From 934ecfdff41d8285ee43a294ca8425328ff96749 Mon Sep 17 00:00:00 2001 From: ChiangYuhsin Date: Fri, 31 Jul 2026 03:28:45 +0000 Subject: [PATCH 1/2] Add NVIDIA SGLang Suite F v6 runner (nvidia_sglang_inference_opt_4fd1e59e) --- .../.vendor/sglang/.clang-format-ignore | 1 + .../.vendor/sglang/.devcontainer/Dockerfile | 35 + .../sglang/.devcontainer/devcontainer.json | 30 + .../.vendor/sglang/.editorconfig | 25 + .../sglang/.github/CI_PERMISSIONS.json | 989 +++ .../.vendor/sglang/.github/CODEOWNERS | 47 + .../.vendor/sglang/.github/FOLDER_README.md | 12 + .../.github/ISSUE_TEMPLATE/1-bug-report.yml | 35 + .../ISSUE_TEMPLATE/2-feature-request.yml | 23 + .../.vendor/sglang/.github/MAINTAINER.md | 67 + .../.vendor/sglang/.github/labeler.yml | 110 + .../sglang/.github/pull_request_template.md | 26 + .../sglang/.github/update_ci_permission.py | 196 + .../sglang/.github/workflows/auto-format.yml | 71 + .../sglang/.github/workflows/auto-tune.yml | 10 + .../bot-bump-kernel-version-to-sglang.yml | 68 + .../workflows/bot-bump-kernel-version.yml | 50 + .../workflows/bot-bump-sglang-version.yml | 61 + .../cancel-all-pending-pr-test-runs.yml | 45 + .../workflows/cancel-pr-workflow-on-merge.yml | 22 + .../.github/workflows/ci-failure-monitor.yml | 64 + .../sglang/.github/workflows/ci-monitor.yml | 111 + .../workflows/close-inactive-issues.yml | 96 + .../.github/workflows/execute-notebook.yml | 60 + .../sglang/.github/workflows/labeler.yml | 20 + .../.vendor/sglang/.github/workflows/lint.yml | 50 + .../workflows/nightly-release-gateway.yml | 196 + .../.github/workflows/nightly-test-amd.yml | 59 + .../.github/workflows/nightly-test-intel.yml | 26 + .../.github/workflows/nightly-test-nvidia.yml | 570 ++ .../workflows/open-pr-copy-from-oss.yml | 28 + .../.github/workflows/open-pr-copy-to-oss.yml | 31 + .../.github/workflows/pr-benchmark-rust.yml | 321 + .../sglang/.github/workflows/pr-gate.yml | 173 + .../sglang/.github/workflows/pr-test-amd.yml | 451 + .../sglang/.github/workflows/pr-test-npu.yml | 205 + .../.github/workflows/pr-test-pd-router.yml | 607 ++ .../sglang/.github/workflows/pr-test-rust.yml | 419 + .../sglang/.github/workflows/pr-test-xeon.yml | 102 + .../sglang/.github/workflows/pr-test-xpu.yml | 118 + .../sglang/.github/workflows/pr-test.yml | 1208 +++ .../workflows/release-docker-amd-nightly.yml | 132 + .../.github/workflows/release-docker-amd.yml | 48 + .../.github/workflows/release-docker-cu13.yml | 119 + .../.github/workflows/release-docker-dev.yml | 122 + .../workflows/release-docker-gateway.yml | 30 + .../workflows/release-docker-npu-nightly.yml | 78 + .../.github/workflows/release-docker-npu.yml | 75 + .../.github/workflows/release-docker-xeon.yml | 35 + .../.github/workflows/release-docker.yml | 138 + .../sglang/.github/workflows/release-docs.yml | 65 + .../.github/workflows/release-fake-tag.yml | 35 + .../workflows/release-pypi-gateway.yml | 167 + .../sglang/.github/workflows/release-pypi.yml | 31 + .../.github/workflows/release-whl-kernel.yml | 197 + .../workflows/slash-command-handler.yml | 46 + .../sglang/.github/workflows/stress-test.yml | 44 + .../.vendor/sglang/.gitignore | 246 + .../.vendor/sglang/.isort.cfg | 3 + .../.vendor/sglang/.pre-commit-config.yaml | 77 + .../3rdparty/amd/profiling/PROFILING.md | 425 + .../sglang/3rdparty/amd/profiling/client.sh | 27 + .../3rdparty/amd/profiling/install_rpd.sh | 10 + .../3rdparty/amd/profiling/loadTracer.sh | 43 + .../sglang/3rdparty/amd/profiling/rpd.patch | 12 + .../profiling/rpd_profile_server_enable.patch | 49 + ...rofile_server_enable_wCPU_activities.patch | 126 + .../sglang/3rdparty/amd/profiling/server.sh | 20 + .../amd/profiling/torch_profiler.patch | 25 + .../sglang/3rdparty/amd/tuning/TUNING.md | 118 + .../3rdparty/amd/tuning/benchmark_moe_rocm.py | 380 + .../.vendor/sglang/CODE_OF_CONDUCT.md | 128 + .../.vendor/sglang/LICENSE | 201 + .../.vendor/sglang/Makefile | 49 + .../.vendor/sglang/README.md | 84 + .../.vendor/sglang/assets/logo.png | Bin 0 -> 402692 bytes .../.vendor/sglang/assets/logo.svg | 1 + .../.vendor/sglang/assets/logo_square.svg | 1 + .../bench_attention_sink_triton.py | 250 + .../bench_in_batch_prefix.py | 130 + .../benchmark_batch/benchmark_batch.py | 193 + .../benchmark_batch/benchmark_tokenizer.py | 126 + .../benchmark/benchmark_vllm_060/README.md | 89 + .../sglang/benchmark/blog_v0_2/405b_sglang.sh | 24 + .../sglang/benchmark/blog_v0_2/405b_trt.sh | 17 + .../sglang/benchmark/blog_v0_2/405b_vllm.sh | 24 + .../sglang/benchmark/blog_v0_2/README.md | 164 + .../sglang/benchmark/blog_v0_2/config.md | 100 + .../.vendor/sglang/benchmark/boolq/README.md | 19 + .../sglang/benchmark/boolq/bench_sglang.py | 124 + .../boolq/convert_parquet_to_json.py | 28 + .../sglang/benchmark/boolq/parquet_to_json.sh | 26 + .../.vendor/sglang/benchmark/ceval/README.md | 15 + .../sglang/benchmark/ceval/bench_sglang.py | 138 + .../sglang/benchmark/deepseek_v3/README.md | 388 + .../.vendor/sglang/benchmark/dspy/README.md | 51 + .../sglang/benchmark/dspy/bench_dspy_intro.py | 192 + .../benchmark/generative_agents/README.md | 38 + .../generative_agents/agent_functions.py | 300 + .../generative_agents/bench_other.py | 80 + .../generative_agents/bench_sglang.py | 74 + .../sglang/benchmark/gpt_oss/README.md | 163 + .../.vendor/sglang/benchmark/gsm8k/README.md | 47 + .../sglang/benchmark/gsm8k/bench_other.py | 151 + .../sglang/benchmark/gsm8k/bench_sglang.py | 148 + .../sglang/benchmark/hellaswag/README.md | 47 + .../sglang/benchmark/hellaswag/bench_other.py | 118 + .../benchmark/hellaswag/bench_sglang.py | 109 + .../.vendor/sglang/benchmark/hf3fs/bench.sh | 59 + .../sglang/benchmark/hf3fs/bench_client.py | 162 + .../sglang/benchmark/hf3fs/bench_storage.py | 258 + .../sglang/benchmark/hf3fs/bench_zerocopy.py | 140 + .../sglang/benchmark/hicache/README.md | 91 + .../benchmark/hicache/bench_long_context.py | 102 + .../sglang/benchmark/hicache/bench_mix.py | 567 ++ .../sglang/benchmark/hicache/bench_mix.sh | 42 + .../benchmark/hicache/bench_multiturn.py | 615 ++ .../sglang/benchmark/hicache/bench_serving.py | 1029 +++ .../benchmark/hicache/data_processing.py | 569 ++ .../sglang/benchmark/hicache/download.sh | 66 + .../sglang/benchmark/hicache/nextqa.py | 159 + .../.vendor/sglang/benchmark/hicache/perf.py | 248 + .../benchmark/json_decode_regex/README.md | 60 + .../json_decode_regex/bench_other.py | 98 + .../json_decode_regex/bench_sglang.py | 101 + .../json_decode_regex/build_dataset.py | 58 + .../benchmark/json_jump_forward/README.md | 88 + .../json_jump_forward/bench_other.py | 288 + .../json_jump_forward/bench_sglang.py | 143 + .../json_jump_forward/build_dataset.py | 58 + .../benchmark/json_jump_forward/dataset.txt | 50 + .../sglang/benchmark/json_schema/README.md | 15 + .../benchmark/json_schema/bench_sglang.py | 146 + .../kernels/all_reduce/benchmark_aiter.py | 330 + .../kernels/all_reduce/benchmark_mscclpp.py | 224 + .../all_reduce/benchmark_torch_symm_mem.py | 251 + .../triton_flashinfer_cudnn.py | 403 + .../benchmark/kernels/deepep/deepep_utils.py | 218 + .../benchmark/kernels/deepep/tuning_deepep.py | 476 ++ .../benchmark/kernels/deepseek/README.md | 19 + .../deepseek/benchmark_deepgemm_fp8_gemm.py | 401 + .../benchmark_deepgemm_fp8_gemm_blackwell.py | 329 + .../benchmark_deepgemm_fp8_group_gemm.py | 487 ++ .../elementwise/benchmark_concat_mla.py | 198 + .../flashinfer_allreduce_fusion/README.md | 102 + .../benchmark_fused_collective.py | 1304 +++ .../kernels/fused_moe_triton/README.md | 210 + .../benchmark_sglang_fused_moe_triton.py | 239 + .../benchmark_torch_compile_fused_moe.py | 305 + ...nchmark_vllm_vs_sglang_fused_moe_triton.py | 264 + .../kernels/fused_moe_triton/common_utils.py | 260 + .../kernels/fused_moe_triton/tuning_client.py | 71 + .../tuning_fused_moe_triton.py | 458 ++ .../tuning_fused_moe_triton_sep.py | 694 ++ .../kernels/fused_moe_triton/tuning_text.json | 1 + .../kernels/quantization/bench_fp4_quant.py | 136 + .../kernels/quantization/bench_int8_quant.py | 94 + .../quantization/tuning_block_wise_kernel.py | 474 ++ .../benchmark_get_last_loc_triton.py | 169 + ...enchmark_write_req_to_token_pool_triton.py | 342 + .../bench_triton_swa_kernel.py | 283 + .../sglang/benchmark/line_retrieval/README.md | 37 + .../benchmark/line_retrieval/bench_sglang.py | 149 + .../benchmark/line_retrieval/gen_data.py | 139 + .../sglang/benchmark/llava_bench/README.md | 61 + .../llava_bench/bench_hf_llava_bench.sh | 9 + .../benchmark/llava_bench/bench_hf_mme.sh | 9 + .../benchmark/llava_bench/bench_sglang.py | 96 + .../benchmark/llava_bench/bench_sglang_mme.sh | 2 + .../benchmark/llava_bench/download_images.py | 20 + .../sglang/benchmark/llm_judge/README.md | 33 + .../sglang/benchmark/llm_judge/bench_other.py | 151 + .../benchmark/llm_judge/bench_sglang.py | 97 + .../benchmark/long_json_decode/README.md | 33 + .../benchmark/long_json_decode/bench_other.py | 89 + .../long_json_decode/bench_sglang.py | 81 + .../long_json_decode/build_dataset.py | 27 + .../sglang/benchmark/lora/launch_server.py | 82 + .../sglang/benchmark/lora/lora_bench.py | 475 ++ .../.vendor/sglang/benchmark/mmlu/README.md | 59 + .../sglang/benchmark/mmlu/bench_other.py | 173 + .../sglang/benchmark/mmlu/bench_sglang.py | 181 + .../sglang/benchmark/mmlu/download_data.sh | 2 + .../.vendor/sglang/benchmark/mmmu/README.md | 49 + .../.vendor/sglang/benchmark/mmmu/bench_hf.py | 169 + .../sglang/benchmark/mmmu/bench_sglang.py | 216 + .../sglang/benchmark/mmmu/data_utils.py | 215 + .../sglang/benchmark/mmmu/eval_utils.py | 683 ++ .../sglang/benchmark/mmmu/prompt_format.yaml | 15 + .../sglang/benchmark/mtbench/README.md | 48 + .../sglang/benchmark/mtbench/bench_other.py | 111 + .../sglang/benchmark/mtbench/bench_sglang.py | 99 + .../benchmark/mtbench/bench_sglang_eagle.py | 137 + .../benchmark/multi_chain_reasoning/README.md | 49 + .../multi_chain_reasoning/bench_other.py | 186 + .../multi_chain_reasoning/bench_sglang.py | 140 + .../benchmark/multi_document_qa/README.md | 47 + .../multi_document_qa/bench_other.py | 114 + .../multi_document_qa/bench_sglang.py | 93 + .../multi_document_qa/build_dataset.py | 70 + .../benchmark/multi_turn_chat/README.md | 66 + .../benchmark/multi_turn_chat/bench_other.py | 93 + .../benchmark/multi_turn_chat/bench_sglang.py | 79 + .../benchmark/multi_turn_chat/data_gen.py | 29 + .../multi_turn_chat/long_prompt_multi_turn.py | 129 + .../prefill_only/bench_embeddings.py | 159 + .../benchmark/prefill_only/bench_score.py | 192 + .../sglang/benchmark/prefill_only/util.py | 813 ++ .../.vendor/sglang/benchmark/react/README.md | 34 + .../sglang/benchmark/react/bench_other.py | 202 + .../sglang/benchmark/react/bench_sglang.py | 153 + .../benchmark/reasoning_benchmark/README.md | 77 + .../reasoning_benchmark/answer_extraction.py | 269 + .../reasoning_benchmark/bench_sglang.py | 135 + .../reasoning_benchmark/eval_utils.py | 206 + .../benchmark/tip_suggestion/.gitignore | 1 + .../sglang/benchmark/tip_suggestion/README.md | 33 + .../benchmark/tip_suggestion/bench_other.py | 133 + .../benchmark/tip_suggestion/bench_sglang.py | 100 + .../benchmark/tip_suggestion/lmql_funcs.py | 50 + .../benchmark/tip_suggestion/topic.jsonl | 50 + .../benchmark/tree_of_thought_deep/README.md | 51 + .../tree_of_thought_deep/bench_other.py | 222 + .../tree_of_thought_deep/bench_sglang.py | 171 + .../tree_of_thought_deep/lmql_funcs.py | 82 + .../benchmark/tree_of_thought_v0/README.md | 43 + .../tree_of_thought_v0/bench_other.py | 179 + .../tree_of_thought_v0/bench_sglang.py | 159 + .../.vendor/sglang/docker/Dockerfile | 326 + .../.vendor/sglang/docker/compose.yaml | 35 + .../.vendor/sglang/docker/configs/.gitconfig | 30 + .../.vendor/sglang/docker/configs/.tmux.conf | 27 + .../.vendor/sglang/docker/configs/.vimrc | 45 + .../.vendor/sglang/docker/configs/.zshrc | 27 + .../.vendor/sglang/docker/configs/yank | 12 + .../sglang/docker/diffusion.Dockerfile | 104 + .../.vendor/sglang/docker/gateway.Dockerfile | 79 + .../docker/k8s-sglang-distributed-sts.yaml | 103 + .../sglang/docker/k8s-sglang-service.yaml | 117 + .../.vendor/sglang/docker/npu.Dockerfile | 100 + .../.vendor/sglang/docker/rocm.Dockerfile | 318 + .../sglang/docker/sagemaker.Dockerfile | 6 + .../.vendor/sglang/docker/serve | 34 + .../.vendor/sglang/docker/xeon.Dockerfile | 56 + .../.vendor/sglang/docker/xpu.Dockerfile | 73 + .../.vendor/sglang/docs/Makefile | 58 + .../.vendor/sglang/docs/README.md | 55 + .../sglang/docs/_static/css/custom_log.css | 29 + .../sglang/docs/_static/css/readthedocs.css | 9 + .../sglang/docs/_static/image/logo.ico | Bin 0 -> 38078 bytes .../sglang/docs/_static/image/logo.png | Bin 0 -> 402692 bytes .../advanced_features/attention_backend.md | 269 + .../advanced_features/checkpoint_engine.md | 254 + .../deterministic_inference.md | 154 + .../dp_for_multi_modal_encoder.md | 69 + .../advanced_features/expert_parallelism.md | 141 + .../docs/advanced_features/forward_hooks.md | 297 + .../sglang/docs/advanced_features/hicache.rst | 8 + .../hicache_best_practices.md | 196 + .../docs/advanced_features/hicache_design.md | 155 + .../hyperparameter_tuning.md | 77 + .../sglang/docs/advanced_features/lora.ipynb | 595 ++ .../docs/advanced_features/observability.md | 35 + .../advanced_features/pd_disaggregation.md | 325 + .../docs/advanced_features/quantization.md | 384 + .../sglang/docs/advanced_features/router.md | 470 ++ .../separate_reasoning.ipynb | 377 + .../advanced_features/server_arguments.md | 467 ++ .../speculative_decoding.ipynb | 370 + .../structured_outputs.ipynb | 998 +++ ...uctured_outputs_for_reasoning_models.ipynb | 830 ++ .../docs/advanced_features/tool_parser.ipynb | 853 ++ .../docs/advanced_features/vlm_query.ipynb | 325 + .../sglang/docs/basic_usage/deepseek_v3.md | 280 + .../sglang/docs/basic_usage/deepseek_v32.md | 234 + .../sglang/docs/basic_usage/gpt_oss.md | 129 + .../.vendor/sglang/docs/basic_usage/llama4.md | 92 + .../sglang/docs/basic_usage/native_api.ipynb | 601 ++ .../docs/basic_usage/offline_engine_api.ipynb | 235 + .../sglang/docs/basic_usage/openai_api.rst | 9 + .../basic_usage/openai_api_completions.ipynb | 534 ++ .../basic_usage/openai_api_embeddings.ipynb | 195 + .../docs/basic_usage/openai_api_vision.ipynb | 254 + .../docs/basic_usage/popular_model_usage.rst | 12 + .../.vendor/sglang/docs/basic_usage/qwen3.md | 33 + .../sglang/docs/basic_usage/qwen3_vl.md | 130 + .../docs/basic_usage/sampling_params.md | 346 + .../docs/basic_usage/send_request.ipynb | 253 + .../.vendor/sglang/docs/conf.py | 205 + .../.vendor/sglang/docs/deploy.py | 22 + .../docs/developer_guide/bench_serving.md | 355 + .../benchmark_and_profiling.md | 456 ++ .../developer_guide/contribution_guide.md | 140 + .../development_guide_using_docker.md | 108 + .../docs/developer_guide/release_process.md | 18 + .../developer_guide/setup_github_runner.md | 51 + .../sglang/docs/get_started/install.md | 185 + .../.vendor/sglang/docs/index.rst | 106 + .../.vendor/sglang/docs/platforms/amd_gpu.md | 159 + .../sglang/docs/platforms/ascend_npu.md | 190 + .../sglang/docs/platforms/cpu_server.md | 216 + .../sglang/docs/platforms/nvidia_jetson.md | 80 + .../.vendor/sglang/docs/platforms/tpu.md | 3 + .../.vendor/sglang/docs/platforms/xpu.md | 92 + .../docs/references/custom_chat_template.md | 51 + .../docs/references/environment_variables.md | 127 + .../.vendor/sglang/docs/references/faq.md | 39 + .../references/frontend/choices_methods.md | 77 + .../references/frontend/frontend_index.rst | 9 + .../frontend/frontend_tutorial.ipynb | 456 ++ .../sglang/docs/references/learn_more.md | 9 + .../docs/references/mindspore_models.md | 164 + .../multi_node_deployment/deploy_on_k8s.md | 337 + .../lws_pd/lws-examples/d-svc.yaml | 12 + .../lws_pd/lws-examples/d.yaml | 290 + .../lws_pd/lws-examples/lb.yaml | 56 + .../lws_pd/lws-examples/p-svc.yaml | 12 + .../lws_pd/lws-examples/p.yaml | 304 + .../lws_pd/lws_pd_deploy.md | 783 ++ .../multi_node_deployment/multi_node.md | 100 + .../multi_node_index.rst | 14 + .../rbg_pd/deepseekv32_pd.md | 569 ++ .../references/post_training_integration.md | 31 + .../docs/references/production_metrics.md | 231 + .../references/production_request_trace.md | 160 + .../docs/references/torch_compile_cache.md | 13 + .../.vendor/sglang/docs/requirements.txt | 20 + .../.vendor/sglang/docs/serve.sh | 3 + .../docs/supported_models/classify_models.md | 162 + .../docs/supported_models/embedding_models.md | 126 + .../supported_models/generative_models.md | 64 + .../docs/supported_models/modelscope.md | 28 + .../multimodal_language_models.md | 111 + .../docs/supported_models/rerank_models.md | 49 + .../docs/supported_models/reward_models.md | 28 + .../supported_models/support_new_models.md | 320 + .../supported_models/transformers_fallback.md | 58 + .../.vendor/sglang/docs/wrap_run_llm.py | 47 + .../.vendor/sglang/examples/assets/.gitignore | 1 + .../sglang/examples/assets/example_image.png | Bin 0 -> 57365 bytes .../tool_chat_template_deepseekr1.jinja | 92 + .../tool_chat_template_deepseekv3.jinja | 92 + .../tool_chat_template_deepseekv31.jinja | 92 + .../tool_chat_template_deepseekv32.jinja | 101 + .../tool_chat_template_llama3.1_json.jinja | 121 + .../tool_chat_template_llama4_pythonic.jinja | 112 + .../vision_template_sarashina_vl.jinja | 9 + .../examples/checkpoint_engine/update.py | 241 + .../quick_start/anthropic_example_chat.py | 73 + .../quick_start/anthropic_example_complete.py | 68 + .../quick_start/azure_openai_example_chat.py | 83 + .../quick_start/gemini_example_chat.py | 73 + .../quick_start/gemini_example_complete.py | 68 + .../gemini_example_multimodal_chat.py | 30 + .../quick_start/images/cat.jpeg | Bin 0 -> 345163 bytes .../quick_start/images/dog.jpeg | Bin 0 -> 416580 bytes .../quick_start/local_example_chat.py | 75 + .../quick_start/local_example_complete.py | 70 + .../quick_start/local_example_llava_next.py | 78 + .../quick_start/openai_example_chat.py | 74 + .../quick_start/openai_example_complete.py | 68 + .../quick_start/openai_example_n.py | 71 + .../quick_start/openai_example_o1.py | 57 + .../quick_start/openrouter_example_chat.py | 81 + .../quick_start/together_example_chat.py | 81 + .../quick_start/together_example_complete.py | 76 + .../frontend_language/usage/chinese_regex.py | 53 + .../usage/choices_logprob.py | 44 + .../frontend_language/usage/cot_decoding.py | 115 + .../frontend_language/usage/json_decode.py | 83 + .../frontend_language/usage/json_logprobs.py | 103 + .../usage/llava_video/srt_example_llava_v.py | 260 + .../usage/llava_video/srt_example_llava_v.sh | 131 + .../usage/openai_chat_speculative.py | 155 + .../usage/openai_speculative.py | 54 + .../usage/parallel_sample.py | 40 + .../trace_and_evaluate_rag_using_parea.ipynb | 408 + .../usage/readme_examples.py | 109 + .../usage/sgl_gen_min_tokens.py | 35 + .../frontend_language/usage/streaming.py | 49 + .../frontend_language/usage/triton/Dockerfile | 10 + .../frontend_language/usage/triton/README.md | 35 + .../models/character_generation/config.pbtxt | 23 + .../sglang/examples/monitoring/README.md | 76 + .../examples/monitoring/docker-compose.yaml | 28 + .../grafana/dashboards/config/dashboard.yaml | 11 + .../dashboards/json/sglang-dashboard.json | 984 +++ .../grafana/datasources/datasource.yaml | 8 + .../examples/monitoring/opentelemetry.yaml | 38 + .../examples/monitoring/prometheus.yaml | 10 + .../examples/monitoring/tracing_compose.yaml | 21 + .../profiler/nsys_profile_tools/README.md | 176 + .../nsys_profile_tools/gputrc2graph.py | 344 + .../sglang_engine_model.json | 61 + .../.vendor/sglang/examples/runtime/README.md | 45 + .../examples/runtime/engine/custom_server.py | 53 + .../examples/runtime/engine/embedding.py | 27 + .../engine/fastapi_engine_inference.py | 195 + .../examples/runtime/engine/launch_engine.py | 17 + .../runtime/engine/offline_batch_inference.py | 43 + .../engine/offline_batch_inference_async.py | 65 + .../engine/offline_batch_inference_eagle.py | 38 + .../engine/offline_batch_inference_qwen_1m.py | 74 + .../engine/offline_batch_inference_vlm.py | 52 + .../sglang/examples/runtime/engine/readme.md | 54 + .../runtime/engine/save_remote_state.py | 59 + .../runtime/engine/save_sharded_state.py | 74 + .../hidden_states/hidden_states_engine.py | 66 + .../hidden_states/hidden_states_server.py | 81 + .../.vendor/sglang/examples/runtime/lora.py | 67 + .../runtime/multimodal/llama3_llava_server.py | 111 + .../multimodal/llava_onevision_server.py | 264 + .../runtime/multimodal/pixtral_server.py | 127 + .../runtime/multimodal/qwen_llava_server.py | 111 + .../examples/runtime/multimodal_embedding.py | 18 + .../openai_chat_with_response_prefill.py | 53 + .../sglang/examples/runtime/reward_model.py | 32 + .../token_in_token_out_llm_engine.py | 43 + .../token_in_token_out_llm_server.py | 68 + .../token_in_token_out_vlm_engine.py | 74 + .../token_in_token_out_vlm_server.py | 78 + .../sglang/examples/runtime/vertex_predict.py | 66 + .../sagemaker/deploy_and_serve_endpoint.py | 69 + .../usage/modelopt_quantize_and_export.py | 303 + .../.vendor/sglang/python/pyproject.toml | 167 + .../.vendor/sglang/python/pyproject_cpu.toml | 134 + .../sglang/python/pyproject_other.toml | 156 + .../.vendor/sglang/python/pyproject_xpu.toml | 138 + .../.vendor/sglang/python/sglang/README.md | 17 + .../.vendor/sglang/python/sglang/__init__.py | 83 + .../python/sglang/bench_offline_throughput.py | 476 ++ .../sglang/python/sglang/bench_one_batch.py | 795 ++ .../python/sglang/bench_one_batch_server.py | 605 ++ .../sglang/python/sglang/bench_serving.py | 2867 +++++++ .../.vendor/sglang/python/sglang/check_env.py | 433 + .../sglang/python/sglang/cli/__init__.py | 0 .../sglang/python/sglang/cli/generate.py | 33 + .../.vendor/sglang/python/sglang/cli/main.py | 26 + .../.vendor/sglang/python/sglang/cli/serve.py | 75 + .../.vendor/sglang/python/sglang/cli/utils.py | 152 + .../sglang/python/sglang/compile_deep_gemm.py | 191 + .../sglang/python/sglang/eval/llama3_eval.py | 315 + .../sglang/python/sglang/eval/loogle_eval.py | 164 + .../sglang/python/sglang/global_config.py | 29 + .../python/sglang/jit_kernel/.clang-format | 19 + .../jit_kernel/csrc/cuda_wait_value.cuh | 38 + .../python/sglang/jit_kernel/csrc/hicache.cuh | 264 + .../sglang/jit_kernel/cuda_wait_value.py | 79 + .../python/sglang/jit_kernel/hicache.py | 138 + .../jit_kernel/include/sgl_kernel/tensor.h | 487 ++ .../jit_kernel/include/sgl_kernel/utils.cuh | 101 + .../jit_kernel/include/sgl_kernel/utils.h | 88 + .../jit_kernel/include/sgl_kernel/warp.cuh | 145 + .../sglang/python/sglang/jit_kernel/utils.py | 103 + .../.vendor/sglang/python/sglang/lang/api.py | 292 + .../python/sglang/lang/backend/anthropic.py | 73 + .../sglang/lang/backend/base_backend.py | 82 + .../python/sglang/lang/backend/litellm.py | 90 + .../python/sglang/lang/backend/openai.py | 475 ++ .../sglang/lang/backend/runtime_endpoint.py | 527 ++ .../python/sglang/lang/backend/vertexai.py | 148 + .../python/sglang/lang/chat_template.py | 668 ++ .../sglang/python/sglang/lang/choices.py | 164 + .../sglang/python/sglang/lang/interpreter.py | 1061 +++ .../.vendor/sglang/python/sglang/lang/ir.py | 643 ++ .../sglang/python/sglang/lang/tracer.py | 279 + .../sglang/python/sglang/launch_server.py | 29 + .../python/sglang/multimodal_gen/README.md | 76 + .../python/sglang/multimodal_gen/__init__.py | 6 + .../multimodal_gen/benchmarks/compare_perf.py | 216 + .../sglang/multimodal_gen/configs/__init__.py | 3 + .../backend/vmoba/wan_1.3B_77_448_832.json | 16 + .../backend/vmoba/wan_1.3B_77_480_832.json | 16 + .../multimodal_gen/configs/models/__init__.py | 8 + .../multimodal_gen/configs/models/base.py | 105 + .../configs/models/dits/__init__.py | 7 + .../configs/models/dits/base.py | 76 + .../configs/models/dits/flux.py | 42 + .../configs/models/dits/hunyuanvideo.py | 184 + .../configs/models/dits/qwenimage.py | 41 + .../configs/models/dits/stepvideo.py | 64 + .../configs/models/dits/wanvideo.py | 102 + .../configs/models/dits/zimage.py | 40 + .../configs/models/encoders/__init__.py | 25 + .../configs/models/encoders/base.py | 86 + .../configs/models/encoders/clip.py | 95 + .../configs/models/encoders/llama.py | 69 + .../configs/models/encoders/qwen_image.py | 68 + .../configs/models/encoders/t5.py | 86 + .../configs/models/vaes/__init__.py | 11 + .../configs/models/vaes/base.py | 156 + .../configs/models/vaes/flux.py | 58 + .../configs/models/vaes/hunyuanvae.py | 41 + .../configs/models/vaes/qwenimage.py | 53 + .../configs/models/vaes/stepvideovae.py | 31 + .../configs/models/vaes/wanvae.py | 88 + .../configs/pipeline_configs/__init__.py | 35 + .../configs/pipeline_configs/base.py | 669 ++ .../configs/pipeline_configs/flux.py | 576 ++ .../configs/pipeline_configs/hunyuan.py | 114 + .../configs/pipeline_configs/qwen_image.py | 330 + .../configs/pipeline_configs/stepvideo.py | 36 + .../configs/pipeline_configs/wan.py | 212 + .../configs/pipeline_configs/zimage.py | 142 + .../multimodal_gen/configs/sample/__init__.py | 5 + .../multimodal_gen/configs/sample/flux.py | 25 + .../multimodal_gen/configs/sample/hunyuan.py | 37 + .../configs/sample/qwenimage.py | 18 + .../configs/sample/sampling_params.py | 629 ++ .../configs/sample/stepvideo.py | 22 + .../multimodal_gen/configs/sample/teacache.py | 43 + .../multimodal_gen/configs/sample/wan.py | 217 + .../multimodal_gen/configs/sample/zimage.py | 33 + .../sglang/multimodal_gen/configs/utils.py | 61 + .../csrc/attn/vmoba_attn/README.md | 31 + .../csrc/attn/vmoba_attn/setup.py | 26 + .../attn/vmoba_attn/tests/test_vmoba_attn.py | 137 + .../csrc/attn/vmoba_attn/vmoba/__init__.py | 2 + .../csrc/attn/vmoba_attn/vmoba/vmoba.py | 1086 +++ .../python/sglang/multimodal_gen/docs/cli.md | 184 + .../multimodal_gen/docs/contributing.md | 56 + .../sglang/multimodal_gen/docs/install.md | 48 + .../sglang/multimodal_gen/docs/openai_api.md | 281 + .../multimodal_gen/docs/support_matrix.md | 48 + .../multimodal_gen/docs/support_new_models.md | 107 + .../python/sglang/multimodal_gen/envs.py | 328 + .../python/sglang/multimodal_gen/registry.py | 423 + .../runtime/distributed/__init__.py | 55 + .../runtime/distributed/communication_op.py | 55 + .../device_communicators/__init__.py | 1 + .../base_device_communicator.py | 297 + .../device_communicators/cpu_communicator.py | 161 + .../device_communicators/cuda_communicator.py | 79 + .../device_communicators/pynccl.py | 258 + .../device_communicators/pynccl_wrapper.py | 450 + .../runtime/distributed/group_coordinator.py | 1226 +++ .../runtime/distributed/parallel_state.py | 1144 +++ .../runtime/distributed/utils.py | 195 + .../runtime/entrypoints/__init__.py | 1 + .../runtime/entrypoints/cli/__init__.py | 1 + .../runtime/entrypoints/cli/cli_types.py | 28 + .../runtime/entrypoints/cli/generate.py | 151 + .../runtime/entrypoints/cli/main.py | 44 + .../runtime/entrypoints/cli/serve.py | 69 + .../runtime/entrypoints/cli/utils.py | 74 + .../entrypoints/diffusion_generator.py | 459 ++ .../runtime/entrypoints/http_server.py | 80 + .../runtime/entrypoints/openai/common_api.py | 62 + .../runtime/entrypoints/openai/image_api.py | 228 + .../runtime/entrypoints/openai/protocol.py | 65 + .../runtime/entrypoints/openai/stores.py | 46 + .../runtime/entrypoints/openai/utils.py | 125 + .../runtime/entrypoints/openai/video_api.py | 265 + .../runtime/entrypoints/utils.py | 42 + .../multimodal_gen/runtime/launch_server.py | 142 + .../multimodal_gen/runtime/layers/__init__.py | 1 + .../runtime/layers/activation.py | 129 + .../layers/attention/STA_configuration.py | 414 + .../runtime/layers/attention/__init__.py | 28 + .../layers/attention/backends/__init__.py | 1 + .../layers/attention/backends/aiter.py | 101 + .../attention/backends/attention_backend.py | 180 + .../layers/attention/backends/flash_attn.py | 139 + .../layers/attention/backends/flash_attn_2.py | 78 + .../layers/attention/backends/sage_attn.py | 70 + .../layers/attention/backends/sage_attn3.py | 78 + .../runtime/layers/attention/backends/sdpa.py | 77 + .../attention/backends/sliding_tile_attn.py | 313 + .../attention/backends/video_sparse_attn.py | 331 + .../layers/attention/backends/vmoba.py | 258 + .../runtime/layers/attention/layer.py | 396 + .../runtime/layers/attention/selector.py | 197 + .../runtime/layers/custom_op.py | 110 + .../runtime/layers/layernorm.py | 429 + .../multimodal_gen/runtime/layers/linear.py | 1057 +++ .../runtime/layers/lora/linear.py | 433 + .../multimodal_gen/runtime/layers/mlp.py | 46 + .../runtime/layers/quantization/__init__.py | 71 + .../layers/quantization/base_config.py | 152 + .../runtime/layers/rotary_embedding.py | 889 ++ .../runtime/layers/triton_ops.py | 948 +++ .../multimodal_gen/runtime/layers/usp.py | 255 + .../multimodal_gen/runtime/layers/utils.py | 24 + .../runtime/layers/visual_embedding.py | 190 + .../layers/vocab_parallel_embedding.py | 480 ++ .../runtime/loader/component_loader.py | 766 ++ .../runtime/loader/fsdp_load.py | 314 + .../multimodal_gen/runtime/loader/utils.py | 102 + .../runtime/loader/weight_utils.py | 300 + .../runtime/managers/forward_context.py | 120 + .../runtime/managers/gpu_worker.py | 191 + .../runtime/managers/scheduler.py | 230 + .../multimodal_gen/runtime/models/__init__.py | 1 + .../runtime/models/dits/base.py | 134 + .../runtime/models/dits/causal_wanvideo.py | 851 ++ .../runtime/models/dits/flux.py | 558 ++ .../runtime/models/dits/flux_2.py | 814 ++ .../runtime/models/dits/hunyuanvideo.py | 961 +++ .../runtime/models/dits/qwen_image.py | 650 ++ .../runtime/models/dits/stepvideo.py | 729 ++ .../runtime/models/dits/wanvideo.py | 945 +++ .../runtime/models/dits/zimage.py | 605 ++ .../runtime/models/encoders/base.py | 71 + .../runtime/models/encoders/bert.py | 46 + .../runtime/models/encoders/clip.py | 700 ++ .../runtime/models/encoders/llama.py | 459 ++ .../runtime/models/encoders/mistral_3.py | 459 ++ .../runtime/models/encoders/qwen2_5vl.py | 1180 +++ .../runtime/models/encoders/stepllm.py | 614 ++ .../runtime/models/encoders/t5.py | 716 ++ .../runtime/models/encoders/vision.py | 96 + .../runtime/models/parameter.py | 423 + .../multimodal_gen/runtime/models/registry.py | 365 + .../runtime/models/schedulers/base.py | 37 + .../scheduling_flow_match_euler_discrete.py | 698 ++ .../scheduling_flow_unipc_multistep.py | 853 ++ .../scheduling_self_forcing_flow_match.py | 142 + .../schedulers/scheduling_unipc_multistep.py | 1207 +++ .../multimodal_gen/runtime/models/utils.py | 148 + .../runtime/models/vaes/autoencoder.py | 585 ++ .../models/vaes/autoencoder_kl_flux2.py | 524 ++ .../models/vaes/autoencoder_kl_qwenimage.py | 1170 +++ .../runtime/models/vaes/common.py | 648 ++ .../runtime/models/vaes/hunyuanvae.py | 852 ++ .../runtime/models/vaes/stepvideovae.py | 1184 +++ .../runtime/models/vaes/wanvae.py | 1343 +++ .../runtime/models/vision_utils.py | 304 + .../runtime/pipelines/__init__.py | 1 + .../multimodal_gen/runtime/pipelines/flux.py | 124 + .../runtime/pipelines/flux_2.py | 124 + .../runtime/pipelines/hunyuan_pipeline.py | 95 + .../runtime/pipelines/qwen_image.py | 187 + .../runtime/pipelines/stepvideo_pipeline.py | 182 + .../pipelines/wan_causal_dmd_pipeline.py | 81 + .../runtime/pipelines/wan_dmd_pipeline.py | 101 + .../runtime/pipelines/wan_i2v_dmd_pipeline.py | 113 + .../runtime/pipelines/wan_i2v_pipeline.py | 118 + .../runtime/pipelines/wan_pipeline.py | 101 + .../runtime/pipelines/zimage_pipeline.py | 116 + .../runtime/pipelines_core/__init__.py | 62 + .../pipelines_core/composed_pipeline_base.py | 365 + .../executors/parallel_executor.py | 92 + .../executors/pipeline_executor.py | 59 + .../pipelines_core/executors/sync_executor.py | 39 + .../runtime/pipelines_core/lora_pipeline.py | 284 + .../runtime/pipelines_core/schedule_batch.py | 241 + .../runtime/pipelines_core/stages/__init__.py | 59 + .../runtime/pipelines_core/stages/base.py | 233 + .../pipelines_core/stages/causal_denoising.py | 508 ++ .../pipelines_core/stages/conditioning.py | 107 + .../runtime/pipelines_core/stages/decoding.py | 213 + .../pipelines_core/stages/denoising.py | 1387 ++++ .../pipelines_core/stages/denoising_dmd.py | 204 + .../runtime/pipelines_core/stages/encoding.py | 105 + .../pipelines_core/stages/image_encoding.py | 357 + .../pipelines_core/stages/input_validation.py | 250 + .../stages/latent_preparation.py | 163 + .../stages/stepvideo_encoding.py | 99 + .../pipelines_core/stages/text_encoding.py | 336 + .../stages/timestep_preparation.py | 148 + .../pipelines_core/stages/validators.py | 522 ++ .../runtime/platforms/__init__.py | 172 + .../multimodal_gen/runtime/platforms/cpu.py | 61 + .../multimodal_gen/runtime/platforms/cuda.py | 438 + .../runtime/platforms/interface.py | 252 + .../multimodal_gen/runtime/platforms/mps.py | 88 + .../multimodal_gen/runtime/platforms/rocm.py | 138 + .../runtime/scheduler_client.py | 149 + .../multimodal_gen/runtime/server_args.py | 1053 +++ .../runtime/sync_scheduler_client.py | 92 + .../multimodal_gen/runtime/utils/common.py | 340 + .../runtime/utils/distributed.py | 231 + .../runtime/utils/hf_diffusers_utils.py | 406 + .../runtime/utils/logging_utils.py | 482 ++ .../runtime/utils/perf_logger.py | 263 + .../sglang/multimodal_gen/test/__init__.py | 1 + .../test/cli/test_generate_common.py | 107 + .../test/cli/test_generate_t2i_perf.py | 84 + .../test/cli/test_generate_t2v_perf.py | 81 + .../test/cli/test_generate_ti2v_perf.py | 71 + .../multimodal_gen/test/cli/test_serve.py | 301 + .../sglang/multimodal_gen/test/run_suite.py | 173 + .../multimodal_gen/test/server/conftest.py | 53 + .../test/server/perf_baselines.json | 1018 +++ .../test/server/test_server_2_gpu_a.py | 25 + .../test/server/test_server_2_gpu_b.py | 25 + .../test/server/test_server_a.py | 31 + .../test/server/test_server_b.py | 31 + .../test/server/test_server_common.py | 384 + .../test/server/test_server_utils.py | 696 ++ .../test/server/testcase_configs.py | 484 ++ .../sglang/multimodal_gen/test/slack_utils.py | 193 + .../test/test_files/launch_flux.json | 11 + .../test/test_files/launch_wan.json | 11 + .../multimodal_gen/test/test_offline_api.py | 75 + .../sglang/multimodal_gen/test/test_utils.py | 420 + .../multimodal_gen/third_party/__init__.py | 1 + .../multimodal_gen/third_party/pynvml.py | 7227 +++++++++++++++++ .../python/sglang/multimodal_gen/utils.py | 813 ++ .../.vendor/sglang/python/sglang/profiler.py | 155 + .../sglang/python/sglang/srt/_custom_ops.py | 165 + .../srt/batch_invariant_ops/__init__.py | 29 + .../batch_invariant_ops.py | 991 +++ .../sglang/srt/batch_overlap/operations.py | 211 + .../srt/batch_overlap/operations_strategy.py | 211 + .../srt/batch_overlap/single_batch_overlap.py | 116 + .../srt/batch_overlap/two_batch_overlap.py | 1027 +++ .../sglang/srt/checkpoint_engine/__init__.py | 9 + .../checkpoint_engine_worker.py | 142 + .../sglang/srt/checkpoint_engine/update.py | 317 + .../python/sglang/srt/compilation/backend.py | 455 ++ .../srt/compilation/compilation_config.py | 29 + .../srt/compilation/compilation_counter.py | 47 + .../python/sglang/srt/compilation/compile.py | 215 + .../srt/compilation/compiler_interface.py | 503 ++ .../srt/compilation/cuda_piecewise_backend.py | 215 + .../srt/compilation/fix_functionalization.py | 134 + .../python/sglang/srt/compilation/fx_utils.py | 83 + .../sglang/srt/compilation/inductor_pass.py | 140 + .../sglang/srt/compilation/pass_manager.py | 66 + .../compilation/piecewise_context_manager.py | 61 + .../python/sglang/srt/configs/__init__.py | 47 + .../python/sglang/srt/configs/chatglm.py | 78 + .../sglang/python/sglang/srt/configs/dbrx.py | 279 + .../python/sglang/srt/configs/deepseek_ocr.py | 794 ++ .../python/sglang/srt/configs/deepseekvl2.py | 687 ++ .../sglang/srt/configs/device_config.py | 19 + .../python/sglang/srt/configs/dots_ocr.py | 64 + .../python/sglang/srt/configs/dots_vlm.py | 134 + .../python/sglang/srt/configs/exaone.py | 195 + .../python/sglang/srt/configs/falcon_h1.py | 310 + .../python/sglang/srt/configs/internvl.py | 706 ++ .../python/sglang/srt/configs/janus_pro.py | 634 ++ .../python/sglang/srt/configs/jet_nemotron.py | 74 + .../python/sglang/srt/configs/jet_vlm.py | 53 + .../python/sglang/srt/configs/kimi_linear.py | 161 + .../python/sglang/srt/configs/kimi_vl.py | 38 + .../sglang/srt/configs/kimi_vl_moonvit.py | 32 + .../python/sglang/srt/configs/load_config.py | 121 + .../sglang/srt/configs/longcat_flash.py | 104 + .../python/sglang/srt/configs/mamba_utils.py | 183 + .../python/sglang/srt/configs/model_config.py | 1043 +++ .../sglang/srt/configs/modelopt_config.py | 30 + .../sglang/srt/configs/nano_nemotron_vl.py | 114 + .../python/sglang/srt/configs/nemotron_h.py | 306 + .../sglang/python/sglang/srt/configs/olmo3.py | 105 + .../sglang/srt/configs/points_v15_chat.py | 29 + .../python/sglang/srt/configs/qwen3_next.py | 291 + .../python/sglang/srt/configs/qwen3_omni.py | 613 ++ .../python/sglang/srt/configs/qwen3_vl.py | 576 ++ .../sglang/python/sglang/srt/configs/radio.py | 106 + .../python/sglang/srt/configs/step3_vl.py | 172 + .../sglang/srt/configs/update_config.py | 174 + .../sglang/python/sglang/srt/configs/utils.py | 25 + .../python/sglang/srt/connector/__init__.py | 58 + .../sglang/srt/connector/base_connector.py | 111 + .../python/sglang/srt/connector/redis.py | 85 + .../sglang/srt/connector/remote_instance.py | 82 + .../sglang/python/sglang/srt/connector/s3.py | 122 + .../sglang/srt/connector/serde/__init__.py | 31 + .../sglang/srt/connector/serde/safe_serde.py | 30 + .../sglang/srt/connector/serde/serde.py | 43 + .../python/sglang/srt/connector/utils.py | 35 + .../sglang/python/sglang/srt/constants.py | 10 + .../srt/constrained/base_grammar_backend.py | 255 + .../srt/constrained/llguidance_backend.py | 178 + .../srt/constrained/outlines_backend.py | 190 + .../srt/constrained/outlines_jump_forward.py | 200 + .../constrained/reasoner_grammar_backend.py | 114 + .../srt/constrained/triton_ops/bitmask_ops.py | 141 + .../python/sglang/srt/constrained/utils.py | 12 + .../srt/constrained/xgrammar_backend.py | 343 + .../sglang/python/sglang/srt/custom_op.py | 102 + .../python/sglang/srt/debug_utils/__init__.py | 0 .../sglang/srt/debug_utils/dump_comparator.py | 300 + .../sglang/srt/debug_utils/dump_loader.py | 105 + .../python/sglang/srt/debug_utils/dumper.py | 172 + .../sglang/srt/debug_utils/log_parser.py | 46 + .../sglang/srt/debug_utils/model_truncator.py | 112 + .../debug_utils/tensor_dump_forward_hook.py | 164 + .../sglang/srt/debug_utils/text_comparator.py | 234 + .../srt/disaggregation/ascend/__init__.py | 6 + .../sglang/srt/disaggregation/ascend/conn.py | 117 + .../disaggregation/ascend/transfer_engine.py | 96 + .../srt/disaggregation/base/__init__.py | 8 + .../sglang/srt/disaggregation/base/conn.py | 159 + .../srt/disaggregation/common/__init__.py | 5 + .../sglang/srt/disaggregation/common/conn.py | 609 ++ .../sglang/srt/disaggregation/common/utils.py | 42 + .../sglang/srt/disaggregation/decode.py | 966 +++ .../decode_kvcache_offload_manager.py | 235 + .../decode_schedule_batch_mixin.py | 183 + .../srt/disaggregation/fake/__init__.py | 1 + .../sglang/srt/disaggregation/fake/conn.py | 93 + .../sglang/srt/disaggregation/kv_events.py | 418 + .../srt/disaggregation/mooncake/__init__.py | 6 + .../srt/disaggregation/mooncake/conn.py | 1325 +++ .../mooncake/transfer_engine.py | 242 + .../srt/disaggregation/mooncake/utils.py | 110 + .../srt/disaggregation/nixl/__init__.py | 6 + .../sglang/srt/disaggregation/nixl/conn.py | 867 ++ .../sglang/srt/disaggregation/prefill.py | 745 ++ .../python/sglang/srt/disaggregation/utils.py | 360 + .../python/sglang/srt/distributed/__init__.py | 3 + .../srt/distributed/communication_op.py | 35 + .../device_communicators/all_reduce_utils.py | 16 + .../device_communicators/cuda_wrapper.py | 183 + .../device_communicators/custom_all_reduce.py | 440 + .../custom_all_reduce_utils.py | 386 + .../device_communicators/hpu_communicator.py | 49 + .../device_communicators/npu_communicator.py | 39 + .../device_communicators/pymscclpp.py | 315 + .../device_communicators/pynccl.py | 382 + .../device_communicators/pynccl_allocator.py | 189 + .../device_communicators/pynccl_wrapper.py | 563 ++ .../device_communicators/quick_all_reduce.py | 275 + .../device_communicators/shm_broadcast.py | 510 ++ .../device_communicators/torch_symm_mem.py | 163 + .../device_communicators/xpu_communicator.py | 48 + .../srt/distributed/naive_distributed.py | 113 + .../sglang/srt/distributed/parallel_state.py | 1957 +++++ .../python/sglang/srt/distributed/utils.py | 230 + .../sglang/srt/dllm/algorithm/__init__.py | 39 + .../python/sglang/srt/dllm/algorithm/base.py | 18 + .../srt/dllm/algorithm/low_confidence.py | 59 + .../sglang/python/sglang/srt/dllm/config.py | 40 + .../sglang/srt/elastic_ep/elastic_ep.py | 74 + .../sglang/srt/entrypoints/EngineBase.py | 73 + .../python/sglang/srt/entrypoints/context.py | 249 + .../python/sglang/srt/entrypoints/engine.py | 934 +++ .../sglang/srt/entrypoints/grpc_server.py | 1006 +++ .../sglang/srt/entrypoints/harmony_utils.py | 368 + .../sglang/srt/entrypoints/http_server.py | 1670 ++++ .../srt/entrypoints/http_server_engine.py | 136 + .../sglang/srt/entrypoints/openai/__init__.py | 0 .../srt/entrypoints/openai/encoding_dsv32.py | 451 + .../sglang/srt/entrypoints/openai/protocol.py | 1231 +++ .../srt/entrypoints/openai/serving_base.py | 269 + .../srt/entrypoints/openai/serving_chat.py | 1254 +++ .../entrypoints/openai/serving_classify.py | 204 + .../entrypoints/openai/serving_completions.py | 486 ++ .../entrypoints/openai/serving_embedding.py | 177 + .../srt/entrypoints/openai/serving_rerank.py | 104 + .../entrypoints/openai/serving_responses.py | 1321 +++ .../srt/entrypoints/openai/serving_score.py | 62 + .../entrypoints/openai/serving_tokenize.py | 144 + .../srt/entrypoints/openai/tool_server.py | 175 + .../srt/entrypoints/openai/usage_processor.py | 84 + .../sglang/srt/entrypoints/openai/utils.py | 72 + .../python/sglang/srt/entrypoints/tool.py | 87 + .../python/sglang/srt/entrypoints/warmup.py | 60 + .../sglang/python/sglang/srt/environ.py | 458 ++ .../sglang/python/sglang/srt/eplb/__init__.py | 0 .../srt/eplb/eplb_algorithms/__init__.py | 80 + .../srt/eplb/eplb_algorithms/deepseek.py | 221 + .../srt/eplb/eplb_algorithms/deepseek_vec.py | 276 + .../eplb/eplb_algorithms/elasticity_aware.py | 87 + .../python/sglang/srt/eplb/eplb_manager.py | 118 + .../srt/eplb/eplb_simulator/__init__.py | 1 + .../sglang/srt/eplb/eplb_simulator/reader.py | 51 + .../sglang/srt/eplb/expert_distribution.py | 1025 +++ .../python/sglang/srt/eplb/expert_location.py | 573 ++ .../srt/eplb/expert_location_dispatch.py | 109 + .../srt/eplb/expert_location_updater.py | 575 ++ .../srt/function_call/base_format_detector.py | 339 + .../sglang/srt/function_call/core_types.py | 34 + .../srt/function_call/deepseekv31_detector.py | 206 + .../srt/function_call/deepseekv32_detector.py | 321 + .../srt/function_call/deepseekv3_detector.py | 209 + .../srt/function_call/function_call_parser.py | 199 + .../srt/function_call/glm4_moe_detector.py | 158 + .../srt/function_call/gpt_oss_detector.py | 241 + .../srt/function_call/json_array_parser.py | 51 + .../srt/function_call/kimik2_detector.py | 239 + .../srt/function_call/llama32_detector.py | 88 + .../sglang/srt/function_call/minimax_m2.py | 354 + .../srt/function_call/mistral_detector.py | 129 + .../srt/function_call/pythonic_detector.py | 224 + .../srt/function_call/qwen25_detector.py | 120 + .../srt/function_call/qwen3_coder_detector.py | 353 + .../srt/function_call/step3_detector.py | 407 + .../python/sglang/srt/function_call/utils.py | 143 + .../sglang/python/sglang/srt/grpc/__init__.py | 1 + .../python/sglang/srt/grpc/compile_proto.py | 248 + .../sglang/srt/grpc/grpc_request_manager.py | 924 +++ .../python/sglang/srt/grpc/health_servicer.py | 189 + .../sglang/srt/grpc/scheduler_launcher.py | 183 + .../sglang/srt/grpc/sglang_scheduler.proto | 459 ++ .../sglang/srt/grpc/sglang_scheduler_pb2.py | 119 + .../sglang/srt/grpc/sglang_scheduler_pb2.pyi | 490 ++ .../srt/grpc/sglang_scheduler_pb2_grpc.py | 327 + .../python/sglang/srt/layers/activation.py | 392 + .../python/sglang/srt/layers/amx_utils.py | 86 + .../srt/layers/attention/aiter_backend.py | 1178 +++ .../srt/layers/attention/ascend_backend.py | 1208 +++ .../layers/attention/attention_registry.py | 230 + .../srt/layers/attention/base_attn_backend.py | 169 + .../layers/attention/cutlass_mla_backend.py | 285 + .../attention/double_sparsity_backend.py | 257 + .../dual_chunk_flashattention_backend.py | 1700 ++++ .../sglang/srt/layers/attention/fla/chunk.py | 241 + .../srt/layers/attention/fla/chunk_delta_h.py | 343 + .../srt/layers/attention/fla/chunk_o.py | 178 + .../attention/fla/chunk_scaled_dot_kkt.py | 151 + .../sglang/srt/layers/attention/fla/cumsum.py | 300 + .../layers/attention/fla/fused_gdn_gating.py | 69 + .../layers/attention/fla/fused_recurrent.py | 723 ++ .../fla/fused_sigmoid_gating_recurrent.py | 232 + .../sglang/srt/layers/attention/fla/index.py | 35 + .../sglang/srt/layers/attention/fla/kda.py | 1359 ++++ .../sglang/srt/layers/attention/fla/l2norm.py | 150 + .../layers/attention/fla/layernorm_gated.py | 350 + .../sglang/srt/layers/attention/fla/op.py | 66 + .../srt/layers/attention/fla/solve_tril.py | 465 ++ .../sglang/srt/layers/attention/fla/utils.py | 328 + .../srt/layers/attention/fla/wy_fast.py | 156 + .../attention/flashattention_backend.py | 2523 ++++++ .../layers/attention/flashinfer_backend.py | 1606 ++++ .../attention/flashinfer_mla_backend.py | 1090 +++ .../srt/layers/attention/flashmla_backend.py | 596 ++ .../layers/attention/hybrid_attn_backend.py | 147 + .../attention/hybrid_linear_attn_backend.py | 994 +++ .../srt/layers/attention/intel_amx_backend.py | 131 + .../layers/attention/mamba/causal_conv1d.py | 129 + .../attention/mamba/causal_conv1d_triton.py | 1170 +++ .../srt/layers/attention/mamba/mamba.py | 586 ++ .../layers/attention/mamba/mamba2_metadata.py | 215 + .../attention/mamba/mixer2_rms_norm_gated.py | 120 + .../layers/attention/mamba/ops/__init__.py | 2 + .../attention/mamba/ops/layernorm_gated.py | 172 + .../layers/attention/mamba/ops/mamba_ssm.py | 442 + .../srt/layers/attention/mamba/ops/ssd_bmm.py | 214 + .../attention/mamba/ops/ssd_chunk_scan.py | 562 ++ .../attention/mamba/ops/ssd_chunk_state.py | 646 ++ .../attention/mamba/ops/ssd_combined.py | 261 + .../attention/mamba/ops/ssd_state_passing.py | 264 + .../srt/layers/attention/merge_state.py | 46 + .../attention/npu_ops/mla_preprocess.py | 393 + .../layers/attention/nsa/dequant_k_cache.py | 295 + .../attention/nsa/index_buf_accessor.py | 365 + .../srt/layers/attention/nsa/nsa_indexer.py | 1129 +++ .../srt/layers/attention/nsa/quant_k_cache.py | 287 + .../layers/attention/nsa/tilelang_kernel.py | 785 ++ .../layers/attention/nsa/transform_index.py | 144 + .../srt/layers/attention/nsa/triton_kernel.py | 136 + .../sglang/srt/layers/attention/nsa/utils.py | 328 + .../srt/layers/attention/nsa_backend.py | 1572 ++++ .../srt/layers/attention/tbo_backend.py | 260 + .../layers/attention/torch_flex_backend.py | 325 + .../layers/attention/torch_native_backend.py | 276 + .../srt/layers/attention/triton_backend.py | 1317 +++ .../attention/triton_ops/decode_attention.py | 1345 +++ .../triton_ops/decode_attention.py.bak | 776 ++ .../triton_ops/decode_attention.py.v5_backup | 777 ++ .../triton_ops/double_sparsity_attention.py | 1106 +++ .../attention/triton_ops/extend_attention.py | 1045 +++ .../attention/triton_ops/merge_state.py | 96 + .../attention/triton_ops/prefill_attention.py | 217 + .../triton_ops/rocm_mla_decode_rope.py | 439 + .../layers/attention/trtllm_mha_backend.py | 717 ++ .../layers/attention/trtllm_mla_backend.py | 1114 +++ .../sglang/srt/layers/attention/utils.py | 181 + .../sglang/srt/layers/attention/vision.py | 754 ++ .../srt/layers/attention/vision_utils.py | 65 + .../srt/layers/attention/wave_backend.py | 627 ++ .../attention/wave_ops/decode_attention.py | 184 + .../attention/wave_ops/extend_attention.py | 147 + .../attention/wave_ops/prefill_attention.py | 79 + .../srt/layers/attention/xpu_backend.py | 1028 +++ .../python/sglang/srt/layers/communicator.py | 937 +++ .../sglang/srt/layers/communicator_nsa_cp.py | 284 + .../srt/layers/deep_gemm_wrapper/__init__.py | 1 + .../layers/deep_gemm_wrapper/compile_utils.py | 248 + .../layers/deep_gemm_wrapper/configurer.py | 25 + .../layers/deep_gemm_wrapper/entrypoint.py | 119 + .../python/sglang/srt/layers/dp_attention.py | 572 ++ .../python/sglang/srt/layers/elementwise.py | 605 ++ .../srt/layers/flashinfer_comm_fusion.py | 232 + .../python/sglang/srt/layers/layernorm.py | 472 ++ .../sglang/python/sglang/srt/layers/linear.py | 1399 ++++ .../sglang/srt/layers/logits_processor.py | 1213 +++ .../sglang/srt/layers/model_parallel.py | 155 + .../sglang/srt/layers/modelopt_utils.py | 11 + .../python/sglang/srt/layers/moe/__init__.py | 30 + .../sglang/srt/layers/moe/cutlass_moe.py | 401 + .../srt/layers/moe/cutlass_moe_params.py | 169 + .../sglang/srt/layers/moe/cutlass_w4a8_moe.py | 540 ++ .../sglang/srt/layers/moe/ep_moe/__init__.py | 0 .../sglang/srt/layers/moe/ep_moe/kernels.py | 1383 ++++ .../sglang/srt/layers/moe/ep_moe/layer.py | 578 ++ .../srt/layers/moe/flashinfer_cutedsl_moe.py | 183 + .../sglang/srt/layers/moe/fused_moe_native.py | 107 + .../layers/moe/fused_moe_triton/__init__.py | 42 + .../moe/fused_moe_triton/configs/README.md | 40 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 146 + ...336,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 218 + ...792,device_name=NVIDIA_A100-SXM4-80GB.json | 218 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 218 + ...VIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json | 218 + ...072,device_name=NVIDIA_H100_80GB_HBM3.json | 218 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 218 + ...584,device_name=NVIDIA_A100-SXM4-80GB.json | 218 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 218 + ...168,device_name=NVIDIA_A100-SXM4-80GB.json | 218 + ...512,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...024,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + .../E=16,N=1024,device_name=NVIDIA_H200.json | 146 + ...344,device_name=NVIDIA_A100-SXM4-40GB.json | 146 + ...344,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...344,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 146 + ...336,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 218 + ...792,device_name=NVIDIA_A100-SXM4-80GB.json | 218 + ...048,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...688,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...688,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 146 + ...VIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 130 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 146 + ...584,device_name=NVIDIA_A100-SXM4-80GB.json | 218 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 130 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 146 + ...168,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...VIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 130 + ...192,device_name=NVIDIA_A800-SXM4-80GB.json | 146 + ...048,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...024,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8.json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...ame=NVIDIA_H20,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...=64,device_name=NVIDIA_A800-SXM4-80GB.json | 146 + ...evice_name=NVIDIA_L20,dtype=int8_w8a8.json | 146 + ...vice_name=NVIDIA_L40S,dtype=int8_w8a8.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...280,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...280,device_name=NVIDIA_A800-SXM4-80GB.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...280,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=64,N=1280,device_name=NVIDIA_H200.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=64,N=2560,device_name=NVIDIA_H200.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...320,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=64,N=320,device_name=NVIDIA_H200.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...640,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...640,device_name=NVIDIA_A800-SXM4-80GB.json | 146 + ...VIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...640,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=64,N=640,device_name=NVIDIA_H200.json | 146 + ...14336,device_name=AMD_Instinct_MI300X.json | 200 + ...14336,device_name=AMD_Instinct_MI325X.json | 200 + ...14336,device_name=AMD_Radeon_Graphics.json | 200 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 138 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=8,N=14336,device_name=NVIDIA_H200.json | 146 + ...=1792,device_name=AMD_Instinct_MI300X.json | 200 + ...=1792,device_name=AMD_Instinct_MI325X.json | 200 + ...=1792,device_name=AMD_Radeon_Graphics.json | 200 + ...792,device_name=NVIDIA_A100-SXM4-40GB.json | 146 + ...792,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...792,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=8,N=1792,device_name=NVIDIA_H200.json | 146 + ...048,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...048,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=8,N=2048,device_name=NVIDIA_H200.json | 146 + ...=3584,device_name=AMD_Instinct_MI300X.json | 200 + ...=3584,device_name=AMD_Instinct_MI325X.json | 200 + ...=3584,device_name=AMD_Radeon_Graphics.json | 200 + ...584,device_name=NVIDIA_A100-SXM4-40GB.json | 146 + ...584,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...VIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...584,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=8,N=3584,device_name=NVIDIA_H200.json | 146 + .../E=8,N=3584,device_name=NVIDIA_L40S.json | 173 + ...me=AMD_Instinct_MI300X,dtype=fp8_w8a8.json | 178 + ...me=AMD_Instinct_MI325X,dtype=fp8_w8a8.json | 178 + ...me=AMD_Radeon_Graphics,dtype=fp8_w8a8.json | 178 + ...096,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...096,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=8,N=4096,device_name=NVIDIA_H200.json | 146 + ...=7168,device_name=AMD_Instinct_MI300X.json | 200 + ...=7168,device_name=AMD_Instinct_MI325X.json | 200 + ...=7168,device_name=AMD_Radeon_Graphics.json | 200 + ...168,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...168,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=8,N=7168,device_name=NVIDIA_H200.json | 146 + ...me=AMD_Instinct_MI300X,dtype=fp8_w8a8.json | 175 + ...me=AMD_Instinct_MI325X,dtype=fp8_w8a8.json | 175 + ...me=AMD_Radeon_Graphics,dtype=fp8_w8a8.json | 175 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + ...192,device_name=NVIDIA_A800-SXM4-80GB.json | 146 + ...192,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + .../E=128,N=192,device_name=NVIDIA_H20.json | 146 + .../E=128,N=192,device_name=NVIDIA_H200.json | 146 + ...384,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + .../E=128,N=384,device_name=NVIDIA_H20.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + .../E=128,N=384,device_name=NVIDIA_H200.json | 146 + ...512,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...768,device_name=NVIDIA_A800-SXM4-80GB.json | 146 + ...768,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + .../E=128,N=768,device_name=NVIDIA_H20.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + .../E=128,N=768,device_name=NVIDIA_H200.json | 146 + .../E=128,N=96,device_name=NVIDIA_H20.json | 146 + ...NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json | 146 + ...NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json | 146 + ...NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...=64,device_name=NVIDIA_A800-SXM4-80GB.json | 146 + ...=64,device_name=NVIDIA_A800-SXM4-80GB.json | 146 + ...168,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + .../E=16,N=1024,device_name=NVIDIA_B200.json | 146 + ...TX_6000_Ada_Generation,dtype=fp8_w8a8.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + .../E=128,N=768,device_name=NVIDIA_H20.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + ...E=160,N=320,device_name=NVIDIA_H20-3e.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...168,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...856,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + .../E=128,N=1856,device_name=NVIDIA_L40S.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + ...TX_5880_Ada_Generation,dtype=fp8_w8a8.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + .../E=128,N=928,device_name=NVIDIA_L40S.json | 146 + ...evice_name=NVIDIA_B200,dtype=fp8_w8a8.json | 146 + ...-Q_Workstation_Edition,dtype=fp8_w8a8.json | 146 + ...evice_name=NVIDIA_B200,dtype=fp8_w8a8.json | 146 + ...evice_name=NVIDIA_B200,dtype=fp8_w8a8.json | 146 + ...dtype=fp8_w8a8,per_channel_quant=True.json | 146 + ...-Q_Workstation_Edition,dtype=fp8_w8a8.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + .../E=256,N=256,device_name=NVIDIA_B200.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + .../E=256,N=512,device_name=NVIDIA_B200.json | 146 + .../E=256,N=512,device_name=NVIDIA_H20.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...=fp8_w8a8,block_shape=[128, 128]_down.json | 164 + ...=64,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...128,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...E=512,N=128,device_name=NVIDIA_H20-3e.json | 146 + .../E=512,N=128,device_name=NVIDIA_H200.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + .../E=512,N=256,device_name=NVIDIA_B200.json | 146 + ...E=512,N=256,device_name=NVIDIA_H20-3e.json | 146 + .../E=512,N=256,device_name=NVIDIA_H200.json | 146 + ...=64,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + .../E=512,N=64,device_name=NVIDIA_H200.json | 146 + .../moe/fused_moe_triton/fused_marlin_moe.py | 233 + .../layers/moe/fused_moe_triton/fused_moe.py | 752 ++ .../fused_moe_triton_config.py | 290 + .../fused_moe_triton_kernels.py | 950 +++ .../srt/layers/moe/fused_moe_triton/layer.py | 1178 +++ .../fused_moe_triton/moe_align_block_size.py | 82 + .../fused_moe_triton/triton_kernels_moe.py | 314 + .../sglang/srt/layers/moe/kt_ep_wrapper.py | 393 + .../srt/layers/moe/moe_runner/__init__.py | 4 + .../sglang/srt/layers/moe/moe_runner/base.py | 280 + .../srt/layers/moe/moe_runner/deep_gemm.py | 585 ++ .../srt/layers/moe/moe_runner/runner.py | 86 + .../srt/layers/moe/moe_runner/triton.py | 456 ++ .../layers/moe/moe_runner/triton_kernels.py | 198 + .../sglang/srt/layers/moe/rocm_moe_utils.py | 140 + .../python/sglang/srt/layers/moe/router.py | 428 + .../layers/moe/token_dispatcher/__init__.py | 53 + .../srt/layers/moe/token_dispatcher/base.py | 353 + .../srt/layers/moe/token_dispatcher/deepep.py | 846 ++ .../srt/layers/moe/token_dispatcher/fuseep.py | 97 + .../layers/moe/token_dispatcher/mooncake.py | 393 + .../layers/moe/token_dispatcher/standard.py | 192 + .../python/sglang/srt/layers/moe/topk.py | 1102 +++ .../python/sglang/srt/layers/moe/utils.py | 277 + .../python/sglang/srt/layers/multimodal.py | 189 + .../python/sglang/srt/layers/parameter.py | 550 ++ .../sglang/python/sglang/srt/layers/pooler.py | 131 + .../srt/layers/quantization/__init__.py | 94 + .../srt/layers/quantization/auto_round.py | 394 + .../sglang/srt/layers/quantization/awq.py | 977 +++ .../srt/layers/quantization/awq_triton.py | 368 + .../srt/layers/quantization/base_config.py | 242 + .../srt/layers/quantization/blockwise_int8.py | 380 + .../quantization/compressed_tensors/README.md | 6 + .../compressed_tensors/compressed_tensors.py | 657 ++ .../compressed_tensors_moe.py | 654 ++ .../compressed_tensors/schemes/__init__.py | 16 + .../schemes/compressed_tensors_scheme.py | 56 + .../schemes/compressed_tensors_w8a16_fp8.py | 135 + .../schemes/compressed_tensors_w8a8_fp8.py | 184 + .../schemes/compressed_tensors_w8a8_int8.py | 173 + .../schemes/compressed_tensors_wNa16.py | 339 + .../quantization/compressed_tensors/utils.py | 218 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + .../sglang/srt/layers/quantization/fp8.py | 1353 +++ .../srt/layers/quantization/fp8_kernel.py | 1863 +++++ .../srt/layers/quantization/fp8_utils.py | 973 +++ .../srt/layers/quantization/fpgemm_fp8.py | 202 + .../sglang/srt/layers/quantization/gguf.py | 566 ++ .../sglang/srt/layers/quantization/gptq.py | 1143 +++ .../srt/layers/quantization/int8_kernel.py | 441 + .../srt/layers/quantization/int8_utils.py | 73 + .../srt/layers/quantization/kv_cache.py | 81 + .../srt/layers/quantization/kvfp4_tensor.py | 112 + .../srt/layers/quantization/marlin_utils.py | 997 +++ .../layers/quantization/marlin_utils_fp8.py | 352 + .../srt/layers/quantization/modelopt_quant.py | 1910 +++++ .../srt/layers/quantization/moe_wna16.py | 501 ++ .../sglang/srt/layers/quantization/mxfp4.py | 870 ++ .../srt/layers/quantization/mxfp4_tensor.py | 135 + .../sglang/srt/layers/quantization/petit.py | 252 + .../srt/layers/quantization/petit_utils.py | 104 + .../sglang/srt/layers/quantization/qoq.py | 245 + .../srt/layers/quantization/quark/__init__.py | 0 .../srt/layers/quantization/quark/quark.py | 433 + .../layers/quantization/quark/quark_moe.py | 527 ++ .../quantization/quark/schemes/__init__.py | 7 + .../quark/schemes/quark_scheme.py | 55 + .../quark/schemes/quark_w4a4_mxfp4.py | 135 + .../quark/schemes/quark_w8a8_fp8.py | 186 + .../srt/layers/quantization/quark/utils.py | 214 + .../layers/quantization/rocm_mxfp4_utils.py | 13 + .../sglang/srt/layers/quantization/unquant.py | 475 ++ .../sglang/srt/layers/quantization/utils.py | 563 ++ .../sglang/srt/layers/quantization/w4afp8.py | 408 + .../srt/layers/quantization/w8a8_fp8.py | 305 + .../srt/layers/quantization/w8a8_int8.py | 1511 ++++ .../sglang/srt/layers/radix_attention.py | 190 + .../sglang/srt/layers/rocm_linear_utils.py | 44 + .../sglang/srt/layers/rotary_embedding.py | 2905 +++++++ .../python/sglang/srt/layers/sampler.py | 581 ++ .../python/sglang/srt/layers/sparse_pooler.py | 98 + .../python/sglang/srt/layers/torchao_utils.py | 135 + .../sglang/python/sglang/srt/layers/utils.py | 58 + .../srt/layers/vocab_parallel_embedding.py | 575 ++ .../sglang/srt/lora/backend/ascend_backend.py | 287 + .../sglang/srt/lora/backend/base_backend.py | 134 + .../srt/lora/backend/chunked_backend.py | 379 + .../sglang/srt/lora/backend/lora_registry.py | 53 + .../sglang/srt/lora/backend/triton_backend.py | 197 + .../python/sglang/srt/lora/eviction_policy.py | 139 + .../sglang/python/sglang/srt/lora/layers.py | 356 + .../sglang/python/sglang/srt/lora/lora.py | 172 + .../python/sglang/srt/lora/lora_config.py | 47 + .../python/sglang/srt/lora/lora_manager.py | 455 ++ .../python/sglang/srt/lora/lora_registry.py | 252 + .../sglang/python/sglang/srt/lora/mem_pool.py | 333 + .../sglang/srt/lora/triton_ops/__init__.py | 15 + .../lora/triton_ops/chunked_sgmv_expand.py | 214 + .../lora/triton_ops/chunked_sgmv_shrink.py | 176 + .../srt/lora/triton_ops/gate_up_lora_b.py | 187 + .../sglang/srt/lora/triton_ops/qkv_lora_b.py | 198 + .../srt/lora/triton_ops/sgemm_lora_a.py | 170 + .../srt/lora/triton_ops/sgemm_lora_b.py | 176 + .../sglang/python/sglang/srt/lora/utils.py | 134 + .../managers/async_dynamic_batch_tokenizer.py | 170 + .../srt/managers/async_mm_data_processor.py | 122 + .../sglang/srt/managers/cache_controller.py | 823 ++ .../sglang/srt/managers/configure_logging.py | 47 + .../srt/managers/data_parallel_controller.py | 550 ++ .../srt/managers/detokenizer_manager.py | 325 + .../sglang/srt/managers/disagg_service.py | 46 + .../python/sglang/srt/managers/io_struct.py | 1638 ++++ .../python/sglang/srt/managers/mm_utils.py | 882 ++ .../srt/managers/multi_tokenizer_mixin.py | 507 ++ .../srt/managers/multimodal_processor.py | 53 + .../sglang/srt/managers/overlap_utils.py | 158 + .../srt/managers/request_metrics_exporter.py | 209 + .../sglang/srt/managers/schedule_batch.py | 2078 +++++ .../sglang/srt/managers/schedule_policy.py | 725 ++ .../python/sglang/srt/managers/scheduler.py | 2698 ++++++ .../srt/managers/scheduler_dp_attn_mixin.py | 201 + .../srt/managers/scheduler_input_blocker.py | 106 + .../srt/managers/scheduler_metrics_mixin.py | 426 + .../scheduler_output_processor_mixin.py | 1049 +++ .../sglang/srt/managers/scheduler_pp_mixin.py | 349 + .../srt/managers/scheduler_profiler_mixin.py | 402 + .../srt/managers/scheduler_recv_skipper.py | 38 + .../scheduler_runtime_checker_mixin.py | 342 + .../scheduler_update_weights_mixin.py | 212 + .../sglang/srt/managers/session_controller.py | 159 + .../sglang/srt/managers/template_manager.py | 308 + .../managers/tokenizer_communicator_mixin.py | 784 ++ .../sglang/srt/managers/tokenizer_manager.py | 2509 ++++++ .../python/sglang/srt/managers/tp_worker.py | 475 ++ .../python/sglang/srt/managers/utils.py | 170 + .../python/sglang/srt/mem_cache/allocator.py | 580 ++ .../sglang/srt/mem_cache/allocator_ascend.py | 186 + .../sglang/srt/mem_cache/base_prefix_cache.py | 150 + .../sglang/srt/mem_cache/cache_init_params.py | 26 + .../sglang/srt/mem_cache/chunk_cache.py | 105 + .../python/sglang/srt/mem_cache/common.py | 507 ++ .../mem_cache/cpp_radix_tree/.clang-format | 1 + .../srt/mem_cache/cpp_radix_tree/common.h | 29 + .../mem_cache/cpp_radix_tree/radix_tree.py | 182 + .../srt/mem_cache/cpp_radix_tree/tree_v2.cpp | 143 + .../srt/mem_cache/cpp_radix_tree/tree_v2.h | 59 + .../cpp_radix_tree/tree_v2_binding.cpp | 32 + .../cpp_radix_tree/tree_v2_debug.cpp | 194 + .../mem_cache/cpp_radix_tree/tree_v2_impl.h | 286 + .../mem_cache/cpp_radix_tree/tree_v2_node.h | 257 + .../sglang/srt/mem_cache/evict_policy.py | 46 + .../sglang/srt/mem_cache/flush_cache.py | 33 + .../sglang/srt/mem_cache/hicache_storage.py | 287 + .../sglang/srt/mem_cache/hiradix_cache.py | 946 +++ .../sglang/srt/mem_cache/mamba_radix_cache.py | 1013 +++ .../sglang/srt/mem_cache/memory_pool.py | 2179 +++++ .../sglang/srt/mem_cache/memory_pool_host.py | 933 +++ .../sglang/srt/mem_cache/multimodal_cache.py | 133 + .../sglang/srt/mem_cache/radix_cache.py | 849 ++ .../sglang/srt/mem_cache/radix_cache_cpp.py | 243 + .../sglang/srt/mem_cache/storage/__init__.py | 10 + .../storage/aibrix_kvcache/README.md | 37 + .../aibrix_kvcache/aibrix_kvcache_storage.py | 157 + .../storage/aibrix_kvcache/unit_test.py | 97 + .../srt/mem_cache/storage/backend_factory.py | 223 + .../srt/mem_cache/storage/eic/README.md | 24 + .../srt/mem_cache/storage/eic/eic_storage.py | 777 ++ .../srt/mem_cache/storage/eic/test_unit.py | 115 + .../mem_cache/storage/hf3fs/docs/README.md | 54 + .../hf3fs/docs/deploy_sglang_3fs_multinode.md | 65 + .../storage/hf3fs/docs/setup_usrbio_client.md | 68 + .../mem_cache/storage/hf3fs/hf3fs_client.py | 163 + .../storage/hf3fs/hf3fs_usrbio_client.py | 200 + .../mem_cache/storage/hf3fs/hf3fs_utils.cpp | 35 + .../storage/hf3fs/mini_3fs_metadata_server.py | 471 ++ .../mem_cache/storage/hf3fs/storage_hf3fs.py | 651 ++ .../storage/hf3fs/test_hf3fs_utils.py | 43 + .../srt/mem_cache/storage/lmcache/README.md | 43 + .../storage/lmcache/example_config.yaml | 7 + .../storage/lmcache/lmc_radix_cache.py | 268 + .../mem_cache/storage/lmcache/unit_test.py | 121 + .../storage/mooncake_store/README.md | 350 + .../storage/mooncake_store/mooncake_store.py | 565 ++ .../mooncake_store/test_mooncake_store.py | 189 + .../srt/mem_cache/storage/nixl/README.md | 172 + .../mem_cache/storage/nixl/hicache_nixl.py | 286 + .../srt/mem_cache/storage/nixl/nixl_utils.py | 204 + .../storage/nixl/test_hicache_nixl_storage.py | 270 + .../sglang/srt/mem_cache/swa_radix_cache.py | 1093 +++ .../python/sglang/srt/mem_cache/utils.py | 253 + .../python/sglang/srt/metrics/collector.py | 1223 +++ .../python/sglang/srt/metrics/func_timer.py | 101 + .../sglang/srt/metrics/label_transform.py | 27 + .../srt/metrics/startup_func_log_and_timer.py | 150 + .../sglang/python/sglang/srt/metrics/utils.py | 55 + .../srt/model_executor/cpu_graph_runner.py | 640 ++ .../srt/model_executor/cuda_graph_runner.py | 914 +++ .../srt/model_executor/forward_batch_info.py | 1208 +++ .../sglang/srt/model_executor/hook_manager.py | 82 + .../srt/model_executor/input_buffers.py | 180 + .../srt/model_executor/mindspore_runner.py | 118 + .../sglang/srt/model_executor/model_runner.py | 2871 +++++++ .../srt/model_executor/npu_graph_runner.py | 172 + .../piecewise_cuda_graph_runner.py | 680 ++ .../sglang/srt/model_loader/__init__.py | 40 + .../python/sglang/srt/model_loader/loader.py | 2103 +++++ .../remote_instance_weight_loader_utils.py | 69 + .../python/sglang/srt/model_loader/utils.py | 171 + .../sglang/srt/model_loader/weight_utils.py | 1225 +++ .../srt/model_loader/weight_validation.py | 220 + .../python/sglang/srt/models/apertus.py | 685 ++ .../sglang/python/sglang/srt/models/arcee.py | 532 ++ .../python/sglang/srt/models/baichuan.py | 441 + .../python/sglang/srt/models/bailing_moe.py | 996 +++ .../sglang/srt/models/bailing_moe_nextn.py | 167 + .../sglang/python/sglang/srt/models/bert.py | 498 ++ .../python/sglang/srt/models/chatglm.py | 427 + .../sglang/python/sglang/srt/models/clip.py | 559 ++ .../python/sglang/srt/models/commandr.py | 418 + .../sglang/python/sglang/srt/models/dbrx.py | 460 ++ .../python/sglang/srt/models/deepseek.py | 489 ++ .../sglang/srt/models/deepseek_janus_pro.py | 2063 +++++ .../sglang/srt/models/deepseek_nextn.py | 250 + .../python/sglang/srt/models/deepseek_ocr.py | 1516 ++++ .../python/sglang/srt/models/deepseek_v2.py | 4124 ++++++++++ .../python/sglang/srt/models/deepseek_vl2.py | 362 + .../python/sglang/srt/models/dots_ocr.py | 171 + .../python/sglang/srt/models/dots_vlm.py | 185 + .../python/sglang/srt/models/dots_vlm_vit.py | 328 + .../sglang/python/sglang/srt/models/ernie4.py | 426 + .../python/sglang/srt/models/ernie4_eagle.py | 203 + .../sglang/python/sglang/srt/models/exaone.py | 377 + .../python/sglang/srt/models/falcon_h1.py | 570 ++ .../sglang/python/sglang/srt/models/gemma.py | 410 + .../sglang/python/sglang/srt/models/gemma2.py | 480 ++ .../python/sglang/srt/models/gemma2_reward.py | 70 + .../python/sglang/srt/models/gemma3_causal.py | 762 ++ .../python/sglang/srt/models/gemma3_mm.py | 460 ++ .../python/sglang/srt/models/gemma3n_audio.py | 949 +++ .../sglang/srt/models/gemma3n_causal.py | 1010 +++ .../python/sglang/srt/models/gemma3n_mm.py | 533 ++ .../sglang/python/sglang/srt/models/glm4.py | 632 ++ .../python/sglang/srt/models/glm4_moe.py | 1279 +++ .../sglang/srt/models/glm4_moe_nextn.py | 162 + .../sglang/python/sglang/srt/models/glm4v.py | 759 ++ .../python/sglang/srt/models/glm4v_moe.py | 232 + .../sglang/python/sglang/srt/models/gpt2.py | 288 + .../python/sglang/srt/models/gpt_bigcode.py | 304 + .../python/sglang/srt/models/gpt_oss.py | 1172 +++ .../python/sglang/srt/models/granite.py | 505 ++ .../python/sglang/srt/models/granitemoe.py | 387 + .../sglang/python/sglang/srt/models/grok.py | 979 +++ .../python/sglang/srt/models/hunyuan.py | 814 ++ .../python/sglang/srt/models/idefics2.py | 341 + .../python/sglang/srt/models/internlm2.py | 357 + .../sglang/srt/models/internlm2_reward.py | 64 + .../python/sglang/srt/models/interns1.py | 274 + .../python/sglang/srt/models/internvl.py | 743 ++ .../python/sglang/srt/models/jet_nemotron.py | 599 ++ .../python/sglang/srt/models/jet_vlm.py | 143 + .../python/sglang/srt/models/kimi_linear.py | 710 ++ .../python/sglang/srt/models/kimi_vl.py | 307 + .../sglang/srt/models/kimi_vl_moonvit.py | 641 ++ .../sglang/python/sglang/srt/models/llada2.py | 941 +++ .../sglang/python/sglang/srt/models/llama.py | 753 ++ .../sglang/python/sglang/srt/models/llama4.py | 562 ++ .../sglang/srt/models/llama_classification.py | 81 + .../python/sglang/srt/models/llama_eagle.py | 149 + .../python/sglang/srt/models/llama_eagle3.py | 276 + .../sglang/srt/models/llama_embedding.py | 87 + .../python/sglang/srt/models/llama_reward.py | 126 + .../sglang/python/sglang/srt/models/llava.py | 845 ++ .../python/sglang/srt/models/llavavid.py | 283 + .../python/sglang/srt/models/longcat_flash.py | 1049 +++ .../sglang/srt/models/longcat_flash_nextn.py | 686 ++ .../sglang/python/sglang/srt/models/mimo.py | 160 + .../python/sglang/srt/models/mimo_mtp.py | 203 + .../python/sglang/srt/models/mindspore.py | 305 + .../python/sglang/srt/models/minicpm.py | 399 + .../python/sglang/srt/models/minicpm3.py | 517 ++ .../python/sglang/srt/models/minicpmo.py | 1916 +++++ .../python/sglang/srt/models/minicpmv.py | 1046 +++ .../python/sglang/srt/models/minimax_m2.py | 913 +++ .../python/sglang/srt/models/mistral.py | 93 + .../python/sglang/srt/models/mixtral.py | 478 ++ .../python/sglang/srt/models/mixtral_quant.py | 430 + .../sglang/python/sglang/srt/models/mllama.py | 1059 +++ .../python/sglang/srt/models/mllama4.py | 1008 +++ .../sglang/srt/models/nano_nemotron_vl.py | 219 + .../python/sglang/srt/models/nemotron_h.py | 739 ++ .../python/sglang/srt/models/nemotron_nas.py | 435 + .../sglang/python/sglang/srt/models/nvila.py | 355 + .../python/sglang/srt/models/nvila_lite.py | 184 + .../sglang/python/sglang/srt/models/olmo.py | 377 + .../sglang/python/sglang/srt/models/olmo2.py | 441 + .../sglang/python/sglang/srt/models/olmoe.py | 437 + .../sglang/python/sglang/srt/models/opt.py | 637 ++ .../sglang/python/sglang/srt/models/orion.py | 369 + .../python/sglang/srt/models/persimmon.py | 330 + .../sglang/python/sglang/srt/models/phi.py | 321 + .../python/sglang/srt/models/phi3_small.py | 476 ++ .../sglang/python/sglang/srt/models/phi4mm.py | 538 ++ .../python/sglang/srt/models/phi4mm_audio.py | 1260 +++ .../python/sglang/srt/models/phi4mm_utils.py | 1917 +++++ .../sglang/python/sglang/srt/models/phimoe.py | 559 ++ .../python/sglang/srt/models/pixtral.py | 459 ++ .../sglang/srt/models/points_v15_chat.py | 186 + .../sglang/python/sglang/srt/models/qwen.py | 355 + .../sglang/python/sglang/srt/models/qwen2.py | 668 ++ .../python/sglang/srt/models/qwen2_5_vl.py | 736 ++ .../python/sglang/srt/models/qwen2_audio.py | 188 + .../sglang/srt/models/qwen2_classification.py | 75 + .../python/sglang/srt/models/qwen2_eagle.py | 146 + .../python/sglang/srt/models/qwen2_moe.py | 859 ++ .../python/sglang/srt/models/qwen2_rm.py | 78 + .../python/sglang/srt/models/qwen2_vl.py | 599 ++ .../sglang/python/sglang/srt/models/qwen3.py | 604 ++ .../sglang/srt/models/qwen3_classification.py | 84 + .../python/sglang/srt/models/qwen3_moe.py | 968 +++ .../python/sglang/srt/models/qwen3_next.py | 1086 +++ .../sglang/srt/models/qwen3_next_mtp.py | 111 + .../sglang/srt/models/qwen3_omni_moe.py | 658 ++ .../python/sglang/srt/models/qwen3_vl.py | 817 ++ .../python/sglang/srt/models/qwen3_vl_moe.py | 339 + .../sglang/python/sglang/srt/models/radio.py | 532 ++ .../python/sglang/srt/models/registry.py | 128 + .../python/sglang/srt/models/roberta.py | 338 + .../sglang/srt/models/sarashina2_vision.py | 268 + .../sglang/python/sglang/srt/models/siglip.py | 281 + .../sglang/python/sglang/srt/models/solar.py | 505 ++ .../python/sglang/srt/models/stablelm.py | 331 + .../python/sglang/srt/models/starcoder2.py | 357 + .../python/sglang/srt/models/step3_vl.py | 1000 +++ .../python/sglang/srt/models/teleflm.py | 104 + .../sglang/srt/models/torch_native_llama.py | 507 ++ .../python/sglang/srt/models/transformers.py | 288 + .../sglang/python/sglang/srt/models/utils.py | 84 + .../sglang/python/sglang/srt/models/xverse.py | 382 + .../python/sglang/srt/models/xverse_moe.py | 478 ++ .../sglang/python/sglang/srt/models/yivl.py | 115 + .../customized_mm_processor_utils.py | 35 + .../sglang/srt/multimodal/internvl_utils.py | 115 + .../python/sglang/srt/multimodal/mm_utils.py | 651 ++ .../multimodal/processors/base_processor.py | 739 ++ .../sglang/srt/multimodal/processors/clip.py | 35 + .../srt/multimodal/processors/deepseek_ocr.py | 37 + .../multimodal/processors/deepseek_vl_v2.py | 62 + .../srt/multimodal/processors/dots_vlm.py | 81 + .../srt/multimodal/processors/gemma3.py | 54 + .../srt/multimodal/processors/gemma3n.py | 71 + .../sglang/srt/multimodal/processors/glm4v.py | 87 + .../srt/multimodal/processors/internvl.py | 280 + .../srt/multimodal/processors/janus_pro.py | 44 + .../srt/multimodal/processors/kimi_vl.py | 46 + .../sglang/srt/multimodal/processors/llava.py | 211 + .../srt/multimodal/processors/minicpm.py | 144 + .../sglang/srt/multimodal/processors/mlama.py | 37 + .../srt/multimodal/processors/mllama4.py | 49 + .../multimodal/processors/nano_nemotron_vl.py | 197 + .../sglang/srt/multimodal/processors/nvila.py | 79 + .../srt/multimodal/processors/phi4mm.py | 100 + .../srt/multimodal/processors/pixtral.py | 100 + .../multimodal/processors/points_v15_chat.py | 42 + .../srt/multimodal/processors/qwen_audio.py | 67 + .../srt/multimodal/processors/qwen_vl.py | 361 + .../processors/sarashina2_vision.py | 81 + .../srt/multimodal/processors/step3_vl.py | 517 ++ .../srt/multiplex/multiplexing_mixin.py | 209 + .../sglang/srt/multiplex/pdmux_context.py | 164 + .../python/sglang/srt/operations_strategy.py | 211 + .../srt/parser/code_completion_parser.py | 132 + .../python/sglang/srt/parser/conversation.py | 1087 +++ .../sglang/srt/parser/harmony_parser.py | 588 ++ .../sglang/srt/parser/jinja_template_utils.py | 203 + .../sglang/srt/parser/reasoning_parser.py | 336 + .../srt/sampling/custom_logit_processor.py | 194 + .../srt/sampling/penaltylib/__init__.py | 11 + .../sampling/penaltylib/frequency_penalty.py | 63 + .../srt/sampling/penaltylib/min_new_tokens.py | 93 + .../srt/sampling/penaltylib/orchestrator.py | 249 + .../sampling/penaltylib/presence_penalty.py | 63 + .../srt/sampling/sampling_batch_info.py | 405 + .../sglang/srt/sampling/sampling_params.py | 240 + .../sglang/python/sglang/srt/server_args.py | 4553 +++++++++++ .../sglang/srt/server_args_config_parser.py | 146 + .../srt/speculative/base_spec_worker.py | 34 + .../srt/speculative/cpp_ngram/.clang-format | 15 + .../srt/speculative/cpp_ngram/ngram.cpp | 381 + .../sglang/srt/speculative/cpp_ngram/ngram.h | 111 + .../srt/speculative/cpp_ngram/ngram_cache.py | 138 + .../cpp_ngram/ngram_cache_binding.cpp | 43 + .../sglang/srt/speculative/cpp_ngram/param.h | 126 + .../sglang/srt/speculative/cpp_ngram/queue.h | 73 + .../sglang/srt/speculative/draft_utils.py | 242 + .../eagle_draft_cuda_graph_runner.py | 390 + .../eagle_draft_extend_cuda_graph_runner.py | 458 ++ .../eagle_draft_extend_npu_graph_runner.py | 68 + .../eagle_draft_npu_graph_runner.py | 81 + .../sglang/srt/speculative/eagle_info.py | 796 ++ .../sglang/srt/speculative/eagle_info_v2.py | 538 ++ .../sglang/srt/speculative/eagle_utils.py | 199 + .../sglang/srt/speculative/eagle_worker.py | 1082 +++ .../sglang/srt/speculative/eagle_worker_v2.py | 811 ++ .../sglang/srt/speculative/ngram_info.py | 447 + .../sglang/srt/speculative/ngram_worker.py | 335 + .../sglang/srt/speculative/spec_info.py | 355 + .../sglang/srt/speculative/spec_utils.py | 697 ++ .../srt/speculative/standalone_worker.py | 102 + .../srt/tokenizer/tiktoken_tokenizer.py | 167 + .../sglang/python/sglang/srt/tracing/trace.py | 737 ++ .../python/sglang/srt/utils/__init__.py | 2 + .../python/sglang/srt/utils/aio_rwlock.py | 104 + .../python/sglang/srt/utils/bench_utils.py | 139 + .../sglang/python/sglang/srt/utils/common.py | 3793 +++++++++ .../srt/utils/cuda_ipc_transport_utils.py | 356 + .../sglang/srt/utils/hf_transformers_utils.py | 556 ++ .../sglang/srt/utils/host_shared_memory.py | 82 + .../python/sglang/srt/utils/numa_utils.py | 56 + .../sglang/srt/utils/nvtx_pytorch_hooks.py | 292 + .../python/sglang/srt/utils/offloader.py | 572 ++ .../python/sglang/srt/utils/patch_torch.py | 116 + .../sglang/srt/utils/poll_based_barrier.py | 31 + .../python/sglang/srt/utils/profile_merger.py | 199 + .../python/sglang/srt/utils/profile_utils.py | 388 + .../python/sglang/srt/utils/rpd_utils.py | 452 ++ .../sglang/srt/utils/slow_rank_detector.py | 71 + .../srt/utils/torch_memory_saver_adapter.py | 112 + .../python/sglang/srt/utils/weight_checker.py | 159 + .../sglang/srt/weight_sync/tensor_bucket.py | 107 + .../python/sglang/srt/weight_sync/utils.py | 119 + .../sglang/python/sglang/test/__init__.py | 0 .../python/sglang/test/attention/__init__.py | 0 .../test/attention/test_flashattn_backend.py | 419 + .../attention/test_flashattn_mla_backend.py | 329 + .../test/attention/test_prefix_chunk_info.py | 224 + .../test/attention/test_trtllm_mla_backend.py | 1436 ++++ .../python/sglang/test/ci/ci_register.py | 200 + .../python/sglang/test/ci/ci_stress_utils.py | 224 + .../sglang/python/sglang/test/ci/ci_utils.py | 134 + .../sglang/python/sglang/test/doc_patch.py | 59 + .../python/sglang/test/few_shot_gsm8k.py | 149 + .../sglang/test/few_shot_gsm8k_engine.py | 143 + .../python/sglang/test/get_logits_ut.py | 57 + .../sglang/python/sglang/test/gsm8k_mixin.py | 44 + .../sglang/test/kits/ebnf_constrained_kit.py | 224 + .../sglang/test/kits/json_constrained_kit.py | 101 + .../sglang/test/kits/matched_stop_kit.py | 157 + .../test/kits/radix_cache_server_kit.py | 50 + .../sglang/test/kits/regex_constrained_kit.py | 130 + .../sglang/python/sglang/test/long_prompt.txt | 1 + .../sglang/test/longbench_v2/__init__.py | 1 + .../longbench_v2/longbench_v2_evaluation.md | 217 + .../longbench_v2/test_longbench_v2_eval.py | 238 + .../longbench_v2/validate_longbench_v2.py | 337 + .../validate_longbench_v2_standalone.py | 306 + .../python/sglang/test/mmmu_vlm_mixin.py | 233 + .../python/sglang/test/nightly_bench_utils.py | 156 + .../sglang/python/sglang/test/run_eval.py | 238 + .../sglang/python/sglang/test/runners.py | 918 +++ .../sglang/python/sglang/test/send_one.py | 209 + .../python/sglang/test/simple_eval_common.py | 480 ++ .../python/sglang/test/simple_eval_gpqa.py | 94 + .../sglang/test/simple_eval_humaneval.py | 131 + .../sglang/test/simple_eval_longbench_v2.py | 344 + .../python/sglang/test/simple_eval_math.py | 74 + .../python/sglang/test/simple_eval_mgsm.py | 203 + .../python/sglang/test/simple_eval_mmlu.py | 122 + .../sglang/test/simple_eval_mmmu_vlm.py | 442 + .../test/speculative/test_spec_utils.py | 348 + .../python/sglang/test/test_activation.py | 105 + .../python/sglang/test/test_block_fp8.py | 660 ++ .../test_block_fp8_deep_gemm_blackwell.py | 251 + .../python/sglang/test/test_custom_ops.py | 148 + .../python/sglang/test/test_cutlass_moe.py | 306 + .../sglang/test/test_cutlass_w16a16_moe.py | 118 + .../sglang/test/test_cutlass_w4a8_moe.py | 285 + .../python/sglang/test/test_deepep_utils.py | 219 + .../python/sglang/test/test_deterministic.py | 690 ++ .../sglang/test/test_deterministic_utils.py | 74 + .../sglang/test/test_disaggregation_utils.py | 158 + .../sglang/test/test_dynamic_grad_mode.py | 58 + .../sglang/test/test_kvfp4_quant_dequant.py | 116 + .../python/sglang/test/test_layernorm.py | 185 + .../python/sglang/test/test_marlin_moe.py | 288 + .../python/sglang/test/test_marlin_utils.py | 171 + .../python/sglang/test/test_programs.py | 578 ++ .../sglang/python/sglang/test/test_utils.py | 1919 +++++ .../.vendor/sglang/python/sglang/utils.py | 576 ++ .../.vendor/sglang/python/sglang/version.py | 1 + .../sglang/scripts/check_vram_clear.sh | 27 + .../.vendor/sglang/scripts/ci/amd_ci_exec.sh | 60 + .../scripts/ci/amd_ci_install_dependency.sh | 81 + .../scripts/ci/amd_ci_start_container.sh | 162 + .../sglang/scripts/ci/ci_install_deepep.sh | 83 + .../scripts/ci/ci_install_dependency.sh | 149 + .../sglang/scripts/ci/ci_install_rust.sh | 24 + .../ci/ci_start_disaggregation_servers.sh | 106 + .../sglang/scripts/ci/cleanup_hf_cache.py | 146 + .../scripts/ci/npu_ci_install_dependency.sh | 66 + .../sglang/scripts/ci/prepare_runner.sh | 14 + .../sglang/scripts/ci/publish_traces.py | 382 + .../scripts/ci/slash_command_handler.py | 283 + .../sglang/scripts/ci_monitor/README.md | 334 + .../sglang/scripts/ci_monitor/ci_analyzer.py | 1206 +++ .../scripts/ci_monitor/ci_analyzer_balance.py | 533 ++ .../scripts/ci_monitor/ci_analyzer_perf.py | 1375 ++++ .../ci_monitor/ci_failures_analysis.py | 2139 +++++ .../sglang/scripts/code_sync/copy_from_oss.py | 299 + .../sglang/scripts/code_sync/copy_to_oss.py | 432 + .../sglang/scripts/code_sync/guideline.md | 27 + .../scripts/code_sync/install_github_cli.sh | 18 + .../sglang/scripts/convert_otel_2_perfetto.py | 375 + .../sglang/scripts/ensure_vram_clear.sh | 103 + .../sglang/scripts/export_deepseek_nextn.py | 115 + .../.vendor/sglang/scripts/killall_sglang.sh | 32 + .../scripts/playground/bench_speculative.py | 320 + .../playground/disaggregation/cli-logprob.py | 22 + .../playground/disaggregation/cli-so.py | 34 + .../scripts/playground/disaggregation/cli.py | 29 + .../playground/frontend_reasoning.ipynb | 241 + .../scripts/playground/load_tokenizer.py | 14 + .../playground/long_context_example.py | 36 + .../scripts/playground/lora/analyzer.py | 77 + .../scripts/playground/lora/lora_hf_play.py | 62 + .../scripts/playground/lora/lora_vllm_play.py | 30 + .../sglang/scripts/playground/reference_hf.py | 197 + .../scripts/playground/replay_request_dump.py | 151 + .../scripts/playground/router/test_tree.py | 207 + .../sglang/scripts/playground/router/tree.py | 292 + .../.vendor/sglang/scripts/release/README.md | 94 + .../scripts/release/bump_kernel_version.py | 32 + .../release/bump_kernel_version_to_sglang.py | 143 + .../scripts/release/bump_sglang_version.py | 39 + .../release/check_kernel_version_to_sglang.py | 148 + .../sglang/scripts/release/commit_and_pr.sh | 72 + .../release/commit_and_pr_kernel_to_sglang.sh | 81 + .../sglang/scripts/release/test_utils.py | 159 + .../.vendor/sglang/scripts/release/utils.py | 220 + .../scripts/sort_testcases_alphabetically.py | 27 + .../sglang/scripts/update_kernel_whl_index.py | 53 + .../sglang/scripts/version_branch_to_tag.sh | 30 + .../.vendor/sglang/sgl-kernel/.clang-format | 15 + .../.vendor/sglang/sgl-kernel/CMakeLists.txt | 598 ++ .../.vendor/sglang/sgl-kernel/LICENSE | 201 + .../.vendor/sglang/sgl-kernel/Makefile | 69 + .../.vendor/sglang/sgl-kernel/README.md | 107 + .../sglang/sgl-kernel/THIRDPARTYNOTICES.txt | 488 ++ .../sgl-kernel/benchmark/bench_activation.py | 211 + .../sgl-kernel/benchmark/bench_awq_dequant.py | 153 + .../sgl-kernel/benchmark/bench_cutlass_mla.py | 178 + .../benchmark/bench_dsv3_fused_a_gemm.py | 76 + .../benchmark/bench_dsv3_router_gemm.py | 154 + .../bench_es_fp8_blockwise_grouped_gemm.py | 338 + .../sgl-kernel/benchmark/bench_fp4_gemm.py | 241 + .../benchmark/bench_fp8_blockwise_gemm.py | 239 + .../bench_fp8_blockwise_group_gemm.py | 343 + .../sgl-kernel/benchmark/bench_fp8_gemm.py | 232 + .../sgl-kernel/benchmark/bench_int8_gemm.py | 185 + .../benchmark/bench_kimi_k2_moe_fused_gate.py | 117 + .../benchmark/bench_moe_align_block_size.py | 445 + .../benchmark/bench_moe_ep_post_reorder.py | 90 + .../benchmark/bench_moe_fused_gate.py | 89 + .../benchmark/bench_moe_topk_sigmoid.py | 171 + .../benchmark/bench_moe_topk_softmax.py | 163 + .../sgl-kernel/benchmark/bench_mrope.py | 250 + .../benchmark/bench_nvfp4_scaled_gemm.py | 192 + .../benchmark/bench_per_tensor_quant_fp8.py | 126 + .../bench_per_token_group_quant_8bit.py | 246 + .../benchmark/bench_per_token_quant_fp8.py | 230 + .../benchmark/bench_qserve_w4a8_gemm.py | 216 + .../sgl-kernel/benchmark/bench_rmsnorm.py | 397 + .../benchmark/bench_rotary_embedding.py | 111 + .../sgl-kernel/benchmark/bench_sum_scale.py | 251 + .../benchmark/bench_top_k_top_p_sampling.py | 147 + .../.vendor/sglang/sgl-kernel/build.sh | 240 + .../sglang/sgl-kernel/cmake/flashmla.cmake | 67 + .../sglang/sgl-kernel/cmake/utils.cmake | 19 + .../csrc/allreduce/custom_all_reduce.cu | 137 + .../csrc/allreduce/custom_all_reduce.cuh | 489 ++ .../csrc/allreduce/custom_all_reduce.hip | 180 + .../csrc/allreduce/custom_all_reduce_hip.cuh | 582 ++ .../csrc/allreduce/mscclpp_allreduce.cu | 140 + .../csrc/allreduce/mscclpp_allreduce.cuh | 779 ++ .../csrc/allreduce/quick_all_reduce.cu | 111 + .../csrc/allreduce/quick_all_reduce.cuh | 634 ++ .../csrc/allreduce/quick_all_reduce.h | 237 + .../csrc/allreduce/quick_all_reduce_base.h | 318 + .../csrc/allreduce/test_mscclpp_allreduce.cu | 153 + .../sgl-kernel/csrc/attention/cascade.cu | 55 + .../csrc/attention/cutlass_mla_kernel.cu | 274 + .../cutlass_sm100_mla/device/sm100_mla.hpp | 358 + .../kernel/sm100_fmha_mla_reduction.hpp | 198 + .../sm100_fmha_mla_tma_warpspecialized.hpp | 2018 +++++ .../kernel/sm100_mla_tile_scheduler.hpp | 160 + .../csrc/attention/merge_attn_states.cu | 204 + .../csrc/attention/vertical_slash_index.cu | 462 ++ .../sgl-kernel/csrc/common_extension.cc | 599 ++ .../sgl-kernel/csrc/common_extension_rocm.cc | 191 + .../sglang/sgl-kernel/csrc/cpu/CMakeLists.txt | 99 + .../sglang/sgl-kernel/csrc/cpu/activation.cpp | 135 + .../sglang/sgl-kernel/csrc/cpu/bmm.cpp | 123 + .../sglang/sgl-kernel/csrc/cpu/common.h | 324 + .../sglang/sgl-kernel/csrc/cpu/decode.cpp | 1744 ++++ .../sglang/sgl-kernel/csrc/cpu/extend.cpp | 723 ++ .../sglang/sgl-kernel/csrc/cpu/gemm.cpp | 525 ++ .../.vendor/sglang/sgl-kernel/csrc/cpu/gemm.h | 202 + .../sglang/sgl-kernel/csrc/cpu/gemm_fp8.cpp | 551 ++ .../sglang/sgl-kernel/csrc/cpu/gemm_int8.cpp | 547 ++ .../sglang/sgl-kernel/csrc/cpu/interface.cpp | 74 + .../sglang/sgl-kernel/csrc/cpu/moe.cpp | 1322 +++ .../sglang/sgl-kernel/csrc/cpu/moe_fp8.cpp | 491 ++ .../sglang/sgl-kernel/csrc/cpu/moe_int8.cpp | 1068 +++ .../sglang/sgl-kernel/csrc/cpu/norm.cpp | 417 + .../sglang/sgl-kernel/csrc/cpu/numa_utils.cpp | 91 + .../sglang/sgl-kernel/csrc/cpu/qkv_proj.cpp | 701 ++ .../sglang/sgl-kernel/csrc/cpu/rope.cpp | 346 + .../sglang/sgl-kernel/csrc/cpu/shm.cpp | 666 ++ .../.vendor/sglang/sgl-kernel/csrc/cpu/shm.h | 11 + .../sglang/sgl-kernel/csrc/cpu/topk.cpp | 665 ++ .../csrc/cpu/torch_extension_cpu.cpp | 378 + .../.vendor/sglang/sgl-kernel/csrc/cpu/vec.h | 304 + .../csrc/cutlass_extensions/common.hpp | 21 + .../detail/collective/mixed_input_utils.hpp | 482 ++ .../epilogue/epilogue_per_row_per_col_scale.h | 309 + .../sm90_gmma_builder_mixed_input.inl | 278 + .../collective_builder_mixed_input.hpp | 52 + .../collective_mma_array_mixed_input.hpp | 53 + ...a_gmma_rs_warpspecialized_mixed_input_.hpp | 1538 ++++ .../gemm/cutlass_gemm_caller.cuh | 62 + .../gemm/dispatch_policy.hpp | 38 + .../gemm/fp8_blockwise_gemm_sm90_dispatch.cuh | 197 + .../gemm/gemm_universal_base_compat.h | 356 + .../gemm/gemm_with_epilogue_visitor.h | 492 ++ .../sgl-kernel/csrc/elementwise/activation.cu | 170 + .../sgl-kernel/csrc/elementwise/cast.cu | 170 + .../sgl-kernel/csrc/elementwise/concat_mla.cu | 217 + .../sgl-kernel/csrc/elementwise/copy.cu | 58 + .../elementwise/fused_add_rms_norm_kernel.cu | 59 + .../sgl-kernel/csrc/elementwise/pos_enc.cuh | 467 ++ .../sgl-kernel/csrc/elementwise/rope.cu | 168 + .../sgl-kernel/csrc/elementwise/topk.cu | 522 ++ .../sgl-kernel/csrc/elementwise/utils.cuh | 72 + .../expert_specialization/es_fp8_blockwise.cu | 167 + .../es_fp8_blockwise_functor.cuh | 268 + .../es_fp8_blockwise_launcher.cuh | 284 + .../es_fp8_blockwise_traits.cuh | 174 + .../sglang/sgl-kernel/csrc/flash_extension.cc | 66 + .../sgl-kernel/csrc/flashmla_extension.cc | 55 + .../sglang/sgl-kernel/csrc/gemm/awq_kernel.cu | 221 + .../sglang/sgl-kernel/csrc/gemm/bmm_fp8.cu | 75 + .../sgl-kernel/csrc/gemm/dsv3_fused_a_gemm.cu | 673 ++ .../csrc/gemm/dsv3_router_gemm_bf16_out.cu | 284 + .../csrc/gemm/dsv3_router_gemm_entry.cu | 161 + .../csrc/gemm/dsv3_router_gemm_float_out.cu | 283 + .../csrc/gemm/fp8_blockwise_gemm_kernel.cu | 465 ++ .../sgl-kernel/csrc/gemm/fp8_gemm_kernel.cu | 1532 ++++ .../sgl-kernel/csrc/gemm/gptq/compat.cuh | 62 + .../sgl-kernel/csrc/gemm/gptq/gptq_kernel.cu | 1950 +++++ .../sgl-kernel/csrc/gemm/gptq/matrix_view.cuh | 269 + .../sgl-kernel/csrc/gemm/gptq/qdq_2.cuh | 74 + .../sgl-kernel/csrc/gemm/gptq/qdq_3.cuh | 146 + .../sgl-kernel/csrc/gemm/gptq/qdq_4.cuh | 114 + .../sgl-kernel/csrc/gemm/gptq/qdq_8.cuh | 30 + .../sgl-kernel/csrc/gemm/gptq/qdq_util.cuh | 53 + .../sgl-kernel/csrc/gemm/int8_gemm_kernel.cu | 747 ++ .../csrc/gemm/marlin/awq_marlin_repack.cu | 253 + .../sgl-kernel/csrc/gemm/marlin/dequant.h | 504 ++ .../csrc/gemm/marlin/gptq_marlin.cu | 1120 +++ .../csrc/gemm/marlin/gptq_marlin_repack.cu | 329 + .../sgl-kernel/csrc/gemm/marlin/kernel.h | 36 + .../sgl-kernel/csrc/gemm/marlin/marlin.cuh | 96 + .../csrc/gemm/marlin/marlin_dtypes.cuh | 82 + .../csrc/gemm/marlin/marlin_template.h | 1629 ++++ .../sglang/sgl-kernel/csrc/gemm/math.hpp | 28 + .../csrc/gemm/nvfp4_expert_quant.cu | 728 ++ .../sgl-kernel/csrc/gemm/nvfp4_quant.cuh | 182 + .../sgl-kernel/csrc/gemm/nvfp4_quant_entry.cu | 77 + .../csrc/gemm/nvfp4_quant_kernels.cu | 242 + .../csrc/gemm/nvfp4_scaled_mm_entry.cu | 64 + .../csrc/gemm/nvfp4_scaled_mm_kernels.cu | 687 ++ .../csrc/gemm/per_tensor_quant_fp8.cu | 123 + .../csrc/gemm/per_token_group_quant_8bit.cu | 217 + .../gemm/per_token_group_quant_8bit_v2.cu | 514 ++ .../csrc/gemm/per_token_quant_fp8.cu | 242 + .../csrc/gemm/qserve_w4a8_per_chn_gemm.cu | 710 ++ .../csrc/gemm/qserve_w4a8_per_group_gemm.cu | 795 ++ .../apply_token_bitmask_inplace_cuda.cu | 271 + .../sgl-kernel/csrc/kvcacheio/transfer.cu | 807 ++ .../sgl-kernel/csrc/mamba/causal_conv1d.cu | 669 ++ .../sgl-kernel/csrc/mamba/causal_conv1d.h | 159 + .../sglang/sgl-kernel/csrc/memory/store.cu | 147 + .../csrc/memory/weak_ref_tensor.cpp | 35 + .../moe/cutlass_moe/w4a8/scaled_mm_entry.cu | 91 + .../w4a8/w4a8_get_group_starts.cuh | 158 + .../cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cu | 386 + .../cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cuh | 279 + .../moe/cutlass_moe/w4a8/w4a8_moe_data.cu | 79 + .../sgl-kernel/csrc/moe/cutlass_moe_helper.cu | 142 + .../csrc/moe/fp8_blockwise_moe_kernel.cu | 810 ++ .../csrc/moe/fused_qknorm_rope_kernel.cu | 408 + .../csrc/moe/kimi_k2_moe_fused_gate.cu | 311 + .../moe/marlin_moe_wna16/generate_kernels.py | 158 + .../csrc/moe/marlin_moe_wna16/kernel.h | 40 + .../moe/marlin_moe_wna16/kernel_bf16_ku4.cuh | 39 + .../marlin_moe_wna16/kernel_bf16_ku4b8.cuh | 47 + .../marlin_moe_wna16/kernel_bf16_ku8b128.cuh | 47 + .../moe/marlin_moe_wna16/kernel_fp16_ku4.cuh | 39 + .../marlin_moe_wna16/kernel_fp16_ku4b8.cuh | 47 + .../marlin_moe_wna16/kernel_fp16_ku8b128.cuh | 47 + .../moe/marlin_moe_wna16/kernel_marlin.cuh | 10 + .../moe/marlin_moe_wna16/marlin_template.h | 1899 +++++ .../csrc/moe/marlin_moe_wna16/ops.cu | 1237 +++ .../sgl-kernel/csrc/moe/moe_align_kernel.cu | 383 + .../sgl-kernel/csrc/moe/moe_fused_gate.cu | 523 ++ .../sglang/sgl-kernel/csrc/moe/moe_sum.cu | 66 + .../sgl-kernel/csrc/moe/moe_sum_reduce.cu | 403 + .../csrc/moe/moe_topk_sigmoid_kernels.cu | 592 ++ .../csrc/moe/moe_topk_softmax_kernels.cu | 704 ++ .../csrc/moe/nvfp4_blockwise_moe.cu | 698 ++ .../sgl-kernel/csrc/moe/prepare_moe_input.cu | 392 + .../csrc/quantization/gguf/dequantize.cuh | 583 ++ .../csrc/quantization/gguf/ggml-common.h | 1029 +++ .../csrc/quantization/gguf/gguf_kernel.cu | 836 ++ .../sgl-kernel/csrc/quantization/gguf/mmq.cuh | 881 ++ .../csrc/quantization/gguf/mmvq.cuh | 352 + .../sgl-kernel/csrc/quantization/gguf/moe.cuh | 1379 ++++ .../csrc/quantization/gguf/moe_vec.cuh | 413 + .../csrc/quantization/gguf/vecdotq.cuh | 2037 +++++ .../sgl-kernel/csrc/spatial/cuda_utils.h | 46 + .../csrc/spatial/greenctx_stream.cu | 98 + .../sgl-kernel/csrc/spatial/greenctx_stream.h | 2 + .../sgl-kernel/csrc/spatial_extension.cc | 29 + .../csrc/speculative/eagle_utils.cu | 407 + .../csrc/speculative/ngram_utils.cu | 105 + .../sgl-kernel/csrc/speculative/packbit.cu | 51 + .../csrc/speculative/speculative_sampling.cu | 151 + .../csrc/speculative/speculative_sampling.cuh | 231 + .../include/hip/hip_act_and_mul.cuh | 87 + .../sgl-kernel/include/hip/hip_math_def.h | 94 + .../sgl-kernel/include/hip/hip_vec_dtypes.h | 101 + .../include/hip/impl/hip_vec_bf16_impl.h | 177 + .../include/hip/impl/hip_vec_fp32_impl.h | 129 + .../include/hip/impl/hip_vec_half_impl.h | 172 + .../include/pytorch_extension_utils_rocm.h | 20 + .../sglang/sgl-kernel/include/scalar_type.hpp | 331 + .../sgl-kernel/include/sgl_flash_kernel_ops.h | 85 + .../sgl-kernel/include/sgl_kernel_ops.h | 981 +++ .../include/sgl_kernel_torch_shim.h | 122 + .../.vendor/sglang/sgl-kernel/include/utils.h | 469 ++ .../sglang/sgl-kernel/kernel-runner-setup.sh | 150 + .../.vendor/sglang/sgl-kernel/pyproject.toml | 42 + .../sglang/sgl-kernel/pyproject_cpu.toml | 35 + .../sglang/sgl-kernel/pyproject_rocm.toml | 32 + .../sgl-kernel/python/sgl_kernel/__init__.py | 142 + .../python/sgl_kernel/_fa4_interface.py | 562 ++ .../sgl-kernel/python/sgl_kernel/allreduce.py | 173 + .../sgl-kernel/python/sgl_kernel/attention.py | 132 + .../python/sgl_kernel/cutlass_moe.py | 112 + .../python/sgl_kernel/elementwise.py | 393 + .../sgl_kernel/expert_specialization.py | 29 + .../python/sgl_kernel/flash_attn.py | 369 + .../sgl-kernel/python/sgl_kernel/flash_mla.py | 155 + .../sgl-kernel/python/sgl_kernel/fused_moe.py | 61 + .../sgl-kernel/python/sgl_kernel/gemm.py | 564 ++ .../sgl-kernel/python/sgl_kernel/grammar.py | 15 + .../sgl-kernel/python/sgl_kernel/hadamard.py | 21 + .../sgl-kernel/python/sgl_kernel/kvcacheio.py | 307 + .../python/sgl_kernel/load_utils.py | 232 + .../sgl-kernel/python/sgl_kernel/mamba.py | 50 + .../sgl-kernel/python/sgl_kernel/marlin.py | 44 + .../sgl-kernel/python/sgl_kernel/memory.py | 26 + .../sgl-kernel/python/sgl_kernel/moe.py | 334 + .../sgl_kernel/quantization/__init__.py | 8 + .../python/sgl_kernel/quantization/gguf.py | 62 + .../sgl-kernel/python/sgl_kernel/sampling.py | 543 ++ .../python/sgl_kernel/scalar_type.py | 352 + .../python/sgl_kernel/sparse_flash_attn.py | 293 + .../sgl-kernel/python/sgl_kernel/spatial.py | 63 + .../python/sgl_kernel/speculative.py | 126 + .../python/sgl_kernel/test_utils.py | 125 + .../python/sgl_kernel/testing/__init__.py | 0 .../sgl_kernel/testing/rotary_embedding.py | 221 + .../sgl-kernel/python/sgl_kernel/top_k.py | 117 + .../sgl-kernel/python/sgl_kernel/utils.py | 45 + .../sgl-kernel/python/sgl_kernel/version.py | 1 + .../sglang/sgl-kernel/rename_wheels.sh | 34 + .../.vendor/sglang/sgl-kernel/setup_rocm.py | 117 + .../sglang/sgl-kernel/tests/conftest.py | 14 + .../tests/spatial/test_greenctx_stream.py | 25 + .../tests/speculative/test_eagle_utils.py | 87 + .../tests/speculative/test_ngram_utils.py | 76 + .../speculative/test_speculative_sampling.py | 129 + .../sgl-kernel/tests/test_activation.py | 39 + .../tests/test_apply_token_bitmask_inplace.py | 23 + .../sgl-kernel/tests/test_awq_dequant.py | 115 + .../sglang/sgl-kernel/tests/test_bmm_fp8.py | 43 + .../sgl-kernel/tests/test_causal_conv1d.py | 489 ++ .../sglang/sgl-kernel/tests/test_copy.py | 16 + .../sgl-kernel/tests/test_custom_allreduce.py | 185 + .../sgl-kernel/tests/test_cutlass_mla.py | 104 + .../tests/test_cutlass_w4a8_moe_mm.py | 283 + .../tests/test_dsv3_fused_a_gemm.py | 32 + .../sgl-kernel/tests/test_dsv3_router_gemm.py | 35 + .../tests/test_es_fp8_blockwise_moe.py | 205 + .../sgl-kernel/tests/test_flash_attention.py | 1368 ++++ .../tests/test_flash_attention_4.py | 1458 ++++ .../sglang/sgl-kernel/tests/test_flashmla.py | 662 ++ .../sglang/sgl-kernel/tests/test_fp4_gemm.py | 154 + .../sgl-kernel/tests/test_fp4_quantize.py | 260 + .../tests/test_fp8_blockwise_gemm.py | 93 + .../tests/test_fp8_blockwise_moe.py | 221 + .../sglang/sgl-kernel/tests/test_fp8_gemm.py | 49 + .../tests/test_fused_qk_norm_rope.py | 225 + .../sglang/sgl-kernel/tests/test_gguf.py | 166 + .../sgl-kernel/tests/test_gptq_kernel.py | 131 + .../sglang/sgl-kernel/tests/test_hadamard.py | 78 + .../sglang/sgl-kernel/tests/test_int8_gemm.py | 48 + .../tests/test_kimi_k2_moe_fused_gate.py | 124 + .../sglang/sgl-kernel/tests/test_kvcacheio.py | 694 ++ .../sgl-kernel/tests/test_marlin_gemm.py | 121 + .../sgl-kernel/tests/test_marlin_repack.py | 148 + .../sgl-kernel/tests/test_merge_state.py | 142 + .../sgl-kernel/tests/test_merge_state_v2.py | 400 + .../sglang/sgl-kernel/tests/test_moe_align.py | 274 + .../sgl-kernel/tests/test_moe_fused_gate.py | 103 + .../sgl-kernel/tests/test_moe_topk_sigmoid.py | 183 + .../sgl-kernel/tests/test_moe_topk_softmax.py | 139 + .../sglang/sgl-kernel/tests/test_mscclpp.py | 146 + .../sglang/sgl-kernel/tests/test_norm.py | 142 + .../tests/test_per_tensor_quant_fp8.py | 67 + .../tests/test_per_token_group_quant_8bit.py | 183 + .../tests/test_per_token_quant_fp8.py | 57 + .../tests/test_qserve_w4a8_per_chn_gemm.py | 118 + .../tests/test_qserve_w4a8_per_group_gemm.py | 183 + .../sgl-kernel/tests/test_rotary_embedding.py | 144 + .../sglang/sgl-kernel/tests/test_sampling.py | 185 + .../tests/test_sparse_flash_attn.py | 492 ++ .../sglang/sgl-kernel/tests/test_topk.py | 252 + .../tests/test_torch_defaults_reset.py | 16 + .../.vendor/sglang/sgl-kernel/tests/utils.py | 9 + .../sglang/sgl-router/.cargo/config.toml | 15 + .../.vendor/sglang/sgl-router/Cargo.toml | 166 + .../.vendor/sglang/sgl-router/LICENSE | 1 + .../.vendor/sglang/sgl-router/Makefile | 159 + .../.vendor/sglang/sgl-router/README.md | 675 ++ .../sgl-router/benches/request_processing.rs | 670 ++ .../sgl-router/benches/tokenizer_benchmark.rs | 1890 +++++ .../benches/tool_parser_benchmark.rs | 887 ++ .../sgl-router/bindings/golang/.gitignore | 24 + .../sgl-router/bindings/golang/Cargo.toml | 47 + .../sgl-router/bindings/golang/Makefile | 103 + .../sgl-router/bindings/golang/README.md | 552 ++ .../sgl-router/bindings/golang/client.go | 510 ++ .../sgl-router/bindings/golang/client_test.go | 325 + .../bindings/golang/examples/simple/main.go | 85 + .../bindings/golang/examples/simple/run.sh | 46 + .../golang/examples/streaming/main.go | 125 + .../bindings/golang/examples/streaming/run.sh | 46 + .../bindings/golang/integration_test.go | 228 + .../bindings/golang/internal/ffi/client.go | 228 + .../sgl-router/bindings/golang/src/client.rs | 279 + .../sgl-router/bindings/golang/src/error.rs | 50 + .../bindings/golang/src/grpc_converter.rs | 758 ++ .../sgl-router/bindings/golang/src/lib.rs | 88 + .../sgl-router/bindings/golang/src/memory.rs | 28 + .../sgl-router/bindings/golang/src/stream.rs | 288 + .../bindings/golang/src/tokenizer.rs | 379 + .../bindings/golang/src/tool_parser.rs | 329 + .../sgl-router/bindings/golang/src/utils.rs | 44 + .../sgl-router/bindings/python/.coveragerc | 9 + .../sgl-router/bindings/python/Cargo.toml | 27 + .../sgl-router/bindings/python/MANIFEST.in | 9 + .../sgl-router/bindings/python/README.md | 71 + .../sgl-router/bindings/python/pyproject.toml | 54 + .../sgl-router/bindings/python/setup.py | 28 + .../bindings/python/sglang_router/__init__.py | 3 + .../bindings/python/sglang_router/__main__.py | 8 + .../bindings/python/sglang_router/cli.py | 107 + .../python/sglang_router/launch_router.py | 113 + .../python/sglang_router/launch_server.py | 213 + .../bindings/python/sglang_router/mini_lb.py | 488 ++ .../bindings/python/sglang_router/router.py | 226 + .../python/sglang_router/router_args.py | 778 ++ .../bindings/python/sglang_router/version.py | 1 + .../sgl-router/bindings/python/src/lib.rs | 730 ++ .../.vendor/sglang/sgl-router/build.rs | 124 + .../sglang/sgl-router/py_test/__init__.py | 1 + .../sglang/sgl-router/py_test/conftest.py | 15 + .../e2e_grpc/basic/test_openai_server.py | 343 + .../sgl-router/py_test/e2e_grpc/conftest.py | 29 + .../e2e_grpc/features/test_enable_thinking.py | 194 + .../features/test_reasoning_content.py | 194 + .../sgl-router/py_test/e2e_grpc/fixtures.py | 342 + .../test_openai_function_calling.py | 950 +++ .../function_call/test_tool_choice.py | 795 ++ .../sgl-router/py_test/e2e_grpc/pytest.ini | 13 + .../sgl-router/py_test/e2e_grpc/util.py | 260 + .../validation/test_large_max_new_tokens.py | 115 + .../test_openai_server_ignore_eos.py | 109 + .../sgl-router/py_test/e2e_http/conftest.py | 807 ++ .../py_test/e2e_http/test_e2e_embeddings.py | 62 + .../py_test/e2e_http/test_pd_router.py | 264 + .../py_test/e2e_http/test_regular_router.py | 228 + .../py_test/e2e_response_api/conftest.py | 132 + .../features/test_basic_crud.py | 241 + .../features/test_state_management.py | 161 + .../features/test_streaming_events.py | 152 + .../features/test_structured_output.py | 172 + .../features/test_tools_call.py | 765 ++ .../e2e_response_api/router_fixtures.py | 565 ++ .../py_test/e2e_response_api/util.py | 81 + .../sgl-router/py_test/fixtures/__init__.py | 1 + .../py_test/fixtures/generate_test_certs.py | 236 + .../py_test/fixtures/mock_worker.py | 285 + .../sgl-router/py_test/fixtures/ports.py | 8 + .../py_test/fixtures/router_manager.py | 238 + .../py_test/integration_mock/__init__.py | 1 + .../py_test/integration_mock/conftest.py | 128 + .../load_balancing/__init__.py | 1 + .../load_balancing/test_cache_aware.py | 73 + .../load_balancing/test_power_of_two.py | 99 + .../load_balancing/test_random.py | 32 + .../load_balancing/test_round_robin.py | 33 + .../py_test/integration_mock/test_api_auth.py | 38 + .../integration_mock/test_circuit_breaker.py | 228 + .../integration_mock/test_fault_tolerance.py | 32 + .../py_test/integration_mock/test_mtls.py | 332 + .../integration_mock/test_payload_size.py | 33 + .../integration_mock/test_pd_routing.py | 126 + .../integration_mock/test_rate_limiting.py | 90 + .../py_test/integration_mock/test_retries.py | 71 + .../test_service_discovery_shim.py | 36 + .../test_worker_management.py | 57 + .../sgl-router/py_test/unit/__init__.py | 7 + .../py_test/unit/test_arg_parser.py | 626 ++ .../py_test/unit/test_router_config.py | 423 + .../py_test/unit/test_startup_sequence.py | 1056 +++ .../py_test/unit/test_validation.py | 509 ++ .../.vendor/sglang/sgl-router/pytest.ini | 5 + .../.vendor/sglang/sgl-router/rustfmt.toml | 8 + .../scripts/generate_gateway_release_notes.sh | 312 + .../sgl-router/scripts/run_benchmarks.py | 250 + .../sgl-router/scripts/setup-sccache.sh | 84 + .../sglang/sgl-router/src/app_context.rs | 514 ++ .../sglang/sgl-router/src/config/builder.rs | 740 ++ .../sglang/sgl-router/src/config/mod.rs | 28 + .../sglang/sgl-router/src/config/types.rs | 1247 +++ .../sgl-router/src/config/validation.rs | 969 +++ .../sgl-router/src/core/circuit_breaker.rs | 523 ++ .../sglang/sgl-router/src/core/error.rs | 236 + .../sglang/sgl-router/src/core/job_queue.rs | 734 ++ .../sgl-router/src/core/metrics_aggregator.rs | 91 + .../.vendor/sglang/sgl-router/src/core/mod.rs | 40 + .../sglang/sgl-router/src/core/model_card.rs | 323 + .../sglang/sgl-router/src/core/model_type.rs | 332 + .../sglang/sgl-router/src/core/retry.rs | 401 + .../sgl-router/src/core/token_bucket.rs | 243 + .../sglang/sgl-router/src/core/worker.rs | 1945 +++++ .../sgl-router/src/core/worker_builder.rs | 534 ++ .../sgl-router/src/core/worker_manager.rs | 456 ++ .../sgl-router/src/core/worker_registry.rs | 546 ++ .../src/core/workflow/definition.rs | 98 + .../sgl-router/src/core/workflow/engine.rs | 484 ++ .../sgl-router/src/core/workflow/event.rs | 188 + .../sgl-router/src/core/workflow/executor.rs | 129 + .../sgl-router/src/core/workflow/mod.rs | 21 + .../sgl-router/src/core/workflow/state.rs | 175 + .../steps/external_worker_registration.rs | 586 ++ .../core/workflow/steps/mcp_registration.rs | 304 + .../sgl-router/src/core/workflow/steps/mod.rs | 32 + .../workflow/steps/worker_registration.rs | 926 +++ .../src/core/workflow/steps/worker_removal.rs | 310 + .../sgl-router/src/core/workflow/types.rs | 271 + .../sgl-router/src/data_connector/common.rs | 78 + .../sgl-router/src/data_connector/core.rs | 483 ++ .../sgl-router/src/data_connector/factory.rs | 140 + .../sgl-router/src/data_connector/memory.rs | 730 ++ .../sgl-router/src/data_connector/mod.rs | 25 + .../sgl-router/src/data_connector/noop.rs | 189 + .../sgl-router/src/data_connector/oracle.rs | 1130 +++ .../sgl-router/src/data_connector/postgres.rs | 699 ++ .../sglang/sgl-router/src/grpc_client/mod.rs | 7 + .../src/grpc_client/sglang_scheduler.rs | 786 ++ .../sgl-router/src/grpc_client/vllm_engine.rs | 736 ++ .../.vendor/sglang/sgl-router/src/lib.rs | 19 + .../.vendor/sglang/sgl-router/src/logging.rs | 132 + .../.vendor/sglang/sgl-router/src/main.rs | 769 ++ .../sglang/sgl-router/src/mcp/config.rs | 528 ++ .../sgl-router/src/mcp/connection_pool.rs | 318 + .../sglang/sgl-router/src/mcp/error.rs | 50 + .../sglang/sgl-router/src/mcp/inventory.rs | 447 + .../sglang/sgl-router/src/mcp/manager.rs | 782 ++ .../.vendor/sglang/sgl-router/src/mcp/mod.rs | 25 + .../sglang/sgl-router/src/mcp/oauth.rs | 194 + .../sglang/sgl-router/src/mcp/proxy.rs | 251 + .../sglang/sgl-router/src/mcp/tool_args.rs | 103 + .../.vendor/sglang/sgl-router/src/metrics.rs | 1046 +++ .../sglang/sgl-router/src/middleware.rs | 556 ++ .../sglang/sgl-router/src/policies/bucket.rs | 1167 +++ .../sgl-router/src/policies/cache_aware.rs | 533 ++ .../sglang/sgl-router/src/policies/factory.rs | 113 + .../sglang/sgl-router/src/policies/mod.rs | 178 + .../sgl-router/src/policies/power_of_two.rs | 213 + .../sglang/sgl-router/src/policies/random.rs | 129 + .../sgl-router/src/policies/registry.rs | 483 ++ .../sgl-router/src/policies/round_robin.rs | 151 + .../sglang/sgl-router/src/policies/tree.rs | 1450 ++++ .../src/proto/sglang_scheduler.proto | 459 ++ .../sgl-router/src/proto/vllm_engine.proto | 218 + .../src/protocols/builders/chat/mod.rs | 7 + .../src/protocols/builders/chat/response.rs | 218 + .../builders/chat/stream_response.rs | 421 + .../sgl-router/src/protocols/builders/mod.rs | 27 + .../src/protocols/builders/responses/mod.rs | 5 + .../protocols/builders/responses/response.rs | 411 + .../sglang/sgl-router/src/protocols/chat.rs | 712 ++ .../sgl-router/src/protocols/classify.rs | 57 + .../sglang/sgl-router/src/protocols/common.rs | 457 ++ .../sgl-router/src/protocols/completion.rs | 214 + .../sgl-router/src/protocols/embedding.rs | 57 + .../sgl-router/src/protocols/generate.rs | 297 + .../sglang/sgl-router/src/protocols/mod.rs | 15 + .../sglang/sgl-router/src/protocols/rerank.rs | 212 + .../sgl-router/src/protocols/responses.rs | 1298 +++ .../src/protocols/sampling_params.rs | 119 + .../sgl-router/src/protocols/validated.rs | 164 + .../sgl-router/src/protocols/worker_spec.rs | 315 + .../sgl-router/src/reasoning_parser/README.md | 474 ++ .../src/reasoning_parser/factory.rs | 594 ++ .../sgl-router/src/reasoning_parser/mod.rs | 10 + .../src/reasoning_parser/parsers/base.rs | 362 + .../reasoning_parser/parsers/deepseek_r1.rs | 118 + .../src/reasoning_parser/parsers/glm45.rs | 124 + .../src/reasoning_parser/parsers/kimi.rs | 142 + .../src/reasoning_parser/parsers/minimax.rs | 166 + .../src/reasoning_parser/parsers/mod.rs | 15 + .../src/reasoning_parser/parsers/qwen3.rs | 188 + .../src/reasoning_parser/parsers/step3.rs | 129 + .../sgl-router/src/reasoning_parser/traits.rs | 135 + .../sglang/sgl-router/src/routers/factory.rs | 134 + .../sgl-router/src/routers/grpc/client.rs | 177 + .../sgl-router/src/routers/grpc/common/mod.rs | 6 + .../grpc/common/response_collection.rs | 85 + .../grpc/common/response_formatting.rs | 29 + .../routers/grpc/common/responses/handlers.rs | 196 + .../src/routers/grpc/common/responses/mod.rs | 9 + .../grpc/common/responses/streaming.rs | 836 ++ .../routers/grpc/common/responses/utils.rs | 166 + .../grpc/common/stages/client_acquisition.rs | 62 + .../grpc/common/stages/dispatch_metadata.rs | 73 + .../src/routers/grpc/common/stages/helpers.rs | 43 + .../src/routers/grpc/common/stages/mod.rs | 39 + .../grpc/common/stages/request_execution.rs | 154 + .../grpc/common/stages/worker_selection.rs | 197 + .../sgl-router/src/routers/grpc/context.rs | 426 + .../sgl-router/src/routers/grpc/error.rs | 184 + .../src/routers/grpc/harmony/builder.rs | 887 ++ .../src/routers/grpc/harmony/detector.rs | 17 + .../src/routers/grpc/harmony/mod.rs | 56 + .../src/routers/grpc/harmony/parser.rs | 535 ++ .../src/routers/grpc/harmony/processor.rs | 323 + .../src/routers/grpc/harmony/responses.rs | 1610 ++++ .../src/routers/grpc/harmony/stages/mod.rs | 14 + .../grpc/harmony/stages/preparation.rs | 414 + .../grpc/harmony/stages/request_building.rs | 156 + .../harmony/stages/response_processing.rs | 155 + .../src/routers/grpc/harmony/streaming.rs | 1151 +++ .../src/routers/grpc/harmony/types.rs | 139 + .../sglang/sgl-router/src/routers/grpc/mod.rs | 23 + .../sgl-router/src/routers/grpc/pd_router.rs | 260 + .../sgl-router/src/routers/grpc/pipeline.rs | 470 ++ .../src/routers/grpc/proto_wrapper.rs | 381 + .../src/routers/grpc/regular/mod.rs | 9 + .../src/routers/grpc/regular/processor.rs | 463 ++ .../routers/grpc/regular/responses/context.rs | 70 + .../grpc/regular/responses/conversions.rs | 427 + .../grpc/regular/responses/handlers.rs | 814 ++ .../src/routers/grpc/regular/responses/mod.rs | 20 + .../grpc/regular/responses/tool_loop.rs | 1203 +++ .../routers/grpc/regular/responses/types.rs | 18 + .../routers/grpc/regular/stages/chat/mod.rs | 12 + .../grpc/regular/stages/chat/preparation.rs | 112 + .../regular/stages/chat/request_building.rs | 113 + .../stages/chat/response_processing.rs | 120 + .../grpc/regular/stages/generate/mod.rs | 12 + .../regular/stages/generate/preparation.rs | 121 + .../stages/generate/request_building.rs | 108 + .../stages/generate/response_processing.rs | 118 + .../src/routers/grpc/regular/stages/mod.rs | 17 + .../grpc/regular/stages/preparation.rs | 59 + .../grpc/regular/stages/request_building.rs | 50 + .../regular/stages/response_processing.rs | 59 + .../src/routers/grpc/regular/streaming.rs | 1260 +++ .../sgl-router/src/routers/grpc/router.rs | 359 + .../sgl-router/src/routers/grpc/utils.rs | 1147 +++ .../sgl-router/src/routers/header_utils.rs | 94 + .../sglang/sgl-router/src/routers/http/mod.rs | 5 + .../sgl-router/src/routers/http/pd_router.rs | 1406 ++++ .../sgl-router/src/routers/http/pd_types.rs | 78 + .../sgl-router/src/routers/http/router.rs | 880 ++ .../sglang/sgl-router/src/routers/mod.rs | 257 + .../src/routers/openai/conversations.rs | 1338 +++ .../sgl-router/src/routers/openai/mcp.rs | 979 +++ .../sgl-router/src/routers/openai/mod.rs | 18 + .../src/routers/openai/responses.rs | 317 + .../sgl-router/src/routers/openai/router.rs | 1261 +++ .../src/routers/openai/streaming.rs | 1550 ++++ .../sgl-router/src/routers/openai/utils.rs | 226 + .../sgl-router/src/routers/router_manager.rs | 752 ++ .../.vendor/sglang/sgl-router/src/server.rs | 974 +++ .../sgl-router/src/service_discovery.rs | 1147 +++ .../sglang/sgl-router/src/tokenizer/README.md | 197 + .../src/tokenizer/cache/fingerprint.rs | 106 + .../sgl-router/src/tokenizer/cache/l0.rs | 220 + .../sgl-router/src/tokenizer/cache/l1.rs | 507 ++ .../sgl-router/src/tokenizer/cache/mod.rs | 362 + .../sgl-router/src/tokenizer/chat_template.rs | 534 ++ .../sgl-router/src/tokenizer/factory.rs | 457 ++ .../sglang/sgl-router/src/tokenizer/hub.rs | 333 + .../sgl-router/src/tokenizer/huggingface.rs | 274 + .../sglang/sgl-router/src/tokenizer/mock.rs | 125 + .../sglang/sgl-router/src/tokenizer/mod.rs | 122 + .../sgl-router/src/tokenizer/sequence.rs | 268 + .../sglang/sgl-router/src/tokenizer/stop.rs | 606 ++ .../sglang/sgl-router/src/tokenizer/stream.rs | 107 + .../sglang/sgl-router/src/tokenizer/tests.rs | 139 + .../sgl-router/src/tokenizer/tiktoken.rs | 279 + .../sglang/sgl-router/src/tokenizer/traits.rs | 89 + .../sgl-router/src/tool_parser/errors.rs | 32 + .../sgl-router/src/tool_parser/factory.rs | 384 + .../sglang/sgl-router/src/tool_parser/mod.rs | 27 + .../src/tool_parser/parsers/deepseek.rs | 328 + .../src/tool_parser/parsers/glm4_moe.rs | 326 + .../src/tool_parser/parsers/helpers.rs | 488 ++ .../src/tool_parser/parsers/json.rs | 305 + .../src/tool_parser/parsers/kimik2.rs | 348 + .../src/tool_parser/parsers/llama.rs | 245 + .../src/tool_parser/parsers/minimax_m2.rs | 549 ++ .../src/tool_parser/parsers/mistral.rs | 287 + .../sgl-router/src/tool_parser/parsers/mod.rs | 32 + .../src/tool_parser/parsers/passthrough.rs | 55 + .../src/tool_parser/parsers/pythonic.rs | 414 + .../src/tool_parser/parsers/qwen.rs | 258 + .../src/tool_parser/parsers/step3.rs | 576 ++ .../src/tool_parser/partial_json.rs | 553 ++ .../sgl-router/src/tool_parser/state.rs | 16 + .../sgl-router/src/tool_parser/tests.rs | 599 ++ .../sgl-router/src/tool_parser/traits.rs | 76 + .../sgl-router/src/tool_parser/types.rs | 88 + .../.vendor/sglang/sgl-router/src/version.rs | 57 + .../sgl-router/tests/api_endpoints_test.rs | 2034 +++++ .../tests/cache_aware_backward_compat_test.rs | 149 + .../tests/chat_template_format_detection.rs | 312 + .../tests/chat_template_integration.rs | 414 + .../sgl-router/tests/chat_template_loading.rs | 230 + .../tests/common/mock_mcp_server.rs | 174 + .../tests/common/mock_openai_server.rs | 240 + .../sgl-router/tests/common/mock_worker.rs | 1271 +++ .../sglang/sgl-router/tests/common/mod.rs | 617 ++ .../tests/common/streaming_helpers.rs | 134 + .../sgl-router/tests/common/test_app.rs | 211 + .../sglang/sgl-router/tests/mcp_test.rs | 543 ++ .../tests/metrics_aggregator_test.rs | 271 + .../tests/policy_registry_integration.rs | 170 + .../sgl-router/tests/request_formats_test.rs | 358 + .../sgl-router/tests/responses_api_test.rs | 1605 ++++ .../sgl-router/tests/spec/chat_completion.rs | 570 ++ .../sgl-router/tests/spec/chat_message.rs | 91 + .../sglang/sgl-router/tests/spec/embedding.rs | 96 + .../sglang/sgl-router/tests/spec/mod.rs | 9 + .../sglang/sgl-router/tests/spec/rerank.rs | 569 ++ .../sglang/sgl-router/tests/spec/responses.rs | 1200 +++ .../sglang/sgl-router/tests/spec_test.rs | 2 + .../sgl-router/tests/streaming_tests.rs | 347 + .../sgl-router/tests/test_openai_routing.rs | 915 +++ .../sgl-router/tests/test_pd_routing.rs | 926 +++ .../tests/tokenizer_cache_correctness_test.rs | 467 ++ .../sgl-router/tests/tokenizer_integration.rs | 560 ++ .../sgl-router/tests/tool_parser_deepseek.rs | 161 + .../tests/tool_parser_edge_cases.rs | 351 + .../sgl-router/tests/tool_parser_fallback.rs | 272 + .../sgl-router/tests/tool_parser_glm4_moe.rs | 169 + .../sgl-router/tests/tool_parser_json.rs | 717 ++ .../sgl-router/tests/tool_parser_kimik2.rs | 157 + .../sgl-router/tests/tool_parser_llama.rs | 455 ++ .../tests/tool_parser_minimax_m2.rs | 780 ++ .../sgl-router/tests/tool_parser_mistral.rs | 274 + .../tests/tool_parser_mixed_edge_cases.rs | 291 + .../tests/tool_parser_partial_json.rs | 156 + .../sgl-router/tests/tool_parser_pythonic.rs | 518 ++ .../sgl-router/tests/tool_parser_qwen.rs | 307 + .../sgl-router/tests/tool_parser_step3.rs | 238 + .../sglang/sgl-router/tests/workflow_test.rs | 320 + .../.vendor/sglang/test/README.md | 60 + .../.vendor/sglang/test/lora_utils.py | 559 ++ .../ascend/test_ascend_w8a8_quantization.py | 104 + .../manual/ascend/test_mindspore_models.py | 64 + .../sglang/test/manual/cpu/test_comm.py | 116 + .../manual/debug_utils/test_log_parser.py | 28 + .../http_server/test_abort_request.py | 205 + .../test/manual/ep/test_deepep_internode.py | 445 + .../test/manual/ep/test_deepep_intranode.py | 378 + .../test/manual/ep/test_deepep_low_latency.py | 325 + .../sglang/test/manual/ep/test_eplb.py | 154 + .../manual/ep/test_hybrid_dp_ep_tp_mtp.py | 2728 +++++++ .../sglang/test/manual/ep/test_moe_deepep.py | 116 + .../ep/test_moe_deepep_eval_accuracy_large.py | 75 + .../test/manual/ep/test_mooncake_ep_small.py | 143 + .../hicache/test_disaggregation_hicache.py | 262 + .../manual/lang_frontend/test_bind_cache.py | 51 + .../test/manual/lang_frontend/test_choices.py | 91 + .../manual/lang_frontend/test_jump_forward.py | 138 + .../lang_frontend/test_openai_backend.py | 92 + .../lang_frontend/test_separate_reasoning.py | 68 + .../test_separate_reasoning_execution.py | 193 + .../attention/nsa/test_act_quant_triton.py | 281 + .../manual/layers/moe/test_moe_runners.py | 191 + .../manual/lora/test_chunked_sgmv_backend.py | 761 ++ .../test/manual/lora/test_lora_cuda_graph.py | 108 + .../test/manual/lora/test_lora_llama4.py | 63 + .../test/manual/lora/test_lora_qwen3_vl.py | 229 + .../manual/lora/test_lora_spec_decoding.py | 71 + .../test/manual/models/test_clip_models.py | 78 + .../manual/models/test_falcon_h1_models.py | 146 + .../manual/models/test_gme_qwen_models.py | 85 + .../test/manual/models/test_grok_models.py | 53 + .../test/manual/models/test_llama4_models.py | 73 + .../test/manual/models/test_mtp_models.py | 58 + .../test/manual/models/test_unsloth_models.py | 213 + .../manual/nightly/test_deepseek_v31_perf.py | 87 + .../manual/nightly/test_deepseek_v32_perf.py | 128 + .../nightly/test_text_models_gsm8k_eval.py | 124 + .../manual/nightly/test_text_models_perf.py | 60 + .../manual/nightly/test_vlms_mmmu_eval.py | 127 + .../test/manual/nightly/test_vlms_perf.py | 88 + .../nightly/test_vlms_piecewise_cuda_graph.py | 266 + .../features/test_cache_report.py | 289 + .../features/test_continuous_usage_stats.py | 105 + .../features/test_structural_tag.py | 126 + ...est_disaggregation_piecewise_cuda_graph.py | 86 + .../test/manual/quant/test_fp8_kvcache.py | 114 + .../test_async_dynamic_batch_tokenizer.py | 295 + .../manual/test_async_mm_data_processor.py | 364 + .../test/manual/test_config_integration.py | 158 + .../test/manual/test_custom_allreduce.py | 182 + .../manual/test_deepseek_chat_templates.py | 318 + .../test_deepseek_v32_cp_single_node.py | 99 + .../test/manual/test_double_sparsity.py | 65 + .../test_eagle_infer_beta_dp_attention.py | 97 + .../test/manual/test_expert_distribution.py | 101 + .../manual/test_expert_location_updater.py | 255 + .../sglang/test/manual/test_fim_completion.py | 72 + .../test/manual/test_forward_split_prefill.py | 305 + .../test/manual/test_get_weights_by_name.py | 183 + .../sglang/test/manual/test_health_check.py | 28 + .../sglang/test/manual/test_kv_events.py | 289 + .../sglang/test/manual/test_logprobs.py | 526 ++ .../.vendor/sglang/test/manual/test_mla_tp.py | 66 + .../sglang/test/manual/test_modelopt.py | 58 + .../test/manual/test_modelopt_fp8kvcache.py | 30 + .../manual/test_models_from_modelscope.py | 40 + .../sglang/test/manual/test_mscclpp.py | 196 + .../test/manual/test_quick_allreduce.py | 303 + .../test/manual/test_sagemaker_server.py | 183 + .../test/manual/test_schedule_policy.py | 202 + .../test/manual/test_session_control.py | 786 ++ .../manual/test_srt_engine_with_quant_args.py | 60 + .../manual/test_tokenizer_batch_encode.py | 121 + .../test/manual/test_tokenizer_manager.py | 386 + .../test_torch_flex_attention_backend.py | 49 + .../sglang/test/manual/test_torch_tp.py | 30 + .../sglang/test/manual/test_tracing.py | 272 + .../manual/test_triton_attention_rocm_mla.py | 259 + .../test/manual/test_triton_moe_wna16.py | 246 + .../test/manual/test_two_batch_overlap.py | 152 + .../test/manual/test_vertex_endpoint.py | 64 + .../sglang/test/manual/test_vlm_accuracy.py | 321 + .../manual/test_wave_attention_backend.py | 61 + .../sglang/test/manual/test_weight_version.py | 227 + .../sglang/test/nightly/nightly_utils.py | 306 + .../test/nightly/test_batch_invariant_ops.py | 261 + .../test/nightly/test_cpp_radix_cache.py | 50 + .../test_deepseek_r1_fp8_trtllm_backend.py | 91 + .../test/nightly/test_deepseek_v31_perf.py | 87 + .../nightly/test_deepseek_v32_nsabackend.py | 282 + .../test/nightly/test_deepseek_v32_perf.py | 128 + .../nightly/test_deepseek_v3_deterministic.py | 57 + .../test_deepseek_v3_fp4_cutlass_moe.py | 75 + .../sglang/test/nightly/test_encoder_dp.py | 275 + ...test_flashinfer_trtllm_gen_attn_backend.py | 65 + .../test_flashinfer_trtllm_gen_moe_backend.py | 68 + .../sglang/test/nightly/test_fp4_moe.py | 458 ++ .../sglang/test/nightly/test_glm_4_6_perf.py | 49 + .../test/nightly/test_gpt_oss_4gpu_perf.py | 61 + .../nightly/test_kimi_k2_thinking_perf.py | 54 + .../test/nightly/test_lora_eviction_policy.py | 193 + .../test/nightly/test_lora_openai_api.py | 330 + .../nightly/test_lora_openai_compatible.py | 282 + .../sglang/test/nightly/test_lora_qwen3.py | 65 + .../test/nightly/test_lora_radix_cache.py | 89 + .../test/nightly/test_minimax_m2_perf.py | 54 + .../sglang/test/nightly/test_nsa_indexer.py | 619 ++ .../test/nightly/test_qwen3_235b_perf.py | 49 + .../nightly/test_qwen3_fp4_trtllm_gen_moe.py | 61 + .../nightly/test_qwen3_next_deterministic.py | 47 + .../nightly/test_text_models_gsm8k_eval.py | 124 + .../test/nightly/test_text_models_perf.py | 60 + .../test/nightly/test_vlms_mmmu_eval.py | 127 + .../sglang/test/nightly/test_vlms_perf.py | 88 + .../.vendor/sglang/test/pytest.ini | 2 + .../test_function_call_parser.py | 2363 ++++++ .../test_json_schema_constraint.py | 567 ++ .../function_call/test_unknown_tool_name.py | 81 + .../stress/test_stress_deepseek_v3.py | 51 + .../registered/stress/test_stress_glm_4_6.py | 51 + .../registered/stress/test_stress_kimi_k2.py | 59 + .../stress/test_stress_qwen3_235b.py | 51 + .../test/registered/test_srt_backend.py | 84 + .../.vendor/sglang/test/run_suite.py | 185 + .../.vendor/sglang/test/run_suite_nightly.py | 94 + .../test/srt/ascend/test_ascend_deepep.py | 118 + .../srt/ascend/test_ascend_deepseek_mtp.py | 99 + .../srt/ascend/test_ascend_graph_tp1_bf16.py | 95 + .../srt/ascend/test_ascend_graph_tp2_bf16.py | 97 + .../srt/ascend/test_ascend_hicache_mha.py | 98 + .../ascend/test_ascend_mla_fia_w8a8int8.py | 103 + .../srt/ascend/test_ascend_mla_w8a8int8.py | 101 + .../ascend/test_ascend_sampling_backend.py | 96 + .../test/srt/ascend/test_ascend_tp1_bf16.py | 74 + .../test/srt/ascend/test_ascend_tp2_bf16.py | 98 + .../srt/ascend/test_ascend_tp2_fia_bf16.py | 101 + .../test/srt/ascend/test_ascend_tp4_bf16.py | 100 + .../sglang/test/srt/configs/deepseek_v3.yaml | 28 + .../srt/configs/deepseek_v3_long_context.yaml | 28 + .../sglang/test/srt/configs/llama_405b.yaml | 28 + .../test/srt/configs/random_config.yaml | 25 + .../random_flashinfer_vs_triton_config.yaml | 25 + .../test/srt/configs/sharegpt_config.yaml | 7 + .../sglang/test/srt/cpu/test_activation.py | 56 + .../sglang/test/srt/cpu/test_binding.py | 27 + .../sglang/test/srt/cpu/test_cpu_graph.py | 86 + .../sglang/test/srt/cpu/test_decode.py | 169 + .../sglang/test/srt/cpu/test_extend.py | 189 + .../.vendor/sglang/test/srt/cpu/test_gemm.py | 188 + .../cpu/test_intel_amx_attention_backend_a.py | 73 + .../cpu/test_intel_amx_attention_backend_b.py | 35 + .../cpu/test_intel_amx_attention_backend_c.py | 53 + .../.vendor/sglang/test/srt/cpu/test_mla.py | 155 + .../.vendor/sglang/test/srt/cpu/test_moe.py | 264 + .../.vendor/sglang/test/srt/cpu/test_norm.py | 135 + .../test/srt/cpu/test_qkv_proj_with_rope.py | 431 + .../.vendor/sglang/test/srt/cpu/test_rope.py | 180 + .../sglang/test/srt/cpu/test_shared_expert.py | 224 + .../.vendor/sglang/test/srt/cpu/test_topk.py | 196 + .../.vendor/sglang/test/srt/cpu/utils.py | 274 + .../test_tensor_dump_forward_hook.py | 95 + ...sparsity-config-Llama-3.1-8B-Instruct.json | 1 + .../sglang/test/srt/ep/test_deepep_large.py | 159 + .../sglang/test/srt/ep/test_deepep_small.py | 388 + .../.vendor/sglang/test/srt/ep/test_moe_ep.py | 90 + .../sglang/test/srt/experiment_runner.py | 364 + .../srt/external_models/custom_qwen2_vl.py | 21 + .../test/srt/hicache/test_hicache_storage.py | 59 + .../test_hicache_storage_3fs_backend.py | 88 + .../test_hicache_storage_file_backend.py | 333 + .../test_hicache_storage_mooncake_backend.py | 283 + .../test/srt/hicache/test_hicache_variants.py | 179 + .../test/srt/kv_cache_scales_llama3_1_8b.json | 42 + .../test/srt/kv_cache_scales_llama3_8b.json | 42 + .../test/srt/kv_cache_scales_qwen2_1_5b.json | 38 + .../attention/mamba/test_causal_conv1d.py | 379 + .../attention/mamba/test_mamba2_mixer.py | 143 + .../layers/attention/mamba/test_mamba_ssm.py | 296 + .../attention/mamba/test_mamba_ssm_ssd.py | 606 ++ .../.vendor/sglang/test/srt/lora/test_lora.py | 57 + .../sglang/test/srt/lora/test_lora_backend.py | 82 + .../test/srt/lora/test_lora_eviction.py | 144 + .../sglang/test/srt/lora/test_lora_tp.py | 83 + .../sglang/test/srt/lora/test_lora_update.py | 1569 ++++ .../test/srt/lora/test_multi_lora_backend.py | 72 + .../.vendor/sglang/test/srt/models/compare.py | 52 + .../models/test_compressed_tensors_models.py | 46 + .../srt/models/test_cross_encoder_models.py | 90 + .../test/srt/models/test_dummy_grok_models.py | 34 + .../test/srt/models/test_embedding_models.py | 145 + .../models/test_encoder_embedding_models.py | 162 + .../test/srt/models/test_generation_models.py | 192 + .../test/srt/models/test_glm4_moe_models.py | 49 + .../test/srt/models/test_kimi_k2_models.py | 68 + .../srt/models/test_kimi_linear_models.py | 46 + .../models/test_nvidia_nemotron_nano_v2.py | 28 + .../models/test_nvidia_nemotron_nano_v2_vl.py | 31 + .../test/srt/models/test_qwen3_next_models.py | 177 + .../test/srt/models/test_qwen_models.py | 77 + .../test/srt/models/test_reward_models.py | 94 + .../srt/models/test_transformers_models.py | 181 + .../sglang/test/srt/models/test_vlm_models.py | 52 + .../test/srt/nightly/test_gsm8k_eval_amd.py | 193 + .../sglang/test/srt/openai_server/__init__.py | 0 .../test/srt/openai_server/basic/__init__.py | 0 .../basic/test_openai_embedding.py | 197 + .../openai_server/basic/test_openai_server.py | 1105 +++ .../srt/openai_server/basic/test_protocol.py | 350 + .../openai_server/basic/test_serving_chat.py | 630 ++ .../basic/test_serving_completions.py | 157 + .../basic/test_serving_embedding.py | 144 + .../srt/openai_server/features/__init__.py | 0 .../features/test_enable_thinking.py | 237 + .../openai_server/features/test_json_mode.py | 131 + .../features/test_openai_server_ebnf.py | 98 + .../test_openai_server_hidden_states.py | 350 + .../features/test_reasoning_content.py | 341 + .../openai_server/function_call/__init__.py | 0 .../test_openai_function_calling.py | 952 +++ .../function_call/test_tool_choice.py | 855 ++ .../srt/openai_server/validation/__init__.py | 0 .../validation/test_large_max_new_tokens.py | 103 + .../validation/test_matched_stop.py | 80 + .../test_openai_server_ignore_eos.py | 84 + .../test_request_length_validation.py | 88 + .../test/srt/ops/test_repeat_interleave.py | 141 + .../.vendor/sglang/test/srt/parse_results.py | 57 + .../sglang/test/srt/quant/test_autoround.py | 62 + .../.vendor/sglang/test/srt/quant/test_awq.py | 78 + .../sglang/test/srt/quant/test_awq_dequant.py | 175 + .../sglang/test/srt/quant/test_block_int8.py | 230 + .../sglang/test/srt/quant/test_fp8_kernel.py | 126 + .../quant/test_fused_rms_fp8_group_quant.py | 148 + .../sglang/test/srt/quant/test_int8_kernel.py | 172 + .../test/srt/quant/test_triton_scaled_mm.py | 93 + .../test/srt/quant/test_w4a8_deepseek_v3.py | 246 + .../test/srt/quant/test_w8a8_quantization.py | 111 + .../sglang/test/srt/rl/test_fp32_lm_head.py | 111 + .../srt/rl/test_update_weights_from_disk.py | 439 + .../test_update_weights_from_distributed.py | 765 ++ .../srt/rl/test_update_weights_from_tensor.py | 295 + .../test/srt/rotary_embedding/test_mrope.py | 143 + .../.vendor/sglang/test/srt/run_suite.py | 530 ++ .../.vendor/sglang/test/srt/test_abort.py | 191 + .../sglang/test/srt/test_bench_one_batch.py | 72 + .../sglang/test/srt/test_bench_serving.py | 562 ++ .../srt/test_bench_typebaseddispatcher.py | 261 + .../.vendor/sglang/test/srt/test_bnb.py | 307 + .../sglang/test/srt/test_build_eagle_tree.py | 308 + .../sglang/test/srt/test_chunked_prefill.py | 31 + .../test/srt/test_constrained_decoding.py | 69 + ...est_constrained_decoding_spec_reasoning.py | 100 + .../sglang/test/srt/test_create_kvindices.py | 74 + .../sglang/test/srt/test_cutedsl_moe.py | 482 ++ .../sglang/test/srt/test_data_parallelism.py | 78 + .../test/srt/test_deepseek_v32_basic.py | 78 + .../sglang/test/srt/test_deepseek_v32_mtp.py | 105 + .../sglang/test/srt/test_deepseek_v3_basic.py | 75 + .../test/srt/test_deepseek_v3_cutedsl_4gpu.py | 162 + .../test/srt/test_deepseek_v3_fp4_4gpu.py | 176 + .../sglang/test/srt/test_deepseek_v3_mtp.py | 108 + .../sglang/test/srt/test_deterministic.py | 61 + .../test/srt/test_disaggregation_basic.py | 436 + .../srt/test_disaggregation_different_tp.py | 301 + .../srt/test_disaggregation_dp_attention.py | 97 + .../test_disaggregation_hybrid_attention.py | 81 + .../sglang/test/srt/test_disaggregation_pp.py | 89 + .../sglang/test/srt/test_dp_attention.py | 128 + .../srt/test_eagle_constrained_decoding.py | 77 + .../test/srt/test_eagle_dp_attention.py | 129 + .../sglang/test/srt/test_eagle_infer_a.py | 438 + .../sglang/test/srt/test_eagle_infer_b.py | 511 ++ .../sglang/test/srt/test_eagle_infer_beta.py | 96 + .../test/srt/test_eval_accuracy_large.py | 92 + .../sglang/test/srt/test_eval_fp8_accuracy.py | 114 + .../sglang/test/srt/test_external_models.py | 28 + .../.vendor/sglang/test/srt/test_fa3.py | 263 + .../sglang/test/srt/test_flash_attention_4.py | 54 + .../.vendor/sglang/test/srt/test_flashmla.py | 132 + .../.vendor/sglang/test/srt/test_fp8_utils.py | 44 + .../.vendor/sglang/test/srt/test_fused_moe.py | 240 + .../.vendor/sglang/test/srt/test_gguf.py | 27 + .../sglang/test/srt/test_gpt_oss_1gpu.py | 31 + .../sglang/test/srt/test_gpt_oss_4gpu.py | 36 + .../sglang/test/srt/test_gpt_oss_common.py | 152 + .../sglang/test/srt/test_gptqmodel_dynamic.py | 204 + .../sglang/test/srt/test_harmony_parser.py | 876 ++ .../sglang/test/srt/test_hidden_states.py | 139 + .../test/srt/test_hybrid_attn_backend.py | 163 + .../sglang/test/srt/test_input_embeddings.py | 151 + .../.vendor/sglang/test/srt/test_io_struct.py | 577 ++ .../test/srt/test_jinja_template_utils.py | 310 + .../sglang/test/srt/test_llama31_fp4.py | 58 + .../test_load_weights_from_remote_instance.py | 383 + .../sglang/test/srt/test_local_attn.py | 72 + .../sglang/test/srt/test_mamba_unittest.py | 337 + .../.vendor/sglang/test/srt/test_metrics.py | 88 + .../sglang/test/srt/test_metrics_utils.py | 137 + .../.vendor/sglang/test/srt/test_mla.py | 52 + .../sglang/test/srt/test_mla_deepseek_v3.py | 191 + .../sglang/test/srt/test_mla_flashinfer.py | 124 + .../.vendor/sglang/test/srt/test_mla_fp8.py | 49 + .../test/srt/test_mla_int8_deepseek_v3.py | 229 + .../sglang/test/srt/test_model_hooks.py | 152 + .../sglang/test/srt/test_modelopt_export.py | 352 + .../sglang/test/srt/test_modelopt_loader.py | 566 ++ .../test/srt/test_moe_eval_accuracy_large.py | 94 + ...ulti_instance_release_memory_occupation.py | 253 + .../sglang/test/srt/test_multi_tokenizer.py | 83 + .../srt/test_ngram_speculative_decoding.py | 117 + .../test/srt/test_no_chunked_prefill.py | 30 + .../test/srt/test_no_overlap_scheduler.py | 35 + .../sglang/test/srt/test_original_logprobs.py | 195 + .../.vendor/sglang/test/srt/test_page_size.py | 47 + .../sglang/test/srt/test_patch_torch.py | 133 + .../.vendor/sglang/test/srt/test_penalty.py | 246 + .../test/srt/test_piecewise_cuda_graph.py | 417 + .../sglang/test/srt/test_pp_single_node.py | 322 + .../test/srt/test_priority_scheduling.py | 421 + .../sglang/test/srt/test_profile_merger.py | 363 + .../test/srt/test_profile_merger_http_api.py | 162 + .../sglang/test/srt/test_profile_v2.py | 102 + .../test/srt/test_pytorch_sampling_backend.py | 90 + .../sglang/test/srt/test_quantization.py | 138 + .../sglang/test/srt/test_radix_attention.py | 87 + .../sglang/test/srt/test_radix_cache_unit.py | 646 ++ .../sglang/test/srt/test_reasoning_parser.py | 603 ++ .../srt/test_release_memory_occupation.py | 405 + .../test/srt/test_request_queue_validation.py | 88 + .../sglang/test/srt/test_retract_decode.py | 117 + .../.vendor/sglang/test/srt/test_rope_rocm.py | 116 + .../.vendor/sglang/test/srt/test_score_api.py | 590 ++ .../sglang/test/srt/test_server_args.py | 274 + .../test/srt/test_skip_tokenizer_init.py | 244 + .../test/srt/test_speculative_registry.py | 149 + .../sglang/test/srt/test_srt_endpoint.py | 743 ++ .../sglang/test/srt/test_srt_engine.py | 219 + .../test_standalone_speculative_decoding.py | 115 + .../sglang/test/srt/test_start_profile.py | 342 + .../sglang/test/srt/test_swa_unittest.py | 339 + .../sglang/test/srt/test_torch_compile.py | 80 + .../sglang/test/srt/test_torch_compile_moe.py | 77 + .../test_torch_native_attention_backend.py | 47 + .../.vendor/sglang/test/srt/test_torchao.py | 95 + .../test/srt/test_triton_attention_backend.py | 66 + .../test/srt/test_triton_attention_kernels.py | 783 ++ .../sglang/test/srt/test_triton_fused_moe.py | 192 + .../srt/test_triton_moe_channel_fp8_kernel.py | 184 + .../test/srt/test_triton_sliding_window.py | 129 + .../test/srt/test_type_based_dispatcher.py | 222 + .../test/srt/test_utils_update_weights.py | 170 + .../sglang/test/srt/test_video_utils.py | 59 + .../test/srt/test_vision_chunked_prefill.py | 200 + .../test/srt/test_vision_openai_server_a.py | 226 + .../srt/test_vision_openai_server_common.py | 583 ++ .../sglang/test/srt/test_vlm_input_format.py | 255 + .../test/srt/test_wave_attention_kernels.py | 322 + .../test/srt/xpu/test_intel_xpu_backend.py | 64 + .../NOTES.md | 65 + .../README.md | 29 + .../meta.json | 23 + .../requirements.txt | 38 + .../runner.py | 236 + .../runner_config.yaml | 8 + .../runner_config_conservative.yaml | 2 + .../test_runner.py | 122 + .../triton_kernel_config.json | 5 + 2923 files changed, 797743 insertions(+) create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.clang-format-ignore create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.devcontainer/Dockerfile create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.devcontainer/devcontainer.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.editorconfig create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/CI_PERMISSIONS.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/CODEOWNERS create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/FOLDER_README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/ISSUE_TEMPLATE/1-bug-report.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/ISSUE_TEMPLATE/2-feature-request.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/MAINTAINER.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/labeler.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/pull_request_template.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/update_ci_permission.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/auto-format.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/auto-tune.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/bot-bump-kernel-version-to-sglang.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/bot-bump-kernel-version.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/bot-bump-sglang-version.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/cancel-all-pending-pr-test-runs.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/cancel-pr-workflow-on-merge.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/ci-failure-monitor.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/ci-monitor.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/close-inactive-issues.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/execute-notebook.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/labeler.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/lint.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/nightly-release-gateway.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/nightly-test-amd.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/nightly-test-intel.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/nightly-test-nvidia.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/open-pr-copy-from-oss.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/open-pr-copy-to-oss.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-benchmark-rust.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-gate.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-amd.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-npu.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-pd-router.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-rust.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-xeon.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-xpu.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-amd-nightly.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-amd.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-cu13.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-dev.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-gateway.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-npu-nightly.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-npu.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-xeon.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docs.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-fake-tag.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-pypi-gateway.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-pypi.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-whl-kernel.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/slash-command-handler.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/stress-test.yml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.gitignore create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.isort.cfg create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.pre-commit-config.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/PROFILING.md create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/client.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/install_rpd.sh create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/loadTracer.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/rpd.patch create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/rpd_profile_server_enable.patch create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/rpd_profile_server_enable_wCPU_activities.patch create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/server.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/torch_profiler.patch create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/tuning/TUNING.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/tuning/benchmark_moe_rocm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/CODE_OF_CONDUCT.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/LICENSE create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/Makefile create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/assets/logo.png create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/assets/logo.svg create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/assets/logo_square.svg create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/bench_attention_sink/bench_attention_sink_triton.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/bench_in_batch_prefix/bench_in_batch_prefix.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/benchmark_batch/benchmark_batch.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/benchmark_batch/benchmark_tokenizer.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/benchmark_vllm_060/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/blog_v0_2/405b_sglang.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/blog_v0_2/405b_trt.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/blog_v0_2/405b_vllm.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/blog_v0_2/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/blog_v0_2/config.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/boolq/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/boolq/bench_sglang.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/boolq/convert_parquet_to_json.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/boolq/parquet_to_json.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/ceval/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/ceval/bench_sglang.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/deepseek_v3/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/dspy/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/dspy/bench_dspy_intro.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/generative_agents/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/generative_agents/agent_functions.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/generative_agents/bench_other.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/generative_agents/bench_sglang.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/gpt_oss/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/gsm8k/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/gsm8k/bench_other.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/gsm8k/bench_sglang.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hellaswag/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hellaswag/bench_other.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hellaswag/bench_sglang.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hf3fs/bench.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hf3fs/bench_client.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hf3fs/bench_storage.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hf3fs/bench_zerocopy.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/bench_long_context.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/bench_mix.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/bench_mix.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/bench_multiturn.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/bench_serving.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/data_processing.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/download.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/nextqa.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/perf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_decode_regex/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_decode_regex/bench_other.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_decode_regex/bench_sglang.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_decode_regex/build_dataset.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_jump_forward/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_jump_forward/bench_other.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_jump_forward/bench_sglang.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_jump_forward/build_dataset.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_jump_forward/dataset.txt create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_schema/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_schema/bench_sglang.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/all_reduce/benchmark_aiter.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/all_reduce/benchmark_mscclpp.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/all_reduce/benchmark_torch_symm_mem.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/decoding_attention_triton/triton_flashinfer_cudnn.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepep/deepep_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepep/tuning_deepep.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepseek/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm_blackwell.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_group_gemm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/elementwise/benchmark_concat_mla.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/flashinfer_allreduce_fusion/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/benchmark_sglang_fused_moe_triton.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/benchmark_torch_compile_fused_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/common_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/tuning_client.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton_sep.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/tuning_text.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/quantization/bench_fp4_quant.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/quantization/bench_int8_quant.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/quantization/tuning_block_wise_kernel.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/scheduler_batch/benchmark_get_last_loc_triton.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/scheduler_batch/benchmark_write_req_to_token_pool_triton.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/sliding_window_attention_triton/bench_triton_swa_kernel.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/line_retrieval/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/line_retrieval/bench_sglang.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/line_retrieval/gen_data.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/README.md create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/bench_hf_llava_bench.sh create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/bench_hf_mme.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/bench_sglang.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/bench_sglang_mme.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/download_images.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llm_judge/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llm_judge/bench_other.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llm_judge/bench_sglang.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/long_json_decode/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/long_json_decode/bench_other.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/long_json_decode/bench_sglang.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/long_json_decode/build_dataset.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/lora/launch_server.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/lora/lora_bench.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmlu/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmlu/bench_other.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmlu/bench_sglang.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmlu/download_data.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/bench_hf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/bench_sglang.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/data_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/eval_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/prompt_format.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mtbench/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mtbench/bench_other.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mtbench/bench_sglang.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mtbench/bench_sglang_eagle.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_chain_reasoning/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_chain_reasoning/bench_other.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_chain_reasoning/bench_sglang.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_document_qa/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_document_qa/bench_other.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_document_qa/bench_sglang.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_document_qa/build_dataset.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_turn_chat/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_turn_chat/bench_other.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_turn_chat/bench_sglang.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_turn_chat/data_gen.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_turn_chat/long_prompt_multi_turn.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/prefill_only/bench_embeddings.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/prefill_only/bench_score.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/prefill_only/util.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/react/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/react/bench_other.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/react/bench_sglang.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/reasoning_benchmark/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/reasoning_benchmark/answer_extraction.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/reasoning_benchmark/bench_sglang.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/reasoning_benchmark/eval_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/.gitignore create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/bench_other.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/bench_sglang.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/lmql_funcs.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/topic.jsonl create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_deep/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_deep/bench_other.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_deep/bench_sglang.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_deep/lmql_funcs.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_v0/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_v0/bench_other.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_v0/bench_sglang.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/Dockerfile create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/compose.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/configs/.gitconfig create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/configs/.tmux.conf create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/configs/.vimrc create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/configs/.zshrc create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/configs/yank create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/diffusion.Dockerfile create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/gateway.Dockerfile create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/k8s-sglang-distributed-sts.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/k8s-sglang-service.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/npu.Dockerfile create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/rocm.Dockerfile create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/sagemaker.Dockerfile create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/serve create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/xeon.Dockerfile create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/xpu.Dockerfile create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/Makefile create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/_static/css/custom_log.css create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/_static/css/readthedocs.css create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/_static/image/logo.ico create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/_static/image/logo.png create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/attention_backend.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/checkpoint_engine.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/deterministic_inference.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/dp_for_multi_modal_encoder.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/expert_parallelism.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/forward_hooks.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/hicache.rst create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/hicache_best_practices.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/hicache_design.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/hyperparameter_tuning.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/lora.ipynb create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/observability.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/pd_disaggregation.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/quantization.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/router.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/separate_reasoning.ipynb create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/server_arguments.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/speculative_decoding.ipynb create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/structured_outputs.ipynb create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/structured_outputs_for_reasoning_models.ipynb create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/tool_parser.ipynb create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/vlm_query.ipynb create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/deepseek_v3.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/deepseek_v32.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/gpt_oss.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/llama4.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/native_api.ipynb create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/offline_engine_api.ipynb create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/openai_api.rst create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/openai_api_completions.ipynb create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/openai_api_embeddings.ipynb create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/openai_api_vision.ipynb create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/popular_model_usage.rst create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/qwen3.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/qwen3_vl.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/sampling_params.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/send_request.ipynb create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/conf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/deploy.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/bench_serving.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/benchmark_and_profiling.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/contribution_guide.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/development_guide_using_docker.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/release_process.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/setup_github_runner.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/get_started/install.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/index.rst create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/amd_gpu.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/ascend_npu.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/cpu_server.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/nvidia_jetson.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/tpu.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/xpu.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/custom_chat_template.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/environment_variables.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/faq.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/frontend/choices_methods.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/frontend/frontend_index.rst create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/frontend/frontend_tutorial.ipynb create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/learn_more.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/mindspore_models.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/deploy_on_k8s.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws-examples/d-svc.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws-examples/d.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws-examples/lb.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws-examples/p-svc.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws-examples/p.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/multi_node.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/multi_node_index.rst create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/rbg_pd/deepseekv32_pd.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/post_training_integration.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/production_metrics.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/production_request_trace.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/torch_compile_cache.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/requirements.txt create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/serve.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/classify_models.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/embedding_models.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/generative_models.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/modelscope.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/multimodal_language_models.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/rerank_models.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/reward_models.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/support_new_models.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/transformers_fallback.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/wrap_run_llm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/assets/.gitignore create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/assets/example_image.png create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_deepseekr1.jinja create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_deepseekv3.jinja create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_deepseekv31.jinja create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_deepseekv32.jinja create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_llama3.1_json.jinja create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_llama4_pythonic.jinja create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/vision_template_sarashina_vl.jinja create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/checkpoint_engine/update.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/anthropic_example_chat.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/anthropic_example_complete.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/azure_openai_example_chat.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/gemini_example_chat.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/gemini_example_complete.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/gemini_example_multimodal_chat.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/images/cat.jpeg create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/images/dog.jpeg create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/local_example_chat.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/local_example_complete.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/local_example_llava_next.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/openai_example_chat.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/openai_example_complete.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/openai_example_n.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/openai_example_o1.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/openrouter_example_chat.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/together_example_chat.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/together_example_complete.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/chinese_regex.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/choices_logprob.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/cot_decoding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/json_decode.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/json_logprobs.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/llava_video/srt_example_llava_v.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/llava_video/srt_example_llava_v.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/openai_chat_speculative.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/openai_speculative.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/parallel_sample.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/rag_using_parea/trace_and_evaluate_rag_using_parea.ipynb create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/readme_examples.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/sgl_gen_min_tokens.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/streaming.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/triton/Dockerfile create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/triton/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/triton/models/character_generation/config.pbtxt create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/docker-compose.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/grafana/dashboards/config/dashboard.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/grafana/dashboards/json/sglang-dashboard.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/grafana/datasources/datasource.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/opentelemetry.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/prometheus.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/tracing_compose.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/profiler/nsys_profile_tools/README.md create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/profiler/nsys_profile_tools/gputrc2graph.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/profiler/nsys_profile_tools/sglang_engine_model.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/custom_server.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/embedding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/fastapi_engine_inference.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/launch_engine.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/offline_batch_inference.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/offline_batch_inference_async.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/offline_batch_inference_eagle.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/offline_batch_inference_qwen_1m.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/offline_batch_inference_vlm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/readme.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/save_remote_state.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/save_sharded_state.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/hidden_states/hidden_states_engine.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/hidden_states/hidden_states_server.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/lora.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/multimodal/llama3_llava_server.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/multimodal/llava_onevision_server.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/multimodal/pixtral_server.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/multimodal/qwen_llava_server.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/multimodal_embedding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/openai_chat_with_response_prefill.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/reward_model.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/token_in_token_out/token_in_token_out_llm_engine.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/token_in_token_out/token_in_token_out_llm_server.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/token_in_token_out/token_in_token_out_vlm_engine.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/token_in_token_out/token_in_token_out_vlm_server.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/vertex_predict.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/sagemaker/deploy_and_serve_endpoint.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/usage/modelopt_quantize_and_export.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/pyproject.toml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/pyproject_cpu.toml create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/pyproject_other.toml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/pyproject_xpu.toml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/bench_offline_throughput.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/bench_one_batch.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/bench_one_batch_server.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/bench_serving.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/check_env.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/cli/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/cli/generate.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/cli/main.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/cli/serve.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/cli/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/compile_deep_gemm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/eval/llama3_eval.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/eval/loogle_eval.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/global_config.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/.clang-format create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/csrc/cuda_wait_value.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/csrc/hicache.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/cuda_wait_value.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/hicache.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/include/sgl_kernel/tensor.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/include/sgl_kernel/utils.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/include/sgl_kernel/utils.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/include/sgl_kernel/warp.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/api.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/anthropic.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/base_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/litellm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/openai.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/runtime_endpoint.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/vertexai.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/chat_template.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/choices.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/interpreter.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/ir.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/tracer.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/launch_server.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/benchmarks/compare_perf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/backend/vmoba/wan_1.3B_77_448_832.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/backend/vmoba/wan_1.3B_77_480_832.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/base.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/base.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/flux.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/hunyuanvideo.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/qwenimage.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/stepvideo.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/wanvideo.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/zimage.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/base.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/clip.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/llama.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/qwen_image.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/t5.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/base.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/flux.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/hunyuanvae.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/qwenimage.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/stepvideovae.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/wanvae.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/base.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/flux.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/hunyuan.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/qwen_image.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/stepvideo.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/zimage.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/flux.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/hunyuan.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/qwenimage.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/sampling_params.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/stepvideo.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/teacache.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/wan.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/zimage.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/csrc/attn/vmoba_attn/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/csrc/attn/vmoba_attn/setup.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/csrc/attn/vmoba_attn/tests/test_vmoba_attn.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/csrc/attn/vmoba_attn/vmoba/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/csrc/attn/vmoba_attn/vmoba/vmoba.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/cli.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/contributing.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/install.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/openai_api.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/support_matrix.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/support_new_models.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/envs.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/registry.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/communication_op.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/base_device_communicator.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/cpu_communicator.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/cuda_communicator.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/pynccl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/pynccl_wrapper.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/cli_types.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/generate.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/main.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/serve.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/diffusion_generator.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/common_api.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/image_api.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/stores.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/launch_server.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/activation.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/STA_configuration.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/aiter.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/attention_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/flash_attn.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/flash_attn_2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/sage_attn.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/sage_attn3.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/sdpa.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/sliding_tile_attn.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/vmoba.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/layer.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/selector.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/custom_op.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/layernorm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/linear.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/lora/linear.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/mlp.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/quantization/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/quantization/base_config.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/rotary_embedding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/triton_ops.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/usp.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/visual_embedding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/vocab_parallel_embedding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/loader/component_loader.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/loader/fsdp_load.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/loader/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/loader/weight_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/managers/forward_context.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/managers/scheduler.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/base.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/causal_wanvideo.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/flux.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/hunyuanvideo.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/stepvideo.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/zimage.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/base.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/bert.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/clip.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/llama.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/mistral_3.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/stepllm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/t5.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/vision.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/parameter.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/registry.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/schedulers/base.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_flow_match_euler_discrete.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_flow_unipc_multistep.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_self_forcing_flow_match.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_unipc_multistep.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_flux2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/common.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/hunyuanvae.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/stepvideovae.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vision_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/flux.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/flux_2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/hunyuan_pipeline.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/qwen_image.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/stepvideo_pipeline.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/wan_causal_dmd_pipeline.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/wan_dmd_pipeline.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/wan_i2v_dmd_pipeline.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/wan_i2v_pipeline.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/wan_pipeline.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/zimage_pipeline.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/composed_pipeline_base.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/executors/parallel_executor.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/executors/pipeline_executor.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/executors/sync_executor.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/lora_pipeline.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/causal_denoising.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/conditioning.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising_dmd.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/encoding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/image_encoding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/input_validation.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/latent_preparation.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/stepvideo_encoding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/text_encoding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/timestep_preparation.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/validators.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/cpu.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/cuda.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/interface.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/mps.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/rocm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/scheduler_client.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/server_args.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/sync_scheduler_client.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/utils/common.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/utils/distributed.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/utils/logging_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/utils/perf_logger.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/cli/test_generate_common.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/cli/test_generate_t2i_perf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/cli/test_generate_t2v_perf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/cli/test_generate_ti2v_perf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/cli/test_serve.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/run_suite.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/conftest.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/perf_baselines.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_2_gpu_a.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_2_gpu_b.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_a.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_b.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_common.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/testcase_configs.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/slack_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/test_files/launch_flux.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/test_files/launch_wan.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/test_offline_api.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/test_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/third_party/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/third_party/pynvml.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/profiler.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/_custom_ops.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_invariant_ops/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_invariant_ops/batch_invariant_ops.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_overlap/operations.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_overlap/operations_strategy.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_overlap/single_batch_overlap.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_overlap/two_batch_overlap.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/checkpoint_engine/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/checkpoint_engine/checkpoint_engine_worker.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/checkpoint_engine/update.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/compilation_config.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/compilation_counter.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/compile.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/compiler_interface.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/cuda_piecewise_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/fix_functionalization.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/fx_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/inductor_pass.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/pass_manager.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/piecewise_context_manager.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/chatglm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/dbrx.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/deepseek_ocr.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/deepseekvl2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/device_config.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/dots_ocr.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/dots_vlm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/exaone.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/falcon_h1.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/internvl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/janus_pro.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/jet_nemotron.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/jet_vlm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/kimi_linear.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/kimi_vl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/kimi_vl_moonvit.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/load_config.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/longcat_flash.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/mamba_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/model_config.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/modelopt_config.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/nano_nemotron_vl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/nemotron_h.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/olmo3.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/points_v15_chat.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/qwen3_next.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/qwen3_omni.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/qwen3_vl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/radio.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/step3_vl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/update_config.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/connector/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/connector/base_connector.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/connector/redis.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/connector/remote_instance.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/connector/s3.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/connector/serde/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/connector/serde/safe_serde.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/connector/serde/serde.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/connector/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/constants.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/constrained/base_grammar_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/constrained/llguidance_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/constrained/outlines_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/constrained/outlines_jump_forward.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/constrained/reasoner_grammar_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/constrained/triton_ops/bitmask_ops.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/constrained/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/constrained/xgrammar_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/custom_op.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/debug_utils/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/debug_utils/dump_comparator.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/debug_utils/dump_loader.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/debug_utils/dumper.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/debug_utils/log_parser.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/debug_utils/model_truncator.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/debug_utils/tensor_dump_forward_hook.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/debug_utils/text_comparator.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/disaggregation/ascend/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/disaggregation/ascend/conn.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/disaggregation/ascend/transfer_engine.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/disaggregation/base/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/disaggregation/base/conn.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/disaggregation/common/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/disaggregation/common/conn.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/disaggregation/common/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/disaggregation/decode.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/disaggregation/fake/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/disaggregation/fake/conn.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/disaggregation/kv_events.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/disaggregation/mooncake/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/disaggregation/mooncake/conn.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/disaggregation/mooncake/transfer_engine.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/disaggregation/mooncake/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/disaggregation/nixl/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/disaggregation/nixl/conn.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/disaggregation/prefill.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/disaggregation/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/distributed/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/distributed/communication_op.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/distributed/device_communicators/all_reduce_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/distributed/device_communicators/cuda_wrapper.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/distributed/device_communicators/custom_all_reduce.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/distributed/device_communicators/custom_all_reduce_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/distributed/device_communicators/hpu_communicator.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/distributed/device_communicators/npu_communicator.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/distributed/device_communicators/pymscclpp.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/distributed/device_communicators/pynccl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/distributed/device_communicators/pynccl_allocator.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/distributed/device_communicators/pynccl_wrapper.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/distributed/device_communicators/quick_all_reduce.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/distributed/device_communicators/shm_broadcast.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/distributed/device_communicators/torch_symm_mem.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/distributed/device_communicators/xpu_communicator.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/distributed/naive_distributed.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/distributed/parallel_state.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/distributed/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/dllm/algorithm/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/dllm/algorithm/base.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/dllm/algorithm/low_confidence.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/dllm/config.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/elastic_ep/elastic_ep.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/EngineBase.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/context.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/engine.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/grpc_server.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/harmony_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/http_server.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/http_server_engine.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/openai/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/openai/encoding_dsv32.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/openai/protocol.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/openai/serving_base.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/openai/serving_chat.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/openai/serving_classify.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/openai/serving_completions.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/openai/serving_embedding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/openai/serving_rerank.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/openai/serving_responses.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/openai/serving_score.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/openai/serving_tokenize.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/openai/tool_server.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/openai/usage_processor.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/openai/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/tool.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/entrypoints/warmup.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/environ.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/eplb/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/eplb/eplb_algorithms/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/eplb/eplb_algorithms/deepseek.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/eplb/eplb_algorithms/deepseek_vec.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/eplb/eplb_algorithms/elasticity_aware.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/eplb/eplb_manager.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/eplb/eplb_simulator/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/eplb/eplb_simulator/reader.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/eplb/expert_distribution.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/eplb/expert_location.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/eplb/expert_location_dispatch.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/eplb/expert_location_updater.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/function_call/base_format_detector.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/function_call/core_types.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/function_call/deepseekv31_detector.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/function_call/deepseekv32_detector.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/function_call/deepseekv3_detector.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/function_call/function_call_parser.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/function_call/glm4_moe_detector.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/function_call/gpt_oss_detector.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/function_call/json_array_parser.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/function_call/kimik2_detector.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/function_call/llama32_detector.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/function_call/minimax_m2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/function_call/mistral_detector.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/function_call/pythonic_detector.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/function_call/qwen25_detector.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/function_call/qwen3_coder_detector.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/function_call/step3_detector.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/function_call/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/grpc/__init__.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/grpc/compile_proto.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/grpc/grpc_request_manager.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/grpc/health_servicer.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/grpc/scheduler_launcher.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/grpc/sglang_scheduler.proto create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/grpc/sglang_scheduler_pb2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/grpc/sglang_scheduler_pb2.pyi create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/grpc/sglang_scheduler_pb2_grpc.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/activation.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/amx_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/aiter_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/ascend_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/attention_registry.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/base_attn_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/cutlass_mla_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/double_sparsity_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/dual_chunk_flashattention_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/fla/chunk.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/fla/chunk_delta_h.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/fla/chunk_o.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/fla/chunk_scaled_dot_kkt.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/fla/cumsum.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/fla/fused_gdn_gating.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/fla/fused_recurrent.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/fla/fused_sigmoid_gating_recurrent.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/fla/index.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/fla/kda.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/fla/l2norm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/fla/layernorm_gated.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/fla/op.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/fla/solve_tril.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/fla/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/fla/wy_fast.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/flashattention_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/flashinfer_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/flashinfer_mla_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/flashmla_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/hybrid_attn_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/intel_amx_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/mamba/causal_conv1d.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/mamba/causal_conv1d_triton.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/mamba/mamba.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/mamba/mamba2_metadata.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/mamba/mixer2_rms_norm_gated.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/mamba/ops/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/mamba/ops/layernorm_gated.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/mamba/ops/mamba_ssm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/mamba/ops/ssd_bmm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/mamba/ops/ssd_chunk_scan.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/mamba/ops/ssd_chunk_state.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/mamba/ops/ssd_combined.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/mamba/ops/ssd_state_passing.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/merge_state.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/npu_ops/mla_preprocess.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/nsa/dequant_k_cache.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/nsa/index_buf_accessor.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/nsa/nsa_indexer.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/nsa/quant_k_cache.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/nsa/tilelang_kernel.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/nsa/transform_index.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/nsa/triton_kernel.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/nsa/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/nsa_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/tbo_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/torch_flex_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/torch_native_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/triton_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/triton_ops/decode_attention.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/triton_ops/decode_attention.py.bak create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/triton_ops/decode_attention.py.v5_backup create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/triton_ops/double_sparsity_attention.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/triton_ops/extend_attention.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/triton_ops/merge_state.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/triton_ops/prefill_attention.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/triton_ops/rocm_mla_decode_rope.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/trtllm_mha_backend.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/trtllm_mla_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/vision.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/vision_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/wave_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/wave_ops/decode_attention.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/wave_ops/extend_attention.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/wave_ops/prefill_attention.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/attention/xpu_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/communicator.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/communicator_nsa_cp.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/deep_gemm_wrapper/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/deep_gemm_wrapper/compile_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/deep_gemm_wrapper/configurer.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/deep_gemm_wrapper/entrypoint.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/dp_attention.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/elementwise.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/flashinfer_comm_fusion.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/layernorm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/linear.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/logits_processor.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/model_parallel.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/modelopt_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/__init__.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/cutlass_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/cutlass_moe_params.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/cutlass_w4a8_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/ep_moe/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/ep_moe/kernels.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/ep_moe/layer.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/flashinfer_cutedsl_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_native.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=14336,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=14336,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=1792,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=1792,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3072,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3072,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3072,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3584,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=7168,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=144,N=512,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1024,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1024,device_name=NVIDIA_H200.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1344,device_name=NVIDIA_A100-SXM4-40GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1344,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1344,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=14336,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=14336,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1792,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1792,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=2048,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=2688,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=2688,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3072,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3072,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3200,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3584,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=6400,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=7168,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=800,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=160,N=192,device_name=NVIDIA_A800-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=20,N=2048,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=24,N=1024,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_H20,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=64,device_name=NVIDIA_A800-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=64,device_name=NVIDIA_L20,dtype=int8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=64,device_name=NVIDIA_L40S,dtype=int8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1024,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_A800-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_H200.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=2560,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=2560,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=2560,device_name=NVIDIA_H200.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=320,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=320,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=320,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=320,device_name=NVIDIA_H200.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_A800-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_H200.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=AMD_Instinct_MI300X.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=AMD_Instinct_MI325X.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=AMD_Radeon_Graphics.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=NVIDIA_H200.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=AMD_Instinct_MI300X.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=AMD_Instinct_MI325X.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=AMD_Radeon_Graphics.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_A100-SXM4-40GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_H200.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_H200.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=AMD_Instinct_MI300X.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=AMD_Instinct_MI325X.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=AMD_Radeon_Graphics.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_A100-SXM4-40GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_H200.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_L40S.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_H200.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=AMD_Instinct_MI300X.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=AMD_Instinct_MI325X.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=AMD_Radeon_Graphics.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_H200.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=192,device_name=NVIDIA_A800-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=192,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=192,device_name=NVIDIA_H20.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=192,device_name=NVIDIA_H200.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H20.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H200.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=512,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_A800-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H20.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H200.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=96,device_name=NVIDIA_H20.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=129,N=352,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=160,N=320,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=161,N=192,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=256,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=264,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=264,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=264,N=256,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=264,N=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=272,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=272,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=272,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=272,N=64,device_name=NVIDIA_A800-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=288,N=64,device_name=NVIDIA_A800-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_0/E=16,N=1024,device_name=NVIDIA_B200.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=352,device_name=NVIDIA_RTX_6000_Ada_Generation,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=384,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=768,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=768,device_name=NVIDIA_H20.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=160,N=192,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=160,N=320,device_name=NVIDIA_H20-3e.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=160,N=384,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=160,N=640,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=128,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=256,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=256,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=384,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=384,N=128,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=384,N=256,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=385,N=128,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=385,N=128,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=128,N=1856,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=128,N=1856,device_name=NVIDIA_L40S.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=128,N=192,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=128,N=352,device_name=NVIDIA_RTX_5880_Ada_Generation,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=128,N=384,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=128,N=928,device_name=NVIDIA_L40S.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=129,N=352,device_name=NVIDIA_B200,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=129,N=352,device_name=NVIDIA_RTX_PRO_6000_Blackwell_Max-Q_Workstation_Edition,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=129,N=704,device_name=NVIDIA_B200,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=160,N=384,device_name=NVIDIA_B200,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=161,N=192,device_name=NVIDIA_H200,dtype=fp8_w8a8,per_channel_quant=True.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=161,N=384,device_name=NVIDIA_RTX_PRO_6000_Blackwell_Max-Q_Workstation_Edition,dtype=fp8_w8a8.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=256,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=256,N=256,device_name=NVIDIA_B200.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=256,N=256,device_name=NVIDIA_H800,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=256,N=512,device_name=NVIDIA_B200.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=256,N=512,device_name=NVIDIA_H20.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=257,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=257,N=256,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=257,N=256,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128]_down.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=257,N=64,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=384,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=128,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=128,device_name=NVIDIA_H20-3e.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=128,device_name=NVIDIA_H200.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=128,device_name=NVIDIA_H800,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=256,device_name=NVIDIA_B200.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=256,device_name=NVIDIA_H20-3e.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=256,device_name=NVIDIA_H200.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=64,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=64,device_name=NVIDIA_H200.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/fused_marlin_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/fused_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/fused_moe_triton_config.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/fused_moe_triton_kernels.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/layer.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/moe_align_block_size.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/fused_moe_triton/triton_kernels_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/kt_ep_wrapper.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/moe_runner/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/moe_runner/base.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/moe_runner/deep_gemm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/moe_runner/runner.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/moe_runner/triton.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/moe_runner/triton_kernels.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/rocm_moe_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/router.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/token_dispatcher/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/token_dispatcher/base.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/token_dispatcher/deepep.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/token_dispatcher/fuseep.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/token_dispatcher/mooncake.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/token_dispatcher/standard.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/topk.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/moe/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/multimodal.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/parameter.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/pooler.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/auto_round.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/awq.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/awq_triton.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/base_config.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/blockwise_int8.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/compressed_tensors/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/compressed_tensors/schemes/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_scheme.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/compressed_tensors/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=24576,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=24576,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=24576,K=1536,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=36864,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=36864,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/fp8.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/fp8_kernel.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/fp8_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/fpgemm_fp8.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/gguf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/gptq.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/int8_kernel.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/int8_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/kv_cache.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/kvfp4_tensor.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/marlin_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/marlin_utils_fp8.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/modelopt_quant.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/moe_wna16.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/mxfp4.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/mxfp4_tensor.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/petit.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/petit_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/qoq.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/quark/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/quark/quark.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/quark/quark_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/quark/schemes/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/quark/schemes/quark_scheme.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/quark/schemes/quark_w4a4_mxfp4.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/quark/schemes/quark_w8a8_fp8.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/quark/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/rocm_mxfp4_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/unquant.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/w4afp8.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/w8a8_fp8.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/quantization/w8a8_int8.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/radix_attention.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/rocm_linear_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/rotary_embedding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/sampler.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/sparse_pooler.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/torchao_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/layers/vocab_parallel_embedding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/lora/backend/ascend_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/lora/backend/base_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/lora/backend/chunked_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/lora/backend/lora_registry.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/lora/backend/triton_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/lora/eviction_policy.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/lora/layers.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/lora/lora.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/lora/lora_config.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/lora/lora_manager.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/lora/lora_registry.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/lora/mem_pool.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/lora/triton_ops/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/lora/triton_ops/chunked_sgmv_expand.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/lora/triton_ops/chunked_sgmv_shrink.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/lora/triton_ops/gate_up_lora_b.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/lora/triton_ops/qkv_lora_b.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/lora/triton_ops/sgemm_lora_a.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/lora/triton_ops/sgemm_lora_b.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/lora/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/async_dynamic_batch_tokenizer.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/async_mm_data_processor.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/cache_controller.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/configure_logging.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/data_parallel_controller.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/detokenizer_manager.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/disagg_service.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/io_struct.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/mm_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/multi_tokenizer_mixin.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/multimodal_processor.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/overlap_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/request_metrics_exporter.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/schedule_batch.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/schedule_policy.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/scheduler.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/scheduler_dp_attn_mixin.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/scheduler_input_blocker.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/scheduler_metrics_mixin.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/scheduler_output_processor_mixin.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/scheduler_pp_mixin.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/scheduler_profiler_mixin.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/scheduler_recv_skipper.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/scheduler_runtime_checker_mixin.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/scheduler_update_weights_mixin.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/session_controller.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/template_manager.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/tokenizer_communicator_mixin.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/tokenizer_manager.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/tp_worker.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/managers/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/allocator.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/allocator_ascend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/base_prefix_cache.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/cache_init_params.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/chunk_cache.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/common.py create mode 120000 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/cpp_radix_tree/.clang-format create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/cpp_radix_tree/common.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/cpp_radix_tree/radix_tree.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2_binding.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2_debug.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2_impl.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2_node.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/evict_policy.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/flush_cache.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/hicache_storage.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/hiradix_cache.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/mamba_radix_cache.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/memory_pool.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/memory_pool_host.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/multimodal_cache.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/radix_cache.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/radix_cache_cpp.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/aibrix_kvcache/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/aibrix_kvcache/aibrix_kvcache_storage.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/aibrix_kvcache/unit_test.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/backend_factory.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/eic/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/eic/eic_storage.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/eic/test_unit.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/hf3fs/docs/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/hf3fs/docs/deploy_sglang_3fs_multinode.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/hf3fs/docs/setup_usrbio_client.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/hf3fs/hf3fs_client.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/hf3fs/hf3fs_usrbio_client.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/hf3fs/hf3fs_utils.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/hf3fs/mini_3fs_metadata_server.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/hf3fs/storage_hf3fs.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/hf3fs/test_hf3fs_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/lmcache/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/lmcache/example_config.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/lmcache/lmc_radix_cache.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/lmcache/unit_test.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/mooncake_store/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_store.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/mooncake_store/test_mooncake_store.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/nixl/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/nixl/hicache_nixl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/nixl/nixl_utils.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/storage/nixl/test_hicache_nixl_storage.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/swa_radix_cache.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/mem_cache/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/metrics/collector.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/metrics/func_timer.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/metrics/label_transform.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/metrics/startup_func_log_and_timer.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/metrics/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/model_executor/cpu_graph_runner.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/model_executor/cuda_graph_runner.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/model_executor/forward_batch_info.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/model_executor/hook_manager.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/model_executor/input_buffers.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/model_executor/mindspore_runner.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/model_executor/model_runner.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/model_executor/npu_graph_runner.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/model_executor/piecewise_cuda_graph_runner.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/model_loader/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/model_loader/loader.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/model_loader/remote_instance_weight_loader_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/model_loader/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/model_loader/weight_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/model_loader/weight_validation.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/apertus.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/arcee.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/baichuan.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/bailing_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/bailing_moe_nextn.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/bert.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/chatglm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/clip.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/commandr.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/dbrx.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/deepseek.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/deepseek_janus_pro.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/deepseek_nextn.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/deepseek_ocr.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/deepseek_v2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/deepseek_vl2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/dots_ocr.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/dots_vlm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/dots_vlm_vit.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/ernie4.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/ernie4_eagle.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/exaone.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/falcon_h1.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/gemma.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/gemma2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/gemma2_reward.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/gemma3_causal.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/gemma3_mm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/gemma3n_audio.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/gemma3n_causal.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/gemma3n_mm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/glm4.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/glm4_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/glm4_moe_nextn.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/glm4v.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/glm4v_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/gpt2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/gpt_bigcode.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/gpt_oss.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/granite.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/granitemoe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/grok.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/hunyuan.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/idefics2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/internlm2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/internlm2_reward.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/interns1.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/internvl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/jet_nemotron.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/jet_vlm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/kimi_linear.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/kimi_vl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/kimi_vl_moonvit.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/llada2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/llama.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/llama4.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/llama_classification.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/llama_eagle.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/llama_eagle3.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/llama_embedding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/llama_reward.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/llava.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/llavavid.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/longcat_flash.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/longcat_flash_nextn.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/mimo.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/mimo_mtp.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/mindspore.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/minicpm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/minicpm3.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/minicpmo.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/minicpmv.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/minimax_m2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/mistral.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/mixtral.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/mixtral_quant.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/mllama.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/mllama4.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/nano_nemotron_vl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/nemotron_h.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/nemotron_nas.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/nvila.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/nvila_lite.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/olmo.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/olmo2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/olmoe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/opt.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/orion.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/persimmon.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/phi.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/phi3_small.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/phi4mm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/phi4mm_audio.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/phi4mm_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/phimoe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/pixtral.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/points_v15_chat.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/qwen.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/qwen2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/qwen2_5_vl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/qwen2_audio.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/qwen2_classification.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/qwen2_eagle.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/qwen2_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/qwen2_rm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/qwen2_vl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/qwen3.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/qwen3_classification.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/qwen3_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/qwen3_next.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/qwen3_next_mtp.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/qwen3_omni_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/qwen3_vl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/qwen3_vl_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/radio.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/registry.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/roberta.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/sarashina2_vision.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/siglip.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/solar.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/stablelm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/starcoder2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/step3_vl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/teleflm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/torch_native_llama.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/transformers.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/xverse.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/xverse_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/models/yivl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/customized_mm_processor_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/internvl_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/mm_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/base_processor.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/clip.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/deepseek_ocr.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/deepseek_vl_v2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/dots_vlm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/gemma3.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/gemma3n.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/glm4v.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/internvl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/janus_pro.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/kimi_vl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/llava.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/minicpm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/mlama.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/mllama4.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/nano_nemotron_vl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/nvila.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/phi4mm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/pixtral.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/points_v15_chat.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/qwen_audio.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/qwen_vl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/sarashina2_vision.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multimodal/processors/step3_vl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multiplex/multiplexing_mixin.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/multiplex/pdmux_context.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/operations_strategy.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/parser/code_completion_parser.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/parser/conversation.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/parser/harmony_parser.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/parser/jinja_template_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/parser/reasoning_parser.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/sampling/custom_logit_processor.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/sampling/penaltylib/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/sampling/penaltylib/frequency_penalty.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/sampling/penaltylib/min_new_tokens.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/sampling/penaltylib/orchestrator.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/sampling/penaltylib/presence_penalty.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/sampling/sampling_batch_info.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/sampling/sampling_params.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/server_args.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/server_args_config_parser.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/speculative/base_spec_worker.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/speculative/cpp_ngram/.clang-format create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/speculative/cpp_ngram/ngram.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/speculative/cpp_ngram/ngram.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/speculative/cpp_ngram/ngram_cache.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/speculative/cpp_ngram/ngram_cache_binding.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/speculative/cpp_ngram/param.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/speculative/cpp_ngram/queue.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/speculative/draft_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/speculative/eagle_draft_extend_npu_graph_runner.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/speculative/eagle_draft_npu_graph_runner.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/speculative/eagle_info.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/speculative/eagle_info_v2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/speculative/eagle_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/speculative/eagle_worker.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/speculative/eagle_worker_v2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/speculative/ngram_info.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/speculative/ngram_worker.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/speculative/spec_info.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/speculative/spec_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/speculative/standalone_worker.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/tokenizer/tiktoken_tokenizer.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/tracing/trace.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/utils/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/utils/aio_rwlock.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/utils/bench_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/utils/common.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/utils/cuda_ipc_transport_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/utils/hf_transformers_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/utils/host_shared_memory.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/utils/numa_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/utils/nvtx_pytorch_hooks.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/utils/offloader.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/utils/patch_torch.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/utils/poll_based_barrier.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/utils/profile_merger.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/utils/profile_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/utils/rpd_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/utils/slow_rank_detector.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/utils/torch_memory_saver_adapter.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/utils/weight_checker.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/weight_sync/tensor_bucket.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/weight_sync/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/attention/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/attention/test_flashattn_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/attention/test_flashattn_mla_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/attention/test_prefix_chunk_info.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/attention/test_trtllm_mla_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/ci/ci_register.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/ci/ci_stress_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/ci/ci_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/doc_patch.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/few_shot_gsm8k.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/few_shot_gsm8k_engine.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/get_logits_ut.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/gsm8k_mixin.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/kits/ebnf_constrained_kit.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/kits/json_constrained_kit.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/kits/matched_stop_kit.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/kits/radix_cache_server_kit.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/kits/regex_constrained_kit.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/long_prompt.txt create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/longbench_v2/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/longbench_v2/longbench_v2_evaluation.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/longbench_v2/test_longbench_v2_eval.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/longbench_v2/validate_longbench_v2.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/longbench_v2/validate_longbench_v2_standalone.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/mmmu_vlm_mixin.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/nightly_bench_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/run_eval.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/runners.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/send_one.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_common.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_gpqa.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_humaneval.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_longbench_v2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_math.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_mgsm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_mmlu.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_mmmu_vlm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/speculative/test_spec_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_activation.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_block_fp8.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_block_fp8_deep_gemm_blackwell.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_custom_ops.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_cutlass_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_cutlass_w16a16_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_cutlass_w4a8_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_deepep_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_deterministic.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_deterministic_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_disaggregation_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_dynamic_grad_mode.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_kvfp4_quant_dequant.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_layernorm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_marlin_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_marlin_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_programs.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/version.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/check_vram_clear.sh create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/amd_ci_exec.sh create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/amd_ci_install_dependency.sh create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/amd_ci_start_container.sh create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/ci_install_deepep.sh create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/ci_install_dependency.sh create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/ci_install_rust.sh create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/ci_start_disaggregation_servers.sh create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/cleanup_hf_cache.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/npu_ci_install_dependency.sh create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/prepare_runner.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/publish_traces.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/slash_command_handler.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci_monitor/README.md create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci_monitor/ci_analyzer.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci_monitor/ci_analyzer_balance.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci_monitor/ci_analyzer_perf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci_monitor/ci_failures_analysis.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/code_sync/copy_from_oss.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/code_sync/copy_to_oss.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/code_sync/guideline.md create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/code_sync/install_github_cli.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/convert_otel_2_perfetto.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ensure_vram_clear.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/export_deepseek_nextn.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/killall_sglang.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/bench_speculative.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/disaggregation/cli-logprob.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/disaggregation/cli-so.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/disaggregation/cli.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/frontend_reasoning.ipynb create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/load_tokenizer.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/long_context_example.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/lora/analyzer.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/lora/lora_hf_play.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/lora/lora_vllm_play.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/reference_hf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/replay_request_dump.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/router/test_tree.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/router/tree.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/README.md create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/bump_kernel_version.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/bump_kernel_version_to_sglang.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/bump_sglang_version.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/check_kernel_version_to_sglang.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/commit_and_pr.sh create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/commit_and_pr_kernel_to_sglang.sh create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/test_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/sort_testcases_alphabetically.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/update_kernel_whl_index.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/version_branch_to_tag.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/.clang-format create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/CMakeLists.txt create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/LICENSE create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/Makefile create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/THIRDPARTYNOTICES.txt create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_activation.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_awq_dequant.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_cutlass_mla.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_dsv3_fused_a_gemm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_dsv3_router_gemm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_es_fp8_blockwise_grouped_gemm.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_fp4_gemm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_fp8_blockwise_gemm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_fp8_blockwise_group_gemm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_fp8_gemm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_int8_gemm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_kimi_k2_moe_fused_gate.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_moe_align_block_size.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_moe_ep_post_reorder.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_moe_fused_gate.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_moe_topk_sigmoid.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_moe_topk_softmax.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_mrope.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_nvfp4_scaled_gemm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_per_tensor_quant_fp8.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_per_token_group_quant_8bit.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_per_token_quant_fp8.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_qserve_w4a8_gemm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_rmsnorm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_rotary_embedding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_sum_scale.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_top_k_top_p_sampling.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/build.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/cmake/flashmla.cmake create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/cmake/utils.cmake create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/custom_all_reduce.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/custom_all_reduce.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/custom_all_reduce.hip create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/custom_all_reduce_hip.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/mscclpp_allreduce.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/mscclpp_allreduce.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/quick_all_reduce.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/quick_all_reduce.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/quick_all_reduce.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/quick_all_reduce_base.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/test_mscclpp_allreduce.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cascade.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cutlass_mla_kernel.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cutlass_sm100_mla/device/sm100_mla.hpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_fmha_mla_reduction.hpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_fmha_mla_tma_warpspecialized.hpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_mla_tile_scheduler.hpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/merge_attn_states.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/vertical_slash_index.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/common_extension.cc create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/common_extension_rocm.cc create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/CMakeLists.txt create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/activation.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/bmm.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/common.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/decode.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/extend.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/gemm.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/gemm.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/gemm_fp8.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/gemm_int8.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/interface.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/moe.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/moe_fp8.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/moe_int8.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/norm.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/numa_utils.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/qkv_proj.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/rope.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/shm.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/shm.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/topk.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/torch_extension_cpu.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/vec.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/common.hpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/detail/collective/mixed_input_utils.hpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/epilogue/epilogue_per_row_per_col_scale.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/collective/builders/sm90_gmma_builder_mixed_input.inl create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/collective/collective_builder_mixed_input.hpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/collective/collective_mma_array_mixed_input.hpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/collective/sm90_mma_array_tma_gmma_rs_warpspecialized_mixed_input_.hpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/cutlass_gemm_caller.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/dispatch_policy.hpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/fp8_blockwise_gemm_sm90_dispatch.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/gemm_universal_base_compat.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/gemm_with_epilogue_visitor.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/activation.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/cast.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/concat_mla.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/copy.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/fused_add_rms_norm_kernel.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/pos_enc.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/rope.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/topk.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/utils.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/expert_specialization/es_fp8_blockwise.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/expert_specialization/es_fp8_blockwise_functor.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/expert_specialization/es_fp8_blockwise_launcher.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/expert_specialization/es_fp8_blockwise_traits.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/flash_extension.cc create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/flashmla_extension.cc create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/awq_kernel.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/bmm_fp8.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/dsv3_fused_a_gemm.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/dsv3_router_gemm_bf16_out.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/dsv3_router_gemm_entry.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/dsv3_router_gemm_float_out.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/fp8_blockwise_gemm_kernel.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/fp8_gemm_kernel.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/compat.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/gptq_kernel.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/matrix_view.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/qdq_2.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/qdq_3.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/qdq_4.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/qdq_8.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/qdq_util.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/int8_gemm_kernel.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/awq_marlin_repack.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/dequant.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/gptq_marlin.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/gptq_marlin_repack.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/kernel.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/marlin.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/marlin_dtypes.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/marlin_template.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/math.hpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_expert_quant.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_quant.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_quant_entry.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_quant_kernels.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_scaled_mm_entry.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_scaled_mm_kernels.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/per_tensor_quant_fp8.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/per_token_group_quant_8bit.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/per_token_group_quant_8bit_v2.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/per_token_quant_fp8.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/qserve_w4a8_per_chn_gemm.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/qserve_w4a8_per_group_gemm.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/grammar/apply_token_bitmask_inplace_cuda.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/kvcacheio/transfer.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/mamba/causal_conv1d.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/mamba/causal_conv1d.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/memory/store.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/memory/weak_ref_tensor.cpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/scaled_mm_entry.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_get_group_starts.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_moe_data.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe_helper.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/fp8_blockwise_moe_kernel.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/fused_qknorm_rope_kernel.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/kimi_k2_moe_fused_gate.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/generate_kernels.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku4.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku4b8.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku8b128.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku4.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku4b8.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku8b128.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_marlin.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/marlin_template.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/ops.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_align_kernel.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_fused_gate.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_sum.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_sum_reduce.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_topk_sigmoid_kernels.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_topk_softmax_kernels.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/nvfp4_blockwise_moe.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/prepare_moe_input.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/dequantize.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/ggml-common.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/gguf_kernel.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/mmq.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/mmvq.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/moe.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/moe_vec.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/vecdotq.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/spatial/cuda_utils.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/spatial/greenctx_stream.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/spatial/greenctx_stream.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/spatial_extension.cc create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/speculative/eagle_utils.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/speculative/ngram_utils.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/speculative/packbit.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/speculative/speculative_sampling.cu create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/speculative/speculative_sampling.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/hip_act_and_mul.cuh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/hip_math_def.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/hip_vec_dtypes.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/impl/hip_vec_bf16_impl.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/impl/hip_vec_fp32_impl.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/impl/hip_vec_half_impl.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/pytorch_extension_utils_rocm.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/scalar_type.hpp create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/sgl_flash_kernel_ops.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/sgl_kernel_ops.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/sgl_kernel_torch_shim.h create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/utils.h create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/kernel-runner-setup.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/pyproject.toml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/pyproject_cpu.toml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/pyproject_rocm.toml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/_fa4_interface.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/allreduce.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/attention.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/cutlass_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/elementwise.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/expert_specialization.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/flash_attn.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/flash_mla.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/fused_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/gemm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/grammar.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/hadamard.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/kvcacheio.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/load_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/mamba.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/marlin.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/memory.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/quantization/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/quantization/gguf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/sampling.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/scalar_type.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/sparse_flash_attn.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/spatial.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/speculative.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/test_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/testing/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/testing/rotary_embedding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/top_k.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/version.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/rename_wheels.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/setup_rocm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/conftest.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/spatial/test_greenctx_stream.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/speculative/test_eagle_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/speculative/test_ngram_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/speculative/test_speculative_sampling.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_activation.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_apply_token_bitmask_inplace.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_awq_dequant.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_bmm_fp8.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_causal_conv1d.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_copy.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_custom_allreduce.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_cutlass_mla.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_cutlass_w4a8_moe_mm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_dsv3_fused_a_gemm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_dsv3_router_gemm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_es_fp8_blockwise_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_flash_attention.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_flash_attention_4.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_flashmla.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fp4_gemm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fp4_quantize.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fp8_blockwise_gemm.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fp8_blockwise_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fp8_gemm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fused_qk_norm_rope.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_gguf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_gptq_kernel.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_hadamard.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_int8_gemm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_kimi_k2_moe_fused_gate.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_kvcacheio.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_marlin_gemm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_marlin_repack.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_merge_state.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_merge_state_v2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_moe_align.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_moe_fused_gate.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_moe_topk_sigmoid.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_moe_topk_softmax.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_mscclpp.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_norm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_per_tensor_quant_fp8.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_per_token_group_quant_8bit.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_per_token_quant_fp8.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_qserve_w4a8_per_chn_gemm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_qserve_w4a8_per_group_gemm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_rotary_embedding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_sampling.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_sparse_flash_attn.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_topk.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_torch_defaults_reset.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/.cargo/config.toml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/Cargo.toml create mode 120000 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/LICENSE create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/Makefile create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/benches/request_processing.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/benches/tokenizer_benchmark.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/benches/tool_parser_benchmark.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/.gitignore create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/Cargo.toml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/Makefile create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/client.go create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/client_test.go create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/examples/simple/main.go create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/examples/simple/run.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/examples/streaming/main.go create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/examples/streaming/run.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/integration_test.go create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/internal/ffi/client.go create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/client.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/error.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/grpc_converter.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/lib.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/memory.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/stream.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/tokenizer.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/tool_parser.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/utils.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/.coveragerc create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/Cargo.toml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/MANIFEST.in create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/pyproject.toml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/setup.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/__main__.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/cli.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/launch_router.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/launch_server.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/mini_lb.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/router.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/router_args.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/version.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/src/lib.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/build.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/conftest.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/basic/test_openai_server.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/conftest.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/features/test_enable_thinking.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/features/test_reasoning_content.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/fixtures.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/function_call/test_openai_function_calling.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/function_call/test_tool_choice.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/pytest.ini create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/util.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/validation/test_large_max_new_tokens.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/validation/test_openai_server_ignore_eos.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_http/conftest.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_http/test_e2e_embeddings.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_http/test_pd_router.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_http/test_regular_router.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/conftest.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/features/test_basic_crud.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/features/test_state_management.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/features/test_streaming_events.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/features/test_structured_output.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/features/test_tools_call.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/router_fixtures.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/util.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/fixtures/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/fixtures/generate_test_certs.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/fixtures/mock_worker.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/fixtures/ports.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/fixtures/router_manager.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/conftest.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/load_balancing/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/load_balancing/test_cache_aware.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/load_balancing/test_power_of_two.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/load_balancing/test_random.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/load_balancing/test_round_robin.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_api_auth.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_circuit_breaker.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_fault_tolerance.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_mtls.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_payload_size.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_pd_routing.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_rate_limiting.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_retries.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_service_discovery_shim.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_worker_management.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/unit/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/unit/test_arg_parser.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/unit/test_router_config.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/unit/test_startup_sequence.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/unit/test_validation.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/pytest.ini create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/rustfmt.toml create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/scripts/generate_gateway_release_notes.sh create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/scripts/run_benchmarks.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/scripts/setup-sccache.sh create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/app_context.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/config/builder.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/config/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/config/types.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/config/validation.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/circuit_breaker.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/error.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/job_queue.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/metrics_aggregator.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/model_card.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/model_type.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/retry.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/token_bucket.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/worker.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/worker_builder.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/worker_manager.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/worker_registry.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/definition.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/engine.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/event.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/executor.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/state.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/steps/external_worker_registration.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/steps/mcp_registration.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/steps/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/steps/worker_registration.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/steps/worker_removal.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/types.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/common.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/core.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/factory.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/memory.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/noop.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/oracle.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/postgres.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/grpc_client/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/grpc_client/sglang_scheduler.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/grpc_client/vllm_engine.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/lib.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/logging.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/main.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/config.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/connection_pool.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/error.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/inventory.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/manager.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/oauth.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/proxy.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/tool_args.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/metrics.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/middleware.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/bucket.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/cache_aware.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/factory.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/power_of_two.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/random.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/registry.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/round_robin.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/tree.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/proto/sglang_scheduler.proto create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/proto/vllm_engine.proto create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/chat/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/chat/response.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/chat/stream_response.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/responses/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/responses/response.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/chat.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/classify.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/common.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/completion.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/embedding.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/generate.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/rerank.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/responses.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/sampling_params.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/validated.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/worker_spec.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/factory.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/base.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/deepseek_r1.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/glm45.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/kimi.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/minimax.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/qwen3.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/step3.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/traits.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/factory.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/client.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/response_collection.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/response_formatting.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/responses/handlers.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/responses/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/responses/streaming.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/responses/utils.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/client_acquisition.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/dispatch_metadata.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/helpers.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/request_execution.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/worker_selection.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/context.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/error.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/builder.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/detector.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/parser.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/processor.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/responses.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/stages/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/stages/preparation.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/stages/request_building.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/stages/response_processing.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/streaming.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/types.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/pd_router.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/pipeline.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/proto_wrapper.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/processor.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/context.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/conversions.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/handlers.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/tool_loop.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/types.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/chat/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/chat/preparation.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/chat/request_building.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/chat/response_processing.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/generate/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/generate/preparation.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/generate/request_building.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/generate/response_processing.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/preparation.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/request_building.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/response_processing.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/streaming.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/router.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/utils.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/header_utils.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/http/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/http/pd_router.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/http/pd_types.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/http/router.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/conversations.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/mcp.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/responses.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/router.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/streaming.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/utils.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/router_manager.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/server.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/service_discovery.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tokenizer/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tokenizer/cache/fingerprint.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tokenizer/cache/l0.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tokenizer/cache/l1.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tokenizer/cache/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tokenizer/chat_template.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tokenizer/factory.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tokenizer/hub.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tokenizer/huggingface.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tokenizer/mock.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tokenizer/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tokenizer/sequence.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tokenizer/stop.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tokenizer/stream.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tokenizer/tests.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tokenizer/tiktoken.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tokenizer/traits.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tool_parser/errors.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tool_parser/factory.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tool_parser/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tool_parser/parsers/deepseek.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tool_parser/parsers/glm4_moe.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tool_parser/parsers/helpers.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tool_parser/parsers/json.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tool_parser/parsers/kimik2.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tool_parser/parsers/llama.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tool_parser/parsers/minimax_m2.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tool_parser/parsers/mistral.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tool_parser/parsers/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tool_parser/parsers/passthrough.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tool_parser/parsers/pythonic.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tool_parser/parsers/qwen.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tool_parser/parsers/step3.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tool_parser/partial_json.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tool_parser/state.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tool_parser/tests.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tool_parser/traits.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/tool_parser/types.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/version.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/api_endpoints_test.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/cache_aware_backward_compat_test.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/chat_template_format_detection.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/chat_template_integration.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/chat_template_loading.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/common/mock_mcp_server.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/common/mock_openai_server.rs create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/common/mock_worker.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/common/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/common/streaming_helpers.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/common/test_app.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/mcp_test.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/metrics_aggregator_test.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/policy_registry_integration.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/request_formats_test.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/responses_api_test.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/spec/chat_completion.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/spec/chat_message.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/spec/embedding.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/spec/mod.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/spec/rerank.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/spec/responses.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/spec_test.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/streaming_tests.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/test_openai_routing.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/test_pd_routing.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/tokenizer_cache_correctness_test.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/tokenizer_integration.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/tool_parser_deepseek.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/tool_parser_edge_cases.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/tool_parser_fallback.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/tool_parser_glm4_moe.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/tool_parser_json.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/tool_parser_kimik2.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/tool_parser_llama.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/tool_parser_minimax_m2.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/tool_parser_mistral.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/tool_parser_mixed_edge_cases.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/tool_parser_partial_json.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/tool_parser_pythonic.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/tool_parser_qwen.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/tool_parser_step3.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/tests/workflow_test.rs create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/lora_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/ascend/test_ascend_w8a8_quantization.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/ascend/test_mindspore_models.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/cpu/test_comm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/debug_utils/test_log_parser.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/entrypoints/http_server/test_abort_request.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/ep/test_deepep_internode.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/ep/test_deepep_intranode.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/ep/test_deepep_low_latency.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/ep/test_eplb.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/ep/test_hybrid_dp_ep_tp_mtp.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/ep/test_moe_deepep.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/ep/test_moe_deepep_eval_accuracy_large.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/ep/test_mooncake_ep_small.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/hicache/test_disaggregation_hicache.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/lang_frontend/test_bind_cache.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/lang_frontend/test_choices.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/lang_frontend/test_jump_forward.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/lang_frontend/test_openai_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/lang_frontend/test_separate_reasoning.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/lang_frontend/test_separate_reasoning_execution.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/layers/attention/nsa/test_act_quant_triton.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/layers/moe/test_moe_runners.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/lora/test_chunked_sgmv_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/lora/test_lora_cuda_graph.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/lora/test_lora_llama4.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/lora/test_lora_qwen3_vl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/lora/test_lora_spec_decoding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/models/test_clip_models.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/models/test_falcon_h1_models.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/models/test_gme_qwen_models.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/models/test_grok_models.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/models/test_llama4_models.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/models/test_mtp_models.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/models/test_unsloth_models.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/nightly/test_deepseek_v31_perf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/nightly/test_deepseek_v32_perf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/nightly/test_text_models_gsm8k_eval.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/nightly/test_text_models_perf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/nightly/test_vlms_mmmu_eval.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/nightly/test_vlms_perf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/nightly/test_vlms_piecewise_cuda_graph.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/openai_server/features/test_cache_report.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/openai_server/features/test_continuous_usage_stats.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/openai_server/features/test_structural_tag.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/quant/test_fp8_kvcache.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_async_dynamic_batch_tokenizer.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_async_mm_data_processor.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_config_integration.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_custom_allreduce.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_deepseek_chat_templates.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_deepseek_v32_cp_single_node.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_double_sparsity.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_eagle_infer_beta_dp_attention.py create mode 100755 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_expert_distribution.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_expert_location_updater.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_fim_completion.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_forward_split_prefill.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_get_weights_by_name.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_health_check.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_kv_events.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_logprobs.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_mla_tp.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_modelopt.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_modelopt_fp8kvcache.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_models_from_modelscope.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_mscclpp.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_quick_allreduce.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_sagemaker_server.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_schedule_policy.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_session_control.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_srt_engine_with_quant_args.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_tokenizer_batch_encode.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_tokenizer_manager.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_torch_flex_attention_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_torch_tp.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_tracing.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_triton_attention_rocm_mla.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_triton_moe_wna16.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_two_batch_overlap.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_vertex_endpoint.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_vlm_accuracy.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_wave_attention_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/manual/test_weight_version.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/nightly_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_batch_invariant_ops.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_cpp_radix_cache.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_deepseek_r1_fp8_trtllm_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_deepseek_v31_perf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_deepseek_v32_nsabackend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_deepseek_v32_perf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_deepseek_v3_deterministic.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_deepseek_v3_fp4_cutlass_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_encoder_dp.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_flashinfer_trtllm_gen_attn_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_flashinfer_trtllm_gen_moe_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_fp4_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_glm_4_6_perf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_gpt_oss_4gpu_perf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_kimi_k2_thinking_perf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_lora_eviction_policy.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_lora_openai_api.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_lora_openai_compatible.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_lora_qwen3.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_lora_radix_cache.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_minimax_m2_perf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_nsa_indexer.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_qwen3_235b_perf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_qwen3_fp4_trtllm_gen_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_qwen3_next_deterministic.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_text_models_gsm8k_eval.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_text_models_perf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_vlms_mmmu_eval.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/nightly/test_vlms_perf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/pytest.ini create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/function_call/test_function_call_parser.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/function_call/test_json_schema_constraint.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/function_call/test_unknown_tool_name.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/stress/test_stress_deepseek_v3.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/stress/test_stress_glm_4_6.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/stress/test_stress_kimi_k2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/stress/test_stress_qwen3_235b.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/test_srt_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/run_suite.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/run_suite_nightly.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_deepep.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_deepseek_mtp.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_graph_tp1_bf16.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_graph_tp2_bf16.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_hicache_mha.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_mla_fia_w8a8int8.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_mla_w8a8int8.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_sampling_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_tp1_bf16.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_tp2_bf16.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_tp2_fia_bf16.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_tp4_bf16.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/deepseek_v3.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/deepseek_v3_long_context.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/llama_405b.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/random_config.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/random_flashinfer_vs_triton_config.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/sharegpt_config.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_activation.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_binding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_cpu_graph.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_decode.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_extend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_gemm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_intel_amx_attention_backend_a.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_intel_amx_attention_backend_b.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_intel_amx_attention_backend_c.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_mla.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_norm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_qkv_proj_with_rope.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_rope.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_shared_expert.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_topk.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/debug_utils/test_tensor_dump_forward_hook.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/double-sparsity-config-Llama-3.1-8B-Instruct.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ep/test_deepep_large.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ep/test_deepep_small.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ep/test_moe_ep.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/experiment_runner.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/external_models/custom_qwen2_vl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/hicache/test_hicache_storage.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/hicache/test_hicache_storage_3fs_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/hicache/test_hicache_storage_file_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/hicache/test_hicache_storage_mooncake_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/hicache/test_hicache_variants.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/kv_cache_scales_llama3_1_8b.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/kv_cache_scales_llama3_8b.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/kv_cache_scales_qwen2_1_5b.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/layers/attention/mamba/test_causal_conv1d.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/layers/attention/mamba/test_mamba2_mixer.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/layers/attention/mamba/test_mamba_ssm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/layers/attention/mamba/test_mamba_ssm_ssd.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_lora.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_lora_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_lora_eviction.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_lora_tp.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_lora_update.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_multi_lora_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/compare.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_compressed_tensors_models.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_cross_encoder_models.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_dummy_grok_models.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_embedding_models.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_encoder_embedding_models.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_generation_models.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_glm4_moe_models.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_kimi_k2_models.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_kimi_linear_models.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_nvidia_nemotron_nano_v2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_nvidia_nemotron_nano_v2_vl.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_qwen3_next_models.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_qwen_models.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_reward_models.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_transformers_models.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_vlm_models.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/nightly/test_gsm8k_eval_amd.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_openai_embedding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_openai_server.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_protocol.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_serving_chat.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_serving_completions.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_serving_embedding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/features/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/features/test_enable_thinking.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/features/test_json_mode.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/features/test_openai_server_ebnf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/features/test_openai_server_hidden_states.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/features/test_reasoning_content.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/function_call/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/function_call/test_openai_function_calling.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/function_call/test_tool_choice.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/validation/__init__.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/validation/test_large_max_new_tokens.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/validation/test_matched_stop.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/validation/test_openai_server_ignore_eos.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/validation/test_request_length_validation.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ops/test_repeat_interleave.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/parse_results.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_autoround.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_awq.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_awq_dequant.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_block_int8.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_fp8_kernel.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_fused_rms_fp8_group_quant.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_int8_kernel.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_triton_scaled_mm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_w4a8_deepseek_v3.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_w8a8_quantization.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/rl/test_fp32_lm_head.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/rl/test_update_weights_from_disk.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/rl/test_update_weights_from_distributed.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/rl/test_update_weights_from_tensor.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/rotary_embedding/test_mrope.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/run_suite.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_abort.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_bench_one_batch.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_bench_serving.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_bench_typebaseddispatcher.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_bnb.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_build_eagle_tree.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_chunked_prefill.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_constrained_decoding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_constrained_decoding_spec_reasoning.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_create_kvindices.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_cutedsl_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_data_parallelism.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v32_basic.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v32_mtp.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v3_basic.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v3_cutedsl_4gpu.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v3_fp4_4gpu.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v3_mtp.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deterministic.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_disaggregation_basic.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_disaggregation_different_tp.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_disaggregation_dp_attention.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_disaggregation_hybrid_attention.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_disaggregation_pp.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_dp_attention.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eagle_constrained_decoding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eagle_dp_attention.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eagle_infer_a.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eagle_infer_b.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eagle_infer_beta.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eval_accuracy_large.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eval_fp8_accuracy.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_external_models.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_fa3.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_flash_attention_4.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_flashmla.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_fp8_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_fused_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_gguf.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_gpt_oss_1gpu.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_gpt_oss_4gpu.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_gpt_oss_common.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_gptqmodel_dynamic.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_harmony_parser.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_hidden_states.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_hybrid_attn_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_input_embeddings.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_io_struct.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_jinja_template_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_llama31_fp4.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_load_weights_from_remote_instance.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_local_attn.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mamba_unittest.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_metrics.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_metrics_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mla.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mla_deepseek_v3.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mla_flashinfer.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mla_fp8.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mla_int8_deepseek_v3.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_model_hooks.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_modelopt_export.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_modelopt_loader.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_moe_eval_accuracy_large.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_multi_instance_release_memory_occupation.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_multi_tokenizer.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_ngram_speculative_decoding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_no_chunked_prefill.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_no_overlap_scheduler.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_original_logprobs.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_page_size.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_patch_torch.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_penalty.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_piecewise_cuda_graph.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_pp_single_node.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_priority_scheduling.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_profile_merger.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_profile_merger_http_api.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_profile_v2.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_pytorch_sampling_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_quantization.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_radix_attention.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_radix_cache_unit.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_reasoning_parser.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_release_memory_occupation.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_request_queue_validation.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_retract_decode.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_rope_rocm.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_score_api.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_server_args.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_skip_tokenizer_init.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_speculative_registry.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_srt_endpoint.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_srt_engine.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_standalone_speculative_decoding.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_start_profile.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_swa_unittest.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_torch_compile.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_torch_compile_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_torch_native_attention_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_torchao.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_triton_attention_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_triton_attention_kernels.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_triton_fused_moe.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_triton_moe_channel_fp8_kernel.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_triton_sliding_window.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_type_based_dispatcher.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_utils_update_weights.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_video_utils.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_vision_chunked_prefill.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_vision_openai_server_a.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_vision_openai_server_common.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_vlm_input_format.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_wave_attention_kernels.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/xpu/test_intel_xpu_backend.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/NOTES.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/README.md create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/meta.json create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/requirements.txt create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/runner.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/runner_config.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/runner_config_conservative.yaml create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/test_runner.py create mode 100644 runners/nvidia_sglang_inference_opt_4fd1e59e/triton_kernel_config.json diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.clang-format-ignore b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.clang-format-ignore new file mode 100644 index 00000000..15c76cc4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.clang-format-ignore @@ -0,0 +1 @@ +sgl-kernel/3rdparty/tensorrt_llm/* diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.devcontainer/Dockerfile b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.devcontainer/Dockerfile new file mode 100644 index 00000000..3c7b67ca --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.devcontainer/Dockerfile @@ -0,0 +1,35 @@ +FROM lmsysorg/sglang:dev + +# Create non-root user with specified UID and GID +# NOTE: Replace with your own UID and GID. This is a workaround from https://github.com/microsoft/vscode-remote-release/issues/49#issuecomment-489060908. +ARG HOST_UID=1003 +ARG HOST_GID=1003 +RUN groupadd -g $HOST_GID devuser && \ + useradd -m -u $HOST_UID -g $HOST_GID -s /bin/zsh devuser + +# Give devuser sudo access +RUN apt-get update && apt-get install -y sudo && \ + echo "devuser ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/devuser && \ + rm -rf /var/lib/apt/lists/* && \ + apt-get clean + +# Set up oh-my-zsh for devuser +RUN cp -r /root/.oh-my-zsh /home/devuser/.oh-my-zsh && \ + cp /root/.zshrc /home/devuser/.zshrc && \ + cp /root/.vimrc /home/devuser/.vimrc && \ + cp /root/.tmux.conf /home/devuser/.tmux.conf && \ + sed -i 's|/root/.oh-my-zsh|/home/devuser/.oh-my-zsh|g' /home/devuser/.zshrc && \ + chown -R devuser:devuser /home/devuser/ + +# Set workspace directory and ownership +WORKDIR /sgl-workspace/sglang +RUN chown -R devuser:devuser /sgl-workspace + +# Switch to devuser +USER devuser + +# Install uv +RUN curl -LsSf https://astral.sh/uv/install.sh | sh + +# Install rust +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.devcontainer/devcontainer.json b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.devcontainer/devcontainer.json new file mode 100644 index 00000000..338b10fe --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.devcontainer/devcontainer.json @@ -0,0 +1,30 @@ +{ + "name": "sglang", + "build": { + "dockerfile": "Dockerfile" + }, + "remoteUser": "devuser", + "customizations": { + "vscode": { + "extensions": [ + // Python development + "ms-python.python", + "charliermarsh.ruff", + // Rust development + "rust-lang.rust-analyzer", + "tamasfe.even-better-toml" + ] + } + }, + "forwardPorts": [], + "runArgs": [ + "--gpus", + "all" + ], + // The two lines below ensures that your local changes in the sglang + // repo is automatically synced to the sglang pip package installed + // in the dev docker container. You can remove / comment out these + // two lines if you prefer to sync code changes manually. + "workspaceMount": "source=${localWorkspaceFolder},target=/sgl-workspace/sglang,type=bind", + "workspaceFolder": "/sgl-workspace/sglang" +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.editorconfig b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.editorconfig new file mode 100644 index 00000000..030a7293 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.editorconfig @@ -0,0 +1,25 @@ +# https://editorconfig.org/ + +root = true + +[*] +charset = utf-8 +end_of_line = lf +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.{json,yaml,yml}] +indent_size = 2 + +[*.md] +indent_size = 2 +x-soft-wrap-text = true + +[*.rst] +indent_size = 4 +x-soft-wrap-text = true + +[Makefile] +indent_style = tab diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/CI_PERMISSIONS.json b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/CI_PERMISSIONS.json new file mode 100644 index 00000000..79d99a23 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/CI_PERMISSIONS.json @@ -0,0 +1,989 @@ +{ + "1pikachu": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "Alcanderian": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "AniZpZ": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "BBuf": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "BHZ-BER": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "ByronHsu": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "CaoE": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "CatherineSue": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "DarkSharpness": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "Edwardf0t1": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "FlamingoPg": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "FrankLeeeee": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "Fridge003": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "HaiShaw": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "HanHan009527": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "HandH1998": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "Hanrui-Wang": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "HydraQYH": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "JeremieMelo": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "Johnsonms": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "JustinTong0323": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "Kangyan-Zhou": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "LorrinWWW": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "Oasis-Git": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "Qiaolin-Yu": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "Qihang-Zhang": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "ShangmingCai": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "SimonCqk": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "TianQiLin666666": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "Ubospica": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "Valentine233": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "XiaotongJiang": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "XucSh": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "Ying1123": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "ZailiWang": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "ZhengdQin": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "acelyc111": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "adarshxs": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "airMeng": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "alisonshao": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "ayrnb": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "azhurkevich": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "b8zhong": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "blzheng": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "byjiang1996": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "cctry": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "ch-wan": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "chunyuan-w": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "cicirori": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "dougyster": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "elfiegg": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "fy1214": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "fzyzcjy": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "gaopengff": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "gongwei-130": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "gongy": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "guapisolo": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "guoyuhong": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "hanming-lu": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "harrisonlimh": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "hebiao064": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "hlu1": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "hnyls2002": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "huaiyuzh": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "huangtingwei9988": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "hubertlu-tw": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "hyhieu": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "hzh0425": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "iforgetmyname": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "ishandhanani": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "ispobock": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "jason-fxz": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "jhinpan": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "jianan-gu": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "jinleic": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "jinmingyi1998": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "kaixih": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "kevin85421": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "key4ng": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "kkHuang-amd": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "kssteven418": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "kushanam": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "lanking520": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "lifuhuang": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "liz-badada": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "merrymercy": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "mickqian": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "mingfeima": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "minleminzui": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "netanel-haber": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "nvcastet": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "ocss884": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "pansicheng": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "pavanimajety": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "ping1jing2": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "pranavm-nvidia": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "pyc96": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "qingquansong": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "qywu": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "rainj-me": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "ravi03071991": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "rkooo567": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "saienduri": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "sglang-bot": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "shaharmor98": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "shanyu-sys": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "shuaills": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "sleepcoo": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "slin1237": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "stmatengss": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "strgrb": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "sundar24295s": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "sunjiweiswift": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "sunxxuns": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "thecodingwizard": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "timmy-feng": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "trevor-m": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "vincentzed": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "wenscarl": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "whybeyoung": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "wisclmy0611": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "xiezhq-hermann": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "xutizhou": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "yanbing-j": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "yangsijia-serena": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "yhyang201": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "yilian49": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "yizhang2077": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "ykcombat": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "ynwang007": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "yuan-luo": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "yundai424": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "yyihuang": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "yzh119": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "zhaochenyang20": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + }, + "zhijian-liu": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "zhuzilin": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "zhyncs": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "zminglei": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + "can_rerun_stage": true + }, + "zyksir": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override", + "can_rerun_stage": true + }, + "zyzshishui": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "custom override", + "can_rerun_stage": true + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/CODEOWNERS b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/CODEOWNERS new file mode 100644 index 00000000..1f009435 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/CODEOWNERS @@ -0,0 +1,47 @@ +.github @merrymercy @Fridge003 @ispobock @Kangyan-Zhou +/docker @Fridge003 @ispobock @HaiShaw @ishandhanani +/docker/npu.Dockerfile @ping1jing2 @iforgetmyname +/python/pyproject.toml @merrymercy @Fridge003 @ispobock +/python/sglang/multimodal_gen @mickqian @yhyang201 +/python/sglang/srt/batch_invariant_ops @Fridge003 @hebiao064 +/python/sglang/srt/constrained @hnyls2002 @DarkSharpness +/python/sglang/srt/compilation @hebiao064 +/python/sglang/srt/disaggregation @ByronHsu @hnyls2002 @ShangmingCai +/python/sglang/srt/disaggregation/ascend @ping1jing2 @iforgetmyname +/python/sglang/srt/distributed @yizhang2077 @merrymercy @ch-wan +/python/sglang/srt/entrypoints @ispobock @CatherineSue @slin1237 @merrymercy @JustinTong0323 +/python/sglang/srt/entrypoints/grpc_server.py @CatherineSue @slin1237 +/python/sglang/srt/eplb @fzyzcjy @ch-wan +/python/sglang/srt/function_call @CatherineSue @JustinTong0323 +/python/sglang/srt/grpc @CatherineSue @slin1237 +/python/sglang/srt/layers @merrymercy @Ying1123 @Fridge003 @ispobock @HaiShaw @ch-wan @BBuf @Edwardf0t1 +/python/sglang/srt/layers/quantization @ch-wan @BBuf @Edwardf0t1 @FlamingoPg @AniZpZ +/python/sglang/srt/layers/attention/ascend_backend.py @ping1jing2 @iforgetmyname +/python/sglang/srt/lora @Ying1123 @Fridge003 @lifuhuang +/python/sglang/srt/managers @merrymercy @Ying1123 @hnyls2002 @xiezhq-hermann @zhyncs +/python/sglang/srt/mem_cache @merrymercy @Ying1123 @hnyls2002 @xiezhq-hermann +/python/sglang/srt/mem_cache/allocator_ascend.py @ping1jing2 @iforgetmyname +/python/sglang/srt/model_executor @merrymercy @Ying1123 @hnyls2002 @Fridge003 @ispobock +/python/sglang/srt/model_executor/npu_graph_runner.py @ping1jing2 @iforgetmyname +/python/sglang/srt/model_executor/piecewise_cuda_graph_runner.py @hebiao064 +/python/sglang/srt/multimodal @mickqian @JustinTong0323 @yhyang201 +/python/sglang/srt/speculative @Ying1123 @merrymercy @hnyls2002 +/sgl-kernel @zhyncs @ispobock @BBuf @yizhang2077 @merrymercy @FlamingoPg @HaiShaw +/sgl-router @slin1237 @CatherineSue +/sgl-router/benches @slin1237 +/sgl-router/bindings/python @CatherineSue @key4ng @slin1237 +/sgl-router/py_test @CatherineSue @key4ng +/sgl-router/src/config @slin1237 +/sgl-router/src/core @slin1237 +/sgl-router/src/data_connector @key4ng +/sgl-router/src/grpc_client @CatherineSue @slin1237 +/sgl-router/src/mcp @key4ng @slin1237 +/sgl-router/src/policies @slin1237 @ByronHsu +/sgl-router/src/proto @CatherineSue @slin1237 +/sgl-router/src/protocols @CatherineSue @key4ng +/sgl-router/src/reasoning_parser @CatherineSue +/sgl-router/src/routers @CatherineSue @key4ng @slin1237 +/sgl-router/src/tokenizer @slin1237 @CatherineSue +/sgl-router/src/tool_parser @slin1237 @CatherineSue +/test/srt/ascend @ping1jing2 @iforgetmyname +/test/srt/test_modelopt* @Edwardf0t1 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/FOLDER_README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/FOLDER_README.md new file mode 100644 index 00000000..ccbf94ec --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/FOLDER_README.md @@ -0,0 +1,12 @@ +# Maintenance Tools + +This folder contains tools and workflows for automating maintenance tasks. + +## CI Permissions + +`CI_PERMISSIONS.json` defines the CI permissions granted to each user. +Maintainers can directly edit the file to add entries with `"reason": "custom override"`. +Maintainers can also run `update_ci_permission.py` to update it with some auto rules (e.g., top contributors in the last 90 days get full permissions). + +## Others +- `MAINTAINER.md` defines the code maintenance model. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/ISSUE_TEMPLATE/1-bug-report.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/ISSUE_TEMPLATE/1-bug-report.yml new file mode 100644 index 00000000..6e3d9a83 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/ISSUE_TEMPLATE/1-bug-report.yml @@ -0,0 +1,35 @@ +name: 🐞 Bug report +description: Report a bug to help us reproduce and fix it. +title: "[Bug] " +labels: ['Bug'] + +body: +- type: checkboxes + attributes: + label: Checklist + options: + - label: I searched related issues but found no solution. + - label: The bug persists in the latest version. + - label: Issues without environment info and a minimal reproducible demo are hard to resolve and may receive no feedback. + - label: If this is not a bug report but a general question, please start a discussion at https://github.com/sgl-project/sglang/discussions. Otherwise, it will be closed. + - label: Please use English. Otherwise, it will be closed. +- type: textarea + attributes: + label: Describe the bug + description: A clear, concise description of the bug. + validations: + required: true +- type: textarea + attributes: + label: Reproduction + description: Command/script run and model used. + placeholder: Paste the command here. + validations: + required: true +- type: textarea + attributes: + label: Environment + description: Run `python3 -m sglang.check_env` and paste output here. Issues without this will be closed. + placeholder: Paste environment output here. + validations: + required: true diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/ISSUE_TEMPLATE/2-feature-request.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/ISSUE_TEMPLATE/2-feature-request.yml new file mode 100644 index 00000000..99f1f4d5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/ISSUE_TEMPLATE/2-feature-request.yml @@ -0,0 +1,23 @@ +name: 🚀 Feature request +description: Suggest an idea for this project +title: "[Feature] " + +body: +- type: checkboxes + attributes: + label: Checklist + options: + - label: If this is not a feature request but a general question, please start a discussion at https://github.com/sgl-project/sglang/discussions. Otherwise, it will be closed. + - label: Please use English. Otherwise, it will be closed. +- type: textarea + attributes: + label: Motivation + description: | + Clearly and concisely describe the feature's motivation. + validations: + required: true +- type: textarea + attributes: + label: Related resources + description: | + Provide official releases or third-party implementations if available. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/MAINTAINER.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/MAINTAINER.md new file mode 100644 index 00000000..634f11cd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/MAINTAINER.md @@ -0,0 +1,67 @@ +# SGLang Code Maintenance Model +This document describes the code maintenance model for the SGLang project. +Since SGLang is a large project involving multiple organizations and hardware platforms, we designed this model with the following goals: +- Ensure a responsive and smooth review process. +- Allow for fast iteration, so maintainers can sometimes bypass flaky CI tests for important PRs. + +## Role Descriptions +There are four roles in this maintenance model. Some are custom roles, while others are predefined by GitHub. + +- **Merge Oncall**: The person who drives the PR merge process. They have strong area-specific expertise and uphold a high bar for code quality. + - Permission: Merge PRs. Bypass branch protection rules if needed. + - Responsibility: Shepherd the merge of PRs assigned to their area. Revert or hotfix any issues related to their merge (especially if they bypass). +- **Codeowner**: The person who protects critical code. Without a bypass, each PR needs at least one Codeowner approval for each modified file protected by [CODEOWNERS](./CODEOWNERS). Please note that this role is not an honor but a significant responsibility because PRs cannot be merged without your approval (except when bypassed by a Merge Oncall). + - Permission: Approve PRs, allowing them to be merged without a bypass. + - Responsibility: Review PRs in a timely manner. +- **Write**: A person with write permission to the SGLang repo. + - Permission: Merge PRs if they have passed required tests and been approved by Codeowners. This role cannot bypass branch protection rules. + - Responsibility: Review and merge PRs in a timely manner. +- **CI Oncall**: A person who manages CI runners for specific hardware platforms. + - Permission: Add CI runners. + - Responsibility: Keep the CI runners up and running. + +__Note__: Difference between Merge Oncall and Codeowner +- The Merge Oncall is an active role held by someone who actively tries to help merge PRs and can bypass CI if needed. +- The Codeowner is a passive protection role provided by GitHub; it prevents accidental changes to critical code. +- The list of Merge Oncalls is attached below. The list of Codeowners is in the [CODEOWNERS](./CODEOWNERS) file. + +__Note__: The permissions to trigger CI tests are defined separately according to these [rules](https://docs.sglang.io/developer_guide/contribution_guide.html#how-to-trigger-ci-tests). + + +## Pull Request Merge Process +1. The author submits a pull request (PR) and fills out the PR checklist. +2. A bot assigns this PR to a Merge Oncall and @-mentions them. At the same time, GitHub will automatically request reviews from Codeowners. +3. Someone tags the PR with a `run-ci` label ([help](https://docs.sglang.io/developer_guide/contribution_guide.html#how-to-trigger-ci-tests)). Then the author can trigger CI by pushing new commits. +4. The Merge Oncall coordinates the review (e.g., asking people to review) and approves the PR; the Codeowners also approve the PR. If the assigned Merge Oncall is not responsive, the author can ping other related Merge Oncalls and Reviewers in the list below. +5. The code can now be merged: + - **Ideal case:** For each modified file, one Codeowner has approved the PR. The PR has also passed the required CI tests. Then, anyone with write permission can merge the PR. + - **Exception:** In cases where it is difficult to meet all requirements (due to flaky CI or slow responses), a Merge Oncall can bypass branch protection to merge the PR. + +If you meet any issues during the merge, you can discuss in [slack channels](https://slack.sglang.ai/): #dev, #pull-request, and #ci-cd-build-release. + +## The List of Merge Oncalls and Reviewers +The format is @github-username (Slack username). + +TODO: fill in the list. + +Now we have many Merge Oncalls mainly because the CI is flaky and the CODEOWNERS is too coarse-grained. +In the future, we hope the CI can be improved and we only need bypass rarely. After that, most Merge Oncalls can be converted back to Write and CODEOWNERS. + +This list is based on the current situation. If you or someone you know would like to take on more responsibility and are qualified, please ping @Lianmin Zheng and @Ying Sheng in the Slack channel. They will start a nomination and internal review process. + +## The List of CI Oncalls +The format is @github-username (Slack username). + +### NVIDIA GPUs +@merrymercy (Lianmin Zheng), @Kangyan-Zhou (Kangyan Zhou), @ch-wan (Cheng Wan), @HanHan009527 (hanhan), @ishandhanani (Ishan Dhanani), @key4ng (Keyang Ru), @slin1237 (Simo Lin), @ShangmingCai (Shangming Cai) + +### AMD GPUs +@saienduri (Sai Enduri), @HaiShaw (Henry HAI) + +### Intel CPU and XPU +@mingfeima (Mingfei Ma), @DiweiSun (Diwei Sun) + +### Ascend NPUs +@iforgetmyname (Even Zhou) + +This list is based on the current situation. If you or someone you know would like to donate machines for CI, they can serve as the CI oncalls for their machines. Please ping @Lianmin Zheng and @Ying Sheng in the Slack channel. They will start a nomination and internal review process. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/labeler.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/labeler.yml new file mode 100644 index 00000000..5151e5e2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/labeler.yml @@ -0,0 +1,110 @@ +# Configuration for the GitHub Labeler action +# Automatically adds labels to PRs based on the files changed + +# Router specific (Rust code in sgl-router) +model-gateway: + - changed-files: + - any-glob-to-any-file: 'sgl-router/**/*' + +# Kernel specific +sgl-kernel: + - changed-files: + - any-glob-to-any-file: 'sgl-kernel/**/*' + +# Documentation +documentation: + - changed-files: + - any-glob-to-any-file: + - '**/*.md' + - 'docs/**/*' + - 'README*' + +# Dependencies +dependencies: + - changed-files: + - any-glob-to-any-file: + - '**/requirements*.txt' + - '**/Cargo.toml' + - '**/Cargo.lock' + - '**/pyproject*.toml' + - '**/setup.py' + - '**/poetry.lock' + - '**/package.json' + - '**/package-lock.json' + +# Multi-modal +Multi-modal: + - changed-files: + - any-glob-to-any-file: + - '**/*multimodal*' + - '**/*vision*' + - '**/*vlm*' + +# Diffusion +diffusion: + - changed-files: + - any-glob-to-any-file: 'python/sglang/multimodal_gen/**/*' + +# LoRA +lora: + - changed-files: + - any-glob-to-any-file: + - '**/*lora*' + +# Quantization +quant: + - changed-files: + - any-glob-to-any-file: + - '**/*quant*' + - '**/*quantization*' + +# Speculative decoding +speculative-decoding: + - changed-files: + - any-glob-to-any-file: + - '**/*speculative*' + +# AMD specific +amd: + - changed-files: + - any-glob-to-any-file: + - '**/*amd*' + - '**/*rocm*' + +# NPU specific +npu: + - changed-files: + - any-glob-to-any-file: + - '**/*npu*' + - '**/*ascend*' + +# Blackwell +blackwell: + - changed-files: + - any-glob-to-any-file: + - '**/*nvfp4*' + - 'sgl-kernel/csrc/attention/cutlass_sm100_mla/**/*' + - 'python/sglang/srt/layers/attention/trtllm_mla_backend.py' + - 'python/sglang/srt/layers/attention/trtllm_mha_backend.py' + +# DeepSeek specific +deepseek: + - changed-files: + - any-glob-to-any-file: + - '**/*deepseek*' + +# HiCache +hicache: + - changed-files: + - any-glob-to-any-file: + - '**/*hicache*' + +# Deterministic +deterministic: + - changed-files: + - any-glob-to-any-file: 'python/sglang/srt/batch_invariant_ops/**/*' + +# Piecewise CUDA Graph +piecewise-cuda-graph: + - changed-files: + - any-glob-to-any-file: 'python/sglang/srt/compilation/**/*' diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/pull_request_template.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/pull_request_template.md new file mode 100644 index 00000000..7696bc51 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/pull_request_template.md @@ -0,0 +1,26 @@ + + +## Motivation + + + +## Modifications + + + +## Accuracy Tests + + + +## Benchmarking and Profiling + + + +## Checklist + +- [ ] Format your code according to the [Format code with pre-commit](https://docs.sglang.io/developer_guide/contribution_guide.html#format-code-with-pre-commit). +- [ ] Add unit tests according to the [Run and add unit tests](https://docs.sglang.io/developer_guide/contribution_guide.html#run-and-add-unit-tests). +- [ ] Update documentation according to [Write documentations](https://docs.sglang.io/developer_guide/contribution_guide.html#write-documentations). +- [ ] Provide accuracy and speed benchmark results according to [Test the accuracy](https://docs.sglang.io/developer_guide/contribution_guide.html#test-the-accuracy) and [Benchmark the speed](https://docs.sglang.io/developer_guide/contribution_guide.html#benchmark-the-speed). +- [ ] Follow the SGLang code style [guidance](https://docs.sglang.io/developer_guide/contribution_guide.html#code-style-guidance). +- [ ] Work with maintainers to merge your PR. See the [PR Merge Process](https://github.com/sgl-project/sglang/blob/main/.github/MAINTAINER.md#pull-request-merge-process) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/update_ci_permission.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/update_ci_permission.py new file mode 100644 index 00000000..2ed84667 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/update_ci_permission.py @@ -0,0 +1,196 @@ +""" +Update the CI permissions configuration file. + +This script updates the `CI_PERMISSIONS.json` file, which defines the CI permissions granted to each user. + +The format of `CI_PERMISSIONS.json` is as follows: + +{ + "username1": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor" + }, + "username2": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override" + } +} + +Permissions are assigned according to the following rules: + +1. Add the top 50 contributors from the last 90 days with full permissions, no cooldown, and the reason "top contributor". +2. Load all users from the existing `CI_PERMISSIONS.json` file and update their entries as follows: + - If a user is already covered by rule 1, skip that user. + - If the old reason of a user is "top contributor" but they are not in the current top contributors list, change their configuration to: + { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "cooldown_interval_minutes": 60, + "reason": "custom override" + } + - For all other cases, preserve the original configuration unchanged. +3. All other users receive no permissions and a 120-minute cooldown (they are omitted from the file). + +Usage: + export GH_TOKEN="your_github_token" + python3 update_ci_permission.py +""" + +import json +import os +from collections import Counter +from datetime import datetime, timedelta, timezone + +import requests + +# Configuration +REPO_OWNER = "sgl-project" +REPO_NAME = "sglang" +FILE_NAME = "CI_PERMISSIONS.json" +GH_TOKEN = os.getenv("GH_TOKEN") + +if not GH_TOKEN: + raise ValueError("Error: GH_TOKEN environment variable is not set.") + +HEADERS = { + "Authorization": f"Bearer {GH_TOKEN}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", +} + + +def github_api_get(endpoint, params=None): + """Helper to make paginated GitHub API requests.""" + results = [] + url = f"https://api.github.com/repos/{REPO_OWNER}/{REPO_NAME}/{endpoint}" + + while url: + response = requests.get(url, headers=HEADERS, params=params) + if response.status_code != 200: + print(f"Error fetching {url}: {response.status_code} {response.text}") + # If we fail to fetch, strictly return what we have or empty to avoid crashing logic + break + + data = response.json() + if isinstance(data, list): + results.extend(data) + else: + return data # Non-list response (not paginated usually) + + # Handle pagination + url = None + if "link" in response.headers: + links = response.headers["link"].split(", ") + for link in links: + if 'rel="next"' in link: + url = link[link.find("<") + 1 : link.find(">")] + params = None # Params are included in the next link + break + return results + + +def get_write_access_users(): + """Fetches users with push (write) or admin access.""" + print("Fetching collaborators with write access...") + # Note: This endpoint usually requires admin rights on the token. + collaborators = github_api_get("collaborators", params={"per_page": 100}) + + writers = set() + for col in collaborators: + perms = col.get("permissions", {}) + # Check for admin, maintain, or push rights + if perms.get("admin") or perms.get("maintain") or perms.get("push"): + writers.add(col["login"]) + + print(f"Found {len(writers)} users with write access.") + return writers + + +def get_top_contributors(days=90, limit=50): + """Fetches top contributors based on commit count in the last N days.""" + print(f"Fetching commits from the last {days} days...") + since_date = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat() + + # Fetch commits + commits = github_api_get("commits", params={"since": since_date, "per_page": 100}) + + author_counts = Counter() + for commit in commits: + # commit['author'] contains the GitHub user object (can be None if not linked) + if commit.get("author") and "login" in commit["author"]: + author_counts[commit["author"]["login"]] += 1 + + top_users = [user for user, _ in author_counts.most_common(limit)] + print(f"Found {len(top_users)} active contributors in the last {days} days.") + return set(top_users) + + +def load_existing_permissions(): + if os.path.exists(FILE_NAME): + try: + with open(FILE_NAME, "r") as f: + return json.load(f) + except json.JSONDecodeError: + print(f"Warning: {FILE_NAME} is invalid JSON. Starting fresh.") + return {} + + +def main(): + # Gather Data + try: + write_access_users = get_write_access_users() + except Exception as e: + print(f"Warning: Could not fetch collaborators (check token scope). Error: {e}") + write_access_users = set() + + top_contributors = get_top_contributors(days=90, limit=50) + old_permissions = load_existing_permissions() + + new_permissions = {} + + # Rule 1: Add Top 50 Contributors + for user in top_contributors: + new_permissions[user] = { + "can_tag_run_ci_label": True, + "can_rerun_failed_ci": True, + "cooldown_interval_minutes": 0, + "reason": "top contributor", + } + + # Rule 2: Process Existing Users (Merge Logic) + for user, config in old_permissions.items(): + if user in new_permissions: + # Already handled by Rule 1 or 2 + continue + + old_reason = config.get("reason", "") + + # If they fell off the top contributor list + if old_reason in ["top contributor"]: + new_permissions[user] = { + "can_tag_run_ci_label": True, + "can_rerun_failed_ci": True, + "cooldown_interval_minutes": 60, + "reason": "custom override", + } + else: + # Preserve custom overrides + new_permissions[user] = config + + # Save and Sort + # Sorting keys for cleaner diffs + sorted_permissions = dict(sorted(new_permissions.items())) + + with open(FILE_NAME, "w") as f: + json.dump(sorted_permissions, f, indent=4) + f.write("\n") # Add trailing newline + + print(f"Successfully updated {FILE_NAME}. Total users: {len(sorted_permissions)}") + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/auto-format.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/auto-format.yml new file mode 100644 index 00000000..7466572a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/auto-format.yml @@ -0,0 +1,71 @@ +name: Auto Format Code + +on: + pull_request: + types: [labeled] + +permissions: + contents: write + pull-requests: write + +jobs: + auto-format: + if: github.event.label.name == 'format' + runs-on: ubuntu-latest + steps: + - name: Checkout PR branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + token: ${{ secrets.GITHUB_TOKEN }} + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install pre-commit hook + run: | + python -m pip install pre-commit + pre-commit install + + - name: Run pre-commit to format code + run: SKIP=no-commit-to-branch pre-commit run --all-files + continue-on-error: true + + - name: Check for changes + id: check_changes + run: | + if [[ -n $(git status -s) ]]; then + echo "has_changes=true" >> $GITHUB_OUTPUT + else + echo "has_changes=false" >> $GITHUB_OUTPUT + fi + + - name: Commit and push changes + if: steps.check_changes.outputs.has_changes == 'true' + run: | + git config --local user.email "github-actions[bot]@users.noreply.github.com" + git config --local user.name "github-actions[bot]" + git add . + git commit -m "🤖 Auto-format code with isort, black, ruff, and clang-format" + git push + + - name: Remove format label + if: always() + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + try { + await github.rest.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + name: 'format' + }); + } catch (error) { + console.log('Label may have already been removed'); + } diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/auto-tune.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/auto-tune.yml new file mode 100644 index 00000000..0afc79bb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/auto-tune.yml @@ -0,0 +1,10 @@ +name: Auto tune + +on: + workflow_dispatch: + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/bot-bump-kernel-version-to-sglang.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/bot-bump-kernel-version-to-sglang.yml new file mode 100644 index 00000000..1621c924 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/bot-bump-kernel-version-to-sglang.yml @@ -0,0 +1,68 @@ +name: Bot Bump Kernel Version to SGLang + +on: + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + bump-kernel-version-to-sglang: + runs-on: ubuntu-latest + outputs: + branch_name: ${{ steps.set_output.outputs.branch_name }} + needs_sync: ${{ steps.check_sync.outputs.needs_sync }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install Python dependencies + run: | + pip install tomli + + - name: Check if sync is needed + id: check_sync + run: | + python scripts/release/check_kernel_version_to_sglang.py + + - name: Configure Git and branch + if: steps.check_sync.outputs.needs_sync == 'true' + id: set_output + run: | + git config user.name "sglang-bot" + git config user.email "sglang-bot@users.noreply.github.com" + RANDOM_SUFFIX=$(echo $RANDOM | md5sum | head -c 4) + KERNEL_VERSION="${{ steps.check_sync.outputs.kernel_version }}" + BRANCH_NAME="bot/bump-kernel-version-to-sglang-${KERNEL_VERSION}-${RANDOM_SUFFIX}" + git checkout -b "$BRANCH_NAME" + echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV + echo "KERNEL_VERSION=$KERNEL_VERSION" >> $GITHUB_ENV + echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT + + - name: Run kernel version bump script + if: steps.check_sync.outputs.needs_sync == 'true' + run: | + python scripts/release/bump_kernel_version_to_sglang.py + + - name: Commit and create PR + if: steps.check_sync.outputs.needs_sync == 'true' + env: + GH_TOKEN: ${{ secrets.GH_PAT_FOR_PULL_REQUEST }} + run: | + bash scripts/release/commit_and_pr_kernel_to_sglang.sh "$KERNEL_VERSION" "$BRANCH_NAME" + + run-nightly-tests: + needs: bump-kernel-version-to-sglang + if: needs.bump-kernel-version-to-sglang.outputs.needs_sync == 'true' + uses: ./.github/workflows/nightly-test-nvidia.yml + with: + ref: ${{ needs.bump-kernel-version-to-sglang.outputs.branch_name }} + secrets: inherit diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/bot-bump-kernel-version.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/bot-bump-kernel-version.yml new file mode 100644 index 00000000..91a808c6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/bot-bump-kernel-version.yml @@ -0,0 +1,50 @@ +name: Bot Bump Kernel Version + +on: + workflow_dispatch: + inputs: + new_version: + description: 'New sgl-kernel version (e.g., 0.3.12)' + required: true + type: string + +permissions: + contents: write + pull-requests: write + +jobs: + bump-kernel-version: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install Python dependencies + run: | + pip install tomli + + - name: Configure Git and branch + run: | + git config user.name "sglang-bot" + git config user.email "sglang-bot@users.noreply.github.com" + RANDOM_SUFFIX=$(echo $RANDOM | md5sum | head -c 4) + BRANCH_NAME="bot/bump-kernel-version-${{ github.event.inputs.new_version }}-${RANDOM_SUFFIX}" + git checkout -b "$BRANCH_NAME" + echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV + + - name: Run kernel version bump script + run: | + python scripts/release/bump_kernel_version.py "${{ github.event.inputs.new_version }}" + + - name: Commit and create PR + env: + GH_TOKEN: ${{ secrets.GH_PAT_FOR_PULL_REQUEST }} + run: | + bash scripts/release/commit_and_pr.sh "sgl-kernel" "${{ github.event.inputs.new_version }}" "$BRANCH_NAME" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/bot-bump-sglang-version.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/bot-bump-sglang-version.yml new file mode 100644 index 00000000..cbfd8e76 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/bot-bump-sglang-version.yml @@ -0,0 +1,61 @@ +name: Bot Bump SGLang Version + +on: + workflow_dispatch: + inputs: + new_version: + description: 'New SGLang version (e.g., 0.5.3 or 0.5.3rc0)' + required: true + type: string + +permissions: + contents: write + pull-requests: write + +jobs: + bump-sglang-version: + runs-on: ubuntu-latest + outputs: + branch_name: ${{ steps.set_output.outputs.branch_name }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install Python dependencies + run: | + pip install tomli + + - name: Configure Git and branch + id: set_output + run: | + git config user.name "sglang-bot" + git config user.email "sglang-bot@users.noreply.github.com" + RANDOM_SUFFIX=$(echo $RANDOM | md5sum | head -c 4) + BRANCH_NAME="bot/bump-sglang-version-${{ github.event.inputs.new_version }}-${RANDOM_SUFFIX}" + git checkout -b "$BRANCH_NAME" + echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV + echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT + + - name: Run SGLang version bump script + run: | + python scripts/release/bump_sglang_version.py "${{ github.event.inputs.new_version }}" + + - name: Commit and create PR + env: + GH_TOKEN: ${{ secrets.GH_PAT_FOR_PULL_REQUEST }} + run: | + bash scripts/release/commit_and_pr.sh "SGLang" "${{ github.event.inputs.new_version }}" "$BRANCH_NAME" + + run-nightly-tests: + needs: bump-sglang-version + uses: ./.github/workflows/nightly-test-nvidia.yml + with: + ref: ${{ needs.bump-sglang-version.outputs.branch_name }} + secrets: inherit diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/cancel-all-pending-pr-test-runs.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/cancel-all-pending-pr-test-runs.yml new file mode 100644 index 00000000..6217542e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/cancel-all-pending-pr-test-runs.yml @@ -0,0 +1,45 @@ +name: Cancel All Pending PR Test Runs + +on: + workflow_dispatch: + inputs: + workflows: + description: 'Space-separated list of workflow filenames to cancel' + required: true + type: string + default: 'pr-test.yml pr-test-xeon.yml' + +permissions: + actions: write # Needed to cancel runs + contents: read # Needed to read repo info + +jobs: + cancel-pending: + runs-on: ubuntu-latest + steps: + - name: Install GitHub CLI + run: sudo apt-get install -y gh jq + + - name: Cancel all pending/waiting runs for specified workflows + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + run: | + # Read the space-separated string from the input into a bash array + WORKFLOW_FILES=(${{ github.event.inputs.workflows }}) + + echo "Targeting ${#WORKFLOW_FILES[@]} workflow(s): ${{ github.event.inputs.workflows }}" + + for workflow_file in "${WORKFLOW_FILES[@]}"; do + echo "--- Checking workflow: $workflow_file ---" + gh run list \ + --repo "$REPO" \ + --workflow "$workflow_file" \ + --json databaseId,status \ + --limit 1000 \ + | jq -r '.[] | select(.status=="queued" or .status=="in_progress") | .databaseId' \ + | while read run_id; do + echo "Cancelling run ID: $run_id for workflow: $workflow_file" + gh run cancel "$run_id" --repo "$REPO" + done + done diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/cancel-pr-workflow-on-merge.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/cancel-pr-workflow-on-merge.yml new file mode 100644 index 00000000..535884ba --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/cancel-pr-workflow-on-merge.yml @@ -0,0 +1,22 @@ +name: Cancel PR Workflows on Merge + +on: + pull_request_target: + types: + - closed + +permissions: + actions: write + +jobs: + cancel: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.12.1 + with: + workflow_id: all + access_token: ${{ secrets.GITHUB_TOKEN }} + ignore_sha: true + pr_number: ${{ github.event.pull_request.number }} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/ci-failure-monitor.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/ci-failure-monitor.yml new file mode 100644 index 00000000..89e770ee --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/ci-failure-monitor.yml @@ -0,0 +1,64 @@ +name: CI Failure Monitor + +on: + schedule: + - cron: '*/30 * * * *' # Every 30 minutes + workflow_dispatch: + inputs: + limit: + description: 'Number of workflow runs to analyze (across all workflows)' + required: false + default: '1000' + type: string + threshold: + description: 'Alert threshold for consecutive failures' + required: false + default: '4' + type: string + +concurrency: + group: ci-failure-monitor-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + actions: read + +jobs: + failure-analysis: + if: github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.14' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install requests + + - name: Run Failure Analysis + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + PYTHONUNBUFFERED: 1 + PYTHONIOENCODING: utf-8 + run: | + cd scripts/ci_monitor + python ci_failures_analysis.py \ + --token $GITHUB_TOKEN \ + --limit ${{ inputs.limit || '1000' }} \ + --threshold ${{ inputs.threshold || '4' }} \ + --output ci_failure_analysis_$(date +%Y%m%d_%H%M%S).json + + - name: Upload Analysis Results + uses: actions/upload-artifact@v4 + with: + name: ci-failure-analysis-${{ github.run_number }} + path: | + scripts/ci_monitor/ci_failure_analysis_*.json + retention-days: 7 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/ci-monitor.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/ci-monitor.yml new file mode 100644 index 00000000..28a198a3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/ci-monitor.yml @@ -0,0 +1,111 @@ +name: CI Monitor + +on: + schedule: + - cron: '0 */12 * * *' # Every 12 hours for main analysis + workflow_dispatch: + inputs: + limit: + description: 'Number of CI runs to analyze' + required: false + default: '1000' + type: string + +concurrency: + group: ci-monitor-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: write + actions: read + +jobs: + ci-monitor: + if: github.repository == 'sgl-project/sglang'|| github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.9' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install requests matplotlib pandas + + - name: Run CI Analysis + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + PYTHONUNBUFFERED: 1 + PYTHONIOENCODING: utf-8 + run: | + cd scripts/ci_monitor + python ci_analyzer.py --token $GITHUB_TOKEN --limit ${{ inputs.limit || '1000' }} --output ci_analysis_$(date +%Y%m%d_%H%M%S).json + + - name: Run Nightly Test Analysis + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + PYTHONUNBUFFERED: 1 + PYTHONIOENCODING: utf-8 + run: | + cd scripts/ci_monitor + python ci_analyzer.py --token $GITHUB_TOKEN --mode nightly --days 2 --output nightly_analysis_$(date +%Y%m%d_%H%M%S).json + + - name: Run Performance Analysis + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + PYTHONUNBUFFERED: 1 + PYTHONIOENCODING: utf-8 + run: | + cd scripts/ci_monitor + python ci_analyzer_perf.py --token $GITHUB_TOKEN --limit ${{ inputs.limit || '1000' }} --output-dir performance_tables_$(date +%Y%m%d_%H%M%S) --upload-to-github + + - name: Upload Analysis Results + uses: actions/upload-artifact@v4 + with: + name: ci-analysis-results-${{ github.run_number }} + path: | + scripts/ci_monitor/ci_analysis_*.json + scripts/ci_monitor/nightly_analysis_*.json + scripts/ci_monitor/performance_tables_* + retention-days: 30 + + ci-monitor-balance: + needs: ci-monitor + if: github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.9' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install requests + + - name: Run Test Balance Analysis + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + PYTHONUNBUFFERED: 1 + PYTHONIOENCODING: utf-8 + run: | + cd scripts/ci_monitor + python ci_analyzer_balance.py --token $GITHUB_TOKEN --limit ${{ inputs.limit || '1000' }} --output test_balance_report_$(date +%Y%m%d_%H%M%S).json + + - name: Upload Balance Analysis Results + uses: actions/upload-artifact@v4 + with: + name: test-balance-results-${{ github.run_number }} + path: | + scripts/ci_monitor/test_balance_report_*.json + scripts/ci_monitor/test_balance_report_*.csv + retention-days: 30 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/close-inactive-issues.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/close-inactive-issues.yml new file mode 100644 index 00000000..048e6c44 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/close-inactive-issues.yml @@ -0,0 +1,96 @@ +name: Close Inactive Issues + +on: + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + +permissions: + issues: write + contents: read + +jobs: + close-inactive-issues: + if: github.repository == 'sgl-project/sglang' + runs-on: ubuntu-latest + steps: + - name: Check and close inactive issues + uses: actions/github-script@v6 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const sixtyDaysAgo = new Date(Date.now() - 60 * 24 * 60 * 60 * 1000); + + const [owner, repo] = process.env.GITHUB_REPOSITORY.split('/'); + console.log(`Owner: ${owner}, Repo: ${repo}`); + + async function fetchIssues(page = 1) { + console.log(`Fetching issues for ${owner}/${repo}, page ${page}`); + return await github.rest.issues.listForRepo({ + owner, + repo, + state: 'open', + sort: 'updated', + direction: 'asc', + per_page: 100, + page: page + }); + } + + async function processIssues() { + console.log('Starting to process issues'); + console.log(`Repository: ${owner}/${repo}`); + + let page = 1; + let hasMoreIssues = true; + while (hasMoreIssues) { + try { + const issues = await fetchIssues(page); + console.log(`Fetched ${issues.data.length} issues on page ${page}`); + + if (issues.data.length === 0) { + hasMoreIssues = false; + break; + } + + for (const issue of issues.data) { + // Skip if the issue has 'good first issue' label + if (issue.labels.some(label => label.name === 'good first issue')) { + console.log(`Skipping issue #${issue.number} as it's marked as 'good first issue'`); + continue; + } + if (new Date(issue.updated_at) < sixtyDaysAgo) { + try { + await github.rest.issues.update({ + owner, + repo, + issue_number: issue.number, + state: 'closed', + labels: [...issue.labels.map(l => l.name), 'inactive'] + }); + await github.rest.issues.createComment({ + owner, + repo, + issue_number: issue.number, + body: 'This issue has been automatically closed due to inactivity. Please feel free to reopen it if needed.' + }); + console.log(`Closed issue #${issue.number} due to inactivity.`); + } catch (error) { + console.error(`Failed to close issue #${issue.number}: ${error.message}`); + } + } else { + console.log(`Issue #${issue.number} is still active. Stopping processing.`); + hasMoreIssues = false; + break; + } + } + page += 1; + } catch (error) { + console.error(`Error fetching issues on page ${page}: ${error.message}`); + hasMoreIssues = false; + } + } + console.log('Finished processing issues'); + } + + await processIssues(); diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/execute-notebook.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/execute-notebook.yml new file mode 100644 index 00000000..52942c77 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/execute-notebook.yml @@ -0,0 +1,60 @@ +name: Execute Notebooks + +on: + pull_request: + branches: [ main ] + paths: + - "python/sglang/**" + - "docs/**" + workflow_dispatch: + + +concurrency: + group: execute-notebook-${{ github.ref }} + cancel-in-progress: true + + +jobs: + run-all-notebooks: + runs-on: 1-gpu-runner + if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-ci') + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + pip install -r docs/requirements.txt + apt-get update && apt-get install -y pandoc parallel retry + ln -sf "$(which python3)" /usr/bin/python + + - name: Setup Jupyter Kernel + run: | + python -m ipykernel install --user --name python3 --display-name "Python 3" + + - name: Execute notebooks + timeout-minutes: 40 + run: | + cd docs + make clean + make compile + + + notebook-finish: + needs: [ + run-all-notebooks + ] + runs-on: ubuntu-latest + steps: + - name: Check all dependent job statuses + run: | + results=(${{ join(needs.*.result, ' ') }}) + for result in "${results[@]}"; do + if [ "$result" = "failure" ] || [ "$result" = "cancelled" ]; then + echo "Job failed with result: $result" + exit 1 + fi + done + echo "All jobs completed successfully" + exit 0 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/labeler.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/labeler.yml new file mode 100644 index 00000000..5509bd41 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/labeler.yml @@ -0,0 +1,20 @@ +name: Auto Label PRs + +on: + pull_request_target: + types: [opened, synchronize, reopened] + +permissions: + contents: read + pull-requests: write + +jobs: + label: + runs-on: ubuntu-latest + steps: + - name: Auto-label by file changes + uses: actions/labeler@v5 + with: + repo-token: "${{ secrets.GITHUB_TOKEN }}" + configuration-path: .github/labeler.yml + sync-labels: false diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/lint.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/lint.yml new file mode 100644 index 00000000..56598470 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/lint.yml @@ -0,0 +1,50 @@ +name: Lint + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install pre-commit hook + run: | + python -m pip install pre-commit + pre-commit install + + - name: Run pre-commit checks + run: SKIP=no-commit-to-branch pre-commit run --all-files --show-diff-on-failure + + - name: Run sgl-kernel clang-format checks + uses: DoozyX/clang-format-lint-action@v0.18.1 + with: + source: sgl-kernel + extensions: h,c,cpp,hpp,cu,cuh,cc + clangFormatVersion: 18 + style: file + + - name: Check proto files are in sync + run: | + if ! diff -q python/sglang/srt/grpc/sglang_scheduler.proto sgl-router/src/proto/sglang_scheduler.proto; then + echo "❌ ERROR: Proto files are out of sync!" + echo "" + echo "The following files must be kept identical:" + echo " - python/sglang/srt/grpc/sglang_scheduler.proto" + echo " - sgl-router/src/proto/sglang_scheduler.proto" + echo "" + echo "Please ensure both files have the same content." + echo "" + echo "Differences:" + diff python/sglang/srt/grpc/sglang_scheduler.proto sgl-router/src/proto/sglang_scheduler.proto || true + exit 1 + fi diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/nightly-release-gateway.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/nightly-release-gateway.yml new file mode 100644 index 00000000..7b5226ba --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/nightly-release-gateway.yml @@ -0,0 +1,196 @@ +# Nightly release workflow for SGLang Model Gateway + +name: Nightly Release SGLang Model Gateway to PyPI + +on: + schedule: + # Run at 2 AM UTC every day + - cron: '0 2 * * *' + workflow_dispatch: # Allow manual trigger + +jobs: + build: + name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }}) + runs-on: ${{ matrix.os }}-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu, macos, windows] + target: [x86_64, aarch64] + manylinux: [auto] + include: + - os: ubuntu + platform: linux + - os: windows + ls: dir + target: x86_64 + python-architecture: x64 + interpreter: 3.9 3.10 3.11 3.12 3.13 + - os: macos + target: aarch64 + interpreter: 3.9 3.10 3.11 3.12 3.13 + - os: ubuntu + platform: linux + target: aarch64 + # musllinux + - os: ubuntu + platform: linux + target: x86_64 + manylinux: musllinux_1_1 + - os: ubuntu + platform: linux + target: aarch64 + manylinux: musllinux_1_1 + exclude: + - os: windows + target: aarch64 + + steps: + - uses: actions/checkout@v4 + with: + path: sglang-repo + + - name: Move sgl-router folder to root and delete sglang-repo + run: | + mv sglang-repo/sgl-router/* . + rm -rf sglang-repo + ls -alt + shell: bash + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + architecture: ${{ matrix.python-architecture || 'x64' }} + + - name: Modify version for nightly release + run: | + # Get current version from pyproject.toml + CURRENT_VERSION=$(python -c "import tomllib; print(tomllib.load(open('bindings/python/pyproject.toml', 'rb'))['project']['version'])" 2>/dev/null || python -c "import tomli; print(tomli.load(open('bindings/python/pyproject.toml', 'rb'))['project']['version'])") + # Create nightly version with date: e.g., 0.2.1.dev20250128 + NIGHTLY_VERSION="${CURRENT_VERSION}.dev$(date +%Y%m%d)" + echo "Nightly version: $NIGHTLY_VERSION" + + # Update pyproject.toml with nightly version (temporary, not committed) + sed -i.bak "s/version = \"${CURRENT_VERSION}\"/version = \"${NIGHTLY_VERSION}\"/" bindings/python/pyproject.toml + + # Verify the change + cat bindings/python/pyproject.toml | grep "^version" + shell: bash + + - name: Install twine and tomli + run: pip install -U twine tomli + + - name: Install protoc (macOS) + if: matrix.os == 'macos' + run: brew install protobuf + + - name: Install protoc (Windows) + if: matrix.os == 'windows' + run: choco install protoc -y + + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + working-directory: bindings/python + target: ${{ matrix.target }} + manylinux: ${{ matrix.manylinux || 'auto' }} + args: --release --out dist --features vendored-openssl --interpreter ${{ matrix.interpreter || '3.9 3.10 3.11 3.12 3.13 3.14' }} + rust-toolchain: stable + docker-options: -e CI -e CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc -e CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ + before-script-linux: | + # Install build dependencies (perl/make for vendored OpenSSL, protoc for gRPC) + if command -v yum &> /dev/null; then + yum update -y && yum install -y wget unzip gcc gcc-c++ perl-core make + # Install cross-compilation toolchain for aarch64 if needed + if [ "${{ matrix.target }}" = "aarch64" ]; then + yum install -y gcc-aarch64-linux-gnu gcc-c++-aarch64-linux-gnu || true + fi + elif command -v apt-get &> /dev/null; then + apt-get update && apt-get install -y wget unzip gcc g++ perl make + # Install cross-compilation toolchain for aarch64 if needed + if [ "${{ matrix.target }}" = "aarch64" ]; then + apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu || true + fi + fi + (cd /tmp && \ + wget https://github.com/protocolbuffers/protobuf/releases/download/v32.0/protoc-32.0-linux-x86_64.zip && \ + unzip protoc-32.0-linux-x86_64.zip -d /usr/local && \ + rm protoc-32.0-linux-x86_64.zip) + protoc --version + + - name: List built packages + run: ${{ matrix.ls || 'ls -lh' }} bindings/python/dist/ + + - name: Check packages + run: twine check --strict bindings/python/dist/* + + - uses: actions/upload-artifact@v4 + with: + name: packages-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.manylinux || 'auto' }} + path: bindings/python/dist/ + + build-sdist: + name: Build SDist + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + path: sglang-repo + + - name: Move sgl-router folder to root and delete sglang-repo + run: | + mv sglang-repo/sgl-router/* . + rm -rf sglang-repo + ls -alt + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Modify version for nightly release + run: | + # Get current version from pyproject.toml + CURRENT_VERSION=$(python -c "import tomllib; print(tomllib.load(open('bindings/python/pyproject.toml', 'rb'))['project']['version'])" 2>/dev/null || python -c "import tomli; print(tomli.load(open('bindings/python/pyproject.toml', 'rb'))['project']['version'])") + # Create nightly version with date: e.g., 0.2.1.dev20250128 + NIGHTLY_VERSION="${CURRENT_VERSION}.dev$(date +%Y%m%d)" + echo "Nightly version: $NIGHTLY_VERSION" + + # Update pyproject.toml with nightly version (temporary, not committed) + sed -i "s/version = \"${CURRENT_VERSION}\"/version = \"${NIGHTLY_VERSION}\"/" bindings/python/pyproject.toml + + # Verify the change + cat bindings/python/pyproject.toml | grep "^version" + + - name: Build SDist + uses: PyO3/maturin-action@v1 + with: + working-directory: bindings/python + command: sdist + args: --out dist + rust-toolchain: stable + + - uses: actions/upload-artifact@v4 + with: + name: sdist + path: bindings/python/dist/*.tar.gz + + upload: + name: Upload to TestPyPI + if: github.repository == 'sgl-project/sglang' # Ensure this job only runs for the sgl-project/sglang repository + needs: [build, build-sdist] + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v4 + with: + path: dist + merge-multiple: true + + - name: Upload to TestPyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.TEST_PYPI_TOKEN_ROUTER }} + run: | + pip install twine + twine upload --repository testpypi dist/* --verbose diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/nightly-test-amd.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/nightly-test-amd.yml new file mode 100644 index 00000000..932aafe8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/nightly-test-amd.yml @@ -0,0 +1,59 @@ +name: Nightly Test (AMD) + +on: + schedule: + - cron: '0 0 * * *' + push: + branches: + - main + paths: + - "python/sglang/version.py" + workflow_dispatch: + +concurrency: + group: nightly-test-amd-${{ github.ref }} + cancel-in-progress: true + +jobs: + nightly-test: + if: github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' + strategy: + matrix: + runner: [linux-mi300-gpu-2, linux-mi325-gpu-2-nightly] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Nightly Test + run: | + bash scripts/ci/amd_ci_exec.sh -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" python3 run_suite.py --suite nightly-amd --timeout-per-file 7200 + echo "$(> $GITHUB_STEP_SUMMARY + + check-all-jobs: + if: always() && (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch') + needs: + - nightly-test + runs-on: ubuntu-latest + steps: + - name: Check if any job failed + run: | + if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then + echo "One or more nightly test jobs failed" + exit 1 + fi + if [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then + echo "One or more nightly test jobs were cancelled" + exit 1 + fi + echo "All nightly test jobs passed" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/nightly-test-intel.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/nightly-test-intel.yml new file mode 100644 index 00000000..b32735dd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/nightly-test-intel.yml @@ -0,0 +1,26 @@ +name: Nightly Test (Intel) + +on: + schedule: + - cron: '0 0 * * *' + push: + branches: + - main + paths: + - "python/sglang/version.py" + workflow_dispatch: + +concurrency: + group: nightly-test-intel-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Placeholder for Intel GPU tests + # Add Intel-specific nightly test workflows here when available + + placeholder: + if: github.repository == 'sgl-project/sglang' + runs-on: ubuntu-latest + steps: + - name: Placeholder + run: echo "Intel nightly tests will be added here" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/nightly-test-nvidia.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/nightly-test-nvidia.yml new file mode 100644 index 00000000..37d36439 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/nightly-test-nvidia.yml @@ -0,0 +1,570 @@ +name: Nightly Test (Nvidia) + +on: + schedule: + - cron: '0 0 * * *' + push: + branches: + - main + paths: + - "python/sglang/version.py" + - "test/run_suite.py" + workflow_dispatch: + inputs: + job_filter: + description: 'Select which job to run (leave empty or "all" to run all jobs)' + required: false + type: choice + default: 'all' + options: + - 'all' + - 'nightly-test-general-1-gpu-runner' + - 'nightly-test-general-4-gpu-h100' + - 'nightly-test-general-8-gpu-h200' + - 'nightly-test-general-8-gpu-h20' + - 'nightly-test-text-accuracy-2-gpu-runner' + - 'nightly-test-text-perf-2-gpu-runner' + - 'nightly-test-vlm-accuracy-2-gpu-runner' + - 'nightly-test-vlm-perf-2-gpu-runner' + - 'nightly-test-multimodal-server-1-gpu' + - 'nightly-test-multimodal-server-2-gpu' + - 'nightly-test-perf-4-gpu-b200' + - 'nightly-test-perf-8-gpu-b200' + workflow_call: + inputs: + ref: + description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' + required: false + type: string + default: '' + job_filter: + description: 'Select which job to run (leave empty or "all" to run all jobs)' + required: false + type: string + default: 'all' + +concurrency: + group: nightly-test-nvidia-${{ github.ref }} + cancel-in-progress: true + +jobs: + # General tests - 1 GPU + nightly-test-general-1-gpu-runner: + if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-1-gpu-runner') + runs-on: 1-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 60 + run: | + cd test + python3 run_suite.py --hw cuda --suite nightly-1-gpu --nightly --continue-on-error + + # General tests - 4 GPU H100 + nightly-test-general-4-gpu-h100: + if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-4-gpu-h100') + runs-on: 4-gpu-h100 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 30 + run: | + cd test + python3 run_suite.py --hw cuda --suite nightly-4-gpu --nightly --continue-on-error + + # General tests - 8 GPU H200 + nightly-test-general-8-gpu-h200: + if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h200') + runs-on: 8-gpu-h200 + env: + RUNNER_LABELS: 8-gpu-h200 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 30 + env: + GPU_CONFIG: "8-gpu-h200" + run: | + cd test + python3 run_suite.py --hw cuda --suite nightly-8-gpu-h200 --nightly --continue-on-error + + - name: Run Qwen3-235B nightly performance test + timeout-minutes: 180 + env: + TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} + PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} + GPU_CONFIG: "8-gpu-h200" + run: | + rm -rf test/performance_profiles_qwen3_235b/ + cd test + python3 nightly/test_qwen3_235b_perf.py + + - name: Publish Qwen3-235B traces to storage repo + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + run: | + python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_qwen3_235b + + - name: Run Kimi-K2-Thinking nightly performance test + timeout-minutes: 180 + env: + TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} + PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} + GPU_CONFIG: "8-gpu-h200" + run: | + rm -rf test/performance_profiles_kimi_k2_thinking/ + cd test + python3 nightly/test_kimi_k2_thinking_perf.py + + - name: Publish Kimi-K2-Thinking traces to storage repo + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + run: | + python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_kimi_k2_thinking + + - name: Run GLM-4.6 nightly performance test + timeout-minutes: 180 + env: + TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} + PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} + GPU_CONFIG: "8-gpu-h200" + run: | + rm -rf test/performance_profiles_glm_4_6/ + cd test + IS_BLACKWELL=1 python3 nightly/test_glm_4_6_perf.py + + - name: Publish GLM-4.6 traces to storage repo + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + run: | + python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_glm_4_6 + + - name: Run MiniMax-M2 nightly performance test + timeout-minutes: 180 + env: + TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} + PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} + GPU_CONFIG: "8-gpu-h200" + run: | + rm -rf test/performance_profiles_minimax_m2/ + cd test + python3 nightly/test_minimax_m2_perf.py + + - name: Publish MiniMax-M2 traces to storage repo + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + run: | + python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_minimax_m2 + + # General tests - 8 GPU H20 + nightly-test-general-8-gpu-h20: + if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h20') + runs-on: 8-gpu-h20 + env: + SGLANG_CI_RDMA_ALL_DEVICES: "mlx5_1,mlx5_2,mlx5_3,mlx5_4" + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 30 + env: + GPU_CONFIG: "8-gpu-h20" + run: | + cd test + python3 run_suite.py --hw cuda --suite nightly-8-gpu-h20 --nightly --continue-on-error + + # Text model accuracy tests + nightly-test-text-accuracy-2-gpu-runner: + if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-text-accuracy-2-gpu-runner') + runs-on: 2-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + + - name: Run eval test for text models + timeout-minutes: 120 + run: | + cd test + python3 nightly/test_text_models_gsm8k_eval.py + + # Text model performance tests + nightly-test-text-perf-2-gpu-runner: + if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-text-perf-2-gpu-runner') + runs-on: 2-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + + - name: Run performance test for text models + timeout-minutes: 180 + env: + TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} + PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} + GPU_CONFIG: "2-gpu-runner" + run: | + cd test + rm -rf performance_profiles_text_models/ + python3 nightly/test_text_models_perf.py + + - name: Publish traces to storage repo + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + run: | + python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_text_models + + # VLM accuracy tests + nightly-test-vlm-accuracy-2-gpu-runner: + if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-vlm-accuracy-2-gpu-runner') + runs-on: 2-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + + - name: Run eval test for VLM models (fixed MMMU-100) + timeout-minutes: 240 + run: | + cd test + python3 nightly/test_vlms_mmmu_eval.py + + # VLM performance tests + nightly-test-vlm-perf-2-gpu-runner: + if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-vlm-perf-2-gpu-runner') + runs-on: 2-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + + - name: Run perf test for VLM models (MMMU) + timeout-minutes: 240 + env: + TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} + PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} + GPU_CONFIG: "2-gpu-runner" + run: | + cd test + rm -rf performance_profiles_vlms/ + python3 nightly/test_vlms_perf.py + + - name: Publish traces to storage repo + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + run: | + python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_vlms + + # diffusion performance tests + nightly-test-multimodal-server-1-gpu: + if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-multimodal-server-1-gpu') + runs-on: 1-gpu-runner + strategy: + fail-fast: false + max-parallel: 5 + matrix: + part: [0, 1] + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh diffusion + pip install slack_sdk + + - name: Run diffusion server tests + env: + SGLANG_DIFFUSION_SLACK_TOKEN: ${{ secrets.SGLANG_DIFFUSION_SLACK_TOKEN }} + GITHUB_RUN_ID: ${{ github.run_id }} + + timeout-minutes: 60 + run: | + cd python + python3 sglang/multimodal_gen/test/run_suite.py \ + --suite 1-gpu \ + --partition-id ${{ matrix.part }} \ + --total-partitions 2 + + + nightly-test-multimodal-server-2-gpu: + if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-multimodal-server-2-gpu') + runs-on: 2-gpu-runner + strategy: + fail-fast: false + max-parallel: 5 + matrix: + part: [0, 1] + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh diffusion + pip install slack_sdk + + - name: Run diffusion server tests + env: + SGLANG_DIFFUSION_SLACK_TOKEN: ${{ secrets.SGLANG_DIFFUSION_SLACK_TOKEN }} + GITHUB_RUN_ID: ${{ github.run_id }} + + timeout-minutes: 60 + run: | + cd python + python3 sglang/multimodal_gen/test/run_suite.py \ + --suite 2-gpu \ + --partition-id ${{ matrix.part }} \ + --total-partitions 2 + + # B200 Performance tests - 4 GPU + nightly-test-perf-4-gpu-b200: + if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-4-gpu-b200') + runs-on: 4-gpu-b200 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Install dependencies + run: | + IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 60 + run: | + cd test + python3 run_suite.py --hw cuda --suite nightly-4-gpu-b200 --nightly --continue-on-error + + # B200 Performance tests - 8 GPU + nightly-test-perf-8-gpu-b200: + if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-8-gpu-b200') + runs-on: 8-gpu-b200 + env: + RUNNER_LABELS: 8-gpu-b200 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Install dependencies + run: | + IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh + + - name: Run DeepSeek v3.1 nightly performance test + timeout-minutes: 180 + env: + TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} + PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} + GPU_CONFIG: "8-gpu-b200" + run: | + rm -rf test/performance_profiles_deepseek_v31/ + cd test + IS_BLACKWELL=1 python3 nightly/test_deepseek_v31_perf.py + + - name: Publish DeepSeek v3.1 traces to storage repo + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + run: | + python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_deepseek_v31 + + - name: Run DeepSeek v3.2 nightly performance test + if: always() + timeout-minutes: 180 + env: + TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} + PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} + GPU_CONFIG: "8-gpu-b200" + run: | + rm -rf test/performance_profiles_deepseek_v32/ + cd test + IS_BLACKWELL=1 python3 nightly/test_deepseek_v32_perf.py + + - name: Publish DeepSeek v3.2 traces to storage repo + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + run: | + python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_deepseek_v32 + + - name: Run Kimi-K2-Thinking nightly performance test + if: always() + timeout-minutes: 180 + env: + TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} + PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} + GPU_CONFIG: "8-gpu-b200" + run: | + rm -rf test/performance_profiles_kimi_k2_thinking/ + cd test + IS_BLACKWELL=1 python3 nightly/test_kimi_k2_thinking_perf.py + + - name: Publish Kimi-K2-Thinking traces to storage repo + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + run: | + python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_kimi_k2_thinking + + - name: Run Qwen3-235B nightly performance test + if: always() + timeout-minutes: 180 + env: + TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} + PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} + GPU_CONFIG: "8-gpu-b200" + run: | + rm -rf test/performance_profiles_qwen3_235b/ + cd test + IS_BLACKWELL=1 python3 nightly/test_qwen3_235b_perf.py + + - name: Publish Qwen3-235B traces to storage repo + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + run: | + python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_qwen3_235b + + - name: Run GLM-4.6 nightly performance test + if: always() + timeout-minutes: 180 + env: + TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} + PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} + GPU_CONFIG: "8-gpu-b200" + run: | + rm -rf test/performance_profiles_glm_4_6/ + cd test + IS_BLACKWELL=1 python3 nightly/test_glm_4_6_perf.py + + - name: Publish GLM-4.6 traces to storage repo + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + run: | + python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_glm_4_6 + + - name: Run MiniMax-M2 nightly performance test + if: always() + timeout-minutes: 180 + env: + TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} + PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} + GPU_CONFIG: "8-gpu-b200" + run: | + rm -rf test/performance_profiles_minimax_m2/ + cd test + IS_BLACKWELL=1 python3 nightly/test_minimax_m2_perf.py + + - name: Publish MiniMax-M2 traces to storage repo + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + run: | + python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_minimax_m2 + + # Final check job + check-all-jobs: + if: github.repository == 'sgl-project/sglang' && always() + needs: + - nightly-test-general-1-gpu-runner + - nightly-test-general-4-gpu-h100 + - nightly-test-general-8-gpu-h200 + - nightly-test-general-8-gpu-h20 + - nightly-test-text-accuracy-2-gpu-runner + - nightly-test-text-perf-2-gpu-runner + - nightly-test-vlm-accuracy-2-gpu-runner + - nightly-test-vlm-perf-2-gpu-runner + - nightly-test-multimodal-server-1-gpu + - nightly-test-multimodal-server-2-gpu + - nightly-test-perf-4-gpu-b200 + - nightly-test-perf-8-gpu-b200 + runs-on: ubuntu-latest + steps: + - name: Check if any job failed + run: | + if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then + echo "One or more nightly test jobs failed" + exit 1 + fi + if [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then + echo "One or more nightly test jobs were cancelled" + exit 1 + fi + echo "All nightly test jobs passed" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/open-pr-copy-from-oss.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/open-pr-copy-from-oss.yml new file mode 100644 index 00000000..05af6ea4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/open-pr-copy-from-oss.yml @@ -0,0 +1,28 @@ +name: Open A PR to Copy Code From OSS + +on: + workflow_dispatch: + # schedule: + # - cron: '0 10 * * *' + +permissions: + contents: write + +jobs: + copy: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: 'main' + + - name: Install GitHub CLI (if not present) + run: | + bash scripts/code_sync/install_github_cli.sh + + - name: Copy from OSS code + env: + GH_TOKEN: ${{ secrets.PAT_FOR_CODE_SYNC_FROM_LIANMIN }} + run: | + python3 scripts/code_sync/copy_from_oss.py diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/open-pr-copy-to-oss.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/open-pr-copy-to-oss.yml new file mode 100644 index 00000000..b3bb6aae --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/open-pr-copy-to-oss.yml @@ -0,0 +1,31 @@ +name: Open A PR to Copy Diff To OSS + +on: + workflow_dispatch: + inputs: + commit_sha: + description: 'The commit SHA to copy. Defaults to LAST to copy the latest commit.' + required: false + default: 'LAST' + +permissions: + contents: write + +jobs: + copy: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install GitHub CLI (if not present) + run: | + bash scripts/code_sync/install_github_cli.sh + + - name: Copy to OSS code + env: + GH_TOKEN: ${{ secrets.PAT_FOR_CODE_SYNC_FROM_LIANMIN }} + run: | + python3 scripts/code_sync/copy_to_oss.py --commit ${{ github.event.inputs.commit_sha }} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-benchmark-rust.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-benchmark-rust.yml new file mode 100644 index 00000000..0b98b774 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-benchmark-rust.yml @@ -0,0 +1,321 @@ +name: PR Benchmark (SMG Components) + +on: + push: + branches: [ main ] + paths: + - "sgl-router/**" + pull_request: + branches: [ main ] + paths: + - "sgl-router/**" + workflow_dispatch: + +concurrency: + group: pr-benchmark-rust-${{ github.ref }} + cancel-in-progress: true + +env: + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + +permissions: + contents: read + pull-requests: write + issues: write + +jobs: + # Quick check job that always runs on PRs + benchmark-compile-check: + name: Benchmark Compilation Check + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_rust.sh + + - name: Configure sccache + uses: mozilla-actions/sccache-action@v0.0.9 + with: + version: "v0.10.0" + + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: sgl-router + # Share cache across all benchmark jobs + shared-key: "rust-cache" + # Save cache even on failure + save-if: true + cache-all-crates: true + cache-on-failure: true + + - name: Check benchmarks compile + run: | + source "$HOME/.cargo/env" + cd sgl-router/ + cargo check --benches + + - name: Show sccache stats + if: always() + run: sccache --show-stats + + # Full benchmark jobs that only run with label or on main branch + benchmark-request-processing: + name: Request Processing Benchmark + if: | + github.repository == 'sgl-project/sglang' && + (github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + (contains(github.event.pull_request.labels.*.name, 'router-benchmark') && + contains(github.event.pull_request.labels.*.name, 'run-ci'))) + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + # Fetch enough history for baseline comparison + fetch-depth: 100 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_rust.sh + + - name: Configure sccache + uses: mozilla-actions/sccache-action@v0.0.9 + with: + version: "v0.10.0" + + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: sgl-router + # Share cache across all benchmark jobs + shared-key: "rust-cache" + cache-all-crates: true + cache-on-failure: true + # Save cache even on failure + save-if: true + + - name: Run request processing benchmark + timeout-minutes: 30 + run: | + source "$HOME/.cargo/env" + cd sgl-router/ + # Try to use sccache, but disable if it fails + if command -v sccache &> /dev/null; then + echo "Testing sccache availability..." + # Try to start sccache and check if it works + export RUSTC_WRAPPER=sccache + export SCCACHE_GHA_ENABLED="true" + if sccache --start-server 2>/dev/null && sccache --show-stats 2>/dev/null; then + echo "sccache is working, using it for compilation" + else + echo "sccache failed to start, falling back to regular cargo" + unset RUSTC_WRAPPER + unset SCCACHE_GHA_ENABLED + fi + else + echo "sccache not available, using regular cargo" + fi + # Run only the summary benchmark for quick validation in PRs + cargo bench --bench request_processing -- benchmark_summary --exact + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v4 + with: + name: request-processing-results-${{ github.sha }} + path: | + sgl-router/target/criterion/benchmark_summary/ + retention-days: 30 + + - name: Show sccache stats + if: always() + run: sccache --show-stats + + benchmark-tokenizer: + name: Tokenizer Benchmark + if: | + github.repository == 'sgl-project/sglang' && + (github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + (contains(github.event.pull_request.labels.*.name, 'router-benchmark') && + contains(github.event.pull_request.labels.*.name, 'run-ci'))) + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 100 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_rust.sh + + - name: Configure sccache + uses: mozilla-actions/sccache-action@v0.0.9 + with: + version: "v0.10.0" + + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: sgl-router + # Share cache across all benchmark jobs + shared-key: "rust-cache" + cache-all-crates: true + cache-on-failure: true + # Save cache even on failure + save-if: true + + - name: Run tokenizer benchmark + timeout-minutes: 30 + run: | + source "$HOME/.cargo/env" + cd sgl-router/ + # Try to use sccache, but disable if it fails + if command -v sccache &> /dev/null; then + echo "Testing sccache availability..." + # Try to start sccache and check if it works + export RUSTC_WRAPPER=sccache + export SCCACHE_GHA_ENABLED="true" + if sccache --start-server 2>/dev/null && sccache --show-stats 2>/dev/null; then + echo "sccache is working, using it for compilation" + else + echo "sccache failed to start, falling back to regular cargo" + unset RUSTC_WRAPPER + unset SCCACHE_GHA_ENABLED + fi + else + echo "sccache not available, using regular cargo" + fi + cargo bench --bench tokenizer_benchmark + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v4 + with: + name: tokenizer-results-${{ github.sha }} + path: | + sgl-router/target/criterion/tokenizer*/ + retention-days: 30 + + benchmark-tool-parser: + name: Tool Parser Benchmark + if: | + github.repository == 'sgl-project/sglang' && + (github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + (contains(github.event.pull_request.labels.*.name, 'router-benchmark') && + contains(github.event.pull_request.labels.*.name, 'run-ci'))) + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 100 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_rust.sh + + - name: Configure sccache + uses: mozilla-actions/sccache-action@v0.0.9 + with: + version: "v0.10.0" + + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: sgl-router + # Share cache across all benchmark jobs + shared-key: "rust-cache" + cache-all-crates: true + cache-on-failure: true + # Save cache even on failure + save-if: true + + - name: Run tool parser benchmark + timeout-minutes: 30 + run: | + source "$HOME/.cargo/env" + cd sgl-router/ + # Try to use sccache, but disable if it fails + if command -v sccache &> /dev/null; then + echo "Testing sccache availability..." + # Try to start sccache and check if it works + export RUSTC_WRAPPER=sccache + export SCCACHE_GHA_ENABLED="true" + if sccache --start-server 2>/dev/null && sccache --show-stats 2>/dev/null; then + echo "sccache is working, using it for compilation" + else + echo "sccache failed to start, falling back to regular cargo" + unset RUSTC_WRAPPER + unset SCCACHE_GHA_ENABLED + fi + else + echo "sccache not available, using regular cargo" + fi + cargo bench --bench tool_parser_benchmark + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v4 + with: + name: tool-parser-results-${{ github.sha }} + path: | + sgl-router/target/criterion/tool_parser*/ + retention-days: 30 + + - name: Show sccache stats + if: always() + run: sccache --show-stats + + benchmark-summary: + name: Benchmark Summary + needs: [benchmark-request-processing, benchmark-tokenizer, benchmark-tool-parser] + if: always() && (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') + runs-on: ubuntu-latest + steps: + - name: Download all benchmark results + uses: actions/download-artifact@v4 + with: + pattern: '*-results-${{ github.sha }}' + path: benchmark-results + + - name: Generate summary + run: | + echo "## Benchmark Results Summary" > summary.md + echo "" >> summary.md + echo "### Request Processing" >> summary.md + if [ -d "benchmark-results/request-processing-results-${{ github.sha }}" ]; then + echo "✅ Completed" >> summary.md + else + echo "❌ Failed or skipped" >> summary.md + fi + echo "" >> summary.md + echo "### Tokenizer" >> summary.md + if [ -d "benchmark-results/tokenizer-results-${{ github.sha }}" ]; then + echo "✅ Completed" >> summary.md + else + echo "❌ Failed or skipped" >> summary.md + fi + echo "" >> summary.md + echo "### Tool Parser" >> summary.md + if [ -d "benchmark-results/tool-parser-results-${{ github.sha }}" ]; then + echo "✅ Completed" >> summary.md + else + echo "❌ Failed or skipped" >> summary.md + fi + cat summary.md + + - name: Upload summary + uses: actions/upload-artifact@v4 + with: + name: benchmark-summary-${{ github.sha }} + path: summary.md + retention-days: 30 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-gate.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-gate.yml new file mode 100644 index 00000000..cffc8f5d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-gate.yml @@ -0,0 +1,173 @@ +on: + workflow_call: + inputs: + require-run-ci: + description: "Whether the PR must have the run-ci label" + type: boolean + default: true + cool-down-minutes: + description: "Cooldown period in minutes for low-permission users; 0 disables rate limiting" + type: number + default: 120 + +jobs: + pr-gate: + # 1. for commits on main: no gating needed + # 2. for workflow_dispatch: this can only be triggered by users with write access + runs-on: ubuntu-latest + steps: + - name: Fetch latest PR info + if: github.event_name == 'pull_request' + id: pr + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const pr = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number + }); + core.setOutput("labels", JSON.stringify(pr.data.labels.map(l => l.name))); + core.setOutput("draft", pr.data.draft); + core.setOutput("user", pr.data.user.login); + + - name: Log PR info + if: github.event_name == 'pull_request' + run: | + echo "===== PR Info =====" + echo "PR Event: ${{ github.event_name }}" + echo "PR Labels: ${{ steps.pr.outputs.labels }}" + echo "PR Draft: ${{ steps.pr.outputs.draft }}" + echo "PR User: ${{ steps.pr.outputs.user }}" + echo "Require run-ci: ${{ inputs.require-run-ci }}" + echo "Cool down minutes: ${{ inputs.cool-down-minutes }}" + echo "===================" + + - name: Block draft PR + if: github.event_name == 'pull_request' && fromJson(steps.pr.outputs.draft) + run: | + echo "PR is draft. Blocking CI." + exit 1 + + - name: Require run-ci label (optional) + if: github.event_name == 'pull_request' && inputs.require-run-ci == true + run: | + labels='${{ steps.pr.outputs.labels }}' + if [[ "${{ contains(fromJson(steps.pr.outputs.labels), 'run-ci') }}" == "false" ]]; then + echo "Missing required label 'run-ci'." + exit 1 + fi + + - name: Enforce rate limit for low-permission actors (optional) + if: github.event_name == 'pull_request' && inputs.cool-down-minutes > 0 + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const DEFAULT_MINUTES = Number("${{ inputs.cool-down-minutes }}"); + const owner = context.repo.owner; + const repo = context.repo.repo; + const eventName = context.eventName; + const curRun = await github.rest.actions.getWorkflowRun({ + owner, repo, run_id: context.runId + }); + let triggeringActor = curRun.data.triggering_actor?.login || context.actor; + if (triggeringActor === "github-actions[bot]") { + triggeringActor = `${{ steps.pr.outputs.user }}`; + core.info( + `triggering_actor is github-actions[bot]; substituting PR author '${triggeringActor}'.` + ); + } + + async function hasHighPermission(username) { + try { + const { data } = await github.rest.repos.getCollaboratorPermissionLevel({ owner, repo, username }); + const perm = data.permission || 'none'; + return perm === 'write' || perm === 'maintain' || perm === 'admin'; + } catch (e) { + if (e.status === 404 || e.status === 403) return false; + throw e; + } + } + + if (await hasHighPermission(triggeringActor)) { + core.info(`Triggering user '${triggeringActor}' has high permission. No rate limit applied.`); + return; + } + + let effectiveCooldownMinutes = DEFAULT_MINUTES; + let perUserCooldownMinutes = null; + + try { + const contentResp = await github.rest.repos.getContent({ + owner, + repo, + path: ".github/CI_PERMISSIONS.json", + ref: "main", + }); + + if (!Array.isArray(contentResp.data) && contentResp.data && "content" in contentResp.data) { + const raw = Buffer.from( + contentResp.data.content, + contentResp.data.encoding || "base64" + ).toString(); + const ciPermissions = JSON.parse(raw); + + const userPerm = ciPermissions[triggeringActor]; + if (userPerm && typeof userPerm.cooldown_interval_minutes === "number") { + perUserCooldownMinutes = userPerm.cooldown_interval_minutes; + core.info( + `Per-user cooldown for '${triggeringActor}' from CI_PERMISSIONS.json: ${perUserCooldownMinutes} minutes.` + ); + } else { + core.info(`No per-user cooldown found for '${triggeringActor}' in CI_PERMISSIONS.json.`); + } + } else { + core.info("CI_PERMISSIONS.json content response is not a file; skipping per-user cooldown."); + } + } catch (e) { + core.info(`CI_PERMISSIONS.json not found or unreadable: ${e.message}. Using default rate limit only.`); + } + + if (perUserCooldownMinutes !== null) { + effectiveCooldownMinutes = Math.min(effectiveCooldownMinutes, perUserCooldownMinutes); + } + + if (effectiveCooldownMinutes <= 0) { + core.info( + `Effective cooldown for '${triggeringActor}' is 0 minutes; no rate limit enforced for this user.` + ); + return; + } + + const cutoff = new Date(Date.now() - effectiveCooldownMinutes * 60 * 1000); + core.info( + `Checking for workflow runs since ${cutoff.toISOString()} (last ${effectiveCooldownMinutes} minutes) for event '${eventName}'.` + ); + + const { data } = await github.rest.actions.listWorkflowRuns({ + owner, + repo, + workflow_id: 'pr-test.yml', + event: eventName, + per_page: 100, + }); + + const runs = data.workflow_runs || []; + const recentFound = runs.find((run) => { + if (String(run.id) === String(context.runId)) return false; + if (new Date(run.created_at) < cutoff) return false; + return (run.actor?.login === triggeringActor) || (run.triggering_actor?.login === triggeringActor); + }); + + if (recentFound) { + core.setFailed( + `User '${triggeringActor}' already triggered '${context.workflow}' via '${eventName}' at ${recentFound.created_at}. ` + + `Please wait ${effectiveCooldownMinutes} minutes before triggering again.` + ); + } else { + core.info( + `No recent runs detected for '${triggeringActor}' within the last ${effectiveCooldownMinutes} minutes; proceeding.` + ); + } diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-amd.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-amd.yml new file mode 100644 index 00000000..8a48f7cb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-amd.yml @@ -0,0 +1,451 @@ +name: PR Test (AMD) + +on: + push: + branches: [ main ] + paths: + - "python/**" + - "!python/sglang/multimodal_gen/**" + - "scripts/ci/**" + - "test/**" + - "sgl-kernel/**" + - ".github/workflows/pr-test-amd.yml" + pull_request: + branches: [ main ] + paths: + - "python/**" + - "!python/sglang/multimodal_gen/**" + - "scripts/ci/**" + - "test/**" + - "sgl-kernel/**" + - ".github/workflows/pr-test-amd.yml" + workflow_dispatch: + +concurrency: + group: pr-test-amd-${{ github.ref }} + cancel-in-progress: true + +jobs: + call-gate: + uses: ./.github/workflows/pr-gate.yml + secrets: inherit + check-changes: + needs: [call-gate] + runs-on: ubuntu-latest + outputs: + main_package: ${{ steps.filter.outputs.main_package }} + sgl_kernel: ${{ steps.filter.outputs.sgl_kernel }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Detect file changes + id: filter + uses: dorny/paths-filter@v3 + with: + filters: | + main_package: + - "python/**" + - "scripts/ci/**" + - "test/**" + - ".github/workflows/pr-test-amd.yml" + sgl_kernel: + - "sgl-kernel/**" + + # =============================================== sgl-kernel ==================================================== + sgl-kernel-unit-test-amd: + needs: [check-changes] + if: needs.check-changes.outputs.sgl_kernel == 'true' + strategy: + fail-fast: false + matrix: + runner: [linux-mi300-gpu-1] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Ensure VRAM is clear + run: bash scripts/ensure_vram_clear.sh rocm + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd_ci_install_dependency.sh + + - name: Run test + timeout-minutes: 14 + run: | + docker exec -w /sglang-checkout/sgl-kernel/tests ci_sglang python3 -m pytest test_moe_align.py + docker exec -w /sglang-checkout/sgl-kernel/tests ci_sglang python3 -m pytest test_moe_topk_softmax.py + docker exec -w /sglang-checkout/sgl-kernel/tests/speculative ci_sglang python3 -m pytest test_eagle_utils.py + docker exec -w /sglang-checkout/sgl-kernel/tests ci_sglang python3 -m pytest test_apply_token_bitmask_inplace.py + docker exec -w /sglang-checkout/sgl-kernel/tests ci_sglang python3 -m pytest test_activation.py + docker exec -w /sglang-checkout/sgl-kernel/tests ci_sglang python3 -m pytest test_kvcacheio.py + + # =============================================== primary ==================================================== + + stage-a-test-1-amd: + needs: [check-changes] + if: always() && !failure() && !cancelled() && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + strategy: + fail-fast: false + matrix: + runner: [linux-mi300-gpu-1] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Ensure VRAM is clear + run: bash scripts/ensure_vram_clear.sh rocm + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd_ci_install_dependency.sh + + - name: Run test + timeout-minutes: 10 + run: | + docker exec -w /sglang-checkout/test ci_sglang python3 run_suite.py --hw amd --suite stage-a-test-1 + + unit-test-backend-1-gpu-amd: + needs: [check-changes, stage-a-test-1-amd] + if: always() && !failure() && !cancelled() && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + strategy: + fail-fast: false + matrix: + runner: [linux-mi300-gpu-1] + part: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Ensure VRAM is clear + run: bash scripts/ensure_vram_clear.sh rocm + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Run test + timeout-minutes: 30 + run: | + bash scripts/ci/amd_ci_exec.sh python3 run_suite.py --suite per-commit-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 12 + + unit-test-backend-2-gpu-amd: + needs: [check-changes, stage-a-test-1-amd] + if: always() && !failure() && !cancelled() && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + strategy: + fail-fast: false + matrix: + runner: [linux-mi300-gpu-2] + part: [0, 1] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Ensure VRAM is clear + run: bash scripts/ensure_vram_clear.sh rocm + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Run test + timeout-minutes: 30 + run: | + bash scripts/ci/amd_ci_exec.sh python3 run_suite.py --suite per-commit-2-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 + + unit-test-backend-8-gpu-amd: + needs: [check-changes, unit-test-backend-2-gpu-amd] + if: always() && !failure() && !cancelled() && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + env: + RUNNER_LABELS: linux-mi300-gpu-8 + strategy: + fail-fast: false + matrix: + runner: [linux-mi300-gpu-8] + part: [0, 1, 2] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Ensure VRAM is clear + run: bash scripts/ensure_vram_clear.sh rocm + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Run test + timeout-minutes: 60 + run: | + bash scripts/ci/amd_ci_exec.sh python3 run_suite.py --suite per-commit-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 3600 + + performance-test-1-gpu-part-1-amd: + needs: [check-changes, stage-a-test-1-amd] + if: always() && !failure() && !cancelled() && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + strategy: + fail-fast: false + matrix: + runner: [linux-mi300-gpu-1] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Ensure VRAM is clear + run: bash scripts/ensure_vram_clear.sh rocm + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Benchmark single latency + timeout-minutes: 20 + run: | + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_one_batch.TestBenchOneBatch.test_bs1_small + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_one_batch.TestBenchOneBatch.test_bs1_default + + - name: Benchmark online latency + timeout-minutes: 15 + run: | + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_online_latency_default + + - name: Benchmark offline throughput + timeout-minutes: 15 + run: | + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_default + + - name: Benchmark offline throughput (Non-streaming, small batch size) + timeout-minutes: 15 + run: | + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_non_stream_small_batch_size + + performance-test-1-gpu-part-2-amd: + needs: [check-changes, stage-a-test-1-amd] + if: always() && !failure() && !cancelled() && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + strategy: + fail-fast: false + matrix: + runner: [linux-mi300-gpu-1] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Ensure VRAM is clear + run: bash scripts/ensure_vram_clear.sh rocm + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Benchmark offline throughput (w/o RadixAttention) + timeout-minutes: 15 + run: | + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_without_radix_cache + + - name: Benchmark offline throughput (w/ Triton) + timeout-minutes: 15 + run: | + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_with_triton_attention_backend + + - name: Benchmark offline throughput (w/ FP8) + timeout-minutes: 15 + run: | + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_default_fp8 + + performance-test-2-gpu-amd: + needs: [check-changes, unit-test-backend-2-gpu-amd] + if: always() && !failure() && !cancelled() && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + strategy: + fail-fast: false + matrix: + runner: [linux-mi300-gpu-2] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Ensure VRAM is clear + run: bash scripts/ensure_vram_clear.sh rocm + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Benchmark dummy grok (TP=2) + timeout-minutes: 30 + run: | + bash scripts/ci/amd_ci_exec.sh python3 models/test_dummy_grok_models.py + + - name: Benchmark single latency (TP=2) + timeout-minutes: 25 + run: | + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_one_batch.TestBenchOneBatch.test_moe_tp2_bs1 + + - name: Benchmark single latency + torch.compile (TP=2) + timeout-minutes: 25 + run: | + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_one_batch.TestBenchOneBatch.test_torch_compile_tp2_bs1 + + - name: Benchmark offline throughput (TP=2) + timeout-minutes: 25 + run: | + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_moe_offline_throughput_default + + - name: Benchmark offline throughput (w/o RadixAttention) (TP=2) + timeout-minutes: 25 + run: | + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_moe_offline_throughput_without_radix_cache + + accuracy-test-1-gpu-amd: + needs: [check-changes, stage-a-test-1-amd] + if: always() && !failure() && !cancelled() && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + strategy: + fail-fast: false + matrix: + runner: [linux-mi300-gpu-1] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Ensure VRAM is clear + run: bash scripts/ensure_vram_clear.sh rocm + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Evaluate Accuracy + timeout-minutes: 30 + run: | + bash scripts/ci/amd_ci_exec.sh -e SGLANG_USE_AITER=0 python3 test_eval_accuracy_large.py + bash scripts/ci/amd_ci_exec.sh python3 test_eval_fp8_accuracy.py + bash scripts/ci/amd_ci_exec.sh python3 models/test_qwen_models.py + + accuracy-test-2-gpu-amd: + needs: [check-changes, accuracy-test-1-gpu-amd] + if: always() && !failure() && !cancelled() && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + strategy: + fail-fast: false + matrix: + runner: [linux-mi300-gpu-2] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Ensure VRAM is clear + run: bash scripts/ensure_vram_clear.sh rocm + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Evaluate accuracy (TP=2) + timeout-minutes: 30 + run: | + bash scripts/ci/amd_ci_exec.sh -e SGLANG_USE_AITER=0 python3 test_moe_eval_accuracy_large.py + + pr-test-amd-finish: + needs: + [ + call-gate, + check-changes, + + sgl-kernel-unit-test-amd, + + stage-a-test-1-amd, + unit-test-backend-1-gpu-amd, + unit-test-backend-2-gpu-amd, + unit-test-backend-8-gpu-amd, + performance-test-1-gpu-part-1-amd, + performance-test-1-gpu-part-2-amd, + performance-test-2-gpu-amd, + accuracy-test-1-gpu-amd, + accuracy-test-2-gpu-amd, + ] + if: always() + runs-on: ubuntu-latest + steps: + - name: Check all dependent job statuses + run: | + # Convert the 'needs' context to a JSON string + json_needs='${{ toJson(needs) }}' + + # Get a list of all job names from the JSON keys + job_names=$(echo "$json_needs" | jq -r 'keys_unsorted[]') + + for job in $job_names; do + # For each job, extract its result + result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result') + + # Print the job name and its result + echo "$job: $result" + + # Check for failure or cancellation and exit if found + if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then + echo "The above jobs failed." + exit 1 + fi + done + + # If the loop completes, all jobs were successful + echo "All jobs completed successfully" + exit 0 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-npu.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-npu.yml new file mode 100644 index 00000000..af19e754 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-npu.yml @@ -0,0 +1,205 @@ +name: PR Test (NPU) + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: + +concurrency: + group: pr-test-npu-${{ github.ref }} + cancel-in-progress: true + +jobs: + # ==================== Check Changes ==================== # + check-changes: + runs-on: ubuntu-latest + outputs: + main_package: ${{ steps.filter.outputs.main_package }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Detect file changes + id: filter + uses: dorny/paths-filter@v3 + with: + filters: | + main_package: + - "python/sglang/!(multimodal_gen)/**" + - "python/*.toml" + - "scripts/ci/npu_ci_install_dependency.sh" + - "test/srt/ascend/**" + - ".github/workflows/pr-test-npu.yml" + + # ==================== PR Gate ==================== # + pr-gate: + needs: check-changes + if: needs.check-changes.outputs.main_package == 'true' + uses: ./.github/workflows/pr-gate.yml + secrets: inherit + + per-commit-1-npu-a2: + needs: [check-changes, pr-gate] + if: needs.check-changes.outputs.main_package == 'true' + runs-on: linux-arm64-npu-1 + container: + image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.3.rc1-910b-ubuntu22.04-py3.11 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + # speed up by using infra cache services + CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" + sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list + pip config set global.index-url http://${CACHING_URL}/pypi/simple + pip config set global.extra-index-url "https://pypi.tuna.tsinghua.edu.cn/simple" + pip config set global.trusted-host "${CACHING_URL} pypi.tuna.tsinghua.edu.cn" + + bash scripts/ci/npu_ci_install_dependency.sh 910b + # copy required file from our daily cache + cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp + # copy download through proxy + curl -o /tmp/test.jsonl -L https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl + + - name: Run test + timeout-minutes: 60 + env: + SGLANG_USE_MODELSCOPE: true + SGLANG_IS_IN_CI: true + HF_ENDPOINT: https://hf-mirror.com + TORCH_EXTENSIONS_DIR: /tmp/torch_extensions + PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True" + STREAMS_PER_DEVICE: 32 + run: | + export PATH="/usr/local/Ascend/8.3.RC1/compiler/bishengir/bin:${PATH}" + cd test/srt + python3 run_suite.py --suite per-commit-1-npu-a2 + + per-commit-2-npu-a2: + needs: [check-changes, pr-gate] + if: needs.check-changes.outputs.main_package == 'true' + runs-on: linux-arm64-npu-2 + strategy: + fail-fast: true + matrix: + part: [0, 1, 2] + container: + image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.3.rc1-910b-ubuntu22.04-py3.11 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + # speed up by using infra cache services + CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" + sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list + pip config set global.index-url http://${CACHING_URL}/pypi/simple + pip config set global.extra-index-url "https://pypi.tuna.tsinghua.edu.cn/simple" + pip config set global.trusted-host "${CACHING_URL} pypi.tuna.tsinghua.edu.cn" + + bash scripts/ci/npu_ci_install_dependency.sh 910b + # copy required file from our daily cache + cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp + # copy download through proxy + curl -o /tmp/test.jsonl -L https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl + + - name: Run test + timeout-minutes: 60 + env: + SGLANG_USE_MODELSCOPE: true + SGLANG_IS_IN_CI: true + HF_ENDPOINT: https://hf-mirror.com + TORCH_EXTENSIONS_DIR: /tmp/torch_extensions + PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True" + STREAMS_PER_DEVICE: 32 + run: | + export PATH="/usr/local/Ascend/8.3.RC1/compiler/bishengir/bin:${PATH}" + cd test/srt + python3 run_suite.py --suite per-commit-2-npu-a2 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 + + per-commit-4-npu-a2: + needs: [check-changes, pr-gate] + if: needs.check-changes.outputs.main_package == 'true' + runs-on: linux-arm64-npu-4 + container: + image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.3.rc1-910b-ubuntu22.04-py3.11 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + # speed up by using infra cache services + CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" + sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list + pip config set global.index-url http://${CACHING_URL}/pypi/simple + pip config set global.extra-index-url "https://pypi.tuna.tsinghua.edu.cn/simple" + pip config set global.trusted-host "${CACHING_URL} pypi.tuna.tsinghua.edu.cn" + + bash scripts/ci/npu_ci_install_dependency.sh 910b + # copy required file from our daily cache + cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp + # copy download through proxy + curl -o /tmp/test.jsonl -L https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl + + - name: Run test + timeout-minutes: 60 + env: + SGLANG_USE_MODELSCOPE: true + SGLANG_IS_IN_CI: true + HF_ENDPOINT: https://hf-mirror.com + TORCH_EXTENSIONS_DIR: /tmp/torch_extensions + PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True" + STREAMS_PER_DEVICE: 32 + run: | + export PATH="/usr/local/Ascend/8.3.RC1/compiler/bishengir/bin:${PATH}" + cd test/srt + python3 run_suite.py --suite per-commit-4-npu-a2 --timeout-per-file 3600 + + per-commit-16-npu-a3: + needs: [check-changes, pr-gate] + if: needs.check-changes.outputs.main_package == 'true' + runs-on: linux-aarch64-a3-16 + strategy: + fail-fast: true + matrix: + part: [0, 1] + container: + image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.3.rc1-a3-ubuntu22.04-py3.11 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + # speed up by using infra cache services + CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" + sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list + pip config set global.index-url http://${CACHING_URL}/pypi/simple + pip config set global.extra-index-url "https://pypi.tuna.tsinghua.edu.cn/simple" + pip config set global.trusted-host "${CACHING_URL} pypi.tuna.tsinghua.edu.cn" + + bash scripts/ci/npu_ci_install_dependency.sh a3 + # copy required file from our daily cache + cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp + # copy download through proxy + curl -o /tmp/test.jsonl -L https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl + + - name: Run test + timeout-minutes: 60 + env: + SGLANG_USE_MODELSCOPE: true + SGLANG_IS_IN_CI: true + HF_ENDPOINT: https://hf-mirror.com + TORCH_EXTENSIONS_DIR: /tmp/torch_extensions + PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True" + STREAMS_PER_DEVICE: 32 + run: | + export PATH="/usr/local/Ascend/8.3.RC1/compiler/bishengir/bin:${PATH}" + cd test/srt + python3 run_suite.py --suite per-commit-16-npu-a3 --timeout-per-file 3600 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-pd-router.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-pd-router.yml new file mode 100644 index 00000000..d95e7153 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-pd-router.yml @@ -0,0 +1,607 @@ +name: PR Benchmark (SMG PD Router) + +on: + push: + branches: [ main ] + paths: + - 'python/sglang/srt/disaggregation/**' + - 'scripts/ci/ci_start_disaggregation_servers.sh' + - 'sgl-router/**' + pull_request: + branches: [ main ] + types: [opened, synchronize, reopened, labeled] + paths: + - 'python/sglang/srt/disaggregation/**' + - 'scripts/ci/ci_start_disaggregation_servers.sh' + - 'sgl-router/**' + workflow_dispatch: + +concurrency: + group: test-disaggregation-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write + issues: write + +jobs: + test-disaggregation: + if: | + github.event_name != 'pull_request' || + (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run-ci') && contains(github.event.pull_request.labels.*.name, 'router-benchmark')) || + (github.event.action == 'labeled' && github.event.label.name == 'run-ci' && contains(github.event.pull_request.labels.*.name, 'router-benchmark')) || + (github.event.action == 'labeled' && github.event.label.name == 'router-benchmark' && contains(github.event.pull_request.labels.*.name, 'run-ci')) + runs-on: [8-gpu-h200-oracle] + timeout-minutes: 45 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 10 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.12' + + - name: Setup Rust + run: | + bash scripts/ci/ci_install_rust.sh + + - name: Cache Rust dependencies + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + sgl-router/target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('sgl-router/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo- + + - name: Cache pip dependencies + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('python/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Validate environment + run: | + echo "=== System Validation ===" + nvidia-smi + echo "GPU count: $(nvidia-smi -L | wc -l)" + if [ $(nvidia-smi -L | wc -l) -lt 8 ]; then + echo "Error: This test requires at least 8 GPUs" + exit 1 + fi + + echo "=== GPU Process Check ===" + # Fail fast if any GPU compute processes are active + if command -v nvidia-smi >/dev/null 2>&1; then + # Try to query compute apps first (preferred and concise) + gpu_procs=$(nvidia-smi --query-compute-apps=pid,process_name,gpu_uuid --format=csv,noheader 2>/dev/null | sed '/^$/d' || true) + + # Fallback to detailed PIDS report if the query returns nothing but there might still be processes + if [ -z "$gpu_procs" ]; then + gpu_procs=$(nvidia-smi -q -d PIDS 2>/dev/null | awk '/Processes/{flag=1;next}/^$/{flag=0}flag' | sed '/^\s*Processes:/d' | sed '/^\s*$/d' || true) + fi + + if [ -n "$gpu_procs" ]; then + echo "Error: Found active GPU processes using the device(s):" + echo "$gpu_procs" + exit 1 + else + echo "No active GPU compute processes detected." + fi + else + echo "Error: nvidia-smi not found; skipping GPU process check." + exit 1 + fi + + echo "=== RDMA Validation ===" + if ! command -v ibv_devices >/dev/null 2>&1; then + echo "Error: InfiniBand tools not found" + exit 1 + fi + + # Check for active IB devices + found_active_device=false + for device in mlx5_{0..11}; do + if ibv_devinfo $device >/dev/null 2>&1; then + state=$(ibv_devinfo $device | grep "state:" | head -1 | awk '{print $2}') + if [[ "$state" == "PORT_ACTIVE" ]]; then + echo "✓ Found active device: $device" + found_active_device=true + break + fi + fi + done + + if [ "$found_active_device" = false ]; then + echo "Error: No active IB devices found" + echo "Available devices:" + ibv_devices || true + exit 1 + fi + + echo "=== Model Validation ===" + if [ ! -d "/raid/models/meta-llama/Llama-3.1-8B-Instruct" ]; then + echo "Error: Model not found" + ls -la /raid/models/ || echo "No models directory" + exit 1 + fi + echo "✓ Model found" + + - name: Install SGLang dependencies + run: | + echo "Installing SGLang with all extras..." + python3 -m pip --no-cache-dir install --upgrade pip + python3 -m pip --no-cache-dir install torch==2.9.1 torchvision torchaudio --index-url https://download.pytorch.org/whl/test/cu128 + python3 -m pip --no-cache-dir install -e "python[all]" --break-system-packages + python3 -m pip --no-cache-dir install mooncake-transfer-engine==0.3.7.post2 + python3 -m pip --no-cache-dir install --user --force-reinstall genai-bench==0.0.2 + + - name: Build and install sgl-router + run: | + source "$HOME/.cargo/env" + echo "Building sgl-router..." + cd sgl-router/bindings/python + pip install maturin + maturin build --release --out dist --features vendored-openssl + pip install --force-reinstall dist/*.whl + + - name: Start disaggregation servers + id: start_servers + run: | + echo "Starting disaggregation servers..." + READY_FILE=".disagg_ready" + rm -f "$READY_FILE" + DISAGG_READY_FILE="$READY_FILE" bash scripts/ci/ci_start_disaggregation_servers.sh & + SERVER_PID=$! + echo "server_pid=$SERVER_PID" >> $GITHUB_OUTPUT + + # Wait until script signals readiness (8/8 healthy) or timeout + TIMEOUT=300 + ELAPSED=0 + while [ $ELAPSED -lt $TIMEOUT ]; do + if [ -f "$READY_FILE" ]; then + echo "✓ All disaggregation servers are healthy (signal detected)" + break + fi + if ! ps -p $SERVER_PID > /dev/null; then + echo "Error: server bootstrap script exited prematurely" + exit 1 + fi + sleep 5 + ELAPSED=$((ELAPSED + 5)) + done + if [ $ELAPSED -ge $TIMEOUT ]; then + echo "❌ Timeout waiting for disaggregation servers to be healthy" + exit 1 + fi + + echo "✓ Servers started (PID: $SERVER_PID)" + + + - name: Test all policies sequentially + timeout-minutes: 30 + run: | + POLICIES=("random" "round_robin" "cache_aware" "power_of_two") + BASE_URL="http://127.0.0.9:8000" + + # Free commonly used ports for router and metrics + echo "Freeing ports 29000 (metrics) and 8000 (API), if in use..." + fuser -k -n tcp 29000 2>/dev/null || true + fuser -k -n tcp 8000 2>/dev/null || true + sleep 1 + + for policy in "${POLICIES[@]}"; do + echo "" + echo "==================================================" + echo "Testing policy: $policy" + echo "==================================================" + + # Free ports before starting router + fuser -k -n tcp 29000 2>/dev/null || true + fuser -k -n tcp 8000 2>/dev/null || true + + # Start router with the current policy + echo "Starting router with policy: $policy..." + RUST_BACKTRACE=1 python3 -m sglang_router.launch_router \ + --pd-disaggregation \ + --policy "$policy" \ + --prefill http://127.0.0.1:30001 9001 \ + --prefill http://127.0.0.2:30002 9002 \ + --prefill http://127.0.0.3:30003 9003 \ + --prefill http://127.0.0.4:30004 9004 \ + --decode http://127.0.0.5:30005 \ + --decode http://127.0.0.6:30006 \ + --decode http://127.0.0.7:30007 \ + --decode http://127.0.0.8:30008 \ + --host 127.0.0.9 \ + --log-level warn \ + --port 8000 & + ROUTER_PID=$! + + # Wait for router to become healthy + echo "Waiting for router to become healthy..." + TIMEOUT=60 + ELAPSED=0 + while [ $ELAPSED -lt $TIMEOUT ]; do + if curl --connect-timeout 5 --silent http://127.0.0.9:8000 > /dev/null 2>&1; then + echo "✓ Router is reachable" + break + fi + if ! ps -p $ROUTER_PID > /dev/null; then + echo "Error: Router process died" + exit 1 + fi + sleep 5 + ELAPSED=$((ELAPSED + 5)) + done + + if [ $ELAPSED -ge $TIMEOUT ]; then + echo "Error: Router health check timeout" + kill $ROUTER_PID 2>/dev/null || true + exit 1 + fi + + # Test API functionality + echo "Testing API completions for $policy..." + response=$(curl -s -X POST "$BASE_URL/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer test-token" \ + -d '{ + "model": "/raid/models/meta-llama/Llama-3.1-8B-Instruct", + "messages": [ + {"role": "user", "content": "Write a Python function to calculate fibonacci numbers recursively"} + ], + "stream": false, + "max_completion_tokens": 100 + }') + + if echo "$response" | jq -e '.choices[0].message.content' > /dev/null 2>&1; then + echo "✓ API test passed for $policy" + else + echo "✗ API test failed for $policy: $response" + kill $ROUTER_PID 2>/dev/null || true + exit 1 + fi + + # Test streaming + echo "Testing streaming API for $policy..." + stream_response=$(timeout 30 curl -s -X POST "$BASE_URL/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer test-token" \ + -d '{ + "model": "/raid/models/meta-llama/Llama-3.1-8B-Instruct", + "messages": [ + {"role": "user", "content": "Count from 1 to 5"} + ], + "stream": true, + "max_completion_tokens": 50 + }') + + if echo "$stream_response" | grep -q "data:"; then + echo "✓ Streaming API test passed for $policy" + else + echo "✗ Streaming API test failed for $policy" + kill $ROUTER_PID 2>/dev/null || true + exit 1 + fi + + # Run genai-bench benchmark + echo "Running genai-bench for $policy..." + genai-bench benchmark \ + --api-backend openai \ + --api-base "http://127.0.0.9:8000" \ + --api-key "dummy-token" \ + --api-model-name "/raid/models/meta-llama/Llama-3.1-8B-Instruct" \ + --model-tokenizer /raid/models/meta-llama/Llama-3.1-8B-Instruct \ + --task text-to-text \ + --num-concurrency 64 \ + --traffic-scenario "D(8000,2000)" \ + --max-requests-per-run 1000 \ + --max-time-per-run 5 \ + --experiment-folder-name "benchmark_${policy}" \ + --experiment-base-dir "." + + # Find the actual experiment folder + actual_folder=$(find . -maxdepth 1 -name "benchmark_${policy}" -type d | head -1) + + if [ -n "$actual_folder" ]; then + # Extract metrics from the Excel summary or JSON files + summary_file="$actual_folder"/*_summary.xlsx + json_files=$(find "$actual_folder" -name "*.json" | grep -v experiment_metadata) + + echo "Genai-bench results saved in: $actual_folder" + + # Extract mean values and validate performance thresholds + echo "📊 Extracting performance metrics for $policy..." + + # Find JSON files excluding experiment metadata + json_files=$(find "$actual_folder" -name "*.json" | grep -v experiment_metadata) + + if [ -n "$json_files" ]; then + # Extract metrics using jq and validate against loose thresholds + for json_file in $json_files; do + echo "Processing: $(basename "$json_file")" + + # Extract mean values for performance validation + ttft_mean=$(jq -r '.aggregated_metrics.stats.ttft.mean' "$json_file") + e2e_latency_mean=$(jq -r '.aggregated_metrics.stats.e2e_latency.mean' "$json_file") + input_throughput_mean=$(jq -r '.aggregated_metrics.stats.input_throughput.mean' "$json_file") + output_throughput_mean=$(jq -r '.aggregated_metrics.stats.output_throughput.mean' "$json_file") + + echo " TTFT mean: ${ttft_mean}s" + echo " E2E Latency mean: ${e2e_latency_mean}s" + echo " Input Throughput mean: ${input_throughput_mean} tokens/s" + echo " Output Throughput mean: ${output_throughput_mean} tokens/s" + + # Set mean thresholds (allowing for reasonable variance) + # These can be adjusted based on your performance requirements + ttft_threshold=4.7 # Max 4.7 seconds for mean TTFT + e2e_latency_threshold=35.0 # Max 35.0 seconds for mean E2E latency + input_throughput_threshold=10000 # Min 02000 tokens/s for mean input throughput + output_throughput_threshold=68 # Min 68 tokens/s for mean output throughput + + + # Validate mean thresholds + validation_passed=true + + if (( $(echo "$ttft_mean > $ttft_threshold" | bc -l) )); then + echo "❌ TTFT validation failed: $ttft_mean > $ttft_threshold" + validation_passed=false + fi + + if (( $(echo "$e2e_latency_mean > $e2e_latency_threshold" | bc -l) )); then + echo "❌ E2E Latency validation failed: $e2e_latency_mean > $e2e_latency_threshold" + validation_passed=false + fi + + if (( $(echo "$input_throughput_mean < $input_throughput_threshold" | bc -l) )); then + echo "❌ Input Throughput validation failed: $input_throughput_mean < $input_throughput_threshold" + validation_passed=false + fi + + if (( $(echo "$output_throughput_mean < $output_throughput_threshold" | bc -l) )); then + echo "❌ Output Throughput validation failed: $output_throughput_mean < $output_throughput_threshold" + validation_passed=false + fi + + if [ "$validation_passed" = true ]; then + echo "✅ Performance validation passed for $policy" + else + echo "❌ Performance validation failed for $policy" + kill $ROUTER_PID 2>/dev/null || true + exit 1 + fi + done + + echo "✓ Genai-bench completed successfully for $policy" + echo "📊 Detailed metrics and plots available in: $actual_folder" + else + echo "✗ Benchmark failed for $policy: No JSON results found" + kill $ROUTER_PID 2>/dev/null || true + exit 1 + fi + else + echo "✗ Benchmark failed for $policy: Experiment folder not found" + kill $ROUTER_PID 2>/dev/null || true + exit 1 + fi + + # Stop router before testing next policy + echo "Stopping router for $policy..." + # First try graceful shutdown + kill $ROUTER_PID 2>/dev/null || true + + # Wait up to 5 seconds for graceful shutdown + for i in {1..5}; do + if ! ps -p $ROUTER_PID > /dev/null 2>&1; then + echo "Router stopped gracefully" + break + fi + sleep 1 + done + + # Force kill if still running + if ps -p $ROUTER_PID > /dev/null 2>&1; then + echo "Force killing router..." + kill -9 $ROUTER_PID 2>/dev/null || true + fi + + # Short delay to ensure port is released + sleep 2 + + echo "✓ Completed testing for $policy" + done + + echo "" + echo "✅ All policies tested successfully!" + + + - name: Upload benchmark results + if: success() + uses: actions/upload-artifact@v4 + with: + name: genai-bench-results-all-policies + path: benchmark_**/ + + - name: Cleanup servers + if: always() + run: | + if [ -n "${{ steps.start_servers.outputs.server_pid }}" ]; then + pkill -P ${{ steps.start_servers.outputs.server_pid }} || true + kill ${{ steps.start_servers.outputs.server_pid }} || true + fi + pkill -f "sglang.launch_server" || true + sleep 5 + remaining=$(ps aux | grep -c "sglang.launch_server" || echo "0") + echo "Cleanup completed. Remaining processes: $remaining" + + summarize-benchmarks: + needs: test-disaggregation + runs-on: ubuntu-latest + if: success() + + steps: + - name: Install jq + run: sudo apt-get update && sudo apt-get install -y jq bc + + - name: Download benchmark results + uses: actions/download-artifact@v4 + with: + name: genai-bench-results-all-policies + + - name: List downloaded contents + run: | + echo "Contents after download:" + ls -la + find . -name "benchmark_*" -type d + echo "JSON files found:" + find . -name "*.json" | head -10 + + - name: Create benchmark summary + run: | + echo "=== DEBUG: Creating benchmark summary ===" + echo "Available benchmark directories:" + find . -name "benchmark_*" -type d + echo "==========================================" + + echo "## PD Router Genai-Bench Results Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "🚀 **Benchmarked with genai-bench for comprehensive LLM serving performance evaluation**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Policy | Status | TTFT (s) | E2E Latency (s) | Input Throughput (tok/s) | Output Throughput (tok/s) |" >> $GITHUB_STEP_SUMMARY + echo "|--------|--------|----------|-----------------|--------------------------|---------------------------|" >> $GITHUB_STEP_SUMMARY + + # First, complete the table with all policies + for policy in random round_robin cache_aware power_of_two; do + # Find genai-bench result folders for this policy (handle zip extraction structure) + result_folder=$(find . -maxdepth 2 -name "benchmark_${policy}" -type d | head -1) + if [ -z "$result_folder" ]; then + # Try alternative patterns in case of different extraction structure + result_folder=$(find . -maxdepth 3 -path "*benchmark_${policy}*" -type d | head -1) + fi + + echo "DEBUG: Policy ${policy} -> Found folder: ${result_folder:-'NOT FOUND'}" + + if [ -n "$result_folder" ] && [ -d "$result_folder" ]; then + # Find JSON file with metrics + json_file=$(find "$result_folder" -name "*.json" -not -name "experiment_metadata.json" | head -1) + + if [ -n "$json_file" ] && [ -f "$json_file" ]; then + # Extract performance metrics + ttft_mean=$(jq -r '.aggregated_metrics.stats.ttft.mean // "N/A"' "$json_file" 2>/dev/null || echo "N/A") + e2e_latency_mean=$(jq -r '.aggregated_metrics.stats.e2e_latency.mean // "N/A"' "$json_file" 2>/dev/null || echo "N/A") + input_throughput_mean=$(jq -r '.aggregated_metrics.stats.input_throughput.mean // "N/A"' "$json_file" 2>/dev/null || echo "N/A") + output_throughput_mean=$(jq -r '.aggregated_metrics.stats.output_throughput.mean // "N/A"' "$json_file" 2>/dev/null || echo "N/A") + + # Format numbers for display (2 decimal places) + if [ "$ttft_mean" != "N/A" ] && [ "$ttft_mean" != "null" ]; then + ttft_display=$(printf "%.2f" "$ttft_mean" 2>/dev/null || echo "$ttft_mean") + else + ttft_display="N/A" + fi + + if [ "$e2e_latency_mean" != "N/A" ] && [ "$e2e_latency_mean" != "null" ]; then + e2e_display=$(printf "%.2f" "$e2e_latency_mean" 2>/dev/null || echo "$e2e_latency_mean") + else + e2e_display="N/A" + fi + + if [ "$input_throughput_mean" != "N/A" ] && [ "$input_throughput_mean" != "null" ]; then + input_display=$(printf "%.0f" "$input_throughput_mean" 2>/dev/null || echo "$input_throughput_mean") + else + input_display="N/A" + fi + + if [ "$output_throughput_mean" != "N/A" ] && [ "$output_throughput_mean" != "null" ]; then + output_display=$(printf "%.0f" "$output_throughput_mean" 2>/dev/null || echo "$output_throughput_mean") + else + output_display="N/A" + fi + + echo "| ${policy} | ✅ Success | $ttft_display | $e2e_display | $input_display | $output_display |" >> $GITHUB_STEP_SUMMARY + else + echo "| ${policy} | ❌ No Data | N/A | N/A | N/A | N/A |" >> $GITHUB_STEP_SUMMARY + fi + else + echo "| ${policy} | ❌ Failed | N/A | N/A | N/A | N/A |" >> $GITHUB_STEP_SUMMARY + fi + done + + # Add performance validation summary + echo "" >> $GITHUB_STEP_SUMMARY + echo "## 📊 Performance Validation" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Thresholds:** TTFT ≤ 2.0s | E2E Latency ≤ 8.0s | Input Throughput ≥ 10,000 tok/s | Output Throughput ≥ 100 tok/s" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + validation_summary="" + for policy in random round_robin cache_aware power_of_two; do + # Use same robust path finding as above + result_folder=$(find . -maxdepth 2 -name "benchmark_${policy}" -type d | head -1) + if [ -z "$result_folder" ]; then + result_folder=$(find . -maxdepth 3 -path "*benchmark_${policy}*" -type d | head -1) + fi + + if [ -n "$result_folder" ] && [ -d "$result_folder" ]; then + json_file=$(find "$result_folder" -name "*.json" -not -name "experiment_metadata.json" | head -1) + if [ -n "$json_file" ] && [ -f "$json_file" ]; then + # Extract metrics for validation + ttft=$(jq -r '.aggregated_metrics.stats.ttft.mean // "N/A"' "$json_file" 2>/dev/null || echo "N/A") + e2e_latency=$(jq -r '.aggregated_metrics.stats.e2e_latency.mean // "N/A"' "$json_file" 2>/dev/null || echo "N/A") + input_throughput=$(jq -r '.aggregated_metrics.stats.input_throughput.mean // "N/A"' "$json_file" 2>/dev/null || echo "N/A") + output_throughput=$(jq -r '.aggregated_metrics.stats.output_throughput.mean // "N/A"' "$json_file" 2>/dev/null || echo "N/A") + + # Check thresholds (using same values as in main workflow) + validation_status="✅" + if [ "$ttft" != "N/A" ] && [ "$ttft" != "null" ]; then + if (( $(echo "$ttft > 4.7" | bc -l 2>/dev/null || echo "0") )); then + validation_status="❌" + fi + fi + if [ "$e2e_latency" != "N/A" ] && [ "$e2e_latency" != "null" ]; then + if (( $(echo "$e2e_latency > 35.0" | bc -l 2>/dev/null || echo "0") )); then + validation_status="❌" + fi + fi + if [ "$input_throughput" != "N/A" ] && [ "$input_throughput" != "null" ]; then + if (( $(echo "$input_throughput < 10000" | bc -l 2>/dev/null || echo "0") )); then + validation_status="❌" + fi + fi + if [ "$output_throughput" != "N/A" ] && [ "$output_throughput" != "null" ]; then + if (( $(echo "$output_throughput < 68" | bc -l 2>/dev/null || echo "0") )); then + validation_status="❌" + fi + fi + + validation_summary="${validation_summary}- **${policy}**: $validation_status\n" + else + validation_summary="${validation_summary}- **${policy}**: ❌ No data\n" + fi + else + validation_summary="${validation_summary}- **${policy}**: ❌ Failed\n" + fi + done + + echo -e "$validation_summary" >> $GITHUB_STEP_SUMMARY + + echo "" >> $GITHUB_STEP_SUMMARY + echo "## 📊 Genai-Bench Features Used" >> $GITHUB_STEP_SUMMARY + echo "- **Token-level Performance**: TTFT, TPOT, End-to-End latency" >> $GITHUB_STEP_SUMMARY + echo "- **Throughput Analysis**: Input/Output/Total token throughput" >> $GITHUB_STEP_SUMMARY + echo "- **Statistical Analysis**: Percentiles, mean, std dev for all metrics" >> $GITHUB_STEP_SUMMARY + echo "- **Visual Reports**: Automated plots and Excel summaries" >> $GITHUB_STEP_SUMMARY + echo "- **SGLang Backend**: Native integration with SGLang serving" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "✅ All policies tested successfully with genai-bench!" >> $GITHUB_STEP_SUMMARY diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-rust.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-rust.yml new file mode 100644 index 00000000..1a67c84f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-rust.yml @@ -0,0 +1,419 @@ +name: PR Test (SMG) + +on: + push: + branches: [ main ] + paths: + - "sgl-router/**" + pull_request: + branches: [ main ] + types: [opened, synchronize, reopened, labeled] + paths: + - "sgl-router/**" + workflow_dispatch: + +concurrency: + group: router-tests-${{ github.ref }} + cancel-in-progress: true + +env: + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + +jobs: + maturin-build-test: + if: | + github.event_name != 'pull_request' || + (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run-ci')) || + (github.event.action == 'labeled' && github.event.label.name == 'run-ci') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + path: sglang-repo + + - name: Move sgl-router folder to root + run: | + mv sglang-repo/sgl-router/* . + rm -rf sglang-repo + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install protoc and dependencies + run: | + sudo apt-get update + sudo apt-get install -y wget unzip gcc g++ perl make + cd /tmp + wget https://github.com/protocolbuffers/protobuf/releases/download/v32.0/protoc-32.0-linux-x86_64.zip + sudo unzip protoc-32.0-linux-x86_64.zip -d /usr/local + rm protoc-32.0-linux-x86_64.zip + protoc --version + + - name: Configure sccache + uses: mozilla-actions/sccache-action@v0.0.9 + with: + version: "v0.10.0" + + - name: Test maturin build + uses: PyO3/maturin-action@v1 + with: + working-directory: bindings/python + args: --release --out dist --features vendored-openssl + rust-toolchain: stable + sccache: true + + - name: List built wheel + run: ls -lh bindings/python/dist/ + + - name: Test wheel install + run: | + pip install bindings/python/dist/*.whl + python -c "import sglang_router; print('Python package: OK')" + python -c "from sglang_router.sglang_router_rs import Router; print('Rust extension: OK')" + python -m sglang_router.launch_router --help > /dev/null && echo "Entry point: OK" + router-unit-tests: + if: | + github.event_name != 'pull_request' || + (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run-ci')) || + (github.event.action == 'labeled' && github.event.label.name == 'run-ci') + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_rust.sh + + - name: Configure sccache + uses: mozilla-actions/sccache-action@v0.0.9 + with: + version: "v0.10.0" + + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: sgl-router + cache-all-crates: true + cache-on-failure: true + + - name: Run lint + run: | + source "$HOME/.cargo/env" + cd sgl-router/ + rustup component add clippy + cargo clippy --all-targets --all-features -- -D warnings + + - name: Run fmt + run: | + source "$HOME/.cargo/env" + cd sgl-router/ + rustup component add --toolchain nightly-x86_64-unknown-linux-gnu rustfmt + rustup toolchain install nightly --profile minimal + cargo +nightly fmt -- --check + + - name: Run Rust tests + timeout-minutes: 20 + run: | + source "$HOME/.cargo/env" + cd sgl-router/ + cargo test + + - name: Check benchmark compilation + run: | + source "$HOME/.cargo/env" + cd sgl-router/ + cargo check --benches + + - name: Quick benchmark sanity check + timeout-minutes: 15 + run: | + source "$HOME/.cargo/env" + cd sgl-router/ + # Run quick benchmarks to ensure they work using Python script + python3 scripts/run_benchmarks.py --quick + + - name: Show sccache stats + if: always() + run: sccache --show-stats + + router-http-tests: + if: | + github.event_name != 'pull_request' || + (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run-ci')) || + (github.event.action == 'labeled' && github.event.label.name == 'run-ci') + runs-on: 4-gpu-a10 + timeout-minutes: 32 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install rust dependencies + run: | + bash scripts/ci/ci_install_rust.sh + + - name: Configure sccache + uses: mozilla-actions/sccache-action@v0.0.9 + with: + version: "v0.10.0" + + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: sgl-router + cache-all-crates: true + cache-on-failure: true + + - name: Install SGLang dependencies + run: | + sudo --preserve-env=PATH bash scripts/ci/ci_install_dependency.sh + + - name: Build python binding + run: | + source "$HOME/.cargo/env" + export RUSTC_WRAPPER=sccache + cd sgl-router/bindings/python + python3 -m pip install --upgrade pip maturin + pip uninstall -y sglang-router + maturin build --profile ci --features vendored-openssl --out dist + pip install dist/*.whl + + - name: Run Python unit tests + run: | + cd sgl-router + source "$HOME/.cargo/env" + python3 -m pip install pytest pytest-cov pytest-xdist + pytest -q py_test/unit --cov=sglang_router --cov-config=bindings/python/.coveragerc --cov-report=term-missing --cov-fail-under=80 + + - name: Run Python integration tests + run: | + cd sgl-router + source "$HOME/.cargo/env" + # Integration tests use FastAPI/uvicorn for mock workers + python3 -m pip install fastapi uvicorn orjson + pytest -q py_test/integration_mock + + - name: Run Python E2E tests + run: | + bash scripts/killall_sglang.sh "nuk_gpus" + cd sgl-router + source "$HOME/.cargo/env" + python3 -m pip --no-cache-dir install --upgrade --ignore-installed blinker + python3 -m pip --no-cache-dir install --upgrade genai-bench==0.0.2 + pytest py_test/e2e_http -s -vv -o log_cli=true --log-cli-level=INFO + + - name: Upload benchmark results + if: success() + uses: actions/upload-artifact@v4 + with: + name: genai-bench-results-all-policies + path: sgl-router/benchmark_**/ + + router-grpc-response-api-tests: + if: | + github.event_name != 'pull_request' || + (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run-ci')) || + (github.event.action == 'labeled' && github.event.label.name == 'run-ci') + runs-on: 4-gpu-a10 + timeout-minutes: 32 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install rust dependencies + run: | + bash scripts/ci/ci_install_rust.sh + + - name: Configure sccache + uses: mozilla-actions/sccache-action@v0.0.9 + with: + version: "v0.10.0" + + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: sgl-router + cache-all-crates: true + cache-on-failure: true + + - name: Install SGLang dependencies + run: | + sudo --preserve-env=PATH bash scripts/ci/ci_install_dependency.sh + + - name: Setup Oracle Instant Client + run: | + sudo apt-get install -y unzip + INSTANT_CLIENT_DIR="/home/ubuntu/instant-client" + INSTANT_CLIENT_ZIP="instantclient-basic-linux.x64-23.9.0.25.07.zip" + + if [ ! -d "$INSTANT_CLIENT_DIR/instantclient_23_9" ]; then + echo "Downloading Oracle Instant Client..." + mkdir -p "$INSTANT_CLIENT_DIR" + cd "$INSTANT_CLIENT_DIR" + wget https://download.oracle.com/otn_software/linux/instantclient/2390000/$INSTANT_CLIENT_ZIP + unzip $INSTANT_CLIENT_ZIP + rm $INSTANT_CLIENT_ZIP + else + echo "Oracle Instant Client already exists, skipping download" + fi + + echo "LD_LIBRARY_PATH=/home/ubuntu/instant-client/instantclient_23_9:\$LD_LIBRARY_PATH" >> $GITHUB_ENV + + - name: Start Oracle Database + run: | + docker run -d -p 1521:1521 -e ORACLE_PASSWORD=oracle --name oracle-db gvenzl/oracle-xe:21-slim + echo "Starting Oracle DB..." + + # Export Oracle connection environment variables + echo "ATP_USER=system" >> $GITHUB_ENV + echo "ATP_PASSWORD=oracle" >> $GITHUB_ENV + echo "ATP_DSN=localhost:1521/XEPDB1" >> $GITHUB_ENV + + - name: Start Brave MCP Server + run: | + docker run -d --rm \ + -p 8001:8080 \ + -e BRAVE_API_KEY \ + --name brave-search-server \ + shoofio/brave-search-mcp-sse:1.0.10 + echo "Starting Brave MCP Server..." + sleep 2 + curl -f --max-time 1 http://localhost:8001/sse > /dev/null 2>&1 && echo "Brave MCP Server is healthy!" || echo "Brave MCP Server responded" + + - name: Build python binding + run: | + source "$HOME/.cargo/env" + export RUSTC_WRAPPER=sccache + cd sgl-router/bindings/python + python3 -m pip install --upgrade pip maturin + pip uninstall -y sglang-router + maturin build --profile ci --features vendored-openssl --out dist + pip install dist/*.whl + + - name: Run Python E2E response API tests + run: | + python3 -m pip install pytest-rerunfailures + bash scripts/killall_sglang.sh "nuk_gpus" + cd sgl-router + source "$HOME/.cargo/env" + SHOW_ROUTER_LOGS=1 pytest --reruns 3 --reruns-delay 2 py_test/e2e_response_api -s -vv -o log_cli=true --log-cli-level=INFO + + - name: Run Python E2E gRPC tests + run: | + bash scripts/killall_sglang.sh "nuk_gpus" + cd sgl-router + source "$HOME/.cargo/env" + SHOW_ROUTER_LOGS=1 ROUTER_LOCAL_MODEL_PATH="/home/ubuntu/models" pytest py_test/e2e_grpc -s -vv -o log_cli=true --log-cli-level=INFO + + - name: Cleanup Brave MCP Server + if: always() + run: | + docker stop brave-search-server || true + docker rm brave-search-server || true + + - name: Cleanup Oracle Database + if: always() + run: | + docker stop oracle-db || true + docker rm oracle-db || true + + + finish: + needs: [maturin-build-test, router-unit-tests, router-http-tests, router-grpc-response-api-tests] + runs-on: ubuntu-latest + steps: + - name: Finish + run: echo "This is an empty step to ensure that all jobs are completed." + + summarize-benchmarks: + needs: router-http-tests + runs-on: ubuntu-latest + if: success() + + steps: + - name: Install jq + run: sudo apt-get update && sudo apt-get install -y jq bc + + - name: Download benchmark results + uses: actions/download-artifact@v4 + with: + name: genai-bench-results-all-policies + + - name: List downloaded contents + run: | + echo "Contents after download:" + ls -la + find . -name "benchmark_*" -type d + echo "JSON files found:" + find . -name "*.json" | head -10 + + - name: Create benchmark summary + run: | + echo "=== DEBUG: Creating benchmark summary ===" + echo "Available benchmark directories:" + find . -name "benchmark_*" -type d || true + echo "==========================================" + + echo "## Router E2E Genai-Bench Results Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Results captured from E2E tests for two scenarios: regular router (2 workers, dp=2) and PD router (2 prefill + 2 decode)." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Scenario | Status | TTFT (s) | E2E Latency (s) | Input Throughput (tok/s) | Output Throughput (tok/s) |" >> $GITHUB_STEP_SUMMARY + echo "|----------|--------|----------|-----------------|--------------------------|---------------------------|" >> $GITHUB_STEP_SUMMARY + + scenarios=$'Regular (dp=2, round_robin)|benchmark_round_robin_regular\nPD (2 prefill + 2 decode, round_robin)|benchmark_round_robin_pd' + + echo "$scenarios" | sed 's/^\s*//' | while IFS='|' read -r label pattern; do + [ -z "$label" ] && continue + # Find the result folder (handle different extraction layouts) + result_folder=$(find . -maxdepth 3 \( -name "$pattern" -o -path "*${pattern}*" \) -type d | head -1) + + if [ -n "$result_folder" ] && [ -d "$result_folder" ]; then + json_file=$(find "$result_folder" -name "*.json" -not -name "experiment_metadata.json" | head -1) + + if [ -n "$json_file" ] && [ -f "$json_file" ]; then + ttft_mean=$(jq -r '.aggregated_metrics.stats.ttft.mean' "$json_file") + e2e_latency_mean=$(jq -r '.aggregated_metrics.stats.e2e_latency.mean' "$json_file") + input_throughput_mean=$(jq -r '.aggregated_metrics.stats.input_throughput.mean' "$json_file") + output_throughput_mean=$(jq -r '.aggregated_metrics.stats.output_throughput.mean' "$json_file") + + ttft_display=$(printf "%.2f" "$ttft_mean" 2>/dev/null || echo "$ttft_mean") + e2e_display=$(printf "%.2f" "$e2e_latency_mean" 2>/dev/null || echo "$e2e_latency_mean") + input_display=$(printf "%.0f" "$input_throughput_mean" 2>/dev/null || echo "$input_throughput_mean") + output_display=$(printf "%.0f" "$output_throughput_mean" 2>/dev/null || echo "$output_throughput_mean") + + echo "| ${label} | ✅ Success | $ttft_display | $e2e_display | $input_display | $output_display |" >> $GITHUB_STEP_SUMMARY + + # Optional GPU utilization table if monitor output exists + gpu_json="$result_folder/gpu_utilization.json" + if [ -f "$gpu_json" ]; then + overall_mean=$(jq -r '.overall.mean // 0' "$gpu_json") + printf "\n#### GPU Utilization — %s\n\n" "$label" >> $GITHUB_STEP_SUMMARY + printf "Overall mean: %.2f%%\n\n" "$overall_mean" >> $GITHUB_STEP_SUMMARY + echo "| GPU | Mean (%) | p5 | p10 | p25 | p50 | p75 | p90 | p95 |" >> $GITHUB_STEP_SUMMARY + echo "|-----|----------|----|-----|-----|-----|-----|-----|-----|" >> $GITHUB_STEP_SUMMARY + jq -r ' + .per_gpu + | to_entries[] + | [ .key, + (.value.mean // 0), + (.value.p5 // 0), + (.value.p10 // 0), + (.value.p25 // 0), + (.value.p50 // 0), + (.value.p75 // 0), + (.value.p90 // 0), + (.value.p95 // 0) + ] + | @tsv' "$gpu_json" \ + | while IFS=$'\t' read -r gpu m p5 p10 p25 p50 p75 p90 p95; do + printf "| %s | %.2f | %.2f | %.2f | %.2f | %.2f | %.2f | %.2f | %.2f |\n" "$gpu" "$m" "$p5" "$p10" "$p25" "$p50" "$p75" "$p90" "$p95" >> $GITHUB_STEP_SUMMARY + done + echo "" >> $GITHUB_STEP_SUMMARY + fi + fi + fi + done diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-xeon.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-xeon.yml new file mode 100644 index 00000000..4eb335a2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-xeon.yml @@ -0,0 +1,102 @@ +name: PR Test (Xeon) + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: + +concurrency: + group: pr-test-xeon-${{ github.ref }} + cancel-in-progress: false + +jobs: + # ==================== Check Changes ==================== # + check-changes: + runs-on: ubuntu-latest + outputs: + main_package: ${{ steps.filter.outputs.main_package }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Detect file changes + id: filter + uses: dorny/paths-filter@v3 + with: + filters: | + main_package: + - "python/sglang/!(multimodal_gen)/**" + - "python/*.toml" + - "scripts/ci/**" + - "test/**" + - "sgl-kernel/**" + - ".github/workflows/pr-test-xeon.yml" + - "docker/xeon.Dockerfile" + + # ==================== PR Gate ==================== # + pr-gate: + needs: check-changes + if: needs.check-changes.outputs.main_package == 'true' + uses: ./.github/workflows/pr-gate.yml + secrets: inherit + + build-test: + needs: [check-changes, pr-gate] + if: needs.check-changes.outputs.main_package == 'true' + runs-on: xeon-gnr + env: + HF_HOME: /home/sdp/.cache/huggingface + strategy: + matrix: + build_type: ['all'] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Build and Push + run: | + version=$(cat python/sglang/version.py | cut -d'"' -f2) + tag=v${version}-xeon + PR_REPO=${{ github.event.pull_request.head.repo.clone_url }} + PR_HEAD_REF=${{ github.head_ref }} + + docker build \ + ${PR_REPO:+--build-arg SGLANG_REPO=$PR_REPO} \ + ${PR_HEAD_REF:+--build-arg VER_SGLANG=$PR_HEAD_REF} \ + . -f docker/xeon.Dockerfile -t sglang_xeon --no-cache + + - name: Run container + run: | + docker run -dt \ + -v ${{ github.workspace }}:/sglang-checkout/ --ipc=host \ + -v ${HF_HOME}:/root/.cache/huggingface \ + --name ci_sglang_xeon \ + sglang_xeon + + - name: Check AMX support + id: check_amx + timeout-minutes: 5 + run: | + docker exec -w /sglang-checkout/ ci_sglang_xeon \ + bash -c "source /opt/.venv/bin/activate && python3 -c 'import torch; import sgl_kernel; assert torch._C._cpu._is_amx_tile_supported(); assert hasattr(torch.ops.sgl_kernel, \"convert_weight_packed\"); '" + + - name: Run unit tests + timeout-minutes: 36 + run: | + docker exec -w /sglang-checkout/ ci_sglang_xeon \ + bash -c "source /opt/.venv/bin/activate && cd ./test/srt && python3 run_suite.py --suite per-commit-cpu --timeout-per-file 1500" + + - name: Change permission + timeout-minutes: 2 + run: | + docker exec -u root ci_sglang_xeon bash -c " + rm -rf /tmp/ci-home && + chown -R $(id -u):$(id -g) /sglang-checkout/ 2>/dev/null || true + " + + - name: Cleanup container + if: always() + run: | + docker rm -f ci_sglang_xeon || true diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-xpu.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-xpu.yml new file mode 100644 index 00000000..e464bd57 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test-xpu.yml @@ -0,0 +1,118 @@ +name: PR Test (XPU) + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: + +concurrency: + group: pr-test-xpu-${{ github.ref }} + cancel-in-progress: true + +jobs: + # ==================== Check Changes ==================== # + check-changes: + runs-on: ubuntu-latest + outputs: + main_package: ${{ steps.filter.outputs.main_package }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Detect file changes + id: filter + uses: dorny/paths-filter@v3 + with: + filters: | + main_package: + - "python/sglang/!(multimodal_gen)/**" + - "python/*.toml" + - "scripts/ci/**" + - "test/**" + - "sgl-kernel/**" + - ".github/workflows/pr-test-xpu.yml" + - "docker/xpu.Dockerfile" + + # ==================== PR Gate ==================== # + pr-gate: + needs: check-changes + if: needs.check-changes.outputs.main_package == 'true' + uses: ./.github/workflows/pr-gate.yml + secrets: inherit + + build-and-test: + needs: [check-changes, pr-gate] + if: needs.check-changes.outputs.main_package == 'true' + runs-on: intel-bmg + env: + HF_HOME: /home/sdp/.cache/huggingface + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build Docker image + run: | + PR_REPO=${{ github.event.pull_request.head.repo.clone_url }} + PR_HEAD_REF=${{ github.head_ref }} + docker build \ + ${PR_REPO:+--build-arg SG_LANG_REPO=$PR_REPO} \ + ${PR_HEAD_REF:+--build-arg SG_LANG_BRANCH=$PR_HEAD_REF} \ + --no-cache --progress=plain -f docker/xpu.Dockerfile -t xpu_sglang_main:bmg . + + - name: Run container + id: start_container + run: | + container_id=$(docker run -dt \ + --group-add 992 \ + --group-add $(getent group video | cut -d: -f3) \ + -v ${HF_HOME}:/root/.cache/huggingface \ + --device /dev/dri \ + -e HF_TOKEN="$(cat ~/huggingface_token.txt)" \ + xpu_sglang_main:bmg) + echo "Started container: $container_id" + echo "container_id=$container_id" >> "$GITHUB_OUTPUT" + + - name: Install Dependency + timeout-minutes: 20 + run: | + cid="${{ steps.start_container.outputs.container_id }}" + docker exec "$cid" /home/sdp/miniforge3/envs/py3.10/bin/python3 -m pip install --upgrade pip + docker exec "$cid" /home/sdp/miniforge3/envs/py3.10/bin/python3 -m pip install pytest expecttest ray huggingface_hub + docker exec "$cid" /home/sdp/miniforge3/envs/py3.10/bin/python3 -m pip uninstall -y flashinfer-python + docker exec "$cid" /bin/bash -c '/home/sdp/miniforge3/envs/py3.10/bin/huggingface-cli login --token ${HF_TOKEN} ' + docker exec -u root "$cid" /bin/bash -c "ln -sf /home/sdp/miniforge3/envs/py3.10/bin/python3 /usr/bin/python3" + + - name: Run E2E Bfloat16 tests + timeout-minutes: 20 + run: | + cid="${{ steps.start_container.outputs.container_id }}" + docker exec -w /home/sdp/sglang/ "$cid" \ + bash -c "LD_LIBRARY_PATH=/home/sdp/miniforge3/envs/py3.10/lib:$LD_LIBRARY_PATH && cd ./test/srt && python3 run_suite.py --suite per-commit-xpu" + + - name: Cleanup container + if: always() + run: | + cid="${{ steps.start_container.outputs.container_id }}" + docker rm -f "$cid" || true + + finish: + if: always() + needs: [build-and-test, pr-gate] + runs-on: ubuntu-latest + steps: + - name: Check job status + run: | + result="${{ needs.build-and-test.result }}" + if [ "$result" != "success" ] && [ "$result" != "skipped" ]; then + echo "Job failed with result: $result" + exit 1 + fi + echo "All jobs completed successfully (result: $result)" + exit 0 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test.yml new file mode 100644 index 00000000..beaa89ac --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/pr-test.yml @@ -0,0 +1,1208 @@ +name: PR Test + +on: + schedule: + - cron: '0 */6 * * *' # Run every 6 hours + pull_request: + branches: [main] + workflow_dispatch: + inputs: + version: + description: "FlashInfer version" + required: true + type: choice + default: "release" + options: + - "release" + - "nightly" + target_stage: + description: "Specific stage to run (optional, for quick testing)" + required: false + type: string + default: "" + +concurrency: + group: pr-test-${{ github.ref }} + cancel-in-progress: true + +jobs: + # =============================================== check changes ==================================================== + check-changes: + runs-on: ubuntu-latest + outputs: + main_package: ${{ steps.filter.outputs.main_package || steps.scheduled.outputs.main_package }} + sgl_kernel: ${{ steps.filter.outputs.sgl_kernel || steps.scheduled.outputs.sgl_kernel }} + multimodal_gen: ${{ steps.filter.outputs.multimodal_gen || steps.scheduled.outputs.multimodal_gen }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Detect file changes + id: filter + uses: dorny/paths-filter@v3 + if: github.event_name != 'schedule' + with: + filters: | + main_package: + - "python/sglang/!(multimodal_gen)/**" + - "python/*.toml" + - "scripts/ci/**" + - "test/**" + - ".github/workflows/pr-test.yml" + sgl_kernel: + - "sgl-kernel/**" + multimodal_gen: + - "python/sglang/multimodal_gen/**" + - "python/sglang/cli/**" + - "python/*.toml" + - ".github/workflows/pr-test.yml" + + - name: Set all changes to true for scheduled runs + id: scheduled + if: github.event_name == 'schedule' + run: | + echo "main_package=true" >> $GITHUB_OUTPUT + echo "sgl_kernel=false" >> $GITHUB_OUTPUT + echo "multimodal_gen=true" >> $GITHUB_OUTPUT + + - name: Show filter results in summary (table) + run: | + { + echo "## Change Detection" + echo "" + echo "| Component | Changed |" + echo "|----------------|---------|" + echo "| main_package | ${{ steps.filter.outputs.main_package || steps.scheduled.outputs.main_package }} |" + echo "| sgl_kernel | ${{ steps.filter.outputs.sgl_kernel || steps.scheduled.outputs.sgl_kernel }} |" + echo "| multimodal_gen | ${{ steps.filter.outputs.multimodal_gen || steps.scheduled.outputs.multimodal_gen }} |" + } >> $GITHUB_STEP_SUMMARY + + # =============================================== PR Gate ==================================================== + call-gate: + needs: check-changes + if: | + needs.check-changes.outputs.main_package == 'true' || + needs.check-changes.outputs.sgl_kernel == 'true' || + needs.check-changes.outputs.multimodal_gen == 'true' + uses: ./.github/workflows/pr-gate.yml + secrets: inherit + + # =============================================== sgl-kernel ==================================================== + + sgl-kernel-build-wheels: + needs: [check-changes, call-gate] + if: needs.check-changes.outputs.sgl_kernel == 'true' + runs-on: x64-kernel-build-node + strategy: + matrix: + include: + - python-version: "3.10" + cuda-version: "12.9" + # Add back when CUDA 13.0 is supported on CI + # - python-version: "3.10" + # cuda-version: "13.0" + name: Build Wheel + steps: + - name: Cleanup + run: | + sudo rm -rf $GITHUB_WORKSPACE/* || true + + - uses: actions/checkout@v4 + with: + submodules: "recursive" + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Build wheel for Python ${{ matrix.python-version }} and CUDA ${{ matrix.cuda-version }} + run: | + cd sgl-kernel + ./build.sh "${{ matrix.python-version }}" "${{ matrix.cuda-version }}" + env: + USE_CCACHE: 1 + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: wheel-python${{ matrix.python-version }}-cuda${{ matrix.cuda-version }} + path: sgl-kernel/dist/* + + sgl-kernel-build-wheels-arm: + needs: [check-changes, call-gate] + if: needs.check-changes.outputs.sgl_kernel == 'true' + runs-on: arm-kernel-build-node + strategy: + matrix: + include: + - python-version: "3.10" + cuda-version: "12.9" + name: Build Wheel Arm + steps: + - name: Cleanup + run: | + if [ -d "$GITHUB_WORKSPACE" ]; then + sudo rm -rf "$GITHUB_WORKSPACE"/* || true + else + echo "$GITHUB_WORKSPACE does not exist, nothing to clean" + fi + + - uses: actions/checkout@v4 + with: + submodules: "recursive" + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Build wheel for Python ${{ matrix.python-version }} and CUDA ${{ matrix.cuda-version }} + run: | + cd sgl-kernel + ./build.sh "${{ matrix.python-version }}" "${{ matrix.cuda-version }}" + env: + USE_CCACHE: 1 + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: wheel-python${{ matrix.python-version }}-cuda${{ matrix.cuda-version }}-aarch64 + path: sgl-kernel/dist/* + + sgl-kernel-unit-test: + needs: [check-changes, call-gate, sgl-kernel-build-wheels] + if: needs.check-changes.outputs.sgl_kernel == 'true' + runs-on: 1-gpu-runner + env: + RUNNER_LABELS: 1-gpu-runner + steps: + - uses: actions/checkout@v4 + + - name: Cleanup + run: | + ls -alh sgl-kernel/dist || true + rm -rf sgl-kernel/dist/* || true + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 30 + run: | + cd sgl-kernel + pytest tests/ + + sgl-kernel-mla-test: + needs: [check-changes, call-gate, sgl-kernel-build-wheels] + if: needs.check-changes.outputs.sgl_kernel == 'true' + runs-on: 1-gpu-runner + env: + RUNNER_LABELS: 1-gpu-runner + steps: + - uses: actions/checkout@v4 + + - name: Cleanup + run: | + ls -alh sgl-kernel/dist || true + rm -rf sgl-kernel/dist/* || true + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 30 + run: | + cd test/srt + python3 test_mla_deepseek_v3.py + + sgl-kernel-benchmark-test: + needs: [check-changes, call-gate, sgl-kernel-build-wheels] + if: needs.check-changes.outputs.sgl_kernel == 'true' + runs-on: 1-gpu-runner + env: + CI: true + RUNNER_LABELS: 1-gpu-runner + steps: + - uses: actions/checkout@v4 + + - name: Cleanup + run: | + ls -alh sgl-kernel/dist || true + rm -rf sgl-kernel/dist/* || true + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh + + - name: Run benchmark tests + timeout-minutes: 45 + run: | + cd sgl-kernel/benchmark + echo "Running sgl-kernel benchmark tests in CI mode..." + + echo "CI environment variable: $CI" + echo "GITHUB_ACTIONS environment variable: $GITHUB_ACTIONS" + + for bench_file in bench_*.py; do + echo "Testing $bench_file..." + timeout 60 python3 "$bench_file" || echo "Warning: $bench_file timed out or failed, continuing..." + echo "Completed $bench_file" + echo "---" + done + + echo "All benchmark tests completed!" + + sgl-kernel-b200-test: + needs: [check-changes, sgl-kernel-build-wheels] + if: needs.check-changes.outputs.sgl_kernel == 'true' + runs-on: 4-gpu-b200 + env: + RUNNER_LABELS: 4-gpu-b200 + steps: + - uses: actions/checkout@v4 + + - name: Cleanup + run: | + ls -alh sgl-kernel/dist || true + rm -rf sgl-kernel/dist/* || true + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh + + - name: Run sgl-kernel unit tests on B200 + timeout-minutes: 30 + run: | + cd sgl-kernel + pytest tests/ + + # Adding a single CUDA13 smoke test to verify that the kernel builds and runs + # TODO: Add back this test when it can pass on CI + # cuda13-kernel-smoke-test: + # needs: [check-changes, sgl-kernel-build-wheels] + # if: needs.check-changes.outputs.sgl_kernel == 'true' + # runs-on: x64-cu13-kernel-tests + # steps: + # - uses: actions/checkout@v4 + + # - name: Cleanup + # run: | + # ls -alh sgl-kernel/dist || true + # rm -rf sgl-kernel/dist/* || true + + # - name: Download CUDA 13.0 artifacts + # uses: actions/download-artifact@v4 + # with: + # path: sgl-kernel/dist/ + # merge-multiple: true + # pattern: wheel-python3.10-cuda13.0 + + # - name: Install dependencies + # run: | + # CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh + + # - name: Run kernel unit tests + # timeout-minutes: 30 + # run: | + # cd sgl-kernel + # pytest tests/ + + # =============================================== primary ==================================================== + + stage-a-test-1: + needs: [check-changes, call-gate, sgl-kernel-build-wheels] + if: | + (inputs.target_stage == 'stage-a-test-1') || + ( + always() && + (github.event_name == 'schedule' || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + runs-on: 1-gpu-runner + env: + RUNNER_LABELS: 1-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 10 + run: | + cd test/ + python3 run_suite.py --hw cuda --suite stage-a-test-1 + # temporarily put backend-independent cpu tests here + python3 run_suite.py --hw cpu --suite default + + + multimodal-gen-test-1-gpu: + needs: [check-changes, call-gate, sgl-kernel-build-wheels] + if: | + (inputs.target_stage == 'multimodal-gen-test-1-gpu') || + ( + always() && + (github.event_name == 'schedule' || (!failure() && !cancelled())) && + needs.check-changes.outputs.multimodal_gen == 'true' + ) + runs-on: 1-gpu-runner + strategy: + fail-fast: false + matrix: + part: [0, 1] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Cleanup + run: | + ls -alh sgl-kernel/dist || true + rm -rf sgl-kernel/dist/* || true + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh diffusion + - name: Run diffusion server tests + timeout-minutes: 60 + run: | + cd python + python3 sglang/multimodal_gen/test/run_suite.py \ + --suite 1-gpu \ + --partition-id ${{ matrix.part }} \ + --total-partitions 2 + + + multimodal-gen-test-2-gpu: + needs: [check-changes, call-gate, sgl-kernel-build-wheels] + if: | + (inputs.target_stage == 'multimodal-gen-test-2-gpu') || + ( + always() && + (github.event_name == 'schedule' || (!failure() && !cancelled())) && + needs.check-changes.outputs.multimodal_gen == 'true' + ) + runs-on: 2-gpu-runner + strategy: + fail-fast: false + matrix: + part: [0, 1] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Cleanup + run: | + ls -alh sgl-kernel/dist || true + rm -rf sgl-kernel/dist/* || true + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh diffusion + + - name: Run diffusion server tests + timeout-minutes: 60 + run: | + cd python + python3 sglang/multimodal_gen/test/run_suite.py \ + --suite 2-gpu \ + --partition-id ${{ matrix.part }} \ + --total-partitions 2 + + quantization-test: + needs: [check-changes, call-gate, stage-a-test-1] + if: | + (inputs.target_stage == 'quantization-test') || + ( + always() && + (github.event_name == 'schedule' || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + runs-on: 1-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + pip install "bitsandbytes>=0.44.0" + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh + - name: Run test + timeout-minutes: 30 + run: | + cd test/srt + python3 run_suite.py --suite quantization_test + + unit-test-backend-1-gpu: + needs: [check-changes, call-gate, stage-a-test-1] + if: | + (inputs.target_stage == 'unit-test-backend-1-gpu') || + ( + always() && + (github.event_name == 'schedule' || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + runs-on: 1-gpu-runner + env: + RUNNER_LABELS: 1-gpu-runner + strategy: + fail-fast: false + max-parallel: 8 + matrix: + part: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 30 + run: | + cd test/srt + python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 15 + + unit-test-backend-2-gpu: + needs: [check-changes, call-gate, unit-test-backend-1-gpu] + if: | + (inputs.target_stage == 'unit-test-backend-2-gpu') || + ( + always() && + (github.event_name == 'schedule' || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + runs-on: 2-gpu-runner + env: + RUNNER_LABELS: 2-gpu-runner + strategy: + fail-fast: false + matrix: + part: [0, 1] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 30 + run: | + cd test/srt + python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 + + unit-test-backend-4-gpu: + needs: [check-changes, call-gate, unit-test-backend-2-gpu] + if: | + (inputs.target_stage == 'unit-test-backend-4-gpu') || + ( + always() && + (github.event_name == 'schedule' || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + runs-on: 4-gpu-h100 + env: + RUNNER_LABELS: 4-gpu-h100 + strategy: + fail-fast: false + matrix: + part: [0, 1, 2] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 20 + run: | + cd test/srt + python3 run_suite.py --suite per-commit-4-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 + + unit-test-backend-8-gpu-h200: + needs: [check-changes, call-gate, unit-test-backend-2-gpu] + if: | + (inputs.target_stage == 'unit-test-backend-8-gpu-h200') || + ( + always() && + (github.event_name == 'schedule' || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + runs-on: 8-gpu-h200 + env: + RUNNER_LABELS: 8-gpu-h200 + strategy: + fail-fast: false + matrix: + part: [0, 1, 2] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 20 + run: | + cd test/srt + python3 run_suite.py --suite per-commit-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 + + unit-test-backend-8-gpu-h20: + needs: [check-changes, call-gate, unit-test-backend-2-gpu] + if: | + (inputs.target_stage == 'unit-test-backend-8-gpu-h20') || + ( + always() && + (github.event_name == 'schedule' || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + runs-on: 8-gpu-h20 + env: + SGLANG_CI_RDMA_ALL_DEVICES: "mlx5_1,mlx5_2,mlx5_3,mlx5_4" + RUNNER_LABELS: 8-gpu-h20 + strategy: + fail-fast: false + matrix: + part: [0, 1] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_deepep.sh + + - name: Run test + timeout-minutes: 20 + run: | + cd test/srt + python3 run_suite.py --suite per-commit-8-gpu-h20 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 + + performance-test-1-gpu-part-1: + needs: [check-changes, call-gate, stage-a-test-1] + if: | + (inputs.target_stage == 'performance-test-1-gpu-part-1') || + ( + always() && + (github.event_name == 'schedule' || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + runs-on: 1-gpu-runner + env: + RUNNER_LABELS: 1-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh + + - name: Benchmark single latency + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_one_batch.TestBenchOneBatch.test_bs1_small + python3 -m unittest test_bench_one_batch.TestBenchOneBatch.test_bs1_default + + - name: Benchmark online latency + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_online_latency_default + + - name: Benchmark offline throughput + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_default + + - name: Benchmark offline throughput (Non-streaming, small batch size) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_non_stream_small_batch_size + + - name: Benchmark online latency (EAGLE) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_online_latency_eagle + + - name: Benchmark online latency (LoRA) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_lora_online_latency + python3 -m unittest test_bench_serving.TestBenchServing.test_lora_online_latency_with_concurrent_adapter_updates + + performance-test-1-gpu-part-2: + needs: [check-changes, call-gate, stage-a-test-1] + if: | + (inputs.target_stage == 'performance-test-1-gpu-part-2') || + ( + always() && + (github.event_name == 'schedule' || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + runs-on: 1-gpu-runner + env: + RUNNER_LABELS: 1-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh + + - name: Benchmark offline throughput (w/o RadixAttention) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_without_radix_cache + + - name: Benchmark offline throughput (w/ Triton) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_with_triton_attention_backend + + - name: Benchmark offline throughput (w/ FP8) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_default_fp8 + + - name: Benchmark VLM offline throughput + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_vlm_offline_throughput + + - name: Benchmark VLM online latency + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_vlm_online_latency + + performance-test-1-gpu-part-3: + needs: [check-changes, call-gate, stage-a-test-1] + if: | + (inputs.target_stage == 'performance-test-1-gpu-part-3') || + ( + always() && + (github.event_name == 'schedule' || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + runs-on: 1-gpu-runner + env: + RUNNER_LABELS: 1-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh + + - name: Benchmark Scores online latency and throughput + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_score_api_latency_throughput + + - name: Benchmark Scores online latency and throughput (batch size scaling) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_score_api_batch_scaling + + - name: Benchmark Embeddings online latency and throughput + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_embeddings_api_latency_throughput + + - name: Benchmark Embeddings online latency and throughput (batch size scaling) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_embeddings_api_batch_scaling + + performance-test-2-gpu: + needs: [check-changes, call-gate, unit-test-backend-2-gpu] + if: | + (inputs.target_stage == 'performance-test-2-gpu') || + ( + always() && + (github.event_name == 'schedule' || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + runs-on: 2-gpu-runner + env: + RUNNER_LABELS: 2-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh + + - name: Benchmark single latency (TP=2) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_one_batch.TestBenchOneBatch.test_moe_tp2_bs1 + + - name: Benchmark single latency + torch.compile (TP=2) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_one_batch.TestBenchOneBatch.test_torch_compile_tp2_bs1 + + - name: Benchmark offline throughput (TP=2) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_moe_offline_throughput_default + + - name: Benchmark offline throughput (w/o RadixAttention) (TP=2) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_moe_offline_throughput_without_radix_cache + + - name: Benchmark offline PP decode throughput (PP=2) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_pp_offline_throughput_default_decode + + - name: Benchmark offline PP prefill throughput (PP=2) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_pp_long_context_prefill + + accuracy-test-1-gpu: + needs: [check-changes, call-gate, stage-a-test-1] + if: | + (inputs.target_stage == 'accuracy-test-1-gpu') || + ( + always() && + (github.event_name == 'schedule' || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + runs-on: 1-gpu-runner + env: + RUNNER_LABELS: 1-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh + git clone https://github.com/merrymercy/human-eval.git + cd human-eval + pip install -e . + + - name: Evaluate accuracy + timeout-minutes: 20 + run: | + cd test/srt + python3 test_eval_accuracy_large.py + + accuracy-test-2-gpu: + needs: [check-changes, call-gate, accuracy-test-1-gpu] + if: | + (inputs.target_stage == 'accuracy-test-2-gpu') || + ( + always() && + (github.event_name == 'schedule' || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + runs-on: 2-gpu-runner + env: + RUNNER_LABELS: 2-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh + git clone https://github.com/merrymercy/human-eval.git + cd human-eval + pip install -e . + + - name: Evaluate accuracy (TP=2) + timeout-minutes: 20 + run: | + cd test/srt + python3 test_moe_eval_accuracy_large.py + + unit-test-deepep-4-gpu: + needs: [check-changes, call-gate, unit-test-backend-2-gpu] + if: | + (inputs.target_stage == 'unit-test-deepep-4-gpu') || + ( + always() && + (github.event_name == 'schedule' || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + runs-on: 4-gpu-h100 + env: + RUNNER_LABELS: 4-gpu-h100 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_deepep.sh + + - name: Run test + timeout-minutes: 20 + run: | + cd test/srt + python3 run_suite.py --suite per-commit-4-gpu-deepep + + unit-test-deepep-8-gpu: + needs: [check-changes, call-gate, unit-test-backend-2-gpu] + if: | + (inputs.target_stage == 'unit-test-deepep-8-gpu') || + ( + always() && + (github.event_name == 'schedule' || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + runs-on: 8-gpu-h200 + env: + RUNNER_LABELS: 8-gpu-h200 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_deepep.sh + + - name: Run test + timeout-minutes: 20 + run: | + cd test/srt + python3 run_suite.py --suite per-commit-8-gpu-h200-deepep + + unit-test-backend-4-gpu-b200: + needs: [check-changes, call-gate, unit-test-backend-2-gpu] + if: | + (inputs.target_stage == 'unit-test-backend-4-gpu-b200') || + ( + always() && + (github.event_name == 'schedule' || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + runs-on: 4-gpu-b200 + env: + RUNNER_LABELS: 4-gpu-b200 + strategy: + fail-fast: false + matrix: + part: [0, 1] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v6 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 30 + run: | + cd test/srt + python3 run_suite.py --suite per-commit-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 + + unit-test-backend-4-gpu-gb200: + needs: [check-changes, call-gate, unit-test-backend-2-gpu, sgl-kernel-build-wheels-arm] + if: | + (inputs.target_stage == 'unit-test-backend-4-gpu-gb200') || + ( + always() && + (github.event_name == 'schedule' || (!failure() && !cancelled())) && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + ) + runs-on: 4-gpu-gb200 + env: + RUNNER_LABELS: 4-gpu-gb200 + strategy: + fail-fast: false + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9-aarch64 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} IS_BLACKWELL=1 GRACE_BLACKWELL=1 bash scripts/ci/ci_install_deepep.sh + + - name: Run test + timeout-minutes: 45 + run: | + cd test/srt + python3 run_suite.py --suite per-commit-4-gpu-gb200 --auto-partition-id 0 --auto-partition-size 1 --timeout-per-file 3600 + + pr-test-finish: + needs: + [ + call-gate, + check-changes, + + sgl-kernel-build-wheels, + sgl-kernel-unit-test, + sgl-kernel-mla-test, + sgl-kernel-benchmark-test, + sgl-kernel-b200-test, + + multimodal-gen-test-1-gpu, + multimodal-gen-test-2-gpu, + + stage-a-test-1, + quantization-test, + unit-test-backend-1-gpu, + unit-test-backend-2-gpu, + unit-test-backend-4-gpu, + unit-test-backend-8-gpu-h20, + unit-test-backend-8-gpu-h200, + performance-test-1-gpu-part-1, + performance-test-1-gpu-part-2, + performance-test-1-gpu-part-3, + performance-test-2-gpu, + accuracy-test-1-gpu, + accuracy-test-2-gpu, + unit-test-deepep-4-gpu, + unit-test-deepep-8-gpu, + unit-test-backend-4-gpu-b200, + unit-test-backend-4-gpu-gb200, + ] + if: always() + runs-on: ubuntu-latest + steps: + - name: Check all dependent job statuses + run: | + # Convert the 'needs' context to a JSON string + json_needs='${{ toJson(needs) }}' + + # Get a list of all job names from the JSON keys + job_names=$(echo "$json_needs" | jq -r 'keys_unsorted[]') + + for job in $job_names; do + # For each job, extract its result + result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result') + + # Print the job name and its result + echo "$job: $result" + + # Check for failure or cancellation and exit if found + if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then + echo "The above jobs failed." + exit 1 + fi + done + # If the loop completes, all jobs were successful + echo "All jobs completed successfully" + exit 0 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-amd-nightly.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-amd-nightly.yml new file mode 100644 index 00000000..47508ac2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-amd-nightly.yml @@ -0,0 +1,132 @@ +name: Release Docker Images Nightly (AMD) +on: + workflow_dispatch: + schedule: + - cron: '0 13 * * *' + +concurrency: + # A PR number if a pull request and otherwise the commit hash. This cancels + # queued and in-progress runs for the same PR (presubmit) or commit + # (postsubmit). The workflow name is prepended to avoid conflicts between + # different workflows. + group: ${{ github.workflow }}-${{ github.event.number || github.sha }} + cancel-in-progress: true + +jobs: + publish: + if: github.repository == 'sgl-project/sglang' + runs-on: amd-docker-scale + environment: 'prod' + strategy: + fail-fast: false + matrix: + gpu_arch: ['gfx942', 'gfx942-rocm700', 'gfx950'] + build_type: ['all'] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: "Set Date" + run: | + echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_AMD_USERNAME }} + password: ${{ secrets.DOCKERHUB_AMD_TOKEN }} + + - name: Build and Push + run: | + version=$(cat python/sglang/version.py | cut -d'"' -f2) + echo "Version: ${version}" + + if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then + rocm_tag="rocm630-mi30x" + elif [ "${{ matrix.gpu_arch }}" = "gfx942-rocm700" ]; then + rocm_tag="rocm700-mi30x" + elif [ "${{ matrix.gpu_arch }}" = "gfx950" ]; then + rocm_tag="rocm700-mi35x" + else + echo "Unsupported gfx arch" + exit 1 + fi + + tag=v${version}-${rocm_tag} + + docker build . -f docker/rocm.Dockerfile --build-arg BUILD_TYPE=${{ matrix.build_type }} --build-arg GPU_ARCH=${{ matrix.gpu_arch }} -t rocm/sgl-dev:${tag}-${{ env.DATE }} --no-cache + docker push rocm/sgl-dev:${tag}-${{ env.DATE }} + + cache: + if: always() && github.repository == 'sgl-project/sglang' + runs-on: linux-mi300-gpu-1 + environment: 'prod' + needs: publish + strategy: + fail-fast: false + matrix: + gpu_arch: ['gfx942', 'gfx942-rocm700'] + build_type: ['all'] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: "Set Date" + run: | + echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_AMD_USERNAME }} + password: ${{ secrets.DOCKERHUB_AMD_TOKEN }} + + - name: Pull and Save Docker Image to Cache + run: | + set -euxo pipefail + + version=$(cat python/sglang/version.py | cut -d'"' -f2) + echo "Version: ${version}" + + if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then + rocm_tag="rocm630-mi30x" + elif [ "${{ matrix.gpu_arch }}" = "gfx942-rocm700" ]; then + rocm_tag="rocm700-mi30x" + else + echo "Unsupported gfx arch" + exit 1 + fi + + tag=v${version}-${rocm_tag} + + if [ "${{ matrix.build_type }}" = "all" ]; then + tag_suffix="" + else + echo "Unsupported build type" + exit 1 + fi + + image="rocm/sgl-dev:${tag}-${{ env.DATE }}${tag_suffix}" + + # Determine target cache file name based on ROCm variant + if [[ "${rocm_tag}" == rocm630* ]]; then + final_path="/home/runner/sgl-data/docker/image.tar" + elif [[ "${rocm_tag}" == rocm700* ]]; then + final_path="/home/runner/sgl-data/docker/image-700.tar" + else + echo "Unexpected ROCm tag: ${rocm_tag}" + exit 1 + fi + + tmp_path="${final_path}.tmp" + + echo "Pulling image: ${image}" + docker pull "${image}" + + echo "Saving to temp file: ${tmp_path}" + docker save "${image}" -o "${tmp_path}" + + echo "Moving to final path: ${final_path}" + mv -f "${tmp_path}" "${final_path}" + + echo "Cache populated successfully at ${final_path}" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-amd.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-amd.yml new file mode 100644 index 00000000..8b4fae51 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-amd.yml @@ -0,0 +1,48 @@ +name: Release Docker Images (AMD) +on: + push: + branches: + - main + paths: + - "python/sglang/version.py" + workflow_dispatch: + +jobs: + publish: + if: github.repository == 'sgl-project/sglang' + runs-on: amd-docker-scale + environment: 'prod' + strategy: + matrix: + gpu_arch: ['gfx942', 'gfx942-rocm700', 'gfx950'] + build_type: ['all'] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and Push + run: | + version=$(cat python/sglang/version.py | cut -d'"' -f2) + echo "Version: ${version}" + + if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then + rocm_tag="rocm630-mi30x" + elif [ "${{ matrix.gpu_arch }}" = "gfx942-rocm700" ]; then + rocm_tag="rocm700-mi30x" + elif [ "${{ matrix.gpu_arch }}" = "gfx950" ]; then + rocm_tag="rocm700-mi35x" + else + echo "Unsupported gfx arch" + exit 1 + fi + + tag=v${version}-${rocm_tag} + + docker build . -f docker/rocm.Dockerfile --build-arg BUILD_TYPE=${{ matrix.build_type }} --build-arg GPU_ARCH=${{ matrix.gpu_arch }} -t lmsysorg/sglang:${tag} --no-cache + docker push lmsysorg/sglang:${tag} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-cu13.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-cu13.yml new file mode 100644 index 00000000..32763cb7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-cu13.yml @@ -0,0 +1,119 @@ +name: Build and Push CUDA 13 Docker Images + +# release this manually via workflow_dispatch for now +on: + workflow_dispatch: + +jobs: + build-dev: + if: ${{ github.repository == 'sgl-project/sglang' }} + runs-on: ${{ matrix.runner }} + strategy: + matrix: + include: + - runner: x64-docker-build-node + platform: linux/amd64 + build_type: all + grace_blackwell: 0 + tag: dev-x86-cu13-$(date +%Y%m%d) + version: 13.0.1 + - runner: arm-docker-build-node + platform: linux/arm64 + build_type: all + grace_blackwell: 1 + tag: dev-arm64-cu13-$(date +%Y%m%d) + version: 13.0.1 + steps: + - name: Delete huge unnecessary tools folder + run: rm -rf /opt/hostedtoolcache + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Free disk space + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + docker-images: true + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and Push Dev Image + run: | + docker buildx build \ + --platform ${{ matrix.platform }} \ + --push \ + -f docker/Dockerfile \ + --build-arg CUDA_VERSION=${{ matrix.version }} \ + --build-arg BUILD_TYPE=${{ matrix.build_type }} \ + --build-arg CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) \ + --build-arg GRACE_BLACKWELL=${{ matrix.grace_blackwell }} \ + --build-arg USE_LATEST_SGLANG=1 \ + -t lmsysorg/sglang:${{ matrix.tag }} \ + --no-cache \ + . + + create-manifests: + runs-on: ubuntu-22.04 + needs: [build-dev] + if: ${{ github.repository == 'sgl-project/sglang' }} + strategy: + matrix: + variant: + - tag: dev-cu13 + x86_tag: dev-x86-cu13 + arm64_tag: dev-arm64-cu13 + steps: + - uses: docker/setup-buildx-action@v3 + + - uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - run: | + docker buildx imagetools create \ + -t lmsysorg/sglang:${{ matrix.variant.tag }} \ + -t lmsysorg/sglang:nightly-${{ matrix.variant.tag }}-$(date +%Y%m%d)-${GITHUB_SHA:0:8} \ + lmsysorg/sglang:${{ matrix.variant.x86_tag }} \ + lmsysorg/sglang:${{ matrix.variant.arm64_tag }} + + - name: Cleanup Old Nightly Builds + run: | + # Get JWT token for Docker Hub API + TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d '{"username": "${{ secrets.DOCKERHUB_USERNAME }}", "password": "${{ secrets.DOCKERHUB_TOKEN }}"}' https://hub.docker.com/v2/users/login/ | jq -r .token) + + # Get all tags for the repository + TAGS_RESPONSE=$(curl -s -H "Authorization: JWT $TOKEN" "https://hub.docker.com/v2/repositories/lmsysorg/sglang/tags/?page_size=100") + + # Extract tags that match our pattern and sort by last_updated timestamp (most recent first) + TAGS=$(echo "$TAGS_RESPONSE" | jq -r '.results[] | select(.name | startswith("nightly-${{ matrix.variant.tag }}-")) | "\(.last_updated)|\(.name)"' | sort -r | cut -d'|' -f2) + + # Count total tags and keep only the 14 most recent + TAG_COUNT=$(echo "$TAGS" | wc -l) + if [ "$TAG_COUNT" -gt 14 ]; then + echo "Found $TAG_COUNT nightly builds, keeping only the 14 most recent" + TAGS_TO_DELETE=$(echo "$TAGS" | tail -n +15) + echo "Tags to delete: $TAGS_TO_DELETE" + + # Delete old tags + for tag in $TAGS_TO_DELETE; do + echo "Deleting tag: $tag" + curl -X DELETE \ + -H "Authorization: JWT $TOKEN" \ + "https://hub.docker.com/v2/repositories/lmsysorg/sglang/tags/$tag/" + done + else + echo "Only $TAG_COUNT nightly builds found, no cleanup needed" + fi diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-dev.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-dev.yml new file mode 100644 index 00000000..dfe346b2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-dev.yml @@ -0,0 +1,122 @@ +name: Build and Push Development Docker Images + +on: + workflow_dispatch: + schedule: + - cron: "0 0 * * *" + +jobs: + build-dev: + if: ${{ github.repository == 'sgl-project/sglang' }} + runs-on: ${{ matrix.runner }} + strategy: + matrix: + include: + - runner: x64-docker-build-node + platform: linux/amd64 + build_type: all + grace_blackwell: 0 + tag: dev-x86 + version: 12.9.1 + - runner: arm-docker-build-node + platform: linux/arm64 + build_type: all + grace_blackwell: 1 + tag: dev-arm64 + version: 12.9.1 + steps: + - name: Delete huge unnecessary tools folder + run: rm -rf /opt/hostedtoolcache + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Free disk space + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + docker-images: true + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and Push Dev Image + run: | + docker buildx build \ + --platform ${{ matrix.platform }} \ + --push \ + -f docker/Dockerfile \ + --build-arg CUDA_VERSION=${{ matrix.version }} \ + --build-arg BUILD_TYPE=${{ matrix.build_type }} \ + --build-arg CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) \ + --build-arg GRACE_BLACKWELL=${{ matrix.grace_blackwell }} \ + --build-arg USE_LATEST_SGLANG=1 \ + --build-arg INSTALL_FLASHINFER_JIT_CACHE=1 \ + -t lmsysorg/sglang:${{ matrix.tag }} \ + --no-cache \ + . + + create-manifests: + runs-on: ubuntu-22.04 + needs: [build-dev] + if: ${{ github.repository == 'sgl-project/sglang' }} + strategy: + matrix: + variant: + - tag: dev + x86_tag: dev-x86 + arm64_tag: dev-arm64 + steps: + - uses: docker/setup-buildx-action@v3 + + - uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - run: | + SHORT_SHA="${{ github.sha }}" + docker buildx imagetools create \ + -t lmsysorg/sglang:${{ matrix.variant.tag }} \ + -t lmsysorg/sglang:nightly-${{ matrix.variant.tag }}-$(date +%Y%m%d)-${SHORT_SHA:0:8} \ + lmsysorg/sglang:${{ matrix.variant.x86_tag }} \ + lmsysorg/sglang:${{ matrix.variant.arm64_tag }} + + - name: Cleanup Old Nightly Builds + run: | + # Get JWT token for Docker Hub API + TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d '{"username": "${{ secrets.DOCKERHUB_USERNAME }}", "password": "${{ secrets.DOCKERHUB_TOKEN }}"}' https://hub.docker.com/v2/users/login/ | jq -r .token) + + # Get all tags for the repository + TAGS_RESPONSE=$(curl -s -H "Authorization: JWT $TOKEN" "https://hub.docker.com/v2/repositories/lmsysorg/sglang/tags/?page_size=100") + + # Extract tags that match our pattern and sort by last_updated timestamp (most recent first) + TAGS=$(echo "$TAGS_RESPONSE" | jq -r '.results[] | select(.name | startswith("nightly-${{ matrix.variant.tag }}-")) | "\(.last_updated)|\(.name)"' | sort -r | cut -d'|' -f2) + + # Count total tags and keep only the 14 most recent + TAG_COUNT=$(echo "$TAGS" | wc -l) + if [ "$TAG_COUNT" -gt 14 ]; then + echo "Found $TAG_COUNT nightly builds, keeping only the 14 most recent" + TAGS_TO_DELETE=$(echo "$TAGS" | tail -n +15) + echo "Tags to delete: $TAGS_TO_DELETE" + + # Delete old tags + for tag in $TAGS_TO_DELETE; do + echo "Deleting tag: $tag" + curl -X DELETE \ + -H "Authorization: JWT $TOKEN" \ + "https://hub.docker.com/v2/repositories/lmsysorg/sglang/tags/$tag/" + done + else + echo "Only $TAG_COUNT nightly builds found, no cleanup needed" + fi diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-gateway.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-gateway.yml new file mode 100644 index 00000000..d1061333 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-gateway.yml @@ -0,0 +1,30 @@ +name: Release SGLang Model Gateway Docker Image +on: + push: + branches: + - main + paths: + - "sgl-router/bindings/python/sglang_router/version.py" + workflow_dispatch: + +jobs: + publish: + if: github.repository == 'sgl-project/sglang' + runs-on: ubuntu-24.04 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and Push + run: | + version=$(cat sgl-router/bindings/python/sglang_router/version.py | cut -d'"' -f2) + tag=v${version} + + docker build . -f docker/gateway.Dockerfile -t lmsysorg/sglang-router:${tag} --no-cache + docker push lmsysorg/sglang-router:${tag} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-npu-nightly.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-npu-nightly.yml new file mode 100644 index 00000000..1873c2c3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-npu-nightly.yml @@ -0,0 +1,78 @@ +name: Release Docker Images Nightly (NPU) +on: + pull_request: + branches: + - main + paths: + - ".github/workflows/release-docker-npu-nightly.yml" + - "docker/npu.Dockerfile" + workflow_dispatch: + schedule: + - cron: "0 0 * * *" + +concurrency: + group: ${{ github.workflow }}-${{ github.sha }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-22.04-arm + strategy: + matrix: + cann_version: ["8.3.rc1"] + device_type: ["910b", "a3"] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Free up disk space + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + with: + tool-cache: true + docker-images: false + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + lmsysorg/sglang + # push with schedule event + # push with workflow_dispatch event + tags: | + type=ref,event=pr + type=ref,event=branch + type=schedule,pattern=main + flavor: | + latest=false + suffix=-cann${{ matrix.cann_version }}-${{ matrix.device_type }},onlatest=true + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Log into docker hub + uses: docker/login-action@v3 + if: ${{ github.repository == 'sgl-project/sglang' && github.event_name != 'pull_request' }} + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v6 + with: + context: docker + file: docker/npu.Dockerfile + # TODO: need add x86 platforms support when memfabric is ready + platforms: linux/arm64 + labels: ${{ steps.meta.outputs.labels }} + tags: ${{ steps.meta.outputs.tags }} + push: ${{ github.repository == 'sgl-project/sglang' && github.event_name != 'pull_request' }} + provenance: false + build-args: | + SGLANG_KERNEL_NPU_TAG=20251128 + CANN_VERSION=${{ matrix.cann_version }} + DEVICE_TYPE=${{ matrix.device_type }} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-npu.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-npu.yml new file mode 100644 index 00000000..dd054c9c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-npu.yml @@ -0,0 +1,75 @@ +name: Release Docker Images (NPU) +on: + push: + tags-ignore: + - "gateway-*" # Exclude gateway/router tags + - "router-*" # Exclude router tags + pull_request: + branches: + - main + paths: + - ".github/workflows/release-docker-npu.yml" + - "docker/npu.Dockerfile" + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-22.04-arm + strategy: + matrix: + cann_version: ["8.3.rc1"] + device_type: ["910b", "a3"] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Free up disk space + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + with: + tool-cache: true + docker-images: false + + # push with tag + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + lmsysorg/sglang + tags: | + type=ref,event=pr + type=ref,event=tag,suffix=-cann${{ matrix.cann_version }}-${{ matrix.device_type }} + flavor: | + latest=false + + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Login to Docker Hub + uses: docker/login-action@v2 + if: ${{ github.repository == 'sgl-project/sglang' && github.event_name != 'pull_request' }} + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Get version + id: get_version + run: | + version=$(cat python/sglang/version.py | cut -d'"' -f2) + echo "TAG=lmsysorg/sglang:v$version-cann${{ matrix.cann_version }}-${{ matrix.device_type }}" >> $GITHUB_OUTPUT + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v6 + with: + context: docker + file: docker/npu.Dockerfile + # TODO: need add x86 platforms support when memfabric is ready + platforms: linux/arm64 + labels: ${{ steps.meta.outputs.labels }} + tags: ${{ steps.meta.outputs.tags || steps.get_version.outputs.TAG }} + push: ${{ github.repository == 'sgl-project/sglang' && github.event_name != 'pull_request' }} + provenance: false + build-args: | + SGLANG_KERNEL_NPU_TAG=20251128 + CANN_VERSION=${{ matrix.cann_version }} + DEVICE_TYPE=${{ matrix.device_type }} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-xeon.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-xeon.yml new file mode 100644 index 00000000..60e24933 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker-xeon.yml @@ -0,0 +1,35 @@ +name: Release Docker Xeon Images +on: + push: + branches: + - main + paths: + - "python/sglang/version.py" + workflow_dispatch: + +jobs: + publish: + if: github.repository == 'sgl-project/sglang' + runs-on: ubuntu-24.04 + environment: 'prod' + strategy: + matrix: + build_type: ['all'] + steps: + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and Push + run: | + version=$(cat python/sglang/version.py | cut -d'"' -f2) + tag=v${version}-xeon + + docker build . -f docker/xeon.Dockerfile -t lmsysorg/sglang:${tag} --no-cache + docker push lmsysorg/sglang:${tag} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker.yml new file mode 100644 index 00000000..59603385 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docker.yml @@ -0,0 +1,138 @@ +name: Release Docker Images +on: + push: + branches: + - main + paths: + - "python/sglang/version.py" + workflow_dispatch: + +jobs: + publish-x86: + if: github.repository == 'sgl-project/sglang' + environment: "prod" + strategy: + matrix: + variant: + - cuda_version: "12.9.1" + build_type: "all" + grace_blackwell: 0 + runs-on: x64-docker-build-node + steps: + - name: Delete huge unnecessary tools folder + run: rm -rf /opt/hostedtoolcache + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Free disk space + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + docker-images: false + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and Push AMD64 + run: | + version=$(cat python/sglang/version.py | cut -d'"' -f2) + tag=v${version}-cu129-amd64 + + docker buildx build \ + --platform linux/amd64 \ + --push \ + -f docker/Dockerfile \ + --build-arg CUDA_VERSION=${{ matrix.variant.cuda_version }} \ + --build-arg BUILD_TYPE=${{ matrix.variant.build_type }} \ + --build-arg GRACE_BLACKWELL=${{ matrix.variant.grace_blackwell }} \ + -t lmsysorg/sglang:${tag} \ + --no-cache \ + . + + publish-arm64: + if: github.repository == 'sgl-project/sglang' + environment: "prod" + strategy: + matrix: + variant: + - cuda_version: "12.9.1" + build_type: "all" + grace_blackwell: 1 + runs-on: arm-docker-build-node + steps: + - name: Delete huge unnecessary tools folder + run: rm -rf /opt/hostedtoolcache + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and Push ARM64 + run: | + version=$(cat python/sglang/version.py | cut -d'"' -f2) + tag=v${version}-cu129-arm64 + + docker buildx build \ + --platform linux/arm64 \ + --push \ + -f docker/Dockerfile \ + --build-arg CUDA_VERSION=${{ matrix.variant.cuda_version }} \ + --build-arg BUILD_TYPE=${{ matrix.variant.build_type }} \ + --build-arg GRACE_BLACKWELL=${{ matrix.variant.grace_blackwell }} \ + -t lmsysorg/sglang:${tag} \ + --no-cache \ + . + + create-manifests: + runs-on: ubuntu-22.04 + needs: [publish-x86, publish-arm64] + if: github.repository == 'sgl-project/sglang' + environment: "prod" + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Create multi-arch manifests + run: | + version=$(cat python/sglang/version.py | cut -d'"' -f2) + + # Create versioned manifest + docker buildx imagetools create \ + -t lmsysorg/sglang:v${version} \ + lmsysorg/sglang:v${version}-cu129-amd64 \ + lmsysorg/sglang:v${version}-cu129-arm64 + + # Create latest manifest + docker buildx imagetools create \ + -t lmsysorg/sglang:latest \ + lmsysorg/sglang:v${version}-cu129-amd64 \ + lmsysorg/sglang:v${version}-cu129-arm64 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docs.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docs.yml new file mode 100644 index 00000000..78fafc60 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-docs.yml @@ -0,0 +1,65 @@ +name: Release Documentation + +on: + push: + branches: + - main + paths: + - "docs/**" + - "python/sglang/version.py" + - "python/sglang/**" + workflow_dispatch: + +concurrency: + group: release-docs-${{ github.ref }} + cancel-in-progress: true + +jobs: + execute-and-deploy: + runs-on: 1-gpu-runner + if: github.repository == 'sgl-project/sglang' + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + pip install -r docs/requirements.txt + apt-get update && apt-get install -y pandoc parallel retry + ln -sf "$(which python3)" /usr/bin/python + + - name: Setup Jupyter Kernel + run: | + python -m ipykernel install --user --name python3 --display-name "Python 3" + + - name: Execute notebooks + timeout-minutes: 40 + run: | + cd docs + make clean + make compile + + - name: Push HTML to sgl-project.github.io + timeout-minutes: 30 + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_DOCUMENTATION }} + run: | + cd docs + make html + python3 wrap_run_llm.py + + cd _build/html + + git clone https://$GITHUB_TOKEN@github.com/sgl-project/sgl-project.github.io.git ../sgl-project.github.io --depth 1 + find ../sgl-project.github.io/ -mindepth 1 -not -path "../sgl-project.github.io/.git*" -not -name CNAME -not -name ".jekyll" -not -name ".nojekyll" -delete + cp -r * ../sgl-project.github.io + cp ../../README.md ../sgl-project.github.io/README.md + cd ../sgl-project.github.io + git config user.name "sglang-bot" + git config user.email "sglangbot@gmail.com" + git add . + git commit -m "Update $(date +'%Y-%m-%d %H:%M:%S')" + git push https://$GITHUB_TOKEN@github.com/sgl-project/sgl-project.github.io.git main + cd .. + rm -rf sgl-project.github.io diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-fake-tag.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-fake-tag.yml new file mode 100644 index 00000000..d1acc6bf --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-fake-tag.yml @@ -0,0 +1,35 @@ +name: Release Fake Tag +on: + push: + branches: + - main + paths: + - "python/sglang/version.py" + workflow_dispatch: + +permissions: + contents: write + +jobs: + publish: + if: github.repository == 'sgl-project/sglang' + runs-on: ubuntu-latest + environment: 'prod' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Get version + id: get_version + run: | + version=$(cat python/sglang/version.py | cut -d'"' -f2) + echo "TAG=v$version" >> $GITHUB_OUTPUT + + - name: Create and push tag + run: | + git config user.name "sglang-bot" + git config user.email "sglang-bot@users.noreply.github.com" + git tag ${{ steps.get_version.outputs.TAG }} + git push origin ${{ steps.get_version.outputs.TAG }} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-pypi-gateway.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-pypi-gateway.yml new file mode 100644 index 00000000..0f051faa --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-pypi-gateway.yml @@ -0,0 +1,167 @@ +name: Release SGLang Model Gateway to PyPI + +on: + push: + branches: + - main + paths: + - sgl-router/bindings/python/pyproject.toml + workflow_dispatch: + +jobs: + build: + name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }}) + runs-on: ${{ matrix.os }}-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu, macos, windows] + target: [x86_64, aarch64] + manylinux: [auto] + include: + - os: ubuntu + platform: linux + - os: windows + ls: dir + target: x86_64 + python-architecture: x64 + interpreter: 3.9 3.10 3.11 3.12 3.13 + - os: macos + target: aarch64 + interpreter: 3.9 3.10 3.11 3.12 3.13 + - os: ubuntu + platform: linux + target: aarch64 + # musllinux + - os: ubuntu + platform: linux + target: x86_64 + manylinux: musllinux_1_1 + - os: ubuntu + platform: linux + target: aarch64 + manylinux: musllinux_1_1 + exclude: + - os: windows + target: aarch64 + + steps: + - uses: actions/checkout@v4 + with: + path: sglang-repo + + - name: Move sgl-router folder to root and delete sglang-repo + run: | + mv sglang-repo/sgl-router/* . + rm -rf sglang-repo + ls -alt + shell: bash + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + architecture: ${{ matrix.python-architecture || 'x64' }} + + - name: Install twine + run: pip install -U twine + + - name: Install protoc (macOS) + if: matrix.os == 'macos' + run: brew install protobuf + + - name: Install protoc (Windows) + if: matrix.os == 'windows' + run: choco install protoc -y + + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + working-directory: bindings/python + target: ${{ matrix.target }} + manylinux: ${{ matrix.manylinux || 'auto' }} + args: --release --out dist --features vendored-openssl --interpreter ${{ matrix.interpreter || '3.9 3.10 3.11 3.12 3.13 3.14' }} + rust-toolchain: stable + docker-options: -e CI -e CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc -e CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ + before-script-linux: | + # Install build dependencies (perl/make for vendored OpenSSL, protoc for gRPC) + if command -v yum &> /dev/null; then + yum update -y && yum install -y wget unzip gcc gcc-c++ perl-core make + # Install cross-compilation toolchain for aarch64 if needed + if [ "${{ matrix.target }}" = "aarch64" ]; then + yum install -y gcc-aarch64-linux-gnu gcc-c++-aarch64-linux-gnu || true + fi + elif command -v apt-get &> /dev/null; then + apt-get update && apt-get install -y wget unzip gcc g++ perl make + # Install cross-compilation toolchain for aarch64 if needed + if [ "${{ matrix.target }}" = "aarch64" ]; then + apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu || true + fi + fi + (cd /tmp && \ + wget https://github.com/protocolbuffers/protobuf/releases/download/v32.0/protoc-32.0-linux-x86_64.zip && \ + unzip protoc-32.0-linux-x86_64.zip -d /usr/local && \ + rm protoc-32.0-linux-x86_64.zip) + protoc --version + + - name: List built packages + run: ${{ matrix.ls || 'ls -lh' }} bindings/python/dist/ + + - name: Check packages + run: twine check --strict bindings/python/dist/* + + - uses: actions/upload-artifact@v4 + with: + name: packages-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.manylinux || 'auto' }} + path: bindings/python/dist/ + + build-sdist: + name: Build SDist + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + path: sglang-repo + + - name: Move sgl-router folder to root and delete sglang-repo + run: | + mv sglang-repo/sgl-router/* . + rm -rf sglang-repo + ls -alt + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Build SDist + uses: PyO3/maturin-action@v1 + with: + working-directory: bindings/python + command: sdist + args: --out dist + rust-toolchain: stable + + - uses: actions/upload-artifact@v4 + with: + name: sdist + path: bindings/python/dist/*.tar.gz + + upload: + name: Upload to PyPI + if: github.repository == 'sgl-project/sglang' # Ensure this job only runs for the sgl-project/sglang repository + needs: [build, build-sdist] + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v4 + with: + path: dist + merge-multiple: true + + - name: Upload to PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN_ROUTER }} + run: | + pip install twine + twine upload dist/* --verbose diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-pypi.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-pypi.yml new file mode 100644 index 00000000..354ee21d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-pypi.yml @@ -0,0 +1,31 @@ +name: Release PyPI +on: + push: + branches: + - main + paths: + - "python/sglang/version.py" + workflow_dispatch: + +jobs: + publish: + if: github.repository == 'sgl-project/sglang' + runs-on: ubuntu-latest + environment: "prod" + steps: + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Upload to pypi + run: | + cd python + cp ../README.md ../LICENSE . + pip install build + python3 -m build + pip install twine + python3 -m twine upload dist/* -u __token__ -p ${{ secrets.PYPI_TOKEN }} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-whl-kernel.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-whl-kernel.yml new file mode 100644 index 00000000..4f0a33ef --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/release-whl-kernel.yml @@ -0,0 +1,197 @@ +name: Release SGLang Kernels + +on: + push: + branches: + - main + paths: + - sgl-kernel/python/sgl_kernel/version.py + workflow_dispatch: + inputs: + tag_name: + type: string + required: false + +concurrency: + group: release-sglang-kernels-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-cu129-matrix: + if: github.repository == 'sgl-project/sglang' + strategy: + matrix: + python-version: ["3.10"] + cuda-version: ["12.9"] + arch: [x86_64, aarch64] + include: + - arch: x86_64 + runner: x64-kernel-build-node + - arch: aarch64 + runner: arm-kernel-build-node + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + with: + submodules: "recursive" + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Build wheels + run: | + cd sgl-kernel + chmod +x ./build.sh + ./build.sh "${{ matrix.python-version }}" "${{ matrix.cuda-version }}" ${{ matrix.arch == 'aarch64' && 'aarch64' || '' }} + env: + USE_CCACHE: 0 + + - name: Upload to PyPI + working-directory: sgl-kernel + run: | + pip install twine + python3 -m twine upload --skip-existing dist/* -u __token__ -p ${{ secrets.PYPI_TOKEN }} + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: wheel-python${{ matrix.python-version }}-cuda${{ matrix.cuda-version }}${{ matrix.arch == 'aarch64' && '-aarch64' || '' }} + path: sgl-kernel/dist/* + + release-cu129: + needs: build-cu129-matrix + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-* + + - name: Set tag name + id: set_tag_name + run: | + if [ -z "${{ inputs.tag_name }}" ]; then + TAG_NAME="v$(cat sgl-kernel/python/sgl_kernel/version.py | cut -d'"' -f2)" + echo "tag_name=$TAG_NAME" >> $GITHUB_OUTPUT + else + echo "tag_name=${{ inputs.tag_name }}" >> $GITHUB_OUTPUT + fi + + - name: Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ steps.set_tag_name.outputs.tag_name }} + repository: sgl-project/whl + token: ${{ secrets.WHL_TOKEN }} + files: | + sgl-kernel/dist/* + + - name: Clone wheel index + run: git clone https://oauth2:${WHL_TOKEN}@github.com/sgl-project/whl.git sgl-whl + env: + WHL_TOKEN: ${{ secrets.WHL_TOKEN }} + + - name: Update wheel index + run: python3 scripts/update_kernel_whl_index.py --cuda 129 + + - name: Push wheel index + run: | + cd sgl-whl + git config --local user.name "sglang-bot" + git config --local user.email "sglangbot@gmail.com" + git add -A + git commit -m "update whl index" + git push + + # for now we do not release CUDA 13.0 wheels to pypi + build-cu130-matrix: + if: github.repository == 'sgl-project/sglang' + strategy: + matrix: + python-version: ["3.10"] + cuda-version: ["13.0"] + arch: [x86_64, aarch64] + include: + - arch: x86_64 + runner: x64-kernel-build-node + - arch: aarch64 + runner: arm-kernel-build-node + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + with: + submodules: "recursive" + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Build wheels + run: | + cd sgl-kernel + chmod +x ./build.sh + ./build.sh "${{ matrix.python-version }}" "${{ matrix.cuda-version }}" ${{ matrix.arch == 'aarch64' && 'aarch64' || '' }} + env: + USE_CCACHE: 0 + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: wheel-python${{ matrix.python-version }}-cuda${{ matrix.cuda-version }}${{ matrix.arch == 'aarch64' && '-aarch64' || '' }} + path: sgl-kernel/dist/* + + release-cu130: + needs: build-cu130-matrix + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-* + + - name: Set tag name + id: set_tag_name + run: | + if [ -z "${{ inputs.tag_name }}" ]; then + TAG_NAME="v$(cat sgl-kernel/python/sgl_kernel/version.py | cut -d'"' -f2)" + echo "tag_name=$TAG_NAME" >> $GITHUB_OUTPUT + else + echo "tag_name=${{ inputs.tag_name }}" >> $GITHUB_OUTPUT + fi + + - name: Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ steps.set_tag_name.outputs.tag_name }} + repository: sgl-project/whl + token: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }} + files: | + sgl-kernel/dist/* + + - name: Clone wheel index + run: git clone https://oauth2:${WHL_TOKEN}@github.com/sgl-project/whl.git sgl-whl + env: + WHL_TOKEN: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }} + + - name: Update wheel index + run: python3 scripts/update_kernel_whl_index.py --cuda 130 + + - name: Push wheel index + run: | + cd sgl-whl + git config --local user.name "sglang-bot" + git config --local user.email "sglangbot@gmail.com" + git add -A + git commit -m "update whl index" + git push diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/slash-command-handler.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/slash-command-handler.yml new file mode 100644 index 00000000..9ba79953 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/slash-command-handler.yml @@ -0,0 +1,46 @@ +name: Slash Command Handler + +on: + issue_comment: + types: [created, edited] + +permissions: + contents: read + pull-requests: write # Required to add labels and reactions + actions: write # Required to rerun workflows + issues: write # Required for comment reactions in some contexts + +jobs: + slash_command: + # Only run if it is a PR and the comment starts with a recognized command + if: > + github.event.issue.pull_request && + (startsWith(github.event.comment.body, '/tag-run-ci-label') || + startsWith(github.event.comment.body, '/rerun-failed-ci') || + startsWith(github.event.comment.body, '/tag-and-rerun-ci') || + startsWith(github.event.comment.body, '/rerun-stage')) + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + pip install PyGithub + + - name: Handle Slash Command + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO_FULL_NAME: ${{ github.repository }} + PR_NUMBER: ${{ github.event.issue.number }} + COMMENT_ID: ${{ github.event.comment.id }} + COMMENT_BODY: ${{ github.event.comment.body }} + USER_LOGIN: ${{ github.event.comment.user.login }} + run: | + python scripts/ci/slash_command_handler.py diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/stress-test.yml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/stress-test.yml new file mode 100644 index 00000000..4e6bc741 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.github/workflows/stress-test.yml @@ -0,0 +1,44 @@ +name: Stress Test + +on: + workflow_dispatch: + inputs: + num_prompts: + description: 'Number of prompts per model' + required: true + default: '50000' + type: string + duration_minutes: + description: 'Timeout per model in minutes' + required: true + default: '45' + type: string + +jobs: + stress-test: + if: github.repository == 'sgl-project/sglang' + runs-on: 8-gpu-h200 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + + - name: Run stress tests + timeout-minutes: 210 + env: + NUM_PROMPTS: ${{ inputs.num_prompts }} + DURATION_MINUTES: ${{ inputs.duration_minutes }} + run: | + cd test + python3 run_suite.py --hw cuda --suite stress + + - name: Upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: stress-test-results + path: | + stress_test_*.jsonl diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.gitignore b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.gitignore new file mode 100644 index 00000000..118dd9ae --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.gitignore @@ -0,0 +1,246 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ + +# Tokenizer cache for tests +.tokenizer_cache/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ + +# MacOS +.DS_Store + +# Vim +*.swp + +# Documentation +docs/_build + +# SGL +benchmark/mmlu/data +benchmark/mmlu/data.tar +benchmark/llava_bench/images +benchmark/llava_bench/mme_pack +*.jsonl +tmp*.txt + +# Torch Compile logs +tl_out/ + +# Plots +*.png +*.pdf + +# personnal +work_dirs/ +*.csv + +!logo.png + +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +compile_commands.json + +*.iml + +# VSCode +.vscode + +1 + +# Autoenv +.env.leave + +# Rust lib +Cargo.lock + +lmms-eval + +**/.claude/ +**/.serena/ diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.isort.cfg b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.isort.cfg new file mode 100644 index 00000000..83509515 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.isort.cfg @@ -0,0 +1,3 @@ +[settings] +profile=black +known_first_party=sglang diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.pre-commit-config.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.pre-commit-config.yaml new file mode 100644 index 00000000..5ddd480f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/.pre-commit-config.yaml @@ -0,0 +1,77 @@ +default_stages: [pre-commit, pre-push, manual] +exclude: ^python/sglang/multimodal_gen/csrc + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-symlinks + - id: destroyed-symlinks + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + args: [--allow-multiple-documents] + - id: check-toml + - id: check-ast + - id: check-added-large-files + - id: check-merge-conflict + - id: check-shebang-scripts-are-executable + - id: detect-private-key + - id: debug-statements + - id: no-commit-to-branch + - repo: https://github.com/PyCQA/isort + rev: 5.13.2 + hooks: + - id: isort + exclude: '^python/sglang/srt/grpc/.*_pb2\.py$|^python/sglang/srt/grpc/.*_pb2_grpc\.py$|^python/sglang/srt/grpc/.*_pb2\.pyi$|^python/sglang/srt/grpc/.*_pb2_grpc\.pyi$' + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.11.7 + hooks: + - id: ruff + args: + - --select=F401,F821 + - --fix + files: ^(benchmark/|docs/|examples/|python/sglang/|sgl-router/py_*|test/) + exclude: | + (?x)^( + .*/__init__\.py$| + .*\.ipynb$| + python/sglang/srt/grpc/.*_pb2\.py$| + python/sglang/srt/grpc/.*_pb2_grpc\.py$| + python/sglang/srt/grpc/.*_pb2\.pyi$| + python/sglang/srt/grpc/.*_pb2_grpc\.pyi$| + )$ + - repo: https://github.com/psf/black + rev: 24.10.0 + hooks: + - id: black-jupyter + exclude: '^python/sglang/srt/grpc/.*_pb2\.py$|^python/sglang/srt/grpc/.*_pb2_grpc\.py$|^python/sglang/srt/grpc/.*_pb2\.pyi$|^python/sglang/srt/grpc/.*_pb2_grpc\.pyi$' + - repo: https://github.com/codespell-project/codespell + rev: v2.4.1 + hooks: + - id: codespell + additional_dependencies: ['tomli'] + args: ['--toml', 'python/pyproject.toml', '-L', 'cann,thi,makro,wil,rouge,PRIS'] + exclude: | + (?x)^( + test/srt/test_reasoning_parser\.py| + docs/advanced_features/vlm_query\.ipynb| + python/sglang/srt/grpc/.*_pb2\.py| + python/sglang/srt/grpc/.*_pb2_grpc\.py| + python/sglang/srt/grpc/.*_pb2\.pyi| + python/sglang/srt/grpc/.*_pb2_grpc\.pyi| + sgl-router/src/tokenizer/chat_template\.rs + )$ + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: v18.1.8 + hooks: + - id: clang-format + types_or: [c++, cuda] + args: [--style=file, --verbose] + - repo: https://github.com/kynan/nbstripout + rev: 0.8.1 + hooks: + - id: nbstripout + args: + - '--keep-output' + - '--extra-keys=metadata.kernelspec metadata.language_info.version' diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/PROFILING.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/PROFILING.md new file mode 100644 index 00000000..7e15ec84 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/PROFILING.md @@ -0,0 +1,425 @@ +## Profiling SGLang Infer System with AMD GPUs +This AppNote describes the SGLang profiling technical, code augment and running steps for systems with AMD Instinct GPUs, nevertheless the same procedure may work with Nvidia GPUs too. +Examples and steps are provided in detail, to facilitate easy reproduce and use to localize performance problem towards optimizations. +Two primary methods are covered: +- [RPD](https://github.com/ROCm/rocmProfileData.git) +- [PyTorch Profiler](https://pytorch.org/tutorials/recipes/recipes/profiler_recipe.html) + +### Profiling SGLang Infer System with RPD Profiler +RPD profiler is a low-overhead cross-platform profiler. Therefore, the same RPD code augment not only works for profiling on ROCm/AMD GPUs, but also works for profiling on CUDA/Nvidia GPUs as well. To do RPD profiling on SGLang repository, please use scripts and patch files included in this directory and follow the steps below: +1. Install RPD with rpd.patch applied during installation using install_rpd.sh, both files are in this directory. + +install_rpd.sh + +```bash +# download and install RPD +apt update && apt install -y sqlite3 libsqlite3-dev libfmt-dev + +# install rpd module +git clone https://github.com/ROCmSoftwarePlatform/rocmProfileData +cd rocmProfileData +git checkout 976899e9c6dbc6dd2bccf770818e4e44125590ac +git apply rpd.patch +make && make install +cd rocpd_python && python setup.py install && cd .. +cd rpd_tracer && make clean;make install && python setup.py install && cd .. +``` + +rpd.patch + +```bash +diff --git a/rpd_tracer/Makefile b/rpd_tracer/Makefile +index e9d9feb..b2e9e1a 100644 +--- a/rpd_tracer/Makefile ++++ b/rpd_tracer/Makefile +@@ -16,7 +16,7 @@ ifneq (,$(HIP_PATH)) + $(info Building with roctracer) + RPD_LIBS += -L/opt/rocm/lib -lroctracer64 -lroctx64 -lamdhip64 -lrocm_smi64 + RPD_INCLUDES += -I/opt/rocm/include -I/opt/rocm/include/roctracer -I/opt/rocm/include/hsa +- RPD_SRCS += RoctracerDataSource.cpp RocmSmiDataSource.cpp ++ RPD_SRCS += RoctracerDataSource.cpp + RPD_INCLUDES += -D__HIP_PLATFORM_AMD__ + endif +``` +2. Add loadTracer.sh file included in this directory to /sglang/python/sglang. + +loadTracer.sh + +```bash +#!/bin/bash +################################################################################ +# Copyright (c) 2021 - 2023 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +################################################################################ +OUTPUT_FILE="trace.rpd" + +if [ "$1" = "-o" ] ; then + OUTPUT_FILE=$2 + shift + shift +fi + +if [ -e ${OUTPUT_FILE} ] ; then + rm ${OUTPUT_FILE} +fi + +python3 -m rocpd.schema --create ${OUTPUT_FILE} +if [ $? != 0 ] ; then + echo "Error: Could not create rpd file. Please run 'python setup.py install' from the rocpd_python dir" + exit +fi + +export RPDT_FILENAME=${OUTPUT_FILE} +export RPDT_AUTOSTART=0 +LD_PRELOAD=librocm-smi_64:librpd_tracer.so "$@" +``` +3. Apply patch (provided in this directory) with "git apply rpd_profile_server_enable.patch" if the main profiling purpose is to get info on gpu kernels as well as limited cpu activity info. + +#### Common Notes 1 +Please note that although we are doing TP=8 in the example, we purposely only log RPD profiling on 2 ranks in the patch file (i.e.tp_rank=0/1) for profiling/visualization convenience, as even Perfetto streaming mode can only load maximal 8GB json file for visualization. With 2 ranks logged in RPD profiling, we could still check whether there are issues among ranks (e.g. load imbalance issue, nccl issue), and at the same time, we could log relatively longer time duration before the json file generated from RPD file hits 8GB size. + +rpd_profile_server_enable.patch + +```bash +diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py +index 62d1ff9..9021c01 100644 +--- a/python/sglang/srt/managers/scheduler.py ++++ b/python/sglang/srt/managers/scheduler.py +@@ -71,6 +71,8 @@ from sglang.srt.utils import ( + suppress_other_loggers, + ) + from sglang.utils import get_exception_traceback ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() + + logger = logging.getLogger(__name__) + +@@ -245,6 +247,7 @@ class Scheduler: + ], + with_stack=True, + ) ++ self.rpd = rpdTracerControl() + + @torch.inference_mode() + def event_loop(self): +@@ -1027,15 +1030,24 @@ class Scheduler: + def start_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.start() ++ #self.profiler.start() #block pytorch profiler for rpd profiler enabling ++ if self.tp_rank == 0 or self.tp_rank == 1: ++ self.rpd.start() ++ self.rpd.rangePush("", "rpd profile range", "") ++ logger.info("rpd is enabled") + + def stop_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.stop() +- self.profiler.export_chrome_trace( +- self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" +- ) ++ #self.profiler.stop() ++ #self.profiler.export_chrome_trace( ++ # self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" ++ #) ++ if self.tp_rank ==0 or self.tp_rank ==1: ++ self.rpd.rangePop() ++ self.rpd.stop() ++ self.rpd.flush() ++ logger.info("rpd is done") + logger.info("Profiler is done") +``` + +#### Advanced Debugging with RPD Profiler +Sometimes, we want to use rpd profiler to capture more CPU and python activities in order to debug some challenging issues (e.g. root cause of load imbalance across gpu processes, root cause of bubbles, etc). Only in such cases, we need to apply patch "git apply rpd_profile_server_enable_wCPU_activities.patch", where 3 files are modified. + +rpd_profile_server_enable_wCPU_activities.patch + +```bash +diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py +index 62d1ff9..2edb427 100644 +--- a/python/sglang/srt/managers/scheduler.py ++++ b/python/sglang/srt/managers/scheduler.py +@@ -71,6 +71,8 @@ from sglang.srt.utils import ( + suppress_other_loggers, + ) + from sglang.utils import get_exception_traceback ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() + + logger = logging.getLogger(__name__) + +@@ -245,6 +247,7 @@ class Scheduler: + ], + with_stack=True, + ) ++ self.rpd = rpdTracerControl() + + @torch.inference_mode() + def event_loop(self): +@@ -1027,15 +1030,26 @@ class Scheduler: + def start_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.start() ++ #self.profiler.start() ++ logger.info("torch profiler is disabled") ++ if self.tp_rank == 0 or self.tp_rank == 1: ++ self.rpd.setPythonTrace(True) ++ self.rpd.start() ++ self.rpd.rangePush("", "scheduler", "") ++ logger.info("rpd is enabled inside scheduler profiling") + + def stop_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.stop() +- self.profiler.export_chrome_trace( +- self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" +- ) ++ #self.profiler.stop() ++ #self.profiler.export_chrome_trace( ++ # self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" ++ #) ++ if self.tp_rank ==0 or self.tp_rank ==1: ++ self.rpd.rangePop() ++ self.rpd.stop() ++ self.rpd.flush() ++ logger.info("rpd is done inside scheduler") + logger.info("Profiler is done") + + +diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py +index 2621ccd..181df85 100644 +--- a/python/sglang/srt/managers/tokenizer_manager.py ++++ b/python/sglang/srt/managers/tokenizer_manager.py +@@ -58,6 +58,10 @@ from sglang.srt.sampling.sampling_params import SamplingParams + from sglang.srt.server_args import PortArgs, ServerArgs + from sglang.srt.utils import is_generation_model, is_multimodal_model + ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() ++ ++ + asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) + + logger = logging.getLogger(__name__) +@@ -514,10 +518,20 @@ class TokenizerManager: + self.send_to_scheduler.send_pyobj(req) + + def start_profile(self): ++ rpd = rpdTracerControl() ++ rpd.setPythonTrace(True) ++ rpd.start() ++ rpd.rangePush("", "tokenizer_manager", "") ++ logger.info("tokenizer_manager rpd profiling started!") + req = ProfileReq.START_PROFILE + self.send_to_scheduler.send_pyobj(req) + + def stop_profile(self): ++ rpd = rpdTracerControl() ++ rpd.rangePop() ++ rpd.stop() ++ rpd.flush() ++ logger.info("rpd profiling is done inside tokenizer_manager!") + req = ProfileReq.STOP_PROFILE + self.send_to_scheduler.send_pyobj(req) + +diff --git a/python/sglang/srt/server.py b/python/sglang/srt/server.py +index 7111c93..2bd722c 100644 +--- a/python/sglang/srt/server.py ++++ b/python/sglang/srt/server.py +@@ -30,6 +30,8 @@ import threading + import time + from http import HTTPStatus + from typing import Dict, List, Optional, Union ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() + + # Fix a bug of Python threading + setattr(threading, "_register_atexit", lambda *args, **kwargs: None) +@@ -152,6 +154,11 @@ async def flush_cache(): + @app.post("/start_profile") + async def start_profile(): + """Start profiling.""" ++ rpd = rpdTracerControl() ++ rpd.setPythonTrace(True) ++ rpd.start() ++ rpd.rangePush("", "server rpd profile range", "") ++ logger.info("rpd profiling started in server.py!") + tokenizer_manager.start_profile() + return Response( + content="Start profiling.\n", +@@ -164,6 +171,11 @@ async def start_profile(): + async def stop_profile(): + """Stop profiling.""" + tokenizer_manager.stop_profile() ++ rpd = rpdTracerControl() ++ rpd.rangePop() ++ rpd.stop() ++ rpd.flush() ++ logger.info("rpd profiling is done in server.py!") + return Response( + content="Stop profiling. This will take some time.\n", + status_code=200, +``` + +4. As an example for grok1 profiling, we create a dummy_grok1 directory with config.json (see content below) inside this directory and copy this directory to the right path for "--model-path" if you want to use the example server.sh file provided. +```bash +cat ../dummy_grok1/config.json +{ + "architectures": [ + "Grok1ModelForCausalLM" + ], + "embedding_multiplier_scale": 78.38367176906169, + "output_multiplier_scale": 0.5773502691896257, + "vocab_size": 131072, + "hidden_size": 6144, + "intermediate_size": 32768, + "max_position_embeddings": 8192, + "num_experts_per_tok": 2, + "num_local_experts": 8, + "num_attention_heads": 48, + "num_hidden_layers": 64, + "num_key_value_heads": 8, + "head_dim": 128, + "rms_norm_eps": 1e-05, + "rope_theta": 10000.0, + "model_type": "mixtral", + "torch_dtype": "bfloat16" +} +``` +5. Launch server with rpd enabled script ./server.sh in one terminal inside the docker container. + +#### Common Notes 2 +- Remember to change model-path to the correct path +- loadTracer.sh is needed to conduct profiling +- SGLANG_TORCH_PROFILER_DIR is used for default torch profiler +- Do not use loadTracer.sh if you are using the torch profiler, simply use python3 -m sglang.launch_server. + + +server.sh + +```bash +#!/bin/bash + +# export SGLANG_TORCH_PROFILER_DIR=/data/sglang/ +export SGLANG_TORCH_PROFILER_DIR=/sgl-workspace/sglang/profile/ + +# Get the current timestamp +TIMESTAMP=$(date +"%Y%m%d_%H%M%S") + +# Define the log file with a timestamp +LOGFILE="sglang_server_log_$TIMESTAMP.json" + +# Run the Python command and save the output to the log file +loadTracer.sh python3 -m sglang.launch_server \ + --model-path /sgl-workspace/sglang/dummy_grok1 \ + --tokenizer-path Xenova/grok-1-tokenizer \ + --load-format dummy \ + --quantization fp8 \ + --tp 8 \ + --port 30000 \ + --disable-radix-cache 2>&1 | tee "$LOGFILE" +``` +6. Open another terminal for the same docker container, and run the rpd enabled ./client.sh after you see "The server is fired up and is ready to roll!" message from server side terminal. + +#### Common Notes 3 +- Use curl http://localhost:30000/start_profile & curl http://localhost:30000/stop_profile to control the start and end of profiling. Check sglang/python/sglang/srt/managers/scheduler.py for more details. +- Please don't use RPD profiler together with PyTorch profiler to avoid interference. +- The rocmProfileData/tools/rpd2tracing.py file is used to generate json file from RPD file. + +client.sh + +```bash +#!/bin/bash + +# Start profiling via API +curl http://localhost:30000/start_profile -H "Content-Type: application/json" + +# Benchmark serving using sglang with random dataset and tokenizer +# Define the log file with a timestamp +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +LOGFILE="sglang_client_log_$TIMESTAMP.json" + +# Run the benchmark with specified parameters and save logs +python3 -m sglang.bench_serving \ + --backend sglang \ + --tokenizer Xenova/grok-1-tokenizer \ + --dataset-name random \ + --random-input 1024\ + --random-output 1024 \ + --num-prompts 120 \ + --request-rate 8 \ + --output-file online.jsonl 2>&1 | tee "$LOGFILE" + +# Stop profiling via API +curl http://localhost:30000/stop_profile -H "Content-Type: application/json" + +# Convert tracing file to csv & json +sqlite3 trace.rpd ".mode csv" ".header on" ".output trace.csv" "select * from top;" ".output stdout" +python3 ./rocmProfileData/tools/rpd2tracing.py trace.rpd trace.json +``` +7. Follow [Perfetto docs](https://perfetto.dev/docs/visualization/large-traces) to visualize large json files. Try to adjust parameters so that the trace.json file size is less than 9GB. + +### Profiling SGLang Infer System with PyTorch Profiler + +Please use the steps as follows: + +1. Apply the patch torch_profiler.patch. Note that you can modify "if self.tp_rank == 0" in the patch to allow more ranks be recorded in profiling. + +torch_profiler.patch +```bash +diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py +index 62d1ff9..6ecd78c 100644 +--- a/python/sglang/srt/managers/scheduler.py ++++ b/python/sglang/srt/managers/scheduler.py +@@ -240,7 +240,6 @@ class Scheduler: + ) + self.profiler = torch.profiler.profile( + activities=[ +- torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ], + with_stack=True, +@@ -1033,9 +1032,11 @@ class Scheduler: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") + self.profiler.stop() +- self.profiler.export_chrome_trace( +- self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" +- ) ++ if self.tp_rank == 0: ++ with open(f"stats_repro_{int(time.time())}.txt", "w") as f: ++ print(self.profiler.key_averages(group_by_input_shape=True).table(sort_by="cuda_time_total", row_limit=-1), file=f) ++ print("Profiling stats done.") ++ + logger.info("Profiler is done") +``` + +2. Create the model path directory and copy it to the right path for "--model-path" if you want to use the server.sh file provided. + +3. Modify the included server.sh by removing "loadTracer.sh" before python command and launch script ./server.sh in one terminal inside the docker container. + +4. Similar to step 6 in RPD profiling section, but remove the last 2 lines in client.sh, which converted rpd file into csv and json files. Run modified client.sh for PyTorch profiling. +------- +- [Torch Profiler](https://pytorch.org/tutorials/recipes/recipes/profiler_recipe.html) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/client.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/client.sh new file mode 100755 index 00000000..150ea9f1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/client.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Start profiling via API +curl http://localhost:30000/start_profile -H "Content-Type: application/json" + +# Benchmark serving using sglang with random dataset and tokenizer +# Define the log file with a timestamp +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +LOGFILE="sglang_client_log_$TIMESTAMP.json" + +# Run the benchmark with specified parameters and save logs +python3 -m sglang.bench_serving \ + --backend sglang \ + --tokenizer Xenova/grok-1-tokenizer \ + --dataset-name random \ + --random-input 1024\ + --random-output 1024 \ + --num-prompts 240 \ + --request-rate 8 \ + --output-file online.jsonl 2>&1 | tee "$LOGFILE" + +# Stop profiling via API +curl http://localhost:30000/stop_profile -H "Content-Type: application/json" + +# Convert tracing file to csv & json +sqlite3 trace.rpd ".mode csv" ".header on" ".output trace.csv" "select * from top;" ".output stdout" +python3 /sgl-workspace/rocmProfileData/tools/rpd2tracing.py trace.rpd trace.json diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/install_rpd.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/install_rpd.sh new file mode 100644 index 00000000..d1b04b98 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/install_rpd.sh @@ -0,0 +1,10 @@ +# download and install RPD +apt update && apt install -y sqlite3 libsqlite3-dev libfmt-dev + +# install rpd module +git clone https://github.com/ROCmSoftwarePlatform/rocmProfileData +cd rocmProfileData +git apply rpd.patch +make && make install +cd rocpd_python && python setup.py install && cd .. +cd rpd_tracer && make clean;make install && python setup.py install && cd .. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/loadTracer.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/loadTracer.sh new file mode 100755 index 00000000..8a95a335 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/loadTracer.sh @@ -0,0 +1,43 @@ +#!/bin/bash +################################################################################ +# Copyright (c) 2021 - 2023 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +################################################################################ +OUTPUT_FILE="trace.rpd" + +if [ "$1" = "-o" ] ; then + OUTPUT_FILE=$2 + shift + shift +fi + +if [ -e ${OUTPUT_FILE} ] ; then + rm ${OUTPUT_FILE} +fi + +python3 -m rocpd.schema --create ${OUTPUT_FILE} +if [ $? != 0 ] ; then + echo "Error: Could not create rpd file. Please run 'python setup.py install' from the rocpd_python dir" + exit +fi + +export RPDT_FILENAME=${OUTPUT_FILE} +export RPDT_AUTOSTART=0 +LD_PRELOAD=librocm-smi_64:librpd_tracer.so "$@" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/rpd.patch b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/rpd.patch new file mode 100644 index 00000000..87917654 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/rpd.patch @@ -0,0 +1,12 @@ +diff --git a/rpd_tracer/Makefile b/rpd_tracer/Makefile +index e9d9feb..b2e9e1a 100644 +--- a/rpd_tracer/Makefile ++++ b/rpd_tracer/Makefile +@@ -16,7 +16,7 @@ ifneq (,$(HIP_PATH)) + $(info Building with roctracer) + RPD_LIBS += -L/opt/rocm/lib -lroctracer64 -lroctx64 -lamdhip64 -lrocm_smi64 + RPD_INCLUDES += -I/opt/rocm/include -I/opt/rocm/include/roctracer -I/opt/rocm/include/hsa +- RPD_SRCS += RoctracerDataSource.cpp RocmSmiDataSource.cpp ++ RPD_SRCS += RoctracerDataSource.cpp + RPD_INCLUDES += -D__HIP_PLATFORM_AMD__ + endif diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/rpd_profile_server_enable.patch b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/rpd_profile_server_enable.patch new file mode 100644 index 00000000..3cd39153 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/rpd_profile_server_enable.patch @@ -0,0 +1,49 @@ +diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py +index 62d1ff9..9021c01 100644 +--- a/python/sglang/srt/managers/scheduler.py ++++ b/python/sglang/srt/managers/scheduler.py +@@ -71,6 +71,8 @@ from sglang.srt.utils import ( + suppress_other_loggers, + ) + from sglang.utils import get_exception_traceback ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() + + logger = logging.getLogger(__name__) + +@@ -245,6 +247,7 @@ class Scheduler: + ], + with_stack=True, + ) ++ self.rpd = rpdTracerControl() + + @torch.inference_mode() + def event_loop(self): +@@ -1027,15 +1030,24 @@ class Scheduler: + def start_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.start() ++ #self.profiler.start() #block pytorch profiler for rpd profiler enabling ++ if self.tp_rank == 0 or self.tp_rank == 1: ++ self.rpd.start() ++ self.rpd.rangePush("", "rpd profile range", "") ++ logger.info("rpd is enabled") + + def stop_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.stop() +- self.profiler.export_chrome_trace( +- self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" +- ) ++ #self.profiler.stop() ++ #self.profiler.export_chrome_trace( ++ # self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" ++ #) ++ if self.tp_rank ==0 or self.tp_rank ==1: ++ self.rpd.rangePop() ++ self.rpd.stop() ++ self.rpd.flush() ++ logger.info("rpd is done") + logger.info("Profiler is done") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/rpd_profile_server_enable_wCPU_activities.patch b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/rpd_profile_server_enable_wCPU_activities.patch new file mode 100644 index 00000000..5416f4d5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/rpd_profile_server_enable_wCPU_activities.patch @@ -0,0 +1,126 @@ +diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py +index 62d1ff9..2edb427 100644 +--- a/python/sglang/srt/managers/scheduler.py ++++ b/python/sglang/srt/managers/scheduler.py +@@ -71,6 +71,8 @@ from sglang.srt.utils import ( + suppress_other_loggers, + ) + from sglang.utils import get_exception_traceback ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() + + logger = logging.getLogger(__name__) + +@@ -245,6 +247,7 @@ class Scheduler: + ], + with_stack=True, + ) ++ self.rpd = rpdTracerControl() + + @torch.inference_mode() + def event_loop(self): +@@ -1027,15 +1030,26 @@ class Scheduler: + def start_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.start() ++ #self.profiler.start() ++ logger.info("torch profiler is disabled") ++ if self.tp_rank == 0 or self.tp_rank == 1: ++ self.rpd.setPythonTrace(True) ++ self.rpd.start() ++ self.rpd.rangePush("", "scheduler", "") ++ logger.info("rpd is enabled inside scheduler profiling") + + def stop_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.stop() +- self.profiler.export_chrome_trace( +- self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" +- ) ++ #self.profiler.stop() ++ #self.profiler.export_chrome_trace( ++ # self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" ++ #) ++ if self.tp_rank ==0 or self.tp_rank ==1: ++ self.rpd.rangePop() ++ self.rpd.stop() ++ self.rpd.flush() ++ logger.info("rpd is done inside scheduler") + logger.info("Profiler is done") + + +diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py +index 2621ccd..181df85 100644 +--- a/python/sglang/srt/managers/tokenizer_manager.py ++++ b/python/sglang/srt/managers/tokenizer_manager.py +@@ -58,6 +58,10 @@ from sglang.srt.sampling.sampling_params import SamplingParams + from sglang.srt.server_args import PortArgs, ServerArgs + from sglang.srt.utils import is_generation_model, is_multimodal_model + ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() ++ ++ + asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) + + logger = logging.getLogger(__name__) +@@ -514,10 +518,20 @@ class TokenizerManager: + self.send_to_scheduler.send_pyobj(req) + + def start_profile(self): ++ rpd = rpdTracerControl() ++ rpd.setPythonTrace(True) ++ rpd.start() ++ rpd.rangePush("", "tokenizer_manager", "") ++ logger.info("tokenizer_manager rpd profiling started!") + req = ProfileReq.START_PROFILE + self.send_to_scheduler.send_pyobj(req) + + def stop_profile(self): ++ rpd = rpdTracerControl() ++ rpd.rangePop() ++ rpd.stop() ++ rpd.flush() ++ logger.info("rpd profiling is done inside tokenizer_manager!") + req = ProfileReq.STOP_PROFILE + self.send_to_scheduler.send_pyobj(req) + +diff --git a/python/sglang/srt/server.py b/python/sglang/srt/server.py +index 7111c93..2bd722c 100644 +--- a/python/sglang/srt/server.py ++++ b/python/sglang/srt/server.py +@@ -30,6 +30,8 @@ import threading + import time + from http import HTTPStatus + from typing import Dict, List, Optional, Union ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() + + # Fix a bug of Python threading + setattr(threading, "_register_atexit", lambda *args, **kwargs: None) +@@ -152,6 +154,11 @@ async def flush_cache(): + @app.post("/start_profile") + async def start_profile(): + """Start profiling.""" ++ rpd = rpdTracerControl() ++ rpd.setPythonTrace(True) ++ rpd.start() ++ rpd.rangePush("", "server rpd profile range", "") ++ logger.info("rpd profiling started in server.py!") + tokenizer_manager.start_profile() + return Response( + content="Start profiling.\n", +@@ -164,6 +171,11 @@ async def start_profile(): + async def stop_profile(): + """Stop profiling.""" + tokenizer_manager.stop_profile() ++ rpd = rpdTracerControl() ++ rpd.rangePop() ++ rpd.stop() ++ rpd.flush() ++ logger.info("rpd profiling is done in server.py!") + return Response( + content="Stop profiling. This will take some time.\n", + status_code=200, diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/server.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/server.sh new file mode 100755 index 00000000..f877e6c7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/server.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# export SGLANG_TORCH_PROFILER_DIR=/data/sglang/ +export SGLANG_TORCH_PROFILER_DIR=/sgl-workspace/sglang/profile/ + +# Get the current timestamp +TIMESTAMP=$(date +"%Y%m%d_%H%M%S") + +# Define the log file with a timestamp +LOGFILE="sglang_server_log_$TIMESTAMP.json" + +# Run the Python command and save the output to the log file +loadTracer.sh python3 -m sglang.launch_server \ + --model-path /sgl-workspace/sglang/dummy_grok1 \ + --tokenizer-path Xenova/grok-1-tokenizer \ + --load-format dummy \ + --quantization fp8 \ + --tp 8 \ + --port 30000 \ + --disable-radix-cache 2>&1 | tee "$LOGFILE" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/torch_profiler.patch b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/torch_profiler.patch new file mode 100644 index 00000000..40f55740 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/profiling/torch_profiler.patch @@ -0,0 +1,25 @@ +diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py +index 62d1ff9..6ecd78c 100644 +--- a/python/sglang/srt/managers/scheduler.py ++++ b/python/sglang/srt/managers/scheduler.py +@@ -240,7 +240,6 @@ class Scheduler: + ) + self.profiler = torch.profiler.profile( + activities=[ +- torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ], + with_stack=True, +@@ -1033,9 +1032,11 @@ class Scheduler: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") + self.profiler.stop() +- self.profiler.export_chrome_trace( +- self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" +- ) ++ if self.tp_rank == 0: ++ with open(f"stats_repro_{int(time.time())}.txt", "w") as f: ++ print(self.profiler.key_averages(group_by_input_shape=True).table(sort_by="cuda_time_total", row_limit=-1), file=f) ++ print("Profiling stats done.") ++ + logger.info("Profiler is done") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/tuning/TUNING.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/tuning/TUNING.md new file mode 100644 index 00000000..e7b9b204 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/tuning/TUNING.md @@ -0,0 +1,118 @@ +## Tuning SGLang Infer System with AMD GPUs +This AppNote describes the SGLang performance tuning technical, code harness and running steps for systems with AMD Instinct GPUs. +Harness code, examples and steps are provided in detail, to facilitate easy reproduce & use to tune performance towards workloads. +Three primary runtime areas are covered: + +## 1. Triton Kernels +To maximize Triton kernel efficiency, several strategies can be employed: + +### Key Environment Variables: +- **num_stages**: Adjusts the number of pipeline stages to optimize kernel efficiency based on the specific type of operations (e.g., General Matrix Multiplication - GEMM). +- **waves_per_eu**: Controls the usage of Vector General Purpose Registers (VGPR) to enhance occupancy, thereby improving latency or throughput. +- **BLOCK_M, BLOCK_N, BLOCK_K**: Tunable tile sizes that assist in balancing memory transfer and computational efficiency. +- **matrix_instr_nonkdim**: Optimizes the usage of Matrix-Fused Multiply-Add (MFMA) instructions for specific kernel types, such as Flash Attention. +- **OPTIMIZE_EPILOGUE**: An environment variable that can be set to `1` to enhance performance by eliminating the `convert_layout` operation in the kernel's epilogue. +```python +@triton.autotune(configs=[ + triton.Config({'waves_per_eu': 1}, num_warps=4, num_stages=1), + triton.Config({'waves_per_eu': 1}, num_warps=8, num_stages=1), + triton.Config({'waves_per_eu': 1}, num_warps=16, num_stages=1), + triton.Config({'waves_per_eu': 2}, num_warps=4, num_stages=1), + triton.Config({'waves_per_eu': 2}, num_warps=8, num_stages=1), + triton.Config({'waves_per_eu': 2}, num_warps=16, num_stages=1), + triton.Config({'waves_per_eu': 4}, num_warps=4, num_stages=1), + triton.Config({'waves_per_eu': 4}, num_warps=8, num_stages=1), + triton.Config({'waves_per_eu': 4}, num_warps=16, num_stages=1), + ], key=['BLOCK_N', 'NUM_TOKEN_BLKS'], use_cuda_graph=True) +@triton.jit +def _triton_kernel_funtion(): + ... +``` +## 2. Torch Tunable Operations +**TunableOp** is a feature in PyTorch that allows for the definition and optimization of custom kernels with tunable parameters. This feature is particularly useful for enhancing the performance of kernels by experimenting with different configurations. + +### Key Environment Variables: +1. **PYTORCH_TUNABLEOP_ENABLED**: + - Default: `0` + - Set to `1` to enable TunableOp. + +2. **PYTORCH_TUNABLEOP_TUNING**: + - Default: `1` + - Set to `0` to disable tuning. If a tuned entry is not found, it will run the tuning step and record the entry when PYTORCH_TUNABLEOP_ENABLED is enabled. + +3. **PYTORCH_TUNABLEOP_VERBOSE**: + - Default: `0` + - Set to `1` to enable verbose output for TunableOp. + +### Usage Example: +To enable TunableOp and tuning, and optionally enable verbose mode, you can run the following command in your terminal: + +```bash +#Tuning +PYTORCH_TUNABLEOP_ENABLED=1 PYTORCH_TUNABLEOP_TUNING=1 your_script.sh + +#Inference with tuning op +PYTORCH_TUNABLEOP_ENABLED=1 PYTORCH_TUNABLEOP_TUNING=0 your_script.sh + +#Print out the log +PYTORCH_TUNABLEOP_ENABLED=1 PYTORCH_TUNABLEOP_TUNING=0 PYTORCH_TUNABLEOP_VERBOSE=1 your_script.sh + +``` +## 3. Torch Compilation + + +The following are suggestions for optimizing matrix multiplication (GEMM) and convolution (conv) operations in PyTorch using Inductor, a part of the PyTorch compilation framework. The goal is to leverage Triton to achieve better performance. + +To tune Triton kernels with GEMM and convolution ops (conv), use the `torch.compile` function with the max-autotune mode. This benchmarks a predefined list of Triton configurations and selects the fastest one for each shape. + +### Key Configurations: +1. **Max Autotune**: + - Set `torch._inductor.config.max_autotune = True` or `TORCHINDUCTOR_MAX_AUTOTUNE=1`. + +2. **Fine-Grained Control**: + - Enable GEMM tuning: `torch._inductor.config.max_autotune_gemm = True`. + - Enable tuning for pointwise/reduction ops: `torch._inductor.config.max_autotune.pointwise = True`. + +3. **Backend Selection**: + - Use `torch._inductor.max_autotune_gemm_backends` to limit backends to TRITON for better performance. + +4. **Freezing for Inference**: + - Use `torch._inductor.config.freezing=True` to enable constant folding optimizations. + +5. **Debugging**: + - Set `TORCH_COMPILE_DEBUG=1` to extract Triton kernels generated by Inductor. + +### Example Code Block: +```bash +#Gemm Tuning +TORCHINDUCTOR_MAX_AUTOTUNE=1 TORCHINDUCTOR_COORDINATE_DESCENT_TUNING=1 your_script.sh + +#Specify your backend to TRITON for Gemm Tuning +TORCHINDUCTOR_MAX_AUTOTUNE=1 TORCHINDUCTOR_COORDINATE_DESCENT_TUNING=1 TORCHINDUCTOR_MAX_AUTOTUNE_GEMM_BACKENDS=TRITON your_script.sh + +#Inference with large improvement on AMD GPU +TORCHINDUCTOR_FREEZING=1 your_script.sh +``` +## 4. Fused MOE kernel +To maximize moe kernel efficiency, need to use below scripts to find out the best launch configuration + +### Key parameters: +- **--model**: what moe model type to do tuning, it will automatically decide the size of d_model, model_intermediate_size, num_layers +- **--tp-size**: simulate the whole model run configuration to set the dimension size using tp correctly +- **--batch**: M dimension size of moe kernel, for prefill moe kernel the value is batch*input_len, for decode moe kernel the value is batch +- **--dtype**: computation type + +```bash +#Tuning +#for example, we have one case like this "python3 -m sglang.bench_latency --model dummy_grok1/ --load-format dummy --tokenizer-path Xenova/grok-1-tokenizer --tp 8 --batch-size 32 --input 1024 --output 8 --attention-backend triton --sampling-backend pytorch --quantization fp8" to run, it defined batch-size 32 input length 1024 and output length 8, from "--batch" in moe view point, the prefill batch is 32*1024 = 32768, the decode batch is 32*1(only one output token generated in each run). +#so we can tune decode moe use below command +python benchmark_moe_rocm.py --model grok1 --tp-size 8 --dtype float8 --batch "32" +# and use this command to tune prefill moe +python benchmark_moe_rocm.py --model grok1 --tp-size 8 --dtype float8 --batch "32768" +``` + +## Reference + +For more detailed information on tuning SGLang performance with AMD GPUs, please refer to the following link: + +[ROCm Documentation: Triton Kernel Performance Optimization](https://rocm.docs.amd.com/en/latest/how-to/tuning-guides/mi300x/workload.html#triton-kernel-performance-optimization) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/tuning/benchmark_moe_rocm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/tuning/benchmark_moe_rocm.py new file mode 100644 index 00000000..af596d21 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/3rdparty/amd/tuning/benchmark_moe_rocm.py @@ -0,0 +1,380 @@ +import argparse +import json +import os +import sys + +import torch +import torch.nn.functional as F +import triton +import triton.language as tl +from tqdm import tqdm +from transformers import AutoConfig + +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import ( + fused_moe, + get_config_file_name, +) + +padding_size = 128 if bool(int(os.getenv("SGLANG_MOE_PADDING", "0"))) else 0 + + +def main(model, tp_size, dtype: str, batches): + method = fused_moe + + for bs in batches: + run_grid(int(bs), model=model, method=method, tp_size=tp_size, dtype=dtype) + + +def prune_configs(M, N, K, configs): + pruned_configs = [] + elemBytes_a = 1 # [DV Note] Hard-coded for float16 (2 bytes) + elemBytes_b = 1 # [DV Note] Hard-coded for float16 (2 bytes) + + mfma = 16 if M < 32 or N < 32 else 32 + + # TODO (zhanglx): figure out the boundary between large and small gemms + large_gemm = False + if M >= 2048 and N >= 2048: + large_gemm = True + + for config in configs: + BLOCK_SIZE_M = config.get("BLOCK_SIZE_M") + BLOCK_SIZE_N = config.get("BLOCK_SIZE_N") + BLOCK_SIZE_K = config.get("BLOCK_SIZE_K") + num_warps = config.get("num_warps") + matrix_instr_nonkdim = config.get("matrix_instr_nonkdim") + # kpack = config.get("kpack") + if matrix_instr_nonkdim > mfma: + continue + if mfma == 4 and BLOCK_SIZE_K < 64: + continue + # some layouts could not work properly in case + # number elements per thread is less 1 + if BLOCK_SIZE_M * BLOCK_SIZE_N < 64: + continue + SPLIT_K = 1 # config.get("SPLIT_K") + GROUP_M = config.get("GROUP_SIZE_M") + if matrix_instr_nonkdim > BLOCK_SIZE_M or matrix_instr_nonkdim > BLOCK_SIZE_N: + continue + if matrix_instr_nonkdim >= M and matrix_instr_nonkdim != BLOCK_SIZE_M: + continue + if matrix_instr_nonkdim >= N and matrix_instr_nonkdim != BLOCK_SIZE_N: + continue + # Skip BLOCK_SIZE that is too large compare to M/N + # unless BLOCK_SIZE is already small enough + if M * 2 < BLOCK_SIZE_M and BLOCK_SIZE_M != 16: + continue + if N * 2 < BLOCK_SIZE_N and BLOCK_SIZE_N != 16: + continue + # skip large split_k when not necessary + if SPLIT_K != 1 and not need_split_k(M, N, K): + continue + # skip split_k that leads to EVEN_K = false + leap = SPLIT_K * BLOCK_SIZE_K + modv = K % leap + if modv != 0: + continue + # skip large GROUP_M + if GROUP_M * BLOCK_SIZE_M > M and GROUP_M != 1: + continue + # out of shared memory resource + # TODO (zhanglx): This does not consider the LDS usage in the epilogue + LDS = ( + BLOCK_SIZE_K * BLOCK_SIZE_M * elemBytes_a + + BLOCK_SIZE_K * BLOCK_SIZE_N * elemBytes_b + ) + if LDS > 65536: + continue + # Skip small block sizes and num_warps for large gemm + # For fp16 and f8, we want to only use BLOCK_SIZE >= 64 + if large_gemm: + if BLOCK_SIZE_M < 64 or BLOCK_SIZE_N < 64: + continue + if BLOCK_SIZE_K < 64: + continue + if num_warps < 4: + continue + + pruned_configs.append(config) + + return pruned_configs + + +def union_of_list_of_dicts(l1, l2): + result = [] + temp_list = l1.copy() + temp_list.extend(l2) + for myDict in temp_list: + if myDict not in result: + result.append(myDict) + + return result + + +def run_grid(bs, model, method, tp_size, dtype: str): + + config = AutoConfig.from_pretrained(model) + + top_k = config.num_experts_per_tok + d_model = config.hidden_size + model_intermediate_size = config.intermediate_size + num_layers = config.num_hidden_layers + hidden_states_dtype = config.torch_dtype + + if config.num_experts_per_tok: + if config.architectures[0] == "Grok1ModelForCausalLM": + num_total_experts = config.num_experts + else: + num_total_experts = config.num_local_experts + else: + raise ValueError(f"Unsupported Mixtral model {model}") + + # tp_size = 2 + num_warmup_calls = 10 + num_calls = 30 + + num_warmup_trials = 1 + num_trials = 1 + + full_configs = [] + + block_m_range = [16, 32, 64, 128, 256] + block_n_range = [16, 32, 64, 128, 256] + block_k_range = [32, 64, 128, 256] # MUST >= 32 + num_warps_range = [1, 2, 4, 8] + group_m_range = [1, 4, 8, 16, 32] + # For now we see better perf with num_stages=0 for all gemm configs we care + # But keep this explicit so that we do not forget we may need to set it to + # other values in the future + num_stage_range = [2] + waves_per_eu_range = [0, 1, 2, 4, 8] + # Remove 32 because of triton compiling error + matrix_instr_nonkdim_range = [16] + kpack_range = [1, 2] + + for block_size_m in block_m_range: + for block_size_n in block_n_range: + for block_size_k in block_k_range: + for group_size_m in group_m_range: + for num_warps in num_warps_range: + for num_stages in num_stage_range: + for waves_per_eu in waves_per_eu_range: + for matrix_instr_nonkdim in matrix_instr_nonkdim_range: + for kpack in kpack_range: + full_configs.append( + { + "BLOCK_SIZE_M": block_size_m, + "BLOCK_SIZE_N": block_size_n, + "BLOCK_SIZE_K": block_size_k, + "GROUP_SIZE_M": group_size_m, + "num_warps": num_warps, + "num_stages": num_stages, + "waves_per_eu": waves_per_eu, + "matrix_instr_nonkdim": matrix_instr_nonkdim, + "kpack": kpack, + } + ) + + M1 = bs * 2 + N1 = model_intermediate_size * 2 // tp_size + K1 = d_model + prune_configs_1 = prune_configs(M1, N1, K1, full_configs) + + M2 = bs * 2 + N2 = d_model + K2 = model_intermediate_size // tp_size + prune_configs_2 = prune_configs(M2, N2, K2, full_configs) + + configs = union_of_list_of_dicts(prune_configs_1, prune_configs_2) + + print( + f"{bs=} || {len(full_configs)=} | {len(prune_configs_1)=} | \ + {len(prune_configs_2)=} | {len(configs)=}" + ) + + best_config = None + best_time_us = 1e20 + + print(f"{tp_size=} {bs=}") + + for config in tqdm(configs): + # warmup + try: + print(config) + for _ in range(num_warmup_trials): + run_timing( + num_calls=num_warmup_calls, + bs=bs, + d_model=d_model, + num_total_experts=num_total_experts, + top_k=top_k, + tp_size=tp_size, + model_intermediate_size=model_intermediate_size, + method=method, + config=config, + dtype=dtype, + hidden_states_dtype=hidden_states_dtype, + ) + except triton.runtime.autotuner.OutOfResources: + continue + + # trial + for _ in range(num_trials): + kernel_dur_ms = run_timing( + num_calls=num_calls, + bs=bs, + d_model=d_model, + num_total_experts=num_total_experts, + top_k=top_k, + tp_size=tp_size, + model_intermediate_size=model_intermediate_size, + method=method, + config=config, + dtype=dtype, + hidden_states_dtype=hidden_states_dtype, + ) + + kernel_dur_us = 1000 * kernel_dur_ms + model_dur_ms = kernel_dur_ms * num_layers + + if kernel_dur_us < best_time_us: + best_config = config + best_time_us = kernel_dur_us + + tqdm.write( + f"{kernel_dur_us=:.1f} {model_dur_ms=:.1f}" + f" {bs=} {tp_size=} {top_k=} {num_total_experts=} " + f"{d_model=} {model_intermediate_size=} {num_layers=}" + ) + + print("best_time_us", best_time_us) + print("best_config", best_config) + + # holds Dict[str, Dict[str, int]] + filename = get_config_file_name( + num_total_experts, + model_intermediate_size // tp_size, + "float8" if dtype == "float8" else None, + ) + print(f"writing config to file {filename}") + existing_content = {} + if os.path.exists(filename): + with open(filename, "r") as f: + existing_content = json.load(f) + existing_content[str(bs)] = best_config + with open(filename, "w") as f: + json.dump(existing_content, f, indent=4) + f.write("\n") + + +def run_timing( + num_calls: int, + bs: int, + d_model: int, + num_total_experts: int, + top_k: int, + tp_size: int, + model_intermediate_size: int, + method, + config, + dtype: str, + hidden_states_dtype, +) -> float: + shard_intermediate_size = model_intermediate_size // tp_size + + hidden_states = torch.rand( + (bs, d_model), + device="cuda:0", + dtype=hidden_states_dtype, + ) + + w1 = torch.rand( + (num_total_experts, 2 * shard_intermediate_size, d_model + padding_size), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + + w2 = torch.rand( + (num_total_experts, d_model, shard_intermediate_size + padding_size), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + + w1_scale = None + w2_scale = None + a1_scale = None + a2_scale = None + + if dtype == "float8": + w1 = w1.to(torch.float8_e4m3fnuz) + w2 = w2.to(torch.float8_e4m3fnuz) + w1_scale = torch.ones( + num_total_experts, device=hidden_states.device, dtype=torch.float32 + ) + w2_scale = torch.ones( + num_total_experts, device=hidden_states.device, dtype=torch.float32 + ) + a1_scale = torch.ones(1, device=hidden_states.device, dtype=torch.float32) + a2_scale = torch.ones(1, device=hidden_states.device, dtype=torch.float32) + + gating_output = F.softmax( + torch.rand( + (num_calls, bs, num_total_experts), + device=hidden_states.device, + dtype=torch.float32, + ), + dim=-1, + ) + + ################################## + + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + + start_event.record() + for i in range(num_calls): + hidden_states = method( + hidden_states=hidden_states, + w1=w1, + w2=w2, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + gating_output=gating_output[0], + topk=top_k, + renormalize=True, + inplace=True, + override_config=config, + use_fp8=dtype == "float8", + ) + + end_event.record() + end_event.synchronize() + + dur_ms = start_event.elapsed_time(end_event) / num_calls + return dur_ms + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="benchmark_mixtral_moe", + description="Benchmark and tune the fused_moe kernel", + ) + parser.add_argument( + "--dtype", + type=str, + default="auto", + choices=["float8", "float16", "bfloat16"], + help="Data type used for fused_moe kernel computations", + ) + parser.add_argument("--model", type=str, default="hpcai-tech/grok-1") + + parser.add_argument("--tp-size", type=int, default=2, help="Tensor paralleli size") + parser.add_argument("-b", "--batches", type=str) + + args = parser.parse_args() + + batches = args.batches.split(",") + + sys.exit(main(args.model, args.tp_size, args.dtype, batches)) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/CODE_OF_CONDUCT.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..18c91471 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/LICENSE b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/LICENSE new file mode 100644 index 00000000..9c422689 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2023-2024 SGLang Team + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/Makefile b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/Makefile new file mode 100644 index 00000000..d6ef1942 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/Makefile @@ -0,0 +1,49 @@ +.PHONY: check-deps install-deps format update help + +# Show help for each target +help: + @echo "Available targets:" + @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' + +check-deps: ## Check and install required Python formatting dependencies + @command -v isort >/dev/null 2>&1 || (echo "Installing isort..." && pip install isort) + @command -v black >/dev/null 2>&1 || (echo "Installing black..." && pip install black) + +install-deps: ## Install Python formatting tools (isort and black) + pip install isort black + +format: check-deps ## Format modified Python files using isort and black + @echo "Formatting modified Python files..." + git diff --name-only --diff-filter=M | grep '\.py$$' | xargs -I {} sh -c 'isort {} && black {}' + +FILES_TO_UPDATE = docker/rocm.Dockerfile \ + python/pyproject.toml \ + python/pyproject_other.toml \ + python/sglang/version.py \ + docs/developer_guide/setup_github_runner.md \ + docs/get_started/install.md \ + docs/platforms/amd_gpu.md \ + docs/platforms/ascend_npu.md \ + docs/platforms/cpu_server.md \ + docs/platforms/xpu.md \ + benchmark/deepseek_v3/README.md + +update: ## Update version numbers across project files. Usage: make update + @if [ -z "$(filter-out $@,$(MAKECMDGOALS))" ]; then \ + echo "Version required. Usage: make update "; \ + exit 1; \ + fi + @OLD_VERSION=$$(grep "version" python/sglang/version.py | cut -d '"' -f2); \ + NEW_VERSION=$(filter-out $@,$(MAKECMDGOALS)); \ + echo "Updating version from $$OLD_VERSION to $$NEW_VERSION"; \ + for file in $(FILES_TO_UPDATE); do \ + if [ "$(shell uname)" = "Darwin" ]; then \ + sed -i '' -e "s/$$OLD_VERSION/$$NEW_VERSION/g" $$file; \ + else \ + sed -i -e "s/$$OLD_VERSION/$$NEW_VERSION/g" $$file; \ + fi \ + done; \ + echo "Version update complete" + +%: + @: diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/README.md new file mode 100644 index 00000000..3ae19493 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/README.md @@ -0,0 +1,84 @@ +
+logo + +[![PyPI](https://img.shields.io/pypi/v/sglang)](https://pypi.org/project/sglang) +![PyPI - Downloads](https://static.pepy.tech/badge/sglang?period=month) +[![license](https://img.shields.io/github/license/sgl-project/sglang.svg)](https://github.com/sgl-project/sglang/tree/main/LICENSE) +[![issue resolution](https://img.shields.io/github/issues-closed-raw/sgl-project/sglang)](https://github.com/sgl-project/sglang/issues) +[![open issues](https://img.shields.io/github/issues-raw/sgl-project/sglang)](https://github.com/sgl-project/sglang/issues) +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/sgl-project/sglang) + +
+ +-------------------------------------------------------------------------------- + +| [**Blog**](https://lmsys.org/blog/) +| [**Documentation**](https://docs.sglang.io/) +| [**Roadmap**](https://roadmap.sglang.io/) +| [**Join Slack**](https://slack.sglang.io/) +| [**Weekly Dev Meeting**](https://meet.sglang.io/) +| [**Slides**](https://github.com/sgl-project/sgl-learning-materials?tab=readme-ov-file#slides) | + +## News +- [2025/11] 🔥 SGLang Diffusion accelerates video and image generation ([blog](https://lmsys.org/blog/2025-11-07-sglang-diffusion/)). +- [2025/10] 🔥 SGLang now runs natively on TPU with the SGLang-Jax backend ([blog](https://lmsys.org/blog/2025-10-29-sglang-jax/)). +- [2025/10] PyTorch Conference 2025 SGLang Talk ([slide](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/sglang_pytorch_2025.pdf)). +- [2025/09] 🔥 Deploying DeepSeek on GB200 NVL72 with PD and Large Scale EP (Part II): 3.8x Prefill, 4.8x Decode Throughput ([blog](https://lmsys.org/blog/2025-09-25-gb200-part-2/)). +- [2025/09] SGLang Day 0 Support for DeepSeek-V3.2 with Sparse Attention ([blog](https://lmsys.org/blog/2025-09-29-deepseek-V32/)). +- [2025/08] SGLang x AMD SF Meetup on 8/22: Hands-on GPU workshop, tech talks by AMD/xAI/SGLang, and networking ([Roadmap](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_sglang_roadmap.pdf), [Large-scale EP](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_sglang_ep.pdf), [Highlights](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_highlights.pdf), [AITER/MoRI](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_aiter_mori.pdf), [Wave](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_wave.pdf)). +- [2025/08] SGLang provides day-0 support for OpenAI gpt-oss model ([instructions](https://github.com/sgl-project/sglang/issues/8833)) +- [2025/05] Deploying DeepSeek with PD Disaggregation and Large-scale Expert Parallelism on 96 H100 GPUs ([blog](https://lmsys.org/blog/2025-05-05-large-scale-ep/)). + +
+More + +- [2025/10] SGLang x Nvidia SF Meetup on 10/2 ([recap](https://x.com/lmsysorg/status/1975339501934510231)). +- [2025/06] SGLang, the high-performance serving infrastructure powering trillions of tokens daily, has been awarded the third batch of the Open Source AI Grant by a16z ([a16z blog](https://a16z.com/advancing-open-source-ai-through-benchmarks-and-bold-experimentation/)). +- [2025/06] Deploying DeepSeek on GB200 NVL72 with PD and Large Scale EP (Part I): 2.7x Higher Decoding Throughput ([blog](https://lmsys.org/blog/2025-06-16-gb200-part-1/)). +- [2025/03] Supercharge DeepSeek-R1 Inference on AMD Instinct MI300X ([AMD blog](https://rocm.blogs.amd.com/artificial-intelligence/DeepSeekR1-Part2/README.html)) +- [2025/03] SGLang Joins PyTorch Ecosystem: Efficient LLM Serving Engine ([PyTorch blog](https://pytorch.org/blog/sglang-joins-pytorch/)) +- [2025/02] Unlock DeepSeek-R1 Inference Performance on AMD Instinct™ MI300X GPU ([AMD blog](https://rocm.blogs.amd.com/artificial-intelligence/DeepSeekR1_Perf/README.html)) +- [2025/01] SGLang provides day one support for DeepSeek V3/R1 models on NVIDIA and AMD GPUs with DeepSeek-specific optimizations. ([instructions](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3), [AMD blog](https://www.amd.com/en/developer/resources/technical-articles/amd-instinct-gpus-power-deepseek-v3-revolutionizing-ai-development-with-sglang.html), [10+ other companies](https://x.com/lmsysorg/status/1887262321636221412)) +- [2024/12] v0.4 Release: Zero-Overhead Batch Scheduler, Cache-Aware Load Balancer, Faster Structured Outputs ([blog](https://lmsys.org/blog/2024-12-04-sglang-v0-4/)). +- [2024/10] The First SGLang Online Meetup ([slides](https://github.com/sgl-project/sgl-learning-materials?tab=readme-ov-file#the-first-sglang-online-meetup)). +- [2024/09] v0.3 Release: 7x Faster DeepSeek MLA, 1.5x Faster torch.compile, Multi-Image/Video LLaVA-OneVision ([blog](https://lmsys.org/blog/2024-09-04-sglang-v0-3/)). +- [2024/07] v0.2 Release: Faster Llama3 Serving with SGLang Runtime (vs. TensorRT-LLM, vLLM) ([blog](https://lmsys.org/blog/2024-07-25-sglang-llama3/)). +- [2024/02] SGLang enables **3x faster JSON decoding** with compressed finite state machine ([blog](https://lmsys.org/blog/2024-02-05-compressed-fsm/)). +- [2024/01] SGLang provides up to **5x faster inference** with RadixAttention ([blog](https://lmsys.org/blog/2024-01-17-sglang/)). +- [2024/01] SGLang powers the serving of the official **LLaVA v1.6** release demo ([usage](https://github.com/haotian-liu/LLaVA?tab=readme-ov-file#demo)). + +
+ +## About +SGLang is a high-performance serving framework for large language models and vision-language models. +It is designed to deliver low-latency and high-throughput inference across a wide range of setups, from a single GPU to large distributed clusters. +Its core features include: + +- **Fast Backend Runtime**: Provides efficient serving with RadixAttention for prefix caching, a zero-overhead CPU scheduler, prefill-decode disaggregation, speculative decoding, continuous batching, paged attention, tensor/pipeline/expert/data parallelism, structured outputs, chunked prefill, quantization (FP4/FP8/INT4/AWQ/GPTQ), and multi-LoRA batching. +- **Extensive Model Support**: Supports a wide range of generative models (Llama, Qwen, DeepSeek, Kimi, GLM, GPT, Gemma, Mistral, etc.), embedding models (e5-mistral, gte, mcdse), reward models (Skywork), and diffusion models (WAN, Qwen-Image), with easy extensibility for integrating new models. Compatible with most Hugging Face models and OpenAI APIs. +- **Extensive Hardware Support**: Runs on NVIDIA GPUs (GB200/B300/H100/A100/Spark), AMD GPUs (MI355/MI300), Intel Xeon CPUs, Google TPUs, Ascend NPUs, and more. +- **Flexible Frontend Language**: Offers an intuitive interface for programming LLM applications, supporting chained generation calls, advanced prompting, control flow, multi-modal inputs, parallelism, and external interactions. +- **Active Community**: SGLang is open-source and supported by a vibrant community with widespread industry adoption, powering over 400,000 GPUs worldwide. + +## Getting Started +- [Install SGLang](https://docs.sglang.io/get_started/install.html) +- [Quick Start](https://docs.sglang.io/basic_usage/send_request.html) +- [Backend Tutorial](https://docs.sglang.io/basic_usage/openai_api_completions.html) +- [Frontend Tutorial](https://docs.sglang.io/references/frontend/frontend_tutorial.html) +- [Contribution Guide](https://docs.sglang.io/developer_guide/contribution_guide.html) + +## Benchmark and Performance +Learn more in the release blogs: [v0.2 blog](https://lmsys.org/blog/2024-07-25-sglang-llama3/), [v0.3 blog](https://lmsys.org/blog/2024-09-04-sglang-v0-3/), [v0.4 blog](https://lmsys.org/blog/2024-12-04-sglang-v0-4/), [Large-scale expert parallelism](https://lmsys.org/blog/2025-05-05-large-scale-ep/), [GB200 rack-scale parallelism](https://lmsys.org/blog/2025-09-25-gb200-part-2/). + +## Adoption and Sponsorship +SGLang has been deployed at large scale, generating trillions of tokens in production each day. It is trusted and adopted by a wide range of leading enterprises and institutions, including xAI, AMD, NVIDIA, Intel, LinkedIn, Cursor, Oracle Cloud, Google Cloud, Microsoft Azure, AWS, Atlas Cloud, Voltage Park, Nebius, DataCrunch, Novita, InnoMatrix, MIT, UCLA, the University of Washington, Stanford, UC Berkeley, Tsinghua University, Jam & Tea Studios, Baseten, and other major technology organizations across North America and Asia. +As an open-source LLM inference engine, SGLang has become the de facto industry standard, with deployments running on over 400,000 GPUs worldwide. +SGLang is currently hosted under the non-profit open-source organization [LMSYS](https://lmsys.org/about/). + +logo + +## Contact Us +For enterprises interested in adopting or deploying SGLang at scale, including technical consulting, sponsorship opportunities, or partnership inquiries, please contact us at sglang@lmsys.org + +## Acknowledgment +We learned the design and reused code from the following projects: [Guidance](https://github.com/guidance-ai/guidance), [vLLM](https://github.com/vllm-project/vllm), [LightLLM](https://github.com/ModelTC/lightllm), [FlashInfer](https://github.com/flashinfer-ai/flashinfer), [Outlines](https://github.com/outlines-dev/outlines), and [LMQL](https://github.com/eth-sri/lmql). diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/assets/logo.png b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/assets/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..2a8bc258f6663112f02b40f04843789585d86d68 GIT binary patch literal 402692 zcmeFZcQ~AD7e2}uj7}It9a5B0qm2@b5JX9onbB*4h~7J+1S3I;lBkgoMkjjjqKoLn z=$#;XJF>I)_uJn&-}&?WcU+h2E#`f%=ULBs*1GR|Efe`jQ<;L4i4+eHk3v;NQ5z4B zL=O)SFa{#Jx)X9pCVzF%e4?jnrJ;exdvy-N1L8B`5ni3)U;WDBGymuOK0Y@d!C&_Q zS1+Lb`~3H({`$dxaCNx4`g6QNbf|!bCy%G92-o$(->7>QdvnzLmWN0ww%WJamyLyz zh;dHI0u;fhpvVZ)=Y(gBUM%Zr6%sz-3~rM1u-PA?@yLs(8yBGrKfiaO5#BNzyDEmK zL(LKStI&9oDU8mo^)>a5EuIKo@jXCuNXRUCR?X*{TwW3XzTR3yMM-I@@wBnAzra9T zVbY;hYFCbwdxLAg^$QF&H9$cg1j6Tl07HZR|BL^RY=BmULlWPT4Z8oo7}__w@#YJ% zPrd=8Ymff57;hpQ1Dr&hi@*BOyCME-nP{Ik2E^`@gvck{_+RUWYlIB!d!CtNUbqu- z_%c(`(h)B;4Tq8bz08T)!0K?q~?=JWB$=U>7j_9Xvy0`g#ab~YImTI~OP zzdVRMvax(-9-~MRiKE0i5;RQ5|9CD=Z$Jz9Gfby3>aqJYisbE=AUgudb%jSU?6zgx z{~oH46|mZAT=+0!>3`asg%T(gVEs`2Q)f*q8H0L zv2G*&{q%XyUj@ouyOcyEwfyX6kHYJIkr5;hkWzcpS`s8zhVb4p-aNTc64E`K^Uq^h zgy<2}&s4vij5jTPy+lhE`JPsXY-I`zzPq4}AJf@S{=L7@s`t|y?{Xb$yW9Ws{C_6$ zd#DFMbo5MD&9%%`nnc4Nv;49(cTB>uTpm7mrDbO1&Ldq@TjB!EBKFCQ*9_pu4(XZ9 zIZBTN(FSPDD5Zeb=l`toW1xX)rsXBzE@aA%&g5W&ab_^ZrMPa%JfHY>i`hm@FXIP< z&VP-77!M6-kikRcsnG%c>koe}gxG)^j&sFI2JLi->s+x+qD3A%YhGcdbWV|ND z?YuPxd6rw;+44A>?RS0G{(grLC&0J4KR&z?<}6}KG;4A0#mK?_JSxt)$eDC*Gh`%y zvi;yq`t+xf|FS|B1sV{ZTs}E;{y#TOXXt(D==X0y*+dk5?qoggv}zQb(FVISV}A3; zo*jZ%pVn>b&i&=^JUNwp<-P@r+^EfoAGW^M=J_bIB9y$-YXG`7vsu0`ntdIhKJm|V zi%_7k!Y{0WuU>ZcQ}RkM$0uwOGSt9uv_mhW+~>_hoQlY9`3t78t0$Ve*gi|>sPH(8 zC}$e}uQ1w(R&Jb+B)%|lBlsUSbgHDWId5l`ymm(+wt!C_bvt?O)_a4wQ!CRK*A;KJ z;U;oNqYRQ=tb4Ctm-VZ@5}nN&GO;Z2&I=Y-8M`+0Vdg zU-j}X89?9$Xw1jOc>5!NofT25^Opd(_`kzBnuFTL|L)7PMbVpb(h*a34c_CLm5knu zY2on7F-hFj1F6UNxRDwxH&Zb4qrUDNpqnhYPU&))i+R_QT^S*}B&5cq& z$2xz_G_0kZUx(k5^oniF=w3?H4@pNK;#vd|NljGQ9p@29Mjkvk=!rs}79gnhjO@o< zCwHT#P6kwA<5eH;&0Z_x0se8C^oT~qz0zBrWqPY^{Yz4z83k+5o%#N2Lmn2&Fjx2?b$wP9U%-wd=Nc&OA|dAfpZxGOjewJXJsM#7Va?=p^= z#y$xNvb)SUm)p*9Egq7G7}gAoe;SV@BS_+v0kS62SAWa;XMyhkH&9;(Od+t@=CFg& z0T=!5W7)KiFGJGG!gNBMMNHb$&+K#^#8qqUnAw@pQ7@ZY)-naMt`kpovX)?CR0U~} zazBQq~9k#7(i2 z>N{9TS1m6n;SN5+P-d75HhNEV5vUU&RB-R+BFV=tfa7Op@6qZ=tkcE`lF5&yJG#VP z+2dC`@-uAxafj;scqP(i=}7&v3fBFoW#ql_>{8^r^k*9fH(QbHZmAX(MnOgD*^6Xf zl3}@w4A6)#EL5&6FTCP+gu0tZgO4^9^0qkh z8~}L%$+^rk)s1Am-Hf5AIXy~s{(l+BDTw-Kdk^IRVqN=RcmMGkv-i}DQf3l)iP*H* zwd2#tIX+^$cP((`pn@nbEMxMXlFk!&m4iYK2*BgZpUREFugExB{pe#t75A_*sy{)3 zdDC*q>omQ>7FoA9DRy9GUpzvuIsLQdmv}UC(p_({8nvD1DeEa>GsZF1MG!izXwVx> zi&T5NNC+^dj@_)0HyedOpx)?42Y92yDoG6?V0;4>jJC7Zokf`AslwsGIh?;YQKAFz z`I5D|;(BsUe2v+B)HGLc+G6lQh8b3y)56hNBgv4L@`f!wkhSe0h_e5|sO|3}_Zi{j+icXdEY?^gnjvRnpvCOLXuY>o`wWS1;qQ zo6!#Bh}*uu-?wVzROoq^POIv>e;*ploxYnwVX%7SGON3gIb~^9a8H!5jP7FFDVmow zQvvkJ&zz3HF+ip-GWE>A!mCfggLdz@&d$m7!!IdhwylK3oG7eW>{J{1v1l7v-rHs* zIyNbr5cPS4OvBlAvRKRr!&UAA;Ft62^k>Ca*ifl;9dj@;-UJzFB`yY|r-Aw*5ygHS z@f3iP(y5nCRpxP$!bb42A82`TQ7@s`vh$#HcJhVd0DSn!xcQ@44L7qM*1ugxNDD+R zkS!H-*2D1dSW~Co2Fq=8NT{P_S={!~U~WIz?`2X_oO5{zS7uxHRMVXRaR7s*>w-~D zbKA+!q7N{=NX7u>!?l*00-T9|-!PAtAVW=pq>1~y{yA)<=nT70jX~`QBWV6lj%%#v z#5tGx*HO9){GDq4O_zqk8qKh;ThOTLY zr?UOHuy<$>IW71=-8Q@w!Bi>AMcs@{6SipE|>_ZPLW)8m?9B1`T2V6+RBA@nga_%?sBa z{O`?yCh*XFq=Xkw#$6dSP5a}*q^a7%r@_R6S4umMB*%}7cijyanT??&4hZV=6my!( zxihQ#&Zdh&olwNSNeA@dZH+tWr({?c;S8hGGvW!Wd@tK>2F2OXbeut1+~ZA#7N>oq zpv99UXQ)FY?OMR3!)0vuds>9IaDBc4b6GG96Rz)n+m3jwQZSP&F6!3~Y@$!Rw4Dd% zf>z)faT!igiR$ooMph4RedO-Sh-!J&yKw}u$P0kWi=vo7d|55m6@$w787m{qa7i2Z z+mp@eP@yq3ud+8mkkhKhgjyB1#n=Ju-%}TI27PH*@iCr=4aQOiJoiXzXy`R(YdT1` zI5b|gcD+vQa+Y@wCYD50uhM0D*rr_{;h#K}*V%nEax|6qBH@Yq1kBYpR=d5;E49DF zBPl9wS19%lSF9sRk(04QU1yV@@R^2P<*&iMxGxEK(A2L($PhQl*_yO~frochcEHp3 zE1dJSXNNt|rsr_n60az1!DXvtP=A4p77D65SV%^V=c(cn1QFO{xc*$hd z!Z>7Mc?-Wx{bFQhrE$L!LNAQ;YyEr3)3i66Urolh$|0$y`0M)u%k)@xyOHL;>lduV z#k0XaPw?-ul`hvj@Q_xy`;xvDkuq8KT-Dfp)Jp49ocM(BQY&eLRG5A}cQ+w`cKNN? z5(Kgn@hX0xktB+TBuYR}oO-M!tm&cLN{vvFF#~mNu=~tDHtnY^7e4%;+l9lBJjG<- zBBZ;Ib>YUXZ6&fl7Fh|FKP4+vBiv#OOWUc|XX=v$ZOU+~ItO|R=aETKu8p;5&#Z;kq1LG4B%e%tMB zc@bp9{Na0{dpTV3g7fA|&N5*;N|Z~jKQ+DDo|R@CO;L0+G;z7zXrttMZz}&_CFapg zWMOoH^`&U?Y23Eou$tGRYyO2yu94fY#+Yn&_pne7VO#Fe+iVsO;6-msM*^iZQT62x zhtuiDn3sjc90t|ruLQ}pd}Rdd1n=Kb z0GtW}Mjp`Xl{RzN9?e173c1Qgw*5z*Rvvg2wI|g6qR|xF+rt)IP;=|Xf6Lv&k z9({t3dE%nZdLk$N1D41pQ=GaYCsJ&%ZDlUSx^fQf&%QXIjNl(up4vEw0|YHiL!3l9 z+VleHWW`At_PiNS_mvD(`$$-+pqP@*FUh_+^JbdV+o5{0m%GNQDWGm=L4l~+T<&_8 zxrg-=G=CQPfUu=S8w%{$sKP){A!|wuI}X}CRq41!F9-_Oxmo7+^Ox)g&Ga# zPDZss>LP%Lcm2*}*be7Lr$uKbviaFWxY)dG7<{=M-=DJu;z}2ZJ)b&STAs(L z1zw~$ZfNfrCs8!xCQznF!!gy!fvYG82YJ=n1{XX?_$!6|-?Yv*q4I$YK=NS`vyP|( zW#Lo8jcO8Xvh*-_Q?(7$H7A&jEB!_st@h3YW47Tv`K^@nRjEIhsx7F^e%riU zs2gUVPoM5Mr<1K}G!U6F?Imzi(t0c1x2NZonybTa&)9$%d=ON!eepdYMqA8~ah6JOZ7Lohj*vmQE@d#rMMVOxWgNAGAKx`QU~&VJ7{ z{T?XdoXqEDDr~<64oaipN=x!z>zinBBpLl$Bhx3!D>2DnA`WEVtD@g=^2_C7aNP`v zI_BiZEV3?7G-GHxrBk28z!|q;iGr;Y!6#jjvr%VJoO$CmLs^jmv}Te&hBf>af!ooJ zk_AtU3uQhJIy6H&N{!w(6*!>0G4L5Jw?~5rIe5)HGGrH#*B6SYq zx7I26ldC3s<{b;kPLt(Y9(Vbo9O^E&L>%uPei!i_wH(p0%jh=sU2ReGoi03~1$>0y zcCxiJMC+`2@dt9m zsckQ}>Q1vzoyku#Uw*hCEN2bfeCtP^A-3MMe2-1u8D1u;8YnV36!4O^)ZX-T<)Q$= zIsY)$(OA9g+J)t_8>W)lQ6>J5T&LZ$N;bWU0|=!J0Q}n!0tXGpno-+ZeZi8Nu*K1Q zpx-a1BSEM@Q3wB)>vDOyy}!eU${;r0YFv`^Olp!emk@X7`Gu@+0RzYlE-!BQ%*w%l z99Z25IsAydVKbU;X}+FvxL$P#$6 zD-s~G?~=nyiuF0Rw_QtKFccx`?1;aJDrisEL4ou z%XQv)T1zf-W8UIIX*WZQ^QW+<;8d!!(Tk3shhIhFmhuVhuhN?c3xCD%+Paudk^%H_@%9s$7s*}DLZsNPP`w4yNku*5%7l4iWawg5 z%t>-L$#hf!YC05EKOZ-*uZfc-@8I>UUhB<&4lWf;>o4m25xG_P!_sypX-mvq5yV0c zObs|1o0&5=kzz#U<|exVZx&6Qd#V6(r#tlSzM#U7k$73CRd(}lQh}r995~UOj(Qb= zw7x#zJGjW=M+o90+B7@E`tH*7@AuYhovmoPm?WN?+kBmioA?}I+N$Ix26b|z0 z?$51C&JLIKyw(AGZ5W`l$PxGY|xL zC$uxzmH{GQDZQATD?$-&SV=89CZ(X{x#EW#sic6LUd{fOBHNyIBlaM)&M#(}*6>XtwTTZKz4W=z^) zxcjb~H>@en3yelc#)4$_e+u7iAl^+$>WPv)U$H!OF|W9L5_Wg{FmC&7JmIib$L(lA zZ8Wfw{T%y{Lx>c>&nesWRZ979X4A-_n10!~1Ppo|*t8zL0IX%G2L28;jrj8E+n$E+ zbcTzbJ~3+SlZ3c|uvk3m@6&<@ud1(cfA4oLjaMnC!? zG51~C3Ls2oxUBb{0Jk1JwOcda!72cKkUaG?m%AO?`VM;UAqRb!p;Id_rYM;}dGnef=S`9{!t>I$bXXh(aNd4UzTWS28wQJmhG%Dfd;zwTYWES9NG;9e=r zp^ERyjg)v*~WN**m6p*nbje8NSXy zx-2MYY4-7Bg+f@fl?ft*gV5;9UqU$S(^dH*%*2%RUL-ai7yQ%VApg6d87d?QlPF^5 z$h!3p*9jnaLsrxeb5rTWQF{CFAZ9>-Oc)~Ni1cCdG>XouSaVc9omOfQzkxJ3 zNMD|66;CZVNEailhq~QtD=7@&)>C4|1;K6j2FFv?b+#C)_>xF5=e0+ zk6w%_@Y%`Lp*E^pCn<>V?^gQKx@gO2f>_YC^QAUv^WmFAU8nR(F#@Lg936T48S+2e z0Y|I~orquv z!)T2ky*e%RxRbzlMTFLuS6vMH^ym=Y_;Sk;h}+E%Tz42wEwC*PX!&ZPF+8(3lw^}3 z=mRP4dIhK0W?s2XGr>`)9KAs06@eacX*mO{=cw@j-=#BY%hhSb)MFgG3G4kvQh(B(lR`*z!eo~Bf7~LA9*wSFj<1{aoN?T$>Gl5{ml?mKgkZ1r+fx$`cGJ2RNFz)I z%~n{Lk1`p$q7ya2c-phI>V3!%P`A6&LWK7*?PsVx!*e6Iws`@_N~nH-P!SlUh$onv zB>4m!5+q(pjMgm<5C(OM;|^X)^QCKC8C5o-={L4e$>`!;^Z^bFH|eqWPvL zEMEEw(B>-{BDC@G4Ixpv`OVmm6wUo--+Sh)-<(5OT61q~1D3l&QPg!a-o9-GL;7!7 za@`*;-0=P&Br55_~*IqUMgme~#Fh;k%c)dCgxIqc7dwmhsl@cOW!W*j@o@=m|<**R%fvK*}NcLjhA3uZw*@nNSz!&4n*cQL65@CFZFwR z-z98U-*s)+4M7VGbdY%pkSYnse+@Ix)VuNmn9Za@dh|{UlC(*HlaFhxB$}e0ES>@M z2@guatWQnd)R;jQLPkWNQiye`&X|v*04)`u$nhqg3DB5~f0xWl`^=QAM4Ai~An$x` z>BsfFBpresqYHVm>L``WXoYJndO!Hn1Pw1AH42>^PHHD-V&t2zXBg0|csnsyqp1HI zcKFMi*#xvdW6{-B8a$1eIUP=bq*FBCytv&Dmb*d9>l-GYd9C(NNCV!K&6dg$sgB%? zyJ2626U~S@9jSA7Qm3bUqJe;JZKqKzqP=ND(q9rhmFt%pA63Vl?lPB~v65nuOFz>U zrZ*)(Ya&^{pt!Z5&*O!B$;rQ)H??W2HwbP+DsgFHuG|6KGQPIr6Kg`O?pB3q$|VP) z+^VEVW>1L6OmA0m=C$ndrp{M_NBxDRkHKmZ(Q&N3Ca(2uFHPjs&UL;B0NaBbb*R{f zL?q~M4ufc5zriP<)jlS`y#LgHQ9AHo@$NPqtL*W&efBNohBckvA)&+hzP-ZoKRrS) zYS81sEhXPW%{JtI4MDO&ib5gEFCJBu7xYI8$m`j*7UG~|ERE|2lIB+5SJU(tM z!-~EgFYz;BJis+~=p@OPjpyCVB_d@E!Hqg1C6|nJ_DtY!aEH&~Bt*S(;UB&(NK11_ zHgUeu(ZLq33q;2*Ra`Y8xD(IA2N0CeL01(beb>@iC^-@5Qn)-sW?YXEsJPRnccpx= zT=(NZYB?=xPYD&RDC!XjklMY0%S|$bU`!yf06vEsBi6iOqOdlzbg;oFcsvL)nY&zoYzu-!4?_wpY~c)$JbgHQNEm$W^Yx8{i>wv5vPTp{F+IvzC^PD5 zCQQM767rL$j~FT~N+Yz^F6*j8AxHKF7J>!?hX)v+qlx z5WEkk&@zJ}g(h%);Lr_f(-cXBciT=XwO{yDWLGYI8i2Ue56j)knFfDKl5@4!JBPk@ zk>JM5=Z)W0^3s}?c{3i-tVuUXhneu#D_ty*6jnMMGgi?PZp%ckXTxZGJzk2(;l!P5 z5-fzivMOyJOEx^XtK|#WT9l4F5kTcuE+1<|6hPWDx`#OxkrEDgC2UU#m1hyH9s|Nk zBe(i3^kviP1jmubxs@nMfeGn=E9%N5mla;6M3U!mIi1jGdnk;gnpc7>i_Y8Y=Ktdp zi4b5l<>LdFEt^b*=9e;oavg!mw8N4$Kdxzkfx6rI5+OCaNtb3K#qRx6uQn^IX{gTY zx|^s|&$oQ_nbkixizR5^nw(61*={c}ba|x@@-OGk;Z%fww(bew3MYe90%pq{NB%C& zp(7O<1JpfFW4CT&=U9c>)YqjQzb1aZTw*k_KaTg0gCZ!y>BGVJM8MV7#?!PkD${y` zrp6Q?z9IY`5%Kj;AthK8WPoUI*Nh{jfV24ZCMgZ_NbgHyqg%+*Nw#uttBO-idTf|+>>srm5)o@)*$cym&$1(htKS8%$GhBM} z<$SGi7;?FS`0^+I#O6Hh-R!?`=>7ZY>v}*#IyvU+s1%qID9Ad$M>hm4Wy1Q745=An z{R+B6zbe{xtAgi?MX>{F%w~s&boOhY7kM+6KnmWE-CN0Onk{=#bGhPawpM;!k)=-2 zxzD^A%A=Zd@1(XJB$s9jI&gKY-LB@nlKmaT$hfrF9@c*ET|n_DB9V3o5}joiL_G2* z)7J43@6^3nCnddVlJPGk( z2iBL3TJTX766qYD|jr%k`cCo88hOcVBB3HuB_A1mD zcooROzVRQm`gC6zK1mc$;qh-YM-8A2kv~pQ4)CrEZOz;Lp41V~$+xp;)~4%}4O7g`aPR$2>~3958fol2#r}lLstLmU2QX+B2fCV?0`_wL~|k4!>JCcjD3#YBeUr;X_l2BS^X93z)tzb-h z_Q;yLxMY#uVKVI`g?tIk4Vc@=lD8qS&cIvT3wLQMb&w64L5sg}XNmjQ7E-_5gr8V{ z^4hPol_%af0R=78D7+@lwQ8Yw$Lza5Zq%7Yh`fxTr^@0#PXdXaYz3`BrG9YSe!Bst>1T5x$VTwdwJcakserD*f_ z^NBpq_YoT{&5AGE>}RT;^@_r}LcFUY7Xd^{Q{{%$?`%Bc`7PdYMAqRI)ox z75YydVsDbkQwR?Qz9Z1It1{m0wI7`*_#pOYC}T>%@<9EC6mVGU)TVZi%$^8v{bwZQ z_*TVHl>mHLp#Gli$pB}O5W@wbW2z>(^>#r(KOufqzoc*_B;ECQr_z^Y1>msDGj0;1 zJB=#B^aL1iL?PpVWn=VmiCo1dS>ei2V-O_FfX+ntXzYPlr~C(s3yOq`=ja9lj&llv z5I0M1DdAsaTr5X^qmlW31?!>6TxtWFVC@gX$9?i_a`gJBzp|Zo?1LV;A##Zdckjpa5JRnH+4A*1(3MZhV0(S_ z3Oj2sbI^4^!nxLSw>qZ#UP}02?}SCff*MX^!OSk|9GL-Gy;WV(>;W^B?0{k>pmFyV z*a$7GQgSP=DinTcY1QLZMrExV`x`7-RO%<70;2~_oQ0fLFPXI-Ec?xf%`BtxhI*%M zsf)q=CbBb0j!$b}ap!CR(Vy964=RpiO(s;CrBEQ)l@Gb7Z2Zu^l2BWa{M&2*Nz?H< z%I_urXeD>$(?th1q&dV1?ObiB+(>&Q&wwRFk#4DjxDpYrGEbH{Gq}2$M zDz$>p6Z%30X2hu=* z9?|lvo+-~pF8k(X1GeoI9wBD$Th#Xw%4V}KYyZXzbxP@JSL&Ei93enBMa0!o9wK=E z7rz2^;Yz@hX*$M%drkwS)yb6@{CjW9E3q{{G5AfdhGx7p$*+p|pJjwpByz74am;oV z@e8eFHgGh&Z1RuNB&RdFB{^zuF`ZerV7hzWAPQE7XyodEn{N zaxGj%a)VKa&;apU+tm)1oYdJ0PGUM@?^f1SDqWTyf3Xnk*BTX6IX;rMyq_mF=aMEJ%dsl-*{cPY=p)h;W%0uPu|axU|-yIgay-m4efSd0uD z0UXqI0|q)}l=SIT#bP*lNwc{NTsX02w`sM*G=LSB5ubOHe&l5~(NwUVS`wyY_gKWs z9SW6^TA#T3hpe+G!hc3it|DWlcEBNz=*)zN9{{YOT(uc{2tTrnT$x_X)dKe&399Db z>3I=oLaP!M(0C2>SBqk!7Y+A$L8&oLXOwE{?s?06Ob260q5Pw;doeMGcfSATY4eKd zS9O=HH*l^i4NcOl>>CcE(y?w+L0_rKQ>AHRqK!O%xR^N29G8V!K(ns#xlA4TBx>6zb%dRqiZtGE zb%^WilD+P8m3z!&KyHiHEHP#b-QIHiT5-VbdALbF2+an1oRLeNk=yFGwzX!eEfeOJ zbo-aKK!T*HWc^~)boW)Mq;q?$=wDNTR+0!y_q8NiNSF+(b!t}lbrgas z_*+Sk>7jC77Y1P4KJ?0F{!ojj8?OYJA74^gW?#Vz$eM8=Jk(2q@A$u^&N>c zb-@m;O=1sq%gR$juUdz8bU`%3QmwjL0$Fpiq&dkEyg^>Pp*FUCJ@5JXLTqoBG*c@- z3h%Uiekc(I4>!^aLl~Aj@tKy95hYiaL^d#mA@#;f*5O_L4kMT3(e^k9u{vfIX$R>_ zJ5D3cNXFTk%c@VwM>5|uhhXS&hKx67?L;Y~WxOnI`Lmrqr!iQR_z0)MW&P>M;s6j3 zZgFEt=iIKd*T!rY$)#NNs5Q#oxB@gPJP=Zova0}~{o!q7A$L$o2S$K=O!XCgzjFcV zNi&pa3loj9cLt|0Qde5aUppU=>@G*U%W=b0PM8mLa)2Z$jJC4}jb>#`x9gG_ZAHw! zYud%R8e#I6-s8R`KWcC7`_CCYpCQC8N^?i_ z@8)P8TMIvTVO-KV@C8UFSY z+o+D7NMbgo-Mkc(bwdaH(MBILQ$|>O`w2AR;!Jvmaqexg?thvP+P@_-(ExlS`N{Vk}O5rfQtk0(7*66SH+NW?u-c?@<-X$B0+&7f6TO^CuGpE>Knu?^f0kChX! zFCSd{ma=9ekUjCD^5rZIC_^D4C)i7T4ErwKs4lRG=35fhw4oq3f9=LTA&y?N&Irt1~G*KNg!s;=1IN>`uWe2>_Mbk}uz8Y`@M z{QEMB@cCi87wKRk6^Ese+xcd6Q~5T8Tf`qVdzGG6=wAvzh6U?L3-2~m*)?LM{{U)olI|8s7a>BtT91 zIg^zVET9G{vD%~F?r5E)r9eFT_=am)HPMIRW{bB%8scF{_Q|~4BfZ#|&aUKsp`xQX zLL;Qi2>L_jRqQmbx-Q4Fa+^0GEBvZD%L1fd%2)>cd{=d&=~)^~U%!|VwID%lqMzDg zxKoLIHxCX&$@cF~7aZNozokxWT9UzEt?NFEdY=51!C-M+9coj^wfxMj1*7dGu*+M% zzE<@m*t=`VwhUha%EI0_Yug^pkWs+(_>gt?Yaw^J#rq`-DYy&2!jrH~Mm5oG^E}$A zbxJ{Fkr?YOh?B0m$!@H%QQ!+}&6cZW_zJ(_oRGaQ|0X;DCqqr?i=8@xn$X5G{#&USC zNi$Y?;5b)>K*bzE*o-}Nslcx7z?@buN09Te8Q7TN7@`XCxmW>2QquTqN`w~h>or9! zcfNZdEO&;j=}Ff z@#MP|%qcIU=+nv&kEDqlzrYkV%FYF$1p=;|Wx%5roazgG8wctz2Rw3>ThvCwyXP`n z+W9?woDjl6grx(&v&iLn5S{hMs-{<23~%RdeH(tD0OkDfSG;2+102+R5Pct$@=(sl zFEW_-Ahfrm=NtPIqOk8y*Z)fuP5*=V@S1O<>ZYRZFk%9Ciy!%AEo_R7pb z;6b#!(|)d@gwFa{HB**hM@DG!!NOOAxxI1DgeQ=Gs=39`yDj^TvOLvJSJc2GqOn{U zG|JY&$&I}x=Ne(d1EdrH3}d|QvzY42tx_0S8vCh^)jUe|3Ro(e}ikSSm7V5Pi9JOz=FB{Pf%ox?s98Yf6*~zaF(woMiZk!U2$0 z%>OEAMn4%MnNUrtssRI0%5}QE^)+BFhkCMNunx&Lp0r?Smi@ui8RG53o*QAJ!Q<)x zON<3H?cr=B`bayEAlH0Adsn2vSa?faVyU@6Tgc0wL4?kTn*rkRmTaaLN;hNcV?xsnnRr0=|mY-XCUU`A5gw~W^B!yf4&79%GYxS zTV;u|zFKkUS~svmGvmdru*xrQ*rrlJma)eL3jfVA&`GE+akL5`d4&+IJLBp0x!P}G%4i`K2DB_P14 zhrOvXanE9~BRtz}G@}PTX17m($_X)gYmzUNk<&G2c2x@>)s{c^Sx;1vKP1%TOdsHV z-q__l;d4VB=8f~Z@!-dGjmNjp;51?|htm-?_o8*^?5gseL1|f*)_%$v*ALRNH$jh_ z#K@Y!j)(QWSn;}au$p;|u2&q{)YmkY4wnq*mZfz43T8`BV=GjFHkpeStMgrx1H8Vx zE*6n1RT;tvKevE#eW`9_!hH}vD97>AG);4{0w-LE&EPD{TPW@76Y zk2UDbaj9U8$5LP4UwC~=pG?XSv_S=6DM$S!c3cT&LOCEeO1oXIYa53N#aGIVvH+eA!mU{8k$-111!d_uAJ% zEjOLoUOpSbV4LasP(&dz9=BPRir0K0Hd)HaQAh+UGt!Z~mTE(O z56z@V*5G~v4B`2D-$28>Iqs7oBMd!;;Xxu_O~!UxSqi+9X1cZi5|S5a#~*n`KO5L5 zn_kh1C7SY!4t4p7ZpCiy{>THQ`#Op7Q;K+;B%pf=eLyivD`(~2E`Wn$uP*9kRk2M^N{7a7Uzv~Bu4r)Fi-Y1R$DX!s zy(P3vP2rx>R2ePRRTO%vobGzY(O}p;$R`()O?p4@C_?u)8Ntxli1u1D^&W}TwlSE* z1g3R|9j_ENYJNN2DuC2MLeaj zPNx^QXtJQVprMs08Tl;*^F3;cm$0Jb={w=R1ZTcvi#CM4iBcbM?yWP|_m4w}_FV3< zU>vDGva_!7CCVCDp)cL8elJCA0(4eEc&jg z>`n?K;LGh&SiNL|Nej659=RN<$)o!HA$}|EKi>u5(5NogzJr*$(SCiXi|>8#m@ONl zaZUWE&!b7JlVon?w$0;fb_TvfKnFjXx4qHACbb}E@bUHbfN9tbs;7Ccxc`XDfAN^p zmw3ccf1@&akUA4!l1Mz|SsX3WIJC%?rkU5|h(!l~DH<_K4Rd~qhkEAhg-s<|AHM14 zfJi8u^2tMUg!ShCYG^+qQbN!gL#lm>krbvlrPMl!vzfeGl@iZh!lBY*7Q~N*xaai4 zlzH-6U6Y(9LbwtaP1{TXzCSgx^qYKMyoCzK2a&?DJ>!@hbcjtF-Jd-lSCq4;hFxWr1A27ywgeZpdF;4C7+3$LA|d0))wE#h zAih4Oeqly0`rzDnG6DjZ2yz$cp*7b?1!p?414AyN@8&-RU30pkDb`!5z0b--gdz^c z9yUed!)2hAC4TYv_wC;FPmll?ia;O(lc;?g@fWn!dK|oZV&>`+&}zM5dZ_cqs`#Fk zuh?SnEp>*Y;$B+?O5bPxvgSVTm2vU%=r~tZZWq+qSZ7gD(8_PAhDWh6nuqT4!<&zh zX&SRN`G(g3mhz3H1mK%@@U-ykDm0^l;k9tJb@r2f`~m(1@>h2YuR$ zF;f_Ki-qGG%5#3rb>r@8^kC;rBgu;i!3{m zR*BEurI9MHz#G5peU^Y~H5BN0SE|e!Fl!+!AW>dV$rnxFTaD^L(!OT+EpXL7UEYUFyD6vi&6|9@uJK`oD-{!g)4;QEv^RJG0zxavjEMoySt*(5 z_%s47mGo@F(XwYxR*CO&5jp9mj*1A`5ct{)OqxmKiYtpmRo_P@$u#CMi&(U!bP)9Q2>x?}k?r2zK;(FW8Y&F3zfgMGLvqBcXNOoihh9h6n)R?UM>1@}b}5<=xtJw%Ds;G}XnMV$;=om%9&p?gjqhx;uFuc+nmBSc^<&wDeI%lKO6~1 zJtp4xw9hk(z^wpLSddH>t#1z2#%KBjc)>h7Wq0j6wU+*~o3qT0#HOt<#h()+y@7&Pj^o0Xe|o47+x!b-`* z{_T50zqk0+bBtgtJ3hB83@*wV9PpSOJCoI4^RXaKC^);r|D{>^h?pAzerPaA&XGzK zoiTbcVt#{57; zD{ZH}I69DMPjoY5cdky54W!owQ#{bO-v?S!%#`l~{77J^Q zEv<>e4AbYy00n);Y^f!K-aM?;P~ zW{M4Bm!u0-W*!9)1!Fa~`e>bsjC36>7fUg+N&yy2!M~bdEv9ZJwfjk2iC!sCse``N z80QAZkOIt0yq$|uhjbrC2IYsk;Vwg7i|DM|$#KB*urgu2-x)I)myy(zN)GuXz?d}T z&}-T(CuVoXNl=gIkYHeM`N77ut9aPEAT^dgvcS_2_F$MY!$rH^DxY~~t6w)C>Mz9w z9J7ZV&{}GvK6ZZzm}C2a1tzp_U%wN*KZ>gA`)-$RQ}XyB7i;^=I7a_8HPd?{<6nt6 zxUF42+dQ@^%|0=}ej2WfeKB}>>SFbsYRQ`!=U;+mMy;*0epu+7_C%<5!Y?E);0diH z$iuh$#BFU-X=CU`mZ*^ze=r6bmX-b=7&t@@%+-TN<`t5E_g#BFtdh4FQB4BN5}tP&IAT<@`dh-KT! zK3x3e0>Hq+FK?`@LHpzLmhCmiAAd_{4Q}s`=b!Hu@SxRu*k8{&R=ja-vgQyMO#}^T z57|SLuCO%l@kdTy_^qQzp+v@@P*na^*RYb`YHCvcyUQPq4pSE&KfY663C3M2r%U`o zyuwTl?Q1fQ|7SUjYb+il94ZtpFRo*1OeVPg9`8i~j-b|(e*)w+mXj%6JpBC(qL<)s z<}ncOx0z64OsSJ9=&xM1(8d^N$+mmM=RFF5c=q6Guc!FxsKRNp z!J|e}6g0)PahjrkAzpT|yF9&#D2ix~U?Y8{_0h7fKYGjK_k_sxQiLI@=eBATKCmR; z|1E-0M^E31yM6%ki0b?&P<ko6~FOCJg@vj|AtV9jjGMgY+cnQD`6haxN zVxU-vQ!N(``e*<)HGO4MiBmD{@a19zruN&x2nk~nPs4fDs4wKCz}ZVtw`x-pFcoml z(~L8#Wp1RD)I2p3tYsdhARF?aIIq}~_V5G^H%Q+!dL42bub6U+e-t6EZ#>RzxY)b5 z+clEBoTPcSXz`)V?}p*teN_A#?aj;kq|ts2qA3{lqh2cId7*1IRI6&c!I(a+9l zA~{4vxXnfQ-#=7d{x*)6fJHSyeeK&a-;Gc(>)$SA@;8>E1CD%x??$mMl^*CrgY2i{ zCU@F@qEdj(BhmWtsLokGdOzE$xI6n`8^Zje0YBy$k$8?-k)Z?~Kd6Y(Mu#>1g~jjt zHtl-BE(bw{Z)idr?XDo~s(WJVpPGc$H}b}qrjWo_Dv4&?qUe;Opgg9YkD5;$yK2Ck zux;p6kULs1tV&>Rgbuj@MwB?0hPqc*f%DqQu~elR8eV*f8>VaP!UhR3@GGyk|;F5FTRRZBVqv$)QDHe-@Y#{dToA17n3 z4^ukDPevJk-W+N2T(9zo>=l^cq~H;`8U^@QP#Ux`M4I}sgDw&lKf=_eQ(R|+*tRM- z<-hGh_wsv}f(m;0A^uQaP3_aDi+Rnq=&HAI3H6S~tiR8qt5$BE;ET@)277%zv(eYC z9BjMX%E$h-Kealxm)t2VbG}6h4^cAsGlS@c-ztpCYST$g;V4N_5!NQ>G0h?8aluVk z!TQS<@1k|6C817AEL~xc!O_yAx)D1@Ba;OBb0!dJ*X#6jy^0&|7aV=-TBO%oU!g8_ zf%Uk#CE`nna^SrgSG$4k%ui~gzm5xS)IB7#YYxz~UW+j#YkKiqMc?0V@92#}O#EDG zI!ZA`O%)mPZ7qdWM$?Pfk4_70Glj97VE!gOl=XV!kMGH9`J5aw`!W!_G5a@#^q-8b zbr^*IiG1yVrKQc~PtHBpj|aVf99(&~&YeijuU%~n^|FlA|EpPqph`rKA65UiZ_qnR z#vY6F+P$O1(uwg|Z!w(x{hcrAQs_!IcpHe~?G44-)YY`_!v>EicEB*EU&oX$wNT}Q zndisTSuOhaT(3ZPSt!RBpUvX!NXp_hQH)SCKHBw2q{+x~wJdjdiObN)->?-C>w9_a z5a|dcwJ66Y>uoiLIdc8>eXews%|B#d2Oo#Wx;Bib0}k;^UEu@0u(j2-5RXzT z-*WPN9EVK)9Qzz$>wMuA&PYK)7a&W|B5;Zk^tc@Y@J%N7H=(nWvFH{I5M~6mq++>< zl^c>{YjWV`q7lLzGM=LNyLx3HPR>$9E)}S@7JPyiZ!W4Y5?RzhuV7&7vO#EGYUqqr z$QQ6KTTnalf?GX%@gISY4GADLg*;GW9%~tY#R3f|q-mtb2n}i_9}#h{oBT9h_j3+) z9LFi`LrV2c;#5Xgc|z|?DCz)d084;u7P~(`B5tr2%Q7E(x*@wwuSmn-PVIy)gK2*tt zm2?TJ>B9x%)y4JawF6hZr}-M7Wd%Y!_F!uyOSH+9a)L!qo`5D)C`}d>=GAFely!o< z#zg9O3TMmYbrnUp^9UrpsHaXanB7MT@C`fWm^prl$4!UeDWBle>askU(!G4>)_NV$ z*6MX}G2ztw`)yh31a`rwV}|cLzYcX&Cv4GZV)Q*9yQ~qdAN>Sw`s}aryP^`Bccb6 zx3J@^6QV%)f$P}sJ8!#}A{AZrIK58s>;P@B(!BdnT2(2OQo5bQZF$u$WG$-RsaHOi z(n&{CUx+V@piss}Q3+1{s;o;O{bT65ff8R43Hxb}er1TBwFq(FLj6}(6xNx*%2Xe^ zo_DD7b%NDv5Clh^Frn&ds04}{K-<;H`qMIMiPx3yB}=_T&;n-6!V6zIzF6Q<@IZiE z+jr{>*|t47o*!TE&KhJ)ywmSGecerp?|(=rI5Hl^!2F$qiU22m~H{c-B97Ctm0u!V(yQ8LiAFkm=OFm$yP zpYPQ^#FPt55$2kaSfa+A92(7<9%IEHmraia-F<$-bo_IvNYdIFc`7?k$QfK&TF}8>UlwIz;W3yx9M`LqG9?7i85n-+ktWq^@8|jmNEUT%&Zzg6TDJ${m49kI z$`ND5yeIrh*pJ*6#seXI4w^!qNvt$Gq6JJFNTW^Cy8YoOzgp$+lir{|dLgcUS8Dvt zrL3{lIKwnXBta8ZYmj|?`$>Y8sxR@a*C4 z^{@$jM~TkL(hYkkZcomQ$8_3f-#&<9w5HeMe0#ss(r0TIhg6S&Wz(d#9#xEJ| zN-=3D0SgL@_QbXr)+-_#boF}69=C>OkH3$$OB_xj!&Mxa9IcksP#W64`ha__& z0@E_PFlR&(RxxgnrNV2;Y56#Lq=2_0b33OEoQ)50&!mHxQQF)El0_3Ef7~*4*OPpR zmlZo`??+KbC%4>!gAzCQP!hWwI7$^N&;K1{eNi;44#5tsswa2e**7>u`77j2yCT_Q%IbGL zm%4#;@l9K!et=ao^Bqupy`?uj^~$d_vT3cQeGFi}UHaIiFHBKgO}pR!J4ZkP+~| z&;$2q8o2UOG3$VJz2vx~(p;uOn5y1$J9eoTkVKX6Q51#2DT9iIV^zm2Y%#up<>0J1 zTUIq75jYD+wCO=f z%l55CHHHZ8)ai3orE8`!->)~Ri;5^se4jpv$))a|&+V4aMG#s$FV;oh6hdAuZQ2uJ z6xXMv`5W*MzH#V(dOD2Yl$Kl3l~QJZ1LMu#&tdG0r7&en!)ryoP53WBwt!6+slo3TP2G?OQi6~b@YA~U zyW`%0_EFVZtb|X^C88Qr*YStqXXr1(&z^KrUpx*zXUuIg7{l*1kY;bPibIk%HallW zeI$JdnsIUB`8^>l?!V;7^iNS+NMEVM$|(tdnGcUF273!TERfgSS$6b8_=#9&)soHJiBI{|lfZaFSD&LlQ$5}Dkh zwfZ@lpHqSAua8@U6jY}KgDacfcsZ)GX6vGjyjjsVfq)mKL8K8lcen&D+D5VErds^ z-8LLtBjEa_P%D9puq4<$nZSB$Tvl#(GO@1FXfVwACYN}K0q2dFTjlXv+(;+;W6JkSZ)u(MJ@27U^6!)RHc%Fgq^RzHst;$_=+Z8QMBnxf zWcf8WYwKpG=$XpBLGQ58=U+;I`yNYECL;xC^PfHf{4%d*VoVvOEj~<5%FN|%8#>u? z=$;C6@DJqjCR|iS{8*{|Lt6^wskVExGuY;Dy#Ep!@D2JkTuA1#{VyVgvlL(n_w%*b z*`5)+Y&R%@U%5F!CvIxMV|$DOJe7~aOA*^D=H&Y&_O^$vQNbvRV|9qG$6-1eWu*yo zS;RR~KhLz|1hUhiZ^A-i!Y%K#$)p$WK8s0x{U~k4Fz>W-wy?kg#8S$p*?h~)&Cx=# zLQOmAE3~72@-G7-Gy3iaDZN)_Q?8iaSZ#E4RmlY(JtNiTsHDe#VA@Qq z5c?@B;V~&21Q%(Q%D-X3o^9nSP>PAu!^UmJEj*_cCJv7q!7gb0L81p`sw_ctc5e%Q zF3altXu?MvVm`y{=3oi-%XOqGA6{v=!cf(2G(I>TYa)3MdQ2i_ru{}l9SZk zn^YfC>B-jmO`Q)Q2}6UInYsodFA*!MSttef_IWC$TF!G^c`(SVn@JM=QMuVyNPSpc z2#i2ff7m2E#7hr==p@b7gNtY}TpHw38)9{s^h{x>>EopiH&~2l|0Jl_8*X2qxu>bu z%O4ruj=Nq_8=}GwM7^qciLXiPOpW9?oT;KH5Gm}tsNy4HAfN8XgV>&K)CYMa5l$)uD^TL{Xgf9yF zJNc8ppgU1CSP(|Xa9Xi@d1E6nw$hQP5B0Tl2SeoT$9Uv66a`*zWcu=FhK*07AVdExS#Tu$PXTC>RWWur^`LkbA5=>f`8hV=M}cwBc``@Kz7+e15&tSjS&~oznI9$Xa;h9~@pOc6 zor`%d_|RCSzj(0@aB?SBuWMNhFtE*hGUtFpE&VIVRT=lP-3gyc`XaH6fduHDCiPI! zA6i+eIwd8NthWBx@c9sam20pv$mTm{yf}C_l?0w6KwV=C1lrU8^(YkuHb%fT+G_@Q zsDi%uGrb06N-pEywNg0W7$|76BHnK8t;Ng1{d^pEK80K!&ZAPdJ3xC0u#*;)UlRhO zB$H*uyW;GrVQjH;A)&#N zES3>wQx+)K)6xLd$%Sk6@SW}16*UIQF!cCHkC_y8DqXOi?aVyA6cNfSjur+atG91G z;Z@ExF`^+wsxtug#SSR&e`PRi_^b$d6xH{ z&n#IW_j4&j?#p;Bk&*I6v-)JI%p&7 zaSb4uF~UbWVUg3crEskfH)xPqsbG=jA3s_@vM8Y_FCt!0p;~t<1+C@-D2oRlUZ&ht zSOGkJ{T|ceuzF%`QFF|DDIKr&|)1tdfO{*WuTt}C3Tef<5cqsF(5w&8eBrk^l9?H zU{j7uRA=kUTnEVq^U(}vAd3=X7_(Kiu%Bpp{FdY1#j|gY!hNq#`oW9RGG}swT>)lb zuqCOoGh9%df?4(6;hE!1e0B>Jux6OG2q7FKG85&>n@7hB%kYnPw<0pw^mSCn0c4-P zc>R{UF`9{3_wt8lnB+IV(?u+>?)TH6!fO#hi)dc}oXGe1T)uX$RotS+Z?=ql2_75e z-khHjOrjf2@v5el2-U(`LzE4FsDK5xG0A2Z)<>X_nvodARBZfuFN5JN>?-qegQ?vz z*EvBBF5*XwZGG@5GsFN5OKL%m8*HbK89WR;Dnm;_C+Lxi_37S5q|OnhF->CiQ1TQj z4qG)R2CHZjSmpASP7LD)AREO<958oW={kkdKyUdiSH)jpWj5YA9B&V()xW=2wTpcU z)__?ub4$<=q1l;O%m6p<%678%Hm zwUcfReS>6cG}Fdgio1!1*LG#D;>tcNIY)M*cf*)UCW6q%g&o?^Bl)F3#gCdg82PG= z!f_XBDS6e}G_=HINn}NJm-#uDTs)prg1lv7#xDUbgnsRqnlQ|4G$Tu-QNt|Z*V>F~>G$h#RtA@PKy zPE0Sl;&E1}a*HpWPR!xGVia4gAz}m-<(hj41HuQwLFwY>hvC-Sf)8trQU&0n*4~}+ zhvo9Of1v$<_z^@W1wuO35b?95VP`WiY2|}6r%JqZciF|0{adz@UMCP@IpV$>udMqX z@W-U({*QNVYuhyji_3`Ww|q!51^_>q@e8CKiHpJQQW;?m?B*-P&(yJvQnjsWQ6wNo zAl{0=>$ULgtRn4rZiY?E^GLFi^}@c%ctENSXii3a1itTGH3XmiAk$m(s$k|03Lw65 zMfA3qxb%G{vb25(2Yq&&BNFi znF`|wwVST2G<-9Ib03=v|GFB154A1WBsG!zxOnr ze;FMDKUj~jQv^KyEQGwVLphD9W!^l!&Ys(T)CDuq+u)f1#hdd#6ENID=HaLf z6yW5o;%>7)<#w>{bT+i6$upT*#=X2|%4lJ-%W_el0lZ@c1ljPb0O!SC~UIVA=-OZlGY<(IDfR4@^ zTJJQ6!yM_Wgv`A+t$*K+u79Jy`Q>}mhlk$?J(oDENs?+7 zTu2dr0a7D;uuK^7&f)$CG1Tw?bf|4@zFS5c4(I*wvxLI$Y^3hU90k8gZ%WHnBJNFo zM_;sKAEeznuPJutBmjF>C{M!8f)RXofMw`pFp3%GVs$Rh@H?vw`S}z^$t0 z+pdt=5jTOSCgefP(WgVhb^>}~C~%@}%|BQ5`CY)tP7bM_>)AX>ukuKawWJb>p88>D z?h8@?Haufw|g&^B;|+!_)strqlt16 zUVDdW`(^X(G>IYqSTBntkud0eyYM&1IStNDC44X8D%Z}*{OV%h&S zTG8?*-sIw!#7n!)mx*Q>))n7)M0F5_j;NE@6Y2j1v3)%1*-WRQZ2!}m^!WMKQ+DzI z+uu6gI2Xf)4yQ9uQa4^JGuU_;@n#;s7D{-}Hs_qU3$B;YYf_(&!Yeq&-Gxsr<8I1O zIlJNOyC#wvKvUTHWIwcneDg#DsnQ&%mgRII?FFcNe=ihun#58a~X^4S7rvRe2rWDQR{ z((eBr98R9UFyDa^>`?=v3oeHulZ3PM(qmfu&PundD9=_APc@7?SCzxxQ($Q#+93&5 zCN#FdZw0c{ZA-&k8vL#JcdIUyB|^3`nn5u#&%&+2*JePT6p>!|j#bUL%)(bz-R7^X zN*ZQ<`xd_=5xklRPbnI~wQLD`UMb@++Z(z%_=Ed|2KX=pDq*|Hm_BF{Jkd`Z?5R2{ zIt)_t)+~S2ET=YfT*)*3@@#}W%=}|JL^K~cx=L^V+9o7@68!2nGdHcJ0r-L?JuE6< zO5c@^7QFWqnf;h{F5-{pbpDWT#Ddg&47eKubL7yT)(63oyy2QN{)d+C@;cDP(k|ak z`T98kPb}47#h`0C4J>*A`f>z6mmZ>2+IWvipHoQ?aJ^r0izb20rlW^ZW?>Kw&XK;; z(B-IDr$@+ZD`rl=ty4vNQpZ`#+*qE`5za^=WS-H;#ru?A8E;(8A;d9?<}JC^*EK3h z!k=72<%rnYd_MYUGf_QL1!7!;p+#Z=|3bSBivmn5*emTH5u1>v_3RxlOeCj1%Rq2r z`#G(TLlsWajjDK$+9{bV38Fd_YJqov7Sn)WX_RTB5i<3_{FKyHrO z;}94fGk2FNWxf0xyGG_Yz}>t^@Ptgp&kzR_;};i-A&ENMcmnG&1Mm0jAF`7@5G^@M zGURzPgmhMt_eRCkIZl9+*{AE;^M>hbhNtuC5$i_3l)qG&oBknbNP*Nz9CxvSD0e0$ zrXq5(Kgd1%e3h@^Fb!n+wpD^!NGADe=k9}FF0@o#l+3g$EJZ&^e}VA%G;Nx4L~wtd z?gBry)o|gbU96M%xpan|aS?3NxU}YO;W?V&a**O-C5j83_cR9y2GlXBubv!~0^vFp z>)IL+1J6GFvp3!6JYW98RSDv?A>!eqAzV4yVKBrue=n~+iZog!GxZr;+=0^S-PC_e*nlh80IciRw={B9u zz;IuylIHi}a9>Sl7p!X#<99zT@^ds@LgIP_<%eFZZ$tk>gC?Ax7~jogtMhY%%T`dY zSZJV%^{FTViUNO~jMG*E;mM8N+ClYNVPjkbR$M1moe252At^S>6k7@~X}@cS4~w_X z>?8dHBXa8}?ou>-?QTiYMSo*z8Wtann*-znWhAEYv6sYK@(Zxgv37M?OHxzov_;-3 z6K}CKif?J)w`}2N>Ds>d6@0$%2}#*z?oTMky%B|OpuiV13XuL;4zQHIA<3TX#VcZn zaX`f}Uzf(GWIE84jsgG%^mVWCm$xfCxvr31djz!`nJr*t@u}6`cf+Yxq>y>Ky`8U( zc&mcyzClxPn-jAOWj)mqsr2G-p|T@hKs)%b0piC^=g~eret8&{t@+jYH>mcNPy#Ac z<#E3w{4J=eL$qm{YAJm7mH(-Wz;4y%?_%J`3W(i;U?8Jgk=)m$O8&F9PYoAUh+*M2 zVLMq&4P95@oAAy=ve*2R)V$f_Ma(rO2}vZuHG&t@&q+d+g~9~`+Le@1eWGsCb83O! zo9Z71jhgG4UC=dPlj7jjtn~8++BN;DYcxDK#)lBC)R0*kZkwsMI^G1_F+~Dt0Xso?yUQ+2R%!)*H zmhg}7n&ccy;G!DJ+&BIwf+<>j{%BI69I%j3*f3cGfMqJ?$d7u4`C@gl8#h5Pi2aVS zbirW=OX>5Jx;b4V?_m>Zed&4agvl8pBKD-1)(klD+jZr}G3$1zK26)({Q2tkK%}`7 zzy1FI19a;+`2FS-``LfMn~l*lDIg&VZ7ckLlXwEgiQCHdXJXvaTB6l||19yGjd`5H zfQ>E8mQAX06qaZp?e6rKL67ym<#0Qx?44809q;aKiZa#3b&{WH;6v&7seiY~Mpe6B zn-m@;T|k_OlKE~Q&hyu1_@luUbRHGs$m~zWHs!=JjWVM6$u&jP!CKFf+q3(G6goGC zd{D|_j!JQyxalWMmB$%~N0nT8czFqs#s>laDj&;#oSEWI^KGyBTx2D1N(fITk>1j~ zpj{;sms;vKKOQeR`a97s5pLBgKOapcvTk5h^hbST2fu z8{&6cA6z~NtF3e$3*S6@izOM4{UCCnqybFwB2fMPi{ElXgWFmdSJ{!G{MUkbrR1L# zk~m?7CIe4eRtNGOdXB!%MPRgyEOSXv$!11aTHWtWFp{x^zorQl0OAiOJ6YKQY;MUvYsPKj3*lX4t3}IKokH zq{P38Ls4|-HwF8TOYM^VL_XK|&pl~=5k#hyaot(A_)UWNmAfrH^JCna1ppGeb)xi7 zmkzedJJhLeJ>OXDN&jATe9NaGh&LJ(ZDU#w3@Bx6|0VF=Yw@0skyQI9D@v*e4O%Vv zWD!uHhZC}S8upco0=6c)#r6urYO@UsomevA?Sv!PkvW)x+5%fJaog?#Pxyr55VpP#Liw1oo8#SzhLx@S1U_wWN5CqMJk=ANfd-(*}jOHnPRcKifrvzpKd|8%SL? zPnqb#g|2CZn94mux`0j}4OV{OA*6k!rs$fOXl^PQw`lSfy%J@f=Sr;rrIo*?p_E4O zTwTaSP9H4TYe_F_EW-YN0D&1-9!XA^H|$`y9-mfJe|wG+Qa!W;8S`E| z7#4c9mNPcjZ)W7m`}{CojPac={J+@4RWPH!@z@!*kaNvc!$1U_ohE#lT${k{T*K$L zf==08x98rU1m61GzHMvg!uF5n+O08409Ri8G6kHkloUAppulnz+ZR6NAIi+5(6Y)5 z1`^XEUWw|3KrBeQ#9%g10Z2GPSU6TZ_Vo@fGdzK#V&|%%>l31snY`A}rBaSSBD1sV zAE)jMyg~aRxP%P!m^|d&rVnoW!LkbQU*Es{hsQkAEcfEhF40@e=y2hOi=72`=8Q2r zh?f(+(#f>`?}T5w43q6+yQs(*MR{nwB^Oi^*AcM&$gRA|Z94FgcKUmWbOF93CLSF7 z;OU=kcn^?6>{_TbUA;BesQXTacNZ zu=7V(C@KTLBT7}lWJWy;Q_(;dls#G2pk8CLZG}Aq<|~*$4wLjhw!QgiG5hyzGPtd( z;$8B-&iTS~li%Jej?PiGd$s65nDg!GAO6!Br-H1u?Vzlik4?wPGl!fvp0}xi-qW|Y z+xXti55(O+5XJ+LVoRBeoC^7mbyQmyQ(M+ht*GmCl``IYRFl__`>l(*BbrNt{foTE z8ASu0@u}7JzfSAtpSG;piTFxJt^d$*W^28!s-xLR@PuV`#u$j|SvKgE(=UQ3d4O`F zI^*$ZKVa-pKku`?*Imc04^&||`&d{(BD;xRTc||SpYGNUIf*ES3u8W`+PcDCs>LiK zpya!9&FTH9`@ROxGw!y$>1AfCEG7PT{r0D@XEFA0;CCE58GXlLe6dQ#7Z|V7;~VFU z5Tip;O`A*MX^_OBD8#@(Ure7{eqX(-bJwjXasSot+YC|Z$sRm^S5Q0@?*%`j8QU;r z*+fkstjUR&mp)Z`x5nj1l`70DK2dgzTGF#u8Rq1UZQj;gQe7Y&v6v?uERzo|ipO}V z(WLq;H8l#g6$%ynr9^eZ3u_nC10j-bhm(49X(_mY6&1GggLqUV6;&`KtgTh^bnBX(iJ@OtX zIUaFc*n}gTwkOaNC+#|LHn`c*x+|0yh3$^O9dw~f8%jXFm&g`(m)6p5q@-L%DRc+kxduC;!h@ze+4{`=M2GCEHlyerg>M z5e8M4g+AEg%LKn85o6k$zdbLa@1x1MZTy>{q@nn~j#a}w7a55@)3Rcr9JK>oF#mAz zlq!Im0c#Ns2oHx2>U1;Y7Yq0Ia~@n6I`)GeRSk|l^=*!i^o$0-AdXqN>+pZgI4-@# zr*ORamKyfT3-xqXlmR88hs~3p>jJ(0J#AYxlO}w8fZ91%8~`OQe@XhxZAiivsN-bd zveeC}E~G`CFc~8`EKT;By*(A=m72?^EMKHU6^r0ednsHWe3QVzc=&A@KB{zvmQzeL zDuOj18MHlL9S=@^0W2{~x=h>L6PUN6C*DI5bz5O*_ZgyopKCZz3URk#=>)R}^&FSa zMs2ATY6EXKIxt^N(^sIY6b3>!7rr%qUw^45I3V2cCfu!35(E=%QqTAHSV|qK86czgH{nmNDoVP&35BMAXCC=%N=-$n=%HUl334)>axb@ol zzVRSmf~a-h!1HT_&W{`812UugANGSfXXlY4l%@MCH?1qn4Ew4^Y`_8{bYeuoL8gtu zztt;#&Uj$DT1LsGbEJN7PdWgdd|ujkROV}`T$<`pQ!=@9_)Rmg^{G_UYIQ7<&|ssG z2DO$??zeiQ@99+6-qY4+0#3Pj(#Hn7z1X#@BFbX}XGa$vMZ`JR#VEtP*x=+Hn8K?U zS1n#P+ckMmqJ{k}7+Xs#J?Pmi zwkxwdUi=r#9!X)6%?gh32*(R;rPq&1_D3_t*QLE*KWOL!RG7;fj{Yb#`WWg>;9vj1p$8@-|@DX2NE`HN4~J1 zR48Sx7zmr&iK+`s*%#Bm7>{6ux89XuJ1?X4M>rQNzPC8^pl&w#lt9>)i*M1Y(x4jn zs6!PeA6{e^BG8nG*w(BRA5w-@7wPz!ksXN?oT;_%Iz()bTF2>v%cBT8G{)49}jF%;TcS7IB~nPT(VGx^l@+ z6-f#|yrt(HMhckRCD8USK+mjw@rXkG!%_#jYx&sXN8PbqJ(-KW{EhP+wc(DIOw|-g z=HB%f2FKX-9(aoc^U>X&ay4>o$$E8zH_yI&+=v@{gL-jK!F{BqKBDbUK}Q@^*EQVZ zDJmyk9ndG0kWa6v`UeO&Rg7!jNv|2v?Fz;hKIMgun$~HmUY9OXzc4w_x{L_+Qp!~t zvL@L`!|sXY^TXsi|30xmt=~8ae{Knt@wafk;i0_LZk%0E`Vl~#t_p;`xh*86n*E)% zMC;uOrZi;mF5Ch~EgW!+mtBzoP!nMTp?ZBKMi2L&%qe&)7VcvsQ)no}_N#;|7@hVk z?$6zkh;44->q_SSZEYYkpn(epU!x>KHw)L9ExtN-pLzYp&h@vTqD^Zp$cQS zU5jY6t@%SJP(b{IDjw0&J3!cpx$kf2e){LdC$9@d6Jb|?3d|*{Ve>>|Q%5*!as2-0 z^KYuCBl{&%<_%rWQs#iI(dVYOE>Jk?XS7QG8ck0pVvG^@wNV}~r`d;|$Ct?<-hL9u zH&tQ-im!q##i3=m2ISB4kvWLpa==%#=&anqAcyDI!jj=UKCbMgj*F497M}8y-|)Q- zF>6ow7-uV5BMAQR)(R!PxE`onzzJwjY9obPT^2$d1=vX#e0=wv5ojnGR`xsK#Q{Ml z&W8!eH1gF1R7}{=SI#onr>AK7itp0_tCUx+&u0tB7|adN^~PGHj(Ib0e{%4oQ|-h; zx1ya#cIZd3L4)G6NAZtPttUmIHznVybVx z6ap~os-NzSul@^PuY+W+b|9Tczqm?tgk(!QUT*k_{Lw1C+Zg47V7cf@J{ynl4khAQ z$L>|>-B5-R5&pPRx-Y-;3pSK&SB>o6anLLcOyJ=&14c1dokKqTVb{ad<&&j@ zfYi#z%EQz{y=>?8eSQdc&^g%Fx4q}AHTF4qbrH5>%)0oV!>&#sk^6S%+Jc!7R5t(#Hvq*NK2@WE`V44iBX%-lzfT=^1okjMgU=ifw%P+SrD zd=x+CGf--L>uM=e5Rv^x@f8GLZAy0DzgH6hz!>DzXDh-RUwuWp@PRG|pi*i_eL@f_ zt1RNDSarH%wj5*idrY-94?60Ke$8 zJ{7t59XZfA_~TtUou>jb_FcY8y1$J?aTW@dmBceISJB}C+S7zAlMXfN{?nwSb%*6O zj^AO}RB&5!ypTX?nf?K0Lq>Almb$RBfe01yIw#7u544 z`vRLupWJ&|UnvowlQk#u^e*!(aYk#QuAT%&&CrE*m8;)kytIrdmwLEsCM;DE#RR#& z_`S5d;tH6-@AIx5<|aodQ`w+gS}Vm~z=%AZ3+mF3(pd4-&>y;=>ZylHZj`xR;1Ij#VOy9szYN z2W;&|<2WKuw|pl!l92LD>_j&(`giy#8J*=L8tUCz^WF0sN9Ju!wTe4t{=9C@im9y)IOHz_1YT!5o>^NkAkz& zIzSz-$$R;=u@}f$xtozQwd#)u=JE)Y=5jub(7axecFszrYqCi_^9Bt$83(Xl%tZe> z+YYrXl94KOUP9ORe6@!m)%&g|14}xC8wC_Dz{!dnnaa?0Z>gz-hmcNuaVF7xCjC>p zU_rJuCterR^fd`$qD!*x^%RdKF%(f0FB1}smInuUFn0pPf1}QRT)3hEkwr0undj)}yXs=cShld}#*%afo#P`q(8G#NA%1zByGAM9z} z3@9qogBY<=SM5*1>r_OB&v`)S=FaDmiZP}Lj|3mn*Ag4WP>TV zas(HigSPw?yx#Mwh~a^cy((IqiHMI=BJz05(~G-9#(XhI+NPgmD_?PFvKo2f!ivN) zn{BC5nx{M6qnBe>*x4KpP9=` z^i>EF57z9~B25hcPGoS4X=%*bdO43laW3{w1J9{p!IaDpxlepV2i2x^Hq*I__ zqWY9{-Ms0hS0<1UWm*ji_yN+a!EHGrDxYawxQ_b6mA}59Phsz5{zl~i5-|cp<49I<*1AyZLJtyHdx>1~+}L z?5Qm7c=*$!M(v*fH>d*is``)5RnPS334TA6)`frrxR*YAs=$nMPUUAlr`qLjJ?K6i zFPZd?_e2Jfe~t@N*9m*p>nM`eA$&Fny7VbVkN-Zg#4PjlJypW_IOBkgfuqI~0C(Bl zna6{;fKY$dGH0`0)Buk0<3XI*hB0=ZW9scDwnT`5~C|^KX#(R5r0;5-uqSHDG z4!P2JyXpCZhWi}RPvrrE+}dkT1adINJ~#UiN^u(7tI%7oJ+Ka5eVfjK0KE z*T6)|NBWqt4R$}(YeOCdaLF#RgD7;%IKGm9;C5iPVVn?ok|J$*&ph8lGQs`@F~dl( zZ)mj@KUzfz1FEQi)UW|wA;EtrwT>R;g=7Bq0e@vyXM8Ux)h>jZ6BUh`O^RISRYiDPW*#Iwld|WQ)E{ zCrV1{SP$Ag2?U0j5crJRp@|$90}L;(dd#;;g}cDFE0BcmssT#-%#`(5PU_)QP=k*z zjsCV%@XpS-EYn$h;rfL3Wp7AXtp_(0e^`)4;D_l`+0FFYmc(FY-3)TU4Czx)xQWndoifJ1+45>w=&@QCiz4ct6J!3v zPj~~YE&cW*dus}h<2kUS;iDyI^0X>2gZtBa(KF6?i<(QLV-cM!ynAuh{s?Y%FbV@k zloe}rqoEr+faSC$KY{Ns$+%ER_<5TTfj0M|jp=y4lX)74W&0)z z&6AqIZX``WLyIY_F3LRzaL`yBZ`|Frxv*&zAcZ=D25pm-Dy=GJhTb@4|wH^*-x~QpF=zeNNUa^cmIHcnmPcOCX&cd2WO>Ln<_K zr~Yde+nN%hZC)|N9;tagE+s=hEQ1r~clUC{u9yTl1yQi$5kr6;OOv;P0^(TN<_QY; zLcvAasQ4dqW7ubn=y8{mPEdGhmMGh|31Z4RYGM1e7w!0Sfm-=5f1kE`vquWwFE@d< z$gkz=nqSjVpPdxbVw8Oqpk5Y&F~lrwHE;h9o4}!nRuK7rrs!CMw=T$h!fR$Il6(m~ zyIZ|ChT`#HekazeQm=WNWXTz0A52zdEn1(p*|u}OPA(c@+N{cG^Up! zhyJEf^#OAed1M@VLXKGbdZ|<9@^Bk zAT1+@`hmgM1?^`cET5m7mzZH5LpuBYwY%3J*=Z5Bvm+6-IhN#4Yl2u`OYp4(=+;oq zQ+ze^e%~_gTC26^-(O6Zr}q&zK$T^6kKe~qs*S?b9mWpl^RIeaD5!zD_cjOu()-Vb zrt~)Dp0TW-k(It&sp_g+iF>95fJyxlkjH3@9j%u_54baoi5)ez{Jwk_P=R}y9>7o0 zG+xGGUhZ08HF;yPKG^9;v-S*?>~MkPu7S%lsLVJ$U#9b;EZH?7XSP|Rlf{Rg36>lnUB>oU>KOJiJLPr86^*~;7q zNS68i2sOR^!`5Nb7P^Dbd4GLf6*6 z5$U5VBfiCh-v_e8W;E79n~p=6ePJ5HKMp{KX-fD{pr>_w@*uXu_FEF(04T%uo|`a8 z9cTQVV<=qp?3rCxgWv?DD4LM(Ryesl`*VwmOQE%?|2Ay4D(DO3*vpHvQJOPda&3UP z} z2y)$&-bf+>>ib#W?bTcz_mm(97_NihXr1jy4?53`H&pxG>6-N$_XEe_o&rXI&ns>88h1g>Vw#^J{ zg{0+FTSeFWia2p-$2L5lz1FF?islSMNY+BW*5_oj`!Y7+mNu6LJpQqL>__!zSdq_E zt9~LTXvN?@FS4_gR$};v56|g`mGV`#Wt2*8N9HSN3brbrYAia+c-j_jhEjosKY;S+ z)z~g5_Ywc;K~$ragxa266g9C-`wvI!Qo=n*@<4C@{1)0KgEmDG{n}u(lEQ8F?y!zJ z3~ba>LmbS@8M|zwM+a>7SQ=E#B-x{L+lxjnmIq39K5y2NZym=q>#%4UXHJ-7DYXJHD20;WwY;gD})VXzvh+>W^xKWT)uMH@ab>_WmNXh&4wlF)@jNLD%}g+ z?aB&W#RW1>3aj41bQ1RjR}UB06DVyI2-PENJswmV1q8h^!vitxwOQ)PT1>GY!5e-7 zbUf-|+Y(v{jOobp1(rDk5DO4u-W`=Zi0= zyBQeuR?*)-Z&Jac}N* z4v)R*59CW^8#&2kw*xm=4spr~1ZAp{%U>U^wXA3WJu9BYlPvM;5a~|vJ40N{yGu)Z zO>N_;iWCD?xnfhk-Ebn7f_BuPomd<4yw`B&igBSc;f&!7tV zE}TL;swZ;jnxTUSH-H`6A;+PYnlRGl`0Qo57h!Y4NY$F0es-}%Wx@!5!Cei5REJ(i zs;_~9+s%c#y&G&C{u<4- zRkP&74um~$$fsRJD|Zv!$)Ayd@+DwriBhlzP`)-xyz3a4%zM^6P(ca1src_JYM2Yy z&(>6{gkdVrS|k8TW5~rz^>79T=m#d}H$QaZizfVkwul!XPUs@ap*P zn5>oDvx8o#PC0~D^|6LPFvW^iVW+Uxfm9p|5s3rj)WV%(wU%e7r#pKfGUPrRA4%Q? z{97zXj&Wtae37)0^^cq1aVRML&dsmn5(UHmBMK^0SJ3?x&11Hr%gxQC!gc|*xRe#x zuE@P?XZ6b+l1I_|hssigr2P2nEL8sn#Cu-Krh9sR&3RFhW zGG2wyeWfYy6za&%1F3}HK+UL(^hc7u|3%VCs4g5ql-MgH5D6F#!_c4h0>8q!shy22_8>CAWLTPCNTl+#4noN_T}B6>R~g7bmZlk<|+^F01Q;+lO!J2~j&_r-KrYe@b=dv|O{B4zwR+l8kMC zeg6mtY9IcB8pE~!J*c560iy}+9#3$2zZnpdV1crs&xFL-gMOb%1@RCe<%dMY(M7ExRAZ+@dhLO9r*yPHjN(A_ES?r}-kFe;w>DKAI6+7uJMd>6*ixgw)@@w9 z@Yjt3(25MiTSlD5v@JU84jitpMqH#P-!3^j}*^I_2;Qv?p z$%uE6l(jim^6qjlih6k1S$%(Tg6Yce?7jHnwJ%1R731;gwz1r%vK1bv=%H1fdygiz z60oEouyX8`Z$eJ#ioFr7axabP;X<)x$kg}x0&&0}MZ|$4BF;V)ZE_tj7ucp7=9Z_8 zM9+VpbyfX7SMp%8W;LwtXFVT2yroq&)ku+U11=VaPeX&V8SkZs@TR zCwT{}Q>KH{tWQJaw4yNhUwdOv-0g%+S~Pf@H$NsVdBF6wvb?CSd*h zk%>i7p%d+QGuY+WU1iY8vnmN##WdYq^k%mN4Knde2w(SX#ni9e*j>ozOVUcjDb1N?kX@(KCqwtNSNh4`^$(b z#@g3x)aUmgJMcdSKf|vfa2kdlBS_=s(dV5~?@xH%u>R z+?#FI-bV#YUCVn%DIx@^6+%Hs(j$poYy%^2HeT1Oin+sVx^2#GdY#?0{to^!DEY?1 z!)RH(YV^lB8vx*D&i!tkW6GJ{9r#jP3@Kk-_;8r5aw}&9$_L10ij<&*JY%2Xrf;hq zOp;%Ks{fbu-s-+2>Nq8{yXVdL(SIL7DBd;da$_ z3FB6`56&O2S~A=m+?HaUNPBL*n2~$U7Fwafpk<`*`pitQ!kJ1HBwJJ9ht#V(LRnK0; z7sj?Ij^*J{xKrL z5(^MXVj((>gb6<%dBq%H8}@5#s@<94v+rz*{YIn0cgOcsf|pNwtrQD4-#&64tS6qA zyzB6Ya^G)ejLxy8ioH5!IiuP5lJrAR&YX(L<@dbSnO!6`jZthPQ}QlSQD*WB>0n+~ zKjv6jrqS&1P)B_NCqDC@-xM*(Sjw!wKAenA?}nNS>1o?rX7gIR!5y}xSpgTx&ox$W z^Ge%9AhC}wHfIMa5d>Li9eeMUT7%*SuU08wJkG_)?ah<%qt~Hx^MOBo7h7r(O9ID7 z`QH|UT~GUeKN@4`ngaA=MSndl2!a~#qQ87SbP9XgKzDB>C9rR~S8f|NCCKVfB|Y7Za4{YZ7RBB+>g{+9}oyQRWae3?#VH7OFBt ztRr>9MBIdu)vYlUJ4_nUS-G7(w-u0|7g$-7eQ(ae!d;j6c2lc`eH-ryA zj*3Xdbu=zQ$CBf6f?pLlK~C?Ozp&HMGGgy>q-M8SC_`3fA6w2mi3m-2pG7gskKflH z8M@GT97b%Ecf^rnZrba3c$Mk1(0%!b^sTKLpX^9Nn{6F{k}h2dEFsw+xETx(JRWj@ z1VSSrp92wO@3z)oyO-0=|H``P_EtOUnr{F7y+IjFMv`6eItZ*<_3D>-As(zGntWug$CF5C?u`I=hg|7UI z6s{!|QQhY()7R*pAz^KX&D#G+#p@(^j&BpWb!EwjNCJn8xtjZ@s+nKceyb|IG7Y6Y zcmk{6wC);vGvFelk>;;mq!YQo|4}Pl7dR_ntA`;Rx7cde%n%Hs7oToEMyl63d-qeC z1lF6hqp{0$iJ1N^YQeOlnpr_-NeF%-CgzI4{2&+Zm|s-B3lIoA4k0lRi*aYuB<2Tj zDQ~bilrG6$Ey?23CdS?DZSUs?qvcP`q!u6IH^;#EeolR6ylbmWQC>w$t%B_@A9=4= zubtoUqJVxComM6b#h8)WP7R+5ndv;4E%K{6-FftLG5Zb8IT_qY;m*>0tI4oFPDW)1 zEtE;_zZcG8_l*$7D|Z8SBG6Kvy&H5bY>ez-k~3jeM#p9v&%Ns~(|4!pMffu7rAAjH z4NF{>GCh3WH``HRDEMRmb*i;YbQr^8fttnZ>IVEXqgX*PWSeh zV{c4oD^p_CBZ^*S=-}Rf;j?`G%ix-}2K%4e-WFTPB3*5kl(;KeUdHa9;q^JUc42R? zpx~POF>3y$ih6lJA2B^O0$AjlRRGr*@hV)SN%>9Wxxv#l2DHfz4@W9Cd)NAH4j57W zEBYpb3Y3qH+hgaai)w>RTy$JSm`d0x9GmJFYRQcsXI!2n(`mnNTnSV=cW6;1t$}=x zYU~)0d&7SScwW25M*wnw)lS%v>!Z;b{4zM18E+qPPc@IJetS$F-n+JqM?}l^rG6dA z`JS1UDKfo682CO2PuHV|&fa&Hfd942?C#TUK$h#^=g=c~=WKBpZCGvBC><=YDBNq5 zB{Ji~FcKr2J1_3qivTLzl!~7YTZL7NW-v3B*%W)|%+*<6c9XvWnN;%AbL5*!i$^=d zW9eSNDT;anNojvG-)M*Y`@(ko`v&)r0C(qE6^({DwrWiH9R(n;WRI6)X*R4@&Rn@Q zb8RD2X*JNC)iH3RO@;pBvKXYjv)~oaE;VxS{>!RZCr;6H6}?vU^>?3!XM?Eh`!d!cDJwY_ak_(y_iz`1Y(UK8 zHG1c^@+6z9f1&~(l`X;Q+7c_2vkDjz>_dlM@TA|Pks%90jgPIdKmlqDTMN`issh~e z9LGK3AY_@et6PXse~6JAT{Pr#%b!5r3#*n#weGF|ifoNWd{g`Q_GI=THUF2*w=`O^ zT-j0>82vm#@_?DSW<=paE#49sN z4Pu%KxJr>u+mlT$&t;3e$_l@W3onUs?CTCLL?@VYK z!7xEY0*Pi)=}}dBts|qit5uk9NVX=$U53`F55K<^P+;g*9X+dbl6m3za#kt&^V+w1 zI#9A=M>P07ezVN8-_L&`p00EHATF7m6a&bCH_57%>cxEVq3A9}g(gAxrZo=H5#qALhFt%Us&FA+^j9D zUAk~d)yAEJ!T@wC2speCR|YEt>XE%e-%wo|F7SdFNiK#wY=^x6AE1*6cW^wLD^$9K zPW9(uXAUvkJ$Y93zg|DSHTw=zU#1ua!_=f=e}9vVhCm^bj992(yEQWEwt!N^6%^hN zVofaZ6CqeLTv!8c#IfLC5Q8Lrl4@Ly-f8T5>@iF*ml1<(n zi|0184?qX3S)L%KI_?7+`7CN)x6jC2`ut7*TA5#hEW!wK2Z|%K39u9MTduv_uhAuH z;eT6Z2|>ed1qU}=C1BXuqK3=?M;nm-#zS5?=XU(J!CEneyky8ftG^Ho-jgKD4!JTHt`(OW>NIIgFVf-Y*~#?B0q54zd3I9O z3OGp9H&sy)jM4tAepeX>k{)od%+14;ax0_+-Ugy}x{pgE?3U_4tg$W>^K=_MSx%4- zUGNT%fj%=rRC_Vqi?Cz;CI&N;hGIsa_@=rcs)5L1Y@T&JQx170>n!_%?vsEL=wuEn z_-*jhdyKES1N^(d4|^!LLUiNkzG*@bY6n-jxBVuiqq|EdYf@2(^F5j2@?GoTFTPo zArfzQZmJ@cL0Go3_kyW=L4Hd|nnTDFEZ5AU8~kx3sgLZ1UpioUeo8F$2MVwVMPG^= zUB%`KbidbN6hM9>kh8sU<-ww98%G~Sm`*57Od*RcbVmw+!O^f@9`=a|Kz-e0<*o^L zCE~~#qPC}afHv-RytP-_o2`1T5s!tcN z2b;{ehtSc7W2n9d3nPBqX&W-P@6Uq+8GZP@*D_do7Ban{qA>luHkBH$K;}lhVStv} zN1njTY*Qmc+`{dr+`z}nTy}44nc9;`o2^Pqu~$$c_vi(`8X=MvfTgcp{>sdl48r70_XiXw48!7*QFrQNP9@}gU>0gV zZ8qMkd~lY(F6FXvDT@>Un#&31ic$<4u?1XibTNdXlVmhH_mliBLjAdft{oenq-**( zN-0ueS`x>qabUEx4M|a}0OGQ{IiWs04>=rb{LY_( z8k}DlZt@SLu=#FedVCTH!WK%c&yO0PG~Iz> zy6zt&_q=ugBUWj9)t{jhVF@=CFc}IjOQ|y(j;AAqb6(nYLkSXM?PZY2&htB{G_>NEfOPwA` zk8;(Pk?j7m{bwJMdVI8ra8U#rF&~p>#~G1~x6(hhvl;eWzdHd1Lr#_XjYjzPep^a0 zkC=VOM#;6Wtg{p53xrm)VrNo~{3hvxvM?e*r3b>f($?e0#*eHxTQ%nu&FpB?jfhs7)1X@0AGy0&xH#lL^--urbQuT{(EPweR6jC+R{f5zrXFNGt@7Wgd81C!QKHufykVZ7LD?|t^JOuGf&yi^V0D`C@sC# zNwP0Hw7p2lK@kKtBwex6cJ1v=WSirwWQtxcAt$fY%)ryuJRHeawthtE0SJgP)}0H$ zNy0^@3O5*ti%hnWQqoLK`iqX*xqjAVen2KX?IG;qCwyQLFv}6Ly+~d@F!z%yDxf5b z$Y@VrPW-Q9WlwkI74;J@zppw)I=7ku|ETYzmdbuHB+n35^g@FQDydNe{G>-@F4wk@ zzBAsRliE0)R+Ff+0lr`D*J8{HaZM@3kr0BXrGuJpdFj zBUkWb`~M9iaqHRtS^KW)B}RII`4?H>ys27syiUE&eqhLcw3BHbD7egISzJo6&wA}T zDSoNxyPob|;Z1@e_Lp#Vo=_pIuT>B5OE zm;yt@p+^jwz-5n+yp6^$fI>%)1B|BE>5fj|sP5mG7Z}hq!WoFjz-VS7<4xgbNC@>) zJrSZwGC%(ps7o={QYwQ_Pu64Ep0#AzelX~l;*HEZG2se4;bG)NQUa@R%6Xnc=2Bze z*j)#vmAJHM-AFnh^W#fuso9<$e^Ve34VC={)snRjtoZ5>Q6^HhAMug%c)TzBey7}USn(g@L^By;mD+8*O;3+Wf*?2uX=QaO%5gMh5v$r9SLw3 zv1WUWaR`w4al9#K*Y{VMm9ZlqvC5uV?Q*;w|5CL$^dg||Q@okhjVDu|Y!D=(VPOy> zQIBLX)k|!63dFk6``-0+`(Z)&_Nx95310IiTYNP9K)i*dlsKnrU{ImZ;CPD2SW=oS z_JXF}&#$7D=mNtq>hxkRRgE9X{GFxL(>r?DO)uuP`Sj8|7($lBRz)v2sj;@yq3YzN zVrflAQEDs2upohhIX}59k5TTsNOF;J|itsSh%UC_*apauLnEmH#Gb0 z&z_gNq;i41)cvp-QfZi_ggTfW?!iwqO`<~JrNylU5NlQJK1y<9>9zN7=~83;cl}YW z6cw}463#3$rJsBX(N(l|ixy-d=^eLx!+{8t@eBQQMZzeB^(!bX*`HQH)%UX>4`2P0 ziC-tKcKN`3IEC&m+sB@C;=vPEvy?VE`=m5jnB6=&5)r9zj=ImDPdr5U7kk7>PCZ0g ztElq)JbZ;~NO=w$rGG+oxCx_b>{I|UqMAEUUk#}Bi3kjFhkh#VH_BB7_j4Kx+*CWc z&Fp70Ax$mlKFsjuqBmirZa(uBl3Z;(9URQRpr4r1k3_&M6?-=%dXXbHk_1a)3LDu9 zV~_?gfi?jsnz?<)XJ<{g{GRbw!PsCpE{;3C*1z+G0)=baDO$Y!(f!!fe`EP$=(k}5 z%2yu$QvzoT(3X%#d)4=E-~>j%05hm$WSTUb(>I050_cCv>&yh(4V82@<`~fhdSt@= z=nteIUxXgK7INv)nSw{e0FivM0(JTZ3eGBx^o~V&RS&NSq7Z-l9XJt+m`g~@E7kw2 z7H?iBd3rH0f?AOm$j>FbZtxI6)EwwY$$Sfu%J3s0M98rB!AW>j94%OcbEtvzUxAo! z|2qa?$tRam00Fy8W3&9Hex-+qzrltDaBEVCQDGJZ{WTEGczNvN%1ycQxZ)4B^~y=I zlAmlHyY!7OOTH+jT<4fVR{U9yx~ZlD0Sz?+|5!O!8F-R1YJoSYTS+^`Kc-OP3P$0& z@oCk_@seg@if6IRC1j=i1U`MB;w4}eOaFfWmNE{oS}Yl{mpk_XSSEt9iS3cR58qX~ zG&L;mR|BNsQ^gFJY!uP<9fsTDT2E8cT|RXuqD|DjkW6T#UPFFRDwn1H5b^1a&a|E9 zo#2}I3gZFU#T=bK{SXv`nsw4DJTfn1ELyz_$1UKvmr#88S)ebzM(W;Hx`SK6I387& zkJ_?*K7`wvR;PGb1+2WY^jRT=yv9Ss#Sv(axhEX%b4J+BO2U&&`^^xN5_4roFbSi* zvl{UtSP2L ziRECd&ZotQ!d7(^J)u-w>qrm8_V52(zclV3s0mJwy&Q|JFE271B7bju>(P4bvR+=y zU&Dd~qHe^@aEh|eXC9lfp3~n;*&mE@AgZq@v^=!|S`fZWyLjE!>>w88pUHJhSNzP+ z#w}#Y+BxTc=!Y-9*w2ZBb7hdx@!0r#V6Wd`K-KgjCn$8H$fR}OcnNie6;N)ZVoTkY zQm%h`fj+@W_w-?qsyTO2PN0F zw(zVYF)ahWmqI-d!^K$wHwL6<#RS#H&r6-&Ugv!b7Z^Suij}$V)KkQ?EzUo!X7ZG) zdJ7g)_?&&<52gBiWLC$cXhp-wn8JrWY1?%nFW+iGd%FVL%}`Dch*j9Dz7Nl-SRh0C z0J39s0XT#)Obq#c6z{ga^RQhTbN+rEa0&Sp@NbG`9RHX2X)P{(THC0De;ItpWJV3& z9JK%8IUpTV0}BY_~^>X!R(D$ncGYTl~Q#z34eiqMAx1em)N z+zIv=iR1D+pWv~s6w|pDSH&^8s1WKosyK3&{Qx%O_3>6F?ym<=nK2_-Xe>o~5SUx7 zBHu866~6mV*NSxu5K@+6;G`T+I^^ZCneuBP3@D(DYesw^Ud6=XkOpX&0TX1qGPcG` zWE1h!&dX-T?&$<`=+S~m9aOxath-Y+zoeX66~qp;Mm`4qLly-=0CeoIrzNgj2&ZCo ziZqE|$Hf#DM|1M3CRLZ>vjLplM6lk z$CK06zo_!PO+q-zf}jE&qQcr(4n1?kZ{wgd4Db5dhC>JY5n4nq>q256NRXa8;cM9K zwN7x7{TjsB<6V&)e4BxFG#OpxnBF0wg+x>xs1gvEBctiRtP<|iqC}Xz452zv{>3V#oYh-Ef%rPn$3!CuN-7sc z8X@lZZ8=YE$#JeDs!3D=HLg@cXA+g{q_U&9Y=8gQGv^a3*&?#X=gTTw*W+c-*FgpC z%lI8sEjK2nFpi5{_C@2iSWKZl4XlSo%V|IALwhHTKAd?S?8!(`uQeh}A+IXEwJ?XndU z=yd#CXpKXKN zcdtG*Afkhi0|ddr8(s7;OhMm*&|8)C4#^Ou0-^!1nlD!m$5BT zlp+Hd<9V0)mXZ#QPPg9X6(eR90y1iyb>_}K4;}O7XZ2|-aYQ`;G|PHSNe|rgZ>doR zW>r1#PP7pB!RKlj#TaSH`>2MY87|AC6fPlNwebi7CWD)52D9D>*!5*}B`5E~BUs=9 zUs94>jxj9s;C=dSX|?gL!xwRPKk}`5%P@u-5#dpN97vDP_eeo){b+1HaIOwQnu^}w zG)|pzw0`n$&X?sHMM7)4S0jS@QbGEm#Lm!;6$~CHPgV%PG$Fv5&d{JGh0cX&B3K_W ziM$}fnpx>jSU>VJX9P(^5^@q9sV_=IPEMyxh0VX}0)f(Vsam&Pc8O#TxQ0@z3KX{b zPsjtwJnJ+HO}c{qQ~$RR!o3+j84cgRuaSq>zbZ9dg_xvaAF`%`1`QUS?vQs4lCk7G zT?MV@y-5K^tVBB_PdW-AJM(t;P41T&`4;5cU9Qm&c0l1ySz~r(e96g6bogi$rgz#l z2e?042O(>TXd}Rqp;HQ_!FbZD2A#;jw2&s~+ijd>Lo`|fDVp)TNz-wBNvz*)%9^x# zyD<0kOnw8U4$*)D){PvUI~ccnDP{sTb`tB?m>7YEgu^!>f3k0d+MoRmNQ%|G2~hzM z;m7Htv);PO6AV)WFv64*7`gzsw^}#}x^J#n>90Ex8+OR_Ik+gZQK6q0-~Y>}&G%*M zx!4UmZ_E*iPUc=0N^ecmsHYQbFh+b^l4u3;#AB_Y;hOo(o!AvdNhW4h2*Zr_ldsI0 zMuZXpq)E`Z_95~@e@WE}?ZnXyEXR_{WNh$fm~mgVq}${aZm>R5BiDJ^KH%*E@FmmC zEp{(cX3>QFszE>J421NQ-ZOhMVWrqZMD>o78AUpQ7jz2kc2Q?jXeKV88`=GL7Wyr) zTvRV>(K?)eM(XPleoc^HOx zVpI*MK&*3!^FK`EJ|V91*syXc#654+G9_eiB>Jt1F)?Zx=(oCKUF%dCU1eQrv6l9* zio`QP=uWPA1xFciMaz2v*97#e7L7qWS_cYrpo1lvuS1x=La}M#z{R)w;dHU5K!kDM z*j$o%EM}ku&0ozz3wfH-K5YRQ$;)q41qnIJ zDtJjA9Ju)HGO-)Vr(rQBA3iK>{H5l7Rha1yMaPk`-2yYH(WYVz$R-=c$Li>pS^Mo( z)lPosLvm2-3%}WV*WhZ_86e?k`A%JJJjZgO$|^C1BWLdyS$(AtsGzaBqWxSu@;kbF z5->gZfa$i4&sD%~^MBe>9^JHj`>An0EBf9^x$JGV2c7=~Bo?m)mse1OYM%nkt~TJK z*!{~7O+@+iyiVagv~=9gb%g*?U$Gq41B!1uby^tTg~BdPKf_3 z396Po%(tI>*y}>*qjs}1{iM$eTy4H&4@sfq^srdAKM*Ypzh;veZ=jG#s;!ml1+bDO z#g>?s&c3R_=;Sid%sZmj?+a%Ra(4E^fQJ8E4-t)@aAQUgtQNqJMFLEbuSl*98p#V= zx{`nZgi#y|FXB*qJtUsJqlL{E&@ddCs?KCWVl|8N^PzEBNzjq&W5mQpbt@JRI&QeS z^OEg|^dfiEAFvgayVo$Blbs#$FSbv-PMClC)dVsq__rhP=YQ_VTWQ8QdRBh|%sD{O zS9L!Q!Eo{C|Jx8&qGtWWJv2ovQ+j96=l3~hM^%`@9rh_~l^lLPbn$t?6J*PEQj(`D zevlj%C|0o{#;o3hato;x{4NpWN9*Fj@UaI%`SZc0{lRRW(;Q4UpbVUt1QeR7G~Kd# z(kI=|8-oD0VEy}&j0nTWk=YfgjNi*5(LlP62aP~Tz z;VM2MTX28X26vI>7SUm#+Bht^HecScT12B0N#Zb^=8f^G`d~ewR#CJ06!0RiDz_8! zm9HVS_QAGS)Kq>v` zonL6WqDF()uH@U4F8>uTd%hbcN)Wtnw>-u^a6eF&kb`ZaZNeSu4tuU=#{b|s?;ZbD z{1nMbvQxr_dfVi?Ioy;L3Y?nFk{2UU>Ht0O*yhxW5{-Mn4+sZ6k=pC-!Nsx~rTwWv zJ(;yjdb^!{P5Z(gZHe|DhFD65o50$H!0P(F*o5k=N8`+-&qBmJ$@r$9L^+oFkdDr) zv2=W}B?i37wacRqQ0$`aN&;L(&gRYBBRiiB9R{Oi89z42VIr4Ny=z25*q}z;ie^Wr z=N$tmREwpLm?K9i(;tkXh9~)zPy0els9V~%{N6*BGR#>#lbzkv|AJGzPeKq>_Wa-? zm}}4fFIRcdHu9dEjm%k0WanG>u(WF^ACX#>jKBk#$LqK>?z(#(H#=@!>BAJQA1N%w z=AkPWn#3o4TOI&AY{k?`Vn_5AL8d-_H&SyN+K1#wbE~RX!V>SlVve^|$P0o>YE>n zjRSCIA8wNI)cSv#@iqFPnZ&b$7ZdY~Q_-9xeu*$tmtG|jq0)1urUf{6fawZhiVUl8 zGH*u6Av2+?tYZ%c^>|d5`WhUbpicT7Fa(K=6RGoV8Ngig$owqA3j+ZzGXSV)5s=k2 z;AnOgaEv%*qvumI$j=w!2RUHxlFQOsUYgt#pxSuc`TCd~sFi^wRJ;Y&K4MBKx-;o6 z0?)AoT2Xlz*30QB&{R}tktsBo12hqGJZvG`okXT-456l1d`CPW^Dd8s8j=rH2CEOP zE%hl;oiAODApY!j^>Mv#!Qx5Dz0>Ul_7@?=lt8aqv3m}|Ws3~GEGU_nXN9NVLxiO$ zJZ=y#%bWC^)ED2xqwZ_EKbdp_I)GVCzyd_SB5e76)L!Gi_=i0#KfIo55#ICIkJF|dO|G{sFuKPST;?&pQexd1-qK{~E3H_pjRJL2_L_}&YMhWY> z;Yd!<|EK#SbPuv>nS3bjJ#@)Eg(M*}Q~J8~{GEfPJ!Qw}7acg@^320Kt*}T7DOF^Y za-~4evYSaC!Ecx>e~@na@+Eh^Q-PowO>l&6xpr)3a-j6(W-I_(CFMw3KyTtz56g2d zv=ILwsi|_QzGPCOL}FRxevdEriix@gK=9rUH1B_4q2BjY^Lr zL$*=%vYZF@zYwTySu@?gQrwmw2E@waf;HZBDggRznN=iI%KwcacM+%4^Y`!>{~5oN z*gSbc?_qwGc7Q8oevv#^{AJl9x_AHVyrMSjbunoz6=`YnBO zd!tU}g8#esE&#GV_g~VCuc(#V^8Vm zK>mqQwG9B8iF&Wy{B+gY<|+pbR(Z8cqMvC-($$0EMJNzYd;BEyVpB<18M%eG{R16K zkxq~E4QZO4ZNFkJnUCTrp!3GAl)ikKf3_B{u)}fK=-YDG!{Up7FF6t0mNcdR6NrO< zO%FdDicZcG6@1pTGdFK(2jtN_MCkGYJgzR=8&zHGdy*!*cBLQH;YTKaSA6)`0L*>2 zg0swW6`i~wkacEays zR*w-qm@hDrCWPP}9M4qY6N!se;t!0)J&bXD_t?=dJI9dbiPS6`@cv-=@czE|57A4( z&9SNFqzIP{oWf-TL%%a<6H;PXA0Lb{nC*o+*(Q?*e`HPtI?yh@J`iih1@pxjZ%p;a z;z)k%ywEk{OQVeuyxZQnuV0{xfB{*E5~bW&lapY=V;-BA*SosF{m3`zFpQt3LFzTL znbPp3r^YY~4pnPC70a~WO+G0n4!{BFq#CAk3(Hv%_t3?lf{|`~>yp5DL;w%Z>s;+) zg(`Vm5W~G;*)@9F1g5UO(M|P%!6^>i9*fs(dXi6p%)72}KIcR6`>zYI{PIhbkT+7C?viAv|3c*WGn%S9!E`5FWe*r zJPx5C94o$?B$T7kmy9-{O%jTK0WZ`+rP$;+P3YIdxdvsCe5}d#YPL|VLp_*0p_+je zL3LoGf5hopijc0F!E~a~NP7~6lmIA5|Ge($W(W?g0Kkt3JGn(BqkHtvtMeQJUf-yd za(U4l1c_bg!HPOlsD%e7VZYYc4R-XtFTSe`wG3c4uLrsyJHec)OL$1E8M}S`CGdak z_zV2nd+&Y~6D`bt4Z2kk)Tkl)iUZtTUb=toU7Z;s`Xl5)>1Fl92_yhTDZNtJ3?aPw zx^+*mN<8(;FSwTB`89YF8gHH)z9@vV2InKc(2&Q@_oR|UXUyVt_SN~@tgUtSox7Ge z*ewjL0d;FQuC7>^n^M|H)z>wEYVG_JG1OxLqt(+lvhs*DYTC?w2jZH)O!E$GOV9d7 z;W7(`NzizkMSWakw=_vmF>#W0w3tKys`bf*#JMm|=uf2ZEiRi!Sw&7Q-$ZAg3+3P! z72ke$WbgKdG#QrQ|(` z*z4)OSrKM-$LuJ`9b{<-Mw=QKM>Ft2D(3q=k{%C1#w0A`4fBo$^86_`zV>0c=qEVF zyJO_RWSTP_q8^NkDyhGnnRM5N8S~2Bz2$@Wd!hqYEe$!X^RfxV$mD?+nMl0e7df|2 z`e`SZ$W~c{kb%rL+}*^N+0d*}W>6t`;_CpYc;zbXXr&%z1UW6GPLwbRm^m;r5I-y# zcQjH{FWb}ok=kXw&mUA8VR6+-SL3F+pCWDDsUwDF9bwQ~&` z4#mLWZb0;=-H)f5a^so<3_AG z@ftGyXN~Xc5q)3~FbyRV!Yqw;_MpX0FQnZ~>b~^U(mBCDrh91SV;LB=O>M5TJzo4x zWD}NLad=>+UDqj$#3szLBkQ?bbef4HFom>W;fOv@AVi{;S?jZq{`}`2t_Hi(^EhUw zvLSBvy^VtJuYNlCO>nfnsrtKD(QUN?i_zZ0 z`cq-UlCY$5&UI7u1DZZ0i=)MQX1HivA-qvomA$jiBqCW$v1Ybk{5LYqQ7U5!<8;>S z>=3D_i#u$o^4d3C_49Cy*?OJwMprfyos9yyf@@=T#OdIYcY}FCM^t!u8s)ld12D1z zHs~0Ek#gfkRB{F?=%xc>isY3Ei%wx)$ICW1JI`l@=Z3RhvXvavzUatVY^gKrCVg}~ za4&TXJDceRj7SqOs)RrMdD$XOdZ)_DXija1z=qumNvVj&h~au9e;qq}U2fJ^IB7&j z2>X>}f>^KgrM0wAm=r=-FmfL z5SjKw*DS%V~jCFRB|)qLpUFVr}oKCbbMJrmCc|`u*fun{9R3mN{`OW3dd9*x#0*O2G|sw3*OCzSneXq^SVeKl8_zAowL0708rJQ^C03TIzM@y(=+&!;a81g0_BUk!G{iz#%~sL&=lY?IJUFX7}JP)+9{*6w20>o)r#v@$CG_bk*C z0ROoC#_Ytf_A_ejzCDy!p|`8t3#8}lUQ?D+!CAxX>*Ie#c8Q6x=PS%fOkxCGqmQ?2 z?eCey>@51tBGn-@T)=B0tN_%>mvC0gp(oxcH^yETlVm&`8t>z>T%pk(b03Nts|>|u%1y7@xplYWQgYsw|Id^h z>8$WUU7bf*M+Kk^DNuSm>OReEx&MzC27IJgYP{d2^PhkE6k6`?)>lT`{E!SOdGQ`D z)Ze%3rFUiUi}S~k9|t(z>f}n z_4S2{0?L1w!L?N5S!5u=6n-gV*lBj1O3f-u23(y1p;3PI8QP6($40c~8ro#}T&3wye`xW;H9_8_G2NHXhf@%#6<*d^B=nERg|tOc7`ySN3?)pd@7xC+cI`ljC4^AWSt?k zfd6kjf)aWRksYR7rxe+z+9H1iOqTrf5yBK8pb{TbXpqKsT2M$3*)hp38J?dOhEPivQgAW#Z|hTbs;hUOSN;FbaCd z2OTf13M*a%Uauf%J|b0)1Ag!2$OwoLn3}1u*OSfTc7#6JaJpp)8JzV_*B~#R`ko+s zs=aEs!OZMa)`G&kNQgeV+QS621Wf?R4oxH2Jz+6x5~fKsqsO#317{U#<)R}y+R%+e zfd77IU7FoPzwz)XOt+shP~Z9oz(% z$K&1uGJtdZM)J1d;PoV|wV5yt`?p+|)Tp@cFg~fdQd@_zgi+N{7V{eSg85OGuIc9w z=Z;PeC>vHYoZWalIB$=vSQPFogp+Ri&V;W!&brTgL{f)@L|Z|%RJ4N zXp_-VTEK7}8vYV%zhcs)Yd2vHLLdOktLl%J8f&{LyS?BYDmM*huRnU2# z29N93`wdj{ujSn2y#G2>)0h}AXW+iF;{ON~ICQxQVgLj;kR3(Q9hNzppkphhigs?A z@?Tb1Vh$lUF!qG-<*cyE)Kg7;_2t$nf@QA z3_Iq$IDR=Y_;L%WiO^Vr>o8%Y0>h(Gqo2{_#|l}pgc}xaM_+aIzMoEnFEM8N{ALE; z@+OMjNsRVh+$wq_XW)%c(Md5GF4DeF_RRh5?LRihJv{D8j&a0z@$HKz4_|_@Zd69U zn-+cHY~-N&R~hup@M}cC=&Y(TYh)y%AfGoXUZPOVsgk3x*42*dPZkFr11E(DM5^E_ zpI}`*ZD-G9th{Edv_Jifiqr~K1gXm(;i{Lv@tzvpwX*_%t-@*Pm6)*H64P(Rvt|vz z_iug-26r%Xjk(|Mte$u?%vz$;#&< zxKiNWr)Zg_(uwiTcm4%#hC4(kQ)Tu&_sk;ChgjT#1IQhAMV_R9PFmor9#Yx5pCXrO zWJ3KHqb!6jo$c7`cdo|oT}x`L{J_wbXK_qK$3EozR6rx{LxGVSRWsTmWa<~NraCth z%XH#CiDmblecc0wAF00CQZM}3#1{9#82V9gva^~RYteg)A|IG2e)xX~d#k88n{HdU zY1}PHa1HJR3r=vC#+~5q4vmvQkl+r%8+QvHAUJ^_!QCx1u7Br!zkT-DW9dnIFfw-B$)rAdPVfF!JBuHWUlf7ixVc0C6fy?x7AymDn=G9oW zp#W1NiQ4<1rSv`VdcSbX6+}`!_hNn|dvCd4heZKhnq^8&xaqI{8G##$E}fOpYrqE< zO5{H{o8}f>IFLWrCMT;%_8of0>D+DC`pu^C8oqtU@J|;NE#>|`9a%9KksWp&go$G( zP2%p-ZYPnMiDDUiY${L|Psf%>kD|9BYDb02+i(cOyh#&LpiB^04GazG*&Qq`R`U4Y z2aH2n8`gR%nhr)|r1x_Eqm?*s%s+Tkpq`rF(=FGtHrRa%*yZYyJwlxj8tT|cP^whvlt<3kK0s8rSGMbaey;n;?HLW zqRTw8=`~rAxOpFh83czW2bTDPg0*t%6FNo%JFgpJU^4zD8PQu;++FCkp-G=G=G!q& zFy;`nZH(|kxGy}cJP@`XO5{9KiCED_HhabizeBP2FrA~M9Xkv2_{Tl;eLCfxI}bXh z6$eRNK$+OXPL0fJ2}yR)f3;Fb|7$Cih5ncPk6Ege|7ZRTRtKbWAneyd(X>l5?hpme z71YK>#&JAO$!L3~z~^i=shA3$5b7bN-J-Ou5=XHN<6ES|5Mi7%SM;$P@nQ+;$=_>FxZBK0O#Gb=F2#9do%hdE{lzTNJ)98N-=LK#g*FIG z4>m~(e0I)gc@YRU;d9o#y}Q4A=;SzSx@pEf3J5Yj&NuccbGXNGxrTY^q8r3*0Brjc z-%HmMphDTQakf#QT9w?d&AQ+^sxxqRPl8NE%!0VGc6An#ID;RMZR{1p?=VtF&dv=? zn=X=mA^6k$Ro#hnqLt~B={*m&$8TGRQF)1(5pC&|Fyf#pwk@4<1jH3}|x4ATK$i}dmkLN4|E_#Uxe z$f4zS;XeZy>BNTAk86xR4pPu0Y+V;Ii$&oy>ypk9-O%Y!D;i)NYN4P;vw05wlgSw4 z7QqTSRNCsDhY1)y+x=lZ>;nZNJ=8j$-gPIY@TDWiMtoj@4J}cFS-}Fzwpf1=t$6JN zFA#;oq7Dqqf$3r;sGOm$6ZH==?t4&yU1*o*vq92}2*X8+td-w&=jjro%*__jQ8&5t zI|kb@n6kmGi20Srk{9Z*7Gu24kXgU0O#09!sg&53SXeEXb2tCJLcMUa4du1rhc+Z4 z7`tV(5X2(b_f1c>7zGNDu+{yW0N)-Lx2ybD-6f6?r~L*w7lIE9-oMC1D$ebEo`{~T zTzDc@TV?A4EqM`1mOjFWPY@xggo!j#`-hnb3fBGYRxRaZF$=)Yck=?Fr=%Z(J-W

U3Wp@(=H?VD41NZCQ>hjA>+p zfYslOvRi-l3bc9fsdetu!{5|#{wRaV+jyhsT4~p0z~an0KdI*<8>+_P=Oq46MUqIO zSKbO>CDkGp!wc-G66%hf5q7}~jp%=uG`A3tp5wM6+;IeaxnNj1?XH0ydyygBVnf;`$| z(~XqIIKaT~6TvT4*=QrM` zp9Nw=OoaS29{=V_Q3oE;{_)R~3K_lsZ!vkjiDX$^aYWeJr+ZKiPC7C|?jMec{-U?C z|F#%^Yqma=^VxReKX9+ZkUpZJ+Me#u^x3u`Sd)JkO$qN=z$H$jg*md-tu2mtr3-mg zx}RRUpKEztWG-mYOilhy_vW4G>khFD$b+GV?@_eX81qLYGP&nE7`ipaL5R_u9?nt;^LTI{`BS?C!|$l8dBQ6)`I=GoF!}=IEHx@S^l*0ZRnuQ+ z9V8OA|J$hBqUIx~*a(B1BPwhjDnFK}PH^;p7j}49|FvL*(EvZ0q-Ou)yAhGi4)er; z;a~?&cI^KrBLXgQ&gTqooTFj|cCx*`M^AJ}z0VA}@1^G0PjY$z>^WMdtjaI!84{qw z^i9N1OMWXBiCcelwb)^R97GKW36h`@4->*goJ@_^jueJUp?e)h)N*SrgW1T9bQGya zMI3!Xft zeVfy-&IzuyYDSLv6r-}IAE~L_{&|1GhZp6xTZBgIGt#83BfT^J9oHw1okZf3)!6E z?!m8mye%1M(}n3d+zpLC-A8I(6UW-EjXO`aj0(M^MWZ3{PTJ)C4C+^GVvkjd`hZw! z029s-;ARS7AA;m-v8ltS8r6w@HJ=Pn)sIrY!d0^2U>~}6vF=azY>Zgoc3$SZX0Nwj zNZy72+u3c2+NTyu^(=Wi)Kzwi6_QrzGo!<;KAb+}wZvFSJTjW|!fIDZ=mEAAEk}Qu+5iqRnk|_0*w2 zp*Pzbg8Dh#diI0w|5|FuVtt3TdtSHyn~xRauZn7Hp}oedzsd6OpE~b<=xP4m=+8J* z1!yniiiX6Sl9P3AtoP|9T*#}V);~%ll_bKmxN2i5EN=SniiW)t^oEdRbH3m(!iO9y zcWOL!)9KineT(hTIpOP^sao35IMK8Bf`0006d=kw^?J=k+fEI`CGk;l<&6&=%1D6z8va3H~aK@3>X58~Y?;kJg|1(}@YK z0~G;wIx#Fcrp*iqQ*mO|`|-!okuYAm@iPnM${>7LQrDAH|BvCu zQz#0#^gQ8T`>FsPtr{U}PL96=864jF(0}-VzdK}x@cjHj`V?OZ-Sdm;blm{f{n&bi zUpuLZNL6ZTA#hO^)bk#hGMYUwiOe&wKz-mf4sq0CsL7>hKd0vb8P$4@^FLTPOnz3g zxjsqUlqvOktv5nn05>!Rd+oSxQxuL=6x=Jlgz>T{P?}}7ke-jZXRt3htc$n#ngnN} z1?Ep+!z8!}){IkWm&Sojd6IR4pJ*HJUQy~)6GIE3m@&~jL(_}o)|+?Snf?j{f1PNA zxhZ4a^c5vh zH}2`z?w5U-eXlom4*n5_V5yGr4vWwD7ea`r2cIFz6|!u zJbjPVwq#d4!>AR`u(Coy_8(!%{r^pLFBRSMRWlOu_lW`+DTgL#r1chdARvq6vjP9q zj7-2sb^Vc15ngbyY4MN)F~Z3GpX4~i3hvu(FC<2nMjKXanulWF&~ME zUj>n-SQ&+T{++i#j3O#g z1RPEvul=C=w2`Ta0* z+|g=7kjKqq9{?T>9_L%pk77n4cIs-Iu+1hqjqG4klULLOn4SB2@g2CN_IHaU;QF(! z<15kQnXW1++)w@|x&!(b4ZJw>*G78FO-hol%rZ;lUeif`L_o#hTrm>6hKhiGYp%@9 zK*Nl}PIHy=OCj71^$-=1Rj$yU!CIjLge6ernjdc0Vo03%9obR&%pTsMTn)U^TiH&m z`{hkSvphj@ndlyH3KU8Zcp7xshJ9NwiTMsWkKX`qT~I%(iseG5!35Rtfg`wU4f0Uk zLgb%9;`j8hw&LyoP}NG*bGVY%Rl8PlZNxF!P~i`X>^!XNJS1xr9;U&r6)Kwwc*;lt zBkPXCd;o}c>DOTkjBs&|MhX8kHQJ}RZmgJJy>Qsh9{6^Tl>=7^w3x6Vo6qztZeYyX zRGo(*?sIiPI3W!BN#wzWl-r@q7fllswv0^Sw9U~@917V^O6!5Sj*9uu>Z;E3^Wo#+ zG{8zxi#P0JCgEv4;R>d;-GW1#oPgt#cO_q9pA-<@dw<3f8aqTXY}=Aooe{J#@Y5em zC3AHkR1vI#%eMP2p9{#uTMFs-UDrcJEuq3xhy5lv7dS zZBp>&`~rni2{xA1{YiN$7<{s;7-WfeKybK(Um@LX*r6GyfJas1o z950;Q*>`_^Z%DHG)rpJ_T(KIOsCB=*weQ}G^ck40)86O2GN29R}MD>GqM z!W~kT{9Q(^Qg3^w>z+N>MGvOcAz1f*_tmEyLwDe-p)7XcS28yyaR^S&yVrxpvpu+J z`8BwjoX)s&ZA%C^97q*dVi8FpPgK%T1_j@gzwMzbeio*7$X7(ez9necNaJA&(TLX< zgTz4BA%$NW#U>b0ebiJ#t6c&@K8C z9N=@xlP5y>C2KdP@`&~rnwI;TDHYo|(DuWf8M+RIzxliozw@^?``<%M(V;dLzn<6p z{qDw6A4x4#6rqly$~VKX{St?570U~m59)s}rD6oa-98#&(}Ab74Yp)1*>t7_+)E~M zlko!=#m7uxi3L50c+5tOM5|9K_Mza-tgUaUW=W8^HJ#}kOX z-!-!bycEPaj}T=@AgBZ(W%38IkJ0{K1?cSncKQFvu-CCaECEldSBdr<`w;vBoy5Ec z=KtWOW1{~0gWF8kpcCDbHGc%B`x(Z{-d$%kEjKcB{TctPj_K?)of+Vy-0wzt-C=6o+H2DN z*q~pda!r9Dc5O+nO<`aKC=aj0d)u}@Ql2m=YnJ-MEINdNhp>Vwevy-gHT+H9V=Z1DL!Q@@gm*RMkVr*Pqg zHVsGV6zz^uLpsL{fU4vjbr{!7UVXHf zb(P?}4w3TVcB)I)>ejE{1gZH&YjZw=h4J&(QXor!+C_x!jiqSH6R#OjlUcc#eD= zxCMPwjuUwe{(!@c&DX>HGoAm#Sg&ikx=hL@11+{pwpAheGbCiqiDk|F^-AAHEAcMs zs~w_EnIPu$QOfST!0iEAOit=o8P7FoJt6e#zbmj(+Wp?6hbOtp~?R1m}9f$u3cBS>ShDh zxv|&Cerk%y?QTcFUIV-z3il+Q7&bo|s`m0$wZN4w?d8~I@zJ&(=%y)P9Vq790gD+_ zJBT?Jilu~XP9}2YM0=LqN@&};B64UZAbA$*VruT zi%M395TLt9OV}#5{7Wt$t$A8rm1Z4M?OnT5+;{5vihuOLxj)X*i=Uv@yqW(}u(Mlk ze&!+JQgGG$LhD{%IuHFeLH!t1KN_*`9%B%W?tq7u>0Oj7+7c_;@=;W0Vcn1QLRRPx zc85)#P6dKb+?<^V*g3$Qo~nDgb;kG@B(!?8_Hx>aO?r%2>vvPi5&M*`mpeSq&)sW$ zweSNiiczWl2Ye+KHGlW$u>vfRtZr8CS)h35|5@oMsE>~vgD;!g>|?Qlo8~==X%}AX zd-bs*$>{dd3$N!mPt(h zYN$TfkT8X@d9i4OZ&K9>+7-BulJ@RwVEUqg;vWDI(FCvvxPWW+gJ}JGHnPK>nGA({ zq&Ov*58)X6lR=X_f|)0heD2D#S<-}2)e;A&r^~~2_MPjq+61q9coArYit9WD3<8>< zQsJ#T;6&?eaI<1$#z`m(Qkj;FWMvx=z5h7UVOoZek7uI@HIOuxGwvfJoX#I5dtR42HE=X&*h*Zzy=bza#-SfQX0ga=>FzP_*Nq?C&o9`d5u0HQi+^Vchk0YQ} z`yh+`jcRl`8gSHLa7Y?Pz-TkE8_nk%pdBZ6&n7}^Ap)p?td~6n)sTL@N)StdD!Qw0 z?hzL@L#{iyz^_^;{K?kHb73f?WEw-8`8wW})+L}vS4jH*N*fmo{WMVJ{#fNtSnj^^(89m zv5RxHMrs#M3-F0aUXQ|;@!nuKT;$~qEHPcp`$I}$V*5D~F4vz8X2%b>) zleI;l_GCdw<)`Of6<_p~%B7;SKZe%Rvo2GXX!QNiZb{#)I9Ids#BRxj9`~KsOv39r zULx;IIIHlemY?FAk!$a3B-bJMF|-ty_o}^=8|iZE0*5X`+jIA$!se)2z1l*TMS$u- z>PRb|R_k?$WT{j}(sda>lK&9X+)wtH(ZaDlghM0wCKwjvk7*)sQHOdaB@Zwl@ZAq8 zylg`VoS?6n6qF~6+PQW7uqO)ry+zep-Zgkr{ZyF>6$?JdNIVHTKUAb0#WrZYSitW% zZTH<8HyRaHk$Cl zL8Ea*&4u@|JldZ_>6cAzuigpMu;+U2BTa@!DE+?=?w--bY|I`zXpG^ zMR^fgyJZex>f(<5j!i`7L%beQ-5T?%LB+;#+U-X^5Ui!fh|J9BUE1O}K%s-RWGTox zGf}*_fX*MjA;iIj_ThaTa!plOa`p2{FlM9`xc;vo+3t zBS!40yC|G!OOf!$L=67J>nR0R5URT@MuI}65O3GG8i@PkyQ`#8NSRHH@IF3bI`e{; z4F@91*R~Ssq0kg~k^<45l5g@^MqaO?ArhT{(y%&ZSDO2grI6BgK&ievqA7&_9Ch9f zO1NIezNC&CeTdN0-dZLo&IrLzU1wUT_Na=?(rC*jS*(83&{8*b>bWH~-lyjW`z`C?r7d9z?huWC=hXtJ3O78bViUM>5XGY+% zbHVsUQ7IlRC;CMixn5aBJHPO|ni}2W(&u42QbSO^PPtfZ+LdCiGuP*Kp`JCiu{Sui zV}RJ}>>2N}c^rR`3?%@E5)DI@(g+70xRI@DHw;}voF)cD2k(D344Tg|Xfrg~vtz&E z_7;BV#Ovn%gO~99y}!*%i{Y>rBujs=6-b6%DRlo3KXFxmDdxTH^yTan(=+AK&5m9T z(U6>eK_Sq8*|zKT3O+ccp=r3;N|#wEp;~3+>uNz`LQsh}O{ds_)GK&x%zbzDx$!Sf zAE<%IS8Xjn_jVNntHVV389S~hmrw@1- z*LRx6U%TBMEQ{oC*}X;3ao*fgHhCUqqoi^9MsroA+=d|BV>jZQFBZrF2U{Wj^l$^C z#{SQfr?VSRuBCE7&u_vx!njZ3G%j)PG7PpG`_cCUKDr5IB`uL{b0(%!%f-^ULA)E0 z0W8_;qweZK&^H*A0u7T}+RMz{xnY^hq^>3Vf?3C!q;zydrBLgUbT_ivk&!WSwROeD(%AcBdv^f7 zWL@@bG#N7iE7t@BqR-F+g5;_SiD zxlfmaM+bZFR?eY|3iWv5DTvKR+J9n%=)}c7p8KF4(iO&2XaICX=;gso_n3O-He0#! zQRW@L0{p-%Tx#ltH|taCmDSgjN+n6CLB-#Bm^iX?Ev{7Sv_DG386NQi#WZXJ!&P(z zcC-fn(#pg8TmqL}X z$r4tB`s$}CXsnUeQKm(@dorbc&`f*kqSb`D&7SaPKWp<^0K-dV+{}!0M!&;^2eOsd zH+TLgk>6y=#bUrl@1*0KM&;HWMg4mZXIRu5*v8J+u@l$OW6^38Dt<4W~4@-I5(O zi9FBN#5akoEqKVSKWY~Kt~hM}q`nSmj~gDnl`(RRx8skM#I7cxr!w(k{o6kdT{J($ zT*C?h0hNE-M;!eP=K05E*LJ7iswC==yqAHzc`urXL}XctIr7&o+tdAXVU|2Rv=EH4 zjM(CcC*%?wjIvhjy8KK{4wRM)RbCQ3QU+s%-x+s|!Q}9h<zd3v zgMcC;0mVuXPz!hW?0(ZcU?rrSPKn4$=?LT8%?h>5%%U?Djb#@M;=Z)6bbfV<&GWeA z6q-a#%rdskqDfK92)_se6cRw07RUT*Ab7PDK{&A8M|tE*)Z3EplGh)H6)->o*xQn| z*PzvquusDS;@`rvBFNRD=t|l^VN^0nBe-tfHLVaWU@YTt25}k2`+>(KRN{eU+R|)z z)nQcXsb4gf-VIfziRWOT0;*CIN@2XXI@G)hC`f-kEZoBuQie8#PV^HCs)CyB#_?zl zSo9#Zt56ycb+Y#_zh5#SBN6&>*lggv&eqa7#&zn|d?b1={s^?Zmp?Yc{ylJAQRCe` zuOXrtdD_$@1>WGPNANJ)JaFeq zckTo0(#Reucnz`7ZgoU8!gY0{;9dNL7uw*@iI4WZ#Ye~8hkCNePD5JGtLQEuBdnY9 zF5KtsAsRUB%0nUxSXtkSZ8vuM%TvLie;R7puL32ig6eJAY>Y@ zsoKpEU5!2=su}Z#0T7}^SXd|@pEU7CWHe-lExch-YHeoB=8kObjnpB-m}1^Hj$8jG z$DyC7=&#^6tY&b}?q>_y!KRQF%I?&;&}t}x-M ziY5zdfjU1GN`?}l;&n-t+$ zcD38Z|BMpMedOdHCvNe4H?c?@T2CJ<;@$uFiW<1Kb4!#;a+fpSlkFMIWKHfHml;rI zeFxY(uAtEQ+k|cT?F|L4)O7xSTiVmQBm%k49A6jOvI+7kzMHS8)ok0cJDzKY*RE4W zisPkE$=IRT5{`V3FinX#ObVe43mOq6wgY92hZcO2X^e-fkfaiZV8hT8tz>*=W$`$Q zGGd35%jKwW z7s`YxTV6F|BQH}$D**72^Yz2c6!zGQw6sG@7QMd$Trh@c;yF6W6BJC@OIR$f)%PW| z!wl}F^nnQ)m5wigCU@CGUK0s#=K<1g7}}bXF13c^D{ap~L^^aSQJ|%5Bu1wK7XNMG z^$L8%Tf2`XWYI_Wq&XZPa?a6jom;7U_uABv`WAh8^BS@cPQ_+!WwXsLOE9{5uv7R< zYZvid2JmU*3MgKQ#IhHeVuPidbGKJr4?H?QbjGt?Q3tsxg`J~~;aaO$ozj&`YHy6C z@=RoJ3$(h{!zF)+4?P%Hfy*xdMB{=bL}rAALMq$QVF)-dIXaVYS%iF>G~C13B9PND zZ`<_aT5bU9uhMsz#vSPu!mV~{%t8KScmk(A%tK1dI0WLUIvF|_50f%A#A;zbjol-v z8d!TYr$f?j-%c*5y35p6xAj*ok@fLGW2vC8;Aq*GEOMp7;02u9N!+7H-p&(i&hpVR z4YBD~rTZWIWcqusg4LBLkX1C!FA&qaLdJV`>2?L`KD}w2eZ?6rG(FJHXa&@Gov+Z0 z>_G+BD0$J5U7A}Q#a_jg5dcG78yXrOTpa@-2V!PGE}no41f57l+UBHgbO1_oa${$XjT4 z&XbMob0e1in%&We;q%DMmzZ$XnE=?JKv9^@tR;%CR$q5Zx)wqM}H@FE8` zN62i9Dn*cWcWD~Ckrc1(Ph`)t7(*Mv^WVFjZG2C<8|oA875Fkaucg(LIA z%q_puL4#}zvj-$`EiCaS)}B|+dxN|K#T?vX@<3oKuR@R3Ef!kn zRh<4-t>cIFyy7Szx``Sb_lG#=zq|l2(vzvE=uT@rG#t-{ga9$A)3QJf%o2h)9-R+B z?go#FKC4-(?E(!J0fObzP@|bY$3?9a;Q<;E`85TXdI!yN{_b_AN4;(@pF2<|bEX=<7|nEgO&&gFccp)b`m!*7SmB+*w^{ zo`x<3)BSnw!&ifmuG*4r2JVJi6a53L6OR5oWYxq!WKIU|Hf%XfxA0q`?n^xhP2VXN zN+JC1=sn5!bbXIM8ZE7S`?FhZNm&Y>JmZr&dFd=on{-IwOI5$Z(r@Nndmfj#`E%8Y zU}Xe+;3}AC$9TN@H~Ed#BV00*rJ4bUf^vY9l_D?J7S8kh0;G6lEW+!3hPM#94}27_ z9eqc})nW=XWvD7^b}RcKVTEKv0x~}ZxcO--skS#hM=`a_v@;m1) z3#%zUpVZ-N%@k8Ipx63`QSrR}QF~>YjBG<@c=!*{D2uXBnP3P)8SpsK!|XiR@*96xT* zA8xyND5FF=ZqRr4b-~XVrVa!>`iN(1|KGfYop)E_pkLOZ8&5eI$VSR$FW+hCT~c<5 z4;(TT&Fetu#L}W0)}O1iET_pW(ZT&BIcS6LcT!B5^^3_Rfo@%y&KPnLNa^`4sASWh zgnUV#=**4wI95s`uuANczS5d@?sln}GBw3WHT0)M7btX%3m(>UYOBGWHdX&Zhc|M# z@E*Nb&K9a^_L6_OPCQ49yYACojGC-ptm(uJ5S8bKHz@e|EXs4X=WRP^9iPxcwV(jH zF&C;l<@tI5Pr#Jy=ZSp9kT!_gtkL3ea!6hkAjVSvi|1g!Il-y4Yglzp@@xylFs{kC zbop3u#^mr$glp2**|uO(+Q{_s68Y*xBJgx@=^Lp};KavOj!uNQ2!vOd#c_@Q@QWsp zty)@`^Iq4J728n2Ch6KT@$C8T+_Ih|LL9`cte)d#J5@;u#&!6hYSFEW@DP~Yg8s!o zUZ?eds?XY7jSI2y1ypSBo3Us@fNBv-*oyHOg2(oe6pkY{_KlC+=k;tQU&Lk9F)GkD z!kJJWu&%gztkBJZ-0-)su{v7H0?@2;CM+CN0y&WG2pn&>>0MC%Rt^T%oB_=0cxQT;$y_2uTH3t`Zn5;?yMU%7N+j2 zo!WP_01#@f`Am;y6(rSQ;PHBy8XUC9H8SYzYERGZcQmT4Dy zXYC2+G9vyZ44zEO#XHYL2y%LYF5&Vb{%$egj9p>_DXxR+)ob|z@d@a|i8frT+uj)# zDg3gVuOCdw_HLObfFl)QtbNpRCQ@9F(zTGHjpNp}_R9;gur_uqfp=QJ&MLqs1!)q{ zeow=H8MC;zPz3Br4*%Zx{=qDUflmg?6)mXtUI=@W?LtqM-S6uIadmQ3f5u9gd>vRD zLJO}Ezw*h`R-d`h*N$~uVNn^ld*Ep)%J7a`E`l>XNw;MCy*uBky!Fd41mR}jX|$;BwFi5a&}~0DV~WN zXhUqVw}cN>l8?3KRapmWs&m;Th3&J?4xIg{Te#Nr58x^we%Xy2%mjWp<7A6g*9q){ zszT(X=B?bE$-7t~JD6Gn*>H$VV5X2VHK1+9um-ri5#x;2CF;OipZg~IqZTx7q5K1B z!bO=7D)jS?=1`B;7+b+E`MBABfRTr2&k7_@mh+6xQ8^vFnvtY`O!>982d9m88TxOx zpUY|UB~<*v{J714xn~-i+()4uC;!6uhxIHr3bNyvPIK=wXrOJ3)X>Ju>wul;+mFFB zVMy&LZuj%$;n50*$I=E-a+Z}~L7K=p#6+^gS#Ui$^%TCevq!t5#j6%iMCr6s-d|zO zjZKRZ*Y>wC_Vuxz$kh+{5Keci#9h0NX^iOKvh3m)j*CE zR9+@dk6{cm{)$aub+c?!4EuI3s1RX$G_DGUY#uJG`UP+Ni@t9qg01UF;>oP*QiS^8 zVgESZZ5!Clz!57nm`h)4l*X-yd|`>aFMiph!DTS zsXmHpqUJMIy{rAjyuphQ?N)H5i$0+#JIWr{ty&wPNSfkd_VK*M((dR3(w8m@F_*98 zaQR3x^1i+!m)BPH{k^ouv+_D2J00{3BbF=JE{eloeOAN?2(W>y2WF0o=t|41v%3L*hZz ziCTi5YXjD9?T-Xh;}r3hoFs%+_D`9Sf1G*S%#xr-$ea){oKrfV+FEf%Ww5E8us|mz zyEpvGEZgHPytb_&mA>^GD;#*?%zi_W_+Tw1)1)L#tOSZQh${nP71%h^v=N&YRE+KrfwQ`z+P$=E5r zzz&8v5KRnrPX(!X&$M5*=RUsyM@$iZ5FCUs*V=J3GkZu_k$Om#;O=6e!!x?6aVZs8 z{LRzm#u8B$Z_G$&h4P!zo6_-ly=_Ktak@2uknd|Y`78j?XKk(;8wmGPJ+NdtXi#n- zn}>L}ZQS!?2!!bHHDHzozTw`2dPVwlTUai;H=Tx6dz`%6Sw$9*Ym)NaKt zV=-B!)Vt-xvnS?K`ouyg_ntj4<GE0%kuRL7k|$`%2^(n+=%?+%#C zaoTh7&%NOl;n=0n$5xrVeo+-yQbI#ab(!YAoVFN z--O~nYvDgDXyf}un!?mmt>Er{ND>J9i`A*2TkbDwvBe74sl2r0NnzZZdfD$f`jy>p z-Zih(`OJo1O~Xo}Y- zDOvP2fVw!gMnRLz>o3vExA}<4T4-NdvqMTM;AOd79_>&Bf@n zNf_05WeW4n?bDAEZxPnluk0MOb9rt@1!^4k@>wvhc1ak?+>=MuI7-QW2@RCRP17W# z!0|3gFH zE0rpahQZG!t4>RecL%1V3p-dk#ws|%`rFIxO_+Ut>8!{AcX2l4Y1lD`d5-zYgpXU&au+FldmEu%gK>iMbn{bT%ez5=)Qo$wCv_h}nC7e4wg1Mgd^;W0Xp%a|)m{np!f=P^F1l+*l~ zCE}%+n=>>{<1G<)Fp_s?SWF-UFo)(qtsGY#em5w|JA=1GV~h74)r#>XGRTq&)2^t; zyXmCw#dbIs2~rQXm|p*tdfDl=N|o&%Ovv>*UAl^I z(=y;A#mZb$Re0fo5h+s*2F!_FI&1paNzn=omC^G2KLv+_nh$EtkQo}ZVuKW;cZSaF z9|d#?aXRI)l=_{!g7#u^oNlFeiuA6<+rKZpyxYG<4~-KJS%GPSvL7xi=L{ai?ex7VS%Ziq6BRN2WS@0^f+{2&OekIx(Z$Y>yegHVq4B*OuTOsaoKeHS#_W<0){X4NUx++^!8-QHP z9JQKqPHMMiu4e;c@z~q^y7!B;81lW^^n5H95WII-N9DwBBt};7acE^D5zj}AyclhR z8qz-X1B6iN1Fa-?2ju)u_XQRu%>55nA>{(2Y-XVIl8+x1))ORfKm_6-MAaOi0R2=l z8MXo{Rk;cxooDqAETOFVPj7WmDUe+*%ri`k%Br&+Wm6_xDTK7Umudy>VL9Lh&@%RJ z;JR)7>+Ec5HrwnkU|ExXXd{`fLJuqQh+p~$NIVtZqOJTJ^7nSNaGqBz+63XAe+t%_ zl?_2Qed`YAr9#)E^V8NzD%Y%2RA8N%VK)Lk7s{>X-C=yWPpbgVksc8J`p9}1xLTnH z6u+TBjp$ocEQlHr9rc^IYQH8`ho)SzuP^*kJ=+6ueZ+}s>0%M4W_8ZJ=UKY64O7;m zWY_1tX1|t(&3cwTTC7ya+N4{xl`!EOX&D z+1Q)$P{?;u^8+zY7aY2U>%?!(Tk8NDfR%2nAZs{KRcb!-%G(25{ZrIRb1+e?&e?iN zH`glZl%#l>IkTxPm4PFAxr7m$Bc(FM81BB?ue8-U@f!uD8~lf|z83-|kYAb(@$Ync z8_aQh-!BZWeP|Bc2jyoGv(fXVcD&eaT(wIqK7MXQB`vplUoGEYPMK~8SMol@mtbTv z!k5sUMtnWL|FKYzQRc-CNCn3LE5%?$Qca}4mnl<-ECLE)t!0d_;sKp!(<}FKMRQJq zDZ2vjc&h_^?j;{qt(%pcQ3S}T=~F27yUZ^!2py%8uk`Qt3%P?4&!5SkP2u(GVo^F(@?L3{ z_f?>O6Hukp44xihJdic(3MKTZ|Fw6C8|`2E{}J|`K;P=b^wskBHqLk!(8fOLo;EhSxwl!72bgd2wExYyO0}*6+L4eSg-yaL+vV z#m3LTYquXe${XP{>30Bww=0(Dj-oH8A;VB}Eh4{1) zKhjv%UB6Z99k_>^YH2H6#T_E^ya6U&)xbmN-hd&C;YLD;)k*55 zJrQk#5QsaKlo8l|(>$-gMm4C)OEn|FB~U7W)J`P;>!kpmF6dp?Vwx7`*9dc8vPGsB zaLknyHI%8$MKgNRO>nzI(wP#!UhqQM#73j)*tm3(O$*C92nRs1rFXez`^wDwJM>5ijsZFlio^@}eHXS47 z+WYt@Jr=1NK9IEZ_A4c?YrnewG7YdQ=KauzDctvoZ0Ig#o(U)AC2bRszr^Fl8-J2N za`sU`PcEG%{B1<3B!Hks*|HACOl3rI#SkuC+8y;?Vn9xOuyJ=zEOvADRxbLAB_4B} z(;nAg(cN*=;kCI^FZtouNFnUyPv4t+_S{~XbH%A@Q}v97L87uacRUwe$ zxan!UN~5WOs1IV{x?H4wyd)W)YDg;8E+wBP8SQ!DQ&`DSx$BVHT|1>BV;@hGRL1d| zLo_?!sWHmW3wYBq$LT-FJuy`e^eqzDe*Q=yKSO=bS%F2t5&*Zdap(`*?N!_vqt{}@ z(qEYm4C2gx7ro&W4JXBq^Ja>6X?s>MbMns`$m*$%$Ls==;9*nCsx6!MimuKx!`?=V@Vuo)n3w z^jL9&Jgp)>F#P;K)}QV4g=d(rok?G=xi37KQt{i5m6Pm9A02^I{eN#g_}|hDFzWZy z|B>QMGmtB&qpF4hv8aN6SBAp8;WoZFLO%j_VFzakFoh<(Ns`&!GDx*jX|jNdm#tYb zgPX~IqcVosnc~+PpXOI8b+^EE8P3^trX30TkUa_^uIB1#2u||YsW>n#<&9mS z@|O_D>2k!Kdz_~Yc-1t%Gj~h-0BFCz5aZMV7y12=4VegWd6G4U-oL3fRt6LUp3Iv` zyx*ndKVhX@?itFjLQSRgQY~ zED**4D~dFgTlTZ>{j|C4k<(~oJ1}fRn1IrHvS2QUPxoD;2&b9#au=mV-q%&)_a-us z&-T|s3ILTi@0w#DmWkHJM}6>ESE{Hpl;6%5NI;R?_4yP9ESJGP1-@MrX6BkVyp!QO zXI5I*g;yBcAlN?H`0Y?cywEqFr+kSVRjrl3^(jfV`h14oF3WE8UDevo4?1_^gsr?( zd?GZFsq=(}Dx3nz%ODhYmkvA|b6B&9g?;*KB_cQJx?aXVzJdrr`DAK482rK#kfKZcCnc?-Y$7wbcxcnh|PWv%Zv$;uC zzw$Hjm%{VHRFs`I2V21%9W&4;8dmlPHC)}=#`EYqz0#kg-(`qB66>cAhlCs2$jbti zWW2)AoTyvkp%&fUHzL*j!nN(i^`@McTtoa@@}eShx63tn+Tl1g=Uhea0cG`-+|=nG zJxPrARo?d^HAPaBQvW^IzBB1e`TX%O;tpaw8LHgVa|BV4UHFAE#hBXuo0MP4mv32; zgEOx1RSN!l4(k(QBfNUj@vXH+5mDc2yaEAoty!L(I8U|tdNBgZjyE) z!tEq>^uBr_PIyBxi6f+GysX-m*mg`LjEQ#*@r_%(B4)%s*%s6IzQb}E)qj*LMLv4~ zpg+xz+P_a5CejuGCQ!Od;OpAV)taq<|}LeaMVSf)kfWpseEwp zybqVJyL_2Gu<`w=oL80~f1E|cH|2SiPXlF))B%?&@3sj<|0+p`C+w_I&Y!uE(NL9E zTxrBcysf)4EAU`U>Zzxl)?;*ZRfrAV2Lqt_c`s4cpYI`#F+bc7&Mz!{ll5@+l5&@irHP%|pVZi0sv%4X8qr+pHND^w z3{Ri1l`kpfliw`Kb&doBpHsnBHcn#4wr-iU1tEsBZ~JOj66zJ1w6kS54!=Y5_$g&i z+J!GZz7l@CTs^U-ATri0Z%1={2$2j>f7Y3&1;*VN)r`FPpx+eXm6nuMYk3|oe-QFK zIAtk$?&2lycbSwgKC{FC-bVy_ve(Z}CdA98i{j|)&h%oK-m#wGk5R$3?bKJ_0i^$-;4fgO_DLEv?e@lhbo1|Ha(b#Zs_MJj@reBN>sVizu`pWiBid;usk zQ?8nG!-#Z!6#hG!7M^g<$5^(Vlt2WPL8(||>YI@~e2fwCkGo=-V50|r*Cvj`vjOKG zPlCoC%zOdseI8=LZwEec>x%ZSZS(XGH`}i3?iiTn9vl(*@I#@xXAz?8+ox*E+E82O z^VcK_-lq!h1HgT_+Qech>&@`3hh6T+Z=bv_MVacOxXaqjXx-mZAt3RPcf)7muh&-Xe{r9`T=@s zMj~&`cf7jSpXIkWe_nDe`&yoPS&Gb>fOx?s<1JKnj=@mmkg}s@M{uBeb||p;CD8B37AiDR^xX+jw)D>sqKx)>%SXV zxV{pLd5%Nf?OAE0RzbhmY306dwAK65oPo0;aLTPDRf6SB`bw-OJ*a7FY03bT8B*x) zC(=aHWK@M%_i_;faD)Tq9h{)c-qc>qa+TK8w`eWrPxt=&M-zz%g@scoGR$Ij4w z79t%XACLk{6m|OoznU;-t`DW`1;U1H@eB&uSDhN~Kd9_4yH(_&SrfO&eWmj=+gp|R z`fBTEJki&rPVI8|;{+^D(QEd+K0!_`v@2ZHp+je)|94wC@6$V+PG6*Udj2v6&;HNJ zzje%izqKGORgwiVh)MHCgrM0Vz9)C96NO6&Y z<1`nIA9bl|fYR}*7058!+0jv71_n{n6Y_^7PV5-y%x9IXpw*@xjL(PR;xY+1c}OKB z+K^cST+X)TWi%Z2a+woBFGQHLWDG<*>)xZXt3-j&uixiV^U1{ZaOKvoMadn0gwi+F zGa#1e%5!tPSxz;6cZ*Ckkr~Xqp)tXqrk}?p*SpK^*P1N%eBRD;@p&(=nptbysgt83 zG9hmK!qkaMm$!^P>LF@a)#R{6bDIn4N+oxmxt|I?fDtssyPH*z)5rE{KN}Ks8W)6J z1b#fH=Bizu@Q^it>O!cMl;D{GvQUcNXT;pmZzs>#PjdH<<;+82zPQJeR-L_}FMNhf! zGQv>wF096}Ls2s#{o??rWOFx0`rAc#%1nRI*b_+xSj8) z@{jjMtW}Tc4d@jt-%m~;ODZev2TbwA3V}oTuOP939+FOgx1LQlcW0ze3RdqeJi-57 zAz`dF*A8Z}$7k24&9>uxI(Zr^;uBJ6zvZV=rFfhH)`OU8L#n?_6DN(UB0_mdZ!Ic8 zLO>yl=V}4IZw0U9boRux8K9lRJ);GX(jy)pZ(wt2K(K|%yq$;-ofiHl>GD&|0hR8$q4asHuhj`?}tT>KVc zN%c3XA*uK73uRvO!fXAWc0raOU;Qm6Qznh8|4uj2HhAgtUDP{ZRQ}n-?&#y4q=Mo2 z9_ue~Ua!@Klj(1@P5Z3v*)#@2Z$k6)B=u5TVhTu`0%AVnqG~r9eCJ&{WI{UtCM?~T z!wzLX&3WqAm|4DP_;Kyzx&1Zk4FW*H6&g==#aMsFZ9Rcx$%uaryhqo?LqbC;yv>9T z!3ub@VI3#&CO^lOqT)`;pAe?D5v|dkf{IcuFX3DiU=R3jm9B=B*c&dOciohT+UTn9Gs-)g-DK^ZhSptmOt*9e?}*#Gbc3ru5(eEDoX$|UwdUAPF~Kfm+!iWuuD0O z_*r`-7E2M6XP{g~>#N`A1`47j}G<<2`Z!(2CQ%pEPZ`qBsDMqR(Vv$d& zsMlxYCp(tmgdluU9)%wC4l$ShRHI*pmntE(R#NDuS4MSh`m_?SOq^{^%+f=a{b_Vy z2#Z2)E&0W~D^tFfZELNeHFpnQjl)t-NzUl|&*Eih{F66}G+9#V%q3FkVD*sNR1wD_ zv#825uqtnWfOblc0KuqW5#8}GA)s@=RH?xy1{MRXP%T897so&`p-h@d6SRfilb^Lc zt!1lY_ak92ib5H|QFz~m00Dl0oeKbjUS>)_GSlv^I}B?wpow3%D=ac8p5E2drfM>E zlks{9P^vffhTTmhM zc5^XC$Nv#@=aZjHTsbP;{%ys@9IZbKC1&F!9-`R(rx5)6Va3yMH<3dWZ~kweU&2R8 z_WS6C5OgDJ*RQes80pVz>X%_Zo(Z;&!aE*%Q%uPJ?jEiE_OKCIgVHXFPX^BbN-e`5 zF3BJ2nXx=@+@5TA6RxHr>?u>@PC^+p-jz2kF7%=AAAp0v>3Z<@#s)#Gg^oHP zpS2g<>sF;U6D@stNn}4n$fMChkyIA_zf%M!RH0!fZU8}GIny_CB2sz^gg`AF5wa(H z$>&2)kex2XZ9{j|41kXPLS=4)r=bQ)Ls9$nEHbQ-eM{{zp?6>)pfYeT_v;$(b1b4l zl9D5vkTO#*a~_Lb^CM-UYx-13o~Zscd*+MM7#+2N$U!4|x%fqZ`V2*Y0M;ZZ@2cc4 z1gjbj;EeU_Cr&GkdL(fADWtTrIi$5&rd@Cd`ApR5?*ox^vpy{R^|BRLBcy%5tTJF@bb8@FNl|75~KA}LPl_Q zIwC;Ox1RJMA(vbhLBuEE_U(+vAo)Q*`BghKK^u-?n}i8V(K+2kzDGhyZnT`r+e^A@ zW9@K9dBn`t4H$_ew%m-aO0UYUHo;;8{&ac-&=%G3@$)L}?$6wV_hGBEzwaLB^TTFa z9_$x1^4BQNhk8 zNg}X`@FFeyOpO&tS*dW>w#`*b8zuF?;xWFsS zCq^^7bB5qlRdNELy!CV&#JFgxN?ywvfZ$F6n#2K6i$aVQ=(I>dhar{n)%$dBz$RLQ z=8hbO6{dr_5QozJy->ghx+XH|>Or+FRg;**_a?QQ$aCBE1>q|BqF;m#5@ewu>U_Elc1_h}!LKBWT`=dJ}K?t#jX=DvUfIl-t8(@z z;2t3BG6aZ_Kc0DGiQMI?{Xu-{myg zBiaTbSX~*!YUfq)ocas>DW7(z0M@lkT=h)|Y+4Yd{XSk?Vd!m{547j$RO0><^Y;!x zPo)dsZ`256cGH)O`W_2m`GAimz_BJO5Y7j~q1nA20v%<_X3SZ$qSHDy{luFOx#NcO+t23Y5iqS@#=5W> zfvd>4V1;uE>!Jb*kYw1miuT^0v&~!85&wXW0-6fRuPOgVfNNvOx>Y^T*Y3q@{dk*7 z6p5+k1!~Wc%7VAOW8fL{p0wtMh+v*SOS40h)BNo3DtZ%IFkFLdoTU$F#)K5G6(Tgi zPa}DptI9+oOyC6_gU;UB2m|#+^#TLDtK|`p3HT}jT$&K0Z3FlL_?~w9)Q!Mt+rx>$ zM>kon5Mbi}EYKhm$<_x)JqMz4RGRv(Gm$SJ4rXsNWTPfxM#5n@p)l5y)J@~tm%lMtoQj{DSyCKn8tacII@&m>(~H7Lb+9uGNI_fB1+cp9cfIA4 z9+3u5!)8QzqG8?Z!ei(5z%Rv)$mzmz^>R5?um3I`C!Jn7+?W(FkWS+tbxO?wXV6*Hh ze~Lar37mC^-DTk0)yH!)4vE*&uG%8 zcyp@bffZvBT$6OCpIMTYRSgFwHUK?emh`ViA_UmqO>nh@!D+(>yP&%dP_UWsuMw=3 zm3y_QSWD;|yFm6YaVFbLs)Wd00SPXJPq_sfKX)}}_x)}R%QwP_@)TXuy&xSzKjUu+ z@yxxg>0*2zGd+BvuIBiN>R~=_3clIQy<%@Dr2}~y{dooW8%Z23 zHd7M}NN>W^<>i9T6P;*a)7V)68k7eX)IP?^SQ2y>SWN#xp(6~$I~j4vxT}trYWd#y z^V(&3K`Z}`{67(o_~(A7A#AB+vA<~Te@VN}|42I(kQLxxr$1-}f)9|)4gRiKsfVXn zmrYOF=C7f)N}ZRGZUY?2vP?l|eP+pzIbu?U)ZK=OMK5SMycXU~F-Z`_rT3zpxZiCS zz*>;Br{jnAJQJ(&`FH1qlFK@ek=f&wQslP3cH|6cD&k-DNQ&uxdHQ#r-&eE;(w9R) z;rnoYgA%WLYB$+QFB$J@%CxUGWzM$RX3X&`#gvE_>43ZYFDx!;<9Mq<{A(;g!ptp( zwKjx=zwBqENNUQBW@?K7MgRbxA+phEd76+DxAZ2@6Xf!BU-MF8MV0wif5P-~Vha7B z{1U1N^_a6GtKC2}Jhf5ctM#WBxE{p6l+G>lmFo>vZ`NZ6uHhc>r!Je96s=u0b4?|S zLz?l8MM`Gt9EV+kybu6`f1m@F#ZOP+@g~>741s%P6|ketFqYMCLNksg71+ix%zHupvtUdy(QRj#Ih8!MK-nIbP@5;UkPzy~T z1o!vG(cz8L`2)<0A88o8q{9l@)kL@8n)fO(+Ycv>NvYcKp1XQ|A8XCE4rtc2N3~64 zF8332qP13fm>D;agAmF;7C95s<;GEHxC}C!*Hn>o8}OKBP3+z6#N%ek^99L$H0PsB zR-K2(R5ouCF6@WdX8+Y~7v4VJJ6O$13>r0cd=RR&X0`7Sns441Mt53qCNaZ~UkRcr zhB=`5kMS0Bwl%lkvsX3(~jLBI_n&lS{&qC!5LpI~sI#Xv;}jKm@SqyH+kFfKsE z^hv)FZw20@;@M#>?RtYth~5Uw2JjM^(&wr<@mjF9*FISEpag zm?lh|%8#r$98);S>%?xg=T8=W-`GE|zFgD;wC%ecn@?}RfQTc4N9C@^t)%4OcZ4qq zR{uu+y|~$#r0e=vZ$^4ro+|rd@3X#47mtFh8;m;90$44ez`fl0Rr%h0qCvzJ%jc-7 zv(0CR1~IlDZ4Iwy5>u!hLuuH$Ia4_{3mnkw+7qjMqqWy?j?<3d6Tbf7y|yBEiAnLKPfp6PZA}9R6xQ zMPcw@fqia`}5O@A%k%=wV#i%>~6nHm@Ppx>2wz0uL8&Igw8t=k)2sZB>W zF1_w-^;8Vx*%8B{S5Bk)e$Iy85XOxc4+fOdJn+3eB95c#s2Jb5U2lxMu{-C@$WbvG zh?u4=d{Hyte(hDI7=<+9WPbLLqrjzr#KZI9Z&O@M4;c*Q#Eq{gcepFYey8oxa75o1lu%A(WuYRZT9QfUlg>bFaUr590Xkc=bFpcEmZQ_w7U7aU(CD}J>R^Kbrv zFGi2;D2(J(H`wX8-r*SnTo7uQ#NIwqfIfAJ5{|ZuBxuNQGg<}=(&E5NKiqL#bZjSm z61_HS8;5idB`iOUnTK0{BW&`}>;`{jbeh2p(vjYdnMg{$1=&gg7gj*;A1Z46R@nxV z9$7@=Xn9ConnIQUaNBZ7h48HkzIrMXDg=o_rN#>dh`oMY-s+GNmusC->d)xYBHN5E zdR=WYyX;DDzdGtGnBE{HU5Kid3<_cC$R{HhMJ>f}RVoLQe~xeXvGHt4)z4jrpN8r| z(2nhK`%8;?u4n3GMFOmZOqNHwAs~kWsrT$rvX9Hvx~E2|5Q0FqxCd7)2*InSoT|Ho zDU=|~R)6865~h>M)%VNHLy2GSC~a-(?f!()vnk0{Ip10a6r!+mQZXEc8wM)HLzodE*7eIz zlO|jcO(*KN&p-c+`PIL?&!LpF?DhX5kTCkhyZsAR?+?gm@AaD&<^D6~!V9u$WX5S) z7!M&nf7WY1ZM}a+6{4zhf}woqcxOWx%K2?^4KpYs__&~}n!aS%lI#If;HJGkqGlg2 zmhf(sjc$L9%Q{f-SMP|xQpAkT+7R!W`d7Brp=a`l^u&Cg|>Y`%o@xexiRR9aAe zC&YgRVw99*nQl{RN&Y-gjke z(TROkFI$np>ks&gfLSO(hc0kHWqSxuR@})9dTl4W;NC#f6#q0Z_<$?iG(7WhJ}A^X zG9KFP6WsoE|8PGnjI#Wl&cuS;Pyhtq@@8TudI!QuxY3NO)J&D8@L#|tS6T-7gl|Q_ zoRN!26-rMLr*$lBH^9( zI6}G>lrIWV6T;-RS+gR7P4jsT%lG!VuqqB8UMbhgR(&8;7y~=z;DbtHQGSOUom_6%0|fKEfCoHHkB`KtQY7?Jc$DINa`6=RL{Yy-L-LnVZ0VWl2aJaT?fR_3I(LwT`-a+$dcyy0W4;-~dHSLv2F!-vVE<tmKjc}p2$2P+crUj&X$euY*=`eR`Ul$?|lzcNi*AQQnNK#wY_kN3vO1J%;Oey)az5 zJ@4Z}{`Vsx57`qBj-BDYlxvRzvj|c?JWdYGM0*h1B1d=e6Qk30pwrI6E6E=f(=BSC z{SHF*U(7QhUm@&)PW_VozdG`Y>x1c58_CE!t^k%tw}k42~aT-Wm8>oXITErY|^;ptwdm}{h$|XM6f>ZWT2?P zuc3SsQFP5aLH0U7Kzq--8$b;zRr6OKIoNXf=Ic|ixp(r;?cNXAUpNWH+s~Zj9O>z| z)9F9et}YyDl%K4(BP+jmQ{f%8yUcqd1%sY=d;=>=V!e2=5IRC{EL+TnN{%Gb23W0~9^2KC zY%J26pAQh4?OiYEk(*lp(dn$eJ+j4{2AH88bm=0A4e|8S-1YQ&v z!VI@F<{qrx;`FH{;WeN6acGBvD5LIJpT?JRBT$0X$)(em8I4xM%`>Ah%!EtI`GIksO(A^f7MBPf_Ghg$mTA~_Bx*tOil zS)L4j&P^w?5E90#$3iO127tm)_ZeXjpA-JB99=Qyg#mPfOOVc~LYtW2*cw(KRVcxLmTyT8N zww_%D8Uw4!R{P$3r2BUB=2G`H-aZ-S?i(h)Jw4997T-09-g?L3Ra!Ml&b8F!HXh0! z+lS1_jZ>bsvjyVwL@#_7ToaJrz#5A|5A61&bbA}XEs4(d#8@G$i4SW62O$-N9}kfU zClVK@Njxf|ZJKUJqM$6^EImF{G#ApKmRfph3WH)CBl0_f6ftw33GD$I!xHcG0^Xih z^+I{Qsw`Knicq6HGSMeKkUL8UnervhzU@ObPhaAN{Hpd*K|6UabZXDMjyewq)@ToU z_|J8!!hYqqT`EoVul2Ly&ujmQMqJwoe@-EbZD7HE`+2Vz?!jx*uAUHGMU4~;vrSw% zv%xSI3`j_+WXJn)+O;IUJwg9mN?ImyvU*~v6mVeU=!kWPx3H4XpYl`{EtjsD-+wUFJsY{| zpm=4CJij;uflL3W0kBDJ2jsVfg>wJq^Z$x0AO$6fQttKTjB3VIB{wG4So&puwC7#R z+~o2n2eU9=4`l}v|MR0yUPk?|3zdbPiy%pw`7t|ayQPjlanT3iZ_$3; z*UQ7%soNK417A8B6C_N3WmofhW4SA#|M(f_Wo`Z`51Za%4|UjlW=9u3}ZTeA(^p3}@{<0*yCbbXYf&s@1o`E8!cH~cX0r}62n zOWtEi^*^dHr9ApsVPL4Cg%}U6JGPHA9>=jON!g@Dh-!fz2unFoV<9NTQ^YU47-?R znFoPyj)k|)R}V?~=LbDzVwbhw6QnMa_zUA1G?scr_k=O|M>Y9UfGi)%*bc`PiOYc5 zJS+ziq?61^XQRnbkfb&=dR)1Rwa1a zfUA&{cud)Y7L*6Ee&C<<2{YcN&s(LxXMCN$9hBS^w$$AMHMMWfKWbbi43T}1teZ78 z9HcMeCj~5Tpa_8$5qI)EyYI-tlC8#nE}|~T^Mz68p@Mc%Mvi7CV?VxmgBq|b_J)<9 zQXBCBBu~jrZ$JsC&!^F4Z4pDV+ybrq0WR;?tHNH$W9L!IzzS`yU{xmmU~5L~g)mi{ z$n|PL5vnIGvIGFh7F-J^KCb(ltk+50T_J-$9iKd4X$j!3=sGM`Y_b*XT+v_;{Rd!p3I)&5`1DPEXocn1aj953b^LYpVD$83$&Nu{ zdyNXG6w0S5?7S@_s?g;755LB7TQ$yu2Nc4ud)LwgrMP~01?Pf?bt2Sv29q$Oz2i#u-VA0$WH!W+{P}`hC^zz+O@;{QG7J|Hwy)V{vmH~ znzNK42@LX%vi{Om@ddr!^9~OxRtyUG3=++V!GAV$KhMjugK*g(84?S2qc(ePfcZlJ zh2(q6t>pP`lQ1p}T6wjG%=Cvic*rGCx;UvTJ7wLS3}q1b6klD_ynq%vM15$qLTz^1 z2JHHZQP{NkBvX99y!nNet#PwhRo>osHiz&fQWIBB0a@(jkgJijc}4%A*28?hALd4? z${Sy2?oO@eWy9Z&TZ6Y)C#7U`2_&7&T(q9#eBCQO-iOfOjQ@-jixOk8-*q?ze8786 z2cL>UZz68#ZHgxHTpUQq?`6Gycp=be>OXmbR}CuUdjxu1!Wq1cB5qx#4PpVoIxos% zLiA>dzsJ{VJ9V&Pv>ga-s%Q*)!z=K{iJu_N!ZMg2la=__(z!xlUa}!33h|*;18|Y zRmf&Vs{+g!|Nf{ZJ<_d!*D{--cKt8(tUn)_M3h!IoZ_DzbxmQzBz#y_VKMdQAM|Cw zKGTq2byou4`6yj|Uj~>|)ma*Lf&I1sFxS~VjhVDjYeoOlk))Rkt?&wm8Q;S~v%-dq z<2bDxz%vjWe=>39qk^YdST*+===;)j3Hhd7b!6{@%eD#bY~UB@GtZaab%gB2%7LLcIH@a+Kw2h3$t2 z$)Ec}+4Fl)CJ0`hlGbWO941QS(OQx4#6;iiPn!Aie@}!mzW+<~4E-Iino$%O(Ksjm z?}7y^(`@8aNX1yEugmbC*z@XN3BCIEGwy#zM@d0or_j>qso?GMIJ)p}_r6{2Ydl;9 zErg`*Q;^?t`RT=Q2vIY!(X5MO4mb<9>Ir*&{6Hiw7T*MZzYEj2DuX*97jCV= zECYPldVKn{K11-IUeiOOFyZl>%u$I%}0xfODKCEHR^HY5>#(T@UFxYCxt?d`|Hqdf?*B$&-VI!NAj z0Kaz~t1tZ;YCd0Nl zWafo863y{v$L#nz@OJ+TOZd7@I$o2jbnk5n2S|xJQI5>gri7UfS!G5(uV#AU`UPy> zNJN%{70txDl|C7drH)-Eaih!6T!D%q9@>_!9gr6n!r!&jlFujpeI^8BVf(V@vNEsP zDtHz0>O7B8l7LntP{xbCG63W#AwzYN9y1Rw4pjPv?^t!GD|F@kvPih~;mOCbVPE)W z=|L%=>dOz$j;BP=xB&Z0pcU9OHhkrvw&MH#T)8G_gK(wr&zMvB_WsL*7hQqw*N2eT zTmL17;qWH2e`#T!4wh>I?*iNoLkvi|ex04X5$^I~2{e1_dF%Mq{pz9v&+#d4?vs8N z_{I9=|8$tjyj)4oSIo}u{9%-?hwnX$OYyv+>{&dGZ2DAxI>QG6g~+3D&)jh7?w^-R z^S=c-4B`kr3hHC;F-Kkg92@ZyI>0ao*&kh#$>P4S50RYsSaFE^qjsY}z#+1>E-lI=y zA-2=C_0G|(jMS}T< zj>GnivA$KKO>W)%UUiMcq?rXEuf`Wv&jm4YJj%uTfRE!)BZFF_Qo!M&9s`+ zx|qDYxmLiSOP}^GkdwtL+G9XmDN~DrIEX6TUDhiz&2T`Z56cfp3Nib9R{YCxdf4P{ zcyckbwtz6?)vXXvG2_Pc5Ssk0KQ0dxQWZAbz<&3n=qHB7AM{Z-=er&$1y!=J0hDGpa}|zq64g zgAmvWxNgcHJcu|fRX6kUopYeQyEVK7FXW`=D=mBVfBnMUivG^;=&39eRQ@I{qfz)gQJ=OCTjqH!I1> z_t@7{M>xsrm?p=P7{~WC{%=~BMn#F*|2M5`p+z*|tYYzx=>OgW`ooxA6Ri7Rp7Ytj zpV5DL&c`&Wpb*cd?A;PX?Q0|9?Gl{L;V*i9KMOn1{P$0ZVyOK0b7HHWL|e|GVGQ^S zNsL}u%{FeLV*BN5^Kmddk(R0S^vXF9*(ng)x$hU(sm;jE7Yo=}qMw~>uF<~f#d$B` z+K6lG^*t;$rkFtDSxV6z9M5B%@zcL)(YhXN_6x| zI3wXmBVGX1T3#}MH71jk^xn1rVKq%C1k{4!We~*HVN6QLTl$VEFxrK#;Ie&Cb;!#f zt@=WP3li&nAI;7QDsd_t$t-HZ5%_WPGI7r_tycR|W>yg~&Vj!YhNgHptoSDPeA9pnltH)RLk+W z+391A4k(?4=D4Wstsh1JYpIY^{V57fQWVNuW?IOxpjdB(9R20k^EED;iYfZGNyyZB zavW*m9uGEQQk7G)A)~Jl7=OEqUy~{CgTNxZ1{uKsyTF=JrFImw+xDG9?(nGkcZi+uQZywF0 zBJ>E6UJLZfG0fW*BLlaM;5tI)1Xymwt%$-ZU!7%Xx5{4x^|rCD-xRDSt$X5A`9dZ( zu)x9;-PM$QpdFro_15xfTa>^3;(UQ>t9Ha0O4WAHAGZ95@8F{j{s-Ubz|+RL5Sfv5 z|1*ho;HAd)YuhrMzOq6>MMGFtUBV9W($II@zouiHpX( z9)HnKgru*gFD!$C9@DcN7g4mwkbIH3h_8p2l8p#qEFwGg@lu_w98Fp)n7!?@5Vg4yGG? z;#NayG}cMN-rgntQp!qhRYC|O*Z%0qnfcwHN{jO@ThSstMq8&m(v(9X3kmVCMBWe8 znPinXPifRvC0cp&xE4x%G76yihOyZ^Bqf~ZW-D5$qNfonEoE30qUOOMD(`S==DB%@ z82Vw|L{j@@NMfF5_yeFXyiJM$U^U={I26Yw5@1L$ak#*&%5E+My$wJSx8R+N4E?-u z2ntr|F3lt{&s2Wy4ygkXH>^pLIvOODT!a_sw`$Q60l=zn* z@Fg8lpYiKIGr2H;#e>@#G4m*u5Q@WZob7cA*@Yp_-{bW7ao#9@A_8FPx>bx@KTi>K zo7&oYZ(jdX)BbeD94 zG{Z172oe&KN=PFm&CnnnGjvLaf`l|99U_PfrKFUc(dU1j_q^vi*Tn}u^5fq3-fOS5 z_PtpOMAUivO_G-2K*kZBhpd2^sfLohL}D{4twyZzMxYy&M&*W6!;(u)SoEZP*~}*~ zLxgQB!H_j_=PtREKl`9F??DZDNXr@hS#Q8u4}lli*xis(EKWWHsq+pKNyhI9m&^yJ zU(6YIk6#n`Y5H}$|M~Ca8pPnglWX4HgQeo6HIPf!7BWXC8n;imVN-!&NHb0peIEF) zidVA^4Ym4DD|VCeK(HROZc(q{vxY^7x17{zN^xt%rIO?V&;@sM!Zt-k;$X?W75BI>Xp>} zeNbKj5l?;&d6SW+2-C556ZHfiex{jt&+iXloBKIbK)X3)$@w%`*z}QVY~0V*UpnOc zzmNm+nr{__&HNHO;Kje>_I*8_qEbgGzp< zk5=!e*1P1iymu~N2Ov;KdwU(?fzKWYW|D?8#${vU1+XkqQz=`#eG$d%5Cg#wDa#^K zI4a#~D5u&xba|m=#lU+{^p?nnU*%1vdC!uG7{`^Pd$tlLmUnUSTgI7444i}aLsTc?iV zWrRZUT|v7W=^IfBOmg0r4#cwQ18yRN6cHP&s2CF5Sy zwUs3aZ2s?IcQ%D5ld~Yy92^~%yfXlk+*fyMES*jILQ)d zz^H*J2X53xu0pP4_#T>?PCwZ)vBrUoYutK6W0)?QBy9XapeRwjpQw-u|K&E$8pSf_ zM^H7S`bj@s3*$>s^S~>((KqUvKlUB2U!}-H({(Vn#c^W z%$0x(c3!7nQOAFN-qQ1-=FG?qovvy_SIz!p{_-W0;`WBkBPjX4C~P~ITXF7r@l+I9 z=+VR$)sL8@M)ZDeH!hJ`d`K#;{iDSx*s{&asbx2r(9Glml(OJhgNo0yxn8nDk3HABLksSUm;@OF$@m9R(Rqj10^ zN<>xOFt}VMVfeSCu%KjIYoB&06n1@j`tOS*6YnY@1oH<@l&=YcW$#mD?II)!@i5yz zsPHARH7ujfEErUvGGLo3)KF9GKxIsda4}lX9V&%+rZ`7mg>>|rC_js7+YUEt7-Kl&V9s~b;=~b zt25d@5l3){%|J0f_Ex~JCBlrV*o@P&(H85bz9W&)uwW3s!KemKhUiRj?i0~0aqLfD z-e&uxQgc$~V2u1G)-|S4mSE>6?OuQPe(^Z$K_drXwaXw#vSpG#DNqY%pXwVBQu~|) z0%m3vS8m&SeqS$(;m*uaR^rS`U>^06l3E2%ERhuZjT>?WV+mg8wxiN^!p?_d*k->< zz+3@9sq+dJvHriDO_E{yYx87nIFloeMt#fT_fzhr|L>S+#u98#WY zUxoQnX$St$~|M@Osd!P^QWp+^Doy~JMW4U11&uuH$IkR z*&&R-P8KF;zg6`SU3%);^k`oO?+m|6=&u+7KdkE!AwNO?LMUTNA7CWWeR@4;uR%U* ztasYHHdUEl?&t)}*95$^^+6z6FP)qV%^2hPC=SIi=~(EV6b$EESjfpQxW+ z^eYk8Ov=3|vLC5_O4;Y%8)4UMW0S>!vxig?fOFfV^-CD}{Go8c2gT*dfPB*5e-IBd z$7zmC1T7e2&~d5-v<&sS2ui&ZG1{TQhSELY2$d`JKuYJr>Ev$e<1GXP`!xfbpix0sX#xbF{dL zTIRe5ytGyr9MdJUg4q8NS`USkFUy%C+yY~P3{m1ml=6vDA+aP^u?6N17m+Nj$7gM7 zAsg;@HE=~~%b)i#C1Lg#VJXA9rp@3IT7$}q`y*VHXj;CwHCs!j;4i5^tybMU4E=IfP`L-bi-ps$7zLiGKTq2RQ{ zAeC%l6Qx@K3B-?SeFe!J?E6(#EN0#IQW9sf2cecUwHpD9pfVKjoVzDy_D`)<_8dqy z@A$1N%VohSY%^nu(pdt9SyHL;+gyY_UMTI#2j8RkKlp=$;MY3GME<_{FG8$F0Kcv`3JPId+a7K%x zwq~T0U6FjJAH8R*$`$oDGZ4UQ2d7AupkVpW@|h^Da%(jqjn|R*TOI^ONEuqczv@_y9_A?ywI`wG^u(d`+V6LNyhOwy&0jgs2UaN9D7Df$1A z<%Ot&R6-nk*p0#PNTUX$tHdAS`-_p(mWoCq*OR#pzf} zBvl8#1+7LM9EvGJj`PLt!@N8E!D~N3$0GUqOm|mcz6hX945H7D2U$U>20xvzt~xeW z^ETpI88PLnKb5O7D)%Vg8TT!evk#1%iXp9UsnZs0KJ0M~cj;@k+GjDXPP&kIhw4Ur z_Uw|#JPaZNYH%~=i*HhNjk5dR`18b|_qtmc)yd$y2>OS0dMuOIc^Dx(R%J$vVXm-D zD=M8_k!f1)-}!;gS}!W0Pr7mfX&i9RoLCGyFtm*Fje4GJgK*dztwc$iR0Exum&E6+ zpIPO}K;vCuH4j?#{KH}sO9EhyqO6fHGlj_S;9P%E5+f=Qgf`u}G9fGHu!5bsrX%tpM$r0=f&r7vEAvAk z`c;qdH~j*nFOCfk3+SMNMvjq}jrWs@Y$Y1&AKHCGddAr-SifHsSXt_aLNpVO;fECG z_*uQu8VzH&8Y#7df~@8ROSDTM)P!;!J`2Qgz;fKj*jX<)(q}QvEtXq^VF|y4YL3H! zk8m5br_ZXf(|(ev@peLqa^}0v?V`#t=-5txZs1SJ$6~%>x1zVP^A%6!?Lq_&Duf-e zBb}P0f!QE-@-ZNcs3|JbwYq(&Z>@yzRg<&NBbx9EoJYB8n|L8E!>h}%g<%9FQQFrg zZ#Wfk0sEC5NycDF-`vrtTUAEa0T=e=|Cas+4X^Z;DOzuY0fZPTww^%l7n^i5h|dgB z@h|~n4&KyP)_;(mt~0Y_%8Ew4P2P`VIiB~wyLUZP8H5lvIc8fgc?l}Rl;!a89~PV% z)v%S(fB*O*B8tpcHD38_^r_6Auu^|pSghf)b?oe97JXESMGj^U{>er6N(W{ajIv|h z2lJW0udPcgryWop4dzp_qGXj)lgh#;Id-TNCg(Tw$$77M3s)oDoCE~lh--gHcc05P zcCzUytw(K_wof<;4#GaOQ<_&g8QgU0$-&lY9`Wf{GHK(TeOGGoewyD*{Y9+Fmr5%$ z>Af#k#6x=JBwc*vngbX7=egVk<+n+ybtvV!T%u;3MNF=YjkxCY_lN3_RerqKd4K)m zZJ2OkrAGkzo}i+5;R@^^MYrOldBN%Hf_NBA2hZG*vr13xIFsGv{1DLUnmFan&rHoY z{(5a9BlL%xM&p=Y3z+d}c?;#jtG}Ysk-+r1x$xZtGbW~rG?{S^F;B4ez*rh93lbjm zPEzOD2G3YfBbLcJ+|$Lp-Uq39SngW&$_Q@PfuT9y?9BcU4#sYb56sp)GU>@Tr=q2X zgT->mu=s(bY&~5vG?qZrQ{Q=3ZuX%Z)ttG53IS>Z$w&@XSw(0_HF4Hlo!)Z66Toi* z!)$*E!=hqqF2`Ka6t0jrA`?q25!umlEP!~gkD%nhJl<0cGUu3#(?Rvou^UX>n-y-% z^T1R+v9uTBniou_lGk|)%yr7@p@`GU-HlkI`^2@BTI>5EpJVj?@Bo3WIbzGuIWP2ke1B(T=7aJo;qirj1GOKrJ!93nnZ!MnZl#+G%JF8Q< z2)y8g(W!#5QKI;K8ggTWl-&MP*A82Z{3p12edXyFs0E~xhY)AG4Fn?KE$K>nX6?& zE@B}5l>oGP~Mz@H%k#~0aJHx{41azrKlx!{t$y;zQGhs;}Q*wAkIGlrwK zO8YbT6WuJ!i;BGSZVSX&hAEmnCTO-++i^0eiuSwHl`#iiRuhep3;BlsPEnfeLBlMg zx`-Dr&heu@hK$-nPg%FY}^p0=iHdX!O62bZ8)tq_}E5{>sf2mkoPX9IoKVk5mH0vE##=%R=?fH0hq)fRwY z9B`1O{nsXgNDU$v#J3@)^Z6m~uOpcYv$V(QbvJr4(!`qV{lD<81J@1(pU&wx_b>Ez^$=6WXrk*e|x1{}8C~BoHbomvF76r*TJ( z&y9aus4@VL6yF?wn_=c z`>N5~xB>+*HGEDV1$NxEDYh=Z)5ga3I~wQ&9XNAWi3^HjuX5U&kGiTmN^OJuR(}~j zT4!hz^=N5c@SDqK>82zL{H0E99iQ(Fe#^fpv;()>hRbl(VTx|cqizl?MWMq&ii(R{_CDIMVwWeGX!IBx8=aCx2N$B8VIKc*v{Z z8cfsSimo~1gpPRJ&-*{<-LQ5e>cI8hD@Yx&nUW<3BOX>ZqVwUzPrxU3U>=WFcPDTdd&-gqu^%6PX#sEt$mM(!}k`b1Vh*65lPq2 z*&uJZW?}T-_%ZX?AXe`$#zt0L;w{x)ySr~|E&ZI_CrBumd@py=<{r{^bG4q-Qc4&; z7xNdZY>2kYO8mnr9GI^2onaWF0BqvVAcKcxsYK5=u2$5W$Z29Sh$Ah}Pi^8txOsWl zM(UngXNG-@)v-|IF)dcqvt87uG26w~A|0hm$rdVu_v@rtOFo%u=^{l8p0Q#Sya-{J zy-~|3=vp4@l!IbTZXbtN%6#vMxAbei&t)c`^*2}9bOB5jHa2X{RWZC1*F``W0p#FH z8|2qdEe7TWyYIg9o`Y6JS%%@CD@Fuj!7t1U>@j64q=)HSy2HYK$Sv&6eva=Y$G!HbwEzeht_Je{NUF$Rtz;ZhJ zEv5_JJ9+F7X=N&U9dD{&yiur}Bi;|-p;&e~6P*uBFt3R!K@(W5&U@Qm!&h+RC=w2a zr^h+(C}qY5Ym(2OsD|lFutJnda-ViYGHyaOP3KW3)>Bi0v%Q&gi9m`1?B%7)LQ;gs zv8K7^3eyu}sO`Gk#eX~n+*wK`2gslRTb!o8iiA4{!en9-&+ap2t! z^Kput?4%#J{vNEgu%z;*zP0s`UBV{=Rl14WptJOo zr8I8B#L%s+*Bz7BG?9L_7qaWY_JY#%Rdd;RUx{xh=okG2nr=3>Ltc=f&+w+Hv`e4Wy3a(N=H8L#nR+O|Zmf|iRV8G)I+|Hiv*lp9CjVCW4meuw% zp)#l+o3XT&R%XNy2WT{{rQd1#{Wc#h8Ez$E$Rz8e={Gq7Kq9d)8A^P%i zehDYTVpOX`%~Z&)s9n`PyZqzHBPc+T-eqW9PiG>G>@kWTJ$Ma?)oLY(XR`b7e1oKU^gpR)OarW3{9uJ z>0c)`Xh`ty(-Gjluz75LGjMG6Dq&>(_1vbx+=!5uqM2&DhGPx;+q8U#ozoMTN|gcY(6Nwkc&+ICE`c z=#d@0Yu}0%9Kuw?jwtcX97Hqpc->v2Zn?r5EdNua>Ur`8=^_=TE_UpbVmnCk*O9~3 zvOuI8@9(NTdedE-f+uRp1sPQF#C?Mm>TB+kW8lcveH0jAO9M*uMGt^us6xk1#2it( zF3T0)1a{MHiKJAV3Nvt?;(Jm88*5lelMg?0nNs+Cw&`9RW749+Ac$LETA5^1*j6mlsVQ% zdqg4enY66s9Xq7@8k|xy9Zr8Cjk3dCRBJ_bqnz1HM^OpelrP_!adPv~V#*@u4M(BXtycM+NOZ%c z^6St-%*!Cchoj8U0fD;+^vINmU>`9xOlB1-lUV{48N@vwGyN%j_aH$jlp2OedqFi> zWV>_TU594seD2cr+$g1T5FKdJ?_K^^F_7dlXZkVf27@}#2-vUd@||&4(zk`*Qshfv z@W%**M8Pxu0wF-GD4(R4js!~=o=Iiit`qZ<4Yn6)f#?k^cnSSl07PP!LV>`m?aZB} zu5^*a2sbjNsOkz1g15HfU{(I+^rXLq{hFffM)Y4{zXUpSWTtodm)3d&hMdiQXip0b z`a@KL<9Q(8)x+qu|L!(Pc3RUk=!54YjZq3KDM`*XSri@h_s9*3j{&>Al(@2v0=5aF-ABwSpFnU4l1QlIRw=1Lr;@G{>@svOv!rG0i-E zSn&E+A_+!8zWEIfgPY||!T&OaZwxdDOUcfP2@hXW;Z^&@JGujdK43)&G77e=cP zy&g~*Ll%lfLIg?^XQCE=%Qt47+ZSQ92 z57~ypZ>@KXP>xc&rrrg#Lp67QfxL=x|7()*<1&F_g|2i#=TN5`dFI(~D|apvbT2Nm zm&m$Q6vv8m(-vNOiyPINcTQi=So2}P6x?wj0JuE3{BRpyv#0?8U4X50&U|lss)V&h z<1ZS&yfPEYf`fBslhtps!qv6x@VI%OA^RLIHL#@B47s#EL|q7v*CaGJ>t$yxKk1*Z zHg)%F&YXYe)OZ=8SuT@9siqd_DX19nsR{b=a*TL7b)zTemNOQ7?A4fLHe|x_3Zfq= zR2*cvQ`)&4PmN;){nB-ZBe7J%`huM7#c|-(de>+}R^NKto&{d!#-s42=S@f*Zj0JS ze;(z<-9P>mx;%PIJ!Z;gA@}3^A5j9*Ixz_xdZpCb-*9%0egf%y%8 z_|%Vf(o^HX-CqNoSWFGyY#TKFm~)cSZ7LL97S5t;bi*jO8-*HiA<+d>S%rH^}o9Z|^&8NUI88Gksvp+%w69etY z)eM)_l5Y7*ch$|YK$=5A2Wxyx{DN)RFYvh!j>(kzP3V$FA{Re6in)-y7h?Y7K$6vh zP=OOGGMXI!E;SQ4aOD=niugg9dm9s^TRlom&zpfPTF;Brgs3LO?>@ICo z6tpGCQ^Y(L_1#qS{SN*T_HH)ya*!|lRSiaZfC(9ca9e|4?aO6DY+-L0Qwpcb<|~%z z`lnPxao@aV{9#>6w%noilC*Ev7HTj)4fEv*c_AnRfEHaXS%Yt30hFuL#!!f(PP@;*neWPuAc@wGqBxGfWYdYX0ZX=qA{V z%H^2V5Fb-(Odg$9CAhpGH>s(iqH?(GbcLZD2*i=_F=B%YYrXI{;W|uXCHt2CV2zb$ z8&52P95st=>QMIrKic?dF)Il%F8^e0U`x_oL}%kKp_s-@*Uy7pO7~w{pLEMCx$e7u zf7xEhWg!>gMir8Clkkv5-=3Cx5UlF?by)qVeH^#E+5{2!#5h#WCM=aR0ycj%tE03Y zSq+B9c|3v;nPKGyB6)(&{krDrMXmo_3l=CdBrotk7kWvtHiQUhSSo4-psE&HYEHE2!K-3v~{FDiNs2-)#4nX)07#1Ga zF1J0^WGG>Gv=MfDykVOcDeE)rQG7&%SiV-eDP2X65nj0`(fe?57Ljd9-~%0VtMCo* zq!o7Xc>kBe#%3&iSS(j2+XUm=wac5|vmyX)3q3jZKEFz%-DC$Hxi1?a~V9jknl z`jsCm;p1nN8<`dDPK}pYN82QJ-iVMOK?i^WdsvJ(%e5(#636$%ud&ol$3=-92JqXy zqx~Y9usEs7_|*|Qnc?`dJa%*JI8V(_bpew3^}{u1VP~*~-Cr~k#wuh5AXe|uMn6-h z$Br4JTen>A3M*XF8v(Ay;+@N-rrJQDmmTVpT zD4lV$*|Fz|e^^#^-sn17YMW{6t?IWMdI+eEECA~Hj z<(R33L20KK%4T4rM!@TPAy83=rDB;BVUC+mFmp({O&g)N7@@!*REbPy$p{-2FYzr_Em zaT#>yn*VmOe%0oA`)Z93^CfZCp$qT~DNls|FAo1I0Hhe=zE+W)gt=Gi3CZi{o;I%% zSYaEqs~vle&4jB8gE5YHW>#0{S{50*N>eZ|N1SKbYJ&vHWufUM?bi5C%EUQkMoO7Z z;DG@hzN0|~f~D-+%uZRYR0I$GQbIg+nu#0w(G**3ZrT1vnPEa3w80tLYoKygetJc^ zY2o_QD*p)!nH=LL)I^FsCXS}sGTBj=S~-K z>~YP=5TsqFy@O#n>nt7k_Rsuk>sG5o9LzHxwu(yxrTs_6M?hZYUOG~T jN-uByR z{e=SSu8m50gN=8~>e|1UtbaQx?;9Qge|ym6{-&Q4@MORh$lW%Ej!ASt6)A?{)XSJ8 z_zN~2H{l1b1pPJ*zKQ6xxQ~e(*5crOU3b4M*^?D|o`?JT`swT559kk9?CPYv`w(xr z!$%5aV0Yz&bN}zR0K#S;K^(1f)tj(54Bp0qm2u_jus!1WZBsthvg{3X%1~UUF=1>G zjerkN^IUG#wj%};P7mEYeB|der4t)o2UXL|NKQ-J04m@KP1URo1b|#4cd4LZ<%F(R zgc`03T!$Do|J*zqZb&k-g31FxS!BbH2PWLi6V;ZP;wIQ8Ar}fGoQkCpV&(3>m~F~3 zm@|fo@hPn#5|M#a*7Z5boVxT#fXwB|68GoZjzqVlNja!ik@^o4^7&Ky|*Hm5BVmT#f2mUb?j?Nrp0N(1DVNncP z1LmiSkjna71IBv}~kyyLH%kSpzGkCfB(%#GVl>KYXTbHZbpi4EIO%DKy` zXcS=SVv|~zair2c>+2%myW4Yv_Ys6M;?|_+`jV;%MuEMr%{P{9S@Evu28XNvT5MOD3Q zE~b44imLNk4z13&WA2v?1cY-E7+b3wPnbX1QRkC1>#DC7F2$bkt1pAHjs2$Uy?J?e z4u8nCIRF{d&`crgnZqb)6)Ou>^s;?vvtKLdYUYe$fNlw^Y0MAU$x{%?CuL7{^SxLT zoTLZSjt)-@p>8~|qsSK4WrY;jE%PD~-Flmp6b@?atx?)NW9&4ckx7q_F}BSF6bmrV zPb?NxX|4NYYMUJXQ$$}V0>#us>TUjj-wN;*$OF*n>>nL?)9OHN*8|tz!_&TM=BVvD z7vBmrtc)$D|UI|Wtu_#)HEJ>_m%TAYuJND`jI!A;~r)ph1-1%%!o zC1m@la8_#x#o*&=h|+@I`nol*W}usK8{Gvqf<$#Mz1CGC!mkOrtf4R9{~Wjk@0mHz z>gGP~j#}Fc-0JH;Vzc6jdsV*IA>^Ia`2I%#RW)eoXSYnF_6_)#)ItSsy%X&Gd>nCs7sc5+b&ek#cdRworvktTnOU@ zMZ+n&;YXVb&&UkEttS>4m6|19Qqhftjt8N&la<|jP*1HV!!Un#F7nkp$qI2pXD6Dm zWVY^TDX`?+0bwUmaZGA^Q=v>r;S6ZYg(4q@v1Juo2BMoIXNXP+_^qsq%2p}52V)yV zX)sHw6Y0aW4gbe~9Nyw=gAy0#fGey@30wadJ^1Chr zW=9;EqXLmt1obk&n~SjJR2b@U8L1!hJG}u?STqN$fG7%7gP~SuulrpzL$8eYwF7&0 z#TuU=646-;YReI z?JJ&-OFZwFp{sQIBQ%O$wRkx*8~r7jIL%TGy`6dOM%hR$6iuG}E@H}cbGZD* zA}eduifTh71hkYd1l)<4xo}HweZP2_ew~wsoWW-}KtGs7iHUZi$c%1g)#+K&skfy? z+rm5)3oW~Be(UoKDm0RH4-hl2pWcqIpWk|Q_^5TymW-I(XYNfEoA~9pl$(~jYqOcU z$TZzf4~sxo>Xf(2^eb&%cHjCL*UO+h6R+8FH6b3iu*M|`wYtt1hp^OL_jt?bozvx! z^ro)`3lFJfPrM7fbKf0?$nWs+%c^XCWJgPE6!)=*I7EL@dp16%D_<8)?(=SyCZp*? z&)%~ikdVuZt57ZD>w3K%zMzqhv8F3ZvY?M3E7%A)^hm+^P82)Uk#&ebRZ{NtPr#di zg0^%-6QPcxf}UZ`i3*@5Skn|rj%%0Qz>4Q3F7-}bi{u==p(+ZH&`#rbL(6!Ubxg&0 zmfut#VniYWec*H>Yp{mL zo$-7${-QOYy%U(NSTguqJc16Pr}Ss&HqRv*`UI-mQTT`(icMc7uK20s?J)u}8n5n( zhnl94Ha*8L4?pQLa|!+T&FCwG9bSP%e5U(pz_efd?yO-HykL`mw;rOjn22V+z$sIV z9oqa)HLA!PwLzvW3Ds^L65cEt3bDp@wS00;BVgZ_U!ODvv`=dYu$+jp&kEZB!Gw*j zA%z%PEimaMQR*+BU1dp(YXi#yX+~4os}j~0Fe72tt|gKa#0Dv7c0LJ+d31xqY^Egd zWl?|3^b`yeMqGK{y`9nYDr~iZPO&?|7@#z{f~xt#^$3 zm}>W(>acbG-uEq1XYC%>G_}nS$?KkgaWCrdymzZE+xhju`zz1CiL4)dF1pbHdyVbU zD_Bb=^z%G*T`CCv>##g|*SJ>v@6H%2DH44R`NkOVThfIe;q_05DX zjc;USfwNymEI5#fAO&H4tG&TaRU>}e=b_!|4qJBT3Pf_DF!lh9UdvlHJ(DNH(xH6hG%y@6bquGg1Yrj&K* z_jS>(RCgg~7|Jar2t9uZT4q1ez$&w7ZZl4!`oYK3JdoB~H{OdA(L=p2DlRFr@(Q94 zpOHnsu7^}XAPX5X*l)Cbl_`umgW1jCX9aIF&*L@jfcaGUsXgO zQ~(X7*}f)T6lxvn+x;rh@u67u-u=+beNPT5{H4t54MiIgP`j~04@LY`5FKf39m^?M1i zrmwdaN4>wRV@6Zhp6jJamjvMhhI#ZB`WtUk5p*gGBJ%&2nWQ2itgJfWjjX8Hv>X&$ zx#=WAW8LEH(;wzJxwUm~S$9=Lsd~+rF0>TGGs#=UaQTK48?#D^?eKLlC(CEn4Q z>@I?PZrz$yc8v5DK8B29<~wugXsQZZacoi&o`l3tw9t{(=Ar zI((#~n2e$26p`qb-f5Pnq_qYeF)a`Z1$4OXq1?WxTtUwTmqZanwKCb?jI&*lqxUCE zjD-#WN<{5!A-bfzH)@BAq@}IdlBdG8sjA`2o~^3}kNm+m-mjBLP5D>O9N4E#Zr%RE zP^NRq-DZ*sPckX39_?(g|AJ8CP=^0{h)#K8*wguhB9I)L?p>@x0CYko!Wc6BXTKY&W*QDC-1=~K zXn(;S$>IKUkbz*sj>BUrwNGdW*yJF_u)rA?t+$0x#yKc=qD%ZAnHn z3W8yQFqz9LS@ifu(uT~+%)gV4R4HLF} zkZ=K2xHKQ8xCoDyY?7H@T{Neyks0O%P0IXmNkCl-pnKlM5f5Ih*NX2AL)VR@>jk<} z4QCYy0B>z7rOnrI*U=q?3SXHgmAVEV#JQEPfb6@zc{Z>agM+G94a}9)LZS!Vg-|t#8w)R1r2;YwWfU%|eyg%z0b_?-qIkYT`^aU?$+N z{NBg}g@x-4%Lh6AjR}kt3|v1yb6a8#=R7Vm8>UY22;3S>sqyaos-nInW6`hN0m=@y zMDCMGsdJUHL9nqqT+>;8%?`U=kZ;3Vg||UHl}Nv(dD2Be%op10*%t(k0K}at#%t(`5E1K4Q&NZ$GaU9#joVDAe$4&= zW2FiA;LR1^7@k!q;~y$Ih}=|{2>o^e_B=u&;nbuc$lS=b_t~-ef$e`G0bclbB!YNv z_@E^wtlusbVa|VCT24^Q%t+Uv{qZc#GCIBKGmO=2hKYJk8_b8pK9ELU9WUWyhgCJ8 zVT7<3BX@UldteAh8-;_cmWMxAdc{V7v!048$I-1QNAYjYWropZoMC5%*?47!wX;HC zaI`#o-CT*G&8AaxVrc2)xY^TiarS_v%^Ap`0Pt69gL~9BQPgB==(fY?CEu+lh^SU? z^zu<39`7k`R=>-5y>YQqpBISiRdSQUI){p-)~(Wbk}aZpz`<6C`nsO+HV75`oLztH z%}Z+=vjgAZ!^U>^ze5(6{$-4gzI4X%1BD|TI+QGUFdb48xnsitc8iAl3aL7Px)q3Qr0g^0g9jJHU z8gB=QB829EtD-50Kqt>#CU(y(JmRj)q=Xm8b^*eH3W$9)&mlwvcvYNPnVP)gB|_?~ z+@n3hkRewz{}BGj5K`m4ui6%&s76w_qE!Yud^gTYb8YgYQo20oCZx~a=?ym+Fd^Rc(#+%c3Iw@X9R^-10IxY!!h%kuOw@<0+ zX0&5IUL_~H1ImDR7n(t|mNRp~W@)wEOo@BwC!8DbsT&?Vk!WH%+%e1y6ExfnOnzRM z@lLnZmCiG2#}3FK3*btth0DS}eS!OR<#Z}~)%JAcJ$3&JOr&~&V&&H2qijFFY8*>q zE=wi!Pv{?sc~3%m&ktL@Xg&R(P2bl(uTEMC+Fj}W_06vtxGHVgR>?Z8fp&?$=5`zY zoA1yxsU)y!X7%{4d-TVryu!;1{;AMYYP6^0VmAbuVk~J7${8 zggppWtC~F*LrNL0±t)^$>@mtZknrN8~-E0~DK@L?cQV@LEPGfX(#zD2hI;2<87hFz$jfD?#*5)w*jFYFs5=v}?!*ZL93DN53ygbd<$- zT3B89tc zkZA?orJbg#6*c2V@eSeEV9dbDN8`_hGs9==PD+?!q5{I?Sf12G*tYKM?y@+&o;~kg zx{Z^w^^2p@YBT>b-%IN!=JubWAE{Qv0jU)WFhyXA#~sa#-877Vo2&|{y{xw3fsik~ zBjA=~adNamL?oYvUIE&H&lMP z!CZ@BQ%uin{3E;ldTCO`obfb?L~V9kt4vgGK3?RhfDvbyx_u=D}}{^Z5K%Te2G3>Cld#rlWfS}UKtd@(KMjU*=Cm=oa6uRGl9 zf6b)&dVU0a^KqJ3q35q3_)I>i%jX}Yy+{b7+c~Ss#7i5`L+ERohW-(h3*LWpzc9lI z1!vW|II{l;==)|muHB%mn_36jvrMg$zbQ(YkrRl53G91hmGp!^y+f=Ea(;M z#;6$lx8WkW#(LNF4t@m^;(&|D2L+(<8|jew#%E+w_&IYOA#GF)+yn4+!}mm9)%^41 z*j?<8e?`9QB32tQDdfZCO``&V$+pdDptSI4C?A@5x=zh{xA!}si)veLV@lA-N@_Q; z5w|S8-hcTz*u8b)A1`kU6ieeCxQ-Bc)_D5=Vc0bKCb$#!e2_v~tvy z+D58|{}0!}T-0B-aBi+G2(7!LZ}c_rF)MUt%(^EIOdl@G1LYSsI+04PK}|nqwRAZ= zK9h`d6Cb;Lz37~)G-!fcq_Nyk7!b}yn{_^+Ti`4Hz}0e#tqjCk*G(idADv3duQbvH{aXh7I^98td%GjSmBTn)MGbY=+_LuJ|hl81!AT z8b>^nA-B?k&tV+P!{kBFOwx>NP!(umOQJM`DUh}H2TR_$#N1D#jPozAmq%)A zE?GvVq*`}#EYEAuIh!uur1c#l=IP-Ne7wK1&jX-aLN#nB65}b50fhNs>o6uz$^WDn z$Y5%tfgTJlmNc{I2)NX9-`jkK#$b=%H!tPgbw0h5t zvqG;pN5|5?rlRlYHEzN&m)A+3{@9NyPsmd{-^i~oldgC453eUDW9|>icP;K)jB1Uh zX?PV6h~vYAy*BNcV4%)JDlt9Gh#@FfP=AWuXLPy6gM*`D;+Q@yGDdNBz_B_|GsbOU z>?7cA&(=Kn)Pm_s4Qqn}V6(7W_%Lf6XnGUG74zB`{kKP7{d^P?WJ6wD@&r8~_vm!! z=X(PVK6(7651roB;babok2)--b~;#&5*~K!u@T6e!grSjZG(c;pXe3Pk6Z_2$$xPw zhc|o;sGCa;&Wu@Y!2T5zW{>#y!BRqiTu8l4gGD0+BM9?RxFr~Oy84w7CtP#S?6!Im z;HeGpfvab8J~z+ha$3B#!vc1U5vc|$*BJ$MfZ5Sj=52VGzbIDLENOsKYPD(&e)-P> z^h^NShZM6usaV$5htUzg6D`z58ng7yZ_siZUY=%Ij-j>gV}@Br)DAfFkncb=O^P2r zn@p!z%>SQ?&!cL4?Kf(ORiMxMUgIK@H!I1-X7m4N^tC`6eWPXe{y(Cvz$@s$edX5e z^0bH1MdNSdgu`R;Uvc~*K~X_ySJQNx>qdfyNXzTTP~Y$7mo|6HkY@&>*GD1x4xg9> zFsI}wUSAH?x2<3Fyq$SZ`_-DqBP-l>)}vj*N- zwdxr7SB}JRn4utEm)-38-}$7xJxkJGF6B*9Kjtxq^m!o25ilU;=J9VHjrNJkD=?>X`ePWo9WIhPMmn&G>pz?k%p1r*B9^I>J&~e^C@qldmz(uUMhk zuaXwJ&g0SpwxTk{NcmC@5~r}B=n{`b&8vADl;lg#61@YteP3bH_S{H!xBtNPGdZ{J z$I_gH#IW?h)RIC`k))Wrs{v5b|2|x8Y=R@iazEE=8LbinT z>_56H6nouW25hpn)@C9bGu*e?W|)r4tCu~)zlc7UF@1z=qzfM$nImU~p$AJn|8%`S zA7-2sGaEO1hXXc{VorX#n|*TP9!Rbj4Gc%W7yCoAk895Vs$)rR)Un^}<~IHj%qIlt z^zQ5Az;ayZheLGh&o9h-E0ebPDAGK4FuvvVfYgtomzh=P1@1mppF*O)OXQoA9>GD} zCv1ptFuQ329)zrz333*LI5pRy^OPm!=kl7?s#KM)C0G!3PV_IE3;)HC=-*`c(WPGF zjshx&UP^7boD)*&G28{gJ-6L5$Bk~C5VWZI)|N5ZUkh-{>Lrsa>@Rk9%MVI>O4qgD zl&5-Vmfsmv*^qAzFU9+=QF3PxL{Ueei>kKU?`G1GGEVM4J)3zcxns5*lDB0PVn^>U zcT$t^hdI8Bsilxvd0YRJO&-=Kof*qpF%g?Iq# zh}$xnq5i=oC6^P&%e0=Ze7g>DKf-8<2jgBrJR9>qgawwXjr9#EE`4vNxVs?Ek%sG` z`mRsD3*A*-L@W+-?xZzEy_n~HTuOXl@wn^7+Zh)|7re5Kx$^b(x#jWzbe}9gI zqA^|f`Zp|o@3Z=(%U{n){VJ$}t|$7PtE5QpWAI?id7gDl)D-jK>^!eh!T6=x19|gt z4GGO<(e%6*2lm}uRwxK>|JdrtpJ)T&Hvi7Mh^S5C5uhrwDMPDOH}-*9odyi-@>yV1 z*^Uk?{6;MapWtW{>C0iCt2|8kSL7b`bI7&^hbr>|Z-K&MaIV@H6HC{P*Hi7PIiM$G z1-l+c0wsGRWd&xr_EfUPHrcwuUlcdWyLty)!aaGp4L%X?m+;8jwk*xG*1yu}AxQjl zbOB)mHYVkogODe)W=VPd%MW;rQH=oPe^Lj+$4w#uxK#;wwLju|*CC#lY3kVrWFut& zNZP3fcpkFl`?;n|9oC#@2>Tf5B7&MafCYBM1nWyzT)GFRdi$X~C{Ew?{?wB6_6H;Z z+7XS)ApyYwCS)9A*%h8}x@pn)ZQZ!NGwxq$1d}t3mP(?DdcP+YUu#lK=%#DQ&sUIM zIfmC!>AV1FPQw@4*lH}=cGH(@BRj&q7494sv^W#La(vtx=)bi9fCFLjmIEGuKLkRE ziBkR*ig9U7iy!?KU|2aEDpMn=BxrcN?04ERxiP@kcOGz}cOjj<^n(p^fUnB;Ah#s= zT0YS~fu0Fu&Z(Un#+U_W?*REYa>oFf8AZCjqXz(sWcU?P3n;UIc}>0!6xq)9KzsV= zacs?`(#l7gI>Zz=FUd6LG{^C6NQ?)QjBUWr2aJtNb^r{Le?LH_9s{0GuY&9#dj#1F zkiq$>!qirDR6ma8_VJa8Y-&6GkLP+DXJ^$J{7;Kb2M_EUTGDHvs>t#t{nr$Kt4)BM zjLbTOa;W!3`eYGCiu{SyqR(_U1b{$&EMwmBbxJGp@pnUh6wq}OY%k5j;gp654P_=%abNAx&q*y2G*$;OLGB-uy!zIC`n?Y*X6r%B|YfNwf|zYd_jV zo~==S>>Tj+Mt6BLKZriK$t2nHP=BF=tyX{W01x`19~e9Oa|~!F(k;(M@S#!q1r~sD zn&k!n2xq9CLHv2JxIwLKaa6kWXRMfkerr8L4z%6tndv>-{lhkAjLqkSTSg7=ZpEse zF{>$UH-9b!exS>@i0y@ih@h0>PtGmw~P`*9T6wEB}$TQIAj;3 zJOW^zx1YDd*|q`fl?B%QYo2|UFm^Y`!!1Lj7J~amN=e_Z#mya7UBg@|fnC7Kzq1|M z?VS?%#XQnB`W1PBIB7^)ZZoes%!F5L36!W%8_2H5h6f{;M^=N2GRMPTCAb4BR-c~q z>*nc83vtY`p0hUPiL=MGlZpQ*A6f+_BkM=rg)klQ3A|KnIF;*Fqap4P#oI?XNpw{R z0HH9wI}+#ive+~-POCKTW}b}{RZ%D=sGL8#`xy)X)_46!A|97l2*c5xq=9b$jP?OM zfd@m1Qa1N^6D+HNq8+s#^|@Q8!U+&Of1ZRD17H>6`ePMJ_Ori`_;RT?3@aI;GN{{w zeKXY7T0{_z8Q$}p&Fg{$&=f^Ua^0Wx^!ZEX#r~e^=g~Lywu5tw`3Z< z2s^fXk#6JNMxP_iQ;cv($`}#;u2w?SYvq}NqYw{MX z>w-MC5*)Ta7GmmttDn1~TF`+cRT&b0EkXE)Px}Lf`ri<@|A&EQdg%WFVoArpsNY{W zo}T$nhlcxU%MQ<04(LWx^QAcdDuFeN`t3skE<$Rhvy{g(BYMq2&tq71X7>4{Ytwy} zlCtL-{Ynn>fc9}T$E13865h1ABg-P-_w730=)z_rqt^dny8ow;H6th)@=eqa_ET;s zWd0#}1Bi6!p0tu?^!1Z$d%^0Tjz4EVReM75%$=Ge3WJ6ooQPx!o3*|#fN%sy6`>z@0=`NKkfGj6x#j3@DG`=2!v zeco9ivzMB{{-)YnK0)ybkz8|oxpJ`P_us=!5U|EgJW>Ln46Z_Iv%kMBd~Z0ZQyRlf zO4qaT+1WQ06*fK3grf!ERL?s58OYdIrnz9Ug;}zx7s7Ce%+$wf=j01cX+gWeXZ==! zD*;BFPyj+xZbGOPKb=FoEY{`hhD1aJ4LLXUZORtalWuCF;H%P7oXW@uJA2#*ps`fmtQnsr`K+$s6!XoggysRtzOwI* zBS{vz4!Py>A-2|Q)@$4A3TUzoqOr&xaI^H~l17=# z7KPzdIfYw25F+_NmvrWJKyO-FHpFAn%my3u3Vw4`2}g6n(R3;UFM%K@02PI#=o}S@ z*C7Vr?nxWlHddFtc4gB4-p1h8c=F5-o}N8A^_%}6OzL^|;HT%XeLUoB_&`TEX=Lbcj_%^#q; zT8=hNPt!{LP?z^%d{tL5OpcMvffu5t5QeWBl$2~B{hw^z(#|f+f(86%M_Y9i_ zX2IM`zF@mhd;d7jkn(}Xh>8~(Vp1>qgFRVp^jjz>;;ntjR(eHxO?pGRpM}HxEFmi_ z>dmX_pT^{wmZqMQmXhAzGTagqGyV=iH|Z-tFp+%g2+|Gz18RiV23p2_sj1X#xg#&b?2b)0h{Fpg~}g?ul`ut^*m1TM>!^Cv(Ft z4;Y;{e2Yb#jp}T$>kxDH#lpNjI=~Jy3;AzI5>5Zx+wK3~di$fB-rmDdJ5&7UgNB5* zs>dZfGZcsS@$z>*0!YTS<-O%MZdXBqg(F2*!(+_4XR~wCUpJmOu6kEsy$>jv2!ysRyy$y-F2}~hG-8Rr(U+ivBxc9@a zATW}8i1WQTDo5%-)VhX|gI)=mz@C|oOfR|l_jWpu9gKNrGL?0?Q_WK|6~~mWDL+p$n|uz4+sN!n1`OQP&RqGS0NBLE~Dw=n?8%iYwM>pI&$>0afyk_992 za7c!duuO3v54%l51I2!PST}}$k4sX=ByEBGYP)oz>ki&hS6)%m|qy8~0ryzN?ikA)dxU zJZ9^+~4y~fwyrwuZRbg5&rKR(B6hm{f-66_+v5V0f)SA})%JxG8l>;K<8&@`3jEr{W7i z*8GISY{1n9g5wOQ8ubb9CUo)wYSKUJcHk~UIlj}ZOig@>>e7+5H{b#Hnv-)Om0T-H zZ-;rxp35T*<_Z8-P!%I!r^U-;!te{%Eo+@rJnU5)mzF6mP2zQUPQ#!>*#oW}``|)1 zytCHUs{dqrIpkJqO)8`Qu#CZyfZPwE{#}rhCndg@DJ@@u1lw9_8}EXi3ZQ}AA40OA zV)u$1dFZrdKG{w-*c}{PkrJZ6WEr;X!;5PjmmdU&talWS5f3pDebBgBWvpd}GNf{5 z8pr$NURpoSpsHI}Zt%pc>zVA=@{!{a{PzfFm|CNM?AssL_`JORe>U-3{|_$Md%93a zpX+pb!_;vUV@Zm_{5DfX6LCO3+4x}VV*dMbs{eU{PsqlfrE5g7Cb9G-nsU2=Cj74pQFXY^COsH>RAJf|H5zH_SP3k zl3Z{usfa(h4MexZ8jMO3r^-mIz#;N(9@tl%+^A|FcTjK|=i9vX_7mr}OSU`$!i<1O z`LUwFNWQ)4!`~n@LiV}LiGaryRu6=l>4~eu+FGp%@M>HQ)Zs^Ts-+r8D4DxWZKWM| zLlQ)vK77W3k6hw_%?0=%bNh1CixI(-QbybGh3;Yz%fCI;5f4wO#IS10=#$*7Hnlnq z>>JnlFW8jyMQse5a5%(-I8-5X6C+JH^BE_VJa-O?08kA= z3X^*73M2>vsZBo$Q+a_eCU-LKsw2+UY+u{_O`l^lXdqT zwI@!58?(ki<{pu`vXY~!`DT0g9eh)H?Y_U7F`QCfzHaZWm}}ZYkBX6_3Mj8q)(j`Q zyHRY=%O+TH)m4AR0JdLDVh(8Hmg#Gz=Q4QUH+Nnz>*AO^HW73C3PN!g-MUP-y~PZ* zAVZd7$`6A`PTzf(kZ6T+hw18dC)RxG_aXn&I9|VX{;5Ctv(6xxEq|q8Ma&AH6K)Vx zJB!;ygD(HCm6K2h*v|{uZ&0-wyFl#`oEH_N}1rW%jVIfLy19Ae{JnW|h zlD_h_AulGB8a_P`0zrXZ)PW$v;f_b#4WN`Mwvl)WkuB~qoxoZLwoD$NQWe0e8VzZ) z$!3nTgp>nq6RVvq2(5Y`iod98BK|Ogtz_d^m@WeRJ3mXqLU;@S-B=hC{XR*CkAM%L zC|HNoFa0k&v2JDQdAyyYn$3j##;Z2z+Bx+{Nb-KLoSryf5bw}zA=|x-_rDiO#tg53 zX0${cg3F7wvp~7(v*^ibxVU4`ur(l6yPe_3+>?Fu^;&1PH_3>k<)Sp<_Wcd(-TZ65 z3lTZRYc(^Nm+B+x!u3ytSJ#a$T<5HKRyr|1Aun6>yztf*hsx%G$VdGRO|DmHdLRHh>8m@A`6#0>5xcI3G*G?~ zG&!jpC~Pd>Aa;)lcJwQXr;*Y^v-lAla@qYSZ{&|_Y2Gu>&{c9aFz+VH)CWNQ|F0<1 zb!_A_z7Zg%MeXK&6b~YuvHvr@1+WIo49e+|qx9W*ZJ>`M-owO=4W3?h9Td;(8*Y1@ zg`|vKx+0RU?FILYJ`$SXbe!>DYF_D~jt+OO=g3g<94h6TEz+hZ6XQ#vU{`!iv~$ED z<89`5h>_m4iw9(?GI#gE+p=YMG!*q5C$#6lhfw0Vrd*P77 ze7T$+@d+X@is36lq*v@K0+quAm9hyDGe>Tkj`V2qcS-9-?w7owU25{K(O~2TcQxV~ zlGODSen15VhPrSI3wbQ|9UYGJn^gq2ny?yE2ZUT=Z~g}Dt}(e-6nhcME*9>?He%ts z5gBG)6cL^o+X~XZ$gVv0zmg|!a-Q=&4M|?^nIgP1;3wJJOh3@j^@t+LaDK<^$}N+Irq{bi508r`(RKFp4J*ErVPpuC12m6h@dNr;Y6Ka)9_AY~F5u^>Z#;YUP zdI>J_QchXwEP&#eGr`f*xtufHb%>%29r|@VAH_}0zZg*;9@FXRw@SZwzP~{x60X;y zhiIhB0L(tJ2G14<3Hdq6&C2WmWSKNM{%@8EC{x6~)4)CjkP|oe%GtUD4fk%Kzv=w{ zE>qe5G76ROG0hsHL-pF2^T8-w!|Kr*;?$j~+vx|dhWN6jCAzk*r;YIZE~;pn=6shd zb~Gt*t~nzEulG2orz=Zja}dzQ7QZ9SUH6u&s!>08*RrE)vLCLael8hiJFv;?y28Y> zQU(%;-~9)EMTFjBQ2*u%oOmkZk+)t!IL7a^jOi`}n>+%}1fqT!7t`11ZRH%WL;Za} z6719s0p81~1p4{x6$eZHxsyjan8y&LM=;WV9^9iMKuNbOxaD?Q`MLVrJ?ewci5)&^N%!d%ZKi8+$#r(n{B z(YHEAH|Og*OV_WvY)@V7p8GUcPrmxM)*Zi<0rD*}=)N(&UWY8~i52USChNQ2q(r=+ z;9eIjMu|o0Lr`XEj(IZ8_A2aI$ptC*CTu*Kw^Hl5cvoYz%(XLDk z9?E1Z^e&f-!#KYuGkX2JD-v)}sEX^|=pFm31kt^Fw_`7qldo$+eDfO`{h_OV1+6>3 z>UMEq#qxgddNV3GWl~fE*{s>N$i7X-p%TLmvcMhU(k~xvrj=sJ=Mq#Q4C$5)b9~M` zG?5_6q8upNrdz!<#T6g3#u5);w?@~>iz_izTMiFG{T&<7Jq)lP~zy%0~2oEx?@0 zjQ|db@Z^&527t?_q{6Fefc!d#o_A@bLeOVv#bWY@gR|X{h>L6As4uXM{ik@+ke$CM zG1TU(`>xf*Rsq<@Q#&72aF;~EN^I7j0XdAx;vCplReAdiN&KLqXT`L*F?NDCQF8#g zkCC`#^KY_b!UL;7Uy`GqH0q>YH``*Jb(vaG4zB-tlPI;?A?qzNAo8oT&FDAI&P->@ z!L@jJdFHV4_29L4$wlwX*NY{d?(0W`!`F*4+qN_u)(W$XuTP%S=v^O9Jiof2y|-~* zLr&I~AJ$eD#49WB2n#T(&k5UkiZG=^(LB&m1+MHU2E7kdbqm0m0QKu)N3OAkz}Y?I z-gAd)R~SBiJ|U8!I3d1#+l2C!7^_|OsMqW)!6O|EzRLE`UlB=DF*CaXai$OpTu$`E zG0)~edSg`0H0GBE4}9A(IR*bG9}9a^Go-XG9?`AOOms)nVrE?OG`V>`eD~yyw(?A* zj%@i;1VP{Jz7L0wxot7gILh!A&fld>nn9y)SDSWy_DEa|qJJ~tY)#Lti-piL2Hf_0 z@2gx%ez_u6eGs?OH^a(k?PkIlL0sC>z7w?nKg*RQxxf)(;cWVW!O@mPB_}DTRLw{- z0-nH^LL2#Jr-ps~u~4=~K3oV6yLLoC`tOw56LaQ*&>mK12OPxa3`#FJszACtqxmo zim?V4z;&C_>YD87d)#)&i^)+4TRTrF#Y>ehovghWwtl*@&vJD$5v~pCd@l@D6J*x` zkYH;}8_zVG8(;##*fMjIa&{L}krqR-GHa}AK*{kKUgY6iRlDsZkFB>~v3?7RoO4~2m-kXNZkJ_1m z6cC(vWcTnZh`L}`IHe*PSyWx;+3M*PZPH` z(hB}bqh(91O~O~Yo*c&qVwA{!PZSS+wqzUW`rfjtAvw}~YW3L^pA##4nM!n*R7uMB z_%K}Sv{kCRH}rUbS%kkW=3|DJk9(Z1O@5CtYrkl6cH&@?dvq)}Hle-A8zz@}4Ub#U z)Np&JcL}~v6dCexfJ?g!)k*qtJcYP4>FCRgrHXm!eG#8RT^}Ha zLg8+s)Q)%yPEBLg1znit+y3=@z`tL9SkJH74E$(j@W@!JqZ!r=&cJ(|pk(CPEuM(c0dc(%n>n^KAj>Ji?{nir+4Q zNQL+0n2n{qovZLIP2zwxrc5IiSCK?6nLd?$(yXQMBc5d{gQ%DyCWv)Z5ylxBVE);V zV!jd!)GZk^l6V?1L?2087M;0H^ePIkyI+}5oWiXs04MNO%I6PBE%(u{{<_qS@qiN$ zeptv2ALZ{DL9Sb+Mta5~NqT^F5<$~4d8I+orQzdLs8*b%V<6Om77qsJRzHqQBT|IR z#f)oLwlG`O$*OL~6cRHGU2Ti$wPu#yjd~gMN%u+i5-T+lBT~1u+gM9FIoTV8Dw}u8Uud~ZlRVn8P3yx!MB0` z3w#p=z&DMM|4oq37qpLVvimca*064I-pr-){vz4ze-R;I6~zsr#s$8Q;{( z8U_xH*WCDVZgk(P%%TN2!_^)#51n!_O8cVxhPe7Ue{6`Cu?My*e#lCpjU~fuB>F(L z6h2tdpYA`;%b@D3T-+Y+-()Jy7MzkxR7~t08M8IGF~dqdeIQ_#{B!sVn`3{px4|}+ zF!?q|q{MX6_REurt;%VijVpSRd1n201GhylC3e@3Cv|XtzsugGEY6rYdGwQ~G0l`} zd_&CRaT(TP_5^6H?qQ3aK~-N~aH}R1Y?2to6r8y^QdElnu5=R@Q09YKzwn6!aB54H zR5UMoGCrx@ZgKBB$Tek8kwH;A>1wh23Y{Nx&;sWEKWk*E9(InQf@gi>)Ph_6HQLhl z`?6Y@xY`B1JDjR&t64bnwKG<89$>Ogx#)zWkExr&Ww1cdNC(ZSsj9YnJ&VbUk!~KG5Q|*vfhgPqxex{U3s&lH)nW1Y44Tg z@+YUMllrZ{7P+QC%g%e4Sa{k(=C@*48*4MvZm)~~-El<8@xW}z<}8T9>OF9!uJ8QP zdTrCsnI3eQ_gR4W>k{v+oDD^jUw*&XeRFN7n90zyT@{W$AND{s=RC4((Oc}=*x^@h zAsdkYvH*mdM#3AVHT=h@HrLA^)MeU}A3=7LJovLMG_s#kupMRdSiFccA>>>F%BrTT zC3n|-oB|zo<>@{$N_;KzdQn7*s|Lbm;8cgBgZ1HFnTmXr4M_r^IPdRsFrUcFu;hf~ zZfM`?KNZvXCD|`Wi3t3yGjXfgUl-kUujrNr(u?8Or1N~;Gg9IlL`{e`M)B$cn=tQ+ zxpZ0vk(=kqfg4a^d920D81=hsReLVmwCQ~xx!3nN*mGh6+o9={8>xvQ9{AbyLk0IBrIv0GPw_2e>)*W zg4i=hHF~?XUmgL;D(np6!v;l|a!VH4+6)tqK5`jg` z@97__9niTP>B#?F29aU~Bs0%Dy}5tv*Y{U{eJw~dU_w`Cb4q?;6k+>&^2&L_heUoa zQoqW3d=pR3m<_X1ikBV(d&G18iGpeEfPil;Ir{jzDg`P=SprK-a9*Ir2%Z^oe%lne;$i*j^?XbfEkD z-uXqMZ#`lxR&r;xsp5%)62M<+1rH7aoG!XK6?({+-ebPMdO zKJ}hvf|s(v9pe+CPjh=RHoit%GH|5iYC=vTy0|o>e= z&voSeA1|@CC-C6otvx7vgO9gby)L2FlfQBe zx_j92<;iXwi?UPLB!PhJ6Bas*Q#kja51Aj!phC4*0IjRz0eP{f1tx^T%JI!{>**|t zhY5Z|X}>z%9;1nae8gF4&>20T{>=_968~N56AqdGMFyM*oZ_y17zdvpxJT^54g>GT z`V7o3Bmwz|jRuJ~-)GN>g^mypZ_$5w!2+X?7oPu<)XWtc9E+?}d*>9UiEN5mVRZ8c zthu_lSDp8SKk=v*<}ev_yf1FzJwklt(m7jGeb$@OA;X$(<(+-~Dfo?5UpC9k7wVYf zTAEztJLU%rTf6ojtNKGGa@DvmOq?hzW8>Rgt2MYsM3 z;X1C9-Rx_7T`A>wsV*l#UfY8-=Y#Du4J$$!`rd z_bLw7ZP0mqeD)RuV+GH=rBFo$5;@Jw-EX$MF1!ydBVCTLIa7d;!GP>QeU_%As*!wj9`GP9r49K*mZX5AD^BOiYeR@StTQaG(2zc`PpSS- zZhbczz}V3GWfSpn3#Sd`s241bz(~82s5|%i*AuI!G;b4aK0R;isjA8~%=Ig7GAMY~ z%MjCp^DE5;{=9Pdxs$Bg+jt909PM$8r}r&O{bF*gG5HCfLV$dHK{d0gFA~hd+cy`? z-R@49UGdqeJJ!cHE~~ipn5`<0Nr|O)^joi~sp4_}7+5*s?y{q4~Qg=Z^RT zTt~e*^SFP1BjnzlI^bZ6oj+J<{VWi#tA z*HC>%`#J*T7&rV|)n6g=w&z9-7lKK?m`ZZFTA^^iGajNBU&|M3)=MUU)e_6N$*z}Z zs~f^DOD69Lc&lmQkZ0d)Q;TTskCV%r=&Nas>L$cTX+o|v^~IDRq|6W>W`(wmivFha zEM#Jm=OZOZL{Cl-7(GIfodc5c)_QDhFr3LyDxMkq1yD`}x8lng+rP>YmTa?3dNEIE znGFq$&aNp`QD{@zqWILq%_-vGLUPKgi3rr{fZEwal)fDW)bxff3>L(9LzKbl24(L} z6NH%Cd1UJdzVU;g8oI$tw~uQ)$)C75Q-Pq6QSSEDH(=N8<@6C%z$5LFZq9zE5?u7g z$EU7YdWM>2v?@hngv0iq;?RQybE^>6P5Wa6nU426R+0(uv41otR=SU?a%7&LCp8=| zZb}>qPtX->zEsC402h{^%pY(;O$Ibe#$GUcfZ6d?e{`-nd+W8P=8^EG1!8kxn)vqP zZakcd?Sk-VbwO8+X)bM1k?9e=!#ZGJM}EY9h0~FU%HZ`+OBg8M_wmbO{k*Msq|1(w zOHtbJo3QFX{3RetJ;YPD=4Cifsl1;mifIS9lg-IJcG7lU{?-A{1Jb%9(vli@UsB8Q z(UKERiMzyr;!D*Kx|8m`&gT6YNCh4pX)L`kwp-8*zKAnc%|ZJ1bc(?u9}`==dL-3x zY0yh9Yv{Aa(8#`BNYuCEVbcol+M!M3I{gx4QJYPZ*=~}IS0!r;`<+D6qUe~C zT8MPi9m+~GZ5jH_0!GK-?Q`e9Ek6M3X(qFtotI*CbMLoUAZ)|u_(2bJn|0>0@R-3w zAQ2@yzubgytbRICjyvL~!Q`7>e*SSQQU^FJ?g#~cs1?2vyKQVCUW@uCla@B~m3sL}kD4H41KFLg0 zWVP)PN5vsiuulMGfvjPbLmt0}7;Ar(E$I?B`Bd_} z4g|7PLnUze>|NQY`}~MPR9Fr_%4fWya2`WEK4UK8yYO{KA2ujwHki# zKXpr;br%IwYR;e0h!-uP-iT1vi`x=_oeC{1{j|*)*m#QUJ(2E zCP(Y|7A@o6!)1fKy^;^M)WLU~`CJe*K)Uwxv3VNuP>K;*WoJ?#eK)c^zx z`CU1*c~9pKXGuQQNxShDp|(+>+?;CBMLF9`Dao|A=^j-I!q3}?uj_h`EQ?DYKKp5t z*MLc?Q)&sB#WV>S;u-Nf zGsMVG2U-@_*=kjvBXu4;dbM#P#-?Y{A8w4qm>ttupD9-U$YZ@!ekGZFn@auiOtHbP z?@w8!h?3D03Ao5Lt(r+%!H0;;SYV@*lEXe(WskD!D^v-omuB(B*-*XVX^%|T&Rs`# zf#fNfVNo|hDtZbU_gKpDtUa3bdq>;XR@yc`M#z$^3()x_hn;W>l*itY}Hr z1Ty4g`MJ_u?2A9cneE$K{M%!bUdlk)sE9w~o&j*G#sR-bfUg8R;rQ%{NRw5e%aHps zvk$@H<)4@|2h&aP>w1t76Vd`f?GpkZT~@vl^ihrn^RwAfY;QPS6)1F-=fd|>JBW07 zfu(n>=PnxID-C+g6n#?ml)4TY%)A5J)uZbCj8$~ zG7iL3@8nY}tu@ggO#05^i1&@zpzi2xw;ncd_W(QRWGUV>5Ygs#a2e)D$XVcPng7&m ze>LGmv>SimH2_B5QRnU@y9L0KQkMfVZ_i%R^)cRFbsb93{cIDjUZossL(YQk69+S~ zKwhNRFBb1>j56w26ciDjFHGZBc<#nQe^<^HMd~y!MmKfN>G`ZY5j4`1J4^e$HsFdt z67m%&YjdlmT`}*i16N4*9({`Xm4LaIEcw$Wq8%cq`D3rGz`wKS`rh`CM#^6%R{)zf zE5Y8_8^Ud9oK7d9Qd!UVJNMEf>I}p9>WAzR8}h-IL|LP48$cxo&UGcme$6$;PF@r) z394AD+J7>uI;<7T0pl$q?t)lCgR$z%uMdWSNXqqf@wmyjA5?2G+C9ZnIZgWm`-%_!w2TNrj+ z0*?!|f*aJQ$QQSVLQe@g<;ProlMhN1K=B@sgL_!;RpY$X!Kx10Y>RI&8ujPrWAm$t z;s8r5DKB}wZmqwRAq;*+gu?e4q#|a#N;IqrCzUIJTR=Jel&k%*Udtp_hL;lXz-h&R z^S)I;^R%9zmo4z6;XFqu970``-~*U@M+Z`sdoI|-d47EKeerQ+EpN_}=?N1G^ByBA~yHv8Zg$>%XV z+XMY)tlJCr_i9g4z4wOuFI~b@4{xiW8&Z9}#P_jC9?!kC=&12|uL`1ansW6~&Pb0! zo=iXA7EDvbE<(CXpb4>mwa6GV3?)$Pt-C%dPSlx8babs_pr~!%({SbX``qTyHvbK` zL3HqM#U&vL*y@Hh79`%yIT{U^P8mt_y_jU(XjY!Fztd2o*t>u1Pr&%)g2foMTZP$8 zhFa8Ym5}I~c&;!0l|lVe86UR<(fsicA0@I@f9p2-AVdJ3;7JC zvSZCKgp2ST?R?-3lfE6cCHfh!lG%MWF?SmOGMuEtY@m;5Hko(2Pqt7y%!=wvJT!#1 zh=xZ4ai@XcIyIS4RVL2Ef->u99#W(d>D*q>I#eC5$&!SF*2c+3IBtISMe64R4|r2T zyUj^G_v_=Ac4Y)EjHLdJs}PzrO7^CN{slJffx~_Gam-_A&fkaGx`3zD)RVXHw$)b` ztJzT`P+V%FkQo-nR&1^JT4oYNT-Kw9X$t?&)?ww^fL;v1Q_~7se(|CdB%ir2&s_`g z41H!fn1lg`n)88H9V#||(%wz9rrU^wQ@2CH!(Dd3+EgiB5Y{lY>A8*X>k-ip3Uuf0 z%ltRHiN@gxnD5fMuF2=qG+V7hC~zu++U-kJ4VzSIJh#z6DzjZIQqAbiSr}&(5SNMV zdlp8s)Oz12U3v;7ce0>1byPM{I2-ML$$W9M8AmF#K2xd=vGmrYS`i62(MSNf#tV0E zJH%SZ6~sMC*!)ckLA<6(P5ESS;yyb+!#GI=+~i7RiOlm`vHo2&tJAcNC~)CJgqd+>8c3 zGaU8pqhh!WpirLMYv5Nq(&_w241FbGr7C7iD zUfFu4s(Sh4{HYLD1G#gM7aABYnW^FwwaL4v9W#Lp^#oEiq+=-LIU<1X^}?VzRzq}Y zsjyt8>^#z8M(^sF+lKdry{6 z_+I9=3EOM>mzNg8?|2~gNK*x2szKtmS&{dPOUXNoQrz?R601>|1!hx4sNG-97tuFDm#TfVk9)Ym^&kM$*a-TIoX zv{_(?CW&|6D(eHl`YhMj!~R*wK1yN|(ekJ@gSq!gS%EnetwtSiGJEFB{R2SG^{;om zxX0d{y*4spfbDwK9K2anT>X|-bzx?7wvDjD6uFT`})t;>chO-!4 zr#ok;1g`U`YR8r_EGh_hYDpTm%6^ttf}t^UkP=u!2_wZhE(rrh1w*g-Vw&a;+f{yy z6BIwK)}42Le=GDhsgP$m+JZ)Z@O()zfHrf)Tpme#lU#fkGXxi-$u@+I`}$SOro(oX zPo2VTzc902!zn!)$4ej~fw#`L>7CEgqvMLSIIf6DDs!Gk7M{@=S8PS2h=Lv?Q%+Bx zjwa9@9-z&_Xq!VeR`#O5RM^lq#%&P)4|(E^P3hR|)EL5$+@bLEkZcxr0mJo?wBYc$X#}q^R@oX$MFyy9^Kw zt)H;lLhLe!#&t4~PnJfJ-UkXFQ{5~g<4Po8+*kHL%Cz<6W!>)u&1=M=yP6Ppv}XF+ z*DIC@t8#(YF;yGe!-v26X!n0kBhYUZihtE?Z$&8%YW<4CgVBAs?Sk_;g`t82*1a`O zfs9YVE(~|WuSDMQmRVh3(UGkUlm~PMw-OMq?h|t-d738JVlLcTLflh=V!TLW0$H|K z!Kq%ix(FAP^Ih%&yX2?nh4m{-7y@RLa~b$5DHi#xg<0=kGJ+@{UghUgKHKjsyS2Zc zx}DIiGz%Fv*|_IQT}}Wu#k@2E%!)+%rHu_QRhKuJhyYW?Jx&y*oG}HUS48~upae4p zSb|aOgsjJYdI<>gi?IT*k0T$cA$+5DZz%u|(!IuV`a9m)z4U_O)B3p=N?xIhg)A17 z%&Lnq$Q39P4j@Oi&=#j17cMEJQC{MUe@hl8lpA>8>92$6P`7;^^h)=qi zvmJlntr`fu>AV`1mb%ovUy@(9W?W#Li+haK_S*!Uq_3+ekj#$tHCgA#`8%28v&wpc z3Is(gyPg}elm@coayh%5rRwqi^WwJfdf33DOm97CX%&_1!K}sb%j15#n9`e*a2Ipn zy-W&l4?@G+asoOQxfwPKkvEUhU=y6Bmmqc?4Gk*ShEVTMeHaRVKBoy zdDZAkP#CU!m#@z!4pGlTp}%5={bS*fuWrD;2L#lHf_IT{as<2(S{CmG`9_3;RQ-S# zfUND>7oL8f4R2;lxX%eolS)YxTSQ#Z5u~cdd@;yo^9kA-ygD^5H5nFM6}+ize~mwi zI2h}{Iz+Zze+~^)X%r=h$zt1I8t((QGy_VQLyBjFJvZJEMHFYsQ z=`-Mg!Y@d$z#}3PUBgC}eMPX##Ob0pco<=bddlhgK|)^@5M=yHITyr^c{OZ}QdekY z`l#nG6I3;a%c-S+^$7jO?@5IfU+Uf?}~#+MJN zneL2w^~6y}hJx`l&wRyxhIZz0*-BPAr_tcWWy^m#kH3h0zZ4C*FDX7|2F^re z6{=wKw6j1mjGNwi2=lWS)u9m{cfpKgM6pAVGIcO%SAUO2?S6Hj6JsE>Caxahwea+a zwRn$$uu2%6f^8&UtvOm}Ue?n3&_PQ%;)!2g_D1`D36DRBlt4NQz=>a+G7b$z* zB~dSy_wJtE7h(>Z82I|!W}`C}_9Mt=+vyP^(T~f=p#Kz>={Nz^0O0&mXPK|sj!G$s zS*EXzTiW9hRe;jj3K-aF^}t1;YQdk^8L8O=RMFgcdSR4BGj!RbaMAH{=|F8rCJA{S zBw;0tLx)R1_gNaK@SNN=<@v%m3d!-cqJO4D8A`*XWAIuDO2e#E`0}Ajd3$@3{6`Z% z;8wbm&tB+tQJK;ksjnIXCLnMU4s*7cyM ziOYv?L5TuLmzd*_WD&AFb3*NT=3CYTSrC*yI}6~g)-lX+T1_Z29YH=Sy2ZMjwc34I zi??6!Jy5s^g*+%k2T@=qhP%=L&32F1=NRek1m3IE#HpTkRZ zBOqg@4Drh4jc83{s3#6YtBvWmc>3gP>58evT-kEt=zRu$Qosp&)hxZM@V##I6LAB5 zE~wZt`?2?ZnJ!4Wwre-e0gb_Kdmv%iHKE!}Em^<7s-IW!{aL>qlf%sMP_;V45#sZ~ zqa*uN*9#}<-e0~+b zdeWKb?dK=sCZm_@&92v8B-fIXN$D>K3Rlq18)rM6q^v&y`9!qi9m96wdkM6Ph5io- zjec(KF`SVI^|ThvhyFz?=$>PTvbJn3Drf$j8&qzyA;uF_1fEjKJ6)xF3StYbM0Z&6 z?|e25+QiM#_NX=gp|)hAV~nBa^3-ADJe}X6!}7a|torb(KOfD=>(PI}RRBKgkLvQy zR?zORcMjg2zY2=il{J*n>hjzWy##X$<0d&C&20(cGQ{^U_2>I&?RQxaDyoO*yX4B{ zhvqbQmv{&h*|Ow6A!-YVvhRAgdhn^PB@}7Yp=SBpU*?tXVs>+DmTP7_g>FVfxVA^^ zWIMsuUq&9Y7cpx1{k=^?4TK3CJx#z5E_w3bESV=fKe|HIjbPSS9x|GmrHx_7f6jj8 zr%gEw-xCS2X%D40q(I3d_H(!$D71f|NDeTo5Cs`y1W?T~coR!(IF-*%swXuZtquaE1~klrCurnV~yG8Wibn zkQx{Sq??(ch6X7SrF)PRl#oVRdR=(m&-;Auw$|FV@7Mf=ne)2NH z1FRcb{i20AzPTL0@<@YPfD$HR7@MwR`uT!*+Au@OJdmU5z6|FgEBwWe&qkpx!?9j) z;gZsE^=c(nFhgX*|%@{o#LCF@W#Q0<)&WqdZfJkW@mMmhN2`GZ6>i*}R) z=eEldRhx!9qz082YMm{q(AJwDZsqR8v>}7WG6RnF*)%IohZ6iU=8={7&%HoFWRP5h zbeRcu&@s&$D%QXxUusXTTF(sJ(m-}_KZ77NALzxEvzIo22!S$I0wbXgH(3F(ey!}C z@H;1_dr%WB*wS4I0}lpOC!@}(!cZKm9Ze+SRdx)A$jeOwuMd~UuezaAEhW8L`#S+T zNhp(h+`2+h&voBevS$T;)VbMq82(eQR_|O1N`w%^@B}F^O_Gh&jDpZQ6?*JQ2 z2o~52&yrfa`x&_~ZonToWRA01dBeAY6h5ZFawer+8G?=p?REh5o8hD`y*3wBpTF1F z<}^!Q{DesX|85hsvO{oFoxu9j%hk4T=KC4`t>lFfI{~RzKB6uZ?<2!yj)yM9daiCl zz}-s4Dhg3xWMEiQit-mhm#~ni<>CxC3q)^$r9S<`_CR;mk$3M#B9d_eN#|pe9iP#G z6+8S+Tvi`Jvxm$#Sm_-J?+H`)f2|40ga+>`Vs9Z2OAHI=at>%hp3s|4-pq4?ri`VA zoAFNWlp_?LC1krXaag%lCC8c_ndlrAH$k0`3fCWr{9Feg1?^k8H`4Ygd#-!dEMXdJcx$U)S`71L$ z+LxK{pS+gl<5L|+kxUdfzgl@a&Vnblp^L7B5+V&B$jMe}`vEvf>w5xNkL?eA`lI(>HJ2&#reR*v{A{ zavw+m;y&AwLhwKkXQGnlVcgKokeEU;tnqNr6uTgzSJpStTfMw~Rd+;A(-eOv6a0nW zILzix-ple|XOGv@3wASRy1&iVJe4rko#23o)ZEhfh2w7Y>$^GAnj*KpjZT%;ao@%7 ztHq&C$$#-yn`;O5!!ub!uZ#=L3U61|ljgEn@vw=l=%#~kqryV`((%)1pkA3xhb$5r zE5zc6n+|Rz?$q{%mG1@)ccMRXY{e+GHRqMtV z(_6!gV1ARozByIXuk3RA>E#98%UZRWCE?@Ur;8tS_2TMO$63s^_doxa9#OicN2cld z0iPcGMfK$VK=#?F=OI#p_G9NwfFF7LXElrbCZFE-{fO)zAu@DyjqCKks{zsaB84ar zRhjPfI5?O&^rcUf7o=*Pfb7eB%hKpR1x*TXJ2|sr$`Lcf-DMKjHV9VX0dTLH8yGRY{HK3@l`-E7`5Zp{rkY}KKwMIa0=%@yLG?qBD zzV4yhslSTbF7K+~?oIM}6a!w_17{7O4!(q~uLF8BT3w__$IBp|ijf;plx6A8lh**Z zO9D~*TLWM8UTQnpfcj16c$!6b8PA#14s(S(dg|K>WwKS#;U3B$9WW_bRNdy+yxA`q zV_~5nT>|`XZiKS-ba@vtN9YZmiq>wi!|hejI??`$MBTq`8-Z8&@)=`1RK_31dYk9w z8xE3Qlo1Fhlu(_#myFRt3t-KO4^o5Ih&a2<%ss=n1D*DdkmP z{Q`VnfSXyY6bJlMTW?mzn1}j?iC`M!?=xQ!ZIT2nKChgnW)85kz^;3{{A%$RMZ#)- zME~A=4gKxn$H!CD=kiaxqDE{!5?^hEnmjo8^Igzi#Z*0U(@Urdhzdqo@*?ysR|&Jd=# zNr4ltK%$t+2rvafIBP2nR4WwP?>%_qp^WVt64uTmKg5@f5BtTzcDIFOY7?YH?m=7p zW4rb+%(Lzl)lbre44G@&{%!?yxQ=X<%wQOYnvp|{RUzS3QgEsp&H(_Y>B}5K$wT-E zQJ;lCt)WT;5A}RXi)Dds*Ru1c+O6aE8K0@(E?wR#-||h9&}@=&PZiZM-m69dbuf%u zd5ZPstjG2pCD@P*#_iXiBgxkGYcZkI~OSatm_;?i|Z*cMp=7d(DdEL{Kx zv9vg*WU_>A7@r>Ab%O#CX1e9;Da0EJOes%lcn4nZQ-Vj$q5q7l6BB{uKm|G8(^|RI zy0wtA3tr!{|Ih%yGr{vOr&IU;!RgQoi~WbwsR?d(<{KY{@lyQ=^W^#rY{M z5Bc7Sv!vVN_b6h|lwSE1C>#bWANW7x4}EO|L=h^%P-7@%_x+&P#U`J!Sd2@ZdfqSW zzWM->^oMW#+YaU_jDMus1=tikDvkrHV$F(_Uz!VQBbG;xx6i%s-{S*N?G@X2!29$pYo_T3Y=x`_9y3eMso^Ms# z3XG=d*x5iU#GHA}+TGk28kluyOx~D4o)sq?OZDr_f8N?|w4(q7_QdXX`VBlZ{JWgE z=up}kwx|@Ce~6RBJu@d0MTC}d>k|7>wqfs&6i8MVl+-wkqijK*6&4>a?GC{PT-u>U zlx(92=1vDRj!^5A$CIVfX$FCpHIK;haB-m>4Ys=p4~GO+5z2RI($K`e1y&>Mm%*Qr zC(v1(O2Yo3JflOc;%U)pK@Q!E+s8*w_dzzba_Ldw5w&6r0uKLiK=4rHkoW@{t(m_X zt$np^q}ttkmWiHGgbx_{ z$fY!08<{dR+2hWBU3s5XnmZj|=RYal2`UQtvlVcBU_GKflXaf+o23)>!wyYCjXCNK z&ZQ>s`YOmxS}4M8dBXeS{b$KiW2j8T=2V1U)!X3pyIR|P7EI3mb*+oJLNYVtX3Kxu zv&@LZ!`s7#IJe7~kCw&OL6&`$b+FQbya=5rYG}9C9}rp z=*AB7tFyiQGi+etKkx{!X12S^1x9=AHb^8%3=I<16q?i~DJ-;jw6^UJZ2fo#^bTz> zo6O5ai3KbzSNQpP&FXK=-CpBm2>Y;k0IXTPH@dFpRa=~(MZhK~kKUUza9~Th;iyyF zdk3Y7e(Ii2`}x;I$mDVdmjX>370MrA5_I*4HwdmktU>IGNQPsvAmYQ9%uYH1uwNe! zJGV`2-CZ1rx&b5Y*J?B^-zzM08|3|EPDR5Nr%)d zG&sifmOim;4HY!j5%Z}xZ{}9abFIOI3k2CFUNsOqJ9Z0DA`Laz;goKTNJllReYBzA zrhkXgOR$=K@AwsXLoR5t3W(hMDB&$9Ni)z#Dnm|?hcHL6yaCC$9d>q27YCW6P3|P) zg>dlsSl6e=>fcX5v6Sc0=*_V-|48Q1O`}OPzS|OB_U}pClmaB*#?5?|+i#2Olwi?H zk`GE6v0#cra22v(a21LMgM}#3BzD4&ktKk(1eb8@PF3_r(sc>9W21NXoZ{S{-0bA# zfJDGS%`ewVX-?2FyH7k&sO^>-(v{*>`+`BgKUyY-Hv8*g&F_0r9=f=vo|4Lo?&~s# z7rt;pKbc6_hfQxCWjFq49|J#PqQAu5h&+&6>7q(A)8qr>_1>Dg;H`IbLu3{4xtfzWu^aXRu4~w3ja3m81D3R}1v>X1vn}+8>33 z8tVz(P0==*B+8Y`8ouBKWwWFoegaE@;2WQ*+<-nYhIMGola$Qn7J(Ea>#cVysx|)* z4&m)p(hG(a872<-jE*!xUw{6p+K3Q6E(BUToB4h}kOIKCq;Bv?0rYBu$}UGLyu#T4hb{i?W{fZzVF)H^MV(m?d3W zZ1el`{8PH-ed@ipB6gbRzf%3qe$$NIxhLX=_-fggeHgJEuvck#ISqJL&q6#d)vuh- zXACd+W#B5rK?R`-M5gchIh-i@l<7Uq0(6ElBI-b{acpIy++{!~JIrg^lCM*Itw|KX zHuDZkpELbh$ap#CDwQ2|Hpi3Y=5@fbqj<`+{o8ML#ivo|Oq+Y=nV;$D_mAsz^ZCJ? zrzRK2VxDmwkqQ^a&R)VZ|MWr95|Pfc%SW~~#tcaumi{u0{C*_Yk)NHwn^PwwjiwV) zuF-4hbtQ&_id3{rqFu0IYPE@tVBTOikf1{&rB3l^)B4!~kA96>g{%scu&k)C&%fPE zlXBt=sH?kH)Z<)g`M(6x=ub9lBWUcW=nL}$o>_t#m^B1h^e=uER4B}GproZrx7h^N zI@1Ou>9qX~-6(aex@VB@k0KyyH24$(b|2o3ppw)Li|l4_IiR|cVgR`JKC4vmT4Pze30``R>t z^GUfpLxjPu?Ywyu*Kfaw!~kytYgmgE`s^ZPn)zR4m&fJFVseC+TiftOB_Y7^7!o-v z9yp`=J9ib~;de499;n~<*3&%%eOG|ujA0nCjwk4GTdqLUg4$LA6kh$`!&l_%pgm)>S z{9?pymJamZwbfv_%e>WScf&7AanI^5(UU$>jD4kX zXr_Gv%KylSso#<}QGokuA>Y4d&;N3|S+_61|FC7 z+aVI;dS8mQb+?XNr?dU>dx|QRkWcSU1Rn|T#7Ntqjrw@Hqn^FK0Z^L0cgFGV@aPfi z>pn6bN5P?+IGx~TTIk6~A{fNUYp>ps^Is4d@Tp$wfNGgv7;-@7d|o)m!qJ~IwKI#Z zN*d)+$oHRxC8|_x#05|#zw**`fXZ+;sM38xB+TFMP7jiM5f=`9P>z5|jM&9y#IfW$aMih$A0XWD`jHIJfq8sfxs_`HCU} zS-DP>UX&i_3kHu=vzR2hwHs!L#(Av-!w|{`F~^ClLtkj6saukX#;{%4X&wW;Kz@XW z$k;~^1Uy~7+>Yu)`D2kY+bL3B@o7u@e>XMYqmmbZ^=o-~BAi|t0!y)kEh9cEv%JxN z|MYorjavq3P}Ci?L8BJry5lLFXmhCK>bKh+W6CN!QI>6D5j;0642yh0;VwU!Xon54 zU|%P^ST$`K3)qU6?7s4v;19_DeH6tBl~`l+YS6l@m1>>7dMVYEcy-9gOj|*;3^h7vup!9i7 z50hp{7l!aal`xEbilOqegtCcBmJw$jC=@x$N}!!<0=1T@o%lt@7y;d-fP?T_e^nB_ z`Ihkl$lBD=uhA8ruOj>H^rf@zFYrDof`9Z#X^yZ5_`?l5;BZ6{YftpFDhK3&o{$v; z_k&E}3PK4sZV~xNdqBZFIoE^}4ni=Jz~Yn9lOXnclo1cUvgosNQVz+Wqm3D?``Pr} zje>mnRA9>vH9;7xPBD>~GNu2@8_Gy4C~BenZ0Q{e!Z(|qlj{S!GOyXm@k4T?9cIK! zpyFo!Aw$jOq>lm}=aNcwYr{4#Q<|^Qa|685ak?XOg*CpLcF@UFtg;Fnt~VBZI)J%+ z5rdceathzu0-D(&Zx2W-?#8T{(wYG7Z-_6R#1ABQHRirr<@X*w)_XwA z*Voq-aX=tYjFJ24&pwUN5@6*({?E$c3t2*5YtFtJe~Y_@Z>w2dbgnrXGA^=Y{;JwC zJLs=qW*6tp8@lChuO`mERo`qeIT{CgTtdMi z93D8kvFQ$o7&W09O7wA$tU$%5=PjnuFrg6u!I53I3$;b8=mp7N6yS~6*+|#G9dmH+ ze-};&?0FRlGsL3h^Tv+LcKjY|3jQ(H3tU>dwA$(+3o>RGFvvC|JDV=QyL4oF}!?kOGL8I25K z1Wg3iqjrro2e#4%Gk?dWA2McJxiuROPahB|b=XOTVA=G`KF(akD*uGQu_e)nUZ+i` zAJ4r&5WCDEI7oGhAX;yk7gx?Ts>L`*33k994jeswGG&+^p)d#J4f>~OY$Du@tXX=1 zzz(Ke2iRcNEomn!Z1TgbBxnC{#D8bxcqu+2_t6K5ZS0*0T(i3#e!2}h0H#AOCB}|nYoIfX$CbAHe{lwypv6TwIJc`$_jy( zdq0)^j*`x2|7+%`$36T1vU0?WwSN1n4b#Wp28o|#u>Phd8kUl98@tm3y*1hWgm9$r z?PHy?(-KCz_6xCqhRgczr&`P!oX4ijja-uWBBUeh?}D)4EN{flryCGkX%bu!1RtS|O+;yte&@Y2r4 zfF-lq6E6>Y*Us|vl0gXK2i#zX5*ug2a4hQ|*GXPF+^_gSGL?t5!c4!;We+;+wG4d_`oiY&6s%DwbNGVdeF6qwXX^r%=t09x+R*9^vQ94PL%NoE;Lv=#;)im>s7eis%9vIVJwD9T4g zHE!|X-5N)0VuEcs``QQF1uGv}&~po+4tyQyMUghxgL~FD(n53qJX zJK(*hilyN>%^9o9kV--wH@I73N#?~S7Vt`T%5LC{ZySgtCB4bxGN>vb-vXh;N5H_2 zo`ke`4~o~vTbVJ$JCa)Wv~P!26NwDrF>eM&c% znCmv*%}_VHNuZ1ao;N;;PZj(v)AUZ<>uliUsyFHHmHNrb-96^yzm# z&6FU|_db2?rDwI)W`dQZ$=E-tFMG|RIT-b0i|uKH*P7q^O1Zj@JUP5)@=O2Y-a}RB zd65mlKbs(>3=nP{*G=orQV+y~)y(S%WT#`z{;{zW=D>>5a96qI%P zeU{mB*$u4)l??v8;Kn545qKCjoaLs>d+d87*4O553 zA2fVv3TBKA&oTXmQmG$-0U5Y?u7r7TJ=pZh=6nO-!zUQC^MJo_e@26XnMpVh2ukcJ z=0qgUjvGR``;y0+S&|Q;-!Wr=o2B1e zBy`L7iT%bhf0V;v{>D^=zbBVZ7eo(Kiv%uy?Yg~n*bIS7Dd+7u6%AY3th6>v=YZfZSyreW2Ju<#!cpozsk&+LLSh@Rp)d z`+6Q0Wf>laSXkwi><$QLy%O`0HdfSHxxYb^jW>=GfLXNbtwXDZnm89>`t@lJZPr80_8ml=U$*UTA71hc z&kSK4lwj)g`6{rVny;j+z{&~>tuF}h#g<1xo>_*)koQr-K@2c0dKpacb51x^La~Ym zuk_DPA~=XJ_^v_^i4#J%dJh)FVPQs_+W6R8I-tXla1fgkWXx$}MxR=w;Cbvs;}3vy zw_%GagP!no=vaW>=)hS0<2*#*Kc^dVtu@hWOY3}XstAl~GF7ZEqLdUqe9~Z@ElvNz z+lBnXK(P$=9wQ)&H9A|BdYz(MNtLMb8O&?14Ft%R=yl=Qw=1cxO-iN`eoeM1}jo$b9W7jF!OG%wpX_`6+<`2LXPVKi_L>uQKEQ%+DSa z)QynfQk0oo{dw!UtSEwMyKezP$WS57gGZpfp3m3_P+O5OSLvT#3O~J|1vo0OP;>ZW z2vvPzA%={41et`jgLw(xQys9Q5kfuJ4$bB=bGriKK{2D*qk@`k**~01KrCp@>pMFd z7l7fXMf-%%{&$+U2oSg&>ipDCG{BFD zrU~B>6LSUvbb*1y>K5InPd~2=O2IT$NbXY(J_R0YP^5xu&{O{h7T8XXf+r)ExBDId zw~_s_(U(?i4lJrDyJfd3IZqc^HW8SAvDIOQD>w09&E{9eAmQm;e&3OC)a|p1mA?|C zgZ}?5LGl75NYgq0LDc_W<1M?(Br!}53zk^tZd5tC?VRQM0~@B85_mOqynP{KLUYAf zsv596m0LyiHY6|-=6q3kB^KbFfAx*-$8Y-8SU)pP#tzCns3Wsp5eN*^t`3%K-(}m9 zs{v~PW2~8iAiX&6ig+4$Pjtv$2<0ylqgzGlfO~&NpD@~&i|V_ri0s`bFvA85~pb#F-Z4aaPWmRB{ zO29y`++!yOhYDz7;Q8dsEN+|xkP;|GF%tPsuEHR!k-U!^S$Xr_By-orN0uNwA&3$z ztQ)3G&qd4u;lYc^aOc4jD~~KuxWZ*uUOE0v&h-kDW|MO_ zqMqm-_SkJ%m`wbi_2&h=M#w2l400bL|vp3fBW+76ldQ!WSrg zccfz4TEOW@q%}($p1%CLM=p!<9hQf1Y^(3qTS_?~a@&Q#UhBK=yu$|T$J`F3AEc*X z%x1N-Ld|G9@OFU+ItYCwJzWob0EfyLI6c+-{wvl%+}Dcy4}CV&ko*++2g3f%3`^)E zvS@Ay^t#9d){W*i(l#evC`xlc^a;0*ldZRZ{&#vY@=3NT+|!Kv9S!@=_Zv6`c{w^k zwZ=K3U)c1u?#b3f=iLY~(kIa7{h5q%M}DHn1HFD{ZN?0RaFlV_)b@(`??hi0j@c;JCQxQBsVw#riyoX&2U%IJeIFE5fPEGP~P|EVGCEG|a-L z7JJD)wRb}oZ(uCL+e0aGn0>g5i&=DyNnn=)uPZKoC`w(@76@L5MC?6yzN%g(!E?E8 z=zM(jBfgxrP5X8v?8!usb`A2b9Sp!(rmO%Q530#YUw{I_zIo|yDWA3&45+BUKq|0% zxLmJp_mFU+z6kyvdl$`ZCKz+^b)1FWFWG8Yt!F*-5_hW#$hns_G0jAJ`_bo@Fv!w=5X4=f{RK4B|0LL_@-5FeC*??+Sk>j z_(@Qo5Vq6T_l8F4?j6i^@=hxPP>UHUuDpBeqm|!Tw*kL)mYEogx#@S~lC%;GcXwrL zQzQ#3nBd+2Em!?wf+FfG9Gw8CXaqnEFh9^66~p?XqXdqpS@8KC+>cE^cPf&>3BQc) zgY;H}9!P;(UrPv9Vo1h|2dqQo&>&z=D=_I%v4gd?NoUL0!7%jB<7lRYj9uc3a50Zl z^L!iktuJAfYf_KpQD1~Bl_v%AuvMqOUXlYPom%)v*eOK2LeQt3{WOVgwr(1ssNbTp zgshss5qn#R+6YiET5*n`p@n+Tp-7zv`caMpwQ#7(l1UfvWw*#Gt97+A z{2uhy95~M33-QrHSd!^+JuCyXK#SnRYR1A<-X684<=tb6qukA$e^zMVu)LwjDK~Lg zbfW#g7=ABzM2q@wOn&CY9dQ#1#(r*7hYVAA-VN$h0;sg|(ZD)pl|a z^dmY|b-F~~{1zORwu!sHWS-37W$^h$=CojC_RwcWYbHlv45Z*mmcGC3kf=TN?T>SOFCxu8qVgMt=))T)?d+rri-J~7?TiwvZ>(tWsgoJ!ys{G7E zaUqp$4mrbhCZ8lmz3VWwo_K2Ff+9T^4rx_?R@O2*{YSV=?e9W)Kc-yerA>n4ljND8ne z4UH0Z*cIEC&wp$kdr)@wCEETcJ__bEPI8?t`jtY|0U!0X8+r+}ut)Jc;?Oldt1mJ* zoYeIa^{{F@o7%ZhKCYi{F*%3h-pAJgtVVarS@fX_fau8KO{sxQIaLps%}fw!g9uHm zC*`oBYL$W%f`5`Mg`p8u<+JEx?> zm%RwNJ&rQ5-za5-#+#p5&4OoOueW=khh_r&ZJ!MB_XH2{y6_C!Eh_I$0+D8vwej3`u&s~1 zOc?N_6++#9#rP_kWcz~cdS*ZBERApaF_hKUa`=%-sJy&E^WwSU)oYaF0@cu>GAv|J zejjh*e)TEMM4PR2)iq72hL2;CQ=wjv<4E$P+uQcTOqFr7y!1q%BT_fY=rAc;^=ggJ zd81ULIJUA-xGYMC>_@cZ{{2I+KJHU3iflFb%meC5)yf; zn&jmRv2LT#Gv>eS&Gt`dd4A5(i$F-GvTHPG4;S1w=|Fs<5>S(`iMD=F;y5vKUjAsD zI7_-J+L}J~(R27q{aG$EOh*(+3b&>O2kPP!w)Hf;edL)LP*YKmz4FjUni84Xl{>mn zsaqc1D`-X0AR8@*_lc5p>}h9R-A%O^Cf1dKqd*LwfKR(}fFUf4wK9MRsFJ%h_nWME zEOO`R1O0nT2WUH(hAJlbtvs1t658d2c=SvUfq8~(D)oRA6|1-I~1GVN;Z`5)VJJa=o! zbn>1xy=A2sSyu-}VuJ@GFCinn#$HAPD)Wmh+72A3kaMk>TCyO^@EbdbE@39+qd?(% zj`-6Yx?8k@WNApPicxn?ZD>8ixW_XW170EqVuX!GIWh_E`_-EqbU~{QhnEFK;WX$5C_xH!YTq4&9cr@h3uDkWKPr{*C7IWH*Wp;1Fc`X7|%d*I| zuo`&vwQC7M5>ZyndWcDa2^foVD#-xjR>YzDw0ZdHNszk#N?h;D8Zy?;sajtWHFohR~)IE_ge->Y>Q|CCiP$}rd zJVoFAJut0fUWs%pNMhmX4{I{q_-TOVah}}7nfdb_ttMv%!8P^U_`Wycmoy0=xg46} zOxAa4;MnRCeF?Ij-T*^9IMC(@G}3dENlcM^ccqt>nCO2B67mac%Fgk}yBD5o&GHlG z`baCO*NVF^tj&t0Y{S2+u=}2Q!arP;+(#I8?wn2{=REx#@`-Lm47xU$Jq*f{JNm^ zAN>2X@4ujQYY*`Ov#g_WlJL5lp>;e&ga{9)I>YM*Hl9233z`(@$3zktbOlEn>~(<& zs7ubdUu{C-zf#LeWUAaHZQ@0J@9Gm$wVxabjiadr5*>56^mDBgQVGta9q?IUPzA)2 zSE|YYxFR}PR5e|u&h3Kj&<>VRHhdH}#u57n_r6zFu8q(=4a(YiSz!vsof533(^NSwC-rGv|b4S2`rUaHvzP8 z=opvNL2bSai=Hhs7+44BsZTl=vofZg0A2G}x z&Xj9b3mYUmodEnY@TUjy*@bqYj+=Ocm#);mkMb1(HhJLiMa(vNh_iJ0sp=i? z?|!^R<`#V~fMq`AONg(tVj(@WC)R0aqf=tBtH9=P(4E%k++nYcde{z6mxjAB!FwgL zGc?rD%#_l}JRvlZn%D${lM=M*|Jq5jK~Qdjvp?}|X>f!VRj8KK>Otm)t0}(9Y`CEZ zzO2zNQnA5C?Vv3M`CMqjC+Qwe-IMtlYv$wMLF9Q%=Xd{79*fcKwto3dEA<9C z6%1XPcwcijNo89sKdLGAG{@qYr!j@;;n9RaWS{r-U}BTQVjyPSSte7NY}y}X&YzGW zGjuJP5{);WUK-anNPB(ko2A-c9)V0>vtf7N;!iNaoSarF=L_fN{n=RHScue^>$mck z+bp0#c?=u_kB8I^kA!z~` z-?+@D?$Oe8^j@h#4v}a<5I9n4)kZx+B|K~74WBmj4x*ab;u$RnB9BFl386)Nw}pFo z)Yu4gPk6i-luWeNj)KkVESgEYXNTTAVYKQoXxvo>^$&zwziC6lU#RTlq+fDZ0K$=X zOz- zM82sXxWIIP%Dz22wHf9}>eP+Wt=it$8xT2}lyH`GF#H#s0X}{|CVK2TTKweO;1f~& z+u-e4U?%3G{^o2lzm^P()3p|I4TeHQ?82_Y`s#=TCVXA)w2b{I8LV4E_Fu#}p3dF$ z=oB;3j}1|6JKwlOzhu>xSQ7-lu9@XOAI*QnxQEqZGcGHm4LF%5`z3jm_J=v&sF0F| z`LTIrSPscUT3L2{fSSNJNX8JBM(X+k`f3Y0m`0=c3m~AKs&x;V zjHKXauM2+=)DHb+g~y4;@fk)m0h5I=yUUN)JwZw6H;MpES8qsg3u+^lh_-E%gO8>~ z!}s3vVq*o+6h&rQk(Pzeq>^4jUy;csBX!}ttIP?h9m4%QRF8t+F~C~z#BV;KOb4BL zLuoMb47B>UGVw-8(q`GR;o}MhG+yKw(1A^_y03n9heY&z$wctfb7=Wb$atG!%88Di zwMQ?qElW2$Yk>H(NlrR5tHDS`&Qm6;GLs0a+JdvzbE`a%$QrXcdEtwHmLD@IN{U?F zYIQ~0pv4jsh3qs}iKb8I#nTzEb{3hatlCJ}cpB-2GB)}hCq7Dr6Wqg8$!)Gj>r$lZ z#_x)s67OPSrH6Ze67OiubWE1JxF`=pi`ITyVDk)!ruaVf)a%vaJ4U4LgP=7MT1HM4 z5ya8RMb~8JzbZ{STpzhWr&If+?HZMLpNcwT11$y|)qa^=oK$rIX&C>NVp~H0100`E z@wsj&Yh2q`HJa2QL%?uWr?>#r0_*g5O4`YlYai`=a-{WK25Xd0Ulx(~eXk>iHhT?* z`(hKy%P;fWcBK3#iFE-!=wj{Wpx@=|UB+{c?)CdtpOjScZV+|Ebw_BcM8O{V%L))$ zApC0>WJ-}n1`GaeA%HZZG=^2bk(UFt_Tbkx%R5h)??Ur?l|=O24hnzgBCFaD%xRhS zM0XwW!`%m|hDK7n5iCtP$5Vl3g*---XuU|)E)zn}AF+0tso}zS9i+E?`~ZMJQDL|z zMiLAgld0eQX-EAgWc{{<-&khOQ^jz{us#tTtU z?I0#V1DnqxJDcU5jbs!A6(Di7LGF@%_KD@Iwp@>Isfq5IQJL)LtLCR>kT0 zNx{be5wv88H#TgjTxa|T$YBlG2N*7%l=yvmtnL0y?636l9}JO-A_vzBX z9ndd}A^Lbccq}sC9a;x;GXX=qpRDebQGcK5+3DVj#E1Hc=FDIDxkqO!mP43BKxj=x zTz}259^nOru<4PVmUQ!(mOJ7wx?vTMH3E{h4@8a@e;m8ThU*9+Icnd#O<(rY`mcSv z;Cx1`3%UBeQ|~t=f#8Q@n?;ZdL%#RD@^5E^%4|EzP+;;vifW<#Z{Ojw>S4oK!nD6> zUWQvCX!%xjGhQwgD0(H(ma5+5K#k82wDY;pazG@D=DBqL8b;%YaY@=wOC1l9CsV6r z1c{8s3}h5%n}!NLATb-8@%L9fb%;npRg)+F&v@{JAG{@GGkHiM-u3Ze3v5uG)0*E0 zfTGvUZg?{Kw68pj@A+mEZ%%agHa<%X&;E|bt5FYjID+5_SM2)kgPqC*bm*#yBc4Ld zbexq!BIYdQODKp1Lp_dYJuK_>HltTMEV+TQK%@UW;@sh$k#>a>+QAMe!42Id`Uvgn zyrICW`1NsFOyXv=0Ly3Pj2y`5hWSvJIh-&xUhZSwVBxJFj(E^tD+=}UfY`)NAoZuW zhxel&k2@xxH`2@ojn&>8K}O_VCu0anM7ZxT0aM>wO;^Sdm4D=ea`@9VH^qlETd!Yc z!dMjib4#`1a&HTZvceYEA!6g`#^IH8i%Pz0{yJ1=N!t$=B0cAjoPB~H3f98BQs~yA z1F4=4an5a0_(;Fc^YCXAAj$j@u&Qk0_1{z^Mlwncd0jolq^CQ); zJ(51HL$GjJJlS7v|C-py5XWyl>|N^;M#@wZiF>tiO6)fBU)MX`LR_mg%6mj7{;mhK zjUNBA9z0uMA%b;E*Ef}_9+)WhKo2t79{zagyYe%Rq||OUB-*90v5HFE!3u?~-IKcf zQTu(!sA>3QNAv2wXYg+ka`}&+U!wY?ZUvC<&}dWA$XswnvXd1Hh!XXXnA666<%@O` zwueIla;$0-`gQB8q2IXm%kr!aKhbeQVzXQv15R91rypK#2Mk2-;v)H6`v92bV(Hh< z`9k>ES?4d7KQfk1fwbvP`Mq8pT^!gws-%+ep{7`RqWf=SHlM)`VwBEQ>w0TN(RmSf zyDJ4&Eyz>9VhQQ%`rcih;+PqKHhtAF%yhl2Ekv?Z9#saLxyw6EIPFVQQ1qy%0NCCO zk9{+rDl&j(Z;Hc3M#tH;XjRLW!N9th<9$RLb$Q7hkP>GBSO~)W+`6=T9oRT*W70v{ zk=>0HBP3c%9Bc|r+Qf%!5ne0h{etZiY%x9f}D(*}Q9tK#X^xeO_eP296v55YaEF(tYu zOPox8$woq}cJ((S5f1-QAHXDRfnQQjJJZn8$!9e3nO;BxfiwO4?1)3|~RAm)i+ z_9FAy2l=DkylvGla$WVZumY1DT+ZYz8=J$1tPnS(S zVgk+r%r1Sl8+6XSZ&Lz%{MW1EvHe$)JTIg?q^jM@jzxNu*`Ksn%zjp|LU#(6Qw zl^f74;^NkWqpR6ID<@d(vwZs48Pm#5$RWjLX*Smo%A64RGzs=tHO>1 zd!s6^hk?U}H1$2hRIQuPX!r`17nu^w8jO|zT+6o9rPm?=j{;0P`wWmScgpmD3{oYO)s$O~7&h0ugB+G@=sf(twj2P{{P za}FohnIvuol5X2g&Fz`mxDX$T`~HIvNm)?73FsK_l#F%efPMV^YM27mBg!Q8@t_jS zEPo6~Y}DWvW#}`%Q%d5C$gIBnQa9o0oc36aRo&s@)!B+O5P#+ayhGlfuf1FVFNHYZ zsVI^B^5Nj&_)sAAJstted#h(Hxa}hHwn-(%g7{XIdA%9>XUt-Chek8ku@Q?=Kx~A& zc=kZn`V@t2l8c1Yoy$vql&+AiQcE;R>-LaJs@pIXMOw6sj0RHsD(&g)aT=-w7W%9y zdTSPsK7W9Syt0MnuzBmg&LwJzG`y-X6nM3&|LaT-wOjcwiYbq^t7l!j=t4#k1 z54+YDOI*X}KH*NLQj^#4Ir+%{hR-ud=qdg_{@m=96>Ih(9X}Z=Is!4>yjPFv0wa_g zRSNRISnO~IG>XlgO}rE*MZ(zhTP}db_Psx5f8IVc?DLyc;?YZ+UVM7cm^$YbK!B z*4KB~RIX^%NvQMoly%BgZh?Yb;FR1m>dfIqi^|TsucRUrL&NO?Uq1r)httTSw0Q_E zPylKa&CwCz4b_XZ>GIZh&q-UEq!XZ+Jqt~B20DHqMae^0y^akAg+7!YD23iJ(=jQ^Eq_ z8l0suxJ4kcfF<-Fm$YG2WH9w%j5taqncs?8sm&5*(u#dn`^?;$Qvr38S<8j1WU#>6 z3#Tvl{Qsfry`$j_zqjA$6Nw-(Q4_rl36YTKy+@51M(^_ozt_ zMDLvlVbrKW@ILbWmG`W3&Og?&mWAP-``P>2*Zy1?Zx|cI{@k8GN_a=*qO*qxKb(0| zVBaF=*h?eUy$E%uT57&FWm)szq~bV|e?}(s-z>d$V@g|GVoSh+gqI7b2H=zkHL5rT zVb{lT(6RHE?-1yL5JRk))BqnQ!#-1mmJ*X@EAOwqYQpp?vV;u>%}_>YYkGdxZ$UY} z+``i;HxK1+r{llHq+7Xl;s9OO7%dO@bG^%HfrA>fkzb$dYtfCLP=S&Oax!CqK3Tu% zs~5N&C0QS?F|Bxf4^Nn>_DQC%N3*=C!OxOWYLkE;D>Z=2C)4iqe>k4oF`^dE-PVN& z$%R?Y?N>l^TR}~@{%L1W&Pt3y;Gy+qLeIIuq&nXNVmI{Es5T~{Y83Wk_>#%pMR)kX zUV;||GwvY;lm;#pb5=98>Z)TxYbLYciT^9g@n z2op&ivhB|^6f9TW?moaQVvFJ`xFrtMX&)WOO23NSuOvkYjBz!BUguduG4tk-2z&73 zW~xjkk6UV_ogQ_mtTGkaZATYG#fE|xZ@FSH%~JU#`IsPE=zVtOU8{p+1k-oORrj!2{ zxpO`C|AXB5|3={WO5G^V_D_9F@R{y~@s9+$mjg_?l!(i&*13-NQ4x9`hP*er#nxiN z>C97;b+3jOy-12}SCwJAhpm^@lIY6^fARfMq7#gSh7qL|YExg!xpBc!$zL`tiY!<6 zKX;U9%YE#9qu`KGGxO~%rUIq(*;DF(il23wjjkR2Ce5(NiSU5*Qa7u_^TmzT z4kit?JBPn^LAjaN5W;X`40a{QAq~U{HBM; zNl5f;KE$vK0fe%A;M9#Yg{mdmiK3ITgLyM1u;c(W*-IP%#H`nd4AJT~!oe!#& zz2Bg*SHyJknC{rE#fR~wmxqYU6m;cdK^{JQf?85fe0$hqnI~d0&*yAD-eX~Q$lP-O z%*FddVtQR%U>NYqQ;^3dZm1_woG{6Yr!`5^vt#6^8cjm4YZ|Nf8UVcEdm_A53f35q zeU>RSy1rXKZKHZ!0?vWjZX!DE>Iw_<+=N0LJEdHhKQWgZ*Zv?urr(~mzTy2RuD{didg&j_ zv^QSA^XOpL$!8=iJ$O8yQk-^`TPfi3^rAP>2l4Xy4m_=~hd~;(sz#dDb+oJO$4dPU z0&}e8v1Oo54ndwHa<`*fdWc#BiASpjN0ehGOw0rwQT%TaVAW+*-r+sXieTA-fDo$k zeB(|#mlBHBktd;i2s3F$mXG8gQWprTkCTev{=Eemr4sVXV7CH$cJy4?EU;MQ|L)eN zkBcwB_DeG&F9%S6C5x^e9bq`%Xo$YDTKz_N(KbtPgSFY{OI(Vr^-N9Mg4Ok1w6uO* zYAIfcX8gTZqg=A!d{ojsM9T%2&fjOlV1XyFXUX>#1ei6%y~qI&R|-yVSn?b{QEluU z%*-*AZ-j8(tKNAf39O?)4yh}M0SH5y?{A@&5HgG*z0Ut|5)vqL3|gnU4h$IT$aFXj z{udP^F8#l#82BF7yRMCE0%y?ibmqYH_V4D~?@VhAhPM1{SGcFfaw2 zj5*xSx$3iNJ`oRWt|T!0I~(XcXq6F4j%Jq7xB#$lGS(W_Bp`po zjkf^-8R7HlF@bzTC_hEUs)JP|>08I!`|v8q0`CqsbQH%I&!6zWmGMpn*Y*$AP$2yx zW+=tu+v}r>gi^xM(YruAN5O&Ore=I85E+mx!jTI;;-=!cwBYVknz`7r-Zr~(2pBm|6{Uzm$YR}FZ-Evre_?I$Cp<2402U8>*r0U^_9 zfnO9@6yavO*t@4^)IE-r#m<%YhVAeB?)#iEkPWK{t)@r7QtbQO7Rh}OxnRwpo*Rm0 zcK@y@qw9e}p6`|CuWcq5!*L9$NS_B*LbF_(Oyi0cPt~<(NnJSzAm!H;7>BW&$`N5N zpN-is#!et8Z5HS%3z*df;Q&|(d+i$EssDet#*NYA|Cejrg3>nKRB-E~(0#Su z@BJz#fzd6P`{XEIHgj}kLH5I+AsqTnEb#b7<4><6EemoY6FW%qO{_l3R>^b#?sg*O zj9@m)50kX(9&0`_6OH)HEqFZ;wJ4Xgpm2e{(b|W|J8!);D@S&9KxE2DLBlVL2cvm5 zHrEFufR&$t(#k3_iZKZRR*>y2!rqkx{Z(pFVeMjvz}Q2+{vo*E_zhY>91?p<*(sbDQ^Jy7iskHla@Xz;BbC^eB?$p-(F}HGgttx%8liNg@#Iu541`FUE@h$U;aPf^7pY*m! zX}sbkO1=124rr!AnDrdK(AZg?*NKLVKtmpa;--^ll7`Lc9X+d0;po!u(Xn2EgY=Ye zcxOpzdGy&|5?XLPWxTVTSSsyq_tmN7plEg1659FiPH9xlzIAka$ZWN4`3Spl%ai$k z*<78Q&f5{1fKC0Y6wFf$SWG&^b^@f#md0h9(E*2Z;DwkTgKBV(h7a}V-O#+``+?r4 zdf%YtN>PSZUrRy)BjsvF+|T*0wwA&e_A&)~Ab+;`*d=3B-ijyXsm~xHa2W5)V1oUk_DtRSVD!pG!%~ z3g~j_5BGz(ZS+2hTX{9O#DxzVj_{Be$}Ynmu%KUmxQG7z>b>!s0#&VIzdsdn1p;2> zM{c0;$%ggEUTP#i5n)557x6s$RCkK)!z5$Gq2+Zv$|oO7|X#VO~+v)QRva#{ld zkg=43=|U@jO~9L{kODxkZ5eCN-*ladFyo-MhQ^kAe7+2gw$cEuy@6cLV%mq>*?jW5Le=a3jJAB~F*9u}l*EEEQM3=V1e<6DO^0 z0@$U?9MzJ1{r2k2)`Myw!YkuZJ}iy7f(JkPY5h~Jo3R#ceHotZ@My7 z1;L&FL?$1i zeaMa=XbYymi3-xt*Q?uNBbLm#<31($sF%TN2AZIS!sx~VH?3&e5H(&*p+HGV$( zuYlkrM*ep8h&nTLp#C#k&MEntv^@9|x)V8uapS8G-y_B|{3!$)yN@EbpwuhYWiQh8 zagZX80Ow@(t+*v&(b=U63tdaHCVShaCj zd>JrV?sUC$zPj}0lJ<3DzY){b4&VlYMq)hlBiT$h(h3sWi-l`9q%@8XucN@ z*5HMnW;VC+1BBg zaXvAgjB41E2dwwa)|Lyd+L~ggXiq`kzkH zr-K#k3U~sW3_h&q4$iT}C07A%CW3`ydi+OBL%r*wmD|66hRg+C?2l+AV!q#X`Ab<- zH$pKPGa|s>7hJw@d9)rFc+i)5^{V3@F}#E;N{ry$9*LH!x^_2{{k>R*G+MKBL$1R*DCtxx2?-RA?Yw zlw&b_G*E|wjtu7rXHtJqbtU?t{54T=Egz)&L7Qli^lGFCsA&Zu5xZ48+hIWaX6o{n z%awKzyS)1K0_M`~uadlNvP}N7+A)Ru-1<$T&e zqD=MfJf&U@4Lb-c%INz|>BsF(U3AWf9<+4axv#BfphqH+%a>sGz`Hj4(_$lOtiuzK zh*w#d#0$R%ZPZ#y>PSLQU)JUamK4&Y zOqUC#P-%fcMwuVCB2(f$SqXRgREu=ujcMm@1|Lm2e^O|H07(6nGnHAN|5D)wo?LIj zib(ve16G(noeP0rpE9chcXwWg*HmSHZB4y~vuOTx)Zb+*6}PE+a07~`fugV~?XxBgv+}ADWk1$i4=_^P`jnXVY)r3`?u2l; z>z?W31Til2Vm%Ld%r~y?tuM14j*! zGC#hI2{Zn?q5#jkdDfewgoECuOrf-%A)R{L16zf(02Wov(Iu^Cyv&|398{0?w!=|L zZ48wh5SS+pkr+C`ntlQ7>LyFM8c}?nzHE6`^H`HRgwKY zG;{UmiMS_+smukItObXAF>pI0BRs;=b>a$V`dKOM$3qPzjszYq1=e6Fq`|lcbxGh) zQuXY;m2iv-R4X?MN%t7PA+AKbu7CmH#^=Y~BD!UT=|{oddF0TWalT*Oc;h`bxbke< zTr0~9l^-QqL4}R$L@`>kzJRuPORRU!S83M-=Yt0at%h0GUfWRh$Nyozb)eyUvA(K5 z3;}kvCXgK=y(5PHU`lKk3P#aC{?L9#j_jko4i~EbbJ|@pF2EF`egof{+#6pa^q%KU z8+Z$fPD!vzw{*W&sO6U(4$1=ER&2rH;h-uRa~K5Fi1{qw4ASKz^weIzB*xVsBlmdAMsPyH-ZKvZ+&Qd zcnHRx^%`38=wRYLnI^NEMub9P)3gAv1u@P*dcR753#Ntj4n{yVgowT7$i^`F>cbf} z=8f}pk@I!eyV_37d*uO_mSMylD)g)LJtbs1*q6odOyZ~_;V{IZ^ zHDgb|F4{0ueCW(MD`ju5Y|?2)t2|==Swzzxw8msTK^F=Up-7bDai-*=mo6nJq{QCc z>UotQBJy3n#44OE7G;Z;thkATUR;*$st;~+N~tL{(@s~oSA4ddwi6W`i;*u={5 z)JFO(TB%~B`BKO0e_x~%Yy6XMelrazP^fyj(Pdq9hy3Z0+=T0t-^Gm$swnO!{`G4a z`DvMua5hk++f%Pg3e`?Zy5Hi64`^Thc1{js7?d^+%G?{LB|K<&EL;Bq)yH#Cq&r-A zjDw~K0e=%x|EOxd#ksNW1#MN>q|N`z{0FET;FS@dmXia&#p|t9Ysx=ALriM$&!0uL zUv_&L2>&uzJiqL%2|f?m`Bk_rx!$__*KwQ|pPV$SWsu?pA%4K$Q;OZivx(nK+XB7- zYHBvuZXX|rOp5FpE{n1}nh(6n4$ezD|GTAiNkW>zgjs(z!2@+AQVM>7bPOAL(X^x` za|gk2&UCZfNWKOF@|N=0Q-RqH$#;}QNRsrFXlUk*a{KYi!jPG@4id74Y#3LVH`$|i z{l8BxY=`TruZ?0M?H?DH`*B`r^L8myJVzyKe~nvuZI)jHe=75c96p@^U{(Zd=+DF6 z=qNR+9L$6w#$|~^_$F}B&1|;U7urj(7d<9VH16Qf$`(p5B)@H+md$ah4XzGIB;t8V03PgI&`_pT z83ts2PS?B7yf@A%aFUvnBQ`;L;3pV9W~(`1JV+E<1&zLEnzW-(^TX+oH=%}Ef7z>s zbr>>Om0ulZE7-kxivpW9&)7K_Y{o0kLy3LJdz*z@Ur-KnQcf7JTZ4EbJFL>fywckD z=vB?g3GhG$65pf(t&86|j8-px91WM6-R|HEEIxSdl>3?KsmVrv;@yNX=i@oZ1dlnI z)4&O1ylNf+i!1)2-(sJj1NJ(Az;C`&h7}qZ1wOq60?|5X6#X+?G-o))YCh=5>u47MX?N9nc=>=G+_#5 zA`GF&uVn$tA{$A-uPA4^mqlqw+~=Z!Z+<^u)olTdz!$54_!=QSpOe5lGv;C%KR7Y% zvt9~Uv*~T~r)^aHWNZsXsHKqsZG;BdxxLd?B0{>cjTWr*^YA$n@57fo~{`e$y4I}B}Q}-IU z8MayZAKp7^Qyut)#mhYZBBap*1c>B+`v@1q<9i9V~t`J4hSm_U`j#;7cC)TdpEbw9mC0J_%2h$uEy!ag-E{Id4 znyONT1oA}&U1>r-95*I(bUTR#FTHm+$X=c#$UglJp=y#dZ6fKiyal(MFhjqnM`^ym zi%=CHe?<9t-fiW@9E;`gr|Skx?(&7K98MuDhIiU{VqW-C4bMt-32^S~>O%P0m-AbX@V#D@-(~sV+CMdr<~H4hd%X-TZPWJ~V=pkeyzSo|Gy>B??P=CbD(j$F!>l{%aP+>KGOvE>1M9UgV_v z#}JsJYr5wh;O*#5iB!ushQ3fVBl`mkpUG*%9F?<oV2Da$n&6lMlTteR=aaf9O}ugFYk{%;mY5v$Luo2fC&t> z)Z3)0ldu`3crCeT8l6L?Ixc->=I09v!>BBhWTd5w=>yOrik3vGU)a#W8lZJFygK8410S;p z_~SyQJdDFf!L%2k=HJfidE|MD^^33E!Wy+|?kcnA>JKbu?L_j4NQxb7N*d+6>s7U%p?%k6txxXiRAdT3FXp?iHpc(vo)mNrOwohYY z>%hQIdjTU%Tl|^4`sbvt6aMK^Pm)=*@frj_`dj=3X-D*SoNb6{?Ys|1{n74xrk{`H z=8Gpr5><)Qk=I(J2jTkFfF<3DnxH0+gOrND4F~(Kc0`0&SI`1czQxp|&$C*BSrYy_ z2#gfsOhO6&QwNC(6-^{4O^1zV;$~c&b&MWbwMHV;1?-XGHjo(l^B}B&z6TOlp*Y4o zN6FW;M~RI3X3DNd*PYqrF=fa1{> zcean}T%D3dq>;|glDoNJmFM4mkbmVV!y7qu!K*{FwBLb?v3H`^T+nU$-WUc^r2A;j z6$57^s>G;5zaQ(z-VVk2StoTOV=0?tSn*boE|ss5(WEAqzANdi7RAgxJ*O-REKWqgSe33t2^M{i zkuckP!;nn%nzTR$Z_FE3{S8^b?EE4XG}K^Xzta2tVFj7aEZ>`LAK&?w747o7_S)sa zEnCh57+tN@62?bI5So5t=7h9V(qwm~Z#uVsO4sE=s5gKbi8#8Xfjl4ym&&Hc#l#djQ>qzRtlXEn2OxjCKZSe5Fq^J$%St4+1g1lio*N?osVPNl#tgR?8{}fSAT%&%(*%1@7KPldra}1IV_yLFF2*}3sD7bKPnEn;>f}`9@~4}usETh1Zv{){?fysa@eEYVB*%bK7(9(qBvF^p$~w^N zxL*XSAWVQ#knNa%Gv)6_-)brHdnjE)C$|%o&9<|#nRK30bwK%{nwD^(VR?=8pBmd8 z88l7!kn_xQ2hNhi^!Ibv&@6G*{a_Qdnlo{tsDdu(FdRnt+bhu-;$YfZK|yEv%3S>N z#ZJ+df}CA0srMX9o10$7tX``G?QBk_%tf!m#NJVKqtK(v%5Z_R1EiS`-SGGl_}sh* z%8<=$QSdT`M#BxkY}c}Q&=BFVbn&=e{nsku2n#BFZ+dL@h$Nl0T&fI@>QH3!3Hyd}k9%e1?iAL>8^y8qj{tp2&+PD6IOo?Yl zJq?VHusw#~*Mgsj;`%{H5I(YZRQYG$`7tgATA8XB?=9KShOAUBUi21ykhM1~)|Gcy z<%IuS%ll>&MlT+RfCg4ypVRvH{{G!nWutncCy6?*J+oh|PhdAH1gCb4-#@sPE5oIw z2CFhZu38-#dCrKwp4`0QFuQwuv~lqZYv9IQh|3*3TfpQ2gb8sn{>=r~6v7FQC0l2E zg%{5jPINzl=NR2-{T-rf3LiAK`7fSMmjGl(askm9iubhm)l7Qt(qC&IHSUaKickhw z(T3v!?N1?>ML-D1*=jw&X$$mOeKX8w_DcS%7>%=5TX2$l!VfAJZi1@mDcZZU5k9Gj zPqJjHdTC1{3HH00;q15Gthziy{YuE)$}<<#$eqjMdczk_^c>f=ald@=PRhf{%D47G zA5N<_y@pB$LE=1Jz?h;=+VoXsbtwT5uU2%rBh)?Mm{^L}_80NvCd{7HA+DjPX*Ha@ zLuFYHpvp$*Xk*h%+D;=u{f#78Tkh|g$>HbtD;68U0)PZ;CcNNM@TrH#DHw(2GK)6HD0n5?M*;r+*Q zm!gHc#_1jNP8tCb??frFz-EYt^;38D`!(tzPE1<`2!Hxq^yk{%uABo+fYgrzR+?zy z?}bJubv)tFuY7ldUy*`nAPeZWstccs+~PL0+P|ZEqwkPP zTPofVerG2sRjP8T>zxbL^9(i<{%)8=ZI#M*{b?&zlX~lYiQw6S{5Ya#)f|y9-Sb9O zgfL8b((so$1>#fwW*qi?wC$={g(LuAXK80$d*HCdxT)xpt6nD3fZgC4!#rD?m_vTk z0T;tBtW9oV@Yz}-RM4oz%PLN|Ya1{wA`kKG4a{#7u1a+`tKghiu8Z|PJvRT1ZlG--%cx!}SQW3ED2pGXPy}NZXsU_YI^UDn6tQKgK#Ua_W}09J)E~Jp%+Ue2kL`9nXbl*TEH;-f%|$%` zOy!WF84gk~A@3GG)LENTJB9e>=3zBu5hr+G{;1^F&5QLN{YDf5st#BzzweKKy|@p^ z3rGEzU{wHwte&werm^xn3Cm>bCqJ9)?8NcPiCx{PvzptcaJxU?pZpU|F@^kI^lScS zO=QG)&1St;gZ69|V_U#g`pn-%tLd{-=^Ist+Voz&t7`vf@bKi*jm3T-)9N{}8F8Iy z^@*}{V&}p-8Gu~c)?8jUDy@yBW`%Hd+g7>``{W>f_}g{~$``yHDrEeU++S9C9`Qu1 z%PEwV>`ABvjz{R9JPy3B~DH8V55{(J{w9*k(Gmtpj8+JeM zR+twNCx9(pd}Y0xAxhIH-A_7BJJS8hios^pMKD%$BY4w2t1sb;$GOj+^je2yp2=Nr zFb~-?M_h$oc>eg{B>_9q;=!YAQQxz(A1Oe)Zt%*ZeZ@#b`z9Gv#=c&W7M zL*8i4MAut@wbQZ_uavdFKJ-m$v{Cj|q&C3_M?jJ^rjys*fBeooB_jiL{j$-*zJK*d#qfJRnO3ls_7L zuz`-NZ!Wq}UzicJHh3(@Q>-`ANn=1V!p-5rAcHB1fKg^4v;4xo#5)qe%Oju^*e)ED zL1JGhxKj65#H-ToGU+hq5ujOgX%@0zN2i5iie%h2V?ntwuBe}=pI}C z7uZVXaCK^YTx3V^ER_2Z+Hxktrob3(FJ#-e(b*Mqvg6e3Y=QX~mS{}T@SEr=WA1VY zZU3fakB>&>FmYx-And)oY!jV9@V!qf&vy%Z*ibk!YkECqi$I%q04TGcZrHQep^wF- z9^(XbGqb|yCp$ZyN<8O1!-}o{fk>-@7*M9r=k0#f(->VfyYqhhXRNyY_?;`CFIr*!dR*Il&Au(SoOy&zcoF9^r7I;D8;z4 zZ%Tak1lA_*nbTbeB%r`^5iXbZ4+kU_4&w3N?h0kzzo4Lk7T@nCsK^|rc(DT7JEtSZ zG5Z){ug+GzXcC;zhU|Lu4((6AiW!+|aud{teuXEN^s;=Pe&88~Fzn zt)hdWhPD~rK?t;L3(k(y^T2XuZBIlh*n@@k6?Y_f;Om<+GIH z_Z&ntfPVB%-BKNfq;SAw3JI++Ts$l_*dJ-O<49ZR7_fahOe$%fppsj~tZ$?=*&EK^ zAU2G1m9J^SABGf1!mg-(E>lZ3tywc;r`Ns0vcUNBPWYi+SwVW&N0wkql z7$%+50ASm6ttaHWWomk)A_H2x92bh&!3*)ny-G)n(`-BG7GX?iU{f)A>;h2PB7HOe z$kUch=?3pebmRu{=I+zSA`9*Y4$d(qLOciks7FBGXu1e;JbU+UaXZivYLV%rTC{Y2 zeC{pT1`pVMuC!oiHVb}`H`PfvU7}2NG8mMOJ9m7E=pZ*}+g6)>9#c_U9djdd(aZW` zxow~9Rk_H;B ziWj##?B`NNAV($TS6){uygSpqFE|JR2|;=1*yIw9(cOz>Tn*g2wu1bg$J@O_Ban6P zMUtg?g3GU2kI+Y{s9Y2m{ zmr3y=NDiGA+E&?X(vJ&04|v3&9@hv=wr9XFwHHZ{`PbKd(hj)B4XdV{%c@!dnJwTSVv&H8zc zqmup*hW_e;ZG+nV>kj<}sK)AEk?IpLhQN>MI60LSuz9LyJLLpX46F2+EIzPXj)e7= z4xI%=z;yQcKR|--yV1?+lR#=^Q7|fT4t+ZUe~Khb+$ON9*hICwtdc4F6GN5@#y#+{ zqJKXMzsZveRtPd?PxOloR=-7Q^SfQM6jGJ@68^&>Lj5DBzK7)l(6Y*%mn{1IK&zKy zD>zsTj0KqW<7Iwa{tj3!Yhg%vB8PFPdeuZn2=wtx`u(ysq-BfAiKY~O$eiyg;3T^- zUOzzNTnvi=bcPQUe>!HVQ*esIA9|;E`pKSne)6x|wq(gUHZ~x{blw_Xtn;6wb8r}t zeiSj5Thz;0W1;H& za~&gUbjhzQGN5A|pCV&d599=;1=MmuS0k14NhTzlH;Cta(OPdb0#=D$+1 z($;qdPEgX-d;Ex~!yGHd9^CzRh3topjnn$#tug6v;f9Zr%Me z2+y5)1=OAGxv)=tx;_>BAcps*g44ZCmx)n*(ZY;tBA))`5yC>50fcf1iG|Jv;Cc!K zFdih6#g(s?b=cy643cB1*o>eZ`)whFPr5t9MMQMh8z;@BU~aZ5PL@aT&u z_31{V*PXz(nW_|x!20`so%C%g0AM4j&r5{5@q-F$kzd^KGC7Rp15uQiVr|K2f8}L7 zVjT3DV+7`77=v(F1(bWgjqw_W7zof9UtGLbsi7q-A< z!C6q%>eeI$!EZXH*6&KuUKE7t7^j?_TIPTH=;Esixc$R(fMM1AwHTIe$uq2i4nn~yr*bFkF`8uw zufabt8Kd`1)kWg4Qz#AX#u$H7eM&5fAs2eHqXfXw8#U;{HwcEv!>Mz9@Q1?;Jjy+c z8c9b>fF&e%S+B_YNZX@fnTL0L8yzBGyOJFIZ!&qZne+vxchdzH)rN6QlAOMS!ir4x zANrY;OfLQ+!%qfIInC<>^$6)k_HV7)-iX0_UXyu|aFOLLFFE;SNui}cA01fqTwCxo z@JWx#SZIKFPS)mdY%&7MNuSI*whmac?jDtNHZ}mhdW8!zH>6{w99>$*fW>0kWzR=D zv8>M%*slJks2}pqNJeFXBJm9af z#%<@=SW2+PD6M=Bmhc9!hn{Eh-strv?)g>cW5t?kU%JMU%%?cB4Pl%UxXq%`jEb(& zUeVn#oZ00Whyg*U@UEi#FUiU4^b`kxxtHsEXtjA@2y|=MtUq636&A~&rBrK771BTP z6Tr<7Y&;jW+?2O`^e_H0f2}dUelrYb2!szjTks$4vG*0^XoK9hafIw&H@pU4o_cMD%@pprVT+x5_ z{r2YigF^M=F~RsOU~-T%`8ukRtCyDG@!f}XmQ(j-OYvx7kEvOB$uwjB#F?(4Yrk^$ zd0G((iYW*`12y}-8NKnA%zC~0TGLzxwHN*2CNC}!dOWIAIwLjSD|Q>dFx9r;JgbKM zZ*Pi7pc+ZLhR)A`nJ5n?aFagKf}5tB4$f|57%#*~{)8trVSnuUWTL`Vu9`a5Z0~r(CD10|q zB<`^skDdX$942>*BH}UeDwh`}K(aJis0zvY_G)q^tx_t~c09xu>?L!uCBccb9Z&!A zdc$nvA@5IhIUXYG;G8I;A>v_`Z7@9Fx!128j7_#nJ>`G-&SF03$!0qpxdA|Brv$iu z{oWb7x+ju-26#ElRQ1~4Mn$+<>?&rlRAl@ldwQ{y2djl<-u-=H+w-ENEU}KYz!Vv! z4wmD&c-1CSnDvFDf>-1_5USKtvK9XNVNB&1rV2=uayRsshmQ@LTu%Jd`}uJNf@cf;?0hf?sxlB!=c_p&60{aSceW8vyPw6y=->)}znpy;r# z%A2S|p!Ei<$-IL!jtSf3$uI?yXF!zhRZ}e{2?b^-Y3sTz%S@K)LvdnyBRq&W;px|B z-{MD?)OS>r<2X_AN}Z%{!HSA(e%<1KbY+txi9Oeh8jee&MeN`b4AHqM2VeakG#Zg9 ziRA>2Sh`I<&zB}vH~Y7efsZ5;_Pt00Epg{?Zk+ilN|nv@)5UcdKV%OT^vS1?>SuPy zHD=F+mhIUZQ*iY6)RBE=_CzWV!(XC0ss+{7`po-`O@beYH1NK*k2<&P+HXFRlgvGzWG)VQWBHc1sR(rOl&XD_^7iO^msk6)1DvZn2@ZB4M~heqGCL z(6OLV@{6~utES47n!rMVBNu|7z5ktnIzdt?=^4MHjlpd+srG$_*T>alx^HZC4o$VG z(e9>7x41>Q0;q%?ofjJEhamwtLjm0;_h8$Cmw)`|bD@A>t32Er0jvH4htvK7xCvu; zgqhO>S*?6=5r>8T_B&FcB{L8C<23Uu8O1xp+1TxQ5QnY4 z#oYk`MH{R4n&rikSXQ~a)|QV8Ni$?C?P~hWMaB!uH1X81tX#ure zfTiE6y2J! z1hYH_ph5E$d@TR9D%7)b=6}8qpRojO(c=Ngx#yPM`)`FZ-+r9e1j57YevP(XwtV{Z zV5B#|y^pV6yW(G7LN}@%34-;5G0Wq?#@h(v#q64UotP{r!jhdGGaGnngPkAGj8zES zGik~C8;X3mn#5!D*qniwl#pfh_8+BA)Rh+g2NXylgTE{UOWou4!?3_A1_fn^KX_oN z)od)6{b}!R!cPY+bw&0_tx8053kr>R$QR9D+477Z5kXx%A&@0tg%1qlYJzq{@cVy9zZTZEb;{4lY>R z4@~W@c4sagSd%x1DscXe0Qxrf@~SW5IA7QW=V$QZn{iKLRb!7F-ST}lo|5{{HjdVgd}VMYts+>H=|kHT3&1YW^XB|{~N z$+)lVnXMZJ4w)NbN7(1@3LM#Wxs~1RxIqh_INiq6Ka}2}HwUXEo9&v*mynuzxKn4Q zv3*d|Nok^ZVHq{&H`^ zSSH@{UBNe9SKQo6xAqYB8%!-dK)c{rywPoM?r<>lff5X2I_ITjCX{JMSX8%|>S1-- z`}}1KE!V?5yWoGsugbeJJb z*zaXXB~mFwf=~t$3|=N{`6>7`o-|=;Y873YZ_fTa#;2xNafoxDu=|}y_zd`_N2uyE ziptjzn9a#8doNE%kRxtrUMH}Xh3SH5BItI{rP(a?@u_N0?DI?u;2bLiFl#;m@j6It z-Jqo#pcNZ<;|3tXlfo&m3ab@*QZ2ni%9UC2t1QUD6ej086~up=fD+<+JKd0SLi(XT z^&CUC$34Oo?0;m9`*>CB&Xmvf?N{*CInptF-r>qUA_bVOnqE%N9r;lQ;@81wMa&*Dl18uZnX1qfdP1n-JY zkDIFFuNLT1&K}8-r47^qDJFW0AIIPq-b+oaRM?!e+oQmu2;2;tOSIQ3<2C1X$x|jD zsapR&B~{Q>`IkLiNQcp`N*My|Hg5lruQo=WLwO7CU(2UvAn%^zzx&tmI_fiuoQxvtSV)pW-fhfn)sMy!{{ z@Ag`V@!iI7StT92b>c%lxh{R(A75FUC>#$gG5Q6^w^G(`i4BDOg^=_ZPylL30(wrE z?vs>wTYC7fuO$nXqqGV^gNTjWi&^tCcYr{Nl=*&^y&jUbsb$oidY?3T)2o$F;XHof zUTxPc;Z{YT^A34acI-vba82HE+pegjfnZeo#TV;cV0MCPZv}>6dZ&@-z3fJ>^ptAj zG{L1(O7YayNV)p`ok4?74Jw@_2lt|%N9YZS?@5yp-=sN1dWSjXjMOa~npHA`dAfYx z-`m13Fh<|ydsK02tLw`P9%`*hT;UYqK)i64R26PjVr5goW!8?VTjLDLU!%!~dslb8 zJJiCGQ&J%8DaI5LeL>Ew8-G3{(g?)Pl^e%g8BZ>aV@!>EN2H#p5(!voGdW0A-D?I+ zGL+6_5$_3&#itJBY-Mh*u1wjtMT4J!Q5PaCEM0HC#OU zv*v;K57u5R-40ym{fbu9Gc5RtY&@B33>4U;50nrCS+i9jzvoa;v%{TEoi(~yx`rj= z4N(7nf@mYx%*Od+zJnd;fN%7Kh2Pae4_jKf~e! z3HDU$xiUJvl#M_Mr8EM!pO-7%Q=%dqgB^B4)!x(=6#ESRu_$RwBW%?H>PRfg^Wo0m zjiY{O4|&Ql9-xefqIk?eIi}x>otsFecdv(Alm+7~gcTdp%NyigcoioT^B=Tgv^ctI z2G~Ef6wdhrO~#TZe|6&5w+X-eqdgYVhKsGr^M0Kb!Bm<{C_eOC!Xm{+;Sdt+JM>yc zh7Mv{#UKNO-{zL*-aN|g_sVINub@|Z&S$6E@b;-js*#>0hM$f`De#FcA$3m`#Kqtf zs=!wR+~{fxAg7HqmmuR6+-tBg=oB>XDK975f{Z{Zeo&N<(?swip{o_Dn~Bg+nXp_2 zkuu1eXh$FI3@(a7q1qUrhjUIU29k!Z_f$n|1&JuU|ASqYlK&PCZ<@7Q`RmnFQs_qI zzx}p9p8l5TLtX#WRuIUK-E!;dTx%aIN%4E|m0`$@lGa_64X#w>r2NcvRbb&nCTsTc|eQ8;9JlwW*CNa>}7@8`pnxxDfFO zrTQS!K{@0@|AOKDbrJmm9Q4#R; zewm#QTw7D5O$3Mp^~D>tSSmJ$vE;YGj7_e|$cB7*0A=c?~d)&zl3(|mqQ z8H=3(kDg8L4U|KPU-WbUxnZ4eF2@Jy+i`hEcc0``N+f$=x%td9W{({neHDz-X0G;6 z`Y!tZTWKtxjzx|`YrR1~JJ&uX2ULQj4~tadS>-V~8M%~Pt(b;1YFU{h?x6=)jDJ+| z-J0B_e`Nr`yL$P@eUtU)dExfyWcyT2Q`@Z*5l`737bKE%Tm0rp23Xz|eiSs%bB`GX z-xstiG2ZCkzTWFe&ICXA7U5%+V^THOge{`+>u(kwC~H#P(EBt5ckN+y?O^r0MD_CZ*Xt+C z;hXj@9i&=)O2bYl@Z^Qh^9gkS9xeUiR(NZ1_4o_X=DTU$LoARHx`W+@-jQ8AlHqft z(Yz*A908p|0@ZTX7sIkIMk=Gu$%?W1cRw7-;nNn#aB#8T%r~*Ui)5;idC$sLvG<@! zLQeeJYL24rFNrchF@kmMUHEGzQlq!rIC!_~_q_^7e+)6y($me^)Ut9rY0#x5j%5+_ zaod5J>X#T&)$xg8j_@?Mu^$`uLnaqIg+GBvlH3%Tjb7zRz`sV z;2%k5z%s0paveAbl^Fi$J1?p(WM{Jho(1|*!062z`A2=lNv1ZgJe zp1F(|ThyF}cRrN<9(*XS`i8&Qps9p8DihZ}kk6pl9bCK)EY00He~(IWLlo}iDdg(B ziBq-{%x1H;ZHc+2@?j;m<8@pnN;(rwngI>lkDppJM=sJ*8=pn)dv0rWmcHlP2o`7k z(0wSdz8f*=cG3v^n%4JHtRkkG*Q7)((v+E+I&heIR2s8CvX(39%d2R6 z{~Gv42Dom5pU6GucA>A3S6pev;N76t;$$+%c9$@W8qOVt8WoQjfy2P&R1{2F)z5K0 zmU-?s!IsiP(&L1b2#VOcCnFyTw?|k~0|uFf6KZ?aiAtr*Kk1vr@UsdpujUmyz4Uv= zxS@XIR{MecUxl$^{L@cdq&Tt;s2>B*eQ7xm8qnN=6oZeSo6vVuhal>wtZbH#2#wu& z8SloLEuR<^@0F1_rdQ(`G?+af%d=)PtvisKB-Vggk);lbXM}&ttA3Ve@1DUftOm4C z4(Yx5j%Zd93TWT?zZoh}-h0el_kT#|I?h_5DCjKw_sI*uqUV!hmJ2vHsA_o|P3o3Z z0iIR%i-j?8`1!kwYZN8=N&>bCF+$%xXY*0?VJgKFOtf+SG%R51@+tm8(Sw%nBSi`Q zGgH#|%?P03c~^)n|GGo_vpyFp=py`9r>r@RJKm&OQanZUV)Y+QqM5D7%kj!^ zNvpc;_LRe6?R+JW1m=QAkwG5!IWz+Ak~b|q|pf( zW}ti#9Xr9ts#v2HRrbB(lTe(#ftA+GabKmOT=%;FUN)QYGi8Q5usDwgcPwzGZeEE? zxc`$D@URF8TPPb%T}uy}zBv{JeeTOjTM*ZEa%8}Mpt6Cc#Ndv~e_{?wpR-DUNm=ES z>X1#3&0_90m0=WLL_60@vEdq-dsEnVS~d8obkTp1sZJ% z1dIDwhek*x)DamC5t7jwEQ1z|Hl&2?jU?F36sNVc92Ca5`a+3T6!<{6eO?FE>(^|}8irbM5fT@UHvFP3&xVPbqK17X8dbVXc zOLpU*l^jR%tZ)!-Y7Xr-nvB(jRAe@J(T+X8h%GElx!rVrb5y6PN#*(-7-SDGAU(@WlvEM8)I-8KY7mVf#{6Hhi z{DCrBi?-yxb_>?2*xjpAI=m0;kef&<9aUtM_k65&c?!91yaqI#S6S;NMP|bYhqn>X z;!|I$xsvCig40)HPWv`;0LYn;5Rxv=RAoN9?$Qmz4IMlx8z&ydXO`IfrIOiqB<;sN zshQ5FmF*u|Svo>Ml`SCM`1P}8u?%gj+mn#b0x$Kq^F=Mf7*vu{(dpqV;m^egt#-Oc z8|sN&o!`{!*D(|>!X{*}jh23nxAi>?EXlZS0suUKHNI^RCGplA7hq1eJ-@~c`4rkPVRbMyjkeR{>A2RgvyjXgf)c6HhT>_Heev*xU#PP3d8jy#()CglLFA;4KgBr2oq|$aFY2bb!!EH{zvrH^J`Nb>{ zqU;J=4%90F>ky|^oldH=hN)wfAoz@4y_x|$O?F17=HI)FR0LQb7mJ8$HgX{A6P;r- z0ZWiw0j`!R5J-b+m*VYEW>&kFAg~vSTn_fV5@5uxPdLqFzlv_ZXTf8Qa8^~j0(+Vg z7Z5LpW(Q2wM|514_}i?qm4s~by0Jnf7uOAw?{u%0i+lD0tt1LftD-B71n$4aP%}^p zv2W8wW~EkJdlG_3qwYjWZPwhc9UMu?>VrrQ+J+hxt7!of#a4s1Qs5s4 z#qwF^Q5z%TWYPQsVDWTq21qWnlEYsiL7E@uxP)9ftCbsrS>7HU=q6Vs@D=aKI=wRI zF{bYhHHka%UBtd`_vcsTOTFeqDtw7r#$PL^hER$@=`~&`P3{ zedy>7`Fmis$7gB59EeaH7p5{{C7wj+n8xg-*z!M!*nV=4^Ow{zSfkcb*1YudHL4GX zAQJU~KJG`8%MLMgHsA4ag=X6B6=vCKsM&7g+iLxO{y_Gp&^vvvd+*xlGD{Wd!bSP| zC4yxNgXAvkn=4H&zm4k1zQ1gGkiz=##^#T27Ujl?rPSkJ_P^>SaF-MMcgO#SDG}1% z6UHL;AFM=JR&z`Lf9$8sORRqKtSM)c{~!CQGhXC1CGgPU10FifK4N!W0n;CQ$nkrR zS%^2RGn@(T zYdc{r9yC__4>j34f>hi0CO(qhNf9u%jQQs--32)t4r*;AIQSntW^io+#=?BUhbjX1 zz|L|XR8ZDe!H4sgx~C{$vBS;;{@wF13}m9W8#;BkSm3;sXe>63T#R&%?Eo)?pV|@n z4~uY|;zdzYVx$sJd3+xIex9d*3-~3+3{q=(3v?3O6vd{s98irTa8bum(05-?l(Roz z*Yv&)aHKV&#$@W3VAYlHFS`nFJM3{9<=Vui*HnExJw_j@s<~jj_>of7f zL5&I%G(+(0oMwSTd9_$56ZWlb1i`(t)CE=Yb6E3f>5|zHpMYyfB=KXcR^)8?eFmk* zC`mTEpLUeldw|1Xy#~~g5j;<I8>NJ?I`k)Uh#Zkt&2+cEN!Po+Fo`vlRqRl14uhkRnE<}z34-GbaFS6h z9Q=M@=koP7ACZ!Q&W;103wC|k>22)^6FyFODOVOg7recwyJxcfr%Ju{H}CC5#lbP_ znCnNhgpK9(t~8O~zU-a!=~!KF`<)?a*Mv*SW=Eh8qy|X_r})f*Y`=osZP-Rfpz09~ z`F)g&B-kN5CStO);s;ndovuQxOG~n9ut*PGKh8|%AMv=Giq!=~%Ce+%!ab#ROoNZ? zUXj%4^pl5w?^RtI8H9c+71Y=#{Q8KNoFM5rfgHtblIjJ%s$9HC-Sg{!cPi&hLO}u# zz9DJDGk|D3X`~L}%TcJDz%GniIc#ndVv4hiyeje*oBK|;yM@I*4PhazgxOM>2w%KE zRL!x^&*&c-C+W+vjUjlF`mNN~;^9KXo(azOh-bz^>mbA7`OqEVZ*C({F%BKDd3R$h zuxfn+K&oMNqNtwPtAkp%b$EmZ3=lC9j=g7s0yVeU*5u*V3)@KZ#2Nn)*--WvI&aCud5@Eqvu7Z%vFgAz zNg;GQbiOoLdn4mHY0ui6ck8=>t5U|Fp`bmKsnxcH z1d0|qX#P_&PL&yLB^PHH@|Jodg*S{Rw^6U%A!Fd~ToW_z3-5-&`T?iBdt{L(i9Hs) zSKXixw|grtOqE|FQJMpgf;?~?64uoB08y#-gOkxDVm4&sQs9r*QUuU7_aVt2rm?Xx z;z>EI{F{Z#Z;FiVkqjF}{|Aa7mam~K){uS)z3z1Q{73fc-8Z=bwM3_3sk?-KLSU2| zsOGt$&PL{MZeMdJ&0qUa=bkm@0W{r@ajF}hxTvO(ZXka;^X}eX+Q!Rji-Ltu%Lmsh z2fQcigs>GX`xU^Yp-|*m&D{UGu8Nhq4eQhzg(qT#ZKIA$(JK^rl=Yq#l1CboYVV7qE)acNu3@mFh z9P0Md1t}Sp4Kpwv1;Lx=yJv(Y(8%Xt6_n(}>#rTdZ(?IuqjO?b!s}9P$#Q_`UQW&y zZ=!j(-wGz2QnKDHzuqmam65{iaD%Rf^$Tnt18{P^`zTZ4(7bHF+5QT+ai3tu{xBHw6n*U`1 zfEsz~U+O$j@gX3i!r%H{*yGG0UNS(zK=F*U8?N*2c7BCp5L2vMm$F6z%HmV}0B;Pa zduu^eN)ctI3V*4=aF=|R`#d%WJ5FRJ=LDOYZM#x~n&-FTXz)`hVlu_bI-TRP4k{`H=v)v0p3}?uDV$sN#_Ib<1FxC~arNqDXL!3zA z1YhHT<(wd{nfvQ9kKe$pq6>rkW==iN)f=~5R%YX#;${SdzfQGIQ(>k>kv%EElb$9m z>rZ{TUGn^UsLWT#oa$%Ibda~~U=+a~0uJ@QN4Peeuk7L1G?~Y^w#?$yyz)=60KDSL z-!}LE8Ed88|Btbjezx_KRxe)JP2WrVK z%EJ14&n&m%W}j6i!Z^$mouQOdbe~b5HH|xHM&Wm2qnM^sx1_`)kMR0HerlVh)yX3= z%xP~!@$h-x9-Sw80Mvvx#<)Y95UiqgE#+$Bxx~q++h3ItE+f#fw`Vl?f<#hUSS3Y1 zXs5gAL&8irbfTXnPf#ix+MLD~&zk>srj7*f$6O z4eT6S&yg!oOHh_OBC38b?QTVVgoT)VH!LfE3cNbC+Q0Tgc|t6m!A^G6HDJ#H+z(Uv zC&Q0ZVHLD=C%7n2y8f8r`G{_ArDAW+o0kLrCQVP%xZ=`2J3)Iy901v?3&o~3I~VzZ zN*r+is*tS_Y7f{lL;EC;UQ!w-I~TyZ?HH8PhZ0@*s*j$a9;Hl5JS9RP{Iw{HeuqC@ z$o@ShvFu;<`s}P=`1n6Ur}uwRA^)7ONA~VX7O~>P|J6AciCdBXFO`#t8QXqq+NHuh zxCZdVeep53WAvbMd%K!|2nm4$d)A)`19LeYKS(Tpm9awcb2Wmz@9w`~b0?c;#PZR^wOUXa(hs zyBxKO)iV@dVdbzx%T>ZnB&%H5r98{LY-a%XrF}k%-I%{2OW%h7d-kITNGRS9JW3Rs z;oukjQ5xYf!#BY`g4oYU$3e3b>e1|?WSXD*%IMA(#x6jBLiBbkR{x)V@6Xjddh_Lc zq}oy3O`+H`gWX#WxN5;vuOEA?Vxs&=XqC1*w;wn9bHl3Hg+p-v(jPc%&9Qcttk1To z+*DEPZfG1vF_hUS9L-BO+g;7a*;dMq%$5Es{!47)P-RX#Rvn;EbPBl`UxJVls3(Y2 z8|+u(=ky`obW!*jtqscBovMWro7{~pWGnXUyWbdiN({1IdNArk$*9;tSm(wuZ#m$j zQo9)UMb`G&De4{5{g~%uPLZasRn0LtVA2$~y`Iby)oS(6#pZ!-4@I>3_>tkkVZ}9yA@MpS=UJE39wFCMP>c~V&K;DAlVevC_BP;+>aol4Wa!cZo-zJnmFoRl)S;{yg7% zE>GR8E}V8|hJkC944lA&yVF`KdCFJ@u3_z{_`cWI3Jh%k`< zcek4L04yTy$X()jkhb|QLH7si?{(KM-y~kRx7hRkEp-sVuIbL2a>g+I8DgZUn~|x_ zEf<0-jyk0`u?Bp`ItY5=9P7MQ*dTJ?tKB7A3RfYn!bXx!_9JMdV?Tp_z}YR(e8%ss z66vdzL&8RSwki2n-tWBMik^0rv@ZNkmNl6NWE(8ldp)-e=F8z{=X+e(aGLvpiM+VekRW*58hWl>pF^0*40ezo0wHsew9 zRMUWn=Zp%KeT=D&nGFe-Nj_i~ryznYq@Z;_4FHDA4s6En5CiBrzkKYHFn{T6IMiAh-*8?QfCL0_YQ->`-CVlHOn3Sl5ZUUxvXS9 zBwGV7_W4huizfe#`mQbxGm$6Z8LieEM=RnUg)&16WRYK0HVC||EnL4|0ri~t$cl#X zhq0c1Dzgie(-&gkS3k|&GG8e^#2BUsqe_U0u8g0(s4~;EQiZ@oHX$!4RRm0Nf#o<~ z{EVKT@1y#eJ)mD;995hlb?Rot+_ty&)yg;ru(RF^+vim3)3pZ#$8TUPcimZWr*uZ= zY~u&WOQOWj;V@}S*M>(Pe#bN&9QU#eiAbT#XTcxpf*XH}ccc5J`=a!q$s>6-uU#gz z&Zw34ar$fblzfceyq5@@n(Dde-MdFo3(|Y}Ru#EBpfe_h%tYJ`vsL#A0R#QD`q%ub>K%2?`8sCkI zEDp#u@St)fPfY2&&Y{#(Mw9V0xntLa-@hsEzV|(L>@vovm>jna1|EeYQXP^BH5ovZ zo%h((gZ30&QMkzVs3?m^`pVY#v9{JK@8-K>dy&%bv$GvWgwhl~J9x5r`^iUX-E*M4m#nha=DLT`2kc6*#*m8dla_Xb zh~Y3QHUr;XBiH)`AtgO#8|6CEQ;B|;(5_lQV}c`kM_bLKHKDElF#u6<^L4Z*TXa}( z-Dnv_GpCq*zxt=VzgGKuc1`hI#lfft*JAZ~IFXvFsO^v!%MnWzCE$A*aP1g$bvN|# zo)}Q8JWXteNLNWHNeC#9XzrUgn{i{$h>@WBmZedXr8hVlR3RC)3h(#6^U|lvG|{2J zp0RK9!G<#ZTVKIgD#DZ`tu2U9hSAbdvWBj~NS`JgrYRw4^Aw>!oi+w_{HYQrGgIi; zMTzhY5*jBW!nDuVepKZqX|H(t%knwMAovrzsGq$0G~}5<<8LxJtUI>T$2(u(7W*2B z&;op`YLT@WCD7H;nJ`zRHB1O7YV&?Wc(`XNHF2`i3b2PfszfHrr_(@ap6OEv{8cb` z(vmFa=x|GZp5lyhxL!vq5uF4YZ%Z&D?bCmi!1)FtvTu2c^lpgKQfR*;el?P*SRJ;`-hAs1!xJGe!DB8W7`y~d5OKu9ue8a)9y=;*d zh|)U=$^@?tU&>6w{lsVok19{aBTv~2WPLwhebfJBE!1}d$k>$o@9M6hOA8O+ch+A6 zd(DHfJSWg1eszQ3$!vFCCot;)8|fQy0C2}?f?56ZL9qD5QPJmm$1^s4DZo@&vf+*1 z9pR||J@`e zh)Jg`b41_s=;fz#Jk=&#K-9~rDNFHRWXb;&Qh7r2xbT2IujdzQH~oE`aGA4rTAewc z?Dyh&S`{LGhC}1MTZVP2sWv!_U3LoZgxh3e5e@lX`U}8_f{5J|jF5z?G%EP?-3lMz z*7BtwKu9jNRxJ{ljAEB2;@E!ni1suo$Oyea;1X0Zfkub{zb?G~8^&~tf6F}%IKA&e zV1YlJ3LNHdRu}1kE+cgXAl727#cshJE_Wlad<&jaLCIgox0ibDtr8a+v1cODZ(o#D zCZ@cZyG9(`#mp6gntcS{&#Ho1^LPwlHtnk2 zhz)5H(3wX5o>*MZNV*eEig{W3T6Safdc38*UL35RGR9J?k2>!CGin3c3pWpoSlDsu zUj*b5eev(mYWTHu+cd`!M|&SulO>x-7{TKfKhy8OO>nd72{NO=yUl-{?<+0x|V*cObq zS?OL zlqk7Jw$U86Jk7oJ{d}p}3$gC=%CNvRav~yj+bA6ymF9SN>=)5*euUs+Gw`q86OTM+ zWWezf0ASVmBT(XWJax1McZYR*`c++4iD5E^gsy#yH|3tq##36Y6fAQEE)?-Tmh}Az z2N2cgfPx%*rRzlJl{*cGaa!I?9n8zI_`||jC`h3;`!b;S9 zeUDB4e#oRYXd~p^!&z+paU+4JublsEX$IB3aye2uZSlx#&lY9uL%6bG{^0WDbQlXh`RHgcq{I=o;rGs1H0Hf3Pd%A_17fH_BdC{MwWA|l`9#Q4M{pJys%vQTfUBLj+ zn*X+irtK^5wPYd@^ZvHkLFD>>Xx7dok%J_k*H6$(Y|0;;Wh-Rm9-Ww+!PeP5PI(kd zI1JLbio+Uz(f2#BQ9VB7!HISR_j~IT@VI?Xr@TS7;B$?DCa|w`rz z2^S`2k0;vQC+UHkXj!_U>y>er?Jl%o$Zugw_ML19kof*O&p3Mz`@cq2$|lw7sP}F4 zUp1GGO14)Ad}Avuv&QvTsP`TJ=M6VZ6C^z@D|g;t(kqSr-^S7O{}`S+As`wx zR$H*2@KQbe+&3)%&F(D)+IZRsnlspV+>Lk5qf(vD(5t_UO{`>!(-r*a=8s6LRzE5E zGz9;JuM@wq2(iTo4W-Kg_EMY0gWFCYaFRvaa*!}2J?$E!96*hQs==TfLeX#@qN#o&kWCM8abN$(MP*6N`_ku1)vGs zziL-~`2xt>Ta~6F5`ZqE+UR3i(DzPh9{De3=(sAg1MuME`$-b-VpJN@%0R6kbBSa%7?1XU3$iUHC$Kbx|V2hS`ld0K`H`fW=7MSVb$rD8*B`=L4)FI zpDD!!+kNYaV~oz;v?j^2X=<}RW4HcPRoHpCBiA*ty`1uN=i%wU;Kdt7+-fV$7xRs! zPg}Y4p5)2EsW(r>+&cLS-%4{qWIAMDlbo3@pM-aMethiP9WxBMkNQFjx8&1*LmvA= zG`=Zno-l4t?=41Ki)Q$>vT@7u%(67`Sw51NQrX-OHz}k@?qbeqMSgIAto4=zbwH0M4Pkz_0D&G z8xORD?9Ii!4Ru}GbwSu3Ve)bz7FS(yQFeBel*lVeJ+X(t!W~U=88)Uw;QbLEn`ROkdDvHia>uS~N-ZCxl1zc3*|Hzj&U!P~`~$SV5y4fdW_SXq z$>C>(LS64u_qyp>=Z@hc(0Y7#;12;kms0H_=O;?D z$ClG?&&i^fK<^nA{PtGx1+egvsA-ZLE(5S1yV|t>Z?<7p112wkS6rFGQYtXe9l@V#4#dTjo;#HHgy{W|#oe&s3|l z{4DYv=x<(#;xHS0mi|x$rS?HfML+^#@TBn=C>1nAjOQi7qU&iVpHO1jTtJ%BuIbKa z{BT&HTpLYQA$LjFwnrlDw0-XU%~d_}2>Pq{zr)sSOiv%)A4uWbNjCt2W!Y0&OK^9= zz6puxF^qgZgCKX-fX-<%n4ZW*8r?UNEO6KwD|xA32WN-rv;Al+er)>DB2(FgLrS8D z!`m_|5*W1J7YzIm+s9I0`OB4rUvu#Mhb!q!hBZiH_GX^NYa61_n~^>w)s(<%S}e%u zXcnJ^#C7+Uf!3vN6@Ot}TgtWlDzy_wGW%}X=M^84YU{}&4=A1A^LS?>tPxq&Z?_4ifOqBo#wfJxW3er#;fZj$@0SM( z-#h#Wxm8{#Z667;D%CUCB!HeK*}g3M5xvZ#w>8G&Ee#epKoMLt0m`@c0;S~ACUW=V z2OA+}GQejg2SBX{kfIdqP#4QxXi%aLn8atq=jI!|-xyQVfh5aTzVLmuO5+cB*LDS4 zQkzI=hSSb<)lPV!#3r)NQBkKFDZhEHwlKNeGV%jJYn2i*y@J}XG&(B#s>#!|V;&^* z1hJb{fxiBEXkUW(vI56Y{#a-RSho}dMv4r~dU!xqZ7lO!SO}+?B0B0-ft~i3L|C>6(f1$uQsI#ify+rfg=k(nTWx-J>TMl-XhR(J`5%DgNSS#wfao*Q=d(`&g=#RE2Po-^qSdQ)63o<=jKi(B$6M-bsoh^rB&fKXgVOR3lh2l z%zJl#v;5=g4=3E2{B$eqGAuFcFShnNC?NUidJ$_zrAk3mjn+7a6SOW#O=4by%k0V(!$nZJiBORK-1Ji5(+d86z+FTHFr$3H_Qm$* zJ*BW}DmnWt$Tjw*h{Pu3dR#1?5?N#nF29Siczt9zDaB{(pU$UTt~C^4)d`tU3!+ie zFykKNGZ$zWeAnT}e^1EzM`q1w2r#1qy({7ifSK}!`n+oCFO|9{jhzjp0J%mqKXC@+ zWls04LK@r&scEgk=-)yi9A=8|j^Qx#1lYsy(6=4-t-K{*I5Z8|ot8kExJ^Oj0I#oE zpRPP{4$Wo@GEtu=?$t|xT@ctwn3QQCxPUuM4B+$$*)N(rli)MiKI(VR9A}qp`vHFr zZ1c@j06^~l?2;;hKA;b~*DH!#Vkny=kD^*rr*l`BxtZ+(0kxEP(<=pYl_X>N!p5*2 z*bOPjxj`oY08NwPr4$x%zz_vc?(_QyB5-`XR+#%)lUpmeV1%6NP8~1c?e9ta_28SM z+Jarw4=wbPfU%=vt{Qi!*%E%+JeS`|rAqeol%>V1Pl!t(7s4%yK^w7-i-#T1hug&q z5O-ve8%K4n0Jc(HVQh{4qwj9^PHYC}Yr!vv&@M1Rr6Z!W>bba&!K1BURBAfHxY~}{ zTtWUF|0jnBKX{YX>*S|8{Bs$P&|5+6W67!`Xxcjae3|XeQCfgsyHEW$e#dJw?DyOp zfd8!O{u@S?{%06@VkzPq1r?xm{bz10rYyQ`IeF-5i)fhaa7&VGI9Xj}QMP%2Eyq?& z>J^T+8{@n{p~NNNZ!(gB5>+_7pZy}JNSjhh60`{sZ7S983~g;DGJ5MDTt}EOFLU)_ zxv5saj;eQ{@N!`y3w*Wk(tS&@UcLf^74~BIEEeX8|GC~X=4Y1?-rWl+a-VrO<8=9I z{15>jFzu)}t}@k2C|_&Rku3n1@*)@!@FJs&rTb_@k`Tc>0QnBR8tt=X$3rD&qBt^9 za@lu*x1fBeh2xy~Usr)RPM3}vMuhLlSBd3np!Am_^7iK2lUv7-a%1kty~qy{Rx2O9 zwivM+)HVbuG2vl>r+MJ1Yoh)$SnyGsQ-v=w=?9F8JM%IOo6s5F>B24QM8J zPkv5nfFm^%BSm!0mx+d#v77l@ z3KftDlt7p7ahx^$KO<3-Xczlbk*8 z`qC9`2^(X3^pVDzIj%mrp_s1b1Y@fvc)67{sCs}j0Y0OM^jXA)TR`hoCXr(vnZZzq zTTku2AHHK`6f{!SCattc*}JBHTkI@ zel|L^rVxOp78;&h8TDumZ>B6715~8I)4y3c*M*#kwvPYcQ{4WKhy*GD@F_Ae|BFw7 z{hu}|K2k@h4`@(Ia^K8o~YLD;cyoVJMa= z0m|QCTo+}u1x42qy{s2?b;q7Pi{)cz)q}t&Sd&YF1O*>13fUzR@^IviH1|;)E6W`% z&6&$No;l=*jjWHGC)Z{J-80HXHxn@0w?ZO{f?m@;<3QaNMR@3z&(*J+z6*BXCR9__ zt0qR$yD7L69V?YV?xIucqAb8l@8FOFr1nBYDF^-FrEo%$?V`{jiJ)rrZ>EI^;{^cU z*T)P@$R5UrP8*)j`r<60k#^DDV&wm_Neoy+vbT;aHY3SH8Zf};u~jIJtNUQu^(^Qz zBo{btechT=E^m-(P(A@{w|nUx?hbJYbo?2IS&*1O8pg&Hk%|T~a1Xm@n9>nI?}|=i zU(qA_dz*E@m$yx{bJIlLH`b?Vyk^tif@_5nfS=UzcKK~>d_G06GIQ%{(^&o&<+e;{ zwo`J-)Ufkc`IOyw#?B^+7^te$yb^3hJbP~495Fs8FIrhbTx1t%#GMWuuQ#Z-BdS7b z0MS^Za+?>dVqig>)vHhongR<|D(Zy@)@K>k11x%@+7i1r0U6_)>f_=HQ>9li+6_;1 z<=7>hvTX_9gy2?N=*n#r$8N3(K3!fTZ!SRemUq{XrC5plel$X^FhUdTzkDW%inv5y)br z0vZTIk9&p8u$=yrZNt1d!Tbz+;b_)-t@!I?_MFvk9l)Fb#wr55$+f41jWNKSERgH> zv+;<)i!%1m&HE?IuoTVG`nO-%74LYcq8E6uqU&EkS&F~+gUD_egFnC6y!CA&c5?XG-(@<` zx=a*V+CV&xQ)T+Ea3c?t!)2B)-j0c;#LRuGih6+S9KdFl7y~s;@2GSJ_AbwJjRK-l zD^%+HYH(@!_@Kg-MbO?VBkdmWQk6M?@%@8A+f-GP>k9e}R)j+eUxb86nm|;Ad=RcOYTlfRQz27{Dk8-e zCB)`B60u$l>pBiu^VOYv-K+iU&iC2X2PZ5WrnK(le zEGhce+Nr4Y@l6p|g0_!rhM#Hnoc0#68Sqe9c}85BED%dpM@JusOtDg_rSoL{Uybt4e?E?X;zF$l1O{4nMw`{8ac_=(nA} z>@_?8_6Qa9i3f{wSs#mn!(cRyaj#M*QfCQZ#1W+Q^jDhfm`ljLs)hG*xpnKuZ8%4r3?Te4vKkVhGw7=jsTUX}2gIS2N(It~!Bo zD6yl1of$k(;^o>8*_jSp7*%YT0@7X4-)WE=(aRFV5BZYJiDA!kzxsNXVP5hEJ5qmF ztUhjUrNyWJ+R>s^(Jgg_fUb6#IvA~&l-}+zy9-RX>o+Y&x|h?WRM$+eJ4B~Uu>{|; z>vd*qPn@n5Jk{+Lz^i-HklY>-)zkhWXtQu?{x@^OY>wRSCS*44aRs(jA%8x#X&;{t zcun&3xWouRvB)nil6B-*+KKkqg?CGSvZ0{+V}g5LOK>_{fX2$7Z;Wa zqI^vOjMjve<1AIvUZD6Q>s}7aJK0yNHYpsb98QmtAA5KKmKx4COcL%~djRXefd9=xfx+J2U~2 z&_$d_aFG-26i_ah69T0-ywI<$jej)11Sf5-WGT0aSmf@S^tR*yR!fN?sJhWgQ=6xlXdd$6#CU|yWRyktnc;Dsg>pT| z$|&&8aOMUJq@e(UK(13b?_|3{?eR<54Qf6U&CXq1dmoJe=;H}gA!NSfebQ@K)AzSg z^X7$-md%7FxI-yDpCA4Fllkn`6>4{P-pKz_Zd!{QwR?!34L&gJ{?phjXazp|6<%0& zo)G~pkPK3JjoNx+H7WKMCBQ_IKF_@EIEwTzKC4@;-FdlkVDs@n758}sP3}8S#-AS8 z^V2LR3n}VeAW7{`t-z@m)eS%h8hiCiorFdG{qg=QTY_?XhfSJD+6RC?^ciA6oqIQv>QA$mPo4` zvsp92btRsLaIy=HzMpovG^pF%N?dPOkH6(rh4}-)5!T)|4ym9qsM=Zi*Qc^eu#TX) zqb|6Is^z?Ei-Q15rpFr7yB2?>=R4=H>}(@myj787wRargu27g64#Q^%s{!~6RHQlLswiV|r|;ddH-DNWO^ z^Zq+Pf;;rmCr#7b{}lGRaVy1#T05*`|0VFXWCN;XKRinc6w-4NT!4v*x@p2*^|(^B zQGgb`0Fr<8+{6xnb52c^5|pjNjAqnTx&#cCw`Z>0r{*_H!+o+sFwA=r3y9*dND3-U zWh^`H8%8{!J=PZcCNHT><@e_OsYaFDLmVe%}@^=*##7$>9sMv z2GMcB?pP0-$$JSffDGN#`QuaKt90*G@k6Z-;(GqjB@?eEGQY}M&k8P5hvv>EiWhqy z)(yhD5`9hZP&|4b+c`Rr1l?%aqNq!RY!T2Rxa09~!zEszvx>`j%PMz}XK_;MVYy2w z;})dUYJRWCVz7-|@x>Aj;P^|2qmk9eCXf&X*E!-}lNHoV&=Y&PdA^f`1OUZ9Nm)vDWa&&tGjPJbZffsj}u~XYz&ZTzum&qX>3O zw%Xs;FPWp=b>`>gI%%lnH{n6UC*rEr$4p+&fIQddGa+Wz7z?qFKf)h=?+C~P{_BM4 zahp+jV&U96>5dHD&qJcZqI;HugWJ};i|K9KVoyu;gcZ32j<+}L`MxM*;HD!uDEQN` z@IWu&P}TarAr7a`QMB%tqA{drhHa-6r`c=tZo0ESO05|wh+BTsfN=M>N-)m9QEeEO z!TBJHd}Y7#2p{xknhSfY9ri{^i1lZgX@o_&0`zbAY|HD`BP-b$SfL{)3tg|X>Uj7@ zp?;v#QRvl6c9ch`ZS!_w@cm+%)&griA zHPKipHH{5xSBR_mbc00o>22=}tPX)yHyhCHF*1L)>wzRi5g#a9S zd~ERaE@HCjyv*OG+?l(Pbx2Atl5uulUs&Cv__**c22z4R6 z*W#7UsaL_f+2h=~%@t;-_?JlV0Fp7Lw2Z3QZeFc-*UE(=C-Xc)mcJ@>&S4+RO;ZQI z_5=5kQB!;Cs&C4v-a)n?Hce!e`i9}~RUon~R{zPKL`~bfECFs*aC*#wVNDI$V0x7w ziizemLVtOlsB{iR%q_5BxD6wQN%b&o{a_#a)Cf&QRl9Nb=eP80Cyf{qc*`qMrlP@#Jnmn6J1?*+^|R~wX;c_GI4 zyh!AIX8FOikji1iB}at9dfIruN;StHhA`jy+MPq=w@r2ReD?&+c2862pGK{Xy9+0U zsith10wm^)f*0#Mzkkj8-!|0OM!5S=wtidB4e8B@Et-1)sRwVqA~3?d`S?yZ;ojNA zi!+Fb&yuS0enc0?d2^n!&1}2ln(SKtPepXP&pgZrtn;gT{ZqO7i_L=(s6kC%se-(h za`l%7ib?ks2Z-k(Lb{F=3I7jOZypb2A9n#~U$d`u6JsAl%1-t@yO}|@Y%#VFLiQzF zLdKSTHw$Ah_E0HCvW_)NA!V)XTfLY2exCPv-~W94V?N_|{jPJ)_nh<1g21g%v|hcv zyFwa_Ye>F(9XAP(0Xv=z_EqF3gG`j|Nq*D5rxV@GNO^Z@iQ}zN*71^tsoA=JFJP|t z%M%@R6^CpTL}{45^#wXClA;Ho;lTSn3!%GU&(4P2nQV@$mGG(YaKn9@g5nQ~UB2JY`iHT40T>;d_t_>{8j(X?9lJr=JCX5H>rE0j2`B<`Hx> zqgr^)ABL5v8AdIAsA!oqBEkvi9Laj5pr=e{0eLsmC-;c59I-X$c73MfSrFfLJwfhg z6>IXD0zSqGuh{Gu<59yAbB(lXm##%eIb0n$&%BT9HXuY7%NUbczX2XlC8?oI>QZtl5xBi&h~UJ2Cw(D!6K zraz3{4Ip{Xo8d{^iNth>R!>-Sk$u$MX4ysq-xb!BiCSwt0FujlV8!GElV4&`(t=CM zdpNk&6PyQO$!2cf@+oVr^MB8z4tkeLR8vjPP-fj6+bQGHZx_Em)fqC&t;#lwLw;T` zk;SSM6>Z%nFEFIU#$G$y(wSY`Eb*6EK~*lX&_} z2=<)KJX|`p_d63cB`+c9)78(u^0wsfq=nSJpv@2)T02aG%{y$WeF@qss1-ALqYWq;nO?*tnSy>73_f>Y&I+pXYa|p zeU^I^#{1^(kc?o($l&nq%Nop&f4IY>v~7w%iJbVXhIIx?t&TV_{x%A+J$dt^s*s-L zNFhJ`+qc{lhUz`hb*^%D@y!jzE!We6W6?u-N8AmL7(K1huo5H?4h|}&$V8=8t;MdW zbgw=!iaJ?fRMGOQ=Cl5-+zDYk^h*# z?~RauY{)=CrAuYp9~m3+T1o)G3xKXNrVPmP?(2Z~-8(K~Ry}#(Ws&hDp36&XLBOoe zHo4OJB0c!;_TYAVSI&jruKi{{T`ko&F_L@t%W!OB^V)$<`}erX#g#c+?|TMr(QWa(BF)s00(DakQoIzy zrXIzmeTDNCIQUM4>kl5O9;J1K5zD&`ev0jlR9#rNwQLEFzKyZJzuoBI7>$}${`sL9 zR%-ngn-$S#!QW+}hdC`?q>L#>-qsh}k1yC8!BuVM5mrHuA>9$VvPfO=<~}YniWiDc zk_;iM`}(&I24SNHd@Y*TlwZ^;^n*yPS;_lTF3McC#FeXs@co3(Ci3uU`IIZW^O~$8 zz%yoCsx;@ac>3$RBF#KPb7p$!dH1 zBJT>4x`lTi=PZJ(P@lX2pgG9$lV#5+YHzh&!QiiPOr#sRX02%^BHmdu!DdyRrjR?NL~J;)u9LVQH6$Jk^!a{?Q! zq@Y~h>8p>VgSIiuh#Gx+W+6i&NT4+#k)!_8(*#OR{)%2YKKp)$TGdBowEXrb58}@u zum?hOzU}iXrQ1(G8Jii{jN~pnv<(fuUOiP7oRc7ke`?2q>1b}50v3k9xM-??@_)Il zu>We_W)j$l=VRK@%e|qo8dZ~9#X=d?9w9I94agCmB%D~PIP?6_-{TRp9s`YpZQKy}fi=#(pS#^W zvbV|eAyaj^ZapV}uxdvb6Ycbahv!Z!LDr;ZB(kSv#|P_7p9!cIs{)(ldjuZ+^6^yM z?$$U6AY{g@+4lV7o+v?H!BHEg!c8@awYS47Ch4Q*s(-^=QSraa=GKxqF&k$~EPlaD^jTSCSPB;dcE(GtIoDVqySw6 zk+57<)l{4$`DuIjFlPOLP0HJkXCUW&{*H)nNvU`5kyMp9xXILi8D)P zK3m=S8#$)MY+JMDsyo-{G+N4*@BJFS(!s`mr1G=z!8$rYhtJnSRoA9zrOkxvE*(<~ zEg7Y)Q-ZPbRE!vO+raaBp0;Pkdx?p~@tL9Za|Ry_hh$@rH@Vw&g^BcT&UE*kI^w&9 zLAjr(`?Z#Sosn0`_SRPt(vx=@!a*iA)A~|xHrwN36aZ1mw5j^<; zbK}L=NJ|5eaP_Niw@41A)n}8i4Qa9L1TiM;WGa;W@VTlJbcPH((36sq(pYEvuw&!f zjR%(OOAFXK&He=&SM(N1{x&03)o4r4*&l({+Z-|W{L=6>JWAppKP*e{n}_dMEO6*SVwKwp; zUELp2j-3QTF#PHLJf?)i>7K{6%mGBZP9x)4hD*uo4oLkhbTmVXeh%IWP&g%0ugxDn zj_Qy8tjvdukAOYYUuqAtz9A*-){;^_La_;!HY6p}P`lDz8IWemV!zPk=ZJk!_>lSJ zsrksS3ZRhSnV2!>`}hL`t4xfoat6j?0NM?Nmj7v{xq@CvD7(^o@pyrAY~%3h2i92v z6DB=Z8`YHK(K5Z%C)OF6i8_soHCVTiArT&cfDV(F<~McVq<1!MuKbe}%g%;+$Jq!ze0gTV$gfy(1D`@hz(%n=tXJSott(=`8909OVAqDI7 zT}wp)>2&Z z*EczQB)rK~iT=hiL%}}iX*rQ5>T(>*6SJ=uYxf_BtmryLPQ7+sd4Fy`GR+nm!EzXV zpX^j7lBh&+*xxA=4^CzRC;3rBaDQA9XhYd(BfOn1B*qxBH$*$0`QlByR5aJRb--40 zVzKAk8hb3wY<&Rt?Yg(|YQlBoN{-FEW3*P@FbK)$$VU~oa#ItYJzIPlACp)plVsBj0#R2*IK@}>S1Du8M?F_3LLTsV*qYf%{;2;9{QLWD$%WGMV?R?;u}qXngwm6u8&$YQJ{WUi z++eL^W#lveD!Vmxpl2E8X`6)-bX;>ah-KGEd&(||a?q^3nGB6>epz9cK>g;+OY=F& zLC8^YXVxIc=;Iye6gj6{q}pNYon-mm;t3G8hdBenN6You)i-6idiRjfGLw0cboJ|< z=gq2rBK4daY^kPv$PUsbJ;vBr5S~(^k17Cx%UEcAC|+OjY!qtIrgx^l16vYu)!Nbv zA6*^`{#eGw7yG2YHyHd7y{3$OoiEM2EpnKpLR%eEa0F>xiNmw;3wqcT5KKl#x_N({ zy%A=bS7%tK@|_!gI%3oqAN;ehd;!s{*vk1}Y%!roN9*}syj_fZubdgjb{~!J7pCdR zJ#^rYPgzOOldp347-5nxT|1icwxs<$ehw1&*^#(}D-EZLdvCda`ONo>2~a*}Lm5@OQWU*@_-^F<749i!_-(k?H57@IIIM_6423LI!+7l>w za`NS6WMtmc#1g0LqN(avEf})1P&|hsq&<6t%!J5IxA;~MNm4!PZ@fyA$x;l^2KWY) zG80vxBoe`zYRnTp>i*zlR0_IwfYh&jn+y=e$F z8-?r2x~Sod8o%H{--!g8xc5=y{80&qh7UZygpxcJeQ7Rvov!1R8B(*)aPwmqXF@@_ zM-_7?PV~ij(qraB7w;c{AwjG}2;* z3SVA4T`z76nTnZry)RB}7FEzZ)ZFE1AJJK)_K4aW1j|372!K@CLR!hwpa3LM^$%bt z4dfd0&Op_!j;k#TThu%Il-UUdK=2y%&MztvRsKbG>`8WOIjcyKxaPC?68ml~Dk%b_ zMxuSA_kwBb>V5KR3X)9pu|T&%{U-Au0c`V}&L_P1BE8qavU%>4oY{!JtJrf3)VTFk zkgaF74-IL? zjZa`(LdJ~Ln<{_A_!V6d>PjL+T}vV(jQtHPCXRrRApgZKI8ud$=Y~2h8fnc^y0_6( zi`ov#NzeAlnZcn5pHff91qrq|$U0^c&A}m>VeXIdCL5un-yqC4H#Ya<8{^-Q@OV6w zN;{o0y(@cIIAFf?CPvA@@-TV@TU&0u&c3LUWCfu(X8%;fw<}(2HfK)YtoV-8J_fML z4)ctZB(l<%XoRf+4U`b5Il_BuDL6R}ko4ibZ)eD=hnTLw0edL>7JGOv$lBii;CnT+ z-S@HeoB$6!d4b1>Ny=y2*`@-Knm4%`tk0{vAo-}@d4_mXO}toOOS)mpRGmuoNOohc zUUsMCuokW$#%WDLIbAH=oT2MiO!}6>$Ve-%=-(Q9MQmHZ830;;wnlz=(9`tl!lx@l zUeq|ac`jl?%j)3!`+gs?TeQCt-oQDC#JwGk~Z5% zN25O0gJKrU=*lhnmFSLx2;zYC6wiaM_W`g`*@WpAg6XFI44j!>|7$5RJUNCqDmt|DRxzS7DaYKo7p{GCu2QD z%>jt7j|rpY)6vJgCpHxhRU16j)O%70f=JT?!cHECOmRJ__+V1Niove-EO&YG6uyw0Ey4e(JQtr zcr(xBQg4ZrBX~#OGGXpfk+I*o5K!0Fe|US0))VX^Ao>Pio!4}_SjK1xh^$}vda8gX zRRF!fbGTyElO>;Ay|$_TnxZwOIXnq^_pFaV+hM>Y1GA-c*KoToT22hc$84Skh#%ze z+b*=Th9i8O9)bKTUfOHnI+|CX;I5)u8{(5UKF(*NYX#@za;_*jX;})Ul2NV{zg`fI zJ!9uv!PRD|r%(c0s40(!m4auee{vp13!fxwK7GqOy5P#lRqDj<%qW@)o+bNcFYniS ziU6b=o##YsJRP*f2QVN=K9EWJK^=NCKwJ=| z+`?*y8J@=2c%c9}(}|B1+70JZv2606tXJ>sZWN}5zWvl4wlasczcyqkH1yPv`W@zgyov{OBnWw9)Pq-Us%m zYt@4Xpvk#-r+;4F_}ao!a*G%+F5`+Sz(KSRTbg+FHbc2ET6lt4i|r4=2OBG5T>C)A zIj)@?BAx`@mP~=R0vp(ADCy*Pf2^b*eKY@{Y=Uo*Sk%r_YnxP?_PZUo zIJ<;-`q+Rf;4&?jBMokzZk)&WKzVf1^1NER>>sWAIx0#z&h|@pR=- zHLt@r?;ENa=LDm%>&?I%h=KlJ=*+x1A@yP}v8uz#0J@_6xp@lWJtpHLd<4dH;LyaU32)dnp&A*KePL-hb~x0-_Li0)Q-OYN&!Hi;)wby%NdPOP$ZnQ%$N`PoRdENN zgAl6C(D}}Yr{o17*mEsaY{wb03zM@saVm%Cxu%1($d_mV!-*7iNGnGAN?)O`>~4OE zZcg7q0v--$K0ycsY_iG*Y&Jb3{qsb_7a=RIE{kM*Wjec>Q4Eo(0o`$s?TO z{}9198}CqYlBBOdFdoH=-{~)qVgZQtC#lNC*;gB)ewYcYBHM$HDm2zx<}U@`L68#- z^kx52al{QiY)8(;TShgL$W6>pw_4}9{I96utY>Y9IXd~4|IT$n}*)YYAI{xqj^OABRU9q<$ z#{hoOuz3#dS;oOuag}r{F2tX+)hP*5O|;vwgJD>>8|rP@izFmS!aYwpI~9TjI+N@U z0!tVi^r(~re~B=)r*P(Ymf5)tK*XTYuZ;>NQ-`exkh?12>YxCK)Q(WzP^phu2I_8F zH=y~0U_mAv1W2M?i&X|{vQd2$%?RTb+LV$Hczam-uMS?S4S0tBsPxF0s7nmO3q9ro zc?8K1ft#~)e*(vVeu>^kCo*HSYqEltrOVm#iz1vMB_91sU~AoB+JKyrQm7jnrqVs( zM(O(T&)Tluqbg@a0stO=BSc7{hOwz^35eCNW$qBLU66rYjC=o;IzvH}BtAwAAH)AY zR-_MawH8lBB>c>qEao@NiRkFzEHuLaV|Q{iLda$dllxF=p3L`IGrwg%OOb;_F!nHd zzN}I+a=#CLHowH{)j0*ivXUAz+kL3wZCi-owqU17qviJDEJI2!-<0b4_@NXx z578joF&A(hOshk)4>@1v%P?qY2o0Pp43ao8R#o>UCi+m`&s?C)R_#jRbb7$(__}*v zPrFp?0QrC;CPb>|APJ^&>=h?L3R7#7J05z`cCh9b>E%BPg8@@UN}s4nAIFnETnqJ6 ztDjp(5~MEby+FP76`X2+pN5m0>Y1KenDQUXI@@lTzoi z%n}f{=s0xSfsg%(lZU&5!>FExHu!QF{{DIIwA5m^`bm?*r#;&&qQV${UCD!bOXyPg zIzYY%w`Pu^dU52;rxHiGnFih|-c&06C!Lspbp-(e(j zqx@O0_ZDXLQQDbU#&`5W8hvo8`QEp#EoKksorSRP{6{s$z_jbCvy@Xrw1YxH`}Qjy zOZjxD3HQAE1b$9q7+!Jck!9WN;bKX({OnVp1J{9%ZB~CRv(L^GEVy1&tAYCrvoM1 z;Z{y_E5Ir}O((rr;%>y3fj6}yo`09quZ_h8b#?Fm(I+BJ@NYiSZ>p1C0ZPoHy&*Hy z*GQq!1ol<%ZN{;b{Vi;bo9--?ZcsU-)vUQdXUy=BPEP^qn-+T6x7~U$xA3>H*$S-<~cr;yEaR-f?txTIbU zJ|KkyN0lSmth6iWHhX0lOHzuz>hf0pxjP1~k>MRu}U5aJ8Ne?hN^B^0B_hctFyNO z8ludmtc1Hp?!2+YlSR55#m` zW#d#gBUDHk8#+$(xJ(v~OQZKsPNv8riq^dbtCf5jYc4)-QjaFudVJ+gFS*cUeb(~f zAAZu-NR_bEULfn;aV{)D`ivP3_3cvATusq(3q5>~jT;xc6_BdVM)Cg|9^U{og?K5B zs=owcr3bLx$FHMKY)`MI-F}lDd7?B+$Su-woQ1#D97PZ5)Mu4LQWi>Y1Z`-e#+9T& zdtvSKboT+nS!a_*gZ%Yo8V0mhbH@vx5{$ch$&=4lVxMear4EzD62gxpV=Lo0mI4 z_PHuQ(-lfh0y-!~J9xj_dKEl8(E>cH1Q};?`b0YfMdOmVH;GPt%4gzZ%207JSpN*E z1OI*0!)z370C9W8#32*)$x$?tdF@3h!wP=d8+LgLIbwvFv4SY!v6c^b>U#_%%ovlq z?0NnHu*S~YFnmlWq#21O-P9IOh-N~`ltY^DuWzk!1vt_YPJUW=!|^0f zHgc|7D>&5NhRV0XooLaCv}yup$EJoowimPZD1L|*Y4otS$#32kk7*o9tR{tmZ){=Z znLI#k4a#bF)_ZQ#kXtQRzvsg9SsGvn)Yi8ke2#+R6sN4Z6nm_OzpXcA*v5wVC29^6!NfzDxvLKvKuDdq{?|d-gKWJ3lWf zFy*LY_%F2Uw?JJDsx-Z z`$eG>aUfyt-;tDnPIOq|zZ#}L-<7cq7zzn`Dh9Ik?Yr%U+yJAs`$+v= zQUAmx!5so;C6+w>aMPF&Ej}z}ik~1$rET9PmkOEilf%BY~K-Wf{4G$w@=%%c! z_-tcT8c(Nr6IhaLsnVnyyR2{@5exZSpMd(fRjqUiW!lDF+FwMNJ-}^1K}Ij_W7aO% zesRp9Jm#0P!(sGXwfQ@7(S}u3W-KhD#Od{M6`(~xOj=6|e zrZCa>%;wt72{I`8k!f?ciXxkt#7)J9NW$PI$sh2Sy0+Lb1Kax98djtFxf;f)MW_q) z9mt>bm&>+F*t^l@=HGnLa&CCE56S_5X;xa%tqEv(Z*n9<+9kKH8*yP8wTOg0zcyihWA||U zI}!ExxrhS~pf9ksZw&m%4T-N63t%^}|Nj#Q&Tb85Yv9J1NQ)M-Hr|_PoW@Hr^NIcv zkGgodbHhHg=O`#miuA=juS`tF@ztVAJ}3kUeZ8JwP!bLFLkw$+YjT>)o0bQ?&v7P? zyoR^6c6i5x`I_&fJl{J%L%9g}#O4i#k;TAtkE6oJa=S==j0tsbsIAL=PlbOZzu%%| zW4I!{)yxu;g{YJuq4B<$tMrAeBAZG)FTT-aqb45=QPhB7%r+I`9GVDSc`Y7d0@*jA zM^${l!3EbCEOQ62qml1kTpru95iViCtBa5X%43SP+mV5|qU}Vxe%Z;gLbf}HUH>tJ z7wRc{R#6Y`-Nw;%1xWHv%x-xCFKy14Zrx^(|7z!bB6@#c8#i8iQUv75giguwQz8IE z*c97e?Pd^JbAyu*I|@iR=(up|s|>CSrdhN&r|3B`JV@E43@s3NT<}Vc;~}l$HICSn z6PKP(GC9x~zgh@MaZCo%l~eSuy2x*FS=1wjEOs9xkjk>5@NmRbcoREv!$x=w>@(3R z!9--p=Be23xW$7HzLI1ItBi9QB{vRujAlPEgp-|`b2PmCi5JP3}})#q;TXBRK7@O-8_`+IYn z$5-e7SVzl&g)UA7So&}f=Ov){VJfcceArLY*B<8Ru-#mYlzIXGBd%r*Aj=BU3EmH& zQ6^GgxXXk+#B#(uphlw~_MA`qg*Zm-KJ0FAo3Rt6&Isqt>E`W={s(xlGf=>P4JbH} zaj66Praai41DTxzGjB`k>O+g)3wpO^ssGE;f~Nr5#d z6Xta7gc(v6e%5xwoao2kO73wh@=wl@f)sTrxpu2aD_0;p;!4++V8`C60oYYS=-T0V zlBMda2}|TPQ~&BuSn7M%V*Kp7I`!L+*Zduy$;3&$UI;26F!3H;%0|rF3^)>huiRK0OD~m~vGi&GC)?j_`1&T4Bir6A!x7jd ze2=2ea46KES%w@o6*ALmFVMEwy{Vv=(|vyYac9d0wKw-;QExfKK@m6l&fXeu!>KNpP`{2Mbz9Y(ER()Qfm-e9K)gqUi9+L~?Ir zL{tu}$??ya@`BpaBinOXr3-VM^Sklb(inal2p~$r5`lOhRKEw}u|*zeg~=wP{$a>H zYfe7~Hqor%ObnH$1q=;ZW=@o{Ck8KPBaMGMWBAT@7FIe#Lzw*uH`?Ju2E z#5B&zpf#{jeZlb=TWhGdS8xw*|7h;Wn&b+)W>Tkvfu?U9}3jA-%?lX;ad>a zi@ZSp3jEpP3=I?+g(R_`3z(yXyM*f2bBGt`rLt&L+U}E;ANY6oP z=fj*+RYn47B7cMSJc*^AN)Zs zT^D%7G9 zB<3j17FthI2@=bVm-+lw+~U0w6{&bZt6K+2A?**YefWM!cMeOLPwmxypiAw7qS&3; zNqvDyOAbCd$4-_zEI2xZ1t`Lr_Di{JryTqRf=V6!TQ z7{-xr<&L2%s34U{j*3QVdTZcZG7fH-Y-_{3_JU%)d|HZHsOhzPk+7@r58-kqhS+kN zSfaEU%+KyR-bi4c`6HKNA2O4!g{lQm&2-bR1S$T9A>#o@*>3mJ*)u}h`*VJNI~xE zhg3O1L~N6j z03DG=fS7bSF3#+5reekfd^Oe>mcusKLFOp4upoQ!LK6P_2K}@(YBp3~1*m_4vX zoP+OJkvZzdOp20yQ&yd`kkB6nKL(Xt6B9&D?v}X|wT9uq%jLk7&%VqZNH}D{@J;kq z-GIzqadXHj=ul}eMF*nkb7rW%)fmIM|G8FbmPJDMxkuoK`!@4Y!+I3^vbFn|Z{}41 zSa1fMz6em8l%mD}GUQoBj+maE`XW{waQvFi?h$IEmFAc5OAW!?Wfh3S_}=8P3L9m2<4TCtOR z^Yz5p#Sjz5rND$uHrHKRBJA8OiN$!Elpr`bFF_RUDLBL+h)pOQB1uqFf`_+i7Tc5Bwax9ngS z?f_hRrMJk1jd59SPL@3L&tu#f0iw4;y~SQmBt?x+s1akTePr-2(hNTxn$J6EkQ=C58$(b zfcrX@_R?}(A*v_c%2gLQ=?zq?V5+&m+~;2fNV4wZAK%Z0?GHZbanu3RwruP7IVt1Q zqLpk_z6NOL`&I5$(XUPQvf1j^dXQ=AV%7UiGH93LPME^iz)+qUd?dXk%~4bk|c*9=yC+&`zbsaINn^gZyy1KzIXnMpt$b zDsNl|yV^wE6@G%zy~5)Pbf{_E-hfC&?cGm}^L1~T2!oFv>Ma~1Zb;lu?bV|gi5vk* z>yqPztY-};xk{o{5jsv+NI|xe7k6I&Y`q~ARYQ1}6XaAJR;HKHYii!=OmCrf50Hpw zffrJx`}Hh#vDq?jvR0%G=5f{|;PDnLrpXRfT^D-2CCSEAD?#R07G_03hD_Vsv-47A zQm3%*Pmq+MRC&wn%~D55yKrfIqw;K;R{TSXHKdiS%K{1IIc;0F*K0w}zv_Y`>X$N5 z#dLq87h}@)VYdO7u_2^CNvJdlayK3vP)|Qd8w(saPs06Ig1mpRu@Xz>R1JD9pDi=3;2fkH;Y{VH2}1Y1JUa)U(#ve^jY#gFn!UGvEAmB z%Q*Kve@lAW&o2EPx3c#T&MEk}CC-8#j!0__x2k5hZqiflU9o|Nlks9;;`ecPHI&nt zy)UA!M${s^v~XOUX!$9db^=#}WUH3H>D;k?#qU8!!>B_MpfddwrA|Z9FaDU*4v=*P zZ}T0f6Q-S^El&EFYLy%}zBPXv%G7-@4wYj7lca(9F7DKwyV3K*G3IyG;$xRhrXNo4 zs**>pBLo0&+tNbqO-lTs_U(UMVhsPe#1O`e3v4PNH(h!0l^#z6 zq3tfC?t!}ctoXz57`?wcTo#aIzrWwv8#mInt{e3uQ>LO4=9&q)(8!3Iivh=Mm0;LVG6`E=XXl@txAn-q(wC^5|R*$ zxK7!a`~KMe)L{CTWhQFMynaKPy&XB9DmMU3vGuKS_-NLrTZfaE^puM!PjJw%^`Ru$ zho2eSViKhDeAtpnbS+(nMbjM8qXpvwjJ;%B#lIRW&r$;CYV^uvqBGkpA&(-fHkQ4` z9Jhh831vc=HLzSvnzd&YmAayVt1zj6(T0)|M<-ab_J?DrcjXJel$}X8s=3aU z`}ob+$L{Akt!!?Ht^m)$TQm|uL%kRuaaH&E*K^<8)YMVvl;OoEGsc-i^JVfJu?Vx9 zJT-WVE?ilbWa%O6IxQZ~tKpImT_7}Lc~ai&%$hgGufy*(iHha>q&Z%b^k-dCg7m&m zn)T4~pG-HYhAon%wI_Z#5C1qQVoa;s@=P`fP&ORYeehQjC}SvV^wpXxIxbDYb!C*|n~(wTgi4^M)Imh+ z+I|KdA^%fy<=6ef&)e9iYQ_u1P^)N@VnX37$q$O%t9f>Y!s{dB8J+5uJDOvE8**ON zKQrKFoG}_OS!m&q{34kvV(a4S78FBW`aQdnVx+CUgTS;YO$PdHy zdDru(RD0pO@iBWZUG!q7)q+&4V+|4l>G&n}Av71w1@1ScUabpB2zeG)hlLM*(Cnto zf)GxlZy97M=mHKcw8Lj32CBP`BJbDgt2gr~KP9;)z=;y8$kn%eJx8ArTg0KlRB!8` z8L$Ay&)<7tM%;wt2uC^vK$`>PYDOl&XDxemM{v3%rI8V2s+{pS zs=kP8I$TsN-sAU})10eh&Szo9!pO3F{{KkS2%<*evHK&7)DIem16L&6Fq;RHN#)zx z9bElEJ!nK77_7k>A2TFhek(LZHX{__VA`Vp8TlQyQsMTk}VynsgBGz3@)md0?bBX zw+kHDs$7GwlgNSdtzoR|kvA+S;j_d1?F71LTCuad4rg=9_3Rc&!gd>!a>P>CQYgrs zz-sSydS?Yy^3^&dP3y3@hcQ$ot?kivP=-45qIAe|Xb1 z4RuI!Bg0GZ%!ja-hpx~0oR)Y#B(ttpK>!kEZxqFCfhfsvv$wYQ*w89?{~Po&!AUqs z{Jn%AoBC1D4JaI1#u>l$9zU5zcx^KHI`Q>*EB(_%D=iz!Pl%itb59?BnHMtkRfVp+ zzZ1aZqq`?kQ}Vwl|4E9wt6ndY0$F}qJP*54ZGjnYdU7r3mfKr#@jr4Y5FF?vB&w+c zyAM4YsY$jw_+5ACR~T)lgX*!Wx9ETX3xTsGXtb-}fTR-w*W7WG+I?V2HuUE!z?PNd zh0%v80T7N4-)RJ&P*mFk3mCCjKx=Ari5r9v=WSnCR-`}sCy8@%vGQ_uyg+>~E{oXT z+BqfjsKb#pJm`$;a!eVaRA{)_s}O9m)W-97BYZlnNBTQ+k_=?Ln%Ar;%N`hh^J1^M zsnpp1CSK6BWpKQC8Tp zjk*h(kbS#n!*#%J6Z=2Ti%c7;OtV*=uRs}&t<}Hz0E3(OZNQ07`y16c)b}NaJ?>q9+Ht3|&># zz$Q$Sp01Tpz*3g^w51K`M*n%5%OI;+U*HvT>v7AaTNTU6G?VN^wAMN-MKNtB6M(06Wl5etHW$BV@cM$Gm*%~ zzr#gI5S*7CP>1o5xkzP=H^w?BDf;=qquoE=2g==oXVc`*DW`+a9(o4r=0rc_hmCz7 z(eKp6q?ly~p!qD>U2+ueYLXP^2szu&kg>>ZCS)$9-d^kGz(+@J0hJ|jo@>m5v@cUG zQEh%UO1u8lpw6zP3(~$p+TeBqwGDtYjeO=^07TI|71Ec&HZ&w&+zKa0y$*eSI_es8 znq%NRL#9f1*)-QGx`vl%u#R|Bv2Bq)6qqH?SAO#nzS(rHwTAS&uK!IpCd{MbWIf$GWu~8Iku!3%Ns8MAhJgM?1Ch+5SHVKlj zxxsfpg#gI=tGv6EQ#tiN1qXi~|HG-C4?A3Ddj}RHJEZ!iQ!9hY1D8teuz{j$QYqnM64bXT2&6& zv1ITAwIl5~zR=$91xgMw(C&F9bSbzHOYV>U_?$lU2IGVS<&o6nr!88NNHMz3xv`z* z1||=~TIW2S<%JNkFAM&)8_+L2j_2NGys-05lLrw|4Nf4NA&Bvy;kf34C z#*GcCP0K3VsxyTXtDmn*P97gqSzrJ&NIGzo4Kmt9A#MW}{8CQ(5R!N#P%`zM=>@H2 zhcg|}juCGPSlbR)-uWDS_5Cga7$UofkFTyFwH)$zgKdAvtRh>8?OUFgeh~`}iQR|3 zmp_4@{jS=Qf^BY30!JDP1mS>B8asNrX0Vv64~5nDn^?~FzwD9^_f73}PzM%b!0t{L z7y|8?=$o$>PIfRaEIOG@`pl0y$DNJ~f~9Ro8kbV>;-2uO!Y zHw+*x&CJjx-O?a2q=-@~AxNs!eeiw%_udb8t^19|Hx@I`bI#d&|8|FgIeV=h(i$J6vUP+H(kGDj`AA^Vw1aa!|Uo-67!Ddwq=Q~gi42{J49d- zPSW#qy2{M-8A~!Its5*pA?Cq@Fbr!1d?R8!c-5bkX)_8}D_Kj~0bP__)Ftw3GC> zGL53zg0IFblFn*wV+{UUNz_M{T3qz9wORyJ+^i7y09oob-nU_LZ%~gvXRdM!%~vf1 z4#bQ64~ZzuMgwl_z}ccTK9Xed*5~~jpWKaB^>{}4v~@8xrVEcf`ZGHwi?%>nS^Th4 z8eELp7pgC3KelMCb95eF-w*{2W^!x(=~4417XMq162)PPM~=o-qvT_M^^!wEFQXw9Nj|lN!Ey zVrGo)?fx??nC})6gyt!5ivnmj9PoFJ$RWZSgR+se*M2$p&R1noh*1)e_y8)jjU|t~ zF}i&=vCkt3pLGa!R!eY<`#~fgSmbh^IO(fYnE^M~`C(W!aR6YQxFT>b_7NUkPUp^T7^ph zv>!w26JKb(Xwy!)MRLou2~k84Z0xBM6aS!Ma;3BOJNE%b0`B_E2or5BIMCibk+W)c z7dU5?`hRua`rNsAUvd?-E-RJO>1b0z)86P_62Mpf6i}Xe0SIPLOm}R40&mJK(_|FC zZJZ~}@`2ipI6N$f;4Elk3iL4pU^V36-W_EVv5%voPLqe-GDNR$9dU+-^PB!i2CQ+w zQ+5DPJlb%xUe4OBt$t+J+9M04Y1Fg*2Vvf>kA2m|Aa+6|ZacF?(NRl(C|e_2Ig+%#UEI1JYJU zAInW2SQ#S-V?sfYqpWVltv9x>2h>vNxY11ExbC?4XHQQ9lQBATCnEy$DHyixH!u0^ zkV9^uTCLZT)aWILHj{poukhK#<-yADYaOF+>SCpJ+u~o3@SBV z7q=0t0WXIsHH7PIDs2iz!^il>`nqE7H^mteX*@PE4VG8%O;F&YyDz7ja5V@@etnlRW)g z2B$rv_YUMS9Q(BR?-cv3rKL3m`4dS}8Q)796;0W~HmMQma_*jRymh+RlSQ0DI5jL`kn$(D3{5>J zL*G6#YHAgsO9%h+EAKMy^g|F5@P3qsufH++g+18h(ec;A=v*?3!kah{rx)-@T?|xK(qIq%?7ZR!N*-wD0}kGySWUKFc%f| zD%d-MG@uN+FQ<%MdWxVW?ohp{R^ zj2jTv{Y6T=QtivUuu5(dFTL~Hr&M1iPg)FD$7b@Iw_!yeUd#@V}zVuiAcpI~(!Wq)UEVWodoS3qQ-oa&qFg`rmnnH~dVQY~B*4 z7SOY_^|()ZKP%9=R>k_Pz9BESTMj~c{_ZN|xVK%4ZJS)rG$;qJ7j4KYbkJWzt;a_O zMGd$G)ddjLGDNBv@Z>V!4jPu9FswVgNV?a+AA2bh?tmQBWK0gyy~Pn_9&K2^%~Rjg zn(@0*L;^7nHIC7PFg}|rzl%sg`U$F*qCV(WYpP)Y^p;Eo3JkxS1`jm{p-bPI@@RMK z%Y!aHOa{qZ;Q56rgB72splr2#&~Wy z?UGT|Jv(ywyA7Xn-jfoE6RYVGcsbc| zoFQ=h_b4mbPK#+lsu)6zBHA5gHjoTUOMASn(wgEAV^mfvY0MwaBH2uPk{jHZC4t4V zK9dc}J9#MD1`rCW0 zOD^&f&gVrQ=Kz5a=CFK|pIqP~+SRC8;+lUdd~}QB*^jeqt_@9!j_*>q8WL7Ia4V&{ zLZrXm3GsYo+5;x)4&mY;AnIN7;y^>Z|zxN9M%{o0n9y+<@XY(E`1R1&`*e>faN zZ=%!yH7MXVBY-we%K=ORQXWBzyBubfCnDr&0|P8dVSb|3l(ck1G#?gB=Dg2KHQ<)o zbL;?3rmJEKzos+Wfx3{! z+=4?j0C$@r z7e@*xq{=E=oCtWjPV0B)t%40WMB;(=v|6%R%P5DE+N#HnbF}IHrV@e(LTH$3n${j4 zKP&Nv<-%v1Pmfd$Jw|D{WaQT? z9qGuC(hEi;zo%i-Tp8y^gZ&(no5CAOg+60l{ua}^zl+|f<&NbwV7wd(?U0HtfWUTJv8100Qa_z}rS zOGo=`MWwfX@p2ipWN9k|M-iq-KAvHS=8ZXHgc2FXT53zhWB4(~$kZeWGt~Z2$)G^5 zu2rJhRYMs}#>&Bi)~awVIx+7g@`eJc0v3eMsVAu2rs+qS&+9XRMC=KO5PLvqDb}>a zX4G6^FDPIMa|?YI$fn&b2KZQ&0r>ERq(S9cBwjN_mq?qZYQHD{kOKiq56w$M1FwJn z*d(+;B}vdMn!{wEgZuj*Kn0s&bA{;>xDAh`DlGv+s^$=El&8h@@;kIM@a-xe3Z^my z!EX?74nESIK_;Y=iGksZ|NBq%E~{oK2(W z`epK}Te{OzGFg62I9D@~j_4!}CIaVaGr+ST3sb^J7xp4$RY=nwwUm{%+puq%?fRD9 z3FCzjU|H`}SkjBb6sFbJ-zbzE^Cq98M@_IHonnX+FBVv*X!>sIzZEJg;cUU^N65`g6J1<5yMr z(H4y)EvQ#KUuO7CfItAj5G-jn?ry=W+-`dF?%Oh?O;%n1Nc(_$pEb;d3t<`9M!JZK z`_6hCX}`Y;7U9G%)g1@s4L}O(`YZB`Do)fD{x=~Cjs;8!9#~BI_tXZh@)(vJh^E>G zq%nUs0XPjbRBU_%7lK$&-KpO}?m+CnaIZgw6VX%I3#O_rQw~#3PRD}#P7C!|1_rmkz3z@^hHA>45e1L4SKSrr(Cwia;oFsoppuZa2b z#;}rR7&eMiRO?p42X}lLJGIb2C}c#q^V`>zPm_Y+@(MarZRTE-;s!7SJ*aB7VQoq- zhvzz4pNenHe&YWlTK%~Ff1CZ7fgsb0Lqti)$#Uw0TKSzkm4axko}0TWEIo=@kQs;$ znh77jo57=Kff&A0m)GRwnj99L7vmQ6|2|NuTr#K~c>L0-h!K=0{qVfyi*@_q+wX6? zeXo~YaAfpyZme3D)=z^|a-I)=pm&d37DX(O;J++!EWpM2Z4Fqllxel9exl8WMMV;g zW-&vZ5e_BYp&Ia!b}OerM-3&YrOfej%IS}t#&yuq24*RFGim_Zh7!{j_Zl@UMrl=| z>Tm)JALIRAEdM5t<8kCMNrk=FkuP)+l2I2TUkNN-xOuN{IFtK>-dISXJ~4}G)aivo z@d2+VjPPAd#g-#6W2;r-yhk4m6&4b?8qa>5;=$kvW8^giuCw#{P6jousptL+`I?C; zK+ml8FyNQjOR+e~%IhVV8Meub{ENqUo=fn!FY(#-ls+M5J^Vk48tm~W z>}!)RsMJFojO$1!!WZyjo&iQf_HH?w*ffj*o}{W^W6`wDo2Il{Ylf*2&OweLxn#_o zg674h^D77MZ0i`^ofm{!t!di%#WX1lgnQ14Frr>aG|fkb1tnwX7WB(W%)PC2mIWb@QDLCT|XT z8%m8XL+fCMS}U2D?_7~mxWE)kvpe}o{8`d-JH!TRVXyRlzhJzrln6vlI4|$=TYiws zPw78n*M(DeUPirIZAS*4_a&y*gAqVpO>L`jxRZX0gdXG(1<)bDCM|h}b!$z1p`5QA zkbco)LcQXecmxEH6qqp~{wXyuy%@W+`q>P?p|BFTqXvu${aJ4Nr8yS$^TF9gs~PsP z_*?a>uO-Vr9#EtGJ9f|8gRXnFQgsJua>+wpZVjAd9YlfaYbyiQ*6qMuy;Jh8iHkd4*{0^gj`R?b zPIeYHe)(5*r*GvMDdmxlY8{HU615N94GPy1MPmRy<2n~Wr&*Cog(Gvxig z6t#lewK2sY4fsP%p$O}NN5Awf1T*2UJ8}6Ahs6P_gEQ$1rlMz`-6|hj0iDpmeU>=| zt9f%USP|E$R9g_2BM2ZbQtsw2k%oXsk#sa=TFOX=3!vS*=`FV^$wS&P-&E1})s6&z z|4=N`eVQcR{k8(sRgvuSxeF@*e8xzqo&fGSFx)Pk6t!kNXDN#YtTd+a>%)s)yL~jLjOrrE_C0gnSq?LBz$_xXeJh zDsDe9Lpa?8qe5OVoL*Zl$$pAaq*VsV4E~3Mx8yItgdSsz$Kv;+LL-6LRJlUz{0&}PEZhK4H)*I`oT}XbU`TAYVlNSUD2)} z!b{C{?Ora1~xVzz~FRZZuc9DOMA%WdR&!PsxV2d9~o_4kd#UAiU}^Wm*P1O?p5W z_-l^}7Tx~(5pz~n^l-}~!9<{S{tRgrLx_y15CE;n*;-DSz-cOUp5@9|97bO z<2ab$Gyr&bw4@nPsCncy4ei;#ucC~&N%c8zv0d<|ma=^kQFD7z;~&Eo(Lop6O$6vy zPIen8|0l}L#UoDMZjeZcp`unRidf?k$5w0d9*NYL0*$I!vt`EzIyK;~fcfVMLIHn* zYh5az)y{k+5=Tbe3YzsGygk`!wYM+Hau3uS8;bW^P z+v>8eUI{8_2UCB@^lz&nIi7{vzj#Ij*!M_t#LM;-q`1*Je^@+`LU_msa;b9tIxSzo zxOeYaGq>QqKxmMQ%+p+h#;KUfkN7}la=Z626#3+qAQU-^qSurCdBg72fHiMfiS(l+ z|0!P9%{aTZmA~uH4!smYO|g3pA)=KruyC7tS6Gu~V+=d4?(c9N9?oq#QP^5r36ISj zZEcKo0w=t21#rmpq=!c8tN#GV5_S0PEw;qvyYMIUx)Erv&%@10=wm6=Z`l&k|DwVG zn5ii4jC$R4ylEkpOF{2kEw+2;=F^HENl@q>)rD$P;Nz!Tx9Y9g=XLWIP z$Rz`ZA0jE`;1m$oG+ex>_=%c24ys$@);aM0QAH2!zmxjtIo+9pCGqchr2(gKZP7X~ zs2lVvE$x5?tQjZpVLMTlZ*o`e;sT4AdGT&=;;UP^T|uOZDtwjfak$g?e*`{8eapTa zd$@f^1sn5I1h_n_NStUY8{>jmH4AfTK9=FfN;m?H2fQBFDwXRcRCdVgDb2unU_T)} z{2Yj0^)NY1i#Tf7k8OE%{y@PkcIVak@m{V139c@cCE$eg2}h1B=+;KpvA~}w_o>rJ zI||+>?GlQt_{s!@b<2>r8J(&_B9Sx&`^e3MpvG=^4-X>Q>E(B3nC|1o4uia_91o-c=v8f$oq1 zfy;HEBsD%9S)5bsfK<&D0#1BuPGFPqDAPCL*f^mHj#$iwp}%z0nCC)5irD1sSnJ=) zzq4OxTpmS-ndQ9)E;ZTi7-5cHIxRQ}ex^+*LsCGzBgX~=a}Q7y=m|@_3pC)@3IY2J ztb^XGesilo32_b*`IFMP9>l1QHt|2u`u>uSDKT0nMgIJYj^u=)p*3WwQM96_5C8fWo#`24gCfxB@G6Bw$ zYoBS*XuBb;Q=S6F7QL`A)`F3eCNwiuYY02$o5(4xaP0b`P?S_X{_dm7CqSL93oySwuNyGQu}BXEn^+pRbl3#vm*QvNcEJmI z_2$1rw>N-zhi$!AFn7xH-YAJUcH1wWnD)eOSUjqdq`bl8@DW8hd;fj)u72JjX(a{o zqI#CYKOP9Npr(E9;{Cd#+A%^zn)!z#xKL1=1)mw7$yilrsuj}>l<|P!wf9m$u^+%m zRJ0ObZWRJLn$MOmNWbK+IvCI{=^@Rz<}<-H1jE}1LLUH@aDqQ5TekScV#Vpx&Z_as znU}%QOn?S$w32RV5GuO;(I#Rgd8r@e7&O=rZ&+LLYPBJ`g3DSm^hsE6dzePnZDj~I1%C~Li;=^GuN~7D&Z#{>qVDfTC zbA4DpD>Qv(bwy-k(=zZhg5Pw4ReoGlIKSxx^Ns{M!PIl>15QTDRhSp`G1jvG5@K&} zXe9AmHFv{dexS+OAc;BN8V^`2wEQfFx>7qtaMf}4C1Zv>n~SY!t+ zqBdgoCzF00E-5-d_ilz(E!bz*)LCdkmHt6+JJKb1AzySvF48tT>eEzV!3~WXa2>bo`KEIbsfT)g=r-Zp*_|vP1|pnpV0#5CGnW>0xhNk;@5d z9LK2-C9~49RJES2Ow)WUha=NNn*sQcB1V}N2)tu>zGM};n*7s5x=pEm@8`F*90=#P z4XjRGme>fUU^fM8InCk;hyrMCGLn+`v(!Q<*ouHx%ct>Pz|%j&YA?uE>Kom#){MV| z;AUfqBgb^*n;7InL)>(KPZIQTx^*bD(R^v}fbcW@2PJQh(J@OT!0Txr}!uIX@aN{;p+rxxBn(O8|-cVDu`*MF+Y*l_X!F9x(*|$WR;O z$J`%b0eXCi$!%S{%bZyFqvT2HzLTq*|6W9fwfdFZI~xQ%sE=r zoO)^B+vh2^zy#f)i7g}ogE1)6)Qv? z1U29>K+|u^kJhzvia`@@!HC)~++AKNNWzdfFyig<$M~g&r_?p{w&$l)n3ft|nbW?H zi?p6FZ+mf~>JM`zuaT}DY0S{qg4Af8Z=HlJ>A8Q+CdsEp#>a`|95CkmW#wv$BQU18 z83cfw%-&As4p1C7NqF!F_h^L8U>Rln;SYAZGskQB+jeJ7$Ca!ZK!&&w!qw%fX=pcd zlnAPNzF|Gvyp=nfqjyqO5%ZSeX#v5M1XjJnbovpsv8u`3tAD#Fw(sGI7$8(aQ2xK~ zcK%iO6NAZHR8_szRhDnx3f~(^Cpyr6yLx!5?MU$-o}E9oJG)wLl_<{~f$y{0K0Wk+ z{rZy;sk!5)y~h0&*K3cnpsnoKIqvuoh+)-=Q=Q8fW z>wxVtpd0?39H^_RDO`w4r_n61|8pxc#zyrtG&r4yNOm*f)5pN2g>|y<O6zoulg(Sb7F&z<}nUfKK^ zcQ$=TJ3o$gKh>}GSbs`odA=CpG0$BA&;3C0vAogn1J5H+>mlyOeU&C~ij(JzVs>C- zvp-d(!<|gt7YaOfHhxwqHn%f*7amVEfZeq&ji0o0SUzw2(ZNXGzfZ7uNUhK|NL;Tn zND3|#!pWulxX}7V8YmrY;g5p7a{53x(yPbbG5jI;-+dDBKK=wQl-v6G(yiUAE!XeQ zEwcVRw!M?dm=Za}CW!!E?!dp81Mx>4QANDgfSe!|Nw_iXffqJ;7N_G`gbs6`H=Y4R?_Em+NqACpPJY^oWVjfJ8%Qds9 z-v1*(SUr{t&USNRaehoeQZtci>h_f0{1;ng%}A&A9Xj9Zp4GlTyfzK=wrwH2>lcq{ zPb~P3KPcBgcpQFI1+i)1aH~(^C{Egx5pFSZ*hi}?C*v>mT z9(~zk$VzV5ZOAp5HFaPfCKivG^2LZ-4w%3D{7a1}(kmm2!)#6JTx$y~)Nr#y>?n+7P?~FnviiUa{SuIcnz9 z!@vmnGw6Ou!1>B?@(PEdhBF~DYT4LOLf3?02VW(^45%;u(Q*KFy<#E|T*TBTQ%my( zhiBTX@zH80gNy}@F@aMQ0FPPcwelnh4_bMgWJmD>xcdds1|5G}cO0?%7ml8L2B$k* zd$>$QqTcIqvRcEaz~ni%MqS^~TLZRN6rCPjGCf4%N>wy0WG_VVZVlq-s(~#-s$z6X zT7G%%YyJaFi>B2?ZLUOM>Rg}o^ zo4+o*qo^x((^I0m%A!MXJqT@Of+Uk|CVE1NGZ5rm*1_FM@l#FJWZUL3Z$)(>Dtnyc znaYb2kH>A>>xw6yE;CDqsxQ2c*jCg0kx^CqDlg>YvI@4pqMxR}^$@8)IUp=x3Dwo@ zsGi8J1zZIkFU6|aQI&U*JeU!L9NoA~BR_fLR2g{5Nnbh07D!N3^zcM;@d7 z@?HHp#!DarI!(;nA5c<=<{M3%}w~FVRB*T=YLZF{*%%zgO}X`(>!SQ#!Nl1%aX0^77fk4WtM6xwBg0cx;pu0kF(i zy-cDj_KPNy9r7NmOZR_9K$}GBq%3SkzVrwTL(TYU9c?Bbg#A)A0YMYi#l%_yeW`e1 zT545)SrgWCb|#;n_+Jn-T@FXDUsciB(*34BBAdlT0Mp(im2C$#*Gp@Vcel#y zn*`y?qv=t?T8jR9aIm6FD2Guv{=>n#v7JUMFMt}KR%uYJJWRrjJbs1HdT#4v{)JCZ z`O`-KvO%4)o}Wkb6Be{XW2cS37BTGUToYQxPIj$ z`b|?H)2r&p`%{1B;g`!Z+-9@$#ohfb9}V{iz`7IP{ny|3w_U7EW-H(epCOtQ1tnVX zRpjWdXcK6!zo=blkB<3Hl8n-2t$bID$o7t^662ZjM#=TKjp>Wz4#;=6gsr)>tV{r@ zX*GZc3+)4V^H0VyMFR1c$FxJCE4@FA|`!s?NV<>AiYNvT4Hhaws?a^`N73oykQ&SwZ) z*A_qJKEv>C+7B-y#KTu|`a))&HBc|ibJ7aD^VrPEjdb?L-sehGJN(1PPTG0=vu%n! znh-9h(lBkypu?{E>&?5#uMyIVHpXFJ)g2&C zY@-SX$jw!MWtii68Rq_j7pwl{J6%N;=Ttrs`%tRSgItFOF4yyWN?fr{E1G<;A69)I zgMN+uI4ER7lK3TsS~g_}Y>qGB9O47M3rj`OyZ4JB%v#T-!*tYAQ9FM$a|S9_BVJ(7 z9(>T}(h>V1w-$ZwLe{eVJJ}^ z#ipZM_%OVxM#J@EW%}Nye_^97JBf;p5(DXLk zOdNf_mjb+VKC+H~0~&z|uex?%zeyL?@Pv`F@$wkIXTP;{C%N3{C3) z3C;=^210D+o9slDBfbaITb0>8LJL(+(w+g%RxTG>BvYKk<%@v838vTyxXe z+k8ym<8(dm$MKd0O_}dCpU$=tO@Fkj?Fs%#gp)-vLEI;k%40ek{n*P-;+;)nMw2mA z9jMp?4(N4v)%hhX2;a^cTa}v_o*7=hr3hIXk4Xqx@&!70zr9Tea(zGq*Lyf_fw!G7 zqdr#_zk64_x(!2nD9db7;#23BK3`VnZ`QR~We&`%^kEW2;)awv_>>_vCnpko%%zac4%y;(+yS@VGh)bR z0k_JK?y`79I$}#-MJm$nqxsnCw_)cU#p0V(-iNz$|FQl)Jp5gnB&Ajb7PVeDF5dmU z#P>LntIi-G>*w)mg}hPryPMZNJ${o~4YV6ACA)|}{cd{b4FZH#x5<64Q({$8_fZwK zivvr?Qrx5Tq8}fUleov(^(ykg-ZG|UWdUz`Ew`6-B*#>hPQVOw8v6ajq4Gxw=GrQ0 z5X>MS&3_BO}_ov75i^K520L$2z-|Ru6g(q9dlxB=p>_I_D!}{ga zPW3&{`|Bq4%xZNh{>UP|TYdu5De37&wU+K@MuZZCehPYVb}*La97jISKz|$=?Ig_S zR}MTcD%72=`$*Y%&~ENCOdLFd5}iP`nlW`*Oyk^I%^OjpUSJF>paGn(QS9nt_N94S1Qig(JAzu;w(8F5YlrIm zYx>GWh01+hGnoh96oOXeuSg-I_y-XNLq^(a8D4mFXc}ROew3WVHcZYz0~S{j`N%m9 zWqQVG@+P>uHG={T=kz966)DJezP}TIL;DE$#RD`7Cg&_>=PKfVqWmj>I?oXg{pV7L zx%1TjAJMjZCA`}yzwM-c)A{P?{8ye@!?Z&*{QTFC1vi=RL_?wnOR^Pz$bSxW%qi7P zdmkk{(I7Nq<3+ew8VlT~;I=AxN5UKy{#SSd(yMdVE!C7b3B&!M;9&68n*vTjvel{9 z9Tf~p13uj*`U}gSg4Fw*`#hZkae~Nk^k+~BTjv{|PQomNDj(oJ$F|#{pq!af?&Kn( z(HT#YJ9pcBa0U||;9l6+b3yqo=5j8M`wgq+cnL}{!H8~8f2;#k2&nT z$z9i(JZ1!%DNjuX_nQ4^f#u^-BhW5Ez3>oMSAf{#nq*tP4h%E2%o68lMR{z>$Hy4% zd#GIsVF6h*fwBiQowofY=|pu7lVtv*UPRR-;)Bc#Kk+u}eRnSw)p;pX_;L$RD|6<4 z4CQ&Jzlq$|vPT_IP+&+OCS~n+)#O%T-vD*S4}r7X(cNktv|I@idORo7iReL`M8#UCSL_vvIB|!QPKTs%ToN98BYjr4C97^!0w5)O+x^v&emW zmQgUG<0RpHdN|;cL6Apq5f3FBsz_Pz-4k#DayW6M;L~*AAJZ2&fQHs^T`5lK-8JAr z8gN6kx6C((HcU&Y3W~gQ zX8B2ASZf|_1KjEE3PxZw;AZoqUTSrs$;6MSZ4PC~%VIv_rhq!knv*dFjxMk9xCJ{B zyl|5-aszr~hF@}9qpIhUG3^wRryY_(4%_#<#+>e?AZdjk#P-X&X~N4M6xP@>3~S-} zix((K?(T}570Q{`a0}wpQl>m{cVAk)*=(4VPTR-JG-hVb>pb~YKW!$HB5in+2Lj{Xzb(;qTt z@?HAE<5uawpZ)AM(QZjf!9DX8s`hu$BCz%+sGb#}$EJOT`0HJkBJJq;f|h0aEQm{D^rfWxWqqG=HpN1nm|OqbgX!_09U&VWnZ1KdsYGDQ_s zGK{>WG%c37)gs#+k4PLwNHO`++N3C%*vffKGJciBj;WYYxYcb?l zDDcjF*X?VL;scWU^8PEs9`er~n19gQ+Z}EZCdj*9eUhSEl)c)XCwDR@`=bc|Ic|85 zIL%dxpBe=N`C?Vh*C+%%)y(}fWp?H)fCw~cW5UzDNx=EKl!JVRIwsJHg$Ig6@%>30JWd2fE{ zesPdK!JZzAgP`H>wVf5O)zm@DWYq!Fsd=I+a2+<7IFBP;`YH9eLU>eZf zT%yJns&Z(%s_2L8);Om_FkL&?|3zWarnn6Y3NJC{xaIvkoK0}1qq|{!|N>>f7M}S2^UY;ZU%K{APb(7 zg|*^I#(WNzQk!%+(*%s)ceYbCw+VqTWT|%C!BE`D*K*N1hpr)*ouNw=o`c8`OJ4Zj z?x>uS>#KH7`H9oZnwwigW%!JK507 zRJ}6rAyCqHe{jpP7qsUvvcc`{cs5yhJfZP^d1)kO<`XtQ-NMgAAvoZu4dA~_U%gMm z_d7E;pFa$Lps`u^){xKSq#9*^0{bn#*?TOdDrCffz7F>``|%c>wZ3AJ0AtA%Gm{Gm znR-qU>o5@djM@@8>trZY8C=EH`nf6OM59eylgMChEBpQD7BQgEhuC6g>}Zdn)5yt( zx7u(UCR0uv+ii%CE$qzEOr2R7_!50_WXmV8WMTEh9U ze-$RRqgd4|M{Wlqe*`v}dN%vxH==cS>yhF2=%;p+zu`6-+@z%FCZE2rm|$6*oCzH) zEVJdo&tAxs7MG@~DmABCf12}_=T$1-KH~RVz69Nts`HTQKZlI)&n?LpqT}{2h(*E} z<#8+`oMdjtP~ge{A0<&%obL}y{1{u3(1FQS->DQwNDcrJp=uq;(JbJ^>6^LIcpyO#@i0+Actrexo8r5$4X2tTo6aigg>kiZmy9tK}C}Q*xnNP7|j68b!fM zC3vQYLJ`3oMQa01_F>%+eEES14<0B#Zh44^J=n$&K-cbdwIs_09-I-0yUg|Ka%R^|b2BzZukuM{pgQSMi=q$_IK!>1fn z!>a*r?FLp>W@Bgx%|Ogc%|2f`*>|?E6wC&xDH$}(6!_`8s<}#?r*;Zbz2>D*bqWTT zz#KmxYyoVyk}qu)heo(lVDB^irTS(`0pPIdJMCeUrX%QO?l-d%C{ShZx7EjuKHO{z z1UZ%I+5@Gp!3_3Yh5@}nZ@MFzj2-t%f>6V=wZhAaC@B@p$VxFc;$03n8{%FjrT~1~ z4m-0))3u~fR$y|~KovovZ6lO0Aie|T;zqr~rQ*(he?C~wHt7*{@ldByF{)QGp8BP@ z3g|BJ4i>?Vk~>L{fc}c@ z{b+mgulQ@j0CY*=WH=xM$2wZ!GpM`m0-7Vu?)TO6`V4TxlsNDJTz}W#oFW|^zFV`9 zezf=!aMNbAQC=AtS1uq^(SXb2cPG`8m}v?+7dh}ijb;_-8V;FMLTRYAip#mEiXoc< z)McoOrP4ntROMsOOGL5fC8DrTvAD?0O#GT6yNrz6V1Rqb(c@Da0VhSAwYi`>M zm}*v{pqO4V9@oYan>U`IejudSPG<))>e??dZN$T(w6oE0VxeI6zpR))cXuB|o#Osi zXSFdQ(gEHXfVluxzl1evvaTcArN9wDPN3^i-MS6)6)oKf^Rpgy-oqdO7so1F(bmm8&X!*ulu z*UW*jg68uEgAEx4Th5a6UrvP@ulOwMt+xIdy<6bmn)rVzVmnw>UljG^g<&EjK)cI7 zt-xDA{2M%~{xw*Ys0e{uNj?ArWCeXR!f`1D(UsALVzB!^2Ty~rBh*#DLBARQC~~hc zs>{bAdJsF@)cks{#~RWGch%S%)xLpi7XKevL0c|>V`A6fb@tQq4mcpKjZZ?9O5FG~ zh*vGy-5;*knBG%^9H?kt2~w|in}n%Ae^@LGmWO;Yf0JvTx(=Ur__Y__V(3rx_<}bgS*~@hRCX~U>Zh#>88V=h+-A&9oN;6+D2V<=(Z(Um zr#cydPt=et%kYbVN2AhpIw|U&GJl*vQg7n9q=Xz2{{D z9}El*eC&Z@TS&Xs)boB#d?LmdTv}krq7L=bPNP30(U^ExxY#@qaiw3nORx#~&|=!4 z_6g&XH)&``=F*N%FMdN5e)2^H}0@X7B*g zAU4J!j;l9pkVyxc4XG4svjv|kH@wLypYRV-v`_2PPnXRHFDHY9hcL_sySa9KFBlCF zMNS6+8=>kK(D?^?haKX5Y#V=7O^Xc>u}2m4yJO0_Uq>jeEdPb-X77O%vSA|5(kFIp|lz+6!3m$=55^`j(M9Rwy(RQHNK>2I@MoWZMBdhE@M;} zN03VS@vr#$k;*nXi6U>qKtju&Voh6(IedvZ(a~zJmv8vqP|^Ju{_c@;o*=5#vS(AF zA2{3O{k13gVNa@dQHt{i1~3>Kt%8+>-=JJJU$ln>ZSe4>Ns`65_ph85KlXql)cO}UiluO-Y z{I-O7?n`O?FI}I>VjdP*p%EIEuDyir;E&FO&Pn?ct7x$pH8S&=(nKOwudS4E1xw6? z2AO_ua*qAl7H{jop7Gr7!zHt9h&*T^YtpX02?*_W`a1Q>j+lxK z-Em}5_2;j;H9-f`vg>T-MfTwAGfH$6w)3u{jE}fZ6_coL0TT$O!rq%B>`JWXA!;D# z-@T}N^_IpK?*$X`fy1~KqhSejWe$^7KS(&fRJ%Dk|AL^Z4u^}TVkWVF(KYFd`)^A~ zs)Jjz&K!1Z0h51ytw_%P_+CPd>D8a8U%`Ig)gZroCe0}Z)C!+Gpko-YC2olC2YdM0 zh1mel?$c`C-)%tbw;JRXvoH|yVa|={99I7VnZr+k(DWSiltEQZ)j;}s+5B*QcI~C8 zW>{$`3Hf4a&AC9VfjrzmB(Rw*tY1T4wNMLIqy;-msqStvYs(7SofsPU_2OLZdY0Ej zZN=zN-24ZlHd1ZcQI#X@)#IjM7S7I8qt2hF4q zStectNc;`cJVvt~qG-X;JhoWZ=-P$1|1)74y1;rh_Q;8J?@Ik~9Fz-;5F=uK8+BM!ZOMdUTPfj0@uHrt z^l0{z8v6T2E|2k>f+=m}PD@@E6`9^TM!<0st2s{p@T|iAurmd zEgVihk_|YH{9!USUH4~#QyozNX$?k@1k*uaKo)SGsATtYg?LXDvdds0b4HUweuJr( zbWWWa-=^q5nsI4q>61)N8b<^&c*<5t8ezF?XedL%BNwg4a1Rfq2*OK^*3YU#x-jWF zh`qFwQYao|)Z-vr@0F`V8CTuPpOp_8PH5kx8>V*Lvc7|d<>hAnmj%%4dPz=Jb!%sn z{rFM2nl`ux&L%bCca)4{9L#7%{!xQ8fZBS{3;lpw$s1o?Mw6o<#DR7YpXjQCc@o9- z;3k?;M;A!_Q=r$h#M$oCrDZYOt*XJ;=YzEW!s5VY-cjQnetf0xKxj4X;j@r*t8lA7 zPeu)BK6PKkJCfUe28HfQ8JT}Y1bVyv_f;VcRjnZ)yVRSmQ2@U1$G{iPd*9C6u71|1 z{fK9R0nF0c$OqJ}W>b*?O6LqFEH!W;X3onVVHRj+A?G6#z+d(n*KnIn9z`-gh-nN1 zTWA5&WYD8mRHG@r6_DMf&&2+p7hAdpdbm{}qaijd6dTHX$F_wI*D;CMmA2Nkvw+We3&ip-2YQK2}f10XvnfW%Ngw4###f@jJLi zEKsaUdJ@`5m4V3<`ghzG@Ca9u8;Y_EsH{Ru#c12)x)rvJI7CdwZHr+{MD|~H-$xvP=Kdc z!5ASIP-g(<24{Hu^}TbHLPyW~0}>wonh&1AuLyME(x6%Ca9m=c4r@ z_{OI&hHY)C{IrsuJ~rMV1Qg3f#sFV=2ZzUw>8kuik(f(A7*g*@qL$RA^snG02uL3q zQR@f1E`NujB-fUrLZVr;1BypNLg_@5zY`k)19d-^%U+4W;%g5@6dYL7!;qf+d;^1J zFI{9%OoTB#k5nLT)x#F#m*W9`6U;gmGl4jv795c0B^90KUy)Llg6cVqnHfI%(nhLn z%M}j6F=Ybd6z3a=_RzNE#5CEn1J?g3=3KD7~iEKaY@?8kvnNV1K( z(2z_|on|eOYJ-8=*EG0M>%kimQ1+L@%Let1HG;LH+wXAE6vuG!Y6Qx&2J!qeuPQTYFO_FkJA267f6^#5?lF2bOr8GA36r}+ zM&TexGn^F`?;S9E_OSjL;OR;oGXm0lrM_MI3(<1$l}{m}O_)u3R}|k*3DPQnAbQon zVlb0UsTbt-Q_!tVQ1c4<zm+_p0JgNTPonAi5G6_ERJDTDfz)zGZ5*;(-m7B)n(?G%Z(ckogA7CJ@*<2z3 zX|&7-wU&AFj@ah`VTr9>uTDqZYBgVdR1l?-Pp;M$$y$tmSuM%*D zgM@(8-Bpxc94rIS{Ut;5Z(Mi z*wEe{-t8pm>83nYGmY9cnGEjAG&qOnz52x12{gf>Gm1gnFwB=zy_qXR$L+gTyxQHp z*D5>BmOL3j5B+9o?YfYeiE)Uy-l5%Z&89idnC$$3a5g{gc^~M^uX>z;Jo_CjdU?ZpQBSChBy0Y^80Ar42S7GK=S+PrR-0Z ziqF$L(O`XO63?&B$G?l*A~B5=w?K^=my%PL+20(TQU4i84k&e!RIpxM%p)3cr{V!( z!ly%m-BItM>nv7iX6mmn8y4cs2LrKaF&P~Fv@1Q3r>6Ew1Nci+1dhaGj$)eOm|#SY zEPx&~*xK|V1;>$OgofET+~o|)5h>tZo^0@12T@XP`aLV#hS ze+Jw{j?Hz+^E&B|tlYh(sv1l97e73_n(C_Z8WG}4_Qr??_cnuQKU(#QyAXTs1GAs^f;P%a*VKmsS+hU^Xiow8aD|PvADdn*Vf=Rcuvqncj4!{&ngR?TUh` zj<0J8vwch@P9^)2w>%m0Vo`X30Ox6F=@dco zgoTBjQG8$qVYmS`h8tDD4Y{p}GwS@Pn;}YVpf4)irk-!g4VUg9NrXeW=45;ugw`X! z)aGsGShv^K5Ik6PRr-ssHkhfkdXO^g2=s0yOXs( zTxW2t`WGo|g!wflgvIzi>YN8~l_{ZJ?O~{*_}xjqy#|J&LaUZTUh=33N;M|(&Al=s z*<7ZEK`QF5&5&HZ1k<(2v&h=k${M#N5O^8>I z)V+QFI6N4u-{F>&!0e~OD{iZ4N#)k6fK`Rxt^1sK6&$heMQroF=scU#s=>yzjpg2e z+EScAwYKdFowIFWYZ*t``f1^UKd40Zj_Ddc1#Rj#0NUA?s%fYla^5#~-AEjQJ*tf> zp}wK3J`8!L_c=pY$wVcBP|F~%1OdIxJR9$12OSD1Wa3w)5$Zmc^nM5cXjPs9p-X`m zF(3VeMEmHIFAf5oekG)M6|iy~Mw9&`NYEifpZ=xVQRU9U0d2c+y*!FplNz7OWicBx z{h4o4ALgrS7*MRmw>%1~?K7*Ptb5fw&0MQt1M=PY5;Vm5E)6f`1gn8W<4;!At>BDl z!XRzY1CKvB3_f2lalF~=+#*KHUv~v2EtAiNTxx8OuGdz$=Jk~OtVOQXQN19xAm48F zTJSxY+xR}!(3AWXlyvT&jD-96w|#p<&}&;A(D;ec{Q`Vqg5o#`l7eA(K|LY6j%7kK;>LeSA3&Z3Y{lA3o-3Wy(QHAZkESYXm;9J zViK=tbSlP_ZNRMn&MDUPVmq3-A(MxG5bKbfB+{~;_M+*_O|dyWYD~B#)-(?;kk8Tr zdX`J2qapDzHR&660n?8cDkRE#!Y-ttu|{4iZ9jbQP_+urRPZ|5WK7wJK4qZygY{m1 z#9%2Y6z~dYh%-{;N+2lkvzTjCL+;A;sO3g`m=Yl_ldxU__YN__wLU=H=31i%1Uryg z?-Cnqy!S`B9Vrqc`U9R>O1mPG|IYB`t@QwAQX z_bYAdNNU>l`G~m#ico@ke%6x^6rqTaEwSh>BpZMdirbyM;$8u2^ETA_q|g{{=4n6$!5w0a`yDQ4g4v62dy;*gI>SxX=W?*svPT1Z!x~bd^|r z!RtAFW{tItI>g?8SR^GJqOV2bTTXv2FAc@3Mz`7hQ(ekneX7%tb@b(=dQ$63mvfKX z3$=0R>Ae>^H2-ln7w|E&>nqj3Y2873E5Gg8yqfvSheb>eXTx62{%7I1D#OI*&=Bf} z%M|wFTrd9uGO~qp!7}~G8K820y~1A90>Bu?L8{sbC}~va^-_0 zzf1mGH5F1%I^^X-P&X`mdqu)qU`c66wZcXEjsN_nJ<--8($sb%h5tgJ8lN_ao`+)A*u=rNNuJYZ%}PIwbB1-_l}j_B zmfofOgjr91r+8UUxnp6hEWltM6IC+NFqj6iY^y8TRWL9uE7h1B2tH=bYz_CA{*b?7 z*pR7tNm@FXmw)14J^PwC!xJ+rwrZDQ`WMBbY>@kXKe?1^=?>}fTlsY8bfm>usZ z%)Ygd;75I(n~++V*4V(6D^HPA>~Z9gI)kCZVFB~svgpc(uSNn)OiWu3B3|34o94)1 zvhx5<*gk0@^c3Fwyh1`64=R|fm3OnAHZQDLDSp|%mYvrKbYZT%|F;Xv7qR=d3sYXz z!i9vVKll2K(=KtHH-pQgC*`isE{}{K$X2bHFXiRl#;Tx|)nODbJ}b z!((A!HLe=@P=wCx0YURE*-72E2I3k>y?Kh7!BC?Tl+8Xb*6l@o56A^Hv|2#xnEl_Sf(3CI#sl>_ zIEG?&l%jnf)`qyCKE%M^wp14zx1&t;iZs|K4)a+B(7+-+~ls;=iRnF z4DaJVzpT%jH6|Gtbl=*l$?iD(tXDH}qt4j$rK^40P-X67DY{d(revZvb$5vYHZ)xt zrc-iOD@UhD@$tR5K_{>D#0wdY2ys2hTi>DXwAiMjFL6^uNYG1oo#~t&!Dy}Pj~53J z`RvH_C}H}2MRnwhxv|;hTty2q^vwfpoE`2Yz{<62G2OS3UAT;|DILkEKb~HAQz|u6 zqer38rFL3X9$wfe2+-r7WC=e5oNtV1E9(gXEMlb^B?q!!c9H{RL>EP=RG;^_%5sn3T1e@nA|Qpux6#Ejn)(yu)gp)c47z ze)H%4U?i@ABbW&sxE(r^uL?v9(HsxF&V&p}0pMO~5p-+s!-gS1wrSq(% zvVuWyyzU}J97XO6uYao60&GmyT>I&%vOSuDdMNM|N}~JAqWn(Prpi)a21rl}c@fw) z9V&|o>tcu zI^#wP1hM1`2kKt6bz&qZKSh^R&bZZH+29d9juqNyykxBH!P z9KKWA=X}L z?QR0>u$nvTE`Xmd)q|gh66#>5Y}}_EJaWq2kkIA@cOtLDN90!rmY81^D}i|@bwnSZ zITWK=fjc3LD1a2$u^7W%8l$Uq4fc{Gg}={&vH8oSk%$q@{sHE;%9Y9{g_3@g|avQ*9*OH>EBn+^$Khj3Ss#e&Ie937&K zyn4h^lxBG-9i7=BE=ov#TX>I#mt<;~`;(+!x?-qL{|Z8VIDuy9kKr~`a1;eA76ufD zLf`QKgKeX5e($>v?)z+;dSGICdEhgtyo(+-XHfOcX&;>v@Va9X45*1iJPXU8i~x=V zd$J0(6j-!D?>gW4djN66(ewbD-MCO2hNXuZlI<~xVU}!5ttVZ*4X`KSd_k^vJOP!P3dh96Bg&_dGFe>m)Zo4)!`X57%1y# ziAmY0DF!T%ZR5#N?wHC}hR6L?ktX0ueM-t6 zxS(|OpNz6E z#jmLCqbD|?6pBby$XqKh8r_lxN6lTlB_b-{os#ULzy7N%ard)kvg_Bf|9Ah#EcHe+ z!H;wjW0Ihi9{^AXXbO(M0S~ihc__oCREYnu(+a<+;fcb@#~v-3kH#$c7DQJLME?2} zf42=J&eW~IwpgF3YjzLzv|h!4*Ek-Yas8(OciX~^wmF^Oa=k%DLwxcU|9X@~L(;2y z((Hw#<_2+GK z`1Z-ybzSx>1S6_*!b4z00ub6%?UNf}=)u@?nQOIrt*|*zhX{<0?`*vvIyLc&$a61g zo!g(O!-Eq2;$zuzGu`JxFQb zt)nIsGokqkxMM$#&bRs5PBF&skJYw%hk>6_Ittt$;4{onai9u&n7%MjWkuePtr^b> zbh~8B#{Be@O=^^j4r08M>+*;-QHeOG|c?sss85E-l(P?F=cOa*+7wI0MqT(0H_=CZzGdm=+NJKVtDz0r-Q zgE%V0kV87kBOP(vtLT|!saQFGs7?5HJbdWH;ZoDSAK?IeIB#2FOD4NEFe_LzIPN_w zjJ#m*sG8gYT*?{1zKH)#Gs(=Q_RW9BIe;FH1wGaN345y``b!(cy}F}l+Kht1|1d_u zB8M3v>ix9o#PQcdFZ6nW6j-6CObN-jS`UcHWCquZAT*Zhfh@)(rCl_HMD+KL;}B5= z6WJzJxgi!VeLqNtl$leykCpEjlF@90by;hKX+>%T#cHU=pSl?paIqvWlmBO&?j0jT zC0@0Egz&kpw=5yEeMObck~z3PiQ1X6_4=>xg$721P2kw+J*O%VTFiqP;_8GXp!Rs1&m@~z}}{aaE4I*ps5 zxV+LLSMOtr(>I*2o~VT}HOtkEO}dRwjSVEFtbPQ1h-4+!-wyqC9uAoXnO~Pn8H!;3 zV?y+@Huedc<(;lFD{*?^>+KZR2}lf;>3swA?^8$^pR`r)sLubX*B%xI5+4C0;{0x_ zcRnyC;1k0o5?S^IN&L`MqP(DuiiK&hrodEQ2p>peo+t;IKoU^c0o&nSD_LLZm3;eY zl76&_vf4bljL@;|H*lb7c9p}ZXEYojAyQ3N~2X55})Hw@LXquhawo;n&8R z;Vk_BMu%ktp{gv#cxXVqUqLnZ{*Va}SJ0%_XQ@D{%wlMCQShfwrN;Sc(}MZR8R~`> zv&v=Rg@A{9GLa7fK1?+Dq}%s*uH$zFW>CMq&z&*py-{ zqnQsvV@wn!EkP76>6)v+f!-inm}O36`vQg-W5znyTk-#}2FwXQj1RtPl4<|2P`lLq zu!2~czJB5Ic$)wC$421@j=#G}aK~fe*t0``traID>nZfXUmpjeV3>>$W^RT*=838N zS_)39ATk9^s1BLGG_hQ@g!YH!JT=veoJyGgI^#_%DHvW+=!LzJc|}%sf#Jb2Ve13XJOlj|a~L7P^uXgAdCiDjxqjTR3;E4{p4Y0t)?@j)i-O z7cCm*)H_3UxY_8GE-yLftXF;|%NlWJbdNQ_&6lA$c9l$w;Jw|P8+GglKict9gr|g& z$AV!j3|O>4f+Ycio-H37#jNR2@Rfp3RWe^}9b}xH>QGALF|9d;_-TWuF2kFx*tRRW zE6vtC31x9N!(t_8ECHQcx6F^TEqt&!bs}6g{w=q)rEF)JIQJNx9C#($mh}@QX>Su$ zz$fo|n1EPpH8le!@VLc*9|aavw!n(n_g#lMSuv1vXW)R}R`+@e&ctCvt_Id0me3*@ zz^*o2K&!;GoQN{2|8AyrZ6W61^`&?Va|fu|qH?_fMz~tyi?{5OKWSAtt-vKl8y|W= zc9J!YWkgQNJC)AAzmEzu_WhT^b1M(5@6J{PY7A|N?eRkas8OF1%GhL7>gKFv=b6`X zT=R4!K@srO|I~^xDIu0yNO%PvufR@@MH>rWvM4g0$8ih`6k>o3fOLxCQp-Q2x|dKy za^%Hu4&L*}&spq83pV#Z8b`#ELqeCRU-JwunHpow;0>5&rn4CD-}Nm6R)5|@<9dAl zLqxerrErX0iM1eQW^7H2*zmBde~5uWaSlne$%Z1!0HA*mKNkQ@rc0|W4%xrjcSNgd zOjro^;c3(Td{7Q)^Yhb%mB6Lwh3m)S=$cdAwNb6F_pw~Tk{a~A1!luD#HOb+pZw;A zfzw%oJ0;B99>1OSk$G<8RjIs4hi@uYbmbf1i)5yXXKg;jdUTdU!0_!uNd+KTTeZ}b zN&Bz)t@>Z{o5z35Z^17AHNUYw3SGd3d_P}h`NPE7ajPyOJkiDX+do>Y?x*m6}W zxQH1n={6C3*7JwR3Kk^}mW9K~%4bsH#VWR}(T1UvC(od0a%ExXb|2vQPdeo)9PzEd z^qex~VCp9_Dqz%u6J3O3eOfAD4LiBj;uNE;!|s7U6%H9ndhq&`S01bS5iba1Gf!5@vD-Di z>EXk`r8CQF~awntAYS14Tu%6X-A@m(-~t`#t}spMTeViX6@FA&y~%$EItaIcevXaIh$?@ z{EQL&F#yP=_0+1*PgZEqE0NQbxBa=P)T^e(;b{FVb2P|6-xx3Vnh3FAAV}@86y--N zFCOL*yUA8l+8iuwagp-o45Qblf*Y1p^N&NB;9j5X{f4gPV$@C2d3W)LMwuthDc`BH zqNmU)UtdYTLCiKh{l#=Q`8X^nK0cb=!FwJi4{UF&^<^nf*>b;Omw{!7^A{0+9q@(+^ z1YV-KD@8frZ348G7z0ekDd4~{4-v5gtft+#HtOM4e!%rb^H>)H=@GCa4LmlWIzjIF; zU<3~c>lAY)!>8yClMvPTTrm(lNGSU@zv}0vg8@)A8=|uSKtJ2P@ex3D{*0yF`l%T4 zND+h))#X=z!!Vk?U55e*Uv(4GQZ&fjV^tzXZ|7$LAbFXDbC`wDZq5?IP>F7$|Mta@6PuBcN1?DjPykzu z5tYF#dQBgR?@lMbMxsk@ZGH9fx$HigHow?Ps@S#rov`i> z9Pn+OOBo9n$PVCV<(jR=uvv z|K04~1@Bcn7}^qK|8;fJM4$p8-_f;s!a)N~nV!+s00IA7UVcm8)l#D5#AQXI)c4DZ zt4-rb1C|odAe8J;DjbLyHek8xh&2F&vGtJ#LORtTF0^OF<){pjKEYzhG-3BT&^o3O ztj11iM!BFVUhna9TbYU(azJE}eka2v&KW?A7f5oCGPC$|>6WxcrCv8{>fo;+-tL#u z>@BlEJAc2?;zEDI;Fc5(nO&w9?0W6?$s34FIaH8gCg*_0KZ~(B-G!BTvv8XFE0PMN zg&>_OT_SB1b%wIVO&w{pdiZ-l=6gn1j+!={-hR-Z)GVAzPcFFA-5Z#omN*#$prj@w zI_W$H;7Dq4B>${ugcTWA-k1zty?%iq^f|4{%;Wpn&X8%gus?Vhy;=dq0@?Nv4Kox0Kg5j;qyNR-_wwtu8a?6ylN#ZXF!Eu~WbRE;+Ee zRm%KTJ4`vap_(Hki_B4=-4}X^c}5aivLOow@7eCNjDt@BE!MGj-|*C=tt4^;U^lj& zsbRjf-9(c6R*4R4rpXOll7}y0^O-=k?z-iyNe*mYe1q?bOU5+DhS!Sz2 zH|F1sWrl8d+|e6=b(N>W-x`M(!1&0dq(%H#XK#g_U6s+WDG7;>g^+8Cl_LNWyxQ>P zU{i{n&0fpXuXK=^CCeRnF6LGWhD_ZAtJIdJnZ!&4cU2CBlIWyw>@ulqM`Oeqv zJ|^v64Wf#l0N2ev+F;`|fU&ZCw+&`~*hy}!L=7PEdX^1153MoYIyX}`RzC=sG*x@K z#D`b&z@)j|5#ueh*oUX?3B_#9Rkn=^$YwJlgw798PUjKSsRgnfT^$1;=Z~(7Gh`t5R5(3#{{%0pFka@9A;#1AR^F z{c;Vq_I?OV2qyj)TFN3{4veNr%k}eB(*QRdM@!7%6TOhxnsC|JM}H?SmvXpjj$DH- z&z8B-6;0J=v8Ek#>1iUbEugRcdIig(e3J!0Va{WUbtx$@q%ojw_8uAZGg1qyodRA{ z@M-mLig9o`JStA;%9n_EHFS)==L&URCxot9ZWxckZM_&gerZ0(bN-*sfalH9rPIHi zLBjU`>I`~)e6CEdB=4`iJ4PZwH&wq(fp{0C=NssCVUr^HTJ6E>v=XC2ZtVoMttYU% zZs*_=!(i*~8}GH;XABK>ZmzqDm#+*(B@S+K!wAd3f*k59lrLo#@*NVjNZ)yl*QiOF zFMyNE1DFvuI2q5yv{Eq344YJcHV2Vd5^Og;aFLT;h)bv1d}7n5HQ>$N{}HfzDNbw> zm6Ij4_8V}ZX&S5Z&t|`Q7QRxfM4zubA3febij(tF+UIm8Ie08L$Wfq`aMznlBcD>Z zFQF;ed-v^Mx<|rP>!z6XO#8i$O9tEp@(<&v7dT(#3EnN4F@^64|3vDS%nAmCbX(XSj;#A#TS({J$fs>a z=+1?&!ky~f+a!A5IQ|_SR{GDy{5vWYj(JW#o7KwG*nk+irN7w2&ulR=#rLoH@;$fs zULsSC`#l06o|i1%0(|gO{PGMNv%S<5px4!!dG(5lpd~=$S%KCbfQup`n!Q!Z>Ri7Z z`AexQ^*2@gm|`H7N{AA0crt8T0SPvwGJIMpt|9Npfw;nDG4I$@nF2XlrVgGk(#Wvi zfy@=r@ed6%t#aExo6yFFP>B&Yh5cq`ex+q2i?J3PZc83L_r}SA)qJq!VoK8Nska`W zdv)D2Sn;aNb>~y@q$r3mVRIF1Sc`qeOZGehJ&ylt8EHx&%<$se%G@-yEn`5TfYi@l zshY$Ju6}}U6VwKGE_?eyFKZGV%*KVFuyu3Rdgh)W*f{^R0HTP%y(;&s#_?nTSNy(- zrSki{xB2?0ZItZNj|DGoy57M>@%aN6l18CV$hLe_Bs6TAN2b=i2|;WRLfQwIEMs@vMGU$-U-hqamXQUhWl^yPn43orpq|>F13Ddv5TCl*EdBBY3*VI z*7Ox6X-Dc_BkacFw*3lQv3LJ=rI_B=N9v20^uUxpXWOb~FHZX6kM`kZi_cLnpda%| z`2|^FX*HX34A#HG{mtD3=WugrrwEk~bPyZ{4CKZu~*=i+AH>YT!WR-FI)Ya~mA0 zOFX3y?jd^9YM)b(jc6ukUpzVJ^$>n;xyIAtPhA~}7lB=b%_8JVSp z!biUBRhtEH>A&mauyr}016WCoZw1O1ub=sSZc(7qHVrUx4Ewlid9};vNoZ0e zZPy-?o%O%=9X}+zUfCVd05hTREMRP-SZ&;V>348GNK|U4WUF>(4rpOiH-Y2AzIp&` zM&2=YuKav8vAPr=5tkze2`${ZHW9E}d|^l&(tgQ#_uUxV_3?PZd2}6wgfSeyp=H*|qLnE5< z8*)3T>PISTu?6vE1@Q^`IXAor2T$p~_n8I4oahBTz|5Q+^fwDTFCW$o2Rl?%A_k6Q zb-c>fZsyyF;CX*lwzkGzOx%v7o;T#R-4&l##HzrUu?p3^gP9D*XTNDPK(U}R%AT<< zTvvd;1^5-jyeOSqt^g^C_$M5f zfc`v)3yly5Q4H^0t3WnR4WzY9&hOlAIB(3AT#xT*l^%Uw#{LZ~-yKogbFKe?#{gPC z+A|#^oaLX>_WUsrKS8FKKvnU8r}%8f`yC+STGPQzj-5{6an8nxvjKB_LEgohpfT#f zm=UNU0!J9GAqszZT@+&+zt>dyT^1OM75S|UH={R|FEuTY_aypFfTrx!A05<@oI+)5 z`FBaF+?IR0SBGAi(amq0pIS?zYf@J#mgGjJ=~GFF4Ous+OZ9Ow?)#5939Y#PooTol z9(_^-t>O6_E_7E0g5fSKHEJ^7XKHvkL6cpH9wJxCZ32MUHX1B0^)Lrm{~EOy5)`M> z@pc45W(?AEYU$Z4EdyT~yRYBga!twXb||SMk@Wi6Yh8*3<>GI>7lM6h%gUctM(aPx zef>7yY-*~75leQuPhARGi*Hf_%tx5L#?0D6)>ZB zro0b~H!VIp`|SSE%)n<6vkN}xU+W9Vu|~XwnkC1M7VsePX4ND2jKn!oOqyp#?jk7& z5mlSV+%ND3@CSB2#Z|1`kTI&j)rAk9K7gXfn$#Nx_l&*xerX=>F-_DosMvHL-u8Z8 zQ@8qy79(C@zrh@OsY6yBx^$X7jNY1X`yf)kT!mrc%Gf}PnWanqu*Q_|R=1b{^$Fo{1t^MtGXJ@;I`}@F*hGY9MQ&RA z#l!^`FIE-+@HfSpo0? zMIk<&;KEbuO(uhi6u8MS8<4}5eWv65Y)c~Qd zRgZp%okkcIeN}~YgCOL8)F7c>*B@?CUE8eV+ed?Wia`z%aW=NBWXMN9fc$W3MMTK0 zc+S{-TEWf}d+Iouc}z!qh~DcaiM8ibJ#D_6r^|aiW-lS3Wk#AyZr?91>Ol}7ePS@~ zOA#%fnE2ul>YkDzE(~DdZ1dNdZKYQ;@!DSrO3yfQg*PnfS(73SPF( zyJ-6k_0CaDn{<0FE!P)>-V_V&YjT##jfk*nzXiNm_%!`nv*tc@duc{coLJ$+7A&9z zdzB?n;~QjK#b>bOG=-s<&IXM43kUUAP@1V95Q09B2FO+QlHod4KUeisle~6+K0b9@ z_Br0^9C|I*ILuxCG)P{3syp%No1-6oe~ZM5{${-Md%Nv%@Dul5m8C?BT3h;OX(NH{ zGHkAM=S;e@KCkVX>7u_5|Ck8uOMJ8c36B5&TiN*H3RaXqS3i+>jW5DQ#QzpM(3qLa)Pk9eaf<7g=B&4pVlpJY-@!`#bIjwS2( zoWgH3Nm~qS9#f9=hN0>04Fisd#{*d9!-x&}ka)WsvAZC>LqfS}mcwV#-aU5@RrGgo zHot7~RgP`Dbk^m$V!Qv+c(mt;WQ}TuCN)9V2PMMiAYEKt4n)=5XK(VGX1WOxN`|qUiHiS-;9<7^3cKQgD+{RDD@k_}o;fh&pq#=k$2K zGM|oZZlv`3diN^f7^8ASO1eL_gh85ray0jlWorhb_t|qb@T}M(fjr#fQF-iioJJ|v z(eF_wU-li}l+RN4JKg^$At$(3O|(5U5BkY|9B`Aruw)~4_RHWb6s7hAXWV9A#9rH; zm?|_xyjnY$&y&ZTn699y4iHk%AUe~tcgsi%2R(ZV8N=H{<1ujYYaGsNK*vK;n>|*lO zfnP^#l8Q%7k4>DskC)_iBWC43J|ewWm&wHSsa)#i;gjwM?dA>VclZn&vnqOnLnXhn zIWf{}E4wdsdPe>@L#12RJOIm4M!4S0QqnNliyjIu);Jpu1YiF|$}xc1d4jX2BZAXa zgUC&!O9Mi{aQi3x7&A3ev+l>ZSQMW7FR#avCWgsVM9=MHJz&4*y=L;vryrr+v@uAV zQvzuWZ0O8&;OJdCp1F^t{#50ZqxQA1+zL?uK?vHO*bhoB<|GMO@dxkY*O{n>)bpd_ zKWn#D+oio?l%G$%tG5!n;J3Z?oyUHf%Ds3stow9Ah|UO;Q|Scf;&4CLDMDJOm-2t4 zBZ&P=EGX8iGw{n39ljN|8azD;wA48CnHGlrY>?^l6LhBDoMI~Fm*9$~!8 zeq*)W6}`;s4jPsZeA+u%XO(ayM_uH~1_9jRlQ?&)8kzSic z)lxV=Vbb<Ie$yV2LCp!IhyR ze~q&reUcvTt*S?3{h;7E!tHoxB!%Bmz&k31fI!m&iIHRX0b68+5!wu%NDTz|mE+yq z9!|9IEc~W}@9JT}cWGPo>&aNv>ers0N12o%REb95!ttL&8&T=WNfcc}?4-Jz(>^Z{ z@C9%S_~r%lI@6KO#OeD$EUV!U$d!w73+)p0r$PFII^o+7v+{VvD zIX7!e+O_smwSc+l6&ykWSH95KggG9rQkF$+`G?bYr?V>)r~XoN{ws!Y6(Y-vu}xa? zI@ev+MI>s^oP=yeJO#O+o=kspz0V=;P0DDTB}jXGKHv`z_Jv`)`xrNfwX|zDN->_o`f%TO)cyt+6fQdw=3;$nxXWs=pf~y z+N6SwTcL46NZ8}7RS~;qS&#pd1@Q6m2m&d83Wo$&nVj50sr8{X$ua^?SV{{c^~Nd2 z6T|XmeB#AD{%}KG-a^^j?>ILF78Kfp>g#8YmQsY5a!7jtCX>za+06q3!)y`E zU9dp|c`(@pXKZoLXg*b?$*bg)eYZsO4Wz_-1{L7Oc3^q^O^!Y+ii<^8g4MdQjfoEr zcjm}{4CjGP%IXZBF-+0~N>1~-*vyiu^6?>M$isF~TG}WiSw_h1~sQu$hX? zi>o~{zG#2>L%ngfJv}!Nd)OK>^*w5zJWKOa9?y^t*Q1sI3E$0=CXh5J-1oD9(}YC; z^nC6=)lEf)`B`C7)ivFcOs9c5C2}-dTMXUj`zsOM!AHgmmjci=4+i^tsJ<|=C6*LY zPP_BTN1e-X7`L0zJsAoYx9H!|jdsB@#h(p@B)5Iqr;X((XMWC!y~PN#A{^Lm4@hNW z@|h^F%ia~|KPMW{ewyTL{rIIrjhkpkM<@Ll*c7Sff3))W$CbxiE+H7nwBM*>o2^b! zr}N0X%R)ZpF7a9ZwJ2EU74?srl9FV1_`vZF<(kR?L}M5V1;}6H5iXJuF8&cNLLZuT zC;j{<#iKGBpK_)KV#FieM^@Shj&+(&u~`EtkI;gnD)HK1eZ)FA zokt(uE$T|Q$-jE@{kIM#eX&W&`CJn7AI6Z`fpLa-Qc)W8~o8cR_>#y zp#<<}198ebhS8Xwk37t6kD8SOnd+uqV_-=l?gaFp2CQa-cHAkz+OskO*|bA3anSE~ zSG7^FzPs+$^(5Tmcflr73aVL}EXpv;FSg`iNc}CRt`yH`(&QRZ9pXyS7gAzaNvdHW zTVMT*&Eo89A~5t44TGGjmiVgn{(#RgLaRSCy(9X5cG@lV%$3i%F===;6jDN-S}Bd3 zx?NJ&%{pDKKw9Yjd!!$JOT-Tsmu}!6S1?#+1ZlKL1!lxb_`c((iuj^o|%$i5P#fPZnplSdJeJGyINz zy>cN$4jnkO2(&!T`x2o2@vO93zuvwdcq{QS4dT}3abfnd{awjkp#B2MzxHL&N&jz? zYeu0NDlWaK=MCgTAG3f!=Zp6j?FBR+UM3=E)|FqE@NSZS7b|x#3rq^WlNiPH#BZS< z`fbc}Qp%1SCgD;DrAAkZls4W#hPQmfI;HPV`@Q5;#q6HeNMT%!!(#H*N`wC!N!2nA zbrHjHIJW|$zZ1e@mbJB+s0Nn*%FQeHm)Hg^V0y@YRp^ekC%lKhQ*oC%U?ZuM$;NgXp;po+VK|~M}Ukh&P zs%L{~&yy7QSDHQS0;4)Mhj*fR&0Q58XgE32^4ul8I;!m{dh+E4H!BapJ6%%mXF)#` zLMG?(d|UrQlTgQkL4B?TCcw~=$*<K35TlD{BC8HaOVYYyLHd zY6ty4CS)qFpD>CAgI?d~oM>|J49GRlF+*NYiA3}M*>(%8j~^vcn?V?0rI9DeBudwf z=wSI2Uy==Lq$D_|f$y>GX7bN&I-F&fUoT+3(#>6ke<&`zhs{K>fbZ?Vi(PM~?i-56 zn-?q>ZTnR3=4@ZB6OA|giYw1rKCw=o{^?ja6PU|N3HVMP8U+dE@b2!ym;8LLjkJk+N0DkLNo{SGdRdg#^N?z=Q zddogi=bcw8JSTTX4@F}6bi>3V8kq9z?SDQg9=vHjzx&!%xy}m5N6_oSfG&X2s!%|7v~4;+YCyXlMY7x`)MvIxU`Kov;OYmnpSbCOKf;k@bkTmq)3d;SFb9UYt3^T%+ot^TmZ>%UuKfB(?Sm(5{+yEb zj#r+W-AmILE`7}R?FdN}VRv->lZWP|qS#b*iWO)4R|Rh$*zl*mL{vaAL&|HJo^=NY zgO)D!ah%1wjQ7kAA1oE`bA9fMKAXHcBuW`@+2b6|h%Ch|k1_H*|D|?gq5-^J-H*SY zOK{P2UJr*G3_(3Ac<_uXfFGs1p2?LJABJU%(LmesIQso*D(DZikRjCf2j$hb+pa;h z<&@91uQcG$%{$7LUGxo5jnj}{5Q9h>lE-e9QJ3olW9&0iLF!UHD5oyP2IxG|9l9K( z73v37XLa^#MH@!au;tz>Qh(O=ImDYnKhL(ibxw7X@OVLnwk5QYp^M);YCoCSdpQ)B zQ|qQ9vMqX~lX;TYc99);w@(=zalazUd$ly@+jaLVe5=b#4wsr%E6H5HpmO;%alcOh z92b)b5oAk{8PE$lHrQb0zHnq!*4??n{gt?!cuU-9oGH&Vfj=o(n(Pe!1zBX-Tovbf z2!MmqVyEhruS$RCTaqEHnE59lQ}+m!k6^@z7UG!CNKD?L5aPL|4}NUXEz0Fr7}l6K zY*?|a^7>)qG?4i)x*qmK-pFb;6^RL__zxU6t0m?nuV|OLx?JG)9=#yXqrHg%{I@An zI0n1jHC@KvyXBY@xf>M@^`$nLp4?`Eu{tkOHZ15cye^f_)hWspU!R#$K}0-ii<3db zX*Od`k1Qynrxqd(2)xj1DfqdA6SOgF;Ptqc771F)%4RSojODaKiiQpthyoQ+TUTTm zezjgzWI^GAweU7~KF~J{ZA9#|h3A?aza@@N?KAYRnU2})Ar!o=zcalj1u)LPmF09P zcsPM{bT%>WFZqk%A?|6{PZ7GWzD2nFV$kN8+NRyw}>u5aNcMYLDJxWYk z;YT>h&4y8`e`I~I6aByf$X%WvnzRF9^1!NJS=Tkm_*fiLfUe>>&h zUALCt`q|Fx<3w~{vbd&Va)0XSdUzczi;2O(MfA7((I8H%)G zCXe#rW8kYNfHu9CAnP}H1d!~YZaun`q9VG{f#S$FV3DhZfin8&4;&X8ml5PkL zi`>h1tMLgOL%oafEggB7 zeEJfm-U-EgCIKj+`Lq;|{vEjM- zBPr(UH>@ajBp011GfCA9L&eMCjKUcRzWfQUw9Z9O-TMoh;<$#;mhR9i9qc|gX05@CMYj^W7b=yNE?@7Kox|j0`YIDXPYkH+1BT|P zdH$^(o3p>4yDdbG=6G#Ph_Ga?4gBU70~0@|1zgh=R~TSo_+Fx_+nb$BBc@kt+hM-0~w$)9BdE(R3~FR2Oh1X$y$C#E@g2 zUAIAnuvs3|^f(d4b0k*@S`_@CfpW_EWu@@C4cwUlKD}N(xd_O;p4)9dypW2LdO=FD z#jLn-p4C^LGWnV@ZvD{=Z2J+UBZBPKX~!}G7kBXc3`cdc@6qQ<79%bNOF&mqB|~hR z$3T+CVR^?#cHny_>^OO@2aS~DBOGy-+4?}6_Lsx#2v*%}7mnGPKnHOt)w^xuQHpQH zAzP`)?uzXDi7y47icWa0ZqWu-?vtC(^Xj)NqThyXdJ2n*-SE1^ zVktHzWLPevIhKFK&XkBI*>4V2I?h=T7|TL^P@z@&1L?Oa>kt#+$D$+%^sL1=K|K@Y zJ29_onVI-I2HwH@=OEZ)%D)<`=YBIo#y8|58BO2rDtL5}PLrwH12@mg-kO=}8b$W5 z1Smg=X{vJ8;V5_;RnGMQ2W#84;p%2Hf2VjeZy?pgz^VJt^+3omn>k|hYIzfCeC#vWLfi+E!eSPBM zBtdYE%vu>m`EvNWTc~Hg*iYKOk0NtYcqwFh+Aj72eUA1)zPD{PG6-8`l}6GcL+|KU z3>OIrZcyZ+ze~OmytB5Q+(ai8djh8qnTqLZ3yC%>ISopzFjAGSB{zfj#p`Wi@v9IL zKgYk}$C47*LGwyR86?`>GU`_P5d0eB8iICQ?PcM-9$5G0w(?l<&=XXh*gibZP4qNB zDu1a|`>O+3uX*Z!6f@NG;Au>y+`oGx6(4xgQd-nitUFJ6lZqA!LL2qIr*ewAj63Ro z=5GC4-&;Sl{i&h6KDN2=nRTkBfbz$(%&WhOix%Qc1k8PvewL{!H*LBj_FU{@m~c<040s3*h7XTQEA(ki=65upu*#c2v!$h|NdTX zt&F4$z@QLNNH1#N=AyT9KcV3btFB}+ZPG~pGs}hq zZE9SG%Tv}hQ0i~ZG~#pMkcU3}Z0`>_pMTm3q-)~bJ_Ts#*w8&MT!L-HGn~lZEmq z;P3duQMu%+!FbD(#yy6*TZVe6DWunq={>hFL!@8`+b2~*(aOMgbW}utZS*&ggS`%V zPhD-nTv;)CY^DMp`($pGeo2T_YGQXNbzgLT9}{zn2|3)5VnRrE2BnOns|}p0yDI_G zi=ljgEB^#@18JzdwPi=>?oOc?Nggq>wc8^8Q$bIJO7*ztV!30keb1(=@8(j>)4Z-T zDIgmXyejda_h-w0n^Tb}=WSvfkEo#-;-avjrbuuz&YAdZ+J^O$N|+|m>&=3_w?WC& ziKjvQ6S9H9r$NQh#YuM-F|0TP~G};zGsl3axBU5Fe5W zlB6l|cvvv>j|+J((iNCkQ~iuda{t0$?F^`nv!Av18D zePJ=%n}9B{WFNmb+)D1uV#D*yfBM!3g~SCKGig1)?EVh*Bve8$H`COiAyiFm$eT2z zP{BmXM~7`19txtmDwX@c(W5%p{a@a^yiT%uJ~}9YGYlUK%G=P&0a9n!Lc%vE(iwci z&sIwQs2x={p5ZA3DQ#_PKNNwO@VM9Gf(HElY1*V-omK;K%vTdXyVaBL3YGZnW=^Xq zwLek$Mtb|4XJ=8PMyOHUx_{b^OhwH2>5y*QiCOmzLV4Ja6oYtWm|7x@Gf z0&pXkYn$z-s;4KyDD9yPE)N}%)4>0v+FT~ceAJeZAJf^|PGy{OLAgsnZzh%GNAbgS z2+E2np9XK3zhH!gKQA()kgLTPObC$-eA`7b;G2Y!FY`{Hak-=0Kssd2L={Wdixn2} zi74L1Wt;9}-cdYZ!`0P)e^laiVSn54vP$nxDs2;kuwG-6 zm2{Z#1>^5W$6^L2G1Rb`S4vzXS}7)`o4z4>(aLZ~txrmM#KrU@gRolc-DaZTcpyaK4+JDw*P`y5fWVW z$CF5(x>#GN!k2UoLNtl4ITr{5yQm{X6k7hish(AT*j5KA{XzR{kXlUwt7W)2#m+-@ zHIuFiE80!iBA+q|A|!<(QC&hg6jL_sWfD4pt7QfC{4W(o3X}=%FlZkO4FvniLxHlT zo2G5Z7NFC6_TkrlVc{o%5RlO1;a0+(51r*>H6oGorjB&%M(Um#l5+Q1By$!gZ^<=g z66$?MCVt*kd-c@d)l`rx#boHI+X6PhzHcJ3(e;~%X&mkIemCdN(Ck@BxBF??=I5Vk zT7Yv+ogRHq#U08p_9ud+7CFJHhO_LIZWK61N#m-jn%YAo`D0$!P^&cLIFFRI?RDKv87@eq`Bvy-yO{Nh0AkD!?_un?mf$*ec@(94be>-EmJ^D(w!I7;G~;7_?Kl^?c)n0oWeD$%lpT97gvpK*yw0kJ zR|4!ic(&@tM6L?U`_;}KZ3*qzWSaz4-4ykF7HC*2D6Fl{Am2U=6yZoyhPI6LFmE7_ zjC`E8E~1M1`x5(;N-^c(w1zziI$(5*0OiZ4{m4|opfZ>hafJXbbnGL&9a_UO!MYBo zZofHiG^2r4amW&^(w?t7mQ_u9zo8xrCfbT|EO{7t7)JjFvmCbhgOK^TO;q#1 zqcnG@O68)c616n0*D(omQO;{{8f2XFh7?)vrptv*ynnS29#cX6AC8NI4vobi_mrc% zGSG;axJN<9n9KHEu2P8w)B;4K66*04H<$@v(!?3DFlf|h+Bm{hHwoB2JiTQoq>9^u zrmFxeS<=|~NNeybY-1g35fc{qrDaA*Pya|#pv<`c7HRBvb;L6F<*Ic?!e^-iKKet- zEt#|YA#wAK|Mw%-`~MR|K;WTRMegG4dfxe0b1{^Zj$jjF(k z(N2AU)MFq>i|~(^cC~7j#`My=nRViYbD^cLFAzRgIO^a{O#a(knbwU>1w3}F;L*M(wvV<$fK^D>xpj$#uJ$Q?&&WS$w zrkd0dLm*O8gj*Gckvx0RRdZ>uobXVE*Z<+0BgyCYDW9=vI;0j&a*o-K^8?RXfcA(- zN7fq0^h%=z-$b31$!t@+9q=R4kc|err!kN@KAr?cuM4tT!ay*7B=i{VVqH-6mt+fu zLgjAa({DtD8~61=E$(SaDEiBv)a@BxIYTYaM!K4t&TUM^PN*;THgP0JKf30kGp0Dp!8M} zqGti7HLhE*OfUL3V_w)0)b}%fUwkj0H4BT|Z-%gE7#?4j*)5|%Qs#~cQEUvT5v8Zu zxfHXkc~76hSe#1T=67#)Oh7QNHWk=T^hn5Be|17=$4_=<58@TM@H}N? z8jS5ygd6XoKtn#3>5`SxCHTS0-N4~R1qK>R&-dWnFLWMLH)dxDkh`iwL6HGODUC%V zDXZrU#l(tC^$Fn9s&Z%LKHyN`R+Oiuq3?xi?S!%nL3cwZ#tWW{u(8g8V2$FmX!QAM zKYeMP_x@>QK8S3@AL^F;8Q`4wkm!^46{-X@G`KJt;Xsj*(RYI(HP?$WI120DZhvI_ zzI`QXme9ap(UV|=HGX8`i^lzPbgY-!Q(8Jit814W>z+iP`t_jYVV|SQ|MseM5^>Qb z{nXeP#=vjl{zRmOl=9nND`($KN5 zpn8jw4aBBz_v;)!=ZgAsD)bAM0yN0}daV-UJ!dQJ*<-<*qkR>EhxUQnQf_%$gR^$IK+T$5bamhv8UUqKRP>vLgKf6kw@ zkS|%{s$-yIQg(XN&M}>(O2lBF#1!`UHz_L>2nIh>SMwp#Eu|0Z7Rt!Ozb-*JD44v5 z2m9Y`slUwUZ3?-i`tfSQa5}yHPw-35^&fZ}_Kcxhiy{AfVGMKte#N7Reh`|^8h{V2 zd&keh(=VEc4bH;1l%czkJ1Hf6L%RBkd9cnYttav;s{VZ1JNi?46EirxQx8a@r^B;F zM4e-oSzisU_vcrz4TUk!<+^83=W^EVyBCxPWlg#N^eXm%`1kK#WnWJa?H?XKi0RK7 zZtUZp5faJ~>my*u1q7`9U1&4yJ+Wa$k(S|QbF;1KBq*7hn8bFn0hz8)z<3%4!qv?r z1R{?<3mzz$NEbCfJE)8Tu~%2(+Tg)1Oz#QKD-(A zO{HUS@O5lL4$B4&?f~YP%UdNNI$|a>{`x~x)!mmeqMi6AzbhTN1VccZ+3(LdmvAxO z07X2R`#(z0;mwz?4od*jU{*^)MH7-{MxOL3%!4<1HYTI}benhi4$Ol8u;(uIF+mDl zKKAx1W}>0m{l@ZRUt&;9vgLUzZLDJaoJ z()%&kX!fZ8@t8vSuD5pWWF>P3 zhgGqwmeqD&ihUDQox;P;;NMaljjm&PZ@&a}9bvm5>Ci{ix%fG1V$VV`Y1-!(5~JxKtp7BQYBj&tD?Mo!!{c}d#^cri-3P>d2nvD8X*=z_V}CJdcuE9U zhKYcS4N3}S0-fes*-LOzv++;WyvEo=Zz{|~Z*HyU*Capw`)zmKz!e0{BS|P)1jTh5 zOQZWHob(4atB%2NDBwbMb0R z7WjPKc}6;${LPL4`sRfO2fDa2_q4awd+3+{t)A%BJ?U`zAN9n4*<9-1>PchAN|v|Z z{P`E>oBg@D{^*8(Bw8K#qxY8|D3&!s6TYsna+7P;QM&lp2yioWAJ;7CLia{g9n0GTs$%XIyeD-SVZ>xJgO z3v%AYshL5^rlNW|W2L$erq7mC>Z08#Mk~s*a-P_bDOF~7SWJeqeD#NN5@rmE*0u;`Rw zhK6N&xw1X*`loSq(GQMjU-pEXbPUL$vkL{hNZ4ZS|E>>D#_E^;W^^`X7n#sPXopR0 z5e<4mg!BpmeGgV+Ve8!e{n=-WO>`ZQYWI-AC-idiJztbw%S`Y*7>t?jLlJTLgaS2w z?x%%B=eA`7g}_ulBHOv46sTI~OMr9RE@!JoFidb{wCg^}tv} z+(Xi?m*}_*VxG)d**)6Ik!B;M@aNunM$TpQ*@hwuRH%UuhwSf+N;J6Ib80%nJ`zI*~`zEP-*%MP9sjaTynl&)`=VC1N9Cv7?9@{KPooEJ03P!soMunJ^ zTkWnZ-!C7e2AR~Y5*3*&JB7K;*!IP|Bu3rxF}*Qs4Y!~DW$`j@HLmutI3@GX=-ATj zk}(OMT9atV*%fHdQIa+|7$<+_RI(Kk{0apawv_$`S#BEOOoBKJdr)A;#387p5r|8V zpLi__3Z(y-F@P$?{R ztFERWy!4bPTAD8OCem?aC;&!UsVbXhxQ#W3CtvBlcm1WHvlt~`?v4-JFY58HDwQ|^ zLp56rVFDRGsxApVcOyIow0q7$0~P!BRTX)BnB{?z_cW5j{(Kk?-hr(|vCPJ=3%ND+ z+$mpI8#`ksmCj2t>f7mWz>fV=G3N+?u%k+S4&9p8FUKue?Yx0);*2L0aT#ePyM4UC zlm3hQr@UgujYKWuxq=y8bNq4Uk-x(!7djER3VgGtO!9%a$9{c_NC_`j43=F|MfdI- zr?OHWGv7ZznhvwwNf?uz7VX}NexkQ&ZS6}n(h^wjntFX8Zke4RadAo+c(uN5Z4S-M zo~2q=B|+thFqqt3ZBwK49{)kyQZ@4ILL|3j;QGL7ElnKGe&o<&A7%sKkcrTmWu!Bk zW}#!0N7<`(mr)MnI+H8qDlW_XD5H@dw*IE9>OO1j%R{lFu^~oO)H?)a38bpNItC}t zN?z>vKFkhHfM2g+gJ4Yo$?aN-M)qru$WlHyY=!oq&6M+ry&|#ckn0DXQ~%_`#%6|; zr-EF$RYTr~aynvT_Ih~toi!a99aHl71iYrI=3Z=V-%v#X-1)!Q{C~N0f%55&f+>a8 zb`(DW$i^h^y#(Vys}9y7uu7^17`_9tl&ZSo-!!~mI!R;KI%x4>%cpeuNY#eq z@eDeqU+@zrnV+X^W>EUxy{@haHIY}TobuVxoiu6(C6x=>ebmuVDYjt4;?}MZulh+2 z5uDE_+Gta~Bl!aClP)Qxo;fg>FUhNQ*t;xWypfc==rwS6|A)tK-~-_Ha#{M_K>-~z zj6WJuB8q_iolH+b{tn$K+lll@=Fa8+*_$n@&wVV=hiWgb$;*{d(CRXF##)%h%=VSf zH2toJTS6c(S3A<@=eIfD&@~togwUnK_#G)?WegOPy(%3hD)j3@r}uh+vHe0`$-)pG ztU++*i9K*Ry$$c4CoK;nWe5>dq0?;I(f@2yGsnrinIY-SZs&8h3uZ)hTx^+ttgo@jbdt5Wo2A0dBXIuV?QlY~0;Yu>h; z9aR-bjK!DGc^d9hdL{Vt*PjF%4sF3pwQ1;DS+&p7?&WdnDB}bgu3~`7bj}ELpfE<{@Q#K;K7CA&Uo{0>4QPmjqxGB7M38NSW`M zEJ_6$g=a~oV_|vKPeM=yB3)FSo<@lq@Zi`8a>Ajxvri>eRXH0Yib|FEbSwvu0LPuf{jI=Vu)_CoaA~nz@5wJZowuy5c?-Pj}96TrcBZY zDtSl|NW-ZadBX`s#ETe4L2H(Trty7csLw)+C&F55G;D0a;nU~Pa&*n$@6Wirui0RQ0EQ?UqA&fbY6(Ib<7P#J;YZdx27|Z*VWPV}(xy<|Bc{+~ zKf~psV4TpOUw5q3gMmfJFwZQ`8VJfqn@Xt+cE$*KRW$zLy{@YC%Jz@jGxZDO5vb1- zRjy@lgQ~9S2e(`%9bRgVt=|~WaYS(bA%?Dr&b4c%*1Gus1Y!^ctScLVmS6YXDfio} zY0s7EaaAZFEY7aS4^p3^(qf(&IhK5VN{IGf;t()hx);u_aQp38kiKD`-2HwNT{;YB za$UxqL0DjE|NP}Er2^0ccv7XOqedGbKgLSp(ArWs#Ta)g>R380HJ4bPQ4yZY1mUtO z0Apd#s7{U~m21`P);aOmUEQm1d5Nk>Q;{?2WId4WBfPA1fUX<_PYoX3v|k%hX+){_ zApMvf1B`GXP{>FsfYv^R{sIHmmZ?WwhTEx?<>-7G-y|??Zm#&Hy%c5pvAgreosTDvqg*e`AqQ@8h3W9RIj=-UZ3+hn1ZyG6kFuNtx#?4?z=R3+Vv&R3IjM zxEUL3WOBLk>)Z;N`WBpFRj&0^(ZeqjFZ`~$Zdv8IS8qlxSK-5U<}&5i#sKayviFju~!u2cy#Pun}E(6@FTDO7@0r4g_zidkgYbW~e_ zn4OUD`V&@^0NdKN#q^5l$~L77*>#V0(!q*yaUBcb3FU%;X1vLWgvbj_K9+p8h7 zm?;#4nsHvUA5>AMXXnc3j(7nEW3kI1EGY8cO0Stwpo*~OzgqvdpN{CpVALY52-y30 zaVh@sc_EO>d<=koB3G1CMm9;APXF%t%Dv0Xg5SW=Ar0LRwSHC4+FWH7mc!JDi|Qf) z2&$K{KdKZxnPM5~$6|>JF$<@oIG-*UY*$+JL^4+>!_MoH6Q+_E_SkfXD&LJOcP5Ir=d*Lpx#>jue5gwyw4dAqIw0Wwx^ zu+MBV>s`2yYjuFY%`9t^CUrQ-$-}B5^C){|?)AZ9z%t6Hd&I4&=WAd)E*0AOhre?R z94!x4T25^Y%NVMq)+L=3=dJ0lbX&3JR^j3b#tRSjuLC%fybW=>0l1nnE{w5sOcc0o1C$`Yk~|4jGq0K+_WuqDC(BlVn7BWn4}&;zZB%A%yDp7Q z(BO~2kbhydqSxqwGK+jcJlcJD%b#%)p})aTHnIPB9q4~g z^Kjz9xs)?Gr)#UDN0b)4OR5I@AO*+#tD{OksJvZqGJHM^os&4~_qck0XPvBSY|oey z)}!Kr0yyLcyt1OBSty;=Lz`(?)b;XiBQTM-$k_mjItnEZF6`RgBd$uN3F{Ol6w8VX zGed7ty{p2l+znK4_0Jxor`O)jc!o3mm;H^5mQ$m9J8q`W?B8Dx1vXXO3TrakRiS2{ z-(LS`grQE5=5-G;Jz`M?c$jAU#F>I2X&A&+IX54k>$Y=yk4Psu6~(hIjw146rI9K!s&NY2{E155!_gh7yE-4$a$yEVq{h~i7KpWP|T zx(&K%G;C14(7n*VxIwzE-|KSmiRC5m4|w)|6#9UpX0fD=)Y~MPZ-J+gmjj)1G)JjA zVPVy)Dx@AD8YA_bK3U8<^C?TuoXLQI^yl0^G}hz7|I3=91R&z+z=Q+*J(&Ux3RLSf z=3$jZcv~r>PIM6P?^ogY)ws}N?9*xT>~Nh*U=TtynZgd;7`VGLK#9`nL4v_2ZziPT z;85bXN@rQEE%4boT2r7wTj7END}h!TXE=H)EyPbZkEtDfSMDmD zEG}T41?lQ>GEcixq!ni81(QR|^!SH(v!MW)`}#Qg>BkN){Y&kdW6_fa3-4s>wMGGz zgNDXGiC+U-1%}@9`A=3JZY?gqf1|Ts)JsMdQ%tsg+K87zIB>RY4b z3gVBZPZ(aEA>9IrolR*ve9|htuZ~M2OniVrfI3Rk#fx@8=0NAwTe>KAyxirWv}e*C zZ)jzG_b(Q3I{uN+rsdNlTO>mav$xSExNklXno%z|ZqIJGXj=lVOQxQB%-h)McO@4$ z&1{fFbauo}7JO|8ENQwwP@|;jy4zUq5k?jxqv``SdVmaSVPM@1I)NZ(25lz1~izZGpKprm;6Jmd&;hr=n%O55QRla8$`)YY%(d$GU~8IMAKk!Z?5BMiU-Nq^3ZJl(JF zU^P+*FI4p=kS}kCeKLc7Kgc-hyD}fm1#0e}0S$hpZ?+#?=%x`XcEiPqDTsR5P7oQV zF0)biY1^(Z-Ib~(2nf8Gzy}7Y>NGmYF%2b1g$qp5gtV#`!~CEGb5gG3sQ6YTpI8g+9qDC?wusu*jA26}n^^e!%pygAU zqn9;3L*rfblH|?V*_~t-k+1SiF7OOvnb0Sx8h6aIS(BlarsGw#!@Kwz;>s{lV0`TM zXWjMjNP~J`^nzH%?aK+pg#*Lfg8cA>k(i$uM?$1ZwU_rqQ@nlA7faVGDm=w#esBNg zJa81F-zq7*VI)i)$h!HXYxexdw4USwWEhiKN?8Zpq+!3`ZTHn}@dR%fhg!ion@7R9 z6G=+aZ{-a##%XC@yy?%B>2$!pNvwBh9vrzY88R5*K5K?YRE5XM;?Yp#4LxE6hYcM<`7n@>@%rm)62JC0-fIcpBA&ZWFKrOala$A@{7Vt_Jb$n^?k#6$oo`joM0xqYKDhRY_9Xv3BcL~nBx1ovUg^yBHad2G=F#W-ps{QyB_ENt| z7k|CDyl8Kf+AFzlE5+qJYD#N=m~(7XAWs>rp2#gP3gQ^ zs6zRFF1*xDI1zBQ-h1E7{M)!|+{phGjo)3Vk5Wj9;b+5bYee8@4Z{=vKg&o4-itZ* zrWMbd+kx?ux@L+y=Qi6&JkDI_>(#i{EX4r2SA|4@?V{JK04q)!LQ(akYWw}qkB>{= z)tJ0%w8}?S@f=ijW>1nT!&K7ys+Tr9wg?B#Qq(n&PO*Q&BD0yH3E`31}jYaKrDSo34WaZv7 z#Y*H}jvuJvUj(_rxBemMmq)cu0DxoCS{zRCQ}HWxritAgEcW=NGOo(X&Az?XHq`rK zqOHx1DYyUX0;BMGelFov_kj(=jo!I$+}Q^A14$u;bp?^>@4)Q;$JkqiMg4C5!!p!R zl0z$9LnGZFNH;Td3R03vNe(UDEg;Pd9g=H3{aeLs^vgOMDdp3(AdTBXPl!Xj^TH{fg zR18GxNK{U{@%JE;e_xUR`CxKjOCKTyXX7F6fxXVn*9hrSSMAAM=yO@QV01fP^~pP? zB`y46FP>`Ip}Hjf-(CP6!~qyVlX~axm75Q`<$_u@dly2e7bqp(56M$&7g0GQ@3w)NOa^tfPNS0~4uUWK zlj4!(j4Z*7QoVM8r>}b4CM04fA4Zl?&Ji=Dzm#9*ket(iTnhnE215zTAC609-U%kD z5BBE(bon)p%6&Osi-yJ*knn=ExMfpu5g1qw+Vq81j!FZAakP4yL;k=4csRXF5GP2W zmj8~eu%b_qct+W`48U!nF%Ub=W;RO09@dE^w}Py|mHRg**m2Yoqk)PFmD*o1ZoH4` zbhEWZv4<)5iIblHINROUk`Z>=n?E<+pF#ANDt@Q1Od2H~Up0#_*teKIIyTj zeF+?y3h|{gKD7(WX}LV3d^Mv0(Uj9?!;=lWpgP`2$LwE{PAxbqab^*PZudm}7Sh#Y z3mJVnr(5LF`O)`TOpWu9sHR`j>nWM)4^5=vb4yIk5GyDl6d~`fOZcE7)lZf( zRyQ{JMy{alsduEg2mY&81_Ldcu(l%J+n0rzpScf-LvnsrFAYG-4_zW=*U+3_NR1Q* zzK_XB4}zflNdD>N3*Bp~FH%XRuYPK*;QiyWUEqcUhFPXpcJE#Y6pt`(a6Jnz2>JOe zs;2wjVseT9D<3D=^rr4V{XBKb^AV89$(>#VFW^J*fd)LD;IO+|r zdLrXpU}E!fh`hz)hZV79yoVSdwNmy&+-qNa{K@)PJC6kq2g07rSL-DsLhW=y4q>>Q zH21#iWFcUl82OwaR(Y>bRe*M2pCZa;A>5zFjih}#Y}qpsVi5O1pB=5PSnY~attyN0 zg(l*-Jx8_+%teC4T{6S%G9 z(B@HVHd4L4G7SJzp3`?_aE6EvzVlP=RM&^e=yXA{^Z^np>Gk@eU-FGdAe|@ zTSLE}sVAyeW1nZ~?!EzY$~4qHCx>aBfHIUZ84^ku>wzM^80Qljv)tb z(QI!Sg?05+elVFZP$|nmz4Nf$Ca_XLsFzry8qeRomB=3<@7JKhonwi#Ke=b_+fkyz zp;ER3xIW=X3aB1I_u%4r;x%-}UFEXnAfMSOYhgTXmJW4)>AJH%-k$RE(JPl`j`l&a zQ{UYYX!|So(ob{H9q-D)pj|pQR1R0wbR*k1w1fNiI^~Z5Hb@GWSPmt1C^~OBs|p(2 zL%w?sK@!{YRnTDZ0Vv$-54bmF;kgQ*=Dyq~-rog&CL+Nip$7w7F$J zB+5BX4A=O3Pee%Go$S%;asCK(mh^Rabr!YgcaTRj6@eYf&F!aeEtkH%&W&{UCqf9G z*}Ym2o!25%9ao=Pl#@&L7qu_9vAGhSx5Bl~82R$z2rN%aW8iRS|FMR27U%(~VR zC=NtQ6)tF5pFK*t7Xih02KQ+7pS}P2XP64}KzqnzrsmcbFy$utk5)mI0-*D<8hjXq z)4Gb_7ur4RFpwV}3_#Dg5r~G0+?jT;<%33LGCHr?EiBwEBQRtEOSfV?o-DMINwMbc8>HQA}%P7^YQ!gP9kX7 z@0XY?=)&da*mF|R`5CSh@Knu|gd)WEe7!Ek5aiCP!^UJ@kWhkhPK@5BEk#^>?-c(t zQh(FA5Mr4U9pumQgLFI7ZX1gYy4eF>-Rp-eu*jR^Mu)K1A4PuI3N=D_7*OD&z2{z- zPete>kF^w1&L2oCrCnwz-yIn!U>wwhUSsxpqTDO`CE!}~TUxwfCYqdSu9Nk{8y`tq zuD8Wqq*~LTW)xO`Ch$yZM5JFIAbu)x<_ekEG>bpqF=GF(dZv@WOA(DZ#+Io&ikZPs zVk73uA`K-B$?x7%CvKYV6&1spH^qofwf@oM{TU6n3!9!4^<+a+x`2|`T-8?*ikA;} zA(j3fc}FgQAa^HO8>Ka{mYHxsS0wYoEjR-HwCxIMk zcAUg+bI!TwJ{^4dP$8P^_L#9fbc97WX|}G9qe_$0k<)wB4K?}G_MkEL_L@V|fTUM2 z0_{B`f;ZDF~*bq&m(H^w~e zNpNT``V{ctb+Ijo<%GrK%kk)RwE4vjx#S+<-k#5I)jsk4OavPAKxoHjGItCUK|_k7 zvH>lFBKr#0^b#9+ZJu?3;kAhSojQp!y_yjYI{?Fydy9#N8>@Ep4IyzhSi}Bv1kj6} z*NKb@!S|9|J1S6X@Cqb=xrk%SZV>1VaX(?-Y2$&@9;UtNPR z3Q(1*YjBZIlJR13bw(EM@`_!6Ub!@aUrUXuG;S=>)}1orcZ9JQZ_D7b?2ut-s#Cn| z{#xjML$%anBcb-yC-u{gyAq>Cv)%7YRfVp9hJ;xI2A+9EKgcCKstIiGXB-Fkz2Uto z&L1&8+c1`Y%+OtKQbx7AWA%VkLYemff-Q>JQU_->G@nHlgTp}=bZYW`y%oh*COUbm zUcc(C^4B9By-mp3c?@aIiz))e-x^NC?Cda>tRlMu0GVYdT~q zrAG%H&}P*|ydPS)e;328FMfT`Y5!L6DX`ybh=#}Nr-3AL^x=8WCxZiGC-YMa{4zFo zR$oH+kt5STlTmzFyom=5lA9>$!mKv;l|KmBgp$B_O*@cjkS-BNgHV_dXh%4L5CAKa zg38Ir|2DVefbqaiMxu(4Rz)5tF06&%SEs|XL2LyiVC)=@aUxeZ2d$YiWv-0wB(cUx zCX=Rz^eB8T>LemfAzQ(q6`);((D6+W8FfzCzq-qz=`GAZb;V{?Gmof}V=MC?Jb9Cm z%&d7Shw?U+SyNG8qzqV3iLz?gr!y$yKa8oKDWfCTIvAykp#SjX21{>)D*8D2T%}Ll z^q^tkiPSD!?yxJ^*9l_`f)}jSyQ|*)Hrz<*p$@`KW^Ravw*4V|N2Tk_Uc>r+A8WV- zXJSgf<9=4QosAj*U5Vw%3H^ZVSVQfO=a;d(`4HRCpp@i3QEe(+sklC9LXY2`{xVm! zzzwUhquz!J{p%PVrLm;|UiE{TFVWn9zE3#Br4FEA-PT-zfW|I%Z zw6a#_9_DfY0^E@}b~63(p^-DL-Q=ne-oIMuRebS{G;Y5K9EX>JJ_kQ>O_|)M3ON5t z)rD*RYX4TyXn9i55Nqus>Ee{h_S|$ts;2@zOPibZMbdG`i+U|dnv~{@nA#9OyroLcUKYAoz!g-(@1KVh~w@W z%5Xs7&1+N$h~`CGBe#V3Wa)=$Ykr~cg(H*iLY+(k9iM(psOwAVqb#2lYP!S0(K7vA zJ6&M{jc{a@qqB*D0UUj;Ua4GSy1B2{o=#3{E|-^k%9w9CIVoxf*vwwL50g&6YIJrF zda+bwwYnyN9MiGi^~3V3f1 z)-BggC_koM*$#|YMS`8)nbIBj1NN~nA3hydE^RcvHxWj38pJcq`UkSfYHpdGTFahd zIdm@PK;4_FXTf(OW$=C4`j2VL*MQ9*2{eQcjEA7HpKJ=HV^F$x6U8~SKTmBA-Ilhh zFs_iw#DI4!dU>8y`y()5={RRs@?-hAPDf;9%0hnkxw}02a?1v%#A?y0)<>(!K- z@Ef@oMMsV{2f}!7+R$sv=`>{An3|<&#t>74ZKswuwgRsrgX#Oq-dm0&R4qg@mVa;a z-G0@P8n|_>^E8vU<7U^%^qJhB0{X_K%TMxY>@6+qd;SJfsz=Q?`UgihM&C6H#nT8b zkfPFPOt*j~fD7O+Py1FewLc_zVIIa4gT9OECI16%JS{Z!Tp)fjO#V< z#lMMW)w_Ch?du?st?~;m9?g1yN;Ht_iem_2+AbR|qE=Mgpf!>Ut%C03= z?qO`|&0OZtcocdad4-YEare8`W!Wz`VgVR741}MW$7?`$D{r>bfH*j^`GJCBv)nSa zV~rjUy9f1V^8LyIGLD}=RhkCG!|vmJBerjABMRnyHxv`lqD(X57GxBa)h8V4GM{BoR{KAv16r?USBb!kf9toiWAxM!l0h`u#PaKFqj+jU@lxfzKZV=Y)(xg{3=9tuTp|7_VrYUSKn`5J#ynn3SK% z3v3sx~N!!?83JQqhhkZpILlearG;pqj7*eul!DD60 zMRSX84=W(#0CWK5k7c|to%;NC(f(Y%eyRS)K79JJ*>zQ2QSc?4FR0@qCR6fTq6^P- zHu*_YdAq~PS_}887-%CE%3|)pQuT~sA@=#ylz4!(HOl~){@xx9HYpYihlIOya}C0- zOjGh9;GozYWwV9Z{=0EjGrz#dNx&s8QEEpXl_3cm)O~xX;UxF6Wq^B2H)Zt8Dstnik*mc;dD{V*W!=tW0VMvtkT0jD|KPZcnXRzMfzs+R3JiY!=&^ePHwU8Sb=8 zF`4+I>ha@B;kgf5Ao{@m<8NZE4O`V8w-S4DcJ#9zaef4_KlS^6G_sI1kx^gyBvSbf z&NRfa{pUMQzWStai}&@r(lgOotfz_6;T@!fn5f1bVshqmHD$l-vjsbBtlHl7}qGL z=`7Wrjv;MOfID3ic<+w%hQZKj=bl`@hm0L=v~ZFiprwK?K)K$&(N|qcT3|4)C8+9U zBD(=wlmr{#;{nb$#$DOkOguuP-;6ROBbaAxBUuD&n{9jEX?$h+6$CM5?}W9tKt9hy zL;qvfgOtzu|K0oLQk^R1(Wf&C*R5+g{v2i#Y(4&{`g{|#=bbsxB}H-G*7!3l!LR-k z^DjJ9@KRJ)C%tf}Nb6AayCkD7LwZ^H=jcD!5@$E)^n>|fDE>HQg{-BxWHJN^Ip90C zcKkm+drJKj!jy13z|gJ71sKLI-#RGz_uff=1==nsa$lOt+k0#_5lP4}YnYqCU!mIM zZ#U52eI!7ETPjd)NYw@AR>cgCQQoR}3=F=c%kx666Z2BlU#|&TQCi+6mrHY5p0Ca4cSnh@t ztGU_`Kgku$AdlO1!Y~c2Gg{}ei6=HOFE1Kty{-SQztwna%Frk|Xd;%;fdrohnvVYI zc=vw)pNvfiXzCNQKCb)UNk_Vqj^JV z_PhL3@0v%a&-S5wvCD5)bb3iv|*_GzjgqkIr+2N#1icH=P`!hILQ$E6B%9_jIc>JHB9#Cb_!S2+j zxnX_eK-wtVJ;_D7T@pfFi#5v=djz!fmvYLU=ncqd3pUw$))?MUMH*(QJpr`X9R4w} z{)|K_orJENqqj3yWOx2E;RU<=*#v-M$l$d(#7$|QuISqS@jc`3DDVD<1){T zOXwB$?k8`lPyEw@0ehH*ZhDX!aOG%@Mj+BC@IQJ7q%}^!?QpAu@gk)iYE%@pM*idl zm0xl>J@QFkQaI_adYgzuNYPW@7V!ztJ`XyuE@yE37U-4)XiW?W1o?vGs`k^Ed8WTI ztL6EE$c_BHz-LE%oy&{LcOERv!;oZ?B?O)PVy7*2ZJZ+?S}Xw8f^id;yMUtz>Q6NC9l_xtRQuYQ>7iUC)A?BNY=>IlUV32^#!Y zcv)FXU+GQEvCf;_EPia;W!|~1T2dRR#Fz{4vEuvX#SDK~C_{RzH9lP|Rk->bp|huR zqAn)DevxZT_o!n1G_JnI&~YnH7qL?L!h`5#Jv}qIjlfH31Kf-h?yBf|23#KRS&n-i zKt7kgsT*sSsw4A>LF3P6UUuiu&BflErNs! zn?v7p(2%NqQJ<3{Kf0*@84Ob*p~@g;fh^8`rMd%^SYOvIz*DRN{;0R^tr4`~X3WC8mqC>#+%^(2IjVA`Ietrc3+(U{T&ovOwnt)7gGxbRfTj1S;ZnJpq7E1NguMw zET>O{Y-BFr_n@0^zZ#Hzxq1i z-~O-KKdkedPO}j297$#KN-QsYQ)J18T5@zP7Rx^NWS_zP zD-zPIh?*8BDgP<~fZC3Efy@mksz?nSm_e|0T`dZ8=33B0JENzmB71KNlB~zKT!V3o zDb?9gW(vD+xuVgU@|>TDTfuh+gZCh$T2+aWk>N*2y*7VnHmaznNH1l~ZThlre`Sx@ zkqt!V+APVB1$}DGxo|Q$?NAu}zAX^}J+IToB`E`4A}+c7oRyj~nQHYyaOJAna#@K{ z&+N-ARBJw5C2$rfO8N48S=rR0s)+((!aR}cKq4VGHs=7F-SwzA>=d-E7ShSfQ;@uN zf-eU%Qc}wu6-2HzN;~eQ8aM|@Q@_;Z*1${Wdz$o1FZZ6dUtNmuP02Ub+owkz+Gud$ z-d5kye+=7n|KAPUGMq~?`d-?w;m`kG{Wqy{)uPUf96%h^`H@jS#S`cCm%A$vqCds& zBlPXFQbX_Q+G*~5r_k}$`PP2uk>WLmH3ENlRZbVqOIRsLtIzqCemeVl`}h_Usnk+G zk5^%1O<~{24))(_)M}v*?84tRAft%Fz~J3rkCduYE8gqL35aZ2#HjtV-nvLJns_pW z^uT`;Uk;Ew9A_QtYaz>5-aM0O-;IFy_KYMh;m0XG$pB}WjQo_zZ}A~x_=N8*U&{0w zFYSi=d%`Kn^@P)dE+csC@x;Q63Gx+Kw&z}cGZYWkyq|?4v|s+AhnBUorub+%pqfCEv0NpM-RPM>gb46+bm18vT7qT5ywDe;> zf;!#x<9UR`m{h+RvCeO1?Q&ep^uPL#pG|w`T#}Rg5=b>s@b>t>g?s~$L=M4DP*HhI zQ7`I5W>}2NFMx1ueUUVq)1q7Mw0!W00mcc?X91_*Ba0SMd^sF^cPeU>p0h2bF5nkS zH8-r<-0Z-Vx>Yv;!~s97)PSicTxl3Q&O>X5vRF&=!zK2)RIZR7yce+&qj=l&)e`vMy!mk zXE6fw25_~pOshQDg8klFoDy8E$K{#~V@U21-M8W zO4}f#LKhqgE~=KnaIAP++l%!|`Vt~ENk|%}*{dwaZcybEYYXz`)LtSljpbm+rt-7? zG5hPcLQ{6S&$ako15tieFelPp-kYqotw0&2B{oTUSH{~t`_%9vb>}8>yV1~b=)c4X zqzj4!AW6D@OpQkXQ5Hbr@U^q0pZC7KUNN@B{K@7kF^_RtYuXg`QS38}+wV43@k&3~ zTnx#vKaRf?-|-)-EV z*QO{|NEI&orey}vd?NDLV0bynaR?Sr9Hccd+W~QBtPbrx=BMIbB_CtP$&m* zJ%8WhEZwpv-C}N#5N9ZPo`M1#O_?2pw8ZuX2V7~Q2%4L^KLwql|dcgUN zf_B@|WqWMN@Tsd4o(8bI)~`W`2O>3pwX$OJgHaA8+I#0k$M!*4r0a!B)T$o8J`d|W z7z#v|NflP4Xit>}B406yFFBdGbL9aq;CWe}%~LS44zBZ*B1bUl%PGh%JeA~>&Fr|` zeYba761nT(4)xpmo%+20AveH?!#jnED@dMI!QLg5@6iutH&OWmxSJ~ZVMq6j%@L6h z5nWTN+S&wu0#*R1&aoBUp+5nx%YB;6Y%|{p*QV@DW)!5$0DRQ7i+>tlBL#rwnHFxC zA@AQZOC~xI^p{mPw~Re?z^9XLgIXM}q)cV7@ac;me8&OJA58ZK`FO~Ou#ve#C#X}a zZ5|(xvgBv#t|wPHclboS87LtVBA~auf^CIR=3Z@}eR3Ygixrt`7vI6CMG6u05k}is z82;4Ud!)?7UQtQvzoS>>iri8LbYL?~**uL*`|3}e3zFZ1UZ#&9eONAjyl^TL&UYP^lg^%rzCZ{bEOeAAn-*!=T8t7D5V{1=%pU<(3=M$L=v48=VP z8sU5=LD8wA9H**77$Dow1^3KbD6$b<^h`7X9b^f4Oi~YnYpbHYjQ;r`D@dtT(B$*0 z6B$E8mw61X6yNHti@SS4v+D9z?Zk>Uho2FU58?quiVUR@!=OJme?F=BD>aSX4V&&u z8S%&_Lysx;75?<@r z>sq0|AJkK!*P`P@(9=)GrRU0KDA$fU6Fon9Pi_6og9{77zI{3oFxQd!pkYDM7={1A z(I*s3A1(UbvZ!xmDLuDzY2xd2?)halycm`G~ok&s{0qP%_*Uqa5A z$86O|f{q{%`aFiF{1uE>^9H0Nx3Z-=ckpK{G=qP}wxsKArj@p{gQUua`*hLQ^NY%~ zpPY-LCT1tb4*wTTVj+ULG8x(tY~RF8v?6CE3FYb=N!L4MCVH8j3eK}#3bgWgJuEi0 zO`oh8>|(Bn%UK?3i}_)=`ywZlSPz!m18rSlmPoKNL8m(M4x^=0DP;FsfMgGxbUp|J zyz~_sNK{Sph3R|s%JSJ9t53-TsU&U*Ml>9ot-`mga{o)w({{l8H>48iRg83*aY=K; z6OmrsyBnTL18v`N_Z^t5 zO|23Rs5demHS^$z3<@6Fbp3R@lwtO)0d*D#?F(o`8p`?UojD*Ec_41IFRdZQnLx;S zK38oT`tPr?d+1sW zP>;Np)!18oRr$6$U4hs;tqpvlyof1D-z^Fj3-%wJMR-$YII8Ob2?U4dckXc*^XwX zUy=P4+f>;-Qdg~O{g3|Zf0Rlv;<&&FXuzrK?!i(T#hrdlvwuNcBE*UfHSU5-b!*@^ zW%O6S#z!h#jE7aN0{TgKqB+c(p<9>v7E26EIYq8qF3(LF9DQRglUxhFFk^lzA`FR* z*;Is*8?-M=e+B=!YxW-b?acG0-&GcZB`eB$}hH%DEoXv9d-P+DonMs4!N2{!&90QH{Uv`leV5{g!mqQ$V{`>O>s ziQ1Yc55|yVPfZc6Uf7Zju|I@c&MK7Pf!Ps_PS3l(VV`E6T@}^4lcf+jvL7cS79t5)p*mgTb zbt|+-_yzZ0z$F8<_harqes2dIpl*8`jF@*?uN91t9AI}9LSaM21KY6983v?s1+zo` z=P&C^$NUuksM%ISQMBvRvSPxCt!H6e7!}d_gq2e>U%G4aMGW2(0y9UJPVdepoV(4z zLndRmk|TpomK6~Otx-Uo$=aU5%k-bRlHKz8dkrEsGKJtv&plm3HXua#BSzU^Lhphk z`EeDAn3=UxAsm34br2Q!+{qN@GRJp&aB({4an4lZpj|3p1@Dv7sqBhEK-a*+Z6OSm zPe<`>hB<9NB*c=TfkR9WCavD9e9c*HMxi0ytIR7a6VIv{p?P5bLOEy)n)*u4Oo4zp zt1VbQIbNp*j1?x2ikWuZiTCB%|5mA_YomRK3@fvtLJAd;;zTcUv^^wq@W_XZrWh}- zwkqf9-nhRu%@{5{1`y3pQ755f?J*UQ%H(L$*A>@!Nif zEh8B(^e}UiSHwmAi-$cA+7=4C)kWMgLrCD>RXVRy7oy8s}bv`hEF|eUvS-5YY z@{=MUbMsN#$vNFR?-3R1xviCuOSrrlyXG6BhXMHs;@7VW#e5a$!?5phlT*g}5h|*w z^kvKqGOPUiTjZG*{=9|i7x(sntxeJPsp#_$cS22f&uOexkXt^GqU+dF<-S32Zo@Ef z`M|2*X4fCUB;{TyuCKzY$C76J-hK=WrJ_tsS#rObDV`4gMJd`XCiZ5}Gobd+I}S;w z{RLvh8CW}1ZxVKFFv}afSp=jh<=&t4jZ{v!iXnA$g;EsZ4v(iSVFePcMfQZc}KkUR8eJ2Odm^b;{8r!z{eR5fvF?E$c%_h+|SY<%Jg=lU8Ga z;wqd2iBcHT{WiYKAqkF!G0kqB^>-|>2e?EuXFu2OPNYw1+}eF?(@icj?i~|sgj3=% zDeO#If~6J~1x#E!DDT)DvB4gld^267>hypQqzeUfHuJgnMUV1{Lyw20upJ}^{HQ&* zt-7AvUr(J}O&@-3jsk$N@ybRMRTE$s^wZFQ*n4q4JM_Ask~#%{L;*N6?|9vJ&MOD` z2g8C$f3SCd2@of{-BvpDreG44$7TdU#?zfsQSNRZ=mBj zmKfTuZ%uBWm5~r*Tk))Jcd%&Cue$l*dB6un1pG;TtfgmGFxk^CqIMTM@LK=)WLfMb zjq*kFp3gwH>U-{Y;v+Xb^p6ocyG`FRT1jA5-Y&VO;v2lTqkTV*Lnn_4j0L`Z=o{Ih zGgw^pszWDne!;C&!Szj1%eih>F3`rYD3;ym{{>U~{>1FRm|D+Ysa*oSJ?0BEW~A4g z{<8J2s28ksh3>*-Yz#{nr&6d=EGAL zzh6A%1BIlIiJMhCl-3#>Ki+S{_g_b`P@zaLYz`5b{+U|qL5i`IfgJ%X4?j-VSC7&( zDyXDO3tLuN*;ovi9;JMor>Xi9-r#NN)kNmo>Qq~QiTM)^+l9tukxt=Iv{On8NSRj$ zA{D(mWHFa>QeAqrl}j|1=Sx5y()4E2WTzO=N?3(|<7B(-5QeqNRJx#iO(Z0vW?Sx! zu64qNj5ftZFR=$70|Pw+gF2*$K1bL!=ZQ9CN`khy#UT3HA$nMrcdOY|qJPYbGpe&E zGKa=AHnwG?K%HTDZ^rK)&(q8dM|S&W96kv$X`RWf{=khE5=cM*kd+1kVJEAD!MNHv?Cp=CY8JXWDa>dgofL%CICn)pUNL4>5J*?t zaAM`Yb1a&Qm7z973j^DI9hCw)9_0UHoIq5=Ki9T5tIuDq^I1&-0?~ z0BO_QhFmwE%RSi{a1-9}SX!Bq*W*2VC`} z_{R3Sr5}@xJ!+oW|2g`txiwRR-x=S`L6%m?et+BDOUqcUW&9mK#?HB?)9;)L{O=Ay zA)RFe@gFA3%rkxH%(wt0?msZQ?B`zKd(fz}uQ4rKZGpH!2yLVaiHjOQuMfFQXjQ60 zoS>~nGW<>;HvJKQpw56B>wQ+=<$8wBJj|%c>q79;9CMwC9yES4@))A`4aLN~`rt>w z4Le#fedkfs{oY;fb`$eF2K0b>$q1VxGP3Q z7jC*L#5VD^coJv%c;5O1i2XN8P>}xsGGv5fJ83#*9Ox=8N<14~GNlG0am3B;0-mYV zjs?x1XsAs05guPWv-7OWXvPv>V)6WxR8g)%MNpa@37Rb1zr$c;Cn`G*2+06#X-cS; zS3f83j%$Hb?MXL-D`9z)-vqWaMshNaA^I85-Bo8gUoLf@w0Wg3vTg|mUD!m(7po(G zif*K|_iKLwuV0o2uvU+Pjz13M;wePqzezkUp330bQj zd{w&F7uo*6IU+@LFM$Ub$*D@6ok)Olz_3leRW2KHf6rai#NX2E$;(Vo-fCL; z{(0D9cU0!kqZXumw$lbip0vv-(q49OnZ=aZc-HlB21e-S9Fs-1UaZ77QsOANScg}< zf^>;uzq0b>EJC@N*^28h5|O1<2wY~kMVDo4CR{o!+Il>J;X%q*n2!SzjVJLJKE@bu zZPjTdujakP`GI7_Yt{q>NdK@|=kMev9bZJv{_UD-xkmLRiQiDHiP~K-*QS9LFX`U& zDlPa^^=zB-PFE7lNxDq8uRX<|T~H(7Kaa=Xum~g6=Wm3WN$p2k3EUCx+T#L@(<=9vqXXNN;iq{Vy3 z%<(Ew;%Wt0^)I^BM?%u1>qXHt){BN|uh0w2^;{3%uPPH*#3*;_lUOZ;zP9iD=I9ps z#`hI!7DP%L=LW(6zxf&U1nbEi;Tl;?gD-1QQzW(!Onm^`XV8`hYme6OS$9OYr>w}n zBzBNq{ZR+*fcV<|rQN_&NaWMZ32tFUZB<}73E<3Qh|-m06EJ}Ah0%BZq!tFSzk12c z{1JNtq5`L%1@Bv?gEGo7`AE}$lfx-mtR;dRI-{KoK2tWuk_jxU;X);#&01r{uy=s z?JQeranf3n9v??R^>nqs{ni2!>-IkUg+AP2z}NVNtT5IRKQZ<{CM=X2$Y zXl|$9io}Vk2!^*Gth9LW&Wg!(3UCim2TIm_!3AFcy5)+MfN*RDqGCHrc?VU6XS8|^ zgc`AVR-p|qBe+9+nk=5(bjCgXusTUQkPB1r)u(WSD{gCz?3Ji-Oy~U(GyaLS)xY83g4PZGBfWLz62PR5va;clZUNx#)?D(So zy?oZr9{7`nc-jG5ESX>SAsK*$z-8H5{YjE=n_s@PJl3+5+GF#NmyO&aaD5BcK+Nc{ z^yTk_$*FX<)|Ss5@{BEl-;gM{5YG(fSC;kBEucPsUOXzp8D-hjboKdD=(6Kw_eCy- z)R56PM-LNf^~sd~^I@^j6l-se%@Y+42c_CBOl z&jb|t*R~hGyA|1cQ-lS3M;jQBl)2sHxW}nYl8v2!tt!(wo5|^Vp>MD?`<^^wF$2RU zwmjZKk@JPi=T3*%df`3Ue7Z7hQrp+n^Fjr!ogTJa>tAnm${sq%PN}9M1v#Z_n||MYCyNeiEoKHX}4^GeyWFVx^bO6+rCoJ@)dp#RJ9ftClb~ z2P#d(JNE)jsH-=?$_A@}g&;vZ14|HwxeGB+52k?e1DV!gR_NN0b8S?y>eeiawv5n* z3Vz=qL0cAELCn^wUFP_K`=7IXHw=IC^S??uWD21~b8Cdr8+^iGPsyM`KFAj}!t;l3 zUbqqqV~Y@#k|Uk!#ht2@;N2E!C<)S+yVhi|CfHCAt^YGj7&5COz77CpvA>ijzH`59UqKd)KF!{&RhBaAp z1H$8zqsv-L-YSpVx&C8~$Lc>cXM5wq-4Li%ykhSqUtu}+1^oG%SJf%f_KXqQWz8Pf zQ0be#hU-P~;N|nHxkwLF*d!vVVf~5av1=bD z?A9@wu%fG_p+eP7EZ86+Ps%$N`D*yxOVM|s zBRLwr!`aR4-piWI^@$UMp2#hIv*OX|{dGZKDKZ!hHm~0k+AgArCW0mrE>^DX&Ylb~ zc9|u;k&)$7!0NHGxuKxPX&eTWh1Xz<#Mj*ky7(}?l<)yEnGQ8sPP&~$=+(r`3o*9> z&*dxUT0HzXB}pi9zjo|~8aIsY{EGS#gggr}0%XGCS_!Y@!hwm<|H{>p zx{~_ibZd{9Jv;w#U-LeD&oY7b+cF6@H2v7^fhv?1G8^m)HXKzmS4BR+F6!A?oob~9 zCKMjTS9H%^5a;p!+sby4*#Bgk|B3SKr!aZVI--HS_qmhu92dx;!$oP!l4om|DoJ_u zl~DZyX$q%|`~p*gtu+=zw`}b@N%`Hy0E7`r7XydtR*Mc!bu+{xL>Vn$Z{2}zkZKCF zj<@EAXhOf8+j3rC4A=eqb(XQ09EiiOu|c*giiP3~ywX2*p2){sqVye%Tw% zp72fwI?R|r+mD`*z1M@N-}+q3y8f+|JF^K&Cxe>Xof&w3-5wdS*r0Q0_xq1^QU|lW z97~B=H_0MFE=xIFeT8&|lCBK;0K!iHPH2L*7AQ7#F0$0DU)j;}JZoBr=&X^b-Ba0B zUXK{%E_mK}IHyc6iQu@%RA5+sWht1OMd0^>cakyMbMX&ehaXrLz3vO*jFL|A=PvYE zHgkoJf=4bl>5to1b|vYmH-N%abBUfWhsluFW0oK<5;W&Kt$hO#7#IMSy48(cIO*Bc)vAk@}r`LyL-{+1KzrJ zDx~R_{I3hZg$9Vqd&87C8F~fhRAr%&QnQh*rJS_jLNQDMTfL!G#~a)z&abC6N-)F_ z`kiV!S!ag@gdTV21nP&rfRxWUXg;8Kf%h_9qCj0C!`I^sKXUzte$%@oMg9$~b^>y> z!nJYRJMs_UfP02+O43^eMs_8jm%_@pXJe-1^R-~wgMv>8h$}`UxjcfWc4?gA8A!P6s4&SD(vOzYT zUbFDZZ?tbAkZFTGtiS2K?n!a2ta_o!D=(_I0`AqrqZ_!0E1I?@~zWpy6q7C#!# zNn6o;BYS>HW_|b44MMuED!}f7;6Uyws9_~QdJLl1Lfj@IniS*p6CsDp$7&NYLa=g8 zC&0&`zi0CA4h^ngzGM%jH-J25aQVc%=dD+E3M z;R%iE%o+FyVT4@l@DbR6^-dzj+G3Gu`S4rshXt4i4`5pZSLih90eR`Fp8%;FQ{Imj zZ$$UNshn|>o|5Jn*>@*0JjpV7H_)25RmBQ8{4}dq4!%h87JNqN^ZltQPbEhgHdfdK zUkh-fA6&!85s&x)GYAfXeBuJM>Q-U@DDto^CBM?qy7XY+3*cV!T^6j``()HvQ_Es@ z)QJ#tBW&6+%j`3Q9MQXEegN^FtkSzscJ0j`3Z(>4XAB&wBltho4w|6-sZ~Tn*mbg- z{Vkw2s968$RNs7*4zL+sesGKn&MiaDJL5mx4ZuU_yQ0voTcS)NumunV+Z^bl;NIlv z=gjt)<|at1>rkIg23CFK3lt(Bbd`v8IXz~(!tBEGmK3kmJqKg&`QJc2RfnxeAdAh*2k zOiM8eFCURnK|25K1uzmKxq$WKVAcboVdREZXxczclgCX48~<<=Y%Xw+f!T^*0y;@_ z=F<`B8dP}+DMQAhN^l?$s&b}P_F2~4ouGQk<9&GI_m3!fny@D^FcAbPN51)UtJbWB87?c#Jix;)wY+z(s|#p@*xuCMCO9bnIscVA z;N3~o*D0Fzt825*6)(~*(9EJ_w-w|(D=7g7PJ+Z_Fc-5In&p%z^xG;PI?6JYZtqy^ zc?!`Jz)F!+-Z8`M5{ZZ%W&&*JGgkcuADJs^>`so07hCG$k53XOJ#U+{F7gJFX`9|j z0i8Fs?vb@bCmA3XMH3!Sp2VfBpYqM(8i0-YEv}^%3LUKxv2WeZ~=$&Ryd^4p+yI>@zh3dp5_`MaY zd;wcmgdVQ>pCJ$dU?{4y%@{+{CkI`(R+|7IXAU8?S1ODgoTBYs3trocQ z&8hP3_C!q$@JhQXbBo^Ps4MOD{5d_vzmn+{JtXey<&6SOF#DI=`JM%L4+AP zq;cqw6e$IyL68zb+HVcleciX$`#jJ0J@5AY^#^X-T5Hbz*y}hWDRQ?|rLd2M#}%9f z&xRqqjtrPcy&63~sxiMZGcX(xWB!C%=NM_MGtMF~zz=rn%8zCX&cs-eHdTIcB8xDX z8d5W^nkAkLq^~Ee*RWAmi&6{(X%K z;n5eYNEPI{C?A?!@m-Wu?V^iu@PnL=dSH~~Hx9MU`gUjc_S4O;U&E8eS)e8Jw|3LH zB6M$1weB^Ry5#-#JK085Z{U{^(UsfiYkl2dkxYl9CpizSCLm|uW=kLWiSBEcU;ABg zT%_-wLxjr&M$6xON!Uh2YIK%7J*oe~fQ;m`xEh?gEPF^u176O(N&BIKn zKcQWoq(;W6;=4d+EA^n9lqFYX_JPeU)?yEu;=5&@PK_EM15WjWhdx9G#aG@=1Ss;# znRv4mx(>^c4_+U{Hw98Q*vyD=zhpH1Jrq6RoO9Z7o4R2`6!VRdr9v74?4FQjDr{s0 zr;#Ar*hi8xB~d!5yg(45gv#lqN_mh+9&`M1C#*=9mkl`PZ#JNqbw|~hp(XteAf_8X z+Z}V<+gSvWb^EhZA#tg%%Ig2X&`?Y?t7H&EYs*9Tb#7dwI9@c<$Pi8)MmLH-i0FxGEIC{2VRDLh1UIN!X^}-&D5pTwEWl4%~(F<-AaN-Ok_=O{p(z zV4f6HA6O|$3USN*SSl8=4|$QPqVX0u4Hx;|+l8vPHBe$mjX4$lJtbWk;hch0l@mlx zz?o;jrgIEkEqb3ksdN%bOz&P}-@2>cvG)a}-`H?@ijszu;su+Cy4?i-2SJsAC?$<= z>FHUqx5tjeb9i_s&XEV4k;iZI?Lj_#7DHHKUqz63XdkZIIRE zo?Q=*aZ4&ILUj3EdEb|2F2~PFSw^%7$EQK+&IlAzz4T8}u{~W~sU7(#Sypx9_+{V@ z*`SBRmzcBon4z0I)Hkq9RldE-DH0I!)HjRBg2!tt8NZ8XAv#rB`&3uJoZO`FH^95U z3k95kd*eNCx`~kE(ik;-%WV#FVz5@_>q+89!p@vr9#vu*Byd(Nz$$h)Dzxhfo59*N znUAYKD|gJO8MkJhc9=1#i8F|&6{-9iCb8B|KlxDyjl^$GdH0pt@cs=(?Zoo1*UrP5 zCyOi<--+L)N%9jRhT@gm=%R}Z<=~uXQ$T1SyRa)V$6JM4O(W{x8eZ2BE;3`<9$&ip z>?vFOUc*=ft1=~kh*ThWa!LL4fTkvV3r`wHpyt{5tifSmIl5csphXCTD2*BqlI+sl zX26xb!{GVF%egQLo@XdW@Bz^mwE@3VGOGIAC)J&lcJ089J#iY`04rTvMmg#}Y+p#l z)Htvn!Mce*?u!Oo#*$?1T)A0vq4cZ;}q8PD5Yy^bFOR?IGO6M*mW}yXYRcHgJ zXH|a!U*I55UIk7A=R9L~FRzC_?_SA&kxUk;3^Ben)hxhsd+x%V+(;HP&hf{iEujRt z(Fv&}L7^qB0aFGyA>-+Il9y`O-Iqu_Or$J?Qwt8RNaJ_s znYxGWYs9(>23I(=hs*`(ba{_uzH=g;heURC7idWgG36m|TFX_^KL88ca z^Un0pcw{J>ueuw3M3Jc5I9oGe((O^5XzCDF00_rBq&$Ood$n*C;k#v7FN%l-k<4ao zV!244RCtwe@DF%!{6ruAgEZSB;BLPx9wrSZ@2S8kr{*P-6}tO1i)L8jz3Ej@~% zsho}edP7hUTMaSfiXigNEcj6MQ}0B(dtka)RGbK7hAw;8JsK1c; ziADqJ(Ex^j7F?x&qa+6Z0}T_b}qw>fNnR!2+^dHk|=1I>#rb%sU4LhxE1~5swyzL zW270J1lJ=Q+-Ccgki6D@rL&LZ&0T)btJbFRpQE@9ke|qn#mR>AIr9UUkywUo*^h$( z9>~+yjT|jU+AvmaHISmW^n5TgFzMkP=Z>qfR1`s$Cp_Ml`Zi9U=AMC@GH^sxrq!4| zOck(CoJE#o9Qwd)2yLFYf7q%&VITtC$l6>ltw7aAjYxQgrCqAEN?9pGwvYEdS*};E|AA zdq)oYqASiI)F3#U$f6<~mz4s*J)9*NFjt#f%z{4NWL8j{2n2luh-7{+Gy?8kPcV># z(YMJbX@AtE`Hms+n>~jju8t!@Kk9$GaHl0 zw>?AF{?wZn%Up8#z}=bN)zZ#!@v!rOtz@M%EppJlG|p-x;F+Cjm&c9UqE9qXjwGPg zPlmt0F-=^U4Y0M``lWEet_DToo1@LcY-3W4wT7VRyEku?QIL=f5{{P?najO~4%s?U zc+u8e86zAN-~Iy!#{+JF-@gy`)jRCJXb_8tn$>HU9vy`MX6OU1=(LRwa`QC-C(Jr( zzN>B99oBNq%b5dO>J>jeK8C}9AK^z9o`BUBHv^53FyQ<~fM+Pfv;~pzV|9-dg_~w4 zb*^J-_&f30R0!a< z+>DP}X^yP@{i$!*aql(6t$ceC`vZA=4|KMbQ?j@0m?c2Id04G=1bM%ezgRb5ev$Sn zTY3q%|Ch5Y;P9uj zOn&^Qvs_5yGA^ILzf`aWdjyOF(MU`L-C@^yz0V!sQA@KV?2Jey^L%UG%(qyY0Y9UN zi5KEKt8E|Txt%r(mTs6%Dl@GoT#mEL`QQNhw#|B^My~2aCDdO3E&n-6^UqT`2r`gl zesIuHCi&hNQj!!c8Q>Nd!z;vq{+StAawVCfRZ40uU!Zx&6!)o-cB*YP)XWD}P^7$msBWJ*6^XX3vfEnRmfsC}PNZq6ozXnNN;gRwAH}b|w&bQxu zBuyuxDDoPMc9jml6dY7JZ$EwEbY61%XFBtCGpxa07G8Zy}=_ud^PPCjY~gTk%*~-+&|{xa zT*i`lk$)+z0{KQdb6A2|J2A^760W%Of?c};7CBcl3E;%-Yv89s6^=)JCL4vm0u5&h z`-Ob;QN+k;w)u#p{2YBmzU#L5hn3~+>X`6F;Qu_5H+E&aK<`-ApUmPLcp8Y`9|kP( z$aTsJCxYJCmP+7CkYXZhXOg%5>)knU=vPj*rkA24Ho~a??})dPtUhh0nWF zwbYU6M!w|PY=kwXMQRh|yG0@dF&3oVN?y+l5jvg5`&QP7Jsd!6JX8Ku&+Gha_NVi_ zoAk@SEPR^*yS$<>zl1~T}jNq&dCUj^IV{OALrUaFqi+ z?G6@(RtWBJ{rmXEDkU^vfL%OO`0%Uajp+)5$+!eDc;+l(*oCWe#`Ol_hrX{u&@4tR zv_73D$DJjdZQQ18&3c8iCEx&FwCWp?Dz7~Nje z;i6fWfOH@iABd-Uwxsd*d!O2$U1g0fzkbuCOJf#T3 zz2_pxJyUk>&!uAE*#y&F5pQcWg^+?3dgdCEhZ9+2a({!c;rKJ)H#?tlwzl5RcA`jr8>Z}?r4rXOE0 z>a&d=== z1r{EpdyiprUTRij4OMB={2v>HOERy~BinX7a~thv{gbW+83%q4@-*2Anp2*V^h;@a z^jVxbR5BHdc_mfi<2S*6%n40|13WW*A+wRhtk81%oSSfs78J9lt{kVs(#>x~DEOVl z#}%)Apz&+u;E`T#4h``}+!#|eH#?H7&d<16RWzphax$7nA0STJkIesvA35~j_>pHq-x~S>v2K2mdH%Zv&D)+0tVT^79hAv_;zBjGh3BaG zuHpS=QD`W?kjnz55icg!6uP{P>ueZp3#ex6tO*K`IvNTO3khKL4lri8dAH$RLDYHQ zgGk)nXgODMXdb9zPGu}C`g4}Us8T3iksvBwQ6GWl29xW!fh%a&3k}#Zg(LmsUc(>0 ziV`fNgniXHU`P+z7hhF=qap`*T@eYoE)OZFyV|qRV zTs1siqv*Ch=#n60CHQZbd`SQKXt<#by)MthzBH&Z8i@GR>~jnpb%Jih&a;C7P+@!%XI>%T2WSGR__z|6VI zM})yo>wc_VOBXIB$HF1!ryK4oDSyJD-NJO#zu<5hCSvjJFF5>}Yw(M;_2iNu3K4o; zp58&3mPWZ;Xf#&s-~pX0na}sB%>(+5L=&Mi6T$Z9(4pGGDjzg1g&;!P zx%^tGnYIVudo8M8lY3klwQ$0UjK&CROzZWn-~p6oQ(N}WDa5!8@;DEFv+wst6NUJ- z6u&y%17c()MChHo28M!fX4-`hSlfKfHe^tz%yDMIr0&ieeFRn=LP=rnAauFlc~AJ4 z=_|ZqG|M*fVq(ilbG~+WSCQ%;&ib3ah|2t+BfTw7Aautg09Tq1LITuLNd0rBSN!c(_ebI1|vi+;KJB!3E+&Gsce!h8lIZ8#Fq z>eOYoSxDLua`5cpmpnwkatbO_*j+G!0e4Ucak>xR3B~+iomaw{?=#%_`XN`RgRLa4 zVF9@2Dgn*o9VdNTn^)cYj0bO&NFf1iA}HUCkB6?sfgyBhjX-@UvvwxQsuPDMnmNf8 z;I5oczYxb#njTSPH2=&7NsdF79WZAw-99zr@mDW5G6_I&Sti37R0o7`xzMsM%ANtq zE51&g-5!kNKfK9T)faGAmGeIj9VcNO%WAn?yR)r*j_9w|r~XY}d7sLAz7l1UZtzV` zd_udny??mHLWJr48dyw^{b`VcETyr?x%tAu3)!A8%2dCRVX1_OkOkOv$IA+-!l+VH zZ1zZh^d9p#O0v0KelE*g&J<4a5dpxa{W6=m6GiVnU^KQ2g7;3_Vg+3&$DfO^YEopv zNssCpzU4dq??H3v@L09U-7bJTTDF2dI*u4m^q&~387IomvE zO#+p*4av$65x?}7ShdjgUgVnXQJP;Pq7WRaQYhv8DU>$LeNU>XuuKeBXts$HNonx- zGg`-ICia}9U#0y6kF9$=q(rK4!~gVnuaO`D(RA(>ZAq;kf3(P9y7y!&w?3lOkZFVV zl;GRWHKMoT#B7eKP3#R(cdVsQrdj&v{MIBm8-$A&E(N(O08G$5vkp%7X)PY_KU9m3 zQQX#?5?^ObcyIk7eVM7#dehi{Yb_iu-`)-?ARh6t^6mYNnU4O){m%k4c2<6>5VKQ0e0o^Hg`@9-w& z0x>v$$+!wFVNISmwD%WFK(|_S84LjsZP5F#7xo-^EErdF#W8?x6)gECV?Y8@VAhE6Zg08j9RK8 zyubuc2LsgV1TDPDApWee2az9hBIzRuCoAutLX;8N0>D_Exx1D>qsHv%Qq?$U8}e{x z>(P(;N%aQtb-qw4bynojdBt z(6R=WEs!3IqoVXy8poq*wU`K%K%T({YB;CWLIvwXk&-KR{fBm?y;|k4UM(Tb+Nc#O z%F-h9aT)qgXfII(qlpDZC#E&EBJDQ^y)R>ozUKPC$!C_BLPd_10<~#z{p5f!Ps{&( znCIW%Y03hKOkF3<<4;*C^dX%A*WT|gRwp9}!VP4D+<8NSpmHTbr0>c=->xM@6S`6K z9^a2M)T$;zXnclr2{U<+PSz61CTYT_X9_ZIzaR8B?u_UNktkWj|><05PFIqD5**JQhGNBBV zXNdQqidP};q2v`oo4n~SGT=)FY5P{ZQJ<+wTYHvlyiNz}EOH^DKfTmto>ytHc^Qpq zAu#g!HpfnbB#3A9@m@WM7k5xY9sS`c0$=$saSq@qaz{}pbWOBG{_7#SNiLw`x>8ORjIokmv2Zd zpwh$kZYGonIP3Sd@4WLhnI7@7tv@KU?I;FZ>%X5k@M&k<0aZIeg~qOaGm*|E0xMs8 z{lu;4nM&_^^#hk*LGN&A5l&ZW-6Z>+1VuN2710$V$BWFnN7lkrf=I-pH$mo)7S;y>03e?*)UgeoMWBN7ti2m$=r=E*mf8B=pHPkM6%{J{DpHP z%KV{!6v%je^LG+F^CTFm^m>gS8#am|7J0Yiv`rG zSO`Q(LXEnc<6QS?^?2}svxsE;fWPfvf zyU+*gqbIRE&f%FWZQODL1XGaRt*#!}aDvS*COQ0mxJC+QT_>c-m2_;0`V*(!F$sfW zNPJAil_dC7F7_Ok;riWYLf;y8a`9tKeaa5M&AIr@hO3YIwD($owTwBLiZg6rGB*T>=Blxj}chb!&bEDWY zW)*q1-Xwqi#;?n}g8O5U@2mW?f@BHrOB!1Q4&TR=p%jizJRJetw1wMeA>!H8V`17u z`37_aoMUEE3B^7UomwT*FX2`Fb>CiwTC?jW>OezkfE7YdL>{EuCicyse&Szeml<%|AGRz5Dtc5Ldxy%cied zgxRH>_oqMAwzKzV?b=uj?_jxRYF=>^kWT=DyC1NMFc1|aLjnOP$5-q8hct3-fl_>M zd_HJO311Ohqbo`wg+s#zFt0fhU*^m_aicpkSObmfQS80gp6iRhTupZ75b zEWS|wPYw$EWfOzM4e(r$eUq$X`k0S0&&hWef{SJcpS||E|0(efO_(Jn;EtzSY&a!% z{woRCkNc}i=Rbo}7A%#Df2X+0jr2}5IDAr*HkKj>#*ySf*Y^fA3;cZut#5_rAr^pJ zR<%qan7m-D$AFq0_Vu1QPYVp1U1~a?*{+OuZ}IM_$sPrUX6M;>DRmplMt0cnYxP>d zdTMzKrG${LEvg};0LE$FRd+M^rtZR4Uk$Cb=n==t3rD!nQ~oq7yaSjb!KwqUcX%{L z9xxhZ@X)AxY0Z0v%M0(K=k1L8-`L890in|ZL-*^6UCoEs zV65Z+9E>fd0D@@665{-ZkPaj&yE|5}DWT-AKSCe<{;y0XWa?12&b&=Nif%I$1Eig#B^2NOnO+ z;Kp6>Lv(T!oQX&daO%fsqkQo0bhwwQ_B&Y%6CLZuv*{2t?>Ue5sU*ucP8+7+O0o26wa{rh5RY#y@~DPE+*3h1TiG zH{jM{RqidVQU@sJ`W^|0ClGfl;{hE-0qGmx{~zfa>_9mA?}5-9$lv_H4ur>6=5Mdy zaSlBDZca;9AtG$zRA7=gW>J?+bxlzPg+l75@?Ft1A4p_NK)r@=f!MyU$N1>dnpt`@ zTT5@Rl2qLHJ%5YT*q?ABIO!f*FgF0w*MA=)fLB9+`zSO3xDgbBgv($WXfr=?qvllm zwR6aIsKE%$y+ZSi+VHp;1Ll0X?{tK$T6xiZm9Nk9o%VD?Lg8d(SK{ZZVeNDtWTzr? zl9fdfO~YIFXxd&dd{9o0B0?5ls&%e5d9KvEd0gIc&-B-y`W4kV7)D*VUdvq^wB8!o z*46ceO zJKm$6f051}siupKg0ubAfAwZ)L}zb-5!A?!CBAaLMGnBTsvv^Qlr~b$8tw|XXU`7h zyPPAsqQbi&n#&K`3)x}7t;?nD4?FJ>{;&~Z&UB2&k4c^#dI4&zju>F@JjJfYL- z`eS6N$b}dW7ObGH#jm~etA!u@mD<5Xv++Fgd^M4VVB_c0nt?b2+Z@2huJuH3jOsQSsl%}OP!mnPlvu~=&g_uA`90J3hYs*cDWS)l&!_jF?u%$VT zccz8YZbk3|!ObTPQEd)Ac#+skSOt*Ll4ES68%4~zQVm2f1i|p`1zz9At(mk5nd;Nq zkc%N<`LK(pK3atz)1bIU?3xV_{Rzj;=vv>?mOdbOf;c^Dx|N9T{(4~g(1iS*VM+M@ z><#(6Uieywx+ftrRojf${-7#ZTyn$p3hY zXG^`R9-dfz8ote1kuJxoqjtscSb84BI=?zYQ?a>G1)yLwP!6%Nv@98~JG|uVwR#<9 zEP0QTpk3MC#vfz8xda#++VvqcKu@3~allypaGYFdqlC1#o24QG7q zQGXb?#=KY=Dm1_-S}i*Awcbr|8V#=MuR{9o>2;nq$8yrf#vglq?aObB5*fMq4oXzG zvL{r%wt+%ReQwJ;`Re#%w?!6&Hv*4-6MYqYwVg4zY0%y=Ec0pL}AGkx#-YfTk@E zW3KT)sjM>%Ordo@im-v`{~Wt_ znL({uC4jbzA!`ceIM$K1`}DfuE;9nBClXdP;>4;!h_&cPw(uvPO~Y+_X!kHn{qP|i z)dN^lBFV3UT>PI!AAyS=B;#1WL^;>vFFO}&?TAx1{SvNlkAbK+_{ydwg<=T7yJWKa zvacyaF%Ym#t?`f~i&!c_BL@tRY8qh-|X2t6~VsF>lo zuRo>BK3lm3Wc}1%0S;6rUJ9rE&FE?013kc8B#xB?dNqSv3LXR>dD*RE3bF!dO}HsN zo!fhL@)t&7I(Q3upl@efg&%rtJ9dwnrSQ;$PLB79zk1yek^JSmU=%?AxkWdiT`}Lh zA!zZB(*Y|M;dDrFhXy2;JOhef)&Ue1Fj6+gwyq9}J|K)l9E(Ar=SULEqli&(UMDuX zWUuT5ot0jAh4b~c3ui52<4vnjAaPAF^*EdoSG)9`b8{at*a2qC_nH~39*KFoR@U#1nlctCKH#lVfYh*>fYe*Y+L+a%^H5fFe zQl=Vu9!`{>KV-^6#VLc?rU8V|5!5XyOmcoVssEuT@d??rN_3 z3hJy^Fb&9EO2qhxLFk1Pc3*wiRJ%U&xcf*G=5b%L_6zV5uo)cl3nAP{${?I;tS`lb zblj^7$OceluG7B5$3i6S6LeA9hzBv!iD;ONCX^6};Yotg<0qoy3@jBckma-{=G!x1#GFJk z`1$t$b@{J!On0)$_DxdS?is%{5m%Fj_9C>ZY|L#D0ut?ILq!BnCWNo{uHNDi5*J<% zvmF**XqiQ}0jj2`_)ko~ZvoLCa|APZC?tqm5c&1@dYb9b7;Kge{465K3=!+S?A_YO z-9f7UzOnqYGAug6xWS33Z-BlA)=6UdVf_k$nSunVN31l4NVWJiSTNREMpso>66ZPM z@s>OSQoGGoA?O?)1Xb zZBN6^^0ny`r@Qwy2f<4PfV@i3;D6GQjhPX*6mkEu$QBg%GE-d7`+q|gXRT8?G~~BJ zlx48;DvDYI&H+Xc;GSicT^_kQ^x(p)1w)^cjjj&vJerYt1EZy}j!~s}TRBS4SlN_J zgb+(8UFS>KfNPKzp!f@91SB?I?xx4i%pW}~{y34$Tu@n=Qt%J2<}Vg*LXpu~yOF9l zD<|64ZhEJfsYO3pzVH|EZC30Ic^>l9bAMg$L3_XLc*7{|XF;z7qp(=B$5p`{k4^{* zX0IHQs+6@qt744Qo>INGN8+o%Ad4oYUMj5O6za*IS>3y~$ThOM0E9=7N)Qaj!xhMq z|B$T^CbJoz+yCD1DsUU?O+L0)d8ExSj#5NbnX%}FQMgD$OFR5*SC0i^UjPzuP@#>% zm0&06F7r2{Wmr;4)n#FlLCHKo-nnezchJ()>$tsZ&$U5grhc*`AKbyWmFcAc9aYBZ z?)NZkXG8e=|Jd2!1cD*Ozm$KfWCI);Xl?A?SK;2Z_WW2k+nJ%{aNm-Vpoz#g=?No` zB>vQN-)!~-EnyOL7VOc)AuQa_y1jA9QxtBI#?6c>^_{uY!&@Kg|9iT_{>cUMAi;t! zaKZk`vJ2Fg;*(HGIwT9t!!S1)5;(p}8H~4t(m~E-9W2H0R#=PPt08{L^6V|JGK2MB zg^v7+T3MKI4LyZv$&uwJ11aXZsKdB`>jLqHmBl4PvS;5gtCex;fxtCsqxO9uLfY`{aDEXm<^auirsa@@cd z>q)|6t;Y|<8jM9Ae~(3^QtO(xsfHIW}AFfy7lcn};_GO(=_*wf3$*JwW03CfTTV45=%x zX^z6$f{kE^&4)XmdRlr>@oc$MdHLvFfVHC?4-i z2D#>b&(Io=^roQi2F~^^m5QMCk8R~u3nMpEhTK9l8=HAF&bQN-@(pzl7(U-8S1hV} zN6NshkH`}~p zv54bMRKVGElV$^6L;v&uR*2NHn7tye5Wei11W4YsSA(I`Yuq**_!+HbNKy5bC}26wTi1NMChBVznE7y$y_2x8Gp>lU^|Ga{H;&;|rq{m&kpuO%KN>4YVytjnk~apCf6 zzat=xw;q-yrDwEd)@@t2@^pWF#$yoBGn6GRdWbU^7mxe;&oK5~98lnyY57V%U90}v zBP;XT%9gD2{k7jbRPiF&ZPF=2S`giJSIT7*L2r`Q6QH9eVFanGZe4%vo{3)kVU~ z7V9IH?&vxKGfp?!dSmi9`gjg~+2b&*IEoyH(lseyuXc}o2mPNVr36}$< z%uAJ3+9>J1o5dx)v@F_Cl>_MIkk%+2(Q~PT*TiqV{jjOkZ2a4)6d)RE+@aFH^gk6O zhUwzFW!t9L+SazZWg@sQ$~W>^&(rGgp8S0L8F2eiecToK9xxZE zA%CCY`+Bpdy>UW{{)Kam`+q`y5M-A$4rfnkmrbG`0Hx8A-pJ8nfvIaj8c@v_NCFzF zXi4{Q{!RSn(IHzHH|grc7({8~nnNS=nuU>DD>GAax`Qd(pef7T8j#qNxS8$tA(dXG^z|d-d z44tRT-ci$Fd3+PZstu1^J>K|nurq@tqWiT)pPQ~9N7c$R>n_}|R)RMR+$rO|sN8^G zr!f(go$N!@qMjy{EWWd5zfI%<*ZU&K-66-EDpu!w=9A0?+942RTrU4W<_U7mgmRX! zV_yK)!gLya&+~6B)^xE0&}4(ht7jcyzlHt|{dBw}7;_MkyYw%G@zi0gU@jO?KQQ?8 z$ng9c=tG#~G?$>er29nKxrv#CQ!CG?p`Kd$PWu*&=-3t0+yHjB(xK@4? z^eX)gmdxM8SKlG=d%p4okN8lAdTx_NAiPok2T!60(XPV9=YG~6)mTj#q#Wv}3?~yu zie^1Y71?*)?y0Pp`aSdITt|X$2t%6RHuT+@1}EDH1n^MeZfrzi&+x~?H% z@EUN_gVm+>)kLQpOrMQ`%L$>dq|Ysw|k3+U>_X8 z`oLBY8DNQ(X)#3a88x7U_DRScBnvdtyIltNpF;gtBmdp(ti1X*^Ahk z3lIgOH}8!oNGAfB$(}r2?7o5+xm*PCsyr~LKk@HaV}M%t7V7gG4Ag z)FQSE!Vm5R8H!$FZBqjlYY!*>S6It{{84ji%9b|=NF)^_5R$v@z*CY~e?6kbCzP;i1(CcBuWhQegZpdAEm1ZF= z`kw~zUo$EQT%WMMs+>V#;#_HIfajrN-=o#XB`nVU#-8!1;$4f_=+DC47F=eH+!pZ` zor|w)8vs`VC?Hb5lj%p`<-=}i%jI5O&g}jEtSXDcc#D2U2k;+_mwNl)HJ-HfgrWUJ zy>NmGPdodC#c}4M8-)`?m4_u`b)NMeGp%A3NaiisR?fHkT~dG5;eC9MlUfbMoy1hK z#kIs3YOPkmoN=yvoEq?-tr3>yX>7}2SW?mM>VPzfucX?Wr;;-YeDL50jU+Kt&HO9} zbrYm&H@L1m6xSnYgH4eIr(c!cXG_~`@~ynFa|X2HBb|p!pUO~=QyqKT{`91yz&xMU z@Ag0sa)IVNTZNifEPw_4Z!$qq!&^`4*R{D2ig$KyVXdYc^QB8SCiA^ySyC2Pl<<& zd_HDG)Vk=K_sEBdNEUX*?At5-pdRv1f-Y&*Y%E~BdKrOxZ3)Z1G(?R*s{gGlcB|lb zb}BT_8+y<%Qg^!voUM)fFZnmjv1j_U;?f1Jp7d%I!6u2cJEm2_qz+7p-&)K4fzPzK zF>HV1b5OS3#lo;}sqVW8dL|P))!oP2D*bPVJz)0Gl8=n5pA&}O`oKl(b%NnFSFfL15W36iz2iN&Dn`=jeHX z7o9Lc@hYA~ov;u`l~s+>D6p#v>U9#lm6T?(;=1QI#sq~gtso`gM?fpk;O-3|R@%IQs(hGh_dk{Y+rnfp%f)L)*V_o6`!$4d$W+ zIT-r4HI{B;>ut*r)f+n5wSlM4SMBR^F}|gHK;A1ysj5mu#Jdn#DU@IWEqx6Rm+#D{ z_I@Djp>{-kbwVa2KY74WtWOD1CqXeCHFf)MMgDJCSx-B;frBH7qa-V(RZ$Po8ON`~ zM8M^_9()dy9pNi79Vd0Uh2-brFiouVZ|3Z|SrWI#J;lEY^gV5pUfyQ|`%&=crif`L zrG}J0dM7O{KaMh`qDwo-GqfnlX1k@`7AIv2y$zn&#(hd4hEgX6A^szm5D`xJu*R#Dn{bnuCA-dJ z{#n}H*cZuVPdH3xs6up7LtBE)@m|6@LHE8z5h1k7L4Ez$kxw>Fr)&HKYo}f#69@YD z2#QH1NEQ^gSUVe9d7J|5OaUxZAe`_&>=pvn{j}M<5m$GYzXWinDWyOrlUqsILg_G; z23X-iqHhZMcL(*}6-_>-{dIlv#;I|-iCj~>{3ToucL#f9!8k7qY22F)(Dv$GBW$es zKOrLkLoh#p9Hc(16Y`0uDMH8QaSWau<2iegPNH6_v2&urbGjqVp6}};&|v)z zm9VJsZMJlw#gge!$m0}t@K9V3n}#lQ`^Q$r8Q)+bO=fR~y)t#LY4x6XxThliuJXHVK)~C-X1H()V3Q3&$DQh? z#iLt`4AqboUJasS7=GMyW0r26JGauuQ>(VPEs|sz5U%Jo^s`< zwHL7Rz{D=Ji&8{Htv<5qbY}Dzwj&>8ej&`hzfX)UN&Ux&y8P@($TNPKvOW`KTa!}t zMCRZ&Wc1p0C*O@y{T8<^lo@hGfA^mg9>eipI!GaS`eMyy!djr zqMh5lcaA)^;!qy%O7tP=$JTzT?7G+TXFfWQ7*38R$fwY(^xK5I&BUknCBOr)u2jcv5+P4X|3p-WA5D2s-TJM0QDHz@GjOmQUXn-_#`? zkNtMvDCJH}0z&+&`de$`yEkU0bvyWb;hVP^qy;3l9ZVH0pLx0kXe+KZ`8YNx2BgVZ z(USe?ZyRqzjDsdnjXU%kORU|~N*_1btIC|pB)oWC~5uHL! zU7fVik#=e{-%*|SI9ymLh@(^`FwS)B=vQ96tPjf5as*ie4VAEboWI<1vteHW14ImS zT38cNx$1hEGWbvN$E?22zc?ht_8SgK6VM+AWS#uGj9kmC@SHh6KP)nq6GX;nR%V{R z)|UizZ3}v9*uiWCyeV{6w#_`^MP&lH*+EA`{Qho!;{ny;JUixNO5~_qU=XIPSfnkVVq9c~j{^-L@Z{D=J@05LMZta^Z{pp~*7sGe#b0Uqw75WP zKb(io^DedPALAIMWth=0$%6mqf9?>3;6~%DjsNvSv5>;hfB(H}DG~12Om2UEmBiMq zQx0E-tp%K9F1N(?W`EWswZtkWG;a@RL_wN_$yhRXHs<4o>Y}27*-3tD^==imj{J`w z2jE%qaMp$|ja)6}r+%T1VY#Kf!Vb*vMNi=#RJ~oYjRfBr0l)7Y_3Hj%N#Jz+na_Qx z8^1yr9G6|ouw1q2RnzB0xKWT{>Z9T3^j8%lCK)Bv$_I(r)&xeLfJPvO$yO^dZ8gpb zjFJ`(Pr%YgRj*60FqX~VB>pxbW=!r8uS)$UwxR{!eaBR0dRS{q{s^$=3Z}F?9BDXB z9z9ME#)jy7;W@bCatEy!_FrZqe)~lAXg>)x%hqU19566MP%lqCx7I=jX9L+=U0`3K z_A!6o>f??m?;5orDT=@M;SdjU#`fAR9UYK$WvsV#F;G2J<^D+bdsV>y=db^Dku)aj z_djH8DqO+0hJ$m(+Q2g{;R9(B=%l)}Iwp2*t7=*O;%FPOB25Wn`AC1GnqQ^LremEJ zX$&o03Tks^`1P3Ym3>}iq9p?nOxxL|_@5=+OUQ0!Gajeye}r3&dWE>l{MoMVB%({k zr^DHXbRgeXGO1*Os5V^q8a2aX+j{T;Y}J)F(UObfhn);sh)IN$L%D~~4O{vtrnvor zTYV-u8Eu>_*=f(3ZmD>%aszBeefQ^6#4-a55k+feGfABumiyt7X|NZnj30B*# zs!R_2Sta^^;|2F)!>j%N1se$fy;zeo{yoyXPXefxEK-c5}1&3Mx;bhPug1ju(-E4EyEjTx}v*We;+X;*cihpXk zJY@ctj_Td0yWa_u>k#9y`?U}QcM)zM|JJ$DM~98S1&HnTe7DN`o@xhKL~}0=iRH3A z-uv7LipkZ-yC(Tc-(nW;rPClxN9s)zU|chN$epq$M6hcc@GZP z+R3Zj;JXlbW>tsHk?&t}bLlddmT)&C^+Honw{)CWPOytyJG4i(Adl~Aw!|iD!ld&| zN9??z)WvY2S!mwJ+4y?QAlI^aHs1M(#*I!I{ zW+8l~7XJis8_hq{;ZFP<=jpg=Di?u*cV`r1JoJa*DkC7R;#M26D8dc zWrHYq{A3h{%)XM@Yqgo1iK$Ref?w;N9ZW#$5elT1E*t?xii?%E*}HSKXS$x-1Pk|0 zAD=h${;U4@JMaKCOZhF|X53pD>=*O=*$N*Wv8jJBRg=gvADtcibNDR2K4no>>6AIU z?i-Gk49GWj&6|oHzByq$jKX}ng1262w%BAxnLbUgl2EHOtJ1x1IVYxmtT|NHZ4U;=@1pvYmA{^zQ-cCqw+JLRS{u(Kso3qS*JM ze0Enqq&Aw=F}tyAkCwGvVd#^j3G!#Vg^-H4Dg7+v5D1i(f#OwVyBt1?B9{L^lwtWBnu3?Y7M<`$XUZ%Qd5Q$JvO!eIe zE#RO^?~Cc-^(ql7fN~|1DkA9gIWj%UOb;}_;R0LjE*J7_tXrl>aW*dh-RBA99hyGd z`o)3P&aVDkT&D)Ca%efu!fVg(P^ zkXAzJhPCMKMjE7Bx;v%W5AXZ?zWwceu5->`z_qyNGv}Cdj5(%zte$LFzR7sT*fU#5 zj3ISw2hL|aOCq8|{UD&{xT7kZdR87xr8iLeL)w+!*JND_Q0Se!c;P3pr}f79HL^bO zKlCWOZr^KW;{3g}UZ~3%F{YVDaR-74+G`}2SK87WOfYDyeK%4LygD!k&a=mK%s4FD z0CZx3A#yLI$QZGbWs2Mn~2{xHtH=JU(HYtz^tM88t`Dd_Zy?Ft82>#L+qp?pMN zbPMG*BC0jRi~}!43`)?;Lix{RPx97{N;%p5e=_uK0=7IO>1c$dm}7G-?j$Bz6uR_w zPgNY_m%`uM?8qkwa_eUDxtZ&Kh_>SXe?(i&T*ZUhXP(-${*&5b>h9F`V)$h#epqv^ z&N=|l-H>b}J@n>op69X0=hs4K6zL?Tj1@JZ>w~<;j^viy+=Wz^p>4oCUP>|(lTF9A zwpYXqMhj|RN&E6`;i~-$o&U#Yp(Ty%#mYF0g;iVL6$|Gla{PGsem4kWBl>F5m_`n| z`0T+RWEE3y{#2)0W2vh~qHO(=eSBSs2(-PIE82wf}(6eog--o}j3S zTOM9xdR+q;?m=cXQS>>jOz11rv9ybz>$wBbqs`oM4LD9 zb~OCguMeP_>9S-3R2j_?g8j|UCwX77({Hak`~I)(2p&XX-M5yU_EYIu>08%YDuJYj z3xK!T_lj?rExoekUE93rD&;V8K5a|j>OF|K0X4_BO+atckTVt&`rXFNK@D?(C*5<3{;oBh%ftL5|2z)UJ169S+w;_!|Jj~%0{*}MH=~VaXN^MrlwLjq zd(JQ!Zs8&%F^TI?mHfWoM0dV-<$K=PU499t&?++#dlen%NvH3fl?Jm3LngUY!4yt3g z=Hh-TGxRQ_tS(NINKg5dChUMaSGkKy$ay-i0ad2}nbIZdAkHMB8e7 zZP2@$`$WG0D1pW{0sqw#dRc9qkFt#+3&_unP-u>UlNiD$eu1S+1<^GnR;j|Wosl27 z2|C=}7dl6JB185qAQ6%K7$#{cDm)%SrgBg>ZAYQN+Fb25w*oR!8U9)m>gU6F!5CZF zgzxOUxwv^rNhOMRUAME&u_(>Yvd$s*}X@%$iiNq=LK=@#(-g)Vtxkzy(+=RiGb zl_WD8_ZW&&`Eswhx zf9PH>u8zS`&rTrqWAI$`O@^S+t9D&6*I+`Sz$KxO3NYh$YQhrPY`_aArUn%wfd{5Z zr*atV50-=CYc$pu?jLhRMm1vG0ccBr&rhSg;mN(Te62GW@6)RvUoF>l}*bn)K?f!P?9oq!#6cH7=jx_^fYWj`aek{QH zm&srmrcYhw|CFNqDPaw?z!!3jKO@3z=QP@;STn%WJ#)5$uBlNhPi0|dc2++qS^@ZU zr7hu40i&vIb}DVojUgfTF#{4}hRkfC|M37UoIqw$V&x=Pm_IkuR|!0^SSQcFGt!ES zPewOeSEi0}^h1$i(%eAXCUNoOWKeTnrYMUv)=n8`pqc{ZJl(X>U2Q^i1jEr=YJ7Xf z64*KoJf?G9JIt!~*7^x0#_j8%(rZ=`K-&YN(@xg1kb2|EL27&k`ZX4C@A!!w&^(zm+Rt!>@xq@mDOW2klkSZ zsYN>=-TY7=Cs0a?c(t`=KIwgWGGEjX3D-|AE&u&QjJq3c1q+j-)z{fz{SwE=%{P^* zW@_Rp{wk_a(c#9}N?(qdl}?Oh2CR}QSxQHS0ONHfepl{8DTb7hy)&)&t=05}yyzC) z#p@+64Y&ZaPsx?|%4<%9ZRK*t-o+ zbX1_|l1j+^H+3`}i-Tz+hWqCET<%?cQVMkn0OGuja6*h<>Bzs+bC6Xg0GEdfZ|~d=3L#Ay}Ar!%?~;bEqJh`*{2<-R<^nbR4W?I(H(= zDH=~Wcv0JncCup)eQ-dLDoo?}S+7CWJIm>eIEWHi@3@-{o;Cd%Y6U5{1n}7HfEWU% zVA!vq-AuOYwut|Z9sx_~+U=_wWw9BSXWMYCmuQmH%#qI?5I#Z|#qf(@(&zgZ*EHCz z|FYyJo)LV&5k_LmlMS-bW}dgGh%4V)po-W$%1FOnS3mFz%NHE>)v1>

Np=bxq!k zM}^#=+AU;1)dyfL9gwneOa8yl*3F*QlgvauI0MI6Xc3dDaS>fKYfv>Of9=yYXwVrX zK&2_Skh!3o>#%K_vYyv`re}qgD5~2r($Z<{ndIOi#>7Xf75txS*W6q*^^~^0Ty@e* zNqpaz*b72|_xx|H)iy?Be;pU>rf|FBZhIv}M0lIS#70Xq^NEFV4&@ik5E~awG*-A= zc?NnESQeU#BE`ZaY=I&*^W;%M2`bhV7;)_*-yOoA1yc zyR8kpGhUeEV+q6LMNd)x^@dS17r2uUI*(%tEE3`^Aak(;Mm4XPw7Rv=xp))aiPVwP zm3h<7Xw?M2@;i8i?&9w`y}X8Ei>Rm5e~FiKF|OD0#9`{be^P(jt~1T~7Ori@L{qJ7 zt-7htF=A|y<4F;}iq-1o1Oak>rhq|8a8bebEPn?-W(mTI-ee9jp3Q;GCV&z3mt+T* zP{HioZUqy%QX_s(ZTpDBe3#Bs(N9V$9lU;Z{EyAz{^b8*vk3eDuvr8lngLaxDLb!5;fuZ>>sb^M=^&wV2!;6JEIEE%=XeW;Sb_XO?9tVG(W=#!Jc?c@H82rc z@sf-KM{UjYk+|}8HY>fDsu??G)j9gd@#tG$tJyfJ@6|ENuhz$A(KjJ2eBBorub%Es z{8uqD#K6BgUo6gxHdk^qQ@CCjkdGE3^ReV=}=rK70zr5GJ7#Yxf1RnD@K1?=8 z;E?`z9vFpqt8{4$g?t6mCDH+qR8wgIrs=4JMfU9AqN$I!y^F|WmnYj9L?vKb#N=n= zvm*UPB-zX2O~AU~WRPB91ztbD!O&Mt<30LpmoH%l&XF;S1Mc8Vi>qqcQ**H zdpBFIqy)>lm|hMd?);rxu{XGug#J;ZtYeSBDouvIvtEg)pDI=7A7do}sudx?Pxn;j zZteUeMn5v%r*cz=M0jx1OD7|L1PQ@l7c#DkeE9BaOu?0*9$m=o{P)0{f4TQ=v;z-n z3y#8x*(Xoqr~PBxG@_yS>osIIJ8vnQynTdXb4h@%j5FY z=y;ySd7f4geG;6g_XCF3>pxs%pOO4phWa_9xcYI+(%1V#j5|DDq0fD)*A*TRlz$f) z(bQ6#e6*VLUbu9Y$l%`SUReBEP(7UT(Wl(^@OTt^5*R-IcXE`+Z$qxv{ul(aHUm~R zpIfAbM~kBapZ)&f^OkCn;kS??rntp@D;K^GY7AM7wf!%h7yQ}$ zNnGy{-Xuwi$zWh>BRJxt*UNG7#TvnTF+l6)UIgnjU+l;wNcm9-$=nt9U{oZK4&S@Za|emXu4x2POK|M-jMg%eLzN^$}ge>h#VFOe2p>^%d}J~=bQA9F;zZ$HIhS-^u1{; z8a3x%{mxrZt!tUbhi|@{KHiygLgn=-7}yhkTE6N$hY`}NpvoQDhQ@SQgpR97GCKf_ zF3Bs^@-f+A`+)lr4z5e$?(0Y*-j^hQdbM0&J=l?lGZDLgD8jr~^loRbbo+~}lK;j> zzBENAM*OdXoNZxJ8Lt`c!%(^~eb&)s%Mp5vVreJaD$-Y2B#Wm7!5d#9wu#(7{^6GV zk;$)1?GQ?r)Du$}jN6ThHf}6C{p~?DQKCvursa!qC+ZlqR@+CLOxBSxEfC}<_3@yT z`ZqQrCV{NwZxLJizl%_0tnrd598Jw_Z&2=!a)+XzSAx>b^1q=bxp^-y& zjU#j{X7||zOo;e&cO=vJf}1o@E+yE(&X1r$Y$C++LjFY)r@g!Xy-9YJ?|zOk1R~X# zxNw=qjePa8c~1nDfaQvr{Rf$ezdZ#LJTJ5Z%W1R}7m|UIMk^jaCv4`_V@9K`Y6e?e z>w{K#&%|ui&xut#%jM)`Ha)TkY)rI!&9MMSa*9JDYc9m^JGN2F-CFI|C(+J^KX{xHSyq z=ZRl5b>z)0wjN$oA#^*d|KpdDOJV58*6G;WrF7Fw{Gv;|#Frm|H=#0oM{r9*jF^vx zV3#;I@I{s3L?gA%TP#C=2|w54t&}V*;*{(6eU2X&wd?)nz}MtdPZDbNh$tAxhTE}| zhe3~8yi7l zc^lNbr#KQu*oh7$@H^>!^R|!LVV&yxxJsWO#(=bYdG5?F`k;oBKaxvZvDu*fEJCsz zQ@R8xPbz8 zHqMlBP3;Y6B7=4nT6*4Lf1*2Sf)X+!rj*NG)Suu!`XG$&;b2<8VAW`xuf3P11rsPSC{zYv?;#+#+ZnYsXScPS2+6nbYhNl!xoe*D7d#hOs=<}`5sK9LDNLoES4$h49mRfO3gwP zx}V&1y3A)Xmsw8apDJe)ba|?jQH%YSlYHKmJt_=i>wYX7d#{pm2dj(j{pQ3b5@YSK z=-d%MhJRqTD^L6VaB@I{GqsTfqJ!7-56({nQ4%O$lWe*Ec=1D$U(cI=*Pl^3vDxfDiWW`;Y)thS?$H1rQ zdtYx|#;LJo(cvfONtA(z{i)4wal5VAthD8yx{u!fb!cw-@e{wgLGDrK+;_0RLye+7 ze*E7UY6J1dqu*T}e4Q?O>&Qe|yf{ID?zUy@j*X3Vq_U7`jKuz)Y%(~qU@H`y(AnQR zZW!6}e*HOcyHvI8!+AJPSv=H%F(H-W0tY5+HH$Zs9au9DN)kqdB(>2K>f~h_)6EB= zBZ3{vvQlK#yYANL7;vd>)hrP*lMu!%{j}T45n!wzo{vxsj!{LF+g#Z&LLEejY2SuJ zitUN%S0ZGgstge-C860VSzOj|f@4G3Gz1JJ%@qGUn%)%?ek9cSL z=jFTOfb^CF5{0GvhnjhtwG*-BmB>E0=FTBoYo%vsTV5}OsI;Z}Rh?oiT%4%n&`s6l z?tC6A(c_$xA$8OICfwex@9Dqq=l#kM8SX7%hDM6mgX;&l?D^0dYEQ&F>f^&;!}Bzf zxi^a}n#?RszIJ;pR?`ki>`i6WgnaK*-`8@@vJH$!-jM~CAD>S6XfQNq?e^oF%SOcN z60(7L3b5m_p&buNXs?5%y)b>;V@3~bEsR3TTGmfJl}Y;KdTuJQI*;Y2&44MOj%(?9 zC9PTLN8p&NKkfl~4L zL;v$aIe#H~lEiV2aQ%>@f_p*O`6PSoV^x^f-)|PqGLAI6Zuqe6V9QzMIreT;@$B)@ zSdU4Ha2!Z*N}`L3i)eMGys+dK)#;>~+3i;3c z^3eD4&^^vYqP~@EYExgO{A7sGkUM+*zG7F3=i}#yO$4jd*x;Gxl#|JUSe0cA*`00u z^MMK!AU17hRpeTOxV6u8AfSpBSf`K7^@jl zT5W+GepeJZ1TTjx7L+agEjiHn;<)##HqV@=hdk52^`UG9M^|`pq_39_uM}w2#nsxq zQ?|aXX7iBTVFtCjG@dK^@cBH_WUF3uQoHwr8LYaZN*;M_JGu}R@|%8Q?D~W<3$s;W zWdA_yF;u6LY2hUCpOV%FO2S0^hKE;RoGf9_gg3(G_zZcS8%Q)XkkMy}8j(UlQXK=b z;p?4NWjqXAPO`<8PitK?ORZoUjj`~b4fJ;plX<6qSPjo;5c~kpaDM`21<-<4qWm+B&N8Q(3wLH}u3ZABSCpN#-j5$)OP)e4R zmjZone0NW2Abp^nQff`a7#Puz`2q-C@>Qa;RCp?Wwn>MjQZV=15M7@T&a@d?G-6U6 z(mpx1gD)wvsW0kBOx?fYr>43jh%~%A{yaTU^vO5=4-?TLlq`?`ap9n(`ev8B9baY# z$2EClTF_b@9W>U)>{Kth&Wv9ww6e>Ct_k<4WjNdUyBl9z0&Z3J`Bg%r6aRy36c#vO z{$B|C3-xanL`9O0NZHYSr)21%A|u{QDd4dB%(FFr_t>sn;~&QcC|DAK@cxflOuSUk z01vPpRVWGP{PQ4?HBmUuO3X0G3TKy1u6$+mrxQD;Oi_DrVmJt(4He(e<@ zn5L}Rke8KGX^3^MQoiaFDi7cFlhxu3jv+ja57q1@3`*s&#tWbb*HkNSkgPsApmhf;L1K$FW8f&e<7eR3%pdmk~zE=j6^y}Vz?24BN zHUWd_wdi$)TS+as_*>IeyuHX{NXg!llNBu1CN%r!zb3(FYRuA z@{Z_&7acu}*F#J8q@laSVY`sKUfactK70UH`%m69N7IMu#zGP-MCO^t`3Kp=0#hyJD}DoLRBD3j4e~H5;*{W9vXR-t zQ`1++r)i4J&h*W`NbfT_KxVq-{f>_S;u-z;Lf0E&G*^cR(;pU{g_Hk0- zMD3O~+0MDVS2{(BpPE3?x9-Q!bFHg8d`G9tc2MK6qswx1wJzIuKFiJ5vB+Pmg}D^0 z5^zc!qSnw66YUsOM#Pq<%LG6Zs&yHn9oMr00(0M@?J#Yd?g^w0yT*tpj(N36-tOS) zmakb(r7MP9_wWq13X^U8wUR6k0aal>pCT*KdtPcQe7MfeZu`9C1t*Kf3i6&Qd~dxG zXVhx!vi`%4&FFuDC=TRdfq$aU&{@DoTah1Z@nI!oTvR6YTPk^YLkoEbcU74GVQNT= zEdD^eacA!j#<=*XZvo{R)?dE*nQ-=|k1BrOoA_*ozi}-dteDEd(%hGTAGJ%$@G4?N~0(cuXOveU88_XzA=PB*T`!%H&~+^ zn$yUsts=!CS12fJVp^&u)!(0EoQpK>LoQX!{Vja5o>!v?Cp5`~2|9|U4JtK?79T26 zYfivX_?VkC`3Q4auCDwXhoAJh**&hDU(sy(FFXu5Pi)C_kaYMftVaj!}L^J#{*!kyPx74)+HUU1qkmuj~FEADtP}SseI*LNA zz%Nau*$W5u8MbwIZHQ=@15a})6w^s3y%dW%`ooGd-+x-A1>=VcE#| z@_Zx+vyeIt^0U;)sE1l?>o)AL^YPhF+Vp7A7fW0Hn2REPOWc*dtp!?D7g8Yg7=*G=nF{^D?3AK6 zLDrFwdsfyn|L*SV&^)_5c&m(IT{$7knC>OB6{>R8SVje?0`gq6h9*G1+X^%ksOlLL z3D?q_YkToq!GB#oWkgsDb77Lq7EIPAar3C(z3~%k7pZ92xS8YH;9zuPCWWTpo-69G zkk@r1Tji{12`5&?%=J4Pq+OhqO!Bp#h6zjC-L6-isp)N#e>?*u%$~yVi1vdOm`&S! z^~Z<0b%q%ii$Y1becuSNqmG`Y%gEoPUlT&`MP3mU^y6Q%(O_qFuPI)WGM9Yq*W=^-e7j+l32WLxuY>LIQx*-Ktnf zo~AVR9LN!d&KynqmJ)_m=)XZr#_x_{km+>IqUYti^j$FJLw$@E`dnBwo8zHe*)ci| zIEP7m9EC>xou4e72OqRtG^g+@yFl&KtE^6Yqt7p|*xL<__E_^yp#GCHhs9{hpKvx< z>Gm(R5P9@HPWvqSDI4a3%16ItHW|gyX_I3pqxZ@Gl8#&$%wp`gN#aU+lNFGL(uIUw z$uc0wrp#!!J16wjM_@mLaxp}gr!z2MRo9M}AugEgH|JlZ9~KP0>>)#{8?ZR(tdU7s z=<)m&!VU=`GF(%~=)wKSsrHygk`o()#?()o&3p1IwgY$v(_g#gn6}?ylR&vf{zb|9 zp;*sipk>hRheP=+@Vc0tCHP0o()|k_W}l*)W~GL7`2;}B_Fs{?TtsucR?DvcDa(B{ zNByBl7@l2sgqD7o_xSnhx%b7ufyleN*$@#YNm$2icFVASI1WB04njB%1`DPD&)dMS z3;38ax)FZZ3oIe$b~!T{wm;_{D`Vh?6?U;^I0Z9u<$jAj0xQn??^yb%?lwYpkNt6OXrez?fCf9Dh)C^iR*8=`{`|~;sEi0y46cN zd(&3gLaX~sbt$tRiVnJg3^uT`GPmZ-AjR599#_%2^-#=sB)(2M6M_RHv}VmS}j0(z% zLUeeER0z@g+`N1?AEX9+YOKa+M;0?Z#EDnN6iovBltp;j=|cfdF?kfOs?H%QlRqdM za5ftjSa>>q{ep_eUkRVneYK^;&gkPa>L+`*&JvEXu=N8PGH94Q9vtSWp3D%ht+OK?_AzVj6}=Lw0;^8gG7^MW~iLgh%?DFVq%(2KvYG(&J;CbNPUs1 zC-pAaeTQ{zs3|#VOGR{FGhXS+$8c>nhgR^{#tt)Zdv{9Cj=vo zL^t|8(TIo-ISP!ZGZaTy411qGUtYhbwfF`63#dI&6bN^W1e9h)ja(MXqar%Y?NDT< zkStB>0++&o5C&D;9%PT4{6lG z-e>R$j3?uLid%X+g)>Tj&7O?r$U5gA$ERcZH`GwaED&9?85E@s2ZnUh zrGB4Pl51AGVNAZmi_3a7Oq)+HCij4;QD#SQiJr^s@`+lRU-%>`QqdnBjn_5aeG}jR z-c~Ke{WHZfAoN?@pKW<9SuFELR+0%4#qHVTm*_mpVKxC?Nip$S3}mgvPF%>GBWIGN zLb5j7D(;}uT(hwImO3YX)Fk~UTCe0XCU^X+s5+C+(|BJ0_x`Ypn_Lpww5xSF9EeX}s_CN7C{3k#9G`QF?;QQJ1K`!jS#2Zhd5W;;t z>&ZT=z;8UrRyBU2l9CW}>Vs$?=pj9DbsNJOOsTw|J0@72DyV);GEzI_($y3-6@;%e zr_5hgH3YUhjYf1D*L1TN>*p%xI%?GOHfv8ZZ-k4?Q<-~jtK8_Y2NkW}a{^fh7i!~5^mlxEiw8pV=Tb6+ekz@gBi zH!$ZDi3q#`ZFh@7=PVQb`7Gd=b$lF&9v1n2DYbn9Xy&wdVdqzEA$h-5z^?mQjt%bo z!pDC)!yCY$46MJoL4Wx9LASW(<0@jpmIlaDyxeF-_sU|Nv57(PJJD%4&PA!dUU}yB zo?BdeN`#vAai^SBIL`0a#6seKLoa=DXMP*~K;i?FSw1f1kMMsZ;JlA>GN5L;a&t9j zp;c;@9RIQ|I(_vAO&RKYSL2_qBd~`a3d%3UK&i-9D~~IdaKU|io-hzUgcOGW2`1E8 zhJ^X{$tQ4?Ma;0Om19yt!(94yh_g>w4ti84_tExK)}i`! zwEU3z^kojyrWNH!_wV8MeOiO66>4;7aKbV0!gv?6wpeBVOIoM*`|iS~B0Zm|Y-Xap z0{xkEPv0cvCl<;8Aj3AEnd)FiBD~_QpoE!78upu#hh3;=yTN{g-!)V$h*Dakk?ai z_X(KC2~~<2rFRo3GlMEnmgmcGfVR*~1|0pZbP@HHm{|%Rei^aex&7mnyM)Oh2PGnm9Jg1!oW~h|jem65^<&n` zq7nZ#fdmIG7j4##2OfI5p9!L!KEi2&%#nGWC1o{Zn#5tAbLV8FEg$8pslHGZazNd+Jon-h&s*MDVEFY#c~!8$LlMRxWba~uvGeUXKDAqgjLw9TrCe)K z|1$zvi5tg7HJuPTLPSb|>2=cSRBpRHLW@2JxA_2FA`ptxcK(Ld(>#rT5BHsAU)_e; zjescrgADMVaIH{UKzs)%Lk6laty|V%B~=20L%R#0?;75rZ*ku!heHZ9yF=-7Z7H5B zzn}+|n0sp1p%z-)ge-QPD>nG{DZnSAA`dc|ZMnv-@H`t&N*Tv-*?S z7y>B8N+++SVAZ<4u>E=O@wVXH=Xhrl=N2By$n6O&rEny){;ylLqF@&=0@iGJ;I}kc z&T~Ob6I6He6;MAZRBFWRt78`G&2VFDKE9EhCSzQPe!@t$5{>($FIu2U;5NOe5XD>5 z@~qIJTPh?UPqxe|A(=uf1*TR3FWv%Ix_g%0FJsa_8QvJVl=!ld(HhZbL1@LPB;hEb z&^LG~U$0MUj`UBAkXp;vE4m?%!gcXmis_^4cFjC9}+IdrX$`oT(u_)Xdn=+i#0;b+Y4^AXU;;2T?;z*mhwj=b2gjgt8a zxSjAP!su&YT1IDve6-B3wWy)spknJJI#nz((eKV3C1mRWjeB4yY=GBz%@PCPqzhf{lx<0L8$)b+@?&sF*IYfsQr%U2Ny97`6^=m@CTU8xA2qQ7HUjE+iFe=EFk+jo zob?0j$)ue{+{c|pfOB<8&_qHPYT|SU7}~Q_!Nw4D>u_D?lU*IbtO>$;Or^)&C?c~S z<<@5V!u&FJ0wt$6_sR?@xn>ps(`JDTAllSeG@DxYBK(VaQ6ka#D(I4qW5-(+&zGqW3n=N)QjEFt-Wz41%cB?ME=JYJJ>W|{HK-@7Ug zOM1LH$Lg-nm{w=hqSk{1{xZ^udgGv5o~h`Yfj7Qta)Y7AA;=nzv#>oPYT+Iy5|Ag( zs#p>65355$ucGOBWqNfxf#Z937thd`t5)v8riou-ipCV*$Z4)(cE#bx!P5K=>5X(( zuy{G9(=!W84g>5*s-ZD4ilJYO+7~ewzsQ4N>^7Fq{^JEO>>6bSsq?$fUfU!wof_f< z%!)%BW_KC}#ZiR+A}igY`hW%$7Xn}V^ZM5)0x})28Fbe3glBDc@{W3i4AHzU$5$Xqpde3_{=k(^@%S81m=u z6rN9A!+hVO{uZ5a_r6PS=B4-6)~6RV$11(Ea!;)rGV?7CF`~@ZHt5y~50Py3?z{e~ zo8Ktl`rQRxz6eU%_7ps{q+s7`)thPR{5v+??@LBn5jp-3*;{rd`KV~+TPz1*Z6Ek< zk@DU}3)-LwJv3b{8q(ZPH`LZ`Jm{V=8|>UQ6Prj-@8upuEEs!YN$i1=R)zXVyz`D} z-fMkuXp%GvT+E`KQ&l_q<$769?uCbthR;A4oTXtdijo*0#Wh&Vb{mW|pa+`u_aIAPK)Je>6)aJ$d-J znstF7VXiJ+5KJ}8Y*>^`I{deM>081TsGE3vXJ*ox%6T}1#xN{2_c#CDt4&1;+!d|g zl)b9fF%q06*L?(A=6$tl)@}f~%e?DZbX55`rB4>WD#>E605^Pl(jX3O;4vt5DH|$K zCu0-v+kly43pMZx##_FHgV~$hH13ADkK5o{EAwiJl-;nTVQdaR3ofYb8Kh$fY%F}O}BzJJ4TIVE{^e$B|43apDWn{W&;xC@#>LZCcow1 zoG2Y^GGSdi9Z<_>*l4f;>GicnUrArXoD2+sKqW%)T$ecerp_dL?e; zqF~9A;2r&lcHThXx4LfKmAEDM$R>*E4!X*ru^oww*WlLf+*)NyEf*P2KvJsH2-iIL z(N2vw6RGRm%koz!GL#+eNiore!2Kmsck)?DLN2vE6a@5N+U{d6={@*<#lt2kP2l?e z_;nbTYiXb`?Q*u*)zP6{li>T))*!{ksRpjwF$b@*{LJ@qKOcD((Sngh9ydxo9>F%RD;Xe z68v$nt&GhSK8Pd!`h(FY9NxHD>#6P?={kyY1s-^ijJ8XZssSZF*YO6tmMaJ}6J+X4 zx?JlCOy9W?y=LVp#_6%wx`0VAokcNyY#cmW6OOX^wc3o8joo;L1w$KrZ6#U91Su&= z+}h8w-6)0V{}%2tA)bd(+pB@TaFj1-XtN3#s`o1KE#+wh=jF~&RWXGa2c=A1AKb1d zZfKbL*#tS2J2Livl$~%Y9b&e4HSe9A=F&3r9R0OLCv_JSAzA1d4%|Oxx#|%E>kv#f zW!X58d8JG0U3{NLaDsFHZ(GD0CrF^QH{uBJB*POGpU4qn-*NLmL>4^x?<6#e9`^F= zOk0_t;TS*LTUIA8#E^G(B&spbRN_U&t3fp8si@m<8P(mGg3Y=Z6~CAwst-FU*z^el zo-w@hdx4luzlZS49VnjMoX?aTKYM}D06id6A>}fN)HpdzeEkR zM1=2NW)|r`@(-h(mC0!K5w3N4V!vn}Ud8;DJ02pcem?-PMrp!jO0zG$|GOsw@&0ep z=;0jw@wBp;E3$Jxjx_t*d$|{+?N)JLKIjLxkw^dv8jF3u`2z|{el_g(GMB%4BTn<~w-on0GQbvuzgy2p)t>^@-p^Z2*4dGz77 zuy)v^g{tBaBU6%t8-$DRB>6|Yq93m1jsou@DEeR&vB4ASVVe|g+0CelT-ssJx*hT+ zm7`meK*I`p;Ns8|ZF$(@K5ORN?#ZcCaoY9u6Y)FQyV2e`*3xNJWAkZVK)?||o#E#E8w&a~(EN$+w*yUU2 zO7w?IJ1fN(9Tvuv)l=IK;xkGhMd|;QBhtiVNKg_+q-o=heX*Hkz!Q6<@BTtbavPVt zz$}Pm3-{WfYHhZNgO7V-cJt-exB}JNm-Hy)Ut=Xyhr4*yrv@xgjs8d!Q(G2q5cxA2 zv}N+s=F9}4kcTgl9j4;-DXg0pj-)Mj)%wJfX{D{R65@V>Gt-CdRGnI>b<6*?U*l^Q z(Y@Inq~voJnLzN9jqmprVS-M8s$3+B3z;5%blK`k{1m}0sa z#U^)kiA_7p2yoAQ2>%9ny`tAJbQKm+E@(3n!I8;CG}u8qY>$Hz!k5%j#XgKuvc^Xq z%w=6_2?V1bMj>A1iF@-Lkj zvcMPVQ%KSgYWOghKx-uh4C5(Xau3AU!Af1isf`L4W==D1?He;}9DfsC9zXPbstKHf z>Tm-u^)2`x`Bq0JhX17g^!}g5%4zDz{rnDd^k}+&#@11Ih|*;O!Xq8>j=`XeO5Ri}U+NK% zkb;r1aqBRNtC=itlt^OXIab{&NQ2;r$G93W42m#{?9cHLO)ly3jdfEoM`TvU8mgZx z^d(=SZCy$7B5*wZg5;HUY8hoTcp5ZA;Wt{V+6X^d!yEvTnpw?`XW!uB`t94VTQzCI zA9_^P^GXNpf*BjGucXVoCO3`}Zv;I#Nas6 ziwD~~$kiFg(hD4TY+~XR#PB1bsv={O^wyB)GHn6(4H&ZwA+C%*5fbGCQbb{4l_$9! znRU7^Rt22A$K(6$%*q|5A9*is1+wjxE^3mYuTrb_!NDV3WK@=LR%z&=YVE0p*yScvRhw@%VQhu3f4x1%0q zkpJ@hsgYT2qF}3UNEuRQf6{?1-We%w3MaPQ-uIcB)^i89b&`G4ELAgNZ&k-Ku2kGf zrqR@Ex|!nDWaj0OD#6sNYgm(-JsG!S*WJy;3furQc!YzTLuIZNOv3Bp~hDU>=P&3kJ7i zPGg{I#lq~nIWleI&7?rj>zyk2WeKJ_+o|LOI3(>nWs4AqavTIL?al*`$)p+5oC1tw zd%xHU2(B%-$V2g{B_qBSF_!7V16|LhEkQucRiHSxGs*-wB&Z@=2&r@~Hn`xj! z&y^%4VWm4o%-(wui<(;O=~|t!;$;)iz$d%AIJGL|+q#AJ zbRCn9Bue&Jv+_N9%-@TvERB$Z{OKf?O{rC2nmkZT^-?SBxk8JDKnaI`W|0l}*I4oE z97Xl`-pnsCNR_{X!kAc7!^hGgf@xEA+;Fa7k*nh+8tonapWB$&Np#$^U|gY|OxXb8 z5kgXXqL8eX?4#?rlm(GM;U--2fp0BH_rF-~CUaC%m3$Q8T0AeAR9=3**%#(c=+%G` z(7wZo4kt=)<{tD$`sp2m)ue<4Mx4Z$V%o)k;hf-t>@mb|&SQte4UNSnb9TSP0ESw* zyFvgGrkkIq(H4Wsu2>FlVIA0}VUHhk_Rk#XGk;|t*MGdi#KUtjm#a_{G1#p*bdQVg zH^0B%xtsXw4fY`zxD~#7hW6p5`?E4qfvT{}-`t&anq^m)ph>bbn)0HbT_cvvxf7lO z(gzH|fj5^i0lIAd;CKpWT~R{jUD`R}da^R&jm(6%Q?((y{&zw)SrpFe&9}pPb7G&& z27^`j0=}bE&dN!B390u|8?fCF7D7}k)I0zKn&^;Ley>c|w@&{m&RpJ}9XI}mIcl8W z%fQ#auyqwYHN0&(JFAvK6YPuv${aw^69C<|jj<6>aJfOmZ)2FZ%E^1a@<$#yv$Fuk zIgbPU)Z=d0E|GGtINqPua^G)8VYOc7UZxv5wz-9egXu9vrWVPx9442`9dI)bzPv^K z@V@RNIlb^DdM47hP$NT$&&6Xe8lVW6+}Jh;jf||;zov^nKPe7`kvJAlSW*`Si^=PS_Sg3s zC_HhQhA?z)O48pA)9T(CfEKwu{~O;juhxRMnckx6hR{ThEDTbZ66BSfGqJoyqw#tn zlF%aiOPPTf>NlDJl6$HP*bA?4YE0+>?n-L|KUmd4R;q&OW%wt4LW4hfk_@w%SN{aqWD zYHr%Q!47vbp&#_UwFK!mV2>cgP4em2#Scg|H|!!_oXOBKpQ`?duM)Lby1Au@9KY@9 zh2;O8RoWZYh=v@bACwgG;$O&d5^9_|sx?aws#|8yX%@JLTQPCPXTO^;7tS~c$Q(0! znV2pSi+7eA@MhQi2b+571$rWj_K2%~cE88q=Z;P~yO4C7dCC8@CVcG13Kpn-150yy z&9K*0F0V9soE1q^;ffp0IxP`QF&MM|GEyPZZ7gje!NW@2$hc8$y`)`*%PfU zCz_??DJ+)VUa36t!g6kenIgZz%_;Zd^&Jm1Z^T%PCj)3LC14&%;**_`-#Pq#_-^n# z7v5b%W>TNSaD2V|x|-v&`~qxz_^x1#xo9a>*`JSEcQM%h;-*LQ^s)gf$957lE8krp zo^0uUcUe93^vk&epDD(<9^`&P*Bx*%KxY}0YMq<*BM>gIr}DrI^0yI7ZdvEnE=I+}+ zngR0_Do?8GBnY2?i%k-Z){x4xyIs63+(^Bs@=Fv?>LVeS1gi!WFrvI(u`qLIK5_%Y z5mu0IwXy|mg+)ZRH?s95#C7<S72z6?i!FBB&1t9rJ12qI+QNyW%(X9Xy^}~Q^$N;u3h97;6n3Ne|KsKqqXakWm%3$`$_lv_ zatzD9FiAb%+?Irp;Jexr0EQ&bO+PCA zVtBkeI_b99apD|17r{9f!RyPg2&sxGe}-k;v?bsuQ{^~7OM=O`Tu2TvV#5Ab_0k%O zO`g%bU7Yicj>^)MxM^|bUD@WJG2V_a6o@=-5_ej)8JCuasuB5zf+Y{V>g+ZJT|VohL*^D|_XbdK40YyfI=`v-9{gvn2d)s-7ogx=*(? zI~k2eO<|-=obI&+6$gkN%6yT|)mr@#q8omXsMgHNF@6N>2OjQ;Qe!EBj^(OSuzz?Q$Ep z*BJP`rh`-*m*9(}5S!cYaLp7&=;K)l*5RBji6VWVZ2)l{PH(@h)#CdUrKG^=;)2KV zhz3GI3`I$6qvwok^Kt z|BRI3qcnxi0^p9snj&^tnW{^BEl%8CnvoclOR`1~V-rYJh2AJVl+|-kGg9V zKN|vD&O-YdBkKewq9#)+#coX+QApI7IjJlZ8VRwt9zURO7BGqE6-OO(Hn3SeDGid) zJ8`3bQqV7rP=197R#Ahc2mtA+O8=uJh$c#1Q0?rdpEN#TK4YHKwiC zH%N{j$^(_2DM0DAzy0&D9VxBrh>yG1`#S4Kj{lzJq1BxKuEB+dyik-okZ;rQWj53j z&MWH26wSoU85Zrvp z7zuw%Aj^xoPgay6=9IlJ#x1D(jUsWJ4_Fz=;+L|`wi9q2q?k-C1+SI^v!%5O%1Mz+ zVBfHJ#ANlzmrKWhnT!;;!weLxx$H4>tp|iO+n9JQ{$N8K1Uy|7AvRMXUkM~HEZXo2 zjmjjuNroLi^SLk@ny!n*6gVmgM<(sI3!^I7S9X39-zPfXQ+z!cp6fWMi51G)rCpWg zlSt}X!Pthuy8R>s{}_Kp6f`kyP23&lvv06PIYRgB^rY% zi=T+wwdLx&_wE#n_rkj@O(1Fja6QMf?n%_B*g4#)Q-($hv(}iEFqsAI`FL|V=wtq*e8`EKLXrOPtykRzW0J`He~y z!Z$R6wG7?s^NO$4sU(_`Eyg9{mG3XdtMQ+7bh!9C*SuKxL^?^LW*xIK^dEGts#d$d zaEJ9q#XP&bIpsCsj0I1(lY~0{Ja1J7j4AcWtd!;|CtFJJ6U0I}K2v=&n)jwFa`+h_ z)9bi*{-PK8ML1FLj%6`f!cZfup0wDv$%=_?D76 z?{~y6MnBjAT)AH!cU+%*cqzZJS(i2YS)H)b)@V}?3-Z9NbcnC{vL;-I!{3qSkb!6l zH<^r$aegmswm~kCCU#_u(|=m=tQbTzp&%6PDfNPklc*A@RsyOd^}6)}{VFB()@?@F zXil`;W$_L@oXcUcNIujC{Z@JgUZFkp^`TBlPm!U5;@#UDsd?0HY^b zoAA|*nmngwGY&RgOq}8lC=v_rv)vy=Qf0pWz&YjH8Ho7R3sb6Hv5+?fTBuA$F%Mc1 zL_j^D&!L?6tD3czB!%4tpcVQnq%dOC3Kp)3^OYw?F&qJpj!ty;wUHjk7gi2|-P@GC z;4@sZ(2M$3`lnA{I!>N(y1k3grf|t}KP%FzB2X2*QUdT|W(k3!$;iPShZW zVi@}Iz37DoRexm!^Hq0e2RjeWolIkKhL%RqN!k9^An&e+7ZO*(T1jvu*$zci?^DUt zhvk?hjyv$&QYYZ>M7yqWWAD*{1c)XkWklx3rMQm`4hyUcoC!LAx)V*z(TiP$7@!b* z{4b@((DufAYgpmGVVi@A|7elSjQ?m68A{4lN8u3XJmeAc2N&fs>FJ=84#%4Ppv7sj zr}=kyfULQ5d4q+^Np53-u@<@QZCh!W*gH1Nnx!75*yLXbCbe^A!2%H~;n~wARmx{6 zpYEN!b~(jl(1a$#G7NaEB0gfH|LhAleS=129#}4oT--QOE{hJ4KAE6mEI{r18P)R~ zOTdP7Y12LHn3-o^)lW+yQEy=~i+$$No5mfchghn*)nZh@c*FFD3{O2Tz zjYLA$bFV1YsxlRFUAz81z!t1jgE}tpvT!0gAIxE*vZ~66PNq@IWb9|dsdo;tL1xfLEVg2UnFkqq8ovxNx(*Fmb_5bm0elBkbWh z(;O614*>7L9Yp5Y%v?uZjRBcDB$>7%*~zajS02G5c$pvAtVDTep%LA&_DOR2Tby7K zGB@t$9Y$!+`{{Z&izkcC$nI(&vcA%$p4;f@HGJ#b5*t4U&6RRHk1|qaY!mJ#6 z78=0}L=+mI0_>&nb4T{K?e^DV8&n#yGo!jB4=*8Wr_7v9+ksj1$*`q$vSHP58dCZW zCPUFXOym!Tzud~aGfcVW_ILX|Aeyz&LRw@sos+pG2iYG+g!2p9BqC7ogUso zU42hCz<|hiw8#!q=t@C))8DqHg|mi!oMi)1^B&yKkf~#x8lBTDETS(IUmYLUlt|>O zhd`qZd516uTHEw1v7h<(DNjt9j7-BeGEcD~3YgVwemV8Pqr3y|JGj zP1k>xXDz6P_L9S@-f_IDva|4vfT13@WU@`vO{qrtjxheOsaC0T7?C({sKoQ+rSyQC z9R~%dHl9_=)Nj8*8Z2qs%_GV7MHqohy`R>*5;?OWGuDVUK@XEo)newzN9USrNk2Xv z_U}5asylGI)!SMEj*)1*0qk?q`cx#tA0j#vYCVvBw`G{oe1j>&Uy`e0T-iT~!4;O; zXyihAD&LVTl#8X0?Whmyh0Rtqvmo(S+MPBh_jX+(PVl~Ef zHw}lWd%ZTstaNs>l4=R=7#sAHjKmGh9<{R(7Xw^yWneJwY^H4L$)V#+ z`Ik$BupDF63&GOm(p43C$flaKh$Q`5*>MmYq2GjK1!L21^(6(l%GV#r)*HS3-bXnM z$D#0jS5324=n{CYI92*1AJkNYxbm2LR~>``x6hdUCrl%at{U!9Yxa)!!{`!W1{SJ9 zR!pV+HM^Y3379dZY4ZKNi6ewb_tkX)N>k-qUHej;&u_$)b)&*yXP@8h@1r{Al5IXz z3nj1F8OVIWU@4z z7=q7v8LxU^HXMdte2T4n)3B);A`rwShWvL9lU}}TOl%_KZWf(hWdb7;gjsGwgekGc zLXI}6ikf|sTxk%8BGIFmN%6p+;gp*b&z!~zCL3B!#BH5Z8$#9!S1^!H6_Sl4cW4dM z+a1;;M97!+lKfGDFd2bU*bM9FMr`Zww_wWpOeT`Jo8|@XuzXh*FND~lswKKO&I@Ic zUNdB<41e*U;Blu^fS+X-XxgO%#4wsH|(dtH=xMmViqoW6d2|WQ-w82U|1= zn71LT8SW}h+lCMhnz`>|wK&rotBdDn$JA+_mLf$cA_p||(v^jnJXXksFpfB-VfF*@ zSYf%{Dz9IkjnV=(Rkz`1?9Ma)^w)E}+w|a)!;a)=A-zHK$`G8zD%Vve!-|^YdyK{k zI|d3+omVmp+Sw~0)Z;kIZ9C5akM(RBWD^wRwfmbbSzi=1qF&C)osIkN5Bm`U=krUc z+x@#HA?<^TVudW$%@BCt4iK#1h->Lb*eQ?_Ms~p2&uF9WadQzoX*vUpJm-oG*HR8{DBm^flY{jY_R zfc1YHC<3z8nOVPd-#)2e5U%W{ZPx+sG-Gm;XTk!6W_tEQL zN2qVKs~yPrYJrBa_9f3)a41EnPocxI)|^yUx{-MOSmF#`y(lAse7$Cwqv7_UDj;D( zzGCu*j*cSe9EZ}vyGcS3T2!5?rN3hV3UCM$=bwPFE4!mcubINuIcz32l@hbdCeDe~ z!K7&>fc@-b_<#&O`^)?_Kji4APIU)_m9En~>iXSYHgAM;ZMRQ3R6$fHQ-8O=ZCxtf zm?p7SD9QoB#s^XPLX3j=2uGfLOM>-aC+?5(^-G0M&!_6<8|2*#b)It|K<@Gi*BXJq z^S%v3VByNwRl}h&3SX%8TZnTxclAC_UuzVL<-g5$(ia_;(^h}0w1~tT+6XhqTj5yP zV01k(Q_9@Ab_eJVpSs-Y=ruS3(>g0V7OkrpO_T(i zU?CYwhAdd!eN$N>$jonfG)IjUz1@daP2QbZ+c4FR<&E+^vojDxKErR~4@_pq|w(yl>%G z+9R@atY@z@Du((AFF}yZ%2)t^0jZjCDhN6 z#5^PLxCM|eLm$xpH`7J&^7g9i_&-ABiY@SK)%(YdXhgfs)S;KzlFL8K>;Dp6wHu3m zNNvn33#8{-3`2fLzM&4)o=4V3xLh5w2GWfpA24mKO0S03(rcFtqsj%osMOM>UzkG` zHrFl_4Oj0f73kh#qSDszCNNn;^K=WoBr9#xL-^)%I zbZ$v*kACeLb+))*$c5RDd8c;gr?9HGY)HE8XMH=W`Z*lP&~LG$_W>t6nLksMnMkVA zs8l)+#+zh2ag=ZYBQN)@zBw}%5$!s}q_bvQCTHf*@_jF}u=#1}w15+R6J7DTH|A{$ z0F%%y!6Ys|dSVzTR-9Z=IBe|uK>!u}LbIVjvH1BJIg~c}PV@8Yx6cHocuIk2^JeX|{Kl(>qD zG5`6tU%b#fNA1jdQ+2ItH_S2O_xJ$K*Iry3x=ZSeXCqdzak_zu)E`5t4w_|Gq2*x; zXD{A@XFiq~4G1rZ8`b)|;c*#DR$#q}GhqoBJN3~CW)V-nZxkU$h+lxfj`lM@ZE5l( zzCGOBEIox@` zCo$3Qwl>!&JN~9v*1-37kA|Im4oDC% za1VZxX*oNlsuik%XQ0Vs>yjfs!g{4w4UO|HjK6vtR&je%AN>Stnm)CmR~b5D!Eu{K z)0qs$^RXs%^S>VMFVE~n$C5RhWImsQZ{?HAw1o$KKiK&d;?%h9Csci3+VRh03fZuP zZ?DGFCEIyvG29t^8D+dtK1WdDL2i1a0&#pXBNb)83#oQlQ}ff+3zYhJRAH1=uk;DLzgZ`g$~f)7hov1_k<2gs6Z3nU7lQJg z6cs#URJ7N1_34~J(2H(#o=_wfYvpEqBiP{1@jm#Q&}IcV@)*+$V2%B!l2QZ+5w*t! zV5V}sV2qL;&9Z3fhOw0EM<)1&P63g{L(KzmA#SH`+2~FlDkpp($nh7NIj5M4h&MLB zxNO;o_&6IFzq6w*V6LwQSYEa1brrsNuF?z&q#h7V)h45N^;;C}-9tBjn>CC0S!^~p z5`H=9J7|n6Bk`tJ>i9$Q+0ta{4(K3Wo>6HUlB!k%`Wh|QGlk+2Um)}5*;n~4T0`~D zw|+52nA;rRv|k_P`r)#j0SoFapA&JuAMC4SV{^x4JEvSkeojZ6C^H#1?YSym^D7=u zSaX)^-8$3SFd2`_HHZ1ZUT?O!OR>#bCsv(7odpLarTkNO)7ulPPUvEty|P?tg7G#T zJAo#v%n!q7b2PX2&esmT5uCe*AAbIAou$`s(RcH^cfz9e0hFp#>)7`=M1+YQW#pBf z397_o2%uQ8t#B$8(njG#_H`&5MO|*dsBXA_iDN~B!f7_?pg08Ok2nOBg!HaMSI>^* zaF65*PeKA{>_jXnFVAEJl99i}I$U&dV{O~Zj`ea!ixeeWC*LqZR}{2l%`DFM-W}a> z{?-2?|G2M8{9sNf>qo9lH|Fq&dRs!l4i4@<*!0`6wDvPHT&?%~f>>2fAZieaBN{|5 zBZIhi+r*KAdY3jWR%thD3xEWunJ^vmzRO5klt&ztK&{E*tYJR(3D=T&s&ARwZfR|> z!#_7wlca!hhtKBC>}^^YE_$U7_vVky7)j@=R!q5+Awd?X;l%Rx21HXhA6C4$%$F;J zN1#J7kPW`idKdU?Df&rqM3tAUlT29Su;mfEKRKLSm%xdk&D;w8Er-!Hei2m8qG_AuNT-_W{GZ z)GwRfl<0BzSJ#eC|B?$yTXrY$Y5H<8&RX9j!4}iQ)}0*?)6+SN(0gE&1so~V8g*Y! z0g^cD3nOp-OVqtrKJwZ?xneW42O$327}GO>Mfugy&e{1BBQFu4`XMFmqF~?2@X13U zLXH3OcsKZ|L71nT@Nc9tgnCc4Oz$Pu0?`^4%|M2kVJ{wyWtc(_RJ%4q3}|3uNTY+E zHB*}xJ%tAq>O@Ae7VwSv9e(^O2na`=xQ|%&kDjt>+xJqeU5XP4TPMd0-6PykFbsOP z-6ceeI$gOh@$|0MZyFw=?O+} zg`O}Ga$z%$i0BT1{ivgfJR*6hA{;_wX4$|P?w1xQmqog#a~8~GQ=qTh2iW@ThB9e5 zZwqmIm-<~ZWtvCS>R~^0dtbE5aK7iKScKy*!V#`25or6;-&CUrtFzMve@!Rpia%0mfL3UpCntIIy zX1o?r9DSKU&q^Oqo#u-hH||%Q(;7JXOPui|dW8_mtWlSy)j3NcWT=!L0ofki5?t1a5;54g&LtG588CvbUhKD@5ll1Ghr2CxzV>(?tJAo>dW&iT>lj6Kk?$ zKf0sXm%>%P0!gw)toO3gJA_1T3;`p?KU^K5)+JVP$Hi?$)mAL1VKrP1Ngo0w&`~2Y zKfk)JM88~7s1GaIK+v}R7N|1CAhj%)4(D)jx!hNK&28w(ehI z8CWrAhyKIaGy%=@W0R4K+k*=Cw`!*KontZ#=BG+xvz4g_NG&QjUKBGZh(ZTOMgF+@lTtkJX z?8bOtE=(1+>o%s3=TktAw85G3BFV14rvix$dB*R0$d^Le{{Kojy{a^GWPbZU;uS9x z8;Fz$*dg|Yl&H%<-UFT~{QD`PdzTbO*PqCe)`lGd?(T)ZYGWkSV4hMmZ+u&k55A$K zAl_bXP7*~xD=6GKR8p9glu`yzPu?{vkoi%}p<5X5ZK_kDMs8JL8nIaXX~TlS`meVp zf%~YD*tA6WIJ1{Q?o>}9D_%pu6|1q!^;V}6BV;rzhW1yYkN+WfFIQ@aEG9zxH||&p z@ZcnOJNfH@3YS&{dg@b-B~b53Oa^xq-(@~%tT0IyN2lT@^kM)#u`4BTmImUH{Y~sM zyq~0+HnM*cz8~`2#Rus;nh_XI>Y+fi5CwH6`I2$Sn9zTi>ed`O1IkzQARfB zl_WyI(Y>=j66YU*>kBNY8+M6R-{z{;vg5Q{RKwcZU`Mtsu3;%mCcng*?0zl_6=6)I znTky(Gx7!^H%@Lf{Vi)+ci4_hoLNDhP8QViF{N7Jd%XVWJIs}{h=3JtQQSpiQR3TY z2Mbg3MKoR*Mspp?j$U#NBecwKd$*-A*YF4yG%T^@>SYhzNa;^>mzErTj@cwE4wt7$ z@pQ-2?+wxWcU8>wm^pRdeHMT#rK~r~q8}V(B4t}tew&SW@*KM!kSENM^tH)IoZ;+n4hd7@@|zV95RLXMDxP%I`AiO1A;zKIFqi{it`b>1C7e|8!Thcz3+h-2oXzBgD;8>0Uox z^_Jx7p|E8mjkr7w5w+2Yr>PA)sx6&V5y9HKc0&L>>QeJB^#Pdr`P(N-Ay-Zs1R;1m z`gde&fDk#S!TlIOb{(pHN=*=FiKeN6RiVoGW`{F5kYX2vrH_}US?3qwl*nOQByl+4 zaI>`$@ZH>iVt^@TK6%5|hr?iv_05I{dL2Fz;a1uhd~eqML7Gdhi*d2rl6O1-K`xbJGsSL|>z(MAw&c5~iW_FK!m9yUJ@s*#L9| zB;hmZzXXh|AIEq9H;c>Y2s{!*>>(AdX#5(oX3 z4qLwLkM2fZc2>-i4tB9LO%8gLNVw5h1ejp{*j1re=>=#|PZf>7H!48&N>@*=1d&$o zpw6**j$xA+=)IhMnTwDwk)bd}BNgZg{nm{7=|Fl;keZ$mN=BB{`}bo?r9d@Ew^I7$ zZ1QlNoZu{38)AH_z-bpv@av-yPZn;}tv~PJ2FL~yiT4JZijstnmvN=pvu3T^Uyca9 zGjq;Eo^#fe(MVL&*SCx4cS~xPCb$!wmPGSJH;pZqgNk~p5nGQS&5i_s2#0N)D9y*} zd2O{$);Uw$)`l!R6PSM`Wx@fbNY0Hv(Nj?D&d!Wk&5cRu6%6}<2BOEqzqqzrSWkzu zp_Na_)$6SPT??G8^qw}Vm=jFxJ4ow$z4q3J6)CY7$VPBxMgAACXuq@OjZ3u98C;g+ z)O8uoP<@o<)-unyf0fX=vZ}G_weAztWy^@_qcG(CgslcWH3$SH)otk?RAQWm9zY!vRnf%^W5+?H^bw=BS zFbgno)d^{V9!Y7&+rCIK7-33-Ptp5Qjy!SV5!k6LZxJqU%cd8-ku%f`Jk%^} zMBc`vR&Mv-Pk9ehs-JIteS1D(6Nrp{pw%Ulpc&I;V|L#JXt!(CP#0>EIbm^Jq7pr7 zHHDd=-xB-O0pLIz{~S<%wT1AhrnV|?_T9XhrMkd=>QV&PN!^aCZ5H!CDk0>guhqAk zh>1>5HL*Hd7ErG_0-;`nXR25NjL$e_zK`@;=APz#y-lp)`w`v&Yt9e0a&;-P66_&D z-&8F!iKc_MCOd;8iLRCYeDnOVoCkQsY?O9VUC+U@t@-bm}%qD)~bwf*XaV8p#BQX0BWm0(((!kYx zo}*^@MsE|v92rEZVV3}|Qw>HdGPYhk8d%nQA_L>~a`zTK{z3`4w&l`{OQge4jT`nI z`^VvmH0@~9Cg}U~SG6*aKH4G<#UA!D@UStJkIV^n`}TaVmFf!}Gy0eq2W%d>9fGX$_1@o86jtgq9Lxv><#^XE^>16EArhCm zR*i|PJTa<;$ z;u96xV<3ccN&1)ad@lJ_O##-cHP3XHe}xGV*UfJ+ff_R znypD1+*Y3!ZRg67MOtpB{Oc1O4|RW#I=*Upagp{M5{j# zQdcWhXNthn3XtIW)7!Ebc;&q}&bV>N^V*N*5@eW3t>NgYCg$z-D+}Lzxc)UKmhw-C zJc8!m$+6~cTIan|nt`TA$au(IykeW<146$}RjUpZDB7W$;qulQ$i9GAAk53~g=z~O zPgbuz+@70&J~$1N-$06#^}At4PlbeB`_QoBv-^$q@!%>@JWl=G{NK|^*z4i&*JsA& zzGtr;{}L4s2|I0~yzouCaSgZ6b;yVg|N16Fw%EBYJA$k5pGm&Xr@0_68!=AFHPDg-I@3{)~7y0Tc(}mt0KgGY@Q&=~-pby)a8q`h9Vh4G{CDV^BeAYv0=9CZJ0=sB2$Fcl;#J$uYS)2- zw8Sv7O-I-v*)LT_y;RUH5mzwLG;E^gv08Oq?^nh21Xp_;{uMItfgD}e`5o(T#JoKA z`cr(Z7Tb^f#>!W@ggfN33|j2Ur#2*hf@MdoSq=%;mU})bP3aGd5ZzX~C8Wp9wGgg_ z%WeP&V|6)%PBho67!%|xjCK84H6vF_`yDRX%pftIl2m9ygaga79e%0ym5WlV;vH|oa+)B1LQeUp8vdGZf3+n_KE zZbH}x`TA*Bw5oEiKf9;p@d2*VN$~xJIH-btFzztOgX2XdB?v`0ZVAJM62Ef8XlS4~ z-re4kQhE~Fuq=aE)imz#j5h^(T2PK*czpeqkqN_K4ygEDzr*+WyY%1vAFvJ1^A?~{ z-Ct@b1Tm}4e(1tzR5xnMsTdkhQq_oXp45rbh6}nWEl=$C?qI^GzaH!dtExABH0vTI z>NvMgKH!<-YW(nU|9f6=C|0m@89rIdv`t?YtW{ke_GKJeEd(xcr5FVyl!&jC)UgY8(X9)Ye)Sw#rie) zUN|_Jh}UXPCCGoU{m;RW!`-*wt3aRge>e9ZiK{UB{`~$VrPkebV|9|gmKtiVwEc}& zZ2uERd(`&|EUFML=Bum3?nd0t3%v$&dgb3T{3kZADmv1Di#q7MMF5a zmEpWE(4B!ONC^bW8g+t7ULM}HBSRT!2&H(lFi3VV0&+3GE`Zxgc#3$@CJ-%zI-5?k$} zTNPo7_W<0IUf2)1pGRO@vQ3=ulcC$n0Bi4t{u$ug(l=JF`I-tVg!(;e|RxV$$+}cjZYpK{h&lhU?-S zd^a|MGp_VZfGnFtc(DnxOJN+lkW-AD$d^&2Q9gC!{>PEcJ88YO&+Naql_NgF`drI7 z1W8%G`xG(#N=PD2*lMDS-tDh6&gdaIFTXPV?hu8eR}dxVne}A;sU%g}OCjhEF*(YE zsMri|DQKCH+aJo)8&4_%inY|8<_mn4*t+!Ur&|wpbV{pOa6>q@ForV8j>4lrD^dTeBSRmH`&8V`*55JPf z`m4JBSMfhah-S~{NHG0m{ohpmgh|aM6!O$u&G5f_hhl@6%rJK(0cp8uMeC%Mb`=zL z(foQ4*|f4jl2yoW1>Qp}Bs7!-GywY@3mhW@do$IFEpJ0tIrbfk=9|Wlx*rKZ@gPm< zB_18`Y{ptmzaiN&@Kfk%TceAtp;yOvmgpdrc-Rdu-G>4dK# zrJ;pQ^+_py)bv}SfpSoH07mHmH!C~X~yVnYNurZ9_z=JcXM#fv)m?c$_8mm}?@M?MUTxBtN zOGs(ayAw&PFjW4)DtW)~rPm`NMv=4uP^v|OHe}QV%ZM63i%3-qowCCCm2({Yl{w8s zn^OFJUigk+oJ79+Y%01_2#^DBSb5+R&PQxsf@ia(}L zPW)4{$h8XUGMD>1UK+6y23*-~Cp}#qc6~W5u}(~{rHW4zfoXD|*wLcxSU%!Iy;tsQ zTIN?P@#{kaZ|GCH@q9*DUVhcs=cGrIBi8!ztNGR6mU(jHU1J+zQOl$ zG;2ipvjy3bomjB^BE-k6)2AtSPz9nVA8!%j4V%7UecN_8q#tV^$<4*}E0I1h0$myo zd1Pefk&myh(PGBnz;&TbRCL{%Y;MaZ<8 z27Iol_0@ zenjl*4PH&9 zk<4;p=~HSnFumC@;U&44u_TL3%=hk%<1MV`|N0O|!HYnNe1@xpKz^P%1ZfsP zlryP{>7ffbRo+J4n#`)5o4*MbwK|(}+Z5FXV!*V6-_skh+c`wHpA#>tVt|=1fI2Fx z&_{N?({xOP9U8uHWckg=H3@0k*}JT*%J(ju1(JXI2^v|t+W~~pSzK{ z$@l9ZUN~OgWQEDAEptBoHKKt+O_AbMP?=txkYY#S(cIPVxe52KygP$Wh_nEyxw|yj z_LW>Uj>CB;;mIerKcb%|Zij_=xX-!&#SFGfEQt)DCm~)6Y>=Ylot0$bgcdCuWW)st zeq1i7>Im#zWIGG@A#^ch5WYt5VPF{QaGehs~OJL)q?c)!Q$SVa%*)(h85V`_^mMn`o!i zi%Hgka-}BJMF;+uzdV>ZL9|UpoXKRK>h?O0x^mrcPQwaIP2H80O8KI~6GzM)?0%4< z*uI#aBGN!3x}i~IM>?!gG{npj^6NO%07kT#;=GTul~^27DP!o8ZO`_MelPBYOJuxP z@%z-wFON_{_R&i(w=fw6M!uP#xFixs%i2b zD_^4FB5l#n4uJlhZZUed>OW`3h?*C06W2%KpU&^|KLZZCbbf;F z-&G6`u3jw-?VS^9jo3coJ6iuqV@8WjY5WiGVnKs^_nPq8k)87*)mSk*Ry*+RJ5LRt zR#HB#oi5hYO5}#3Z-(XCQVAguWjl|StTUYQoKqyVUy=5oTD~rt7m8-$5b1&l&`3u6 z1P9`GlA2VxI(cS+#yK=E^2?)iE}GkyeVJpk>c;75PN+eM>X7;G8g$g%v3UPpylNwu z#mwU7XE(*8BtFxC3N334_-i;4bcH8B4Cqvcp$*Ws+sVup|BaEaTS|6a*IH)Y#SSVt zichP6ErieV5mpTGhV`N^LJf)+6Gi)Z;7A{)+POR)b+$?XXuiXV*Azje*PkNsO;g}| z7r>jeX~CCWroy;MyXD{`!nn!E>{PV?%RLEEn5*B)WL2o&a<=?za~cI(pN>!ko1fio zhE{l$F=7N2WApQ4xrt=yJLJfsIf8s8$`dSF2Jp%871RNIe_rC$RlP3Tv80*tb<0}1 zh8*fQ{%HVhz3u&=4G>k_&V2Avci$eDc5;eU7?OlBNJ+k zxl#e66UPPDdTy;|yvag8$0iC7kmeq5cxW`>pFhrPWCQdXvcZMH*tcgjcV6ZiDVD>T zy!jGpu-et>{J`qNB}+5jAHGm@;&Sf|e3{tM`cvBEHR9#~KIn^JEo zjL541sQ;CW5Ic^s{wB&G2&t!jFu4EL&$ttp1>W0^AGL0~H6sJ3O;{(yogb3T!v6C9 zdw;R?_qL{cuScT(hJ1hl&VM2M8diLsv+#)}Aw3 zHmvxuVeqt1`NL27H_3r8=m2v2806QpuYF9`pfjRHXlTeL_PmKZ>5@16+RVn~1YA%!@8-`>>)w-Ay98^|uc!&~8J4zm`$LN}YhHg^#zc&Gc z*!uj0)k}hroFB_`c3Bk}(JOf84bA%uR(=sGo;-0jVI4S}?7c9>jgetJ{Z^|v7WdNi zwymZ3JBA7q$u6RRZTM)Xt|}&nOq-GErJBZbP|1i&$?k}D$?i3aRB7(j9bKigAB{D!hW29(db{>&=+9uhEkp(($~me~Up zqV0{h_<%X#E+Ln?=9Sc#0iYf;YKy|WXyUAHk-bnn6 zUE9%4uEe6c`G7Mch+PQlN(9ptUo{<29~3YlMUlBOR+qx#zlQ50nPl4QKs#^nYg=zy zZwksk+|M=@wqyv^64gQr&5C)OCzH1{>{KM=E>sFOZn?Rg%_0~!2N%Win&P_uGz3z( zQTTM>iqkfSW9$+Jl5&ikE!OXru*8wyVAL8|)B!1;wc=XT@r+F%+7Sm*>y;C^nQNbO zb3a%6T|BtAl)>xk>p|UfIhTls!1Mb)e>0i-!e`?7f?6W_P083VZDHlQQpPB#lEa`v zr!#rZt+a3Dy1J51ls`NT1WS!M#wunjr;9aOqHgnEmQ-PwnsC~j$)zU9HHw8m=jJX} z$>b4;obRDH1*=k!@h+v+OJVwvZ`zIiMD5-ofl6#grF`$7jt=?unhGk2imrKSB)WoXY2di$}Q3$Y{QVK8OI8seM`UEa#}@nj?f_;nZ#}Tl-}vlzF#x zmF^i+$!cm2yIt+sAN-n4MaA|&4LLF+ytUd!Xq8rS7M~b_E508*QgA(at=dL!_V~RX%yPj zb~q9AQFLloZY~5ebA;t+NhDS=i6U6MBa1L2JI7cesN*Yun@}ylR6X=rjBOVmXPV2^ z@0G-|jPSgHa$|67)ZA>9&TJLmY!$gELj<3MBJ5=Z3F#(0mpvIi`i=6B0%(-rSqt^- zp&SmME1sV!zw?YU>YzT0(^f3?aRh0$Z0c{Zk8^+i%3Z1o#eKl(1+(^&9eTSj0l6p?71Pe17JiikqX1FfZ^tr-_ZtJ6rUviXTo`+mngwy$5xRY z^(3KWk#GGb{~Mi!jzYo@IDIe>M;F8jES&yyBd5?BDl|_?zJ&@^3_iui19d7KATA_)%nx~`EDkf{ z>`JF5zcMAd#5osNEw(LT4|4^A+^Q0i=aLi3ax25&;9XVD*l!4uUq};qByzHOe zH3IJMc+F;&#AFwku6LR-K)MzjZrWH7*VKyy7i!rE+23y60`YmmEt=N+_PD#c%#w7j z_(O)5OaGHE_7;AczpP#lR=+!DI@ww4^lW#Rvv2_jq$s)%*mS+Iw*aHj8u7`&K3A3N zJG8;=i634drr*U8pwOWlf09K5G5nQgu_FFEex+yACUwls!wkcuyv$Kb|L~1W;>JmM z@(Q_A4<~doTSTJhv|n4r1NRpXoNKbPL;C!wbRcg@{f$#`fY2Hu-pIgh=LA9DftR zIR~cLgz8qT1n}#BLCQhFHEa%%WojRN`)~a+6LrtD<1W$0dQHDbjz6CO*E?;vV)=Zi zhmOE_K~dzMiPdYs<7#Y-0u`5R-B8a=Spr@f9J;Q+T;=iP(K|jb8rt_vwR~ zyP1@ZnY<63Q#&-CzhwfQ4%+!^oIX0fm(-3GbuzQ%pin9aRW1?A!NdO`X8x~fT_hlR zeY7Uc{m8W%B~7uK`dKj%yH2S;Y}T?Fjfto{^@5gUG>zqxSgu;GMsjonz# zPipO)+|rd5n*hs={D<=o{P2c)yTxC5jX;p(Dc(Om!b!Y)d2&|Qg_xu2JNX^SfBsH` z$`E64!vTxo@Ha^CtXlnYbQ7 zq2}}NxopP)xowGN^uceQG_DVi4Gf??Nj~-tSl}b%-x5};%)<)za(N3Fv8Xi0rel|~ ztl6>o#?I@<1F9FRorxJZi0Wq-g1(Hh8imG~8lM7QcQVmQSpxpDANP1yE>|Lh zw;Fnq7&faW(lawOrgQ@2ahapVuzR2aKz#PS`AQgzc}!w|_&ud(Us=S4;K-28M@LBq z=IK&Fbt9zQHCsNMEOnM1GYwofA`uQyvichBiMI(;kLEmJa>~nfrIW>dso6}sE4-WG z3V^}^_@o-Ti=Y+>CBsD)2sdvJdI&kR3U&YB0hcvmr5odFU5Ck-0xNHk&G<)7)Yq*k z3K&@7960@@*i8kHZQz^MZa|@=_v6CD`-87l9b3f??Rmq!|7=#9d6x6CZAq!5df!xQ ziWx?C+u?TL$)_N@3BFFhJ!R`pPm;GUy(DN_>`PC(uX2K4%){^98Q>X@UlWEv#vLOm z6>eAs2*Ty9x{#fb7-8s$bjqQinbryPGnE{hFh0!94qiC}jK> zgw-%a(Eo3(`~CF+@Bkdw_2;;)P|}~n@#E}<)I|wa1S{T`hPvyfvAVeTZjYXwGLor{ zFze!Lk4w?oN-GV7Ty665*r9YFAsfKiVb)CMFcsz4eoMp@*Y*lUhemJkZDa(yXvb?y zCb?&2`rl?@FGP|misCTn$SL5kj69cGdxB_is!d*&>qm(i)WUj#X`T~gVtXZdJi57F zIN?R0!OJYQ7feCBH_D(tettVtBDNB8cbOz2o!!{w>Ajm@ zgSKhUCwWPES8pm({={2iwp`RN5!Gt!szKNw4R>5*(!~5`-$q;d$dx@xHq~b6TXjA+ zKBT^-_M_{Q&koP=f`6Mw<=kboWpQgpgu;4qi#?en*8~g4tItBq_R0`K#$|=SM(x|* zD^jI;hvQVA26V=%FSTSsPKUibuO#8Iu(#7zX`H!j^LiGq^oxR#MWe=lAX)12K< zV0^kjWS@ZjoN7WNct_ou_sJ$FjA@)cgR=hFf4%>)FY8?>3y=9MA+8^gBj$@ff=$+3 zo!WuG&c?J6CH(dbC+zu{V?^CXl#0C2)ElcmRAAzt2+tuP&Z-5!eG?B>X*OIW#nDx$ z(#Tv;*9_hnlr*Iq@r|-J!1Lt-GaE85w=m%uVNtFQpewL(Kb^R$v+Nk_v@&R#to+58 zN>rWiTr=lrcI0QEm(v0&{ zGGqe;D5FcB-E&O@4?%;yI zy7a$a_27n(XjkWdvxp#KyO^zrXB`V|oe}E6ZIm>_h~ojPd1wr1wiw9o*k=#$Z+ybd z$53chaaEcBg8LorF|XQM;NWGELmY;i&tyZPUj}h~*kyFvb*{n}DGSub+anMy3y5MJ z&AEDVW;eDAj%qKQpU2h%$hMSeDrtn>mk0dZPiz`-Ta@eowfxGrJo_L3(~OYNh@?pC zzf5Ep(lxu|2CeK8KJl2enH)2}j$~QwZ2uDv6PXYH5EZDXF5Y?oe4TJH`DS!)*O$_a z+kb~gD$1u*IW}om`y(ydBp?Yk6n2)#hD$VrkCg=1v{7<6R*@JKDQN8rgVCN1W%hSb z+srCWy;UEgoOhnwY0{AYI5kN*uNQ#T#p>;Ir(s~7@m8m1;*&mQ)P28ni+IBj3@ZLCR6K^QJNTh9uLmNocyJ zoV5xQIi{;_!WyX=5$~rnKEGa8$*-l5)!fG6Hf%`1s0C6KFGTVDl*y;?_l%ud`r+Rk zo6}0UuQFI+pHAth4P6F$m4D@DeBS96m!#}j>irU`P?eb(ZL(ICcIiHRcO3x12C!HR zgY4_dg`Vl2=a9qq!i6KLL_7sp!21k2iy(xt!kc%UqlAJ&I1NV?>fZD6EeF(r{mhPq zzvxH4z9)8DyrK9064b=$DUFPd3&G|x4=;X*>aIz+03&$tl7hBjwG%O~d|E47@80kH=+d(<8Z zQx6>Qr*f}h5PmjKYDnXL(hFjE!Tlg0ypYBVimhw8AvE)>LKe5W{F@%V?ALoXPEvk` ze)=(Guyzw^IqtfrVG!HnXjeF$5~1@OyY-m&3N!($zhT(yLp6N&n$qp2Hj#Y%ECK7O zrSUvgx=*-0(2>h<^7ryREbTHK{knlGd7csUx@6kARRc^$6u_#L7x8f;MQ+94EMFB? z@sd#H%e88M3NLC*_GrkQf&LCkpF~HSI_2Q=wtRg>2=Ff6k5_xQOm_qzMetX5lx$JN z^9sR11Y%_}$FubBCB`|+aBL0B_cms{m@sWSGMuz%xK55-{URV5QBcpcuXz+Bj@K!w z>*_Q1^5p@O=j$n>@LO7fcJ>6R4s;$0$`~HD%0joFhJI%&n~~oK#XTnsCbTW~ zgU;U?jo*Gb09uL3Pjs)jd*lL-ejmP;Gv(RtGc{ygmS>@7uJSAMgb-w*8sV|WI$Oc~7cdng4P0UOc{}2< z_;bCjCQ`=wOiK50hR_*h4T|qnq2c_c?8#0@k=zJ*U zxh}3?bSniquYI{peJn{y|9Ry*;1CFE8!BCyZd*JR>b}%k`7SeldXX<4SYYY;nhr6r zu(+IzRGgg zQZbyg+QUW3VX4^cdMe%H=7pp6(j_wVcq!Vcy)*02sLAqY#*Qlz#nPYd4ySzHM5oOY z*ZiH=!}C`xt(3J7Q9?>2Z}+8_e*MVucHxkc;V%`_TGx%R#lU#!{@#do7t*rKoV)HK z_WiYM+MsV!V3;OIvHe3TT;NYzC;Kf`zM&hR1%l=GSWIu}IbzDi&}I83D$+22{-p={ zT9DZdGwPxaLYwt}Q7iF`sdxXwtk9@3*xQ#TkyJ#+mI40>SY(k9^GfPJ4fMztcna(r zq4^%4cuX1GtVddSPAP~Yl0x$ALc~Yk22w~Neh>IuA&j+YpQOQX4xMojOXpFH3(JDd zy%T%=^2#QdwOhl?I+(Tk``_gjs~%wGFXQKMMS*fOzu%omOc^IV;4remTLhq&F)(`u zD&l&YKzswATLehbcC&8^%}kTxo&Crd0{N7@dWY=G_LRkOdv+<97pI|kxP`1juXJIK z5mX57*R}_J=n@Dgo9krRg!P}fhPqIYSGkDX`jxT58{~UND5^Z#q#doo=onW6|U zpl*u_1q+Je95sPcz8m``e3izQFdqZV3c)A_G)33WA488ej5ymjc; z%+pL;G?TCV%~(t(%2G_=4qQUMV*+))N`#9eci4reQ*N$)fS)I7+Q@}OTWV=NB@JL5 zujN^?jNWNmDl2c1j+s?fn#S(#t5@CoAXfGLY8GD}M=Vre+0_ex$jZi~O_z^tB<9oP z{j;y%)%tiKUz6bBJ&(#(FfdO4=Va2CVFA0Zeh%waqttKL6xiWh>|DI{Q?Cgbi%Zdb z(pzj)0{JcEjdahPO@(Urb4J)kq?s4o>*iZC8^nZ6IS%k-H4HeHq(C!{WwEp4?c=IU z9?YF{tG|CM4==7{VM>OQ5H-ebok&Y_ah<#vhRwSr)En%xB035Ezt$&7e-KTE9!zbc zA>=VWsH6s7>K}Z$M|Fd8DozLc-n~WVvHglm6Rms{SI+ebOM>gj#nUl!TGLZr0!mB4 zkY1)w49`^NbYzMRsNxuShHuZK9O3=n~rh9$UTSOrlNfxj?KfxK!HFxX8mz54QTVIkMw!?Roswbmx=Gtmh%8I4suel4FnTOW;hlxy}~+Bm2hbb7Mmhq)MgZ(_4My@bnhpf1LN*&2GE->!+Zy zJgLej1O9IkR`uOKdF4r$qE`fRts%jqBsvqP;7F2~WKT-b8BsIdnn_CZ0&(Hk4;rGR zdR)sWT(t6_uI%$axA`Z=X_>FR3hOE(bS2e)0V&-+%o8t}UGM&8pftW-BJuEm9A#b*)nLu| zvl{ZMR%Q$udh53t7U?+P%$Tecc&5IZG*7v{R+DZOZAB>t6p*WENV%mzC5uO?3}yi~ z^!p!kLB<;gexRQla(7m96fL=879PHkuM3Q0#_U)DGHwB}MfG$$aJK=k>QX<-sUA|@ zDBY^y0^A(A5lY=Njj}`sn^W+R;y@KQFFSoUb34Pyl=);%OFKp87iP{aKaSA-Vb|u8 zt0SlE>UN8E56t5pk6icyguN6o)$J#XU4DDRS5!aNx9M)l9;s>z`X-!_DL%WW-@Hs@ zzHDR2PChPpH4OqluG-7v=VY)aei-0)krTaNaWwJe> z@;82r1$MnSF-{Zo^Wv6A(k_mK(Cc1o{ZaShuT9vYBFfm=Mjazj-P8LTU$@dbvttiO zTUgtz%}^C@emUvP%GYMT)^wGm;TL=Xv5~7Z27e~9)YPIw9fpa} zgyP`w3n5wrj+?b7W!I{Nny1S?k6_e76#c19-X8YJ@+^0?ZdlU}y~F&uxy!&3e{x89y8RAj|OOIR)ew^|uS4WWqEZ?u9{bnmYPJ49rKO>es1Qu^ zhapzgU{{0Ml8tAHcmXBG)-+q)ob@9+j2zBRR-KqQHh!28P+B#goaqx&ck|ZRquud> z!HQ#xVpipa*8CO@7>EtNM6R*t+Rxq)&8~~;;Ux?ct;_(0u;E4D^c48c&{}9~=9?>9 z`HvTvoQm}89DQVg;2$c3DG7tCDHZhhJhv$ZpPshVY=|?MxP8k?vyCt|VdJKP=+&Y) zV7`w}{U5G$h#0sBYv!2T+75=nBdkR#AjKUXPU9V`k0bN1cjEqCU^5#R)SIO8j(j{2 zQO#D|O^iWK*Ci47HLl)!U2r7z5e(uQ@Ma5YA-?^C4#$kT*;?Dh+75z?YGB+h&%itQ z%f(uzQ+r9KxB**lm)VqIB(6e&>p8u#14)fU6u)F=s;k80G4DCSBjEu$9+u+(4Q2d% zZLURy-b6Z|JuOv)}#46?-0ygwdNZ?+ACc!)Ia zYhShcLFuDHt2Y?JJUV&zd#prvCsV3~!?Gj(4hsD?+&?fA$up)@W&zy=z~vgcW< z*UPrBGmx3(*QPGeX&D&C&Two)o_~0pHTYE5Rsz4c{OpKgrT3S5Fc@Kc`C>FB=WeN~ zcQ2J@Ug3)ws#?FjZkH%vmA$w&>I8m_n+&abFAsS7q8j9&DGF#U@l*mGWW6AJ%f1Qx zW}RGKfrA<}gOZaRT_6mr$>=8lHWf%%Jv$G@;K3vhq%9GW?1--zDoBLosms93+e6I4 zsO|L0@0bjdlK9Oo2lzNxrY5b3RciESB@P=5IDi0hdnSWvq6xFvmrM{6Sz^1)dtFE^ zh14&W(y!YpM1p3fC^eNVn+Lxgt6bQRsz_ftezekT!M!Tyz^b4mZ4(zV6)kvIt;|^X z1ccBjn_MgHD*6;G&|<|2GJ0K-?#gwhO@mvU{d)9hU!jj6gM84H*3@}|6s<(|A84|a zP8VY&$oaIS@C~#n|h2$4RmEf67>I|NE!4Om&PeZ zP>BLiP{%m7Be;KLm-mUJ>)=jRf5+6~)#L*5k-wXm8WXilNNUAVfUHWFQ^dWiY^TrC z<7&y{BKhL&+y^@saNU_`NL&=HG1+6DI+x6Vr|8irmPta|iuwLB82{cLVz2+zrPxp0 zc%sxv{2L)MSX4yzh@xtN-N@C6rOX<5kIaP%XV66k<8U($CNJoe8|6Fg;OJpM;h#dS zf&*#8Pk#LjNVmWZPA9L9RRQA)h)0sDWVMq0{aM3`oQ4Z92#vK5&P^hu>6wc>825ZV}kvN9(Y3Q|RPWx0%r; z34()%T299&fSphGZEs|{bGoM4fjjHFnjT=jo%Ti-!?uGo$I<>Ht4r9#pDoJl7Z+pF z++Ex8a#E6ey+C6l1McVNVeI^Qe|;XgM<~NlAL8IzGbfHM^-%R@al07Pw_M8{`Axs} zQ%dQ?X7zRHii|z+3x2a40M!qG-a3m7R#@fB?N-qiQsJ}eqlq|++BsAD3Um;>(Hz}tYOx7dXlC5eh8;p`b=bL+MPeMguh6vESmGP63P`s zD#;aOajrh+~ylJ92izTM1c!ieIZb@nm>p=gO}x3S3C_@ciP^^?Lgei zl>j*TnSS{lvTN;l)v$WKXK_A&U6KEjl^ZMHaWH+*RzotZ>CxrJ$d4{g-0|ICukLWbp`euY#ZKe`RsJ zpVbYhecD+umNUgbRgBNotNB28hZbtVxQ{RfQsO=g0j+Y#wD0$84^F@Wn8iu(zyB4( zn9lxl`_E1-hyDk1Qlnb2yaMF)uJcmr%bIO+bZE_c%PiDeD0bu*J7uc z9?pqV+8|4eH0cCMHxKv)%A}F*k1}y-&81W>F$IKT1H*``4#cy6EBI=s%Hj&A0gL9ekr`;Jmzvvtn^yw!=p2Sq&Ang8TqPLN$9ME^0Nm&} zA&)F<5%C{d?e;T|FUfN`sH8A`@w}iZtg1VxwX^qtx}!f7o#to`^GJt z`^TH*q0wJit5?^eO4>vSx5l>_GgYVKjkmNDEm(dd5Vw)Z*$&>vjW-id^2rTPnCo*1 zr%Dz*{rJ_XfB1>MS+Yf-Y|xgo_(M8W#8|}%I(WjcfqsO0U-BQbI)NkM6*tu{-H6Zm zG3zTq_4I*sWFJZ7S%ZNXk`L_FtI@q=4a^=i9I@t6>cwBmF`KK;`%{v`|hRdS6x~M^^$V2RIN0#{e4a2ag`2z z_1Yfmf#&`fm*Id2g7A%F=vD3)+}vpPvj0Hbu)FbXa!_MNV_q5jGcC%%#rJl%zY1BchLP9Gr*cbvsY z6#vd_%r|4ry`cnc5Esfz4`KhkP^;X4D4T;F&(B263*T)^P1!I`Cg<)o`HS2kDM6aK zNdK|%Tes)yVUg4fP7ZF8+!s={Uv#?~i?4=zg6k`<-MH1_^e{+fQ%1NmqKvA%j{s_~ zKf2KH^#^Gu%C1e$wtfD?e~VO{#Q!F3aB;3Wc_o+AG3*`L@ujiDhQtSXT+W&=O2uc5!M|Q_k5HT0bX!FzG7FFjasr}fgj{@au*K3A%1gG?a8evaLg&izX zHSulH(-N(sZ}wI}g!*fIRRP&j&4V3eIfT=b6%j5JI+!-Z7%$$wA0nUJmfeK$MM>&B z6o>C%b1U_)(I67Jk&E!*K0WMfPtcna9^c0MTezFxtuWu!F4|aUg%W5EXc*>EEEpz1 z)hXCRP%HmMr3ml=s{t+_&aIu~Q~cPb)NU`N+DvV)xwA9mCVq={t1<$vW7vUnj8(c? z)5Ejr4GsKu(?HLF=)9GFRwzpPo_Zf>?UD*rZ?$Q#C-}Q6-xPT%x-zYg;OMhh*tIDQ z%`|$4*hLTvswkL)zLL?Ktr&0sA_5=Ms1}xPyJ>G7th#bDMALscNnLkHY>eNRl`G(^ zb5&S;J?R9PsBd@%+!ikkt{z`KT|kt8zD~RO-?`i9s3=KY#MYq4Z6#XBW}aYG#L1nN z1cxiW54}%RJ{Undj4K)${tDafM@`X-j{Nf(`xV(?+*k_cnuB3&19+d^YsykpsQw)7 zmLsiH#)7DB#Yu(r^(4vjYWG$0$6wsioEmOxI=;V7PI1l;dH*=;o1uD+v#e0#L)@w63Bvs1ikPfR@$y@)f*G#K*W@p}sAuzwS_ygk&u2${0 zNPbY&=uhTdd#X|9j(T`6bgaoeuAOLwuZ(M$kr8rp!d3d31kPkI{EJ<>Q5{aaVrLXX z<{YLv-k{d01g5GQ?%`-&w%6-eym6Tj2NFPpo%1 z^8p>E-bg78N<=~YgWS?o1<>8N{MAR(%3z(qha8zEj=wW>1w_LXb<0Q=f8~Fqj&~^CclNv&G9yhQBE&Wl-SiK@_PjW5R(RUfhS8q0z<1EHtXC%)I#v z&?4D9HmoH>-c8{j$sdJfTn80CT!{C^-ZgZlxmPUWEchECbdJttwZBjw^6h*F*1{Of z=Y7ncL}gDN*Tj+&dbjQlU)x{fm-Ez54FOSN9$LN=Azz(W&!}RB*{}1yQb}I`31nro zH4(8LnRw`EY?T?+8!_1QDIAHI=6`ozDYb|94pTbMILpy0wOIw9S3hgNPX71WD$SY=T( z8b;xKT&8ALr`>6>=(i$=3k{O-slZE=IbTp_Bc!>*MDtgcKJFpDwc+6$U z{fQ}3$V>e0!LNgk)-av32g-qCDoCd7vV$HJB4I_l}(B?gt$Tgxmz>-GN@vI^|Q}Qs}7{_RV@g;U&PZ!fC#@_c~xc zC#{7KNspw7f#STvO`(A8%5b&9C}I7%^*yoCID02C?%!Ov(?RFS!BM}w$NU=}t@gOu z*oANTVC@{|^**_;ht8ANg!Yx_bwlQR2B~oNvc0&v`k{Qal{vB@h^ds`&isG;NREAc zj`v%1{#=?)Kd5@(QB1;R$H&?sU=*Wo!1$jug(}⩔AKZtHIqj5~3Z^JR8-seM_XE z95ffB$Sw|FZQ?RYnBjv+ ztK>7cp3%SK^>ma)d52PLA7M>0&!*2S3mu37a#Y+219YpH%o`Le&3$MV$?xQBWy*2! zWm{ohx;2@&#J#Pu1e9|4C0Ziz%_$S}5Q8E?jdpYW!D)f5`Pb?d68V_vd%H>Wzr%UU zjbilJXP}47CJikUm_+E@CNYe2CG>c&t#4<@s+ZDI;L5$^LDh^ns)Lc5;cnL4zVj%R zE`cfvz>3#4#c1z+L!SXX!aG)R45{PKK7ttvKE4ZXJRm3IK8RP}`NsWAY}j#cd0BZe z-K`jQ@gFlnLQn!>#ib`i+E!1QW5ey&Wm>R#4 zuJ9pzg}EaOref>(7XINWTa!`6SKfH;4}4O6-+B);9AZhVqoOJiY^zuK0yd~@$Xv@r zJI6K4b`LD1>6MA_6AThzuvrf0P0Ta)L@C*umQ|XapQE_~oodQ^+#Fh!>tWE1>Q+Ps zPkQ-(4h^V9M*O361;IeXvis6 z5s*nNB-5AV} z&&&j;wK{Rh(868ohNl$yz_w7x3fudu)E>)}s`aWj2BSI|g2atYwEtAwwx$}i%6cTO zqeeaamppt|ld6iLWljEc{_$j0Y z&(I+I3^poPzdGqekZwS{iml)HugLO)EE6#ewbzABs*a>MYSTXHyI(JGxyuIHC3k}! zXLcH$!G#AC3$!&v0^+ZXea|NLZ+Iaauy6n}>b)bjOa7hNumqj)tqL1=yOem8UhT;| zf(5nO-tzVLojF!nRpRHlgJOgutg#ZO2;axgQhfpD@eRe~jRE{-{smNawOiF_0Mr&>;`Nxap}@Sfr@sZmcHCS1Z2wVwWYA)Pp6PnG6FK4Pz!&&Uy3bzMQXQ zG8_)Fnt8jD3i(lGo%nT6%*xO`n*-0I2gk36!c$a!#Ps8{OG{=puZ1u4f{`;6B^F|A zjU2JV`~=&8mk=h?SHi^^BBk%68D#Q-S(!|E#YfunV&7hwJ$RFq*mf4C%sRlm@TzLv z$ino29XA8t6U4}_-@vd}AjQiOwadK6yr@XI;zir~a@vj!*rX6_$-cjS_H+1-tnrd? zO#PDc)o|>(5j5ysm;XbFLJ2vE!zY~AaV%ysZm+{hFH&W>X2Vb`p17U2gO4kBoeA_sQhi8b-cKqq7JE=-| z)G9K}wIrWxo%j*3UfSGjm_5>V`|)@MIe+g$&c1OzMBfunuEI+aB||?sitYUnEJ@2lyAEeqyNEHWo^GdukO44 zSNVA@gmy-+5xZ*GBEBB{dA!3diGwv3RW!IG7Tm8HMyQcQukT5(f0LZnOJM&)vbijm zfoIl~$p{vyfP8*VoM$%`ZGMdSTR>;Q4dmsv#RrOUx(X(uKAEg)!+#6X^91oLoCo*QTVz5!_V+&Ia~=5{T;icc33z^g$9{x5O?d9Pj+nh3)>kq) zzyL8}PUXlkhuF{pO@aBXEKryMyD}3>MX7N^xZk{`Vh7AqY9g(~mA>`emFEOE=cARh z#+h&Cq_T7xNHN6UB@=6S?7&|vmu~2=!)x)ELp|#?DQYd^^3`69g!=!<0zhhyC%Fb1 z4Nj6kfI#^ocwD-nw*BQuFwr{rNv(rBJV;$0@J8sF3UFf*-<8cTAs@L{E39gWIh{5L>yQhM-e{@LHI|Tpg zKU#wtwV272H&olSK?O=?e{1Py6 zivzEnw|GDPW?WSruXEsI`rRUeKOkIKk!S^v?Cz8cVeB6Bdbu(L+R6X%(P#dE7-3EI zoinx%j@bvm{8Z;EORy#ikz!TU$(oMMw#DF7MjH*FX-DH?G#U`EHw+Xn;UW`>0~hhptGW5^g!7!c%6l%j&j9iFRf@RhIU}CQ)@;{Bxe8V? z8Sm>!vzV7l3&n2_jXtvcTZs{^3lLrs_s-wXvBLIvq&TJ%-Ti`%Mf3w4 z9lV4W^5GPKE*q=|Dr}PLu&;EW(n#xc)$dRtl)L=q@#+HMkaY9u~aU% zBDjQMiUPF|_86D2Rn}8W#~!srKcIUpd@K<|vbD_EQ`M$!A>y>~zP1M)R~EU*6itdm zGhsH6Wwsen^^TpRJg3Xg>6F1G#_j?M^M&pySX=IjB|p>s=PNvFqq{%W7Hq`&MM9}H zYMuMTh(%Cur7}o@PM+C=xHMvxPW_dPd^6d)B_8rP!MDZ2#rFM)3E&1KEhC{&1GKh8D>BS!5S7Wkp<&Pju{k&39-_9@=;S&Kku%e3qbZzTF(vMWijm#V< z<;)j>)}LcgYE{ut082%+A~9Rp!&e^fi8XP(%1ut+lTWn@6jCH(8{8T^f-l28t}Yi6 z>;qlDJy`L5TX%K{l$_m0@sD-nTwC?zFJJty*y1-?cn!g4eWFYCWQ(vCmQe13-RCSu z=`T3f6dD7qwg_0e+jZU{Is66C=XJrZLA3kM%=TOooiD^;&?CXSbf1Jahl0!ot#mwP zWb}n>)4yt#!l?f!N917tk8&iTpCYndrF@+SNg*OfW$7P}zZ7loPZGI7Vmo2%?M@}o z9O1p@VF%yes&N$y+IAB2y+)HhT66liKj1yldit>D+(w7!>REMvtcA2ocSLd4Z?3=v zS`WIO3M4N!ES&D++hP^|Ry9%m!g+O*Y1kYvU91|%4_o}Pbde*yZTi7YG$y8V{`f9u z^BX%>hj+!PzdZgkIa3#OY4@$)_!fV{7zIs-Wv~YPhe5DTN6woNHR^J6Kr~n|W-Ulj zisLcEV5k2bp9I`f>XhY7^1JSj7Q(z=vgkFlN-P-W zDmnvAt)fmq>dDV7woT9N!T6?F4FHX*gEB~r4|`RTw*dt?0)}sT_}Wd+x0dqqx40Kz z2ux2s1m068bxJ&pa=}0uO)2*{$qP~drYh)%2)o&Hv!N662e)5g*nl-@6G_ve_54=m z%H5ySZ(k%wsos=lU)ks2jnVql-+S2Z2)#uF@^{YTygc;YpazKdrc%(GvwYyjV6O~t zxOH}k_l)H`HT{sZWIWj&izr0Tq%mR^-}2z3b+|-tJq)@Us8^L0>qermwri#YbqADG;(cHQ=Gk`xG z3Q|I{=GNP^RP;0VevZXC<$(6YL(D?r) zBwV(HQvXqqT&Psa{)cyzz(#-5fby^HGO}BfcqqA%%xwR>>9HFUxYLrYJ7@IK)U#Xd zyDy^cVVbComGQ5ZnCH4&_T=5OD7Uq7P{Co~p@WH6-TED&^N$BC3hkA%6FNkz_t!f) z?^DKymQeLqs<)$)f&CDvJ2kPVg=?c)aiVXq&0*^%sdE|c?XWiogdMwD?+9F@^!}3! zk!Oicy_n1WlRS%tdU6;v5hXF46Y+)dJD){J)2ElG{SsM~M>W;}UtXiT@9lR8eh7|J zfUW%3kR9WTyDw)!Z_ThyqfMfz_0pnLf_)5bQEYvmeiH7hge^zE@%yODQ4Ao-U=p}h z{&RFF0jF+GTcIGqIJ4Huzp{HLbU0EGS=-!JKXGrybEVEZq7Y6&-V%F$WT@O}-c2_N zqR^SOm)Cq z)H%#6S-YIxr9vB9tL$-r8XGvwGbGds<_) zOGvRyn+;$=q{wm038_=|U_A(OQa+FkppqNR6Ev7AakI(m({F!|*6H0!AR&!Zu;*^e znQp#(PfD&oWu$a~bhS(&U{%n1drd3kN3XcoPzF|jmw_9!5N_It)+~qCU&7oXTsf~B z0zOMVy-YWmb-H)p0kxe~eOeG5ehk{{97?GeIk|2`YkAu|)4~LKNT*s)+xhiEn2^7IVytCW8}KzGjB=2?^&e(2jXUJYHrOr0~BDz$98^cF8 z$DHLk+V>2??HN(MsNIE26tRVlB1PN|IT&Vy@oId{=s&nUZ4B0Xg@?DFJ4-4HU0-Nr zm&>C!8&Wnn<7bv*B?@<9dfXAaZ=MeBEURtvD8|HY!5dh2Jc zE&h+?2IAa>_Fv0QQ?OIbx4F=u4&ScA#kK1g8yi0Pjg{X^&ymBn#PZK;$`az$jKq|@ ztckLq*1ubeKOCNZE&kZq&AqfNII$(}|JC^f@xI(~Zp%V$%DdnLs_*$&&mq!2Q47D; zA3NKE9(@*x#|=iF^LM=l6FS28&Isz zlnZd4I?%AAWZt(*R#VS!1En{Iuz(~v)wHCqA(d<a5TAC9_D@^0c@f-2n_-1 zcApE#(i#np7CWTY0gxIn6+NVbuAaR?9wDBG3p6f&J2Yd>fTb7Zurulvqc6T>o2en8 zxeaUo_*x^lSZN3APXh_hv>$U$SBB(B>b%iN5cOp!5({-njLq3>A22YuU_`RZJwxyL zQ^+eg0z%AqM&<8(Bt?6OY}V;~8opCpaNShx$MvL`({~faRHI75W|pc+HArStE__iH zs|PJnO@z~G#d$^p6VYg$xLt8JQMBy!4@!+aS6W6DvHj&Tj!RoQQ(A|1=2naOPHP%w zh?hnue$TvJ9;Ap;#w)ZJo0M(DZMx19;CsZW^q#9SUijM=y2s9!pr&Y(uM*=J>P4}w z^H(`3k6c?*9*(4nGy}^E9%>i?sb z#1xM|(DR@f&$&KG$y2WAFBiij5jWB5lt4r0YD)BZnipkg?N?5nvZ-UoQJ=Q$Cgv>I zU1Q#QWzhAuKOG=gL+4FmN(GZ9P{DN3p9|)02jMA*Thj!ONQ8Lm2K0he8pDTFcxIXIWTvj`u)u!& z^HWbLB=Eo`zwZiR9P~)^<9-T|+#(iPqWw~=5?}uS;su~ynu`Q9$EMhdSuEq`FNyN{I86tB&7AlQy@G=5R zr>koKb)D9dl2sbwgS-idCVniQXA}I`dVhmN?V%!W))*KiF*QRP9c_2E_w9^(zlaGO z2enuTk(+NEyH9(e`q5YlNerFX|Z0P4wN-{kmA46(Noo1&wgpkJslPHb; zg90jLICroU#i2y&&>uG5K_!X7zkgkPcC6g7(t;Cpz)Ti)MbBHPH9EB)xJQr!gMpQC z<SVCu;oB&WY5l)nhZ}hqMaVgYGoZms71Y5fX~W7{JJsQi%tuMT_{}$Ms?1 zUi5>pznxhA3~L(yl4F@;nfij529%bmrQ;X~b)c)Sg*j3I3J3*d>%__s(`iUc03y zjrTpFrjp}wyo7558L4EG)~hPHjDp~6GBo3k<{R{frY5S}6uvPV#_4yG%E5^dO1g~G z$Yy}*e-7h2#4-rpvU5{7d)8to+{Degq`m2}7Qb~}wB8n^u~PM05eCeBrW>=WPdAgA zx@*rB+E{E|yv~R;NZ&o@1J>do{l-|V&JVA1UrD=6La+;8_qG;KzuWji7Nv2kIFOj- z`_HmsR0tGv{9l$8y8mTaAz%c7{_b4zRKDBy@P6jn(Svqkqm5K#VD^o~{!5NKWw*2L`xLJ%VsV;z!U`j(PFN-avg5`Ze zQW5bFk6HU>9$lKkRvje; z($^eIA=4wXbA?+dHu89lcJ}!8ps^h% zkvy+XT2bZxLkwp={fap}!hvw~V5vE)eYeBReSBx0s_PvcVm~nG2isWH*N3#h-$@!l zD-1!u(Gn{zR-FGOx8a9wGXF+Skd5EKS(Nsz7vU#Eu5bC@Yau!~@)bmT<~_njOXCq2 zU4)0}2ujPZ5?_ex+F8=xjr!;4@Myza|}`k#^goXBqo(nL;?8hZc~ z|B1$}+ow$elThJt`&IhYNFi5&xmAoDUQfjV!mUxs#6NQ8#1TTbsHf$Da*^!Airomv zX@9Y_$|H-Qr`~eO=paTm84pZ!YG`?~LCYc7NcVTb-+q=aw%xtgz#7nMM{wnJp`fcN z-&^eB9}VC|9*f%t5-vHMmM5AI&H|`N^%gx`VO71FXGz_c%xd%PS958>pYZ#Un*93i zvL0>ip{ei~NzJ)pkix47OvgBReJW=7m*PrshnuMrs3ZNyd8q#GNrB%d@i0zbADMlh@VfJt z?(0bL>^~k7+XgxrfCXQM!Q=8a&MqFj+X}_97W*lJ}t$`SChpqxhwMAz%~Ws6SVBPLzL!d5g*ssp@WjtIrDQusoPpy)Gp z7y3+ga(;yDmoP7YD7SH%@4>5oBFBLZ^Hx^6Wy{Dldyb1kZw!KGf*RNl87CdY&m7EQj<=2UCgLoW$;$|%8NaczSeLPw1V>t|W(Lfw_!ZJQm4I)%as%a+ z4q<7$W5^Kqr*|JyfgS)GuwP=jo-@HxIdv9TWgIcxKsQX#?k~*?j_?eDgi-{kLN$z> zOcb1mbUOk|VLsZ?zO#ZL#B`?75UQgnQu&E|bXt>^Hiq2wLB139*vpLlC6>u4cvmg= z_ryY*!?9;|)qHlTj^kY(PEmiHSQ|cn<+873z`E|1q8!#DYVJfc^S@)`^bmcJvQdzl+!Q=!}L8Ow#0ceTGjZ;MvVJ^@1CxNFjnjkI9A zHSPi^5-2Zb3Ft25gI<5GF27!v3;c{iOrTD}+HSmVzPuAwnmLFWGQ7|%$pz%10Lm3` ziMyDk;sY&b;TOmA;v3~kfXc)&qG8S@4*hof0erozB4~=+Fi6h%JGMZptM@eh)qYPx z!mcq==AI=-P{{6WL6`cvEbJGG}+leup0&Oy%76I7djbHxKJBom{;;n?r`r1~rc zQh3S1Z~&jz;t#wJafqAkRr|UAmP&JCV8k#uF#Wu2)4c6Ri$8n)&Gff>#3htFGBy(4 zTKCI4$JpIKK}D??OW)eg_-@`~e|w{6ozOJkqy=W4}{ zf7C|x+i6VUbXAw5H)jKJWNJp-$0NDXZ(DMHkWR({H&8rlu7MsxbH}9cPJsl3h4==! zr0NL{SK2gwSHH9Mq&73Gy95T|bYx&w4@H8}xj?&Jux3Tl=9ng1R=btvRg#oX%nmUF2|@o0 z2}?hsJEL3kFSmfdbaTG|ZypGqJQ8(|K&h1_6|B*bsX{Nvj2||2@v4C)k=AbD!5y4f z9lrbZu01qW^aYqjMi3}=k- zA;E;Wy+UA!Et}#9O7@{r9WqW))3U|T_>T;-$}=VY(DpuW_M}Z?mSrGOIfMDO>D4VG9C0;XjE^F;_BZfc3`7<-%xahwqIpl^Q{%TxP9V8KPaZgOaXMMEi(o zV9Y*PNY+Xz)0Dz{bVxd4UARraKWa!X!7200=DlUvKe^hV+x5&> zuiMIM!`stH6Q)Oj5_8u+Xltku=%HC*+080><`ml~y%+G4-fxke-}TQpNu%#^*6wOI z(Gz`G}dx1!26VU}0Ny0sh!X)SV*@`N_8{cmTl&@W~YTN=5{V~9<#{&(&3(_41rY8gB&>Cg5OFz!(KBE-hoKA zD6%4158)3gkuueY=-&7c5DJNU$oGi?xV_t=`2H0nl<)wK53J?Wu8v6`Z*`WH5-R#` zq(_Sot&gba2l@fmZ&67lvgSNI7MPd(XvAkBUnT@N5C<82`sZM1H%g73s|tdI1pvM6 zJgC;q5IsdxNrg?w91}+o_;*ex+WXxYZxtBvu>`iF2nANOf*Yoo>PGesLq2T(QBM^`S-u+ zY=Jp{Qz(_fZ1uOEnWlc?-%3ExFv*yt?sOY;Z|W`JPzi0R&E#+fQhH8pMl!dj+gQnD zSq*Hy2`+^lVfJ|5ma65_`K{78$Q(w`maDwLo3oq-%sg*TC|O~%t-I09?n2$|g~J6V zJQm|fKtt>N>&8*7rYh7_Kw?wzYz~=>^?RtPqpGpDFDbxu@p^jk77; z_kYL*u_G4n8S7o3C}As(;Hcy2c>0bxlhhNK{{8kq&YOP8~)1*TAM5XO* z#O^1b6igC*r+aXTIzogE`p25nuHsHfONc}z{s8}ssMq$Y|Mf%RZB8K}dS|7`_PMQH zHV7H7Zei=C9HzD>h=Hscd|4`y9}V;?^eEfL4db(kn8dj`K8Ctxv&iJjLzg&oAxMq7 zFRltK{d0C?mcd6X{uTl`t)H7viBrlLFQhj95=s>#322Z1O(=B$kb+hVdD}47FKJTz z6Y075S?1$3u(&U^q*|&)(aUYg0q(ZjHg-FF&`b)CT{zpSX#F*K{pp52Wa!&xTc=%I zk$7{j36h&~I|=D4g}unA>_Z%E(9Idj4vVq~ zvw`6G5b_~E+86VYA=OJ1tLxZ-d3EA-8Vyay!%{A`d+$M(9JEE`{(4F!Q2J=zx)$Hm zERiLO)vt!`nLm-an~gp>0`&te1jj!uv!sCCkzMSq$W@8cIx!s^ooO(ZsT({EE$cM9 z$%j5@#)l7+P?y@B%p_M=e1N(OOO+GuL}O_fQpQYb)3;W2^1zr>O@~N9q;7!ZHXy_c z>;dSslSV~X*AP=9S9xtVb|pR~@gY-?WSVD&@-P7==q$B0{*s+%+EHOna=Dd2*jCjo z2vGi(GGUKY?et>r#IAFinFOr2^MV)H1$eHZt4|CsV@!DD=(RKB zzME=%)5diCd3{aF`D5#Ym5Ad;dcf%mefQ!e#+2IG@7ttmAWq!O$x7 zu1mf6rKEtTjqerqI7b1$8kyOCN8Xil#$!;k(#^)a(7^L~o8KqPm((XC;x}zGEN%PN zvyLRo&A-kr!{fibW-x1lCa=h^%Qa2kGi%qIusREupe9(J z_X*fzN;$j&dPGv9B-FJYAJVzqbCR=>c+6cwFX`HG<8Bes2(w57q> zG^`Sc4R31Qhniv6-QxSvCiQK5q3n*tZm85-@Oc%5ft~Q&vStQv0ACs+h&uvt5yVN7 zP=EIT^@!7wVEZ1~ zwm^cnO1F^&zf&BZ{6vcMMZ3Ai3fW{;la|ty=+{y(S1wdz$23Nd@x?Q&xjS(64}MDV zg=m~5t0=L=V$2t1k~!t_ITGV2x7ZPzaG`;~>&b>iep|{-@VTC}pabXj{I4^O(Z|Vd zoyLK?Iq7~w_Bco*83#1!nHb!X8IM+IZ8L6*+Py8j8F4WE1 zgr@2%XRLqtDR!d%uqR+>3RL7pI>!qWMVoelBe1--aw3zq2||V&gf;WxeT^tpk-!}v zt)aPrN6r<1@UG6;aIjVhY=FKy15Qp4K)PKw-t z7R|>zCh}aUOid!~-IVJ*Mp!Zg_{t)GZbVrX|B`($jQ>EQ_7xEPd;OQ z%HwfGOZ6POW<}2o2QIa*iZaz^!U-sKg4dzAA0*Oh8BNCkW@c){f0?@TANWKR&F{jf7GY(4=v0|pjh zj#t=I-}$q$pw&AfS!^(1ldSml=Uq`b=Roc;gXH|CRAnCgLo%6Vey$@G&^;CYs=rV%r zpa6Hfruy+3Pu0u<%-7>%y^zX+OI)(=LlJnP+^G?G7Sw`*urH?6f6|H0d{`jW9deVJZP49xe9_(?iWhqemK-CYi8fGO6*MHV z5}46xiRN7bZX%(D4pB?x`5#=y+Tx&5 zHB5#Ym3NBe34uVDQ->r8b%Q~=MOafhGfCi$__e(hmD7yt2S1OM?re+hry*I_?Q(Gz zeEWu#A%73pb|B93cY+1nLnPng*lwG>-yz}Q-n&+3kXGq*R=tZwg)i{Gu$)np#DUW(FDX5NLA+LJ|0IqwD(2S;dYGO6U$q;n5mKqSevr>+uhieD)3@LUADgul0~iZ_72?ZyODDx!DDL2J_N0g)X&Rz-|*);%7t+&}zV1wN9+1 zqRej%Umo%WWr)1+61nb5s1stjLtFa2|Mtn`ng8Ba^=G(JS(UzJweic>knMp4Hop2i zja~0JVPNKx5;pHTImbt3LuEz)ROMPm_s*)5rLe`kjjxot#7HQ8(hWeHtF=lC_AbXA zIV0j~<^9swc3G!<(Fg3^xR*g-ZG}P(6-@?_$|Qb6Vd4Tzaw+U_;Q~x#C}B4H3lkZe zc0Ewi7;q%Bo=G}6eSB7cS*%{I2ncjs42rzShZzgTonuu9%?<|_H-xB;TqG3zCVL7( z{(e^Xd|L#oV~&8Z>{Y|`nV;t85EK0H3yQiJONW3OQVwa|a<2Ej;>!WOZSjUH-})@R z1RoGh!6r?syDN(bnhf9usTK;f(L#if?Ow!kEwG2{wbbM0M@%V8D9i%Il~HhBn4)LG zQTHphX79en4coQdJQ>WIwfDgFQ=rh;!1lD}ES>(;CxU%GU$;qtKd0>CwblodJXC?V z5NRjf&v6GB9Rrv-PJ)b(G3f9ofLA7yU}T3~v3mgmpY9B9stEZkA0Ks{c81fcyg#7@ zv!s+yXWrG+E?=^ zZtdv{7~9>YsNrC@bMquTn@LljB$?udjv6MK^-x@j+VfZLU9a*NbLIViX zA7eIC27n$IXK8D-vyq@GPCdxluwZ&&k~JJWP%DK^nf)fGJ`Xfu>=08%LIU4WuNrZF zEP@^GwE|6Ms`SB0YkOO-O8WG|e=UNdBe0<5d3;}P$a1eD67%>l=|{^0kk_$DQ*>Bf|Oao9=TZoX64xv9SG;6jyW z?9E5TfEoJ0y^C3(y!0BD<~J;-eL}iTvATB=wC`*XePO|(ppu+4Cd}iXs)6YUnW3kA z;=)3zr+a&6g@FaHtb$9U*6*>9?oQb)a~kTYM?S+Ezc*jPof~*2mNn`SqdvVL9NX@h zi2V7TCBnXl16I0g@?&1Ad_>hMP^N^PwS-+>R+0t^ETF(X1CK}9cDWigMk`RzYeE(k zpg<$o_VHGnqA(MMiRKDW)rC8L}H@oEAuekrc~!kf$X zha6iI5u{X@jfML=E*RIl#B2{Ik@emYUtoQ?p{W0I3cME|>kZnUn&h1fhySFk1173d zAF| zzrOn~c5~&vo#8UnsO8V1M|_#mrl=C`rySBMlj6!N%vG&?Q`{$%nK$SSrKfZF!2oCJ zQTxMV;`dk2>aUEIiQ4Uyr>JYzHZw7NfhB*~2R7(G7I0F}1k1S!M{_lT%@l_X#ee5D zo?KG3Eq~k%aG6LzhU!$~Y(x%6^bKr%zPgv@V$o3QH@DP<{WY$tDz+2 zQ=Y(;`9(cyZD&sL#%cDWD$*zFX3%=^V=^lTU4gc4w0Q(AmLDCQEB1uAjGq6g_zMm% z71@Pc8ch3%nIt|Qi!-AYRxs$UkNran;k$6r&|tbCbb$ezJO$1Ex`)40#(r;N@i~+Q z_Btm}Q!SwrYZ=V!SA2QMiX{AjS)?b6dncOkOVvUghFj`SBy)N)a*=--@qh-zgjsm+ zN3>L=9(PeWS<#Ln`-@*pQrNG?Jfen!>v_gUkmMz>mz~qVwT>>`8%duIVLUbNHt+$V zscjY+hkm!>0GaJU!9~b7pDbD%V5>TRRPc6|CHfDvmJ>qT%WT@%r?s=u6+)k}R67$B zaVa`0*nC>RqMoh({Cj_dqnhLL^ydJ!Ft9V+G}1O_I55|s`{Oa}lw}MtiG2OU4Y5q?Pn7Q z7LL;Q8#g;HHl=}bu;^Q2G3yG#pZkU=y>5b*2MV=Ovc@q(G0yQL^WlI zgpv`7^7{pdB<=T}`pbw93cc+mslVw)Nb24N%Cx+vOF7a-)^n0<4 zm-Wde`N^821G9dKJA0Zeng^v6SDqmi_-tsL7{2pLVj->mB74EVy4HVvMHSC8S$<2b z;CN#UW!BH?Z$?(~frhXY{6-!UH$7>rfhQ~)!`Lj`i4$)s3@}ONSLd(S?iFiz?hgtI zKL8lZ$jyb%9r1+dzhl)uB-;TVW3Z(r4VcY|Z&xMF3AJ^^k2EZ!708{Bx|mXB0g4u! zeNRJ((_2IOs%b0lrUugDBaFR=sHlYOnAuPSp_7zVle_;vj1 zv%@ip0~@mwMy?ZW?w&=~g+rMywD~+QG`u4BgUv^K&QrLo(+TV}G5t=}u65H3o*s`2 z1gc3&@q6&59xFm;%u973jJmVe>k)AlwM-Ocl}t|o{XM>=C!rY+h*MCYE{Xj08C``= z%^f5f|66={>QG}s1xw7<^qhTZWK6lJLzcTk#jHZ}+>VqKz-?aJ?K z4A&S2MSlmB(5yq;Fc#~+Nn`g5N8U|^RcdCpbA%|5Q5GP`ACj`&&VS>xFz3mA{sDlK z$6UXr3oxCVKc&qzq($I4sOIi@*3CWkp0QMOf5r|w-)y$*JVLwqkxo7YaT0+VM7>4_ zt49gA(E8n+jbjfDhX%Rs*F^J$9o5E@tdXoh^^d1R3cfS;gjWNm;C~}Q4K6URwL#y3 zR+kC!L+`!W<_rSpyt|_^h)% z65tbo4qC}}3RGQ4F12iIKxE%~h^wEt zFc)~OTkH}~tKf_2z$vnr>HO5Loz9yA<>a^gra2y`K9OJdHY4A1`#%77Hvx~$0;J!T zCe4|mJH%9H;H2BlXrMM(KkdeCty8tlctr6H-2^=RGJ(9Ib<6y5Wo44@Smh?+(VrwF zVZ69&F>BJ+7}9_8oq=J8@TJR1Vs{E5P!E#eb6}zQ3S<(n5y0LEh}h{#KYcjkOQ+fDdVu zB~V!DTD(>&jKeSv$^79|Q0v5ZOC|AXbKP?Io}5_11*~jbn_Bqw^hsG%#^^*+IJvpf z9%@Vn5S|Q+U7^Kd372@4A7#Im=T&jpb~9`!d@}Q~4>IQ4HS2nL=VUVP+TLk#U9|Z= z8(H$|eB<&H@e@$-l+v4zC1R^l^5kB#;_FOBDdeUq2$P-;!Hb?CG~L_)y*B+`NK-;S3Zsz_K2`3P8Ux()cXKK+uOOgOB@${igaw$K zE=Ynqk!FyV`-f>PnyH<*uM}d2uuL&Y#uq>EJtVjE=g&fs)3|uPd|{$mE|e_ldT-6F zhv}3e{qVLnEnlgD*MD4jWdC4$IFbi95m|vWY%!=9 zGU4FiV{Se&zWK4WT6S|0#5WOm`tHuRqH%ouk+&pRBiBoL&wGM4_Wp1+Mq_hUaCP7v zTzvF!)OXw9URu*f`|UMMY_}X{9E20rXYYN5i{d*AJa7BsHE>yRC2haE@7A3t;5hXC=~zmH3b^(L-8 zKLMHa!HByty2s(?fW^c6LF4gw_S4uQOd#QVtsiuKI5}N$^(C_I-Q~nVl70Z+FU>cC z-0NhT58DsHUP16WU`m#B-Vg8QSTm>h)|k?bSF{x9Y_xH6>!KNEr0(U7u*Cf4E4Baq zYdWDniQNtn-n2#^9-Jr7P~`Bk&sA^ayOle>d0(uyg|>1!sM8KcK!C|KoJH^g`5SGw zN22UQUr%7cAdrm6$03A(zrIi%dsr#eWHuu^1|CAda9uG*DD1jlZsUtBJ~yx~F$xoU z!i6R0FEF@fN*?K({_QL3m(i#WlwCz6!o-deokg^k0>Q(Fuc@kL_c4p)=K8p}NYD%? z(%pPgqj>_Bk{}>}zX1C8vWx`|=TBel6ZrjC3&8iC%DXotK^>P-H-)W&7s5Ou&U4oT zYg#RP%K_})quPGgI-)=S{_VBEb(G^cRPT;OxaY2R`D>c!s65~pes$vWNCaA;id>_c zm`XF~uG{uEM#dppNlCG;!rFg#dwt&;x(25pjU2NKQn5E|Zmt+%qGz0yxFRFX7{9ju zO!vUPr;4Ay)tsHE>#0~s6Il?Fv5j)2 z@(KQvnJ=?8ILnA1@2PNc1&r$P=vt;|Txi`O$S&wrEEz`P?nbP$DFjvB8Zk}v?`b;Z z7}K8{GQrq-r6?#&A#4xzNqSv)Lp?a|z#iV&jct=}#cCAAD~-O$rTQgR0>vF$swCw~ zk*NM%)z0hUN*4DkXLiGsr`0iZ6C%yOx)ea}*FIW|JXw(h{jZmHc+;LbVKlX>d=Wts zY%*42;Kv#8gZ(~g6QpxCCOkXd$PsVk;8C#mN-J)&##b!TfDEOFt8x_9p3f+u-_F62 zHPLo`cN=p}C2}W?C>1GD&fKwf&3;oy-MAB??IJB55_WVOwoqbc@f!3mY{CD5L0P^xJ8 zP$Yd}S(;{2Q2WnVHE$(~f$xv!&{kh>G)34L9VN1SmNEH8F%{cSLJ#L1M@Sl|+Fyr7 z*rrb#Vg{0jXd~U|shEl4A?1Vx29_X{4b=n5Mne09()Xq%u+&#+g}5Bt%CUYwT5Am` zo|2t1^+65~?xQfRk>kO1-IRUpN1+R4g+CLW823{@#ki}=dT@P0vqRyTvl7JrPLaeA zQ$iTMJJZ^&0NEEq96qL+3T$Ost1o&wQ<_ZLke|qshntDtRAVH5GL`oCJk!Gqc@0{d z)x1lsPD@l^CIZZ@vn#m>V)TLO+!leJqdm7Rylb5aif%1FlRp=a#K3PqiiT(QgV~7^wyUj`jm>97UyxftB=|!n5_G zbd}JP9Q5PBz~1|QyAf_<>kGnY+xLYNsDRHV<(I!s^k{fw3qD#s?q=Td~U9L5*g)r zzfAkTZv00WE4B}n1Uiz`tvqfRI3YV%!1dbQ}9pwF5#` zGs7Cc<{(cdTd#U`9xWphrO13gqN&JGbv>`FLnY=})SIpo&rEg&4tzcQ$cv4{oR1lT zdg&s3M{M45!loj^xVkqxW!;bP+uk~N4~UEP91UC#1y07^?WkM2gIm9WLOz;{x5cR<99VCGx$H+sof@oN;K)ZmmBE%zSL#@`{)Fb zW*G&zeYD1u#X}S(xn`;%><;8!uXYWAhbSqMu~Z#zD*6l9fI{eAtX;m&ik?zCV{C~X z#7zpA9jd{-8%OmdqML)H8ZvjfP_q=lmVoSJ6Zptya)$`;w<;g%oUAP508G&A&E;n1=TVQk+p4Z|6?MS|8V`R=NeuD+e8uxnHK=lvBU-kTaIh__!!wb~tIc^bZEGkNlXq%`30B**|~>J>6YPDddcfKIW=I|x?JNgU0A=uo6g3n44g z)Zy*vQ`gnk(y2!mGm^SFFAlP2n~oynx9ZJChU9XEU%EbuxmFtz$&meIr}fW1o=jNy z4CNI@sUX&)UWo22xx2NyZmnU3;pfZA1dHHC(c2V+M}v7jU#qW|2)8?V_}HkvY2Vy^ zsl^Vdkl%6oh8AvVU7_8?y8T@&_zzLxQ$Jo(0vpoX z+zp_jCs?zLZr|y+MMj;11dJ!R8VNTV6wOFdfX{0~();?Z8)zhOoHF4!81p-pEv`;8 zb+-KiOL*SmxL%{(Cr#6;k!{ zlA1`ek$IuNu$qHXsTC_T`U7~ErB%ux+$#}#JJ|wP7AX})3DHdE7JRvb#1B#`y)P}h38#iNDN?|(Y6TBsF&dU!K&V^W=TtVDiM*&`!`VdK=H>hd(tK`{Vu;&4wKYu^7@PX|@~)Fcd9`oHWru}Ps)6hdSRRt7h&v6U zKuM%kLxUML5|w4`+v&J5tJI`Yw;C$(Jxs{g0YN7M@7VERgET3s@nEA2hQqGk`))4? z?c1x8r~)v6p$BcT_k4^y_Nw&d99&N^Gjt{$3XD?kDr`NJ`^9NZ1@z$eXImUCjI}4& z6fxuc{cLZdFoK4P4GCY?v)~{@mvjHzJW3AV%zM&ob~ikX5E?u6;B80T%U6t$s3H04 zP9QrcX{UNAJ(J%_%Llen-$#kT<$f6@p~ACI7mNDo_dPZLOqgM4*j(CA&Dk*Yk{qmeDtfyj)n8*H-q z2pJ;-J{&vggLfIcat|+Lm+B-d7cn~B`tav3Y;1-yGCFYqMDgwhCiqjytW zU;^mWJr>{ZQ8qp2_n-Qz`ZnFtd-C?DCk+Cw-YV;WYsd5hDL<-RYwJ&JG6VkdFPr`I z6GALjm}TNmDG7?TfvH+T|j(H?>` z258E|?ZO5SZmG36H5Q!fnvIOwxU9Mh=)07t7G_6G(Mhbde3LM9%%6yml@fppyR(ew z>od>>H$3Zufb1m_kgKt}8T_vFZmCJZM`9wP-sR*_IGty)LV_qkZ4>Li2NaE#xF4Z= z(WVlx{LD|JH1xsuLMM);XQZX_O;RpaCp-K*IM{U9rFI-u1bI35x2Btz{V$B83%t9LlazKy!djb&L@Sj)ld1mxUl>%w!};)Y!3VKwe?e6(MZ z-OksjDH5(X_Q5dB0Jj>1vZkBFex*^aSpHrQ^yG~l1m_x*g3O0n?qK-LBq6X+Rfg|H zO{NggFh*D^F$z-}{|k6x8aqDNfKTrav%Ep~0-r#xn06$18FA)aYDX;Z25NqN?qNBw zV^{h|U~06T}UsGM2 zCzbya=cBbMKTM0X^i zHy(GvR5s3gRJ`JRA>=24)fNF8^e_X^X(4ADp1zeT{|yXXv&N z?~pV(`)U$~0WFJu8}Y#lWzy^bL4eVbUWgBz_AU~y zq8R&P5hf2&;NQ90lrwql))m<3YZCw=w<}{9eRadUKhqcWT=!3)?`j9vI|$H|dh0)x zL}Y@eE&)df6l1AedH-gv_e<9&$hxiCn6nmEIuwjJJK%3Re+W9yi+@iA8oAX0azE5sL*g6hYY_pJS-p;NhZ1u=}EV%5wRagmODCb+n zLY-~?NC+!L*H5C}AquJy3`k@bNEvXzEpYAIIq=y=XjMh3aKeUxik-M>oMgiJJ@NmF z+cn`pnLbia;PAny`=OO7*<_?it^YyGc)RU;PYO96e#C0C#wI0Payi;$jKDjG3ac3x zv&VJd)bJ$h;^B?G)s3~7^`(D7TmdWvRUXp&=WE3ci~G`Ve8-x^YqpMQ;={gX=gcCK zz@H=bQ(>iV)Q^O0-Tc0e>}e};T&FS|H=oEYl~$^jP!ak>5R&RBfKd$I*XSuW zG|`rcA0qx1Pqo@}St;_FF0PZt1+^@TR0p=~_RC@RLW+JI9(-G$pUHkkH2SDqx_nw* z%1r_s5F=5H!cwU2@7F{)cffl7dT~HBMo{rzK(K^^WPPa5pp?4j1cQtr2>S@(dNN5M z@0}U~)~RSksNkz_5=jWNdI~w6Puxf|*^n^=`5s%JU2hbR5n^U#@JE5ZS;?ri)4Rad zl}q27njc;SzY2fbOy}hKT=vyuY5V;Lpt2KKSdvZJoa&OIXlC747Q&zUjz3|e=ZDIMhtoj>?TOuR`dLc4QDsCV&TD37cJ_wzNyqn~i z@LuYHF%zsH7g8FcZz5Y3y$lAXgKMK@kda`h<^A#!GswLz=DNOW-l9rneIVoatCxfE zzi9wS;C?S8Ik~Y!+hqwLYQ}>f83|c0PJtVv1LjlyEEVhq>Y9^iYf9&9LE+o`XGE5X zC`~^%JNbXlG%XqI3Uf55Yb3spXn5-FP8RW_d}`@kk3>FmZ7d~xhNu1E(~mhOCW|5z zsBjxHosYh$Q70&1=j;SRK(N=_{=4zp0z31vfq>!bUyQ-{V#0!jKji&YQOZPs(XyFt9)7(T}dyx9zxb!j?XbU*197QDxZYW1q< zs9sfP0!ylxg2sdI+?t9@4F=s3Kgh{Xb(Y_nuq~|r8}u|1ASTX1ls|=UBa{RSPW8EH zZXLIKq3h3FGIi8VPX_M*(0RQ!p(hJh7g#N0!~F zlelw6QzX}G@$naf0qj6{mDvQlb-3R z1VMsDEeSt73Q(1qFy=RPbzDN5C7X9QPpx)|pO?G(&(9oow^@93YNNGN&$&viKK~K1 zdX9F^sO?6ooa7k?moD>pE|(m*ojsC6XWBi}8A z0hl!6`>gm4=o877s(AXUAMzdGU?1WztgeA>f32Or6bWzwOCludQ=H_>g`TeLOuokp z<;W6LO7fuFQtgAd5TQ6=iQ1NZP)2QC;Om7lnuS?Xec_~vcYb;Y=UYFI-{~WaBk=ZG z+*>cM85{W#JN;SNy+vW_p-=C$T?keaC zM%a9hJ$1?k*f_VT4BGW>!ZcehZK53gVOFI>l8-_5VqR{G%9MMLl!rtn+wx1+{%>Gu zuOpD}u+z8kkRrJI1dCNyC191weIoC&dP@AT^1y^@axhDP6~oZicJOjtAaKCnZQ*ki zmOuH9$=PyFv@e7+;6&qWpW#9`x!5;#v_Ey~UhE`U7sJ)p3VmfX8?2EPz+At}#g8me zYEcHMJLXzjQu)QCf6)RZJa2w~R2YJ&J|B%x$seCV$fmf4-Y`K}6Q}Hqa!wR0J z4NxC+DF}l}%n(xZJFtQ~Tm}6ISO-}`KV*Ss8$*RGl}-n%iL4J&^xIAQArc60F-CO3 z2DQG_L@CIU4-!ZkFl>woraPijoENh@K@EI#g66&+Ay3y7#Vx4+n29c)qjmT~(U1B29L}b^ZP3_S3z_ zDhn5qnPl7Nn}?KhyuT#%(-;jG5VFF!F@?>%H~elW`ypf0A(XpART@BKdFyqxrj~J) zcXcNbsn@Os2?OAKaK00NjBD(!MnlGb(Zlc%fk(0dUtP~tx-JKd?F)hxE~hV{L~q|g z2^dBUb+brgFHyo7%)5ejuMD#Qx`L}+1=iP>7&>A6pk8wv?6VXmtzN<0h!9+SMQ+K% z+-PS9X24~HUCksO{wVEI*KEg^TSf$4Pb90(D^z#VN)xlwk2atqVAzSM5e!YoziCk@ zRstyBr3QBdJEEUWs|-W>ps}M3E|j%W)ggFfsH_hA3LPvxv(zY29GjZEtfgR2CD$*n zsbgSlI%Pr!MhSpq0DG(V#z*1V8u#<>9|LDRpu$(N7`Kf6?9Kl_qpv6@ytOu_IHV02 z82+8CJ!>@0A;n**Md&7CX9{;QXFI0t#QB%Y!aL!hW7kh7gXUk4KGl-x1m4J1;`@)v z6xDuo?bKIZZSM-2xw|JE-0G`hoNg(LG&q`jc{t}BFlyF&g4X8-!hSR)?~&>57qw~Y zA9lhBp4%`n85`US$s+r4OZxy$tv(PQ8skKfgp8j{3^FY~!F&C{r!FoDxGMcWY`u3p z)zKe6&UNig7uj1~6v--k%MACrMzXWV6;Who?{1L@E$+25vt?&huDvqLs3^Pmo~u5e z-{<>%{Qk=Qhuiy{_jx^E&+$4??p{Rdeo{y}F7Li=i6eipZ@W+eRW}#v58rjxd~tV@ zo_A)YllrAp?mdX>l&_?{_0NvH~v;si48rI32gjFY%jxe1ZyS?RHLQy9d1<=FA%(&MB-a;314n#F;kD3dZBs5 zjrm;ZjYX}oavF1QwnRRNbru-ozo_AX^)+ug{$d>}FdT=%wlJD}|Jw}aO;K_nH| zwp$pFOP(8({$}0n9nX<`Cf+=|>7qS?&>ICvL6R3EQhzcVD^6ym!!AKqx?6r)@lu75 zTtlv2LR2o)F1fGXC(>);ne-kWP->x=tE3P8yh`Kw>u&x?Ixdz=(xW=n8Z!<7wfYZO zi4yz5L%>fk^eiM@1p~R3Kjr(L?$nQ}d2_u8{ZU`n2iD6-3=TA>EjIyYpayG>Z-w|o zT9fV~9uj^F>AH-VUFJFS4=fCG>T2mYUN5IOK>1*D;>CHzi6iYU!V}A)z-N_G%bb4A zI5bS$4XDJ8X4Pc+J&2WGfO!#S{S?&a_VO7+;KIgE_Z9rj^A`S8t~1)F9itz#vht2W zS-X0N=W&8jdNhtbN`U>IyNw(qYwsuM(_f#To z&woCI;(#h3{Cz6d76Wedi+kNHWBxsIYg4=H()F^#1wB0;e1rFPf4-BF!y)3=1Zk__DOKy82LoKQX1`t_dGvxQ8t)9U})j`|!LKPvaNkYQY$;!9G) z%=QauLqf$>xx>H)MZ?l?2BUbm`Bq?obu=x|iK|J{aj9+z)9`vKCW`ANXbj1Hvl8*V z#dfN4H|jbvG=>_9a4Jg0DHNKgSz~~mIrg*#;uKq4SBPpQm$6|6C515wfU3PajVL7bS_>nr%szpDgv8fM}y519wJbR2_w_~KyT zK*b9mN#+ec#v0lP-J`VZF5jY&&X?2`LA2QA*b*q$8#V~BTGa`$twQ-E#*g`b7Pc=O0<`WDR9CoQ&L*_ymf|1f+(fpp+35u+}#sBF=Bv`o83{ z-2dSGfw=noU)Xa)JIpa2h$A7M)H--L#3IFd`6EmDArCV;HOJnB?opWBb-mIEd)VZg zYd{ryVUKsVQemwLs|F6tGbmA&bR3$pzK^)R4=l=q3WH@JGR%_JyB!+yO~zYm?qFX_ zCR@6TXN4w5U}QudU}sY+5|X?g96pALu#@Ub%k z`REGEkA~8PIEFHQ^>&z%g$b;t$eEORbzHrirNK$BDjw1iU|d#t4Jjm&94uB&ZEf8V zEfJN)y}K}_1&8h}$xzn+`6awOywo`QUjP=y(8C@cNjrk8d>g6=sGM(K*>0VZIO}zW zpVkl_+ECnGm$j!1<4zQErS7}o*8Y1#Cz_}I=PI=)U+T*nwkYdp#d%TZep(UT`wFDws^f~NsZi9J(GRduj{vWao%D;!Gm+iosJsM z1lUhr##Zl|2{AI^gjO-*fBem{Y8VzVBWNm-R%Wk1h z^BXo`3B_DTdl5VzS$;S#gE35pt8)gjcMl1LL!~BreWa=3;9kC|!$`;}I{R~4>kU7r zb$B)MI8E=J3%X3BA>iW{xtzA&v+$H zlFDAse$v_vYy3RbauB?>=c!Tt_IJ#j?w{Q2oK53}*Y)3XxtqPl|5RP7(I{VQHbUYA z%n0cHl4h7jPKdGQC0?tuj$BH2e-=~5yNl0#f?NC<^q%l;yYpw`nUuqwS$AvYG4$2! zi*?6?*(<<3uK9B1C@SiK{EuNCS@p#u8QHN9pj4Khqv0f}{C!sZIk)y&I&6b2O%N8_FG@S2=dT(KOC zUhq*^Pk9~kYhEQ?R_2=}_&Ezc2lH15UR|V}W<5}!8*h-qa1zsf1QaI-2qgrp5tuDR zm$03!3EP8cptZStNI-*z##@{Aw2t zmr(Co?p3swq{DNO=RZ%sRhx>fN5?&!sX$TEoXngQA0q9B<|UTn9~Kb`vt>z&OoeP? z;et0SXgieR%){e;q8_WfVpnTQ_g+wRAJ<+4rQboGqVi}mlP4P!8rG=CBxB#8+JI^o44?#TOVj>={%7sMTLU*0>;0<*iqdlr|sWzAjCDS#q( zy?b<1@fNVcFxrS>_jc;yw6cZy)qZ7=aiW6@E7lkBVfGGXqED_v@{WX7~R zVI#AeE zmE+rmehi%#A_9xs>ac~#eBJK22+kfe7#fDa?R%1Uh^FE=J+$}jD3Pkhd={z|*hhqY zRNyCC4=)rlSCzD*3z;^or+VE=tv+NH)yhHICgK>8HK zysi7RhtFxkJbshljrCsfr}gD7`d&z!)_K*=ehta^>N;Z=pwu-)w@d8DA?P>-X8I#T zg@uMKwjRW1<7n%h`>@T&>v_p+`c7dCf!~%ovcw)pDSm2F&|q1-{!d{5OG*9nRyHmZ zUDD|{!BD*B@3(9s11i$qQaz?R|4-7|?S)1PWR8l(9#Ea*iFA>RcX(u_jE+4S?tSat z8{KWz(lYjbyWbRg>;C@9@9)mb3k^EtRgdZG4kI7;C%ykJl|OtNxI!kZ zR|U2RA@l1c@w=hOYN{o;M0qh9BXs?GQM`sI1C9xeS+|=yCJ(^)sskzF1N$;IUVQ$V zoJlpQ{zCxc?=DvISP?)5kWs^=c|D2a-hGwANY`5Y7^`o(Xo`8if&=|m*Cy;%?>ie0|>%U$Z>RNauL0{QWdqUptKi=AF)sqIza zDAlGuxCQ2EVDx6|1z;iMFi$!~6)t#%D^=XL(l!RshgJ4vR7{8eKor;@w*jo656%Bn zTeX+n_+6v>dxCsjee&EH*$%yj1Gu+lSIiuxt zd#n{z+!Yi0qaf(#L8voQsd4xcgi&m#w0ZYvrK|nrs}_9v!-rE+ZeAx}pSd60cfC=* z{rM=M|M9UcU0I7rJHDjau)jNnQ<{GMY~tLXv%f1@>#@>MgcSvi3}l~6GP{$)i7}HD z`&WnNaDCIWoDB1dN`79<44YrJGI9ZXRo4b+a8b9P*G|< zZaKNSvQu`#{N}As#Y-8Mp`mQ2@eNJYqOa?X`F4KuZO<;%r&YWv(|xnBhDlhVP2Abu zYnhU%k4L3&H@{Q(RhujMympQjCaEu7c>20s(o$K%CK4n67IAJ>UOsdGtBkd#XxL}{ zSj4oAQBSyjrziU5298e99Kq~$p@gXIX~7R-xCI*ag3gvU*7Gu$UP1M4oKn0Vt5y>a zVIv{y3?sLrr%=Nr@Ni>rZgcRx(cSO0&7RV)!hE7uox`+f$Qg^IZ4@4 z8|Q!*MeR_H`$Mtye?C0(#@?UA@F#ATcprN^6FYdC*Y~~S-Q|4yPT@EHq;o^wr_M8> ztE&4I!J+m=94e8r?=q>#)CTv|`0H57+hgEmESP!}R{vCf>_UA!6}fe$iyo7&ex1+~ zyfmnhUxRc95^EuOKqiEZhw;`_nDd#1qckz_bnNh}YO>qYg#~7$7`Q}*N>TfLsFg|~ zYPqRgUs`m8>|GzNS7v23^o6u7XqFKV&wZuh zRGZ0W{}z5|{j!=tVkPd)7|*-aTcsH&V_G9YoFSM`drG0pF!ZeLc{jkHb11rpipKq9 z>#l$`&@Y%RFozv{inoLfSqb4<;pW$ysAh*pStL^Yb6LV?Ejd|>Y?r$TwuGT~HVjMH zy@?+|$CiWAJF*B3H#dkGC`FunP_`+MUP~acF`0*ECu+isYy%_^HOU$L} zzjstHPt`P_x7uq~1JLb&`uQJZ!EyeK9ovj$+jKrb9VHkZTv$=#cZSo8wd^d+rB>&x zN${3-t^?Mgs|J3geTD-M@hKOwQ-=PuK73RrYdSOP>lY+N#*b=Jh8W0>YH4DTH$qow zQ+tq8b#K#+^4-SY-8_0Wko#4=oOGmYy(Vl#Q_Yr@H|%WJ#0?33+UtM8go#?18{*cS z*b^@tGL0a^4nGp-lsT*4*;Hp7e&*NYd7hF`Weqs1!v zqMSA&!(r_-3|Iu?S2Mm5x()UQV_bnIF9zGr3x9V7JicKlFEZSrlLvuD(A9(qnlE$l zyuYRcjKwU3Pfdx(!b|W-LGGA#jag8zJinQ@_)Yjtn=?X6_!-z=%P3E;Eqmfwd$Ho< zx6}S|iymsN>8lxmbgR}3h3T0?rQ$_k(sgW(&Z~k;MM4(g*56ZIqFqv`kj7;TXcAAF zcicXd>yp5v%EzYpe41>^xl}iJ_0Z~f{-t3Gg;P0#$j=BC?kcP>iWC1W&dL~`T87LhUjWWbeP?w*g!+YdfnFY!mw;zc|PJ4mwE(c~#YDjEWeXSK!8QYKY=u&FB(z~Xi(!n$u9t|BIl8%JL%8+;u$z_^-LCH z$&Ow%D4=XAP|j}BtyR;NwwB$htj4{lt>4z~pTAI?pR6X$-YUG~d*)$3jGl(7y5AsFu7cEe3ohW{)g6}bev9M zyLl$|E+QIWZg`Jnw4f1Ulx)hlbU`wvTfO)l(KSj{rGR`3O<5Hb>7)<^M51`QlG+}_ zx{vEj)es$tEwrxYs$WZbe_tP%d&Tn};w=9N7T&)K^h;93%jYesNpbSh|eoM{q&0k66c1vC^#LUR?3g1tcah|$6lWZqpnS}BO!Kw$HY)j&Wn&w zL*Uwj+$q0=>qq121mh9bS9wkAFFeE*)AAM*;eZRCD+=@+|Hy7$yH?7`luG(EyN8^XuSd5q8N{See2O%&?8kK8g z+izc=p7T-^j|2ci|;1N9@b+@ zcc!R(o+g+`H4jn&nxj|?(hGR34i-+DPmYJPiiuvxHK#%YZ%KqT0_b_cLNt9|X7Hfv zVU9P{PKgOFqWV5w*V@~q>2(=s#GT_C8@mkw8IE}B-4_ji4%~l(tpxgm_p(o(UG*f5 zBQc$=N3OlLc)ay2X-nskcEuKNtm!V>u3|c_+$Sl{R*d)z&WC0V@x!YKw70Ygtq}z8 z0vuJpaJI|+yWe7?WH*S=$#&;l3%WAV`gUi`#r?wQN~x;t-Ib-%vp6z1VVxKt z9RIQ;qo)WW6ISmA6r-dz&&j@c>t*&kUY|<5Cjb1`h{~o_Yok%_#swl^;P9b^R4sC} z9M#G1-a1Bq5){3Eo(n?h=C<-vTHLd#{#VsjuX zPsdpr45pzvk358tX~vjBdqLOQBEQjEidNe0d!qp6n1r#zjkU5 za(Rih)b+HDA|2?3F>FqXR^ z^2`yVW@P}awr zR@L?4`r`wVWh)HkKAi+)DCenMM(2-Lx48QYEdHYsC?=~Q*APi!5Z0llCzO|V#2{bg zFq-mPBV#LT)eV(0RW1LlCPEg_8tU9qVE(tgMUl#I+4no8-%#k*d*_p_m_+5vRh!w0 zf}G3UT?4;JlwF1M$43gIUP?O}yM2OB-L#9dVm5U?iPKJ0R;Uc=qL8Ov&x$y zBY7#mC8=5ym2X}(rL}6dKsm+bgz+$&wqND99Lhd^85Wlu|N4o;m)&g+&E5UqOc?lM zsg(aBDZK_CTolm=smF+^oypGk7|%4^_^X>`b~&fvjmrsuVanLcxqRoO9jHzD zwxz9;7p)t@y~{bo=c&tkH*+q>gl;QK65RS39A~HG>IabbJ=*2&4`!lWv)QF%(hw6; zA>;Z$T6_{YGtr;4QSFd8{9Ui*5YufkGGFEa_V6@_+DF4;aU7EUxl9^m7zOiH>-tXpmP32Uia!8yd zkZ;YzZMO%6CscaWF45Ah%o|7&KM-Ueyh7K5*fkMpcc*cIoR#yzSqGljL0)4WVGBS* zT9BWFu>Zi>9%=VthaalX!IAZai)t0vST|mzJonnY>up>4&edmvj((qOS?a(Um%OzR zGRG)!#+RUBV#KSq#p(n_W&bwfE-7((ao&8`w~p6WXI|5nA)E1BlVz-lSCX2(fh=|F zJpZtsuPyGJ6+dE!Uk_WBrM_&Zhi&H(n<6Z0( z`CEF0fSH;I*UV#P&9{ZwsjA$1obWi&BI0v@c*zxOrxtJjYu$$&#KDQ&Grt(8w!#2v zL3eu=5_T4$Fd9Ki8y_`m+6wHz`7zPp-w6D6=blNZ&R<<+F6k9K{O0o~1JAlS9`U23 z=^wiRaGMH)nBAP*%Zf>qz3kETP0qPA=P2CbMxBe!rEMw!=cO8fQ7xa7xM!CEb1`Ek zE1az8qI7rhNZ9&D(GixRs#H}^ue1Hhv_@xlGFP?LiDcq z{Zx0}Z?%&Ox@cCKw;O1YM9Cozn(1Ef>arKx>N5j7wgZ)rJHhR&o7MS9z|k@~(TpKz1k z&bLHx|15~BWDO}_=`=MxI9IpPOHAjnG9V{ZcfMU1hO>LeGqd-P(^$wo{yK|h2;#YwiOq}?&2>K#t z=62r1NS%c0qYGFRdfIE|hV-l?TCJdPown*c>JH#dbT&^nn8Dz!^ zJly-*AZ~D12eW4UBIr63Mb zeBgBfBr#0GgTF_*M6*gckR-Hz{%vUuzYB5@i|Z?I>+KYzXwdj0_i{=F@BTH$bIF6@ z{!$_9I@qnr5|P^)6eGL@!-uocKdGSHiUE~rbh9aboC}z4-ZD6AmOhaSu%9{tKeX#5 z`%TTB5doRp@ey|WnyU9Y`4sQI?Dhr7f7!MCggD(KaELrZOrfw7&EwtoJ#AA1o8`bZFLCyS4*({a0M;%lZi_cNS-f7-)8-AppzL57(XEfl14LzIM#3vDrH zjzc_i-##WVRDYc)ghS_$I^0GO$*7u+b53T-b!8aZ>_9lU~Gdc^TY7a^^=gayWm34iEY-O7de zp_}Kj)sI5NntoQAjq$`^IXxK1J1J5SG8#f^)9Hq7QWu|{-*WzO!I77eiK<{Eunoz3>=yW`) zQzldA`G{GiCxR-ygFQv+!vmogCa}>sH_T$zxmbQV=wx3x>E%&&hV^pJkHQR(!jymw z*F0PcgMvidi1q}XBJ1F#)Hj!$7+z18(_r9Oq^6-oxFy6&Ds$J1F6gelIR@v49^M0E1+jKOL}d zt#s~%ApRojad`-j4XuHi(s+>j*BYO|D|`A(Knl%a{nzR)FLVLRw6KWZhla7~eTP&WU2&pD zcS?Wv#plnnJ&B)nRTxlS`O)EF(Opo918Fr`hA*V)u+TBx0j#VN{aQ45>p@%(*g_4H zn`LCCgKz2;Yb&q`&@JgbTCzSECnMoL8s&eh)kL*p%lO4~;PPrKMH?~TV7z&Ce&v~I zPPMw%-ADJtnO{8n^V>s7a;&U=agOXigMm!SE3R0e+$`CKzxZ&9lcQ8V0Vl6`NJ6XD z8e+qm1*Cr|kndUN+}R&Hi)s;4zi*p44EX5utDyuI1{;f&1h3LvmhcK7NY@o^I71;c zxnFDgCbc8PRUIfY;Ii0I$8O#|gOB*O2o5l2G(^O|8tA?ANGP~s1IEmC$}ipK2)7YN zR)aS0%RrY$b?=*j4mcc!5@m}RjIz+Iu+_uE!7vmjb(@j`WOP@F8*GG@e=Mc7`;Jd~ zK$xr*M9cU^fg8r0>;qLz7_Xsf36=&AFIMzw_-u*l$eEEq`9A-h$-|Pf>;hdwEK3$5 z#re*X>$R@)nj7%IHD1!k*iy>zWWOsXeM)i^#QrB&a_g}~)awcl)l$op8@`sB6u$%s zm5T9UlrdZym-{=m&=j2Tzt|uk*4d@iSo>DmtSj;9pEr_ro+2)^29*d*p3x;3+zSOY zdD(}}i3mh=`2xTEpr!Q>No5M^wzz4Nr*|6bpv&;Vv&ku{(YKvupI(Uxc$IX(v>?av z4nJ!;MG{REu{9CSo=OI0lMq_^8=AbLU{NCbB5d(raz$A$UdE>P9>~S3#(gibUmTfT zOD9Y*Ib}A%vu#{kii?S3Zh1Rk=_D4UpE-RuI+-F|Rf|_CXZMRn`8AH~f1{u-JjB9% zgE!BWWzbA|pU)O`Ju5}XPdy*aB3f)0oNz|NU^w9I$Q7vXLob&U>2*RF;!PPbSX;_S zX;zVlOhEYYUeDKMDy(~@O!E*6>5hYVv{4LqKG8FqNVcVm6{3jbrRO49yyryUz);@4 zFd-!IGWEU99QC$wMc+G?a!xUAAr#~h0WjPHeggmjUG4@&T6acG6M|J>42~pYef+t? zUsAxy&xadGL61@CA_cRs|MqGK^pUrtl+b9Rn(LojX{GmazJB8AKNt`B8gWnk^6lLy z5h$w&r4)xExMe4ItZ6ed3S)}1i4OdbUMgX(laA0hYAs++0oOuu$v(GM1#$1A#;qU+ zqzL1C-+KuQ9=`f9QLWB|^r)B8ix9BS322J+FSnvOPDfB@{&6dw&n%Hk@Ugo8pn|`X zI=0-Jj%2QgfMuij`Y%{=o0Rx!c!Vp9Vea&$sR(XBc_`Z}J;bHcu>?mx{xluPNWPtV zWS9(a?%%Z}pw0U#VI6QLs!9|7@$+P6B~NmDnugDPekAx_##F$=geGTPzgR8g)tJY} z#DK8I3sHvv#4FGhkZ-#QQ~j7^U$f*;DHQ-A2{fXbszbC8`I+sbM;*KS1S-% zk+E2Lw+kg(E7NeK2_k5mI?_kH?_iwnhV^Qs%d*zXt!0-pFW@LLtxjLrS#mT&V!6A` z=^R%g@*(KTDXJp_iT}J@AdacXvPKy6pT8g^2M>u>DLZG;d#SukNoK7W>|J2)cW5@G z6^dg$D}r?B=qnL*dzFV5VA7bsH6%a-ZGda3+~OSBQ+XSaSX51ssQO}ih6pyz_V8$Y zAkBKYxfFk!*~9f>QDE_llHi`7n-;i-m>G5yg!$k@gdGyLkYtnoT&}xlF1bm zf6gaQ8Iuq&7$BNLDICGQZ9V)1cW71^Vu;@d7)15I1uYbn4dC4aojhGy?tBu>-f@!+ z{g!Z1yqCW`sLt%)6of06-J8G3{F_J|oYP{mmTt#^x z{zs*Xp+4SKb33&p4_|>a85S?Z7FzlfnEM^AnCZttOhS@YCU>MNsE!IieYaGD&>nS;7So^5xK| z0*sJlxST&uXx>e-514JN!%Sd3X7{+5iXRoctmIhLv$-svl`tDEG~X?7zmW=!&w0KI zyajrIbp{klhHqR>|5?9`ju$-*gN8x`CDNR}k_yr1x{-4zHvVIW= zCRwkpSYN@xiT3f-okC=^E!}7S6^e=^Hipqq21(kP@t+JAC$AV6+s9H~Ld-r~Fque* z0d4`cQHf;C?5LRsQ|6 zfmCzOii4eUy0o(u@6nCtC{0S7+ku_`0F+JE=viqoE~c6Z5)wlcQ9IGRUUfz6?~7%{ z-x|eMLR9aV#yPe9N}T&eMBC@D(^6oD$o`aL#!Zj6?Clc`spmSQ-Jl3f2+mNm`C-yt zO$CXDaAJ{^J9%s(`zT7x0$X4V3pZY_W2lPIfeC9t(u-R2sS3?Vog(@cYC&&VhRKFY(Di0K*OVc}-(8R7jN)lfoFeKvZz;tnP{KNJa z-`TD-;~aakzr-ONMdpl8{yPig@u6v3|9nCc`_2r- z!~NTO4E0xqQN8>UGq(yx|DIoY22ihbVr>rnx7Rax&kG{6S_fOk)t_oqa9Fd7Ruimu zpD#`cI<}1YdHl))nlJiMy)-Z0fL?zQEzdNtJsmZ@|X!dfax zym8{fsKcRBUR?g&VXFGb7}w{Gf~GmM9Z1;Ll)bW^@oO>QKN@-YyclH6zV=HeL)oSKz)WzbF|gl1KlXdq;^=PB ztVFZaCs@NRjxi@`pXw=p=#PV}&ATEa#D9ww8{4U312qQl@&6eL0#8bgx5bdf|9u(D z6Bv56 zZKsp=RO{iH+YWlA{1&0;SxX520n0_7JYI9(YT9}?E{@UbjF89(z`?R5fFX!mCt>sl z99EIyRyObEUz|b47Vg!NGSwM&HC#T9r#69$QJdXob8PhwDy2dQhAmd)t+m_J zzSK-D`=v+KtoQK+)k-Rr;t27oYMR@H=+1i8#zM*m_Z1>`;1mdlAdw3Z*8cS#{Zx}= zultO%Zqv2Rh}Fol8t`~&X&hPc0S*F7|QN5mQybL)N1dw)+d=DO_5Ne17sto2v*(Z&-yjuf}Uq}~pA5rp%Kx9%h_BN#OZgWL!L zOk`;L>jp*R^y2*^jG$-9zsS#dmS`{ehZP6?g3f+t94FG8rt~Z6p1Vpa$tT8AzY|yY5XX zt?P5Y?+WL5I5O@JQV-oi&w(Dx+~(Lzcu(O}6O#VDL2~rH!GE`67B-M`1TsfgYbZ2= zaX^DrQRXd|c-H*bT{+PzWcmihxGWeWAllm_-@CB2DOK#rUv`C9UEYU>`9g(Nd|P8Y zb)aaHcV63+5O=qgYwu0~82X$$Q`8$XbJs*0m=XAmvlyg zSZ(4$j$v~3PpAuJ9gth&Dg@=b0)ejy&mnCEb3h{YcfkyA!?VOpcctB zS;y*>f}W6AH&m*A%{BRmMhn5LkCgRJR>tkS2tmmh{|zsD{%3V~b4#e@mTSXb{rbxt zX!@LMA`df@>+5c_p86s%L!mPzkt<_At>U?OA_t?g+TcjnW192wJTB^59f=CJdXa*N zphN*z`dE3Db5lQ`$f?{oH{+Kpaly2W>OK1-gvcy)Izn!#gO$*N%j9`gh9Es^X$}7a zMNY}WnM9)fBZ9TxF|0Mg<>!aXQ#mJHV*?w?`>$PPifW=%tSElG$<(yFbRnyLenbw2 zD^HO>WB9^M2Wh9jVCGbjajukgbxVdFt1dYm7ej4a^|q7SXSP2dvT5BMrTI?;J3Ue? z_**;JVC0!+)*n~qxFOWA3vx21tZWTif$41Tl-4wS+=>`Vu|bzmb!Ix-$Y`RmR2L@% zxzAI|OI59z-ZOLzk3t!j3i-##KU1w1A)`;!=%Khw579!C>z!1GuTZUTD%TKZMfN$o zG!Fu2qXiN<4$Z)=tuXgx#qx5_!n1PkKTssMF){cq?LG>7oo6W>^J@V@XeD~&(mm1* z;%cLAIaTm-6DT%b;E(J%Ve{I<6vxVkuA5{Mpasd5m0P?!3pbUO^29vqqX9fN@)`Njm=9TCVr1pyP&oC$e_m z>vD95H&*Df4g4ZgtSx5E;kttw(M{;qz(s0KM`nU^_zePDncl=Xv)EKU%PKtkWAtnZ zQIU1$5|<-wY9Mu5syfvGbU%Zt&71q(3(CsC%rM^t>Hf*Y`I0Jz4B^kl7cLyxaMB!X zxpA-dmV)6=Bp@TPTVzDhYv@Fz#Ks;0UR%X*q6~X83ix+j0kth zh;+)8mc$|C2A9~5jq%`8`6_Q!+O8mmvtd7ZMyFd9+diqe(bQa|(=4t_+Bo~kU4D*I zPjXTD#YY9s=wJjJsi0$<00YQLMe`%%tu@_ff0Yi?tJFa9`=mO{6`=Nr!FSR&W)b4^ z+;`+H`6{Teh4nl!2=>CSQiPfp$#!x#Lxug44L z7v!K4n2IXkfxKYjVkVUF)Ax*#(?=hwBml1eDGpt6lMDDO#O-s(bSzTv$)7!#^!phB5x*aImx}&KA9BLHaE&K1N6rw zIJ-Kj`sY{y6!v96B{T-54u~ljWPf<@)r=iu@kcE)Le`5gp%cG^dcLZ1we|U(rJFiy zc;Rlg$DS9gn1S~SQFGy=ElTRXx)O~byu-VZ%!K3-k`&S;WX5+<3lrhr2^W6v!x&x8 z&+7W7$AKluRR#qufOUl1(RG~fxXbs8%7^Qt>-g&kyzxc^3ZZpFR_u&1c?UMz3>R-) z!afURTOhHpo%?;t6#xX*mlFBjSVBTwQk>YRjj8;JOF~O@TQJb~IFxhA?m-u^;+)Q# z%{9@jUYe&CK#AS@c#OYyp~aSkAUvY7U6`G4CjO#={9%wCW%|DUdGy*wtnJX?0iR42 zR7|h`)*^mi^14Eakhx@T6(9eC8ZdIHej&JbqCz^7ZCYKV`W&vdD&&*fW3ojT#camJ z1adnR79=Q;$Cp-UF)i-*#7~u==t=B9JB?bN5`^3+ zB>#SN;-Det)wJvHr)1ZGjppdhM9$gs;~QJa-eKcrgefi~jImsU?_S&Jcv%4LC_rK- z8QT`FVf*d-$v}Ts=MsmAdFHh@-na?fGpU6VcOo40M>-`h)3g>;>RuW0M zTBEG}$pnh|%zc>?7Ou_b@k8Ah^Pa$t{T(l{n%sMatkJE!a^(C0OI?G`Sv&QB0;7qq zJ(CoA=Dn`~-e5gRF^&U(@E}hnWy@U!*B^M)r8u6=-#p0^C*>;&yf=$MJkx%I!c-ZW zD)`m(@XQF@%e0!x5k9GH%7xJoy&A3~Z7PN!E)HB+ukG2+b&m~S9E904I`%hg zEU_&A`kK9Pwezl(?MPV+T=2)Qd&-JGZ4Z;Mntu`Smmp|m-zT2HIwdfscL&>hR3n0N zdTHZ-N-5nNp1bcT#=L_CTGOxVfd9K^NQE5We|c9ePn#k0=;)*=>koO0+nLtA9+C;6 znY_D}$n?h!0^i=EAioHY%B94a5qiGiI}~OJVw3N+vuj3NCIn2` zN+d8$k0qIEYnh;v&`qqk54_Gt`cb}~x*+_)sosdkC0y*K(3x#Ha{Q{!`Tk*P=o|cUhFR&H7K+A*5g&9QqFM(TKo&+)l>IB0kZ7__P59M&rEe|4`!&j_W3Od1@jePW zARxLlEl!EtSquf7u zHz3_bHdIzF@W=Me5;DEiS@`41?9&M5zZQF(F1#51a%$$Hd~@J8yyF{(b&MYCm4LEI zjuPPR-t)>TDK7aiE75B&;+UEGuHBK7(tKC^^2K=T#(Xkgh*;+Jwztrr@0`LWbxUav7dc$5AtpgV2Z)J0hS}b% zM2(InIXoPJMHd<_L*)xiRRiHYs!des2-R59B`liz$SyWS2U%ZVo){=GpTfk_@W2gG z1SQ3BJ_7UCfX$V)Pw&GICFPEKe>}FRDA-Y)ShW<)$GgmKpN=aGuym(%F^uFwlL?fZ zPqe7G_o?E^Cqfu5j}XVZsdSI)#PP1)K>SQb0Y>Q+0Wrt5A|)M^yOKK2#j$4iGf9o6 zZ)en{t#6KC*_6>kH%sH_!!EPw;IFb10*5j9EIjiqz0!UcuLGseeq=J*oB25vk_b%{dp8# z-EuVw+*XulkjH#-lymv|_esn4_gl)iux4)Ll`(FJ;1>S=;H3M@ErfQ;QYf>Z(9^co zr#hNVRLsJe8sQ7F!kWST1uDrz%$G7H+C)7HDdK#JC4-pSgj3&=iSBnh=h;->ag^Uy z+Fe{`%MgCHySR98u-_LUDPuQUCyM2R-eo5?qD{+usnpV5J0J)RX8) zT2ku=5Pq0&OcL(Q4hmf_Q0U~|3b)Gs^s=6;%2gEksm55tXNlCE8U6w;ma3!{6=JB^FjA>D46 zF$`Z&bhUS*#9&#z!8w$?%|q?&;zWONe}z__Jxry^Hjo;R<={nf&8bz#_mSRgpU<>@ z(}9&5FMQ;d?#cChsg5vuU_?959A7-{e%=xv8gNAv^uRwU1byvWx)Xb2yS3ns!QZ@- z&t_cF-RP-Hdnps3VO+;3m4-x|U*B4MyKr82UFJb?KZ$*r+Lv1-nI{gvfoo|@aNVi^ z=J1ZwL0gP>BZk{vL+6QQ0fGU;=%9ulc~?<7N6@r3>&5UPx|N>G@PCDa3}Uflz%fCBe(HU)^0SD(ma%PZo9DjB&OQF0 zNA}ObFna%D!f2#LC&PcYAE=Ns&Ofe7Zrn)!&9r(Y$ld<#iX(Xp){2%L^020Hdr~xO zg4hdKgJbp3n2?O5-nZj!{NpdUR!s9yoyFWeX2GK38kdrm;7lcT$qt6OfsX>Ep`@a$ zNQnrDmJ&n+B_#wlT0lxV1SJFs2^l2Vtr zKhOQVc;4^Dw%_;nz0P&cxz06g*+{KLM-$yDv>5{#V5C__OL7gGF>eGSW0d299Amer zsP$z+Ndz85$veh_Y~KI^8GHb#7O@*#e>_d!!k88-06cKgQp}y*z($}GJmPb5FTY?x z-e+xmrYsfSkAgu|Tf(spB=2P3)9^-AdXYOj56JwyXH*$c*}i2x>gkKe4if;pu544k z!(%HK8m07a;G!8!arbk}qbR-&f971QJ5+q~5{2Trfwd(evVDBAk2brPqYjlEp78A`R?s8dEB_FrVQ69qaEd;)Q;T zR*&puwV%ab2Dn4PA$u(n&fi*sv<+EO_Iv%mgQ>aGIe`q*v7_7Y(7lvQVW`DG=q<`^ zqMB!0IS~C@bPfj&x7Kh(vW7*tQQ2CY?f~2x)XSNkDS+N|UUd;ti37cEb)%qjHz}WW zZsG$G6B8Jd6+U_Aq4T%p!7Y%x_rKtv*ls^92h-;+xliGSp}yk!(Z6Qatz*}j>Fk7V zb0fYmL1baJU6LZiWF1Y9-G1pD3qP-9p5+d58@A-Gu(X6iGm^Y#-VMb=4p-*6$KPp% z;UoSn`6M+>vA{1F3iIiG=@+E&A{cnKdJXFWejb5c!w@_TlTq^bAkw_8)*pY+D?8Yt z-?V)sP?YX&5$H=NFwX$|{f_~txqY}eOj@-zdh>POQ~KCjtMTrCFby#+TKy^0`!L0` zyjFY^TYn1vQ7Wvq@Wa0`2?UJIXiiv(y{~|)M~_r>S=#XbuxtB3$uQi5fnb4bVcZBU z5V+@QiP3O^edUhVnN*1cy)$Z z3q&lYPwYCuw0rA$B)SwcijPZEcVjUz4CFqUm%pVG$gpkd80CL$QAXCTz(2@7fAMW@ z6n0OWfY!>xKi^uL-+cYEXnhi_RQxMOuO_Jj`lra$^;w3VYt~)DU%F;0d*9Yso+26_ z3TMA!?_(u>2Y3*LwKh4u|C4d)XS@lynu2X|c8M8&EkV8geleVHyu!E}WQ(_j=>JV2 zr*pNaL{P4Fd8Ag(()*#N z5ELga4SHDtyT$E3c#nDvve1osJid1Yda4PZ{k)s`{j9-FB=ZX(1@!dA$47Gs9F;AT zfo?)*Ac5@es;nMr|`t8{;lg^G3w^^k&V<{c*FgcB=5E#^S+}KD-L`AS-Sn77Z4(5*L5dm7+gi1pZUb@woDy?P^}Nu?QMq(BZrW>(1mZt%s~6dvoTg=6#}(j zWinU2@S7E=N#pQXP1Moh4k#H0u5m2mcdny~hg7cNO`sxyxm;V!LuQ+Hh&s z3!m=s#-9Hy7gEjG6f(<{GR`r%PkI1<*>fhlI$4-_&_lQZZZK%EK$}IA-+irCL#|^@ zoq|JZ2QV;#r;13KEKkd^E2R6`ujzou3X{v7 zMAz$jlNnGj3qq`EP5qDtwSm*$wR_PB~9!t;W-htKycEEl8(vPmr_gjmGPFVb671SS-<7d z6U!hH-z}xEt>^>)}%I70iIXHZ(daN0lhV|)HYgNx2AI6TviS7%cJT|<_Hm5jEUV@l42<2WJ}SOD#h_Jaj#HSuV;gG9Nh4Xga-Zr zf-v*oSC~~fNlPbTyMep-c!Ld@fFlf#tYCGoYi?I{vN0g; zS*P7=qu;-?kpdgc=8dC!@Z;zHV1#iwtg)8{_j;_F3`x#T*OB2%6d4=KsVot~BSQ9T zfIF)LviQY7(8&Gb#TLO<1Z<071<4>tIso^pmK=a8BKtjiDHH#4;_r^`73lsK?==&e&;Ss6n$H|WMQ82% zJ>35n)cw&(pJV1s@h^5J)*+kcqSEMTMS`YQ3ba0VltkJkBTM*I+3g)v{i-Unm0*Z? z?kq&n51e;D@j71pc7SBm6CfRE^>17xKGz?jh9ojUC04i`@Fgrf)OZ62QlmESPh-|R z8>>jIViiBF+;2Lw$x)aPFe&4g3lOy)qVtrvUw0HW^xM6!<~=LAqygGL;C2t?);$ce znl8+4`nR!ug|`STeWkFoU+WQWg(JrXBa$i_??2$IAWI((g@^}WHRc&B=Cu>GvkBgU zV%I;mT5(ShPim(L^okG<;sW#%L|i=7O)0LVdL1YdxmSsF%@eVvOmCD2*C}42r(C79 z5&CyM;S^>Jgsw7k=GF`|X3w1U_h_MF4(zg10O2A+FlJkiggJUgYV?!wdLG#Pl(o^; zv9kj?KDhwS@l;Z4K`h0I5Cr_(v27w-V2?06yj#D}GrrS+!TP3bVdl-` z(sV3>?y{$@k`hg*rgpZXA6RK_{izsMsCk}eHs6k}uK83s&enQFxvN)hVPb zNV5ql4EBQ~30+v#P$o)qW-%r>0ketl!dFwMT>09%(?T89NE=Uvaof)n<;r znzZ@4AW2>O##j)f$W)r@B7+s{|8}RvD&Nk-5mX2rToFXAu5DkmAhR)6X(L+9hv#|mMMY}aYj%NGn45kOr^k;>%|f_M^*TdV|c0l2G*>D1YJ z8qN^B2I777+u1UYP}`5|KCrJ_LF?>?p51xmQ06_|w7VC3QT#ybru%`!lvjMWB`+#( zvFa!Wx(R>7uwdv@_Hk|oqBsv0GFgXXy0<;-4?=5=L)VK92!96Y*<13v|7wFDCt}nl zi^^c3ydnkA!sxJ^FF{{4?vHS^nmE1ypHEklT}UbNvKt=eh`+_<2UjJv@+GzfsSLD1 z&-jog!WFbsPuZinW6#!~Bk{!s(G>ODZT|E2oP3>14m9iNa@B1n%ApH=G`ZKeakO}A zU%SOBL*zcXU6^x_g1E)%;Jn(91{i1eV{Y$fFP%N-MV|bd|K2QLba6As2klk|H4^t& z@_10@tC+H(^Kx+Bixx6vzb6xdS4+#vPyF27ClarY#XifceqYX*)_v8DVP*1`0}Up1 z&)$%VBX06rty38+bAQe80MOHHt^o{Hv~>kQdzAQ#QdzZbC$a(^a_3)~mT~+qLE^;v zuba%C9?06s+~Js+6yTt2<^?U2{l(xH@=Ujk5oO-ng%2@Gw(rwD1I&n9rICXLT9m{K zU)(F_mgE3?pnP6y!!Jw29kaL~BFPvyw19hI#9!vY;Q_f8 zqP4v+P;azNDL6KWp@ww}pb!WeYoJ~*i)ed6RYy9D(SA!9TUX!*qLs88!mpnTA_xw|naIl{qa zDl*X-T4m-x$9I3x!5vlptd+=5wRexA{v=~f zL26E8HB9FLv2-*7C5!6Q$B|w_Xk%CEyz==R#o~kDxzFD9d>|Wd`Em&$7y6BoXOrXBQs-zB?5oIwBftarmZ`NqG)i|BeM;gJhxj8ua~BBgrRB zEF4~p;oL;m>;Sl((nYc(ma9jo(;0X~`n&m<2c2aV<3VEJx9HT$H2}$%S9jTKtjj0J z#(>N*GEb+X-U5#}utLpundy^h=tW}T$4YLLxrt3>!Z8k}@C{!)I)T=BMK726Dk9{d zMgDC^;@<-EvZn}Cl7gcoR-#{R$N7Pt+`}ennUSdo*?(yqZN~Qo_>)jHLB=D3oDjda z{-yO}l8BTQmdvzj7@`OQy)!$V23>HQK3f9mt&VaXBsd)U4%jqOv(LLO{K zs;jEsR+zT#*P0#SMXEZD#1~O0C0|YaWC_};ID*OpPfjZ2jKL(hK*1^+{!VB}-3y0Ga|ai{baFZ0dV{fH;# zLDa9AymOrytPYU)vI7p-kpb;;5d?-m854(Flldf3xM^>yvqcAvuRi*0UvguXz1SrQslBNrS#Uc8`0d*`z zxeRVJRxr|tVe3~chrRazmqnIV zgxLoLh9WX7v{2Aq6HGD0(2GIb{>&-^PJa`^7%pDfmqjmqkv}f~&O+}huORi$j82Y( zeT}rd-95!y5iL#{k+8k0Zik#y^{_@TgXqE4ZX1~I1y z`#k%Y+$Ut;nbq=^<9D`1X9$HmyqfOWswJHW*C@4HjH|@8Y_HB}2pE^+Qi5+yAARv| zF$t=DUYJ-J*+;!S@^!g%sp!8I2+jRJ*jDlLha?4sr}XGP5F%G$neBEB^sy?tyr#q%Z0XQ4DE}9=k?k7Z-cwyj7o`pWT)C% ztfwc5tGaT{#O!LO-{CPqyH$&o)W<=<0_$!>R#%kJz~Jw>c5c*XNX`$UHx0NXuF;J{ z;~Rg1O6QXpuWWB{3Mm0)N`Ai@3!(F7_Ks>xuy}-l7TnsDSqKA*c460SC=mwGl#)T9 zbTn}0sf;8pD#aWwXvs}f3QbQ|a-kW(#0r>Lt|b|F@Npq;fFi1=>29?1KJA1O(VzSy z@n8WVWEC^LffuT3H6lJ(Y%al+bF9Qj$>M1pEGiKDT}zK@tH2I|XyR!>tgxEoSG&?9 zdd6h^HPyzyiU)IrdVCFzg3OBCOmPiIlZ;qN`@XBjKLYkApfqf577x}*WH)n}_5FW3 zpT~#1)K-v%(|%5=EH)z`0SU(H z3!i0n)Wwsm(8XG==dY))M^rd#acZVhi<7;vd@;3U*>AC)k4ry#L-sQQ_Acn)*{}4D zdZPms1gUm0(Jr_XY!*R0JN5WOuCeVs6X7vx=vevjhrP!mBoYdY#T{m)M5RGlNmx!@ z)-9a+X7C98AIQTCHA_X@=k>ZpfTaa}g&?@tHMhZvlzhaovi zpzS7HsxDW}4S&CRxR6+F-^(PEQpYdEJJS0Yry1tAcz^&W+r%jK!-elqtb#m&&Y@QP z`mwr8KkqZ}%HgO$2+x)*9pBRQr>nmPbza!dRc$Qj(e=?wN!;%)CN+Jr{N9^G(YL;o zFb=($l^eSFp<6oJ?d+}QpRtj%7<44jGw#JCEYnU)onkYU9|hy3-~dd)uAMx{%xKI>V^G$pzRoKu8hEo`VrLS}$y=+i|edKr>=Z}Wuj zX4a#b4>lC}hABo>@3QcYj{^-2B z;WnNY`-h8|le(*S)*jZb>i-|S-1+ryfgpcPuM+O@a%S;bd)K4$#j&v>nu{H~9N&nJ zF5U|+S0MI!&iWa*q&#Tfb+r0dNy7um`>8Wp2xr1?0!q)prbr^e_%5x_ zRAV@6jUGBbU(qrxRIP>&q!@CjqItHV1!x^?f z14swT6$#MC?moS#z&7(Q{z<}iKcxU+0Mclwej-1C8hAh|?>-khiJ~}PhNc0B@V)5A z3aJJh&X(5GLW{(*LT`I9IxQJ8a<~MDWj=BFVH})V%{ptMo~#!$Q@PV~5}0m}#)swn zt=s}^si~=7vEHXgR6Ff3MciogRNG47nH*0w2isW0rKq8g9HnZ;1L{l-wXVNrp|!{Q z`8zG62D^$2iQjE=l9$^6?=c?|F^^AqLV0r7lIZNv($6JUHe%rwVE6TxTL8jRG8Yb7 z!3j;hI-!W@+gWb`>nhstPyzE;gt5y;|6c-R<+>_LS6S*6x+stj1Ygf}lqY zB>_(#FvNF<(nOX0-7!(NwRcW8uIj6|?IU`@oVIbRg9cB`(Ha5Y1FI%V^{5DIVLl}k z@nmL;7ID)r001?BzLvUW0sfp}hJH8GS@Fw(tN)DJwq^SF8nagtNIbM4{4*>oo`&p4#JtBg?AH=iw9*nU34ZPH6#`?jT z#u6a#)+TYxN*pg^&0C(-;-sWY>S!S&rr*i?-adl%!9nSQ#q%Ql?uCgMp;x^~moQ|5 zL7l{6I7}#5?Oipp<0OGYK)Yy!}G6q0vHNx`?w)iVt!>oW@5fPcW{)T)yuC@9H0C_3(?+G z!0YcEO3V)k;=7+fC7%+#xoFAbeNthUSI+uohXR*eV?1j2xQF+OV1B@e638foy%u6& zFR9x`A0Fhga}!8x(nCoS=SHsR7%gwX(G!x8K?NEU%DbC9!hkJ}0Vv!0uf#@mNF2%v zL}(V<`V#?<5rIdy@^O$8I-)qzzeZBT-ueO;RvgUGI|B}(MLjh=h~6l(lej^{+&eeH zm3D4ZOP)w2KSESh@d5zPV{bHo9H)C-@r7{^Rn_*3okXb@(;K1|2j_iD$oF#aYF%*T zI|X`68^Rx&=<(y)e*!R28Uj~G64^wA zkaa7u@7O(xj;OhEq_7s41rBp~L)J56;*~fwj(DP|LtPgi zK0KHeyMt?yYC8L$6D1A~Kbk+a08s_i^2lfE(I^@^G(9@}P#-C3opZTTr|*b-Otq8k zET+%+s@tn7ao#1qn;Y6Kfx0}=VG=1NqY!Q&i*DcPsV=&{TpGAZ{7}o@>-tTY5?#-O zdaQDOsH)+Q${UL(bD!5R6Q;e25-{TY$)#0>HPRSnu6bd?Sve1 z1pE=Q+GM#m!bnyrOIbub^izZN>wLQFmrwYf9fiRN>RnL?%&Iwv`o5`-lNE>9POr!p z^}?2Y@dt_}7VfKrz;}-izf(q!W7nd{u<0kS#!BJ682Dm`{V4}I(R<&2G>9=RaO#D8 z-DPXmtzR35yJyxx?_s>I7rG1t8;QAp^KLWLS}de(%W<8Am{|^ZE55h9U`xV$Y=B3N zS4*m-a|Yf{;|+@8=t${#%ZmZ_{3=cUk;EQy!YnS4MwgaYy=S?RjQL0|Q#0Y1Jj9b9 z1h=td=o!ZJv(Y8CqTCbxA5L_qi|$cGTvM%NJ{=_hr%Xt zV|XHc`VJbK>0C$8bqXTlFJeA=mFNlHNeC(s4)_RQZ-B3+M<{u&HpXYWWa5n9L#Rdm zUnSK6oRZRAikCs!qx8X5@%ceJKgJU%U+JlW_gbGCDx>vBd%S=}^DpAN0iaVNV<*6C zm}xXIBq99C#_pc-je^1c_u-q-sI}@(5XA=YSFaWCO~y5*h>h^yS?r&bk+wYpck611 zx1?SM;~oGU8JOJC@5fUij)jc&_3Ug-*7!!#1K&~wFjxTH0$7Kt`-fG&Yt3?obzrmJp`mbSV zgh;A`Uy#<)lP<6V+i50>a-LQ`aJKOKVkr)e?UGBX7@>ZF?l=m;^ezJe**cI#Db8Wh zE?}SG1F)wjnVKT=I>b^cCk_GHgv>>!adc4FeWjVB-Gs#sr;9_x5fo3}?3NR_pi+iL z%TQP^HN|DBL*h`Ie}EoxS`CA=sf)vVC3+5iDrNZN+r2RJ;PAirx#xv%EV53KPsrbmKKuJ{9anOfK4{-z=bz444~1grUOlzn znu*>Y*ijE%K|l&3M8TVWsx+N{-relkNnT35nYP9?>kFg;V&Sy~_Ed%K;ms#x?AW>% z-ZWC{M8|Hb@k(yJgiBd-cgbhH7zz!MZ}F%)jmK&pm6Oa1AV0YN#9il7QXgJ-2SQJR zE#xtGbm%j!<3`GWX=Co1`TLy(K{RE8at;Sp)pK*j!=xi=FK(|U%PTeYhsPc|mR z*JJbMVyy(%W11J}L3m8AE-mkst1in-K>YC~)6Q}vE?meH;3|9lCcbVMudFfeJBg-v z+A)K6c0XAU*$chwRg5kA=`--_NnUwH33bR@N5&!36X(J$B#sq8C7?neP6VYYO$N}u zTQ~fX(4D*HmIpzmqU%Nd+m+^pN;R~oG(-h47+ysk^h@wN2)naVx6b_MW}KpA%*m^7 zLKi+H==xiURm~ZLt^}UQ=Jdfbbkv;A3)Qy7lUvzjk~V6l6o18fF%~l5mmx7u9yN>z zZpj~s`vAW7X}rG11V5GvO@x zHH1*wep%Azt6?Dxlk%H)iE83M2?c4hh~!O>?rboxJhe_bIpyJs;}eXXOafNtRbIMq z?L4A~H@WU8I9r<9Tc}b%;sUS zD^whQI_nOxS)!q8)hCE7h*lL3B)`rXi_#s_d2~S)3!i(gDWFFrlm_Qa=+FpILH!gQ z`>QIvmZriwWnW0obUJT;_4_Es{h`PI5UzFP=-Hdq@#vk%Yxnl~z4t3SH{tF$`kP~t zaT3*w>qEk!U*(4nubic-KIQb{+=9pYW zy|uu@$gr)O565CcW}QzFcpfH069B|mjfv?jw}w}qOtnDZD?A4!qj9->)amj%=3^ja zxOyl=y~7q<0uvup(~H~SZFv>+O~Ax1L=iE)TH3r6yAOL?!f@`jWgI3$K0Y&fOVNsw z)~b4?A3IRE>9iJ|sY@l-4%Ku}28XY0+}_D~Dk1E5kO&WYADEc(cZrC$1-WDwzQPbD zS}~pc>^Fxm$Wqo-z2V{_PgV`je@N3sn>Gxnvd{{xaejOXYBAk-cLU!-3N;sNG9*GM zF#Q8$y5@!AhmPcfvFCNc;&Px*v~?nZb*wS2kb5f^Ag|L}6gGOi0`xKbJ*87P=vIMaifS9X`w_` z=&c3PsMog_ECA~9eDl`nCutVY*?Un;BBL; z&7c9+`QcmNbaoSi5@Uo5fAhH~FO78Hq*T9UlMQ$;|F6RUa~KEh5n=z8!o0oO|=DwjulZ;a2#fKdCM_X~YzOb>7Vdg>Z;15s6v z&wjo7IWBJR(?QW=`TfTTgi&K3(NX6C#QLj{GQ}dJ)Z19%$%GrDMHm)XAqdCc0{L-? zUgYEwa&3sF$wqjLH^aPb=xbm3ISroIwlhBMjZs&5SmxNNuTjk&(s8P&2L`;{xN=(A z`f0uNR992gl`P|3lJ|3NKvC_%W|F5-#QlbPaw=b0pt-(U{$ihX{m;h;2UuLhA|=ob zS8s#RsN^xgqgV%PeEm#klbofGqVVI#c?5HSapF-EEYdPDnU4q>V)4|&Kwmv9c1I0- zGO(thnnfP%yZ}xcg(@Qab!^zu2%9)c{c8O6%su{ewpIwB>jlhv8Xw~1AuY1v!yCQ6 zfJ(q8^?GzKW^%}ty%v;+cb3I>n}UpsC+?~r+`=T)hTr$fv&>NXX19^3)f2kv62aKb zvaF0piPvTCzT{qk;Ha2Yr`HBFsbZ72r$)$jyl@`BP*2vgA4OA*^(igDqa-d+SsD;h z?*NaX1A4fbGFRf1atqsl1*d%}8$7EbN&Lt$UG2d}fKJt~m426G$lm+@{G=cmqM~s& zZ#}LL(;65L@ShSl3U!7&;{ao%mI{d!oURyiBqK_!;gY3hJ?)|7$S?&szciZye8;zb zUymKaY<6wT3I1flg7litykiz;o!HGY6m9G-BdSs^(j>0Wvxn}Kjk^C=tTKe0{%krZ z2X`n&cWJkz}aKGGp z^h6uMyuVEHtY%rM>F`FmAusN||N9cF$eD8i^C0Eq+D|52sS9;jMN)8g^>$&l@zBlY zro@oy%9TJUw{b$@icp2)8={vh1oZ8})z8c4|J0?H|ENo!q6!49Kt4Njd=i(j?o_|M zZi#xAw2!;@B3@5Ma~30@1y7xl^HF%m2z-= zfT5N=uFDE95z(baNi;WDMkid9;7HzMYWk^z>zAYN!G&xVBtfEz@)U0lj9qx1!7#P)MB;JM|UzcrTj66$OvNL()#-qi&A1OYb-7K-!Gxy zI^vvUJ9rP*R_M79yNNyUz7?kyYj+5bg0*$ic6{*no@yIVq*c?y|G}02qpF`I;Q3>T zy}j-WWiMJF)-k*Bf9*LYjCr{q`{`bPWv>zdRT<&cq-X%4@abE@v5k%OA(~D-OZgd7g|DIepF1lnEI_oElRWevthX*{=BhLVUP( zF7*8AeGQ-4!&YOKZ`H*!OTumkwnx^#>wh0{PgKMqyTEQg^3s#2$F3BzcP1It#5NW& z&?mh-Mr>(e z$=gk)22*_TMX-?vDhvJNkMr`WR(-4%?wYl_RTK45i5A}hg%BL!b{0r=Bu0{uf^llO+txV%o3I!jSYpwCFpR={A z69~7tZv|8)eRy($N4zy)1@jgMv_x+;eMh=?&{0-zW=aF)YAG&t{n>}Do3E93wc)US z!AMIp20ce{s&dN4=2v_WAKI;(AN2(Zi_^F6856z*;JvzqK!`B$z)CQou@z~{nsjKX z#EAc3Q#j$9#)NLJfod;uKR7ixmHNK?4B%<+Z)k!ue4l4W8>WVs-CwmmhdsQjeIO#Y zuuH1m-P+b3$u#un)3Y1wGS_CXeD;5(TPgMbnulioi6M1f|1l4Beq}nt_elurI$G* z&y{p8@`mRxTv9Z1^;|jld2>4}y%Mpu?x{Rvs;~GXCt6YEm_RN(u(6-eY%wd3Z|LK@ z{Xm$->iwN}ZDFNN>_ngwR=zFbo;*o|y&jNFQva84+wNGa^>SB7yH@+5Zzm2AhxA{I zfxq!g@7acVQ3_hpoHu|y3X&x1*@c$oyqh2#C;(bz79e!LyTzPN0x|08RGYjB(YBx!J-|8WYT19&Gy8u8+0+$Uxc>My>ng?Sc_1TRv&U((FqM6et-geJ*k5B<9Fd9Z#3 zRqG68EQXC8;Ld(h25Kmu-Q6^!g-&SF-q}~S>~|&m4UdKyT?{#gf=|Ud1T5|i_ z3mzAeRm6%!YesYbFzw+rlASs=Lp~ewZ{=U>V`Ymw$>TEU35C##eE8`+@)LlZVuCjX z1ak~kRA5q%@9ZY`2_rGmV65Az2SKP)@&g-O>UQ93UuVknt&(~jz0MY}dc^BT!hnOE zZWcwj@WVwjvk0#D7@S7>G7w6S2;kWFo(mL9ABcrhk%TGyuQ7@OJ$vu2_>psBDL_{evKM zYEi3@@Gn`Rz%IU;d*r?F@JHbXk0BA1_Chpy;OHSpRm+uD^Sf^pX5J*$t3S7WFyxdgec^KJ~Qfb>f&B zHE%Rli8Ib?CHKUTCeycR@%Zz8#D*_W!Owz|3-o%|yw%gx5Pq5Ga1c+frv9Dmj{u{Y zTdyq@oPo*fls8py0TLUtO&yKtkRnvTHokN<0`0nm#lc5SydMQgjm_TpBEY2Qx0YMwJC5 zNbnncc#_IpjK@AnGRL@tmW8hLfECT&b2#dQ{q7a_-aq68|DNjPgd6MPu(Jq=V!8leKC z@(=+S`%@m{i4?*BBX}WUB{b^A{Lbm$Frph!wegqV zMn{N`0VB%g;TjuRZLVG8!dEAM>W7|mk+E$gZgl>+4(R&p*%c9hQL?+(A?LuO$%nfeN<3Tu5|!06|TLPjr(VD_cmuU{d%?q;C`qAxW9UYZlji|_!8}h zUp}ha22I&0lv&Gg)iOR&i3q9rWAIJ!Q=DJ{`~Rr}q`J6Ci0MR%!V7hdS)U5Zs z^arP`KXV)%oz+2yKpksr=P?VABoWX_agtbdT7Bv5y$eL(T;e2hc^V$GqmyUJ6|k?W z`Fz$$i&_!4UoRwTc}y2iz?A6(Q17C@z?sLKa-rdbT$}l#&v=xqA~Za&M&bKJ+(=naMmO^*t2tmy(=rQP zf(~<)1AY8j4bzDdHG7YjpP)T-tdvHCEnvs#eCEgJLgjYTW1cOUJcNcGEG&X2mh(ysRV z1NM9WH2vC8`7|_t{Y5(pnPK&iu^3|jfgf;ds7RwrL)SyYgOoGUOwAt?!Cewb4fTiR zZZ&_VyK0P-jz4q|Wt_yhB(?4L3Jvu<2d0a|eKe(`t2hLnE7`X4yzk{$foX49-X2Vn zYUXKjYV$ze))(mL+Oq0%j-? zE~MwK_E~J|PDFTxIagJw9YgY;A^0wG&CXwn3ZI+#`xa=QuvywqRQ&&^8)W}+U)w_< zg9nw2ME+0!}>UbwIN(lxp&o`}CCZv*w#HY#SCaRFLEwy(7y&uGtBmo>mtO7OY) zXHJyhBDAVacL*j*B4w?xjM-qq*-jUGGcekz^iFw;-KwuvER z@EmpkH!ita(ml~3v&wUIwq`DE$54J;mU}{??rZIn*Skx*8>dicF$^>uIzXNGY}tX8 ziMa72%yh%ep(sVuJl!D)tJoaxycA$;svV*65W|@bBYoNPB#NL~1ue}^L>O&eMB%GU zGSwYRpr`C+S`M?f)aC+?-80@?EeE)eZ6e4KHsHjJXJ3qDWA;D9(T@ugx@;rhppVS# zx2Gg1aR>!P#;A9`5W{ejR-*4eNCs!-(wsbz>{KUy_}f{Ri@*RM9#UxbO6KKX>c88+ zehCs0Y$UA*Gi9$S#0jqc7#>!7P=u9EqLyEQ&*L% zki)}zC}2pkM6(nV>cmyO9@arz`khb#WL5IwZmSbHQi|AWU74qzC&op!g^L}lR19NS zx%daslc)`d`MEn2F*)auziMNN{LTRt+caxyw*4FQuNakt0e93RuqdUC@$|Rpoa=~z z0L@e01c?9tN8&Q1N9&h?*&lq#gjiGF-kcdNq-K-Oh*19U*pG+K1Z5sn_Zr$F-^5+V zh1a+^hgR6CMJ_ceg$ciDykP*c^f07^ZPh>QyVYWIPi?;YFLz^c$+T#v@p8-rCf8^$_j*tw|UG&EUuu z`0$huJ>s|G5GDaOI#sEer>LIWBxmf{>zj0s`UFv=GLhMOAYfTIDuA#>wRbE+3lyP}wc zH1}c2D^79bqjyfC;RDIl4);4Xe1Fd>r>#1KxK z8wxf7F~#N5Pwp~oT?-XqVs2}w6p{pJG|vafmKiWiT}-E+KV#3Ryct2#ytH}L8?pbP z;`>-&X(H9OBe4Ni*{6u3vKWqp8{g=1M|>c)f+>q`ksmx6Yq2a%tD#+++Gy-!d!Mdy z?wD|I+}57^Vg6qJ{7F z*Z&z5z%lbqD1K0+Nx(R!e+~E^!)WFkdDMelz8ak(B(l8yqya|RUZ{TcPaz{YUiP=I z?|jrXHFto2kdmaJ{p$M7hnBGj#V^UTG3mW@Z8p^o7n9CTI$z01uQs2|T!46VA)z;C>qA|_TJ@1XzX$I0S_Q=L$jQF!v-g=&>6 za31FN!Ph06-^OS`JZgK4V>X1CplzW-yjU#1C*bsO;3$%`z@c4|X;VQJ9L=~*jR)GG zV!TUe1Vq4yUsakpz{rkm#vxs7`Z^`5R&DkPJ@BZj7msTd!7U7~cN8MoW*3($nL0Pw zYw#K<4nF~eH#)Hq5Py<2)scG9Mvb!<6QnQC!FXgx5ZBM?!m~i1P=;Udb8TM_I)KB8 zppl0gg~b3=H4Im`DrSIjo%U#ivcsvb3CDwsoxBeAR`Tvj>D_%p`;1TCIJ0kGP~-t)mM@Yhd5 z#SN$Fv!v-;n@Mw#zcXN0{ihUmyXm4T1M(%)I>BDMZEIeAF<;&W~(fa|b^!w;^Sh0o65aVsqiA z<^gAGxx=LEVkVg(D_xUe=7|?Jl8@l}2c_4Ay{$nrmkiFkwSB@Z*Rw?L&W2MH_g#s( z3Y9p%_a^%pz67%5Hgz&|w4I9yt4A5GM*2j83&A2V&y0LaJ`00qYZ2J5rnvV$cO=nO zEhr$ThUu8WccscFm9yi1nz7zlQq~7&kUU5#-3cY`W*mDcq`LX@Nn1oiNf3W;_sAvE z8P}Pqdi-nr;p&5;jU83yEf}XsD;C@b2FshWX8PlgoGxPR&F;6G32Hiz2H!g=7LpFN zfcu1WK}!)z-KZC-8bte$x)kU_vOIB=eE-uG^Yoa)QK-dJcb=Z`B*DYiMF+D_7vdSd z`9{K`MjM~`3siIt;zy+-ZNtINlx21zcal@Ox+l%%dU2DX7j^Jg2(Ff= z$&X$9>+m5~nUUraH&9BK0@~Gu+&d8x|qGx{5-OMi7M0z93YStK}EpLr} z|7rO~&gk4?MaG3xk6Cgx?dV;aWleKy5ZC|{Ojs5z4+W2VwJd624V_uvnYPtz^Tdi2 z)84vBmnvH=PhYVp*{D3+*bu!7fzq3C6!!ItcF;=ilRgBZKgFLuZLxW!<+vf$!c9l_ z=?%NNh+Ew#%}JJZ_yokM`MSUIo?7r#4kz?r$AV6J15EL9=864W+{ zmiAh%yFvmq4WViM*1OekM}-6(W4{a0#6j_t1o@Xw%*sPYpiG&bvn|myyOjsQ{yR}w zK?lGz&fRy?CamqQA@it&>9hs^Sns~OnXNx=0&rJ?C)9Q1Bi5JNI-%|!Mh63~&VR0) zXv(O~x4ch2i&}T17!hjHOI+MAJ-yG%l_MB6V_s5IG8R_S;*AXr8jrx4n&ix$~M`5n+qBis$4nb(>lh{y;uT6(vGD zrk9Q(-dlm%;SYd$vDVNEa~nt|bJ+FXMRqa+P!DaSfiDDEenZdbozwFzz9sxBhq0z0 zk{*HV*4p6@5T|B;@-U3YaZ|7k9D+t`EDavfG$~S$)=gSm?GYDf;Z!H9s5mJ4C@Rp&WeS0u$snHsbS@*!g^KpAnbx z6F}&+;IV}Ulw649%CCHxcY2u(^%l3h_xGk|k9 zB7D)0-kp7tEj(vO2vcl!GrhW+mIHCob)VKeM=;LfTz20g(J`~x456!HWd|7T*pgGh zOH~{sz(!;xLEel8^zp)iT%i*On%?`pJ=2>Vx0%OTD8uHF&0m{ux_+i{@w6mD6qn8H zz2I-KF@?XOcxh?T^d+z?C5*jpH9-W3xPb6V!pFS*b|n;>LAYD-}K6 zox2(j-IF~6F~SW@P)3kZn1?nPR@imfB~Q++5U6q>g7Y-Gmg`U>5#6~OhUa_qGg(j4 z9jc<3s& zVc;St{urth1(UyB(rj`-1fQ@Eb@`C65=OKXlL%IsG3F89p5{4~Td2QRGV?MBGrUrr zl^8-DeB4WdXcLbZCBGIe?)Oa<(YbBkCOSqtN>A`CK!69mc-`xAj3pvJELa($Q(9;m zh=4e#C&q;f*3o%kVP5=q!c!fLzmAYOz7Y)S)s$Q;@;61;kdp~^t9-9v6ZN6u}()Z88AVgYkZhZ5oPuWhWH2h05%8K`M63-(~ zxNM4VXxozRmxZ*7SIn%#y($UHP|q8d{jvN7

0ybUymw)5FscP zZr3HVbTS=WK?jI;jf{|`($9H*@>UG_nBWv7eA<4vD)jBd*gK2K2}l}C@K_NPNrXEX z0$sBgE>VjTsK&T;9aBr99k^APPvWinZsUXNdH06S{WkEcDq zgMiPt`H^}tZlFcO1k*AS zv8(yiL*oNYa*dl!W=rBjc0;N)S$Hzrx2?lmxjWZbC2fn=#*-?UcGY*BMkF=r)*VhxKT%)OlWsW zUTB~Nmg^x%(o<0|>CRylS{%inD>P>(wDAjE-a_IL)(w42UDLNo6ZdfZS^3q!g&oR0L%VEJwAF4TwA|>( z1w1~b5KtF=UsmC%9kI<}Gxo49PFNKN$vIGi_0S2%*@T3X%UqV>NB~PP#OF&l3D>0f zi-);&Ye<0GxA~Y1!LaANb@R<4vB@Fsm$RvsL=Wv(WnlV#mcIkTLLeirLPxz(&W!~V zik)y3^r3|+d?vaurYI-y4JQJaec9lP+=Rp6J~!>oMH{8V3y_^;_NzOzd^nSZ*oLQQ zcIJHreNbkI$JWN?r^ajjNuvwD<$a;HK^HoJ1Gz=EXY_=g3aT>(^;6Vx-s!3Hua|=MLV02d5&Urv1f+DAR$fjAK zd4z3)1(~un8}Fv1W9a8+p2`qVkt`Hr4ZX%+J|>dqu}0e9&0cDjEgVl@V^AA9aSG}o z&h>ng=;(8^x1u%o7mQ;L4Fq;GKN#JcSTw`-SmU;}+YC);<^9v050w&tQQtTFmt3w=hcU=5Gz9!Dl=qX6K_>6Ibm zxw1NWQC~A{3L#K<6GMN58JalvZ2ltzaYFQ~eo)P`-GxrUG>E%Vtkr&6ythlDgY9WJ zD&qD`EkDy_6A?*}ja=3rrwH(89s(_@E`~4?bH6FRcck)N15;B0YbA=O`-4 zrzNBhfwJClHMhNov_m&99WDJ(FAMaM>>aRoKNF|%e8)tYwJ3c7THM@7Z#ce@rV+g!miDacZZ4-ui@ayr9&AgOfKh49ynZ2W61&0MURQ(}?uTcxm7d#guEiN#r| z8WV->kD7CFgS(-WiEC1NFR}OoHL7>!!=1Lx&{xxX7azUz2 zEv6WJ)-#}y+U(E!xonPReweV9Rqp9UG8wkj9?Y(vQ2)PTs-vR?D*x!aa1>$`@LulX(WvFTx7$Q}}_U>TWne|87)nnUtXQt~(#|hF`@@E~7 zOY2}nVOgU0GPGq%IW6x`vsy$%9rqA0h<2j}(!5Le zzsJ@O^D>{e7+u<4X^WOKITYhumrv&t#d5vGWT{ON(_7%P0ZinSM^CIBw#4*Su}}Pa z1e0T#yAIg{6bN=2VXIch2r6&C8`{~gjTc1?k_b1M0}kC>)eKPdVN`~A=Q_SEgxoQj zuX%ADjD<;(#lNBPoD0V~gyLT##x5&0J9RZEnM4Gsye7G!_fjJ98~L`3v;Bj&pJ|eV zs^~!Zv`U&QY)2LZc2aI3G<%xoQp&O^lI(ppIgsdxk8Hmu3jngEDA`0Lh|U+)6fiYb zw^N<XP`prNsRWmDTE{#KcWP(t>i`6o~SBCVg<{;LYNmLSzuH z+imKkoeCRZLK!6}^RdTI)?muX=`U?Hcyjsw?p=Zlh>QOY$8K|BY`;=pY)uT0-oTeQ zZH#ILDRhJkpp-@s7?BE(7r*{SIMiOnTR`q zQEyln%dxTRL$(7t`A>5k4dT?jgr@{{6q4_7LI&qRlk!h9mwDOKp^yxv?-|lfkz6J+ z859x^Gt%%GEo4&^fIiYaLZC1G)efFM&{uciah@w@9}?E#(hfIU&?6@q8)BtON`b>5 z#AN}aTLz@70Nx$Jqb%oC2f6MLkObH|EIK{Zu*(_(00Aq}b-%{7jkT^rpk1 zL6YF9k;YU0rpGh~?z95qGJlEa-%Af^+?A?11M#edrIz5tr?^uP zLTLH%PYGYj&S%0kt)s^VkR^1>*0167H2L)WSMONcD^9QL`Lp*{b4>xPV#yU(#Fi)25HBn1BcIkwE(jlri4MHAjIwBQS>(M z7OM!dLPyfpJs?Gc&u0&zbExII+zw-()}9{_Ab8+hCM8f|qiEsAm`ay|Pm1Qs^KcH}lms7b=U2}w{J=TS!7^7qH^JQtBv}aPp`q;07i{xB(?Qp}_WTSyZt^$l z-eDuGQ+w*xxQvJ|poXsw_sL*@tM)#0G2cR%nM@^au;ZTer&0wMlwh& zlOvYbiaDJu3=e$Sio2vclpJE3p8f8{nOF~#Mk}r&8=^GURCx#2`PdclbmIVsk9#Hx zRU{J~IxEH;(^#a|-mQ>;DoJ)AX@(HOZu@)yHdktPF76VCE{YXe}KC#hp` z|Dd7^Fbt%Wn8F&v_|9KQcC}Z*%Tyw?oySl*w2#_Ah?ci^gsPs$)b#k)2GEKks%JdZ zTEI*3>Hdd=pefv^am|pprl>>Pv=F|&h$7kEswd6Cbb6UJfmZ`FJ-E7PO=P7I!a}q^w;+=nUgAwc;h(UKHFiLwD5y4UIS@0D43%hCOu4T#TT{G zA7qi9m&o`AT$7DNyK!=~eDqM6jgroTzPZJbzifm&rtTe{*MZlh_9BH)^ix8*LqlKL zq#xkBz?~8NgTdkG_;d%LI(-iJE_r{07y_L#1W)G z@#F-3w`w0M=BqEZm9P9Ad{m$cuv`h_v$+%q?&|lh9nqkz5q^K9xo35CwZ-EV-KJCZ zdfBTJw6|yZn+~?~71K5y?gu0Yakl?lW&fZ0a#2$q`Ig$8yDe0Ivc#`r&(ye4-K1qD zmPq^%+^bS{Y^tfz-GrY6h(5URP;{H7+YC)hW_hJ{5we_Ql5Zmh>f<{Uv#jbXrWc&Umo6Md%81T-fW_X zr9P6Dj0Xs>4{?osjN9T*@K&Dr=GZTXqVmG%t*k}l`gW#o;xj5?DqKNaIWM@FUl&-s z%A!$|-Ye(N&QU_D4SY~i#qyNum;RcH4NcqiZxLVLp(SoVQK*tI?x+f$6ZGjfSE`a# z#OmkH7_&XUM)qQ>L738yd$Z#vSRdElA`x3XkT?;M3GO1sDZuH znDh2{ebdR|bZAJO_bd>>i9`S=8^YkhU8dVIJO?O#AJtuFQ0>aBue0IV6SS;0#)QaF zKAb>sThg6Dq)(rHTBu{m*zF@eyZ)qxodt5`R{R}0)TYX&s_g5553mk>=Y*<8WymfO zR~)1!|8S}nZcV+}fCs_FIh@UC<=Q}&3&OvX_une8o~ci@k;+zqlVeU_m54VzJ5?JR zd?kS3H}oz*YtRG{&T~R?Pj?Je-#$(`%zPAKrv1Gs;LB@tOauyO;jZORDV=4@KhJjy zaQ(La`2rXK3m(0|_T3>~|KtuXJmNnn;9{~yvyx$-2-HAa%MH5~>CmR zQ^2Y(Bqm2y&XyF*3!WrStpMth5|OsI$zD9NHd7qI?|nX{l1;e|OCn8v#I?uS*HYop zGF(Jp=j75=@bg@{?z*dc!K1%VAzq3A%h~xWoq53#%e(CSA-trn22wp7O*Yk(9GQ@P z{tw68QojkqfMtY3XhhVWX$!dO9x?Cf9j2@zH-I(c6CHsku2U|RtbDJJvC z_EvyuHG_;g&#za?{So>VuIAlPtr6d^b}T>fS4aY<5`wq08jt!z&Y}@C#}@YXNt{7@ zoos>BK6AM6cOA$=u6t&7~s&oy!H>Fy?6q_ zR$4!HfuogHIwicANx9HDu}UVY*w9xoxdLg*O`KdSyKgdno=StGVDH`7x@fsfcI1wIJIV`76`GBJ{s~Q zpMVNg;(M=XoiOZPLptZL8#_DZnTJX(^r-Ql%psbY`LFw2#dcKo-7j;PVrgWEr?uEa z=HoqN66L{GN38+W`m@RNFE1Y?+182XdSx5n4|BzjhNc&0eyY`(4NxNQ<}GU9Zy?qt zE&X4{<3Z8oJ zQg|K{4pzx#`kRVuE!nLa%lwBOx>eHF><-Nan%7+r4eIv)m7u2n)v&#Il&$k%>94>2 z*VbH(33VG^?>V2j?r;wre@}#JScxHBh@*5A6W%3Y&|YWk2NS~ySWouSc- z$aDIM=l6@R3nylOKku?N2`$=%77sgk-r_&tfR<+^Z!{XwoWDB6eE#A?XgzU$Lbz4B z<-CDlP8={00_oM2nKD&w6IhJB2`jU)Br+HS3V+uPxv(XlqZouU>QazVkJz)r%Hj)p zD3It2mYeAAoH+DKeaZbyH+y4Ib(ibudm_avB=#&(oG;!U<#LKGFvr86 zCx^rhX`UoTpawCP?c1XRVX%D^FXQ~^H+SAeo&K`cfSW%98{Lp=r_wpd)lUQvnGX7~ z6g3zyiMkURN*@0B?oX1ITWZF^0g5bybSO8hc1=o7<}eR`LAaqq)mm^y>W>85Pc{&` z$;a!kh=J3sLpHx#H?;IVSZ3CcXD&a7W|o;XxXdH%+?+%HCQezh(7Hn|9T)B@?59d- zFWjpf069CNEiJ#eG#@`+`A~hfJ09YdxU^8)DsZxJU~#sqPU3Bm;KVs3xF-L<;cgfg zmznRv@J-6qB{yu*9-%?N9&75{8My`P>p6u_7ODvL4U&?Y5VA7e+q4#NSqpt*qWYqk zcw>Ra6%?|}>r*g!xer%kntBlbmo^y}Hh)YX-^q6_jY)%GV=-0JE|xai>AUp_=j?A_ZnfIqH% zHX@9#L&5??Y?5S#sNacvGO)p0i~|l^@+mk=yY{(#yI~YX>fNV zd*@J4>9Wj14bO6(C;|Iyb>dG9oun3$||UD*x%n1<|75Yw%X%4*y{>m6J$t)Z*4iE_Bq z(}vfeQ~rh4OJAcY8b6r?A}0KVqBA)_0$<9V;golLmpQ}q;Jdnw>~*#?dfe{oR(JOx zWgnR*-eo5}i_3gjygt(0tfxM=-VK&fsZ{lR?`ex11h!hwi(IsDSC^2z)?W*$m^NYzxWPP`te_%{1-g+xI6N z0~BRb(1{vjEp(sOa~o$Y2BER3j)aA0ZnTD{TcgmN$3MkX4~naGzH~{{v!nL~3g*n5 z0#!^tc5r@g!Q#T$WqmfazB!SybAq(2yaN@4>J5Ge8gwB4&tX&9f}MZ{T(*wG|Hn{L z6XDM`J~nw|QU8tVb@5A4lL!N6qZFZUl1kWPa*N%uB~O01G?)ojUS6^B#p)C1WK~$r49>|VbW-A zzIf1TP|TIExU5i{R!Q}DG#1c06EV{UDCd?hFQqb$`O9|#Zlj0ln6;uWz+QCy#^IB> zFWr-%B_WcA6Dx#y!u(9mCUZ$j^bx+)^v=FCr);1I{4ib+ne*DIDcBCukg zpa*$Uv69YT9^5%<5vNMT=&~1#hUP#eyulBZ^oHfWnLA0LI3rZ#?#wQMV8L$1Sus7n ztjl-DTXzsORG=uK;XBX@BrK$%jLpxXuE|&LJdjS{ao|zFdeL5K164F+*NJ8L4RZc~ zR$%3*VbAPOt@s6=fzU4Vk%0-jTyb0X7At$~5k}2+6rDy{ZH|zZ1F09Abg$oPxaHE2 z7sv@2+ThqFGksqpbBeUx2Criy@F8^ru+h}Z^5Dk1nQTRUD&5efO)k76`y*~Vk2@Z{ z=7EFrX+77YY+oMcdVHcTyCSuFc>k3qWNAm{KG=7)lf}O*g1pwec6Ba*IeQ|}_MnHx zUkf2H3?6)+agvxLFdM*d=gsc60)LtfPBea5Cb`0r=w1&U30Xlh@05xi^+v$edjboYXbMB1md~~pv&D2_@m=$l3v3!Pi=5-%4xdzzWOmJVfB}+7 zAKmP!D{9`B5OSnD_Oz|VW%Li`pLxjpn+quX-?#w2iV__y=5v?O0JAjm6PSMl54^(q z&L$|=Z_o!cN1y&Gi!(1tGxpDp`#19XexK_z~-0vl_IuiOakL`Xs=je7u z8Dj+2>iwFk(F-G5sHkP=B1wJVEXs1EnW?W07CGUFr}gVrys`7oIvaQ02pc;nBaU`s z&MgM*L-8IdN?I(xmBb`%s~C%{pU8Y{FaAvOmE{!DM|WuWr>}X2udj0ydC~P8)2+Pq zn=c(H^{60Tu>)t@#$E&gH@Ia2YXeWLZ%iR?UZu# zPV8*tsK##VC=2<)tnai5*O2&L@O^#A>WKkDc$quQY*51{CF;+-@pWe4f)#2@DK~51H}lAzk`&4y!+*_ z>z0SE5#JYgk`(oSQpQHRP06`_JZ{B>9E5x{s!M*l(R*Xe^@3g8$hnm`aOB~~W&b|u zZ>a9yi#ikjY)1xso31QL6c2ERcDfNT_P1e>Z<5B?V~wpJv8AI6B&9Xe?)7ujkqF5e z;_{vfO~0s{+TUA)z3Hq@Z-(F7{y^;tpA(7~thYQO5cX!7k){f`%OPs~)sE<3Inf(J z8n^|dHRs4DC@6T~ei>O4tmI9bpOhb{lHNI^ic*lnt zA4B6W-t(2qgi~TzbR-ODI>>yRm5mGnybOz)9@c$|#)af@mFd4Ax+605q>$xMoj3zd zLnn{`ryg&!MAWStEk+X&%b0CtCclL!!d+zVcwy&(`636WHzF3YC}y(8iTGhtg|7{d z@;tO%y1DD+-`_jz+u?GxUh4ZGR&Jnf6t5p|{Ns6}UhF(E<8}wP*;mD2TCoLSUT2rj zVs!D!PuLY~)irfkir~IXzMvGkjRvEMFsNFO<|R%SMP+%Y_9mes_nnL8Vw^YJMMT_H z_fw)^&W0_wR^T>h->;?Ad3F-u1ePB*JrZ_6onajRC8Gb)MgXWo^Ojon&f@0bRJeKV zKP2P#_5#?I1n3FLyPo`g0@&(3kpo-H2>9wiD0n6KD-tJ!9T%S+ew_Ygb42=OZtE8E zOPag6Sy_2mn8-e+;4R(?>jF6hWV|}E8{yU5dY-`$xjNX)w9-f$i=zy8|CA#NDn6jU zk0QD^P8#{;I=K-8Rq{`T>Py0HFAK>{ZutG!KjBKNs&QlPbaGT#`~Cr&U6JTCl8R5o z5VW_FDVTe@A{D=8$DOQL#kM=}oI$xZe?>5IV2-3^G|xux+pq)D8=b$cdgByDZuH&)}edXNpl?B;&5;tNR?A)+#XaJ!m>hE*aNMwwt-wOD>Q>? zVQ*3-1DT$`IJT2H?0HRzf|A}3Opo;i*dNyAzY};uLbS^rb|6-p9KJddO^B{3+#aZ4 zE86f`l*1gKTu<{1)b!8y=!G;E@(5e#n#5plW+B#_pQ0WKo9iWvR5baGIsO@p+bWtbwHuAr=!6}!Fyx%dphh$2apZ;anWGQ`sOO0Vkj1$b!(ZyIq_5>ycIS8T`9A@ti^ziX6scDu z^W_a-34IX>1uWTA(Hg zrL_p}6{lbk&NgnbgM7LFX>}~lJQJT?qKInc10#OmZX7L7WCSF*%p&HiRm5uk zW={xL^HL6^VNQB6+inYWPOFrF7fX#GdbO)&{904y$48S+yKa#ni=Pw~(v8Ar_H3HN zDtM)xA8F>}&WxKB$i)^8zw?N^4=MhJ+iHs!&>*o{sdj+a`%v%;w;N=fwz~HKvmF!c znAGpva%b9n$m+@ra^TLk38GcM-;_0$fRow`cqpa}sYxMTIb4sLuPm6!#C6bvfe|%- znCrg=pPvcewd7zS+{*W&1M0uo*#rpzht91VN^pgzOyxEsjM z(-yX~e(5wR@^!$6{DbjGwn_d+TIZj^wU)1`E9nL8d(?=`7k$=oEj8#f>qn4)?arro zJ+7h5NV_R|8F$Ml{o*2Nr1#6q&y$^AbW|)O&4b0`a6D-?xeTd_?SgCV-{OLvHB%}`FY2aNymkNzXtR8R@Y|7-5 zmrLhm9ZoaZxW6Vv%ZHEZp7dd+#1HWB*lwK}W7*VaW8tq84G-p{R)suOs;K6o$C|Qv z3Ul5F=h^QR9^Vd5LFF8tBx-x{ExybVWX$3X4iD1a`K*w|>(P`IiVT7)Kdzc?{`kg*j0Io2&0~`| zY?Y2)oLmsdWYtEH;=P#@-HmzbVUtO1U5LuaV9*Bt*su6e^A%~Awe?@mD&TuGPCIX{ zu%i$<*zk?T=9GAAyg2XWw`PJ6R#IDAxOm)JMkuo{B+}5i^pY;-^an{Gr)c&Zv!c?s zmD|{<6{ySyPSY|NsC%@LNdnw?u_wjWVeA`0g;OJC@=O1)VW0$G0(AS5=~f!t4URQI zl^x3uFBJ$$H=bB2S@wf?yR8=iBO)4Xrc2gfXR5efg!6%+wdXXGwflw0EZdEX zLlH0TxiILII1gMSPbgk^)(Fw$FB*I4XN}q{{3ZU#7WrU;<*Z2b7CT0T!8bP7&nwpo z$w8!e|AX*rRuu2{sD9)=JAzRkQzBet&*ndOEbBBmkihHFt|u{k*-T3&YfJTi9Fp=| z(NZsiGfr__$XZnPYJb_c0HxKCjgtcqD$_u>DAXV=!L%U2Z*P z4klQK4N?V}fcQG}w0S%p4(4Y@49h5|iv^%rOO*wh`e^))))>?=F}9&a`=AzP8q@uO zvgJe%ajo&75y3}n3kp~Xsm*PmA;I=1RXis5o+>sIjmI*6zWioMGH%pL9Csed>puG6(+w542iF#lw=Xd!x)nJd{wDt6<>;=R{Gf-8{X zk^O7x_aEnK3d&6_rVBj=TIPp>UoQqr^CU=R-eLI9UHog<5<;tQu1c9s{$XDK6>+QT(KK~_+Wq$i7k_Er*U(irtxskvdi76lAFPbzAU9@JP3#~cK*)SVZ%#c zBg}B$9Y;E?p+a7U7v5pdCrE%FMN6e%c94grqBIZpzFFh68wzjybV^*ZG zd&8LrnIrw94WCdjabVqpHf{~r66WS0==thm=BfFl>$u9Cy+5EYrUO0C)@##Vmdm(F`O!5DVU2>u#-%p163cq7zkeV@>AQDHqL z!WnK$T~xxpI{FYKgN!M3Kdg~ef3G6Zx@Z;kj(buIYW}da-pVk!i-Th=%`B?=Kb8V_; zPUz-=N3TVrN%wUMDdEc(;?G6u8o773%_N-qy_r%5E8e$MjQ3NaY4@R3pAbnMAdL1@ zjfZ8{lt%lHTeCk`!96V)UD|0sl{Rpv1`LI-nBp&XAb0<7Q2wtg$|DUMIEWCeyI%jN zVCEkW1@xF51C`6Iw-cDJd6W#ic^*NNJb>T74Is~X+myFWxJBGp>pzhGxF} zqfI-rlfxzW(2~1NKC{G=$zx`9P6~Vi5b5u4#wQRHPCPX~iYgn(rMvj4->>*R3X>?^ zcu6D;^s61d0RQdM8rcktmrjT}}?3q!8uK(n7mt%kW2>@MqRWQM^^eT1yF~y8REHYj#P2N*o*vd#->N#_Yf|ACNhp$;GOUEH%k>= z8#@#~E_)@6il<+>7srhIM7ED{C6Tas-2qQ{=vDc5aTlxHR3P|1oi@FcV=nmbvcse< zG25Q4n?&&ZM5T-JHE;VAnrR|)>ncn6x(&?lRlj&p80-)vAY?E1jenx&O@K>6QuaW0 zR-t%t>=y?++aeT5^NUk=Ijs7sAmbxNgJ|bTTLL(xl)^&W=p^UfYBHGYkKD$IbSx*U z1Fe^u{J{C|ukHgYDPi{1H1qG$u6Xt#>BIKAY0vu_aoLbw^@7zjDaiiBCC(1L$1!yC z^6gLGSaJU&5H6+cXr-;-e%m{?|GhF*f_b?Xk-Md_BREOFH3TMSBkkSjfW$qr_qhrs z_s0yfB)v^vxR-2+@>SkD?K#z?f7?^#lw@Vsl;F|nUO&=a7`VIzbN`Z1r1eMm zf$7aZSI&Nj0d@W#Qav>rYOhu`NWL^_B!AoHVj2?q#= zTuCR1^=SW^`<;2y|N3F2nHNf179NGQ0sm2S5UCdE>IOr7k5^!Z)*2J#RYZAt|Ml)F zx72jBC|`qP#Ps7F*jg&ab-SGMna^HVW$$?KpHEx)64)BMA6U)#g^FxlD4^3^tCu|| zm@{uaLCR~?{a`#T-bwrK-ub;>KM-Iae;RWc`_Go*QnGrlw0EkqfB*gbx-86fHGva= ze(TBfJile^$}3K$3*t+t-AGDzQy{9IcYdS#pD%2tC%ZA_p1Dl^8uG6S;}evLehqYe zp7AqeC4Ka&)?VjLocJ`6Ki=52gE{eoqnmaO!B)TRIWkeF!srpxXeVtxA`0{}c| zKdCV74aL4>kuf*xJ};Av{lGelze_(Sh_xQMOi+TQcgC&H4vpLAKlx)L{v+l8DCvLh h`2Sz;*_ diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/assets/logo_square.svg b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/assets/logo_square.svg new file mode 100644 index 00000000..a82fa0ae --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/assets/logo_square.svg @@ -0,0 +1 @@ + diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/bench_attention_sink/bench_attention_sink_triton.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/bench_attention_sink/bench_attention_sink_triton.py new file mode 100644 index 00000000..21bc2a59 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/bench_attention_sink/bench_attention_sink_triton.py @@ -0,0 +1,250 @@ +import argparse + +import torch +import triton + +from sglang.srt.layers.attention.triton_ops.decode_attention import ( + decode_attention_fwd_grouped, +) +from sglang.srt.layers.attention.triton_ops.extend_attention import extend_attention_fwd + +# gpt oss +head_num = 64 +head_dim = 64 +head_kv_num = 8 + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["S"], # sequence length on x-axis + x_vals=[128, 256, 512, 1024, 2048, 4096], + x_log=True, + line_arg="B", # batch size as different lines + line_vals=[1, 8, 32, 128], + line_names=["B=1", "B=8", "B=32", "B=128"], + styles=[ + ("blue", "-"), + ("green", "-"), + ("red", "-"), + ("cyan", "-"), + ], + ylabel="TFLOPS", + plot_name="attention-sink-triton-decode", + args={}, + ) +) +def benchmark_decode(B, S, H_Q, H_KV, D): + D_V = D + dtype = torch.bfloat16 + seq_len = S + total_tokens = B * seq_len + device = torch.device("cuda") + sm_scale = 1.0 / (D**0.5) + max_kv_splits = 8 + num_kv_splits = torch.full((B,), 4, dtype=torch.int32, device="cuda") + + # q represents the new token being generated, one per batch + q = torch.randn(B, H_Q, D, dtype=dtype, device="cuda") + + # k_buffer and v_buffer represent all previous tokens + k_buffer = torch.randn(total_tokens, H_KV, D, dtype=dtype, device="cuda") + v_buffer = torch.randn(total_tokens, H_KV, D, dtype=dtype, device="cuda") + + o = torch.zeros(B, H_Q, D_V, dtype=dtype, device="cuda") + + b_seq_len = torch.full((B,), seq_len, device="cuda") + + kv_indptr = torch.zeros((B + 1,), dtype=torch.int32, device="cuda") + kv_indptr[1 : B + 1] = torch.cumsum(b_seq_len, dim=0) + kv_indices = torch.arange(total_tokens, device="cuda") + + attn_logits1 = torch.empty( + (B, H_Q, max_kv_splits, D_V), + dtype=torch.float32, + device="cuda", + ) + attn_lse1 = torch.empty( + (B, H_Q, max_kv_splits, D_V), + dtype=torch.float32, + device="cuda", + ) + sink = torch.randn(H_Q, device=device, dtype=torch.float32) + + # warmup + for _ in range(5): + decode_attention_fwd_grouped( + q, + k_buffer, + v_buffer, + o, + kv_indptr, + kv_indices, + attn_logits1, + attn_lse1, + num_kv_splits, + max_kv_splits, + sm_scale, + logit_cap=0.0, + sinks=sink, + ) + + # benchmark + run_step = 500 + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + start_event.record() + for _ in range(run_step): + decode_attention_fwd_grouped( + q, + k_buffer, + v_buffer, + o, + kv_indptr, + kv_indices, + attn_logits1, + attn_lse1, + num_kv_splits, + max_kv_splits, + sm_scale, + logit_cap=0.0, + sinks=sink, + ) + end_event.record() + end_event.synchronize() + torch.cuda.synchronize() + ms = start_event.elapsed_time(end_event) / run_step + tflops = lambda ms: (2 * B * S * H_Q * D) * 1e-9 / ms # must be causal + return tflops(ms) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["S"], # sequence length on x-axis + x_vals=[128, 256, 512, 1024, 2048, 4096], + x_log=True, + line_arg="B", # batch size as different lines + line_vals=[1, 8, 32, 128], + line_names=["B=1", "B=8", "B=32", "B=128"], + styles=[ + ("blue", "-"), + ("green", "-"), + ("red", "-"), + ("cyan", "-"), + ], + ylabel="TFLOPS", + plot_name="attention-sink-triton-extend", + args={}, + ) +) +def benchmark_extend(B, S, H_Q, H_KV, D): + # S here represents N_CTX from the test + dtype = torch.bfloat16 + device = "cuda" + + # Split S into prefix and extend lengths + prefill_len = S // 2 # Similar to test's N_CTX // 2 + extend_len = S // 4 # Make extend length smaller than prefix + + # Calculate total tokens and extend tokens + total_extend_tokens = B * extend_len + total_prefix_tokens = B * prefill_len + + # Create query, key, value tensors for extension + q_extend = torch.randn(total_extend_tokens, H_Q, D, dtype=dtype, device=device) + k_extend = torch.randn(total_extend_tokens, H_KV, D, dtype=dtype, device=device) + v_extend = torch.randn(total_extend_tokens, H_KV, D, dtype=dtype, device=device) + o_extend = torch.empty_like(q_extend) + + # Create key-value buffers for prefix + k_buffer = torch.randn(total_prefix_tokens, H_KV, D, dtype=dtype, device=device) + v_buffer = torch.randn(total_prefix_tokens, H_KV, D, dtype=dtype, device=device) + + # Create index pointers + qo_indptr = torch.arange(0, (B + 1) * extend_len, extend_len, device=device).to( + torch.int32 + ) + kv_indptr = torch.arange(0, (B + 1) * prefill_len, prefill_len, device=device).to( + torch.int32 + ) + kv_indices = torch.arange(0, total_prefix_tokens, device=device).to(torch.int32) + + sm_scale = 1.0 / (D**0.5) + # sliding_window = 128 # From GPT-OSS config, skip for now + sliding_window = -1 + + sink = torch.randn(H_Q, device=device, dtype=torch.float32) + + # warmup + for _ in range(5): + extend_attention_fwd( + q_extend, + k_extend, + v_extend, + o_extend, + k_buffer, + v_buffer, + qo_indptr, + kv_indptr, + kv_indices, + custom_mask=None, + is_causal=True, + mask_indptr=None, + max_len_extend=extend_len, + sm_scale=sm_scale, + sliding_window_size=sliding_window, + sinks=sink, + ) + + # benchmark + run_step = 500 + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + start_event.record() + for _ in range(run_step): + extend_attention_fwd( + q_extend, + k_extend, + v_extend, + o_extend, + k_buffer, + v_buffer, + qo_indptr, + kv_indptr, + kv_indices, + custom_mask=None, + is_causal=True, + mask_indptr=None, + max_len_extend=extend_len, + sm_scale=sm_scale, + sliding_window_size=sliding_window, + sinks=sink, + ) + end_event.record() + end_event.synchronize() + torch.cuda.synchronize() + ms = start_event.elapsed_time(end_event) / run_step + + # FLOPS calculation: each attention operation requires 2 multiplications per element + total_flops = 2 * total_extend_tokens * H_Q * (prefill_len + extend_len / 2) * D + tflops = lambda ms: total_flops * 1e-12 / (ms * 1e-3) # convert to TFLOPS + return tflops(ms) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--bench", type=str, default="all", help="all, extend, decode") + args = parser.parse_args() + + kwargs = { + "H_Q": head_num, + "H_KV": head_kv_num, + "D": head_dim, + } + + if args.bench in ["all", "decode"]: + benchmark_decode.run(print_data=True, show_plots=False, **kwargs) + + if args.bench in ["all", "extend"]: + benchmark_extend.run(print_data=True, show_plots=False, **kwargs) + + print("Benchmark finished!") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/bench_in_batch_prefix/bench_in_batch_prefix.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/bench_in_batch_prefix/bench_in_batch_prefix.py new file mode 100644 index 00000000..28209711 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/bench_in_batch_prefix/bench_in_batch_prefix.py @@ -0,0 +1,130 @@ +# Benchmark with lots of common prefixes. Used to benchmark prefix caching performance. +# +# Launch a server: +# python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 --log-level-http warning + +import random +import string +import time + +from tqdm import tqdm +from transformers import AutoTokenizer + +import sglang as sgl +from sglang import set_default_backend +from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint + + +def generate_random_string(token_length: int) -> str: + random_string = "".join( + random.choices(string.ascii_letters + string.digits, k=token_length * 100) + ) + tokenized_output = tokenizer.encode(random_string, add_special_tokens=False)[ + :token_length + ] + + if len(tokenized_output) < token_length: + tokenized_output = tokenized_output + [tokenizer.pad_token_id] * ( + token_length - len(tokenized_output) + ) + + decoded_string = tokenizer.decode(tokenized_output, skip_special_tokens=False) + return decoded_string + + +def generate_unique_prefix(base_text, index): + return str(index) + base_text[len(str(index)) :] + + +@sgl.function +def text_qa(s, question, gen_len): + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n", temperature=0, max_tokens=gen_len) + + +def prepare_prompts(num_prefix, num_samples_per_prefix, prefix_length, suffix_length): + base_prefix = generate_random_string(prefix_length) + + tot_input_len = 0 + all_prompts = [] + for i in tqdm(range(num_prefix), desc="prepare prompts"): + unique_prefix = generate_unique_prefix(base_prefix, i) + prompt_list = [] + for j in range(num_samples_per_prefix): + suffix = generate_random_string(suffix_length) + prompt = unique_prefix + suffix + prompt_list.append(prompt) + tot_input_len += len(tokenizer.encode(prompt)) + all_prompts.append(prompt_list) + return all_prompts, tot_input_len + + +def test_batch_by_batch(all_prompts, gen_len): + backend.flush_cache() + + tot_time = 0 + for i in range(len(all_prompts)): + tic = time.perf_counter() + text_qa.run_batch( + list(zip(all_prompts[i], [gen_len] * len(all_prompts[i]))), + ) + tot_time += time.perf_counter() - tic + + return tot_time + + +def test_batch_by_batch_with_hint(all_prompts, gen_len): + backend.flush_cache() + + tot_time = 0 + for i in range(len(all_prompts)): + tic = time.perf_counter() + # Send a hint to cache the prefix + text_qa.run_batch(list(zip(all_prompts[i][:1], [gen_len]))) + # Send the batch + text_qa.run_batch(list(zip(all_prompts[i], [gen_len] * len(all_prompts[i])))) + + tot_time += time.perf_counter() - tic + + return tot_time + + +def test_send_all(all_prompts, gen_len): + backend.flush_cache() + + all_prompts = [x for prompt_list in all_prompts for x in prompt_list] + + tic = time.perf_counter() + text_qa.run_batch( + list(zip(all_prompts, [gen_len] * len(all_prompts))), + ) + tot_time = time.perf_counter() - tic + + return tot_time + + +if __name__ == "__main__": + tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer") + backend = RuntimeEndpoint("http://127.0.0.1:30000") + set_default_backend(backend) + + random.seed(0) + num_prefix = 10 + num_samples_per_prefix = 32 + prefix_length = 1024 + suffix_length = 128 + gen_len = 1 + all_prompts, tot_input_len = prepare_prompts( + num_prefix, num_samples_per_prefix, prefix_length, suffix_length + ) + + print(f"Total input token length: {tot_input_len}\n") + + cost = test_batch_by_batch(all_prompts, gen_len) + print(f"Latency of test_batch_by_batch : {cost:.4f} s\n") + + cost = test_batch_by_batch_with_hint(all_prompts, gen_len) + print(f"Latency of test_batch_by_batch_with_hint: {cost:.4f} s\n") + + cost = test_send_all(all_prompts, gen_len) + print(f"Latency of test_send_all : {cost:.4f} s\n") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/benchmark_batch/benchmark_batch.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/benchmark_batch/benchmark_batch.py new file mode 100644 index 00000000..a8592d48 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/benchmark_batch/benchmark_batch.py @@ -0,0 +1,193 @@ +import concurrent.futures +import os +import random +import time +from concurrent.futures import ProcessPoolExecutor +from statistics import mean + +import requests +from tqdm import tqdm +from transformers import AutoTokenizer + +from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint + +############################################################################### +# CONFIG +############################################################################### +ENDPOINT_URL = "http://127.0.0.1:30000" +TOKENIZER_DIR = "/models/meta-llama/Llama-3.2-3B" + +# Benchmark configurations +NUM_REQUESTS = 10 # Total number of requests (each with BATCH_SIZE prompts) +NUM_TOKENS = 32000 # Tokens per prompt +BATCH_SIZE = 8 # Number of prompts per request +GEN_TOKENS = 0 # Tokens to generate per prompt + + +############################################################################### +# REQUEST GENERATION (in parallel) +############################################################################### +def generate_random_prompt(index, tokenizer_dir, num_tokens): + """Generate a single random prompt with specified token count.""" + tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir) + vocab_size = tokenizer.vocab_size + + def generate_random_text(num_toks): + random_token_ids = [random.randint(0, vocab_size - 1) for _ in range(num_toks)] + return tokenizer.decode(random_token_ids, clean_up_tokenization_spaces=True) + + random_text = generate_random_text(num_tokens) + return f"Prompt {index}: {random_text}" + + +def prepare_all_prompts(num_requests, batch_size, num_tokens, tokenizer_dir): + """Generate prompts for all requests in parallel.""" + total_prompts = num_requests * batch_size + all_prompts = [None] * total_prompts + max_workers = min(os.cpu_count() or 1, total_prompts) + + with ProcessPoolExecutor(max_workers=max_workers) as executor: + futures = [ + executor.submit(generate_random_prompt, i, tokenizer_dir, num_tokens) + for i in range(total_prompts) + ] + for future in tqdm( + concurrent.futures.as_completed(futures), + total=total_prompts, + desc="Generating prompts", + ): + index = futures.index(future) + all_prompts[index] = future.result() + + batched_prompts = [ + all_prompts[i * batch_size : (i + 1) * batch_size] for i in range(num_requests) + ] + + print( + f"Generated {total_prompts} prompts with {num_tokens} tokens each, grouped into {num_requests} requests of {batch_size} prompts.\n" + ) + return batched_prompts + + +############################################################################### +# HTTP CALLS +############################################################################### +def send_batch_request(endpoint, prompts, gen_tokens, request_id): + """Send a batch of prompts to the /generate endpoint synchronously.""" + sampling_params = { + "max_new_tokens": gen_tokens, + "temperature": 0.7, + "stop": "\n", + } + data = {"text": prompts, "sampling_params": sampling_params} + + start_time = time.perf_counter() + try: + response = requests.post( + endpoint.base_url + "/generate", json=data, timeout=3600 + ) + if response.status_code != 200: + error = response.json() + raise RuntimeError(f"Request {request_id} failed: {error}") + result = response.json() + elapsed_time = (time.perf_counter() - start_time) * 1000 # Convert to ms + avg_per_prompt = elapsed_time / len(prompts) if prompts else 0 + return request_id, elapsed_time, avg_per_prompt, True, len(prompts) + except Exception as e: + print(f"[Request] Error for request {request_id}: {e}") + return request_id, 0, 0, False, len(prompts) + + +def run_benchmark(endpoint, batched_prompts, batch_size, gen_tokens): + """Run the benchmark sequentially.""" + results = [] + num_requests = len(batched_prompts) + + # Record start time for total latency + benchmark_start_time = time.perf_counter() + + for i, batch_prompts in enumerate(batched_prompts): + request_id = i + 1 + assert ( + len(batch_prompts) == batch_size + ), f"Request {request_id} should have {batch_size} prompts, got {len(batch_prompts)}" + + print( + f"[Request] Sending request {request_id}/{num_requests} with {len(batch_prompts)} prompts at {int(time.time()*1000)}" + ) + result = send_batch_request(endpoint, batch_prompts, gen_tokens, request_id) + results.append(result) + + # Calculate total latency + total_latency = (time.perf_counter() - benchmark_start_time) * 1000 # Convert to ms + + return results, total_latency + + +############################################################################### +# RESULTS +############################################################################### +def process_results(results, total_latency, num_requests): + """Process and display benchmark results.""" + total_time = 0 + successful_requests = 0 + failed_requests = 0 + request_latencies = [] + per_prompt_latencies = [] + total_prompts = 0 + + for request_id, elapsed_time, avg_per_prompt, success, batch_size in results: + if success: + successful_requests += 1 + total_prompts += batch_size + request_latencies.append(elapsed_time) + per_prompt_latencies.append(avg_per_prompt) + total_time += elapsed_time / 1000 # Convert to seconds + else: + failed_requests += 1 + + avg_request_latency = mean(request_latencies) if request_latencies else 0 + avg_per_prompt_latency = mean(per_prompt_latencies) if per_prompt_latencies else 0 + throughput = total_prompts / total_time if total_time > 0 else 0 + + print("\nBenchmark Summary:") + print(f" Total requests sent: {len(results)}") + print(f" Total prompts sent: {total_prompts}") + print(f" Successful requests: {successful_requests}") + print(f" Failed requests: {failed_requests}") + print(f" Total latency (all requests): {total_latency:.2f} ms") + print(f" Avg per request latency: {avg_request_latency:.2f} ms") + print(f" Avg per prompt latency: {avg_per_prompt_latency:.2f} ms") + print(f" Throughput: {throughput:.2f} prompts/second\n") + + +############################################################################### +# MAIN +############################################################################### +def main(): + # Initialize endpoint + endpoint = RuntimeEndpoint(ENDPOINT_URL) + + # Generate prompts + batched_prompts = prepare_all_prompts( + NUM_REQUESTS, BATCH_SIZE, NUM_TOKENS, TOKENIZER_DIR + ) + + # Flush cache before benchmark + # endpoint.flush_cache() + + # Run benchmark + print( + f"Starting benchmark: NUM_TOKENS={NUM_TOKENS}, BATCH_SIZE={BATCH_SIZE}, NUM_REQUESTS={NUM_REQUESTS}\n" + ) + results, total_latency = run_benchmark( + endpoint, batched_prompts, BATCH_SIZE, GEN_TOKENS + ) + + # Process and display results + process_results(results, total_latency, NUM_REQUESTS) + + +if __name__ == "__main__": + random.seed(0) + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/benchmark_batch/benchmark_tokenizer.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/benchmark_batch/benchmark_tokenizer.py new file mode 100644 index 00000000..88a5820b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/benchmark_batch/benchmark_tokenizer.py @@ -0,0 +1,126 @@ +import random +import time +from statistics import mean + +from transformers import AutoTokenizer + +# CONFIG +TOKENIZER_DIR = ( + "/shared/public/sharing/fait360brew/training/models/meta-llama/Llama-3.2-3B" +) +NUM_TOKENS = 20000 # Each prompt should contain this many tokens +BATCH_SIZES = [1, 2, 4, 8] # Test different batch sizes +NUM_RUNS = 5 # Number of runs for each batch size to get reliable measurements + + +def generate_random_prompts(num_prompts, num_tokens, tokenizer): + """Generate random prompts with specified token count.""" + vocab_size = tokenizer.vocab_size + all_prompts = [] + + print(f"Generating {num_prompts} random prompts with {num_tokens} tokens each...") + for i in range(num_prompts): + # Generate random token IDs - this directly gives us the exact token count + random_token_ids = [ + random.randint(0, vocab_size - 1) for _ in range(num_tokens) + ] + random_text = tokenizer.decode( + random_token_ids, clean_up_tokenization_spaces=True + ) + + prompt = f"Prompt {i}: {random_text}" + tokens = tokenizer.encode(prompt) + print(f" Prompt {i}: {len(tokens)} tokens") + all_prompts.append(prompt) + + return all_prompts + + +def benchmark_sequential_vs_batch(prompts, batch_size, tokenizer): + """Compare sequential vs batch tokenization for a given batch size.""" + + # Sequential tokenization using encode() + sequential_times = [] + for run in range(NUM_RUNS): + batch_prompts = prompts[:batch_size] # Use same prompts for fair comparison + + start_time = time.perf_counter() + for prompt in batch_prompts: + tokens = tokenizer.encode(prompt) + sequential_time = (time.perf_counter() - start_time) * 1000 + sequential_times.append(sequential_time) + + # Batch tokenization using tokenizer() + batch_times = [] + for run in range(NUM_RUNS): + batch_prompts = prompts[:batch_size] # Use same prompts for fair comparison + + start_time = time.perf_counter() + tokens = tokenizer(batch_prompts) + batch_time = (time.perf_counter() - start_time) * 1000 + batch_times.append(batch_time) + + return { + "batch_size": batch_size, + "avg_sequential_ms": mean(sequential_times), + "avg_batch_ms": mean(batch_times), + "speedup_factor": ( + mean(sequential_times) / mean(batch_times) if mean(batch_times) > 0 else 0 + ), + "sequential_runs": sequential_times, + "batch_runs": batch_times, + } + + +def main(): + print("Tokenizer Benchmark: Sequential vs Batch Processing") + print("-" * 60) + print(f"Tokenizer: {TOKENIZER_DIR}") + print(f"Tokens per prompt: {NUM_TOKENS}") + print(f"Number of runs per batch size: {NUM_RUNS}") + print("-" * 60) + + # Load tokenizer once for all operations + tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_DIR) + + # The largest batch size determines how many prompts we need + max_batch_size = max(BATCH_SIZES) + all_prompts = generate_random_prompts(max_batch_size, NUM_TOKENS, tokenizer) + + results = [] + print("\nRunning benchmark...") + + for batch_size in BATCH_SIZES: + print(f"\nBenchmarking batch size: {batch_size}") + result = benchmark_sequential_vs_batch(all_prompts, batch_size, tokenizer) + results.append(result) + + print(f" Sequential tokenization (encode):") + for i, run_time in enumerate(result["sequential_runs"]): + print(f" Run {i+1}: {run_time:.2f} ms") + print(f" Average: {result['avg_sequential_ms']:.2f} ms") + + print(f" Batch tokenization (tokenizer):") + for i, run_time in enumerate(result["batch_runs"]): + print(f" Run {i+1}: {run_time:.2f} ms") + print(f" Average: {result['avg_batch_ms']:.2f} ms") + + print(f" Speedup factor: {result['speedup_factor']:.2f}x") + + print("\n" + "=" * 60) + print("SUMMARY OF RESULTS") + print("=" * 60) + print( + f"{'Batch Size':<10} {'Sequential (ms)':<18} {'Batch (ms)':<18} {'Speedup':<10}" + ) + print("-" * 60) + + for result in results: + print( + f"{result['batch_size']:<10} {result['avg_sequential_ms']:.2f} ms{' ' * 8} {result['avg_batch_ms']:.2f} ms{' ' * 8} {result['speedup_factor']:.2f}x" + ) + + +if __name__ == "__main__": + random.seed(0) + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/benchmark_vllm_060/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/benchmark_vllm_060/README.md new file mode 100644 index 00000000..b480dabf --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/benchmark_vllm_060/README.md @@ -0,0 +1,89 @@ +## How to reproduce the benchmark results for SGLang v0.3.0 compared to vLLM v0.6.0 + +In short, with multi step enabled, in online scenarios that we benchmarked, the Median TTFT of vLLM is **3 times** that of SGLang, and the Median ITL is **10 times** that of SGLang. Lower Median TTFT and ITL are better. vLLM's multi-step optimization did not improve throughput while ensuring lower Median TTFT and ITL. Also, under maximum throughput benchmark, if vLLM does not set gpu util to 0.95 separately and uses the default configuration instead, its maximum throughput is **lower** than that of SGLang. + +## Online benchmark results + +### Llama 3.1 8B Instruct 1 x A100 80G + +| RPS | Num prompts | Engine | Median E2E Latency | Median TTFT | Median TPOT | Median ITL | +|------|-------------|--------|--------------------|-------------|-------------|------------| +| 4 | 1200 | SGLang | 1564.17 | **31.98** | 13.17 | **11.93** | +| 4 | 1200 | vLLM | 1691.97 | **100.48** | 14.14 | **129.32** | +| 8 | 2400 | SGLang | 2175.02 | **35.68** | 17.85 | **14.41** | +| 8 | 2400 | vLLM | 2137.16 | **120.39** | 17.09 | **158.63** | + +### Llama 3.1 70B Insruct 4 x H100 80G + +| RPS | Num Prompts | Engine | Median E2E Latency | Median TTFT | Median TPOT | Median ITL | +|------|-------------|--------|--------------------|-------------|-------------|------------| +| 4 | 1200 | SGLang | 3005.24 | **53.94** | 25.03 | **21.67** | +| 4 | 1200 | vLLM | 2915.60 | **179.15** | 23.58 | **231.23** | +| 8 | 2400 | SGLang | 4064.98 | **58.11** | 33.07 | **24.45** | +| 8 | 2400 | vLLM | 3752.38 | **207.12** | 29.15 | **275.32** | + +## Offline benchmark results + +### Llama 3.1 8B Instruct 1 x A100 80G + +| RPS | Num Prompts | Engine | Request throughput | Output token throughput | +|------|-------------|--------|--------------------|-------------------------| +| inf | 5000 | SGLang | 22.03 | **4281.51** | +| inf | 5000 | vLLM | 21.27 | **4132.37** | + +### Llama 3.1 70B Insruct 4 x H100 80G + +| RPS | Num Prompts | Engine | Request throughput | Output token throughput | +|------|-------------|--------|--------------------|-------------------------| +| inf | 5000 | SGLang | 19.84 | **3856.01** | +| inf | 5000 | vLLM | 19.04 | **3700.64** | + +## Installation + +```bash +# install sglang v0.3.0 +pip install --upgrade pip +pip install "sglang[all]"==0.3.0 +pip install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.4/ + +# install vllm v0.6.0 +pip install vllm==0.6.0 +``` + +## Notes + +We referred to the reproduction method in https://github.com/vllm-project/vllm/issues/8176, and added the `--num-scheduler-steps 10` parameter when starting the vLLM server. The `gpu_memory_utilization` of vLLM is by default 0.9 at both TP 1 and TP 4, while SGLang's `mem_frac` is 0.88 at TP 1 and 0.85 at TP 4, so we manually set it to 0.88 at TP 4. + +## Online benchmarks + +```bash +# Llama 3.1 8B Instruct on 1 x A100 +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --enable-torch-compile --disable-radix-cache +python -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-8B-Instruct --disable-log-requests --num-scheduler-steps 10 --max_model_len 4096 + +# Llama 3.1 70B Instruct on 4 x H100 +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-70B-Instruct --disable-radix-cache --tp 4 +python -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-70B-Instruct --disable-log-requests --num-scheduler-steps 10 --tensor 4 --max_model_len 4096 + +# bench serving +python3 -m sglang.bench_serving --backend sglang --dataset-name sharegpt --num-prompts 1200 --request-rate 4 +python3 -m sglang.bench_serving --backend sglang --dataset-name sharegpt --num-prompts 2400 --request-rate 8 +python3 -m sglang.bench_serving --backend vllm --dataset-name sharegpt --num-prompts 1200 --request-rate 4 +python3 -m sglang.bench_serving --backend vllm --dataset-name sharegpt --num-prompts 2400 --request-rate 8 +``` + +## Offline benchmarks + +```bash +# Llama 3.1 8B Instruct on 1 x A100 +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --enable-torch-compile --disable-radix-cache +python -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-8B-Instruct --disable-log-requests --num-scheduler-steps 10 --max_model_len 4096 + +# Llama 3.1 70B Instruct on 4 x H100 +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-70B-Instruct --disable-radix-cache --tp 4 --mem-frac 0.88 +python -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-70B-Instruct --disable-log-requests --num-scheduler-steps 10 --tensor 4 --max_model_len 4096 + +# bench serving +python3 -m sglang.bench_serving --backend sglang --dataset-name sharegpt --num-prompts 5000 +python3 -m sglang.bench_serving --backend vllm --dataset-name sharegpt --num-prompts 5000 +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/blog_v0_2/405b_sglang.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/blog_v0_2/405b_sglang.sh new file mode 100644 index 00000000..49185378 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/blog_v0_2/405b_sglang.sh @@ -0,0 +1,24 @@ +# Create dummy weights: +# 1. Create a folder `~/llama-3.1-405b-fp8-dummy` and create `config.json` and tokenizer under this folder. +# 2. Get `config.json`` from ./config.md +# 3. Download the tokenizer +# wget https://huggingface.co/neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8/resolve/main/tokenizer.json +# wget https://huggingface.co/neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8/resolve/main/tokenizer_config.json + +# Launch sglang +# python -m sglang.launch_server --model-path ~/llama-3.1-405b-fp8-dummy/ --load-format dummy --tp 8 --quantization fp8 --disable-radix --mem-frac 0.87 + +# offline +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 3000 --random-input 1024 --random-output 1024 > sglang_log11 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 4000 --random-input 1024 --random-output 512 > sglang_log12 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 800 --random-input 4096 --random-output 2048 > sglang_log13 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 1500 --random-input 4096 --random-output 1024 > sglang_log14 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 6000 --random-input 256 --random-output 512 > sglang_log15 +python3 -m sglang.bench_serving --backend sglang --dataset-name sharegpt --num-prompt 2000 > sglang_log21 + +# online +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 300 --request-rate 1 --random-input 1024 --random-output 1024 > sglang_log31 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 600 --request-rate 2 --random-input 1024 --random-output 1024 > sglang_log32 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 1200 --request-rate 4 --random-input 1024 --random-output 1024 > sglang_log33 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 2400 --request-rate 8 --random-input 1024 --random-output 1024 > sglang_log34 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 3200 --request-rate 16 --random-input 1024 --random-output 1024 > sglang_log35 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/blog_v0_2/405b_trt.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/blog_v0_2/405b_trt.sh new file mode 100644 index 00000000..1950bc92 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/blog_v0_2/405b_trt.sh @@ -0,0 +1,17 @@ +# Launch trtllm +# https://github.com/sgl-project/tensorrt-demo + +# offline +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 3000 --random-input 1024 --random-output 1024 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log11 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 4000 --random-input 1024 --random-output 512 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log12 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 800 --random-input 4096 --random-output 2048 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log13 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 1500 --random-input 4096 --random-output 1024 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log14 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 6000 --random-input 256 --random-output 512 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log15 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name sharegpt --num-prompt 2000 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log21 + +# online +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 300 --request-rate 1 --random-input 1024 --random-output 1024 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log31 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 600 --request-rate 2 --random-input 1024 --random-output 1024 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log32 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 1200 --request-rate 4 --random-input 1024 --random-output 1024 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log33 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 2400 --request-rate 8 --random-input 1024 --random-output 1024 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log34 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 3200 --request-rate 16 --random-input 1024 --random-output 1024 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log35 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/blog_v0_2/405b_vllm.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/blog_v0_2/405b_vllm.sh new file mode 100644 index 00000000..d1ef048e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/blog_v0_2/405b_vllm.sh @@ -0,0 +1,24 @@ +# Create dummy weights: +# 1. Create a folder `~/llama-3.1-405b-fp8-dummy` and create `config.json` and tokenizer under this folder. +# 2. Get `config.json`` from ./config.md +# 3. Download the tokenizer +# wget https://huggingface.co/neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8/resolve/main/tokenizer.json +# wget https://huggingface.co/neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8/resolve/main/tokenizer_config.json + +# Launch vllm +# python3 -m vllm.entrypoints.openai.api_server --model ~/llama-3.1-405b-fp8-dummy/ --load-format dummy --disable-log-requests --tensor-parallel-size 8 --max-model-len 10000 + +# offline +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 3000 --random-input 1024 --random-output 1024 > vllm_log11 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 4000 --random-input 1024 --random-output 512 > vllm_log12 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 800 --random-input 4096 --random-output 2048 > vllm_log13 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 1500 --random-input 4096 --random-output 1024 > vllm_log14 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 6000 --random-input 256 --random-output 512 > vllm_log15 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name sharegpt --num-prompt 2000 > vllm_log21 + +# online +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 300 --request-rate 1 --random-input 1024 --random-output 1024 > vllm_log31 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 600 --request-rate 2 --random-input 1024 --random-output 1024 > vllm_log32 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 1200 --request-rate 4 --random-input 1024 --random-output 1024 > vllm_log33 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 2400 --request-rate 8 --random-input 1024 --random-output 1024 > vllm_log34 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 3200 --request-rate 16 --random-input 1024 --random-output 1024 > vllm_log35 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/blog_v0_2/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/blog_v0_2/README.md new file mode 100644 index 00000000..7448554e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/blog_v0_2/README.md @@ -0,0 +1,164 @@ +# How to reproduce the benchmark results of SGLang + +## Prerequisite + +### Install the latest SGLang + +```bash +git clone https://github.com/sgl-project/sglang.git +cd sglang +git checkout v0.2.7 + +pip install --upgrade pip +pip install -e "python[all]" + +pip install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.3/ +``` + +### Set up ulimit and HF_TOKEN + +```bash +ulimit -n 65535 +# Change the token to a real and usable one, with access permissions for the Llama 3 models. +export HF_TOKEN=hf_token +``` + +### Launch the server + +```bash +# Meta-Llama-3.1-8B-Instruct +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --enable-torch-compile --disable-radix-cache + +# Meta-Llama-3.1-70B-Instruct +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-70B-Instruct --disable-radix-cache --tp 8 + +# Meta-Llama-3-70B-Instruct-FP8 +python -m sglang.launch_server --model-path neuralmagic/Meta-Llama-3-70B-Instruct-FP8 --disable-radix-cache --tp 8 +``` + +## Benchmark + +### Hardware Requirements + +- 8B models: Single NVIDIA A100 80GB GPU +- 70B models: 8 x NVIDIA A100 80GB GPUs with Tensor Parallelism (TP) 8 +- 70B FP8 models: 8 x NVIDIA H100 GPUs with Tensor Parallelism (TP) 8 + +Please ensure you have the appropriate hardware before running the benchmarks. + +#### Offline benchmark + +```bash +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 4000 --random-input 1024 --random-output 1024 --output-file offline.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 5000 --random-input 1024 --random-output 512 --output-file offline.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 1000 --random-input 4096 --random-output 2048 --output-file offline.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 2000 --random-input 4096 --random-output 1024 --output-file offline.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 6000 --random-input 256 --random-output 512 --output-file offline.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name sharegpt --num-prompts 3000 --output-file offline.jsonl +cat offline.jsonl | cut -d':' -f12 | cut -d',' -f1 +``` + +#### Online benchmark + +```bash +python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 300 --request-rate 1 --output-file online.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 600 --request-rate 2 --output-file online.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 1200 --request-rate 4 --output-file online.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 2400 --request-rate 8 --output-file online.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 3200 --request-rate 16 --output-file online.jsonl +cat online.jsonl | cut -d':' -f9 | cut -d',' -f1 +``` + +## Other + +We tried using vLLM 0.5.3.post1, but it often crashes under high loads, and it seems to have similar or worse performance compared to vLLM 0.5.2 from our partial benchmarking, so we are using the older version, vLLM 0.5.2. + +Preparation for TensorRT LLM can refer to https://github.com/sgl-project/tensorrt-demo. Specifically, we used a batch size of 512, a max input length of 8192, and a max number of tokens of 8192. The instance count for preprocessing and postprocessing in Triton Server is 16. + +```bash +# vLLM +pip install vllm==0.5.2 +pip install jsonschema==4.21.1 + +# Meta-Llama-3-8B-Instruct +python -m vllm.entrypoints.openai.api_server --model meta-llama/Meta-Llama-3-8B-Instruct --disable-log-requests + +# meta-llama/Meta-Llama-3-70B-Instruct +python -m vllm.entrypoints.openai.api_server --model meta-llama/Meta-Llama-3-70B-Instruct --disable-log-requests --tensor 8 + +# neuralmagic/Meta-Llama-3-70B-Instruct-FP8 +python -m vllm.entrypoints.openai.api_server --model neuralmagic/Meta-Llama-3-70B-Instruct-FP8 --disable-log-requests --tensor 8 +``` + +```bash +wget https://raw.githubusercontent.com/sgl-project/sglang/main/python/sglang/bench_serving.py +``` + +```bash +# vLLM Offline + +python3 bench_serving.py --backend vllm --dataset-name random --num-prompts 4000 --random-input 1024 --random-output 1024 --output-file offline_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --num-prompts 5000 --random-input 1024 --random-output 512 --output-file offline_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --num-prompts 1000 --random-input 4096 --random-output 2048 --output-file offline_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --num-prompts 2000 --random-input 4096 --random-output 1024 --output-file offline_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --num-prompts 6000 --random-input 256 --random-output 512 --output-file offline_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name sharegpt --num-prompts 3000 --output-file offline_vllm.jsonl +cat offline_vllm.jsonl | cut -d':' -f12 | cut -d',' -f1 +``` + +```bash +# vLLM Online + +python3 bench_serving.py --backend vllm --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 300 --request-rate 1 --output-file online_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 600 --request-rate 2 --output-file online_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 1200 --request-rate 4 --output-file online_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 2400 --request-rate 8 --output-file online_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 3200 --request-rate 16 --output-file online_vllm.jsonl +cat online_vllm.jsonl | cut -d':' -f9 | cut -d',' -f1 +``` + +```bash +# TensorRT LLM Offline 8B + +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --num-prompts 4000 --random-input 1024 --random-output 1024 --output-file offline_trt_8b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --num-prompts 5000 --random-input 1024 --random-output 512 --output-file offline_trt_8b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --num-prompts 1000 --random-input 4096 --random-output 2048 --output-file offline_trt_8b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --num-prompts 2000 --random-input 4096 --random-output 1024 --output-file offline_trt_8b.jsonl +python3 bench_serving.py --backend trt --dataset-name random --num-prompts 6000 --random-input 256 --random-output 512 --output-file offline_trt_8b.jsonl --model meta-llama/Meta-Llama-3-8B-Instruct +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name sharegpt --num-prompts 3000 --output-file offline_trt_8b.jsonl +cat offline_trt_8b.jsonl | cut -d':' -f12 | cut -d',' -f1 +``` + +```bash +# TensorRT LLM Online 8B + +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 300 --request-rate 1 --output-file online_trt_8b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 600 --request-rate 2 --output-file online_trt_8b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 1200 --request-rate 4 --output-file online_trt_8b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 2400 --request-rate 8 --output-file online_trt_8b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 3200 --request-rate 16 --output-file online_trt_8b.jsonl +cat online_trt_8b.jsonl | cut -d':' -f9 | cut -d',' -f1 +``` + +```bash +# TensorRT LLM Offline 70B + +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --num-prompts 4000 --random-input 1024 --random-output 1024 --output-file offline_trt_70b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --num-prompts 5000 --random-input 1024 --random-output 512 --output-file offline_trt_70b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --num-prompts 1000 --random-input 4096 --random-output 2048 --output-file offline_trt_70b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --num-prompts 2000 --random-input 4096 --random-output 1024 --output-file offline_trt_70b.jsonl +python3 bench_serving.py --backend trt --dataset-name random --num-prompts 6000 --random-input 256 --random-output 512 --output-file offline_trt_70b.jsonl --model meta-llama/Meta-Llama-3-70B-Instruct +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name sharegpt --num-prompts 3000 --output-file offline_trt_70b.jsonl +cat offline_trt_70b.jsonl | cut -d':' -f12 | cut -d',' -f1 +``` + +```bash +# TensorRT LLM Online 70B + +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 300 --request-rate 1 --output-file online_trt_70b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 600 --request-rate 2 --output-file online_trt_70b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 1200 --request-rate 4 --output-file online_trt_70b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 2400 --request-rate 8 --output-file online_trt_70b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 3200 --request-rate 16 --output-file online_trt_70b.jsonl +cat online_trt_70b.jsonl | cut -d':' -f9 | cut -d',' -f1 +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/blog_v0_2/config.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/blog_v0_2/config.md new file mode 100644 index 00000000..3faf6009 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/blog_v0_2/config.md @@ -0,0 +1,100 @@ +### used for TensorRT LLM + +``` +{ + "architecture": "LlamaForCausalLM", + "dtype": "float16", + "logits_dtype": "float32", + "vocab_size": 128256, + "max_position_embeddings": 8192, + "hidden_size": 16384, + "num_hidden_layers": 126, + "num_attention_heads": 128, + "num_key_value_heads": 16, + "head_size": 128, + "qk_layernorm": false, + "hidden_act": "silu", + "intermediate_size": 53248, + "norm_epsilon": 1e-05, + "position_embedding_type": "rope_gpt_neox", + "use_parallel_embedding": false, + "embedding_sharding_dim": 0, + "share_embedding_table": false, + "mapping": { + "world_size": 8, + "tp_size": 8, + "pp_size": 1, + "gpus_per_node": 8 + }, + "quantization": { + "quant_algo": "FP8", + "kv_cache_quant_algo": null, + "group_size": 128, + "smoothquant_val": null, + "has_zero_point": false, + "pre_quant_scale": false, + "exclude_modules": [ + "lm_head" + ] + }, + "kv_dtype": "float16", + "rotary_scaling": null, + "residual_mlp": false, + "moe_normalization_mode": null, + "rotary_base": 500000.0, + "moe_num_experts": 0, + "moe_top_k": 0, + "moe_tp_mode": 2, + "attn_bias": false, + "disable_weight_only_quant_plugin": false, + "mlp_bias": false +} +``` + +### used for vLLM and SGLang + +``` +{ + "_name_or_path": "dummy_fp8", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128009, + "hidden_act": "silu", + "hidden_size": 16384, + "initializer_range": 0.02, + "intermediate_size": 53248, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 128, + "num_hidden_layers": 126, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "quantization_config": { + "activation_scheme": "static", + "ignored_layers": [ + "lm_head" + ], + "quant_method": "fp8" + }, + "rope_scaling": { + "factor": 8.0, + "low_freq_factor": 1.0, + "high_freq_factor": 4.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "max_position_embeddings": 131072, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.41.1", + "use_cache": true, + "vocab_size": 128256 +} +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/boolq/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/boolq/README.md new file mode 100644 index 00000000..3704742e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/boolq/README.md @@ -0,0 +1,19 @@ +## Download data +``` +git clone https://hf-mirror.com/datasets/google/boolq +``` + +## Convert parquet to json +``` +bash parquet_to_json.sh +``` +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path ramblingpolymath/Qwen3-32B-W8A8 --port 30000 +``` + +``` +python3 bench_sglang.py +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/boolq/bench_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/boolq/bench_sglang.py new file mode 100644 index 00000000..b3ce3c99 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/boolq/bench_sglang.py @@ -0,0 +1,124 @@ +import argparse +import json +import time + +import numpy as np + +from sglang.api import set_default_backend +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import read_jsonl + + +def get_example(lines, i, answer): + prompt = "Question: " + lines[i]["question"] + lines[i]["passage"] + "\nAnswer:" + if answer: + prompt += str(lines[i]["answer"]) + return prompt + + +def few_shot_examples(lines, k): + prompts = "" + for i in range(k): + prompts += get_example(lines, i, True) + "\n\n" + return prompts + + +def main(args): + # Select backend + set_default_backend(select_sglang_backend(args)) + + # Read data + train_data_path = args.train_data_path + test_data_path = args.test_data_path + lines_train = list(read_jsonl(train_data_path)) + lines_test = list(read_jsonl(test_data_path)) + + # Construct prompts + num_questions = args.num_questions + num_shots = args.num_shots + few_shots = few_shot_examples(lines_train, num_shots) + + questions = [] + answer = [] + for i in range(len(lines_test[:num_questions])): + questions.append(get_example(lines_test, i, False)) + answer.append(str(lines_test[i]["answer"])) + arguments = [{"question": q} for q in questions] + + ##################################### + ######### SGL Program Begin ######### + ##################################### + + import sglang as sgl + + @sgl.function + def few_shot_boolq(s, question): + s += few_shots + question + s += sgl.gen("answer", max_tokens=5, stop=["\n"]) + + ##################################### + ########## SGL Program End ########## + ##################################### + + # Run requests + tic = time.perf_counter() + states = few_shot_boolq.run_batch( + arguments, + temperature=0, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + + preds = [] + for i in range(len(states)): + preds.append(states[i]["answer"]) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(answer)) + + # Compute speed + num_output_tokens = sum( + s.get_meta_info("answer")["completion_tokens"] for s in states + ) + output_throughput = num_output_tokens / latency + + # Print results + print(f"Accuracy: {acc:.3f}") + print(f"Latency: {latency:.3f} s") + print(f"Output throughput: {output_throughput:.3f} token/s") + + # Results + with open(args.result_file, "a") as fout: + value = { + "task": "boolq", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--num-shots", type=int, default=5) + parser.add_argument( + "--train-data-path", type=str, default="./boolq/data/train-00000-of-00001.json" + ) + parser.add_argument( + "--test-data-path", + type=str, + default="./boolq/data/validation-00000-of-00001.json", + ) + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/boolq/convert_parquet_to_json.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/boolq/convert_parquet_to_json.py new file mode 100644 index 00000000..e3e69cb3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/boolq/convert_parquet_to_json.py @@ -0,0 +1,28 @@ +import sys + +import pyarrow.parquet as pq + + +def convert_parquet_to_json(input_file, output_file): + # read parquet file + table = pq.read_table(input_file) + + # turn parquet data to dataframe + df = table.to_pandas() + + # turn dataframe to json form + json_data = df.to_json(orient="records", lines=True) + + # write json to file + with open(output_file, "w") as f: + f.write(json_data) + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("Usage:python convert_parquet_to_json.py ") + + input_file = sys.argv[1] + output_file = sys.argv[2] + + convert_parquet_to_json(input_file, output_file) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/boolq/parquet_to_json.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/boolq/parquet_to_json.sh new file mode 100755 index 00000000..9aaf087f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/boolq/parquet_to_json.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +#define input and output direction +input_dir="./boolq/data" +output_dir="./boolq/data" + +#define files needed to be handled +files=( + "train-00000-of-00001.parquet" + "validation-00000-of-00001.parquet" +) + +#foe files above, use python script to convert the form +for file in "${files[@]}"; do + input_file="${input_dir}/${file}" + output_file="${output_dir}/${file%.parquet}.json" + + echo "Converting ${input_file} to ${output_file} ..." + python3 convert_parquet_to_json.py "${input_file}" "${output_file}" + + if [ $? -eq 0 ]; then + echo "Conversion successful: ${output_file}" + else + echo "Conversion failed: ${input_file}" + fi +done diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/ceval/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/ceval/README.md new file mode 100644 index 00000000..b822e43c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/ceval/README.md @@ -0,0 +1,15 @@ +## Download data +``` +git lfs clone https://huggingface.co/datasets/ceval/ceval-exam +``` + +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path ramblingpolymath/Qwen3-32B-W8A8 --port 30000 +``` + +``` +python3 bench_sglang.py +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/ceval/bench_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/ceval/bench_sglang.py new file mode 100644 index 00000000..bcebd55c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/ceval/bench_sglang.py @@ -0,0 +1,138 @@ +import argparse +import json +import os +import random +import re +import time + +import numpy as np +from datasets import load_dataset + +from sglang.lang.api import set_default_backend +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) + +choices = ["A", "B", "C", "D"] + + +def get_one_example(line, include_answer): + res = line["question"] + res += f"\nA. {line['A']}" + res += f"\nB. {line['B']}" + res += f"\nC. {line['C']}" + res += f"\nD. {line['D']}" + + if include_answer: + res += f"\nAnswer: {line['answer']} \n\n" + return res + + +def get_few_shot_examples(lines): + res = "" + for line in lines: + res += get_one_example(line, True) + "\n\n" + return res + + +def get_answer_value(response): + pattern = r"(Answer:|answer:|答案是|答案是:|正确答案是:|答案:|Assistant:)\s*([A-D])(?![\w])" + match = re.search(pattern, response) + + if match: + return match.group(2) + + return random.choice(choices) + + +def main(args): + # Read data && Construct prompts + arguments = [] + labels = [] + examples = "examples:\n" + data_path = args.data_path + for subject in os.listdir(data_path): + subject_path = os.path.join(data_path, subject) + if os.path.isdir(subject_path) and subject != ".git": + dataset = load_dataset(data_path, name=subject) + dev_lines_temp = dataset["dev"] + val_lines_temp = dataset["val"] + few_shot_examples = get_few_shot_examples(dev_lines_temp) + examples += f"{few_shot_examples}" + for val_line in val_lines_temp: + arguments.append( + { + "examples": few_shot_examples, + "question": get_one_example(val_line, False), + } + ) + labels.append(val_line["answer"]) + + ##################################### + ######### SGL Program Begin ######### + ##################################### + + import sglang as sgl + + @sgl.function + def few_shot_ceval(s, examples, question): + s += examples + question + sgl.gen("Answer") + + ##################################### + ########## SGL Program End ########## + ##################################### + + num_questions = args.num_questions if args.num_questions else len(arguments) + + # Select backend + set_default_backend(select_sglang_backend(args)) + + # Run requests + tic = time.perf_counter() + states = few_shot_ceval.run_batch( + arguments[:num_questions], + temperature=0, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + + preds = [get_answer_value(states[i]["Answer"]) for i in range(num_questions)] + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels[:num_questions])) + + # Compute speed + num_output_tokens = sum( + s.get_meta_info("Answer")["completion_tokens"] for s in states + ) + output_throughput = num_output_tokens / latency + + # Print results + print(f"Accuracy: {acc:.3f}") + print(f"Latency: {latency:.3f} s") + print(f"Output throughput: {output_throughput:.3f} token/s") + + # Write results + with open(args.result_file, "a") as fout: + value = { + "task": "ceval", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="ceval/ceval-exam") + parser.add_argument("--num-questions", type=int, default=None) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/deepseek_v3/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/deepseek_v3/README.md new file mode 100644 index 00000000..59459180 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/deepseek_v3/README.md @@ -0,0 +1,388 @@ +# DeepSeek V3.1/V3/R1 Support + +The SGLang and DeepSeek teams collaborated to get DeepSeek V3 FP8 running on NVIDIA and AMD GPUs **from day one**. SGLang also supports [MLA optimization](https://lmsys.org/blog/2024-09-04-sglang-v0-3/#deepseek-multi-head-latent-attention-mla-throughput-optimizations) and [DP attention](https://lmsys.org/blog/2024-12-04-sglang-v0-4/#data-parallelism-attention-for-deepseek-models), making SGLang one of the best open-source LLM engines for running DeepSeek models. SGLang is the inference engine recommended by the official [DeepSeek team](https://github.com/deepseek-ai/DeepSeek-V3/tree/main?tab=readme-ov-file#62-inference-with-sglang-recommended). + +Special thanks to Meituan's Search & Recommend Platform Team and Baseten's Model Performance Team for implementing the model, and DataCrunch for providing GPU resources. + +For optimizations made on the DeepSeek series models regarding SGLang, please refer to [DeepSeek Model Optimizations in SGLang](https://docs.sglang.io/basic_usage/deepseek.html). + +## Installation & Launch + +If you encounter errors when starting the server, ensure the weights have finished downloading. It's recommended to download them beforehand or restart multiple times until all weights are downloaded. + +### Using Docker (Recommended) + +```bash +# Pull latest image +# https://hub.docker.com/r/lmsysorg/sglang/tags +docker pull lmsysorg/sglang:latest + +# Launch +docker run --gpus all --shm-size 32g -p 30000:30000 -v ~/.cache/huggingface:/root/.cache/huggingface --ipc=host --network=host --privileged lmsysorg/sglang:latest \ + python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code --port 30000 +``` + +If you are using RDMA, please note that: + +1. `--network host` and `--privileged` are required by RDMA. If you don't need RDMA, you can remove them. +2. You may need to set `NCCL_IB_GID_INDEX` if you are using RoCE, for example: `export NCCL_IB_GID_INDEX=3`. + +Add [performance optimization options](#performance-optimization-options) as needed. + +### Using pip + +```bash +# Installation +pip install "sglang[all]>=0.5.6" + +# Launch +python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code +``` + +Add [performance optimization options](#performance-optimization-options) as needed. + + + +### Performance Optimization Options + +[MLA optimizations](https://lmsys.org/blog/2024-09-04-sglang-v0-3/#deepseek-multi-head-latent-attention-mla-throughput-optimizations) are enabled by default. Here are some optional optimizations can be enabled as needed. + +- [Data Parallelism Attention](https://lmsys.org/blog/2024-12-04-sglang-v0-4/#data-parallelism-attention-for-deepseek-models): For high QPS scenarios, add the `--enable-dp-attention` argument to boost throughput. +- [Torch.compile Optimization](https://lmsys.org/blog/2024-09-04-sglang-v0-3/#torchcompile-latency-optimizations): Add `--enable-torch-compile` argument to enable it. This will take some time while server starts. The maximum batch size for torch.compile optimization can be controlled with `--torch-compile-max-bs`. It's recommended to set it between `1` and `8`. (e.g., `--torch-compile-max-bs 8`) + +### Usage: Chat with DeepSeek + +#### DeepSeek V3/R1 + +```python3 +import openai +client = openai.Client( + base_url="http://127.0.0.1:30000/v1", api_key="EMPTY") + +# Chat completion +response = client.chat.completions.create( + model="default", + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "List 3 countries and their capitals."}, + ], + temperature=0, + max_tokens=64, +) +print(response) +``` + +#### DeepSeek V3.1 +On top of the basic usage similar to the DeepSeek V3/R1 example, DeepSeek V3.1 supports a request-level thinking/non-thinking toggle. Simply switch the `"thinking"` field in `extra_body={"chat_template_kwargs": {"thinking": True}}` to enable/disable the thinking mode. + +##### Non Thinking +```python3 +import openai +client = openai.Client( + base_url="http://127.0.0.1:30000/v1", api_key="EMPTY") + +# Chat completion +response = client.chat.completions.create( + model="default", + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "Answer the following with the second letter of the correct answer only: What is the capital of France?"}, + ], + temperature=0, + max_tokens=1024, + extra_body = {"chat_template_kwargs": {"thinking": False}} +) +print(response.choices[0].message.content) +``` +Answer: +``` +h +``` +* The correct response should be 'A', as the correct answer to the question is 'Paris'. +##### Thinking +```python3 +import openai +client = openai.Client( + base_url="http://127.0.0.1:30000/v1", api_key="EMPTY") + +# Chat completion +response = client.chat.completions.create( + model="default", + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "Answer the following with the second letter of the correct answer only: What is the capital of France?"}, + ], + temperature=0, + max_tokens=1024, + extra_body = {"chat_template_kwargs": {"thinking": True}} +) +print(response) +``` +Answer: +``` +First, the question is: "What is the capital of France?" I know that the capital of France is Paris. + +The user says: "Answer the following with the second letter of the correct answer only." So, I need to provide only the second letter of the correct answer. + +The correct answer is "Paris". Now, I need to find the second letter of "Paris". + +Let's spell it out: P-A-R-I-S. + +- First letter: P + +- Second letter: A + +- Third letter: R + +- Fourth letter: I + +- Fifth letter: S + +So, the second letter is "A". + +I should only output the second letter, which is "A". No additional text or explanation, just the letter. + +The user emphasized "the second letter of the correct answer only", so my response should be just "A". + +Finally, I need to make sure that this is the correct answer. Yes, Paris is indeed the capital of France.A +``` +* The response contains `` thinking trace and model was able to derive the correct answer from it. + +### Example: Serving with two H20\*8 nodes + +For example, there are two H20 nodes, each with 8 GPUs. The first node's IP is `10.0.0.1`, and the second node's IP is `10.0.0.2`. Please **use the first node's IP** for both commands. + +If the command fails, try setting the `GLOO_SOCKET_IFNAME` parameter. For more information, see [Common Environment Variables](https://pytorch.org/docs/stable/distributed.html#common-environment-variables). + +If the multi nodes support NVIDIA InfiniBand and encounter hanging issues during startup, consider adding the parameter `export NCCL_IB_GID_INDEX=3`. For more information, see [this](https://github.com/sgl-project/sglang/issues/3516#issuecomment-2668493307). + +```bash +# node 1 +python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3 --tp 16 --dist-init-addr 10.0.0.1:5000 --nnodes 2 --node-rank 0 --trust-remote-code + +# node 2 +python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3 --tp 16 --dist-init-addr 10.0.0.1:5000 --nnodes 2 --node-rank 1 --trust-remote-code +``` + +If you have two H100 nodes, the usage is similar to the aforementioned H20. + +> **Note that the launch command here does not enable Data Parallelism Attention or `torch.compile` Optimization**. For optimal performance, please refer to the command options in [Performance Optimization Options](#option_args). + +### Example: Serving with two H200\*8 nodes and docker + +There are two H200 nodes, each with 8 GPUs. The first node's IP is `192.168.114.10`, and the second node's IP is `192.168.114.11`. Configure the endpoint to expose it to another Docker container using `--host 0.0.0.0` and `--port 40000`, and set up communications with `--dist-init-addr 192.168.114.10:20000`. +A single H200 with 8 devices can run DeepSeek V3, the dual H200 setup is just to demonstrate multi-node usage. + +```bash +# node 1 +docker run --gpus all \ + --shm-size 32g \ + --network=host \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --name sglang_multinode1 \ + -it \ + --rm \ + --env "HF_TOKEN=$HF_TOKEN" \ + --ipc=host \ + lmsysorg/sglang:latest \ + python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3 --tp 16 --dist-init-addr 192.168.114.10:20000 --nnodes 2 --node-rank 0 --trust-remote-code --host 0.0.0.0 --port 40000 +``` + +```bash +# node 2 +docker run --gpus all \ + --shm-size 32g \ + --network=host \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --name sglang_multinode2 \ + -it \ + --rm \ + --env "HF_TOKEN=$HF_TOKEN" \ + --ipc=host \ + lmsysorg/sglang:latest \ + python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3 --tp 16 --dist-init-addr 192.168.114.10:20000 --nnodes 2 --node-rank 1 --trust-remote-code --host 0.0.0.0 --port 40000 +``` + +To ensure functionality, we include a test from a client Docker container. + +```bash +docker run --gpus all \ + --shm-size 32g \ + --network=host \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --name sglang_multinode_client \ + -it \ + --rm \ + --env "HF_TOKEN=$HF_TOKEN" \ + --ipc=host \ + lmsysorg/sglang:latest \ + python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 1 --random-output 512 --random-range-ratio 1 --num-prompts 1 --host 0.0.0.0 --port 40000 --output-file "deepseekv3_multinode.jsonl" +``` + +> **Note that the launch command here does not enable Data Parallelism Attention or `torch.compile` Optimization**. For optimal performance, please refer to the command options in [Performance Optimization Options](#option_args). + +### Example: Serving with four A100\*8 nodes + +To serve DeepSeek-V3 with A100 GPUs, we need to convert the [FP8 model checkpoints](https://huggingface.co/deepseek-ai/DeepSeek-V3) to BF16 with [script](https://github.com/deepseek-ai/DeepSeek-V3/blob/main/inference/fp8_cast_bf16.py) mentioned [here](https://github.com/deepseek-ai/DeepSeek-V3/blob/main/inference/fp8_cast_bf16.py) first. + +Since the BF16 model is over 1.3 TB, we need to prepare four A100 nodes, each with 8 80GB GPUs. Assume the first node's IP is `10.0.0.1`, and the converted model path is `/path/to/DeepSeek-V3-BF16`, we can have following commands to launch the server. + +```bash +# node 1 +python3 -m sglang.launch_server --model-path /path/to/DeepSeek-V3-BF16 --tp 32 --dist-init-addr 10.0.0.1:5000 --nnodes 4 --node-rank 0 --trust-remote-code --host 0.0.0.0 --port 30000 + +# node 2 +python3 -m sglang.launch_server --model-path /path/to/DeepSeek-V3-BF16 --tp 32 --dist-init-addr 10.0.0.1:5000 --nnodes 4 --node-rank 1 --trust-remote-code + +# node 3 +python3 -m sglang.launch_server --model-path /path/to/DeepSeek-V3-BF16 --tp 32 --dist-init-addr 10.0.0.1:5000 --nnodes 4 --node-rank 2 --trust-remote-code + +# node 4 +python3 -m sglang.launch_server --model-path /path/to/DeepSeek-V3-BF16 --tp 32 --dist-init-addr 10.0.0.1:5000 --nnodes 4 --node-rank 3 --trust-remote-code +``` + +> **Note that the launch command here does not enable Data Parallelism Attention or `torch.compile` Optimization**. For optimal performance, please refer to the command options in [Performance Optimization Options](#option_args). + +Then we can benchmark the accuracy and latency by accessing the first node's exposed port with the following example commands. + +```bash +# bench accuracy +python3 benchmark/gsm8k/bench_sglang.py --num-questions 1319 --host http://10.0.0.1 --port 30000 + +# bench latency +python3 -m sglang.bench_one_batch_server --model None --base-url http://10.0.0.1:30000 --batch-size 1 --input-len 128 --output-len 128 +``` + + +### Example: Serving with 8 A100/A800 with AWQ Quantization + +**Recommended Usage** + +Add `--quantization moe_wna16` flag to enable moe wna16 kernel for better performance. +One example is as follows: + +```bash +python3 -m sglang.launch_server --model cognitivecomputations/DeepSeek-R1-AWQ --tp 8 --trust-remote-code --quantization moe_wna16 +``` + +Alternatively, you can use `--quantization awq_marlin` as follows: + +```bash +python3 -m sglang.launch_server --model cognitivecomputations/DeepSeek-R1-AWQ --tp 8 --trust-remote-code --quantization awq_marlin --dtype float16 +``` + +Note that `awq_marlin` only supports `float16` now, which may lead to some precision loss. + +### Example: Serving with 16 A100/A800 with int8 Quantization + +There are block-wise and per-channel quantization methods, and the quantization parameters have already been uploaded to Huggingface. One example is as follows: + +- [meituan/DeepSeek-R1-Block-INT8](https://huggingface.co/meituan/DeepSeek-R1-Block-INT8) +- [meituan/DeepSeek-R1-Channel-INT8](https://huggingface.co/meituan/DeepSeek-R1-Channel-INT8) + +Assuming that master node IP is `MASTER_IP`, checkpoint path is `/path/to/DeepSeek-R1-INT8` and port=5000, we can have following commands to launch the server: +```bash +#master +python3 -m sglang.launch_server \ + --model meituan/DeepSeek-R1-Block-INT8 --tp 16 --dist-init-addr \ + MASTER_IP:5000 --nnodes 2 --node-rank 0 --trust-remote-code --enable-torch-compile --torch-compile-max-bs 8 +#cluster +python3 -m sglang.launch_server \ + --model meituan/DeepSeek-R1-Block-INT8 --tp 16 --dist-init-addr \ + MASTER_IP:5000 --nnodes 2 --node-rank 1 --trust-remote-code --enable-torch-compile --torch-compile-max-bs 8 +``` + +> **Note that the launch command here enables `torch.compile` Optimization**. For optimal performance, please refer to the command options in [Performance Optimization Options](#option_args). + +Then on the **master node**, supposing the ShareGPT data is located at `/path/to/ShareGPT_V3_unfiltered_cleaned_split.json`, you can run the following commands to benchmark the launched server: + +```bash +# bench accuracy +python3 benchmark/gsm8k/bench_sglang.py --num-questions 1319 + +# bench serving +python3 -m sglang.bench_serving --dataset-path /path/to/ShareGPT_V3_unfiltered_cleaned_split.json --dataset-name random --random-input 128 --random-output 128 --num-prompts 1000 --request-rate 128 --random-range-ratio 1.0 +``` + +> **Note: using `--parallel 200` can accelerate accuracy benchmarking**. + +### Example: Serving with 32 L40S with int8 Quantization + +Running with per-channel quantization model: + +- [meituan/DeepSeek-R1-Channel-INT8](https://huggingface.co/meituan/DeepSeek-R1-Channel-INT8) + +Assuming that master node IP is `MASTER_IP`, checkpoint path is `/path/to/DeepSeek-R1-Channel-INT8` and port=5000, we can have following commands to launch the server: + +```bash +#master +python3 -m sglang.launch_server --model meituan/DeepSeek-R1-Channel-INT8 --tp 32 --quantization w8a8_int8 \ + --dist-init-addr MASTER_IP:5000 --nnodes 4 --node-rank 0 --trust-remote \ + --enable-torch-compile --torch-compile-max-bs 32 +#cluster +python3 -m sglang.launch_server --model meituan/DeepSeek-R1-Channel-INT8 --tp 32 --quantization w8a8_int8 \ + --dist-init-addr MASTER_IP:5000 --nnodes 4 --node-rank 1 --trust-remote \ + --enable-torch-compile --torch-compile-max-bs 32 +python3 -m sglang.launch_server --model meituan/DeepSeek-R1-Channel-INT8 --tp 32 --quantization w8a8_int8 \ + --dist-init-addr MASTER_IP:5000 --nnodes 4 --node-rank 2 --trust-remote \ + --enable-torch-compile --torch-compile-max-bs 32 +python3 -m sglang.launch_server --model meituan/DeepSeek-R1-Channel-INT8 --tp 32 --quantization w8a8_int8 \ + --dist-init-addr MASTER_IP:5000 --nnodes 4 --node-rank 3 --trust-remote \ + --enable-torch-compile --torch-compile-max-bs 32 +``` + +The benchmarking method is the same as describted in the previous [16 x A100](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-16-a100a800-with-int8-quantization) example. + +### Example: Serving on any cloud or Kubernetes with SkyPilot + +SkyPilot helps find cheapest available GPUs across any cloud or existing Kubernetes clusters and launch distributed serving with a single command. See details [here](https://github.com/skypilot-org/skypilot/tree/master/llm/deepseek-r1). + +To serve on multiple nodes: + +```bash +git clone https://github.com/skypilot-org/skypilot.git +# Serve on 2 H100/H200x8 nodes +sky launch -c r1 llm/deepseek-r1/deepseek-r1-671B.yaml --retry-until-up +# Serve on 4 A100x8 nodes +sky launch -c r1 llm/deepseek-r1/deepseek-r1-671B-A100.yaml --retry-until-up +``` + +#### Troubleshooting + +If you encounter the following error with fp16/bf16 checkpoint: + +```bash +ValueError: Weight output_partition_size = 576 is not divisible by weight quantization block_n = 128. +``` + +edit your `config.json` and remove the `quantization_config` block. For example: + +```json +"quantization_config": { + "activation_scheme": "dynamic", + "fmt": "e4m3", + "quant_method": "fp8", + "weight_block_size": [128, 128] +}, +``` + +Removing this block typically resolves the error. For more details, see the discussion in [sgl-project/sglang#3491](https://github.com/sgl-project/sglang/issues/3491#issuecomment-2650779851). + +# Example: Serving with 4 H200 with w4fp8 Quantization +There are mixed-precision quantization methods where MoE layers are computed using W4(int)A(FP)8 quantization while the dense layers remain in FP8 precision. Users can run these models efficiently on 4xH200 GPUs (or potentially 8xH100 GPUs), as the pre-quantized weights are already available on Hugging Face. Here's an example: + +```bash +python -m sglang.launch_server --model novita/Deepseek-V3-0324-W4AFP8 --mem-fraction-static 0.85 --disable-shared-experts-fusion --tp-size 4 +``` + +Other variants of pre-quantized DeepSeek models are also available: + +- [novita/Deepseek-V3.1-W4AFP8](https://huggingface.co/novita/Deepseek-V3.1-W4AFP8) +- [novita/Deepseek-R1-0528-W4AFP8](https://huggingface.co/novita/Deepseek-R1-0528-W4AFP8) +- [novita/Deepseek-R1-W4AFP8](https://huggingface.co/novita/Deepseek-R1-W4AFP8) +- [novita/Deepseek-V3-0324-W4AFP8](https://huggingface.co/novita/Deepseek-V3-0324-W4AFP8) + + +## DeepSeek V3 Optimization Plan + +https://github.com/sgl-project/sglang/issues/2591 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/dspy/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/dspy/README.md new file mode 100644 index 00000000..a2b213aa --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/dspy/README.md @@ -0,0 +1,51 @@ +## Install + +``` +pip3 install dspy-ai +``` + +Turn off cache at https://github.com/stanfordnlp/dspy/blob/34d8420383ec752037aa271825c1d3bf391e1277/dsp/modules/cache_utils.py#L10. +``` +cache_turn_on = False +``` + +or set the environment variable + +``` +export DSP_CACHEBOOL=false +``` + +## Benchmark SGLang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_dspy_intro.py --backend sglang +``` + + +## Benchmark TGI +``` +docker run --name tgi --rm -ti --gpus all --network host \ + -v /home/ubuntu/model_weights/Llama-2-7b-chat-hf:/Llama-2-7b-chat-hf \ + ghcr.io/huggingface/text-generation-inference:1.3.0 \ + --model-id /Llama-2-7b-chat-hf --num-shard 1 --trust-remote-code \ + --max-input-length 2048 --max-total-tokens 4096 \ + --port 24000 +``` + +``` +python3 bench_dspy_intro.py --backend tgi +``` + + + +## Benchmark vLLM +``` +python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_dspy_intro.py --backend vllm +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/dspy/bench_dspy_intro.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/dspy/bench_dspy_intro.py new file mode 100644 index 00000000..2b0936ed --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/dspy/bench_dspy_intro.py @@ -0,0 +1,192 @@ +""" +Adapted from +https://github.com/stanfordnlp/dspy/blob/34d8420383ec752037aa271825c1d3bf391e1277/intro.ipynb#L9 +""" + +import argparse + +import dspy +from dspy.datasets import HotPotQA + + +class BasicQA(dspy.Signature): + """Answer questions with short factoid answers.""" + + question = dspy.InputField() + answer = dspy.OutputField(desc="often between 1 and 5 words") + + +class GenerateAnswer(dspy.Signature): + """Answer questions with short factoid answers.""" + + context = dspy.InputField(desc="may contain relevant facts") + question = dspy.InputField() + answer = dspy.OutputField(desc="often between 1 and 5 words") + + +class RAG(dspy.Module): + def __init__(self, num_passages=3): + super().__init__() + + self.retrieve = dspy.Retrieve(k=num_passages) + self.generate_answer = dspy.ChainOfThought(GenerateAnswer) + + def forward(self, question): + context = self.retrieve(question).passages + prediction = self.generate_answer(context=context, question=question) + return dspy.Prediction(context=context, answer=prediction.answer) + + +def main(args): + # lm = dspy.OpenAI(model='gpt-3.5-turbo') + if args.backend == "tgi": + lm = dspy.HFClientTGI( + model="meta-llama/Llama-2-7b-chat-hf", + port=args.port, + url="http://localhost", + ) + elif args.backend == "sglang": + lm = dspy.HFClientSGLang( + model="meta-llama/Llama-2-7b-chat-hf", + port=args.port, + url="http://localhost", + ) + elif args.backend == "vllm": + lm = dspy.HFClientVLLM( + model="meta-llama/Llama-2-7b-chat-hf", + port=args.port, + url="http://localhost", + ) + else: + raise ValueError(f"Invalid backend: {args.backend}") + + colbertv2_wiki17_abstracts = dspy.ColBERTv2( + url="http://20.102.90.50:2017/wiki17_abstracts" + ) + dspy.settings.configure(lm=lm, rm=colbertv2_wiki17_abstracts) + + # Load the dataset. + dataset = HotPotQA( + train_seed=1, train_size=20, eval_seed=2023, dev_size=args.dev_size, test_size=0 + ) + + # Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata. + trainset = [x.with_inputs("question") for x in dataset.train] + devset = [x.with_inputs("question") for x in dataset.dev] + + print(len(trainset), len(devset)) + + train_example = trainset[0] + print(f"Question: {train_example.question}") + print(f"Answer: {train_example.answer}") + + dev_example = devset[18] + print(f"Question: {dev_example.question}") + print(f"Answer: {dev_example.answer}") + print(f"Relevant Wikipedia Titles: {dev_example.gold_titles}") + + print( + f"For this dataset, training examples have input keys {train_example.inputs().keys()} and label keys {train_example.labels().keys()}" + ) + print( + f"For this dataset, dev examples have input keys {dev_example.inputs().keys()} and label keys {dev_example.labels().keys()}" + ) + + # Define the predictor. + generate_answer = dspy.Predict(BasicQA) + + # Call the predictor on a particular input. + pred = generate_answer(question=dev_example.question) + + # Print the input and the prediction. + print(f"Question: {dev_example.question}") + print(f"Predicted Answer: {pred.answer}") + + lm.inspect_history(n=1) + + # Define the predictor. Notice we're just changing the class. The signature BasicQA is unchanged. + generate_answer_with_chain_of_thought = dspy.ChainOfThought(BasicQA) + + # Call the predictor on the same input. + pred = generate_answer_with_chain_of_thought(question=dev_example.question) + + # Print the input, the chain of thought, and the prediction. + print(f"Question: {dev_example.question}") + print(f"Thought: {pred.rationale.split('.', 1)[1].strip()}") + print(f"Predicted Answer: {pred.answer}") + + retrieve = dspy.Retrieve(k=3) + topK_passages = retrieve(dev_example.question).passages + + print( + f"Top {retrieve.k} passages for question: {dev_example.question} \n", + "-" * 30, + "\n", + ) + + for idx, passage in enumerate(topK_passages): + print(f"{idx+1}]", passage, "\n") + + retrieve("When was the first FIFA World Cup held?").passages[0] + + from dspy.teleprompt import BootstrapFewShot + + # Validation logic: check that the predicted answer is correct. + # Also check that the retrieved context does actually contain that answer. + def validate_context_and_answer(example, pred, trace=None): + answer_EM = dspy.evaluate.answer_exact_match(example, pred) + answer_PM = dspy.evaluate.answer_passage_match(example, pred) + return answer_EM and answer_PM + + # Set up a basic teleprompter, which will compile our RAG program. + teleprompter = BootstrapFewShot(metric=validate_context_and_answer) + + # Compile! + compiled_rag = teleprompter.compile(RAG(), trainset=trainset) + + # Ask any question you like to this simple RAG program. + my_question = "What castle did David Gregory inherit?" + + # Get the prediction. This contains `pred.context` and `pred.answer`. + pred = compiled_rag(my_question) + + # Print the contexts and the answer. + print(f"Question: {my_question}") + print(f"Predicted Answer: {pred.answer}") + print(f"Retrieved Contexts (truncated): {[c[:200] + '...' for c in pred.context]}") + + from dspy.evaluate.evaluate import Evaluate + + # Set up the `evaluate_on_hotpotqa` function. We'll use this many times below. + evaluate_on_hotpotqa = Evaluate( + devset=devset, + num_threads=args.num_threads, + display_progress=True, + display_table=5, + ) + + # Evaluate the `compiled_rag` program with the `answer_exact_match` metric. + metric = dspy.evaluate.answer_exact_match + evaluate_on_hotpotqa(compiled_rag, metric=metric) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--port", type=int) + parser.add_argument("--num-threads", type=int, default=32) + parser.add_argument("--dev-size", type=int, default=150) + parser.add_argument( + "--backend", type=str, choices=["sglang", "tgi", "vllm"], default="sglang" + ) + args = parser.parse_args() + + if args.port is None: + default_port = { + "vllm": 21000, + "lightllm": 22000, + "tgi": 24000, + "sglang": 30000, + } + args.port = default_port.get(args.backend, None) + + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/generative_agents/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/generative_agents/README.md new file mode 100644 index 00000000..393a9ce8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/generative_agents/README.md @@ -0,0 +1,38 @@ +## Download the dataset + +``` +wget -O agent_calls.jsonl https://drive.google.com/uc?export=download&id=19qLpD45e9JGTKF2cUjJJegwzSUEZEKht +``` + +## Run benchmark + +Ensure that this benchmark is run in a serial manner (using --parallel 1) to preserve any potential dependencies between requests. + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-events 1000 --parallel 1 +``` + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-events 1000 --backend vllm --parallel 1 +``` + +### Benchmark guidance +``` +python3 bench_other.py --num-events 1000 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +### Benchmark lmql + +``` +python3 bench_other.py --num-events 1000 --backend lmql --parallel 1 +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/generative_agents/agent_functions.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/generative_agents/agent_functions.py new file mode 100644 index 00000000..9785fc1e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/generative_agents/agent_functions.py @@ -0,0 +1,300 @@ +import sglang as sgl + +# here are the top five agent functions contributing ~70% LLM calls +# reference: https://github.com/joonspk-research/generative_agents/ + + +@sgl.function +def poignancy_event(s, persona_name, persona_iss, event): + s += "Here is a brief description of " + persona_name + ".\n" + s += persona_iss + "\n" + s += "On the scale of 1 to 10, where 1 is purely mundane (e.g., brushing teeth, making bed) and 10 is extremely poignant (e.g., a break up, college acceptance), rate the likely poignancy of the following event for" + s += persona_name + ".\n\n" + s += "Event: " + event + s += "Rate (return a number between 1 to 10):" + s += sgl.gen(name="Rate", max_tokens=2) + + +def poignancy_event_prompt(persona_name, persona_iss, event): + # return prompt and max_tokens + s = "" + s += "Here is a brief description of " + persona_name + ".\n" + s += persona_iss + "\n" + s += "On the scale of 1 to 10, where 1 is purely mundane (e.g., brushing teeth, making bed) and 10 is extremely poignant (e.g., a break up, college acceptance), rate the likely poignancy of the following event for" + s += persona_name + ".\n\n" + s += "Event: " + event + s += "Rate (return a number between 1 to 10):" + return {"prompt": s, "max_tokens": 2, "stop": None} + + +@sgl.function +def generate_event_triple(s, persona_name, action): + s += """Task: Turn the input into (subject, predicate, object). +Input: Sam Johnson is eating breakfast. +Output: (Dolores Murphy, eat, breakfast) +--- +Input: Joon Park is brewing coffee. +Output: (Joon Park, brew, coffee) +--- +Input: Jane Cook is sleeping. +Output: (Jane Cook, is, sleep) +--- +Input: Michael Bernstein is writing email on a computer. +Output: (Michael Bernstein, write, email) +--- +Input: Percy Liang is teaching students in a classroom. +Output: (Percy Liang, teach, students) +--- +Input: Merrie Morris is running on a treadmill. +Output: (Merrie Morris, run, treadmill) +---""" + s += persona_name + "is" + action + ".\n" + s += "(" + persona_name + "," + s += sgl.gen(name="Triple", max_tokens=20, stop=")") + + +def generate_event_triple_prompt(persona_name, action): + s = "" + s += """Task: Turn the input into (subject, predicate, object). +Input: Sam Johnson is eating breakfast. +Output: (Dolores Murphy, eat, breakfast) +--- +Input: Joon Park is brewing coffee. +Output: (Joon Park, brew, coffee) +--- +Input: Jane Cook is sleeping. +Output: (Jane Cook, is, sleep) +--- +Input: Michael Bernstein is writing email on a computer. +Output: (Michael Bernstein, write, email) +--- +Input: Percy Liang is teaching students in a classroom. +Output: (Percy Liang, teach, students) +--- +Input: Merrie Morris is running on a treadmill. +Output: (Merrie Morris, run, treadmill) +---""" + s += persona_name + "is" + action + ".\n" + s += "(" + persona_name + "," + return {"prompt": s, "max_tokens": 20, "stop": ")"} + + +@sgl.function +def generate_pronunciatio(s, action): + s += "Convert an action description to an emoji (important: use two or less emojis).\n" + s += "Action description: " + action + ".\n" + s += "Emoji:" + sgl.gen(name="Emoji", max_tokens=6) + + +def generate_pronunciatio_prompt(action): + s = "" + s += "Convert an action description to an emoji (important: use two or less emojis).\n" + s += "Action description: " + action + ".\n" + s += "Emoji:" + return {"prompt": s, "max_tokens": 6, "stop": None} + + +@sgl.function +def action_location_sector( + s, + persona_name, + living_sector, + living_sector_areas, + current_sector, + current_sector_areas, + daily_plan, + sector_options, + current_action, + next_action, +): + s += """Task -- choose an appropriate area from the area options for a task at hand. +Sam Kim lives in {Sam Kim's house} that has Sam Kim's room, bathroom, kitchen. +Sam Kim is currently in {Sam Kim's house} that has Sam Kim's room, bathroom, kitchen. +Area options: {Sam Kim's house, The Rose and Crown Pub, Hobbs Cafe, Oak Hill College, Johnson Park, Harvey Oak Supply Store, The Willows Market and Pharmacy}. +* Stay in the current area if the activity can be done there. Only go out if the activity needs to take place in another place. +* Must be one of the "Area options," verbatim. +For taking a walk, Sam Kim should go to the following area: {Johnson Park} +--- +Jane Anderson lives in {Oak Hill College Student Dormatory} that has Jane Anderson's room. +Jane Anderson is currently in {Oak Hill College} that has a classroom, library +Area options: {Oak Hill College Student Dormatory, The Rose and Crown Pub, Hobbs Cafe, Oak Hill College, Johnson Park, Harvey Oak Supply Store, The Willows Market and Pharmacy}. +* Stay in the current area if the activity can be done there. Only go out if the activity needs to take place in another place. +* Must be one of the "Area options," verbatim. +For eating dinner, Jane Anderson should go to the following area: {Hobbs Cafe} +---""" + s += ( + persona_name + + " lives in " + + living_sector + + " that has " + + living_sector_areas + + ".\n" + ) + s += ( + persona_name + + " is currently in " + + current_sector + + " that has " + + current_sector_areas + + ".\n" + ) + s += daily_plan + ".\n" + s += "Area options: " + sector_options + ".\n" + s += """* Stay in the current area if the activity can be done there. Only go out if the activity needs to take place in another place. +* Must be one of the "Area options," verbatim.\n""" + s += ( + persona_name + + " is " + + current_action + + ". For " + + next_action + + ", " + + persona_name + + " should go to the following area: {" + ) + s += sgl.gen(name="Location", max_tokens=10, stop="}") + + +def action_location_sector_prompt( + persona_name, + living_sector, + living_sector_areas, + current_sector, + current_sector_areas, + daily_plan, + sector_options, + current_action, + next_action, +): + s = "" + s += """Task -- choose an appropriate area from the area options for a task at hand. +Sam Kim lives in {Sam Kim's house} that has Sam Kim's room, bathroom, kitchen. +Sam Kim is currently in {Sam Kim's house} that has Sam Kim's room, bathroom, kitchen. +Area options: {Sam Kim's house, The Rose and Crown Pub, Hobbs Cafe, Oak Hill College, Johnson Park, Harvey Oak Supply Store, The Willows Market and Pharmacy}. +* Stay in the current area if the activity can be done there. Only go out if the activity needs to take place in another place. +* Must be one of the "Area options," verbatim. +For taking a walk, Sam Kim should go to the following area: {Johnson Park} +--- +Jane Anderson lives in {Oak Hill College Student Dormatory} that has Jane Anderson's room. +Jane Anderson is currently in {Oak Hill College} that has a classroom, library +Area options: {Oak Hill College Student Dormatory, The Rose and Crown Pub, Hobbs Cafe, Oak Hill College, Johnson Park, Harvey Oak Supply Store, The Willows Market and Pharmacy}. +* Stay in the current area if the activity can be done there. Only go out if the activity needs to take place in another place. +* Must be one of the "Area options," verbatim. +For eating dinner, Jane Anderson should go to the following area: {Hobbs Cafe} +---""" + s += ( + persona_name + + " lives in " + + living_sector + + " that has " + + living_sector_areas + + ".\n" + ) + s += ( + persona_name + + " is currently in " + + current_sector + + " that has " + + current_sector_areas + + ".\n" + ) + s += daily_plan + ".\n" + s += "Area options: " + sector_options + ".\n" + s += """* Stay in the current area if the activity can be done there. Only go out if the activity needs to take place in another place. +* Must be one of the "Area options," verbatim.\n""" + s += ( + persona_name + + " is " + + current_action + + ". For " + + next_action + + ", " + + persona_name + + " should go to the following area: {" + ) + return {"prompt": s, "max_tokens": 10, "stop": "}"} + + +@sgl.function +def action_location_object( + s, persona_name, target_sector, target_sector_areas, current_action, next_action +): + s += """ +Jane Anderson is in kitchen in Jane Anderson's house. +Jane Anderson is going to Jane Anderson's house that has the following areas: {kitchen, bedroom, bathroom} +Stay in the current area if the activity can be done there. Never go into other people's rooms unless necessary. +For cooking, Jane Anderson should go to the following area in Jane Anderson's house: +Answer: {kitchen} +--- +Tom Watson is in common room in Tom Watson's apartment. +Tom Watson is going to Hobbs Cafe that has the following areas: {cafe} +Stay in the current area if the activity can be done there. Never go into other people's rooms unless necessary. +For getting coffee, Tom Watson should go to the following area in Hobbs Cafe: +Answer: {cafe} +---""" + s += ( + persona_name + + " is going to " + + target_sector + + " that has the following areas: {" + + target_sector_areas + + "}\n" + ) + s += """* Stay in the current area if the activity can be done there. +* NEVER go into other people's rooms unless necessary.""" + s += ( + persona_name + + " is " + + current_action + + ". For " + + next_action + + ", " + + persona_name + + "should go to the following area in " + + target_sector + ) + s += " (MUST pick one of {" + target_sector_areas + "}):\n" + s += "Answer: {" + sgl.gen(name="Area", max_tokens=5, stop="}") + + +def action_location_object_prompt( + persona_name, target_sector, target_sector_areas, current_action, next_action +): + s = "" + s += """ +Jane Anderson is in kitchen in Jane Anderson's house. +Jane Anderson is going to Jane Anderson's house that has the following areas: {kitchen, bedroom, bathroom} +Stay in the current area if the activity can be done there. Never go into other people's rooms unless necessary. +For cooking, Jane Anderson should go to the following area in Jane Anderson's house: +Answer: {kitchen} +--- +Tom Watson is in common room in Tom Watson's apartment. +Tom Watson is going to Hobbs Cafe that has the following areas: {cafe} +Stay in the current area if the activity can be done there. Never go into other people's rooms unless necessary. +For getting coffee, Tom Watson should go to the following area in Hobbs Cafe: +Answer: {cafe} +---""" + s += ( + persona_name + + " is going to " + + target_sector + + " that has the following areas: {" + + target_sector_areas + + "}\n" + ) + s += """* Stay in the current area if the activity can be done there. +* NEVER go into other people's rooms unless necessary.""" + s += ( + persona_name + + " is " + + current_action + + ". For " + + next_action + + ", " + + persona_name + + "should go to the following area in " + + target_sector + ) + s += " (MUST pick one of {" + target_sector_areas + "}):\n" + s += "Answer: {" + return {"prompt": s, "max_tokens": 5, "stop": "}"} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/generative_agents/bench_other.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/generative_agents/bench_other.py new file mode 100644 index 00000000..c0b3a340 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/generative_agents/bench_other.py @@ -0,0 +1,80 @@ +import argparse +import json +import time + +from agent_functions import ( + action_location_object_prompt, + action_location_sector_prompt, + generate_event_triple_prompt, + generate_pronunciatio_prompt, + poignancy_event_prompt, +) +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_events] + mapping = { + "poignancy_event": poignancy_event_prompt, + "generate_event_triple": generate_event_triple_prompt, + "generate_pronunciatio": generate_pronunciatio_prompt, + "action_location_sector": action_location_sector_prompt, + "action_location_object": action_location_object_prompt, + } + + arguments = [mapping[k](**v) for l in lines for k, v in l.items()] + states = [] + + # Select backend + call_generate = get_call_generate(args) + + def get_one_answer(arg): + answer = call_generate(**arg, temperature=0) + states.append(answer) + + async def get_one_answer_async(arg): + answer = await call_generate(**arg, temperature=0) + states.append(answer) + + tic = time.perf_counter() + # we always sequentially execute agent calls to maintain its dependency + if args.backend != "lmql": + for arg in tqdm(arguments): + get_one_answer(arg) + else: + import asyncio + + loop = asyncio.get_event_loop() + for arg in tqdm(arguments): + loop.run_until_complete(get_one_answer_async(arg)) + latency = time.perf_counter() - tic + + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "Generative Agents", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + # to pack weighted functions as a single agent + "num_requests": len(arguments) / len(mapping), + "other": { + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="agent_calls.jsonl") + parser.add_argument("--num-events", type=int, default=10) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/generative_agents/bench_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/generative_agents/bench_sglang.py new file mode 100644 index 00000000..034b1659 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/generative_agents/bench_sglang.py @@ -0,0 +1,74 @@ +import argparse +import json +import time + +from agent_functions import ( + action_location_object, + action_location_sector, + generate_event_triple, + generate_pronunciatio, + poignancy_event, +) + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_events] + mapping = { + "poignancy_event": poignancy_event, + "generate_event_triple": generate_event_triple, + "generate_pronunciatio": generate_pronunciatio, + "action_location_sector": action_location_sector, + "action_location_object": action_location_object, + } + arguments = [{mapping[k]: v for k, v in l.items()} for l in lines] + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + states = [] + # Run requests + tic = time.perf_counter() + for a in arguments: + # only a single key in the dict + for func, arg in a.items(): + result = func.run(**arg) + result.sync() + states.append(result) + latency = time.perf_counter() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "Generative Agents", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + # to pack weighted functions as a single agent + "num_requests": len(arguments) / len(mapping), + "other": { + "num_events": args.num_events, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="agent_calls.jsonl") + parser.add_argument("--num-events", type=int, default=10) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/gpt_oss/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/gpt_oss/README.md new file mode 100644 index 00000000..4d1b00e9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/gpt_oss/README.md @@ -0,0 +1,163 @@ +# How to reproduce the result of GPT-OSS with SGLang + +### Install the latest SGLang + +```bash +git clone https://github.com/sgl-project/sglang.git +cd sglang +git checkout v0.5.1.post3 + +pip install --upgrade pip +pip install -e "python[all]" +``` + +### Reproduce the benchmark throughput result (Batch Size 1) + +Launch Command + +```bash +# MXFP4 120B on H100 +python3 -m sglang.launch_server --model openai/gpt-oss-120b --tp 8 --attention-backend triton + +# BF16 120B on H100 +python3 -m sglang.launch_server --model lmsys/gpt-oss-120b-bf16 --tp 8 --attention-backend triton + +# MXFP4 120B on B200 +python3 -m sglang.launch_server --model openai/gpt-oss-120b --tp 4 + +# BF16 120B on B200 +python3 -m sglang.launch_server --model lmsys/gpt-oss-120b-bf16 --tp 4 +``` + +Benchmark Command + +```bash + +# MXFP4 120B on H100 +python3 -m sglang.bench_one_batch_server --model openai/gpt-oss-120b --base-url http://localhost:30000 --batch-size 1 --input-len 1024 --output-len 512 --show-report +``` + +### Reproduce the benchmark throughput result (Batch Size 32) + +Launch Command + +```bash +# MXFP4 120B on H100 +python3 -m sglang.launch_server --model openai/gpt-oss-120b --tp 8 + +# BF16 120B on H100 +python3 -m sglang.launch_server --model lmsys/gpt-oss-120b-bf16 --tp 8 + +# MXFP4 120B on B200 +python3 -m sglang.launch_server --model openai/gpt-oss-120b --tp 4 + +# BF16 120B on B200 +python3 -m sglang.launch_server --model lmsys/gpt-oss-120b-bf16 --tp 4 +``` + +Benchmark Command + +```bash +python3 -m sglang.bench_one_batch_server --model openai/gpt-oss-120b --base-url http://localhost:30000 --batch-size 32 --input-len 1024 8192 --output-len 512 --show-report +``` + +### Reproduce the evaluation result + +Install gpt-oss + +```bash +git clone https://github.com/openai/gpt-oss.git +cd gpt-oss +pip install -e . +``` + +Evaluation Command + +```bash +DATASET=gpqa +BASE_URL=YOUR_BASE_URL +OPENAI_API_KEY=dummy python -m gpt_oss.evals \ + --base-url ${BASE_URL}/v1 \ + --model dummy \ + --reasoning-effort low,medium,high \ + --eval $DATASET \ + --n-threads 1000 +``` + +### Reproduce the benchmark result of acceptance length +> Note: On B200, if top k is 1, set `--attention-backend trtllm_mha` +```bash +git clone https://github.com/sgl-project/SpecForge.git +cd SpecForge/benchmarks +config_list=( + "1,0,0,0" + "1,3,1,4" + "1,5,4,8" +) +python3 bench_model_speedup.py \ + --model-path openai/gpt-oss-120b \ + --speculative-draft-model-path lmsys/EAGLE3-gpt-oss-120b-bf16 \ + --port 20001 \ + --trust-remote-code \ + --mem-fraction-static 0.8 \ + --tp-size 4 \ + --attention-backend fa3 \ + --config-list "${config_list[@]}" \ + --benchmark-list mtbench:80 gsm8k:200 humaneval:200 math500:200 \ + --output lmsys_gpt-oss-120b_Eagle3_result.jsonl + +python3 bench_model_speedup.py \ + --model-path openai/gpt-oss-120b \ + --speculative-draft-model-path nvidia/gpt-oss-120b-Eagle3 \ + --port 20001 \ + --trust-remote-code \ + --mem-fraction-static 0.8 \ + --tp-size 4 \ + --attention-backend fa3 \ + --config-list "${config_list[@]}" \ + --benchmark-list mtbench:80 gsm8k:200 humaneval:200 math500:200 \ + --output nv_gpt-oss-120b_Eagle3_result.jsonl +``` + +### Reproduce the result of speculative decoding speedup + +Launch Command + +```bash +# On Hopper: +# - Tree decoding (topk > 1) and chain decoding (topk = 1) are supported on both FA3 and Triton backends. +python3 -m sglang.launch_server --model openai/gpt-oss-120b --speculative-algorithm EAGLE3 --speculative-draft-model-path lmsys/EAGLE3-gpt-oss-120b-bf16 --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --tp 4 +python3 -m sglang.launch_server --model openai/gpt-oss-120b --speculative-algorithm EAGLE3 --speculative-draft-model-path lmsys/EAGLE3-gpt-oss-120b-bf16 --speculative-num-steps 5 --speculative-eagle-topk 4 --speculative-num-draft-tokens 8 --tp 4 + +# On Blackwell: +# - Chain decoding (topk = 1) is supported on TRTLLM-MHA backend. Tree decoding (topk > 1) is in progress, stay tuned! +# - Both tree decoding (topk > 1) and chain decoding (topk = 1) are supported on the Triton backend. +python3 -m sglang.launch_server --model openai/gpt-oss-120b --speculative-algo EAGLE3 --speculative-draft-model-path lmsys/EAGLE3-gpt-oss-120b-bf16 --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --tp 4 +python3 -m sglang.launch_server --model openai/gpt-oss-120b --speculative-algo EAGLE3 --speculative-draft-model-path lmsys/EAGLE3-gpt-oss-120b-bf16 --speculative-num-steps 5 --speculative-eagle-topk 4 --speculative-num-draft-tokens 8 --attention-backend triton --tp 4 +``` + +Benchmark Command + +```bash +config_list=( + "1,0,0,0" + "1,3,1,4" + "1,5,4,8" +) +python3 bench_model_speedup.py \ + --model-path openai/gpt-oss-120b \ + --speculative-draft-model-path lmsys/EAGLE3-gpt-oss-120b-bf16 \ + --port 20001 \ + --trust-remote-code \ + --mem-fraction-static 0.8 \ + --tp-size 4 \ + --attention-backend fa3 \ + --config-list "${config_list[@]}" \ + --benchmark-list gsm8k:200 humaneval:200 math500:200 \ + --output lmsys_gpt-oss-120b_Eagle3_result.jsonl +``` + +We can gain the best speedup with the following settings: + +- **1.39x** speedup with the `--speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4` setting. +- **1.52x** speedup with the `--speculative-num-steps 5 --speculative-eagle-topk 4 --speculative-num-draft-tokens 8` setting. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/gsm8k/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/gsm8k/README.md new file mode 100644 index 00000000..c110f533 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/gsm8k/README.md @@ -0,0 +1,47 @@ +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 200 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-questions 200 --backend vllm +``` + + +### Benchmark lightllm +``` +# A10G +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 16000 --port 22000 +``` + +``` +python3 bench_other.py --num-questions 200 --backend lightllm +``` + + +### Benchmark guidance +``` +python3 bench_other.py --num-questions 200 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + + +### Benchmark lmql +``` +CUDA_VISIBLE_DEVICES=0,1 lmql serve-model meta-llama/Llama-2-7b-chat-hf --cuda --port 23000 +``` + +``` +python3 bench_other.py --num-questions 100 --backend lmql --parallel 2 +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/gsm8k/bench_other.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/gsm8k/bench_other.py new file mode 100644 index 00000000..6dcb9ad7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/gsm8k/bench_other.py @@ -0,0 +1,151 @@ +import argparse +import ast +import asyncio +import json +import re +import time +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import download_and_cache_file, dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_one_example(lines, i, include_answer): + ret = "Question: " + lines[i]["question"] + "\nAnswer:" + if include_answer: + ret += " " + lines[i]["answer"] + return ret + + +def get_few_shot_examples(lines, k): + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def main(args): + # Select backend + call_generate = get_call_generate(args) + + # Read data + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + filename = download_and_cache_file(url) + lines = list(read_jsonl(filename)) + + # Construct prompts + num_questions = args.num_questions + num_shots = args.num_shots + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + + states = [None] * len(labels) + + # Run requests + if args.backend != "lmql": + # Use thread pool + def get_one_answer(i): + answer = call_generate( + prompt=few_shot_examples + questions[i], + temperature=0, + max_tokens=256, + stop=["Question", "Assistant:", "<|separator|>"], + ) + states[i] = answer + + tic = time.perf_counter() + if args.parallel == 1: + for i in tqdm(range(len(questions))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(questions)))), + total=len(questions), + ) + ) + + else: + # Use asyncio + async def batched_call(batch_size): + for i in range(0, len(questions), batch_size): + tasks = [] + for q in questions[i : i + batch_size]: + tasks.append( + call_generate( + few_shot_examples + q, + temperature=0, + max_tokens=256, + stop="Question", + ) + ) + rets = await asyncio.gather(*tasks) + for j in range(len(rets)): + states[i + j] = rets[j] + + tic = time.perf_counter() + asyncio.run(batched_call(batch_size=args.parallel)) + latency = time.perf_counter() - tic + + preds = [] + for i in range(len(states)): + preds.append(get_answer_value(states[i])) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + + # Print results + print(f"Accuracy: {acc:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Latency: {latency:.3f} s") + + # Dump results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--num-shots", type=int, default=5) + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/gsm8k/bench_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/gsm8k/bench_sglang.py new file mode 100644 index 00000000..9cdc2cf8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/gsm8k/bench_sglang.py @@ -0,0 +1,148 @@ +import argparse +import ast +import json +import os +import re +import time + +import numpy as np + +from sglang.lang.api import set_default_backend +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + dump_bench_raw_result, + select_sglang_backend, +) +from sglang.utils import download_and_cache_file, dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_one_example(lines, i, include_answer): + ret = "Question: " + lines[i]["question"] + "\nAnswer:" + if include_answer: + ret += " " + lines[i]["answer"] + return ret + + +def get_few_shot_examples(lines, k): + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def main(args): + # Select backend + set_default_backend(select_sglang_backend(args)) + + # Read data + data_path = args.data_path + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + if not os.path.isfile(data_path): + data_path = download_and_cache_file(url) + lines = list(read_jsonl(data_path)) + + # Construct prompts + num_questions = args.num_questions + num_shots = args.num_shots + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q} for q in questions] + + ##################################### + ######### SGL Program Begin ######### + ##################################### + + import sglang as sgl + + @sgl.function + def few_shot_gsm8k(s, question): + s += few_shot_examples + question + s += sgl.gen( + "answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"] + ) + + ##################################### + ########## SGL Program End ########## + ##################################### + + # Run requests + tic = time.perf_counter() + states = few_shot_gsm8k.run_batch( + arguments, + temperature=0, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + + preds = [] + for i in range(len(states)): + preds.append(get_answer_value(states[i]["answer"])) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + + # Compute speed + num_output_tokens = sum( + s.get_meta_info("answer")["completion_tokens"] for s in states + ) + output_throughput = num_output_tokens / latency + + # Print results + print(f"Accuracy: {acc:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Latency: {latency:.3f} s") + print(f"Output throughput: {output_throughput:.3f} token/s") + + # Dump results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + dump_bench_raw_result( + path=args.raw_result_file, + states=states, + preds=preds, + labels=labels, + ) + + with open(args.result_file, "a") as fout: + value = { + "task": "gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--num-shots", type=int, default=5) + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hellaswag/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hellaswag/README.md new file mode 100644 index 00000000..cb7e6536 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hellaswag/README.md @@ -0,0 +1,47 @@ +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 200 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-questions 200 --backend vllm +``` + + +### Benchmark lightllm +``` +# A10G +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 16000 --port 22000 +``` + +``` +python3 bench_other.py --num-questions 200 --backend lightllm +``` + + +### Benchmark guidance +``` +CUDA_VISIBLE_DEVICES=0,1 python3 bench_other.py --num-questions 200 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + + +### Benchmark lmql +``` +lmql serve-model meta-llama/Llama-2-7b-chat-hf --cuda --port 23000 +``` + +``` +python3 bench_other.py --num-questions 200 --backend lmql --port 23000 --parallel 1 +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hellaswag/bench_other.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hellaswag/bench_other.py new file mode 100644 index 00000000..cde0794b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hellaswag/bench_other.py @@ -0,0 +1,118 @@ +import argparse +import asyncio +import json +import time +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_select +from sglang.utils import download_and_cache_file, read_jsonl + + +def get_one_example(lines, i, include_answer): + ret = lines[i]["activity_label"] + ": " + lines[i]["ctx"] + " " + if include_answer: + ret += lines[i]["endings"][lines[i]["label"]] + return ret + + +def get_few_shot_examples(lines, k): + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def main(args): + # Select backend + call_select = get_call_select(args) + + # Read data + url = "https://raw.githubusercontent.com/rowanz/hellaswag/master/data/hellaswag_val.jsonl" + filename = download_and_cache_file(url) + lines = list(read_jsonl(filename)) + + # Construct prompts + num_questions = args.num_questions + num_shots = args.num_shots + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + choices = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + choices.append(lines[i]["endings"]) + labels.append(lines[i]["label"]) + + preds = [None] * len(labels) + + # Run requests + if args.backend != "lmql": + # Use thread pool + def get_one_answer(i): + preds[i] = call_select( + context=few_shot_examples + questions[i], choices=choices[i] + ) + + tic = time.perf_counter() + if args.parallel == 1: + for i in tqdm(range(len(questions))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(questions)))), + total=len(questions), + ) + ) + else: + # Use asyncio + async def batched_call(batch_size): + for i in range(0, len(questions), batch_size): + tasks = [] + for q, c in zip( + questions[i : i + batch_size], choices[i : i + batch_size] + ): + tasks.append(call_select(context=few_shot_examples + q, choices=c)) + rets = await asyncio.gather(*tasks) + for j in range(len(rets)): + preds[i + j] = rets[j] + + tic = time.perf_counter() + asyncio.run(batched_call(batch_size=args.parallel)) + + latency = time.perf_counter() - tic + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + print(f"Latency: {latency:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + with open(args.result_file, "a") as fout: + value = { + "task": "hellaswag", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--num-shots", type=int, default=20) + parser.add_argument("--data-path", type=str, default="hellaswag_val.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hellaswag/bench_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hellaswag/bench_sglang.py new file mode 100644 index 00000000..2adce99b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hellaswag/bench_sglang.py @@ -0,0 +1,109 @@ +import argparse +import json +import os +import time + +import numpy as np + +from sglang.lang.api import set_default_backend +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import download_and_cache_file, read_jsonl + + +def get_one_example(lines, i, include_answer): + ret = lines[i]["activity_label"] + ": " + lines[i]["ctx"] + " " + if include_answer: + ret += lines[i]["endings"][lines[i]["label"]] + return ret + + +def get_few_shot_examples(lines, k): + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def main(args): + # Select backend + set_default_backend(select_sglang_backend(args)) + + # Read data + data_path = args.data_path + url = "https://raw.githubusercontent.com/rowanz/hellaswag/master/data/hellaswag_val.jsonl" + if not os.path.isfile(data_path): + data_path = download_and_cache_file(url) + lines = list(read_jsonl(data_path)) + + # Construct prompts + num_questions = args.num_questions + num_shots = args.num_shots + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + choices = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + choices.append(lines[i]["endings"]) + labels.append(lines[i]["label"]) + arguments = [{"question": q, "choices": c} for q, c in zip(questions, choices)] + + ##################################### + ######### SGL Program Begin ######### + ##################################### + + import sglang as sgl + + @sgl.function + def few_shot_hellaswag(s, question, choices): + s += few_shot_examples + question + s += sgl.select("answer", choices=choices) + + ##################################### + ########## SGL Program End ########## + ##################################### + + # Run requests + tic = time.perf_counter() + rets = few_shot_hellaswag.run_batch( + arguments, + temperature=0, + num_threads=args.parallel, + progress_bar=True, + ) + preds = [choices[i].index(rets[i]["answer"]) for i in range(len(rets))] + latency = time.perf_counter() - tic + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + print(f"Latency: {latency:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + with open(args.result_file, "a") as fout: + value = { + "task": "hellaswag", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--num-shots", type=int, default=20) + parser.add_argument("--data-path", type=str, default="hellaswag_val.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hf3fs/bench.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hf3fs/bench.sh new file mode 100644 index 00000000..049116b8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hf3fs/bench.sh @@ -0,0 +1,59 @@ +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/python3.12/dist-packages:/usr/local/lib/python3.12/dist-packages/torch/lib +python3 benchmark/hf3fs/bench_client.py + +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/python3.12/dist-packages:/usr/local/lib/python3.12/dist-packages/torch/lib +SGLANG_HICACHE_HF3FS_CONFIG_PATH=/sgl-workspace/sglang/benchmark/hf3fs/hf3fs.json \ +python3 benchmark/hf3fs/bench_storage.py + +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/python3.12/dist-packages:/usr/local/lib/python3.12/dist-packages/torch/lib +export SGLANG_HICACHE_HF3FS_CONFIG_PATH=/sgl-workspace/sglang/benchmark/hf3fs/hf3fs.json +echo '{"file_path_prefix": "/data/hf3fs-test-0", "file_size": 1099511627776, "numjobs": 16, "entries": 8}' > \ +${SGLANG_HICACHE_HF3FS_CONFIG_PATH} +python3 benchmark/hf3fs/bench_zerocopy.py + +#################################################################################################### + +rm -rf nohup.out && \ +nohup python3 -m sglang.launch_server \ + --model-path /code/models/Qwen3-32B/ \ + --host 0.0.0.0 --port 33301 \ + --page-size 64 \ + --enable-hierarchical-cache \ + --hicache-ratio 2 --hicache-size 0 \ + --hicache-write-policy write_through \ + --hicache-storage-backend hf3fs & + +rm -rf bench_multiturn.out && \ +nohup python3 benchmark/hicache/bench_multiturn.py \ + --model-path /code/models/Qwen3-32B \ + --dataset-path /code/models/ShareGPT_V3_unfiltered_cleaned_split.json \ + --port 33301 \ + --request-length 2048 --num-clients 512 --num-rounds 3 --max-parallel 8 \ + > bench_multiturn.out & + +#################################################################################################### + +rm -rf nohup.out && \ +nohup python3 -m sglang.launch_server \ + --model-path /code/models/DeepSeek-R1/ \ + --tp 16 --nnodes 2 --node-rank 0 \ + --dist-init-addr 10.74.249.153:5000 \ + --host 0.0.0.0 --port 33301 \ + --page-size 64 \ + --enable-hierarchical-cache \ + --hicache-ratio 2 --hicache-size 60 \ + --hicache-write-policy write_through \ + --hicache-storage-backend hf3fs & + +rm -rf bench_multiturn.out && \ +nohup python3 benchmark/hicache/bench_multiturn.py \ + --model-path /code/models/Qwen3-32B \ + --dataset-path /code/models/ShareGPT_V3_unfiltered_cleaned_split.json \ + --port 33301 \ + --request-length 2048 --num-clients 1024 --num-rounds 3 --max-parallel 8 \ + > bench_multiturn.out & + +#################################################################################################### + +ps aux | grep "sglang.launch_server" | grep -v grep | awk '{print $2}' | xargs kill -9 +ps aux | grep "bench_multiturn.py" | grep -v grep | awk '{print $2}' | xargs kill -9 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hf3fs/bench_client.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hf3fs/bench_client.py new file mode 100644 index 00000000..0af3c80c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hf3fs/bench_client.py @@ -0,0 +1,162 @@ +import concurrent.futures +import logging +import random +import time +from typing import List + +import torch +from tqdm import tqdm + +from sglang.srt.mem_cache.storage.hf3fs.hf3fs_usrbio_client import Hf3fsUsrBioClient + + +def print_stats(x: List[int]): + x = sorted(x) + lenx = len(x) + print( + f"mean = {sum(x)/len(x):.2f}, " + f"min = {min(x):.2f}, " + f"p25 = {x[int(lenx*0.25)]:.2f}, " + f"p50 = {x[int(lenx*0.5)]:.2f}, " + f"p75 = {x[int(lenx*0.75)]:.2f}, " + f"max = {max(x):.2f}" + ) + + +def test(): + # /path/to/hf3fs + file_path = "/data/bench.bin" + file_size = 1 << 40 + bytes_per_page = 16 << 20 + entries = 32 + file_ops = Hf3fsUsrBioClient(file_path, file_size, bytes_per_page, entries) + + print("test batch_read / batch_write") + num_pages = 128 + dtype = torch.bfloat16 + numel = bytes_per_page // dtype.itemsize + offsets = list(range(file_size // bytes_per_page)) + random.shuffle(offsets) + offsets = offsets[:num_pages] + offsets = [i * bytes_per_page for i in offsets] + tensor_writes = [ + torch.randn(numel, dtype=dtype) + for _ in tqdm(range(num_pages), desc="prepare tensor") + ] + for i in tqdm(range(0, num_pages, file_ops.entries), desc="batch_write"): + results = file_ops.batch_write( + offsets[i : i + file_ops.entries], tensor_writes[i : i + file_ops.entries] + ) + assert all([result == numel * dtype.itemsize for result in results]) + tensor_reads = [ + torch.empty(numel, dtype=dtype) + for _ in tqdm(range(num_pages), desc="prepare tensor") + ] + for i in tqdm(range(0, num_pages, file_ops.entries), desc="batch_read"): + results = file_ops.batch_read( + offsets[i : i + file_ops.entries], tensor_reads[i : i + file_ops.entries] + ) + assert all([result == numel * dtype.itemsize for result in results]) + assert all([torch.allclose(r, w) for r, w in zip(tensor_reads, tensor_writes)]) + + file_ops.close() + print("test done") + + +def bench(): + file_path = "/data/bench.bin" + file_size = 1 << 40 + bytes_per_page = 16 << 20 + entries = 8 + numjobs = 16 + + dtype = torch.bfloat16 + numel = bytes_per_page // dtype.itemsize + + file_ops = [ + Hf3fsUsrBioClient(file_path, file_size, bytes_per_page, entries) + for _ in range(numjobs) + ] + + num_page = entries + + offsets = list(range(file_size // bytes_per_page)) + tensors_write = [torch.randn(numel, dtype=dtype)] * num_page + tensors_read = [torch.empty(numel, dtype=dtype)] * num_page + random.shuffle(offsets) + + warmup = 50 + iteration = 100 + + executor = concurrent.futures.ThreadPoolExecutor(max_workers=numjobs) + + w_bw = [] + w_size = num_page * numjobs * bytes_per_page / (1 << 30) + for i in tqdm(range(warmup + iteration), desc="Benchmarking write (GB/s)"): + _offsets = [ + [ + offset * bytes_per_page + for offset in offsets[ + (i * numjobs + j) * num_page : (i * numjobs + j + 1) * num_page + ] + ] + for j in range(numjobs) + ] + tik = time.perf_counter() + futures = [ + executor.submit(file_ops[j].batch_write, offset, tensors_write) + for j, offset in enumerate(_offsets) + ] + results = [future.result() for future in futures] + tok = time.perf_counter() + if i < warmup: + continue + w_bw.append(w_size / (tok - tik)) + results = [ + _result == bytes_per_page for result in results for _result in result + ] + assert all(results) + print_stats(w_bw) + + r_bw = [] + r_size = w_size + for i in tqdm(range(warmup + iteration), desc="Benchmarking read (GB/s)"): + _offsets = [ + [ + offset * bytes_per_page + for offset in offsets[ + (i * numjobs + j) * num_page : (i * numjobs + j + 1) * num_page + ] + ] + for j in range(numjobs) + ] + tik = time.perf_counter() + futures = [ + executor.submit(file_ops[j].batch_read, offset, tensors_read) + for j, offset in enumerate(_offsets) + ] + results = [future.result() for future in futures] + tok = time.perf_counter() + if i < warmup: + continue + r_bw.append(r_size / (tok - tik)) + results = [ + _result == bytes_per_page for result in results for _result in result + ] + assert all(results) + print_stats(r_bw) + + executor.shutdown(wait=True) + for _file_ops in file_ops: + _file_ops.close() + print("bench done") + + +def main(): + logging.basicConfig(level=logging.INFO) + test() + bench() + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hf3fs/bench_storage.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hf3fs/bench_storage.py new file mode 100644 index 00000000..f0ce171b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hf3fs/bench_storage.py @@ -0,0 +1,258 @@ +import json +import logging +import os +import random +import time +from typing import List + +import torch +from tqdm import tqdm + +from sglang.srt.mem_cache.storage.hf3fs.mini_3fs_metadata_server import ( + Hf3fsLocalMetadataClient, +) +from sglang.srt.mem_cache.storage.hf3fs.storage_hf3fs import HiCacheHF3FS + + +def print_stats(x: List[int]): + x = sorted(x) + lenx = len(x) + print( + f"mean = {sum(x)/len(x):.2f}, " + f"min = {min(x):.2f}, " + f"p25 = {x[int(lenx*0.25)]:.2f}, " + f"p50 = {x[int(lenx*0.5)]:.2f}, " + f"p75 = {x[int(lenx*0.75)]:.2f}, " + f"max = {max(x):.2f}" + ) + + +def test(): + # Qwen3-32B + layer_num = 64 + head_num, head_dim = 8, 128 + kv_lora_rank, qk_rope_head_dim = 0, 0 + store_dtype = torch.bfloat16 + tokens_per_page = 64 + + file_path_prefix = "/data/test" + file_size = 128 << 20 + numjobs = 16 + bytes_per_page = 16 << 20 + entries = 2 + dtype = store_dtype + + config_path = os.getenv(HiCacheHF3FS.default_env_var) + assert config_path + try: + with open(config_path, "w") as f: + json.dump( + { + "file_path_prefix": file_path_prefix, + "file_size": file_size, + "numjobs": numjobs, + "entries": entries, + }, + f, + ) + except Exception as e: + raise RuntimeError(f"Failed to dump config to {config_path}: {str(e)}") + hicache_hf3fs = HiCacheHF3FS.from_env_config(bytes_per_page, dtype) + + numel = 2 * tokens_per_page * layer_num * head_num * head_dim + assert numel * dtype.itemsize == bytes_per_page + + num_pages = 10 + tensors = {} + for i in range(num_pages): + k = f"key_{i}" + v = torch.randn((numel,)).to(dtype=dtype) + ok = hicache_hf3fs.set(k, v) + if i < (file_size // bytes_per_page): + assert ok, f"Failed to insert {k}" + else: + assert not ok + tensors[k] = v + assert hicache_hf3fs.get("key_8") is None + assert hicache_hf3fs.get("key_9") is None + + start = 0 + for i in range(start, start + hicache_hf3fs.num_pages): + k = f"key_{i}" + assert hicache_hf3fs.exists(k) + out = hicache_hf3fs.get(k) + assert out is not None + v = tensors[k] + assert torch.allclose(v, out, atol=1e-3), f"Tensor mismatch for {k}" + + assert not hicache_hf3fs.exists("not_exists") + + hicache_hf3fs.delete("key_7") + v2 = torch.randn((numel,)).to(dtype=dtype) + assert hicache_hf3fs.set("key_new", v2) + assert torch.allclose(hicache_hf3fs.get("key_new"), v2, atol=1e-3) + + hicache_hf3fs.clear() + assert ( + len(hicache_hf3fs.metadata_client.rank_metadata.free_pages) + == hicache_hf3fs.metadata_client.rank_metadata.num_pages + ) + + # batch + num_pages = 10 + tensors = {} + keys = [] + values = [] + for i in range(num_pages): + k = f"key_{i}" + keys.append(k) + v = torch.randn((numel,)).to(dtype=dtype) + values.append(v) + + ok = hicache_hf3fs.batch_set(keys, values) + assert not ok + assert hicache_hf3fs.get("key_8") is None + assert hicache_hf3fs.get("key_9") is None + + results = hicache_hf3fs.batch_get(keys[: hicache_hf3fs.num_pages]) + for result, key, value in zip( + results, keys[: hicache_hf3fs.num_pages], values[: hicache_hf3fs.num_pages] + ): + assert torch.allclose(value, result, atol=1e-3), f"Tensor mismatch for {key}" + + hicache_hf3fs.close() + os.remove(hicache_hf3fs.file_path) + + print("All test cases passed.") + + +def bench(): + # Qwen3-32B + layer_num = 64 + head_num, head_dim = 8, 128 + kv_lora_rank, qk_rope_head_dim = 0, 0 + store_dtype = torch.bfloat16 + tokens_per_page = 64 + + file_path = "/data/test.bin" + file_size = 1 << 40 + numjobs = 16 + bytes_per_page = 16 << 20 + entries = 8 + dtype = store_dtype + hicache_hf3fs = HiCacheHF3FS( + rank=0, + file_path=file_path, + file_size=file_size, + numjobs=numjobs, + bytes_per_page=bytes_per_page, + entries=entries, + dtype=dtype, + metadata_client=Hf3fsLocalMetadataClient(), + ) + + numel = 2 * tokens_per_page * layer_num * head_num * head_dim + assert numel * dtype.itemsize == bytes_per_page + + num_page = 128 + values = [torch.randn((numel,)).to(dtype=dtype) for _ in tqdm(range(num_page))] + + warmup = 50 + iteration = 100 + + w_bw = [] + w_size = num_page * bytes_per_page / (1 << 30) + for i in tqdm(range(warmup + iteration), desc="Benchmarking write (GB/s)"): + keys = [f"{j}" for j in range(i * num_page, (i + 1) * num_page)] + tik = time.perf_counter() + ok = hicache_hf3fs.batch_set(keys, values) + tok = time.perf_counter() + if i < warmup: + continue + w_bw.append(w_size / (tok - tik)) + assert ok + print_stats(w_bw) + + r_bw = [] + r_size = num_page * bytes_per_page / (1 << 30) + for i in tqdm(range(warmup + iteration), desc="Benchmarking read (GB/s)"): + keys = random.sample( + list(hicache_hf3fs.metadata_client.rank_metadata.key_to_index.keys()), + num_page, + ) + tik = time.perf_counter() + results = hicache_hf3fs.batch_get(keys) + tok = time.perf_counter() + if i < warmup: + continue + r_bw.append(r_size / (tok - tik)) + assert all([r is not None for r in results]) + print_stats(r_bw) + + hicache_hf3fs.close() + + +def allclose(): + # Qwen3-32B + layer_num = 64 + head_num, head_dim = 8, 128 + kv_lora_rank, qk_rope_head_dim = 0, 0 + store_dtype = torch.bfloat16 + tokens_per_page = 64 + + file_path = "/data/test.bin" + file_size = 1 << 40 + numjobs = 16 + bytes_per_page = 16 << 20 + entries = 8 + dtype = store_dtype + hicache_hf3fs = HiCacheHF3FS( + rank=0, + file_path=file_path, + file_size=file_size, + numjobs=numjobs, + bytes_per_page=bytes_per_page, + entries=entries, + dtype=dtype, + metadata_client=Hf3fsLocalMetadataClient(), + ) + + numel = 2 * tokens_per_page * layer_num * head_num * head_dim + assert numel * dtype.itemsize == bytes_per_page + + num_page = 128 + values = [torch.randn((numel,)).to(dtype=dtype) for _ in tqdm(range(num_page))] + + iteration = 100 + + for i in tqdm(range(iteration), desc="Benchmarking write (GB/s)"): + keys = [f"{j}" for j in range(i * num_page, (i + 1) * num_page)] + ok = hicache_hf3fs.batch_set(keys, values) + assert ok + + read_keys, read_results = [], [] + for i in tqdm(range(iteration), desc="Benchmarking read (GB/s)"): + keys = random.sample( + list(hicache_hf3fs.metadata_client.rank_metadata.key_to_index.keys()), + num_page, + ) + results = hicache_hf3fs.batch_get(keys) + read_keys.extend(keys) + read_results.extend(results) + assert all([r is not None for r in results]) + + for key, result in tqdm(zip(read_keys, read_results)): + assert torch.allclose(values[int(key) % num_page], result, atol=1e-3) + + hicache_hf3fs.close() + + +def main(): + logging.basicConfig(level=logging.INFO) + test() + bench() + allclose() + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hf3fs/bench_zerocopy.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hf3fs/bench_zerocopy.py new file mode 100644 index 00000000..bfa7bff0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hf3fs/bench_zerocopy.py @@ -0,0 +1,140 @@ +import threading +import time + +import torch +from tqdm import tqdm + +from sglang.srt.distributed import ( + get_world_group, + init_distributed_environment, + initialize_model_parallel, +) +from sglang.srt.managers.cache_controller import ( + HiCacheController, + PrefetchOperation, + StorageOperation, +) +from sglang.srt.mem_cache.allocator import TokenToKVPoolAllocator +from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool +from sglang.srt.mem_cache.memory_pool_host import MHATokenToKVPoolHost + +init_distributed_environment( + world_size=1, + rank=0, + distributed_init_method="tcp://127.0.0.1:23456", + local_rank=0, + backend="gloo", +) + +initialize_model_parallel( + tensor_model_parallel_size=1, + pipeline_model_parallel_size=1, +) + +group = get_world_group().cpu_group + +max_total_num_tokens = 524288 +page_size = 64 +kv_cache_dtype = torch.bfloat16 +layer_num = 64 +head_num, head_dim = 8, 128 +device = "cuda" +hicache_ratio = 2 +hicache_size = 0 +hicache_mem_layout = "page_first" +# hicache_mem_layout = "layer_first" +hicache_write_policy = "write_through" +hicache_io_backend = "kernel" +hicache_storage_backend = "hf3fs" +prefetch_threshold = 256 + +op_size = 1024 +op_num = 16 + +token_to_kv_pool = MHATokenToKVPool( + max_total_num_tokens, + page_size=page_size, + dtype=kv_cache_dtype, + head_num=head_num, + head_dim=head_dim, + layer_num=layer_num, + device=device, + enable_memory_saver=True, +) + +token_to_kv_pool_allocator = TokenToKVPoolAllocator( + max_total_num_tokens, + dtype=kv_cache_dtype, + device=device, + kvcache=token_to_kv_pool, + need_sort=False, +) + +kv_cache = token_to_kv_pool_allocator.get_kvcache() +token_to_kv_pool_host = MHATokenToKVPoolHost( + kv_cache, + hicache_ratio, + hicache_size, + page_size, + hicache_mem_layout, +) + +load_cache_event = threading.Event() +cache_controller = HiCacheController( + token_to_kv_pool_allocator, + token_to_kv_pool_host, + page_size, + group, + load_cache_event=load_cache_event, + write_policy=hicache_write_policy, + io_backend=hicache_io_backend, + storage_backend=hicache_storage_backend, + prefetch_threshold=prefetch_threshold, +) + +operations = [ + StorageOperation( + torch.tensor(list(range(i, i + op_size))), + list(range(i, i + op_size)), + hash_value=[f"{j}" for j in range(i, i + op_size, page_size)], + ) + for i in tqdm(range(0, op_num * op_size, op_size)) +] + +tik = time.monotonic() +if hicache_mem_layout == "page_first": + for operation in operations: + cache_controller.zerocopy_page_backup(operation, batch_size=128) +elif hicache_mem_layout == "layer_first": + for operation in operations: + cache_controller.generic_page_backup(operation, batch_size=128) +tok = time.monotonic() +print(f"{tok-tik:.6f} s") + +operations = [ + PrefetchOperation( + f"{i}", + torch.tensor(list(range(i, i + op_size))), + list(range(i, i + op_size)), + f"{i}", + ) + for i in tqdm(range(0, op_num * op_size, op_size)) +] + +for operation in operations: + operation.hash_value = [ + f"{j}" + for j in range( + int(operation.last_hash), int(operation.last_hash) + op_size, page_size + ) + ] + +tik = time.monotonic() +if hicache_mem_layout == "page_first": + for operation in operations: + cache_controller.zerocopy_page_transfer(operation, batch_size=128) +elif hicache_mem_layout == "layer_first": + for operation in operations: + cache_controller.generic_page_transfer(operation, batch_size=128) +tok = time.monotonic() +print(f"{tok-tik:.6f} s") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/README.md new file mode 100644 index 00000000..f2aa6ff0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/README.md @@ -0,0 +1,91 @@ +## Run synthetic multi-turn benchmark + +``` +# SGLang server with radix cache disabled +python -m sglang.launch_server --model-path Qwen/Qwen2.5-14B-Instruct --port 30000 --disable-radix-cache + +# SGLang server with radix cache on and first-come-first-serve policy +python -m sglang.launch_server --model-path Qwen/Qwen2.5-14B-Instruct --port 30000 --schedule-policy fcfs + +# The default SGLang server with radix cache on and long-prefix-match policy +python -m sglang.launch_server --model-path Qwen/Qwen2.5-14B-Instruct --port 30000 + +# SGLang server with hierarchical radix cache enabled +python -m sglang.launch_server --model-path Qwen/Qwen2.5-14B-Instruct --port 30000 --enable-hierarchical-cache + +``` + +``` +python bench_multiturn.py --model-path Qwen/Qwen2.5-14B-Instruct +``` + +Note: The performance gain of hierarchical caching depends on the ratio of reusable tokens to GPU memory capacity. The more tokens to be reused, the larger the model, and the more constrained the GPU memory size, the greater the benefit one can expect from hierarchical caching. + + +# Benchmark with more datasets +## Download Dataset +```bash +./download.sh {sharegpt|ultragpt|loogle|nextqa|all} +``` +This script will automatically download the required dataset to the current working directory + +## Multiturn Benchmark +### Supported Datasets +- sharegpt +- ultrachat +- loogle +### Example Usage: +```bash +python3 bench_serving.py --model mistralai/Mistral-7B-Instruct-v0.3 --backend sglang \ +--dataset-path longdep_qa.json --dataset-name loogle --request-rate 10 --num-prompts 10 \ +--port 8001 --enable-multiturn --disable-shuffle +``` +This uses `mistralai/Mistral-7B-Instruct-v0.3` model with `sglang` as backend. The dataset +is `longdep_qa.json`. We send `10 conversations` with `10 req/s` to port 8001. We enable +multiturn chat without shuffling the order of conversations (i.e. following the original +order in the dataset file). + +### Note: +The requests of multiple conversations are sent in a round robin fashion. +For example, if we have 3 conversations A, B, C whose rounds are `[2, 3, 4]` correspondingly, +multiturn chat will send the requests to the backend in the following order: `[A1, B1, C1, A2, B2, C2, B3, C3, C4]` +This has implications on the cache reuse patterns: the cache reuse distance is the largest +under this request pattern (which means a prefix-aware local scheduler in the backend can +yield the most benefit compared to a FIFO scheduler) + +## Shared Prefix Benchmark +### Supported Datasets +- loogle +### Example Usage: +```bash +python3 bench_serving.py --model mistralai/Mistral-7B-Instruct-v0.3 --backend sglang \ +--dataset-path longdep_qa.json --dataset-name loogle --request-rate 10 --num-prompts 10 \ +--port 8001 --enable-shared-prefix --disable-shuffle +``` +### Note: +Shared Prefix benchmark sends the questions for the same prompt together. For example, +if we have 3 shared prefix A, B, C, which have [2, 3, 4] questions correspondingly, +the shared prefix benchmark will send the requests to the +backend in the following order: `[A+Q1, A+Q2, B+Q1, B+Q2, B+Q3, C+Q1, C+Q2, C+Q3]`. + + +## Multi Modality Benchmark (WIP) +### Supported Datasets: +- nextqa +### Example Usage: +```bash +Server: +python3 -m sglang.launch_server --model-path lmms-lab/LLaVA-NeXT-Video-7B --tp 2 --dp 1 --port 8001 \ +--host 0.0.0.0 --mem-fraction-static 0.9 --tokenizer-path llava-hf/llava-1.5-7b-hf \ +--json-model-override-args "{\"architectures\": [\"LlavaVidForCausalLM\"], \"model_type\":\"llava\", \"mm_spatial_pool_stride\":2}" + +Client: +python3 bench_serving.py --model lmms-lab/LLaVA-NeXT-Video-7B --backend sglang --dataset-path \ +NExTVideo --dataset-name nextqa --request-rate 10 --num-prompts 1 --disable-shuffle --port 8001 \ --enable-multiturn --max-frames 16 --tokenizer llava-hf/llava-1.5-7b-hf --fixed-output-len 2048 +``` +Note: for the server args, `tokenizer-path`, overriding architecture are necessary. + +## Supported Backend +- sglang (oai) +- vllm (oai) +- lmdeploy (oai) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/bench_long_context.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/bench_long_context.py new file mode 100644 index 00000000..a3656cef --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/bench_long_context.py @@ -0,0 +1,102 @@ +import json +import queue +import time + +import requests +from bench_multiturn import ( + ReadyQueue, + WorkloadGenerator, + gen_payload, + log_to_jsonl_file, + parse_args, +) +from tqdm.asyncio import tqdm + +from sglang.bench_serving import get_tokenizer + + +class ContextWorkloadGenerator(WorkloadGenerator): + def __init__(self, args): + # Construct the base URL for requests + self.baseurl = f"http://{args.host}:{args.port}/" + self.url = self.baseurl + "generate" + + self.tokenizer = get_tokenizer(args.model_path) + self.distribution = args.distribution + self.request_rate = args.request_rate + self.start_time = None + self.finished_time = None + + self.sent_requests = 0 + self.completed_requests = 0 + + self.dataset = json.load(open(args.dataset_path)) + num_requests = min(args.num_clients, len(self.dataset["queries"])) + + init_requests = [] + for i in range(num_requests): + context_id = self.dataset["queries"][i]["context"] + init_requests.append( + ( + i, + gen_payload( + self.dataset["contexts"][context_id] + + self.dataset["queries"][i]["question"], + len( + self.tokenizer( + self.dataset["queries"][i]["reference_answer"] + )["input_ids"] + ), + ), + ) + ) + self.ready_queue = ReadyQueue(init_requests=init_requests) + + self.response_queue = queue.Queue() + self.pbar = tqdm(total=num_requests) + self.performance_metrics = { + "ttft": [], + "latency": [], + "itl": [], + "prompt_len": [], + "cached_tokens": [], + "generated_len": [], + } + + self.max_parallel = args.max_parallel + self.logfile = args.log_file + self.enable_round_barrier = False + + def response_handler(self): + while True: + try: + client_id, response = self.response_queue.get( + timeout=10 + ) # Block until response is available + if not response.success: + raise ValueError(f"Request failed with error: {response.error}") + self.performance_metrics["ttft"].append(response.ttft) + self.performance_metrics["itl"].extend(response.itl) + self.performance_metrics["latency"].append(response.latency) + self.performance_metrics["prompt_len"].append(response.prompt_len) + self.performance_metrics["cached_tokens"].append(response.cached_tokens) + self.performance_metrics["generated_len"].append(response.generated_len) + self.completed_requests += 1 + + except queue.Empty: + if self.pbar.n == self.pbar.total: + break + + +if __name__ == "__main__": + args = parse_args() + args.num_rounds = 1 + args.max_parallel = 24 + flush_cache_url = f"http://{args.host}:{args.port}/flush_cache" + + for request_rate in [24, 16, 12, 8, 4, 2, 1]: + args.request_rate = request_rate + requests.post(flush_cache_url) + time.sleep(1) + performance_data = ContextWorkloadGenerator(args).run() + log_to_jsonl_file(performance_data, args.log_file, args.tag) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/bench_mix.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/bench_mix.py new file mode 100644 index 00000000..cfd25bc4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/bench_mix.py @@ -0,0 +1,567 @@ +import argparse +import asyncio +import json +import logging +import os +import queue +import random +import threading +import time +from dataclasses import dataclass +from functools import wraps + +import aiohttp + +from sglang.bench_serving import ( + RequestFuncOutput, + get_tokenizer, + remove_prefix, + sample_random_requests, +) + +# Set up logger +logger = logging.getLogger(__name__) + +# Set up JSONL file for debug logging +debug_log_file = None +# Create a lock for thread-safe debug log writing +debug_log_lock = threading.Lock() + + +def write_debug_log(data): + global debug_log_file + + """Write debug information to a JSONL file""" + if debug_log_file is None: + return + + # Acquire lock for thread-safe writing + with debug_log_lock: + # Write as JSONL (JSON Line format) + debug_log_file.write(json.dumps(data) + "\n") + debug_log_file.flush() + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Script to benchmark concurrent requests to a server." + ) + parser.add_argument( + "--model-path", + type=str, + default="/data/models/Qwen3-0.6B", + help="model path compatible with Hugging Face Transformers", + ) + parser.add_argument( + "--dataset-path", + type=str, + default="/data/models/ShareGPT_V3_unfiltered_cleaned_split/ShareGPT_V3_unfiltered_cleaned_split.json", + help="local dataset to sample tokens from", + ) + parser.add_argument( + "--host", + type=str, + default="localhost", + help="Server hostname or IP (default: localhost)", + ) + parser.add_argument( + "--port", + type=int, + default=30000, + help="Server port (default: 30000)", + ) + parser.add_argument( + "--duration", + type=int, + default=600, + help="Duration to run the benchmark in seconds (default: 300 seconds)", + ) + parser.add_argument( + "--log-level", + type=str, + default="info", + choices=["debug", "info"], + help="Set the logging level (default: info)", + ) + parser.add_argument( + "--debug-log-file", + type=str, + default="debug.log.jsonl", + help="File to write debug logs in JSONL format", + ) + return parser.parse_args() + + +def load_config(): + config_path = os.getenv("CONFIG_PATH") + if not config_path: + raise ValueError("Environment variable 'CONFIG_PATH' is not set.") + + with open(config_path, "r") as f: + config = json.load(f) + + required_keys = [ + "num_rounds", + "num_clients", + "round_ratios", + "mean_new_tokens_per_round", + "mean_return_tokens_per_round", + "mean_inter_round_interval", + ] + + for key in required_keys: + if key not in config: + raise KeyError(f"Missing required configuration key: {key}") + + num_rounds = config["num_rounds"] + assert len(config["round_ratios"]) == num_rounds + assert len(config["mean_new_tokens_per_round"]) == num_rounds + assert len(config["mean_return_tokens_per_round"]) == num_rounds + assert len(config["mean_inter_round_interval"]) == num_rounds + + print(config) + + return config + + +@dataclass +class UserData: + user_id: int + current_round: int + total_rounds: int + prompt: str + return_tokens: int + start: int + + +def synchronized(): + def _decorator(func): + @wraps(func) + def wrapper(self, *args, **kwargs): + with self.lock: + return func(self, *args, **kwargs) + + return wrapper + + return _decorator + + +class UserGenerator: + def __init__(self, config, model_path, dataset_path): + self.tokenizer_path = model_path + self.tokenizer = get_tokenizer(self.tokenizer_path) + self.dataset_path = dataset_path + + self.user_id = 0 + self.lock = threading.Lock() + + self.num_rounds = config["num_rounds"] + + self.cumulative_ratios = [ + sum(config["round_ratios"][: i + 1]) + for i in range(len(config["round_ratios"])) + ] + self.mean_new_tokens_per_round = config["mean_new_tokens_per_round"] + self.mean_return_tokens_per_round = config["mean_return_tokens_per_round"] + self.mean_inter_round_interval = config["mean_inter_round_interval"] + + self.sigma = 100 + self.range_ratio = 0.8 + assert self.range_ratio <= 1 + + self.candidate_inputs = [ + [ + r + for r in sample_random_requests( + input_len=( + self.mean_new_tokens_per_round[i] * (2 - self.range_ratio) + ), + output_len=( + self.mean_return_tokens_per_round[i] * (2 - self.range_ratio) + ), + num_prompts=config["num_clients"], + range_ratio=self.range_ratio / (2 - self.range_ratio), + tokenizer=self.tokenizer, + dataset_path=self.dataset_path, + random_sample=False, + ) + ] + for i in range(self.num_rounds) + ] + + self.multiturn_queue = [] + + self.user_stats = [0 for _ in range(self.num_rounds)] + self.input_stats = [[0, 0] for _ in range(self.num_rounds)] + self.output_stats = [[0, 0] for _ in range(self.num_rounds)] + + def gen(self): + user_id = self.user_id + self.user_id += 1 + + rand_ratio = random.randint(0, self.cumulative_ratios[-1]) + i = len(self.cumulative_ratios) + for idx, cumulative_ratio in enumerate(self.cumulative_ratios): + if rand_ratio >= cumulative_ratio: + continue + else: + i = idx + 1 + break + total_rounds = i + current_round = 0 + + candidate_input = random.sample(self.candidate_inputs[current_round], 1)[0] + self.input_stats[0][0] += candidate_input.prompt_len + self.input_stats[0][1] += 1 + prompt = f"{user_id} " + candidate_input.prompt + return_tokens = int( + random.gauss(self.mean_return_tokens_per_round[current_round], self.sigma) + ) + if return_tokens <= 0: + return_tokens = self.mean_return_tokens_per_round[current_round] + start = 0 + + user_data = UserData( + user_id, current_round, total_rounds, prompt, return_tokens, start + ) + + self.user_stats[total_rounds - 1] += 1 + + return user_data + + @synchronized() + def push(self, user_data, generated_text, len_itl): + self.output_stats[user_data.current_round][0] += len_itl + 1 + self.output_stats[user_data.current_round][1] += 1 + user_data.current_round += 1 + if user_data.current_round >= user_data.total_rounds: + return + + candidate_input = random.sample( + self.candidate_inputs[user_data.current_round], 1 + )[0] + self.input_stats[user_data.current_round][0] += candidate_input.prompt_len + self.input_stats[user_data.current_round][1] += 1 + user_data.prompt += generated_text + candidate_input.prompt + user_data.return_tokens = int( + random.gauss( + self.mean_return_tokens_per_round[user_data.current_round], self.sigma + ) + ) + if user_data.return_tokens <= 0: + user_data.return_tokens = self.mean_return_tokens_per_round[ + user_data.current_round + ] + interval = random.gauss( + self.mean_inter_round_interval[user_data.current_round], self.sigma + ) + if interval <= 0: + interval = self.mean_inter_round_interval[user_data.current_round] + user_data.start = time.perf_counter() + interval + + if len(self.multiturn_queue) == 0: + self.multiturn_queue.append(user_data) + else: + i = len(self.multiturn_queue) + for idx, d in enumerate(self.multiturn_queue): + if user_data.start < d.start: + i = idx + break + self.multiturn_queue.insert(idx, user_data) + + @synchronized() + def pop(self): + if ( + len(self.multiturn_queue) + and time.perf_counter() > self.multiturn_queue[0].start + ): + return self.multiturn_queue.pop(0) + return self.gen() + + +def gen_payload(prompt, output_len): + payload = { + "text": prompt, + "sampling_params": { + "temperature": 0.0, + "max_new_tokens": output_len, + "ignore_eos": True, + }, + "stream": True, + "stream_options": {"include_usage": True}, + "lora_path": "", + "return_logprob": False, + "logprob_start_len": -1, + } + return payload + + +AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=20 * 60 * 60) + + +async def async_request_sglang_generate( + user_data, + url, + atomic_counter, +): + """ + Sends a streaming request to the server. Gathers text token-by-token. + """ + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session: + headers = {} + generated_text = "" + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + output = RequestFuncOutput() + payload = gen_payload(user_data.prompt, user_data.return_tokens) + write_debug_log({"timestamp": st, "user_data": user_data.__dict__}) + + try: + async with session.post(url=url, json=payload, headers=headers) as response: + if response.status == 200: + prompt_tokens = 0 + cached_tokens = 0 + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + if data.get("text"): + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft = ttft + prompt_tokens = (data.get("meta_info") or {}).get( + "prompt_tokens", 0 + ) + cached_tokens = (data.get("meta_info") or {}).get( + "cached_tokens", 0 + ) + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + most_recent_timestamp = timestamp + generated_text = data["text"] + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.prompt_len = prompt_tokens + output.cached_tokens = cached_tokens + else: + output.error = response.reason or "" + output.success = False + except Exception as e: + output.success = False + output.error = str(e) + print(f"Request failed: {e}") + + atomic_counter.increment(1) + return output + + +class AtomicCounter: + def __init__(self, initial_value=0): + self._value = initial_value + self.lock = threading.Lock() + + @synchronized() + def increment(self, amount=1): + self._value += amount + + @synchronized() + def get(self): + return self._value + + +class WorkloadGenerator: + def __init__(self, args): + config = load_config() + user_generator = UserGenerator( + config, + args.model_path, + args.dataset_path, + ) + + self.url = f"http://{args.host}:{args.port}/generate" + + self.tokenizer = user_generator.tokenizer + self.start_time = None + self.finished_time = None + self.duration = args.duration + self.done = False + + self.sent_requests = 0 + self.completed_requests = 0 + + self.user_generator = user_generator + self.response_queue = queue.Queue() + self.performance_metrics = { + "ttft": [], + "latency": [], + "prompt_len": [], + "cached_tokens": [], + } + self.max_parallel = config["num_clients"] + + self.atomic_counter = AtomicCounter() + + async def handle_request(self, user_data): + try: + response = await async_request_sglang_generate( + user_data, self.url, self.atomic_counter + ) + self.response_queue.put((user_data, response)) + except Exception as e: + print(f"Request failed: {e}") + self.completed_requests += 1 + + def request_sender(self): + async def request_loop(): + while True: + if self.sent_requests - self.completed_requests < self.max_parallel: + new_request = self.user_generator.pop() + if new_request: + asyncio.create_task(self.handle_request(new_request)) + self.sent_requests += 1 + else: + await asyncio.sleep(0.05) + continue + + if time.perf_counter() - self.start_time > self.duration: + self.done = True + break + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(request_loop()) + loop.close() + + def response_handler(self): + while True: + try: + user_data, response = self.response_queue.get(timeout=10) + logger.info( + f"{((time.perf_counter()-self.start_time)/self.duration*100):.2f}%" + ) + if not response.success: + raise ValueError(f"Request failed with error: {response.error}") + + self.user_generator.push( + user_data, response.generated_text, len(response.itl) + ) + self.performance_metrics["ttft"].append(response.ttft) + self.performance_metrics["latency"].append(response.latency) + self.performance_metrics["prompt_len"].append(response.prompt_len) + self.performance_metrics["cached_tokens"].append(response.cached_tokens) + self.completed_requests += 1 + self.finished_time = time.perf_counter() + + except queue.Empty: + if self.done: + break + except ValueError as e: + print(f"Error processing response for client {user_data}: {e}") + continue + + def run(self): + request_thread = threading.Thread(target=self.request_sender, daemon=True) + response_thread = threading.Thread(target=self.response_handler, daemon=True) + + self.start_time = time.perf_counter() + request_thread.start() + response_thread.start() + + request_thread.join() + response_thread.join() + + performance_data = { + "summary": { + "total_requests": len(self.performance_metrics["ttft"]), + "average_ttft": sum(self.performance_metrics["ttft"]) + / len(self.performance_metrics["ttft"]), + "p90_ttft": sorted(self.performance_metrics["ttft"])[ + int(0.9 * len(self.performance_metrics["ttft"])) + ], + "median_ttft": sorted(self.performance_metrics["ttft"])[ + len(self.performance_metrics["ttft"]) // 2 + ], + "average_latency": sum(self.performance_metrics["latency"]) + / len(self.performance_metrics["latency"]), + "p90_latency": sorted(self.performance_metrics["latency"])[ + int(0.9 * len(self.performance_metrics["latency"])) + ], + "median_latency": sorted(self.performance_metrics["latency"])[ + len(self.performance_metrics["latency"]) // 2 + ], + "throughput": self.atomic_counter.get() + / (self.finished_time - self.start_time), + "cache_hit_rate": ( + 0 + if sum(self.performance_metrics["prompt_len"]) == 0 + else sum(self.performance_metrics["cached_tokens"]) + / sum(self.performance_metrics["prompt_len"]) + ), + }, + } + print("All requests completed") + print("Performance metrics summary:") + print(f" Total requests: {performance_data['summary']['total_requests']}") + print(f" Average TTFT: {performance_data['summary']['average_ttft']:.2f}") + print(f" P90 TTFT: {performance_data['summary']['p90_ttft']:.2f}") + print(f" Median TTFT: {performance_data['summary']['median_ttft']:.2f}") + print( + f" Average latency: {performance_data['summary']['average_latency']:.2f}" + ) + print(f" P90 latency: {performance_data['summary']['p90_latency']:.2f}") + print(f" Median latency: {performance_data['summary']['median_latency']:.2f}") + print( + f" Throughput: {performance_data['summary']['throughput']:.2f} requests per second" + ) + print(f" Cache Hit Rate: {performance_data['summary']['cache_hit_rate']:.6f}") + + user_stats = self.user_generator.user_stats + input_stats = self.user_generator.input_stats + output_stats = self.user_generator.output_stats + print(f"round_ratios: {user_stats}") + print( + f"mean_new_tokens_per_round: {[int(a/b) if b > 0 else 0 for a, b in input_stats]}" + ) + print( + f"mean_return_tokens_per_round: {[int(a/b) if b > 0 else 0 for a, b in output_stats]}" + ) + return performance_data + + +def main(): + global debug_log_file + + args = parse_args() + if args.log_level == "debug": + logging.basicConfig(level=logging.DEBUG) + logger.info("use log_level debug") + # Initialize debug log file + debug_log_file = open(args.debug_log_file, "w") + else: + logging.basicConfig(level=logging.INFO) + logger.info("use log_level info") + performance_data = WorkloadGenerator(args).run() + + # Close debug log file if it was opened + if debug_log_file: + debug_log_file.close() + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/bench_mix.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/bench_mix.sh new file mode 100755 index 00000000..5ff6dca9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/bench_mix.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/python3.12/dist-packages:/usr/local/lib/python3.12/dist-packages/torch/lib +rm -rf nohup.out && \ +nohup python3 -m sglang.launch_server \ + --attention-backend triton \ + --model-path /code/models/Qwen3-32B/ \ + --log-level info \ + --tp 4 --mem-frac 0.25 \ + --host 0.0.0.0 --port 33301 \ + --enable-metrics --enable-cache-report \ + --page-size 64 \ + --enable-hierarchical-cache \ + --hicache-ratio 2.5 --hicache-size 0 \ + --hicache-io-backend kernel \ + --hicache-mem-layout layer_first \ + --hicache-write-policy write_through \ + & + +################################################## + +export CONFIG_PATH=/tmp/bench_mix_config.json + +# num_clients: Maximum number of concurrent client requests to be simulated +# round_ratios: Distribution of requests across rounds. Given sum(round_ratios) total requests, +# round_ratios[i] denotes the number of requests that will execute for (i+1) rounds +echo '{ + "num_rounds": 10, + "num_clients": 60, + "round_ratios": [50, 25, 15, 15, 10, 10, 9, 8, 7, 6], + "mean_new_tokens_per_round": [1000, 400, 350, 300, 280, 260, 240, 220, 210, 200], + "mean_return_tokens_per_round": [100, 100, 100, 100, 100, 100, 100, 100, 100, 100], + "mean_inter_round_interval": [30, 30, 30, 30, 30, 30, 30, 30, 30, 30] +}' > ${CONFIG_PATH} + +rm -rf bench_mix.out && \ +nohup python3 /sgl-workspace/sglang/benchmark/hicache/bench_mix.py \ + --model-path /code/models/Qwen3-32B/ \ + --dataset-path /code/models/ShareGPT_V3_unfiltered_cleaned_split.json \ + --port 33301 \ + --duration 600 \ +> bench_mix.out & diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/bench_multiturn.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/bench_multiturn.py new file mode 100644 index 00000000..fe154d6b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/bench_multiturn.py @@ -0,0 +1,615 @@ +import argparse +import asyncio +import json +import queue +import random +import threading +import time +from datetime import datetime +from typing import Optional + +import aiohttp +import numpy as np +import requests +from tqdm.asyncio import tqdm + +from sglang.bench_serving import ( + RequestFuncOutput, + get_tokenizer, + remove_prefix, + sample_random_requests, +) + +AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=20 * 60 * 60) + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Script to benchmark concurrent requests to a server." + ) + parser.add_argument( + "--num-clients", + type=int, + default=256, + help="Number of concurrent clients", + ) + parser.add_argument( + "--max-parallel", + type=int, + default=128, + help="Maximum number of parallel requests", + ) + parser.add_argument( + "--request-length", + type=int, + default=512, + help="Length of each new request", + ) + parser.add_argument( + "--output-length", + type=int, + default=64, + help="Length of each output", + ) + parser.add_argument( + "--num-rounds", + type=int, + default=5, + help="Number of rounds per client", + ) + parser.add_argument( + "--distribution", + type=str, + default="poisson", + choices=["poisson", "uniform"], + help="Distribution type for request intervals (poisson or uniform)", + ) + parser.add_argument( + "--request-rate", + type=float, + default=1.0, + help="Average number of requests per second", + ) + parser.add_argument( + "--host", + type=str, + default="localhost", + help="Server hostname or IP (default: localhost)", + ) + parser.add_argument( + "--port", + type=int, + default=30000, + help="Server port (default: 30000)", + ) + parser.add_argument( + "--model-path", + type=str, + default="meta-llama/Llama-3.1-8B-Instruct", + help="model path compatible with Hugging Face Transformers", + ) + parser.add_argument( + "--dataset-path", + type=str, + default="", + help="local dataset to sample tokens from", + ) + parser.add_argument( + "--log-file", + type=str, + default="performance_metrics.jsonl", + help="File to log performance metrics", + ) + parser.add_argument( + "--disable-auto-run", + action="store_true", + help="If set, disable automatically testing with a range of request rates.", + ) + parser.add_argument( + "--disable-random-sample", + action="store_true", + help="If set, disable random sampling of requests from the ShareGPT dataset.", + ) + parser.add_argument( + "--enable-round-barrier", + action="store_true", + help="If set, only send i-th turn requests after all (i-1)-th turn requests finished.", + ) + parser.add_argument( + "--sub-question-input-length", + type=int, + default=0, + help="Length of the sub question input for each request, if set 0 use request_length", + ) + parser.add_argument( + "--ready-queue-policy", + type=str, + default="random", + help="Policy for popping requests from the ready queue (random or fifo)", + ) + parser.add_argument( + "--tag", + type=str, + default="", + help="Tag of a certain run in the log file", + ) + parser.add_argument("--seed", type=int, default=1, help="The random seed.") + parser.add_argument( + "--lora-path", + type=str, + default="", + help="String of LoRA path. Currently we only support benchmarking on a single LoRA adaptor.", + ) + return parser.parse_args() + + +async def async_request_sglang_generate( + payload, + url, + pbar: Optional[tqdm] = None, +): + """ + Sends a streaming request to the server. Gathers text token-by-token. + """ + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session: + headers = {} + generated_text = "" + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + output = RequestFuncOutput() + + try: + async with session.post(url=url, json=payload, headers=headers) as response: + if response.status == 200: + prompt_tokens = 0 + cached_tokens = 0 + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + if data["text"]: + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft = ttft + prompt_tokens = (data.get("meta_info") or {}).get( + "prompt_tokens", 0 + ) + cached_tokens = (data.get("meta_info") or {}).get( + "cached_tokens", 0 + ) + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + most_recent_timestamp = timestamp + generated_text = data["text"] + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.prompt_len = prompt_tokens + output.cached_tokens = cached_tokens + output.generated_len = len(output.itl) + 1 + else: + output.error = response.reason or "" + output.success = False + except Exception as e: + output.success = False + output.error = str(e) + print(f"Request failed: {e}") + + if pbar: + pbar.update(1) + return output + + +def gen_payload(prompt, output_len, lora_path=""): + payload = { + "text": prompt, + "sampling_params": { + "temperature": 0.0, + "max_new_tokens": output_len, + "ignore_eos": True, + }, + "stream": True, + "stream_options": {"include_usage": True}, + "lora_path": lora_path, + "return_logprob": False, + "logprob_start_len": -1, + } + return payload + + +def log_to_jsonl_file(data, file_path="performance_metrics.jsonl", tag=""): + """Append the data with a timestamp and tag to the specified JSONL file.""" + timestamped_data = {"timestamp": datetime.now().isoformat(), "tag": tag, **data} + try: + with open(file_path, "a") as file: + file.write( + json.dumps(timestamped_data) + "\n" + ) # Write as a single line in JSONL format + except IOError as e: + print(f"Error writing to JSONL file: {e}") + + +class ReadyQueue: + """ + Thread-safe queue that can pop requests in different orders based on given policy. + """ + + def __init__(self, init_requests=None, policy="random"): + self.lock = threading.Lock() + self.requests = init_requests or [] + self.policy = policy + + def append(self, item): + with self.lock: + self.requests.append(item) + + def pop(self): + with self.lock: + if not self.requests: + return None + if self.policy == "random": + index = random.randrange(len(self.requests)) + return self.requests.pop(index) + elif self.policy == "fifo": + return self.requests.pop(0) + else: + # todo, varying thinking time of clients + raise ValueError(f"{self.policy} not implemented") + + +class WorkloadGenerator: + def __init__(self, args): + # Construct the base URL for requests + self.url = f"http://{args.host}:{args.port}/generate" + + self.tokenizer = get_tokenizer(args.model_path) + self.distribution = args.distribution + self.request_rate = args.request_rate + self.start_time = None + self.finished_time = None + + self.sent_requests = 0 + self.completed_requests = 0 + + self.candidate_inputs = sample_random_requests( + input_len=args.request_length, + output_len=args.output_length, + num_prompts=args.num_clients, + range_ratio=1.0, + tokenizer=self.tokenizer, + dataset_path=args.dataset_path, + random_sample=not args.disable_random_sample, + ) + self.candidate_inputs = [i.prompt for i in self.candidate_inputs] + + if args.sub_question_input_length != 0: + sub_question_input_length = args.sub_question_input_length + else: + sub_question_input_length = args.request_length + + self.sub_question_inputs = sample_random_requests( + input_len=sub_question_input_length, + output_len=args.output_length, + num_prompts=args.num_clients * max(args.num_rounds - 1, 1), + range_ratio=1.0, + tokenizer=self.tokenizer, + dataset_path=args.dataset_path, + random_sample=not args.disable_random_sample, + ) + + init_requests = [ + ( + i, + gen_payload( + self.candidate_inputs[i], args.output_length, args.lora_path + ), + ) + for i in range(args.num_clients) + ] + self.client_records = { + i: {"round": 0, "history": init_requests[i][1]["text"]} + for i in range(args.num_clients) + } + self.ready_queue = ReadyQueue( + init_requests=init_requests, policy=args.ready_queue_policy + ) + self.candidate_inputs = self.candidate_inputs[args.num_clients :] + + self.response_queue = queue.Queue() + self.pbar = tqdm(total=args.num_clients * args.num_rounds) + self.performance_metrics = { + "ttft": [], + "latency": [], + "prompt_len": [], + "cached_tokens": [], + "generated_len": [], + } + self.enable_round_barrier = args.enable_round_barrier + if self.enable_round_barrier: + # Add round-specific metrics while preserving the original structure + for i in range(args.num_rounds): + self.performance_metrics[f"round_{i}"] = { + "ttft": [], + "latency": [], + "prompt_len": [], + "cached_tokens": [], + "generated_len": [], + } + self.num_clients = args.num_clients + + self.num_rounds = args.num_rounds + self.max_parallel = args.max_parallel + self.output_length = args.output_length + + async def handle_request(self, item): + try: + client_id, payload = item + response = await async_request_sglang_generate(payload, self.url, self.pbar) + if self.pbar.n == self.pbar.total: + self.finished_time = time.perf_counter() + self.response_queue.put((client_id, response)) + except Exception as e: + print(f"Request failed: {e}") + + def request_sender(self): + async def request_loop(): + while True: + if self.sent_requests - self.completed_requests < self.max_parallel: + new_request = self.ready_queue.pop() + if new_request: + asyncio.create_task(self.handle_request(new_request)) + self.sent_requests += 1 + else: + await asyncio.sleep(0.05) + continue + + if self.pbar.n == self.pbar.total: + break + + # Calculate Poisson-distributed wait time + if self.distribution == "poisson": + sleep_time = random.expovariate(self.request_rate) + elif self.distribution == "uniform": + avg_interval = ( + 1.0 / self.request_rate if self.request_rate > 0 else 1.0 + ) + sleep_time = random.uniform(0, 2 * avg_interval) + else: + raise ValueError("Invalid distribution type") + await asyncio.sleep(sleep_time) # Wait before sending the next request + + # Create and run the event loop for asynchronous requests + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(request_loop()) + loop.close() + + def response_handler(self): + next_round_reqs = [] + while True: + try: + client_id, response = self.response_queue.get( + timeout=10 + ) # Block until response is available + if not response.success: + raise ValueError(f"Request failed with error: {response.error}") + self.client_records[client_id]["history"] += response.generated_text + current_round = self.client_records[client_id]["round"] + self.client_records[client_id]["round"] += 1 + self.performance_metrics["ttft"].append(response.ttft) + self.performance_metrics["latency"].append(response.latency) + self.performance_metrics["prompt_len"].append(response.prompt_len) + self.performance_metrics["cached_tokens"].append(response.cached_tokens) + self.performance_metrics["generated_len"].append(response.generated_len) + if self.enable_round_barrier: + self.performance_metrics[f"round_{current_round}"]["ttft"].append( + response.ttft + ) + self.performance_metrics[f"round_{current_round}"][ + "latency" + ].append(response.latency) + self.performance_metrics[f"round_{current_round}"][ + "prompt_len" + ].append(response.prompt_len) + self.performance_metrics[f"round_{current_round}"][ + "cached_tokens" + ].append(response.cached_tokens) + self.performance_metrics[f"round_{current_round}"][ + "generated_len" + ].append(response.generated_len) + self.completed_requests += 1 + + if self.client_records[client_id]["round"] < self.num_rounds: + # append new request to client's history + self.client_records[client_id][ + "history" + ] += self.sub_question_inputs.pop().prompt + new_req = ( + client_id, + gen_payload( + self.client_records[client_id]["history"], + self.output_length, + args.lora_path, + ), + ) + if self.enable_round_barrier: + next_round_reqs.append(new_req) + if len(next_round_reqs) == self.num_clients: + for req in next_round_reqs: + self.ready_queue.append(req) + next_round_reqs = [] + else: + self.ready_queue.append(new_req) + except queue.Empty: + if self.pbar.n == self.pbar.total: + break + except ValueError as e: + print(f"Error processing response for client {client_id}: {e}") + continue + + def run(self): + request_thread = threading.Thread(target=self.request_sender, daemon=True) + response_thread = threading.Thread(target=self.response_handler, daemon=True) + + self.start_time = time.perf_counter() + request_thread.start() + response_thread.start() + + request_thread.join() + response_thread.join() + self.pbar.close() + + duration = self.finished_time - self.start_time + performance_data = { + "summary": { + "total_requests": len(self.performance_metrics["ttft"]), + "request_rate": self.request_rate, + "average_prompt_len": ( + sum(self.performance_metrics["prompt_len"]) + / len(self.performance_metrics["prompt_len"]) + if self.performance_metrics["prompt_len"] + else 0.0 + ), + "average_output_len": ( + sum(self.performance_metrics["generated_len"]) + / len(self.performance_metrics["generated_len"]) + if self.performance_metrics["generated_len"] + else 0.0 + ), + "average_ttft": sum(self.performance_metrics["ttft"]) + / len(self.performance_metrics["ttft"]), + "p90_ttft": sorted(self.performance_metrics["ttft"])[ + int(0.9 * len(self.performance_metrics["ttft"])) + ], + "median_ttft": sorted(self.performance_metrics["ttft"])[ + len(self.performance_metrics["ttft"]) // 2 + ], + "average_latency": sum(self.performance_metrics["latency"]) + / len(self.performance_metrics["latency"]), + "p90_latency": sorted(self.performance_metrics["latency"])[ + int(0.9 * len(self.performance_metrics["latency"])) + ], + "median_latency": sorted(self.performance_metrics["latency"])[ + len(self.performance_metrics["latency"]) // 2 + ], + "input_token_throughput": sum(self.performance_metrics["prompt_len"]) + / duration, + "output_token_throughput": sum( + self.performance_metrics["generated_len"] + ) + / duration, + "throughput": self.pbar.total / duration, + "cache_hit_rate": ( + 0 + if sum(self.performance_metrics["prompt_len"]) == 0 + else sum(self.performance_metrics["cached_tokens"]) + / sum(self.performance_metrics["prompt_len"]) + ), + }, + } + if self.enable_round_barrier: + performance_data["round"] = {} + for round_num in range(args.num_rounds): + round_key = f"round_{round_num}" + round_metrics = self.performance_metrics[round_key] + performance_data["round"][round_key] = { + "average_ttft": ( + sum(round_metrics["ttft"]) / len(round_metrics["ttft"]) + if round_metrics["ttft"] + else 0 + ), + "cache_hit_rate": ( + 0 + if sum(round_metrics["prompt_len"]) == 0 + else sum(round_metrics["cached_tokens"]) + / sum(round_metrics["prompt_len"]) + ), + "request_count": len(round_metrics["ttft"]), + } + print("All requests completed") + print("Performance metrics summary:") + print( + f" Total requests: {performance_data['summary']['total_requests']} at {performance_data['summary']['request_rate']} requests per second" + ) + print( + f" Average Prompt Length: {performance_data['summary']['average_prompt_len']:.2f} tokens" + ) + print( + f" Average Output Length: {performance_data['summary']['average_output_len']:.2f} tokens" + ) + print(f" Average TTFT: {performance_data['summary']['average_ttft']:.2f}") + print(f" P90 TTFT: {performance_data['summary']['p90_ttft']:.2f}") + print(f" Median TTFT: {performance_data['summary']['median_ttft']:.2f}") + print( + f" Average latency: {performance_data['summary']['average_latency']:.2f}" + ) + print(f" P90 latency: {performance_data['summary']['p90_latency']:.2f}") + print(f" Median latency: {performance_data['summary']['median_latency']:.2f}") + print( + f" Input token throughput: {performance_data['summary']['input_token_throughput']:.2f} tokens per second" + ) + print( + f" Output token throughput: {performance_data['summary']['output_token_throughput']:.2f} tokens per second" + ) + print( + f" Request Throughput: {performance_data['summary']['throughput']:.2f} requests per second" + ) + print(f" Cache Hit Rate: {performance_data['summary']['cache_hit_rate']:.6f}") + + if self.enable_round_barrier: + # Print round-basedsummary + print("Per-round metrics:") + if "round" in performance_data: + for round_num in range(self.num_rounds): + round_key = f"round_{round_num}" + if round_key in performance_data["round"]: + round_data = performance_data["round"][round_key] + avg_ttft = round_data["average_ttft"] + cache_hit_rate = round_data["cache_hit_rate"] + request_count = round_data["request_count"] + print( + f" Round {round_num}: Average TTFT = {avg_ttft:.2f}s, " + f"Cache Hit Rate = {cache_hit_rate:.6f} " + f"({request_count} requests)" + ) + else: + print(f" Round {round_num}: No requests completed") + + return performance_data + + +if __name__ == "__main__": + args = parse_args() + flush_cache_url = f"http://{args.host}:{args.port}/flush_cache" + + random.seed(args.seed) + np.random.seed(args.seed) + + if args.disable_auto_run: + print("Running with specified request rate...") + request_rates = [args.request_rate] + else: + print("Auto-running with different request rates...") + request_rates = [16, 14, 12, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1] + + for rate in request_rates: + args.request_rate = rate + requests.post(flush_cache_url) + time.sleep(1) + performance_data = WorkloadGenerator(args).run() + log_to_jsonl_file(performance_data, args.log_file, tag=args.tag) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/bench_serving.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/bench_serving.py new file mode 100644 index 00000000..e38d0d0e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/bench_serving.py @@ -0,0 +1,1029 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/6366efc67b0aedd2c1721c14385370e50b297fb3/benchmarks/backend_request_func.py +# Adapted from https://github.com/vllm-project/vllm/blob/6366efc67b0aedd2c1721c14385370e50b297fb3/benchmarks/benchmark_serving.py + +""" +Benchmark online serving with dynamic requests. + +Usage: +python3 -m sglang.bench_serving --backend sglang --num-prompt 10 + +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 3000 --random-input 1024 --random-output 1024 --random-range-ratio 0.5 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --request-rate-range 1,2,4,8,16,32 --random-input 4096 --random-output 1024 --random-range-ratio 0.125 --multi +""" + +import argparse +import asyncio +import json +import os +import random +import sys +import time +import traceback +import warnings +from argparse import ArgumentParser +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple + +import aiohttp +import numpy as np +import requests +from data_processing import MsgContent, SampleOutput, get_dataset +from tqdm.asyncio import tqdm +from transformers import PreTrainedTokenizerBase + +from sglang.bench_serving import get_tokenizer, remove_prefix, set_ulimit + +AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=20 * 60 * 60) + +global args + + +@dataclass +class RequestFuncInput: + prompts: List[Tuple[MsgContent, int, int]] + api_url: str + model: str + lora_name: str + extra_request_body: Dict[str, Any] + + # For multiturn chat, store the context + prev_messages: List = field(default_factory=list) + finished_prompts: int = 0 + + +@dataclass +class RequestFuncOutput: + generated_text: List[str] = field(default_factory=list) + prompt_len: List[int] = field(default_factory=list) + output_len: List[int] = field(default_factory=list) + latency: List[float] = field(default_factory=list) + ttft: List[float] = field(default_factory=list) + itl: List[float] = field(default_factory=list) # List of inter-token latencies + + success: bool = False + error: str = "" + + +# set ignore_eos True by default +async def async_request_openai_completions( + request_func_input: RequestFuncInput, + queue: asyncio.Queue, + tokenizer: PreTrainedTokenizerBase, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + assert api_url.endswith( + "completions" + ), "OpenAI Completions API URL must end with 'completions'." + + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session: + payload = { + "model": request_func_input.model, + "temperature": 0.0, + "best_of": 1, + "stream": not args.disable_stream, + "stream_options": {"include_usage": True}, + "ignore_eos": not args.disable_ignore_eos, + **request_func_input.extra_request_body, + } + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}", + } + + output = RequestFuncOutput() + + prompt_idx = request_func_input.finished_prompts + messages = request_func_input.prev_messages + prompt, input_len, max_tokens = request_func_input.prompts[prompt_idx] + prompt_len = sum( + prompt[1] + prompt[2] # input_len + output_len + for prompt in request_func_input.prompts[:prompt_idx] + ) + prompt_len += input_len + + # Messages + messages.append( + { + "role": "user", + "content": prompt, + } + ) + payload["messages"] = messages + payload["max_tokens"] = max_tokens + + # output.prompt_len = request_func_input.prompt_len + # print(payload) + + generated_text = "" + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + actual_prompt_len = prompt_len - 1 + actual_output_len = 0 + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + timestamp = time.perf_counter() + # NOTE: Some completion API might have a last + # usage summary response without a token so we + # want to check a token was generated + if data["usage"] is not None and len(data["usage"]) > 0: + actual_prompt_len = data["usage"]["prompt_tokens"] + actual_output_len = data["usage"]["completion_tokens"] + continue + delta = data["choices"][0]["delta"] + + if delta.get("content", None): + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft.append(ttft) + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + generated_text += delta["content"] + most_recent_timestamp = timestamp + + output.prompt_len.append(actual_prompt_len) # truncate + output.output_len.append(actual_output_len) + output.generated_text.append(generated_text) + output.success = True + output.latency.append(latency) + + # Prepare for the new request + request_func_input.prompts[prompt_idx] = ( + prompt, + input_len, + actual_output_len, # changes from max_tokens to output_len + ) + prompt_idx += 1 + messages.append( + { + "role": "assistant", + "content": generated_text, + } + ) + + # Move the new request to the end of the queue + if prompt_idx < len(request_func_input.prompts): + request_func_input.finished_prompts = prompt_idx + request_func_input.prev_messages = messages + await queue.put(request_func_input) + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +async def async_request_profile(api_url: str) -> RequestFuncOutput: + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session: + output = RequestFuncOutput() + try: + async with session.post(url=api_url) as response: + if response.status == 200: + output.success = True + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + return output + + +ASYNC_REQUEST_FUNCS = { + "sglang": async_request_openai_completions, + "vllm": async_request_openai_completions, + "lmdeploy": async_request_openai_completions, +} + + +@dataclass +class BenchmarkMetrics: + completed: int + total_input: int + total_output: int + total_output_retokenized: int + request_throughput: float + input_throughput: float + output_throughput: float + output_throughput_retokenized: float + total_throughput: float + total_throughput_retokenized: float + mean_ttft_ms: float + median_ttft_ms: float + std_ttft_ms: float + p90_ttft_ms: float + p99_ttft_ms: float + mean_tpot_ms: float + median_tpot_ms: float + std_tpot_ms: float + p90_tpot_ms: float + p99_tpot_ms: float + mean_itl_ms: float + median_itl_ms: float + std_itl_ms: float + p90_itl_ms: float + p99_itl_ms: float + mean_e2e_latency_ms: float + median_e2e_latency_ms: float + std_e2e_latency_ms: float + p99_e2e_latency_ms: float + concurrency: float + + +async def get_requests( + input_requests_queue: asyncio.Queue, + request_rate: float, + num_actual_requests: int, +) -> AsyncGenerator[RequestFuncInput, None]: + for _ in range(num_actual_requests): + try: + request = await asyncio.wait_for( + input_requests_queue.get(), timeout=300 + ) # Wait for 5 minutes then abort + except Exception as e: + print(f"exception: {e}") + break + + yield request + + if request_rate == float("inf"): + continue + + interval = np.random.exponential(1.0 / request_rate) + await asyncio.sleep(interval) + + +def calculate_metrics( + outputs: List[RequestFuncOutput], + dur_s: float, + tokenizer: PreTrainedTokenizerBase, + backend: str, +) -> Tuple[BenchmarkMetrics, List[int]]: + output_lens: List[int] = [] + retokenized_output_lens: List[int] = [] + total_input = 0 + completed = 0 + itls: List[float] = [] + tpots: List[float] = [] + ttfts: List[float] = [] + e2e_latencies: List[float] = [] + output_success = 0 + for i in range(len(outputs)): + if outputs[i].success: + output_success += 1 + assert len(outputs[i].generated_text) == len(outputs[i].latency) + assert len(outputs[i].generated_text) == len(outputs[i].ttft) + for j in range(len(outputs[i].generated_text)): + output_len = outputs[i].output_len[j] + output_lens.append(output_len) + retokenized_output_len = len( + tokenizer.encode( + outputs[i].generated_text[j], add_special_tokens=False + ) + ) + retokenized_output_lens.append(retokenized_output_len) + total_input += outputs[i].prompt_len[j] + if output_len > 1: + tpots.append( + (outputs[i].latency[j] - outputs[i].ttft[j]) / (output_len - 1) + ) + + completed += 1 + itls += outputs[i].itl + ttfts += outputs[i].ttft + e2e_latencies += outputs[i].latency + + else: + output_lens.append(0) + retokenized_output_lens.append(0) + + if completed == 0: + warnings.warn( + "All requests failed. This is likely due to a misconfiguration " + "on the benchmark arguments.", + stacklevel=2, + ) + metrics = BenchmarkMetrics( + completed=completed, + total_input=total_input, + total_output=sum(output_lens), + total_output_retokenized=sum(retokenized_output_lens), + request_throughput=completed / dur_s, + input_throughput=total_input / dur_s, + output_throughput=sum(output_lens) / dur_s, + output_throughput_retokenized=sum(retokenized_output_lens) / dur_s, + total_throughput=(total_input + sum(output_lens)) / dur_s, + total_throughput_retokenized=(total_input + sum(retokenized_output_lens)) + / dur_s, + mean_ttft_ms=np.mean(ttfts or 0) + * 1000, # ttfts is empty if streaming is not supported by backend + median_ttft_ms=np.median(ttfts or 0) * 1000, + std_ttft_ms=np.std(ttfts or 0) * 1000, + p90_ttft_ms=np.percentile(ttfts or 0, 90) * 1000, + p99_ttft_ms=np.percentile(ttfts or 0, 99) * 1000, + mean_tpot_ms=np.mean(tpots or 0) * 1000, + median_tpot_ms=np.median(tpots or 0) * 1000, + std_tpot_ms=np.std(tpots or 0) * 1000, + p90_tpot_ms=np.percentile(tpots or 0, 90) * 1000, + p99_tpot_ms=np.percentile(tpots or 0, 99) * 1000, + mean_itl_ms=np.mean(itls or 0) * 1000, + median_itl_ms=np.median(itls or 0) * 1000, + std_itl_ms=np.std(itls or 0) * 1000, + p90_itl_ms=np.percentile(itls or 0, 90) * 1000, + p99_itl_ms=np.percentile(itls or 0, 99) * 1000, + mean_e2e_latency_ms=np.mean(e2e_latencies) * 1000, + median_e2e_latency_ms=np.median(e2e_latencies) * 1000, + std_e2e_latency_ms=np.std(e2e_latencies) * 1000, + p99_e2e_latency_ms=np.percentile(e2e_latencies, 99) * 1000, + concurrency=np.sum(e2e_latencies) / dur_s, + ) + + return metrics, output_lens + + +async def benchmark( + backend: str, + api_url: str, + base_url: str, + model_id: str, + tokenizer: PreTrainedTokenizerBase, + input_requests: SampleOutput, + request_rate: float, + max_concurrency: Optional[int], + disable_tqdm: bool, + lora_name: str, + extra_request_body: Dict[str, Any], + profile: bool, + enable_shared_prefix: bool, +): + if backend in ASYNC_REQUEST_FUNCS: + request_func = ASYNC_REQUEST_FUNCS[backend] + else: + raise ValueError(f"Unknown backend: {backend}") + + # Limit concurrency + # From https://github.com/vllm-project/vllm/pull/9390 + semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else None + + async def limited_request_func(request_func_input, queue, tokenizer, pbar): + if semaphore is None: + return await request_func( + request_func_input=request_func_input, + queue=queue, + tokenizer=tokenizer, + pbar=pbar, + ) + async with semaphore: + return await request_func( + request_func_input=request_func_input, + queue=queue, + tokenizer=tokenizer, + pbar=pbar, + ) + + num_actual_requests = sum(len(r) for r in input_requests) + print(f"Num of shared prefixes or conversations: {len(input_requests)}") + print(f"Num of total requests: {num_actual_requests}") + + # flatten the requests for shared prefix + if enable_shared_prefix: + input_requests = [[r] for requests in input_requests for r in requests] + inputs_requests_queue = asyncio.Queue(maxsize=len(input_requests)) + print("Starting initial single prompt test run...") + # NOTE: Just use the first request of the first conversation for warmup + test_input = RequestFuncInput( + model=model_id, + prompts=input_requests[0][:1], + api_url=api_url, + lora_name=lora_name, + extra_request_body=extra_request_body, + ) + test_output = await request_func( + request_func_input=test_input, queue=inputs_requests_queue, tokenizer=tokenizer + ) + if not test_output.success: + raise ValueError( + "Initial test run failed - Please make sure benchmark arguments " + f"are correctly specified. Error: {test_output.error}" + ) + else: + print("Initial test run completed. Starting main benchmark run...") + + # Check the states + assert inputs_requests_queue.empty() + + # Flush cache + if "sglang" in backend: + requests.post(base_url + "/flush_cache") + + time.sleep(1.0) + + # Start profiler + if profile: + print("Starting profiler...") + profile_output = await async_request_profile( + api_url=base_url + "/start_profile" + ) + if profile_output.success: + print("Profiler started") + + for request in input_requests: + request_func_input = RequestFuncInput( + model=model_id, + prompts=request, + api_url=api_url, + lora_name=lora_name, + extra_request_body=extra_request_body, + ) + inputs_requests_queue.put_nowait(request_func_input) + if ( + not args.enable_multiturn + and not args.enable_shared_prefix + and not args.dataset_name == "generated-shared-prefix" + ): + assert len(input_requests) == num_actual_requests + + pbar = None if disable_tqdm else tqdm(total=num_actual_requests) + + benchmark_start_time = time.perf_counter() + tasks: List[asyncio.Task] = [] + async for request in get_requests( + inputs_requests_queue, request_rate, num_actual_requests + ): + tasks.append( + asyncio.create_task( + limited_request_func( + request_func_input=request, + queue=inputs_requests_queue, + tokenizer=tokenizer, + pbar=pbar, + ) + ) + ) + outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks) + + # Stop profiler + if profile: + print("Stopping profiler...") + profile_output = await async_request_profile(api_url=base_url + "/stop_profile") + if profile_output.success: + print("Profiler stopped") + + if pbar is not None: + pbar.close() + + # Compute metrics and print results + benchmark_duration = time.perf_counter() - benchmark_start_time + metrics, output_lens = calculate_metrics( + outputs=outputs, + dur_s=benchmark_duration, + tokenizer=tokenizer, + backend=backend, + ) + + print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="=")) + print("{:<40} {:<10}".format("Backend:", backend)) + print("{:<40} {:<10}".format("Traffic request rate:", request_rate)) + print( + "{:<40} {:<10}".format( + "Max request concurrency:", + max_concurrency if max_concurrency else "not set", + ) + ) + print("{:<40} {:<10}".format("Successful requests:", metrics.completed)) + print("{:<40} {:<10.2f}".format("Benchmark duration (s):", benchmark_duration)) + print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input)) + print("{:<40} {:<10}".format("Total generated tokens:", metrics.total_output)) + print( + "{:<40} {:<10}".format( + "Total generated tokens (retokenized):", metrics.total_output_retokenized + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Request throughput (req/s):", metrics.request_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Input token throughput (tok/s):", metrics.input_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Output token throughput (tok/s):", metrics.output_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Total token throughput (tok/s):", metrics.total_throughput + ) + ) + print("{:<40} {:<10.2f}".format("Concurrency:", metrics.concurrency)) + print("{s:{c}^{n}}".format(s="End-to-End Latency", n=50, c="-")) + print( + "{:<40} {:<10.2f}".format("Mean E2E Latency (ms):", metrics.mean_e2e_latency_ms) + ) + print( + "{:<40} {:<10.2f}".format( + "Median E2E Latency (ms):", metrics.median_e2e_latency_ms + ) + ) + print("{s:{c}^{n}}".format(s="Time to First Token", n=50, c="-")) + print("{:<40} {:<10.2f}".format("Mean TTFT (ms):", metrics.mean_ttft_ms)) + print("{:<40} {:<10.2f}".format("Median TTFT (ms):", metrics.median_ttft_ms)) + print("{:<40} {:<10.2f}".format("P90 TTFT (ms):", metrics.p90_ttft_ms)) + print("{:<40} {:<10.2f}".format("P99 TTFT (ms):", metrics.p99_ttft_ms)) + print( + "{s:{c}^{n}}".format(s="Time per Output Token (excl. 1st token)", n=50, c="-") + ) + print("{:<40} {:<10.2f}".format("Mean TPOT (ms):", metrics.mean_tpot_ms)) + print("{:<40} {:<10.2f}".format("Median TPOT (ms):", metrics.median_tpot_ms)) + print("{:<40} {:<10.2f}".format("P90 TPOT (ms):", metrics.p90_tpot_ms)) + print("{:<40} {:<10.2f}".format("P99 TPOT (ms):", metrics.p99_tpot_ms)) + print("{s:{c}^{n}}".format(s="Inter-token Latency", n=50, c="-")) + print("{:<40} {:<10.2f}".format("Mean ITL (ms):", metrics.mean_itl_ms)) + print("{:<40} {:<10.2f}".format("Median ITL (ms):", metrics.median_itl_ms)) + print("{:<40} {:<10.2f}".format("P90 ITL (ms):", metrics.p90_itl_ms)) + print("{:<40} {:<10.2f}".format("P99 ITL (ms):", metrics.p99_itl_ms)) + print("=" * 50) + + if ( + metrics.median_ttft_ms is not None + and metrics.mean_itl_ms is not None + and metrics.output_throughput is not None + ): + result = { + # Arguments + "backend": args.backend, + "dataset_name": args.dataset_name, + "request_rate": request_rate, + "max_concurrency": max_concurrency, + "fixed_output_len": args.fixed_output_len, + "random_input_len": args.random_input_len, + "random_output_len": args.random_output_len, + "random_range_ratio": args.random_range_ratio, + # Results + "duration": benchmark_duration, + "completed": metrics.completed, + "total_input_tokens": metrics.total_input, + "total_output_tokens": metrics.total_output, + "total_output_tokens_retokenized": metrics.total_output_retokenized, + "request_throughput": metrics.request_throughput, + "input_throughput": metrics.input_throughput, + "output_throughput": metrics.output_throughput, + "mean_e2e_latency_ms": metrics.mean_e2e_latency_ms, + "median_e2e_latency_ms": metrics.median_e2e_latency_ms, + "std_e2e_latency_ms": metrics.std_e2e_latency_ms, + "p99_e2e_latency_ms": metrics.p99_e2e_latency_ms, + "mean_ttft_ms": metrics.mean_ttft_ms, + "median_ttft_ms": metrics.median_ttft_ms, + "std_ttft_ms": metrics.std_ttft_ms, + "p99_ttft_ms": metrics.p99_ttft_ms, + "mean_tpot_ms": metrics.mean_tpot_ms, + "median_tpot_ms": metrics.median_tpot_ms, + "std_tpot_ms": metrics.std_tpot_ms, + "p99_tpot_ms": metrics.p99_tpot_ms, + "mean_itl_ms": metrics.mean_itl_ms, + "median_itl_ms": metrics.median_itl_ms, + "std_itl_ms": metrics.std_itl_ms, + "p99_itl_ms": metrics.p99_itl_ms, + "concurrency": metrics.concurrency, + "input_throughput": metrics.input_throughput, + "output_throughput": metrics.output_throughput, + "fixed_output_len": args.fixed_output_len, + "random_input_len": args.random_input_len, + "random_output_len": args.random_output_len, + "random_range_ratio": args.random_range_ratio, + "duration": benchmark_duration, + "completed": metrics.completed, + } + else: + print(f"Error running benchmark for request rate: {request_rate}") + print("-" * 30) + + # Determine output file name + if args.output_file: + output_file_name = args.output_file + else: + now = datetime.now().strftime("%m%d") + if args.dataset_name == "random": + output_file_name = f"{args.backend}_{now}_{args.num_prompts}_{args.random_input_len}_{args.random_output_len}.jsonl" + else: + output_file_name = ( + f"{args.backend}_{now}_{args.num_prompts}_{args.dataset_name}.jsonl" + ) + + # Append results to a JSONL file + with open(output_file_name, "a") as file: + file.write(json.dumps(result) + "\n") + + result = { + "duration": benchmark_duration, + "completed": metrics.completed, + "total_input_tokens": metrics.total_input, + "total_output_tokens": metrics.total_output, + "total_output_tokens_retokenized": metrics.total_output_retokenized, + "request_throughput": metrics.request_throughput, + "input_throughput": metrics.input_throughput, + "output_throughput": metrics.output_throughput, + "mean_ttft_ms": metrics.mean_ttft_ms, + "median_ttft_ms": metrics.median_ttft_ms, + "std_ttft_ms": metrics.std_ttft_ms, + "p90_ttft_ms": metrics.p90_ttft_ms, + "p99_ttft_ms": metrics.p99_ttft_ms, + "mean_tpot_ms": metrics.mean_tpot_ms, + "median_tpot_ms": metrics.median_tpot_ms, + "std_tpot_ms": metrics.std_tpot_ms, + "p90_tpot_ms": metrics.p90_tpot_ms, + "p99_tpot_ms": metrics.p99_tpot_ms, + "mean_itl_ms": metrics.mean_itl_ms, + "median_itl_ms": metrics.median_itl_ms, + "std_itl_ms": metrics.std_itl_ms, + "p90_itl_ms": metrics.p90_itl_ms, + "p99_itl_ms": metrics.p99_itl_ms, + "input_lens": [output.prompt_len for output in outputs], + "output_lens": output_lens, + "ttfts": [output.ttft for output in outputs], + "itls": [output.itl for output in outputs], + "generated_texts": [output.generated_text for output in outputs], + "errors": [output.error for output in outputs], + "mean_e2e_latency_ms": metrics.mean_e2e_latency_ms, + "median_e2e_latency_ms": metrics.median_e2e_latency_ms, + } + return result + + +def run_benchmark(args_: argparse.Namespace): + global args + args = args_ + + # Set default value for max_concurrency if not present + if not hasattr(args, "max_concurrency"): + args.max_concurrency = None + + # Set global environments + set_ulimit() + random.seed(args.seed) + np.random.seed(args.seed) + + extra_request_body = {} + if args.extra_request_body: + extra_request_body = json.loads(args.extra_request_body) + + # Set url + if args.port is None: + args.port = { + "sglang": 30000, + "lmdeploy": 23333, + "vllm": 8000, + }.get(args.backend, 30000) + + model_url = ( + f"{args.base_url}/v1/models" + if args.base_url + else f"http://{args.host}:{args.port}/v1/models" + ) + + if args.backend in ["sglang", "vllm", "lmdeploy"]: + api_url = ( + f"{args.base_url}/v1/chat/completions" + if args.base_url + else f"http://{args.host}:{args.port}/v1/chat/completions" + ) + base_url = ( + f"http://{args.host}:{args.port}" if args.base_url is None else args.base_url + ) + + # Get model name + if args.model is None: + if args.backend == "truss": + print( + "Please provide a model with `--model` when using truss backend. e.g. --model meta-llama/Llama-3.1-8B-Instruct" + ) + sys.exit(1) + try: + response = requests.get(model_url) + model_list = response.json().get("data", []) + args.model = model_list[0]["id"] if model_list else None + except Exception as e: + print(f"Failed to fetch model from {model_url}. Error: {e}") + print( + "Please specify the correct host and port using `--host` and `--port`." + ) + sys.exit(1) + + if args.model is None: + print("No model specified or found. Please provide a model using `--model`.") + sys.exit(1) + + # Dataset compatibility check + if args.enable_multiturn: + # TODO: Support multiturn for random + if args.dataset_name not in ["sharegpt", "ultrachat", "loogle", "nextqa"]: + print( + "Multiturn conversation is only supported for sharegpt, ultrachat, loogle, and nextqa datasets." + ) + sys.exit(1) + + if args.enable_shared_prefix: + if args.dataset_name not in ["loogle", "nextqa"]: + print("Shared prefix is only supported for loogle and nextqa datasets.") + sys.exit(1) + + print(f"{args}\n") + + # Read dataset + backend = args.backend + model_id = args.model + tokenizer_id = args.tokenizer if args.tokenizer is not None else args.model + + tokenizer = get_tokenizer(tokenizer_id) + + input_requests = get_dataset(args, tokenizer) + + return asyncio.run( + benchmark( + backend=backend, + api_url=api_url, + base_url=base_url, + model_id=model_id, + tokenizer=tokenizer, + input_requests=input_requests, + request_rate=args.request_rate, + max_concurrency=args.max_concurrency, + disable_tqdm=args.disable_tqdm, + lora_name=args.lora_name, + extra_request_body=extra_request_body, + profile=args.profile, + enable_shared_prefix=args.enable_shared_prefix, + ) + ) + + +if __name__ == "__main__": + parser = ArgumentParser(description="Benchmark the online serving throughput.") + parser.add_argument( + "--backend", + type=str, + choices=list(ASYNC_REQUEST_FUNCS.keys()), + default="sglang", + help="Must specify a backend, depending on the LLM Inference Engine.", + ) + parser.add_argument( + "--base-url", + type=str, + default=None, + help="Server or API base url if not using http host and port.", + ) + parser.add_argument( + "--host", type=str, default="0.0.0.0", help="Default host is 0.0.0.0." + ) + parser.add_argument( + "--port", + type=int, + help="If not set, the default port is configured according to its default value for different LLM Inference Engines.", + ) + parser.add_argument( + "--dataset-name", + type=str, + default="sharegpt", + choices=[ + "sharegpt", + "random", + "generated-shared-prefix", + "ultrachat", + "loogle", + "nextqa", + ], + help="Name of the dataset to benchmark on.", + ) + parser.add_argument( + "--dataset-path", type=str, default="", help="Path to the dataset." + ) + parser.add_argument( + "--model", + type=str, + help="Name or path of the model. If not set, the default model will request /v1/models for conf.", + ) + parser.add_argument( + "--tokenizer", + type=str, + help="Name or path of the tokenizer. If not set, using the model conf.", + ) + parser.add_argument( + "--chat-template", + type=str, + help="The buliltin chat template name or the path of the chat template file. This is only used for OpenAI-compatible API server.", + ) + parser.add_argument( + "--num-prompts", + type=int, + default=1000, + help="Number of prompts to process. Default is 1000.", + ) + parser.add_argument( + "--fixed-output-len", + type=int, + default=None, + help="Output length for each request. Overrides the output length from the dataset.", + ) + parser.add_argument( + "--sharegpt-context-len", + type=int, + default=None, + help="The context length of the model for the ShareGPT dataset. Requests longer than the context length will be dropped.", + ) + parser.add_argument( + "--random-input-len", + type=int, + default=1024, + help="Number of input tokens per request, used only for random dataset.", + ) + parser.add_argument( + "--random-output-len", + default=1024, + type=int, + help="Number of output tokens per request, used only for random dataset.", + ) + parser.add_argument( + "--random-range-ratio", + type=float, + default=0.0, + help="Range of sampled ratio of input/output length, " + "used only for random dataset.", + ) + parser.add_argument( + "--request-rate", + type=float, + default=float("inf"), + help="Number of requests per second. If this is inf, then all the requests are sent at time 0. " + "Otherwise, we use Poisson process to synthesize the request arrival times. Default is inf.", + ) + parser.add_argument( + "--max-concurrency", + type=int, + default=None, + help="Maximum number of concurrent requests. This can be used " + "to help simulate an environment where a higher level component " + "is enforcing a maximum number of concurrent requests. While the " + "--request-rate argument controls the rate at which requests are " + "initiated, this argument will control how many are actually allowed " + "to execute at a time. This means that when used in combination, the " + "actual request rate may be lower than specified with --request-rate, " + "if the server is not processing requests fast enough to keep up.", + ) + parser.add_argument( + "--multi", + action="store_true", + help="Use request rate range rather than single value.", + ) + parser.add_argument( + "--request-rate-range", + type=str, + default="2,34,2", + help="Range of request rates in the format start,stop,step. Default is 2,34,2. It also supports a list of request rates, requiring the parameters to not equal three.", + ) + parser.add_argument("--output-file", type=str, help="Output JSONL file name.") + parser.add_argument( + "--enable-multiturn", + action="store_true", + help="Enable multiturn chat for online serving benchmarking. " + "This option is effective on the following datasets: " + "sharegpt, ultrachat, loogle, nextqa", + ) + parser.add_argument( + "--enable-shared-prefix", + action="store_true", + help="Enable shared prefix for online serving benchmarking. " + "This option is effective on the following datasets: " + "loogle, nextqa", + ) + + parser.add_argument( + "--disable-shuffle", + action="store_true", + help="Disable shuffling datasets. This is useful to generate stable output " + "in benchmarking", + ) + parser.add_argument( + "--disable-tqdm", + action="store_true", + help="Specify to disable tqdm progress bar.", + ) + parser.add_argument( + "--disable-stream", + action="store_true", + help="Disable streaming mode.", + ) + parser.add_argument( + "--return-logprob", + action="store_true", + help="Return logprob.", + ) + parser.add_argument("--seed", type=int, default=1, help="The random seed.") + parser.add_argument( + "--disable-ignore-eos", + action="store_true", + help="Disable ignoring EOS.", + ) + parser.add_argument( + "--extra-request-body", + metavar='{"key1": "value1", "key2": "value2"}', + type=str, + help="Append given JSON object to the request payload. You can use this to specify" + "additional generate params like sampling params.", + ) + parser.add_argument( + "--apply-chat-template", + action="store_true", + help="Apply chat template", + ) + parser.add_argument( + "--profile", + action="store_true", + help="Use Torch Profiler. The endpoint must be launched with " + "SGLANG_TORCH_PROFILER_DIR to enable profiler.", + ) + parser.add_argument( + "--lora-name", + type=str, + default=None, + help="The name of LoRA adapter", + ) + + group = parser.add_argument_group("generated-shared-prefix dataset arguments") + group.add_argument( + "--gsp-num-groups", + type=int, + default=64, + help="Number of system prompt groups for generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-prompts-per-group", + type=int, + default=16, + help="Number of prompts per system prompt group for generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-system-prompt-len", + type=int, + default=2048, + help="Target length in tokens for system prompts in generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-question-len", + type=int, + default=128, + help="Target length in tokens for questions in generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-output-len", + type=int, + default=256, + help="Target length in tokens for outputs in generated-shared-prefix dataset", + ) + # videos specific + parser.add_argument( + "--max-frames", + type=int, + default=sys.maxsize, + help="The maximum number of frames to extract from each video. " + "This option is specific to the nextqa dataset (video benchmark). ", + ) + args = parser.parse_args() + + if args.enable_multiturn and args.enable_shared_prefix: + parser.error( + "--enable-multiturn and --enable-shared-prefix cannot be set at the same time." + ) + + run_benchmark(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/data_processing.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/data_processing.py new file mode 100644 index 00000000..1fb3650c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/data_processing.py @@ -0,0 +1,569 @@ +import json +import os +import pickle +import random +from typing import List, Optional, Tuple, Union + +import numpy as np +from nextqa import NExTQALoader + +# from nextqa.video import , VideoPrompt +from tqdm.asyncio import tqdm +from transformers import PreTrainedTokenizerBase + +SHAREGPT_URL = "https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json" + +from sglang.bench_serving import ( + download_and_cache_file, + gen_prompt, + get_gen_prefix_cache_path, +) +from sglang.lang.chat_template import get_chat_template, get_chat_template_by_model_path +from sglang.srt.entrypoints.openai.protocol import ChatCompletionMessageContentPart +from sglang.utils import encode_video_base64 + +# type of content fields, can be only prompts or with images/videos +MsgContent = Union[str, List[ChatCompletionMessageContentPart]] + +# A list of all the conversations. Each conversation is a list of +# tuples. If multiturn is not enabled, the length of list is 1, +# containing only the first Q&A pair. +# For the shared prefix workload (synthetic, loogle, nextqa), it +# is a list of conversations sharing the same prefix (synthetic, +# doc, video) +SampleOutput = List[List[Tuple[MsgContent, int, int]]] + + +def common_filter_chat( + num_requests: int, + new_dataset: List, + tokenizer: PreTrainedTokenizerBase, + min_prompt_len: Optional[int], + min_output_len: Optional[int], + max_prompt_len: Optional[int], + max_output_len: Optional[int], + fixed_output_len: Optional[int], +) -> SampleOutput: + # Filter out sequences that are too long or too short + filtered_dataset: SampleOutput = [] + l = 0 + input_tokens = 0 + output_tokens = 0 + while l < num_requests: + for i in range(len(new_dataset)): + if l == num_requests: + break + processed = [] + for j in new_dataset[i]: + # Tokenize the prompts and completions. + prompt = j[0] + prompt_token_ids = tokenizer.encode(prompt) + prompt_len = len(prompt_token_ids) + + completion = j[1] + completion_token_ids = tokenizer.encode(completion) + output_len = ( + len(completion_token_ids) + if fixed_output_len is None + else fixed_output_len + ) + if ( + min_prompt_len is not None + and prompt_len < min_prompt_len + or min_output_len is not None + and output_len < min_output_len + or max_prompt_len is not None + and prompt_len > max_prompt_len + or max_output_len is not None + and output_len > max_output_len + ): + # Prune too short sequences. + continue + input_tokens += prompt_len + output_tokens += output_len + processed.append((prompt, prompt_len, output_len)) + if len(processed) != 0: + filtered_dataset.append(processed) + l += 1 + + print(f"#Input tokens: {input_tokens}") + print(f"#Output tokens: {output_tokens}") + return filtered_dataset + + +def sample_sharegpt_requests( + dataset_path: str, + num_requests: int, + tokenizer: PreTrainedTokenizerBase, + disable_shuffle: bool = False, + enable_multiturn: bool = True, + fixed_output_len: Optional[int] = None, +) -> SampleOutput: + if fixed_output_len is not None and fixed_output_len < 4: + raise ValueError("output_len too small") + + # Download sharegpt if necessary + if not os.path.isfile(dataset_path): + dataset_path = download_and_cache_file(SHAREGPT_URL) + + # Load the dataset. + with open(dataset_path) as f: + dataset = json.load(f) + # Filter out the conversations with less than 2 turns. + dataset = [data for data in dataset if len(data["conversations"]) >= 2] + + # Keep one conversation in one list + new_dataset = [] + for data in dataset: + if len(data["conversations"]) % 2 != 0: + continue + if data["conversations"][0]["from"] != "human": + continue + chat = [] + total_len = 2 + if enable_multiturn: + total_len = len(data["conversations"]) + for i in range(0, total_len, 2): + # One user One Assistant + chat.append( + ( + data["conversations"][i]["value"], + data["conversations"][i + 1]["value"], + ) + ) + new_dataset.append(chat) + + if not disable_shuffle: + # Shuffle the dataset. + random.shuffle(new_dataset) + + # Filter out sequences that are too long or too short + filtered_dataset: SampleOutput = common_filter_chat( + num_requests, new_dataset, tokenizer, 4, 4, None, None, fixed_output_len + ) + return filtered_dataset + + +def sample_ultrachat_requests( + dataset_path: str, + num_requests: int, + tokenizer: PreTrainedTokenizerBase, + disable_shuffle: bool = False, + enable_multiturn: bool = True, + fixed_output_len: Optional[int] = None, +) -> SampleOutput: + if fixed_output_len is not None and fixed_output_len < 4: + raise ValueError("output_len too small") + + # Load the dataset + dataset = [] + with open(dataset_path) as f: + while True: + line = f.readline() + if not line: + break + dataset.append(json.loads(line)) + + # Filter out the conversations with less than 2 turns. + dataset = [data for data in dataset if len(data["data"]) >= 2] + + # Keep one conversation in one list + new_dataset = [] + for data in dataset: + if len(data["data"]) % 2 != 0: + continue + chat = [] + total_len = 2 + if enable_multiturn: + total_len = len(data["data"]) + for i in range(0, total_len, 2): + # One user One Assistant + chat.append((data["data"][i], data["data"][i + 1])) + new_dataset.append(chat) + + # Shuffle the dataset. + if not disable_shuffle: + random.shuffle(new_dataset) + + # Filter out sequences that are too long or too short + filtered_dataset: SampleOutput = common_filter_chat( + num_requests, new_dataset, tokenizer, 4, 4, None, None, fixed_output_len + ) + return filtered_dataset + + +def sample_loogle_requests( + dataset_path: str, + num_requests: int, + tokenizer: PreTrainedTokenizerBase, + disable_shuffle: bool = False, + enable_multiturn: bool = True, + enable_shared_prefix: bool = False, + fixed_output_len: Optional[int] = None, +) -> SampleOutput: + if fixed_output_len is not None and fixed_output_len < 4: + raise ValueError("output_len too small") + + # Load the dataset + dataset = [] + with open(dataset_path) as f: + while True: + line = f.readline() + if not line: + break + dataset.append(json.loads(line)) + + # Keep one conversation in one list + new_dataset = [] + # TODO: Add shared prefix support for loogle + # NOTE: Now we preprocess it only for chat + for data in dataset: + chat = [] + if ( + "qa_pairs" not in data + or data["qa_pairs"] == "none" + or len(data["qa_pairs"]) == 0 + ): + # If Q is none (for summarization), + # We add a question for summarization + # And keep the summary up to 1024 words + chat.append( + ( + "Input: " + + data["input"] + + " Question: " + + "Please summarize the input", + data["input"][:1024], + ) + ) + new_dataset.append(chat) + else: + qa_pairs = eval(data["qa_pairs"]) + for i, qa in enumerate(qa_pairs): + if i == 0 or enable_shared_prefix: + # Combine input with the first Q + chat.append( + ("Input: " + data["input"] + " Question: " + qa["Q"], qa["A"]) + ) + elif enable_multiturn: + chat.append((qa["Q"], qa["A"])) + + new_dataset.append(chat) + + # Shuffle the dataset. + if not disable_shuffle: + random.shuffle(new_dataset) + + # Filter out sequences that are too long or too short + filtered_dataset: SampleOutput = common_filter_chat( + num_requests, new_dataset, tokenizer, 4, None, None, None, fixed_output_len + ) + return filtered_dataset + + +def sample_nextqa_requests( + dataset_path: str, + num_requests: int, + tokenizer: PreTrainedTokenizerBase, + max_frames: int, # Specific for video + model_path: str, + disable_shuffle: bool = False, + enable_multiturn: bool = True, # No multiturn support for now + backend: str = "sglang-oai", + chat_template_name: Optional[str] = None, + fixed_output_len: Optional[int] = None, +) -> SampleOutput: + """ + Example of messages: + message = { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": base64_data}}, + {"type": "text", "text": video.prompt}, + ], + } + """ + + if fixed_output_len is None: + fixed_output_len = 4096 + + # TODO: Check for multiturn + dataset = NExTQALoader(video_dir=dataset_path, max_frames=max_frames) + new_dataset = [] + for v in dataset: + new_dataset.append(v) + + if not disable_shuffle: + random.shuffle(new_dataset) + + # TODO: prompt len can get from server side + filtered_dataset = [] + l = 0 + while l < num_requests: + for i in range(len(new_dataset)): + if l == num_requests: + break + + video = new_dataset[i] + + # text prompt + prompt = video.prompt + + # NOTE: Chat Template is a must for video benchmark because we have to + # add special image token for later expansion + if backend == "sglang" or backend == "sglang-native": + if "chat_template" in tokenizer.init_kwargs: + chat_template = get_chat_template(tokenizer.get_chat_template()) + elif chat_template_name is not None: + chat_template = get_chat_template(chat_template_name) + else: + chat_template = get_chat_template_by_model_path(model_path) + prompt = chat_template.image_token + prompt + + prompt_token_ids = tokenizer(prompt).input_ids + prompt_len = len(prompt_token_ids) + output_len = fixed_output_len # max output len, not real output len + + # video input + base64_data = encode_video_base64(video.path, video.num_frames) + + # NOTE: This will be replaced by the expanded length from the server + prompt_len += video.num_frames + + # add to content + content = [ + {"type": "image_url", "image_url": {"url": base64_data}}, + {"type": "text", "text": prompt}, + ] + + filtered_dataset.append([(content, prompt_len, output_len)]) + l += 1 + return filtered_dataset + + +def sample_random_requests( + input_len: int, + output_len: int, + num_prompts: int, + range_ratio: float, + tokenizer: PreTrainedTokenizerBase, + dataset_path: str, + disable_shuffle: bool = False, +) -> SampleOutput: + + input_lens = np.random.randint( + max(int(input_len * range_ratio), 1), + input_len + 1, + size=num_prompts, + ) + output_lens = np.random.randint( + int(output_len * range_ratio), + output_len + 1, + size=num_prompts, + ) + + if True: + # Sample token ids from ShareGPT and repeat/truncate them to satisfy the input_lens + + # Download sharegpt if necessary + if not os.path.isfile(dataset_path): + dataset_path = download_and_cache_file(SHAREGPT_URL) + + # Load the dataset. + with open(dataset_path) as f: + dataset = json.load(f) + # Filter out the conversations with less than 2 turns. + dataset = [data for data in dataset if len(data["conversations"]) >= 2] + # Only keep the first two turns of each conversation. + dataset = [ + (data["conversations"][0]["value"], data["conversations"][1]["value"]) + for data in dataset + ] + + if not disable_shuffle: + # Shuffle the dataset. + random.shuffle(dataset) + + # Filter out sequences that are too long or too short + input_requests: SampleOutput = [] + for data in dataset: + i = len(input_requests) + if i == num_prompts: + break + + # Tokenize the prompts and completions. + prompt = data[0] + prompt_token_ids = tokenizer.encode(prompt) + prompt_len = len(prompt_token_ids) + + # Skip empty prompt + if prompt_len == 0: + continue + + if prompt_len > input_lens[i]: + input_ids = prompt_token_ids[: input_lens[i]] + else: + ratio = (input_lens[i] + prompt_len - 1) // prompt_len + input_ids = (prompt_token_ids * ratio)[: input_lens[i]] + prompt = tokenizer.decode(input_ids) + input_requests.append([(prompt, int(input_lens[i]), int(output_lens[i]))]) + else: + # Sample token ids from random integers. This can cause some NaN issues. + offsets = np.random.randint(0, tokenizer.vocab_size, size=num_prompts) + input_requests = [] + for i in range(num_prompts): + prompt = tokenizer.decode( + [ + (offsets[i] + i + j) % tokenizer.vocab_size + for j in range(input_lens[i]) + ] + ) + input_requests.append([(prompt, int(input_lens[i]), int(output_lens[i]))]) + + print(f"#Input tokens: {np.sum(input_lens)}") + print(f"#Output tokens: {np.sum(output_lens)}") + return input_requests + + +def sample_generated_shared_prefix_requests( + num_groups: int, + prompts_per_group: int, + system_prompt_len: int, + question_len: int, + output_len: int, + tokenizer: PreTrainedTokenizerBase, + args, + disable_shuffle: bool = False, +) -> SampleOutput: + """Generate benchmark requests with shared system prompts using random tokens and caching.""" + cache_path = get_gen_prefix_cache_path(args, tokenizer) + + # Try to load from cache first + if cache_path.exists(): + print(f"\nLoading cached generated input data from {cache_path}") + with open(cache_path, "rb") as f: + return pickle.load(f) + + print("\nGenerating new input data...") + + # Generate system prompts for each group + system_prompts = [] + for _ in range(num_groups): + system_prompt = gen_prompt(tokenizer, system_prompt_len) + system_prompts.append(system_prompt) + + # Generate questions + questions = [] + for _ in range(num_groups * prompts_per_group): + question = gen_prompt(tokenizer, question_len) + questions.append(question) + + # Combine system prompts with questions + input_requests = [] + total_input_tokens = 0 + total_output_tokens = 0 + + for group_idx in tqdm(range(num_groups), desc="Generating system prompt"): + system_prompt = system_prompts[group_idx] + input_requests.append([]) + for prompt_idx in tqdm( + range(prompts_per_group), desc="Generating questions", leave=False + ): + question = questions[group_idx * prompts_per_group + prompt_idx] + full_prompt = f"{system_prompt}\n\n{question}" + prompt_len = len(tokenizer.encode(full_prompt)) + input_requests[-1].append((full_prompt, prompt_len, output_len)) + total_input_tokens += prompt_len + total_output_tokens += output_len + + if not disable_shuffle: + # Shuffle questions + random.shuffle(input_requests) + + # Print statistics + print(f"\nGenerated shared prefix dataset statistics:") + print(f"Number of groups: {num_groups}") + print(f"Prompts per group: {prompts_per_group}") + print(f"Total prompts: {len(input_requests) * prompts_per_group}") + print(f"Total input tokens: {total_input_tokens}") + print(f"Total output tokens: {total_output_tokens}") + print( + f"Average system prompt length: {sum(len(tokenizer.encode(sp)) for sp in system_prompts) / len(system_prompts):.1f} tokens" + ) + print( + f"Average question length: {sum(len(tokenizer.encode(q)) for q in questions) / len(questions):.1f} tokens\n" + ) + + # Save to cache + cache_path.parent.mkdir(parents=True, exist_ok=True) + print(f"Caching generated input data to {cache_path}") + with open(cache_path, "wb") as f: + pickle.dump(input_requests, f) + + return input_requests + + +def get_dataset(args, tokenizer): + if args.dataset_name == "sharegpt": + input_requests = sample_sharegpt_requests( + dataset_path=args.dataset_path, + num_requests=args.num_prompts, + tokenizer=tokenizer, + disable_shuffle=args.disable_shuffle, + enable_multiturn=args.enable_multiturn, + fixed_output_len=args.fixed_output_len, + ) + elif args.dataset_name == "ultrachat": + input_requests = sample_ultrachat_requests( + dataset_path=args.dataset_path, + num_requests=args.num_prompts, + tokenizer=tokenizer, + disable_shuffle=args.disable_shuffle, + enable_multiturn=args.enable_multiturn, + fixed_output_len=args.fixed_output_len, + ) + elif args.dataset_name == "loogle": + input_requests = sample_loogle_requests( + dataset_path=args.dataset_path, + num_requests=args.num_prompts, + tokenizer=tokenizer, + disable_shuffle=args.disable_shuffle, + enable_multiturn=args.enable_multiturn, + enable_shared_prefix=args.enable_shared_prefix, + fixed_output_len=args.fixed_output_len, + ) + elif args.dataset_name == "nextqa": + input_requests = sample_nextqa_requests( + dataset_path=args.dataset_path, + num_requests=args.num_prompts, + tokenizer=tokenizer, + max_frames=args.max_frames, + model_path=args.model, + disable_shuffle=args.disable_shuffle, + enable_multiturn=args.enable_multiturn, + backend=args.backend, + chat_template_name=args.chat_template, + fixed_output_len=args.fixed_output_len, + ) + elif args.dataset_name == "random": + input_requests = sample_random_requests( + input_len=args.random_input_len, + output_len=args.random_output_len, + num_prompts=args.num_prompts, + range_ratio=args.random_range_ratio, + tokenizer=tokenizer, + dataset_path=args.dataset_path, + ) + elif args.dataset_name == "generated-shared-prefix": + input_requests = sample_generated_shared_prefix_requests( + num_groups=args.gsp_num_groups, + prompts_per_group=args.gsp_prompts_per_group, + system_prompt_len=args.gsp_system_prompt_len, + question_len=args.gsp_question_len, + output_len=args.gsp_output_len, + args=args, + tokenizer=tokenizer, + ) + else: + raise ValueError(f"Unknown dataset: {args.dataset_name}") + return input_requests diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/download.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/download.sh new file mode 100755 index 00000000..340bcc3b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/download.sh @@ -0,0 +1,66 @@ +#!/usr/bin/bash + +# The usage function +usage() { + echo "Usage: $0 {sharegpt|ultragpt|loogle|nextqa|all}" + exit 1 +} + +# The download function +download() { + case "$1" in + sharegpt) + echo $1 + wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json + ;; + ultragpt) + echo $1 + # Questions about the world + wget https://cloud.tsinghua.edu.cn/seafhttp/files/be1d7b87-22ca-449e-a6a7-c61d1ea7e010/ultrachat_release_230407.json + # Writing and Creation + wget https://cloud.tsinghua.edu.cn/seafhttp/files/61742d2a-25e2-4d08-b2b9-15f47ae50ace/ultrachat_material_release_230417.json + wget https://cloud.tsinghua.edu.cn/seafhttp/files/f71f6aa6-d346-4b16-85b7-8502efa3d608/ultrachat_material_release_230412.json + # External materials + wget https://cloud.tsinghua.edu.cn/seafhttp/files/42d22e28-e899-4975-a70f-5eda163e265d/ultrachat_existent_material_release_230420.json.gz + gunzip ultrachat_existent_material_release_230420.json.gz + ;; + loogle) + echo $1 + git lfs install + git clone git@hf.co:datasets/bigainlco/LooGLE + unzip LooGLE/data.zip + ;; + nextqa) + echo $1 + git lfs install + git clone https://huggingface.co/datasets/lmms-lab/NExTQA + unzip NExTQA/videos.zip + ;; + *) + usage + exit 1 + ;; + esac +} + +# Arg check +if [ "$#" -ne 1 ]; then + usage +fi + +# Invoke + +case "$1" in + sharegpt|ultragpt|loogle|nextqa) + download "$1" + ;; + all) + download sharegpt + download ultragpt + download loogle + download nextqa + ;; + *) + usage + ;; +esac diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/nextqa.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/nextqa.py new file mode 100644 index 00000000..4db6caa1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/nextqa.py @@ -0,0 +1,159 @@ +import os +import sys +from typing import List + +import av +from datasets import load_dataset + + +def find_video_files(video_dir) -> List[str]: + if os.path.isfile(video_dir): + return [video_dir] + + video_files = [] + for root, dirs, files in os.walk(video_dir): + for file in files: + if file.endswith((".mp4", ".avi", ".mov")): + video_files.append(os.path.join(root, file)) + # if file is dir + elif os.path.isdir(file): + video_files.extend(find_video_files(file)) + return video_files + + +def video_frames(video_path, max_frames) -> int: + container = av.open(video_path) + total_frames = container.streams.video[0].frames + return min(total_frames, max_frames) + + +class Video: + def __init__(self, video_path, num_frames): + self.path = video_path + self.num_frames = num_frames + + def __str__(self): + return f"Video({self.path}, {self.num_frames})" + + def __iter__(self): + return iter((self.path, self.num_frames)) + + +class VideoPrompt(Video): + def __init__(self, video_path, num_frames, prompt): + super().__init__(video_path, num_frames) + self.prompt = prompt + + def __str__(self): + return f"VideoPrompt({self.path}, {self.num_frames}, {self.prompt})" + + def __iter__(self): + return iter((self.path, self.num_frames, self.prompt)) + + +class VideoLoader: + pass + + +class VideoFileLoader(VideoLoader): + """ + Load all the videos in a directory + """ + + def __init__(self, video_dir, batch_size=1, max_frames=sys.maxsize): + super().__init__() + self.video_dir = video_dir + self.video_files = find_video_files(video_dir) + self.batch_size = batch_size + self.max_frames = max_frames + print(f"batch_size: {batch_size}, max_frames: {max_frames}") + + def __iter__(self): # (file, number of frames) + if self.batch_size == 1: + for video_file in self.video_files: + yield Video(video_file, video_frames(video_file, self.max_frames)) + else: + batch = [] + for video_file in self.video_files: + video = Video(video_file, video_frames(video_file, self.max_frames)) + batch.append(video) + if len(batch) == self.batch_size: + yield batch + batch = [] + + +class NExTQALoader(VideoLoader): + """ + Load vdideos and prompts from NExT dataset + set: train, test or validation + """ + + def __init__( + self, video_dir, batch_size=1, max_frames=sys.maxsize, dset="test", task="OE" + ): + """ + task: 'MV' or 'OE' + """ + super().__init__() + self.task = task + print(f"Loading the {dset} data of {task} from lmms-lab/NExTQA") + self.ds = load_dataset("lmms-lab/NExTQA", task) + self.ds = self.ds[dset] + + # self.n = ds.num_rows + self.video_dir = video_dir + self.video_files = find_video_files(video_dir) + self.video_to_path = dict() + for video_file in self.video_files: + video_id = video_file.split("/")[-1].split(".")[0] + self.video_to_path[video_id] = video_file + + self.batch_size = batch_size + self.max_frames = max_frames + + def get_video_prompt(self, entry, max_frames) -> VideoPrompt: + # Get video + video_id = entry["video"] + video_path = self.video_to_path[video_id] + assert os.path.exists(video_path), f"Video not found: {video_path}" + num_frames = min(entry["frame_count"], max_frames) + video = Video(video_path, num_frames) + prompt = entry["question"] + "?" + if self.task == "MC": # add choices + prompt += f' a0: {entry["a0"]}, a1: {entry["a1"]}, a2: {entry["a2"]}, a3: {entry["a3"]}' + return VideoPrompt(video_path, num_frames, prompt) + + def __iter__(self): + if self.batch_size == 1: + for entry in self.ds: + yield self.get_video_prompt(entry, self.max_frames) + else: + batch = [] + for entry in self.ds: + video = self.get_video_prompt(entry, self.max_frames) + batch.append(video) + if len(batch) == self.batch_size: + yield batch + batch = [] + + +# main +if __name__ == "__main__": + video_dir = "./videos" + # video_loader = VideoFileLoader(video_dir, batch_size=16) + # for batch in video_loader: + # print(f"Number of videos in batch: {len(batch)}") + # for video_file, num_frames in batch: + # print(f"Video: {video_file} number of frames: {num_frames}") + + video_loader = NExTQALoader(video_dir, batch_size=16, dset="test", task="OE") + for batch in video_loader: + print(f"Number of videos in batch: {len(batch)}") + for video_file, num_frames, prompt in batch: + print( + f"Video: {video_file} number of frames: {num_frames}, prompt: {prompt}" + ) + # break + # for video_file, prompt in batch: + # print(f"Video: {video_file} prompt: {prompt}") + # break diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/perf.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/perf.py new file mode 100644 index 00000000..2349af4b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/hicache/perf.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +from typing import Any, Callable, NamedTuple + +import torch + + +def jit_hicache_impl( + k_cache_dst: torch.Tensor, + v_cache_dst: torch.Tensor, + indices_dst: torch.Tensor, + k_cache_src: torch.Tensor, + v_cache_src: torch.Tensor, + indices_src: torch.Tensor, + item_bytes: int, + block_quota: int, +) -> None: + from sglang.jit_kernel.hicache import transfer_hicache_one_layer + + _ = item_bytes + + transfer_hicache_one_layer( + k_cache_dst=k_cache_dst, + v_cache_dst=v_cache_dst, + indices_dst=indices_dst, + k_cache_src=k_cache_src, + v_cache_src=v_cache_src, + indices_src=indices_src, + block_quota=block_quota, + ) + + +def ref_hicache_impl( + k_cache_dst: torch.Tensor, + v_cache_dst: torch.Tensor, + indices_dst: torch.Tensor, + k_cache_src: torch.Tensor, + v_cache_src: torch.Tensor, + indices_src: torch.Tensor, + item_bytes: int, + block_quota: int, +) -> None: + from sgl_kernel import transfer_kv_per_layer + + transfer_kv_per_layer( + src_k=k_cache_src, + src_v=v_cache_src, + dst_k=k_cache_dst, + dst_v=v_cache_dst, + src_indices=indices_src, + dst_indices=indices_dst, + item_size=item_bytes, + block_quota=block_quota, + ) + + +class HicacheBenchArgs(NamedTuple): + cache_item_size: int + dtype: torch.dtype + block_quota: int + + +def perf(f: Callable[[], Any], loop: int = 100) -> float: + tic = torch.cuda.Event(enable_timing=True) + toc = torch.cuda.Event(enable_timing=True) + torch.cuda.synchronize() + # warm up + f() + torch.cuda._sleep(10**8) + tic.record() + for _ in range(loop): + f() + toc.record() + toc.synchronize() + return tic.elapsed_time(toc) / loop + + +@torch.inference_mode() +def test_hicache_kernel(args: HicacheBenchArgs) -> None: + CACHE_ITEM_SIZE, DTYPE, BLOCK_QUOTA = args + + CUDA_CACHE_SIZE = 1024 * 1024 + HOST_CACHE_SIZE = CUDA_CACHE_SIZE * 2 + + cuda_cache = torch.randn( + (2, CUDA_CACHE_SIZE, CACHE_ITEM_SIZE), + dtype=DTYPE, + device="cuda", + ) + host_cache = torch.empty( + (2, HOST_CACHE_SIZE, CACHE_ITEM_SIZE), + dtype=DTYPE, + device="cpu", + pin_memory=True, + ) + + ITEM_BYTES = cuda_cache.element_size() * CACHE_ITEM_SIZE + + def _gen_indices(size: int, bs: int) -> torch.Tensor: + assert bs <= size + result = ( + (torch.randperm(size, dtype=torch.int64, device="cuda")[:bs]).sort().values + ) + if not (torch.all(result >= 0) and torch.all(result < size)): + where = (result < 0) | (result >= size) + place = where.nonzero(as_tuple=False) + print("Invalid indices at positions:", place) + print("Invalid indices values:", result[place]) + raise ValueError("Generated invalid indices") + return result + + def _calc_tput(dur: float) -> float: + return (MEM / (1024**3)) / (dur / 1000) # GB/s + + def _gain_str(aot_dur: float, jit_dur: float) -> str: + gain = 100 * (aot_dur / jit_dur - 1) + if gain >= 0: + return f"+{gain:>6.2f}%" + else: + return f"-{-gain:>6.2f}%" + + print(f"{CACHE_ITEM_SIZE = }, {DTYPE = }, {BLOCK_QUOTA = }") + + def _fast_test_correctness(bs: int): + src_indices = _gen_indices(CUDA_CACHE_SIZE, bs) + dst_indices = _gen_indices(HOST_CACHE_SIZE, bs) + host_cache_cuda = torch.randn_like(host_cache, device="cuda") + host_cache.copy_(host_cache_cuda, non_blocking=True) + + # copy from cuda to host + jit_hicache_impl( + k_cache_dst=host_cache[0], + v_cache_dst=host_cache[1], + indices_dst=dst_indices, + k_cache_src=cuda_cache[0], + v_cache_src=cuda_cache[1], + indices_src=src_indices, + item_bytes=ITEM_BYTES, + block_quota=BLOCK_QUOTA, + ) + dst_indices = dst_indices.cpu() + assert torch.all( + host_cache[0][dst_indices].cuda() == cuda_cache[0][src_indices] + ) + + BS_RANGE = [2**n for n in range(8, 18)] + for bs in BS_RANGE: + _fast_test_correctness(bs) + + print("Correctness passed! Start HiCache kernel performance test...") + print("=" * 70) + + for bs in BS_RANGE: + indices_dst = _gen_indices(CUDA_CACHE_SIZE, bs) + indices_src = _gen_indices(HOST_CACHE_SIZE, bs) + MEM = 2 * bs * ITEM_BYTES + + def _run_kernel_h2d(impl): + return impl( + k_cache_dst=cuda_cache[0], + v_cache_dst=cuda_cache[1], + indices_dst=indices_dst, + k_cache_src=host_cache[0], + v_cache_src=host_cache[1], + indices_src=indices_src, + item_bytes=ITEM_BYTES, + block_quota=BLOCK_QUOTA, + ) + + our_h2d_dur = perf(lambda: _run_kernel_h2d(jit_hicache_impl)) + ref_h2d_dur = perf(lambda: _run_kernel_h2d(ref_hicache_impl)) + print( + f"{bs = :6d}, H->D", + f"| aot {_calc_tput(ref_h2d_dur):<6.2f} GB/s", + f"| jit {_calc_tput(our_h2d_dur):<6.2f} GB/s", + f"| {_gain_str(ref_h2d_dur, our_h2d_dur)}", + ) + + print("=" * 70) + + for bs in BS_RANGE: + indices_dst = _gen_indices(HOST_CACHE_SIZE, bs) + indices_src = _gen_indices(CUDA_CACHE_SIZE, bs) + MEM = 2 * bs * ITEM_BYTES + + def _run_kernel_d2h(impl): + return impl( + k_cache_dst=host_cache[0], + v_cache_dst=host_cache[1], + indices_dst=indices_dst, + k_cache_src=cuda_cache[0], + v_cache_src=cuda_cache[1], + indices_src=indices_src, + item_bytes=ITEM_BYTES, + block_quota=BLOCK_QUOTA, + ) + + our_d2h_dur = perf(lambda: _run_kernel_d2h(jit_hicache_impl)) + ref_d2h_dur = perf(lambda: _run_kernel_d2h(ref_hicache_impl)) + print( + f"{bs = :6d}, D->H", + f"| aot {_calc_tput(ref_d2h_dur):<6.2f} GB/s", + f"| jit {_calc_tput(our_d2h_dur):<6.2f} GB/s", + f"| {_gain_str(ref_d2h_dur, our_d2h_dur)}", + ) + + print("=" * 70) + + +def main() -> None: + torch.cuda.set_device(0) + stream = torch.cuda.Stream() + torch.cuda.set_stream(stream) + + tic = torch.cuda.Event(enable_timing=True) + toc = torch.cuda.Event(enable_timing=True) + + BUF_SIZE = 1024 * 1024 * 1024 + cuda_mem = torch.empty(BUF_SIZE, dtype=torch.uint8, device="cuda") + host_mem = torch.empty(BUF_SIZE, dtype=torch.uint8, device="cpu", pin_memory=True) + + # test peak bandwidth + tic.record() + cuda_mem.copy_(host_mem, non_blocking=True) + toc.record() + toc.synchronize() + dur = tic.elapsed_time(toc) + print(f"Peak H->D Bandwidth: {(BUF_SIZE / (1024**3)) / (dur / 1000):.2f} GB/s") + + tic.record() + host_mem.copy_(cuda_mem, non_blocking=True) + toc.record() + toc.synchronize() + dur = tic.elapsed_time(toc) + print(f"Peak D->H Bandwidth: {(BUF_SIZE / (1024**3)) / (dur / 1000):.2f} GB/s") + + for block_quota in [1, 2, 3, 4]: + for cache_item_size in [128, 256, 512, 1024]: + args = HicacheBenchArgs( + cache_item_size=cache_item_size, + dtype=torch.float16, + block_quota=block_quota, + ) + test_hicache_kernel(args) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_decode_regex/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_decode_regex/README.md new file mode 100644 index 00000000..26acabae --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_decode_regex/README.md @@ -0,0 +1,60 @@ +## Run benchmark + +### Build dataset +``` +pip install wikipedia +python3 build_dataset.py +``` + +### Dependencies + +``` +llama_cpp_python 0.2.19 +guidance 0.1.10 +vllm 0.2.5 +outlines 0.0.22 +``` + +### Benchmark sglang + +Run Llama-7B + +``` +python3 -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +Run Mixtral-8x7B + +``` +python3 -m sglang.launch_server --model-path mistralai/Mixtral-8x7B-Instruct-v0.1 --port 30000 --tp-size 8 +``` + +Benchmark + +``` +python3 bench_sglang.py --num-questions 10 +``` + + +### Benchmark Outlines + vLLM + +Run Llama-7B + +``` +python3 -m outlines.serve.serve --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +Benchmark + +``` +python3 bench_other.py --backend outlines --num-questions 10 +``` + + +### Benchmark guidance + +Run Llama-7B and benchmark + +``` +python3 bench_other.py --backend guidance --num-questions 10 --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_decode_regex/bench_other.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_decode_regex/bench_other.py new file mode 100644 index 00000000..87051ea8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_decode_regex/bench_other.py @@ -0,0 +1,98 @@ +import argparse +import json +import time +from concurrent.futures import ThreadPoolExecutor +from functools import partial + +from tqdm import tqdm + +from sglang.lang.ir import REGEX_FLOAT, REGEX_INT, REGEX_STR +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +REGEX_LIST = r"\[(" + REGEX_STR + ", )*" + REGEX_STR + r"\]" + + +# fmt: off +def json_decode(document, generate): + s = "Please extract the information of a city from the following wikipedia page.\n" + s += "Page begin.\n" + document + "Page end.\n" + s += "Here is the name, country, and symbol of the city in JSON format.\n" + s += "{\n" + s += ' "name": ' + s += generate(s, max_tokens=8, regex=REGEX_STR + ",") + "\n" + s += ' "country": ' + s += generate(s, max_tokens=8, regex=REGEX_STR + ",") + "\n" + s += ' "latitude": ' + s += generate(s, max_tokens=8, regex=REGEX_FLOAT + ",") + "\n" + s += ' "population": ' + s += generate(s, max_tokens=8, regex=REGEX_INT + ",") + "\n" + s += ' "top 3 landmarks": ' + s += generate(s, max_tokens=24, regex=REGEX_LIST) + "\n" + s += "}\n" + + return s +# fmt: on + + +def main(args): + lines = read_jsonl(args.data_path) + arguments = [] + for i in range(len(lines[: args.num_questions])): + arguments.append( + { + "document": lines[i]["document"], + } + ) + states = [None] * len(arguments) + + # Select backend + call_generate = partial(get_call_generate(args), temperature=0) + + # Run requests + def get_one_answer(i): + states[i] = json_decode(generate=call_generate, **arguments[i]) + + tic = time.perf_counter() + if args.parallel == 1: + for i in tqdm(range(len(arguments))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + rets = list( + tqdm( + executor.map(get_one_answer, list(range(len(arguments)))), + total=len(arguments), + ) + ) + for _ in rets: + pass + + latency = time.perf_counter() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "json_decode_regex", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="questions.jsonl") + parser.add_argument("--num-questions", type=int, default=20) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_decode_regex/bench_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_decode_regex/bench_sglang.py new file mode 100644 index 00000000..9aab11e4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_decode_regex/bench_sglang.py @@ -0,0 +1,101 @@ +import argparse +import json +import time + +import sglang as sgl +from sglang.lang.ir import REGEX_FLOAT, REGEX_INT, REGEX_STR +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + +REGEX_LIST = r"\[(" + REGEX_STR + ", )*" + REGEX_STR + r"\]" + +# fmt: off +@sgl.function +def json_warm_up(s): + s += "The information about Hogwarts is in the following JSON format.\n" + with s.var_scope("json_output"): + s += "{\n" + s += ' "name": ' + sgl.gen("name", max_tokens=8, regex=REGEX_STR + ",") + "\n" + s += ' "country": ' + sgl.gen("country", max_tokens=8, regex=REGEX_STR + ",") + "\n" + s += ' "latitude": ' + sgl.gen("latitude", max_tokens=8, regex=REGEX_FLOAT + ",") + "\n" + s += ' "population": ' + sgl.gen("population", max_tokens=8, regex=REGEX_INT + ",") + "\n" + s += ' "top 3 landmarks": ' + sgl.gen( "landmarks", max_tokens=24, regex=REGEX_LIST) + "\n" + s += "}\n" + print(f'The warmp up json result is:\n{s["json_output"]}') +# fmt: on + +# fmt: off +@sgl.function +def json_decode(s, document): + s += "Please extract the information of a city from the following wikipedia page.\n" + s += "Page begin.\n" + document + "Page end.\n" + s += "Here is the name, country, and symbol of the city in JSON format.\n" + with s.var_scope("json_output"): + s += "{\n" + s += ' "name": ' + sgl.gen("name", max_tokens=8, regex=REGEX_STR + ",") + "\n" + s += ' "country": ' + sgl.gen("country", max_tokens=8, regex=REGEX_STR + ",") + "\n" + s += ' "latitude": ' + sgl.gen("latitude", max_tokens=8, regex=REGEX_FLOAT + ",") + "\n" + s += ' "population": ' + sgl.gen("population", max_tokens=8, regex=REGEX_INT + ",") + "\n" + s += ' "top 3 landmarks": ' + sgl.gen( "landmarks", max_tokens=24, regex=REGEX_LIST) + "\n" + s += "}\n" +# fmt: on + + +def main(args): + lines = read_jsonl(args.data_path) + lines = list(lines) + arguments = [] + for i in range(len(lines[: args.num_questions])): + arguments.append( + { + "document": lines[i]["document"], + } + ) + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Warm up + json_warm_up.run().sync() + + # Run requests + tic = time.perf_counter() + states = json_decode.run_batch( + arguments, temperature=0, num_threads=args.parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(f"tmp_{args.backend}_json_results.txt", "w") as fout: + for state in states: + fout.write(state["json_output"] + "\n") + + with open(args.result_file, "a") as fout: + value = { + "task": "json_decode_regex", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="questions.jsonl") + parser.add_argument("--num-questions", type=int, default=20) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_decode_regex/build_dataset.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_decode_regex/build_dataset.py new file mode 100644 index 00000000..1396e5ed --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_decode_regex/build_dataset.py @@ -0,0 +1,58 @@ +import json + +import transformers +import wikipedia + +model_path = "meta-llama/Llama-2-7b-chat-hf" +t = transformers.AutoTokenizer.from_pretrained(model_path) +city_names = [ + "los angles", + "london", + "tokyo", + "beijing", + "singapore", + "paris", + "dubai", + "sydney", + "moscow", + "rome", + "toronto", + "rio de janeiro", + "istanbul", + "berlin", + "auckland", + "buenos aires", + "mexico city", + "mumbai", + "seoul", + "bangkok", + "cairo", + "athens", + "jerusalem", +] + + +def get_content(city_name): + content = str(wikipedia.page(city_name).content) + content = content.replace("\n\n", "\n") + + tokens = t.encode(content) + + expected_tokens = 3000 + truncate_len = int((expected_tokens / len(tokens)) * len(content)) + truncate_content = content[:truncate_len] + truncate_tokens = t.encode(truncate_content) + + # Count token + print( + f"city_name: {city_name}, #tokens: {len(tokens)}, #truncate tokens: {len(truncate_tokens)}" + ) + + return truncate_content + + +if __name__ == "__main__": + with open("questions.jsonl", "w") as fout: + for city_name in city_names: + truncate_content = get_content(city_name) + fout.write(json.dumps({"document": truncate_content}) + "\n") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_jump_forward/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_jump_forward/README.md new file mode 100644 index 00000000..38fb67e8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_jump_forward/README.md @@ -0,0 +1,88 @@ +## Run benchmark + +### Dependencies + +``` +llama_cpp_python 0.2.38 +guidance 0.1.10 +vllm 0.2.7 +outlines 0.0.25 +``` + +### Build dataset + +When benchmarking long document information retrieval, run the following command to build the dataset: + +```bash +pip install wikipedia +python3 build_dataset.py +``` + +### Benchmark sglang + +Run Llama-7B + +```bash +python3 -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +Benchmark Character Generation + +```bash +python3 bench_sglang.py --mode character +``` + +Benchmark City Information Retrieval + +```bash +python3 bench_sglang.py --mode city +``` + + +### Benchmark Outlines + vLLM + +Run Llama-7B + +```bash +python3 -m outlines.serve.serve --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +Benchmark Character Generation + +```bash +python3 bench_other.py --mode character --backend outlines +``` + +Benchmark City Information Retrieval + +```bash +python3 bench_other.py --mode city --backend outlines +``` + +### Benchmark guidance + +Run Llama-7B and benchmark character generation + +```bash +python3 bench_other.py --mode character --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +Run Llama-7B and benchmark city information retrieval + +```bash +python3 bench_other.py --mode city --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +### Benchmark lmql + +Run Llama-7B and benchmark character generation + +``` +python3 bench_other.py --mode character --backend lmql --parallel 1 +``` + +Run Llama-7B and benchmark city information retrieval + +``` +python3 bench_other.py --mode city --backend lmql --parallel 1 +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_jump_forward/bench_other.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_jump_forward/bench_other.py new file mode 100644 index 00000000..a64e950d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_jump_forward/bench_other.py @@ -0,0 +1,288 @@ +import argparse +import json +import time +from concurrent.futures import ThreadPoolExecutor +from functools import partial + +import guidance +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +# there are some FSM bugs with json regex converted from pydantic model +# here use a string regex instead +# regex_string = build_regex_from_object(HarryPoterRole) +character_regex = ( + r"""\{\n""" + + r""" "name": "[\w\d\s]{1,16}",\n""" + + r""" "house": "(Gryffindor|Slytherin|Ravenclaw|Hufflepuff)",\n""" + + r""" "blood status": "(Pure-blood|Half-blood|Muggle-born)",\n""" + + r""" "occupation": "(student|teacher|auror|ministry of magic|death eater|order of the phoenix)",\n""" + + r""" "wand": \{\n""" + + r""" "wood": "[\w\d\s]{1,16}",\n""" + + r""" "core": "[\w\d\s]{1,16}",\n""" + + r""" "length": [0-9]{1,2}\.[0-9]{0,2}\n""" + + r""" \},\n""" + + r""" "alive": "(Alive|Deceased)",\n""" + + r""" "patronus": "[\w\d\s]{1,16}",\n""" + + r""" "bogart": "[\w\d\s]{1,16}"\n""" + + r"""\}""" +) + +city_regex = ( + r"""\{\n""" + + r""" "name": "[\w\d\s]{1,16}",\n""" + + r""" "country": "[\w\d\s]{1,16}",\n""" + + r""" "latitude": [-+]?[0-9]*\.?[0-9]{0,2},\n""" + + r""" "population": [-+]?[0-9]{1,9},\n""" + + r""" "top 3 landmarks": \["[\w\d\s]{1,16}", "[\w\d\s]{1,16}", "[\w\d\s]{1,16}"\]\n""" + + r"""\}""" +) + +# fmt: off +def character_gen(name, generate): + s = name + " is a character in Harry Potter. Please fill in the following information about this character.\n" + s += generate(s, max_tokens=256, regex=character_regex) + return s +# fmt: on + +# fmt: off +def city_gen(document, generate): + s = "Please extract the information of a city from the following wikipedia page.\n" + s += "Page begin.\n" + document + "Page end.\n" + s += "Here is the name, country, and symbol of the city in JSON format.\n" + s += generate(s, max_tokens=256, regex=city_regex) + return s +# fmt: on + + +@guidance +def character_maker(lm, name): + regex_str_no_quote = r"[\w\d\s]+" + regex_float = r"[0-9]+\.[0-9]+" + lm += f"""\ + {name} is a character in Harry Potter. Please fill in the following information about this character. + {{ + "name": "{guidance.gen("name", max_tokens=16, regex=regex_str_no_quote)}", + "house": "{guidance.select(options=['Gryffindor', 'Slytherin', 'Ravenclaw', 'Hufflepuff'], name='house')}", + "blood status": "{guidance.select(options=['Pure-blood', 'Half-blood', 'Muggle-born'], name='blood status')}", + "occupation": "{guidance.select(options=['student', 'teacher', 'auror', 'ministry of magic', 'death eater', 'order of the phoenix'], name='occupation')}", + "wand": {{ + "wood": "{guidance.gen("wood", max_tokens=16, regex=regex_str_no_quote)}", + "core": "{guidance.gen('core', max_tokens=16, regex=regex_str_no_quote)}", + "length": {guidance.gen('length', max_tokens=10, regex=regex_float)} + }}, + "alive": "{guidance.select(options=['Alive', 'Deceased'], name='alive')}", + "patronus": "{guidance.gen('patronus', max_tokens=16, regex=regex_str_no_quote)}", + "bogart": "{guidance.gen('bogart', max_tokens=16, regex=regex_str_no_quote)}" + }} + """ + + return lm + + +async def call_generate_lmql( + prompt, temperature, max_tokens, regex, max_len=4096, model=None, **kwargs +): + assert model is not None + import lmql + + @lmql.query(model=model) + async def program(question, max_tokens, regex): + '''lmql + """{question}[ANSWER]""" where len(TOKENS(ANSWER)) < max_tokens and REGEX(ANSWER, regex) + return ANSWER + ''' + + return await program( + question=prompt, + temperature=temperature, + max_tokens=max_tokens, + max_len=max_len, + regex=regex, + **kwargs, + ) + + +@guidance +def city_maker(lm, document): + regex_str_no_quote = r"[\w\d\s]+" + regex_float = r"[0-9]+\.[0-9]+" + lm += f"""\ + Please extract the information of a city from the following wikipedia page. + Page begin. + {document} + Page end. + Here is the name, country, and symbol of the city in JSON format. + {{ + "name": "{guidance.gen("name", max_tokens=16, regex=regex_str_no_quote)}", + "country": "{guidance.gen("country", max_tokens=16, regex=regex_str_no_quote)}", + "latitude": {guidance.gen("latitude", max_tokens=10, regex=regex_float)}, + "population": {guidance.gen("population", max_tokens=10, regex=r"[0-9]+")}, + "top 3 landmarks": [ + "{guidance.gen("landmark1", max_tokens=16, regex=regex_str_no_quote)}", "{guidance.gen("landmark2", max_tokens=16, regex=regex_str_no_quote)}", "{guidance.gen("landmark3", max_tokens=16, regex=regex_str_no_quote)}" + ] + }} + """ + + return lm + + +def bench_character(args): + arguments = [] + with open(args.data_path, "r") as f: + for line in f: + arguments.append({"name": line.strip()}) + arguments = arguments[: args.num_jsons] + + states = [None] * len(arguments) + + # Select backend + if args.backend == "outlines": + call_generate = partial(get_call_generate(args), temperature=0) + + def get_one_answer(i): + states[i] = character_gen(**arguments[i], generate=call_generate) + + elif args.backend == "guidance": + model = guidance.models.LlamaCpp( + args.model_path, + n_gpu_layers=-1, + n_ctx=args.n_ctx, + ) + + def get_one_answer(i): + lm = model + character_maker(**arguments[i]) + states[i] = lm + + elif args.backend == "lmql": + import asyncio + + import lmql + + model = lmql.model(args.model_path, endpoint=f"{args.host}:{args.port}") + call_generate = partial( + call_generate_lmql, + model=model, + max_tokens=256, + regex=character_regex, + ) + + async def get_one_answer_async(i): + states[i] = await call_generate(prompt=arguments[i]["name"], temperature=0) + + else: + raise ValueError(f"Invalid backend: {args.backend}") + + tic = time.perf_counter() + + if args.backend != "lmql": + if args.parallel == 1: + for i in tqdm(range(len(arguments))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + rets = list( + tqdm( + executor.map(get_one_answer, list(range(len(arguments)))), + total=len(arguments), + ) + ) + for _ in rets: + pass + else: + batches = [] + for i in range(0, len(arguments), args.parallel): + batches.append(list(range(i, min(i + args.parallel, len(arguments))))) + loop = asyncio.get_event_loop() + + for bt in tqdm(batches): + loop.run_until_complete( + asyncio.gather(*[get_one_answer_async(i) for i in bt]) + ) + + latency = time.perf_counter() - tic + + return states, latency + + +def bench_city_doc(args): + arguments = [] + for line in read_jsonl(args.data_path): + arguments.append({"document": line["document"]}) + arguments = arguments[: args.num_jsons] + + states = [None] * len(arguments) + + # Select backend + if args.backend == "outlines": + call_generate = partial(get_call_generate(args), temperature=0) + + def get_one_answer(i): + states[i] = city_gen(**arguments[i], generate=call_generate) + + elif args.backend == "guidance": + model = guidance.models.LlamaCpp( + args.model_path, + n_gpu_layers=-1, + n_ctx=args.n_ctx, + ) + + def get_one_answer(i): + lm = model + city_maker(**arguments[i]) + states[i] = lm + + else: + raise ValueError(f"Invalid backend: {args.backend}") + + tic = time.perf_counter() + if args.parallel == 1: + for i in tqdm(range(len(arguments))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + rets = executor.map(get_one_answer, list(range(len(arguments)))) + for _ in rets: + pass + + latency = time.perf_counter() - tic + + return states, latency + + +def main(args): + if args.mode == "character": + args.data_path = "dataset.txt" + states, latency = bench_character(args) + elif args.mode == "city": + args.data_path = "questions.jsonl" + states, latency = bench_city_doc(args) + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}_{args.mode}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "json_jump_forward", + "backend": args.backend, + "latency": round(latency, 3), + "num_jsons": args.num_jsons, + "mode": args.mode, + "parallel": args.parallel, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str) + parser.add_argument("--num-jsons", type=int, default=50) + parser.add_argument( + "--mode", type=str, default="character", choices=["character", "city"] + ) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_jump_forward/bench_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_jump_forward/bench_sglang.py new file mode 100644 index 00000000..29f635f7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_jump_forward/bench_sglang.py @@ -0,0 +1,143 @@ +import argparse +import json +import time + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + +# there are some FSM bugs with json regex converted from pydantic model +# here use a string regex instead +# regex_string = build_regex_from_object(HarryPoterRole) +character_regex = ( + r"""\{\n""" + + r""" "name": "[\w\d\s]{1,16}",\n""" + + r""" "house": "(Gryffindor|Slytherin|Ravenclaw|Hufflepuff)",\n""" + + r""" "blood status": "(Pure-blood|Half-blood|Muggle-born)",\n""" + + r""" "occupation": "(student|teacher|auror|ministry of magic|death eater|order of the phoenix)",\n""" + + r""" "wand": \{\n""" + + r""" "wood": "[\w\d\s]{1,16}",\n""" + + r""" "core": "[\w\d\s]{1,16}",\n""" + + r""" "length": [0-9]{1,2}\.[0-9]{0,2}\n""" + + r""" \},\n""" + + r""" "alive": "(Alive|Deceased)",\n""" + + r""" "patronus": "[\w\d\s]{1,16}",\n""" + + r""" "bogart": "[\w\d\s]{1,16}"\n""" + + r"""\}""" +) + +city_regex = ( + r"""\{\n""" + + r""" "name": "[\w\d\s]{1,16}",\n""" + + r""" "country": "[\w\d\s]{1,16}",\n""" + + r""" "latitude": [-+]?[0-9]*\.?[0-9]{0,2},\n""" + + r""" "population": [-+]?[0-9]{1,9},\n""" + + r""" "top 3 landmarks": \["[\w\d\s]{1,16}", "[\w\d\s]{1,16}", "[\w\d\s]{1,16}"\]\n""" + + r"""\}""" +) + +# fmt: off +@sgl.function +def character_gen(s, name): + s += name + " is a character in Harry Potter. Please fill in the following information about this character.\n" + s += sgl.gen("json_output", max_tokens=256, regex=character_regex) +# fmt: on + +# fmt: off +@sgl.function +def city_gen(s, document): + s += "Please extract the information of a city from the following wikipedia page.\n" + s += "Page begin.\n" + document + "Page end.\n" + s += "Here is the name, country, and symbol of the city in JSON format.\n" + s += sgl.gen("json_output",max_tokens=256, regex=city_regex) +# fmt: on + + +def bench_city_doc(args): + arguments = [] + for line in read_jsonl(args.data_path): + arguments.append({"document": line["document"]}) + arguments = arguments[: args.num_jsons] + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.perf_counter() + states = city_gen.run_batch( + arguments, + temperature=0, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + + return states, latency + + +def bench_character(args): + arguments = [] + with open(args.data_path, "r") as f: + for line in f: + arguments.append({"name": line.strip()}) + arguments = arguments[: args.num_jsons] + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.perf_counter() + states = character_gen.run_batch( + arguments, + temperature=0, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + + return states, latency + + +def main(args): + if args.mode == "character": + args.data_path = "dataset.txt" + states, latency = bench_character(args) + elif args.mode == "city": + args.data_path = "questions.jsonl" + states, latency = bench_city_doc(args) + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}_{args.mode}.txt", states) + with open(f"{args.backend}_{args.mode}.json", "w") as fout: + for state in states: + fout.write(state["json_output"] + "\n") + + with open(args.result_file, "a") as fout: + value = { + "task": "json_jump_forward", + "backend": args.backend, + "latency": round(latency, 3), + "num_jsons": args.num_jsons, + "mode": args.mode, + "parallel": args.parallel, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str) + parser.add_argument("--num-jsons", type=int, default=50) + parser.add_argument( + "--mode", type=str, default="character", choices=["character", "city"] + ) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_jump_forward/build_dataset.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_jump_forward/build_dataset.py new file mode 100644 index 00000000..1396e5ed --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_jump_forward/build_dataset.py @@ -0,0 +1,58 @@ +import json + +import transformers +import wikipedia + +model_path = "meta-llama/Llama-2-7b-chat-hf" +t = transformers.AutoTokenizer.from_pretrained(model_path) +city_names = [ + "los angles", + "london", + "tokyo", + "beijing", + "singapore", + "paris", + "dubai", + "sydney", + "moscow", + "rome", + "toronto", + "rio de janeiro", + "istanbul", + "berlin", + "auckland", + "buenos aires", + "mexico city", + "mumbai", + "seoul", + "bangkok", + "cairo", + "athens", + "jerusalem", +] + + +def get_content(city_name): + content = str(wikipedia.page(city_name).content) + content = content.replace("\n\n", "\n") + + tokens = t.encode(content) + + expected_tokens = 3000 + truncate_len = int((expected_tokens / len(tokens)) * len(content)) + truncate_content = content[:truncate_len] + truncate_tokens = t.encode(truncate_content) + + # Count token + print( + f"city_name: {city_name}, #tokens: {len(tokens)}, #truncate tokens: {len(truncate_tokens)}" + ) + + return truncate_content + + +if __name__ == "__main__": + with open("questions.jsonl", "w") as fout: + for city_name in city_names: + truncate_content = get_content(city_name) + fout.write(json.dumps({"document": truncate_content}) + "\n") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_jump_forward/dataset.txt b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_jump_forward/dataset.txt new file mode 100644 index 00000000..c12421e5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_jump_forward/dataset.txt @@ -0,0 +1,50 @@ +Harry Potter +Hermione Granger +Ron Weasley +Albus Dumbledore +Severus Snape +Rubeus Hagrid +Draco Malfoy +Ginny Weasley +Fred Weasley +George Weasley +Percy Weasley +Sirius Black +Remus Lupin +Neville Longbottom +Luna Lovegood +Cedric Diggory +Cho Chang +Lord Voldemort +Minerva McGonagall +Filius Flitwick +Dolores Umbridge +Bellatrix Lestrange +Lucius Malfoy +Molly Weasley +Arthur Weasley +Nymphadora Tonks +Dobby +Moaning Myrtle +Peter Pettigrew +Alastor 'Mad-Eye' Moody +Horace Slughorn +Vernon Dursley +Petunia Dursley +Dudley Dursley +Argus Filch +Sybill Trelawney +Gilderoy Lockhart +Fleur Delacour +Viktor Krum +Bill Weasley +Oliver Wood +Cornelius Fudge +Barty Crouch Sr. +Barty Crouch Jr. +Kingsley Shacklebolt +Quirinus Quirrell +Nearly Headless Nick +Aunt Marge +Griphook +Ludo Bagman diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_schema/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_schema/README.md new file mode 100644 index 00000000..e6790e1f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_schema/README.md @@ -0,0 +1,15 @@ +## Run benchmark + +### Benchmark sglang + +Run Llama-8b + +```bash +python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --port 30000 +``` + +Benchmark + +```bash +python3 bench_sglang.py +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_schema/bench_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_schema/bench_sglang.py new file mode 100644 index 00000000..8de68df3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/json_schema/bench_sglang.py @@ -0,0 +1,146 @@ +import argparse +import json +import time +from typing import List, Tuple + +import jsonschema +from datasets import load_dataset + +import sglang as sgl +from sglang.global_config import global_config +from sglang.srt.utils.hf_transformers_utils import get_tokenizer +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text + + +@sgl.function +def schema_gen(s, message: Tuple[str, str], json_schema: str): + system, user = message + s += sgl.system(system) + s += sgl.user(user) + s += sgl.assistant( + sgl.gen("json_output", temperature=0, max_tokens=256, json_schema=json_schema) + ) + + +def contains_formats(schema, formats: List[str]): + if isinstance(schema, dict): + if schema.get("format", None) in formats: + return True + for value in schema.values(): + if contains_formats(value, formats): + return True + elif isinstance(schema, list): + for item in schema: + if contains_formats(item, formats): + return True + return False + + +def convert_dataset(path: str): + raw_dataset = load_dataset(path) + dataset = [] + for data in raw_dataset["train"]: + messages = data["prompt"] + schema = data["schema"] + obj = json.loads(schema) + + # skip some corrupted examples + if obj.get("type", None) is None: + continue + + # skip schema with format "email" + # which is not supported by outlines for now + if contains_formats(obj, ["email"]): + continue + + system = messages[0] + user = messages[1] + assert system["role"] == "system", "invalid role" + assert user["role"] == "user", "invalid role" + assert len(messages) == 2, "invalid message length" + message = json.dumps(system["content"]), json.dumps(user["content"]) + dataset.append( + { + "message": message, + "json_schema": schema, + } + ) + + return dataset + + +def bench_schema(args): + arguments = convert_dataset(args.data_path) + + if args.num_jsons < 0 or args.num_jsons > len(arguments): + args.num_jsons = len(arguments) + arguments = arguments[: args.num_jsons] + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.perf_counter() + states = schema_gen.run_batch( + arguments, + temperature=0, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + + # Check if the outputs are valid + indexes = [] + for i, state in enumerate(states): + try: + schema = json.loads(arguments[i]["json_schema"]) + obj = json.loads(state["json_output"]) + assert jsonschema.validate(obj, schema) is None + except Exception as e: + print(e) + indexes.append(i) + + return states, latency + + +def main(args): + states, latency = bench_schema(args) + + # Compute accuracy + tokenizer = get_tokenizer( + global_config.default_backend.get_server_info()["tokenizer_path"] + ) + output_jsons = [state["json_output"] for state in states] + num_output_tokens = sum(len(tokenizer.encode(x)) for x in output_jsons) + print(f"Latency: {latency:.3f}") + print(f"Output throughput: {num_output_tokens / latency:.3f} token/s") + print(f"#output tokens: {num_output_tokens}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + with open(f"{args.backend}.jsonl", "w") as fout: + for state in states: + fout.write(state["json_output"] + "\n") + + with open(args.result_file, "a") as fout: + value = { + "task": "json_schema", + "backend": args.backend, + "latency": round(latency, 3), + "num_jsons": args.num_jsons, + "parallel": args.parallel, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="NousResearch/json-mode-eval") + parser.add_argument("--num-jsons", type=int, default=-1) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/all_reduce/benchmark_aiter.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/all_reduce/benchmark_aiter.py new file mode 100644 index 00000000..bca45620 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/all_reduce/benchmark_aiter.py @@ -0,0 +1,330 @@ +""" +Benchmark SGLang vs Aiter custom all-reduce across message sizes. +Usage: + torchrun --nproc_per_node=2 benchmark_aiter.py + torchrun --nproc_per_node=4 benchmark_aiter.py + torchrun --nproc_per_node=8 benchmark_aiter.py +""" + +import argparse +import os +import sys +import time +from typing import List, Optional, Tuple + +import torch +import torch.distributed as dist + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Benchmark SGLang vs Aiter custom all-reduce across message sizes." + ) + parser.add_argument( + "--backend", + type=str, + default="gloo", + help="Process group backend for the custom-AR control path (must NOT be nccl).", + ) + parser.add_argument( + "--warmup", + type=int, + default=5, + help="Warmup iterations per size per implementation.", + ) + parser.add_argument( + "--iters-small", + type=int, + default=50, + help="Benchmark iterations for sizes <= 1MB.", + ) + parser.add_argument( + "--iters-large", + type=int, + default=20, + help="Benchmark iterations for sizes > 1MB.", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Print per-iteration timings on rank 0 for debugging.", + ) + return parser.parse_args() + + +def get_env_rank_world() -> Tuple[int, int, int]: + rank = int(os.environ.get("RANK", "0")) + world_size = int(os.environ.get("WORLD_SIZE", "1")) + local_rank = int(os.environ.get("LOCAL_RANK", str(rank))) + return rank, world_size, local_rank + + +def init_dist(backend: str): + rank, world_size, _ = get_env_rank_world() + if not dist.is_initialized(): + dist.init_process_group( + backend=backend, + init_method="env://", + rank=rank, + world_size=world_size, + ) + + +def get_device(local_rank: int) -> torch.device: + torch.cuda.set_device(local_rank) + return torch.device(f"cuda:{local_rank}") + + +def human_size(num_bytes: int) -> str: + units = [("B", 1), ("K", 1024), ("M", 1024 * 1024), ("G", 1024 * 1024 * 1024)] + for suf, base in reversed(units): + if num_bytes % base == 0 and num_bytes >= base: + val = num_bytes // base + return f"{val}{suf}" + return f"{num_bytes}B" + + +def get_message_sizes() -> List[int]: + return [ + 32 * 1024, + 64 * 1024, + 128 * 1024, + 256 * 1024, + 512 * 1024, + 1 * 1024 * 1024, + 2 * 1024 * 1024, + 4 * 1024 * 1024, + 8 * 1024 * 1024, + 16 * 1024 * 1024, + 32 * 1024 * 1024, + 64 * 1024 * 1024, + ] + + +@torch.inference_mode() +def run_once(comm, inp: torch.Tensor) -> Optional[torch.Tensor]: + if hasattr(comm, "all_reduce_unreg"): + return comm.all_reduce_unreg(inp) + if hasattr(comm, "custom_all_reduce"): + return comm.custom_all_reduce(inp) + raise RuntimeError("No known all-reduce method found on the communicator.") + + +@torch.inference_mode() +def bench_impl( + name: str, + comm, + sizes: List[int], + device: torch.device, + warmup: int, + iters_small: int, + iters_large: int, + verbose: bool, + pg: Optional[dist.ProcessGroup] = None, +) -> List[Tuple[int, Optional[float]]]: + rank = dist.get_rank() + world_size = dist.get_world_size() + results: List[Tuple[int, Optional[float]]] = [] + + for size_bytes in sizes: + elems = size_bytes // 2 # float16: 2 bytes per element + inp = torch.empty(elems, dtype=torch.float16, device=device) + inp.uniform_(0, 1) + + disabled = False + dist.barrier(group=pg) + for _ in range(warmup): + torch.cuda.synchronize() + out = run_once(comm, inp) + torch.cuda.synchronize() + if out is None: + disabled = True + break + dist.barrier(group=pg) + + if disabled: + if rank == 0: + print( + f"[{name}] {human_size(size_bytes)}: custom AR disabled (skipped)" + ) + results.append((size_bytes, None)) + continue + + num_iters = iters_small if size_bytes <= (1 * 1024 * 1024) else iters_large + + times_ms: List[float] = [] + for it in range(num_iters): + dist.barrier(group=pg) + torch.cuda.synchronize() + t0 = time.perf_counter() + out = run_once(comm, inp) + torch.cuda.synchronize() + t1 = time.perf_counter() + dist.barrier(group=pg) + + if out is None: + disabled = True + break + + dt_ms = (t1 - t0) * 1000.0 + times_ms.append(dt_ms) + + if verbose and rank == 0: + print( + f"[{name}] size={human_size(size_bytes)} iter={it} time={dt_ms:.3f} ms" + ) + + if disabled or not times_ms: + if rank == 0: + print( + f"[{name}] {human_size(size_bytes)}: custom AR disabled (no timings)" + ) + results.append((size_bytes, None)) + continue + + avg_ms_local = sum(times_ms) / len(times_ms) + avg_tensor = torch.tensor([avg_ms_local], dtype=torch.float64, device=device) + gather_list = [torch.zeros_like(avg_tensor) for _ in range(world_size)] + dist.all_gather(gather_list, avg_tensor, group=pg) + if rank == 0: + avg_ms = float(torch.stack(gather_list).mean().item()) + print( + f"[{name}] {human_size(size_bytes)}: {avg_ms:.3f} ms (avg across ranks)" + ) + results.append((size_bytes, avg_ms)) + else: + results.append((size_bytes, None)) + + return results + + +def main(): + args = parse_args() + rank, world_size, local_rank = get_env_rank_world() + + if world_size not in (2, 4, 6, 8): + print( + f"[rank {rank}] WARNING: world_size={world_size} not in supported set (2,4,6,8). " + "Custom AR may disable itself.", + file=sys.stderr, + ) + + init_dist(args.backend) + device = get_device(local_rank) + + # Import after dist init; some libs query torch dist state on import + sgl_comm = None + aiter_comm = None + HAVE_SGLANG = False + HAVE_AITER = False + + try: + from sglang.srt.distributed.device_communicators.custom_all_reduce import ( + CustomAllreduce as SGLCustomAllreduce, + ) + + HAVE_SGLANG = True + except Exception as e: + if rank == 0: + print(f"SGLang CustomAllreduce import failed: {e}", file=sys.stderr) + + try: + from aiter.dist.device_communicators.custom_all_reduce import ( + CustomAllreduce as AiterCustomAllreduce, + ) + + HAVE_AITER = True + except Exception as e: + if rank == 0: + print(f"Aiter CustomAllreduce import failed: {e}", file=sys.stderr) + + if rank == 0: + print(f"Initialized PG backend={args.backend} world_size={world_size}") + print(f"Device: {device.type}:{device.index}") + print(f"SGLang available: {HAVE_SGLANG}, Aiter available: {HAVE_AITER}") + + pg = dist.group.WORLD + sizes = get_message_sizes() + max_size = max(sizes) if sizes else (64 * 1024 * 1024) + + if HAVE_SGLANG: + try: + sgl_comm = SGLCustomAllreduce(group=pg, device=device, max_size=max_size) + except Exception as e: + if rank == 0: + print( + f"Failed to construct SGLang CustomAllreduce: {e}", file=sys.stderr + ) + sgl_comm = None + + if HAVE_AITER: + try: + aiter_comm = AiterCustomAllreduce( + group=pg, device=device, max_size=max_size + ) + except Exception as e: + if rank == 0: + print( + f"Failed to construct Aiter CustomAllreduce: {e}", file=sys.stderr + ) + aiter_comm = None + + sgl_results: List[Tuple[int, Optional[float]]] = [] + aiter_results: List[Tuple[int, Optional[float]]] = [] + + if sgl_comm is not None: + sgl_results = bench_impl( + name="SGLang", + comm=sgl_comm, + sizes=sizes, + device=device, + warmup=args.warmup, + iters_small=args.iters_small, + iters_large=args.iters_large, + verbose=args.verbose, + pg=pg, + ) + + if aiter_comm is not None: + aiter_results = bench_impl( + name="Aiter", + comm=aiter_comm, + sizes=sizes, + device=device, + warmup=args.warmup, + iters_small=args.iters_small, + iters_large=args.iters_large, + verbose=args.verbose, + pg=pg, + ) + + for comm in (sgl_comm, aiter_comm): + if comm is not None and hasattr(comm, "close"): + try: + comm.close() + except Exception: + pass + + if dist.get_rank() == 0: + print("\nResults (avg ms across ranks; None = disabled/unavailable):") + header = f"{'Size':>8} {'SGLang(ms)':>12} {'Aiter(ms)':>11}" + print(header) + print("-" * len(header)) + + sgl_map = {s: v for s, v in sgl_results if v is not None} + aiter_map = {s: v for s, v in aiter_results if v is not None} + + for s in sizes: + sgl_ms = sgl_map.get(s, None) + aiter_ms = aiter_map.get(s, None) + print( + f"{human_size(s):>8} {('%.3f' % sgl_ms) if sgl_ms is not None else 'None':>12} " + f"{('%.3f' % aiter_ms) if aiter_ms is not None else 'None':>11}" + ) + + dist.barrier() + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/all_reduce/benchmark_mscclpp.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/all_reduce/benchmark_mscclpp.py new file mode 100644 index 00000000..eebbd00c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/all_reduce/benchmark_mscclpp.py @@ -0,0 +1,224 @@ +"""For Now, MSCCL is only supported on TP16 and TP8 case + +export WORLD_SIZE=1 +export RANK=0 +export MASTER_ADDR=127.0.0.1 +export MASTER_PORT=12345 + +torchrun --nproc_per_node gpu \ +--nnodes $WORLD_SIZE \ +--node_rank $RANK \ +--master_addr $MASTER_ADDR \ +--master_port $MASTER_PORT benchmark/kernels/all_reduce/benchmark_mscclpp.py +""" + +import os +from contextlib import nullcontext +from typing import List + +import torch +import torch.distributed as dist +from torch.distributed import ProcessGroup + +from sglang.srt.distributed import init_distributed_environment +from sglang.srt.distributed.device_communicators.pymscclpp import PyMscclppCommunicator +from sglang.srt.distributed.device_communicators.pynccl import PyNcclCommunicator +from sglang.srt.distributed.parallel_state import ( + get_tensor_model_parallel_group, + graph_capture, + initialize_model_parallel, + set_mscclpp_all_reduce, +) + + +def torch_allreduce(torch_input: torch.Tensor, group: ProcessGroup) -> torch.Tensor: + dist.all_reduce(torch_input, group=group) + return torch_input + + +def msccl_allreduce( + msccl_input: torch.Tensor, msccl_comm: PyMscclppCommunicator +) -> torch.Tensor: + return msccl_comm.all_reduce(msccl_input) + + +def pynccl_allreduce( + msccl_input: torch.Tensor, pynccl_comm: PyNcclCommunicator +) -> torch.Tensor: + pynccl_comm.all_reduce(msccl_input) + return msccl_input + + +def _bench_graph_time(func, inp_randn, warmup_loop=2, graph_loop=10, test_loop=10): + graph_input = inp_randn.clone() + with graph_capture() as graph_capture_context: + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=graph_capture_context.stream): + for _ in range(graph_loop): + graph_out = func(graph_input) + + graph.replay() + func_output = graph_out.clone() + + for _ in range(warmup_loop): + graph.replay() + torch.cuda.synchronize() + + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + + latencies: List[float] = [] + for _ in range(test_loop): + torch.cuda.synchronize() + dist.barrier() + start_event.record() + graph.replay() + end_event.record() + end_event.synchronize() + latencies.append(start_event.elapsed_time(end_event)) + func_cost_us = sum(latencies) / len(latencies) / graph_loop * 1000 + graph.reset() + return func_output, func_cost_us + + +def _bench_eager_time(func, inp_randn, warmup_loop=2, test_loop=10): + eager_input = inp_randn.clone() + eager_output = func(eager_input) + func_output = eager_output.clone() + + for _ in range(warmup_loop): + func(eager_input) + torch.cuda.synchronize() + + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + torch.cuda.synchronize() + start_event.record() + for _ in range(test_loop): + func(eager_input) + end_event.record() + torch.cuda.synchronize() + func_cost_us = start_event.elapsed_time(end_event) / test_loop * 1000 + + return func_output, func_cost_us + + +def get_torch_prof_ctx(do_prof: bool): + ctx = ( + torch.profiler.profile( + activities=[ + torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ], + record_shapes=True, + with_stack=True, + ) + if do_prof + else nullcontext() + ) + return ctx + + +def human_readable_size(size, decimal_places=1): + for unit in ["B", "KiB", "MiB", "GiB", "TiB", "PiB"]: + if size < 1024.0 or unit == "PiB": + break + size /= 1024.0 + return f"{size:.{decimal_places}f} {unit}" + + +try: + from tabulate import tabulate +except ImportError: + print("tabulate not installed, skipping table printing") + tabulate = None + + +def print_markdown_table(data): + if tabulate is not None: + print(tabulate(data, headers="keys", tablefmt="github")) + return + headers = data[0].keys() + header_row = "| " + " | ".join(headers) + " |" + separator = "| " + " | ".join(["---"] * len(headers)) + " |" + rows = [] + for item in data: + row = "| " + " | ".join(str(item[key]) for key in headers) + " |" + rows.append(row) + markdown_table = "\n".join([header_row, separator] + rows) + print(markdown_table) + + +if __name__ == "__main__": + import logging + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + force=True, + ) + if not dist.is_initialized(): + dist.init_process_group(backend="nccl") + world, world_size = dist.group.WORLD, dist.get_world_size() + rank = dist.get_rank() + torch.cuda.set_device(rank % 8) + device = torch.cuda.current_device() + set_mscclpp_all_reduce(True) + init_distributed_environment( + world_size=world_size, + rank=rank, + local_rank=rank % 8, + ) + initialize_model_parallel(tensor_model_parallel_size=world_size) + group = get_tensor_model_parallel_group().device_group + cpu_group = get_tensor_model_parallel_group().cpu_group + pynccl_comm = get_tensor_model_parallel_group().pynccl_comm + pymscclpp_comm = get_tensor_model_parallel_group().pymscclpp_comm + dist.barrier() + profile = False + dtype = torch.bfloat16 + ctx = get_torch_prof_ctx(profile) + result = [] + + with ctx: + for i in range(10, 20): + sz = 2**i + if sz * dtype.itemsize > 2**20: + break + inp_randn = torch.randint(1, 16, (sz,), dtype=dtype, device=device) + + memory = torch.empty_like(inp_randn) + memory_out = torch.empty_like(memory) + torch_eager_output, torch_eager_time = _bench_eager_time( + lambda inp: torch_allreduce(inp, group), inp_randn + ) + msccl_eager_output, msccl_eager_time = _bench_eager_time( + lambda inp: msccl_allreduce(inp, pymscclpp_comm), inp_randn + ) + msccl_graph_output, msccl_graph_time = _bench_graph_time( + lambda inp: msccl_allreduce(inp, pymscclpp_comm), inp_randn + ) + # since pynccl is inplace op, this return result is not correct if graph loop > 1 + _, pynccl_graph_time = _bench_graph_time( + lambda inp: pynccl_allreduce(inp, pynccl_comm), inp_randn + ) + torch.testing.assert_close(torch_eager_output, msccl_graph_output) + torch.testing.assert_close(torch_eager_output, msccl_eager_output) + result.append( + { + "msg_size": human_readable_size(inp_randn.nbytes), + "torch eager time": torch_eager_time, + "msccl eager time": msccl_eager_time, + "msccl graph time": msccl_graph_time, + "pynccl graph time": pynccl_graph_time, + } + ) + if rank == 0: + print(f"sz={sz}, dtype={dtype}: correctness check PASS!") + if rank == 0: + print_markdown_table(result) + if profile: + prof_dir = f"prof/msccl" + os.makedirs(prof_dir, exist_ok=True) + ctx.export_chrome_trace(f"{prof_dir}/trace_rank{dist.get_rank()}.json.gz") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/all_reduce/benchmark_torch_symm_mem.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/all_reduce/benchmark_torch_symm_mem.py new file mode 100644 index 00000000..030fd5bb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/all_reduce/benchmark_torch_symm_mem.py @@ -0,0 +1,251 @@ +"""For Now, TORCH_SYMM_MEM is only supported on following limited tp case + +SM90: { + 2: 64 * MiB, # 64 MB + 4: 64 * MiB, # 64 MB + 6: 128 * MiB, # 128 MB + 8: 128 * MiB, # 128 MB +}, +SM100: { + 2: 64 * MiB, # 64 MB + 4: 64 * MiB, # 64 MB + 6: 128 * MiB, # 128 MB + 8: 128 * MiB, # 128 MB +} + +export WORLD_SIZE=8 +export RANK=0 +export MASTER_ADDR=127.0.0.1 +export MASTER_PORT=12345 + +torchrun --nproc_per_node gpu \ +--nnodes $WORLD_SIZE \ +--node_rank $RANK \ +--master_addr $MASTER_ADDR \ +--master_port $MASTER_PORT ./benchmark/kernels/all_reduce/benchmark_torch_symm_mem.py +""" + +import os +from contextlib import nullcontext +from typing import List + +import torch +import torch.distributed as dist +from torch.distributed import ProcessGroup + +from sglang.srt.distributed import init_distributed_environment +from sglang.srt.distributed.device_communicators.pynccl import PyNcclCommunicator +from sglang.srt.distributed.device_communicators.torch_symm_mem import ( + TorchSymmMemCommunicator, +) +from sglang.srt.distributed.parallel_state import ( + get_tensor_model_parallel_group, + graph_capture, + initialize_model_parallel, + set_torch_symm_mem_all_reduce, +) + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + + +def torch_allreduce(torch_input: torch.Tensor, group: ProcessGroup) -> torch.Tensor: + dist.all_reduce(torch_input, group=group) + return torch_input + + +def torch_symm_mem_allreduce( + torch_symm_mem_input: torch.Tensor, torch_symm_mem_comm: TorchSymmMemCommunicator +) -> torch.Tensor: + return torch_symm_mem_comm.all_reduce(torch_symm_mem_input) + + +def pynccl_allreduce( + pynccl_input: torch.Tensor, pynccl_comm: PyNcclCommunicator +) -> torch.Tensor: + pynccl_comm.all_reduce(pynccl_input) + return pynccl_input + + +def _bench_graph_time(func, inp_randn, warmup_loop=2, graph_loop=10, test_loop=10): + graph_input = inp_randn.clone() + with graph_capture() as graph_capture_context: + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=graph_capture_context.stream): + for _ in range(graph_loop): + graph_out = func(graph_input) + + graph.replay() + func_output = graph_out.clone() + + for _ in range(warmup_loop): + graph.replay() + torch.cuda.synchronize() + + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + + latencies: List[float] = [] + for _ in range(test_loop): + torch.cuda.synchronize() + dist.barrier() + start_event.record() + graph.replay() + end_event.record() + end_event.synchronize() + latencies.append(start_event.elapsed_time(end_event)) + func_cost_us = sum(latencies) / len(latencies) / graph_loop * 1000 + graph.reset() + return func_output, func_cost_us + + +def _bench_eager_time(func, inp_randn, warmup_loop=2, test_loop=10): + eager_input = inp_randn.clone() + eager_output = func(eager_input) + func_output = eager_output.clone() + + for _ in range(warmup_loop): + func(eager_input) + torch.cuda.synchronize() + + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + torch.cuda.synchronize() + start_event.record() + for _ in range(test_loop): + func(eager_input) + end_event.record() + torch.cuda.synchronize() + func_cost_us = start_event.elapsed_time(end_event) / test_loop * 1000 + + return func_output, func_cost_us + + +def get_torch_prof_ctx(do_prof: bool): + ctx = ( + torch.profiler.profile( + activities=[ + torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ], + record_shapes=True, + with_stack=True, + ) + if do_prof + else nullcontext() + ) + return ctx + + +def human_readable_size(size, decimal_places=1): + for unit in ["B", "KiB", "MiB", "GiB", "TiB", "PiB"]: + if size < 1024.0 or unit == "PiB": + break + size /= 1024.0 + return f"{size:.{decimal_places}f} {unit}" + + +try: + from tabulate import tabulate +except ImportError: + print("tabulate not installed, skipping table printing") + tabulate = None + + +def print_markdown_table(data): + if tabulate is not None: + print(tabulate(data, headers="keys", tablefmt="github")) + return + headers = data[0].keys() + header_row = "| " + " | ".join(headers) + " |" + separator = "| " + " | ".join(["---"] * len(headers)) + " |" + rows = [] + for item in data: + row = "| " + " | ".join(str(item[key]) for key in headers) + " |" + rows.append(row) + markdown_table = "\n".join([header_row, separator] + rows) + print(markdown_table) + + +if __name__ == "__main__": + import logging + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + force=True, + ) + if not dist.is_initialized(): + dist.init_process_group(backend="nccl") + world, world_size = dist.group.WORLD, dist.get_world_size() + rank = dist.get_rank() + torch.cuda.set_device(rank % 8) + device = torch.cuda.current_device() + set_torch_symm_mem_all_reduce(True) + init_distributed_environment( + world_size=world_size, + rank=rank, + local_rank=rank % 8, + ) + initialize_model_parallel(tensor_model_parallel_size=world_size) + group = get_tensor_model_parallel_group().device_group + cpu_group = get_tensor_model_parallel_group().cpu_group + pynccl_comm = get_tensor_model_parallel_group().pynccl_comm + torch_symm_mem_comm = get_tensor_model_parallel_group().torch_symm_mem_comm + dist.barrier() + profile = False + dtype = torch.bfloat16 + ctx = get_torch_prof_ctx(profile) + result = [] + + with ctx: + if IS_CI: + i_range = range(10, 11) + else: + i_range = range(10, 20) + for i in i_range: + sz = 2**i + if sz * dtype.itemsize > 2**24: + break + inp_randn = torch.randint(1, 16, (sz,), dtype=dtype, device=device) + + memory = torch.empty_like(inp_randn) + memory_out = torch.empty_like(memory) + torch_eager_output, torch_eager_time = _bench_eager_time( + lambda inp: torch_allreduce(inp, group), inp_randn + ) + symm_mem_eager_output, symm_mem_eager_time = _bench_eager_time( + lambda inp: torch_symm_mem_allreduce(inp, torch_symm_mem_comm), + inp_randn, + ) + symm_mem_graph_output, symm_mem_graph_time = _bench_graph_time( + lambda inp: torch_symm_mem_allreduce(inp, torch_symm_mem_comm), + inp_randn, + ) + # since pynccl is inplace op, this return result is not correct if graph loop > 1 + _, pynccl_graph_time = _bench_graph_time( + lambda inp: pynccl_allreduce(inp, pynccl_comm), inp_randn + ) + torch.testing.assert_close(torch_eager_output, symm_mem_graph_output) + torch.testing.assert_close(torch_eager_output, symm_mem_eager_output) + result.append( + { + "msg_size": human_readable_size(inp_randn.nbytes), + "torch eager time": torch_eager_time, + "symm mem eager time": symm_mem_eager_time, + "symm mem graph time": symm_mem_graph_time, + "pynccl graph time": pynccl_graph_time, + } + ) + if rank == 0: + print(f"sz={sz}, dtype={dtype}: correctness check PASS!") + if rank == 0: + print_markdown_table(result) + if profile: + prof_dir = f"prof/torch_symm_mem" + os.makedirs(prof_dir, exist_ok=True) + ctx.export_chrome_trace(f"{prof_dir}/trace_rank{dist.get_rank()}.json.gz") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/decoding_attention_triton/triton_flashinfer_cudnn.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/decoding_attention_triton/triton_flashinfer_cudnn.py new file mode 100644 index 00000000..b61240b0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/decoding_attention_triton/triton_flashinfer_cudnn.py @@ -0,0 +1,403 @@ +import itertools +import math + +import cudnn +import torch +import torch.utils.benchmark as benchmark +from flashinfer import BatchDecodeWithPagedKVCacheWrapper + +from sglang.srt.layers.attention.triton_ops.decode_attention import decode_attention_fwd +from sglang.srt.utils import should_use_tensor_core + + +def benchmark_forward( + fn, + *inputs, + repeats=10, + amp=False, + amp_dtype=torch.float16, + **kwinputs, +): + def amp_wrapper(*inputs, **kwinputs): + with torch.autocast(device_type="cuda", dtype=amp_dtype, enabled=amp): + fn(*inputs, **kwinputs) + + t = benchmark.Timer( + stmt="fn_amp(*inputs, **kwinputs)", + globals={"fn_amp": amp_wrapper, "inputs": inputs, "kwinputs": kwinputs}, + num_threads=torch.get_num_threads(), + ) + m = t.timeit(repeats) + return t, m + + +def time_fwd(func, *args, **kwargs): + time_f = benchmark_forward(func, *args, **kwargs) + return time_f[1].mean * 1e6 + + +def decode_attention_sglang( + q, + kv_data, + batch_size, + kv_len, + head_num_q, + head_num_kv, + head_dim, + num_kv_splits, + warmup=10, +): + + k_buffer = kv_data[0].view(-1, head_num_kv, head_dim) + v_buffer = kv_data[1].view(-1, head_num_kv, head_dim) + o = torch.empty_like(q) + total_tokens = batch_size * kv_len + req_to_token = torch.arange(0, total_tokens).to(0).int().view(batch_size, kv_len) + b_req_idx = torch.arange(0, batch_size).to(0).int() + b_seq_len = torch.full((batch_size,), kv_len, dtype=torch.int32, device="cuda") + max_len_in_batch = kv_len + sm_scale = 1.0 / (head_dim**0.5) + + attn_logits = torch.empty( + (batch_size, head_num_q, num_kv_splits, head_dim + 1), + dtype=torch.float32, + device="cuda", + ) + + for _ in range(warmup): + decode_attention_fwd( + q, + k_buffer, + v_buffer, + o, + req_to_token, + b_req_idx, + b_seq_len, + attn_logits, + num_kv_splits, + sm_scale, + ) + + f = time_fwd( + decode_attention_fwd, + q, + k_buffer, + v_buffer, + o, + req_to_token, + b_req_idx, + b_seq_len, + attn_logits, + num_kv_splits, + sm_scale, + ) + + return f, o + + +def decode_attention_flashinfer(dtype, head_num_q, head_num_kv): + workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8, device="cuda") + use_tensor_cores = should_use_tensor_core( + kv_cache_dtype=dtype, + num_attention_heads=head_num_q, + num_kv_heads=head_num_kv, + ) + flashinfer_decode_wrapper = BatchDecodeWithPagedKVCacheWrapper( + workspace_buffer, "NHD", use_tensor_cores=use_tensor_cores + ) + + class FlashinferAttention(torch.autograd.Function): + @staticmethod + def forward( + ctx, + q, + kv_data, + batch_size, + kv_len, + head_num_q, + head_num_kv, + head_dim, + dtype, + warmup=10, + ): + total_tokens = batch_size * kv_len + kv_indptr = torch.arange(0, batch_size + 1).to(0).int() * kv_len + kv_indices = torch.arange(0, total_tokens).to(0).int() + kv_last_page_len = torch.full( + (batch_size,), 1, dtype=torch.int32, device="cuda" + ) + + flashinfer_decode_wrapper.end_forward() + flashinfer_decode_wrapper.begin_forward( + kv_indptr, + kv_indices, + kv_last_page_len, + head_num_q, + head_num_kv, + head_dim, + 1, + pos_encoding_mode="NONE", + data_type=dtype, + ) + + for _ in range(warmup): + o = flashinfer_decode_wrapper.forward( + q.contiguous().view(-1, head_num_q, head_dim), kv_data + ) + + f = time_fwd( + flashinfer_decode_wrapper.forward, + q.contiguous().view(-1, head_num_q, head_dim), + kv_data, + ) + + return f, o + + return FlashinferAttention + + +def convert_to_cudnn_type(torch_type): + if torch_type == torch.float16: + return cudnn.data_type.HALF + elif torch_type == torch.bfloat16: + return cudnn.data_type.BFLOAT16 + elif torch_type == torch.float32: + return cudnn.data_type.FLOAT + elif torch_type == torch.int32: + return cudnn.data_type.INT32 + elif torch_type == torch.int64: + return cudnn.data_type.INT64 + else: + raise ValueError("Unsupported tensor data type.") + + +def decode_attention_cudnn( + q, kv_data, batch_size, kv_len, head_num_q, head_num_kv, head_dim, dtype, warmup=10 +): + # Prepare data: continuous q,k,v + dims_q = (batch_size, head_num_q, 1, head_dim) + strides_q = (head_num_q * head_dim, head_dim, head_num_q * head_dim, 1) + q_gpu = q.as_strided(dims_q, strides_q) + o_gpu = ( + torch.empty(batch_size * head_num_q * head_dim) + .half() + .cuda() + .as_strided(dims_q, strides_q) + ) + + dims_kv = (batch_size, head_num_kv, kv_len, head_dim) + strides_kv = ( + kv_len * head_num_kv * head_dim, + head_dim, + head_num_kv * head_dim, + 1, + ) + k_gpu = kv_data[0].as_strided(dims_kv, strides_kv) + v_gpu = kv_data[1].as_strided(dims_kv, strides_kv) + + seq_len_q_gpu = torch.full((batch_size, 1, 1, 1), 1, device="cuda") + seq_len_kv_gpu = torch.full((batch_size, 1, 1, 1), kv_len, device="cuda") + attn_scale = 1.0 / (head_dim**0.5) + + # Prepare data: paged k,v + block_size = 1 + blocks_per_batch = math.ceil(kv_len / block_size) + # [num_blocks, head_num_kv, block_size, head_dim], num_blocks = batch_size * blocks_per_batch + container_k_gpu = torch.cat(k_gpu.chunk(blocks_per_batch, dim=2), dim=0) + container_v_gpu = torch.cat(v_gpu.chunk(blocks_per_batch, dim=2), dim=0) + page_table_k_gpu = ( + torch.linspace( + 0, + batch_size * blocks_per_batch - 1, + batch_size * blocks_per_batch, + device="cuda", + dtype=torch.int32, + ) + .reshape(blocks_per_batch, 1, batch_size, 1) + .transpose(0, 2) + ) + page_table_v_gpu = page_table_k_gpu.clone() + + graph = cudnn.pygraph( + io_data_type=convert_to_cudnn_type(dtype), + intermediate_data_type=cudnn.data_type.FLOAT, + compute_data_type=cudnn.data_type.FLOAT, + ) + + q = graph.tensor_like(q_gpu) + container_k = graph.tensor_like(container_k_gpu) + container_v = graph.tensor_like(container_v_gpu) + page_table_k = graph.tensor_like(page_table_k_gpu) + page_table_v = graph.tensor_like(page_table_v_gpu) + + seq_len_q = graph.tensor_like(seq_len_q_gpu) + seq_len_kv = graph.tensor_like(seq_len_kv_gpu) + + o, _ = graph.sdpa( + name="sdpa", + q=q, + k=container_k, # Container K: non contiguous container with K blocks + v=container_v, # Container V: non contiguous container with V blocks + is_inference=True, + attn_scale=attn_scale, + use_causal_mask=False, + use_padding_mask=True, + seq_len_q=seq_len_q, + seq_len_kv=seq_len_kv, + paged_attention_k_table=page_table_k, # Page Table K: Tensor containing offsets to the container with K blocks + paged_attention_v_table=page_table_v, # Page Table V: Tensor containing offsets to the container with V blocks + paged_attention_max_seq_len_kv=kv_len, # The maximum sequence length for K caches (this is optional, but recommended) + ) + + o.set_output(True).set_dim(dims_q).set_stride(strides_q) + + graph.validate() + graph.build_operation_graph() + graph.create_execution_plans([cudnn.heur_mode.A]) + graph.check_support() + graph.build_plans() + + workspace = torch.empty( + graph.get_workspace_size(), device="cuda", dtype=torch.uint8 + ) + + variant_pack = { + q: q_gpu, + container_k: container_k_gpu, + container_v: container_v_gpu, + page_table_k: page_table_k_gpu, + page_table_v: page_table_v_gpu, + seq_len_q: seq_len_q_gpu, + seq_len_kv: seq_len_kv_gpu, + o: o_gpu, + } + + for _ in range(warmup): + graph.execute(variant_pack, workspace) + + f = time_fwd( + graph.execute, + variant_pack, + workspace, + ) + + return f, o_gpu.squeeze(dim=2) + + +def calculate_diff(): + + dtype = torch.float16 + batch_size = 64 + kv_len = 4096 + head_num_q = 64 + head_num_kv = 8 + head_dim = 128 + + q = torch.randn(batch_size, head_num_q, head_dim, dtype=dtype, device="cuda") + kv_data = ( + torch.randn( + batch_size * kv_len, head_num_kv, head_dim, dtype=dtype, device="cuda" + ), + torch.randn( + batch_size * kv_len, head_num_kv, head_dim, dtype=dtype, device="cuda" + ), + ) + + _, output_sglang = decode_attention_sglang( + q, + kv_data, + batch_size, + kv_len, + head_num_q, + head_num_kv, + head_dim, + num_kv_splits=8, + ) + + attn_flashinfer = decode_attention_flashinfer(dtype, head_num_q, head_num_kv).apply + _, output_flashinfer = attn_flashinfer( + q, kv_data, batch_size, kv_len, head_num_q, head_num_kv, head_dim, dtype + ) + + _, output_cudnn = decode_attention_cudnn( + q, kv_data, batch_size, kv_len, head_num_q, head_num_kv, head_dim, dtype + ) + + print(f"SGLang output={output_sglang}") + print(f"FlashInfer output={output_flashinfer}") + print(f"cuDNN output={output_cudnn}") + if torch.allclose(output_sglang, output_flashinfer, atol=1e-2, rtol=1e-2): + print("✅ SGLang[Triton] and FlashInfer match") + else: + print("❌ SGLang[Triton] and FlashInfer differ") + + if torch.allclose(output_sglang, output_cudnn, atol=1e-2, rtol=1e-2): + print("✅ SGLang[Triton] and cuDNN match") + else: + print("❌ SGLang[Triton] and cuDNN differ") + + +if __name__ == "__main__": + calculate_diff() + + head_dim = 128 + dtype = torch.float16 + batch_size_range = [2**i for i in range(0, 8, 2)] + kv_len_range = [2**i for i in range(6, 13, 1)] + configs = list(itertools.product(batch_size_range, kv_len_range)) + + for head_num_q, head_num_kv in [[32, 32], [64, 8], [40, 8]]: + attn_flashinfer = decode_attention_flashinfer( + dtype, head_num_q, head_num_kv + ).apply + for batch_size, kv_len in configs: + q = torch.randn( + batch_size, head_num_q, head_dim, dtype=dtype, device="cuda" + ) + kv_data = ( + torch.randn( + batch_size * kv_len, + head_num_kv, + head_dim, + dtype=dtype, + device="cuda", + ), + torch.randn( + batch_size * kv_len, + head_num_kv, + head_dim, + dtype=dtype, + device="cuda", + ), + ) + us_cudnn, output_cudnn = decode_attention_cudnn( + q, kv_data, batch_size, kv_len, head_num_q, head_num_kv, head_dim, dtype + ) + us_sglang, output_sglang = decode_attention_sglang( + q, + kv_data, + batch_size, + kv_len, + head_num_q, + head_num_kv, + head_dim, + num_kv_splits=8, + ) + us_flashinfer, _ = attn_flashinfer( + q, kv_data, batch_size, kv_len, head_num_q, head_num_kv, head_dim, dtype + ) + print( + head_num_q, + " ", + head_num_kv, + " ", + batch_size, + " ", + kv_len, + " ", + us_cudnn, + " ", + us_sglang, + " ", + us_flashinfer, + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepep/deepep_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepep/deepep_utils.py new file mode 100644 index 00000000..169529ef --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepep/deepep_utils.py @@ -0,0 +1,218 @@ +# ADAPTED FROM https://github.com/deepseek-ai/DeepEP/blob/main/tests/utils.py + +import os +import sys +from typing import Optional + +import numpy as np +import torch +import torch.distributed as dist + + +def init_dist(local_rank: int, num_local_ranks: int, args): + ip = args.master_addr + port = args.master_port + num_nodes = args.nnodes + node_rank = args.node_rank + assert (num_local_ranks < 8 and num_nodes == 1) or num_local_ranks == 8 + + dist.init_process_group( + backend="nccl", + init_method=f"tcp://{ip}:{port}", + world_size=num_nodes * num_local_ranks, + rank=node_rank * num_local_ranks + local_rank, + ) + torch.set_default_dtype(torch.bfloat16) + torch.set_default_device("cuda") + torch.cuda.set_device(local_rank) + + return ( + dist.get_rank(), + dist.get_world_size(), + dist.new_group(list(range(num_local_ranks * num_nodes))), + ) + + +def calc_diff(x: torch.Tensor, y: torch.Tensor): + x, y = x.double() + 1, y.double() + 1 + denominator = (x * x + y * y).sum() + sim = 2 * (x * y).sum() / denominator + return (1 - sim).item() + + +def per_token_cast_to_fp8(x: torch.Tensor): + assert x.dim() == 2 and x.size(1) % 128 == 0 + m, n = x.shape + x_view = x.view(m, -1, 128) + x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4) + return (x_view * (448.0 / x_amax.unsqueeze(2))).to(torch.float8_e4m3fn).view( + m, n + ), (x_amax / 448.0).view(m, -1) + + +def per_token_cast_back(x_fp8: torch.Tensor, x_scales: torch.Tensor): + x_fp32 = x_fp8.to(torch.float32).view(x_fp8.size(0), -1, 128) + x_scales = x_scales.view(x_fp8.size(0), -1, 1) + return (x_fp32 * x_scales).view(x_fp8.shape).to(torch.bfloat16) + + +def inplace_unique(x: torch.Tensor, num_slots: int): + assert x.dim() == 2 + mask = x < 0 + x_padded = x.masked_fill(mask, num_slots) + bin_count = torch.zeros((x.size(0), num_slots + 1), dtype=x.dtype, device=x.device) + bin_count.scatter_add_(1, x_padded, torch.ones_like(x_padded)) + bin_count = bin_count[:, :num_slots] + sorted_bin_count, sorted_bin_idx = torch.sort(bin_count, dim=-1, descending=True) + sorted_bin_idx.masked_fill_(sorted_bin_count == 0, -1) + sorted_bin_idx = torch.sort(sorted_bin_idx, descending=True, dim=-1).values + x[:, :].fill_(-1) + valid_len = min(num_slots, x.size(1)) + x[:, :valid_len] = sorted_bin_idx[:, :valid_len] + + +def create_grouped_scores( + scores: torch.Tensor, group_idx: torch.Tensor, num_groups: int +): + num_tokens, num_experts = scores.shape + scores = scores.view(num_tokens, num_groups, -1) + mask = torch.zeros((num_tokens, num_groups), dtype=torch.bool, device=scores.device) + mask = mask.scatter_(1, group_idx, True).unsqueeze(-1).expand_as(scores) + return (scores * mask).view(num_tokens, num_experts) + + +def bench(fn, num_warmups: int = 20, num_tests: int = 30, post_fn=None): + # Flush L2 cache with 256 MB data + torch.cuda.synchronize() + cache = torch.empty(int(256e6 // 4), dtype=torch.int, device="cuda") + + # Warmup + for _ in range(num_warmups): + fn() + + # Flush L2 + cache.zero_() + + # Testing + start_events = [torch.cuda.Event(enable_timing=True) for _ in range(num_tests)] + end_events = [torch.cuda.Event(enable_timing=True) for _ in range(num_tests)] + for i in range(num_tests): + # Record + start_events[i].record() + fn() + end_events[i].record() + if post_fn is not None: + post_fn() + torch.cuda.synchronize() + + times = np.array( + [s.elapsed_time(e) / 1e3 for s, e in zip(start_events, end_events)] + )[1:] + return np.average(times), np.min(times), np.max(times) + + +class empty_suppress: + def __enter__(self): + return self + + def __exit__(self, *_): + pass + + +class suppress_stdout_stderr: + def __enter__(self): + self.outnull_file = open(os.devnull, "w") + self.errnull_file = open(os.devnull, "w") + + self.old_stdout_fileno_undup = sys.stdout.fileno() + self.old_stderr_fileno_undup = sys.stderr.fileno() + + self.old_stdout_fileno = os.dup(sys.stdout.fileno()) + self.old_stderr_fileno = os.dup(sys.stderr.fileno()) + + self.old_stdout = sys.stdout + self.old_stderr = sys.stderr + + os.dup2(self.outnull_file.fileno(), self.old_stdout_fileno_undup) + os.dup2(self.errnull_file.fileno(), self.old_stderr_fileno_undup) + + sys.stdout = self.outnull_file + sys.stderr = self.errnull_file + return self + + def __exit__(self, *_): + sys.stdout = self.old_stdout + sys.stderr = self.old_stderr + + os.dup2(self.old_stdout_fileno, self.old_stdout_fileno_undup) + os.dup2(self.old_stderr_fileno, self.old_stderr_fileno_undup) + + os.close(self.old_stdout_fileno) + os.close(self.old_stderr_fileno) + + self.outnull_file.close() + self.errnull_file.close() + + +def bench_kineto( + fn, + kernel_names, + num_tests: int = 30, + suppress_kineto_output: bool = False, + trace_path: Optional[str] = None, + barrier_comm_profiling: bool = False, +): + # Profile + suppress = suppress_stdout_stderr if suppress_kineto_output else empty_suppress + with suppress(): + schedule = torch.profiler.schedule(wait=0, warmup=1, active=1, repeat=1) + with torch.profiler.profile( + activities=[torch.profiler.ProfilerActivity.CUDA], schedule=schedule + ) as prof: + for i in range(2): + # NOTES: use a large kernel and a barrier to eliminate the unbalanced CPU launch overhead + if barrier_comm_profiling: + lhs = torch.randn((8192, 8192), dtype=torch.float, device="cuda") + rhs = torch.randn((8192, 8192), dtype=torch.float, device="cuda") + lhs @ rhs + dist.all_reduce(torch.ones(1, dtype=torch.float, device="cuda")) + for _ in range(num_tests): + fn() + prof.step() + + # Parse the profiling table + assert isinstance(kernel_names, str) or isinstance(kernel_names, tuple) + is_tupled = isinstance(kernel_names, tuple) + prof_lines = ( + prof.key_averages() + .table(sort_by="cuda_time_total", max_name_column_width=100) + .split("\n") + ) + kernel_names = (kernel_names,) if isinstance(kernel_names, str) else kernel_names + assert all([isinstance(name, str) for name in kernel_names]) + for name in kernel_names: + assert ( + sum([name in line for line in prof_lines]) == 1 + ), f"Errors of the kernel {name} in the profiling table" + + # Save chrome traces + if trace_path is not None: + prof.export_chrome_trace(trace_path) + + # Return average kernel times + units = {"ms": 1e3, "us": 1e6} + kernel_times = [] + for name in kernel_names: + for line in prof_lines: + if name in line: + time_str = line.split()[-2] + for unit, scale in units.items(): + if unit in time_str: + kernel_times.append(float(time_str.replace(unit, "")) / scale) + break + break + return tuple(kernel_times) if is_tupled else kernel_times[0] + + +def hash_tensor(t: torch.Tensor): + return t.view(torch.int64).sum().item() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepep/tuning_deepep.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepep/tuning_deepep.py new file mode 100644 index 00000000..db08a8f1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepep/tuning_deepep.py @@ -0,0 +1,476 @@ +# MODIFIED FROM https://github.com/deepseek-ai/DeepEP/blob/main/tests/test_internode.py + +""" +Example usage: +python tuning_deepep.py --nnodes 4 --node-rank $MY_NODE_RANK --master-addr 1.2.3.4 +Then check `deepep_tuned.json` +""" + +import argparse +import json +import time +from copy import deepcopy +from pathlib import Path + +# noinspection PyUnresolvedReferences +import deep_ep +import torch +import torch.distributed as dist +from deepep_utils import ( + bench, + calc_diff, + create_grouped_scores, + init_dist, + inplace_unique, + per_token_cast_back, + per_token_cast_to_fp8, +) + + +def test_main( + num_sms: int, + local_rank: int, + num_local_ranks: int, + num_ranks: int, + num_nodes: int, + rank: int, + buffer: deep_ep.Buffer, + group: dist.ProcessGroup, + args, +): + # Settings + num_tokens, hidden, num_topk_groups, num_topk, num_experts = ( + 4096, + 7168, + min(num_nodes, 4), + 8, + (256 // num_ranks) * num_ranks, + ) + assert num_experts % num_ranks == 0 and num_local_ranks == 8 + if local_rank == 0: + print( + f"[config] num_tokens={num_tokens}, hidden={hidden}, num_topk_groups={num_topk_groups}, num_topk={num_topk}", + flush=True, + ) + + # Random data + x = torch.ones((num_tokens, hidden), dtype=torch.bfloat16, device="cuda") * rank + x_pure_rand = torch.randn((num_tokens, hidden), dtype=torch.bfloat16, device="cuda") + x_e4m3 = per_token_cast_to_fp8(x) + scores = ( + torch.randn((num_tokens, num_experts), dtype=torch.float32, device="cuda").abs() + + 1 + ) + group_scores = scores.view(num_tokens, num_nodes, -1).amax(dim=-1) + group_idx = torch.topk( + group_scores, k=num_topk_groups, dim=-1, sorted=False + ).indices + masked_scores = create_grouped_scores(scores, group_idx, num_nodes) + topk_idx = torch.topk(masked_scores, num_topk, dim=-1, largest=True, sorted=False)[ + 1 + ] + topk_weights = ( + torch.ones((num_tokens, num_topk), dtype=torch.float32, device="cuda") * rank + ) + topk_weights_pure_rand = torch.randn( + (num_tokens, num_topk), dtype=torch.float32, device="cuda" + ) + rank_idx = topk_idx // (num_experts // num_ranks) + rank_idx.masked_fill_(topk_idx == -1, -1) + inplace_unique(rank_idx, num_ranks) + rdma_rank_idx = rank_idx // num_local_ranks + rdma_rank_idx.masked_fill_(rank_idx == -1, -1) + inplace_unique(rdma_rank_idx, num_nodes) + + # RDMA dispatch counts + rdma_idx = topk_idx // (num_experts // num_nodes) + rdma_idx.masked_fill_(topk_idx == -1, -1) + inplace_unique(rdma_idx, num_nodes) + num_rdma_token_sent = rdma_idx.ne(-1).sum().item() + + # Expert meta + num_tokens_per_expert = torch.zeros((num_experts,), dtype=torch.int, device="cuda") + for i in range(num_experts): + num_tokens_per_expert[i] = (topk_idx == i).sum() + gbl_num_tokens_per_expert = num_tokens_per_expert.clone() + dist.all_reduce(gbl_num_tokens_per_expert, group=group) + + # Rank layout meta + num_tokens_per_rank = torch.empty((num_ranks,), dtype=torch.int, device="cuda") + num_tokens_per_rdma_rank = torch.empty((num_nodes,), dtype=torch.int, device="cuda") + token_idx_in_rank = torch.full( + (num_ranks, num_tokens), -1, dtype=torch.long, device="cuda" + ) + for i in range(num_ranks): + num_tokens_per_rank[i] = (rank_idx == i).sum() + token_sel = (rank_idx == i).max(dim=-1)[0] + count = token_sel.sum().item() + tokens = torch.sort(token_sel.to(torch.int), descending=True)[1] + tokens[:count] = torch.sort(tokens[:count])[0] + token_idx_in_rank[i][tokens[:count]] = torch.arange( + count, dtype=torch.long, device="cuda" + ) + for i in range(num_nodes): + num_tokens_per_rdma_rank[i] = (rdma_rank_idx == i).sum() + token_idx_in_rank = token_idx_in_rank.T.contiguous().to(torch.int) + is_token_in_rank = token_idx_in_rank >= 0 + gbl_num_tokens_per_rank = num_tokens_per_rank.clone() + dist.all_reduce(gbl_num_tokens_per_rank, group=group) + + ( + ref_num_tokens_per_rank, + ref_num_tokens_per_rdma_rank, + ref_num_tokens_per_expert, + ref_is_token_in_rank, + _, + ) = buffer.get_dispatch_layout(topk_idx, num_experts) + assert torch.allclose(ref_num_tokens_per_rank, num_tokens_per_rank) + assert torch.allclose(ref_num_tokens_per_rdma_rank, num_tokens_per_rdma_rank) + assert torch.allclose(ref_num_tokens_per_expert, num_tokens_per_expert) + assert torch.allclose(ref_is_token_in_rank, is_token_in_rank) + t = bench(lambda: buffer.get_dispatch_layout(topk_idx, num_experts))[0] + if local_rank == 0: + print(f"[layout] Kernel performance: {t * 1000:.3f} ms", flush=True) + print("", flush=True) + group.barrier() + time.sleep(1) + + # Config + rdma_buffer_size, nvl_buffer_size = 128, (720 if num_ranks in (144, 160) else 512) + config = deep_ep.Config(num_sms, 8, nvl_buffer_size, 16, rdma_buffer_size) + + # Test dispatch + # noinspection PyShadowingNames + def check_data(check_x, recv_gbl_rank_prefix_sum): + assert torch.allclose(check_x.amin(dim=1), check_x.amax(dim=1)) + check_start = 0 + for i in range(num_ranks): + check_end = recv_gbl_rank_prefix_sum[i].item() + assert (check_x[check_start:check_end, :].int() - i).sum().item() == 0 + check_start = check_end + + for previous_mode in (False, True): + for async_mode in (False, True): + for current_x in (x_pure_rand, x, x_e4m3): + for with_topk in (False, True): + if local_rank == 0: + print( + f'[testing] Running with {"FP8" if isinstance(current_x, tuple) else "BF16"}, {"with" if with_topk else "without"} top-k (async={async_mode}, previous={previous_mode}) ...', + flush=True, + end="", + ) + dispatch_args = { + "x": current_x, + "num_tokens_per_rank": num_tokens_per_rank, + "num_tokens_per_rdma_rank": num_tokens_per_rdma_rank, + "is_token_in_rank": is_token_in_rank, + "num_tokens_per_expert": num_tokens_per_expert, + "config": config, + "async_finish": async_mode, + } + if with_topk: + dispatch_args.update( + { + "topk_idx": topk_idx, + "topk_weights": ( + topk_weights_pure_rand + if current_x is x_pure_rand + else topk_weights + ), + } + ) + if previous_mode: + dispatch_args.update({"previous_event": buffer.capture()}) + ( + recv_x, + recv_topk_idx, + recv_topk_weights, + recv_num_tokens_per_expert_list, + handle, + event, + ) = buffer.dispatch(**dispatch_args) + event.current_stream_wait() if async_mode else () + recv_x = ( + per_token_cast_back(*recv_x) + if isinstance(recv_x, tuple) + else recv_x + ) + + # Checks + recv_gbl_rank_prefix_sum = handle[-4] + assert gbl_num_tokens_per_rank[rank].item() == recv_x.size( + 0 + ), f"{gbl_num_tokens_per_rank[rank].item()} != {recv_x.size(0)}" + assert ( + gbl_num_tokens_per_expert.view(num_ranks, -1)[rank].tolist() + == recv_num_tokens_per_expert_list + ) + if current_x is not x_pure_rand: + check_data(recv_x, recv_gbl_rank_prefix_sum) + if with_topk: + # Check `topk_idx` + assert ( + recv_topk_idx.eq(-1) + | ( + (recv_topk_idx >= 0) + & (recv_topk_idx < (num_experts // num_ranks)) + ) + ).sum().item() == recv_topk_idx.numel() + for i, count in enumerate(recv_num_tokens_per_expert_list): + assert recv_topk_idx.eq(i).sum().item() == count + + # Check `topk_weights` + if current_x is not x_pure_rand: + recv_topk_weights[recv_topk_idx.eq(-1)] = ( + recv_topk_weights.amax(dim=1, keepdim=True).expand_as( + recv_topk_weights + )[recv_topk_idx.eq(-1)] + ) + check_data(recv_topk_weights, recv_gbl_rank_prefix_sum) + + # Test cached dispatch (must without top-k staffs) + if not with_topk: + dispatch_args = { + "x": current_x, + "handle": handle, + "config": config, + "async_finish": async_mode, + } + if previous_mode: + dispatch_args.update({"previous_event": buffer.capture()}) + recv_x, _, _, _, _, event = buffer.dispatch(**dispatch_args) + event.current_stream_wait() if async_mode else () + recv_x = ( + per_token_cast_back(*recv_x) + if isinstance(recv_x, tuple) + else recv_x + ) + if current_x is not x_pure_rand: + check_data(recv_x, recv_gbl_rank_prefix_sum) + + # Test combine + combine_args = { + "x": recv_x, + "handle": handle, + "config": config, + "async_finish": async_mode, + } + if with_topk: + combine_args.update({"topk_weights": recv_topk_weights}) + if previous_mode: + combine_args.update({"previous_event": buffer.capture()}) + combined_x, combined_topk_weights, event = buffer.combine( + **combine_args + ) + event.current_stream_wait() if async_mode else () + check_x = combined_x.float() / is_token_in_rank.sum( + dim=1 + ).unsqueeze(1) + ref_x = x_pure_rand if current_x is x_pure_rand else x + assert calc_diff(check_x, ref_x) < 5e-6 + if with_topk: + check_topk_weights = ( + combined_topk_weights + if (current_x is x_pure_rand) + else ( + combined_topk_weights + / is_token_in_rank.sum(dim=1).unsqueeze(1) + ) + ) + ref_topk_weights = ( + topk_weights_pure_rand + if current_x is x_pure_rand + else topk_weights + ) + assert calc_diff(check_topk_weights, ref_topk_weights) < 1e-9 + + # For later tuning + dispatch_bf16_rdma_send_bytes = num_rdma_token_sent * hidden * 2 + dispatch_bf16_nvl_recv_bytes = recv_x.numel() * 2 + combine_bf16_nvl_send_bytes = dispatch_bf16_nvl_recv_bytes + combine_bf16_rdma_recv_bytes = dispatch_bf16_rdma_send_bytes + + if local_rank == 0: + print(" passed", flush=True) + if local_rank == 0: + print("", flush=True) + + output_data = {} + + # Tune dispatch performance + best_dispatch_results = None + fp8_factor = (1 + 4 / 128) / 2 + for current_x in (x_e4m3, x): + best_time, best_results = 1e10, None + rdma_send_bytes = ( + (dispatch_bf16_rdma_send_bytes * fp8_factor) + if isinstance(current_x, tuple) + else dispatch_bf16_rdma_send_bytes + ) + nvl_recv_bytes = ( + (dispatch_bf16_nvl_recv_bytes * fp8_factor) + if isinstance(current_x, tuple) + else dispatch_bf16_nvl_recv_bytes + ) + for nvl_chunk_size in range(4, 33, 4): + for rdma_chunk_size in range(4, 33, 4): + config_kwargs = { + "num_sms": num_sms, + "num_max_nvl_chunked_send_tokens": nvl_chunk_size, + "num_max_nvl_chunked_recv_tokens": nvl_buffer_size, + "num_max_rdma_chunked_send_tokens": rdma_chunk_size, + "num_max_rdma_chunked_recv_tokens": rdma_buffer_size, + } + config = deep_ep.Config(**config_kwargs) + tune_args = {"x": current_x, "handle": handle, "config": config} + t = bench(lambda: buffer.dispatch(**tune_args))[0] + if t < best_time: + best_time, best_results = t, ( + num_sms, + nvl_chunk_size, + rdma_chunk_size, + config_kwargs, + ) + if local_rank == 0: + print( + f"[tuning] SMs {num_sms}, NVL chunk {nvl_chunk_size}, RDMA chunk {rdma_chunk_size}: {rdma_send_bytes / 1e9 / t:.2f} GB/s (RDMA), {nvl_recv_bytes / 1e9 / t:.2f} GB/s (NVL) ", + flush=True, + ) + if local_rank == 0: + print( + f'[tuning] Best dispatch ({"FP8" if isinstance(current_x, tuple) else "BF16"}): SMs {best_results[0]}, NVL chunk {best_results[1]}, RDMA chunk {best_results[2]}: {rdma_send_bytes / 1e9 / best_time:.2f} GB/s (RDMA), {nvl_recv_bytes / 1e9 / best_time:.2f} GB/s (NVL)', + flush=True, + ) + print("", flush=True) + is_fp8 = isinstance(current_x, tuple) + if is_fp8: + output_data["normal_dispatch"] = deepcopy(best_results[3]) + + if isinstance(current_x, tuple): + # Gather FP8 the best config from rank 0 + best_dispatch_results = torch.tensor( + [best_results[0], best_results[1], best_results[2]], + dtype=torch.int32, + device="cuda", + ) + all_best_fp8_results_list = [ + torch.zeros_like(best_dispatch_results) + for _ in range(torch.distributed.get_world_size()) + ] + dist.all_gather( + all_best_fp8_results_list, best_dispatch_results, group=group + ) + best_dispatch_results = all_best_fp8_results_list[0].tolist() + dispatch_config = deep_ep.Config( + best_dispatch_results[0], + best_dispatch_results[1], + nvl_buffer_size, + best_dispatch_results[2], + rdma_buffer_size, + ) + + dispatch_args = { + "x": x, + "num_tokens_per_rank": num_tokens_per_rank, + "num_tokens_per_rdma_rank": num_tokens_per_rdma_rank, + "is_token_in_rank": is_token_in_rank, + "num_tokens_per_expert": num_tokens_per_expert, + "config": dispatch_config if dispatch_config is not None else config, + } + recv_x, _, _, _, handle, _ = buffer.dispatch(**dispatch_args) + + # Tune combine performance + best_time, best_results = 1e10, None + for nvl_chunk_size in range(1, 8, 1): + for rdma_chunk_size in range(12 if num_nodes == 2 else 8, 33, 4): + config_kwargs = { + "num_sms": num_sms, + "num_max_nvl_chunked_send_tokens": nvl_chunk_size, + "num_max_nvl_chunked_recv_tokens": nvl_buffer_size, + "num_max_rdma_chunked_send_tokens": rdma_chunk_size, + "num_max_rdma_chunked_recv_tokens": rdma_buffer_size, + } + config = deep_ep.Config(**config_kwargs) + tune_args = {"x": recv_x, "handle": handle, "config": config} + t = bench(lambda: buffer.combine(**tune_args))[0] + if local_rank == 0: + print( + f"[tuning] SMs {num_sms}, NVL chunk {nvl_chunk_size}, RDMA chunk {rdma_chunk_size}: {combine_bf16_rdma_recv_bytes / 1e9 / t:.2f} GB/s (RDMA), {combine_bf16_nvl_send_bytes / 1e9 / t:.2f} GB/s (NVL) ", + flush=True, + ) + if t < best_time: + best_time, best_results = t, ( + num_sms, + nvl_chunk_size, + rdma_chunk_size, + config_kwargs, + ) + + if local_rank == 0: + print( + f"[tuning] Best combine: SMs {best_results[0]}, NVL chunk {best_results[1]}, RDMA chunk {best_results[2]}: {combine_bf16_rdma_recv_bytes / 1e9 / best_time:.2f} GB/s (RDMA), {combine_bf16_nvl_send_bytes / 1e9 / best_time:.2f} GB/s (NVL)", + flush=True, + ) + print("", flush=True) + output_data["normal_combine"] = deepcopy(best_results[3]) + + if rank == 0 and local_rank == 0: + _write_output(args, output_data) + + +def _write_output(args, output_data): + text = json.dumps(output_data, indent=4) + output_path = args.output_path + print(f"Write to {output_path} with {text}") + Path(output_path).write_text(text) + + +# noinspection PyUnboundLocalVariable +def test_loop(local_rank: int, num_local_ranks: int, args): + num_nodes = args.nnodes + rank, num_ranks, group = init_dist(local_rank, num_local_ranks, args) + + num_sms = args.num_sms + num_qps_per_rank = num_sms // 2 + + buffer = deep_ep.Buffer( + group, + int(1e9), + int(1e9), + low_latency_mode=False, + num_qps_per_rank=num_qps_per_rank, + ) + assert num_local_ranks == 8 and num_ranks > 8 + torch.manual_seed(rank) + + for i in (num_sms,): + test_main( + i, + local_rank, + num_local_ranks, + num_ranks, + num_nodes, + rank, + buffer, + group, + args, + ) + if local_rank == 0: + print("", flush=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--num-sms", type=int, default=24) + parser.add_argument("--output-path", type=str, default="deepep_tuned.json") + parser.add_argument("--nnodes", type=int, default=1) + parser.add_argument("--node-rank", type=int, default=0) + parser.add_argument("--master-addr", type=str, default="127.0.0.1") + parser.add_argument("--master-port", type=int, default=8361) + args = parser.parse_args() + print(f"Start system with {args=}") + + num_processes = 8 + torch.multiprocessing.spawn( + test_loop, args=(num_processes, args), nprocs=num_processes + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepseek/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepseek/README.md new file mode 100644 index 00000000..fba775f2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepseek/README.md @@ -0,0 +1,19 @@ +## DeepSeek kernels benchmark + + +### Prerequisites +- You should install [DeepGemm](https://github.com/deepseek-ai/DeepGEMM) from source before run `benchmark_deepgemm_fp8_gemm.py` and `benchmark_deepgemm_fp8_group_gemm.py`. + +### Benchmark +- `benchmark_deepgemm_fp8_gemm.py` + ```bash + python benchmark_deepgemm_fp8_gemm.py --run_correctness --tp_size 1 + ``` + +- `benchmark_deepgemm_fp8_group_gemm.py` + ```bash + python benchmark_deepgemm_fp8_group_gemm.py --run_correctness --tp_size 1 + ``` + + - You can use the `--run_correctness` parameter to verify all kernels results's correctness. + - You can use the `--tp_size` parameter to benchmark all FP8 w8a8 block-wise matrix multiplications involved in DeepSeek V3/R1 under the current tensor parallelism (TP) setting. This benchmark compares DeepSeek's open-source [DeepGemm](https://github.com/deepseek-ai/DeepGEMM) implementation with SGLang's and VLLM Triton implementation. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm.py new file mode 100644 index 00000000..bd02e2ae --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm.py @@ -0,0 +1,401 @@ +from typing import Tuple + +import deep_gemm +import tilelang +import tilelang.language as T +import torch +import triton +from deep_gemm import ceil_div +from deep_gemm.utils.layout import get_mn_major_tma_aligned_tensor +from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + w8a8_block_fp8_matmul as vllm_w8a8_block_fp8_matmul, +) + +from sglang.srt.layers.quantization.fp8_kernel import ( + w8a8_block_fp8_matmul_deepgemm as w8a8_block_fp8_matmul, +) + + +# Adapted from https://github.com/tile-ai/tilelang/blob/a8cfdce92795cb861c9033573534653ee040b5ed/examples/deepseek_deepgemm/example_deepgemm_fp8_2xAcc.py#L1 +def tl_gemm( + M, + N, + K, + in_dtype, + out_dtype, + accum_dtype, +): + assert in_dtype in [ + "e4m3_float8", + ], "Currently only e4m3_float8 is supported" + assert out_dtype in [ + "bfloat16", + "float16", + ], "Currently only bfloat16 and float16 are supported" + + TILE_SIZE = (128, 128, 128) + block_M = TILE_SIZE[0] + block_N = TILE_SIZE[1] + block_K = TILE_SIZE[2] + + A_shape = (M, K) + Scales_A_shape = (M, T.ceildiv(K, block_K)) + B_shape = (N, K) + Scales_B_shape = (T.ceildiv(N, block_N), T.ceildiv(K, block_K)) + A_shared_shape = (block_M, block_K) + B_shared_shape = (block_N, block_K) + C_shared_shape = (block_M, block_N) + + @T.prim_func + def main( + A: T.Buffer(A_shape, in_dtype), + scales_a: T.Buffer(Scales_A_shape, "float32"), + B: T.Buffer(B_shape, in_dtype), + scales_b: T.Buffer(Scales_B_shape, "float32"), + C: T.Buffer((M, N), out_dtype), + ): + with T.Kernel(T.ceildiv(N, block_N), T.ceildiv(M, block_M), threads=128) as ( + bx, + by, + ): + + A_shared = T.alloc_shared(A_shared_shape, in_dtype) + B_shared = T.alloc_shared(B_shared_shape, in_dtype) + C_shared = T.alloc_shared(C_shared_shape, out_dtype) + Scale_C_shared = T.alloc_shared((block_M), "float32") + C_local = T.alloc_fragment(C_shared_shape, accum_dtype) + C_local_accum = T.alloc_fragment(C_shared_shape, accum_dtype) + + # Improve L2 Cache + T.use_swizzle(panel_size=10) + + T.clear(C_local) + T.clear(C_local_accum) + K_iters = T.ceildiv(K, block_K) + for k in T.Pipelined(K_iters, num_stages=4): + # Load A into shared memory + T.copy(A[by * block_M, k * block_K], A_shared) + # Load B into shared memory + T.copy(B[bx * block_N, k * block_K], B_shared) + # Load scale into shared memory + Scale_B = scales_b[bx, k] + for i in T.Parallel(block_M): + Scale_C_shared[i] = scales_a[by * block_M + i, k] * Scale_B + + T.gemm(A_shared, B_shared, C_local, transpose_B=True) + # Promote to enable 2xAcc + for i, j in T.Parallel(block_M, block_N): + C_local_accum[i, j] += C_local[i, j] * Scale_C_shared[i] + T.clear(C_local) + # TMA store + T.copy(C_local_accum, C_shared) + T.copy(C_shared, C[by * block_M, bx * block_N]) + + return main + + +def per_token_cast_to_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 and x.size(1) % 128 == 0 + m, n = x.shape + x_view = x.view(m, -1, 128) + x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4) + return (x_view * (448.0 / x_amax.unsqueeze(2))).to(torch.float8_e4m3fn).view( + m, n + ), (x_amax / 448.0).view(m, -1) + + +def per_block_cast_to_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 + m, n = x.shape + x_padded = torch.zeros( + (ceil_div(m, 128) * 128, ceil_div(n, 128) * 128), dtype=x.dtype, device=x.device + ) + x_padded[:m, :n] = x + x_view = x_padded.view(-1, 128, x_padded.size(1) // 128, 128) + x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4) + x_scaled = (x_view * (448.0 / x_amax)).to(torch.float8_e4m3fn) + return x_scaled.view_as(x_padded)[:m, :n].contiguous(), (x_amax / 448.0).view( + x_view.size(0), x_view.size(2) + ) + + +def fp8_gemm_deepgemm( + x_fp8: torch.Tensor, + x_scale: torch.Tensor, + y_fp8: torch.Tensor, + y_scale: torch.Tensor, + m: int, + n: int, + k: int, +): + """DeepGEMM implementation of FP8 GEMM""" + out = torch.empty((m, n), device="cuda", dtype=torch.bfloat16) + + # Run DeepGEMM kernel + deep_gemm.fp8_gemm_nt((x_fp8, x_scale), (y_fp8, y_scale), out) + return out + + +def fp8_gemm_sglang( + x_fp8: torch.Tensor, + x_scale: torch.Tensor, + y_fp8: torch.Tensor, + y_scale: torch.Tensor, + m: int, + n: int, + k: int, +): + """SGLang implementation of FP8 GEMM""" + block_size = [128, 128] # Matches the block size in per_block_cast_to_fp8 + + # Run SGLang kernel + out = w8a8_block_fp8_matmul( + x_fp8, y_fp8, x_scale, y_scale, block_size, torch.bfloat16 + ) + return out + + +def fp8_gemm_vllm( + x_fp8: torch.Tensor, + x_scale: torch.Tensor, + y_fp8: torch.Tensor, + y_scale: torch.Tensor, + m: int, + n: int, + k: int, +): + """vLLM implementation of FP8 GEMM""" + block_size = [128, 128] # Matches the block size in per_block_cast_to_fp8 + + # Run vLLM kernel + out = vllm_w8a8_block_fp8_matmul( + x_fp8, y_fp8, x_scale, y_scale, block_size, torch.bfloat16 + ) + return out + + +def calculate_diff(m: int, n: int, k: int): + x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) + y = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) + + x_fp8, x_scale = per_token_cast_to_fp8(x.clone()) + y_fp8, y_scale = per_block_cast_to_fp8(y.clone()) + x_scale_col_major = get_mn_major_tma_aligned_tensor(x_scale.clone()) + + out_deepgemm = fp8_gemm_deepgemm( + x_fp8.clone(), + x_scale_col_major.clone(), + y_fp8.clone(), + y_scale.clone(), + m, + n, + k, + ) + out_sglang = fp8_gemm_sglang( + x_fp8.clone(), x_scale.clone(), y_fp8.clone(), y_scale.clone(), m, n, k + ) + + tilelang_func = tl_gemm(m, n, k, "e4m3_float8", "bfloat16", "float32") + tilelang_kernel = tilelang.compile(tilelang_func, out_idx=[-1]) + out_tilelang = tilelang_kernel( + x_fp8.clone(), x_scale.clone(), y_fp8.clone(), y_scale.clone() + ) + + diff_sglang_deepgemm = torch.abs(out_deepgemm - out_sglang).mean().item() + diff_tilelang_deepgemm = torch.abs(out_deepgemm - out_tilelang).mean().item() + diff_tilelang_sglang = torch.abs(out_tilelang - out_sglang).mean().item() + + print(f"Shape m={m}, n={n}, k={k}:") + print(f"DeepGEMM output: {out_deepgemm[0, 0:5]}") + print(f"SGLang output: {out_sglang[0, 0:5]}") + print(f"TileLang output: {out_tilelang[0, 0:5]}") + print(f"Mean absolute difference (SGLang-DeepGEMM): {diff_sglang_deepgemm}") + print(f"Mean absolute difference (TileLang-DeepGEMM): {diff_tilelang_deepgemm}") + print(f"Mean absolute difference (TileLang-SGLang): {diff_tilelang_sglang}") + + sglang_deepgemm_match = torch.allclose( + out_deepgemm, out_sglang, atol=1e-2, rtol=1e-2 + ) + tilelang_deepgemm_match = torch.allclose( + out_deepgemm, out_tilelang, atol=1e-2, rtol=1e-2 + ) + tilelang_sglang_match = torch.allclose( + out_tilelang, out_sglang, atol=1e-2, rtol=1e-2 + ) + + if sglang_deepgemm_match and tilelang_deepgemm_match and tilelang_sglang_match: + print("✅ All implementations match\n") + else: + print("❌ Some implementations differ:") + print(f" - SGLang vs DeepGEMM: {'✅' if sglang_deepgemm_match else '❌'}") + print(f" - TileLang vs DeepGEMM: {'✅' if tilelang_deepgemm_match else '❌'}") + print(f" - TileLang vs SGLang: {'✅' if tilelang_sglang_match else '❌'}\n") + + +def get_weight_shapes(tp_size): + # cannot TP + total = [ + (512 + 64, 7168), + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (7168, 16384), + (7168, 18432), + ] + # N can TP + n_tp = [ + (18432 * 2, 7168), + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (24576, 1536), + (4096, 7168), + ] + # K can TP + k_tp = [(7168, 18432), (7168, 16384), (7168, 2048)] + + weight_shapes = [] + for t in total: + weight_shapes.append(t) + for n_t in n_tp: + new_t = (n_t[0] // tp_size, n_t[1]) + weight_shapes.append(new_t) + for k_t in k_tp: + new_t = (k_t[0], k_t[1] // tp_size) + weight_shapes.append(new_t) + + return weight_shapes + + +def create_benchmark_configs(tp_size): + configs = [] + weight_shapes = get_weight_shapes(tp_size) + batch_sizes = [8, 16, 32, 64, 128, 256, 1024, 2048, 4096] + + for n, k in weight_shapes: + for m in batch_sizes: + configs.append((m, n, k, tp_size)) + + return configs + + +def get_benchmark(tp_size): + all_configs = create_benchmark_configs(tp_size) + + @triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["m", "n", "k", "tp_size"], + x_vals=[list(config) for config in all_configs], + line_arg="provider", + line_vals=["deepgemm", "sglang", "tilelang"], + line_names=["DeepGEMM", "SGLang", "TileLang"], + styles=[("blue", "-"), ("red", "-"), ("green", "-")], + ylabel="ms", + plot_name=f"fp8-gemm-performance-comparison-tp{tp_size}", + args={}, + ) + ) + def benchmark(m, n, k, tp_size, provider): + print(f"Shape (m={m}, n={n}, k={k}, tp={tp_size}), Provider: {provider}") + x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) + y = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) + + # Preprocess data before benchmarking + x_fp8, x_scale = per_token_cast_to_fp8(x) + y_fp8, y_scale = per_block_cast_to_fp8(y) + x_scale_col_major = get_mn_major_tma_aligned_tensor(x_scale.clone()) + + quantiles = [0.5, 0.2, 0.8] + + if provider == "deepgemm": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: fp8_gemm_deepgemm( + x_fp8.clone(), + x_scale_col_major.clone(), + y_fp8.clone(), + y_scale.clone(), + m, + n, + k, + ), + quantiles=quantiles, + ) + elif provider == "sglang": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: fp8_gemm_sglang( + x_fp8.clone(), + x_scale.clone(), + y_fp8.clone(), + y_scale.clone(), + m, + n, + k, + ), + quantiles=quantiles, + ) + else: # tilelang + tilelang_func = tl_gemm(m, n, k, "e4m3_float8", "bfloat16", "float32") + tilelang_kernel = tilelang.compile(tilelang_func, out_idx=[-1]) + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: tilelang_kernel( + x_fp8.clone(), + x_scale.clone(), + y_fp8.clone(), + y_scale.clone(), + ), + quantiles=quantiles, + ) + + # Calculate TFLOPS + flops = 2 * m * n * k # multiply-adds + tflops = flops / (ms * 1e-3) / 1e12 + + # Print shape-specific results with TFLOPS + print(f"Time: {ms*1000:.2f} ms, TFLOPS: {tflops:.2f}") + return ms * 1000, max_ms * 1000, min_ms * 1000 # convert to ms + + return benchmark + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", + type=str, + default="./configs/benchmark_ops/fp8_gemm/", + help="Path to save fp8 gemm benchmark results", + ) + parser.add_argument( + "--run_correctness", + action="store_true", + default=True, + help="Whether to run correctness test", + ) + parser.add_argument( + "--tp_size", + type=int, + default=1, + help="Tensor parallelism size to benchmark (default: 1)", + ) + args = parser.parse_args() + + # Set random seed for reproducibility + torch.manual_seed(0) + torch.cuda.manual_seed(0) + + # Enable TF32, adapted from https://github.com/deepseek-ai/DeepGEMM/blob/main/tests/test_core.py#L148 + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.allow_tf32 = True + + # Run correctness tests on a few examples + if args.run_correctness: + print("Running correctness tests...") + calculate_diff(64, 512, 7168) # Small test + calculate_diff(64, 7168, 16384) # Medium test + calculate_diff(64, 18432, 7168) # Large test + + # Get the benchmark function with the specified tp_size + benchmark = get_benchmark(args.tp_size) + + print(f"Running performance benchmark for TP size = {args.tp_size}...") + benchmark.run(print_data=True, save_path=args.save_path) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm_blackwell.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm_blackwell.py new file mode 100644 index 00000000..de14bd90 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm_blackwell.py @@ -0,0 +1,329 @@ +import argparse +from typing import Tuple + +import torch +import triton +from deep_gemm import ceil_div +from flashinfer.gemm import gemm_fp8_nt_groupwise + +from sglang.srt.layers.quantization.fp8_kernel import ( + sglang_per_token_group_quant_fp8, + w8a8_block_fp8_matmul_deepgemm, +) +from sglang.srt.layers.quantization.fp8_utils import requant_weight_ue8m0 + +BLOCK_SIZE = 128 + + +def per_block_cast_to_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 + assert BLOCK_SIZE == 128 + m, n = x.shape + x_padded = torch.zeros( + (ceil_div(m, 128) * 128, ceil_div(n, 128) * 128), dtype=x.dtype, device=x.device + ) + x_padded[:m, :n] = x + x_view = x_padded.view(-1, 128, x_padded.size(1) // 128, 128) + x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4) + x_scaled = (x_view * (448.0 / x_amax)).to(torch.float8_e4m3fn) + return x_scaled.view_as(x_padded)[:m, :n].contiguous(), (x_amax / 448.0).view( + x_view.size(0), x_view.size(2) + ) + + +def get_weight_shapes(tp_size): + # cannot TP + total = [ + (512 + 64, 7168), + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (7168, 16384), + (7168, 18432), + ] + # N can TP + n_tp = [ + (18432 * 2, 7168), + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (24576, 1536), + (4096, 7168), + ] + # K can TP + k_tp = [(7168, 18432), (7168, 16384), (7168, 2048)] + + weight_shapes = [] + for t in total: + weight_shapes.append(t) + for n_t in n_tp: + new_t = (n_t[0] // tp_size, n_t[1]) + weight_shapes.append(new_t) + for k_t in k_tp: + new_t = (k_t[0], k_t[1] // tp_size) + weight_shapes.append(new_t) + + return weight_shapes + + +def create_benchmark_configs(tp_size): + configs = [] + weight_shapes = get_weight_shapes(tp_size) + batch_sizes = [8, 16, 32, 64, 128, 256, 1024, 2048, 4096] + + for n, k in weight_shapes: + for m in batch_sizes: + configs.append((m, n, k, tp_size)) + + return configs + + +def fp8_gemm_flashinfer( + x_fp8: torch.Tensor, + x_scale: torch.Tensor, + y_fp8: torch.Tensor, + y_scale: torch.Tensor, +): + """Flashinfer implementation of FP8 GEMM""" + output = gemm_fp8_nt_groupwise( + x_fp8, + y_fp8, + x_scale, + y_scale, + out_dtype=torch.bfloat16, + backend="trtllm", + ) + return output + + +def fp8_gemm_deepgemm_blackwell( + x_fp8: torch.Tensor, + x_scale: torch.Tensor, + y_fp8: torch.Tensor, + y_scale: torch.Tensor, +): + """DeepGEMM implementation of FP8 GEMM""" + block_size = [BLOCK_SIZE, BLOCK_SIZE] + output = w8a8_block_fp8_matmul_deepgemm( + x_fp8, y_fp8, x_scale, y_scale, block_size, output_dtype=torch.bfloat16 + ) + return output + + +def check_accuracy(a, b, atol, rtol, percent): + """Unified accuracy checking function with detailed error reporting.""" + if not torch.isfinite(a).all(): + print("Non-finite values in reference output") + return False + if not torch.isfinite(b).all(): + print("Non-finite values in actual output") + return False + assert a.shape == b.shape, f"Shape mismatch: {a.shape} vs {b.shape}" + + close = torch.isclose(a, b, atol=atol, rtol=rtol) + match_ratio = close.float().mean() + if match_ratio >= percent: + return True + + mismatch_percent = 1.0 - match_ratio.item() + if mismatch_percent > 1 - percent: + print( + f"Mismatch percentage is {mismatch_percent:.4f} for rtol {rtol} " + f"(threshold: {1 - percent:.4f})" + ) + return False + + +def calculate_diff(m: int, n: int, k: int): + x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) + y = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) + + y_fp8, y_scale = per_block_cast_to_fp8(y) + x_fp8, x_scale = sglang_per_token_group_quant_fp8( + x, BLOCK_SIZE, column_major_scales=True + ) + out_flashinfer = fp8_gemm_flashinfer( + x_fp8, + x_scale, + y_fp8, + y_scale, + ) + + dg_x_fp8, dg_x_scale = sglang_per_token_group_quant_fp8( + x, + BLOCK_SIZE, + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + ) + # We can directly quantize y here, but to mimic the behavior of the actual + # implementations, we requant it here. + dg_y_fp8, dg_y_scale = requant_weight_ue8m0( + y_fp8, y_scale, [BLOCK_SIZE, BLOCK_SIZE] + ) + out_deepgemm = fp8_gemm_deepgemm_blackwell( + dg_x_fp8, dg_x_scale, dg_y_fp8, dg_y_scale + ) + + print(f"Shape m={m}, n={n}, k={k}:") + print(f"Flashinfer output: {out_flashinfer[0, 0:5]}") + print(f"DeepGEMM output: {out_deepgemm[0, 0:5]}") + + flashinfer_deepgemm_match = check_accuracy( + out_flashinfer, out_deepgemm, 0.1, 0.6, 0.95 + ) + print("Correctness check:") + print(f" - Flashinfer vs DeepGEMM: {'✅' if flashinfer_deepgemm_match else '❌'}") + + +def _benchmark(m, n, k, tp_size, provider): + print(f"Shape (m={m}, n={n}, k={k}, tp={tp_size}), Provider: {provider}") + x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) + y = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) + + # Preprocess data before benchmarking + y_fp8, y_scale = per_block_cast_to_fp8(y) + x_fp8, x_scale = sglang_per_token_group_quant_fp8( + x, BLOCK_SIZE, column_major_scales=True + ) + dg_x_fp8, dg_x_scale = sglang_per_token_group_quant_fp8( + x, + BLOCK_SIZE, + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + ) + dg_y_fp8, dg_y_scale = requant_weight_ue8m0( + y_fp8, y_scale, [BLOCK_SIZE, BLOCK_SIZE] + ) + + quantiles = [0.5, 0.2, 0.8] + + if provider == "deepgemm": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: fp8_gemm_deepgemm_blackwell( + dg_x_fp8, + dg_x_scale, + dg_y_fp8, + dg_y_scale, + ), + quantiles=quantiles, + ) + elif provider == "flashinfer": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: fp8_gemm_flashinfer( + x_fp8, + x_scale, + y_fp8, + y_scale, + ), + quantiles=quantiles, + ) + + # Calculate TFLOPS + flops = 2 * m * n * k # multiply-adds + tflops = flops / (ms * 1e-3) / 1e12 + + # Print shape-specific results with TFLOPS + print(f"Time: {ms*1000:.2f} us, TFLOPS: {tflops:.2f}") + return ms, max_ms, min_ms + + +def get_benchmark_plot_friendly(tp_size): + all_configs = create_benchmark_configs(tp_size) + x_vals = list(range(len(all_configs))) + + @triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["cfg_id"], + x_vals=x_vals, + line_arg="provider", + line_vals=["deepgemm", "flashinfer"], + line_names=["DeepGEMM", "Flashinfer"], + styles=[("blue", "-"), ("red", "-")], + ylabel="us", + plot_name=f"fp8-gemm-performance-comparison-tp{tp_size}", + args={}, + ) + ) + def benchmark(cfg_id, provider): + m, n, k, tp_size = all_configs[cfg_id] + ms, min_ms, max_ms = _benchmark(m, n, k, tp_size, provider) + return ms * 1000, max_ms * 1000, min_ms * 1000 # convert to ms + + return benchmark + + +def get_benchmark(tp_size): + all_configs = create_benchmark_configs(tp_size) + + @triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["m", "n", "k", "tp_size"], + x_vals=[list(config) for config in all_configs], + line_arg="provider", + line_vals=["deepgemm", "flashinfer"], + line_names=["DeepGEMM", "Flashinfer"], + styles=[("blue", "-"), ("red", "-")], + ylabel="us", + plot_name=f"fp8-gemm-performance-comparison-tp{tp_size}", + args={}, + ) + ) + def benchmark(m, n, k, tp_size, provider): + ms, min_ms, max_ms = _benchmark(m, n, k, tp_size, provider) + return ms * 1000, max_ms * 1000, min_ms * 1000 # convert to ms + + return benchmark + + +if __name__ == "__main__": + if not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10: + print("Skipping benchmark because the device is not supported") + exit(0) + + parser = argparse.ArgumentParser() + parser.add_argument( + "--save-path", + type=str, + default="./configs/benchmark_ops/fp8_gemm/", + help="Path to save fp8 gemm benchmark results", + ) + parser.add_argument( + "--run-correctness", + action="store_true", + default=True, + help="Whether to run correctness test", + ) + parser.add_argument( + "--tp-size", + type=int, + default=1, + help="Tensor parallelism size to benchmark (default: 1)", + ) + parser.add_argument( + "--plot-friendly", + action="store_true", + default=False, + help="Plot x axis as the config index instead of the m", + ) + args = parser.parse_args() + + # Set random seed for reproducibility + torch.manual_seed(0) + torch.cuda.manual_seed(0) + + # Run correctness tests on a few examples + if args.run_correctness: + print("Running correctness tests...") + calculate_diff(64, 512, 7168) # Small test + calculate_diff(64, 7168, 16384) # Medium test + calculate_diff(64, 18432, 7168) # Large test + + # Get the benchmark function with the specified tp_size + benchmark = ( + get_benchmark_plot_friendly(args.tp_size) + if args.plot_friendly + else get_benchmark(args.tp_size) + ) + + print(f"Running performance benchmark for TP size = {args.tp_size}...") + benchmark.run(print_data=True, save_path=args.save_path) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_group_gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_group_gemm.py new file mode 100644 index 00000000..b2cea070 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_group_gemm.py @@ -0,0 +1,487 @@ +from typing import Tuple + +import deep_gemm +import torch +import triton +import triton.language as tl +from deep_gemm import calc_diff +from deep_gemm.utils.layout import get_mn_major_tma_aligned_tensor + +# Import shared functionality from the regular GEMM benchmark +from sglang.benchmark.kernels.deepseek.benchmark_deepgemm_fp8_gemm import ( + per_block_cast_to_fp8, + per_token_cast_to_fp8, +) + + +def construct_grouped_and_flat_fp8( + x: torch.Tensor, y: torch.Tensor, num_groups: int, is_masked: bool +) -> Tuple[ + Tuple[torch.Tensor, torch.Tensor], # grouped x_fp8 + Tuple[torch.Tensor, torch.Tensor], # grouped y_fp8 + Tuple[torch.Tensor, torch.Tensor], # flat x_fp8 + Tuple[torch.Tensor, torch.Tensor], # flat y_fp8 + torch.Tensor, # output + torch.Tensor, # reference output +]: + # Verify input shapes + m, k = x.shape + n, k_y = y.shape + assert k == k_y, f"Incompatible shapes: x({m}, {k}), y({n}, {k_y})" + assert m % num_groups == 0, f"m({m}) must be divisible by num_groups({num_groups})" + assert m % 4 == 0, f"TMA alignment error: {m}" + + # Reshape inputs for grouped processing + m_per_group = m // num_groups + x_grouped = x.view(num_groups, m_per_group, k) + y_grouped = y.unsqueeze(0).expand(num_groups, n, k) + + # Initialize output tensors + out = torch.empty((num_groups, m_per_group, n), device="cuda", dtype=torch.bfloat16) + ref_out = torch.einsum("gmk,gnk->gmn", x_grouped, y_grouped) + + # Quantize grouped tensors + x_fp8_grouped = ( + torch.empty_like(x_grouped, dtype=torch.float8_e4m3fn), + torch.empty( + (num_groups, m_per_group, k // 128), device="cuda", dtype=torch.float + ), + ) + y_fp8_grouped = ( + torch.empty_like(y_grouped, dtype=torch.float8_e4m3fn), + torch.empty( + (num_groups, (n + 127) // 128, k // 128), device="cuda", dtype=torch.float + ), + ) + for i in range(num_groups): + x_fp8_grouped[0][i], x_fp8_grouped[1][i] = per_token_cast_to_fp8(x_grouped[i]) + y_fp8_grouped[0][i], y_fp8_grouped[1][i] = per_block_cast_to_fp8(y_grouped[i]) + + # Quantize flat tensors + x_fp8_flat = per_token_cast_to_fp8(x) + y_fp8_flat = per_block_cast_to_fp8(y) + + # For non-masked input, merge the group and M dims in output + if not is_masked: + x_fp8_grouped = ( + x_fp8_grouped[0].view(-1, k), + per_token_cast_to_fp8(x_grouped.view(-1, k))[1], + ) + out, ref_out = out.view(-1, n), ref_out.view(-1, n) + + # Transpose earlier for testing + x_fp8_grouped = ( + x_fp8_grouped[0], + get_mn_major_tma_aligned_tensor(x_fp8_grouped[1]), + ) + x_fp8_flat = (x_fp8_flat[0], get_mn_major_tma_aligned_tensor(x_fp8_flat[1])) + + return x_fp8_grouped, y_fp8_grouped, x_fp8_flat, y_fp8_flat, out, ref_out + + +# Since we don't have a group gemm kernel in SGLang/vLLM, we implemented a +# custom kernel based on the Triton tutorial. +# https://triton-lang.org/main/getting-started/tutorials/03-matrix-multiplication.html +@triton.jit +def fp8_gemm_group_triton_kernel( + # Pointers to matrices + a_ptr, + b_ptr, + c_ptr, + # Pointers to scaling factors + a_scale_ptr, + b_scale_ptr, + # Matrix dimensions + M, + N, + K, + # The stride variables represent how much to increase the ptr by when moving by 1 + # element in a particular dimension. + stride_am, + stride_ak, + stride_bk, + stride_bn, + stride_cm, + stride_cn, + # Strides for scaling factors + stride_a_scale_m, + stride_a_scale_k, + stride_b_scale_n, + stride_b_scale_k, + # Meta-parameters + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + GROUP_SIZE_M: tl.constexpr, +): + """Kernel for computing the matmul C = A x B with FP8 inputs and scaling factors. + A has shape (M, K), B has shape (K, N) and C has shape (M, N) + + Note: Block sizes must be multiples of 32 for optimal TMA performance. + """ + # Map program ids to the block of C it should compute + pid_group = tl.program_id(axis=0) # Group ID + pid_n = tl.program_id(axis=1) # N dimension ID + + # Compute the M block ID within this group + group_size_m = min(M - pid_group * GROUP_SIZE_M, GROUP_SIZE_M) + pid_m_within_group = tl.program_id(axis=2) % group_size_m + pid_m = pid_group * GROUP_SIZE_M + pid_m_within_group + + # Create pointers for the first blocks of A and B + offs_am = (pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)) % M + offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N + offs_k = tl.arange(0, BLOCK_SIZE_K) + a_ptrs = a_ptr + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak) + b_ptrs = b_ptr + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) + + # Initialize accumulator + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + + # Main loop + for k_block in range(0, tl.cdiv(K, BLOCK_SIZE_K)): + k_offset = k_block * BLOCK_SIZE_K + + # Load the next block of A and B, with masks + a = tl.load(a_ptrs, mask=offs_k[None, :] < K - k_offset, other=0.0) + b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k_offset, other=0.0) + + # Calculate indices for scaling factors for this K block + a_scale_ptrs = a_scale_ptr + ( + offs_am * stride_a_scale_m + k_block * stride_a_scale_k + ) + b_scale_ptrs = b_scale_ptr + ( + pid_n * stride_b_scale_n + k_block * stride_b_scale_k + ) + + # Perform matrix multiplication in FP8 + res = tl.dot(a, b) + + # Load scaling factors for the current block + a_scale = tl.load(a_scale_ptrs)[:, None] # [BLOCK_SIZE_M, 1] + b_scale = tl.load(b_scale_ptrs) + + # Apply scaling factors to the accumulated result + accumulator += res * a_scale * b_scale + + # Advance pointers + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + + # Convert to bfloat16 for output + c = accumulator.to(tl.bfloat16) + + # Write back the result + offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + c_ptrs = c_ptr + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] + c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) + tl.store(c_ptrs, c, mask=c_mask) + + +def fp8_gemm_group_triton(a_tuple, b_tuple, c, num_groups): + """ + Perform matrix multiplication with FP8 inputs and proper scaling. + + Args: + a_tuple: Tuple of (quantized_tensor, scale_factors) for input A + b_tuple: Tuple of (quantized_tensor, scale_factors) for input B + c: Output tensor in BF16 format + num_groups: Number of groups for grouped GEMM + + Returns: + Result tensor in BF16 format + """ + # Unpack the tuples + a, a_scale = a_tuple + b, b_scale = b_tuple + + M, K = a.shape + _, N = b.shape + + # Configure block sizes - must be multiples of 32 for TMA alignment + BLOCK_SIZE_M = 128 + BLOCK_SIZE_N = 128 + BLOCK_SIZE_K = 128 + + # Calculate grid dimensions + num_pid_m = triton.cdiv(M, BLOCK_SIZE_M) + num_pid_n = triton.cdiv(N, BLOCK_SIZE_N) + num_groups_grid = triton.cdiv(num_pid_m, num_groups) + + # 3D grid launch - (group, n_blocks, m_blocks_per_group) + grid = (num_groups_grid, num_pid_n, min(num_groups, num_pid_m)) + + fp8_gemm_group_triton_kernel[grid]( + a, + b, + c, + a_scale, + b_scale, + M, + N, + K, + a.stride(0), + a.stride(1), + b.stride(0), + b.stride(1), + c.stride(0), + c.stride(1), + a_scale.stride(0), + 1, # Stride in the K dimension may be 1 + b_scale.stride(0), + 1 if b_scale.dim() > 1 else 0, + BLOCK_SIZE_M=BLOCK_SIZE_M, + BLOCK_SIZE_N=BLOCK_SIZE_N, + BLOCK_SIZE_K=BLOCK_SIZE_K, + GROUP_SIZE_M=num_groups, + ) + + return c + + +def fp8_gemm_group_deepgemm(x_fp8_grouped, y_fp8_grouped, out, m_indices): + deep_gemm.m_grouped_fp8_gemm_nt_contiguous( + x_fp8_grouped, + y_fp8_grouped, + out, + m_indices, + ) + return out + + +def calculate_diff(m: int, n: int, k: int, num_groups: int): + print(f"Shape (m={m}, n={n}, k={k}") + x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) + y = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) + x_fp8_grouped, y_fp8_grouped, x_fp8_flat, y_fp8_flat, out, out_torch = ( + construct_grouped_and_flat_fp8(x, y, num_groups, is_masked=False) + ) + m_per_group = m // num_groups + out_deepgemm = out.clone() + m_indices = torch.arange(0, num_groups, device="cuda", dtype=torch.int) + m_indices = ( + m_indices.unsqueeze(-1).expand(num_groups, m_per_group).contiguous().view(-1) + ) + + fp8_gemm_group_deepgemm( + x_fp8_grouped, + y_fp8_grouped, + out_deepgemm, + m_indices, + ) + torch.cuda.synchronize() + + # Prepare inputs for Triton + a, a_scale = x_fp8_flat + b, b_scale = y_fp8_flat + b = b.T.contiguous() + # Ensure scales are in the right format and contiguous + a_scale, b_scale = a_scale.contiguous(), b_scale.contiguous() + M, _ = a.shape + _, N = b.shape + c = torch.empty((M, N), device=a.device, dtype=torch.bfloat16) + out_triton = fp8_gemm_group_triton((a, a_scale), (b, b_scale), c, num_groups) + torch.cuda.synchronize() + + diff_torch_deepgemm = torch.abs(out_torch - out_deepgemm).mean().item() + diff_torch_triton = torch.abs(out_torch - out_triton).mean().item() + diff_deepgemm_triton = torch.abs(out_deepgemm - out_triton).mean().item() + + print(f"Shape m={m}, n={n}, k={k}:") + print(f"Torch output: {out_torch[0, 0:5]}") + print(f"DeepGEMM output: {out_deepgemm[0, 0:5]}") + print(f"Triton output: {out_triton[0, 0:5]}") + print(f"Mean absolute difference (Torch-DeepGEMM): {diff_torch_deepgemm}") + print(f"Mean absolute difference (Torch-Triton): {diff_torch_triton}") + print(f"Mean absolute difference (DeepGEMM-Triton): {diff_deepgemm_triton}") + + deepgemm_torch_diff = calc_diff(out_deepgemm, out_torch) + triton_torch_diff = calc_diff(out_triton, out_torch) + deepgemm_triton_diff = calc_diff(out_deepgemm, out_triton) + + DIFF_THRESHOLD = 0.001 + all_match = ( + deepgemm_torch_diff < DIFF_THRESHOLD + and triton_torch_diff < DIFF_THRESHOLD + and deepgemm_triton_diff < DIFF_THRESHOLD + ) + if all_match: + print("✅ All implementations match\n") + else: + print("❌ Some implementations differ:") + print( + f" - Torch vs DeepGEMM: {'✅' if deepgemm_torch_diff < DIFF_THRESHOLD else '❌'}" + f" - Torch vs Triton: {'✅' if triton_torch_diff < DIFF_THRESHOLD else '❌'}" + f" - DeepGEMM vs Triton: {'✅' if deepgemm_triton_diff < DIFF_THRESHOLD else '❌'}" + ) + + +def get_weight_shapes(tp_size): + # cannot TP + total = [ + (512 + 64, 7168), + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (7168, 16384), + (7168, 18432), + ] + # N can TP + n_tp = [ + (18432 * 2, 7168), + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (24576, 1536), + (4096, 7168), + ] + # K can TP + k_tp = [(7168, 18432), (7168, 16384), (7168, 2048)] + + weight_shapes = [] + for t in total: + weight_shapes.append(t) + for n_t in n_tp: + new_t = (n_t[0] // tp_size, n_t[1]) + weight_shapes.append(new_t) + for k_t in k_tp: + new_t = (k_t[0], k_t[1] // tp_size) + weight_shapes.append(new_t) + + return weight_shapes + + +def create_benchmark_configs(tp_size): + configs = [] + weight_shapes = get_weight_shapes(tp_size) + batch_sizes = [2048, 4096] + group_sizes = [4, 8] + for n, k in weight_shapes: + for m in batch_sizes: + for num_groups in group_sizes: + configs.append((m, n, k, num_groups, tp_size)) + + return configs + + +def get_benchmark(tp_size): + all_configs = create_benchmark_configs(tp_size) + + @triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["m", "n", "k", "num_groups", "tp_size"], + x_vals=[config for config in all_configs], + line_arg="provider", + line_vals=["deepgemm", "triton"], + line_names=["DeepGEMM", "Triton"], + styles=[("blue", "-"), ("red", "-")], + ylabel="ms", + plot_name=f"fp8-group-gemm-performance-comparison-tp{tp_size}", + args={}, + ) + ) + def benchmark(m, n, k, num_groups, tp_size, provider): + print( + f"Shape (m={m}, n={n}, k={k}, tp={tp_size}, num_groups={num_groups}, Provider: {provider}" + ) + x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) + y = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) + x_fp8_grouped, y_fp8_grouped, x_fp8_flat, y_fp8_flat, out, out_torch = ( + construct_grouped_and_flat_fp8(x, y, num_groups, is_masked=False) + ) + m_per_group = m // num_groups + m_indices = torch.arange(0, num_groups, device="cuda", dtype=torch.int) + m_indices = ( + m_indices.unsqueeze(-1) + .expand(num_groups, m_per_group) + .contiguous() + .view(-1) + ) + + quantiles = [0.5, 0.2, 0.8] + + if provider == "deepgemm": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: fp8_gemm_group_deepgemm( + x_fp8_grouped, + y_fp8_grouped, + out, + m_indices, + ), + quantiles=quantiles, + ) + elif provider == "triton": + # Prepare inputs for Triton + # We did it outside of the lambda function to make it fair comparison like deepgemm + a, a_scale = x_fp8_flat + b, b_scale = y_fp8_flat + b = b.T.contiguous() + # Ensure scales are in the right format and contiguous + a_scale, b_scale = a_scale.contiguous(), b_scale.contiguous() + M, _ = a.shape + _, N = b.shape + c = torch.empty((M, N), device=a.device, dtype=torch.bfloat16) + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: fp8_gemm_group_triton( + (a, a_scale), + (b, b_scale), + c, + num_groups, + ), + quantiles=quantiles, + ) + + # Calculate TFLOPS + flops = 2 * m * n * k # multiply-adds + tflops = flops / (ms * 1e-3) / 1e12 + + print(f"Time: {ms*1000:.2f} ms, TFLOPS: {tflops:.2f}") + return ms * 1000, max_ms * 1000, min_ms * 1000 # convert to ms + + return benchmark + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", + type=str, + default="./configs/benchmark_ops/fp8_group_gemm/", + help="Path to save deepgemm fp8 group gemm benchmark results", + ) + parser.add_argument( + "--run_correctness", + action="store_true", + help="Whether to run correctness test", + ) + parser.add_argument( + "--tp_size", + type=int, + default=1, + help="Tensor parallelism size to benchmark (default: 1)", + ) + args = parser.parse_args() + + # Set random seed for reproducibility + torch.manual_seed(0) + torch.cuda.manual_seed(0) + + # Enable TF32, adapted from https://github.com/deepseek-ai/DeepGEMM/blob/main/tests/test_core.py#L148 + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.allow_tf32 = True + + # Run correctness tests on a few examples + if args.run_correctness: + print("Running correctness tests...") + calculate_diff(8192, 7168, 4096, 4) + calculate_diff(8192, 2048, 7168, 4) + calculate_diff(4096, 7168, 4096, 8) + calculate_diff(4096, 2048, 7168, 8) + calculate_diff(4096, 576, 7168, 8) + + # Get the benchmark function with the specified tp_size + benchmark = get_benchmark(args.tp_size) + + print(f"Running performance benchmark for TP size = {args.tp_size}...") + benchmark.run(print_data=True, save_path=args.save_path) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/elementwise/benchmark_concat_mla.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/elementwise/benchmark_concat_mla.py new file mode 100644 index 00000000..c4d7bb1c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/elementwise/benchmark_concat_mla.py @@ -0,0 +1,198 @@ +import torch +import triton +import triton.language as tl +from sgl_kernel import concat_mla_k as concat_mla_k_cuda + +DEVICE = triton.runtime.driver.active.get_active_torch_device() + +num_local_heads = 128 +qk_nope_head_dim = 128 +qk_rope_head_dim = 64 + + +def create_data(num_tokens): + k_nope_container = torch.randn( + (num_tokens, num_local_heads, qk_nope_head_dim + 128), + dtype=torch.bfloat16, + device="cuda", + ) + k_nope = k_nope_container[:, :, :qk_nope_head_dim] + + k_rope_container = torch.randn( + (num_tokens, 1, 128 + qk_rope_head_dim), dtype=torch.bfloat16, device="cuda" + ) + k_rope = k_rope_container[:, :, -qk_rope_head_dim:] + + k = torch.empty( + (num_tokens, num_local_heads, qk_nope_head_dim + qk_rope_head_dim), + dtype=torch.bfloat16, + device="cuda", + ) + return dict(k=k, k_nope=k_nope, k_rope=k_rope) + + +def fn_torch(k, k_nope, k_rope): + k[..., :qk_nope_head_dim] = k_nope + k[..., qk_nope_head_dim:] = k_rope + + +def fn_hack_non_strided(k, k_nope, k_rope): + k_flatten_view = k.flatten() + k_flatten_view[: k_nope.numel()] = k_nope.flatten() + + k2 = k_flatten_view[k_nope.numel() :].view(k_rope.numel(), -1) + k2 = k_rope.flatten()[:, None] + + +@torch.compile(dynamic=True) +def fn_torch_compiled(k, k_nope, k_rope): + return fn_torch(k, k_nope, k_rope) + + +def fn_cuda(k, k_nope, k_rope): + concat_mla_k_cuda(k, k_nope, k_rope) + + +@triton.jit +def fn_triton_kernel( + k_ptr, + k_nope_ptr, + k_rope_ptr, + num_tokens, + QK_NOPE_HEAD_DIM: tl.constexpr, + QK_ROPE_HEAD_DIM: tl.constexpr, + NUM_LOCAL_HEADS: tl.constexpr, + K_NOPE_STRIDE_0: tl.constexpr, + K_NOPE_STRIDE_1: tl.constexpr, + K_STRIDE_0: tl.constexpr, + K_STRIDE_1: tl.constexpr, + K_ROPE_STRIDE_0: tl.constexpr, + BLOCK_ROWS: tl.constexpr, +): + pid = tl.program_id(axis=0) + + token_id = pid * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS) + token_mask = token_id < num_tokens + + head_id = tl.arange(0, NUM_LOCAL_HEADS) + + # nope + nope_sub_id = tl.arange(0, QK_NOPE_HEAD_DIM) + offs_nope = ( + token_id[:, None, None] * K_NOPE_STRIDE_0 + + head_id[None, :, None] * K_NOPE_STRIDE_1 + + nope_sub_id[None, None, :] + ) + offs_k = ( + token_id[:, None, None] * K_STRIDE_0 + + head_id[None, :, None] * K_STRIDE_1 + + nope_sub_id[None, None, :] + ) + vals_nope = tl.load(k_nope_ptr + offs_nope, mask=token_mask[:, None, None]) + tl.store(k_ptr + offs_k, vals_nope, mask=token_mask[:, None, None]) + + # rope + rope_sub_id = tl.arange(0, QK_ROPE_HEAD_DIM) + offs_rope = token_id[:, None, None] * K_ROPE_STRIDE_0 + rope_sub_id[None, None, :] + offs_k = ( + token_id[:, None, None] * K_STRIDE_0 + + head_id[None, :, None] * K_STRIDE_1 + + rope_sub_id[None, None, :] + + QK_NOPE_HEAD_DIM + ) + vals_rope = tl.load(k_rope_ptr + offs_rope, mask=token_mask[:, None, None]) + tl.store(k_ptr + offs_k, vals_rope, mask=token_mask[:, None, None]) + + +def fn_triton(k, k_nope, k_rope): + assert k.device == DEVICE and k_nope.device == DEVICE and k_rope.device == DEVICE + num_tokens, _, _ = k.shape + grid = lambda meta: (triton.cdiv(num_tokens, meta["BLOCK_ROWS"]),) + fn_triton_kernel[grid]( + k, + k_nope, + k_rope, + num_tokens, + QK_NOPE_HEAD_DIM=qk_nope_head_dim, + QK_ROPE_HEAD_DIM=qk_rope_head_dim, + NUM_LOCAL_HEADS=num_local_heads, + K_NOPE_STRIDE_0=k_nope.stride(0), + K_NOPE_STRIDE_1=k_nope.stride(1), + K_STRIDE_0=k.stride(0), + K_STRIDE_1=k.stride(1), + K_ROPE_STRIDE_0=k_rope.stride(0), + BLOCK_ROWS=16, + ) + + +def execute_and_get_output(f, data): + data["k"].zero_() + f(**data) + assert data["k"].sum().item() != 0 + return data["k"].clone() + + +torch.manual_seed(0) +data = create_data(num_tokens=32768) +output_ref = execute_and_get_output(fn_torch, data) +output_exp = execute_and_get_output(fn_cuda, data) +# print(output_ref) +# print(output_exp) +if not torch.all(output_ref == output_exp): + abs_delta = torch.abs(output_ref - output_exp) + raise AssertionError( + f"{output_ref=} {output_exp=} " + f"{abs_delta=} " + f"{torch.argwhere(abs_delta != 0.0)=} " + ) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["num_tokens"], # Argument names to use as an x-axis for the plot. + x_vals=[ + 2048, + 4096, + 8192, + 16384, + 32768, + ], # Different possible values for `x_name`. + x_log=False, # x axis is logarithmic. + line_arg="provider", # Argument name whose value corresponds to a different line in the plot. + line_vals=[ + "torch", + "torch_compiled", + "triton", + "hack_non_strided", + "cuda", + ], # Possible values for `line_arg`. + line_names=[ + "torch", + "torch_compiled", + "triton", + "hack_non_strided", + "cuda", + ], # Label name for the lines. + plot_name="vector-add-performance", # Name for the plot. Used also as a file name for saving the plot. + args={}, # Values for function arguments not in `x_names` and `y_name`. + ) +) +def benchmark(num_tokens, provider): + data = create_data(num_tokens=num_tokens) + quantiles = [0.5, 0.2, 0.8] + fn = { + "torch": fn_torch, + "torch_compiled": fn_torch_compiled, + "triton": fn_triton, + "hack_non_strided": fn_hack_non_strided, + "cuda": fn_cuda, + }[provider] + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: fn(**data), quantiles=quantiles + ) + return ms, min_ms, max_ms + + +torch.cuda.cudart().cudaProfilerStart() +benchmark.run(print_data=True, show_plots=True) +torch.cuda.cudart().cudaProfilerStop() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/flashinfer_allreduce_fusion/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/flashinfer_allreduce_fusion/README.md new file mode 100644 index 00000000..e651604c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/flashinfer_allreduce_fusion/README.md @@ -0,0 +1,102 @@ +# FlashInfer Fused AllReduce + RMSNorm Benchmark + +This benchmark script is modified from the [original implementation](https://github.com/vllm-project/vllm/blob/237e1fb887c7f5a579420fa0295097f24b006594/benchmarks/kernels/benchmark_fused_collective.py) by the vLLM community. It aims to compare the performance differences between FlashInfer fused operators in SGLang (trtllm_allreduce_fusion: AllReduce + Residual Add + RMSNorm + optional quantization) and conventional implementations (standard `tensor_model_parallel_all_reduce` + separate RMSNorm/quantization). Specifically, this script tests the timing performance of two implementation paths: 1) Standard AllReduce and RMSNorm executed separately; 2) FlashInfer's fused operator combining AllReduce, Residual Add, RMSNorm, and optional quantization operations. + +This benchmark script helps us tune the ipc workspace size of the `flashinfer_allreduce_residual_rmsnorm` operator in SGLang and prepare for applications with FP8/FP4 quantized fused operators. + +Script path: `benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py` + +## Feature Overview + +- Compare average execution time (ms) and calculate speedup ratios for the following paths: + - standard_allreduce_rmsnorm (Standard AllReduce + RMSNorm) + - flashinfer_fused_allreduce_rmsnorm (Fused AllReduce + RMSNorm), including oneshot and twoshot modes + - Optionally compare FP8/FP4 quantized fused paths with standard paths +- Use CUDA Graph capture and batch replay to reduce measurement noise +- Automatically select the faster "standard baseline" (native/compiled version) as the denominator for speedup calculation +- Optionally export results in Markdown format + +## Runtime Environment and Prerequisites + +- At least 2 GPUs, and launch multi-process distributed training using `torchrun` (NCCL backend) +- Properly install/compile sglang along with sgl-kernel and custom operators + +## Quick Start (Command Examples) + +The following examples use world_size=2. You can modify `--nproc_per_node` and parameters according to your machine: + +- Regular paths only (no quantization): +``` +torchrun --nproc_per_node=2 \ +benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py \ +--no-quant --hidden-dim 1024 --seq-lens 512 1024 2048 4096 --trials 100 +``` + +- FP8 quantization paths only: +``` +torchrun --nproc_per_node=2 \ +benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py \ +--quant-fp8 --hidden-dim 1024 --seq-lens 512 1024 2048 4096 --trials 100 +``` + +- FP4 quantization paths only: +``` +torchrun --nproc_per_node=2 \ +benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py \ +--quant-fp4 --hidden-dim 1024 --seq-lens 512 1024 2048 4096 --trials 100 +``` + +- Larger hidden dimensions: +``` +torchrun --nproc_per_node=2 \ +benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py \ +--no-quant --hidden-dim 4096 --seq-lens 512 1024 2048 4096 --trials 100 +``` + +## Parameter Description +- `--seq-lens`: List of sequence lengths to test (default: 128 512 1024 2048) +- `--hidden-dim`: Hidden dimension (default: 8192) +- `--dtypes`: Data type list, `float16|bfloat16|float32` (default: bfloat16) +- `--no-residual`: Only test "no residual" scenarios (default tests both "with/without residual") +- Mutually exclusive quantization options: + - `--no-quant`: No quantization testing + - `--quant-fp8`: Only FP8 quantization testing + - `--quant-fp4`: Only FP4 quantization testing + - `--quant-all`: Test all (default) +- FlashInfer related: + - `--disable-oneshot`: Disable oneshot mode (default enables oneshot and tests twoshot simultaneously) +- Runtime configuration: + - `--warmup`: Warmup count before graph capture and before graph replay (default 5) + - `--trials`: Benchmark iteration count (default 20; internally each `graph.replay()` will batch replay multiple times) + - `--output-file`: Save results as Markdown file (only rank0 takes effect) + +## Output Example + +Each configuration group prints a table showing average execution time and relative speedup ratios (baseline is the faster standard implementation). For example: +``` +================================================================================ +Results: seq_len=1024, hidden_dim=1024 +dtype=torch.bfloat16, residual=yes, quant_mode=none +================================================================================ +Operation Time (ms) Speedup +-------------------------------------------------------------------------------- +standard_allreduce_rmsnorm 0.024 0.98x +standard_allreduce_rmsnorm_native_compiled 0.023 baseline +flashinfer_fused_allreduce_rmsnorm_oneshot 0.011 2.19x +flashinfer_fused_allreduce_rmsnorm_twoshot 0.041 0.57x +``` + +If `--output-file` is specified, all configurations will be summarized in Markdown tables in that file. + +## Important Notes and Recommendations + +- Distributed: The script uses `torchrun` environment variables to initialize distributed training and binds tensors/communication groups to the current rank's corresponding device. +- World size: Requires `WORLD_SIZE > 1` to perform communication operator benchmarks. Otherwise, the script will error and prompt. +- FlashInfer: + - If not installed or interfaces are missing, the script will only run standard paths and provide prompts in the logs. + - The fused operator internally uses "oneshot"/"twoshot" two trigger methods; oneshot is enabled by default and twoshot is tested simultaneously. +- FP8/FP4: + - FP8 uses sglang's FP8 tools and dtype, with underlying platform selection of `e4m3`/`e4m3fnuz` etc. + - FP4 uses sgl-kernel's `scaled_fp4_quant`, requiring corresponding platform support. +- CUDA Graph: + - Uses sglang's `graph_capture()` to prepare capture-ready state for communication, then uses `torch.cuda.graph` to capture kernels, reducing measurement jitter. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py new file mode 100644 index 00000000..4aebf62b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py @@ -0,0 +1,1304 @@ +# Modified from https://github.com/vllm-project/vllm/blob/237e1fb887c7f5a579420fa0295097f24b006594/benchmarks/kernels/benchmark_fused_collective.py + +""" +Benchmark for FlashInfer fused collective operations vs standard operations. + +This benchmark compares: +1. FlashInfer's trtllm_allreduce_fusion (fused allreduce + rmsnorm + optional quant) +2. Standard tensor_model_parallel_all_reduce + separate rmsnorm/quant operations + +Usage with torchrun: + torchrun --nproc_per_node=2 benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py --no-quant --hidden-dim 1024 --seq-len 512 1024 2048 4096 --trials 100 + torchrun --nproc_per_node=2 benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py --quant-fp8 --hidden-dim 1024 --seq-len 512 1024 2048 4096 --trials 100 + torchrun --nproc_per_node=2 benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py --quant-fp4 --hidden-dim 1024 --seq-len 512 1024 2048 4096 --trials 100 + + torchrun --nproc_per_node=2 benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py --no-quant --hidden-dim 4096 --seq-len 512 1024 2048 4096 --trials 100 + torchrun --nproc_per_node=2 benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py --quant-fp8 --hidden-dim 4096 --seq-len 512 1024 2048 4096 --trials 100 + torchrun --nproc_per_node=2 benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py --quant-fp4 --hidden-dim 4096 --seq-len 512 1024 2048 4096 --trials 100 +""" + +import argparse +import contextlib +import itertools +import logging +import os +import time +from typing import Optional + +import torch # type: ignore +import torch.distributed as dist # type: ignore + +from sglang.srt.distributed import get_tp_group, tensor_model_parallel_all_reduce +from sglang.srt.distributed.parallel_state import ( + cleanup_dist_env_and_memory, + graph_capture, + init_distributed_environment, + initialize_model_parallel, +) +from sglang.srt.layers.layernorm import RMSNorm # noqa +from sglang.srt.layers.quantization.fp8_kernel import fp8_dtype as SGLANG_FP8_DTYPE +from sglang.srt.layers.quantization.fp8_kernel import static_quant_fp8 + +try: + from sgl_kernel import fused_add_rmsnorm as SGL_FUSED_ADD_RMS_NORM + from sgl_kernel import rmsnorm as SGL_RMS_NORM + from sgl_kernel import scaled_fp4_quant as SGL_SCALED_FP4_QUANT +except Exception: # pragma: no cover - fallback on non-supported platforms + SGL_FUSED_ADD_RMS_NORM = None + SGL_RMS_NORM = None + SGL_SCALED_FP4_QUANT = None + +FP8_DTYPE = SGLANG_FP8_DTYPE + +logger = logging.getLogger(__name__) + +# Try to import FlashInfer +try: + import flashinfer.comm as flashinfer_comm # type: ignore + + if not hasattr(flashinfer_comm, "trtllm_allreduce_fusion"): + flashinfer_comm = None + logger.warning( + "FlashInfer comm module found but missing trtllm_allreduce_fusion" + ) +except ImportError: + flashinfer_comm = None + logger.warning("FlashInfer not found, only benchmarking standard operations") + +# Constants +MiB = 1024 * 1024 + +# FlashInfer max sizes per world size +# Enable 64MB for 2, 4, 8 world sizes to verify large input sizes +# use --disable-oneshot to disable oneshot mode for very large input sizes +_FI_MAX_SIZES = { + 2: 64 * MiB, # 64MB + 4: 64 * MiB, # 64MB + 8: 64 * MiB, # 64MB +} + +# Global workspace tensor for FlashInfer +_FI_WORKSPACE_TENSOR = None + + +def setup_flashinfer_workspace( + world_size: int, + rank: int, + hidden_dim: int, + max_token_num: int, + use_fp32_lamport: bool = False, +): + """Setup FlashInfer workspace for fused allreduce operations.""" + global _FI_WORKSPACE_TENSOR + + if flashinfer_comm is None: + return None, None + + if world_size not in _FI_MAX_SIZES: + logger.warning("FlashInfer not supported for world size %s", world_size) + return None, None + + try: + # Create IPC workspace + ipc_handles, workspace_tensor = ( + flashinfer_comm.trtllm_create_ipc_workspace_for_all_reduce_fusion( + tp_rank=rank, + tp_size=world_size, + max_token_num=max_token_num, + hidden_dim=hidden_dim, + group=get_tp_group().device_group, + use_fp32_lamport=use_fp32_lamport, + ) + ) + + _FI_WORKSPACE_TENSOR = workspace_tensor + return ipc_handles, workspace_tensor + except Exception as e: + logger.error("Failed to setup FlashInfer workspace: %s", e) + return None, None + + +def cleanup_flashinfer_workspace(ipc_handles): + """Cleanup FlashInfer workspace.""" + if flashinfer_comm is None or ipc_handles is None: + return + + try: + group = get_tp_group().device_group + flashinfer_comm.trtllm_destroy_ipc_workspace_for_all_reduce(ipc_handles, group) + except Exception as e: + logger.error("Failed to cleanup FlashInfer workspace: %s", e) + + +class FlashInferFusedAllReduceParams: + """Parameters for FlashInfer fused allreduce operations.""" + + def __init__( + self, + rank: int, + world_size: int, + use_fp32_lamport: bool = False, + max_token_num: int = 1024, + ): + self.rank = rank + self.world_size = world_size + self.use_fp32_lamport = use_fp32_lamport + self.trigger_completion_at_end = True + self.launch_with_pdl = True + self.fp32_acc = True + self.max_token_num = max_token_num + + def get_trtllm_fused_allreduce_kwargs(self): + return { + "world_rank": self.rank, + "world_size": self.world_size, + "launch_with_pdl": self.launch_with_pdl, + "trigger_completion_at_end": self.trigger_completion_at_end, + "fp32_acc": self.fp32_acc, + } + + +def flashinfer_fused_allreduce_rmsnorm( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rms_gamma: torch.Tensor, + rms_eps: float, + allreduce_params: "FlashInferFusedAllReduceParams", + use_oneshot: bool, + norm_out: Optional[torch.Tensor] = None, +): + """FlashInfer fused allreduce + rmsnorm operation.""" + if flashinfer_comm is None or _FI_WORKSPACE_TENSOR is None: + raise RuntimeError("FlashInfer not available or workspace not initialized") + + if norm_out is None: + norm_out = input_tensor + residual_out = residual + else: + residual_out = input_tensor + + flashinfer_comm.trtllm_allreduce_fusion( + allreduce_in=input_tensor, + token_num=input_tensor.shape[0], + residual_in=residual, + residual_out=residual_out, + norm_out=norm_out, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + hidden_dim=input_tensor.shape[-1], + workspace_ptrs=_FI_WORKSPACE_TENSOR, + pattern_code=flashinfer_comm.AllReduceFusionPattern.kARResidualRMSNorm, + allreduce_out=None, + quant_out=None, + scale_out=None, + layout_code=None, + scale_factor=None, + use_oneshot=use_oneshot, + **allreduce_params.get_trtllm_fused_allreduce_kwargs(), + ) + + +def flashinfer_fused_allreduce_rmsnorm_fp8_quant( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rms_gamma: torch.Tensor, + rms_eps: float, + scale_factor: torch.Tensor, + allreduce_params: FlashInferFusedAllReduceParams, + use_oneshot: bool = True, + norm_out: Optional[torch.Tensor] = None, + quant_out: Optional[torch.Tensor] = None, +): + """FlashInfer fused allreduce + rmsnorm + FP8 quantization.""" + if flashinfer_comm is None or _FI_WORKSPACE_TENSOR is None: + raise RuntimeError("FlashInfer not available or workspace not initialized") + + if norm_out is None: + norm_out = input_tensor + residual_out = residual + else: + residual_out = input_tensor + + flashinfer_comm.trtllm_allreduce_fusion( + allreduce_in=input_tensor, + token_num=input_tensor.shape[0], + residual_in=residual, + residual_out=residual_out, + norm_out=norm_out, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + hidden_dim=input_tensor.shape[-1], + workspace_ptrs=_FI_WORKSPACE_TENSOR, + pattern_code=flashinfer_comm.AllReduceFusionPattern.kARResidualRMSNormFP8Quant, + allreduce_out=None, + quant_out=quant_out, + scale_out=None, + layout_code=None, + scale_factor=scale_factor, + use_oneshot=use_oneshot, + **allreduce_params.get_trtllm_fused_allreduce_kwargs(), + ) + + +def flashinfer_fused_allreduce_rmsnorm_fp4_quant( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rms_gamma: torch.Tensor, + rms_eps: float, + input_global_scale: torch.Tensor, + allreduce_params: FlashInferFusedAllReduceParams, + quant_out: torch.Tensor, + use_oneshot: bool, + output_scale: torch.Tensor, + norm_out: Optional[torch.Tensor] = None, +): + """FlashInfer fused allreduce + rmsnorm + FP4 quantization.""" + if flashinfer_comm is None or _FI_WORKSPACE_TENSOR is None: + raise RuntimeError("FlashInfer not available or workspace not initialized") + + if norm_out is None: + norm_out = input_tensor + residual_out = residual + else: + residual_out = input_tensor + + flashinfer_comm.trtllm_allreduce_fusion( + allreduce_in=input_tensor, + token_num=input_tensor.shape[0], + residual_in=residual, + residual_out=residual_out, + norm_out=norm_out, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + hidden_dim=input_tensor.shape[-1], + workspace_ptrs=_FI_WORKSPACE_TENSOR, + pattern_code=flashinfer_comm.AllReduceFusionPattern.kARResidualRMSNormFP4Quant, + allreduce_out=None, + quant_out=quant_out, + scale_out=output_scale, + layout_code=None, + scale_factor=input_global_scale, + use_oneshot=use_oneshot, + **allreduce_params.get_trtllm_fused_allreduce_kwargs(), + ) + + +def standard_allreduce_rmsnorm( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rms_gamma: torch.Tensor, + rms_eps: float, + norm_out: Optional[torch.Tensor] = None, +): + """Standard allreduce + rmsnorm operations.""" + # All-reduce first + allreduce_out = tensor_model_parallel_all_reduce(input_tensor) + # Then RMS norm + if residual is not None: + # Fused add + RMS norm (in-place on allreduce_out) + if SGL_FUSED_ADD_RMS_NORM is not None: + SGL_FUSED_ADD_RMS_NORM(allreduce_out, residual, rms_gamma, rms_eps) + else: + rms = RMSNorm(allreduce_out.shape[-1], eps=rms_eps) + rms.weight.data = rms_gamma + rms.forward_native(allreduce_out, residual) + else: + # Just RMS norm + if SGL_RMS_NORM is not None: + _ = SGL_RMS_NORM(allreduce_out, rms_gamma, rms_eps) + else: + rms = RMSNorm(allreduce_out.shape[-1], eps=rms_eps) + rms.weight.data = rms_gamma + _ = rms.forward_native(allreduce_out) + + +def standard_allreduce_rmsnorm_fp8_quant( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rms_gamma: torch.Tensor, + rms_eps: float, + scale_factor: torch.Tensor, + norm_out: Optional[torch.Tensor] = None, + quant_out: Optional[torch.Tensor] = None, +): + """Standard allreduce + rmsnorm + FP8 quantization.""" + # All-reduce first + allreduce_out = tensor_model_parallel_all_reduce(input_tensor) + + # Then RMS norm + static FP8 quantization + if residual is not None: + if SGL_FUSED_ADD_RMS_NORM is not None: + SGL_FUSED_ADD_RMS_NORM(allreduce_out, residual, rms_gamma, rms_eps) + quant_out, _ = static_quant_fp8( + allreduce_out, scale_factor, repeat_scale=False + ) + else: + rms = RMSNorm(allreduce_out.shape[-1], eps=rms_eps) + rms.weight.data = rms_gamma + normed, _ = rms.forward_native(allreduce_out, residual) + quant_out, _ = static_quant_fp8(normed, scale_factor, repeat_scale=False) + return quant_out, residual + else: + if SGL_RMS_NORM is not None: + normed = SGL_RMS_NORM(allreduce_out, rms_gamma, rms_eps) + else: + rms = RMSNorm(allreduce_out.shape[-1], eps=rms_eps) + rms.weight.data = rms_gamma + normed = rms.forward_native(allreduce_out) + quant_out, _ = static_quant_fp8(normed, scale_factor, repeat_scale=False) + return quant_out + + +def standard_allreduce_rmsnorm_fp4_quant( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rms_gamma: torch.Tensor, + rms_eps: float, + input_global_scale: torch.Tensor, + quant_out: torch.Tensor, + output_scale: torch.Tensor, + norm_out: Optional[torch.Tensor] = None, +): + """Standard allreduce + rmsnorm + FP4 quantization.""" + + # All-reduce first + allreduce_out = tensor_model_parallel_all_reduce(input_tensor) + + # Then RMS norm + if residual is not None: + if SGL_FUSED_ADD_RMS_NORM is not None: + SGL_FUSED_ADD_RMS_NORM(allreduce_out, residual, rms_gamma, rms_eps) + quant_input = allreduce_out + else: + rms = RMSNorm(allreduce_out.shape[-1], eps=rms_eps) + rms.weight.data = rms_gamma + quant_input, _ = rms.forward_native(allreduce_out, residual) + residual_out = residual + else: + if SGL_RMS_NORM is not None: + quant_input = SGL_RMS_NORM(allreduce_out, rms_gamma, rms_eps) + else: + rms = RMSNorm(allreduce_out.shape[-1], eps=rms_eps) + rms.weight.data = rms_gamma + quant_input = rms.forward_native(allreduce_out) + residual_out = allreduce_out + + # Finally FP4 quantization + if SGL_SCALED_FP4_QUANT is None: + raise RuntimeError("scaled_fp4_quant is not available on this platform") + quant_res, output_scale_res = SGL_SCALED_FP4_QUANT(quant_input, input_global_scale) + if residual is not None: + return quant_res, residual_out, output_scale_res + else: + return quant_res, quant_input + + +def standard_allreduce_rmsnorm_native( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rmsnorm_layer: RMSNorm, + norm_out: Optional[torch.Tensor] = None, +): + """Standard allreduce + rmsnorm operations using native RMSNorm forward.""" + # All-reduce first + allreduce_out = tensor_model_parallel_all_reduce(input_tensor) + # Apply native RMSNorm + if residual is not None: + result = rmsnorm_layer.forward_native(allreduce_out, residual) + return result # Returns (norm_out, residual_out) + else: + result = rmsnorm_layer.forward_native(allreduce_out) + return result # Returns norm_out + + +def standard_allreduce_rmsnorm_fp8_quant_native( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rmsnorm_layer: RMSNorm, + scale_factor: torch.Tensor, + norm_out: Optional[torch.Tensor] = None, + quant_out: Optional[torch.Tensor] = None, +): + """Standard allreduce + rmsnorm + FP8 quantization using native implementations.""" + # All-reduce first + allreduce_out = tensor_model_parallel_all_reduce(input_tensor) + + # Apply native RMSNorm + if residual is not None: + norm_out, residual_out = rmsnorm_layer.forward_native(allreduce_out, residual) + else: + norm_out = rmsnorm_layer.forward_native(allreduce_out) + residual_out = allreduce_out + + # Apply native FP8 quantization + quant_out, _ = static_quant_fp8(norm_out, scale_factor, repeat_scale=False) + + if residual is not None: + return quant_out, residual_out + else: + return quant_out + + +def standard_allreduce_rmsnorm_fp4_quant_native( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rmsnorm_layer: RMSNorm, + input_global_scale: torch.Tensor, + quant_out: torch.Tensor, + output_scale: torch.Tensor, + norm_out: Optional[torch.Tensor] = None, +): + """Standard allreduce + rmsnorm + FP4 quantization using native RMSNorm.""" + # All-reduce first + allreduce_out = tensor_model_parallel_all_reduce(input_tensor) + + # Apply native RMSNorm + if residual is not None: + norm_out, residual_out = rmsnorm_layer.forward_native(allreduce_out, residual) + quant_input = norm_out + else: + norm_out = rmsnorm_layer.forward_native(allreduce_out) + quant_input = norm_out + residual_out = allreduce_out + + # Apply FP4 quantization (still using fused CUDA op as there's no native FP4) + if SGL_SCALED_FP4_QUANT is None: + raise RuntimeError("scaled_fp4_quant is not available on this platform") + quant_res, output_scale_res = SGL_SCALED_FP4_QUANT(quant_input, input_global_scale) + + if residual is not None: + return quant_res, residual_out, output_scale_res + else: + return quant_res, norm_out + + +# Compiled versions of native functions +@torch.compile +def standard_allreduce_rmsnorm_native_compiled( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rmsnorm_layer: RMSNorm, + norm_out: Optional[torch.Tensor] = None, +): + """Compiled version of standard allreduce + rmsnorm.""" + return standard_allreduce_rmsnorm_native( + input_tensor, residual, rmsnorm_layer, norm_out + ) + + +@torch.compile +def standard_allreduce_rmsnorm_fp8_quant_native_compiled( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rmsnorm_layer: RMSNorm, + scale_factor: torch.Tensor, + norm_out: Optional[torch.Tensor] = None, + quant_out: Optional[torch.Tensor] = None, +): + """Compiled version of standard allreduce + rmsnorm + FP8 quantization.""" + return standard_allreduce_rmsnorm_fp8_quant_native( + input_tensor, + residual, + rmsnorm_layer, + scale_factor, + norm_out, + quant_out, + ) + + +@torch.compile +def standard_allreduce_rmsnorm_fp4_quant_native_compiled( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rmsnorm_layer: RMSNorm, + input_global_scale: torch.Tensor, + quant_out: torch.Tensor, + output_scale: torch.Tensor, + norm_out: Optional[torch.Tensor] = None, +): + """Compiled version of standard allreduce + rmsnorm + FP4 quantization.""" + return standard_allreduce_rmsnorm_fp4_quant_native( + input_tensor, + residual, + rmsnorm_layer, + input_global_scale, + quant_out, + output_scale, + norm_out, + ) + + +def create_test_tensors( + seq_len: int, hidden_dim: int, dtype: torch.dtype, use_residual: bool = True +): + """Create test tensors for benchmarking.""" + input_tensor = torch.randn(seq_len, hidden_dim, dtype=dtype) + residual = ( + torch.randn_like(input_tensor) + if use_residual + else torch.zeros_like(input_tensor) + ) + rms_gamma = torch.ones(hidden_dim, dtype=dtype) + norm_out = None if use_residual else torch.empty_like(input_tensor) + + # Quantization scales + scale_fp8 = torch.tensor(1.0, dtype=torch.float32) + scale_fp4 = torch.tensor(1.0, dtype=torch.float32) + quant_out_fp8 = torch.empty_like(input_tensor, dtype=FP8_DTYPE) + # Pre-allocate FP4 output tensors (to avoid allocation overhead in benchmarks) + fp4_quant_out = torch.empty((seq_len, hidden_dim // 2), dtype=torch.uint8) + fp4_output_scale = torch.empty((128, 4), dtype=torch.int32) + + return ( + input_tensor, + norm_out, + residual, + rms_gamma, + scale_fp8, + quant_out_fp8, + scale_fp4, + fp4_quant_out, + fp4_output_scale, + ) + + +def benchmark_operation( + operation_func, *args, warmup: int = 5, trials: int = 20, **kwargs +): + """Benchmark a single operation using CUDA graphs.""" + # Warmup before graph capture + for _ in range(warmup): + operation_func(*args, **kwargs) + torch.cuda.synchronize() + + # Create CUDA graph + graph = torch.cuda.CUDAGraph() + num_op_per_cudagraph = 10 + + # Use sglang's graph_capture to make tensor_model_parallel_all_reduce graph-safe + with graph_capture() as graph_capture_context: + with torch.cuda.graph(graph, stream=graph_capture_context.stream): + for _ in range(num_op_per_cudagraph): + operation_func(*args, **kwargs) + + # Graph warmup + torch.cuda.synchronize() + for _ in range(warmup): + graph.replay() + + # Benchmark with CUDA graph + torch.cuda.synchronize() + start_time = time.perf_counter() + + for _ in range(trials // num_op_per_cudagraph): + # operation_func(*args, **kwargs) + graph.replay() + + torch.cuda.synchronize() + end_time = time.perf_counter() + + avg_time_ms = ((end_time - start_time) / trials) * 1000 + return avg_time_ms + + +def run_benchmarks( + seq_len: int, + hidden_dim: int, + dtype: torch.dtype, + use_residual: bool, + allreduce_params: Optional[FlashInferFusedAllReduceParams], + quant_mode: str = "all", + disable_oneshot: bool = False, +): + """Run all benchmarks for given configuration. + + Args: + quant_mode: "none", "fp8_only", "fp4_only", or "all" + """ + ( + input_tensor, + norm_out, + residual, + rms_gamma, + scale_fp8, + quant_out_fp8, + scale_fp4, + fp4_quant_out, + fp4_output_scale, + ) = create_test_tensors(seq_len, hidden_dim, dtype, use_residual) + + rms_eps = 1e-6 + results = {} + + # Create RMSNorm once for native benchmarks + rmsnorm_layer = RMSNorm(hidden_dim, eps=rms_eps) + rmsnorm_layer.weight.data = rms_gamma + + if quant_mode in ["all", "none"]: + # Standard AllReduce + RMSNorm + try: + time_ms = benchmark_operation( + standard_allreduce_rmsnorm, + input_tensor, + norm_out=norm_out, + residual=residual, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + ) + results["standard_allreduce_rmsnorm"] = time_ms + except Exception as e: + logger.error("Standard AllReduce+RMSNorm failed: %s", e) + results["standard_allreduce_rmsnorm"] = float("inf") + + # Standard AllReduce + RMSNorm Native Compiled + try: + time_ms = benchmark_operation( + standard_allreduce_rmsnorm_native_compiled, + input_tensor, + residual=residual, + rmsnorm_layer=rmsnorm_layer, + norm_out=norm_out, + ) + results["standard_allreduce_rmsnorm_native_compiled"] = time_ms + except Exception as e: + logger.error("Standard AllReduce+RMSNorm Native Compiled failed: %s", e) + results["standard_allreduce_rmsnorm_native_compiled"] = float("inf") + + # FlashInfer Fused AllReduce + RMSNorm Oneshot + if flashinfer_comm is not None and allreduce_params is not None: + try: + if not disable_oneshot: + time_ms = benchmark_operation( + flashinfer_fused_allreduce_rmsnorm, + input_tensor, + residual=residual, + norm_out=norm_out, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + allreduce_params=allreduce_params, + use_oneshot=True, + ) + results["flashinfer_fused_allreduce_rmsnorm_oneshot"] = time_ms + except Exception as e: + logger.error("FlashInfer Fused AllReduce+RMSNorm Oneshot failed: %s", e) + results["flashinfer_fused_allreduce_rmsnorm_oneshot"] = float("inf") + + # FlashInfer Fused AllReduce + RMSNorm Two-shot + try: + time_ms = benchmark_operation( + flashinfer_fused_allreduce_rmsnorm, + input_tensor, + residual=residual, + norm_out=norm_out, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + allreduce_params=allreduce_params, + use_oneshot=False, + ) + results["flashinfer_fused_allreduce_rmsnorm_twoshot"] = time_ms + except Exception as e: + logger.error( + "FlashInfer Fused AllReduce+RMSNorm Two-shot failed: %s", e + ) + results["flashinfer_fused_allreduce_rmsnorm_twoshot"] = float("inf") + + if quant_mode in ["all", "fp8_only"]: + # Standard AllReduce + RMSNorm + FP8 Quant + try: + time_ms = benchmark_operation( + standard_allreduce_rmsnorm_fp8_quant, + input_tensor, + norm_out=norm_out, + residual=residual, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + scale_factor=scale_fp8, + quant_out=quant_out_fp8, + ) + results["standard_allreduce_rmsnorm_fp8_quant"] = time_ms + except Exception as e: + logger.error("Standard AllReduce+RMSNorm+FP8 failed: %s", e) + results["standard_allreduce_rmsnorm_fp8_quant"] = float("inf") + + # Standard AllReduce + RMSNorm + FP8 Quant Native Compiled + try: + time_ms = benchmark_operation( + standard_allreduce_rmsnorm_fp8_quant_native_compiled, + input_tensor, + residual=residual, + rmsnorm_layer=rmsnorm_layer, + # quant_fp8_layer removed in sglang version; static_quant_fp8 is used within the function + scale_factor=scale_fp8, + norm_out=norm_out, + quant_out=quant_out_fp8, + ) + results["standard_allreduce_rmsnorm_fp8_quant_native_compiled"] = time_ms + except Exception as e: + logger.error("Standard AllReduce+RMSNorm+FP8 Native Compiled failed: %s", e) + results["standard_allreduce_rmsnorm_fp8_quant_native_compiled"] = float( + "inf" + ) + + # FlashInfer Fused AllReduce + RMSNorm + FP8 Quant Oneshot + if flashinfer_comm is not None and allreduce_params is not None: + try: + if not disable_oneshot: + time_ms = benchmark_operation( + flashinfer_fused_allreduce_rmsnorm_fp8_quant, + input_tensor, + norm_out=norm_out, + residual=residual, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + scale_factor=scale_fp8, + quant_out=quant_out_fp8, + allreduce_params=allreduce_params, + use_oneshot=True, + ) + results["flashinfer_fused_allreduce_rmsnorm_fp8_quant_oneshot"] = ( + time_ms + ) + except Exception as e: + logger.error( + "FlashInfer Fused AllReduce+RMSNorm+FP8 Oneshot failed: %s", + e, + ) + results["flashinfer_fused_allreduce_rmsnorm_fp8_quant_oneshot"] = float( + "inf" + ) + # FlashInfer Fused AllReduce + RMSNorm + FP8 Quant Two-shot + try: + time_ms = benchmark_operation( + flashinfer_fused_allreduce_rmsnorm_fp8_quant, + input_tensor, + norm_out=norm_out, + residual=residual, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + scale_factor=scale_fp8, + quant_out=quant_out_fp8, + allreduce_params=allreduce_params, + use_oneshot=False, + ) + results["flashinfer_fused_allreduce_rmsnorm_fp8_quant_twoshot"] = ( + time_ms + ) + except Exception as e: + logger.error( + "FlashInfer Fused AllReduce+RMSNorm+FP8 Two-shot failed: %s", + e, + ) + results["flashinfer_fused_allreduce_rmsnorm_fp8_quant_twoshot"] = float( + "inf" + ) + + if quant_mode in ["all", "fp4_only"]: + # Standard AllReduce + RMSNorm + FP4 Quant + try: + time_ms = benchmark_operation( + standard_allreduce_rmsnorm_fp4_quant, + input_tensor, + norm_out=norm_out, + residual=residual, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + input_global_scale=scale_fp4, + quant_out=fp4_quant_out, + output_scale=fp4_output_scale, + ) + results["standard_allreduce_rmsnorm_fp4_quant"] = time_ms + except Exception as e: + logger.error("Standard AllReduce+RMSNorm+FP4 failed: %s", e) + results["standard_allreduce_rmsnorm_fp4_quant"] = float("inf") + + # Standard AllReduce + RMSNorm + FP4 Quant Native Compiled + try: + time_ms = benchmark_operation( + standard_allreduce_rmsnorm_fp4_quant_native_compiled, + input_tensor, + residual=residual, + rmsnorm_layer=rmsnorm_layer, + input_global_scale=scale_fp4, + quant_out=fp4_quant_out, + output_scale=fp4_output_scale, + norm_out=norm_out, + ) + results["standard_allreduce_rmsnorm_fp4_quant_native_compiled"] = time_ms + except Exception as e: + logger.error("Standard AllReduce+RMSNorm+FP4 Native Compiled failed: %s", e) + results["standard_allreduce_rmsnorm_fp4_quant_native_compiled"] = float( + "inf" + ) + + # FlashInfer Fused AllReduce + RMSNorm + FP4 Quant Oneshot + if flashinfer_comm is not None and allreduce_params is not None: + try: + if not disable_oneshot: + time_ms = benchmark_operation( + flashinfer_fused_allreduce_rmsnorm_fp4_quant, + input_tensor, + residual=residual, + norm_out=norm_out, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + input_global_scale=scale_fp4, + allreduce_params=allreduce_params, + quant_out=fp4_quant_out, + output_scale=fp4_output_scale, + use_oneshot=True, + ) + results["flashinfer_fused_allreduce_rmsnorm_fp4_quant_oneshot"] = ( + time_ms + ) + except Exception as e: + logger.error( + "FlashInfer Fused AllReduce+RMSNorm+FP4 Oneshot failed: %s", + e, + ) + results["flashinfer_fused_allreduce_rmsnorm_fp4_quant_oneshot"] = float( + "inf" + ) + + # FlashInfer Fused AllReduce + RMSNorm + FP4 Quant Two-shot + if flashinfer_comm is not None and allreduce_params is not None: + try: + time_ms = benchmark_operation( + flashinfer_fused_allreduce_rmsnorm_fp4_quant, + input_tensor, + residual=residual, + norm_out=norm_out, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + input_global_scale=scale_fp4, + allreduce_params=allreduce_params, + quant_out=fp4_quant_out, + output_scale=fp4_output_scale, + use_oneshot=False, + ) + results["flashinfer_fused_allreduce_rmsnorm_fp4_quant_twoshot"] = ( + time_ms + ) + except Exception as e: + logger.error( + "FlashInfer Fused AllReduce+RMSNorm+FP4 Two-shot failed: %s", + e, + ) + results["flashinfer_fused_allreduce_rmsnorm_fp4_quant_twoshot"] = float( + "inf" + ) + + return results + + +def prepare_results_with_speedups(results_dict): + """Prepare results with speedup calculations based on dynamic baseline selection.""" + prepared_results = [] + + # Determine the fastest baseline for each operation type + def get_fastest_baseline(op_name, results_dict): + """Get the fastest baseline between standard and native_compiled versions.""" + if "fp8_quant" in op_name: + candidates = [ + "standard_allreduce_rmsnorm_fp8_quant", + "standard_allreduce_rmsnorm_fp8_quant_native_compiled", + ] + elif "fp4_quant" in op_name: + candidates = [ + "standard_allreduce_rmsnorm_fp4_quant", + "standard_allreduce_rmsnorm_fp4_quant_native_compiled", + ] + else: + candidates = [ + "standard_allreduce_rmsnorm", + "standard_allreduce_rmsnorm_native_compiled", + ] + + # Find the fastest among available candidates + fastest_time = float("inf") + fastest_baseline = None + + for candidate in candidates: + if ( + candidate in results_dict + and results_dict[candidate] != float("inf") + and results_dict[candidate] < fastest_time + ): + fastest_time = results_dict[candidate] + fastest_baseline = candidate + + return fastest_baseline + + # Create dynamic baseline mapping + dynamic_baseline_mapping = {} + for op_name in results_dict: + if ( + op_name.startswith("flashinfer_") + or op_name.startswith("standard_") + and not op_name.endswith("_native_compiled") + ): + dynamic_baseline_mapping[op_name] = get_fastest_baseline( + op_name, results_dict + ) + + for op_name, time_ms in results_dict.items(): + if time_ms == float("inf"): + speedup_str = "FAILED" + time_str = "FAILED" + else: + time_str = f"{time_ms:.3f}" + # Find the appropriate baseline for this operation + baseline_op = dynamic_baseline_mapping.get(op_name) + if baseline_op and baseline_op in results_dict: + baseline_time = results_dict[baseline_op] + if baseline_time != float("inf") and baseline_time > 0: + speedup = baseline_time / time_ms + speedup_str = f"{speedup:.2f}x" + else: + speedup_str = "N/A" + else: + # For baseline operations, determine if this is the fastest baseline + if op_name.endswith("_native_compiled") or ( + op_name.startswith("standard_") + and not op_name.endswith("_native_compiled") + ): + fastest_baseline = get_fastest_baseline(op_name, results_dict) + if fastest_baseline == op_name: + speedup_str = "baseline" + else: + if fastest_baseline and fastest_baseline in results_dict: + baseline_time = results_dict[fastest_baseline] + if baseline_time != float("inf") and baseline_time > 0: + speedup = baseline_time / time_ms + speedup_str = f"{speedup:.2f}x" + else: + speedup_str = "N/A" + else: + speedup_str = "N/A" + else: + speedup_str = "N/A" + + prepared_results.append( + { + "operation": op_name, + "time_ms": time_ms, + "time_str": time_str, + "speedup_str": speedup_str, + } + ) + + return prepared_results + + +def print_results(results_dict, seq_len, hidden_dim, dtype, use_residual, quant_mode): + """Print benchmark results in a formatted table.""" + print(f"\n{'=' * 80}") + print(f"Results: seq_len={seq_len}, hidden_dim={hidden_dim}") + print( + f"dtype={dtype}, residual={'yes' if use_residual else 'no'}, " + f"quant_mode={quant_mode}" + ) + print(f"{'=' * 80}") + print(f"{'Operation':<50} {'Time (ms)':<12} {'Speedup':<10}") + print(f"{'-' * 80}") + + # Prepare results with speedup calculations + prepared_results = prepare_results_with_speedups(results_dict) + + for result in prepared_results: + if result["time_ms"] == float("inf"): + time_display = result["time_str"] + else: + time_display = f"{result['time_ms']:.3f}" + + print( + f"{result['operation']:<50} {time_display:<12} {result['speedup_str']:<10}" + ) + + +def format_results_markdown( + all_results: list[dict], world_size: int, args: argparse.Namespace +) -> str: + """Format all benchmark results as markdown.""" + markdown = f"""# FlashInfer Fused Collective Operations Benchmark Results + +**World Size:** {world_size} +**Hidden Dimension:** {args.hidden_dim} +**Warmup Iterations:** {args.warmup} +**Benchmark Trials:** {args.trials} +**Quantization Mode:** {all_results[0]["quant_mode"] if all_results else "N/A"} + +--- + +""" + + for result in all_results: + seq_len = result["seq_len"] + dtype = result["dtype"] + use_residual = result["use_residual"] + results_dict = result["results"] + + residual_str = "with residual" if use_residual else "no residual" + + markdown += f""" +## Configuration: seq_len={seq_len}, dtype={dtype}, {residual_str} + +| Operation | Time (ms) | Speedup | +|-----------|-----------|---------| +""" + + # Prepare results with speedup calculations + prepared_results = prepare_results_with_speedups(results_dict) + + for result in prepared_results: + # Format operation name for better readability + formatted_op_name = result["operation"].replace("_", " ").title() + markdown += f"| {formatted_op_name} | {result['time_str']} |" + markdown += f"{result['speedup_str']} |\n" + + markdown += "\n" + + return markdown + + +def save_results_to_file( + all_results: list[dict], world_size: int, args: argparse.Namespace, rank: int +): + """Save benchmark results to markdown file (only on rank 0).""" + if rank != 0: + return + + if not all_results: + logger.warning("No results to save") + return + + output_path = args.output_file + + try: + markdown_content = format_results_markdown(all_results, world_size, args) + + with open(output_path, "w") as f: + f.write(markdown_content) + + except Exception as e: + logger.error("Failed to save results to file: %s", e) + + +def main(): + parser = argparse.ArgumentParser( + description="Benchmark fused collective operations" + ) + parser.add_argument( + "--seq-lens", + type=int, + nargs="+", + default=[128, 512, 1024, 2048], + help="Sequence lengths to test", + ) + parser.add_argument( + "--hidden-dim", type=int, default=8192, help="Hidden dimension size" + ) + parser.add_argument( + "--dtypes", + type=str, + nargs="+", + default=["bfloat16"], + choices=["float16", "bfloat16", "float32"], + help="Data types to test", + ) + parser.add_argument( + "--no-residual", + action="store_true", + help="Skip residual connection tests", + ) + + # Quantization mode options (mutually exclusive with --no-quant) + quant_group = parser.add_mutually_exclusive_group() + quant_group.add_argument( + "--no-quant", action="store_true", help="Skip all quantization tests" + ) + quant_group.add_argument( + "--quant-fp8", action="store_true", help="Only run FP8 quantization tests" + ) + quant_group.add_argument( + "--quant-fp4", action="store_true", help="Only run FP4 quantization tests" + ) + quant_group.add_argument( + "--quant-all", + action="store_true", + help="Run all quantization tests (default)", + ) + + parser.add_argument( + "--disable-oneshot", + action="store_true", + help="Disable oneshot mode for FlashInfer operations", + ) + parser.add_argument( + "--warmup", type=int, default=5, help="Number of warmup iterations" + ) + parser.add_argument( + "--trials", type=int, default=20, help="Number of benchmark trials" + ) + parser.add_argument( + "--output-file", + type=str, + help="""Output file path for markdown results + (default: benchmark_results_.md) + """, + ) + + args = parser.parse_args() + + # Check if running with torchrun (required for collective operations) + if "RANK" not in os.environ or "WORLD_SIZE" not in os.environ: + raise RuntimeError( + "Must run with torchrun for distributed benchmarking. " + "Example: torchrun --nproc_per_node=2 benchmark_fused_collective.py" + ) + + # Initialize distributed environment + rank = int(os.environ["RANK"]) + world_size = int(os.environ["WORLD_SIZE"]) + + device = torch.device(f"cuda:{rank}") + torch.cuda.set_device(device) + torch.set_default_device(device) + + init_distributed_environment( + world_size=world_size, + rank=rank, + local_rank=rank, + backend="nccl", + ) + initialize_model_parallel(tensor_model_parallel_size=world_size) + + # Validate world size (must be > 1 for collective operations) + if world_size <= 1: + raise ValueError( + "World size must be > 1 for collective operations benchmarking. " + f"Current world size: {world_size}. Use torchrun with --nproc_per_node > 1." + ) + + # Determine quantization mode + if args.no_quant: + quant_mode = "none" + elif args.quant_fp8: + quant_mode = "fp8_only" + elif args.quant_fp4: + quant_mode = "fp4_only" + else: # args.quant_all or default + quant_mode = "all" + + if rank == 0: + logger.info("Running benchmark with world_size=%s, rank=%s", world_size, rank) + logger.info("Quantization mode: %s", quant_mode) + if flashinfer_comm is not None: + oneshot_status = "enabled" if not args.disable_oneshot else "disabled" + logger.info( + "FlashInfer available - will benchmark fused operations (oneshot: %s)", + oneshot_status, + ) + else: + logger.info( + "FlashInfer not available - only benchmarking standard operations" + ) + + # Convert dtype strings to torch dtypes + dtype_map = { + "float16": torch.float16, + "bfloat16": torch.bfloat16, + "float32": torch.float32, + } + dtypes = [dtype_map[dt] for dt in args.dtypes] + + # Test configurations + residual_options = [True] if not args.no_residual else [False] + if not args.no_residual: + residual_options.append(False) + + configs = list(itertools.product(args.seq_lens, dtypes, residual_options)) + + # Setup FlashInfer workspace if available + ipc_handles = None + allreduce_params = None + + if flashinfer_comm is not None: + # Use the largest hidden dimension for workspace setup + max_num_token = _FI_MAX_SIZES.get(world_size) // ( + args.hidden_dim * world_size * 2 + ) + + ipc_handles, workspace_tensor = setup_flashinfer_workspace( + world_size, rank, args.hidden_dim, max_num_token + ) + + if workspace_tensor is not None: + allreduce_params = FlashInferFusedAllReduceParams( + rank=rank, + world_size=world_size, + max_token_num=max_num_token, + ) + + # Collect all results for markdown export + all_results = [] + + try: + # Run benchmarks + for seq_len, dtype, use_residual in configs: + if rank == 0: + logger.info( + "\nTesting: seq_len=%s, hidden_dim=%s, dtype=%s, residual=%s", + seq_len, + args.hidden_dim, + dtype, + use_residual, + ) + + results = run_benchmarks( + seq_len, + args.hidden_dim, + dtype, + use_residual, + allreduce_params, + quant_mode=quant_mode, + disable_oneshot=args.disable_oneshot, + ) + + # Store results for markdown export + if rank == 0: + all_results.append( + { + "seq_len": seq_len, + "hidden_dim": args.hidden_dim, + "dtype": str(dtype).replace("torch.", ""), + "use_residual": use_residual, + "quant_mode": quant_mode, + "results": results, + } + ) + + print_results( + results, + seq_len, + args.hidden_dim, + dtype, + use_residual, + quant_mode, + ) + + # Save results to markdown file + if args.output_file and rank == 0: + save_results_to_file(all_results, world_size, args, rank) + + finally: + # Cleanup + if ipc_handles is not None: + cleanup_flashinfer_workspace(ipc_handles) + + with contextlib.suppress(Exception): + dist.barrier() + cleanup_dist_env_and_memory(shutdown_ray=False) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/README.md new file mode 100644 index 00000000..f11c6541 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/README.md @@ -0,0 +1,210 @@ +## Tuning Triton MoE Kernels + +This directory contains benchmarking tools for MoE (Mixture of Experts) kernels. + +### Overview + +The tuning tools support both **Tensor Parallelism (TP)** and **Expert Parallelism (EP)** modes: + +- **TP Mode**: Traditional tensor parallelism where intermediate layers are sharded across GPUs +- **EP Mode**: Expert parallelism where experts are distributed across GPUs. Can be combined with TP mode (e.g., `--tp-size 8 --ep-size 2`) +- **MLLM Support**: Multi-modal Large Language Models with text encoders (e.g., Llama4, Qwen3VL) + +### Tuning Tools + +#### 1. `tuning_fused_moe_triton.py` +A unified tool for tuning the `fused_moe_triton` kernel. Adapted from [vllm's benchmark_moe.py](https://github.com/vllm-project/vllm/blob/main/benchmarks/kernels/benchmark_moe.py), with support for EP mode and various model architectures. + +#### 2. `tuning_fused_moe_triton_sep.py` +A specialized tool for separate kernel tuning, optimizing the first and second MoE kernels independently with TMA (Tensor Memory Accelerator) support. + +### Usage Examples + +#### Basic TP Mode Tuning +```bash +# Tune Mixtral-8x7B with default TP settings +python benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py \ + --model mistralai/Mixtral-8x7B-Instruct-v0.1 \ + --tune + +# Tune Qwen2-57B with FP8 and TP=4 +python benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py \ + --model Qwen/Qwen2-57B-A14B-Instruct \ + --tp-size 4 \ + --dtype fp8_w8a8 \ + --tune + +# Tune DeepSeek-V3 with FP8 and TP=8 +python benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py \ + --model deepseek-ai/DeepSeek-V3-0324 \ + --tp-size 8 \ + --dtype fp8_w8a8 \ + --tune +``` + +#### EP Mode Tuning (Expert Parallelism) +**Note**: EP mode can be used alone or combined with TP mode. When using both, ensure `tp_size` is divisible by `ep_size`. + +```bash +# Tune Mixtral-8x7B with EP=2 only +python benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py \ + --model mistralai/Mixtral-8x7B-Instruct-v0.1 \ + --tp-size 2 \ + --ep-size 2 \ + --tune + +# Tune Qwen2-57B with TP=8 and EP=4 (combined mode) +python benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py \ + --model Qwen/Qwen2-57B-A14B-Instruct \ + --tp-size 8 \ + --ep-size 4 \ + --dtype fp8_w8a8 \ + --tune +``` + +#### MLLM Model Tuning (Multi-modal) +```bash +python benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py \ + --model Qwen/Qwen3-VL-30B-A3B-Instruct \ + --tp-size 2 \ + --tune +``` + +#### Separate Kernel Tuning with `tuning_fused_moe_triton_sep.py` + +This tool requires pre-generated topk_ids files and supports both TP and EP modes: + +Edit the code file (such as srt/models/deepseek_v2.py) in the Python site package and add the logic for saving topk_ids: + +```python +# import get_tensor_model_parallel_rank +# DeepseekV2MoE::forward_normal +if hidden_states.shape[0] >= 4096 and get_tensor_model_parallel_rank() == 0: + topk_ids_dir = xxxx + if not hasattr(self, "save_idx"): + self.save_idx = 0 + if self.save_idx <= 1: + torch.save(topk_output.topk_ids, f"{topk_ids_dir}/topk_ids_layer{self.layer_id}_idx{self.save_idx}.pt") + self.save_idx += 1 +``` + +Launch sglang server and send request using `benchmark/kernels/fused_moe_triton/tuning_client.py` +```bash +python benchmark/kernels/fused_moe_triton/tuning_client.py --port 8000 +``` + +```bash +# TP Mode: Tune separate kernels with TP=4 +python benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton_sep.py \ + --model Qwen/Qwen2-57B-A14B-Instruct \ + --tp-size 4 \ + --topk-ids-dir /path/to/topk_ids \ + --tune + +# EP Mode: Tune separate kernels with TP=4 and EP=2 +python benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton_sep.py \ + --model mistralai/Mixtral-8x7B-Instruct-v0.1 \ + --tp-size 4 \ + --ep-size 2 \ + --topk-ids-dir /path/to/topk_ids \ + --tune + +# MLLM: Tune DeepSeek-V3 with separate kernels, TP=8 and EP=4 +python benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton_sep.py \ + --model deepseek-ai/DeepSeek-V3-0324 \ + --tp-size 8 \ + --ep-size 4 \ + --dtype fp8_w8a8 \ + --topk-ids-dir /path/to/topk_ids \ + --tune + +# Benchmark specific config without tuning +python benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton_sep.py \ + --model deepseek-ai/DeepSeek-V3-0324 \ + --tp-size 4 \ + --batch-size 1024 \ + --dtype fp8_w8a8 \ + --configs 128 256 128 16 8 4 \ + --topk-ids-dir /path/to/topk_ids +``` + +#### Advanced Options +```bash +# Channel-wise quantization +python benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py \ + --model meituan/DeepSeek-R1-Channel-INT8 \ + --tp-size 16 \ + --dtype int8_w8a8 \ + --per-channel-quant \ + --tune + +# Specific batch size tuning +python benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py \ + --model mistralai/Mixtral-8x7B-Instruct-v0.1 \ + --batch-size 2048 \ + --tune +``` + +### Configuration Files + +After tuning, configuration files will be generated: +- **Standard tuning**: `E=64,N=640,device_name=NVIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json` +- **Separate kernel tuning**: Two files for up/down kernels with TMA optimization flags + +Move these files to `sglang/srt/layers/moe/fused_moe_triton/configs/triton_version/` directory to use them in SGLang. + +### Supported Models + +- **Mixtral**: mistralai/Mixtral-8x7B-Instruct-v0.1, mixtral-8x22b +- **Qwen**: Qwen2-57B, Qwen3-235B, Qwen3VL (MLLM) +- **DeepSeek**: DeepSeek-V2, DeepSeek-V3, DeepSeek-R1 +- **Llama**: Llama4-Vision (MLLM) +- **DBRX**: databricks/dbrx-instruct +- **Jamba**: ai21labs/AI21-Jamba +- **Grok**: xai-org/grok-1 +- **GLM**: THUDM/glm-4-9b-chat +- **Bailing**: Custom MoE models + +### Parameters Reference + +- `--model`: HuggingFace model name or local path +- `--tp-size`: Tensor parallelism size (default: 2) +- `--ep-size`: Expert parallelism size (default: 1, can be combined with TP mode, ensure tp_size is divisible by ep_size) +- `--dtype`: Data type (`auto`, `fp8_w8a8`, `int8_w8a16`, `int8_w8a8`) +- `--batch-size`: Specific batch size for tuning (optional) +- `--tune`: Enable tuning mode +- `--per-channel-quant`: Enable per-channel quantization +- `--disable-shared-experts-fusion`: Disable shared expert fusion for some models +- `--topk-ids-dir`: Directory containing pre-generated topk_ids (for sep tool only) +- `--configs`: Manual config specification [BLOCK_M, BLOCK_N, BLOCK_K, GROUP_M, warps, stages] + +### Performance Comparison Tool + +- `benchmark_vllm_vs_sglang_fused_moe_triton.py`: A tool for comparing the performance of fused MoE kernels between vllm and sglang implementations. Supports various model architectures and data types. + +Example usage: +```bash +# Compare with default settings (Mixtral model) +python benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py + +# Compare with FP8 mode for Qwen2-57B +python benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py \ + --model Qwen/Qwen2-57B-A14B-Instruct \ + --use-fp8-w8a8 + +# Compare with custom TP size +python benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py \ + --model deepseek-ai/DeepSeek-V3-0324 \ + --tp-size 8 + +# Compare with custom TP size +python benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py \ + --model deepseek-ai/DeepSeek-V3-0324 \ + --tp-size 8 +``` + +The benchmark results will be saved as plots and data files in the specified output directory (default: `./configs/benchmark_ops/vllm_sglang_fused_moe/`). + +- `benchmark_torch_compile_fused_moe.py`: A tool for benchmarking the performance of the fused MoE kernel with `torch.compile` and original fused MoE kernel. + +Usage is similar to `benchmark_vllm_vs_sglang_fused_moe_triton.py`, note that `torch.compile` does not support `fp8_w8a8` and `int8_w8a8` fused_moe_kernel. Both tools now support EP mode with `--ep-size` parameter. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/benchmark_sglang_fused_moe_triton.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/benchmark_sglang_fused_moe_triton.py new file mode 100644 index 00000000..b418855a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/benchmark_sglang_fused_moe_triton.py @@ -0,0 +1,239 @@ +# python3 benchmark/kernels/fused_moe_triton/sglang_fused_moe_triton.py --model /DeepSeek-V3/ --tp-size 8 +import argparse + +import torch +import triton +from common_utils import get_model_config + +from sglang.srt.distributed.parallel_state import ( + destroy_distributed_environment, + destroy_model_parallel, + init_distributed_environment, + initialize_model_parallel, +) +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import ( + fused_moe as fused_moe_sglang, +) +from sglang.srt.layers.moe.fused_moe_triton.triton_kernels_moe import ( + triton_kernel_moe_forward, +) +from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig +from sglang.srt.layers.moe.topk import TopK, TopKConfig, select_experts + + +def fused_moe_triton_api( + x, + w1, + w2, + input_gating, + topk, +): + topk_op = TopK( + top_k=topk, + renormalize=False, + use_grouped_topk=False, + ) + topk_op.use_triton_kernels = True + triton_topk_output = topk_op.forward_cuda( + hidden_states=x, + router_logits=input_gating, + ) + + moe_runner_config = MoeRunnerConfig( + inplace=False, + ) + return triton_kernel_moe_forward( + x, + w1, + w2, + triton_topk_output, + moe_runner_config, + ) + + +def fused_moe_sglang_api( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=False, + w1_scale=None, + w2_scale=None, + a1_scale=None, + a2_scale=None, + block_shape=None, +): + topk_output = select_experts( + hidden_states=x, + router_logits=input_gating, + topk_config=TopKConfig(top_k=topk, renormalize=False), + ) + return fused_moe_sglang( + x, + w1, + w2, + topk_output, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + block_shape=block_shape, + ) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=list([128, 256, 512, 1024, 2048, 4096, 8192]), + line_arg="provider", + line_vals=[ + "sglang_fused_moe_triton_v340", + "sglang_fused_moe_triton", + ], + line_names=[ + "sglang_fused_moe_triton_v340", + "sglang_fused_moe_triton", + ], + styles=[ + ("blue", "-"), + ("green", "-"), + ], + ylabel="Time (ms)", + plot_name="fused-moe-performance", + args={}, + ) +) +def benchmark( + batch_size, + provider, + model_config, + use_fp8_w8a8=False, + use_cuda_graph: bool = False, +): + print(f"benchmark {provider} with batch_size={batch_size}") + torch.set_default_device("cuda") + torch.cuda.manual_seed_all(0) + + num_tokens = batch_size + num_experts = model_config["num_experts"] + hidden_size = model_config["hidden_size"] + shard_intermediate_size = model_config["shard_intermediate_size"] + topk = model_config["topk"] + dtype = model_config["dtype"] + block_shape = model_config["block_shape"] + + x = torch.randn(num_tokens, hidden_size, dtype=dtype) + + w1 = torch.randn(num_experts, shard_intermediate_size, hidden_size, dtype=dtype) + w2 = torch.randn( + num_experts, hidden_size, shard_intermediate_size // 2, dtype=dtype + ) + + w1_tri = w1.clone() + w2_tri = w2.clone() + w1_tri = w1_tri.transpose(-2, -1).contiguous() + w2_tri = w2_tri.transpose(-2, -1).contiguous() + + input_gating = torch.randn(num_tokens, num_experts, dtype=torch.float32) + + if provider == "sglang_fused_moe_triton_v340": + api_func = fused_moe_triton_api + api_kwargs = { + "x": x, + "w1": w1_tri, + "w2": w2_tri, + "input_gating": input_gating, + "topk": topk, + } + else: + api_func = fused_moe_sglang_api + api_kwargs = { + "x": x, + "w1": w1, + "w2": w2, + "input_gating": input_gating, + "topk": topk, + "use_fp8_w8a8": use_fp8_w8a8, + "block_shape": block_shape, + } + + # Warmup + for _ in range(10): + _ = api_func(**api_kwargs) + torch.cuda.synchronize() + + if use_cuda_graph: + stream = torch.cuda.Stream() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=stream): + api_func(**api_kwargs) + torch.cuda.synchronize() + + bench_lambda = lambda: graph.replay() + else: + bench_lambda = lambda: api_func(**api_kwargs) + + quantiles = [0.5, 0.2, 0.8] + ms, min_ms, max_ms = triton.testing.do_bench(bench_lambda, quantiles=quantiles) + return ms, min_ms, max_ms + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--model", type=str, default="mistralai/Mixtral-8x7B-Instruct-v0.1" + ) + parser.add_argument("--tp-size", "--tp", type=int, default=2) + parser.add_argument("--ep-size", "--ep", type=int, default=1) + parser.add_argument("--use-fp8-w8a8", action="store_true") + parser.add_argument( + "--use-cuda-graph", action="store_true", help="Enable CUDA Graph capture/replay" + ) + parser.add_argument( + "--save-path", + type=str, + default="./configs/benchmark_ops/sglang_fused_moe/", + ) + parser.add_argument("--trust-remote-code", action="store_true") + args = parser.parse_args() + + try: + if not torch.distributed.is_initialized(): + torch.distributed.init_process_group( + backend="nccl" if torch.cuda.is_available() else "gloo", + init_method="tcp://127.0.0.1:23456", + world_size=1, + rank=0, + ) + + init_distributed_environment( + world_size=1, + rank=0, + distributed_init_method="tcp://127.0.0.1:23456", + local_rank=0, + backend="nccl" if torch.cuda.is_available() else "gloo", + ) + + initialize_model_parallel( + tensor_model_parallel_size=args.ep_size, + pipeline_model_parallel_size=args.tp_size, + ) + + model_config = get_model_config(args.model, args.tp_size, args.ep_size) + benchmark.run( + show_plots=True, + print_data=True, + save_path=args.save_path, + model_config=model_config, + use_fp8_w8a8=args.use_fp8_w8a8, + use_cuda_graph=args.use_cuda_graph, + ) + finally: + destroy_model_parallel() + destroy_distributed_environment() + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/benchmark_torch_compile_fused_moe.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/benchmark_torch_compile_fused_moe.py new file mode 100644 index 00000000..2b4faa24 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/benchmark_torch_compile_fused_moe.py @@ -0,0 +1,305 @@ +# python3 benchmark/kernels/fused_moe_triton/benchmark_torch_compile_fused_moe.py --model /DeepSeek-V3/ --tp-size 8 --use-fp8-w8a8 +import argparse + +import torch +import triton +from torch.nn import functional as F +from transformers import AutoConfig + +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import ( + fused_moe as fused_moe_triton, +) +from sglang.srt.model_executor.cuda_graph_runner import set_torch_compile_config + + +def get_model_config(model_name: str, tp_size: int): + """Get model configuration parameters""" + config = AutoConfig.from_pretrained(model_name, trust_remote_code=True) + + if config.architectures[0] == "DbrxForCausalLM": + E = config.ffn_config.moe_num_experts + topk = config.ffn_config.moe_top_k + intermediate_size = config.ffn_config.ffn_hidden_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] == "JambaForCausalLM": + E = config.num_experts + topk = config.num_experts_per_tok + intermediate_size = config.intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] == "Qwen2MoeForCausalLM": + E = config.num_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] == "Qwen3MoeForCausalLM": + E = config.n_routed_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] in ["DeepseekV2ForCausalLM", "DeepseekV3ForCausalLM"]: + E = config.n_routed_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] == "Llama4ForConditionalGeneration": + E = config.text_config.num_local_experts + topk = config.text_config.num_experts_per_tok + intermediate_size = config.text_config.intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] in [ + "Grok1ForCausalLM", + "Grok1ImgGen", + "Grok1AForCausalLM", + ]: + E = config.num_local_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + else: + # Default: Mixtral + E = config.num_local_experts + topk = config.num_experts_per_tok + intermediate_size = config.intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + + shape_configs = { + "num_experts": E, + "topk": topk, + "hidden_size": config.hidden_size, + "shard_intermediate_size": shard_intermediate_size, + "dtype": config.torch_dtype, + } + print(f"{shape_configs=}") + return shape_configs + + +def fused_topk_native( + hidden_states: torch.Tensor, + gating_output: torch.Tensor, + topk: int, + renormalize: bool, +): + assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch" + M, _ = hidden_states.shape + topk_weights = torch.empty( + M, topk, dtype=torch.float32, device=hidden_states.device + ) + topk_ids = torch.empty(M, topk, dtype=torch.int32, device=hidden_states.device) + topk_weights = F.softmax(gating_output.float(), dim=-1) + topk_weights, topk_ids = torch.topk(topk_weights, topk, dim=-1) + if renormalize: + topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True) + return topk_weights, topk_ids + + +@torch.compile(dynamic=False) +def fused_moe_torch( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=False, + w1_scale=None, + w2_scale=None, + a1_scale=None, + a2_scale=None, +) -> torch.Tensor: + assert not use_fp8_w8a8, "Fp8_w8a8 fused_moe is not supported for torch compile" + + topk_weights, topk_ids = fused_topk_native( + hidden_states=x, + gating_output=input_gating, + topk=topk, + renormalize=True, + ) + w13_weights = w1[topk_ids] + w1_weights, w3_weights = torch.chunk(w13_weights, 2, dim=2) + w2_weights = w2[topk_ids] + x1 = torch.einsum("ti,taoi -> tao", x, w1_weights) + x1 = F.silu(x1) + x3 = torch.einsum("ti, taoi -> tao", x, w3_weights) + expert_outs = torch.einsum("tao, taio -> tai", (x1 * x3), w2_weights) + return torch.einsum("tai,ta -> ti", expert_outs, topk_weights.to(expert_outs.dtype)) + + +def fused_moe_torch_compile( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=False, + w1_scale=None, + w2_scale=None, + a1_scale=None, + a2_scale=None, +): + return fused_moe_torch( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + ) + + +def fused_moe_sglang_api( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=False, + w1_scale=None, + w2_scale=None, + a1_scale=None, + a2_scale=None, +): + return fused_moe_triton( + x, + w1, + w2, + input_gating, + topk, + renormalize=True, + inplace=True, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + ) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=list(range(1, 5)), + line_arg="provider", + line_vals=[ + "fused_moe_triton", + "fused_moe_torch_compile", + ], + line_names=[ + "fused_moe_triton", + "fused_moe_torch_compile", + ], + styles=[ + ("blue", "-"), + ("green", "-"), + ], + ylabel="Time (ms)", + plot_name="fused-moe-performance", + args={}, + ) +) +def benchmark(batch_size, provider, model_config, use_fp8_w8a8=False): + print(f"benchmark {provider} with batch_size={batch_size}") + torch.set_default_device("cuda") + torch.cuda.manual_seed_all(0) + set_torch_compile_config() + + num_tokens = batch_size + num_experts = model_config["num_experts"] + hidden_size = model_config["hidden_size"] + shard_intermediate_size = model_config["shard_intermediate_size"] + topk = model_config["topk"] + dtype = model_config["dtype"] + + x = torch.randn(num_tokens, hidden_size, dtype=dtype) + + if use_fp8_w8a8: + init_dtype = dtype + w1 = torch.randn( + num_experts, shard_intermediate_size, hidden_size, dtype=init_dtype + ) + w2 = torch.randn( + num_experts, hidden_size, shard_intermediate_size // 2, dtype=init_dtype + ) + w1 = w1.to(torch.float8_e4m3fn) + w2 = w2.to(torch.float8_e4m3fn) + w1_scale = torch.randn(num_experts, dtype=torch.float32) + w2_scale = torch.randn(num_experts, dtype=torch.float32) + a1_scale = torch.randn(1, dtype=torch.float32) + a2_scale = torch.randn(1, dtype=torch.float32) + else: + w1 = torch.randn(num_experts, shard_intermediate_size, hidden_size, dtype=dtype) + w2 = torch.randn( + num_experts, hidden_size, shard_intermediate_size // 2, dtype=dtype + ) + w1_scale = w2_scale = a1_scale = a2_scale = None + + input_gating = torch.randn(num_tokens, num_experts, dtype=torch.float32) + + # Warmup + api_func = ( + fused_moe_torch_compile + if provider == "fused_moe_torch_compile" + else fused_moe_sglang_api + ) + for _ in range(10): + y = api_func( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + ) + torch.cuda.synchronize() + + quantiles = [0.5, 0.2, 0.8] + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: api_func( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + )[0], + quantiles=quantiles, + ) + return ms, min_ms, max_ms + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--model", type=str, default="mistralai/Mixtral-8x7B-Instruct-v0.1" + ) + parser.add_argument("--tp-size", type=int, default=2) + parser.add_argument("--use-fp8-w8a8", action="store_true") + parser.add_argument( + "--save-path", + type=str, + default="./configs/benchmark_ops/fused_moe_torch_compile/", + ) + args = parser.parse_args() + + model_config = get_model_config(args.model, args.tp_size) + benchmark.run( + show_plots=True, + print_data=True, + save_path=args.save_path, + model_config=model_config, + use_fp8_w8a8=args.use_fp8_w8a8, + ) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py new file mode 100644 index 00000000..206ee2a8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py @@ -0,0 +1,264 @@ +# python3 benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py --model /DeepSeek-V3/ --tp-size 8 --use-fp8-w8a8 +import argparse + +import torch +import triton +from vllm.model_executor.layers.fused_moe.fused_moe import fused_moe as fused_moe_vllm + +from sglang.srt.distributed.parallel_state import ( + destroy_distributed_environment, + destroy_model_parallel, + init_distributed_environment, + initialize_model_parallel, +) +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import ( + fused_moe as fused_moe_sglang, +) + +from .common_utils import get_model_config + + +def fused_moe_vllm_api( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=False, + w1_scale=None, + w2_scale=None, + a1_scale=None, + a2_scale=None, + block_shape=None, +): + if block_shape is not None: + return fused_moe_vllm( + x, + w1, + w2, + input_gating, + topk, + renormalize=True, + inplace=True, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + block_shape=block_shape, + ) + else: + return fused_moe_vllm( + x, + w1, + w2, + input_gating, + topk, + renormalize=True, + inplace=True, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + ) + + +def fused_moe_sglang_api( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=False, + w1_scale=None, + w2_scale=None, + a1_scale=None, + a2_scale=None, + block_shape=None, +): + return fused_moe_sglang( + x, + w1, + w2, + input_gating, + topk, + renormalize=True, + inplace=True, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + block_shape=block_shape, + ) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=list(range(1, 513)), + line_arg="provider", + line_vals=[ + "vllm_fused_moe_triton", + "sglang_fused_moe_triton", + ], + line_names=[ + "vllm_fused_moe_triton", + "sglang_fused_moe_triton", + ], + styles=[ + ("blue", "-"), + ("green", "-"), + ], + ylabel="Time (ms)", + plot_name="fused-moe-performance", + args={}, + ) +) +def benchmark(batch_size, provider, model_config, use_fp8_w8a8=False): + print(f"benchmark {provider} with batch_size={batch_size}") + torch.set_default_device("cuda") + torch.cuda.manual_seed_all(0) + + num_tokens = batch_size + num_experts = model_config["num_experts"] + hidden_size = model_config["hidden_size"] + shard_intermediate_size = model_config["shard_intermediate_size"] + topk = model_config["topk"] + dtype = model_config["dtype"] + block_shape = model_config["block_shape"] + + x = torch.randn(num_tokens, hidden_size, dtype=dtype) + w1_scale = w2_scale = a1_scale = a2_scale = None + + if use_fp8_w8a8: + init_dtype = dtype + w1 = torch.randn( + num_experts, shard_intermediate_size, hidden_size, dtype=init_dtype + ) + w2 = torch.randn( + num_experts, hidden_size, shard_intermediate_size // 2, dtype=init_dtype + ) + w1 = w1.to(torch.float8_e4m3fn) + w2 = w2.to(torch.float8_e4m3fn) + + if block_shape is None: + w1_scale = torch.randn(num_experts, dtype=torch.float32) + w2_scale = torch.randn(num_experts, dtype=torch.float32) + a1_scale = torch.randn(1, dtype=torch.float32) + a2_scale = torch.randn(1, dtype=torch.float32) + else: + block_n, block_k = block_shape[0], block_shape[1] + n_tiles_w1 = (shard_intermediate_size + block_n - 1) // block_n + n_tiles_w2 = (hidden_size + block_n - 1) // block_n + k_tiles_w1 = (hidden_size + block_k - 1) // block_k + k_tiles_w2 = (shard_intermediate_size // 2 + block_k - 1) // block_k + w1_scale = torch.rand( + (num_experts, n_tiles_w1, k_tiles_w1), dtype=torch.float32 + ) + w2_scale = torch.rand( + (num_experts, n_tiles_w2, k_tiles_w2), dtype=torch.float32 + ) + else: + w1 = torch.randn(num_experts, shard_intermediate_size, hidden_size, dtype=dtype) + w2 = torch.randn( + num_experts, hidden_size, shard_intermediate_size // 2, dtype=dtype + ) + + input_gating = torch.randn(num_tokens, num_experts, dtype=torch.float32) + + # Warmup + api_func = ( + fused_moe_vllm_api + if provider == "vllm_fused_moe_triton" + else fused_moe_sglang_api + ) + for _ in range(10): + y = api_func( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + block_shape=block_shape, + ) + torch.cuda.synchronize() + + quantiles = [0.5, 0.2, 0.8] + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: api_func( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + block_shape=block_shape, + )[0], + quantiles=quantiles, + ) + return ms, min_ms, max_ms + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--model", type=str, default="mistralai/Mixtral-8x7B-Instruct-v0.1" + ) + parser.add_argument("--tp-size", "--tp", type=int, default=2) + parser.add_argument("--ep-size", "--ep", type=int, default=1) + parser.add_argument("--use-fp8-w8a8", action="store_true") + parser.add_argument( + "--save-path", + type=str, + default="./configs/benchmark_ops/vllm_sglang_fused_moe/", + ) + args = parser.parse_args() + + try: + if not torch.distributed.is_initialized(): + torch.distributed.init_process_group( + backend="nccl" if torch.cuda.is_available() else "gloo", + init_method="tcp://127.0.0.1:23456", + world_size=1, + rank=0, + ) + + init_distributed_environment( + world_size=1, + rank=0, + distributed_init_method="tcp://127.0.0.1:23456", + local_rank=0, + backend="nccl" if torch.cuda.is_available() else "gloo", + ) + + initialize_model_parallel( + tensor_model_parallel_size=1, + pipeline_model_parallel_size=1, + ) + + shape_configs = get_model_config(args.model, args.tp_size, args.ep_size) + benchmark.run( + show_plots=True, + print_data=True, + save_path=args.save_path, + model_config=shape_configs, + use_fp8_w8a8=args.use_fp8_w8a8, + ) + finally: + destroy_model_parallel() + destroy_distributed_environment() + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/common_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/common_utils.py new file mode 100644 index 00000000..40f8697f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/common_utils.py @@ -0,0 +1,260 @@ +import json +from typing import Dict, List, TypedDict + +import torch +from transformers import AutoConfig + +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import get_config_dtype_str +from sglang.srt.layers.moe.fused_moe_triton.fused_moe_triton_config import ( + get_config_file_name, +) +from sglang.srt.utils import is_hip + + +class BenchmarkConfig(TypedDict): + BLOCK_SIZE_M: int + BLOCK_SIZE_N: int + BLOCK_SIZE_K: int + GROUP_SIZE_M: int + num_warps: int + num_stages: int + + +def calculate_shard_intermediate_size( + intermediate_size: int, tp_size: int, ep_size: int = 1 +) -> int: + assert tp_size % ep_size == 0 + moe_tp_size = tp_size // ep_size + assert intermediate_size % moe_tp_size == 0 + return 2 * intermediate_size // moe_tp_size + + +def get_model_config( + model_name: str, + tp_size: int, + ep_size: int = 1, + disable_shared_experts_fusion: bool = False, + topk_ids_dir: str = None, +) -> Dict: + config = AutoConfig.from_pretrained(model_name, trust_remote_code=True) + + block_shape = None + if ( + hasattr(config, "quantization_config") + and "weight_block_size" in config.quantization_config + ): + block_shape = config.quantization_config["weight_block_size"] + assert len(block_shape) == 2 + + architecture = config.architectures[0] + + # Replace config with text_config for encoder-decoder models after getting block_shape and architecture + if hasattr(config, "text_config"): + config = config.get_text_config() + + if architecture == "DbrxForCausalLM": + E = config.ffn_config.moe_num_experts // ep_size + topk = config.ffn_config.moe_top_k + intermediate_size = config.ffn_config.ffn_hidden_size + elif architecture == "JambaForCausalLM": + E = config.num_experts // ep_size + topk = config.num_experts_per_tok + intermediate_size = config.intermediate_size + elif architecture in [ + "Qwen2MoeForCausalLM", + "Qwen3MoeForCausalLM", + "Qwen3NextForCausalLM", + "Qwen3VLMoeForConditionalGeneration", + ]: + E = config.num_experts // ep_size + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + elif architecture in [ + "DeepseekV2ForCausalLM", + "DeepseekV3ForCausalLM", + "Glm4MoeForCausalLM", + ]: + E = (config.n_routed_experts // ep_size) + ( + 0 + if disable_shared_experts_fusion + or architecture not in ["DeepseekV3ForCausalLM", "Glm4MoeForCausalLM"] + else 1 + ) + topk = config.num_experts_per_tok + ( + 0 if disable_shared_experts_fusion or topk_ids_dir is None else 1 + ) + intermediate_size = config.moe_intermediate_size + elif architecture == "Llama4ForConditionalGeneration": + E = config.num_local_experts // ep_size + ( + 0 if disable_shared_experts_fusion else 1 + ) + topk = config.num_experts_per_tok + ( + 0 if disable_shared_experts_fusion or topk_ids_dir is None else 1 + ) + intermediate_size = config.intermediate_size + elif architecture in [ + "Grok1ForCausalLM", + "Grok1ImgGen", + "Grok1AForCausalLM", + ]: + E = config.num_local_experts // ep_size + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + elif architecture in [ + "BailingMoEForCausalLM", + "BailingMoeForCausalLM", + "BailingMoeV2ForCausalLM", + ]: + E = config.num_experts // ep_size + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + elif architecture == "NemotronHForCausalLM": + E = config.n_routed_experts // ep_size + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + else: + # Default: Mixtral + E = config.num_local_experts // ep_size + topk = config.num_experts_per_tok + intermediate_size = config.intermediate_size + + shard_intermediate_size = calculate_shard_intermediate_size( + intermediate_size, tp_size, ep_size + ) + + return { + "num_experts": E, + "topk": topk, + "hidden_size": config.hidden_size, + "shard_intermediate_size": shard_intermediate_size, + "dtype": config.torch_dtype, + "block_shape": block_shape, + "architecture": architecture, + } + + +def get_rocm_configs_compute_bound() -> List[Dict[str, int]]: + configs: List[BenchmarkConfig] = [] + waves_per_eu_range = 0 + for num_stages in [2]: + for block_m in [32, 64, 128, 256]: + for block_k in [32, 64, 128, 256]: + for block_n in [16, 32, 64, 128, 256]: + for num_warps in [1, 2, 4, 8]: + for group_size in [1, 4, 8, 16, 32]: + configs.append( + { + "BLOCK_SIZE_M": block_m, + "BLOCK_SIZE_N": block_n, + "BLOCK_SIZE_K": block_k, + "GROUP_SIZE_M": group_size, + "num_warps": num_warps, + "num_stages": num_stages, + "waves_per_eu": waves_per_eu_range, + } + ) + return configs + + +def get_configs_compute_bound() -> List[Dict[str, int]]: + configs: List[BenchmarkConfig] = [] + if is_hip(): + configs = get_rocm_configs_compute_bound() + else: + for num_stages in [2, 3, 4, 5]: + for block_m in [16, 32, 64, 128, 256]: + for block_k in [64, 128, 256]: + for block_n in [32, 64, 128, 256]: + for num_warps in [4, 8]: + for group_size in [1, 16, 32, 64]: + configs.append( + { + "BLOCK_SIZE_M": block_m, + "BLOCK_SIZE_N": block_n, + "BLOCK_SIZE_K": block_k, + "GROUP_SIZE_M": group_size, + "num_warps": num_warps, + "num_stages": num_stages, + } + ) + return configs + + +def sort_config(config: BenchmarkConfig) -> BenchmarkConfig: + return { + "BLOCK_SIZE_M": config["BLOCK_SIZE_M"], + "BLOCK_SIZE_N": config["BLOCK_SIZE_N"], + "BLOCK_SIZE_K": config["BLOCK_SIZE_K"], + "GROUP_SIZE_M": config["GROUP_SIZE_M"], + "num_warps": config["num_warps"], + "num_stages": config["num_stages"], + **( + {"waves_per_eu": config["waves_per_eu"]} if "waves_per_eu" in config else {} + ), + **({"USE_TMA": config["USE_TMA"]} if "USE_TMA" in config else {}), + } + + +def save_configs( + configs: Dict[int, BenchmarkConfig], + filename: str, +) -> None: + print(f"Writing best config to {filename}...") + with open(filename, "w") as f: + json.dump(configs, f, indent=4) + f.write("\n") + + +def get_config_filename( + num_experts: int, + shard_intermediate_size: int, + hidden_size: int, + topk: int, + dtype: torch.dtype, + use_fp8_w8a8: bool, + use_int8_w8a8: bool, + use_int8_w8a16: bool, + per_channel_quant: bool, + block_shape: List[int], +) -> str: + dtype_str = get_config_dtype_str( + dtype, + use_int8_w8a16=use_int8_w8a16, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a8=use_int8_w8a8, + ) + + # NOTE(woosuk): The current naming convention uses w2.shape[2], which + # is the intermediate size after silu_and_mul. + filename = get_config_file_name( + num_experts, + shard_intermediate_size // 2, + dtype_str, + block_shape, + per_channel_quant, + ) + + return filename + + +def get_default_batch_sizes() -> List[int]: + return [ + 1, + 2, + 4, + 8, + 16, + 24, + 32, + 48, + 64, + 96, + 128, + 256, + 512, + 1024, + 1536, + 2048, + 3072, + 4096, + ] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/tuning_client.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/tuning_client.py new file mode 100644 index 00000000..68cbfa73 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/tuning_client.py @@ -0,0 +1,71 @@ +import argparse +import os +import time + +import openai + +""" +# Edit the code file srt/models/deepseek_v2.py in the Python site package and add the logic for saving topk_ids: +# import get_tensor_model_parallel_rank +# DeepseekV2MoE::forward_normal +if hidden_states.shape[0] >= 4096 and get_tensor_model_parallel_rank() == 0: + topk_ids_dir = xxxx + if not hasattr(self, "save_idx"): + self.save_idx = 0 + if self.save_idx <= 1: + torch.save(topk_output.topk_ids, f"{topk_ids_dir}/topk_ids_layer{self.layer_id}_idx{self.save_idx}.pt") + self.save_idx += 1 +""" + + +def read_long_prompt(): + import json + + current_dir = os.path.dirname(os.path.abspath(__file__)) + with open(f"{current_dir}/tuning_text.json", "r") as fp: + text = fp.read() + rst = json.loads(text) + return rst["prompt"] + + +def openai_stream_test(model, ip, port): + client = openai.Client(base_url=f"http://{ip}:{port}/v1", api_key="None") + qst = read_long_prompt() + + messages = [ + {"role": "user", "content": qst}, + ] + msg2 = dict( + model=model, + messages=messages, + temperature=0.6, + top_p=0.75, + max_tokens=100, + ) + response = client.chat.completions.create(**msg2, stream=True) + time_start = time.time() + time_cost = [] + for chunk in response: + time_end = time.time() + # if chunk.choices[0].delta.content: + # print(chunk.choices[0].delta.content, end="", flush=True) + time_cost.append(time_end - time_start) + time_start = time.time() + + ttft = time_cost[0] + time_cost[1] + tpot = sum(time_cost[2:]) / len(time_cost[2:]) + print(f"\nTTFT {ttft}, TPOT {tpot}") + return ttft, tpot + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--model", type=str, default="auto") + parser.add_argument( + "--ip", + type=str, + default="127.0.0.1", + ) + parser.add_argument("--port", type=int, default=8188) + args = parser.parse_args() + openai_stream_test(args.model, args.ip, args.port) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py new file mode 100644 index 00000000..aef7ed8f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py @@ -0,0 +1,458 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/main/benchmarks/kernels/benchmark_moe.py +import argparse +import time +from contextlib import nullcontext +from datetime import datetime +from typing import Any, Dict, List, Tuple + +import ray +import torch +import triton +from common_utils import ( + BenchmarkConfig, + get_config_filename, + get_configs_compute_bound, + get_default_batch_sizes, + get_model_config, + save_configs, + sort_config, +) +from ray.experimental.tqdm_ray import tqdm + +from sglang.srt.layers.moe.fused_moe_triton import override_config +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import fused_moe +from sglang.srt.layers.moe.fused_moe_triton.fused_moe_triton_config import ( + get_config_dtype_str, + get_default_config, + get_moe_configs, +) +from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig +from sglang.srt.layers.moe.topk import TopKConfig, select_experts +from sglang.srt.utils import is_hip + +_is_hip = is_hip() + + +def benchmark_config( + config: BenchmarkConfig, + num_tokens: int, + num_experts: int, + shard_intermediate_size: int, + hidden_size: int, + topk: int, + dtype: torch.dtype, + use_fp8_w8a8: bool, + use_int8_w8a8: bool, + use_int8_w8a16: bool, + per_channel_quant: bool, + block_shape: List[int] = None, + num_iters: int = 100, +) -> float: + init_dtype = torch.float16 if use_fp8_w8a8 else dtype + x = torch.randn(num_tokens, hidden_size, dtype=dtype) + if use_int8_w8a16 or use_int8_w8a8: + w1 = torch.randint( + -127, + 127, + ( + num_experts, + shard_intermediate_size, + hidden_size, + ), + dtype=torch.int8, + ) + w2 = torch.randint( + -127, + 127, + ( + num_experts, + hidden_size, + shard_intermediate_size // 2, + ), + dtype=torch.int8, + ) + else: + w1 = torch.randn( + num_experts, shard_intermediate_size, hidden_size, dtype=init_dtype + ) + w2 = torch.randn( + num_experts, hidden_size, shard_intermediate_size // 2, dtype=init_dtype + ) + gating_output = torch.randn(num_iters, num_tokens, num_experts, dtype=torch.float32) + + w1_scale = None + w2_scale = None + a1_scale = None + a2_scale = None + if use_int8_w8a16: + w1_scale = torch.randn( + (num_experts, 2 * shard_intermediate_size), dtype=torch.float32 + ) + w2_scale = torch.randn((hidden_size, num_experts), dtype=torch.float32) + if use_fp8_w8a8 or use_int8_w8a8: + if use_int8_w8a8 and block_shape is None: + w1_scale = torch.randn( + num_experts, shard_intermediate_size, dtype=torch.float32 + ) + w2_scale = torch.randn(num_experts, hidden_size, dtype=torch.float32) + elif block_shape is None: + w1_scale = torch.randn(num_experts, dtype=torch.float32) + w2_scale = torch.randn(num_experts, dtype=torch.float32) + a1_scale = torch.randn(1, dtype=torch.float32) + a2_scale = torch.randn(1, dtype=torch.float32) + else: + block_n, block_k = block_shape[0], block_shape[1] + n_tiles_w1 = (shard_intermediate_size + block_n - 1) // block_n + n_tiles_w2 = (hidden_size + block_n - 1) // block_n + k_tiles_w1 = (hidden_size + block_k - 1) // block_k + k_tiles_w2 = (shard_intermediate_size // 2 + block_k - 1) // block_k + w1_scale = torch.rand( + (num_experts, n_tiles_w1, k_tiles_w1), dtype=torch.float32 + ) + w2_scale = torch.rand( + (num_experts, n_tiles_w2, k_tiles_w2), dtype=torch.float32 + ) + + if use_fp8_w8a8: + w1 = w1.to(torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn) + w2 = w2.to(torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn) + + input_gating = torch.randn(num_tokens, num_experts, dtype=torch.float32) + topk_config = TopKConfig( + top_k=topk, + renormalize=True, + ) + topk_output = select_experts(x, input_gating, topk_config) + + def prepare(i: int): + input_gating = gating_output[i] + new_topk_output = select_experts(x, input_gating, topk_config) + topk_output.topk_weights.copy_(new_topk_output.topk_weights) + topk_output.topk_ids.copy_(new_topk_output.topk_ids) + topk_output.router_logits.copy_(new_topk_output.router_logits) + + def run(): + moe_runner_config = MoeRunnerConfig( + inplace=True, + ) + + with override_config(config): + fused_moe( + x, + w1, + w2, + topk_output, + moe_runner_config=moe_runner_config, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a8=use_int8_w8a8, + use_int8_w8a16=use_int8_w8a16, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + per_channel_quant=per_channel_quant, + block_shape=block_shape, + ) + + # JIT compilation & warmup + run() + torch.cuda.synchronize() + + # Capture 10 invocations with CUDA graph + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + for _ in range(10): + run() + torch.cuda.synchronize() + + # Warmup + for _ in range(5): + graph.replay() + torch.cuda.synchronize() + + # Flush L2 cache with 256 MB data + cache_flush = torch.empty(int(256e6 // 4), dtype=torch.int, device="cuda") + cache_flush.zero_() + + start_events = [torch.cuda.Event(enable_timing=True) for _ in range(num_iters)] + end_events = [torch.cuda.Event(enable_timing=True) for _ in range(num_iters)] + + for i in range(num_iters): + prepare(i) + start_events[i].record() + graph.replay() + end_events[i].record() + torch.cuda.synchronize() + + latencies: List[float] = [] + for i in range(num_iters): + latencies.append(start_events[i].elapsed_time(end_events[i])) + avg = sum(latencies) / (num_iters * 10) * 1000 # us + graph.reset() + return avg + + +@ray.remote(num_gpus=1) +class BenchmarkWorker: + + def __init__(self, seed: int) -> None: + torch.set_default_device("cuda") + torch.cuda.manual_seed_all(0) + self.seed = seed + # Get the device ID to allocate tensors and kernels + # on the respective GPU. + self.device_id = int(ray.get_gpu_ids()[0]) + + def benchmark( + self, + num_tokens: int, + num_experts: int, + shard_intermediate_size: int, + hidden_size: int, + topk: int, + dtype: torch.dtype, + use_fp8_w8a8: bool, + use_int8_w8a8: bool, + use_int8_w8a16: bool, + per_channel_quant: bool, + block_shape: List[int], + ) -> Tuple[Dict[str, int], float]: + torch.cuda.manual_seed_all(0) + dtype_str = get_config_dtype_str( + dtype, use_int8_w8a16=use_int8_w8a16, use_fp8_w8a8=use_fp8_w8a8 + ) + # NOTE(woosuk): The current naming convention uses w2.shape[2], which + # is the intermediate size after silu_and_mul. + block_n = block_shape[0] if block_shape else 0 + block_k = block_shape[1] if block_shape else 0 + op_config = get_moe_configs( + num_experts, + shard_intermediate_size // 2, + dtype_str, + block_n, + block_k, + per_channel_quant, + ) + if op_config is None: + config = get_default_config( + num_tokens, + num_experts, + shard_intermediate_size, + hidden_size, + topk, + dtype_str, + False, + block_shape, + ) + else: + config = op_config[min(op_config.keys(), key=lambda x: abs(x - num_tokens))] + with torch.cuda.device(self.device_id) if is_hip() else nullcontext(): + kernel_time = benchmark_config( + config, + num_tokens, + num_experts, + shard_intermediate_size, + hidden_size, + topk, + dtype, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + per_channel_quant, + block_shape, + ) + return config, kernel_time + + def tune( + self, + num_tokens: int, + num_experts: int, + shard_intermediate_size: int, + hidden_size: int, + topk: int, + dtype: torch.dtype, + use_fp8_w8a8: bool, + use_int8_w8a8: bool, + use_int8_w8a16: bool, + per_channel_quant: bool, + block_shape: List[int], + search_space: List[Dict[str, int]], + ) -> Dict[str, int]: + best_config = None + best_time = float("inf") + with torch.cuda.device(self.device_id) if is_hip() else nullcontext(): + for config in tqdm(search_space): + try: + kernel_time = benchmark_config( + config, + num_tokens, + num_experts, + shard_intermediate_size, + hidden_size, + topk, + dtype, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + per_channel_quant, + block_shape, + num_iters=10, + ) + except (triton.runtime.autotuner.OutOfResources, RuntimeError): + # Some configurations may be invalid and fail to compile. + continue + + if kernel_time < best_time: + best_time = kernel_time + best_config = config + now = datetime.now() + print(f"{now.ctime()}] Completed tuning for batch_size={num_tokens}") + assert best_config is not None + return best_config + + +def main(args: argparse.Namespace): + print(args) + + model_config = get_model_config( + args.model, args.tp_size, args.ep_size, args.disable_shared_experts_fusion + ) + + E = model_config["num_experts"] + topk = model_config["topk"] + hidden_size = model_config["hidden_size"] + shard_intermediate_size = model_config["shard_intermediate_size"] + dtype = model_config["dtype"] + block_shape = model_config["block_shape"] + + use_fp8_w8a8 = args.dtype == "fp8_w8a8" + use_int8_w8a8 = args.dtype == "int8_w8a8" + use_int8_w8a16 = args.dtype == "int8_w8a16" + per_channel_quant = args.per_channel_quant + + if args.batch_size is None: + batch_sizes = get_default_batch_sizes() + else: + batch_sizes = [args.batch_size] + + ray.init() + num_gpus = int(ray.available_resources()["GPU"]) + workers = [BenchmarkWorker.remote(args.seed) for _ in range(num_gpus)] + + def _distribute(method: str, inputs: List[Any]) -> List[Any]: + outputs = [] + worker_idx = 0 + for input_args in inputs: + worker = workers[worker_idx] + worker_method = getattr(worker, method) + output = worker_method.remote(*input_args) + outputs.append(output) + worker_idx = (worker_idx + 1) % num_gpus + return ray.get(outputs) + + if args.tune: + search_space = get_configs_compute_bound() + if block_shape is not None: + block_n, block_k = block_shape[0], block_shape[1] + search_space = [ + config + for config in search_space + if block_k % config["BLOCK_SIZE_K"] == 0 + ] + + filename = get_config_filename( + E, + shard_intermediate_size, + hidden_size, + topk, + dtype, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + per_channel_quant, + block_shape, + ) + print( + f"Start tuning over {len(search_space)} configurations to create {filename}..." + ) + + start = time.perf_counter() + configs = _distribute( + "tune", + [ + ( + batch_size, + E, + shard_intermediate_size, + hidden_size, + topk, + dtype, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + per_channel_quant, + block_shape, + search_space, + ) + for batch_size in batch_sizes + ], + ) + best_configs = { + M: sort_config(config) for M, config in zip(batch_sizes, configs) + } + save_configs( + best_configs, + filename, + ) + end = time.perf_counter() + print(f"Tuning took {end - start:.2f} seconds") + else: + outputs = _distribute( + "benchmark", + [ + ( + batch_size, + E, + shard_intermediate_size, + hidden_size, + topk, + dtype, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + per_channel_quant, + block_shape, + ) + for batch_size in batch_sizes + ], + ) + + for batch_size, (config, kernel_time) in zip(batch_sizes, outputs): + print(f"Batch size: {batch_size}, config: {config}") + print(f"Kernel time: {kernel_time:.2f} us") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--model", type=str, default="mistralai/Mixtral-8x7B-Instruct-v0.1" + ) + parser.add_argument("--tp-size", "--tp", type=int, default=2) + parser.add_argument("--ep-size", "--ep", type=int, default=1) + parser.add_argument( + "--dtype", + type=str, + choices=["auto", "fp8_w8a8", "int8_w8a16", "int8_w8a8"], + default="auto", + ) + parser.add_argument( + "--per-channel-quant", + action="store_true", + ) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--batch-size", type=int, required=False) + parser.add_argument("--tune", action="store_true") + parser.add_argument("--disable-shared-experts-fusion", action="store_true") + args = parser.parse_args() + + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton_sep.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton_sep.py new file mode 100644 index 00000000..afee7994 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton_sep.py @@ -0,0 +1,694 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/main/benchmarks/kernels/benchmark_moe.py +import argparse +import json +import os +import time +from contextlib import nullcontext +from datetime import datetime +from typing import Any, Dict, List, Tuple + +import ray +import torch +import triton +import triton.language as tl +from common_utils import ( + BenchmarkConfig, + get_config_filename, + get_configs_compute_bound, + get_default_batch_sizes, + get_model_config, + sort_config, +) +from ray.experimental.tqdm_ray import tqdm +from sgl_kernel import silu_and_mul + +from sglang.srt.layers.moe.fused_moe_triton import override_config +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import ( + get_config_dtype_str, + invoke_fused_moe_kernel, + moe_align_block_size, +) +from sglang.srt.layers.moe.fused_moe_triton.fused_moe_triton_config import ( + get_config_file_name, +) +from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig +from sglang.srt.layers.moe.topk import TopKConfig, select_experts +from sglang.srt.utils import is_hip + +_is_hip = is_hip() + + +def benchmark_config( + config: BenchmarkConfig, + num_tokens: int, + num_experts: int, + shard_intermediate_size: int, + hidden_size: int, + topk: int, + dtype: torch.dtype, + use_fp8_w8a8: bool, + use_int8_w8a8: bool, + use_int8_w8a16: bool, + topk_ids_dir: str, + block_shape: List[int] = None, + num_iters: int = 100, +) -> float: + ncu_enable = os.getenv("NCU_ENABLE", "0") == "1" + if ncu_enable: + num_iters = 1 + init_dtype = torch.float16 if use_fp8_w8a8 else dtype + hidden_states = torch.randn(num_tokens, hidden_size, dtype=dtype) + if use_int8_w8a16 or use_int8_w8a8: + w1 = torch.randint( + -127, + 127, + ( + num_experts, + shard_intermediate_size, + hidden_size, + ), + dtype=torch.int8, + ) + w2 = torch.randint( + -127, + 127, + ( + num_experts, + hidden_size, + shard_intermediate_size // 2, + ), + dtype=torch.int8, + ) + else: + w1 = torch.randn( + num_experts, shard_intermediate_size, hidden_size, dtype=init_dtype + ) + w2 = torch.randn( + num_experts, hidden_size, shard_intermediate_size // 2, dtype=init_dtype + ) + gating_output = torch.randn(num_iters, num_tokens, num_experts, dtype=torch.float32) + + w1_scale = None + w2_scale = None + a1_scale = None + a2_scale = None + if use_int8_w8a16: + w1_scale = torch.randn( + (num_experts, 2 * shard_intermediate_size), dtype=torch.float32 + ) + w2_scale = torch.randn((hidden_size, num_experts), dtype=torch.float32) + if use_fp8_w8a8 or use_int8_w8a8: + if use_int8_w8a8 and block_shape is None: + w1_scale = torch.randn( + num_experts, shard_intermediate_size, dtype=torch.float32 + ) + w2_scale = torch.randn(num_experts, hidden_size, dtype=torch.float32) + elif block_shape is None: + w1_scale = torch.randn(num_experts, dtype=torch.float32) + w2_scale = torch.randn(num_experts, dtype=torch.float32) + a1_scale = torch.randn(1, dtype=torch.float32) + a2_scale = torch.randn(1, dtype=torch.float32) + else: + block_n, block_k = block_shape[0], block_shape[1] + n_tiles_w1 = (shard_intermediate_size + block_n - 1) // block_n + n_tiles_w2 = (hidden_size + block_n - 1) // block_n + k_tiles_w1 = (hidden_size + block_k - 1) // block_k + k_tiles_w2 = (shard_intermediate_size // 2 + block_k - 1) // block_k + w1_scale = torch.rand( + (num_experts, n_tiles_w1, k_tiles_w1), dtype=torch.float32 + ) + w2_scale = torch.rand( + (num_experts, n_tiles_w2, k_tiles_w2), dtype=torch.float32 + ) + + if use_fp8_w8a8: + w1 = w1.to(torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn) + w2 = w2.to(torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn) + + input_gating = torch.randn(num_tokens, num_experts, dtype=torch.float32) + topk_config = TopKConfig( + top_k=topk, + renormalize=True, + ) + topk_output = select_experts(hidden_states, input_gating, topk_config) + + def prepare(i: int): + input_gating = gating_output[i] + topk_ids = torch.load(f"{topk_ids_dir}/topk_ids_layer{i%58+3}_idx{i//58}.pt") + new_topk_output = select_experts(hidden_states, input_gating, topk_config) + topk_output.topk_weights.copy_(new_topk_output.topk_weights) + tokens, _topk = topk_output.topk_ids.shape + topk_output.topk_ids.copy_(topk_ids[:tokens, :_topk]) + topk_output.router_logits.copy_(new_topk_output.router_logits) + + moe_use_tma = False + + def run(): + moe_runner_config = MoeRunnerConfig( + inplace=True, + ) + topk_weights, topk_ids, _ = topk_output + + sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size( + topk_ids, config["BLOCK_SIZE_M"], num_experts + ) + M = hidden_states.shape[0] + E, N, _ = w1.shape + + topk = topk_ids.shape[1] + padded_tokens = ( + min(M * topk, E + 1) * (config["BLOCK_SIZE_M"] - 1) if moe_use_tma else 0 + ) + total_tokens = M * topk + padded_tokens + cache = torch.empty( + total_tokens * max(N, w2.shape[1]), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + intermediate_cache1 = cache[: total_tokens * N].view( + (total_tokens, N), + ) + intermediate_cache2 = torch.empty( + (total_tokens, N // 2), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + intermediate_cache3 = cache[: M * topk * w2.shape[1]].view( + (M, topk, w2.shape[1]), + ) + + compute_type = ( + tl.bfloat16 if hidden_states.dtype == torch.bfloat16 else tl.float16 + ) + apply_router_weight_on_input = moe_runner_config.apply_router_weight_on_input + + with override_config(config): + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + torch.cuda.synchronize() + start_event.record() + for _ in range(10 if not ncu_enable else 1): + invoke_fused_moe_kernel( + hidden_states, + w1, + None, + intermediate_cache1, + None, + w1_scale, + None, + topk_weights, + topk_ids, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + apply_router_weight_on_input, + topk_ids.shape[1], + config, + compute_type=compute_type, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a8=False, + use_int8_w8a16=False, + use_int4_w4a16=False, + per_channel_quant=False, + block_shape=block_shape, + b_use_tma=moe_use_tma, + c_sorted=moe_use_tma, + filter_expert=False, + ) + end_event.record() + end_event.synchronize() + time_cost0 = start_event.elapsed_time(end_event) + + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + torch.cuda.synchronize() + start_event.record() + + silu_and_mul(intermediate_cache1.view(-1, N), intermediate_cache2) + for _ in range(10 if not ncu_enable else 1): + invoke_fused_moe_kernel( + intermediate_cache2, + w2, + None, + intermediate_cache3, + a2_scale, + w2_scale, + None, + topk_weights, + topk_ids, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + not apply_router_weight_on_input, + 1, + config, + compute_type=compute_type, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a8=False, + use_int8_w8a16=False, + use_int4_w4a16=False, + per_channel_quant=False, + block_shape=block_shape, + a_use_tma=moe_use_tma, + b_use_tma=moe_use_tma, + filter_expert=False, + ) + end_event.record() + end_event.synchronize() + time_cost1 = start_event.elapsed_time(end_event) + return time_cost0, time_cost1 + + # JIT compilation & warmup + if not ncu_enable: + moe_use_tma = False + run() + moe_use_tma = True + run() + latencies: List[float] = [] + latencies1: List[float] = [] + latencies_tma: List[float] = [] + latencies1_tma: List[float] = [] + + for i in range(num_iters): + prepare(i) + torch.cuda.synchronize() + moe_use_tma = False + t0, t1 = run() + torch.cuda.synchronize() + latencies.append(t0) + latencies1.append(t1) + + moe_use_tma = True + t0, t1 = run() + torch.cuda.synchronize() + latencies_tma.append(t0) + latencies1_tma.append(t1) + + avg = sum(latencies) / (num_iters * 10) * 1000 # us + avg_tma = sum(latencies_tma) / (num_iters * 10) * 1000 # us + avg1 = sum(latencies1) / (num_iters * 10) * 1000 # us + avg1_tma = sum(latencies1_tma) / (num_iters * 10) * 1000 # us + + return avg, avg_tma, avg1, avg1_tma + + +class BestConfigTrace: + def __init__(self, name): + self.name = name + self.config = None + self.time_cost = float("inf") + self.time_cost_all = None # kernel0 without tma,, kernel0 with tma, kernel1 without tma, kernel1 with tma + + def update(self, config, time_cost, time_cost_all): + if time_cost < self.time_cost: + print( + f"New best config for {self.name}: {config}, {time_cost=}, {time_cost_all=}, org: {self.config}, {self.time_cost_all}", + flush=True, + ) + self.config = config + self.time_cost = time_cost + self.time_cost_all = time_cost_all + + @property + def total_time(self): + return self.time_cost_all[0] + min(self.time_cost_all[2], self.time_cost_all[3]) + + def config_dict(self, down_moe=False): + if not down_moe: + return self.config + else: + return { + **self.config, + "USE_TMA": self.time_cost_all[2] > self.time_cost_all[3], + } + + +class BenchmarkWorker: + + def __init__(self, seed: int) -> None: + torch.set_default_device("cuda") + torch.cuda.manual_seed_all(0) + self.seed = seed + # Get the device ID to allocate tensors and kernels + # on the respective GPU. + self.device_id = 0 # int(ray.get_gpu_ids()[0]) + + def benchmark( + self, + num_tokens: int, + num_experts: int, + shard_intermediate_size: int, + hidden_size: int, + topk: int, + dtype: torch.dtype, + use_fp8_w8a8: bool, + use_int8_w8a8: bool, + use_int8_w8a16: bool, + block_shape: List[int], + cfg: Dict[str, int], + topk_ids_dir: str, + ) -> Tuple[Dict[str, int], float]: + torch.cuda.manual_seed_all(0) + dtype_str = get_config_dtype_str( + dtype, use_int8_w8a16=use_int8_w8a16, use_fp8_w8a8=use_fp8_w8a8 + ) + # NOTE(woosuk): The current naming convention uses w2.shape[2], which + # is the intermediate size after silu_and_mul. + block_n = block_shape[0] if block_shape else 0 + block_k = block_shape[1] if block_shape else 0 + with torch.cuda.device(self.device_id) if is_hip() else nullcontext(): + kernel_time = benchmark_config( + cfg, + num_tokens, + num_experts, + shard_intermediate_size, + hidden_size, + topk, + dtype, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + topk_ids_dir, + block_shape, + ) + return cfg, kernel_time + + def tune( + self, + num_tokens: int, + num_experts: int, + shard_intermediate_size: int, + hidden_size: int, + topk: int, + dtype: torch.dtype, + use_fp8_w8a8: bool, + use_int8_w8a8: bool, + use_int8_w8a16: bool, + block_shape: List[int], + search_space: List[Dict[str, int]], + topk_ids_dir: str, + ) -> Dict[str, int]: + trace0 = BestConfigTrace("kernel0") + trace1 = BestConfigTrace("kernel1") + trace2 = BestConfigTrace("kernel all") + + with torch.cuda.device(self.device_id) if is_hip() else nullcontext(): + for config in tqdm(search_space): + try: + kt0_no_tma, kt0_tma, kt1_no_tma, kt1_tma = benchmark_config( + config, + num_tokens, + num_experts, + shard_intermediate_size, + hidden_size, + topk, + dtype, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + topk_ids_dir, + block_shape, + num_iters=10, + ) + except triton.runtime.autotuner.OutOfResources: + # Some configurations may be invalid and fail to compile. + continue + kt0 = kt0_no_tma + kt1 = min(kt1_no_tma, kt1_tma) + trace0.update( + config, + kt0, + (kt0_no_tma, kt0_tma, kt1_no_tma, kt1_tma), + ) + trace1.update( + config, + kt1, + (kt0_no_tma, kt0_tma, kt1_no_tma, kt1_tma), + ) + trace2.update( + config, + kt0 + kt1, + (kt0_no_tma, kt0_tma, kt1_no_tma, kt1_tma), + ) + + now = datetime.now() + print(f"{now.ctime()}] Completed tuning for batch_size={num_tokens}") + assert trace0.config is not None + assert trace1.config is not None + print( + f"{num_tokens=}, {trace0.config=}, {trace0.time_cost_all=}, {trace1.config=}, {trace1.time_cost_all=}" + ) + if trace0.config["BLOCK_SIZE_M"] != trace1.config["BLOCK_SIZE_M"]: + best_trace = trace0 if trace0.total_time < trace1.total_time else trace1 + best_trace = ( + best_trace if best_trace.total_time < trace2.total_time else trace2 + ) + return ( + best_trace.config_dict(), + best_trace.config_dict(True), + best_trace.time_cost_all, + best_trace.time_cost_all, + ) + return ( + trace0.config_dict(), + trace1.config_dict(True), + trace0.time_cost_all, + trace1.time_cost_all, + ) + + +def save_configs_sep( + configs: Dict[int, BenchmarkConfig], + num_experts: int, + shard_intermediate_size: int, + hidden_size: int, + topk: int, + dtype: torch.dtype, + use_fp8_w8a8: bool, + use_int8_w8a8: bool, + use_int8_w8a16: bool, + block_shape: List[int], + down_moe: bool = False, +) -> None: + dtype_str = get_config_dtype_str( + dtype, + use_int8_w8a16=use_int8_w8a16, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a8=use_int8_w8a8, + ) + + # NOTE(woosuk): The current naming convention uses w2.shape[2], which + # is the intermediate size after silu_and_mul. + filename = get_config_file_name( + num_experts, + shard_intermediate_size // 2, + dtype_str, + block_shape, + down_moe=down_moe, + ) + + print(f"Writing best config to {filename}...") + with open(filename, "w") as f: + json.dump(configs, f, indent=4) + f.write("\n") + + +def main(args: argparse.Namespace): + print(args) + + model_config = get_model_config( + args.model, + args.tp_size, + args.ep_size, + args.disable_shared_experts_fusion, + args.topk_ids_dir, + ) + + E = model_config["num_experts"] + topk = model_config["topk"] + hidden_size = model_config["hidden_size"] + shard_intermediate_size = model_config["shard_intermediate_size"] + dtype = model_config["dtype"] + block_shape = model_config["block_shape"] + + use_fp8_w8a8 = args.dtype == "fp8_w8a8" + use_int8_w8a8 = args.dtype == "int8_w8a8" + use_int8_w8a16 = args.dtype == "int8_w8a16" + + topk_ids_dir = args.topk_ids_dir + if args.batch_size is None: + batch_sizes = get_default_batch_sizes() + batch_sizes.reverse() + else: + batch_sizes = [args.batch_size] + if len(batch_sizes) == 1: + worker = BenchmarkWorker(args.seed) + if args.tune: + search_space = get_configs_compute_bound() + worker.tune( + batch_sizes[0], + E, + shard_intermediate_size, + hidden_size, + topk, + dtype, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + block_shape, + search_space, + topk_ids_dir, + ) + else: + cfg = { + "BLOCK_SIZE_M": args.configs[0], + "BLOCK_SIZE_N": args.configs[1], + "BLOCK_SIZE_K": args.configs[2], + "GROUP_SIZE_M": args.configs[3], + "num_warps": args.configs[4], + "num_stages": args.configs[5], + } + + _, (t0, t0_tma, t1, t1_tma) = worker.benchmark( + args.batch_size, + E, + shard_intermediate_size, + hidden_size, + topk, + dtype, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + block_shape, + cfg, + topk_ids_dir, + ) + print(f"{t0=}, {t0_tma=}, {t1=}, {t1_tma=}") + return + + assert args.tune + + ray.init() + num_gpus = int(ray.available_resources()["GPU"]) + workers = [ + ray.remote(num_gpus=1)(BenchmarkWorker).remote(args.seed) + for _ in range(num_gpus) + ] + + def _distribute(method: str, inputs: List[Any]) -> List[Any]: + outputs = [] + worker_idx = 0 + for input_args in inputs: + worker = workers[worker_idx] + worker_method = getattr(worker, method) + output = worker_method.remote(*input_args) + outputs.append(output) + worker_idx = (worker_idx + 1) % num_gpus + return ray.get(outputs) + + search_space = get_configs_compute_bound() + if block_shape is not None: + block_n, block_k = block_shape[0], block_shape[1] + search_space = [ + config for config in search_space if block_k % config["BLOCK_SIZE_K"] == 0 + ] + filename = get_config_filename( + E, + shard_intermediate_size, + hidden_size, + topk, + dtype, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + False, + block_shape, + ) + print( + f"Start tuning over {len(search_space)} configurations to create {filename}..." + ) + + start = time.perf_counter() + configs = _distribute( + "tune", + [ + ( + batch_size, + E, + shard_intermediate_size, + hidden_size, + topk, + dtype, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + block_shape, + search_space, + topk_ids_dir, + ) + for batch_size in batch_sizes + ], + ) + print(f"{configs=}", flush=True) + cur_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + with open(f"tuning_result_{cur_time}.txt", "w") as f: + print(configs, file=f) + batch_sizes.reverse() + configs0 = [config[0] for config in configs] + configs1 = [config[1] for config in configs] + configs0.reverse() + configs1.reverse() + best_configs0 = {M: sort_config(config) for M, config in zip(batch_sizes, configs0)} + save_configs_sep( + best_configs0, + E, + shard_intermediate_size, + hidden_size, + topk, + dtype, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + block_shape, + ) + + best_configs1 = {M: sort_config(config) for M, config in zip(batch_sizes, configs1)} + save_configs_sep( + best_configs1, + E, + shard_intermediate_size, + hidden_size, + topk, + dtype, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + block_shape, + down_moe=True, + ) + end = time.perf_counter() + print(f"Tuning took {end - start:.2f} seconds") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--model", type=str, default="mistralai/Mixtral-8x7B-Instruct-v0.1" + ) + parser.add_argument("--tp-size", "--tp", type=int, default=2) + parser.add_argument("--ep-size", "--ep", type=int, default=1) + parser.add_argument( + "--dtype", + type=str, + choices=["auto", "fp8_w8a8", "int8_w8a16", "int8_w8a8"], + default="auto", + ) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--batch-size", type=int, required=False) + parser.add_argument("--tune", action="store_true") + parser.add_argument("--disable-shared-experts-fusion", action="store_true") + parser.add_argument("--configs", type=int, nargs="+", required=False) + parser.add_argument("--topk-ids-dir", type=str, required=True) + args = parser.parse_args() + + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/tuning_text.json b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/tuning_text.json new file mode 100644 index 00000000..80242160 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/fused_moe_triton/tuning_text.json @@ -0,0 +1 @@ +{"prompt": "Here are the relevant Wikipedia articles:\nThe president of the United States (POTUS) is the head of state and head of government of the United States of America. The president directs the executive branch of the federal government and is the commander-in-chief of the United States Armed Forces.\nThe power of the presidency has grown substantially since the first president, George Washington, took office in 1789. While presidential power has ebbed and flowed over time, the presidency has played an increasingly significant role in American political life since the beginning of the 20th century, carrying over into the 21st century with notable expansions during the presidencies of Franklin D. Roosevelt and George W. Bush. In modern times, the president is one of the world's most powerful political figures and the leader of the world's only remaining superpower. As the leader of the nation with the largest economy by nominal GDP, the president possesses significant domestic and international hard and soft power. For much of the 20th century, especially during the Cold War, the U.S. president was often called \"the leader of the free world\".\nArticle II of the Constitution establishes the executive branch of the federal government and vests executive power in the president. The power includes the execution and enforcement of federal law and the responsibility to appoint federal executive, diplomatic, regulatory, and judicial officers. Based on constitutional provisions empowering the president to appoint and receive ambassadors and conclude treaties with foreign powers, and on subsequent laws enacted by Congress, the modern presidency has primary responsibility for conducting U.S. foreign policy. The role includes responsibility for directing the world's most expensive military, which has the second-largest nuclear arsenal.\nThe president also plays a leading role in federal legislation and domestic policymaking. As part of the system of separation of powers, Article I, Section 7 of the Constitution gives the president the power to sign or veto federal legislation. Since modern presidents are typically viewed as leaders of their political parties, major policymaking is significantly shaped by the outcome of presidential elections, with presidents taking an active role in promoting their policy priorities to members of Congress who are often electorally dependent on the president. In recent decades, presidents have also made increasing use of executive orders, agency regulations, and judicial appointments to shape domestic policy.\nThe president is elected indirectly through the Electoral College to a four-year term, along with the vice president. Under the Twenty-second Amendment, ratified in 1951, no person who has been elected to two presidential terms may be elected to a third. In addition, nine vice presidents have become president by virtue of a president's intra-term death or resignation. In all, 45 individuals have served 46 presidencies spanning 58 four-year terms. Joe Biden is the 46th and current president, having assumed office on January 20, 2021.\n\nHistory and development\nOrigins\nDuring the American Revolutionary War, the Thirteen Colonies, represented by the Second Continental Congress in Philadelphia, declared themselves to be independent sovereign states and no longer under British rule. The affirmation was made in the Declaration of Independence, which was written predominantly by Thomas Jefferson and adopted unanimously on July 4, 1776, by the Second Continental Congress. Recognizing the necessity of closely coordinating their efforts against the British, the Continental Congress simultaneously began the process of drafting a constitution that would bind the states together. There were long debates on a number of issues, including representation and voting, and the exact powers to be given the central government. Congress finished work on the Articles of Confederation to establish a perpetual union between the states in November 1777 and sent it to the states for ratification.\nUnder the Articles, which took effect on March 1, 1781, the Congress of the Confederation was a central political authority without any legislative power. It could make its own resolutions, determinations, and regulations, but not any laws, and could not impose any taxes or enforce local commercial regulations upon its citizens. This institutional design reflected how Americans believed the deposed British system of Crown and Parliament ought to have functioned with respect to the royal dominion: a superintending body for matters that concerned the entire empire. The states were out from under any monarchy and assigned some formerly royal prerogatives (e.g., making war, receiving ambassadors, etc.) to Congress; the remaining prerogatives were lodged within their own respective state governments. The members of Congress elected a president of the United States in Congress Assembled to preside over its deliberation as a neutral discussion moderator. Unrelated to and quite dissimilar from the later office of president of the United States, it was a largely ceremonial position without much influence.\nIn 1783, the Treaty of Paris secured independence for each of the former colonies. With peace at hand, the states each turned toward their own internal affairs. By 1786, Americans found their continental borders besieged and weak and their respective economies in crises as neighboring states agitated trade rivalries with one another. They witnessed their hard currency pouring into foreign markets to pay for imports, their Mediterranean commerce preyed upon by North African pirates, and their foreign-financed Revolutionary War debts unpaid and accruing interest. Civil and political unrest loomed. Events such as the Newburgh Conspiracy and Shays' Rebellion demonstrated that the Articles of Confederation were not working.\nFollowing the successful resolution of commercial and fishing disputes between Virginia and Maryland at the Mount Vernon Conference in 1785, Virginia called for a trade conference between all the states, set for September 1786 in Annapolis, Maryland, with an aim toward resolving further-reaching interstate commercial antagonisms. When the convention failed for lack of attendance due to suspicions among most of the other states, Alexander Hamilton of New York led the Annapolis delegates in a call for a convention to offer revisions to the Articles, to be held the next spring in Philadelphia. Prospects for the next convention appeared bleak until James Madison and Edmund Randolph succeeded in securing George Washington's attendance to Philadelphia as a delegate for Virginia.\nWhen the Constitutional Convention convened in May 1787, the 12 state delegations in attendance (Rhode Island did not send delegates) brought with them an accumulated experience over a diverse set of institutional arrangements between legislative and executive branches from within their respective state governments. Most states maintained a weak executive without veto or appointment powers, elected annually by the legislature to a single term only, sharing power with an executive council, and countered by a strong legislature. New York offered the greatest exception, having a strong, unitary governor with veto and appointment power elected to a three-year term, and eligible for reelection to an indefinite number of terms thereafter. It was through the closed-door negotiations at Philadelphia that the presidency framed in the U.S. Constitution emerged.\n\n1789–1933\nAs the nation's first president, George Washington established many norms that would come to define the office. His decision to retire after two terms helped address fears that the nation would devolve into monarchy, and established a precedent that would not be broken until 1940 and would eventually be made permanent by the Twenty-Second Amendment. By the end of his presidency, political parties had developed, with John Adams defeating Thomas Jefferson in 1796, the first truly contested presidential election. After Jefferson defeated Adams in 1800, he and his fellow Virginians James Madison and James Monroe would each serve two terms, eventually dominating the nation's politics during the Era of Good Feelings until Adams' son John Quincy Adams won election in 1824 after the Democratic-Republican Party split.\nThe election of Andrew Jackson in 1828 was a significant milestone, as Jackson was not part of the Virginia and Massachusetts elite that had held the presidency for its first 40 years. Jacksonian democracy sought to strengthen the presidency at the expense of Congress, while broadening public participation as the nation rapidly expanded westward. However, his successor, Martin Van Buren, became unpopular after the Panic of 1837, and the death of William Henry Harrison and subsequent poor relations between John Tyler and Congress led to further weakening of the office. Including Van Buren, in the 24 years between 1837 and 1861, six presidential terms would be filled by eight different men, with none serving two terms. The Senate played an important role during this period, with the Great Triumvirate of Henry Clay, Daniel Webster, and John C. Calhoun playing key roles in shaping national policy in the 1830s and 1840s until debates over slavery began pulling the nation apart in the 1850s.\nAbraham Lincoln's leadership during the Civil War has led historians to regard him as one of the nation's greatest presidents. The circumstances of the war and Republican domination of Congress made the office very powerful, and Lincoln's re-election in 1864 was the first time a president had been re-elected since Jackson in 1832. After Lincoln's assassination, his successor Andrew Johnson lost all political support and was nearly removed from office, with Congress remaining powerful during the two-term presidency of Civil War general Ulysses S. Grant. After the end of Reconstruction, Grover Cleveland would eventually become the first Democratic president elected since before the war, running in three consecutive elections (1884, 1888, 1892) and winning twice. In 1900, William McKinley became the first incumbent to win re-election since Grant in 1872.\nAfter McKinley's assassination by Leon Czolgosz in 1901, Theodore Roosevelt became a dominant figure in American politics. Historians believe Roosevelt permanently changed the political system by strengthening the presidency, with some key accomplishments including breaking up trusts, conservationism, labor reforms, making personal character as important as the issues, and hand-picking his successor, William Howard Taft. The following decade, Woodrow Wilson led the nation to victory during World War I, although Wilson's proposal for the League of Nations was rejected by the Senate. Warren Harding, while popular in office, would see his legacy tarnished by scandals, especially Teapot Dome, and Herbert Hoover quickly became very unpopular after failing to alleviate the Great Depression.\n\nImperial presidency\nThe ascendancy of Franklin D. Roosevelt in 1933 led further toward what historians now describe as the Imperial presidency. Backed by enormous Democratic majorities in Congress and public support for major change, Roosevelt's New Deal dramatically increased the size and scope of the federal government, including more executive agencies.: 211–12  The traditionally small presidential staff was greatly expanded, with the Executive Office of the President being created in 1939, none of whom require Senate confirmation.: 229–231  Roosevelt's unprecedented re-election to a third and fourth term, the victory of the United States in World War II, and the nation's growing economy all helped established the office as a position of global leadership.: 269  His successors, Harry Truman and Dwight D. Eisenhower, each served two terms as the Cold War led the presidency to be viewed as the \"leader of the free world\", while John F. Kennedy was a youthful and popular leader who benefited from the rise of television in the 1960s.\nAfter Lyndon B. Johnson lost popular support due to the Vietnam War and Richard Nixon's presidency collapsed in the Watergate scandal, Congress enacted a series of reforms intended to reassert itself. These included the War Powers Resolution, enacted over Nixon's veto in 1973, and the Congressional Budget and Impoundment Control Act of 1974 that sought to strengthen congressional fiscal powers. By 1976, Gerald Ford conceded that \"the historic pendulum\" had swung toward Congress, raising the possibility of a \"disruptive\" erosion of his ability to govern. Ford failed to win election to a full term and his successor, Jimmy Carter, failed to win re-election. Ronald Reagan, who had been an actor before beginning his political career, used his talent as a communicator to help reshape the American agenda away from New Deal policies toward more conservative ideology.\nWith the Cold War ending and the United States becoming the world's undisputed leading power, Bill Clinton, George W. Bush, and Barack Obama each served two terms as president. Meanwhile, Congress and the nation gradually became more politically polarized, especially following the 1994 mid-term elections that saw Republicans control the House for the first time in 40 years, and the rise of routine filibusters in the Senate in recent decades. Recent presidents have thus increasingly focused on executive orders, agency regulations, and judicial appointments to implement major policies, at the expense of legislation and congressional power. Presidential elections in the 21st century have reflected this continuing polarization, with no candidate except Obama in 2008 winning by more than five percent of the popular vote and two, George W. Bush and Donald Trump, winning in the Electoral College while losing the popular vote.\n\nCritics of presidency's evolution\nThe nation's Founding Fathers expected the Congress, which was the first branch of government described in the Constitution, to be the dominant branch of government; however, they did not expect a strong executive department. However, presidential power has shifted over time, which has resulted in claims that the modern presidency has become too powerful, unchecked, unbalanced, and \"monarchist\" in nature. In 2008 professor Dana D. Nelson expressed belief that presidents over the previous thirty years worked towards \"undivided presidential control of the executive branch and its agencies\". She criticized proponents of the unitary executive theory for expanding \"the many existing uncheckable executive powers—such as executive orders, decrees, memorandums, proclamations, national security directives and legislative signing statements—that already allow presidents to enact a good deal of foreign and domestic policy without aid, interference or consent from Congress\". Bill Wilson, board member of Americans for Limited Government, opined that the expanded presidency was \"the greatest threat ever to individual freedom and democratic rule\".\n\nLegislative powers\nArticle I, Section 1 of the Constitution vests all lawmaking power in Congress's hands, and Article 1, Section 6, Clause 2 prevents the president (and all other executive branch officers) from simultaneously being a member of Congress. Nevertheless, the modern presidency exerts significant power over legislation, both due to constitutional provisions and historical developments over time.\n\nSigning and vetoing bills\nThe president's most significant legislative power derives from the Presentment Clause, which gives the president the power to veto any bill passed by Congress. While Congress can override a presidential veto, it requires a two-thirds vote of both houses, which is usually very difficult to achieve except for widely supported bipartisan legislation. The framers of the Constitution feared that Congress would seek to increase its power and enable a \"tyranny of the majority\", so giving the indirectly elected president a veto was viewed as an important check on the legislative power. While George Washington believed the veto should only be used in cases where a bill was unconstitutional, it is now routinely used in cases where presidents have policy disagreements with a bill. The veto – or threat of a veto – has thus evolved to make the modern presidency a central part of the American legislative process.\nSpecifically, under the Presentment Clause, once a bill has been presented by Congress, the president has three options:\n\nSign the legislation within ten days, excluding Sundays, the bill becomes law.\nVeto the legislation within the above timeframe and return it to the house of Congress from which it originated, expressing any objections, the bill does not become law, unless both houses of Congress vote to override the veto by a two-thirds vote.\nTake no action on the legislation within the above timeframe—the bill becomes law, as if the president had signed it, unless Congress is adjourned at the time, in which case it does not become law, which is known as a pocket veto.\nIn 1996, Congress attempted to enhance the president's veto power with the Line Item Veto Act. The legislation empowered the president to sign any spending bill into law while simultaneously striking certain spending items within the bill, particularly any new spending, any amount of discretionary spending, or any new limited tax benefit. Congress could then repass that particular item. If the president then vetoed the new legislation, Congress could override the veto by its ordinary means, a two-thirds vote in both houses. In Clinton v. City of New York, 524 U.S. 417 (1998), the U.S. Supreme Court ruled such a legislative alteration of the veto power to be unconstitutional.\n\nSetting the agenda\nFor most of American history, candidates for president have sought election on the basis of a promised legislative agenda. Article II, Section 3, Clause 2 requires the president to recommend such measures to Congress which the president deems \"necessary and expedient\". This is done through the constitutionally-based State of the Union address, which usually outlines the president's legislative proposals for the coming year, and through other formal and informal communications with Congress.\nThe president can be involved in crafting legislation by suggesting, requesting, or even insisting that Congress enact laws that the president believes are needed. Additionally, the president can attempt to shape legislation during the legislative process by exerting influence on individual members of Congress. Presidents possess this power because the Constitution is silent about who can write legislation, but the power is limited because only members of Congress can introduce legislation.\nThe president or other officials of the executive branch may draft legislation and then ask senators or representatives to introduce these drafts into Congress. Additionally, the president may attempt to have Congress alter proposed legislation by threatening to veto that legislation unless requested changes are made.\n\nPromulgating regulations\nMany laws enacted by Congress do not address every possible detail, and either explicitly or implicitly delegate powers of implementation to an appropriate federal agency. As the head of the executive branch, presidents control a vast array of agencies that can issue regulations with little oversight from Congress.\nIn the 20th century, critics charged that too many legislative and budgetary powers that should have belonged to Congress had slid into the hands of presidents. One critic charged that presidents could appoint a \"virtual army of 'czars'—each wholly unaccountable to Congress yet tasked with spearheading major policy efforts for the White House\". Presidents have been criticized for making signing statements when signing congressional legislation about how they understand a bill or plan to execute it. This practice has been criticized by the American Bar Association as unconstitutional. Conservative commentator George Will wrote of an \"increasingly swollen executive branch\" and \"the eclipse of Congress\".\n\nConvening and adjourning Congress\nTo allow the government to act quickly in case of a major domestic or international crisis arising when Congress is not in session, the president is empowered by Article II, Section 3 of the Constitution to call a special session of one or both houses of Congress. Since John Adams first did so in 1797, the president has called the full Congress to convene for a special session on 27 occasions. Harry S. Truman was the most recent to do so in July 1948, known as the Turnip Day Session. In addition, prior to ratification of the Twentieth Amendment in 1933, which brought forward the date on which Congress convenes from December to January, newly inaugurated presidents would routinely call the Senate to meet to confirm nominations or ratify treaties. In practice, the power has fallen into disuse in the modern era as Congress now formally remains in session year-round, convening pro forma sessions every three days even when ostensibly in recess. Correspondingly, the president is authorized to adjourn Congress if the House and Senate cannot agree on the time of adjournment; no president has ever had to exercise this power.\n\nExecutive powers\nThe president is head of the executive branch of the federal government and is constitutionally obligated to \"take care that the laws be faithfully executed\". The executive branch has over four million employees, including the military.\n\nAdministrative powers\nPresidents make political appointments. An incoming president may make up to 4,000 upon taking office, 1200 of which must be confirmed by the U.S. Senate. Ambassadors, members of the Cabinet, and various officers, are among the positions filled by presidential appointment with Senate confirmation.\nThe power of a president to fire executive officials has long been a contentious political issue. Generally, a president may remove executive officials at will. However, Congress can curtail and constrain a president's authority to fire commissioners of independent regulatory agencies and certain inferior executive officers by statute.\nTo manage the growing federal bureaucracy, presidents have gradually surrounded themselves with many layers of staff, who were eventually organized into the Executive Office of the President of the United States. Within the Executive Office, the president's innermost layer of aides, and their assistants, are located in the White House Office.\nThe president also possesses the power to manage operations of the federal government by issuing various types of directives, such as presidential proclamation and executive orders. When the president is lawfully exercising one of the constitutionally conferred presidential responsibilities, the scope of this power is broad. Even so, these directives are subject to judicial review by U.S. federal courts, which can find them to be unconstitutional. Congress can overturn an executive order through legislation.\n\nForeign affairs\nArticle II, Section 3, Clause 4 requires the president to \"receive Ambassadors.\" This clause, known as the Reception Clause, has been interpreted to imply that the president possesses broad power over matters of foreign policy, and to provide support for the president's exclusive authority to grant recognition to a foreign government. The Constitution also empowers the president to appoint United States ambassadors, and to propose and chiefly negotiate agreements between the United States and other countries. Such agreements, upon receiving the advice and consent of the U.S. Senate (by a two-thirds majority vote), become binding with the force of federal law.\nWhile foreign affairs has always been a significant element of presidential responsibilities, advances in technology since the Constitution's adoption have increased presidential power. Where formerly ambassadors were vested with significant power to independently negotiate on behalf of the United States, presidents now routinely meet directly with leaders of foreign countries.\n\nCommander-in-chief\nOne of the most important of executive powers is the president's role as commander-in-chief of the United States Armed Forces. The power to declare war is constitutionally vested in Congress, but the president has ultimate responsibility for the direction and disposition of the military. The exact degree of authority that the Constitution grants to the president as commander-in-chief has been the subject of much debate throughout history, with Congress at various times granting the president wide authority and at others attempting to restrict that authority. The framers of the Constitution took care to limit the president's powers regarding the military; Alexander Hamilton explained this in Federalist No. 69:The President is to be commander-in-chief of the army and navy of the United States. ... It would amount to nothing more than the supreme command and direction of the military and naval forces ... while that [the power] of the British king extends to the DECLARING of war and to the RAISING and REGULATING of fleets and armies, all [of] which ... would appertain to the legislature. [Emphasis in the original.]\nIn the modern era, pursuant to the War Powers Resolution, Congress must authorize any troop deployments longer than 60 days, although that process relies on triggering mechanisms that have never been employed, rendering it ineffectual. Additionally, Congress provides a check to presidential military power through its control over military spending and regulation. Presidents have historically initiated the process for going to war, but critics have charged that there have been several conflicts in which presidents did not get official declarations, including Theodore Roosevelt's military move into Panama in 1903, the Korean War, the Vietnam War, and the invasions of Grenada in 1983 and Panama in 1989.\nThe amount of military detail handled personally by the president in wartime has varied greatly. George Washington, the first U.S. president, firmly established military subordination under civilian authority. In 1794, Washington used his constitutional powers to assemble 12,000 militia to quell the Whiskey Rebellion, a conflict in Western Pennsylvania involving armed farmers and distillers who refused to pay an excise tax on spirits. According to historian Joseph Ellis, this was the \"first and only time a sitting American president led troops in the field\", though James Madison briefly took control of artillery units in defense of Washington, D.C., during the War of 1812. Abraham Lincoln was deeply involved in overall strategy and in day-to-day operations during the American Civil War, 1861–1865; historians have given Lincoln high praise for his strategic sense and his ability to select and encourage commanders such as Ulysses S. Grant.\nThe present-day operational command of the Armed Forces is delegated to the Department of Defense and is normally exercised through the secretary of defense. The chairman of the Joint Chiefs of Staff and the Combatant Commands assist with the operation as outlined in the presidentially approved Unified Command Plan (UCP).\n\nJuridical powers and privileges\nThe president has the power to nominate federal judges, including members of the United States courts of appeals and the Supreme Court of the United States. However, these nominations require Senate confirmation before they may take office. Securing Senate approval can provide a major obstacle for presidents who wish to orient the federal judiciary toward a particular ideological stance. When nominating judges to U.S. district courts, presidents often respect the long-standing tradition of senatorial courtesy. Presidents may also grant pardons and reprieves. Gerald Ford pardoned Richard Nixon a month after taking office. Presidents often grant pardons shortly before leaving office, like when Bill Clinton pardoned Patty Hearst on his last day in office; this is often controversial.\nTwo doctrines concerning executive power have developed that enable the president to exercise executive power with a degree of autonomy. The first is executive privilege, which allows the president to withhold from disclosure any communications made directly to the president in the performance of executive duties. George Washington first claimed the privilege when Congress requested to see Chief Justice John Jay's notes from an unpopular treaty negotiation with Great Britain. While not enshrined in the Constitution or any other law, Washington's action created the precedent for the privilege. When Nixon tried to use executive privilege as a reason for not turning over subpoenaed evidence to Congress during the Watergate scandal, the Supreme Court ruled in United States v. Nixon, 418 U.S. 683 (1974), that executive privilege did not apply in cases where a president was attempting to avoid criminal prosecution. When Bill Clinton attempted to use executive privilege regarding the Lewinsky scandal, the Supreme Court ruled in Clinton v. Jones, 520 U.S. 681 (1997), that the privilege also could not be used in civil suits. These cases established the legal precedent that executive privilege is valid, although the exact extent of the privilege has yet to be clearly defined. Additionally, federal courts have allowed this privilege to radiate outward and protect other executive branch employees but have weakened that protection for those executive branch communications that do not involve the president.\nThe state secrets privilege allows the president and the executive branch to withhold information or documents from discovery in legal proceedings if such release would harm national security. Precedent for the privilege arose early in the 19th century when Thomas Jefferson refused to release military documents in the treason trial of Aaron Burr and again in Totten v. United States 92 U.S. 105 (1876), when the Supreme Court dismissed a case brought by a former Union spy. However, the privilege was not formally recognized by the U.S. Supreme Court until United States v. Reynolds 345 U.S. 1 (1953), where it was held to be a common law evidentiary privilege. Before the September 11 attacks, use of the privilege had been rare, but increasing in frequency. Since 2001, the government has asserted the privilege in more cases and at earlier stages of the litigation, thus in some instances causing dismissal of the suits before reaching the merits of the claims, as in the Ninth Circuit's ruling in Mohamed v. Jeppesen Dataplan, Inc. Critics of the privilege claim its use has become a tool for the government to cover up illegal or embarrassing government actions.\nThe degree to which the president personally has absolute immunity from court cases is contested and has been the subject of several Supreme Court decisions. Nixon v. Fitzgerald (1982) dismissed a civil lawsuit against by-then former president Richard Nixon based on his official actions. Clinton v. Jones (1997) decided that a president has no immunity against civil suits for actions taken before becoming president and ruled that a sexual harassment suit could proceed without delay, even against a sitting president. The 2019 Mueller report on Russian interference in the 2016 presidential election detailed evidence of possible obstruction of justice, but investigators declined to refer Donald Trump for prosecution based on a United States Department of Justice policy against indicting an incumbent president. The report noted that impeachment by Congress was available as a remedy. As of October 2019, a case was pending in the federal courts regarding access to personal tax returns in a criminal case brought against Donald Trump by the New York County District Attorney alleging violations of New York state law.\n\nLeadership roles\nHead of state\nAs head of state, the president represents the United States government to its own people and represents the nation to the rest of the world. For example, during a state visit by a foreign head of state, the president typically hosts a State Arrival Ceremony held on the South Lawn, a custom begun by John F. Kennedy in 1961. This is followed by a state dinner given by the president which is held in the State Dining Room later in the evening.\n\nAs a national leader, the president also fulfills many less formal ceremonial duties. For example, William Howard Taft started the tradition of throwing out the ceremonial first pitch in 1910 at Griffith Stadium, Washington, D.C., on the Washington Senators's Opening Day. Every president since Taft, except for Jimmy Carter, threw out at least one ceremonial first ball or pitch for Opening Day, the All-Star Game, or the World Series, usually with much fanfare. Every president since Theodore Roosevelt has served as honorary president of the Boy Scouts of America.\nOther presidential traditions are associated with American holidays. Rutherford B. Hayes began in 1878 the first White House egg rolling for local children. Beginning in 1947, during the Harry S. Truman administration, every Thanksgiving the president is presented with a live domestic turkey during the annual National Thanksgiving Turkey Presentation held at the White House. Since 1989, when the custom of \"pardoning\" the turkey was formalized by George H. W. Bush, the turkey has been taken to a farm where it will live out the rest of its natural life.\nPresidential traditions also involve the president's role as head of government. Many outgoing presidents since James Buchanan traditionally give advice to their successor during the presidential transition. Ronald Reagan and his successors have also left a private message on the desk of the Oval Office on Inauguration Day for the incoming president.\nThe modern presidency holds the president as one of the nation's premier celebrities. Some argue that images of the presidency have a tendency to be manipulated by administration public relations officials as well as by presidents themselves. One critic described the presidency as \"propagandized leadership\" which has a \"mesmerizing power surrounding the office\". Administration public relations managers staged carefully crafted photo-ops of smiling presidents with smiling crowds for television cameras. One critic wrote the image of John F. Kennedy was described as carefully framed \"in rich detail\" which \"drew on the power of myth\" regarding the incident of PT 109 and wrote that Kennedy understood how to use images to further his presidential ambitions. As a result, some political commentators have opined that American voters have unrealistic expectations of presidents: voters expect a president to \"drive the economy, vanquish enemies, lead the free world, comfort tornado victims, heal the national soul and protect borrowers from hidden credit-card fees\".\n\nHead of party\nThe president is typically considered to be the head of their political party. Since the entire House of Representatives and at least one-third of the Senate is elected simultaneously with the president, candidates from a political party inevitably have their electoral success intertwined with the performance of the party's presidential candidate. The coattail effect, or lack thereof, will also often impact a party's candidates at state and local levels of government as well. However, there are often tensions between a president and others in the party, with presidents who lose significant support from their party's caucus in Congress generally viewed to be weaker and less effective.\n\nGlobal leader\nWith the rise of the United States as a superpower in the 20th century, and the United States having the world's largest economy into the 21st century, the president is typically viewed as a global leader, and at times the world's most powerful political figure. The position of the United States as the leading member of NATO, and the country's strong relationships with other wealthy or democratic nations like those comprising the European Union, have led to the moniker that the president is the \"leader of the free world\".\n\nSelection process\nEligibility\nArticle II, Section 1, Clause 5 of the Constitution sets three qualifications for holding the presidency. To serve as president, one must:\n\nbe a natural-born citizen of the United States;\nbe at least 35 years old;\nbe a resident in the United States for at least 14 years.\nA person who meets the above qualifications would, however, still be disqualified from holding the office of president under any of the following conditions:\n\nUnder Article I, Section 3, Clause 7, having been impeached, convicted and disqualified from holding further public office, although there is some legal debate as to whether the disqualification clause also includes the presidential office: the only previous persons disqualified under this clause were three federal judges.\nUnder Section 3 of the Fourteenth Amendment, no person who swore an oath to support the Constitution, and later rebelled against the United States, is eligible to hold any office. However, this disqualification can be lifted by a two-thirds vote of each house of Congress. There is, again, some debate as to whether the clause as written allows disqualification from the presidential position, or whether it would first require litigation outside of Congress, although there is precedent for use of this amendment outside of the original intended purpose of excluding Confederates from public office after the Civil War.\nUnder the Twenty-second Amendment, no person can be elected president more than twice. The amendment also specifies that if any eligible person serves as president or acting president for more than two years of a term for which some other eligible person was elected president, the former can only be elected president once.\n\nCampaigns and nomination\nThe modern presidential campaign begins before the primary elections, which the two major political parties use to clear the field of candidates before their national nominating conventions, where the most successful candidate is made the party's presidential nominee. Typically, the party's presidential candidate chooses a vice presidential nominee, and this choice is rubber-stamped by the convention. The most common previous profession of presidents is lawyer.\nNominees participate in nationally televised debates, and while the debates are usually restricted to the Democratic and Republican nominees, third party candidates may be invited, such as Ross Perot in the 1992 debates. Nominees campaign across the country to explain their views, convince voters and solicit contributions. Much of the modern electoral process is concerned with winning swing states through frequent visits and mass media advertising drives.\n\nElection\nThe president is elected indirectly by the voters of each state and the District of Columbia through the Electoral College, a body of electors formed every four years for the sole purpose of electing the president and vice president to concurrent four-year terms. As prescribed by Article II, Section 1, Clause 2, each state is entitled to a number of electors equal to the size of its total delegation in both houses of Congress. Additionally, the Twenty-third Amendment provides that the District of Columbia is entitled to the number it would have if it were a state, but in no case more than that of the least populous state. Currently, all states and the District of Columbia select their electors based on a popular election. In all but two states, the party whose presidential–vice presidential ticket receives a plurality of popular votes in the state has its entire slate of elector nominees chosen as the state's electors. Maine and Nebraska deviate from this winner-take-all practice, awarding two electors to the statewide winner and one to the winner in each congressional district.\nOn the first Monday after the second Wednesday in December, about six weeks after the election, the electors convene in their respective state capitals (and in Washington, D.C.) to vote for president and, on a separate ballot, for vice president. They typically vote for the candidates of the party that nominated them. While there is no constitutional mandate or federal law requiring them to do so, the District of Columbia and 32 states have laws requiring that their electors vote for the candidates to whom they are pledged. The constitutionality of these laws was upheld in Chiafalo v. Washington (2020). Following the vote, each state then sends a certified record of their electoral votes to Congress. The votes of the electors are opened and counted during a joint session of Congress, held in the first week of January. If a candidate has received an absolute majority of electoral votes for president (currently 270 of 538), that person is declared the winner. Otherwise, the House of Representatives must meet to elect a president using a contingent election procedure in which representatives, voting by state delegation, with each state casting a single vote, choose between the top three electoral vote-getters for president. To win the presidency, a candidate must receive the votes of an absolute majority of states (currently 26 of 50).\nThere have been two contingent presidential elections in the nation's history. A 73–73 electoral vote tie between Thomas Jefferson and fellow Democratic-Republican Aaron Burr in the election of 1800 necessitated the first. Conducted under the original procedure established by Article II, Section 1, Clause 3 of the Constitution, which stipulates that if two or three persons received a majority vote and an equal vote, the House of Representatives would choose one of them for president; the runner-up would become vice president. On February 17, 1801, Jefferson was elected president on the 36th ballot, and Burr elected vice president. Afterward, the system was overhauled through the Twelfth Amendment in time to be used in the 1804 election. A quarter-century later, the choice for president again devolved to the House when no candidate won an absolute majority of electoral votes (131 of 261) in the election of 1824. Under the Twelfth Amendment, the House was required to choose a president from among the top three electoral vote recipients: Andrew Jackson, John Quincy Adams, and William H. Crawford. Held February 9, 1825, this second and most recent contingent election resulted in John Quincy Adams being elected president on the first ballot.\n\nInauguration\nPursuant to the Twentieth Amendment, the four-year term of office for both the president and the vice president begins at noon on January 20, in the year following the preceding presidential election. The first presidential and vice presidential terms to begin on this date, known as Inauguration Day, were the second terms of President Franklin D. Roosevelt and Vice President John Nance Garner in 1937. Previously, Inauguration Day was on March 4. As a result of the date change, the first term (1933–37) of both men had been shortened by 43 days.\nBefore executing the powers of the office, a president is required to recite the presidential Oath of Office, found in Article II, Section 1, Clause 8 of the Constitution. This is the only component in the inauguration ceremony mandated by the Constitution:\n\nI do solemnly swear (or affirm) that I will faithfully execute the Office of President of the United States, and will to the best of my ability, preserve, protect, and defend the Constitution of the United States.\nPresidents have traditionally placed one hand upon a Bible while taking the oath, and have added \"So help me God\" to the end of the oath. Although the oath may be administered by any person authorized by law to administer oaths, presidents are traditionally sworn in by the chief justice of the United States.\n\nIncumbency\nTerm limit\nWhen the first president, George Washington, announced in his Farewell Address that he was not running for a third term, he established a \"two terms then out\" precedent. Precedent became tradition after Thomas Jefferson publicly embraced the principle a decade later during his second term, as did his two immediate successors, James Madison and James Monroe. In spite of the strong two-term tradition, Ulysses S. Grant sought nomination at the 1880 Republican National Convention for a non-consecutive third term, but was unsuccessful.\nIn 1940, after leading the nation through the Great Depression and focused on supporting U.S. allied nations at war with the Axis powers, Franklin Roosevelt was elected to a third term, breaking the long-standing precedent. Four years later, with the U.S. engaged in World War II, he was re-elected again despite his declining physical health; he died 82 days into his fourth term on April 12, 1945.\nIn response to the unprecedented length of Roosevelt's presidency, the Twenty-second Amendment was adopted in 1951. The amendment bars anyone from being elected president more than twice, or once if that person served more than two years (24 months) of another president's four-year term. Harry S. Truman, the president at the time it was submitted to the states by the Congress, was exempted from its limitations. Without the exemption, he would not have been eligible to run for a second full term in 1952 (which he briefly sought), as he had served nearly all of Franklin Roosevelt's unexpired 1945–1949 term and had been elected to a full four-year term beginning in 1949. Since becoming operative in 1951, the amendment has been applicable to six twice-elected presidents: Dwight D. Eisenhower, Richard Nixon, Ronald Reagan, Bill Clinton, George W. Bush, and Barack Obama.\n\nVacancies and succession\nUnder Section 1 of the Twenty-fifth Amendment, ratified in 1967, the vice president becomes president upon the removal from office, death, or resignation of the president. Deaths have occurred a number of times, resignation has occurred only once, and removal from office has never occurred.\nBefore the ratification of the Twenty-fifth amendment (which clarified the matter of succession), Article II, Section 1, Clause 6, stated only that the vice president assumes the \"powers and duties\" of the presidency in the event of a president's removal, death, resignation, or inability. Under this clause, there was ambiguity about whether the vice president would actually become president in the event of a vacancy, or simply act as president, potentially resulting in a special election. Upon the death of President William Henry Harrison in 1841, Vice President John Tyler declared that he had succeeded to the office itself, refusing to accept any papers addressed to the \"Acting President\", and Congress ultimately accepted it.\nIn the event of a double vacancy, Article II, Section 1, Clause 6 also authorizes Congress to declare who shall become acting president in the \"Case of Removal, Death, Resignation or Inability, both of the president and vice president\". The Presidential Succession Act of 1947 (codified as 3 U.S.C. § 19) provides that if both the president and vice president have left office or are both otherwise unavailable to serve during their terms of office, the presidential line of succession follows the order of: speaker of the House, then, if necessary, the president pro tempore of the Senate, and then if necessary, the eligible heads of federal executive departments who form the president's cabinet. The cabinet currently has 15 members, of which the secretary of state is first in line; the other Cabinet secretaries follow in the order in which their department (or the department of which their department is the successor) was created. Those individuals who are constitutionally ineligible to be elected to the presidency are also disqualified from assuming the powers and duties of the presidency through succession. No statutory successor has yet been called upon to act as president.\n\nDeclarations of inability\nUnder the Twenty-fifth Amendment, the president may temporarily transfer the presidential powers and duties to the vice president, who then becomes acting president, by transmitting to the speaker of the House and the president pro tempore of the Senate a statement that he is unable to discharge his duties. The president resumes his or her powers upon transmitting a second declaration stating that he is again able. The mechanism has been used by Ronald Reagan (once), George W. Bush (twice), and Joe Biden (once), each in anticipation of surgery.\nThe Twenty-fifth Amendment also provides that the vice president, together with a majority of certain members of the Cabinet, may transfer the presidential powers and duties to the vice president by transmitting a written declaration, to the speaker of the House and the president pro tempore of the Senate, to the effect that the president is unable to discharge his or her powers and duties. If the president then declares that no such inability exist, he or she resumes the presidential powers unless the vice president and Cabinet make a second declaration of presidential inability, in which case Congress decides the question.\n\nRemoval\nArticle II, Section 4 of the Constitution allows for the removal of high federal officials, including the president, from office for \"treason, bribery, or other high crimes and misdemeanors\". Article I, Section 2, Clause 5 authorizes the House of Representatives to serve as a \"grand jury\" with the power to impeach said officials by a majority vote. Article I, Section 3, Clause 6 authorizes the Senate to serve as a court with the power to remove impeached officials from office, by a two-thirds vote to convict.\nThree presidents have been impeached by the House of Representatives: Andrew Johnson in 1868, Bill Clinton in 1998, and Donald Trump in 2019 and 2021; none have been convicted by the Senate. Additionally, the House Judiciary Committee conducted an impeachment inquiry against Richard Nixon in 1973–74 and reported three articles of impeachment to the House of Representatives for final action; however, he resigned from office before the House voted on them.\n\nCircumvention of authority\nControversial measures have sometimes been taken short of removal to deal with perceived recklessness on the part of the president, or with a long-term disability. In some cases, staff have intentionally failed to deliver messages to or from the president, typically to avoid executing or promoting the president to write certain orders. This has ranged from Richard Nixon's Chief of Staff not transmitting orders to the Cabinet due to the president's heavy drinking, to staff removing memos from Donald Trump's desk. Decades before the Twenty-fifth Amendment, in 1919, President Woodrow Wilson had a stroke that left him partly incapacitated. First lady Edith Wilson kept this condition a secret from the public for a while, and controversially became the sole gatekeeper for access to the president (aside from his doctor), assisting him with paperwork and deciding which information was \"important\" enough to share with him.\n\nCompensation\nSince 2001, the president's annual salary has been $400,000, along with a: $50,000 expense allowance; $100,000 nontaxable travel account, and $19,000 entertainment account. The president's salary is set by Congress, and under Article II, Section 1, Clause 7 of the Constitution, any increase or reduction in presidential salary cannot take effect before the next presidential term of office.\n\nResidence\nThe Executive Residence of the White House in Washington, D.C. is the official residence of the president. The site was selected by George Washington, and the cornerstone was laid in 1792. Every president since John Adams (in 1800) has lived there. At various times in U.S. history, it has been known as the \"President's Palace\", the \"President's House\", and the \"Executive Mansion\". Theodore Roosevelt officially gave the White House its current name in 1901. The federal government pays for state dinners and other official functions, but the president pays for personal, family, and guest dry cleaning and food.\nCamp David, officially titled Naval Support Facility Thurmont, a mountain-based military camp in Frederick County, Maryland, is the president's country residence. A place of solitude and tranquility, the site has been used extensively to host foreign dignitaries since the 1940s.\nPresident's Guest House, located next to the Eisenhower Executive Office Building at the White House Complex and Lafayette Park, serves as the president's official guest house and as a secondary residence for the president if needed. Four interconnected, 19th-century houses—Blair House, Lee House, and 700 and 704 Jackson Place—with a combined floor space exceeding 70,000 square feet (6,500 m2) comprise the property.\n\n\tPresidential residences\n\nTravel\nThe primary means of long-distance air travel for the president is one of two identical Boeing VC-25 aircraft, which are extensively modified Boeing 747 airliners and are referred to as Air Force One while the president is on board (although any U.S. Air Force aircraft the president is aboard is designated as \"Air Force One\" for the duration of the flight). In-country trips are typically handled with just one of the two planes, while overseas trips are handled with both, one primary and one backup. The president also has access to smaller Air Force aircraft, most notably the Boeing C-32, which are used when the president must travel to airports that cannot support a jumbo jet. Any civilian aircraft the president is aboard is designated Executive One for the flight.\nFor short-distance air travel, the president has access to a fleet of U.S. Marine Corps helicopters of varying models, designated Marine One when the president is aboard any particular one in the fleet. Flights are typically handled with as many as five helicopters all flying together and frequently swapping positions as to disguise which helicopter the president is actually aboard to any would-be threats.\nFor ground travel, the president uses the presidential state car, which is an armored limousine designed to look like a Cadillac sedan, but built on a truck chassis. The U.S. Secret Service operates and maintains the fleet of several limousines. The president also has access to two armored motorcoaches, which are primarily used for touring trips.\n\n\tPresidential transportation\n\nProtection\nThe U.S. Secret Service is charged with protecting the president and the first family. As part of their protection, presidents, first ladies, their children and other immediate family members, and other prominent persons and locations are assigned Secret Service codenames. The use of such names was originally for security purposes and dates to a time when sensitive electronic communications were not routinely encrypted; today, the names simply serve for purposes of brevity, clarity, and tradition.\n\nPost-presidency\nActivities\nSome former presidents have had significant careers after leaving office. Prominent examples include William Howard Taft's tenure as chief justice of the United States and Herbert Hoover's work on government reorganization after World War II. Grover Cleveland, whose bid for reelection failed in 1888, was elected president again four years later in 1892. Two former presidents served in Congress after leaving the White House: John Quincy Adams was elected to the House of Representatives, serving there for 17 years, and Andrew Johnson returned to the Senate in 1875, though he died soon after. Some ex-presidents were very active, especially in international affairs, most notably Theodore Roosevelt; Herbert Hoover; Richard Nixon; and Jimmy Carter.\nPresidents may use their predecessors as emissaries to deliver private messages to other nations or as official representatives of the United States to state funerals and other important foreign events. Richard Nixon made multiple foreign trips to countries including China and Russia and was lauded as an elder statesman. Jimmy Carter has become a global human rights campaigner, international arbiter, and election monitor, as well as a recipient of the Nobel Peace Prize. Bill Clinton has also worked as an informal ambassador, most recently in the negotiations that led to the release of two American journalists, Laura Ling and Euna Lee, from North Korea. During his presidency, George W. Bush called on former Presidents Bush and Clinton to assist with humanitarian efforts after the 2004 Indian Ocean earthquake and tsunami. President Obama followed suit by asking Presidents Clinton and Bush to lead efforts to aid Haiti after an earthquake devastated that country in 2010.\nClinton was active politically since his presidential term ended, working with his wife Hillary on her 2008 and 2016 presidential bids and President Obama on his 2012 reelection campaign. Obama was also active politically since his presidential term ended, having worked with his former vice president Joe Biden on his 2020 election campaign. Trump has continued to make appearances in the media and at conferences and rallies since leaving office in 2021. He is currently running for a non-consecutive second term in the upcoming 2024 presidential election.\n\nPension and other benefits\nThe Former Presidents Act (FPA), enacted in 1958, grants lifetime benefits to former presidents and their widows, including a monthly pension, medical care in military facilities, health insurance, and Secret Service protection; also provided is funding for a certain number of staff and for office expenses. The act has been amended several times to provide increases in presidential pensions and in the allowances for office staff. The FPA excludes any president who was removed from office by impeachment.\nAccording to a 2008 report by the Congressional Research Service:\n\nChief executives leaving office prior to 1958 often entered retirement pursuing various occupations and received no federal assistance. When industrialist Andrew Carnegie announced a plan in 1912 to offer $25,000 annual pensions to former Presidents, many Members of Congress deemed it inappropriate that such a pension would be provided by a private corporation executive. That same year, legislation was first introduced to create presidential pensions, but it was not enacted. In 1955, such legislation was considered by Congress because of former President Harry S. Truman's financial limitations in hiring an office staff\nThe pension has increased numerous times with congressional approval. Retired presidents receive a pension based on the salary of the current administration's cabinet secretaries, which was $199,700 per year in 2012. Former presidents who served in Congress may also collect congressional pensions. The act also provides former presidents with travel funds and franking privileges.\nPrior to 1997, all former presidents, their spouses, and their children until age 16 were protected by the Secret Service until the president's death. In 1997, Congress passed legislation limiting Secret Service protection to no more than 10 years from the date a president leaves office. On January 10, 2013, President Obama signed legislation reinstating lifetime Secret Service protection for him, George W. Bush, and all subsequent presidents. A first spouse who remarries is no longer eligible for Secret Service protection.\n\nPresidential libraries\nEvery president since Herbert Hoover has created a repository known as a presidential library for preserving and making available his papers, records, and other documents and materials. Completed libraries are deeded to and maintained by the National Archives and Records Administration (NARA); the initial funding for building and equipping each library must come from private, non-federal sources. There are currently thirteen presidential libraries in the NARA system. There are also presidential libraries maintained by state governments and private foundations and Universities of Higher Education, including:\n\nThe Abraham Lincoln Presidential Library and Museum, which is run by the State of Illinois;\nThe George W. Bush Presidential Library and Museum, which is run by Southern Methodist University;\nThe George H. W. Bush Presidential Library and Museum, which is run by Texas A&M University; and\nThe Lyndon Baines Johnson Presidential Library and Museum, which is run by the University of Texas at Austin.\nSeveral former presidents have overseen the building and opening of their own presidential libraries. Some even made arrangements for their own burial at the site. Several presidential libraries contain the graves of the president they document: \n\nThe Harry S. Truman Presidential Library and Museum in Independence, Missouri;\nThe Dwight D. Eisenhower Presidential Library, Museum and Boyhood Home in Abilene, Kansas;\nThe Richard Nixon Presidential Library and Museum in Yorba Linda, California; and\nThe Ronald Reagan Presidential Library and Museum in Simi Valley, California.\nThese gravesites are open to the general public.\n\nPolitical affiliation\nPolitical parties have dominated American politics for most of the nation's history. Though the Founding Fathers generally spurned political parties as divisive and disruptive, and their rise had not been anticipated when the U.S. Constitution was drafted in 1787, organized political parties developed in the U.S. in the mid-1790s nonetheless. They evolved from political factions, which began to appear almost immediately after the Federal government came into existence. Those who supported the Washington administration were referred to as \"pro-administration\" and would eventually form the Federalist Party, while those in opposition largely joined the emerging Democratic-Republican Party.\nGreatly concerned about the very real capacity of political parties to destroy the fragile unity holding the nation together, Washington remained unaffiliated with any political faction or party throughout his eight-year presidency. He was, and remains, the only U.S. president never to be affiliated with a political party. Since Washington, every U.S. president has been affiliated with a political party at the time of assuming office.\nThe number of presidents per political party by their affiliation at the time they were first sworn into office (alphabetical, by last name) are:\n\nTimeline of presidents\nThe following timeline depicts the progression of the presidents and their political affiliation at the time of assuming office.\n\nSee also\nOutline of American politics\n\nNotes\nReferences\nFurther reading\nExternal links\n\nWhite House homepage\nUnited States Presidents Collection. General Collection, Beinecke Rare Book and Manuscript Library, Yale University\n\nJames Buchanan Jr. ( bew-KAN-ən; April 23, 1791 – June 1, 1868) was the 15th president of the United States, serving from 1857 to 1861. Buchanan also served as the secretary of state from 1845 to 1849 and represented Pennsylvania in both houses of the U.S. Congress. He was an advocate for states' rights, particularly regarding slavery, and minimized the role of the federal government preceding the Civil War.\nBuchanan was a lawyer in Pennsylvania and won his first election to the state's House of Representatives as a Federalist. He was elected to the U.S. House of Representatives in 1820 and retained that post for five terms, aligning with Andrew Jackson's Democratic Party. Buchanan served as Jackson's minister to Russia in 1832. He won the election in 1834 as a U.S. senator from Pennsylvania and continued in that position for 11 years. He was appointed to serve as President James K. Polk's secretary of state in 1845, and eight years later was named as President Franklin Pierce's minister to the United Kingdom.\nBeginning in 1844, Buchanan became a regular contender for the Democratic Party's presidential nomination. He was nominated and won the 1856 presidential election. As President, Buchanan intervened to assure the Supreme Court's majority ruling in the pro-slavery decision in the Dred Scott case. He acceded to Southern attempts to engineer Kansas' entry into the Union as a slave state under the Lecompton Constitution, and angered not only Republicans but also Northern Democrats. Buchanan honored his pledge to serve only one term and supported Breckinridge's unsuccessful candidacy in the 1860 presidential election. He failed to reconcile the fractured Democratic Party amid the grudge against Stephen Douglas, leading to the election of Republican and former Congressman Abraham Lincoln.\nBuchanan's leadership during his lame duck period, before the American Civil War, has been widely criticized. He simultaneously angered the North by not stopping secession and the South by not yielding to their demands. He supported the Corwin Amendment in an effort to reconcile the country. He made an unsuccessful attempt to reinforce Fort Sumter, but otherwise refrained from preparing the military. In his personal life, Buchanan never married and was the only U.S. president to remain a lifelong bachelor, leading some historians and authors to question his sexual orientation. His failure to forestall the Civil War has been described as incompetence, and he spent his last years defending his reputation. Historians and scholars rank Buchanan as among the worst presidents in American history.\n\nEarly life\nChildhood and education\nJames Buchanan Jr. was born into a Scottish-Irish family on April 23, 1791, in a log cabin on a farm called Stony Batter, near Cove Gap, Peters Township, in the Allegheny Mountains of southern Pennsylvania. He was the last president born in the 18th century and, until the election of Joe Biden in 2020, the only one born in Pennsylvania. Buchanan was the second of eleven children with six sisters and four brothers, and the eldest son of James Buchanan Sr. (1761–1821) and his wife Elizabeth Speer (1767–1833). James Buchanan Sr., was an Ulster-Scot from just outside Ramelton, a small town in the north-east of County Donegal in the north-west of Ulster, the northern province in Ireland, who emigrated to the newly formed United States in 1783, having sailed from Derry. He belonged to the Clan Buchanan, whose members had emigrated in large numbers from the Scottish Highlands to Ulster in the north of Ireland during the Plantation of Ulster in the seventeenth century and, later, largely because of poverty and persecution by the Crown due to their Presbyterian faith, had further emigrated in large numbers from Ulster to America from the early eighteenth century onwards. Shortly after Buchanan's birth, the family relocated to a farm near Mercersburg, Pennsylvania, and later settled in the town in 1794. His father became the area's wealthiest resident, working as a merchant, farmer, and real estate investor. Buchanan attributed his early education primarily to his mother, whereas his father had a greater influence on his character. His mother had discussed politics with him as a child and had an interest in poetry, quoting John Milton and William Shakespeare to Buchanan.\nBuchanan attended the Old Stone Academy in Mercersburg and then Dickinson College in Carlisle, Pennsylvania. In 1808, he was nearly expelled for disorderly conduct; he and his fellow students had attracted negative attention for drinking in local taverns, disturbing the peace at night and committing acts of vandalism, but he pleaded for a second chance and ultimately graduated with honors in 1809. Later that year, he moved to the state capital at Lancaster, to train as a lawyer for two and a half years with the well-known James Hopkins. Following the fashion of the time, Buchanan studied the United States Code and the Constitution of the United States as well as legal authorities such as William Blackstone during his education.\n\nEarly law practice and Pennsylvania House of Representatives\nIn 1812, Buchanan passed the bar exam and after being admitted to the bar, he remained in Lancaster, even when Harrisburg became the new capital of Pennsylvania. Buchanan quickly established himself as a prominent legal representative in the city. His income rapidly rose after he established his practice, and by 1821 he was earning over $11,000 per year (equivalent to $250,000 in 2023). At this time, Buchanan became a Freemason, and served as the Worshipful Master of Masonic Lodge No. 43 in Lancaster and as a District Deputy Grand Master of the Grand Lodge of Pennsylvania.\nBuchanan also served as chairman of the Lancaster chapter of the Federalist Party. Like his father, he supported their political program, which provided federal funds for building projects and import duties as well as the re-establishment of a central bank after the First Bank of the United States' license expired in 1811. He became a strong critic of Democratic-Republican President James Madison during the War of 1812. Although he did not himself serve in a militia during the War of 1812, during the British occupation he joined a group of young men who stole horses for the United States Army in the Baltimore area. He was the last president involved in the War of 1812.\nIn 1814, he was elected for the Federalists to the Pennsylvania House of Representatives, where he was the youngest member, and held this seat until 1816. Since the sessions in the Pennsylvania General Assembly lasted only three months, Buchanan continued practicing law at a profit by charging higher fees, and his service helped him acquire more clients. In 1815, Buchanan defended District Judge Walter Franklin in an impeachment trial before the Pennsylvania Senate, over alleged judicial misconduct. Impeachments were more common at the time because the line between abuse of office and a wrong legal decision was determined by the ruling parties' preferences and the popularity of the judge's decision. Buchanan persuaded the senators that only judicial crimes and clear violations of the law justified impeachment.\n\nCongressional career\nU.S. House of Representatives\nIn the congressional elections of 1820, Buchanan ran for a seat in the House of Representatives. Shortly after his election victory, his father died in a carriage accident. As a young Representative, Buchanan was one of the most prominent leaders of the \"Amalgamator party\" faction of Pennsylvanian politics, named that because it was made up of both Democratic-Republicans and former Federalists, which transitioned from the First Party System to the Era of Good Feelings. During this era, the Democratic-Republicans became the most influential party. Buchanan's Federalist convictions were weak, and he switched parties after opposing a nativist Federalist bill. During the 1824 presidential election, Buchanan initially supported Henry Clay, but switched to Andrew Jackson (with Clay as a second choice) when it became clear that the Pennsylvanian public overwhelmingly preferred Jackson. After Jackson lost the 1824 election, he joined his faction, but Jackson had contempt for Buchanan due to his misinterpretation of his efforts to mediate between the Clay and Jackson camps.\nIn Washington, Buchanan became an avid defender of states' rights, and was close with many southern Congressmen, viewing some New England Congressmen as dangerous radicals. Buchanan's close proximity to his constituency allowed him to establish a Democratic coalition in Pennsylvania, consisting of former Federalist farmers, Philadelphia artisans, and Ulster-Scots-Americans. In the 1828 presidential election, he secured Pennsylvania, while the \"Jacksonian Democrats\", an independent party after splitting from the National Republican Party, won an easy victory in the parallel congressional election.\n\nBuchanan gained most attention during an impeachment trial where he acted as prosecutor for federal district judge James H. Peck; however, the Senate rejected Buchanan's plea and acquitted Peck by a majority vote. He was appointed to the Agriculture Committee in his first year, and he eventually became chairman of the Judiciary Committee. In 1831, Buchanan declined a nomination for the 22nd United States Congress from his constituency consisting of Dauphin, Lebanon, and Lancaster counties. He still had political ambitions and some Pennsylvania Democrats put him forward as a candidate for the vice presidency in the 1832 election.\n\nMinister to Russia\nAfter Jackson was re-elected in 1832, he offered Buchanan the position of United States Ambassador to Russia. Buchanan was reluctant to leave the country, as the distant St. Petersburg was a kind of political exile, which was the intention of Jackson, who considered Buchanan to be an \"incompetent busybody\" and untrustworthy, but he ultimately agreed. His work focused on concluding a trade and shipping treaty with Russia. While Buchanan was successful with the former, negotiating an agreement on free merchant shipping with Foreign Minister Karl Nesselrode proved difficult. He had denounced Tsar Nicholas I as a despot merely a year prior during his tenure in Congress; many Americans had reacted negatively to Russia's reaction to the 1830 Polish uprising.\n\nU.S. Senator\nBuchanan returned home and lost the election in the State Legislature for a full six-year term in the 23rd Congress, but was appointed by the Pennsylvania state legislature to succeed William Wilkins in the U.S. Senate. Wilkins, in turn, replaced Buchanan as the ambassador to Russia. The Jacksonian Buchanan, who was re-elected in 1836 and 1842, opposed the re-chartering of the Second Bank of the United States and sought to expunge a congressional censure of Jackson stemming from the Bank War. Buchanan served in the Senate until March 1845 and was twice confirmed in office. To unite Pennsylvania Democrats at the State Convention, he was chosen as their candidate for the National Convention. Buchanan maintained a strict adherence to the Pennsylvania State Legislature's guidelines and sometimes voted against positions in Congress which he promoted in his own speeches, despite open ambitions for the White House.\nBuchanan was known for his commitment to states' rights and the Manifest Destiny ideology. He rejected President Martin Van Buren's offer to become United States Attorney General and chaired prestigious Senate committees such as the Committee on the Judiciary and the Committee on Foreign Relations. Buchanan was one of only a few senators to vote against the Webster–Ashburton Treaty for its \"surrender\" of lands to the United Kingdom, as he demanded the entire Aroostook River Valley for the United States. In the Oregon Boundary Dispute, Buchanan adopted the maximum demand of 54°40′ as the northern border and spoke out in favor of annexing the Republic of Texas. During the contentious 1838 Pennsylvania gubernatorial election, Buchanan chose to support the Democratic challenger, David Rittenhouse Porter, who was elected by fewer than 5,500 votes as Pennsylvania's first governor under the state's revised Constitution of 1838.\nBuchanan also opposed a gag rule sponsored by John C. Calhoun that would have suppressed anti-slavery petitions. He joined the majority in blocking the rule, with most senators of the belief that it would have the reverse effect of strengthening the abolitionists. He said, \"We have just as little right to interfere with slavery in the South, as we have to touch the right of petition.\" Buchanan thought that the issue of slavery was the domain of the states, and he faulted abolitionists for exciting passions over the issue. In the lead-up to the 1844 Democratic National Convention, Buchanan positioned himself as a potential alternative to former President Martin Van Buren, but the nomination went to James K. Polk, who won the election.\n\nDiplomatic career\nSecretary of State\nBuchanan was offered the position of Secretary of State in the Polk administration or, as the alternative, a seat on the Supreme Court, to compensate him for his support in the election campaign but also in order to eliminate him as an internal party rival. He accepted the State Department post and served for the duration of Polk's single term in office. During his tenure, the United States recorded its largest territorial gain in history through the Oregon Treaty and the Treaty of Guadalupe Hidalgo, which included territory that is now Texas, California, Nevada, New Mexico, Arizona, Utah, and Colorado. In negotiations with Britain over Oregon, Buchanan initially favored the 49th parallel as the boundary of Oregon Territory, while Polk called for a more northerly boundary line. When Northern Democrats rallied around the popular slogan Fifty-Four Forty or Fight (\"54°40′ or war\") in the 1844 election campaign, Buchanan adopted this position, but later followed Polk's direction, leading to the Oregon Compromise of 1846, which established the 49th parallel as the boundary in the Pacific Northwest.\nIn regards to Mexico, Buchanan maintained a dubious view that its attack on American troops on the other side of the Rio Grande in April 1846 constituted a border violation and a legitimate reason for war. During the Mexican-American War, Buchanan initially advised against claiming territory south of the Rio Grande, fearing war with Britain and France. However, as the war came to an end, Buchanan changed his mind and argued for the annexation of further territory, arguing that Mexico was to blame for the war and that the compensation negotiated for the American losses was too low. Buchanan sought the nomination at the 1848 Democratic National Convention, as Polk had promised to serve only one term, but he only won the support of the Pennsylvania and Virginia delegations, so Senator Lewis Cass of Michigan was nominated.\n\nCivilian life and 1852 presidential election\nWith the 1848 election of Whig Zachary Taylor, Buchanan returned to private life. Buchanan was getting on in years and still dressed in the old-fashioned style of his adolescence, earning him the nickname \"Old Public Functionary\" from the press. Slavery opponents in the North mocked him as a relic of prehistoric man because of his moral values. He bought the house of Wheatland on the outskirts of Lancaster and entertained various visitors while monitoring political events. During this period, Buchanan became the center of a family network consisting of 22 nieces, nephews and their descendants, seven of whom were orphans. He found public service jobs for some through patronage, and for those in his favor, he took on the role of surrogate father. He formed the strongest emotional bond with his niece Harriet Lane, who later became First Lady for Buchanan in the White House.\nIn 1852, he was named president of the Board of Trustees of Franklin and Marshall College in Lancaster, and he served in this capacity until 1866. Buchanan did not completely leave politics. He intended to publish a collection of speeches and an autobiography, but his political comeback was thwarted by the 1852 presidential election. Buchanan traveled to Washington to discuss Pennsylvania Democratic Party politics, which were divided into two camps led by Simon Cameron and George Dallas. He quietly campaigned for the 1852 Democratic presidential nomination. In light of the Compromise of 1850, which had led to the admission of California into the Union as a free state and a stricter Fugitive Slave Act, Buchanan now rejected the Missouri Compromise and welcomed Congress's rejection of the Wilmot Proviso, which prohibited slavery in all territories gained in the Mexican-American War. Buchanan criticized abolitionism as a fanatical attitude and believed that slavery should be decided by state legislatures, not Congress. He disliked abolitionist Northerners due to his party affiliation, and became known as a \"doughface\" due to his sympathy toward the South. Buchanan emerged as a promising candidate for the Democratic presidential nomination, alongside Lewis Cass, Stephen Douglas, and William L. Marcy; however, the Pennsylvania convention did not vote unanimously in his favor, with over 30 delegates protesting against him. At the 1852 Democratic National Convention, he won the support of many southern delegates but failed to win the two-thirds support needed for the presidential nomination, which went to Franklin Pierce. Buchanan declined to serve as the vice presidential nominee, and the convention instead nominated his close friend, William R. King.\n\nMinister to the United Kingdom\nPierce won the election in 1852, and six months later, Buchanan accepted the position of United States Minister to the United Kingdom, a position that represented a step backward in his career and that he had twice previously rejected. Buchanan sailed for England in the summer of 1853, and he remained abroad for the next three years. In 1850, the United States and Great Britain signed the Clayton–Bulwer Treaty, which committed both countries to joint control of any future canal that would connect the Atlantic and Pacific Oceans through Central America. Buchanan met repeatedly with Lord Clarendon, the British foreign minister, in hopes of pressuring the British to withdraw from Central America. He was able to reduce British influence in Honduras and Nicaragua while also raising the kingdom's awareness of American interests in the region. He also focused on the potential annexation of Cuba, which had long interested him.\nAt Pierce's prompting, Buchanan met in Ostend, Belgium, with U.S. Ambassador to Spain Pierre Soulé and U.S. Ambassador to France John Mason, to work out a plan for the acquisition of Cuba. A memorandum draft resulted, called the Ostend Manifesto, which proposed the purchase of Cuba from Spain, then in the midst of revolution and near bankruptcy. The document declared the island \"as necessary to the North American republic as any of its present ... family of states\". Against Buchanan's recommendation, the final draft of the manifesto suggested that \"wresting it from Spain\", if Spain refused to sell, would be justified \"by every law, human and Divine\". The manifesto was met with a divided response and was never acted upon. It weakened the Pierce administration and reduced support for Manifest Destiny. In 1855, as Buchanan's desire to return home grew, Pierce asked him to hold the fort in London in light of the relocation of a British fleet to the Caribbean.\n\nElection of 1856\nBuchanan's service abroad allowed him to conveniently avoid the debate over the Kansas–Nebraska Act then roiling the country in the slavery dispute. While he did not overtly seek the presidency, he assented to the movement on his behalf. While still in England, he campaigned by praising John Joseph Hughes, who was Archbishop of New York, to a Catholic archbishop. The latter campaigned for Buchanan among high-ranking Catholics as soon as he heard about it. When Buchanan arrived home at the end of April 1856, he led on the first ballot, supported by powerful Senators John Slidell, Jesse Bright, and Thomas F. Bayard, who presented Buchanan as an experienced leader appealing to the North and South. The 1856 Democratic National Convention met in June 1856, producing a platform that reflected Buchanan's views, including support for the Fugitive Slave Law, which required the return of escaped slaves. The platform also called for an end to anti-slavery agitation and U.S. \"ascendancy in the Gulf of Mexico\". President Pierce hoped for re-nomination, while Senator Stephen A. Douglas also loomed as a strong candidate. He won the nomination after seventeen ballots after Douglas' resignation. He was joined on the ticket by John C. Breckinridge of Kentucky in order to maintain regional proportional representation, placating supporters of Pierce and Douglas, also allies of Breckinridge.\nBuchanan faced two candidates in the general election: former Whig President Millard Fillmore ran as the candidate for the anti-Catholic, anti-immigrant American Party (or \"Know-Nothing\"), while John C. Frémont ran as the Republican nominee. The contrast between Buchanan and Frémont was particularly stark, with opposing caricaturists drawing the Democratic candidate as a fussy old man in drag. Buchanan did not actively campaign, but he wrote letters and pledged to uphold the Democratic platform. In the election, he carried every slave state except for Maryland, as well as five slavery-free states, including his home state of Pennsylvania. He won 45 percent of the popular vote and decisively won the electoral vote, taking 174 of 296 votes. His election made him the first president from Pennsylvania. In a combative victory speech, Buchanan denounced Republicans, calling them a \"dangerous\" and \"geographical\" party that had unfairly attacked the South. He also declared, \"the object of my administration will be to destroy sectional party, North or South, and to restore harmony to the Union under a national and conservative government.\" He set about this initially by feigning a sectional balance in his cabinet appointments.\n\nPresidency (1857–1861)\nInauguration\nBuchanan was inaugurated on March 4, 1857, taking the oath of office from Chief Justice Roger B. Taney. In his lengthy inaugural address, Buchanan committed himself to serving only one term, as his predecessor had done. He abhorred the growing divisions over slavery and its status in the territories, saying that Congress should play no role in determining the status of slavery in the states or territories. He proposed a solution based on the Kansas-Nebraska Act, which stated that the principle of popular sovereignty was decisive, and Congress had no say in the matter. Buchanan recommended that a federal slave code be enacted to protect the rights of slaveowners in federal territories. He alluded to a then-pending Supreme Court case, Dred Scott v. Sandford, which he said would permanently settle the issue of slavery. Dred Scott was a slave who was temporarily taken from a slave state to a free territory by his owner, John Sanford. After Scott returned to the slave state, he filed a petition for his freedom based on his time in the free territory.\n\nAssociate Justice Robert C. Grier leaked the decision in the \"Dred Scott\" case early to Buchanan. In his inaugural address, Buchanan declared that the issue of slavery in the territories would be \"speedily and finally settled\" by the Supreme Court. According to historian Paul Finkelman: Buchanan already knew what the Court was going to decide. In a major breach of Court etiquette, Justice Grier, who, like Buchanan, was from Pennsylvania, had kept the President-elect fully informed about the progress of the case and the internal debates within the Court. When Buchanan urged the nation to support the decision, he already knew what Taney would say. Republican suspicions of impropriety turned out to be fully justified.\nHistorians agree that the court decision was a major disaster because it dramatically inflamed tensions, leading to the Civil War. In 2022, historian David W. Blight argued that the year 1857 was, \"the great pivot on the road to disunion...largely because of the Dred Scott case, which stoked the fear, distrust and conspiratorial hatred already common in both the North and the South to new levels of intensity.\"\n\nPersonnel\nCabinet and administration\nAs his inauguration approached, Buchanan sought to establish an obedient, harmonious cabinet to avoid the in-fighting that had plagued Andrew Jackson's administration. The cabinet's composition had to do justice to the proportional representation within the party and between the regions of the country. Buchanan first worked on this task in Wheatland until he traveled to the capital in January 1857. There, like many other guests at the National Hotel, he contracted severe dysentery, from which he did not fully recover until several months later. Dozens of those who fell ill died, including Buchanan's nephew and private secretary Eskridge Lane.\nThe cabinet selection was disastrous, with four Southern ministers being large-scale slaveholders who later became loyal to the Confederate States of America. Secretary of the Treasury Howell Cobb was considered the greatest political talent in the Cabinet, while the three department heads from the northern states were all considered to be doughfaces. His objective was to dominate the cabinet, and he chose men who would agree with his views. Buchanan had a troubled relationship with his vice president from the beginning, when he did not receive him during his inaugural visit but referred him to his niece and First Lady, which Breckinridge never forgave him for and saw as disrespectful. He left out the influential Stephen A. Douglas, who had made Buchanan's nomination possible by resigning at the National Convention the previous year, when filling the post. Concentrating on foreign policy, he appointed the aging Lewis Cass as Secretary of State. Buchanan's appointment of Southerners and their allies alienated many in the North, and his failure to appoint any followers of Douglas divided the party. Outside of the cabinet, he left in place many of Pierce's appointments but removed a disproportionate number of Northerners who had ties to Democratic opponents Pierce or Douglas.\n\nJudicial appointments\nBuchanan appointed one Justice, Nathan Clifford, to the Supreme Court of the United States. He appointed seven other federal judges to United States district courts. He also appointed two judges to the United States Court of Claims.\n\nIntervention in the Dred Scott case\nThe case of Dred Scott v. Sandford, to which Buchanan referred to in his inaugural address, dated back to 1846. Scott sued for his release in Missouri, claiming he lived in service to the proprietor in Illinois and Wisconsin Territory. The case reached the Supreme Court and gained national attention by 1856. Buchanan consulted with Judge John Catron in January 1857, inquiring about the outcome of the case and suggesting that a broader decision, beyond the specifics of the case, would be more prudent. Buchanan hoped that a broad decision protecting slavery in the territories could lay the issue to rest, allowing him to focus on other issues.\nCatron replied on February 10, saying that the Supreme Court's Southern majority would decide against Scott, but would likely have to publish the decision on narrow grounds unless Buchanan could convince his fellow Pennsylvanian, Justice Robert Cooper Grier, to join the majority of the court. Buchanan then wrote to Grier and prevailed upon him, providing the majority leverage to issue a broad-ranging decision sufficient to render the Missouri Compromise of 1820 unconstitutional.\nTwo days after Buchanan was sworn in as president, Chief Justice Taney delivered the Dred Scott decision, which denied the petitioner's request to be set free from slavery. The ruling broadly asserted that Congress had no constitutional power to exclude slavery in the territories. According to this decision, slaves were forever the property of their owners without rights and no African American could ever be a full citizen of the United States, even if they had full civil rights in a state. Buchanan's letters were not made public at the time, but he was seen conversing quietly with the Chief Justice during his inauguration. When the decision was issued, Republicans began spreading the word that Taney had informed Buchanan of the impending outcome. Rather than destroying the Republican platform as Buchanan had hoped, the decision infuriated Northerners, who condemned it.\n\nPanic of 1857\nThe Panic of 1857 began in the summer of that year, when the New York branch of Ohio Life Insurance and Trust Company announced its insolvency. The crisis spread rapidly, and by the fall, 1,400 state banks and 5,000 businesses had gone bankrupt. Unemployment and hunger became common in northern cities, but the agricultural south was more resilient. Buchanan agreed with the southerners who attributed the economic collapse to over-speculation.\nBuchanan acted in accordance with Jacksonian Democracy principles, which restricted paper money issuance, and froze federal funds for public works projects, causing resentment among some of the population due to his refusal to implement an economic stimulus program. While the government was \"without the power to extend relief\", it would continue to pay its debts in specie, and while it would not curtail public works, none would be added. In hopes of reducing paper money supplies and inflation, he urged the states to restrict the banks to a credit level of $3 to $1 of specie and discouraged the use of federal or state bonds as security for bank note issues. The economy recovered in several years, though many Americans suffered as a result of the panic. Buchanan had hoped to reduce the deficit, but by the time he left office the federal budget grew by 15%.\n\nUtah War\nIn the spring of 1857, the Latter-day Saints and their leader Brigham Young had been challenging federal representatives in Utah Territory, causing harassment and violence against non-Mormons. Young harassed federal officers and discouraged outsiders from settling in the Salt Lake City area. In September 1857, the Utah Territorial Militia, associated with the Latter-day Saints, perpetrated the Mountain Meadows massacre, in which Young's militia attacked a wagon train and killed 125 settlers. Buchanan was offended by the militarism and polygamous behavior of Young. With reports of violence against non-Mormons, Buchanan authorized a military expedition into Utah Territory in late March 1857 to replace Young as governor. The force consisted of 2,500 men, including Alfred Cumming and his staff, and was commanded by General William S. Harney. Complicating matters, Young's notice of his replacement was not delivered because the Pierce administration had annulled the Utah mail contract, and Young portrayed the approaching forces as an unauthorized overthrow.\nBuchanan's personnel decision incited resistance from the Mormons around Young, as Harney was known for his volatility and brutality. In August 1857, Albert S. Johnston replaced him for organizational reasons. Young reacted to the military action by mustering a two-week expedition, destroying wagon trains, oxen, and other Army property. Buchanan then dispatched Thomas L. Kane as a private agent to negotiate peace. The mission was successful, a peaceful agreement to replace Governor Young with Cumming was reached, and the Utah War ended. The President granted amnesty to inhabitants affirming loyalty to the government, and placed the federal troops at a peaceable distance for the balance of his administration.\nBuchanan did not comment on the conflict again until his State of the Union Address in December 1857, leaving open the question of whether it was a rebellion in Utah. One of Buchanan's last official acts in March 1861 was to reduce the size of Utah Territory in favor of Nevada, Colorado, and Nebraska. While the Latter-day Saints had frequently defied federal authority, some historians consider Buchanan's action was an inappropriate response to uncorroborated reports.\n\nTransatlantic telegraph cable\nBuchanan was the first recipient of an official telegram transmitted across the Atlantic. Following the dispatch of test and configuration telegrams, on August 16, 1858 Queen Victoria sent a 98-word message to Buchanan at his summer residence in the Bedford Springs Hotel in Pennsylvania, expressing hope that the newly laid cable would prove \"an additional link between the nations whose friendship is founded on their common interest and reciprocal esteem\". Queen Victoria's message took 16 hours to send.\nBuchanan responded: \"It is a triumph more glorious, because far more useful to mankind, than was ever won by conqueror on the field of battle. May the Atlantic telegraph, under the blessing of Heaven, prove to be a bond of perpetual peace and friendship between the kindred nations, and an instrument destined by Divine Providence to diffuse religion, civilization, liberty, and law throughout the world.\"\n\nBleeding Kansas and constitutional dispute\nThe Kansas–Nebraska Act of 1854 created the Kansas Territory and allowed the settlers there to decide whether to allow slavery. This resulted in violence between \"Free-Soil\" (antislavery) and pro-slavery settlers, which developed into the \"Bleeding Kansas\" period. The antislavery settlers, with the help of Northern abolitionists, organized their own territorial government in Topeka. The more numerous proslavery settlers, many from the neighboring slave state Missouri, established a government in Lecompton, giving the Territory two different governments for a time, with two distinct constitutions, each claiming legitimacy. The admission of Kansas as a state required a constitution be submitted to Congress with the approval of a majority of its residents. Under President Pierce, a series of violent confrontations escalated over who had the right to vote in Kansas. The situation drew national attention, and some in Georgia and Mississippi advocated secession should Kansas be admitted as a free state. Buchanan chose to endorse the pro-slavery Lecompton government.\nBuchanan appointed Robert J. Walker to replace John W. Geary as Territorial Governor, and there ensued conflicting referendums from Topeka and Lecompton, where election fraud occurred. In October 1857, the Lecompton government framed the pro-slavery Lecompton Constitution that agreed to a referendum limited solely to the slavery question. However, the vote against slavery, as provided by the Lecompton Convention, would still permit existing slaves, and all their issue, to be enslaved, so there was no referendum that permitted the majority anti-slavery residents to prohibit slavery in Kansas. As a result, anti-slavery residents boycotted the referendum since it did not provide a meaningful choice.\nDespite the protests of Walker and two former Kansas governors, Buchanan decided to accept the Lecompton Constitution. In a December 1857 meeting with Stephen A. Douglas, the chairman of the Senate Committee on Territories, Buchanan demanded that all Democrats support the administration's position of admitting Kansas under the Lecompton Constitution. On February 2, he transmitted the Lecompton Constitution to Congress. He also transmitted a message that attacked the \"revolutionary government\" in Topeka, conflating them with the Mormons in Utah. Buchanan made every effort to secure congressional approval, offering favors, patronage appointments, and even cash for votes. The Lecompton Constitution won the approval of the Senate in March, but a combination of Know-Nothings, Republicans, and Northern Democrats defeated the bill in the House.\nBuchanan never forgave Douglas, as the Northern Democrats' rejection was the deciding factor in the House's decision, and he removed all Douglas supporters from his patronage in Illinois and Washington, D.C., installing pro-administration Democrats, including postmasters. Rather than accepting defeat, Buchanan backed the 1858 English Bill, which offered Kansas immediate statehood and vast public lands in exchange for accepting the Lecompton Constitution. In August 1858, Kansans by referendum strongly rejected the Lecompton Constitution. The territory received an abolitionist constitution, which was bitterly opposed in Congress by representatives and senators from the southern states until Kansas was admitted to the Union in January 1861.\nThe dispute over Kansas became the battlefront for control of the Democratic Party. On one side were Buchanan, the majority of Southern Democrats, and the \"doughfaces\". On the other side were Douglas and the majority of northern Democrats, as well as a few Southerners. Douglas's faction continued to support the doctrine of popular sovereignty, while Buchanan insisted that Democrats respect the Dred Scott decision and its repudiation of federal interference with slavery in the territories.\n\n1858 mid-term elections\nDouglas's Senate term was coming to an end in 1859, with the Illinois legislature, elected in 1858, determining whether Douglas would win re-election. The Senate seat was the primary issue of the legislative election, marked by the famous debates between Douglas and his Republican opponent for the seat, Abraham Lincoln. Buchanan, working through federal patronage appointees in Illinois, ran candidates for the legislature in competition with both the Republicans and the Douglas Democrats. This could easily have thrown the election to the Republicans, and showed the depth of Buchanan's animosity toward Douglas. In the end, Douglas Democrats won the legislative election and Douglas was re-elected to the Senate. In that year's elections, Douglas forces took control throughout the North, except in Buchanan's home state of Pennsylvania. Buchanan's support was otherwise reduced to a narrow base of southerners.\nThe division between northern and southern Democrats allowed the Republicans to win a plurality of the House in the 1858 elections, and allowed them to block most of Buchanan's agenda. Buchanan, in turn, added to the hostility with his veto of six substantial pieces of Republican legislation. Among these measures were the Homestead Act, which would have given 160 acres of public land to settlers who remained on the land for five years, and the Morrill Act, which would have granted public lands to establish land-grant colleges. Buchanan argued that these acts were unconstitutional. In the western and northwestern United States, where the Homestead Act was very popular, even many Democrats condemned the president's policies, while many Americans who considered education an important asset resented Buchanan's veto of agricultural colleges.\n\nForeign policy\nBuchanan took office with an ambitious foreign policy, designed to establish U.S. hegemony over Central America at the expense of Great Britain. Buchanan sought to revitalize Manifest Destiny and to enforce the Monroe Doctrine, which had been under attack from the Spanish, French, and especially the British in the 1850s. He hoped to re-negotiate the Clayton–Bulwer Treaty to counter European imperialism in the Western Hemisphere, which he thought limited U.S. influence in the region. He also sought to establish American protectorates over the Mexican states of Chihuahua and Sonora to secure American citizens and investments, and most importantly, he hoped to achieve his long-term goal of acquiring Cuba. However, Buchanan's ambitions in Cuba and Mexico were largely blocked by the House of Representatives. After long negotiations with the British, he convinced them to cede the Bay Islands to Honduras and the Mosquito Coast to Nicaragua.\nIn 1858, Buchanan ordered the Paraguay expedition to punish Paraguay for firing on the USS Water Witch, ordering 2,500 marines and 19 warships there. This costly expedition took months to reach Asunción, which successfully resulted in a Paraguayan apology and payment of an indemnity. The chiefs of Raiatea and Tahaa in the South Pacific, refusing to accept the rule of King Tamatoa V, unsuccessfully petitioned the United States to accept the islands under a protectorate in June 1858. Buchanan also considered buying Alaska from the Russian Empire, as whaling in the waters there had become of great economic importance to the United States. Buchanan fueled this by spreading the rumor to the Russian ambassador Eduard de Stoeckl in December 1857 that a large amount of Mormons intended to emigrate to Russian Alaska. In the winter of 1859, an initial purchase offer of $5,000,000 (equivalent to $169,560,000 in 2023) was made. Although the project ultimately failed due to the reservations of Foreign Minister Alexander Gorchakov, the talks formed the basis for the later negotiations to purchase Alaska.\nBuchanan sought trade agreements with the Qing Dynasty and Japan. In China, his envoy William Bradford Reed succeeded in having the United States included as a party to the Treaty of Tianjin. In May 1860, Buchanan received a Japanese delegation consisting of several princes who carried the Harris Treaty negotiated by Townsend Harris for mutual ratification. Buchanan was offered a herd of elephants by King Rama IV of Siam, though the letter arrived after Buchanan's departure from office and Buchanan's successor Abraham Lincoln declined the offer stating that the U.S. had an unsuitable climate. Other presidential pets included a pair of bald eagles and a Newfoundland dog.\n\nCovode Committee\nIn March 1860, the House impaneled the Covode Committee to investigate the Buchanan administration's patronage system for alleged impeachable offenses, such as bribery and extortion of representatives. Buchanan supporters accused the committee, consisting of three Republicans and two Democrats, of being blatantly partisan, and claimed its chairman, Republican Rep. John Covode, was acting on a personal grudge stemming from a disputed land grant designed to benefit Covode's railroad company. The Democratic committee members, as well as Democratic witnesses, were enthusiastic in their condemnation of Buchanan.\nThe committee was unable to establish grounds for impeaching Buchanan; however, the majority report issued on June 17 alleged corruption and abuse of power among members of his cabinet. The committee gathered evidence that Buchanan had tried to bribe members of Congress in his favor through intermediaries in the spring of 1858 in connection with the pro-slavery Lecompton Constitution of Kansas, and threatened their relatives with losing their posts if they did not vote in favor of the Lecompton Constitution. Witnesses also testified that the federal government used public funds to strengthen the intra-party faction of Douglas's opponents in Illinois. The Democrats pointed out that evidence was scarce, but did not refute the allegations; one of the Democratic members, Rep. James Robinson, stated that he agreed with the Republicans, though he did not sign it.\nThe public was shocked by the extent of the bribery, which affected all levels and agencies of government. Buchanan claimed to have \"passed triumphantly through this ordeal\" with complete vindication. Republican operatives distributed thousands of copies of the Covode Committee report throughout the nation as campaign material in that year's presidential election.\n\nElection of 1860\nAs he had promised in his inaugural address, Buchanan did not seek re-election. He went so far as to tell his ultimate successor, \"If you are as happy in entering the White House as I shall feel on returning to Wheatland, you are a happy man.\"\nAt the 1860 Democratic National Convention in Charleston, the party split over the issue of slavery in the territories, damaging Buchanan's reputation as the main person responsible for this issue. Though Douglas led after every ballot, he was unable to win the two-thirds majority required. The convention adjourned after 53 ballots, and re-convened in Baltimore in June. After Douglas finally won the nomination, several Southerners refused to accept the outcome, and nominated Vice President Breckinridge as their own candidate. Douglas and Breckinridge agreed on most issues except the protection of slavery. Buchanan, nursing a grudge against Douglas, failed to reconcile the party, and tepidly supported Breckinridge. With the splintering of the Democratic Party, Republican nominee Abraham Lincoln won a four-way election that also included John Bell of the Constitutional Union Party. Lincoln's support in the North was enough to give him an Electoral College majority. Buchanan became the last Democrat to win a presidential election until Grover Cleveland in 1884.\nAs early as October, the army's Commanding General, Winfield Scott, an opponent of Buchanan, warned him that Lincoln's election would likely cause at least seven states to secede from the union. He recommended that massive amounts of federal troops and artillery be deployed to those states to protect federal property, although he also warned that few reinforcements were available. Since 1857, Congress had failed to heed calls for a stronger militia and allowed the army to fall into deplorable condition. Buchanan distrusted Scott and ignored his recommendations. After Lincoln's election, Buchanan directed Secretary of War John B. Floyd to reinforce southern forts with such provisions, arms, and men as were available; however, Floyd persuaded him to revoke the order.\n\nSecession\nWith Lincoln's victory, talk of secession and disunion reached a boiling point, putting the burden on Buchanan to address it in his final speech to Congress on December 10. In his message, which was anticipated by both factions, Buchanan denied the right of states to secede but maintained the federal government was without power to prevent them. He placed the blame for the crisis solely on \"intemperate interference of the Northern people with the question of slavery in the Southern States,\" and suggested that if they did not \"repeal their unconstitutional and obnoxious enactments ... the injured States, after having first used all peaceful and constitutional means to obtain redress, would be justified in revolutionary resistance to the Government of the Union.\" Buchanan's only suggestion to solve the crisis was \"an explanatory amendment\" affirming the constitutionality of slavery in the states, the fugitive slave laws, and popular sovereignty in the territories. His address was sharply criticized both by the North, for its refusal to stop secession, and the South, for denying its right to secede. Five days after the address was delivered, Treasury Secretary Howell Cobb resigned, as his views had become irreconcilable with the President's. Even as the formation of the Confederacy by the secessionist states became increasingly apparent in the winter of 1860, the president continued to surround himself with Southerners and ignore the Republicans.\n\nSouth Carolina, long the most radical Southern state, seceded from the Union on December 20, 1860. However, Unionist sentiment remained strong among many in the South, and Buchanan sought to appeal to the Southern moderates who might prevent secession in other states. He met with South Carolinian commissioners in an attempt to resolve the situation at Fort Sumter, which federal forces remained in control of despite its location in Charleston, South Carolina. Buchanan saw Congress, not himself, as responsible for finding a solution to the secession crisis. As a compromise for the southern states, Buchanan envisioned the adoption of amendments to the United States Constitution that would guarantee the right to slavery in the southern states and territories and strengthen the right of slave owners to reclaim escaped slaves as property in the northern states.\nHe refused to dismiss Interior Secretary Jacob Thompson after the latter was chosen as Mississippi's agent to discuss secession, and he refused to fire Secretary of War John B. Floyd despite an embezzlement scandal. Floyd ended up resigning, but not before sending numerous firearms to Southern states, where they eventually fell into the hands of the Confederacy. Despite Floyd's resignation, Buchanan continued to seek the advice of counselors from the Deep South, including Jefferson Davis and William Henry Trescot. Buchanan's friend Rose O'Neal Greenhow took advantage of the proximity to the president and spied for the Confederacy, which had already established a sophisticated network for gathering information from its eventual opponent before its formation.\nEfforts were made in vain by Sen. John J. Crittenden, Rep. Thomas Corwin, and former president John Tyler to negotiate a compromise to stop secession, with Buchanan's support. Failed attempts were also made by a group of governors meeting in New York. Buchanan secretly asked President-elect Lincoln to call for a national referendum on the issue of slavery, but Lincoln declined. In December 1860, when the second session of the 36th Congress was convened, The Committee of Thirty-Three was established by the House of Representatives to prevent further states from seceding. They proposed the Corwin Amendment, which would bar Congress from interfering with slavery in states. Despite opposition from Republicans, it passed both houses of Congress and was proposed to states for ratification, but it was never ratified by the requisite number of states.\nDespite the efforts of Buchanan and others, six more slave states seceded by the end of January 1861. Buchanan replaced the departed Southern cabinet members with John Adams Dix, Edwin M. Stanton, and Joseph Holt, all of whom were committed to preserving the Union. When Buchanan considered surrendering Fort Sumter, the new cabinet members threatened to resign, and Buchanan relented. On January 5, Buchanan decided to reinforce Fort Sumter, sending the Star of the West with 250 men and supplies. However, he failed to ask Major Robert Anderson to provide covering fire for the ship, and it was forced to return North without delivering troops or supplies. Buchanan chose not to respond to this act of war, and instead sought to find a compromise to avoid secession. He received a March 3 message from Anderson, that supplies were running low, but the response became Lincoln's to make, as the latter succeeded to the presidency the next day.\n\nStates admitted to the Union\nThree new states were admitted to the Union while Buchanan was in office:\n\nMinnesota – May 11, 1858\nOregon – February 14, 1859\nKansas – January 29, 1861\n\nFinal years and death (1861–1868)\nAfter leaving office, Buchanan retired to private life in Wheatland, where he spent most of his time in his study, reading books and writing letters. The Civil War erupted within two months of Buchanan's retirement. He supported the Union and the war effort, writing to former colleagues that, \"the assault upon Sumter was the commencement of war by the Confederate states, and no alternative was left but to prosecute it with vigor on our part.\" Buchanan supported Lincoln's introduction of universal conscription in the northern states, but was an opponent of his Emancipation Proclamation. Although he recognized constitutional violations in some of the president's executive orders, he never criticized them in public. He also wrote a letter to his fellow Pennsylvania Democrats in Harrisburg, urging them and all young men to enlist in the Union army and \"join the many thousands of brave & patriotic volunteers who are already in the field.\"\nBuchanan was dedicated to defending his actions prior to the Civil War, which was referred to by some as \"Buchanan's War\". He received hate mail and threatening letters daily, and stores in Lancaster displayed Buchanan's likeness with the eyes inked red, a noose drawn around his neck and the word \"TRAITOR\" written across his forehead. The Senate proposed a resolution of condemnation which ultimately failed, and newspapers accused him of colluding with the Confederacy. His former cabinet members, five of whom had been given jobs in the Lincoln administration, refused to defend Buchanan publicly.\nBuchanan became distraught by the vitriolic attacks levied against him, and fell sick and depressed. In October 1862, he defended himself in an exchange of letters with Winfield Scott, published in the National Intelligencer. He soon began writing his fullest public defense, in the form of his memoir Mr. Buchanan's Administration on the Eve of Rebellion, which was published in 1866, one year after the Civil War ended. Buchanan attributed secession to the \"malign influence\" of Republicans and the abolitionist movement. He discussed his foreign policy successes and expressed satisfaction with his decisions, even during the secession crisis. He blamed Robert Anderson, Winfield Scott, and Congress for the unresolved issue. Two years after the publication of the memoir, Buchanan caught a cold in May 1868, which quickly worsened due to his advanced age. He died on June 1, 1868, of respiratory failure at the age of 77 at his home at Wheatland. He was interred in Woodward Hill Cemetery in Lancaster.\n\nPolitical views\nBuchanan was often considered by anti-slavery northerners a \"doughface\", a northerner with pro-southern principles. Buchanan's sympathies for the Southern states went beyond political expediency for his path to the White House. He identified with cultural and social values that he found reflected in the honor code and lifestyle of the planter class and with which he increasingly came into contact in his retirement community beginning in 1834. Shortly after his election, he said that the \"great object\" of his administration was \"to arrest, if possible, the agitation of the Slavery question in the North and to destroy sectional parties\". Although Buchanan was personally opposed to slavery, he believed that the abolitionists were preventing the solution to the slavery problem. He stated, \"Before [the abolitionists] commenced this agitation, a very large and growing party existed in several of the slave states in favor of the gradual abolition of slavery; and now not a voice is heard there in support of such a measure. The abolitionists have postponed the emancipation of the slaves in three or four states for at least half a century.\" In deference to the intentions of the typical slaveholder, he was willing to provide the benefit of the doubt. In his third annual message to Congress, the president claimed that the slaves were \"treated with kindness and humanity. ... Both the philanthropy and the self-interest of the master have combined to produce this humane result.\"\n\nBuchanan thought restraint was the essence of good self-government. He believed the constitution comprised \"... restraints, imposed not by arbitrary authority, but by the people upon themselves and their representatives. ... In an enlarged view, the people's interests may seem identical, but to the eye of local and sectional prejudice, they always appear to be conflicting ... and the jealousies that will perpetually arise can be repressed only by the mutual forbearance which pervades the constitution.\" Regarding slavery and the Constitution, he stated: \"Although in Pennsylvania we are all opposed to slavery in the abstract, we can never violate the constitutional compact we have with our sister states. Their rights will be held sacred by us. Under the constitution it is their own question; and there let it remain.\"\nOne of the prominent issues of the day was tariffs. Buchanan was conflicted by free trade as well as prohibitive tariffs, since either would benefit one section of the country to the detriment of the other. As a senator from Pennsylvania, he said: \"I am viewed as the strongest advocate of protection in other states, whilst I am denounced as its enemy in Pennsylvania.\"\nBuchanan was also torn between his desire to expand the country for the general welfare of the nation, and to guarantee the rights of the people settling particular areas. On territorial expansion, he said, \"What, sir? Prevent the people from crossing the Rocky Mountains? You might just as well command the Niagara not to flow. We must fulfill our destiny.\" On the resulting spread of slavery, through unconditional expansion, he stated: \"I feel a strong repugnance by any act of mine to extend the present limits of the Union over a new slave-holding territory.\" For instance, he hoped the acquisition of Texas would \"be the means of limiting, not enlarging, the dominion of slavery.\"\n\nPersonal life\nBuchanan suffered from esotropia. In addition, one eye was short-sighted and the other far-sighted. To cover this, he bent his head forward and leaned it to one side during social interactions. This led to ridicule, which Henry Clay, among others, used ruthlessly during a congressional debate.\nIn 1818, Buchanan met Anne Caroline Coleman at a grand ball in Lancaster, and the two began courting. Anne was the daughter of the wealthy iron manufacturer Robert Coleman; Robert, like Buchanan's father, was from County Donegal in Ulster. Anne was also the sister-in-law of Philadelphia judge Joseph Hemphill, one of Buchanan's colleagues. By 1819, the two were engaged, but spent little time together. Buchanan was busy with his law firm and political projects during the Panic of 1819, which took him away from Coleman for weeks at a time. Rumors abounded, as some suggested that he was involved with other (unidentified) women. Letters from Coleman revealed she was aware of several rumors, and she accused him of only being interested in her money. She broke off the engagement, and soon afterward, on December 9, 1819, inexplicably died of \"hysterical convulsions\" resulting from an overdose of laudanum, at the age of 23. It was never established if the drug was taken by instruction, by accident, or by intent. Buchanan wrote to her father for permission to attend the funeral, which was refused. At the time of her funeral, he said that, \"I feel happiness has fled from me forever.\" Afterwards, Buchanan claimed that he remained unmarried out of devotion to his only love, who had died young.\n\nIn 1833 and the 1840s, he spoke of plans to marry, but these came to nothing and may merely have been due to his ambitions for a seat in the federal Senate or the White House. In the latter case, the aspirant was 19-year-old Anna Payne, the niece of former First Lady Dolley Madison. During his presidency, an orphaned niece, Harriet Lane, whom he had adopted, served as official White House hostess. There was an unfounded rumor that he had an affair with President Polk's widow, Sarah Childress Polk.\nBuchanan had a close relationship with William Rufus King, which became a popular target of gossip. King was an Alabama politician who briefly served as vice president under Franklin Pierce. Buchanan and King lived together in a Washington boardinghouse and attended social functions together from 1834 until 1844. Such a living arrangement was then common, though Buchanan once referred to the relationship as a \"communion\". Andrew Jackson mockingly called them \"Miss Nancy\" and \"Aunt Fancy\", the former being a 19th-century euphemism for an effeminate man. Buchanan's Postmaster General, Aaron V. Brown, also referred to King as \"Aunt Fancy\", as well as Buchanan's \"better half\", and \"wife\". King died of tuberculosis shortly after Pierce's inauguration, four years before Buchanan became president. Buchanan described him as \"among the best, the purest and most consistent public men I have known\". Biographer Baker opines that both men's nieces may have destroyed correspondence between the two men. However, she believes that their surviving letters illustrate only \"the affection of a special friendship\".\nBuchanan's lifelong bachelorhood after Anne Coleman's death has drawn interest and speculation. Some conjecture that Anne's death merely served to deflect questions about Buchanan's sexuality and bachelorhood. One of his biographers, Jean Baker, suggests that Buchanan was celibate, if not asexual. Several writers have surmised that he was homosexual, including James W. Loewen, Robert P. Watson, and Shelley Ross. Loewen indicated that Buchanan, late in life, wrote a letter acknowledging that he might marry a woman who could accept his \"lack of ardent or romantic affection\".\n\nLegacy\nHistorical reputation\nThough Buchanan predicted that \"history will vindicate my memory,\" historians have criticized Buchanan for his unwillingness or inability to act in the face of secession. Historical rankings of presidents of the United States without exception place Buchanan among the least successful presidents. When scholars are surveyed, he ranks at or near the bottom in terms of vision/agenda-setting, domestic leadership, foreign policy leadership, moral authority, and positive historical significance of their legacy. According to surveys taken by American scholars and political scientists between 1948 and 1982, Buchanan ranks every time among the worst presidents of the United States, alongside Harding, Fillmore and Nixon.\nBuchanan biographer Philip S. Klein focused in 1962, during the Civil Rights movement, upon challenges Buchanan faced:\n\nBuchanan assumed leadership ... when an unprecedented wave of angry passion was sweeping over the nation. That he held the hostile sections in check during these revolutionary times was in itself a remarkable achievement. His weaknesses in the stormy years of his presidency were magnified by enraged partisans of the North and South. His many talents, which in a quieter era might have gained for him a place among the great presidents, were quickly overshadowed by the cataclysmic events of civil war and by the towering Abraham Lincoln.\nBiographer Jean Baker is less charitable to Buchanan, saying in 2004:\n\nAmericans have conveniently misled themselves about the presidency of James Buchanan, preferring to classify him as indecisive and inactive ... In fact Buchanan's failing during the crisis over the Union was not inactivity, but rather his partiality for the South, a favoritism that bordered on disloyalty in an officer pledged to defend all the United States. He was that most dangerous of chief executives, a stubborn, mistaken ideologue whose principles held no room for compromise. His experience in government had only rendered him too self-confident to consider other views. In his betrayal of the national trust, Buchanan came closer to committing treason than any other president in American history.Other historians, such as Robert May, argued that his politics were \"anything but pro-slavery\", nevertheless, a very negative view is to be found in Michael Birkner's works about Buchanan. For Lori Cox Han, he ranks among scholars \"as either the worst president in [American] history or as part of a lowest ranking failure category\".\n\nMemorials\nA bronze and granite memorial near the southeast corner of Washington, D.C.'s Meridian Hill Park was designed by architect William Gorden Beecher and sculpted by Maryland artist Hans Schuler. It was commissioned in 1916 but not approved by the U.S. Congress until 1918, and not completed and unveiled until June 26, 1930. The memorial features a statue of Buchanan, bookended by male and female classical figures representing law and diplomacy, with engraved text reading: \"The incorruptible statesman whose walk was upon the mountain ranges of the law,\" a quote from a member of Buchanan's cabinet, Jeremiah S. Black.\n\nAn earlier monument was constructed in 1907–1908 and dedicated in 1911, on the site of Buchanan's birthplace in Stony Batter, Pennsylvania. Part of the original 18.5-acre (75,000 m2) memorial site is a 250-ton pyramid structure that stands on the site of the original cabin where Buchanan was born. The monument was designed to show the original weathered surface of the native rubble and mortar.\nThree counties are named in his honor, in Iowa, Missouri, and Virginia. Another in Texas was christened in 1858 but renamed Stephens County, after the newly elected vice president of the Confederate States of America, Alexander Stephens, in 1861. The city of Buchanan, Michigan, was also named after him. Several other communities are named after him: the unincorporated community of Buchanan, Indiana, the city of Buchanan, Georgia, the town of Buchanan, Wisconsin, and the townships of Buchanan Township, Michigan, and Buchanan, Missouri.\nJames Buchanan High School is a small, rural high school located on the outskirts of his childhood hometown, Mercersburg, Pennsylvania.\n\nPopular culture depictions\nBuchanan and his legacy are central to the film Raising Buchanan (2019). He is portrayed by René Auberjonois.\n\nSee also\nHistorical rankings of presidents of the United States\nList of presidents of the United States\nList of presidents of the United States by previous experience\nPresidents of the United States on U.S. postage stamps\nList of federal political sex scandals in the United States\n\nReferences\nWorks cited\nFurther reading\nExternal links\n\nUnited States Congress. \"James Buchanan (id: B001005)\". Biographical Directory of the United States Congress.\nJames Buchanan: A Resource Guide from the Library of Congress\nThe James Buchanan papers, spanning the entirety of his legal, political and diplomatic career, are available for research use at the Historical Society of Pennsylvania.\nUniversity of Virginia article: Buchanan biography\nWheatland\nJames Buchanan at Tulane University\nEssay on James Buchanan and his presidency from the Miller Center of Public Affairs\nBuchanan's Birthplace State Park, Franklin County, Pennsylvania\n\"Life Portrait of James Buchanan\", from C-SPAN's American Presidents: Life Portraits, June 21, 1999\nPrimary sources\n\nWorks by James Buchanan at Project Gutenberg\nWorks by James Buchanan at LibriVox (public domain audiobooks) \nWorks by or about James Buchanan at the Internet Archive\nJames Buchanan Ill with Dysentery Before Inauguration: Original Letters Shapell Manuscript Foundation\nMr. Buchanans Administration on the Eve of the Rebellion. President Buchanans memoirs.\nInaugural Address Archived August 9, 2020, at the Wayback Machine\nFourth Annual Message to Congress, December 3, 1860\n\nHarriet Rebecca Lane Johnston (May 9, 1830 – July 3, 1903) acted as first lady of the United States during the administration of her uncle, lifelong bachelor president James Buchanan, from 1857 to 1861. She has been described as the first of the modern first ladies, being a notably charming and diplomatic hostess, whose dress-styles were copied, and who promoted deserving causes. In her will, she left funds for a new school on the grounds of Washington National Cathedral. Several ships have been named in her honor, including the cutter USCGC Harriet Lane, still in service.\n\nStatus\nLane is the only person to have served as First Lady to a bachelor president, Buchanan being the only U.S. president never to have married. She is among 11 women who have served as First Lady, but were not married to the president, with most of the other women being relatives of widowed presidents.\n\nEarly life\nHarriet Lane's family was from Franklin County, Pennsylvania. She was the youngest child of Elliott Tole Lane, a merchant, and Jane Ann Buchanan Lane. She lost her mother when she was nine; when her father's death two years later made her an orphan, she requested that her favorite uncle, James Buchanan, be appointed as her legal guardian. Buchanan, an unmarried Democratic senator from Pennsylvania, indulged his niece and her sister, enrolling them in boarding schools in Charles Town, Virginia (later for two years at the Georgetown Visitation Monastery in the Georgetown section of Washington, D.C.) By this time, Buchanan was Secretary of State, and, as he had promised, he introduced her to fashionable and political circles.\nIn 1854, she joined him in London, where he was minister to the Court of St. James's. Queen Victoria gave \"dear Miss Lane\" the rank of ambassador's wife; admiring suitors gave her the fame of a beauty. In appearance \"Hal\" Lane was of medium height, with masses of light, almost golden-colored hair. She had eyes that were described as \"violet colored\".\n\nActing First Lady of the United States\nThe capital welcomed its new \"Democratic Queen\" to the White House in 1857. Harriet was a popular hostess during the four years of the Buchanan presidency. Women copied her hair and clothing styles (especially when she lowered the neckline on her inaugural gown by 2.5 inches), parents named their daughters for her, and a popular song (\"Listen to the Mockingbird\") was dedicated to her. While in the White House, she used her position to promote social causes, such as improving the living conditions of Native Americans in reservations. She also made a point of inviting artists and musicians to White House functions. For both her popularity and her advocacy work, she has been described as the first of the modern first ladies, and her popularity at the time is compared to that of Jacqueline Kennedy in the 1960s. The presidential yacht was named for her—the first of several ships to be named after her, one of which remains in service.\n\nAs sectional tensions increased, she worked out seating arrangements for her weekly formal dinner parties with special care, to give dignitaries their proper precedence and still keep political foes apart. Her tact did not falter, but her task became impossible—as did her uncle's. Seven states had seceded by the time Buchanan retired from office and returned with his niece to his spacious country home, Wheatland, near Lancaster, Pennsylvania.\nIn the 1982 Siena College Research Institute survey asking historians to assess American first ladies, Lane and several other \"acting\" first ladies were included. The first ladies survey, which has been conducted periodically since, ranks first ladies according to a cumulative score on the independent criteria of their background, value to the country, intelligence, courage, accomplishments, integrity, leadership, being their own women, public image, and value to the president. In the 1982 survey, out of 42 first ladies and acting first ladies, Lane was assessed as the 29th most highly regarded among historians. Acting first ladies such as Lane have been excluded from subsequent iterations of this survey.\n\nRomance and marriage\nDuring her time in England, Sir Fitzroy Kelly, then Prime Minister Palmerston's attorney general, proposed marriage to her; Queen Victoria was strongly in favor of this match, as it would keep Lane in England.\nLane considered the advantages of a number of bachelors. Her uncle cautioned Lane against \"rushing precipitately into matrimonial connections\" as his ward found her potential suitors \"pleasant but dreadfully troublesome\". Lane eventually married Baltimore banker Henry Elliott Johnston at the age of 36. They had two sons: James Buchanan Johnston (1866–1881) and Henry Elliot Johnston (1869–1882), but within the 18 years from 1867 to 1885, her uncle, her husband, and her children all died.\n\nLater life and death\nHarriet wrote her will in 1895 and lived another eight years, during which the country's general prosperity greatly increased the value of her estate. She added a codicil in 1899 directing that a school building be constructed on the grounds of the Washington National Cathedral property and asked that it be called the Lane-Johnston Building \"to the end that the family names of my husband and myself may be associated with the bequest made in loving memory of our sons.\" A codicil of 1903 increased her gift by one third but said that only half the total was to be spent on the building. The remainder was \"specially to provide for the free maintenance, education and training of choirboys, primarily those in service of the Cathedral.\" This bequest founded the prestigious boys' school that today is called St. Albans School, which opened in October 1909. \nAt Harriet Lane Johnston's funeral, services were conducted by Bishop Satterlee and Canon DeVries of the Washington National Cathedral. She was buried in Green Mount Cemetery, Baltimore, Maryland, her grave marked with a Celtic cross like the Peace Cross on the cathedral close. In 1905, guests were invited to see the cornerstone of the first St. Albans School building, laid for what the invitation referred to as \"The Lane Johnston Choir School for Boys of the Washington Cathedral\".\n\nLegacy\nLane left bequests in her will that established a children's hospital and a boys' school, and she donated her collection of artwork to the Smithsonian. Several Navy and Coast Guard ships have been named in her honor.\nHer birthplace, the Lane House, was listed on the National Register of Historic Places in 1972.\n\nHospital and school\nShe dedicated $400,000 (equivalent to $13,600,000 in 2023) to establish the Harriet Lane Home for Invalid Children at the Johns Hopkins Hospital in Baltimore, Maryland as a memorial to two sons who had died in childhood. In October 1912 the Harriet Lane Home officially opened. It was the first children's clinic in the United States that was associated with a medical school. Eventually treating over 60,000 children a year, the Harriet Lane Home became a pioneer treatment, teaching, and research clinic.\nFrom 1930 to 1963 Helen Taussig, who helped to develop the blue baby operation, headed the pediatric cardiac clinic. Child psychiatrist Leo Kanner did studies of autistic children. Lawson Wilkins established an endocrine clinic that developed procedures used universally to treat children with certain glandular disorders, including dwarfism. John E. Bordley and William G. Hardy broke ground in detecting hearing impairments in very young children. It became a renowned pediatric facility; the Harriet Lane Outpatient Clinics serve thousands of children today, and the widely used manual for pediatric house officers, The Harriet Lane Handbook, bears her name.\nThe Harriet Lane Outpatient Clinics continue to operate in countries throughout the world.\nThe pediatric medicine Harriet Lane Handbook series continues in print and online, with multiple titles. The original title (subtitled A Manual for Pediatric House Officers) is in its 22nd edition, published by Mosby.\n\nArt collection\nShe had an art collection based on European works which she left to the U.S. government. The Smithsonian Institution called her the \"First Lady of the National Collection of Fine Arts\" after her collection was accepted into public ownership.\n\nNamesake ships\nThe United States Coast Guard has had three cutters named in her honor. The first was the USRC Harriet Lane, commissioned into the United States Revenue Cutter Service (predecessor of the USCG) in 1857. This cutter was transferred to the United States Navy in 1861 because of the American Civil War.\nThe second cutter named for Harriet Lane was the 125 foot USCGC Harriet Lane (WSC-141), commissioned in 1926 and decommissioned in 1946.\nThe third cutter named for Harriet Lane is the USCGC Harriet Lane (WMEC-903). The cutter was commissioned in May 1984, and as of 2021, is still in active service.\n\nFootnotes\nReferences\nFurther reading\nBalcerski, Thomas J. \"Harriet Rebecca Lane Johnston.\" in A Companion to First Ladies (2016): 197-213.\nRosenberger, Homer Tope. \"To what Extent Did Harriet Lance Influence the Public Policies of James Buchanan?\" Lancaster County Historical Society, 1970. online\nUpdike, John (1974). Buchanan Dying (play). (Ms. Johnston is a character in Updike's fictional play about President Buchanan.)\n\nExternal links\nWorks by or about Harriet Lane at the Internet Archive\n\"Harriet Lane\". First Ladies: Influence & Image. firstladies.org. CNN.\n\nSince the office was established in 1789, 45 persons have served as president of the United States. Of these, eight have died in office: four were assassinated, and four died of natural causes. In each of these instances, the vice president has succeeded to the presidency. This practice is now governed by Section One of the Twenty-fifth Amendment to the United States Constitution, ratified in 1967, which declares that, \"the Vice President shall become President\" if the president is removed from office, dies, or resigns. The initial authorization for this practice was provided by Article II, Section 1, Clause 6, of the U.S. Constitution.\nThe first incumbent U.S. president to die was William Henry Harrison, on April 4, 1841, only one month after Inauguration Day. He died from complications of what at the time was believed to be pneumonia. The second American president to die in office, Zachary Taylor, died on July 9, 1850, from acute gastroenteritis. Abraham Lincoln was the first U.S. president to be killed while in office. He was shot by John Wilkes Booth on the night of April 14, 1865, and died the following morning. Sixteen years later, on July 2, 1881, James A. Garfield was shot by Charles J. Guiteau, surviving for over two months before dying on September 19, 1881.\nOn September 14, 1901, William McKinley died, eight days after being shot by Leon Czolgosz. Next, Warren G. Harding suffered a heart attack, and died on August 2, 1923. On April 12, 1945, Franklin D. Roosevelt (who had just begun his fourth term in office) collapsed and died as a result of a cerebral hemorrhage. The most recent U.S. president to die in office was John F. Kennedy, who was shot by Lee Harvey Oswald on November 22, 1963, in Dallas, Texas.\n\n1841: William Henry Harrison\nOn March 26, 1841, William Henry Harrison became ill with a cold after being caught in a torrential downpour without cover. His symptoms grew progressively worse over the ensuing two days, at which time a team of doctors was called in to treat him. After making a diagnosis of right lower lobe pneumonia, they proceeded to place heated suction cups on his bare torso and to administer a series of bloodlettings, to supposedly draw out the disease. When those procedures failed to bring about improvement, the doctors treated him with ipecac, Castor oil, calomel, and finally with a boiled mixture of crude petroleum and Virginia snakeroot. All this only weakened Harrison further.\nInitially, no official announcement was made concerning Harrison's illness, which, the longer he remained out of public view, fueled public speculation and concern. By the end of the month large crowds were gathering outside the White House, holding vigil while awaiting any news about the president's condition. On the evening of April 4, 1841, nine days after becoming ill, and exactly one month after taking the oath of office, Harrison died at age 68. His last words were to his attending doctor, though assumed to be directed at Vice President John Tyler:\n\nSir, I wish you to understand the true principles of the government. I wish them carried out. I ask nothing more.\nA 30-day period of mourning commenced following the president's death. Various public ceremonies, modeled after European royal funeral practices, were held. An invitation-only funeral service was also held, on April 7 in the East Room of the White House, after which Harrison's coffin was brought to Congressional Cemetery in Washington, D.C., where it was placed in a temporary receiving vault.\nThat June, Harrison's body was transported by train and river barge to North Bend, Ohio. Then, on July 7, 1841, the nation's 9th president was buried in a family tomb at the summit of Mt. Nebo, overlooking the Ohio River – the William Henry Harrison Tomb State Memorial.\nHarrison's death sparked a brief constitutional crisis regarding succession to the presidency, as the U.S. Constitution was unclear as to whether Vice President John Tyler should assume the office of president or merely execute the duties of the vacant office. Tyler claimed a constitutional mandate to carry out the full powers and duties of the presidency and took the presidential oath of office, setting an important precedent for an orderly transfer of presidential power when a president leaves office intra-term.\nCoincidentally, all but one of the presidents who later died in office had, like Harrison, won a presidential election in a year ending in a zero (1840 through 1960). This pattern of tragedies came to be known as the Curse of Tippecanoe, or the Curse of Tecumseh, the name of the Shawnee leader against whom Harrison fought in the 1811 Battle of Tippecanoe. Also sometimes referred to as the Zero Factor legend, the pattern was disrupted by Ronald Reagan, who survived an assassination attempt in 1981 (69 days after taking office) and lived to complete two full terms.\n\n1850: Zachary Taylor\nZachary Taylor was known to have consumed copious amounts of ice water, cold milk, green apples, and cherries on July 4, 1850, after attending holiday celebrations and the laying of the cornerstone of the Washington Monument. That same evening, he became severely ill with an unknown digestive ailment. Doctors used popular treatments of the time. On the morning of July 9, the president asked his wife Margaret not to grieve saying:\n\nI have always done my duty, I am ready to die. My only regret is for the friends I leave behind me.\nTaylor died late that evening, five days after becoming ill, at age 65. Contemporary reports listed the cause of death as \"bilious diarrhea or a bilious cholera.\" He was succeeded by Vice President Millard Fillmore.\nTaylor's funeral took place on July 13, and like Harrison's nine years earlier, was held in the East Room of the White House. Afterward, an estimated 100,000 people gathered along the funeral route to Congressional Cemetery where his coffin was placed temporarily in the Public Vault; that October it was transported to Louisville, Kentucky. On November 1, 1850, Taylor was buried in his family's burial ground on the Taylor estate, Springfield, which became the Zachary Taylor National Cemetery.\nAlmost immediately after his death, rumors began to circulate that Taylor had been poisoned by pro-slavery Southerners, and various conspiracy theories persisted into the late-20th century. The cause of Taylor's death was definitively established in 1991, when his remains were exhumed and an autopsy conducted by Kentucky's chief medical examiner. Subsequent neutron activation analysis conducted at Oak Ridge National Laboratory revealed no evidence of poisoning, as arsenic levels were too low. The analysis concluded Taylor had contracted cholera morbus (acute gastroenteritis), as Washington had open sewers, and his food or drink may have been contaminated.\n\n1865: Abraham Lincoln\nThe assassination of Abraham Lincoln took place on Good Friday, April 14, 1865, as the Civil War was drawing to a close. He died the following morning at the age of 56. The assassination occurred five days after General Robert E. Lee and the Army of Northern Virginia surrendered to General Ulysses S. Grant and the Army of the Potomac following the Battle of Appomattox Court House. Lincoln was the first American president to be killed by an assassin. (The first U.S. president to be confronted by a would-be assassin was Andrew Jackson 30 years earlier, in January 1835.)\nThe assassination of President Lincoln was planned and carried out by the well-known stage actor John Wilkes Booth, a Confederate sympathizer, vehement in his denunciation of Lincoln, and a strong opponent of the abolition of slavery in the United States. Booth and a group of co-conspirators originally plotted to kidnap Lincoln, but later planned to kill him, Vice President Andrew Johnson, and Secretary of State William H. Seward in a bid to help the Confederacy's cause. Johnson's would-be-assassin, George Atzerodt did not carry out his part of the plan, and Johnson succeeded Lincoln as president while Lewis Powell only managed to wound Seward.\nLincoln was shot once in the back of his head while watching the play Our American Cousin with his wife Mary Todd Lincoln at Ford's Theatre in Washington, D.C., on the night of April 14, 1865. An army surgeon who happened to be at Ford's, Doctor Charles Leale, assessed Lincoln's wound as mortal. The unconscious president was then carried across the street from the theater to the Petersen House, where he remained in a coma for eight hours before dying the following morning.\nWithin two weeks of the manhunt for Lincoln's killers, on April 26, 1865, Booth and David Herold were caught in a tobacco barn in Port Conway, Virginia. While Herold surrendered, Booth was shot to death by Boston Corbett, a Union Corporal.\nA three-week series of official functions were held following the president's death. He lay in state in the East Room of the White House which was open to the public on April 18. A funeral service was held the next day, and then the coffin was transported in a procession down Pennsylvania Avenue to the United States Capitol, where a ceremonial burial service was held in the rotunda. After lying in state at the Capitol, Lincoln's remains were transported by train to Springfield, Illinois, for burial. He was interred on May 4, 1865, at Oak Ridge Cemetery in Springfield – the Lincoln Tomb State Historic Site since 1895.\n\n1881: James A. Garfield\nThe assassination of James A. Garfield happened in Washington, D.C., on July 2, 1881. Garfield was shot by Charles J. Guiteau at 9:30 a.m., less than four months into his term as the nation's 20th president. He died 11 weeks later on September 19, 1881, at the age of 49. Vice President Chester A. Arthur succeeded him as president. Garfield was scheduled to leave Washington on July 2, 1881, for his summer vacation. On that day, Guiteau lay in wait for the president at the Baltimore and Potomac Railroad station, on the southwest corner of present-day Sixth Street and Constitution Avenue NW, Washington, D.C.\nPresident Garfield came to the Sixth Street Station on his way to his alma mater, Williams College, where he was scheduled to deliver a speech. Garfield was accompanied by two of his sons, James and Harry, and Secretary of State James G. Blaine. Secretary of War Robert Todd Lincoln waited at the station to see the president off. Garfield had no bodyguard or security detail; with the exception of Abraham Lincoln during the Civil War, early U.S. presidents never used any guards.\nAs President Garfield entered the waiting room of the station, Guiteau stepped forward and pulled the trigger from behind at point-blank range. \"My God, what is that?!\" Garfield cried out, flinging up his arms. Guiteau fired again and Garfield collapsed. One bullet grazed Garfield's shoulder; the other hit him in the back, passing the first lumbar vertebra but missing the spinal cord before coming to rest behind his pancreas.\nGarfield, conscious but in shock, was carried to an upstairs floor of the train station. Lincoln sent for D.C. Bliss, a prominent Washington physician, who soon arrived and examined Garfield's wounds several times, probing for the bullet that remained lodged in the president's body with his fingers and metal probes. Two additional doctors were summoned, and they also probed the entry wound. Eventually there were about twenty people in the room, including at least ten physicians. As Garfield was being cared for, Lincoln, thinking back to the death of his father, said \"How many hours of sorrow I have passed in this town.\"\nGarfield was carried back to the White House. Although doctors told him that he would not survive the night, the president remained conscious and alert. The next morning his vital signs were good and doctors began to hope for recovery. A long vigil began, with Garfield's doctors issuing regular bulletins that the American public followed closely throughout the summer of 1881. His condition fluctuated. Fevers came and went. Garfield struggled to keep down solid food and spent most of the summer eating little, and that only liquids.\nGarfield had been a regular visitor to the shore town of Long Branch, New Jersey, one of the nation's premier summer vacation spots until World War I. In early September, it was decided to bring him to Elberon, a quiet beach town just to the south of Long Branch, in hopes that the beach air would help him recover. When they heard that the president was being brought to their town, local citizens built more than half a mile of tracks in less than 24 hours, enabling Garfield to be brought directly to the door of the oceanfront Franklyn cottage, rather than being moved by carriage from the local Elberon train station. However, Garfield died 12 days later. A granite marker on Garfield Road identifies the former site of the cottage, which was demolished in 1950. Throughout the five-month drama, anxious Americans across the country were kept informed of developments by the news media. The publisher of Frank Leslie's Illustrated Newspaper, Miriam Leslie, was especially quick to publish fully illustrated accounts of key moments, from Garfield's shooting to the embalming of his body.\nChester Arthur was at his home in New York City on the night of September 19, when word came that Garfield had died. After first getting the news, Arthur said \"I hope—my God, I do hope it is a mistake.\" But confirmation by telegram came soon after. Arthur took the presidential oath of office, administered by a New York Supreme Court judge, then left for Long Branch to pay his respects before traveling on to Washington. Garfield's body was taken to Washington, where it lay in state for two days in the Capitol Rotunda before being taken to Cleveland, where the funeral was held on September 26.\nWhen the tracks that had been hastily built to the Franklyn cottage were later torn up, actor Oliver Byron bought the wooden ties, and had local carpenter William Presley build them into a small tea house, in commemoration of the president. The red & white (originally red, white & blue) \"Garfield Tea House\" still survives, resting a couple of blocks away from the site of the cottage on the grounds of the Long Branch Historical Museum, a former Episcopal Church. The church is nicknamed \"The Church of the Presidents,\" as it had been attended by, in addition to Garfield, presidents Chester A. Arthur, Ulysses S. Grant, Benjamin Harrison, Rutherford Hayes, William McKinley, and Woodrow Wilson, during their own visits to Long Branch.\n\n1901: William McKinley\nWilliam McKinley was assassinated on September 6, 1901, inside the Temple of Music on the grounds of the Pan-American Exposition in Buffalo, New York. McKinley was shaking hands with the public when Leon Czolgosz, a Polish-American anarchist, shot him. The 58-year-old president died eight days later on September 14 from gangrene caused by the bullet wounds.\nMcKinley had been elected for a second term in 1900. He enjoyed meeting the public, and was reluctant to accept the security available to his office. The secretary to the president, George B. Cortelyou, feared an assassination attempt would take place during a visit to the Temple of Music, and twice took it off the schedule. McKinley restored it each time.\nCzolgosz had lost his job during the economic Panic of 1893 and turned to anarchism, a political philosophy whose adherents had previously killed foreign leaders. Regarding McKinley as a symbol of oppression, Czolgosz felt it was his duty as an anarchist to kill him. Unable to get near McKinley during the earlier part of the presidential visit, Czolgosz shot McKinley twice as the President reached to shake his hand in the reception line at the temple. One bullet grazed McKinley; the other entered his abdomen and was never found.\nMcKinley initially appeared to be recovering, but took a turn for the worse on September 13 as his wounds became gangrenous, and died early the next morning; Vice President Theodore Roosevelt succeeded him. Roosevelt was hiking near the top of Mt. Marcy, in New York's Adirondack region, when a runner located him to convey the news. After McKinley's murder, for which Czolgosz was put to death in the electric chair, the United States Congress passed legislation to officially charge the Secret Service with the responsibility for protecting the president.\n\n1923: Warren G. Harding\nWarren G. Harding died from a sudden heart attack in his hotel suite while visiting San Francisco on the evening of August 2, 1923, at the age of 57. His death quickly led to theories that he had been poisoned or committed suicide. Rumors of poisoning were fueled, in part, by a book called The Strange Death of President Harding by private detective and former Ohio Gang member Gaston Means, who suggested First Lady Florence Harding had poisoned her husband after learning of his infidelity. Mrs. Harding's refusal to allow an autopsy on President Harding only added to the speculation. According to the physicians attending Harding, however, the symptoms in the days prior to his death all pointed to congestive heart failure. Harding's biographer, Samuel H. Adams, concluded that \"Warren G. Harding died a natural death which, in any case, could not have been long postponed.\"\nImmediately after President Harding's death, Mrs. Harding returned to Washington, D.C., and briefly stayed in the White House with the new president Calvin Coolidge and first lady. For a month, former first lady Harding gathered and destroyed by fire President Harding's correspondence and documents, both official and unofficial. Upon her return to Marion, Ohio, Mrs. Harding hired a number of secretaries to collect and burn President Harding's personal papers. According to Mrs. Harding, she took these actions to protect her husband's legacy. The remaining papers were held and kept from public view by the Harding Memorial Association in Marion.\n\n1945: Franklin D. Roosevelt\nOn March 29, 1945, Franklin D. Roosevelt went to the Little White House in Warm Springs, Georgia, to rest before his anticipated appearance at the founding conference of the United Nations in late April in San Francisco. At around 1:00 pm on April 12, Roosevelt said, \"I have a terrific pain in the back of my head,\" which were his last words. He then slumped forward in his chair, unconscious, and was carried into his bedroom. The president's attending cardiologist, Howard Bruenn, diagnosed a massive cerebral hemorrhage (stroke). The 63-year-old Roosevelt died a few hours later, without regaining consciousness. As Allen Drury later said, \"so ended an era, and so began another.\" After Roosevelt's death, an editorial in The New York Times declared, \"Men will thank God on their knees a hundred years from now that Franklin D. Roosevelt was in the White House.\"\nIn his later years at the White House, when Roosevelt was increasingly overworked, his daughter Anna Roosevelt Boettiger had moved in to provide her father companionship and support. Anna had also arranged for her father to meet with his former mistress, the then widowed Lucy Mercer Rutherfurd. A close friend of both Roosevelt and Mercer who was present, Elizabeth Shoumatoff, rushed Mercer away to avoid negative publicity and implications of infidelity. When Eleanor heard about her husband's death, she was also faced with the news that Anna had been arranging these meetings with Mercer and that Mercer had been with Franklin when he died.\nOn the morning of April 13, Roosevelt's body was placed in a flag-draped coffin and loaded onto the presidential train. After a White House funeral on April 14, Roosevelt was transported back to Hyde Park by train, guarded by four servicemen, one each from the Army, Navy, Marines, and Coast Guard. As was his wish, Roosevelt was buried in the Rose Garden of the Springwood estate, the Roosevelt family home in Hyde Park on April 15. Eleanor died in November 1962 and was buried next to him.\nRoosevelt's death was met with shock and grief across the U.S. and around the world. His declining health had not been known to the general public. Roosevelt had been president for more than 12 years, longer than any other person, and had led the country through some of its greatest crises to the impending defeat of Nazi Germany and within sight of the defeat of Japan as well.\nLess than a month after his death, on May 8, the war in Europe ended. President Harry S. Truman dedicated Victory in Europe Day and its celebrations to Roosevelt's memory, and kept the flags across the U.S. at half-staff for the remainder of the 30-day mourning period. In doing so, Truman said that his only wish was \"that Franklin D. Roosevelt had lived to witness this day.\"\n\n1963: John F. Kennedy\nThe most recent U.S. president to die in office is John F. Kennedy, who was assassinated on November 22, 1963, in Dallas, Texas. He was fatally shot by Lee Harvey Oswald, who fired three shots from a sixth floor window of the Texas School Book Depository at 12:30 p.m. as the presidential motorcade passed through Dealey Plaza. Riding in the vehicle with the president were First Lady Jackie Kennedy, Texas governor John Connally, and Connally's wife Nellie; Governor Connally was also seriously wounded in the attack. The motorcade rushed to Parkland Memorial Hospital, where Kennedy was pronounced dead about 30 minutes later, at the age of 46. Connally recovered from his injuries.\nVice President Lyndon B. Johnson, who was a few cars behind the president in the motorcade, became U.S. president upon Kennedy's death. He took the presidential oath of office onboard Air Force One as it sat on the runway at Dallas Love Field. Oswald was arrested by the Dallas Police Department that afternoon, and was charged under Texas state law with the murder of Kennedy, as well as that of Dallas policeman J. D. Tippit, who had been fatally shot a short time after the assassination. Two days later, on November 24, 1963, as live television cameras were covering his transfer from the city jail to the county jail, Oswald was fatally shot in the basement of Dallas Police Headquarters by Dallas nightclub operator Jack Ruby. Ruby was convicted of Oswald's murder, though it was later overturned on appeal, and Ruby died in prison in 1967 while awaiting a new trial.\nIn 1964, after a 10-month investigation into the assassination, the Warren Commission concluded that President Kennedy was assassinated by Lee Harvey Oswald and that Oswald had acted entirely alone. It also concluded that Jack Ruby acted alone when he killed Oswald in police custody. Nonetheless, speculation over \"what really happened\" on November 22, 1963, in Dallas captured the public imagination during the decades that followed. Polls conducted from 1966 to 2004 found that as many as 80 percent of Americans have suspected that there was a criminal conspiracy or cover-up. Numerous books, films, television specials and websites have examined the assassination in minute detail, and numerous conspiracy theories have been advanced. Parties as varied as the FBI, the CIA, the Mafia, the Cuban and the Soviet governments, along with Kennedy's successor, Lyndon Johnson, have been identified as Suspect. In an article published prior to the 50th anniversary of Kennedy's assassination, author Vincent Bugliosi estimates that a total of 42 groups, 82 assassins, and 214 people have been accused in conspiracy theories challenging the \"lone gunman\" theory.\n\nSee also\nList of United States presidential assassination attempts\nCurse of Tippecanoe\n\nNotes\nReferences\nBibliography\nBauer, K. Jack (1985). Zachary Taylor: Soldier, Planter, Statesman of the Old Southwest. Louisiana State University Press. ISBN 0-8071-1237-2.\nCleaves, Freeman (1939). Old Tippecanoe: William Henry Harrison and His Time. New York, NY: C. Scribner's Sons.\nLeech, Margaret (1959). In the Days of McKinley. New York: Harper and Brothers. pp. 594–600. OCLC 456809.\nMcCullough, David (1992). Truman. Simon & Schuster. ISBN 0-671-86920-5.\nMillard, Candice (2011). Destiny of the Republic. Doubleday. ISBN 978-0-385-53500-7.\nMiller, Scott (2011). The President and the Assassin. New York: Random House. pp. 56–60. ISBN 978-1-4000-6752-7.\nPeskin, Allan (1978). Garfield. Kent State University Press. ISBN 0-87338-210-2.\nVowell, Sarah (2005). Assassination Vacation. Simon and Schuster. ISBN 0-7432-6003-1.\n\nExternal links\nThe Mortal Presidency Archived June 3, 2015, at the Wayback Machine (Shapell Manuscript Foundation)\n\nJames Abram Garfield (November 19, 1831 – September 19, 1881) was the 20th president of the United States, serving from March 1881 until his assassination in September that year. A preacher, lawyer, and Civil War general, Garfield served nine terms in the United States House of Representatives and is the only sitting member of the House to be elected president. Before his candidacy for the presidency, he had been elected to the U.S. Senate by the Ohio General Assembly—a position he declined when he became president-elect.\nGarfield was born into poverty in a log cabin and grew up in northeastern Ohio. After graduating from Williams College, he studied law and became an attorney. He was a preacher in the Stone–Campbell Movement and president of the Western Reserve Eclectic Institute, affiliated with the Disciples. Garfield was elected as a Republican member of the Ohio State Senate in 1859, serving until 1861. He opposed Confederate secession, was a major general in the Union Army during the American Civil War, and fought in the battles of Middle Creek, Shiloh, and Chickamauga. He was elected to Congress in 1862 to represent Ohio's 19th district. Throughout his congressional service, he firmly supported the gold standard and gained a reputation as a skilled orator. He initially agreed with Radical Republican views on Reconstruction but later favored a Moderate Republican–aligned approach to civil rights enforcement for freedmen. Garfield's aptitude for mathematics extended to his own proof of the Pythagorean theorem, which he published in 1876.\nAt the 1880 Republican National Convention, delegates chose Garfield, who had not sought the White House, as a compromise presidential nominee on the 36th ballot. In the 1880 presidential election, he conducted a low-key front porch campaign and narrowly defeated the Democratic nominee, Winfield Scott Hancock. Garfield's accomplishments as president included his assertion of presidential authority against senatorial courtesy in executive appointments, a purge of corruption in the Post Office, and his appointment of a Supreme Court justice. He advocated for agricultural technology, an educated electorate, and civil rights for African Americans. He also proposed substantial civil service reforms, which were passed by Congress in 1883 as the Pendleton Civil Service Reform Act and signed into law by his successor, Chester A. Arthur.\nGarfield was a member of the intraparty \"Half-Breed\" faction who used the powers of the presidency to defy the powerful \"Stalwart\" Senator Roscoe Conkling from New York. He did this by appointing Blaine faction leader William H. Robertson to the lucrative post of Collector of the Port of New York. The ensuing political battle resulted in Robertson's confirmation and the resignations of Conkling and Thomas C. Platt from the Senate.\nOn July 2, 1881, Charles J. Guiteau, a disappointed and delusional office seeker, shot Garfield at the Baltimore and Potomac Railroad Station in Washington. The wound was not immediately fatal, but an infection caused by his doctors' unsanitary methods in treating the wound killed Garfield on September 19. Due to his brief tenure in office, historians tend to rank Garfield as a below-average president, though he has earned praise for anti-corruption and pro-civil rights stances.\n\nChildhood and early life\nJames Abram Garfield was born the youngest of five children on November 19, 1831, in a log cabin in Orange Township, now Moreland Hills, Ohio. Garfield's ancestor Edward Garfield migrated from Hillmorton, Warwickshire, England, to Massachusetts around 1630. James's father Abram was born in Worcester, New York, and came to Ohio to woo his childhood sweetheart, Mehitabel Ballou, only to find her married. He instead wed her sister Eliza, who was born in New Hampshire. James was named after an earlier son of Eliza and Abram who had died in infancy.\nIn early 1833, Abram and Eliza Garfield joined a Stone-Campbell church, a decision that influenced their youngest son's life. Abram died later that year, and James was raised in poverty in a household led by his strong-willed mother. He was her favorite child and the two remained close for the rest of his life. Eliza remarried in 1842, but soon left her second husband, Warren (or Alfred) Belden, and a scandalous divorce was awarded in 1850. James took his mother's side in the matter and noted Belden's 1880 death with satisfaction in his diary. Garfield also enjoyed his mother's stories about his ancestry, especially those about his Welsh great-great-grandfathers and an ancestor who served as a knight of Caerphilly Castle.\nPoor and fatherless, Garfield was mocked by his peers and became sensitive to slights throughout his life; he sought escape through voracious reading. He left home at age 16 in 1847 and was rejected for work on the only ship in port in Cleveland. Garfield instead found work on a canal boat, managing the mules that pulled it. Horatio Alger later used this labor to good effect when he wrote Garfield's campaign biography in 1880.\nAfter six weeks, illness forced Garfield to return home, and during his recuperation, his mother and a local school official secured his promise to forgo canal work for a year of school. In 1848, he began at Geauga Seminary, in nearby Chester Township, Geauga County, Ohio. Garfield later said of his childhood, \"I lament that I was born to poverty, and in this chaos of childhood, seventeen years passed before I caught any inspiration ... a precious 17 years when a boy with a father and some wealth might have become fixed in manly ways.\"\n\nEducation, marriage and early career\nGarfield attended Geauga Seminary from 1848 to 1850 and learned academic subjects for which he had not previously had time. He excelled as a student and was especially interested in languages and elocution. He began to appreciate the power a speaker had over an audience, writing that the speaker's platform \"creates some excitement. I love agitation and investigation and glory in defending unpopular truth against popular error.\" Geauga was coeducational, and Garfield was attracted to one of his classmates, Lucretia Rudolph, whom he later married. To support himself at Geauga, he worked as a carpenter's assistant and teacher. The need to go from town to town to find work as a teacher aggravated Garfield, and he developed a dislike of what he called \"place-seeking\", which became, he said, \"the law of my life.\" In later years, he astounded his friends by disregarding positions that could have been his with little politicking. Garfield had attended church more to please his mother than to worship God, but in his late teens he underwent a religious awakening. He attended many camp meetings, which led to his being born again on March 4, 1850, when he was baptized into Christ by being submerged in the icy waters of the Chagrin River.\nAfter he left Geauga, Garfield worked for a year at various jobs, including teaching jobs. Finding that some New Englanders worked their way through college, Garfield determined to do the same and sought a school that could prepare him for the entrance examinations. From 1851 to 1854, he attended the Western Reserve Eclectic Institute (later named Hiram College) in Hiram, Ohio, a school founded by and still affiliated with the Christian Church (Disciples of Christ). While there, he was most interested in the study of Greek and Latin but was inclined to learn about and discuss any new thing he encountered. Securing a position on entry as janitor, he obtained a teaching position while he was still a student there. Lucretia Rudolph also enrolled at the Institute and Garfield wooed her while teaching her Greek. He developed a regular preaching circuit at neighboring churches and, in some cases, earned one gold dollar per service. By 1854, Garfield had learned all the Institute could teach him and was a full-time teacher. Garfield then enrolled at Williams College in Williamstown, Massachusetts, as a third-year student; he received credit for two years' study at the Institute after passing a cursory examination. Garfield was also impressed with the college president, Mark Hopkins, who had responded warmly to Garfield's letter inquiring about admission. He said of Hopkins, \"The ideal college is Mark Hopkins on one end of a log with a student on the other.\" Hopkins later said of Garfield in his student days, \"There was a large general capacity applicable to any subject. There was no pretense of genius, or alternation of spasmodic effort, but a satisfactory accomplishment in all directions.\" After his first term, Garfield was hired to teach penmanship to the students of nearby Pownal, Vermont, a post Chester A. Arthur previously held.\n\nGarfield graduated Phi Beta Kappa from Williams in August 1856, was named salutatorian, and spoke at the commencement. His biographer Ira Rutkow writes that Garfield's years at Williams gave him the opportunity to know and respect those of different social backgrounds, and that, despite his origin as an unsophisticated Westerner, socially conscious New Englanders liked and respected him. \"In short,\" Rutkow writes, \"Garfield had an extensive and positive first experience with the world outside the Western Reserve of Ohio.\"\nUpon his return to Ohio, the degree from a prestigious Eastern college made Garfield a man of distinction. He returned to Hiram to teach at the Institute and in 1857 was made its principal, though he did not see education as a field that would realize his full potential. The abolitionist atmosphere at Williams had enlightened him politically, after which he began to consider politics as a career. He campaigned for Republican presidential candidate John C. Frémont in 1856. In 1858, he married Lucretia, and they had seven children, five of whom survived infancy. Soon after the wedding, he registered to read law at the office of attorney Albert Gallatin Riddle in Cleveland, though he did his studying in Hiram. He was admitted to the bar in 1861.\nLocal Republican leaders invited Garfield to enter politics upon the death of Cyrus Prentiss, the presumptive nominee for the local state senate seat. He was nominated at the party convention on the sixth ballot and was elected, serving from 1860 to 1861. Garfield's major effort in the state senate was an unsuccessful bill providing for Ohio's first geological survey to measure its mineral resources.\n\nCivil War\nAfter Abraham Lincoln's election as president, several Southern states announced their secession from the Union to form a new government, the Confederate States of America. Garfield read military texts while anxiously awaiting the war effort, which he regarded as a holy crusade against the Slave Power. In April 1861, the rebels bombarded Fort Sumter, one of the South's last federal outposts, beginning the Civil War. Although he had no military training, Garfield knew his place was in the Union Army.\nAt Governor William Dennison's request, Garfield deferred his military ambitions to remain in the legislature, where he helped appropriate the funds to raise and equip Ohio's volunteer regiments. When the legislature adjourned Garfield spent the spring and early summer on a speaking tour of northeastern Ohio, encouraging enlistment in the new regiments. Following a trip to Illinois to purchase muskets, Garfield returned to Ohio and, in August 1861, received a commission as a colonel in the 42nd Ohio Infantry regiment. The 42nd Ohio existed only on paper, so Garfield's first task was to fill its ranks. He did so quickly, recruiting many of his neighbors and former students. The regiment traveled to Camp Chase, outside Columbus, Ohio, to complete training. In December, Garfield was ordered to bring the 42nd to Kentucky, where they joined the Army of the Ohio under Brigadier General Don Carlos Buell.\n\nBuell's command\nBuell quickly assigned Garfield the task of driving Confederate forces out of eastern Kentucky, giving him the 18th Brigade for the campaign, which, besides his own 42nd, included the 40th Ohio Infantry, two Kentucky infantry regiments and two cavalry units. They departed Catlettsburg, Kentucky, in mid-December, advancing through the valley of the Big Sandy River. The march was uneventful until Union forces reached Paintsville, Kentucky, on January 6, 1862, where Garfield's cavalry engaged the rebels at Jenny's Creek. Confederate troops under Brigadier General Humphrey Marshall held the town in numbers roughly equal to Garfield's own, but Garfield positioned his troops so as to deceive Marshall into believing the rebels were outnumbered. Marshall ordered his troops to withdraw to the forks of Middle Creek, on the road to Virginia, and Garfield ordered his troops to take up the pursuit. They attacked the rebel positions on January 9, 1862, in the Battle of Middle Creek, the only pitched battle Garfield commanded personally. At the fighting's end, the Confederates withdrew from the field and Garfield sent his troops to Prestonsburg to reprovision.\n\nIn recognition of his success, Garfield was promoted to brigadier general. After Marshall's retreat, Garfield's command was the sole remaining Union force in eastern Kentucky and he announced that any men who had fought for the Confederacy would be granted amnesty if they returned to their homes, lived peaceably, and remained loyal to the Union. The proclamation was surprisingly lenient, as Garfield now believed the war was a crusade for eradication of slavery. Following a brief skirmish at Pound Gap, the last rebel units in the area were outflanked and retreated to Virginia.\nGarfield's promotion gave him command of the 20th Brigade of the Army of the Ohio, which received orders to join Major General Ulysses S. Grant's forces as they advanced on Corinth, Mississippi, in early 1862. Before the 20th Brigade arrived, however, Confederate forces under General Albert Sidney Johnston surprised Grant's men in their camps, driving them back. Garfield's troops received word of the battle and advanced quickly, joining the rest of the army on the second day to drive the Confederates back across the field and into retreat. The action, later known as the Battle of Shiloh, was the bloodiest of the war to date; Garfield was exposed to fire for much of the day, but emerged uninjured. Major General Henry W. Halleck, Grant's superior, took charge of the combined armies and advanced ponderously toward Corinth; when they arrived, the Confederates had fled.\nThat summer, Garfield suffered from jaundice and significant weight loss. He was forced to return home, where his wife nursed him back to health. While he was home, Garfield's friends worked to gain him the Republican nomination for Congress, but he refused to campaign with the delegates. He returned to military duty that autumn and went to Washington to await his next assignment. During this period of idleness, a rumor of an extramarital affair caused friction in the Garfields' marriage until Lucretia eventually chose to overlook it. Garfield repeatedly received tentative assignments that were quickly withdrawn, to his frustration. In the meantime, he served on the court-martial of Fitz John Porter for his tardiness at the Second Battle of Bull Run. He was convinced of Porter's guilt and voted with his fellow generals to convict Porter. The trial lasted almost two months, from November 1862 to January 1863, and, by its end, Garfield had procured an assignment as chief of staff to Major General William S. Rosecrans.\n\nChief of staff for Rosecrans\nGenerals' chiefs of staff were usually more junior officers, but Garfield's influence with Rosecrans was greater than usual, with duties extending beyond communication of orders to actual management of his Army of the Cumberland. Rosecrans had a voracious appetite for conversation, especially when unable to sleep; in Garfield, he found \"the first well read person in the Army\" and the ideal candidate for discussions that ran deep into the night. They discussed everything, especially religion, and the two became close despite Garfield's being 12 years his junior. Rosecrans, who had converted from Methodism to Roman Catholicism, softened Garfield's view of his faith.\nGarfield recommended that Rosecrans replace wing commanders Alexander McCook and Thomas Crittenden, as he believed they were ineffective, but Rosecrans ignored the suggestion. With Rosecrans, Garfield devised the Tullahoma Campaign to pursue and trap Confederate General Braxton Bragg in Tullahoma. After initial Union success, Bragg retreated toward Chattanooga, where Rosecrans stalled and requested more troops and supplies. Garfield argued for an immediate advance, in line with demands from Halleck and Lincoln. After a council of war and lengthy deliberations, Rosecrans agreed to attack.\nAt the ensuing Battle of Chickamauga on September 19 and 20, 1863, confusion among the wing commanders over Rosecrans's orders created a gap in the lines, resulting in a rout of the right flank. Rosecrans concluded that the battle was lost and fell back on Chattanooga to establish a defensive line. Garfield, however, thought part of the army had held and, with Rosecrans's approval, headed across Missionary Ridge to survey the scene. Garfield's hunch was correct. Consequently, his ride became legendary and Rosecrans's error reignited criticism about the latter's leadership. While Rosecrans's army had avoided disaster, they were stranded in Chattanooga, surrounded by Bragg's army. Garfield sent a telegram to Secretary of War Edwin M. Stanton alerting Washington to the need for reinforcements to avoid annihilation. Lincoln and Halleck responded to the request for reinforcements by sending 20,000 troops to Garfield by rail within nine days. In the meantime, Grant was promoted to command of the western armies and quickly replaced Rosecrans with George H. Thomas. Garfield was ordered to report to Washington, where he was promoted to major general. According to historian Jean Edward Smith, Grant and Garfield had a \"guarded relationship\" since Grant promoted Thomas, rather than Garfield, to command of the Army of the Cumberland after Rosecrans's dismissal.\n\nCongressional career\nElection in 1862; Civil War years\nWhile he served in the Army in early 1862, friends of Garfield approached him about running for Congress from Ohio's newly redrawn and heavily Republican 19th district. He worried that he and other state-appointed generals would receive obscure assignments, and running for Congress would allow him to resume his political career. That the new Congress would not hold its first regular session until December 1863 allowed him to continue his war service for a time. Home on medical leave, he refused to campaign for the nomination, leaving that to political managers who secured it at the local convention in September 1862 on the eighth ballot. In the October general election, he defeated D.B. Woods by a two-to-one margin for a seat in the 38th Congress.\nDays before his Congressional term began, Garfield lost his eldest daughter, three-year-old Eliza, and became anxious and conflicted, saying his \"desolation of heart\" might require his return to \"the wild life of the army.\" He also assumed that the war would end before his joining the House, but it had not, and he felt strongly that he belonged in the field, rather than in Congress. He also thought he could expect a favorable command, so he decided to see President Lincoln. During their meeting, Lincoln recommended he take his House seat, as there was an excess of generals and a shortage of administration congressmen, especially those with knowledge of military affairs. Garfield accepted this recommendation and resigned his military commission to do so.\nGarfield met and befriended Treasury Secretary Salmon P. Chase, who saw Garfield as a younger version of himself. The two agreed politically and both were part of the Radical wing of the Republican Party. Once he took his seat in December 1863, Garfield was frustrated at Lincoln's reluctance to press the South hard. Many radicals, led in the House by Pennsylvania's Thaddeus Stevens, wanted rebel-owned lands confiscated, but Lincoln threatened to veto any bill that proposed to do so on a widespread basis. In debate on the House floor, Garfield supported such legislation and, discussing England's Glorious Revolution, hinted that Lincoln might be thrown out of office for resisting it. Garfield had supported Lincoln's Emancipation Proclamation and marveled at the \"strange phenomenon in the world's history, when a second-rate Illinois lawyer is the instrument to utter words which shall form an epoch memorable in all future ages.\"\nGarfield not only favored the abolition of slavery, but also believed the leaders of the rebellion had forfeited their constitutional rights. He supported the confiscation of Southern plantations and even exile or execution of rebellion leaders as a means to ensure a permanent end to slavery. Garfield felt Congress had an obligation \"to determine what legislation is necessary to secure equal justice to all loyal persons, without regard to color.\" He was more supportive of Lincoln when he took action against slavery.\nGarfield showed leadership early in his congressional career; he was initially the only Republican vote to terminate the use of bounties in military recruiting. Some financially able recruits had used the bounty system to buy their way out of service (called commutation), which Garfield considered reprehensible. He gave a speech pointing out the flaws in the existing conscription law: 300,000 recruits had been called upon to enlist, but barely 10,000 had done so, with the remainder claiming exemption, providing money, or recruiting a substitute. Lincoln appeared before the Military Affairs committee on which Garfield served, demanding a more effective bill; even if it cost him reelection, Lincoln was confident he could win the war before his term expired. After many false starts, Garfield, with Lincoln's support, procured the passage of a conscription bill that excluded commutation.\nUnder Chase's influence, Garfield became a staunch proponent of a dollar backed by a gold standard, and strongly opposed the \"greenback\". He also accepted the necessity of suspension of payment in gold or silver during the Civil War with strong reluctance. He voted with the Radical Republicans in passing the Wade–Davis Bill, designed to give Congress more authority over Reconstruction, but Lincoln defeated it with a pocket veto.\nGarfield did not consider Lincoln very worthy of reelection, but there seemed to be no viable alternative. \"He will probably be the man, though I think we could do better\", he said. Garfield attended the party convention and promoted Rosecrans as Lincoln's running mate, but delegates chose Military Governor of Tennessee Andrew Johnson. Lincoln was reelected, as was Garfield. By then, Chase had left the Cabinet and been appointed Chief Justice, and his relations with Garfield became more distant.\nGarfield took up the practice of law in 1865 to improve his personal finances. His efforts took him to Wall Street where, the day after Lincoln's assassination, a riotous crowd drew him into an impromptu speech to calm their passions: \"Fellow citizens! Clouds and darkness are round about Him! His pavilion is dark waters and thick clouds of the skies! Justice and judgment are the establishment of His throne! Mercy and truth shall go before His face! Fellow citizens! God reigns, and the Government at Washington still lives!\" The speech, with no mention or praise of Lincoln, was, according to Garfield biographer Robert G. Caldwell, \"quite as significant for what it did not contain as for what it did.\" In the following years, Garfield had more praise for Lincoln; a year after Lincoln's death, Garfield said, \"Greatest among all these developments were the character and fame of Abraham Lincoln,\" and in 1878 he called Lincoln \"one of the few great rulers whose wisdom increased with his power\".\nWhen in Washington, Garfield attended Vermont Avenue Christian Church, which later became National City Christian Church, a building constructed and funded by the Disciples.\n\nReconstruction\nIn 1864, the U.S. Senate passed the 13th Amendment, which abolished slavery throughout the Union. The bill failed to pass the House by a two-thirds majority until January 31, 1865, when it was then sent to the states for ratification. The Amendment opened other issues concerning African American civil rights. Garfield asked, \"[What] is freedom? Is it the bare privilege of not being chained?...If this is all, then freedom is a bitter mockery, a cruel delusion.\"\nGarfield supported black suffrage as firmly as he supported abolition. President Johnson sought the rapid restoration of the Southern states during the months between his accession and the meeting of Congress in December 1865; Garfield hesitantly supported this policy as an experiment. Johnson, an old friend, sought Garfield's backing and their conversations led Garfield to assume Johnson's differences with Congress were not large. When Congress assembled in December (to Johnson's chagrin, without the elected representatives of the Southern states, who were excluded), Garfield urged conciliation on his colleagues, although he feared that Johnson, a former Democrat, might join other Democrats to gain political control. Garfield foresaw conflict even before February 1866, when Johnson vetoed a bill to extend the life of the Freedmen's Bureau, charged with aiding the former slaves. By April, Garfield had concluded that Johnson was either \"crazy or drunk with opium.\"\n\nThe conflict between Congress and President Johnson was the major issue of the 1866 campaign, with Johnson taking to the campaign trail in a Swing Around the Circle and Garfield facing opposition within the Republican party in his home district. With the South still disenfranchised and Northern public opinion behind the Republicans, they gained a two-thirds majority in both houses of Congress. Garfield, having overcome his challengers at the district nominating convention, won reelection easily.\nGarfield opposed the proposed impeachment of Johnson initially when Congress convened in December 1866, but supported legislation to limit Johnson's powers, such as the Tenure of Office Act, which restricted Johnson's ability to remove presidential appointees. Distracted by committee duties, Garfield spoke about these bills rarely, but was a loyal Republican vote against Johnson.\nOn January 7, 1867, Garfield voted in support of the resolution that launched the first impeachment inquiry against Johnson (run by the House Committee on the Judiciary). On December 7, 1867, he voted against the unsuccessful resolution to impeach Johnson that the House Committee on the Judiciary had sent the full House. On January 27, 1868, he voted to pass the resolution that authorized the second impeachment inquiry against Johnson (run by the House Select Committee on Reconstruction). Due to a court case, he was absent on February 24, 1868, when the House impeached Johnson, but gave a speech aligning himself with Thaddeus Stevens and others who sought Johnson's removal shortly thereafter. Garfield was present on March 2 and 3, 1868, when the House voted on specific articles of impeachment, and voted in support of all 11 articles. During the March 2 debate on the articles, Garfield argued that what he characterized as Johnson's attempts to render Ulysses S. Grant, William Tecumseh Sherman, and William H. Emory personal tools of his demonstrated Johnson's intent to disregard the law and override the Constitution, suggesting that Johnson's trial perhaps could be expedited to last only a day in order to hasten his removal. When Johnson was acquitted in his trial before the Senate, Garfield was shocked and blamed the outcome on the trial's presiding officer, Chief Justice Chase, his onetime mentor.\nBy the time Grant succeeded Johnson in 1869, Garfield had moved away from the remaining radicals (Stevens, their leader, had died in 1868). By this time, many in the Republican Party wanted to remove the \"Negro question\" from national affairs. Garfield hailed the ratification of the 15th Amendment in 1870 as a triumph and favored Georgia's readmission to the Union as a matter of right, not politics. An influential Republican, Garfield said, \"[The] Fifteen Amendment confers on the African race the care of its own destiny. It places their fortunes in their own hands.\" In 1871, Congress took up the Ku Klux Klan Act, which was designed to combat attacks on African Americans' suffrage rights. Garfield opposed the act, saying, \"I have never been more perplexed by a piece of legislation.\" He was torn between his indignation at the Klan, whom he called \"terrorists\", and his concern for the power given the president to enforce the act through suspension of habeas corpus.\n\nTariffs and finance\nThroughout his political career, Garfield favored the gold standard and decried attempts to increase the money supply through the issuance of paper money not backed by gold, and later, through the free and unlimited coinage of silver. In 1865, he was put on the House Ways and Means Committee, a long-awaited opportunity to focus on financial and economic issues. He reprised his opposition to the greenback, saying, \"Any party which commits itself to paper money will go down amid the general disaster, covered with the curses of a ruined people.\" In 1868 Garfield gave a two-hour speech on currency in the House, which was widely applauded as his best oratory to that point; in it, he advocated a gradual resumption of specie payments, that is, the government paying out silver and gold, rather than paper money that could not be redeemed.\nTariffs had been raised to high levels during the Civil War. Afterward, Garfield, who made a close study of financial affairs, advocated moving toward free trade, though the standard Republican position was a protective tariff that would allow American industries to grow. This break with his party likely cost him his place on the Ways and Means Committee in 1867, and though Republicans held the majority in the House until 1875, Garfield remained off that committee. Garfield came to chair the powerful House Appropriations Committee, but it was Ways and Means, with its influence over fiscal policy, that he really wanted to lead. One reason he was denied a place on Ways and Means was the opposition of the influential Republican editor Horace Greeley.\n\nStarting in January 1870, Garfield, then chairman of the House Banking Committee, led an investigation into the Black Friday Gold Panic scandal. In 1869, during Grant's first term in office, two New York conspirators, Jay Gould and James Fisk, launched a scheme to corner the gold market. The conspiracy was broken on Friday, September 24, 1869, when Grant and Treasury Secretary George Boutwell released gold into the market, causing widespread financial panic. During the investigation, rumors spread that Grant's family might have been involved. In order not to force Grant's wife to testify, Garfield had a private meeting with Grant at the White House. When Garfield showed Grant testimony about him and his family, Grant thanked Garfield but refused to read it or give a response. Grant personally resented Garfield for investigating Black Friday and his wife Julia concerning possible involvement in the scandal.\nGarfield's investigation and final majority report, released on September 12, 1870, were thorough but found no indictable offenses and exonerated Grant and Julia of wrongdoing. Garfield thought the scandal was enabled by the greenbacks that financed the speculation. Garfield was not at all enthused about President Grant's reelection in 1872—until Greeley, who emerged as the candidate of the Democrats and Liberal Republicans, became the only serious alternative. Garfield said, \"I would say Grant was not fit to be nominated and Greeley is not fit to be elected.\" Both Grant and Garfield were overwhelmingly reelected.\n\nCrédit Mobilier scandal; salary grab\nThe Crédit Mobilier of America scandal involved corruption in the financing of the Union Pacific Railroad, part of the transcontinental railroad which was completed in 1869. Union Pacific officers and directors secretly purchased control of the Crédit Mobilier of America company, then contracted with it to undertake construction of the railroad. The railroad paid the company's grossly inflated invoices with federal funds appropriated to subsidize the project, and the company was allowed to purchase Union Pacific securities at par value, well below the market rate. Crédit Mobilier showed large profits and stock gains, and distributed substantial dividends. The high expenses meant Congress was called upon to appropriate more funds. One of the railroad officials who controlled Crédit Mobilier was also a congressman, Oakes Ames of Massachusetts. He offered some of his colleagues the opportunity to buy Crédit Mobilier stock at par value, well below what it sold for on the market, and the railroad got its additional appropriations.\n\nThe story broke in July 1872, in the middle of the presidential campaign. Among those named were Vice President Schuyler Colfax, Massachusetts Senator Henry Wilson (the Republican candidate for vice president), Speaker James G. Blaine of Maine, and Garfield. Greeley had little luck taking advantage of the scandal. When Congress reconvened after the election, Blaine, seeking to clear his name, demanded a House investigation. Evidence before the special committee exonerated Blaine. Garfield had said in September 1872 that Ames had offered him stock but he had repeatedly refused it. Testifying before the committee in January, Ames said he had offered Garfield ten shares of stock at par value, but that Garfield had never taken them or paid for them, though a year passed, from 1867 to 1868, before Garfield had finally refused. Appearing before the committee on January 14, 1873, Garfield confirmed much of this. Ames testified several weeks later that Garfield agreed to take the stock on credit, and that it was paid for by the company's huge dividends. The two men differed over $300 that Garfield received and later paid back, with Garfield deeming it a loan and Ames a dividend.\nGarfield's biographers have been unwilling to exonerate him in the scandal. Allan Peskin writes, \"Did Garfield lie? Not exactly. Did he tell the truth? Not completely. Was he corrupted? Not really. Even Garfield's enemies never claimed that his involvement in the affair influenced his behavior.\" Rutkow writes, \"Garfield's real offense was that he knowingly denied to the House investigating committee that he had agreed to accept the stock and that he had also received a dividend of $329.\" Caldwell suggests Garfield \"told the truth [before the committee, but] certainly failed to tell the whole truth, clearly evading an answer to certain vital questions and thus giving the impression of worse faults than those of which he was guilty.\" That Crédit Mobilier was a corrupt organization had been a badly kept secret, even mentioned on the floor of Congress, and editor Sam Bowles wrote at the time that Garfield, in his positions on committees dealing with finance, \"had no more right to be ignorant in a matter of such grave importance as this, than the sentinel has to snore on his post.\"\nAnother issue that caused Garfield trouble in his 1874 reelection bid was the so-called \"Salary Grab\" of 1873, which increased the compensation for members of Congress by 50%, retroactive to 1871. As chairman of the Appropriations Committee, Garfield was responsible for shepherding the appropriations bill through the House; during the debate in February 1873, Massachusetts Representative Benjamin Butler offered the increase as an amendment, and despite Garfield's opposition, it passed the House and eventually became law. The law was very popular in the House, as almost half the members were lame ducks, but the public was outraged, and many of Garfield's constituents blamed him, though he personally refused to accept the increase. In a bad year for Republicans, who lost control of the House for the first time since the Civil War, Garfield had his closest congressional election, winning with only 57% of the vote.\n\nFloor leader; Hayes administration\nThe Democratic takeover of the House of Representatives in 1875 meant the loss of Garfield's chairmanship of the Appropriations Committee, though the Democrats did put him on the Ways and Means Committee. With many of his leadership rivals defeated in the 1874 Democratic landslide, and Blaine elected to the Senate, Garfield was seen as the Republican floor leader, and the likely Speaker, should the party regain control of the chamber.\nGarfield thought the land grants given to expanding railroads was an unjust practice. He also opposed monopolistic practices by corporations, as well as the power sought by workers' unions. He supported the proposed establishment of the United States civil service as a means of ridding officials of the annoyance of aggressive office seekers. He especially wished to eliminate the practice of forcing government workers, in exchange for their positions, to kick back a percentage of their wages as political contributions.\nAs the 1876 presidential election approached, Garfield was loyal to the candidacy of Senator Blaine, and fought for the former Speaker's nomination at the 1876 Republican National Convention in Cincinnati. When it became clear, after six ballots, that Blaine could not prevail, the convention nominated Ohio Governor Rutherford B. Hayes. Although Garfield had supported Blaine, he had kept good relations with Hayes, and wholeheartedly supported the governor. Garfield had hoped to retire from politics after his term expired to devote himself full-time to the practice of law, but to help his party, he sought re-election, and won it easily that October. Any celebration was short-lived, as Garfield's youngest son, Neddie, fell ill with whooping cough shortly after the congressional election, and soon died.\n\nWhen Hayes appeared to have lost the presidential election the following month to Democrat Samuel Tilden, the Republicans launched efforts to reverse the results in South Carolina, Louisiana, and Florida, where they held the governorship. If Hayes won all three states, he would take the election by a single electoral vote. Grant asked Garfield to serve as a \"neutral observer\" of the recount in Louisiana. The observers soon recommended to the state electoral commissions that Hayes be declared the winner—Garfield recommended the entire vote of West Feliciana Parish, which had given Tilden a sizable majority, be thrown out. The Republican governors of the three states certified that Hayes had won their states, to the outrage of Democrats, who had the state legislatures submit rival returns, and threatened to prevent the counting of the electoral vote—under the Constitution, Congress is the final arbiter of the election. Congress then established an Electoral Commission, consisting of eight Republicans and seven Democrats, to determine the winner. Despite his objection to the Commission, Garfield was appointed to it. He felt Congress should count the vote and proclaim Hayes victorious. Hayes emerged the victor by a party line vote of 8–7. In exchange for recognizing Hayes as president, Southern Democrats secured the removal of federal troops from the South, ending Reconstruction.\nAlthough an Ohio Senate seat would be vacated by the resignation of John Sherman to become Treasury Secretary, Hayes needed Garfield's expertise to protect him from the agenda of a hostile Congress, and asked him not to seek it. Garfield agreed. As Hayes's key legislator in the House, he gained considerable prestige and respect for his role there. When Congress debated the Bland–Allison Act, to have the government purchase large quantities of silver and strike it into legal tender dollar coins, Garfield opposed it as a deviation from the gold standard; it was enacted over Hayes's veto in February 1878.\nIn 1876, Garfield purchased the property in Mentor that reporters later dubbed Lawnfield, where he conducted the first successful front porch campaign for the presidency. Hayes suggested that Garfield run for governor in 1879, seeing that as a road likely to take Garfield to the White House. Garfield preferred to seek election as a U.S. senator. Rivals were spoken of for the seat, such as Secretary Sherman, but he had presidential ambitions (for which he sought Garfield's support), and other candidates fell by the wayside. The General Assembly elected Garfield to the Senate in January 1880, though his term was not scheduled to commence until March 4, 1881.\n\nLegal career and other activities\nIn 1865, Garfield became a partner in the law firm of a fellow Disciple of Christ, Jeremiah Black. They had much in common, except politics: Black was an avid Democrat, having served in the cabinet of President James Buchanan. The next year, Black was retained by some pro-Confederate northern civilians who had been found guilty of treason in a military court and sentenced to death. Black saw an opportunity to strike a blow against military courts and the Republicans. He had heard Garfield's military speeches, and learned of not only his oratory skills but also his resistance to expansive powers of military commissions. Black assigned the case to Garfield one week before arguments were to be made before the U. S. Supreme Court. When Black warned him of the political peril, Garfield responded, \"It don't make any difference. I believe in English liberty and English law.\" In this landmark case, Ex parte Milligan, Garfield successfully argued that civilians could not be tried before military tribunals, despite a declaration of martial law, as long as civil courts were still operating. In his first court appearance, Garfield's oral argument lasted over two hours, and though his wealthy clients refused to pay him, he had established himself as a preeminent lawyer.\nDuring Grant's first term, Garfield was discontented with public service and in 1872 again pursued opportunities in the law. But he declined a partnership offer from a Cleveland law firm when told his prospective partner was of \"intemperate and licentious\" reputation. In 1873, after Chase's death, Garfield appealed to Grant to appoint Justice Noah H. Swayne Chief Justice, but Grant appointed Morrison R. Waite.\n\nIn 1871, Garfield traveled to Montana Territory to negotiate the removal of the Bitterroot Salish tribe to the Flathead Indian Reservation. Having been told that the people would happily move, Garfield expected an easy task. Instead, he found the Salish determined to stay in their Bitterroot Valley homeland. His attempts to coerce Chief Charlo to sign the agreement nearly brought about a military clash. In the end, he convinced two subchiefs to sign and move to the reservation with a few of the Salish people. Garfield never convinced Charlo to sign, although the official treaty document voted on by Congress bore his forged mark.\nIn 1876, Garfield developed a trapezoid proof of the Pythagorean theorem, which was published in the New England Journal of Education. Mathematics historian William Dunham wrote that Garfield's trapezoid work was \"really a very clever proof.\" According to the Journal, Garfield arrived at the proof \"in mathematical amusements and discussions with other members of congress.\"\nAfter his conversion experience in 1850, religious inquiry was a high priority for Garfield. He read widely and moved beyond the confines of his early experience as a member of the Disciples of Christ. His new, broader perspective was rooted in his devotion to freedom of inquiry and his study of history. The intensity of Garfield's religious thought was also influenced by his experience in combat and his interaction with voters.\n\nPresidential election of 1880\nRepublican nomination\nHaving just been elected to the Senate with John Sherman's support, Garfield was committed to Sherman for the 1880 Republican presidential nomination. Before the convention began, however, a few Republicans, including Wharton Barker of Philadelphia, thought Garfield the best choice for the nomination. Garfield denied any interest in the position, but the attention was enough to make Sherman suspicious of his lieutenant's ambitions. Besides Sherman, the early favorites for the nomination were Blaine, former President Grant; several other candidates attracted delegates as well.\nThe Republican Party at the time was split into two factions: the \"Stalwarts\", who supported the existing federal government patronage system, and the \"Half-Breeds\", who wanted civil service reform. As the convention began, New York Senator Roscoe Conkling, floor leader for the Stalwarts, who supported former President Ulysses S. Grant, proposed that the delegates pledge to back the eventual nominee in the general election. When three West Virginia delegates declined to be so bound, Conkling sought to expel them from the convention. Garfield rose to defend the men, giving a passionate speech in defense of their right to reserve judgment. The crowd turned against Conkling, and he withdrew the motion. The performance delighted Garfield's boosters, who were then convinced he was the only one who could attract a majority of the delegates' votes.\nAfter speeches in favor of the other front-runners, Garfield rose to place Sherman's name in nomination; his speech was well-received, but the delegates mustered little excitement for Sherman as the next president. The first ballot showed Grant leading with 304 votes to Blaine's 284, and Sherman's 93 votes placed him in a distant third. Subsequent ballots demonstrated a deadlock between Grant and Blaine, with neither having the 379 votes needed for nomination. Jeremiah McLain Rusk, a member of the Wisconsin delegation, and Benjamin Harrison, an Indiana delegate, sought to break the deadlock by shifting a few of the anti-Grant votes to a dark horse candidate—Garfield. Garfield gained 50 votes on the 35th ballot, and a stampede began. Garfield protested to the Ohio delegation that he did not seek the nomination and would not betray Sherman, but they overruled his objections and cast their ballots for him. In the next round of voting, nearly all the Sherman and Blaine delegates shifted their support to Garfield, giving him 399 votes, and the Republican nomination. Most of the Grant forces backed the former president to the end, creating a disgruntled Stalwart minority in the party. To obtain that faction's support for the ticket, Chester A. Arthur, a former New York customs collector and member of Conkling's political machine, was chosen as the vice presidential nominee.\n\nCampaign against Hancock\nEven with a Stalwart on the ticket, animosity between the Republican factions carried over from the convention, so Garfield traveled to New York to meet with party leaders. After convincing the Stalwart crowd to put aside their differences and unite for the coming campaign, Garfield returned to Ohio, leaving the active campaigning to others, as was traditional at the time. Meanwhile, the Democrats settled on their nominee, Major General Winfield Scott Hancock of Pennsylvania, a career military officer. Hancock and the Democrats expected to carry the Solid South, while much of the North was considered safe territory for Garfield and the Republicans; most of the campaign focused on a few close states, including New York and Indiana.\nPractical differences between the candidates were few, but Republicans began the campaign with the familiar theme of waving the bloody shirt. They reminded Northern voters the Democratic Party was responsible for secession and four years of civil war, and Democrats would reverse the gains of that war, dishonor Union veterans, and pay Confederate veterans pensions out of the federal treasury. Fifteen years had passed since the end of the war, and with Union generals at the head of both tickets, the bloody shirt was of diminishing value in exciting the voters. With a few months to go before the election, the Republicans switched tactics to emphasize the tariff. Seizing on the Democratic platform's call for a \"tariff for revenue only\", Republicans told Northern workers a Hancock presidency would weaken the tariff protection that kept them in good jobs. Hancock made the situation worse when, attempting to strike a moderate stance, he said, \"The tariff question is a local question.\" The Republican ploy proved effective in uniting the North behind Garfield. Ultimately, of the more than 9.2 million popular votes cast, fewer than 2,000 separated the two candidates. But in the Electoral College, Garfield had an easy victory over Hancock, 214 to 155. The election made Garfield the only sitting member of the House ever to be elected to the presidency.\n\nPresidency (1881)\nCabinet and inauguration\nBefore his inauguration, Garfield was occupied with assembling a cabinet that might engender peace between the party's Conkling and Blaine factions. Blaine's delegates had provided much of the support for Garfield's nomination, so the Maine senator received the place of honor as Secretary of State. Blaine was not only the president's closest advisor, but he was also obsessed with knowing all that took place in the White House, and allegedly posted spies there in his absence. Garfield nominated William Windom of Minnesota as Secretary of the Treasury, William H. Hunt of Louisiana as Secretary of the Navy, Robert Todd Lincoln as Secretary of War, and Samuel J. Kirkwood of Iowa as Secretary of the Interior. New York was represented by Thomas Lemuel James as Postmaster General. Garfield appointed Pennsylvania's Wayne MacVeagh, an adversary of Blaine's, as Attorney General. Blaine tried to sabotage the appointment by convincing Garfield to name an opponent of MacVeagh, William E. Chandler, as Solicitor General under MacVeagh. Only Chandler's rejection by the Senate forestalled MacVeagh's resignation over the matter.\nBecause Garfield was distracted by cabinet maneuvering, his inaugural address was a \"compendium of platitudes\" and fell below expectations. At one high point, however, Garfield emphasized the civil rights of African-Americans, saying \"Freedom can never yield its fullness of blessings so long as the law or its administration places the smallest obstacle in the pathway of any virtuous citizen.\" After discussing the gold standard, the need for education, and an unexpected denunciation of Mormon polygamy, the speech ended. The crowd applauded, but the speech, according to Peskin, \"however sincerely intended, betrayed its hasty composition by the flatness of its tone and the conventionality of its subject matter.\"\nGarfield's appointment of James infuriated Conkling, a factional opponent of the Postmaster General, who demanded a compensatory appointment for his faction, such as the position of Secretary of the Treasury. The resulting squabble occupied much of Garfield's brief presidency. The feud with Conkling reached a climax when the president, at Blaine's instigation, nominated Conkling's enemy, Judge William H. Robertson, to be Collector of the Port of New York. This was one of the prize patronage positions below cabinet level and was then held by Edwin A. Merritt. Conkling raised the time-honored principle of senatorial courtesy in an attempt to defeat the nomination, to no avail. Garfield, who believed the practice was corrupt, would not back down and threatened to withdraw all nominations unless Robertson was confirmed, intending to \"settle the question whether the president is registering clerk of the Senate or the Executive of the United States.\" Ultimately, Conkling and his New York colleague, Senator Thomas C. Platt, resigned their Senate seats to seek vindication but found only further humiliation when the New York legislature elected others in their places. Robertson was confirmed as Collector and Garfield's victory was clear. To Blaine's chagrin, the victorious Garfield returned to his goal of balancing the interests of party factions and nominated a number of Conkling's Stalwart friends to offices.\nWith his cabinet complete, Garfield had to contend with myriad office seekers. He exclaimed, \"My God! What is there in this place that a man should ever get into it.\" Garfield's family happily settled into the White House, but he found presidential duties exasperating.\n\nRefinance of national debt\nGarfield ordered the Secretary of the Treasury William Windom to refund (refinance) the national debt by calling in outstanding U.S. bonds paying 6% interest. Holders would have the option of accepting cash or new bonds at 3%, closer to the interest rates of the time. Taxpayers were saved an estimated $10 million. By comparison, federal expenditures in 1881 were below $261 million (~$7.09 billion in 2023).\n\nSupreme Court nomination\nIn 1880, President Hayes had nominated Stanley Matthews to the Supreme Court but the Senate declined to act on the nomination. In March 1881, Garfield re-nominated Matthews to the Court and the Senate confirmed Matthews by a vote of 24–23. According to The New York Times, \"opposition to Matthews's Supreme Court appointment ... stemmed from his prosecution in 1859 of a newspaper editor who had assisted two runaway slaves.\" Because Matthews was \"a professed abolitionist at the time, the matter was later framed as political expediency triumphing over moral principle.\" Matthews served on the Court until his death in 1889.\n\nReforms\nGrant and Hayes had both advocated civil service reform, and by 1881 such reform associations had organized with renewed energy across the nation. Garfield sympathized with them, believing the spoils system damaged the presidency and often eclipsed more important concerns. Some reformers became disappointed when Garfield promoted limited tenure only to minor office seekers and gave appointments to his old friends.\nCorruption in the post office also cried out for reform. In April 1880, there had been a congressional investigation of corruption in the Post Office Department, where profiteering rings allegedly stole millions of dollars, securing bogus mail contracts on star routes. After obtaining contracts with the lowest bid, costs to run the mail routes would be escalated and profits would be divided among ring members. Shortly after taking office, Garfield received word of postal corruption by an alleged star route ringleader, Assistant Postmaster General Thomas J. Brady. Garfield demanded Brady's resignation and ordered prosecutions that ended in trials for conspiracy. When told that his party, including his campaign manager, Stephen W. Dorsey, was involved, Garfield directed that the corruption in the Post Office be rooted out \"to the bone\", regardless of where it might lead. Brady resigned and was indicted for conspiracy, though jury trials in 1882 and 1883 found Brady not guilty.\n\nCivil rights and education\nGarfield believed the key to improving the state of African American civil rights was government supported education. During Reconstruction, freedmen had gained citizenship and suffrage, which enabled them to participate in government, but Garfield believed their rights were being eroded by Southern white resistance and illiteracy, and he was concerned that blacks would become America's permanent \"peasantry\". He proposed a \"universal\" education system funded by the federal government. In February 1866, as a congressman from Ohio, Garfield and Ohio School Commissioner Emerson Edward White had drafted a bill for the National Department of Education. They believed that through the use of statistics they could push the US Congress to establish a federal agency for school reform. But by the time of Garfield's presidency, Congress and the northern white public had lost interest in African-American rights, and Congress did not pass federal funding for universal education during his term. Garfield also worked to appoint several African Americans to prominent positions: Frederick Douglass, recorder of deeds in Washington; Robert Elliot, special agent to the Treasury; John M. Langston, Haitian minister; and Blanche K. Bruce, register to the Treasury. Garfield believed Southern support for the Republican Party could be gained by \"commercial and industrial\" interests rather than race issues and began to reverse Hayes's policy of conciliating Southern Democrats. He appointed William H. Hunt, a Republican from Louisiana, as Secretary of the Navy. To break the hold of the resurgent Democratic Party in the Solid South, Garfield took patronage advice from Virginia Senator William Mahone of the biracial independent Readjuster Party, hoping to add the independents' strength to the Republicans' there.\n\nForeign policy and naval reform\nGarfield had little foreign policy experience, so he leaned heavily on Blaine. They agreed on the need to promote freer trade, especially within the Western Hemisphere. Garfield and Blaine believed increasing trade with Latin America would be the best way to keep the United Kingdom of Great Britain and Ireland from dominating the region. And by encouraging exports, they believed they could increase American prosperity. Garfield authorized Blaine to call for a Pan-American conference in 1882 to mediate disputes among the Latin American nations and to serve as a forum for talks on increasing trade.\nAt the same time, they hoped to negotiate a peace in the War of the Pacific then being fought by Bolivia, Chile, and Peru. Blaine favored a resolution that would result in Peru yielding no territory, but Chile by 1881 had occupied the Peruvian capital of Lima, and rejected any settlement that restored the previous status quo.\nGarfield sought to expand American influence in other areas, calling for renegotiation of the Clayton–Bulwer Treaty to allow the United States to construct a canal through Panama without British involvement and attempting to reduce British influence in the strategically located Kingdom of Hawaii. Garfield's and Blaine's plans for the United States' involvement in the world stretched even beyond the Western Hemisphere, as he sought commercial treaties with Korea and Madagascar. Garfield also considered enhancing U.S. military strength abroad, asking Navy Secretary Hunt to investigate the navy's condition with an eye toward expansion and modernization. In the end, these ambitious plans came to nothing after Garfield was assassinated. Nine countries had accepted invitations to the Pan-American conference, but the invitations were withdrawn in April 1882 after Blaine resigned from the cabinet and Arthur, Garfield's successor, cancelled the conference. Naval reform continued under Arthur, on a more modest scale than Garfield and Hunt had envisioned, ultimately ending in the construction of the Squadron of Evolution.\n\nAssassination\nGuiteau and shooting\nCharles J. Guiteau had followed various professions in his life, but in 1880 had determined to gain federal office by supporting what he expected would be the winning Republican ticket. He composed a speech, \"Garfield vs. Hancock\", and got it printed by the Republican National Committee. One means of persuading the voters in that era was through orators expounding on the candidate's merits, but with the Republicans seeking more famous men, Guiteau received few opportunities to speak. On one occasion, according to Kenneth D. Ackerman, Guiteau was unable to finish his speech due to nerves. Guiteau, who considered himself a Stalwart, deemed his contribution to Garfield's victory sufficient to justify his appointment to the position of consul in Paris, despite the fact that he spoke no French, nor any foreign language. One medical expert has since described Guiteau as possibly a narcissistic schizophrenic; neuroscientist Kent Kiehl assessed him as a clinical psychopath.\n\nOne of Garfield's more wearying duties was seeing office-seekers, and he saw Guiteau at least once. White House officials suggested to Guiteau that he approach Blaine, as the consulship was within the Department of State. Blaine also saw the public regularly, and Guiteau became a regular at these sessions. Blaine, who had no intention of giving Guiteau a position he was unqualified for and had not earned, simply said the deadlock in the Senate over Robertson's nomination made it impossible to consider the Paris consulship, which required Senate confirmation. Once the New York senators had resigned, and Robertson had been confirmed as Collector, Guiteau pressed his claim, and Blaine told him he would not receive the position.\nGuiteau came to believe he had lost the position because he was a Stalwart. He decided the only way to end the Republican Party's internecine warfare was for Garfield to die—though he had nothing personal against the president. Arthur's succession would restore peace, he felt, and lead to rewards for fellow Stalwarts, including Guiteau.\nThe assassination of Abraham Lincoln was deemed a fluke due to the Civil War, and Garfield, like most people, saw no reason the president should be guarded; his movements and plans were often printed in the newspapers. Guiteau knew Garfield would leave Washington for a cooler climate on July 2, 1881, and made plans to kill him before then. He purchased a gun he thought would look good in a museum, and followed Garfield several times, but each time his plans were frustrated, or he lost his nerve. His opportunities dwindled to one—Garfield's departure by train for New Jersey on the morning of July 2.\nGuiteau concealed himself by the ladies' waiting room at the Sixth Street Station of the Baltimore and Potomac Railroad, from where Garfield was scheduled to depart. Most of Garfield's cabinet planned to accompany him at least part of the way. Blaine, who was to remain in Washington, came to the station to see him off. The two men were deep in conversation and did not notice Guiteau before he took out his revolver and shot Garfield twice, once in the back and once in the arm. Guiteau attempted to leave the station but was quickly captured. As Blaine recognized him, Guiteau was led away, and said, \"I did it. I will go to jail for it. I am a Stalwart and Arthur will be President.\" News of his motivation to benefit the Stalwarts reached many with the news of the shooting, causing rage against that faction.\n\nTreatment and death\nGarfield was struck by two shots: one glanced off his arm while the other pierced his back, shattering a rib and embedding itself in his abdomen. \"My God, what is this?\" he exclaimed. Among those at the station was Robert Todd Lincoln, who was deeply upset, thinking back to when his father Abraham Lincoln was assassinated 16 years earlier. Garfield was taken on a mattress upstairs to a private office, where several doctors examined him. At his request, Garfield was taken back to the White House, and his wife, then in New Jersey, was sent for. Blaine sent word to Vice President Arthur in New York City, who received threats against his life because of his animosity toward Garfield and Guiteau's statements.\nAlthough Joseph Lister's pioneering work in antisepsis was known to American doctors, few of them had confidence in it, and none of his advocates were among Garfield's treating physicians. The physician who took charge at the depot and then at the White House was Doctor Willard Bliss. A noted physician and surgeon, Bliss was an old friend of Garfield, and about a dozen doctors, led by Bliss, were soon probing the wound with unsterilized fingers and instruments. Garfield was given morphine for the pain, and asked Bliss to frankly tell him his chances, which Bliss put at one in a hundred. \"Well, Doctor, we'll take that chance.\"\nOver the next few days, Garfield made some improvement, as the nation viewed the news from the capital and prayed. Although he never stood again, he was able to sit up and write several times, and his recovery was viewed so positively that a steamer was fitted out as a seagoing hospital to aid with his convalescence. He was nourished on oatmeal porridge (which he detested) and milk from a cow on the White House lawn. When told that Indian chief Sitting Bull, a prisoner of the army, was starving, Garfield said, \"Let him starve...\" initially, but a few moments later said, \"No, send him my oatmeal.\"\n\nX-ray imaging, which could have assisted physicians in precisely locating the bullet in Garfield's body, would not be invented for another 14 years. Alexander Graham Bell tried to locate the bullet with a primitive metal detector, but was unsuccessful, though the device had been effective when tested on others. But Bliss limited its use on Garfield, ensuring he remained in charge. Because Bliss insisted the bullet rested someplace it did not, the detector could not locate it. Bell shortly returned after adjusting his device, which emitted an unusual tone in the area where Bliss believed the bullet was lodged. Bliss took this as confirmation that the bullet was where he declared it to be. Bliss recorded the test as a success, saying it was: now unanimously agreed that the location of the ball has been ascertained with reasonable certainty, and that it lies, as heretofore stated, in the front wall of the abdomen, immediately over the groin, about five inches [130 mm] below and to the right of the navel.\nOne means of keeping Garfield comfortable in Washington's summer heat was one of the first successful air conditioning units: air propelled by fans over ice and then dried reduced the temperature in the sickroom by 20 °F (11 °C). Engineers from the navy, and other scientists, worked together to develop it, though there were problems to solve, such as excessive noise and increased humidity.\nOn July 23, Garfield took a turn for the worse when his temperature increased to 104 °F (40 °C); doctors, concerned by an abscess at the wound, inserted a drainage tube. This initially helped, and the bedridden Garfield held a brief cabinet meeting on July 29; members were under orders from Bliss to discuss nothing that might excite Garfield. Doctors probed the abscess, hoping to find the bullet; they likely made the infections worse. Garfield performed only one official act in August, signing an extradition paper. By the end of the month, he was much feebler than he had been, and his weight had decreased from 210 pounds (95 kg) to 130 pounds (59 kg).\nGarfield had long been anxious to escape hot, unhealthy Washington, and in early September the doctors agreed to move him to Elberon, part of Long Branch, New Jersey, where his wife had recovered earlier in the summer. He left the White House for the last time on September 5, traveling in a specially cushioned railway car; a spur line to the Francklyn Cottage, a seaside mansion given over to his use, was built in a night by volunteers. After arriving in Elberon the next day, Garfield was moved from the train car to a bedroom where he could see the ocean as officials and reporters maintained what became (after an initial rally) a death watch. Garfield's personal secretary, Joe Stanley Brown, wrote forty years later, \"to this day I cannot hear the sound of the low slow roll of the Atlantic on the shore, the sound which filled my ears as I walked from my cottage to his bedside, without recalling again that ghastly tragedy.\"\n\nOn September 18, Garfield asked Colonel A.F. Rockwell, a friend, if he would have a place in history. Rockwell assured him he would and told Garfield he had much work still before him. But his response was, \"No, my work is done.\" The following day, Garfield, then suffering also from pneumonia and hypertension, marveled that he could not pick up a glass despite feeling well and went to sleep without discomfort. He awoke that evening around 10:15 p.m. complaining of great pain in his chest to his chief of staff General David Swaim, who was watching him, as he placed his hand over his heart. The president then requested a drink of water from Swaim. After finishing his glass, Garfield said, \"Oh Swaim, this terrible pain—press your hand on it.\" As Swaim put his hand on Garfield's chest, Garfield's hands went up reflexively. Clutching his heart, he exclaimed, \"Oh, Swaim, can't you stop this? Oh, oh, Swaim!\" Those were Garfield's last words. Swaim ordered another attendant to send for Bliss, who found Garfield unconscious. Despite efforts to revive him, Garfield never awoke, and he was pronounced dead at about 10:30 p.m. Learning from a reporter of Garfield's death the following day, Chester A. Arthur took the presidential oath of office administered by New York Supreme Court Justice John R. Brady.\nAccording to some historians and medical experts, Garfield might have survived his wounds had the doctors attending him had at their disposal today's medical research, knowledge, techniques, and equipment. Standard medical practice at the time dictated that priority be given to locating the path of the bullet. Several of his doctors inserted their unsterilized fingers into the wound to probe for the bullet, a common practice in the 1880s. Historians agree that massive infection was a significant factor in Garfield's demise. Biographer Peskin said medical malpractice did not contribute to Garfield's death; the inevitable infection and blood poisoning that would ensue from a deep bullet wound resulted in damage to multiple organs and spinal fragmentation. Rutkow, a professor of surgery at the University of Medicine and Dentistry of New Jersey, has argued that starvation also played a role. Rutkow suggests \"Garfield had such a nonlethal wound. In today's world, he would have gone home in a matter of two or three days.\" The conventional narrative regarding Garfield's post-shooting medical condition was challenged by Theodore Pappas and Shahrzad Joharifard in a 2013 article in The American Journal of Surgery. They argued that Garfield died from a late rupture of a splenic artery pseudoaneurysm, which developed secondary to the path of the bullet adjacent to the splenic artery. They also argued that his sepsis was actually caused by post-traumatic acute acalculous cholecystitis. Based on the autopsy report, the authors speculate that his gallbladder subsequently ruptured, leading to the development of a large bile-containing abscess adjacent to the gallbladder. Pappas and Joharifard say this caused the septic decline in Garfield's condition that was visible starting from July 23, 1881. Pappas and Joharifard also state that they don't believe that Garfield's doctors could have saved him even if they had been aware of his cholecystitis, since the first successful cholecystectomy (surgical removal of the gallbladder) was performed a year after Garfield's death.\nGuiteau was indicted on October 14, 1881, for the murder of the president. During his trial, Guiteau declared that he was not responsible for Garfield's death, admitting to the shooting but not the killing. In his defense, Guiteau wrote: \"General Garfield died from malpractice. According to his own physicians, he was not fatally shot. The doctors who mistreated him ought to bear the odium of his death, and not his assailant. They ought to be indicted for murdering James A. Garfield, and not me.\" After a chaotic trial in which Guiteau often interrupted and argued, and in which his counsel used the insanity defense, the jury found him guilty on January 25, 1882, and he was sentenced to death by hanging. Guiteau may have had neurosyphilis, a disease that causes physiological mental impairment. He was executed on June 30, 1882.\n\nFuneral, memorials and commemorations\nGarfield's funeral train left Long Branch on the same special track that had brought him there, traveling over tracks blanketed with flowers and past houses adorned with flags. His body was transported to the Capitol and then continued on to Cleveland for burial. Shocked by his death, Marine Band leader John Philip Sousa composed the march \"In Memoriam\", which was played when Garfield's body was received in Washington, D.C. More than 70,000 citizens, some waiting over three hours, passed by Garfield's coffin as his body lay in state from September 21 to 23, 1881, at the United States Capitol rotunda; on September 25, in Cleveland, Garfield's casket was paraded down Euclid Avenue from Wilson Avenue to Public Square, with those in attendance including former presidents Grant and Hayes, and Generals William Sherman, Sheridan and Hancock. More than 150,000—a number equal to the city's population—likewise paid their respects, and Sousa's march was again played. Garfield's body was temporarily interred in the Schofield family vault in Cleveland's Lake View Cemetery until his permanent memorial was built.\nMemorials to Garfield were erected across the country. On April 10, 1882, seven months after Garfield's death, the U.S. Post Office Department issued a postage stamp in his honor. In 1884, sculptor Frank Happersberger completed a monument on the grounds of the San Francisco Conservatory of Flowers. In 1887, the James A. Garfield Monument was dedicated in Washington. Another monument, in Philadelphia's Fairmount Park, was erected in 1896. In Victoria, Australia, Cannibal Creek was renamed Garfield in his honor.\n\nOn May 19, 1890, Garfield's body was permanently interred, with great solemnity and fanfare, in a mausoleum in Lake View Cemetery. Attending the dedication ceremonies were former President Hayes, President Benjamin Harrison, and future president William McKinley. Garfield's Treasury Secretary, William Windom, also attended. Harrison said Garfield was always a \"student and instructor\" and that his life works and death would \"continue to be instructive and inspiring incidents in American history\". Three panels on the monument display Garfield as a teacher, Union major general, and orator; another shows him taking the presidential oath, and a fifth shows his body lying in state at the Capitol rotunda in Washington, D.C.\nGarfield's murder by a deranged office-seeker awakened public awareness of the need for civil service reform legislation. Senator George H. Pendleton, a Democrat from Ohio, launched a reform effort that resulted in the Pendleton Act in January 1883. This act reversed the \"spoils system\" where office seekers paid up or gave political service to obtain or keep federally appointed positions. Under the act, appointments were awarded on merit and competitive examination. To ensure the reform was implemented, Congress and Arthur established and funded the Civil Service Commission. The Pendleton Act, however, covered only 10% of federal government workers. For Arthur, previously known for having been a \"veteran spoilsman\", civil service reform became his most noteworthy achievement.\nA marble statue of Garfield by Charles Niehaus was added to the National Statuary Hall Collection in the Capitol in Washington D.C., a gift from the State of Ohio in 1886.\nGarfield is honored with a life-size bronze sculpture inside the Cuyahoga County Soldiers' and Sailors' Monument in Cleveland, Ohio.\nOn March 2, 2019, the National Park Service erected exhibit panels in Washington to mark the site of his assassination.\n\nLegacy and historical view\nFor a few years after his assassination, Garfield's life story was seen as an exemplar of the American success story—that even the poorest boy might someday become President of the United States. Peskin wrote: \"In mourning Garfield, Americans were not only honoring a president; they were paying tribute to a man whose life story embodied their own most cherished aspirations.\" As the rivalry between Stalwarts and Half-Breeds faded from the scene in the late 1880s and after, so too did memories of Garfield. In the 1890s, Americans became disillusioned with politicians, and looked elsewhere for inspiration, focusing on industrialists, labor leaders, scientists, and others as their heroes. Increasingly, Garfield's short time as president was forgotten.\n\nThe 20th century saw no revival for Garfield. Thomas Wolfe deemed the presidents of the Gilded Age, including Garfield, \"lost Americans\" whose \"gravely vacant and bewhiskered faces mixed, melted, swam together\". The politicians of the Gilded Age faded from the public eye, their luster eclipsed by those who had influenced America outside of political office during that time; the robber barons, the inventors, those who had sought social reform, and others who had lived as America rapidly changed. Current events and more recent figures occupied America's attention. According to Ackerman, \"the busy Twentieth Century has made Garfield's era seem remote and irrelevant, its leaders ridiculed for their very obscurity.\"\nGarfield's biographers, and those who have studied his presidency, tend to think well of him, and that his presidency saw a promising start before its untimely end. Historian Justus D. Doenecke, while deeming Garfield a bit of an enigma, chronicles his achievements: \"by winning a victory over the Stalwarts, he enhanced both the power and prestige of his office. As a man, he was intelligent, sensitive, and alert, and his knowledge of how government worked was unmatched.\" Doenecke criticizes Garfield's dismissal of Merritt in Robertson's favor, and wonders if the president was truly in command of the situation even after the latter's confirmation. In 1931, Caldwell wrote: \"If Garfield lives in history, it will be partly on account of the charm of his personality—but also because in life and in death, he struck the first shrewd blows against a dangerous system of boss rule which seemed for a time about to engulf the politics of the nation. Perhaps if he had lived he could have done no more.\" Rutkow writes that \"James Abram Garfield's presidency is reduced to a tantalizing 'what if.'\"\nIn 2002, historian Bernard A. Weisberger said, \"[Garfield] was, to some extent, a perfect moderate. He read widely (and unobtrusively) without its visibly affecting his Christianity, his Republicanism, or his general laissez-faire orthodoxy. He was not so much a scholar in politics as a politic scholar.\" Peskin believes Garfield deserves more credit for his political career than he has received: \"True, his accomplishments were neither bold nor heroic, but his was not an age that called for heroism. His stormy presidency was brief, and in some respects, unfortunate, but he did leave the office stronger than he found it. As a public man he had a hand in almost every issue of national importance for almost two decades, while as a party leader he, along with Blaine, forged the Republican Party into the instrument that would lead the United States into the twentieth century.\"\n\nNotes\nReferences\nWorks cited\nFurther reading\nFuller, Corydon E. (2022) [1887]. Reminiscences of James A. Garfield. Hansebooks. ISBN 978-3-34807-944-0.\nGoodyear, C. W. (2023). President Garfield: From Radical to Unifier. New York, New York: Simon & Schuster.\nGraff Henry F., ed. The Presidents: A Reference History (3rd ed. 2002) online\nHammond, William A.; Ashhurst, Jr., John; Sims, J. Marion; Hodgen, John T. (December 1881). \"The Surgical Treatment of President Garfield\". The North American Review. 133 (301): 578–610. JSTOR 25101018.\nHoudek, John Thomas. \"James A. Garfield and Rutherford B. Hayes: A Study in State and National Politics\" (PhD dissertation, Michigan State University; Proquest Dissertations Publishing, 1970. 7111871).\nMenke, Richard. \"Media in America, 1881: Garfield, Guiteau, Bell, Whitman.\" Critical Inquiry 31.3 (2005): 638–664.\nMillard, Candice (2012). Destiny of the Republic: A Tale of Madness, Medicine and the Murder of a President. New York, New York: Anchor Books. ISBN 978-0-7679-2971-4.\nNorth, Ira Lutts. \"A rhetorical criticism of the speaking of James Abram Garfield, 1876-1880\" (PhD dissertation, Louisiana State University; ProQuest Dissertations Publishing, 1953. DP69446).\nRushford, Jerry Bryant. \"Political Disciple: The Relationship Between James A. Garfield And The Disciples Of Christ\" (PhD dissertation, University of California, Santa Barbara; ProQuest Dissertations Publishing, 1977. 7807029).\nSkidmore, Max J. \"James A. Garfield and Chester A. Arthur.\" in Maligned Presidents: The Late 19th Century (Palgrave Macmillan, New York, 2014) pp. 63–79.\nSutton, Thomas C. \"James A. Garfield.\" in The Presidents and the Constitution (Volume One. New York University Press, 2020) pp. 266–275.\nUhler, Kevin A. \"The demise of patronage: Garfield, the midterm election, and the passage of the Pendleton Civil Service Act\" (PhD. Diss. The Florida State University, 2011) online.\nVermilya, Daniel J. James Garfield and the Civil War: For Ohio and the Union (Arcadia Publishing, 2015).\n\nExternal links\n\nGarfield, James Abram, (1831–1881) Congressional Biography\nJames Garfield: A Resource Guide from the Library of Congress\nJames A. Garfield at the Database of Classical Scholars\n[http://millercenter.org/president/garfield Brief essays on James A. Garfield and his administration from the Miller Center of Public Affairs\n\"Life Portrait of James Garfield\", from C-SPAN's American Presidents: Life Portraits, July 26, 1999\nWorks by or about James A. Garfield at the Internet Archive\nWorks by James A. Garfield at LibriVox (public domain audiobooks) \nNotable alumni of Delta Upsilon fraternity, including Garfield\nJames A. Garfield Personal Manuscripts\nJames A. Garfield Collection at Williams College Chapin Library\nJames A. Garfield Collection at Williams College Archives and Special Collections\nOfficial medical bulletins relating to the health of U.S. President James Garfield from the U.S. National Library of Medicine. Contains medical bulletins issued by attending physicians D. Hayes Agnes, J.K. Barnes, D. W. Bliss, Frank H. Hamilton, Robert Reyburn, and J.J. Woodward between July 6 – September 19, 1881.\n\nBased on all the information, answer the query. \n\nQuery: If my future wife has the same first name as the 15th first lady of the United States' mother and her surname is the same as the second assassinated president's mother's maiden name, what is my future wife's name? \n\n"} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/quantization/bench_fp4_quant.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/quantization/bench_fp4_quant.py new file mode 100644 index 00000000..afc12dd8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/quantization/bench_fp4_quant.py @@ -0,0 +1,136 @@ +import argparse +import itertools + +import torch +import triton +from flashinfer import ( + scaled_fp4_grouped_quantize, + silu_and_mul_scaled_nvfp4_experts_quantize, +) +from sgl_kernel.elementwise import silu_and_mul + +from sglang.srt.layers import deep_gemm_wrapper +from sglang.srt.layers.moe.ep_moe.kernels import silu_and_mul_masked_post_quant_fwd + + +def _test_accuracy_once(E, M, K, input_dtype, device): + x = torch.randn(E, M, K, device=device, dtype=input_dtype) + glb_scales = torch.ones((E,), dtype=torch.float32, device=device) + masks = torch.full((E,), M, dtype=torch.int32, device=device) + out, blk_scales = silu_and_mul_scaled_nvfp4_experts_quantize(x, masks, glb_scales) + out1, blk_scales1 = scaled_fp4_grouped_quantize( + silu_and_mul(x), + masks, + glb_scales, + ) + + torch.testing.assert_close(out, out1) + torch.testing.assert_close(blk_scales, blk_scales1) + print(f"E: {E}, M: {M}, K: {K}, type: {input_dtype} OK") + + +NUM_RANKS = 48 +M_PER_RANKs = [128, 256, 512, 1024] +Ms = [M_PER_RANK * NUM_RANKS for M_PER_RANK in M_PER_RANKs] +Ks = [2048, 4096, 7168] + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["M", "K"], + x_vals=list(itertools.product(Ms, Ks)), + x_log=False, + line_arg="provider", + line_vals=["triton_fp8", "cuda_unfused_fp4", "cuda_fused_fp4"], + line_names=["triton_fp8", "cuda_unfused_fp4", "cuda_fused_fp4"], + styles=[("blue", "-"), ("orange", "-"), ("green", "-")], + ylabel="ms", + plot_name="fp4 quant", + args={}, + ) +) +def benchmark(M, K, provider): + E = 6 + device = "cuda" + x = torch.randn(E, M, K, device=device, dtype=torch.bfloat16) + glb_scales = torch.ones((E,), dtype=torch.float32, device=device) + masks = torch.randint(1, 4096, (E,), dtype=torch.int32, device=device) + fp8_out = torch.empty( + ( + x.shape[0], + x.shape[1], + x.shape[2] // 2, + ), + device=x.device, + dtype=torch.float8_e4m3fn, + ) + scale_block_size = 128 + fp8_scales = torch.empty( + ( + x.shape[0], + x.shape[1], + x.shape[2] // 2 // scale_block_size, + ), + device=x.device, + dtype=torch.float32, + ) + + quantiles = [0.5, 0.2, 0.8] + if provider == "triton_fp8": + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: silu_and_mul_masked_post_quant_fwd( + x, + fp8_out, + fp8_scales, + scale_block_size, + masks, + scale_ue8m0=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0, + ), + quantiles=quantiles, + ) + if provider == "cuda_unfused_fp4": + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: scaled_fp4_grouped_quantize( + silu_and_mul(x), + masks, + glb_scales, + ), + quantiles=quantiles, + ) + if provider == "cuda_fused_fp4": + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: silu_and_mul_scaled_nvfp4_experts_quantize( + x, + masks, + glb_scales, + ), + quantiles=quantiles, + ) + + return ms, min_ms, max_ms + + +def test_accuracy(): + E = 6 + N_RANKS = 48 + Ms = [128, 256, 512, 1024] + Ks = [2048, 4096, 7168] + input_dtype = torch.bfloat16 + for M in Ms: + for K in Ks: + _test_accuracy_once(E, N_RANKS * M, K, input_dtype, "cuda") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", + type=str, + default="./bench_fp4_quant_res", + help="Path to save fp4 quant benchmark results", + ) + args = parser.parse_args() + + test_accuracy() + + benchmark.run(print_data=True, show_plots=True, save_path=args.save_path) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/quantization/bench_int8_quant.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/quantization/bench_int8_quant.py new file mode 100644 index 00000000..94b79569 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/quantization/bench_int8_quant.py @@ -0,0 +1,94 @@ +import argparse + +import torch +import triton +from vllm._custom_ops import scaled_int8_quant as vllm_scaled_int8_quant + +from sglang.srt.layers.quantization.int8_kernel import per_token_quant_int8 + + +@torch.compile(backend="inductor") +def torch_int8_quant(x): + int8_max = torch.iinfo(torch.int8).max + + abs_max = x.abs().max(dim=-1, keepdim=True).values + scales = abs_max.to(torch.float32) / float(int8_max) + + q_x = (x / scales).round().to(torch.int8) + + return q_x, scales + + +def _test_accuracy_once(M, K, input_dtype, device): + x = torch.randn(M, K, dtype=input_dtype, device=device) * 5000 + out, scales, _ = vllm_scaled_int8_quant(x, symmetric=True) + out1, scales1 = per_token_quant_int8(x) + out2, scales2 = torch_int8_quant(x) + torch.testing.assert_close(out, out2, atol=1, rtol=0) + torch.testing.assert_close(out, out1, atol=1, rtol=0) + torch.testing.assert_close(scales, scales2) + torch.testing.assert_close(scales1, scales2) + print(f"M: {M}, K: {K}, type: {input_dtype} OK") + + +def test_accuracy(): + Ms = [1, 13, 128, 1024, 2048, 4096] + Ks = [512, 1024, 2048, 8192] + input_dtypes = [torch.float16, torch.bfloat16] + for M in Ms: + for K in Ks: + for input_dtype in input_dtypes: + _test_accuracy_once(M, K, input_dtype, "cuda") + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=[1, 16, 32, 64, 128, 256, 512, 1024, 2048], + x_log=False, + line_arg="provider", + line_vals=["vllm op", "triton", "torch.compile"], + line_names=["vllm op", "triton", "torch.compile"], + styles=[("blue", "-"), ("orange", "-"), ("red", "-")], + ylabel="ms", + plot_name="int8 per token quant", + args={}, + ) +) +def benchmark(batch_size, provider): + M, K = batch_size, 16384 + x = torch.randn(M, K, dtype=torch.float16, device="cuda") * 1000 + + quantiles = [0.5, 0.2, 0.8] + if provider == "vllm op": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: vllm_scaled_int8_quant(x, symmetric=True), + quantiles=quantiles, + ) + if provider == "triton": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: per_token_quant_int8(x), + quantiles=quantiles, + ) + if provider == "torch.compile": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: torch_int8_quant(x), + quantiles=quantiles, + ) + + return ms, min_ms, max_ms + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", + type=str, + default="./bench_int8_quant_res", + help="Path to save int8 quant benchmark results", + ) + args = parser.parse_args() + + test_accuracy() + + benchmark.run(print_data=True, show_plots=True, save_path=args.save_path) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/quantization/tuning_block_wise_kernel.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/quantization/tuning_block_wise_kernel.py new file mode 100644 index 00000000..1b51e54b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/quantization/tuning_block_wise_kernel.py @@ -0,0 +1,474 @@ +# Copyright 2025 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import argparse +import json +import multiprocessing as mp +import os +import time +from datetime import datetime +from typing import Any, Dict, List + +import torch +import triton +from tqdm import tqdm + +mp.set_start_method("spawn", force=True) + +from sglang.srt.layers.quantization.fp8_kernel import ( + _w8a8_block_fp8_matmul, + _w8a8_block_fp8_matmul_unrolledx4, +) +from sglang.srt.layers.quantization.int8_kernel import _w8a8_block_int8_matmul +from sglang.srt.utils import get_device_core_count, get_device_name, is_hip + +_is_hip = is_hip() + +DTYPE_MAP = { + "float32": torch.float32, + "float16": torch.float16, + "half": torch.half, + "bfloat16": torch.bfloat16, +} + + +def w8a8_block_matmul( + A: torch.Tensor, + B: torch.Tensor, + As: torch.Tensor, + Bs: torch.Tensor, + block_size: List[int], + config: Dict[str, Any], + output_dtype: torch.dtype = torch.float16, +) -> torch.Tensor: + """This function performs matrix multiplication with block-wise quantization. + + It takes two input tensors `A` and `B` with scales `As` and `Bs`. + The output is returned in the specified `output_dtype`. + + Args: + A: The input tensor, e.g., activation. + B: The input tensor, e.g., weight. + As: The per-token-group quantization scale for `A`. + Bs: The per-block quantization scale for `B`. + block_size: The block size for per-block quantization. It should be 2-dim, e.g., [128, 128]. + output_dytpe: The dtype of the returned tensor. + + Returns: + torch.Tensor: The result of matmul. + """ + assert len(block_size) == 2 + block_n, block_k = block_size[0], block_size[1] + + assert A.shape[-1] == B.shape[-1] + assert A.shape[:-1] == As.shape[:-1] and A.is_contiguous() + assert triton.cdiv(A.shape[-1], block_k) == As.shape[-1] + M = A.numel() // A.shape[-1] + + assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2 + N, K = B.shape + assert triton.cdiv(N, block_n) == Bs.shape[0] + assert triton.cdiv(K, block_k) == Bs.shape[1] + + C_shape = A.shape[:-1] + (N,) + C = A.new_empty(C_shape, dtype=output_dtype) + + def grid(META): + return ( + triton.cdiv(M, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]), + ) + + # Use manually unrolledx4 kernel on AMD GPU when the grid size is small. + # Empirical testing shows the sweet spot lies when it's less than the # of + # compute units available on the device. + num_workgroups = triton.cdiv(M, config["BLOCK_SIZE_M"]) * triton.cdiv( + N, config["BLOCK_SIZE_N"] + ) + + if A.dtype == torch.float8_e4m3fnuz or A.dtype == torch.float8_e4m3fn: + kernel = ( + _w8a8_block_fp8_matmul_unrolledx4 + if (_is_hip == True and num_workgroups <= get_device_core_count()) + else _w8a8_block_fp8_matmul + ) + else: + kernel = _w8a8_block_int8_matmul + + kernel[grid]( + A, + B, + C, + As, + Bs, + M, + N, + K, + block_n, + block_k, + A.stride(-2), + A.stride(-1), + B.stride(1), + B.stride(0), + C.stride(-2), + C.stride(-1), + As.stride(-2), + As.stride(-1), + Bs.stride(1), + Bs.stride(0), + **config, + ) + + return C + + +def get_rocm_configs_compute_bound(): + configs = [] + waves_per_eu_range = 0 + for num_stages in [2]: + for block_m in [32, 64, 128, 256]: + for block_k in [32, 64, 128, 256]: + for block_n in [16, 32, 64, 128, 256]: + for num_warps in [4, 8]: + for group_size in [1, 4, 8, 16, 32]: + configs.append( + { + "BLOCK_SIZE_M": block_m, + "BLOCK_SIZE_N": block_n, + "BLOCK_SIZE_K": block_k, + "GROUP_SIZE_M": group_size, + "num_warps": num_warps, + "num_stages": num_stages, + "waves_per_eu": waves_per_eu_range, + } + ) + return configs + + +def get_configs_compute_bound(): + configs = [] + if _is_hip: + configs = get_rocm_configs_compute_bound() + else: + for num_stages in [2, 3, 4, 5]: + for block_m in [16, 32, 64, 128, 256]: + for block_k in [64, 128]: + for block_n in [32, 64, 128, 256]: + for num_warps in [4, 8]: + for group_size in [1, 16, 32, 64]: + configs.append( + { + "BLOCK_SIZE_M": block_m, + "BLOCK_SIZE_N": block_n, + "BLOCK_SIZE_K": block_k, + "GROUP_SIZE_M": group_size, + "num_warps": num_warps, + "num_stages": num_stages, + } + ) + return configs + + +def get_weight_shapes(tp_size): + # NOTE(HandH1998): The weight shapes only works for DeepSeek-V3. Modify them, if you tune for another different model. + # cannot TP + total = [ + (512 + 64, 7168), + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (7168, 16384), + (7168, 18432), + ] + # N can TP + n_tp = [ + (18432 * 2, 7168), + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (24576, 1536), + (4096, 7168), + ] + # K can TP + k_tp = [(7168, 18432), (7168, 16384), (7168, 2048)] + + weight_shapes = [] + for t in total: + weight_shapes.append(t) + for n_t in n_tp: + new_t = (n_t[0] // tp_size, n_t[1]) + weight_shapes.append(new_t) + for k_t in k_tp: + new_t = (k_t[0], k_t[1] // tp_size) + weight_shapes.append(new_t) + return weight_shapes + + +def benchmark_config( + A, B, As, Bs, block_size, config, out_dtype=torch.float16, num_iters=10 +): + def run(): + w8a8_block_matmul(A, B, As, Bs, block_size, config, out_dtype) + + torch.cuda.synchronize() + # JIT complication & warmup + for _ in range(5): + run() + torch.cuda.synchronize() + + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + + latencies: List[float] = [] + for i in range(num_iters): + torch.cuda.synchronize() + start_event.record() + run() + end_event.record() + end_event.synchronize() + latencies.append(start_event.elapsed_time(end_event)) + avg = sum(latencies) / (num_iters * 10) * 1000 # us + return avg + + +def tune(M, N, K, block_size, out_dtype, search_space, input_type): + factor_for_scale = 1e-2 + + if input_type == "fp8": + fp8_info = torch.finfo( + torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn + ) + fp8_max, fp8_min = fp8_info.max, fp8_info.min + + A_fp32 = ( + (torch.rand(M, K, dtype=torch.float32, device="cuda") - 0.5) * 2 * fp8_max + ) + A = A_fp32.clamp(min=fp8_min, max=fp8_max).to( + torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn + ) + + B_fp32 = ( + (torch.rand(N, K, dtype=torch.float32, device="cuda") - 0.5) * 2 * fp8_max + ) + B = B_fp32.clamp(min=fp8_min, max=fp8_max).to( + torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn + ) + else: + int8_info = torch.iinfo(torch.int8) + int8_max, int8_min = int8_info.max, int8_info.min + + A_fp32 = ( + (torch.rand(M, K, dtype=torch.float32, device="cuda") - 0.5) * 2 * int8_max + ) + A = A_fp32.clamp(min=int8_min, max=int8_max).to(torch.int8) + + B_fp32 = ( + (torch.rand(N, K, dtype=torch.float32, device="cuda") - 0.5) * 2 * int8_max + ) + B = B_fp32.clamp(min=int8_min, max=int8_max).to(torch.int8) + + block_n, block_k = block_size[0], block_size[1] + n_tiles = (N + block_n - 1) // block_n + k_tiles = (K + block_k - 1) // block_k + + As = torch.rand(M, k_tiles, dtype=torch.float32, device="cuda") * factor_for_scale + Bs = ( + torch.rand(n_tiles, k_tiles, dtype=torch.float32, device="cuda") + * factor_for_scale + ) + + best_config = None + best_time = float("inf") + for config in tqdm(search_space): + try: + kernel_time = benchmark_config( + A, + B, + As, + Bs, + block_size, + config, + out_dtype, + num_iters=10, + ) + except triton.runtime.autotuner.OutOfResources: + # Some configurations may be invalid and fail to compile. + continue + + if kernel_time < best_time: + best_time = kernel_time + best_config = config + now = datetime.now() + print(f"{now.ctime()}] Completed tuning for batch_size={M}") + assert best_config is not None + return best_config + + +def save_configs( + N, + K, + block_n, + block_k, + configs, + save_path, + input_type="fp8", +) -> None: + os.makedirs(save_path, exist_ok=True) + device_name = get_device_name().replace(" ", "_") + json_file_name = f"N={N},K={K},device_name={device_name},dtype={input_type}_w8a8,block_shape=[{block_n}, {block_k}].json" + + config_file_path = os.path.join(save_path, json_file_name) + print(f"Writing best config to {config_file_path}...") + + with open(config_file_path, "w") as f: + json.dump(configs, f, indent=4) + f.write("\n") + + +def get_available_gpu_count(): + """Get the number of available GPUs.""" + return torch.cuda.device_count() + + +def tune_on_gpu(args_dict): + """Run tuning on a specific GPU.""" + gpu_id = args_dict["gpu_id"] + batch_sizes = args_dict["batch_sizes"] + weight_shapes = args_dict["weight_shapes"] + args = args_dict["args"] + + torch.cuda.set_device(gpu_id) + print(f"Starting tuning on GPU {gpu_id} with batch sizes {batch_sizes}") + + block_n = args.block_n + block_k = args.block_k + out_dtype = DTYPE_MAP[args.out_dtype] + save_path = args.save_path + input_type = args.input_type + + search_space = get_configs_compute_bound() + search_space = [ + config for config in search_space if block_k % config["BLOCK_SIZE_K"] == 0 + ] + + start = time.perf_counter() + results = {} + for shape in tqdm(weight_shapes, desc=f"GPU {gpu_id} - Shapes"): + N, K = shape[0], shape[1] + print(f"[GPU {gpu_id}] Tune for weight shape of `N: {N}, K: {K}`") + benchmark_results = [ + tune( + batch_size, + N, + K, + [block_n, block_k], + out_dtype, + search_space, + input_type, + ) + for batch_size in tqdm(batch_sizes, desc=f"GPU {gpu_id} - Batch sizes") + ] + best_configs = {M: config for M, config in zip(batch_sizes, benchmark_results)} + save_configs(N, K, block_n, block_k, best_configs, save_path, input_type) + + end = time.perf_counter() + print(f"Tuning on GPU {gpu_id} took {end - start:.2f} seconds") + + +def distribute_batch_sizes(batch_sizes, num_gpus): + """Distribute batch sizes across available GPUs.""" + batches_per_gpu = [] + for i in range(num_gpus): + start_idx = i * len(batch_sizes) // num_gpus + end_idx = (i + 1) * len(batch_sizes) // num_gpus + batches_per_gpu.append(batch_sizes[start_idx:end_idx]) + return batches_per_gpu + + +def main(args): + print(args) + + num_gpus = get_available_gpu_count() + if num_gpus == 0: + raise RuntimeError("No GPU available for tuning") + print(f"Found {num_gpus} GPUs for parallel tuning") + + torch.cuda.init() + + if args.batch_size is None: + batch_sizes = [ + 1, + 2, + 4, + 8, + 16, + 24, + 32, + 48, + 64, + 96, + 128, + 256, + 512, + 1024, + 1536, + 2048, + 3072, + 4096, + ] + else: + batch_sizes = [args.batch_size] + num_gpus = 1 # If only one batch size, use only one GPU + + weight_shapes = get_weight_shapes(args.tp_size) + + batches_per_gpu = distribute_batch_sizes(batch_sizes, num_gpus) + + process_args = [] + for gpu_id in range(num_gpus): + process_args.append( + { + "gpu_id": gpu_id, + "batch_sizes": batches_per_gpu[gpu_id], + "weight_shapes": weight_shapes, # Each GPU processes all weight shapes + "args": args, + } + ) + + ctx = mp.get_context("spawn") + with ctx.Pool(num_gpus) as pool: + pool.map(tune_on_gpu, process_args) + + print("Multi-GPU tuning completed") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("--tp-size", "-tp", type=int, default=8) + parser.add_argument( + "--input-type", type=str, choices=["fp8", "int8"], default="fp8" + ) + parser.add_argument( + "--out-dtype", + type=str, + choices=["float32", "float16", "bfloat16", "half"], + default="float16", + ) + parser.add_argument("--block-n", type=int, default=128) + parser.add_argument("--block-k", type=int, default=128) + parser.add_argument("--batch-size", type=int, required=False) + parser.add_argument( + "--save-path", type=str, default="python/sglang/srt/layers/quantization/configs" + ) + args = parser.parse_args() + + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/scheduler_batch/benchmark_get_last_loc_triton.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/scheduler_batch/benchmark_get_last_loc_triton.py new file mode 100644 index 00000000..3e17205e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/scheduler_batch/benchmark_get_last_loc_triton.py @@ -0,0 +1,169 @@ +import os + +import torch +import triton +import triton.language as tl + + +@torch.compile(dynamic=True) +def get_last_loc_torch( + req_to_token: torch.Tensor, + req_pool_indices_tensor: torch.Tensor, + prefix_lens_tensor: torch.Tensor, +) -> torch.Tensor: + return torch.where( + prefix_lens_tensor > 0, + req_to_token[req_pool_indices_tensor, prefix_lens_tensor - 1], + torch.full_like(prefix_lens_tensor, -1), + ) + + +@triton.jit +def get_last_loc_kernel( + req_to_token, + req_pool_indices_tensor, + prefix_lens_tensor, + result, + num_tokens, + req_to_token_stride, + BLOCK_SIZE: tl.constexpr, +): + pid = tl.program_id(0) + offset = tl.arange(0, BLOCK_SIZE) + pid * BLOCK_SIZE + mask = offset < num_tokens + + prefix_lens = tl.load(prefix_lens_tensor + offset, mask=mask, other=0) + req_pool_indices = tl.load(req_pool_indices_tensor + offset, mask=mask, other=0) + + token_mask = prefix_lens > 0 + token_index = req_pool_indices * req_to_token_stride + (prefix_lens - 1) + tokens = tl.load(req_to_token + token_index, mask=token_mask, other=-1) + + tl.store(result + offset, tokens, mask=mask) + + +def get_last_loc_triton( + req_to_token: torch.Tensor, + req_pool_indices_tensor: torch.Tensor, + prefix_lens_tensor: torch.Tensor, +) -> torch.Tensor: + BLOCK_SIZE = 256 + num_tokens = prefix_lens_tensor.shape[0] + result = torch.empty_like(prefix_lens_tensor) + grid = (triton.cdiv(num_tokens, BLOCK_SIZE),) + + get_last_loc_kernel[grid]( + req_to_token, + req_pool_indices_tensor, + prefix_lens_tensor, + result, + num_tokens, + req_to_token.stride(0), + BLOCK_SIZE, + ) + return result + + +def test_get_last_loc(): + max_batch = 4097 + max_context_len = 6148 + batch_size = 20 + + # Initialize input tensors + req_to_token = torch.zeros( + (max_batch, max_context_len), dtype=torch.int32, device="cuda" + ) + req_pool_indices = torch.arange(batch_size, dtype=torch.int64, device="cuda") + pre_lens = torch.randint( + -max_context_len // 2, + max_context_len, + (batch_size,), + dtype=torch.int64, + device="cuda", + ) + + last_loc_res = get_last_loc_triton(req_to_token, req_pool_indices, pre_lens) + last_loc_ref = get_last_loc_torch(req_to_token, req_pool_indices, pre_lens) + + # Compare results + torch.testing.assert_close(last_loc_res, last_loc_ref) + + +def get_benchmark(): + batch_sizes = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] + + @triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=batch_sizes, + line_arg="provider", + line_vals=["reference", "triton"], + line_names=["PyTorch", "Triton"], + styles=[("blue", "-"), ("green", "-")], + ylabel="us", + plot_name="get-last-loc-performance", + args={}, + ) + ) + def benchmark(batch_size, provider): + max_batch = 2048 + max_context_len = 16384 + + req_to_token = torch.zeros( + (max_batch, max_context_len), dtype=torch.int32, device="cuda" + ) + req_pool_indices = torch.arange(batch_size, dtype=torch.int64, device="cuda") + pre_lens = torch.randint( + -max_context_len // 2, + max_context_len, + (batch_size,), + dtype=torch.int64, + device="cuda", + ) + + quantiles = [0.5, 0.2, 0.8] + + if provider == "reference": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: get_last_loc_torch(req_to_token, req_pool_indices, pre_lens), + quantiles=quantiles, + ) + elif provider == "triton": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: get_last_loc_triton(req_to_token, req_pool_indices, pre_lens), + quantiles=quantiles, + ) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + return benchmark + + +def run_benchmark(save_path: str = "./configs/benchmark_ops/get_last_loc/"): + """Run benchmark and save results""" + + # Ensure save path exists + os.makedirs(save_path, exist_ok=True) + + # Run correctness test + test_get_last_loc() + print("Correctness test passed!") + + # Run performance test + benchmark = get_benchmark() + benchmark.run(print_data=True, save_path=save_path) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", + type=str, + default="./configs/benchmark_ops/get_last_loc/", + help="Path to save benchmark results", + ) + args = parser.parse_args() + + run_benchmark(args.save_path) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/scheduler_batch/benchmark_write_req_to_token_pool_triton.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/scheduler_batch/benchmark_write_req_to_token_pool_triton.py new file mode 100644 index 00000000..1ce43c8b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/scheduler_batch/benchmark_write_req_to_token_pool_triton.py @@ -0,0 +1,342 @@ +import itertools +import os + +import torch +import triton +import triton.language as tl + + +@triton.jit +def write_req_to_token_pool_triton( + req_to_token_ptr, # [max_batch, max_context_len] + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + req_to_token_ptr_stride: tl.constexpr, +): + BLOCK_SIZE: tl.constexpr = 512 + pid = tl.program_id(0) + + req_pool_index = tl.load(req_pool_indices + pid) + pre_len = tl.load(pre_lens + pid) + seq_len = tl.load(seq_lens + pid) + + # TODO: optimize this? + cumsum_start = 0 + for i in range(pid): + cumsum_start += tl.load(extend_lens + i) + + num_loop = tl.cdiv(seq_len - pre_len, BLOCK_SIZE) + for i in range(num_loop): + offset = tl.arange(0, BLOCK_SIZE) + i * BLOCK_SIZE + mask = offset < (seq_len - pre_len) + value = tl.load(out_cache_loc + cumsum_start + offset, mask=mask) + tl.store( + req_to_token_ptr + + req_pool_index * req_to_token_ptr_stride + + offset + + pre_len, + value, + mask=mask, + ) + + +@triton.jit +def write_req_to_token_pool_triton_optimize( + req_to_token_ptr, # [max_batch, max_context_len] + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + req_to_token_ptr_stride: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + pid_batch = tl.program_id(0) + pid_token = tl.program_id(1) + + req_pool_index = tl.load(req_pool_indices + pid_batch) + pre_len = tl.load(pre_lens + pid_batch) + seq_len = tl.load(seq_lens + pid_batch) + extend_len = seq_len - pre_len + + cumsum_start = 0 + for i in range(pid_batch): + cumsum_start += tl.load(extend_lens + i) + + token_start = pid_token * BLOCK_SIZE + + offset = tl.arange(0, BLOCK_SIZE) + actual_offset = token_start + offset + mask = actual_offset < extend_len + + src_ptr = out_cache_loc + cumsum_start + actual_offset + src_ptr = tl.max_contiguous(tl.multiple_of(src_ptr, BLOCK_SIZE), BLOCK_SIZE) + value = tl.load(src_ptr, mask=mask) + dst_ptr = ( + req_to_token_ptr + + req_pool_index * req_to_token_ptr_stride + + actual_offset + + pre_len + ) + dst_ptr = tl.max_contiguous(tl.multiple_of(dst_ptr, BLOCK_SIZE), BLOCK_SIZE) + + tl.store(dst_ptr, value, mask=mask) + + +def write_req_to_token_pool_reference( + req_to_token: torch.Tensor, + req_pool_indices: torch.Tensor, + pre_lens: torch.Tensor, + seq_lens: torch.Tensor, + extend_lens: torch.Tensor, + out_cache_loc: torch.Tensor, +) -> None: + """Reference implementation using PyTorch""" + for i in range(len(req_pool_indices)): + req_pool_idx = req_pool_indices[i].item() + pre_len = pre_lens[i].item() + seq_len = seq_lens[i].item() + extend_len = extend_lens[i].item() + + cumsum_start = sum(extend_lens[:i].tolist()) + + # Copy values from out_cache_loc to req_to_token + req_to_token[req_pool_idx, pre_len:seq_len] = out_cache_loc[ + cumsum_start : cumsum_start + extend_len + ] + + +def test_write_req_to_token_pool(): + max_batch = 4097 + max_context_len = 6148 + batch_size = 1 + extend_len = 14 + + # Initialize input tensors + req_to_token = torch.zeros( + (max_batch, max_context_len), dtype=torch.int32, device="cuda" + ) + req_pool_indices = torch.tensor([42], dtype=torch.int32, device="cuda") + pre_lens = torch.tensor([8], dtype=torch.int32, device="cuda") + seq_lens = torch.tensor([22], dtype=torch.int32, device="cuda") + extend_lens = torch.tensor([extend_len], dtype=torch.int32, device="cuda") + out_cache_loc = torch.arange(extend_len, dtype=torch.int32, device="cuda") + + # Create copies for reference implementation + req_to_token_ref = req_to_token.clone() + req_to_token_opt = req_to_token.clone() + + # Run original triton kernel + write_req_to_token_pool_triton[(batch_size,)]( + req_to_token, + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + max_context_len, + ) + + # Run optimized triton kernel + def grid(batch_size, extend_len): + num_token_blocks = triton.cdiv(extend_len, 512) + return (batch_size, num_token_blocks) + + write_req_to_token_pool_triton_optimize[grid(batch_size, extend_len)]( + req_to_token_opt, + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + max_context_len, + BLOCK_SIZE=512, + ) + + # Run reference implementation + write_req_to_token_pool_reference( + req_to_token_ref, + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + ) + + # Compare results + torch.testing.assert_close(req_to_token, req_to_token_ref) + torch.testing.assert_close(req_to_token_opt, req_to_token_ref) + + # Test case 2: batch size > 1 + batch_size = 3 + extend_lens_list = [14, 20, 30] + total_extend_len = sum(extend_lens_list) + + req_to_token = torch.zeros( + (max_batch, max_context_len), dtype=torch.int32, device="cuda" + ) + req_pool_indices = torch.tensor([42, 100, 200], dtype=torch.int32, device="cuda") + pre_lens = torch.tensor([8, 10, 15], dtype=torch.int32, device="cuda") + seq_lens = torch.tensor([22, 30, 45], dtype=torch.int32, device="cuda") + extend_lens = torch.tensor(extend_lens_list, dtype=torch.int32, device="cuda") + out_cache_loc = torch.arange(total_extend_len, dtype=torch.int32, device="cuda") + + req_to_token_ref = req_to_token.clone() + req_to_token_opt = req_to_token.clone() + + # Run original triton kernel + write_req_to_token_pool_triton[(batch_size,)]( + req_to_token, + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + max_context_len, + ) + + # Run optimized triton kernel + max_extend_len = max(extend_lens_list) + write_req_to_token_pool_triton_optimize[grid(batch_size, max_extend_len)]( + req_to_token_opt, + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + max_context_len, + BLOCK_SIZE=512, + ) + + # Run reference implementation + write_req_to_token_pool_reference( + req_to_token_ref, + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + ) + + # Compare results + torch.testing.assert_close(req_to_token, req_to_token_ref) + torch.testing.assert_close(req_to_token_opt, req_to_token_ref) + + +def get_benchmark(): + batch_sizes = [1, 2, 4, 8, 16, 32, 64, 128] + extend_lens = [32, 64, 128, 256, 512, 1024, 2048, 4096, 8192] + configs = list(itertools.product(batch_sizes, extend_lens)) + + @triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size", "extend_len"], + x_vals=configs, + line_arg="provider", + line_vals=["reference", "triton", "triton_optimize"], + line_names=["PyTorch", "Triton", "Triton Optimized"], + styles=[("blue", "-"), ("green", "-"), ("red", "-")], + ylabel="us", + plot_name="write-req-to-token-pool-performance", + args={}, + ) + ) + def benchmark(batch_size, extend_len, provider): + max_batch = 256 + max_context_len = 16384 + + extend_lens_list = [extend_len] * batch_size + total_extend_len = sum(extend_lens_list) + + req_to_token = torch.zeros( + (max_batch, max_context_len), dtype=torch.int32, device="cuda" + ) + req_pool_indices = torch.arange(batch_size, dtype=torch.int32, device="cuda") + pre_lens = torch.ones(batch_size, dtype=torch.int32, device="cuda") * 8 + seq_lens = pre_lens + extend_len + extend_lens = torch.tensor(extend_lens_list, dtype=torch.int32, device="cuda") + out_cache_loc = torch.arange(total_extend_len, dtype=torch.int32, device="cuda") + + quantiles = [0.5, 0.2, 0.8] + + if provider == "reference": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: write_req_to_token_pool_reference( + req_to_token.clone(), + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + ), + quantiles=quantiles, + ) + elif provider == "triton": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: write_req_to_token_pool_triton[(batch_size,)]( + req_to_token.clone(), + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + max_context_len, + ), + quantiles=quantiles, + ) + else: + + def run_optimized(): + block_size = 128 if extend_len <= 1024 else 512 + grid_config = (batch_size, triton.cdiv(extend_len, block_size)) + write_req_to_token_pool_triton_optimize[grid_config]( + req_to_token.clone(), + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + max_context_len, + BLOCK_SIZE=block_size, + ) + + ms, min_ms, max_ms = triton.testing.do_bench( + run_optimized, quantiles=quantiles + ) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + return benchmark + + +def run_benchmark(save_path: str = "./configs/benchmark_ops/write_req_to_token_pool/"): + """Run benchmark and save results""" + + # Ensure save path exists + os.makedirs(save_path, exist_ok=True) + + # Run correctness test + test_write_req_to_token_pool() + print("Correctness test passed!") + + # Run performance test + benchmark = get_benchmark() + benchmark.run(print_data=True, save_path=save_path) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", + type=str, + default="./configs/benchmark_ops/write_req_to_token_pool/", + help="Path to save benchmark results", + ) + args = parser.parse_args() + + run_benchmark(args.save_path) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/sliding_window_attention_triton/bench_triton_swa_kernel.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/sliding_window_attention_triton/bench_triton_swa_kernel.py new file mode 100644 index 00000000..98144d47 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/kernels/sliding_window_attention_triton/bench_triton_swa_kernel.py @@ -0,0 +1,283 @@ +import itertools + +import torch +import torch.nn.functional as F +import triton.testing as tt + +from sglang.srt.layers.attention.triton_ops.extend_attention import extend_attention_fwd + + +def extend_attention_fwd_torch( + q: torch.Tensor, # [extend_tokens, H_Q, D] + k: torch.Tensor, # [extend_tokens, H_KV, D] + v: torch.Tensor, # [extend_tokens, H_KV, D] + o: torch.Tensor, # [extend_tokens, H_Q, D] + k_cache: torch.Tensor, # [total_tokens, H_KV, D] + v_cache: torch.Tensor, # [total_tokens, H_KV, D] + qo_indptr: torch.Tensor, # [B+1] + kv_indptr: torch.Tensor, # [B+1] + kv_indices: torch.Tensor, # [prefix_tokens] + sliding_window_size: int, +): + B = qo_indptr.size(0) - 1 + _, H_Q, D = q.shape + _, H_KV, _ = k.shape + + group_size = H_Q // H_KV + scale = 1.0 / D**0.5 + + for i in range(B): + q_start = int(qo_indptr[i].item()) + q_end = int(qo_indptr[i + 1].item()) + kv_start = int(kv_indptr[i].item()) + kv_end = int(kv_indptr[i + 1].item()) + + prefix_indices = kv_indices[kv_start:kv_end] + k_prefix = k_cache[prefix_indices] # [prefix_len, H_KV, D] + v_prefix = v_cache[prefix_indices] # [prefix_len, H_KV, D] + + k_extend = k[q_start:q_end] # [extend_len, H_KV, D] + v_extend = v[q_start:q_end] # [extend_len, H_KV, D] + q_extend = q[q_start:q_end] # [extend_len, H_Q, D] + + k_full = torch.cat([k_prefix, k_extend], dim=0) # [total_len, H_KV, D] + v_full = torch.cat([v_prefix, v_extend], dim=0) # [total_len, H_KV, D] + + if group_size != 1: + k_full_hq = k_full.repeat_interleave( + group_size, dim=1 + ) # [total_len, H_Q, D] + v_full_hq = v_full.repeat_interleave( + group_size, dim=1 + ) # [total_len, H_Q, D] + else: + k_full_hq = k_full + v_full_hq = v_full + + prefix_len = k_prefix.size(0) + extend_len = k_extend.size(0) + total_len = prefix_len + extend_len + + # causal + pos_keys = torch.arange(total_len, device=q.device) + t = prefix_len + torch.arange(extend_len, device=q.device) # [extend_len] + causal_mask = pos_keys.unsqueeze(0) <= t.unsqueeze(1) + + # sliding window + if sliding_window_size is not None and sliding_window_size > 0: + start = (t - (sliding_window_size)).clamp_min(0) # [extend_len] + else: + start = torch.zeros_like(t) + window_mask = pos_keys.unsqueeze(0) >= start.unsqueeze(1) + + final_mask = causal_mask & window_mask + + attn_scores = ( + torch.einsum("qhd,khd->qhk", q_extend, k_full_hq) * scale + ) # [extend_len, H_Q, total_len] + attn_scores = attn_scores.masked_fill(~final_mask.unsqueeze(1), float("-inf")) + + attn_weights = F.softmax(attn_scores, dim=-1) + o[q_start:q_end] = torch.einsum("qhk,khd->qhd", attn_weights, v_full_hq) + + +def _build_batch( + B, N_CTX, H_Q, H_KV, D, WINDOW_SIZE, dtype=torch.bfloat16, device="cuda" +): + b_seq_len_prefix = torch.randint( + 1, max(2, N_CTX // 2), (B,), dtype=torch.int32, device=device + ) + b_seq_len_extend = torch.randint( + 1, max(2, N_CTX // 2), (B,), dtype=torch.int32, device=device + ) + b_seq_len = b_seq_len_prefix + b_seq_len_extend + + b_start_loc = torch.zeros((B,), dtype=torch.int32, device=device) + b_start_loc[1:] = torch.cumsum(b_seq_len[:-1], 0) + b_start_loc_extend = torch.zeros((B,), dtype=torch.int32, device=device) + b_start_loc_extend[1:] = torch.cumsum(b_seq_len_extend[:-1], 0) + + kv_indptr = torch.zeros((B + 1,), dtype=torch.int32, device=device) + kv_indptr[1 : B + 1] = torch.cumsum(b_seq_len_prefix[:B], dim=0) + + kv_indices = torch.zeros( + (int(b_seq_len_prefix.sum().item()),), dtype=torch.int32, device=device + ) + for i in range(B): + s = kv_indptr[i].item() + e = kv_indptr[i + 1].item() + kv_indices[s:e] = torch.arange( + b_start_loc[i], + b_start_loc[i] + b_seq_len_prefix[i], + dtype=torch.int32, + device=device, + ) + + total_token_num = int(torch.sum(b_seq_len).item()) + extend_token_num = int(torch.sum(b_seq_len_extend).item()) + + k_buffer = torch.empty( + (total_token_num, H_KV, D), dtype=dtype, device=device + ).normal_(mean=0.1, std=0.2) + v_buffer = torch.empty( + (total_token_num, H_KV, D), dtype=dtype, device=device + ).normal_(mean=0.1, std=0.2) + + k_extend = torch.empty((extend_token_num, H_KV, D), dtype=dtype, device=device) + v_extend = torch.empty((extend_token_num, H_KV, D), dtype=dtype, device=device) + q_extend = torch.empty((extend_token_num, H_Q, D), dtype=dtype, device=device) + + for i in range(B): + extend_start_in_buffer = b_start_loc[i] + b_seq_len_prefix[i] + extend_end_in_buffer = b_start_loc[i] + b_seq_len[i] + extend_start = b_start_loc_extend[i] + extend_end = b_start_loc_extend[i] + b_seq_len_extend[i] + + k_extend[extend_start:extend_end] = k_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + v_extend[extend_start:extend_end] = v_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + q_extend[extend_start:extend_end] = torch.empty( + (int(b_seq_len_extend[i].item()), H_Q, D), dtype=dtype, device=device + ).normal_(mean=0.1, std=0.2) + + o_extend_triton = torch.empty( + (extend_token_num, H_Q, D), dtype=dtype, device=device + ) + o_extend_torch = torch.empty((extend_token_num, H_Q, D), dtype=dtype, device=device) + + b_seq_len_extend = b_seq_len - b_seq_len_prefix + max_len_extend = int(torch.max(b_seq_len_extend, 0)[0].item()) + qo_indptr = torch.zeros((B + 1,), dtype=torch.int32, device=device) + qo_indptr[1 : B + 1] = torch.cumsum(b_seq_len_extend[:B], dim=0) + + inputs = dict( + q_extend=q_extend, + k_extend=k_extend, + v_extend=v_extend, + k_buffer=k_buffer, + v_buffer=v_buffer, + o_extend_triton=o_extend_triton, + o_extend_torch=o_extend_torch, + qo_indptr=qo_indptr, + kv_indptr=kv_indptr, + kv_indices=kv_indices, + max_len_extend=max_len_extend, + WINDOW_SIZE=WINDOW_SIZE, + ) + meta = dict( + B=B, N_CTX=N_CTX, H_Q=H_Q, H_KV=H_KV, D=D, extend_token_num=extend_token_num + ) + return inputs, meta + + +def _run_triton(inputs): + extend_attention_fwd( + inputs["q_extend"], + inputs["k_extend"], + inputs["v_extend"], + inputs["o_extend_triton"], + inputs["k_buffer"], + inputs["v_buffer"], + inputs["qo_indptr"], + inputs["kv_indptr"], + inputs["kv_indices"], + custom_mask=None, + is_causal=True, + mask_indptr=None, + max_len_extend=inputs["max_len_extend"], + sliding_window_size=inputs["WINDOW_SIZE"], + ) + + +def _run_torch_ref(inputs): + extend_attention_fwd_torch( + inputs["q_extend"], + inputs["k_extend"], + inputs["v_extend"], + inputs["o_extend_torch"], + inputs["k_buffer"], + inputs["v_buffer"], + inputs["qo_indptr"], + inputs["kv_indptr"], + inputs["kv_indices"], + inputs["WINDOW_SIZE"], + ) + + +N_CTXS = [1024, 2048, 4096, 8192] +WINDOW_SIZES = [-1, 127, 256, 512] + +CONFIGS = list(itertools.product(N_CTXS, WINDOW_SIZES)) + +PROVIDERS = ["torch", "triton"] + + +@tt.perf_report( + tt.Benchmark( + x_names=["N_CTX", "WINDOW_SIZE"], + x_vals=CONFIGS, + line_arg="provider", + line_vals=PROVIDERS, + line_names=PROVIDERS, + ylabel="Runtime (ms)", + plot_name="extend_attention_triton_vs_torch", + args={ + "B": 32, + "H_Q": 64, + "H_KV": 8, + "D": 128, + "dtype": "bf16", + "device": "cuda", + "check_correctness": False, + "warmup": 25, + "rep": 100, + }, + ) +) +def bench( + N_CTX, + provider, + B, + H_Q, + H_KV, + D, + dtype, + device, + WINDOW_SIZE, + check_correctness, + warmup, + rep, +): + torch.manual_seed(0) + torch.cuda.manual_seed(0) + dtype_map = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32} + dt = dtype_map[dtype] + + inputs, _ = _build_batch( + B, N_CTX, H_Q, H_KV, D, WINDOW_SIZE, dtype=dt, device=device + ) + + if check_correctness and provider == "triton": + _run_triton(inputs) + _run_torch_ref(inputs) + torch.cuda.synchronize() + if not torch.allclose( + inputs["o_extend_triton"], inputs["o_extend_torch"], rtol=1e-3, atol=1e-3 + ): + raise AssertionError("Mismatch between triton and torch reference.") + + if provider == "triton": + ms = tt.do_bench(lambda: _run_triton(inputs), warmup=warmup, rep=rep) + elif provider == "torch": + ms = tt.do_bench(lambda: _run_torch_ref(inputs), warmup=warmup, rep=rep) + else: + raise ValueError(provider) + + return ms + + +if __name__ == "__main__": + bench.run(print_data=True, show_plots=False) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/line_retrieval/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/line_retrieval/README.md new file mode 100644 index 00000000..3c81a63c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/line_retrieval/README.md @@ -0,0 +1,37 @@ +## Download data + +``` +wget https://raw.githubusercontent.com/merrymercy/merrymercy.github.io/master/files/random_words.json +python3 gen_data.py --number 1000 +``` + +## Run benchmark + +### Benchmark sglang +``` +python3 -m sglang.launch_server --model-path codellama/CodeLlama-7b-hf --port 30000 +``` + +``` +python3 bench_sglang.py --src-index 600 --num-q 50 --parallel 1 +``` + + +### + +``` +# original +Accuracy: 0.940, latency: 332.83 s + +# parallel encoding (no_adjust, offset = 1000) +Accuracy: 0.760, latency: 238.46 s + +# parallel encoding (no_adjust, offset = 3000) +Accuracy: 0.760, latency: 238.46 s + +# parallel encoding (no_adjust, offset = 0) +Accuracy: 0.520, latency: 238.46 s + +# parallel encoding (adjust_cache) +Accuracy: 0.460, latency: 257.66 s +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/line_retrieval/bench_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/line_retrieval/bench_sglang.py new file mode 100644 index 00000000..e974e7dd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/line_retrieval/bench_sglang.py @@ -0,0 +1,149 @@ +import argparse +import json +import re +import time + +import numpy as np + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text + + +@sgl.function +def line_retrieval(s, prefix, suffix, body_0, body_1, body_2, body_3): + s += prefix + "\n" + + contexts = [body_0, body_1, body_2, body_3] + position_ids_offset = [i * 1000 for i in range(len(contexts))] + forks = s.fork(len(contexts), position_ids_offset) + forks += lambda i: contexts[i] + "\n" + forks.join(mode="concate_and_append") + + s += "\n" + suffix + s += sgl.gen("answer", max_tokens=16) + + +def eval_model(args, line_obj, num_hoops, src_indices, dst_percents): + arguments = [] + labels = [] + sum_src_indices = [] + sum_dst_indices = [] + + for i in range(len(src_indices)): + for j in range(len(dst_percents)): + src_index = src_indices[i] + dst_percent = dst_percents[j] + + query_indices = line_obj["group_by_num_hoops"][str(num_hoops)] + query_indices = [ + q + for q in query_indices + if all(l <= src_index for l in line_obj["links"][q]) and q < src_index + ] + dst_index = query_indices[ + min(int(len(query_indices) * dst_percent), len(query_indices) - 1) + ] + label = line_obj["values"][dst_index] + + body = line_obj["lines"][: src_index + 1] + suffix = line_obj["suffix"].replace("???", line_obj["indices"][dst_index]) + body_part_len = len(body) // 4 + + arguments.append( + { + "prefix": line_obj["prefix"], + "body_0": "\n".join(body[:body_part_len]), + "body_1": "\n".join(body[body_part_len : 2 * body_part_len]), + "body_2": "\n".join(body[2 * body_part_len : 3 * body_part_len]), + "body_3": "\n".join(body[3 * body_part_len :]), + "suffix": suffix, + } + ) + labels.append(label) + sum_src_indices.append(src_index) + sum_dst_indices.append(dst_index) + + # Select backend + backend = select_sglang_backend(args) + + tic = time.perf_counter() + states = line_retrieval.run_batch( + arguments, + temperature=0, + backend=backend, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + + corrects = [] + for i in range(len(arguments)): + output = states[i]["answer"] + prompt_len = states[i].get_meta_info("answer").get("prompt_length", -1) + label = labels[i] + + # Try all numbers + findall = re.findall("\d+", output) + if not findall: + response_number = output + else: + for response_number in findall: + if response_number == label: + break + + correct = response_number == label + corrects.append(correct) + + # Log results + summary = ( + f"Line index: {sum_src_indices[i]} -> {sum_dst_indices[i]}, " + f"Prompt len: {prompt_len}, " + f"Correct: {correct}, " + f"Label: {label}, Predicted: {response_number}, " + ) + print(summary) + + accuracy = np.mean(corrects) + print(f"Accuracy: {accuracy:.3f}, latency: {latency:.2f} s") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "line_retrieval", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": len(arguments), + "other": { + "num_questions": len(arguments), + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +def main(args): + line_obj = json.load(open(args.data_path, "r")) + + num_hoops = args.num_hoops + for src_index in args.src_index: + src_indices = [src_index] + num_queries = args.num_queries_per_src + dst_percents = [i * (1 / (num_queries)) for i in range(num_queries)] + eval_model(args, line_obj, num_hoops, src_indices, dst_percents) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="lines_1000_0.0.json") + parser.add_argument("--src-index", type=int, nargs="+", default=[100]) + parser.add_argument("--num-queries-per-src", type=int, default=10) + parser.add_argument("--num-hoops", type=int, default=1) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/line_retrieval/gen_data.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/line_retrieval/gen_data.py new file mode 100644 index 00000000..a01d40a0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/line_retrieval/gen_data.py @@ -0,0 +1,139 @@ +""" +Generate line data for line retrieval task. + +Usage: +python3 gen_data.py --number 1000 +""" + +import argparse +import json +from collections import defaultdict + +import numpy as np +from tqdm import tqdm + + +def generate_lines(random_words, num_lines, redirect_ratio): + prefix = "Here is a list of lines, each with its corresponding REGISTER_CONTENT value. Please memorize them. Be prepared to provide the REGISTER_CONTENT value for a specific line index when I ask." + suffix = "The list has ended. Please give the final REGISTER_CONTENT value for a specific line after resolving the redirections and references. For example, the REGISTER_CONTENT of Line __idx0__ is __val0__. The REGISTER_CONTENT of Line __idx1__ is __val1__. The REGISTER_CONTENT of Line __idx2__ is __val2__. The REGISTER_CONTENT of Line ??? is" + + # Raw lines + visited_indices = set([None]) + visited_values = set([None]) + + lines = [] + redirects = [] + indices = [] + values = [] + for i in tqdm(range(num_lines)): + line_index = None + while line_index in visited_indices: + line_index = "-".join(np.random.choice(random_words, size=(2,))) + visited_indices.add(line_index) + + line_value = np.random.randint(low=0, high=999999) + line_value = f"{line_value:06}" + + line = f"Line {line_index}: The REGISTER_CONTENT is {line_value}." + lines.append(line) + redirects.append(None) + indices.append(line_index) + values.append(line_value) + + # Add redirect + if redirect_ratio > 0: + num_redirect_lines = int(len(lines) * redirect_ratio) + redirect_indices = np.random.choice( + np.arange(len(lines)), size=(num_redirect_lines,), replace=False + ) + for i in redirect_indices: + target_idx = np.random.choice(min(i * 2 + 100, num_lines)) + lines[i] = ( + f"Line {indices[i]}: The REGISTER_CONTENT is the same as Line {indices[target_idx]}." + ) + redirects[i] = target_idx + + # Build links and find sources + links = [[] for _ in range(num_lines)] + contains_ring = set() + for i in range(num_lines): + if redirects[i] is None: + continue + + tmp_link = [] + cur = i + visited = set() + while redirects[cur] is not None: + visited.add(cur) + tmp_link.append(redirects[cur]) + cur = redirects[cur] + + if cur in visited: + contains_ring.add(i) + tmp_link = None + break + values[i] = values[cur] + links[i] = tmp_link + + # Group by num_links + group_by_num_hoops = defaultdict(list) + for i in range(num_lines): + if i in contains_ring: + continue + group_by_num_hoops[len(links[i]) + 1].append(i) + + keys = sorted(list(group_by_num_hoops.keys())) + for num_links in keys: + print(f"#links: {num_links}, #lines: {len(group_by_num_hoops[num_links])}") + + # Append few-shot examples + hoop1_candidates = list(group_by_num_hoops[1]) + hoop1_candidate_keys = {c: max([c] + links[c]) for c in hoop1_candidates} + hoop1_candidates.sort(key=lambda c: hoop1_candidate_keys[c]) + hoop2_candidates = list(group_by_num_hoops[2]) + hoop2_candidate_keys = {c: max([c] + links[c]) for c in hoop2_candidates} + hoop2_candidates.sort(key=lambda c: hoop2_candidate_keys[c]) + + i = hoop1_candidates[5] + suffix = suffix.replace("__idx0__", indices[i]).replace("__val0__", values[i]) + if len(hoop2_candidates): + i = hoop2_candidates[0] + suffix = suffix.replace("__idx1__", indices[i]).replace("__val1__", values[i]) + i = hoop2_candidates[1] + suffix = suffix.replace("__idx2__", indices[i]).replace("__val2__", values[i]) + else: + i = hoop1_candidates[1] + suffix = suffix.replace("__idx1__", indices[i]).replace("__val1__", values[i]) + i = hoop1_candidates[10] + suffix = suffix.replace("__idx2__", indices[i]).replace("__val2__", values[i]) + + obj = { + "prefix": prefix, + "suffix": suffix, + "lines": lines, + "indices": indices, + "values": values, + "links": links, + "group_by_num_hoops": group_by_num_hoops, + "contains_ring": sorted(list(contains_ring)), + } + return obj + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--number", type=int) + parser.add_argument("--redirect-ratio", type=float, default=0.0) + args = parser.parse_args() + + num_lines = args.number + + random_words_filename = "random_words.json" + random_words = json.load(open(random_words_filename, "r")) + + np.random.seed(42) + obj = generate_lines(random_words, num_lines, args.redirect_ratio) + + fout = f"lines_{num_lines}_{args.redirect_ratio:.1f}.json" + with open(fout, "w") as fout: + json.dump(obj, fout, indent=2) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/README.md new file mode 100644 index 00000000..a71f339d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/README.md @@ -0,0 +1,61 @@ +## Download benchmark images + +``` +python3 download_images.py +``` + +image benchmark source: https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild + +### Other Dependency +``` +pip3 install "sglang[all]" +pip3 install "torch>=2.1.2" "transformers>=4.36" pillow +``` + +## Run benchmark + +### Benchmark sglang +Launch a server +``` +python3 -m sglang.launch_server --model-path liuhaotian/llava-v1.6-vicuna-7b --tokenizer-path llava-hf/llava-1.5-7b-hf --port 30000 +``` + +Run benchmark +``` +# Run with local models +python3 bench_sglang.py --num-questions 60 + +# Run with OpenAI models +python3 bench_sglang.py --num-questions 60 --backend gpt-4-vision-preview +``` + +### Bench LLaVA original code +``` +git clone git@github.com:haotian-liu/LLaVA.git +cd LLaVA +git reset --hard 9a26bd1435b4ac42c282757f2c16d34226575e96 +pip3 install -e . + +cd ~/sglang/benchmark/llava_bench +CUDA_VISIBLE_DEVICES=0 bash bench_hf_llava_bench.sh +``` + + +### Benchmark llama.cpp + +``` +# Install +CMAKE_ARGS="-DLLAMA_CUBLAS=on" pip install llama-cpp-python +pip install sse_starlette starlette_context pydantic_settings + +# Download weights +mkdir -p ~/model_weights/llava-v1.5-7b/ +wget https://huggingface.co/mys/ggml_llava-v1.5-7b/resolve/main/ggml-model-f16.gguf -O ~/model_weights/llava-v1.5-7b/ggml-model-f16.gguf +wget https://huggingface.co/mys/ggml_llava-v1.5-7b/resolve/main/mmproj-model-f16.gguf -O ~/model_weights/llava-v1.5-7b/mmproj-model-f16.gguf +``` + +``` +python3 -m llama_cpp.server --model ~/model_weights/llava-v1.5-7b/ggml-model-f16.gguf --clip_model_path ~/model_weights/llava-v1.5-7b/mmproj-model-f16.gguf --chat_format llava-1-5 --port 23000 + +OPENAI_BASE_URL=http://localhost:23000/v1 python3 bench_sglang.py --backend gpt-4-vision-preview --num-q 1 +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/bench_hf_llava_bench.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/bench_hf_llava_bench.sh new file mode 100755 index 00000000..a51a715b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/bench_hf_llava_bench.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +python -m llava.eval.model_vqa \ + --model-path liuhaotian/llava-v1.5-7b \ + --question-file ./questions.jsonl \ + --image-folder ./images \ + --answers-file ./answers_hf.jsonl \ + --temperature 0 \ + --conv-mode vicuna_v1 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/bench_hf_mme.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/bench_hf_mme.sh new file mode 100755 index 00000000..6ed332fe --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/bench_hf_mme.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +python -m llava.eval.model_vqa_loader \ + --model-path liuhaotian/llava-v1.5-7b \ + --question-file ./mme_pack/llava_mme_bench_replace.jsonl \ + --image-folder ./mme_pack/MME_Benchmark_release_version \ + --answers-file ./answers_hf_mme.jsonl \ + --temperature 0 \ + --conv-mode vicuna_v1 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/bench_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/bench_sglang.py new file mode 100644 index 00000000..b9e8c140 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/bench_sglang.py @@ -0,0 +1,96 @@ +import argparse +import json +import os +import time + +import tqdm + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + + +@sgl.function +def image_qa(s, image_file, question): + s += sgl.user(sgl.image(image_file) + question) + s += sgl.assistant(sgl.gen("answer", max_tokens=args.max_tokens)) + + +def main(args): + lines = list(read_jsonl(args.question_file))[: args.num_questions] + arguments = [ + { + "image_file": os.path.abspath(args.image_folder + "/" + l["image"]), + "question": l["text"], + } + for l in lines + ] + # arguments = [ + # {"image_file": + # Image.open(os.path.abspath(args.image_folder + "/" + l["image"])), + # "question": l["text"]} for l in lines + # ] + + states = [None] * len(lines) + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.perf_counter() + if args.parallel == 1: + for i in tqdm.tqdm(range(len(lines))): + image_file = arguments[i]["image_file"] + question = arguments[i]["question"] + ret = image_qa.run(image_file=image_file, question=question, temperature=0) + states[i] = ret + else: + states = image_qa.run_batch( + arguments, temperature=0, num_threads=args.parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + print(f"Write output to {args.answer_file}") + with open(args.answer_file, "w") as fout: + for i in range(len(lines)): + value = { + "question_id": lines[i]["question_id"], + "prompt": lines[i]["text"], + "text": states[i]["answer"].strip(), + "model_id": backend.model_info["model_path"], + "answer_id": i, + "metadata": {}, + } + fout.write(json.dumps(value) + "\n") + + with open(args.result_file, "a") as fout: + value = { + "task": "llava_bench", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": len(lines), + "parallel": args.parallel, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--question-file", type=str, default="questions.jsonl") + parser.add_argument("--answer-file", type=str, default="answers.jsonl") + parser.add_argument("--image-folder", type=str, default="./images") + parser.add_argument("--temperature", type=float, default=0.0) + parser.add_argument("--num-questions", type=int, default=None) + parser.add_argument("--max-tokens", type=int, default=768) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/bench_sglang_mme.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/bench_sglang_mme.sh new file mode 100644 index 00000000..8c8c09cc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/bench_sglang_mme.sh @@ -0,0 +1,2 @@ +MME_FOLDER=./mme_pack +python3 bench_sglang.py --num-questions 5000 --question-file $MME_FOLDER/llava_mme_bench_replace.jsonl --answer-file answer_mme.jsonl --image-folder $MME_FOLDER/MME_Benchmark_release_version --max-tokens 4 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/download_images.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/download_images.py new file mode 100644 index 00000000..701190a0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llava_bench/download_images.py @@ -0,0 +1,20 @@ +import os + +# Create the 'images' directory if it doesn't exist +if not os.path.exists("images"): + os.makedirs("images") + +# Base URL +base_url = "https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild/resolve/main/images/" + +# Loop through image numbers +for i in range(1, 25): + # Format the image number with leading zeros + image_number = str(i).zfill(3) + image_url = base_url + image_number + ".jpg" + image_path = "images/" + image_number + ".jpg" + + # Download the image using wget + os.system(f"wget -O {image_path} {image_url}") + +print("Download complete.") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llm_judge/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llm_judge/README.md new file mode 100644 index 00000000..164b19a0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llm_judge/README.md @@ -0,0 +1,33 @@ +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 25 --parallel 8 +python3 bench_sglang.py --num-questions 16 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --backend vllm --num-questions 25 +``` + + +### Benchmark guidance +``` +python3 bench_other.py --backend guidance --num-questions 25 --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +### Benchmark lmql + +``` +python3 bench_other.py --backend lmql --num-questions 25 --parallel 1 +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llm_judge/bench_other.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llm_judge/bench_other.py new file mode 100644 index 00000000..8e602906 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llm_judge/bench_other.py @@ -0,0 +1,151 @@ +import argparse +import json +import time +from concurrent.futures import ThreadPoolExecutor +from functools import partial + +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +system_prompt = "Please serve as an impartial judge and rigorously evaluate the quality of the following article. Apply the most stringent standards possible, showing no leniency." + +dimension_prompts = [ + "Content: This refers to the essences of the essay. The substance should be well researched, accurate, relevant to the topic and should show a thorough understanding of the subject. The essay should also reflect a clear goal or purpose.", + "Organization and Structure: An essay needs to be properly structured with a clear introduction, body, and conclusion. The essay should flow naturally, with one paragraph leading seamlessly into the next.", + "Argument and Analysis: The argument made in the essay should be logical, coherent and clearly articulated. Each point made should be backed up by solid evidence and thorough analysis.", + "Clarity and Precision: The essay should be written in a clear and concise manner. The points made should be easily understood by the reader. The language used should also be precise and unambiguous.", + "Grammar and Punctuation: Proper use of grammar and punctuation is vital in an academic essay. Errors in grammar and punctuation not only distract the reader but can also negatively impact the meaning and interpretation of the content.", + "Referencing and Citation: An essay should contain proper citations and references for all sources used. This not only prevents accusations of plagiarism but also gives credit to the authors of the works that have contributed to the essay. The citation should adhere to a specific format as required by the academic institution or specified by the professor.", +] + + +def multi_dimension_judge(article, generate): + s = system_prompt + s += "\n```\n" + article + "\n```\n\n" + + judges = [] + for i in range(len(dimension_prompts)): + comp = generate( + s + + "USER: Please judge the quality based on the following metric. " + + dimension_prompts[i] + + " Please provide a single-paragraph judgement. " + + "Focus on the provided metric and do not say other things. " + 'End your judgement paragraph with the word "END"\nJUDGE:', + max_tokens=256, + stop="END", + ) + judges.append(comp) + + s += "I will judge the quality based on the following metrics.\n" + for i in range(len(dimension_prompts)): + s += dimension_prompts[i].split(":")[0] + ": " + judges[i].strip() + "\n" + + s += "In summary, on a scale of 1 to 10, I would give the article a score of" + s += generate(s, max_tokens=2, stop=None) + + return s + + +async def multi_dimension_judge_async(article, generate): + s = system_prompt + s += "\n```\n" + article + "\n```\n\n" + + judges = [] + for i in range(len(dimension_prompts)): + comp = await generate( + s + + "USER: Please judge the quality based on the following metric. " + + dimension_prompts[i] + + " Please provide a single-paragraph judgement. " + + "Focus on the provided metric and do not say other things. " + 'End your judgement paragraph with the word "END"\nJUDGE:', + max_tokens=256, + stop="END", + ) + judges.append(comp) + + s += "I will judge the quality based on the following metrics.\n" + for i in range(len(dimension_prompts)): + s += dimension_prompts[i].split(":")[0] + ": " + judges[i].strip() + "\n" + + s += "In summary, on a scale of 1 to 10, I would give the article a score of" + s += await generate(s, max_tokens=2, stop=None) + + return s + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_questions] + states = [None] * len(lines) + + # Select backend + call_generate = partial(get_call_generate(args), temperature=0) + + # Run requests + tic = time.perf_counter() + + if args.backend != "lmql": + + def get_one_answer(i): + states[i] = multi_dimension_judge(lines[i], call_generate) + + if args.parallel == 1: + for i in tqdm(range(len(lines))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(lines)))), + total=len(lines), + ) + ) + + else: + import asyncio + + async def get_one_answer_async(i): + states[i] = await multi_dimension_judge_async(lines[i], call_generate) + + batches = [] + for i in range(0, len(lines), args.parallel): + batches.append(list(range(i, min(i + args.parallel, len(lines))))) + + loop = asyncio.get_event_loop() + for bt in tqdm(batches): + loop.run_until_complete( + asyncio.gather(*[get_one_answer_async(i) for i in bt]) + ) + + latency = time.perf_counter() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "llm_judge", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="articles.jsonl") + parser.add_argument("--num-questions", type=int, default=20) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llm_judge/bench_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llm_judge/bench_sglang.py new file mode 100644 index 00000000..97e6c397 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/llm_judge/bench_sglang.py @@ -0,0 +1,97 @@ +import argparse +import json +import time + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + +system_prompt = "Please serve as an impartial judge and rigorously evaluate the quality of the following article. Apply the most stringent standards possible, showing no leniency." + +dimension_prompts = [ + "Content: This refers to the essences of the essay. The substance should be well researched, accurate, relevant to the topic and should show a thorough understanding of the subject. The essay should also reflect a clear goal or purpose.", + "Organization and Structure: An essay needs to be properly structured with a clear introduction, body, and conclusion. The essay should flow naturally, with one paragraph leading seamlessly into the next.", + "Argument and Analysis: The argument made in the essay should be logical, coherent and clearly articulated. Each point made should be backed up by solid evidence and thorough analysis.", + "Clarity and Precision: The essay should be written in a clear and concise manner. The points made should be easily understood by the reader. The language used should also be precise and unambiguous.", + "Grammar and Punctuation: Proper use of grammar and punctuation is vital in an academic essay. Errors in grammar and punctuation not only distract the reader but can also negatively impact the meaning and interpretation of the content.", + "Referencing and Citation: An essay should contain proper citations and references for all sources used. This not only prevents accusations of plagiarism but also gives credit to the authors of the works that have contributed to the essay. The citation should adhere to a specific format as required by the academic institution or specified by the professor.", +] + + +@sgl.function +def multi_dimension_judge(s, article): + s += system_prompt + s += "\n```\n" + article + "\n```\n\n" + + forks = s.fork(len(dimension_prompts)) + for i in range(len(dimension_prompts)): + forks[i] += ( + "USER: Please judge the quality based on the following metric. " + + dimension_prompts[i] + + " Please provide a single-paragraph judgement. " + + "Focus on the provided metric and do not say other things. " + 'End your judgement paragraph with the word "END"\nJUDGE:' + ) + forks[i] += sgl.gen("judgement", max_tokens=256, stop="END") + forks.join() + + s += "I will judge the quality based on the following metrics.\n" + for i in range(len(dimension_prompts)): + s += ( + dimension_prompts[i].split(":")[0] + + ": " + + forks[i]["judgement"].strip() + + "\n" + ) + + s += "In summary, on a scale of 1 to 10, I would give the article a score of" + s += sgl.gen("score", max_tokens=2) + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_questions] + arguments = [{"article": l} for l in lines] + + # Select backend + backend = select_sglang_backend(args) + + # Run requests + tic = time.perf_counter() + states = multi_dimension_judge.run_batch( + arguments, + temperature=0, + backend=backend, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "llm_judge", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="articles.jsonl") + parser.add_argument("--num-questions", type=int, default=20) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/long_json_decode/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/long_json_decode/README.md new file mode 100644 index 00000000..37fceee1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/long_json_decode/README.md @@ -0,0 +1,33 @@ +## Run benchmark + +### Benchmark sglang +``` +python3 -m sglang.launch_server --model-path codellama/CodeLlama-7b-instruct-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 5 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model codellama/CodeLlama-7b-instruct-hf --disable-log-requests --port 21000 --gpu 0.97 +``` + +``` +python3 bench_other.py --backend vllm --num-questions 5 +``` + + +### Benchmark guidance +``` +python3 bench_other.py --backend guidance --num-questions 5 --parallel 1 --n-ctx 11000 --model-path path/to/code-llama/gguf +``` + + +### Build dataset +``` +pip install wikipedia +python3 build_dataset.py +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/long_json_decode/bench_other.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/long_json_decode/bench_other.py new file mode 100644 index 00000000..0ad38a01 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/long_json_decode/bench_other.py @@ -0,0 +1,89 @@ +import argparse +import json +import time +from concurrent.futures import ThreadPoolExecutor +from functools import partial + +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + + +def json_decode(document, generate): + s = "Please extract the information of a city from the following wikipedia page.\n" + s += "Page begin.\n" + document + "Page end.\n" + s += "Here is the name, country, and symbol of the city in JSON format.\n" + s += "{\n" + s += ' "name": "' + s += generate(s, max_tokens=8, stop='"') + '",\n' + s += ' "country": "' + s += generate(s, max_tokens=8, stop='"') + '",\n' + s += ' "air port code": "' + s += generate(s, max_tokens=8, stop='"') + '",\n' + s += ' "top 3 landmarks": "' + s += generate(s, max_tokens=24, stop='"') + '",\n' + s += "}\n" + return s + + +def main(args): + lines = read_jsonl(args.data_path) + arguments = [] + for i in range(len(lines[: args.num_questions])): + arguments.append( + { + "document": lines[i]["document"], + } + ) + states = [None] * len(arguments) + + # Select backend + call_generate = partial(get_call_generate(args), temperature=0) + + # Run requests + def get_one_answer(i): + states[i] = json_decode(generate=call_generate, **arguments[i]) + + tic = time.perf_counter() + if args.parallel == 1: + for i in tqdm(range(len(arguments))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(arguments)))), + total=len(arguments), + ) + ) + + latency = time.perf_counter() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "long_json_decode", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="questions.jsonl") + parser.add_argument("--num-questions", type=int, default=100) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/long_json_decode/bench_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/long_json_decode/bench_sglang.py new file mode 100644 index 00000000..8394cfc2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/long_json_decode/bench_sglang.py @@ -0,0 +1,81 @@ +import argparse +import json +import time + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + + +@sgl.function +def json_decode(s, document): + s += "Please extract the information of a city from the following wikipedia page.\n" + s += "Page begin.\n" + document + "Page end.\n" + s += "Here is the name, country, and symbol of the city in JSON format.\n" + s += "{\n" + s += ' "name": "' + sgl.gen("name", max_tokens=8, stop='"') + '",\n' + s += ' "country": "' + sgl.gen("country", max_tokens=8, stop='"') + '",\n' + s += ( + ' "air port code": "' + + sgl.gen("air port code", max_tokens=8, stop='"') + + '",\n' + ) + s += ( + ' "top 3 landmarks": "' + + sgl.gen("landmarks", max_tokens=24, stop='"') + + '",\n' + ) + s += "}\n" + + +def main(args): + lines = read_jsonl(args.data_path) + arguments = [] + for i in range(len(lines[: args.num_questions])): + arguments.append( + { + "document": lines[i]["document"], + } + ) + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.perf_counter() + states = json_decode.run_batch( + arguments, temperature=0, num_threads=args.parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "long_json_decode", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="questions.jsonl") + parser.add_argument("--num-questions", type=int, default=10) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/long_json_decode/build_dataset.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/long_json_decode/build_dataset.py new file mode 100644 index 00000000..7b15d395 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/long_json_decode/build_dataset.py @@ -0,0 +1,27 @@ +import json + +import transformers +import wikipedia + +name = "meta-llama/Llama-2-7b-chat-hf" +t = transformers.AutoTokenizer.from_pretrained(name) +city_names = ["los angles", "london", "tokyo", "beijing", "singapore"] + + +for city_name in city_names: + content = str(wikipedia.page(city_name).content) + content = content.replace("\n\n", "\n") + + tokens = t.encode(content) + + truncate_len = int((10000 / len(tokens)) * len(content)) + truncate_content = content[:truncate_len] + truncate_tokens = t.encode(truncate_content) + + # Count token + print( + f"city_name: {city_name}, #tokens: {len(tokens)}, #truncate tokens: {len(truncate_tokens)}" + ) + + with open("questions.jsonl", "a") as fout: + fout.write(json.dumps({"document": truncate_content}) + "\n") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/lora/launch_server.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/lora/launch_server.py new file mode 100644 index 00000000..5dcf66ad --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/lora/launch_server.py @@ -0,0 +1,82 @@ +import argparse +import os + +NUM_LORAS = 4 +LORA_PATH = { + "base": "meta-llama/Llama-2-7b-hf", + "lora": "winddude/wizardLM-LlaMA-LoRA-7B", +} + + +def launch_server(args): + base_path = LORA_PATH["base"] + lora_path = LORA_PATH["lora"] + + if args.base_only: + cmd = f"python3 -m sglang.launch_server --model {base_path} " + else: + cmd = f"python3 -m sglang.launch_server --model {base_path} --lora-paths " + for i in range(NUM_LORAS): + lora_name = f"lora{i}" + cmd += f"{lora_name}={lora_path} " + cmd += f"--disable-radix " + cmd += f"--max-loras-per-batch {args.max_loras_per_batch} " + cmd += f"--max-running-requests {args.max_running_requests} " + cmd += f"--lora-backend {args.lora_backend} " + cmd += f"--tp-size {args.tp_size} " + if args.disable_custom_all_reduce: + cmd += "--disable-custom-all-reduce" + if args.enable_mscclpp: + cmd += "--enable-mscclpp" + if args.enable_torch_symm_mem: + cmd += "--enable-torch-symm-mem" + print(cmd) + os.system(cmd) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--base-only", + action="store_true", + ) + parser.add_argument( + "--max-loras-per-batch", + type=int, + default=8, + ) + parser.add_argument( + "--max-running-requests", + type=int, + default=8, + ) + parser.add_argument( + "--lora-backend", + type=str, + default="csgmv", + ) + parser.add_argument( + "--tp-size", + type=int, + default=1, + help="Tensor parallel size for distributed inference", + ) + # disable_custom_all_reduce + parser.add_argument( + "--disable-custom-all-reduce", + action="store_true", + help="Disable custom all reduce when device does not support p2p communication", + ) + parser.add_argument( + "--enable-mscclpp", + action="store_true", + help="Enable using mscclpp for small messages for all-reduce kernel and fall back to NCCL.", + ) + parser.add_argument( + "--enable-torch-symm-mem", + action="store_true", + help="Enable using torch symm mem for all-reduce kernel and fall back to NCCL.", + ) + args = parser.parse_args() + + launch_server(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/lora/lora_bench.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/lora/lora_bench.py new file mode 100644 index 00000000..4f380c70 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/lora/lora_bench.py @@ -0,0 +1,475 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import argparse +import asyncio +import json +import random +import resource +import sys +import time +import traceback +from argparse import ArgumentParser +from datetime import datetime +from typing import Any, Dict, List, Optional, Tuple + +import numpy as np +from launch_server import LORA_PATH, NUM_LORAS +from tqdm.asyncio import tqdm +from transformers import PreTrainedTokenizerBase + +from sglang.bench_serving import ( + RequestFuncInput, + RequestFuncOutput, + _create_bench_client_session, + calculate_metrics, + get_request, + get_tokenizer, + remove_prefix, + sample_random_requests, +) + +global args + + +# set ignore_eos True by default +async def async_request_openai_completions( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + # assert api_url.endswith( + # "completions" + # ), "OpenAI Completions API URL must end with 'completions'." + + prompt = request_func_input.prompt + + async with _create_bench_client_session() as session: + # payload = { + # "model": request_func_input.model, + # "prompt": prompt, + # "temperature": 0.0, + # "best_of": 1, + # "max_tokens": request_func_input.output_len, + # "stream": not args.disable_stream, + # "ignore_eos": not args.disable_ignore_eos, + # **request_func_input.extra_request_body, + # } + # headers = {"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"} + if args.base_only: + payload = { + "text": prompt, + "sampling_params": {"max_new_tokens": request_func_input.output_len}, + } + else: + payload = { + "text": prompt, + "sampling_params": {"max_new_tokens": request_func_input.output_len}, + "lora_path": f"lora{random.randint(0, NUM_LORAS - 1)}", + } + headers = {"Authorization": ""} + + output = RequestFuncOutput() + output.prompt_len = request_func_input.prompt_len + + generated_text = "" + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + # NOTE: Some completion API might have a last + # usage summary response without a token so we + # want to check a token was generated + if data["text"]: + # if data["choices"][0]["text"]: + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft = ttft + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + most_recent_timestamp = timestamp + # generated_text += data["choices"][0]["text"] + generated_text += data["text"] + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.output_len = request_func_input.output_len + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +ASYNC_REQUEST_FUNCS = { + "sglang": async_request_openai_completions, +} + + +async def benchmark( + backend: str, + api_url: str, + model_id: str, + tokenizer: PreTrainedTokenizerBase, + input_requests: List[Tuple[str, int, int]], + request_rate: float, + disable_tqdm: bool, + extra_request_body: Dict[str, Any], +): + if backend in ASYNC_REQUEST_FUNCS: + request_func = ASYNC_REQUEST_FUNCS[backend] + else: + raise ValueError(f"Unknown backend: {backend}") + + print("Starting initial single prompt test run...") + test_request = input_requests[0] + test_input = RequestFuncInput( + model=model_id, + prompt=test_request.prompt, + api_url=api_url, + prompt_len=test_request.prompt_len, + output_len=test_request.output_len, + lora_name="dummy", # the lora_name argument will not be used + image_data=None, + extra_request_body=extra_request_body, + ) + test_output = await request_func(request_func_input=test_input) + if not test_output.success: + raise ValueError( + "Initial test run failed - Please make sure benchmark arguments " + f"are correctly specified. Error: {test_output.error}" + ) + else: + print("Initial test run completed. Starting main benchmark run...") + + pbar = None if disable_tqdm else tqdm(total=len(input_requests)) + + benchmark_start_time = time.perf_counter() + tasks: List[asyncio.Task] = [] + async for request in get_request(input_requests, request_rate): + request_func_input = RequestFuncInput( + model=model_id, + prompt=request.prompt, + api_url=api_url, + prompt_len=request.prompt_len, + output_len=request.output_len, + lora_name="dummy", + image_data=None, + extra_request_body=extra_request_body, + ) + tasks.append( + asyncio.create_task( + request_func(request_func_input=request_func_input, pbar=pbar) + ) + ) + outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks) + + if pbar is not None: + pbar.close() + + benchmark_duration = time.perf_counter() - benchmark_start_time + + metrics, output_lens = calculate_metrics( + input_requests=input_requests, + outputs=outputs, + dur_s=benchmark_duration, + tokenizer=tokenizer, + backend=backend, + ) + + print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="=")) + print("{:<40} {:<10}".format("Backend:", backend)) + print("{:<40} {:<10}".format("Traffic request rate:", request_rate)) + print("{:<40} {:<10}".format("Successful requests:", metrics.completed)) + print("{:<40} {:<10.2f}".format("Benchmark duration (s):", benchmark_duration)) + print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input)) + print("{:<40} {:<10}".format("Total generated tokens:", metrics.total_output)) + print( + "{:<40} {:<10}".format( + "Total generated tokens (retokenized):", metrics.total_output_retokenized + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Request throughput (req/s):", metrics.request_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Input token throughput (tok/s):", metrics.input_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Output token throughput (tok/s):", metrics.output_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format("Total throughput (tok/s):", metrics.total_throughput) + ) + print("{s:{c}^{n}}".format(s="End-to-End Latency", n=50, c="-")) + print( + "{:<40} {:<10.2f}".format("Mean E2E Latency (ms):", metrics.mean_e2e_latency_ms) + ) + print( + "{:<40} {:<10.2f}".format( + "Median E2E Latency (ms):", metrics.median_e2e_latency_ms + ) + ) + print("{s:{c}^{n}}".format(s="Time to First Token", n=50, c="-")) + print("{:<40} {:<10.2f}".format("Mean TTFT (ms):", metrics.mean_ttft_ms)) + print("{:<40} {:<10.2f}".format("Median TTFT (ms):", metrics.median_ttft_ms)) + print("{:<40} {:<10.2f}".format("P99 TTFT (ms):", metrics.p99_ttft_ms)) + print( + "{s:{c}^{n}}".format(s="Time per Output Token (excl. 1st token)", n=50, c="-") + ) + print("{:<40} {:<10.2f}".format("Mean TPOT (ms):", metrics.mean_tpot_ms)) + print("{:<40} {:<10.2f}".format("Median TPOT (ms):", metrics.median_tpot_ms)) + print("{:<40} {:<10.2f}".format("P99 TPOT (ms):", metrics.p99_tpot_ms)) + print("{s:{c}^{n}}".format(s="Inter-token Latency", n=50, c="-")) + print("{:<40} {:<10.2f}".format("Mean ITL (ms):", metrics.mean_itl_ms)) + print("{:<40} {:<10.2f}".format("Median ITL (ms):", metrics.median_itl_ms)) + print("{:<40} {:<10.2f}".format("P99 ITL (ms):", metrics.p99_itl_ms)) + print("=" * 50) + + if ( + metrics.median_ttft_ms is not None + and metrics.mean_itl_ms is not None + and metrics.output_throughput is not None + ): + result = { + "backend": args.backend, + "request_rate": request_rate, + "total_input_tokens": metrics.total_input, + "total_output_tokens": metrics.total_output, + "total_output_tokens_retokenized": metrics.total_output_retokenized, + "mean_e2e_latency_ms": metrics.mean_e2e_latency_ms, + "median_e2e_latency_ms": metrics.median_e2e_latency_ms, + "median_ttft_ms": metrics.median_ttft_ms, + "median_itl_ms": metrics.median_itl_ms, + "output_throughput": metrics.output_throughput, + "random_input_len": args.random_input_len, + "random_output_len": args.random_output_len, + "random_range_ratio": args.random_range_ratio, + "duration": benchmark_duration, + "completed": metrics.completed, + } + else: + print(f"Error running benchmark for request rate: {request_rate}") + print("-" * 30) + + # Determine output file name + if args.output_file: + output_file_name = args.output_file + else: + now = datetime.now().strftime("%m%d") + output_file_name = f"{args.backend}_{now}_{args.num_prompts}_{args.random_input_len}_{args.random_output_len}.jsonl" + + # Append results to a JSONL file + with open(output_file_name, "a") as file: + file.write(json.dumps(result) + "\n") + + result = { + "duration": benchmark_duration, + "completed": metrics.completed, + "total_input_tokens": metrics.total_input, + "total_output_tokens": metrics.total_output, + "total_output_tokens_retokenized": metrics.total_output_retokenized, + "request_throughput": metrics.request_throughput, + "input_throughput": metrics.input_throughput, + "output_throughput": metrics.output_throughput, + "mean_ttft_ms": metrics.mean_ttft_ms, + "median_ttft_ms": metrics.median_ttft_ms, + "std_ttft_ms": metrics.std_ttft_ms, + "p99_ttft_ms": metrics.p99_ttft_ms, + "mean_tpot_ms": metrics.mean_tpot_ms, + "median_tpot_ms": metrics.median_tpot_ms, + "std_tpot_ms": metrics.std_tpot_ms, + "p99_tpot_ms": metrics.p99_tpot_ms, + "mean_itl_ms": metrics.mean_itl_ms, + "median_itl_ms": metrics.median_itl_ms, + "std_itl_ms": metrics.std_itl_ms, + "p99_itl_ms": metrics.p99_itl_ms, + "input_lens": [output.prompt_len for output in outputs], + "output_lens": output_lens, + "ttfts": [output.ttft for output in outputs], + "itls": [output.itl for output in outputs], + "generated_texts": [output.generated_text for output in outputs], + "errors": [output.error for output in outputs], + "mean_e2e_latency_ms": metrics.mean_e2e_latency_ms, + "median_e2e_latency_ms": metrics.median_e2e_latency_ms, + } + return result + + +def run_benchmark(args_: argparse.Namespace): + global args + args = args_ + + # Set global environments + set_ulimit() + random.seed(args.seed) + np.random.seed(args.seed) + + # Set url + if args.port is None: + args.port = { + "sglang": 30000, + }.get(args.backend, 30000) + + # api_url = ( + # f"{args.base_url}/v1/completions" + # if args.base_url + # else f"http://{args.host}:{args.port}/v1/completions" + # ) + api_url = ( + f"{args.base_url}/generate" + if args.base_url + else f"http://{args.host}:{args.port}/generate" + ) + + print(f"{args}\n") + + # Read dataset + backend = args.backend + model_id = args.model = LORA_PATH["base"] + tokenizer_id = args.model + + tokenizer = get_tokenizer(tokenizer_id) + + input_requests = sample_random_requests( + input_len=args.random_input_len, + output_len=args.random_output_len, + num_prompts=args.num_prompts, + range_ratio=args.random_range_ratio, + tokenizer=tokenizer, + dataset_path="", + ) + + return asyncio.run( + benchmark( + backend=backend, + api_url=api_url, + model_id=model_id, + tokenizer=tokenizer, + input_requests=input_requests, + request_rate=args.request_rate, + disable_tqdm=False, + extra_request_body={}, + ) + ) + + +def set_ulimit(target_soft_limit=65535): + resource_type = resource.RLIMIT_NOFILE + current_soft, current_hard = resource.getrlimit(resource_type) + + if current_soft < target_soft_limit: + try: + resource.setrlimit(resource_type, (target_soft_limit, current_hard)) + except ValueError as e: + print(f"Fail to set RLIMIT_NOFILE: {e}") + + +if __name__ == "__main__": + parser = ArgumentParser(description="Benchmark the online lora serving throughput.") + parser.add_argument( + "--backend", + type=str, + choices=list(ASYNC_REQUEST_FUNCS.keys()), + default="sglang", + help="Must specify a backend, depending on the LLM Inference Engine.", + ) + parser.add_argument( + "--base-url", + type=str, + default=None, + help="Server or API base url if not using http host and port.", + ) + parser.add_argument( + "--host", type=str, default="0.0.0.0", help="Default host is 0.0.0.0." + ) + parser.add_argument( + "--port", + type=int, + help="If not set, the default port is configured according to its default value for different LLM Inference Engines.", + ) + parser.add_argument( + "--num-prompts", + type=int, + default=50, + help="Number of prompts to process. Default is 1000.", + ) + parser.add_argument( + "--random-input-len", + type=int, + default=1024, + help="Number of input tokens per request, used only for random dataset.", + ) + parser.add_argument( + "--random-output-len", + type=int, + default=128, + help="Number of output tokens per request, used only for random dataset.", + ) + parser.add_argument( + "--random-range-ratio", + type=float, + default=0.0, + help="Range of sampled ratio of input/output length, " + "used only for random dataset.", + ) + parser.add_argument( + "--request-rate", + type=float, + default=float("inf"), + help="Number of requests per second. If this is inf, then all the requests are sent at time 0. " + "Otherwise, we use Poisson process to synthesize the request arrival times. Default is inf.", + ) + parser.add_argument( + "--base-only", + action="store_true", + ) + parser.add_argument("--output-file", type=str, help="Output JSONL file name.") + parser.add_argument("--seed", type=int, default=1, help="The random seed.") + args = parser.parse_args() + run_benchmark(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmlu/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmlu/README.md new file mode 100644 index 00000000..16de20cd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmlu/README.md @@ -0,0 +1,59 @@ +## Download data +``` +bash download_data.sh +``` + +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --nsub 10 +``` + +``` +# OpenAI models +python3 bench_sglang.py --backend gpt-3.5-turbo --parallel 8 +``` + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --nsub 10 --backend vllm +``` + + +### Benchmark lightllm +``` +# A10G +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 16000 --port 22000 + +# V100 +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 4500 --port 22000 +``` + +``` +python3 bench_other.py --nsub 10 --backend lightllm +``` + + +### Benchmark guidance +``` +python3 bench_other.py --nsub 10 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + + +### Benchmark lmql +``` +CUDA_VISIBLE_DEVICES=0,1 lmql serve-model meta-llama/Llama-2-7b-chat-hf --cuda --port 23000 +``` + +``` +python3 bench_other.py --nsub 10 --backend lmql --parallel 2 +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmlu/bench_other.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmlu/bench_other.py new file mode 100644 index 00000000..f1b166c2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmlu/bench_other.py @@ -0,0 +1,173 @@ +import argparse +import asyncio +import json +import os +import time +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +import pandas as pd +import tiktoken +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate + +choices = ["A", "B", "C", "D"] + +tokenizer = tiktoken.encoding_for_model("gpt-3.5-turbo") + + +def format_subject(subject): + l = subject.split("_") + s = "" + for entry in l: + s += " " + entry + return s + + +def format_example(df, idx, include_answer=True): + prompt = df.iloc[idx, 0] + k = df.shape[1] - 2 + for j in range(k): + prompt += "\n{}. {}".format(choices[j], df.iloc[idx, j + 1]) + prompt += "\nAnswer:" + if include_answer: + prompt += " {}\n\n".format(df.iloc[idx, k + 1]) + return prompt + + +def gen_prompt(train_df, subject, k=-1): + prompt = "The following are multiple choice questions (with answers) about{}.\n\n".format( + format_subject(subject) + ) + if k == -1: + k = train_df.shape[0] + for i in range(k): + prompt += format_example(train_df, i) + return prompt + + +def evaluate(args, subject, dev_df, test_df, call_generate): + prompts = [] + labels = [] + + # Construct prompts + k = args.ntrain + train_prompt = gen_prompt(dev_df, subject, k) + while len(tokenizer.encode(train_prompt)) > 1536: + k -= 1 + train_prompt = gen_prompt(dev_df, subject, k) + + for i in range(test_df.shape[0]): + prompt_end = format_example(test_df, i, include_answer=False) + prompt = train_prompt + prompt_end + prompts.append(prompt) + + label = test_df.iloc[i, test_df.shape[1] - 1] + labels.append(label) + + preds = [None] * len(prompts) + max_tokens = 1 + + # Run requests + if args.backend != "lmql": + # Use thread pool + def get_one_answer(i): + pred = call_generate(prompts[i], temperature=0, max_tokens=max_tokens) + preds[i] = pred.strip()[0] + + tic = time.perf_counter() + if args.parallel == 1: + for i in range(len(prompts)): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + executor.map(get_one_answer, list(range(len(prompts)))) + else: + # Use asyncio + async def batched_call(batch_size): + for i in range(0, len(prompts), batch_size): + tasks = [] + for p in prompts[i : i + batch_size]: + tasks.append(call_generate(p, temperature=0, max_tokens=max_tokens)) + rets = await asyncio.gather(*tasks) + for j in range(len(rets)): + preds[i + j] = rets[j].strip()[0] + + tic = time.perf_counter() + asyncio.run(batched_call(batch_size=args.parallel)) + latency = time.perf_counter() - tic + + # Compute accuracy + cors = [pred == label for pred, label in zip(preds, labels)] + acc = np.mean(cors) + cors = np.array(cors) + + print( + "Average accuracy {:.3f}, latency {:.2f}, #q: {} - {}".format( + acc, latency, len(prompts), subject + ) + ) + + return cors, acc, latency + + +def main(args): + subjects = sorted( + [ + f.split("_test.csv")[0] + for f in os.listdir(os.path.join(args.data_dir, "test")) + if "_test.csv" in f + ] + ) + + all_cors = [] + all_latencies = [] + num_requests = 0 + + # Select backend + call_generate = get_call_generate(args) + + for subject in tqdm(subjects[: args.nsub]): + dev_df = pd.read_csv( + os.path.join(args.data_dir, "dev", subject + "_dev.csv"), header=None + )[: args.ntrain] + test_df = pd.read_csv( + os.path.join(args.data_dir, "test", subject + "_test.csv"), header=None + ) + + cors, acc, latency = evaluate(args, subject, dev_df, test_df, call_generate) + all_cors.append(cors) + all_latencies.append(latency) + num_requests += len(test_df) + + total_latency = np.sum(all_latencies) + print("Total latency: {:.3f}".format(total_latency)) + + weighted_acc = np.mean(np.concatenate(all_cors)) + print("Average accuracy: {:.3f}".format(weighted_acc)) + + # Write results + with open(args.result_file, "a") as fout: + value = { + "task": "mmlu", + "backend": args.backend, + "num_gpus": 1, + "latency": round(total_latency, 3), + "accuracy": round(weighted_acc, 3), + "num_requests": num_requests, + "other": { + "nsub": args.nsub, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--ntrain", type=int, default=5) + parser.add_argument("--data_dir", type=str, default="data") + parser.add_argument("--nsub", type=int, default=60) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmlu/bench_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmlu/bench_sglang.py new file mode 100644 index 00000000..23057be4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmlu/bench_sglang.py @@ -0,0 +1,181 @@ +import argparse +import json +import os +import time + +import numpy as np +import pandas as pd +import tiktoken + +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + dump_bench_raw_result, + select_sglang_backend, +) + +choices = ["A", "B", "C", "D"] + +tokenizer = tiktoken.encoding_for_model("gpt-3.5-turbo") + + +def format_subject(subject): + l = subject.split("_") + s = "" + for entry in l: + s += " " + entry + return s + + +def format_example(df, idx, include_answer=True): + prompt = df.iloc[idx, 0] + k = df.shape[1] - 2 + for j in range(k): + prompt += "\n{}. {}".format(choices[j], df.iloc[idx, j + 1]) + prompt += "\nAnswer:" + if include_answer: + prompt += " {}\n\n".format(df.iloc[idx, k + 1]) + return prompt + + +def gen_prompt(train_df, subject, k=-1): + prompt = "The following are multiple choice questions (with answers) about{}.\n\n".format( + format_subject(subject) + ) + if k == -1: + k = train_df.shape[0] + for i in range(k): + prompt += format_example(train_df, i) + return prompt + + +def main(args): + subjects = sorted( + [ + f.split("_test.csv")[0] + for f in os.listdir(os.path.join(args.data_dir, "test")) + if "_test.csv" in f + ] + ) + + # Build prompts + arguments = [] + labels = [] + num_questions = [] + + for subject in subjects[: args.nsub]: + dev_df = pd.read_csv( + os.path.join(args.data_dir, "dev", subject + "_dev.csv"), header=None + )[: args.ntrain] + test_df = pd.read_csv( + os.path.join(args.data_dir, "test", subject + "_test.csv"), header=None + ) + num_questions.append(test_df.shape[0]) + + k = args.ntrain + few_shot_examples = gen_prompt(dev_df, subject, k) + while len(tokenizer.encode(few_shot_examples)) > 1536: + k -= 1 + few_shot_examples = gen_prompt(dev_df, subject, k) + + for i in range(test_df.shape[0]): + prompt_end = format_example(test_df, i, include_answer=False) + + arguments.append( + { + "examples": few_shot_examples, + "question": prompt_end, + } + ) + + label = test_df.iloc[i, test_df.shape[1] - 1] + labels.append(label) + + ##################################### + ######### SGL Program Begin ######### + ##################################### + + import sglang as sgl + + if args.backend.startswith("gpt-"): + + @sgl.function + def few_shot_mmlu(s, examples, question): + s += sgl.user(examples + question) + s += sgl.assistant(sgl.gen("answer")) + + else: + + @sgl.function + def few_shot_mmlu(s, examples, question): + s += examples + question + sgl.gen("answer") + + ##################################### + ########## SGL Program End ########## + ##################################### + + # Select backend + backend = select_sglang_backend(args) + + # Run + tic = time.perf_counter() + states = few_shot_mmlu.run_batch( + arguments, + temperature=0, + max_new_tokens=1, + backend=backend, + num_threads=args.parallel, + progress_bar=True, + ) + preds = [ + s["answer"].strip()[0] if len(s["answer"].strip()) > 0 else "" for s in states + ] + latency = time.perf_counter() - tic + + # Compute accuracy + cors = [pred == label for pred, label in zip(preds, labels)] + + pt = 0 + for subject, num_qs in zip(subjects[: args.nsub], num_questions): + print( + f"subject: {subject}, #q:{num_qs}, acc: {np.mean(cors[pt: pt + num_qs]):.3f}" + ) + pt += num_qs + assert pt == len(cors) + weighted_acc = np.mean(cors) + + dump_bench_raw_result( + path=args.raw_result_file, + states=states, + preds=preds, + labels=labels, + ) + + # Print results + print("Total latency: {:.3f}".format(latency)) + print("Average accuracy: {:.3f}".format(weighted_acc)) + + # Write results + with open(args.result_file, "a") as fout: + value = { + "task": "mmlu", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(weighted_acc, 3), + "num_requests": len(arguments), + "other": { + "nsub": args.nsub, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--ntrain", "-k", type=int, default=5) + parser.add_argument("--data_dir", "-d", type=str, default="data") + parser.add_argument("--save_dir", "-s", type=str, default="results") + parser.add_argument("--nsub", type=int, default=60) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmlu/download_data.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmlu/download_data.sh new file mode 100644 index 00000000..fb224165 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmlu/download_data.sh @@ -0,0 +1,2 @@ +wget https://people.eecs.berkeley.edu/~hendrycks/data.tar +tar xf data.tar diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/README.md new file mode 100644 index 00000000..61fea8bc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/README.md @@ -0,0 +1,49 @@ +## Run evaluation + +### Evaluate sglang + +Host the VLM: + +``` +python -m sglang.launch_server --model-path Qwen/Qwen2-VL-7B-Instruct --port 30000 +``` + +It's recommended to reduce the memory usage by appending something like `--mem-fraction-static 0.6` to the command above. + +Benchmark: + +``` +python benchmark/mmmu/bench_sglang.py --port 30000 --concurrency 16 +``` + +You can adjust the `--concurrency` to control the number of concurrent OpenAI calls. + +You can use `--lora-path` to specify the LoRA adapter to apply during benchmarking. E.g., +``` +# Launch server with LoRA enabled +python -m sglang.launch_server --model-path microsoft/Phi-4-multimodal-instruct --port 30000 --trust-remote-code --disable-radix-cache --lora-paths vision= + +# Apply LoRA adapter during inferencing +python -m benchmark/mmmu/bench_sglang.py --concurrency 8 --lora-path vision +``` + +You can use `--response-answer-regex` to specify how to extract the answer from the response string. E.g., +``` +python3 -m sglang.launch_server --model-path zai-org/GLM-4.1V-9B-Thinking --reasoning-parser glm45 + +python3 bench_sglang.py --response-answer-regex "<\|begin_of_box\|>(.*)<\|end_of_box\|>" --concurrency 64 +``` + +You can use `--extra-request-body` to specify additional OpenAI request parameters. E.g., +``` +python3 bench_sglang.py --extra-request-body '{"max_new_tokens": 128, "temperature": 0.01}' +``` + +### Evaluate HF + +``` +python benchmark/mmmu/bench_hf.py --model-path Qwen/Qwen2-VL-7B-Instruct +``` + +# Profiling MMMU +You should use the standard instructions found in the [dedicated profiling doc](../../docs/developer_guide/benchmark_and_profiling.md) if running this benchmark with the profile option. We recommend using `--concurrency 1` for consistency, which makes profiling and debugging easier. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/bench_hf.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/bench_hf.py new file mode 100644 index 00000000..c841f444 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/bench_hf.py @@ -0,0 +1,169 @@ +import argparse + +import PIL +import torch +from data_utils import save_json +from eval_utils import ( + EvalArgs, + eval_result, + get_sampling_params, + prepare_samples, + process_result, +) +from tqdm import tqdm +from transformers import AutoModel, AutoProcessor, GenerationConfig + + +@torch.no_grad() +def eval_mmmu(args): + eval_args = EvalArgs.from_cli_args(args) + + sampling_params = get_sampling_params(eval_args) + generation_config = GenerationConfig( + max_new_tokens=sampling_params["max_new_tokens"], + do_sample=False, + ) + + try: + from transformers import AutoModelForImageTextToText + + model = AutoModelForImageTextToText.from_pretrained( + args.model_path, + torch_dtype="auto", + trust_remote_code=True, + ) + except Exception as first_exception: + try: + # check if the model is belongs to internvl + if "InternVL" in args.model_path: + from transformers import AutoTokenizer + + from sglang.srt.multimodal.internvl_utils import image_to_pixel_values + + tokenizer = AutoTokenizer.from_pretrained(args.model_path) + model = AutoModel.from_pretrained( + args.model_path, + torch_dtype="auto", + trust_remote_code=True, + ) + generation_config_internvl = dict( + max_new_tokens=sampling_params["max_new_tokens"], do_sample=False + ) + + else: + model = AutoModel.from_pretrained( + args.model_path, + torch_dtype="auto", + trust_remote_code=True, + init_tts=False, + ) + except Exception as second_exception: + raise RuntimeError( + f"Failed to load model: First attempt failed with {first_exception}, " + f"second attempt failed with {second_exception}" + ) from second_exception + + model = model.eval().cuda() + + processor = AutoProcessor.from_pretrained( + args.model_path, torch_dtype="auto", device_map="auto", trust_remote_code=True + ) + + samples = prepare_samples(eval_args) + out_samples = dict() + + answer_dict = {} + for sample in tqdm(samples): + prompt = sample["final_input_prompt"] + image = sample["image"] + prefix = prompt.split("<")[0] + suffix = prompt.split(">")[1] + assert image is not None + + if "InternVL" in args.model_path: + image = PIL.Image.open(sample["image_path"]).convert("RGB") + pixel_values = image_to_pixel_values( + image, input_size=448, max_num=12, use_thumbnail=True + ) + pixel_values = pixel_values.to(device="cuda", dtype=torch.bfloat16) + contents = "" + if prefix: + contents += prefix + contents += "\n" + if suffix: + contents += suffix + response = model.chat( + tokenizer, pixel_values, contents, generation_config_internvl + ) + print(f"response: {response}") + process_result(response, sample, answer_dict, out_samples) + continue + + contents = [] + if prefix: + contents += [{"type": "text", "text": prefix}] + contents += [ + { + "type": "image", + "image": sample["image_path"], + } + ] + if suffix: + contents += [{"type": "text", "text": suffix}] + messages = [{"role": "user", "content": contents}] + try: + model_inputs = processor.tokenizer.apply_chat_template( + messages, + tokenize=True, + return_dict=True, + add_generation_prompt=True, + return_tensors="pt", + ).to(model.device) + input_len = model_inputs["input_ids"].shape[-1] + generation = model.generate( + **model_inputs, generation_config=generation_config + ) + generation = generation[0][input_len:] + response = processor.decode(generation, skip_special_tokens=True) + except: + contents = [] + if prefix: + contents += [prefix] + image = PIL.Image.open(sample["image_path"]) + contents += [image] + if suffix: + contents += [suffix] + messages = [{"role": "user", "content": contents}] + response = model.chat( + msgs=messages, + tokenizer=processor.tokenizer, + sampling=False, + max_new_tokens=sampling_params["max_new_tokens"], + use_tts_template=False, + generate_audio=False, + temperature=0.0, + ) + print(f"response: {response}") + process_result(response, sample, answer_dict, out_samples) + + args.output_path = f"{args.model_path}_answer_hf.json" + save_json(args.output_path, out_samples) + eval_result( + model_answer_path=args.output_path, + answer_dict=answer_dict, + eval_output_path=f"{args.model_path}_val_hf.json", + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--model-path", + type=str, + help="The path of the model weights. This can be a local folder or a Hugging Face repo ID.", + required=True, + ) + EvalArgs.add_cli_args(parser) + args = parser.parse_args() + + eval_mmmu(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/bench_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/bench_sglang.py new file mode 100644 index 00000000..9a0bf452 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/bench_sglang.py @@ -0,0 +1,216 @@ +""" +Bench the sglang-hosted vLM with benchmark MMMU + +Usage: + Host the VLM: python -m sglang.launch_server --model-path Qwen/Qwen2-VL-7B-Instruct --port 30000 + + Benchmark: python benchmark/mmmu/bench_sglang.py --port 30000 --concurrency 16 + +The eval output will be logged +""" + +import argparse +import asyncio +import re +import sys +import time +import traceback +from dataclasses import dataclass, field +from typing import Any, List, Optional, Tuple + +import aiohttp +import openai +from data_utils import save_json +from eval_utils import ( + EvalArgs, + eval_result, + get_sampling_params, + prepare_samples, + process_result, +) +from tqdm import tqdm + +from sglang.test.test_utils import add_common_sglang_args_and_parse + +AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=20 * 60 * 60) + + +@dataclass +class RequestFuncOutput: + generated_text: List[str] = field(default_factory=list) + prompt_len: List[int] = field(default_factory=list) + output_len: List[int] = field(default_factory=list) + latency: List[float] = field(default_factory=list) + ttft: List[float] = field(default_factory=list) + itl: List[float] = field(default_factory=list) # List of inter-token latencies + + success: bool = False + error: str = "" + + +async def async_request_profile(api_url: str) -> RequestFuncOutput: + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session: + output = RequestFuncOutput() + try: + async with session.post(url=api_url) as response: + if response.status == 200: + output.success = True + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + return output + + +def _get_prefix_suffix(prompt: str) -> Tuple[str, str]: + """Split the prompt into prefix and suffix.""" + prefix = prompt.split("<")[0] + suffix = prompt.split(">", 1)[1] + return prefix, suffix + + +async def process_sample( + client: Any, sample: dict, sampling_params: dict, lora_path: Optional[str] = None +) -> Tuple[dict, str]: + """Send a single sample to the LLM and return (sample, response).""" + prompt = sample["final_input_prompt"] + prefix, suffix = _get_prefix_suffix(prompt) + image = sample["image"] + assert image is not None + image_path = sample["image_path"] + extra_body = None if lora_path is None else {"lora_path": lora_path} + response = await client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": prefix}, + {"type": "image_url", "image_url": {"url": image_path}}, + {"type": "text", "text": suffix}, + ], + } + ], + temperature=0, + max_completion_tokens=sampling_params["max_new_tokens"], + max_tokens=sampling_params["max_new_tokens"], + extra_body=extra_body, + ) + return sample, response.choices[0].message.content + + +async def process_sample_with_semaphore( + semaphore: asyncio.Semaphore, + client: Any, + sample: dict, + sampling_params: dict, + lora_path: Optional[str] = None, +) -> Tuple[dict, str]: + """Wrap process_sample with a semaphore for concurrency control.""" + async with semaphore: + return await process_sample(client, sample, sampling_params, lora_path) + + +async def eval_mmmu(args) -> None: + """Main evaluation loop with concurrency control.""" + eval_args = EvalArgs.from_cli_args(args) + sampling_params = get_sampling_params(eval_args) + samples = prepare_samples(eval_args) + lora_path = eval_args.lora_path + answer_dict = {} + out_samples = {} + client = openai.AsyncOpenAI( + api_key="sk", + base_url=f"http://127.0.0.1:{args.port}/v1", + timeout=20 * 60 * 60, + ) + start = time.perf_counter() + base_url = f"http://127.0.0.1:{args.port}" + + if args.profile: + print("Starting profiler...") + profile_output = await async_request_profile( + api_url=f"{base_url}/start_profile" + ) + if profile_output.success: + print("Profiler started") + + samples = samples[: args.profile_number] + + if args.concurrency == 1: + # For concurrency == 1, run in sequential mode to ensure consistent order + # this is mainly for profiling + for sample in tqdm(samples): + _, response = await process_sample( + client, sample, sampling_params, lora_path + ) + sample["original_response"] = response + answer = ( + re.search(args.response_answer_regex, response) + if response is not None + else None + ) + process_result( + answer.group(1).strip() if answer else response, + sample, + answer_dict, + out_samples, + ) + else: + semaphore = asyncio.Semaphore(args.concurrency) + tasks = [ + process_sample_with_semaphore( + semaphore, client, sample, sampling_params, lora_path + ) + for sample in samples + ] + + for coro in tqdm(asyncio.as_completed(tasks), total=len(tasks)): + sample, response = await coro + sample["original_response"] = response + answer = ( + re.search(args.response_answer_regex, response) + if response is not None + else None + ) + process_result( + answer.group(1).strip() if answer else response, + sample, + answer_dict, + out_samples, + ) + + if args.profile: + print("Stopping profiler...") + profile_output = await async_request_profile(api_url=f"{base_url}/stop_profile") + if profile_output.success: + print("Profiler stopped") + + print(f"Benchmark time: {time.perf_counter() - start}") + args.output_path = "./answer_sglang.json" + save_json(args.output_path, out_samples) + eval_result( + model_answer_path=args.output_path, + answer_dict=answer_dict, + eval_output_path="./val_sglang.json", + ) + + +def parse_args(): + parser = argparse.ArgumentParser() + EvalArgs.add_cli_args(parser) + args = add_common_sglang_args_and_parse(parser) + return args + + +def main(): + args = parse_args() + asyncio.run(eval_mmmu(args)) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/data_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/data_utils.py new file mode 100644 index 00000000..8c36768d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/data_utils.py @@ -0,0 +1,215 @@ +"""Utils for data load, save, and process (e.g., prompt construction)""" + +import json +import os +import re + +import yaml + +DOMAIN_CAT2SUB_CAT = { + "Art and Design": ["Art", "Art_Theory", "Design", "Music"], + "Business": ["Accounting", "Economics", "Finance", "Manage", "Marketing"], + "Science": [ + "Biology", + "Chemistry", + "Geography", + "Math", + "Physics", + ], + "Health and Medicine": [ + "Basic_Medical_Science", + "Clinical_Medicine", + "Diagnostics_and_Laboratory_Medicine", + "Pharmacy", + "Public_Health", + ], + "Humanities and Social Science": [ + "History", + "Literature", + "Sociology", + "Psychology", + ], + "Tech and Engineering": [ + "Agriculture", + "Architecture_and_Engineering", + "Computer_Science", + "Electronics", + "Energy_and_Power", + "Materials", + "Mechanical_Engineering", + ], +} + + +CAT_SHORT2LONG = { + "acc": "Accounting", + "agri": "Agriculture", + "arch": "Architecture_and_Engineering", + "art": "Art", + "art_theory": "Art_Theory", + "bas_med": "Basic_Medical_Science", + "bio": "Biology", + "chem": "Chemistry", + "cli_med": "Clinical_Medicine", + "cs": "Computer_Science", + "design": "Design", + "diag_med": "Diagnostics_and_Laboratory_Medicine", + "econ": "Economics", + "elec": "Electronics", + "ep": "Energy_and_Power", + "fin": "Finance", + "geo": "Geography", + "his": "History", + "liter": "Literature", + "manage": "Manage", + "mark": "Marketing", + "mate": "Materials", + "math": "Math", + "mech": "Mechanical_Engineering", + "music": "Music", + "phar": "Pharmacy", + "phys": "Physics", + "psy": "Psychology", + "pub_health": "Public_Health", + "socio": "Sociology", +} + + +def get_multi_choice_info(options): + """ + Given the list of options for multiple choice question + Return the index2ans and all_choices + """ + + start_chr = "A" + all_choices = [] + index2ans = {} + for i, option in enumerate(options): + index2ans[chr(ord(start_chr) + i)] = option + all_choices.append(chr(ord(start_chr) + i)) + + return index2ans, all_choices + + +def load_yaml(file_path): + with open(file_path, "r") as stream: + try: + yaml_dict = yaml.safe_load(stream) + except yaml.YAMLError as exc: + print(exc) + + return yaml_dict + + +def parse_img_path(text): + matches = re.findall("", text) + return matches + + +def process_single_sample(data): + question = data["question"] + o_imgs_paths = [] + for option in data["options"]: + current_o_imgs_paths = parse_img_path(option) + for img_path in current_o_imgs_paths: + o_imgs_paths.append(img_path) + + if len(o_imgs_paths) > 1: # multiple images in options, used for random selection + return { + "id": data["id"], + "question": question, + "options": data["options"], + "answer": data["answer"], + "image": None, + "question_type": data["question_type"], + } + else: + return { + "id": data["id"], + "question": question, + "options": data["options"], + "answer": data["answer"], + "image": data["image_1"], + "question_type": data["question_type"], + } + + +# DATA SAVING +def save_json(filename, ds): + print(f"answers saved to: {filename}") + os.makedirs(os.path.dirname(filename), exist_ok=True) + with open(filename, "w") as f: + json.dump(ds, f, indent=4) + + +def save_jsonl(filename, data): + """ + Save a dictionary of data to a JSON Lines file with the filename as key and caption as value. + + Args: + filename (str): The path to the file where the data should be saved. + data (dict): The dictionary containing the data to save where key is the image path and value is the caption. + """ + with open(filename, "w", encoding="utf-8") as f: + for img_path, caption in data.items(): + # Extract the base filename without the extension + base_filename = os.path.basename(img_path) + # Create a JSON object with the filename as the key and caption as the value + json_record = json.dumps({base_filename: caption}, ensure_ascii=False) + # Write the JSON object to the file, one per line + f.write(json_record + "\n") + + +def save_args(args, path_dir): + argsDict = args.__dict__ + with open(path_dir + "setting.txt", "w") as f: + f.writelines("------------------ start ------------------" + "\n") + for eachArg, value in argsDict.items(): + f.writelines(eachArg + " : " + str(value) + "\n") + f.writelines("------------------- end -------------------") + + +# DATA PROCESSING +def construct_prompt(sample, config): + question = sample["question"] + options = eval(sample["options"]) + example = "" + if sample["question_type"] == "multiple-choice": + start_chr = "A" + prediction_range = [] + index2ans = {} + for option in options: + prediction_range.append(start_chr) + example += f"({start_chr}) {option}\n" + index2ans[start_chr] = option + start_chr = chr(ord(start_chr) + 1) + empty_prompt_sample_structure = config["multi_choice_example_format"] + empty_prompt = empty_prompt_sample_structure.format(question, example) + res_dict = {} + res_dict["index2ans"] = index2ans + res_dict["correct_choice"] = sample["answer"] + res_dict["all_choices"] = prediction_range + res_dict["empty_prompt"] = empty_prompt + if config["task_instructions"]: + res_dict["final_input_prompt"] = ( + config["task_instructions"].strip() + "\n\n" + empty_prompt + ) + else: + res_dict["final_input_prompt"] = empty_prompt + + res_dict["gt_content"] = options[ord(sample["answer"].upper()) - ord("A")] + else: + empty_prompt_sample_structure = config["short_ans_example_format"] + empty_prompt = empty_prompt_sample_structure.format(question) + res_dict = {} + res_dict["empty_prompt"] = empty_prompt + if config["task_instructions"]: + res_dict["final_input_prompt"] = ( + config["task_instructions"].strip() + "\n\n" + empty_prompt + ) + else: + res_dict["final_input_prompt"] = empty_prompt + res_dict["gt_content"] = sample["answer"] + + res_dict.update(sample) + return res_dict diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/eval_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/eval_utils.py new file mode 100644 index 00000000..955a3bfa --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/eval_utils.py @@ -0,0 +1,683 @@ +"""Response Parsing and Evaluation for various models""" + +import argparse +import dataclasses +import json +import os +import pprint +import random +import re +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Dict, Optional + +import numpy as np +import torch +from data_utils import ( + CAT_SHORT2LONG, + DOMAIN_CAT2SUB_CAT, + construct_prompt, + load_yaml, + process_single_sample, + save_json, +) +from datasets import concatenate_datasets, load_dataset +from tqdm import tqdm + + +@dataclasses.dataclass +class EvalArgs: + seed: int = 42 + split: str = "validation" + image_pixels_limit: int = -1 + result_filename: str = f"./val_sglang.json" + prompt_format_file: str = "prompt_format.yaml" + dataset_path: str = "MMMU/MMMU" + extra_request_body: Optional[str] = None + profile: bool = False + profile_number: int = 5 + concurrency: int = 1 + max_new_tokens: int = 30 + response_answer_regex: str = "(.*)" + lora_path: Optional[str] = None + + @staticmethod + def add_cli_args(parser: argparse.ArgumentParser): + parser.add_argument( + "--result-filename", + type=str, + default=EvalArgs.result_filename, + help="The filename to save the evaluation results.", + ) + parser.add_argument( + "--image-pixels-limit", + type=int, + default=EvalArgs.image_pixels_limit, + help="The maximum number of pixels allowed for an image. If an image exceeds this limit, it will be skipped during evaluation.", + ) + parser.add_argument( + "--dataset-path", + type=str, + default=EvalArgs.dataset_path, + help="path to the dataset", + ) + parser.add_argument("--seed", type=int, default=1, help="The random seed.") + parser.add_argument( + "--prompt-format-file", + type=str, + help="The path to the prompt format of mmmu. If not, a default format llava_config.yaml will be used", + ) + parser.add_argument( + "--split", + type=str, + default=EvalArgs.split, + help='Split of the dataset to use for evaluation. Default is "validation".', + ) + parser.add_argument( + "--extra-request-body", + metavar='{"key1": "value1", "key2": "value2"}', + type=str, + default=EvalArgs.extra_request_body, + help="Append given JSON object to the request payload. You can use this to specify" + "additional generate params like sampling params.", + ) + parser.add_argument( + "--profile", action="store_true", help="enable mmmu profile" + ) + parser.add_argument( + "--profile-number", + type=int, + default=EvalArgs.profile_number, + help="Number of samples to profile. If not set, will profile all samples.", + ) + parser.add_argument( + "--concurrency", + type=int, + default=EvalArgs.concurrency, + help="Number of concurrent requests to make during evaluation. Default is 1, which means no concurrency.", + ) + parser.add_argument( + "--max-new-tokens", + type=int, + default=EvalArgs.max_new_tokens, + help="Maximum number of new tokens to generate per sample.", + ) + parser.add_argument( + "--response-answer-regex", + type=str, + default=EvalArgs.response_answer_regex, + help="Specific regex to capture the answer from the response, string", + ) + parser.add_argument( + "--lora-path", + type=str, + default=EvalArgs.lora_path, + help="Specify the LoRA path to use for evaluation. If specified, the value will be specified in the body of every request as `lora-path`.", + ) + + @classmethod + def from_cli_args(cls, args: argparse.Namespace): + attrs = [attr.name for attr in dataclasses.fields(cls)] + return cls(**{attr: getattr(args, attr) for attr in attrs}) + + +def set_seed(seed_value): + """ + Set the seed for PyTorch (both CPU and CUDA), Python, and NumPy for reproducible results. + + :param seed_value: An integer value to be used as the seed. + """ + torch.manual_seed(seed_value) + if torch.cuda.is_available(): + torch.cuda.manual_seed(seed_value) + torch.cuda.manual_seed_all(seed_value) # For multi-GPU setups + random.seed(seed_value) + np.random.seed(seed_value) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + +def prepare_samples(eval_args: EvalArgs): + print("Preparing samples...") + # Build prompts + set_seed(eval_args.seed) + + prompt_format_file = ( + eval_args.prompt_format_file + if eval_args.prompt_format_file is not None + else os.path.join(os.path.dirname(__file__), "prompt_format.yaml") + ) + # load config and process to one value + eval_args.config = load_yaml(prompt_format_file) + for key, value in eval_args.config.items(): + if key != "eval_params" and type(value) == list: + assert len(value) == 1, "key {} has more than one value".format(key) + eval_args.config[key] = value[0] + + # run for each subject in parallel + sub_dataset_list = [] + subjects = list(CAT_SHORT2LONG.values()) # Get a fixed list of subjects + + print(f"Loading datasets for {len(subjects)} subjects...") + with ThreadPoolExecutor() as executor: + # Submit all load_dataset tasks + future_to_subject = { + executor.submit( + load_dataset, eval_args.dataset_path, subject, split=eval_args.split + ): subject + for subject in subjects + } + + # Collect results as they complete + results = {} + for future in tqdm( + as_completed(future_to_subject), + total=len(subjects), + desc="Loading datasets", + ): + subject = future_to_subject[future] + try: + results[subject] = future.result() + except Exception as exc: + print(f"{subject} generated an exception: {exc}") + + # Ensure datasets are added in the original order for consistency + for subject in subjects: + if subject in results: + sub_dataset_list.append(results[subject]) + else: + # Handle cases where a dataset failed to load (optional, depends on desired behavior) + print(f"Warning: Dataset for subject '{subject}' could not be loaded.") + + # merge all dataset + dataset = concatenate_datasets(sub_dataset_list) + + # Prepare images in parallel + images_path = os.path.expanduser("~/.cache/mmmu/images") + os.makedirs(images_path, exist_ok=True) + print(f"Saving images to: {images_path}") + + samples = [] + skip_count = 0 + + def process_sample(i, sample): + sample = process_single_sample(sample) + sample = construct_prompt(sample, eval_args.config) + image = sample["image"] + width, height = image.size + if 0 < eval_args.image_pixels_limit <= width * height: + return None, True + # Use a unique identifier for the image path to avoid potential collisions if indices reset + image_path = f"{images_path}/image_{sample['id']}.png" + if not os.path.exists(image_path): + image.save(image_path) + sample["image_path"] = image_path + return sample, False + + print("Processing samples...") + with ThreadPoolExecutor() as executor: + # Pass the sample itself to process_sample, index is less reliable now + futures = [ + executor.submit( + process_sample, i, sample + ) # Keep index i for tqdm maybe? Or remove it. Let's keep it for now. + for i, sample in enumerate(dataset) + ] + for future in tqdm( + as_completed(futures), total=len(dataset), desc="Processing samples" + ): + sample, skipped = future.result() + if skipped: + skip_count += 1 + elif sample: + samples.append(sample) + + samples.sort(key=lambda x: x["final_input_prompt"]) + + print( + f"Skipping {skip_count} samples with large images, {round((float(skip_count) / len(dataset)) * 100, 2)}% of dataset" + ) + print("Samples have been prepared") + return samples + + +def get_sampling_params(eval_args): + max_new_tokens = eval_args.max_new_tokens + temperature = 0.001 + + extra_request_body = {} + if eval_args.extra_request_body: + extra_request_body = json.loads(eval_args.extra_request_body) + + return { + "temperature": temperature, + "max_new_tokens": max_new_tokens, + **extra_request_body, + } + + +# ----------- Process Multi-choice ------------- +def parse_multi_choice_response(response, all_choices, index2ans): + """ + Parse the prediction from the generated response. + Return the predicted index e.g., A, B, C, D. + """ + for char in [",", ".", "!", "?", ";", ":", "'"]: + response = response.strip(char) + response = " " + response + " " # add space to avoid partial match + + index_ans = True + ans_with_brack = False + candidates = [] + for choice in all_choices: # e.g., (A) (B) (C) (D) + if f"({choice})" in response: + candidates.append(choice) + ans_with_brack = True + + if len(candidates) == 0: + for choice in all_choices: # e.g., A B C D + if f" {choice} " in response: + candidates.append(choice) + + # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example + if len(candidates) == 0 and len(response.split()) > 5: + for index, ans in index2ans.items(): + if ans.lower() in response.lower(): + candidates.append(index) + index_ans = False # it's content ans. + + if len(candidates) == 0: # still not get answer, randomly choose one. + pred_index = random.choice(all_choices) + elif len(candidates) > 1: + start_indexes = [] + if index_ans: + if ans_with_brack: + for can in candidates: + index = response.rfind(f"({can})") + start_indexes.append(index) # -1 will be ignored anyway + # start_indexes = [generated_response.index(f'({can})') for can in candidates] + else: + for can in candidates: + index = response.rfind(f" {can} ") + start_indexes.append(index) + else: + for can in candidates: + index = response.lower().rfind(index2ans[can].lower()) + start_indexes.append(index) + # get the last one + pred_index = candidates[np.argmax(start_indexes)] + else: # if only one candidate, use it. + pred_index = candidates[0] + + return pred_index + + +# ----------- Process Open ------------- +def check_is_number(string): + """ + Check if the given string a number. + """ + try: + float(string.replace(",", "")) + return True + except ValueError: + # check if there's comma inside + return False + + +def normalize_str(string): + """ + Normalize the str to lower case and make them float numbers if possible. + """ + # check if characters in the string + + # if number, numerize it. + string = string.strip() + + is_number = check_is_number(string) + + if is_number: + string = string.replace(",", "") + string = float(string) + # leave 2 decimal + string = round(string, 2) + return [string] + else: # it's likely to be a string + # lower it + string = string.lower() + if len(string) == 1: + return [" " + string, string + " "] # avoid trivial matches + return [string] + + +def extract_numbers(string): + """ + Exact all forms of numbers from a string with regex. + """ + # Pattern for numbers with commas + pattern_commas = r"-?\b\d{1,3}(?:,\d{3})+\b" + # Pattern for scientific notation + pattern_scientific = r"-?\d+(?:\.\d+)?[eE][+-]?\d+" + # Pattern for simple numbers without commas + pattern_simple = r"-?(?:\d+\.\d+|\.\d+|\d+\b)(?![eE][+-]?\d+)(?![,\d])" + + # Extract numbers with commas + numbers_with_commas = re.findall(pattern_commas, string) + # Extract numbers in scientific notation + numbers_scientific = re.findall(pattern_scientific, string) + # Extract simple numbers without commas + numbers_simple = re.findall(pattern_simple, string) + + # Combine all extracted numbers + all_numbers = numbers_with_commas + numbers_scientific + numbers_simple + return all_numbers + + +def parse_open_response(response): + """ + Parse the prediction from the generated response. + Return a list of predicted strings or numbers. + """ + + # content = content.strip("\n").strip(".").strip(" ") + def get_key_subresponses(response): + key_responses = [] + response = response.strip().strip(".").lower() + sub_responses = re.split(r"\.\s(?=[A-Z])|\n", response) + indicators_of_keys = [ + "could be ", + "so ", + "is ", + "thus ", + "therefore ", + "final ", + "answer ", + "result ", + ] + key_responses = [] + for index, resp in enumerate(sub_responses): + # if last one, accept it's an equation (the entire response can be just one sentence with equation) + if index == len(sub_responses) - 1: + indicators_of_keys.extend(["="]) + shortest_key_response = None # the shortest response that may contain the answer (tail part of the response) + for indicator in indicators_of_keys: + if indicator in resp: + if not shortest_key_response: + shortest_key_response = resp.split(indicator)[-1].strip() + else: + if len(resp.split(indicator)[-1].strip()) < len( + shortest_key_response + ): + shortest_key_response = resp.split(indicator)[-1].strip() + # key_responses.append(resp.split(indicator)[1].strip()) + + if shortest_key_response: + # and it's not trivial + if shortest_key_response.strip() not in [ + ":", + ",", + ".", + "!", + "?", + ";", + ":", + "'", + ]: + key_responses.append(shortest_key_response) + if len(key_responses) == 0: # did not found any + return [response] + return key_responses + + # pdb.set_trace() + key_responses = get_key_subresponses(response) + + pred_list = key_responses.copy() # keep the original string response + for resp in key_responses: + pred_list.extend(extract_numbers(resp)) + + tmp_pred_list = [] + for i in range(len(pred_list)): + tmp_pred_list.extend(normalize_str(pred_list[i])) + pred_list = tmp_pred_list + + # remove duplicates + pred_list = list(set(pred_list)) + + return pred_list + + +# ----------- Evaluation ------------- + + +def eval_multi_choice(gold_i, pred_i): + """ + Evaluate a multiple choice instance. + """ + correct = False + # for case like Answer: A, Answer is A, answer is A, answer: A + for _exp in ["Answer:", "Answer is ", "answer is ", "answer: "]: + if _exp in pred_i: + pred_i = pred_i.split(_exp)[1].strip() + break + # for case like (A), (B), (C), (D) ...... + if "(" in pred_i and ")" in pred_i: + try: + pred_i = re.search(r"\(([A-Z])\)", pred_i).group(1) + except: + print(f"Error to extract answer from: {pred_i}") + pass + # only they are exactly the same, we consider it as correct + if isinstance(gold_i, list): + for answer in gold_i: + if answer == pred_i: + correct = True + break + else: # gold_i is a string + if gold_i == pred_i: + correct = True + return correct + + +def eval_open(gold_i, pred_i): + """ + Evaluate an open question instance + """ + correct = False + if isinstance(gold_i, list): + # use float to avoid trivial matches + norm_answers = [] + for answer in gold_i: + norm_answers.extend(normalize_str(answer)) + else: + norm_answers = normalize_str(gold_i) + for pred in pred_i: # pred is already normalized in parse response phase + if isinstance(pred, str): # if it's a string, then find if ans in the pred_i + for norm_ans in norm_answers: + # only see if the string answer in the string pred + if isinstance(norm_ans, str) and norm_ans in pred: + if not correct: + correct = True + break + else: # it's a float number + if pred in norm_answers: + if not correct: + correct = True + break + return correct + + +# ----------- Batch Evaluation ------------- +def evaluate(samples): + """ + Batch evaluation for multiple choice and open questions. + """ + pred_correct = 0 + judge_dict = dict() + for sample in samples: + gold_i = sample["answer"] + pred_i = sample["parsed_pred"] + if sample["question_type"] == "multiple-choice": + correct = eval_multi_choice(gold_i, pred_i) + else: # open question + correct = eval_open(gold_i, pred_i) + + if correct: + judge_dict[sample["id"]] = "Correct" + pred_correct += 1 + else: + # print(f"Wrong! expected {pred_i}, answered with {gold_i}") + judge_dict[sample["id"]] = "Wrong" + + if len(samples) == 0: + return {"acc": 0} + return judge_dict, {"acc": pred_correct / len(samples)} + + +# ----------- Calculate Accuracy ------------- +def calculate_ins_level_acc(results: Dict): + """Calculate the instruction level accuracy for given Subject results""" + acc = 0 + ins_num = 0 + for cat_results in results.values(): + acc += cat_results["acc"] * cat_results["num_example"] + ins_num += cat_results["num_example"] + if ins_num == 0: + return 0 + return acc / ins_num + + +def process_result(response, sample, answer_dict, out_samples): + if response is None: + return + if sample["question_type"] == "multiple-choice": + pred_ans = parse_multi_choice_response( + response, sample["all_choices"], sample["index2ans"] + ) + else: # open question + pred_ans = response + + out_samples[sample["id"]] = { + "pred_ans": pred_ans, + "original_response": sample["original_response"], + "ground_truth": sample["answer"], + "question_type": sample["question_type"], + } + + # set ground truth answer + answer_dict[sample["id"]] = { + "question_type": sample["question_type"], + "ground_truth": sample["answer"], + } + + +def eval_result(model_answer_path, answer_dict, eval_output_path=None): + if eval_output_path is None: + eval_output_path = model_answer_path + print("Evaluating...") + output_dict = json.load(open(model_answer_path)) + # answer_dict = json.load(open(answer_path)) + + # group by category + output_dict_w_cat = {} + for data_id, parsed_pred in output_dict.items(): + if isinstance(parsed_pred, str): + parsed_pred = parsed_pred + elif isinstance(parsed_pred, dict): + parsed_pred = parsed_pred["pred_ans"] + else: + raise ValueError(f"Unknown type of parsed_pred: {type(parsed_pred)}") + category = "_".join(data_id.split("_")[1:-1]) + if category not in output_dict_w_cat: + output_dict_w_cat.update({category: {}}) + output_dict_w_cat[category].update({data_id: parsed_pred}) + + # group by category + answer_dict_w_cat = {} + for data_id, parsed_pred in answer_dict.items(): + category = "_".join(data_id.split("_")[1:-1]) + if category not in answer_dict_w_cat: + answer_dict_w_cat.update({category: {}}) + answer_dict_w_cat[category].update({data_id: parsed_pred}) + + evaluation_result = {} + + for category in CAT_SHORT2LONG.values(): + # print("Evaluating: {}".format(category)) + # get cat_outputs and cat_answers + try: + cat_outputs = output_dict_w_cat[category] + cat_answers = answer_dict_w_cat[category] + except KeyError: + # print("Skipping {} for not found".format(category)) + continue + + exampels_to_eval = [] + for data_id, parsed_pred in cat_outputs.items(): + question_type = cat_answers[data_id]["question_type"] + if question_type != "multiple-choice": + parsed_pred = parse_open_response( + parsed_pred + ) # mainly for type consistency (make it number, etc.) + else: + parsed_pred = parsed_pred + + exampels_to_eval.append( + { + "id": data_id, + "question_type": question_type, + "answer": cat_answers[data_id]["ground_truth"], + "parsed_pred": parsed_pred, + } + ) + + judge_dict, metric_dict = evaluate(exampels_to_eval) + metric_dict.update({"num_example": len(exampels_to_eval)}) + for key, value in judge_dict.items(): + output_dict[key]["judge"] = value + + evaluation_result[category] = metric_dict + + save_json(model_answer_path, output_dict) + printable_results = {} + # pdb.set_trace() + # add domain Subject + for domain, in_domain_cats in DOMAIN_CAT2SUB_CAT.items(): + in_domain_cat_results = {} + for cat_name in in_domain_cats: # use the order in DOMAIN_CAT2SUB_CAT + if cat_name in evaluation_result.keys(): + in_domain_cat_results[cat_name] = evaluation_result[cat_name] + else: + pass + in_domain_ins_acc = calculate_ins_level_acc(in_domain_cat_results) + in_domain_data_num = sum( + [ + cat_results["num_example"] + for cat_results in in_domain_cat_results.values() + ] + ) + printable_results["Overall-" + domain] = { + "num": int(in_domain_data_num), + "acc": round(in_domain_ins_acc, 3), + } + # add sub category + for cat_name, cat_results in in_domain_cat_results.items(): + printable_results[cat_name] = { + "num": int(cat_results["num_example"]), + "acc": round(cat_results["acc"], 3), + } + + # table.append(["-----------------------------", "-----", "----"]) + all_ins_acc = calculate_ins_level_acc(evaluation_result) + overall_acc = round(all_ins_acc, 3) + printable_results["Overall"] = { + "num": sum( + [cat_results["num_example"] for cat_results in evaluation_result.values()] + ), + "acc": overall_acc, + } + pprint.pprint(printable_results) + out = eval_output_path + with open(out, "w", encoding="utf-8") as outfile: + json.dump(printable_results, outfile) + print(f"eval out saved to {out}") + + print(f"Overall accuracy: {overall_acc}") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/prompt_format.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/prompt_format.yaml new file mode 100644 index 00000000..1a0f7211 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mmmu/prompt_format.yaml @@ -0,0 +1,15 @@ +task_instructions: +- "" +multi_choice_example_format: +- "{} + +{} + +Answer with the option's letter from the given choices directly." + +short_ans_example_format: +- "{} + +Answer the question using a single word or phrase." +temperature: +- 0 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mtbench/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mtbench/README.md new file mode 100644 index 00000000..fc37caee --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mtbench/README.md @@ -0,0 +1,48 @@ +## Download Dataset + +```sh +wget -O question.jsonl https://raw.githubusercontent.com/lm-sys/FastChat/main/fastchat/llm_judge/data/mt_bench/question.jsonl +``` + +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 80 +``` + +### Benchmark sglang EAGLE +``` +python3 -m sglang.launch_server --model meta-llama/Meta-Llama-3-8B-Instruct --speculative-algo EAGLE \ + --speculative-draft-model-path lmsys/sglang-EAGLE-LLaMA3-Instruct-8B --speculative-num-steps 5 \ + --speculative-eagle-topk 8 --speculative-num-draft-tokens 64 --dtype float16 --port 30000 +``` + +``` +python3 bench_sglang_eagle.py --num-questions 80 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-questions 80 --backend vllm +``` + + +### Benchmark lightllm +``` +# A10G +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 16000 --port 22000 +``` + +``` +python3 bench_other.py --num-questions 80 --backend lightllm +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mtbench/bench_other.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mtbench/bench_other.py new file mode 100644 index 00000000..5e579e9a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mtbench/bench_other.py @@ -0,0 +1,111 @@ +import argparse +import json +import os +import time +import uuid +from concurrent.futures import ThreadPoolExecutor + +from fastchat.model import get_conversation_template +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate + + +def load_questions(filename): + questions = [] + with open(filename, "r") as fin: + for line in fin: + obj = json.loads(line) + questions.append(obj) + return questions + + +def write_answers(filename, model_id, questions, answers): + with open(os.path.expanduser(filename), "w") as fout: + for i in range(len(answers)): + ans_json = { + "question_id": questions[i]["question_id"], + "answer_id": uuid.uuid4().hex, + "model_id": model_id, + "choices": { + "index": 0, + "turns": [answers[i][0], answers[i][1]], + }, + "tstamp": time.time(), + } + fout.write(json.dumps(ans_json) + "\n") + + +def main(args): + questions = load_questions(args.question_file) + questions = (questions * 10)[: args.num_questions] + max_tokens = 256 + model_id = "llama-2-chat" + + conv_main = get_conversation_template(model_id) + + # Select backend + call_generate = get_call_generate(args) + + answers = [None] * len(questions) + + def get_answer(i): + conv = conv_main.copy() + cur_answers = [] + for j in range(2): + q = questions[i]["turns"][j] + conv.append_message(conv.roles[0], q) + conv.append_message(conv.roles[1], None) + + prompt = conv.get_prompt() + output = call_generate(prompt, temperature=0, max_tokens=max_tokens).strip() + + cur_answers.append(output) + conv.update_last_message(output) + + answers[i] = cur_answers + + # Run requests + tic = time.perf_counter() + if args.parallel == 1: + for i in tqdm(range(len(questions))): + get_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_answer, list(range(len(questions)))), + total=len(questions), + ) + ) + + latency = time.perf_counter() - tic + + print(f"#questions: {len(questions)}, Latency: {latency:.2f}") + + # Write results + answer_file = args.answer_file or f"tmp_output_{args.backend}.txt" + write_answers(answer_file, model_id, questions, answers) + + with open(args.result_file, "a") as fout: + value = { + "task": "mtbench", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--question-file", type=str, default="question.jsonl") + parser.add_argument("--answer-file", type=str, default=None) + parser.add_argument("--num-questions", type=int, default=80) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mtbench/bench_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mtbench/bench_sglang.py new file mode 100644 index 00000000..0d0545b3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mtbench/bench_sglang.py @@ -0,0 +1,99 @@ +import argparse +import json +import os +import time +import uuid + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) + + +def load_questions(filename): + questions = [] + with open(filename, "r") as fin: + for line in fin: + obj = json.loads(line) + questions.append(obj) + return questions + + +def write_answers(filename, model_id, questions, answers): + with open(os.path.expanduser(filename), "w") as fout: + for i in range(len(answers)): + ans_json = { + "question_id": questions[i]["question_id"], + "answer_id": uuid.uuid4().hex, + "model_id": model_id, + "choices": { + "index": 0, + "turns": [answers[i][0], answers[i][1]], + }, + "tstamp": time.time(), + } + fout.write(json.dumps(ans_json) + "\n") + + +@sgl.function +def answer_mt_bench(s, question_1, question_2): + s += sgl.system() + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1")) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2")) + + +def main(args): + # Construct prompts + questions = load_questions(args.question_file)[: args.num_questions] + arguments = [ + {"question_1": q["turns"][0], "question_2": q["turns"][1]} for q in questions + ] + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.perf_counter() + rets = answer_mt_bench.run_batch( + arguments, + temperature=0, + max_new_tokens=256, + num_threads=args.parallel, + progress_bar=True, + ) + answers = [[s["answer_1"], s["answer_2"]] for s in rets] + latency = time.perf_counter() - tic + + print(f"#questions: {len(questions)}, Latency: {latency:.2f}") + + # Write results + model_id = backend.model_info["model_path"] + answer_file = args.answer_file or f"tmp_output_{args.backend}.txt" + write_answers(answer_file, model_id, questions, answers) + + with open(args.result_file, "a") as fout: + value = { + "task": "mtbench", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--question-file", type=str, default="question.jsonl") + parser.add_argument("--answer-file", type=str, default=None) + parser.add_argument("--num-questions", type=int, default=80) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mtbench/bench_sglang_eagle.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mtbench/bench_sglang_eagle.py new file mode 100644 index 00000000..7ad3dcb1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/mtbench/bench_sglang_eagle.py @@ -0,0 +1,137 @@ +""" +Adapted from https://github.com/chromecast56/sglang/blob/6f145d2eadb93a116134f703358ce76f15381045/benchmark/mtbench/bench_sglang.py + +Benchmark SGLang EAGLE/EAGLE3 Speculative Decoding + +Usage: +python3 benchmark/mtbench/bench_sglang_eagle.py --num-questions 80 --parallel 1 +""" + +import argparse +import json +import os +import time +import uuid + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) + + +def load_questions(filename): + questions = [] + with open(filename, "r") as fin: + for line in fin: + obj = json.loads(line) + questions.append(obj) + return questions + + +def write_answers(filename, model_id, questions, answers): + with open(os.path.expanduser(filename), "w") as fout: + for i in range(len(answers)): + ans_json = { + "question_id": questions[i]["question_id"], + "answer_id": uuid.uuid4().hex, + "model_id": model_id, + "choices": { + "index": 0, + "prompt": [answers[i][0], answers[i][1]], + }, + "tstamp": time.time(), + } + fout.write(json.dumps(ans_json) + "\n") + + +@sgl.function +def answer_mt_bench(s, question_1, question_2): + s += sgl.system( + "You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information." + ) + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1")) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2")) + + +def main(args): + # Construct prompts + questions = load_questions(args.question_file)[: args.num_questions] + arguments = [ + {"question_1": q["prompt"][0], "question_2": q["prompt"][1]} for q in questions + ] + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.perf_counter() + rets = answer_mt_bench.run_batch( + arguments, + temperature=0, + max_new_tokens=2048, + num_threads=args.parallel, + progress_bar=True, + ) + answers = [[s["answer_1"], s["answer_2"]] for s in rets] + + latency = time.perf_counter() - tic + num_output_tokens = sum( + s.get_meta_info("answer_1")["completion_tokens"] + + s.get_meta_info("answer_2")["completion_tokens"] + for s in rets + ) + + # NOTE: acceptance length is just completion_tokens / spec_verify_ct + # {'id': '3bb9c5ead109488d8ed5ee9cbecaec29', 'finish_reason': {'type': 'length', 'length': 256}, 'prompt_tokens': 37, 'spec_verify_ct': 101, 'completion_tokens': 256, 'cached_tokens': 0} + + output_throughput = num_output_tokens / latency + + has_verify = "spec_verify_ct" in rets[0].get_meta_info("answer_1") + if has_verify: + num_verify_tokens = sum( + s.get_meta_info("answer_1")["spec_verify_ct"] + + s.get_meta_info("answer_2")["spec_verify_ct"] + for s in rets + ) + + accept_length = num_output_tokens / num_verify_tokens + else: + accept_length = 1.0 + + print( + f"#questions: {len(questions)}, Throughput: {output_throughput:.2f} token/s, Acceptance length: {accept_length:.2f}" + ) + + # Write results + model_id = backend.model_info["model_path"] + answer_file = args.answer_file or f"tmp_output_{args.backend}.txt" + write_answers(answer_file, model_id, questions, answers) + + with open(args.result_file, "a") as fout: + value = { + "task": "mtbench", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "throughput": round(output_throughput, 3), + "accept_length": round(accept_length, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--question-file", type=str, default="question.jsonl") + parser.add_argument("--answer-file", type=str, default=None) + parser.add_argument("--num-questions", type=int, default=80) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_chain_reasoning/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_chain_reasoning/README.md new file mode 100644 index 00000000..4c9f740f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_chain_reasoning/README.md @@ -0,0 +1,49 @@ +## Download data +``` +wget https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl +``` + +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 --schedule-conservativeness 1.3 +``` + +``` +python3 bench_sglang.py --num-questions 64 +python3 bench_sglang.py --num-questions 32 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-questions 64 --backend vllm +``` + + +### Benchmark lightllm +``` +# A10G +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 16000 --port 22000 +``` + +``` +python3 bench_other.py --num-questions 64 --backend lightllm +``` + + +### Benchmark guidance +``` +python3 bench_other.py --num-questions 8 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +### Benchmark lmql + +``` +python3 bench_other.py --num-questions 64 --backend lmql --parallel 1 +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_chain_reasoning/bench_other.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_chain_reasoning/bench_other.py new file mode 100644 index 00000000..f361496a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_chain_reasoning/bench_other.py @@ -0,0 +1,186 @@ +import argparse +import ast +import asyncio +import json +import re +import time +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +prompt_lib = [ + "Let us think step by step.", + "Approach this methodically. Let's dissect the problem into smaller, more manageable parts.", + "It's important to proceed step by step, ensuring accuracy at each stage.", + "Take a deep breath and break this down.", + "A little bit of arithmetic and a logical approach will help us quickly arrive at the solution to this problem.", + "I am extremely good at math.", +] + + +def multi_chain_gsm8k(question, num_chains, call_generate): + s = "Question: " + question + "\n" + # s += call_generate(s + "Answer: " + prompt_lib[0], max_tokens=256, + # stop="Question", temperature=0) + # return s + + comps = [] + for i in range(num_chains): + comps.append( + call_generate( + s + "Answer: " + prompt_lib[i % num_chains], + max_tokens=256, + temperature=0.3, + stop="Question", + ) + ) + + s += "Answer: To answer this question, here are some possible solutions. " + s += "After considering all of them, I will do a majority vote.\n\n" + for i in range(num_chains): + s += f"Solution {i+1}: " + comps[i].strip() + "\n\n" + s += "\nBy considering the above solutions and doing a majority vote, I think the final answer (a single integer number) is " + s += call_generate(s, max_tokens=16, temperature=0, stop=None) + return s + + +async def multi_chain_gsm8k_async(question, num_chains, call_generate): + s = "Question: " + question + "\n" + # s += call_generate(s + "Answer: " + prompt_lib[0], max_tokens=256, + # stop="Question", temperature=0) + # return s + + comps = [] + for i in range(num_chains): + comps.append( + await call_generate( + s + "Answer: " + prompt_lib[i % num_chains], + max_tokens=256, + temperature=0.3, + stop="Question", + ) + ) + + s += "Answer: To answer this question, here are some possible solutions. " + s += "After considering all of them, I will do a majority vote.\n\n" + for i in range(num_chains): + s += f"Solution {i+1}: " + comps[i].strip() + "\n\n" + s += "\nBy considering the above solutions and doing a majority vote, I think the final answer (a single integer number) is " + s += await call_generate(s, max_tokens=16, temperature=0, stop=None) + return s + + +def main(args): + lines = read_jsonl(args.data_path) + + # Construct prompts + k = args.num_shot + + questions = [] + labels = [] + for i in range(len(lines[: args.num_questions])): + questions.append(lines[i]["question"]) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + + states = [None] * len(labels) + + # Select backend + call_generate = get_call_generate(args) + + # Run requests + if args.backend != "lmql": + # Use thread pool + def get_one_answer(i): + answer = multi_chain_gsm8k(questions[i], args.num_chains, call_generate) + states[i] = answer + + tic = time.perf_counter() + if args.parallel == 1: + for i in tqdm(range(len(questions))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(questions)))), + total=len(questions), + ) + ) + + else: + # Use asyncio + async def get_one_answer_asyncio(i): + answer = await multi_chain_gsm8k_async( + questions[i], args.num_chains, call_generate + ) + states[i] = answer + + tic = time.perf_counter() + loop = asyncio.get_event_loop() + batches = [ + list(range(i, min(i + args.parallel, len(questions)))) + for i in range(0, len(questions), args.parallel) + ] + for bt in tqdm(batches): + tasks = [get_one_answer_asyncio(k) for k in bt] + loop.run_until_complete(asyncio.gather(*tasks)) + + latency = time.perf_counter() - tic + + preds = [] + for i in range(len(states)): + preds.append(get_answer_value(states[i])) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + print(f"Latency: {latency:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "multi_chain_gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--num-shot", type=int, default=0) + parser.add_argument("--num-chains", type=int, default=5) + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=50) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_chain_reasoning/bench_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_chain_reasoning/bench_sglang.py new file mode 100644 index 00000000..1d3129db --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_chain_reasoning/bench_sglang.py @@ -0,0 +1,140 @@ +import argparse +import ast +import json +import re +import time + +import numpy as np + +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +prompt_lib = [ + "Let us think step by step.", + "Approach this methodically. Let's dissect the problem into smaller, more manageable parts.", + "It's important to proceed step by step, ensuring accuracy at each stage.", + "Take a deep breath and break this down.", + "A little bit of arithmetic and a logical approach will help us quickly arrive at the solution to this problem.", + "I am extremely good at math.", +] + + +def main(args): + lines = read_jsonl(args.data_path) + + # Construct prompts + # k = args.num_shot + # few_shot_examples = get_few_shot_examples(lines, k) + + questions = [] + labels = [] + for i in range(len(lines[: args.num_questions])): + questions.append(lines[i]["question"]) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q} for q in questions] + + num_chains = args.num_chains + + ##################################### + ######### SGL Program Begin ######### + ##################################### + + import sglang as sgl + + @sgl.function + def multi_chain_gsm8k(s, question): + s += "Question: " + question + "\n" + # s += "Answer: " + prompt_lib[0] + sgl.gen("answer", max_tokens=256, stop="Question", + # temperature=0) + # return + + forks = s.fork(num_chains) + for i in range(num_chains): + forks[i] += ( + "Answer: " + + prompt_lib[i % num_chains] + + sgl.gen("chain", max_tokens=256, temperature=0.3, stop="Question") + ) + forks.join() + + s += "Answer: To answer this question, here are some possible solutions. " + s += "After considering all of them, I will do a majority vote.\n\n" + for i in range(num_chains): + s += f"Solution {i+1}: " + forks[i]["chain"].strip() + "\n\n" + s += "\nBy considering the above solutions and doing a majority vote, I think the final answer (a single integer number) is " + s += sgl.gen("answer", max_tokens=16) + + ##################################### + ########## SGL Program End ########## + ##################################### + + # Select backend + backend = select_sglang_backend(args) + + # Run requests + tic = time.perf_counter() + states = multi_chain_gsm8k.run_batch( + arguments, + temperature=0, + backend=backend, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + + preds = [] + for i in range(len(states)): + preds.append(get_answer_value(states[i]["answer"])) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + print(f"Latency: {latency:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "multi_chain_gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--num-shot", type=int, default=0) + parser.add_argument("--num-chains", type=int, default=5) + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=50) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_document_qa/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_document_qa/README.md new file mode 100644 index 00000000..09f49c78 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_document_qa/README.md @@ -0,0 +1,47 @@ +## Run benchmark + +### Benchmark sglang +``` +python3 -m sglang.launch_server --model-path codellama/CodeLlama-7b-instruct-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 10 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model codellama/CodeLlama-7b-instruct-hf --disable-log-requests --port 21000 --gpu 0.97 +``` + +``` +python3 bench_other.py --backend vllm --num-questions 64 +``` + + +### Benchmark guidance +``` +python3 bench_other.py --backend guidance --num-questions 32 --parallel 1 --n-ctx 11000 --model-path path/to/code-llama/gguf +``` + + + +### Build dataset + +``` +pip install PyPDF2 +python3 build_dataset.py +``` + +```python +import PyPDF2 + +with open('llama2.pdf', 'rb') as file: + reader = PyPDF2.PdfReader(file) + text = '' + for page_num in range(len(reader.pages)): + text += reader.pages[page_num].extract_text() + with open('output.txt', 'w') as text_file: + text_file.write(text) +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_document_qa/bench_other.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_document_qa/bench_other.py new file mode 100644 index 00000000..627837c5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_document_qa/bench_other.py @@ -0,0 +1,114 @@ +import argparse +import json +import time +from concurrent.futures import ThreadPoolExecutor +from functools import partial + +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +USER_PREFIX = "[INST] " +USER_SUFFIX = " [/INST]" +ASSISTANT_PREFIX = "" +ASSISTANT_SUFFIX = " " + + +def multi_document_qa(docs, question, generate): + s = USER_PREFIX + s += "Please answer a question according to given documents.\n" + s += "Question:" + question + "Documents begin.\n" + + s += "".join(docs) + + s += "\nDocuments end." + s += ( + "\n\nBased on the above documents, please answer this question:\n" + + question + + "\nAnswer in three words or fewer." + ) + s += USER_SUFFIX + s += ASSISTANT_PREFIX + answer = generate(s, max_tokens=16, stop=None) + return answer + + +def main(args): + lines = read_jsonl(args.data_path) + l = lines[0] + arguments = [] + labels = [] + + num_docs = 10 + if args.backend == "guidance": + num_docs = 7 # due to OOM + + for i in range(len(l["questions"][: args.num_questions])): + arguments.append( + { + "docs": l["documents"][:num_docs], + "question": l["questions"][i], + } + ) + labels.append(l["answers"][i]) + states = [None] * len(arguments) + + # Select backend + call_generate = partial(get_call_generate(args), temperature=0) + + # Run requests + def get_one_answer(i): + states[i] = multi_document_qa(generate=call_generate, **arguments[i]) + + tic = time.perf_counter() + if args.parallel == 1: + for i in tqdm(range(len(labels))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(labels)))), + total=len(labels), + ) + ) + + latency = time.perf_counter() - tic + + # Compute accuracy + print(states) + correct = 0 + for s, label in zip(states, labels): + answer = s.lower() + if all(x in answer for x in label.lower().split(" ")): + correct += 1 + accuracy = correct / len(labels) + print(f"Accuracy: {accuracy:.3f}") + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "multi_document_qa", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "accuracy": accuracy, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="questions.jsonl") + parser.add_argument("--num-questions", type=int, default=100) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_document_qa/bench_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_document_qa/bench_sglang.py new file mode 100644 index 00000000..0b4b0dbc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_document_qa/bench_sglang.py @@ -0,0 +1,93 @@ +import argparse +import json +import time + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + + +@sgl.function +def multi_document_qa(s, docs, question): + s += sgl.user_begin() + s += "Please answer a question according to given documents.\n" + s += "Question:" + question + "Documents begin.\n" + + forks = s.fork(len(docs)) + forks += lambda i: docs[i] + forks.join("concate_and_append") + + s += "\nDocuments end." + s += ( + "\n\nBased on the above documents, please answer this question:\n" + + question + + "\nAnswer in three words or fewer." + ) + s += sgl.user_end() + s += sgl.assistant(sgl.gen("answer", max_tokens=16)) + + +def main(args): + lines = read_jsonl(args.data_path) + l = lines[0] + arguments = [] + labels = [] + for i in range(len(l["questions"][: args.num_questions])): + arguments.append( + { + "docs": l["documents"][:10], + "question": l["questions"][i], + } + ) + labels.append(l["answers"][i]) + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.perf_counter() + states = multi_document_qa.run_batch( + arguments, temperature=0, num_threads=args.parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + # Compute accuracy + print([s["answer"] for s in states]) + correct = 0 + for s, label in zip(states, labels): + answer = s["answer"].lower() + if all(x in answer for x in label.lower().split(" ")): + correct += 1 + accuracy = correct / len(labels) + print(f"Accuracy: {accuracy:.3f}") + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "multi_document_qa", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "accuracy": accuracy, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="questions.jsonl") + parser.add_argument("--num-questions", type=int, default=100) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_document_qa/build_dataset.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_document_qa/build_dataset.py new file mode 100644 index 00000000..27df9474 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_document_qa/build_dataset.py @@ -0,0 +1,70 @@ +import json + +import transformers + +content = "\n".join( + open("llama2.txt", "r", encoding="utf-8", errors="ignore").readlines() +) +content = content.replace("\n\n", "\n") + +# Count token +name = "meta-llama/Llama-2-7b-chat-hf" +t = transformers.AutoTokenizer.from_pretrained(name) +print(f"num tokens: {len(t.encode(content))}") + +# Segment +SEP = "\n\n" +parts = content.split(SEP) +print(f"num segments: {len(parts)}") + +segment_len = 1100 + +segments = [] +tmp = [] +tmp_len = 0 +for i in range(len(parts)): + tmp.append(parts[i]) + tmp_len += len(t.encode(parts[i])) + + if tmp_len > segment_len: + segments.append(SEP.join(tmp)) + tmp = [] + tmp_len = 0 + +for i, s in enumerate(segments): + print(i, len(t.encode(segments[i]))) + +# Dump +with open("questions.jsonl", "w") as fout: + fout.write( + json.dumps( + { + "documents": segments[:30], + "questions": [ + "What is the name of the fine-tuned LLMs?", + "Which figure shows the helpfulness human evaluation results for Llama 2-Chat?", + "What is the number of parameters in the largest Llama 2 model?", + "What is the batch size of fine-tuning?", + "Where can we find the details of potential data contamination?", + "What is the full name of MPT?", + "What is the power consumption of RSC in Watt?", + "How many tokens of data do they train on?", + "Which model's release is delayed due to a lack of time to sufficiently red team?", + "Which activation function is used in Llama?", + ], + "answers": [ + "Llama 2 Chat", + "1", + "70 B", + "64", + "A 6", + "MosaicML", + "400", + "2 trillion", + "34 B", + "SwiGLU", + ], + } + ) + + "\n" + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_turn_chat/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_turn_chat/README.md new file mode 100644 index 00000000..0fb5b21f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_turn_chat/README.md @@ -0,0 +1,66 @@ +### Benchmark sglang + +Run Llama-7B + +``` +python3 -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +Run Mixtral-8x7B +(When there is a CUDA out-of-memory error, try to reduce the `--mem-fraction-static`) + +``` +python3 -m sglang.launch_server --model-path mistralai/Mixtral-8x7B-Instruct-v0.1 --port 30000 --tp-size 8 +``` + +Benchmark(short output) + +``` +python3 bench_sglang.py --tokenizer meta-llama/Llama-2-7b-chat-hf +``` + +Benchmark(long output) + +``` +python3 bench_sglang.py --tokenizer meta-llama/Llama-2-7b-chat-hf --long +``` + +### Benchmark vLLM + +Run Llama-7B + +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +Run Mixtral-8x7B + +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model mistralai/Mixtral-8x7B-Instruct-v0.1 --disable-log-requests --port 21000 --tensor-parallel-size 8 +``` + +Benchmark(short output) + +``` +python3 bench_other.py --tokenizer meta-llama/Llama-2-7b-chat-hf --backend vllm +``` + +Benchmark(long output) + +``` +python3 bench_other.py --tokenizer meta-llama/Llama-2-7b-chat-hf --backend vllm --long +``` + +### Benchmark guidance + +Benchmark Llama-7B (short output) + +``` +python3 bench_other.py --tokenizer meta-llama/Llama-2-7b-chat-hf --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +Benchmark Llama-7B (long output) + +``` +python3 bench_other.py --tokenizer meta-llama/Llama-2-7b-chat-hf --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf --long +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_turn_chat/bench_other.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_turn_chat/bench_other.py new file mode 100644 index 00000000..9189af5b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_turn_chat/bench_other.py @@ -0,0 +1,93 @@ +import json +import time +from argparse import ArgumentParser +from concurrent.futures import ThreadPoolExecutor +from functools import partial + +from data_gen import gen_arguments +from tqdm import tqdm +from vllm.transformers_utils.tokenizer import get_tokenizer + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text + + +def multi_turns(generate, qas): + s = "" + for qa in qas: + s += qa["prompt"] + s += generate(s, max_tokens=qa["new_tokens"]) + + return s + + +def main(args): + print(args) + + tokenizer = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code) + + multi_qas = gen_arguments(args, tokenizer) + + states = [None] * args.num_qa + + call_generate = partial(get_call_generate(args), temperature=0) + + def get_one_answer(i): + states[i] = multi_turns(generate=call_generate, **multi_qas[i]) + + tic = time.perf_counter() + if args.parallel == 1: + for i in tqdm(range(len(multi_qas))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + rets = list( + tqdm( + executor.map(get_one_answer, list(range(len(multi_qas)))), + total=len(multi_qas), + ) + ) + for _ in rets: + pass + + latency = time.perf_counter() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "multi_turn_chat", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_qa, + "num_turns": args.turns, + "other": { + "parallel": args.parallel, + "output_mode": "long" if args.long else "short", + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument("--turns", type=int, default=4) + parser.add_argument("--num-qa", type=int, default=20) + parser.add_argument("--min-len-q", type=int, default=256) + parser.add_argument("--max-len-q", type=int, default=512) + parser.add_argument("--min-len-a", type=int, default=4) + parser.add_argument("--max-len-a", type=int, default=8) + parser.add_argument("--tokenizer", type=str, required=True) + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--long", action="store_true") + args = add_common_other_args_and_parse(parser) + + if args.long: + args.min_len_a = 256 + args.max_len_a = 512 + args.num_qa = 20 + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_turn_chat/bench_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_turn_chat/bench_sglang.py new file mode 100644 index 00000000..1051bf19 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_turn_chat/bench_sglang.py @@ -0,0 +1,79 @@ +import json +import time +from argparse import ArgumentParser + +from data_gen import gen_arguments +from vllm.transformers_utils.tokenizer import get_tokenizer + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text + + +@sgl.function +def multi_turns(s, qas): + for qa in qas: + s += qa["prompt"] + s += sgl.gen(max_tokens=qa["new_tokens"], ignore_eos=True) + + +def main(args): + tokenizer = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code) + + multi_qas = gen_arguments(args, tokenizer) + + backend = select_sglang_backend(args) + + tic = time.perf_counter() + states = multi_turns.run_batch( + multi_qas, + temperature=0, + backend=backend, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + + print(f"Latency: {latency:.3f}") + + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "multi_turn_chat", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_qa, + "num_turns": args.turns, + "other": { + "parallel": args.parallel, + "output_mode": "long" if args.long else "short", + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument("--turns", type=int, default=4) + parser.add_argument("--num-qa", type=int, default=20) + parser.add_argument("--min-len-q", type=int, default=256) + parser.add_argument("--max-len-q", type=int, default=512) + parser.add_argument("--min-len-a", type=int, default=4) + parser.add_argument("--max-len-a", type=int, default=8) + parser.add_argument("--tokenizer", type=str, required=True) + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--long", action="store_true") + args = add_common_sglang_args_and_parse(parser) + + if args.long: + args.min_len_a = 256 + args.max_len_a = 512 + args.num_qa = 20 + + print(args) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_turn_chat/data_gen.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_turn_chat/data_gen.py new file mode 100644 index 00000000..043c07a7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_turn_chat/data_gen.py @@ -0,0 +1,29 @@ +import random +import string + +random.seed(42) + + +def gen_prompt(tokenizer, token_num): + cha_set = string.ascii_letters + string.digits + ret = "".join(random.choices(cha_set, k=token_num)) + while len(tokenizer(ret).input_ids) < token_num: + ret += random.choice(cha_set) + return ret + + +def gen_arguments(args, tokenizer): + multi_qas = [{"qas": []} for _ in range(args.num_qa)] + for i in range(args.num_qa): + qas = multi_qas[i]["qas"] + for _ in range(args.turns): + prompt_len = random.randint(args.min_len_q, args.max_len_q) + new_tokens = random.randint(args.min_len_a, args.max_len_a) + qas.append( + { + "prompt": gen_prompt(tokenizer, prompt_len), + "new_tokens": new_tokens, + } + ) + + return multi_qas diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_turn_chat/long_prompt_multi_turn.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_turn_chat/long_prompt_multi_turn.py new file mode 100644 index 00000000..88eba70c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/multi_turn_chat/long_prompt_multi_turn.py @@ -0,0 +1,129 @@ +import json +import random +import time +from argparse import ArgumentParser +from pathlib import Path + +from tqdm import tqdm + +import sglang as sgl +from sglang.srt.utils.hf_transformers_utils import get_tokenizer +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text + + +def gen_prompt(tokenizer, token_num): + all_available_tokens = list(tokenizer.get_vocab().values()) + selected_tokens = random.choices(all_available_tokens, k=token_num) + ret = tokenizer.decode(selected_tokens) + return ret + + +def get_cache_path(args): + # Create cache directory under ~/.cache/sglang + cache_dir = Path.home() / ".cache" / "sglang" + + # Create a unique cache filename based on the arguments that affect generation + cache_key = f"qa_{args.num_qa}_{args.turns}_{args.system_prompt_len}_{args.len_q}_{args.len_a}_{args.tokenizer.replace('/', '_')}.json" + return cache_dir / cache_key + + +def gen_arguments(args, tokenizer): + cache_path = get_cache_path(args) + + # Try to load from cache first + if cache_path.exists(): + print(f"Loading cached arguments from {cache_path}") + with open(cache_path, "r") as f: + return json.load(f) + + print("Generating new arguments...") + # First progress bar for system prompts + multi_qas = [] + for _ in tqdm(range(args.num_qa), desc="Generating system prompts"): + multi_qas.append( + {"system_prompt": gen_prompt(tokenizer, args.system_prompt_len), "qas": []} + ) + + # Nested progress bars for QA pairs + for i in tqdm(range(args.num_qa), desc="Generating QA pairs"): + qas = multi_qas[i]["qas"] + for j in range(args.turns): + qas.append( + { + "prompt": gen_prompt(tokenizer, args.len_q), + "new_tokens": args.len_a, + } + ) + + # Save to cache + cache_path.parent.mkdir(parents=True, exist_ok=True) + with open(cache_path, "w") as f: + json.dump(multi_qas, f) + print(f"Cached arguments saved to {cache_path}") + + return multi_qas + + +@sgl.function +def multi_turns(s, system_prompt, qas): + s += system_prompt + + for i, qa in enumerate(qas): + s += qa["prompt"] + s += sgl.gen(max_tokens=qa["new_tokens"], ignore_eos=True) + + +def main(args): + tokenizer = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code) + + multi_qas = gen_arguments(args, tokenizer) + + backend = select_sglang_backend(args) + + tic = time.perf_counter() + states = multi_turns.run_batch( + multi_qas, + temperature=0, + backend=backend, + num_threads="auto", + progress_bar=True, + ) + latency = time.perf_counter() - tic + + print(f"Latency: {latency:.3f}") + + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "multi_turn_system_prompt_chat", + "backend": args.backend, + "latency": round(latency, 3), + "num_requests": args.num_qa, + "num_turns": args.turns, + "other": { + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument("--turns", type=int, default=8) + parser.add_argument("--num-qa", type=int, default=128) + parser.add_argument("--system-prompt-len", type=int, default=2048) + parser.add_argument("--len-q", type=int, default=32) + parser.add_argument("--len-a", type=int, default=128) + parser.add_argument( + "--tokenizer", type=str, default="meta-llama/Meta-Llama-3-8B-Instruct" + ) + parser.add_argument("--trust-remote-code", action="store_true") + args = add_common_sglang_args_and_parse(parser) + + print(args) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/prefill_only/bench_embeddings.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/prefill_only/bench_embeddings.py new file mode 100644 index 00000000..74d8a582 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/prefill_only/bench_embeddings.py @@ -0,0 +1,159 @@ +""" +SGLang Embeddings Benchmark Script + +This script benchmarks SGLang's /v1/embeddings API performance using HTTP requests. + +Features: +- HTTP-only implementation +- Uses /v1/embeddings API endpoint directly +- Configurable RPS, duration, and batch sizes +- Progress tracking and detailed metrics +- Poisson and constant request distributions + +Usage: +- Update configuration variables at the top of the file +- Ensure SGLang server is running on the configured HTTP_URL +- Run: python bench_embeddings.py +""" + +import asyncio +import logging +from typing import Optional + +from transformers import AutoTokenizer +from util import ( + BenchmarkConfig, + generate_text_with_token_count, + run_benchmark_main, + run_generic_benchmark, +) + +# Configure logging +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger(__name__) + +############################################################################### +# CONFIG +############################################################################### +# Create benchmark configuration +config = BenchmarkConfig() +config.rps_values = [500] +config.duration_secs_values = [60] +config.num_unique_requests = 100 +config.distribution = "POISSON" +config.profile = False +config.freeze_gc = True # Enable GC freeze functionality +# Profiler output directory - by default uses present working directory (pwd) +# Uncomment and customize the line below to override the default location: +# config.profiler_dir = "/sglang-oss-trace" + +# HTTP Configuration +HTTP_URL = "http://localhost:30000/v1/embeddings" + +# Embeddings API Config +EMBEDDINGS_MODEL_PATH = "Qwen/Qwen3-Embedding-0.6B" +BATCH_SIZE = [1] # Number of items per request (batch size) + +# Configurable input token length +EMBEDDINGS_INPUT_TOKENS = 500 # Default token length +MATRYOSHKA_DIMENSIONS: Optional[int] = ( + None # Set to None to disable matryoshka embeddings +) + +# Load tokenizer once for embeddings text generation +print("Loading tokenizer for embeddings input generation...") +embeddings_tokenizer = AutoTokenizer.from_pretrained(EMBEDDINGS_MODEL_PATH) + +# Generate input text with the specified token length using pre-loaded tokenizer +EMBEDDINGS_INPUT_TEXT = generate_text_with_token_count( + EMBEDDINGS_MODEL_PATH, + EMBEDDINGS_INPUT_TOKENS, + config.special_replicated_token, + tokenizer=embeddings_tokenizer, +) + + +############################################################################### +# REQUEST GENERATION (in parallel) +############################################################################### +def build_embeddings_request(index: int, item_count: int) -> tuple: + """Build a single embeddings request.""" + try: + # For embeddings, input can be a string or list of strings + if item_count == 1: + input_data = EMBEDDINGS_INPUT_TEXT + else: + input_data = [EMBEDDINGS_INPUT_TEXT for _ in range(item_count)] + req = { + "input": input_data, + "model": EMBEDDINGS_MODEL_PATH, + "dimensions": MATRYOSHKA_DIMENSIONS, + } + return (index, req) + except Exception as e: + logger.error(f"Error building request {index}: {e}") + return (index, None) + + +def validate_embeddings_response(response_data: dict) -> bool: + """Validate embeddings API response.""" + return ( + "data" in response_data + and len(response_data["data"][0]["embedding"]) == MATRYOSHKA_DIMENSIONS + if MATRYOSHKA_DIMENSIONS + else True + ) + + +def build_warmup_embeddings_request() -> dict: + """Build a warmup request for the embeddings API.""" + return { + "input": EMBEDDINGS_INPUT_TEXT, + "model": EMBEDDINGS_MODEL_PATH, + "dimensions": MATRYOSHKA_DIMENSIONS, + } + + +############################################################################### +# MAIN +############################################################################### +async def run_benchmark(rps, duration_secs, item_count): + """Run a single embeddings benchmark with the given RPS value.""" + return await run_generic_benchmark( + rps=rps, + duration_secs=duration_secs, + item_count=item_count, + config=config, + http_url=HTTP_URL, + build_request_func=build_embeddings_request, + response_validator=validate_embeddings_response, + api_name="EMBEDDINGS", + request_description="embeddings requests", + ) + + +async def main(): + additional_info = { + "Input text length": f"{EMBEDDINGS_INPUT_TOKENS} tokens", + "Input text preview": ( + EMBEDDINGS_INPUT_TEXT[:100] + "..." + if len(EMBEDDINGS_INPUT_TEXT) > 100 + else EMBEDDINGS_INPUT_TEXT + ), + } + + await run_benchmark_main( + config, + run_benchmark, + "EMBEDDINGS", + HTTP_URL, + BATCH_SIZE, + additional_info, + build_warmup_embeddings_request, + ) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/prefill_only/bench_score.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/prefill_only/bench_score.py new file mode 100644 index 00000000..117335ea --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/prefill_only/bench_score.py @@ -0,0 +1,192 @@ +""" +SGLang Scoring Benchmark Script + +This script benchmarks SGLang's scoring API performance using HTTP requests. + +Current Features: +- HTTP-only implementation (open source compatible) +- Uses /v1/score API endpoint directly +- Single item scoring with batching support +- Configurable RPS, duration, and batch sizes +- Progress tracking and detailed metrics +- Poisson and constant request distributions + +Usage: +- Update configuration variables at the top of the file +- Ensure SGLang server is running on the configured HTTP_URL +- Run: python bench_score.py +- Each request will contain ITEM_COUNT_VALUES items for batch scoring + +""" + +import asyncio + +from transformers import AutoTokenizer +from util import ( + BenchmarkConfig, + generate_text_with_token_count, + run_benchmark_main, + run_generic_benchmark, +) + +############################################################################### +# CONFIG +############################################################################### +# Create benchmark configuration +config = BenchmarkConfig() +config.rps_values = [160] +config.duration_secs_values = [60] +config.num_unique_requests = 100 +config.distribution = "POISSON" +config.profile = False +config.freeze_gc = True # Enable GC freeze functionality +# Profiler output directory - by default uses present working directory (pwd) +# Uncomment and customize the line below to override the default location: +# config.profiler_dir = "/sglang-oss-trace" + +# HTTP Configuration +HTTP_URL = "http://localhost:30000/v1/score" # Use score API directly + +# Score API Config +# ITEM_COUNT_VALUES determines number of items per score request (batch size) +SCORE_QUERY_TOKENS = 120 +SCORE_ITEM_TOKENS = 180 +SCORE_MODEL_PATH = "Qwen/Qwen3-0.6B" +SCORE_LABEL_TOKEN_IDS = [9454, 2753] # Yes/No token IDs +ITEM_COUNT_VALUES = [10] # Number of items per request + +# Special token to replicate for precise token counting +SPECIAL_REPLICATED_TOKEN = "<|im_start|>" + + +############################################################################### +# REQUEST GENERATION (in parallel) +############################################################################### +def create_score_request_builder(): + """Create a score request builder function with shared tokenizer.""" + # Load tokenizer once here to verify special token and get precise counts + print("Loading tokenizer...") + tokenizer = AutoTokenizer.from_pretrained(SCORE_MODEL_PATH) + + # Verify that our special token produces exactly 1 token + special_token_count = len( + tokenizer.encode(config.special_replicated_token, add_special_tokens=False) + ) + print( + f"Special token '{config.special_replicated_token}' produces " + f"{special_token_count} token(s)" + ) + + def generate_text_with_token_count_local(num_toks): + """Generate text with precise token count using replicated token.""" + return generate_text_with_token_count( + SCORE_MODEL_PATH, + num_toks, + config.special_replicated_token, + tokenizer=tokenizer, + ) + + def build_score_request(index: int, item_count: int) -> tuple: + """Build a single score request.""" + try: + # Generate query and items for score API + query = generate_text_with_token_count_local(SCORE_QUERY_TOKENS) + items = [ + generate_text_with_token_count_local(SCORE_ITEM_TOKENS) + for _ in range(item_count) + ] + + # Return as dict for score API format + score_data = { + "query": query, + "items": items, + "label_token_ids": SCORE_LABEL_TOKEN_IDS, + "model": SCORE_MODEL_PATH, + } + return (index, score_data) + + except Exception as e: + print(f"Error building request {index}: {e}") + return (index, None) + + return build_score_request + + +def validate_score_response(response_data: dict) -> bool: + """Validate score API response.""" + return "scores" in response_data or "logprobs" in response_data + + +def build_warmup_score_request() -> dict: + """Build a warmup request for the score API.""" + # Load tokenizer once for warmup generation + tokenizer = AutoTokenizer.from_pretrained(SCORE_MODEL_PATH) + + warmup_query = generate_text_with_token_count( + SCORE_MODEL_PATH, + SCORE_QUERY_TOKENS, + config.special_replicated_token, + tokenizer=tokenizer, + ) + warmup_items = [ + generate_text_with_token_count( + SCORE_MODEL_PATH, + SCORE_ITEM_TOKENS, + config.special_replicated_token, + tokenizer=tokenizer, + ) + for _ in range(3) + ] + + return { + "query": warmup_query, + "items": warmup_items, + "label_token_ids": SCORE_LABEL_TOKEN_IDS, + "model": SCORE_MODEL_PATH, + # Add missing parameters for consistency with the original warmup + "apply_softmax": True, + "item_first": False, + } + + +############################################################################### +# MAIN +############################################################################### +async def run_benchmark(rps, duration_secs, item_count): + """Run a single benchmark with the given RPS value.""" + # Create the request builder function with shared tokenizer + build_request_func = create_score_request_builder() + + return await run_generic_benchmark( + rps=rps, + duration_secs=duration_secs, + item_count=item_count, + config=config, + http_url=HTTP_URL, + build_request_func=build_request_func, + response_validator=validate_score_response, + api_name="SINGLE_ITEM_SCORING", + request_description="score requests", + ) + + +async def main(): + """Main function that runs benchmarks for all RPS values.""" + additional_info = { + "Query tokens per request": SCORE_QUERY_TOKENS, + "Item tokens per item": SCORE_ITEM_TOKENS, + } + + await run_benchmark_main( + config, + run_benchmark, + "SINGLE_ITEM_SCORING", + HTTP_URL, + ITEM_COUNT_VALUES, + additional_info, + build_warmup_score_request, + ) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/prefill_only/util.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/prefill_only/util.py new file mode 100644 index 00000000..3b385591 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/prefill_only/util.py @@ -0,0 +1,813 @@ +""" +Common utilities for SGLang benchmark scripts. + +This module contains shared code for benchmarking different SGLang APIs +including scoring, embeddings, and other endpoints. +""" + +import asyncio +import concurrent.futures +import json +import os +import random +from statistics import mean +from typing import Any, Callable, Dict, List, Optional, Tuple + +import aiohttp +import numpy as np +from tqdm import tqdm +from transformers import AutoTokenizer + + +class BenchmarkConfig: + """Configuration for benchmark parameters.""" + + def __init__(self): + # Common benchmark settings + self.server_type = "HTTP" + self.rps_values = [70] + self.duration_secs_values = [60] + self.num_unique_requests = 100 + self.distribution = "POISSON" # Options: "CONSTANT", "POISSON" + self.profile = False + + # Garbage Collection Control + self.freeze_gc = True # Enable/disable garbage collection freezing + + # Profiler configuration + self.profiler_dir = ( + os.getcwd() + ) # Default profiler output directory (current working directory) + + # Special token for text generation + self.special_replicated_token = "<|im_start|>" + + +def generate_text_with_token_count( + model_path: str, + num_tokens: int, + special_token: str = "<|im_start|>", + tokenizer: Optional[Any] = None, +) -> str: + """ + Generate text with precise token count using a replicated token. + + Args: + model_path: Path to the model for tokenizer + num_tokens: Target number of tokens + special_token: Token to replicate + tokenizer: Optional pre-loaded tokenizer to avoid repeated loading + + Returns: + Generated text with approximately the target token count + """ + if tokenizer is None: + tokenizer = AutoTokenizer.from_pretrained(model_path) + + # Verify token count + special_token_count = len(tokenizer.encode(special_token, add_special_tokens=False)) + + if special_token_count == 1: + # Simple case: token maps to exactly 1 token + return special_token * num_tokens + else: + print(f"Special token '{special_token}' produces {special_token_count} tokens") + # Handle case where special token produces multiple tokens + repetitions = (num_tokens + special_token_count - 1) // special_token_count + text = special_token * repetitions + + # Verify we got the expected token count + actual_tokens = len(tokenizer.encode(text, add_special_tokens=False)) + if actual_tokens < num_tokens: + print(f"Warning: Generated {actual_tokens} tokens, expected {num_tokens}") + + return text + + +def setup_profiler(config: BenchmarkConfig, benchmark_name: str) -> None: + """ + Set up profiler environment if profiling is enabled. + + Args: + config: Benchmark configuration + benchmark_name: Name of the benchmark (used in directory path) + """ + if config.profile: + # Create benchmark-specific subdirectory + profiler_path = os.path.join( + config.profiler_dir, benchmark_name.lower().replace("_", "-") + ) + os.environ["SGLANG_TORCH_PROFILER_DIR"] = profiler_path + print(f"Profiler enabled. Output directory: {profiler_path}") + else: + print("Profiler disabled") + + +def prepare_all_requests_parallel( + num_requests: int, + item_count: int, + build_request_func: Callable[[int, int], Tuple[int, Any]], + config: BenchmarkConfig, + description: str = "requests", +) -> List[Any]: + """ + Generic function to generate unique requests in parallel, then reuse them. + + Args: + num_requests: Total number of requests needed + item_count: Number of items per request (batch size) + build_request_func: Function that takes (index, item_count) and returns (index, request_data) + config: Benchmark configuration + description: Description for progress bars + + Returns: + List of request data objects + """ + + def build_request_wrapper(index): + """Wrapper to call the provided build_request_func.""" + try: + return build_request_func(index, item_count) + except Exception as e: + print(f"Error building request {index}: {e}") + return (index, None) + + # Generate only the unique requests + unique_requests = [None] * config.num_unique_requests + max_workers = min(8, os.cpu_count() or 1) # Limit to 8 threads max + + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [] + for i in tqdm( + range(config.num_unique_requests), + desc=f"Submitting {description} generation tasks", + ): + future = executor.submit(build_request_wrapper, i) + futures.append(future) + + # Collect results as they complete + for f in tqdm( + concurrent.futures.as_completed(futures), + desc=f"Building unique {description}", + total=config.num_unique_requests, + ): + try: + index, req_data = f.result() + if req_data is not None: + unique_requests[index] = req_data + else: + print(f"Failed to build request {index}") + except Exception as e: + print(f"Error processing request result: {e}") + + # Check if we have any valid requests + valid_requests = [req for req in unique_requests if req is not None] + if not valid_requests: + raise RuntimeError("Failed to generate any valid requests") + + print( + f"Successfully generated {len(valid_requests)} out of " + f"{config.num_unique_requests} unique {description}" + ) + + # Create the full request list by cycling through unique requests + print( + f"Reusing {len(valid_requests)} unique {description} to create " + f"{num_requests} total requests..." + ) + all_requests = [] + for i in tqdm(range(num_requests), desc=f"Reusing {description}"): + unique_index = i % len(valid_requests) + all_requests.append(valid_requests[unique_index]) + + print(f"All {description} prepared.\n") + return all_requests + + +async def sleep_with_distribution(distribution: str, rps: float) -> None: + """ + Sleep according to the specified distribution pattern. + + Args: + distribution: "CONSTANT" or "POISSON" + rps: Requests per second rate + """ + if distribution == "CONSTANT": + interval = 1 / rps + await asyncio.sleep(interval) + elif distribution == "POISSON": + # For Poisson process, inter-arrival times follow exponential distribution + interval = random.expovariate(rps) + await asyncio.sleep(interval) + else: + raise ValueError( + f"Unknown distribution: {distribution}. Use 'CONSTANT' or 'POISSON'." + ) + + +def build_http_request_json(request_data: Any) -> str: + """ + Generic function to build HTTP request JSON. + + Args: + request_data: The data to serialize to JSON + + Returns: + JSON string representation of the request data + """ + return json.dumps(request_data) + + +async def make_http_call( + session: aiohttp.ClientSession, + request_data: Any, + request_id: int, + results_queue: asyncio.Queue, + http_url: str, + response_validator: Callable[[Dict[str, Any]], bool], + api_name: str = "API", +) -> None: + """ + Generic HTTP call function for API requests. + + Args: + session: aiohttp client session + request_data: Data to send in the request + request_id: Unique identifier for this request + results_queue: Queue to put results + http_url: URL to send the request to + response_validator: Function to validate the response JSON + api_name: Name of the API for error messages + """ + try: + start_time = asyncio.get_running_loop().time() + + request_json = build_http_request_json(request_data) + headers = {"Content-Type": "application/json"} + + async with session.post(http_url, data=request_json, headers=headers) as resp: + resp_text = await resp.text() + + if resp.status != 200: + print( + f"[HTTP] {api_name} Request {request_id} failed with status " + f"{resp.status}: {resp_text}" + ) + completion_time = asyncio.get_running_loop().time() + await results_queue.put((request_id, 0, False, completion_time)) + return + + # Parse and validate response + try: + response_data = json.loads(resp_text) + success = response_validator(response_data) + if not success: + print( + f"[HTTP] {api_name} Request {request_id} failed response validation" + ) + except json.JSONDecodeError: + print( + f"[HTTP] {api_name} Request {request_id} failed to parse JSON response" + ) + success = False + + completion_time = asyncio.get_running_loop().time() + elapsed_time = (completion_time - start_time) * 1000 + await results_queue.put((request_id, elapsed_time, success, completion_time)) + + except Exception as e: + print(f"[HTTP] {api_name} Error for request {request_id}: {e}") + completion_time = asyncio.get_running_loop().time() + await results_queue.put((request_id, 0, False, completion_time)) + + +async def send_profile_request( + profile_text: str, http_url: str, session: Optional[aiohttp.ClientSession] = None +) -> None: + """ + Send a profile request (START_PROFILE or STOP_PROFILE) and wait for completion. + + Args: + profile_text: "START_PROFILE" or "STOP_PROFILE" + http_url: Base HTTP URL (will derive profile endpoints from this) + session: Optional aiohttp session to use + """ + try: + if session: + print(f"Sending {profile_text} request via HTTP...") + + # Determine the correct endpoint + if "/v1/" in http_url: + base_url = http_url.rsplit("/v1/", 1)[0] # Remove /v1/xxx + else: + base_url = http_url.rsplit("/", 1)[0] # Remove last path component + + if profile_text == "START_PROFILE": + endpoint_url = f"{base_url}/start_profile" + elif profile_text == "STOP_PROFILE": + endpoint_url = f"{base_url}/stop_profile" + else: + print(f"Unknown profile request: {profile_text}") + return + + headers = {"Content-Type": "application/json"} + + async with session.post(endpoint_url, headers=headers) as resp: + resp_text = await resp.text() + if resp.status == 200: + print(f"{profile_text} request completed") + else: + print( + f"{profile_text} request failed with status " + f"{resp.status}: {resp_text}" + ) + else: + print(f"Cannot send {profile_text} request - missing session") + + except Exception as e: + print(f"Error sending {profile_text} request: {e}") + + +async def call_freeze_gc_http(session: aiohttp.ClientSession, http_url: str) -> None: + """ + Call the /freeze_gc HTTP endpoint. + + Args: + session: aiohttp client session + http_url: Base HTTP URL to derive the freeze_gc endpoint from + """ + try: + # Derive freeze_gc endpoint from the API URL + if "/v1/" in http_url: + freeze_gc_url = http_url.rsplit("/v1/", 1)[0] + "/freeze_gc" + else: + freeze_gc_url = http_url.rsplit("/", 1)[0] + "/freeze_gc" + + print(f"Calling freeze_gc endpoint: {freeze_gc_url}") + + async with session.post(freeze_gc_url) as resp: + if resp.status == 200: + print("freeze_gc called successfully") + else: + resp_text = await resp.text() + print(f"freeze_gc failed with status {resp.status}: {resp_text}") + + except Exception as e: + print(f"Failed to call freeze_gc: {e}") + + +async def send_warmup_requests( + session: aiohttp.ClientSession, + http_url: str, + build_warmup_request_func: Callable[[], Any], + num_warmup: int = 3, +) -> None: + """ + Send warmup requests to HTTP server. + + Args: + session: aiohttp client session + http_url: URL to send warmup requests to + build_warmup_request_func: Function that returns a warmup request object + num_warmup: Number of warmup requests to send + """ + print(f"Sending {num_warmup} HTTP warmup requests...") + + for i in range(num_warmup): + try: + warmup_data = build_warmup_request_func() + request_json = build_http_request_json(warmup_data) + headers = {"Content-Type": "application/json"} + + async with session.post( + http_url, data=request_json, headers=headers + ) as resp: + if resp.status == 200: + print(f"Warmup request {i+1}/{num_warmup} completed successfully") + else: + print( + f"Warmup request {i+1}/{num_warmup} failed with status {resp.status}" + ) + + except Exception as e: + print(f"Warmup request {i+1}/{num_warmup} failed with error: {e}") + + print("HTTP warmup requests completed") + + +async def perform_global_warmup_and_freeze( + config: BenchmarkConfig, + http_url: str, + build_warmup_request_func: Callable[[], Any], +) -> None: + """ + Perform warmup and optionally GC freeze operations once before all benchmark runs. + + Args: + config: Benchmark configuration + http_url: URL for API requests + build_warmup_request_func: Function that returns a warmup request object + """ + print("=" * 80) + print(f"PERFORMING GLOBAL WARMUP{' AND GC FREEZE' if config.freeze_gc else ''}") + print("=" * 80) + + print(f"Performing HTTP warmup{' and GC freeze' if config.freeze_gc else ''}...") + async with aiohttp.ClientSession() as session: + await send_warmup_requests(session, http_url, build_warmup_request_func) + if config.freeze_gc: + await call_freeze_gc_http(session, http_url) + print( + f"HTTP warmup{' and GC freeze' if config.freeze_gc else ''} completed successfully." + ) + + print( + f"Global warmup{' and GC freeze' if config.freeze_gc else ''} operations completed." + ) + print("=" * 80) + + +async def process_results( + results_queue: asyncio.Queue, + num_requests: int, + send_duration: float, + total_duration: float, + rps: int, + duration_secs: int, + item_count: int, + test_start_time: float, + config: BenchmarkConfig, + http_mode: str = "UNKNOWN", +) -> List[Dict[str, Any]]: + """ + Process benchmark results and group them by minute intervals. + + Args: + results_queue: Queue containing result tuples + num_requests: Total number of requests sent + send_duration: Time taken to send all requests + total_duration: Total time for all requests to complete + rps: Target requests per second + duration_secs: Test duration in seconds + item_count: Number of items per request + test_start_time: Start time of the test + config: Benchmark configuration + http_mode: Description of the HTTP mode/API being tested + + Returns: + List of dictionaries containing minute-by-minute results + """ + all_results = [] + + # Collect all results + for _ in range(num_requests): + result = await results_queue.get() + request_id, elapsed_time, success, completion_time = result + all_results.append( + { + "request_id": request_id, + "elapsed_time": elapsed_time, + "success": success, + "completion_time": completion_time, + } + ) + + # Group results by minute intervals + minute_results = [] + num_minutes = int(duration_secs // 60) + (1 if duration_secs % 60 > 0 else 0) + + for minute in range(num_minutes): + minute_start = test_start_time + (minute * 60) + minute_end = test_start_time + ((minute + 1) * 60) + + # Filter results that completed in this minute + minute_data = [ + r for r in all_results if minute_start <= r["completion_time"] < minute_end + ] + + response_times = [r["elapsed_time"] for r in minute_data if r["success"]] + successful_requests = len([r for r in minute_data if r["success"]]) + failed_requests = len([r for r in minute_data if not r["success"]]) + + avg_response_time = mean(response_times) if response_times else 0 + + # Calculate percentiles using numpy + if response_times: + p50 = np.percentile(response_times, 50) + p90 = np.percentile(response_times, 90) + p99 = np.percentile(response_times, 99) + else: + p50 = p90 = p99 = 0 + + minute_result = { + "test_duration_secs": duration_secs, + "minute_interval": minute + 1, + "target_rps": rps, + "item_count": item_count, + "server_type": config.server_type, + "distribution": config.distribution, + "unique_requests": config.num_unique_requests, + "total_requests": len(minute_data), + "successful_requests": successful_requests, + "failed_requests": failed_requests, + "send_duration_secs": send_duration, + "total_duration_secs": total_duration, + "avg_response_time_ms": avg_response_time, + "p50_response_time_ms": p50, + "p90_response_time_ms": p90, + "p99_response_time_ms": p99, + } + + minute_results.append(minute_result) + + print( + f"\nMinute {minute + 1} Summary for RPS {rps}, " + f"Duration {duration_secs}s, Item Count {item_count}:" + ) + print(f" Requests completed in minute: {len(minute_data)}") + print(f" Successful requests: {successful_requests}") + print(f" Failed requests: {failed_requests}") + print(f" Average response time: {avg_response_time:.2f} ms") + print(f" P50 response time: {p50:.2f} ms") + print(f" P90 response time: {p90:.2f} ms") + print(f" P99 response time: {p99:.2f} ms") + + # Print overall summary + all_response_times = [r["elapsed_time"] for r in all_results if r["success"]] + total_successful = len([r for r in all_results if r["success"]]) + total_failed = len([r for r in all_results if not r["success"]]) + + overall_avg = mean(all_response_times) if all_response_times else 0 + if all_response_times: + overall_p50 = np.percentile(all_response_times, 50) + overall_p90 = np.percentile(all_response_times, 90) + overall_p99 = np.percentile(all_response_times, 99) + else: + overall_p50 = overall_p90 = overall_p99 = 0 + + print( + f"\nOverall Summary for RPS {rps}, Duration {duration_secs}s, " + f"Item Count {item_count}:" + ) + print(f" Test duration: {duration_secs} seconds") + print(f" Server type: {config.server_type}") + print(f" HTTP mode: {http_mode}") + print(f" Target RPS: {rps}") + print(f" Item count: {item_count}") + print(f" Distribution: {config.distribution}") + print(f" Unique requests generated: {config.num_unique_requests}") + print(f" Total requests sent: {num_requests}") + print(f" Successful requests: {total_successful}") + print(f" Failed requests: {total_failed}") + print(f" Time to send all requests: {send_duration:.2f} seconds") + print(f" Time for all requests to complete: {total_duration:.2f} seconds") + print(f" Average response time: {overall_avg:.2f} ms") + print(f" P50 response time: {overall_p50:.2f} ms") + print(f" P90 response time: {overall_p90:.2f} ms") + print(f" P99 response time: {overall_p99:.2f} ms\n") + + return minute_results + + +def print_csv_results(all_results: List[Dict[str, Any]]) -> None: + """ + Print benchmark results in CSV format. + + Args: + all_results: List of result dictionaries from process_results + """ + print("\n" + "=" * 80) + print("FINAL CSV RESULTS:") + print("=" * 80) + + # CSV Header + headers = [ + "test_duration_secs", + "minute_interval", + "target_rps", + "item_count", + "server_type", + "distribution", + "unique_requests", + "total_requests", + "successful_requests", + "failed_requests", + "send_duration_secs", + "total_duration_secs", + "avg_response_time_ms", + "p50_response_time_ms", + "p90_response_time_ms", + "p99_response_time_ms", + ] + print(",".join(headers)) + + # CSV Data + for result in all_results: + row = [ + result["test_duration_secs"], + result["minute_interval"], + result["target_rps"], + result["item_count"], + result["server_type"], + result["distribution"], + result["unique_requests"], + result["total_requests"], + result["successful_requests"], + result["failed_requests"], + f"{result['send_duration_secs']:.2f}", + f"{result['total_duration_secs']:.2f}", + f"{result['avg_response_time_ms']:.2f}", + f"{result['p50_response_time_ms']:.2f}", + f"{result['p90_response_time_ms']:.2f}", + f"{result['p99_response_time_ms']:.2f}", + ] + print(",".join(map(str, row))) + + +async def run_benchmark_main( + config: BenchmarkConfig, + run_single_benchmark_func, + benchmark_name: str, + http_url: str, + item_count_values: List[int], + additional_info: Optional[Dict[str, Any]] = None, + build_warmup_request_func: Optional[Callable[[], Any]] = None, +) -> None: + """ + Main benchmark orchestration function. + + Args: + config: Benchmark configuration + run_single_benchmark_func: Async function to run a single benchmark + benchmark_name: Name of the benchmark (e.g., "SCORING", "EMBEDDINGS") + http_url: URL of the API endpoint + item_count_values: List of item counts to test + additional_info: Additional information to print in the header + build_warmup_request_func: Optional function to build warmup requests + """ + total_combinations = ( + len(config.duration_secs_values) + * len(config.rps_values) + * len(item_count_values) + ) + + print( + f"Running benchmarks for {len(config.duration_secs_values)} duration " + f"values, {len(config.rps_values)} RPS values, and " + f"{len(item_count_values)} item count values = " + f"{total_combinations} total combinations" + ) + print(f"Server Type: {config.server_type}") + print(f"HTTP Mode: {benchmark_name}") + print(f"API URL: {http_url}") + + if additional_info: + for key, value in additional_info.items(): + print(f"{key}: {value}") + + print(f"Items per request (batch size): {item_count_values}") + print(f"Profiling Enabled: {config.profile}") + print(f"Duration values: {config.duration_secs_values}") + print(f"RPS values: {config.rps_values}") + print(f"Item count values: {item_count_values}") + print("=" * 80) + + # Set up profiler environment + setup_profiler(config, benchmark_name) + + # Perform global warmup and GC freeze operations if warmup function is provided + if build_warmup_request_func is not None: + await perform_global_warmup_and_freeze( + config, http_url, build_warmup_request_func + ) + + all_results = [] + + for duration_secs in config.duration_secs_values: + for rps in config.rps_values: + for item_count in item_count_values: + result = await run_single_benchmark_func(rps, duration_secs, item_count) + all_results.extend(result) # Extend with minute results + + print_csv_results(all_results) + + +async def run_generic_benchmark( + rps: int, + duration_secs: int, + item_count: int, + config: BenchmarkConfig, + http_url: str, + build_request_func: Callable[[int, int], Tuple[int, Any]], + response_validator: Callable[[Dict[str, Any]], bool], + api_name: str, + request_description: str = "requests", +) -> List[Dict[str, Any]]: + """ + Generic benchmark runner that can be used for different APIs. + + Args: + rps: Requests per second + duration_secs: Duration of the test in seconds + item_count: Number of items per request (batch size) + config: Benchmark configuration + http_url: URL of the API endpoint + build_request_func: Function to build individual requests + response_validator: Function to validate API responses + api_name: Name of the API for logging + request_description: Description for progress bars + + Returns: + List of dictionaries containing minute-by-minute results + """ + num_requests = int(rps * duration_secs) + print( + f"Starting benchmark with RPS={rps}, Duration={duration_secs}s, " + f"Item Count={item_count}, num_requests={num_requests}" + ) + print(f"Server Type: {config.server_type}") + print(f"HTTP Mode: {api_name}") + print(f"Profiling Enabled: {config.profile}") + + # Build requests in parallel (unmeasured) + all_requests = prepare_all_requests_parallel( + num_requests, item_count, build_request_func, config, request_description + ) + + results_queue = asyncio.Queue() + tasks = [] + + # Track timing for sending requests + send_start_time = asyncio.get_running_loop().time() + + # HTTP implementation + async with aiohttp.ClientSession( + timeout=aiohttp.ClientTimeout(total=300) + ) as session: + + # Send START_PROFILE if profiling is enabled + if config.profile: + await send_profile_request("START_PROFILE", http_url, session=session) + + # Add progress bar for sending requests + with tqdm( + total=len(all_requests), + desc=f"Sending HTTP {request_description} at {rps} RPS", + unit="req", + ) as pbar: + for i, request_data in enumerate(all_requests): + request_id = i + 1 + tasks.append( + asyncio.create_task( + make_http_call( + session, + request_data, + request_id, + results_queue, + http_url, + response_validator, + api_name, + ) + ) + ) + + # Update progress bar + pbar.update(1) + + # Throttle based on distribution + if i < len(all_requests) - 1: + await sleep_with_distribution(config.distribution, rps) + + send_end_time = asyncio.get_running_loop().time() + send_duration = send_end_time - send_start_time + + # Wait for all requests to complete with progress tracking + print(f"Waiting for {len(tasks)} HTTP {request_description} to complete...") + with tqdm( + total=len(tasks), desc=f"Completing HTTP {request_description}", unit="req" + ) as completion_pbar: + completed_tasks = [] + for task in asyncio.as_completed(tasks): + await task + completed_tasks.append(task) + completion_pbar.update(1) + + # Send STOP_PROFILE if profiling is enabled + if config.profile: + await send_profile_request("STOP_PROFILE", http_url, session=session) + + completion_end_time = asyncio.get_running_loop().time() + total_duration = completion_end_time - send_start_time + + return await process_results( + results_queue, + num_requests, + send_duration, + total_duration, + rps, + duration_secs, + item_count, + send_start_time, + config, + api_name, + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/react/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/react/README.md new file mode 100644 index 00000000..51bcaa44 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/react/README.md @@ -0,0 +1,34 @@ +## Run benchmark + +NOTE: This is an implementation for replaying a given trace for throughput/latency benchmark purposes. It is not an actual ReAct agent implementation. + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 100 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-questions 100 --backend vllm +``` + + +### Benchmark guidance +``` +python3 bench_other.py --num-questions 100 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +### Benchmark lmql + +``` +python3 bench_other.py --num-questions 100 --backend lmql --parallel 1 +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/react/bench_other.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/react/bench_other.py new file mode 100644 index 00000000..08666662 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/react/bench_other.py @@ -0,0 +1,202 @@ +import argparse +import json +import time +from concurrent.futures import ThreadPoolExecutor + +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + + +def get_prompt(question): + prompt = ( + """Solve a question answering task with interleaving Thought, Action, Observation steps. Thought can reason about the current situation, and Action can be three types: +(1) Search[entity], which searches the exact entity on Wikipedia and returns the first paragraph if it exists. If not, it will return some similar entities to search. +(2) Lookup[keyword], which returns the next sentence containing keyword in the current passage. +(3) Finish[answer], which returns the answer and finishes the task. +Here are some examples. +Question: What is the elevation range for the area that the eastern sector of the Colorado orogeny extends into? +Thought 1: I need to search Colorado orogeny, find the area that the eastern sector of the Colorado orogeny extends into, then find the elevation range of the area. +Action 1: Search[Colorado orogeny] +Observation 1: The Colorado orogeny was an episode of mountain building (an orogeny) in Colorado and surrounding areas. +Thought 2: It does not mention the eastern sector. So I need to look up eastern sector. +Action 2: Lookup[eastern sector] +Observation 2: (Result 1 / 1) The eastern sector extends into the High Plains and is called the Central Plains orogeny. +Thought 3: The eastern sector of Colorado orogeny extends into the High Plains. So I need to search High Plains and find its elevation range. +Action 3: Search[High Plains] +Observation 3: High Plains refers to one of two distinct land regions: +Thought 4: I need to instead search High Plains (United States). +Action 4: Search[High Plains (United States)] +Observation 4: The High Plains are a subregion of the Great Plains. From east to west, the High Plains rise in elevation from around 1,800 to 7,000 ft (550 to 2,130 m).[3] +Thought 5: High Plains rise in elevation from around 1,800 to 7,000 ft, so the answer is 1,800 to 7,000 ft. +Action 5: Finish[1,800 to 7,000 ft] +Question: Musician and satirist Allie Goertz wrote a song about the "The Simpsons" character Milhouse, who Matt Groening named after who? +Thought 1: The question simplifies to "The Simpsons" character Milhouse is named after who. I only need to search Milhouse and find who it is named after. +Action 1: Search[Milhouse] +Observation 1: Milhouse Mussolini Van Houten is a recurring character in the Fox animated television series The Simpsons voiced by Pamela Hayden and created by Matt Groening. +Thought 2: The paragraph does not tell who Milhouse is named after, maybe I can look up "named after". +Action 2: Lookup[named after] +Observation 2: (Result 1 / 1) Milhouse was named after U.S. president Richard Nixon, whose middle name was Milhous. +Thought 3: Milhouse was named after U.S. president Richard Nixon, so the answer is Richard Nixon. +Action 3: Finish[Richard Nixon] +Question: Which documentary is about Finnish rock groups, Adam Clayton Powell or The Saimaa Gesture? +Thought 1: I need to search Adam Clayton Powell and The Saimaa Gesture, and find which documentary is about Finnish rock groups. +Action 1: Search[Adam Clayton Powell] +Observation 1: Could not find [Adam Clayton Powell]. Similar: ['Adam Clayton Powell III', 'Seventh Avenue (Manhattan)', 'Adam Clayton Powell Jr. State Office Building', 'Isabel Washington Powell', 'Adam Powell', 'Adam Clayton Powell (film)', 'Giancarlo Esposito']. +Thought 2: To find the documentary, I can search Adam Clayton Powell (film). +Action 2: Search[Adam Clayton Powell (film)] +Observation 2: Adam Clayton Powell is a 1989 American documentary film directed by Richard Kilberg. +The film is about the rise and fall of influential African-American politician Adam Clayton Powell Jr.[3][4] It was later aired as part of the PBS series The American Experience. +Thought 3: Adam Clayton Powell (film) is a documentary about an African-American politician, not Finnish rock groups. So the documentary about Finnish rock groups must instead be The Saimaa Gesture. +Action 3: Finish[The Saimaa Gesture] +Question: What profession does Nicholas Ray and Elia Kazan have in common? +Thought 1: I need to search Nicholas Ray and Elia Kazan, find their professions, then find the profession they have in common. +Action 1: Search[Nicholas Ray] +Observation 1: Nicholas Ray (born Raymond Nicholas Kienzle Jr., August 7, 1911 – June 16, 1979) was an American film director, screenwriter, and actor best known for the 1955 film Rebel Without a Cause. +Thought 2: Professions of Nicholas Ray are director, screenwriter, and actor. I need to search Elia Kazan next and find his professions. +Action 2: Search[Elia Kazan] +Observation 2: Elia Kazan was an American film and theatre director, producer, screenwriter and actor. +Thought 3: Professions of Elia Kazan are director, producer, screenwriter, and actor. So profession Nicholas Ray and Elia Kazan have in common is director, screenwriter, and actor. +Action 3: Finish[director, screenwriter, actor] +Question: Which magazine was started first Arthur's Magazine or First for Women? +Thought 1: I need to search Arthur's Magazine and First for Women, and find which was started first. +Action 1: Search[Arthur's Magazine] +Observation 1: Arthur's Magazine (1844-1846) was an American literary periodical published in Philadelphia in the 19th century. +Thought 2: Arthur's Magazine was started in 1844. I need to search First for Women next. +Action 2: Search[First for Women] +Observation 2: First for Women is a woman's magazine published by Bauer Media Group in the USA.[1] The magazine was started in 1989. +Thought 3: First for Women was started in 1989. 1844 (Arthur's Magazine) < 1989 (First for Women), so Arthur's Magazine was started first. +Action 3: Finish[Arthur's Magazine] +Question: Were Pavel Urysohn and Leonid Levin known for the same type of work? +Thought 1: I need to search Pavel Urysohn and Leonid Levin, find their types of work, then find if they are the same. +Action 1: Search[Pavel Urysohn] +Observation 1: Pavel Samuilovich Urysohn (February 3, 1898 â August 17, 1924) was a Soviet mathematician who is best known for his contributions in dimension theory. +Thought 2: Pavel Urysohn is a mathematician. I need to search Leonid Levin next and find its type of work. +Action 2: Search[Leonid Levin] +Observation 2: Leonid Anatolievich Levin is a Soviet-American mathematician and computer scientist. +Thought 3: Leonid Levin is a mathematician and computer scientist. So Pavel Urysohn and Leonid Levin have the same type of work. +Action 3: Finish[yes] +""" + + question + ) + return prompt + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_questions] + arguments = [{"question": k, "triplets": v} for l in lines for k, v in l.items()] + + states = [] + + # Select backend + call_generate = get_call_generate(args) + + def run_single_agent(argument): + question = argument["question"] + triplets = argument["triplets"] + prompt = get_prompt(question) + for i in range(1, len(triplets) + 2): + prompt += "Thought " + str(i) + ":" + states.append(prompt) + answer = call_generate( + prompt, max_tokens=200, temperature=0, stop="Observation" + ) + if i > len(triplets): + break + prompt += ( + triplets[i - 1]["thought"] + + "\nAction " + + str(i) + + ":" + + triplets[i - 1]["action"] + + "\nObservation " + + str(i) + + ":" + + triplets[i - 1]["observation"] + + "\n" + ) + + states.append(answer) + + async def run_single_agent_async(argument): + question = argument["question"] + triplets = argument["triplets"] + prompt = get_prompt(question) + for i in range(1, len(triplets) + 2): + prompt += "Thought " + str(i) + ":" + states.append(prompt) + answer = await call_generate( + prompt, max_tokens=200, temperature=0, stop="Observation", max_len=4096 + ) + if i > len(triplets): + break + prompt += ( + triplets[i - 1]["thought"] + + "\nAction " + + str(i) + + ":" + + triplets[i - 1]["action"] + + "\nObservation " + + str(i) + + ":" + + triplets[i - 1]["observation"] + + "\n" + ) + + states.append(answer) + + tic = time.perf_counter() + + if args.backend != "lmql": + if args.parallel == 1: + for arg in tqdm(arguments): + run_single_agent(arg) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(run_single_agent, arguments), total=len(arguments) + ) + ) + + else: + import asyncio + + loop = asyncio.get_event_loop() + batches = [ + [] for _ in range((len(arguments) + args.parallel - 1) // args.parallel) + ] + for i, arg in enumerate(arguments): + batches[i // args.parallel].append(arg) + for bt in tqdm(batches): + tasks = [run_single_agent_async(arg) for arg in bt] + loop.run_until_complete(asyncio.gather(*tasks)) + + latency = time.perf_counter() - tic + + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "ReAct Agents", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": len(arguments), + "other": { + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="hotpotqa_100.jsonl") + parser.add_argument("--num-questions", type=int, default=10) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/react/bench_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/react/bench_sglang.py new file mode 100644 index 00000000..331638e9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/react/bench_sglang.py @@ -0,0 +1,153 @@ +import argparse +import json +import time + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + + +@sgl.function +def webthink(s, question, triplets): + s += ( + """Solve a question answering task with interleaving Thought, Action, Observation steps. Thought can reason about the current situation, and Action can be three types: +(1) Search[entity], which searches the exact entity on Wikipedia and returns the first paragraph if it exists. If not, it will return some similar entities to search. +(2) Lookup[keyword], which returns the next sentence containing keyword in the current passage. +(3) Finish[answer], which returns the answer and finishes the task. +Here are some examples. +Question: What is the elevation range for the area that the eastern sector of the Colorado orogeny extends into? +Thought 1: I need to search Colorado orogeny, find the area that the eastern sector of the Colorado orogeny extends into, then find the elevation range of the area. +Action 1: Search[Colorado orogeny] +Observation 1: The Colorado orogeny was an episode of mountain building (an orogeny) in Colorado and surrounding areas. +Thought 2: It does not mention the eastern sector. So I need to look up eastern sector. +Action 2: Lookup[eastern sector] +Observation 2: (Result 1 / 1) The eastern sector extends into the High Plains and is called the Central Plains orogeny. +Thought 3: The eastern sector of Colorado orogeny extends into the High Plains. So I need to search High Plains and find its elevation range. +Action 3: Search[High Plains] +Observation 3: High Plains refers to one of two distinct land regions: +Thought 4: I need to instead search High Plains (United States). +Action 4: Search[High Plains (United States)] +Observation 4: The High Plains are a subregion of the Great Plains. From east to west, the High Plains rise in elevation from around 1,800 to 7,000 ft (550 to 2,130 m).[3] +Thought 5: High Plains rise in elevation from around 1,800 to 7,000 ft, so the answer is 1,800 to 7,000 ft. +Action 5: Finish[1,800 to 7,000 ft] +Question: Musician and satirist Allie Goertz wrote a song about the "The Simpsons" character Milhouse, who Matt Groening named after who? +Thought 1: The question simplifies to "The Simpsons" character Milhouse is named after who. I only need to search Milhouse and find who it is named after. +Action 1: Search[Milhouse] +Observation 1: Milhouse Mussolini Van Houten is a recurring character in the Fox animated television series The Simpsons voiced by Pamela Hayden and created by Matt Groening. +Thought 2: The paragraph does not tell who Milhouse is named after, maybe I can look up "named after". +Action 2: Lookup[named after] +Observation 2: (Result 1 / 1) Milhouse was named after U.S. president Richard Nixon, whose middle name was Milhous. +Thought 3: Milhouse was named after U.S. president Richard Nixon, so the answer is Richard Nixon. +Action 3: Finish[Richard Nixon] +Question: Which documentary is about Finnish rock groups, Adam Clayton Powell or The Saimaa Gesture? +Thought 1: I need to search Adam Clayton Powell and The Saimaa Gesture, and find which documentary is about Finnish rock groups. +Action 1: Search[Adam Clayton Powell] +Observation 1: Could not find [Adam Clayton Powell]. Similar: ['Adam Clayton Powell III', 'Seventh Avenue (Manhattan)', 'Adam Clayton Powell Jr. State Office Building', 'Isabel Washington Powell', 'Adam Powell', 'Adam Clayton Powell (film)', 'Giancarlo Esposito']. +Thought 2: To find the documentary, I can search Adam Clayton Powell (film). +Action 2: Search[Adam Clayton Powell (film)] +Observation 2: Adam Clayton Powell is a 1989 American documentary film directed by Richard Kilberg. +The film is about the rise and fall of influential African-American politician Adam Clayton Powell Jr.[3][4] It was later aired as part of the PBS series The American Experience. +Thought 3: Adam Clayton Powell (film) is a documentary about an African-American politician, not Finnish rock groups. So the documentary about Finnish rock groups must instead be The Saimaa Gesture. +Action 3: Finish[The Saimaa Gesture] +Question: What profession does Nicholas Ray and Elia Kazan have in common? +Thought 1: I need to search Nicholas Ray and Elia Kazan, find their professions, then find the profession they have in common. +Action 1: Search[Nicholas Ray] +Observation 1: Nicholas Ray (born Raymond Nicholas Kienzle Jr., August 7, 1911 – June 16, 1979) was an American film director, screenwriter, and actor best known for the 1955 film Rebel Without a Cause. +Thought 2: Professions of Nicholas Ray are director, screenwriter, and actor. I need to search Elia Kazan next and find his professions. +Action 2: Search[Elia Kazan] +Observation 2: Elia Kazan was an American film and theatre director, producer, screenwriter and actor. +Thought 3: Professions of Elia Kazan are director, producer, screenwriter, and actor. So profession Nicholas Ray and Elia Kazan have in common is director, screenwriter, and actor. +Action 3: Finish[director, screenwriter, actor] +Question: Which magazine was started first Arthur's Magazine or First for Women? +Thought 1: I need to search Arthur's Magazine and First for Women, and find which was started first. +Action 1: Search[Arthur's Magazine] +Observation 1: Arthur's Magazine (1844-1846) was an American literary periodical published in Philadelphia in the 19th century. +Thought 2: Arthur's Magazine was started in 1844. I need to search First for Women next. +Action 2: Search[First for Women] +Observation 2: First for Women is a woman's magazine published by Bauer Media Group in the USA.[1] The magazine was started in 1989. +Thought 3: First for Women was started in 1989. 1844 (Arthur's Magazine) < 1989 (First for Women), so Arthur's Magazine was started first. +Action 3: Finish[Arthur's Magazine] +Question: Were Pavel Urysohn and Leonid Levin known for the same type of work? +Thought 1: I need to search Pavel Urysohn and Leonid Levin, find their types of work, then find if they are the same. +Action 1: Search[Pavel Urysohn] +Observation 1: Pavel Samuilovich Urysohn (February 3, 1898 â August 17, 1924) was a Soviet mathematician who is best known for his contributions in dimension theory. +Thought 2: Pavel Urysohn is a mathematician. I need to search Leonid Levin next and find its type of work. +Action 2: Search[Leonid Levin] +Observation 2: Leonid Anatolievich Levin is a Soviet-American mathematician and computer scientist. +Thought 3: Leonid Levin is a mathematician and computer scientist. So Pavel Urysohn and Leonid Levin have the same type of work. +Action 3: Finish[yes] +""" + + question + ) + for i in range(1, len(triplets) + 2): + s += "Thought " + str(i) + ":" + # NOTE: This is an implementation for replaying a given trace for benchmark purposes. It is not an actual ReAct agent implementation. + ss = s.fork(1) + ss[0] += sgl.gen(name="thought_action", max_tokens=200, stop="Observation") + ss.join() + # to verify the correctness of output, this should be collected + # print(ss[0]["thought_action"]) + if i > len(triplets): + break + s += ( + triplets[i - 1]["thought"] + + "\nAction " + + str(i) + + ":" + + triplets[i - 1]["action"] + + "\nObservation " + + str(i) + + ":" + + triplets[i - 1]["observation"] + + "\n" + ) + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_questions] + arguments = [{"question": k, "triplets": v} for l in lines for k, v in l.items()] + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + states = [] + tic = time.perf_counter() + states = webthink.run_batch( + arguments, + temperature=0, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "ReAct Agents", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": len(arguments), + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="hotpotqa_100.jsonl") + parser.add_argument("--num-questions", type=int, default=10) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/reasoning_benchmark/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/reasoning_benchmark/README.md new file mode 100644 index 00000000..f9d26e0c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/reasoning_benchmark/README.md @@ -0,0 +1,77 @@ +# Run benchmark + +This benchmark is primarily intended to be used with reasoning models like `DeepSeek-R1` and its distilled models like `DeepSeek-R1-Distill-Qwen-1.5B`. Please use + +```bash +pip install antlr4-python3-runtime +``` + +for `parse_latex` which we use for symbolic equality check. + +## Benchmark sglang + +1. Launch the Server +```bash +python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B --port 30000 +``` + +Note that depending on the GPU this benchmark will take quiet some time. To employ data parallelism please use: + +```bash +python3 -m sglang_router.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B --port 30000 --dp-size 4 +``` + + +2. Benchmarking + +We use [suggested](https://github.com/deepseek-ai/DeepSeek-R1) parameters of `temperature=0.6`, `top_p=.95`, `max_new_tokens=32768`. The command line argument `num-tries` can be used to evaluate the model multiple times on the same question. We use the suggested `64` from the repo for AIME 2024. For LIMO, we use `8` as the number of tries due to the size of the dataset. + +By default evaluate on LIMO dataset. + +```bash +python3 bench_sglang.py --parallel 256 --num-tries 64 --port 30000 +``` + +Evaluate on AIME 2024 dataset. + +```bash +python3 bench_sglang.py --parallel 256 --port 30000 --data-path Maxwell-Jia/AIME_2024 --question-key Problem --answer-key Answer --num-tries 64 +``` + +Evaluate on [AIME 2025 I dataset](https://huggingface.co/datasets/opencompass/AIME2025). For benchmark result see [here](https://matharena.ai/). + +```bash +python3 bench_sglang.py --parallel 256 --port 30000 --data-path opencompass/AIME2025 --question-key question --answer-key answer --num-tries 64 +``` +## Results + +### Evaluation Results +| Dataset | Num Tries | Accuracy | Reference | Standard Error | +|------------|-----------|----------|-----------|-----------| +| LIMO | 8 | 47.7% | ? | ? | +| AIME 2024 | 64 | 33.2% | 28.9% | 3.4% | +| AIME 2025 I| 64 | 29.9% | 25.0% | ? | + +### Statistic Analysis Results +Set up SGLang engine for statistic analysis, for high efficiency we use `--dp-size 8` for data parallelism: +```bash +python3 -m sglang_router.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B --port 30000 --dp-size 8 +``` +**Experiment 1**: +We fixed the number of attempts (num_tries) and conducted multiple runs to assess the consistency of the model's performance. The results show that all recorded accuracies lie within ± one standard error deviation from the mean. This suggests that **our metric serves as an effective upper bound for the deviation of reported accuracy**. + +To collect the accuracy, run the following command 30 times: +```bash +python3 bench_sglang.py --parallel 64 --port 30000 --data-path Maxwell-Jia/AIME_2024 --question-key Problem --answer-key Answer --num-tries 64 +``` + +![acc_hist](figure/Acc_histplot.png) + + +**Experiment 2**: We explored the relationship between the number of attempts (num_tries) and the standard error (SE) by varying num_tries across a range (e.g., 8, 16, 32, ..., 256) and performing a single run for each value. The results demonstrate that as the number of attempts increases, the standard error decreases, leading to **greater stability in answer accuracy**. + +To reveal the relationship, run the command 6 times and adjust the parameter `--num-tries` for each run: +```bash +python3 bench_sglang.py --parallel 64 --port 30000 --data-path Maxwell-Jia/AIME_2024 --question-key Problem --answer-key Answer --num-tries +``` +![SE_num_tries](figure/SE_numtries.png) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/reasoning_benchmark/answer_extraction.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/reasoning_benchmark/answer_extraction.py new file mode 100644 index 00000000..45ce59af --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/reasoning_benchmark/answer_extraction.py @@ -0,0 +1,269 @@ +# Adapted from https://github.com/deepseek-ai/DeepSeek-Math/blob/main/evaluation/data_processing/answer_extraction.py + +import re + +import regex + + +def _fix_fracs(string): + substrs = string.split("\\frac") + new_str = substrs[0] + if len(substrs) > 1: + substrs = substrs[1:] + for substr in substrs: + new_str += "\\frac" + if len(substr) > 0 and substr[0] == "{": + new_str += substr + else: + try: + assert len(substr) >= 2 + except: + return string + a = substr[0] + b = substr[1] + if b != "{": + if len(substr) > 2: + post_substr = substr[2:] + new_str += "{" + a + "}{" + b + "}" + post_substr + else: + new_str += "{" + a + "}{" + b + "}" + else: + if len(substr) > 2: + post_substr = substr[2:] + new_str += "{" + a + "}" + b + post_substr + else: + new_str += "{" + a + "}" + b + string = new_str + return string + + +def _fix_a_slash_b(string): + if len(string.split("/")) != 2: + return string + a = string.split("/")[0] + b = string.split("/")[1] + try: + if "sqrt" not in a: + a = int(a) + if "sqrt" not in b: + b = int(b) + assert string == "{}/{}".format(a, b) + new_string = "\\frac{" + str(a) + "}{" + str(b) + "}" + return new_string + except: + return string + + +def _fix_sqrt(string): + _string = re.sub(r"\\sqrt(-?[0-9.a-zA-Z]+)", r"\\sqrt{\1}", string) + _string = re.sub(r"\\sqrt\s+(\w+)$", r"\\sqrt{\1}", _string) + return _string + + +def _fix_tan(string): + _string = re.sub(r"\\tan(-?[0-9.a-zA-Z]+)", r"\\tan{\1}", string) + _string = re.sub(r"\\tan\s+(\w+)$", r"\\tan{\1}", _string) + return _string + + +def strip_string(string): + string = str(string).strip() + # linebreaks + string = string.replace("\n", "") + + # right "." + string = string.rstrip(".") + + # remove inverse spaces + string = string.replace("\\!", "") + # string = string.replace("\\ ", "") + + # replace \\ with \ + # string = string.replace("\\\\", "\\") + # string = string.replace("\\\\", "\\") + + if string.startswith("\\text{") and string.endswith("}"): + string = string.split("{", 1)[1][:-1] + + # replace tfrac and dfrac with frac + string = string.replace("tfrac", "frac") + string = string.replace("dfrac", "frac") + string = string.replace("cfrac", "frac") + + # remove \left and \right + string = string.replace("\\left", "") + string = string.replace("\\right", "") + + # Remove unit: miles, dollars if after is not none + _string = re.sub(r"\\text{.*?}$", "", string).strip() + if _string != "" and _string != string: + # print("Warning: unit not removed: '{}' -> '{}'".format(string, _string)) + string = _string + + # Remove circ (degrees) + string = string.replace("^{\\circ}", "").strip() + string = string.replace("^\\circ", "").strip() + + string = regex.sub(r"\{(c|m)?m\}(\^(2|3))?", "", string).strip() + string = regex.sub(r"p\.m\.$", "", string).strip() + string = regex.sub(r"(\d)\s*t$", r"\1", string).strip() + + # remove dollar signs + string = string.replace("\\$", "") + string = string.replace("$", "") + + # string = string.replace("\\text", "") + string = string.replace("x\\in", "") + + # remove percentage + string = string.replace("\\%", "%") + string = string.replace("\%", "%") + # string = string.replace("%", "") + + # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string + string = string.replace(" .", " 0.") + string = string.replace("{.", "{0.") + + # cdot + string = string.replace("\\cdot", "") + + # inf + string = string.replace("infinity", "\\infty") + if "\\infty" not in string: + string = string.replace("inf", "\\infty") + string = string.replace("+\\inity", "\\infty") + + # and + # string = string.replace("and", "") + string = string.replace("\\mathbf", "") + string = string.replace("\\mathrm", "") + + # use regex to remove \mbox{...} + string = re.sub(r"\\mbox{.*?}", "", string) + + # quote + string.replace("'", "") + string.replace('"', "") + + # i, j + if "j" in string and "i" not in string: + string = string.replace("j", "i") + + # replace a.000b where b is not number or b is end, with ab, use regex + string = re.sub(r"(\d+)\.0+([^\d])", r"\1\2", string) + string = re.sub(r"(\d+)\.0+$", r"\1", string) + + # if empty, return empty string + if len(string) == 0: + return string + if string[0] == ".": + string = "0" + string + + # to consider: get rid of e.g. "k = " or "q = " at beginning + # if len(string.split("=")) == 2: + # if len(string.split("=")[0]) <= 2: + # string = string.split("=")[1] + + string = _fix_sqrt(string) + string = _fix_tan(string) + string = string.replace(" ", "") + + # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b} + string = _fix_fracs(string) + + # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y + string = _fix_a_slash_b(string) + + string = regex.sub(r"(\\|,|\.)+$", "", string) + + return string + + +def extract_boxed_answers(text): + answers = [] + for piece in text.split("boxed{")[1:]: + n = 0 + for i in range(len(piece)): + if piece[i] == "{": + n += 1 + elif piece[i] == "}": + n -= 1 + if n < 0: + if i + 1 < len(piece) and piece[i + 1] == "%": + answers.append(piece[: i + 1]) + else: + answers.append(piece[:i]) + break + return answers + + +def extract_program_output(pred_str): + """ + extract output between the last ```output\n...\n``` + """ + if "```output" not in pred_str: + return "" + if "```output" in pred_str: + pred_str = pred_str.split("```output")[-1] + if "```" in pred_str: + pred_str = pred_str.split("```")[0] + output = pred_str.strip() + return output + + +def extract_answer(pred_str, exhaust=False): + pred = [] + if "final answer is $" in pred_str and "$. I hope" in pred_str: + tmp = pred_str.split("final answer is $", 1)[1] + pred = [tmp.split("$. I hope", 1)[0].strip()] + elif "boxed" in pred_str: + pred = extract_boxed_answers(pred_str) + elif "he answer is" in pred_str: + pred = [pred_str.split("he answer is")[-1].strip()] + else: + program_output = extract_program_output(pred_str) + if program_output != "": + # fall back to program + pred.append(program_output) + else: # use the last number + pattern = "-?\d*\.?\d+" + ans = re.findall(pattern, pred_str.replace(",", "")) + if len(ans) >= 1: + ans = ans[-1] + else: + ans = "" + if ans: + pred.append(ans) + + # multiple line + _pred = [] + for ans in pred: + ans = ans.strip().split("\n")[0] + ans = ans.lstrip(":") + ans = ans.rstrip(".") + ans = ans.rstrip("/") + ans = strip_string(ans) + _pred.append(ans) + if exhaust: + return _pred + else: + return _pred[-1] if _pred else "" + + +def extract_math_answer(question, reasoning, task): + answer = [] + for ans in extract_answer(reasoning, exhaust=True): + if "separated by commas" in question and all(ch not in ans for ch in "()[]"): + answer.extend([a.strip() for a in ans.split(",")]) + elif regex.search(r"\\text\{\s*and\s*\}", ans): + answer.extend( + [ + a.strip() + for a in regex.sub(r"\\text\{\s*and\s*\}", "[SEP]", ans).split( + "[SEP]" + ) + ] + ) + else: + answer.append(ans.strip()) + return answer diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/reasoning_benchmark/bench_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/reasoning_benchmark/bench_sglang.py new file mode 100644 index 00000000..ccbff9d1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/reasoning_benchmark/bench_sglang.py @@ -0,0 +1,135 @@ +import argparse +import json +import time + +import answer_extraction +import eval_utils +import numpy as np +from datasets import load_dataset + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text + + +@sgl.function +def reasoning_gen(s, question: str): + s += sgl.user( + question + + "\nPlease reason step by step, and put your final answer within \boxed{}." + ) + s += sgl.assistant( + sgl.gen( + "answer", + ) + ) + + +def convert_dataset(path: str, question_key: str, answer_key: str, num_tries: int): + raw_dataset = load_dataset(path) + questions = [] + answers = [] + for data in raw_dataset["train"]: + question = data[question_key] + answer = data[answer_key] + for _ in range(num_tries): + questions.append({"question": question}) + answers.append({"answer": answer}) + return questions, answers + + +def main(args): + # Select backend + sgl.set_default_backend(select_sglang_backend(args)) + + # Get dataset + questions, answers = convert_dataset( + args.data_path, args.question_key, args.answer_key, args.num_tries + ) + + # Run requests + tic = time.perf_counter() + states = reasoning_gen.run_batch( + questions, + num_threads=args.parallel, + progress_bar=True, + temperature=0.6, + max_new_tokens=32768, + top_p=0.95, + ) + latency = time.perf_counter() - tic + + # Extract results and record outcomes in a list. + outcomes = [] + for i, state in enumerate(states): + try: + pred_answer = answer_extraction.extract_math_answer( + questions[i]["question"], state["answer"], "limo" + ) + gt_answer = str(answers[i]["answer"]) + pred_answer = ( + pred_answer[-1] if isinstance(pred_answer, list) else pred_answer + ) + is_correct = 1 if eval_utils.math_equal(pred_answer, gt_answer) else 0 + except Exception as e: + print(f"Error extracting answer: {e}") + is_correct = 0 + + outcomes.append(is_correct) + + # Calculate overall accuracy using numpy + overall_accuracy = np.mean(outcomes) + print(f"Overall Accuracy: {overall_accuracy}") + + # Calculate mean standard error over questions if num_tries >= 2 + if args.num_tries > 1: + outcomes_np = np.array(outcomes).reshape(-1, args.num_tries) + # Using sample standard deviation with ddof=1 + std_per_question = np.std(outcomes_np, axis=1, ddof=1) + # Compute the standard error for each question: std / sqrt(num_tries) + se_per_question = std_per_question / np.sqrt(args.num_tries) + mean_se = se_per_question.mean() + print(f"Mean Standard Error of Accuracy across questions: {mean_se}") + else: + mean_se = None + print("Not enough samples per question to compute standard error.") + + # Calculate output throughput + num_output_tokens = sum( + s.get_meta_info("answer")["completion_tokens"] for s in states + ) + output_throughput = num_output_tokens / latency + print(f"Output throughput: {output_throughput} token/s") + + # Dump results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + # Write results + with open(args.result_file, "a") as fout: + value = { + "task": "limo", + "backend": args.backend, + "latency": round(latency, 3), + "overall_accuracy": round(overall_accuracy, 3), + "mean_se_accuracy": round(mean_se, 3) if mean_se is not None else None, + "num_requests": len(questions), + "other": { + "num_questions": len(questions), + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="GAIR/LIMO") + parser.add_argument("--question-key", type=str, default="question") + parser.add_argument("--answer-key", type=str, default="answer") + parser.add_argument("--num-tries", type=int, default=1) + add_common_sglang_args_and_parse(parser) + args = parser.parse_args() + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/reasoning_benchmark/eval_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/reasoning_benchmark/eval_utils.py new file mode 100644 index 00000000..ab736954 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/reasoning_benchmark/eval_utils.py @@ -0,0 +1,206 @@ +# Adapted from https://github.com/deepseek-ai/DeepSeek-Math/blob/main/evaluation/eval/eval_utils.py + +from math import isclose + +import regex +from sympy import N, simplify +from sympy.parsing.latex import parse_latex +from sympy.parsing.sympy_parser import parse_expr + + +def parse_digits(num): + # format: 234.23 || 23% + num = regex.sub(",", "", str(num)) + try: + return float(num) + except: + if num.endswith("%"): + num = num[:-1] + if num.endswith("\\"): + num = num[:-1] + try: + return float(num) / 100 + except: + pass + return None + + +def is_digit(num): + # paired with parse_digits + return parse_digits(num) is not None + + +def symbolic_equal(a, b): + def _parse(s): + for f in [parse_latex, parse_expr]: + try: + return f(s) + except: + pass + return s + + a = _parse(a) + b = _parse(b) + + try: + if simplify(a - b) == 0: + return True + except: + pass + + try: + if isclose(N(a), N(b), abs_tol=1e-3): + return True + except: + pass + return False + + +def math_equal(prediction, reference, include_percentage=True, is_close=True): + """ + Exact match of math if and only if: + 1. numerical equal: both can convert to float and are equal + 2. symbolic equal: both can convert to sympy expression and are equal + """ + if str(prediction) == str(reference): + return True + + try: # 1. numerical equal + if is_digit(prediction) and is_digit(reference): + prediction = parse_digits(prediction) + reference = parse_digits(reference) + # number questions + if include_percentage: + gt_result = [reference / 100, reference, reference * 100] + else: + gt_result = [reference] + for item in gt_result: + try: + if is_close: + if isclose(item, prediction, abs_tol=1e-3): + return True + else: + if item == prediction: + return True + except Exception: + continue + return False + except: + pass + + if not prediction and prediction not in [0, False]: + return False + + # 2. symbolic equal + reference = str(reference).strip() + prediction = str(prediction).strip() + + if ( + regex.match(r"(\(|\[).+(\)|\])", prediction) is not None + and regex.match(r"(\(|\[).+(\)|\])", reference) is not None + ): + pred_parts = prediction[1:-1].split(",") + ref_parts = reference[1:-1].split(",") + if len(pred_parts) == len(ref_parts): + if all( + [ + math_equal( + pred_parts[i], ref_parts[i], include_percentage, is_close + ) + for i in range(len(pred_parts)) + ] + ): + return True + + # Add back matrix comparison + if ( + ( + prediction.startswith("\\begin{pmatrix}") + or prediction.startswith("\\begin{bmatrix}") + ) + and ( + prediction.endswith("\\end{pmatrix}") + or prediction.endswith("\\end{bmatrix}") + ) + and ( + reference.startswith("\\begin{pmatrix}") + or reference.startswith("\\begin{bmatrix}") + ) + and ( + reference.endswith("\\end{pmatrix}") or reference.endswith("\\end{bmatrix}") + ) + ): + pred_lines = [ + line.strip() + for line in prediction[ + len("\\begin{pmatrix}") : -len("\\end{pmatrix}") + ].split("\\\\") + if line.strip() + ] + ref_lines = [ + line.strip() + for line in reference[ + len("\\begin{pmatrix}") : -len("\\end{pmatrix}") + ].split("\\\\") + if line.strip() + ] + matched = True + if len(pred_lines) == len(ref_lines): + for pred_line, ref_line in zip(pred_lines, ref_lines): + pred_parts = pred_line.split("&") + ref_parts = ref_line.split("&") + if len(pred_parts) == len(ref_parts): + if not all( + [ + math_equal( + pred_parts[i], + ref_parts[i], + include_percentage, + is_close, + ) + for i in range(len(pred_parts)) + ] + ): + matched = False + break + else: + matched = False + if not matched: + break + else: + matched = False + if matched: + return True + + # Add back equation comparison + if prediction.count("=") == 1 and reference.count("=") == 1: + pred = prediction.split("=") + pred = f"{pred[0].strip()} - ({pred[1].strip()})" + ref = reference.split("=") + ref = f"{ref[0].strip()} - ({ref[1].strip()})" + if symbolic_equal(pred, ref) or symbolic_equal(f"-({pred})", ref): + return True + elif ( + prediction.count("=") == 1 + and len(prediction.split("=")[0].strip()) <= 2 + and "=" not in reference + ): + if math_equal( + prediction.split("=")[1], reference, include_percentage, is_close + ): + return True + elif ( + reference.count("=") == 1 + and len(reference.split("=")[0].strip()) <= 2 + and "=" not in prediction + ): + if math_equal( + prediction, reference.split("=")[1], include_percentage, is_close + ): + return True + + # symbolic equal with sympy + if symbolic_equal(prediction, reference): + return True + + return False diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/.gitignore b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/.gitignore new file mode 100644 index 00000000..3322de9b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/.gitignore @@ -0,0 +1 @@ +!topic.jsonl diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/README.md new file mode 100644 index 00000000..97ebe1d8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/README.md @@ -0,0 +1,33 @@ +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 64 +python3 bench_sglang.py --num-questions 32 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --backend vllm --num-questions 64 +``` + + +### Benchmark guidance +``` +python3 bench_other.py --backend guidance --num-questions 32 --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +### Benchmark lmql + +``` +python3 bench_other.py --backend lmql --num-questions 32 --parallel 1 +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/bench_other.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/bench_other.py new file mode 100644 index 00000000..2630081b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/bench_other.py @@ -0,0 +1,133 @@ +import argparse +import json +import time +from concurrent.futures import ThreadPoolExecutor +from functools import partial + +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +number = 5 + + +def expand_tip(topic, tip, generate): + s = ( + """Please expand a tip for a topic into a detailed paragraph. + +Topic: staying healthy +Tip: Regular Exercise +Paragraph: Incorporate physical activity into your daily routine. This doesn't necessarily mean intense gym workouts; it can be as simple as walking, cycling, or yoga. Regular exercise helps in maintaining a healthy weight, improves cardiovascular health, boosts mental health, and can enhance cognitive function, which is crucial for fields that require intense intellectual engagement. + +Topic: building a campfire +Tip: Choose the Right Location +Paragraph: Always build your campfire in a safe spot. This means selecting a location that's away from trees, bushes, and other flammable materials. Ideally, use a fire ring if available. If you're building a fire pit, it should be on bare soil or on a bed of stones, not on grass or near roots which can catch fire underground. Make sure the area above is clear of low-hanging branches. + +Topic: writing a blog post +Tip: structure your content effectively +Paragraph: A well-structured post is easier to read and more enjoyable. Start with an engaging introduction that hooks the reader and clearly states the purpose of your post. Use headings and subheadings to break up the text and guide readers through your content. Bullet points and numbered lists can make information more digestible. Ensure each paragraph flows logically into the next, and conclude with a summary or call-to-action that encourages reader engagement. + +Topic: """ + + topic + + "\nTip: " + + tip + + "\nParagraph:" + ) + return generate(s, max_tokens=128, stop=["\n\n"]) + + +def suggest_tips(topic, generate): + s = "Please act as a helpful assistant. Your job is to provide users with useful tips on a specific topic.\n" + s += "USER: Give some tips for " + topic + ".\n" + s += ( + "ASSISTANT: Okay. Here are " + + str(number) + + " concise tips, each under 8 words:\n" + ) + + tips = [] + for i in range(1, 1 + number): + s += f"{i}." + tip = generate(s, max_tokens=24, stop=[".", "\n"]) + s += tip + ".\n" + tips.append(tip) + + paragraphs = [expand_tip(topic, tip, generate=generate) for tip in tips] + + for i in range(1, 1 + number): + s += f"Tip {i}:" + paragraphs[i - 1] + "\n" + return s + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_questions] + states = [None] * len(lines) + + # Select backend + call_generate = partial(get_call_generate(args), temperature=0) + + # Run requests + tic = time.perf_counter() + if args.backend != "lmql": + + def get_one_answer(i): + states[i] = suggest_tips(lines[i]["topic"], call_generate) + + if args.parallel == 1: + for i in tqdm(range(len(lines))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(lines)))), + total=len(lines), + ) + ) + + else: + import asyncio + + from lmql_funcs import suggest_tips_async + + async def get_one_answer_async(i): + states[i] = await suggest_tips_async(lines[i]["topic"], call_generate) + + batches = [] + for i in range(0, len(lines), args.parallel): + batches.append(list(range(i, min(i + args.parallel, len(lines))))) + loop = asyncio.get_event_loop() + for batch in tqdm(batches): + loop.run_until_complete( + asyncio.gather(*[get_one_answer_async(i) for i in batch]) + ) + latency = time.perf_counter() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "tip_suggestion", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="topic.jsonl") + parser.add_argument("--num-questions", type=int, default=100) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/bench_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/bench_sglang.py new file mode 100644 index 00000000..86c476f9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/bench_sglang.py @@ -0,0 +1,100 @@ +import argparse +import json +import time + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + +number = 5 + + +@sgl.function +def expand_tip(s, topic, tip): + s += ( + """Please expand a tip for a topic into a detailed paragraph. + +Topic: staying healthy +Tip: Regular Exercise +Paragraph: Incorporate physical activity into your daily routine. This doesn't necessarily mean intense gym workouts; it can be as simple as walking, cycling, or yoga. Regular exercise helps in maintaining a healthy weight, improves cardiovascular health, boosts mental health, and can enhance cognitive function, which is crucial for fields that require intense intellectual engagement. + +Topic: building a campfire +Tip: Choose the Right Location +Paragraph: Always build your campfire in a safe spot. This means selecting a location that's away from trees, bushes, and other flammable materials. Ideally, use a fire ring if available. If you're building a fire pit, it should be on bare soil or on a bed of stones, not on grass or near roots which can catch fire underground. Make sure the area above is clear of low-hanging branches. + +Topic: writing a blog post +Tip: structure your content effectively +Paragraph: A well-structured post is easier to read and more enjoyable. Start with an engaging introduction that hooks the reader and clearly states the purpose of your post. Use headings and subheadings to break up the text and guide readers through your content. Bullet points and numbered lists can make information more digestible. Ensure each paragraph flows logically into the next, and conclude with a summary or call-to-action that encourages reader engagement. + +Topic: """ + + topic + + "\nTip: " + + tip + + "\nParagraph:" + ) + s += sgl.gen("paragraph", max_tokens=128, stop=["\n\n"], temperature=0) + + +@sgl.function +def suggest_tips(s, topic): + s += "Please act as a helpful assistant. Your job is to provide users with useful tips on a specific topic.\n" + s += "USER: Give some tips for " + topic + ".\n" + s += ( + "ASSISTANT: Okay. Here are " + + str(number) + + " concise tips, each under 8 words:\n" + ) + + paragraphs = [] + for i in range(1, 1 + number): + s += f"{i}." + sgl.gen(f"tip_{i}", max_tokens=24, stop=[".", "\n"]) + ".\n" + paragraphs.append(expand_tip(topic=topic, tip=s[f"tip_{i}"])) + + for i in range(1, 1 + number): + s += f"Tip {i}:" + paragraphs[i - 1]["paragraph"] + "\n" + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_questions] + arguments = [{"topic": l["topic"]} for l in lines] + + # Select backend + sgl.set_default_backend(select_sglang_backend(args)) + + # Run requests + tic = time.perf_counter() + states = suggest_tips.run_batch( + arguments, temperature=0, num_threads=args.parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "tip_suggestion", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="topic.jsonl") + parser.add_argument("--num-questions", type=int, default=100) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/lmql_funcs.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/lmql_funcs.py new file mode 100644 index 00000000..7790bbe9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/lmql_funcs.py @@ -0,0 +1,50 @@ +number = 5 + + +async def expand_tip_async(topic, tip, generate): + s = ( + """Please expand a tip for a topic into a detailed paragraph. + +Topic: staying healthy +Tip: Regular Exercise +Paragraph: Incorporate physical activity into your daily routine. This doesn't necessarily mean intense gym workouts; it can be as simple as walking, cycling, or yoga. Regular exercise helps in maintaining a healthy weight, improves cardiovascular health, boosts mental health, and can enhance cognitive function, which is crucial for fields that require intense intellectual engagement. + +Topic: building a campfire +Tip: Choose the Right Location +Paragraph: Always build your campfire in a safe spot. This means selecting a location that's away from trees, bushes, and other flammable materials. Ideally, use a fire ring if available. If you're building a fire pit, it should be on bare soil or on a bed of stones, not on grass or near roots which can catch fire underground. Make sure the area above is clear of low-hanging branches. + +Topic: writing a blog post +Tip: structure your content effectively +Paragraph: A well-structured post is easier to read and more enjoyable. Start with an engaging introduction that hooks the reader and clearly states the purpose of your post. Use headings and subheadings to break up the text and guide readers through your content. Bullet points and numbered lists can make information more digestible. Ensure each paragraph flows logically into the next, and conclude with a summary or call-to-action that encourages reader engagement. + +Topic: """ + + topic + + "\nTip: " + + tip + + "\nParagraph:" + ) + return await generate(s, max_tokens=128, stop="\n\n") + + +async def suggest_tips_async(topic, generate): + s = "Please act as a helpful assistant. Your job is to provide users with useful tips on a specific topic.\n" + s += "USER: Give some tips for " + topic + ".\n" + s += ( + "ASSISTANT: Okay. Here are " + + str(number) + + " concise tips, each under 8 words:\n" + ) + + tips = [] + for i in range(1, 1 + number): + s += f"{i}." + # NOTE: stop is different due to lmql does not support a list of stop tokens + tip = await generate(s, max_tokens=24, stop=".\n") + s += tip + ".\n" + tips.append(tip) + + paragraphs = [await expand_tip_async(topic, tip, generate=generate) for tip in tips] + + for i in range(1, 1 + number): + s += f"Tip {i}:" + paragraphs[i - 1] + "\n" + return s diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/topic.jsonl b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/topic.jsonl new file mode 100644 index 00000000..c8ac6d12 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tip_suggestion/topic.jsonl @@ -0,0 +1,50 @@ +{"topic": "organizing a successful charity event", "number": 6} +{"topic": "improving personal credit scores", "number": 7} +{"topic": "staying motivated during job searches", "number": 5} +{"topic": "maintaining a work-life balance", "number": 9} +{"topic": "reducing carbon footprint at home", "number": 8} +{"topic": "starting a book club", "number": 5} +{"topic": "learning to play a musical instrument", "number": 7} +{"topic": "getting into freelance writing", "number": 6} +{"topic": "beginner yoga poses", "number": 8} +{"topic": "preparing for graduate school exams", "number": 5} +{"topic": "exploring minimalist living", "number": 9} +{"topic": "effective grocery shopping", "number": 7} +{"topic": "winter camping", "number": 5} +{"topic": "starting a podcast on a budget", "number": 8} +{"topic": "creating a capsule wardrobe", "number": 6} +{"topic": "improving your writing skills", "number": 7} +{"topic": "learning a new software quickly", "number": 9} +{"topic": "reducing anxiety before public speaking", "number": 5} +{"topic": "planning a solo travel adventure", "number": 8} +{"topic": "beginner skateboarders", "number": 6} +{"topic": "studying abroad", "number": 7} +{"topic": "planting a vegetable garden", "number": 5} +{"topic": "adopting a shelter pet", "number": 9} +{"topic": "learning to cook ethnic cuisines", "number": 8} +{"topic": "effective conflict resolution", "number": 5} +{"topic": "starting a vlog", "number": 7} +{"topic": "keeping a daily journal", "number": 6} +{"topic": "improving sleep hygiene", "number": 8} +{"topic": "beginner mountain climbers", "number": 5} +{"topic": "creating a mobile app", "number": 9} +{"topic": "maintaining a saltwater aquarium", "number": 7} +{"topic": "preparing for a baby's arrival", "number": 6} +{"topic": "writing a fantasy novel", "number": 5} +{"topic": "effective team leadership", "number": 8} +{"topic": "making a documentary film", "number": 9} +{"topic": "learning about historical events", "number": 7} +{"topic": "baking gluten-free treats", "number": 6} +{"topic": "improving mental arithmetic skills", "number": 5} +{"topic": "building a treehouse", "number": 8} +{"topic": "getting started with watercolor painting", "number": 9} +{"topic": "creating a YouTube tutorial series", "number": 7} +{"topic": "landscape photography", "number": 5} +{"topic": "navigating cultural differences", "number": 6} +{"topic": "preparing for a marathon", "number": 8} +{"topic": "building an online business", "number": 9} +{"topic": "learning to dance at home", "number": 5} +{"topic": "self-publishing a book", "number": 7} +{"topic": "starting an urban farm", "number": 6} +{"topic": "improving your memory", "number": 8} +{"topic": "creating a personal brand online", "number": 9} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_deep/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_deep/README.md new file mode 100644 index 00000000..bf5ab163 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_deep/README.md @@ -0,0 +1,51 @@ +## Download data +``` +wget https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl +``` + +## Run benchmark + +NOTE: This is an implementation for throughput/latency benchmark purposes. The prompts are not tuned to achieve good accuracy on the GSM-8K tasks. + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 32 +python3 bench_sglang.py --num-questions 16 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-questions 32 --backend vllm +``` + + +### Benchmark lightllm +``` +# A10G +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 16000 --port 22000 +``` + +``` +python3 bench_other.py --num-questions 32 --backend lightllm +``` + + +### Benchmark guidance +``` +python3 bench_other.py --num-questions 8 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +### Benchmark lmql + +``` +python3 bench_other.py --num-questions 8 --backend lmql --parallel 1 +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_deep/bench_other.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_deep/bench_other.py new file mode 100644 index 00000000..0ef8c636 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_deep/bench_other.py @@ -0,0 +1,222 @@ +import argparse +import ast +import json +import re +import time +from collections import Counter +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def most_frequent_number(numbers): + if not numbers: + return None + + frequency = Counter(numbers) + most_frequent = max(frequency, key=frequency.get) + return most_frequent + + +USER_PREFIX = "[INST] " +USER_SUFFIX = " [/INST]" +ASSISTANT_PREFIX = "" +ASSISTANT_SUFFIX = " " + +# Use a low temp to make the results more deterministic and the comparison more fair. +temp = 0.001 + + +def propose_plan(s, question, num_branches, call_generate): + s += ( + USER_PREFIX + + """Please generate a high-level plan for solving the following question. As the first step, just say what method and idea you will use to solve the question. You can reorganize the information in the question. Do not do the actual calculation. Keep your response concise and within 80 words. Question: """ + + question + + USER_SUFFIX + ) + + s += ASSISTANT_PREFIX + comps = call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +def execute_plan(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """The plan looks good! Now, use real numbers and do the calculation. Please solve the question step-by-step according to the high-level plan. Give me the final answer. Make your response short.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +def reflect_solution(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """Okay. Now, evaluate your own solution and give it a score on a scale of 1 to 5. Please do rigorous check of the correctness.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +def get_final_answer(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """Based on your reflection, do you change your mind? Now, give me the final answer after careful consideration.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +def tree_search(question, num_branches, call_generate): + plan_forks = propose_plan("", question, num_branches, call_generate) + + sol_states = [] + for plan in plan_forks: + forks = execute_plan(plan, num_branches, call_generate) + sol_states.extend(forks) + + ref_states = [] + for sol in sol_states: + forks = reflect_solution(sol, num_branches, call_generate) + ref_states.extend(forks) + + solutions = [] + for sol in ref_states: + ans = get_final_answer(sol, num_branches, call_generate) + solutions.append(ans) + + return solutions + + +def main(args): + lines = read_jsonl(args.data_path) + + # Construct prompts + num_branches = 2 + questions = [] + labels = [] + for i in range(len(lines[: args.num_questions])): + questions.append(lines[i]["question"]) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q, "num_branches": num_branches} for q in questions] + + # Select backend + call_generate = get_call_generate(args) + + # Run requests + states = [None] * len(questions) + + tic = time.perf_counter() + if args.backend != "lmql": + + def get_one_answer(i): + states[i] = tree_search(**arguments[i], call_generate=call_generate) + + if args.parallel == 1: + for i in tqdm(range(len(questions))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(questions)))), + total=len(questions), + ) + ) + + else: + import asyncio + + from lmql_funcs import tree_search_async + + async def get_one_answer_async(i): + states[i] = await tree_search_async( + **arguments[i], call_generate=call_generate + ) + + batches = [ + [] for _ in range((len(questions) + args.parallel - 1) // args.parallel) + ] + for i in range(len(questions)): + batches[i // args.parallel].append(i) + + loop = asyncio.get_event_loop() + for bt in tqdm(batches): + tasks = [get_one_answer_async(k) for k in bt] + loop.run_until_complete(asyncio.gather(*tasks)) + + latency = time.perf_counter() - tic + + answers_text = [] + for s in states: + answers_text.append([x for xs in s for x in xs]) + + preds = [] + for i in range(len(states)): + answers = [get_answer_value(v) for v in answers_text[i]] + preds.append(most_frequent_number(answers)) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + print(f"Latency: {latency:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", answers_text) + + with open(args.result_file, "a") as fout: + value = { + "task": "tree_of_thought_gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_deep/bench_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_deep/bench_sglang.py new file mode 100644 index 00000000..bcdb6e54 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_deep/bench_sglang.py @@ -0,0 +1,171 @@ +import argparse +import ast +import json +import re +import time +from collections import Counter + +import numpy as np + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def most_frequent_number(numbers): + if not numbers: + return None + + frequency = Counter(numbers) + most_frequent = max(frequency, key=frequency.get) + return most_frequent + + +# Use a low temp to make the results more deterministic and the comparison more fair. +temp = 0.001 + + +def propose_plan(s, question, num_branches): + s += sgl.user( + """Please generate a high-level plan for solving the following question. As the first step, just say what method and idea you will use to solve the question. You can reorganize the information in the question. Do not do the actual calculation. Keep your response concise and within 80 words. Question: """ + + question + ) + forks = s.fork(num_branches) + forks += sgl.assistant(sgl.gen("plan", max_tokens=256, temperature=temp)) + return forks + + +def execute_plan(s, num_branches): + s += sgl.user( + """The plan looks good! Now, use real numbers and do the calculation. Please solve the question step-by-step according to the high-level plan. Give me the final answer. Make your response short.""" + ) + forks = s.fork(num_branches) + forks += sgl.assistant(sgl.gen("answer", max_tokens=256, temperature=temp)) + return forks + + +def reflect_solution(s, num_branches): + s += sgl.user( + """Okay. Now, evaluate your own solution and give it a score on a scale of 1 to 5. Please do rigorous check of the correctness.""" + ) + forks = s.fork(num_branches) + forks += sgl.assistant(sgl.gen("score", max_tokens=256, temperature=temp)) + return forks + + +def get_final_answer(s, num_branches): + s += sgl.user( + """Based on your reflection, do you change your mind? Now, give me the final answer after careful consideration.""" + ) + forks = s.fork(num_branches) + forks += sgl.assistant(sgl.gen("final_answer", max_tokens=256, temperature=temp)) + return forks + + +@sgl.function +def tree_search(s, question, num_branches): + plan_forks = propose_plan(s, question, num_branches) + + sol_states = [] + for plan in plan_forks: + forks = execute_plan(plan, num_branches) + sol_states.extend(forks) + + ref_states = [] + for sol in sol_states: + forks = reflect_solution(sol, num_branches) + ref_states.extend(forks) + + solutions = [] + for sol in ref_states: + forks = get_final_answer(sol, num_branches) + solutions.append(forks) + solutions = [[s.text() for s in forks] for forks in solutions] + + return solutions + + +def main(args): + lines = read_jsonl(args.data_path) + lines = list(lines) + + # Construct prompts + num_branches = 2 + questions = [] + labels = [] + for i in range(len(lines[: args.num_questions])): + questions.append(lines[i]["question"]) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q, "num_branches": num_branches} for q in questions] + + # Select backend + backend = select_sglang_backend(args) + + # Run requests + tic = time.perf_counter() + states = tree_search.run_batch( + arguments, + temperature=0, + backend=backend, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + answers_text = [] + for s in states: + answers_text.append([x for xs in s.ret_value for x in xs]) + + preds = [] + for i in range(len(states)): + answers = [get_answer_value(v) for v in answers_text[i]] + preds.append(most_frequent_number(answers)) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + print(f"Latency: {latency:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", answers_text) + + with open(args.result_file, "a") as fout: + value = { + "task": "tree_of_thought_gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_deep/lmql_funcs.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_deep/lmql_funcs.py new file mode 100644 index 00000000..c783cdbe --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_deep/lmql_funcs.py @@ -0,0 +1,82 @@ +from bench_other import ( + ASSISTANT_PREFIX, + ASSISTANT_SUFFIX, + USER_PREFIX, + USER_SUFFIX, + temp, +) + + +async def propose_plan_async(s, question, num_branches, call_generate): + s += ( + USER_PREFIX + + """Please generate a high-level plan for solving the following question. As the first step, just say what method and idea you will use to solve the question. You can reorganize the information in the question. Do not do the actual calculation. Keep your response concise and within 80 words. Question: """ + + question + + USER_SUFFIX + ) + + s += ASSISTANT_PREFIX + comps = await call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +async def execute_plan_async(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """The plan looks good! Now, use real numbers and do the calculation. Please solve the question step-by-step according to the high-level plan. Give me the final answer. Make your response short.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = await call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +async def reflect_solution_async(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """Okay. Now, evaluate your own solution and give it a score on a scale of 1 to 5. Please do rigorous check of the correctness.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = await call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +async def get_final_answer_async(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """Based on your reflection, do you change your mind? Now, give me the final answer after careful consideration.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = await call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +async def tree_search_async(question, num_branches, call_generate): + plan_forks = await propose_plan_async("", question, num_branches, call_generate) + + sol_states = [] + for plan in plan_forks: + forks = await execute_plan_async(plan, num_branches, call_generate) + sol_states.extend(forks) + + ref_states = [] + for sol in sol_states: + forks = await reflect_solution_async(sol, num_branches, call_generate) + ref_states.extend(forks) + + solutions = [] + for sol in ref_states: + ans = await get_final_answer_async(sol, num_branches, call_generate) + solutions.append(ans) + + return solutions diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_v0/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_v0/README.md new file mode 100644 index 00000000..821bb20d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_v0/README.md @@ -0,0 +1,43 @@ +## Download data +``` +wget https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl +``` + +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 32 --parallel 16 +python3 bench_sglang.py --num-questions 10 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-questions 32 --backend vllm +``` + + +### Benchmark lightllm +``` +# A10G +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 16000 --port 22000 +``` + +``` +python3 bench_other.py --num-questions 32 --backend lightllm +``` + + +### Benchmark guidance +``` +python3 bench_other.py --num-questions 32 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_v0/bench_other.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_v0/bench_other.py new file mode 100644 index 00000000..703ecd7f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_v0/bench_other.py @@ -0,0 +1,179 @@ +import argparse +import ast +import json +import re +import time +from collections import Counter +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def most_frequent_number(numbers): + if not numbers: + return None + + frequency = Counter(numbers) + most_frequent = max(frequency, key=frequency.get) + return most_frequent + + +USER_PREFIX = "[INST] " +USER_SUFFIX = " [/INST]" +ASSISTANT_PREFIX = "" +ASSISTANT_SUFFIX = " " + +# Use a low temp to make the results more deterministic and the comparison more fair. +temp = 0.3 + + +def propose_plan(s, question, num_branches, call_generate): + s += ( + USER_PREFIX + + """Please generate a high-level plan for solving the following question. As the first step, just say what method and idea you will use to solve the question. You can reorganize the information in the question. Do not do the actual calculation. Keep your response concise and within 80 words. Question: """ + + question + + USER_SUFFIX + ) + + s += ASSISTANT_PREFIX + comps = call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +def execute_plan(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """The plan looks good! Now, use real numbers and do the calculation. Please solve the question step-by-step according to the high-level plan. Give me the final answer. Make your response short.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +def reflect_solution(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """Okay. Now you evaluate your own solution and give it a score on a scale of 1 to 5. Please do rigorous check of the correctness.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +def tree_search(question, num_branches, call_generate): + s = "" + solutions = [] + + plan_forks = propose_plan(s, question, num_branches, call_generate) + for plan in plan_forks: + sol_forks = execute_plan(plan, num_branches, call_generate) + for sol in sol_forks: + score_forks = reflect_solution(sol, num_branches, call_generate) + solutions.append(sol_forks) + + return solutions + + +def main(args): + lines = read_jsonl(args.data_path) + + # Construct prompts + num_branches = 3 + questions = [] + labels = [] + for i in range(len(lines[: args.num_questions])): + questions.append(lines[i]["question"]) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q, "num_branches": num_branches} for q in questions] + + # Select backend + call_generate = get_call_generate(args) + + # Run requests + states = [None] * len(questions) + + def get_one_answer(i): + states[i] = tree_search(**arguments[i], call_generate=call_generate) + + tic = time.perf_counter() + if args.parallel == 1: + for i in tqdm(range(len(questions))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(questions)))), + total=len(questions), + ) + ) + + latency = time.perf_counter() - tic + + answers_text = [] + for s in states: + answers_text.append([x for xs in s for x in xs]) + + preds = [] + for i in range(len(states)): + answers = [get_answer_value(v) for v in answers_text[i]] + preds.append(most_frequent_number(answers)) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + print(f"Latency: {latency:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", answers_text) + + with open(args.result_file, "a") as fout: + value = { + "task": "tree_of_thought_gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_v0/bench_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_v0/bench_sglang.py new file mode 100644 index 00000000..6d7575f3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/benchmark/tree_of_thought_v0/bench_sglang.py @@ -0,0 +1,159 @@ +import argparse +import ast +import json +import re +import time +from collections import Counter + +import numpy as np + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def most_frequent_number(numbers): + if not numbers: + return None + + frequency = Counter(numbers) + most_frequent = max(frequency, key=frequency.get) + return most_frequent + + +# Use a low temp to make the results more deterministic and the comparison more fair. +temp = 0.3 + + +def propose_plan(s, question, num_branches): + s += sgl.user( + """Please generate a high-level plan for solving the following question. As the first step, just say what method and idea you will use to solve the question. You can reorganize the information in the question. Do not do the actual calculation. Keep your response concise and within 80 words. Question: """ + + question + ) + forks = s.fork(num_branches) + forks += sgl.assistant(sgl.gen("plan", max_tokens=256, temperature=temp)) + return forks + + +def execute_plan(s, num_branches): + s += sgl.user( + """The plan looks good! Now, use real numbers and do the calculation. Please solve the question step-by-step according to the high-level plan. Give me the final answer. Make your response short.""" + ) + forks = s.fork(num_branches) + forks += sgl.assistant(sgl.gen("answer", max_tokens=256, temperature=temp)) + return forks + + +def reflect_solution(s, num_branches): + s += sgl.user( + """Okay. Now you evaluate your own solution and give it a score on a scale of 1 to 5. Please do rigorous check of the correctness.""" + ) + forks = s.fork(num_branches) + forks += sgl.assistant(sgl.gen("score", max_tokens=256, temperature=temp)) + return forks + + +@sgl.function +def tree_search(s, question, num_branches): + forks_to_join = [] + + plan_forks = propose_plan(s, question, num_branches) + forks_to_join.append(plan_forks) + + sol_states = [] + for plan in plan_forks: + forks = execute_plan(plan, num_branches) + forks_to_join.append(forks) + sol_states.extend(forks) + + for sol in sol_states: + forks = reflect_solution(sol, num_branches) + forks_to_join.append(forks) + + for f in reversed(forks_to_join): + f.join() + + +def main(args): + lines = read_jsonl(args.data_path) + + # Construct prompts + num_branches = 3 + questions = [] + labels = [] + for i in range(len(lines[: args.num_questions])): + questions.append(lines[i]["question"]) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q, "num_branches": num_branches} for q in questions] + + # Select backend + backend = select_sglang_backend(args) + + # Run requests + tic = time.perf_counter() + states = tree_search.run_batch( + arguments, + temperature=0, + backend=backend, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + answers_text = [] + for s in states: + answers_text.append([x for xs in s["answer"] for x in xs]) + + preds = [] + for i in range(len(states)): + answers = [get_answer_value(v) for v in answers_text[i]] + preds.append(most_frequent_number(answers)) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + print(f"Latency: {latency:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", answers_text) + + with open(args.result_file, "a") as fout: + value = { + "task": "tree_of_thought_gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/Dockerfile b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/Dockerfile new file mode 100644 index 00000000..7121a4d5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/Dockerfile @@ -0,0 +1,326 @@ +ARG CUDA_VERSION=12.9.1 +FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu22.04 AS base + +ARG TARGETARCH +ARG BUILD_TYPE=all +ARG BRANCH_TYPE=remote +ARG GRACE_BLACKWELL=0 + +ARG GRACE_BLACKWELL_DEEPEP_BRANCH=gb200_blog_part_2 +ARG DEEPEP_COMMIT=9af0e0d0e74f3577af1979c9b9e1ac2cad0104ee +ARG BUILD_AND_DOWNLOAD_PARALLEL=8 +ARG SGL_KERNEL_VERSION=0.3.18.post2 +ARG SGL_VERSION=0.5.6 +ARG USE_LATEST_SGLANG=0 +ARG GDRCOPY_VERSION=2.5.1 +ARG PIP_DEFAULT_INDEX +ARG UBUNTU_MIRROR +ARG GITHUB_ARTIFACTORY=github.com +ARG INSTALL_FLASHINFER_JIT_CACHE=0 +ARG FLASHINFER_VERSION=0.5.3 +ARG NVSHMEM_VERSION=3.4.5 + +ENV DEBIAN_FRONTEND=noninteractive \ + CUDA_HOME=/usr/local/cuda \ + GDRCOPY_HOME=/usr/src/gdrdrv-${GDRCOPY_VERSION}/ \ + FLASHINFER_VERSION=${FLASHINFER_VERSION} +# Add GKE default lib and bin locations. +ENV PATH="${PATH}:/usr/local/nvidia/bin" \ + LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/nvidia/lib:/usr/local/nvidia/lib64" + +# Replace Ubuntu sources if it is specified +RUN if [ -n "$UBUNTU_MIRROR" ]; then \ + sed -i "s|http://.*archive.ubuntu.com|$UBUNTU_MIRROR|g" /etc/apt/sources.list && \ + sed -i "s|http://.*security.ubuntu.com|$UBUNTU_MIRROR|g" /etc/apt/sources.list; \ +fi + +RUN --mount=type=cache,target=/var/cache/apt apt update && apt install wget -y && apt install software-properties-common -y \ + && add-apt-repository ppa:deadsnakes/ppa -y \ + && apt install python3.12-full python3.12-dev python3.10-venv -y \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 2 \ + && update-alternatives --set python3 /usr/bin/python3.12 \ + && wget https://bootstrap.pypa.io/get-pip.py \ + && python3 get-pip.py \ + # Fix for `apt-add-repository` + && cd /usr/lib/python3/dist-packages/ \ + && ln -s apt_pkg.cpython-310-*-linux-gnu.so apt_pkg.so + +# Set timezone and install all packages +RUN --mount=type=cache,target=/var/cache/apt echo 'tzdata tzdata/Areas select America' | debconf-set-selections \ + && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \ + && apt-get update && apt-get install -y --no-install-recommends \ + tzdata \ + software-properties-common netcat-openbsd kmod unzip openssh-server \ + curl wget lsof zsh ccache tmux htop git-lfs tree \ + build-essential cmake perl \ + libopenmpi-dev libnuma1 libnuma-dev numactl \ + libibverbs-dev libibverbs1 libibumad3 \ + librdmacm1 libnl-3-200 libnl-route-3-200 libnl-route-3-dev libnl-3-dev \ + ibverbs-providers infiniband-diags perftest \ + libgoogle-glog-dev libgtest-dev libjsoncpp-dev libunwind-dev \ + libboost-all-dev libssl-dev \ + libgrpc-dev libgrpc++-dev libprotobuf-dev protobuf-compiler protobuf-compiler-grpc \ + pybind11-dev \ + libhiredis-dev libcurl4-openssl-dev \ + libczmq4 libczmq-dev \ + libfabric-dev \ + patchelf \ + nvidia-dkms-550 \ + devscripts debhelper fakeroot dkms check libsubunit0 libsubunit-dev \ + && ln -sf /usr/bin/python3.12 /usr/bin/python \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean + +# Replace pip global cache if it is specified +RUN if [ -n "${PIP_DEFAULT_INDEX}" ]; then \ + python3 -m pip config set global.index-url ${PIP_DEFAULT_INDEX}; \ +fi + +# GDRCopy installation +RUN mkdir -p /tmp/gdrcopy && cd /tmp \ + && wget -q https://${GITHUB_ARTIFACTORY}/NVIDIA/gdrcopy/archive/refs/tags/v${GDRCOPY_VERSION}.tar.gz \ + && tar -xzf v${GDRCOPY_VERSION}.tar.gz && rm v${GDRCOPY_VERSION}.tar.gz \ + && cd gdrcopy-${GDRCOPY_VERSION}/packages \ + && CUDA=/usr/local/cuda ./build-deb-packages.sh \ + && dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb \ + && cd / && rm -rf /tmp/gdrcopy + +# Fix DeepEP IBGDA symlink +RUN ln -sf /usr/lib/$(uname -m)-linux-gnu/libmlx5.so.1 /usr/lib/$(uname -m)-linux-gnu/libmlx5.so + +FROM scratch AS local_src +COPY . /src + +FROM base AS build-image +# Install SGLang +# Until torch 2.9 and cu13 are stable we manually update torch if you are on CUDA 13 +WORKDIR /sgl-workspace +ARG BRANCH_TYPE +COPY --from=local_src /src /tmp/local_src +RUN if [ "$BRANCH_TYPE" = "local" ]; then \ + cp -r /tmp/local_src /sgl-workspace/sglang; \ + elif [ "$USE_LATEST_SGLANG" = "1" ]; then \ + git clone --depth=1 https://github.com/sgl-project/sglang.git /sgl-workspace/sglang; \ + else \ + git clone --depth=1 --branch v${SGL_VERSION} https://github.com/sgl-project/sglang.git /sgl-workspace/sglang; \ + fi \ + && rm -rf /tmp/local_src +RUN --mount=type=cache,target=/root/.cache/pip python3 -m pip install --upgrade pip setuptools wheel html5lib six \ + && cd sglang \ + && case "$CUDA_VERSION" in \ + 12.6.1) CUINDEX=126 ;; \ + 12.8.1) CUINDEX=128 ;; \ + 12.9.1) CUINDEX=129 ;; \ + 13.0.1) CUINDEX=130 ;; \ + *) echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 ;; \ + esac \ + && if [ "$CUDA_VERSION" = "12.6.1" ]; then \ + python3 -m pip install https://${GITHUB_ARTIFACTORY}/sgl-project/whl/releases/download/v${SGL_KERNEL_VERSION}/sgl_kernel-${SGL_KERNEL_VERSION}+cu124-cp310-abi3-manylinux2014_$(uname -m).whl --force-reinstall --no-deps \ + ; \ + elif [ "$CUDA_VERSION" = "12.8.1" ] || [ "$CUDA_VERSION" = "12.9.1" ]; then \ + python3 -m pip install sgl-kernel==${SGL_KERNEL_VERSION} \ + ; \ + elif [ "$CUDA_VERSION" = "13.0.1" ]; then \ + python3 -m pip install https://github.com/sgl-project/whl/releases/download/v${SGL_KERNEL_VERSION}/sgl_kernel-${SGL_KERNEL_VERSION}+cu130-cp310-abi3-manylinux2014_$(uname -m).whl --force-reinstall --no-deps \ + ; \ + else \ + echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 \ + ; \ + fi \ + && python3 -m pip install -e "python[${BUILD_TYPE}]" --extra-index-url https://download.pytorch.org/whl/cu${CUINDEX} \ + && if [ "$INSTALL_FLASHINFER_JIT_CACHE" = "1" ]; then \ + python3 -m pip install flashinfer-jit-cache==${FLASHINFER_VERSION} --index-url https://flashinfer.ai/whl/cu${CUINDEX} ; \ + fi \ + && FLASHINFER_CUBIN_DOWNLOAD_THREADS=${BUILD_AND_DOWNLOAD_PARALLEL} FLASHINFER_LOGGING_LEVEL=warning python3 -m flashinfer --download-cubin + +# We use Tom's DeepEP fork for GB200 for now; the 1fd57b0276311d035d16176bb0076426166e52f3 commit is https://github.com/fzyzcjy/DeepEP/tree/gb200_blog_part_2 +RUN set -eux; \ + if [ "${CUDA_VERSION%%.*}" = "12" ]; then \ + pip install nvidia-nvshmem-cu12==${NVSHMEM_VERSION} ; \ + elif [ "${CUDA_VERSION%%.*}" = "13" ]; then \ + pip install nvidia-nvshmem-cu13==${NVSHMEM_VERSION} ; \ + else \ + echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 ; \ + fi && \ + if [ "$GRACE_BLACKWELL" = "1" ]; then \ + git clone https://github.com/fzyzcjy/DeepEP.git && \ + cd DeepEP && \ + git checkout ${GRACE_BLACKWELL_DEEPEP_BRANCH} && \ + sed -i 's/#define NUM_CPU_TIMEOUT_SECS 100/#define NUM_CPU_TIMEOUT_SECS 1000/' csrc/kernels/configs.cuh && \ + cd .. ; \ + else \ + wget -q https://${GITHUB_ARTIFACTORY}/deepseek-ai/DeepEP/archive/${DEEPEP_COMMIT}.zip && \ + unzip ${DEEPEP_COMMIT}.zip && rm ${DEEPEP_COMMIT}.zip && mv DeepEP-${DEEPEP_COMMIT} DeepEP && cd DeepEP && \ + sed -i 's/#define NUM_CPU_TIMEOUT_SECS 100/#define NUM_CPU_TIMEOUT_SECS 1000/' csrc/kernels/configs.cuh && \ + cd .. ; \ + fi + +# Install DeepEP +# CTK13 requires the cccl include +RUN --mount=type=cache,target=/root/.cache/pip cd /sgl-workspace/DeepEP && \ + case "$CUDA_VERSION" in \ + 12.6.1) \ + CHOSEN_TORCH_CUDA_ARCH_LIST='9.0' \ + ;; \ + 12.8.1|12.9.1|13.0.1) \ + CHOSEN_TORCH_CUDA_ARCH_LIST='9.0;10.0;10.3' \ + ;; \ + *) \ + echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 \ + ;; \ + esac && \ + if [ "${CUDA_VERSION%%.*}" = "13" ]; then \ + sed -i "/^ include_dirs = \['csrc\/'\]/a\ include_dirs.append('${CUDA_HOME}/include/cccl')" setup.py; \ + fi && \ + TORCH_CUDA_ARCH_LIST="${CHOSEN_TORCH_CUDA_ARCH_LIST}" MAX_JOBS=${BUILD_AND_DOWNLOAD_PARALLEL} pip install --no-build-isolation . + +# Python tools +RUN --mount=type=cache,target=/root/.cache/pip python3 -m pip install \ + datamodel_code_generator \ + mooncake-transfer-engine==0.3.7.post2 \ + pre-commit \ + pytest \ + black \ + isort \ + icdiff \ + uv \ + wheel \ + scikit-build-core \ + nixl \ + py-spy + +# Some patching packages +# TODO: Remove this when torch version covers these packages +# Move cutlass-dsl to pyproject.toml after drivers on CI runners are updated +RUN --mount=type=cache,target=/root/.cache/pip if [ "${CUDA_VERSION%%.*}" = "12" ]; then \ + python3 -m pip install nvidia-nccl-cu12==2.28.3 --force-reinstall --no-deps ; \ + python3 -m pip install nvidia-cudnn-cu12==9.16.0.29 --force-reinstall --no-deps; \ + python3 -m pip install nvidia-nvshmem-cu12==${NVSHMEM_VERSION} --force-reinstall --no-deps; \ + python3 -m pip install nvidia-cutlass-dsl==4.3.0 --force-reinstall --no-deps; \ +elif [ "${CUDA_VERSION%%.*}" = "13" ]; then \ + python3 -m pip install nvidia-nccl-cu13==2.28.3 --force-reinstall --no-deps ; \ + python3 -m pip install nvidia-cublas==13.1.0.3 --force-reinstall --no-deps ; \ + python3 -m pip install nixl-cu13 ; \ + python3 -m pip install nvidia-nvshmem-cu13==${NVSHMEM_VERSION} --force-reinstall --no-deps; \ + python3 -m pip install nvidia-cutlass-dsl==4.3.1 --force-reinstall --no-deps; \ +fi + +# Install development tools and utilities +RUN --mount=type=cache,target=/var/cache/apt apt-get update && apt-get install -y \ + gdb \ + ninja-build \ + vim \ + tmux \ + htop \ + wget \ + curl \ + locales \ + lsof \ + git \ + git-lfs \ + zsh \ + tree \ + silversearcher-ag \ + cloc \ + unzip \ + pkg-config \ + libssl-dev \ + bear \ + ccache \ + less \ + gnuplot \ + && apt install -y rdma-core infiniband-diags openssh-server perftest ibverbs-providers libibumad3 libibverbs1 libnl-3-200 libnl-route-3-200 librdmacm1 \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean + +RUN --mount=type=cache,target=/var/cache/apt apt update -y \ + && apt install -y --no-install-recommends gnupg \ + && echo "deb http://developer.download.nvidia.com/devtools/repos/ubuntu2004/$(if [ "$(uname -m)" = "aarch64" ]; then echo "arm64"; else echo "amd64"; fi) /" | tee /etc/apt/sources.list.d/nvidia-devtools.list \ + && apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/$(if [ "$(uname -m)" = "aarch64" ]; then echo "arm64"; else echo "x86_64"; fi)/7fa2af80.pub \ + && apt update -y \ + && apt install nsight-systems-cli -y + +# Set up locale +RUN locale-gen en_US.UTF-8 +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US:en +ENV LC_ALL=en_US.UTF-8 + +# Install minimal Python packages +RUN --mount=type=cache,target=/root/.cache/pip python3 -m pip install --break-system-packages \ + pytest \ + black \ + isort \ + icdiff \ + scikit-build-core \ + uv \ + pre-commit \ + pandas \ + matplotlib \ + tabulate \ + termplotlib + +# Install diff-so-fancy +RUN curl -LSso /usr/local/bin/diff-so-fancy https://${GITHUB_ARTIFACTORY}/so-fancy/diff-so-fancy/releases/download/v1.4.4/diff-so-fancy \ + && chmod +x /usr/local/bin/diff-so-fancy + +# Install clang-format +RUN curl -LSso /usr/local/bin/clang-format https://${GITHUB_ARTIFACTORY}/muttleyxd/clang-tools-static-binaries/releases/download/master-32d3ac78/clang-format-16_linux-amd64 \ + && chmod +x /usr/local/bin/clang-format + +# Install clangd +RUN curl -L https://${GITHUB_ARTIFACTORY}/clangd/clangd/releases/download/18.1.3/clangd-linux-18.1.3.zip -o clangd.zip \ + && unzip clangd.zip \ + && cp -r clangd_18.1.3/bin/* /usr/local/bin/ \ + && cp -r clangd_18.1.3/lib/* /usr/local/lib/ \ + && rm -rf clangd_18.1.3 clangd.zip + +# Install CMake +RUN CMAKE_VERSION=3.31.1 \ + && ARCH=$(uname -m) \ + && CMAKE_INSTALLER="cmake-${CMAKE_VERSION}-linux-${ARCH}" \ + && wget -q "https://${GITHUB_ARTIFACTORY}/Kitware/CMake/releases/download/v${CMAKE_VERSION}/${CMAKE_INSTALLER}.tar.gz" \ + && tar -xzf "${CMAKE_INSTALLER}.tar.gz" \ + && cp -r "${CMAKE_INSTALLER}/bin/"* /usr/local/bin/ \ + && cp -r "${CMAKE_INSTALLER}/share/"* /usr/local/share/ \ + && rm -rf "${CMAKE_INSTALLER}" "${CMAKE_INSTALLER}.tar.gz" + +# Build and install sgl-router (Rust toolchain removed after build to save space) +RUN --mount=type=cache,target=/root/.cache/pip curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \ + && export PATH="/root/.cargo/bin:${PATH}" \ + && rustc --version && cargo --version \ + && python3 -m pip install maturin \ + && cd /sgl-workspace/sglang/sgl-router/bindings/python \ + && ulimit -n 65536 && maturin build --release --features vendored-openssl --out dist \ + && python3 -m pip install --force-reinstall dist/*.whl \ + && rm -rf /root/.cargo /root/.rustup target dist ~/.cargo \ + && sed -i '/\.cargo\/env/d' /root/.profile /root/.bashrc /root/.zshenv 2>/dev/null || true + + +# Add yank script +COPY --chown=root:root --chmod=755 docker/configs/yank /usr/local/bin/yank + +# Install oh-my-zsh and plugins +RUN sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" "" --unattended \ + && git clone https://github.com/zsh-users/zsh-autosuggestions ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/zsh-autosuggestions \ + && git clone https://github.com/zsh-users/zsh-syntax-highlighting.git ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/zsh-syntax-highlighting + +# Configure Vim and tmux +COPY docker/configs/.vimrc /root/.vimrc +COPY docker/configs/.tmux.conf /root/.tmux.conf + +# Configure Git +COPY docker/configs/.gitconfig /tmp/.gitconfig +RUN cat /tmp/.gitconfig >> /root/.gitconfig && rm /tmp/.gitconfig + +# Configure zsh +COPY docker/configs/.zshrc /root/.zshrc + +RUN set -euxo ; \ + curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | \ + sed "s|https://github.com|https://${GITHUB_ARTIFACTORY}|g" | \ + bash -s -- --tag 1.42.4 --to /usr/local/bin + +# Set workspace directory +WORKDIR /sgl-workspace/sglang diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/compose.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/compose.yaml new file mode 100644 index 00000000..f7ff1fbd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/compose.yaml @@ -0,0 +1,35 @@ +services: + sglang: + image: lmsysorg/sglang:latest + container_name: sglang + volumes: + - ${HOME}/.cache/huggingface:/root/.cache/huggingface + # If you use modelscope, you need mount this directory + # - ${HOME}/.cache/modelscope:/root/.cache/modelscope + restart: always + network_mode: host # required by RDMA + privileged: true # required by RDMA + # Or you can only publish port 30000 + # ports: + # - 30000:30000 + environment: + HF_TOKEN: + # if you use modelscope to download model, you need set this environment + # - SGLANG_USE_MODELSCOPE: true + entrypoint: python3 -m sglang.launch_server + command: --model-path meta-llama/Llama-3.1-8B-Instruct + --host 0.0.0.0 + --port 30000 + ulimits: + memlock: -1 + stack: 67108864 + ipc: host + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:30000/health || exit 1"] + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: ["0"] + capabilities: [gpu] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/configs/.gitconfig b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/configs/.gitconfig new file mode 100644 index 00000000..8150e40d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/configs/.gitconfig @@ -0,0 +1,30 @@ +[core] + editor = vim + whitespace = fix,-indent-with-non-tab,trailing-space,cr-at-eol + pager = diff-so-fancy | less --tabs=4 -RFX + +[color] + ui = true + +[color "diff-highlight"] + oldNormal = red bold + oldHighlight = red bold 52 + newNormal = green bold + newHighlight = green bold 22 + +[color "diff"] + meta = 11 + frag = magenta bold + commit = yellow bold + old = red bold + new = green bold + whitespace = red reverse + +[alias] + lg = log --color --graph --pretty=format:'%Cred%h%Creset - %s %Cgreen(%cr) %C(bold blue)<%an>%Creset%C(auto)%d%Creset' --abbrev-commit -- + +[http] + sslVerify = false + +[pull] + rebase = true diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/configs/.tmux.conf b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/configs/.tmux.conf new file mode 100644 index 00000000..89f20064 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/configs/.tmux.conf @@ -0,0 +1,27 @@ +# Pane border styling +set -g pane-border-style fg='#742727',bg=black +set -g pane-active-border-style fg=red,bg=black + +# Status bar styling +set -g status-style bg='#0C8A92',fg=black + +# Change prefix key to backtick +set-option -g prefix ` +unbind C-b +bind-key ` send-prefix + +# Split panes using - and = with current path +unbind '"' +bind - splitw -v -c '#{pane_current_path}' +unbind '%' +bind = splitw -h -c '#{pane_current_path}' + +# Vi mode settings +bind-key -T copy-mode-vi Y send-keys -X copy-pipe 'yank > #{pane_tty}' +set-window-option -g mode-keys vi + +# Other settings +set-option -g escape-time 0 +set-option -g base-index 1 +set-window-option -g mouse on +set -g history-limit 100000 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/configs/.vimrc b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/configs/.vimrc new file mode 100644 index 00000000..d4414000 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/configs/.vimrc @@ -0,0 +1,45 @@ +function! Yank(text) abort + let escape = system('yank', a:text) + if v:shell_error + echoerr escape + else + call writefile([escape], '/dev/tty', 'b') + endif +endfunction + +noremap y y:call Yank(@0) + +" automatically run yank(1) whenever yanking in Vim +function! CopyYank() abort + call Yank(join(v:event.regcontents, "\n")) +endfunction + +autocmd TextYankPost * call CopyYank() + +" Basic settings +set number +syntax on +set mouse=a +filetype indent on + +" Indentation +set autoindent nosmartindent +set smarttab +set expandtab +set shiftwidth=4 +set softtabstop=4 + +" Visual guides +set colorcolumn=120 +highlight ColorColumn ctermbg=5 + +" Status line +set laststatus=2 +set statusline=%<%f\ %h%m%r%=%{\"[\".(&fenc==\"\"?&enc:&fenc).((exists(\"+bomb\")\ &&\ &bomb)?\",B\":\"\").\"]\ \"}%k\ %-14.(%l,%c%V%)\ %P + +" Backspace behavior +set backspace=2 + +" Encoding +set encoding=utf-8 +set fileencoding=utf-8 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/configs/.zshrc b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/configs/.zshrc new file mode 100644 index 00000000..5c7113e0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/configs/.zshrc @@ -0,0 +1,27 @@ +export ZSH="/root/.oh-my-zsh" + +# Theme +ZSH_THEME="robbyrussell" + +# Plugins +plugins=( + git + z + zsh-autosuggestions + zsh-syntax-highlighting +) + +source $ZSH/oh-my-zsh.sh + +# Aliases +alias ll='ls -alF' +alias la='ls -A' +alias l='ls -CF' +alias vi='vim' + +# Enhanced history +HISTSIZE=10000 +SAVEHIST=10000 +setopt HIST_IGNORE_ALL_DUPS +setopt HIST_FIND_NO_DUPS +setopt INC_APPEND_HISTORY diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/configs/yank b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/configs/yank new file mode 100755 index 00000000..c9de641b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/configs/yank @@ -0,0 +1,12 @@ +#!/bin/bash +put() { + esc=$1 + test -n "$TMUX" -o -z "${TERM##screen*}" && esc="\033Ptmux;\033$esc\033\\" + printf "$esc" +} +put "\033]52;c;!\a" +buf=$( cat "$@" ) +len=$( printf %s "$buf" | wc -c ) max=74994 +test $len -gt $max && echo "$0: input is $(( len - max )) bytes too long" >&2 +put "\033]52;c;$( printf %s "$buf" | head -c $max | base64 | tr -d '\r\n' )\a" +test -n "$TMUX" && tmux set-buffer "$buf" ||: diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/diffusion.Dockerfile b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/diffusion.Dockerfile new file mode 100644 index 00000000..d8af45b7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/diffusion.Dockerfile @@ -0,0 +1,104 @@ +FROM nvidia/cuda:12.8.0-cudnn-devel-ubuntu22.04 + +ENV DEBIAN_FRONTEND=noninteractive + +SHELL ["/bin/bash", "-c"] + +WORKDIR /sgl-workspace/sglang + +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget \ + git \ + ca-certificates \ + openssh-server \ + zsh \ + vim \ + curl \ + gcc-11 \ + g++-11 \ + clang-11 \ + libnuma1 libnuma-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install oh-my-zsh and plugins +RUN sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" "" --unattended \ + && git clone https://github.com/zsh-users/zsh-autosuggestions ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/zsh-autosuggestions \ + && git clone https://github.com/zsh-users/zsh-syntax-highlighting.git ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/zsh-syntax-highlighting + + +# Set up C++20 compilers for ThunderKittens +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 --slave /usr/bin/g++ g++ /usr/bin/g++-11 + +# Set CUDA environment variables +ENV CUDA_HOME=/usr/local/cuda-12.8 +ENV PATH=${CUDA_HOME}/bin:${PATH} +ENV LD_LIBRARY_PATH=${CUDA_HOME}/lib64:$LD_LIBRARY_PATH + +# Install uv and source its environment +RUN curl -LsSf https://astral.sh/uv/install.sh | sh && \ + echo 'source $HOME/.local/bin/env' >> /root/.zshrc + +# Copy just the pyproject.toml first to leverage Docker cache +COPY python/pyproject.toml python/ + +# Create a dummy README to satisfy the installation +RUN mkdir -p python && echo "# Placeholder" > python/README.md + +# Create and activate virtual environment with specific Python version and seed +RUN source $HOME/.local/bin/env && \ + uv venv --python 3.12 --seed /opt/venv && \ + source /opt/venv/bin/activate && \ + uv pip install nvitop && \ + uv pip install --no-cache-dir --upgrade pip && \ + uv pip install --no-cache-dir --prerelease=allow ./python[diffusion] + +COPY . . + +# Install dependencies using uv and set up shell configuration +RUN source $HOME/.local/bin/env && \ + source /opt/venv/bin/activate && \ + git config --unset-all http.https://github.com/.extraheader || true && \ + echo 'source /opt/venv/bin/activate' >> /root/.zshrc && \ + echo 'if [ -n "$ZSH_VERSION" ] && [ -f ~/.zshrc ]; then . ~/.zshrc; elif [ -f ~/.bashrc ]; then . ~/.bashrc; fi' > /root/.profile + +# Set PATH to include venv bin +ENV PATH=/opt/venv/bin:$PATH + +# Configure zsh +COPY --chown=root:root <<-"EOF" /root/.zshrc +export ZSH="/root/.oh-my-zsh" + +source $HOME/.local/bin/env +source /opt/venv/bin/activate + +## Theme +ZSH_THEME="robbyrussell" + +## Plugins +plugins=( + git + z + zsh-autosuggestions + zsh-syntax-highlighting +) + +source $ZSH/oh-my-zsh.sh + +## Aliases +alias ll='ls -alF' +alias la='ls -A' +alias l='ls -CF' +alias vi='vim' + +## Enhanced history +HISTSIZE=10000 +SAVEHIST=10000 +setopt HIST_IGNORE_ALL_DUPS +setopt HIST_FIND_NO_DUPS +setopt INC_APPEND_HISTORY +EOF + + +EXPOSE 22 + +CMD ["/bin/zsh"] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/gateway.Dockerfile b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/gateway.Dockerfile new file mode 100644 index 00000000..e63bf0db --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/gateway.Dockerfile @@ -0,0 +1,79 @@ +######################## BASE IMAGE ########################## +FROM ubuntu:24.04 AS base + +ARG PYTHON_VERSION=3.12 + +# set the environment variables +ENV PATH="/root/.local/bin:${PATH}" +ENV DEBIAN_FRONTEND=noninteractive + +# uv environment variables +ENV UV_HTTP_TIMEOUT=500 +ENV VIRTUAL_ENV="/opt/venv" +ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python +ENV UV_LINK_MODE="copy" +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + + +# install dependencies +RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \ + && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \ + && apt update -y \ + && apt install -y curl \ + && rm -rf /var/lib/apt/lists/* \ + && apt clean + +# install uv +RUN curl -LsSf https://astral.sh/uv/install.sh | sh + +# install python +RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV} + +FROM scratch AS local_src +COPY . /src + +######################### BUILD IMAGE ######################### +FROM base AS build-image + +# set the environment variables +ENV PATH="/root/.cargo/bin:${PATH}" + +# install dependencies +RUN apt update -y \ + && apt install -y git build-essential libssl-dev pkg-config protobuf-compiler \ + && rm -rf /var/lib/apt/lists/* \ + && apt clean + +# install rustup from rustup.rs +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \ + && rustc --version && cargo --version && protoc --version + +# copy source code +COPY --from=local_src /src /opt/sglang + +# working directory +WORKDIR /opt/sglang/sgl-router + +# install maturin and build the wheel with vendored OpenSSL +RUN uv pip install maturin \ + && cargo clean \ + && rm -rf bindings/python/dist/ \ + && cd bindings/python \ + && ulimit -n 65536 && maturin build --release --features vendored-openssl --out dist \ + && rm -rf /root/.cache + +######################### ROUTER IMAGE ######################### +FROM base AS router-image + +# Copy the built package from the build image +COPY --from=build-image /opt/sglang/sgl-router/bindings/python/dist/*.whl dist/ + +# Build the package and install +RUN uv pip install --force-reinstall dist/*.whl + +# Clean up unnecessary files to reduce the image size +RUN rm -rf /root/.cache dist/ \ + && apt purge -y --auto-remove curl + +# Set the entrypoint to the main command +ENTRYPOINT ["python3", "-m", "sglang_router.launch_router"] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/k8s-sglang-distributed-sts.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/k8s-sglang-distributed-sts.yaml new file mode 100644 index 00000000..4252363c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/k8s-sglang-distributed-sts.yaml @@ -0,0 +1,103 @@ +# Two Nodes Sglang example + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: distributed-sglang +spec: + replicas: 2 # number of nodes/pods to run distributed sglang + selector: + matchLabels: + app: distributed-sglang + serviceName: "" + template: + metadata: + labels: + app: distributed-sglang + spec: + containers: + - name: sglang-container + image: docker.io/lmsysorg/sglang:latest + imagePullPolicy: Always # image may be replaced by official CI versioned image + command: + - /bin/bash + - -c + # please modify the sglang serving arguments below, as necessary. + # NOTE: the --expert-parallel-size is for MoE model like DeepSeek-R1 + args: + - | + python3 -m sglang.launch_server \ + --model /llm-folder \ + --dist-init-addr sglang-master-pod:5000 \ + --tensor-parallel-size 16 \ + --nnodes 2 \ + --node-rank $POD_INDEX \ + --trust-remote-code \ + --host 0.0.0.0 \ + --port 8000 \ + --enable-metrics \ + --expert-parallel-size 16 + env: + - name: POD_INDEX # reflects the node-rank + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.labels['apps.kubernetes.io/pod-index'] + - name: NCCL_DEBUG + value: INFO + resources: + limits: + nvidia.com/gpu: "8" + requests: + volumeMounts: + - mountPath: /dev/shm + name: dshm + - mountPath: /llm-folder + name: llm + securityContext: + privileged: true # to leverage RDMA/InfiniBand device, co-work with HostNetwork=true + hostNetwork: true + volumes: + - emptyDir: + medium: Memory + sizeLimit: 10Gi + name: dshm + - hostPath: + path: /llm-folder # replace with PVC or hostPath with your model weights + type: DirectoryOrCreate + name: llm + #- persistentVolumeClaim: + # claimName: llm-pvc + # name: llm +--- +apiVersion: v1 +kind: Service +metadata: + name: sglang-master-pod +spec: + type: ClusterIP + selector: + app: distributed-sglang + apps.kubernetes.io/pod-index: "0" + ports: + - name: dist-port + port: 5000 + targetPort: 5000 +--- +# the serving service +apiVersion: v1 +kind: Service +metadata: + name: sglang-serving-on-master +spec: + type: NodePort + selector: + app: distributed-sglang + apps.kubernetes.io/pod-index: "0" + ports: + - name: serving + port: 8000 + targetPort: 8000 + - name: metrics + port: 8080 + targetPort: 8080 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/k8s-sglang-service.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/k8s-sglang-service.yaml new file mode 100644 index 00000000..866d50be --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/k8s-sglang-service.yaml @@ -0,0 +1,117 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: llama-31-8b-sglang +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 30Gi + storageClassName: default # change this to your preferred storage class + volumeMode: Filesystem +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: nvidia +handler: nvidia +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: meta-llama-31-8b-instruct-sglang +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: meta-llama-31-8b-instruct-sglang + template: + metadata: + labels: + app: meta-llama-31-8b-instruct-sglang + model: meta-llama-31-8b-instruct + engine: sglang + spec: + restartPolicy: Always + runtimeClassName: nvidia + containers: + - name: meta-llama-31-8b-instruct-sglang + image: docker.io/lmsysorg/sglang:latest + imagePullPolicy: Always # IfNotPresent or Never + ports: + - containerPort: 30000 + command: ["python3", "-m", "sglang.launch_server"] + args: + [ + "--model-path", + "meta-llama/Llama-3.1-8B-Instruct", + "--host", + "0.0.0.0", + "--port", + "30000", + ] + env: + - name: HF_TOKEN + value: + resources: + limits: + nvidia.com/gpu: 1 + cpu: 8 + memory: 40Gi + requests: + cpu: 2 + memory: 16Gi + nvidia.com/gpu: 1 + volumeMounts: + - name: shm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + - name: localtime + mountPath: /etc/localtime + readOnly: true + livenessProbe: + httpGet: + path: /health + port: 30000 + initialDelaySeconds: 120 + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /health_generate + port: 30000 + initialDelaySeconds: 120 + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 3 + successThreshold: 1 + volumes: + - name: shm + emptyDir: + medium: Memory + sizeLimit: 10Gi + - name: hf-cache + persistentVolumeClaim: + claimName: llama-31-8b-sglang + - name: localtime + hostPath: + path: /etc/localtime + type: File +--- +apiVersion: v1 +kind: Service +metadata: + name: meta-llama-31-8b-instruct-sglang +spec: + selector: + app: meta-llama-31-8b-instruct-sglang + ports: + - protocol: TCP + port: 80 # port on host + targetPort: 30000 # port in container + type: LoadBalancer # change to ClusterIP if needed diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/npu.Dockerfile b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/npu.Dockerfile new file mode 100644 index 00000000..54261e70 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/npu.Dockerfile @@ -0,0 +1,100 @@ +ARG CANN_VERSION=8.3.rc1 +ARG DEVICE_TYPE=a3 +ARG OS=ubuntu22.04 +ARG PYTHON_VERSION=py3.11 + +FROM quay.io/ascend/cann:$CANN_VERSION-$DEVICE_TYPE-$OS-$PYTHON_VERSION + +# Update pip & apt sources +ARG PIP_INDEX_URL="https://pypi.org/simple/" +ARG APTMIRROR="" +ARG PYTORCH_VERSION="2.8.0" +ARG TORCHVISION_VERSION="0.23.0" +ARG PTA_URL="https://sglang-ascend.obs.cn-east-3.myhuaweicloud.com/sglang/torch_npu/torch_npu-2.8.0.post2.dev20251113-cp311-cp311-manylinux_2_28_aarch64.whl" +ARG TRITON_ASCEND_URL="https://sglang-ascend.obs.cn-east-3.myhuaweicloud.com/sglang/triton_ascend/triton_ascend-3.2.0.dev2025112116-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl" +ARG BISHENG_NAME="Ascend-BiSheng-toolkit_aarch64_20251121.run" +ARG BISHENG_URL="https://sglang-ascend.obs.cn-east-3.myhuaweicloud.com/sglang/triton_ascend/${BISHENG_NAME}" +ARG SGLANG_TAG=main +ARG ASCEND_CANN_PATH=/usr/local/Ascend/ascend-toolkit +ARG SGLANG_KERNEL_NPU_TAG=main + +ARG PIP_INSTALL="python3 -m pip install --no-cache-dir" +ARG DEVICE_TYPE + +WORKDIR /workspace + +# Define environments +ENV DEBIAN_FRONTEND=noninteractive + +RUN pip config set global.index-url $PIP_INDEX_URL +RUN if [ -n "$APTMIRROR" ];then sed -i "s|.*.ubuntu.com|$APTMIRROR|g" /etc/apt/sources.list ;fi + +# Install development tools and utilities +RUN apt-get update -y && apt upgrade -y && apt-get install -y \ + build-essential \ + cmake \ + vim \ + wget \ + curl \ + net-tools \ + zlib1g-dev \ + lld \ + clang \ + locales \ + ccache \ + openssl \ + libssl-dev \ + pkg-config \ + ca-certificates \ + && rm -rf /var/cache/apt/* \ + && rm -rf /var/lib/apt/lists/* \ + && update-ca-certificates \ + && locale-gen en_US.UTF-8 + +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US:en +ENV LC_ALL=en_US.UTF-8 + + +### Install MemFabric +RUN ${PIP_INSTALL} mf-adapter==1.0.0 +### Install SGLang Model Gateway +RUN ${PIP_INSTALL} sglang-router + + +### Install PyTorch and PTA +RUN (${PIP_INSTALL} torch==${PYTORCH_VERSION} torchvision==${TORCHVISION_VERSION} --index-url https://download.pytorch.org/whl/cpu) \ + && (${PIP_INSTALL} ${PTA_URL}) + + +# TODO: install from pypi released triton-ascend +RUN (${PIP_INSTALL} pybind11) \ + && (${PIP_INSTALL} ${TRITON_ASCEND_URL}) + +# Install SGLang +RUN git clone https://github.com/sgl-project/sglang --branch $SGLANG_TAG && \ + (cd sglang/python && rm -rf pyproject.toml && mv pyproject_other.toml pyproject.toml && ${PIP_INSTALL} -v .[srt_npu]) && \ + rm -rf sglang + +# Install Deep-ep +# pin wheel to 0.45.1 ref: https://github.com/pypa/wheel/issues/662 +RUN ${PIP_INSTALL} wheel==0.45.1 && git clone --branch $SGLANG_KERNEL_NPU_TAG https://github.com/sgl-project/sgl-kernel-npu.git \ + && export LD_LIBRARY_PATH=${ASCEND_CANN_PATH}/latest/runtime/lib64/stub:$LD_LIBRARY_PATH && \ + source ${ASCEND_CANN_PATH}/set_env.sh && \ + cd sgl-kernel-npu && \ + bash build.sh \ + && ${PIP_INSTALL} output/deep_ep*.whl output/sgl_kernel_npu*.whl \ + && cd .. && rm -rf sgl-kernel-npu \ + && cd "$(python3 -m pip show deep-ep | awk '/^Location:/ {print $2}')" && ln -s deep_ep/deep_ep_cpp*.so + +# Install CustomOps +RUN wget https://sglang-ascend.obs.cn-east-3.myhuaweicloud.com/ops/CANN-custom_ops-8.2.0.0-$DEVICE_TYPE-linux.aarch64.run && \ + chmod a+x ./CANN-custom_ops-8.2.0.0-$DEVICE_TYPE-linux.aarch64.run && \ + ./CANN-custom_ops-8.2.0.0-$DEVICE_TYPE-linux.aarch64.run --quiet --install-path=/usr/local/Ascend/ascend-toolkit/latest/opp && \ + wget https://sglang-ascend.obs.cn-east-3.myhuaweicloud.com/ops/custom_ops-1.0.$DEVICE_TYPE-cp311-cp311-linux_aarch64.whl && \ + ${PIP_INSTALL} ./custom_ops-1.0.$DEVICE_TYPE-cp311-cp311-linux_aarch64.whl + +# Install Bisheng +RUN wget -O "${BISHENG_NAME}" "${BISHENG_URL}" && chmod a+x "${BISHENG_NAME}" && "./${BISHENG_NAME}" --install && rm "${BISHENG_NAME}" + +CMD ["/bin/bash"] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/rocm.Dockerfile b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/rocm.Dockerfile new file mode 100644 index 00000000..b7fcf770 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/rocm.Dockerfile @@ -0,0 +1,318 @@ +# Usage (to build SGLang ROCm docker image): +# docker build --build-arg SGL_BRANCH=v0.5.6 --build-arg GPU_ARCH=gfx942 -t v0.5.6-rocm630-mi30x -f rocm.Dockerfile . +# docker build --build-arg SGL_BRANCH=v0.5.6 --build-arg GPU_ARCH=gfx942-rocm700 -t v0.5.6-rocm700-mi30x -f rocm.Dockerfile . +# docker build --build-arg SGL_BRANCH=v0.5.6 --build-arg GPU_ARCH=gfx950 -t v0.5.6-rocm700-mi35x -f rocm.Dockerfile . + + +# Default base images +ARG BASE_IMAGE_942="rocm/sgl-dev:vllm20250114" +ARG BASE_IMAGE_942_ROCM700="rocm/sgl-dev:rocm7-vllm-20250904" +ARG BASE_IMAGE_950="rocm/sgl-dev:rocm7-vllm-20250904" + +# This is necessary for scope purpose +ARG GPU_ARCH=gfx950 + +# =============================== +# Base image 942 with rocm630 and args +FROM $BASE_IMAGE_942 AS gfx942 +ENV BUILD_VLLM="0" +ENV BUILD_TRITON="1" +ENV BUILD_LLVM="0" +ENV BUILD_AITER_ALL="1" +ENV BUILD_MOONCAKE="1" +ENV AITER_COMMIT="v0.1.4" +ENV NO_DEPS_FLAG="" + +# =============================== +# Base image 942 and args +FROM $BASE_IMAGE_942_ROCM700 AS gfx942-rocm700 +ENV BUILD_VLLM="0" +ENV BUILD_TRITON="0" +ENV BUILD_LLVM="0" +ENV BUILD_AITER_ALL="1" +ENV BUILD_MOONCAKE="1" +ENV AITER_COMMIT="v0.1.7.post1" +ENV NO_DEPS_FLAG="" + +# =============================== +# Base image 950 and args +FROM $BASE_IMAGE_950 AS gfx950 +ENV BUILD_VLLM="0" +ENV BUILD_TRITON="0" +ENV BUILD_LLVM="0" +ENV BUILD_AITER_ALL="0" +ENV BUILD_MOONCAKE="1" +ENV AITER_COMMIT="v0.1.7.post2" +ENV NO_DEPS_FLAG="" +# =============================== +# Chosen arch and args +FROM ${GPU_ARCH} + +# This is necessary for scope purpose, again +ARG GPU_ARCH=gfx950 +ENV GPU_ARCH_LIST=${GPU_ARCH%-*} + +ARG SGL_REPO="https://github.com/sgl-project/sglang.git" +ARG SGL_DEFAULT="main" +ARG SGL_BRANCH=${SGL_DEFAULT} + +ARG TRITON_REPO="https://github.com/ROCm/triton.git" +ARG TRITON_COMMIT="improve_fa_decode_3.0.0" + +ARG AITER_REPO="https://github.com/ROCm/aiter.git" + +ARG LLVM_REPO="https://github.com/jrbyrnes/llvm-project.git" +ARG LLVM_BRANCH="MainOpSelV2" +ARG LLVM_COMMIT="6520ace8227ffe2728148d5f3b9872a870b0a560" + +ARG MOONCAKE_REPO="https://github.com/kvcache-ai/Mooncake.git" +ARG MOONCAKE_COMMIT="b6a841dc78c707ec655a563453277d969fb8f38d" + +ARG TILELANG_REPO="https://github.com/HaiShaw/tilelang.git" +ARG TILELANG_BRANCH="dsv32-mi35x" +ARG TILELANG_COMMIT="ae938cf885743f165a19656d1122ad42bb0e30b8" + +ARG FHT_REPO="https://github.com/jeffdaily/fast-hadamard-transform.git" +ARG FHT_BRANCH="rocm" +ARG FHT_COMMIT="46efb7d776d38638fc39f3c803eaee3dd7016bd1" +USER root + +# Install some basic utilities +RUN python -m pip install --upgrade pip && pip install setuptools_scm +RUN apt-get purge -y sccache; python -m pip uninstall -y sccache; rm -f "$(which sccache)" + +WORKDIR /sgl-workspace + +# ----------------------- +# llvm +RUN if [ "$BUILD_LLVM" = "1" ]; then \ + ENV HIP_CLANG_PATH="/sgl-workspace/llvm-project/build/bin/" \ + git clone --single-branch ${LLVM_REPO} -b ${LLVM_BRANCH} \ + && cd llvm-project \ + && git checkout ${LLVM_COMMIT} \ + && mkdir build \ + && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=1 -DLLVM_TARGETS_TO_BUILD="AMDGPU;X86" -DLLVM_ENABLE_PROJECTS="clang;lld;" -DLLVM_ENABLE_RUNTIMES="compiler-rt" ../llvm \ + && make -j$(nproc); \ + fi + +# ----------------------- +# AITER +RUN pip uninstall -y aiter +RUN git clone ${AITER_REPO} \ + && cd aiter \ + && git checkout ${AITER_COMMIT} \ + && git submodule update --init --recursive +RUN cd aiter \ + && echo "[AITER] GPU_ARCH=${GPU_ARCH}" \ + && if [ "$BUILD_AITER_ALL" = "1" ] && [ "$BUILD_LLVM" = "1" ]; then \ + sh -c "HIP_CLANG_PATH=/sgl-workspace/llvm-project/build/bin/ PREBUILD_KERNELS=1 GPU_ARCHS=$GPU_ARCH_LIST python setup.py develop"; \ + elif [ "$BUILD_AITER_ALL" = "1" ]; then \ + sh -c "PREBUILD_KERNELS=1 GPU_ARCHS=$GPU_ARCH_LIST python setup.py develop"; \ + else \ + sh -c "GPU_ARCHS=$GPU_ARCH_LIST python setup.py develop"; \ + fi + +# ----------------------- +# Triton +RUN if [ "$BUILD_TRITON" = "1" ]; then \ + pip uninstall -y triton \ + && git clone ${TRITON_REPO} \ + && cd triton \ + && git checkout ${TRITON_COMMIT} \ + && cd python \ + && python setup.py install; \ + fi + +# ----------------------- +# Build vLLM +ARG VLLM_REPO="https://github.com/ROCm/vllm.git" +ARG VLLM_BRANCH="9f6b92db47c3444b7a7d67451ba0c3a2d6af4c2c" +RUN if [ "$BUILD_VLLM" = "1" ]; then \ + git clone ${VLLM_REPO} \ + && cd vllm \ + && git checkout ${VLLM_BRANCH} \ + && python -m pip install -r requirements/rocm.txt \ + && python setup.py clean --all \ + && python setup.py develop; \ + fi + +# ----------------------- +# Build Mooncake +ENV PATH=$PATH:/usr/local/go/bin + +RUN if [ "$BUILD_MOONCAKE" = "1" ]; then \ + apt update && apt install -y zip unzip wget && \ + apt install -y gcc make libtool autoconf librdmacm-dev rdmacm-utils infiniband-diags ibverbs-utils perftest ethtool libibverbs-dev rdma-core && \ + apt install -y openssh-server openmpi-bin openmpi-common libopenmpi-dev && \ + git clone ${MOONCAKE_REPO} && \ + cd Mooncake && \ + git checkout ${MOONCAKE_COMMIT} && \ + git submodule update --init --recursive && \ + bash dependencies.sh -y && \ + rm -rf /usr/local/go && \ + wget https://go.dev/dl/go1.22.2.linux-amd64.tar.gz && \ + tar -C /usr/local -xzf go1.22.2.linux-amd64.tar.gz && \ + rm go1.22.2.linux-amd64.tar.gz && \ + mkdir -p build && \ + cd build && \ + cmake .. -DUSE_HIP=ON -DUSE_ETCD=ON && \ + make -j "$(nproc)" && make install; \ + fi + +# ----------------------- +# Build SGLang +ARG BUILD_TYPE=all + +RUN pip install IPython \ + && pip install orjson \ + && pip install python-multipart \ + && pip install torchao==0.9.0 \ + && pip install pybind11 + +RUN pip uninstall -y sgl_kernel sglang +RUN git clone ${SGL_REPO} \ + && cd sglang \ + && if [ "${SGL_BRANCH}" = ${SGL_DEFAULT} ]; then \ + echo "Using ${SGL_DEFAULT}, default branch."; \ + git checkout ${SGL_DEFAULT}; \ + else \ + echo "Using ${SGL_BRANCH} branch."; \ + git checkout ${SGL_BRANCH}; \ + fi \ + && cd sgl-kernel \ + && rm -f pyproject.toml \ + && mv pyproject_rocm.toml pyproject.toml \ + && AMDGPU_TARGET=$GPU_ARCH_LIST python setup_rocm.py install \ + && cd .. \ + && rm -rf python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml \ + && if [ "$BUILD_TYPE" = "srt" ]; then \ + python -m pip --no-cache-dir install -e "python[srt_hip]" ${NO_DEPS_FLAG}; \ + else \ + python -m pip --no-cache-dir install -e "python[all_hip]" ${NO_DEPS_FLAG}; \ + fi + +RUN python -m pip cache purge + +# Copy config files to support MI300X in virtualized environments (MI300X_VF). Symlinks will not be created in image build. +RUN find /sgl-workspace/sglang/python/sglang/srt/layers/quantization/configs/ \ + /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/ \ + -type f -name '*MI300X*' | xargs -I {} sh -c 'vf_config=$(echo "$1" | sed "s/MI300X/MI300X_VF/"); cp "$1" "$vf_config"' -- {} + +# Install Rust toolchain for sgl-router +ENV PATH="/root/.cargo/bin:${PATH}" +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \ + && rustc --version && cargo --version + +# Build and install sgl-router +RUN python3 -m pip install --no-cache-dir setuptools-rust \ + && cd /sgl-workspace/sglang/sgl-router/bindings/python \ + && cargo build --release \ + && python3 -m pip install --no-cache-dir . \ + && rm -rf /root/.cache + +# ----------------------- +# TileLang +ENV DEBIAN_FRONTEND=noninteractive +ENV LIBGL_ALWAYS_INDIRECT=1 +RUN echo "LC_ALL=en_US.UTF-8" >> /etc/environment + +RUN /bin/bash -lc 'set -euo pipefail; \ + # Build TileLang only for gfx950 + if [ "${GPU_ARCH:-}" != "gfx950" ]; then \ + echo "[TileLang] Skipping (GPU_ARCH=${GPU_ARCH:-unset})"; \ + exit 0; \ + fi; \ + echo "[TileLang] Building TileLang for ${GPU_ARCH}"; \ + \ + # System dependencies (NO llvm-dev to avoid llvm-config-16 shadowing) + apt-get update && apt-get install -y --no-install-recommends \ + build-essential git wget curl ca-certificates gnupg \ + libgtest-dev libgmock-dev \ + libprotobuf-dev protobuf-compiler libgflags-dev libsqlite3-dev \ + python3 python3-dev python3-setuptools python3-pip \ + gcc libtinfo-dev zlib1g-dev libedit-dev libxml2-dev \ + cmake ninja-build pkg-config libstdc++6 \ + && rm -rf /var/lib/apt/lists/*; \ + \ + # Build GoogleTest static libs (Ubuntu package ships sources only) + cmake -S /usr/src/googletest -B /tmp/build-gtest -DBUILD_GTEST=ON -DBUILD_GMOCK=ON -DCMAKE_BUILD_TYPE=Release && \ + cmake --build /tmp/build-gtest -j"$(nproc)" && \ + cp -v /tmp/build-gtest/lib/*.a /usr/lib/x86_64-linux-gnu/ && \ + rm -rf /tmp/build-gtest; \ + \ + # Keep setuptools < 80 (compat with base image) + python3 -m pip install --upgrade "setuptools>=77.0.3,<80" wheel cmake ninja && \ + python3 -m pip cache purge || true; \ + \ + # Locate ROCm llvm-config; fallback to installing LLVM 18 if missing + LLVM_CONFIG_PATH=""; \ + for p in /opt/rocm/llvm/bin/llvm-config /opt/rocm/llvm-*/bin/llvm-config /opt/rocm-*/llvm*/bin/llvm-config; do \ + if [ -x "$p" ]; then LLVM_CONFIG_PATH="$p"; break; fi; \ + done; \ + if [ -z "$LLVM_CONFIG_PATH" ]; then \ + echo "[TileLang] ROCm llvm-config not found; installing LLVM 18..."; \ + curl -fsSL https://apt.llvm.org/llvm.sh -o /tmp/llvm.sh; \ + chmod +x /tmp/llvm.sh; \ + /tmp/llvm.sh 18; \ + LLVM_CONFIG_PATH="$(command -v llvm-config-18)"; \ + if [ -z "$LLVM_CONFIG_PATH" ]; then echo "ERROR: llvm-config-18 not found after install"; exit 1; fi; \ + fi; \ + echo "[TileLang] Using LLVM_CONFIG at: $LLVM_CONFIG_PATH"; \ + export PATH="$(dirname "$LLVM_CONFIG_PATH"):/usr/local/bin:${PATH}"; \ + export LLVM_CONFIG="$LLVM_CONFIG_PATH"; \ + \ + # Optional shim for tools that expect llvm-config-16 + mkdir -p /usr/local/bin && \ + printf "#!/usr/bin/env bash\nexec \"%s\" \"\$@\"\n" "$LLVM_CONFIG_PATH" > /usr/local/bin/llvm-config-16 && \ + chmod +x /usr/local/bin/llvm-config-16; \ + \ + # TVM Python bits need Cython + python3 -m pip install --no-cache-dir "cython>=0.29.36,<3.0"; \ + \ + # Clone + pin TileLang (bundled TVM), then build + git clone --recursive --branch "${TILELANG_BRANCH}" "${TILELANG_REPO}" /opt/tilelang && \ + cd /opt/tilelang && \ + git fetch --depth=1 origin "${TILELANG_COMMIT}" || true && \ + git checkout -f "${TILELANG_COMMIT}" && \ + git submodule update --init --recursive && \ + export CMAKE_ARGS="-DLLVM_CONFIG=${LLVM_CONFIG} ${CMAKE_ARGS:-}" && \ + bash ./install_rocm.sh' + +# ----------------------- +# Hadamard-transform (HIP build) +RUN /bin/bash -lc 'set -euo pipefail; \ + git clone --branch "${FHT_BRANCH}" "${FHT_REPO}" fast-hadamard-transform; \ + cd fast-hadamard-transform; \ + git checkout -f "${FHT_COMMIT}"; \ + python setup.py install' + +# ----------------------- +# Python tools +RUN python3 -m pip install --no-cache-dir \ + py-spy \ + pre-commit + +# ----------------------- +# Performance environment variable. + +ENV HIP_FORCE_DEV_KERNARG=1 +ENV HSA_NO_SCRATCH_RECLAIM=1 +ENV SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1 +ENV SGLANG_INT4_WEIGHT=0 +ENV SGLANG_MOE_PADDING=1 +ENV SGLANG_ROCM_DISABLE_LINEARQUANT=0 +ENV SGLANG_ROCM_FUSED_DECODE_MLA=1 +ENV SGLANG_SET_CPU_AFFINITY=1 +ENV SGLANG_USE_AITER=1 +ENV SGLANG_USE_ROCM700A=1 + +ENV NCCL_MIN_NCHANNELS=112 +ENV VLLM_FP8_PADDING=1 +ENV VLLM_FP8_ACT_PADDING=1 +ENV VLLM_FP8_WEIGHT_PADDING=1 +ENV VLLM_FP8_REDUCE_CONV=1 +ENV TORCHINDUCTOR_MAX_AUTOTUNE=1 +ENV TORCHINDUCTOR_MAX_AUTOTUNE_POINTWISE=1 + +CMD ["/bin/bash"] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/sagemaker.Dockerfile b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/sagemaker.Dockerfile new file mode 100644 index 00000000..5fbff509 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/sagemaker.Dockerfile @@ -0,0 +1,6 @@ +FROM lmsysorg/sglang:latest + +COPY serve /usr/bin/serve +RUN chmod 777 /usr/bin/serve + +ENTRYPOINT [ "/usr/bin/serve" ] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/serve b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/serve new file mode 100755 index 00000000..9f464bf4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/serve @@ -0,0 +1,34 @@ +#!/bin/bash +echo "Starting server" + +PREFIX="SM_SGLANG_" +ARG_PREFIX="--" + +ARGS=() + +while IFS='=' read -r key value; do + arg_name=$(echo "${key#"${PREFIX}"}" | tr '[:upper:]' '[:lower:]' | tr '_' '-') + + ARGS+=("${ARG_PREFIX}${arg_name}") + if [ -n "$value" ]; then + ARGS+=("$value") + fi +done < <(env | grep "^${PREFIX}") + +# Add default port only if not already set +if ! [[ " ${ARGS[@]} " =~ " --port " ]]; then + ARGS+=(--port "${SM_SGLANG_PORT:-8080}") +fi + +# Add default host only if not already set +if ! [[ " ${ARGS[@]} " =~ " --host " ]]; then + ARGS+=(--host "${SM_SGLANG_HOST:-0.0.0.0}") +fi + +# Add default model-path only if not already set +if ! [[ " ${ARGS[@]} " =~ " --model-path " ]]; then + ARGS+=(--model-path "${SM_SGLANG_MODEL_PATH:-/opt/ml/model}") +fi + +echo "Running command: exec python3 -m sglang.launch_server ${ARGS[@]}" +exec python3 -m sglang.launch_server "${ARGS[@]}" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/xeon.Dockerfile b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/xeon.Dockerfile new file mode 100644 index 00000000..da2f4a0f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/xeon.Dockerfile @@ -0,0 +1,56 @@ +FROM ubuntu:24.04 +SHELL ["/bin/bash", "-c"] + +ARG SGLANG_REPO=https://github.com/sgl-project/sglang.git +ARG VER_SGLANG=main + +ARG VER_TORCH=2.9.0 +ARG VER_TORCHVISION=0.24.0 +ARG VER_TRITON=3.5.0 + +RUN apt-get update && \ + apt-get full-upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + ca-certificates \ + git \ + curl \ + wget \ + vim \ + gcc \ + g++ \ + make \ + libsqlite3-dev \ + google-perftools \ + libtbb-dev \ + libnuma-dev \ + numactl + +WORKDIR /opt + +RUN curl -LsSf https://astral.sh/uv/install.sh | sh && \ + source $HOME/.local/bin/env && \ + uv venv --python 3.12 + +RUN echo -e '[[index]]\nname = "torch"\nurl = "https://download.pytorch.org/whl/cpu"\n\n[[index]]\nname = "torchvision"\nurl = "https://download.pytorch.org/whl/cpu"\n\n[[index]]\nname = "triton"\nurl = "https://download.pytorch.org/whl/cpu"' > .venv/uv.toml + +ENV UV_CONFIG_FILE=/opt/.venv/uv.toml + +WORKDIR /sgl-workspace +RUN source $HOME/.local/bin/env && \ + source /opt/.venv/bin/activate && \ + git clone ${SGLANG_REPO} sglang && \ + cd sglang && \ + git checkout ${VER_SGLANG} && \ + cd python && \ + cp pyproject_cpu.toml pyproject.toml && \ + uv pip install . && \ + uv pip install torch==${VER_TORCH} torchvision==${VER_TORCHVISION} triton==${VER_TRITON} --force-reinstall && \ + cd ../sgl-kernel && \ + cp pyproject_cpu.toml pyproject.toml && \ + uv pip install . + +ENV SGLANG_USE_CPU_ENGINE=1 +ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc.so.4:/usr/lib/x86_64-linux-gnu/libtbbmalloc.so:/opt/.venv/lib/libiomp5.so +RUN echo 'source /opt/.venv/bin/activate' >> /root/.bashrc + +WORKDIR /sgl-workspace/sglang diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/xpu.Dockerfile b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/xpu.Dockerfile new file mode 100644 index 00000000..5aa57b3d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docker/xpu.Dockerfile @@ -0,0 +1,73 @@ +# If the device is Battlemage, we need to set UBUNTU_VERSION to 24.10 + +# Usage: docker build --build-arg UBUNTU_VERSION=24.04 --build-arg PYTHON_VERSION=3.10 -t sglang:xpu_kernel -f xpu.Dockerfile --no-cache . + +# Use Intel deep learning essentials base image with Ubuntu 24.04 +FROM intel/deep-learning-essentials:2025.2.2-0-devel-ubuntu24.04 + +# Avoid interactive prompts during package install +ENV DEBIAN_FRONTEND=noninteractive + +# Define build arguments +ARG PYTHON_VERSION=3.10 + +ARG SG_LANG_REPO=https://github.com/sgl-project/sglang.git +ARG SG_LANG_BRANCH=main + +ARG SG_LANG_KERNEL_REPO=https://github.com/sgl-project/sgl-kernel-xpu.git +ARG SG_LANG_KERNEL_BRANCH=main + +RUN useradd -m -d /home/sdp -s /bin/bash sdp && \ + chown -R sdp:sdp /home/sdp + +# Switch to non-root user 'sdp' +USER sdp + +# Set HOME and WORKDIR to user's home directory +ENV HOME=/home/sdp +WORKDIR /home/sdp + +RUN curl -fsSL -v -o miniforge.sh -O https://github.com/conda-forge/miniforge/releases/download/25.1.1-0/Miniforge3-Linux-x86_64.sh && \ + bash miniforge.sh -b -p ./miniforge3 && \ + rm miniforge.sh && \ + # Initialize conda environment and install pip + . ./miniforge3/bin/activate && \ + conda create -y -n py${PYTHON_VERSION} python=${PYTHON_VERSION} && \ + conda activate py${PYTHON_VERSION} && \ + conda install pip && \ + # Append environment activation to .bashrc for interactive shells + echo ". /home/sdp/miniforge3/bin/activate; conda activate py${PYTHON_VERSION}; . /opt/intel/oneapi/setvars.sh; cd /home/sdp" >> /home/sdp/.bashrc + +USER root +RUN apt-get update && apt install -y intel-ocloc + +# Switch back to user sdp +USER sdp + +RUN --mount=type=secret,id=github_token \ + cd /home/sdp && \ + . /home/sdp/miniforge3/bin/activate && \ + conda activate py${PYTHON_VERSION} && \ + pip3 install torch==2.9.0+xpu torchao torchvision torchaudio pytorch-triton-xpu==3.5.0 --index-url https://download.pytorch.org/whl/xpu + +RUN --mount=type=secret,id=github_token \ + cd /home/sdp && \ + . /home/sdp/miniforge3/bin/activate && \ + conda activate py${PYTHON_VERSION} && \ + echo "Cloning ${SG_LANG_BRANCH} from ${SG_LANG_REPO}" && \ + git clone --branch ${SG_LANG_BRANCH} --single-branch ${SG_LANG_REPO} && \ + cd sglang && cd python && \ + cp pyproject_xpu.toml pyproject.toml && \ + pip install . && \ + pip install xgrammar --no-deps && \ + pip install msgspec blake3 py-cpuinfo compressed_tensors gguf partial_json_parser einops --root-user-action=ignore && \ + conda install libsqlite=3.48.0 -y && \ + # Add environment setup commands to .bashrc again (in case it was overwritten) + echo ". /home/sdp/miniforge3/bin/activate; conda activate py${PYTHON_VERSION}; cd /home/sdp" >> /home/sdp/.bashrc + +# Use bash as default shell with initialization from .bashrc +SHELL ["bash", "-c"] + +# Start an interactive bash shell with all environment set up +USER sdp +CMD ["bash", "-c", "source /home/sdp/.bashrc && exec bash"] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/Makefile b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/Makefile new file mode 100644 index 00000000..6b8792c4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/Makefile @@ -0,0 +1,58 @@ +# Minimal Makefile for Sphinx documentation +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SPHINXAUTOBUILD ?= sphinx-autobuild +SOURCEDIR = . +BUILDDIR = _build +PORT ?= 8003 + +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + @echo "" + @echo "Additional targets:" + @echo " serve to build and serve documentation with auto-build and live reload" + +# Compile Notebook files and record execution time +compile: + @set -e; \ + echo "Starting Notebook compilation..."; \ + mkdir -p logs; \ + echo "Notebook execution timings:" > logs/timing.log; \ + START_TOTAL=$$(date +%s); \ + find $(SOURCEDIR) -path "*/_build/*" -prune -o -name "*.ipynb" -print0 | \ + parallel -0 -j3 --halt soon,fail=1 ' \ + NB_NAME=$$(basename {}); \ + START_TIME=$$(date +%s); \ + retry --delay=0 --times=2 -- \ + jupyter nbconvert --to notebook --execute --inplace "{}" \ + --ExecutePreprocessor.timeout=600 \ + --ExecutePreprocessor.kernel_name=python3; \ + RET_CODE=$$?; \ + END_TIME=$$(date +%s); \ + ELAPSED_TIME=$$((END_TIME - START_TIME)); \ + echo "$${NB_NAME}: $${ELAPSED_TIME}s" >> logs/timing.log; \ + exit $$RET_CODE' || exit 1; \ + END_TOTAL=$$(date +%s); \ + TOTAL_ELAPSED=$$((END_TOTAL - START_TOTAL)); \ + echo "---------------------------------" >> logs/timing.log; \ + echo "Total execution time: $${TOTAL_ELAPSED}s" >> logs/timing.log; \ + echo "All Notebook execution timings:" && cat logs/timing.log + +# Serve documentation with auto-build and live reload +serve: + @echo "Starting auto-build server at http://0.0.0.0:$(PORT)" + @$(SPHINXAUTOBUILD) "$(SOURCEDIR)" "$(BUILDDIR)/html" \ + --host 0.0.0.0 \ + --port $(PORT) \ + --watch $(SOURCEDIR) \ + --re-ignore ".*\.(ipynb_checkpoints|pyc|pyo|pyd|git)" + +.PHONY: help Makefile compile clean serve + +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +clean: + find . -name "*.ipynb" -exec nbstripout {} \; + rm -rf $(BUILDDIR) + rm -rf logs diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/README.md new file mode 100644 index 00000000..2cad1335 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/README.md @@ -0,0 +1,55 @@ +# SGLang Documentation + +We recommend new contributors start from writing documentation, which helps you quickly understand SGLang codebase. +Most documentation files are located under the `docs/` folder. + +## Docs Workflow + +### Install Dependency + +```bash +apt-get update && apt-get install -y pandoc parallel retry +pip install -r requirements.txt +``` + +### Update Documentation + +Update your Jupyter notebooks in the appropriate subdirectories under `docs/`. If you add new files, remember to update `index.rst` (or relevant `.rst` files) accordingly. + +- **`pre-commit run --all-files`** manually runs all configured checks, applying fixes if possible. If it fails the first time, re-run it to ensure lint errors are fully resolved. Make sure your code passes all checks **before** creating a Pull Request. + +```bash +# 1) Compile all Jupyter notebooks +make compile # This step can take a long time (10+ mins). You can consider skipping this step if you can make sure your added files are correct. +make html + +# 2) Compile and Preview documentation locally with auto-build +# This will automatically rebuild docs when files change +# Open your browser at the displayed port to view the docs +bash serve.sh + +# 2a) Alternative ways to serve documentation +# Directly use make serve +make serve +# With custom port +PORT=8080 make serve + +# 3) Clean notebook outputs +# nbstripout removes notebook outputs so your PR stays clean +pip install nbstripout +find . -name '*.ipynb' -exec nbstripout {} \; + +# 4) Pre-commit checks and create a PR +# After these checks pass, push your changes and open a PR on your branch +pre-commit run --all-files +``` +--- + +## Documentation Style Guidelines + +- For common functionalities, we prefer **Jupyter Notebooks** over Markdown so that all examples can be executed and validated by our docs CI pipeline. For complex features (e.g., distributed serving), Markdown is preferred. +- Keep in mind the documentation execution time when writing interactive Jupyter notebooks. Each interactive notebook will be run and compiled against every commit to ensure they are runnable, so it is important to apply some tips to reduce the documentation compilation time: + - Use small models (e.g., `qwen/qwen2.5-0.5b-instruct`) for most cases to reduce server launch time. + - Reuse the launched server as much as possible to reduce server launch time. +- Do not use absolute links (e.g., `https://docs.sglang.io/get_started/install.html`). Always prefer relative links (e.g., `../get_started/install.md`). +- Follow the existing examples to learn how to launch a server, send a query and other common styles. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/_static/css/custom_log.css b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/_static/css/custom_log.css new file mode 100644 index 00000000..61f65d01 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/_static/css/custom_log.css @@ -0,0 +1,29 @@ +.output_area { + color: #615656; +} + +table.autosummary td { + width: 50% + } + + img.align-center { + display: block; + margin-left: auto; + margin-right: auto; +} + +.output_area.stderr { + color: #d3d3d3 !important; +} + +.output_area.stdout { + color: #d3d3d3 !important; +} + +div.output_area.stderr { + color: #d3d3d3 !important; +} + +div.output_area.stdout { + color: #d3d3d3 !important; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/_static/css/readthedocs.css b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/_static/css/readthedocs.css new file mode 100644 index 00000000..aca6649b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/_static/css/readthedocs.css @@ -0,0 +1,9 @@ +table.autosummary td { + width: 50% +} + +img.align-center { + display: block; + margin-left: auto; + margin-right: auto; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/_static/image/logo.ico b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/_static/image/logo.ico new file mode 100644 index 0000000000000000000000000000000000000000..7a0b5b2f00e4b626e250e5c4a03de4136d96943a GIT binary patch literal 38078 zcmeHQcX(B0o{l?zIQz`A&+L)}#lGwm-JNk~$604reROtoq=YJnVxbC{kuFFRK|rbj z1nJET1Of;eq=uB6p4^*q(|d1(^j`7Z_jkfg&MkrDdgVUP?)i9L&bjA)<(&8XeZTgd zBrY!Z@!zy*F8u!q7vC>kTn4+ixI9Vll8cUz>q^<#eYn&A09XYon0?iId+a#2#zKISlx4Sjp>X#W@TZIBdOhhSo+U z1Cm4T(wJ}>1Cmb`Uq~y?VLz)zsA3F8F23Cn*3}0JC_0Hj;_hS#=mnJaPH`u{A>I>mjUOF zuF3yh`2U`}BR>>(7{V_8Z}En@ZkdB0?^>ivb4w?R%9>zEs>IdMQk=V1h~s{FICef4 zCobgUoPQC5^%Y3VYrxIg4z#x4!ma*vt{PqH->7UsjPWME3oM4ug**A$5tsWZ^GfzS@`mJ`hc%V12eMnzStGLIJH z42XYuuE_YC_`*9K+YYCiVfXP27*cD{*=74#s%zVD_)HewUK)v)W`yE56NB;S$Ur3 z*MjiS$l#v-9~`OUm;w0B>%sWbj1YLPkHg{f1&GURwAp_`S~cEW6oySlvatO`9+BHG z?C{CMg3U&ZnG=XLUnJnv#XKaWR-vQgmikuZd_eqH*R|oVYh&>7muXnNHyvyCW#F^J zS=ewW3-dlpz`Tv|s3K3citzHsHv{nV5m9iRmUGGBFgac%n-PZ2*IH7@k&27`+uGNkBfurh;Tgg z>NU(-mw?Oqdfec8Q7uJzs*YCkx@_TZD6AHTayz9VhcKYmEVARnnz(5xp~UVFU(^ zjm6-}Y37`B4&2}9hB;qpaF@9l6@{n91z_c#Oz4stQBA#JsWNmi<@nFZ!Eha8FzegY zpSz4a;TNmZm2=7Z}3h!~b9S|6D+F{6Uhu(xNo~t0= zUDelN3i)5M$$)()GY}GU1EK>g%7FQwzrldj2eROGx(K^D54+BlVEMi*tlpoA*=u9_ zx1tAzr9i&|68ZmJvN7u0iJT*Q58^6j& zL~0Y7T6@>B$i)QCt+Z3Oai7F-#^~WbA`}mK=oGk(42L_%Hq~>RXvDw25sY15mm)T+ z*-FRl^~r$O@m$2`brKzjOl!sZgPHg%b=SPl;<0dZ0zTYk#L~|bd+?E05*BVs!0;IX znDcQY^>rGqgp{C+_i0fE#J||Ix$6w@xmJP07t8StWq$|f!TV|jcKH_Iy-y4%ueL4! zGVZZ+Ie284zoOs82E4mckLBAFvD+sLC%(&v53%E`ba;Mh!0*V3`&e^6T}MS?m}dlz zU#~(|No$YY6#e?>D4m%;6OJGA2*N9~!!e(_c(+eB4xP=$^3N0SxG}!om$P*tjndzTf2`D5~5F zI?BBG8Cm)D5E*G{wbIQ4;=h^gH||TutkrtV-4KVV%cC)UWsGt@Wmy!wyfe|*()a6h z8F}?MN)7^JDv*@Z*r$IZ>poO}1Ai7>K92h8w%(9+Gjwzy)*Z=1NK%vH8%bT6FIURB zy53_Rf$-dzgm1&EQC8i7PB9QHHT7+nuqYC)v=65GV)y55O~J*Odek(UbCi)=M>&r} zXiBrfzv!QpU!>rzMIqRJG!>V0#mFYVc9sG0FJsi!w<9I17V)Xon7>|+PxqxFj($&S zb{*;(I~@8l(JuN~!i|Y1WZGAopPCfu%Ze_syn>uWwJzcUn78gQpm@YZ|xc5W`{Zr}{mECmrh``KG zjW{1shl;vRS$Jz!sX!iY!@toI5-TtrJ1S6iF7=)c?OJJ~JgAbg_#O9;fiu}tO@Q-c4BFR=!PtNU_Pktfj>C8XRZL=>1JDVyR!6Rd%V1G1e}|cWd{iqEY3C^k=|3 z)sfgOr=T7?j-}z*w`hOInA>0B>tV)cOLt|!KfXcnfu#QLCky|*^6x$@Ske8W&&5}j zSY7mZw6PKqlvR1IUyb&M`|UPjgCAkPpB&7wX@f5X7h}n`MB3nP|5JR;CG>gbuZw}i zxxvxpC@Qt4mi_MH%Uzn=HgC3#uPfhx~H$D^J^@XV+ICSv_lJmv?0sV?*yfBSE z>ubgy`FEv{IdyfsqN60H5*s~_Tn+W`$G?oz;fc{fkU0{2Zn6PlXBK*;b=zRhnYxIr zo4Y<15%fJxF~DkaGkaA8Htna44=7T6obN4z5_KoG&#m~6OR2(pA4e=r4nh14K|IyUvR`RmT!y0vG2O$-~`%$Z$qnT zgTt`$^X~kD*c2Q2g}c#U#sA;F$-$@OU+j!&tw}t+XiFkK;2MliujBog&$Gx^40Y>l z%82)cLd7q?+rTx`za-@Gf&6y0~hq{ETHoKkCfKM~gfro9P>O<^v>O@LT#LGuBYHu9hLAsD$r(KE5rE0d4e-8PhD_$e zn(1HN&8~?Jc>5#fRaG&t_>{jI8>r-q#9mnD6>QztovtNk)+jzn7~^2k3!)#^>`lYu zg(28_i1A=h5z^=j_a!5J^linz)a#@jk@3!6DS$8cEaK^l%05Vc%Jqm+jPcY#pIT?O zhShCljy>1KD!IoVokN{Ggt_|J8;tOyZ(m;1F`#ET%{xbj!c!p`oLh-}EkEuBAJ-bEokv zzpJcn#iYfN7&Kay2V$QO(B_I?;}cMcyz=&b&w-_?6JKM&20e~mC{=tDk^i$nm3VJu zB-ZUVDn7f!jFJ}^Kye8FqECDnGkAU)i}N8>2uW_n>42MlsH)QPX~e$8u9vqP{v`=%HCd3 zR2f7L22dQre@$JxlA}69d64{+=tnuqzQr`=M{`Q7^4|29tMMO`f|$4Ijho!*Yd*@H z@v!&9v0_(>;?G_UE5lXVwv){9Zrsn@>nD1Q_6)&czhdY!o2l1(V?T+rWbaY-9M!Rv ztS!++zZeq;j}IcSc&ia#9M7cRlBM)He`6d*c!uGZD;wCoXwaQ=$R24sUB!oSGQJo={B4E2g#GTEn#k+pc2YalVZ z!Ac~=26!D!BUjABa6iyw2ZT3?pJXjOP9IU?)uDv!&$x{U>$VxM1}ga-B?oE|t4e-$ zKJFJOFZ{* z-(PEB{MT~Z26tf1o-};0CJI{*CPRF>Oy=0{<}axFbO`?q)CH0YJ>pjiW5Mk`QtC&E z5odExvbe%_ZmNp8-48d%De|hsSxTN$_dmA1S$u>1z`3byzJ*xo@B@H>q3128&#Y{L32O#h75?++bxN=FH`M z?z2}9#0EQrf4N6lo6}cBVJG*2zYea%mSZ`Xyd)Ce1eKwK_O`DgahT+=o_T|Mo3c2F zc?37E2elv5vo<&%vN!)v9?^J_IiI84vz2_Y+6I`$X`?Q>$h_Hv1>tyPWRPO(B}d&o zpB8f={mPy^_YnT^UF16@+>aMolew6-EJDde`!dEh zBvlP04t5y-;?so3RbbWjcbp z520RooH44zS(2mkTpv$;SBJuz?Kb&0&4b9SUqB)ChF*E^MbnL(8>p}L{2dhgav1=7UH6=6j6!X`z>rl z75!R?sU)}DpCUd^QCTDR6)KcC1(<5L|*tV->QCnowQOd*?Is)o)r( zos#boU+4g1O*u-;DesX*#8*J}dIzu<<8c1_!^QuFIx_2O*!fHv|8^4Rpk^BX4kFUd zZZ!VwB+fz2H2xh#q@CSp{M$*KgPLjlJBUa-yV3Z!lQ;)8)A)A~k#=^Y@oy(_4r-?H z?;s-W>_+3?PU0NYOyl1{MB3Sn#=o7!IjEV&zk`Uhvm1?nJBf2pGmU=-5ou>P8vk|@ z=b&a9{|+M3&TfQ%^Os!g>_}^77X$LLmi5a~c6FsSvV(zgw#l@koz}!L48*aUi;n3^ z>%|TReA#9dVfzI^J37>w^qT?sfz^4$ABlWIEBBUx3=a1+@ejnIdpoo?8X4F@$lv{y zzkw=$bG($$%DrJAkHb7qnEv+l&v?F_=-&@{v7%P&W+rMqMdF)Ee2`CQaWYA`OKIizImBM#CBo$6_wcXg-140Dd+MpqB}S7t`C~%zMOlRW z;xSD>b?5IThb%Tz{+9bP;tzz#t`(P-eAo3}_<2F^`Te@y^PjBkJ-@H_R6Elc&=}Ad XFlS(G@5lRez2`qE=smw*t#ketBIbCo literal 0 HcmV?d00001 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/_static/image/logo.png b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/_static/image/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..2a8bc258f6663112f02b40f04843789585d86d68 GIT binary patch literal 402692 zcmeFZcQ~AD7e2}uj7}It9a5B0qm2@b5JX9onbB*4h~7J+1S3I;lBkgoMkjjjqKoLn z=$#;XJF>I)_uJn&-}&?WcU+h2E#`f%=ULBs*1GR|Efe`jQ<;L4i4+eHk3v;NQ5z4B zL=O)SFa{#Jx)X9pCVzF%e4?jnrJ;exdvy-N1L8B`5ni3)U;WDBGymuOK0Y@d!C&_Q zS1+Lb`~3H({`$dxaCNx4`g6QNbf|!bCy%G92-o$(->7>QdvnzLmWN0ww%WJamyLyz zh;dHI0u;fhpvVZ)=Y(gBUM%Zr6%sz-3~rM1u-PA?@yLs(8yBGrKfiaO5#BNzyDEmK zL(LKStI&9oDU8mo^)>a5EuIKo@jXCuNXRUCR?X*{TwW3XzTR3yMM-I@@wBnAzra9T zVbY;hYFCbwdxLAg^$QF&H9$cg1j6Tl07HZR|BL^RY=BmULlWPT4Z8oo7}__w@#YJ% zPrd=8Ymff57;hpQ1Dr&hi@*BOyCME-nP{Ik2E^`@gvck{_+RUWYlIB!d!CtNUbqu- z_%c(`(h)B;4Tq8bz08T)!0K?q~?=JWB$=U>7j_9Xvy0`g#ab~YImTI~OP zzdVRMvax(-9-~MRiKE0i5;RQ5|9CD=Z$Jz9Gfby3>aqJYisbE=AUgudb%jSU?6zgx z{~oH46|mZAT=+0!>3`asg%T(gVEs`2Q)f*q8H0L zv2G*&{q%XyUj@ouyOcyEwfyX6kHYJIkr5;hkWzcpS`s8zhVb4p-aNTc64E`K^Uq^h zgy<2}&s4vij5jTPy+lhE`JPsXY-I`zzPq4}AJf@S{=L7@s`t|y?{Xb$yW9Ws{C_6$ zd#DFMbo5MD&9%%`nnc4Nv;49(cTB>uTpm7mrDbO1&Ldq@TjB!EBKFCQ*9_pu4(XZ9 zIZBTN(FSPDD5Zeb=l`toW1xX)rsXBzE@aA%&g5W&ab_^ZrMPa%JfHY>i`hm@FXIP< z&VP-77!M6-kikRcsnG%c>koe}gxG)^j&sFI2JLi->s+x+qD3A%YhGcdbWV|ND z?YuPxd6rw;+44A>?RS0G{(grLC&0J4KR&z?<}6}KG;4A0#mK?_JSxt)$eDC*Gh`%y zvi;yq`t+xf|FS|B1sV{ZTs}E;{y#TOXXt(D==X0y*+dk5?qoggv}zQb(FVISV}A3; zo*jZ%pVn>b&i&=^JUNwp<-P@r+^EfoAGW^M=J_bIB9y$-YXG`7vsu0`ntdIhKJm|V zi%_7k!Y{0WuU>ZcQ}RkM$0uwOGSt9uv_mhW+~>_hoQlY9`3t78t0$Ve*gi|>sPH(8 zC}$e}uQ1w(R&Jb+B)%|lBlsUSbgHDWId5l`ymm(+wt!C_bvt?O)_a4wQ!CRK*A;KJ z;U;oNqYRQ=tb4Ctm-VZ@5}nN&GO;Z2&I=Y-8M`+0Vdg zU-j}X89?9$Xw1jOc>5!NofT25^Opd(_`kzBnuFTL|L)7PMbVpb(h*a34c_CLm5knu zY2on7F-hFj1F6UNxRDwxH&Zb4qrUDNpqnhYPU&))i+R_QT^S*}B&5cq& z$2xz_G_0kZUx(k5^oniF=w3?H4@pNK;#vd|NljGQ9p@29Mjkvk=!rs}79gnhjO@o< zCwHT#P6kwA<5eH;&0Z_x0se8C^oT~qz0zBrWqPY^{Yz4z83k+5o%#N2Lmn2&Fjx2?b$wP9U%-wd=Nc&OA|dAfpZxGOjewJXJsM#7Va?=p^= z#y$xNvb)SUm)p*9Egq7G7}gAoe;SV@BS_+v0kS62SAWa;XMyhkH&9;(Od+t@=CFg& z0T=!5W7)KiFGJGG!gNBMMNHb$&+K#^#8qqUnAw@pQ7@ZY)-naMt`kpovX)?CR0U~} zazBQq~9k#7(i2 z>N{9TS1m6n;SN5+P-d75HhNEV5vUU&RB-R+BFV=tfa7Op@6qZ=tkcE`lF5&yJG#VP z+2dC`@-uAxafj;scqP(i=}7&v3fBFoW#ql_>{8^r^k*9fH(QbHZmAX(MnOgD*^6Xf zl3}@w4A6)#EL5&6FTCP+gu0tZgO4^9^0qkh z8~}L%$+^rk)s1Am-Hf5AIXy~s{(l+BDTw-Kdk^IRVqN=RcmMGkv-i}DQf3l)iP*H* zwd2#tIX+^$cP((`pn@nbEMxMXlFk!&m4iYK2*BgZpUREFugExB{pe#t75A_*sy{)3 zdDC*q>omQ>7FoA9DRy9GUpzvuIsLQdmv}UC(p_({8nvD1DeEa>GsZF1MG!izXwVx> zi&T5NNC+^dj@_)0HyedOpx)?42Y92yDoG6?V0;4>jJC7Zokf`AslwsGIh?;YQKAFz z`I5D|;(BsUe2v+B)HGLc+G6lQh8b3y)56hNBgv4L@`f!wkhSe0h_e5|sO|3}_Zi{j+icXdEY?^gnjvRnpvCOLXuY>o`wWS1;qQ zo6!#Bh}*uu-?wVzROoq^POIv>e;*ploxYnwVX%7SGON3gIb~^9a8H!5jP7FFDVmow zQvvkJ&zz3HF+ip-GWE>A!mCfggLdz@&d$m7!!IdhwylK3oG7eW>{J{1v1l7v-rHs* zIyNbr5cPS4OvBlAvRKRr!&UAA;Ft62^k>Ca*ifl;9dj@;-UJzFB`yY|r-Aw*5ygHS z@f3iP(y5nCRpxP$!bb42A82`TQ7@s`vh$#HcJhVd0DSn!xcQ@44L7qM*1ugxNDD+R zkS!H-*2D1dSW~Co2Fq=8NT{P_S={!~U~WIz?`2X_oO5{zS7uxHRMVXRaR7s*>w-~D zbKA+!q7N{=NX7u>!?l*00-T9|-!PAtAVW=pq>1~y{yA)<=nT70jX~`QBWV6lj%%#v z#5tGx*HO9){GDq4O_zqk8qKh;ThOTLY zr?UOHuy<$>IW71=-8Q@w!Bi>AMcs@{6SipE|>_ZPLW)8m?9B1`T2V6+RBA@nga_%?sBa z{O`?yCh*XFq=Xkw#$6dSP5a}*q^a7%r@_R6S4umMB*%}7cijyanT??&4hZV=6my!( zxihQ#&Zdh&olwNSNeA@dZH+tWr({?c;S8hGGvW!Wd@tK>2F2OXbeut1+~ZA#7N>oq zpv99UXQ)FY?OMR3!)0vuds>9IaDBc4b6GG96Rz)n+m3jwQZSP&F6!3~Y@$!Rw4Dd% zf>z)faT!igiR$ooMph4RedO-Sh-!J&yKw}u$P0kWi=vo7d|55m6@$w787m{qa7i2Z z+mp@eP@yq3ud+8mkkhKhgjyB1#n=Ju-%}TI27PH*@iCr=4aQOiJoiXzXy`R(YdT1` zI5b|gcD+vQa+Y@wCYD50uhM0D*rr_{;h#K}*V%nEax|6qBH@Yq1kBYpR=d5;E49DF zBPl9wS19%lSF9sRk(04QU1yV@@R^2P<*&iMxGxEK(A2L($PhQl*_yO~frochcEHp3 zE1dJSXNNt|rsr_n60az1!DXvtP=A4p77D65SV%^V=c(cn1QFO{xc*$hd z!Z>7Mc?-Wx{bFQhrE$L!LNAQ;YyEr3)3i66Urolh$|0$y`0M)u%k)@xyOHL;>lduV z#k0XaPw?-ul`hvj@Q_xy`;xvDkuq8KT-Dfp)Jp49ocM(BQY&eLRG5A}cQ+w`cKNN? z5(Kgn@hX0xktB+TBuYR}oO-M!tm&cLN{vvFF#~mNu=~tDHtnY^7e4%;+l9lBJjG<- zBBZ;Ib>YUXZ6&fl7Fh|FKP4+vBiv#OOWUc|XX=v$ZOU+~ItO|R=aETKu8p;5&#Z;kq1LG4B%e%tMB zc@bp9{Na0{dpTV3g7fA|&N5*;N|Z~jKQ+DDo|R@CO;L0+G;z7zXrttMZz}&_CFapg zWMOoH^`&U?Y23Eou$tGRYyO2yu94fY#+Yn&_pne7VO#Fe+iVsO;6-msM*^iZQT62x zhtuiDn3sjc90t|ruLQ}pd}Rdd1n=Kb z0GtW}Mjp`Xl{RzN9?e173c1Qgw*5z*Rvvg2wI|g6qR|xF+rt)IP;=|Xf6Lv&k z9({t3dE%nZdLk$N1D41pQ=GaYCsJ&%ZDlUSx^fQf&%QXIjNl(up4vEw0|YHiL!3l9 z+VleHWW`At_PiNS_mvD(`$$-+pqP@*FUh_+^JbdV+o5{0m%GNQDWGm=L4l~+T<&_8 zxrg-=G=CQPfUu=S8w%{$sKP){A!|wuI}X}CRq41!F9-_Oxmo7+^Ox)g&Ga# zPDZss>LP%Lcm2*}*be7Lr$uKbviaFWxY)dG7<{=M-=DJu;z}2ZJ)b&STAs(L z1zw~$ZfNfrCs8!xCQznF!!gy!fvYG82YJ=n1{XX?_$!6|-?Yv*q4I$YK=NS`vyP|( zW#Lo8jcO8Xvh*-_Q?(7$H7A&jEB!_st@h3YW47Tv`K^@nRjEIhsx7F^e%riU zs2gUVPoM5Mr<1K}G!U6F?Imzi(t0c1x2NZonybTa&)9$%d=ON!eepdYMqA8~ah6JOZ7Lohj*vmQE@d#rMMVOxWgNAGAKx`QU~&VJ7{ z{T?XdoXqEDDr~<64oaipN=x!z>zinBBpLl$Bhx3!D>2DnA`WEVtD@g=^2_C7aNP`v zI_BiZEV3?7G-GHxrBk28z!|q;iGr;Y!6#jjvr%VJoO$CmLs^jmv}Te&hBf>af!ooJ zk_AtU3uQhJIy6H&N{!w(6*!>0G4L5Jw?~5rIe5)HGGrH#*B6SYq zx7I26ldC3s<{b;kPLt(Y9(Vbo9O^E&L>%uPei!i_wH(p0%jh=sU2ReGoi03~1$>0y zcCxiJMC+`2@dt9m zsckQ}>Q1vzoyku#Uw*hCEN2bfeCtP^A-3MMe2-1u8D1u;8YnV36!4O^)ZX-T<)Q$= zIsY)$(OA9g+J)t_8>W)lQ6>J5T&LZ$N;bWU0|=!J0Q}n!0tXGpno-+ZeZi8Nu*K1Q zpx-a1BSEM@Q3wB)>vDOyy}!eU${;r0YFv`^Olp!emk@X7`Gu@+0RzYlE-!BQ%*w%l z99Z25IsAydVKbU;X}+FvxL$P#$6 zD-s~G?~=nyiuF0Rw_QtKFccx`?1;aJDrisEL4ou z%XQv)T1zf-W8UIIX*WZQ^QW+<;8d!!(Tk3shhIhFmhuVhuhN?c3xCD%+Paudk^%H_@%9s$7s*}DLZsNPP`w4yNku*5%7l4iWawg5 z%t>-L$#hf!YC05EKOZ-*uZfc-@8I>UUhB<&4lWf;>o4m25xG_P!_sypX-mvq5yV0c zObs|1o0&5=kzz#U<|exVZx&6Qd#V6(r#tlSzM#U7k$73CRd(}lQh}r995~UOj(Qb= zw7x#zJGjW=M+o90+B7@E`tH*7@AuYhovmoPm?WN?+kBmioA?}I+N$Ix26b|z0 z?$51C&JLIKyw(AGZ5W`l$PxGY|xL zC$uxzmH{GQDZQATD?$-&SV=89CZ(X{x#EW#sic6LUd{fOBHNyIBlaM)&M#(}*6>XtwTTZKz4W=z^) zxcjb~H>@en3yelc#)4$_e+u7iAl^+$>WPv)U$H!OF|W9L5_Wg{FmC&7JmIib$L(lA zZ8Wfw{T%y{Lx>c>&nesWRZ979X4A-_n10!~1Ppo|*t8zL0IX%G2L28;jrj8E+n$E+ zbcTzbJ~3+SlZ3c|uvk3m@6&<@ud1(cfA4oLjaMnC!? zG51~C3Ls2oxUBb{0Jk1JwOcda!72cKkUaG?m%AO?`VM;UAqRb!p;Id_rYM;}dGnef=S`9{!t>I$bXXh(aNd4UzTWS28wQJmhG%Dfd;zwTYWES9NG;9e=r zp^ERyjg)v*~WN**m6p*nbje8NSXy zx-2MYY4-7Bg+f@fl?ft*gV5;9UqU$S(^dH*%*2%RUL-ai7yQ%VApg6d87d?QlPF^5 z$h!3p*9jnaLsrxeb5rTWQF{CFAZ9>-Oc)~Ni1cCdG>XouSaVc9omOfQzkxJ3 zNMD|66;CZVNEailhq~QtD=7@&)>C4|1;K6j2FFv?b+#C)_>xF5=e0+ zk6w%_@Y%`Lp*E^pCn<>V?^gQKx@gO2f>_YC^QAUv^WmFAU8nR(F#@Lg936T48S+2e z0Y|I~orquv z!)T2ky*e%RxRbzlMTFLuS6vMH^ym=Y_;Sk;h}+E%Tz42wEwC*PX!&ZPF+8(3lw^}3 z=mRP4dIhK0W?s2XGr>`)9KAs06@eacX*mO{=cw@j-=#BY%hhSb)MFgG3G4kvQh(B(lR`*z!eo~Bf7~LA9*wSFj<1{aoN?T$>Gl5{ml?mKgkZ1r+fx$`cGJ2RNFz)I z%~n{Lk1`p$q7ya2c-phI>V3!%P`A6&LWK7*?PsVx!*e6Iws`@_N~nH-P!SlUh$onv zB>4m!5+q(pjMgm<5C(OM;|^X)^QCKC8C5o-={L4e$>`!;^Z^bFH|eqWPvL zEMEEw(B>-{BDC@G4Ixpv`OVmm6wUo--+Sh)-<(5OT61q~1D3l&QPg!a-o9-GL;7!7 za@`*;-0=P&Br55_~*IqUMgme~#Fh;k%c)dCgxIqc7dwmhsl@cOW!W*j@o@=m|<**R%fvK*}NcLjhA3uZw*@nNSz!&4n*cQL65@CFZFwR z-z98U-*s)+4M7VGbdY%pkSYnse+@Ix)VuNmn9Za@dh|{UlC(*HlaFhxB$}e0ES>@M z2@guatWQnd)R;jQLPkWNQiye`&X|v*04)`u$nhqg3DB5~f0xWl`^=QAM4Ai~An$x` z>BsfFBpresqYHVm>L``WXoYJndO!Hn1Pw1AH42>^PHHD-V&t2zXBg0|csnsyqp1HI zcKFMi*#xvdW6{-B8a$1eIUP=bq*FBCytv&Dmb*d9>l-GYd9C(NNCV!K&6dg$sgB%? zyJ2626U~S@9jSA7Qm3bUqJe;JZKqKzqP=ND(q9rhmFt%pA63Vl?lPB~v65nuOFz>U zrZ*)(Ya&^{pt!Z5&*O!B$;rQ)H??W2HwbP+DsgFHuG|6KGQPIr6Kg`O?pB3q$|VP) z+^VEVW>1L6OmA0m=C$ndrp{M_NBxDRkHKmZ(Q&N3Ca(2uFHPjs&UL;B0NaBbb*R{f zL?q~M4ufc5zriP<)jlS`y#LgHQ9AHo@$NPqtL*W&efBNohBckvA)&+hzP-ZoKRrS) zYS81sEhXPW%{JtI4MDO&ib5gEFCJBu7xYI8$m`j*7UG~|ERE|2lIB+5SJU(tM z!-~EgFYz;BJis+~=p@OPjpyCVB_d@E!Hqg1C6|nJ_DtY!aEH&~Bt*S(;UB&(NK11_ zHgUeu(ZLq33q;2*Ra`Y8xD(IA2N0CeL01(beb>@iC^-@5Qn)-sW?YXEsJPRnccpx= zT=(NZYB?=xPYD&RDC!XjklMY0%S|$bU`!yf06vEsBi6iOqOdlzbg;oFcsvL)nY&zoYzu-!4?_wpY~c)$JbgHQNEm$W^Yx8{i>wv5vPTp{F+IvzC^PD5 zCQQM767rL$j~FT~N+Yz^F6*j8AxHKF7J>!?hX)v+qlx z5WEkk&@zJ}g(h%);Lr_f(-cXBciT=XwO{yDWLGYI8i2Ue56j)knFfDKl5@4!JBPk@ zk>JM5=Z)W0^3s}?c{3i-tVuUXhneu#D_ty*6jnMMGgi?PZp%ckXTxZGJzk2(;l!P5 z5-fzivMOyJOEx^XtK|#WT9l4F5kTcuE+1<|6hPWDx`#OxkrEDgC2UU#m1hyH9s|Nk zBe(i3^kviP1jmubxs@nMfeGn=E9%N5mla;6M3U!mIi1jGdnk;gnpc7>i_Y8Y=Ktdp zi4b5l<>LdFEt^b*=9e;oavg!mw8N4$Kdxzkfx6rI5+OCaNtb3K#qRx6uQn^IX{gTY zx|^s|&$oQ_nbkixizR5^nw(61*={c}ba|x@@-OGk;Z%fww(bew3MYe90%pq{NB%C& zp(7O<1JpfFW4CT&=U9c>)YqjQzb1aZTw*k_KaTg0gCZ!y>BGVJM8MV7#?!PkD${y` zrp6Q?z9IY`5%Kj;AthK8WPoUI*Nh{jfV24ZCMgZ_NbgHyqg%+*Nw#uttBO-idTf|+>>srm5)o@)*$cym&$1(htKS8%$GhBM} z<$SGi7;?FS`0^+I#O6Hh-R!?`=>7ZY>v}*#IyvU+s1%qID9Ad$M>hm4Wy1Q745=An z{R+B6zbe{xtAgi?MX>{F%w~s&boOhY7kM+6KnmWE-CN0Onk{=#bGhPawpM;!k)=-2 zxzD^A%A=Zd@1(XJB$s9jI&gKY-LB@nlKmaT$hfrF9@c*ET|n_DB9V3o5}joiL_G2* z)7J43@6^3nCnddVlJPGk( z2iBL3TJTX766qYD|jr%k`cCo88hOcVBB3HuB_A1mD zcooROzVRQm`gC6zK1mc$;qh-YM-8A2kv~pQ4)CrEZOz;Lp41V~$+xp;)~4%}4O7g`aPR$2>~3958fol2#r}lLstLmU2QX+B2fCV?0`_wL~|k4!>JCcjD3#YBeUr;X_l2BS^X93z)tzb-h z_Q;yLxMY#uVKVI`g?tIk4Vc@=lD8qS&cIvT3wLQMb&w64L5sg}XNmjQ7E-_5gr8V{ z^4hPol_%af0R=78D7+@lwQ8Yw$Lza5Zq%7Yh`fxTr^@0#PXdXaYz3`BrG9YSe!Bst>1T5x$VTwdwJcakserD*f_ z^NBpq_YoT{&5AGE>}RT;^@_r}LcFUY7Xd^{Q{{%$?`%Bc`7PdYMAqRI)ox z75YydVsDbkQwR?Qz9Z1It1{m0wI7`*_#pOYC}T>%@<9EC6mVGU)TVZi%$^8v{bwZQ z_*TVHl>mHLp#Gli$pB}O5W@wbW2z>(^>#r(KOufqzoc*_B;ECQr_z^Y1>msDGj0;1 zJB=#B^aL1iL?PpVWn=VmiCo1dS>ei2V-O_FfX+ntXzYPlr~C(s3yOq`=ja9lj&llv z5I0M1DdAsaTr5X^qmlW31?!>6TxtWFVC@gX$9?i_a`gJBzp|Zo?1LV;A##Zdckjpa5JRnH+4A*1(3MZhV0(S_ z3Oj2sbI^4^!nxLSw>qZ#UP}02?}SCff*MX^!OSk|9GL-Gy;WV(>;W^B?0{k>pmFyV z*a$7GQgSP=DinTcY1QLZMrExV`x`7-RO%<70;2~_oQ0fLFPXI-Ec?xf%`BtxhI*%M zsf)q=CbBb0j!$b}ap!CR(Vy964=RpiO(s;CrBEQ)l@Gb7Z2Zu^l2BWa{M&2*Nz?H< z%I_urXeD>$(?th1q&dV1?ObiB+(>&Q&wwRFk#4DjxDpYrGEbH{Gq}2$M zDz$>p6Z%30X2hu=* z9?|lvo+-~pF8k(X1GeoI9wBD$Th#Xw%4V}KYyZXzbxP@JSL&Ei93enBMa0!o9wK=E z7rz2^;Yz@hX*$M%drkwS)yb6@{CjW9E3q{{G5AfdhGx7p$*+p|pJjwpByz74am;oV z@e8eFHgGh&Z1RuNB&RdFB{^zuF`ZerV7hzWAPQE7XyodEn{N zaxGj%a)VKa&;apU+tm)1oYdJ0PGUM@?^f1SDqWTyf3Xnk*BTX6IX;rMyq_mF=aMEJ%dsl-*{cPY=p)h;W%0uPu|axU|-yIgay-m4efSd0uD z0UXqI0|q)}l=SIT#bP*lNwc{NTsX02w`sM*G=LSB5ubOHe&l5~(NwUVS`wyY_gKWs z9SW6^TA#T3hpe+G!hc3it|DWlcEBNz=*)zN9{{YOT(uc{2tTrnT$x_X)dKe&399Db z>3I=oLaP!M(0C2>SBqk!7Y+A$L8&oLXOwE{?s?06Ob260q5Pw;doeMGcfSATY4eKd zS9O=HH*l^i4NcOl>>CcE(y?w+L0_rKQ>AHRqK!O%xR^N29G8V!K(ns#xlA4TBx>6zb%dRqiZtGE zb%^WilD+P8m3z!&KyHiHEHP#b-QIHiT5-VbdALbF2+an1oRLeNk=yFGwzX!eEfeOJ zbo-aKK!T*HWc^~)boW)Mq;q?$=wDNTR+0!y_q8NiNSF+(b!t}lbrgas z_*+Sk>7jC77Y1P4KJ?0F{!ojj8?OYJA74^gW?#Vz$eM8=Jk(2q@A$u^&N>c zb-@m;O=1sq%gR$juUdz8bU`%3QmwjL0$Fpiq&dkEyg^>Pp*FUCJ@5JXLTqoBG*c@- z3h%Uiekc(I4>!^aLl~Aj@tKy95hYiaL^d#mA@#;f*5O_L4kMT3(e^k9u{vfIX$R>_ zJ5D3cNXFTk%c@VwM>5|uhhXS&hKx67?L;Y~WxOnI`Lmrqr!iQR_z0)MW&P>M;s6j3 zZgFEt=iIKd*T!rY$)#NNs5Q#oxB@gPJP=Zova0}~{o!q7A$L$o2S$K=O!XCgzjFcV zNi&pa3loj9cLt|0Qde5aUppU=>@G*U%W=b0PM8mLa)2Z$jJC4}jb>#`x9gG_ZAHw! zYud%R8e#I6-s8R`KWcC7`_CCYpCQC8N^?i_ z@8)P8TMIvTVO-KV@C8UFSY z+o+D7NMbgo-Mkc(bwdaH(MBILQ$|>O`w2AR;!Jvmaqexg?thvP+P@_-(ExlS`N{Vk}O5rfQtk0(7*66SH+NW?u-c?@<-X$B0+&7f6TO^CuGpE>Knu?^f0kChX! zFCSd{ma=9ekUjCD^5rZIC_^D4C)i7T4ErwKs4lRG=35fhw4oq3f9=LTA&y?N&Irt1~G*KNg!s;=1IN>`uWe2>_Mbk}uz8Y`@M z{QEMB@cCi87wKRk6^Ese+xcd6Q~5T8Tf`qVdzGG6=wAvzh6U?L3-2~m*)?LM{{U)olI|8s7a>BtT91 zIg^zVET9G{vD%~F?r5E)r9eFT_=am)HPMIRW{bB%8scF{_Q|~4BfZ#|&aUKsp`xQX zLL;Qi2>L_jRqQmbx-Q4Fa+^0GEBvZD%L1fd%2)>cd{=d&=~)^~U%!|VwID%lqMzDg zxKoLIHxCX&$@cF~7aZNozokxWT9UzEt?NFEdY=51!C-M+9coj^wfxMj1*7dGu*+M% zzE<@m*t=`VwhUha%EI0_Yug^pkWs+(_>gt?Yaw^J#rq`-DYy&2!jrH~Mm5oG^E}$A zbxJ{Fkr?YOh?B0m$!@H%QQ!+}&6cZW_zJ(_oRGaQ|0X;DCqqr?i=8@xn$X5G{#&USC zNi$Y?;5b)>K*bzE*o-}Nslcx7z?@buN09Te8Q7TN7@`XCxmW>2QquTqN`w~h>or9! zcfNZdEO&;j=}Ff z@#MP|%qcIU=+nv&kEDqlzrYkV%FYF$1p=;|Wx%5roazgG8wctz2Rw3>ThvCwyXP`n z+W9?woDjl6grx(&v&iLn5S{hMs-{<23~%RdeH(tD0OkDfSG;2+102+R5Pct$@=(sl zFEW_-Ahfrm=NtPIqOk8y*Z)fuP5*=V@S1O<>ZYRZFk%9Ciy!%AEo_R7pb z;6b#!(|)d@gwFa{HB**hM@DG!!NOOAxxI1DgeQ=Gs=39`yDj^TvOLvJSJc2GqOn{U zG|JY&$&I}x=Ne(d1EdrH3}d|QvzY42tx_0S8vCh^)jUe|3Ro(e}ikSSm7V5Pi9JOz=FB{Pf%ox?s98Yf6*~zaF(woMiZk!U2$0 z%>OEAMn4%MnNUrtssRI0%5}QE^)+BFhkCMNunx&Lp0r?Smi@ui8RG53o*QAJ!Q<)x zON<3H?cr=B`bayEAlH0Adsn2vSa?faVyU@6Tgc0wL4?kTn*rkRmTaaLN;hNcV?xsnnRr0=|mY-XCUU`A5gw~W^B!yf4&79%GYxS zTV;u|zFKkUS~svmGvmdru*xrQ*rrlJma)eL3jfVA&`GE+akL5`d4&+IJLBp0x!P}G%4i`K2DB_P14 zhrOvXanE9~BRtz}G@}PTX17m($_X)gYmzUNk<&G2c2x@>)s{c^Sx;1vKP1%TOdsHV z-q__l;d4VB=8f~Z@!-dGjmNjp;51?|htm-?_o8*^?5gseL1|f*)_%$v*ALRNH$jh_ z#K@Y!j)(QWSn;}au$p;|u2&q{)YmkY4wnq*mZfz43T8`BV=GjFHkpeStMgrx1H8Vx zE*6n1RT;tvKevE#eW`9_!hH}vD97>AG);4{0w-LE&EPD{TPW@76Y zk2UDbaj9U8$5LP4UwC~=pG?XSv_S=6DM$S!c3cT&LOCEeO1oXIYa53N#aGIVvH+eA!mU{8k$-111!d_uAJ% zEjOLoUOpSbV4LasP(&dz9=BPRir0K0Hd)HaQAh+UGt!Z~mTE(O z56z@V*5G~v4B`2D-$28>Iqs7oBMd!;;Xxu_O~!UxSqi+9X1cZi5|S5a#~*n`KO5L5 zn_kh1C7SY!4t4p7ZpCiy{>THQ`#Op7Q;K+;B%pf=eLyivD`(~2E`Wn$uP*9kRk2M^N{7a7Uzv~Bu4r)Fi-Y1R$DX!s zy(P3vP2rx>R2ePRRTO%vobGzY(O}p;$R`()O?p4@C_?u)8Ntxli1u1D^&W}TwlSE* z1g3R|9j_ENYJNN2DuC2MLeaj zPNx^QXtJQVprMs08Tl;*^F3;cm$0Jb={w=R1ZTcvi#CM4iBcbM?yWP|_m4w}_FV3< zU>vDGva_!7CCVCDp)cL8elJCA0(4eEc&jg z>`n?K;LGh&SiNL|Nej659=RN<$)o!HA$}|EKi>u5(5NogzJr*$(SCiXi|>8#m@ONl zaZUWE&!b7JlVon?w$0;fb_TvfKnFjXx4qHACbb}E@bUHbfN9tbs;7Ccxc`XDfAN^p zmw3ccf1@&akUA4!l1Mz|SsX3WIJC%?rkU5|h(!l~DH<_K4Rd~qhkEAhg-s<|AHM14 zfJi8u^2tMUg!ShCYG^+qQbN!gL#lm>krbvlrPMl!vzfeGl@iZh!lBY*7Q~N*xaai4 zlzH-6U6Y(9LbwtaP1{TXzCSgx^qYKMyoCzK2a&?DJ>!@hbcjtF-Jd-lSCq4;hFxWr1A27ywgeZpdF;4C7+3$LA|d0))wE#h zAih4Oeqly0`rzDnG6DjZ2yz$cp*7b?1!p?414AyN@8&-RU30pkDb`!5z0b--gdz^c z9yUed!)2hAC4TYv_wC;FPmll?ia;O(lc;?g@fWn!dK|oZV&>`+&}zM5dZ_cqs`#Fk zuh?SnEp>*Y;$B+?O5bPxvgSVTm2vU%=r~tZZWq+qSZ7gD(8_PAhDWh6nuqT4!<&zh zX&SRN`G(g3mhz3H1mK%@@U-ykDm0^l;k9tJb@r2f`~m(1@>h2YuR$ zF;f_Ki-qGG%5#3rb>r@8^kC;rBgu;i!3{m zR*BEurI9MHz#G5peU^Y~H5BN0SE|e!Fl!+!AW>dV$rnxFTaD^L(!OT+EpXL7UEYUFyD6vi&6|9@uJK`oD-{!g)4;QEv^RJG0zxavjEMoySt*(5 z_%s47mGo@F(XwYxR*CO&5jp9mj*1A`5ct{)OqxmKiYtpmRo_P@$u#CMi&(U!bP)9Q2>x?}k?r2zK;(FW8Y&F3zfgMGLvqBcXNOoihh9h6n)R?UM>1@}b}5<=xtJw%Ds;G}XnMV$;=om%9&p?gjqhx;uFuc+nmBSc^<&wDeI%lKO6~1 zJtp4xw9hk(z^wpLSddH>t#1z2#%KBjc)>h7Wq0j6wU+*~o3qT0#HOt<#h()+y@7&Pj^o0Xe|o47+x!b-`* z{_T50zqk0+bBtgtJ3hB83@*wV9PpSOJCoI4^RXaKC^);r|D{>^h?pAzerPaA&XGzK zoiTbcVt#{57; zD{ZH}I69DMPjoY5cdky54W!owQ#{bO-v?S!%#`l~{77J^Q zEv<>e4AbYy00n);Y^f!K-aM?;P~ zW{M4Bm!u0-W*!9)1!Fa~`e>bsjC36>7fUg+N&yy2!M~bdEv9ZJwfjk2iC!sCse``N z80QAZkOIt0yq$|uhjbrC2IYsk;Vwg7i|DM|$#KB*urgu2-x)I)myy(zN)GuXz?d}T z&}-T(CuVoXNl=gIkYHeM`N77ut9aPEAT^dgvcS_2_F$MY!$rH^DxY~~t6w)C>Mz9w z9J7ZV&{}GvK6ZZzm}C2a1tzp_U%wN*KZ>gA`)-$RQ}XyB7i;^=I7a_8HPd?{<6nt6 zxUF42+dQ@^%|0=}ej2WfeKB}>>SFbsYRQ`!=U;+mMy;*0epu+7_C%<5!Y?E);0diH z$iuh$#BFU-X=CU`mZ*^ze=r6bmX-b=7&t@@%+-TN<`t5E_g#BFtdh4FQB4BN5}tP&IAT<@`dh-KT! zK3x3e0>Hq+FK?`@LHpzLmhCmiAAd_{4Q}s`=b!Hu@SxRu*k8{&R=ja-vgQyMO#}^T z57|SLuCO%l@kdTy_^qQzp+v@@P*na^*RYb`YHCvcyUQPq4pSE&KfY663C3M2r%U`o zyuwTl?Q1fQ|7SUjYb+il94ZtpFRo*1OeVPg9`8i~j-b|(e*)w+mXj%6JpBC(qL<)s z<}ncOx0z64OsSJ9=&xM1(8d^N$+mmM=RFF5c=q6Guc!FxsKRNp z!J|e}6g0)PahjrkAzpT|yF9&#D2ix~U?Y8{_0h7fKYGjK_k_sxQiLI@=eBATKCmR; z|1E-0M^E31yM6%ki0b?&P<ko6~FOCJg@vj|AtV9jjGMgY+cnQD`6haxN zVxU-vQ!N(``e*<)HGO4MiBmD{@a19zruN&x2nk~nPs4fDs4wKCz}ZVtw`x-pFcoml z(~L8#Wp1RD)I2p3tYsdhARF?aIIq}~_V5G^H%Q+!dL42bub6U+e-t6EZ#>RzxY)b5 z+clEBoTPcSXz`)V?}p*teN_A#?aj;kq|ts2qA3{lqh2cId7*1IRI6&c!I(a+9l zA~{4vxXnfQ-#=7d{x*)6fJHSyeeK&a-;Gc(>)$SA@;8>E1CD%x??$mMl^*CrgY2i{ zCU@F@qEdj(BhmWtsLokGdOzE$xI6n`8^Zje0YBy$k$8?-k)Z?~Kd6Y(Mu#>1g~jjt zHtl-BE(bw{Z)idr?XDo~s(WJVpPGc$H}b}qrjWo_Dv4&?qUe;Opgg9YkD5;$yK2Ck zux;p6kULs1tV&>Rgbuj@MwB?0hPqc*f%DqQu~elR8eV*f8>VaP!UhR3@GGyk|;F5FTRRZBVqv$)QDHe-@Y#{dToA17n3 z4^ukDPevJk-W+N2T(9zo>=l^cq~H;`8U^@QP#Ux`M4I}sgDw&lKf=_eQ(R|+*tRM- z<-hGh_wsv}f(m;0A^uQaP3_aDi+Rnq=&HAI3H6S~tiR8qt5$BE;ET@)277%zv(eYC z9BjMX%E$h-Kealxm)t2VbG}6h4^cAsGlS@c-ztpCYST$g;V4N_5!NQ>G0h?8aluVk z!TQS<@1k|6C817AEL~xc!O_yAx)D1@Ba;OBb0!dJ*X#6jy^0&|7aV=-TBO%oU!g8_ zf%Uk#CE`nna^SrgSG$4k%ui~gzm5xS)IB7#YYxz~UW+j#YkKiqMc?0V@92#}O#EDG zI!ZA`O%)mPZ7qdWM$?Pfk4_70Glj97VE!gOl=XV!kMGH9`J5aw`!W!_G5a@#^q-8b zbr^*IiG1yVrKQc~PtHBpj|aVf99(&~&YeijuU%~n^|FlA|EpPqph`rKA65UiZ_qnR z#vY6F+P$O1(uwg|Z!w(x{hcrAQs_!IcpHe~?G44-)YY`_!v>EicEB*EU&oX$wNT}Q zndisTSuOhaT(3ZPSt!RBpUvX!NXp_hQH)SCKHBw2q{+x~wJdjdiObN)->?-C>w9_a z5a|dcwJ66Y>uoiLIdc8>eXews%|B#d2Oo#Wx;Bib0}k;^UEu@0u(j2-5RXzT z-*WPN9EVK)9Qzz$>wMuA&PYK)7a&W|B5;Zk^tc@Y@J%N7H=(nWvFH{I5M~6mq++>< zl^c>{YjWV`q7lLzGM=LNyLx3HPR>$9E)}S@7JPyiZ!W4Y5?RzhuV7&7vO#EGYUqqr z$QQ6KTTnalf?GX%@gISY4GADLg*;GW9%~tY#R3f|q-mtb2n}i_9}#h{oBT9h_j3+) z9LFi`LrV2c;#5Xgc|z|?DCz)d084;u7P~(`B5tr2%Q7E(x*@wwuSmn-PVIy)gK2*tt zm2?TJ>B9x%)y4JawF6hZr}-M7Wd%Y!_F!uyOSH+9a)L!qo`5D)C`}d>=GAFely!o< z#zg9O3TMmYbrnUp^9UrpsHaXanB7MT@C`fWm^prl$4!UeDWBle>askU(!G4>)_NV$ z*6MX}G2ztw`)yh31a`rwV}|cLzYcX&Cv4GZV)Q*9yQ~qdAN>Sw`s}aryP^`Bccb6 zx3J@^6QV%)f$P}sJ8!#}A{AZrIK58s>;P@B(!BdnT2(2OQo5bQZF$u$WG$-RsaHOi z(n&{CUx+V@piss}Q3+1{s;o;O{bT65ff8R43Hxb}er1TBwFq(FLj6}(6xNx*%2Xe^ zo_DD7b%NDv5Clh^Frn&ds04}{K-<;H`qMIMiPx3yB}=_T&;n-6!V6zIzF6Q<@IZiE z+jr{>*|t47o*!TE&KhJ)ywmSGecerp?|(=rI5Hl^!2F$qiU22m~H{c-B97Ctm0u!V(yQ8LiAFkm=OFm$yP zpYPQ^#FPt55$2kaSfa+A92(7<9%IEHmraia-F<$-bo_IvNYdIFc`7?k$QfK&TF}8>UlwIz;W3yx9M`LqG9?7i85n-+ktWq^@8|jmNEUT%&Zzg6TDJ${m49kI z$`ND5yeIrh*pJ*6#seXI4w^!qNvt$Gq6JJFNTW^Cy8YoOzgp$+lir{|dLgcUS8Dvt zrL3{lIKwnXBta8ZYmj|?`$>Y8sxR@a*C4 z^{@$jM~TkL(hYkkZcomQ$8_3f-#&<9w5HeMe0#ss(r0TIhg6S&Wz(d#9#xEJ| zN-=3D0SgL@_QbXr)+-_#boF}69=C>OkH3$$OB_xj!&Mxa9IcksP#W64`ha__& z0@E_PFlR&(RxxgnrNV2;Y56#Lq=2_0b33OEoQ)50&!mHxQQF)El0_3Ef7~*4*OPpR zmlZo`??+KbC%4>!gAzCQP!hWwI7$^N&;K1{eNi;44#5tsswa2e**7>u`77j2yCT_Q%IbGL zm%4#;@l9K!et=ao^Bqupy`?uj^~$d_vT3cQeGFi}UHaIiFHBKgO}pR!J4ZkP+~| z&;$2q8o2UOG3$VJz2vx~(p;uOn5y1$J9eoTkVKX6Q51#2DT9iIV^zm2Y%#up<>0J1 zTUIq75jYD+wCO=f z%l55CHHHZ8)ai3orE8`!->)~Ri;5^se4jpv$))a|&+V4aMG#s$FV;oh6hdAuZQ2uJ z6xXMv`5W*MzH#V(dOD2Yl$Kl3l~QJZ1LMu#&tdG0r7&en!)ryoP53WBwt!6+slo3TP2G?OQi6~b@YA~U zyW`%0_EFVZtb|X^C88Qr*YStqXXr1(&z^KrUpx*zXUuIg7{l*1kY;bPibIk%HallW zeI$JdnsIUB`8^>l?!V;7^iNS+NMEVM$|(tdnGcUF273!TERfgSS$6b8_=#9&)soHJiBI{|lfZaFSD&LlQ$5}Dkh zwfZ@lpHqSAua8@U6jY}KgDacfcsZ)GX6vGjyjjsVfq)mKL8K8lcen&D+D5VErds^ z-8LLtBjEa_P%D9puq4<$nZSB$Tvl#(GO@1FXfVwACYN}K0q2dFTjlXv+(;+;W6JkSZ)u(MJ@27U^6!)RHc%Fgq^RzHst;$_=+Z8QMBnxf zWcf8WYwKpG=$XpBLGQ58=U+;I`yNYECL;xC^PfHf{4%d*VoVvOEj~<5%FN|%8#>u? z=$;C6@DJqjCR|iS{8*{|Lt6^wskVExGuY;Dy#Ep!@D2JkTuA1#{VyVgvlL(n_w%*b z*`5)+Y&R%@U%5F!CvIxMV|$DOJe7~aOA*^D=H&Y&_O^$vQNbvRV|9qG$6-1eWu*yo zS;RR~KhLz|1hUhiZ^A-i!Y%K#$)p$WK8s0x{U~k4Fz>W-wy?kg#8S$p*?h~)&Cx=# zLQOmAE3~72@-G7-Gy3iaDZN)_Q?8iaSZ#E4RmlY(JtNiTsHDe#VA@Qq z5c?@B;V~&21Q%(Q%D-X3o^9nSP>PAu!^UmJEj*_cCJv7q!7gb0L81p`sw_ctc5e%Q zF3altXu?MvVm`y{=3oi-%XOqGA6{v=!cf(2G(I>TYa)3MdQ2i_ru{}l9SZk zn^YfC>B-jmO`Q)Q2}6UInYsodFA*!MSttef_IWC$TF!G^c`(SVn@JM=QMuVyNPSpc z2#i2ff7m2E#7hr==p@b7gNtY}TpHw38)9{s^h{x>>EopiH&~2l|0Jl_8*X2qxu>bu z%O4ruj=Nq_8=}GwM7^qciLXiPOpW9?oT;KH5Gm}tsNy4HAfN8XgV>&K)CYMa5l$)uD^TL{Xgf9yF zJNc8ppgU1CSP(|Xa9Xi@d1E6nw$hQP5B0Tl2SeoT$9Uv66a`*zWcu=FhK*07AVdExS#Tu$PXTC>RWWur^`LkbA5=>f`8hV=M}cwBc``@Kz7+e15&tSjS&~oznI9$Xa;h9~@pOc6 zor`%d_|RCSzj(0@aB?SBuWMNhFtE*hGUtFpE&VIVRT=lP-3gyc`XaH6fduHDCiPI! zA6i+eIwd8NthWBx@c9sam20pv$mTm{yf}C_l?0w6KwV=C1lrU8^(YkuHb%fT+G_@Q zsDi%uGrb06N-pEywNg0W7$|76BHnK8t;Ng1{d^pEK80K!&ZAPdJ3xC0u#*;)UlRhO zB$H*uyW;GrVQjH;A)&#N zES3>wQx+)K)6xLd$%Sk6@SW}16*UIQF!cCHkC_y8DqXOi?aVyA6cNfSjur+atG91G z;Z@ExF`^+wsxtug#SSR&e`PRi_^b$d6xH{ z&n#IW_j4&j?#p;Bk&*I6v-)JI%p&7 zaSb4uF~UbWVUg3crEskfH)xPqsbG=jA3s_@vM8Y_FCt!0p;~t<1+C@-D2oRlUZ&ht zSOGkJ{T|ceuzF%`QFF|DDIKr&|)1tdfO{*WuTt}C3Tef<5cqsF(5w&8eBrk^l9?H zU{j7uRA=kUTnEVq^U(}vAd3=X7_(Kiu%Bpp{FdY1#j|gY!hNq#`oW9RGG}swT>)lb zuqCOoGh9%df?4(6;hE!1e0B>Jux6OG2q7FKG85&>n@7hB%kYnPw<0pw^mSCn0c4-P zc>R{UF`9{3_wt8lnB+IV(?u+>?)TH6!fO#hi)dc}oXGe1T)uX$RotS+Z?=ql2_75e z-khHjOrjf2@v5el2-U(`LzE4FsDK5xG0A2Z)<>X_nvodARBZfuFN5JN>?-qegQ?vz z*EvBBF5*XwZGG@5GsFN5OKL%m8*HbK89WR;Dnm;_C+Lxi_37S5q|OnhF->CiQ1TQj z4qG)R2CHZjSmpASP7LD)AREO<958oW={kkdKyUdiSH)jpWj5YA9B&V()xW=2wTpcU z)__?ub4$<=q1l;O%m6p<%678%Hm zwUcfReS>6cG}Fdgio1!1*LG#D;>tcNIY)M*cf*)UCW6q%g&o?^Bl)F3#gCdg82PG= z!f_XBDS6e}G_=HINn}NJm-#uDTs)prg1lv7#xDUbgnsRqnlQ|4G$Tu-QNt|Z*V>F~>G$h#RtA@PKy zPE0Sl;&E1}a*HpWPR!xGVia4gAz}m-<(hj41HuQwLFwY>hvC-Sf)8trQU&0n*4~}+ zhvo9Of1v$<_z^@W1wuO35b?95VP`WiY2|}6r%JqZciF|0{adz@UMCP@IpV$>udMqX z@W-U({*QNVYuhyji_3`Ww|q!51^_>q@e8CKiHpJQQW;?m?B*-P&(yJvQnjsWQ6wNo zAl{0=>$ULgtRn4rZiY?E^GLFi^}@c%ctENSXii3a1itTGH3XmiAk$m(s$k|03Lw65 zMfA3qxb%G{vb25(2Yq&&BNFi znF`|wwVST2G<-9Ib03=v|GFB154A1WBsG!zxOnr ze;FMDKUj~jQv^KyEQGwVLphD9W!^l!&Ys(T)CDuq+u)f1#hdd#6ENID=HaLf z6yW5o;%>7)<#w>{bT+i6$upT*#=X2|%4lJ-%W_el0lZ@c1ljPb0O!SC~UIVA=-OZlGY<(IDfR4@^ zTJJQ6!yM_Wgv`A+t$*K+u79Jy`Q>}mhlk$?J(oDENs?+7 zTu2dr0a7D;uuK^7&f)$CG1Tw?bf|4@zFS5c4(I*wvxLI$Y^3hU90k8gZ%WHnBJNFo zM_;sKAEeznuPJutBmjF>C{M!8f)RXofMw`pFp3%GVs$Rh@H?vw`S}z^$t0 z+pdt=5jTOSCgefP(WgVhb^>}~C~%@}%|BQ5`CY)tP7bM_>)AX>ukuKawWJb>p88>D z?h8@?Haufw|g&^B;|+!_)strqlt16 zUVDdW`(^X(G>IYqSTBntkud0eyYM&1IStNDC44X8D%Z}*{OV%h&S zTG8?*-sIw!#7n!)mx*Q>))n7)M0F5_j;NE@6Y2j1v3)%1*-WRQZ2!}m^!WMKQ+DzI z+uu6gI2Xf)4yQ9uQa4^JGuU_;@n#;s7D{-}Hs_qU3$B;YYf_(&!Yeq&-Gxsr<8I1O zIlJNOyC#wvKvUTHWIwcneDg#DsnQ&%mgRII?FFcNe=ihun#58a~X^4S7rvRe2rWDQR{ z((eBr98R9UFyDa^>`?=v3oeHulZ3PM(qmfu&PundD9=_APc@7?SCzxxQ($Q#+93&5 zCN#FdZw0c{ZA-&k8vL#JcdIUyB|^3`nn5u#&%&+2*JePT6p>!|j#bUL%)(bz-R7^X zN*ZQ<`xd_=5xklRPbnI~wQLD`UMb@++Z(z%_=Ed|2KX=pDq*|Hm_BF{Jkd`Z?5R2{ zIt)_t)+~S2ET=YfT*)*3@@#}W%=}|JL^K~cx=L^V+9o7@68!2nGdHcJ0r-L?JuE6< zO5c@^7QFWqnf;h{F5-{pbpDWT#Ddg&47eKubL7yT)(63oyy2QN{)d+C@;cDP(k|ak z`T98kPb}47#h`0C4J>*A`f>z6mmZ>2+IWvipHoQ?aJ^r0izb20rlW^ZW?>Kw&XK;; z(B-IDr$@+ZD`rl=ty4vNQpZ`#+*qE`5za^=WS-H;#ru?A8E;(8A;d9?<}JC^*EK3h z!k=72<%rnYd_MYUGf_QL1!7!;p+#Z=|3bSBivmn5*emTH5u1>v_3RxlOeCj1%Rq2r z`#G(TLlsWajjDK$+9{bV38Fd_YJqov7Sn)WX_RTB5i<3_{FKyHrO z;}94fGk2FNWxf0xyGG_Yz}>t^@Ptgp&kzR_;};i-A&ENMcmnG&1Mm0jAF`7@5G^@M zGURzPgmhMt_eRCkIZl9+*{AE;^M>hbhNtuC5$i_3l)qG&oBknbNP*Nz9CxvSD0e0$ zrXq5(Kgd1%e3h@^Fb!n+wpD^!NGADe=k9}FF0@o#l+3g$EJZ&^e}VA%G;Nx4L~wtd z?gBry)o|gbU96M%xpan|aS?3NxU}YO;W?V&a**O-C5j83_cR9y2GlXBubv!~0^vFp z>)IL+1J6GFvp3!6JYW98RSDv?A>!eqAzV4yVKBrue=n~+iZog!GxZr;+=0^S-PC_e*nlh80IciRw={B9u zz;IuylIHi}a9>Sl7p!X#<99zT@^ds@LgIP_<%eFZZ$tk>gC?Ax7~jogtMhY%%T`dY zSZJV%^{FTViUNO~jMG*E;mM8N+ClYNVPjkbR$M1moe252At^S>6k7@~X}@cS4~w_X z>?8dHBXa8}?ou>-?QTiYMSo*z8Wtann*-znWhAEYv6sYK@(Zxgv37M?OHxzov_;-3 z6K}CKif?J)w`}2N>Ds>d6@0$%2}#*z?oTMky%B|OpuiV13XuL;4zQHIA<3TX#VcZn zaX`f}Uzf(GWIE84jsgG%^mVWCm$xfCxvr31djz!`nJr*t@u}6`cf+Yxq>y>Ky`8U( zc&mcyzClxPn-jAOWj)mqsr2G-p|T@hKs)%b0piC^=g~eret8&{t@+jYH>mcNPy#Ac z<#E3w{4J=eL$qm{YAJm7mH(-Wz;4y%?_%J`3W(i;U?8Jgk=)m$O8&F9PYoAUh+*M2 zVLMq&4P95@oAAy=ve*2R)V$f_Ma(rO2}vZuHG&t@&q+d+g~9~`+Le@1eWGsCb83O! zo9Z71jhgG4UC=dPlj7jjtn~8++BN;DYcxDK#)lBC)R0*kZkwsMI^G1_F+~Dt0Xso?yUQ+2R%!)*H zmhg}7n&ccy;G!DJ+&BIwf+<>j{%BI69I%j3*f3cGfMqJ?$d7u4`C@gl8#h5Pi2aVS zbirW=OX>5Jx;b4V?_m>Zed&4agvl8pBKD-1)(klD+jZr}G3$1zK26)({Q2tkK%}`7 zzy1FI19a;+`2FS-``LfMn~l*lDIg&VZ7ckLlXwEgiQCHdXJXvaTB6l||19yGjd`5H zfQ>E8mQAX06qaZp?e6rKL67ym<#0Qx?44809q;aKiZa#3b&{WH;6v&7seiY~Mpe6B zn-m@;T|k_OlKE~Q&hyu1_@luUbRHGs$m~zWHs!=JjWVM6$u&jP!CKFf+q3(G6goGC zd{D|_j!JQyxalWMmB$%~N0nT8czFqs#s>laDj&;#oSEWI^KGyBTx2D1N(fITk>1j~ zpj{;sms;vKKOQeR`a97s5pLBgKOapcvTk5h^hbST2fu z8{&6cA6z~NtF3e$3*S6@izOM4{UCCnqybFwB2fMPi{ElXgWFmdSJ{!G{MUkbrR1L# zk~m?7CIe4eRtNGOdXB!%MPRgyEOSXv$!11aTHWtWFp{x^zorQl0OAiOJ6YKQY;MUvYsPKj3*lX4t3}IKokH zq{P38Ls4|-HwF8TOYM^VL_XK|&pl~=5k#hyaot(A_)UWNmAfrH^JCna1ppGeb)xi7 zmkzedJJhLeJ>OXDN&jATe9NaGh&LJ(ZDU#w3@Bx6|0VF=Yw@0skyQI9D@v*e4O%Vv zWD!uHhZC}S8upco0=6c)#r6urYO@UsomevA?Sv!PkvW)x+5%fJaog?#Pxyr55VpP#Liw1oo8#SzhLx@S1U_wWN5CqMJk=ANfd-(*}jOHnPRcKifrvzpKd|8%SL? zPnqb#g|2CZn94mux`0j}4OV{OA*6k!rs$fOXl^PQw`lSfy%J@f=Sr;rrIo*?p_E4O zTwTaSP9H4TYe_F_EW-YN0D&1-9!XA^H|$`y9-mfJe|wG+Qa!W;8S`E| z7#4c9mNPcjZ)W7m`}{CojPac={J+@4RWPH!@z@!*kaNvc!$1U_ohE#lT${k{T*K$L zf==08x98rU1m61GzHMvg!uF5n+O08409Ri8G6kHkloUAppulnz+ZR6NAIi+5(6Y)5 z1`^XEUWw|3KrBeQ#9%g10Z2GPSU6TZ_Vo@fGdzK#V&|%%>l31snY`A}rBaSSBD1sV zAE)jMyg~aRxP%P!m^|d&rVnoW!LkbQU*Es{hsQkAEcfEhF40@e=y2hOi=72`=8Q2r zh?f(+(#f>`?}T5w43q6+yQs(*MR{nwB^Oi^*AcM&$gRA|Z94FgcKUmWbOF93CLSF7 z;OU=kcn^?6>{_TbUA;BesQXTacNZ zu=7V(C@KTLBT7}lWJWy;Q_(;dls#G2pk8CLZG}Aq<|~*$4wLjhw!QgiG5hyzGPtd( z;$8B-&iTS~li%Jej?PiGd$s65nDg!GAO6!Br-H1u?Vzlik4?wPGl!fvp0}xi-qW|Y z+xXti55(O+5XJ+LVoRBeoC^7mbyQmyQ(M+ht*GmCl``IYRFl__`>l(*BbrNt{foTE z8ASu0@u}7JzfSAtpSG;piTFxJt^d$*W^28!s-xLR@PuV`#u$j|SvKgE(=UQ3d4O`F zI^*$ZKVa-pKku`?*Imc04^&||`&d{(BD;xRTc||SpYGNUIf*ES3u8W`+PcDCs>LiK zpya!9&FTH9`@ROxGw!y$>1AfCEG7PT{r0D@XEFA0;CCE58GXlLe6dQ#7Z|V7;~VFU z5Tip;O`A*MX^_OBD8#@(Ure7{eqX(-bJwjXasSot+YC|Z$sRm^S5Q0@?*%`j8QU;r z*+fkstjUR&mp)Z`x5nj1l`70DK2dgzTGF#u8Rq1UZQj;gQe7Y&v6v?uERzo|ipO}V z(WLq;H8l#g6$%ynr9^eZ3u_nC10j-bhm(49X(_mY6&1GggLqUV6;&`KtgTh^bnBX(iJ@OtX zIUaFc*n}gTwkOaNC+#|LHn`c*x+|0yh3$^O9dw~f8%jXFm&g`(m)6p5q@-L%DRc+kxduC;!h@ze+4{`=M2GCEHlyerg>M z5e8M4g+AEg%LKn85o6k$zdbLa@1x1MZTy>{q@nn~j#a}w7a55@)3Rcr9JK>oF#mAz zlq!Im0c#Ns2oHx2>U1;Y7Yq0Ia~@n6I`)GeRSk|l^=*!i^o$0-AdXqN>+pZgI4-@# zr*ORamKyfT3-xqXlmR88hs~3p>jJ(0J#AYxlO}w8fZ91%8~`OQe@XhxZAiivsN-bd zveeC}E~G`CFc~8`EKT;By*(A=m72?^EMKHU6^r0ednsHWe3QVzc=&A@KB{zvmQzeL zDuOj18MHlL9S=@^0W2{~x=h>L6PUN6C*DI5bz5O*_ZgyopKCZz3URk#=>)R}^&FSa zMs2ATY6EXKIxt^N(^sIY6b3>!7rr%qUw^45I3V2cCfu!35(E=%QqTAHSV|qK86czgH{nmNDoVP&35BMAXCC=%N=-$n=%HUl334)>axb@ol zzVRSmf~a-h!1HT_&W{`812UugANGSfXXlY4l%@MCH?1qn4Ew4^Y`_8{bYeuoL8gtu zztt;#&Uj$DT1LsGbEJN7PdWgdd|ujkROV}`T$<`pQ!=@9_)Rmg^{G_UYIQ7<&|ssG z2DO$??zeiQ@99+6-qY4+0#3Pj(#Hn7z1X#@BFbX}XGa$vMZ`JR#VEtP*x=+Hn8K?U zS1n#P+ckMmqJ{k}7+Xs#J?Pmi zwkxwdUi=r#9!X)6%?gh32*(R;rPq&1_D3_t*QLE*KWOL!RG7;fj{Yb#`WWg>;9vj1p$8@-|@DX2NE`HN4~J1 zR48Sx7zmr&iK+`s*%#Bm7>{6ux89XuJ1?X4M>rQNzPC8^pl&w#lt9>)i*M1Y(x4jn zs6!PeA6{e^BG8nG*w(BRA5w-@7wPz!ksXN?oT;_%Iz()bTF2>v%cBT8G{)49}jF%;TcS7IB~nPT(VGx^l@+ z6-f#|yrt(HMhckRCD8USK+mjw@rXkG!%_#jYx&sXN8PbqJ(-KW{EhP+wc(DIOw|-g z=HB%f2FKX-9(aoc^U>X&ay4>o$$E8zH_yI&+=v@{gL-jK!F{BqKBDbUK}Q@^*EQVZ zDJmyk9ndG0kWa6v`UeO&Rg7!jNv|2v?Fz;hKIMgun$~HmUY9OXzc4w_x{L_+Qp!~t zvL@L`!|sXY^TXsi|30xmt=~8ae{Knt@wafk;i0_LZk%0E`Vl~#t_p;`xh*86n*E)% zMC;uOrZi;mF5Ch~EgW!+mtBzoP!nMTp?ZBKMi2L&%qe&)7VcvsQ)no}_N#;|7@hVk z?$6zkh;44->q_SSZEYYkpn(epU!x>KHw)L9ExtN-pLzYp&h@vTqD^Zp$cQS zU5jY6t@%SJP(b{IDjw0&J3!cpx$kf2e){LdC$9@d6Jb|?3d|*{Ve>>|Q%5*!as2-0 z^KYuCBl{&%<_%rWQs#iI(dVYOE>Jk?XS7QG8ck0pVvG^@wNV}~r`d;|$Ct?<-hL9u zH&tQ-im!q##i3=m2ISB4kvWLpa==%#=&anqAcyDI!jj=UKCbMgj*F497M}8y-|)Q- zF>6ow7-uV5BMAQR)(R!PxE`onzzJwjY9obPT^2$d1=vX#e0=wv5ojnGR`xsK#Q{Ml z&W8!eH1gF1R7}{=SI#onr>AK7itp0_tCUx+&u0tB7|adN^~PGHj(Ib0e{%4oQ|-h; zx1ya#cIZd3L4)G6NAZtPttUmIHznVybVx z6ap~os-NzSul@^PuY+W+b|9Tczqm?tgk(!QUT*k_{Lw1C+Zg47V7cf@J{ynl4khAQ z$L>|>-B5-R5&pPRx-Y-;3pSK&SB>o6anLLcOyJ=&14c1dokKqTVb{ad<&&j@ zfYi#z%EQz{y=>?8eSQdc&^g%Fx4q}AHTF4qbrH5>%)0oV!>&#sk^6S%+Jc!7R5t(#Hvq*NK2@WE`V44iBX%-lzfT=^1okjMgU=ifw%P+SrD zd=x+CGf--L>uM=e5Rv^x@f8GLZAy0DzgH6hz!>DzXDh-RUwuWp@PRG|pi*i_eL@f_ zt1RNDSarH%wj5*idrY-94?60Ke$8 zJ{7t59XZfA_~TtUou>jb_FcY8y1$J?aTW@dmBceISJB}C+S7zAlMXfN{?nwSb%*6O zj^AO}RB&5!ypTX?nf?K0Lq>Almb$RBfe01yIw#7u544 z`vRLupWJ&|UnvowlQk#u^e*!(aYk#QuAT%&&CrE*m8;)kytIrdmwLEsCM;DE#RR#& z_`S5d;tH6-@AIx5<|aodQ`w+gS}Vm~z=%AZ3+mF3(pd4-&>y;=>ZylHZj`xR;1Ij#VOy9szYN z2W;&|<2WKuw|pl!l92LD>_j&(`giy#8J*=L8tUCz^WF0sN9Ju!wTe4t{=9C@im9y)IOHz_1YT!5o>^NkAkz& zIzSz-$$R;=u@}f$xtozQwd#)u=JE)Y=5jub(7axecFszrYqCi_^9Bt$83(Xl%tZe> z+YYrXl94KOUP9ORe6@!m)%&g|14}xC8wC_Dz{!dnnaa?0Z>gz-hmcNuaVF7xCjC>p zU_rJuCterR^fd`$qD!*x^%RdKF%(f0FB1}smInuUFn0pPf1}QRT)3hEkwr0undj)}yXs=cShld}#*%afo#P`q(8G#NA%1zByGAM9z} z3@9qogBY<=SM5*1>r_OB&v`)S=FaDmiZP}Lj|3mn*Ag4WP>TV zas(HigSPw?yx#Mwh~a^cy((IqiHMI=BJz05(~G-9#(XhI+NPgmD_?PFvKo2f!ivN) zn{BC5nx{M6qnBe>*x4KpP9=` z^i>EF57z9~B25hcPGoS4X=%*bdO43laW3{w1J9{p!IaDpxlepV2i2x^Hq*I__ zqWY9{-Ms0hS0<1UWm*ji_yN+a!EHGrDxYawxQ_b6mA}59Phsz5{zl~i5-|cp<49I<*1AyZLJtyHdx>1~+}L z?5Qm7c=*$!M(v*fH>d*is``)5RnPS334TA6)`frrxR*YAs=$nMPUUAlr`qLjJ?K6i zFPZd?_e2Jfe~t@N*9m*p>nM`eA$&Fny7VbVkN-Zg#4PjlJypW_IOBkgfuqI~0C(Bl zna6{;fKY$dGH0`0)Buk0<3XI*hB0=ZW9scDwnT`5~C|^KX#(R5r0;5-uqSHDG z4!P2JyXpCZhWi}RPvrrE+}dkT1adINJ~#UiN^u(7tI%7oJ+Ka5eVfjK0KE z*T6)|NBWqt4R$}(YeOCdaLF#RgD7;%IKGm9;C5iPVVn?ok|J$*&ph8lGQs`@F~dl( zZ)mj@KUzfz1FEQi)UW|wA;EtrwT>R;g=7Bq0e@vyXM8Ux)h>jZ6BUh`O^RISRYiDPW*#Iwld|WQ)E{ zCrV1{SP$Ag2?U0j5crJRp@|$90}L;(dd#;;g}cDFE0BcmssT#-%#`(5PU_)QP=k*z zjsCV%@XpS-EYn$h;rfL3Wp7AXtp_(0e^`)4;D_l`+0FFYmc(FY-3)TU4Czx)xQWndoifJ1+45>w=&@QCiz4ct6J!3v zPj~~YE&cW*dus}h<2kUS;iDyI^0X>2gZtBa(KF6?i<(QLV-cM!ynAuh{s?Y%FbV@k zloe}rqoEr+faSC$KY{Ns$+%ER_<5TTfj0M|jp=y4lX)74W&0)z z&6AqIZX``WLyIY_F3LRzaL`yBZ`|Frxv*&zAcZ=D25pm-Dy=GJhTb@4|wH^*-x~QpF=zeNNUa^cmIHcnmPcOCX&cd2WO>Ln<_K zr~Yde+nN%hZC)|N9;tagE+s=hEQ1r~clUC{u9yTl1yQi$5kr6;OOv;P0^(TN<_QY; zLcvAasQ4dqW7ubn=y8{mPEdGhmMGh|31Z4RYGM1e7w!0Sfm-=5f1kE`vquWwFE@d< z$gkz=nqSjVpPdxbVw8Oqpk5Y&F~lrwHE;h9o4}!nRuK7rrs!CMw=T$h!fR$Il6(m~ zyIZ|ChT`#HekazeQm=WNWXTz0A52zdEn1(p*|u}OPA(c@+N{cG^Up! zhyJEf^#OAed1M@VLXKGbdZ|<9@^Bk zAT1+@`hmgM1?^`cET5m7mzZH5LpuBYwY%3J*=Z5Bvm+6-IhN#4Yl2u`OYp4(=+;oq zQ+ze^e%~_gTC26^-(O6Zr}q&zK$T^6kKe~qs*S?b9mWpl^RIeaD5!zD_cjOu()-Vb zrt~)Dp0TW-k(It&sp_g+iF>95fJyxlkjH3@9j%u_54baoi5)ez{Jwk_P=R}y9>7o0 zG+xGGUhZ08HF;yPKG^9;v-S*?>~MkPu7S%lsLVJ$U#9b;EZH?7XSP|Rlf{Rg36>lnUB>oU>KOJiJLPr86^*~;7q zNS68i2sOR^!`5Nb7P^Dbd4GLf6*6 z5$U5VBfiCh-v_e8W;E79n~p=6ePJ5HKMp{KX-fD{pr>_w@*uXu_FEF(04T%uo|`a8 z9cTQVV<=qp?3rCxgWv?DD4LM(Ryesl`*VwmOQE%?|2Ay4D(DO3*vpHvQJOPda&3UP z} z2y)$&-bf+>>ib#W?bTcz_mm(97_NihXr1jy4?53`H&pxG>6-N$_XEe_o&rXI&ns>88h1g>Vw#^J{ zg{0+FTSeFWia2p-$2L5lz1FF?islSMNY+BW*5_oj`!Y7+mNu6LJpQqL>__!zSdq_E zt9~LTXvN?@FS4_gR$};v56|g`mGV`#Wt2*8N9HSN3brbrYAia+c-j_jhEjosKY;S+ z)z~g5_Ywc;K~$ragxa266g9C-`wvI!Qo=n*@<4C@{1)0KgEmDG{n}u(lEQ8F?y!zJ z3~ba>LmbS@8M|zwM+a>7SQ=E#B-x{L+lxjnmIq39K5y2NZym=q>#%4UXHJ-7DYXJHD20;WwY;gD})VXzvh+>W^xKWT)uMH@ab>_WmNXh&4wlF)@jNLD%}g+ z?aB&W#RW1>3aj41bQ1RjR}UB06DVyI2-PENJswmV1q8h^!vitxwOQ)PT1>GY!5e-7 zbUf-|+Y(v{jOobp1(rDk5DO4u-W`=Zi0= zyBQeuR?*)-Z&Jac}N* z4v)R*59CW^8#&2kw*xm=4spr~1ZAp{%U>U^wXA3WJu9BYlPvM;5a~|vJ40N{yGu)Z zO>N_;iWCD?xnfhk-Ebn7f_BuPomd<4yw`B&igBSc;f&!7tV zE}TL;swZ;jnxTUSH-H`6A;+PYnlRGl`0Qo57h!Y4NY$F0es-}%Wx@!5!Cei5REJ(i zs;_~9+s%c#y&G&C{u<4- zRkP&74um~$$fsRJD|Zv!$)Ayd@+DwriBhlzP`)-xyz3a4%zM^6P(ca1src_JYM2Yy z&(>6{gkdVrS|k8TW5~rz^>79T=m#d}H$QaZizfVkwul!XPUs@ap*P zn5>oDvx8o#PC0~D^|6LPFvW^iVW+Uxfm9p|5s3rj)WV%(wU%e7r#pKfGUPrRA4%Q? z{97zXj&Wtae37)0^^cq1aVRML&dsmn5(UHmBMK^0SJ3?x&11Hr%gxQC!gc|*xRe#x zuE@P?XZ6b+l1I_|hssigr2P2nEL8sn#Cu-Krh9sR&3RFhW zGG2wyeWfYy6za&%1F3}HK+UL(^hc7u|3%VCs4g5ql-MgH5D6F#!_c4h0>8q!shy22_8>CAWLTPCNTl+#4noN_T}B6>R~g7bmZlk<|+^F01Q;+lO!J2~j&_r-KrYe@b=dv|O{B4zwR+l8kMC zeg6mtY9IcB8pE~!J*c560iy}+9#3$2zZnpdV1crs&xFL-gMOb%1@RCe<%dMY(M7ExRAZ+@dhLO9r*yPHjN(A_ES?r}-kFe;w>DKAI6+7uJMd>6*ixgw)@@w9 z@Yjt3(25MiTSlD5v@JU84jitpMqH#P-!3^j}*^I_2;Qv?p z$%uE6l(jim^6qjlih6k1S$%(Tg6Yce?7jHnwJ%1R731;gwz1r%vK1bv=%H1fdygiz z60oEouyX8`Z$eJ#ioFr7axabP;X<)x$kg}x0&&0}MZ|$4BF;V)ZE_tj7ucp7=9Z_8 zM9+VpbyfX7SMp%8W;LwtXFVT2yroq&)ku+U11=VaPeX&V8SkZs@TR zCwT{}Q>KH{tWQJaw4yNhUwdOv-0g%+S~Pf@H$NsVdBF6wvb?CSd*h zk%>i7p%d+QGuY+WU1iY8vnmN##WdYq^k%mN4Knde2w(SX#ni9e*j>ozOVUcjDb1N?kX@(KCqwtNSNh4`^$(b z#@g3x)aUmgJMcdSKf|vfa2kdlBS_=s(dV5~?@xH%u>R z+?#FI-bV#YUCVn%DIx@^6+%Hs(j$poYy%^2HeT1Oin+sVx^2#GdY#?0{to^!DEY?1 z!)RH(YV^lB8vx*D&i!tkW6GJ{9r#jP3@Kk-_;8r5aw}&9$_L10ij<&*JY%2Xrf;hq zOp;%Ks{fbu-s-+2>Nq8{yXVdL(SIL7DBd;da$_ z3FB6`56&O2S~A=m+?HaUNPBL*n2~$U7Fwafpk<`*`pitQ!kJ1HBwJJ9ht#V(LRnK0; z7sj?Ij^*J{xKrL z5(^MXVj((>gb6<%dBq%H8}@5#s@<94v+rz*{YIn0cgOcsf|pNwtrQD4-#&64tS6qA zyzB6Ya^G)ejLxy8ioH5!IiuP5lJrAR&YX(L<@dbSnO!6`jZthPQ}QlSQD*WB>0n+~ zKjv6jrqS&1P)B_NCqDC@-xM*(Sjw!wKAenA?}nNS>1o?rX7gIR!5y}xSpgTx&ox$W z^Ge%9AhC}wHfIMa5d>Li9eeMUT7%*SuU08wJkG_)?ah<%qt~Hx^MOBo7h7r(O9ID7 z`QH|UT~GUeKN@4`ngaA=MSndl2!a~#qQ87SbP9XgKzDB>C9rR~S8f|NCCKVfB|Y7Za4{YZ7RBB+>g{+9}oyQRWae3?#VH7OFBt ztRr>9MBIdu)vYlUJ4_nUS-G7(w-u0|7g$-7eQ(ae!d;j6c2lc`eH-ryA zj*3Xdbu=zQ$CBf6f?pLlK~C?Ozp&HMGGgy>q-M8SC_`3fA6w2mi3m-2pG7gskKflH z8M@GT97b%Ecf^rnZrba3c$Mk1(0%!b^sTKLpX^9Nn{6F{k}h2dEFsw+xETx(JRWj@ z1VSSrp92wO@3z)oyO-0=|H``P_EtOUnr{F7y+IjFMv`6eItZ*<_3D>-As(zGntWug$CF5C?u`I=hg|7UI z6s{!|QQhY()7R*pAz^KX&D#G+#p@(^j&BpWb!EwjNCJn8xtjZ@s+nKceyb|IG7Y6Y zcmk{6wC);vGvFelk>;;mq!YQo|4}Pl7dR_ntA`;Rx7cde%n%Hs7oToEMyl63d-qeC z1lF6hqp{0$iJ1N^YQeOlnpr_-NeF%-CgzI4{2&+Zm|s-B3lIoA4k0lRi*aYuB<2Tj zDQ~bilrG6$Ey?23CdS?DZSUs?qvcP`q!u6IH^;#EeolR6ylbmWQC>w$t%B_@A9=4= zubtoUqJVxComM6b#h8)WP7R+5ndv;4E%K{6-FftLG5Zb8IT_qY;m*>0tI4oFPDW)1 zEtE;_zZcG8_l*$7D|Z8SBG6Kvy&H5bY>ez-k~3jeM#p9v&%Ns~(|4!pMffu7rAAjH z4NF{>GCh3WH``HRDEMRmb*i;YbQr^8fttnZ>IVEXqgX*PWSeh zV{c4oD^p_CBZ^*S=-}Rf;j?`G%ix-}2K%4e-WFTPB3*5kl(;KeUdHa9;q^JUc42R? zpx~POF>3y$ih6lJA2B^O0$AjlRRGr*@hV)SN%>9Wxxv#l2DHfz4@W9Cd)NAH4j57W zEBYpb3Y3qH+hgaai)w>RTy$JSm`d0x9GmJFYRQcsXI!2n(`mnNTnSV=cW6;1t$}=x zYU~)0d&7SScwW25M*wnw)lS%v>!Z;b{4zM18E+qPPc@IJetS$F-n+JqM?}l^rG6dA z`JS1UDKfo682CO2PuHV|&fa&Hfd942?C#TUK$h#^=g=c~=WKBpZCGvBC><=YDBNq5 zB{Ji~FcKr2J1_3qivTLzl!~7YTZL7NW-v3B*%W)|%+*<6c9XvWnN;%AbL5*!i$^=d zW9eSNDT;anNojvG-)M*Y`@(ko`v&)r0C(qE6^({DwrWiH9R(n;WRI6)X*R4@&Rn@Q zb8RD2X*JNC)iH3RO@;pBvKXYjv)~oaE;VxS{>!RZCr;6H6}?vU^>?3!XM?Eh`!d!cDJwY_ak_(y_iz`1Y(UK8 zHG1c^@+6z9f1&~(l`X;Q+7c_2vkDjz>_dlM@TA|Pks%90jgPIdKmlqDTMN`issh~e z9LGK3AY_@et6PXse~6JAT{Pr#%b!5r3#*n#weGF|ifoNWd{g`Q_GI=THUF2*w=`O^ zT-j0>82vm#@_?DSW<=paE#49sN z4Pu%KxJr>u+mlT$&t;3e$_l@W3onUs?CTCLL?@VYK z!7xEY0*Pi)=}}dBts|qit5uk9NVX=$U53`F55K<^P+;g*9X+dbl6m3za#kt&^V+w1 zI#9A=M>P07ezVN8-_L&`p00EHATF7m6a&bCH_57%>cxEVq3A9}g(gAxrZo=H5#qALhFt%Us&FA+^j9D zUAk~d)yAEJ!T@wC2speCR|YEt>XE%e-%wo|F7SdFNiK#wY=^x6AE1*6cW^wLD^$9K zPW9(uXAUvkJ$Y93zg|DSHTw=zU#1ua!_=f=e}9vVhCm^bj992(yEQWEwt!N^6%^hN zVofaZ6CqeLTv!8c#IfLC5Q8Lrl4@Ly-f8T5>@iF*ml1<(n zi|0184?qX3S)L%KI_?7+`7CN)x6jC2`ut7*TA5#hEW!wK2Z|%K39u9MTduv_uhAuH z;eT6Z2|>ed1qU}=C1BXuqK3=?M;nm-#zS5?=XU(J!CEneyky8ftG^Ho-jgKD4!JTHt`(OW>NIIgFVf-Y*~#?B0q54zd3I9O z3OGp9H&sy)jM4tAepeX>k{)od%+14;ax0_+-Ugy}x{pgE?3U_4tg$W>^K=_MSx%4- zUGNT%fj%=rRC_Vqi?Cz;CI&N;hGIsa_@=rcs)5L1Y@T&JQx170>n!_%?vsEL=wuEn z_-*jhdyKES1N^(d4|^!LLUiNkzG*@bY6n-jxBVuiqq|EdYf@2(^F5j2@?GoTFTPo zArfzQZmJ@cL0Go3_kyW=L4Hd|nnTDFEZ5AU8~kx3sgLZ1UpioUeo8F$2MVwVMPG^= zUB%`KbidbN6hM9>kh8sU<-ww98%G~Sm`*57Od*RcbVmw+!O^f@9`=a|Kz-e0<*o^L zCE~~#qPC}afHv-RytP-_o2`1T5s!tcN z2b;{ehtSc7W2n9d3nPBqX&W-P@6Uq+8GZP@*D_do7Ban{qA>luHkBH$K;}lhVStv} zN1njTY*Qmc+`{dr+`z}nTy}44nc9;`o2^Pqu~$$c_vi(`8X=MvfTgcp{>sdl48r70_XiXw48!7*QFrQNP9@}gU>0gV zZ8qMkd~lY(F6FXvDT@>Un#&31ic$<4u?1XibTNdXlVmhH_mliBLjAdft{oenq-**( zN-0ueS`x>qabUEx4M|a}0OGQ{IiWs04>=rb{LY_( z8k}DlZt@SLu=#FedVCTH!WK%c&yO0PG~Iz> zy6zt&_q=ugBUWj9)t{jhVF@=CFc}IjOQ|y(j;AAqb6(nYLkSXM?PZY2&htB{G_>NEfOPwA` zk8;(Pk?j7m{bwJMdVI8ra8U#rF&~p>#~G1~x6(hhvl;eWzdHd1Lr#_XjYjzPep^a0 zkC=VOM#;6Wtg{p53xrm)VrNo~{3hvxvM?e*r3b>f($?e0#*eHxTQ%nu&FpB?jfhs7)1X@0AGy0&xH#lL^--urbQuT{(EPweR6jC+R{f5zrXFNGt@7Wgd81C!QKHufykVZ7LD?|t^JOuGf&yi^V0D`C@sC# zNwP0Hw7p2lK@kKtBwex6cJ1v=WSirwWQtxcAt$fY%)ryuJRHeawthtE0SJgP)}0H$ zNy0^@3O5*ti%hnWQqoLK`iqX*xqjAVen2KX?IG;qCwyQLFv}6Ly+~d@F!z%yDxf5b z$Y@VrPW-Q9WlwkI74;J@zppw)I=7ku|ETYzmdbuHB+n35^g@FQDydNe{G>-@F4wk@ zzBAsRliE0)R+Ff+0lr`D*J8{HaZM@3kr0BXrGuJpdFj zBUkWb`~M9iaqHRtS^KW)B}RII`4?H>ys27syiUE&eqhLcw3BHbD7egISzJo6&wA}T zDSoNxyPob|;Z1@e_Lp#Vo=_pIuT>B5OE zm;yt@p+^jwz-5n+yp6^$fI>%)1B|BE>5fj|sP5mG7Z}hq!WoFjz-VS7<4xgbNC@>) zJrSZwGC%(ps7o={QYwQ_Pu64Ep0#AzelX~l;*HEZG2se4;bG)NQUa@R%6Xnc=2Bze z*j)#vmAJHM-AFnh^W#fuso9<$e^Ve34VC={)snRjtoZ5>Q6^HhAMug%c)TzBey7}USn(g@L^By;mD+8*O;3+Wf*?2uX=QaO%5gMh5v$r9SLw3 zv1WUWaR`w4al9#K*Y{VMm9ZlqvC5uV?Q*;w|5CL$^dg||Q@okhjVDu|Y!D=(VPOy> zQIBLX)k|!63dFk6``-0+`(Z)&_Nx95310IiTYNP9K)i*dlsKnrU{ImZ;CPD2SW=oS z_JXF}&#$7D=mNtq>hxkRRgE9X{GFxL(>r?DO)uuP`Sj8|7($lBRz)v2sj;@yq3YzN zVrflAQEDs2upohhIX}59k5TTsNOF;J|itsSh%UC_*apauLnEmH#Gb0 z&z_gNq;i41)cvp-QfZi_ggTfW?!iwqO`<~JrNylU5NlQJK1y<9>9zN7=~83;cl}YW z6cw}463#3$rJsBX(N(l|ixy-d=^eLx!+{8t@eBQQMZzeB^(!bX*`HQH)%UX>4`2P0 ziC-tKcKN`3IEC&m+sB@C;=vPEvy?VE`=m5jnB6=&5)r9zj=ImDPdr5U7kk7>PCZ0g ztElq)JbZ;~NO=w$rGG+oxCx_b>{I|UqMAEUUk#}Bi3kjFhkh#VH_BB7_j4Kx+*CWc z&Fp70Ax$mlKFsjuqBmirZa(uBl3Z;(9URQRpr4r1k3_&M6?-=%dXXbHk_1a)3LDu9 zV~_?gfi?jsnz?<)XJ<{g{GRbw!PsCpE{;3C*1z+G0)=baDO$Y!(f!!fe`EP$=(k}5 z%2yu$QvzoT(3X%#d)4=E-~>j%05hm$WSTUb(>I050_cCv>&yh(4V82@<`~fhdSt@= z=nteIUxXgK7INv)nSw{e0FivM0(JTZ3eGBx^o~V&RS&NSq7Z-l9XJt+m`g~@E7kw2 z7H?iBd3rH0f?AOm$j>FbZtxI6)EwwY$$Sfu%J3s0M98rB!AW>j94%OcbEtvzUxAo! z|2qa?$tRam00Fy8W3&9Hex-+qzrltDaBEVCQDGJZ{WTEGczNvN%1ycQxZ)4B^~y=I zlAmlHyY!7OOTH+jT<4fVR{U9yx~ZlD0Sz?+|5!O!8F-R1YJoSYTS+^`Kc-OP3P$0& z@oCk_@seg@if6IRC1j=i1U`MB;w4}eOaFfWmNE{oS}Yl{mpk_XSSEt9iS3cR58qX~ zG&L;mR|BNsQ^gFJY!uP<9fsTDT2E8cT|RXuqD|DjkW6T#UPFFRDwn1H5b^1a&a|E9 zo#2}I3gZFU#T=bK{SXv`nsw4DJTfn1ELyz_$1UKvmr#88S)ebzM(W;Hx`SK6I387& zkJ_?*K7`wvR;PGb1+2WY^jRT=yv9Ss#Sv(axhEX%b4J+BO2U&&`^^xN5_4roFbSi* zvl{UtSP2L ziRECd&ZotQ!d7(^J)u-w>qrm8_V52(zclV3s0mJwy&Q|JFE271B7bju>(P4bvR+=y zU&Dd~qHe^@aEh|eXC9lfp3~n;*&mE@AgZq@v^=!|S`fZWyLjE!>>w88pUHJhSNzP+ z#w}#Y+BxTc=!Y-9*w2ZBb7hdx@!0r#V6Wd`K-KgjCn$8H$fR}OcnNie6;N)ZVoTkY zQm%h`fj+@W_w-?qsyTO2PN0F zw(zVYF)ahWmqI-d!^K$wHwL6<#RS#H&r6-&Ugv!b7Z^Suij}$V)KkQ?EzUo!X7ZG) zdJ7g)_?&&<52gBiWLC$cXhp-wn8JrWY1?%nFW+iGd%FVL%}`Dch*j9Dz7Nl-SRh0C z0J39s0XT#)Obq#c6z{ga^RQhTbN+rEa0&Sp@NbG`9RHX2X)P{(THC0De;ItpWJV3& z9JK%8IUpTV0}BY_~^>X!R(D$ncGYTl~Q#z34eiqMAx1em)N z+zIv=iR1D+pWv~s6w|pDSH&^8s1WKosyK3&{Qx%O_3>6F?ym<=nK2_-Xe>o~5SUx7 zBHu866~6mV*NSxu5K@+6;G`T+I^^ZCneuBP3@D(DYesw^Ud6=XkOpX&0TX1qGPcG` zWE1h!&dX-T?&$<`=+S~m9aOxath-Y+zoeX66~qp;Mm`4qLly-=0CeoIrzNgj2&ZCo ziZqE|$Hf#DM|1M3CRLZ>vjLplM6lk z$CK06zo_!PO+q-zf}jE&qQcr(4n1?kZ{wgd4Db5dhC>JY5n4nq>q256NRXa8;cM9K zwN7x7{TjsB<6V&)e4BxFG#OpxnBF0wg+x>xs1gvEBctiRtP<|iqC}Xz452zv{>3V#oYh-Ef%rPn$3!CuN-7sc z8X@lZZ8=YE$#JeDs!3D=HLg@cXA+g{q_U&9Y=8gQGv^a3*&?#X=gTTw*W+c-*FgpC z%lI8sEjK2nFpi5{_C@2iSWKZl4XlSo%V|IALwhHTKAd?S?8!(`uQeh}A+IXEwJ?XndU z=yd#CXpKXKN zcdtG*Afkhi0|ddr8(s7;OhMm*&|8)C4#^Ou0-^!1nlD!m$5BT zlp+Hd<9V0)mXZ#QPPg9X6(eR90y1iyb>_}K4;}O7XZ2|-aYQ`;G|PHSNe|rgZ>doR zW>r1#PP7pB!RKlj#TaSH`>2MY87|AC6fPlNwebi7CWD)52D9D>*!5*}B`5E~BUs=9 zUs94>jxj9s;C=dSX|?gL!xwRPKk}`5%P@u-5#dpN97vDP_eeo){b+1HaIOwQnu^}w zG)|pzw0`n$&X?sHMM7)4S0jS@QbGEm#Lm!;6$~CHPgV%PG$Fv5&d{JGh0cX&B3K_W ziM$}fnpx>jSU>VJX9P(^5^@q9sV_=IPEMyxh0VX}0)f(Vsam&Pc8O#TxQ0@z3KX{b zPsjtwJnJ+HO}c{qQ~$RR!o3+j84cgRuaSq>zbZ9dg_xvaAF`%`1`QUS?vQs4lCk7G zT?MV@y-5K^tVBB_PdW-AJM(t;P41T&`4;5cU9Qm&c0l1ySz~r(e96g6bogi$rgz#l z2e?042O(>TXd}Rqp;HQ_!FbZD2A#;jw2&s~+ijd>Lo`|fDVp)TNz-wBNvz*)%9^x# zyD<0kOnw8U4$*)D){PvUI~ccnDP{sTb`tB?m>7YEgu^!>f3k0d+MoRmNQ%|G2~hzM z;m7Htv);PO6AV)WFv64*7`gzsw^}#}x^J#n>90Ex8+OR_Ik+gZQK6q0-~Y>}&G%*M zx!4UmZ_E*iPUc=0N^ecmsHYQbFh+b^l4u3;#AB_Y;hOo(o!AvdNhW4h2*Zr_ldsI0 zMuZXpq)E`Z_95~@e@WE}?ZnXyEXR_{WNh$fm~mgVq}${aZm>R5BiDJ^KH%*E@FmmC zEp{(cX3>QFszE>J421NQ-ZOhMVWrqZMD>o78AUpQ7jz2kc2Q?jXeKV88`=GL7Wyr) zTvRV>(K?)eM(XPleoc^HOx zVpI*MK&*3!^FK`EJ|V91*syXc#654+G9_eiB>Jt1F)?Zx=(oCKUF%dCU1eQrv6l9* zio`QP=uWPA1xFciMaz2v*97#e7L7qWS_cYrpo1lvuS1x=La}M#z{R)w;dHU5K!kDM z*j$o%EM}ku&0ozz3wfH-K5YRQ$;)q41qnIJ zDtJjA9Ju)HGO-)Vr(rQBA3iK>{H5l7Rha1yMaPk`-2yYH(WYVz$R-=c$Li>pS^Mo( z)lPosLvm2-3%}WV*WhZ_86e?k`A%JJJjZgO$|^C1BWLdyS$(AtsGzaBqWxSu@;kbF z5->gZfa$i4&sD%~^MBe>9^JHj`>An0EBf9^x$JGV2c7=~Bo?m)mse1OYM%nkt~TJK z*!{~7O+@+iyiVagv~=9gb%g*?U$Gq41B!1uby^tTg~BdPKf_3 z396Po%(tI>*y}>*qjs}1{iM$eTy4H&4@sfq^srdAKM*Ypzh;veZ=jG#s;!ml1+bDO z#g>?s&c3R_=;Sid%sZmj?+a%Ra(4E^fQJ8E4-t)@aAQUgtQNqJMFLEbuSl*98p#V= zx{`nZgi#y|FXB*qJtUsJqlL{E&@ddCs?KCWVl|8N^PzEBNzjq&W5mQpbt@JRI&QeS z^OEg|^dfiEAFvgayVo$Blbs#$FSbv-PMClC)dVsq__rhP=YQ_VTWQ8QdRBh|%sD{O zS9L!Q!Eo{C|Jx8&qGtWWJv2ovQ+j96=l3~hM^%`@9rh_~l^lLPbn$t?6J*PEQj(`D zevlj%C|0o{#;o3hato;x{4NpWN9*Fj@UaI%`SZc0{lRRW(;Q4UpbVUt1QeR7G~Kd# z(kI=|8-oD0VEy}&j0nTWk=YfgjNi*5(LlP62aP~Tz z;VM2MTX28X26vI>7SUm#+Bht^HecScT12B0N#Zb^=8f^G`d~ewR#CJ06!0RiDz_8! zm9HVS_QAGS)Kq>v` zonL6WqDF()uH@U4F8>uTd%hbcN)Wtnw>-u^a6eF&kb`ZaZNeSu4tuU=#{b|s?;ZbD z{1nMbvQxr_dfVi?Ioy;L3Y?nFk{2UU>Ht0O*yhxW5{-Mn4+sZ6k=pC-!Nsx~rTwWv zJ(;yjdb^!{P5Z(gZHe|DhFD65o50$H!0P(F*o5k=N8`+-&qBmJ$@r$9L^+oFkdDr) zv2=W}B?i37wacRqQ0$`aN&;L(&gRYBBRiiB9R{Oi89z42VIr4Ny=z25*q}z;ie^Wr z=N$tmREwpLm?K9i(;tkXh9~)zPy0els9V~%{N6*BGR#>#lbzkv|AJGzPeKq>_Wa-? zm}}4fFIRcdHu9dEjm%k0WanG>u(WF^ACX#>jKBk#$LqK>?z(#(H#=@!>BAJQA1N%w z=AkPWn#3o4TOI&AY{k?`Vn_5AL8d-_H&SyN+K1#wbE~RX!V>SlVve^|$P0o>YE>n zjRSCIA8wNI)cSv#@iqFPnZ&b$7ZdY~Q_-9xeu*$tmtG|jq0)1urUf{6fawZhiVUl8 zGH*u6Av2+?tYZ%c^>|d5`WhUbpicT7Fa(K=6RGoV8Ngig$owqA3j+ZzGXSV)5s=k2 z;AnOgaEv%*qvumI$j=w!2RUHxlFQOsUYgt#pxSuc`TCd~sFi^wRJ;Y&K4MBKx-;o6 z0?)AoT2Xlz*30QB&{R}tktsBo12hqGJZvG`okXT-456l1d`CPW^Dd8s8j=rH2CEOP zE%hl;oiAODApY!j^>Mv#!Qx5Dz0>Ul_7@?=lt8aqv3m}|Ws3~GEGU_nXN9NVLxiO$ zJZ=y#%bWC^)ED2xqwZ_EKbdp_I)GVCzyd_SB5e76)L!Gi_=i0#KfIo55#ICIkJF|dO|G{sFuKPST;?&pQexd1-qK{~E3H_pjRJL2_L_}&YMhWY> z;Yd!<|EK#SbPuv>nS3bjJ#@)Eg(M*}Q~J8~{GEfPJ!Qw}7acg@^320Kt*}T7DOF^Y za-~4evYSaC!Ecx>e~@na@+Eh^Q-PowO>l&6xpr)3a-j6(W-I_(CFMw3KyTtz56g2d zv=ILwsi|_QzGPCOL}FRxevdEriix@gK=9rUH1B_4q2BjY^Lr zL$*=%vYZF@zYwTySu@?gQrwmw2E@waf;HZBDggRznN=iI%KwcacM+%4^Y`!>{~5oN z*gSbc?_qwGc7Q8oevv#^{AJl9x_AHVyrMSjbunoz6=`YnBO zd!tU}g8#esE&#GV_g~VCuc(#V^8Vm zK>mqQwG9B8iF&Wy{B+gY<|+pbR(Z8cqMvC-($$0EMJNzYd;BEyVpB<18M%eG{R16K zkxq~E4QZO4ZNFkJnUCTrp!3GAl)ikKf3_B{u)}fK=-YDG!{Up7FF6t0mNcdR6NrO< zO%FdDicZcG6@1pTGdFK(2jtN_MCkGYJgzR=8&zHGdy*!*cBLQH;YTKaSA6)`0L*>2 zg0swW6`i~wkacEays zR*w-qm@hDrCWPP}9M4qY6N!se;t!0)J&bXD_t?=dJI9dbiPS6`@cv-=@czE|57A4( z&9SNFqzIP{oWf-TL%%a<6H;PXA0Lb{nC*o+*(Q?*e`HPtI?yh@J`iih1@pxjZ%p;a z;z)k%ywEk{OQVeuyxZQnuV0{xfB{*E5~bW&lapY=V;-BA*SosF{m3`zFpQt3LFzTL znbPp3r^YY~4pnPC70a~WO+G0n4!{BFq#CAk3(Hv%_t3?lf{|`~>yp5DL;w%Z>s;+) zg(`Vm5W~G;*)@9F1g5UO(M|P%!6^>i9*fs(dXi6p%)72}KIcR6`>zYI{PIhbkT+7C?viAv|3c*WGn%S9!E`5FWe*r zJPx5C94o$?B$T7kmy9-{O%jTK0WZ`+rP$;+P3YIdxdvsCe5}d#YPL|VLp_*0p_+je zL3LoGf5hopijc0F!E~a~NP7~6lmIA5|Ge($W(W?g0Kkt3JGn(BqkHtvtMeQJUf-yd za(U4l1c_bg!HPOlsD%e7VZYYc4R-XtFTSe`wG3c4uLrsyJHec)OL$1E8M}S`CGdak z_zV2nd+&Y~6D`bt4Z2kk)Tkl)iUZtTUb=toU7Z;s`Xl5)>1Fl92_yhTDZNtJ3?aPw zx^+*mN<8(;FSwTB`89YF8gHH)z9@vV2InKc(2&Q@_oR|UXUyVt_SN~@tgUtSox7Ge z*ewjL0d;FQuC7>^n^M|H)z>wEYVG_JG1OxLqt(+lvhs*DYTC?w2jZH)O!E$GOV9d7 z;W7(`NzizkMSWakw=_vmF>#W0w3tKys`bf*#JMm|=uf2ZEiRi!Sw&7Q-$ZAg3+3P! z72ke$WbgKdG#QrQ|(` z*z4)OSrKM-$LuJ`9b{<-Mw=QKM>Ft2D(3q=k{%C1#w0A`4fBo$^86_`zV>0c=qEVF zyJO_RWSTP_q8^NkDyhGnnRM5N8S~2Bz2$@Wd!hqYEe$!X^RfxV$mD?+nMl0e7df|2 z`e`SZ$W~c{kb%rL+}*^N+0d*}W>6t`;_CpYc;zbXXr&%z1UW6GPLwbRm^m;r5I-y# zcQjH{FWb}ok=kXw&mUA8VR6+-SL3F+pCWDDsUwDF9bwQ~&` z4#mLWZb0;=-H)f5a^so<3_AG z@ftGyXN~Xc5q)3~FbyRV!Yqw;_MpX0FQnZ~>b~^U(mBCDrh91SV;LB=O>M5TJzo4x zWD}NLad=>+UDqj$#3szLBkQ?bbef4HFom>W;fOv@AVi{;S?jZq{`}`2t_Hi(^EhUw zvLSBvy^VtJuYNlCO>nfnsrtKD(QUN?i_zZ0 z`cq-UlCY$5&UI7u1DZZ0i=)MQX1HivA-qvomA$jiBqCW$v1Ybk{5LYqQ7U5!<8;>S z>=3D_i#u$o^4d3C_49Cy*?OJwMprfyos9yyf@@=T#OdIYcY}FCM^t!u8s)ld12D1z zHs~0Ek#gfkRB{F?=%xc>isY3Ei%wx)$ICW1JI`l@=Z3RhvXvavzUatVY^gKrCVg}~ za4&TXJDceRj7SqOs)RrMdD$XOdZ)_DXija1z=qumNvVj&h~au9e;qq}U2fJ^IB7&j z2>X>}f>^KgrM0wAm=r=-FmfL z5SjKw*DS%V~jCFRB|)qLpUFVr}oKCbbMJrmCc|`u*fun{9R3mN{`OW3dd9*x#0*O2G|sw3*OCzSneXq^SVeKl8_zAowL0708rJQ^C03TIzM@y(=+&!;a81g0_BUk!G{iz#%~sL&=lY?IJUFX7}JP)+9{*6w20>o)r#v@$CG_bk*C z0ROoC#_Ytf_A_ejzCDy!p|`8t3#8}lUQ?D+!CAxX>*Ie#c8Q6x=PS%fOkxCGqmQ?2 z?eCey>@51tBGn-@T)=B0tN_%>mvC0gp(oxcH^yETlVm&`8t>z>T%pk(b03Nts|>|u%1y7@xplYWQgYsw|Id^h z>8$WUU7bf*M+Kk^DNuSm>OReEx&MzC27IJgYP{d2^PhkE6k6`?)>lT`{E!SOdGQ`D z)Ze%3rFUiUi}S~k9|t(z>f}n z_4S2{0?L1w!L?N5S!5u=6n-gV*lBj1O3f-u23(y1p;3PI8QP6($40c~8ro#}T&3wye`xW;H9_8_G2NHXhf@%#6<*d^B=nERg|tOc7`ySN3?)pd@7xC+cI`ljC4^AWSt?k zfd6kjf)aWRksYR7rxe+z+9H1iOqTrf5yBK8pb{TbXpqKsT2M$3*)hp38J?dOhEPivQgAW#Z|hTbs;hUOSN;FbaCd z2OTf13M*a%Uauf%J|b0)1Ag!2$OwoLn3}1u*OSfTc7#6JaJpp)8JzV_*B~#R`ko+s zs=aEs!OZMa)`G&kNQgeV+QS621Wf?R4oxH2Jz+6x5~fKsqsO#317{U#<)R}y+R%+e zfd77IU7FoPzwz)XOt+shP~Z9oz(% z$K&1uGJtdZM)J1d;PoV|wV5yt`?p+|)Tp@cFg~fdQd@_zgi+N{7V{eSg85OGuIc9w z=Z;PeC>vHYoZWalIB$=vSQPFogp+Ri&V;W!&brTgL{f)@L|Z|%RJ4N zXp_-VTEK7}8vYV%zhcs)Yd2vHLLdOktLl%J8f&{LyS?BYDmM*huRnU2# z29N93`wdj{ujSn2y#G2>)0h}AXW+iF;{ON~ICQxQVgLj;kR3(Q9hNzppkphhigs?A z@?Tb1Vh$lUF!qG-<*cyE)Kg7;_2t$nf@QA z3_Iq$IDR=Y_;L%WiO^Vr>o8%Y0>h(Gqo2{_#|l}pgc}xaM_+aIzMoEnFEM8N{ALE; z@+OMjNsRVh+$wq_XW)%c(Md5GF4DeF_RRh5?LRihJv{D8j&a0z@$HKz4_|_@Zd69U zn-+cHY~-N&R~hup@M}cC=&Y(TYh)y%AfGoXUZPOVsgk3x*42*dPZkFr11E(DM5^E_ zpI}`*ZD-G9th{Edv_Jifiqr~K1gXm(;i{Lv@tzvpwX*_%t-@*Pm6)*H64P(Rvt|vz z_iug-26r%Xjk(|Mte$u?%vz$;#&< zxKiNWr)Zg_(uwiTcm4%#hC4(kQ)Tu&_sk;ChgjT#1IQhAMV_R9PFmor9#Yx5pCXrO zWJ3KHqb!6jo$c7`cdo|oT}x`L{J_wbXK_qK$3EozR6rx{LxGVSRWsTmWa<~NraCth z%XH#CiDmblecc0wAF00CQZM}3#1{9#82V9gva^~RYteg)A|IG2e)xX~d#k88n{HdU zY1}PHa1HJR3r=vC#+~5q4vmvQkl+r%8+QvHAUJ^_!QCx1u7Br!zkT-DW9dnIFfw-B$)rAdPVfF!JBuHWUlf7ixVc0C6fy?x7AymDn=G9oW zp#W1NiQ4<1rSv`VdcSbX6+}`!_hNn|dvCd4heZKhnq^8&xaqI{8G##$E}fOpYrqE< zO5{H{o8}f>IFLWrCMT;%_8of0>D+DC`pu^C8oqtU@J|;NE#>|`9a%9KksWp&go$G( zP2%p-ZYPnMiDDUiY${L|Psf%>kD|9BYDb02+i(cOyh#&LpiB^04GazG*&Qq`R`U4Y z2aH2n8`gR%nhr)|r1x_Eqm?*s%s+Tkpq`rF(=FGtHrRa%*yZYyJwlxj8tT|cP^whvlt<3kK0s8rSGMbaey;n;?HLW zqRTw8=`~rAxOpFh83czW2bTDPg0*t%6FNo%JFgpJU^4zD8PQu;++FCkp-G=G=G!q& zFy;`nZH(|kxGy}cJP@`XO5{9KiCED_HhabizeBP2FrA~M9Xkv2_{Tl;eLCfxI}bXh z6$eRNK$+OXPL0fJ2}yR)f3;Fb|7$Cih5ncPk6Ege|7ZRTRtKbWAneyd(X>l5?hpme z71YK>#&JAO$!L3~z~^i=shA3$5b7bN-J-Ou5=XHN<6ES|5Mi7%SM;$P@nQ+;$=_>FxZBK0O#Gb=F2#9do%hdE{lzTNJ)98N-=LK#g*FIG z4>m~(e0I)gc@YRU;d9o#y}Q4A=;SzSx@pEf3J5Yj&NuccbGXNGxrTY^q8r3*0Brjc z-%HmMphDTQakf#QT9w?d&AQ+^sxxqRPl8NE%!0VGc6An#ID;RMZR{1p?=VtF&dv=? zn=X=mA^6k$Ro#hnqLt~B={*m&$8TGRQF)1(5pC&|Fyf#pwk@4<1jH3}|x4ATK$i}dmkLN4|E_#Uxe z$f4zS;XeZy>BNTAk86xR4pPu0Y+V;Ii$&oy>ypk9-O%Y!D;i)NYN4P;vw05wlgSw4 z7QqTSRNCsDhY1)y+x=lZ>;nZNJ=8j$-gPIY@TDWiMtoj@4J}cFS-}Fzwpf1=t$6JN zFA#;oq7Dqqf$3r;sGOm$6ZH==?t4&yU1*o*vq92}2*X8+td-w&=jjro%*__jQ8&5t zI|kb@n6kmGi20Srk{9Z*7Gu24kXgU0O#09!sg&53SXeEXb2tCJLcMUa4du1rhc+Z4 z7`tV(5X2(b_f1c>7zGNDu+{yW0N)-Lx2ybD-6f6?r~L*w7lIE9-oMC1D$ebEo`{~T zTzDc@TV?A4EqM`1mOjFWPY@xggo!j#`-hnb3fBGYRxRaZF$=)Yck=?Fr=%Z(J-W

U3Wp@(=H?VD41NZCQ>hjA>+p zfYslOvRi-l3bc9fsdetu!{5|#{wRaV+jyhsT4~p0z~an0KdI*<8>+_P=Oq46MUqIO zSKbO>CDkGp!wc-G66%hf5q7}~jp%=uG`A3tp5wM6+;IeaxnNj1?XH0ydyygBVnf;`$| z(~XqIIKaT~6TvT4*=QrM` zp9Nw=OoaS29{=V_Q3oE;{_)R~3K_lsZ!vkjiDX$^aYWeJr+ZKiPC7C|?jMec{-U?C z|F#%^Yqma=^VxReKX9+ZkUpZJ+Me#u^x3u`Sd)JkO$qN=z$H$jg*md-tu2mtr3-mg zx}RRUpKEztWG-mYOilhy_vW4G>khFD$b+GV?@_eX81qLYGP&nE7`ipaL5R_u9?nt;^LTI{`BS?C!|$l8dBQ6)`I=GoF!}=IEHx@S^l*0ZRnuQ+ z9V8OA|J$hBqUIx~*a(B1BPwhjDnFK}PH^;p7j}49|FvL*(EvZ0q-Ou)yAhGi4)er; z;a~?&cI^KrBLXgQ&gTqooTFj|cCx*`M^AJ}z0VA}@1^G0PjY$z>^WMdtjaI!84{qw z^i9N1OMWXBiCcelwb)^R97GKW36h`@4->*goJ@_^jueJUp?e)h)N*SrgW1T9bQGya zMI3!Xft zeVfy-&IzuyYDSLv6r-}IAE~L_{&|1GhZp6xTZBgIGt#83BfT^J9oHw1okZf3)!6E z?!m8mye%1M(}n3d+zpLC-A8I(6UW-EjXO`aj0(M^MWZ3{PTJ)C4C+^GVvkjd`hZw! z029s-;ARS7AA;m-v8ltS8r6w@HJ=Pn)sIrY!d0^2U>~}6vF=azY>Zgoc3$SZX0Nwj zNZy72+u3c2+NTyu^(=Wi)Kzwi6_QrzGo!<;KAb+}wZvFSJTjW|!fIDZ=mEAAEk}Qu+5iqRnk|_0*w2 zp*Pzbg8Dh#diI0w|5|FuVtt3TdtSHyn~xRauZn7Hp}oedzsd6OpE~b<=xP4m=+8J* z1!yniiiX6Sl9P3AtoP|9T*#}V);~%ll_bKmxN2i5EN=SniiW)t^oEdRbH3m(!iO9y zcWOL!)9KineT(hTIpOP^sao35IMK8Bf`0006d=kw^?J=k+fEI`CGk;l<&6&=%1D6z8va3H~aK@3>X58~Y?;kJg|1(}@YK z0~G;wIx#Fcrp*iqQ*mO|`|-!okuYAm@iPnM${>7LQrDAH|BvCu zQz#0#^gQ8T`>FsPtr{U}PL96=864jF(0}-VzdK}x@cjHj`V?OZ-Sdm;blm{f{n&bi zUpuLZNL6ZTA#hO^)bk#hGMYUwiOe&wKz-mf4sq0CsL7>hKd0vb8P$4@^FLTPOnz3g zxjsqUlqvOktv5nn05>!Rd+oSxQxuL=6x=Jlgz>T{P?}}7ke-jZXRt3htc$n#ngnN} z1?Ep+!z8!}){IkWm&Sojd6IR4pJ*HJUQy~)6GIE3m@&~jL(_}o)|+?Snf?j{f1PNA zxhZ4a^c5vh zH}2`z?w5U-eXlom4*n5_V5yGr4vWwD7ea`r2cIFz6|!u zJbjPVwq#d4!>AR`u(Coy_8(!%{r^pLFBRSMRWlOu_lW`+DTgL#r1chdARvq6vjP9q zj7-2sb^Vc15ngbyY4MN)F~Z3GpX4~i3hvu(FC<2nMjKXanulWF&~ME zUj>n-SQ&+T{++i#j3O#g z1RPEvul=C=w2`Ta0* z+|g=7kjKqq9{?T>9_L%pk77n4cIs-Iu+1hqjqG4klULLOn4SB2@g2CN_IHaU;QF(! z<15kQnXW1++)w@|x&!(b4ZJw>*G78FO-hol%rZ;lUeif`L_o#hTrm>6hKhiGYp%@9 zK*Nl}PIHy=OCj71^$-=1Rj$yU!CIjLge6ernjdc0Vo03%9obR&%pTsMTn)U^TiH&m z`{hkSvphj@ndlyH3KU8Zcp7xshJ9NwiTMsWkKX`qT~I%(iseG5!35Rtfg`wU4f0Uk zLgb%9;`j8hw&LyoP}NG*bGVY%Rl8PlZNxF!P~i`X>^!XNJS1xr9;U&r6)Kwwc*;lt zBkPXCd;o}c>DOTkjBs&|MhX8kHQJ}RZmgJJy>Qsh9{6^Tl>=7^w3x6Vo6qztZeYyX zRGo(*?sIiPI3W!BN#wzWl-r@q7fllswv0^Sw9U~@917V^O6!5Sj*9uu>Z;E3^Wo#+ zG{8zxi#P0JCgEv4;R>d;-GW1#oPgt#cO_q9pA-<@dw<3f8aqTXY}=Aooe{J#@Y5em zC3AHkR1vI#%eMP2p9{#uTMFs-UDrcJEuq3xhy5lv7dS zZBp>&`~rni2{xA1{YiN$7<{s;7-WfeKybK(Um@LX*r6GyfJas1o z950;Q*>`_^Z%DHG)rpJ_T(KIOsCB=*weQ}G^ck40)86O2GN29R}MD>GqM z!W~kT{9Q(^Qg3^w>z+N>MGvOcAz1f*_tmEyLwDe-p)7XcS28yyaR^S&yVrxpvpu+J z`8BwjoX)s&ZA%C^97q*dVi8FpPgK%T1_j@gzwMzbeio*7$X7(ez9necNaJA&(TLX< zgTz4BA%$NW#U>b0ebiJ#t6c&@K8C z9N=@xlP5y>C2KdP@`&~rnwI;TDHYo|(DuWf8M+RIzxliozw@^?``<%M(V;dLzn<6p z{qDw6A4x4#6rqly$~VKX{St?570U~m59)s}rD6oa-98#&(}Ab74Yp)1*>t7_+)E~M zlko!=#m7uxi3L50c+5tOM5|9K_Mza-tgUaUW=W8^HJ#}kOX z-!-!bycEPaj}T=@AgBZ(W%38IkJ0{K1?cSncKQFvu-CCaECEldSBdr<`w;vBoy5Ec z=KtWOW1{~0gWF8kpcCDbHGc%B`x(Z{-d$%kEjKcB{TctPj_K?)of+Vy-0wzt-C=6o+H2DN z*q~pda!r9Dc5O+nO<`aKC=aj0d)u}@Ql2m=YnJ-MEINdNhp>Vwevy-gHT+H9V=Z1DL!Q@@gm*RMkVr*Pqg zHVsGV6zz^uLpsL{fU4vjbr{!7UVXHf zb(P?}4w3TVcB)I)>ejE{1gZH&YjZw=h4J&(QXor!+C_x!jiqSH6R#OjlUcc#eD= zxCMPwjuUwe{(!@c&DX>HGoAm#Sg&ikx=hL@11+{pwpAheGbCiqiDk|F^-AAHEAcMs zs~w_EnIPu$QOfST!0iEAOit=o8P7FoJt6e#zbmj(+Wp?6hbOtp~?R1m}9f$u3cBS>ShDh zxv|&Cerk%y?QTcFUIV-z3il+Q7&bo|s`m0$wZN4w?d8~I@zJ&(=%y)P9Vq790gD+_ zJBT?Jilu~XP9}2YM0=LqN@&};B64UZAbA$*VruT zi%M395TLt9OV}#5{7Wt$t$A8rm1Z4M?OnT5+;{5vihuOLxj)X*i=Uv@yqW(}u(Mlk ze&!+JQgGG$LhD{%IuHFeLH!t1KN_*`9%B%W?tq7u>0Oj7+7c_;@=;W0Vcn1QLRRPx zc85)#P6dKb+?<^V*g3$Qo~nDgb;kG@B(!?8_Hx>aO?r%2>vvPi5&M*`mpeSq&)sW$ zweSNiiczWl2Ye+KHGlW$u>vfRtZr8CS)h35|5@oMsE>~vgD;!g>|?Qlo8~==X%}AX zd-bs*$>{dd3$N!mPt(h zYN$TfkT8X@d9i4OZ&K9>+7-BulJ@RwVEUqg;vWDI(FCvvxPWW+gJ}JGHnPK>nGA({ zq&Ov*58)X6lR=X_f|)0heD2D#S<-}2)e;A&r^~~2_MPjq+61q9coArYit9WD3<8>< zQsJ#T;6&?eaI<1$#z`m(Qkj;FWMvx=z5h7UVOoZek7uI@HIOuxGwvfJoX#I5dtR42HE=X&*h*Zzy=bza#-SfQX0ga=>FzP_*Nq?C&o9`d5u0HQi+^Vchk0YQ} z`yh+`jcRl`8gSHLa7Y?Pz-TkE8_nk%pdBZ6&n7}^Ap)p?td~6n)sTL@N)StdD!Qw0 z?hzL@L#{iyz^_^;{K?kHb73f?WEw-8`8wW})+L}vS4jH*N*fmo{WMVJ{#fNtSnj^^(89m zv5RxHMrs#M3-F0aUXQ|;@!nuKT;$~qEHPcp`$I}$V*5D~F4vz8X2%b>) zleI;l_GCdw<)`Of6<_p~%B7;SKZe%Rvo2GXX!QNiZb{#)I9Ids#BRxj9`~KsOv39r zULx;IIIHlemY?FAk!$a3B-bJMF|-ty_o}^=8|iZE0*5X`+jIA$!se)2z1l*TMS$u- z>PRb|R_k?$WT{j}(sda>lK&9X+)wtH(ZaDlghM0wCKwjvk7*)sQHOdaB@Zwl@ZAq8 zylg`VoS?6n6qF~6+PQW7uqO)ry+zep-Zgkr{ZyF>6$?JdNIVHTKUAb0#WrZYSitW% zZTH<8HyRaHk$Cl zL8Ea*&4u@|JldZ_>6cAzuigpMu;+U2BTa@!DE+?=?w--bY|I`zXpG^ zMR^fgyJZex>f(<5j!i`7L%beQ-5T?%LB+;#+U-X^5Ui!fh|J9BUE1O}K%s-RWGTox zGf}*_fX*MjA;iIj_ThaTa!plOa`p2{FlM9`xc;vo+3t zBS!40yC|G!OOf!$L=67J>nR0R5URT@MuI}65O3GG8i@PkyQ`#8NSRHH@IF3bI`e{; z4F@91*R~Ssq0kg~k^<45l5g@^MqaO?ArhT{(y%&ZSDO2grI6BgK&ievqA7&_9Ch9f zO1NIezNC&CeTdN0-dZLo&IrLzU1wUT_Na=?(rC*jS*(83&{8*b>bWH~-lyjW`z`C?r7d9z?huWC=hXtJ3O78bViUM>5XGY+% zbHVsUQ7IlRC;CMixn5aBJHPO|ni}2W(&u42QbSO^PPtfZ+LdCiGuP*Kp`JCiu{Sui zV}RJ}>>2N}c^rR`3?%@E5)DI@(g+70xRI@DHw;}voF)cD2k(D344Tg|Xfrg~vtz&E z_7;BV#Ovn%gO~99y}!*%i{Y>rBujs=6-b6%DRlo3KXFxmDdxTH^yTan(=+AK&5m9T z(U6>eK_Sq8*|zKT3O+ccp=r3;N|#wEp;~3+>uNz`LQsh}O{ds_)GK&x%zbzDx$!Sf zAE<%IS8Xjn_jVNntHVV389S~hmrw@1- z*LRx6U%TBMEQ{oC*}X;3ao*fgHhCUqqoi^9MsroA+=d|BV>jZQFBZrF2U{Wj^l$^C z#{SQfr?VSRuBCE7&u_vx!njZ3G%j)PG7PpG`_cCUKDr5IB`uL{b0(%!%f-^ULA)E0 z0W8_;qweZK&^H*A0u7T}+RMz{xnY^hq^>3Vf?3C!q;zydrBLgUbT_ivk&!WSwROeD(%AcBdv^f7 zWL@@bG#N7iE7t@BqR-F+g5;_SiD zxlfmaM+bZFR?eY|3iWv5DTvKR+J9n%=)}c7p8KF4(iO&2XaICX=;gso_n3O-He0#! zQRW@L0{p-%Tx#ltH|taCmDSgjN+n6CLB-#Bm^iX?Ev{7Sv_DG386NQi#WZXJ!&P(z zcC-fn(#pg8TmqL}X z$r4tB`s$}CXsnUeQKm(@dorbc&`f*kqSb`D&7SaPKWp<^0K-dV+{}!0M!&;^2eOsd zH+TLgk>6y=#bUrl@1*0KM&;HWMg4mZXIRu5*v8J+u@l$OW6^38Dt<4W~4@-I5(O zi9FBN#5akoEqKVSKWY~Kt~hM}q`nSmj~gDnl`(RRx8skM#I7cxr!w(k{o6kdT{J($ zT*C?h0hNE-M;!eP=K05E*LJ7iswC==yqAHzc`urXL}XctIr7&o+tdAXVU|2Rv=EH4 zjM(CcC*%?wjIvhjy8KK{4wRM)RbCQ3QU+s%-x+s|!Q}9h<zd3v zgMcC;0mVuXPz!hW?0(ZcU?rrSPKn4$=?LT8%?h>5%%U?Djb#@M;=Z)6bbfV<&GWeA z6q-a#%rdskqDfK92)_se6cRw07RUT*Ab7PDK{&A8M|tE*)Z3EplGh)H6)->o*xQn| z*PzvquusDS;@`rvBFNRD=t|l^VN^0nBe-tfHLVaWU@YTt25}k2`+>(KRN{eU+R|)z z)nQcXsb4gf-VIfziRWOT0;*CIN@2XXI@G)hC`f-kEZoBuQie8#PV^HCs)CyB#_?zl zSo9#Zt56ycb+Y#_zh5#SBN6&>*lggv&eqa7#&zn|d?b1={s^?Zmp?Yc{ylJAQRCe` zuOXrtdD_$@1>WGPNANJ)JaFeq zckTo0(#Reucnz`7ZgoU8!gY0{;9dNL7uw*@iI4WZ#Ye~8hkCNePD5JGtLQEuBdnY9 zF5KtsAsRUB%0nUxSXtkSZ8vuM%TvLie;R7puL32ig6eJAY>Y@ zsoKpEU5!2=su}Z#0T7}^SXd|@pEU7CWHe-lExch-YHeoB=8kObjnpB-m}1^Hj$8jG z$DyC7=&#^6tY&b}?q>_y!KRQF%I?&;&}t}x-M ziY5zdfjU1GN`?}l;&n-t+$ zcD38Z|BMpMedOdHCvNe4H?c?@T2CJ<;@$uFiW<1Kb4!#;a+fpSlkFMIWKHfHml;rI zeFxY(uAtEQ+k|cT?F|L4)O7xSTiVmQBm%k49A6jOvI+7kzMHS8)ok0cJDzKY*RE4W zisPkE$=IRT5{`V3FinX#ObVe43mOq6wgY92hZcO2X^e-fkfaiZV8hT8tz>*=W$`$Q zGGd35%jKwW z7s`YxTV6F|BQH}$D**72^Yz2c6!zGQw6sG@7QMd$Trh@c;yF6W6BJC@OIR$f)%PW| z!wl}F^nnQ)m5wigCU@CGUK0s#=K<1g7}}bXF13c^D{ap~L^^aSQJ|%5Bu1wK7XNMG z^$L8%Tf2`XWYI_Wq&XZPa?a6jom;7U_uABv`WAh8^BS@cPQ_+!WwXsLOE9{5uv7R< zYZvid2JmU*3MgKQ#IhHeVuPidbGKJr4?H?QbjGt?Q3tsxg`J~~;aaO$ozj&`YHy6C z@=RoJ3$(h{!zF)+4?P%Hfy*xdMB{=bL}rAALMq$QVF)-dIXaVYS%iF>G~C13B9PND zZ`<_aT5bU9uhMsz#vSPu!mV~{%t8KScmk(A%tK1dI0WLUIvF|_50f%A#A;zbjol-v z8d!TYr$f?j-%c*5y35p6xAj*ok@fLGW2vC8;Aq*GEOMp7;02u9N!+7H-p&(i&hpVR z4YBD~rTZWIWcqusg4LBLkX1C!FA&qaLdJV`>2?L`KD}w2eZ?6rG(FJHXa&@Gov+Z0 z>_G+BD0$J5U7A}Q#a_jg5dcG78yXrOTpa@-2V!PGE}no41f57l+UBHgbO1_oa${$XjT4 z&XbMob0e1in%&We;q%DMmzZ$XnE=?JKv9^@tR;%CR$q5Zx)wqM}H@FE8` zN62i9Dn*cWcWD~Ckrc1(Ph`)t7(*Mv^WVFjZG2C<8|oA875Fkaucg(LIA z%q_puL4#}zvj-$`EiCaS)}B|+dxN|K#T?vX@<3oKuR@R3Ef!kn zRh<4-t>cIFyy7Szx``Sb_lG#=zq|l2(vzvE=uT@rG#t-{ga9$A)3QJf%o2h)9-R+B z?go#FKC4-(?E(!J0fObzP@|bY$3?9a;Q<;E`85TXdI!yN{_b_AN4;(@pF2<|bEX=<7|nEgO&&gFccp)b`m!*7SmB+*w^{ zo`x<3)BSnw!&ifmuG*4r2JVJi6a53L6OR5oWYxq!WKIU|Hf%XfxA0q`?n^xhP2VXN zN+JC1=sn5!bbXIM8ZE7S`?FhZNm&Y>JmZr&dFd=on{-IwOI5$Z(r@Nndmfj#`E%8Y zU}Xe+;3}AC$9TN@H~Ed#BV00*rJ4bUf^vY9l_D?J7S8kh0;G6lEW+!3hPM#94}27_ z9eqc})nW=XWvD7^b}RcKVTEKv0x~}ZxcO--skS#hM=`a_v@;m1) z3#%zUpVZ-N%@k8Ipx63`QSrR}QF~>YjBG<@c=!*{D2uXBnP3P)8SpsK!|XiR@*96xT* zA8xyND5FF=ZqRr4b-~XVrVa!>`iN(1|KGfYop)E_pkLOZ8&5eI$VSR$FW+hCT~c<5 z4;(TT&Fetu#L}W0)}O1iET_pW(ZT&BIcS6LcT!B5^^3_Rfo@%y&KPnLNa^`4sASWh zgnUV#=**4wI95s`uuANczS5d@?sln}GBw3WHT0)M7btX%3m(>UYOBGWHdX&Zhc|M# z@E*Nb&K9a^_L6_OPCQ49yYACojGC-ptm(uJ5S8bKHz@e|EXs4X=WRP^9iPxcwV(jH zF&C;l<@tI5Pr#Jy=ZSp9kT!_gtkL3ea!6hkAjVSvi|1g!Il-y4Yglzp@@xylFs{kC zbop3u#^mr$glp2**|uO(+Q{_s68Y*xBJgx@=^Lp};KavOj!uNQ2!vOd#c_@Q@QWsp zty)@`^Iq4J728n2Ch6KT@$C8T+_Ih|LL9`cte)d#J5@;u#&!6hYSFEW@DP~Yg8s!o zUZ?eds?XY7jSI2y1ypSBo3Us@fNBv-*oyHOg2(oe6pkY{_KlC+=k;tQU&Lk9F)GkD z!kJJWu&%gztkBJZ-0-)su{v7H0?@2;CM+CN0y&WG2pn&>>0MC%Rt^T%oB_=0cxQT;$y_2uTH3t`Zn5;?yMU%7N+j2 zo!WP_01#@f`Am;y6(rSQ;PHBy8XUC9H8SYzYERGZcQmT4Dy zXYC2+G9vyZ44zEO#XHYL2y%LYF5&Vb{%$egj9p>_DXxR+)ob|z@d@a|i8frT+uj)# zDg3gVuOCdw_HLObfFl)QtbNpRCQ@9F(zTGHjpNp}_R9;gur_uqfp=QJ&MLqs1!)q{ zeow=H8MC;zPz3Br4*%Zx{=qDUflmg?6)mXtUI=@W?LtqM-S6uIadmQ3f5u9gd>vRD zLJO}Ezw*h`R-d`h*N$~uVNn^ld*Ep)%J7a`E`l>XNw;MCy*uBky!Fd41mR}jX|$;BwFi5a&}~0DV~WN zXhUqVw}cN>l8?3KRapmWs&m;Th3&J?4xIg{Te#Nr58x^we%Xy2%mjWp<7A6g*9q){ zszT(X=B?bE$-7t~JD6Gn*>H$VV5X2VHK1+9um-ri5#x;2CF;OipZg~IqZTx7q5K1B z!bO=7D)jS?=1`B;7+b+E`MBABfRTr2&k7_@mh+6xQ8^vFnvtY`O!>982d9m88TxOx zpUY|UB~<*v{J714xn~-i+()4uC;!6uhxIHr3bNyvPIK=wXrOJ3)X>Ju>wul;+mFFB zVMy&LZuj%$;n50*$I=E-a+Z}~L7K=p#6+^gS#Ui$^%TCevq!t5#j6%iMCr6s-d|zO zjZKRZ*Y>wC_Vuxz$kh+{5Keci#9h0NX^iOKvh3m)j*CE zR9+@dk6{cm{)$aub+c?!4EuI3s1RX$G_DGUY#uJG`UP+Ni@t9qg01UF;>oP*QiS^8 zVgESZZ5!Clz!57nm`h)4l*X-yd|`>aFMiph!DTS zsXmHpqUJMIy{rAjyuphQ?N)H5i$0+#JIWr{ty&wPNSfkd_VK*M((dR3(w8m@F_*98 zaQR3x^1i+!m)BPH{k^ouv+_D2J00{3BbF=JE{eloeOAN?2(W>y2WF0o=t|41v%3L*hZz ziCTi5YXjD9?T-Xh;}r3hoFs%+_D`9Sf1G*S%#xr-$ea){oKrfV+FEf%Ww5E8us|mz zyEpvGEZgHPytb_&mA>^GD;#*?%zi_W_+Tw1)1)L#tOSZQh${nP71%h^v=N&YRE+KrfwQ`z+P$=E5r zzz&8v5KRnrPX(!X&$M5*=RUsyM@$iZ5FCUs*V=J3GkZu_k$Om#;O=6e!!x?6aVZs8 z{LRzm#u8B$Z_G$&h4P!zo6_-ly=_Ktak@2uknd|Y`78j?XKk(;8wmGPJ+NdtXi#n- zn}>L}ZQS!?2!!bHHDHzozTw`2dPVwlTUai;H=Tx6dz`%6Sw$9*Ym)NaKt zV=-B!)Vt-xvnS?K`ouyg_ntj4<GE0%kuRL7k|$`%2^(n+=%?+%#C zaoTh7&%NOl;n=0n$5xrVeo+-yQbI#ab(!YAoVFN z--O~nYvDgDXyf}un!?mmt>Er{ND>J9i`A*2TkbDwvBe74sl2r0NnzZZdfD$f`jy>p z-Zih(`OJo1O~Xo}Y- zDOvP2fVw!gMnRLz>o3vExA}<4T4-NdvqMTM;AOd79_>&Bf@n zNf_05WeW4n?bDAEZxPnluk0MOb9rt@1!^4k@>wvhc1ak?+>=MuI7-QW2@RCRP17W# z!0|3gFH zE0rpahQZG!t4>RecL%1V3p-dk#ws|%`rFIxO_+Ut>8!{AcX2l4Y1lD`d5-zYgpXU&au+FldmEu%gK>iMbn{bT%ez5=)Qo$wCv_h}nC7e4wg1Mgd^;W0Xp%a|)m{np!f=P^F1l+*l~ zCE}%+n=>>{<1G<)Fp_s?SWF-UFo)(qtsGY#em5w|JA=1GV~h74)r#>XGRTq&)2^t; zyXmCw#dbIs2~rQXm|p*tdfDl=N|o&%Ovv>*UAl^I z(=y;A#mZb$Re0fo5h+s*2F!_FI&1paNzn=omC^G2KLv+_nh$EtkQo}ZVuKW;cZSaF z9|d#?aXRI)l=_{!g7#u^oNlFeiuA6<+rKZpyxYG<4~-KJS%GPSvL7xi=L{ai?ex7VS%Ziq6BRN2WS@0^f+{2&OekIx(Z$Y>yegHVq4B*OuTOsaoKeHS#_W<0){X4NUx++^!8-QHP z9JQKqPHMMiu4e;c@z~q^y7!B;81lW^^n5H95WII-N9DwBBt};7acE^D5zj}AyclhR z8qz-X1B6iN1Fa-?2ju)u_XQRu%>55nA>{(2Y-XVIl8+x1))ORfKm_6-MAaOi0R2=l z8MXo{Rk;cxooDqAETOFVPj7WmDUe+*%ri`k%Br&+Wm6_xDTK7Umudy>VL9Lh&@%RJ z;JR)7>+Ec5HrwnkU|ExXXd{`fLJuqQh+p~$NIVtZqOJTJ^7nSNaGqBz+63XAe+t%_ zl?_2Qed`YAr9#)E^V8NzD%Y%2RA8N%VK)Lk7s{>X-C=yWPpbgVksc8J`p9}1xLTnH z6u+TBjp$ocEQlHr9rc^IYQH8`ho)SzuP^*kJ=+6ueZ+}s>0%M4W_8ZJ=UKY64O7;m zWY_1tX1|t(&3cwTTC7ya+N4{xl`!EOX&D z+1Q)$P{?;u^8+zY7aY2U>%?!(Tk8NDfR%2nAZs{KRcb!-%G(25{ZrIRb1+e?&e?iN zH`glZl%#l>IkTxPm4PFAxr7m$Bc(FM81BB?ue8-U@f!uD8~lf|z83-|kYAb(@$Ync z8_aQh-!BZWeP|Bc2jyoGv(fXVcD&eaT(wIqK7MXQB`vplUoGEYPMK~8SMol@mtbTv z!k5sUMtnWL|FKYzQRc-CNCn3LE5%?$Qca}4mnl<-ECLE)t!0d_;sKp!(<}FKMRQJq zDZ2vjc&h_^?j;{qt(%pcQ3S}T=~F27yUZ^!2py%8uk`Qt3%P?4&!5SkP2u(GVo^F(@?L3{ z_f?>O6Hukp44xihJdic(3MKTZ|Fw6C8|`2E{}J|`K;P=b^wskBHqLk!(8fOLo;EhSxwl!72bgd2wExYyO0}*6+L4eSg-yaL+vV z#m3LTYquXe${XP{>30Bww=0(Dj-oH8A;VB}Eh4{1) zKhjv%UB6Z99k_>^YH2H6#T_E^ya6U&)xbmN-hd&C;YLD;)k*55 zJrQk#5QsaKlo8l|(>$-gMm4C)OEn|FB~U7W)J`P;>!kpmF6dp?Vwx7`*9dc8vPGsB zaLknyHI%8$MKgNRO>nzI(wP#!UhqQM#73j)*tm3(O$*C92nRs1rFXez`^wDwJM>5ijsZFlio^@}eHXS47 z+WYt@Jr=1NK9IEZ_A4c?YrnewG7YdQ=KauzDctvoZ0Ig#o(U)AC2bRszr^Fl8-J2N za`sU`PcEG%{B1<3B!Hks*|HACOl3rI#SkuC+8y;?Vn9xOuyJ=zEOvADRxbLAB_4B} z(;nAg(cN*=;kCI^FZtouNFnUyPv4t+_S{~XbH%A@Q}v97L87uacRUwe$ zxan!UN~5WOs1IV{x?H4wyd)W)YDg;8E+wBP8SQ!DQ&`DSx$BVHT|1>BV;@hGRL1d| zLo_?!sWHmW3wYBq$LT-FJuy`e^eqzDe*Q=yKSO=bS%F2t5&*Zdap(`*?N!_vqt{}@ z(qEYm4C2gx7ro&W4JXBq^Ja>6X?s>MbMns`$m*$%$Ls==;9*nCsx6!MimuKx!`?=V@Vuo)n3w z^jL9&Jgp)>F#P;K)}QV4g=d(rok?G=xi37KQt{i5m6Pm9A02^I{eN#g_}|hDFzWZy z|B>QMGmtB&qpF4hv8aN6SBAp8;WoZFLO%j_VFzakFoh<(Ns`&!GDx*jX|jNdm#tYb zgPX~IqcVosnc~+PpXOI8b+^EE8P3^trX30TkUa_^uIB1#2u||YsW>n#<&9mS z@|O_D>2k!Kdz_~Yc-1t%Gj~h-0BFCz5aZMV7y12=4VegWd6G4U-oL3fRt6LUp3Iv` zyx*ndKVhX@?itFjLQSRgQY~ zED**4D~dFgTlTZ>{j|C4k<(~oJ1}fRn1IrHvS2QUPxoD;2&b9#au=mV-q%&)_a-us z&-T|s3ILTi@0w#DmWkHJM}6>ESE{Hpl;6%5NI;R?_4yP9ESJGP1-@MrX6BkVyp!QO zXI5I*g;yBcAlN?H`0Y?cywEqFr+kSVRjrl3^(jfV`h14oF3WE8UDevo4?1_^gsr?( zd?GZFsq=(}Dx3nz%ODhYmkvA|b6B&9g?;*KB_cQJx?aXVzJdrr`DAK482rK#kfKZcCnc?-Y$7wbcxcnh|PWv%Zv$;uC zzw$Hjm%{VHRFs`I2V21%9W&4;8dmlPHC)}=#`EYqz0#kg-(`qB66>cAhlCs2$jbti zWW2)AoTyvkp%&fUHzL*j!nN(i^`@McTtoa@@}eShx63tn+Tl1g=Uhea0cG`-+|=nG zJxPrARo?d^HAPaBQvW^IzBB1e`TX%O;tpaw8LHgVa|BV4UHFAE#hBXuo0MP4mv32; zgEOx1RSN!l4(k(QBfNUj@vXH+5mDc2yaEAoty!L(I8U|tdNBgZjyE) z!tEq>^uBr_PIyBxi6f+GysX-m*mg`LjEQ#*@r_%(B4)%s*%s6IzQb}E)qj*LMLv4~ zpg+xz+P_a5CejuGCQ!Od;OpAV)taq<|}LeaMVSf)kfWpseEwp zybqVJyL_2Gu<`w=oL80~f1E|cH|2SiPXlF))B%?&@3sj<|0+p`C+w_I&Y!uE(NL9E zTxrBcysf)4EAU`U>Zzxl)?;*ZRfrAV2Lqt_c`s4cpYI`#F+bc7&Mz!{ll5@+l5&@irHP%|pVZi0sv%4X8qr+pHND^w z3{Ri1l`kpfliw`Kb&doBpHsnBHcn#4wr-iU1tEsBZ~JOj66zJ1w6kS54!=Y5_$g&i z+J!GZz7l@CTs^U-ATri0Z%1={2$2j>f7Y3&1;*VN)r`FPpx+eXm6nuMYk3|oe-QFK zIAtk$?&2lycbSwgKC{FC-bVy_ve(Z}CdA98i{j|)&h%oK-m#wGk5R$3?bKJ_0i^$-;4fgO_DLEv?e@lhbo1|Ha(b#Zs_MJj@reBN>sVizu`pWiBid;usk zQ?8nG!-#Z!6#hG!7M^g<$5^(Vlt2WPL8(||>YI@~e2fwCkGo=-V50|r*Cvj`vjOKG zPlCoC%zOdseI8=LZwEec>x%ZSZS(XGH`}i3?iiTn9vl(*@I#@xXAz?8+ox*E+E82O z^VcK_-lq!h1HgT_+Qech>&@`3hh6T+Z=bv_MVacOxXaqjXx-mZAt3RPcf)7muh&-Xe{r9`T=@s zMj~&`cf7jSpXIkWe_nDe`&yoPS&Gb>fOx?s<1JKnj=@mmkg}s@M{uBeb||p;CD8B37AiDR^xX+jw)D>sqKx)>%SXV zxV{pLd5%Nf?OAE0RzbhmY306dwAK65oPo0;aLTPDRf6SB`bw-OJ*a7FY03bT8B*x) zC(=aHWK@M%_i_;faD)Tq9h{)c-qc>qa+TK8w`eWrPxt=&M-zz%g@scoGR$Ij4w z79t%XACLk{6m|OoznU;-t`DW`1;U1H@eB&uSDhN~Kd9_4yH(_&SrfO&eWmj=+gp|R z`fBTEJki&rPVI8|;{+^D(QEd+K0!_`v@2ZHp+je)|94wC@6$V+PG6*Udj2v6&;HNJ zzje%izqKGORgwiVh)MHCgrM0Vz9)C96NO6&Y z<1`nIA9bl|fYR}*7058!+0jv71_n{n6Y_^7PV5-y%x9IXpw*@xjL(PR;xY+1c}OKB z+K^cST+X)TWi%Z2a+woBFGQHLWDG<*>)xZXt3-j&uixiV^U1{ZaOKvoMadn0gwi+F zGa#1e%5!tPSxz;6cZ*Ckkr~Xqp)tXqrk}?p*SpK^*P1N%eBRD;@p&(=nptbysgt83 zG9hmK!qkaMm$!^P>LF@a)#R{6bDIn4N+oxmxt|I?fDtssyPH*z)5rE{KN}Ks8W)6J z1b#fH=Bizu@Q^it>O!cMl;D{GvQUcNXT;pmZzs>#PjdH<<;+82zPQJeR-L_}FMNhf! zGQv>wF096}Ls2s#{o??rWOFx0`rAc#%1nRI*b_+xSj8) z@{jjMtW}Tc4d@jt-%m~;ODZev2TbwA3V}oTuOP939+FOgx1LQlcW0ze3RdqeJi-57 zAz`dF*A8Z}$7k24&9>uxI(Zr^;uBJ6zvZV=rFfhH)`OU8L#n?_6DN(UB0_mdZ!Ic8 zLO>yl=V}4IZw0U9boRux8K9lRJ);GX(jy)pZ(wt2K(K|%yq$;-ofiHl>GD&|0hR8$q4asHuhj`?}tT>KVc zN%c3XA*uK73uRvO!fXAWc0raOU;Qm6Qznh8|4uj2HhAgtUDP{ZRQ}n-?&#y4q=Mo2 z9_ue~Ua!@Klj(1@P5Z3v*)#@2Z$k6)B=u5TVhTu`0%AVnqG~r9eCJ&{WI{UtCM?~T z!wzLX&3WqAm|4DP_;Kyzx&1Zk4FW*H6&g==#aMsFZ9Rcx$%uaryhqo?LqbC;yv>9T z!3ub@VI3#&CO^lOqT)`;pAe?D5v|dkf{IcuFX3DiU=R3jm9B=B*c&dOciohT+UTn9Gs-)g-DK^ZhSptmOt*9e?}*#Gbc3ru5(eEDoX$|UwdUAPF~Kfm+!iWuuD0O z_*r`-7E2M6XP{g~>#N`A1`47j}G<<2`Z!(2CQ%pEPZ`qBsDMqR(Vv$d& zsMlxYCp(tmgdluU9)%wC4l$ShRHI*pmntE(R#NDuS4MSh`m_?SOq^{^%+f=a{b_Vy z2#Z2)E&0W~D^tFfZELNeHFpnQjl)t-NzUl|&*Eih{F66}G+9#V%q3FkVD*sNR1wD_ zv#825uqtnWfOblc0KuqW5#8}GA)s@=RH?xy1{MRXP%T897so&`p-h@d6SRfilb^Lc zt!1lY_ak92ib5H|QFz~m00Dl0oeKbjUS>)_GSlv^I}B?wpow3%D=ac8p5E2drfM>E zlks{9P^vffhTTmhM zc5^XC$Nv#@=aZjHTsbP;{%ys@9IZbKC1&F!9-`R(rx5)6Va3yMH<3dWZ~kweU&2R8 z_WS6C5OgDJ*RQes80pVz>X%_Zo(Z;&!aE*%Q%uPJ?jEiE_OKCIgVHXFPX^BbN-e`5 zF3BJ2nXx=@+@5TA6RxHr>?u>@PC^+p-jz2kF7%=AAAp0v>3Z<@#s)#Gg^oHP zpS2g<>sF;U6D@stNn}4n$fMChkyIA_zf%M!RH0!fZU8}GIny_CB2sz^gg`AF5wa(H z$>&2)kex2XZ9{j|41kXPLS=4)r=bQ)Ls9$nEHbQ-eM{{zp?6>)pfYeT_v;$(b1b4l zl9D5vkTO#*a~_Lb^CM-UYx-13o~Zscd*+MM7#+2N$U!4|x%fqZ`V2*Y0M;ZZ@2cc4 z1gjbj;EeU_Cr&GkdL(fADWtTrIi$5&rd@Cd`ApR5?*ox^vpy{R^|BRLBcy%5tTJF@bb8@FNl|75~KA}LPl_Q zIwC;Ox1RJMA(vbhLBuEE_U(+vAo)Q*`BghKK^u-?n}i8V(K+2kzDGhyZnT`r+e^A@ zW9@K9dBn`t4H$_ew%m-aO0UYUHo;;8{&ac-&=%G3@$)L}?$6wV_hGBEzwaLB^TTFa z9_$x1^4BQNhk8 zNg}X`@FFeyOpO&tS*dW>w#`*b8zuF?;xWFsS zCq^^7bB5qlRdNELy!CV&#JFgxN?ywvfZ$F6n#2K6i$aVQ=(I>dhar{n)%$dBz$RLQ z=8hbO6{dr_5QozJy->ghx+XH|>Or+FRg;**_a?QQ$aCBE1>q|BqF;m#5@ewu>U_Elc1_h}!LKBWT`=dJ}K?t#jX=DvUfIl-t8(@z z;2t3BG6aZ_Kc0DGiQMI?{Xu-{myg zBiaTbSX~*!YUfq)ocas>DW7(z0M@lkT=h)|Y+4Yd{XSk?Vd!m{547j$RO0><^Y;!x zPo)dsZ`256cGH)O`W_2m`GAimz_BJO5Y7j~q1nA20v%<_X3SZ$qSHDy{luFOx#NcO+t23Y5iqS@#=5W> zfvd>4V1;uE>!Jb*kYw1miuT^0v&~!85&wXW0-6fRuPOgVfNNvOx>Y^T*Y3q@{dk*7 z6p5+k1!~Wc%7VAOW8fL{p0wtMh+v*SOS40h)BNo3DtZ%IFkFLdoTU$F#)K5G6(Tgi zPa}DptI9+oOyC6_gU;UB2m|#+^#TLDtK|`p3HT}jT$&K0Z3FlL_?~w9)Q!Mt+rx>$ zM>kon5Mbi}EYKhm$<_x)JqMz4RGRv(Gm$SJ4rXsNWTPfxM#5n@p)l5y)J@~tm%lMtoQj{DSyCKn8tacII@&m>(~H7Lb+9uGNI_fB1+cp9cfIA4 z9+3u5!)8QzqG8?Z!ei(5z%Rv)$mzmz^>R5?um3I`C!Jn7+?W(FkWS+tbxO?wXV6*Hh ze~Lar37mC^-DTk0)yH!)4vE*&uG%8 zcyp@bffZvBT$6OCpIMTYRSgFwHUK?emh`ViA_UmqO>nh@!D+(>yP&%dP_UWsuMw=3 zm3y_QSWD;|yFm6YaVFbLs)Wd00SPXJPq_sfKX)}}_x)}R%QwP_@)TXuy&xSzKjUu+ z@yxxg>0*2zGd+BvuIBiN>R~=_3clIQy<%@Dr2}~y{dooW8%Z23 zHd7M}NN>W^<>i9T6P;*a)7V)68k7eX)IP?^SQ2y>SWN#xp(6~$I~j4vxT}trYWd#y z^V(&3K`Z}`{67(o_~(A7A#AB+vA<~Te@VN}|42I(kQLxxr$1-}f)9|)4gRiKsfVXn zmrYOF=C7f)N}ZRGZUY?2vP?l|eP+pzIbu?U)ZK=OMK5SMycXU~F-Z`_rT3zpxZiCS zz*>;Br{jnAJQJ(&`FH1qlFK@ek=f&wQslP3cH|6cD&k-DNQ&uxdHQ#r-&eE;(w9R) z;rnoYgA%WLYB$+QFB$J@%CxUGWzM$RX3X&`#gvE_>43ZYFDx!;<9Mq<{A(;g!ptp( zwKjx=zwBqENNUQBW@?K7MgRbxA+phEd76+DxAZ2@6Xf!BU-MF8MV0wif5P-~Vha7B z{1U1N^_a6GtKC2}Jhf5ctM#WBxE{p6l+G>lmFo>vZ`NZ6uHhc>r!Je96s=u0b4?|S zLz?l8MM`Gt9EV+kybu6`f1m@F#ZOP+@g~>741s%P6|ketFqYMCLNksg71+ix%zHupvtUdy(QRj#Ih8!MK-nIbP@5;UkPzy~T z1o!vG(cz8L`2)<0A88o8q{9l@)kL@8n)fO(+Ycv>NvYcKp1XQ|A8XCE4rtc2N3~64 zF8332qP13fm>D;agAmF;7C95s<;GEHxC}C!*Hn>o8}OKBP3+z6#N%ek^99L$H0PsB zR-K2(R5ouCF6@WdX8+Y~7v4VJJ6O$13>r0cd=RR&X0`7Sns441Mt53qCNaZ~UkRcr zhB=`5kMS0Bwl%lkvsX3(~jLBI_n&lS{&qC!5LpI~sI#Xv;}jKm@SqyH+kFfKsE z^hv)FZw20@;@M#>?RtYth~5Uw2JjM^(&wr<@mjF9*FISEpag zm?lh|%8#r$98);S>%?xg=T8=W-`GE|zFgD;wC%ecn@?}RfQTc4N9C@^t)%4OcZ4qq zR{uu+y|~$#r0e=vZ$^4ro+|rd@3X#47mtFh8;m;90$44ez`fl0Rr%h0qCvzJ%jc-7 zv(0CR1~IlDZ4Iwy5>u!hLuuH$Ia4_{3mnkw+7qjMqqWy?j?<3d6Tbf7y|yBEiAnLKPfp6PZA}9R6xQ zMPcw@fqia`}5O@A%k%=wV#i%>~6nHm@Ppx>2wz0uL8&Igw8t=k)2sZB>W zF1_w-^;8Vx*%8B{S5Bk)e$Iy85XOxc4+fOdJn+3eB95c#s2Jb5U2lxMu{-C@$WbvG zh?u4=d{Hyte(hDI7=<+9WPbLLqrjzr#KZI9Z&O@M4;c*Q#Eq{gcepFYey8oxa75o1lu%A(WuYRZT9QfUlg>bFaUr590Xkc=bFpcEmZQ_w7U7aU(CD}J>R^Kbrv zFGi2;D2(J(H`wX8-r*SnTo7uQ#NIwqfIfAJ5{|ZuBxuNQGg<}=(&E5NKiqL#bZjSm z61_HS8;5idB`iOUnTK0{BW&`}>;`{jbeh2p(vjYdnMg{$1=&gg7gj*;A1Z46R@nxV z9$7@=Xn9ConnIQUaNBZ7h48HkzIrMXDg=o_rN#>dh`oMY-s+GNmusC->d)xYBHN5E zdR=WYyX;DDzdGtGnBE{HU5Kid3<_cC$R{HhMJ>f}RVoLQe~xeXvGHt4)z4jrpN8r| z(2nhK`%8;?u4n3GMFOmZOqNHwAs~kWsrT$rvX9Hvx~E2|5Q0FqxCd7)2*InSoT|Ho zDU=|~R)6865~h>M)%VNHLy2GSC~a-(?f!()vnk0{Ip10a6r!+mQZXEc8wM)HLzodE*7eIz zlO|jcO(*KN&p-c+`PIL?&!LpF?DhX5kTCkhyZsAR?+?gm@AaD&<^D6~!V9u$WX5S) z7!M&nf7WY1ZM}a+6{4zhf}woqcxOWx%K2?^4KpYs__&~}n!aS%lI#If;HJGkqGlg2 zmhf(sjc$L9%Q{f-SMP|xQpAkT+7R!W`d7Brp=a`l^u&Cg|>Y`%o@xexiRR9aAe zC&YgRVw99*nQl{RN&Y-gjke z(TROkFI$np>ks&gfLSO(hc0kHWqSxuR@})9dTl4W;NC#f6#q0Z_<$?iG(7WhJ}A^X zG9KFP6WsoE|8PGnjI#Wl&cuS;Pyhtq@@8TudI!QuxY3NO)J&D8@L#|tS6T-7gl|Q_ zoRN!26-rMLr*$lBH^9( zI6}G>lrIWV6T;-RS+gR7P4jsT%lG!VuqqB8UMbhgR(&8;7y~=z;DbtHQGSOUom_6%0|fKEfCoHHkB`KtQY7?Jc$DINa`6=RL{Yy-L-LnVZ0VWl2aJaT?fR_3I(LwT`-a+$dcyy0W4;-~dHSLv2F!-vVE<tmKjc}p2$2P+crUj&X$euY*=`eR`Ul$?|lzcNi*AQQnNK#wY_kN3vO1J%;Oey)az5 zJ@4Z}{`Vsx57`qBj-BDYlxvRzvj|c?JWdYGM0*h1B1d=e6Qk30pwrI6E6E=f(=BSC z{SHF*U(7QhUm@&)PW_VozdG`Y>x1c58_CE!t^k%tw}k42~aT-Wm8>oXITErY|^;ptwdm}{h$|XM6f>ZWT2?P zuc3SsQFP5aLH0U7Kzq--8$b;zRr6OKIoNXf=Ic|ixp(r;?cNXAUpNWH+s~Zj9O>z| z)9F9et}YyDl%K4(BP+jmQ{f%8yUcqd1%sY=d;=>=V!e2=5IRC{EL+TnN{%Gb23W0~9^2KC zY%J26pAQh4?OiYEk(*lp(dn$eJ+j4{2AH88bm=0A4e|8S-1YQ&v z!VI@F<{qrx;`FH{;WeN6acGBvD5LIJpT?JRBT$0X$)(em8I4xM%`>Ah%!EtI`GIksO(A^f7MBPf_Ghg$mTA~_Bx*tOil zS)L4j&P^w?5E90#$3iO127tm)_ZeXjpA-JB99=Qyg#mPfOOVc~LYtW2*cw(KRVcxLmTyT8N zww_%D8Uw4!R{P$3r2BUB=2G`H-aZ-S?i(h)Jw4997T-09-g?L3Ra!Ml&b8F!HXh0! z+lS1_jZ>bsvjyVwL@#_7ToaJrz#5A|5A61&bbA}XEs4(d#8@G$i4SW62O$-N9}kfU zClVK@Njxf|ZJKUJqM$6^EImF{G#ApKmRfph3WH)CBl0_f6ftw33GD$I!xHcG0^Xih z^+I{Qsw`Knicq6HGSMeKkUL8UnervhzU@ObPhaAN{Hpd*K|6UabZXDMjyewq)@ToU z_|J8!!hYqqT`EoVul2Ly&ujmQMqJwoe@-EbZD7HE`+2Vz?!jx*uAUHGMU4~;vrSw% zv%xSI3`j_+WXJn)+O;IUJwg9mN?ImyvU*~v6mVeU=!kWPx3H4XpYl`{EtjsD-+wUFJsY{| zpm=4CJij;uflL3W0kBDJ2jsVfg>wJq^Z$x0AO$6fQttKTjB3VIB{wG4So&puwC7#R z+~o2n2eU9=4`l}v|MR0yUPk?|3zdbPiy%pw`7t|ayQPjlanT3iZ_$3; z*UQ7%soNK417A8B6C_N3WmofhW4SA#|M(f_Wo`Z`51Za%4|UjlW=9u3}ZTeA(^p3}@{<0*yCbbXYf&s@1o`E8!cH~cX0r}62n zOWtEi^*^dHr9ApsVPL4Cg%}U6JGPHA9>=jON!g@Dh-!fz2unFoV<9NTQ^YU47-?R znFoPyj)k|)R}V?~=LbDzVwbhw6QnMa_zUA1G?scr_k=O|M>Y9UfGi)%*bc`PiOYc5 zJS+ziq?61^XQRnbkfb&=dR)1Rwa1a zfUA&{cud)Y7L*6Ee&C<<2{YcN&s(LxXMCN$9hBS^w$$AMHMMWfKWbbi43T}1teZ78 z9HcMeCj~5Tpa_8$5qI)EyYI-tlC8#nE}|~T^Mz68p@Mc%Mvi7CV?VxmgBq|b_J)<9 zQXBCBBu~jrZ$JsC&!^F4Z4pDV+ybrq0WR;?tHNH$W9L!IzzS`yU{xmmU~5L~g)mi{ z$n|PL5vnIGvIGFh7F-J^KCb(ltk+50T_J-$9iKd4X$j!3=sGM`Y_b*XT+v_;{Rd!p3I)&5`1DPEXocn1aj953b^LYpVD$83$&Nu{ zdyNXG6w0S5?7S@_s?g;755LB7TQ$yu2Nc4ud)LwgrMP~01?Pf?bt2Sv29q$Oz2i#u-VA0$WH!W+{P}`hC^zz+O@;{QG7J|Hwy)V{vmH~ znzNK42@LX%vi{Om@ddr!^9~OxRtyUG3=++V!GAV$KhMjugK*g(84?S2qc(ePfcZlJ zh2(q6t>pP`lQ1p}T6wjG%=Cvic*rGCx;UvTJ7wLS3}q1b6klD_ynq%vM15$qLTz^1 z2JHHZQP{NkBvX99y!nNet#PwhRo>osHiz&fQWIBB0a@(jkgJijc}4%A*28?hALd4? z${Sy2?oO@eWy9Z&TZ6Y)C#7U`2_&7&T(q9#eBCQO-iOfOjQ@-jixOk8-*q?ze8786 z2cL>UZz68#ZHgxHTpUQq?`6Gycp=be>OXmbR}CuUdjxu1!Wq1cB5qx#4PpVoIxos% zLiA>dzsJ{VJ9V&Pv>ga-s%Q*)!z=K{iJu_N!ZMg2la=__(z!xlUa}!33h|*;18|Y zRmf&Vs{+g!|Nf{ZJ<_d!*D{--cKt8(tUn)_M3h!IoZ_DzbxmQzBz#y_VKMdQAM|Cw zKGTq2byou4`6yj|Uj~>|)ma*Lf&I1sFxS~VjhVDjYeoOlk))Rkt?&wm8Q;S~v%-dq z<2bDxz%vjWe=>39qk^YdST*+===;)j3Hhd7b!6{@%eD#bY~UB@GtZaab%gB2%7LLcIH@a+Kw2h3$t2 z$)Ec}+4Fl)CJ0`hlGbWO941QS(OQx4#6;iiPn!Aie@}!mzW+<~4E-Iino$%O(Ksjm z?}7y^(`@8aNX1yEugmbC*z@XN3BCIEGwy#zM@d0or_j>qso?GMIJ)p}_r6{2Ydl;9 zErg`*Q;^?t`RT=Q2vIY!(X5MO4mb<9>Ir*&{6Hiw7T*MZzYEj2DuX*97jCV= zECYPldVKn{K11-IUeiOOFyZl>%u$I%}0xfODKCEHR^HY5>#(T@UFxYCxt?d`|Hqdf?*B$&-VI!NAj z0Kaz~t1tZ;YCd0Nl zWafo863y{v$L#nz@OJ+TOZd7@I$o2jbnk5n2S|xJQI5>gri7UfS!G5(uV#AU`UPy> zNJN%{70txDl|C7drH)-Eaih!6T!D%q9@>_!9gr6n!r!&jlFujpeI^8BVf(V@vNEsP zDtHz0>O7B8l7LntP{xbCG63W#AwzYN9y1Rw4pjPv?^t!GD|F@kvPih~;mOCbVPE)W z=|L%=>dOz$j;BP=xB&Z0pcU9OHhkrvw&MH#T)8G_gK(wr&zMvB_WsL*7hQqw*N2eT zTmL17;qWH2e`#T!4wh>I?*iNoLkvi|ex04X5$^I~2{e1_dF%Mq{pz9v&+#d4?vs8N z_{I9=|8$tjyj)4oSIo}u{9%-?hwnX$OYyv+>{&dGZ2DAxI>QG6g~+3D&)jh7?w^-R z^S=c-4B`kr3hHC;F-Kkg92@ZyI>0ao*&kh#$>P4S50RYsSaFE^qjsY}z#+1>E-lI=y zA-2=C_0G|(jMS}T< zj>GnivA$KKO>W)%UUiMcq?rXEuf`Wv&jm4YJj%uTfRE!)BZFF_Qo!M&9s`+ zx|qDYxmLiSOP}^GkdwtL+G9XmDN~DrIEX6TUDhiz&2T`Z56cfp3Nib9R{YCxdf4P{ zcyckbwtz6?)vXXvG2_Pc5Ssk0KQ0dxQWZAbz<&3n=qHB7AM{Z-=er&$1y!=J0hDGpa}|zq64g zgAmvWxNgcHJcu|fRX6kUopYeQyEVK7FXW`=D=mBVfBnMUivG^;=&39eRQ@I{qfz)gQJ=OCTjqH!I1> z_t@7{M>xsrm?p=P7{~WC{%=~BMn#F*|2M5`p+z*|tYYzx=>OgW`ooxA6Ri7Rp7Ytj zpV5DL&c`&Wpb*cd?A;PX?Q0|9?Gl{L;V*i9KMOn1{P$0ZVyOK0b7HHWL|e|GVGQ^S zNsL}u%{FeLV*BN5^Kmddk(R0S^vXF9*(ng)x$hU(sm;jE7Yo=}qMw~>uF<~f#d$B` z+K6lG^*t;$rkFtDSxV6z9M5B%@zcL)(YhXN_6x| zI3wXmBVGX1T3#}MH71jk^xn1rVKq%C1k{4!We~*HVN6QLTl$VEFxrK#;Ie&Cb;!#f zt@=WP3li&nAI;7QDsd_t$t-HZ5%_WPGI7r_tycR|W>yg~&Vj!YhNgHptoSDPeA9pnltH)RLk+W z+391A4k(?4=D4Wstsh1JYpIY^{V57fQWVNuW?IOxpjdB(9R20k^EED;iYfZGNyyZB zavW*m9uGEQQk7G)A)~Jl7=OEqUy~{CgTNxZ1{uKsyTF=JrFImw+xDG9?(nGkcZi+uQZywF0 zBJ>E6UJLZfG0fW*BLlaM;5tI)1Xymwt%$-ZU!7%Xx5{4x^|rCD-xRDSt$X5A`9dZ( zu)x9;-PM$QpdFro_15xfTa>^3;(UQ>t9Ha0O4WAHAGZ95@8F{j{s-Ubz|+RL5Sfv5 z|1*ho;HAd)YuhrMzOq6>MMGFtUBV9W($II@zouiHpX( z9)HnKgru*gFD!$C9@DcN7g4mwkbIH3h_8p2l8p#qEFwGg@lu_w98Fp)n7!?@5Vg4yGG? z;#NayG}cMN-rgntQp!qhRYC|O*Z%0qnfcwHN{jO@ThSstMq8&m(v(9X3kmVCMBWe8 znPinXPifRvC0cp&xE4x%G76yihOyZ^Bqf~ZW-D5$qNfonEoE30qUOOMD(`S==DB%@ z82Vw|L{j@@NMfF5_yeFXyiJM$U^U={I26Yw5@1L$ak#*&%5E+My$wJSx8R+N4E?-u z2ntr|F3lt{&s2Wy4ygkXH>^pLIvOODT!a_sw`$Q60l=zn* z@Fg8lpYiKIGr2H;#e>@#G4m*u5Q@WZob7cA*@Yp_-{bW7ao#9@A_8FPx>bx@KTi>K zo7&oYZ(jdX)BbeD94 zG{Z172oe&KN=PFm&CnnnGjvLaf`l|99U_PfrKFUc(dU1j_q^vi*Tn}u^5fq3-fOS5 z_PtpOMAUivO_G-2K*kZBhpd2^sfLohL}D{4twyZzMxYy&M&*W6!;(u)SoEZP*~}*~ zLxgQB!H_j_=PtREKl`9F??DZDNXr@hS#Q8u4}lli*xis(EKWWHsq+pKNyhI9m&^yJ zU(6YIk6#n`Y5H}$|M~Ca8pPnglWX4HgQeo6HIPf!7BWXC8n;imVN-!&NHb0peIEF) zidVA^4Ym4DD|VCeK(HROZc(q{vxY^7x17{zN^xt%rIO?V&;@sM!Zt-k;$X?W75BI>Xp>} zeNbKj5l?;&d6SW+2-C556ZHfiex{jt&+iXloBKIbK)X3)$@w%`*z}QVY~0V*UpnOc zzmNm+nr{__&HNHO;Kje>_I*8_qEbgGzp< zk5=!e*1P1iymu~N2Ov;KdwU(?fzKWYW|D?8#${vU1+XkqQz=`#eG$d%5Cg#wDa#^K zI4a#~D5u&xba|m=#lU+{^p?nnU*%1vdC!uG7{`^Pd$tlLmUnUSTgI7444i}aLsTc?iV zWrRZUT|v7W=^IfBOmg0r4#cwQ18yRN6cHP&s2CF5Sy zwUs3aZ2s?IcQ%D5ld~Yy92^~%yfXlk+*fyMES*jILQ)d zz^H*J2X53xu0pP4_#T>?PCwZ)vBrUoYutK6W0)?QBy9XapeRwjpQw-u|K&E$8pSf_ zM^H7S`bj@s3*$>s^S~>((KqUvKlUB2U!}-H({(Vn#c^W z%$0x(c3!7nQOAFN-qQ1-=FG?qovvy_SIz!p{_-W0;`WBkBPjX4C~P~ITXF7r@l+I9 z=+VR$)sL8@M)ZDeH!hJ`d`K#;{iDSx*s{&asbx2r(9Glml(OJhgNo0yxn8nDk3HABLksSUm;@OF$@m9R(Rqj10^ zN<>xOFt}VMVfeSCu%KjIYoB&06n1@j`tOS*6YnY@1oH<@l&=YcW$#mD?II)!@i5yz zsPHARH7ujfEErUvGGLo3)KF9GKxIsda4}lX9V&%+rZ`7mg>>|rC_js7+YUEt7-Kl&V9s~b;=~b zt25d@5l3){%|J0f_Ex~JCBlrV*o@P&(H85bz9W&)uwW3s!KemKhUiRj?i0~0aqLfD z-e&uxQgc$~V2u1G)-|S4mSE>6?OuQPe(^Z$K_drXwaXw#vSpG#DNqY%pXwVBQu~|) z0%m3vS8m&SeqS$(;m*uaR^rS`U>^06l3E2%ERhuZjT>?WV+mg8wxiN^!p?_d*k->< zz+3@9sq+dJvHriDO_E{yYx87nIFloeMt#fT_fzhr|L>S+#u98#WY zUxoQnX$St$~|M@Osd!P^QWp+^Doy~JMW4U11&uuH$IkR z*&&R-P8KF;zg6`SU3%);^k`oO?+m|6=&u+7KdkE!AwNO?LMUTNA7CWWeR@4;uR%U* ztasYHHdUEl?&t)}*95$^^+6z6FP)qV%^2hPC=SIi=~(EV6b$EESjfpQxW+ z^eYk8Ov=3|vLC5_O4;Y%8)4UMW0S>!vxig?fOFfV^-CD}{Go8c2gT*dfPB*5e-IBd z$7zmC1T7e2&~d5-v<&sS2ui&ZG1{TQhSELY2$d`JKuYJr>Ev$e<1GXP`!xfbpix0sX#xbF{dL zTIRe5ytGyr9MdJUg4q8NS`USkFUy%C+yY~P3{m1ml=6vDA+aP^u?6N17m+Nj$7gM7 zAsg;@HE=~~%b)i#C1Lg#VJXA9rp@3IT7$}q`y*VHXj;CwHCs!j;4i5^tybMU4E=IfP`L-bi-ps$7zLiGKTq2RQ{ zAeC%l6Qx@K3B-?SeFe!J?E6(#EN0#IQW9sf2cecUwHpD9pfVKjoVzDy_D`)<_8dqy z@A$1N%VohSY%^nu(pdt9SyHL;+gyY_UMTI#2j8RkKlp=$;MY3GME<_{FG8$F0Kcv`3JPId+a7K%x zwq~T0U6FjJAH8R*$`$oDGZ4UQ2d7AupkVpW@|h^Da%(jqjn|R*TOI^ONEuqczv@_y9_A?ywI`wG^u(d`+V6LNyhOwy&0jgs2UaN9D7Df$1A z<%Ot&R6-nk*p0#PNTUX$tHdAS`-_p(mWoCq*OR#pzf} zBvl8#1+7LM9EvGJj`PLt!@N8E!D~N3$0GUqOm|mcz6hX945H7D2U$U>20xvzt~xeW z^ETpI88PLnKb5O7D)%Vg8TT!evk#1%iXp9UsnZs0KJ0M~cj;@k+GjDXPP&kIhw4Ur z_Uw|#JPaZNYH%~=i*HhNjk5dR`18b|_qtmc)yd$y2>OS0dMuOIc^Dx(R%J$vVXm-D zD=M8_k!f1)-}!;gS}!W0Pr7mfX&i9RoLCGyFtm*Fje4GJgK*dztwc$iR0Exum&E6+ zpIPO}K;vCuH4j?#{KH}sO9EhyqO6fHGlj_S;9P%E5+f=Qgf`u}G9fGHu!5bsrX%tpM$r0=f&r7vEAvAk z`c;qdH~j*nFOCfk3+SMNMvjq}jrWs@Y$Y1&AKHCGddAr-SifHsSXt_aLNpVO;fECG z_*uQu8VzH&8Y#7df~@8ROSDTM)P!;!J`2Qgz;fKj*jX<)(q}QvEtXq^VF|y4YL3H! zk8m5br_ZXf(|(ev@peLqa^}0v?V`#t=-5txZs1SJ$6~%>x1zVP^A%6!?Lq_&Duf-e zBb}P0f!QE-@-ZNcs3|JbwYq(&Z>@yzRg<&NBbx9EoJYB8n|L8E!>h}%g<%9FQQFrg zZ#Wfk0sEC5NycDF-`vrtTUAEa0T=e=|Cas+4X^Z;DOzuY0fZPTww^%l7n^i5h|dgB z@h|~n4&KyP)_;(mt~0Y_%8Ew4P2P`VIiB~wyLUZP8H5lvIc8fgc?l}Rl;!a89~PV% z)v%S(fB*O*B8tpcHD38_^r_6Auu^|pSghf)b?oe97JXESMGj^U{>er6N(W{ajIv|h z2lJW0udPcgryWop4dzp_qGXj)lgh#;Id-TNCg(Tw$$77M3s)oDoCE~lh--gHcc05P zcCzUytw(K_wof<;4#GaOQ<_&g8QgU0$-&lY9`Wf{GHK(TeOGGoewyD*{Y9+Fmr5%$ z>Af#k#6x=JBwc*vngbX7=egVk<+n+ybtvV!T%u;3MNF=YjkxCY_lN3_RerqKd4K)m zZJ2OkrAGkzo}i+5;R@^^MYrOldBN%Hf_NBA2hZG*vr13xIFsGv{1DLUnmFan&rHoY z{(5a9BlL%xM&p=Y3z+d}c?;#jtG}Ysk-+r1x$xZtGbW~rG?{S^F;B4ez*rh93lbjm zPEzOD2G3YfBbLcJ+|$Lp-Uq39SngW&$_Q@PfuT9y?9BcU4#sYb56sp)GU>@Tr=q2X zgT->mu=s(bY&~5vG?qZrQ{Q=3ZuX%Z)ttG53IS>Z$w&@XSw(0_HF4Hlo!)Z66Toi* z!)$*E!=hqqF2`Ka6t0jrA`?q25!umlEP!~gkD%nhJl<0cGUu3#(?Rvou^UX>n-y-% z^T1R+v9uTBniou_lGk|)%yr7@p@`GU-HlkI`^2@BTI>5EpJVj?@Bo3WIbzGuIWP2ke1B(T=7aJo;qirj1GOKrJ!93nnZ!MnZl#+G%JF8Q< z2)y8g(W!#5QKI;K8ggTWl-&MP*A82Z{3p12edXyFs0E~xhY)AG4Fn?KE$K>nX6?& zE@B}5l>oGP~Mz@H%k#~0aJHx{41azrKlx!{t$y;zQGhs;}Q*wAkIGlrwK zO8YbT6WuJ!i;BGSZVSX&hAEmnCTO-++i^0eiuSwHl`#iiRuhep3;BlsPEnfeLBlMg zx`-Dr&heu@hK$-nPg%FY}^p0=iHdX!O62bZ8)tq_}E5{>sf2mkoPX9IoKVk5mH0vE##=%R=?fH0hq)fRwY z9B`1O{nsXgNDU$v#J3@)^Z6m~uOpcYv$V(QbvJr4(!`qV{lD<81J@1(pU&wx_b>Ez^$=6WXrk*e|x1{}8C~BoHbomvF76r*TJ( z&y9aus4@VL6yF?wn_=c z`>N5~xB>+*HGEDV1$NxEDYh=Z)5ga3I~wQ&9XNAWi3^HjuX5U&kGiTmN^OJuR(}~j zT4!hz^=N5c@SDqK>82zL{H0E99iQ(Fe#^fpv;()>hRbl(VTx|cqizl?MWMq&ii(R{_CDIMVwWeGX!IBx8=aCx2N$B8VIKc*v{Z z8cfsSimo~1gpPRJ&-*{<-LQ5e>cI8hD@Yx&nUW<3BOX>ZqVwUzPrxU3U>=WFcPDTdd&-gqu^%6PX#sEt$mM(!}k`b1Vh*65lPq2 z*&uJZW?}T-_%ZX?AXe`$#zt0L;w{x)ySr~|E&ZI_CrBumd@py=<{r{^bG4q-Qc4&; z7xNdZY>2kYO8mnr9GI^2onaWF0BqvVAcKcxsYK5=u2$5W$Z29Sh$Ah}Pi^8txOsWl zM(UngXNG-@)v-|IF)dcqvt87uG26w~A|0hm$rdVu_v@rtOFo%u=^{l8p0Q#Sya-{J zy-~|3=vp4@l!IbTZXbtN%6#vMxAbei&t)c`^*2}9bOB5jHa2X{RWZC1*F``W0p#FH z8|2qdEe7TWyYIg9o`Y6JS%%@CD@Fuj!7t1U>@j64q=)HSy2HYK$Sv&6eva=Y$G!HbwEzeht_Je{NUF$Rtz;ZhJ zEv5_JJ9+F7X=N&U9dD{&yiur}Bi;|-p;&e~6P*uBFt3R!K@(W5&U@Qm!&h+RC=w2a zr^h+(C}qY5Ym(2OsD|lFutJnda-ViYGHyaOP3KW3)>Bi0v%Q&gi9m`1?B%7)LQ;gs zv8K7^3eyu}sO`Gk#eX~n+*wK`2gslRTb!o8iiA4{!en9-&+ap2t! z^Kput?4%#J{vNEgu%z;*zP0s`UBV{=Rl14WptJOo zr8I8B#L%s+*Bz7BG?9L_7qaWY_JY#%Rdd;RUx{xh=okG2nr=3>Ltc=f&+w+Hv`e4Wy3a(N=H8L#nR+O|Zmf|iRV8G)I+|Hiv*lp9CjVCW4meuw% zp)#l+o3XT&R%XNy2WT{{rQd1#{Wc#h8Ez$E$Rz8e={Gq7Kq9d)8A^P%i zehDYTVpOX`%~Z&)s9n`PyZqzHBPc+T-eqW9PiG>G>@kWTJ$Ma?)oLY(XR`b7e1oKU^gpR)OarW3{9uJ z>0c)`Xh`ty(-Gjluz75LGjMG6Dq&>(_1vbx+=!5uqM2&DhGPx;+q8U#ozoMTN|gcY(6Nwkc&+ICE`c z=#d@0Yu}0%9Kuw?jwtcX97Hqpc->v2Zn?r5EdNua>Ur`8=^_=TE_UpbVmnCk*O9~3 zvOuI8@9(NTdedE-f+uRp1sPQF#C?Mm>TB+kW8lcveH0jAO9M*uMGt^us6xk1#2it( zF3T0)1a{MHiKJAV3Nvt?;(Jm88*5lelMg?0nNs+Cw&`9RW749+Ac$LETA5^1*j6mlsVQ% zdqg4enY66s9Xq7@8k|xy9Zr8Cjk3dCRBJ_bqnz1HM^OpelrP_!adPv~V#*@u4M(BXtycM+NOZ%c z^6St-%*!Cchoj8U0fD;+^vINmU>`9xOlB1-lUV{48N@vwGyN%j_aH$jlp2OedqFi> zWV>_TU594seD2cr+$g1T5FKdJ?_K^^F_7dlXZkVf27@}#2-vUd@||&4(zk`*Qshfv z@W%**M8Pxu0wF-GD4(R4js!~=o=Iiit`qZ<4Yn6)f#?k^cnSSl07PP!LV>`m?aZB} zu5^*a2sbjNsOkz1g15HfU{(I+^rXLq{hFffM)Y4{zXUpSWTtodm)3d&hMdiQXip0b z`a@KL<9Q(8)x+qu|L!(Pc3RUk=!54YjZq3KDM`*XSri@h_s9*3j{&>Al(@2v0=5aF-ABwSpFnU4l1QlIRw=1Lr;@G{>@svOv!rG0i-E zSn&E+A_+!8zWEIfgPY||!T&OaZwxdDOUcfP2@hXW;Z^&@JGujdK43)&G77e=cP zy&g~*Ll%lfLIg?^XQCE=%Qt47+ZSQ92 z57~ypZ>@KXP>xc&rrrg#Lp67QfxL=x|7()*<1&F_g|2i#=TN5`dFI(~D|apvbT2Nm zm&m$Q6vv8m(-vNOiyPINcTQi=So2}P6x?wj0JuE3{BRpyv#0?8U4X50&U|lss)V&h z<1ZS&yfPEYf`fBslhtps!qv6x@VI%OA^RLIHL#@B47s#EL|q7v*CaGJ>t$yxKk1*Z zHg)%F&YXYe)OZ=8SuT@9siqd_DX19nsR{b=a*TL7b)zTemNOQ7?A4fLHe|x_3Zfq= zR2*cvQ`)&4PmN;){nB-ZBe7J%`huM7#c|-(de>+}R^NKto&{d!#-s42=S@f*Zj0JS ze;(z<-9P>mx;%PIJ!Z;gA@}3^A5j9*Ixz_xdZpCb-*9%0egf%y%8 z_|%Vf(o^HX-CqNoSWFGyY#TKFm~)cSZ7LL97S5t;bi*jO8-*HiA<+d>S%rH^}o9Z|^&8NUI88Gksvp+%w69etY z)eM)_l5Y7*ch$|YK$=5A2Wxyx{DN)RFYvh!j>(kzP3V$FA{Re6in)-y7h?Y7K$6vh zP=OOGGMXI!E;SQ4aOD=niugg9dm9s^TRlom&zpfPTF;Brgs3LO?>@ICo z6tpGCQ^Y(L_1#qS{SN*T_HH)ya*!|lRSiaZfC(9ca9e|4?aO6DY+-L0Qwpcb<|~%z z`lnPxao@aV{9#>6w%noilC*Ev7HTj)4fEv*c_AnRfEHaXS%Yt30hFuL#!!f(PP@;*neWPuAc@wGqBxGfWYdYX0ZX=qA{V z%H^2V5Fb-(Odg$9CAhpGH>s(iqH?(GbcLZD2*i=_F=B%YYrXI{;W|uXCHt2CV2zb$ z8&52P95st=>QMIrKic?dF)Il%F8^e0U`x_oL}%kKp_s-@*Uy7pO7~w{pLEMCx$e7u zf7xEhWg!>gMir8Clkkv5-=3Cx5UlF?by)qVeH^#E+5{2!#5h#WCM=aR0ycj%tE03Y zSq+B9c|3v;nPKGyB6)(&{krDrMXmo_3l=CdBrotk7kWvtHiQUhSSo4-psE&HYEHE2!K-3v~{FDiNs2-)#4nX)07#1Ga zF1J0^WGG>Gv=MfDykVOcDeE)rQG7&%SiV-eDP2X65nj0`(fe?57Ljd9-~%0VtMCo* zq!o7Xc>kBe#%3&iSS(j2+XUm=wac5|vmyX)3q3jZKEFz%-DC$Hxi1?a~V9jknl z`jsCm;p1nN8<`dDPK}pYN82QJ-iVMOK?i^WdsvJ(%e5(#636$%ud&ol$3=-92JqXy zqx~Y9usEs7_|*|Qnc?`dJa%*JI8V(_bpew3^}{u1VP~*~-Cr~k#wuh5AXe|uMn6-h z$Br4JTen>A3M*XF8v(Ay;+@N-rrJQDmmTVpT zD4lV$*|Fz|e^^#^-sn17YMW{6t?IWMdI+eEECA~Hj z<(R33L20KK%4T4rM!@TPAy83=rDB;BVUC+mFmp({O&g)N7@@!*REbPy$p{-2FYzr_Em zaT#>yn*VmOe%0oA`)Z93^CfZCp$qT~DNls|FAo1I0Hhe=zE+W)gt=Gi3CZi{o;I%% zSYaEqs~vle&4jB8gE5YHW>#0{S{50*N>eZ|N1SKbYJ&vHWufUM?bi5C%EUQkMoO7Z z;DG@hzN0|~f~D-+%uZRYR0I$GQbIg+nu#0w(G**3ZrT1vnPEa3w80tLYoKygetJc^ zY2o_QD*p)!nH=LL)I^FsCXS}sGTBj=S~-K z>~YP=5TsqFy@O#n>nt7k_Rsuk>sG5o9LzHxwu(yxrTs_6M?hZYUOG~T jN-uByR z{e=SSu8m50gN=8~>e|1UtbaQx?;9Qge|ym6{-&Q4@MORh$lW%Ej!ASt6)A?{)XSJ8 z_zN~2H{l1b1pPJ*zKQ6xxQ~e(*5crOU3b4M*^?D|o`?JT`swT559kk9?CPYv`w(xr z!$%5aV0Yz&bN}zR0K#S;K^(1f)tj(54Bp0qm2u_jus!1WZBsthvg{3X%1~UUF=1>G zjerkN^IUG#wj%};P7mEYeB|der4t)o2UXL|NKQ-J04m@KP1URo1b|#4cd4LZ<%F(R zgc`03T!$Do|J*zqZb&k-g31FxS!BbH2PWLi6V;ZP;wIQ8Ar}fGoQkCpV&(3>m~F~3 zm@|fo@hPn#5|M#a*7Z5boVxT#fXwB|68GoZjzqVlNja!ik@^o4^7&Ky|*Hm5BVmT#f2mUb?j?Nrp0N(1DVNncP z1LmiSkjna71IBv}~kyyLH%kSpzGkCfB(%#GVl>KYXTbHZbpi4EIO%DKy` zXcS=SVv|~zair2c>+2%myW4Yv_Ys6M;?|_+`jV;%MuEMr%{P{9S@Evu28XNvT5MOD3Q zE~b44imLNk4z13&WA2v?1cY-E7+b3wPnbX1QRkC1>#DC7F2$bkt1pAHjs2$Uy?J?e z4u8nCIRF{d&`crgnZqb)6)Ou>^s;?vvtKLdYUYe$fNlw^Y0MAU$x{%?CuL7{^SxLT zoTLZSjt)-@p>8~|qsSK4WrY;jE%PD~-Flmp6b@?atx?)NW9&4ckx7q_F}BSF6bmrV zPb?NxX|4NYYMUJXQ$$}V0>#us>TUjj-wN;*$OF*n>>nL?)9OHN*8|tz!_&TM=BVvD z7vBmrtc)$D|UI|Wtu_#)HEJ>_m%TAYuJND`jI!A;~r)ph1-1%%!o zC1m@la8_#x#o*&=h|+@I`nol*W}usK8{Gvqf<$#Mz1CGC!mkOrtf4R9{~Wjk@0mHz z>gGP~j#}Fc-0JH;Vzc6jdsV*IA>^Ia`2I%#RW)eoXSYnF_6_)#)ItSsy%X&Gd>nCs7sc5+b&ek#cdRworvktTnOU@ zMZ+n&;YXVb&&UkEttS>4m6|19Qqhftjt8N&la<|jP*1HV!!Un#F7nkp$qI2pXD6Dm zWVY^TDX`?+0bwUmaZGA^Q=v>r;S6ZYg(4q@v1Juo2BMoIXNXP+_^qsq%2p}52V)yV zX)sHw6Y0aW4gbe~9Nyw=gAy0#fGey@30wadJ^1Chr zW=9;EqXLmt1obk&n~SjJR2b@U8L1!hJG}u?STqN$fG7%7gP~SuulrpzL$8eYwF7&0 z#TuU=646-;YReI z?JJ&-OFZwFp{sQIBQ%O$wRkx*8~r7jIL%TGy`6dOM%hR$6iuG}E@H}cbGZD* zA}eduifTh71hkYd1l)<4xo}HweZP2_ew~wsoWW-}KtGs7iHUZi$c%1g)#+K&skfy? z+rm5)3oW~Be(UoKDm0RH4-hl2pWcqIpWk|Q_^5TymW-I(XYNfEoA~9pl$(~jYqOcU z$TZzf4~sxo>Xf(2^eb&%cHjCL*UO+h6R+8FH6b3iu*M|`wYtt1hp^OL_jt?bozvx! z^ro)`3lFJfPrM7fbKf0?$nWs+%c^XCWJgPE6!)=*I7EL@dp16%D_<8)?(=SyCZp*? z&)%~ikdVuZt57ZD>w3K%zMzqhv8F3ZvY?M3E7%A)^hm+^P82)Uk#&ebRZ{NtPr#di zg0^%-6QPcxf}UZ`i3*@5Skn|rj%%0Qz>4Q3F7-}bi{u==p(+ZH&`#rbL(6!Ubxg&0 zmfut#VniYWec*H>Yp{mL zo$-7${-QOYy%U(NSTguqJc16Pr}Ss&HqRv*`UI-mQTT`(icMc7uK20s?J)u}8n5n( zhnl94Ha*8L4?pQLa|!+T&FCwG9bSP%e5U(pz_efd?yO-HykL`mw;rOjn22V+z$sIV z9oqa)HLA!PwLzvW3Ds^L65cEt3bDp@wS00;BVgZ_U!ODvv`=dYu$+jp&kEZB!Gw*j zA%z%PEimaMQR*+BU1dp(YXi#yX+~4os}j~0Fe72tt|gKa#0Dv7c0LJ+d31xqY^Egd zWl?|3^b`yeMqGK{y`9nYDr~iZPO&?|7@#z{f~xt#^$3 zm}>W(>acbG-uEq1XYC%>G_}nS$?KkgaWCrdymzZE+xhju`zz1CiL4)dF1pbHdyVbU zD_Bb=^z%G*T`CCv>##g|*SJ>v@6H%2DH44R`NkOVThfIe;q_05DX zjc;USfwNymEI5#fAO&H4tG&TaRU>}e=b_!|4qJBT3Pf_DF!lh9UdvlHJ(DNH(xH6hG%y@6bquGg1Yrj&K* z_jS>(RCgg~7|Jar2t9uZT4q1ez$&w7ZZl4!`oYK3JdoB~H{OdA(L=p2DlRFr@(Q94 zpOHnsu7^}XAPX5X*l)Cbl_`umgW1jCX9aIF&*L@jfcaGUsXgO zQ~(X7*}f)T6lxvn+x;rh@u67u-u=+beNPT5{H4t54MiIgP`j~04@LY`5FKf39m^?M1i zrmwdaN4>wRV@6Zhp6jJamjvMhhI#ZB`WtUk5p*gGBJ%&2nWQ2itgJfWjjX8Hv>X&$ zx#=WAW8LEH(;wzJxwUm~S$9=Lsd~+rF0>TGGs#=UaQTK48?#D^?eKLlC(CEn4Q z>@I?PZrz$yc8v5DK8B29<~wugXsQZZacoi&o`l3tw9t{(=Ar zI((#~n2e$26p`qb-f5Pnq_qYeF)a`Z1$4OXq1?WxTtUwTmqZanwKCb?jI&*lqxUCE zjD-#WN<{5!A-bfzH)@BAq@}IdlBdG8sjA`2o~^3}kNm+m-mjBLP5D>O9N4E#Zr%RE zP^NRq-DZ*sPckX39_?(g|AJ8CP=^0{h)#K8*wguhB9I)L?p>@x0CYko!Wc6BXTKY&W*QDC-1=~K zXn(;S$>IKUkbz*sj>BUrwNGdW*yJF_u)rA?t+$0x#yKc=qD%ZAnHn z3W8yQFqz9LS@ifu(uT~+%)gV4R4HLF} zkZ=K2xHKQ8xCoDyY?7H@T{Neyks0O%P0IXmNkCl-pnKlM5f5Ih*NX2AL)VR@>jk<} z4QCYy0B>z7rOnrI*U=q?3SXHgmAVEV#JQEPfb6@zc{Z>agM+G94a}9)LZS!Vg-|t#8w)R1r2;YwWfU%|eyg%z0b_?-qIkYT`^aU?$+N z{NBg}g@x-4%Lh6AjR}kt3|v1yb6a8#=R7Vm8>UY22;3S>sqyaos-nInW6`hN0m=@y zMDCMGsdJUHL9nqqT+>;8%?`U=kZ;3Vg||UHl}Nv(dD2Be%op10*%t(k0K}at#%t(`5E1K4Q&NZ$GaU9#joVDAe$4&= zW2FiA;LR1^7@k!q;~y$Ih}=|{2>o^e_B=u&;nbuc$lS=b_t~-ef$e`G0bclbB!YNv z_@E^wtlusbVa|VCT24^Q%t+Uv{qZc#GCIBKGmO=2hKYJk8_b8pK9ELU9WUWyhgCJ8 zVT7<3BX@UldteAh8-;_cmWMxAdc{V7v!048$I-1QNAYjYWropZoMC5%*?47!wX;HC zaI`#o-CT*G&8AaxVrc2)xY^TiarS_v%^Ap`0Pt69gL~9BQPgB==(fY?CEu+lh^SU? z^zu<39`7k`R=>-5y>YQqpBISiRdSQUI){p-)~(Wbk}aZpz`<6C`nsO+HV75`oLztH z%}Z+=vjgAZ!^U>^ze5(6{$-4gzI4X%1BD|TI+QGUFdb48xnsitc8iAl3aL7Px)q3Qr0g^0g9jJHU z8gB=QB829EtD-50Kqt>#CU(y(JmRj)q=Xm8b^*eH3W$9)&mlwvcvYNPnVP)gB|_?~ z+@n3hkRewz{}BGj5K`m4ui6%&s76w_qE!Yud^gTYb8YgYQo20oCZx~a=?ym+Fd^Rc(#+%c3Iw@X9R^-10IxY!!h%kuOw@<0+ zX0&5IUL_~H1ImDR7n(t|mNRp~W@)wEOo@BwC!8DbsT&?Vk!WH%+%e1y6ExfnOnzRM z@lLnZmCiG2#}3FK3*btth0DS}eS!OR<#Z}~)%JAcJ$3&JOr&~&V&&H2qijFFY8*>q zE=wi!Pv{?sc~3%m&ktL@Xg&R(P2bl(uTEMC+Fj}W_06vtxGHVgR>?Z8fp&?$=5`zY zoA1yxsU)y!X7%{4d-TVryu!;1{;AMYYP6^0VmAbuVk~J7${8 zggppWtC~F*LrNL0±t)^$>@mtZknrN8~-E0~DK@L?cQV@LEPGfX(#zD2hI;2<87hFz$jfD?#*5)w*jFYFs5=v}?!*ZL93DN53ygbd<$- zT3B89tc zkZA?orJbg#6*c2V@eSeEV9dbDN8`_hGs9==PD+?!q5{I?Sf12G*tYKM?y@+&o;~kg zx{Z^w^^2p@YBT>b-%IN!=JubWAE{Qv0jU)WFhyXA#~sa#-877Vo2&|{y{xw3fsik~ zBjA=~adNamL?oYvUIE&H&lMP z!CZ@BQ%uin{3E;ldTCO`obfb?L~V9kt4vgGK3?RhfDvbyx_u=D}}{^Z5K%Te2G3>Cld#rlWfS}UKtd@(KMjU*=Cm=oa6uRGl9 zf6b)&dVU0a^KqJ3q35q3_)I>i%jX}Yy+{b7+c~Ss#7i5`L+ERohW-(h3*LWpzc9lI z1!vW|II{l;==)|muHB%mn_36jvrMg$zbQ(YkrRl53G91hmGp!^y+f=Ea(;M z#;6$lx8WkW#(LNF4t@m^;(&|D2L+(<8|jew#%E+w_&IYOA#GF)+yn4+!}mm9)%^41 z*j?<8e?`9QB32tQDdfZCO``&V$+pdDptSI4C?A@5x=zh{xA!}si)veLV@lA-N@_Q; z5w|S8-hcTz*u8b)A1`kU6ieeCxQ-Bc)_D5=Vc0bKCb$#!e2_v~tvy z+D58|{}0!}T-0B-aBi+G2(7!LZ}c_rF)MUt%(^EIOdl@G1LYSsI+04PK}|nqwRAZ= zK9h`d6Cb;Lz37~)G-!fcq_Nyk7!b}yn{_^+Ti`4Hz}0e#tqjCk*G(idADv3duQbvH{aXh7I^98td%GjSmBTn)MGbY=+_LuJ|hl81!AT z8b>^nA-B?k&tV+P!{kBFOwx>NP!(umOQJM`DUh}H2TR_$#N1D#jPozAmq%)A zE?GvVq*`}#EYEAuIh!uur1c#l=IP-Ne7wK1&jX-aLN#nB65}b50fhNs>o6uz$^WDn z$Y5%tfgTJlmNc{I2)NX9-`jkK#$b=%H!tPgbw0h5t zvqG;pN5|5?rlRlYHEzN&m)A+3{@9NyPsmd{-^i~oldgC453eUDW9|>icP;K)jB1Uh zX?PV6h~vYAy*BNcV4%)JDlt9Gh#@FfP=AWuXLPy6gM*`D;+Q@yGDdNBz_B_|GsbOU z>?7cA&(=Kn)Pm_s4Qqn}V6(7W_%Lf6XnGUG74zB`{kKP7{d^P?WJ6wD@&r8~_vm!! z=X(PVK6(7651roB;babok2)--b~;#&5*~K!u@T6e!grSjZG(c;pXe3Pk6Z_2$$xPw zhc|o;sGCa;&Wu@Y!2T5zW{>#y!BRqiTu8l4gGD0+BM9?RxFr~Oy84w7CtP#S?6!Im z;HeGpfvab8J~z+ha$3B#!vc1U5vc|$*BJ$MfZ5Sj=52VGzbIDLENOsKYPD(&e)-P> z^h^NShZM6usaV$5htUzg6D`z58ng7yZ_siZUY=%Ij-j>gV}@Br)DAfFkncb=O^P2r zn@p!z%>SQ?&!cL4?Kf(ORiMxMUgIK@H!I1-X7m4N^tC`6eWPXe{y(Cvz$@s$edX5e z^0bH1MdNSdgu`R;Uvc~*K~X_ySJQNx>qdfyNXzTTP~Y$7mo|6HkY@&>*GD1x4xg9> zFsI}wUSAH?x2<3Fyq$SZ`_-DqBP-l>)}vj*N- zwdxr7SB}JRn4utEm)-38-}$7xJxkJGF6B*9Kjtxq^m!o25ilU;=J9VHjrNJkD=?>X`ePWo9WIhPMmn&G>pz?k%p1r*B9^I>J&~e^C@qldmz(uUMhk zuaXwJ&g0SpwxTk{NcmC@5~r}B=n{`b&8vADl;lg#61@YteP3bH_S{H!xBtNPGdZ{J z$I_gH#IW?h)RIC`k))Wrs{v5b|2|x8Y=R@iazEE=8LbinT z>_56H6nouW25hpn)@C9bGu*e?W|)r4tCu~)zlc7UF@1z=qzfM$nImU~p$AJn|8%`S zA7-2sGaEO1hXXc{VorX#n|*TP9!Rbj4Gc%W7yCoAk895Vs$)rR)Un^}<~IHj%qIlt z^zQ5Az;ayZheLGh&o9h-E0ebPDAGK4FuvvVfYgtomzh=P1@1mppF*O)OXQoA9>GD} zCv1ptFuQ329)zrz333*LI5pRy^OPm!=kl7?s#KM)C0G!3PV_IE3;)HC=-*`c(WPGF zjshx&UP^7boD)*&G28{gJ-6L5$Bk~C5VWZI)|N5ZUkh-{>Lrsa>@Rk9%MVI>O4qgD zl&5-Vmfsmv*^qAzFU9+=QF3PxL{Ueei>kKU?`G1GGEVM4J)3zcxns5*lDB0PVn^>U zcT$t^hdI8Bsilxvd0YRJO&-=Kof*qpF%g?Iq# zh}$xnq5i=oC6^P&%e0=Ze7g>DKf-8<2jgBrJR9>qgawwXjr9#EE`4vNxVs?Ek%sG` z`mRsD3*A*-L@W+-?xZzEy_n~HTuOXl@wn^7+Zh)|7re5Kx$^b(x#jWzbe}9gI zqA^|f`Zp|o@3Z=(%U{n){VJ$}t|$7PtE5QpWAI?id7gDl)D-jK>^!eh!T6=x19|gt z4GGO<(e%6*2lm}uRwxK>|JdrtpJ)T&Hvi7Mh^S5C5uhrwDMPDOH}-*9odyi-@>yV1 z*^Uk?{6;MapWtW{>C0iCt2|8kSL7b`bI7&^hbr>|Z-K&MaIV@H6HC{P*Hi7PIiM$G z1-l+c0wsGRWd&xr_EfUPHrcwuUlcdWyLty)!aaGp4L%X?m+;8jwk*xG*1yu}AxQjl zbOB)mHYVkogODe)W=VPd%MW;rQH=oPe^Lj+$4w#uxK#;wwLju|*CC#lY3kVrWFut& zNZP3fcpkFl`?;n|9oC#@2>Tf5B7&MafCYBM1nWyzT)GFRdi$X~C{Ew?{?wB6_6H;Z z+7XS)ApyYwCS)9A*%h8}x@pn)ZQZ!NGwxq$1d}t3mP(?DdcP+YUu#lK=%#DQ&sUIM zIfmC!>AV1FPQw@4*lH}=cGH(@BRj&q7494sv^W#La(vtx=)bi9fCFLjmIEGuKLkRE ziBkR*ig9U7iy!?KU|2aEDpMn=BxrcN?04ERxiP@kcOGz}cOjj<^n(p^fUnB;Ah#s= zT0YS~fu0Fu&Z(Un#+U_W?*REYa>oFf8AZCjqXz(sWcU?P3n;UIc}>0!6xq)9KzsV= zacs?`(#l7gI>Zz=FUd6LG{^C6NQ?)QjBUWr2aJtNb^r{Le?LH_9s{0GuY&9#dj#1F zkiq$>!qirDR6ma8_VJa8Y-&6GkLP+DXJ^$J{7;Kb2M_EUTGDHvs>t#t{nr$Kt4)BM zjLbTOa;W!3`eYGCiu{SyqR(_U1b{$&EMwmBbxJGp@pnUh6wq}OY%k5j;gp654P_=%abNAx&q*y2G*$;OLGB-uy!zIC`n?Y*X6r%B|YfNwf|zYd_jV zo~==S>>Tj+Mt6BLKZriK$t2nHP=BF=tyX{W01x`19~e9Oa|~!F(k;(M@S#!q1r~sD zn&k!n2xq9CLHv2JxIwLKaa6kWXRMfkerr8L4z%6tndv>-{lhkAjLqkSTSg7=ZpEse zF{>$UH-9b!exS>@i0y@ih@h0>PtGmw~P`*9T6wEB}$TQIAj;3 zJOW^zx1YDd*|q`fl?B%QYo2|UFm^Y`!!1Lj7J~amN=e_Z#mya7UBg@|fnC7Kzq1|M z?VS?%#XQnB`W1PBIB7^)ZZoes%!F5L36!W%8_2H5h6f{;M^=N2GRMPTCAb4BR-c~q z>*nc83vtY`p0hUPiL=MGlZpQ*A6f+_BkM=rg)klQ3A|KnIF;*Fqap4P#oI?XNpw{R z0HH9wI}+#ive+~-POCKTW}b}{RZ%D=sGL8#`xy)X)_46!A|97l2*c5xq=9b$jP?OM zfd@m1Qa1N^6D+HNq8+s#^|@Q8!U+&Of1ZRD17H>6`ePMJ_Ori`_;RT?3@aI;GN{{w zeKXY7T0{_z8Q$}p&Fg{$&=f^Ua^0Wx^!ZEX#r~e^=g~Lywu5tw`3Z< z2s^fXk#6JNMxP_iQ;cv($`}#;u2w?SYvq}NqYw{MX z>w-MC5*)Ta7GmmttDn1~TF`+cRT&b0EkXE)Px}Lf`ri<@|A&EQdg%WFVoArpsNY{W zo}T$nhlcxU%MQ<04(LWx^QAcdDuFeN`t3skE<$Rhvy{g(BYMq2&tq71X7>4{Ytwy} zlCtL-{Ynn>fc9}T$E13865h1ABg-P-_w730=)z_rqt^dny8ow;H6th)@=eqa_ET;s zWd0#}1Bi6!p0tu?^!1Z$d%^0Tjz4EVReM75%$=Ge3WJ6ooQPx!o3*|#fN%sy6`>z@0=`NKkfGj6x#j3@DG`=2!v zeco9ivzMB{{-)YnK0)ybkz8|oxpJ`P_us=!5U|EgJW>Ln46Z_Iv%kMBd~Z0ZQyRlf zO4qaT+1WQ06*fK3grf!ERL?s58OYdIrnz9Ug;}zx7s7Ce%+$wf=j01cX+gWeXZ==! zD*;BFPyj+xZbGOPKb=FoEY{`hhD1aJ4LLXUZORtalWuCF;H%P7oXW@uJA2#*ps`fmtQnsr`K+$s6!XoggysRtzOwI* zBS{vz4!Py>A-2|Q)@$4A3TUzoqOr&xaI^H~l17=# z7KPzdIfYw25F+_NmvrWJKyO-FHpFAn%my3u3Vw4`2}g6n(R3;UFM%K@02PI#=o}S@ z*C7Vr?nxWlHddFtc4gB4-p1h8c=F5-o}N8A^_%}6OzL^|;HT%XeLUoB_&`TEX=Lbcj_%^#q; zT8=hNPt!{LP?z^%d{tL5OpcMvffu5t5QeWBl$2~B{hw^z(#|f+f(86%M_Y9i_ zX2IM`zF@mhd;d7jkn(}Xh>8~(Vp1>qgFRVp^jjz>;;ntjR(eHxO?pGRpM}HxEFmi_ z>dmX_pT^{wmZqMQmXhAzGTagqGyV=iH|Z-tFp+%g2+|Gz18RiV23p2_sj1X#xg#&b?2b)0h{Fpg~}g?ul`ut^*m1TM>!^Cv(Ft z4;Y;{e2Yb#jp}T$>kxDH#lpNjI=~Jy3;AzI5>5Zx+wK3~di$fB-rmDdJ5&7UgNB5* zs>dZfGZcsS@$z>*0!YTS<-O%MZdXBqg(F2*!(+_4XR~wCUpJmOu6kEsy$>jv2!ysRyy$y-F2}~hG-8Rr(U+ivBxc9@a zATW}8i1WQTDo5%-)VhX|gI)=mz@C|oOfR|l_jWpu9gKNrGL?0?Q_WK|6~~mWDL+p$n|uz4+sN!n1`OQP&RqGS0NBLE~Dw=n?8%iYwM>pI&$>0afyk_992 za7c!duuO3v54%l51I2!PST}}$k4sX=ByEBGYP)oz>ki&hS6)%m|qy8~0ryzN?ikA)dxU zJZ9^+~4y~fwyrwuZRbg5&rKR(B6hm{f-66_+v5V0f)SA})%JxG8l>;K<8&@`3jEr{W7i z*8GISY{1n9g5wOQ8ubb9CUo)wYSKUJcHk~UIlj}ZOig@>>e7+5H{b#Hnv-)Om0T-H zZ-;rxp35T*<_Z8-P!%I!r^U-;!te{%Eo+@rJnU5)mzF6mP2zQUPQ#!>*#oW}``|)1 zytCHUs{dqrIpkJqO)8`Qu#CZyfZPwE{#}rhCndg@DJ@@u1lw9_8}EXi3ZQ}AA40OA zV)u$1dFZrdKG{w-*c}{PkrJZ6WEr;X!;5PjmmdU&talWS5f3pDebBgBWvpd}GNf{5 z8pr$NURpoSpsHI}Zt%pc>zVA=@{!{a{PzfFm|CNM?AssL_`JORe>U-3{|_$Md%93a zpX+pb!_;vUV@Zm_{5DfX6LCO3+4x}VV*dMbs{eU{PsqlfrE5g7Cb9G-nsU2=Cj74pQFXY^COsH>RAJf|H5zH_SP3k zl3Z{usfa(h4MexZ8jMO3r^-mIz#;N(9@tl%+^A|FcTjK|=i9vX_7mr}OSU`$!i<1O z`LUwFNWQ)4!`~n@LiV}LiGaryRu6=l>4~eu+FGp%@M>HQ)Zs^Ts-+r8D4DxWZKWM| zLlQ)vK77W3k6hw_%?0=%bNh1CixI(-QbybGh3;Yz%fCI;5f4wO#IS10=#$*7Hnlnq z>>JnlFW8jyMQse5a5%(-I8-5X6C+JH^BE_VJa-O?08kA= z3X^*73M2>vsZBo$Q+a_eCU-LKsw2+UY+u{_O`l^lXdqT zwI@!58?(ki<{pu`vXY~!`DT0g9eh)H?Y_U7F`QCfzHaZWm}}ZYkBX6_3Mj8q)(j`Q zyHRY=%O+TH)m4AR0JdLDVh(8Hmg#Gz=Q4QUH+Nnz>*AO^HW73C3PN!g-MUP-y~PZ* zAVZd7$`6A`PTzf(kZ6T+hw18dC)RxG_aXn&I9|VX{;5Ctv(6xxEq|q8Ma&AH6K)Vx zJB!;ygD(HCm6K2h*v|{uZ&0-wyFl#`oEH_N}1rW%jVIfLy19Ae{JnW|h zlD_h_AulGB8a_P`0zrXZ)PW$v;f_b#4WN`Mwvl)WkuB~qoxoZLwoD$NQWe0e8VzZ) z$!3nTgp>nq6RVvq2(5Y`iod98BK|Ogtz_d^m@WeRJ3mXqLU;@S-B=hC{XR*CkAM%L zC|HNoFa0k&v2JDQdAyyYn$3j##;Z2z+Bx+{Nb-KLoSryf5bw}zA=|x-_rDiO#tg53 zX0${cg3F7wvp~7(v*^ibxVU4`ur(l6yPe_3+>?Fu^;&1PH_3>k<)Sp<_Wcd(-TZ65 z3lTZRYc(^Nm+B+x!u3ytSJ#a$T<5HKRyr|1Aun6>yztf*hsx%G$VdGRO|DmHdLRHh>8m@A`6#0>5xcI3G*G?~ zG&!jpC~Pd>Aa;)lcJwQXr;*Y^v-lAla@qYSZ{&|_Y2Gu>&{c9aFz+VH)CWNQ|F0<1 zb!_A_z7Zg%MeXK&6b~YuvHvr@1+WIo49e+|qx9W*ZJ>`M-owO=4W3?h9Td;(8*Y1@ zg`|vKx+0RU?FILYJ`$SXbe!>DYF_D~jt+OO=g3g<94h6TEz+hZ6XQ#vU{`!iv~$ED z<89`5h>_m4iw9(?GI#gE+p=YMG!*q5C$#6lhfw0Vrd*P77 ze7T$+@d+X@is36lq*v@K0+quAm9hyDGe>Tkj`V2qcS-9-?w7owU25{K(O~2TcQxV~ zlGODSen15VhPrSI3wbQ|9UYGJn^gq2ny?yE2ZUT=Z~g}Dt}(e-6nhcME*9>?He%ts z5gBG)6cL^o+X~XZ$gVv0zmg|!a-Q=&4M|?^nIgP1;3wJJOh3@j^@t+LaDK<^$}N+Irq{bi508r`(RKFp4J*ErVPpuC12m6h@dNr;Y6Ka)9_AY~F5u^>Z#;YUP zdI>J_QchXwEP&#eGr`f*xtufHb%>%29r|@VAH_}0zZg*;9@FXRw@SZwzP~{x60X;y zhiIhB0L(tJ2G14<3Hdq6&C2WmWSKNM{%@8EC{x6~)4)CjkP|oe%GtUD4fk%Kzv=w{ zE>qe5G76ROG0hsHL-pF2^T8-w!|Kr*;?$j~+vx|dhWN6jCAzk*r;YIZE~;pn=6shd zb~Gt*t~nzEulG2orz=Zja}dzQ7QZ9SUH6u&s!>08*RrE)vLCLael8hiJFv;?y28Y> zQU(%;-~9)EMTFjBQ2*u%oOmkZk+)t!IL7a^jOi`}n>+%}1fqT!7t`11ZRH%WL;Za} z6719s0p81~1p4{x6$eZHxsyjan8y&LM=;WV9^9iMKuNbOxaD?Q`MLVrJ?ewci5)&^N%!d%ZKi8+$#r(n{B z(YHEAH|Og*OV_WvY)@V7p8GUcPrmxM)*Zi<0rD*}=)N(&UWY8~i52USChNQ2q(r=+ z;9eIjMu|o0Lr`XEj(IZ8_A2aI$ptC*CTu*Kw^Hl5cvoYz%(XLDk z9?E1Z^e&f-!#KYuGkX2JD-v)}sEX^|=pFm31kt^Fw_`7qldo$+eDfO`{h_OV1+6>3 z>UMEq#qxgddNV3GWl~fE*{s>N$i7X-p%TLmvcMhU(k~xvrj=sJ=Mq#Q4C$5)b9~M` zG?5_6q8upNrdz!<#T6g3#u5);w?@~>iz_izTMiFG{T&<7Jq)lP~zy%0~2oEx?@0 zjQ|db@Z^&527t?_q{6Fefc!d#o_A@bLeOVv#bWY@gR|X{h>L6As4uXM{ik@+ke$CM zG1TU(`>xf*Rsq<@Q#&72aF;~EN^I7j0XdAx;vCplReAdiN&KLqXT`L*F?NDCQF8#g zkCC`#^KY_b!UL;7Uy`GqH0q>YH``*Jb(vaG4zB-tlPI;?A?qzNAo8oT&FDAI&P->@ z!L@jJdFHV4_29L4$wlwX*NY{d?(0W`!`F*4+qN_u)(W$XuTP%S=v^O9Jiof2y|-~* zLr&I~AJ$eD#49WB2n#T(&k5UkiZG=^(LB&m1+MHU2E7kdbqm0m0QKu)N3OAkz}Y?I z-gAd)R~SBiJ|U8!I3d1#+l2C!7^_|OsMqW)!6O|EzRLE`UlB=DF*CaXai$OpTu$`E zG0)~edSg`0H0GBE4}9A(IR*bG9}9a^Go-XG9?`AOOms)nVrE?OG`V>`eD~yyw(?A* zj%@i;1VP{Jz7L0wxot7gILh!A&fld>nn9y)SDSWy_DEa|qJJ~tY)#Lti-piL2Hf_0 z@2gx%ez_u6eGs?OH^a(k?PkIlL0sC>z7w?nKg*RQxxf)(;cWVW!O@mPB_}DTRLw{- z0-nH^LL2#Jr-ps~u~4=~K3oV6yLLoC`tOw56LaQ*&>mK12OPxa3`#FJszACtqxmo zim?V4z;&C_>YD87d)#)&i^)+4TRTrF#Y>ehovghWwtl*@&vJD$5v~pCd@l@D6J*x` zkYH;}8_zVG8(;##*fMjIa&{L}krqR-GHa}AK*{kKUgY6iRlDsZkFB>~v3?7RoO4~2m-kXNZkJ_1m z6cC(vWcTnZh`L}`IHe*PSyWx;+3M*PZPH` z(hB}bqh(91O~O~Yo*c&qVwA{!PZSS+wqzUW`rfjtAvw}~YW3L^pA##4nM!n*R7uMB z_%K}Sv{kCRH}rUbS%kkW=3|DJk9(Z1O@5CtYrkl6cH&@?dvq)}Hle-A8zz@}4Ub#U z)Np&JcL}~v6dCexfJ?g!)k*qtJcYP4>FCRgrHXm!eG#8RT^}Ha zLg8+s)Q)%yPEBLg1znit+y3=@z`tL9SkJH74E$(j@W@!JqZ!r=&cJ(|pk(CPEuM(c0dc(%n>n^KAj>Ji?{nir+4Q zNQL+0n2n{qovZLIP2zwxrc5IiSCK?6nLd?$(yXQMBc5d{gQ%DyCWv)Z5ylxBVE);V zV!jd!)GZk^l6V?1L?2087M;0H^ePIkyI+}5oWiXs04MNO%I6PBE%(u{{<_qS@qiN$ zeptv2ALZ{DL9Sb+Mta5~NqT^F5<$~4d8I+orQzdLs8*b%V<6Om77qsJRzHqQBT|IR z#f)oLwlG`O$*OL~6cRHGU2Ti$wPu#yjd~gMN%u+i5-T+lBT~1u+gM9FIoTV8Dw}u8Uud~ZlRVn8P3yx!MB0` z3w#p=z&DMM|4oq37qpLVvimca*064I-pr-){vz4ze-R;I6~zsr#s$8Q;{( z8U_xH*WCDVZgk(P%%TN2!_^)#51n!_O8cVxhPe7Ue{6`Cu?My*e#lCpjU~fuB>F(L z6h2tdpYA`;%b@D3T-+Y+-()Jy7MzkxR7~t08M8IGF~dqdeIQ_#{B!sVn`3{px4|}+ zF!?q|q{MX6_REurt;%VijVpSRd1n201GhylC3e@3Cv|XtzsugGEY6rYdGwQ~G0l`} zd_&CRaT(TP_5^6H?qQ3aK~-N~aH}R1Y?2to6r8y^QdElnu5=R@Q09YKzwn6!aB54H zR5UMoGCrx@ZgKBB$Tek8kwH;A>1wh23Y{Nx&;sWEKWk*E9(InQf@gi>)Ph_6HQLhl z`?6Y@xY`B1JDjR&t64bnwKG<89$>Ogx#)zWkExr&Ww1cdNC(ZSsj9YnJ&VbUk!~KG5Q|*vfhgPqxex{U3s&lH)nW1Y44Tg z@+YUMllrZ{7P+QC%g%e4Sa{k(=C@*48*4MvZm)~~-El<8@xW}z<}8T9>OF9!uJ8QP zdTrCsnI3eQ_gR4W>k{v+oDD^jUw*&XeRFN7n90zyT@{W$AND{s=RC4((Oc}=*x^@h zAsdkYvH*mdM#3AVHT=h@HrLA^)MeU}A3=7LJovLMG_s#kupMRdSiFccA>>>F%BrTT zC3n|-oB|zo<>@{$N_;KzdQn7*s|Lbm;8cgBgZ1HFnTmXr4M_r^IPdRsFrUcFu;hf~ zZfM`?KNZvXCD|`Wi3t3yGjXfgUl-kUujrNr(u?8Or1N~;Gg9IlL`{e`M)B$cn=tQ+ zxpZ0vk(=kqfg4a^d920D81=hsReLVmwCQ~xx!3nN*mGh6+o9={8>xvQ9{AbyLk0IBrIv0GPw_2e>)*W zg4i=hHF~?XUmgL;D(np6!v;l|a!VH4+6)tqK5`jg` z@97__9niTP>B#?F29aU~Bs0%Dy}5tv*Y{U{eJw~dU_w`Cb4q?;6k+>&^2&L_heUoa zQoqW3d=pR3m<_X1ikBV(d&G18iGpeEfPil;Ir{jzDg`P=SprK-a9*Ir2%Z^oe%lne;$i*j^?XbfEkD z-uXqMZ#`lxR&r;xsp5%)62M<+1rH7aoG!XK6?({+-ebPMdO zKJ}hvf|s(v9pe+CPjh=RHoit%GH|5iYC=vTy0|o>e= z&voSeA1|@CC-C6otvx7vgO9gby)L2FlfQBe zx_j92<;iXwi?UPLB!PhJ6Bas*Q#kja51Aj!phC4*0IjRz0eP{f1tx^T%JI!{>**|t zhY5Z|X}>z%9;1nae8gF4&>20T{>=_968~N56AqdGMFyM*oZ_y17zdvpxJT^54g>GT z`V7o3Bmwz|jRuJ~-)GN>g^mypZ_$5w!2+X?7oPu<)XWtc9E+?}d*>9UiEN5mVRZ8c zthu_lSDp8SKk=v*<}ev_yf1FzJwklt(m7jGeb$@OA;X$(<(+-~Dfo?5UpC9k7wVYf zTAEztJLU%rTf6ojtNKGGa@DvmOq?hzW8>Rgt2MYsM3 z;X1C9-Rx_7T`A>wsV*l#UfY8-=Y#Du4J$$!`rd z_bLw7ZP0mqeD)RuV+GH=rBFo$5;@Jw-EX$MF1!ydBVCTLIa7d;!GP>QeU_%As*!wj9`GP9r49K*mZX5AD^BOiYeR@StTQaG(2zc`PpSS- zZhbczz}V3GWfSpn3#Sd`s241bz(~82s5|%i*AuI!G;b4aK0R;isjA8~%=Ig7GAMY~ z%MjCp^DE5;{=9Pdxs$Bg+jt909PM$8r}r&O{bF*gG5HCfLV$dHK{d0gFA~hd+cy`? z-R@49UGdqeJJ!cHE~~ipn5`<0Nr|O)^joi~sp4_}7+5*s?y{q4~Qg=Z^RT zTt~e*^SFP1BjnzlI^bZ6oj+J<{VWi#tA z*HC>%`#J*T7&rV|)n6g=w&z9-7lKK?m`ZZFTA^^iGajNBU&|M3)=MUU)e_6N$*z}Z zs~f^DOD69Lc&lmQkZ0d)Q;TTskCV%r=&Nas>L$cTX+o|v^~IDRq|6W>W`(wmivFha zEM#Jm=OZOZL{Cl-7(GIfodc5c)_QDhFr3LyDxMkq1yD`}x8lng+rP>YmTa?3dNEIE znGFq$&aNp`QD{@zqWILq%_-vGLUPKgi3rr{fZEwal)fDW)bxff3>L(9LzKbl24(L} z6NH%Cd1UJdzVU;g8oI$tw~uQ)$)C75Q-Pq6QSSEDH(=N8<@6C%z$5LFZq9zE5?u7g z$EU7YdWM>2v?@hngv0iq;?RQybE^>6P5Wa6nU426R+0(uv41otR=SU?a%7&LCp8=| zZb}>qPtX->zEsC402h{^%pY(;O$Ibe#$GUcfZ6d?e{`-nd+W8P=8^EG1!8kxn)vqP zZakcd?Sk-VbwO8+X)bM1k?9e=!#ZGJM}EY9h0~FU%HZ`+OBg8M_wmbO{k*Msq|1(w zOHtbJo3QFX{3RetJ;YPD=4Cifsl1;mifIS9lg-IJcG7lU{?-A{1Jb%9(vli@UsB8Q z(UKERiMzyr;!D*Kx|8m`&gT6YNCh4pX)L`kwp-8*zKAnc%|ZJ1bc(?u9}`==dL-3x zY0yh9Yv{Aa(8#`BNYuCEVbcol+M!M3I{gx4QJYPZ*=~}IS0!r;`<+D6qUe~C zT8MPi9m+~GZ5jH_0!GK-?Q`e9Ek6M3X(qFtotI*CbMLoUAZ)|u_(2bJn|0>0@R-3w zAQ2@yzubgytbRICjyvL~!Q`7>e*SSQQU^FJ?g#~cs1?2vyKQVCUW@uCla@B~m3sL}kD4H41KFLg0 zWVP)PN5vsiuulMGfvjPbLmt0}7;Ar(E$I?B`Bd_} z4g|7PLnUze>|NQY`}~MPR9Fr_%4fWya2`WEK4UK8yYO{KA2ujwHki# zKXpr;br%IwYR;e0h!-uP-iT1vi`x=_oeC{1{j|*)*m#QUJ(2E zCP(Y|7A@o6!)1fKy^;^M)WLU~`CJe*K)Uwxv3VNuP>K;*WoJ?#eK)c^zx z`CU1*c~9pKXGuQQNxShDp|(+>+?;CBMLF9`Dao|A=^j-I!q3}?uj_h`EQ?DYKKp5t z*MLc?Q)&sB#WV>S;u-Nf zGsMVG2U-@_*=kjvBXu4;dbM#P#-?Y{A8w4qm>ttupD9-U$YZ@!ekGZFn@auiOtHbP z?@w8!h?3D03Ao5Lt(r+%!H0;;SYV@*lEXe(WskD!D^v-omuB(B*-*XVX^%|T&Rs`# zf#fNfVNo|hDtZbU_gKpDtUa3bdq>;XR@yc`M#z$^3()x_hn;W>l*itY}Hr z1Ty4g`MJ_u?2A9cneE$K{M%!bUdlk)sE9w~o&j*G#sR-bfUg8R;rQ%{NRw5e%aHps zvk$@H<)4@|2h&aP>w1t76Vd`f?GpkZT~@vl^ihrn^RwAfY;QPS6)1F-=fd|>JBW07 zfu(n>=PnxID-C+g6n#?ml)4TY%)A5J)uZbCj8$~ zG7iL3@8nY}tu@ggO#05^i1&@zpzi2xw;ncd_W(QRWGUV>5Ygs#a2e)D$XVcPng7&m ze>LGmv>SimH2_B5QRnU@y9L0KQkMfVZ_i%R^)cRFbsb93{cIDjUZossL(YQk69+S~ zKwhNRFBb1>j56w26ciDjFHGZBc<#nQe^<^HMd~y!MmKfN>G`ZY5j4`1J4^e$HsFdt z67m%&YjdlmT`}*i16N4*9({`Xm4LaIEcw$Wq8%cq`D3rGz`wKS`rh`CM#^6%R{)zf zE5Y8_8^Ud9oK7d9Qd!UVJNMEf>I}p9>WAzR8}h-IL|LP48$cxo&UGcme$6$;PF@r) z394AD+J7>uI;<7T0pl$q?t)lCgR$z%uMdWSNXqqf@wmyjA5?2G+C9ZnIZgWm`-%_!w2TNrj+ z0*?!|f*aJQ$QQSVLQe@g<;ProlMhN1K=B@sgL_!;RpY$X!Kx10Y>RI&8ujPrWAm$t z;s8r5DKB}wZmqwRAq;*+gu?e4q#|a#N;IqrCzUIJTR=Jel&k%*Udtp_hL;lXz-h&R z^S)I;^R%9zmo4z6;XFqu970``-~*U@M+Z`sdoI|-d47EKeerQ+EpN_}=?N1G^ByBA~yHv8Zg$>%XV z+XMY)tlJCr_i9g4z4wOuFI~b@4{xiW8&Z9}#P_jC9?!kC=&12|uL`1ansW6~&Pb0! zo=iXA7EDvbE<(CXpb4>mwa6GV3?)$Pt-C%dPSlx8babs_pr~!%({SbX``qTyHvbK` zL3HqM#U&vL*y@Hh79`%yIT{U^P8mt_y_jU(XjY!Fztd2o*t>u1Pr&%)g2foMTZP$8 zhFa8Ym5}I~c&;!0l|lVe86UR<(fsicA0@I@f9p2-AVdJ3;7JC zvSZCKgp2ST?R?-3lfE6cCHfh!lG%MWF?SmOGMuEtY@m;5Hko(2Pqt7y%!=wvJT!#1 zh=xZ4ai@XcIyIS4RVL2Ef->u99#W(d>D*q>I#eC5$&!SF*2c+3IBtISMe64R4|r2T zyUj^G_v_=Ac4Y)EjHLdJs}PzrO7^CN{slJffx~_Gam-_A&fkaGx`3zD)RVXHw$)b` ztJzT`P+V%FkQo-nR&1^JT4oYNT-Kw9X$t?&)?ww^fL;v1Q_~7se(|CdB%ir2&s_`g z41H!fn1lg`n)88H9V#||(%wz9rrU^wQ@2CH!(Dd3+EgiB5Y{lY>A8*X>k-ip3Uuf0 z%ltRHiN@gxnD5fMuF2=qG+V7hC~zu++U-kJ4VzSIJh#z6DzjZIQqAbiSr}&(5SNMV zdlp8s)Oz12U3v;7ce0>1byPM{I2-ML$$W9M8AmF#K2xd=vGmrYS`i62(MSNf#tV0E zJH%SZ6~sMC*!)ckLA<6(P5ESS;yyb+!#GI=+~i7RiOlm`vHo2&tJAcNC~)CJgqd+>8c3 zGaU8pqhh!WpirLMYv5Nq(&_w241FbGr7C7iD zUfFu4s(Sh4{HYLD1G#gM7aABYnW^FwwaL4v9W#Lp^#oEiq+=-LIU<1X^}?VzRzq}Y zsjyt8>^#z8M(^sF+lKdry{6 z_+I9=3EOM>mzNg8?|2~gNK*x2szKtmS&{dPOUXNoQrz?R601>|1!hx4sNG-97tuFDm#TfVk9)Ym^&kM$*a-TIoX zv{_(?CW&|6D(eHl`YhMj!~R*wK1yN|(ekJ@gSq!gS%EnetwtSiGJEFB{R2SG^{;om zxX0d{y*4spfbDwK9K2anT>X|-bzx?7wvDjD6uFT`})t;>chO-!4 zr#ok;1g`U`YR8r_EGh_hYDpTm%6^ttf}t^UkP=u!2_wZhE(rrh1w*g-Vw&a;+f{yy z6BIwK)}42Le=GDhsgP$m+JZ)Z@O()zfHrf)Tpme#lU#fkGXxi-$u@+I`}$SOro(oX zPo2VTzc902!zn!)$4ej~fw#`L>7CEgqvMLSIIf6DDs!Gk7M{@=S8PS2h=Lv?Q%+Bx zjwa9@9-z&_Xq!VeR`#O5RM^lq#%&P)4|(E^P3hR|)EL5$+@bLEkZcxr0mJo?wBYc$X#}q^R@oX$MFyy9^Kw zt)H;lLhLe!#&t4~PnJfJ-UkXFQ{5~g<4Po8+*kHL%Cz<6W!>)u&1=M=yP6Ppv}XF+ z*DIC@t8#(YF;yGe!-v26X!n0kBhYUZihtE?Z$&8%YW<4CgVBAs?Sk_;g`t82*1a`O zfs9YVE(~|WuSDMQmRVh3(UGkUlm~PMw-OMq?h|t-d738JVlLcTLflh=V!TLW0$H|K z!Kq%ix(FAP^Ih%&yX2?nh4m{-7y@RLa~b$5DHi#xg<0=kGJ+@{UghUgKHKjsyS2Zc zx}DIiGz%Fv*|_IQT}}Wu#k@2E%!)+%rHu_QRhKuJhyYW?Jx&y*oG}HUS48~upae4p zSb|aOgsjJYdI<>gi?IT*k0T$cA$+5DZz%u|(!IuV`a9m)z4U_O)B3p=N?xIhg)A17 z%&Lnq$Q39P4j@Oi&=#j17cMEJQC{MUe@hl8lpA>8>92$6P`7;^^h)=qi zvmJlntr`fu>AV`1mb%ovUy@(9W?W#Li+haK_S*!Uq_3+ekj#$tHCgA#`8%28v&wpc z3Is(gyPg}elm@coayh%5rRwqi^WwJfdf33DOm97CX%&_1!K}sb%j15#n9`e*a2Ipn zy-W&l4?@G+asoOQxfwPKkvEUhU=y6Bmmqc?4Gk*ShEVTMeHaRVKBoy zdDZAkP#CU!m#@z!4pGlTp}%5={bS*fuWrD;2L#lHf_IT{as<2(S{CmG`9_3;RQ-S# zfUND>7oL8f4R2;lxX%eolS)YxTSQ#Z5u~cdd@;yo^9kA-ygD^5H5nFM6}+ize~mwi zI2h}{Iz+Zze+~^)X%r=h$zt1I8t((QGy_VQLyBjFJvZJEMHFYsQ z=`-Mg!Y@d$z#}3PUBgC}eMPX##Ob0pco<=bddlhgK|)^@5M=yHITyr^c{OZ}QdekY z`l#nG6I3;a%c-S+^$7jO?@5IfU+Uf?}~#+MJN zneL2w^~6y}hJx`l&wRyxhIZz0*-BPAr_tcWWy^m#kH3h0zZ4C*FDX7|2F^re z6{=wKw6j1mjGNwi2=lWS)u9m{cfpKgM6pAVGIcO%SAUO2?S6Hj6JsE>Caxahwea+a zwRn$$uu2%6f^8&UtvOm}Ue?n3&_PQ%;)!2g_D1`D36DRBlt4NQz=>a+G7b$z* zB~dSy_wJtE7h(>Z82I|!W}`C}_9Mt=+vyP^(T~f=p#Kz>={Nz^0O0&mXPK|sj!G$s zS*EXzTiW9hRe;jj3K-aF^}t1;YQdk^8L8O=RMFgcdSR4BGj!RbaMAH{=|F8rCJA{S zBw;0tLx)R1_gNaK@SNN=<@v%m3d!-cqJO4D8A`*XWAIuDO2e#E`0}Ajd3$@3{6`Z% z;8wbm&tB+tQJK;ksjnIXCLnMU4s*7cyM ziOYv?L5TuLmzd*_WD&AFb3*NT=3CYTSrC*yI}6~g)-lX+T1_Z29YH=Sy2ZMjwc34I zi??6!Jy5s^g*+%k2T@=qhP%=L&32F1=NRek1m3IE#HpTkRZ zBOqg@4Drh4jc83{s3#6YtBvWmc>3gP>58evT-kEt=zRu$Qosp&)hxZM@V##I6LAB5 zE~wZt`?2?ZnJ!4Wwre-e0gb_Kdmv%iHKE!}Em^<7s-IW!{aL>qlf%sMP_;V45#sZ~ zqa*uN*9#}<-e0~+b zdeWKb?dK=sCZm_@&92v8B-fIXN$D>K3Rlq18)rM6q^v&y`9!qi9m96wdkM6Ph5io- zjec(KF`SVI^|ThvhyFz?=$>PTvbJn3Drf$j8&qzyA;uF_1fEjKJ6)xF3StYbM0Z&6 z?|e25+QiM#_NX=gp|)hAV~nBa^3-ADJe}X6!}7a|torb(KOfD=>(PI}RRBKgkLvQy zR?zORcMjg2zY2=il{J*n>hjzWy##X$<0d&C&20(cGQ{^U_2>I&?RQxaDyoO*yX4B{ zhvqbQmv{&h*|Ow6A!-YVvhRAgdhn^PB@}7Yp=SBpU*?tXVs>+DmTP7_g>FVfxVA^^ zWIMsuUq&9Y7cpx1{k=^?4TK3CJx#z5E_w3bESV=fKe|HIjbPSS9x|GmrHx_7f6jj8 zr%gEw-xCS2X%D40q(I3d_H(!$D71f|NDeTo5Cs`y1W?T~coR!(IF-*%swXuZtquaE1~klrCurnV~yG8Wibn zkQx{Sq??(ch6X7SrF)PRl#oVRdR=(m&-;Auw$|FV@7Mf=ne)2NH z1FRcb{i20AzPTL0@<@YPfD$HR7@MwR`uT!*+Au@OJdmU5z6|FgEBwWe&qkpx!?9j) z;gZsE^=c(nFhgX*|%@{o#LCF@W#Q0<)&WqdZfJkW@mMmhN2`GZ6>i*}R) z=eEldRhx!9qz082YMm{q(AJwDZsqR8v>}7WG6RnF*)%IohZ6iU=8={7&%HoFWRP5h zbeRcu&@s&$D%QXxUusXTTF(sJ(m-}_KZ77NALzxEvzIo22!S$I0wbXgH(3F(ey!}C z@H;1_dr%WB*wS4I0}lpOC!@}(!cZKm9Ze+SRdx)A$jeOwuMd~UuezaAEhW8L`#S+T zNhp(h+`2+h&voBevS$T;)VbMq82(eQR_|O1N`w%^@B}F^O_Gh&jDpZQ6?*JQ2 z2o~52&yrfa`x&_~ZonToWRA01dBeAY6h5ZFawer+8G?=p?REh5o8hD`y*3wBpTF1F z<}^!Q{DesX|85hsvO{oFoxu9j%hk4T=KC4`t>lFfI{~RzKB6uZ?<2!yj)yM9daiCl zz}-s4Dhg3xWMEiQit-mhm#~ni<>CxC3q)^$r9S<`_CR;mk$3M#B9d_eN#|pe9iP#G z6+8S+Tvi`Jvxm$#Sm_-J?+H`)f2|40ga+>`Vs9Z2OAHI=at>%hp3s|4-pq4?ri`VA zoAFNWlp_?LC1krXaag%lCC8c_ndlrAH$k0`3fCWr{9Feg1?^k8H`4Ygd#-!dEMXdJcx$U)S`71L$ z+LxK{pS+gl<5L|+kxUdfzgl@a&Vnblp^L7B5+V&B$jMe}`vEvf>w5xNkL?eA`lI(>HJ2&#reR*v{A{ zavw+m;y&AwLhwKkXQGnlVcgKokeEU;tnqNr6uTgzSJpStTfMw~Rd+;A(-eOv6a0nW zILzix-ple|XOGv@3wASRy1&iVJe4rko#23o)ZEhfh2w7Y>$^GAnj*KpjZT%;ao@%7 ztHq&C$$#-yn`;O5!!ub!uZ#=L3U61|ljgEn@vw=l=%#~kqryV`((%)1pkA3xhb$5r zE5zc6n+|Rz?$q{%mG1@)ccMRXY{e+GHRqMtV z(_6!gV1ARozByIXuk3RA>E#98%UZRWCE?@Ur;8tS_2TMO$63s^_doxa9#OicN2cld z0iPcGMfK$VK=#?F=OI#p_G9NwfFF7LXElrbCZFE-{fO)zAu@DyjqCKks{zsaB84ar zRhjPfI5?O&^rcUf7o=*Pfb7eB%hKpR1x*TXJ2|sr$`Lcf-DMKjHV9VX0dTLH8yGRY{HK3@l`-E7`5Zp{rkY}KKwMIa0=%@yLG?qBD zzV4yhslSTbF7K+~?oIM}6a!w_17{7O4!(q~uLF8BT3w__$IBp|ijf;plx6A8lh**Z zO9D~*TLWM8UTQnpfcj16c$!6b8PA#14s(S(dg|K>WwKS#;U3B$9WW_bRNdy+yxA`q zV_~5nT>|`XZiKS-ba@vtN9YZmiq>wi!|hejI??`$MBTq`8-Z8&@)=`1RK_31dYk9w z8xE3Qlo1Fhlu(_#myFRt3t-KO4^o5Ih&a2<%ss=n1D*DdkmP z{Q`VnfSXyY6bJlMTW?mzn1}j?iC`M!?=xQ!ZIT2nKChgnW)85kz^;3{{A%$RMZ#)- zME~A=4gKxn$H!CD=kiaxqDE{!5?^hEnmjo8^Igzi#Z*0U(@Urdhzdqo@*?ysR|&Jd=# zNr4ltK%$t+2rvafIBP2nR4WwP?>%_qp^WVt64uTmKg5@f5BtTzcDIFOY7?YH?m=7p zW4rb+%(Lzl)lbre44G@&{%!?yxQ=X<%wQOYnvp|{RUzS3QgEsp&H(_Y>B}5K$wT-E zQJ;lCt)WT;5A}RXi)Dds*Ru1c+O6aE8K0@(E?wR#-||h9&}@=&PZiZM-m69dbuf%u zd5ZPstjG2pCD@P*#_iXiBgxkGYcZkI~OSatm_;?i|Z*cMp=7d(DdEL{Kx zv9vg*WU_>A7@r>Ab%O#CX1e9;Da0EJOes%lcn4nZQ-Vj$q5q7l6BB{uKm|G8(^|RI zy0wtA3tr!{|Ih%yGr{vOr&IU;!RgQoi~WbwsR?d(<{KY{@lyQ=^W^#rY{M z5Bc7Sv!vVN_b6h|lwSE1C>#bWANW7x4}EO|L=h^%P-7@%_x+&P#U`J!Sd2@ZdfqSW zzWM->^oMW#+YaU_jDMus1=tikDvkrHV$F(_Uz!VQBbG;xx6i%s-{S*N?G@X2!29$pYo_T3Y=x`_9y3eMso^Ms# z3XG=d*x5iU#GHA}+TGk28kluyOx~D4o)sq?OZDr_f8N?|w4(q7_QdXX`VBlZ{JWgE z=up}kwx|@Ce~6RBJu@d0MTC}d>k|7>wqfs&6i8MVl+-wkqijK*6&4>a?GC{PT-u>U zlx(92=1vDRj!^5A$CIVfX$FCpHIK;haB-m>4Ys=p4~GO+5z2RI($K`e1y&>Mm%*Qr zC(v1(O2Yo3JflOc;%U)pK@Q!E+s8*w_dzzba_Ldw5w&6r0uKLiK=4rHkoW@{t(m_X zt$np^q}ttkmWiHGgbx_{ z$fY!08<{dR+2hWBU3s5XnmZj|=RYal2`UQtvlVcBU_GKflXaf+o23)>!wyYCjXCNK z&ZQ>s`YOmxS}4M8dBXeS{b$KiW2j8T=2V1U)!X3pyIR|P7EI3mb*+oJLNYVtX3Kxu zv&@LZ!`s7#IJe7~kCw&OL6&`$b+FQbya=5rYG}9C9}rp z=*AB7tFyiQGi+etKkx{!X12S^1x9=AHb^8%3=I<16q?i~DJ-;jw6^UJZ2fo#^bTz> zo6O5ai3KbzSNQpP&FXK=-CpBm2>Y;k0IXTPH@dFpRa=~(MZhK~kKUUza9~Th;iyyF zdk3Y7e(Ii2`}x;I$mDVdmjX>370MrA5_I*4HwdmktU>IGNQPsvAmYQ9%uYH1uwNe! zJGV`2-CZ1rx&b5Y*J?B^-zzM08|3|EPDR5Nr%)d zG&sifmOim;4HY!j5%Z}xZ{}9abFIOI3k2CFUNsOqJ9Z0DA`Laz;goKTNJllReYBzA zrhkXgOR$=K@AwsXLoR5t3W(hMDB&$9Ni)z#Dnm|?hcHL6yaCC$9d>q27YCW6P3|P) zg>dlsSl6e=>fcX5v6Sc0=*_V-|48Q1O`}OPzS|OB_U}pClmaB*#?5?|+i#2Olwi?H zk`GE6v0#cra22v(a21LMgM}#3BzD4&ktKk(1eb8@PF3_r(sc>9W21NXoZ{S{-0bA# zfJDGS%`ewVX-?2FyH7k&sO^>-(v{*>`+`BgKUyY-Hv8*g&F_0r9=f=vo|4Lo?&~s# z7rt;pKbc6_hfQxCWjFq49|J#PqQAu5h&+&6>7q(A)8qr>_1>Dg;H`IbLu3{4xtfzWu^aXRu4~w3ja3m81D3R}1v>X1vn}+8>33 z8tVz(P0==*B+8Y`8ouBKWwWFoegaE@;2WQ*+<-nYhIMGola$Qn7J(Ea>#cVysx|)* z4&m)p(hG(a872<-jE*!xUw{6p+K3Q6E(BUToB4h}kOIKCq;Bv?0rYBu$}UGLyu#T4hb{i?W{fZzVF)H^MV(m?d3W zZ1el`{8PH-ed@ipB6gbRzf%3qe$$NIxhLX=_-fggeHgJEuvck#ISqJL&q6#d)vuh- zXACd+W#B5rK?R`-M5gchIh-i@l<7Uq0(6ElBI-b{acpIy++{!~JIrg^lCM*Itw|KX zHuDZkpELbh$ap#CDwQ2|Hpi3Y=5@fbqj<`+{o8ML#ivo|Oq+Y=nV;$D_mAsz^ZCJ? zrzRK2VxDmwkqQ^a&R)VZ|MWr95|Pfc%SW~~#tcaumi{u0{C*_Yk)NHwn^PwwjiwV) zuF-4hbtQ&_id3{rqFu0IYPE@tVBTOikf1{&rB3l^)B4!~kA96>g{%scu&k)C&%fPE zlXBt=sH?kH)Z<)g`M(6x=ub9lBWUcW=nL}$o>_t#m^B1h^e=uER4B}GproZrx7h^N zI@1Ou>9qX~-6(aex@VB@k0KyyH24$(b|2o3ppw)Li|l4_IiR|cVgR`JKC4vmT4Pze30``R>t z^GUfpLxjPu?Ywyu*Kfaw!~kytYgmgE`s^ZPn)zR4m&fJFVseC+TiftOB_Y7^7!o-v z9yp`=J9ib~;de499;n~<*3&%%eOG|ujA0nCjwk4GTdqLUg4$LA6kh$`!&l_%pgm)>S z{9?pymJamZwbfv_%e>WScf&7AanI^5(UU$>jD4kX zXr_Gv%KylSso#<}QGokuA>Y4d&;N3|S+_61|FC7 z+aVI;dS8mQb+?XNr?dU>dx|QRkWcSU1Rn|T#7Ntqjrw@Hqn^FK0Z^L0cgFGV@aPfi z>pn6bN5P?+IGx~TTIk6~A{fNUYp>ps^Is4d@Tp$wfNGgv7;-@7d|o)m!qJ~IwKI#Z zN*d)+$oHRxC8|_x#05|#zw**`fXZ+;sM38xB+TFMP7jiM5f=`9P>z5|jM&9y#IfW$aMih$A0XWD`jHIJfq8sfxs_`HCU} zS-DP>UX&i_3kHu=vzR2hwHs!L#(Av-!w|{`F~^ClLtkj6saukX#;{%4X&wW;Kz@XW z$k;~^1Uy~7+>Yu)`D2kY+bL3B@o7u@e>XMYqmmbZ^=o-~BAi|t0!y)kEh9cEv%JxN z|MYorjavq3P}Ci?L8BJry5lLFXmhCK>bKh+W6CN!QI>6D5j;0642yh0;VwU!Xon54 zU|%P^ST$`K3)qU6?7s4v;19_DeH6tBl~`l+YS6l@m1>>7dMVYEcy-9gOj|*;3^h7vup!9i7 z50hp{7l!aal`xEbilOqegtCcBmJw$jC=@x$N}!!<0=1T@o%lt@7y;d-fP?T_e^nB_ z`Ihkl$lBD=uhA8ruOj>H^rf@zFYrDof`9Z#X^yZ5_`?l5;BZ6{YftpFDhK3&o{$v; z_k&E}3PK4sZV~xNdqBZFIoE^}4ni=Jz~Yn9lOXnclo1cUvgosNQVz+Wqm3D?``Pr} zje>mnRA9>vH9;7xPBD>~GNu2@8_Gy4C~BenZ0Q{e!Z(|qlj{S!GOyXm@k4T?9cIK! zpyFo!Aw$jOq>lm}=aNcwYr{4#Q<|^Qa|685ak?XOg*CpLcF@UFtg;Fnt~VBZI)J%+ z5rdceathzu0-D(&Zx2W-?#8T{(wYG7Z-_6R#1ABQHRirr<@X*w)_XwA z*Voq-aX=tYjFJ24&pwUN5@6*({?E$c3t2*5YtFtJe~Y_@Z>w2dbgnrXGA^=Y{;JwC zJLs=qW*6tp8@lChuO`mERo`qeIT{CgTtdMi z93D8kvFQ$o7&W09O7wA$tU$%5=PjnuFrg6u!I53I3$;b8=mp7N6yS~6*+|#G9dmH+ ze-};&?0FRlGsL3h^Tv+LcKjY|3jQ(H3tU>dwA$(+3o>RGFvvC|JDV=QyL4oF}!?kOGL8I25K z1Wg3iqjrro2e#4%Gk?dWA2McJxiuROPahB|b=XOTVA=G`KF(akD*uGQu_e)nUZ+i` zAJ4r&5WCDEI7oGhAX;yk7gx?Ts>L`*33k994jeswGG&+^p)d#J4f>~OY$Du@tXX=1 zzz(Ke2iRcNEomn!Z1TgbBxnC{#D8bxcqu+2_t6K5ZS0*0T(i3#e!2}h0H#AOCB}|nYoIfX$CbAHe{lwypv6TwIJc`$_jy( zdq0)^j*`x2|7+%`$36T1vU0?WwSN1n4b#Wp28o|#u>Phd8kUl98@tm3y*1hWgm9$r z?PHy?(-KCz_6xCqhRgczr&`P!oX4ijja-uWBBUeh?}D)4EN{flryCGkX%bu!1RtS|O+;yte&@Y2r4 zfF-lq6E6>Y*Us|vl0gXK2i#zX5*ug2a4hQ|*GXPF+^_gSGL?t5!c4!;We+;+wG4d_`oiY&6s%DwbNGVdeF6qwXX^r%=t09x+R*9^vQ94PL%NoE;Lv=#;)im>s7eis%9vIVJwD9T4g zHE!|X-5N)0VuEcs``QQF1uGv}&~po+4tyQyMUghxgL~FD(n53qJX zJK(*hilyN>%^9o9kV--wH@I73N#?~S7Vt`T%5LC{ZySgtCB4bxGN>vb-vXh;N5H_2 zo`ke`4~o~vTbVJ$JCa)Wv~P!26NwDrF>eM&c% znCmv*%}_VHNuZ1ao;N;;PZj(v)AUZ<>uliUsyFHHmHNrb-96^yzm# z&6FU|_db2?rDwI)W`dQZ$=E-tFMG|RIT-b0i|uKH*P7q^O1Zj@JUP5)@=O2Y-a}RB zd65mlKbs(>3=nP{*G=orQV+y~)y(S%WT#`z{;{zW=D>>5a96qI%P zeU{mB*$u4)l??v8;Kn545qKCjoaLs>d+d87*4O553 zA2fVv3TBKA&oTXmQmG$-0U5Y?u7r7TJ=pZh=6nO-!zUQC^MJo_e@26XnMpVh2ukcJ z=0qgUjvGR``;y0+S&|Q;-!Wr=o2B1e zBy`L7iT%bhf0V;v{>D^=zbBVZ7eo(Kiv%uy?Yg~n*bIS7Dd+7u6%AY3th6>v=YZfZSyreW2Ju<#!cpozsk&+LLSh@Rp)d z`+6Q0Wf>laSXkwi><$QLy%O`0HdfSHxxYb^jW>=GfLXNbtwXDZnm89>`t@lJZPr80_8ml=U$*UTA71hc z&kSK4lwj)g`6{rVny;j+z{&~>tuF}h#g<1xo>_*)koQr-K@2c0dKpacb51x^La~Ym zuk_DPA~=XJ_^v_^i4#J%dJh)FVPQs_+W6R8I-tXla1fgkWXx$}MxR=w;Cbvs;}3vy zw_%GagP!no=vaW>=)hS0<2*#*Kc^dVtu@hWOY3}XstAl~GF7ZEqLdUqe9~Z@ElvNz z+lBnXK(P$=9wQ)&H9A|BdYz(MNtLMb8O&?14Ft%R=yl=Qw=1cxO-iN`eoeM1}jo$b9W7jF!OG%wpX_`6+<`2LXPVKi_L>uQKEQ%+DSa z)QynfQk0oo{dw!UtSEwMyKezP$WS57gGZpfp3m3_P+O5OSLvT#3O~J|1vo0OP;>ZW z2vvPzA%={41et`jgLw(xQys9Q5kfuJ4$bB=bGriKK{2D*qk@`k**~01KrCp@>pMFd z7l7fXMf-%%{&$+U2oSg&>ipDCG{BFD zrU~B>6LSUvbb*1y>K5InPd~2=O2IT$NbXY(J_R0YP^5xu&{O{h7T8XXf+r)ExBDId zw~_s_(U(?i4lJrDyJfd3IZqc^HW8SAvDIOQD>w09&E{9eAmQm;e&3OC)a|p1mA?|C zgZ}?5LGl75NYgq0LDc_W<1M?(Br!}53zk^tZd5tC?VRQM0~@B85_mOqynP{KLUYAf zsv596m0LyiHY6|-=6q3kB^KbFfAx*-$8Y-8SU)pP#tzCns3Wsp5eN*^t`3%K-(}m9 zs{v~PW2~8iAiX&6ig+4$Pjtv$2<0ylqgzGlfO~&NpD@~&i|V_ri0s`bFvA85~pb#F-Z4aaPWmRB{ zO29y`++!yOhYDz7;Q8dsEN+|xkP;|GF%tPsuEHR!k-U!^S$Xr_By-orN0uNwA&3$z ztQ)3G&qd4u;lYc^aOc4jD~~KuxWZ*uUOE0v&h-kDW|MO_ zqMqm-_SkJ%m`wbi_2&h=M#w2l400bL|vp3fBW+76ldQ!WSrg zccfz4TEOW@q%}($p1%CLM=p!<9hQf1Y^(3qTS_?~a@&Q#UhBK=yu$|T$J`F3AEc*X z%x1N-Ld|G9@OFU+ItYCwJzWob0EfyLI6c+-{wvl%+}Dcy4}CV&ko*++2g3f%3`^)E zvS@Ay^t#9d){W*i(l#evC`xlc^a;0*ldZRZ{&#vY@=3NT+|!Kv9S!@=_Zv6`c{w^k zwZ=K3U)c1u?#b3f=iLY~(kIa7{h5q%M}DHn1HFD{ZN?0RaFlV_)b@(`??hi0j@c;JCQxQBsVw#riyoX&2U%IJeIFE5fPEGP~P|EVGCEG|a-L z7JJD)wRb}oZ(uCL+e0aGn0>g5i&=DyNnn=)uPZKoC`w(@76@L5MC?6yzN%g(!E?E8 z=zM(jBfgxrP5X8v?8!usb`A2b9Sp!(rmO%Q530#YUw{I_zIo|yDWA3&45+BUKq|0% zxLmJp_mFU+z6kyvdl$`ZCKz+^b)1FWFWG8Yt!F*-5_hW#$hns_G0jAJ`_bo@Fv!w=5X4=f{RK4B|0LL_@-5FeC*??+Sk>j z_(@Qo5Vq6T_l8F4?j6i^@=hxPP>UHUuDpBeqm|!Tw*kL)mYEogx#@S~lC%;GcXwrL zQzQ#3nBd+2Em!?wf+FfG9Gw8CXaqnEFh9^66~p?XqXdqpS@8KC+>cE^cPf&>3BQc) zgY;H}9!P;(UrPv9Vo1h|2dqQo&>&z=D=_I%v4gd?NoUL0!7%jB<7lRYj9uc3a50Zl z^L!iktuJAfYf_KpQD1~Bl_v%AuvMqOUXlYPom%)v*eOK2LeQt3{WOVgwr(1ssNbTp zgshss5qn#R+6YiET5*n`p@n+Tp-7zv`caMpwQ#7(l1UfvWw*#Gt97+A z{2uhy95~M33-QrHSd!^+JuCyXK#SnRYR1A<-X684<=tb6qukA$e^zMVu)LwjDK~Lg zbfW#g7=ABzM2q@wOn&CY9dQ#1#(r*7hYVAA-VN$h0;sg|(ZD)pl|a z^dmY|b-F~~{1zORwu!sHWS-37W$^h$=CojC_RwcWYbHlv45Z*mmcGC3kf=TN?T>SOFCxu8qVgMt=))T)?d+rri-J~7?TiwvZ>(tWsgoJ!ys{G7E zaUqp$4mrbhCZ8lmz3VWwo_K2Ff+9T^4rx_?R@O2*{YSV=?e9W)Kc-yerA>n4ljND8ne z4UH0Z*cIEC&wp$kdr)@wCEETcJ__bEPI8?t`jtY|0U!0X8+r+}ut)Jc;?Oldt1mJ* zoYeIa^{{F@o7%ZhKCYi{F*%3h-pAJgtVVarS@fX_fau8KO{sxQIaLps%}fw!g9uHm zC*`oBYL$W%f`5`Mg`p8u<+JEx?> zm%RwNJ&rQ5-za5-#+#p5&4OoOueW=khh_r&ZJ!MB_XH2{y6_C!Eh_I$0+D8vwej3`u&s~1 zOc?N_6++#9#rP_kWcz~cdS*ZBERApaF_hKUa`=%-sJy&E^WwSU)oYaF0@cu>GAv|J zejjh*e)TEMM4PR2)iq72hL2;CQ=wjv<4E$P+uQcTOqFr7y!1q%BT_fY=rAc;^=ggJ zd81ULIJUA-xGYMC>_@cZ{{2I+KJHU3iflFb%meC5)yf; zn&jmRv2LT#Gv>eS&Gt`dd4A5(i$F-GvTHPG4;S1w=|Fs<5>S(`iMD=F;y5vKUjAsD zI7_-J+L}J~(R27q{aG$EOh*(+3b&>O2kPP!w)Hf;edL)LP*YKmz4FjUni84Xl{>mn zsaqc1D`-X0AR8@*_lc5p>}h9R-A%O^Cf1dKqd*LwfKR(}fFUf4wK9MRsFJ%h_nWME zEOO`R1O0nT2WUH(hAJlbtvs1t658d2c=SvUfq8~(D)oRA6|1-I~1GVN;Z`5)VJJa=o! zbn>1xy=A2sSyu-}VuJ@GFCinn#$HAPD)Wmh+72A3kaMk>TCyO^@EbdbE@39+qd?(% zj`-6Yx?8k@WNApPicxn?ZD>8ixW_XW170EqVuX!GIWh_E`_-EqbU~{QhnEFK;WX$5C_xH!YTq4&9cr@h3uDkWKPr{*C7IWH*Wp;1Fc`X7|%d*I| zuo`&vwQC7M5>ZyndWcDa2^foVD#-xjR>YzDw0ZdHNszk#N?h;D8Zy?;sajtWHFohR~)IE_ge->Y>Q|CCiP$}rd zJVoFAJut0fUWs%pNMhmX4{I{q_-TOVah}}7nfdb_ttMv%!8P^U_`Wycmoy0=xg46} zOxAa4;MnRCeF?Ij-T*^9IMC(@G}3dENlcM^ccqt>nCO2B67mac%Fgk}yBD5o&GHlG z`baCO*NVF^tj&t0Y{S2+u=}2Q!arP;+(#I8?wn2{=REx#@`-Lm47xU$Jq*f{JNm^ zAN>2X@4ujQYY*`Ov#g_WlJL5lp>;e&ga{9)I>YM*Hl9233z`(@$3zktbOlEn>~(<& zs7ubdUu{C-zf#LeWUAaHZQ@0J@9Gm$wVxabjiadr5*>56^mDBgQVGta9q?IUPzA)2 zSE|YYxFR}PR5e|u&h3Kj&<>VRHhdH}#u57n_r6zFu8q(=4a(YiSz!vsof533(^NSwC-rGv|b4S2`rUaHvzP8 z=opvNL2bSai=Hhs7+44BsZTl=vofZg0A2G}x z&Xj9b3mYUmodEnY@TUjy*@bqYj+=Ocm#);mkMb1(HhJLiMa(vNh_iJ0sp=i? z?|!^R<`#V~fMq`AONg(tVj(@WC)R0aqf=tBtH9=P(4E%k++nYcde{z6mxjAB!FwgL zGc?rD%#_l}JRvlZn%D${lM=M*|Jq5jK~Qdjvp?}|X>f!VRj8KK>Otm)t0}(9Y`CEZ zzO2zNQnA5C?Vv3M`CMqjC+Qwe-IMtlYv$wMLF9Q%=Xd{79*fcKwto3dEA<9C z6%1XPcwcijNo89sKdLGAG{@qYr!j@;;n9RaWS{r-U}BTQVjyPSSte7NY}y}X&YzGW zGjuJP5{);WUK-anNPB(ko2A-c9)V0>vtf7N;!iNaoSarF=L_fN{n=RHScue^>$mck z+bp0#c?=u_kB8I^kA!z~` z-?+@D?$Oe8^j@h#4v}a<5I9n4)kZx+B|K~74WBmj4x*ab;u$RnB9BFl386)Nw}pFo z)Yu4gPk6i-luWeNj)KkVESgEYXNTTAVYKQoXxvo>^$&zwziC6lU#RTlq+fDZ0K$=X zOz- zM82sXxWIIP%Dz22wHf9}>eP+Wt=it$8xT2}lyH`GF#H#s0X}{|CVK2TTKweO;1f~& z+u-e4U?%3G{^o2lzm^P()3p|I4TeHQ?82_Y`s#=TCVXA)w2b{I8LV4E_Fu#}p3dF$ z=oB;3j}1|6JKwlOzhu>xSQ7-lu9@XOAI*QnxQEqZGcGHm4LF%5`z3jm_J=v&sF0F| z`LTIrSPscUT3L2{fSSNJNX8JBM(X+k`f3Y0m`0=c3m~AKs&x;V zjHKXauM2+=)DHb+g~y4;@fk)m0h5I=yUUN)JwZw6H;MpES8qsg3u+^lh_-E%gO8>~ z!}s3vVq*o+6h&rQk(Pzeq>^4jUy;csBX!}ttIP?h9m4%QRF8t+F~C~z#BV;KOb4BL zLuoMb47B>UGVw-8(q`GR;o}MhG+yKw(1A^_y03n9heY&z$wctfb7=Wb$atG!%88Di zwMQ?qElW2$Yk>H(NlrR5tHDS`&Qm6;GLs0a+JdvzbE`a%$QrXcdEtwHmLD@IN{U?F zYIQ~0pv4jsh3qs}iKb8I#nTzEb{3hatlCJ}cpB-2GB)}hCq7Dr6Wqg8$!)Gj>r$lZ z#_x)s67OPSrH6Ze67OiubWE1JxF`=pi`ITyVDk)!ruaVf)a%vaJ4U4LgP=7MT1HM4 z5ya8RMb~8JzbZ{STpzhWr&If+?HZMLpNcwT11$y|)qa^=oK$rIX&C>NVp~H0100`E z@wsj&Yh2q`HJa2QL%?uWr?>#r0_*g5O4`YlYai`=a-{WK25Xd0Ulx(~eXk>iHhT?* z`(hKy%P;fWcBK3#iFE-!=wj{Wpx@=|UB+{c?)CdtpOjScZV+|Ebw_BcM8O{V%L))$ zApC0>WJ-}n1`GaeA%HZZG=^2bk(UFt_Tbkx%R5h)??Ur?l|=O24hnzgBCFaD%xRhS zM0XwW!`%m|hDK7n5iCtP$5Vl3g*---XuU|)E)zn}AF+0tso}zS9i+E?`~ZMJQDL|z zMiLAgld0eQX-EAgWc{{<-&khOQ^jz{us#tTtU z?I0#V1DnqxJDcU5jbs!A6(Di7LGF@%_KD@Iwp@>Isfq5IQJL)LtLCR>kT0 zNx{be5wv88H#TgjTxa|T$YBlG2N*7%l=yvmtnL0y?636l9}JO-A_vzBX z9ndd}A^Lbccq}sC9a;x;GXX=qpRDebQGcK5+3DVj#E1Hc=FDIDxkqO!mP43BKxj=x zTz}259^nOru<4PVmUQ!(mOJ7wx?vTMH3E{h4@8a@e;m8ThU*9+Icnd#O<(rY`mcSv z;Cx1`3%UBeQ|~t=f#8Q@n?;ZdL%#RD@^5E^%4|EzP+;;vifW<#Z{Ojw>S4oK!nD6> zUWQvCX!%xjGhQwgD0(H(ma5+5K#k82wDY;pazG@D=DBqL8b;%YaY@=wOC1l9CsV6r z1c{8s3}h5%n}!NLATb-8@%L9fb%;npRg)+F&v@{JAG{@GGkHiM-u3Ze3v5uG)0*E0 zfTGvUZg?{Kw68pj@A+mEZ%%agHa<%X&;E|bt5FYjID+5_SM2)kgPqC*bm*#yBc4Ld zbexq!BIYdQODKp1Lp_dYJuK_>HltTMEV+TQK%@UW;@sh$k#>a>+QAMe!42Id`Uvgn zyrICW`1NsFOyXv=0Ly3Pj2y`5hWSvJIh-&xUhZSwVBxJFj(E^tD+=}UfY`)NAoZuW zhxel&k2@xxH`2@ojn&>8K}O_VCu0anM7ZxT0aM>wO;^Sdm4D=ea`@9VH^qlETd!Yc z!dMjib4#`1a&HTZvceYEA!6g`#^IH8i%Pz0{yJ1=N!t$=B0cAjoPB~H3f98BQs~yA z1F4=4an5a0_(;Fc^YCXAAj$j@u&Qk0_1{z^Mlwncd0jolq^CQ); zJ(51HL$GjJJlS7v|C-py5XWyl>|N^;M#@wZiF>tiO6)fBU)MX`LR_mg%6mj7{;mhK zjUNBA9z0uMA%b;E*Ef}_9+)WhKo2t79{zagyYe%Rq||OUB-*90v5HFE!3u?~-IKcf zQTu(!sA>3QNAv2wXYg+ka`}&+U!wY?ZUvC<&}dWA$XswnvXd1Hh!XXXnA666<%@O` zwueIla;$0-`gQB8q2IXm%kr!aKhbeQVzXQv15R91rypK#2Mk2-;v)H6`v92bV(Hh< z`9k>ES?4d7KQfk1fwbvP`Mq8pT^!gws-%+ep{7`RqWf=SHlM)`VwBEQ>w0TN(RmSf zyDJ4&Eyz>9VhQQ%`rcih;+PqKHhtAF%yhl2Ekv?Z9#saLxyw6EIPFVQQ1qy%0NCCO zk9{+rDl&j(Z;Hc3M#tH;XjRLW!N9th<9$RLb$Q7hkP>GBSO~)W+`6=T9oRT*W70v{ zk=>0HBP3c%9Bc|r+Qf%!5ne0h{etZiY%x9f}D(*}Q9tK#X^xeO_eP296v55YaEF(tYu zOPox8$woq}cJ((S5f1-QAHXDRfnQQjJJZn8$!9e3nO;BxfiwO4?1)3|~RAm)i+ z_9FAy2l=DkylvGla$WVZumY1DT+ZYz8=J$1tPnS(S zVgk+r%r1Sl8+6XSZ&Lz%{MW1EvHe$)JTIg?q^jM@jzxNu*`Ksn%zjp|LU#(6Qw zl^f74;^NkWqpR6ID<@d(vwZs48Pm#5$RWjLX*Smo%A64RGzs=tHO>1 zd!s6^hk?U}H1$2hRIQuPX!r`17nu^w8jO|zT+6o9rPm?=j{;0P`wWmScgpmD3{oYO)s$O~7&h0ugB+G@=sf(twj2P{{P za}FohnIvuol5X2g&Fz`mxDX$T`~HIvNm)?73FsK_l#F%efPMV^YM27mBg!Q8@t_jS zEPo6~Y}DWvW#}`%Q%d5C$gIBnQa9o0oc36aRo&s@)!B+O5P#+ayhGlfuf1FVFNHYZ zsVI^B^5Nj&_)sAAJstted#h(Hxa}hHwn-(%g7{XIdA%9>XUt-Chek8ku@Q?=Kx~A& zc=kZn`V@t2l8c1Yoy$vql&+AiQcE;R>-LaJs@pIXMOw6sj0RHsD(&g)aT=-w7W%9y zdTSPsK7W9Syt0MnuzBmg&LwJzG`y-X6nM3&|LaT-wOjcwiYbq^t7l!j=t4#k1 z54+YDOI*X}KH*NLQj^#4Ir+%{hR-ud=qdg_{@m=96>Ih(9X}Z=Is!4>yjPFv0wa_g zRSNRISnO~IG>XlgO}rE*MZ(zhTP}db_Psx5f8IVc?DLyc;?YZ+UVM7cm^$YbK!B z*4KB~RIX^%NvQMoly%BgZh?Yb;FR1m>dfIqi^|TsucRUrL&NO?Uq1r)httTSw0Q_E zPylKa&CwCz4b_XZ>GIZh&q-UEq!XZ+Jqt~B20DHqMae^0y^akAg+7!YD23iJ(=jQ^Eq_ z8l0suxJ4kcfF<-Fm$YG2WH9w%j5taqncs?8sm&5*(u#dn`^?;$Qvr38S<8j1WU#>6 z3#Tvl{Qsfry`$j_zqjA$6Nw-(Q4_rl36YTKy+@51M(^_ozt_ zMDLvlVbrKW@ILbWmG`W3&Og?&mWAP-``P>2*Zy1?Zx|cI{@k8GN_a=*qO*qxKb(0| zVBaF=*h?eUy$E%uT57&FWm)szq~bV|e?}(s-z>d$V@g|GVoSh+gqI7b2H=zkHL5rT zVb{lT(6RHE?-1yL5JRk))BqnQ!#-1mmJ*X@EAOwqYQpp?vV;u>%}_>YYkGdxZ$UY} z+``i;HxK1+r{llHq+7Xl;s9OO7%dO@bG^%HfrA>fkzb$dYtfCLP=S&Oax!CqK3Tu% zs~5N&C0QS?F|Bxf4^Nn>_DQC%N3*=C!OxOWYLkE;D>Z=2C)4iqe>k4oF`^dE-PVN& z$%R?Y?N>l^TR}~@{%L1W&Pt3y;Gy+qLeIIuq&nXNVmI{Es5T~{Y83Wk_>#%pMR)kX zUV;||GwvY;lm;#pb5=98>Z)TxYbLYciT^9g@n z2op&ivhB|^6f9TW?moaQVvFJ`xFrtMX&)WOO23NSuOvkYjBz!BUguduG4tk-2z&73 zW~xjkk6UV_ogQ_mtTGkaZATYG#fE|xZ@FSH%~JU#`IsPE=zVtOU8{p+1k-oORrj!2{ zxpO`C|AXB5|3={WO5G^V_D_9F@R{y~@s9+$mjg_?l!(i&*13-NQ4x9`hP*er#nxiN z>C97;b+3jOy-12}SCwJAhpm^@lIY6^fARfMq7#gSh7qL|YExg!xpBc!$zL`tiY!<6 zKX;U9%YE#9qu`KGGxO~%rUIq(*;DF(il23wjjkR2Ce5(NiSU5*Qa7u_^TmzT z4kit?JBPn^LAjaN5W;X`40a{QAq~U{HBM; zNl5f;KE$vK0fe%A;M9#Yg{mdmiK3ITgLyM1u;c(W*-IP%#H`nd4AJT~!oe!#& zz2Bg*SHyJknC{rE#fR~wmxqYU6m;cdK^{JQf?85fe0$hqnI~d0&*yAD-eX~Q$lP-O z%*FddVtQR%U>NYqQ;^3dZm1_woG{6Yr!`5^vt#6^8cjm4YZ|Nf8UVcEdm_A53f35q zeU>RSy1rXKZKHZ!0?vWjZX!DE>Iw_<+=N0LJEdHhKQWgZ*Zv?urr(~mzTy2RuD{didg&j_ zv^QSA^XOpL$!8=iJ$O8yQk-^`TPfi3^rAP>2l4Xy4m_=~hd~;(sz#dDb+oJO$4dPU z0&}e8v1Oo54ndwHa<`*fdWc#BiASpjN0ehGOw0rwQT%TaVAW+*-r+sXieTA-fDo$k zeB(|#mlBHBktd;i2s3F$mXG8gQWprTkCTev{=Eemr4sVXV7CH$cJy4?EU;MQ|L)eN zkBcwB_DeG&F9%S6C5x^e9bq`%Xo$YDTKz_N(KbtPgSFY{OI(Vr^-N9Mg4Ok1w6uO* zYAIfcX8gTZqg=A!d{ojsM9T%2&fjOlV1XyFXUX>#1ei6%y~qI&R|-yVSn?b{QEluU z%*-*AZ-j8(tKNAf39O?)4yh}M0SH5y?{A@&5HgG*z0Ut|5)vqL3|gnU4h$IT$aFXj z{udP^F8#l#82BF7yRMCE0%y?ibmqYH_V4D~?@VhAhPM1{SGcFfaw2 zj5*xSx$3iNJ`oRWt|T!0I~(XcXq6F4j%Jq7xB#$lGS(W_Bp`po zjkf^-8R7HlF@bzTC_hEUs)JP|>08I!`|v8q0`CqsbQH%I&!6zWmGMpn*Y*$AP$2yx zW+=tu+v}r>gi^xM(YruAN5O&Ore=I85E+mx!jTI;;-=!cwBYVknz`7r-Zr~(2pBm|6{Uzm$YR}FZ-Evre_?I$Cp<2402U8>*r0U^_9 zfnO9@6yavO*t@4^)IE-r#m<%YhVAeB?)#iEkPWK{t)@r7QtbQO7Rh}OxnRwpo*Rm0 zcK@y@qw9e}p6`|CuWcq5!*L9$NS_B*LbF_(Oyi0cPt~<(NnJSzAm!H;7>BW&$`N5N zpN-is#!et8Z5HS%3z*df;Q&|(d+i$EssDet#*NYA|Cejrg3>nKRB-E~(0#Su z@BJz#fzd6P`{XEIHgj}kLH5I+AsqTnEb#b7<4><6EemoY6FW%qO{_l3R>^b#?sg*O zj9@m)50kX(9&0`_6OH)HEqFZ;wJ4Xgpm2e{(b|W|J8!);D@S&9KxE2DLBlVL2cvm5 zHrEFufR&$t(#k3_iZKZRR*>y2!rqkx{Z(pFVeMjvz}Q2+{vo*E_zhY>91?p<*(sbDQ^Jy7iskHla@Xz;BbC^eB?$p-(F}HGgttx%8liNg@#Iu541`FUE@h$U;aPf^7pY*m! zX}sbkO1=124rr!AnDrdK(AZg?*NKLVKtmpa;--^ll7`Lc9X+d0;po!u(Xn2EgY=Ye zcxOpzdGy&|5?XLPWxTVTSSsyq_tmN7plEg1659FiPH9xlzIAka$ZWN4`3Spl%ai$k z*<78Q&f5{1fKC0Y6wFf$SWG&^b^@f#md0h9(E*2Z;DwkTgKBV(h7a}V-O#+``+?r4 zdf%YtN>PSZUrRy)BjsvF+|T*0wwA&e_A&)~Ab+;`*d=3B-ijyXsm~xHa2W5)V1oUk_DtRSVD!pG!%~ z3g~j_5BGz(ZS+2hTX{9O#DxzVj_{Be$}Ynmu%KUmxQG7z>b>!s0#&VIzdsdn1p;2> zM{c0;$%ggEUTP#i5n)557x6s$RCkK)!z5$Gq2+Zv$|oO7|X#VO~+v)QRva#{ld zkg=43=|U@jO~9L{kODxkZ5eCN-*ladFyo-MhQ^kAe7+2gw$cEuy@6cLV%mq>*?jW5Le=a3jJAB~F*9u}l*EEEQM3=V1e<6DO^0 z0@$U?9MzJ1{r2k2)`Myw!YkuZJ}iy7f(JkPY5h~Jo3R#ceHotZ@My7 z1;L&FL?$1i zeaMa=XbYymi3-xt*Q?uNBbLm#<31($sF%TN2AZIS!sx~VH?3&e5H(&*p+HGV$( zuYlkrM*ep8h&nTLp#C#k&MEntv^@9|x)V8uapS8G-y_B|{3!$)yN@EbpwuhYWiQh8 zagZX80Ow@(t+*v&(b=U63tdaHCVShaCj zd>JrV?sUC$zPj}0lJ<3DzY){b4&VlYMq)hlBiT$h(h3sWi-l`9q%@8XucN@ z*5HMnW;VC+1BBg zaXvAgjB41E2dwwa)|Lyd+L~ggXiq`kzkH zr-K#k3U~sW3_h&q4$iT}C07A%CW3`ydi+OBL%r*wmD|66hRg+C?2l+AV!q#X`Ab<- zH$pKPGa|s>7hJw@d9)rFc+i)5^{V3@F}#E;N{ry$9*LH!x^_2{{k>R*G+MKBL$1R*DCtxx2?-RA?Yw zlw&b_G*E|wjtu7rXHtJqbtU?t{54T=Egz)&L7Qli^lGFCsA&Zu5xZ48+hIWaX6o{n z%awKzyS)1K0_M`~uadlNvP}N7+A)Ru-1<$T&e zqD=MfJf&U@4Lb-c%INz|>BsF(U3AWf9<+4axv#BfphqH+%a>sGz`Hj4(_$lOtiuzK zh*w#d#0$R%ZPZ#y>PSLQU)JUamK4&Y zOqUC#P-%fcMwuVCB2(f$SqXRgREu=ujcMm@1|Lm2e^O|H07(6nGnHAN|5D)wo?LIj zib(ve16G(noeP0rpE9chcXwWg*HmSHZB4y~vuOTx)Zb+*6}PE+a07~`fugV~?XxBgv+}ADWk1$i4=_^P`jnXVY)r3`?u2l; z>z?W31Til2Vm%Ld%r~y?tuM14j*! zGC#hI2{Zn?q5#jkdDfewgoECuOrf-%A)R{L16zf(02Wov(Iu^Cyv&|398{0?w!=|L zZ48wh5SS+pkr+C`ntlQ7>LyFM8c}?nzHE6`^H`HRgwKY zG;{UmiMS_+smukItObXAF>pI0BRs;=b>a$V`dKOM$3qPzjszYq1=e6Fq`|lcbxGh) zQuXY;m2iv-R4X?MN%t7PA+AKbu7CmH#^=Y~BD!UT=|{oddF0TWalT*Oc;h`bxbke< zTr0~9l^-QqL4}R$L@`>kzJRuPORRU!S83M-=Yt0at%h0GUfWRh$Nyozb)eyUvA(K5 z3;}kvCXgK=y(5PHU`lKk3P#aC{?L9#j_jko4i~EbbJ|@pF2EF`egof{+#6pa^q%KU z8+Z$fPD!vzw{*W&sO6U(4$1=ER&2rH;h-uRa~K5Fi1{qw4ASKz^weIzB*xVsBlmdAMsPyH-ZKvZ+&Qd zcnHRx^%`38=wRYLnI^NEMub9P)3gAv1u@P*dcR753#Ntj4n{yVgowT7$i^`F>cbf} z=8f}pk@I!eyV_37d*uO_mSMylD)g)LJtbs1*q6odOyZ~_;V{IZ^ zHDgb|F4{0ueCW(MD`ju5Y|?2)t2|==Swzzxw8msTK^F=Up-7bDai-*=mo6nJq{QCc z>UotQBJy3n#44OE7G;Z;thkATUR;*$st;~+N~tL{(@s~oSA4ddwi6W`i;*u={5 z)JFO(TB%~B`BKO0e_x~%Yy6XMelrazP^fyj(Pdq9hy3Z0+=T0t-^Gm$swnO!{`G4a z`DvMua5hk++f%Pg3e`?Zy5Hi64`^Thc1{js7?d^+%G?{LB|K<&EL;Bq)yH#Cq&r-A zjDw~K0e=%x|EOxd#ksNW1#MN>q|N`z{0FET;FS@dmXia&#p|t9Ysx=ALriM$&!0uL zUv_&L2>&uzJiqL%2|f?m`Bk_rx!$__*KwQ|pPV$SWsu?pA%4K$Q;OZivx(nK+XB7- zYHBvuZXX|rOp5FpE{n1}nh(6n4$ezD|GTAiNkW>zgjs(z!2@+AQVM>7bPOAL(X^x` za|gk2&UCZfNWKOF@|N=0Q-RqH$#;}QNRsrFXlUk*a{KYi!jPG@4id74Y#3LVH`$|i z{l8BxY=`TruZ?0M?H?DH`*B`r^L8myJVzyKe~nvuZI)jHe=75c96p@^U{(Zd=+DF6 z=qNR+9L$6w#$|~^_$F}B&1|;U7urj(7d<9VH16Qf$`(p5B)@H+md$ah4XzGIB;t8V03PgI&`_pT z83ts2PS?B7yf@A%aFUvnBQ`;L;3pV9W~(`1JV+E<1&zLEnzW-(^TX+oH=%}Ef7z>s zbr>>Om0ulZE7-kxivpW9&)7K_Y{o0kLy3LJdz*z@Ur-KnQcf7JTZ4EbJFL>fywckD z=vB?g3GhG$65pf(t&86|j8-px91WM6-R|HEEIxSdl>3?KsmVrv;@yNX=i@oZ1dlnI z)4&O1ylNf+i!1)2-(sJj1NJ(Az;C`&h7}qZ1wOq60?|5X6#X+?G-o))YCh=5>u47MX?N9nc=>=G+_#5 zA`GF&uVn$tA{$A-uPA4^mqlqw+~=Z!Z+<^u)olTdz!$54_!=QSpOe5lGv;C%KR7Y% zvt9~Uv*~T~r)^aHWNZsXsHKqsZG;BdxxLd?B0{>cjTWr*^YA$n@57fo~{`e$y4I}B}Q}-IU z8MayZAKp7^Qyut)#mhYZBBap*1c>B+`v@1q<9i9V~t`J4hSm_U`j#;7cC)TdpEbw9mC0J_%2h$uEy!ag-E{Id4 znyONT1oA}&U1>r-95*I(bUTR#FTHm+$X=c#$UglJp=y#dZ6fKiyal(MFhjqnM`^ym zi%=CHe?<9t-fiW@9E;`gr|Skx?(&7K98MuDhIiU{VqW-C4bMt-32^S~>O%P0m-AbX@V#D@-(~sV+CMdr<~H4hd%X-TZPWJ~V=pkeyzSo|Gy>B??P=CbD(j$F!>l{%aP+>KGOvE>1M9UgV_v z#}JsJYr5wh;O*#5iB!ushQ3fVBl`mkpUG*%9F?<oV2Da$n&6lMlTteR=aaf9O}ugFYk{%;mY5v$Luo2fC&t> z)Z3)0ldu`3crCeT8l6L?Ixc->=I09v!>BBhWTd5w=>yOrik3vGU)a#W8lZJFygK8410S;p z_~SyQJdDFf!L%2k=HJfidE|MD^^33E!Wy+|?kcnA>JKbu?L_j4NQxb7N*d+6>s7U%p?%k6txxXiRAdT3FXp?iHpc(vo)mNrOwohYY z>%hQIdjTU%Tl|^4`sbvt6aMK^Pm)=*@frj_`dj=3X-D*SoNb6{?Ys|1{n74xrk{`H z=8Gpr5><)Qk=I(J2jTkFfF<3DnxH0+gOrND4F~(Kc0`0&SI`1czQxp|&$C*BSrYy_ z2#gfsOhO6&QwNC(6-^{4O^1zV;$~c&b&MWbwMHV;1?-XGHjo(l^B}B&z6TOlp*Y4o zN6FW;M~RI3X3DNd*PYqrF=fa1{> zcean}T%D3dq>;|glDoNJmFM4mkbmVV!y7qu!K*{FwBLb?v3H`^T+nU$-WUc^r2A;j z6$57^s>G;5zaQ(z-VVk2StoTOV=0?tSn*boE|ss5(WEAqzANdi7RAgxJ*O-REKWqgSe33t2^M{i zkuckP!;nn%nzTR$Z_FE3{S8^b?EE4XG}K^Xzta2tVFj7aEZ>`LAK&?w747o7_S)sa zEnCh57+tN@62?bI5So5t=7h9V(qwm~Z#uVsO4sE=s5gKbi8#8Xfjl4ym&&Hc#l#djQ>qzRtlXEn2OxjCKZSe5Fq^J$%St4+1g1lio*N?osVPNl#tgR?8{}fSAT%&%(*%1@7KPldra}1IV_yLFF2*}3sD7bKPnEn;>f}`9@~4}usETh1Zv{){?fysa@eEYVB*%bK7(9(qBvF^p$~w^N zxL*XSAWVQ#knNa%Gv)6_-)brHdnjE)C$|%o&9<|#nRK30bwK%{nwD^(VR?=8pBmd8 z88l7!kn_xQ2hNhi^!Ibv&@6G*{a_Qdnlo{tsDdu(FdRnt+bhu-;$YfZK|yEv%3S>N z#ZJ+df}CA0srMX9o10$7tX``G?QBk_%tf!m#NJVKqtK(v%5Z_R1EiS`-SGGl_}sh* z%8<=$QSdT`M#BxkY}c}Q&=BFVbn&=e{nsku2n#BFZ+dL@h$Nl0T&fI@>QH3!3Hyd}k9%e1?iAL>8^y8qj{tp2&+PD6IOo?Yl zJq?VHusw#~*Mgsj;`%{H5I(YZRQYG$`7tgATA8XB?=9KShOAUBUi21ykhM1~)|Gcy z<%IuS%ll>&MlT+RfCg4ypVRvH{{G!nWutncCy6?*J+oh|PhdAH1gCb4-#@sPE5oIw z2CFhZu38-#dCrKwp4`0QFuQwuv~lqZYv9IQh|3*3TfpQ2gb8sn{>=r~6v7FQC0l2E zg%{5jPINzl=NR2-{T-rf3LiAK`7fSMmjGl(askm9iubhm)l7Qt(qC&IHSUaKickhw z(T3v!?N1?>ML-D1*=jw&X$$mOeKX8w_DcS%7>%=5TX2$l!VfAJZi1@mDcZZU5k9Gj zPqJjHdTC1{3HH00;q15Gthziy{YuE)$}<<#$eqjMdczk_^c>f=ald@=PRhf{%D47G zA5N<_y@pB$LE=1Jz?h;=+VoXsbtwT5uU2%rBh)?Mm{^L}_80NvCd{7HA+DjPX*Ha@ zLuFYHpvp$*Xk*h%+D;=u{f#78Tkh|g$>HbtD;68U0)PZ;CcNNM@TrH#DHw(2GK)6HD0n5?M*;r+*Q zm!gHc#_1jNP8tCb??frFz-EYt^;38D`!(tzPE1<`2!Hxq^yk{%uABo+fYgrzR+?zy z?}bJubv)tFuY7ldUy*`nAPeZWstccs+~PL0+P|ZEqwkPP zTPofVerG2sRjP8T>zxbL^9(i<{%)8=ZI#M*{b?&zlX~lYiQw6S{5Ya#)f|y9-Sb9O zgfL8b((so$1>#fwW*qi?wC$={g(LuAXK80$d*HCdxT)xpt6nD3fZgC4!#rD?m_vTk z0T;tBtW9oV@Yz}-RM4oz%PLN|Ya1{wA`kKG4a{#7u1a+`tKghiu8Z|PJvRT1ZlG--%cx!}SQW3ED2pGXPy}NZXsU_YI^UDn6tQKgK#Ua_W}09J)E~Jp%+Ue2kL`9nXbl*TEH;-f%|$%` zOy!WF84gk~A@3GG)LENTJB9e>=3zBu5hr+G{;1^F&5QLN{YDf5st#BzzweKKy|@p^ z3rGEzU{wHwte&werm^xn3Cm>bCqJ9)?8NcPiCx{PvzptcaJxU?pZpU|F@^kI^lScS zO=QG)&1St;gZ69|V_U#g`pn-%tLd{-=^Ist+Voz&t7`vf@bKi*jm3T-)9N{}8F8Iy z^@*}{V&}p-8Gu~c)?8jUDy@yBW`%Hd+g7>``{W>f_}g{~$``yHDrEeU++S9C9`Qu1 z%PEwV>`ABvjz{R9JPy3B~DH8V55{(J{w9*k(Gmtpj8+JeM zR+twNCx9(pd}Y0xAxhIH-A_7BJJS8hios^pMKD%$BY4w2t1sb;$GOj+^je2yp2=Nr zFb~-?M_h$oc>eg{B>_9q;=!YAQQxz(A1Oe)Zt%*ZeZ@#b`z9Gv#=c&W7M zL*8i4MAut@wbQZ_uavdFKJ-m$v{Cj|q&C3_M?jJ^rjys*fBeooB_jiL{j$-*zJK*d#qfJRnO3ls_7L zuz`-NZ!Wq}UzicJHh3(@Q>-`ANn=1V!p-5rAcHB1fKg^4v;4xo#5)qe%Oju^*e)ED zL1JGhxKj65#H-ToGU+hq5ujOgX%@0zN2i5iie%h2V?ntwuBe}=pI}C z7uZVXaCK^YTx3V^ER_2Z+Hxktrob3(FJ#-e(b*Mqvg6e3Y=QX~mS{}T@SEr=WA1VY zZU3fakB>&>FmYx-And)oY!jV9@V!qf&vy%Z*ibk!YkECqi$I%q04TGcZrHQep^wF- z9^(XbGqb|yCp$ZyN<8O1!-}o{fk>-@7*M9r=k0#f(->VfyYqhhXRNyY_?;`CFIr*!dR*Il&Au(SoOy&zcoF9^r7I;D8;z4 zZ%Tak1lA_*nbTbeB%r`^5iXbZ4+kU_4&w3N?h0kzzo4Lk7T@nCsK^|rc(DT7JEtSZ zG5Z){ug+GzXcC;zhU|Lu4((6AiW!+|aud{teuXEN^s;=Pe&88~Fzn zt)hdWhPD~rK?t;L3(k(y^T2XuZBIlh*n@@k6?Y_f;Om<+GIH z_Z&ntfPVB%-BKNfq;SAw3JI++Ts$l_*dJ-O<49ZR7_fahOe$%fppsj~tZ$?=*&EK^ zAU2G1m9J^SABGf1!mg-(E>lZ3tywc;r`Ns0vcUNBPWYi+SwVW&N0wkql z7$%+50ASm6ttaHWWomk)A_H2x92bh&!3*)ny-G)n(`-BG7GX?iU{f)A>;h2PB7HOe z$kUch=?3pebmRu{=I+zSA`9*Y4$d(qLOciks7FBGXu1e;JbU+UaXZivYLV%rTC{Y2 zeC{pT1`pVMuC!oiHVb}`H`PfvU7}2NG8mMOJ9m7E=pZ*}+g6)>9#c_U9djdd(aZW` zxow~9Rk_H;B ziWj##?B`NNAV($TS6){uygSpqFE|JR2|;=1*yIw9(cOz>Tn*g2wu1bg$J@O_Ban6P zMUtg?g3GU2kI+Y{s9Y2m{ zmr3y=NDiGA+E&?X(vJ&04|v3&9@hv=wr9XFwHHZ{`PbKd(hj)B4XdV{%c@!dnJwTSVv&H8zc zqmup*hW_e;ZG+nV>kj<}sK)AEk?IpLhQN>MI60LSuz9LyJLLpX46F2+EIzPXj)e7= z4xI%=z;yQcKR|--yV1?+lR#=^Q7|fT4t+ZUe~Khb+$ON9*hICwtdc4F6GN5@#y#+{ zqJKXMzsZveRtPd?PxOloR=-7Q^SfQM6jGJ@68^&>Lj5DBzK7)l(6Y*%mn{1IK&zKy zD>zsTj0KqW<7Iwa{tj3!Yhg%vB8PFPdeuZn2=wtx`u(ysq-BfAiKY~O$eiyg;3T^- zUOzzNTnvi=bcPQUe>!HVQ*esIA9|;E`pKSne)6x|wq(gUHZ~x{blw_Xtn;6wb8r}t zeiSj5Thz;0W1;H& za~&gUbjhzQGN5A|pCV&d599=;1=MmuS0k14NhTzlH;Cta(OPdb0#=D$+1 z($;qdPEgX-d;Ex~!yGHd9^CzRh3topjnn$#tug6v;f9Zr%Me z2+y5)1=OAGxv)=tx;_>BAcps*g44ZCmx)n*(ZY;tBA))`5yC>50fcf1iG|Jv;Cc!K zFdih6#g(s?b=cy643cB1*o>eZ`)whFPr5t9MMQMh8z;@BU~aZ5PL@aT&u z_31{V*PXz(nW_|x!20`so%C%g0AM4j&r5{5@q-F$kzd^KGC7Rp15uQiVr|K2f8}L7 zVjT3DV+7`77=v(F1(bWgjqw_W7zof9UtGLbsi7q-A< z!C6q%>eeI$!EZXH*6&KuUKE7t7^j?_TIPTH=;Esixc$R(fMM1AwHTIe$uq2i4nn~yr*bFkF`8uw zufabt8Kd`1)kWg4Qz#AX#u$H7eM&5fAs2eHqXfXw8#U;{HwcEv!>Mz9@Q1?;Jjy+c z8c9b>fF&e%S+B_YNZX@fnTL0L8yzBGyOJFIZ!&qZne+vxchdzH)rN6QlAOMS!ir4x zANrY;OfLQ+!%qfIInC<>^$6)k_HV7)-iX0_UXyu|aFOLLFFE;SNui}cA01fqTwCxo z@JWx#SZIKFPS)mdY%&7MNuSI*whmac?jDtNHZ}mhdW8!zH>6{w99>$*fW>0kWzR=D zv8>M%*slJks2}pqNJeFXBJm9af z#%<@=SW2+PD6M=Bmhc9!hn{Eh-strv?)g>cW5t?kU%JMU%%?cB4Pl%UxXq%`jEb(& zUeVn#oZ00Whyg*U@UEi#FUiU4^b`kxxtHsEXtjA@2y|=MtUq636&A~&rBrK771BTP z6Tr<7Y&;jW+?2O`^e_H0f2}dUelrYb2!szjTks$4vG*0^XoK9hafIw&H@pU4o_cMD%@pprVT+x5_ z{r2YigF^M=F~RsOU~-T%`8ukRtCyDG@!f}XmQ(j-OYvx7kEvOB$uwjB#F?(4Yrk^$ zd0G((iYW*`12y}-8NKnA%zC~0TGLzxwHN*2CNC}!dOWIAIwLjSD|Q>dFx9r;JgbKM zZ*Pi7pc+ZLhR)A`nJ5n?aFagKf}5tB4$f|57%#*~{)8trVSnuUWTL`Vu9`a5Z0~r(CD10|q zB<`^skDdX$942>*BH}UeDwh`}K(aJis0zvY_G)q^tx_t~c09xu>?L!uCBccb9Z&!A zdc$nvA@5IhIUXYG;G8I;A>v_`Z7@9Fx!128j7_#nJ>`G-&SF03$!0qpxdA|Brv$iu z{oWb7x+ju-26#ElRQ1~4Mn$+<>?&rlRAl@ldwQ{y2djl<-u-=H+w-ENEU}KYz!Vv! z4wmD&c-1CSnDvFDf>-1_5USKtvK9XNVNB&1rV2=uayRsshmQ@LTu%Jd`}uJNf@cf;?0hf?sxlB!=c_p&60{aSceW8vyPw6y=->)}znpy;r# z%A2S|p!Ei<$-IL!jtSf3$uI?yXF!zhRZ}e{2?b^-Y3sTz%S@K)LvdnyBRq&W;px|B z-{MD?)OS>r<2X_AN}Z%{!HSA(e%<1KbY+txi9Oeh8jee&MeN`b4AHqM2VeakG#Zg9 ziRA>2Sh`I<&zB}vH~Y7efsZ5;_Pt00Epg{?Zk+ilN|nv@)5UcdKV%OT^vS1?>SuPy zHD=F+mhIUZQ*iY6)RBE=_CzWV!(XC0ss+{7`po-`O@beYH1NK*k2<&P+HXFRlgvGzWG)VQWBHc1sR(rOl&XD_^7iO^msk6)1DvZn2@ZB4M~heqGCL z(6OLV@{6~utES47n!rMVBNu|7z5ktnIzdt?=^4MHjlpd+srG$_*T>alx^HZC4o$VG z(e9>7x41>Q0;q%?ofjJEhamwtLjm0;_h8$Cmw)`|bD@A>t32Er0jvH4htvK7xCvu; zgqhO>S*?6=5r>8T_B&FcB{L8C<23Uu8O1xp+1TxQ5QnY4 z#oYk`MH{R4n&rikSXQ~a)|QV8Ni$?C?P~hWMaB!uH1X81tX#ure zfTiE6y2J! z1hYH_ph5E$d@TR9D%7)b=6}8qpRojO(c=Ngx#yPM`)`FZ-+r9e1j57YevP(XwtV{Z zV5B#|y^pV6yW(G7LN}@%34-;5G0Wq?#@h(v#q64UotP{r!jhdGGaGnngPkAGj8zES zGik~C8;X3mn#5!D*qniwl#pfh_8+BA)Rh+g2NXylgTE{UOWou4!?3_A1_fn^KX_oN z)od)6{b}!R!cPY+bw&0_tx8053kr>R$QR9D+477Z5kXx%A&@0tg%1qlYJzq{@cVy9zZTZEb;{4lY>R z4@~W@c4sagSd%x1DscXe0Qxrf@~SW5IA7QW=V$QZn{iKLRb!7F-ST}lo|5{{HjdVgd}VMYts+>H=|kHT3&1YW^XB|{~N z$+)lVnXMZJ4w)NbN7(1@3LM#Wxs~1RxIqh_INiq6Ka}2}HwUXEo9&v*mynuzxKn4Q zv3*d|Nok^ZVHq{&H`^ zSSH@{UBNe9SKQo6xAqYB8%!-dK)c{rywPoM?r<>lff5X2I_ITjCX{JMSX8%|>S1-- z`}}1KE!V?5yWoGsugbeJJb z*zaXXB~mFwf=~t$3|=N{`6>7`o-|=;Y873YZ_fTa#;2xNafoxDu=|}y_zd`_N2uyE ziptjzn9a#8doNE%kRxtrUMH}Xh3SH5BItI{rP(a?@u_N0?DI?u;2bLiFl#;m@j6It z-Jqo#pcNZ<;|3tXlfo&m3ab@*QZ2ni%9UC2t1QUD6ej086~up=fD+<+JKd0SLi(XT z^&CUC$34Oo?0;m9`*>CB&Xmvf?N{*CInptF-r>qUA_bVOnqE%N9r;lQ;@81wMa&*Dl18uZnX1qfdP1n-JY zkDIFFuNLT1&K}8-r47^qDJFW0AIIPq-b+oaRM?!e+oQmu2;2;tOSIQ3<2C1X$x|jD zsapR&B~{Q>`IkLiNQcp`N*My|Hg5lruQo=WLwO7CU(2UvAn%^zzx&tmI_fiuoQxvtSV)pW-fhfn)sMy!{{ z@Ag`V@!iI7StT92b>c%lxh{R(A75FUC>#$gG5Q6^w^G(`i4BDOg^=_ZPylL30(wrE z?vs>wTYC7fuO$nXqqGV^gNTjWi&^tCcYr{Nl=*&^y&jUbsb$oidY?3T)2o$F;XHof zUTxPc;Z{YT^A34acI-vba82HE+pegjfnZeo#TV;cV0MCPZv}>6dZ&@-z3fJ>^ptAj zG{L1(O7YayNV)p`ok4?74Jw@_2lt|%N9YZS?@5yp-=sN1dWSjXjMOa~npHA`dAfYx z-`m13Fh<|ydsK02tLw`P9%`*hT;UYqK)i64R26PjVr5goW!8?VTjLDLU!%!~dslb8 zJJiCGQ&J%8DaI5LeL>Ew8-G3{(g?)Pl^e%g8BZ>aV@!>EN2H#p5(!voGdW0A-D?I+ zGL+6_5$_3&#itJBY-Mh*u1wjtMT4J!Q5PaCEM0HC#OU zv*v;K57u5R-40ym{fbu9Gc5RtY&@B33>4U;50nrCS+i9jzvoa;v%{TEoi(~yx`rj= z4N(7nf@mYx%*Od+zJnd;fN%7Kh2Pae4_jKf~e! z3HDU$xiUJvl#M_Mr8EM!pO-7%Q=%dqgB^B4)!x(=6#ESRu_$RwBW%?H>PRfg^Wo0m zjiY{O4|&Ql9-xefqIk?eIi}x>otsFecdv(Alm+7~gcTdp%NyigcoioT^B=Tgv^ctI z2G~Ef6wdhrO~#TZe|6&5w+X-eqdgYVhKsGr^M0Kb!Bm<{C_eOC!Xm{+;Sdt+JM>yc zh7Mv{#UKNO-{zL*-aN|g_sVINub@|Z&S$6E@b;-js*#>0hM$f`De#FcA$3m`#Kqtf zs=!wR+~{fxAg7HqmmuR6+-tBg=oB>XDK975f{Z{Zeo&N<(?swip{o_Dn~Bg+nXp_2 zkuu1eXh$FI3@(a7q1qUrhjUIU29k!Z_f$n|1&JuU|ASqYlK&PCZ<@7Q`RmnFQs_qI zzx}p9p8l5TLtX#WRuIUK-E!;dTx%aIN%4E|m0`$@lGa_64X#w>r2NcvRbb&nCTsTc|eQ8;9JlwW*CNa>}7@8`pnxxDfFO zrTQS!K{@0@|AOKDbrJmm9Q4#R; zewm#QTw7D5O$3Mp^~D>tSSmJ$vE;YGj7_e|$cB7*0A=c?~d)&zl3(|mqQ z8H=3(kDg8L4U|KPU-WbUxnZ4eF2@Jy+i`hEcc0``N+f$=x%td9W{({neHDz-X0G;6 z`Y!tZTWKtxjzx|`YrR1~JJ&uX2ULQj4~tadS>-V~8M%~Pt(b;1YFU{h?x6=)jDJ+| z-J0B_e`Nr`yL$P@eUtU)dExfyWcyT2Q`@Z*5l`737bKE%Tm0rp23Xz|eiSs%bB`GX z-xstiG2ZCkzTWFe&ICXA7U5%+V^THOge{`+>u(kwC~H#P(EBt5ckN+y?O^r0MD_CZ*Xt+C z;hXj@9i&=)O2bYl@Z^Qh^9gkS9xeUiR(NZ1_4o_X=DTU$LoARHx`W+@-jQ8AlHqft z(Yz*A908p|0@ZTX7sIkIMk=Gu$%?W1cRw7-;nNn#aB#8T%r~*Ui)5;idC$sLvG<@! zLQeeJYL24rFNrchF@kmMUHEGzQlq!rIC!_~_q_^7e+)6y($me^)Ut9rY0#x5j%5+_ zaod5J>X#T&)$xg8j_@?Mu^$`uLnaqIg+GBvlH3%Tjb7zRz`sV z;2%k5z%s0paveAbl^Fi$J1?p(WM{Jho(1|*!062z`A2=lNv1ZgJe zp1F(|ThyF}cRrN<9(*XS`i8&Qps9p8DihZ}kk6pl9bCK)EY00He~(IWLlo}iDdg(B ziBq-{%x1H;ZHc+2@?j;m<8@pnN;(rwngI>lkDppJM=sJ*8=pn)dv0rWmcHlP2o`7k z(0wSdz8f*=cG3v^n%4JHtRkkG*Q7)((v+E+I&heIR2s8CvX(39%d2R6 z{~Gv42Dom5pU6GucA>A3S6pev;N76t;$$+%c9$@W8qOVt8WoQjfy2P&R1{2F)z5K0 zmU-?s!IsiP(&L1b2#VOcCnFyTw?|k~0|uFf6KZ?aiAtr*Kk1vr@UsdpujUmyz4Uv= zxS@XIR{MecUxl$^{L@cdq&Tt;s2>B*eQ7xm8qnN=6oZeSo6vVuhal>wtZbH#2#wu& z8SloLEuR<^@0F1_rdQ(`G?+af%d=)PtvisKB-Vggk);lbXM}&ttA3Ve@1DUftOm4C z4(Yx5j%Zd93TWT?zZoh}-h0el_kT#|I?h_5DCjKw_sI*uqUV!hmJ2vHsA_o|P3o3Z z0iIR%i-j?8`1!kwYZN8=N&>bCF+$%xXY*0?VJgKFOtf+SG%R51@+tm8(Sw%nBSi`Q zGgH#|%?P03c~^)n|GGo_vpyFp=py`9r>r@RJKm&OQanZUV)Y+QqM5D7%kj!^ zNvpc;_LRe6?R+JW1m=QAkwG5!IWz+Ak~b|q|pf( zW}ti#9Xr9ts#v2HRrbB(lTe(#ftA+GabKmOT=%;FUN)QYGi8Q5usDwgcPwzGZeEE? zxc`$D@URF8TPPb%T}uy}zBv{JeeTOjTM*ZEa%8}Mpt6Cc#Ndv~e_{?wpR-DUNm=ES z>X1#3&0_90m0=WLL_60@vEdq-dsEnVS~d8obkTp1sZJ% z1dIDwhek*x)DamC5t7jwEQ1z|Hl&2?jU?F36sNVc92Ca5`a+3T6!<{6eO?FE>(^|}8irbM5fT@UHvFP3&xVPbqK17X8dbVXc zOLpU*l^jR%tZ)!-Y7Xr-nvB(jRAe@J(T+X8h%GElx!rVrb5y6PN#*(-7-SDGAU(@WlvEM8)I-8KY7mVf#{6Hhi z{DCrBi?-yxb_>?2*xjpAI=m0;kef&<9aUtM_k65&c?!91yaqI#S6S;NMP|bYhqn>X z;!|I$xsvCig40)HPWv`;0LYn;5Rxv=RAoN9?$Qmz4IMlx8z&ydXO`IfrIOiqB<;sN zshQ5FmF*u|Svo>Ml`SCM`1P}8u?%gj+mn#b0x$Kq^F=Mf7*vu{(dpqV;m^egt#-Oc z8|sN&o!`{!*D(|>!X{*}jh23nxAi>?EXlZS0suUKHNI^RCGplA7hq1eJ-@~c`4rkPVRbMyjkeR{>A2RgvyjXgf)c6HhT>_Heev*xU#PP3d8jy#()CglLFA;4KgBr2oq|$aFY2bb!!EH{zvrH^J`Nb>{ zqU;J=4%90F>ky|^oldH=hN)wfAoz@4y_x|$O?F17=HI)FR0LQb7mJ8$HgX{A6P;r- z0ZWiw0j`!R5J-b+m*VYEW>&kFAg~vSTn_fV5@5uxPdLqFzlv_ZXTf8Qa8^~j0(+Vg z7Z5LpW(Q2wM|514_}i?qm4s~by0Jnf7uOAw?{u%0i+lD0tt1LftD-B71n$4aP%}^p zv2W8wW~EkJdlG_3qwYjWZPwhc9UMu?>VrrQ+J+hxt7!of#a4s1Qs5s4 z#qwF^Q5z%TWYPQsVDWTq21qWnlEYsiL7E@uxP)9ftCbsrS>7HU=q6Vs@D=aKI=wRI zF{bYhHHka%UBtd`_vcsTOTFeqDtw7r#$PL^hER$@=`~&`P3{ zedy>7`Fmis$7gB59EeaH7p5{{C7wj+n8xg-*z!M!*nV=4^Ow{zSfkcb*1YudHL4GX zAQJU~KJG`8%MLMgHsA4ag=X6B6=vCKsM&7g+iLxO{y_Gp&^vvvd+*xlGD{Wd!bSP| zC4yxNgXAvkn=4H&zm4k1zQ1gGkiz=##^#T27Ujl?rPSkJ_P^>SaF-MMcgO#SDG}1% z6UHL;AFM=JR&z`Lf9$8sORRqKtSM)c{~!CQGhXC1CGgPU10FifK4N!W0n;CQ$nkrR zS%^2RGn@(T zYdc{r9yC__4>j34f>hi0CO(qhNf9u%jQQs--32)t4r*;AIQSntW^io+#=?BUhbjX1 zz|L|XR8ZDe!H4sgx~C{$vBS;;{@wF13}m9W8#;BkSm3;sXe>63T#R&%?Eo)?pV|@n z4~uY|;zdzYVx$sJd3+xIex9d*3-~3+3{q=(3v?3O6vd{s98irTa8bum(05-?l(Roz z*Yv&)aHKV&#$@W3VAYlHFS`nFJM3{9<=Vui*HnExJw_j@s<~jj_>of7f zL5&I%G(+(0oMwSTd9_$56ZWlb1i`(t)CE=Yb6E3f>5|zHpMYyfB=KXcR^)8?eFmk* zC`mTEpLUeldw|1Xy#~~g5j;<I8>NJ?I`k)Uh#Zkt&2+cEN!Po+Fo`vlRqRl14uhkRnE<}z34-GbaFS6h z9Q=M@=koP7ACZ!Q&W;103wC|k>22)^6FyFODOVOg7recwyJxcfr%Ju{H}CC5#lbP_ znCnNhgpK9(t~8O~zU-a!=~!KF`<)?a*Mv*SW=Eh8qy|X_r})f*Y`=osZP-Rfpz09~ z`F)g&B-kN5CStO);s;ndovuQxOG~n9ut*PGKh8|%AMv=Giq!=~%Ce+%!ab#ROoNZ? zUXj%4^pl5w?^RtI8H9c+71Y=#{Q8KNoFM5rfgHtblIjJ%s$9HC-Sg{!cPi&hLO}u# zz9DJDGk|D3X`~L}%TcJDz%GniIc#ndVv4hiyeje*oBK|;yM@I*4PhazgxOM>2w%KE zRL!x^&*&c-C+W+vjUjlF`mNN~;^9KXo(azOh-bz^>mbA7`OqEVZ*C({F%BKDd3R$h zuxfn+K&oMNqNtwPtAkp%b$EmZ3=lC9j=g7s0yVeU*5u*V3)@KZ#2Nn)*--WvI&aCud5@Eqvu7Z%vFgAz zNg;GQbiOoLdn4mHY0ui6ck8=>t5U|Fp`bmKsnxcH z1d0|qX#P_&PL&yLB^PHH@|Jodg*S{Rw^6U%A!Fd~ToW_z3-5-&`T?iBdt{L(i9Hs) zSKXixw|grtOqE|FQJMpgf;?~?64uoB08y#-gOkxDVm4&sQs9r*QUuU7_aVt2rm?Xx z;z>EI{F{Z#Z;FiVkqjF}{|Aa7mam~K){uS)z3z1Q{73fc-8Z=bwM3_3sk?-KLSU2| zsOGt$&PL{MZeMdJ&0qUa=bkm@0W{r@ajF}hxTvO(ZXka;^X}eX+Q!Rji-Ltu%Lmsh z2fQcigs>GX`xU^Yp-|*m&D{UGu8Nhq4eQhzg(qT#ZKIA$(JK^rl=Yq#l1CboYVV7qE)acNu3@mFh z9P0Md1t}Sp4Kpwv1;Lx=yJv(Y(8%Xt6_n(}>#rTdZ(?IuqjO?b!s}9P$#Q_`UQW&y zZ=!j(-wGz2QnKDHzuqmam65{iaD%Rf^$Tnt18{P^`zTZ4(7bHF+5QT+ai3tu{xBHw6n*U`1 zfEsz~U+O$j@gX3i!r%H{*yGG0UNS(zK=F*U8?N*2c7BCp5L2vMm$F6z%HmV}0B;Pa zduu^eN)ctI3V*4=aF=|R`#d%WJ5FRJ=LDOYZM#x~n&-FTXz)`hVlu_bI-TRP4k{`H=v)v0p3}?uDV$sN#_Ib<1FxC~arNqDXL!3zA z1YhHT<(wd{nfvQ9kKe$pq6>rkW==iN)f=~5R%YX#;${SdzfQGIQ(>k>kv%EElb$9m z>rZ{TUGn^UsLWT#oa$%Ibda~~U=+a~0uJ@QN4Peeuk7L1G?~Y^w#?$yyz)=60KDSL z-!}LE8Ed88|Btbjezx_KRxe)JP2WrVK z%EJ14&n&m%W}j6i!Z^$mouQOdbe~b5HH|xHM&Wm2qnM^sx1_`)kMR0HerlVh)yX3= z%xP~!@$h-x9-Sw80Mvvx#<)Y95UiqgE#+$Bxx~q++h3ItE+f#fw`Vl?f<#hUSS3Y1 zXs5gAL&8irbfTXnPf#ix+MLD~&zk>srj7*f$6O z4eT6S&yg!oOHh_OBC38b?QTVVgoT)VH!LfE3cNbC+Q0Tgc|t6m!A^G6HDJ#H+z(Uv zC&Q0ZVHLD=C%7n2y8f8r`G{_ArDAW+o0kLrCQVP%xZ=`2J3)Iy901v?3&o~3I~VzZ zN*r+is*tS_Y7f{lL;EC;UQ!w-I~TyZ?HH8PhZ0@*s*j$a9;Hl5JS9RP{Iw{HeuqC@ z$o@ShvFu;<`s}P=`1n6Ur}uwRA^)7ONA~VX7O~>P|J6AciCdBXFO`#t8QXqq+NHuh zxCZdVeep53WAvbMd%K!|2nm4$d)A)`19LeYKS(Tpm9awcb2Wmz@9w`~b0?c;#PZR^wOUXa(hs zyBxKO)iV@dVdbzx%T>ZnB&%H5r98{LY-a%XrF}k%-I%{2OW%h7d-kITNGRS9JW3Rs z;oukjQ5xYf!#BY`g4oYU$3e3b>e1|?WSXD*%IMA(#x6jBLiBbkR{x)V@6Xjddh_Lc zq}oy3O`+H`gWX#WxN5;vuOEA?Vxs&=XqC1*w;wn9bHl3Hg+p-v(jPc%&9Qcttk1To z+*DEPZfG1vF_hUS9L-BO+g;7a*;dMq%$5Es{!47)P-RX#Rvn;EbPBl`UxJVls3(Y2 z8|+u(=ky`obW!*jtqscBovMWro7{~pWGnXUyWbdiN({1IdNArk$*9;tSm(wuZ#m$j zQo9)UMb`G&De4{5{g~%uPLZasRn0LtVA2$~y`Iby)oS(6#pZ!-4@I>3_>tkkVZ}9yA@MpS=UJE39wFCMP>c~V&K;DAlVevC_BP;+>aol4Wa!cZo-zJnmFoRl)S;{yg7% zE>GR8E}V8|hJkC944lA&yVF`KdCFJ@u3_z{_`cWI3Jh%k`< zcek4L04yTy$X()jkhb|QLH7si?{(KM-y~kRx7hRkEp-sVuIbL2a>g+I8DgZUn~|x_ zEf<0-jyk0`u?Bp`ItY5=9P7MQ*dTJ?tKB7A3RfYn!bXx!_9JMdV?Tp_z}YR(e8%ss z66vdzL&8RSwki2n-tWBMik^0rv@ZNkmNl6NWE(8ldp)-e=F8z{=X+e(aGLvpiM+VekRW*58hWl>pF^0*40ezo0wHsew9 zRMUWn=Zp%KeT=D&nGFe-Nj_i~ryznYq@Z;_4FHDA4s6En5CiBrzkKYHFn{T6IMiAh-*8?QfCL0_YQ->`-CVlHOn3Sl5ZUUxvXS9 zBwGV7_W4huizfe#`mQbxGm$6Z8LieEM=RnUg)&16WRYK0HVC||EnL4|0ri~t$cl#X zhq0c1Dzgie(-&gkS3k|&GG8e^#2BUsqe_U0u8g0(s4~;EQiZ@oHX$!4RRm0Nf#o<~ z{EVKT@1y#eJ)mD;995hlb?Rot+_ty&)yg;ru(RF^+vim3)3pZ#$8TUPcimZWr*uZ= zY~u&WOQOWj;V@}S*M>(Pe#bN&9QU#eiAbT#XTcxpf*XH}ccc5J`=a!q$s>6-uU#gz z&Zw34ar$fblzfceyq5@@n(Dde-MdFo3(|Y}Ru#EBpfe_h%tYJ`vsL#A0R#QD`q%ub>K%2?`8sCkI zEDp#u@St)fPfY2&&Y{#(Mw9V0xntLa-@hsEzV|(L>@vovm>jna1|EeYQXP^BH5ovZ zo%h((gZ30&QMkzVs3?m^`pVY#v9{JK@8-K>dy&%bv$GvWgwhl~J9x5r`^iUX-E*M4m#nha=DLT`2kc6*#*m8dla_Xb zh~Y3QHUr;XBiH)`AtgO#8|6CEQ;B|;(5_lQV}c`kM_bLKHKDElF#u6<^L4Z*TXa}( z-Dnv_GpCq*zxt=VzgGKuc1`hI#lfft*JAZ~IFXvFsO^v!%MnWzCE$A*aP1g$bvN|# zo)}Q8JWXteNLNWHNeC#9XzrUgn{i{$h>@WBmZedXr8hVlR3RC)3h(#6^U|lvG|{2J zp0RK9!G<#ZTVKIgD#DZ`tu2U9hSAbdvWBj~NS`JgrYRw4^Aw>!oi+w_{HYQrGgIi; zMTzhY5*jBW!nDuVepKZqX|H(t%knwMAovrzsGq$0G~}5<<8LxJtUI>T$2(u(7W*2B z&;op`YLT@WCD7H;nJ`zRHB1O7YV&?Wc(`XNHF2`i3b2PfszfHrr_(@ap6OEv{8cb` z(vmFa=x|GZp5lyhxL!vq5uF4YZ%Z&D?bCmi!1)FtvTu2c^lpgKQfR*;el?P*SRJ;`-hAs1!xJGe!DB8W7`y~d5OKu9ue8a)9y=;*d zh|)U=$^@?tU&>6w{lsVok19{aBTv~2WPLwhebfJBE!1}d$k>$o@9M6hOA8O+ch+A6 zd(DHfJSWg1eszQ3$!vFCCot;)8|fQy0C2}?f?56ZL9qD5QPJmm$1^s4DZo@&vf+*1 z9pR||J@`e zh)Jg`b41_s=;fz#Jk=&#K-9~rDNFHRWXb;&Qh7r2xbT2IujdzQH~oE`aGA4rTAewc z?Dyh&S`{LGhC}1MTZVP2sWv!_U3LoZgxh3e5e@lX`U}8_f{5J|jF5z?G%EP?-3lMz z*7BtwKu9jNRxJ{ljAEB2;@E!ni1suo$Oyea;1X0Zfkub{zb?G~8^&~tf6F}%IKA&e zV1YlJ3LNHdRu}1kE+cgXAl727#cshJE_Wlad<&jaLCIgox0ibDtr8a+v1cODZ(o#D zCZ@cZyG9(`#mp6gntcS{&#Ho1^LPwlHtnk2 zhz)5H(3wX5o>*MZNV*eEig{W3T6Safdc38*UL35RGR9J?k2>!CGin3c3pWpoSlDsu zUj*b5eev(mYWTHu+cd`!M|&SulO>x-7{TKfKhy8OO>nd72{NO=yUl-{?<+0x|V*cObq zS?OL zlqk7Jw$U86Jk7oJ{d}p}3$gC=%CNvRav~yj+bA6ymF9SN>=)5*euUs+Gw`q86OTM+ zWWezf0ASVmBT(XWJax1McZYR*`c++4iD5E^gsy#yH|3tq##36Y6fAQEE)?-Tmh}Az z2N2cgfPx%*rRzlJl{*cGaa!I?9n8zI_`||jC`h3;`!b;S9 zeUDB4e#oRYXd~p^!&z+paU+4JublsEX$IB3aye2uZSlx#&lY9uL%6bG{^0WDbQlXh`RHgcq{I=o;rGs1H0Hf3Pd%A_17fH_BdC{MwWA|l`9#Q4M{pJys%vQTfUBLj+ zn*X+irtK^5wPYd@^ZvHkLFD>>Xx7dok%J_k*H6$(Y|0;;Wh-Rm9-Ww+!PeP5PI(kd zI1JLbio+Uz(f2#BQ9VB7!HISR_j~IT@VI?Xr@TS7;B$?DCa|w`rz z2^S`2k0;vQC+UHkXj!_U>y>er?Jl%o$Zugw_ML19kof*O&p3Mz`@cq2$|lw7sP}F4 zUp1GGO14)Ad}Avuv&QvTsP`TJ=M6VZ6C^z@D|g;t(kqSr-^S7O{}`S+As`wx zR$H*2@KQbe+&3)%&F(D)+IZRsnlspV+>Lk5qf(vD(5t_UO{`>!(-r*a=8s6LRzE5E zGz9;JuM@wq2(iTo4W-Kg_EMY0gWFCYaFRvaa*!}2J?$E!96*hQs==TfLeX#@qN#o&kWCM8abN$(MP*6N`_ku1)vGs zziL-~`2xt>Ta~6F5`ZqE+UR3i(DzPh9{De3=(sAg1MuME`$-b-VpJN@%0R6kbBSa%7?1XU3$iUHC$Kbx|V2hS`ld0K`H`fW=7MSVb$rD8*B`=L4)FI zpDD!!+kNYaV~oz;v?j^2X=<}RW4HcPRoHpCBiA*ty`1uN=i%wU;Kdt7+-fV$7xRs! zPg}Y4p5)2EsW(r>+&cLS-%4{qWIAMDlbo3@pM-aMethiP9WxBMkNQFjx8&1*LmvA= zG`=Zno-l4t?=41Ki)Q$>vT@7u%(67`Sw51NQrX-OHz}k@?qbeqMSgIAto4=zbwH0M4Pkz_0D&G z8xORD?9Ii!4Ru}GbwSu3Ve)bz7FS(yQFeBel*lVeJ+X(t!W~U=88)Uw;QbLEn`ROkdDvHia>uS~N-ZCxl1zc3*|Hzj&U!P~`~$SV5y4fdW_SXq z$>C>(LS64u_qyp>=Z@hc(0Y7#;12;kms0H_=O;?D z$ClG?&&i^fK<^nA{PtGx1+egvsA-ZLE(5S1yV|t>Z?<7p112wkS6rFGQYtXe9l@V#4#dTjo;#HHgy{W|#oe&s3|l z{4DYv=x<(#;xHS0mi|x$rS?HfML+^#@TBn=C>1nAjOQi7qU&iVpHO1jTtJ%BuIbKa z{BT&HTpLYQA$LjFwnrlDw0-XU%~d_}2>Pq{zr)sSOiv%)A4uWbNjCt2W!Y0&OK^9= zz6puxF^qgZgCKX-fX-<%n4ZW*8r?UNEO6KwD|xA32WN-rv;Al+er)>DB2(FgLrS8D z!`m_|5*W1J7YzIm+s9I0`OB4rUvu#Mhb!q!hBZiH_GX^NYa61_n~^>w)s(<%S}e%u zXcnJ^#C7+Uf!3vN6@Ot}TgtWlDzy_wGW%}X=M^84YU{}&4=A1A^LS?>tPxq&Z?_4ifOqBo#wfJxW3er#;fZj$@0SM( z-#h#Wxm8{#Z667;D%CUCB!HeK*}g3M5xvZ#w>8G&Ee#epKoMLt0m`@c0;S~ACUW=V z2OA+}GQejg2SBX{kfIdqP#4QxXi%aLn8atq=jI!|-xyQVfh5aTzVLmuO5+cB*LDS4 zQkzI=hSSb<)lPV!#3r)NQBkKFDZhEHwlKNeGV%jJYn2i*y@J}XG&(B#s>#!|V;&^* z1hJb{fxiBEXkUW(vI56Y{#a-RSho}dMv4r~dU!xqZ7lO!SO}+?B0B0-ft~i3L|C>6(f1$uQsI#ify+rfg=k(nTWx-J>TMl-XhR(J`5%DgNSS#wfao*Q=d(`&g=#RE2Po-^qSdQ)63o<=jKi(B$6M-bsoh^rB&fKXgVOR3lh2l z%zJl#v;5=g4=3E2{B$eqGAuFcFShnNC?NUidJ$_zrAk3mjn+7a6SOW#O=4by%k0V(!$nZJiBORK-1Ji5(+d86z+FTHFr$3H_Qm$* zJ*BW}DmnWt$Tjw*h{Pu3dR#1?5?N#nF29Siczt9zDaB{(pU$UTt~C^4)d`tU3!+ie zFykKNGZ$zWeAnT}e^1EzM`q1w2r#1qy({7ifSK}!`n+oCFO|9{jhzjp0J%mqKXC@+ zWls04LK@r&scEgk=-)yi9A=8|j^Qx#1lYsy(6=4-t-K{*I5Z8|ot8kExJ^Oj0I#oE zpRPP{4$Wo@GEtu=?$t|xT@ctwn3QQCxPUuM4B+$$*)N(rli)MiKI(VR9A}qp`vHFr zZ1c@j06^~l?2;;hKA;b~*DH!#Vkny=kD^*rr*l`BxtZ+(0kxEP(<=pYl_X>N!p5*2 z*bOPjxj`oY08NwPr4$x%zz_vc?(_QyB5-`XR+#%)lUpmeV1%6NP8~1c?e9ta_28SM z+Jarw4=wbPfU%=vt{Qi!*%E%+JeS`|rAqeol%>V1Pl!t(7s4%yK^w7-i-#T1hug&q z5O-ve8%K4n0Jc(HVQh{4qwj9^PHYC}Yr!vv&@M1Rr6Z!W>bba&!K1BURBAfHxY~}{ zTtWUF|0jnBKX{YX>*S|8{Bs$P&|5+6W67!`Xxcjae3|XeQCfgsyHEW$e#dJw?DyOp zfd8!O{u@S?{%06@VkzPq1r?xm{bz10rYyQ`IeF-5i)fhaa7&VGI9Xj}QMP%2Eyq?& z>J^T+8{@n{p~NNNZ!(gB5>+_7pZy}JNSjhh60`{sZ7S983~g;DGJ5MDTt}EOFLU)_ zxv5saj;eQ{@N!`y3w*Wk(tS&@UcLf^74~BIEEeX8|GC~X=4Y1?-rWl+a-VrO<8=9I z{15>jFzu)}t}@k2C|_&Rku3n1@*)@!@FJs&rTb_@k`Tc>0QnBR8tt=X$3rD&qBt^9 za@lu*x1fBeh2xy~Usr)RPM3}vMuhLlSBd3np!Am_^7iK2lUv7-a%1kty~qy{Rx2O9 zwivM+)HVbuG2vl>r+MJ1Yoh)$SnyGsQ-v=w=?9F8JM%IOo6s5F>B24QM8J zPkv5nfFm^%BSm!0mx+d#v77l@ z3KftDlt7p7ahx^$KO<3-Xczlbk*8 z`qC9`2^(X3^pVDzIj%mrp_s1b1Y@fvc)67{sCs}j0Y0OM^jXA)TR`hoCXr(vnZZzq zTTku2AHHK`6f{!SCattc*}JBHTkI@ zel|L^rVxOp78;&h8TDumZ>B6715~8I)4y3c*M*#kwvPYcQ{4WKhy*GD@F_Ae|BFw7 z{hu}|K2k@h4`@(Ia^K8o~YLD;cyoVJMa= z0m|QCTo+}u1x42qy{s2?b;q7Pi{)cz)q}t&Sd&YF1O*>13fUzR@^IviH1|;)E6W`% z&6&$No;l=*jjWHGC)Z{J-80HXHxn@0w?ZO{f?m@;<3QaNMR@3z&(*J+z6*BXCR9__ zt0qR$yD7L69V?YV?xIucqAb8l@8FOFr1nBYDF^-FrEo%$?V`{jiJ)rrZ>EI^;{^cU z*T)P@$R5UrP8*)j`r<60k#^DDV&wm_Neoy+vbT;aHY3SH8Zf};u~jIJtNUQu^(^Qz zBo{btechT=E^m-(P(A@{w|nUx?hbJYbo?2IS&*1O8pg&Hk%|T~a1Xm@n9>nI?}|=i zU(qA_dz*E@m$yx{bJIlLH`b?Vyk^tif@_5nfS=UzcKK~>d_G06GIQ%{(^&o&<+e;{ zwo`J-)Ufkc`IOyw#?B^+7^te$yb^3hJbP~495Fs8FIrhbTx1t%#GMWuuQ#Z-BdS7b z0MS^Za+?>dVqig>)vHhongR<|D(Zy@)@K>k11x%@+7i1r0U6_)>f_=HQ>9li+6_;1 z<=7>hvTX_9gy2?N=*n#r$8N3(K3!fTZ!SRemUq{XrC5plel$X^FhUdTzkDW%inv5y)br z0vZTIk9&p8u$=yrZNt1d!Tbz+;b_)-t@!I?_MFvk9l)Fb#wr55$+f41jWNKSERgH> zv+;<)i!%1m&HE?IuoTVG`nO-%74LYcq8E6uqU&EkS&F~+gUD_egFnC6y!CA&c5?XG-(@<` zx=a*V+CV&xQ)T+Ea3c?t!)2B)-j0c;#LRuGih6+S9KdFl7y~s;@2GSJ_AbwJjRK-l zD^%+HYH(@!_@Kg-MbO?VBkdmWQk6M?@%@8A+f-GP>k9e}R)j+eUxb86nm|;Ad=RcOYTlfRQz27{Dk8-e zCB)`B60u$l>pBiu^VOYv-K+iU&iC2X2PZ5WrnK(le zEGhce+Nr4Y@l6p|g0_!rhM#Hnoc0#68Sqe9c}85BED%dpM@JusOtDg_rSoL{Uybt4e?E?X;zF$l1O{4nMw`{8ac_=(nA} z>@_?8_6Qa9i3f{wSs#mn!(cRyaj#M*QfCQZ#1W+Q^jDhfm`ljLs)hG*xpnKuZ8%4r3?Te4vKkVhGw7=jsTUX}2gIS2N(It~!Bo zD6yl1of$k(;^o>8*_jSp7*%YT0@7X4-)WE=(aRFV5BZYJiDA!kzxsNXVP5hEJ5qmF ztUhjUrNyWJ+R>s^(Jgg_fUb6#IvA~&l-}+zy9-RX>o+Y&x|h?WRM$+eJ4B~Uu>{|; z>vd*qPn@n5Jk{+Lz^i-HklY>-)zkhWXtQu?{x@^OY>wRSCS*44aRs(jA%8x#X&;{t zcun&3xWouRvB)nil6B-*+KKkqg?CGSvZ0{+V}g5LOK>_{fX2$7Z;Wa zqI^vOjMjve<1AIvUZD6Q>s}7aJK0yNHYpsb98QmtAA5KKmKx4COcL%~djRXefd9=xfx+J2U~2 z&_$d_aFG-26i_ah69T0-ywI<$jej)11Sf5-WGT0aSmf@S^tR*yR!fN?sJhWgQ=6xlXdd$6#CU|yWRyktnc;Dsg>pT| z$|&&8aOMUJq@e(UK(13b?_|3{?eR<54Qf6U&CXq1dmoJe=;H}gA!NSfebQ@K)AzSg z^X7$-md%7FxI-yDpCA4Fllkn`6>4{P-pKz_Zd!{QwR?!34L&gJ{?phjXazp|6<%0& zo)G~pkPK3JjoNx+H7WKMCBQ_IKF_@EIEwTzKC4@;-FdlkVDs@n758}sP3}8S#-AS8 z^V2LR3n}VeAW7{`t-z@m)eS%h8hiCiorFdG{qg=QTY_?XhfSJD+6RC?^ciA6oqIQv>QA$mPo4` zvsp92btRsLaIy=HzMpovG^pF%N?dPOkH6(rh4}-)5!T)|4ym9qsM=Zi*Qc^eu#TX) zqb|6Is^z?Ei-Q15rpFr7yB2?>=R4=H>}(@myj787wRargu27g64#Q^%s{!~6RHQlLswiV|r|;ddH-DNWO^ z^Zq+Pf;;rmCr#7b{}lGRaVy1#T05*`|0VFXWCN;XKRinc6w-4NT!4v*x@p2*^|(^B zQGgb`0Fr<8+{6xnb52c^5|pjNjAqnTx&#cCw`Z>0r{*_H!+o+sFwA=r3y9*dND3-U zWh^`H8%8{!J=PZcCNHT><@e_OsYaFDLmVe%}@^=*##7$>9sMv z2GMcB?pP0-$$JSffDGN#`QuaKt90*G@k6Z-;(GqjB@?eEGQY}M&k8P5hvv>EiWhqy z)(yhD5`9hZP&|4b+c`Rr1l?%aqNq!RY!T2Rxa09~!zEszvx>`j%PMz}XK_;MVYy2w z;})dUYJRWCVz7-|@x>Aj;P^|2qmk9eCXf&X*E!-}lNHoV&=Y&PdA^f`1OUZ9Nm)vDWa&&tGjPJbZffsj}u~XYz&ZTzum&qX>3O zw%Xs;FPWp=b>`>gI%%lnH{n6UC*rEr$4p+&fIQddGa+Wz7z?qFKf)h=?+C~P{_BM4 zahp+jV&U96>5dHD&qJcZqI;HugWJ};i|K9KVoyu;gcZ32j<+}L`MxM*;HD!uDEQN` z@IWu&P}TarAr7a`QMB%tqA{drhHa-6r`c=tZo0ESO05|wh+BTsfN=M>N-)m9QEeEO z!TBJHd}Y7#2p{xknhSfY9ri{^i1lZgX@o_&0`zbAY|HD`BP-b$SfL{)3tg|X>Uj7@ zp?;v#QRvl6c9ch`ZS!_w@cm+%)&griA zHPKipHH{5xSBR_mbc00o>22=}tPX)yHyhCHF*1L)>wzRi5g#a9S zd~ERaE@HCjyv*OG+?l(Pbx2Atl5uulUs&Cv__**c22z4R6 z*W#7UsaL_f+2h=~%@t;-_?JlV0Fp7Lw2Z3QZeFc-*UE(=C-Xc)mcJ@>&S4+RO;ZQI z_5=5kQB!;Cs&C4v-a)n?Hce!e`i9}~RUon~R{zPKL`~bfECFs*aC*#wVNDI$V0x7w ziizemLVtOlsB{iR%q_5BxD6wQN%b&o{a_#a)Cf&QRl9Nb=eP80Cyf{qc*`qMrlP@#Jnmn6J1?*+^|R~wX;c_GI4 zyh!AIX8FOikji1iB}at9dfIruN;StHhA`jy+MPq=w@r2ReD?&+c2862pGK{Xy9+0U zsith10wm^)f*0#Mzkkj8-!|0OM!5S=wtidB4e8B@Et-1)sRwVqA~3?d`S?yZ;ojNA zi!+Fb&yuS0enc0?d2^n!&1}2ln(SKtPepXP&pgZrtn;gT{ZqO7i_L=(s6kC%se-(h za`l%7ib?ks2Z-k(Lb{F=3I7jOZypb2A9n#~U$d`u6JsAl%1-t@yO}|@Y%#VFLiQzF zLdKSTHw$Ah_E0HCvW_)NA!V)XTfLY2exCPv-~W94V?N_|{jPJ)_nh<1g21g%v|hcv zyFwa_Ye>F(9XAP(0Xv=z_EqF3gG`j|Nq*D5rxV@GNO^Z@iQ}zN*71^tsoA=JFJP|t z%M%@R6^CpTL}{45^#wXClA;Ho;lTSn3!%GU&(4P2nQV@$mGG(YaKn9@g5nQ~UB2JY`iHT40T>;d_t_>{8j(X?9lJr=JCX5H>rE0j2`B<`Hx> zqgr^)ABL5v8AdIAsA!oqBEkvi9Laj5pr=e{0eLsmC-;c59I-X$c73MfSrFfLJwfhg z6>IXD0zSqGuh{Gu<59yAbB(lXm##%eIb0n$&%BT9HXuY7%NUbczX2XlC8?oI>QZtl5xBi&h~UJ2Cw(D!6K zraz3{4Ip{Xo8d{^iNth>R!>-Sk$u$MX4ysq-xb!BiCSwt0FujlV8!GElV4&`(t=CM zdpNk&6PyQO$!2cf@+oVr^MB8z4tkeLR8vjPP-fj6+bQGHZx_Em)fqC&t;#lwLw;T` zk;SSM6>Z%nFEFIU#$G$y(wSY`Eb*6EK~*lX&_} z2=<)KJX|`p_d63cB`+c9)78(u^0wsfq=nSJpv@2)T02aG%{y$WeF@qss1-ALqYWq;nO?*tnSy>73_f>Y&I+pXYa|p zeU^I^#{1^(kc?o($l&nq%Nop&f4IY>v~7w%iJbVXhIIx?t&TV_{x%A+J$dt^s*s-L zNFhJ`+qc{lhUz`hb*^%D@y!jzE!We6W6?u-N8AmL7(K1huo5H?4h|}&$V8=8t;MdW zbgw=!iaJ?fRMGOQ=Cl5-+zDYk^h*# z?~RauY{)=CrAuYp9~m3+T1o)G3xKXNrVPmP?(2Z~-8(K~Ry}#(Ws&hDp36&XLBOoe zHo4OJB0c!;_TYAVSI&jruKi{{T`ko&F_L@t%W!OB^V)$<`}erX#g#c+?|TMr(QWa(BF)s00(DakQoIzy zrXIzmeTDNCIQUM4>kl5O9;J1K5zD&`ev0jlR9#rNwQLEFzKyZJzuoBI7>$}${`sL9 zR%-ngn-$S#!QW+}hdC`?q>L#>-qsh}k1yC8!BuVM5mrHuA>9$VvPfO=<~}YniWiDc zk_;iM`}(&I24SNHd@Y*TlwZ^;^n*yPS;_lTF3McC#FeXs@co3(Ci3uU`IIZW^O~$8 zz%yoCsx;@ac>3$RBF#KPb7p$!dH1 zBJT>4x`lTi=PZJ(P@lX2pgG9$lV#5+YHzh&!QiiPOr#sRX02%^BHmdu!DdyRrjR?NL~J;)u9LVQH6$Jk^!a{?Q! zq@Y~h>8p>VgSIiuh#Gx+W+6i&NT4+#k)!_8(*#OR{)%2YKKp)$TGdBowEXrb58}@u zum?hOzU}iXrQ1(G8Jii{jN~pnv<(fuUOiP7oRc7ke`?2q>1b}50v3k9xM-??@_)Il zu>We_W)j$l=VRK@%e|qo8dZ~9#X=d?9w9I94agCmB%D~PIP?6_-{TRp9s`YpZQKy}fi=#(pS#^W zvbV|eAyaj^ZapV}uxdvb6Ycbahv!Z!LDr;ZB(kSv#|P_7p9!cIs{)(ldjuZ+^6^yM z?$$U6AY{g@+4lV7o+v?H!BHEg!c8@awYS47Ch4Q*s(-^=QSraa=GKxqF&k$~EPlaD^jTSCSPB;dcE(GtIoDVqySw6 zk+57<)l{4$`DuIjFlPOLP0HJkXCUW&{*H)nNvU`5kyMp9xXILi8D)P zK3m=S8#$)MY+JMDsyo-{G+N4*@BJFS(!s`mr1G=z!8$rYhtJnSRoA9zrOkxvE*(<~ zEg7Y)Q-ZPbRE!vO+raaBp0;Pkdx?p~@tL9Za|Ry_hh$@rH@Vw&g^BcT&UE*kI^w&9 zLAjr(`?Z#Sosn0`_SRPt(vx=@!a*iA)A~|xHrwN36aZ1mw5j^<; zbK}L=NJ|5eaP_Niw@41A)n}8i4Qa9L1TiM;WGa;W@VTlJbcPH((36sq(pYEvuw&!f zjR%(OOAFXK&He=&SM(N1{x&03)o4r4*&l({+Z-|W{L=6>JWAppKP*e{n}_dMEO6*SVwKwp; zUELp2j-3QTF#PHLJf?)i>7K{6%mGBZP9x)4hD*uo4oLkhbTmVXeh%IWP&g%0ugxDn zj_Qy8tjvdukAOYYUuqAtz9A*-){;^_La_;!HY6p}P`lDz8IWemV!zPk=ZJk!_>lSJ zsrksS3ZRhSnV2!>`}hL`t4xfoat6j?0NM?Nmj7v{xq@CvD7(^o@pyrAY~%3h2i92v z6DB=Z8`YHK(K5Z%C)OF6i8_soHCVTiArT&cfDV(F<~McVq<1!MuKbe}%g%;+$Jq!ze0gTV$gfy(1D`@hz(%n=tXJSott(=`8909OVAqDI7 zT}wp)>2&Z z*EczQB)rK~iT=hiL%}}iX*rQ5>T(>*6SJ=uYxf_BtmryLPQ7+sd4Fy`GR+nm!EzXV zpX^j7lBh&+*xxA=4^CzRC;3rBaDQA9XhYd(BfOn1B*qxBH$*$0`QlByR5aJRb--40 zVzKAk8hb3wY<&Rt?Yg(|YQlBoN{-FEW3*P@FbK)$$VU~oa#ItYJzIPlACp)plVsBj0#R2*IK@}>S1Du8M?F_3LLTsV*qYf%{;2;9{QLWD$%WGMV?R?;u}qXngwm6u8&$YQJ{WUi z++eL^W#lveD!Vmxpl2E8X`6)-bX;>ah-KGEd&(||a?q^3nGB6>epz9cK>g;+OY=F& zLC8^YXVxIc=;Iye6gj6{q}pNYon-mm;t3G8hdBenN6You)i-6idiRjfGLw0cboJ|< z=gq2rBK4daY^kPv$PUsbJ;vBr5S~(^k17Cx%UEcAC|+OjY!qtIrgx^l16vYu)!Nbv zA6*^`{#eGw7yG2YHyHd7y{3$OoiEM2EpnKpLR%eEa0F>xiNmw;3wqcT5KKl#x_N({ zy%A=bS7%tK@|_!gI%3oqAN;ehd;!s{*vk1}Y%!roN9*}syj_fZubdgjb{~!J7pCdR zJ#^rYPgzOOldp347-5nxT|1icwxs<$ehw1&*^#(}D-EZLdvCda`ONo>2~a*}Lm5@OQWU*@_-^F<749i!_-(k?H57@IIIM_6423LI!+7l>w za`NS6WMtmc#1g0LqN(avEf})1P&|hsq&<6t%!J5IxA;~MNm4!PZ@fyA$x;l^2KWY) zG80vxBoe`zYRnTp>i*zlR0_IwfYh&jn+y=e$F z8-?r2x~Sod8o%H{--!g8xc5=y{80&qh7UZygpxcJeQ7Rvov!1R8B(*)aPwmqXF@@_ zM-_7?PV~ij(qraB7w;c{AwjG}2;* z3SVA4T`z76nTnZry)RB}7FEzZ)ZFE1AJJK)_K4aW1j|372!K@CLR!hwpa3LM^$%bt z4dfd0&Op_!j;k#TThu%Il-UUdK=2y%&MztvRsKbG>`8WOIjcyKxaPC?68ml~Dk%b_ zMxuSA_kwBb>V5KR3X)9pu|T&%{U-Au0c`V}&L_P1BE8qavU%>4oY{!JtJrf3)VTFk zkgaF74-IL? zjZa`(LdJ~Ln<{_A_!V6d>PjL+T}vV(jQtHPCXRrRApgZKI8ud$=Y~2h8fnc^y0_6( zi`ov#NzeAlnZcn5pHff91qrq|$U0^c&A}m>VeXIdCL5un-yqC4H#Ya<8{^-Q@OV6w zN;{o0y(@cIIAFf?CPvA@@-TV@TU&0u&c3LUWCfu(X8%;fw<}(2HfK)YtoV-8J_fML z4)ctZB(l<%XoRf+4U`b5Il_BuDL6R}ko4ibZ)eD=hnTLw0edL>7JGOv$lBii;CnT+ z-S@HeoB$6!d4b1>Ny=y2*`@-Knm4%`tk0{vAo-}@d4_mXO}toOOS)mpRGmuoNOohc zUUsMCuokW$#%WDLIbAH=oT2MiO!}6>$Ve-%=-(Q9MQmHZ830;;wnlz=(9`tl!lx@l zUeq|ac`jl?%j)3!`+gs?TeQCt-oQDC#JwGk~Z5% zN25O0gJKrU=*lhnmFSLx2;zYC6wiaM_W`g`*@WpAg6XFI44j!>|7$5RJUNCqDmt|DRxzS7DaYKo7p{GCu2QD z%>jt7j|rpY)6vJgCpHxhRU16j)O%70f=JT?!cHECOmRJ__+V1Niove-EO&YG6uyw0Ey4e(JQtr zcr(xBQg4ZrBX~#OGGXpfk+I*o5K!0Fe|US0))VX^Ao>Pio!4}_SjK1xh^$}vda8gX zRRF!fbGTyElO>;Ay|$_TnxZwOIXnq^_pFaV+hM>Y1GA-c*KoToT22hc$84Skh#%ze z+b*=Th9i8O9)bKTUfOHnI+|CX;I5)u8{(5UKF(*NYX#@za;_*jX;})Ul2NV{zg`fI zJ!9uv!PRD|r%(c0s40(!m4auee{vp13!fxwK7GqOy5P#lRqDj<%qW@)o+bNcFYniS ziU6b=o##YsJRP*f2QVN=K9EWJK^=NCKwJ=| z+`?*y8J@=2c%c9}(}|B1+70JZv2606tXJ>sZWN}5zWvl4wlasczcyqkH1yPv`W@zgyov{OBnWw9)Pq-Us%m zYt@4Xpvk#-r+;4F_}ao!a*G%+F5`+Sz(KSRTbg+FHbc2ET6lt4i|r4=2OBG5T>C)A zIj)@?BAx`@mP~=R0vp(ADCy*Pf2^b*eKY@{Y=Uo*Sk%r_YnxP?_PZUo zIJ<;-`q+Rf;4&?jBMokzZk)&WKzVf1^1NER>>sWAIx0#z&h|@pR=- zHLt@r?;ENa=LDm%>&?I%h=KlJ=*+x1A@yP}v8uz#0J@_6xp@lWJtpHLd<4dH;LyaU32)dnp&A*KePL-hb~x0-_Li0)Q-OYN&!Hi;)wby%NdPOP$ZnQ%$N`PoRdENN zgAl6C(D}}Yr{o17*mEsaY{wb03zM@saVm%Cxu%1($d_mV!-*7iNGnGAN?)O`>~4OE zZcg7q0v--$K0ycsY_iG*Y&Jb3{qsb_7a=RIE{kM*Wjec>Q4Eo(0o`$s?TO z{}9198}CqYlBBOdFdoH=-{~)qVgZQtC#lNC*;gB)ewYcYBHM$HDm2zx<}U@`L68#- z^kx52al{QiY)8(;TShgL$W6>pw_4}9{I96utY>Y9IXd~4|IT$n}*)YYAI{xqj^OABRU9q<$ z#{hoOuz3#dS;oOuag}r{F2tX+)hP*5O|;vwgJD>>8|rP@izFmS!aYwpI~9TjI+N@U z0!tVi^r(~re~B=)r*P(Ymf5)tK*XTYuZ;>NQ-`exkh?12>YxCK)Q(WzP^phu2I_8F zH=y~0U_mAv1W2M?i&X|{vQd2$%?RTb+LV$Hczam-uMS?S4S0tBsPxF0s7nmO3q9ro zc?8K1ft#~)e*(vVeu>^kCo*HSYqEltrOVm#iz1vMB_91sU~AoB+JKyrQm7jnrqVs( zM(O(T&)Tluqbg@a0stO=BSc7{hOwz^35eCNW$qBLU66rYjC=o;IzvH}BtAwAAH)AY zR-_MawH8lBB>c>qEao@NiRkFzEHuLaV|Q{iLda$dllxF=p3L`IGrwg%OOb;_F!nHd zzN}I+a=#CLHowH{)j0*ivXUAz+kL3wZCi-owqU17qviJDEJI2!-<0b4_@NXx z578joF&A(hOshk)4>@1v%P?qY2o0Pp43ao8R#o>UCi+m`&s?C)R_#jRbb7$(__}*v zPrFp?0QrC;CPb>|APJ^&>=h?L3R7#7J05z`cCh9b>E%BPg8@@UN}s4nAIFnETnqJ6 ztDjp(5~MEby+FP76`X2+pN5m0>Y1KenDQUXI@@lTzoi z%n}f{=s0xSfsg%(lZU&5!>FExHu!QF{{DIIwA5m^`bm?*r#;&&qQV${UCD!bOXyPg zIzYY%w`Pu^dU52;rxHiGnFih|-c&06C!Lspbp-(e(j zqx@O0_ZDXLQQDbU#&`5W8hvo8`QEp#EoKksorSRP{6{s$z_jbCvy@Xrw1YxH`}Qjy zOZjxD3HQAE1b$9q7+!Jck!9WN;bKX({OnVp1J{9%ZB~CRv(L^GEVy1&tAYCrvoM1 z;Z{y_E5Ir}O((rr;%>y3fj6}yo`09quZ_h8b#?Fm(I+BJ@NYiSZ>p1C0ZPoHy&*Hy z*GQq!1ol<%ZN{;b{Vi;bo9--?ZcsU-)vUQdXUy=BPEP^qn-+T6x7~U$xA3>H*$S-<~cr;yEaR-f?txTIbU zJ|KkyN0lSmth6iWHhX0lOHzuz>hf0pxjP1~k>MRu}U5aJ8Ne?hN^B^0B_hctFyNO z8ludmtc1Hp?!2+YlSR55#m` zW#d#gBUDHk8#+$(xJ(v~OQZKsPNv8riq^dbtCf5jYc4)-QjaFudVJ+gFS*cUeb(~f zAAZu-NR_bEULfn;aV{)D`ivP3_3cvATusq(3q5>~jT;xc6_BdVM)Cg|9^U{og?K5B zs=owcr3bLx$FHMKY)`MI-F}lDd7?B+$Su-woQ1#D97PZ5)Mu4LQWi>Y1Z`-e#+9T& zdtvSKboT+nS!a_*gZ%Yo8V0mhbH@vx5{$ch$&=4lVxMear4EzD62gxpV=Lo0mI4 z_PHuQ(-lfh0y-!~J9xj_dKEl8(E>cH1Q};?`b0YfMdOmVH;GPt%4gzZ%207JSpN*E z1OI*0!)z370C9W8#32*)$x$?tdF@3h!wP=d8+LgLIbwvFv4SY!v6c^b>U#_%%ovlq z?0NnHu*S~YFnmlWq#21O-P9IOh-N~`ltY^DuWzk!1vt_YPJUW=!|^0f zHgc|7D>&5NhRV0XooLaCv}yup$EJoowimPZD1L|*Y4otS$#32kk7*o9tR{tmZ){=Z znLI#k4a#bF)_ZQ#kXtQRzvsg9SsGvn)Yi8ke2#+R6sN4Z6nm_OzpXcA*v5wVC29^6!NfzDxvLKvKuDdq{?|d-gKWJ3lWf zFy*LY_%F2Uw?JJDsx-Z z`$eG>aUfyt-;tDnPIOq|zZ#}L-<7cq7zzn`Dh9Ik?Yr%U+yJAs`$+v= zQUAmx!5so;C6+w>aMPF&Ej}z}ik~1$rET9PmkOEilf%BY~K-Wf{4G$w@=%%c! z_-tcT8c(Nr6IhaLsnVnyyR2{@5exZSpMd(fRjqUiW!lDF+FwMNJ-}^1K}Ij_W7aO% zesRp9Jm#0P!(sGXwfQ@7(S}u3W-KhD#Od{M6`(~xOj=6|e zrZCa>%;wt72{I`8k!f?ciXxkt#7)J9NW$PI$sh2Sy0+Lb1Kax98djtFxf;f)MW_q) z9mt>bm&>+F*t^l@=HGnLa&CCE56S_5X;xa%tqEv(Z*n9<+9kKH8*yP8wTOg0zcyihWA||U zI}!ExxrhS~pf9ksZw&m%4T-N63t%^}|Nj#Q&Tb85Yv9J1NQ)M-Hr|_PoW@Hr^NIcv zkGgodbHhHg=O`#miuA=juS`tF@ztVAJ}3kUeZ8JwP!bLFLkw$+YjT>)o0bQ?&v7P? zyoR^6c6i5x`I_&fJl{J%L%9g}#O4i#k;TAtkE6oJa=S==j0tsbsIAL=PlbOZzu%%| zW4I!{)yxu;g{YJuq4B<$tMrAeBAZG)FTT-aqb45=QPhB7%r+I`9GVDSc`Y7d0@*jA zM^${l!3EbCEOQ62qml1kTpru95iViCtBa5X%43SP+mV5|qU}Vxe%Z;gLbf}HUH>tJ z7wRc{R#6Y`-Nw;%1xWHv%x-xCFKy14Zrx^(|7z!bB6@#c8#i8iQUv75giguwQz8IE z*c97e?Pd^JbAyu*I|@iR=(up|s|>CSrdhN&r|3B`JV@E43@s3NT<}Vc;~}l$HICSn z6PKP(GC9x~zgh@MaZCo%l~eSuy2x*FS=1wjEOs9xkjk>5@NmRbcoREv!$x=w>@(3R z!9--p=Be23xW$7HzLI1ItBi9QB{vRujAlPEgp-|`b2PmCi5JP3}})#q;TXBRK7@O-8_`+IYn z$5-e7SVzl&g)UA7So&}f=Ov){VJfcceArLY*B<8Ru-#mYlzIXGBd%r*Aj=BU3EmH& zQ6^GgxXXk+#B#(uphlw~_MA`qg*Zm-KJ0FAo3Rt6&Isqt>E`W={s(xlGf=>P4JbH} zaj66Praai41DTxzGjB`k>O+g)3wpO^ssGE;f~Nr5#d z6Xta7gc(v6e%5xwoao2kO73wh@=wl@f)sTrxpu2aD_0;p;!4++V8`C60oYYS=-T0V zlBMda2}|TPQ~&BuSn7M%V*Kp7I`!L+*Zduy$;3&$UI;26F!3H;%0|rF3^)>huiRK0OD~m~vGi&GC)?j_`1&T4Bir6A!x7jd ze2=2ea46KES%w@o6*ALmFVMEwy{Vv=(|vyYac9d0wKw-;QExfKK@m6l&fXeu!>KNpP`{2Mbz9Y(ER()Qfm-e9K)gqUi9+L~?Ir zL{tu}$??ya@`BpaBinOXr3-VM^Sklb(inal2p~$r5`lOhRKEw}u|*zeg~=wP{$a>H zYfe7~Hqor%ObnH$1q=;ZW=@o{Ck8KPBaMGMWBAT@7FIe#Lzw*uH`?Ju2E z#5B&zpf#{jeZlb=TWhGdS8xw*|7h;Wn&b+)W>Tkvfu?U9}3jA-%?lX;ad>a zi@ZSp3jEpP3=I?+g(R_`3z(yXyM*f2bBGt`rLt&L+U}E;ANY6oP z=fj*+RYn47B7cMSJc*^AN)Zs zT^D%7G9 zB<3j17FthI2@=bVm-+lw+~U0w6{&bZt6K+2A?**YefWM!cMeOLPwmxypiAw7qS&3; zNqvDyOAbCd$4-_zEI2xZ1t`Lr_Di{JryTqRf=V6!TQ z7{-xr<&L2%s34U{j*3QVdTZcZG7fH-Y-_{3_JU%)d|HZHsOhzPk+7@r58-kqhS+kN zSfaEU%+KyR-bi4c`6HKNA2O4!g{lQm&2-bR1S$T9A>#o@*>3mJ*)u}h`*VJNI~xE zhg3O1L~N6j z03DG=fS7bSF3#+5reekfd^Oe>mcusKLFOp4upoQ!LK6P_2K}@(YBp3~1*m_4vX zoP+OJkvZzdOp20yQ&yd`kkB6nKL(Xt6B9&D?v}X|wT9uq%jLk7&%VqZNH}D{@J;kq z-GIzqadXHj=ul}eMF*nkb7rW%)fmIM|G8FbmPJDMxkuoK`!@4Y!+I3^vbFn|Z{}41 zSa1fMz6em8l%mD}GUQoBj+maE`XW{waQvFi?h$IEmFAc5OAW!?Wfh3S_}=8P3L9m2<4TCtOR z^Yz5p#Sjz5rND$uHrHKRBJA8OiN$!Elpr`bFF_RUDLBL+h)pOQB1uqFf`_+i7Tc5Bwax9ngS z?f_hRrMJk1jd59SPL@3L&tu#f0iw4;y~SQmBt?x+s1akTePr-2(hNTxn$J6EkQ=C58$(b zfcrX@_R?}(A*v_c%2gLQ=?zq?V5+&m+~;2fNV4wZAK%Z0?GHZbanu3RwruP7IVt1Q zqLpk_z6NOL`&I5$(XUPQvf1j^dXQ=AV%7UiGH93LPME^iz)+qUd?dXk%~4bk|c*9=yC+&`zbsaINn^gZyy1KzIXnMpt$b zDsNl|yV^wE6@G%zy~5)Pbf{_E-hfC&?cGm}^L1~T2!oFv>Ma~1Zb;lu?bV|gi5vk* z>yqPztY-};xk{o{5jsv+NI|xe7k6I&Y`q~ARYQ1}6XaAJR;HKHYii!=OmCrf50Hpw zffrJx`}Hh#vDq?jvR0%G=5f{|;PDnLrpXRfT^D-2CCSEAD?#R07G_03hD_Vsv-47A zQm3%*Pmq+MRC&wn%~D55yKrfIqw;K;R{TSXHKdiS%K{1IIc;0F*K0w}zv_Y`>X$N5 z#dLq87h}@)VYdO7u_2^CNvJdlayK3vP)|Qd8w(saPs06Ig1mpRu@Xz>R1JD9pDi=3;2fkH;Y{VH2}1Y1JUa)U(#ve^jY#gFn!UGvEAmB z%Q*Kve@lAW&o2EPx3c#T&MEk}CC-8#j!0__x2k5hZqiflU9o|Nlks9;;`ecPHI&nt zy)UA!M${s^v~XOUX!$9db^=#}WUH3H>D;k?#qU8!!>B_MpfddwrA|Z9FaDU*4v=*P zZ}T0f6Q-S^El&EFYLy%}zBPXv%G7-@4wYj7lca(9F7DKwyV3K*G3IyG;$xRhrXNo4 zs**>pBLo0&+tNbqO-lTs_U(UMVhsPe#1O`e3v4PNH(h!0l^#z6 zq3tfC?t!}ctoXz57`?wcTo#aIzrWwv8#mInt{e3uQ>LO4=9&q)(8!3Iivh=Mm0;LVG6`E=XXl@txAn-q(wC^5|R*$ zxK7!a`~KMe)L{CTWhQFMynaKPy&XB9DmMU3vGuKS_-NLrTZfaE^puM!PjJw%^`Ru$ zho2eSViKhDeAtpnbS+(nMbjM8qXpvwjJ;%B#lIRW&r$;CYV^uvqBGkpA&(-fHkQ4` z9Jhh831vc=HLzSvnzd&YmAayVt1zj6(T0)|M<-ab_J?DrcjXJel$}X8s=3aU z`}ob+$L{Akt!!?Ht^m)$TQm|uL%kRuaaH&E*K^<8)YMVvl;OoEGsc-i^JVfJu?Vx9 zJT-WVE?ilbWa%O6IxQZ~tKpImT_7}Lc~ai&%$hgGufy*(iHha>q&Z%b^k-dCg7m&m zn)T4~pG-HYhAon%wI_Z#5C1qQVoa;s@=P`fP&ORYeehQjC}SvV^wpXxIxbDYb!C*|n~(wTgi4^M)Imh+ z+I|KdA^%fy<=6ef&)e9iYQ_u1P^)N@VnX37$q$O%t9f>Y!s{dB8J+5uJDOvE8**ON zKQrKFoG}_OS!m&q{34kvV(a4S78FBW`aQdnVx+CUgTS;YO$PdHy zdDru(RD0pO@iBWZUG!q7)q+&4V+|4l>G&n}Av71w1@1ScUabpB2zeG)hlLM*(Cnto zf)GxlZy97M=mHKcw8Lj32CBP`BJbDgt2gr~KP9;)z=;y8$kn%eJx8ArTg0KlRB!8` z8L$Ay&)<7tM%;wt2uC^vK$`>PYDOl&XDxemM{v3%rI8V2s+{pS zs=kP8I$TsN-sAU})10eh&Szo9!pO3F{{KkS2%<*evHK&7)DIem16L&6Fq;RHN#)zx z9bElEJ!nK77_7k>A2TFhek(LZHX{__VA`Vp8TlQyQsMTk}VynsgBGz3@)md0?bBX zw+kHDs$7GwlgNSdtzoR|kvA+S;j_d1?F71LTCuad4rg=9_3Rc&!gd>!a>P>CQYgrs zz-sSydS?Yy^3^&dP3y3@hcQ$ot?kivP=-45qIAe|Xb1 z4RuI!Bg0GZ%!ja-hpx~0oR)Y#B(ttpK>!kEZxqFCfhfsvv$wYQ*w89?{~Po&!AUqs z{Jn%AoBC1D4JaI1#u>l$9zU5zcx^KHI`Q>*EB(_%D=iz!Pl%itb59?BnHMtkRfVp+ zzZ1aZqq`?kQ}Vwl|4E9wt6ndY0$F}qJP*54ZGjnYdU7r3mfKr#@jr4Y5FF?vB&w+c zyAM4YsY$jw_+5ACR~T)lgX*!Wx9ETX3xTsGXtb-}fTR-w*W7WG+I?V2HuUE!z?PNd zh0%v80T7N4-)RJ&P*mFk3mCCjKx=Ari5r9v=WSnCR-`}sCy8@%vGQ_uyg+>~E{oXT z+BqfjsKb#pJm`$;a!eVaRA{)_s}O9m)W-97BYZlnNBTQ+k_=?Ln%Ar;%N`hh^J1^M zsnpp1CSK6BWpKQC8Tp zjk*h(kbS#n!*#%J6Z=2Ti%c7;OtV*=uRs}&t<}Hz0E3(OZNQ07`y16c)b}NaJ?>q9+Ht3|&># zz$Q$Sp01Tpz*3g^w51K`M*n%5%OI;+U*HvT>v7AaTNTU6G?VN^wAMN-MKNtB6M(06Wl5etHW$BV@cM$Gm*%~ zzr#gI5S*7CP>1o5xkzP=H^w?BDf;=qquoE=2g==oXVc`*DW`+a9(o4r=0rc_hmCz7 z(eKp6q?ly~p!qD>U2+ueYLXP^2szu&kg>>ZCS)$9-d^kGz(+@J0hJ|jo@>m5v@cUG zQEh%UO1u8lpw6zP3(~$p+TeBqwGDtYjeO=^07TI|71Ec&HZ&w&+zKa0y$*eSI_es8 znq%NRL#9f1*)-QGx`vl%u#R|Bv2Bq)6qqH?SAO#nzS(rHwTAS&uK!IpCd{MbWIf$GWu~8Iku!3%Ns8MAhJgM?1Ch+5SHVKlj zxxsfpg#gI=tGv6EQ#tiN1qXi~|HG-C4?A3Ddj}RHJEZ!iQ!9hY1D8teuz{j$QYqnM64bXT2&6& zv1ITAwIl5~zR=$91xgMw(C&F9bSbzHOYV>U_?$lU2IGVS<&o6nr!88NNHMz3xv`z* z1||=~TIW2S<%JNkFAM&)8_+L2j_2NGys-05lLrw|4Nf4NA&Bvy;kf34C z#*GcCP0K3VsxyTXtDmn*P97gqSzrJ&NIGzo4Kmt9A#MW}{8CQ(5R!N#P%`zM=>@H2 zhcg|}juCGPSlbR)-uWDS_5Cga7$UofkFTyFwH)$zgKdAvtRh>8?OUFgeh~`}iQR|3 zmp_4@{jS=Qf^BY30!JDP1mS>B8asNrX0Vv64~5nDn^?~FzwD9^_f73}PzM%b!0t{L z7y|8?=$o$>PIfRaEIOG@`pl0y$DNJ~f~9Ro8kbV>;-2uO!Y zHw+*x&CJjx-O?a2q=-@~AxNs!eeiw%_udb8t^19|Hx@I`bI#d&|8|FgIeV=h(i$J6vUP+H(kGDj`AA^Vw1aa!|Uo-67!Ddwq=Q~gi42{J49d- zPSW#qy2{M-8A~!Its5*pA?Cq@Fbr!1d?R8!c-5bkX)_8}D_Kj~0bP__)Ftw3GC> zGL53zg0IFblFn*wV+{UUNz_M{T3qz9wORyJ+^i7y09oob-nU_LZ%~gvXRdM!%~vf1 z4#bQ64~ZzuMgwl_z}ccTK9Xed*5~~jpWKaB^>{}4v~@8xrVEcf`ZGHwi?%>nS^Th4 z8eELp7pgC3KelMCb95eF-w*{2W^!x(=~4417XMq162)PPM~=o-qvT_M^^!wEFQXw9Nj|lN!Ey zVrGo)?fx??nC})6gyt!5ivnmj9PoFJ$RWZSgR+se*M2$p&R1noh*1)e_y8)jjU|t~ zF}i&=vCkt3pLGa!R!eY<`#~fgSmbh^IO(fYnE^M~`C(W!aR6YQxFT>b_7NUkPUp^T7^ph zv>!w26JKb(Xwy!)MRLou2~k84Z0xBM6aS!Ma;3BOJNE%b0`B_E2or5BIMCibk+W)c z7dU5?`hRua`rNsAUvd?-E-RJO>1b0z)86P_62Mpf6i}Xe0SIPLOm}R40&mJK(_|FC zZJZ~}@`2ipI6N$f;4Elk3iL4pU^V36-W_EVv5%voPLqe-GDNR$9dU+-^PB!i2CQ+w zQ+5DPJlb%xUe4OBt$t+J+9M04Y1Fg*2Vvf>kA2m|Aa+6|ZacF?(NRl(C|e_2Ig+%#UEI1JYJU zAInW2SQ#S-V?sfYqpWVltv9x>2h>vNxY11ExbC?4XHQQ9lQBATCnEy$DHyixH!u0^ zkV9^uTCLZT)aWILHj{poukhK#<-yADYaOF+>SCpJ+u~o3@SBV z7q=0t0WXIsHH7PIDs2iz!^il>`nqE7H^mteX*@PE4VG8%O;F&YyDz7ja5V@@etnlRW)g z2B$rv_YUMS9Q(BR?-cv3rKL3m`4dS}8Q)796;0W~HmMQma_*jRymh+RlSQ0DI5jL`kn$(D3{5>J zL*G6#YHAgsO9%h+EAKMy^g|F5@P3qsufH++g+18h(ec;A=v*?3!kah{rx)-@T?|xK(qIq%?7ZR!N*-wD0}kGySWUKFc%f| zD%d-MG@uN+FQ<%MdWxVW?ohp{R^ zj2jTv{Y6T=QtivUuu5(dFTL~Hr&M1iPg)FD$7b@Iw_!yeUd#@V}zVuiAcpI~(!Wq)UEVWodoS3qQ-oa&qFg`rmnnH~dVQY~B*4 z7SOY_^|()ZKP%9=R>k_Pz9BESTMj~c{_ZN|xVK%4ZJS)rG$;qJ7j4KYbkJWzt;a_O zMGd$G)ddjLGDNBv@Z>V!4jPu9FswVgNV?a+AA2bh?tmQBWK0gyy~Pn_9&K2^%~Rjg zn(@0*L;^7nHIC7PFg}|rzl%sg`U$F*qCV(WYpP)Y^p;Eo3JkxS1`jm{p-bPI@@RMK z%Y!aHOa{qZ;Q56rgB72splr2#&~Wy z?UGT|Jv(ywyA7Xn-jfoE6RYVGcsbc| zoFQ=h_b4mbPK#+lsu)6zBHA5gHjoTUOMASn(wgEAV^mfvY0MwaBH2uPk{jHZC4t4V zK9dc}J9#MD1`rCW0 zOD^&f&gVrQ=Kz5a=CFK|pIqP~+SRC8;+lUdd~}QB*^jeqt_@9!j_*>q8WL7Ia4V&{ zLZrXm3GsYo+5;x)4&mY;AnIN7;y^>Z|zxN9M%{o0n9y+<@XY(E`1R1&`*e>faN zZ=%!yH7MXVBY-we%K=ORQXWBzyBubfCnDr&0|P8dVSb|3l(ck1G#?gB=Dg2KHQ<)o zbL;?3rmJEKzos+Wfx3{! z+=4?j0C$@r z7e@*xq{=E=oCtWjPV0B)t%40WMB;(=v|6%R%P5DE+N#HnbF}IHrV@e(LTH$3n${j4 zKP&Nv<-%v1Pmfd$Jw|D{WaQT? z9qGuC(hEi;zo%i-Tp8y^gZ&(no5CAOg+60l{ua}^zl+|f<&NbwV7wd(?U0HtfWUTJv8100Qa_z}rS zOGo=`MWwfX@p2ipWN9k|M-iq-KAvHS=8ZXHgc2FXT53zhWB4(~$kZeWGt~Z2$)G^5 zu2rJhRYMs}#>&Bi)~awVIx+7g@`eJc0v3eMsVAu2rs+qS&+9XRMC=KO5PLvqDb}>a zX4G6^FDPIMa|?YI$fn&b2KZQ&0r>ERq(S9cBwjN_mq?qZYQHD{kOKiq56w$M1FwJn z*d(+;B}vdMn!{wEgZuj*Kn0s&bA{;>xDAh`DlGv+s^$=El&8h@@;kIM@a-xe3Z^my z!EX?74nESIK_;Y=iGksZ|NBq%E~{oK2(W z`epK}Te{OzGFg62I9D@~j_4!}CIaVaGr+ST3sb^J7xp4$RY=nwwUm{%+puq%?fRD9 z3FCzjU|H`}SkjBb6sFbJ-zbzE^Cq98M@_IHonnX+FBVv*X!>sIzZEJg;cUU^N65`g6J1<5yMr z(H4y)EvQ#KUuO7CfItAj5G-jn?ry=W+-`dF?%Oh?O;%n1Nc(_$pEb;d3t<`9M!JZK z`_6hCX}`Y;7U9G%)g1@s4L}O(`YZB`Do)fD{x=~Cjs;8!9#~BI_tXZh@)(vJh^E>G zq%nUs0XPjbRBU_%7lK$&-KpO}?m+CnaIZgw6VX%I3#O_rQw~#3PRD}#P7C!|1_rmkz3z@^hHA>45e1L4SKSrr(Cwia;oFsoppuZa2b z#;}rR7&eMiRO?p42X}lLJGIb2C}c#q^V`>zPm_Y+@(MarZRTE-;s!7SJ*aB7VQoq- zhvzz4pNenHe&YWlTK%~Ff1CZ7fgsb0Lqti)$#Uw0TKSzkm4axko}0TWEIo=@kQs;$ znh77jo57=Kff&A0m)GRwnj99L7vmQ6|2|NuTr#K~c>L0-h!K=0{qVfyi*@_q+wX6? zeXo~YaAfpyZme3D)=z^|a-I)=pm&d37DX(O;J++!EWpM2Z4Fqllxel9exl8WMMV;g zW-&vZ5e_BYp&Ia!b}OerM-3&YrOfej%IS}t#&yuq24*RFGim_Zh7!{j_Zl@UMrl=| z>Tm)JALIRAEdM5t<8kCMNrk=FkuP)+l2I2TUkNN-xOuN{IFtK>-dISXJ~4}G)aivo z@d2+VjPPAd#g-#6W2;r-yhk4m6&4b?8qa>5;=$kvW8^giuCw#{P6jousptL+`I?C; zK+ml8FyNQjOR+e~%IhVV8Meub{ENqUo=fn!FY(#-ls+M5J^Vk48tm~W z>}!)RsMJFojO$1!!WZyjo&iQf_HH?w*ffj*o}{W^W6`wDo2Il{Ylf*2&OweLxn#_o zg674h^D77MZ0i`^ofm{!t!di%#WX1lgnQ14Frr>aG|fkb1tnwX7WB(W%)PC2mIWb@QDLCT|XT z8%m8XL+fCMS}U2D?_7~mxWE)kvpe}o{8`d-JH!TRVXyRlzhJzrln6vlI4|$=TYiws zPw78n*M(DeUPirIZAS*4_a&y*gAqVpO>L`jxRZX0gdXG(1<)bDCM|h}b!$z1p`5QA zkbco)LcQXecmxEH6qqp~{wXyuy%@W+`q>P?p|BFTqXvu${aJ4Nr8yS$^TF9gs~PsP z_*?a>uO-Vr9#EtGJ9f|8gRXnFQgsJua>+wpZVjAd9YlfaYbyiQ*6qMuy;Jh8iHkd4*{0^gj`R?b zPIeYHe)(5*r*GvMDdmxlY8{HU615N94GPy1MPmRy<2n~Wr&*Cog(Gvxig z6t#lewK2sY4fsP%p$O}NN5Awf1T*2UJ8}6Ahs6P_gEQ$1rlMz`-6|hj0iDpmeU>=| zt9f%USP|E$R9g_2BM2ZbQtsw2k%oXsk#sa=TFOX=3!vS*=`FV^$wS&P-&E1})s6&z z|4=N`eVQcR{k8(sRgvuSxeF@*e8xzqo&fGSFx)Pk6t!kNXDN#YtTd+a>%)s)yL~jLjOrrE_C0gnSq?LBz$_xXeJh zDsDe9Lpa?8qe5OVoL*Zl$$pAaq*VsV4E~3Mx8yItgdSsz$Kv;+LL-6LRJlUz{0&}PEZhK4H)*I`oT}XbU`TAYVlNSUD2)} z!b{C{?Ora1~xVzz~FRZZuc9DOMA%WdR&!PsxV2d9~o_4kd#UAiU}^Wm*P1O?p5W z_-l^}7Tx~(5pz~n^l-}~!9<{S{tRgrLx_y15CE;n*;-DSz-cOUp5@9|97bO z<2ab$Gyr&bw4@nPsCncy4ei;#ucC~&N%c8zv0d<|ma=^kQFD7z;~&Eo(Lop6O$6vy zPIen8|0l}L#UoDMZjeZcp`unRidf?k$5w0d9*NYL0*$I!vt`EzIyK;~fcfVMLIHn* zYh5az)y{k+5=Tbe3YzsGygk`!wYM+Hau3uS8;bW^P z+v>8eUI{8_2UCB@^lz&nIi7{vzj#Ij*!M_t#LM;-q`1*Je^@+`LU_msa;b9tIxSzo zxOeYaGq>QqKxmMQ%+p+h#;KUfkN7}la=Z626#3+qAQU-^qSurCdBg72fHiMfiS(l+ z|0!P9%{aTZmA~uH4!smYO|g3pA)=KruyC7tS6Gu~V+=d4?(c9N9?oq#QP^5r36ISj zZEcKo0w=t21#rmpq=!c8tN#GV5_S0PEw;qvyYMIUx)Erv&%@10=wm6=Z`l&k|DwVG zn5ii4jC$R4ylEkpOF{2kEw+2;=F^HENl@q>)rD$P;Nz!Tx9Y9g=XLWIP z$Rz`ZA0jE`;1m$oG+ex>_=%c24ys$@);aM0QAH2!zmxjtIo+9pCGqchr2(gKZP7X~ zs2lVvE$x5?tQjZpVLMTlZ*o`e;sT4AdGT&=;;UP^T|uOZDtwjfak$g?e*`{8eapTa zd$@f^1sn5I1h_n_NStUY8{>jmH4AfTK9=FfN;m?H2fQBFDwXRcRCdVgDb2unU_T)} z{2Yj0^)NY1i#Tf7k8OE%{y@PkcIVak@m{V139c@cCE$eg2}h1B=+;KpvA~}w_o>rJ zI||+>?GlQt_{s!@b<2>r8J(&_B9Sx&`^e3MpvG=^4-X>Q>E(B3nC|1o4uia_91o-c=v8f$oq1 zfy;HEBsD%9S)5bsfK<&D0#1BuPGFPqDAPCL*f^mHj#$iwp}%z0nCC)5irD1sSnJ=) zzq4OxTpmS-ndQ9)E;ZTi7-5cHIxRQ}ex^+*LsCGzBgX~=a}Q7y=m|@_3pC)@3IY2J ztb^XGesilo32_b*`IFMP9>l1QHt|2u`u>uSDKT0nMgIJYj^u=)p*3WwQM96_5C8fWo#`24gCfxB@G6Bw$ zYoBS*XuBb;Q=S6F7QL`A)`F3eCNwiuYY02$o5(4xaP0b`P?S_X{_dm7CqSL93oySwuNyGQu}BXEn^+pRbl3#vm*QvNcEJmI z_2$1rw>N-zhi$!AFn7xH-YAJUcH1wWnD)eOSUjqdq`bl8@DW8hd;fj)u72JjX(a{o zqI#CYKOP9Npr(E9;{Cd#+A%^zn)!z#xKL1=1)mw7$yilrsuj}>l<|P!wf9m$u^+%m zRJ0ObZWRJLn$MOmNWbK+IvCI{=^@Rz<}<-H1jE}1LLUH@aDqQ5TekScV#Vpx&Z_as znU}%QOn?S$w32RV5GuO;(I#Rgd8r@e7&O=rZ&+LLYPBJ`g3DSm^hsE6dzePnZDj~I1%C~Li;=^GuN~7D&Z#{>qVDfTC zbA4DpD>Qv(bwy-k(=zZhg5Pw4ReoGlIKSxx^Ns{M!PIl>15QTDRhSp`G1jvG5@K&} zXe9AmHFv{dexS+OAc;BN8V^`2wEQfFx>7qtaMf}4C1Zv>n~SY!t+ zqBdgoCzF00E-5-d_ilz(E!bz*)LCdkmHt6+JJKb1AzySvF48tT>eEzV!3~WXa2>bo`KEIbsfT)g=r-Zp*_|vP1|pnpV0#5CGnW>0xhNk;@5d z9LK2-C9~49RJES2Ow)WUha=NNn*sQcB1V}N2)tu>zGM};n*7s5x=pEm@8`F*90=#P z4XjRGme>fUU^fM8InCk;hyrMCGLn+`v(!Q<*ouHx%ct>Pz|%j&YA?uE>Kom#){MV| z;AUfqBgb^*n;7InL)>(KPZIQTx^*bD(R^v}fbcW@2PJQh(J@OT!0Txr}!uIX@aN{;p+rxxBn(O8|-cVDu`*MF+Y*l_X!F9x(*|$WR;O z$J`%b0eXCi$!%S{%bZyFqvT2HzLTq*|6W9fwfdFZI~xQ%sE=r zoO)^B+vh2^zy#f)i7g}ogE1)6)Qv? z1U29>K+|u^kJhzvia`@@!HC)~++AKNNWzdfFyig<$M~g&r_?p{w&$l)n3ft|nbW?H zi?p6FZ+mf~>JM`zuaT}DY0S{qg4Af8Z=HlJ>A8Q+CdsEp#>a`|95CkmW#wv$BQU18 z83cfw%-&As4p1C7NqF!F_h^L8U>Rln;SYAZGskQB+jeJ7$Ca!ZK!&&w!qw%fX=pcd zlnAPNzF|Gvyp=nfqjyqO5%ZSeX#v5M1XjJnbovpsv8u`3tAD#Fw(sGI7$8(aQ2xK~ zcK%iO6NAZHR8_szRhDnx3f~(^Cpyr6yLx!5?MU$-o}E9oJG)wLl_<{~f$y{0K0Wk+ z{rZy;sk!5)y~h0&*K3cnpsnoKIqvuoh+)-=Q=Q8fW z>wxVtpd0?39H^_RDO`w4r_n61|8pxc#zyrtG&r4yNOm*f)5pN2g>|y<O6zoulg(Sb7F&z<}nUfKK^ zcQ$=TJ3o$gKh>}GSbs`odA=CpG0$BA&;3C0vAogn1J5H+>mlyOeU&C~ij(JzVs>C- zvp-d(!<|gt7YaOfHhxwqHn%f*7amVEfZeq&ji0o0SUzw2(ZNXGzfZ7uNUhK|NL;Tn zND3|#!pWulxX}7V8YmrY;g5p7a{53x(yPbbG5jI;-+dDBKK=wQl-v6G(yiUAE!XeQ zEwcVRw!M?dm=Za}CW!!E?!dp81Mx>4QANDgfSe!|Nw_iXffqJ;7N_G`gbs6`H=Y4R?_Em+NqACpPJY^oWVjfJ8%Qds9 z-v1*(SUr{t&USNRaehoeQZtci>h_f0{1;ng%}A&A9Xj9Zp4GlTyfzK=wrwH2>lcq{ zPb~P3KPcBgcpQFI1+i)1aH~(^C{Egx5pFSZ*hi}?C*v>mT z9(~zk$VzV5ZOAp5HFaPfCKivG^2LZ-4w%3D{7a1}(kmm2!)#6JTx$y~)Nr#y>?n+7P?~FnviiUa{SuIcnz9 z!@vmnGw6Ou!1>B?@(PEdhBF~DYT4LOLf3?02VW(^45%;u(Q*KFy<#E|T*TBTQ%my( zhiBTX@zH80gNy}@F@aMQ0FPPcwelnh4_bMgWJmD>xcdds1|5G}cO0?%7ml8L2B$k* zd$>$QqTcIqvRcEaz~ni%MqS^~TLZRN6rCPjGCf4%N>wy0WG_VVZVlq-s(~#-s$z6X zT7G%%YyJaFi>B2?ZLUOM>Rg}o^ zo4+o*qo^x((^I0m%A!MXJqT@Of+Uk|CVE1NGZ5rm*1_FM@l#FJWZUL3Z$)(>Dtnyc znaYb2kH>A>>xw6yE;CDqsxQ2c*jCg0kx^CqDlg>YvI@4pqMxR}^$@8)IUp=x3Dwo@ zsGi8J1zZIkFU6|aQI&U*JeU!L9NoA~BR_fLR2g{5Nnbh07D!N3^zcM;@d7 z@?HHp#!DarI!(;nA5c<=<{M3%}w~FVRB*T=YLZF{*%%zgO}X`(>!SQ#!Nl1%aX0^77fk4WtM6xwBg0cx;pu0kF(i zy-cDj_KPNy9r7NmOZR_9K$}GBq%3SkzVrwTL(TYU9c?Bbg#A)A0YMYi#l%_yeW`e1 zT545)SrgWCb|#;n_+Jn-T@FXDUsciB(*34BBAdlT0Mp(im2C$#*Gp@Vcel#y zn*`y?qv=t?T8jR9aIm6FD2Guv{=>n#v7JUMFMt}KR%uYJJWRrjJbs1HdT#4v{)JCZ z`O`-KvO%4)o}Wkb6Be{XW2cS37BTGUToYQxPIj$ z`b|?H)2r&p`%{1B;g`!Z+-9@$#ohfb9}V{iz`7IP{ny|3w_U7EW-H(epCOtQ1tnVX zRpjWdXcK6!zo=blkB<3Hl8n-2t$bID$o7t^662ZjM#=TKjp>Wz4#;=6gsr)>tV{r@ zX*GZc3+)4V^H0VyMFR1c$FxJCE4@FA|`!s?NV<>AiYNvT4Hhaws?a^`N73oykQ&SwZ) z*A_qJKEv>C+7B-y#KTu|`a))&HBc|ibJ7aD^VrPEjdb?L-sehGJN(1PPTG0=vu%n! znh-9h(lBkypu?{E>&?5#uMyIVHpXFJ)g2&C zY@-SX$jw!MWtii68Rq_j7pwl{J6%N;=Ttrs`%tRSgItFOF4yyWN?fr{E1G<;A69)I zgMN+uI4ER7lK3TsS~g_}Y>qGB9O47M3rj`OyZ4JB%v#T-!*tYAQ9FM$a|S9_BVJ(7 z9(>T}(h>V1w-$ZwLe{eVJJ}^ z#ipZM_%OVxM#J@EW%}Nye_^97JBf;p5(DXLk zOdNf_mjb+VKC+H~0~&z|uex?%zeyL?@Pv`F@$wkIXTP;{C%N3{C3) z3C;=^210D+o9slDBfbaITb0>8LJL(+(w+g%RxTG>BvYKk<%@v838vTyxXe z+k8ym<8(dm$MKd0O_}dCpU$=tO@Fkj?Fs%#gp)-vLEI;k%40ek{n*P-;+;)nMw2mA z9jMp?4(N4v)%hhX2;a^cTa}v_o*7=hr3hIXk4Xqx@&!70zr9Tea(zGq*Lyf_fw!G7 zqdr#_zk64_x(!2nD9db7;#23BK3`VnZ`QR~We&`%^kEW2;)awv_>>_vCnpko%%zac4%y;(+yS@VGh)bR z0k_JK?y`79I$}#-MJm$nqxsnCw_)cU#p0V(-iNz$|FQl)Jp5gnB&Ajb7PVeDF5dmU z#P>LntIi-G>*w)mg}hPryPMZNJ${o~4YV6ACA)|}{cd{b4FZH#x5<64Q({$8_fZwK zivvr?Qrx5Tq8}fUleov(^(ykg-ZG|UWdUz`Ew`6-B*#>hPQVOw8v6ajq4Gxw=GrQ0 z5X>MS&3_BO}_ov75i^K520L$2z-|Ru6g(q9dlxB=p>_I_D!}{ga zPW3&{`|Bq4%xZNh{>UP|TYdu5De37&wU+K@MuZZCehPYVb}*La97jISKz|$=?Ig_S zR}MTcD%72=`$*Y%&~ENCOdLFd5}iP`nlW`*Oyk^I%^OjpUSJF>paGn(QS9nt_N94S1Qig(JAzu;w(8F5YlrIm zYx>GWh01+hGnoh96oOXeuSg-I_y-XNLq^(a8D4mFXc}ROew3WVHcZYz0~S{j`N%m9 zWqQVG@+P>uHG={T=kz966)DJezP}TIL;DE$#RD`7Cg&_>=PKfVqWmj>I?oXg{pV7L zx%1TjAJMjZCA`}yzwM-c)A{P?{8ye@!?Z&*{QTFC1vi=RL_?wnOR^Pz$bSxW%qi7P zdmkk{(I7Nq<3+ew8VlT~;I=AxN5UKy{#SSd(yMdVE!C7b3B&!M;9&68n*vTjvel{9 z9Tf~p13uj*`U}gSg4Fw*`#hZkae~Nk^k+~BTjv{|PQomNDj(oJ$F|#{pq!af?&Kn( z(HT#YJ9pcBa0U||;9l6+b3yqo=5j8M`wgq+cnL}{!H8~8f2;#k2&nT z$z9i(JZ1!%DNjuX_nQ4^f#u^-BhW5Ez3>oMSAf{#nq*tP4h%E2%o68lMR{z>$Hy4% zd#GIsVF6h*fwBiQowofY=|pu7lVtv*UPRR-;)Bc#Kk+u}eRnSw)p;pX_;L$RD|6<4 z4CQ&Jzlq$|vPT_IP+&+OCS~n+)#O%T-vD*S4}r7X(cNktv|I@idORo7iReL`M8#UCSL_vvIB|!QPKTs%ToN98BYjr4C97^!0w5)O+x^v&emW zmQgUG<0RpHdN|;cL6Apq5f3FBsz_Pz-4k#DayW6M;L~*AAJZ2&fQHs^T`5lK-8JAr z8gN6kx6C((HcU&Y3W~gQ zX8B2ASZf|_1KjEE3PxZw;AZoqUTSrs$;6MSZ4PC~%VIv_rhq!knv*dFjxMk9xCJ{B zyl|5-aszr~hF@}9qpIhUG3^wRryY_(4%_#<#+>e?AZdjk#P-X&X~N4M6xP@>3~S-} zix((K?(T}570Q{`a0}wpQl>m{cVAk)*=(4VPTR-JG-hVb>pb~YKW!$HB5in+2Lj{Xzb(;qTt z@?HAE<5uawpZ)AM(QZjf!9DX8s`hu$BCz%+sGb#}$EJOT`0HJkBJJq;f|h0aEQm{D^rfWxWqqG=HpN1nm|OqbgX!_09U&VWnZ1KdsYGDQ_s zGK{>WG%c37)gs#+k4PLwNHO`++N3C%*vffKGJciBj;WYYxYcb?l zDDcjF*X?VL;scWU^8PEs9`er~n19gQ+Z}EZCdj*9eUhSEl)c)XCwDR@`=bc|Ic|85 zIL%dxpBe=N`C?Vh*C+%%)y(}fWp?H)fCw~cW5UzDNx=EKl!JVRIwsJHg$Ig6@%>30JWd2fE{ zesPdK!JZzAgP`H>wVf5O)zm@DWYq!Fsd=I+a2+<7IFBP;`YH9eLU>eZf zT%yJns&Z(%s_2L8);Om_FkL&?|3zWarnn6Y3NJC{xaIvkoK0}1qq|{!|N>>f7M}S2^UY;ZU%K{APb(7 zg|*^I#(WNzQk!%+(*%s)ceYbCw+VqTWT|%C!BE`D*K*N1hpr)*ouNw=o`c8`OJ4Zj z?x>uS>#KH7`H9oZnwwigW%!JK507 zRJ}6rAyCqHe{jpP7qsUvvcc`{cs5yhJfZP^d1)kO<`XtQ-NMgAAvoZu4dA~_U%gMm z_d7E;pFa$Lps`u^){xKSq#9*^0{bn#*?TOdDrCffz7F>``|%c>wZ3AJ0AtA%Gm{Gm znR-qU>o5@djM@@8>trZY8C=EH`nf6OM59eylgMChEBpQD7BQgEhuC6g>}Zdn)5yt( zx7u(UCR0uv+ii%CE$qzEOr2R7_!50_WXmV8WMTEh9U ze-$RRqgd4|M{Wlqe*`v}dN%vxH==cS>yhF2=%;p+zu`6-+@z%FCZE2rm|$6*oCzH) zEVJdo&tAxs7MG@~DmABCf12}_=T$1-KH~RVz69Nts`HTQKZlI)&n?LpqT}{2h(*E} z<#8+`oMdjtP~ge{A0<&%obL}y{1{u3(1FQS->DQwNDcrJp=uq;(JbJ^>6^LIcpyO#@i0+Actrexo8r5$4X2tTo6aigg>kiZmy9tK}C}Q*xnNP7|j68b!fM zC3vQYLJ`3oMQa01_F>%+eEES14<0B#Zh44^J=n$&K-cbdwIs_09-I-0yUg|Ka%R^|b2BzZukuM{pgQSMi=q$_IK!>1fn z!>a*r?FLp>W@Bgx%|Ogc%|2f`*>|?E6wC&xDH$}(6!_`8s<}#?r*;Zbz2>D*bqWTT zz#KmxYyoVyk}qu)heo(lVDB^irTS(`0pPIdJMCeUrX%QO?l-d%C{ShZx7EjuKHO{z z1UZ%I+5@Gp!3_3Yh5@}nZ@MFzj2-t%f>6V=wZhAaC@B@p$VxFc;$03n8{%FjrT~1~ z4m-0))3u~fR$y|~KovovZ6lO0Aie|T;zqr~rQ*(he?C~wHt7*{@ldByF{)QGp8BP@ z3g|BJ4i>?Vk~>L{fc}c@ z{b+mgulQ@j0CY*=WH=xM$2wZ!GpM`m0-7Vu?)TO6`V4TxlsNDJTz}W#oFW|^zFV`9 zezf=!aMNbAQC=AtS1uq^(SXb2cPG`8m}v?+7dh}ijb;_-8V;FMLTRYAip#mEiXoc< z)McoOrP4ntROMsOOGL5fC8DrTvAD?0O#GT6yNrz6V1Rqb(c@Da0VhSAwYi`>M zm}*v{pqO4V9@oYan>U`IejudSPG<))>e??dZN$T(w6oE0VxeI6zpR))cXuB|o#Osi zXSFdQ(gEHXfVluxzl1evvaTcArN9wDPN3^i-MS6)6)oKf^Rpgy-oqdO7so1F(bmm8&X!*ulu z*UW*jg68uEgAEx4Th5a6UrvP@ulOwMt+xIdy<6bmn)rVzVmnw>UljG^g<&EjK)cI7 zt-xDA{2M%~{xw*Ys0e{uNj?ArWCeXR!f`1D(UsALVzB!^2Ty~rBh*#DLBARQC~~hc zs>{bAdJsF@)cks{#~RWGch%S%)xLpi7XKevL0c|>V`A6fb@tQq4mcpKjZZ?9O5FG~ zh*vGy-5;*knBG%^9H?kt2~w|in}n%Ae^@LGmWO;Yf0JvTx(=Ur__Y__V(3rx_<}bgS*~@hRCX~U>Zh#>88V=h+-A&9oN;6+D2V<=(Z(Um zr#cydPt=et%kYbVN2AhpIw|U&GJl*vQg7n9q=Xz2{{D z9}El*eC&Z@TS&Xs)boB#d?LmdTv}krq7L=bPNP30(U^ExxY#@qaiw3nORx#~&|=!4 z_6g&XH)&``=F*N%FMdN5e)2^H}0@X7B*g zAU4J!j;l9pkVyxc4XG4svjv|kH@wLypYRV-v`_2PPnXRHFDHY9hcL_sySa9KFBlCF zMNS6+8=>kK(D?^?haKX5Y#V=7O^Xc>u}2m4yJO0_Uq>jeEdPb-X77O%vSA|5(kFIp|lz+6!3m$=55^`j(M9Rwy(RQHNK>2I@MoWZMBdhE@M;} zN03VS@vr#$k;*nXi6U>qKtju&Voh6(IedvZ(a~zJmv8vqP|^Ju{_c@;o*=5#vS(AF zA2{3O{k13gVNa@dQHt{i1~3>Kt%8+>-=JJJU$ln>ZSe4>Ns`65_ph85KlXql)cO}UiluO-Y z{I-O7?n`O?FI}I>VjdP*p%EIEuDyir;E&FO&Pn?ct7x$pH8S&=(nKOwudS4E1xw6? z2AO_ua*qAl7H{jop7Gr7!zHt9h&*T^YtpX02?*_W`a1Q>j+lxK z-Em}5_2;j;H9-f`vg>T-MfTwAGfH$6w)3u{jE}fZ6_coL0TT$O!rq%B>`JWXA!;D# z-@T}N^_IpK?*$X`fy1~KqhSejWe$^7KS(&fRJ%Dk|AL^Z4u^}TVkWVF(KYFd`)^A~ zs)Jjz&K!1Z0h51ytw_%P_+CPd>D8a8U%`Ig)gZroCe0}Z)C!+Gpko-YC2olC2YdM0 zh1mel?$c`C-)%tbw;JRXvoH|yVa|={99I7VnZr+k(DWSiltEQZ)j;}s+5B*QcI~C8 zW>{$`3Hf4a&AC9VfjrzmB(Rw*tY1T4wNMLIqy;-msqStvYs(7SofsPU_2OLZdY0Ej zZN=zN-24ZlHd1ZcQI#X@)#IjM7S7I8qt2hF4q zStectNc;`cJVvt~qG-X;JhoWZ=-P$1|1)74y1;rh_Q;8J?@Ik~9Fz-;5F=uK8+BM!ZOMdUTPfj0@uHrt z^l0{z8v6T2E|2k>f+=m}PD@@E6`9^TM!<0st2s{p@T|iAurmd zEgVihk_|YH{9!USUH4~#QyozNX$?k@1k*uaKo)SGsATtYg?LXDvdds0b4HUweuJr( zbWWWa-=^q5nsI4q>61)N8b<^&c*<5t8ezF?XedL%BNwg4a1Rfq2*OK^*3YU#x-jWF zh`qFwQYao|)Z-vr@0F`V8CTuPpOp_8PH5kx8>V*Lvc7|d<>hAnmj%%4dPz=Jb!%sn z{rFM2nl`ux&L%bCca)4{9L#7%{!xQ8fZBS{3;lpw$s1o?Mw6o<#DR7YpXjQCc@o9- z;3k?;M;A!_Q=r$h#M$oCrDZYOt*XJ;=YzEW!s5VY-cjQnetf0xKxj4X;j@r*t8lA7 zPeu)BK6PKkJCfUe28HfQ8JT}Y1bVyv_f;VcRjnZ)yVRSmQ2@U1$G{iPd*9C6u71|1 z{fK9R0nF0c$OqJ}W>b*?O6LqFEH!W;X3onVVHRj+A?G6#z+d(n*KnIn9z`-gh-nN1 zTWA5&WYD8mRHG@r6_DMf&&2+p7hAdpdbm{}qaijd6dTHX$F_wI*D;CMmA2Nkvw+We3&ip-2YQK2}f10XvnfW%Ngw4###f@jJLi zEKsaUdJ@`5m4V3<`ghzG@Ca9u8;Y_EsH{Ru#c12)x)rvJI7CdwZHr+{MD|~H-$xvP=Kdc z!5ASIP-g(<24{Hu^}TbHLPyW~0}>wonh&1AuLyME(x6%Ca9m=c4r@ z_{OI&hHY)C{IrsuJ~rMV1Qg3f#sFV=2ZzUw>8kuik(f(A7*g*@qL$RA^snG02uL3q zQR@f1E`NujB-fUrLZVr;1BypNLg_@5zY`k)19d-^%U+4W;%g5@6dYL7!;qf+d;^1J zFI{9%OoTB#k5nLT)x#F#m*W9`6U;gmGl4jv795c0B^90KUy)Llg6cVqnHfI%(nhLn z%M}j6F=Ybd6z3a=_RzNE#5CEn1J?g3=3KD7~iEKaY@?8kvnNV1K( z(2z_|on|eOYJ-8=*EG0M>%kimQ1+L@%Let1HG;LH+wXAE6vuG!Y6Qx&2J!qeuPQTYFO_FkJA267f6^#5?lF2bOr8GA36r}+ zM&TexGn^F`?;S9E_OSjL;OR;oGXm0lrM_MI3(<1$l}{m}O_)u3R}|k*3DPQnAbQon zVlb0UsTbt-Q_!tVQ1c4<zm+_p0JgNTPonAi5G6_ERJDTDfz)zGZ5*;(-m7B)n(?G%Z(ckogA7CJ@*<2z3 zX|&7-wU&AFj@ah`VTr9>uTDqZYBgVdR1l?-Pp;M$$y$tmSuM%*D zgM@(8-Bpxc94rIS{Ut;5Z(Mi z*wEe{-t8pm>83nYGmY9cnGEjAG&qOnz52x12{gf>Gm1gnFwB=zy_qXR$L+gTyxQHp z*D5>BmOL3j5B+9o?YfYeiE)Uy-l5%Z&89idnC$$3a5g{gc^~M^uX>z;Jo_CjdU?ZpQBSChBy0Y^80Ar42S7GK=S+PrR-0Z ziqF$L(O`XO63?&B$G?l*A~B5=w?K^=my%PL+20(TQU4i84k&e!RIpxM%p)3cr{V!( z!ly%m-BItM>nv7iX6mmn8y4cs2LrKaF&P~Fv@1Q3r>6Ew1Nci+1dhaGj$)eOm|#SY zEPx&~*xK|V1;>$OgofET+~o|)5h>tZo^0@12T@XP`aLV#hS ze+Jw{j?Hz+^E&B|tlYh(sv1l97e73_n(C_Z8WG}4_Qr??_cnuQKU(#QyAXTs1GAs^f;P%a*VKmsS+hU^Xiow8aD|PvADdn*Vf=Rcuvqncj4!{&ngR?TUh` zj<0J8vwch@P9^)2w>%m0Vo`X30Ox6F=@dco zgoTBjQG8$qVYmS`h8tDD4Y{p}GwS@Pn;}YVpf4)irk-!g4VUg9NrXeW=45;ugw`X! z)aGsGShv^K5Ik6PRr-ssHkhfkdXO^g2=s0yOXs( zTxW2t`WGo|g!wflgvIzi>YN8~l_{ZJ?O~{*_}xjqy#|J&LaUZTUh=33N;M|(&Al=s z*<7ZEK`QF5&5&HZ1k<(2v&h=k${M#N5O^8>I z)V+QFI6N4u-{F>&!0e~OD{iZ4N#)k6fK`Rxt^1sK6&$heMQroF=scU#s=>yzjpg2e z+EScAwYKdFowIFWYZ*t``f1^UKd40Zj_Ddc1#Rj#0NUA?s%fYla^5#~-AEjQJ*tf> zp}wK3J`8!L_c=pY$wVcBP|F~%1OdIxJR9$12OSD1Wa3w)5$Zmc^nM5cXjPs9p-X`m zF(3VeMEmHIFAf5oekG)M6|iy~Mw9&`NYEifpZ=xVQRU9U0d2c+y*!FplNz7OWicBx z{h4o4ALgrS7*MRmw>%1~?K7*Ptb5fw&0MQt1M=PY5;Vm5E)6f`1gn8W<4;!At>BDl z!XRzY1CKvB3_f2lalF~=+#*KHUv~v2EtAiNTxx8OuGdz$=Jk~OtVOQXQN19xAm48F zTJSxY+xR}!(3AWXlyvT&jD-96w|#p<&}&;A(D;ec{Q`Vqg5o#`l7eA(K|LY6j%7kK;>LeSA3&Z3Y{lA3o-3Wy(QHAZkESYXm;9J zViK=tbSlP_ZNRMn&MDUPVmq3-A(MxG5bKbfB+{~;_M+*_O|dyWYD~B#)-(?;kk8Tr zdX`J2qapDzHR&660n?8cDkRE#!Y-ttu|{4iZ9jbQP_+urRPZ|5WK7wJK4qZygY{m1 z#9%2Y6z~dYh%-{;N+2lkvzTjCL+;A;sO3g`m=Yl_ldxU__YN__wLU=H=31i%1Uryg z?-Cnqy!S`B9Vrqc`U9R>O1mPG|IYB`t@QwAQX z_bYAdNNU>l`G~m#ico@ke%6x^6rqTaEwSh>BpZMdirbyM;$8u2^ETA_q|g{{=4n6$!5w0a`yDQ4g4v62dy;*gI>SxX=W?*svPT1Z!x~bd^|r z!RtAFW{tItI>g?8SR^GJqOV2bTTXv2FAc@3Mz`7hQ(ekneX7%tb@b(=dQ$63mvfKX z3$=0R>Ae>^H2-ln7w|E&>nqj3Y2873E5Gg8yqfvSheb>eXTx62{%7I1D#OI*&=Bf} z%M|wFTrd9uGO~qp!7}~G8K820y~1A90>Bu?L8{sbC}~va^-_0 zzf1mGH5F1%I^^X-P&X`mdqu)qU`c66wZcXEjsN_nJ<--8($sb%h5tgJ8lN_ao`+)A*u=rNNuJYZ%}PIwbB1-_l}j_B zmfofOgjr91r+8UUxnp6hEWltM6IC+NFqj6iY^y8TRWL9uE7h1B2tH=bYz_CA{*b?7 z*pR7tNm@FXmw)14J^PwC!xJ+rwrZDQ`WMBbY>@kXKe?1^=?>}fTlsY8bfm>usZ z%)Ygd;75I(n~++V*4V(6D^HPA>~Z9gI)kCZVFB~svgpc(uSNn)OiWu3B3|34o94)1 zvhx5<*gk0@^c3Fwyh1`64=R|fm3OnAHZQDLDSp|%mYvrKbYZT%|F;Xv7qR=d3sYXz z!i9vVKll2K(=KtHH-pQgC*`isE{}{K$X2bHFXiRl#;Tx|)nODbJ}b z!((A!HLe=@P=wCx0YURE*-72E2I3k>y?Kh7!BC?Tl+8Xb*6l@o56A^Hv|2#xnEl_Sf(3CI#sl>_ zIEG?&l%jnf)`qyCKE%M^wp14zx1&t;iZs|K4)a+B(7+-+~ls;=iRnF z4DaJVzpT%jH6|Gtbl=*l$?iD(tXDH}qt4j$rK^40P-X67DY{d(revZvb$5vYHZ)xt zrc-iOD@UhD@$tR5K_{>D#0wdY2ys2hTi>DXwAiMjFL6^uNYG1oo#~t&!Dy}Pj~53J z`RvH_C}H}2MRnwhxv|;hTty2q^vwfpoE`2Yz{<62G2OS3UAT;|DILkEKb~HAQz|u6 zqer38rFL3X9$wfe2+-r7WC=e5oNtV1E9(gXEMlb^B?q!!c9H{RL>EP=RG;^_%5sn3T1e@nA|Qpux6#Ejn)(yu)gp)c47z ze)H%4U?i@ABbW&sxE(r^uL?v9(HsxF&V&p}0pMO~5p-+s!-gS1wrSq(% zvVuWyyzU}J97XO6uYao60&GmyT>I&%vOSuDdMNM|N}~JAqWn(Prpi)a21rl}c@fw) z9V&|o>tcu zI^#wP1hM1`2kKt6bz&qZKSh^R&bZZH+29d9juqNyykxBH!P z9KKWA=X}L z?QR0>u$nvTE`Xmd)q|gh66#>5Y}}_EJaWq2kkIA@cOtLDN90!rmY81^D}i|@bwnSZ zITWK=fjc3LD1a2$u^7W%8l$Uq4fc{Gg}={&vH8oSk%$q@{sHE;%9Y9{g_3@g|avQ*9*OH>EBn+^$Khj3Ss#e&Ie937&K zyn4h^lxBG-9i7=BE=ov#TX>I#mt<;~`;(+!x?-qL{|Z8VIDuy9kKr~`a1;eA76ufD zLf`QKgKeX5e($>v?)z+;dSGICdEhgtyo(+-XHfOcX&;>v@Va9X45*1iJPXU8i~x=V zd$J0(6j-!D?>gW4djN66(ewbD-MCO2hNXuZlI<~xVU}!5ttVZ*4X`KSd_k^vJOP!P3dh96Bg&_dGFe>m)Zo4)!`X57%1y# ziAmY0DF!T%ZR5#N?wHC}hR6L?ktX0ueM-t6 zxS(|OpNz6E z#jmLCqbD|?6pBby$XqKh8r_lxN6lTlB_b-{os#ULzy7N%ard)kvg_Bf|9Ah#EcHe+ z!H;wjW0Ihi9{^AXXbO(M0S~ihc__oCREYnu(+a<+;fcb@#~v-3kH#$c7DQJLME?2} zf42=J&eW~IwpgF3YjzLzv|h!4*Ek-Yas8(OciX~^wmF^Oa=k%DLwxcU|9X@~L(;2y z((Hw#<_2+GK z`1Z-ybzSx>1S6_*!b4z00ub6%?UNf}=)u@?nQOIrt*|*zhX{<0?`*vvIyLc&$a61g zo!g(O!-Eq2;$zuzGu`JxFQb zt)nIsGokqkxMM$#&bRs5PBF&skJYw%hk>6_Ittt$;4{onai9u&n7%MjWkuePtr^b> zbh~8B#{Be@O=^^j4r08M>+*;-QHeOG|c?sss85E-l(P?F=cOa*+7wI0MqT(0H_=CZzGdm=+NJKVtDz0r-Q zgE%V0kV87kBOP(vtLT|!saQFGs7?5HJbdWH;ZoDSAK?IeIB#2FOD4NEFe_LzIPN_w zjJ#m*sG8gYT*?{1zKH)#Gs(=Q_RW9BIe;FH1wGaN345y``b!(cy}F}l+Kht1|1d_u zB8M3v>ix9o#PQcdFZ6nW6j-6CObN-jS`UcHWCquZAT*Zhfh@)(rCl_HMD+KL;}B5= z6WJzJxgi!VeLqNtl$leykCpEjlF@90by;hKX+>%T#cHU=pSl?paIqvWlmBO&?j0jT zC0@0Egz&kpw=5yEeMObck~z3PiQ1X6_4=>xg$721P2kw+J*O%VTFiqP;_8GXp!Rs1&m@~z}}{aaE4I*ps5 zxV+LLSMOtr(>I*2o~VT}HOtkEO}dRwjSVEFtbPQ1h-4+!-wyqC9uAoXnO~Pn8H!;3 zV?y+@Huedc<(;lFD{*?^>+KZR2}lf;>3swA?^8$^pR`r)sLubX*B%xI5+4C0;{0x_ zcRnyC;1k0o5?S^IN&L`MqP(DuiiK&hrodEQ2p>peo+t;IKoU^c0o&nSD_LLZm3;eY zl76&_vf4bljL@;|H*lb7c9p}ZXEYojAyQ3N~2X55})Hw@LXquhawo;n&8R z;Vk_BMu%ktp{gv#cxXVqUqLnZ{*Va}SJ0%_XQ@D{%wlMCQShfwrN;Sc(}MZR8R~`> zv&v=Rg@A{9GLa7fK1?+Dq}%s*uH$zFW>CMq&z&*py-{ zqnQsvV@wn!EkP76>6)v+f!-inm}O36`vQg-W5znyTk-#}2FwXQj1RtPl4<|2P`lLq zu!2~czJB5Ic$)wC$421@j=#G}aK~fe*t0``traID>nZfXUmpjeV3>>$W^RT*=838N zS_)39ATk9^s1BLGG_hQ@g!YH!JT=veoJyGgI^#_%DHvW+=!LzJc|}%sf#Jb2Ve13XJOlj|a~L7P^uXgAdCiDjxqjTR3;E4{p4Y0t)?@j)i-O z7cCm*)H_3UxY_8GE-yLftXF;|%NlWJbdNQ_&6lA$c9l$w;Jw|P8+GglKict9gr|g& z$AV!j3|O>4f+Ycio-H37#jNR2@Rfp3RWe^}9b}xH>QGALF|9d;_-TWuF2kFx*tRRW zE6vtC31x9N!(t_8ECHQcx6F^TEqt&!bs}6g{w=q)rEF)JIQJNx9C#($mh}@QX>Su$ zz$fo|n1EPpH8le!@VLc*9|aavw!n(n_g#lMSuv1vXW)R}R`+@e&ctCvt_Id0me3*@ zz^*o2K&!;GoQN{2|8AyrZ6W61^`&?Va|fu|qH?_fMz~tyi?{5OKWSAtt-vKl8y|W= zc9J!YWkgQNJC)AAzmEzu_WhT^b1M(5@6J{PY7A|N?eRkas8OF1%GhL7>gKFv=b6`X zT=R4!K@srO|I~^xDIu0yNO%PvufR@@MH>rWvM4g0$8ih`6k>o3fOLxCQp-Q2x|dKy za^%Hu4&L*}&spq83pV#Z8b`#ELqeCRU-JwunHpow;0>5&rn4CD-}Nm6R)5|@<9dAl zLqxerrErX0iM1eQW^7H2*zmBde~5uWaSlne$%Z1!0HA*mKNkQ@rc0|W4%xrjcSNgd zOjro^;c3(Td{7Q)^Yhb%mB6Lwh3m)S=$cdAwNb6F_pw~Tk{a~A1!luD#HOb+pZw;A zfzw%oJ0;B99>1OSk$G<8RjIs4hi@uYbmbf1i)5yXXKg;jdUTdU!0_!uNd+KTTeZ}b zN&Bz)t@>Z{o5z35Z^17AHNUYw3SGd3d_P}h`NPE7ajPyOJkiDX+do>Y?x*m6}W zxQH1n={6C3*7JwR3Kk^}mW9K~%4bsH#VWR}(T1UvC(od0a%ExXb|2vQPdeo)9PzEd z^qex~VCp9_Dqz%u6J3O3eOfAD4LiBj;uNE;!|s7U6%H9ndhq&`S01bS5iba1Gf!5@vD-Di z>EXk`r8CQF~awntAYS14Tu%6X-A@m(-~t`#t}spMTeViX6@FA&y~%$EItaIcevXaIh$?@ z{EQL&F#yP=_0+1*PgZEqE0NQbxBa=P)T^e(;b{FVb2P|6-xx3Vnh3FAAV}@86y--N zFCOL*yUA8l+8iuwagp-o45Qblf*Y1p^N&NB;9j5X{f4gPV$@C2d3W)LMwuthDc`BH zqNmU)UtdYTLCiKh{l#=Q`8X^nK0cb=!FwJi4{UF&^<^nf*>b;Omw{!7^A{0+9q@(+^ z1YV-KD@8frZ348G7z0ekDd4~{4-v5gtft+#HtOM4e!%rb^H>)H=@GCa4LmlWIzjIF; zU<3~c>lAY)!>8yClMvPTTrm(lNGSU@zv}0vg8@)A8=|uSKtJ2P@ex3D{*0yF`l%T4 zND+h))#X=z!!Vk?U55e*Uv(4GQZ&fjV^tzXZ|7$LAbFXDbC`wDZq5?IP>F7$|Mta@6PuBcN1?DjPykzu z5tYF#dQBgR?@lMbMxsk@ZGH9fx$HigHow?Ps@S#rov`i> z9Pn+OOBo9n$PVCV<(jR=uvv z|K04~1@Bcn7}^qK|8;fJM4$p8-_f;s!a)N~nV!+s00IA7UVcm8)l#D5#AQXI)c4DZ zt4-rb1C|odAe8J;DjbLyHek8xh&2F&vGtJ#LORtTF0^OF<){pjKEYzhG-3BT&^o3O ztj11iM!BFVUhna9TbYU(azJE}eka2v&KW?A7f5oCGPC$|>6WxcrCv8{>fo;+-tL#u z>@BlEJAc2?;zEDI;Fc5(nO&w9?0W6?$s34FIaH8gCg*_0KZ~(B-G!BTvv8XFE0PMN zg&>_OT_SB1b%wIVO&w{pdiZ-l=6gn1j+!={-hR-Z)GVAzPcFFA-5Z#omN*#$prj@w zI_W$H;7Dq4B>${ugcTWA-k1zty?%iq^f|4{%;Wpn&X8%gus?Vhy;=dq0@?Nv4Kox0Kg5j;qyNR-_wwtu8a?6ylN#ZXF!Eu~WbRE;+Ee zRm%KTJ4`vap_(Hki_B4=-4}X^c}5aivLOow@7eCNjDt@BE!MGj-|*C=tt4^;U^lj& zsbRjf-9(c6R*4R4rpXOll7}y0^O-=k?z-iyNe*mYe1q?bOU5+DhS!Sz2 zH|F1sWrl8d+|e6=b(N>W-x`M(!1&0dq(%H#XK#g_U6s+WDG7;>g^+8Cl_LNWyxQ>P zU{i{n&0fpXuXK=^CCeRnF6LGWhD_ZAtJIdJnZ!&4cU2CBlIWyw>@ulqM`Oeqv zJ|^v64Wf#l0N2ev+F;`|fU&ZCw+&`~*hy}!L=7PEdX^1153MoYIyX}`RzC=sG*x@K z#D`b&z@)j|5#ueh*oUX?3B_#9Rkn=^$YwJlgw798PUjKSsRgnfT^$1;=Z~(7Gh`t5R5(3#{{%0pFka@9A;#1AR^F z{c;Vq_I?OV2qyj)TFN3{4veNr%k}eB(*QRdM@!7%6TOhxnsC|JM}H?SmvXpjj$DH- z&z8B-6;0J=v8Ek#>1iUbEugRcdIig(e3J!0Va{WUbtx$@q%ojw_8uAZGg1qyodRA{ z@M-mLig9o`JStA;%9n_EHFS)==L&URCxot9ZWxckZM_&gerZ0(bN-*sfalH9rPIHi zLBjU`>I`~)e6CEdB=4`iJ4PZwH&wq(fp{0C=NssCVUr^HTJ6E>v=XC2ZtVoMttYU% zZs*_=!(i*~8}GH;XABK>ZmzqDm#+*(B@S+K!wAd3f*k59lrLo#@*NVjNZ)yl*QiOF zFMyNE1DFvuI2q5yv{Eq344YJcHV2Vd5^Og;aFLT;h)bv1d}7n5HQ>$N{}HfzDNbw> zm6Ij4_8V}ZX&S5Z&t|`Q7QRxfM4zubA3febij(tF+UIm8Ie08L$Wfq`aMznlBcD>Z zFQF;ed-v^Mx<|rP>!z6XO#8i$O9tEp@(<&v7dT(#3EnN4F@^64|3vDS%nAmCbX(XSj;#A#TS({J$fs>a z=+1?&!ky~f+a!A5IQ|_SR{GDy{5vWYj(JW#o7KwG*nk+irN7w2&ulR=#rLoH@;$fs zULsSC`#l06o|i1%0(|gO{PGMNv%S<5px4!!dG(5lpd~=$S%KCbfQup`n!Q!Z>Ri7Z z`AexQ^*2@gm|`H7N{AA0crt8T0SPvwGJIMpt|9Npfw;nDG4I$@nF2XlrVgGk(#Wvi zfy@=r@ed6%t#aExo6yFFP>B&Yh5cq`ex+q2i?J3PZc83L_r}SA)qJq!VoK8Nska`W zdv)D2Sn;aNb>~y@q$r3mVRIF1Sc`qeOZGehJ&ylt8EHx&%<$se%G@-yEn`5TfYi@l zshY$Ju6}}U6VwKGE_?eyFKZGV%*KVFuyu3Rdgh)W*f{^R0HTP%y(;&s#_?nTSNy(- zrSki{xB2?0ZItZNj|DGoy57M>@%aN6l18CV$hLe_Bs6TAN2b=i2|;WRLfQwIEMs@vMGU$-U-hqamXQUhWl^yPn43orpq|>F13Ddv5TCl*EdBBY3*VI z*7Ox6X-Dc_BkacFw*3lQv3LJ=rI_B=N9v20^uUxpXWOb~FHZX6kM`kZi_cLnpda%| z`2|^FX*HX34A#HG{mtD3=WugrrwEk~bPyZ{4CKZu~*=i+AH>YT!WR-FI)Ya~mA0 zOFX3y?jd^9YM)b(jc6ukUpzVJ^$>n;xyIAtPhA~}7lB=b%_8JVSp z!biUBRhtEH>A&mauyr}016WCoZw1O1ub=sSZc(7qHVrUx4Ewlid9};vNoZ0e zZPy-?o%O%=9X}+zUfCVd05hTREMRP-SZ&;V>348GNK|U4WUF>(4rpOiH-Y2AzIp&` zM&2=YuKav8vAPr=5tkze2`${ZHW9E}d|^l&(tgQ#_uUxV_3?PZd2}6wgfSeyp=H*|qLnE5< z8*)3T>PISTu?6vE1@Q^`IXAor2T$p~_n8I4oahBTz|5Q+^fwDTFCW$o2Rl?%A_k6Q zb-c>fZsyyF;CX*lwzkGzOx%v7o;T#R-4&l##HzrUu?p3^gP9D*XTNDPK(U}R%AT<< zTvvd;1^5-jyeOSqt^g^C_$M5f zfc`v)3yly5Q4H^0t3WnR4WzY9&hOlAIB(3AT#xT*l^%Uw#{LZ~-yKogbFKe?#{gPC z+A|#^oaLX>_WUsrKS8FKKvnU8r}%8f`yC+STGPQzj-5{6an8nxvjKB_LEgohpfT#f zm=UNU0!J9GAqszZT@+&+zt>dyT^1OM75S|UH={R|FEuTY_aypFfTrx!A05<@oI+)5 z`FBaF+?IR0SBGAi(amq0pIS?zYf@J#mgGjJ=~GFF4Ous+OZ9Ow?)#5939Y#PooTol z9(_^-t>O6_E_7E0g5fSKHEJ^7XKHvkL6cpH9wJxCZ32MUHX1B0^)Lrm{~EOy5)`M> z@pc45W(?AEYU$Z4EdyT~yRYBga!twXb||SMk@Wi6Yh8*3<>GI>7lM6h%gUctM(aPx zef>7yY-*~75leQuPhARGi*Hf_%tx5L#?0D6)>ZB zro0b~H!VIp`|SSE%)n<6vkN}xU+W9Vu|~XwnkC1M7VsePX4ND2jKn!oOqyp#?jk7& z5mlSV+%ND3@CSB2#Z|1`kTI&j)rAk9K7gXfn$#Nx_l&*xerX=>F-_DosMvHL-u8Z8 zQ@8qy79(C@zrh@OsY6yBx^$X7jNY1X`yf)kT!mrc%Gf}PnWanqu*Q_|R=1b{^$Fo{1t^MtGXJ@;I`}@F*hGY9MQ&RA z#l!^`FIE-+@HfSpo0? zMIk<&;KEbuO(uhi6u8MS8<4}5eWv65Y)c~Qd zRgZp%okkcIeN}~YgCOL8)F7c>*B@?CUE8eV+ed?Wia`z%aW=NBWXMN9fc$W3MMTK0 zc+S{-TEWf}d+Iouc}z!qh~DcaiM8ibJ#D_6r^|aiW-lS3Wk#AyZr?91>Ol}7ePS@~ zOA#%fnE2ul>YkDzE(~DdZ1dNdZKYQ;@!DSrO3yfQg*PnfS(73SPF( zyJ-6k_0CaDn{<0FE!P)>-V_V&YjT##jfk*nzXiNm_%!`nv*tc@duc{coLJ$+7A&9z zdzB?n;~QjK#b>bOG=-s<&IXM43kUUAP@1V95Q09B2FO+QlHod4KUeisle~6+K0b9@ z_Br0^9C|I*ILuxCG)P{3syp%No1-6oe~ZM5{${-Md%Nv%@Dul5m8C?BT3h;OX(NH{ zGHkAM=S;e@KCkVX>7u_5|Ck8uOMJ8c36B5&TiN*H3RaXqS3i+>jW5DQ#QzpM(3qLa)Pk9eaf<7g=B&4pVlpJY-@!`#bIjwS2( zoWgH3Nm~qS9#f9=hN0>04Fisd#{*d9!-x&}ka)WsvAZC>LqfS}mcwV#-aU5@RrGgo zHot7~RgP`Dbk^m$V!Qv+c(mt;WQ}TuCN)9V2PMMiAYEKt4n)=5XK(VGX1WOxN`|qUiHiS-;9<7^3cKQgD+{RDD@k_}o;fh&pq#=k$2K zGM|oZZlv`3diN^f7^8ASO1eL_gh85ray0jlWorhb_t|qb@T}M(fjr#fQF-iioJJ|v z(eF_wU-li}l+RN4JKg^$At$(3O|(5U5BkY|9B`Aruw)~4_RHWb6s7hAXWV9A#9rH; zm?|_xyjnY$&y&ZTn699y4iHk%AUe~tcgsi%2R(ZV8N=H{<1ujYYaGsNK*vK;n>|*lO zfnP^#l8Q%7k4>DskC)_iBWC43J|ewWm&wHSsa)#i;gjwM?dA>VclZn&vnqOnLnXhn zIWf{}E4wdsdPe>@L#12RJOIm4M!4S0QqnNliyjIu);Jpu1YiF|$}xc1d4jX2BZAXa zgUC&!O9Mi{aQi3x7&A3ev+l>ZSQMW7FR#avCWgsVM9=MHJz&4*y=L;vryrr+v@uAV zQvzuWZ0O8&;OJdCp1F^t{#50ZqxQA1+zL?uK?vHO*bhoB<|GMO@dxkY*O{n>)bpd_ zKWn#D+oio?l%G$%tG5!n;J3Z?oyUHf%Ds3stow9Ah|UO;Q|Scf;&4CLDMDJOm-2t4 zBZ&P=EGX8iGw{n39ljN|8azD;wA48CnHGlrY>?^l6LhBDoMI~Fm*9$~!8 zeq*)W6}`;s4jPsZeA+u%XO(ayM_uH~1_9jRlQ?&)8kzSic z)lxV=Vbb<Ie$yV2LCp!IhyR ze~q&reUcvTt*S?3{h;7E!tHoxB!%Bmz&k31fI!m&iIHRX0b68+5!wu%NDTz|mE+yq z9!|9IEc~W}@9JT}cWGPo>&aNv>ers0N12o%REb95!ttL&8&T=WNfcc}?4-Jz(>^Z{ z@C9%S_~r%lI@6KO#OeD$EUV!U$d!w73+)p0r$PFII^o+7v+{VvD zIX7!e+O_smwSc+l6&ykWSH95KggG9rQkF$+`G?bYr?V>)r~XoN{ws!Y6(Y-vu}xa? zI@ev+MI>s^oP=yeJO#O+o=kspz0V=;P0DDTB}jXGKHv`z_Jv`)`xrNfwX|zDN->_o`f%TO)cyt+6fQdw=3;$nxXWs=pf~y z+N6SwTcL46NZ8}7RS~;qS&#pd1@Q6m2m&d83Wo$&nVj50sr8{X$ua^?SV{{c^~Nd2 z6T|XmeB#AD{%}KG-a^^j?>ILF78Kfp>g#8YmQsY5a!7jtCX>za+06q3!)y`E zU9dp|c`(@pXKZoLXg*b?$*bg)eYZsO4Wz_-1{L7Oc3^q^O^!Y+ii<^8g4MdQjfoEr zcjm}{4CjGP%IXZBF-+0~N>1~-*vyiu^6?>M$isF~TG}WiSw_h1~sQu$hX? zi>o~{zG#2>L%ngfJv}!Nd)OK>^*w5zJWKOa9?y^t*Q1sI3E$0=CXh5J-1oD9(}YC; z^nC6=)lEf)`B`C7)ivFcOs9c5C2}-dTMXUj`zsOM!AHgmmjci=4+i^tsJ<|=C6*LY zPP_BTN1e-X7`L0zJsAoYx9H!|jdsB@#h(p@B)5Iqr;X((XMWC!y~PN#A{^Lm4@hNW z@|h^F%ia~|KPMW{ewyTL{rIIrjhkpkM<@Ll*c7Sff3))W$CbxiE+H7nwBM*>o2^b! zr}N0X%R)ZpF7a9ZwJ2EU74?srl9FV1_`vZF<(kR?L}M5V1;}6H5iXJuF8&cNLLZuT zC;j{<#iKGBpK_)KV#FieM^@Shj&+(&u~`EtkI;gnD)HK1eZ)FA zokt(uE$T|Q$-jE@{kIM#eX&W&`CJn7AI6Z`fpLa-Qc)W8~o8cR_>#y zp#<<}198ebhS8Xwk37t6kD8SOnd+uqV_-=l?gaFp2CQa-cHAkz+OskO*|bA3anSE~ zSG7^FzPs+$^(5Tmcflr73aVL}EXpv;FSg`iNc}CRt`yH`(&QRZ9pXyS7gAzaNvdHW zTVMT*&Eo89A~5t44TGGjmiVgn{(#RgLaRSCy(9X5cG@lV%$3i%F===;6jDN-S}Bd3 zx?NJ&%{pDKKw9Yjd!!$JOT-Tsmu}!6S1?#+1ZlKL1!lxb_`c((iuj^o|%$i5P#fPZnplSdJeJGyINz zy>cN$4jnkO2(&!T`x2o2@vO93zuvwdcq{QS4dT}3abfnd{awjkp#B2MzxHL&N&jz? zYeu0NDlWaK=MCgTAG3f!=Zp6j?FBR+UM3=E)|FqE@NSZS7b|x#3rq^WlNiPH#BZS< z`fbc}Qp%1SCgD;DrAAkZls4W#hPQmfI;HPV`@Q5;#q6HeNMT%!!(#H*N`wC!N!2nA zbrHjHIJW|$zZ1e@mbJB+s0Nn*%FQeHm)Hg^V0y@YRp^ekC%lKhQ*oC%U?ZuM$;NgXp;po+VK|~M}Ukh&P zs%L{~&yy7QSDHQS0;4)Mhj*fR&0Q58XgE32^4ul8I;!m{dh+E4H!BapJ6%%mXF)#` zLMG?(d|UrQlTgQkL4B?TCcw~=$*<K35TlD{BC8HaOVYYyLHd zY6ty4CS)qFpD>CAgI?d~oM>|J49GRlF+*NYiA3}M*>(%8j~^vcn?V?0rI9DeBudwf z=wSI2Uy==Lq$D_|f$y>GX7bN&I-F&fUoT+3(#>6ke<&`zhs{K>fbZ?Vi(PM~?i-56 zn-?q>ZTnR3=4@ZB6OA|giYw1rKCw=o{^?ja6PU|N3HVMP8U+dE@b2!ym;8LLjkJk+N0DkLNo{SGdRdg#^N?z=Q zddogi=bcw8JSTTX4@F}6bi>3V8kq9z?SDQg9=vHjzx&!%xy}m5N6_oSfG&X2s!%|7v~4;+YCyXlMY7x`)MvIxU`Kov;OYmnpSbCOKf;k@bkTmq)3d;SFb9UYt3^T%+ot^TmZ>%UuKfB(?Sm(5{+yEb zj#r+W-AmILE`7}R?FdN}VRv->lZWP|qS#b*iWO)4R|Rh$*zl*mL{vaAL&|HJo^=NY zgO)D!ah%1wjQ7kAA1oE`bA9fMKAXHcBuW`@+2b6|h%Ch|k1_H*|D|?gq5-^J-H*SY zOK{P2UJr*G3_(3Ac<_uXfFGs1p2?LJABJU%(LmesIQso*D(DZikRjCf2j$hb+pa;h z<&@91uQcG$%{$7LUGxo5jnj}{5Q9h>lE-e9QJ3olW9&0iLF!UHD5oyP2IxG|9l9K( z73v37XLa^#MH@!au;tz>Qh(O=ImDYnKhL(ibxw7X@OVLnwk5QYp^M);YCoCSdpQ)B zQ|qQ9vMqX~lX;TYc99);w@(=zalazUd$ly@+jaLVe5=b#4wsr%E6H5HpmO;%alcOh z92b)b5oAk{8PE$lHrQb0zHnq!*4??n{gt?!cuU-9oGH&Vfj=o(n(Pe!1zBX-Tovbf z2!MmqVyEhruS$RCTaqEHnE59lQ}+m!k6^@z7UG!CNKD?L5aPL|4}NUXEz0Fr7}l6K zY*?|a^7>)qG?4i)x*qmK-pFb;6^RL__zxU6t0m?nuV|OLx?JG)9=#yXqrHg%{I@An zI0n1jHC@KvyXBY@xf>M@^`$nLp4?`Eu{tkOHZ15cye^f_)hWspU!R#$K}0-ii<3db zX*Od`k1Qynrxqd(2)xj1DfqdA6SOgF;Ptqc771F)%4RSojODaKiiQpthyoQ+TUTTm zezjgzWI^GAweU7~KF~J{ZA9#|h3A?aza@@N?KAYRnU2})Ar!o=zcalj1u)LPmF09P zcsPM{bT%>WFZqk%A?|6{PZ7GWzD2nFV$kN8+NRyw}>u5aNcMYLDJxWYk z;YT>h&4y8`e`I~I6aByf$X%WvnzRF9^1!NJS=Tkm_*fiLfUe>>&h zUALCt`q|Fx<3w~{vbd&Va)0XSdUzczi;2O(MfA7((I8H%)G zCXe#rW8kYNfHu9CAnP}H1d!~YZaun`q9VG{f#S$FV3DhZfin8&4;&X8ml5PkL zi`>h1tMLgOL%oafEggB7 zeEJfm-U-EgCIKj+`Lq;|{vEjM- zBPr(UH>@ajBp011GfCA9L&eMCjKUcRzWfQUw9Z9O-TMoh;<$#;mhR9i9qc|gX05@CMYj^W7b=yNE?@7Kox|j0`YIDXPYkH+1BT|P zdH$^(o3p>4yDdbG=6G#Ph_Ga?4gBU70~0@|1zgh=R~TSo_+Fx_+nb$BBc@kt+hM-0~w$)9BdE(R3~FR2Oh1X$y$C#E@g2 zUAIAnuvs3|^f(d4b0k*@S`_@CfpW_EWu@@C4cwUlKD}N(xd_O;p4)9dypW2LdO=FD z#jLn-p4C^LGWnV@ZvD{=Z2J+UBZBPKX~!}G7kBXc3`cdc@6qQ<79%bNOF&mqB|~hR z$3T+CVR^?#cHny_>^OO@2aS~DBOGy-+4?}6_Lsx#2v*%}7mnGPKnHOt)w^xuQHpQH zAzP`)?uzXDi7y47icWa0ZqWu-?vtC(^Xj)NqThyXdJ2n*-SE1^ zVktHzWLPevIhKFK&XkBI*>4V2I?h=T7|TL^P@z@&1L?Oa>kt#+$D$+%^sL1=K|K@Y zJ29_onVI-I2HwH@=OEZ)%D)<`=YBIo#y8|58BO2rDtL5}PLrwH12@mg-kO=}8b$W5 z1Smg=X{vJ8;V5_;RnGMQ2W#84;p%2Hf2VjeZy?pgz^VJt^+3omn>k|hYIzfCeC#vWLfi+E!eSPBM zBtdYE%vu>m`EvNWTc~Hg*iYKOk0NtYcqwFh+Aj72eUA1)zPD{PG6-8`l}6GcL+|KU z3>OIrZcyZ+ze~OmytB5Q+(ai8djh8qnTqLZ3yC%>ISopzFjAGSB{zfj#p`Wi@v9IL zKgYk}$C47*LGwyR86?`>GU`_P5d0eB8iICQ?PcM-9$5G0w(?l<&=XXh*gibZP4qNB zDu1a|`>O+3uX*Z!6f@NG;Au>y+`oGx6(4xgQd-nitUFJ6lZqA!LL2qIr*ewAj63Ro z=5GC4-&;Sl{i&h6KDN2=nRTkBfbz$(%&WhOix%Qc1k8PvewL{!H*LBj_FU{@m~c<040s3*h7XTQEA(ki=65upu*#c2v!$h|NdTX zt&F4$z@QLNNH1#N=AyT9KcV3btFB}+ZPG~pGs}hq zZE9SG%Tv}hQ0i~ZG~#pMkcU3}Z0`>_pMTm3q-)~bJ_Ts#*w8&MT!L-HGn~lZEmq z;P3duQMu%+!FbD(#yy6*TZVe6DWunq={>hFL!@8`+b2~*(aOMgbW}utZS*&ggS`%V zPhD-nTv;)CY^DMp`($pGeo2T_YGQXNbzgLT9}{zn2|3)5VnRrE2BnOns|}p0yDI_G zi=ljgEB^#@18JzdwPi=>?oOc?Nggq>wc8^8Q$bIJO7*ztV!30keb1(=@8(j>)4Z-T zDIgmXyejda_h-w0n^Tb}=WSvfkEo#-;-avjrbuuz&YAdZ+J^O$N|+|m>&=3_w?WC& ziKjvQ6S9H9r$NQh#YuM-F|0TP~G};zGsl3axBU5Fe5W zlB6l|cvvv>j|+J((iNCkQ~iuda{t0$?F^`nv!Av18D zePJ=%n}9B{WFNmb+)D1uV#D*yfBM!3g~SCKGig1)?EVh*Bve8$H`COiAyiFm$eT2z zP{BmXM~7`19txtmDwX@c(W5%p{a@a^yiT%uJ~}9YGYlUK%G=P&0a9n!Lc%vE(iwci z&sIwQs2x={p5ZA3DQ#_PKNNwO@VM9Gf(HElY1*V-omK;K%vTdXyVaBL3YGZnW=^Xq zwLek$Mtb|4XJ=8PMyOHUx_{b^OhwH2>5y*QiCOmzLV4Ja6oYtWm|7x@Gf z0&pXkYn$z-s;4KyDD9yPE)N}%)4>0v+FT~ceAJeZAJf^|PGy{OLAgsnZzh%GNAbgS z2+E2np9XK3zhH!gKQA()kgLTPObC$-eA`7b;G2Y!FY`{Hak-=0Kssd2L={Wdixn2} zi74L1Wt;9}-cdYZ!`0P)e^laiVSn54vP$nxDs2;kuwG-6 zm2{Z#1>^5W$6^L2G1Rb`S4vzXS}7)`o4z4>(aLZ~txrmM#KrU@gRolc-DaZTcpyaK4+JDw*P`y5fWVW z$CF5(x>#GN!k2UoLNtl4ITr{5yQm{X6k7hish(AT*j5KA{XzR{kXlUwt7W)2#m+-@ zHIuFiE80!iBA+q|A|!<(QC&hg6jL_sWfD4pt7QfC{4W(o3X}=%FlZkO4FvniLxHlT zo2G5Z7NFC6_TkrlVc{o%5RlO1;a0+(51r*>H6oGorjB&%M(Um#l5+Q1By$!gZ^<=g z66$?MCVt*kd-c@d)l`rx#boHI+X6PhzHcJ3(e;~%X&mkIemCdN(Ck@BxBF??=I5Vk zT7Yv+ogRHq#U08p_9ud+7CFJHhO_LIZWK61N#m-jn%YAo`D0$!P^&cLIFFRI?RDKv87@eq`Bvy-yO{Nh0AkD!?_un?mf$*ec@(94be>-EmJ^D(w!I7;G~;7_?Kl^?c)n0oWeD$%lpT97gvpK*yw0kJ zR|4!ic(&@tM6L?U`_;}KZ3*qzWSaz4-4ykF7HC*2D6Fl{Am2U=6yZoyhPI6LFmE7_ zjC`E8E~1M1`x5(;N-^c(w1zziI$(5*0OiZ4{m4|opfZ>hafJXbbnGL&9a_UO!MYBo zZofHiG^2r4amW&^(w?t7mQ_u9zo8xrCfbT|EO{7t7)JjFvmCbhgOK^TO;q#1 zqcnG@O68)c616n0*D(omQO;{{8f2XFh7?)vrptv*ynnS29#cX6AC8NI4vobi_mrc% zGSG;axJN<9n9KHEu2P8w)B;4K66*04H<$@v(!?3DFlf|h+Bm{hHwoB2JiTQoq>9^u zrmFxeS<=|~NNeybY-1g35fc{qrDaA*Pya|#pv<`c7HRBvb;L6F<*Ic?!e^-iKKet- zEt#|YA#wAK|Mw%-`~MR|K;WTRMegG4dfxe0b1{^Zj$jjF(k z(N2AU)MFq>i|~(^cC~7j#`My=nRViYbD^cLFAzRgIO^a{O#a(knbwU>1w3}F;L*M(wvV<$fK^D>xpj$#uJ$Q?&&WS$w zrkd0dLm*O8gj*Gckvx0RRdZ>uobXVE*Z<+0BgyCYDW9=vI;0j&a*o-K^8?RXfcA(- zN7fq0^h%=z-$b31$!t@+9q=R4kc|err!kN@KAr?cuM4tT!ay*7B=i{VVqH-6mt+fu zLgjAa({DtD8~61=E$(SaDEiBv)a@BxIYTYaM!K4t&TUM^PN*;THgP0JKf30kGp0Dp!8M} zqGti7HLhE*OfUL3V_w)0)b}%fUwkj0H4BT|Z-%gE7#?4j*)5|%Qs#~cQEUvT5v8Zu zxfHXkc~76hSe#1T=67#)Oh7QNHWk=T^hn5Be|17=$4_=<58@TM@H}N? z8jS5ygd6XoKtn#3>5`SxCHTS0-N4~R1qK>R&-dWnFLWMLH)dxDkh`iwL6HGODUC%V zDXZrU#l(tC^$Fn9s&Z%LKHyN`R+Oiuq3?xi?S!%nL3cwZ#tWW{u(8g8V2$FmX!QAM zKYeMP_x@>QK8S3@AL^F;8Q`4wkm!^46{-X@G`KJt;Xsj*(RYI(HP?$WI120DZhvI_ zzI`QXme9ap(UV|=HGX8`i^lzPbgY-!Q(8Jit814W>z+iP`t_jYVV|SQ|MseM5^>Qb z{nXeP#=vjl{zRmOl=9nND`($KN5 zpn8jw4aBBz_v;)!=ZgAsD)bAM0yN0}daV-UJ!dQJ*<-<*qkR>EhxUQnQf_%$gR^$IK+T$5bamhv8UUqKRP>vLgKf6kw@ zkS|%{s$-yIQg(XN&M}>(O2lBF#1!`UHz_L>2nIh>SMwp#Eu|0Z7Rt!Ozb-*JD44v5 z2m9Y`slUwUZ3?-i`tfSQa5}yHPw-35^&fZ}_Kcxhiy{AfVGMKte#N7Reh`|^8h{V2 zd&keh(=VEc4bH;1l%czkJ1Hf6L%RBkd9cnYttav;s{VZ1JNi?46EirxQx8a@r^B;F zM4e-oSzisU_vcrz4TUk!<+^83=W^EVyBCxPWlg#N^eXm%`1kK#WnWJa?H?XKi0RK7 zZtUZp5faJ~>my*u1q7`9U1&4yJ+Wa$k(S|QbF;1KBq*7hn8bFn0hz8)z<3%4!qv?r z1R{?<3mzz$NEbCfJE)8Tu~%2(+Tg)1Oz#QKD-(A zO{HUS@O5lL4$B4&?f~YP%UdNNI$|a>{`x~x)!mmeqMi6AzbhTN1VccZ+3(LdmvAxO z07X2R`#(z0;mwz?4od*jU{*^)MH7-{MxOL3%!4<1HYTI}benhi4$Ol8u;(uIF+mDl zKKAx1W}>0m{l@ZRUt&;9vgLUzZLDJaoJ z()%&kX!fZ8@t8vSuD5pWWF>P3 zhgGqwmeqD&ihUDQox;P;;NMaljjm&PZ@&a}9bvm5>Ci{ix%fG1V$VV`Y1-!(5~JxKtp7BQYBj&tD?Mo!!{c}d#^cri-3P>d2nvD8X*=z_V}CJdcuE9U zhKYcS4N3}S0-fes*-LOzv++;WyvEo=Zz{|~Z*HyU*Capw`)zmKz!e0{BS|P)1jTh5 zOQZWHob(4atB%2NDBwbMb0R z7WjPKc}6;${LPL4`sRfO2fDa2_q4awd+3+{t)A%BJ?U`zAN9n4*<9-1>PchAN|v|Z z{P`E>oBg@D{^*8(Bw8K#qxY8|D3&!s6TYsna+7P;QM&lp2yioWAJ;7CLia{g9n0GTs$%XIyeD-SVZ>xJgO z3v%AYshL5^rlNW|W2L$erq7mC>Z08#Mk~s*a-P_bDOF~7SWJeqeD#NN5@rmE*0u;`Rw zhK6N&xw1X*`loSq(GQMjU-pEXbPUL$vkL{hNZ4ZS|E>>D#_E^;W^^`X7n#sPXopR0 z5e<4mg!BpmeGgV+Ve8!e{n=-WO>`ZQYWI-AC-idiJztbw%S`Y*7>t?jLlJTLgaS2w z?x%%B=eA`7g}_ulBHOv46sTI~OMr9RE@!JoFidb{wCg^}tv} z+(Xi?m*}_*VxG)d**)6Ik!B;M@aNunM$TpQ*@hwuRH%UuhwSf+N;J6Ib80%nJ`zI*~`zEP-*%MP9sjaTynl&)`=VC1N9Cv7?9@{KPooEJ03P!soMunJ^ zTkWnZ-!C7e2AR~Y5*3*&JB7K;*!IP|Bu3rxF}*Qs4Y!~DW$`j@HLmutI3@GX=-ATj zk}(OMT9atV*%fHdQIa+|7$<+_RI(Kk{0apawv_$`S#BEOOoBKJdr)A;#387p5r|8V zpLi__3Z(y-F@P$?{R ztFERWy!4bPTAD8OCem?aC;&!UsVbXhxQ#W3CtvBlcm1WHvlt~`?v4-JFY58HDwQ|^ zLp56rVFDRGsxApVcOyIow0q7$0~P!BRTX)BnB{?z_cW5j{(Kk?-hr(|vCPJ=3%ND+ z+$mpI8#`ksmCj2t>f7mWz>fV=G3N+?u%k+S4&9p8FUKue?Yx0);*2L0aT#ePyM4UC zlm3hQr@UgujYKWuxq=y8bNq4Uk-x(!7djER3VgGtO!9%a$9{c_NC_`j43=F|MfdI- zr?OHWGv7ZznhvwwNf?uz7VX}NexkQ&ZS6}n(h^wjntFX8Zke4RadAo+c(uN5Z4S-M zo~2q=B|+thFqqt3ZBwK49{)kyQZ@4ILL|3j;QGL7ElnKGe&o<&A7%sKkcrTmWu!Bk zW}#!0N7<`(mr)MnI+H8qDlW_XD5H@dw*IE9>OO1j%R{lFu^~oO)H?)a38bpNItC}t zN?z>vKFkhHfM2g+gJ4Yo$?aN-M)qru$WlHyY=!oq&6M+ry&|#ckn0DXQ~%_`#%6|; zr-EF$RYTr~aynvT_Ih~toi!a99aHl71iYrI=3Z=V-%v#X-1)!Q{C~N0f%55&f+>a8 zb`(DW$i^h^y#(Vys}9y7uu7^17`_9tl&ZSo-!!~mI!R;KI%x4>%cpeuNY#eq z@eDeqU+@zrnV+X^W>EUxy{@haHIY}TobuVxoiu6(C6x=>ebmuVDYjt4;?}MZulh+2 z5uDE_+Gta~Bl!aClP)Qxo;fg>FUhNQ*t;xWypfc==rwS6|A)tK-~-_Ha#{M_K>-~z zj6WJuB8q_iolH+b{tn$K+lll@=Fa8+*_$n@&wVV=hiWgb$;*{d(CRXF##)%h%=VSf zH2toJTS6c(S3A<@=eIfD&@~togwUnK_#G)?WegOPy(%3hD)j3@r}uh+vHe0`$-)pG ztU++*i9K*Ry$$c4CoK;nWe5>dq0?;I(f@2yGsnrinIY-SZs&8h3uZ)hTx^+ttgo@jbdt5Wo2A0dBXIuV?QlY~0;Yu>h; z9aR-bjK!DGc^d9hdL{Vt*PjF%4sF3pwQ1;DS+&p7?&WdnDB}bgu3~`7bj}ELpfE<{@Q#K;K7CA&Uo{0>4QPmjqxGB7M38NSW`M zEJ_6$g=a~oV_|vKPeM=yB3)FSo<@lq@Zi`8a>Ajxvri>eRXH0Yib|FEbSwvu0LPuf{jI=Vu)_CoaA~nz@5wJZowuy5c?-Pj}96TrcBZY zDtSl|NW-ZadBX`s#ETe4L2H(Trty7csLw)+C&F55G;D0a;nU~Pa&*n$@6Wirui0RQ0EQ?UqA&fbY6(Ib<7P#J;YZdx27|Z*VWPV}(xy<|Bc{+~ zKf~psV4TpOUw5q3gMmfJFwZQ`8VJfqn@Xt+cE$*KRW$zLy{@YC%Jz@jGxZDO5vb1- zRjy@lgQ~9S2e(`%9bRgVt=|~WaYS(bA%?Dr&b4c%*1Gus1Y!^ctScLVmS6YXDfio} zY0s7EaaAZFEY7aS4^p3^(qf(&IhK5VN{IGf;t()hx);u_aQp38kiKD`-2HwNT{;YB za$UxqL0DjE|NP}Er2^0ccv7XOqedGbKgLSp(ArWs#Ta)g>R380HJ4bPQ4yZY1mUtO z0Apd#s7{U~m21`P);aOmUEQm1d5Nk>Q;{?2WId4WBfPA1fUX<_PYoX3v|k%hX+){_ zApMvf1B`GXP{>FsfYv^R{sIHmmZ?WwhTEx?<>-7G-y|??Zm#&Hy%c5pvAgreosTDvqg*e`AqQ@8h3W9RIj=-UZ3+hn1ZyG6kFuNtx#?4?z=R3+Vv&R3IjM zxEUL3WOBLk>)Z;N`WBpFRj&0^(ZeqjFZ`~$Zdv8IS8qlxSK-5U<}&5i#sKayviFju~!u2cy#Pun}E(6@FTDO7@0r4g_zidkgYbW~e_ zn4OUD`V&@^0NdKN#q^5l$~L77*>#V0(!q*yaUBcb3FU%;X1vLWgvbj_K9+p8h7 zm?;#4nsHvUA5>AMXXnc3j(7nEW3kI1EGY8cO0Stwpo*~OzgqvdpN{CpVALY52-y30 zaVh@sc_EO>d<=koB3G1CMm9;APXF%t%Dv0Xg5SW=Ar0LRwSHC4+FWH7mc!JDi|Qf) z2&$K{KdKZxnPM5~$6|>JF$<@oIG-*UY*$+JL^4+>!_MoH6Q+_E_SkfXD&LJOcP5Ir=d*Lpx#>jue5gwyw4dAqIw0Wwx^ zu+MBV>s`2yYjuFY%`9t^CUrQ-$-}B5^C){|?)AZ9z%t6Hd&I4&=WAd)E*0AOhre?R z94!x4T25^Y%NVMq)+L=3=dJ0lbX&3JR^j3b#tRSjuLC%fybW=>0l1nnE{w5sOcc0o1C$`Yk~|4jGq0K+_WuqDC(BlVn7BWn4}&;zZB%A%yDp7Q z(BO~2kbhydqSxqwGK+jcJlcJD%b#%)p})aTHnIPB9q4~g z^Kjz9xs)?Gr)#UDN0b)4OR5I@AO*+#tD{OksJvZqGJHM^os&4~_qck0XPvBSY|oey z)}!Kr0yyLcyt1OBSty;=Lz`(?)b;XiBQTM-$k_mjItnEZF6`RgBd$uN3F{Ol6w8VX zGed7ty{p2l+znK4_0Jxor`O)jc!o3mm;H^5mQ$m9J8q`W?B8Dx1vXXO3TrakRiS2{ z-(LS`grQE5=5-G;Jz`M?c$jAU#F>I2X&A&+IX54k>$Y=yk4Psu6~(hIjw146rI9K!s&NY2{E155!_gh7yE-4$a$yEVq{h~i7KpWP|T zx(&K%G;C14(7n*VxIwzE-|KSmiRC5m4|w)|6#9UpX0fD=)Y~MPZ-J+gmjj)1G)JjA zVPVy)Dx@AD8YA_bK3U8<^C?TuoXLQI^yl0^G}hz7|I3=91R&z+z=Q+*J(&Ux3RLSf z=3$jZcv~r>PIM6P?^ogY)ws}N?9*xT>~Nh*U=TtynZgd;7`VGLK#9`nL4v_2ZziPT z;85bXN@rQEE%4boT2r7wTj7END}h!TXE=H)EyPbZkEtDfSMDmD zEG}T41?lQ>GEcixq!ni81(QR|^!SH(v!MW)`}#Qg>BkN){Y&kdW6_fa3-4s>wMGGz zgNDXGiC+U-1%}@9`A=3JZY?gqf1|Ts)JsMdQ%tsg+K87zIB>RY4b z3gVBZPZ(aEA>9IrolR*ve9|htuZ~M2OniVrfI3Rk#fx@8=0NAwTe>KAyxirWv}e*C zZ)jzG_b(Q3I{uN+rsdNlTO>mav$xSExNklXno%z|ZqIJGXj=lVOQxQB%-h)McO@4$ z&1{fFbauo}7JO|8ENQwwP@|;jy4zUq5k?jxqv``SdVmaSVPM@1I)NZ(25lz1~izZGpKprm;6Jmd&;hr=n%O55QRla8$`)YY%(d$GU~8IMAKk!Z?5BMiU-Nq^3ZJl(JF zU^P+*FI4p=kS}kCeKLc7Kgc-hyD}fm1#0e}0S$hpZ?+#?=%x`XcEiPqDTsR5P7oQV zF0)biY1^(Z-Ib~(2nf8Gzy}7Y>NGmYF%2b1g$qp5gtV#`!~CEGb5gG3sQ6YTpI8g+9qDC?wusu*jA26}n^^e!%pygAU zqn9;3L*rfblH|?V*_~t-k+1SiF7OOvnb0Sx8h6aIS(BlarsGw#!@Kwz;>s{lV0`TM zXWjMjNP~J`^nzH%?aK+pg#*Lfg8cA>k(i$uM?$1ZwU_rqQ@nlA7faVGDm=w#esBNg zJa81F-zq7*VI)i)$h!HXYxexdw4USwWEhiKN?8Zpq+!3`ZTHn}@dR%fhg!ion@7R9 z6G=+aZ{-a##%XC@yy?%B>2$!pNvwBh9vrzY88R5*K5K?YRE5XM;?Yp#4LxE6hYcM<`7n@>@%rm)62JC0-fIcpBA&ZWFKrOala$A@{7Vt_Jb$n^?k#6$oo`joM0xqYKDhRY_9Xv3BcL~nBx1ovUg^yBHad2G=F#W-ps{QyB_ENt| z7k|CDyl8Kf+AFzlE5+qJYD#N=m~(7XAWs>rp2#gP3gQ^ zs6zRFF1*xDI1zBQ-h1E7{M)!|+{phGjo)3Vk5Wj9;b+5bYee8@4Z{=vKg&o4-itZ* zrWMbd+kx?ux@L+y=Qi6&JkDI_>(#i{EX4r2SA|4@?V{JK04q)!LQ(akYWw}qkB>{= z)tJ0%w8}?S@f=ijW>1nT!&K7ys+Tr9wg?B#Qq(n&PO*Q&BD0yH3E`31}jYaKrDSo34WaZv7 z#Y*H}jvuJvUj(_rxBemMmq)cu0DxoCS{zRCQ}HWxritAgEcW=NGOo(X&Az?XHq`rK zqOHx1DYyUX0;BMGelFov_kj(=jo!I$+}Q^A14$u;bp?^>@4)Q;$JkqiMg4C5!!p!R zl0z$9LnGZFNH;Td3R03vNe(UDEg;Pd9g=H3{aeLs^vgOMDdp3(AdTBXPl!Xj^TH{fg zR18GxNK{U{@%JE;e_xUR`CxKjOCKTyXX7F6fxXVn*9hrSSMAAM=yO@QV01fP^~pP? zB`y46FP>`Ip}Hjf-(CP6!~qyVlX~axm75Q`<$_u@dly2e7bqp(56M$&7g0GQ@3w)NOa^tfPNS0~4uUWK zlj4!(j4Z*7QoVM8r>}b4CM04fA4Zl?&Ji=Dzm#9*ket(iTnhnE215zTAC609-U%kD z5BBE(bon)p%6&Osi-yJ*knn=ExMfpu5g1qw+Vq81j!FZAakP4yL;k=4csRXF5GP2W zmj8~eu%b_qct+W`48U!nF%Ub=W;RO09@dE^w}Py|mHRg**m2Yoqk)PFmD*o1ZoH4` zbhEWZv4<)5iIblHINROUk`Z>=n?E<+pF#ANDt@Q1Od2H~Up0#_*teKIIyTj zeF+?y3h|{gKD7(WX}LV3d^Mv0(Uj9?!;=lWpgP`2$LwE{PAxbqab^*PZudm}7Sh#Y z3mJVnr(5LF`O)`TOpWu9sHR`j>nWM)4^5=vb4yIk5GyDl6d~`fOZcE7)lZf( zRyQ{JMy{alsduEg2mY&81_Ldcu(l%J+n0rzpScf-LvnsrFAYG-4_zW=*U+3_NR1Q* zzK_XB4}zflNdD>N3*Bp~FH%XRuYPK*;QiyWUEqcUhFPXpcJE#Y6pt`(a6Jnz2>JOe zs;2wjVseT9D<3D=^rr4V{XBKb^AV89$(>#VFW^J*fd)LD;IO+|r zdLrXpU}E!fh`hz)hZV79yoVSdwNmy&+-qNa{K@)PJC6kq2g07rSL-DsLhW=y4q>>Q zH21#iWFcUl82OwaR(Y>bRe*M2pCZa;A>5zFjih}#Y}qpsVi5O1pB=5PSnY~attyN0 zg(l*-Jx8_+%teC4T{6S%G9 z(B@HVHd4L4G7SJzp3`?_aE6EvzVlP=RM&^e=yXA{^Z^np>Gk@eU-FGdAe|@ zTSLE}sVAyeW1nZ~?!EzY$~4qHCx>aBfHIUZ84^ku>wzM^80Qljv)tb z(QI!Sg?05+elVFZP$|nmz4Nf$Ca_XLsFzry8qeRomB=3<@7JKhonwi#Ke=b_+fkyz zp;ER3xIW=X3aB1I_u%4r;x%-}UFEXnAfMSOYhgTXmJW4)>AJH%-k$RE(JPl`j`l&a zQ{UYYX!|So(ob{H9q-D)pj|pQR1R0wbR*k1w1fNiI^~Z5Hb@GWSPmt1C^~OBs|p(2 zL%w?sK@!{YRnTDZ0Vv$-54bmF;kgQ*=Dyq~-rog&CL+Nip$7w7F$J zB+5BX4A=O3Pee%Go$S%;asCK(mh^Rabr!YgcaTRj6@eYf&F!aeEtkH%&W&{UCqf9G z*}Ym2o!25%9ao=Pl#@&L7qu_9vAGhSx5Bl~82R$z2rN%aW8iRS|FMR27U%(~VR zC=NtQ6)tF5pFK*t7Xih02KQ+7pS}P2XP64}KzqnzrsmcbFy$utk5)mI0-*D<8hjXq z)4Gb_7ur4RFpwV}3_#Dg5r~G0+?jT;<%33LGCHr?EiBwEBQRtEOSfV?o-DMINwMbc8>HQA}%P7^YQ!gP9kX7 z@0XY?=)&da*mF|R`5CSh@Knu|gd)WEe7!Ek5aiCP!^UJ@kWhkhPK@5BEk#^>?-c(t zQh(FA5Mr4U9pumQgLFI7ZX1gYy4eF>-Rp-eu*jR^Mu)K1A4PuI3N=D_7*OD&z2{z- zPete>kF^w1&L2oCrCnwz-yIn!U>wwhUSsxpqTDO`CE!}~TUxwfCYqdSu9Nk{8y`tq zuD8Wqq*~LTW)xO`Ch$yZM5JFIAbu)x<_ekEG>bpqF=GF(dZv@WOA(DZ#+Io&ikZPs zVk73uA`K-B$?x7%CvKYV6&1spH^qofwf@oM{TU6n3!9!4^<+a+x`2|`T-8?*ikA;} zA(j3fc}FgQAa^HO8>Ka{mYHxsS0wYoEjR-HwCxIMk zcAUg+bI!TwJ{^4dP$8P^_L#9fbc97WX|}G9qe_$0k<)wB4K?}G_MkEL_L@V|fTUM2 z0_{B`f;ZDF~*bq&m(H^w~e zNpNT``V{ctb+Ijo<%GrK%kk)RwE4vjx#S+<-k#5I)jsk4OavPAKxoHjGItCUK|_k7 zvH>lFBKr#0^b#9+ZJu?3;kAhSojQp!y_yjYI{?Fydy9#N8>@Ep4IyzhSi}Bv1kj6} z*NKb@!S|9|J1S6X@Cqb=xrk%SZV>1VaX(?-Y2$&@9;UtNPR z3Q(1*YjBZIlJR13bw(EM@`_!6Ub!@aUrUXuG;S=>)}1orcZ9JQZ_D7b?2ut-s#Cn| z{#xjML$%anBcb-yC-u{gyAq>Cv)%7YRfVp9hJ;xI2A+9EKgcCKstIiGXB-Fkz2Uto z&L1&8+c1`Y%+OtKQbx7AWA%VkLYemff-Q>JQU_->G@nHlgTp}=bZYW`y%oh*COUbm zUcc(C^4B9By-mp3c?@aIiz))e-x^NC?Cda>tRlMu0GVYdT~q zrAG%H&}P*|ydPS)e;328FMfT`Y5!L6DX`ybh=#}Nr-3AL^x=8WCxZiGC-YMa{4zFo zR$oH+kt5STlTmzFyom=5lA9>$!mKv;l|KmBgp$B_O*@cjkS-BNgHV_dXh%4L5CAKa zg38Ir|2DVefbqaiMxu(4Rz)5tF06&%SEs|XL2LyiVC)=@aUxeZ2d$YiWv-0wB(cUx zCX=Rz^eB8T>LemfAzQ(q6`);((D6+W8FfzCzq-qz=`GAZb;V{?Gmof}V=MC?Jb9Cm z%&d7Shw?U+SyNG8qzqV3iLz?gr!y$yKa8oKDWfCTIvAykp#SjX21{>)D*8D2T%}Ll z^q^tkiPSD!?yxJ^*9l_`f)}jSyQ|*)Hrz<*p$@`KW^Ravw*4V|N2Tk_Uc>r+A8WV- zXJSgf<9=4QosAj*U5Vw%3H^ZVSVQfO=a;d(`4HRCpp@i3QEe(+sklC9LXY2`{xVm! zzzwUhquz!J{p%PVrLm;|UiE{TFVWn9zE3#Br4FEA-PT-zfW|I%Z zw6a#_9_DfY0^E@}b~63(p^-DL-Q=ne-oIMuRebS{G;Y5K9EX>JJ_kQ>O_|)M3ON5t z)rD*RYX4TyXn9i55Nqus>Ee{h_S|$ts;2@zOPibZMbdG`i+U|dnv~{@nA#9OyroLcUKYAoz!g-(@1KVh~w@W z%5Xs7&1+N$h~`CGBe#V3Wa)=$Ykr~cg(H*iLY+(k9iM(psOwAVqb#2lYP!S0(K7vA zJ6&M{jc{a@qqB*D0UUj;Ua4GSy1B2{o=#3{E|-^k%9w9CIVoxf*vwwL50g&6YIJrF zda+bwwYnyN9MiGi^~3V3f1 z)-BggC_koM*$#|YMS`8)nbIBj1NN~nA3hydE^RcvHxWj38pJcq`UkSfYHpdGTFahd zIdm@PK;4_FXTf(OW$=C4`j2VL*MQ9*2{eQcjEA7HpKJ=HV^F$x6U8~SKTmBA-Ilhh zFs_iw#DI4!dU>8y`y()5={RRs@?-hAPDf;9%0hnkxw}02a?1v%#A?y0)<>(!K- z@Ef@oMMsV{2f}!7+R$sv=`>{An3|<&#t>74ZKswuwgRsrgX#Oq-dm0&R4qg@mVa;a z-G0@P8n|_>^E8vU<7U^%^qJhB0{X_K%TMxY>@6+qd;SJfsz=Q?`UgihM&C6H#nT8b zkfPFPOt*j~fD7O+Py1FewLc_zVIIa4gT9OECI16%JS{Z!Tp)fjO#V< z#lMMW)w_Ch?du?st?~;m9?g1yN;Ht_iem_2+AbR|qE=Mgpf!>Ut%C03= z?qO`|&0OZtcocdad4-YEare8`W!Wz`VgVR741}MW$7?`$D{r>bfH*j^`GJCBv)nSa zV~rjUy9f1V^8LyIGLD}=RhkCG!|vmJBerjABMRnyHxv`lqD(X57GxBa)h8V4GM{BoR{KAv16r?USBb!kf9toiWAxM!l0h`u#PaKFqj+jU@lxfzKZV=Y)(xg{3=9tuTp|7_VrYUSKn`5J#ynn3SK% z3v3sx~N!!?83JQqhhkZpILlearG;pqj7*eul!DD60 zMRSX84=W(#0CWK5k7c|to%;NC(f(Y%eyRS)K79JJ*>zQ2QSc?4FR0@qCR6fTq6^P- zHu*_YdAq~PS_}887-%CE%3|)pQuT~sA@=#ylz4!(HOl~){@xx9HYpYihlIOya}C0- zOjGh9;GozYWwV9Z{=0EjGrz#dNx&s8QEEpXl_3cm)O~xX;UxF6Wq^B2H)Zt8Dstnik*mc;dD{V*W!=tW0VMvtkT0jD|KPZcnXRzMfzs+R3JiY!=&^ePHwU8Sb=8 zF`4+I>ha@B;kgf5Ao{@m<8NZE4O`V8w-S4DcJ#9zaef4_KlS^6G_sI1kx^gyBvSbf z&NRfa{pUMQzWStai}&@r(lgOotfz_6;T@!fn5f1bVshqmHD$l-vjsbBtlHl7}qGL z=`7Wrjv;MOfID3ic<+w%hQZKj=bl`@hm0L=v~ZFiprwK?K)K$&(N|qcT3|4)C8+9U zBD(=wlmr{#;{nb$#$DOkOguuP-;6ROBbaAxBUuD&n{9jEX?$h+6$CM5?}W9tKt9hy zL;qvfgOtzu|K0oLQk^R1(Wf&C*R5+g{v2i#Y(4&{`g{|#=bbsxB}H-G*7!3l!LR-k z^DjJ9@KRJ)C%tf}Nb6AayCkD7LwZ^H=jcD!5@$E)^n>|fDE>HQg{-BxWHJN^Ip90C zcKkm+drJKj!jy13z|gJ71sKLI-#RGz_uff=1==nsa$lOt+k0#_5lP4}YnYqCU!mIM zZ#U52eI!7ETPjd)NYw@AR>cgCQQoR}3=F=c%kx666Z2BlU#|&TQCi+6mrHY5p0Ca4cSnh@t ztGU_`Kgku$AdlO1!Y~c2Gg{}ei6=HOFE1Kty{-SQztwna%Frk|Xd;%;fdrohnvVYI zc=vw)pNvfiXzCNQKCb)UNk_Vqj^JV z_PhL3@0v%a&-S5wvCD5)bb3iv|*_GzjgqkIr+2N#1icH=P`!hILQ$E6B%9_jIc>JHB9#Cb_!S2+j zxnX_eK-wtVJ;_D7T@pfFi#5v=djz!fmvYLU=ncqd3pUw$))?MUMH*(QJpr`X9R4w} z{)|K_orJENqqj3yWOx2E;RU<=*#v-M$l$d(#7$|QuISqS@jc`3DDVD<1){T zOXwB$?k8`lPyEw@0ehH*ZhDX!aOG%@Mj+BC@IQJ7q%}^!?QpAu@gk)iYE%@pM*idl zm0xl>J@QFkQaI_adYgzuNYPW@7V!ztJ`XyuE@yE37U-4)XiW?W1o?vGs`k^Ed8WTI ztL6EE$c_BHz-LE%oy&{LcOERv!;oZ?B?O)PVy7*2ZJZ+?S}Xw8f^id;yMUtz>Q6NC9l_xtRQuYQ>7iUC)A?BNY=>IlUV32^#!Y zcv)FXU+GQEvCf;_EPia;W!|~1T2dRR#Fz{4vEuvX#SDK~C_{RzH9lP|Rk->bp|huR zqAn)DevxZT_o!n1G_JnI&~YnH7qL?L!h`5#Jv}qIjlfH31Kf-h?yBf|23#KRS&n-i zKt7kgsT*sSsw4A>LF3P6UUuiu&BflErNs! zn?v7p(2%NqQJ<3{Kf0*@84Ob*p~@g;fh^8`rMd%^SYOvIz*DRN{;0R^tr4`~X3WC8mqC>#+%^(2IjVA`Ietrc3+(U{T&ovOwnt)7gGxbRfTj1S;ZnJpq7E1NguMw zET>O{Y-BFr_n@0^zZ#Hzxq1i z-~O-KKdkedPO}j297$#KN-QsYQ)J18T5@zP7Rx^NWS_zP zD-zPIh?*8BDgP<~fZC3Efy@mksz?nSm_e|0T`dZ8=33B0JENzmB71KNlB~zKT!V3o zDb?9gW(vD+xuVgU@|>TDTfuh+gZCh$T2+aWk>N*2y*7VnHmaznNH1l~ZThlre`Sx@ zkqt!V+APVB1$}DGxo|Q$?NAu}zAX^}J+IToB`E`4A}+c7oRyj~nQHYyaOJAna#@K{ z&+N-ARBJw5C2$rfO8N48S=rR0s)+((!aR}cKq4VGHs=7F-SwzA>=d-E7ShSfQ;@uN zf-eU%Qc}wu6-2HzN;~eQ8aM|@Q@_;Z*1${Wdz$o1FZZ6dUtNmuP02Ub+owkz+Gud$ z-d5kye+=7n|KAPUGMq~?`d-?w;m`kG{Wqy{)uPUf96%h^`H@jS#S`cCm%A$vqCds& zBlPXFQbX_Q+G*~5r_k}$`PP2uk>WLmH3ENlRZbVqOIRsLtIzqCemeVl`}h_Usnk+G zk5^%1O<~{24))(_)M}v*?84tRAft%Fz~J3rkCduYE8gqL35aZ2#HjtV-nvLJns_pW z^uT`;Uk;Ew9A_QtYaz>5-aM0O-;IFy_KYMh;m0XG$pB}WjQo_zZ}A~x_=N8*U&{0w zFYSi=d%`Kn^@P)dE+csC@x;Q63Gx+Kw&z}cGZYWkyq|?4v|s+AhnBUorub+%pqfCEv0NpM-RPM>gb46+bm18vT7qT5ywDe;> zf;!#x<9UR`m{h+RvCeO1?Q&ep^uPL#pG|w`T#}Rg5=b>s@b>t>g?s~$L=M4DP*HhI zQ7`I5W>}2NFMx1ueUUVq)1q7Mw0!W00mcc?X91_*Ba0SMd^sF^cPeU>p0h2bF5nkS zH8-r<-0Z-Vx>Yv;!~s97)PSicTxl3Q&O>X5vRF&=!zK2)RIZR7yce+&qj=l&)e`vMy!mk zXE6fw25_~pOshQDg8klFoDy8E$K{#~V@U21-M8W zO4}f#LKhqgE~=KnaIAP++l%!|`Vt~ENk|%}*{dwaZcybEYYXz`)LtSljpbm+rt-7? zG5hPcLQ{6S&$ako15tieFelPp-kYqotw0&2B{oTUSH{~t`_%9vb>}8>yV1~b=)c4X zqzj4!AW6D@OpQkXQ5Hbr@U^q0pZC7KUNN@B{K@7kF^_RtYuXg`QS38}+wV43@k&3~ zTnx#vKaRf?-|-)-EV z*QO{|NEI&orey}vd?NDLV0bynaR?Sr9Hccd+W~QBtPbrx=BMIbB_CtP$&m* zJ%8WhEZwpv-C}N#5N9ZPo`M1#O_?2pw8ZuX2V7~Q2%4L^KLwql|dcgUN zf_B@|WqWMN@Tsd4o(8bI)~`W`2O>3pwX$OJgHaA8+I#0k$M!*4r0a!B)T$o8J`d|W z7z#v|NflP4Xit>}B406yFFBdGbL9aq;CWe}%~LS44zBZ*B1bUl%PGh%JeA~>&Fr|` zeYba761nT(4)xpmo%+20AveH?!#jnED@dMI!QLg5@6iutH&OWmxSJ~ZVMq6j%@L6h z5nWTN+S&wu0#*R1&aoBUp+5nx%YB;6Y%|{p*QV@DW)!5$0DRQ7i+>tlBL#rwnHFxC zA@AQZOC~xI^p{mPw~Re?z^9XLgIXM}q)cV7@ac;me8&OJA58ZK`FO~Ou#ve#C#X}a zZ5|(xvgBv#t|wPHclboS87LtVBA~auf^CIR=3Z@}eR3Ygixrt`7vI6CMG6u05k}is z82;4Ud!)?7UQtQvzoS>>iri8LbYL?~**uL*`|3}e3zFZ1UZ#&9eONAjyl^TL&UYP^lg^%rzCZ{bEOeAAn-*!=T8t7D5V{1=%pU<(3=M$L=v48=VP z8sU5=LD8wA9H**77$Dow1^3KbD6$b<^h`7X9b^f4Oi~YnYpbHYjQ;r`D@dtT(B$*0 z6B$E8mw61X6yNHti@SS4v+D9z?Zk>Uho2FU58?quiVUR@!=OJme?F=BD>aSX4V&&u z8S%&_Lysx;75?<@r z>sq0|AJkK!*P`P@(9=)GrRU0KDA$fU6Fon9Pi_6og9{77zI{3oFxQd!pkYDM7={1A z(I*s3A1(UbvZ!xmDLuDzY2xd2?)halycm`G~ok&s{0qP%_*Uqa5A z$86O|f{q{%`aFiF{1uE>^9H0Nx3Z-=ckpK{G=qP}wxsKArj@p{gQUua`*hLQ^NY%~ zpPY-LCT1tb4*wTTVj+ULG8x(tY~RF8v?6CE3FYb=N!L4MCVH8j3eK}#3bgWgJuEi0 zO`oh8>|(Bn%UK?3i}_)=`ywZlSPz!m18rSlmPoKNL8m(M4x^=0DP;FsfMgGxbUp|J zyz~_sNK{Sph3R|s%JSJ9t53-TsU&U*Ml>9ot-`mga{o)w({{l8H>48iRg83*aY=K; z6OmrsyBnTL18v`N_Z^t5 zO|23Rs5demHS^$z3<@6Fbp3R@lwtO)0d*D#?F(o`8p`?UojD*Ec_41IFRdZQnLx;S zK38oT`tPr?d+1sW zP>;Np)!18oRr$6$U4hs;tqpvlyof1D-z^Fj3-%wJMR-$YII8Ob2?U4dckXc*^XwX zUy=P4+f>;-Qdg~O{g3|Zf0Rlv;<&&FXuzrK?!i(T#hrdlvwuNcBE*UfHSU5-b!*@^ zW%O6S#z!h#jE7aN0{TgKqB+c(p<9>v7E26EIYq8qF3(LF9DQRglUxhFFk^lzA`FR* z*;Is*8?-M=e+B=!YxW-b?acG0-&GcZB`eB$}hH%DEoXv9d-P+DonMs4!N2{!&90QH{Uv`leV5{g!mqQ$V{`>O>s ziQ1Yc55|yVPfZc6Uf7Zju|I@c&MK7Pf!Ps_PS3l(VV`E6T@}^4lcf+jvL7cS79t5)p*mgTb zbt|+-_yzZ0z$F8<_harqes2dIpl*8`jF@*?uN91t9AI}9LSaM21KY6983v?s1+zo` z=P&C^$NUuksM%ISQMBvRvSPxCt!H6e7!}d_gq2e>U%G4aMGW2(0y9UJPVdepoV(4z zLndRmk|TpomK6~Otx-Uo$=aU5%k-bRlHKz8dkrEsGKJtv&plm3HXua#BSzU^Lhphk z`EeDAn3=UxAsm34br2Q!+{qN@GRJp&aB({4an4lZpj|3p1@Dv7sqBhEK-a*+Z6OSm zPe<`>hB<9NB*c=TfkR9WCavD9e9c*HMxi0ytIR7a6VIv{p?P5bLOEy)n)*u4Oo4zp zt1VbQIbNp*j1?x2ikWuZiTCB%|5mA_YomRK3@fvtLJAd;;zTcUv^^wq@W_XZrWh}- zwkqf9-nhRu%@{5{1`y3pQ755f?J*UQ%H(L$*A>@!Nif zEh8B(^e}UiSHwmAi-$cA+7=4C)kWMgLrCD>RXVRy7oy8s}bv`hEF|eUvS-5YY z@{=MUbMsN#$vNFR?-3R1xviCuOSrrlyXG6BhXMHs;@7VW#e5a$!?5phlT*g}5h|*w z^kvKqGOPUiTjZG*{=9|i7x(sntxeJPsp#_$cS22f&uOexkXt^GqU+dF<-S32Zo@Ef z`M|2*X4fCUB;{TyuCKzY$C76J-hK=WrJ_tsS#rObDV`4gMJd`XCiZ5}Gobd+I}S;w z{RLvh8CW}1ZxVKFFv}afSp=jh<=&t4jZ{v!iXnA$g;EsZ4v(iSVFePcMfQZc}KkUR8eJ2Odm^b;{8r!z{eR5fvF?E$c%_h+|SY<%Jg=lU8Ga z;wqd2iBcHT{WiYKAqkF!G0kqB^>-|>2e?EuXFu2OPNYw1+}eF?(@icj?i~|sgj3=% zDeO#If~6J~1x#E!DDT)DvB4gld^267>hypQqzeUfHuJgnMUV1{Lyw20upJ}^{HQ&* zt-7AvUr(J}O&@-3jsk$N@ybRMRTE$s^wZFQ*n4q4JM_Ask~#%{L;*N6?|9vJ&MOD` z2g8C$f3SCd2@of{-BvpDreG44$7TdU#?zfsQSNRZ=mBj zmKfTuZ%uBWm5~r*Tk))Jcd%&Cue$l*dB6un1pG;TtfgmGFxk^CqIMTM@LK=)WLfMb zjq*kFp3gwH>U-{Y;v+Xb^p6ocyG`FRT1jA5-Y&VO;v2lTqkTV*Lnn_4j0L`Z=o{Ih zGgw^pszWDne!;C&!Szj1%eih>F3`rYD3;ym{{>U~{>1FRm|D+Ysa*oSJ?0BEW~A4g z{<8J2s28ksh3>*-Yz#{nr&6d=EGAL zzh6A%1BIlIiJMhCl-3#>Ki+S{_g_b`P@zaLYz`5b{+U|qL5i`IfgJ%X4?j-VSC7&( zDyXDO3tLuN*;ovi9;JMor>Xi9-r#NN)kNmo>Qq~QiTM)^+l9tukxt=Iv{On8NSRj$ zA{D(mWHFa>QeAqrl}j|1=Sx5y()4E2WTzO=N?3(|<7B(-5QeqNRJx#iO(Z0vW?Sx! zu64qNj5ftZFR=$70|Pw+gF2*$K1bL!=ZQ9CN`khy#UT3HA$nMrcdOY|qJPYbGpe&E zGKa=AHnwG?K%HTDZ^rK)&(q8dM|S&W96kv$X`RWf{=khE5=cM*kd+1kVJEAD!MNHv?Cp=CY8JXWDa>dgofL%CICn)pUNL4>5J*?t zaAM`Yb1a&Qm7z973j^DI9hCw)9_0UHoIq5=Ki9T5tIuDq^I1&-0?~ z0BO_QhFmwE%RSi{a1-9}SX!Bq*W*2VC`} z_{R3Sr5}@xJ!+oW|2g`txiwRR-x=S`L6%m?et+BDOUqcUW&9mK#?HB?)9;)L{O=Ay zA)RFe@gFA3%rkxH%(wt0?msZQ?B`zKd(fz}uQ4rKZGpH!2yLVaiHjOQuMfFQXjQ60 zoS>~nGW<>;HvJKQpw56B>wQ+=<$8wBJj|%c>q79;9CMwC9yES4@))A`4aLN~`rt>w z4Le#fedkfs{oY;fb`$eF2K0b>$q1VxGP3Q z7jC*L#5VD^coJv%c;5O1i2XN8P>}xsGGv5fJ83#*9Ox=8N<14~GNlG0am3B;0-mYV zjs?x1XsAs05guPWv-7OWXvPv>V)6WxR8g)%MNpa@37Rb1zr$c;Cn`G*2+06#X-cS; zS3f83j%$Hb?MXL-D`9z)-vqWaMshNaA^I85-Bo8gUoLf@w0Wg3vTg|mUD!m(7po(G zif*K|_iKLwuV0o2uvU+Pjz13M;wePqzezkUp330bQj zd{w&F7uo*6IU+@LFM$Ub$*D@6ok)Olz_3leRW2KHf6rai#NX2E$;(Vo-fCL; z{(0D9cU0!kqZXumw$lbip0vv-(q49OnZ=aZc-HlB21e-S9Fs-1UaZ77QsOANScg}< zf^>;uzq0b>EJC@N*^28h5|O1<2wY~kMVDo4CR{o!+Il>J;X%q*n2!SzjVJLJKE@bu zZPjTdujakP`GI7_Yt{q>NdK@|=kMev9bZJv{_UD-xkmLRiQiDHiP~K-*QS9LFX`U& zDlPa^^=zB-PFE7lNxDq8uRX<|T~H(7Kaa=Xum~g6=Wm3WN$p2k3EUCx+T#L@(<=9vqXXNN;iq{Vy3 z%<(Ew;%Wt0^)I^BM?%u1>qXHt){BN|uh0w2^;{3%uPPH*#3*;_lUOZ;zP9iD=I9ps z#`hI!7DP%L=LW(6zxf&U1nbEi;Tl;?gD-1QQzW(!Onm^`XV8`hYme6OS$9OYr>w}n zBzBNq{ZR+*fcV<|rQN_&NaWMZ32tFUZB<}73E<3Qh|-m06EJ}Ah0%BZq!tFSzk12c z{1JNtq5`L%1@Bv?gEGo7`AE}$lfx-mtR;dRI-{KoK2tWuk_jxU;X);#&01r{uy=s z?JQeranf3n9v??R^>nqs{ni2!>-IkUg+AP2z}NVNtT5IRKQZ<{CM=X2$Y zXl|$9io}Vk2!^*Gth9LW&Wg!(3UCim2TIm_!3AFcy5)+MfN*RDqGCHrc?VU6XS8|^ zgc`AVR-p|qBe+9+nk=5(bjCgXusTUQkPB1r)u(WSD{gCz?3Ji-Oy~U(GyaLS)xY83g4PZGBfWLz62PR5va;clZUNx#)?D(So zy?oZr9{7`nc-jG5ESX>SAsK*$z-8H5{YjE=n_s@PJl3+5+GF#NmyO&aaD5BcK+Nc{ z^yTk_$*FX<)|Ss5@{BEl-;gM{5YG(fSC;kBEucPsUOXzp8D-hjboKdD=(6Kw_eCy- z)R56PM-LNf^~sd~^I@^j6l-se%@Y+42c_CBOl z&jb|t*R~hGyA|1cQ-lS3M;jQBl)2sHxW}nYl8v2!tt!(wo5|^Vp>MD?`<^^wF$2RU zwmjZKk@JPi=T3*%df`3Ue7Z7hQrp+n^Fjr!ogTJa>tAnm${sq%PN}9M1v#Z_n||MYCyNeiEoKHX}4^GeyWFVx^bO6+rCoJ@)dp#RJ9ftClb~ z2P#d(JNE)jsH-=?$_A@}g&;vZ14|HwxeGB+52k?e1DV!gR_NN0b8S?y>eeiawv5n* z3Vz=qL0cAELCn^wUFP_K`=7IXHw=IC^S??uWD21~b8Cdr8+^iGPsyM`KFAj}!t;l3 zUbqqqV~Y@#k|Uk!#ht2@;N2E!C<)S+yVhi|CfHCAt^YGj7&5COz77CpvA>ijzH`59UqKd)KF!{&RhBaAp z1H$8zqsv-L-YSpVx&C8~$Lc>cXM5wq-4Li%ykhSqUtu}+1^oG%SJf%f_KXqQWz8Pf zQ0be#hU-P~;N|nHxkwLF*d!vVVf~5av1=bD z?A9@wu%fG_p+eP7EZ86+Ps%$N`D*yxOVM|s zBRLwr!`aR4-piWI^@$UMp2#hIv*OX|{dGZKDKZ!hHm~0k+AgArCW0mrE>^DX&Ylb~ zc9|u;k&)$7!0NHGxuKxPX&eTWh1Xz<#Mj*ky7(}?l<)yEnGQ8sPP&~$=+(r`3o*9> z&*dxUT0HzXB}pi9zjo|~8aIsY{EGS#gggr}0%XGCS_!Y@!hwm<|H{>p zx{~_ibZd{9Jv;w#U-LeD&oY7b+cF6@H2v7^fhv?1G8^m)HXKzmS4BR+F6!A?oob~9 zCKMjTS9H%^5a;p!+sby4*#Bgk|B3SKr!aZVI--HS_qmhu92dx;!$oP!l4om|DoJ_u zl~DZyX$q%|`~p*gtu+=zw`}b@N%`Hy0E7`r7XydtR*Mc!bu+{xL>Vn$Z{2}zkZKCF zj<@EAXhOf8+j3rC4A=eqb(XQ09EiiOu|c*giiP3~ywX2*p2){sqVye%Tw% zp72fwI?R|r+mD`*z1M@N-}+q3y8f+|JF^K&Cxe>Xof&w3-5wdS*r0Q0_xq1^QU|lW z97~B=H_0MFE=xIFeT8&|lCBK;0K!iHPH2L*7AQ7#F0$0DU)j;}JZoBr=&X^b-Ba0B zUXK{%E_mK}IHyc6iQu@%RA5+sWht1OMd0^>cakyMbMX&ehaXrLz3vO*jFL|A=PvYE zHgkoJf=4bl>5to1b|vYmH-N%abBUfWhsluFW0oK<5;W&Kt$hO#7#IMSy48(cIO*Bc)vAk@}r`LyL-{+1KzrJ zDx~R_{I3hZg$9Vqd&87C8F~fhRAr%&QnQh*rJS_jLNQDMTfL!G#~a)z&abC6N-)F_ z`kiV!S!ag@gdTV21nP&rfRxWUXg;8Kf%h_9qCj0C!`I^sKXUzte$%@oMg9$~b^>y> z!nJYRJMs_UfP02+O43^eMs_8jm%_@pXJe-1^R-~wgMv>8h$}`UxjcfWc4?gA8A!P6s4&SD(vOzYT zUbFDZZ?tbAkZFTGtiS2K?n!a2ta_o!D=(_I0`AqrqZ_!0E1I?@~zWpy6q7C#!# zNn6o;BYS>HW_|b44MMuED!}f7;6Uyws9_~QdJLl1Lfj@IniS*p6CsDp$7&NYLa=g8 zC&0&`zi0CA4h^ngzGM%jH-J25aQVc%=dD+E3M z;R%iE%o+FyVT4@l@DbR6^-dzj+G3Gu`S4rshXt4i4`5pZSLih90eR`Fp8%;FQ{Imj zZ$$UNshn|>o|5Jn*>@*0JjpV7H_)25RmBQ8{4}dq4!%h87JNqN^ZltQPbEhgHdfdK zUkh-fA6&!85s&x)GYAfXeBuJM>Q-U@DDto^CBM?qy7XY+3*cV!T^6j``()HvQ_Es@ z)QJ#tBW&6+%j`3Q9MQXEegN^FtkSzscJ0j`3Z(>4XAB&wBltho4w|6-sZ~Tn*mbg- z{Vkw2s968$RNs7*4zL+sesGKn&MiaDJL5mx4ZuU_yQ0voTcS)NumunV+Z^bl;NIlv z=gjt)<|at1>rkIg23CFK3lt(Bbd`v8IXz~(!tBEGmK3kmJqKg&`QJc2RfnxeAdAh*2k zOiM8eFCURnK|25K1uzmKxq$WKVAcboVdREZXxczclgCX48~<<=Y%Xw+f!T^*0y;@_ z=F<`B8dP}+DMQAhN^l?$s&b}P_F2~4ouGQk<9&GI_m3!fny@D^FcAbPN51)UtJbWB87?c#Jix;)wY+z(s|#p@*xuCMCO9bnIscVA z;N3~o*D0Fzt825*6)(~*(9EJ_w-w|(D=7g7PJ+Z_Fc-5In&p%z^xG;PI?6JYZtqy^ zc?!`Jz)F!+-Z8`M5{ZZ%W&&*JGgkcuADJs^>`so07hCG$k53XOJ#U+{F7gJFX`9|j z0i8Fs?vb@bCmA3XMH3!Sp2VfBpYqM(8i0-YEv}^%3LUKxv2WeZ~=$&Ryd^4p+yI>@zh3dp5_`MaY zd;wcmgdVQ>pCJ$dU?{4y%@{+{CkI`(R+|7IXAU8?S1ODgoTBYs3trocQ z&8hP3_C!q$@JhQXbBo^Ps4MOD{5d_vzmn+{JtXey<&6SOF#DI=`JM%L4+AP zq;cqw6e$IyL68zb+HVcleciX$`#jJ0J@5AY^#^X-T5Hbz*y}hWDRQ?|rLd2M#}%9f z&xRqqjtrPcy&63~sxiMZGcX(xWB!C%=NM_MGtMF~zz=rn%8zCX&cs-eHdTIcB8xDX z8d5W^nkAkLq^~Ee*RWAmi&6{(X%K z;n5eYNEPI{C?A?!@m-Wu?V^iu@PnL=dSH~~Hx9MU`gUjc_S4O;U&E8eS)e8Jw|3LH zB6M$1weB^Ry5#-#JK085Z{U{^(UsfiYkl2dkxYl9CpizSCLm|uW=kLWiSBEcU;ABg zT%_-wLxjr&M$6xON!Uh2YIK%7J*oe~fQ;m`xEh?gEPF^u176O(N&BIKn zKcQWoq(;W6;=4d+EA^n9lqFYX_JPeU)?yEu;=5&@PK_EM15WjWhdx9G#aG@=1Ss;# znRv4mx(>^c4_+U{Hw98Q*vyD=zhpH1Jrq6RoO9Z7o4R2`6!VRdr9v74?4FQjDr{s0 zr;#Ar*hi8xB~d!5yg(45gv#lqN_mh+9&`M1C#*=9mkl`PZ#JNqbw|~hp(XteAf_8X z+Z}V<+gSvWb^EhZA#tg%%Ig2X&`?Y?t7H&EYs*9Tb#7dwI9@c<$Pi8)MmLH-i0FxGEIC{2VRDLh1UIN!X^}-&D5pTwEWl4%~(F<-AaN-Ok_=O{p(z zV4f6HA6O|$3USN*SSl8=4|$QPqVX0u4Hx;|+l8vPHBe$mjX4$lJtbWk;hch0l@mlx zz?o;jrgIEkEqb3ksdN%bOz&P}-@2>cvG)a}-`H?@ijszu;su+Cy4?i-2SJsAC?$<= z>FHUqx5tjeb9i_s&XEV4k;iZI?Lj_#7DHHKUqz63XdkZIIRE zo?Q=*aZ4&ILUj3EdEb|2F2~PFSw^%7$EQK+&IlAzz4T8}u{~W~sU7(#Sypx9_+{V@ z*`SBRmzcBon4z0I)Hkq9RldE-DH0I!)HjRBg2!tt8NZ8XAv#rB`&3uJoZO`FH^95U z3k95kd*eNCx`~kE(ik;-%WV#FVz5@_>q+89!p@vr9#vu*Byd(Nz$$h)Dzxhfo59*N znUAYKD|gJO8MkJhc9=1#i8F|&6{-9iCb8B|KlxDyjl^$GdH0pt@cs=(?Zoo1*UrP5 zCyOi<--+L)N%9jRhT@gm=%R}Z<=~uXQ$T1SyRa)V$6JM4O(W{x8eZ2BE;3`<9$&ip z>?vFOUc*=ft1=~kh*ThWa!LL4fTkvV3r`wHpyt{5tifSmIl5csphXCTD2*BqlI+sl zX26xb!{GVF%egQLo@XdW@Bz^mwE@3VGOGIAC)J&lcJ089J#iY`04rTvMmg#}Y+p#l z)Htvn!Mce*?u!Oo#*$?1T)A0vq4cZ;}q8PD5Yy^bFOR?IGO6M*mW}yXYRcHgJ zXH|a!U*I55UIk7A=R9L~FRzC_?_SA&kxUk;3^Ben)hxhsd+x%V+(;HP&hf{iEujRt z(Fv&}L7^qB0aFGyA>-+Il9y`O-Iqu_Or$J?Qwt8RNaJ_s znYxGWYs9(>23I(=hs*`(ba{_uzH=g;heURC7idWgG36m|TFX_^KL88ca z^Un0pcw{J>ueuw3M3Jc5I9oGe((O^5XzCDF00_rBq&$Ood$n*C;k#v7FN%l-k<4ao zV!244RCtwe@DF%!{6ruAgEZSB;BLPx9wrSZ@2S8kr{*P-6}tO1i)L8jz3Ej@~% zsho}edP7hUTMaSfiXigNEcj6MQ}0B(dtka)RGbK7hAw;8JsK1c; ziADqJ(Ex^j7F?x&qa+6Z0}T_b}qw>fNnR!2+^dHk|=1I>#rb%sU4LhxE1~5swyzL zW270J1lJ=Q+-Ccgki6D@rL&LZ&0T)btJbFRpQE@9ke|qn#mR>AIr9UUkywUo*^h$( z9>~+yjT|jU+AvmaHISmW^n5TgFzMkP=Z>qfR1`s$Cp_Ml`Zi9U=AMC@GH^sxrq!4| zOck(CoJE#o9Qwd)2yLFYf7q%&VITtC$l6>ltw7aAjYxQgrCqAEN?9pGwvYEdS*};E|AA zdq)oYqASiI)F3#U$f6<~mz4s*J)9*NFjt#f%z{4NWL8j{2n2luh-7{+Gy?8kPcV># z(YMJbX@AtE`Hms+n>~jju8t!@Kk9$GaHl0 zw>?AF{?wZn%Up8#z}=bN)zZ#!@v!rOtz@M%EppJlG|p-x;F+Cjm&c9UqE9qXjwGPg zPlmt0F-=^U4Y0M``lWEet_DToo1@LcY-3W4wT7VRyEku?QIL=f5{{P?najO~4%s?U zc+u8e86zAN-~Iy!#{+JF-@gy`)jRCJXb_8tn$>HU9vy`MX6OU1=(LRwa`QC-C(Jr( zzN>B99oBNq%b5dO>J>jeK8C}9AK^z9o`BUBHv^53FyQ<~fM+Pfv;~pzV|9-dg_~w4 zb*^J-_&f30R0!a< z+>DP}X^yP@{i$!*aql(6t$ceC`vZA=4|KMbQ?j@0m?c2Id04G=1bM%ezgRb5ev$Sn zTY3q%|Ch5Y;P9uj zOn&^Qvs_5yGA^ILzf`aWdjyOF(MU`L-C@^yz0V!sQA@KV?2Jey^L%UG%(qyY0Y9UN zi5KEKt8E|Txt%r(mTs6%Dl@GoT#mEL`QQNhw#|B^My~2aCDdO3E&n-6^UqT`2r`gl zesIuHCi&hNQj!!c8Q>Nd!z;vq{+StAawVCfRZ40uU!Zx&6!)o-cB*YP)XWD}P^7$msBWJ*6^XX3vfEnRmfsC}PNZq6ozXnNN;gRwAH}b|w&bQxu zBuyuxDDoPMc9jml6dY7JZ$EwEbY61%XFBtCGpxa07G8Zy}=_ud^PPCjY~gTk%*~-+&|{xa zT*i`lk$)+z0{KQdb6A2|J2A^760W%Of?c};7CBcl3E;%-Yv89s6^=)JCL4vm0u5&h z`-Ob;QN+k;w)u#p{2YBmzU#L5hn3~+>X`6F;Qu_5H+E&aK<`-ApUmPLcp8Y`9|kP( z$aTsJCxYJCmP+7CkYXZhXOg%5>)knU=vPj*rkA24Ho~a??})dPtUhh0nWF zwbYU6M!w|PY=kwXMQRh|yG0@dF&3oVN?y+l5jvg5`&QP7Jsd!6JX8Ku&+Gha_NVi_ zoAk@SEPR^*yS$<>zl1~T}jNq&dCUj^IV{OALrUaFqi+ z?G6@(RtWBJ{rmXEDkU^vfL%OO`0%Uajp+)5$+!eDc;+l(*oCWe#`Ol_hrX{u&@4tR zv_73D$DJjdZQQ18&3c8iCEx&FwCWp?Dz7~Nje z;i6fWfOH@iABd-Uwxsd*d!O2$U1g0fzkbuCOJf#T3 zz2_pxJyUk>&!uAE*#y&F5pQcWg^+?3dgdCEhZ9+2a({!c;rKJ)H#?tlwzl5RcA`jr8>Z}?r4rXOE0 z>a&d=== z1r{EpdyiprUTRij4OMB={2v>HOERy~BinX7a~thv{gbW+83%q4@-*2Anp2*V^h;@a z^jVxbR5BHdc_mfi<2S*6%n40|13WW*A+wRhtk81%oSSfs78J9lt{kVs(#>x~DEOVl z#}%)Apz&+u;E`T#4h``}+!#|eH#?H7&d<16RWzphax$7nA0STJkIesvA35~j_>pHq-x~S>v2K2mdH%Zv&D)+0tVT^79hAv_;zBjGh3BaG zuHpS=QD`W?kjnz55icg!6uP{P>ueZp3#ex6tO*K`IvNTO3khKL4lri8dAH$RLDYHQ zgGk)nXgODMXdb9zPGu}C`g4}Us8T3iksvBwQ6GWl29xW!fh%a&3k}#Zg(LmsUc(>0 ziV`fNgniXHU`P+z7hhF=qap`*T@eYoE)OZFyV|qRV zTs1siqv*Ch=#n60CHQZbd`SQKXt<#by)MthzBH&Z8i@GR>~jnpb%Jih&a;C7P+@!%XI>%T2WSGR__z|6VI zM})yo>wc_VOBXIB$HF1!ryK4oDSyJD-NJO#zu<5hCSvjJFF5>}Yw(M;_2iNu3K4o; zp58&3mPWZ;Xf#&s-~pX0na}sB%>(+5L=&Mi6T$Z9(4pGGDjzg1g&;!P zx%^tGnYIVudo8M8lY3klwQ$0UjK&CROzZWn-~p6oQ(N}WDa5!8@;DEFv+wst6NUJ- z6u&y%17c()MChHo28M!fX4-`hSlfKfHe^tz%yDMIr0&ieeFRn=LP=rnAauFlc~AJ4 z=_|ZqG|M*fVq(ilbG~+WSCQ%;&ib3ah|2t+BfTw7Aautg09Tq1LITuLNd0rBSN!c(_ebI1|vi+;KJB!3E+&Gsce!h8lIZ8#Fq z>eOYoSxDLua`5cpmpnwkatbO_*j+G!0e4Ucak>xR3B~+iomaw{?=#%_`XN`RgRLa4 zVF9@2Dgn*o9VdNTn^)cYj0bO&NFf1iA}HUCkB6?sfgyBhjX-@UvvwxQsuPDMnmNf8 z;I5oczYxb#njTSPH2=&7NsdF79WZAw-99zr@mDW5G6_I&Sti37R0o7`xzMsM%ANtq zE51&g-5!kNKfK9T)faGAmGeIj9VcNO%WAn?yR)r*j_9w|r~XY}d7sLAz7l1UZtzV` zd_udny??mHLWJr48dyw^{b`VcETyr?x%tAu3)!A8%2dCRVX1_OkOkOv$IA+-!l+VH zZ1zZh^d9p#O0v0KelE*g&J<4a5dpxa{W6=m6GiVnU^KQ2g7;3_Vg+3&$DfO^YEopv zNssCpzU4dq??H3v@L09U-7bJTTDF2dI*u4m^q&~387IomvE zO#+p*4av$65x?}7ShdjgUgVnXQJP;Pq7WRaQYhv8DU>$LeNU>XuuKeBXts$HNonx- zGg`-ICia}9U#0y6kF9$=q(rK4!~gVnuaO`D(RA(>ZAq;kf3(P9y7y!&w?3lOkZFVV zl;GRWHKMoT#B7eKP3#R(cdVsQrdj&v{MIBm8-$A&E(N(O08G$5vkp%7X)PY_KU9m3 zQQX#?5?^ObcyIk7eVM7#dehi{Yb_iu-`)-?ARh6t^6mYNnU4O){m%k4c2<6>5VKQ0e0o^Hg`@9-w& z0x>v$$+!wFVNISmwD%WFK(|_S84LjsZP5F#7xo-^EErdF#W8?x6)gECV?Y8@VAhE6Zg08j9RK8 zyubuc2LsgV1TDPDApWee2az9hBIzRuCoAutLX;8N0>D_Exx1D>qsHv%Qq?$U8}e{x z>(P(;N%aQtb-qw4bynojdBt z(6R=WEs!3IqoVXy8poq*wU`K%K%T({YB;CWLIvwXk&-KR{fBm?y;|k4UM(Tb+Nc#O z%F-h9aT)qgXfII(qlpDZC#E&EBJDQ^y)R>ozUKPC$!C_BLPd_10<~#z{p5f!Ps{&( znCIW%Y03hKOkF3<<4;*C^dX%A*WT|gRwp9}!VP4D+<8NSpmHTbr0>c=->xM@6S`6K z9^a2M)T$;zXnclr2{U<+PSz61CTYT_X9_ZIzaR8B?u_UNktkWj|><05PFIqD5**JQhGNBBV zXNdQqidP};q2v`oo4n~SGT=)FY5P{ZQJ<+wTYHvlyiNz}EOH^DKfTmto>ytHc^Qpq zAu#g!HpfnbB#3A9@m@WM7k5xY9sS`c0$=$saSq@qaz{}pbWOBG{_7#SNiLw`x>8ORjIokmv2Zd zpwh$kZYGonIP3Sd@4WLhnI7@7tv@KU?I;FZ>%X5k@M&k<0aZIeg~qOaGm*|E0xMs8 z{lu;4nM&_^^#hk*LGN&A5l&ZW-6Z>+1VuN2710$V$BWFnN7lkrf=I-pH$mo)7S;y>03e?*)UgeoMWBN7ti2m$=r=E*mf8B=pHPkM6%{J{DpHP z%KV{!6v%je^LG+F^CTFm^m>gS8#am|7J0Yiv`rG zSO`Q(LXEnc<6QS?^?2}svxsE;fWPfvf zyU+*gqbIRE&f%FWZQODL1XGaRt*#!}aDvS*COQ0mxJC+QT_>c-m2_;0`V*(!F$sfW zNPJAil_dC7F7_Ok;riWYLf;y8a`9tKeaa5M&AIr@hO3YIwD($owTwBLiZg6rGB*T>=Blxj}chb!&bEDWY zW)*q1-Xwqi#;?n}g8O5U@2mW?f@BHrOB!1Q4&TR=p%jizJRJetw1wMeA>!H8V`17u z`37_aoMUEE3B^7UomwT*FX2`Fb>CiwTC?jW>OezkfE7YdL>{EuCicyse&Szeml<%|AGRz5Dtc5Ldxy%cied zgxRH>_oqMAwzKzV?b=uj?_jxRYF=>^kWT=DyC1NMFc1|aLjnOP$5-q8hct3-fl_>M zd_HJO311Ohqbo`wg+s#zFt0fhU*^m_aicpkSObmfQS80gp6iRhTupZ75b zEWS|wPYw$EWfOzM4e(r$eUq$X`k0S0&&hWef{SJcpS||E|0(efO_(Jn;EtzSY&a!% z{woRCkNc}i=Rbo}7A%#Df2X+0jr2}5IDAr*HkKj>#*ySf*Y^fA3;cZut#5_rAr^pJ zR<%qan7m-D$AFq0_Vu1QPYVp1U1~a?*{+OuZ}IM_$sPrUX6M;>DRmplMt0cnYxP>d zdTMzKrG${LEvg};0LE$FRd+M^rtZR4Uk$Cb=n==t3rD!nQ~oq7yaSjb!KwqUcX%{L z9xxhZ@X)AxY0Z0v%M0(K=k1L8-`L890in|ZL-*^6UCoEs zV65Z+9E>fd0D@@665{-ZkPaj&yE|5}DWT-AKSCe<{;y0XWa?12&b&=Nif%I$1Eig#B^2NOnO+ z;Kp6>Lv(T!oQX&daO%fsqkQo0bhwwQ_B&Y%6CLZuv*{2t?>Ue5sU*ucP8+7+O0o26wa{rh5RY#y@~DPE+*3h1TiG zH{jM{RqidVQU@sJ`W^|0ClGfl;{hE-0qGmx{~zfa>_9mA?}5-9$lv_H4ur>6=5Mdy zaSlBDZca;9AtG$zRA7=gW>J?+bxlzPg+l75@?Ft1A4p_NK)r@=f!MyU$N1>dnpt`@ zTT5@Rl2qLHJ%5YT*q?ABIO!f*FgF0w*MA=)fLB9+`zSO3xDgbBgv($WXfr=?qvllm zwR6aIsKE%$y+ZSi+VHp;1Ll0X?{tK$T6xiZm9Nk9o%VD?Lg8d(SK{ZZVeNDtWTzr? zl9fdfO~YIFXxd&dd{9o0B0?5ls&%e5d9KvEd0gIc&-B-y`W4kV7)D*VUdvq^wB8!o z*46ceO zJKm$6f051}siupKg0ubAfAwZ)L}zb-5!A?!CBAaLMGnBTsvv^Qlr~b$8tw|XXU`7h zyPPAsqQbi&n#&K`3)x}7t;?nD4?FJ>{;&~Z&UB2&k4c^#dI4&zju>F@JjJfYL- z`eS6N$b}dW7ObGH#jm~etA!u@mD<5Xv++Fgd^M4VVB_c0nt?b2+Z@2huJuH3jOsQSsl%}OP!mnPlvu~=&g_uA`90J3hYs*cDWS)l&!_jF?u%$VT zccz8YZbk3|!ObTPQEd)Ac#+skSOt*Ll4ES68%4~zQVm2f1i|p`1zz9At(mk5nd;Nq zkc%N<`LK(pK3atz)1bIU?3xV_{Rzj;=vv>?mOdbOf;c^Dx|N9T{(4~g(1iS*VM+M@ z><#(6Uieywx+ftrRojf${-7#ZTyn$p3hY zXG^`R9-dfz8ote1kuJxoqjtscSb84BI=?zYQ?a>G1)yLwP!6%Nv@98~JG|uVwR#<9 zEP0QTpk3MC#vfz8xda#++VvqcKu@3~allypaGYFdqlC1#o24QG7q zQGXb?#=KY=Dm1_-S}i*Awcbr|8V#=MuR{9o>2;nq$8yrf#vglq?aObB5*fMq4oXzG zvL{r%wt+%ReQwJ;`Re#%w?!6&Hv*4-6MYqYwVg4zY0%y=Ec0pL}AGkx#-YfTk@E zW3KT)sjM>%Ordo@im-v`{~Wt_ znL({uC4jbzA!`ceIM$K1`}DfuE;9nBClXdP;>4;!h_&cPw(uvPO~Y+_X!kHn{qP|i z)dN^lBFV3UT>PI!AAyS=B;#1WL^;>vFFO}&?TAx1{SvNlkAbK+_{ydwg<=T7yJWKa zvacyaF%Ym#t?`f~i&!c_BL@tRY8qh-|X2t6~VsF>lo zuRo>BK3lm3Wc}1%0S;6rUJ9rE&FE?013kc8B#xB?dNqSv3LXR>dD*RE3bF!dO}HsN zo!fhL@)t&7I(Q3upl@efg&%rtJ9dwnrSQ;$PLB79zk1yek^JSmU=%?AxkWdiT`}Lh zA!zZB(*Y|M;dDrFhXy2;JOhef)&Ue1Fj6+gwyq9}J|K)l9E(Ar=SULEqli&(UMDuX zWUuT5ot0jAh4b~c3ui52<4vnjAaPAF^*EdoSG)9`b8{at*a2qC_nH~39*KFoR@U#1nlctCKH#lVfYh*>fYe*Y+L+a%^H5fFe zQl=Vu9!`{>KV-^6#VLc?rU8V|5!5XyOmcoVssEuT@d??rN_3 z3hJy^Fb&9EO2qhxLFk1Pc3*wiRJ%U&xcf*G=5b%L_6zV5uo)cl3nAP{${?I;tS`lb zblj^7$OceluG7B5$3i6S6LeA9hzBv!iD;ONCX^6};Yotg<0qoy3@jBckma-{=G!x1#GFJk z`1$t$b@{J!On0)$_DxdS?is%{5m%Fj_9C>ZY|L#D0ut?ILq!BnCWNo{uHNDi5*J<% zvmF**XqiQ}0jj2`_)ko~ZvoLCa|APZC?tqm5c&1@dYb9b7;Kge{465K3=!+S?A_YO z-9f7UzOnqYGAug6xWS33Z-BlA)=6UdVf_k$nSunVN31l4NVWJiSTNREMpso>66ZPM z@s>OSQoGGoA?O?)1Xb zZBN6^^0ny`r@Qwy2f<4PfV@i3;D6GQjhPX*6mkEu$QBg%GE-d7`+q|gXRT8?G~~BJ zlx48;DvDYI&H+Xc;GSicT^_kQ^x(p)1w)^cjjj&vJerYt1EZy}j!~s}TRBS4SlN_J zgb+(8UFS>KfNPKzp!f@91SB?I?xx4i%pW}~{y34$Tu@n=Qt%J2<}Vg*LXpu~yOF9l zD<|64ZhEJfsYO3pzVH|EZC30Ic^>l9bAMg$L3_XLc*7{|XF;z7qp(=B$5p`{k4^{* zX0IHQs+6@qt744Qo>INGN8+o%Ad4oYUMj5O6za*IS>3y~$ThOM0E9=7N)Qaj!xhMq z|B$T^CbJoz+yCD1DsUU?O+L0)d8ExSj#5NbnX%}FQMgD$OFR5*SC0i^UjPzuP@#>% zm0&06F7r2{Wmr;4)n#FlLCHKo-nnezchJ()>$tsZ&$U5grhc*`AKbyWmFcAc9aYBZ z?)NZkXG8e=|Jd2!1cD*Ozm$KfWCI);Xl?A?SK;2Z_WW2k+nJ%{aNm-Vpoz#g=?No` zB>vQN-)!~-EnyOL7VOc)AuQa_y1jA9QxtBI#?6c>^_{uY!&@Kg|9iT_{>cUMAi;t! zaKZk`vJ2Fg;*(HGIwT9t!!S1)5;(p}8H~4t(m~E-9W2H0R#=PPt08{L^6V|JGK2MB zg^v7+T3MKI4LyZv$&uwJ11aXZsKdB`>jLqHmBl4PvS;5gtCex;fxtCsqxO9uLfY`{aDEXm<^auirsa@@cd z>q)|6t;Y|<8jM9Ae~(3^QtO(xsfHIW}AFfy7lcn};_GO(=_*wf3$*JwW03CfTTV45=%x zX^z6$f{kE^&4)XmdRlr>@oc$MdHLvFfVHC?4-i z2D#>b&(Io=^roQi2F~^^m5QMCk8R~u3nMpEhTK9l8=HAF&bQN-@(pzl7(U-8S1hV} zN6NshkH`}~p zv54bMRKVGElV$^6L;v&uR*2NHn7tye5Wei11W4YsSA(I`Yuq**_!+HbNKy5bC}26wTi1NMChBVznE7y$y_2x8Gp>lU^|Ga{H;&;|rq{m&kpuO%KN>4YVytjnk~apCf6 zzat=xw;q-yrDwEd)@@t2@^pWF#$yoBGn6GRdWbU^7mxe;&oK5~98lnyY57V%U90}v zBP;XT%9gD2{k7jbRPiF&ZPF=2S`giJSIT7*L2r`Q6QH9eVFanGZe4%vo{3)kVU~ z7V9IH?&vxKGfp?!dSmi9`gjg~+2b&*IEoyH(lseyuXc}o2mPNVr36}$< z%uAJ3+9>J1o5dx)v@F_Cl>_MIkk%+2(Q~PT*TiqV{jjOkZ2a4)6d)RE+@aFH^gk6O zhUwzFW!t9L+SazZWg@sQ$~W>^&(rGgp8S0L8F2eiecToK9xxZE zA%CCY`+Bpdy>UW{{)Kam`+q`y5M-A$4rfnkmrbG`0Hx8A-pJ8nfvIaj8c@v_NCFzF zXi4{Q{!RSn(IHzHH|grc7({8~nnNS=nuU>DD>GAax`Qd(pef7T8j#qNxS8$tA(dXG^z|d-d z44tRT-ci$Fd3+PZstu1^J>K|nurq@tqWiT)pPQ~9N7c$R>n_}|R)RMR+$rO|sN8^G zr!f(go$N!@qMjy{EWWd5zfI%<*ZU&K-66-EDpu!w=9A0?+942RTrU4W<_U7mgmRX! zV_yK)!gLya&+~6B)^xE0&}4(ht7jcyzlHt|{dBw}7;_MkyYw%G@zi0gU@jO?KQQ?8 z$ng9c=tG#~G?$>er29nKxrv#CQ!CG?p`Kd$PWu*&=-3t0+yHjB(xK@4? z^eX)gmdxM8SKlG=d%p4okN8lAdTx_NAiPok2T!60(XPV9=YG~6)mTj#q#Wv}3?~yu zie^1Y71?*)?y0Pp`aSdITt|X$2t%6RHuT+@1}EDH1n^MeZfrzi&+x~?H% z@EUN_gVm+>)kLQpOrMQ`%L$>dq|Ysw|k3+U>_X8 z`oLBY8DNQ(X)#3a88x7U_DRScBnvdtyIltNpF;gtBmdp(ti1X*^Ahk z3lIgOH}8!oNGAfB$(}r2?7o5+xm*PCsyr~LKk@HaV}M%t7V7gG4Ag z)FQSE!Vm5R8H!$FZBqjlYY!*>S6It{{84ji%9b|=NF)^_5R$v@z*CY~e?6kbCzP;i1(CcBuWhQegZpdAEm1ZF= z`kw~zUo$EQT%WMMs+>V#;#_HIfajrN-=o#XB`nVU#-8!1;$4f_=+DC47F=eH+!pZ` zor|w)8vs`VC?Hb5lj%p`<-=}i%jI5O&g}jEtSXDcc#D2U2k;+_mwNl)HJ-HfgrWUJ zy>NmGPdodC#c}4M8-)`?m4_u`b)NMeGp%A3NaiisR?fHkT~dG5;eC9MlUfbMoy1hK z#kIs3YOPkmoN=yvoEq?-tr3>yX>7}2SW?mM>VPzfucX?Wr;;-YeDL50jU+Kt&HO9} zbrYm&H@L1m6xSnYgH4eIr(c!cXG_~`@~ynFa|X2HBb|p!pUO~=QyqKT{`91yz&xMU z@Ag0sa)IVNTZNifEPw_4Z!$qq!&^`4*R{D2ig$KyVXdYc^QB8SCiA^ySyC2Pl<<& zd_HDG)Vk=K_sEBdNEUX*?At5-pdRv1f-Y&*Y%E~BdKrOxZ3)Z1G(?R*s{gGlcB|lb zb}BT_8+y<%Qg^!voUM)fFZnmjv1j_U;?f1Jp7d%I!6u2cJEm2_qz+7p-&)K4fzPzK zF>HV1b5OS3#lo;}sqVW8dL|P))!oP2D*bPVJz)0Gl8=n5pA&}O`oKl(b%NnFSFfL15W36iz2iN&Dn`=jeHX z7o9Lc@hYA~ov;u`l~s+>D6p#v>U9#lm6T?(;=1QI#sq~gtso`gM?fpk;O-3|R@%IQs(hGh_dk{Y+rnfp%f)L)*V_o6`!$4d$W+ zIT-r4HI{B;>ut*r)f+n5wSlM4SMBR^F}|gHK;A1ysj5mu#Jdn#DU@IWEqx6Rm+#D{ z_I@Djp>{-kbwVa2KY74WtWOD1CqXeCHFf)MMgDJCSx-B;frBH7qa-V(RZ$Po8ON`~ zM8M^_9()dy9pNi79Vd0Uh2-brFiouVZ|3Z|SrWI#J;lEY^gV5pUfyQ|`%&=crif`L zrG}J0dM7O{KaMh`qDwo-GqfnlX1k@`7AIv2y$zn&#(hd4hEgX6A^szm5D`xJu*R#Dn{bnuCA-dJ z{#n}H*cZuVPdH3xs6up7LtBE)@m|6@LHE8z5h1k7L4Ez$kxw>Fr)&HKYo}f#69@YD z2#QH1NEQ^gSUVe9d7J|5OaUxZAe`_&>=pvn{j}M<5m$GYzXWinDWyOrlUqsILg_G; z23X-iqHhZMcL(*}6-_>-{dIlv#;I|-iCj~>{3ToucL#f9!8k7qY22F)(Dv$GBW$es zKOrLkLoh#p9Hc(16Y`0uDMH8QaSWau<2iegPNH6_v2&urbGjqVp6}};&|v)z zm9VJsZMJlw#gge!$m0}t@K9V3n}#lQ`^Q$r8Q)+bO=fR~y)t#LY4x6XxThliuJXHVK)~C-X1H()V3Q3&$DQh? z#iLt`4AqboUJasS7=GMyW0r26JGauuQ>(VPEs|sz5U%Jo^s`< zwHL7Rz{D=Ji&8{Htv<5qbY}Dzwj&>8ej&`hzfX)UN&Ux&y8P@($TNPKvOW`KTa!}t zMCRZ&Wc1p0C*O@y{T8<^lo@hGfA^mg9>eipI!GaS`eMyy!djr zqMh5lcaA)^;!qy%O7tP=$JTzT?7G+TXFfWQ7*38R$fwY(^xK5I&BUknCBOr)u2jcv5+P4X|3p-WA5D2s-TJM0QDHz@GjOmQUXn-_#`? zkNtMvDCJH}0z&+&`de$`yEkU0bvyWb;hVP^qy;3l9ZVH0pLx0kXe+KZ`8YNx2BgVZ z(USe?ZyRqzjDsdnjXU%kORU|~N*_1btIC|pB)oWC~5uHL! zU7fVik#=e{-%*|SI9ymLh@(^`FwS)B=vQ96tPjf5as*ie4VAEboWI<1vteHW14ImS zT38cNx$1hEGWbvN$E?22zc?ht_8SgK6VM+AWS#uGj9kmC@SHh6KP)nq6GX;nR%V{R z)|UizZ3}v9*uiWCyeV{6w#_`^MP&lH*+EA`{Qho!;{ny;JUixNO5~_qU=XIPSfnkVVq9c~j{^-L@Z{D=J@05LMZta^Z{pp~*7sGe#b0Uqw75WP zKb(io^DedPALAIMWth=0$%6mqf9?>3;6~%DjsNvSv5>;hfB(H}DG~12Om2UEmBiMq zQx0E-tp%K9F1N(?W`EWswZtkWG;a@RL_wN_$yhRXHs<4o>Y}27*-3tD^==imj{J`w z2jE%qaMp$|ja)6}r+%T1VY#Kf!Vb*vMNi=#RJ~oYjRfBr0l)7Y_3Hj%N#Jz+na_Qx z8^1yr9G6|ouw1q2RnzB0xKWT{>Z9T3^j8%lCK)Bv$_I(r)&xeLfJPvO$yO^dZ8gpb zjFJ`(Pr%YgRj*60FqX~VB>pxbW=!r8uS)$UwxR{!eaBR0dRS{q{s^$=3Z}F?9BDXB z9z9ME#)jy7;W@bCatEy!_FrZqe)~lAXg>)x%hqU19566MP%lqCx7I=jX9L+=U0`3K z_A!6o>f??m?;5orDT=@M;SdjU#`fAR9UYK$WvsV#F;G2J<^D+bdsV>y=db^Dku)aj z_djH8DqO+0hJ$m(+Q2g{;R9(B=%l)}Iwp2*t7=*O;%FPOB25Wn`AC1GnqQ^LremEJ zX$&o03Tks^`1P3Ym3>}iq9p?nOxxL|_@5=+OUQ0!Gajeye}r3&dWE>l{MoMVB%({k zr^DHXbRgeXGO1*Os5V^q8a2aX+j{T;Y}J)F(UObfhn);sh)IN$L%D~~4O{vtrnvor zTYV-u8Eu>_*=f(3ZmD>%aszBeefQ^6#4-a55k+feGfABumiyt7X|NZnj30B*# zs!R_2Sta^^;|2F)!>j%N1se$fy;zeo{yoyXPXefxEK-c5}1&3Mx;bhPug1ju(-E4EyEjTx}v*We;+X;*cihpXk zJY@ctj_Td0yWa_u>k#9y`?U}QcM)zM|JJ$DM~98S1&HnTe7DN`o@xhKL~}0=iRH3A z-uv7LipkZ-yC(Tc-(nW;rPClxN9s)zU|chN$epq$M6hcc@GZP z+R3Zj;JXlbW>tsHk?&t}bLlddmT)&C^+Honw{)CWPOytyJG4i(Adl~Aw!|iD!ld&| zN9??z)WvY2S!mwJ+4y?QAlI^aHs1M(#*I!I{ zW+8l~7XJis8_hq{;ZFP<=jpg=Di?u*cV`r1JoJa*DkC7R;#M26D8dc zWrHYq{A3h{%)XM@Yqgo1iK$Ref?w;N9ZW#$5elT1E*t?xii?%E*}HSKXS$x-1Pk|0 zAD=h${;U4@JMaKCOZhF|X53pD>=*O=*$N*Wv8jJBRg=gvADtcibNDR2K4no>>6AIU z?i-Gk49GWj&6|oHzByq$jKX}ng1262w%BAxnLbUgl2EHOtJ1x1IVYxmtT|NHZ4U;=@1pvYmA{^zQ-cCqw+JLRS{u(Kso3qS*JM ze0Enqq&Aw=F}tyAkCwGvVd#^j3G!#Vg^-H4Dg7+v5D1i(f#OwVyBt1?B9{L^lwtWBnu3?Y7M<`$XUZ%Qd5Q$JvO!eIe zE#RO^?~Cc-^(ql7fN~|1DkA9gIWj%UOb;}_;R0LjE*J7_tXrl>aW*dh-RBA99hyGd z`o)3P&aVDkT&D)Ca%efu!fVg(P^ zkXAzJhPCMKMjE7Bx;v%W5AXZ?zWwceu5->`z_qyNGv}Cdj5(%zte$LFzR7sT*fU#5 zj3ISw2hL|aOCq8|{UD&{xT7kZdR87xr8iLeL)w+!*JND_Q0Se!c;P3pr}f79HL^bO zKlCWOZr^KW;{3g}UZ~3%F{YVDaR-74+G`}2SK87WOfYDyeK%4LygD!k&a=mK%s4FD z0CZx3A#yLI$QZGbWs2Mn~2{xHtH=JU(HYtz^tM88t`Dd_Zy?Ft82>#L+qp?pMN zbPMG*BC0jRi~}!43`)?;Lix{RPx97{N;%p5e=_uK0=7IO>1c$dm}7G-?j$Bz6uR_w zPgNY_m%`uM?8qkwa_eUDxtZ&Kh_>SXe?(i&T*ZUhXP(-${*&5b>h9F`V)$h#epqv^ z&N=|l-H>b}J@n>op69X0=hs4K6zL?Tj1@JZ>w~<;j^viy+=Wz^p>4oCUP>|(lTF9A zwpYXqMhj|RN&E6`;i~-$o&U#Yp(Ty%#mYF0g;iVL6$|Gla{PGsem4kWBl>F5m_`n| z`0T+RWEE3y{#2)0W2vh~qHO(=eSBSs2(-PIE82wf}(6eog--o}j3S zTOM9xdR+q;?m=cXQS>>jOz11rv9ybz>$wBbqs`oM4LD9 zb~OCguMeP_>9S-3R2j_?g8j|UCwX77({Hak`~I)(2p&XX-M5yU_EYIu>08%YDuJYj z3xK!T_lj?rExoekUE93rD&;V8K5a|j>OF|K0X4_BO+atckTVt&`rXFNK@D?(C*5<3{;oBh%ftL5|2z)UJ169S+w;_!|Jj~%0{*}MH=~VaXN^MrlwLjq zd(JQ!Zs8&%F^TI?mHfWoM0dV-<$K=PU499t&?++#dlen%NvH3fl?Jm3LngUY!4yt3g z=Hh-TGxRQ_tS(NINKg5dChUMaSGkKy$ay-i0ad2}nbIZdAkHMB8e7 zZP2@$`$WG0D1pW{0sqw#dRc9qkFt#+3&_unP-u>UlNiD$eu1S+1<^GnR;j|Wosl27 z2|C=}7dl6JB185qAQ6%K7$#{cDm)%SrgBg>ZAYQN+Fb25w*oR!8U9)m>gU6F!5CZF zgzxOUxwv^rNhOMRUAME&u_(>Yvd$s*}X@%$iiNq=LK=@#(-g)Vtxkzy(+=RiGb zl_WD8_ZW&&`Eswhx zf9PH>u8zS`&rTrqWAI$`O@^S+t9D&6*I+`Sz$KxO3NYh$YQhrPY`_aArUn%wfd{5Z zr*atV50-=CYc$pu?jLhRMm1vG0ccBr&rhSg;mN(Te62GW@6)RvUoF>l}*bn)K?f!P?9oq!#6cH7=jx_^fYWj`aek{QH zm&srmrcYhw|CFNqDPaw?z!!3jKO@3z=QP@;STn%WJ#)5$uBlNhPi0|dc2++qS^@ZU zr7hu40i&vIb}DVojUgfTF#{4}hRkfC|M37UoIqw$V&x=Pm_IkuR|!0^SSQcFGt!ES zPewOeSEi0}^h1$i(%eAXCUNoOWKeTnrYMUv)=n8`pqc{ZJl(X>U2Q^i1jEr=YJ7Xf z64*KoJf?G9JIt!~*7^x0#_j8%(rZ=`K-&YN(@xg1kb2|EL27&k`ZX4C@A!!w&^(zm+Rt!>@xq@mDOW2klkSZ zsYN>=-TY7=Cs0a?c(t`=KIwgWGGEjX3D-|AE&u&QjJq3c1q+j-)z{fz{SwE=%{P^* zW@_Rp{wk_a(c#9}N?(qdl}?Oh2CR}QSxQHS0ONHfepl{8DTb7hy)&)&t=05}yyzC) z#p@+64Y&ZaPsx?|%4<%9ZRK*t-o+ zbX1_|l1j+^H+3`}i-Tz+hWqCET<%?cQVMkn0OGuja6*h<>Bzs+bC6Xg0GEdfZ|~d=3L#Ay}Ar!%?~;bEqJh`*{2<-R<^nbR4W?I(H(= zDH=~Wcv0JncCup)eQ-dLDoo?}S+7CWJIm>eIEWHi@3@-{o;Cd%Y6U5{1n}7HfEWU% zVA!vq-AuOYwut|Z9sx_~+U=_wWw9BSXWMYCmuQmH%#qI?5I#Z|#qf(@(&zgZ*EHCz z|FYyJo)LV&5k_LmlMS-bW}dgGh%4V)po-W$%1FOnS3mFz%NHE>)v1>

Np=bxq!k zM}^#=+AU;1)dyfL9gwneOa8yl*3F*QlgvauI0MI6Xc3dDaS>fKYfv>Of9=yYXwVrX zK&2_Skh!3o>#%K_vYyv`re}qgD5~2r($Z<{ndIOi#>7Xf75txS*W6q*^^~^0Ty@e* zNqpaz*b72|_xx|H)iy?Be;pU>rf|FBZhIv}M0lIS#70Xq^NEFV4&@ik5E~awG*-A= zc?NnESQeU#BE`ZaY=I&*^W;%M2`bhV7;)_*-yOoA1yc zyR8kpGhUeEV+q6LMNd)x^@dS17r2uUI*(%tEE3`^Aak(;Mm4XPw7Rv=xp))aiPVwP zm3h<7Xw?M2@;i8i?&9w`y}X8Ei>Rm5e~FiKF|OD0#9`{be^P(jt~1T~7Ori@L{qJ7 zt-7htF=A|y<4F;}iq-1o1Oak>rhq|8a8bebEPn?-W(mTI-ee9jp3Q;GCV&z3mt+T* zP{HioZUqy%QX_s(ZTpDBe3#Bs(N9V$9lU;Z{EyAz{^b8*vk3eDuvr8lngLaxDLb!5;fuZ>>sb^M=^&wV2!;6JEIEE%=XeW;Sb_XO?9tVG(W=#!Jc?c@H82rc z@sf-KM{UjYk+|}8HY>fDsu??G)j9gd@#tG$tJyfJ@6|ENuhz$A(KjJ2eBBorub%Es z{8uqD#K6BgUo6gxHdk^qQ@CCjkdGE3^ReV=}=rK70zr5GJ7#Yxf1RnD@K1?=8 z;E?`z9vFpqt8{4$g?t6mCDH+qR8wgIrs=4JMfU9AqN$I!y^F|WmnYj9L?vKb#N=n= zvm*UPB-zX2O~AU~WRPB91ztbD!O&Mt<30LpmoH%l&XF;S1Mc8Vi>qqcQ**H zdpBFIqy)>lm|hMd?);rxu{XGug#J;ZtYeSBDouvIvtEg)pDI=7A7do}sudx?Pxn;j zZteUeMn5v%r*cz=M0jx1OD7|L1PQ@l7c#DkeE9BaOu?0*9$m=o{P)0{f4TQ=v;z-n z3y#8x*(Xoqr~PBxG@_yS>osIIJ8vnQynTdXb4h@%j5FY z=y;ySd7f4geG;6g_XCF3>pxs%pOO4phWa_9xcYI+(%1V#j5|DDq0fD)*A*TRlz$f) z(bQ6#e6*VLUbu9Y$l%`SUReBEP(7UT(Wl(^@OTt^5*R-IcXE`+Z$qxv{ul(aHUm~R zpIfAbM~kBapZ)&f^OkCn;kS??rntp@D;K^GY7AM7wf!%h7yQ}$ zNnGy{-Xuwi$zWh>BRJxt*UNG7#TvnTF+l6)UIgnjU+l;wNcm9-$=nt9U{oZK4&S@Za|emXu4x2POK|M-jMg%eLzN^$}ge>h#VFOe2p>^%d}J~=bQA9F;zZ$HIhS-^u1{; z8a3x%{mxrZt!tUbhi|@{KHiygLgn=-7}yhkTE6N$hY`}NpvoQDhQ@SQgpR97GCKf_ zF3Bs^@-f+A`+)lr4z5e$?(0Y*-j^hQdbM0&J=l?lGZDLgD8jr~^loRbbo+~}lK;j> zzBENAM*OdXoNZxJ8Lt`c!%(^~eb&)s%Mp5vVreJaD$-Y2B#Wm7!5d#9wu#(7{^6GV zk;$)1?GQ?r)Du$}jN6ThHf}6C{p~?DQKCvursa!qC+ZlqR@+CLOxBSxEfC}<_3@yT z`ZqQrCV{NwZxLJizl%_0tnrd598Jw_Z&2=!a)+XzSAx>b^1q=bxp^-y& zjU#j{X7||zOo;e&cO=vJf}1o@E+yE(&X1r$Y$C++LjFY)r@g!Xy-9YJ?|zOk1R~X# zxNw=qjePa8c~1nDfaQvr{Rf$ezdZ#LJTJ5Z%W1R}7m|UIMk^jaCv4`_V@9K`Y6e?e z>w{K#&%|ui&xut#%jM)`Ha)TkY)rI!&9MMSa*9JDYc9m^JGN2F-CFI|C(+J^KX{xHSyq z=ZRl5b>z)0wjN$oA#^*d|KpdDOJV58*6G;WrF7Fw{Gv;|#Frm|H=#0oM{r9*jF^vx zV3#;I@I{s3L?gA%TP#C=2|w54t&}V*;*{(6eU2X&wd?)nz}MtdPZDbNh$tAxhTE}| zhe3~8yi7l zc^lNbr#KQu*oh7$@H^>!^R|!LVV&yxxJsWO#(=bYdG5?F`k;oBKaxvZvDu*fEJCsz zQ@R8xPbz8 zHqMlBP3;Y6B7=4nT6*4Lf1*2Sf)X+!rj*NG)Suu!`XG$&;b2<8VAW`xuf3P11rsPSC{zYv?;#+#+ZnYsXScPS2+6nbYhNl!xoe*D7d#hOs=<}`5sK9LDNLoES4$h49mRfO3gwP zx}V&1y3A)Xmsw8apDJe)ba|?jQH%YSlYHKmJt_=i>wYX7d#{pm2dj(j{pQ3b5@YSK z=-d%MhJRqTD^L6VaB@I{GqsTfqJ!7-56({nQ4%O$lWe*Ec=1D$U(cI=*Pl^3vDxfDiWW`;Y)thS?$H1rQ zdtYx|#;LJo(cvfONtA(z{i)4wal5VAthD8yx{u!fb!cw-@e{wgLGDrK+;_0RLye+7 ze*E7UY6J1dqu*T}e4Q?O>&Qe|yf{ID?zUy@j*X3Vq_U7`jKuz)Y%(~qU@H`y(AnQR zZW!6}e*HOcyHvI8!+AJPSv=H%F(H-W0tY5+HH$Zs9au9DN)kqdB(>2K>f~h_)6EB= zBZ3{vvQlK#yYANL7;vd>)hrP*lMu!%{j}T45n!wzo{vxsj!{LF+g#Z&LLEejY2SuJ zitUN%S0ZGgstge-C860VSzOj|f@4G3Gz1JJ%@qGUn%)%?ek9cSL z=jFTOfb^CF5{0GvhnjhtwG*-BmB>E0=FTBoYo%vsTV5}OsI;Z}Rh?oiT%4%n&`s6l z?tC6A(c_$xA$8OICfwex@9Dqq=l#kM8SX7%hDM6mgX;&l?D^0dYEQ&F>f^&;!}Bzf zxi^a}n#?RszIJ;pR?`ki>`i6WgnaK*-`8@@vJH$!-jM~CAD>S6XfQNq?e^oF%SOcN z60(7L3b5m_p&buNXs?5%y)b>;V@3~bEsR3TTGmfJl}Y;KdTuJQI*;Y2&44MOj%(?9 zC9PTLN8p&NKkfl~4L zL;v$aIe#H~lEiV2aQ%>@f_p*O`6PSoV^x^f-)|PqGLAI6Zuqe6V9QzMIreT;@$B)@ zSdU4Ha2!Z*N}`L3i)eMGys+dK)#;>~+3i;3c z^3eD4&^^vYqP~@EYExgO{A7sGkUM+*zG7F3=i}#yO$4jd*x;Gxl#|JUSe0cA*`00u z^MMK!AU17hRpeTOxV6u8AfSpBSf`K7^@jl zT5W+GepeJZ1TTjx7L+agEjiHn;<)##HqV@=hdk52^`UG9M^|`pq_39_uM}w2#nsxq zQ?|aXX7iBTVFtCjG@dK^@cBH_WUF3uQoHwr8LYaZN*;M_JGu}R@|%8Q?D~W<3$s;W zWdA_yF;u6LY2hUCpOV%FO2S0^hKE;RoGf9_gg3(G_zZcS8%Q)XkkMy}8j(UlQXK=b z;p?4NWjqXAPO`<8PitK?ORZoUjj`~b4fJ;plX<6qSPjo;5c~kpaDM`21<-<4qWm+B&N8Q(3wLH}u3ZABSCpN#-j5$)OP)e4R zmjZone0NW2Abp^nQff`a7#Puz`2q-C@>Qa;RCp?Wwn>MjQZV=15M7@T&a@d?G-6U6 z(mpx1gD)wvsW0kBOx?fYr>43jh%~%A{yaTU^vO5=4-?TLlq`?`ap9n(`ev8B9baY# z$2EClTF_b@9W>U)>{Kth&Wv9ww6e>Ct_k<4WjNdUyBl9z0&Z3J`Bg%r6aRy36c#vO z{$B|C3-xanL`9O0NZHYSr)21%A|u{QDd4dB%(FFr_t>sn;~&QcC|DAK@cxflOuSUk z01vPpRVWGP{PQ4?HBmUuO3X0G3TKy1u6$+mrxQD;Oi_DrVmJt(4He(e<@ zn5L}Rke8KGX^3^MQoiaFDi7cFlhxu3jv+ja57q1@3`*s&#tWbb*HkNSkgPsApmhf;L1K$FW8f&e<7eR3%pdmk~zE=j6^y}Vz?24BN zHUWd_wdi$)TS+as_*>IeyuHX{NXg!llNBu1CN%r!zb3(FYRuA z@{Z_&7acu}*F#J8q@laSVY`sKUfactK70UH`%m69N7IMu#zGP-MCO^t`3Kp=0#hyJD}DoLRBD3j4e~H5;*{W9vXR-t zQ`1++r)i4J&h*W`NbfT_KxVq-{f>_S;u-z;Lf0E&G*^cR(;pU{g_Hk0- zMD3O~+0MDVS2{(BpPE3?x9-Q!bFHg8d`G9tc2MK6qswx1wJzIuKFiJ5vB+Pmg}D^0 z5^zc!qSnw66YUsOM#Pq<%LG6Zs&yHn9oMr00(0M@?J#Yd?g^w0yT*tpj(N36-tOS) zmakb(r7MP9_wWq13X^U8wUR6k0aal>pCT*KdtPcQe7MfeZu`9C1t*Kf3i6&Qd~dxG zXVhx!vi`%4&FFuDC=TRdfq$aU&{@DoTah1Z@nI!oTvR6YTPk^YLkoEbcU74GVQNT= zEdD^eacA!j#<=*XZvo{R)?dE*nQ-=|k1BrOoA_*ozi}-dteDEd(%hGTAGJ%$@G4?N~0(cuXOveU88_XzA=PB*T`!%H&~+^ zn$yUsts=!CS12fJVp^&u)!(0EoQpK>LoQX!{Vja5o>!v?Cp5`~2|9|U4JtK?79T26 zYfivX_?VkC`3Q4auCDwXhoAJh**&hDU(sy(FFXu5Pi)C_kaYMftVaj!}L^J#{*!kyPx74)+HUU1qkmuj~FEADtP}SseI*LNA zz%Nau*$W5u8MbwIZHQ=@15a})6w^s3y%dW%`ooGd-+x-A1>=VcE#| z@_Zx+vyeIt^0U;)sE1l?>o)AL^YPhF+Vp7A7fW0Hn2REPOWc*dtp!?D7g8Yg7=*G=nF{^D?3AK6 zLDrFwdsfyn|L*SV&^)_5c&m(IT{$7knC>OB6{>R8SVje?0`gq6h9*G1+X^%ksOlLL z3D?q_YkToq!GB#oWkgsDb77Lq7EIPAar3C(z3~%k7pZ92xS8YH;9zuPCWWTpo-69G zkk@r1Tji{12`5&?%=J4Pq+OhqO!Bp#h6zjC-L6-isp)N#e>?*u%$~yVi1vdOm`&S! z^~Z<0b%q%ii$Y1becuSNqmG`Y%gEoPUlT&`MP3mU^y6Q%(O_qFuPI)WGM9Yq*W=^-e7j+l32WLxuY>LIQx*-Ktnf zo~AVR9LN!d&KynqmJ)_m=)XZr#_x_{km+>IqUYti^j$FJLw$@E`dnBwo8zHe*)ci| zIEP7m9EC>xou4e72OqRtG^g+@yFl&KtE^6Yqt7p|*xL<__E_^yp#GCHhs9{hpKvx< z>Gm(R5P9@HPWvqSDI4a3%16ItHW|gyX_I3pqxZ@Gl8#&$%wp`gN#aU+lNFGL(uIUw z$uc0wrp#!!J16wjM_@mLaxp}gr!z2MRo9M}AugEgH|JlZ9~KP0>>)#{8?ZR(tdU7s z=<)m&!VU=`GF(%~=)wKSsrHygk`o()#?()o&3p1IwgY$v(_g#gn6}?ylR&vf{zb|9 zp;*sipk>hRheP=+@Vc0tCHP0o()|k_W}l*)W~GL7`2;}B_Fs{?TtsucR?DvcDa(B{ zNByBl7@l2sgqD7o_xSnhx%b7ufyleN*$@#YNm$2icFVASI1WB04njB%1`DPD&)dMS z3;38ax)FZZ3oIe$b~!T{wm;_{D`Vh?6?U;^I0Z9u<$jAj0xQn??^yb%?lwYpkNt6OXrez?fCf9Dh)C^iR*8=`{`|~;sEi0y46cN zd(&3gLaX~sbt$tRiVnJg3^uT`GPmZ-AjR599#_%2^-#=sB)(2M6M_RHv}VmS}j0(z% zLUeeER0z@g+`N1?AEX9+YOKa+M;0?Z#EDnN6iovBltp;j=|cfdF?kfOs?H%QlRqdM za5ftjSa>>q{ep_eUkRVneYK^;&gkPa>L+`*&JvEXu=N8PGH94Q9vtSWp3D%ht+OK?_AzVj6}=Lw0;^8gG7^MW~iLgh%?DFVq%(2KvYG(&J;CbNPUs1 zC-pAaeTQ{zs3|#VOGR{FGhXS+$8c>nhgR^{#tt)Zdv{9Cj=vo zL^t|8(TIo-ISP!ZGZaTy411qGUtYhbwfF`63#dI&6bN^W1e9h)ja(MXqar%Y?NDT< zkStB>0++&o5C&D;9%PT4{6lG z-e>R$j3?uLid%X+g)>Tj&7O?r$U5gA$ERcZH`GwaED&9?85E@s2ZnUh zrGB4Pl51AGVNAZmi_3a7Oq)+HCij4;QD#SQiJr^s@`+lRU-%>`QqdnBjn_5aeG}jR z-c~Ke{WHZfAoN?@pKW<9SuFELR+0%4#qHVTm*_mpVKxC?Nip$S3}mgvPF%>GBWIGN zLb5j7D(;}uT(hwImO3YX)Fk~UTCe0XCU^X+s5+C+(|BJ0_x`Ypn_Lpww5xSF9EeX}s_CN7C{3k#9G`QF?;QQJ1K`!jS#2Zhd5W;;t z>&ZT=z;8UrRyBU2l9CW}>Vs$?=pj9DbsNJOOsTw|J0@72DyV);GEzI_($y3-6@;%e zr_5hgH3YUhjYf1D*L1TN>*p%xI%?GOHfv8ZZ-k4?Q<-~jtK8_Y2NkW}a{^fh7i!~5^mlxEiw8pV=Tb6+ekz@gBi zH!$ZDi3q#`ZFh@7=PVQb`7Gd=b$lF&9v1n2DYbn9Xy&wdVdqzEA$h-5z^?mQjt%bo z!pDC)!yCY$46MJoL4Wx9LASW(<0@jpmIlaDyxeF-_sU|Nv57(PJJD%4&PA!dUU}yB zo?BdeN`#vAai^SBIL`0a#6seKLoa=DXMP*~K;i?FSw1f1kMMsZ;JlA>GN5L;a&t9j zp;c;@9RIQ|I(_vAO&RKYSL2_qBd~`a3d%3UK&i-9D~~IdaKU|io-hzUgcOGW2`1E8 zhJ^X{$tQ4?Ma;0Om19yt!(94yh_g>w4ti84_tExK)}i`! zwEU3z^kojyrWNH!_wV8MeOiO66>4;7aKbV0!gv?6wpeBVOIoM*`|iS~B0Zm|Y-Xap z0{xkEPv0cvCl<;8Aj3AEnd)FiBD~_QpoE!78upu#hh3;=yTN{g-!)V$h*Dakk?ai z_X(KC2~~<2rFRo3GlMEnmgmcGfVR*~1|0pZbP@HHm{|%Rei^aex&7mnyM)Oh2PGnm9Jg1!oW~h|jem65^<&n` zq7nZ#fdmIG7j4##2OfI5p9!L!KEi2&%#nGWC1o{Zn#5tAbLV8FEg$8pslHGZazNd+Jon-h&s*MDVEFY#c~!8$LlMRxWba~uvGeUXKDAqgjLw9TrCe)K z|1$zvi5tg7HJuPTLPSb|>2=cSRBpRHLW@2JxA_2FA`ptxcK(Ld(>#rT5BHsAU)_e; zjescrgADMVaIH{UKzs)%Lk6laty|V%B~=20L%R#0?;75rZ*ku!heHZ9yF=-7Z7H5B zzn}+|n0sp1p%z-)ge-QPD>nG{DZnSAA`dc|ZMnv-@H`t&N*Tv-*?S z7y>B8N+++SVAZ<4u>E=O@wVXH=Xhrl=N2By$n6O&rEny){;ylLqF@&=0@iGJ;I}kc z&T~Ob6I6He6;MAZRBFWRt78`G&2VFDKE9EhCSzQPe!@t$5{>($FIu2U;5NOe5XD>5 z@~qIJTPh?UPqxe|A(=uf1*TR3FWv%Ix_g%0FJsa_8QvJVl=!ld(HhZbL1@LPB;hEb z&^LG~U$0MUj`UBAkXp;vE4m?%!gcXmis_^4cFjC9}+IdrX$`oT(u_)Xdn=+i#0;b+Y4^AXU;;2T?;z*mhwj=b2gjgt8a zxSjAP!su&YT1IDve6-B3wWy)spknJJI#nz((eKV3C1mRWjeB4yY=GBz%@PCPqzhf{lx<0L8$)b+@?&sF*IYfsQr%U2Ny97`6^=m@CTU8xA2qQ7HUjE+iFe=EFk+jo zob?0j$)ue{+{c|pfOB<8&_qHPYT|SU7}~Q_!Nw4D>u_D?lU*IbtO>$;Or^)&C?c~S z<<@5V!u&FJ0wt$6_sR?@xn>ps(`JDTAllSeG@DxYBK(VaQ6ka#D(I4qW5-(+&zGqW3n=N)QjEFt-Wz41%cB?ME=JYJJ>W|{HK-@7Ug zOM1LH$Lg-nm{w=hqSk{1{xZ^udgGv5o~h`Yfj7Qta)Y7AA;=nzv#>oPYT+Iy5|Ag( zs#p>65355$ucGOBWqNfxf#Z937thd`t5)v8riou-ipCV*$Z4)(cE#bx!P5K=>5X(( zuy{G9(=!W84g>5*s-ZD4ilJYO+7~ewzsQ4N>^7Fq{^JEO>>6bSsq?$fUfU!wof_f< z%!)%BW_KC}#ZiR+A}igY`hW%$7Xn}V^ZM5)0x})28Fbe3glBDc@{W3i4AHzU$5$Xqpde3_{=k(^@%S81m=u z6rN9A!+hVO{uZ5a_r6PS=B4-6)~6RV$11(Ea!;)rGV?7CF`~@ZHt5y~50Py3?z{e~ zo8Ktl`rQRxz6eU%_7ps{q+s7`)thPR{5v+??@LBn5jp-3*;{rd`KV~+TPz1*Z6Ek< zk@DU}3)-LwJv3b{8q(ZPH`LZ`Jm{V=8|>UQ6Prj-@8upuEEs!YN$i1=R)zXVyz`D} z-fMkuXp%GvT+E`KQ&l_q<$769?uCbthR;A4oTXtdijo*0#Wh&Vb{mW|pa+`u_aIAPK)Je>6)aJ$d-J znstF7VXiJ+5KJ}8Y*>^`I{deM>081TsGE3vXJ*ox%6T}1#xN{2_c#CDt4&1;+!d|g zl)b9fF%q06*L?(A=6$tl)@}f~%e?DZbX55`rB4>WD#>E605^Pl(jX3O;4vt5DH|$K zCu0-v+kly43pMZx##_FHgV~$hH13ADkK5o{EAwiJl-;nTVQdaR3ofYb8Kh$fY%F}O}BzJJ4TIVE{^e$B|43apDWn{W&;xC@#>LZCcow1 zoG2Y^GGSdi9Z<_>*l4f;>GicnUrArXoD2+sKqW%)T$ecerp_dL?e; zqF~9A;2r&lcHThXx4LfKmAEDM$R>*E4!X*ru^oww*WlLf+*)NyEf*P2KvJsH2-iIL z(N2vw6RGRm%koz!GL#+eNiore!2Kmsck)?DLN2vE6a@5N+U{d6={@*<#lt2kP2l?e z_;nbTYiXb`?Q*u*)zP6{li>T))*!{ksRpjwF$b@*{LJ@qKOcD((Sngh9ydxo9>F%RD;Xe z68v$nt&GhSK8Pd!`h(FY9NxHD>#6P?={kyY1s-^ijJ8XZssSZF*YO6tmMaJ}6J+X4 zx?JlCOy9W?y=LVp#_6%wx`0VAokcNyY#cmW6OOX^wc3o8joo;L1w$KrZ6#U91Su&= z+}h8w-6)0V{}%2tA)bd(+pB@TaFj1-XtN3#s`o1KE#+wh=jF~&RWXGa2c=A1AKb1d zZfKbL*#tS2J2Livl$~%Y9b&e4HSe9A=F&3r9R0OLCv_JSAzA1d4%|Oxx#|%E>kv#f zW!X58d8JG0U3{NLaDsFHZ(GD0CrF^QH{uBJB*POGpU4qn-*NLmL>4^x?<6#e9`^F= zOk0_t;TS*LTUIA8#E^G(B&spbRN_U&t3fp8si@m<8P(mGg3Y=Z6~CAwst-FU*z^el zo-w@hdx4luzlZS49VnjMoX?aTKYM}D06id6A>}fN)HpdzeEkR zM1=2NW)|r`@(-h(mC0!K5w3N4V!vn}Ud8;DJ02pcem?-PMrp!jO0zG$|GOsw@&0ep z=;0jw@wBp;E3$Jxjx_t*d$|{+?N)JLKIjLxkw^dv8jF3u`2z|{el_g(GMB%4BTn<~w-on0GQbvuzgy2p)t>^@-p^Z2*4dGz77 zuy)v^g{tBaBU6%t8-$DRB>6|Yq93m1jsou@DEeR&vB4ASVVe|g+0CelT-ssJx*hT+ zm7`meK*I`p;Ns8|ZF$(@K5ORN?#ZcCaoY9u6Y)FQyV2e`*3xNJWAkZVK)?||o#E#E8w&a~(EN$+w*yUU2 zO7w?IJ1fN(9Tvuv)l=IK;xkGhMd|;QBhtiVNKg_+q-o=heX*Hkz!Q6<@BTtbavPVt zz$}Pm3-{WfYHhZNgO7V-cJt-exB}JNm-Hy)Ut=Xyhr4*yrv@xgjs8d!Q(G2q5cxA2 zv}N+s=F9}4kcTgl9j4;-DXg0pj-)Mj)%wJfX{D{R65@V>Gt-CdRGnI>b<6*?U*l^Q z(Y@Inq~voJnLzN9jqmprVS-M8s$3+B3z;5%blK`k{1m}0sa z#U^)kiA_7p2yoAQ2>%9ny`tAJbQKm+E@(3n!I8;CG}u8qY>$Hz!k5%j#XgKuvc^Xq z%w=6_2?V1bMj>A1iF@-Lkj zvcMPVQ%KSgYWOghKx-uh4C5(Xau3AU!Af1isf`L4W==D1?He;}9DfsC9zXPbstKHf z>Tm-u^)2`x`Bq0JhX17g^!}g5%4zDz{rnDd^k}+&#@11Ih|*;O!Xq8>j=`XeO5Ri}U+NK% zkb;r1aqBRNtC=itlt^OXIab{&NQ2;r$G93W42m#{?9cHLO)ly3jdfEoM`TvU8mgZx z^d(=SZCy$7B5*wZg5;HUY8hoTcp5ZA;Wt{V+6X^d!yEvTnpw?`XW!uB`t94VTQzCI zA9_^P^GXNpf*BjGucXVoCO3`}Zv;I#Nas6 ziwD~~$kiFg(hD4TY+~XR#PB1bsv={O^wyB)GHn6(4H&ZwA+C%*5fbGCQbb{4l_$9! znRU7^Rt22A$K(6$%*q|5A9*is1+wjxE^3mYuTrb_!NDV3WK@=LR%z&=YVE0p*yScvRhw@%VQhu3f4x1%0q zkpJ@hsgYT2qF}3UNEuRQf6{?1-We%w3MaPQ-uIcB)^i89b&`G4ELAgNZ&k-Ku2kGf zrqR@Ex|!nDWaj0OD#6sNYgm(-JsG!S*WJy;3furQc!YzTLuIZNOv3Bp~hDU>=P&3kJ7i zPGg{I#lq~nIWleI&7?rj>zyk2WeKJ_+o|LOI3(>nWs4AqavTIL?al*`$)p+5oC1tw zd%xHU2(B%-$V2g{B_qBSF_!7V16|LhEkQucRiHSxGs*-wB&Z@=2&r@~Hn`xj! z&y^%4VWm4o%-(wui<(;O=~|t!;$;)iz$d%AIJGL|+q#AJ zbRCn9Bue&Jv+_N9%-@TvERB$Z{OKf?O{rC2nmkZT^-?SBxk8JDKnaI`W|0l}*I4oE z97Xl`-pnsCNR_{X!kAc7!^hGgf@xEA+;Fa7k*nh+8tonapWB$&Np#$^U|gY|OxXb8 z5kgXXqL8eX?4#?rlm(GM;U--2fp0BH_rF-~CUaC%m3$Q8T0AeAR9=3**%#(c=+%G` z(7wZo4kt=)<{tD$`sp2m)ue<4Mx4Z$V%o)k;hf-t>@mb|&SQte4UNSnb9TSP0ESw* zyFvgGrkkIq(H4Wsu2>FlVIA0}VUHhk_Rk#XGk;|t*MGdi#KUtjm#a_{G1#p*bdQVg zH^0B%xtsXw4fY`zxD~#7hW6p5`?E4qfvT{}-`t&anq^m)ph>bbn)0HbT_cvvxf7lO z(gzH|fj5^i0lIAd;CKpWT~R{jUD`R}da^R&jm(6%Q?((y{&zw)SrpFe&9}pPb7G&& z27^`j0=}bE&dN!B390u|8?fCF7D7}k)I0zKn&^;Ley>c|w@&{m&RpJ}9XI}mIcl8W z%fQ#auyqwYHN0&(JFAvK6YPuv${aw^69C<|jj<6>aJfOmZ)2FZ%E^1a@<$#yv$Fuk zIgbPU)Z=d0E|GGtINqPua^G)8VYOc7UZxv5wz-9egXu9vrWVPx9442`9dI)bzPv^K z@V@RNIlb^DdM47hP$NT$&&6Xe8lVW6+}Jh;jf||;zov^nKPe7`kvJAlSW*`Si^=PS_Sg3s zC_HhQhA?z)O48pA)9T(CfEKwu{~O;juhxRMnckx6hR{ThEDTbZ66BSfGqJoyqw#tn zlF%aiOPPTf>NlDJl6$HP*bA?4YE0+>?n-L|KUmd4R;q&OW%wt4LW4hfk_@w%SN{aqWD zYHr%Q!47vbp&#_UwFK!mV2>cgP4em2#Scg|H|!!_oXOBKpQ`?duM)Lby1Au@9KY@9 zh2;O8RoWZYh=v@bACwgG;$O&d5^9_|sx?aws#|8yX%@JLTQPCPXTO^;7tS~c$Q(0! znV2pSi+7eA@MhQi2b+571$rWj_K2%~cE88q=Z;P~yO4C7dCC8@CVcG13Kpn-150yy z&9K*0F0V9soE1q^;ffp0IxP`QF&MM|GEyPZZ7gje!NW@2$hc8$y`)`*%PfU zCz_??DJ+)VUa36t!g6kenIgZz%_;Zd^&Jm1Z^T%PCj)3LC14&%;**_`-#Pq#_-^n# z7v5b%W>TNSaD2V|x|-v&`~qxz_^x1#xo9a>*`JSEcQM%h;-*LQ^s)gf$957lE8krp zo^0uUcUe93^vk&epDD(<9^`&P*Bx*%KxY}0YMq<*BM>gIr}DrI^0yI7ZdvEnE=I+}+ zngR0_Do?8GBnY2?i%k-Z){x4xyIs63+(^Bs@=Fv?>LVeS1gi!WFrvI(u`qLIK5_%Y z5mu0IwXy|mg+)ZRH?s95#C7<S72z6?i!FBB&1t9rJ12qI+QNyW%(X9Xy^}~Q^$N;u3h97;6n3Ne|KsKqqXakWm%3$`$_lv_ zatzD9FiAb%+?Irp;Jexr0EQ&bO+PCA zVtBkeI_b99apD|17r{9f!RyPg2&sxGe}-k;v?bsuQ{^~7OM=O`Tu2TvV#5Ab_0k%O zO`g%bU7Yicj>^)MxM^|bUD@WJG2V_a6o@=-5_ej)8JCuasuB5zf+Y{V>g+ZJT|VohL*^D|_XbdK40YyfI=`v-9{gvn2d)s-7ogx=*(? zI~k2eO<|-=obI&+6$gkN%6yT|)mr@#q8omXsMgHNF@6N>2OjQ;Qe!EBj^(OSuzz?Q$Ep z*BJP`rh`-*m*9(}5S!cYaLp7&=;K)l*5RBji6VWVZ2)l{PH(@h)#CdUrKG^=;)2KV zhz3GI3`I$6qvwok^Kt z|BRI3qcnxi0^p9snj&^tnW{^BEl%8CnvoclOR`1~V-rYJh2AJVl+|-kGg9V zKN|vD&O-YdBkKewq9#)+#coX+QApI7IjJlZ8VRwt9zURO7BGqE6-OO(Hn3SeDGid) zJ8`3bQqV7rP=197R#Ahc2mtA+O8=uJh$c#1Q0?rdpEN#TK4YHKwiC zH%N{j$^(_2DM0DAzy0&D9VxBrh>yG1`#S4Kj{lzJq1BxKuEB+dyik-okZ;rQWj53j z&MWH26wSoU85Zrvp z7zuw%Aj^xoPgay6=9IlJ#x1D(jUsWJ4_Fz=;+L|`wi9q2q?k-C1+SI^v!%5O%1Mz+ zVBfHJ#ANlzmrKWhnT!;;!weLxx$H4>tp|iO+n9JQ{$N8K1Uy|7AvRMXUkM~HEZXo2 zjmjjuNroLi^SLk@ny!n*6gVmgM<(sI3!^I7S9X39-zPfXQ+z!cp6fWMi51G)rCpWg zlSt}X!Pthuy8R>s{}_Kp6f`kyP23&lvv06PIYRgB^rY% zi=T+wwdLx&_wE#n_rkj@O(1Fja6QMf?n%_B*g4#)Q-($hv(}iEFqsAI`FL|V=wtq*e8`EKLXrOPtykRzW0J`He~y z!Z$R6wG7?s^NO$4sU(_`Eyg9{mG3XdtMQ+7bh!9C*SuKxL^?^LW*xIK^dEGts#d$d zaEJ9q#XP&bIpsCsj0I1(lY~0{Ja1J7j4AcWtd!;|CtFJJ6U0I}K2v=&n)jwFa`+h_ z)9bi*{-PK8ML1FLj%6`f!cZfup0wDv$%=_?D76 z?{~y6MnBjAT)AH!cU+%*cqzZJS(i2YS)H)b)@V}?3-Z9NbcnC{vL;-I!{3qSkb!6l zH<^r$aegmswm~kCCU#_u(|=m=tQbTzp&%6PDfNPklc*A@RsyOd^}6)}{VFB()@?@F zXil`;W$_L@oXcUcNIujC{Z@JgUZFkp^`TBlPm!U5;@#UDsd?0HY^b zoAA|*nmngwGY&RgOq}8lC=v_rv)vy=Qf0pWz&YjH8Ho7R3sb6Hv5+?fTBuA$F%Mc1 zL_j^D&!L?6tD3czB!%4tpcVQnq%dOC3Kp)3^OYw?F&qJpj!ty;wUHjk7gi2|-P@GC z;4@sZ(2M$3`lnA{I!>N(y1k3grf|t}KP%FzB2X2*QUdT|W(k3!$;iPShZW zVi@}Iz37DoRexm!^Hq0e2RjeWolIkKhL%RqN!k9^An&e+7ZO*(T1jvu*$zci?^DUt zhvk?hjyv$&QYYZ>M7yqWWAD*{1c)XkWklx3rMQm`4hyUcoC!LAx)V*z(TiP$7@!b* z{4b@((DufAYgpmGVVi@A|7elSjQ?m68A{4lN8u3XJmeAc2N&fs>FJ=84#%4Ppv7sj zr}=kyfULQ5d4q+^Np53-u@<@QZCh!W*gH1Nnx!75*yLXbCbe^A!2%H~;n~wARmx{6 zpYEN!b~(jl(1a$#G7NaEB0gfH|LhAleS=129#}4oT--QOE{hJ4KAE6mEI{r18P)R~ zOTdP7Y12LHn3-o^)lW+yQEy=~i+$$No5mfchghn*)nZh@c*FFD3{O2Tz zjYLA$bFV1YsxlRFUAz81z!t1jgE}tpvT!0gAIxE*vZ~66PNq@IWb9|dsdo;tL1xfLEVg2UnFkqq8ovxNx(*Fmb_5bm0elBkbWh z(;O614*>7L9Yp5Y%v?uZjRBcDB$>7%*~zajS02G5c$pvAtVDTep%LA&_DOR2Tby7K zGB@t$9Y$!+`{{Z&izkcC$nI(&vcA%$p4;f@HGJ#b5*t4U&6RRHk1|qaY!mJ#6 z78=0}L=+mI0_>&nb4T{K?e^DV8&n#yGo!jB4=*8Wr_7v9+ksj1$*`q$vSHP58dCZW zCPUFXOym!Tzud~aGfcVW_ILX|Aeyz&LRw@sos+pG2iYG+g!2p9BqC7ogUso zU42hCz<|hiw8#!q=t@C))8DqHg|mi!oMi)1^B&yKkf~#x8lBTDETS(IUmYLUlt|>O zhd`qZd516uTHEw1v7h<(DNjt9j7-BeGEcD~3YgVwemV8Pqr3y|JGj zP1k>xXDz6P_L9S@-f_IDva|4vfT13@WU@`vO{qrtjxheOsaC0T7?C({sKoQ+rSyQC z9R~%dHl9_=)Nj8*8Z2qs%_GV7MHqohy`R>*5;?OWGuDVUK@XEo)newzN9USrNk2Xv z_U}5asylGI)!SMEj*)1*0qk?q`cx#tA0j#vYCVvBw`G{oe1j>&Uy`e0T-iT~!4;O; zXyihAD&LVTl#8X0?Whmyh0Rtqvmo(S+MPBh_jX+(PVl~Ef zHw}lWd%ZTstaNs>l4=R=7#sAHjKmGh9<{R(7Xw^yWneJwY^H4L$)V#+ z`Ik$BupDF63&GOm(p43C$flaKh$Q`5*>MmYq2GjK1!L21^(6(l%GV#r)*HS3-bXnM z$D#0jS5324=n{CYI92*1AJkNYxbm2LR~>``x6hdUCrl%at{U!9Yxa)!!{`!W1{SJ9 zR!pV+HM^Y3379dZY4ZKNi6ewb_tkX)N>k-qUHej;&u_$)b)&*yXP@8h@1r{Al5IXz z3nj1F8OVIWU@4z z7=q7v8LxU^HXMdte2T4n)3B);A`rwShWvL9lU}}TOl%_KZWf(hWdb7;gjsGwgekGc zLXI}6ikf|sTxk%8BGIFmN%6p+;gp*b&z!~zCL3B!#BH5Z8$#9!S1^!H6_Sl4cW4dM z+a1;;M97!+lKfGDFd2bU*bM9FMr`Zww_wWpOeT`Jo8|@XuzXh*FND~lswKKO&I@Ic zUNdB<41e*U;Blu^fS+X-XxgO%#4wsH|(dtH=xMmViqoW6d2|WQ-w82U|1= zn71LT8SW}h+lCMhnz`>|wK&rotBdDn$JA+_mLf$cA_p||(v^jnJXXksFpfB-VfF*@ zSYf%{Dz9IkjnV=(Rkz`1?9Ma)^w)E}+w|a)!;a)=A-zHK$`G8zD%Vve!-|^YdyK{k zI|d3+omVmp+Sw~0)Z;kIZ9C5akM(RBWD^wRwfmbbSzi=1qF&C)osIkN5Bm`U=krUc z+x@#HA?<^TVudW$%@BCt4iK#1h->Lb*eQ?_Ms~p2&uF9WadQzoX*vUpJm-oG*HR8{DBm^flY{jY_R zfc1YHC<3z8nOVPd-#)2e5U%W{ZPx+sG-Gm;XTk!6W_tEQL zN2qVKs~yPrYJrBa_9f3)a41EnPocxI)|^yUx{-MOSmF#`y(lAse7$Cwqv7_UDj;D( zzGCu*j*cSe9EZ}vyGcS3T2!5?rN3hV3UCM$=bwPFE4!mcubINuIcz32l@hbdCeDe~ z!K7&>fc@-b_<#&O`^)?_Kji4APIU)_m9En~>iXSYHgAM;ZMRQ3R6$fHQ-8O=ZCxtf zm?p7SD9QoB#s^XPLX3j=2uGfLOM>-aC+?5(^-G0M&!_6<8|2*#b)It|K<@Gi*BXJq z^S%v3VByNwRl}h&3SX%8TZnTxclAC_UuzVL<-g5$(ia_;(^h}0w1~tT+6XhqTj5yP zV01k(Q_9@Ab_eJVpSs-Y=ruS3(>g0V7OkrpO_T(i zU?CYwhAdd!eN$N>$jonfG)IjUz1@daP2QbZ+c4FR<&E+^vojDxKErR~4@_pq|w(yl>%G z+9R@atY@z@Du((AFF}yZ%2)t^0jZjCDhN6 z#5^PLxCM|eLm$xpH`7J&^7g9i_&-ABiY@SK)%(YdXhgfs)S;KzlFL8K>;Dp6wHu3m zNNvn33#8{-3`2fLzM&4)o=4V3xLh5w2GWfpA24mKO0S03(rcFtqsj%osMOM>UzkG` zHrFl_4Oj0f73kh#qSDszCNNn;^K=WoBr9#xL-^)%I zbZ$v*kACeLb+))*$c5RDd8c;gr?9HGY)HE8XMH=W`Z*lP&~LG$_W>t6nLksMnMkVA zs8l)+#+zh2ag=ZYBQN)@zBw}%5$!s}q_bvQCTHf*@_jF}u=#1}w15+R6J7DTH|A{$ z0F%%y!6Ys|dSVzTR-9Z=IBe|uK>!u}LbIVjvH1BJIg~c}PV@8Yx6cHocuIk2^JeX|{Kl(>qD zG5`6tU%b#fNA1jdQ+2ItH_S2O_xJ$K*Iry3x=ZSeXCqdzak_zu)E`5t4w_|Gq2*x; zXD{A@XFiq~4G1rZ8`b)|;c*#DR$#q}GhqoBJN3~CW)V-nZxkU$h+lxfj`lM@ZE5l( zzCGOBEIox@` zCo$3Qwl>!&JN~9v*1-37kA|Im4oDC% za1VZxX*oNlsuik%XQ0Vs>yjfs!g{4w4UO|HjK6vtR&je%AN>Stnm)CmR~b5D!Eu{K z)0qs$^RXs%^S>VMFVE~n$C5RhWImsQZ{?HAw1o$KKiK&d;?%h9Csci3+VRh03fZuP zZ?DGFCEIyvG29t^8D+dtK1WdDL2i1a0&#pXBNb)83#oQlQ}ff+3zYhJRAH1=uk;DLzgZ`g$~f)7hov1_k<2gs6Z3nU7lQJg z6cs#URJ7N1_34~J(2H(#o=_wfYvpEqBiP{1@jm#Q&}IcV@)*+$V2%B!l2QZ+5w*t! zV5V}sV2qL;&9Z3fhOw0EM<)1&P63g{L(KzmA#SH`+2~FlDkpp($nh7NIj5M4h&MLB zxNO;o_&6IFzq6w*V6LwQSYEa1brrsNuF?z&q#h7V)h45N^;;C}-9tBjn>CC0S!^~p z5`H=9J7|n6Bk`tJ>i9$Q+0ta{4(K3Wo>6HUlB!k%`Wh|QGlk+2Um)}5*;n~4T0`~D zw|+52nA;rRv|k_P`r)#j0SoFapA&JuAMC4SV{^x4JEvSkeojZ6C^H#1?YSym^D7=u zSaX)^-8$3SFd2`_HHZ1ZUT?O!OR>#bCsv(7odpLarTkNO)7ulPPUvEty|P?tg7G#T zJAo#v%n!q7b2PX2&esmT5uCe*AAbIAou$`s(RcH^cfz9e0hFp#>)7`=M1+YQW#pBf z397_o2%uQ8t#B$8(njG#_H`&5MO|*dsBXA_iDN~B!f7_?pg08Ok2nOBg!HaMSI>^* zaF65*PeKA{>_jXnFVAEJl99i}I$U&dV{O~Zj`ea!ixeeWC*LqZR}{2l%`DFM-W}a> z{?-2?|G2M8{9sNf>qo9lH|Fq&dRs!l4i4@<*!0`6wDvPHT&?%~f>>2fAZieaBN{|5 zBZIhi+r*KAdY3jWR%thD3xEWunJ^vmzRO5klt&ztK&{E*tYJR(3D=T&s&ARwZfR|> z!#_7wlca!hhtKBC>}^^YE_$U7_vVky7)j@=R!q5+Awd?X;l%Rx21HXhA6C4$%$F;J zN1#J7kPW`idKdU?Df&rqM3tAUlT29Su;mfEKRKLSm%xdk&D;w8Er-!Hei2m8qG_AuNT-_W{GZ z)GwRfl<0BzSJ#eC|B?$yTXrY$Y5H<8&RX9j!4}iQ)}0*?)6+SN(0gE&1so~V8g*Y! z0g^cD3nOp-OVqtrKJwZ?xneW42O$327}GO>Mfugy&e{1BBQFu4`XMFmqF~?2@X13U zLXH3OcsKZ|L71nT@Nc9tgnCc4Oz$Pu0?`^4%|M2kVJ{wyWtc(_RJ%4q3}|3uNTY+E zHB*}xJ%tAq>O@Ae7VwSv9e(^O2na`=xQ|%&kDjt>+xJqeU5XP4TPMd0-6PykFbsOP z-6ceeI$gOh@$|0MZyFw=?O+} zg`O}Ga$z%$i0BT1{ivgfJR*6hA{;_wX4$|P?w1xQmqog#a~8~GQ=qTh2iW@ThB9e5 zZwqmIm-<~ZWtvCS>R~^0dtbE5aK7iKScKy*!V#`25or6;-&CUrtFzMve@!Rpia%0mfL3UpCntIIy zX1o?r9DSKU&q^Oqo#u-hH||%Q(;7JXOPui|dW8_mtWlSy)j3NcWT=!L0ofki5?t1a5;54g&LtG588CvbUhKD@5ll1Ghr2CxzV>(?tJAo>dW&iT>lj6Kk?$ zKf0sXm%>%P0!gw)toO3gJA_1T3;`p?KU^K5)+JVP$Hi?$)mAL1VKrP1Ngo0w&`~2Y zKfk)JM88~7s1GaIK+v}R7N|1CAhj%)4(D)jx!hNK&28w(ehI z8CWrAhyKIaGy%=@W0R4K+k*=Cw`!*KontZ#=BG+xvz4g_NG&QjUKBGZh(ZTOMgF+@lTtkJX z?8bOtE=(1+>o%s3=TktAw85G3BFV14rvix$dB*R0$d^Le{{Kojy{a^GWPbZU;uS9x z8;Fz$*dg|Yl&H%<-UFT~{QD`PdzTbO*PqCe)`lGd?(T)ZYGWkSV4hMmZ+u&k55A$K zAl_bXP7*~xD=6GKR8p9glu`yzPu?{vkoi%}p<5X5ZK_kDMs8JL8nIaXX~TlS`meVp zf%~YD*tA6WIJ1{Q?o>}9D_%pu6|1q!^;V}6BV;rzhW1yYkN+WfFIQ@aEG9zxH||&p z@ZcnOJNfH@3YS&{dg@b-B~b53Oa^xq-(@~%tT0IyN2lT@^kM)#u`4BTmImUH{Y~sM zyq~0+HnM*cz8~`2#Rus;nh_XI>Y+fi5CwH6`I2$Sn9zTi>ed`O1IkzQARfB zl_WyI(Y>=j66YU*>kBNY8+M6R-{z{;vg5Q{RKwcZU`Mtsu3;%mCcng*?0zl_6=6)I znTky(Gx7!^H%@Lf{Vi)+ci4_hoLNDhP8QViF{N7Jd%XVWJIs}{h=3JtQQSpiQR3TY z2Mbg3MKoR*Mspp?j$U#NBecwKd$*-A*YF4yG%T^@>SYhzNa;^>mzErTj@cwE4wt7$ z@pQ-2?+wxWcU8>wm^pRdeHMT#rK~r~q8}V(B4t}tew&SW@*KM!kSENM^tH)IoZ;+n4hd7@@|zV95RLXMDxP%I`AiO1A;zKIFqi{it`b>1C7e|8!Thcz3+h-2oXzBgD;8>0Uox z^_Jx7p|E8mjkr7w5w+2Yr>PA)sx6&V5y9HKc0&L>>QeJB^#Pdr`P(N-Ay-Zs1R;1m z`gde&fDk#S!TlIOb{(pHN=*=FiKeN6RiVoGW`{F5kYX2vrH_}US?3qwl*nOQByl+4 zaI>`$@ZH>iVt^@TK6%5|hr?iv_05I{dL2Fz;a1uhd~eqML7Gdhi*d2rl6O1-K`xbJGsSL|>z(MAw&c5~iW_FK!m9yUJ@s*#L9| zB;hmZzXXh|AIEq9H;c>Y2s{!*>>(AdX#5(oX3 z4qLwLkM2fZc2>-i4tB9LO%8gLNVw5h1ejp{*j1re=>=#|PZf>7H!48&N>@*=1d&$o zpw6**j$xA+=)IhMnTwDwk)bd}BNgZg{nm{7=|Fl;keZ$mN=BB{`}bo?r9d@Ew^I7$ zZ1QlNoZu{38)AH_z-bpv@av-yPZn;}tv~PJ2FL~yiT4JZijstnmvN=pvu3T^Uyca9 zGjq;Eo^#fe(MVL&*SCx4cS~xPCb$!wmPGSJH;pZqgNk~p5nGQS&5i_s2#0N)D9y*} zd2O{$);Uw$)`l!R6PSM`Wx@fbNY0Hv(Nj?D&d!Wk&5cRu6%6}<2BOEqzqqzrSWkzu zp_Na_)$6SPT??G8^qw}Vm=jFxJ4ow$z4q3J6)CY7$VPBxMgAACXuq@OjZ3u98C;g+ z)O8uoP<@o<)-unyf0fX=vZ}G_weAztWy^@_qcG(CgslcWH3$SH)otk?RAQWm9zY!vRnf%^W5+?H^bw=BS zFbgno)d^{V9!Y7&+rCIK7-33-Ptp5Qjy!SV5!k6LZxJqU%cd8-ku%f`Jk%^} zMBc`vR&Mv-Pk9ehs-JIteS1D(6Nrp{pw%Ulpc&I;V|L#JXt!(CP#0>EIbm^Jq7pr7 zHHDd=-xB-O0pLIz{~S<%wT1AhrnV|?_T9XhrMkd=>QV&PN!^aCZ5H!CDk0>guhqAk zh>1>5HL*Hd7ErG_0-;`nXR25NjL$e_zK`@;=APz#y-lp)`w`v&Yt9e0a&;-P66_&D z-&8F!iKc_MCOd;8iLRCYeDnOVoCkQsY?O9VUC+U@t@-bm}%qD)~bwf*XaV8p#BQX0BWm0(((!kYx zo}*^@MsE|v92rEZVV3}|Qw>HdGPYhk8d%nQA_L>~a`zTK{z3`4w&l`{OQge4jT`nI z`^VvmH0@~9Cg}U~SG6*aKH4G<#UA!D@UStJkIV^n`}TaVmFf!}Gy0eq2W%d>9fGX$_1@o86jtgq9Lxv><#^XE^>16EArhCm zR*i|PJTa<;$ z;u96xV<3ccN&1)ad@lJ_O##-cHP3XHe}xGV*UfJ+ff_R znypD1+*Y3!ZRg67MOtpB{Oc1O4|RW#I=*Upagp{M5{j# zQdcWhXNthn3XtIW)7!Ebc;&q}&bV>N^V*N*5@eW3t>NgYCg$z-D+}Lzxc)UKmhw-C zJc8!m$+6~cTIan|nt`TA$au(IykeW<146$}RjUpZDB7W$;qulQ$i9GAAk53~g=z~O zPgbuz+@70&J~$1N-$06#^}At4PlbeB`_QoBv-^$q@!%>@JWl=G{NK|^*z4i&*JsA& zzGtr;{}L4s2|I0~yzouCaSgZ6b;yVg|N16Fw%EBYJA$k5pGm&Xr@0_68!=AFHPDg-I@3{)~7y0Tc(}mt0KgGY@Q&=~-pby)a8q`h9Vh4G{CDV^BeAYv0=9CZJ0=sB2$Fcl;#J$uYS)2- zw8Sv7O-I-v*)LT_y;RUH5mzwLG;E^gv08Oq?^nh21Xp_;{uMItfgD}e`5o(T#JoKA z`cr(Z7Tb^f#>!W@ggfN33|j2Ur#2*hf@MdoSq=%;mU})bP3aGd5ZzX~C8Wp9wGgg_ z%WeP&V|6)%PBho67!%|xjCK84H6vF_`yDRX%pftIl2m9ygaga79e%0ym5WlV;vH|oa+)B1LQeUp8vdGZf3+n_KE zZbH}x`TA*Bw5oEiKf9;p@d2*VN$~xJIH-btFzztOgX2XdB?v`0ZVAJM62Ef8XlS4~ z-re4kQhE~Fuq=aE)imz#j5h^(T2PK*czpeqkqN_K4ygEDzr*+WyY%1vAFvJ1^A?~{ z-Ct@b1Tm}4e(1tzR5xnMsTdkhQq_oXp45rbh6}nWEl=$C?qI^GzaH!dtExABH0vTI z>NvMgKH!<-YW(nU|9f6=C|0m@89rIdv`t?YtW{ke_GKJeEd(xcr5FVyl!&jC)UgY8(X9)Ye)Sw#rie) zUN|_Jh}UXPCCGoU{m;RW!`-*wt3aRge>e9ZiK{UB{`~$VrPkebV|9|gmKtiVwEc}& zZ2uERd(`&|EUFML=Bum3?nd0t3%v$&dgb3T{3kZADmv1Di#q7MMF5a zmEpWE(4B!ONC^bW8g+t7ULM}HBSRT!2&H(lFi3VV0&+3GE`Zxgc#3$@CJ-%zI-5?k$} zTNPo7_W<0IUf2)1pGRO@vQ3=ulcC$n0Bi4t{u$ug(l=JF`I-tVg!(;e|RxV$$+}cjZYpK{h&lhU?-S zd^a|MGp_VZfGnFtc(DnxOJN+lkW-AD$d^&2Q9gC!{>PEcJ88YO&+Naql_NgF`drI7 z1W8%G`xG(#N=PD2*lMDS-tDh6&gdaIFTXPV?hu8eR}dxVne}A;sU%g}OCjhEF*(YE zsMri|DQKCH+aJo)8&4_%inY|8<_mn4*t+!Ur&|wpbV{pOa6>q@ForV8j>4lrD^dTeBSRmH`&8V`*55JPf z`m4JBSMfhah-S~{NHG0m{ohpmgh|aM6!O$u&G5f_hhl@6%rJK(0cp8uMeC%Mb`=zL z(foQ4*|f4jl2yoW1>Qp}Bs7!-GywY@3mhW@do$IFEpJ0tIrbfk=9|Wlx*rKZ@gPm< zB_18`Y{ptmzaiN&@Kfk%TceAtp;yOvmgpdrc-Rdu-G>4dK# zrJ;pQ^+_py)bv}SfpSoH07mHmH!C~X~yVnYNurZ9_z=JcXM#fv)m?c$_8mm}?@M?MUTxBtN zOGs(ayAw&PFjW4)DtW)~rPm`NMv=4uP^v|OHe}QV%ZM63i%3-qowCCCm2({Yl{w8s zn^OFJUigk+oJ79+Y%01_2#^DBSb5+R&PQxsf@ia(}L zPW)4{$h8XUGMD>1UK+6y23*-~Cp}#qc6~W5u}(~{rHW4zfoXD|*wLcxSU%!Iy;tsQ zTIN?P@#{kaZ|GCH@q9*DUVhcs=cGrIBi8!ztNGR6mU(jHU1J+zQOl$ zG;2ipvjy3bomjB^BE-k6)2AtSPz9nVA8!%j4V%7UecN_8q#tV^$<4*}E0I1h0$myo zd1Pefk&myh(PGBnz;&TbRCL{%Y;MaZ<8 z27Iol_0@ zenjl*4PH&9 zk<4;p=~HSnFumC@;U&44u_TL3%=hk%<1MV`|N0O|!HYnNe1@xpKz^P%1ZfsP zlryP{>7ffbRo+J4n#`)5o4*MbwK|(}+Z5FXV!*V6-_skh+c`wHpA#>tVt|=1fI2Fx z&_{N?({xOP9U8uHWckg=H3@0k*}JT*%J(ju1(JXI2^v|t+W~~pSzK{ z$@l9ZUN~OgWQEDAEptBoHKKt+O_AbMP?=txkYY#S(cIPVxe52KygP$Wh_nEyxw|yj z_LW>Uj>CB;;mIerKcb%|Zij_=xX-!&#SFGfEQt)DCm~)6Y>=Ylot0$bgcdCuWW)st zeq1i7>Im#zWIGG@A#^ch5WYt5VPF{QaGehs~OJL)q?c)!Q$SVa%*)(h85V`_^mMn`o!i zi%Hgka-}BJMF;+uzdV>ZL9|UpoXKRK>h?O0x^mrcPQwaIP2H80O8KI~6GzM)?0%4< z*uI#aBGN!3x}i~IM>?!gG{npj^6NO%07kT#;=GTul~^27DP!o8ZO`_MelPBYOJuxP z@%z-wFON_{_R&i(w=fw6M!uP#xFixs%i2b zD_^4FB5l#n4uJlhZZUed>OW`3h?*C06W2%KpU&^|KLZZCbbf;F z-&G6`u3jw-?VS^9jo3coJ6iuqV@8WjY5WiGVnKs^_nPq8k)87*)mSk*Ry*+RJ5LRt zR#HB#oi5hYO5}#3Z-(XCQVAguWjl|StTUYQoKqyVUy=5oTD~rt7m8-$5b1&l&`3u6 z1P9`GlA2VxI(cS+#yK=E^2?)iE}GkyeVJpk>c;75PN+eM>X7;G8g$g%v3UPpylNwu z#mwU7XE(*8BtFxC3N334_-i;4bcH8B4Cqvcp$*Ws+sVup|BaEaTS|6a*IH)Y#SSVt zichP6ErieV5mpTGhV`N^LJf)+6Gi)Z;7A{)+POR)b+$?XXuiXV*Azje*PkNsO;g}| z7r>jeX~CCWroy;MyXD{`!nn!E>{PV?%RLEEn5*B)WL2o&a<=?za~cI(pN>!ko1fio zhE{l$F=7N2WApQ4xrt=yJLJfsIf8s8$`dSF2Jp%871RNIe_rC$RlP3Tv80*tb<0}1 zh8*fQ{%HVhz3u&=4G>k_&V2Avci$eDc5;eU7?OlBNJ+k zxl#e66UPPDdTy;|yvag8$0iC7kmeq5cxW`>pFhrPWCQdXvcZMH*tcgjcV6ZiDVD>T zy!jGpu-et>{J`qNB}+5jAHGm@;&Sf|e3{tM`cvBEHR9#~KIn^JEo zjL541sQ;CW5Ic^s{wB&G2&t!jFu4EL&$ttp1>W0^AGL0~H6sJ3O;{(yogb3T!v6C9 zdw;R?_qL{cuScT(hJ1hl&VM2M8diLsv+#)}Aw3 zHmvxuVeqt1`NL27H_3r8=m2v2806QpuYF9`pfjRHXlTeL_PmKZ>5@16+RVn~1YA%!@8-`>>)w-Ay98^|uc!&~8J4zm`$LN}YhHg^#zc&Gc z*!uj0)k}hroFB_`c3Bk}(JOf84bA%uR(=sGo;-0jVI4S}?7c9>jgetJ{Z^|v7WdNi zwymZ3JBA7q$u6RRZTM)Xt|}&nOq-GErJBZbP|1i&$?k}D$?i3aRB7(j9bKigAB{D!hW29(db{>&=+9uhEkp(($~me~Up zqV0{h_<%X#E+Ln?=9Sc#0iYf;YKy|WXyUAHk-bnn6 zUE9%4uEe6c`G7Mch+PQlN(9ptUo{<29~3YlMUlBOR+qx#zlQ50nPl4QKs#^nYg=zy zZwksk+|M=@wqyv^64gQr&5C)OCzH1{>{KM=E>sFOZn?Rg%_0~!2N%Win&P_uGz3z( zQTTM>iqkfSW9$+Jl5&ikE!OXru*8wyVAL8|)B!1;wc=XT@r+F%+7Sm*>y;C^nQNbO zb3a%6T|BtAl)>xk>p|UfIhTls!1Mb)e>0i-!e`?7f?6W_P083VZDHlQQpPB#lEa`v zr!#rZt+a3Dy1J51ls`NT1WS!M#wunjr;9aOqHgnEmQ-PwnsC~j$)zU9HHw8m=jJX} z$>b4;obRDH1*=k!@h+v+OJVwvZ`zIiMD5-ofl6#grF`$7jt=?unhGk2imrKSB)WoXY2di$}Q3$Y{QVK8OI8seM`UEa#}@nj?f_;nZ#}Tl-}vlzF#x zmF^i+$!cm2yIt+sAN-n4MaA|&4LLF+ytUd!Xq8rS7M~b_E508*QgA(at=dL!_V~RX%yPj zb~q9AQFLloZY~5ebA;t+NhDS=i6U6MBa1L2JI7cesN*Yun@}ylR6X=rjBOVmXPV2^ z@0G-|jPSgHa$|67)ZA>9&TJLmY!$gELj<3MBJ5=Z3F#(0mpvIi`i=6B0%(-rSqt^- zp&SmME1sV!zw?YU>YzT0(^f3?aRh0$Z0c{Zk8^+i%3Z1o#eKl(1+(^&9eTSj0l6p?71Pe17JiikqX1FfZ^tr-_ZtJ6rUviXTo`+mngwy$5xRY z^(3KWk#GGb{~Mi!jzYo@IDIe>M;F8jES&yyBd5?BDl|_?zJ&@^3_iui19d7KATA_)%nx~`EDkf{ z>`JF5zcMAd#5osNEw(LT4|4^A+^Q0i=aLi3ax25&;9XVD*l!4uUq};qByzHOe zH3IJMc+F;&#AFwku6LR-K)MzjZrWH7*VKyy7i!rE+23y60`YmmEt=N+_PD#c%#w7j z_(O)5OaGHE_7;AczpP#lR=+!DI@ww4^lW#Rvv2_jq$s)%*mS+Iw*aHj8u7`&K3A3N zJG8;=i634drr*U8pwOWlf09K5G5nQgu_FFEex+yACUwls!wkcuyv$Kb|L~1W;>JmM z@(Q_A4<~doTSTJhv|n4r1NRpXoNKbPL;C!wbRcg@{f$#`fY2Hu-pIgh=LA9DftR zIR~cLgz8qT1n}#BLCQhFHEa%%WojRN`)~a+6LrtD<1W$0dQHDbjz6CO*E?;vV)=Zi zhmOE_K~dzMiPdYs<7#Y-0u`5R-B8a=Spr@f9J;Q+T;=iP(K|jb8rt_vwR~ zyP1@ZnY<63Q#&-CzhwfQ4%+!^oIX0fm(-3GbuzQ%pin9aRW1?A!NdO`X8x~fT_hlR zeY7Uc{m8W%B~7uK`dKj%yH2S;Y}T?Fjfto{^@5gUG>zqxSgu;GMsjonz# zPipO)+|rd5n*hs={D<=o{P2c)yTxC5jX;p(Dc(Om!b!Y)d2&|Qg_xu2JNX^SfBsH` z$`E64!vTxo@Ha^CtXlnYbQ7 zq2}}NxopP)xowGN^uceQG_DVi4Gf??Nj~-tSl}b%-x5};%)<)za(N3Fv8Xi0rel|~ ztl6>o#?I@<1F9FRorxJZi0Wq-g1(Hh8imG~8lM7QcQVmQSpxpDANP1yE>|Lh zw;Fnq7&faW(lawOrgQ@2ahapVuzR2aKz#PS`AQgzc}!w|_&ud(Us=S4;K-28M@LBq z=IK&Fbt9zQHCsNMEOnM1GYwofA`uQyvichBiMI(;kLEmJa>~nfrIW>dso6}sE4-WG z3V^}^_@o-Ti=Y+>CBsD)2sdvJdI&kR3U&YB0hcvmr5odFU5Ck-0xNHk&G<)7)Yq*k z3K&@7960@@*i8kHZQz^MZa|@=_v6CD`-87l9b3f??Rmq!|7=#9d6x6CZAq!5df!xQ ziWx?C+u?TL$)_N@3BFFhJ!R`pPm;GUy(DN_>`PC(uX2K4%){^98Q>X@UlWEv#vLOm z6>eAs2*Ty9x{#fb7-8s$bjqQinbryPGnE{hFh0!94qiC}jK> zgw-%a(Eo3(`~CF+@Bkdw_2;;)P|}~n@#E}<)I|wa1S{T`hPvyfvAVeTZjYXwGLor{ zFze!Lk4w?oN-GV7Ty665*r9YFAsfKiVb)CMFcsz4eoMp@*Y*lUhemJkZDa(yXvb?y zCb?&2`rl?@FGP|misCTn$SL5kj69cGdxB_is!d*&>qm(i)WUj#X`T~gVtXZdJi57F zIN?R0!OJYQ7feCBH_D(tettVtBDNB8cbOz2o!!{w>Ajm@ zgSKhUCwWPES8pm({={2iwp`RN5!Gt!szKNw4R>5*(!~5`-$q;d$dx@xHq~b6TXjA+ zKBT^-_M_{Q&koP=f`6Mw<=kboWpQgpgu;4qi#?en*8~g4tItBq_R0`K#$|=SM(x|* zD^jI;hvQVA26V=%FSTSsPKUibuO#8Iu(#7zX`H!j^LiGq^oxR#MWe=lAX)12K< zV0^kjWS@ZjoN7WNct_ou_sJ$FjA@)cgR=hFf4%>)FY8?>3y=9MA+8^gBj$@ff=$+3 zo!WuG&c?J6CH(dbC+zu{V?^CXl#0C2)ElcmRAAzt2+tuP&Z-5!eG?B>X*OIW#nDx$ z(#Tv;*9_hnlr*Iq@r|-J!1Lt-GaE85w=m%uVNtFQpewL(Kb^R$v+Nk_v@&R#to+58 zN>rWiTr=lrcI0QEm(v0&{ zGGqe;D5FcB-E&O@4?%;yI zy7a$a_27n(XjkWdvxp#KyO^zrXB`V|oe}E6ZIm>_h~ojPd1wr1wiw9o*k=#$Z+ybd z$53chaaEcBg8LorF|XQM;NWGELmY;i&tyZPUj}h~*kyFvb*{n}DGSub+anMy3y5MJ z&AEDVW;eDAj%qKQpU2h%$hMSeDrtn>mk0dZPiz`-Ta@eowfxGrJo_L3(~OYNh@?pC zzf5Ep(lxu|2CeK8KJl2enH)2}j$~QwZ2uDv6PXYH5EZDXF5Y?oe4TJH`DS!)*O$_a z+kb~gD$1u*IW}om`y(ydBp?Yk6n2)#hD$VrkCg=1v{7<6R*@JKDQN8rgVCN1W%hSb z+srCWy;UEgoOhnwY0{AYI5kN*uNQ#T#p>;Ir(s~7@m8m1;*&mQ)P28ni+IBj3@ZLCR6K^QJNTh9uLmNocyJ zoV5xQIi{;_!WyX=5$~rnKEGa8$*-l5)!fG6Hf%`1s0C6KFGTVDl*y;?_l%ud`r+Rk zo6}0UuQFI+pHAth4P6F$m4D@DeBS96m!#}j>irU`P?eb(ZL(ICcIiHRcO3x12C!HR zgY4_dg`Vl2=a9qq!i6KLL_7sp!21k2iy(xt!kc%UqlAJ&I1NV?>fZD6EeF(r{mhPq zzvxH4z9)8DyrK9064b=$DUFPd3&G|x4=;X*>aIz+03&$tl7hBjwG%O~d|E47@80kH=+d(<8Z zQx6>Qr*f}h5PmjKYDnXL(hFjE!Tlg0ypYBVimhw8AvE)>LKe5W{F@%V?ALoXPEvk` ze)=(Guyzw^IqtfrVG!HnXjeF$5~1@OyY-m&3N!($zhT(yLp6N&n$qp2Hj#Y%ECK7O zrSUvgx=*-0(2>h<^7ryREbTHK{knlGd7csUx@6kARRc^$6u_#L7x8f;MQ+94EMFB? z@sd#H%e88M3NLC*_GrkQf&LCkpF~HSI_2Q=wtRg>2=Ff6k5_xQOm_qzMetX5lx$JN z^9sR11Y%_}$FubBCB`|+aBL0B_cms{m@sWSGMuz%xK55-{URV5QBcpcuXz+Bj@K!w z>*_Q1^5p@O=j$n>@LO7fcJ>6R4s;$0$`~HD%0joFhJI%&n~~oK#XTnsCbTW~ zgU;U?jo*Gb09uL3Pjs)jd*lL-ejmP;Gv(RtGc{ygmS>@7uJSAMgb-w*8sV|WI$Oc~7cdng4P0UOc{}2< z_;bCjCQ`=wOiK50hR_*h4T|qnq2c_c?8#0@k=zJ*U zxh}3?bSniquYI{peJn{y|9Ry*;1CFE8!BCyZd*JR>b}%k`7SeldXX<4SYYY;nhr6r zu(+IzRGgg zQZbyg+QUW3VX4^cdMe%H=7pp6(j_wVcq!Vcy)*02sLAqY#*Qlz#nPYd4ySzHM5oOY z*ZiH=!}C`xt(3J7Q9?>2Z}+8_e*MVucHxkc;V%`_TGx%R#lU#!{@#do7t*rKoV)HK z_WiYM+MsV!V3;OIvHe3TT;NYzC;Kf`zM&hR1%l=GSWIu}IbzDi&}I83D$+22{-p={ zT9DZdGwPxaLYwt}Q7iF`sdxXwtk9@3*xQ#TkyJ#+mI40>SY(k9^GfPJ4fMztcna(r zq4^%4cuX1GtVddSPAP~Yl0x$ALc~Yk22w~Neh>IuA&j+YpQOQX4xMojOXpFH3(JDd zy%T%=^2#QdwOhl?I+(Tk``_gjs~%wGFXQKMMS*fOzu%omOc^IV;4remTLhq&F)(`u zD&l&YKzswATLehbcC&8^%}kTxo&Crd0{N7@dWY=G_LRkOdv+<97pI|kxP`1juXJIK z5mX57*R}_J=n@Dgo9krRg!P}fhPqIYSGkDX`jxT58{~UND5^Z#q#doo=onW6|U zpl*u_1q+Je95sPcz8m``e3izQFdqZV3c)A_G)33WA488ej5ymjc; z%+pL;G?TCV%~(t(%2G_=4qQUMV*+))N`#9eci4reQ*N$)fS)I7+Q@}OTWV=NB@JL5 zujN^?jNWNmDl2c1j+s?fn#S(#t5@CoAXfGLY8GD}M=Vre+0_ex$jZi~O_z^tB<9oP z{j;y%)%tiKUz6bBJ&(#(FfdO4=Va2CVFA0Zeh%waqttKL6xiWh>|DI{Q?Cgbi%Zdb z(pzj)0{JcEjdahPO@(Urb4J)kq?s4o>*iZC8^nZ6IS%k-H4HeHq(C!{WwEp4?c=IU z9?YF{tG|CM4==7{VM>OQ5H-ebok&Y_ah<#vhRwSr)En%xB035Ezt$&7e-KTE9!zbc zA>=VWsH6s7>K}Z$M|Fd8DozLc-n~WVvHglm6Rms{SI+ebOM>gj#nUl!TGLZr0!mB4 zkY1)w49`^NbYzMRsNxuShHuZK9O3=n~rh9$UTSOrlNfxj?KfxK!HFxX8mz54QTVIkMw!?Roswbmx=Gtmh%8I4suel4FnTOW;hlxy}~+Bm2hbb7Mmhq)MgZ(_4My@bnhpf1LN*&2GE->!+Zy zJgLej1O9IkR`uOKdF4r$qE`fRts%jqBsvqP;7F2~WKT-b8BsIdnn_CZ0&(Hk4;rGR zdR)sWT(t6_uI%$axA`Z=X_>FR3hOE(bS2e)0V&-+%o8t}UGM&8pftW-BJuEm9A#b*)nLu| zvl{ZMR%Q$udh53t7U?+P%$Tecc&5IZG*7v{R+DZOZAB>t6p*WENV%mzC5uO?3}yi~ z^!p!kLB<;gexRQla(7m96fL=879PHkuM3Q0#_U)DGHwB}MfG$$aJK=k>QX<-sUA|@ zDBY^y0^A(A5lY=Njj}`sn^W+R;y@KQFFSoUb34Pyl=);%OFKp87iP{aKaSA-Vb|u8 zt0SlE>UN8E56t5pk6icyguN6o)$J#XU4DDRS5!aNx9M)l9;s>z`X-!_DL%WW-@Hs@ zzHDR2PChPpH4OqluG-7v=VY)aei-0)krTaNaWwJe> z@;82r1$MnSF-{Zo^Wv6A(k_mK(Cc1o{ZaShuT9vYBFfm=Mjazj-P8LTU$@dbvttiO zTUgtz%}^C@emUvP%GYMT)^wGm;TL=Xv5~7Z27e~9)YPIw9fpa} zgyP`w3n5wrj+?b7W!I{Nny1S?k6_e76#c19-X8YJ@+^0?ZdlU}y~F&uxy!&3e{x89y8RAj|OOIR)ew^|uS4WWqEZ?u9{bnmYPJ49rKO>es1Qu^ zhapzgU{{0Ml8tAHcmXBG)-+q)ob@9+j2zBRR-KqQHh!28P+B#goaqx&ck|ZRquud> z!HQ#xVpipa*8CO@7>EtNM6R*t+Rxq)&8~~;;Ux?ct;_(0u;E4D^c48c&{}9~=9?>9 z`HvTvoQm}89DQVg;2$c3DG7tCDHZhhJhv$ZpPshVY=|?MxP8k?vyCt|VdJKP=+&Y) zV7`w}{U5G$h#0sBYv!2T+75=nBdkR#AjKUXPU9V`k0bN1cjEqCU^5#R)SIO8j(j{2 zQO#D|O^iWK*Ci47HLl)!U2r7z5e(uQ@Ma5YA-?^C4#$kT*;?Dh+75z?YGB+h&%itQ z%f(uzQ+r9KxB**lm)VqIB(6e&>p8u#14)fU6u)F=s;k80G4DCSBjEu$9+u+(4Q2d% zZLURy-b6Z|JuOv)}#46?-0ygwdNZ?+ACc!)Ia zYhShcLFuDHt2Y?JJUV&zd#prvCsV3~!?Gj(4hsD?+&?fA$up)@W&zy=z~vgcW< z*UPrBGmx3(*QPGeX&D&C&Two)o_~0pHTYE5Rsz4c{OpKgrT3S5Fc@Kc`C>FB=WeN~ zcQ2J@Ug3)ws#?FjZkH%vmA$w&>I8m_n+&abFAsS7q8j9&DGF#U@l*mGWW6AJ%f1Qx zW}RGKfrA<}gOZaRT_6mr$>=8lHWf%%Jv$G@;K3vhq%9GW?1--zDoBLosms93+e6I4 zsO|L0@0bjdlK9Oo2lzNxrY5b3RciESB@P=5IDi0hdnSWvq6xFvmrM{6Sz^1)dtFE^ zh14&W(y!YpM1p3fC^eNVn+Lxgt6bQRsz_ftezekT!M!Tyz^b4mZ4(zV6)kvIt;|^X z1ccBjn_MgHD*6;G&|<|2GJ0K-?#gwhO@mvU{d)9hU!jj6gM84H*3@}|6s<(|A84|a zP8VY&$oaIS@C~#n|h2$4RmEf67>I|NE!4Om&PeZ zP>BLiP{%m7Be;KLm-mUJ>)=jRf5+6~)#L*5k-wXm8WXilNNUAVfUHWFQ^dWiY^TrC z<7&y{BKhL&+y^@saNU_`NL&=HG1+6DI+x6Vr|8irmPta|iuwLB82{cLVz2+zrPxp0 zc%sxv{2L)MSX4yzh@xtN-N@C6rOX<5kIaP%XV66k<8U($CNJoe8|6Fg;OJpM;h#dS zf&*#8Pk#LjNVmWZPA9L9RRQA)h)0sDWVMq0{aM3`oQ4Z92#vK5&P^hu>6wc>825ZV}kvN9(Y3Q|RPWx0%r; z34()%T299&fSphGZEs|{bGoM4fjjHFnjT=jo%Ti-!?uGo$I<>Ht4r9#pDoJl7Z+pF z++Ex8a#E6ey+C6l1McVNVeI^Qe|;XgM<~NlAL8IzGbfHM^-%R@al07Pw_M8{`Axs} zQ%dQ?X7zRHii|z+3x2a40M!qG-a3m7R#@fB?N-qiQsJ}eqlq|++BsAD3Um;>(Hz}tYOx7dXlC5eh8;p`b=bL+MPeMguh6vESmGP63P`s zD#;aOajrh+~ylJ92izTM1c!ieIZb@nm>p=gO}x3S3C_@ciP^^?Lgei zl>j*TnSS{lvTN;l)v$WKXK_A&U6KEjl^ZMHaWH+*RzotZ>CxrJ$d4{g-0|ICukLWbp`euY#ZKe`RsJ zpVbYhecD+umNUgbRgBNotNB28hZbtVxQ{RfQsO=g0j+Y#wD0$84^F@Wn8iu(zyB4( zn9lxl`_E1-hyDk1Qlnb2yaMF)uJcmr%bIO+bZE_c%PiDeD0bu*J7uc z9?pqV+8|4eH0cCMHxKv)%A}F*k1}y-&81W>F$IKT1H*``4#cy6EBI=s%Hj&A0gL9ekr`;Jmzvvtn^yw!=p2Sq&Ang8TqPLN$9ME^0Nm&} zA&)F<5%C{d?e;T|FUfN`sH8A`@w}iZtg1VxwX^qtx}!f7o#to`^GJt z`^TH*q0wJit5?^eO4>vSx5l>_GgYVKjkmNDEm(dd5Vw)Z*$&>vjW-id^2rTPnCo*1 zr%Dz*{rJ_XfB1>MS+Yf-Y|xgo_(M8W#8|}%I(WjcfqsO0U-BQbI)NkM6*tu{-H6Zm zG3zTq_4I*sWFJZ7S%ZNXk`L_FtI@q=4a^=i9I@t6>cwBmF`KK;`%{v`|hRdS6x~M^^$V2RIN0#{e4a2ag`2z z_1Yfmf#&`fm*Id2g7A%F=vD3)+}vpPvj0Hbu)FbXa!_MNV_q5jGcC%%#rJl%zY1BchLP9Gr*cbvsY z6#vd_%r|4ry`cnc5Esfz4`KhkP^;X4D4T;F&(B263*T)^P1!I`Cg<)o`HS2kDM6aK zNdK|%Tes)yVUg4fP7ZF8+!s={Uv#?~i?4=zg6k`<-MH1_^e{+fQ%1NmqKvA%j{s_~ zKf2KH^#^Gu%C1e$wtfD?e~VO{#Q!F3aB;3Wc_o+AG3*`L@ujiDhQtSXT+W&=O2uc5!M|Q_k5HT0bX!FzG7FFjasr}fgj{@au*K3A%1gG?a8evaLg&izX zHSulH(-N(sZ}wI}g!*fIRRP&j&4V3eIfT=b6%j5JI+!-Z7%$$wA0nUJmfeK$MM>&B z6o>C%b1U_)(I67Jk&E!*K0WMfPtcna9^c0MTezFxtuWu!F4|aUg%W5EXc*>EEEpz1 z)hXCRP%HmMr3ml=s{t+_&aIu~Q~cPb)NU`N+DvV)xwA9mCVq={t1<$vW7vUnj8(c? z)5Ejr4GsKu(?HLF=)9GFRwzpPo_Zf>?UD*rZ?$Q#C-}Q6-xPT%x-zYg;OMhh*tIDQ z%`|$4*hLTvswkL)zLL?Ktr&0sA_5=Ms1}xPyJ>G7th#bDMALscNnLkHY>eNRl`G(^ zb5&S;J?R9PsBd@%+!ikkt{z`KT|kt8zD~RO-?`i9s3=KY#MYq4Z6#XBW}aYG#L1nN z1cxiW54}%RJ{Undj4K)${tDafM@`X-j{Nf(`xV(?+*k_cnuB3&19+d^YsykpsQw)7 zmLsiH#)7DB#Yu(r^(4vjYWG$0$6wsioEmOxI=;V7PI1l;dH*=;o1uD+v#e0#L)@w63Bvs1ikPfR@$y@)f*G#K*W@p}sAuzwS_ygk&u2${0 zNPbY&=uhTdd#X|9j(T`6bgaoeuAOLwuZ(M$kr8rp!d3d31kPkI{EJ<>Q5{aaVrLXX z<{YLv-k{d01g5GQ?%`-&w%6-eym6Tj2NFPpo%1 z^8p>E-bg78N<=~YgWS?o1<>8N{MAR(%3z(qha8zEj=wW>1w_LXb<0Q=f8~Fqj&~^CclNv&G9yhQBE&Wl-SiK@_PjW5R(RUfhS8q0z<1EHtXC%)I#v z&?4D9HmoH>-c8{j$sdJfTn80CT!{C^-ZgZlxmPUWEchECbdJttwZBjw^6h*F*1{Of z=Y7ncL}gDN*Tj+&dbjQlU)x{fm-Ez54FOSN9$LN=Azz(W&!}RB*{}1yQb}I`31nro zH4(8LnRw`EY?T?+8!_1QDIAHI=6`ozDYb|94pTbMILpy0wOIw9S3hgNPX71WD$SY=T( z8b;xKT&8ALr`>6>=(i$=3k{O-slZE=IbTp_Bc!>*MDtgcKJFpDwc+6$U z{fQ}3$V>e0!LNgk)-av32g-qCDoCd7vV$HJB4I_l}(B?gt$Tgxmz>-GN@vI^|Q}Qs}7{_RV@g;U&PZ!fC#@_c~xc zC#{7KNspw7f#STvO`(A8%5b&9C}I7%^*yoCID02C?%!Ov(?RFS!BM}w$NU=}t@gOu z*oANTVC@{|^**_;ht8ANg!Yx_bwlQR2B~oNvc0&v`k{Qal{vB@h^ds`&isG;NREAc zj`v%1{#=?)Kd5@(QB1;R$H&?sU=*Wo!1$jug(}⩔AKZtHIqj5~3Z^JR8-seM_XE z95ffB$Sw|FZQ?RYnBjv+ ztK>7cp3%SK^>ma)d52PLA7M>0&!*2S3mu37a#Y+219YpH%o`Le&3$MV$?xQBWy*2! zWm{ohx;2@&#J#Pu1e9|4C0Ziz%_$S}5Q8E?jdpYW!D)f5`Pb?d68V_vd%H>Wzr%UU zjbilJXP}47CJikUm_+E@CNYe2CG>c&t#4<@s+ZDI;L5$^LDh^ns)Lc5;cnL4zVj%R zE`cfvz>3#4#c1z+L!SXX!aG)R45{PKK7ttvKE4ZXJRm3IK8RP}`NsWAY}j#cd0BZe z-K`jQ@gFlnLQn!>#ib`i+E!1QW5ey&Wm>R#4 zuJ9pzg}EaOref>(7XINWTa!`6SKfH;4}4O6-+B);9AZhVqoOJiY^zuK0yd~@$Xv@r zJI6K4b`LD1>6MA_6AThzuvrf0P0Ta)L@C*umQ|XapQE_~oodQ^+#Fh!>tWE1>Q+Ps zPkQ-(4h^V9M*O361;IeXvis6 z5s*nNB-5AV} z&&&j;wK{Rh(868ohNl$yz_w7x3fudu)E>)}s`aWj2BSI|g2atYwEtAwwx$}i%6cTO zqeeaamppt|ld6iLWljEc{_$j0Y z&(I+I3^poPzdGqekZwS{iml)HugLO)EE6#ewbzABs*a>MYSTXHyI(JGxyuIHC3k}! zXLcH$!G#AC3$!&v0^+ZXea|NLZ+Iaauy6n}>b)bjOa7hNumqj)tqL1=yOem8UhT;| zf(5nO-tzVLojF!nRpRHlgJOgutg#ZO2;axgQhfpD@eRe~jRE{-{smNawOiF_0Mr&>;`Nxap}@Sfr@sZmcHCS1Z2wVwWYA)Pp6PnG6FK4Pz!&&Uy3bzMQXQ zG8_)Fnt8jD3i(lGo%nT6%*xO`n*-0I2gk36!c$a!#Ps8{OG{=puZ1u4f{`;6B^F|A zjU2JV`~=&8mk=h?SHi^^BBk%68D#Q-S(!|E#YfunV&7hwJ$RFq*mf4C%sRlm@TzLv z$ino29XA8t6U4}_-@vd}AjQiOwadK6yr@XI;zir~a@vj!*rX6_$-cjS_H+1-tnrd? zO#PDc)o|>(5j5ysm;XbFLJ2vE!zY~AaV%ysZm+{hFH&W>X2Vb`p17U2gO4kBoeA_sQhi8b-cKqq7JE=-| z)G9K}wIrWxo%j*3UfSGjm_5>V`|)@MIe+g$&c1OzMBfunuEI+aB||?sitYUnEJ@2lyAEeqyNEHWo^GdukO44 zSNVA@gmy-+5xZ*GBEBB{dA!3diGwv3RW!IG7Tm8HMyQcQukT5(f0LZnOJM&)vbijm zfoIl~$p{vyfP8*VoM$%`ZGMdSTR>;Q4dmsv#RrOUx(X(uKAEg)!+#6X^91oLoCo*QTVz5!_V+&Ia~=5{T;icc33z^g$9{x5O?d9Pj+nh3)>kq) zzyL8}PUXlkhuF{pO@aBXEKryMyD}3>MX7N^xZk{`Vh7AqY9g(~mA>`emFEOE=cARh z#+h&Cq_T7xNHN6UB@=6S?7&|vmu~2=!)x)ELp|#?DQYd^^3`69g!=!<0zhhyC%Fb1 z4Nj6kfI#^ocwD-nw*BQuFwr{rNv(rBJV;$0@J8sF3UFf*-<8cTAs@L{E39gWIh{5L>yQhM-e{@LHI|Tpg zKU#wtwV272H&olSK?O=?e{1Py6 zivzEnw|GDPW?WSruXEsI`rRUeKOkIKk!S^v?Cz8cVeB6Bdbu(L+R6X%(P#dE7-3EI zoinx%j@bvm{8Z;EORy#ikz!TU$(oMMw#DF7MjH*FX-DH?G#U`EHw+Xn;UW`>0~hhptGW5^g!7!c%6l%j&j9iFRf@RhIU}CQ)@;{Bxe8V? z8Sm>!vzV7l3&n2_jXtvcTZs{^3lLrs_s-wXvBLIvq&TJ%-Ti`%Mf3w4 z9lV4W^5GPKE*q=|Dr}PLu&;EW(n#xc)$dRtl)L=q@#+HMkaY9u~aU% zBDjQMiUPF|_86D2Rn}8W#~!srKcIUpd@K<|vbD_EQ`M$!A>y>~zP1M)R~EU*6itdm zGhsH6Wwsen^^TpRJg3Xg>6F1G#_j?M^M&pySX=IjB|p>s=PNvFqq{%W7Hq`&MM9}H zYMuMTh(%Cur7}o@PM+C=xHMvxPW_dPd^6d)B_8rP!MDZ2#rFM)3E&1KEhC{&1GKh8D>BS!5S7Wkp<&Pju{k&39-_9@=;S&Kku%e3qbZzTF(vMWijm#V< z<;)j>)}LcgYE{ut082%+A~9Rp!&e^fi8XP(%1ut+lTWn@6jCH(8{8T^f-l28t}Yi6 z>;qlDJy`L5TX%K{l$_m0@sD-nTwC?zFJJty*y1-?cn!g4eWFYCWQ(vCmQe13-RCSu z=`T3f6dD7qwg_0e+jZU{Is66C=XJrZLA3kM%=TOooiD^;&?CXSbf1Jahl0!ot#mwP zWb}n>)4yt#!l?f!N917tk8&iTpCYndrF@+SNg*OfW$7P}zZ7loPZGI7Vmo2%?M@}o z9O1p@VF%yes&N$y+IAB2y+)HhT66liKj1yldit>D+(w7!>REMvtcA2ocSLd4Z?3=v zS`WIO3M4N!ES&D++hP^|Ry9%m!g+O*Y1kYvU91|%4_o}Pbde*yZTi7YG$y8V{`f9u z^BX%>hj+!PzdZgkIa3#OY4@$)_!fV{7zIs-Wv~YPhe5DTN6woNHR^J6Kr~n|W-Ulj zisLcEV5k2bp9I`f>XhY7^1JSj7Q(z=vgkFlN-P-W zDmnvAt)fmq>dDV7woT9N!T6?F4FHX*gEB~r4|`RTw*dt?0)}sT_}Wd+x0dqqx40Kz z2ux2s1m068bxJ&pa=}0uO)2*{$qP~drYh)%2)o&Hv!N662e)5g*nl-@6G_ve_54=m z%H5ySZ(k%wsos=lU)ks2jnVql-+S2Z2)#uF@^{YTygc;YpazKdrc%(GvwYyjV6O~t zxOH}k_l)H`HT{sZWIWj&izr0Tq%mR^-}2z3b+|-tJq)@Us8^L0>qermwri#YbqADG;(cHQ=Gk`xG z3Q|I{=GNP^RP;0VevZXC<$(6YL(D?r) zBwV(HQvXqqT&Psa{)cyzz(#-5fby^HGO}BfcqqA%%xwR>>9HFUxYLrYJ7@IK)U#Xd zyDy^cVVbComGQ5ZnCH4&_T=5OD7Uq7P{Co~p@WH6-TED&^N$BC3hkA%6FNkz_t!f) z?^DKymQeLqs<)$)f&CDvJ2kPVg=?c)aiVXq&0*^%sdE|c?XWiogdMwD?+9F@^!}3! zk!Oicy_n1WlRS%tdU6;v5hXF46Y+)dJD){J)2ElG{SsM~M>W;}UtXiT@9lR8eh7|J zfUW%3kR9WTyDw)!Z_ThyqfMfz_0pnLf_)5bQEYvmeiH7hge^zE@%yODQ4Ao-U=p}h z{&RFF0jF+GTcIGqIJ4Huzp{HLbU0EGS=-!JKXGrybEVEZq7Y6&-V%F$WT@O}-c2_N zqR^SOm)Cq z)H%#6S-YIxr9vB9tL$-r8XGvwGbGds<_) zOGvRyn+;$=q{wm038_=|U_A(OQa+FkppqNR6Ev7AakI(m({F!|*6H0!AR&!Zu;*^e znQp#(PfD&oWu$a~bhS(&U{%n1drd3kN3XcoPzF|jmw_9!5N_It)+~qCU&7oXTsf~B z0zOMVy-YWmb-H)p0kxe~eOeG5ehk{{97?GeIk|2`YkAu|)4~LKNT*s)+xhiEn2^7IVytCW8}KzGjB=2?^&e(2jXUJYHrOr0~BDz$98^cF8 z$DHLk+V>2??HN(MsNIE26tRVlB1PN|IT&Vy@oId{=s&nUZ4B0Xg@?DFJ4-4HU0-Nr zm&>C!8&Wnn<7bv*B?@<9dfXAaZ=MeBEURtvD8|HY!5dh2Jc zE&h+?2IAa>_Fv0QQ?OIbx4F=u4&ScA#kK1g8yi0Pjg{X^&ymBn#PZK;$`az$jKq|@ ztckLq*1ubeKOCNZE&kZq&AqfNII$(}|JC^f@xI(~Zp%V$%DdnLs_*$&&mq!2Q47D; zA3NKE9(@*x#|=iF^LM=l6FS28&Isz zlnZd4I?%AAWZt(*R#VS!1En{Iuz(~v)wHCqA(d<a5TAC9_D@^0c@f-2n_-1 zcApE#(i#np7CWTY0gxIn6+NVbuAaR?9wDBG3p6f&J2Yd>fTb7Zurulvqc6T>o2en8 zxeaUo_*x^lSZN3APXh_hv>$U$SBB(B>b%iN5cOp!5({-njLq3>A22YuU_`RZJwxyL zQ^+eg0z%AqM&<8(Bt?6OY}V;~8opCpaNShx$MvL`({~faRHI75W|pc+HArStE__iH zs|PJnO@z~G#d$^p6VYg$xLt8JQMBy!4@!+aS6W6DvHj&Tj!RoQQ(A|1=2naOPHP%w zh?hnue$TvJ9;Ap;#w)ZJo0M(DZMx19;CsZW^q#9SUijM=y2s9!pr&Y(uM*=J>P4}w z^H(`3k6c?*9*(4nGy}^E9%>i?sb z#1xM|(DR@f&$&KG$y2WAFBiij5jWB5lt4r0YD)BZnipkg?N?5nvZ-UoQJ=Q$Cgv>I zU1Q#QWzhAuKOG=gL+4FmN(GZ9P{DN3p9|)02jMA*Thj!ONQ8Lm2K0he8pDTFcxIXIWTvj`u)u!& z^HWbLB=Eo`zwZiR9P~)^<9-T|+#(iPqWw~=5?}uS;su~ynu`Q9$EMhdSuEq`FNyN{I86tB&7AlQy@G=5R zr>koKb)D9dl2sbwgS-idCVniQXA}I`dVhmN?V%!W))*KiF*QRP9c_2E_w9^(zlaGO z2enuTk(+NEyH9(e`q5YlNerFX|Z0P4wN-{kmA46(Noo1&wgpkJslPHb; zg90jLICroU#i2y&&>uG5K_!X7zkgkPcC6g7(t;Cpz)Ti)MbBHPH9EB)xJQr!gMpQC z<SVCu;oB&WY5l)nhZ}hqMaVgYGoZms71Y5fX~W7{JJsQi%tuMT_{}$Ms?1 zUi5>pznxhA3~L(yl4F@;nfij529%bmrQ;X~b)c)Sg*j3I3J3*d>%__s(`iUc03y zjrTpFrjp}wyo7558L4EG)~hPHjDp~6GBo3k<{R{frY5S}6uvPV#_4yG%E5^dO1g~G z$Yy}*e-7h2#4-rpvU5{7d)8to+{Degq`m2}7Qb~}wB8n^u~PM05eCeBrW>=WPdAgA zx@*rB+E{E|yv~R;NZ&o@1J>do{l-|V&JVA1UrD=6La+;8_qG;KzuWji7Nv2kIFOj- z`_HmsR0tGv{9l$8y8mTaAz%c7{_b4zRKDBy@P6jn(Svqkqm5K#VD^o~{!5NKWw*2L`xLJ%VsV;z!U`j(PFN-avg5`Ze zQW5bFk6HU>9$lKkRvje; z($^eIA=4wXbA?+dHu89lcJ}!8ps^h% zkvy+XT2bZxLkwp={fap}!hvw~V5vE)eYeBReSBx0s_PvcVm~nG2isWH*N3#h-$@!l zD-1!u(Gn{zR-FGOx8a9wGXF+Skd5EKS(Nsz7vU#Eu5bC@Yau!~@)bmT<~_njOXCq2 zU4)0}2ujPZ5?_ex+F8=xjr!;4@Myza|}`k#^goXBqo(nL;?8hZc~ z|B1$}+ow$elThJt`&IhYNFi5&xmAoDUQfjV!mUxs#6NQ8#1TTbsHf$Da*^!Airomv zX@9Y_$|H-Qr`~eO=paTm84pZ!YG`?~LCYc7NcVTb-+q=aw%xtgz#7nMM{wnJp`fcN z-&^eB9}VC|9*f%t5-vHMmM5AI&H|`N^%gx`VO71FXGz_c%xd%PS958>pYZ#Un*93i zvL0>ip{ei~NzJ)pkix47OvgBReJW=7m*PrshnuMrs3ZNyd8q#GNrB%d@i0zbADMlh@VfJt z?(0bL>^~k7+XgxrfCXQM!Q=8a&MqFj+X}_97W*lJ}t$`SChpqxhwMAz%~Ws6SVBPLzL!d5g*ssp@WjtIrDQusoPpy)Gp z7y3+ga(;yDmoP7YD7SH%@4>5oBFBLZ^Hx^6Wy{Dldyb1kZw!KGf*RNl87CdY&m7EQj<=2UCgLoW$;$|%8NaczSeLPw1V>t|W(Lfw_!ZJQm4I)%as%a+ z4q<7$W5^Kqr*|JyfgS)GuwP=jo-@HxIdv9TWgIcxKsQX#?k~*?j_?eDgi-{kLN$z> zOcb1mbUOk|VLsZ?zO#ZL#B`?75UQgnQu&E|bXt>^Hiq2wLB139*vpLlC6>u4cvmg= z_ryY*!?9;|)qHlTj^kY(PEmiHSQ|cn<+873z`E|1q8!#DYVJfc^S@)`^bmcJvQdzl+!Q=!}L8Ow#0ceTGjZ;MvVJ^@1CxNFjnjkI9A zHSPi^5-2Zb3Ft25gI<5GF27!v3;c{iOrTD}+HSmVzPuAwnmLFWGQ7|%$pz%10Lm3` ziMyDk;sY&b;TOmA;v3~kfXc)&qG8S@4*hof0erozB4~=+Fi6h%JGMZptM@eh)qYPx z!mcq==AI=-P{{6WL6`cvEbJGG}+leup0&Oy%76I7djbHxKJBom{;;n?r`r1~rc zQh3S1Z~&jz;t#wJafqAkRr|UAmP&JCV8k#uF#Wu2)4c6Ri$8n)&Gff>#3htFGBy(4 zTKCI4$JpIKK}D??OW)eg_-@`~e|w{6ozOJkqy=W4}{ zf7C|x+i6VUbXAw5H)jKJWNJp-$0NDXZ(DMHkWR({H&8rlu7MsxbH}9cPJsl3h4==! zr0NL{SK2gwSHH9Mq&73Gy95T|bYx&w4@H8}xj?&Jux3Tl=9ng1R=btvRg#oX%nmUF2|@o0 z2}?hsJEL3kFSmfdbaTG|ZypGqJQ8(|K&h1_6|B*bsX{Nvj2||2@v4C)k=AbD!5y4f z9lrbZu01qW^aYqjMi3}=k- zA;E;Wy+UA!Et}#9O7@{r9WqW))3U|T_>T;-$}=VY(DpuW_M}Z?mSrGOIfMDO>D4VG9C0;XjE^F;_BZfc3`7<-%xahwqIpl^Q{%TxP9V8KPaZgOaXMMEi(o zV9Y*PNY+Xz)0Dz{bVxd4UARraKWa!X!7200=DlUvKe^hV+x5&> zuiMIM!`stH6Q)Oj5_8u+Xltku=%HC*+080><`ml~y%+G4-fxke-}TQpNu%#^*6wOI z(Gz`G}dx1!26VU}0Ny0sh!X)SV*@`N_8{cmTl&@W~YTN=5{V~9<#{&(&3(_41rY8gB&>Cg5OFz!(KBE-hoKA zD6%4158)3gkuueY=-&7c5DJNU$oGi?xV_t=`2H0nl<)wK53J?Wu8v6`Z*`WH5-R#` zq(_Sot&gba2l@fmZ&67lvgSNI7MPd(XvAkBUnT@N5C<82`sZM1H%g73s|tdI1pvM6 zJgC;q5IsdxNrg?w91}+o_;*ex+WXxYZxtBvu>`iF2nANOf*Yoo>PGesLq2T(QBM^`S-u+ zY=Jp{Qz(_fZ1uOEnWlc?-%3ExFv*yt?sOY;Z|W`JPzi0R&E#+fQhH8pMl!dj+gQnD zSq*Hy2`+^lVfJ|5ma65_`K{78$Q(w`maDwLo3oq-%sg*TC|O~%t-I09?n2$|g~J6V zJQm|fKtt>N>&8*7rYh7_Kw?wzYz~=>^?RtPqpGpDFDbxu@p^jk77; z_kYL*u_G4n8S7o3C}As(;Hcy2c>0bxlhhNK{{8kq&YOP8~)1*TAM5XO* z#O^1b6igC*r+aXTIzogE`p25nuHsHfONc}z{s8}ssMq$Y|Mf%RZB8K}dS|7`_PMQH zHV7H7Zei=C9HzD>h=Hscd|4`y9}V;?^eEfL4db(kn8dj`K8Ctxv&iJjLzg&oAxMq7 zFRltK{d0C?mcd6X{uTl`t)H7viBrlLFQhj95=s>#322Z1O(=B$kb+hVdD}47FKJTz z6Y075S?1$3u(&U^q*|&)(aUYg0q(ZjHg-FF&`b)CT{zpSX#F*K{pp52Wa!&xTc=%I zk$7{j36h&~I|=D4g}unA>_Z%E(9Idj4vVq~ zvw`6G5b_~E+86VYA=OJ1tLxZ-d3EA-8Vyay!%{A`d+$M(9JEE`{(4F!Q2J=zx)$Hm zERiLO)vt!`nLm-an~gp>0`&te1jj!uv!sCCkzMSq$W@8cIx!s^ooO(ZsT({EE$cM9 z$%j5@#)l7+P?y@B%p_M=e1N(OOO+GuL}O_fQpQYb)3;W2^1zr>O@~N9q;7!ZHXy_c z>;dSslSV~X*AP=9S9xtVb|pR~@gY-?WSVD&@-P7==q$B0{*s+%+EHOna=Dd2*jCjo z2vGi(GGUKY?et>r#IAFinFOr2^MV)H1$eHZt4|CsV@!DD=(RKB zzME=%)5diCd3{aF`D5#Ym5Ad;dcf%mefQ!e#+2IG@7ttmAWq!O$x7 zu1mf6rKEtTjqerqI7b1$8kyOCN8Xil#$!;k(#^)a(7^L~o8KqPm((XC;x}zGEN%PN zvyLRo&A-kr!{fibW-x1lCa=h^%Qa2kGi%qIusREupe9(J z_X*fzN;$j&dPGv9B-FJYAJVzqbCR=>c+6cwFX`HG<8Bes2(w57q> zG^`Sc4R31Qhniv6-QxSvCiQK5q3n*tZm85-@Oc%5ft~Q&vStQv0ACs+h&uvt5yVN7 zP=EIT^@!7wVEZ1~ zwm^cnO1F^&zf&BZ{6vcMMZ3Ai3fW{;la|ty=+{y(S1wdz$23Nd@x?Q&xjS(64}MDV zg=m~5t0=L=V$2t1k~!t_ITGV2x7ZPzaG`;~>&b>iep|{-@VTC}pabXj{I4^O(Z|Vd zoyLK?Iq7~w_Bco*83#1!nHb!X8IM+IZ8L6*+Py8j8F4WE1 zgr@2%XRLqtDR!d%uqR+>3RL7pI>!qWMVoelBe1--aw3zq2||V&gf;WxeT^tpk-!}v zt)aPrN6r<1@UG6;aIjVhY=FKy15Qp4K)PKw-t z7R|>zCh}aUOid!~-IVJ*Mp!Zg_{t)GZbVrX|B`($jQ>EQ_7xEPd;OQ z%HwfGOZ6POW<}2o2QIa*iZaz^!U-sKg4dzAA0*Oh8BNCkW@c){f0?@TANWKR&F{jf7GY(4=v0|pjh zj#t=I-}$q$pw&AfS!^(1ldSml=Uq`b=Roc;gXH|CRAnCgLo%6Vey$@G&^;CYs=rV%r zpa6Hfruy+3Pu0u<%-7>%y^zX+OI)(=LlJnP+^G?G7Sw`*urH?6f6|H0d{`jW9deVJZP49xe9_(?iWhqemK-CYi8fGO6*MHV z5}46xiRN7bZX%(D4pB?x`5#=y+Tx&5 zHB5#Ym3NBe34uVDQ->r8b%Q~=MOafhGfCi$__e(hmD7yt2S1OM?re+hry*I_?Q(Gz zeEWu#A%73pb|B93cY+1nLnPng*lwG>-yz}Q-n&+3kXGq*R=tZwg)i{Gu$)np#DUW(FDX5NLA+LJ|0IqwD(2S;dYGO6U$q;n5mKqSevr>+uhieD)3@LUADgul0~iZ_72?ZyODDx!DDL2J_N0g)X&Rz-|*);%7t+&}zV1wN9+1 zqRej%Umo%WWr)1+61nb5s1stjLtFa2|Mtn`ng8Ba^=G(JS(UzJweic>knMp4Hop2i zja~0JVPNKx5;pHTImbt3LuEz)ROMPm_s*)5rLe`kjjxot#7HQ8(hWeHtF=lC_AbXA zIV0j~<^9swc3G!<(Fg3^xR*g-ZG}P(6-@?_$|Qb6Vd4Tzaw+U_;Q~x#C}B4H3lkZe zc0Ewi7;q%Bo=G}6eSB7cS*%{I2ncjs42rzShZzgTonuu9%?<|_H-xB;TqG3zCVL7( z{(e^Xd|L#oV~&8Z>{Y|`nV;t85EK0H3yQiJONW3OQVwa|a<2Ej;>!WOZSjUH-})@R z1RoGh!6r?syDN(bnhf9usTK;f(L#if?Ow!kEwG2{wbbM0M@%V8D9i%Il~HhBn4)LG zQTHphX79en4coQdJQ>WIwfDgFQ=rh;!1lD}ES>(;CxU%GU$;qtKd0>CwblodJXC?V z5NRjf&v6GB9Rrv-PJ)b(G3f9ofLA7yU}T3~v3mgmpY9B9stEZkA0Ks{c81fcyg#7@ zv!s+yXWrG+E?=^ zZtdv{7~9>YsNrC@bMquTn@LljB$?udjv6MK^-x@j+VfZLU9a*NbLIViX zA7eIC27n$IXK8D-vyq@GPCdxluwZ&&k~JJWP%DK^nf)fGJ`Xfu>=08%LIU4WuNrZF zEP@^GwE|6Ms`SB0YkOO-O8WG|e=UNdBe0<5d3;}P$a1eD67%>l=|{^0kk_$DQ*>Bf|Oao9=TZoX64xv9SG;6jyW z?9E5TfEoJ0y^C3(y!0BD<~J;-eL}iTvATB=wC`*XePO|(ppu+4Cd}iXs)6YUnW3kA z;=)3zr+a&6g@FaHtb$9U*6*>9?oQb)a~kTYM?S+Ezc*jPof~*2mNn`SqdvVL9NX@h zi2V7TCBnXl16I0g@?&1Ad_>hMP^N^PwS-+>R+0t^ETF(X1CK}9cDWigMk`RzYeE(k zpg<$o_VHGnqA(MMiRKDW)rC8L}H@oEAuekrc~!kf$X zha6iI5u{X@jfML=E*RIl#B2{Ik@emYUtoQ?p{W0I3cME|>kZnUn&h1fhySFk1173d zAF| zzrOn~c5~&vo#8UnsO8V1M|_#mrl=C`rySBMlj6!N%vG&?Q`{$%nK$SSrKfZF!2oCJ zQTxMV;`dk2>aUEIiQ4Uyr>JYzHZw7NfhB*~2R7(G7I0F}1k1S!M{_lT%@l_X#ee5D zo?KG3Eq~k%aG6LzhU!$~Y(x%6^bKr%zPgv@V$o3QH@DP<{WY$tDz+2 zQ=Y(;`9(cyZD&sL#%cDWD$*zFX3%=^V=^lTU4gc4w0Q(AmLDCQEB1uAjGq6g_zMm% z71@Pc8ch3%nIt|Qi!-AYRxs$UkNran;k$6r&|tbCbb$ezJO$1Ex`)40#(r;N@i~+Q z_Btm}Q!SwrYZ=V!SA2QMiX{AjS)?b6dncOkOVvUghFj`SBy)N)a*=--@qh-zgjsm+ zN3>L=9(PeWS<#Ln`-@*pQrNG?Jfen!>v_gUkmMz>mz~qVwT>>`8%duIVLUbNHt+$V zscjY+hkm!>0GaJU!9~b7pDbD%V5>TRRPc6|CHfDvmJ>qT%WT@%r?s=u6+)k}R67$B zaVa`0*nC>RqMoh({Cj_dqnhLL^ydJ!Ft9V+G}1O_I55|s`{Oa}lw}MtiG2OU4Y5q?Pn7Q z7LL;Q8#g;HHl=}bu;^Q2G3yG#pZkU=y>5b*2MV=Ovc@q(G0yQL^WlI zgpv`7^7{pdB<=T}`pbw93cc+mslVw)Nb24N%Cx+vOF7a-)^n0<4 zm-Wde`N^821G9dKJA0Zeng^v6SDqmi_-tsL7{2pLVj->mB74EVy4HVvMHSC8S$<2b z;CN#UW!BH?Z$?(~frhXY{6-!UH$7>rfhQ~)!`Lj`i4$)s3@}ONSLd(S?iFiz?hgtI zKL8lZ$jyb%9r1+dzhl)uB-;TVW3Z(r4VcY|Z&xMF3AJ^^k2EZ!708{Bx|mXB0g4u! zeNRJ((_2IOs%b0lrUugDBaFR=sHlYOnAuPSp_7zVle_;vj1 zv%@ip0~@mwMy?ZW?w&=~g+rMywD~+QG`u4BgUv^K&QrLo(+TV}G5t=}u65H3o*s`2 z1gc3&@q6&59xFm;%u973jJmVe>k)AlwM-Ocl}t|o{XM>=C!rY+h*MCYE{Xj08C``= z%^f5f|66={>QG}s1xw7<^qhTZWK6lJLzcTk#jHZ}+>VqKz-?aJ?K z4A&S2MSlmB(5yq;Fc#~+Nn`g5N8U|^RcdCpbA%|5Q5GP`ACj`&&VS>xFz3mA{sDlK z$6UXr3oxCVKc&qzq($I4sOIi@*3CWkp0QMOf5r|w-)y$*JVLwqkxo7YaT0+VM7>4_ zt49gA(E8n+jbjfDhX%Rs*F^J$9o5E@tdXoh^^d1R3cfS;gjWNm;C~}Q4K6URwL#y3 zR+kC!L+`!W<_rSpyt|_^h)% z65tbo4qC}}3RGQ4F12iIKxE%~h^wEt zFc)~OTkH}~tKf_2z$vnr>HO5Loz9yA<>a^gra2y`K9OJdHY4A1`#%77Hvx~$0;J!T zCe4|mJH%9H;H2BlXrMM(KkdeCty8tlctr6H-2^=RGJ(9Ib<6y5Wo44@Smh?+(VrwF zVZ69&F>BJ+7}9_8oq=J8@TJR1Vs{E5P!E#eb6}zQ3S<(n5y0LEh}h{#KYcjkOQ+fDdVu zB~V!DTD(>&jKeSv$^79|Q0v5ZOC|AXbKP?Io}5_11*~jbn_Bqw^hsG%#^^*+IJvpf z9%@Vn5S|Q+U7^Kd372@4A7#Im=T&jpb~9`!d@}Q~4>IQ4HS2nL=VUVP+TLk#U9|Z= z8(H$|eB<&H@e@$-l+v4zC1R^l^5kB#;_FOBDdeUq2$P-;!Hb?CG~L_)y*B+`NK-;S3Zsz_K2`3P8Ux()cXKK+uOOgOB@${igaw$K zE=Ynqk!FyV`-f>PnyH<*uM}d2uuL&Y#uq>EJtVjE=g&fs)3|uPd|{$mE|e_ldT-6F zhv}3e{qVLnEnlgD*MD4jWdC4$IFbi95m|vWY%!=9 zGU4FiV{Se&zWK4WT6S|0#5WOm`tHuRqH%ouk+&pRBiBoL&wGM4_Wp1+Mq_hUaCP7v zTzvF!)OXw9URu*f`|UMMY_}X{9E20rXYYN5i{d*AJa7BsHE>yRC2haE@7A3t;5hXC=~zmH3b^(L-8 zKLMHa!HByty2s(?fW^c6LF4gw_S4uQOd#QVtsiuKI5}N$^(C_I-Q~nVl70Z+FU>cC z-0NhT58DsHUP16WU`m#B-Vg8QSTm>h)|k?bSF{x9Y_xH6>!KNEr0(U7u*Cf4E4Baq zYdWDniQNtn-n2#^9-Jr7P~`Bk&sA^ayOle>d0(uyg|>1!sM8KcK!C|KoJH^g`5SGw zN22UQUr%7cAdrm6$03A(zrIi%dsr#eWHuu^1|CAda9uG*DD1jlZsUtBJ~yx~F$xoU z!i6R0FEF@fN*?K({_QL3m(i#WlwCz6!o-deokg^k0>Q(Fuc@kL_c4p)=K8p}NYD%? z(%pPgqj>_Bk{}>}zX1C8vWx`|=TBel6ZrjC3&8iC%DXotK^>P-H-)W&7s5Ou&U4oT zYg#RP%K_})quPGgI-)=S{_VBEb(G^cRPT;OxaY2R`D>c!s65~pes$vWNCaA;id>_c zm`XF~uG{uEM#dppNlCG;!rFg#dwt&;x(25pjU2NKQn5E|Zmt+%qGz0yxFRFX7{9ju zO!vUPr;4Ay)tsHE>#0~s6Il?Fv5j)2 z@(KQvnJ=?8ILnA1@2PNc1&r$P=vt;|Txi`O$S&wrEEz`P?nbP$DFjvB8Zk}v?`b;Z z7}K8{GQrq-r6?#&A#4xzNqSv)Lp?a|z#iV&jct=}#cCAAD~-O$rTQgR0>vF$swCw~ zk*NM%)z0hUN*4DkXLiGsr`0iZ6C%yOx)ea}*FIW|JXw(h{jZmHc+;LbVKlX>d=Wts zY%*42;Kv#8gZ(~g6QpxCCOkXd$PsVk;8C#mN-J)&##b!TfDEOFt8x_9p3f+u-_F62 zHPLo`cN=p}C2}W?C>1GD&fKwf&3;oy-MAB??IJB55_WVOwoqbc@f!3mY{CD5L0P^xJ8 zP$Yd}S(;{2Q2WnVHE$(~f$xv!&{kh>G)34L9VN1SmNEH8F%{cSLJ#L1M@Sl|+Fyr7 z*rrb#Vg{0jXd~U|shEl4A?1Vx29_X{4b=n5Mne09()Xq%u+&#+g}5Bt%CUYwT5Am` zo|2t1^+65~?xQfRk>kO1-IRUpN1+R4g+CLW823{@#ki}=dT@P0vqRyTvl7JrPLaeA zQ$iTMJJZ^&0NEEq96qL+3T$Ost1o&wQ<_ZLke|qshntDtRAVH5GL`oCJk!Gqc@0{d z)x1lsPD@l^CIZZ@vn#m>V)TLO+!leJqdm7Rylb5aif%1FlRp=a#K3PqiiT(QgV~7^wyUj`jm>97UyxftB=|!n5_G zbd}JP9Q5PBz~1|QyAf_<>kGnY+xLYNsDRHV<(I!s^k{fw3qD#s?q=Td~U9L5*g)r zzfAkTZv00WE4B}n1Uiz`tvqfRI3YV%!1dbQ}9pwF5#` zGs7Cc<{(cdTd#U`9xWphrO13gqN&JGbv>`FLnY=})SIpo&rEg&4tzcQ$cv4{oR1lT zdg&s3M{M45!loj^xVkqxW!;bP+uk~N4~UEP91UC#1y07^?WkM2gIm9WLOz;{x5cR<99VCGx$H+sof@oN;K)ZmmBE%zSL#@`{)Fb zW*G&zeYD1u#X}S(xn`;%><;8!uXYWAhbSqMu~Z#zD*6l9fI{eAtX;m&ik?zCV{C~X z#7zpA9jd{-8%OmdqML)H8ZvjfP_q=lmVoSJ6Zptya)$`;w<;g%oUAP508G&A&E;n1=TVQk+p4Z|6?MS|8V`R=NeuD+e8uxnHK=lvBU-kTaIh__!!wb~tIc^bZEGkNlXq%`30B**|~>J>6YPDddcfKIW=I|x?JNgU0A=uo6g3n44g z)Zy*vQ`gnk(y2!mGm^SFFAlP2n~oynx9ZJChU9XEU%EbuxmFtz$&meIr}fW1o=jNy z4CNI@sUX&)UWo22xx2NyZmnU3;pfZA1dHHC(c2V+M}v7jU#qW|2)8?V_}HkvY2Vy^ zsl^Vdkl%6oh8AvVU7_8?y8T@&_zzLxQ$Jo(0vpoX z+zp_jCs?zLZr|y+MMj;11dJ!R8VNTV6wOFdfX{0~();?Z8)zhOoHF4!81p-pEv`;8 zb+-KiOL*SmxL%{(Cr#6;k!{ zlA1`ek$IuNu$qHXsTC_T`U7~ErB%ux+$#}#JJ|wP7AX})3DHdE7JRvb#1B#`y)P}h38#iNDN?|(Y6TBsF&dU!K&V^W=TtVDiM*&`!`VdK=H>hd(tK`{Vu;&4wKYu^7@PX|@~)Fcd9`oHWru}Ps)6hdSRRt7h&v6U zKuM%kLxUML5|w4`+v&J5tJI`Yw;C$(Jxs{g0YN7M@7VERgET3s@nEA2hQqGk`))4? z?c1x8r~)v6p$BcT_k4^y_Nw&d99&N^Gjt{$3XD?kDr`NJ`^9NZ1@z$eXImUCjI}4& z6fxuc{cLZdFoK4P4GCY?v)~{@mvjHzJW3AV%zM&ob~ikX5E?u6;B80T%U6t$s3H04 zP9QrcX{UNAJ(J%_%Llen-$#kT<$f6@p~ACI7mNDo_dPZLOqgM4*j(CA&Dk*Yk{qmeDtfyj)n8*H-q z2pJ;-J{&vggLfIcat|+Lm+B-d7cn~B`tav3Y;1-yGCFYqMDgwhCiqjytW zU;^mWJr>{ZQ8qp2_n-Qz`ZnFtd-C?DCk+Cw-YV;WYsd5hDL<-RYwJ&JG6VkdFPr`I z6GALjm}TNmDG7?TfvH+T|j(H?>` z258E|?ZO5SZmG36H5Q!fnvIOwxU9Mh=)07t7G_6G(Mhbde3LM9%%6yml@fppyR(ew z>od>>H$3Zufb1m_kgKt}8T_vFZmCJZM`9wP-sR*_IGty)LV_qkZ4>Li2NaE#xF4Z= z(WVlx{LD|JH1xsuLMM);XQZX_O;RpaCp-K*IM{U9rFI-u1bI35x2Btz{V$B83%t9LlazKy!djb&L@Sj)ld1mxUl>%w!};)Y!3VKwe?e6(MZ z-OksjDH5(X_Q5dB0Jj>1vZkBFex*^aSpHrQ^yG~l1m_x*g3O0n?qK-LBq6X+Rfg|H zO{NggFh*D^F$z-}{|k6x8aqDNfKTrav%Ep~0-r#xn06$18FA)aYDX;Z25NqN?qNBw zV^{h|U~06T}UsGM2 zCzbya=cBbMKTM0X^i zHy(GvR5s3gRJ`JRA>=24)fNF8^e_X^X(4ADp1zeT{|yXXv&N z?~pV(`)U$~0WFJu8}Y#lWzy^bL4eVbUWgBz_AU~y zq8R&P5hf2&;NQ90lrwql))m<3YZCw=w<}{9eRadUKhqcWT=!3)?`j9vI|$H|dh0)x zL}Y@eE&)df6l1AedH-gv_e<9&$hxiCn6nmEIuwjJJK%3Re+W9yi+@iA8oAX0azE5sL*g6hYY_pJS-p;NhZ1u=}EV%5wRagmODCb+n zLY-~?NC+!L*H5C}AquJy3`k@bNEvXzEpYAIIq=y=XjMh3aKeUxik-M>oMgiJJ@NmF z+cn`pnLbia;PAny`=OO7*<_?it^YyGc)RU;PYO96e#C0C#wI0Payi;$jKDjG3ac3x zv&VJd)bJ$h;^B?G)s3~7^`(D7TmdWvRUXp&=WE3ci~G`Ve8-x^YqpMQ;={gX=gcCK zz@H=bQ(>iV)Q^O0-Tc0e>}e};T&FS|H=oEYl~$^jP!ak>5R&RBfKd$I*XSuW zG|`rcA0qx1Pqo@}St;_FF0PZt1+^@TR0p=~_RC@RLW+JI9(-G$pUHkkH2SDqx_nw* z%1r_s5F=5H!cwU2@7F{)cffl7dT~HBMo{rzK(K^^WPPa5pp?4j1cQtr2>S@(dNN5M z@0}U~)~RSksNkz_5=jWNdI~w6Puxf|*^n^=`5s%JU2hbR5n^U#@JE5ZS;?ri)4Rad zl}q27njc;SzY2fbOy}hKT=vyuY5V;Lpt2KKSdvZJoa&OIXlC747Q&zUjz3|e=ZDIMhtoj>?TOuR`dLc4QDsCV&TD37cJ_wzNyqn~i z@LuYHF%zsH7g8FcZz5Y3y$lAXgKMK@kda`h<^A#!GswLz=DNOW-l9rneIVoatCxfE zzi9wS;C?S8Ik~Y!+hqwLYQ}>f83|c0PJtVv1LjlyEEVhq>Y9^iYf9&9LE+o`XGE5X zC`~^%JNbXlG%XqI3Uf55Yb3spXn5-FP8RW_d}`@kk3>FmZ7d~xhNu1E(~mhOCW|5z zsBjxHosYh$Q70&1=j;SRK(N=_{=4zp0z31vfq>!bUyQ-{V#0!jKji&YQOZPs(XyFt9)7(T}dyx9zxb!j?XbU*197QDxZYW1q< zs9sfP0!ylxg2sdI+?t9@4F=s3Kgh{Xb(Y_nuq~|r8}u|1ASTX1ls|=UBa{RSPW8EH zZXLIKq3h3FGIi8VPX_M*(0RQ!p(hJh7g#N0!~F zlelw6QzX}G@$naf0qj6{mDvQlb-3R z1VMsDEeSt73Q(1qFy=RPbzDN5C7X9QPpx)|pO?G(&(9oow^@93YNNGN&$&viKK~K1 zdX9F^sO?6ooa7k?moD>pE|(m*ojsC6XWBi}8A z0hl!6`>gm4=o877s(AXUAMzdGU?1WztgeA>f32Or6bWzwOCludQ=H_>g`TeLOuokp z<;W6LO7fuFQtgAd5TQ6=iQ1NZP)2QC;Om7lnuS?Xec_~vcYb;Y=UYFI-{~WaBk=ZG z+*>cM85{W#JN;SNy+vW_p-=C$T?keaC zM%a9hJ$1?k*f_VT4BGW>!ZcehZK53gVOFI>l8-_5VqR{G%9MMLl!rtn+wx1+{%>Gu zuOpD}u+z8kkRrJI1dCNyC191weIoC&dP@AT^1y^@axhDP6~oZicJOjtAaKCnZQ*ki zmOuH9$=PyFv@e7+;6&qWpW#9`x!5;#v_Ey~UhE`U7sJ)p3VmfX8?2EPz+At}#g8me zYEcHMJLXzjQu)QCf6)RZJa2w~R2YJ&J|B%x$seCV$fmf4-Y`K}6Q}Hqa!wR0J z4NxC+DF}l}%n(xZJFtQ~Tm}6ISO-}`KV*Ss8$*RGl}-n%iL4J&^xIAQArc60F-CO3 z2DQG_L@CIU4-!ZkFl>woraPijoENh@K@EI#g66&+Ay3y7#Vx4+n29c)qjmT~(U1B29L}b^ZP3_S3z_ zDhn5qnPl7Nn}?KhyuT#%(-;jG5VFF!F@?>%H~elW`ypf0A(XpART@BKdFyqxrj~J) zcXcNbsn@Os2?OAKaK00NjBD(!MnlGb(Zlc%fk(0dUtP~tx-JKd?F)hxE~hV{L~q|g z2^dBUb+brgFHyo7%)5ejuMD#Qx`L}+1=iP>7&>A6pk8wv?6VXmtzN<0h!9+SMQ+K% z+-PS9X24~HUCksO{wVEI*KEg^TSf$4Pb90(D^z#VN)xlwk2atqVAzSM5e!YoziCk@ zRstyBr3QBdJEEUWs|-W>ps}M3E|j%W)ggFfsH_hA3LPvxv(zY29GjZEtfgR2CD$*n zsbgSlI%Pr!MhSpq0DG(V#z*1V8u#<>9|LDRpu$(N7`Kf6?9Kl_qpv6@ytOu_IHV02 z82+8CJ!>@0A;n**Md&7CX9{;QXFI0t#QB%Y!aL!hW7kh7gXUk4KGl-x1m4J1;`@)v z6xDuo?bKIZZSM-2xw|JE-0G`hoNg(LG&q`jc{t}BFlyF&g4X8-!hSR)?~&>57qw~Y zA9lhBp4%`n85`US$s+r4OZxy$tv(PQ8skKfgp8j{3^FY~!F&C{r!FoDxGMcWY`u3p z)zKe6&UNig7uj1~6v--k%MACrMzXWV6;Who?{1L@E$+25vt?&huDvqLs3^Pmo~u5e z-{<>%{Qk=Qhuiy{_jx^E&+$4??p{Rdeo{y}F7Li=i6eipZ@W+eRW}#v58rjxd~tV@ zo_A)YllrAp?mdX>l&_?{_0NvH~v;si48rI32gjFY%jxe1ZyS?RHLQy9d1<=FA%(&MB-a;314n#F;kD3dZBs5 zjrm;ZjYX}oavF1QwnRRNbru-ozo_AX^)+ug{$d>}FdT=%wlJD}|Jw}aO;K_nH| zwp$pFOP(8({$}0n9nX<`Cf+=|>7qS?&>ICvL6R3EQhzcVD^6ym!!AKqx?6r)@lu75 zTtlv2LR2o)F1fGXC(>);ne-kWP->x=tE3P8yh`Kw>u&x?Ixdz=(xW=n8Z!<7wfYZO zi4yz5L%>fk^eiM@1p~R3Kjr(L?$nQ}d2_u8{ZU`n2iD6-3=TA>EjIyYpayG>Z-w|o zT9fV~9uj^F>AH-VUFJFS4=fCG>T2mYUN5IOK>1*D;>CHzi6iYU!V}A)z-N_G%bb4A zI5bS$4XDJ8X4Pc+J&2WGfO!#S{S?&a_VO7+;KIgE_Z9rj^A`S8t~1)F9itz#vht2W zS-X0N=W&8jdNhtbN`U>IyNw(qYwsuM(_f#To z&woCI;(#h3{Cz6d76Wedi+kNHWBxsIYg4=H()F^#1wB0;e1rFPf4-BF!y)3=1Zk__DOKy82LoKQX1`t_dGvxQ8t)9U})j`|!LKPvaNkYQY$;!9G) z%=QauLqf$>xx>H)MZ?l?2BUbm`Bq?obu=x|iK|J{aj9+z)9`vKCW`ANXbj1Hvl8*V z#dfN4H|jbvG=>_9a4Jg0DHNKgSz~~mIrg*#;uKq4SBPpQm$6|6C515wfU3PajVL7bS_>nr%szpDgv8fM}y519wJbR2_w_~KyT zK*b9mN#+ec#v0lP-J`VZF5jY&&X?2`LA2QA*b*q$8#V~BTGa`$twQ-E#*g`b7Pc=O0<`WDR9CoQ&L*_ymf|1f+(fpp+35u+}#sBF=Bv`o83{ z-2dSGfw=noU)Xa)JIpa2h$A7M)H--L#3IFd`6EmDArCV;HOJnB?opWBb-mIEd)VZg zYd{ryVUKsVQemwLs|F6tGbmA&bR3$pzK^)R4=l=q3WH@JGR%_JyB!+yO~zYm?qFX_ zCR@6TXN4w5U}QudU}sY+5|X?g96pALu#@Ub%k z`REGEkA~8PIEFHQ^>&z%g$b;t$eEORbzHrirNK$BDjw1iU|d#t4Jjm&94uB&ZEf8V zEfJN)y}K}_1&8h}$xzn+`6awOywo`QUjP=y(8C@cNjrk8d>g6=sGM(K*>0VZIO}zW zpVkl_+ECnGm$j!1<4zQErS7}o*8Y1#Cz_}I=PI=)U+T*nwkYdp#d%TZep(UT`wFDws^f~NsZi9J(GRduj{vWao%D;!Gm+iosJsM z1lUhr##Zl|2{AI^gjO-*fBem{Y8VzVBWNm-R%Wk1h z^BXo`3B_DTdl5VzS$;S#gE35pt8)gjcMl1LL!~BreWa=3;9kC|!$`;}I{R~4>kU7r zb$B)MI8E=J3%X3BA>iW{xtzA&v+$H zlFDAse$v_vYy3RbauB?>=c!Tt_IJ#j?w{Q2oK53}*Y)3XxtqPl|5RP7(I{VQHbUYA z%n0cHl4h7jPKdGQC0?tuj$BH2e-=~5yNl0#f?NC<^q%l;yYpw`nUuqwS$AvYG4$2! zi*?6?*(<<3uK9B1C@SiK{EuNCS@p#u8QHN9pj4Khqv0f}{C!sZIk)y&I&6b2O%N8_FG@S2=dT(KOC zUhq*^Pk9~kYhEQ?R_2=}_&Ezc2lH15UR|V}W<5}!8*h-qa1zsf1QaI-2qgrp5tuDR zm$03!3EP8cptZStNI-*z##@{Aw2t zmr(Co?p3swq{DNO=RZ%sRhx>fN5?&!sX$TEoXngQA0q9B<|UTn9~Kb`vt>z&OoeP? z;et0SXgieR%){e;q8_WfVpnTQ_g+wRAJ<+4rQboGqVi}mlP4P!8rG=CBxB#8+JI^o44?#TOVj>={%7sMTLU*0>;0<*iqdlr|sWzAjCDS#q( zy?b<1@fNVcFxrS>_jc;yw6cZy)qZ7=aiW6@E7lkBVfGGXqED_v@{WX7~R zVI#AeE zmE+rmehi%#A_9xs>ac~#eBJK22+kfe7#fDa?R%1Uh^FE=J+$}jD3Pkhd={z|*hhqY zRNyCC4=)rlSCzD*3z;^or+VE=tv+NH)yhHICgK>8HK zysi7RhtFxkJbshljrCsfr}gD7`d&z!)_K*=ehta^>N;Z=pwu-)w@d8DA?P>-X8I#T zg@uMKwjRW1<7n%h`>@T&>v_p+`c7dCf!~%ovcw)pDSm2F&|q1-{!d{5OG*9nRyHmZ zUDD|{!BD*B@3(9s11i$qQaz?R|4-7|?S)1PWR8l(9#Ea*iFA>RcX(u_jE+4S?tSat z8{KWz(lYjbyWbRg>;C@9@9)mb3k^EtRgdZG4kI7;C%ykJl|OtNxI!kZ zR|U2RA@l1c@w=hOYN{o;M0qh9BXs?GQM`sI1C9xeS+|=yCJ(^)sskzF1N$;IUVQ$V zoJlpQ{zCxc?=DvISP?)5kWs^=c|D2a-hGwANY`5Y7^`o(Xo`8if&=|m*Cy;%?>ie0|>%U$Z>RNauL0{QWdqUptKi=AF)sqIza zDAlGuxCQ2EVDx6|1z;iMFi$!~6)t#%D^=XL(l!RshgJ4vR7{8eKor;@w*jo656%Bn zTeX+n_+6v>dxCsjee&EH*$%yj1Gu+lSIiuxt zd#n{z+!Yi0qaf(#L8voQsd4xcgi&m#w0ZYvrK|nrs}_9v!-rE+ZeAx}pSd60cfC=* z{rM=M|M9UcU0I7rJHDjau)jNnQ<{GMY~tLXv%f1@>#@>MgcSvi3}l~6GP{$)i7}HD z`&WnNaDCIWoDB1dN`79<44YrJGI9ZXRo4b+a8b9P*G|< zZaKNSvQu`#{N}As#Y-8Mp`mQ2@eNJYqOa?X`F4KuZO<;%r&YWv(|xnBhDlhVP2Abu zYnhU%k4L3&H@{Q(RhujMympQjCaEu7c>20s(o$K%CK4n67IAJ>UOsdGtBkd#XxL}{ zSj4oAQBSyjrziU5298e99Kq~$p@gXIX~7R-xCI*ag3gvU*7Gu$UP1M4oKn0Vt5y>a zVIv{y3?sLrr%=Nr@Ni>rZgcRx(cSO0&7RV)!hE7uox`+f$Qg^IZ4@4 z8|Q!*MeR_H`$Mtye?C0(#@?UA@F#ATcprN^6FYdC*Y~~S-Q|4yPT@EHq;o^wr_M8> ztE&4I!J+m=94e8r?=q>#)CTv|`0H57+hgEmESP!}R{vCf>_UA!6}fe$iyo7&ex1+~ zyfmnhUxRc95^EuOKqiEZhw;`_nDd#1qckz_bnNh}YO>qYg#~7$7`Q}*N>TfLsFg|~ zYPqRgUs`m8>|GzNS7v23^o6u7XqFKV&wZuh zRGZ0W{}z5|{j!=tVkPd)7|*-aTcsH&V_G9YoFSM`drG0pF!ZeLc{jkHb11rpipKq9 z>#l$`&@Y%RFozv{inoLfSqb4<;pW$ysAh*pStL^Yb6LV?Ejd|>Y?r$TwuGT~HVjMH zy@?+|$CiWAJF*B3H#dkGC`FunP_`+MUP~acF`0*ECu+isYy%_^HOU$L} zzjstHPt`P_x7uq~1JLb&`uQJZ!EyeK9ovj$+jKrb9VHkZTv$=#cZSo8wd^d+rB>&x zN${3-t^?Mgs|J3geTD-M@hKOwQ-=PuK73RrYdSOP>lY+N#*b=Jh8W0>YH4DTH$qow zQ+tq8b#K#+^4-SY-8_0Wko#4=oOGmYy(Vl#Q_Yr@H|%WJ#0?33+UtM8go#?18{*cS z*b^@tGL0a^4nGp-lsT*4*;Hp7e&*NYd7hF`Weqs1!v zqMSA&!(r_-3|Iu?S2Mm5x()UQV_bnIF9zGr3x9V7JicKlFEZSrlLvuD(A9(qnlE$l zyuYRcjKwU3Pfdx(!b|W-LGGA#jag8zJinQ@_)Yjtn=?X6_!-z=%P3E;Eqmfwd$Ho< zx6}S|iymsN>8lxmbgR}3h3T0?rQ$_k(sgW(&Z~k;MM4(g*56ZIqFqv`kj7;TXcAAF zcicXd>yp5v%EzYpe41>^xl}iJ_0Z~f{-t3Gg;P0#$j=BC?kcP>iWC1W&dL~`T87LhUjWWbeP?w*g!+YdfnFY!mw;zc|PJ4mwE(c~#YDjEWeXSK!8QYKY=u&FB(z~Xi(!n$u9t|BIl8%JL%8+;u$z_^-LCH z$&Ow%D4=XAP|j}BtyR;NwwB$htj4{lt>4z~pTAI?pR6X$-YUG~d*)$3jGl(7y5AsFu7cEe3ohW{)g6}bev9M zyLl$|E+QIWZg`Jnw4f1Ulx)hlbU`wvTfO)l(KSj{rGR`3O<5Hb>7)<^M51`QlG+}_ zx{vEj)es$tEwrxYs$WZbe_tP%d&Tn};w=9N7T&)K^h;93%jYesNpbSh|eoM{q&0k66c1vC^#LUR?3g1tcah|$6lWZqpnS}BO!Kw$HY)j&Wn&w zL*Uwj+$q0=>qq121mh9bS9wkAFFeE*)AAM*;eZRCD+=@+|Hy7$yH?7`luG(EyN8^XuSd5q8N{See2O%&?8kK8g z+izc=p7T-^j|2ci|;1N9@b+@ zcc!R(o+g+`H4jn&nxj|?(hGR34i-+DPmYJPiiuvxHK#%YZ%KqT0_b_cLNt9|X7Hfv zVU9P{PKgOFqWV5w*V@~q>2(=s#GT_C8@mkw8IE}B-4_ji4%~l(tpxgm_p(o(UG*f5 zBQc$=N3OlLc)ay2X-nskcEuKNtm!V>u3|c_+$Sl{R*d)z&WC0V@x!YKw70Ygtq}z8 z0vuJpaJI|+yWe7?WH*S=$#&;l3%WAV`gUi`#r?wQN~x;t-Ib-%vp6z1VVxKt z9RIQ;qo)WW6ISmA6r-dz&&j@c>t*&kUY|<5Cjb1`h{~o_Yok%_#swl^;P9b^R4sC} z9M#G1-a1Bq5){3Eo(n?h=C<-vTHLd#{#VsjuX zPsdpr45pzvk358tX~vjBdqLOQBEQjEidNe0d!qp6n1r#zjkU5 za(Rih)b+HDA|2?3F>FqXR^ z^2`yVW@P}awr zR@L?4`r`wVWh)HkKAi+)DCenMM(2-Lx48QYEdHYsC?=~Q*APi!5Z0llCzO|V#2{bg zFq-mPBV#LT)eV(0RW1LlCPEg_8tU9qVE(tgMUl#I+4no8-%#k*d*_p_m_+5vRh!w0 zf}G3UT?4;JlwF1M$43gIUP?O}yM2OB-L#9dVm5U?iPKJ0R;Uc=qL8Ov&x$y zBY7#mC8=5ym2X}(rL}6dKsm+bgz+$&wqND99Lhd^85Wlu|N4o;m)&g+&E5UqOc?lM zsg(aBDZK_CTolm=smF+^oypGk7|%4^_^X>`b~&fvjmrsuVanLcxqRoO9jHzD zwxz9;7p)t@y~{bo=c&tkH*+q>gl;QK65RS39A~HG>IabbJ=*2&4`!lWv)QF%(hw6; zA>;Z$T6_{YGtr;4QSFd8{9Ui*5YufkGGFEa_V6@_+DF4;aU7EUxl9^m7zOiH>-tXpmP32Uia!8yd zkZ;YzZMO%6CscaWF45Ah%o|7&KM-Ueyh7K5*fkMpcc*cIoR#yzSqGljL0)4WVGBS* zT9BWFu>Zi>9%=VthaalX!IAZai)t0vST|mzJonnY>up>4&edmvj((qOS?a(Um%OzR zGRG)!#+RUBV#KSq#p(n_W&bwfE-7((ao&8`w~p6WXI|5nA)E1BlVz-lSCX2(fh=|F zJpZtsuPyGJ6+dE!Uk_WBrM_&Zhi&H(n<6Z0( z`CEF0fSH;I*UV#P&9{ZwsjA$1obWi&BI0v@c*zxOrxtJjYu$$&#KDQ&Grt(8w!#2v zL3eu=5_T4$Fd9Ki8y_`m+6wHz`7zPp-w6D6=blNZ&R<<+F6k9K{O0o~1JAlS9`U23 z=^wiRaGMH)nBAP*%Zf>qz3kETP0qPA=P2CbMxBe!rEMw!=cO8fQ7xa7xM!CEb1`Ek zE1az8qI7rhNZ9&D(GixRs#H}^ue1Hhv_@xlGFP?LiDcq z{Zx0}Z?%&Ox@cCKw;O1YM9Cozn(1Ef>arKx>N5j7wgZ)rJHhR&o7MS9z|k@~(TpKz1k z&bLHx|15~BWDO}_=`=MxI9IpPOHAjnG9V{ZcfMU1hO>LeGqd-P(^$wo{yK|h2;#YwiOq}?&2>K#t z=62r1NS%c0qYGFRdfIE|hV-l?TCJdPown*c>JH#dbT&^nn8Dz!^ zJly-*AZ~D12eW4UBIr63Mb zeBgBfBr#0GgTF_*M6*gckR-Hz{%vUuzYB5@i|Z?I>+KYzXwdj0_i{=F@BTH$bIF6@ z{!$_9I@qnr5|P^)6eGL@!-uocKdGSHiUE~rbh9aboC}z4-ZD6AmOhaSu%9{tKeX#5 z`%TTB5doRp@ey|WnyU9Y`4sQI?Dhr7f7!MCggD(KaELrZOrfw7&EwtoJ#AA1o8`bZFLCyS4*({a0M;%lZi_cNS-f7-)8-AppzL57(XEfl14LzIM#3vDrH zjzc_i-##WVRDYc)ghS_$I^0GO$*7u+b53T-b!8aZ>_9lU~Gdc^TY7a^^=gayWm34iEY-O7de zp_}Kj)sI5NntoQAjq$`^IXxK1J1J5SG8#f^)9Hq7QWu|{-*WzO!I77eiK<{Eunoz3>=yW`) zQzldA`G{GiCxR-ygFQv+!vmogCa}>sH_T$zxmbQV=wx3x>E%&&hV^pJkHQR(!jymw z*F0PcgMvidi1q}XBJ1F#)Hj!$7+z18(_r9Oq^6-oxFy6&Ds$J1F6gelIR@v49^M0E1+jKOL}d zt#s~%ApRojad`-j4XuHi(s+>j*BYO|D|`A(Knl%a{nzR)FLVLRw6KWZhla7~eTP&WU2&pD zcS?Wv#plnnJ&B)nRTxlS`O)EF(Opo918Fr`hA*V)u+TBx0j#VN{aQ45>p@%(*g_4H zn`LCCgKz2;Yb&q`&@JgbTCzSECnMoL8s&eh)kL*p%lO4~;PPrKMH?~TV7z&Ce&v~I zPPMw%-ADJtnO{8n^V>s7a;&U=agOXigMm!SE3R0e+$`CKzxZ&9lcQ8V0Vl6`NJ6XD z8e+qm1*Cr|kndUN+}R&Hi)s;4zi*p44EX5utDyuI1{;f&1h3LvmhcK7NY@o^I71;c zxnFDgCbc8PRUIfY;Ii0I$8O#|gOB*O2o5l2G(^O|8tA?ANGP~s1IEmC$}ipK2)7YN zR)aS0%RrY$b?=*j4mcc!5@m}RjIz+Iu+_uE!7vmjb(@j`WOP@F8*GG@e=Mc7`;Jd~ zK$xr*M9cU^fg8r0>;qLz7_Xsf36=&AFIMzw_-u*l$eEEq`9A-h$-|Pf>;hdwEK3$5 z#re*X>$R@)nj7%IHD1!k*iy>zWWOsXeM)i^#QrB&a_g}~)awcl)l$op8@`sB6u$%s zm5T9UlrdZym-{=m&=j2Tzt|uk*4d@iSo>DmtSj;9pEr_ro+2)^29*d*p3x;3+zSOY zdD(}}i3mh=`2xTEpr!Q>No5M^wzz4Nr*|6bpv&;Vv&ku{(YKvupI(Uxc$IX(v>?av z4nJ!;MG{REu{9CSo=OI0lMq_^8=AbLU{NCbB5d(raz$A$UdE>P9>~S3#(gibUmTfT zOD9Y*Ib}A%vu#{kii?S3Zh1Rk=_D4UpE-RuI+-F|Rf|_CXZMRn`8AH~f1{u-JjB9% zgE!BWWzbA|pU)O`Ju5}XPdy*aB3f)0oNz|NU^w9I$Q7vXLob&U>2*RF;!PPbSX;_S zX;zVlOhEYYUeDKMDy(~@O!E*6>5hYVv{4LqKG8FqNVcVm6{3jbrRO49yyryUz);@4 zFd-!IGWEU99QC$wMc+G?a!xUAAr#~h0WjPHeggmjUG4@&T6acG6M|J>42~pYef+t? zUsAxy&xadGL61@CA_cRs|MqGK^pUrtl+b9Rn(LojX{GmazJB8AKNt`B8gWnk^6lLy z5h$w&r4)xExMe4ItZ6ed3S)}1i4OdbUMgX(laA0hYAs++0oOuu$v(GM1#$1A#;qU+ zqzL1C-+KuQ9=`f9QLWB|^r)B8ix9BS322J+FSnvOPDfB@{&6dw&n%Hk@Ugo8pn|`X zI=0-Jj%2QgfMuij`Y%{=o0Rx!c!Vp9Vea&$sR(XBc_`Z}J;bHcu>?mx{xluPNWPtV zWS9(a?%%Z}pw0U#VI6QLs!9|7@$+P6B~NmDnugDPekAx_##F$=geGTPzgR8g)tJY} z#DK8I3sHvv#4FGhkZ-#QQ~j7^U$f*;DHQ-A2{fXbszbC8`I+sbM;*KS1S-% zk+E2Lw+kg(E7NeK2_k5mI?_kH?_iwnhV^Qs%d*zXt!0-pFW@LLtxjLrS#mT&V!6A` z=^R%g@*(KTDXJp_iT}J@AdacXvPKy6pT8g^2M>u>DLZG;d#SukNoK7W>|J2)cW5@G z6^dg$D}r?B=qnL*dzFV5VA7bsH6%a-ZGda3+~OSBQ+XSaSX51ssQO}ih6pyz_V8$Y zAkBKYxfFk!*~9f>QDE_llHi`7n-;i-m>G5yg!$k@gdGyLkYtnoT&}xlF1bm zf6gaQ8Iuq&7$BNLDICGQZ9V)1cW71^Vu;@d7)15I1uYbn4dC4aojhGy?tBu>-f@!+ z{g!Z1yqCW`sLt%)6of06-J8G3{F_J|oYP{mmTt#^x z{zs*Xp+4SKb33&p4_|>a85S?Z7FzlfnEM^AnCZttOhS@YCU>MNsE!IieYaGD&>nS;7So^5xK| z0*sJlxST&uXx>e-514JN!%Sd3X7{+5iXRoctmIhLv$-svl`tDEG~X?7zmW=!&w0KI zyajrIbp{klhHqR>|5?9`ju$-*gN8x`CDNR}k_yr1x{-4zHvVIW= zCRwkpSYN@xiT3f-okC=^E!}7S6^e=^Hipqq21(kP@t+JAC$AV6+s9H~Ld-r~Fque* z0d4`cQHf;C?5LRsQ|6 zfmCzOii4eUy0o(u@6nCtC{0S7+ku_`0F+JE=viqoE~c6Z5)wlcQ9IGRUUfz6?~7%{ z-x|eMLR9aV#yPe9N}T&eMBC@D(^6oD$o`aL#!Zj6?Clc`spmSQ-Jl3f2+mNm`C-yt zO$CXDaAJ{^J9%s(`zT7x0$X4V3pZY_W2lPIfeC9t(u-R2sS3?Vog(@cYC&&VhRKFY(Di0K*OVc}-(8R7jN)lfoFeKvZz;tnP{KNJa z-`TD-;~aakzr-ONMdpl8{yPig@u6v3|9nCc`_2r- z!~NTO4E0xqQN8>UGq(yx|DIoY22ihbVr>rnx7Rax&kG{6S_fOk)t_oqa9Fd7Ruimu zpD#`cI<}1YdHl))nlJiMy)-Z0fL?zQEzdNtJsmZ@|X!dfax zym8{fsKcRBUR?g&VXFGb7}w{Gf~GmM9Z1;Ll)bW^@oO>QKN@-YyclH6zV=HeL)oSKz)WzbF|gl1KlXdq;^=PB ztVFZaCs@NRjxi@`pXw=p=#PV}&ATEa#D9ww8{4U312qQl@&6eL0#8bgx5bdf|9u(D z6Bv56 zZKsp=RO{iH+YWlA{1&0;SxX520n0_7JYI9(YT9}?E{@UbjF89(z`?R5fFX!mCt>sl z99EIyRyObEUz|b47Vg!NGSwM&HC#T9r#69$QJdXob8PhwDy2dQhAmd)t+m_J zzSK-D`=v+KtoQK+)k-Rr;t27oYMR@H=+1i8#zM*m_Z1>`;1mdlAdw3Z*8cS#{Zx}= zultO%Zqv2Rh}Fol8t`~&X&hPc0S*F7|QN5mQybL)N1dw)+d=DO_5Ne17sto2v*(Z&-yjuf}Uq}~pA5rp%Kx9%h_BN#OZgWL!L zOk`;L>jp*R^y2*^jG$-9zsS#dmS`{ehZP6?g3f+t94FG8rt~Z6p1Vpa$tT8AzY|yY5XX zt?P5Y?+WL5I5O@JQV-oi&w(Dx+~(Lzcu(O}6O#VDL2~rH!GE`67B-M`1TsfgYbZ2= zaX^DrQRXd|c-H*bT{+PzWcmihxGWeWAllm_-@CB2DOK#rUv`C9UEYU>`9g(Nd|P8Y zb)aaHcV63+5O=qgYwu0~82X$$Q`8$XbJs*0m=XAmvlyg zSZ(4$j$v~3PpAuJ9gth&Dg@=b0)ejy&mnCEb3h{YcfkyA!?VOpcctB zS;y*>f}W6AH&m*A%{BRmMhn5LkCgRJR>tkS2tmmh{|zsD{%3V~b4#e@mTSXb{rbxt zX!@LMA`df@>+5c_p86s%L!mPzkt<_At>U?OA_t?g+TcjnW192wJTB^59f=CJdXa*N zphN*z`dE3Db5lQ`$f?{oH{+Kpaly2W>OK1-gvcy)Izn!#gO$*N%j9`gh9Es^X$}7a zMNY}WnM9)fBZ9TxF|0Mg<>!aXQ#mJHV*?w?`>$PPifW=%tSElG$<(yFbRnyLenbw2 zD^HO>WB9^M2Wh9jVCGbjajukgbxVdFt1dYm7ej4a^|q7SXSP2dvT5BMrTI?;J3Ue? z_**;JVC0!+)*n~qxFOWA3vx21tZWTif$41Tl-4wS+=>`Vu|bzmb!Ix-$Y`RmR2L@% zxzAI|OI59z-ZOLzk3t!j3i-##KU1w1A)`;!=%Khw579!C>z!1GuTZUTD%TKZMfN$o zG!Fu2qXiN<4$Z)=tuXgx#qx5_!n1PkKTssMF){cq?LG>7oo6W>^J@V@XeD~&(mm1* z;%cLAIaTm-6DT%b;E(J%Ve{I<6vxVkuA5{Mpasd5m0P?!3pbUO^29vqqX9fN@)`Njm=9TCVr1pyP&oC$e_m z>vD95H&*Df4g4ZgtSx5E;kttw(M{;qz(s0KM`nU^_zePDncl=Xv)EKU%PKtkWAtnZ zQIU1$5|<-wY9Mu5syfvGbU%Zt&71q(3(CsC%rM^t>Hf*Y`I0Jz4B^kl7cLyxaMB!X zxpA-dmV)6=Bp@TPTVzDhYv@Fz#Ks;0UR%X*q6~X83ix+j0kth zh;+)8mc$|C2A9~5jq%`8`6_Q!+O8mmvtd7ZMyFd9+diqe(bQa|(=4t_+Bo~kU4D*I zPjXTD#YY9s=wJjJsi0$<00YQLMe`%%tu@_ff0Yi?tJFa9`=mO{6`=Nr!FSR&W)b4^ z+;`+H`6{Teh4nl!2=>CSQiPfp$#!x#Lxug44L z7v!K4n2IXkfxKYjVkVUF)Ax*#(?=hwBml1eDGpt6lMDDO#O-s(bSzTv$)7!#^!phB5x*aImx}&KA9BLHaE&K1N6rw zIJ-Kj`sY{y6!v96B{T-54u~ljWPf<@)r=iu@kcE)Le`5gp%cG^dcLZ1we|U(rJFiy zc;Rlg$DS9gn1S~SQFGy=ElTRXx)O~byu-VZ%!K3-k`&S;WX5+<3lrhr2^W6v!x&x8 z&+7W7$AKluRR#qufOUl1(RG~fxXbs8%7^Qt>-g&kyzxc^3ZZpFR_u&1c?UMz3>R-) z!afURTOhHpo%?;t6#xX*mlFBjSVBTwQk>YRjj8;JOF~O@TQJb~IFxhA?m-u^;+)Q# z%{9@jUYe&CK#AS@c#OYyp~aSkAUvY7U6`G4CjO#={9%wCW%|DUdGy*wtnJX?0iR42 zR7|h`)*^mi^14Eakhx@T6(9eC8ZdIHej&JbqCz^7ZCYKV`W&vdD&&*fW3ojT#camJ z1adnR79=Q;$Cp-UF)i-*#7~u==t=B9JB?bN5`^3+ zB>#SN;-Det)wJvHr)1ZGjppdhM9$gs;~QJa-eKcrgefi~jImsU?_S&Jcv%4LC_rK- z8QT`FVf*d-$v}Ts=MsmAdFHh@-na?fGpU6VcOo40M>-`h)3g>;>RuW0M zTBEG}$pnh|%zc>?7Ou_b@k8Ah^Pa$t{T(l{n%sMatkJE!a^(C0OI?G`Sv&QB0;7qq zJ(CoA=Dn`~-e5gRF^&U(@E}hnWy@U!*B^M)r8u6=-#p0^C*>;&yf=$MJkx%I!c-ZW zD)`m(@XQF@%e0!x5k9GH%7xJoy&A3~Z7PN!E)HB+ukG2+b&m~S9E904I`%hg zEU_&A`kK9Pwezl(?MPV+T=2)Qd&-JGZ4Z;Mntu`Smmp|m-zT2HIwdfscL&>hR3n0N zdTHZ-N-5nNp1bcT#=L_CTGOxVfd9K^NQE5We|c9ePn#k0=;)*=>koO0+nLtA9+C;6 znY_D}$n?h!0^i=EAioHY%B94a5qiGiI}~OJVw3N+vuj3NCIn2` zN+d8$k0qIEYnh;v&`qqk54_Gt`cb}~x*+_)sosdkC0y*K(3x#Ha{Q{!`Tk*P=o|cUhFR&H7K+A*5g&9QqFM(TKo&+)l>IB0kZ7__P59M&rEe|4`!&j_W3Od1@jePW zARxLlEl!EtSquf7u zHz3_bHdIzF@W=Me5;DEiS@`41?9&M5zZQF(F1#51a%$$Hd~@J8yyF{(b&MYCm4LEI zjuPPR-t)>TDK7aiE75B&;+UEGuHBK7(tKC^^2K=T#(Xkgh*;+Jwztrr@0`LWbxUav7dc$5AtpgV2Z)J0hS}b% zM2(InIXoPJMHd<_L*)xiRRiHYs!des2-R59B`liz$SyWS2U%ZVo){=GpTfk_@W2gG z1SQ3BJ_7UCfX$V)Pw&GICFPEKe>}FRDA-Y)ShW<)$GgmKpN=aGuym(%F^uFwlL?fZ zPqe7G_o?E^Cqfu5j}XVZsdSI)#PP1)K>SQb0Y>Q+0Wrt5A|)M^yOKK2#j$4iGf9o6 zZ)en{t#6KC*_6>kH%sH_!!EPw;IFb10*5j9EIjiqz0!UcuLGseeq=J*oB25vk_b%{dp8# z-EuVw+*XulkjH#-lymv|_esn4_gl)iux4)Ll`(FJ;1>S=;H3M@ErfQ;QYf>Z(9^co zr#hNVRLsJe8sQ7F!kWST1uDrz%$G7H+C)7HDdK#JC4-pSgj3&=iSBnh=h;->ag^Uy z+Fe{`%MgCHySR98u-_LUDPuQUCyM2R-eo5?qD{+usnpV5J0J)RX8) zT2ku=5Pq0&OcL(Q4hmf_Q0U~|3b)Gs^s=6;%2gEksm55tXNlCE8U6w;ma3!{6=JB^FjA>D46 zF$`Z&bhUS*#9&#z!8w$?%|q?&;zWONe}z__Jxry^Hjo;R<={nf&8bz#_mSRgpU<>@ z(}9&5FMQ;d?#cChsg5vuU_?959A7-{e%=xv8gNAv^uRwU1byvWx)Xb2yS3ns!QZ@- z&t_cF-RP-Hdnps3VO+;3m4-x|U*B4MyKr82UFJb?KZ$*r+Lv1-nI{gvfoo|@aNVi^ z=J1ZwL0gP>BZk{vL+6QQ0fGU;=%9ulc~?<7N6@r3>&5UPx|N>G@PCDa3}Uflz%fCBe(HU)^0SD(ma%PZo9DjB&OQF0 zNA}ObFna%D!f2#LC&PcYAE=Ns&Ofe7Zrn)!&9r(Y$ld<#iX(Xp){2%L^020Hdr~xO zg4hdKgJbp3n2?O5-nZj!{NpdUR!s9yoyFWeX2GK38kdrm;7lcT$qt6OfsX>Ep`@a$ zNQnrDmJ&n+B_#wlT0lxV1SJFs2^l2Vtr zKhOQVc;4^Dw%_;nz0P&cxz06g*+{KLM-$yDv>5{#V5C__OL7gGF>eGSW0d299Amer zsP$z+Ndz85$veh_Y~KI^8GHb#7O@*#e>_d!!k88-06cKgQp}y*z($}GJmPb5FTY?x z-e+xmrYsfSkAgu|Tf(spB=2P3)9^-AdXYOj56JwyXH*$c*}i2x>gkKe4if;pu544k z!(%HK8m07a;G!8!arbk}qbR-&f971QJ5+q~5{2Trfwd(evVDBAk2brPqYjlEp78A`R?s8dEB_FrVQ69qaEd;)Q;T zR*&puwV%ab2Dn4PA$u(n&fi*sv<+EO_Iv%mgQ>aGIe`q*v7_7Y(7lvQVW`DG=q<`^ zqMB!0IS~C@bPfj&x7Kh(vW7*tQQ2CY?f~2x)XSNkDS+N|UUd;ti37cEb)%qjHz}WW zZsG$G6B8Jd6+U_Aq4T%p!7Y%x_rKtv*ls^92h-;+xliGSp}yk!(Z6Qatz*}j>Fk7V zb0fYmL1baJU6LZiWF1Y9-G1pD3qP-9p5+d58@A-Gu(X6iGm^Y#-VMb=4p-*6$KPp% z;UoSn`6M+>vA{1F3iIiG=@+E&A{cnKdJXFWejb5c!w@_TlTq^bAkw_8)*pY+D?8Yt z-?V)sP?YX&5$H=NFwX$|{f_~txqY}eOj@-zdh>POQ~KCjtMTrCFby#+TKy^0`!L0` zyjFY^TYn1vQ7Wvq@Wa0`2?UJIXiiv(y{~|)M~_r>S=#XbuxtB3$uQi5fnb4bVcZBU z5V+@QiP3O^edUhVnN*1cy)$Z z3q&lYPwYCuw0rA$B)SwcijPZEcVjUz4CFqUm%pVG$gpkd80CL$QAXCTz(2@7fAMW@ z6n0OWfY!>xKi^uL-+cYEXnhi_RQxMOuO_Jj`lra$^;w3VYt~)DU%F;0d*9Yso+26_ z3TMA!?_(u>2Y3*LwKh4u|C4d)XS@lynu2X|c8M8&EkV8geleVHyu!E}WQ(_j=>JV2 zr*pNaL{P4Fd8Ag(()*#N z5ELga4SHDtyT$E3c#nDvve1osJid1Yda4PZ{k)s`{j9-FB=ZX(1@!dA$47Gs9F;AT zfo?)*Ac5@es;nMr|`t8{;lg^G3w^^k&V<{c*FgcB=5E#^S+}KD-L`AS-Sn77Z4(5*L5dm7+gi1pZUb@woDy?P^}Nu?QMq(BZrW>(1mZt%s~6dvoTg=6#}(j zWinU2@S7E=N#pQXP1Moh4k#H0u5m2mcdny~hg7cNO`sxyxm;V!LuQ+Hh&s z3!m=s#-9Hy7gEjG6f(<{GR`r%PkI1<*>fhlI$4-_&_lQZZZK%EK$}IA-+irCL#|^@ zoq|JZ2QV;#r;13KEKkd^E2R6`ujzou3X{v7 zMAz$jlNnGj3qq`EP5qDtwSm*$wR_PB~9!t;W-htKycEEl8(vPmr_gjmGPFVb671SS-<7d z6U!hH-z}xEt>^>)}%I70iIXHZ(daN0lhV|)HYgNx2AI6TviS7%cJT|<_Hm5jEUV@l42<2WJ}SOD#h_Jaj#HSuV;gG9Nh4Xga-Zr zf-v*oSC~~fNlPbTyMep-c!Ld@fFlf#tYCGoYi?I{vN0g; zS*P7=qu;-?kpdgc=8dC!@Z;zHV1#iwtg)8{_j;_F3`x#T*OB2%6d4=KsVot~BSQ9T zfIF)LviQY7(8&Gb#TLO<1Z<071<4>tIso^pmK=a8BKtjiDHH#4;_r^`73lsK?==&e&;Ss6n$H|WMQ82% zJ>35n)cw&(pJV1s@h^5J)*+kcqSEMTMS`YQ3ba0VltkJkBTM*I+3g)v{i-Unm0*Z? z?kq&n51e;D@j71pc7SBm6CfRE^>17xKGz?jh9ojUC04i`@Fgrf)OZ62QlmESPh-|R z8>>jIViiBF+;2Lw$x)aPFe&4g3lOy)qVtrvUw0HW^xM6!<~=LAqygGL;C2t?);$ce znl8+4`nR!ug|`STeWkFoU+WQWg(JrXBa$i_??2$IAWI((g@^}WHRc&B=Cu>GvkBgU zV%I;mT5(ShPim(L^okG<;sW#%L|i=7O)0LVdL1YdxmSsF%@eVvOmCD2*C}42r(C79 z5&CyM;S^>Jgsw7k=GF`|X3w1U_h_MF4(zg10O2A+FlJkiggJUgYV?!wdLG#Pl(o^; zv9kj?KDhwS@l;Z4K`h0I5Cr_(v27w-V2?06yj#D}GrrS+!TP3bVdl-` z(sV3>?y{$@k`hg*rgpZXA6RK_{izsMsCk}eHs6k}uK83s&enQFxvN)hVPb zNV5ql4EBQ~30+v#P$o)qW-%r>0ketl!dFwMT>09%(?T89NE=Uvaof)n<;r znzZ@4AW2>O##j)f$W)r@B7+s{|8}RvD&Nk-5mX2rToFXAu5DkmAhR)6X(L+9hv#|mMMY}aYj%NGn45kOr^k;>%|f_M^*TdV|c0l2G*>D1YJ z8qN^B2I777+u1UYP}`5|KCrJ_LF?>?p51xmQ06_|w7VC3QT#ybru%`!lvjMWB`+#( zvFa!Wx(R>7uwdv@_Hk|oqBsv0GFgXXy0<;-4?=5=L)VK92!96Y*<13v|7wFDCt}nl zi^^c3ydnkA!sxJ^FF{{4?vHS^nmE1ypHEklT}UbNvKt=eh`+_<2UjJv@+GzfsSLD1 z&-jog!WFbsPuZinW6#!~Bk{!s(G>ODZT|E2oP3>14m9iNa@B1n%ApH=G`ZKeakO}A zU%SOBL*zcXU6^x_g1E)%;Jn(91{i1eV{Y$fFP%N-MV|bd|K2QLba6As2klk|H4^t& z@_10@tC+H(^Kx+Bixx6vzb6xdS4+#vPyF27ClarY#XifceqYX*)_v8DVP*1`0}Up1 z&)$%VBX06rty38+bAQe80MOHHt^o{Hv~>kQdzAQ#QdzZbC$a(^a_3)~mT~+qLE^;v zuba%C9?06s+~Js+6yTt2<^?U2{l(xH@=Ujk5oO-ng%2@Gw(rwD1I&n9rICXLT9m{K zU)(F_mgE3?pnP6y!!Jw29kaL~BFPvyw19hI#9!vY;Q_f8 zqP4v+P;azNDL6KWp@ww}pb!WeYoJ~*i)ed6RYy9D(SA!9TUX!*qLs88!mpnTA_xw|naIl{qa zDl*X-T4m-x$9I3x!5vlptd+=5wRexA{v=~f zL26E8HB9FLv2-*7C5!6Q$B|w_Xk%CEyz==R#o~kDxzFD9d>|Wd`Em&$7y6BoXOrXBQs-zB?5oIwBftarmZ`NqG)i|BeM;gJhxj8ua~BBgrRB zEF4~p;oL;m>;Sl((nYc(ma9jo(;0X~`n&m<2c2aV<3VEJx9HT$H2}$%S9jTKtjj0J z#(>N*GEb+X-U5#}utLpundy^h=tW}T$4YLLxrt3>!Z8k}@C{!)I)T=BMK726Dk9{d zMgDC^;@<-EvZn}Cl7gcoR-#{R$N7Pt+`}ennUSdo*?(yqZN~Qo_>)jHLB=D3oDjda z{-yO}l8BTQmdvzj7@`OQy)!$V23>HQK3f9mt&VaXBsd)U4%jqOv(LLO{K zs;jEsR+zT#*P0#SMXEZD#1~O0C0|YaWC_};ID*OpPfjZ2jKL(hK*1^+{!VB}-3y0Ga|ai{baFZ0dV{fH;# zLDa9AymOrytPYU)vI7p-kpb;;5d?-m854(Flldf3xM^>yvqcAvuRi*0UvguXz1SrQslBNrS#Uc8`0d*`z zxeRVJRxr|tVe3~chrRazmqnIV zgxLoLh9WX7v{2Aq6HGD0(2GIb{>&-^PJa`^7%pDfmqjmqkv}f~&O+}huORi$j82Y( zeT}rd-95!y5iL#{k+8k0Zik#y^{_@TgXqE4ZX1~I1y z`#k%Y+$Ut;nbq=^<9D`1X9$HmyqfOWswJHW*C@4HjH|@8Y_HB}2pE^+Qi5+yAARv| zF$t=DUYJ-J*+;!S@^!g%sp!8I2+jRJ*jDlLha?4sr}XGP5F%G$neBEB^sy?tyr#q%Z0XQ4DE}9=k?k7Z-cwyj7o`pWT)C% ztfwc5tGaT{#O!LO-{CPqyH$&o)W<=<0_$!>R#%kJz~Jw>c5c*XNX`$UHx0NXuF;J{ z;~Rg1O6QXpuWWB{3Mm0)N`Ai@3!(F7_Ks>xuy}-l7TnsDSqKA*c460SC=mwGl#)T9 zbTn}0sf;8pD#aWwXvs}f3QbQ|a-kW(#0r>Lt|b|F@Npq;fFi1=>29?1KJA1O(VzSy z@n8WVWEC^LffuT3H6lJ(Y%al+bF9Qj$>M1pEGiKDT}zK@tH2I|XyR!>tgxEoSG&?9 zdd6h^HPyzyiU)IrdVCFzg3OBCOmPiIlZ;qN`@XBjKLYkApfqf577x}*WH)n}_5FW3 zpT~#1)K-v%(|%5=EH)z`0SU(H z3!i0n)Wwsm(8XG==dY))M^rd#acZVhi<7;vd@;3U*>AC)k4ry#L-sQQ_Acn)*{}4D zdZPms1gUm0(Jr_XY!*R0JN5WOuCeVs6X7vx=vevjhrP!mBoYdY#T{m)M5RGlNmx!@ z)-9a+X7C98AIQTCHA_X@=k>ZpfTaa}g&?@tHMhZvlzhaovi zpzS7HsxDW}4S&CRxR6+F-^(PEQpYdEJJS0Yry1tAcz^&W+r%jK!-elqtb#m&&Y@QP z`mwr8KkqZ}%HgO$2+x)*9pBRQr>nmPbza!dRc$Qj(e=?wN!;%)CN+Jr{N9^G(YL;o zFb=($l^eSFp<6oJ?d+}QpRtj%7<44jGw#JCEYnU)onkYU9|hy3-~dd)uAMx{%xKI>V^G$pzRoKu8hEo`VrLS}$y=+i|edKr>=Z}Wuj zX4a#b4>lC}hABo>@3QcYj{^-2B z;WnNY`-h8|le(*S)*jZb>i-|S-1+ryfgpcPuM+O@a%S;bd)K4$#j&v>nu{H~9N&nJ zF5U|+S0MI!&iWa*q&#Tfb+r0dNy7um`>8Wp2xr1?0!q)prbr^e_%5x_ zRAV@6jUGBbU(qrxRIP>&q!@CjqItHV1!x^?f z14swT6$#MC?moS#z&7(Q{z<}iKcxU+0Mclwej-1C8hAh|?>-khiJ~}PhNc0B@V)5A z3aJJh&X(5GLW{(*LT`I9IxQJ8a<~MDWj=BFVH})V%{ptMo~#!$Q@PV~5}0m}#)swn zt=s}^si~=7vEHXgR6Ff3MciogRNG47nH*0w2isW0rKq8g9HnZ;1L{l-wXVNrp|!{Q z`8zG62D^$2iQjE=l9$^6?=c?|F^^AqLV0r7lIZNv($6JUHe%rwVE6TxTL8jRG8Yb7 z!3j;hI-!W@+gWb`>nhstPyzE;gt5y;|6c-R<+>_LS6S*6x+stj1Ygf}lqY zB>_(#FvNF<(nOX0-7!(NwRcW8uIj6|?IU`@oVIbRg9cB`(Ha5Y1FI%V^{5DIVLl}k z@nmL;7ID)r001?BzLvUW0sfp}hJH8GS@Fw(tN)DJwq^SF8nagtNIbM4{4*>oo`&p4#JtBg?AH=iw9*nU34ZPH6#`?jT z#u6a#)+TYxN*pg^&0C(-;-sWY>S!S&rr*i?-adl%!9nSQ#q%Ql?uCgMp;x^~moQ|5 zL7l{6I7}#5?Oipp<0OGYK)Yy!}G6q0vHNx`?w)iVt!>oW@5fPcW{)T)yuC@9H0C_3(?+G z!0YcEO3V)k;=7+fC7%+#xoFAbeNthUSI+uohXR*eV?1j2xQF+OV1B@e638foy%u6& zFR9x`A0Fhga}!8x(nCoS=SHsR7%gwX(G!x8K?NEU%DbC9!hkJ}0Vv!0uf#@mNF2%v zL}(V<`V#?<5rIdy@^O$8I-)qzzeZBT-ueO;RvgUGI|B}(MLjh=h~6l(lej^{+&eeH zm3D4ZOP)w2KSESh@d5zPV{bHo9H)C-@r7{^Rn_*3okXb@(;K1|2j_iD$oF#aYF%*T zI|X`68^Rx&=<(y)e*!R28Uj~G64^wA zkaa7u@7O(xj;OhEq_7s41rBp~L)J56;*~fwj(DP|LtPgi zK0KHeyMt?yYC8L$6D1A~Kbk+a08s_i^2lfE(I^@^G(9@}P#-C3opZTTr|*b-Otq8k zET+%+s@tn7ao#1qn;Y6Kfx0}=VG=1NqY!Q&i*DcPsV=&{TpGAZ{7}o@>-tTY5?#-O zdaQDOsH)+Q${UL(bD!5R6Q;e25-{TY$)#0>HPRSnu6bd?Sve1 z1pE=Q+GM#m!bnyrOIbub^izZN>wLQFmrwYf9fiRN>RnL?%&Iwv`o5`-lNE>9POr!p z^}?2Y@dt_}7VfKrz;}-izf(q!W7nd{u<0kS#!BJ682Dm`{V4}I(R<&2G>9=RaO#D8 z-DPXmtzR35yJyxx?_s>I7rG1t8;QAp^KLWLS}de(%W<8Am{|^ZE55h9U`xV$Y=B3N zS4*m-a|Yf{;|+@8=t${#%ZmZ_{3=cUk;EQy!YnS4MwgaYy=S?RjQL0|Q#0Y1Jj9b9 z1h=td=o!ZJv(Y8CqTCbxA5L_qi|$cGTvM%NJ{=_hr%Xt zV|XHc`VJbK>0C$8bqXTlFJeA=mFNlHNeC(s4)_RQZ-B3+M<{u&HpXYWWa5n9L#Rdm zUnSK6oRZRAikCs!qx8X5@%ceJKgJU%U+JlW_gbGCDx>vBd%S=}^DpAN0iaVNV<*6C zm}xXIBq99C#_pc-je^1c_u-q-sI}@(5XA=YSFaWCO~y5*h>h^yS?r&bk+wYpck611 zx1?SM;~oGU8JOJC@5fUij)jc&_3Ug-*7!!#1K&~wFjxTH0$7Kt`-fG&Yt3?obzrmJp`mbSV zgh;A`Uy#<)lP<6V+i50>a-LQ`aJKOKVkr)e?UGBX7@>ZF?l=m;^ezJe**cI#Db8Wh zE?}SG1F)wjnVKT=I>b^cCk_GHgv>>!adc4FeWjVB-Gs#sr;9_x5fo3}?3NR_pi+iL z%TQP^HN|DBL*h`Ie}EoxS`CA=sf)vVC3+5iDrNZN+r2RJ;PAirx#xv%EV53KPsrbmKKuJ{9anOfK4{-z=bz444~1grUOlzn znu*>Y*ijE%K|l&3M8TVWsx+N{-relkNnT35nYP9?>kFg;V&Sy~_Ed%K;ms#x?AW>% z-ZWC{M8|Hb@k(yJgiBd-cgbhH7zz!MZ}F%)jmK&pm6Oa1AV0YN#9il7QXgJ-2SQJR zE#xtGbm%j!<3`GWX=Co1`TLy(K{RE8at;Sp)pK*j!=xi=FK(|U%PTeYhsPc|mR z*JJbMVyy(%W11J}L3m8AE-mkst1in-K>YC~)6Q}vE?meH;3|9lCcbVMudFfeJBg-v z+A)K6c0XAU*$chwRg5kA=`--_NnUwH33bR@N5&!36X(J$B#sq8C7?neP6VYYO$N}u zTQ~fX(4D*HmIpzmqU%Nd+m+^pN;R~oG(-h47+ysk^h@wN2)naVx6b_MW}KpA%*m^7 zLKi+H==xiURm~ZLt^}UQ=Jdfbbkv;A3)Qy7lUvzjk~V6l6o18fF%~l5mmx7u9yN>z zZpj~s`vAW7X}rG11V5GvO@x zHH1*wep%Azt6?Dxlk%H)iE83M2?c4hh~!O>?rboxJhe_bIpyJs;}eXXOafNtRbIMq z?L4A~H@WU8I9r<9Tc}b%;sUS zD^whQI_nOxS)!q8)hCE7h*lL3B)`rXi_#s_d2~S)3!i(gDWFFrlm_Qa=+FpILH!gQ z`>QIvmZriwWnW0obUJT;_4_Es{h`PI5UzFP=-Hdq@#vk%Yxnl~z4t3SH{tF$`kP~t zaT3*w>qEk!U*(4nubic-KIQb{+=9pYW zy|uu@$gr)O565CcW}QzFcpfH069B|mjfv?jw}w}qOtnDZD?A4!qj9->)amj%=3^ja zxOyl=y~7q<0uvup(~H~SZFv>+O~Ax1L=iE)TH3r6yAOL?!f@`jWgI3$K0Y&fOVNsw z)~b4?A3IRE>9iJ|sY@l-4%Ku}28XY0+}_D~Dk1E5kO&WYADEc(cZrC$1-WDwzQPbD zS}~pc>^Fxm$Wqo-z2V{_PgV`je@N3sn>Gxnvd{{xaejOXYBAk-cLU!-3N;sNG9*GM zF#Q8$y5@!AhmPcfvFCNc;&Px*v~?nZb*wS2kb5f^Ag|L}6gGOi0`xKbJ*87P=vIMaifS9X`w_` z=&c3PsMog_ECA~9eDl`nCutVY*?Un;BBL; z&7c9+`QcmNbaoSi5@Uo5fAhH~FO78Hq*T9UlMQ$;|F6RUa~KEh5n=z8!o0oO|=DwjulZ;a2#fKdCM_X~YzOb>7Vdg>Z;15s6v z&wjo7IWBJR(?QW=`TfTTgi&K3(NX6C#QLj{GQ}dJ)Z19%$%GrDMHm)XAqdCc0{L-? zUgYEwa&3sF$wqjLH^aPb=xbm3ISroIwlhBMjZs&5SmxNNuTjk&(s8P&2L`;{xN=(A z`f0uNR992gl`P|3lJ|3NKvC_%W|F5-#QlbPaw=b0pt-(U{$ihX{m;h;2UuLhA|=ob zS8s#RsN^xgqgV%PeEm#klbofGqVVI#c?5HSapF-EEYdPDnU4q>V)4|&Kwmv9c1I0- zGO(thnnfP%yZ}xcg(@Qab!^zu2%9)c{c8O6%su{ewpIwB>jlhv8Xw~1AuY1v!yCQ6 zfJ(q8^?GzKW^%}ty%v;+cb3I>n}UpsC+?~r+`=T)hTr$fv&>NXX19^3)f2kv62aKb zvaF0piPvTCzT{qk;Ha2Yr`HBFsbZ72r$)$jyl@`BP*2vgA4OA*^(igDqa-d+SsD;h z?*NaX1A4fbGFRf1atqsl1*d%}8$7EbN&Lt$UG2d}fKJt~m426G$lm+@{G=cmqM~s& zZ#}LL(;65L@ShSl3U!7&;{ao%mI{d!oURyiBqK_!;gY3hJ?)|7$S?&szciZye8;zb zUymKaY<6wT3I1flg7litykiz;o!HGY6m9G-BdSs^(j>0Wvxn}Kjk^C=tTKe0{%krZ z2X`n&cWJkz}aKGGp z^h6uMyuVEHtY%rM>F`FmAusN||N9cF$eD8i^C0Eq+D|52sS9;jMN)8g^>$&l@zBlY zro@oy%9TJUw{b$@icp2)8={vh1oZ8})z8c4|J0?H|ENo!q6!49Kt4Njd=i(j?o_|M zZi#xAw2!;@B3@5Ma~30@1y7xl^HF%m2z-= zfT5N=uFDE95z(baNi;WDMkid9;7HzMYWk^z>zAYN!G&xVBtfEz@)U0lj9qx1!7#P)MB;JM|UzcrTj66$OvNL()#-qi&A1OYb-7K-!Gxy zI^vvUJ9rP*R_M79yNNyUz7?kyYj+5bg0*$ic6{*no@yIVq*c?y|G}02qpF`I;Q3>T zy}j-WWiMJF)-k*Bf9*LYjCr{q`{`bPWv>zdRT<&cq-X%4@abE@v5k%OA(~D-OZgd7g|DIepF1lnEI_oElRWevthX*{=BhLVUP( zF7*8AeGQ-4!&YOKZ`H*!OTumkwnx^#>wh0{PgKMqyTEQg^3s#2$F3BzcP1It#5NW& z&?mh-Mr>(e z$=gk)22*_TMX-?vDhvJNkMr`WR(-4%?wYl_RTK45i5A}hg%BL!b{0r=Bu0{uf^llO+txV%o3I!jSYpwCFpR={A z69~7tZv|8)eRy($N4zy)1@jgMv_x+;eMh=?&{0-zW=aF)YAG&t{n>}Do3E93wc)US z!AMIp20ce{s&dN4=2v_WAKI;(AN2(Zi_^F6856z*;JvzqK!`B$z)CQou@z~{nsjKX z#EAc3Q#j$9#)NLJfod;uKR7ixmHNK?4B%<+Z)k!ue4l4W8>WVs-CwmmhdsQjeIO#Y zuuH1m-P+b3$u#un)3Y1wGS_CXeD;5(TPgMbnulioi6M1f|1l4Beq}nt_elurI$G* z&y{p8@`mRxTv9Z1^;|jld2>4}y%Mpu?x{Rvs;~GXCt6YEm_RN(u(6-eY%wd3Z|LK@ z{Xm$->iwN}ZDFNN>_ngwR=zFbo;*o|y&jNFQva84+wNGa^>SB7yH@+5Zzm2AhxA{I zfxq!g@7acVQ3_hpoHu|y3X&x1*@c$oyqh2#C;(bz79e!LyTzPN0x|08RGYjB(YBx!J-|8WYT19&Gy8u8+0+$Uxc>My>ng?Sc_1TRv&U((FqM6et-geJ*k5B<9Fd9Z#3 zRqG68EQXC8;Ld(h25Kmu-Q6^!g-&SF-q}~S>~|&m4UdKyT?{#gf=|Ud1T5|i_ z3mzAeRm6%!YesYbFzw+rlASs=Lp~ewZ{=U>V`Ymw$>TEU35C##eE8`+@)LlZVuCjX z1ak~kRA5q%@9ZY`2_rGmV65Az2SKP)@&g-O>UQ93UuVknt&(~jz0MY}dc^BT!hnOE zZWcwj@WVwjvk0#D7@S7>G7w6S2;kWFo(mL9ABcrhk%TGyuQ7@OJ$vu2_>psBDL_{evKM zYEi3@@Gn`Rz%IU;d*r?F@JHbXk0BA1_Chpy;OHSpRm+uD^Sf^pX5J*$t3S7WFyxdgec^KJ~Qfb>f&B zHE%Rli8Ib?CHKUTCeycR@%Zz8#D*_W!Owz|3-o%|yw%gx5Pq5Ga1c+frv9Dmj{u{Y zTdyq@oPo*fls8py0TLUtO&yKtkRnvTHokN<0`0nm#lc5SydMQgjm_TpBEY2Qx0YMwJC5 zNbnncc#_IpjK@AnGRL@tmW8hLfECT&b2#dQ{q7a_-aq68|DNjPgd6MPu(Jq=V!8leKC z@(=+S`%@m{i4?*BBX}WUB{b^A{Lbm$Frph!wegqV zMn{N`0VB%g;TjuRZLVG8!dEAM>W7|mk+E$gZgl>+4(R&p*%c9hQL?+(A?LuO$%nfeN<3Tu5|!06|TLPjr(VD_cmuU{d%?q;C`qAxW9UYZlji|_!8}h zUp}ha22I&0lv&Gg)iOR&i3q9rWAIJ!Q=DJ{`~Rr}q`J6Ci0MR%!V7hdS)U5Zs z^arP`KXV)%oz+2yKpksr=P?VABoWX_agtbdT7Bv5y$eL(T;e2hc^V$GqmyUJ6|k?W z`Fz$$i&_!4UoRwTc}y2iz?A6(Q17C@z?sLKa-rdbT$}l#&v=xqA~Za&M&bKJ+(=naMmO^*t2tmy(=rQP zf(~<)1AY8j4bzDdHG7YjpP)T-tdvHCEnvs#eCEgJLgjYTW1cOUJcNcGEG&X2mh(ysRV z1NM9WH2vC8`7|_t{Y5(pnPK&iu^3|jfgf;ds7RwrL)SyYgOoGUOwAt?!Cewb4fTiR zZZ&_VyK0P-jz4q|Wt_yhB(?4L3Jvu<2d0a|eKe(`t2hLnE7`X4yzk{$foX49-X2Vn zYUXKjYV$ze))(mL+Oq0%j-? zE~MwK_E~J|PDFTxIagJw9YgY;A^0wG&CXwn3ZI+#`xa=QuvywqRQ&&^8)W}+U)w_< zg9nw2ME+0!}>UbwIN(lxp&o`}CCZv*w#HY#SCaRFLEwy(7y&uGtBmo>mtO7OY) zXHJyhBDAVacL*j*B4w?xjM-qq*-jUGGcekz^iFw;-KwuvER z@EmpkH!ita(ml~3v&wUIwq`DE$54J;mU}{??rZIn*Skx*8>dicF$^>uIzXNGY}tX8 ziMa72%yh%ep(sVuJl!D)tJoaxycA$;svV*65W|@bBYoNPB#NL~1ue}^L>O&eMB%GU zGSwYRpr`C+S`M?f)aC+?-80@?EeE)eZ6e4KHsHjJXJ3qDWA;D9(T@ugx@;rhppVS# zx2Gg1aR>!P#;A9`5W{ejR-*4eNCs!-(wsbz>{KUy_}f{Ri@*RM9#UxbO6KKX>c88+ zehCs0Y$UA*Gi9$S#0jqc7#>!7P=u9EqLyEQ&*L% zki)}zC}2pkM6(nV>cmyO9@arz`khb#WL5IwZmSbHQi|AWU74qzC&op!g^L}lR19NS zx%daslc)`d`MEn2F*)auziMNN{LTRt+caxyw*4FQuNakt0e93RuqdUC@$|Rpoa=~z z0L@e01c?9tN8&Q1N9&h?*&lq#gjiGF-kcdNq-K-Oh*19U*pG+K1Z5sn_Zr$F-^5+V zh1a+^hgR6CMJ_ceg$ciDykP*c^f07^ZPh>QyVYWIPi?;YFLz^c$+T#v@p8-rCf8^$_j*tw|UG&EUuu z`0$huJ>s|G5GDaOI#sEer>LIWBxmf{>zj0s`UFv=GLhMOAYfTIDuA#>wRbE+3lyP}wc zH1}c2D^79bqjyfC;RDIl4);4Xe1Fd>r>#1KxK z8wxf7F~#N5Pwp~oT?-XqVs2}w6p{pJG|vafmKiWiT}-E+KV#3Ryct2#ytH}L8?pbP z;`>-&X(H9OBe4Ni*{6u3vKWqp8{g=1M|>c)f+>q`ksmx6Yq2a%tD#+++Gy-!d!Mdy z?wD|I+}57^Vg6qJ{7F z*Z&z5z%lbqD1K0+Nx(R!e+~E^!)WFkdDMelz8ak(B(l8yqya|RUZ{TcPaz{YUiP=I z?|jrXHFto2kdmaJ{p$M7hnBGj#V^UTG3mW@Z8p^o7n9CTI$z01uQs2|T!46VA)z;C>qA|_TJ@1XzX$I0S_Q=L$jQF!v-g=&>6 za31FN!Ph06-^OS`JZgK4V>X1CplzW-yjU#1C*bsO;3$%`z@c4|X;VQJ9L=~*jR)GG zV!TUe1Vq4yUsakpz{rkm#vxs7`Z^`5R&DkPJ@BZj7msTd!7U7~cN8MoW*3($nL0Pw zYw#K<4nF~eH#)Hq5Py<2)scG9Mvb!<6QnQC!FXgx5ZBM?!m~i1P=;Udb8TM_I)KB8 zppl0gg~b3=H4Im`DrSIjo%U#ivcsvb3CDwsoxBeAR`Tvj>D_%p`;1TCIJ0kGP~-t)mM@Yhd5 z#SN$Fv!v-;n@Mw#zcXN0{ihUmyXm4T1M(%)I>BDMZEIeAF<;&W~(fa|b^!w;^Sh0o65aVsqiA z<^gAGxx=LEVkVg(D_xUe=7|?Jl8@l}2c_4Ay{$nrmkiFkwSB@Z*Rw?L&W2MH_g#s( z3Y9p%_a^%pz67%5Hgz&|w4I9yt4A5GM*2j83&A2V&y0LaJ`00qYZ2J5rnvV$cO=nO zEhr$ThUu8WccscFm9yi1nz7zlQq~7&kUU5#-3cY`W*mDcq`LX@Nn1oiNf3W;_sAvE z8P}Pqdi-nr;p&5;jU83yEf}XsD;C@b2FshWX8PlgoGxPR&F;6G32Hiz2H!g=7LpFN zfcu1WK}!)z-KZC-8bte$x)kU_vOIB=eE-uG^Yoa)QK-dJcb=Z`B*DYiMF+D_7vdSd z`9{K`MjM~`3siIt;zy+-ZNtINlx21zcal@Ox+l%%dU2DX7j^Jg2(Ff= z$&X$9>+m5~nUUraH&9BK0@~Gu+&d8x|qGx{5-OMi7M0z93YStK}EpLr} z|7rO~&gk4?MaG3xk6Cgx?dV;aWleKy5ZC|{Ojs5z4+W2VwJd624V_uvnYPtz^Tdi2 z)84vBmnvH=PhYVp*{D3+*bu!7fzq3C6!!ItcF;=ilRgBZKgFLuZLxW!<+vf$!c9l_ z=?%NNh+Ew#%}JJZ_yokM`MSUIo?7r#4kz?r$AV6J15EL9=864W+{ zmiAh%yFvmq4WViM*1OekM}-6(W4{a0#6j_t1o@Xw%*sPYpiG&bvn|myyOjsQ{yR}w zK?lGz&fRy?CamqQA@it&>9hs^Sns~OnXNx=0&rJ?C)9Q1Bi5JNI-%|!Mh63~&VR0) zXv(O~x4ch2i&}T17!hjHOI+MAJ-yG%l_MB6V_s5IG8R_S;*AXr8jrx4n&ix$~M`5n+qBis$4nb(>lh{y;uT6(vGD zrk9Q(-dlm%;SYd$vDVNEa~nt|bJ+FXMRqa+P!DaSfiDDEenZdbozwFzz9sxBhq0z0 zk{*HV*4p6@5T|B;@-U3YaZ|7k9D+t`EDavfG$~S$)=gSm?GYDf;Z!H9s5mJ4C@Rp&WeS0u$snHsbS@*!g^KpAnbx z6F}&+;IV}Ulw649%CCHxcY2u(^%l3h_xGk|k9 zB7D)0-kp7tEj(vO2vcl!GrhW+mIHCob)VKeM=;LfTz20g(J`~x456!HWd|7T*pgGh zOH~{sz(!;xLEel8^zp)iT%i*On%?`pJ=2>Vx0%OTD8uHF&0m{ux_+i{@w6mD6qn8H zz2I-KF@?XOcxh?T^d+z?C5*jpH9-W3xPb6V!pFS*b|n;>LAYD-}K6 zox2(j-IF~6F~SW@P)3kZn1?nPR@imfB~Q++5U6q>g7Y-Gmg`U>5#6~OhUa_qGg(j4 z9jc<3s& zVc;St{urth1(UyB(rj`-1fQ@Eb@`C65=OKXlL%IsG3F89p5{4~Td2QRGV?MBGrUrr zl^8-DeB4WdXcLbZCBGIe?)Oa<(YbBkCOSqtN>A`CK!69mc-`xAj3pvJELa($Q(9;m zh=4e#C&q;f*3o%kVP5=q!c!fLzmAYOz7Y)S)s$Q;@;61;kdp~^t9-9v6ZN6u}()Z88AVgYkZhZ5oPuWhWH2h05%8K`M63-(~ zxNM4VXxozRmxZ*7SIn%#y($UHP|q8d{jvN7

0ybUymw)5FscP zZr3HVbTS=WK?jI;jf{|`($9H*@>UG_nBWv7eA<4vD)jBd*gK2K2}l}C@K_NPNrXEX z0$sBgE>VjTsK&T;9aBr99k^APPvWinZsUXNdH06S{WkEcDq zgMiPt`H^}tZlFcO1k*AS zv8(yiL*oNYa*dl!W=rBjc0;N)S$Hzrx2?lmxjWZbC2fn=#*-?UcGY*BMkF=r)*VhxKT%)OlWsW zUTB~Nmg^x%(o<0|>CRylS{%inD>P>(wDAjE-a_IL)(w42UDLNo6ZdfZS^3q!g&oR0L%VEJwAF4TwA|>( z1w1~b5KtF=UsmC%9kI<}Gxo49PFNKN$vIGi_0S2%*@T3X%UqV>NB~PP#OF&l3D>0f zi-);&Ye<0GxA~Y1!LaANb@R<4vB@Fsm$RvsL=Wv(WnlV#mcIkTLLeirLPxz(&W!~V zik)y3^r3|+d?vaurYI-y4JQJaec9lP+=Rp6J~!>oMH{8V3y_^;_NzOzd^nSZ*oLQQ zcIJHreNbkI$JWN?r^ajjNuvwD<$a;HK^HoJ1Gz=EXY_=g3aT>(^;6Vx-s!3Hua|=MLV02d5&Urv1f+DAR$fjAK zd4z3)1(~un8}Fv1W9a8+p2`qVkt`Hr4ZX%+J|>dqu}0e9&0cDjEgVl@V^AA9aSG}o z&h>ng=;(8^x1u%o7mQ;L4Fq;GKN#JcSTw`-SmU;}+YC);<^9v050w&tQQtTFmt3w=hcU=5Gz9!Dl=qX6K_>6Ibm zxw1NWQC~A{3L#K<6GMN58JalvZ2ltzaYFQ~eo)P`-GxrUG>E%Vtkr&6ythlDgY9WJ zD&qD`EkDy_6A?*}ja=3rrwH(89s(_@E`~4?bH6FRcck)N15;B0YbA=O`-4 zrzNBhfwJClHMhNov_m&99WDJ(FAMaM>>aRoKNF|%e8)tYwJ3c7THM@7Z#ce@rV+g!miDacZZ4-ui@ayr9&AgOfKh49ynZ2W61&0MURQ(}?uTcxm7d#guEiN#r| z8WV->kD7CFgS(-WiEC1NFR}OoHL7>!!=1Lx&{xxX7azUz2 zEv6WJ)-#}y+U(E!xonPReweV9Rqp9UG8wkj9?Y(vQ2)PTs-vR?D*x!aa1>$`@LulX(WvFTx7$Q}}_U>TWne|87)nnUtXQt~(#|hF`@@E~7 zOY2}nVOgU0GPGq%IW6x`vsy$%9rqA0h<2j}(!5Le zzsJ@O^D>{e7+u<4X^WOKITYhumrv&t#d5vGWT{ON(_7%P0ZinSM^CIBw#4*Su}}Pa z1e0T#yAIg{6bN=2VXIch2r6&C8`{~gjTc1?k_b1M0}kC>)eKPdVN`~A=Q_SEgxoQj zuX%ADjD<;(#lNBPoD0V~gyLT##x5&0J9RZEnM4Gsye7G!_fjJ98~L`3v;Bj&pJ|eV zs^~!Zv`U&QY)2LZc2aI3G<%xoQp&O^lI(ppIgsdxk8Hmu3jngEDA`0Lh|U+)6fiYb zw^N<XP`prNsRWmDTE{#KcWP(t>i`6o~SBCVg<{;LYNmLSzuH z+imKkoeCRZLK!6}^RdTI)?muX=`U?Hcyjsw?p=Zlh>QOY$8K|BY`;=pY)uT0-oTeQ zZH#ILDRhJkpp-@s7?BE(7r*{SIMiOnTR`q zQEyln%dxTRL$(7t`A>5k4dT?jgr@{{6q4_7LI&qRlk!h9mwDOKp^yxv?-|lfkz6J+ z859x^Gt%%GEo4&^fIiYaLZC1G)efFM&{uciah@w@9}?E#(hfIU&?6@q8)BtON`b>5 z#AN}aTLz@70Nx$Jqb%oC2f6MLkObH|EIK{Zu*(_(00Aq}b-%{7jkT^rpk1 zL6YF9k;YU0rpGh~?z95qGJlEa-%Af^+?A?11M#edrIz5tr?^uP zLTLH%PYGYj&S%0kt)s^VkR^1>*0167H2L)WSMONcD^9QL`Lp*{b4>xPV#yU(#Fi)25HBn1BcIkwE(jlri4MHAjIwBQS>(M z7OM!dLPyfpJs?Gc&u0&zbExII+zw-()}9{_Ab8+hCM8f|qiEsAm`ay|Pm1Qs^KcH}lms7b=U2}w{J=TS!7^7qH^JQtBv}aPp`q;07i{xB(?Qp}_WTSyZt^$l z-eDuGQ+w*xxQvJ|poXsw_sL*@tM)#0G2cR%nM@^au;ZTer&0wMlwh& zlOvYbiaDJu3=e$Sio2vclpJE3p8f8{nOF~#Mk}r&8=^GURCx#2`PdclbmIVsk9#Hx zRU{J~IxEH;(^#a|-mQ>;DoJ)AX@(HOZu@)yHdktPF76VCE{YXe}KC#hp` z|Dd7^Fbt%Wn8F&v_|9KQcC}Z*%Tyw?oySl*w2#_Ah?ci^gsPs$)b#k)2GEKks%JdZ zTEI*3>Hdd=pefv^am|pprl>>Pv=F|&h$7kEswd6Cbb6UJfmZ`FJ-E7PO=P7I!a}q^w;+=nUgAwc;h(UKHFiLwD5y4UIS@0D43%hCOu4T#TT{G zA7qi9m&o`AT$7DNyK!=~eDqM6jgroTzPZJbzifm&rtTe{*MZlh_9BH)^ix8*LqlKL zq#xkBz?~8NgTdkG_;d%LI(-iJE_r{07y_L#1W)G z@#F-3w`w0M=BqEZm9P9Ad{m$cuv`h_v$+%q?&|lh9nqkz5q^K9xo35CwZ-EV-KJCZ zdfBTJw6|yZn+~?~71K5y?gu0Yakl?lW&fZ0a#2$q`Ig$8yDe0Ivc#`r&(ye4-K1qD zmPq^%+^bS{Y^tfz-GrY6h(5URP;{H7+YC)hW_hJ{5we_Ql5Zmh>f<{Uv#jbXrWc&Umo6Md%81T-fW_X zr9P6Dj0Xs>4{?osjN9T*@K&Dr=GZTXqVmG%t*k}l`gW#o;xj5?DqKNaIWM@FUl&-s z%A!$|-Ye(N&QU_D4SY~i#qyNum;RcH4NcqiZxLVLp(SoVQK*tI?x+f$6ZGjfSE`a# z#OmkH7_&XUM)qQ>L738yd$Z#vSRdElA`x3XkT?;M3GO1sDZuH znDh2{ebdR|bZAJO_bd>>i9`S=8^YkhU8dVIJO?O#AJtuFQ0>aBue0IV6SS;0#)QaF zKAb>sThg6Dq)(rHTBu{m*zF@eyZ)qxodt5`R{R}0)TYX&s_g5553mk>=Y*<8WymfO zR~)1!|8S}nZcV+}fCs_FIh@UC<=Q}&3&OvX_une8o~ci@k;+zqlVeU_m54VzJ5?JR zd?kS3H}oz*YtRG{&T~R?Pj?Je-#$(`%zPAKrv1Gs;LB@tOauyO;jZORDV=4@KhJjy zaQ(La`2rXK3m(0|_T3>~|KtuXJmNnn;9{~yvyx$-2-HAa%MH5~>CmR zQ^2Y(Bqm2y&XyF*3!WrStpMth5|OsI$zD9NHd7qI?|nX{l1;e|OCn8v#I?uS*HYop zGF(Jp=j75=@bg@{?z*dc!K1%VAzq3A%h~xWoq53#%e(CSA-trn22wp7O*Yk(9GQ@P z{tw68QojkqfMtY3XhhVWX$!dO9x?Cf9j2@zH-I(c6CHsku2U|RtbDJJvC z_EvyuHG_;g&#za?{So>VuIAlPtr6d^b}T>fS4aY<5`wq08jt!z&Y}@C#}@YXNt{7@ zoos>BK6AM6cOA$=u6t&7~s&oy!H>Fy?6q_ zR$4!HfuogHIwicANx9HDu}UVY*w9xoxdLg*O`KdSyKgdno=StGVDH`7x@fsfcI1wIJIV`76`GBJ{s~Q zpMVNg;(M=XoiOZPLptZL8#_DZnTJX(^r-Ql%psbY`LFw2#dcKo-7j;PVrgWEr?uEa z=HoqN66L{GN38+W`m@RNFE1Y?+182XdSx5n4|BzjhNc&0eyY`(4NxNQ<}GU9Zy?qt zE&X4{<3Z8oJ zQg|K{4pzx#`kRVuE!nLa%lwBOx>eHF><-Nan%7+r4eIv)m7u2n)v&#Il&$k%>94>2 z*VbH(33VG^?>V2j?r;wre@}#JScxHBh@*5A6W%3Y&|YWk2NS~ySWouSc- z$aDIM=l6@R3nylOKku?N2`$=%77sgk-r_&tfR<+^Z!{XwoWDB6eE#A?XgzU$Lbz4B z<-CDlP8={00_oM2nKD&w6IhJB2`jU)Br+HS3V+uPxv(XlqZouU>QazVkJz)r%Hj)p zD3It2mYeAAoH+DKeaZbyH+y4Ib(ibudm_avB=#&(oG;!U<#LKGFvr86 zCx^rhX`UoTpawCP?c1XRVX%D^FXQ~^H+SAeo&K`cfSW%98{Lp=r_wpd)lUQvnGX7~ z6g3zyiMkURN*@0B?oX1ITWZF^0g5bybSO8hc1=o7<}eR`LAaqq)mm^y>W>85Pc{&` z$;a!kh=J3sLpHx#H?;IVSZ3CcXD&a7W|o;XxXdH%+?+%HCQezh(7Hn|9T)B@?59d- zFWjpf069CNEiJ#eG#@`+`A~hfJ09YdxU^8)DsZxJU~#sqPU3Bm;KVs3xF-L<;cgfg zmznRv@J-6qB{yu*9-%?N9&75{8My`P>p6u_7ODvL4U&?Y5VA7e+q4#NSqpt*qWYqk zcw>Ra6%?|}>r*g!xer%kntBlbmo^y}Hh)YX-^q6_jY)%GV=-0JE|xai>AUp_=j?A_ZnfIqH% zHX@9#L&5??Y?5S#sNacvGO)p0i~|l^@+mk=yY{(#yI~YX>fNV zd*@J4>9Wj14bO6(C;|Iyb>dG9oun3$||UD*x%n1<|75Yw%X%4*y{>m6J$t)Z*4iE_Bq z(}vfeQ~rh4OJAcY8b6r?A}0KVqBA)_0$<9V;golLmpQ}q;Jdnw>~*#?dfe{oR(JOx zWgnR*-eo5}i_3gjygt(0tfxM=-VK&fsZ{lR?`ex11h!hwi(IsDSC^2z)?W*$m^NYzxWPP`te_%{1-g+xI6N z0~BRb(1{vjEp(sOa~o$Y2BER3j)aA0ZnTD{TcgmN$3MkX4~naGzH~{{v!nL~3g*n5 z0#!^tc5r@g!Q#T$WqmfazB!SybAq(2yaN@4>J5Ge8gwB4&tX&9f}MZ{T(*wG|Hn{L z6XDM`J~nw|QU8tVb@5A4lL!N6qZFZUl1kWPa*N%uB~O01G?)ojUS6^B#p)C1WK~$r49>|VbW-A zzIf1TP|TIExU5i{R!Q}DG#1c06EV{UDCd?hFQqb$`O9|#Zlj0ln6;uWz+QCy#^IB> zFWr-%B_WcA6Dx#y!u(9mCUZ$j^bx+)^v=FCr);1I{4ib+ne*DIDcBCukg zpa*$Uv69YT9^5%<5vNMT=&~1#hUP#eyulBZ^oHfWnLA0LI3rZ#?#wQMV8L$1Sus7n ztjl-DTXzsORG=uK;XBX@BrK$%jLpxXuE|&LJdjS{ao|zFdeL5K164F+*NJ8L4RZc~ zR$%3*VbAPOt@s6=fzU4Vk%0-jTyb0X7At$~5k}2+6rDy{ZH|zZ1F09Abg$oPxaHE2 z7sv@2+ThqFGksqpbBeUx2Criy@F8^ru+h}Z^5Dk1nQTRUD&5efO)k76`y*~Vk2@Z{ z=7EFrX+77YY+oMcdVHcTyCSuFc>k3qWNAm{KG=7)lf}O*g1pwec6Ba*IeQ|}_MnHx zUkf2H3?6)+agvxLFdM*d=gsc60)LtfPBea5Cb`0r=w1&U30Xlh@05xi^+v$edjboYXbMB1md~~pv&D2_@m=$l3v3!Pi=5-%4xdzzWOmJVfB}+7 zAKmP!D{9`B5OSnD_Oz|VW%Li`pLxjpn+quX-?#w2iV__y=5v?O0JAjm6PSMl54^(q z&L$|=Z_o!cN1y&Gi!(1tGxpDp`#19XexK_z~-0vl_IuiOakL`Xs=je7u z8Dj+2>iwFk(F-G5sHkP=B1wJVEXs1EnW?W07CGUFr}gVrys`7oIvaQ02pc;nBaU`s z&MgM*L-8IdN?I(xmBb`%s~C%{pU8Y{FaAvOmE{!DM|WuWr>}X2udj0ydC~P8)2+Pq zn=c(H^{60Tu>)t@#$E&gH@Ia2YXeWLZ%iR?UZu# zPV8*tsK##VC=2<)tnai5*O2&L@O^#A>WKkDc$quQY*51{CF;+-@pWe4f)#2@DK~51H}lAzk`&4y!+*_ z>z0SE5#JYgk`(oSQpQHRP06`_JZ{B>9E5x{s!M*l(R*Xe^@3g8$hnm`aOB~~W&b|u zZ>a9yi#ikjY)1xso31QL6c2ERcDfNT_P1e>Z<5B?V~wpJv8AI6B&9Xe?)7ujkqF5e z;_{vfO~0s{+TUA)z3Hq@Z-(F7{y^;tpA(7~thYQO5cX!7k){f`%OPs~)sE<3Inf(J z8n^|dHRs4DC@6T~ei>O4tmI9bpOhb{lHNI^ic*lnt zA4B6W-t(2qgi~TzbR-ODI>>yRm5mGnybOz)9@c$|#)af@mFd4Ax+605q>$xMoj3zd zLnn{`ryg&!MAWStEk+X&%b0CtCclL!!d+zVcwy&(`636WHzF3YC}y(8iTGhtg|7{d z@;tO%y1DD+-`_jz+u?GxUh4ZGR&Jnf6t5p|{Ns6}UhF(E<8}wP*;mD2TCoLSUT2rj zVs!D!PuLY~)irfkir~IXzMvGkjRvEMFsNFO<|R%SMP+%Y_9mes_nnL8Vw^YJMMT_H z_fw)^&W0_wR^T>h->;?Ad3F-u1ePB*JrZ_6onajRC8Gb)MgXWo^Ojon&f@0bRJeKV zKP2P#_5#?I1n3FLyPo`g0@&(3kpo-H2>9wiD0n6KD-tJ!9T%S+ew_Ygb42=OZtE8E zOPag6Sy_2mn8-e+;4R(?>jF6hWV|}E8{yU5dY-`$xjNX)w9-f$i=zy8|CA#NDn6jU zk0QD^P8#{;I=K-8Rq{`T>Py0HFAK>{ZutG!KjBKNs&QlPbaGT#`~Cr&U6JTCl8R5o z5VW_FDVTe@A{D=8$DOQL#kM=}oI$xZe?>5IV2-3^G|xux+pq)D8=b$cdgByDZuH&)}edXNpl?B;&5;tNR?A)+#XaJ!m>hE*aNMwwt-wOD>Q>? zVQ*3-1DT$`IJT2H?0HRzf|A}3Opo;i*dNyAzY};uLbS^rb|6-p9KJddO^B{3+#aZ4 zE86f`l*1gKTu<{1)b!8y=!G;E@(5e#n#5plW+B#_pQ0WKo9iWvR5baGIsO@p+bWtbwHuAr=!6}!Fyx%dphh$2apZ;anWGQ`sOO0Vkj1$b!(ZyIq_5>ycIS8T`9A@ti^ziX6scDu z^W_a-34IX>1uWTA(Hg zrL_p}6{lbk&NgnbgM7LFX>}~lJQJT?qKInc10#OmZX7L7WCSF*%p&HiRm5uk zW={xL^HL6^VNQB6+inYWPOFrF7fX#GdbO)&{904y$48S+yKa#ni=Pw~(v8Ar_H3HN zDtM)xA8F>}&WxKB$i)^8zw?N^4=MhJ+iHs!&>*o{sdj+a`%v%;w;N=fwz~HKvmF!c znAGpva%b9n$m+@ra^TLk38GcM-;_0$fRow`cqpa}sYxMTIb4sLuPm6!#C6bvfe|%- znCrg=pPvcewd7zS+{*W&1M0uo*#rpzht91VN^pgzOyxEsjM z(-yX~e(5wR@^!$6{DbjGwn_d+TIZj^wU)1`E9nL8d(?=`7k$=oEj8#f>qn4)?arro zJ+7h5NV_R|8F$Ml{o*2Nr1#6q&y$^AbW|)O&4b0`a6D-?xeTd_?SgCV-{OLvHB%}`FY2aNymkNzXtR8R@Y|7-5 zmrLhm9ZoaZxW6Vv%ZHEZp7dd+#1HWB*lwK}W7*VaW8tq84G-p{R)suOs;K6o$C|Qv z3Ul5F=h^QR9^Vd5LFF8tBx-x{ExybVWX$3X4iD1a`K*w|>(P`IiVT7)Kdzc?{`kg*j0Io2&0~`| zY?Y2)oLmsdWYtEH;=P#@-HmzbVUtO1U5LuaV9*Bt*su6e^A%~Awe?@mD&TuGPCIX{ zu%i$<*zk?T=9GAAyg2XWw`PJ6R#IDAxOm)JMkuo{B+}5i^pY;-^an{Gr)c&Zv!c?s zmD|{<6{ySyPSY|NsC%@LNdnw?u_wjWVeA`0g;OJC@=O1)VW0$G0(AS5=~f!t4URQI zl^x3uFBJ$$H=bB2S@wf?yR8=iBO)4Xrc2gfXR5efg!6%+wdXXGwflw0EZdEX zLlH0TxiILII1gMSPbgk^)(Fw$FB*I4XN}q{{3ZU#7WrU;<*Z2b7CT0T!8bP7&nwpo z$w8!e|AX*rRuu2{sD9)=JAzRkQzBet&*ndOEbBBmkihHFt|u{k*-T3&YfJTi9Fp=| z(NZsiGfr__$XZnPYJb_c0HxKCjgtcqD$_u>DAXV=!L%U2Z*P z4klQK4N?V}fcQG}w0S%p4(4Y@49h5|iv^%rOO*wh`e^))))>?=F}9&a`=AzP8q@uO zvgJe%ajo&75y3}n3kp~Xsm*PmA;I=1RXis5o+>sIjmI*6zWioMGH%pL9Csed>puG6(+w542iF#lw=Xd!x)nJd{wDt6<>;=R{Gf-8{X zk^O7x_aEnK3d&6_rVBj=TIPp>UoQqr^CU=R-eLI9UHog<5<;tQu1c9s{$XDK6>+QT(KK~_+Wq$i7k_Er*U(irtxskvdi76lAFPbzAU9@JP3#~cK*)SVZ%#c zBg}B$9Y;E?p+a7U7v5pdCrE%FMN6e%c94grqBIZpzFFh68wzjybV^*ZG zd&8LrnIrw94WCdjabVqpHf{~r66WS0==thm=BfFl>$u9Cy+5EYrUO0C)@##Vmdm(F`O!5DVU2>u#-%p163cq7zkeV@>AQDHqL z!WnK$T~xxpI{FYKgN!M3Kdg~ef3G6Zx@Z;kj(buIYW}da-pVk!i-Th=%`B?=Kb8V_; zPUz-=N3TVrN%wUMDdEc(;?G6u8o773%_N-qy_r%5E8e$MjQ3NaY4@R3pAbnMAdL1@ zjfZ8{lt%lHTeCk`!96V)UD|0sl{Rpv1`LI-nBp&XAb0<7Q2wtg$|DUMIEWCeyI%jN zVCEkW1@xF51C`6Iw-cDJd6W#ic^*NNJb>T74Is~X+myFWxJBGp>pzhGxF} zqfI-rlfxzW(2~1NKC{G=$zx`9P6~Vi5b5u4#wQRHPCPX~iYgn(rMvj4->>*R3X>?^ zcu6D;^s61d0RQdM8rcktmrjT}}?3q!8uK(n7mt%kW2>@MqRWQM^^eT1yF~y8REHYj#P2N*o*vd#->N#_Yf|ACNhp$;GOUEH%k>= z8#@#~E_)@6il<+>7srhIM7ED{C6Tas-2qQ{=vDc5aTlxHR3P|1oi@FcV=nmbvcse< zG25Q4n?&&ZM5T-JHE;VAnrR|)>ncn6x(&?lRlj&p80-)vAY?E1jenx&O@K>6QuaW0 zR-t%t>=y?++aeT5^NUk=Ijs7sAmbxNgJ|bTTLL(xl)^&W=p^UfYBHGYkKD$IbSx*U z1Fe^u{J{C|ukHgYDPi{1H1qG$u6Xt#>BIKAY0vu_aoLbw^@7zjDaiiBCC(1L$1!yC z^6gLGSaJU&5H6+cXr-;-e%m{?|GhF*f_b?Xk-Md_BREOFH3TMSBkkSjfW$qr_qhrs z_s0yfB)v^vxR-2+@>SkD?K#z?f7?^#lw@Vsl;F|nUO&=a7`VIzbN`Z1r1eMm zf$7aZSI&Nj0d@W#Qav>rYOhu`NWL^_B!AoHVj2?q#= zTuCR1^=SW^`<;2y|N3F2nHNf179NGQ0sm2S5UCdE>IOr7k5^!Z)*2J#RYZAt|Ml)F zx72jBC|`qP#Ps7F*jg&ab-SGMna^HVW$$?KpHEx)64)BMA6U)#g^FxlD4^3^tCu|| zm@{uaLCR~?{a`#T-bwrK-ub;>KM-Iae;RWc`_Go*QnGrlw0EkqfB*gbx-86fHGva= ze(TBfJile^$}3K$3*t+t-AGDzQy{9IcYdS#pD%2tC%ZA_p1Dl^8uG6S;}evLehqYe zp7AqeC4Ka&)?VjLocJ`6Ki=52gE{eoqnmaO!B)TRIWkeF!srpxXeVtxA`0{}c| zKdCV74aL4>kuf*xJ};Av{lGelze_(Sh_xQMOi+TQcgC&H4vpLAKlx)L{v+l8DCvLh h`2Sz;*_ 1 (native)** | **FP8 KV Cache** | **Spec topk=1** | **Spec topk>1** | **Sliding Window** | **MultiModal** | +|---------------------------------|-----------------------------|------------------|-----------------|-----------------|--------------------|----------------| +| **FlashInfer** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | +| **FA3 (FlashAttention 3)** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| **FA4 (FlashAttention 4)** | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | +| **Triton** | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | +| **Torch Native (SDPA)** | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| **FlexAttention (PyTorch)** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| **TRTLLM MHA** | 16, 32 or 64 | ✅ | ✅ | ❌ | ✅ | ❌ | +| **Dual Chunk FlashAttention** | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | +| **AITER (ROCm)** | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | +| **Wave (ROCm)** | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | +| **Ascend (NPU)** | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | +| **Intel XPU** | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | + +### MLA Backends + +| **Backend** | **Native Page Sizes** | **FP8 KV Cache** | **Chunked Prefix Cache** | **Spec topk=1** | **Spec topk>1** | +|----------------------------|---------------------------|------------------|--------------------------|-----------------|-----------------| +| **FlashInfer MLA** | 1 | ❌ | ✅ | ✅ | ❌ | +| **FlashMLA** | 64 | ✅ | ✅ | ✅ | ❌ | +| **Cutlass MLA** | 128 | ✅ | ✅ | ✅ | ❌ | +| **TRTLLM MLA (Blackwell)** | 32 or 64 | ✅ | ✅ | ✅ | ❌ | +| **FA3 (FlashAttention 3)** | n/a | ❌ | ✅ | ✅ | ⚠️ (page_size=1 only) | +| **Triton** | n/a | ❌ | ❌ | ✅ | ⚠️ (page_size=1 only) | +| **FA4** | 128 | ❌ | ❌ | ❌ | ❌ | +| **Ascend MLA (NPU)** | 128 | ❌ | ❌ | ❌ | ❌ | + +```{note} +Multimodal attention is selected by `--mm-attention-backend`. The "MultiModal" column indicates whether a corresponding multimodal implementation exists for that backend family. +``` + +```{warning} +FlashMLA FP8 KV cache is currently not working. See upstream issue [#8856](https://github.com/sgl-project/sglang/pull/8856). Use non-FP8 KV or another backend when FP8 KV cache is required. +``` + +```{note} +- FlashAttention 4 is prefill-only for now. +- NSA is specifically designed for [DeepSeek V3.2 DSA](https://lmsys.org/blog/2025-09-29-deepseek-V32/). +``` + +```{tip} +Speculative decoding topk: `topk` is the number of draft tokens sampled per step from the draft model. `topk = 1` follows classic EAGLE; `topk > 1` explores multiple branches and requires backend support in both draft and verification paths. +``` + +Note: Many backends that do not natively operate on pages can emulate `page_size > 1` at the wrapper layer by expanding page tables to per-token indices. The "Page Size > 1 (native)" column indicates true in-kernel paging. Some backends require fixed native page sizes and cannot be reduced/emulated differently: TRTLLM MHA (16/32/64), TRTLLM MLA (32/64), FlashMLA (64), Cutlass MLA (128), FA4 (128), Ascend (128). + +MLA page-size constraints: +- FlashInfer MLA: page_size = 1. +- FlashMLA: page_size = 64. +- Cutlass MLA: page_size = 128. +- TRTLLM MLA: page_size ∈ {32, 64}. +- FA4: page_size = 128. + +### Hybrid attention (different backends for prefill vs decode) (Experimental) + +```{warning} +Hybrid attention is an experimental feature. +``` + +You can mix-and-match attention backends for prefill and decode. This is useful when one backend excels at prefill and another excels at decode. For the implementation details, please see `python/sglang/srt/layers/attention/hybrid_attn_backend.py`. + +```bash +# Example: Prefill with FA4, Decode with TRTLLM MLA (Blackwell) +python3 -m sglang.launch_server \ + --model-path nvidia/DeepSeek-R1-FP4 \ + --tp 8 \ + --attention-backend trtllm_mla \ + --moe-runner-backend flashinfer_trtllm \ + --quantization modelopt_fp4 \ + --prefill-attention-backend fa4 +``` + +#### Speculative decoding with hybrid attention + +Hybrid attention also works with speculative decoding. The backend used for draft decoding and target verification depends on `--speculative-attention-mode`: + +- `--speculative-attention-mode decode` (recommended): draft/verify use the decode backend. +- `--speculative-attention-mode prefill` (default): draft/verify use the prefill backend. + +Constraints when combining hybrid attention with speculative decoding: + +- If any attention backend is `trtllm_mha`, speculative decoding supports only `--speculative-eagle-topk 1`. +- For paged MHA backends with `--page-size > 1` and `--speculative-eagle-topk > 1`, only `flashinfer` is supported. +- `flex_attention` is not supported with speculative decoding. +- For MLA backends, `trtllm_mla` supports `topk > 1`; `flashmla` and `flashinfer_mla` support only `topk = 1`. +- CUDA Graph: the decode backend is always captured; the prefill backend is captured only when `--speculative-attention-mode prefill`. + + +```{tip} +If you set only one of `--prefill-attention-backend` or `--decode-attention-backend`, the unspecified phase inherits `--attention-backend`. +If both are specified and differ, SGLang automatically enables a hybrid wrapper to dispatch to the chosen backend per phase. +``` + +## User Guide + +### Launch Command for Different Attention Backends + +- FlashInfer (Default for Non-Hopper Machines, e.g., A100, A40) +```bash +python3 -m sglang.launch_server \ + --model meta-llama/Meta-Llama-3.1-8B-Instruct \ + --attention-backend flashinfer +python3 -m sglang.launch_server \ + --tp 8 \ + --model deepseek-ai/DeepSeek-V3 \ + --attention-backend flashinfer \ + --trust-remote-code +``` + +- FlashAttention 3 (Default for Hopper Machines, e.g., H100, H200, H20) +```bash +python3 -m sglang.launch_server \ + --model meta-llama/Meta-Llama-3.1-8B-Instruct \ + --attention-backend fa3 +python3 -m sglang.launch_server \ + --tp 8 \ + --model deepseek-ai/DeepSeek-V3 \ + --trust-remote-code \ + --attention-backend fa3 +``` + +- Triton +```bash +python3 -m sglang.launch_server \ + --model meta-llama/Meta-Llama-3.1-8B-Instruct \ + --attention-backend triton +python3 -m sglang.launch_server \ + --tp 8 \ + --model deepseek-ai/DeepSeek-V3 \ + --attention-backend triton \ + --trust-remote-code +``` + +- Torch Native +```bash +python3 -m sglang.launch_server \ + --model meta-llama/Meta-Llama-3.1-8B-Instruct \ + --attention-backend torch_native +``` + +- FlashMLA +```bash +python3 -m sglang.launch_server \ + --tp 8 \ + --model deepseek-ai/DeepSeek-R1 \ + --attention-backend flashmla \ + --trust-remote-code +python3 -m sglang.launch_server \ + --tp 8 \ + --model deepseek-ai/DeepSeek-R1 \ + --attention-backend flashmla \ + --kv-cache-dtype fp8_e4m3 \ + --trust-remote-code +``` + +- TRTLLM MLA (Optimized for Blackwell Architecture, e.g., B200) +```bash +python3 -m sglang.launch_server \ + --tp 8 \ + --model deepseek-ai/DeepSeek-R1 \ + --attention-backend trtllm_mla \ + --trust-remote-code +``` + +- TRTLLM MLA with FP8 KV Cache (Higher concurrency, lower memory footprint) +```bash +python3 -m sglang.launch_server \ + --tp 8 \ + --model deepseek-ai/DeepSeek-R1 \ + --attention-backend trtllm_mla \ + --kv-cache-dtype fp8_e4m3 \ + --trust-remote-code +``` + +- Ascend +```bash +python3 -m sglang.launch_server \ + --model meta-llama/Meta-Llama-3.1-8B-Instruct \ + --attention-backend ascend +``` + +- Intel XPU +```bash +python3 -m sglang.launch_server \ + --model meta-llama/Meta-Llama-3.1-8B-Instruct \ + --attention-backend intel_xpu +``` + +- Wave +```bash +python3 -m sglang.launch_server \ + --model meta-llama/Meta-Llama-3.1-8B-Instruct \ + --attention-backend wave +``` + +- FlexAttention +```bash +python3 -m sglang.launch_server \ + --model meta-llama/Meta-Llama-3.1-8B-Instruct \ + --attention-backend flex_attention +``` + +- Dual Chunk FlashAttention +```bash +python3 -m sglang.launch_server \ + --model Qwen/Qwen2.5-14B-Instruct-1M \ + --attention-backend dual_chunk_flash_attn +``` + +- Cutlass MLA +```bash +python3 -m sglang.launch_server \ + --tp 8 \ + --model deepseek-ai/DeepSeek-R1 \ + --attention-backend cutlass_mla \ + --trust-remote-code +``` + +- FlashAttention 4 (MHA & MLA) +```bash +python3 -m sglang.launch_server \ + --tp 8 \ + --model deepseek-ai/DeepSeek-R1 \ + --prefill-attention-backend fa4 \ + --trust-remote-code +``` + +## Steps to add a new attention backend +To add a new attention backend, you can learn from the existing backends +(`python/sglang/srt/layers/attention/triton_backend.py`, `python/sglang/srt/layers/attention/flashattention_backend.py`) +and follow the steps below. + +1. Run without cuda graph. Support the two forward functions + - forward_extend + - Will be used for prefill, prefill with KV cache, and target verification + - It will be called once per layer + - forward_decode + - Will be used for normal decode, and draft decode + - It will be called once per layer + - init_forward_metadata + - Initialize the class and common metadata shared by all layers + - Call the plan function for optimizations like split_kv + - It will be called once per forward +2. Run with cuda graph. It has two phases (capture and replay) and you need to implement three functions + - init_cuda_graph_state + - It will be called once during life time + - Create all common shared buffers + - init_forward_metadata_capture_cuda_graph + - It will be called before capturing a cuda graph + - It is similar to init_forward_metadata but write the medatada to some pre-defined buffers + - init_forward_metadata_replay_cuda_graph + - It will be called before replaying a cuda graph + - This function is in the critical path and needs to be fast diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/checkpoint_engine.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/checkpoint_engine.md new file mode 100644 index 00000000..5e39a7ee --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/checkpoint_engine.md @@ -0,0 +1,254 @@ +# Checkpoint Engine Integration + +The SGLang checkpoint engine integration provides an efficient way to load model weights using a distributed checkpoint loading system. This feature significantly reduces model loading time, especially for large models and multi-node setups, by parallelizing the weight loading process across multiple processes and nodes. + +## Overview + +The checkpoint engine integration allows SGLang to: +- Load model weights in parallel using multiple processes +- Distribute weight loading across multiple nodes to increase effective disk bandwidth +- Overlap weight loading with other initialization tasks like CUDA graph capture +- Support both single-node and multi-node deployments + +## Installation + +First, install the checkpoint engine package: + +```bash +pip install 'checkpoint-engine[p2p]' +``` + +## Architecture + +The system consists of two main components: + +1. **SGLang Server**: Runs with `--wait-for-initial-weights` flag to wait for weights before becoming ready +2. **Checkpoint Engine Workers**: Separate processes (managed by torchrun) that load and distribute model weights + +The checkpoint engine uses a parameter server architecture with support for: +- **Broadcast mode**: Weights are broadcast from loading processes to inference processes +- **P2P mode**: Direct peer-to-peer weight transfer between processes +- **All mode**: Combination of both broadcast and P2P methods + +## Usage Examples + +### Single Node Setup + +**Terminal 1 - Launch SGLang Server:** +```bash +python -m sglang.launch_server \ + --model-path Qwen/Qwen3-8B \ + --tp 8 \ + --load-format dummy \ + --wait-for-initial-weights +``` + +**Terminal 2 - Run Checkpoint Engine:** + +Using sglang entrypoint: +```bash +python -m sglang.srt.checkpoint_engine.update \ + --update-method broadcast \ + --checkpoint-path /path/to/Qwen/Qwen3-8B/ \ + --inference-parallel-size 8 +``` + +Using torchrun directly: +```bash +torchrun --nproc-per-node 8 \ + examples/checkpoint_engine/update.py \ + --update-method broadcast \ + --checkpoint-path /path/to/Qwen/Qwen3-8B/ \ + --inference-parallel-size 8 +``` + +### Multi-Node Setup (2 Nodes) + +**Node 0:** + +Launch SGLang server: +```bash +python -m sglang.launch_server \ + --model-path Qwen/Qwen3-8B \ + --tp 8 \ + --load-format dummy \ + --wait-for-initial-weights \ + --host [IP] +``` + +Run checkpoint engine: + +Using sglang entrypoint (recommended): +```bash +python -m sglang.srt.checkpoint_engine.update \ + --update-method broadcast \ + --checkpoint-path /path/to/Qwen/Qwen3-8B/ \ + --inference-parallel-size 8 +``` + +Using torchrun directly: +```bash +torchrun --nproc-per-node 8 \ + --nnodes 2 \ + --node-rank 0 \ + --master-addr [IP] \ + --master-port 29500 \ + examples/checkpoint_engine/update.py \ + --update-method broadcast \ + --checkpoint-path /path/to/Qwen/Qwen3-8B/ \ + --inference-parallel-size 8 +``` + +**Node 1:** + +Launch SGLang server: +```bash +python -m sglang.launch_server \ + --model-path Qwen/Qwen3-8B \ + --tp 8 \ + --load-format dummy \ + --wait-for-initial-weights \ + --host [IP] +``` + +Run checkpoint engine: + +Using sglang entrypoint (recommended): +```bash +python -m sglang.srt.checkpoint_engine.update \ + --update-method broadcast \ + --checkpoint-path /path/to/Qwen/Qwen3-8B/ \ + --inference-parallel-size 8 +``` + +Using torchrun directly: +```bash +torchrun --nproc-per-node 8 \ + --nnodes 2 \ + --node-rank 1 \ + --master-addr [IP] \ + --master-port 29500 \ + examples/checkpoint_engine/update.py \ + --update-method broadcast \ + --checkpoint-path /path/to/Qwen/Qwen3-8B/ \ + --inference-parallel-size 8 +``` + +### Multi-Node Setup with Tensor Parallelism (TP=16) + +**Node 0:** + +Launch SGLang server: +```bash +python -m sglang.launch_server \ + --model-path Qwen/Qwen3-8B \ + --tp 8 \ + --load-format dummy \ + --wait-for-initial-weights \ + --host [IP] \ + --dist-init-addr [IP]:9120 \ + --nnodes 2 \ + --node-rank 0 +``` + +Run checkpoint engine: + +Using sglang entrypoint (recommended): +```bash +python -m sglang.srt.checkpoint_engine.update \ + --update-method broadcast \ + --checkpoint-path /path/to/Qwen/Qwen3-8B/ \ + --inference-parallel-size 16 +``` + +Using torchrun directly: +```bash +torchrun --nproc-per-node 8 \ + --nnodes 2 \ + --node-rank 0 \ + --master-addr [IP] \ + --master-port 29500 \ + examples/checkpoint_engine/update.py \ + --update-method broadcast \ + --checkpoint-path /path/to/Qwen/Qwen3-8B/ \ + --inference-parallel-size 16 +``` + +**Node 1:** + +Launch SGLang server: +```bash +python -m sglang.launch_server \ + --model-path Qwen/Qwen3-8B \ + --tp 8 \ + --load-format dummy \ + --wait-for-initial-weights \ + --host [IP] \ + --dist-init-addr [IP]:9120 \ + --nnodes 2 \ + --node-rank 1 +``` + +Run checkpoint engine: + +Using sglang entrypoint (recommended): +```bash +python -m sglang.srt.checkpoint_engine.update \ + --update-method broadcast \ + --checkpoint-path /path/to/Qwen/Qwen3-8B/ \ + --inference-parallel-size 16 +``` + +Using torchrun directly: +```bash +torchrun --nproc-per-node 8 \ + --nnodes 2 \ + --node-rank 1 \ + --master-addr [IP] \ + --master-port 29500 \ + examples/checkpoint_engine/update.py \ + --update-method broadcast \ + --checkpoint-path /path/to/Qwen/Qwen3-8B/ \ + --inference-parallel-size 16 +``` + +## Configuration Options + +### SGLang Server Options + +- `--load-format dummy`: Use dummy format for initial loading (allows overlapping with other tasks) +- `--wait-for-initial-weights`: Wait for checkpoint engine to provide weights before becoming ready +- `--host`: Host address for multi-node setups +- `--dist-init-addr`: Distributed initialization address for tensor parallelism + +### Checkpoint Engine Options + +- `--update-method`: Weight update method (`broadcast`, `p2p`, or `all`) +- `--checkpoint-path`: Path to model checkpoint directory +- `--inference-parallel-size`: Number of inference parallel processes +- `--endpoint`: SGLang server endpoint (default: `http://localhost:19730`) +- `--checkpoint-name`: Name for the checkpoint (default: `my-checkpoint-iter-0`) +- `--save-metas-file`: File to save checkpoint metadata +- `--load-metas-file`: File to load checkpoint metadata from +- `--uds`: Unix domain socket path for communication +- `--weight-version`: Version identifier for weights + +## Performance Benefits + +The checkpoint engine provides significant time savings in two main aspects: + +1. **Multi-node Loading**: Each node only loads a portion of weights from disk, effectively increasing disk bandwidth. More participating nodes provide greater acceleration. Preliminary tests show 20-second acceleration when loading DeepSeek-R1 on H20-3e with two nodes. + +2. **Single Process Optimization**: Using dummy format allows overlapping disk-to-CPU transfer with CUDA graph capture and other initialization tasks, providing additional time savings. + +## Troubleshooting + +- Ensure checkpoint engine package is installed: `pip install 'checkpoint-engine[p2p]'` +- Verify network connectivity between nodes in multi-node setups +- Check that the checkpoint path contains valid model files +- Monitor logs for connection errors between SGLang server and checkpoint engine +- Use `--sleep-time` parameter to add delays if needed for debugging + +## References + +- [Checkpoint Engine Repository](https://github.com/MoonshotAI/checkpoint-engine) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/deterministic_inference.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/deterministic_inference.md new file mode 100644 index 00000000..b5b6b521 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/deterministic_inference.md @@ -0,0 +1,154 @@ +# Deterministic Inference + +## Why Deterministic Inference Matters + +Deterministic inference ensures consistent LLM outputs across runs, which is critical for: +- **Reinforcement Learning**: Ensures consistent logprobs across runs, reducing stochastic noise and making RL training more stable, reproducible, and debuggable. +- **Testing & Debugging**: Enables reproducible validation +- **Production**: Improves reliability and user experience + +Even with `temperature=0`, standard LLM inference can produce different outputs due to dynamic batching and varying reduction orders in GPU kernels. + +## The Root Cause of Non-Determinism + +The main source is **varying batch sizes**. Different batch sizes cause GPU kernels to split reduction operations differently, leading to different addition orders. Due to floating-point non-associativity (`(a + b) + c ≠ a + (b + c)`), this produces different results even for identical inputs. + + +## SGLang's Solution + +Building on [Thinking Machines Lab's batch-invariant operators](https://github.com/thinking-machines-lab/batch_invariant_ops), SGLang achieves fully deterministic inference while maintaining compatibility with chunked prefill, CUDA graphs, radix cache, and non-greedy sampling. The development roadmap for deterministic inference features can be found in this [issue](https://github.com/sgl-project/sglang/issues/10278). + +### Supported Backends + +Deterministic inference is only supported with the following three attention backends: **FlashInfer**, **FlashAttention 3 (FA3)**, and **Triton**. + +The following table shows feature compatibility for deterministic inference across different attention backends: + +| Attention Backend | CUDA Graph | Chunked Prefill | Radix Cache | Non-greedy Sampling (Temp > 0) | +|-------------------|------------|-----------------|-------------|---------------------| +| **FlashInfer** | ✅ Yes | ✅ Yes | ❌ No | ✅ Yes | +| **FlashAttention 3 (FA3)** | ✅ Yes | ✅ Yes | ✅ Yes | ✅ Yes | +| **Triton** | ✅ Yes | ✅ Yes | ✅ Yes | ✅ Yes | + +## Usage + +### Basic Usage + +Enable deterministic inference by adding the `--enable-deterministic-inference` flag: + +```bash +python3 -m sglang.launch_server \ + --model-path Qwen/Qwen3-8B \ + --attention-backend fa3 \ + --enable-deterministic-inference +``` + +### Server Arguments + +| Argument | Type/Default | Description | +|----------|--------------|-------------| +| `--enable-deterministic-inference` | flag; default: disabled | Enable deterministic inference with batch-invariant operations | +| `--attention-backend` | string; default: fa3 | Choose attention backend (flashinfer, fa3, or triton) | + +### Example Configurations + +#### Qwen3-8B +```bash +python3 -m sglang.launch_server \ + --model-path Qwen/Qwen3-8B \ + --attention-backend flashinfer \ + --enable-deterministic-inference +``` + +#### Llama Models +```bash +python3 -m sglang.launch_server \ + --model-path meta-llama/Llama-3.1-8B-Instruct \ + --attention-backend fa3 \ + --enable-deterministic-inference +``` + +#### Qwen3-30B-A3B (MoE Model) +```bash +python3 -m sglang.launch_server \ + --model-path Qwen/Qwen3-30B-A3B \ + --attention-backend fa3 \ + --enable-deterministic-inference +``` + +### Deterministic Inference with Non-Greedy Sampling (Temperature > 0) + +SGLang supports deterministic inference even with non-greedy sampling by using sampling seeds. This is particularly useful for reinforcement learning scenarios like GRPO (Group Relative Policy Optimization) where you need multiple diverse but reproducible responses. + +#### Default Behavior + +By default, SGLang uses a sampling seed of `42` for reproducible sampling: + +```python +import requests + +response = requests.post( + "http://localhost:30000/generate", + json={ + "text": "Tell me a joke", + "sampling_params": { + "temperature": 0.8, # Non-greedy sampling + "max_new_tokens": 128, + }, + }, +) +print(response.json()) +# This will always produce the same response across runs +``` + +#### Generating Multiple Reproducible Responses + +To sample different responses from the same prompt while maintaining reproducibility (e.g., for GRPO training), provide different sampling seeds in your requests: + +```python +import requests + +# Prepare a list of sampling seeds for different responses +sampling_seeds = [42, 43, 44, 45, 46] + +responses = [] +for seed in sampling_seeds: + response = requests.post( + "http://localhost:30000/generate", + json={ + "text": "Tell me a joke", + "sampling_params": { + "temperature": 0.8, + "max_new_tokens": 128, + "sampling_seed": seed, # Specify sampling seed + }, + }, + ) + responses.append(response.json()) + +# Each seed will produce a different but reproducible response +# Using the same seed will always produce the same response +``` + +This approach ensures that: +- Different seeds produce diverse responses +- The same seed always produces the same response across different runs +- Results are reproducible for debugging and evaluation + + +## Verification + +Run deterministic tests to verify consistent outputs: + +```bash +# Single test: same prompt, varying batch sizes +python3 -m sglang.test.test_deterministic --test-mode single --n-trials 50 + +# Prefix test: prompts with different prefix lengths +python3 -m sglang.test.test_deterministic --test-mode prefix --n-trials 50 + +# Radix Cache Consistency mode: test radix cache determinism (cached vs uncached prefill) +python3 -m sglang.test.test_deterministic --test-mode radix_cache +``` + +Expected result: All tests should show `Unique samples: 1` (perfectly deterministic). diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/dp_for_multi_modal_encoder.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/dp_for_multi_modal_encoder.md new file mode 100644 index 00000000..ad6132c6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/dp_for_multi_modal_encoder.md @@ -0,0 +1,69 @@ +# DP for Multi-Modal Encoder in SGLang + +A typical VLM architecture involves two main components: an multi-modal encoder and a text decoder. + +Most VLMs utilize a Vision Transformer (ViT) as their multi-modal encoder, it is responsible for processing visual data, extracting features (objects, colors, textures, etc.), and transforming them into a format that can be understood by the model. + +The text deocoder is based on LLM. It processes textual data and generates output based on the encoded visual features. + +However, since the size of ViT is very small compared to language decoders, +there is relatively little gain from TP. On the other hand, TP incurs significant communication +overhead because of all-reduce being performed after every layer. + +Placing the ViT in data parallel while keeping the LLM in tensor parallel consistently lowers TTFT and boosts end-to-end throughput. In this hybrid layout, the vision front-end becomes parallel and lightweight, while scarce interconnect bandwidth and collective ops are reserved for the LLM. + +Data parallelism replicates the entire model across multiple GPU sets and processes different batches of requests in parallel. + +## Pros and Cons for DP Multi-Modal Encoder + +- Unfavorable compute/communication ratio for small ViTs +ViTs used in multimodal stacks are typically modest in parameter count and activation sizes. TP introduces per-layer all-reduce collectives (attention/MLP) whose latency and synchronization overhead outweigh the speedup of splitting relatively small GEMMs. With DP, each GPU runs a full ViT locally—no inference-time collectives—so latency is dominated by compute, not wire time. + +- Graph-capture gaps amplify TP overhead +In production, the vision path often has dynamic shapes (pre/post-processing, variable resolution, patching) that break CUDA Graphs and limit torch.compile fusion. Without capture, we need to pay extra kernel-launch and framework overhead; TP then multiplies that cost with additional NCCL synchronizations. Keeping ViT in DP avoids layering collective latency on top of non-captured kernels. + +- Better interconnect hygiene for the true bottleneck (the LLM) +The LLM’s prefill and decode phases benefit materially from TP on fast links. Offloading ViT to DP eliminates “chatty” small collectives on the same fabric, reducing congestion and jitter for the LLM’s large, bandwidth-hungry all-reduces. + +- Shorter and steadier critical path → lower TTFT +TTFT ≈ T(image encode via ViT) + T(LLM prefill) + T(softmax/sample) +DP has several advantages: +(a) batch and prefetch ViT encodes independently, +(b) overlap them with other requests’ LLM decodes on separate streams, +(c) hand off compact visual embeddings to the TP LLM with minimal queuing. + +- For vision encoders that use hardware-unoptimized Conv3D operations, +batch-level DP can provide another 40% improvement compared to regular TP. + +- Nevertheless, since the weights of the multi-modal encoder are replicated across each TP rank, +there will be a minor increase in memory consumption and may cause OOM if you can barely fit the model already. + +## Command Example +You can enable batch-level DP by setting `mm-enable-dp-encoder`, for example: +``` +SGLANG_MM_FEATURE_CACHE_MB=4096 \ +SGLANG_USE_CUDA_IPC_TRANSPORT=1 \ +SGLANG_VLM_CACHE_SIZE_MB=512 \ +python3 -m sglang.launch_server --host 127.0.0.1 \ + --mem-fraction-static 0.7 \ + --port 30000 \ + --trust-remote-code \ + --dtype auto \ + --max-running-requests 4 \ + --chunked-prefill-size 8192 \ + --attention-backend flashinfer \ + --tp 4 \ + --enable-multimodal \ + --chat-template internvl-2-5 \ + --model OpenGVLab/InternVL2_5-8B \ + --disable-radix-cache \ + --mm-enable-dp-encoder +``` +!!! important + Batch-level multi-modal DP is not to be confused with API request-level DP + (which is instead controlled by `data_parallel_size`). + +## Known supported models +- Qwen2.5-VL () +- Qwen3-VL () +- InternVL () diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/expert_parallelism.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/expert_parallelism.md new file mode 100644 index 00000000..3d9075ff --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/expert_parallelism.md @@ -0,0 +1,141 @@ +# Expert Parallelism + +Expert Parallelism (EP) in SGLang distributes expert weights across multiple devices in Mixture-of-Experts (MoE) models, addressing memory bottlenecks and enabling efficient scaling for high-performance inference. It is particularly vital for serving large-scale MoE models where tokens are dynamically routed to specialized experts across GPUs. By leveraging optimized all-to-all communication and grouped matrix multiplications (GEMMs), EP reduces latency, boosts throughput, and minimizes idle GPU time. SGLang's EP offers strong extensibility through its modular framework, allowing seamless integration of custom kernels, backends, and optimizations without refactoring core logic, supporting diverse hardware and quantization schemes. + +## Supported Backends and Selection Guidance + +SGLang's EP integrates diverse, highly efficient backends for different use cases, allowing fine-grained control over performance trade-offs. Users specify backends via command-line flags: +- `--moe-a2a-backend`: Selects the backend for all-to-all communication. +- `--moe-runner-backend`: Selects the backend for MoE computation. + +### Backends for All-to-All Communication + +| Backend | Description | Use Cases | +|--------------|-----------------------------------------------------------------------------|------------------------------------| +| **`none` (default)** | Disables all-to-all for EP. Uses All-Reduce or All-Gather for token dispatch. | Hybrid EP and TP setups. | +| `deepep` | DeepEP, a communication library for efficient token shuffling in MoE models. | Large-scale EP deployments. | +| `mooncake` | An extension of DeepEP for elastic inference, leveraging RDMA for high-performance data transfers. | Elastic EP serving. | + +DeepEP and Mooncake backends support two modes for token dispatch: `normal` mode (optimized for prefill workloads with high throughput) and `low_latency` mode (optimized for decode workloads with low latency and CUDA Graph compatibility). Users are recommended to set `--deepep-mode auto` to enable automatic dispatch mode switching during runtime. Setting `--deepep-mode normal` or `--deepep-mode low_latency` is useful for debugging or development purposes. + +Currently, DeepEP and Mooncake only support cases where `ep_size = tp_size`. For hybrid EP and TP (i.e., `ep_size < tp_size`), only the `none` backend (All-Reduce or All-Gather-based dispatching) is supported. + +### Backends for MoE Computation + +| Backend | Description | Use Cases | +|--------------------------|-----------------------------------------------------------------------------|------------------------------------| +| **`auto` (default)** | Automatically selects the optimal backend based on model architecture, hardware (e.g., NVIDIA architecture like Ampere, Hopper, Blackwell), quantization scheme (e.g., FP8, FP4), and runtime conditions. | General-purpose deployments; ensures compatibility and performance without user intervention. | +| `triton` | Triton-based implementation for grouped GEMMs. To achieve higher performance, it's highly recommended to create [tuned configurations](https://github.com/sgl-project/sglang/blob/main/benchmark/kernels/fused_moe_triton/README.md). | Custom kernel development or scenarios requiring high extensibility with Torch compilation support. | +| `deep_gemm` | DeepGEMM backend optimized for MoE matrix multiplications, supporting contiguous layouts for prefill and masked layouts for decode; often JIT-compiled for performance. | Large-scale EP deployments with FP8 block-wise quantization. | +| `cutlass` | CUTLASS-based backend for efficient GEMMs. | NVIDIA architectures with CUTLASS support. | +| `flashinfer_trtllm` | FlashInfer integrated with TensorRT-LLM for accelerated MoE computations, supporting FP4 communication operators and high-performance GEMMs. | SM100+ with TRT-LLM. | +| `flashinfer_cutlass` | FlashInfer combined with CUTLASS for high-performance grouped GEMMs in MoE layers, handling FP4/FP8 quantization efficiently. | SM100+ with FP4/FP8 models. | +| `flashinfer_mxfp4` | FlashInfer variant optimized for MXFP4 (mixed FP4) quantization in MoE runners, focusing on memory-efficient low-precision inference. | Low-precision models with MXFP4. | +| `flashinfer_cutedsl` | FlashInfer with a custom DSL for flexible and efficient MoE kernel generation, integrated with ModelOpt FP4 quantization. | Low-precision models with NVFP4. | + +### Examples + +Launch with DeepEP and DeepGEMM for DeepSeek-V3: + +```bash +python -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3 --moe-a2a-backend deepep --moe-runner-backend deep_gemm --tp 8 --ep 8 +``` + +## Extensible EP Framework + +SGLang's EP framework provides modular abstractions for easy integration of custom kernels, backends, and optimizations. It decouples the MoE forward pass into stages (dispatch → pre-permute → core runner → post-permute → combine), enabling seamless extensions without refactoring core logic. + +### Framework Overview + +The framework centers on `FusedMoE` as the unified entry point for a single, extensible structure. Key components include: +- **Dispatcher**: Manages dispatch/combine for backends like DeepEP (implements `BaseDispatcher` subclasses). +- **MoeRunner**: Orchestrates grouped-GEMM execution via `MoeRunnerCore` implementations (e.g., `TritonRunnerCore`). +- **PermuteMethodPool**: Auto-registers layout conversions (e.g., pre/post-permute via `register_pre_permute` and `register_post_permute` for dynamic modes, or `register_fused_func` for static, torch.compile-compatible fused operations). +- **TopK Router**: Backend-agnostic expert selection. + +This design supports multiple backends via `--moe-a2a-backend` and `--moe-runner-backend`, with quantization integrated through a standardized `apply()` method. The computation flow ensures modularity: + +``` +[input_hidden_states] + | + v + TopK.forward -> select_experts / triton_kernels.routing / bypass + | + v + [TopKOutput] + | + v + FusedMoE.forward -> Dispatcher.dispatch -> DeepEP / bypass + | | + | v + | [DispatchOutput] + | | + | v + | quant_method.apply -> MoeRunner.forward + | | | + | | v + | | pre-permute + grouped_gemm + post-permute + | | | + | |-------------- + | v + | [CombineInput] + | | + | v + | Dispatcher.combine -> DeepEP / bypass + | | + |--------------------- + v +[final_hidden_states] +``` + +For details, see the [MoE Refactor Roadmap](https://github.com/sgl-project/sglang/issues/8715). + +### Implementing New Backends + +To add a new backend: +1. For a new all-to-all dispatcher, implement a `BaseDispatcher` subclass with `dispatch` and `combine` methods. +2. For a new MoE runner backend, define a `MoeRunnerCore` subclass for core operations (e.g., grouped GEMMs). +3. Define new input/output formats for the dispatcher or model runner (e.g., `RunnerInput`, `RunnerOutput`). +4. Register permute/unpermute methods to ensure compatibility: + - **Fused Mode** (static, torch.compile-compatible): Use `register_fused_func` for end-to-end operations. + - **Permute Mode** (dynamic): Register `register_pre_permute` and `register_post_permute` for flexible layouts. + +See the [MoE Refactor Implementation PR](https://github.com/sgl-project/sglang/pull/9269) for full changes, including type hints and config expansions. + +### Examples + +For an example implementation, see [moe_runner/triton.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/layers/moe/moe_runner/triton.py), which demonstrates Triton-based grouped GEMMs with registered fused and permutation functions. + +## Computation and Communication Overlap + +SGLang's EP employs advanced overlap techniques to hide communication latency behind computation, maximizing GPU utilization in MoE layers. + +### Two-Batch Overlap (TBO) + +TBO splits requests into micro-batches, interleaving attention computation with dispatch/combine operations. Yield points in the execution graph allow pausing for overlaps, increasing overall throughput without peak memory spikes: + +```python +operations = [ + self._forward_attn, + YieldOperation(), # Overlap with dispatch of prior micro-batch + self._forward_dispatch, + self._forward_mlp, + YieldOperation(), # Overlap with combine + self._forward_combine, +] +``` + +Users need to specify `--enable-two-batch-overlap` to unlock up to 2x throughput. For details, see the [Large-Scale EP Blog](https://lmsys.org/blog/2025-05-05-large-scale-ep/#two-batch-overlap). + +### Single-Batch Overlap (SBO) + +SGLang introduces a dispatcher-hook system for Single-Batch Overlap (SBO), enabling the overlap of operations within a single batch—such as shared experts computation with communication—while decentralizing logic to enhance modularity. These hooks execute before and after the `dispatch` and `combine` operations without modifying core MoE modules. This design simplifies interfaces, reduces coupling, and improves extensibility. For implementation details and an example of overlapping shared experts with DeepEP's combine operation, refer to [PR #13327](https://github.com/sgl-project/sglang/pull/13327). Users can set `--enable-single-batch-overlap` to enable this feature. + + +## Workload Balancer + +SGLang integrates the [Expert Parallelism Load Balancer (EPLB)](https://github.com/deepseek-ai/EPLB) from DeepSeek to address routing imbalances in MoE models. By analyzing expert activation statistics, EPLB computes an optimal expert arrangement, strategically placing or replicating experts to minimize GPU utilization variance, reduce idle cycles, and enhance scalability. + +To enable EPLB, use the flags `--enable-eplb true --load-balance-method eplb`. For optimal performance, increase batch sizes to stabilize activation statistics and configure periodic rebalancing (e.g., every 1000 requests) to adapt to evolving workloads. Simulations demonstrate significant improvements in load balancedness (ratio of mean to max computation time), correlating strongly with throughput gains. + +For more details, refer to the [EPLB Section in the Large-Scale EP Blog](https://lmsys.org/blog/2025-05-05-large-scale-ep/#expert-parallelism-load-balancer) and the [EPLB Repository](https://github.com/deepseek-ai/eplb). diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/forward_hooks.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/forward_hooks.md new file mode 100644 index 00000000..4dc170b3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/forward_hooks.md @@ -0,0 +1,297 @@ +## Model Hooks + +SGLang supports attaching PyTorch forward hooks to specific submodules in the loaded model, configured entirely via `server_args` JSON. + +This is useful for: + +* Logging intermediate activations +* Debugging model internals +* Exporting hidden states to external tooling + +Hooks are attached once during `ModelRunner.initialize` and run on every forward pass. + +--- + +### Configuration overview + +Hooks are configured via a `ServerArgs` field: + +```python +class ServerArgs: + ... + # For forward hooks + forward_hooks: Optional[List[dict[str, Any]]] = None +```` + +In JSON form, a minimal configuration looks like: + +```jsonc +{ + "forward_hooks": [ + { + "name": "outer_linear_hooks", + "target_modules": ["outer.0", "outer.1"], + "hook_factory": "my_project.hooks:dummy_hook_factory", + "config": { + "tag": "outer-layer" + } + } + ] +} +``` + +#### Top-level fields + +* `forward_hooks` (optional list of objects) + Each element is a hook spec describing: + + * Which modules to target + * Which Python factory to call + * What configuration to pass into that factory + +--- + +### Hook spec schema + +Each entry in `forward_hooks` is a JSON object with the following shape: + +```jsonc +{ + "name": "optional-descriptive-name", + "target_modules": ["pattern1", "pattern2", "..."], + "hook_factory": "module.submodule:factory_name", + "config": { + "...": "arbitrary JSON" + } +} +``` + +#### `name` (optional) + +* Human-readable name for logging. +* Used only in log messages such as: + + ```text + Registered forward hook 'outer_linear_hooks' on outer.0 + ``` + +#### `target_modules` (required) + +* List of **module name patterns** used to match entries in `model.named_modules()`. +* Patterns are matched using `fnmatch.fnmatch`, so: + + * `"outer.0"` matches exactly `"outer.0"`. + * `"outer.*"` matches `"outer.0"`, `"outer.1"`, `"outer.inner"`, etc. + * `"outer.inner.*"` matches children under `outer.inner`. + +> If no modules match the given patterns, hook registration does **not** fail. +> Instead, SGLang logs a warning and continues: +> +> ```text +> No modules matched hook spec 'name' patterns=['...'] +> ``` + +#### `hook_factory` (required) + +* String path to the Python factory function that creates the hook. +* Supported formats: + + * `"package.module:factory_name"` + * `"package.module.submodule.factory_name"` + +The path is resolved via: + +```python +def resolve_callable(path: Optional[str]) -> Optional[Callable]: + if path is None: + return None + + if ":" in path: + module_name, fn_name = path.split(":", 1) + else: + parts = path.split(".") + if len(parts) < 2: + raise ValueError( + f"Invalid hook callable path '{path}'. " + "Expected 'module.submodule:factory' or 'module.submodule.factory'." + ) + *mod_parts, fn_name = parts + module_name = ".".join(mod_parts) + + module = importlib.import_module(module_name) + try: + return getattr(module, fn_name) + except AttributeError as e: + raise AttributeError( + f"Module '{module_name}' has no attribute '{fn_name}' " + f"(from hook path '{path}')" + ) from e +``` + +**Failure modes**: + +* If the path is malformed (not enough dots and no `:`), a `ValueError` is raised at startup. +* If the module imports but the attribute is missing, an `AttributeError` is raised with a clear error message. +* If the hook factory returns `None`, a warning is logged and no hook is registered for that spec (initialization continues). + +The first two cause initialization to fail fast with a descriptive error; the last one is non-fatal. + +#### `config` (optional) + +* Arbitrary JSON object. +* Passed directly to the hook factory as a Python `dict`. +* This lets you parameterize hook behavior from config (e.g. tags, log levels, sampling rates, etc.). + +--- + +### Hook lifecycle and behavior + +Hooks are registered in `ModelRunner.initialize()`: + +```python +if server_args.forward_hooks: + register_forward_hooks(self.model, server_args.forward_hooks) +``` + +The actual registration logic is implemented by `register_forward_hooks`: + +```python +def register_forward_hooks(model: nn.Module, hook_specs: List[dict[str, Any]]) -> None: + """ + hook_specs is a list of dicts from server_args.forward_hooks. + Attaches forward hooks to the matching modules. + """ + name_to_module = dict(model.named_modules()) + + for spec in hook_specs: + spec_name = spec.get("name", "") + target_patterns = spec.get("target_modules", []) + if not target_patterns: + logger.warning( + f"Hook spec '{spec_name}' has no 'target_modules', skipping" + ) + continue + + hook_factory_path = spec.get("hook_factory") + if not hook_factory_path: + logger.warning( + f"Hook spec '{spec_name}' has no 'hook_factory', skipping" + ) + continue + + config = spec.get("config") or {} + hook_factory = resolve_callable(hook_factory_path) + + hook = hook_factory(config) if hook_factory else None + if hook is None: + logger.warning( + f"Hook factory '{hook_factory_path}' for spec '{spec_name}' " + "returned None, not registering any hook" + ) + continue + + # Resolve patterns like "model.layers.*.mlp" + matched = [] + for name, module in name_to_module.items(): + if any(fnmatch.fnmatch(name, pattern) for pattern in target_patterns): + matched.append((name, module)) + + if not matched: + logger.warning( + f"No modules matched hook spec '{spec_name}' " + f"patterns={target_patterns}" + ) + continue + + for module_name, module in matched: + if hook: + _ = module.register_forward_hook(hook) + logger.info( + f"Registered forward hook '{spec_name}' " + f"on {module_name}" + ) +``` + +Key points: + +* Hooks are **forward hooks only** (via `module.register_forward_hook`). +* They are attached once at initialization. +* Hook handles are currently not stored on `ModelRunner` (they cannot be removed later via this API). +* Failure to match any modules is non-fatal; a warning is logged instead. +* If a hook factory returns `None`, a warning is logged and that spec is skipped. + +--- + +### Writing a hook factory + +A hook factory is a regular Python function: + +* Takes a `config: dict` (from JSON) +* Returns a forward hook function with signature `(module, inputs, output)` + +Example: + +```python +HOOK_CALLS = [] + +def dummy_hook_factory(config): + """Factory that returns a forward hook capturing a tag from config.""" + tag = config.get("tag", "default") + + def hook(module, inputs, output): + HOOK_CALLS.append( + { + "module_type": type(module).__name__, + "tag": tag, + "shape": tuple(output.shape), + } + ) + return output # must return output if you don’t want to modify the tensor + + return hook +``` + +In JSON: + +```jsonc +{ + "forward_hooks": [ + { + "name": "capture_outer", + "target_modules": ["outer.0", "outer.1"], + "hook_factory": "my_project.hooks:dummy_hook_factory", + "config": { + "tag": "outer" + } + } + ] +} +``` + +This will: + +* Resolve `my_project.hooks:dummy_hook_factory` to a Python callable. +* Call it with `config = {"tag": "outer"}`. +* Use the returned hook for all modules matching `outer.0` and `outer.1`. +* Append metadata about each call to `HOOK_CALLS`. + +--- + +### Summary + +* Define `forward_hooks` as a list of specs in `ServerArgs` to turn on the feature. + +* Each spec: + + * selects modules via `target_modules` (glob patterns over `model.named_modules()`), + * points to a hook factory via `hook_factory`, + * passes arbitrary `config` into that factory. + +* Hook factories are resolved via `resolve_callable`, which supports `module:factory` and `module.submodule.factory`. + +* Hooks are standard PyTorch forward hooks, attached once at startup and invoked on every forward pass. + +* Misconfiguration is either: + + * **fatal and explicit** (bad path / missing attribute), or + * **non-fatal with clear warnings** (no targets matched, or factory returned `None`). diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/hicache.rst b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/hicache.rst new file mode 100644 index 00000000..b2bd08b7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/hicache.rst @@ -0,0 +1,8 @@ +Hierarchical KV Caching (HiCache) +================================= + +.. toctree:: + :maxdepth: 1 + + hicache_best_practices.md + hicache_design.md diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/hicache_best_practices.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/hicache_best_practices.md new file mode 100644 index 00000000..cb1baa01 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/hicache_best_practices.md @@ -0,0 +1,196 @@ +# SGLang HiCache Best Practices + +## Why HiCache Matters + +SGLang HiCache extends the traditional RadixAttention with a three-tier hierarchical KV caching system that dramatically improves performance for long-context and multi-turn conversation scenarios. By intelligently managing KV caches across GPU memory, host memory, and external storage backends, HiCache addresses the fundamental capacity bottleneck that limits cache hit rates in conventional systems. + +## Configuration Guidelines + +## Core HiCache Parameters + +```bash +# Essential HiCache flags +--page-size 64 # Page size for cache management +--enable-hierarchical-cache # Enable HiCache +--hicache-ratio 2 # Host memory ratio (2x GPU memory) +--hicache-size 100 # Host memory size in GBs, will override the above ratio +--hicache-io-backend kernel # The I/O backend of moving data between CPU and GPU +--hicache-write-policy write_through # Cache write policy from GPU to CPU +--hicache-storage-backend # Optional storage backend (e.g., hf3fs, mooncake, etc.) +``` + +## Key Configurations with Storage Backends Enabled + +### Memory Layout Optimization + +```bash +# Page-first: Optimized for I/O efficiency with zero-copy (recommended with kernel backend) +--hicache-mem-layout page_first +# Page-first-direct: Optimized for direct I/O operations (Compatible with fa3 and same zero-copy performance as page_first) +--hicache-mem-layout page_first_direct +# Layer-first +--hicache-mem-layout layer_first +``` +**Layout Compatibility:** +- `page_first`: Only compatible with `kernel` I/O backend, automatically switches to `layer_first` with `direct` backend +- `page_first_direct`: Specifically designed for `direct` I/O backend with optimized memory organization + +### Prefetch Policies + +```bash +# Best-effort: Terminate prefetch when needed +--hicache-storage-prefetch-policy best_effort +# Wait-complete: Ensure complete prefetch, higher cache reuse +--hicache-storage-prefetch-policy wait_complete +# Timeout: Balance between completion and best-effort +--hicache-storage-prefetch-policy timeout +``` + +### Integration with PD Disaggregation + +HiCache works seamlessly with PD Disaggregation. You can choose between two configurations: + +1. **Prefill-only HiCache**: Enable HiCache only on Prefill nodes, allowing KV cache sharing among Prefill instances +2. **Full HiCache with async offloading**: Enable HiCache on Prefill nodes and async KV cache offloading on Decode nodes, allowing Prefill nodes to reuse KV caches from Decode nodes in multi-turn dialogue scenarios + +```bash +# Prefill node with HiCache enabled for cross-prefill sharing (ideal for SystemPrompt scenarios) +python3 -m sglang.launch_server \ + --model-path /xxx/DeepSeek-R1/ \ + --tp 8 \ + --host 0.0.0.0 \ + --port 10000 \ + --enable-metrics \ + --enable-cache-report \ + --mem-fraction-static 0.85 \ + --page-size 64 \ + --enable-hierarchical-cache \ + --hicache-ratio 2 \ + --hicache-size 0 \ + --hicache-mem-layout page_first_direct \ + --hicache-io-backend direct \ + --hicache-write-policy write_through \ + --hicache-storage-backend hf3fs \ + --hicache-storage-prefetch-policy wait_complete \ + --disaggregation-ib-device mlx5_0 \ + --disaggregation-mode prefill \ + --disaggregation-transfer-backend mooncake + +# Decode node with async offloading enabled for KV cache reuse by Prefill (ideal for multi-turn conversations) +python3 -m sglang.launch_server \ + --model-path /xxx/DeepSeek-R1/ \ + --tp 8 \ + --host 0.0.0.0 \ + --port 10000 \ + --enable-metrics \ + --enable-cache-report \ + --page-size 64 \ + --hicache-ratio 2 \ + --hicache-size 0 \ + --hicache-mem-layout page_first_direct \ + --hicache-io-backend direct \ + --hicache-write-policy write_through \ + --hicache-storage-backend hf3fs \ + --hicache-storage-prefetch-policy wait_complete \ + --disaggregation-decode-enable-offload-kvcache \ # Enable async KV cache offloading in decode node + --disaggregation-ib-device mlx5_0 \ + --disaggregation-mode decode \ + --disaggregation-transfer-backend mooncake +``` + + +### Deployment with HF3FS + +Here is an example of deploying DeepSeek-R1 with HiCache-HF3FS. For more details, see the [HF3FS Documentation](../../python/sglang/srt/mem_cache/storage/hf3fs/docs/README.md). + +```bash +python3 -m sglang.launch_server \ + --model-path /xxx/DeepSeek-R1/ \ + --log-level info \ + --tp 8 \ + --host 0.0.0.0 \ + --port 10000 \ + --enable-metrics \ + --enable-cache-report \ + --page-size 64 \ + --mem-fraction-static 0.85 \ + --enable-hierarchical-cache \ + --hicache-ratio 2 \ + --hicache-size 0 \ + --hicache-mem-layout page_first_direct \ + --hicache-io-backend direct \ + --hicache-write-policy write_through \ + --hicache-storage-backend hf3fs \ + --hicache-storage-prefetch-policy wait_complete \ +``` + +### Deployment with Mooncake + +Here is an example of deploying Qwen3-235B-A22B-Instruct-2507 with Mooncake. For more details, see the [Mooncake Documentation](../../python/sglang/srt/mem_cache/storage/mooncake_store/README.md). + +```bash +# Set Mooncake environment variables +export MOONCAKE_TE_META_DATA_SERVER="http://127.0.0.1:8080/metadata" +export MOONCAKE_GLOBAL_SEGMENT_SIZE=816043786240 +export MOONCAKE_PROTOCOL="rdma" +export MOONCAKE_DEVICE="$DEVICE_LIST" +export MOONCAKE_MASTER=127.0.0.1:50051 + +# Launch SGLang server with Mooncake backend +python3 -m sglang.launch_server \ + --model-path $MODEL_PATH \ + --tp 8 \ + --page-size 64 \ + --enable-hierarchical-cache \ + --hicache-ratio 2 \ + --hicache-mem-layout page_first_direct \ + --hicache-io-backend direct \ + --hicache-storage-backend mooncake \ + --hicache-write-policy write_through \ + --hicache-storage-prefetch-policy timeout +``` + + +## Custom Storage Backend Integration + +To integrate a new storage backend: + +1. **Implement three core methods:** + - `get(key)`: Retrieve value by key + - `exists(key)`: Check key existence + - `set(key, value)`: Store key-value pair + +2. **Register your backend:** Add your storage backend to the HiCache [BackendFactory](../../python/sglang/srt/mem_cache/storage/backend_factory.py#L188) + +The HiCache controller handles all scheduling and synchronization automatically. + +### Dynamic Backend Loading + +Alternatively, you can use dynamic loading to avoid hard-coding your backend in the repository: + +```bash +python3 -m sglang.launch_server \ + --model-path your-model \ + --enable-hierarchical-cache \ + --hicache-storage-backend dynamic \ + --hicache-storage-backend-extra-config '{"backend_name":"custom_backend_name", "module_path": "your_module_path", "class_name": "YourHiCacheClassName"}' +``` + +**Configuration Parameters:** +- `--hicache-storage-backend`: Set to `dynamic` +- `--hicache-storage-backend-extra-config`: JSON configuration with: + - `backend_name`: Custom backend identifier + - `module_path`: Python module path to your implementation + - `class_name`: Your HiCache implementation class name + - `interface_v1`: 0 (disable) or 1 (enable) to control usage of batch_get_v1 and batch_set_v1 methods + + +## Community and Support + +- **GitHub Issues**: Report bugs and feature requests +- **Slack Channel**: Join community discussions in #sgl-kv-cache-store +- **Documentation**: Refer to storage backend-specific guides + +--- + +*This document will be continuously updated based on community feedback and new features. Contributions and suggestions are welcome!* diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/hicache_design.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/hicache_design.md new file mode 100644 index 00000000..b775cef5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/hicache_design.md @@ -0,0 +1,155 @@ +# HiCache System Design and Optimization + +This document provides a comprehensive overview of SGLang HiCache, covering its system architecture, workflow and key components. It also details configuration parameters, optimization techniques, and integration with various L3 storage backends, serving as a complete reference for users and developers to understand and tune HiCache for efficient LLM inference. + +## Why and What is HiCache? + +In large language model inference, the prefill phase is often time-consuming: input sequences need to be first converted into Key-Value cache (KV cache) for subsequent decoding. When multiple requests share the same prefix, the KV cache for that prefix is identical. By caching and reusing these shared KV caches, redundant computation can be avoided. To address this, SGLang introduced RadixAttention, which leverages idle GPU memory to cache and reuse prefix KV caches, and **HiCache**, which extends this idea to host memory and distributed storage. + +Inspired by the classic three-level cache design of modern CPUs, HiCache organizes GPU memory as L1, host memory as L2, and distributed storage as L3. This hierarchy enables HiCache to fully exploit the "idle" storage space of GPUs and CPUs, while integrating distributed cache systems such as Mooncake, 3FS, NIXL, and AIBrix KVCache for global KV cache storage and scheduling. As a result, HiCache significantly expands KV cache capacity while maintaining strong read performance—especially in workloads such as multi-QA and long-context inference, where KV cache reuse is frequent. For detailed benchmark results, see [this blog](https://lmsys.org/blog/2025-09-10-sglang-hicache/). + + +## System Design + +### Overall Architecture + +In many modern CPU architectures, the small but fast L1 and L2 caches are private to each core, enabling rapid access to the hottest data, while the larger L3 cache is shared across all cores to significantly reduce redundancy within the cache. Similarly, in HiCache, the L1 and L2 KV caches are private to each inference instance, whereas the L3 KV cache is shared among all inference instances within the cluster. + +### HiRadixTree: Metadata Organization in HiCache + +For KV cache data organization, HiCache builds upon the RadixTree structure introduced in RadixAttention and proposes HiRadixTree. In RadixAttention, each node of the RadixTree corresponds to the KV cache of a consecutive span of tokens in GPU memory. A path from the root to a leaf node represents the prefix of a request, and shared prefixes across multiple requests can reuse the same nodes, thereby avoiding redundant storage. + +HiRadixTree extends this idea: each node corresponds to the KV cache of a span of consecutive tokens and records where that KV cache is stored—whether in local GPU memory, CPU memory, L3 storage, or multiple of these tiers. If stored locally, HiRadixTree maintains precise metadata, including the exact storage address. However, to reduce overhead, HiRadixTree does not store or continuously synchronize metadata for L3 KV cache. Instead, when accessing L3 data, it queries the backend in real time to retrieve the necessary metadata, such as whether the data exists and on which server and location it resides. + +### Overall Workflow + +The workflow of HiCache mainly involves three key operations: **local match**, **prefetch** and **write-back**. When the system receives a new request, it first searches the local L1 and L2 caches for matching KV caches. For parts not found locally, it attempts to prefetch from L3. After prefetching, all required KV caches are loaded into the GPU for computation. Once the prefill computation is complete, the system considers storing the newly generated data into L2 or L3. + +![HiCache Workflow](https://lmsys.org/images/blog/hicache/hicache_overview.png) + +### Local Match + +Local matching is the first step in HiCache's workflow, where incoming request tokens are matched against the HiRadixTree to locate cached KV data in local memory tiers (L1 GPU memory and L2 host memory). + +The matching algorithm traverses the HiRadixTree from the root node, following child nodes that match the token sequence prefix. At each node, the incoming token sequence is compared with the node’s stored token sequence. When `page_size > 1`, matching is performed at the page granularity to optimize memory access patterns. If a match terminates within a node’s stored sequence, the node is automatically split to create an exact boundary, improving the efficiency of future matches. + +The algorithm returns a continuous prefix of the request, with the first part residing in L1 and the latter part in L2. + +Since the process only requires traversing the local HiRadixTree and does not involve any actual data copying, local matching is extremely fast. + +### Prefetch from L3 + +Data prefetching is one of HiCache’s core optimization techniques, designed to proactively load KV caches from L3 storage into local L2 memory, thereby reducing access latency during subsequent operations. + +**Prefetch Trigger Conditions**: +After local matching, for the parts not found in L1 or L2, the system queries L3 to retrieve metadata for the next continuous matching KV caches. If the length of hit cache in L3 exceeds a threshold (default: 256 tokens, configurable), a prefetch operation is triggered. + +**Prefetch Strategies**: HiCache provides three different prefetch termination strategies to address different scenario needs: +- **best_effort**: Terminates immediately when GPU can execute prefill computation, with no waiting time, suitable for scenarios extremely sensitive to latency. +- **wait_complete**: Must wait for all prefetch operations to complete, suitable for scenarios requiring high cache hit rates. +- **timeout**: Terminates after specified time or when complete, balancing latency and cache hit rate needs. + +After prefetching stops, the data already fetched is used together with the local data for the prefill computation. + +For **timeout** strategy, HiCache introduces two configuration parameters to support fine-grained control over prefetch timeout conditions: + +* `prefetch_timeout_base`: the base timeout, representing overhead unrelated to the number of tokens (e.g., scheduling and synchronization). +* `prefetch_timeout_per_ki_token`: the incremental timeout per thousand tokens. + +The timeout is computed as: + +``` +timeout = prefetch_timeout_base + prefetch_timeout_per_ki_token * num_token_to_fetch / 1024 +``` + +### Data Write-back + +The write-back mechanism is responsible for moving frequently accessed KV caches from L1 to L2 and L3, enabling larger and longer-term storage as well as cache sharing across instances. + +**Configurable Write-back Policies**: HiCache supports three write-back strategies: + +* **write_through**: Every access is immediately written back to the next level. When bandwidth is sufficient, this strategy provides the strongest caching benefit. +* **write_through_selective**: Data is written back only after the access frequency exceeds a threshold. This strategy backs up only hot data, reducing I/O overhead. +* **write_back**: Data is written back to the next level only when it is evicted from the upper level. This strategy alleviates storage pressure and is suitable for scenarios where storage capacity is limited but memory utilization must be maximized. + +**Cross-instance Sharing**: When data is written back from L2 to L3, only data not already present in L3 is transferred. KV caches stored in L3 can then be shared across all SGLang instances in the cluster (depending on the L3 backend implementation), significantly improving cache hit rates within the same memory budget. + +### Multi-Rank Synchronization + +During multi-GPU parallel computation, such as tensor parallelism (TP), HiCache must ensure consistent states across different ranks. Therefore, critical computation steps require the use of `all_reduce` for state synchronization. + +For example, during prefetching, `all_reduce(op=min)` is used to ensure that all ranks obtain the same number of L3 hits, preventing inconsistent judgments about whether the prefetch threshold has been reached. Similarly, after prefetching completes or terminates, `all_reduce(op=min)` is again required to guarantee consensus among ranks on the prefix length of the successfully retrieved KV cache. + +### Data Transfer Optimization + +**Zero-Copy Data Transfers**: Both prefetching and write-back involve substantial data movement. Minimizing the number of data copies can significantly improve system performance. HiCache supports passing memory addresses and sizes directly when transferring data from L2 memory to an L3 backend. + +**“Batch-Oriented” Data Organization**: The granularity of data reads and writes has a major impact on performance. To address this, HiCache L3 stores and transfers KV cache data at the granularity of **pages** and supports different data layouts beyond the existing `layer first` scheme, including `page first` and `page first direct`. Under the `page first` and `page first direct` layouts, all KV cache data belonging to the same page is placed in contiguous memory, allowing it to be passed as a single object to L3 using zero-copy transfers. + +![HiCache L2 MEM layout](https://lmsys.org/images/blog/hicache/hicache_layout.png) + +However, because GPU KV computation is naturally performed layer by layer, the GPU inherently operates in a `layer first` layout. When transferring `page first` data from L2 to the GPU, data must be transferred at the granularity of one token per layer. The `page first direct` layout mitigates this issue by grouping together all tokens of a given layer within a page, allowing transfers from L2 to GPU to be aggregated at the page-layer level. + +**CPU-to-GPU Transfer Optimizations**: In HiCache, moving data from CPU memory to GPU is as performance-critical as prefetching data from L3 to L2. HiCache employs several optimizations for this process: + +* **Compute-Transfer Overlap**: During the prefill phase, when transferring data from CPU to GPU, HiCache overlaps layers by concurrently loading the KV cache of layer N+1 while computing layer N. This effectively hides data transfer latency. +* **GPU-assisted I/O Kernels**: On top of `cudaMemcpyAsync`, HiCache implements a set of GPU-assisted I/O kernels specifically optimized for KV cache transfers between CPU and GPU. Compared to the baseline approach, these kernels achieve up to 3x higher transfer speed. + +**Write-back Optimization for MLA**: For MHA (Multi-Head Attention) models under multi-TP, each rank holds `1/tp_size` of a token’s KV data. In contrast, for MLA (Multi-Layer Attention) models, all ranks hold the complete and identical KV data for each token. HiCache includes a dedicated optimization for MLA: only one rank initiates the write-back operation, ensuring that data is not redundantly stored across ranks. + +### Integration with PD-Disaggregation Deployment Mode + +SGLang supports a PD (Prefill-Decode) disaggregation deployment mode through the Mooncake TransferEngine (for details, see [this doc](https://docs.sglang.io/advanced_features/pd_disaggregation.html)). In the PD-disaggregation deployment mode, HiCache can be enabled on both the prefill nodes and decode nodes to optimize prefill performance. If enabled on decode nodes, the decode output will also be written back to L3. + +### Unified Interfaces and Rich L3 Storage Backends + +HiCache encapsulates all read, write, and query operations on L3 backends within the `class HiCacheStorage(ABC)`, exposing a set of simple and consistent interfaces. This design supports a wide range of L3 storage backends and allows users to select the one that best fits their specific use cases. + +- **Mooncake**: Mooncake is a high-performance caching system for LLM inference that leverages RDMA and multi-NIC resources to enable zero-copy, ultra-fast data transfers. Try Mooncake [here](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/mem_cache/storage/mooncake_store). + +- **DeepSeek 3FS (HF3FS)**: HF3FS is a Kubernetes-native distributed storage solution with operator-based deployment. Try HF3FS [here](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/mem_cache/storage/hf3fs). + +- **NIXL**: NIXL provides a unified API for accessing various storage plugins, including but not limited to DeepSeek's 3FS, GPU Direct Storage (GDS) and Amazon S3-compatible object storage. Try NIXL [here](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/mem_cache/storage/nixl). + +- **AIBrix KVCache**: AIBrix KVCache is a production-ready KVCache Offloading Framework, which enables efficient memory tiering and low-overhead cross-engine reuse. Try AIBrix KVCache [here](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/mem_cache/storage/aibrix_kvcache). + +- **HiCacheFile**: A simple file-based storage backend for demonstration purposes. + +Specifically, **LMCache**, an efficient KV cache layer for enterprise-scale LLM inference, provides an alternative solution to HiCache. Try LMCache [here](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/mem_cache/storage/lmcache). + +## Related Parameters + +- **`--enable-hierarchical-cache`**: Enable hierarchical cache functionality. This is required to use HiCache. + +- **`--hicache-ratio HICACHE_RATIO`**: The ratio of the size of host KV cache memory pool to the size of device pool. For example, a value of 2 means the host memory pool is twice as large as the device memory pool. The value of this parameter must be greater than 1, as the current implementation requires the host memory allocated for the KV cache to be larger than the device memory allocated for the KV cache. + +- **`--hicache-size HICACHE_SIZE`**: The size of host KV cache memory pool in gigabytes. This parameter overrides `hicache-ratio` if set. For example, `--hicache-size 30` allocates 30GB (1GB = 1e9 bytes) for the host memory pool **for each rank**. If there are 8 ranks, then the total memory size is 240GB. Just like `hicache-ratio`, the value of this parameter must be larger than the size of device memory allocated for KV cache. + +**Note**: `--hicache-ratio` and `--hicache-size` are two critical parameters. In general, a larger HiCache size leads to a higher cache hit rate, which improves prefill performance. However, the relationship between cache size and hit rate is not linear. Once most reusable KV data—especially hot tokens—are already cached, further increasing the size may yield only marginal performance gains. Users can set these parameters based on their workload characteristics and performance requirements. + +- **`--page-size PAGE_SIZE`**: The number of tokens per page. This parameter determines the granularity of KV cache storage and retrieval. Larger page sizes reduce metadata overhead and improve I/O efficiency for storage backends, but may lower the cache hit rate when only part of a page matches the stored KV cache. For workloads with long common prefixes, larger pages can improve performance, while workloads with more diverse prefixes may benefit from smaller pages. See [Data Transfer Optimization](#data-transfer-optimization) for how page granularity affects I/O performance. + +- **`--hicache-storage-prefetch-policy {best_effort,wait_complete,timeout}`**: Controls when prefetching from storage should stop. See [Prefetch from L3](#prefetch-from-l3) for details. + - `best_effort`: Prefetch as much as possible without blocking + - `wait_complete`: Wait for prefetch to complete before proceeding + - `timeout`: Terminates after specified time or when complete (Recommended for production environments, as setting an appropriate timeout helps the system meet required SLOs) + +- **`--hicache-write-policy {write_back,write_through,write_through_selective}`**: Controls how data is written from faster to slower memory tiers. See [Data Write-back](#data-write-back) for details. + - `write_through`: Immediately writes data to all tiers (strongest caching benefits) + - `write_through_selective`: Uses hit-count tracking to back up only frequently accessed data + - `write_back`: Writes data back to slower tiers only when eviction is needed (reduces I/O load) + +- **`--hicache-io-backend {direct,kernel}`**: Choose the I/O backend for KV cache transfer between CPU and GPU. See [Data Transfer Optimization](#data-transfer-optimization) for details. + - `direct`: Standard CUDA memory copy operations + - `kernel`: GPU-assisted I/O kernels (recommended for better performance) + +- **`--hicache-mem-layout {layer_first,page_first,page_first_direct}`**: Memory layout for the host memory pool. See [Data Transfer Optimization](#data-transfer-optimization) for details. + - `layer_first`: Compatible with GPU computation kernels (default for GPU memory) + - `page_first`: Optimized for I/O efficiency + - `page_first_direct`: Groups all tokens of a given layer within a page, allowing transfers from L2 to GPU to be aggregated at the page-layer level + +- **`--hicache-storage-backend {file,mooncake,hf3fs,nixl,aibrix,dynamic}`**: Choose the storage backend for the L3 tier. Built-in backends: file, mooncake, hf3fs, nixl, aibrix. For dynamic backend, use --hicache-storage-backend-extra-config to specify: `backend_name` (custom name), `module_path` (Python module path), `class_name` (backend class name). See [Unified Interfaces and Rich L3 Storage Backends](#unified-interfaces-and-rich-l3-storage-backends) for available backends. + +- **`--enable-lmcache`**: Using LMCache as an alternative hierarchical cache solution. + +- **`--hicache-storage-backend-extra-config HICACHE_STORAGE_BACKEND_EXTRA_CONFIG`**: JSON string containing extra configuration for the storage backend, e.g., `--hicache-storage-backend-extra-config '{"prefetch_threshold":512, "prefetch_timeout_base": 0.5, "prefetch_timeout_per_ki_token": 0.25}' ` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/hyperparameter_tuning.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/hyperparameter_tuning.md new file mode 100644 index 00000000..d9461e19 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/hyperparameter_tuning.md @@ -0,0 +1,77 @@ +# Hyperparameter Tuning + +## Achieving high throughput for offline batch inference + +Achieving a large batch size is the most important thing for attaining high throughput in offline batch inference. +When the server is running at full load in a steady state, look for the following in the log: + +```Decode batch. #running-req: 233, #token: 370959, token usage: 0.82, cuda graph: True, gen throughput (token/s): 4594.01, #queue-req: 317``` + +### Adjust the request submission speed to control `#queue-req` + +`#queue-req` indicates the number of requests in the queue. +If you frequently see `#queue-req: 0`, it suggests that your client code is submitting requests too slowly. +A healthy range for `#queue-req` is `100 - 2000`. +However, avoid making `#queue-req` too large, as this will increase the scheduling overhead on the server. + +### Achieve a high `token usage` + +`token usage` indicates the KV cache memory utilization of the server. `token usage > 0.9` means good utilization. + +If you frequently see `token usage < 0.9` and `#queue-req > 0`, it means the server is too conservative about taking in new requests. You can decrease `--schedule-conservativeness` to a value like 0.3. +The case of a server being too conservative can happen when users send many requests with a large `max_new_tokens` but the requests stop very early due to EOS or stop strings. + +On the other hand, if you see `token usage` very high and you frequently see warnings like +`KV cache pool is full. Retract requests. #retracted_reqs: 1, #new_token_ratio: 0.9998 -> 1.0000`, you can increase `--schedule-conservativeness` to a value like 1.3. +If you see `KV cache pool is full. Retract requests.` occasionally but not frequently (~1 time per minute), it is okay. + +### Tune `--mem-fraction-static` to increase KV cache pool capacity +SGLang allocates memory as follows: + +Total memory usage = model weights + KV cache pool + CUDA graph buffers + activations + +The `--mem-fraction-static` parameter determines how much memory is allocated to the first two components: + +mem_fraction_static = (model weights + KV cache pool) / GPU memory capacity + +To support higher concurrency, you should maximize the KV cache pool capacity by setting `--mem-fraction-static` as high as possible while still reserving enough memory for activations and CUDA graph buffers. + +SGLang uses simple heuristics to set the default value of `--mem-fraction-static`, but you can optimize it for your use cases. +As a rule of thumb, reserving 5–8 GB of memory for activations is typically sufficient. You can check this by inspecting the logs just before the server is ready. +Look for log entries like this: + +``` +[2025-08-11 17:17:03] max_total_num_tokens=665690, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=4096, context_len=65536, available_gpu_mem=13.50 GB +``` + +Check the `available_gpu_mem` value. +- If it is between 5–8 GB, the setting is good. +- If it is too high (e.g., 10 - 20 GB), increase `--mem-fraction-static` to allocate more memory to the KV cache. +- If it is too low, you risk out-of-memory (OOM) errors later, so decrease `--mem-fraction-static`. + +Another straightforward approach is to increase `--mem-fraction-static` in increments of 0.01 until you encounter OOM errors for your workloads. + +### Avoid out-of-memory errors by tuning `--chunked-prefill-size`, `--mem-fraction-static`, and `--max-running-requests` + +If you encounter out-of-memory (OOM) errors, you can adjust the following parameters: + +- If OOM occurs during prefill, try reducing `--chunked-prefill-size` to `4096` or `2048`. This saves memory but slows down the prefill speed for long prompts. +- If OOM occurs during decoding, try lowering `--max-running-requests`. +- You can also reduce `--mem-fraction-static` to a smaller value, such as 0.8 or 0.7. This decreases the memory usage of the KV cache memory pool and helps prevent OOM errors during both prefill and decoding. However, it limits maximum concurrency and reduces peak throughput. + +### Tune `--cuda-graph-max-bs` +By default, CUDA graph is enabled only for small batch sizes (e.g., less than 160 or 256). +However, for some models, especially at large tensor parallelism sizes, CUDA graph can be useful for batch sizes up to 512 or 768. +Therefore, it may be beneficial to increase `--cuda-graph-max-bs` to a larger value. +Note that CUDA graph consumes more memory, so you may need to reduce `--mem-fraction-static` at the same time. + +### Tune `--dp-size` and `--tp-size` + +Data parallelism is better for throughput. When there is enough GPU memory, always favor data parallelism for throughput. Refer to [sglang router](../advanced_features/router.md) for a better data parallelism rather than using `dp_size` parameter. + +### Try other options + +- `torch.compile` accelerates small models on small batch sizes. You can enable it with `--enable-torch-compile`. +- Try other quantization (e.g. FP8 quantization with `--quantization fp8`) +- Try other parallelism strategies (e.g. [expert parallelism](https://lmsys.org/blog/2025-05-05-large-scale-ep/)) or DP attention for deepseek models (with `--enable-dp-attention --dp-size 8`). +- If the workload has many shared prefixes, try `--schedule-policy lpm`. Here, `lpm` stands for longest prefix match. It reorders requests to encourage more cache hits but introduces more scheduling overhead. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/lora.ipynb b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/lora.ipynb new file mode 100644 index 00000000..da25e988 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/lora.ipynb @@ -0,0 +1,595 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# LoRA Serving" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "SGLang enables the use of [LoRA adapters](https://arxiv.org/abs/2106.09685) with a base model. By incorporating techniques from [S-LoRA](https://arxiv.org/pdf/2311.03285) and [Punica](https://arxiv.org/pdf/2310.18547), SGLang can efficiently support multiple LoRA adapters for different sequences within a single batch of inputs." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Arguments for LoRA Serving" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following server arguments are relevant for multi-LoRA serving:\n", + "\n", + "* `enable_lora`: Enable LoRA support for the model. This argument is automatically set to True if `--lora-paths` is provided for backward compatibility.\n", + "\n", + "* `lora_paths`: The list of LoRA adapters to load. Each adapter must be specified in one of the following formats: | = | JSON with schema {\"lora_name\":str,\"lora_path\":str,\"pinned\":bool}.\n", + "\n", + "* `max_loras_per_batch`: Maximum number of adaptors used by each batch. This argument can affect the amount of GPU memory reserved for multi-LoRA serving, so it should be set to a smaller value when memory is scarce. Defaults to be 8.\n", + "\n", + "* `max_loaded_loras`: If specified, it limits the maximum number of LoRA adapters loaded in CPU memory at a time. The value must be greater than or equal to `max-loras-per-batch`.\n", + "\n", + "* `lora_eviction_policy`: LoRA adapter eviction policy when GPU memory pool is full. `lru`: Least Recently Used (default, better cache efficiency). `fifo`: First-In-First-Out.\n", + "\n", + "* `lora_backend`: The backend of running GEMM kernels for Lora modules. Currently we support Triton LoRA backend (`triton`) and Chunked SGMV backend (`csgmv`). In the future, faster backend built upon Cutlass or Cuda kernels will be added.\n", + "\n", + "* `max_lora_rank`: The maximum LoRA rank that should be supported. If not specified, it will be automatically inferred from the adapters provided in `--lora-paths`. This argument is needed when you expect to dynamically load adapters of larger LoRA rank after server startup.\n", + "\n", + "* `lora_target_modules`: The union set of all target modules where LoRA should be applied (e.g., `q_proj`, `k_proj`, `gate_proj`). If not specified, it will be automatically inferred from the adapters provided in `--lora-paths`. This argument is needed when you expect to dynamically load adapters of different target modules after server startup. You can also set it to `all` to enable LoRA for all supported modules. However, enabling LoRA on additional modules introduces a minor performance overhead. If your application is performance-sensitive, we recommend only specifying the modules for which you plan to load adapters.\n", + "\n", + "* `--max-lora-chunk-size`: Maximum chunk size for the ChunkedSGMV LoRA backend. Only used when --lora-backend is 'csgmv'. Choosing a larger value might improve performance. Please tune this value based on your hardware and workload as needed. Defaults to 16.\n", + "\n", + "* `tp_size`: LoRA serving along with Tensor Parallelism is supported by SGLang. `tp_size` controls the number of GPUs for tensor parallelism. More details on the tensor sharding strategy can be found in [S-Lora](https://arxiv.org/pdf/2311.03285) paper.\n", + "\n", + "From client side, the user needs to provide a list of strings as input batch, and a list of adaptor names that each input sequence corresponds to." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Usage\n", + "\n", + "### Serving Single Adaptor" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Note:** SGLang supports LoRA adapters through two APIs:\n", + "\n", + "1. **OpenAI-Compatible API** (`/v1/chat/completions`, `/v1/completions`): Use the `model:adapter-name` syntax. See [OpenAI API with LoRA](../basic_usage/openai_api_completions.ipynb#Using-LoRA-Adapters) for examples.\n", + "\n", + "2. **Native API** (`/generate`): Pass `lora_path` in the request body (shown below)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import requests\n", + "\n", + "from sglang.test.doc_patch import launch_server_cmd\n", + "from sglang.utils import wait_for_server, terminate_process" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", + " --enable-lora \\\n", + " --lora-paths lora0=algoprog/fact-generation-llama-3.1-8b-instruct-lora \\\n", + " --max-loras-per-batch 1 \\\n", + " --log-level warning \\\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://127.0.0.1:{port}\"\n", + "json_data = {\n", + " \"text\": [\n", + " \"List 3 countries and their capitals.\",\n", + " \"List 3 countries and their capitals.\",\n", + " ],\n", + " \"sampling_params\": {\"max_new_tokens\": 32, \"temperature\": 0},\n", + " # The first input uses lora0, and the second input uses the base model\n", + " \"lora_path\": [\"lora0\", None],\n", + "}\n", + "response = requests.post(\n", + " url + \"/generate\",\n", + " json=json_data,\n", + ")\n", + "print(f\"Output 0: {response.json()[0]['text']}\")\n", + "print(f\"Output 1: {response.json()[1]['text']}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Serving Multiple Adaptors" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", + " --enable-lora \\\n", + " --lora-paths lora0=algoprog/fact-generation-llama-3.1-8b-instruct-lora \\\n", + " lora1=Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16 \\\n", + " --max-loras-per-batch 2 \\\n", + " --log-level warning \\\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://127.0.0.1:{port}\"\n", + "json_data = {\n", + " \"text\": [\n", + " \"List 3 countries and their capitals.\",\n", + " \"List 3 countries and their capitals.\",\n", + " ],\n", + " \"sampling_params\": {\"max_new_tokens\": 32, \"temperature\": 0},\n", + " # The first input uses lora0, and the second input uses lora1\n", + " \"lora_path\": [\"lora0\", \"lora1\"],\n", + "}\n", + "response = requests.post(\n", + " url + \"/generate\",\n", + " json=json_data,\n", + ")\n", + "print(f\"Output 0: {response.json()[0]['text']}\")\n", + "print(f\"Output 1: {response.json()[1]['text']}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Dynamic LoRA loading" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Instead of specifying all adapters during server startup via `--lora-paths`. You can also load & unload LoRA adapters dynamically via the `/load_lora_adapter` and `/unload_lora_adapter` API.\n", + "\n", + "When using dynamic LoRA loading, it's recommended to explicitly specify both `--max-lora-rank` and `--lora-target-modules` at startup. For backward compatibility, SGLang will infer these values from `--lora-paths` if they are not explicitly provided. However, in that case, you would have to ensure that all dynamically loaded adapters share the same shape (rank and target modules) as those in the initial `--lora-paths` or are strictly \"smaller\"." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "lora0 = \"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16\" # rank - 4, target modules - q_proj, k_proj, v_proj, o_proj, gate_proj\n", + "lora1 = \"algoprog/fact-generation-llama-3.1-8b-instruct-lora\" # rank - 64, target modules - q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj\n", + "lora0_new = \"philschmid/code-llama-3-1-8b-text-to-sql-lora\" # rank - 256, target modules - q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj\n", + "\n", + "\n", + "# The `--target-lora-modules` param below is technically not needed, as the server will infer it from lora0 which already has all the target modules specified.\n", + "# We are adding it here just to demonstrate usage.\n", + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + " python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", + " --enable-lora \\\n", + " --cuda-graph-max-bs 2 \\\n", + " --max-loras-per-batch 2 \\\n", + " --max-lora-rank 256\n", + " --lora-target-modules all\n", + " --log-level warning\n", + " \"\"\"\n", + ")\n", + "\n", + "url = f\"http://127.0.0.1:{port}\"\n", + "wait_for_server(url)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Load adapter lora0" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = requests.post(\n", + " url + \"/load_lora_adapter\",\n", + " json={\n", + " \"lora_name\": \"lora0\",\n", + " \"lora_path\": lora0,\n", + " },\n", + ")\n", + "\n", + "if response.status_code == 200:\n", + " print(\"LoRA adapter loaded successfully.\", response.json())\n", + "else:\n", + " print(\"Failed to load LoRA adapter.\", response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Load adapter lora1:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = requests.post(\n", + " url + \"/load_lora_adapter\",\n", + " json={\n", + " \"lora_name\": \"lora1\",\n", + " \"lora_path\": lora1,\n", + " },\n", + ")\n", + "\n", + "if response.status_code == 200:\n", + " print(\"LoRA adapter loaded successfully.\", response.json())\n", + "else:\n", + " print(\"Failed to load LoRA adapter.\", response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Check inference output:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://127.0.0.1:{port}\"\n", + "json_data = {\n", + " \"text\": [\n", + " \"List 3 countries and their capitals.\",\n", + " \"List 3 countries and their capitals.\",\n", + " ],\n", + " \"sampling_params\": {\"max_new_tokens\": 32, \"temperature\": 0},\n", + " # The first input uses lora0, and the second input uses lora1\n", + " \"lora_path\": [\"lora0\", \"lora1\"],\n", + "}\n", + "response = requests.post(\n", + " url + \"/generate\",\n", + " json=json_data,\n", + ")\n", + "print(f\"Output from lora0: \\n{response.json()[0]['text']}\\n\")\n", + "print(f\"Output from lora1 (updated): \\n{response.json()[1]['text']}\\n\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Unload lora0 and replace it with a different adapter:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = requests.post(\n", + " url + \"/unload_lora_adapter\",\n", + " json={\n", + " \"lora_name\": \"lora0\",\n", + " },\n", + ")\n", + "\n", + "response = requests.post(\n", + " url + \"/load_lora_adapter\",\n", + " json={\n", + " \"lora_name\": \"lora0\",\n", + " \"lora_path\": lora0_new,\n", + " },\n", + ")\n", + "\n", + "if response.status_code == 200:\n", + " print(\"LoRA adapter loaded successfully.\", response.json())\n", + "else:\n", + " print(\"Failed to load LoRA adapter.\", response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Check output again:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://127.0.0.1:{port}\"\n", + "json_data = {\n", + " \"text\": [\n", + " \"List 3 countries and their capitals.\",\n", + " \"List 3 countries and their capitals.\",\n", + " ],\n", + " \"sampling_params\": {\"max_new_tokens\": 32, \"temperature\": 0},\n", + " # The first input uses lora0, and the second input uses lora1\n", + " \"lora_path\": [\"lora0\", \"lora1\"],\n", + "}\n", + "response = requests.post(\n", + " url + \"/generate\",\n", + " json=json_data,\n", + ")\n", + "print(f\"Output from lora0: \\n{response.json()[0]['text']}\\n\")\n", + "print(f\"Output from lora1 (updated): \\n{response.json()[1]['text']}\\n\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### OpenAI-compatible API usage\n", + "\n", + "You can use LoRA adapters via the OpenAI-compatible APIs by specifying the adapter in the `model` field using the `base-model:adapter-name` syntax (for example, `qwen/qwen2.5-0.5b-instruct:adapter_a`). For more details and examples, see the “Using LoRA Adapters” section in the OpenAI API documentation: [openai_api_completions.ipynb](../basic_usage/openai_api_completions.ipynb).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### LoRA GPU Pinning" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Another advanced option is to specify adapters as `pinned` during loading. When an adapter is pinned, it is permanently assigned to one of the available GPU pool slots (as configured by `--max-loras-per-batch`) and will not be evicted from GPU memory during runtime. Instead, it remains resident until it is explicitly unloaded.\n", + "\n", + "This can improve performance in scenarios where the same adapter is frequently used across requests, by avoiding repeated memory transfers and reinitialization overhead. However, since GPU pool slots are limited, pinning adapters reduces the flexibility of the system to dynamically load other adapters on demand. If too many adapters are pinned, it may lead to degraded performance, or in the most extreme case (`Number of pinned adapters == max-loras-per-batch`), halt all unpinned requests. Therefore, currently SGLang limits maximal number of pinned adapters to `max-loras-per-batch - 1` to prevent unexpected starvations. \n", + "\n", + "In the example below, we start a server with `lora1` loaded as pinned, `lora2` and `lora3` loaded as regular (unpinned) adapters. Please note that, we intentionally specify `lora2` and `lora3` in two different formats to demonstrate that both are supported." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + " python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", + " --enable-lora \\\n", + " --cuda-graph-max-bs 8 \\\n", + " --max-loras-per-batch 3 \\\n", + " --max-lora-rank 256 \\\n", + " --lora-target-modules all \\\n", + " --lora-paths \\\n", + " {\"lora_name\":\"lora0\",\"lora_path\":\"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16\",\"pinned\":true} \\\n", + " {\"lora_name\":\"lora1\",\"lora_path\":\"algoprog/fact-generation-llama-3.1-8b-instruct-lora\"} \\\n", + " lora2=philschmid/code-llama-3-1-8b-text-to-sql-lora\n", + " --log-level warning\n", + " \"\"\"\n", + ")\n", + "\n", + "\n", + "url = f\"http://127.0.0.1:{port}\"\n", + "wait_for_server(url)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also specify adapter as pinned during dynamic adapter loading. In the example below, we reload `lora2` as pinned adapter:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = requests.post(\n", + " url + \"/unload_lora_adapter\",\n", + " json={\n", + " \"lora_name\": \"lora1\",\n", + " },\n", + ")\n", + "\n", + "response = requests.post(\n", + " url + \"/load_lora_adapter\",\n", + " json={\n", + " \"lora_name\": \"lora1\",\n", + " \"lora_path\": \"algoprog/fact-generation-llama-3.1-8b-instruct-lora\",\n", + " \"pinned\": True, # Pin the adapter to GPU\n", + " },\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Verify that the results are expected:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://127.0.0.1:{port}\"\n", + "json_data = {\n", + " \"text\": [\n", + " \"List 3 countries and their capitals.\",\n", + " \"List 3 countries and their capitals.\",\n", + " \"List 3 countries and their capitals.\",\n", + " ],\n", + " \"sampling_params\": {\"max_new_tokens\": 32, \"temperature\": 0},\n", + " # The first input uses lora0, and the second input uses lora1\n", + " \"lora_path\": [\"lora0\", \"lora1\", \"lora2\"],\n", + "}\n", + "response = requests.post(\n", + " url + \"/generate\",\n", + " json=json_data,\n", + ")\n", + "print(f\"Output from lora0 (pinned): \\n{response.json()[0]['text']}\\n\")\n", + "print(f\"Output from lora1 (pinned): \\n{response.json()[1]['text']}\\n\")\n", + "print(f\"Output from lora2 (not pinned): \\n{response.json()[2]['text']}\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Choosing LoRA Backend\n", + "\n", + "SGLang supports two LoRA backends that you can choose from using the `--lora-backend` argument:\n", + "\n", + "- `triton`: Default basic Triton-based backend.\n", + "- `csgmv`: Chunked SGMV backend optimized for high concurrency scenarios.\n", + "\n", + "The `csgmv` backend was recently introduced to improve performance especially at high-concurrency scenarios. Our benchmark shows that it achieves 20% to 80% latency improvements over the basic triton backend.\n", + "Currently it is at preview phase, we expect to make it our the default LoRA backend in future release. Before that, you can adopt it by manually setting the `--lora-backend` server config." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + " python3 -m sglang.launch_server \\\n", + " --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", + " --enable-lora \\\n", + " --lora-backend csgmv \\\n", + " --max-loras-per-batch 16 \\\n", + " --lora-paths lora1=path/to/lora1 lora2=path/to/lora2\n", + " \"\"\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Future Works\n", + "\n", + "The development roadmap for LoRA-related features can be found in this [issue](https://github.com/sgl-project/sglang/issues/2929). Other features, including Embedding Layer, Unified Paging, Cutlass backend are still under development." + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/observability.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/observability.md new file mode 100644 index 00000000..9c5d2e17 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/observability.md @@ -0,0 +1,35 @@ +# Observability + +## Production Metrics +SGLang exposes the following metrics via Prometheus. You can enable them by adding `--enable-metrics` when launching the server. +You can query them by: +``` +curl http://localhost:30000/metrics +``` + +See [Production Metrics](../references/production_metrics.md) and [Production Request Tracing](../references/production_request_trace.md) for more details. + +## Logging + +By default, SGLang does not log any request contents. You can log them by using `--log-requests`. +You can control the verbosity by using `--log-request-level`. +See [Logging](server_arguments.md#logging) for more details. + +## Request Dump and Replay + +You can dump all requests and replay them later for benchmarking or other purposes. + +To start dumping, use the following command to send a request to a server: +``` +python3 -m sglang.srt.managers.configure_logging --url http://localhost:30000 --dump-requests-folder /tmp/sglang_request_dump --dump-requests-threshold 100 +``` +The server will dump the requests into a pickle file for every 100 requests. + +To replay the request dump, use `scripts/playground/replay_request_dump.py`. + +## Crash Dump and Replay +Sometimes the server might crash, and you may want to debug the cause of the crash. +SGLang supports crash dumping, which will dump all requests from the 5 minutes before the crash, allowing you to replay the requests and debug the reason later. + +To enable crash dumping, use `--crash-dump-folder /tmp/crash_dump`. +To replay the crash dump, use `scripts/playground/replay_request_dump.py`. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/pd_disaggregation.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/pd_disaggregation.md new file mode 100644 index 00000000..ec437ecb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/pd_disaggregation.md @@ -0,0 +1,325 @@ +# PD Disaggregation + +## Why and What is PD Disaggregation? + +Large Language Model (LLM) inference comprises two distinct phases: **Prefill** and **Decode**. The Prefill phase is computation-intensive, processing the entire input sequence, while the Decode phase is memory-intensive, managing the Key-Value (KV) cache for token generation. Traditionally, these phases are handled within a unified engine, where combined scheduling of prefill and decode batches introduces inefficiencies. To address these challenges, we introduce **Prefill and Decoding (PD) Disaggregation** in SGLang. + +### Issues with Unified Scheduling + +The conventional unified engine, which processes prefill and decode batches together, results in two significant problems: + +1. **Prefill Interruption**: Incoming prefill batches frequently interrupt ongoing decode batches, causing substantial delays in token generation. +2. **DP Attention Imbalance**: In data-parallel (DP) attention, one DP worker may process a prefill batch while another handles a decode batch simultaneously, leading to increased decode latency. + +PD Disaggregation resolves these by separating the two stages, enabling tailored optimizations for each. + +For the design details, please refer to [link](https://docs.google.com/document/d/1rQXJwKd5b9b1aOzLh98mnyMhBMhlxXA5ATZTHoQrwvc/edit?tab=t.0). + +Currently, we support Mooncake and NIXL as the transfer engine. + +## Profiling in PD Disaggregation Mode + +When you need to profile prefill or decode workers in PD disaggregation mode, please refer to the [Profile In PD Disaggregation Mode](https://docs.sglang.io/developer_guide/benchmark_and_profiling.html#profile-in-pd-disaggregation-mode) section in the Benchmark and Profiling guide. Due to torch profiler limitations, prefill and decode workers must be profiled separately using dedicated command-line options. + +## Router Integration + +For deploying PD disaggregation at scale with load balancing and fault tolerance, SGLang provides a router. The router can distribute requests between prefill and decode instances using various routing policies. For detailed information on setting up routing with PD disaggregation, including configuration options and deployment patterns, see the [SGLang Router documentation](router.md#mode-3-prefill-decode-disaggregation). + + +## Mooncake +### Requirements + +```bash +uv pip install mooncake-transfer-engine +``` + +### Usage + +### Llama Single Node + +```bash +python -m sglang.launch_server \ + --model-path meta-llama/Llama-3.1-8B-Instruct \ + --disaggregation-mode prefill \ + --port 30000 \ + --disaggregation-ib-device mlx5_roce0 +python -m sglang.launch_server \ + --model-path meta-llama/Llama-3.1-8B-Instruct \ + --disaggregation-mode decode \ + --port 30001 \ + --base-gpu-id 1 \ + --disaggregation-ib-device mlx5_roce0 +python -m sglang_router.launch_router --pd-disaggregation --prefill http://127.0.0.1:30000 --decode http://127.0.0.1:30001 --host 0.0.0.0 --port 8000 +``` + +### DeepSeek Multi-Node + +```bash +# prefill 0 +python -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-V3-0324 \ + --disaggregation-ib-device ${device_name} \ + --disaggregation-mode prefill \ + --host ${local_ip} \ + --port 30000 \ + --trust-remote-code \ + --dist-init-addr ${prefill_master_ip}:5000 \ + --nnodes 2 \ + --node-rank 0 \ + --tp-size 16 \ + --dp-size 8 \ + --enable-dp-attention \ + --moe-a2a-backend deepep \ + --mem-fraction-static 0.8 +# prefill 1 +python -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-V3-0324 \ + --disaggregation-ib-device ${device_name} \ + --disaggregation-mode prefill \ + --host ${local_ip} \ + --port 30000 \ + --trust-remote-code \ + --dist-init-addr ${prefill_master_ip}:5000 \ + --nnodes 2 \ + --node-rank 1 \ + --tp-size 16 \ + --dp-size 8 \ + --enable-dp-attention \ + --moe-a2a-backend deepep \ + --mem-fraction-static 0.8 +# decode 0 +python -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-V3-0324 \ + --disaggregation-ib-device ${device_name} \ + --disaggregation-mode decode \ + --host ${local_ip} \ + --port 30001 \ + --trust-remote-code \ + --dist-init-addr ${decode_master_ip}:5000 \ + --nnodes 2 \ + --node-rank 0 \ + --tp-size 16 \ + --dp-size 8 \ + --enable-dp-attention \ + --moe-a2a-backend deepep \ + --mem-fraction-static 0.8 \ + --max-running-requests 128 +# decode 1 +python -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-V3-0324 \ + --disaggregation-ib-device ${device_name} \ + --disaggregation-mode decode \ + --host ${local_ip} \ + --port 30001 \ + --trust-remote-code \ + --dist-init-addr ${decode_master_ip}:5000 \ + --nnodes 2 \ + --node-rank 1 \ + --tp-size 16 \ + --dp-size 8 \ + --enable-dp-attention \ + --moe-a2a-backend deepep \ + --mem-fraction-static 0.8 \ + --max-running-requests 128 +``` +### Advanced Configuration + +PD Disaggregation with Mooncake supports the following environment variables for fine-grained control over system behavior. + +#### NVLink Transport Configuration +To enable NVLink transport for KV cache transfers with the mooncake backend (recommended for NVL72 deployments), set the following environment variables. Note that auxiliary data transfer will still use TCP as a temporary workaround. + +```bash +export SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True +export MC_FORCE_MNNVL=True +``` + +#### Prefill Server Configuration +| Variable | Description | Default | +|:--------:|:-----------:|:--------: +| **`SGLANG_DISAGGREGATION_THREAD_POOL_SIZE`** | Controls the total number of worker threads for KVCache transfer operations per TP rank | A dynamic value calculated by `int(0.75 * os.cpu_count()) // 8)`, which is limited to be larger than 4 and less than 12 to ensure efficiency and prevent thread race conditions | +| **`SGLANG_DISAGGREGATION_QUEUE_SIZE`** | Sets the number of parallel transfer queues. KVCache transfer requests from multiple decode instances will be sharded into these queues so that they can share the threads and the transfer bandwidth at the same time. If it is set to `1`, then we transfer requests one by one according to fcfs strategy | `4` | +| **`SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT`** | Timeout (seconds) for receiving destination KV indices during request initialization | `300` | + +If a greater mean TTFT is acceptable, you can `export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600` (10 minutes) to relax the timeout condition. +Please be aware that this setting will cause prefill instances to take a longer time to clean up the affected memory resources when a running decode node loses connection. + +#### Decode Server Configuration +| Variable | Description | Default | +|:--------:|:-----------:|:--------: +| **`SGLANG_DISAGGREGATION_HEARTBEAT_INTERVAL`** | Interval (seconds) between health checks to prefill bootstrap servers | `5.0` | +| **`SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE`** | Consecutive heartbeat failures before marking prefill server offline | `2` | +| **`SGLANG_DISAGGREGATION_WAITING_TIMEOUT`** | Timeout (seconds) for receiving KV Cache after request initialization | `300` | + +If a greater mean TTFT is acceptable, you can `export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=600` (10 minutes) to relax the timeout condition. + + +## NIXL +### Requirements + +Install via pip. + +```bash +pip install nixl +``` + +Or build from source - may be required if you already have UCX installed. + +```bash +git clone https://github.com/ai-dynamo/nixl.git +cd nixl +pip install . --config-settings=setup-args="-Ducx_path=/path/to/ucx" +``` + + +### Usage + +### Llama Single Node + +```bash +python -m sglang.launch_server \ + --model-path meta-llama/Llama-3.1-8B-Instruct \ + --disaggregation-mode prefill \ + --port 30000 \ + --disaggregation-transfer-backend nixl +python -m sglang.launch_server \ + --model-path meta-llama/Llama-3.1-8B-Instruct \ + --disaggregation-mode decode \ + --port 30001 \ + --base-gpu-id 1 \ + --disaggregation-transfer-backend nixl +python -m sglang_router.launch_router --pd-disaggregation --prefill http://127.0.0.1:30000 --decode http://127.0.0.1:30001 --host 0.0.0.0 --port 8000 +``` + +### DeepSeek Multi-Node + +```bash +# prefill 0 +python -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-V3-0324 \ + --disaggregation-transfer-backend nixl \ + --disaggregation-mode prefill \ + --host ${local_ip} \ + --port 30000 \ + --trust-remote-code \ + --dist-init-addr ${prefill_master_ip}:5000 \ + --nnodes 2 \ + --node-rank 0 \ + --tp-size 16 \ + --dp-size 8 \ + --enable-dp-attention \ + --moe-a2a-backend deepep \ + --mem-fraction-static 0.8 +# prefill 1 +python -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-V3-0324 \ + --disaggregation-transfer-backend nixl \ + --disaggregation-mode prefill \ + --host ${local_ip} \ + --port 30000 \ + --trust-remote-code \ + --dist-init-addr ${prefill_master_ip}:5000 \ + --nnodes 2 \ + --node-rank 1 \ + --tp-size 16 \ + --dp-size 8 \ + --enable-dp-attention \ + --moe-a2a-backend deepep \ + --mem-fraction-static 0.8 +# decode 0 +python -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-V3-0324 \ + --disaggregation-transfer-backend nixl \ + --disaggregation-mode decode \ + --host ${local_ip} \ + --port 30001 \ + --trust-remote-code \ + --dist-init-addr ${decode_master_ip}:5000 \ + --nnodes 2 \ + --node-rank 0 \ + --tp-size 16 \ + --dp-size 8 \ + --enable-dp-attention \ + --moe-a2a-backend deepep \ + --mem-fraction-static 0.8 \ + --max-running-requests 128 +# decode 1 +python -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-V3-0324 \ + --disaggregation-transfer-backend nixl \ + --disaggregation-mode decode \ + --host ${local_ip} \ + --port 30001 \ + --trust-remote-code \ + --dist-init-addr ${decode_master_ip}:5000 \ + --nnodes 2 \ + --node-rank 1 \ + --tp-size 16 \ + --dp-size 8 \ + --enable-dp-attention \ + --moe-a2a-backend deepep \ + --mem-fraction-static 0.8 \ + --max-running-requests 128 +``` + +## ASCEND + +### Usage + +Use ascend backend with [mf_adapter(download link)](https://sglang-ascend.obs.cn-east-3.myhuaweicloud.com:443/sglang/mf_adapter-1.0.0-cp311-cp311-linux_aarch64.whl?AccessKeyId=HPUAXT4YM0U8JNTERLST&Expires=1783151861&Signature=3j10QDUjqk70enaq8lostYV2bEA%3D) and ASCEND_MF_STORE_URL being set + +```bash +pip install mf_adapter-1.0.0-cp311-cp311-linux_aarch64.whl --force-reinstall +export ASCEND_MF_STORE_URL="tcp://xxx.xx.xxx.xxx:xxxx" +``` +Use mooncake backend, more details can be found in mooncake section. +```bash +export ENABLE_ASCEND_TRANSFER_WITH_MOONCAKE=true +``` + + +### Llama Single Node + +```bash +python -m sglang.launch_server \ + --model-path meta-llama/Llama-3.1-8B-Instruct \ + --disaggregation-mode prefill \ + --port 30000 \ + --disaggregation-transfer-backend ascend +python -m sglang.launch_server \ + --model-path meta-llama/Llama-3.1-8B-Instruct \ + --disaggregation-mode decode \ + --port 30001 \ + --base-gpu-id 1 \ + --disaggregation-transfer-backend ascend +python -m sglang_router.launch_router --pd-disaggregation --prefill http://127.0.0.1:30000 --decode http://127.0.0.1:30001 --host 0.0.0.0 --port 8000 +``` + +### DeepSeek Multi-Node + +```bash +# prefill 0 +python -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-V3-0324 \ + --disaggregation-transfer-backend ascend \ + --disaggregation-mode prefill \ + --host ${local_ip} \ + --port 30000 \ + --trust-remote-code \ + --dist-init-addr ${prefill_master_ip}:5000 \ + --nnodes 1 \ + --node-rank 0 \ + --tp-size 16 +# decode 0 +python -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-V3-0324 \ + --disaggregation-transfer-backend ascend \ + --disaggregation-mode decode \ + --host ${local_ip} \ + --port 30001 \ + --trust-remote-code \ + --dist-init-addr ${decode_master_ip}:5000 \ + --nnodes 1 \ + --node-rank 0 \ + --tp-size 16 +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/quantization.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/quantization.md new file mode 100644 index 00000000..18ef0e8a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/quantization.md @@ -0,0 +1,384 @@ +# Quantization + +SGLang supports various quantization methods, including offline quantization and online dynamic quantization. + +Offline quantization loads pre-quantized model weights directly during inference. This is required for quantization methods +such as GPTQ and AWQ, which collect and pre-compute various statistics from the original weights using the calibration dataset. + +Online quantization dynamically computes scaling parameters—such as the maximum/minimum values of model weights—during runtime. +Like NVIDIA FP8 training's [delayed scaling](https://docs.nvidia.com/deeplearning/transformer-engine/user-guide/examples/fp8_primer.html#Mixed-precision-training-with-FP8) mechanism, online quantization calculates the appropriate scaling factors +on-the-fly to convert high-precision weights into a lower-precision format. + +**Note: For better performance, usability and convenience, offline quantization is recommended over online quantization.** + +If you use a pre-quantized model, do not add `--quantization` to enable online quantization at the same time. +For popular pre-quantized models, please visit [Unsloth](https://huggingface.co/unsloth), [ModelCloud](https://huggingface.co/collections/ModelCloud/vortex-673743382af0a52b2a8b9fe2) +or [NeuralMagic](https://huggingface.co/collections/neuralmagic) collections on HF for some +popular quality validated quantized models. Quantized models must be validated via benchmarks post-quantization +to guard against abnormal quantization loss regressions. + +## Offline Quantization + +To load already quantized models, simply load the model weights and config. **Again, if the model has been quantized offline, +there's no need to add `--quantization` argument when starting the engine. The quantization method will be parsed from the +downloaded Hugging Face config. For example, DeepSeek V3/R1 models are already in FP8, so do not add redundant parameters.** + +```bash +python3 -m sglang.launch_server \ + --model-path hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4 \ + --port 30000 --host 0.0.0.0 +``` + +Take note, if your model is **per-channel quantized (INT8 or FP8) with per-token dynamic quantization activation**, you can opt to include `--quantization w8a8_int8` or `--quantization w8a8_fp8` to invoke the corresponding CUTLASS int8_kernel or fp8_kernel in sgl-kernel. This action will ignore the Hugging Face config's quantization settings. For instance, with `neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8-dynamic`, if you execute with `--quantization w8a8_fp8`, the system will use the `W8A8Fp8Config` from SGLang to invoke the sgl-kernel, rather than the `CompressedTensorsConfig` for vLLM kernels. + +```bash +python3 -m sglang.launch_server \ + --model-path neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8-dynamic \ + --quantization w8a8_fp8 \ + --port 30000 --host 0.0.0.0 +``` + +### Examples of Offline Model Quantization + +#### Using [Unsloth](https://docs.unsloth.ai/basics/inference-and-deployment/sglang-guide) + +We strongly suggest the use of Unsloth to quantize and load the model. Please refer to [SGLang Deployment & Inference Guide with Unsloth](https://docs.unsloth.ai/basics/inference-and-deployment/sglang-guide). + +#### Using [auto-round](https://github.com/intel/auto-round) + +```bash +# Install +pip install auto-round +``` + +- LLM quantization + +```py +# for LLM +from auto_round import AutoRound +model_id = "meta-llama/Llama-3.2-1B-Instruct" +quant_path = "Llama-3.2-1B-Instruct-autoround-4bit" +# Scheme examples: "W2A16", "W3A16", "W4A16", "W8A16", "NVFP4", "MXFP4" (no real kernels), "GGUF:Q4_K_M", etc. +scheme = "W4A16" +format = "auto_round" +autoround = AutoRound(model_id, scheme=scheme) +autoround.quantize_and_save(quant_path, format=format) # quantize and save + +``` + +- VLM quantization +```py +# for VLMs +from auto_round import AutoRoundMLLM +model_name = "Qwen/Qwen2-VL-2B-Instruct" +quant_path = "Qwen2-VL-2B-Instruct-autoround-4bit" +scheme = "W4A16" +format = "auto_round" +autoround = AutoRoundMLLM(model_name, scheme) +autoround.quantize_and_save(quant_path, format=format) # quantize and save + +``` + +- Command Line Usage (Gaudi/CPU/Intel GPU/CUDA) + +```bash +auto-round \ + --model meta-llama/Llama-3.2-1B-Instruct \ + --bits 4 \ + --group_size 128 \ + --format "auto_round" \ + --output_dir ./tmp_autoround +``` + +- known issues + +Several limitations currently affect offline quantized model loading in sglang, These issues might be resolved in future updates of sglang. If you experience any problems, consider using Hugging Face Transformers as an alternative. + +1. Mixed-bit Quantization Limitations + + Mixed-bit quantization is not fully supported. Due to vLLM's layer fusion (e.g., QKV fusion), applying different bit-widths to components within the same fused layer can lead to compatibility issues. + + +2. Limited Support for Quantized MoE Models + + Quantized MoE models may encounter inference issues due to kernel limitations (e.g., lack of support for mlp.gate layer quantization). please try to skip quantizing these layers to avoid such errors. + + +3. Limited Support for Quantized VLMs +

+ VLM failure cases + + Qwen2.5-VL-7B + + auto_round:auto_gptq format: Accuracy is close to zero. + + GPTQ format: Fails with: + ``` + The output size is not aligned with the quantized weight shape + ``` + auto_round:auto_awq and AWQ format: These work as expected. +
+ +#### Using [GPTQModel](https://github.com/ModelCloud/GPTQModel) + +```bash +# install +pip install gptqmodel --no-build-isolation -v +``` + +```py +from datasets import load_dataset +from gptqmodel import GPTQModel, QuantizeConfig + +model_id = "meta-llama/Llama-3.2-1B-Instruct" +quant_path = "Llama-3.2-1B-Instruct-gptqmodel-4bit" + +calibration_dataset = load_dataset( + "allenai/c4", data_files="en/c4-train.00001-of-01024.json.gz", + split="train" + ).select(range(1024))["text"] + +quant_config = QuantizeConfig(bits=4, group_size=128) # quantization config +model = GPTQModel.load(model_id, quant_config) # load model + +model.quantize(calibration_dataset, batch_size=2) # quantize +model.save(quant_path) # save model +``` + +#### Using [LLM Compressor](https://github.com/vllm-project/llm-compressor/) + +```bash +# install +pip install llmcompressor +``` + +Here, we take quantize `meta-llama/Meta-Llama-3-8B-Instruct` to `FP8` as an example to elaborate on how to do offline quantization. + +```python +from transformers import AutoTokenizer +from llmcompressor.transformers import SparseAutoModelForCausalLM +from llmcompressor.transformers import oneshot +from llmcompressor.modifiers.quantization import QuantizationModifier + +# Step 1: Load the original model. +MODEL_ID = "meta-llama/Meta-Llama-3-8B-Instruct" + +model = SparseAutoModelForCausalLM.from_pretrained( + MODEL_ID, device_map="auto", torch_dtype="auto") +tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) + +# Step 2: Perform offline quantization. +# Step 2.1: Configure the simple PTQ quantization. +recipe = QuantizationModifier( + targets="Linear", scheme="FP8_DYNAMIC", ignore=["lm_head"]) + +# Step 2.2: Apply the quantization algorithm. +oneshot(model=model, recipe=recipe) + +# Step 3: Save the model. +SAVE_DIR = MODEL_ID.split("/")[1] + "-FP8-Dynamic" +model.save_pretrained(SAVE_DIR) +tokenizer.save_pretrained(SAVE_DIR) +``` + +Then, you can directly use the quantized model with `SGLang`, by using the following command: + +```bash +python3 -m sglang.launch_server \ + --model-path $PWD/Meta-Llama-3-8B-Instruct-FP8-Dynamic \ + --port 30000 --host 0.0.0.0 +``` + +#### Using [NVIDIA ModelOpt](https://github.com/NVIDIA/TensorRT-Model-Optimizer) + +NVIDIA Model Optimizer (ModelOpt) provides advanced quantization techniques optimized for NVIDIA hardware. SGLang includes a streamlined workflow for quantizing models with ModelOpt and automatically exporting them for deployment. + +##### Installation + +First, install ModelOpt. You can either install it directly or as an optional SGLang dependency: + +```bash +# Option 1: Install ModelOpt directly +pip install nvidia-modelopt + +# Option 2: Install SGLang with ModelOpt support (recommended) +pip install sglang[modelopt] +``` + +##### Quantization and Export Workflow + +SGLang provides an example script that demonstrates the complete ModelOpt quantization and export workflow: + +```bash +# Quantize and export a model using ModelOpt FP8 quantization +python examples/usage/modelopt_quantize_and_export.py quantize \ + --model-path TinyLlama/TinyLlama-1.1B-Chat-v1.0 \ + --export-dir ./quantized_tinyllama_fp8 \ + --quantization-method modelopt_fp8 + +# For FP4 quantization +python examples/usage/modelopt_quantize_and_export.py quantize \ + --model-path TinyLlama/TinyLlama-1.1B-Chat-v1.0 \ + --export-dir ./quantized_tinyllama_fp4 \ + --quantization-method modelopt_fp4 +``` + +##### Available Quantization Methods + +- `modelopt_fp8`: FP8 quantization with optimal performance on NVIDIA Hopper and Blackwell GPUs +- `modelopt_fp4`: FP4 quantization with optimal performance on Nvidia Blackwell GPUs + +##### Python API Usage + +You can also use ModelOpt quantization programmatically: + +```python +import sglang as sgl +from sglang.srt.configs.device_config import DeviceConfig +from sglang.srt.configs.load_config import LoadConfig +from sglang.srt.configs.model_config import ModelConfig +from sglang.srt.model_loader.loader import get_model_loader + +# Configure model with ModelOpt quantization and export +model_config = ModelConfig( + model_path="TinyLlama/TinyLlama-1.1B-Chat-v1.0", + quantization="modelopt_fp8", # or "modelopt_fp4" + trust_remote_code=True, +) + +load_config = LoadConfig( + modelopt_export_path="./exported_model", + modelopt_checkpoint_save_path="./checkpoint.pth", # optional, fake quantized checkpoint +) +device_config = DeviceConfig(device="cuda") + +# Load and quantize the model (export happens automatically) +model_loader = get_model_loader(load_config, model_config) +quantized_model = model_loader.load_model( + model_config=model_config, + device_config=device_config, +) +``` + +##### Deploying Quantized Models + +After quantization and export, you can deploy the model with SGLang: + +```bash +# Deploy the exported quantized model +python -m sglang.launch_server \ + --model-path ./quantized_tinyllama_fp8 \ + --quantization modelopt \ + --port 30000 --host 0.0.0.0 +``` + +Or using the Python API: + +```python +import sglang as sgl + +# Deploy exported ModelOpt quantized model +llm = sgl.Engine( + model_path="./quantized_tinyllama_fp8", + quantization="modelopt" +) + +# Run inference +prompts = ["Hello, how are you?", "What is the capital of France?"] +sampling_params = {"temperature": 0.8, "top_p": 0.95, "max_new_tokens": 100} +outputs = llm.generate(prompts, sampling_params) + +for i, output in enumerate(outputs): + print(f"Prompt: {prompts[i]}") + print(f"Output: {output.outputs[0].text}") +``` + +##### Advanced Features + +**Checkpoint Management**: Save and restore fake quantized checkpoints for reuse: + +```bash +# Save the fake quantized checkpoint during quantization +python examples/usage/modelopt_quantize_and_export.py quantize \ + --model-path meta-llama/Llama-3.2-1B-Instruct \ + --export-dir ./quantized_model \ + --quantization-method modelopt_fp8 \ + --checkpoint-save-path ./my_checkpoint.pth + +# The checkpoint can be reused for future quantization runs and skip calibration +``` + +**Export-only Workflow**: If you have a pre-existing fake quantized ModelOpt checkpoint, you can export it directly: + +```python +from sglang.srt.configs.device_config import DeviceConfig +from sglang.srt.configs.load_config import LoadConfig +from sglang.srt.configs.model_config import ModelConfig +from sglang.srt.model_loader.loader import get_model_loader + +model_config = ModelConfig( + model_path="meta-llama/Llama-3.2-1B-Instruct", + quantization="modelopt_fp8", + trust_remote_code=True, +) + +load_config = LoadConfig( + modelopt_checkpoint_restore_path="./my_checkpoint.pth", + modelopt_export_path="./exported_model", +) + +# Load and export the model +model_loader = get_model_loader(load_config, model_config) +model_loader.load_model(model_config=model_config, device_config=DeviceConfig()) +``` + +##### Benefits of ModelOpt + +- **Hardware Optimization**: Specifically optimized for NVIDIA GPU architectures +- **Advanced Quantization**: Supports cutting-edge FP8 and FP4 quantization techniques +- **Seamless Integration**: Automatic export to HuggingFace format for easy deployment +- **Calibration-based**: Uses calibration datasets for optimal quantization quality +- **Production Ready**: Enterprise-grade quantization with NVIDIA support + +## Online Quantization + +To enable online quantization, you can simply specify `--quantization` in the command line. For example, you can launch the server with the following command to enable `FP8` quantization for model `meta-llama/Meta-Llama-3.1-8B-Instruct`: + +```bash +python3 -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ + --quantization fp8 \ + --port 30000 --host 0.0.0.0 +``` + +Our team is working on supporting more online quantization methods. SGLang will soon support methods including but not limited to `["awq", "gptq", "marlin", "gptq_marlin", "awq_marlin", "bitsandbytes", "gguf"]`. + +SGLang also supports quantization methods based on [torchao](https://github.com/pytorch/ao). You can simply specify `--torchao-config` in the command line to support this feature. For example, if you want to enable `int4wo-128` for model `meta-llama/Meta-Llama-3.1-8B-Instruct`, you can launch the server with the following command: + +```bash +python3 -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ + --torchao-config int4wo-128 \ + --port 30000 --host 0.0.0.0 +``` + +SGLang supports the following quantization methods based on torchao `["int8dq", "int8wo", "fp8wo", "fp8dq-per_tensor", "fp8dq-per_row", "int4wo-32", "int4wo-64", "int4wo-128", "int4wo-256"]`. + +Note: According to [this issue](https://github.com/sgl-project/sglang/issues/2219#issuecomment-2561890230), `"int8dq"` method currently has some bugs when using together with cuda graph capture. So we suggest to disable cuda graph capture when using `"int8dq"` method. Namely, please use the following command: + +```bash +python3 -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ + --torchao-config int8dq \ + --disable-cuda-graph \ + --port 30000 --host 0.0.0.0 +``` + +## Reference + +- [GPTQModel](https://github.com/ModelCloud/GPTQModel) +- [LLM Compressor](https://github.com/vllm-project/llm-compressor/) +- [NVIDIA Model Optimizer (ModelOpt)](https://github.com/NVIDIA/TensorRT-Model-Optimizer) +- [Torchao: PyTorch Architecture Optimization](https://github.com/pytorch/ao) +- [vLLM Quantization](https://docs.vllm.ai/en/latest/quantization/) +- [auto-round](https://github.com/intel/auto-round) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/router.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/router.md new file mode 100644 index 00000000..388b86cd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/router.md @@ -0,0 +1,470 @@ +# SGLang Model Gateway (formerly SGLang Router) + +SGLang Model Gateway is a high-performance model-routing gateway for large-scale LLM deployments. It centralizes worker lifecycle management, balances traffic across heterogeneous protocols (HTTP, gRPC, OpenAI-compatible), and provides enterprise-ready control over history storage, MCP tooling, and privacy-sensitive workflows. The router is deeply optimized for the SGLang serving runtime, but can route to any OpenAI-compatible backend. + +--- + +## Table of Contents +1. [Overview](#overview) +2. [Architecture](#architecture) + - [Control Plane](#control-plane) + - [Data Plane](#data-plane) + - [Storage & Privacy](#storage--privacy) +3. [Deployment Modes](#deployment-modes) + - [Co-launch Router + Workers](#co-launch-router--workers) + - [Separate Launch (HTTP)](#separate-launch-http) + - [gRPC Launch](#grpc-launch) + - [Prefill/Decode Disaggregation](#prefilldecode-disaggregation) + - [OpenAI Backend Proxy](#openai-backend-proxy) +4. [Worker Lifecycle & Dynamic Scaling](#worker-lifecycle--dynamic-scaling) +5. [Reliability & Flow Control](#reliability--flow-control) +6. [Load Balancing Policies](#load-balancing-policies) +7. [Service Discovery (Kubernetes)](#service-discovery-kubernetes) +8. [Security & Authentication](#security--authentication) +9. [History & Data Connectors](#history--data-connectors) +10. [MCP & Advanced Tooling](#mcp--advanced-tooling) +11. [API Surface](#api-surface) +12. [Configuration Reference](#configuration-reference) +13. [Observability](#observability) +14. [Troubleshooting](#troubleshooting) + +--- + +## Overview +- **Unified control plane** for registering, monitoring, and orchestrating regular, prefill, and decode workers across heterogeneous model fleets. +- **Multi-protocol data plane** that routes traffic across HTTP, PD (prefill/decode), gRPC, and OpenAI-compatible backends with shared reliability primitives. +- **Industry-first gRPC pipeline** with native Rust tokenization, reasoning parsers, and tool-call execution for high-throughput, OpenAI-compatible serving; supports both single-stage and PD topologies. +- **Inference Gateway Mode (`--enable-igw`)** dynamically instantiates multiple router stacks (HTTP regular/PD, gRPC) and applies per-model policies for multi-tenant deployments. +- **Conversation & responses connectors** centralize chat history inside the router so the same context can be reused across models and MCP loops without leaking data to upstream vendors (memory, none, Oracle ATP). +- **Enterprise privacy**: agentic multi-turn `/v1/responses`, native MCP client (STDIO/HTTP/SSE/Streamable), and history storage all operate within the router boundary. +- **Reliability core**: retries with jitter, worker-scoped circuit breakers, token-bucket rate limiting with queuing, background health checks, and cache-aware load monitoring. +- **Observability**: Prometheus metrics, structured tracing, request ID propagation, and detailed job queue stats. + +--- + +## Architecture + +### Control Plane +- **Worker Manager** discovers capabilities (`/get_server_info`, `/get_model_info`), tracks load, and registers/removes workers in the shared registry. +- **Job Queue** serializes add/remove requests and exposes status (`/workers/{url}`) so clients can track onboarding progress. +- **Load Monitor** feeds cache-aware and power-of-two policies with live worker load statistics. +- **Health Checker** continuously probes workers and updates readiness, circuit breaker state, and router metrics. + +### Data Plane +- **HTTP routers** (regular & PD) implement `/generate`, `/v1/chat/completions`, `/v1/completions`, `/v1/responses`, `/v1/embeddings`, `/v1/rerank`, and associated admin endpoints. +- **gRPC router** streams tokenized requests directly to SRT gRPC workers, running fully in Rust—tokenizer, reasoning parser, and tool parser all reside in-process. Supports both single-stage and PD routing. +- **OpenAI router** proxies OpenAI-compatible endpoints to external vendors (OpenAI, xAI, etc.) while keeping chat history and multi-turn orchestration local. + +### Storage & Privacy +- Conversation and response history is stored at the router tier (memory, none, or Oracle ATP). The same history can power multiple models or MCP loops without sending data to upstream vendors. +- `/v1/responses` agentic flows, MCP sessions, and conversation APIs share the same storage layer, enabling compliance for regulated workloads. + +--- + +## Deployment Modes + +### Co-launch Router + Workers +Launch the router and a fleet of SGLang workers in one process (ideal for single-node or quick starts). The CLI accepts two namespaces of arguments: +- **Worker arguments** (no prefix) configure the SGLang runtime (`--model`, `--tp-size`, `--dp-size`, `--grpc-mode`, etc.). +- **Router arguments** are prefixed with `--router-` and map directly to `launch_router` flags (`--router-policy`, `--router-model-path`, `--router-log-level`, ...). + +```bash +python -m sglang_router.launch_server \ + --model meta-llama/Meta-Llama-3.1-8B-Instruct \ + --dp-size 4 \ + --host 0.0.0.0 \ + --port 30000 +``` + +Comprehensive example: +```bash +python3 -m sglang_router.launch_server \ + --host 0.0.0.0 \ + --port 8080 \ + --model meta-llama/Llama-3.1-8B-Instruct \ + --tp-size 1 \ + --dp-size 8 \ + --grpc-mode \ + --log-level debug \ + --router-prometheus-port 10001 \ + --router-tool-call-parser llama \ + --router-health-success-threshold 2 \ + --router-health-check-timeout-secs 6000 \ + --router-health-check-interval-secs 60 \ + --router-model-path meta-llama/Llama-3.1-8B-Instruct \ + --router-policy round_robin \ + --router-log-level debug +``` + +### Separate Launch (HTTP) +Run workers independently and point the router at their HTTP endpoints. + +```bash +# Worker nodes +python -m sglang.launch_server --model meta-llama/Meta-Llama-3.1-8B-Instruct --port 8000 +python -m sglang.launch_server --model meta-llama/Meta-Llama-3.1-8B-Instruct --port 8001 + +# Router node +python -m sglang_router.launch_router \ + --worker-urls http://worker1:8000 http://worker2:8001 \ + --policy cache_aware \ + --host 0.0.0.0 --port 30000 +``` + +### gRPC Launch +Use SRT gRPC workers to unlock the highest throughput and access native reasoning/tool pipelines. + +```bash +# Workers expose gRPC endpoints +python -m sglang.launch_server \ + --model meta-llama/Llama-3.1-8B-Instruct \ + --grpc-mode \ + --port 20000 + +# Router +python -m sglang_router.launch_router \ + --worker-urls grpc://127.0.0.1:20000 \ + --model-path meta-llama/Llama-3.1-8B-Instruct \ + --reasoning-parser deepseek-r1 \ + --tool-call-parser json \ + --host 0.0.0.0 --port 8080 +``` + +> gRPC router supports both single-stage and PD serving. Provide `--tokenizer-path` or `--model-path` (HF repo or local directory) plus optional `--chat-template`. + +### Prefill/Decode Disaggregation +Split prefill and decode workers for PD-aware caching and balancing. + +```bash +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --prefill http://prefill1:30001 9001 \ + --decode http://decode1:30011 \ + --policy cache_aware \ + --prefill-policy cache_aware \ + --decode-policy power_of_two +``` + +### OpenAI Backend Proxy +Proxy OpenAI-compatible endpoints (OpenAI, xAI, etc.) while keeping history and MCP sessions local. + +```bash +python -m sglang_router.launch_router \ + --backend openai \ + --worker-urls https://api.openai.com \ + --history-backend memory +``` + +> OpenAI backend mode expects exactly one `--worker-urls` entry per router instance. + +--- + +## Worker Lifecycle & Dynamic Scaling + +Add or remove workers at runtime using the REST APIs. Jobs are queued and tracked for eventual consistency. + +```bash +# Add a worker (HTTP or gRPC) +curl -X POST http://localhost:30000/workers \ + -H "Content-Type: application/json" \ + -d '{"url":"grpc://0.0.0.0:31000","worker_type":"regular"}' + +# Inspect registry +curl http://localhost:30000/workers + +# Remove a worker +curl -X DELETE http://localhost:30000/workers/grpc://0.0.0.0:31000 +``` + +Legacy endpoints (`/add_worker`, `/remove_worker`, `/list_workers`) remain available but will be deprecated. `/workers/{url}` returns both registry data and queued job status. + +--- + +## Reliability & Flow Control + +### Retries +```bash +python -m sglang_router.launch_router \ + --worker-urls http://worker1:8000 http://worker2:8001 \ + --retry-max-retries 5 \ + --retry-initial-backoff-ms 50 \ + --retry-max-backoff-ms 30000 \ + --retry-backoff-multiplier 1.5 \ + --retry-jitter-factor 0.2 +``` + +### Circuit Breaker +```bash +python -m sglang_router.launch_router \ + --worker-urls http://worker1:8000 http://worker2:8001 \ + --cb-failure-threshold 5 \ + --cb-success-threshold 2 \ + --cb-timeout-duration-secs 30 \ + --cb-window-duration-secs 60 +``` + +### Rate Limiting & Queuing +```bash +python -m sglang_router.launch_router \ + --worker-urls http://worker1:8000 http://worker2:8001 \ + --max-concurrent-requests 256 \ + --rate-limit-tokens-per-second 512 \ + --queue-size 128 \ + --queue-timeout-secs 30 +``` + +Requests beyond the concurrency limit wait in a FIFO queue (up to `queue-size`). A `429` is returned when the queue is full; `408` is returned when `queue-timeout-secs` expires. + +--- + +## Load Balancing Policies + +| Policy | Description | Usage | +|--------------------|--------------------------------------------------------------------------------------------------|-------------------------------| +| `random` | Uniform random selection. | `--policy random` | +| `round_robin` | Cycles through workers in order. | `--policy round_robin` | +| `power_of_two` | Samples two workers and picks the lighter one (requires Load Monitor). | `--policy power_of_two` | +| `cache_aware` | Default policy; combines cache locality with load balancing, falling back to shortest queue. | `--policy cache_aware` + tuning flags | + +Key tuning flags: +```bash +--cache-threshold 0.5 \ +--balance-abs-threshold 32 \ +--balance-rel-threshold 1.5 \ +--eviction-interval-secs 120 \ +--max-tree-size 67108864 +``` + +--- + +## Service Discovery (Kubernetes) + +Enable automatic worker discovery via Kubernetes pod selectors. + +```bash +python -m sglang_router.launch_router \ + --service-discovery \ + --selector app=sglang-worker role=inference \ + --service-discovery-namespace production \ + --service-discovery-port 8000 +``` + +PD deployments can specify `--prefill-selector` and `--decode-selector` plus the `sglang.ai/bootstrap-port` annotation for prefill bootstrap ports. Ensure RBAC grants `get/list/watch` on pods. + +--- + +## Security & Authentication + +- **Router API key (`--api-key`)**: clients must supply `Authorization: Bearer `. +- **Worker API keys**: when adding workers dynamically, include `api_key` in the payload; workers listed via CLI inherit the router key. +- **Full-stack auth**: start router with `--api-key`, then add workers with their own keys: + ```bash + curl -H "Authorization: Bearer router-key" \ + -X POST http://localhost:30000/workers \ + -H "Content-Type: application/json" \ + -d '{"url":"http://worker:8000","api_key":"worker-key"}' + ``` +- **Privacy**: All conversation history, `/v1/responses` state, and MCP sessions stay inside the router. Nothing is persisted at remote model vendors unless explicitly proxied. + +--- + +## History & Data Connectors + +| Backend | Description | Usage | +|---------|-------------|-------| +| `memory` (default) | In-memory storage for quick prototyping. | `--history-backend memory` | +| `none` | No persistence; APIs operate but store nothing. | `--history-backend none` | +| `oracle` | Oracle Autonomous Database-backed storage (pooled connections). | `--history-backend oracle` | +| `postgres` | PostgreSQL Database-backed storage (pooled connections). | `--history-backend postgres` | + +Oracle configuration (choose DSN *or* TNS alias): +Install the Oracle Instant Client and set `LD_LIBRARY_PATH` accordingly. +Choose **one** connection method: +```bash +# Option 1: Full connection descriptor +export ATP_DSN="(description=(address=(protocol=tcps)(port=1522)(host=adb.region.oraclecloud.com))(connect_data=(service_name=service_name)))" + +# Option 2: TNS alias (requires wallet) +export ATP_TNS_ALIAS="sglroutertestatp_high" +export ATP_WALLET_PATH="/path/to/wallet" +``` +Provide database credentials and optional pool sizing: +```bash +export ATP_USER="admin" +export ATP_PASSWORD="secret" +export ATP_POOL_MIN=4 +export ATP_POOL_MAX=32 + +python -m sglang_router.launch_router \ + --backend openai \ + --worker-urls https://api.openai.com \ + --history-backend oracle +``` + +> History backends currently apply to OpenAI router mode. gRPC parity for `/v1/responses` is on the roadmap. + +--- + +## MCP & Advanced Tooling + +- Native MCP client supports **STDIO**, **HTTP**, **SSE**, and **Streamable** transports—no external config files required. +- Tool-call parsers cover JSON, Pythonic, XML, and custom schemas with streaming/non-streaming execution loops. +- Reasoning parsers ship for DeepSeek-R1, Qwen3, Step-3, GLM4, Llama families, Kimi K2, GPT-OSS, Mistral, and more (`src/reasoning_parser`). +- Tokenizer factory accepts HuggingFace IDs, local directories, and explicit `tokenizer.json` files with chat template overrides (`src/tokenizer`). + +Use CLI flags to select parsers: +```bash +--reasoning-parser deepseek-r1 \ +--tool-call-parser json \ +--chat-template /path/to/template.json +``` + +--- + +## API Surface + +| Method | Path | Description | +|-----------------------|------------------------------------------|------------------------------------------------| +| `POST` | `/generate` | SGLang generate API. | +| `POST` | `/v1/chat/completions` | OpenAI-compatible chat (streaming/tool calls). | +| `POST` | `/v1/completions` | OpenAI-compatible text completions. | +| `POST` | `/v1/responses` | Create background responses (agentic loops). | +| `GET` | `/v1/responses/{id}` | Retrieve stored responses. | +| `POST` | `/v1/embeddings` | Forward embedding requests. | +| `POST` | `/v1/rerank` | Ranking endpoint (`/rerank` synonym). | +| `POST` | `/v1/conversations` | Create conversation metadata. | +| `GET`/`POST`/`DELETE` | `/v1/conversations/{id}` | Get/update/delete conversation. | +| `GET`/`POST` | `/v1/conversations/{id}/items` | List or append conversation items. | +| `GET`/`DELETE` | `/v1/conversations/{id}/items/{item_id}` | Inspect/delete conversation item. | +| `GET` | `/workers` | List registered workers with health/load. | +| `POST` | `/workers` | Queue worker registration. | +| `DELETE` | `/workers/{url}` | Queue worker removal. | +| `POST` | `/flush_cache` | Flush worker caches (HTTP workers). | +| `GET` | `/get_loads` | Retrieve worker load snapshot. | +| `GET` | `/liveness` / `/readiness` / `/health` | Health probes. | + +--- + +## Configuration Reference + +### Core Settings + +| Parameter | Type | Default | Description | +|-----------------------------|------|-------------|--------------------------------------------------------------------------| +| `--host` | str | 127.0.0.1 | Router host. | +| `--port` | int | 30000 | Router port. | +| `--worker-urls` | list | [] | Worker URLs (HTTP or gRPC). | +| `--policy` | str | cache_aware | Routing policy (`random`, `round_robin`, `cache_aware`, `power_of_two`). | +| `--max-concurrent-requests` | int | -1 | Concurrency limit (-1 disables rate limiting). | +| `--request-timeout-secs` | int | 600 | Request timeout. | +| `--max-payload-size` | int | 256MB | Maximum request payload. | + +### Cache-Aware Tuning + +| Parameter | Type | Default | Description | +|----------------------------|-------|----------|-----------------------------| +| `--cache-threshold` | float | 0.3 | Minimum prefix match ratio. | +| `--balance-abs-threshold` | int | 64 | Absolute load threshold. | +| `--balance-rel-threshold` | float | 1.5 | Relative load ratio. | +| `--eviction-interval-secs` | int | 120 | Cache eviction cadence. | +| `--max-tree-size` | int | 67108864 | Max nodes in cache tree. | + +### Fault Tolerance + +| Parameter | Type | Default | Description | +|------------------------------|-------|---------|----------------------------------| +| `--retry-max-retries` | int | 5 | Max retries. | +| `--retry-initial-backoff-ms` | int | 50 | Initial backoff (ms). | +| `--retry-max-backoff-ms` | int | 30000 | Max backoff (ms). | +| `--retry-backoff-multiplier` | float | 1.5 | Backoff multiplier. | +| `--retry-jitter-factor` | float | 0.2 | Retry jitter (0.0-1.0). | +| `--disable-retries` | flag | False | Disable retries. | +| `--cb-failure-threshold` | int | 5 | Failures before opening circuit. | +| `--cb-success-threshold` | int | 2 | Successes to close circuit. | +| `--cb-timeout-duration-secs` | int | 30 | Cooldown period. | +| `--cb-window-duration-secs` | int | 60 | Window size. | +| `--disable-circuit-breaker` | flag | False | Disable circuit breaker. | + +### Prefill/Decode + +| Parameter | Type | Default | Description | +|-----------------------------------|------|---------|------------------------------------------| +| `--pd-disaggregation` | flag | False | Enable PD mode. | +| `--prefill` | list | [] | Prefill URLs + optional bootstrap ports. | +| `--decode` | list | [] | Decode URLs. | +| `--prefill-policy` | str | None | Override policy for prefill nodes. | +| `--decode-policy` | str | None | Override policy for decode nodes. | +| `--worker-startup-timeout-secs` | int | 600 | Worker init timeout. | +| `--worker-startup-check-interval` | int | 30 | Polling interval. | + +### Kubernetes Discovery + +| Parameter | Type | Description | +|--------------------------------------------|------|--------------------------------------------------------------------| +| `--service-discovery` | flag | Enable discovery. | +| `--selector key=value ...` | list | Label selectors (regular mode). | +| `--prefill-selector` / `--decode-selector` | list | Label selectors for PD mode. | +| `--service-discovery-namespace` | str | Namespace to watch. | +| `--service-discovery-port` | int | Worker port (default 80). | +| `--bootstrap-port-annotation` | str | Prefill bootstrap annotation (default `sglang.ai/bootstrap-port`). | + +--- + +## Observability + +Enable Prometheus metrics: +```bash +python -m sglang_router.launch_router \ + --worker-urls http://worker1:8000 http://worker2:8001 \ + --prometheus-host 0.0.0.0 \ + --prometheus-port 29000 +``` + +Key metrics: + +| Metric | Type | Description | +|--------|------|-------------| +| `sgl_router_requests_total` | Counter | Total requests by endpoint/method. | +| `sgl_router_processed_requests_total` | Counter | Requests processed per worker. | +| `sgl_router_active_workers` | Gauge | Healthy worker count. | +| `sgl_router_running_requests` | Gauge | In-flight requests per worker. | +| `sgl_router_cache_hits_total` / `misses_total` | Counter | Cache-aware routing hits/misses. | +| `sgl_router_generate_duration_seconds` | Histogram | Request latency distribution. | + +Enable request ID propagation: +```bash +python -m sglang_router.launch_router \ + --worker-urls http://worker1:8000 \ + --request-id-headers x-request-id x-trace-id +``` + +--- + +## Troubleshooting + +1. **Workers never ready** + Increase `--worker-startup-timeout-secs` or ensure health probes respond before router startup. + +2. **Load imbalance / hot workers** + Inspect `sgl_router_processed_requests_total` and tune cache-aware thresholds (`--balance-*`, `--cache-threshold`). + +3. **Circuit breaker flapping** + Increase `--cb-failure-threshold` or extend the timeout/window durations. Consider temporarily disabling retries. + +4. **Queue overflow (429)** + Increase `--queue-size` or reduce client concurrency. Ensure `--max-concurrent-requests` matches downstream capacity. + +5. **Memory growth** + Reduce `--max-tree-size` or lower `--eviction-interval-secs` for more aggressive cache pruning. + +6. **Debugging** + ```bash + python -m sglang_router.launch_router \ + --worker-urls http://worker1:8000 \ + --log-level debug \ + --log-dir ./router_logs + ``` + +--- + +SGLang Model Gateway continues to evolve alongside the SGLang runtime. Keep CLI flags, integrations, and documentation aligned when adopting new features or contributing improvements. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/separate_reasoning.ipynb b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/separate_reasoning.ipynb new file mode 100644 index 00000000..fa24e63b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/separate_reasoning.ipynb @@ -0,0 +1,377 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Reasoning Parser\n", + "\n", + "SGLang supports parsing reasoning content out from \"normal\" content for reasoning models such as [DeepSeek R1](https://huggingface.co/deepseek-ai/DeepSeek-R1).\n", + "\n", + "## Supported Models & Parsers\n", + "\n", + "| Model | Reasoning tags | Parser | Notes |\n", + "|---------|-----------------------------|------------------|-------|\n", + "| [DeepSeek‑R1 series](https://huggingface.co/collections/deepseek-ai/deepseek-r1-678e1e131c0169c0bc89728d) | `` … `` | `deepseek-r1` | Supports all variants (R1, R1-0528, R1-Distill) |\n", + "| [DeepSeek‑V3 series](https://huggingface.co/deepseek-ai/DeepSeek-V3.1) | `` … `` | `deepseek-v3` | Including [DeepSeek‑V3.2](https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp). Supports `thinking` parameter |\n", + "| [Standard Qwen3 models](https://huggingface.co/collections/Qwen/qwen3-67dd247413f0e2e4f653967f) | `` … `` | `qwen3` | Supports `enable_thinking` parameter |\n", + "| [Qwen3-Thinking models](https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507) | `` … `` | `qwen3` or `qwen3-thinking` | Always generates thinking content |\n", + "| [Kimi models](https://huggingface.co/moonshotai/models) | `◁think▷` … `◁/think▷` | `kimi` | Uses special thinking delimiters |\n", + "| [GPT OSS](https://huggingface.co/openai/gpt-oss-120b) | `<\\|channel\\|>analysis<\\|message\\|>` … `<\\|end\\|>` | `gpt-oss` | N/A |\n", + "### Model-Specific Behaviors\n", + "\n", + "**DeepSeek-R1 Family:**\n", + "- DeepSeek-R1: No `` start tag, jumps directly to thinking content\n", + "- DeepSeek-R1-0528: Generates both `` start and `` end tags\n", + "- Both are handled by the same `deepseek-r1` parser\n", + "\n", + "**DeepSeek-V3 Family:**\n", + "- DeepSeek-V3.1/V3.2: Hybrid model supporting both thinking and non-thinking modes, use the `deepseek-v3` parser and `thinking` parameter (NOTE: not `enable_thinking`)\n", + "\n", + "**Qwen3 Family:**\n", + "- Standard Qwen3 (e.g., Qwen3-2507): Use `qwen3` parser, supports `enable_thinking` in chat templates\n", + "- Qwen3-Thinking (e.g., Qwen3-235B-A22B-Thinking-2507): Use `qwen3` or `qwen3-thinking` parser, always thinks\n", + "\n", + "**Kimi:**\n", + "- Kimi: Uses special `◁think▷` and `◁/think▷` tags\n", + "\n", + "**GPT OSS:**\n", + "- GPT OSS: Uses special `<|channel|>analysis<|message|>` and `<|end|>` tags" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Usage\n", + "\n", + "### Launching the Server" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Specify the `--reasoning-parser` option." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "from openai import OpenAI\n", + "from sglang.test.doc_patch import launch_server_cmd\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-7B --host 0.0.0.0 --reasoning-parser deepseek-r1 --log-level warning\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that `--reasoning-parser` defines the parser used to interpret responses." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### OpenAI Compatible API\n", + "\n", + "Using the OpenAI compatible API, the contract follows the [DeepSeek API design](https://api-docs.deepseek.com/guides/reasoning_model) established with the release of DeepSeek-R1:\n", + "\n", + "- `reasoning_content`: The content of the CoT.\n", + "- `content`: The content of the final answer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize OpenAI-like client\n", + "client = OpenAI(api_key=\"None\", base_url=f\"http://0.0.0.0:{port}/v1\")\n", + "model_name = client.models.list().data[0].id\n", + "\n", + "messages = [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"What is 1+3?\",\n", + " }\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Non-Streaming Request" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response_non_stream = client.chat.completions.create(\n", + " model=model_name,\n", + " messages=messages,\n", + " temperature=0.6,\n", + " top_p=0.95,\n", + " stream=False, # Non-streaming\n", + " extra_body={\"separate_reasoning\": True},\n", + ")\n", + "print_highlight(\"==== Reasoning ====\")\n", + "print_highlight(response_non_stream.choices[0].message.reasoning_content)\n", + "\n", + "print_highlight(\"==== Text ====\")\n", + "print_highlight(response_non_stream.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Streaming Request" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response_stream = client.chat.completions.create(\n", + " model=model_name,\n", + " messages=messages,\n", + " temperature=0.6,\n", + " top_p=0.95,\n", + " stream=True, # Non-streaming\n", + " extra_body={\"separate_reasoning\": True},\n", + ")\n", + "\n", + "reasoning_content = \"\"\n", + "content = \"\"\n", + "for chunk in response_stream:\n", + " if chunk.choices[0].delta.content:\n", + " content += chunk.choices[0].delta.content\n", + " if chunk.choices[0].delta.reasoning_content:\n", + " reasoning_content += chunk.choices[0].delta.reasoning_content\n", + "\n", + "print_highlight(\"==== Reasoning ====\")\n", + "print_highlight(reasoning_content)\n", + "\n", + "print_highlight(\"==== Text ====\")\n", + "print_highlight(content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Optionally, you can buffer the reasoning content to the last reasoning chunk (or the first chunk after the reasoning content)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response_stream = client.chat.completions.create(\n", + " model=model_name,\n", + " messages=messages,\n", + " temperature=0.6,\n", + " top_p=0.95,\n", + " stream=True, # Non-streaming\n", + " extra_body={\"separate_reasoning\": True, \"stream_reasoning\": False},\n", + ")\n", + "\n", + "reasoning_content = \"\"\n", + "content = \"\"\n", + "for chunk in response_stream:\n", + " if chunk.choices[0].delta.content:\n", + " content += chunk.choices[0].delta.content\n", + " if chunk.choices[0].delta.reasoning_content:\n", + " reasoning_content += chunk.choices[0].delta.reasoning_content\n", + "\n", + "print_highlight(\"==== Reasoning ====\")\n", + "print_highlight(reasoning_content)\n", + "\n", + "print_highlight(\"==== Text ====\")\n", + "print_highlight(content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The reasoning separation is enable by default when specify . \n", + "**To disable it, set the `separate_reasoning` option to `False` in request.**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response_non_stream = client.chat.completions.create(\n", + " model=model_name,\n", + " messages=messages,\n", + " temperature=0.6,\n", + " top_p=0.95,\n", + " stream=False, # Non-streaming\n", + " extra_body={\"separate_reasoning\": False},\n", + ")\n", + "\n", + "print_highlight(\"==== Original Output ====\")\n", + "print_highlight(response_non_stream.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### SGLang Native API " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoTokenizer\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n", + "input = tokenizer.apply_chat_template(\n", + " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", + ")\n", + "\n", + "gen_url = f\"http://localhost:{port}/generate\"\n", + "gen_data = {\n", + " \"text\": input,\n", + " \"sampling_params\": {\n", + " \"skip_special_tokens\": False,\n", + " \"max_new_tokens\": 1024,\n", + " \"temperature\": 0.6,\n", + " \"top_p\": 0.95,\n", + " },\n", + "}\n", + "gen_response = requests.post(gen_url, json=gen_data).json()[\"text\"]\n", + "\n", + "print_highlight(\"==== Original Output ====\")\n", + "print_highlight(gen_response)\n", + "\n", + "parse_url = f\"http://localhost:{port}/separate_reasoning\"\n", + "separate_reasoning_data = {\n", + " \"text\": gen_response,\n", + " \"reasoning_parser\": \"deepseek-r1\",\n", + "}\n", + "separate_reasoning_response_json = requests.post(\n", + " parse_url, json=separate_reasoning_data\n", + ").json()\n", + "print_highlight(\"==== Reasoning ====\")\n", + "print_highlight(separate_reasoning_response_json[\"reasoning_text\"])\n", + "print_highlight(\"==== Text ====\")\n", + "print_highlight(separate_reasoning_response_json[\"text\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Offline Engine API" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sglang as sgl\n", + "from sglang.srt.parser.reasoning_parser import ReasoningParser\n", + "from sglang.utils import print_highlight\n", + "\n", + "llm = sgl.Engine(model_path=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n", + "tokenizer = AutoTokenizer.from_pretrained(\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n", + "input = tokenizer.apply_chat_template(\n", + " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", + ")\n", + "sampling_params = {\n", + " \"max_new_tokens\": 1024,\n", + " \"skip_special_tokens\": False,\n", + " \"temperature\": 0.6,\n", + " \"top_p\": 0.95,\n", + "}\n", + "result = llm.generate(prompt=input, sampling_params=sampling_params)\n", + "\n", + "generated_text = result[\"text\"] # Assume there is only one prompt\n", + "\n", + "print_highlight(\"==== Original Output ====\")\n", + "print_highlight(generated_text)\n", + "\n", + "parser = ReasoningParser(\"deepseek-r1\")\n", + "reasoning_text, text = parser.parse_non_stream(generated_text)\n", + "print_highlight(\"==== Reasoning ====\")\n", + "print_highlight(reasoning_text)\n", + "print_highlight(\"==== Text ====\")\n", + "print_highlight(text)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "llm.shutdown()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Supporting New Reasoning Model Schemas\n", + "\n", + "For future reasoning models, you can implement the reasoning parser as a subclass of `BaseReasoningFormatDetector` in `python/sglang/srt/reasoning_parser.py` and specify the reasoning parser for new reasoning model schemas accordingly." + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/server_arguments.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/server_arguments.md new file mode 100644 index 00000000..891cbacf --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/server_arguments.md @@ -0,0 +1,467 @@ +# Server Arguments + +This page provides a list of server arguments used in the command line to configure the behavior +and performance of the language model server during deployment. These arguments enable users to +customize key aspects of the server, including model selection, parallelism policies, +memory management, and optimization techniques. +You can find all arguments by `python3 -m sglang.launch_server --help` + +## Common launch commands + +- To use a configuration file, create a YAML file with your server arguments and specify it with `--config`. CLI arguments will override config file values. + + ```bash + # Create config.yaml + cat > config.yaml << EOF + model-path: meta-llama/Meta-Llama-3-8B-Instruct + host: 0.0.0.0 + port: 30000 + tensor-parallel-size: 2 + enable-metrics: true + log-requests: true + EOF + + # Launch server with config file + python -m sglang.launch_server --config config.yaml + ``` + +- To enable multi-GPU tensor parallelism, add `--tp 2`. If it reports the error "peer access is not supported between these two devices", add `--enable-p2p-check` to the server launch command. + + ```bash + python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --tp 2 + ``` + +- To enable multi-GPU data parallelism, add `--dp 2`. Data parallelism is better for throughput if there is enough memory. It can also be used together with tensor parallelism. The following command uses 4 GPUs in total. We recommend [SGLang Router](../advanced_features/router.md) for data parallelism. + + ```bash + python -m sglang_router.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --dp 2 --tp 2 + ``` + +- If you see out-of-memory errors during serving, try to reduce the memory usage of the KV cache pool by setting a smaller value of `--mem-fraction-static`. The default value is `0.9`. + + ```bash + python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --mem-fraction-static 0.7 + ``` + +- See [hyperparameter tuning](hyperparameter_tuning.md) on tuning hyperparameters for better performance. +- For docker and Kubernetes runs, you need to set up shared memory which is used for communication between processes. See `--shm-size` for docker and `/dev/shm` size update for Kubernetes manifests. +- If you see out-of-memory errors during prefill for long prompts, try to set a smaller chunked prefill size. + + ```bash + python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --chunked-prefill-size 4096 + ``` + +- To enable `torch.compile` acceleration, add `--enable-torch-compile`. It accelerates small models on small batch sizes. By default, the cache path is located at `/tmp/torchinductor_root`, you can customize it using environment variable `TORCHINDUCTOR_CACHE_DIR`. For more details, please refer to [PyTorch official documentation](https://pytorch.org/tutorials/recipes/torch_compile_caching_tutorial.html) and [Enabling cache for torch.compile](https://docs.sglang.io/references/torch_compile_cache.html). +- To enable torchao quantization, add `--torchao-config int4wo-128`. It supports other [quantization strategies (INT8/FP8)](https://github.com/sgl-project/sglang/blob/v0.3.6/python/sglang/srt/server_args.py#L671) as well. +- To enable fp8 weight quantization, add `--quantization fp8` on a fp16 checkpoint or directly load a fp8 checkpoint without specifying any arguments. +- To enable fp8 kv cache quantization, add `--kv-cache-dtype fp8_e5m2`. +- To enable deterministic inference and batch invariant operations, add `--enable-deterministic-inference`. More details can be found in [deterministic inference document](../advanced_features/deterministic_inference.md). +- If the model does not have a chat template in the Hugging Face tokenizer, you can specify a [custom chat template](../references/custom_chat_template.md). +- To run tensor parallelism on multiple nodes, add `--nnodes 2`. If you have two nodes with two GPUs on each node and want to run TP=4, let `sgl-dev-0` be the hostname of the first node and `50000` be an available port, you can use the following commands. If you meet deadlock, please try to add `--disable-cuda-graph` + + ```bash + # Node 0 + python -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3-8B-Instruct \ + --tp 4 \ + --dist-init-addr sgl-dev-0:50000 \ + --nnodes 2 \ + --node-rank 0 + + # Node 1 + python -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3-8B-Instruct \ + --tp 4 \ + --dist-init-addr sgl-dev-0:50000 \ + --nnodes 2 \ + --node-rank 1 + ``` + +Please consult the documentation below and [server_args.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/server_args.py) to learn more about the arguments you may provide when launching a server. + +## Model and tokenizer +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--model-path`
`--model` | The path of the model weights. This can be a local folder or a Hugging Face repo ID. | `None` | Type: str | +| `--tokenizer-path` | The path of the tokenizer. | `None` | Type: str | +| `--tokenizer-mode` | Tokenizer mode. 'auto' will use the fast tokenizer if available, and 'slow' will always use the slow tokenizer. | `auto` | `auto`, `slow` | +| `--tokenizer-worker-num` | The worker num of the tokenizer manager. | `1` | Type: int | +| `--skip-tokenizer-init` | If set, skip init tokenizer and pass input_ids in generate request. | `False` | bool flag (set to enable) | +| `--load-format` | The format of the model weights to load. "auto" will try to load the weights in the safetensors format and fall back to the pytorch bin format if safetensors format is not available. "pt" will load the weights in the pytorch bin format. "safetensors" will load the weights in the safetensors format. "npcache" will load the weights in pytorch format and store a numpy cache to speed up the loading. "dummy" will initialize the weights with random values, which is mainly for profiling."gguf" will load the weights in the gguf format. "bitsandbytes" will load the weights using bitsandbytes quantization."layered" loads weights layer by layer so that one can quantize a layer before loading another to make the peak memory envelope smaller. | `auto` | `auto`, `pt`, `safetensors`, `npcache`, `dummy`, `sharded_state`, `gguf`, `bitsandbytes`, `layered`, `remote`, `remote_instance` | +| `--model-loader-extra-config` | Extra config for model loader. This will be passed to the model loader corresponding to the chosen load_format. | `{}` | Type: str | +| `--trust-remote-code` | Whether or not to allow for custom models defined on the Hub in their own modeling files. | `False` | bool flag (set to enable) | +| `--context-length` | The model's maximum context length. Defaults to None (will use the value from the model's config.json instead). | `None` | Type: int | +| `--is-embedding` | Whether to use a CausalLM as an embedding model. | `False` | bool flag (set to enable) | +| `--enable-multimodal` | Enable the multimodal functionality for the served model. If the model being served is not multimodal, nothing will happen | `None` | bool flag (set to enable) | +| `--revision` | The specific model version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version. | `None` | Type: str | +| `--model-impl` | Which implementation of the model to use. * "auto" will try to use the SGLang implementation if it exists and fall back to the Transformers implementation if no SGLang implementation is available. * "sglang" will use the SGLang model implementation. * "transformers" will use the Transformers model implementation. | `auto` | Type: str | + +## HTTP server +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--host` | The host of the HTTP server. | `127.0.0.1` | Type: str | +| `--port` | The port of the HTTP server. | `30000` | Type: int | +| `--skip-server-warmup` | If set, skip warmup. | `False` | bool flag (set to enable) | +| `--warmups` | Specify custom warmup functions (csv) to run before server starts eg. --warmups=warmup_name1,warmup_name2 will run the functions `warmup_name1` and `warmup_name2` specified in warmup.py before the server starts listening for requests | `None` | Type: str | +| `--nccl-port` | The port for NCCL distributed environment setup. Defaults to a random port. | `None` | Type: int | + +## Quantization and data type +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--dtype` | Data type for model weights and activations. * "auto" will use FP16 precision for FP32 and FP16 models, and BF16 precision for BF16 models. * "half" for FP16. Recommended for AWQ quantization. * "float16" is the same as "half". * "bfloat16" for a balance between precision and range. * "float" is shorthand for FP32 precision. * "float32" for FP32 precision. | `auto` | `auto`, `half`, `float16`, `bfloat16`, `float`, `float32` | +| `--quantization` | The quantization method. | `None` | `awq`, `fp8`, `gptq`, `marlin`, `gptq_marlin`, `awq_marlin`, `bitsandbytes`, `gguf`, `modelopt`, `modelopt_fp4`, `petit_nvfp4`, `w8a8_int8`, `w8a8_fp8`, `moe_wna16`, `qoq`, `w4afp8`, `mxfp4` | +| `--quantization-param-path` | Path to the JSON file containing the KV cache scaling factors. This should generally be supplied, when KV cache dtype is FP8. Otherwise, KV cache scaling factors default to 1.0, which may cause accuracy issues. | `None` | Type: Optional[str] | +| `--modelopt-quant` | The ModelOpt quantization configuration. Supported values: 'fp8', 'int4_awq', 'w4a8_awq', 'nvfp4', 'nvfp4_awq'. This requires the NVIDIA Model Optimizer library to be installed: pip install nvidia-modelopt | `None` | Type: str | +| `--modelopt-checkpoint-restore-path` | Path to restore a previously saved ModelOpt quantized checkpoint. If provided, the quantization process will be skipped and the model will be loaded from this checkpoint. | `None` | Type: str | +| `--modelopt-checkpoint-save-path` | Path to save the ModelOpt quantized checkpoint after quantization. This allows reusing the quantized model in future runs. | `None` | Type: str | +| `--kv-cache-dtype` | Data type for kv cache storage. "auto" will use model data type. "fp8_e5m2" and "fp8_e4m3" is supported for CUDA 11.8+. | `auto` | `auto`, `fp8_e5m2`, `fp8_e4m3` | +| `--enable-fp32-lm-head` | If set, the LM head outputs (logits) are in FP32. | `False` | bool flag (set to enable) | + +## Memory and scheduling +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--mem-fraction-static` | The fraction of the memory used for static allocation (model weights and KV cache memory pool). Use a smaller value if you see out-of-memory errors. | `None` | Type: float | +| `--max-running-requests` | The maximum number of running requests. | `None` | Type: int | +| `--max-queued-requests` | The maximum number of queued requests. This option is ignored when using disaggregation-mode. | `None` | Type: int | +| `--max-total-tokens` | The maximum number of tokens in the memory pool. If not specified, it will be automatically calculated based on the memory usage fraction. This option is typically used for development and debugging purposes. | `None` | Type: int | +| `--chunked-prefill-size` | The maximum number of tokens in a chunk for the chunked prefill. Setting this to -1 means disabling chunked prefill. | `None` | Type: int | +| `--max-prefill-tokens` | The maximum number of tokens in a prefill batch. The real bound will be the maximum of this value and the model's maximum context length. | `16384` | Type: int | +| `--schedule-policy` | The scheduling policy of the requests. | `fcfs` | `lpm`, `random`, `fcfs`, `dfs-weight`, `lof`, `priority` | +| `--enable-priority-scheduling` | Enable priority scheduling. Requests with higher priority integer values will be scheduled first by default. | `False` | bool flag (set to enable) | +| `--schedule-low-priority-values-first` | If specified with --enable-priority-scheduling, the scheduler will schedule requests with lower priority integer values first. | `False` | bool flag (set to enable) | +| `--priority-scheduling-preemption-threshold` | Minimum difference in priorities for an incoming request to have to preempt running request(s). | `10` | Type: int | +| `--schedule-conservativeness` | How conservative the schedule policy is. A larger value means more conservative scheduling. Use a larger value if you see requests being retracted frequently. | `1.0` | Type: float | +| `--page-size` | The number of tokens in a page. | `1` | Type: int | +| `--hybrid-kvcache-ratio` | Mix ratio in [0,1] between uniform and hybrid kv buffers (0.0 = pure uniform: swa_size / full_size = 1)(1.0 = pure hybrid: swa_size / full_size = local_attention_size / context_length) | `None` | Optional[float] | +| `--swa-full-tokens-ratio` | The ratio of SWA layer KV tokens / full layer KV tokens, regardless of the number of swa:full layers. It should be between 0 and 1. E.g. 0.5 means if each swa layer has 50 tokens, then each full layer has 100 tokens. | `0.8` | Type: float | +| `--disable-hybrid-swa-memory` | Disable the hybrid SWA memory. | `False` | bool flag (set to enable) | + +## Runtime options +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--device` | The device to use ('cuda', 'xpu', 'hpu', 'npu', 'cpu'). Defaults to auto-detection if not specified. | `None` | Type: str | +| `--elastic-ep-backend` | Select the collective communication backend for elastic EP. Currently supports 'mooncake'. | None | N/A | +| `--mooncake-ib-device` | The InfiniBand devices for Mooncake Backend, accepts multiple comma-separated devices. Default is None, which triggers automatic device detection when Mooncake Backend is enabled. | None | N/A | +| `--tensor-parallel-size`
`--tp-size` | The tensor parallelism size. | `1` | Type: int | +| `--pipeline-parallel-size`
`--pp-size` | The pipeline parallelism size. | `1` | Type: int | +| `--pp-max-micro-batch-size` | The maximum micro batch size in pipeline parallelism. | `None` | Type: int | +| `--stream-interval` | The interval (or buffer size) for streaming in terms of the token length. A smaller value makes streaming smoother, while a larger value makes the throughput higher | `1` | Type: int | +| `--stream-output` | Whether to output as a sequence of disjoint segments. | `False` | bool flag (set to enable) | +| `--random-seed` | The random seed. | `None` | Type: int | +| `--constrained-json-whitespace-pattern` | (outlines and llguidance backends only) Regex pattern for syntactic whitespaces allowed in JSON constrained output. For example, to allow the model to generate consecutive whitespaces, set the pattern to [\n\t ]* | `None` | Type: str | +| `--constrained-json-disable-any-whitespace` | (xgrammar and llguidance backends only) Enforce compact representation in JSON constrained output. | `False` | bool flag (set to enable) | +| `--watchdog-timeout` | Set watchdog timeout in seconds. If a forward batch takes longer than this, the server will crash to prevent hanging. | `300` | Type: float | +| `--dist-timeout` | Set timeout for torch.distributed initialization. | `None` | Type: int | +| `--download-dir` | Model download directory for huggingface. | `None` | Type: str | +| `--base-gpu-id` | The base GPU ID to start allocating GPUs from. Useful when running multiple instances on the same machine. | `0` | Type: int | +| `--gpu-id-step` | The delta between consecutive GPU IDs that are used. For example, setting it to 2 will use GPU 0,2,4,... | `1` | Type: int | +| `--sleep-on-idle` | Reduce CPU usage when sglang is idle. | `False` | bool flag (set to enable) | +| `--mm-process-config` | A JSON string for multimodal preprocessing configuration. It can contain keys: `image`, `video`, `audio`. | `{}` | + +## Logging +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--log-level` | The logging level of all loggers. | `info` | Type: str | +| `--log-level-http` | The logging level of HTTP server. If not set, reuse --log-level by default. | `None` | Type: str | +| `--log-requests` | Log metadata, inputs, outputs of all requests. The verbosity is decided by --log-requests-level | `False` | bool flag (set to enable) | +| `--log-requests-level` | 0: Log metadata (no sampling parameters). 1: Log metadata and sampling parameters. 2: Log metadata, sampling parameters and partial input/output. 3: Log every input/output. | `2` | `0`, `1`, `2`, `3` | +| `--crash-dump-folder` | Folder path to dump requests from the last 5 min before a crash (if any). If not specified, crash dumping is disabled. | `None` | Type: str | +| `--crash-on-nan` | Crash the server on nan logprobs. | `False` | Type: str | +| `--show-time-cost` | Show time cost of custom marks. | `False` | bool flag (set to enable) | +| `--enable-metrics` | Enable log prometheus metrics. | `False` | bool flag (set to enable) | +| `--enable-metrics-for-all-schedulers` | Enable --enable-metrics-for-all-schedulers when you want schedulers on all TP ranks (not just TP 0) to record request metrics separately. This is especially useful when dp_attention is enabled, as otherwise all metrics appear to come from TP 0. | `False` | bool flag (set to enable) | +| `--tokenizer-metrics-custom-labels-header` | Specify the HTTP header for passing custom labels for tokenizer metrics. | `x-custom-labels` | Type: str | +| `--tokenizer-metrics-allowed-custom-labels` | The custom labels allowed for tokenizer metrics. The labels are specified via a dict in '--tokenizer-metrics-custom-labels-header' field in HTTP requests, e.g., {'label1': 'value1', 'label2': 'value2'} is allowed if '--tokenizer-metrics-allowed-custom-labels label1 label2' is set. | `None` | List[str] | +| `--bucket-time-to-first-token` | The buckets of time to first token, specified as a list of floats. | `None` | List[float] | +| `--bucket-inter-token-latency` | The buckets of inter-token latency, specified as a list of floats. | `None` | List[float] | +| `--bucket-e2e-request-latency` | The buckets of end-to-end request latency, specified as a list of floats. | `None` | List[float] | +| `--collect-tokens-histogram` | Collect prompt/generation tokens histogram. | `False` | bool flag (set to enable) | +| `--prompt-tokens-buckets` | The buckets rule of prompt tokens. Supports 3 rule types: 'default' uses predefined buckets; 'tse ' generates two sides exponential distributed buckets (e.g., 'tse 1000 2 8' generates buckets [984.0, 992.0, 996.0, 998.0, 1000.0, 1002.0, 1004.0, 1008.0, 1016.0]).); 'custom ...' uses custom bucket values (e.g., 'custom 10 50 100 500'). | `None` | List[str] | +| `--generation-tokens-buckets` | The buckets rule for generation tokens histogram. Supports 3 rule types: 'default' uses predefined buckets; 'tse ' generates two sides exponential distributed buckets (e.g., 'tse 1000 2 8' generates buckets [984.0, 992.0, 996.0, 998.0, 1000.0, 1002.0, 1004.0, 1008.0, 1016.0]).); 'custom ...' uses custom bucket values (e.g., 'custom 10 50 100 500'). | `None` | List[str] | +| `--gc-warning-threshold-secs` | The threshold for long GC warning. If a GC takes longer than this, a warning will be logged. Set to 0 to disable. | `0.0` | Type: float | +| `--decode-log-interval` | The log interval of decode batch. | `40` | Type: int | +| `--enable-request-time-stats-logging` | Enable per request time stats logging | `False` | bool flag (set to enable) | +| `--kv-events-config` | Config in json format for NVIDIA dynamo KV event publishing. Publishing will be enabled if this flag is used. | `None` | Type: str | +| `--enable-trace` | Enable opentelemetry trace | `False` | bool flag (set to enable) | +| `--oltp-traces-endpoint` | Config opentelemetry collector endpoint if --enable-trace is set. format: : | `localhost:4317` | Type: str | + +## API related +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--api-key` | Set API key of the server. It is also used in the OpenAI API compatible server. | `None` | Type: str | +| `--served-model-name` | Override the model name returned by the v1/models endpoint in OpenAI API server. | `None` | Type: str | +| `--weight-version` | Version identifier for the model weights. Defaults to 'default' if not specified. | `default` | Type: str | +| `--chat-template` | The buliltin chat template name or the path of the chat template file. This is only used for OpenAI-compatible API server. | `None` | Type: str | +| `--completion-template` | The buliltin completion template name or the path of the completion template file. This is only used for OpenAI-compatible API server. only for code completion currently. | `None` | Type: str | +| `--file-storage-path` | The path of the file storage in backend. | `sglang_storage` | Type: str | +| `--enable-cache-report` | Return number of cached tokens in usage.prompt_tokens_details for each openai request. | `False` | bool flag (set to enable) | +| `--reasoning-parser` | Specify the parser for reasoning models. Supported parsers: [deepseek-r1, deepseek-v3, glm45, gpt-oss, kimi, qwen3, qwen3-thinking, step3]. | `None` | `deepseek-r1`, `deepseek-v3`, `glm45`, `gpt-oss`, `kimi`, `qwen3`, `qwen3-thinking`, `step3` | +| `--tool-call-parser` | Specify the parser for handling tool-call interactions. Supported parsers: [deepseekv3, deepseekv31, glm, glm45, gpt-oss, kimi_k2, llama3, mistral, pythonic, qwen, qwen25, qwen3_coder, step3]. | `None` | `deepseekv3`, `deepseekv31`, `glm`, `glm45`, `gpt-oss`, `kimi_k2`, `llama3`, `mistral`, `pythonic`, `qwen`, `qwen25`, `qwen3_coder`, `step3` | +| `--sampling-defaults` | Where to get default sampling parameters. 'openai' uses SGLang/OpenAI defaults (temperature=1.0, top_p=1.0, etc.). 'model' uses the model's generation_config.json to get the recommended sampling parameters if available. Default is 'model'. | `model` | `openai`, `model` | +| `--tool-server` | Either 'demo' or a comma-separated list of tool server urls to use for the model. If not specified, no tool server will be used. | `None` | Type: str | + +## Data parallelism +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--data-parallel-size`
`--dp-size` | The data parallelism size. | `1` | Type: int | +| `--load-balance-method` | The load balancing strategy for data parallelism. The Minimum Token algorithm can only be used when DP attention is applied. This algorithm performs load balancing based on the real-time token load of the DP workers. | `round_robin` | `round_robin`, `shortest_queue`, `minimum_tokens` | +| `--load-watch-interval` | The interval of load watching in seconds. | `0.1` | Type: float | +| `--prefill-round-robin-balance` | Prefill is round robin balanced. This is used to promise decode server can get the correct dp rank. | `False` | bool flag (set to enable) | + +## Multi-node distributed serving +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--dist-init-addr`
`--nccl-init-addr` | The host address for initializing distributed backend (e.g., `192.168.0.2:25000`). | `None` | Type: str | +| `--nnodes` | The number of nodes. | `1` | Type: int | +| `--node-rank` | The node rank. | `0` | Type: int | + +## Model override args +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--json-model-override-args` | A dictionary in JSON string format used to override default model configurations. | `{}` | Type: str | +| `--preferred-sampling-params` | json-formatted sampling settings that will be returned in /get_model_info | `None` | Type: str | + +## LoRA +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--enable-lora` | Enable LoRA support for the model. This argument is automatically set to `True` if `--lora-paths` is provided for backward compatibility. | `False` | Bool flag (set to enable) | +| `--max-lora-rank` | The maximum LoRA rank that should be supported. If not specified, it will be automatically inferred from the adapters provided in `--lora-paths`. This argument is needed when you expect to dynamically load adapters of larger LoRA rank after server startup. | `None` | Type: int | +| `--lora-target-modules` | The union set of all target modules where LoRA should be applied (e.g., `q_proj`, `k_proj`, `gate_proj`). If not specified, it will be automatically inferred from the adapters provided in `--lora-paths`. You can also set it to `all` to enable LoRA for all supported modules; note this may introduce minor performance overhead. | `None` | `q_proj`, `k_proj`, `v_proj`, `o_proj`, `gate_proj`, `up_proj`, `down_proj`, `qkv_proj`, `gate_up_proj`, `all` | +| `--lora-paths` | The list of LoRA adapters to load. Each adapter must be specified in one of the following formats: `` \| `=` \| JSON with schema `{"lora_name": str, "lora_path": str, "pinned": bool}`. | `None` | Type: List[str] / JSON objects | +| `--max-loras-per-batch` | Maximum number of adapters for a running batch, including base-only requests. | `8` | Type: int | +| `--max-loaded-loras` | If specified, limits the maximum number of LoRA adapters loaded in CPU memory at a time. Must be ≥ `--max-loras-per-batch`. | `None` | Type: int | +| `--lora-eviction-policy` | LoRA adapter eviction policy when the GPU memory pool is full. | `lru` | `lru`, `fifo` | +| `--lora-backend` | Choose the kernel backend for multi-LoRA serving. | `triton` | `triton`, `csgmv` | +| `--max-lora-chunk-size` | Maximum chunk size for the ChunkedSGMV LoRA backend. Only used when `--lora-backend` is `csgmv`. Larger values may improve performance. | `16` | `16`, `32`, `64`, `128` | + +## Kernel backend +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--attention-backend` | Choose the kernels for attention layers. | `None` | `triton`, `torch_native`, `flex_attention`, `nsa`, `cutlass_mla`, `fa3`, `fa4`, `flashinfer`, `flashmla`, `trtllm_mla`, `trtllm_mha`, `dual_chunk_flash_attn`, `aiter`, `wave`, `intel_amx`, `ascend` | +| `--prefill-attention-backend` | Choose the kernels for prefill attention layers (have priority over --attention-backend). | `None` | `triton`, `torch_native`, `flex_attention`, `nsa`, `cutlass_mla`, `fa3`, `fa4`, `flashinfer`, `flashmla`, `trtllm_mla`, `trtllm_mha`, `dual_chunk_flash_attn`, `aiter`, `wave`, `intel_amx`, `ascend` | +| `--decode-attention-backend` | Choose the kernels for decode attention layers (have priority over --attention-backend). | `None` | `triton`, `torch_native`, `flex_attention`, `nsa`, `cutlass_mla`, `fa3`, `fa4`, `flashinfer`, `flashmla`, `trtllm_mla`, `trtllm_mha`, `dual_chunk_flash_attn`, `aiter`, `wave`, `intel_amx`, `ascend` | +| `--sampling-backend` | Choose the kernels for sampling layers. | `None` | `flashinfer`, `pytorch`, `ascend` | +| `--grammar-backend` | Choose the backend for grammar-guided decoding. | `None` | `xgrammar`, `outlines`, `llguidance`, `none` | +| `--mm-attention-backend` | Set multimodal attention backend. | `None` | `sdpa`, `fa3`, `triton_attn`, `ascend_attn`, `aiter_attn` | +| `--nsa-prefill` | Choose the NSA backend for the prefill stage (overrides `--attention-backend` when running DeepSeek NSA-style attention). | `flashmla_sparse` | `flashmla_sparse`, `flashmla_decode`, `fa3`, `tilelang`, `aiter` | +| `--nsa-decode` | Choose the NSA backend for the decode stage when running DeepSeek NSA-style attention. Overrides `--attention-backend` for decoding. | `flashmla_kv` | `flashmla_prefill`, `flashmla_kv`, `fa3`, `tilelang`, `aiter` | + +## Speculative decoding +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--speculative-algorithm` | Speculative algorithm. | `None` | `EAGLE`, `EAGLE3`, `NEXTN`, `STANDALONE`, `NGRAM` | +| `--speculative-draft-model-path`
`--speculative-draft-model` | The path of the draft model weights. This can be a local folder or a Hugging Face repo ID. | `None` | Type: str | +| `--speculative-draft-model-revision` | The specific draft model version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version. | `None` | Type: str | +| `--speculative-num-steps` | The number of steps sampled from draft model in Speculative Decoding. | `None` | Type: int | +| `--speculative-eagle-topk` | The number of tokens sampled from the draft model in eagle2 each step. | `None` | Type: int | +| `--speculative-num-draft-tokens` | The number of tokens sampled from the draft model in Speculative Decoding. | `None` | Type: int | +| `--speculative-accept-threshold-single` | Accept a draft token if its probability in the target model is greater than this threshold. | `1.0` | Type: float | +| `--speculative-accept-threshold-acc` | The accept probability of a draft token is raised from its target probability p to min(1, p / threshold_acc). | `1.0` | Type: float | +| `--speculative-token-map` | The path of the draft model's small vocab table. | `None` | Type: str | +| `--speculative-attention-mode` | Attention backend for speculative decoding operations (both target verify and draft extend). Can be one of 'prefill' (default) or 'decode'. | `prefill` | `prefill`, `decode` | +| `--speculative-moe-runner-backend` | MOE backend for EAGLE speculative decoding, see --moe-runner-backend for options. Same as moe runner backend if unset. | None | + +## Ngram speculative decoding +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--speculative-ngram-min-match-window-size` | The minimum window size for pattern matching in ngram speculative decoding. | `1` | Type: int | +| `--speculative-ngram-max-match-window-size` | The maximum window size for pattern matching in ngram speculative decoding. | `12` | Type: int | +| `--speculative-ngram-min-bfs-breadth` | The minimum breadth for BFS (Breadth-First Search) in ngram speculative decoding. | `1` | Type: int | +| `--speculative-ngram-max-bfs-breadth` | The maximum breadth for BFS (Breadth-First Search) in ngram speculative decoding. | `10` | Type: int | +| `--speculative-ngram-match-type` | The match type for cache tree. | `BFS` | `BFS`, `PROB` | +| `--speculative-ngram-branch-length` | The branch length for ngram speculative decoding. | `18` | Type: int | +| `--speculative-ngram-capacity` | The cache capacity for ngram speculative decoding. | `10000000` | Type: int | + +## MoE +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--expert-parallel-size`
`--ep-size`
`--ep` | The expert parallelism size. | `1` | Type: int | +| `--moe-a2a-backend` | Select the backend for all-to-all communication for expert parallelism. | `none` | `none`, `deepep` | +| `--moe-runner-backend` | Choose the runner backend for MoE. | `auto` | `auto`, `deep_gemm`, `triton`, `triton_kernel`, `flashinfer_trtllm`, `flashinfer_cutlass`, `flashinfer_mxfp4`, `flashinfer_cutedsl` | +| `--flashinfer-mxfp4-moe-precision` | Choose the computation precision of flashinfer mxfp4 moe | `default` | `default`, `bf16` | +| `--enable-flashinfer-allreduce-fusion` | Enable FlashInfer allreduce fusion with Residual RMSNorm. | `False` | bool flag (set to enable) | +| `--deepep-mode` | Select the mode when enable DeepEP MoE, could be `normal`, `low_latency` or `auto`. Default is `auto`, which means `low_latency` for decode batch and `normal` for prefill batch. | `auto` | `normal`, `low_latency`, `auto` | +| `--ep-num-redundant-experts` | Allocate this number of redundant experts in expert parallel. | `0` | Type: int | +| `--ep-dispatch-algorithm` | The algorithm to choose ranks for redundant experts in expert parallel. | `None` | Type: str | +| `--init-expert-location` | Initial location of EP experts. | `trivial` | Type: str | +| `--enable-eplb` | Enable EPLB algorithm | `False` | bool flag (set to enable) | +| `--eplb-algorithm` | Chosen EPLB algorithm | `auto` | Type: str | +| `--eplb-rebalance-num-iterations` | Number of iterations to automatically trigger a EPLB re-balance. | `1000` | Type: int | +| `--eplb-rebalance-layers-per-chunk` | Number of layers to rebalance per forward pass. | `None` | Type: int | +| `--eplb-min-rebalancing-utilization-threshold` | Minimum threshold for GPU average utilization to trigger EPLB rebalancing. Must be in the range [0.0, 1.0]. | `1.0` | Type: float | +| `--expert-distribution-recorder-mode` | Mode of expert distribution recorder. | `None` | Type: str | +| `--expert-distribution-recorder-buffer-size` | Circular buffer size of expert distribution recorder. Set to -1 to denote infinite buffer. | `None` | Type: int | +| `--enable-expert-distribution-metrics` | Enable logging metrics for expert balancedness | `False` | bool flag (set to enable) | +| `--deepep-config` | Tuned DeepEP config suitable for your own cluster. It can be either a string with JSON content or a file path. | `None` | Type: str | +| `--moe-dense-tp-size` | TP size for MoE dense MLP layers. This flag is useful when, with large TP size, there are errors caused by weights in MLP layers having dimension smaller than the min dimension GEMM supports. | `None` | Type: int | + +## Mamba Cache +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--max-mamba-cache-size` | The maximum size of the mamba cache. | `None` | Type: int | +| `--mamba-ssm-dtype` | The data type of the SSM states in mamba cache. | `float32` | `float32`, `bfloat16` | +| `--mamba-full-memory-ratio` | The ratio of mamba state memory to full kv cache memory. | `0.2` | Type: float | + +## Args for multi-item scoring +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--multi-item-scoring-delimiter` | Delimiter token ID for multi-item scoring. Used to combine Query and Items into a single sequence: QueryItem1Item2... This enables efficient batch processing of multiple items against a single query. | `None` | Type: int | + +## Hierarchical cache +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--enable-hierarchical-cache` | Enable hierarchical cache | `False` | bool flag (set to enable) | +| `--hicache-ratio` | The ratio of the size of host KV cache memory pool to the size of device pool. | `2.0` | Type: float | +| `--hicache-size` | The size of host KV cache memory pool in gigabytes, which will override the hicache_ratio if set. | `0` | Type: int | +| `--hicache-write-policy` | The write policy of hierarchical cache. | `write_through` | `write_back`, `write_through`, `write_through_selective` | +| `--radix-eviction-policy` | The eviction policy of radix trees. 'lru' stands for Least Recently Used, 'lfu' stands for Least Frequently Used. | `lru` | `lru`, `lfu` | +| `--hicache-io-backend` | The IO backend for KV cache transfer between CPU and GPU | `kernel` | `direct`, `kernel`, `kernel_ascend` | +| `--hicache-mem-layout` | The layout of host memory pool for hierarchical cache. | `layer_first` | `layer_first`, `page_first`, `page_first_direct`, `page_first_kv_split` | +| `--hicache-storage-backend` | The storage backend for hierarchical KV cache. Built-in backends: file, mooncake, hf3fs, nixl, aibrix. For dynamic backend, use --hicache-storage-backend-extra-config to specify: backend_name (custom name), module_path (Python module path), class_name (backend class name). | `None` | `file`, `mooncake`, `hf3fs`, `nixl`, `aibrix`, `dynamic`, `eic` | +| `--hicache-storage-prefetch-policy` | Control when prefetching from the storage backend should stop. | `best_effort` | `best_effort`, `wait_complete`, `timeout` | +| `--hicache-storage-backend-extra-config` | A dictionary in JSON string format containing extra configuration for the storage backend. | `None` | Type: str | + +## LMCache +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--enable-lmcache` | Using LMCache as an alternative hierarchical cache solution | `False` | bool flag (set to enable) | + +## Double Sparsity +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--enable-double-sparsity` | Enable double sparsity attention | `False` | bool flag (set to enable) | +| `--ds-channel-config-path` | The path of the double sparsity channel config | `None` | Type: str | +| `--ds-heavy-channel-num` | The number of heavy channels in double sparsity attention | `32` | Type: int | +| `--ds-heavy-token-num` | The number of heavy tokens in double sparsity attention | `256` | Type: int | +| `--ds-heavy-channel-type` | The type of heavy channels in double sparsity attention | `qk` | Type: str | +| `--ds-sparse-decode-threshold` | The minimum decode sequence length required before the double-sparsity backend switches from the dense fallback to the sparse decode kernel. | `4096` | Type: int | + +## Offloading +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--cpu-offload-gb` | How many GBs of RAM to reserve for CPU offloading. | `0` | Type: int | +| `--offload-group-size` | Number of layers per group in offloading. | `-1` | Type: int | +| `--offload-num-in-group` | Number of layers to be offloaded within a group. | `1` | Type: int | +| `--offload-prefetch-step` | Steps to prefetch in offloading. | `1` | Type: int | +| `--offload-mode` | Mode of offloading. | `cpu` | Type: str | + +## Optimization/debug options +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--disable-radix-cache` | Disable RadixAttention for prefix caching. | `False` | bool flag (set to enable) | +| `--cuda-graph-max-bs` | Set the maximum batch size for cuda graph. It will extend the cuda graph capture batch size to this value. | `None` | Type: int | +| `--cuda-graph-bs` | Set the list of batch sizes for cuda graph. | `None` | List[int] | +| `--disable-cuda-graph` | Disable cuda graph. | `False` | bool flag (set to enable) | +| `--disable-cuda-graph-padding` | Disable cuda graph when padding is needed. Still uses cuda graph when padding is not needed. | `False` | bool flag (set to enable) | +| `--enable-profile-cuda-graph` | Enable profiling of cuda graph capture. | `False` | bool flag (set to enable) | +| `--enable-cudagraph-gc` | Enable garbage collection during CUDA graph capture. If disabled (default), GC is frozen during capture to speed up the process. | `False` | bool flag (set to enable) | +| `--enable-nccl-nvls` | Enable NCCL NVLS for prefill heavy requests when available. | `False` | bool flag (set to enable) | +| `--enable-symm-mem` | Enable NCCL symmetric memory for fast collectives. | `False` | bool flag (set to enable) | +| `--disable-flashinfer-cutlass-moe-fp4-allgather` | Disables quantize before all-gather for flashinfer cutlass moe. | `False` | bool flag (set to enable) | +| `--enable-tokenizer-batch-encode` | Enable batch tokenization for improved performance when processing multiple text inputs. Do not use with image inputs, pre-tokenized input_ids, or input_embeds. | `False` | bool flag (set to enable) | +| `--disable-outlines-disk-cache` | Disable disk cache of outlines to avoid possible crashes related to file system or high concurrency. | `False` | bool flag (set to enable) | +| `--disable-custom-all-reduce` | Disable the custom all-reduce kernel and fall back to NCCL. | `False` | bool flag (set to enable) | +| `--enable-mscclpp` | Enable using mscclpp for small messages for all-reduce kernel and fall back to NCCL. | `False` | bool flag (set to enable) | +| `--enable-torch-symm-mem` | Enable using torch symm mem for all-reduce kernel and fall back to NCCL. Only supports CUDA device SM90 and above. SM90 supports world size 4, 6, 8. SM10 supports world size 6, 8. | `False` | bool flag (set to enable) | +| `--disable-overlap-schedule` | Disable the overlap scheduler, which overlaps the CPU scheduler with GPU model worker. | `False` | bool flag (set to enable) | +| `--enable-mixed-chunk` | Enabling mixing prefill and decode in a batch when using chunked prefill. | `False` | bool flag (set to enable) | +| `--enable-dp-attention` | Enabling data parallelism for attention and tensor parallelism for FFN. The dp size should be equal to the tp size. Currently DeepSeek-V2 and Qwen 2/3 MoE models are supported. | `False` | bool flag (set to enable) | +| `--enable-dp-lm-head` | Enable vocabulary parallel across the attention TP group to avoid all-gather across DP groups, optimizing performance under DP attention. | `False` | bool flag (set to enable) | +| `--enable-two-batch-overlap` | Enabling two micro batches to overlap. | `False` | bool flag (set to enable) | +| `--enable-single-batch-overlap` | Let computation and communication overlap within one micro batch. | `False` | bool flag (set to enable) | +| `--tbo-token-distribution-threshold` | The threshold of token distribution between two batches in micro-batch-overlap, determines whether to two-batch-overlap or two-chunk-overlap. Set to 0 denote disable two-chunk-overlap. | `0.48` | Type: float | +| `--enable-torch-compile` | Optimize the model with torch.compile. Experimental feature. | `False` | bool flag (set to enable) | +| `--enable-torch-compile-debug-mode` | Enable debug mode for torch compile. | `False` | bool flag (set to enable) | +| `--enable-piecewise-cuda-graph` | Optimize the model with piecewise cuda graph for extend/prefill only. Experimental feature. | `False` | bool flag (set to enable) | +| `--piecewise-cuda-graph-tokens` | Set the list of tokens when using piecewise cuda graph. | `None` | Type: JSON list | +| `--torch-compile-max-bs` | Set the maximum batch size when using torch compile. | `32` | Type: int | +| `--piecewise-cuda-graph-max-tokens` | Set the maximum tokens when using piecewise cuda graph. | `4096` | Type: int | +| `--torchao-config` | Optimize the model with torchao. Experimental feature. Current choices are: int8dq, int8wo, int4wo-, fp8wo, fp8dq-per_tensor, fp8dq-per_row | `` | Type: str | +| `--enable-nan-detection` | Enable the NaN detection for debugging purposes. | `False` | bool flag (set to enable) | +| `--enable-p2p-check` | Enable P2P check for GPU access, otherwise the p2p access is allowed by default. | `False` | bool flag (set to enable) | +| `--triton-attention-reduce-in-fp32` | Cast the intermediate attention results to fp32 to avoid possible crashes related to fp16. This only affects Triton attention kernels. | `False` | bool flag (set to enable) | +| `--triton-attention-num-kv-splits` | The number of KV splits in flash decoding Triton kernel. Larger value is better in longer context scenarios. The default value is 8. | `8` | Type: int | +| `--triton-attention-split-tile-size` | The size of split KV tile in flash decoding Triton kernel. Used for deterministic inference. | `None` | Type: int | +| `--num-continuous-decode-steps` | Run multiple continuous decoding steps to reduce scheduling overhead. This can potentially increase throughput but may also increase time-to-first-token latency. The default value is 1, meaning only run one decoding step at a time. | `1` | Type: int | +| `--delete-ckpt-after-loading` | Delete the model checkpoint after loading the model. | `False` | bool flag (set to enable) | +| `--enable-memory-saver` | Allow saving memory using release_memory_occupation and resume_memory_occupation | `False` | bool flag (set to enable) | +| `--enable-weights-cpu-backup` | Save model weights to CPU memory during release_weights_occupation and resume_weights_occupation | `False` | bool flag (set to enable) | +| `--allow-auto-truncate` | Allow automatically truncating requests that exceed the maximum input length instead of returning an error. | `False` | bool flag (set to enable) | +| `--enable-custom-logit-processor` | Enable users to pass custom logit processors to the server (disabled by default for security) | `False` | bool flag (set to enable) | +| `--flashinfer-mla-disable-ragged` | Not using ragged prefill wrapper when running flashinfer mla | `False` | bool flag (set to enable) | +| `--disable-shared-experts-fusion` | Disable shared experts fusion optimization for deepseek v3/r1. | `False` | bool flag (set to enable) | +| `--disable-chunked-prefix-cache` | Disable chunked prefix cache feature for deepseek, which should save overhead for short sequences. | `False` | bool flag (set to enable) | +| `--disable-fast-image-processor` | Adopt base image processor instead of fast image processor. | `False` | bool flag (set to enable) | +| `--keep-mm-feature-on-device` | Keep multimodal feature tensors on device after processing to save D2H copy. | `False` | bool flag (set to enable) | +| `--enable-return-hidden-states` | Enable returning hidden states with responses. | `False` | bool flag (set to enable) | +| `--scheduler-recv-interval` | The interval to poll requests in scheduler. Can be set to >1 to reduce the overhead of this. | `1` | Type: int | +| `--numa-node` | Sets the numa node for the subprocesses. i-th element corresponds to i-th subprocess. | `None` | List[int] | +| `--enable-layerwise-nvtx-marker` | Enable layerwise NVTX profiling annotations for the model. This adds NVTX markers to every layer for detailed per-layer performance analysis with Nsight Systems. | `False` | bool flag (set to enable) | +| `--enable-attn-tp-input-scattered` | Allow input of attention to be scattered when only using tensor parallelism, to reduce the computational load of operations such as qkv latent. | `False` | bool flag (set to enable) | +| `--enable-nsa-prefill-context-parallel` | Context parallelism used in the long sequence prefill phase of DeepSeek v3.2 | `False` | bool flag (set to enable) | + +## Forward hooks +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--forward-hooks` | JSON-formatted list of forward hook specifications. Each element must include `target_modules` (list of glob patterns matched against `model.named_modules()` names) and `hook_factory` (Python import path to a factory, e.g. `my_package.hooks:make_hook`). An optional `name` field is used for logging, and an optional `config` object is passed as a `dict` to the factory. | `None` | Type: JSON list | + +## Debug tensor dumps +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--debug-tensor-dump-output-folder` | The output folder for dumping tensors. | `None` | Type: str | +| `--debug-tensor-dump-input-file` | The input filename for dumping tensors | `None` | Type: str | +| `--debug-tensor-dump-inject` | Inject the outputs from jax as the input of every layer. | `False` | Type: str | +| `--enable-dynamic-batch-tokenizer` | Enable async dynamic batch tokenizer for improved performance when multiple requests arrive concurrently. | `False` | bool flag (set to enable) | +| `--dynamic-batch-tokenizer-batch-size` | [Only used if --enable-dynamic-batch-tokenizer is set] Maximum batch size for dynamic batch tokenizer. | `32` | Type: int | +| `--dynamic-batch-tokenizer-batch-timeout` | [Only used if --enable-dynamic-batch-tokenizer is set] Timeout in seconds for batching tokenization requests. | `0.002` | Type: float | + +## PD disaggregation +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--disaggregation-mode` | Only used for PD disaggregation. "prefill" for prefill-only server, and "decode" for decode-only server. If not specified, it is not PD disaggregated | `null` | `null`, `prefill`, `decode` | +| `--disaggregation-transfer-backend` | The backend for disaggregation transfer. Default is mooncake. | `mooncake` | `mooncake`, `nixl`, `ascend`, `fake` | +| `--disaggregation-bootstrap-port` | Bootstrap server port on the prefill server. Default is 8998. | `8998` | Type: int | +| `--disaggregation-decode-tp` | Decode tp size. If not set, it matches the tp size of the current engine. This is only set on the prefill server. | `None` | Type: int | +| `--disaggregation-decode-dp` | Decode dp size. If not set, it matches the dp size of the current engine. This is only set on the prefill server. | `None` | Type: int | +| `--disaggregation-prefill-pp` | Prefill pp size. If not set, it is default to 1. This is only set on the decode server. | `1` | Type: int | +| `--disaggregation-ib-device` | The InfiniBand devices for disaggregation transfer, accepts single device (e.g., --disaggregation-ib-device mlx5_0) or multiple comma-separated devices (e.g., --disaggregation-ib-device mlx5_0,mlx5_1). Default is None, which triggers automatic device detection when mooncake backend is enabled. | `None` | Type: str | +| `--disaggregation-decode-enable-offload-kvcache` | Enable async KV cache offloading on decode server (PD mode). | `False` | bool flag (set to enable) | +| `--num-reserved-decode-tokens` | Number of decode tokens that will have memory reserved when adding new request to the running batch. | `512` | Type: int | +| `--disaggregation-decode-polling-interval` | The interval to poll requests in decode server. Can be set to >1 to reduce the overhead of this. | `1` | Type: int | + +## Custom weight loader +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--custom-weight-loader` | The custom dataloader which used to update the model. Should be set with a valid import path, such as my_package.weight_load_func | `None` | List[str] | +| `--weight-loader-disable-mmap` | Disable mmap while loading weight using safetensors. | `False` | bool flag (set to enable) | +| `--remote-instance-weight-loader-seed-instance-ip` | The ip of the seed instance for loading weights from remote instance. | `None` | Type: str | +| `--remote-instance-weight-loader-seed-instance-service-port` | The service port of the seed instance for loading weights from remote instance. | `None` | Type: int | +| `--remote-instance-weight-loader-send-weights-group-ports` | The communication group ports for loading weights from remote instance. | `None` | Type: JSON list | + +## For PD-Multiplexing +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--enable-pdmux` | Enable PD-Multiplexing, PD running on greenctx stream. | `False` | bool flag (set to enable) | +| `--pdmux-config-path` | The path of the PD-Multiplexing config file. | `None` | Type: str | +| `--sm-group-num` | Number of sm partition groups. | `8` | Type: int | + +## For deterministic inference +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--enable-deterministic-inference` | Enable deterministic inference mode with batch invariant ops. | `False` | bool flag (set to enable) | + +## Deprecated arguments +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--enable-ep-moe` | NOTE: --enable-ep-moe is deprecated. Please set `--ep-size` to the same value as `--tp-size` instead. | `None` | N/A | +| `--enable-deepep-moe` | NOTE: --enable-deepep-moe is deprecated. Please set `--moe-a2a-backend` to 'deepep' instead. | `None` | N/A | +| `--enable-flashinfer-cutlass-moe` | NOTE: --enable-flashinfer-cutlass-moe is deprecated. Please set `--moe-runner-backend` to 'flashinfer_cutlass' instead. | `None` | N/A | +| `--enable-flashinfer-cutedsl-moe` | NOTE: --enable-flashinfer-cutedsl-moe is deprecated. Please set `--moe-runner-backend` to 'flashinfer_cutedsl' instead. | `None` | N/A | +| `--enable-flashinfer-trtllm-moe` | NOTE: --enable-flashinfer-trtllm-moe is deprecated. Please set `--moe-runner-backend` to 'flashinfer_trtllm' instead. | `None` | N/A | +| `--enable-triton-kernel-moe` | NOTE: --enable-triton-kernel-moe is deprecated. Please set `--moe-runner-backend` to 'triton_kernel' instead. | `None` | N/A | +| `--enable-flashinfer-mxfp4-moe` | NOTE: --enable-flashinfer-mxfp4-moe is deprecated. Please set `--moe-runner-backend` to 'flashinfer_mxfp4' instead. | `None` | N/A | + +## Configuration file support +| Argument | Description | Defaults | Options | +| --- | --- | --- | --- | +| `--config` | Read CLI options from a config file. Must be a YAML file with configuration options. | `None` | Type: str | diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/speculative_decoding.ipynb b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/speculative_decoding.ipynb new file mode 100644 index 00000000..aa62b897 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/speculative_decoding.ipynb @@ -0,0 +1,370 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Speculative Decoding\n", + "\n", + "SGLang now provides an EAGLE-based (EAGLE-2/EAGLE-3) speculative decoding option. Our implementation aims to maximize speed and efficiency and is considered to be among the fastest in open-source LLM engines.\n", + "\n", + "### Performance Highlights\n", + "\n", + "Please see below for the huge improvements on throughput for LLaMA-Instruct 3.1 8B tested on MT bench that can be achieved via EAGLE3 decoding.\n", + "For further details please see the [EAGLE3 paper](https://arxiv.org/pdf/2503.01840).\n", + "\n", + "| Method | Throughput (tokens/s) |\n", + "|--------|----------------|\n", + "| SGLang (w/o speculative, 1x H100) | 158.34 tokens/s |\n", + "| SGLang + EAGLE-2 (1x H100) | 244.10 tokens/s |\n", + "| SGLang + EAGLE-3 (1x H100) | 373.25 tokens/s |" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## EAGLE Decoding\n", + "\n", + "To enable EAGLE speculative decoding the following parameters are relevant:\n", + "* `speculative_draft_model_path`: Specifies draft model. This parameter is required.\n", + "* `speculative_num_steps`: Depth of autoregressive drafting. Increases speculation range but risks rejection cascades. Default is 5.\n", + "* `speculative_eagle_topk`: Branching factor per step. Improves candidate diversity, will lead to higher acceptance rate, but more lead to higher memory/compute consumption. Default is 4.\n", + "* `speculative_num_draft_tokens`: Maximum parallel verification capacity. Allows deeper tree evaluation but will lead to higher GPU memory usage. Default is 8.\n", + "\n", + "These parameters are the same for EAGLE-2 and EAGLE-3.\n", + "\n", + "You can find the best combinations of these parameters with [bench_speculative.py](https://github.com/sgl-project/sglang/blob/main/scripts/playground/bench_speculative.py).\n", + "\n", + "In the documentation below, we set `--cuda-graph-max-bs` to be a small value for faster engine startup. For your own workloads, please tune the above parameters together with `--cuda-graph-max-bs`, `--max-running-requests`, `--mem-fraction-static` for the best performance. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### EAGLE-2 decoding\n", + "\n", + "You can enable EAGLE-2 decoding by setting `--speculative-algorithm EAGLE` and choosing an appropriate model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sglang.test.doc_patch import launch_server_cmd\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "import openai" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model meta-llama/Llama-2-7b-chat-hf --speculative-algorithm EAGLE \\\n", + " --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B --speculative-num-steps 3 \\\n", + " --speculative-eagle-topk 4 --speculative-num-draft-tokens 16 --cuda-graph-max-bs 8 --log-level warning\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Llama-2-7b-chat-hf\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=64,\n", + ")\n", + "\n", + "print_highlight(f\"Response: {response}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### EAGLE-2 Decoding with `torch.compile`\n", + "\n", + "You can also enable `torch.compile` for further optimizations and optionally set `--torch-compile-max-bs`:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model meta-llama/Llama-2-7b-chat-hf --speculative-algorithm EAGLE \\\n", + " --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B --speculative-num-steps 5 \\\n", + " --speculative-eagle-topk 8 --speculative-num-draft-tokens 64 --mem-fraction 0.6 \\\n", + " --enable-torch-compile --torch-compile-max-bs 2 --log-level warning\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Llama-2-7b-chat-hf\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=64,\n", + ")\n", + "\n", + "print_highlight(f\"Response: {response}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### EAGLE-2 Decoding via Frequency-Ranked Speculative Sampling\n", + "\n", + "By employing a truncated high-frequency token vocabulary in the draft model, Eagle speculative decoding reduces `lm_head` computational overhead while accelerating the pipeline without quality degradation. For more details, checkout [the paper](https://arxiv.org/pdf/arXiv:2502.14856).\n", + "\n", + "In our implementation, set `--speculative-token-map` to enable the optimization. You can get the high-frequency token in FR-Spec from [this model](https://huggingface.co/thunlp/LLaMA3-Instruct-8B-FR-Spec). Or you can obtain high-frequency token by directly downloading these token from [this repo](https://github.com/thunlp/FR-Spec/tree/main?tab=readme-ov-file#prepare-fr-spec-vocabulary-subset).\n", + "\n", + "Thanks for the contribution from [Weilin Zhao](https://github.com/Achazwl) and [Zhousx](https://github.com/Zhou-sx). " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model meta-llama/Meta-Llama-3-8B-Instruct --speculative-algorithm EAGLE \\\n", + " --speculative-draft-model-path lmsys/sglang-EAGLE-LLaMA3-Instruct-8B --speculative-num-steps 5 \\\n", + " --speculative-eagle-topk 8 --speculative-num-draft-tokens 64 --speculative-token-map thunlp/LLaMA3-Instruct-8B-FR-Spec/freq_32768.pt \\\n", + " --mem-fraction 0.7 --cuda-graph-max-bs 2 --dtype float16 --log-level warning\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3-8B-Instruct\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=64,\n", + ")\n", + "\n", + "print_highlight(f\"Response: {response}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### EAGLE-3 Decoding\n", + "\n", + "You can enable EAGLE-3 decoding by setting `--speculative-algorithm EAGLE3` and choosing an appropriate model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model meta-llama/Llama-3.1-8B-Instruct --speculative-algorithm EAGLE3 \\\n", + " --speculative-draft-model-path jamesliu1/sglang-EAGLE3-Llama-3.1-Instruct-8B --speculative-num-steps 5 \\\n", + " --speculative-eagle-topk 8 --speculative-num-draft-tokens 32 --mem-fraction 0.6 \\\n", + " --cuda-graph-max-bs 2 --dtype float16 --log-level warning\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=64,\n", + ")\n", + "\n", + "print_highlight(f\"Response: {response}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multi Token Prediction\n", + "\n", + "We support [MTP(Multi-Token Prediction)](https://arxiv.org/pdf/2404.19737) in SGLang by using speculative decoding. We use Xiaomi/MiMo-7B-RL model as example here (deepseek mtp usage refer to [deepseek doc](../basic_usage/deepseek.md#multi-token-prediction))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + " python3 -m sglang.launch_server --model-path XiaomiMiMo/MiMo-7B-RL --host 0.0.0.0 --trust-remote-code \\\n", + " --speculative-algorithm EAGLE --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \\\n", + " --mem-fraction 0.5 --log-level warning\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "\n", + "url = f\"http://localhost:{port}/v1/chat/completions\"\n", + "\n", + "data = {\n", + " \"model\": \"XiaomiMiMo/MiMo-7B-RL\",\n", + " \"messages\": [{\"role\": \"user\", \"content\": \"What is the capital of France?\"}],\n", + "}\n", + "\n", + "response = requests.post(url, json=data)\n", + "print_highlight(response.json())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## References\n", + "\n", + "EAGLE process is as follows:\n", + "\n", + "- Within EAGLE the draft model predicts the next feature vector, i.e. the last hidden state of the original LLM, using the feature sequence $(f_1, ..., f_k)$ and the token sequence $(t_2, ..., t_{k+1})$. \n", + "- The next token is then sampled from $p_{k+2}=\\text{LMHead}(f_{k+1})$. Afterwards, the two sequences are extended in a tree style—branching out multiple potential continuations, with the branching factor per step controlled by the `speculative_eagle_topk` parameter—to ensure a more coherent connection of context, and are given as input again.\n", + "- EAGLE-2 additionally uses the draft model to evaluate how probable certain branches in the draft tree are, dynamically stopping the expansion of unlikely branches. After the expansion phase, reranking is employed to select only the top `speculative_num_draft_tokens` final nodes as draft tokens.\n", + "- EAGLE-3 removes the feature prediction objective, incorporates low and mid-layer features, and is trained in an on-policy manner.\n", + "\n", + "This enhances drafting accuracy by operating on the features instead of tokens for more regular inputs and passing the tokens from the next timestep additionally to minimize randomness effects from sampling. Furthermore the dynamic adjustment of the draft tree and selection of reranked final nodes increases acceptance rate of draft tokens further. For more details see [EAGLE-2](https://arxiv.org/abs/2406.16858) and [EAGLE-3](https://arxiv.org/abs/2503.01840) paper.\n", + "\n", + "\n", + "For guidance how to train your own EAGLE model please see the [EAGLE repo](https://github.com/SafeAILab/EAGLE/tree/main?tab=readme-ov-file#train)." + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/structured_outputs.ipynb b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/structured_outputs.ipynb new file mode 100644 index 00000000..b0ec5e6c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/structured_outputs.ipynb @@ -0,0 +1,998 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Structured Outputs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can specify a JSON schema, [regular expression](https://en.wikipedia.org/wiki/Regular_expression) or [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) to constrain the model output. The model output will be guaranteed to follow the given constraints. Only one constraint parameter (`json_schema`, `regex`, or `ebnf`) can be specified for a request.\n", + "\n", + "SGLang supports three grammar backends:\n", + "\n", + "- [XGrammar](https://github.com/mlc-ai/xgrammar)(default): Supports JSON schema, regular expression, and EBNF constraints.\n", + "- [Outlines](https://github.com/dottxt-ai/outlines): Supports JSON schema and regular expression constraints.\n", + "- [Llguidance](https://github.com/guidance-ai/llguidance): Supports JSON schema, regular expression, and EBNF constraints.\n", + "\n", + "We suggest using XGrammar for its better performance and utility. XGrammar currently uses the [GGML BNF format](https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md). For more details, see [XGrammar technical overview](https://blog.mlc.ai/2024/11/22/achieving-efficient-flexible-portable-structured-generation-with-xgrammar).\n", + "\n", + "To use Outlines, simply add `--grammar-backend outlines` when launching the server.\n", + "To use llguidance, add `--grammar-backend llguidance` when launching the server.\n", + "If no backend is specified, XGrammar will be used as the default.\n", + "\n", + "For better output quality, **It's advisable to explicitly include instructions in the prompt to guide the model to generate the desired format.** For example, you can specify, 'Please generate the output in the following JSON format: ...'.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## OpenAI Compatible API" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "import os\n", + "\n", + "from sglang.test.doc_patch import launch_server_cmd\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", + "\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --host 0.0.0.0 --log-level warning\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")\n", + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### JSON\n", + "\n", + "you can directly define a JSON schema or use [Pydantic](https://docs.pydantic.dev/latest/) to define and validate the response." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Using Pydantic**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pydantic import BaseModel, Field\n", + "\n", + "\n", + "# Define the schema using Pydantic\n", + "class CapitalInfo(BaseModel):\n", + " name: str = Field(..., pattern=r\"^\\w+$\", description=\"Name of the capital city\")\n", + " population: int = Field(..., description=\"Population of the capital city\")\n", + "\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Please generate the information of the capital of France in the JSON format.\",\n", + " },\n", + " ],\n", + " temperature=0,\n", + " max_tokens=128,\n", + " response_format={\n", + " \"type\": \"json_schema\",\n", + " \"json_schema\": {\n", + " \"name\": \"foo\",\n", + " # convert the pydantic model to json schema\n", + " \"schema\": CapitalInfo.model_json_schema(),\n", + " },\n", + " },\n", + ")\n", + "\n", + "response_content = response.choices[0].message.content\n", + "# validate the JSON response by the pydantic model\n", + "capital_info = CapitalInfo.model_validate_json(response_content)\n", + "print_highlight(f\"Validated response: {capital_info.model_dump_json()}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**JSON Schema Directly**\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "json_schema = json.dumps(\n", + " {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"name\": {\"type\": \"string\", \"pattern\": \"^[\\\\w]+$\"},\n", + " \"population\": {\"type\": \"integer\"},\n", + " },\n", + " \"required\": [\"name\", \"population\"],\n", + " }\n", + ")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Give me the information of the capital of France in the JSON format.\",\n", + " },\n", + " ],\n", + " temperature=0,\n", + " max_tokens=128,\n", + " response_format={\n", + " \"type\": \"json_schema\",\n", + " \"json_schema\": {\"name\": \"foo\", \"schema\": json.loads(json_schema)},\n", + " },\n", + ")\n", + "\n", + "print_highlight(response.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### EBNF" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ebnf_grammar = \"\"\"\n", + "root ::= city | description\n", + "city ::= \"London\" | \"Paris\" | \"Berlin\" | \"Rome\"\n", + "description ::= city \" is \" status\n", + "status ::= \"the capital of \" country\n", + "country ::= \"England\" | \"France\" | \"Germany\" | \"Italy\"\n", + "\"\"\"\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=[\n", + " {\"role\": \"system\", \"content\": \"You are a helpful geography bot.\"},\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Give me the information of the capital of France.\",\n", + " },\n", + " ],\n", + " temperature=0,\n", + " max_tokens=32,\n", + " extra_body={\"ebnf\": ebnf_grammar},\n", + ")\n", + "\n", + "print_highlight(response.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Regular expression" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"What is the capital of France?\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=128,\n", + " extra_body={\"regex\": \"(Paris|London)\"},\n", + ")\n", + "\n", + "print_highlight(response.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Structural Tag" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tool_get_current_weather = {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"get_current_weather\",\n", + " \"description\": \"Get the current weather in a given location\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"city\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The city to find the weather for, e.g. 'San Francisco'\",\n", + " },\n", + " \"state\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"the two-letter abbreviation for the state that the city is\"\n", + " \" in, e.g. 'CA' which would mean 'California'\",\n", + " },\n", + " \"unit\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The unit to fetch the temperature in\",\n", + " \"enum\": [\"celsius\", \"fahrenheit\"],\n", + " },\n", + " },\n", + " \"required\": [\"city\", \"state\", \"unit\"],\n", + " },\n", + " },\n", + "}\n", + "\n", + "tool_get_current_date = {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"get_current_date\",\n", + " \"description\": \"Get the current date and time for a given timezone\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"timezone\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The timezone to fetch the current date and time for, e.g. 'America/New_York'\",\n", + " }\n", + " },\n", + " \"required\": [\"timezone\"],\n", + " },\n", + " },\n", + "}\n", + "\n", + "schema_get_current_weather = tool_get_current_weather[\"function\"][\"parameters\"]\n", + "schema_get_current_date = tool_get_current_date[\"function\"][\"parameters\"]\n", + "\n", + "\n", + "def get_messages():\n", + " return [\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": f\"\"\"\n", + "# Tool Instructions\n", + "- Always execute python code in messages that you share.\n", + "- When looking for real time information use relevant functions if available else fallback to brave_search\n", + "You have access to the following functions:\n", + "Use the function 'get_current_weather' to: Get the current weather in a given location\n", + "{tool_get_current_weather[\"function\"]}\n", + "Use the function 'get_current_date' to: Get the current date and time for a given timezone\n", + "{tool_get_current_date[\"function\"]}\n", + "If a you choose to call a function ONLY reply in the following format:\n", + "<{{start_tag}}={{function_name}}>{{parameters}}{{end_tag}}\n", + "where\n", + "start_tag => ` a JSON dict with the function argument name as key and function argument value as value.\n", + "end_tag => ``\n", + "Here is an example,\n", + "{{\"example_name\": \"example_value\"}}\n", + "Reminder:\n", + "- Function calls MUST follow the specified format\n", + "- Required parameters MUST be specified\n", + "- Only call one function at a time\n", + "- Put the entire function call reply on one line\n", + "- Always add your sources when using search results to answer the user query\n", + "You are a helpful assistant.\"\"\",\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"You are in New York. Please get the current date and time, and the weather.\",\n", + " },\n", + " ]\n", + "\n", + "\n", + "messages = get_messages()\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=messages,\n", + " response_format={\n", + " \"type\": \"structural_tag\",\n", + " \"structures\": [\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_weather,\n", + " \"end\": \"\",\n", + " },\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_date,\n", + " \"end\": \"\",\n", + " },\n", + " ],\n", + " \"triggers\": [\"\",\n", + " \"content\": {\n", + " \"type\": \"json_schema\",\n", + " \"json_schema\": schema_get_current_weather,\n", + " },\n", + " \"end\": \"\",\n", + " },\n", + " {\n", + " \"begin\": \"\",\n", + " \"content\": {\n", + " \"type\": \"json_schema\",\n", + " \"json_schema\": schema_get_current_date,\n", + " },\n", + " \"end\": \"\",\n", + " },\n", + " ],\n", + " \"at_least_one\": False,\n", + " \"stop_after_first\": False,\n", + " },\n", + " },\n", + ")\n", + "\n", + "print_highlight(response.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Native API and SGLang Runtime (SRT)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### JSON" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Using Pydantic**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "import json\n", + "from pydantic import BaseModel, Field\n", + "\n", + "from transformers import AutoTokenizer\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Meta-Llama-3.1-8B-Instruct\")\n", + "\n", + "\n", + "# Define the schema using Pydantic\n", + "class CapitalInfo(BaseModel):\n", + " name: str = Field(..., pattern=r\"^\\w+$\", description=\"Name of the capital city\")\n", + " population: int = Field(..., description=\"Population of the capital city\")\n", + "\n", + "\n", + "# Make API request\n", + "messages = [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Here is the information of the capital of France in the JSON format.\\n\",\n", + " }\n", + "]\n", + "text = tokenizer.apply_chat_template(\n", + " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", + ")\n", + "response = requests.post(\n", + " f\"http://localhost:{port}/generate\",\n", + " json={\n", + " \"text\": text,\n", + " \"sampling_params\": {\n", + " \"temperature\": 0,\n", + " \"max_new_tokens\": 64,\n", + " \"json_schema\": json.dumps(CapitalInfo.model_json_schema()),\n", + " },\n", + " },\n", + ")\n", + "print_highlight(response.json())\n", + "\n", + "\n", + "response_data = json.loads(response.json()[\"text\"])\n", + "# validate the response by the pydantic model\n", + "capital_info = CapitalInfo.model_validate(response_data)\n", + "print_highlight(f\"Validated response: {capital_info.model_dump_json()}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**JSON Schema Directly**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "json_schema = json.dumps(\n", + " {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"name\": {\"type\": \"string\", \"pattern\": \"^[\\\\w]+$\"},\n", + " \"population\": {\"type\": \"integer\"},\n", + " },\n", + " \"required\": [\"name\", \"population\"],\n", + " }\n", + ")\n", + "\n", + "# JSON\n", + "response = requests.post(\n", + " f\"http://localhost:{port}/generate\",\n", + " json={\n", + " \"text\": text,\n", + " \"sampling_params\": {\n", + " \"temperature\": 0,\n", + " \"max_new_tokens\": 64,\n", + " \"json_schema\": json_schema,\n", + " },\n", + " },\n", + ")\n", + "\n", + "print_highlight(response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### EBNF" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "messages = [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Give me the information of the capital of France.\",\n", + " }\n", + "]\n", + "text = tokenizer.apply_chat_template(\n", + " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", + ")\n", + "response = requests.post(\n", + " f\"http://localhost:{port}/generate\",\n", + " json={\n", + " \"text\": text,\n", + " \"sampling_params\": {\n", + " \"max_new_tokens\": 128,\n", + " \"temperature\": 0,\n", + " \"n\": 3,\n", + " \"ebnf\": (\n", + " \"root ::= city | description\\n\"\n", + " 'city ::= \"London\" | \"Paris\" | \"Berlin\" | \"Rome\"\\n'\n", + " 'description ::= city \" is \" status\\n'\n", + " 'status ::= \"the capital of \" country\\n'\n", + " 'country ::= \"England\" | \"France\" | \"Germany\" | \"Italy\"'\n", + " ),\n", + " },\n", + " \"stream\": False,\n", + " \"return_logprob\": False,\n", + " },\n", + ")\n", + "\n", + "print_highlight(response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Regular expression" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "messages = [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Paris is the capital of\",\n", + " }\n", + "]\n", + "text = tokenizer.apply_chat_template(\n", + " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", + ")\n", + "response = requests.post(\n", + " f\"http://localhost:{port}/generate\",\n", + " json={\n", + " \"text\": text,\n", + " \"sampling_params\": {\n", + " \"temperature\": 0,\n", + " \"max_new_tokens\": 64,\n", + " \"regex\": \"(France|England)\",\n", + " },\n", + " },\n", + ")\n", + "print_highlight(response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Structural Tag" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoTokenizer\n", + "\n", + "# generate an answer\n", + "tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Meta-Llama-3.1-8B-Instruct\")\n", + "\n", + "text = tokenizer.apply_chat_template(\n", + " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", + ")\n", + "payload = {\n", + " \"text\": text,\n", + " \"sampling_params\": {\n", + " \"structural_tag\": json.dumps(\n", + " {\n", + " \"type\": \"structural_tag\",\n", + " \"structures\": [\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_weather,\n", + " \"end\": \"\",\n", + " },\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_date,\n", + " \"end\": \"\",\n", + " },\n", + " ],\n", + " \"triggers\": [\"\",\n", + " \"content\": {\n", + " \"type\": \"json_schema\",\n", + " \"json_schema\": schema_get_current_weather,\n", + " },\n", + " \"end\": \"\",\n", + " },\n", + " {\n", + " \"begin\": \"\",\n", + " \"content\": {\n", + " \"type\": \"json_schema\",\n", + " \"json_schema\": schema_get_current_date,\n", + " },\n", + " \"end\": \"\",\n", + " },\n", + " ],\n", + " \"at_least_one\": False,\n", + " \"stop_after_first\": False,\n", + " },\n", + " }\n", + " )\n", + " },\n", + "}\n", + "\n", + "\n", + "# Send POST request to the API endpoint\n", + "response = requests.post(f\"http://localhost:{port}/generate\", json=payload)\n", + "print_highlight(response.json())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Offline Engine API" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sglang as sgl\n", + "\n", + "llm = sgl.Engine(\n", + " model_path=\"meta-llama/Meta-Llama-3.1-8B-Instruct\", grammar_backend=\"xgrammar\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### JSON" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Using Pydantic**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "from pydantic import BaseModel, Field\n", + "\n", + "\n", + "prompts = [\n", + " \"Give me the information of the capital of China in the JSON format.\",\n", + " \"Give me the information of the capital of France in the JSON format.\",\n", + " \"Give me the information of the capital of Ireland in the JSON format.\",\n", + "]\n", + "\n", + "\n", + "# Define the schema using Pydantic\n", + "class CapitalInfo(BaseModel):\n", + " name: str = Field(..., pattern=r\"^\\w+$\", description=\"Name of the capital city\")\n", + " population: int = Field(..., description=\"Population of the capital city\")\n", + "\n", + "\n", + "sampling_params = {\n", + " \"temperature\": 0.1,\n", + " \"top_p\": 0.95,\n", + " \"json_schema\": json.dumps(CapitalInfo.model_json_schema()),\n", + "}\n", + "\n", + "outputs = llm.generate(prompts, sampling_params)\n", + "for prompt, output in zip(prompts, outputs):\n", + " print_highlight(\"===============================\")\n", + " print_highlight(f\"Prompt: {prompt}\") # validate the output by the pydantic model\n", + " capital_info = CapitalInfo.model_validate_json(output[\"text\"])\n", + " print_highlight(f\"Validated output: {capital_info.model_dump_json()}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**JSON Schema Directly**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "prompts = [\n", + " \"Give me the information of the capital of China in the JSON format.\",\n", + " \"Give me the information of the capital of France in the JSON format.\",\n", + " \"Give me the information of the capital of Ireland in the JSON format.\",\n", + "]\n", + "\n", + "json_schema = json.dumps(\n", + " {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"name\": {\"type\": \"string\", \"pattern\": \"^[\\\\w]+$\"},\n", + " \"population\": {\"type\": \"integer\"},\n", + " },\n", + " \"required\": [\"name\", \"population\"],\n", + " }\n", + ")\n", + "\n", + "sampling_params = {\"temperature\": 0.1, \"top_p\": 0.95, \"json_schema\": json_schema}\n", + "\n", + "outputs = llm.generate(prompts, sampling_params)\n", + "for prompt, output in zip(prompts, outputs):\n", + " print_highlight(\"===============================\")\n", + " print_highlight(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### EBNF\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "prompts = [\n", + " \"Give me the information of the capital of France.\",\n", + " \"Give me the information of the capital of Germany.\",\n", + " \"Give me the information of the capital of Italy.\",\n", + "]\n", + "\n", + "sampling_params = {\n", + " \"temperature\": 0.8,\n", + " \"top_p\": 0.95,\n", + " \"ebnf\": (\n", + " \"root ::= city | description\\n\"\n", + " 'city ::= \"London\" | \"Paris\" | \"Berlin\" | \"Rome\"\\n'\n", + " 'description ::= city \" is \" status\\n'\n", + " 'status ::= \"the capital of \" country\\n'\n", + " 'country ::= \"England\" | \"France\" | \"Germany\" | \"Italy\"'\n", + " ),\n", + "}\n", + "\n", + "outputs = llm.generate(prompts, sampling_params)\n", + "for prompt, output in zip(prompts, outputs):\n", + " print_highlight(\"===============================\")\n", + " print_highlight(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Regular expression" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "prompts = [\n", + " \"Please provide information about London as a major global city:\",\n", + " \"Please provide information about Paris as a major global city:\",\n", + "]\n", + "\n", + "sampling_params = {\"temperature\": 0.8, \"top_p\": 0.95, \"regex\": \"(France|England)\"}\n", + "\n", + "outputs = llm.generate(prompts, sampling_params)\n", + "for prompt, output in zip(prompts, outputs):\n", + " print_highlight(\"===============================\")\n", + " print_highlight(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Structural Tag" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "text = tokenizer.apply_chat_template(\n", + " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", + ")\n", + "prompts = [text]\n", + "\n", + "\n", + "sampling_params = {\n", + " \"temperature\": 0.8,\n", + " \"top_p\": 0.95,\n", + " \"structural_tag\": json.dumps(\n", + " {\n", + " \"type\": \"structural_tag\",\n", + " \"structures\": [\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_weather,\n", + " \"end\": \"\",\n", + " },\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_date,\n", + " \"end\": \"\",\n", + " },\n", + " ],\n", + " \"triggers\": [\"\",\n", + " \"content\": {\n", + " \"type\": \"json_schema\",\n", + " \"json_schema\": schema_get_current_weather,\n", + " },\n", + " \"end\": \"\",\n", + " },\n", + " {\n", + " \"begin\": \"\",\n", + " \"content\": {\n", + " \"type\": \"json_schema\",\n", + " \"json_schema\": schema_get_current_date,\n", + " },\n", + " \"end\": \"\",\n", + " },\n", + " ],\n", + " \"at_least_one\": False,\n", + " \"stop_after_first\": False,\n", + " },\n", + " }\n", + " ),\n", + "}\n", + "\n", + "\n", + "# Send POST request to the API endpoint\n", + "outputs = llm.generate(prompts, sampling_params)\n", + "for prompt, output in zip(prompts, outputs):\n", + " print_highlight(\"===============================\")\n", + " print_highlight(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "llm.shutdown()" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/structured_outputs_for_reasoning_models.ipynb b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/structured_outputs_for_reasoning_models.ipynb new file mode 100644 index 00000000..9cdcc29e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/structured_outputs_for_reasoning_models.ipynb @@ -0,0 +1,830 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Structured Outputs For Reasoning Models\n", + "\n", + "When working with reasoning models that use special tokens like `...` to denote reasoning sections, you might want to allow free-form text within these sections while still enforcing grammar constraints on the rest of the output.\n", + "\n", + "SGLang provides a feature to disable grammar restrictions within reasoning sections. This is particularly useful for models that need to perform complex reasoning steps before providing a structured output.\n", + "\n", + "To enable this feature, use the `--reasoning-parser` flag which decide the think_end_token, such as ``, when launching the server. You can also specify the reasoning parser using the `--reasoning-parser` flag.\n", + "\n", + "## Supported Models\n", + "\n", + "Currently, SGLang supports the following reasoning models:\n", + "- [DeepSeek R1 series](https://huggingface.co/collections/deepseek-ai/deepseek-r1-678e1e131c0169c0bc89728d): The reasoning content is wrapped with `` and `` tags.\n", + "- [QwQ](https://huggingface.co/Qwen/QwQ-32B): The reasoning content is wrapped with `` and `` tags.\n", + "\n", + "\n", + "## Usage\n", + "\n", + "## OpenAI Compatible API" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Specify the `--grammar-backend`, `--reasoning-parser` option." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "import os\n", + "\n", + "from sglang.test.doc_patch import launch_server_cmd\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", + "\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"python -m sglang.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-7B --host 0.0.0.0 --reasoning-parser deepseek-r1 --log-level warning\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")\n", + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### JSON\n", + "\n", + "you can directly define a JSON schema or use [Pydantic](https://docs.pydantic.dev/latest/) to define and validate the response." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Using Pydantic**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pydantic import BaseModel, Field\n", + "\n", + "\n", + "# Define the schema using Pydantic\n", + "class CapitalInfo(BaseModel):\n", + " name: str = Field(..., pattern=r\"^\\w+$\", description=\"Name of the capital city\")\n", + " population: int = Field(..., description=\"Population of the capital city\")\n", + "\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\",\n", + " messages=[\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Give me the information and population of the capital of France in the JSON format.\",\n", + " },\n", + " ],\n", + " temperature=0,\n", + " max_tokens=2048,\n", + " response_format={\n", + " \"type\": \"json_schema\",\n", + " \"json_schema\": {\n", + " \"name\": \"foo\",\n", + " # convert the pydantic model to json schema\n", + " \"schema\": CapitalInfo.model_json_schema(),\n", + " },\n", + " },\n", + ")\n", + "\n", + "print_highlight(\n", + " f\"reasoing_content: {response.choices[0].message.reasoning_content}\\n\\ncontent: {response.choices[0].message.content}\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**JSON Schema Directly**\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "json_schema = json.dumps(\n", + " {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"name\": {\"type\": \"string\", \"pattern\": \"^[\\\\w]+$\"},\n", + " \"population\": {\"type\": \"integer\"},\n", + " },\n", + " \"required\": [\"name\", \"population\"],\n", + " }\n", + ")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\",\n", + " messages=[\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Give me the information and population of the capital of France in the JSON format.\",\n", + " },\n", + " ],\n", + " temperature=0,\n", + " max_tokens=2048,\n", + " response_format={\n", + " \"type\": \"json_schema\",\n", + " \"json_schema\": {\"name\": \"foo\", \"schema\": json.loads(json_schema)},\n", + " },\n", + ")\n", + "\n", + "print_highlight(\n", + " f\"reasoing_content: {response.choices[0].message.reasoning_content}\\n\\ncontent: {response.choices[0].message.content}\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### EBNF" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ebnf_grammar = \"\"\"\n", + "root ::= city | description\n", + "city ::= \"London\" | \"Paris\" | \"Berlin\" | \"Rome\"\n", + "description ::= city \" is \" status\n", + "status ::= \"the capital of \" country\n", + "country ::= \"England\" | \"France\" | \"Germany\" | \"Italy\"\n", + "\"\"\"\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\",\n", + " messages=[\n", + " {\"role\": \"system\", \"content\": \"You are a helpful geography bot.\"},\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Give me the information and population of the capital of France in the JSON format.\",\n", + " },\n", + " ],\n", + " temperature=0,\n", + " max_tokens=2048,\n", + " extra_body={\"ebnf\": ebnf_grammar},\n", + ")\n", + "\n", + "print_highlight(\n", + " f\"reasoing_content: {response.choices[0].message.reasoning_content}\\n\\ncontent: {response.choices[0].message.content}\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Regular expression" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = client.chat.completions.create(\n", + " model=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\",\n", + " messages=[\n", + " {\"role\": \"assistant\", \"content\": \"What is the capital of France?\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=2048,\n", + " extra_body={\"regex\": \"(Paris|London)\"},\n", + ")\n", + "\n", + "print_highlight(\n", + " f\"reasoing_content: {response.choices[0].message.reasoning_content}\\n\\ncontent: {response.choices[0].message.content}\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Structural Tag" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tool_get_current_weather = {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"get_current_weather\",\n", + " \"description\": \"Get the current weather in a given location\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"city\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The city to find the weather for, e.g. 'San Francisco'\",\n", + " },\n", + " \"state\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"the two-letter abbreviation for the state that the city is\"\n", + " \" in, e.g. 'CA' which would mean 'California'\",\n", + " },\n", + " \"unit\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The unit to fetch the temperature in\",\n", + " \"enum\": [\"celsius\", \"fahrenheit\"],\n", + " },\n", + " },\n", + " \"required\": [\"city\", \"state\", \"unit\"],\n", + " },\n", + " },\n", + "}\n", + "\n", + "tool_get_current_date = {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"get_current_date\",\n", + " \"description\": \"Get the current date and time for a given timezone\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"timezone\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The timezone to fetch the current date and time for, e.g. 'America/New_York'\",\n", + " }\n", + " },\n", + " \"required\": [\"timezone\"],\n", + " },\n", + " },\n", + "}\n", + "\n", + "schema_get_current_weather = tool_get_current_weather[\"function\"][\"parameters\"]\n", + "schema_get_current_date = tool_get_current_date[\"function\"][\"parameters\"]\n", + "\n", + "\n", + "def get_messages():\n", + " return [\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": f\"\"\"\n", + "# Tool Instructions\n", + "- Always execute python code in messages that you share.\n", + "- When looking for real time information use relevant functions if available else fallback to brave_search\n", + "You have access to the following functions:\n", + "Use the function 'get_current_weather' to: Get the current weather in a given location\n", + "{tool_get_current_weather[\"function\"]}\n", + "Use the function 'get_current_date' to: Get the current date and time for a given timezone\n", + "{tool_get_current_date[\"function\"]}\n", + "If a you choose to call a function ONLY reply in the following format:\n", + "<{{start_tag}}={{function_name}}>{{parameters}}{{end_tag}}\n", + "where\n", + "start_tag => ` a JSON dict with the function argument name as key and function argument value as value.\n", + "end_tag => ``\n", + "Here is an example,\n", + "{{\"example_name\": \"example_value\"}}\n", + "Reminder:\n", + "- Function calls MUST follow the specified format\n", + "- Required parameters MUST be specified\n", + "- Only call one function at a time\n", + "- Put the entire function call reply on one line\n", + "- Always add your sources when using search results to answer the user query\n", + "You are a helpful assistant.\"\"\",\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"You are in New York. Please get the current date and time, and the weather.\",\n", + " },\n", + " ]\n", + "\n", + "\n", + "messages = get_messages()\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\",\n", + " messages=messages,\n", + " response_format={\n", + " \"type\": \"structural_tag\",\n", + " \"max_new_tokens\": 2048,\n", + " \"structures\": [\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_weather,\n", + " \"end\": \"\",\n", + " },\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_date,\n", + " \"end\": \"\",\n", + " },\n", + " ],\n", + " \"triggers\": [\"\")[0]\n", + "content = response.json()[\"text\"].split(\"\")[1]\n", + "print_highlight(f\"reasoing_content: {reasoing_content}\\n\\ncontent: {content}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**JSON Schema Directly**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "json_schema = json.dumps(\n", + " {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"name\": {\"type\": \"string\", \"pattern\": \"^[\\\\w]+$\"},\n", + " \"population\": {\"type\": \"integer\"},\n", + " },\n", + " \"required\": [\"name\", \"population\"],\n", + " }\n", + ")\n", + "\n", + "# JSON\n", + "text = tokenizer.apply_chat_template(\n", + " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", + ")\n", + "response = requests.post(\n", + " f\"http://localhost:{port}/generate\",\n", + " json={\n", + " \"text\": text,\n", + " \"sampling_params\": {\n", + " \"temperature\": 0,\n", + " \"max_new_tokens\": 2048,\n", + " \"json_schema\": json_schema,\n", + " },\n", + " },\n", + ")\n", + "\n", + "print_highlight(response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### EBNF" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = requests.post(\n", + " f\"http://localhost:{port}/generate\",\n", + " json={\n", + " \"text\": \"Give me the information of the capital of France.\",\n", + " \"sampling_params\": {\n", + " \"max_new_tokens\": 2048,\n", + " \"temperature\": 0,\n", + " \"n\": 3,\n", + " \"ebnf\": (\n", + " \"root ::= city | description\\n\"\n", + " 'city ::= \"London\" | \"Paris\" | \"Berlin\" | \"Rome\"\\n'\n", + " 'description ::= city \" is \" status\\n'\n", + " 'status ::= \"the capital of \" country\\n'\n", + " 'country ::= \"England\" | \"France\" | \"Germany\" | \"Italy\"'\n", + " ),\n", + " },\n", + " \"stream\": False,\n", + " \"return_logprob\": False,\n", + " },\n", + ")\n", + "\n", + "print(response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Regular expression" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = requests.post(\n", + " f\"http://localhost:{port}/generate\",\n", + " json={\n", + " \"text\": \"Paris is the capital of\",\n", + " \"sampling_params\": {\n", + " \"temperature\": 0,\n", + " \"max_new_tokens\": 2048,\n", + " \"regex\": \"(France|England)\",\n", + " },\n", + " },\n", + ")\n", + "print(response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Structural Tag" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "text = tokenizer.apply_chat_template(\n", + " messages, tokenize=False, add_generation_prompt=True, return_dict=False\n", + ")\n", + "payload = {\n", + " \"text\": text,\n", + " \"sampling_params\": {\n", + " \"max_new_tokens\": 2048,\n", + " \"structural_tag\": json.dumps(\n", + " {\n", + " \"type\": \"structural_tag\",\n", + " \"structures\": [\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_weather,\n", + " \"end\": \"\",\n", + " },\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_date,\n", + " \"end\": \"\",\n", + " },\n", + " ],\n", + " \"triggers\": [\"\",\n", + " \"schema\": schema_get_current_weather,\n", + " \"end\": \"\",\n", + " },\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_date,\n", + " \"end\": \"\",\n", + " },\n", + " ],\n", + " \"triggers\": [\" is not trimmed.\n", + "\n", + "sampling_params = {\n", + " \"max_new_tokens\": 1024,\n", + " \"temperature\": 0,\n", + " \"top_p\": 0.95,\n", + " \"skip_special_tokens\": False,\n", + "}\n", + "\n", + "# 1) Offline generation\n", + "result = llm.generate(input_ids=input_ids, sampling_params=sampling_params)\n", + "generated_text = result[\"text\"] # Assume there is only one prompt\n", + "\n", + "print_highlight(\"=== Offline Engine Output Text ===\")\n", + "print_highlight(generated_text)\n", + "\n", + "\n", + "# 2) Parse using FunctionCallParser\n", + "def convert_dict_to_tool(tool_dict: dict) -> Tool:\n", + " function_dict = tool_dict.get(\"function\", {})\n", + " return Tool(\n", + " type=tool_dict.get(\"type\", \"function\"),\n", + " function=Function(\n", + " name=function_dict.get(\"name\"),\n", + " description=function_dict.get(\"description\"),\n", + " parameters=function_dict.get(\"parameters\"),\n", + " ),\n", + " )\n", + "\n", + "\n", + "tools = [convert_dict_to_tool(raw_tool) for raw_tool in tools]\n", + "\n", + "parser = FunctionCallParser(tools=tools, tool_call_parser=\"qwen25\")\n", + "normal_text, calls = parser.parse_non_stream(generated_text)\n", + "\n", + "print_highlight(\"=== Parsing Result ===\")\n", + "print(\"Normal text portion:\", normal_text)\n", + "print_highlight(\"Function call portion:\")\n", + "for call in calls:\n", + " # call: ToolCallItem\n", + " print_highlight(f\" - tool name: {call.name}\")\n", + " print_highlight(f\" parameters: {call.parameters}\")\n", + "\n", + "# 3) If needed, perform additional logic on the parsed functions, such as automatically calling the corresponding function to obtain a return value, etc." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "llm.shutdown()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tool Choice Mode\n", + "\n", + "SGLang supports OpenAI's `tool_choice` parameter to control when and which tools the model should call. This feature is implemented using EBNF (Extended Backus-Naur Form) grammar to ensure reliable tool calling behavior.\n", + "\n", + "### Supported Tool Choice Options\n", + "\n", + "- **`tool_choice=\"required\"`**: Forces the model to call at least one tool\n", + "- **`tool_choice={\"type\": \"function\", \"function\": {\"name\": \"specific_function\"}}`**: Forces the model to call a specific function\n", + "\n", + "### Backend Compatibility\n", + "\n", + "Tool choice is fully supported with the **Xgrammar backend**, which is the default grammar backend (`--grammar-backend xgrammar`). However, it may not be fully supported with other backends such as `outlines`.\n", + "\n", + "### Example: Required Tool Choice" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from openai import OpenAI\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "from sglang.test.doc_patch import launch_server_cmd\n", + "\n", + "# Start a new server session for tool choice examples\n", + "server_process_tool_choice, port_tool_choice = launch_server_cmd(\n", + " \"python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --tool-call-parser qwen25 --host 0.0.0.0 --log-level warning\"\n", + ")\n", + "wait_for_server(f\"http://localhost:{port_tool_choice}\")\n", + "\n", + "# Initialize client for tool choice examples\n", + "client_tool_choice = OpenAI(\n", + " api_key=\"None\", base_url=f\"http://0.0.0.0:{port_tool_choice}/v1\"\n", + ")\n", + "model_name_tool_choice = client_tool_choice.models.list().data[0].id\n", + "\n", + "# Example with tool_choice=\"required\" - forces the model to call a tool\n", + "messages_required = [\n", + " {\"role\": \"user\", \"content\": \"Hello, what is the capital of France?\"}\n", + "]\n", + "\n", + "# Define tools\n", + "tools = [\n", + " {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"get_current_weather\",\n", + " \"description\": \"Get the current weather in a given location\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"city\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The city to find the weather for, e.g. 'San Francisco'\",\n", + " },\n", + " \"unit\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The unit to fetch the temperature in\",\n", + " \"enum\": [\"celsius\", \"fahrenheit\"],\n", + " },\n", + " },\n", + " \"required\": [\"city\", \"unit\"],\n", + " },\n", + " },\n", + " }\n", + "]\n", + "\n", + "response_required = client_tool_choice.chat.completions.create(\n", + " model=model_name_tool_choice,\n", + " messages=messages_required,\n", + " temperature=0,\n", + " max_tokens=1024,\n", + " tools=tools,\n", + " tool_choice=\"required\", # Force the model to call a tool\n", + ")\n", + "\n", + "print_highlight(\"Response with tool_choice='required':\")\n", + "print(\"Content:\", response_required.choices[0].message.content)\n", + "print(\"Tool calls:\", response_required.choices[0].message.tool_calls)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Example: Specific Function Choice\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example with specific function choice - forces the model to call a specific function\n", + "messages_specific = [\n", + " {\"role\": \"user\", \"content\": \"What are the most attactive places in France?\"}\n", + "]\n", + "\n", + "response_specific = client_tool_choice.chat.completions.create(\n", + " model=model_name_tool_choice,\n", + " messages=messages_specific,\n", + " temperature=0,\n", + " max_tokens=1024,\n", + " tools=tools,\n", + " tool_choice={\n", + " \"type\": \"function\",\n", + " \"function\": {\"name\": \"get_current_weather\"},\n", + " }, # Force the model to call the specific get_current_weather function\n", + ")\n", + "\n", + "print_highlight(\"Response with specific function choice:\")\n", + "print(\"Content:\", response_specific.choices[0].message.content)\n", + "print(\"Tool calls:\", response_specific.choices[0].message.tool_calls)\n", + "\n", + "if response_specific.choices[0].message.tool_calls:\n", + " tool_call = response_specific.choices[0].message.tool_calls[0]\n", + " print_highlight(f\"Called function: {tool_call.function.name}\")\n", + " print_highlight(f\"Arguments: {tool_call.function.arguments}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process_tool_choice)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Pythonic Tool Call Format (Llama-3.2 / Llama-3.3 / Llama-4)\n", + "\n", + "Some Llama models (such as Llama-3.2-1B, Llama-3.2-3B, Llama-3.3-70B, and Llama-4) support a \"pythonic\" tool call format, where the model outputs function calls as Python code, e.g.:\n", + "\n", + "```python\n", + "[get_current_weather(city=\"San Francisco\", state=\"CA\", unit=\"celsius\")]\n", + "```\n", + "\n", + "- The output is a Python list of function calls, with arguments as Python literals (not JSON).\n", + "- Multiple tool calls can be returned in the same list:\n", + "```python\n", + "[get_current_weather(city=\"San Francisco\", state=\"CA\", unit=\"celsius\"),\n", + " get_current_weather(city=\"New York\", state=\"NY\", unit=\"fahrenheit\")]\n", + "```\n", + "\n", + "For more information, refer to Meta’s documentation on [Zero shot function calling](https://github.com/meta-llama/llama-models/blob/main/models/llama4/prompt_format.md#zero-shot-function-calling---system-message).\n", + "\n", + "Note that this feature is still under development on Blackwell.\n", + "\n", + "### How to enable\n", + "- Launch the server with `--tool-call-parser pythonic`\n", + "- You may also specify --chat-template with the improved template for the model (e.g., `--chat-template=examples/chat_template/tool_chat_template_llama4_pythonic.jinja`).\n", + "This is recommended because the model expects a special prompt format to reliably produce valid pythonic tool call outputs. The template ensures that the prompt structure (e.g., special tokens, message boundaries like `<|eom|>`, and function call delimiters) matches what the model was trained or fine-tuned on. If you do not use the correct chat template, tool calling may fail or produce inconsistent results.\n", + "\n", + "#### Forcing Pythonic Tool Call Output Without a Chat Template\n", + "If you don't want to specify a chat template, you must give the model extremely explicit instructions in your messages to enforce pythonic output. For example, for `Llama-3.2-1B-Instruct`, you need:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \" python3 -m sglang.launch_server --model-path meta-llama/Llama-3.2-1B-Instruct --tool-call-parser pythonic --tp 1 --log-level warning\" # llama-3.2-1b-instruct\n", + ")\n", + "wait_for_server(f\"http://localhost:{port}\")\n", + "\n", + "tools = [\n", + " {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"get_weather\",\n", + " \"description\": \"Get the current weather for a given location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"location\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The name of the city or location.\",\n", + " }\n", + " },\n", + " \"required\": [\"location\"],\n", + " },\n", + " },\n", + " },\n", + " {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"get_tourist_attractions\",\n", + " \"description\": \"Get a list of top tourist attractions for a given city.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"city\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The name of the city to find attractions for.\",\n", + " }\n", + " },\n", + " \"required\": [\"city\"],\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "\n", + "def get_messages():\n", + " return [\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": (\n", + " \"You are a travel assistant. \"\n", + " \"When asked to call functions, ALWAYS respond ONLY with a python list of function calls, \"\n", + " \"using this format: [func_name1(param1=value1, param2=value2), func_name2(param=value)]. \"\n", + " \"Do NOT use JSON, do NOT use variables, do NOT use any other format. \"\n", + " \"Here is an example:\\n\"\n", + " '[get_weather(location=\"Paris\"), get_tourist_attractions(city=\"Paris\")]'\n", + " ),\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": (\n", + " \"I'm planning a trip to Tokyo next week. What's the weather like and what are some top tourist attractions? \"\n", + " \"Propose parallel tool calls at once, using the python list of function calls format as shown above.\"\n", + " ),\n", + " },\n", + " ]\n", + "\n", + "\n", + "messages = get_messages()\n", + "\n", + "client = openai.Client(base_url=f\"http://localhost:{port}/v1\", api_key=\"xxxxxx\")\n", + "model_name = client.models.list().data[0].id\n", + "\n", + "\n", + "response_non_stream = client.chat.completions.create(\n", + " model=model_name,\n", + " messages=messages,\n", + " temperature=0,\n", + " top_p=0.9,\n", + " stream=False, # Non-streaming\n", + " tools=tools,\n", + ")\n", + "print_highlight(\"Non-stream response:\")\n", + "print_highlight(response_non_stream)\n", + "\n", + "response_stream = client.chat.completions.create(\n", + " model=model_name,\n", + " messages=messages,\n", + " temperature=0,\n", + " top_p=0.9,\n", + " stream=True,\n", + " tools=tools,\n", + ")\n", + "texts = \"\"\n", + "tool_calls = []\n", + "name = \"\"\n", + "arguments = \"\"\n", + "\n", + "for chunk in response_stream:\n", + " if chunk.choices[0].delta.content:\n", + " texts += chunk.choices[0].delta.content\n", + " if chunk.choices[0].delta.tool_calls:\n", + " tool_calls.append(chunk.choices[0].delta.tool_calls[0])\n", + "\n", + "print_highlight(\"Streaming Response:\")\n", + "print_highlight(\"==== Text ====\")\n", + "print_highlight(texts)\n", + "\n", + "print_highlight(\"==== Tool Call ====\")\n", + "for tool_call in tool_calls:\n", + " print_highlight(tool_call)\n", + "\n", + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "> **Note:** \n", + "> The model may still default to JSON if it was heavily finetuned on that format. Prompt engineering (including examples) is the only way to increase the chance of pythonic output if you are not using a chat template." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## How to support a new model?\n", + "1. Update the TOOLS_TAG_LIST in sglang/srt/function_call_parser.py with the model’s tool tags. Currently supported tags include:\n", + "```\n", + "\tTOOLS_TAG_LIST = [\n", + "\t “<|plugin|>“,\n", + "\t ““,\n", + "\t “<|python_tag|>“,\n", + "\t “[TOOL_CALLS]”\n", + "\t]\n", + "```\n", + "2. Create a new detector class in sglang/srt/function_call_parser.py that inherits from BaseFormatDetector. The detector should handle the model’s specific function call format. For example:\n", + "```\n", + " class NewModelDetector(BaseFormatDetector):\n", + "```\n", + "3. Add the new detector to the MultiFormatParser class that manages all the format detectors." + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/vlm_query.ipynb b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/vlm_query.ipynb new file mode 100644 index 00000000..c753f2fd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/advanced_features/vlm_query.ipynb @@ -0,0 +1,325 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Query Vision Language Model" + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## Querying Qwen-VL" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "import nest_asyncio\n", + "\n", + "nest_asyncio.apply() # Run this first.\n", + "\n", + "model_path = \"Qwen/Qwen2.5-VL-3B-Instruct\"\n", + "chat_template = \"qwen2-vl\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "# Lets create a prompt.\n", + "\n", + "from io import BytesIO\n", + "import requests\n", + "from PIL import Image\n", + "\n", + "from sglang.srt.parser.conversation import chat_templates\n", + "\n", + "image = Image.open(\n", + " BytesIO(\n", + " requests.get(\n", + " \"https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true\"\n", + " ).content\n", + " )\n", + ")\n", + "\n", + "conv = chat_templates[chat_template].copy()\n", + "conv.append_message(conv.roles[0], f\"What's shown here: {conv.image_token}?\")\n", + "conv.append_message(conv.roles[1], \"\")\n", + "conv.image_data = [image]\n", + "\n", + "print(conv.get_prompt())\n", + "image" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, + "source": [ + "### Query via the offline Engine API" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "from sglang import Engine\n", + "\n", + "llm = Engine(\n", + " model_path=model_path, chat_template=chat_template, mem_fraction_static=0.8\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "out = llm.generate(prompt=conv.get_prompt(), image_data=[image])\n", + "print(out[\"text\"])" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "### Query via the offline Engine API, but send precomputed embeddings" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "# Compute the image embeddings using Huggingface.\n", + "\n", + "from transformers import AutoProcessor\n", + "from transformers import Qwen2_5_VLForConditionalGeneration\n", + "\n", + "processor = AutoProcessor.from_pretrained(model_path, use_fast=True)\n", + "vision = (\n", + " Qwen2_5_VLForConditionalGeneration.from_pretrained(model_path).eval().visual.cuda()\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "processed_prompt = processor(\n", + " images=[image], text=conv.get_prompt(), return_tensors=\"pt\"\n", + ")\n", + "input_ids = processed_prompt[\"input_ids\"][0].detach().cpu().tolist()\n", + "precomputed_embeddings = vision(\n", + " processed_prompt[\"pixel_values\"].cuda(), processed_prompt[\"image_grid_thw\"].cuda()\n", + ")\n", + "\n", + "mm_item = dict(\n", + " modality=\"IMAGE\",\n", + " image_grid_thw=processed_prompt[\"image_grid_thw\"],\n", + " precomputed_embeddings=precomputed_embeddings,\n", + ")\n", + "out = llm.generate(input_ids=input_ids, image_data=[mm_item])\n", + "print(out[\"text\"])" + ] + }, + { + "cell_type": "markdown", + "id": "10", + "metadata": {}, + "source": [ + "## Querying Llama 4 (Vision)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "import nest_asyncio\n", + "\n", + "nest_asyncio.apply() # Run this first.\n", + "\n", + "model_path = \"meta-llama/Llama-4-Scout-17B-16E-Instruct\"\n", + "chat_template = \"llama-4\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "# Lets create a prompt.\n", + "\n", + "from io import BytesIO\n", + "import requests\n", + "from PIL import Image\n", + "\n", + "from sglang.srt.parser.conversation import chat_templates\n", + "\n", + "image = Image.open(\n", + " BytesIO(\n", + " requests.get(\n", + " \"https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true\"\n", + " ).content\n", + " )\n", + ")\n", + "\n", + "conv = chat_templates[chat_template].copy()\n", + "conv.append_message(conv.roles[0], f\"What's shown here: {conv.image_token}?\")\n", + "conv.append_message(conv.roles[1], \"\")\n", + "conv.image_data = [image]\n", + "\n", + "print(conv.get_prompt())\n", + "print(f\"Image size: {image.size}\")\n", + "\n", + "image" + ] + }, + { + "cell_type": "markdown", + "id": "13", + "metadata": {}, + "source": [ + "### Query via the offline Engine API" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14", + "metadata": {}, + "outputs": [], + "source": [ + "from sglang.test.test_utils import is_in_ci\n", + "\n", + "if not is_in_ci():\n", + " from sglang import Engine\n", + "\n", + " llm = Engine(\n", + " model_path=model_path,\n", + " trust_remote_code=True,\n", + " enable_multimodal=True,\n", + " mem_fraction_static=0.8,\n", + " tp_size=4,\n", + " attention_backend=\"fa3\",\n", + " context_length=65536,\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "if not is_in_ci():\n", + " out = llm.generate(prompt=conv.get_prompt(), image_data=[image])\n", + " print(out[\"text\"])" + ] + }, + { + "cell_type": "markdown", + "id": "16", + "metadata": {}, + "source": [ + "### Query via the offline Engine API, but send precomputed embeddings" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [ + "if not is_in_ci():\n", + " # Compute the image embeddings using Huggingface.\n", + "\n", + " from transformers import AutoProcessor\n", + " from transformers import Llama4ForConditionalGeneration\n", + "\n", + " processor = AutoProcessor.from_pretrained(model_path, use_fast=True)\n", + " model = Llama4ForConditionalGeneration.from_pretrained(\n", + " model_path, torch_dtype=\"auto\"\n", + " ).eval()\n", + " vision = model.vision_model.cuda()\n", + " multi_modal_projector = model.multi_modal_projector.cuda()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18", + "metadata": {}, + "outputs": [], + "source": [ + "if not is_in_ci():\n", + " processed_prompt = processor(\n", + " images=[image], text=conv.get_prompt(), return_tensors=\"pt\"\n", + " )\n", + " print(f'{processed_prompt[\"pixel_values\"].shape=}')\n", + " input_ids = processed_prompt[\"input_ids\"][0].detach().cpu().tolist()\n", + "\n", + " image_outputs = vision(\n", + " processed_prompt[\"pixel_values\"].to(\"cuda\"), output_hidden_states=False\n", + " )\n", + " image_features = image_outputs.last_hidden_state\n", + " vision_flat = image_features.view(-1, image_features.size(-1))\n", + " precomputed_embeddings = multi_modal_projector(vision_flat)\n", + "\n", + " mm_item = dict(modality=\"IMAGE\", precomputed_embeddings=precomputed_embeddings)\n", + " out = llm.generate(input_ids=input_ids, image_data=[mm_item])\n", + " print(out[\"text\"])" + ] + } + ], + "metadata": { + "jupytext": { + "cell_metadata_filter": "-all", + "custom_cell_magics": "kql", + "encoding": "# -*- coding: utf-8 -*-" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/deepseek_v3.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/deepseek_v3.md new file mode 100644 index 00000000..285a318f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/deepseek_v3.md @@ -0,0 +1,280 @@ +# DeepSeek V3/V3.1/R1 Usage + +SGLang provides many optimizations specifically designed for the DeepSeek models, making it the inference engine recommended by the official [DeepSeek team](https://github.com/deepseek-ai/DeepSeek-V3/tree/main?tab=readme-ov-file#62-inference-with-sglang-recommended) from Day 0. + +This document outlines current optimizations for DeepSeek. +For an overview of the implemented features see the completed [Roadmap](https://github.com/sgl-project/sglang/issues/2591). + +## Launch DeepSeek V3.1/V3/R1 with SGLang + +To run DeepSeek V3.1/V3/R1 models, the recommended settings are as follows: + +| Weight Type | Configuration | +|------------|-------------------| +| **Full precision FP8**
*(recommended)* | 8 x H200 | +| | 8 x MI300X | +| | 2 x 8 x H100/800/20 | +| | Xeon 6980P CPU | +| **Full precision BF16** | 2 x 8 x H200 | +| | 2 x 8 x MI300X | +| | 4 x 8 x H100/800/20 | +| | 4 x 8 x A100/A800 | +| **Quantized weights (AWQ)** | 8 x H100/800/20 | +| | 8 x A100/A800 | +| **Quantized weights (int8)** | 16 x A100/800 | +| | 32 x L40S | +| | Xeon 6980P CPU | +| | 2 x Atlas 800I A3 | + + + +Detailed commands for reference: + +- [8 x H200](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#using-docker-recommended) +- [8 x MI300X](../platforms/amd_gpu.md#running-deepseek-v3) +- [2 x 8 x H200](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-two-h208-nodes) +- [4 x 8 x A100](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-four-a1008-nodes) +- [8 x A100 (AWQ)](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-8-a100a800-with-awq-quantization) +- [16 x A100 (int8)](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-16-a100a800-with-int8-quantization) +- [32 x L40S (int8)](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-32-l40s-with-int8-quantization) +- [Xeon 6980P CPU](../platforms/cpu_server.md#example-running-deepseek-r1) +- [2 x Atlas 800I A3 (int8)](../platforms/ascend_npu.md#running-deepseek-v3) + +### Download Weights +If you encounter errors when starting the server, ensure the weights have finished downloading. It's recommended to download them beforehand or restart multiple times until all weights are downloaded. Please refer to [DeepSeek V3](https://huggingface.co/deepseek-ai/DeepSeek-V3-Base#61-inference-with-deepseek-infer-demo-example-only) official guide to download the weights. + +### Launch with one node of 8 x H200 +Please refer to [the example](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#installation--launch). +**Note that Deepseek V3 is already in FP8**, so we should not run it with any quantization arguments like `--quantization fp8 --kv-cache-dtype fp8_e5m2`. + +### Running examples on Multi-node + +- [Serving with two H20*8 nodes](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-two-h208-nodes). + +- [Serving with two H200*8 nodes and docker](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-two-h2008-nodes-and-docker). + +- [Serving with four A100*8 nodes](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-four-a1008-nodes). + +## Optimizations + +### Multi-head Latent Attention (MLA) Throughput Optimizations + +**Description**: [MLA](https://arxiv.org/pdf/2405.04434) is an innovative attention mechanism introduced by the DeepSeek team, aimed at improving inference efficiency. SGLang has implemented specific optimizations for this, including: + +- **Weight Absorption**: By applying the associative law of matrix multiplication to reorder computation steps, this method balances computation and memory access and improves efficiency in the decoding phase. + +- **MLA Attention Backends**: Currently SGLang supports different optimized MLA attention backends, including [FlashAttention3](https://github.com/Dao-AILab/flash-attention), [Flashinfer](https://docs.flashinfer.ai/api/attention.html#flashinfer-mla), [FlashMLA](https://github.com/deepseek-ai/FlashMLA), [CutlassMLA](https://github.com/sgl-project/sglang/pull/5390), **TRTLLM MLA** (optimized for Blackwell architecture), and [Triton](https://github.com/triton-lang/triton) backends. The default FA3 provides good performance across wide workloads. + +- **FP8 Quantization**: W8A8 FP8 and KV Cache FP8 quantization enables efficient FP8 inference. Additionally, we have implemented Batched Matrix Multiplication (BMM) operator to facilitate FP8 inference in MLA with weight absorption. + +- **CUDA Graph & Torch.compile**: Both MLA and Mixture of Experts (MoE) are compatible with CUDA Graph and Torch.compile, which reduces latency and accelerates decoding speed for small batch sizes. + +- **Chunked Prefix Cache**: Chunked prefix cache optimization can increase throughput by cutting prefix cache into chunks, processing them with multi-head attention and merging their states. Its improvement can be significant when doing chunked prefill on long sequences. Currently this optimization is only available for FlashAttention3 backend. + +Overall, with these optimizations, we have achieved up to **7x** acceleration in output throughput compared to the previous version. + +

+ Multi-head Latent Attention for DeepSeek Series Models +

+ +**Usage**: MLA optimization is enabled by default. For MLA models on Blackwell architecture (e.g., B200), the default backend is FlashInfer. To use the optimized TRTLLM MLA backend for prefill and decode operations, explicitly specify `--attention-backend trtllm_mla`. + +**Reference**: Check [Blog](https://lmsys.org/blog/2024-09-04-sglang-v0-3/#deepseek-multi-head-latent-attention-mla-throughput-optimizations) and [Slides](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/lmsys_1st_meetup_deepseek_mla.pdf) for more details. + +### Data Parallelism Attention + +**Description**: This optimization involves data parallelism (DP) for the MLA attention mechanism of DeepSeek Series Models, which allows for a significant reduction in the KV cache size, enabling larger batch sizes. Each DP worker independently handles different types of batches (prefill, decode, idle), which are then synchronized before and after processing through the Mixture-of-Experts (MoE) layer. If you do not use DP attention, KV cache will be duplicated among all TP ranks. + +

+ Data Parallelism Attention for DeepSeek Series Models +

+ +With data parallelism attention enabled, we have achieved up to **1.9x** decoding throughput improvement compared to the previous version. + +

+ Data Parallelism Attention Performance Comparison +

+ +**Usage**: +- Append `--enable-dp-attention --tp 8 --dp 8` to the server arguments when using 8 H200 GPUs. This optimization improves peak throughput in high batch size scenarios where the server is limited by KV cache capacity. However, it is not recommended for low-latency, small-batch use cases. +- DP and TP attention can be flexibly combined. For example, to deploy DeepSeek-V3/R1 on 2 nodes with 8 H100 GPUs each, you can specify `--enable-dp-attention --tp 16 --dp 2`. This configuration runs attention with 2 DP groups, each containing 8 TP GPUs. + +**Reference**: Check [Blog](https://lmsys.org/blog/2024-12-04-sglang-v0-4/#data-parallelism-attention-for-deepseek-models). + +### Multi Node Tensor Parallelism + +**Description**: For users with limited memory on a single node, SGLang supports serving DeepSeek Series Models, including DeepSeek V3, across multiple nodes using tensor parallelism. This approach partitions the model parameters across multiple GPUs or nodes to handle models that are too large for one node's memory. + +**Usage**: Check [here](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-2-h208) for usage examples. + +### Block-wise FP8 + +**Description**: SGLang implements block-wise FP8 quantization with two key optimizations: + +- **Activation**: E4M3 format using per-token-per-128-channel sub-vector scales with online casting. + +- **Weight**: Per-128x128-block quantization for better numerical stability. + +- **DeepGEMM**: The [DeepGEMM](https://github.com/deepseek-ai/DeepGEMM) kernel library optimized for FP8 matrix multiplications. + +**Usage**: The activation and weight optimization above are turned on by default for DeepSeek V3 models. DeepGEMM is enabled by default on NVIDIA Hopper GPUs and disabled by default on other devices. DeepGEMM can also be manually turned off by setting the environment variable `SGLANG_ENABLE_JIT_DEEPGEMM=0`. + +Before serving the DeepSeek model, precompile the DeepGEMM kernels using: +```bash +python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code +``` +The precompilation process typically takes around 10 minutes to complete. + +### Multi-token Prediction +**Description**: SGLang implements DeepSeek V3 Multi-Token Prediction (MTP) based on [EAGLE speculative decoding](https://docs.sglang.io/advanced_features/speculative_decoding.html#EAGLE-Decoding). With this optimization, the decoding speed can be improved by **1.8x** for batch size 1 and **1.5x** for batch size 32 respectively on H200 TP8 setting. + +**Usage**: +Add arguments `--speculative-algorithm`, `--speculative-num-steps`, `--speculative-eagle-topk` and `--speculative-num-draft-tokens` to enable this feature. For example: +``` +python3 -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-V3-0324 \ + --speculative-algorithm EAGLE \ + --speculative-num-steps 1 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 2 \ + --trust-remote-code \ + --tp 8 +``` +- The best configuration for `--speculative-num-steps`, `--speculative-eagle-topk` and `--speculative-num-draft-tokens` can be searched with [bench_speculative.py](https://github.com/sgl-project/sglang/blob/main/scripts/playground/bench_speculative.py) script for given batch size. The minimum configuration is `--speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2`, which can achieve speedup for larger batch sizes. +- FlashAttention3, FlashMLA, and Triton backend fully supports MTP usage. For FlashInfer backend (`--attention-backend flashinfer`) with speculative decoding,`--speculative-eagle-topk` parameter should be set to `1`. MTP support for the CutlassMLA and TRTLLM MLA backends are still under development. +- To enable DeepSeek MTP for large batch sizes (>32), there are some parameters should be changed (Reference [this discussion](https://github.com/sgl-project/sglang/issues/4543#issuecomment-2737413756)): + - Adjust `--max-running-requests` to a larger number. The default value is `48` for MTP. For larger batch sizes, you should increase this value beyond the default value. + - Set `--cuda-graph-bs`. It's a list of batch sizes for cuda graph capture. The default captured batch sizes for speculative decoding is set [here](https://github.com/sgl-project/sglang/blob/49420741746c8f3e80e0eb17e7d012bfaf25793a/python/sglang/srt/model_executor/cuda_graph_runner.py#L126). You can include more batch sizes into it. + + +### Reasoning Content for DeepSeek R1 & V3.1 + +See [Reasoning Parser](https://docs.sglang.io/advanced_features/separate_reasoning.html) and [Thinking Parameter for DeepSeek V3.1](https://docs.sglang.io/basic_usage/openai_api_completions.html#Example:-DeepSeek-V3-Models). + + +### Function calling for DeepSeek Models + +Add arguments `--tool-call-parser deepseekv3` and `--chat-template ./examples/chat_template/tool_chat_template_deepseekv3.jinja`(recommended) to enable this feature. For example (running on 1 * H20 node): + +``` +python3 -m sglang.launch_server \ + --model deepseek-ai/DeepSeek-V3-0324 \ + --tp 8 \ + --port 30000 \ + --host 0.0.0.0 \ + --mem-fraction-static 0.9 \ + --tool-call-parser deepseekv3 \ + --chat-template ./examples/chat_template/tool_chat_template_deepseekv3.jinja +``` + +Sample Request: + +``` +curl "http://127.0.0.1:30000/v1/chat/completions" \ +-H "Content-Type: application/json" \ +-d '{"temperature": 0, "max_tokens": 100, "model": "deepseek-ai/DeepSeek-V3-0324", "tools": [{"type": "function", "function": {"name": "query_weather", "description": "Get weather of an city, the user should supply a city first", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city, e.g. Beijing"}}, "required": ["city"]}}}], "messages": [{"role": "user", "content": "Hows the weather like in Qingdao today"}]}' +``` + +Expected Response + +``` +{"id":"6501ef8e2d874006bf555bc80cddc7c5","object":"chat.completion","created":1745993638,"model":"deepseek-ai/DeepSeek-V3-0324","choices":[{"index":0,"message":{"role":"assistant","content":null,"reasoning_content":null,"tool_calls":[{"id":"0","index":null,"type":"function","function":{"name":"query_weather","arguments":"{\"city\": \"Qingdao\"}"}}]},"logprobs":null,"finish_reason":"tool_calls","matched_stop":null}],"usage":{"prompt_tokens":116,"total_tokens":138,"completion_tokens":22,"prompt_tokens_details":null}} + +``` +Sample Streaming Request: +``` +curl "http://127.0.0.1:30000/v1/chat/completions" \ +-H "Content-Type: application/json" \ +-d '{"temperature": 0, "max_tokens": 100, "model": "deepseek-ai/DeepSeek-V3-0324","stream":true,"tools": [{"type": "function", "function": {"name": "query_weather", "description": "Get weather of an city, the user should supply a city first", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city, e.g. Beijing"}}, "required": ["city"]}}}], "messages": [{"role": "user", "content": "Hows the weather like in Qingdao today"}]}' +``` +Expected Streamed Chunks (simplified for clarity): +``` +data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"{\""}}]}}]} +data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"city"}}]}}]} +data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"\":\""}}]}}]} +data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"Q"}}]}}]} +data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"ing"}}]}}]} +data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"dao"}}]}}]} +data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"\"}"}}]}}]} +data: {"choices":[{"delta":{"tool_calls":null}}], "finish_reason": "tool_calls"} +data: [DONE] +``` +The client needs to concatenate all arguments fragments to reconstruct the complete tool call: +``` +{"city": "Qingdao"} +``` +Important Notes: +1. Use a lower `"temperature"` value for better results. +2. To receive more consistent tool call results, it is recommended to use `--chat-template examples/chat_template/tool_chat_template_deepseekv3.jinja`. It provides an improved unified prompt. + + +### Thinking Budget for DeepSeek R1 + +In SGLang, we can implement thinking budget with `CustomLogitProcessor`. + +Launch a server with `--enable-custom-logit-processor` flag on. + +``` +python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-R1 --tp 8 --port 30000 --host 0.0.0.0 --mem-fraction-static 0.9 --disable-cuda-graph --reasoning-parser deepseek-r1 --enable-custom-logit-processor +``` + +Sample Request: + +```python +import openai +from rich.pretty import pprint +from sglang.srt.sampling.custom_logit_processor import DeepSeekR1ThinkingBudgetLogitProcessor + + +client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="*") +response = client.chat.completions.create( + model="deepseek-ai/DeepSeek-R1", + messages=[ + { + "role": "user", + "content": "Question: Is Paris the Capital of France?", + } + ], + max_tokens=1024, + extra_body={ + "custom_logit_processor": DeepSeekR1ThinkingBudgetLogitProcessor().to_str(), + "custom_params": { + "thinking_budget": 512, + }, + }, +) +pprint(response) +``` + +## FAQ + +**Q: Model loading is taking too long, and I'm encountering an NCCL timeout. What should I do?** + +A: If you're experiencing extended model loading times and an NCCL timeout, you can try increasing the timeout duration. Add the argument `--dist-timeout 3600` when launching your model. This will set the timeout to one hour, which often resolves the issue. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/deepseek_v32.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/deepseek_v32.md new file mode 100644 index 00000000..229c7c5a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/deepseek_v32.md @@ -0,0 +1,234 @@ +# DeepSeek V3.2 Usage + +[DeepSeek-V3.2-Exp](https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp) equips DeepSeek-V3.1-Terminus with DeepSeek Sparse Attention (DSA) through continued training. With DSA, a fine-grained sparse attention mechanism powered by a lightning indexer, DeepSeek-V3.2 achieves efficiency improvements in long-context scenarios. + +For reporting issues or tracking upcoming features, please refer to this [Roadmap](https://github.com/sgl-project/sglang/issues/11060). + +## Installation + +### Docker + +```bash +# H200/B200 +docker pull lmsysorg/sglang:latest + +# MI350/MI355 +docker pull lmsysorg/sglang:dsv32-rocm + +# NPUs +docker pull lmsysorg/sglang:dsv32-a2 +docker pull lmsysorg/sglang:dsv32-a3 +``` + +### Build From Source + +```bash +# Install SGLang +git clone https://github.com/sgl-project/sglang +cd sglang +pip3 install pip --upgrade +pip3 install -e "python" +``` +## Launch DeepSeek V3.2 with SGLang + +To serve DeepSeek-V3.2-Exp on 8xH200/B200 GPUs: + +```bash +# Launch with TP + DP (Recommended) +python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --dp 8 --enable-dp-attention + +# Launch with EP + DP +python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --ep 8 --dp 8 --enable-dp-attention + +# Launch with Pure TP +python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 +``` + +### Configuration Tips +- **DP Attention (Recommended)**: For DeepSeek V3.2 model, the kernels are customized for the use case of `dp_size=8`, so DP attention (`--dp 8 --enable-dp-attention`) is the recommended configuration for better stability and performance. All test cases use this configuration by default. +- **Pure TP Mode**: Launching with pure TP (without `--dp` and `--enable-dp-attention`) is also supported. Note that this mode has not been fully validated in PD disaggregation scenarios. +- **Short-sequence MHA prefill (adaptive)**: For short prefill sequences (default threshold: **2048 tokens**), the NSA backend uses standard MHA automatically (no extra flags). On H200 (SM90) this path uses the FlashAttention variable-length kernel; on B200 (SM100) it uses TRT-LLM ragged MHA. MHA uses `MHA_ONE_SHOT` for best performance. `MHA_ONE_SHOT` computes multi-head attention over all tokens (both cached prefix and newly extended tokens) in a single kernel invocation, avoiding the overhead of chunked KV cache processing. This achieves optimal throughput for short sequences where total sequence length fits within the chunk capacity limit. +- **Choices of Attention Kernels**: The attention backend is automatically set to `nsa` attention backend for DeepSeek V3.2 model. In this backend, different kernels for sparse prefilling/decoding are implemented, which can be specified by `--nsa-prefill-backend` and `--nsa-decode-backend` server arguments. The choices of nsa prefill/decode attention kernels include: + - `flashmla_sparse`: `flash_mla_sparse_fwd` kernel from `flash_mla` library. Can run on both Hopper and Blackwell GPUs. It requires bf16 q, kv inputs. + - `flashmla_kv`: `flash_mla_with_kvcache` kernel from `flash_mla` library. Can run on both Hopper and Blackwell GPUs. It requires bf16 q, fp8 k_cache inputs. + - `fa3`: `flash_attn_with_kvcache` kernel from `flash_attn` library. Can only run on Hopper GPUs. It requires bf16 q, kv inputs. + - `tilelang`: `tilelang` implementation that can run on GPU, HPU and NPU. + - `alter`: Alter kernel on AMD HPUs. Can only be used as decode kernel. +- On the basis of performance benchmarks, the default configuration on H200 and B200 are set as follows : + - H200: `flashmla_sparse` prefill attention (short-seq prefill uses MHA via FlashAttention varlen), `fa3` decode attention, `bf16` kv cache dtype. + - B200: `flashmla_auto` prefill attention (short-seq prefill uses MHA via TRT-LLM ragged), `flashmla_kv` decode attention, `fp8_e4m3` kv cache dtype. `flashmla_auto` enables automatic selection of either `flashmla_sparse` or `flashmla_kv` kernel for prefill based on KV cache dtype, hardware, and heuristics. When FP8 KV cache is enabled and `total_kv_tokens < total_q_tokens * 512`, it uses the `flashmla_sparse` kernel; otherwise, it falls back to the `flashmla_kv` kernel. The heuristics may need to be tuned if the performance of either the `flashmla_sparse` or `flashmla_kv` kernel changes significantly. + +## Multi-token Prediction +SGLang implements Multi-Token Prediction (MTP) for DeepSeek V3.2 based on [EAGLE speculative decoding](https://docs.sglang.io/advanced_features/speculative_decoding.html#EAGLE-Decoding). With this optimization, the decoding speed can be improved significantly on small batch sizes. Please look at [this PR](https://github.com/sgl-project/sglang/pull/11652) for more information. + +Example usage: +```bash +python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --dp 8 --enable-dp-attention --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 +``` +- The best configuration for `--speculative-num-steps`, `--speculative-eagle-topk` and `--speculative-num-draft-tokens` can be searched with [bench_speculative.py](https://github.com/sgl-project/sglang/blob/main/scripts/playground/bench_speculative.py) script for given batch size. The minimum configuration is `--speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2`, which can achieve speedup for larger batch sizes. +- The default value of `--max-running-requests` is set to `48` for MTP. For larger batch sizes, this value should be increased beyond the default value. + + +## Function Calling and Reasoning Parser +The usage of function calling and reasoning parser is the same as DeepSeek V3.1. Please refer to [Reasoning Parser](https://docs.sglang.io/advanced_features/separate_reasoning.html) and [Tool Parser](https://docs.sglang.io/advanced_features/tool_parser.html) documents. + +## PD Disaggregation + +Prefill Command: +```bash +python -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-V3.2-Exp \ + --disaggregation-mode prefill \ + --host $LOCAL_IP \ + --port $PORT \ + --tp 8 \ + --dp 8 \ + --enable-dp-attention \ + --dist-init-addr ${HOST}:${DIST_PORT} \ + --trust-remote-code \ + --disaggregation-bootstrap-port 8998 \ + --mem-fraction-static 0.9 \ +``` + +Decode command: +```bash +python -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-V3.2-Exp \ + --disaggregation-mode decode \ + --host $LOCAL_IP \ + --port $PORT \ + --tp 8 \ + --dp 8 \ + --enable-dp-attention \ + --dist-init-addr ${HOST}:${DIST_PORT} \ + --trust-remote-code \ + --mem-fraction-static 0.9 \ +``` + +Router command: +```bash +python -m sglang_router.launch_router --pd-disaggregation \ + --prefill $PREFILL_ADDR 8998 \ + --decode $DECODE_ADDR \ + --host 127.0.0.1 \ + --port 8000 \ +``` + +If you need more advanced deployment methods or production-ready deployment methods, such as RBG or LWS-based deployment, please refer to [references/multi_node_deployment/rbg_pd/deepseekv32_pd.md](../references/multi_node_deployment/rbg_pd/deepseekv32_pd.md). Additionally, you can also find startup commands for DeepEP-based EP parallelism in the aforementioned documentation. + + +## Benchmarking Results + +### Accuracy Test with `gsm8k` +A simple accuracy benchmark can be tested with `gsm8k` dataset: +```bash +python3 benchmark/gsm8k/bench_sglang.py --num-shots 8 --num-questions 1319 --parallel 1319 +``` + +The result is 0.956, which matches our expectation: +```bash +Accuracy: 0.956 +Invalid: 0.000 +Latency: 25.109 s +Output throughput: 5226.235 token/s +``` + +To test long-context accuracy, run gsm8k with `--num-shots 20`. The results are very close to the 8 shots results: +``` +Accuracy: 0.956 +Invalid: 0.000 +Latency: 29.545 s +Output throughput: 4418.617 token/s +``` + +### Accuracy Test with `gpqa-diamond` + +Accuracy benchmark on long context can be tested on GPQA-diamond dataset with long output tokens and thinking enabled: +```bash +python3 -m sglang.test.run_eval --port 30000 --eval-name gpqa --num-examples 198 --max-tokens 120000 --repeat 8 --thinking-mode deepseek-v3 +``` + +The mean accuracy over 8 runs shows 0.797, which matches the number 79.9 in official tech report. +```bash +Repeat: 8, mean: 0.797 +Scores: ['0.808', '0.798', '0.808', '0.798', '0.783', '0.788', '0.803', '0.793'] +``` + +### Accuracy Test with `aime 2025` + +Prepare the environment by installing NeMo-Skills in the docker or your own virtual environment: + +``` +pip install git+https://github.com/NVIDIA/NeMo-Skills.git --ignore-installed blinker +``` + +Modify the [`jinja chat_template`](https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp/blob/main/tokenizer_config.json#L34) by replacing + +``` +{% set thinking = false %} +``` +with +``` +{% set thinking = true %} +``` +and save it to `chat_template_thinking.jinja`. + +Launch the SGLang server with the modified chat-template file: +``` +python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --dp 8 --enable-dp-attention --chat-template chat_template_thinking.jinja +``` + +Run the following script to evaluate AIME 2025: +``` +#! /bin/bash +export NEMO_SKILLS_DISABLE_UNCOMMITTED_CHANGES_CHECK=1 + +ns prepare_data aime25 + +PORT=30000 +BACKEND=sglang +MODEL="deepseek-ai/DeepSeek-V3.2-Exp" +MODEL_NAME="dsv32-fp8" + +echo "Starting AIME25 evaluation with model $MODEL on port $PORT using backend $BACKEND..." +ns eval \ + --benchmarks=aime25:4 \ + --server_type=$BACKEND \ + --model=$MODEL \ + --server_address=http://localhost:${PORT}/v1 \ + --output_dir=nemo_skills_aime25_${MODEL_NAME}_output_${BACKEND}_$(date +%Y%m%d_%H%M%S) \ + ++max_concurrent_requests=512 \ + ++server.api_key=dummy \ + ++inference.tokens_to_generate=64000 +``` + +Test results: + + +| evaluation_mode | num_entries | avg_tokens | gen_seconds | symbolic_correct | no_answer | +|--------------------|-------------|------------|-------------|-----------------------|-----------| +| pass@1[avg-of-4] | 30 | 14410 | 1758 | 85.83% ± 4.19% | 0.00% | +| majority@4 | 30 | 14410 | 1758 | 90.00% | 0.00% | +| pass@4 | 30 | 14410 | 1758 | 93.33% | 0.00% | + +Note that the result of problem#3 with id `aime25-2` is marked as false by nemo-skills but is actually correct because nemo-skills fails to match predicted_answer `016` with expected_answer `16`. If we add 1/30 = 3.33% to the results, the pass@1[avg-of-4] result matches with reference which is 89.3. + + +## DSA long sequence context parallel optimization(experimental) + +Accuracy benchmark on long context can be tested on GPQA-diamond dataset with long output tokens and thinking enabled: + +Example usage: +```bash +# Launch with EP + DP +python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --ep 8 --dp 2 --enable-dp-attention --enable-nsa-prefill-context-parallel --max-running-requests 32 +``` +### Context-parallel Tips +`CP_size` reuses `atten_tp_size`, which is equal to `TP_size` / `DP_size`. +Some features are still not supported at present. +- **Multi-batch prefill**: Currently, only single-request processing is supported during the prefill process. +- **disaggregation**: P/D disaggregation. +- **Cross-machine support**: - Currently only tested on a single machine (TP=8,EP=8). +- **Other Args**: Currently only supports moe_dense_tp_size=1, kv_cache_dtype = "bf16", moe_a2a_backend = "deepep", +- **DP_size**: `CP_size` reuses `atten_tp_size`, which is equal to `TP_size` / `DP_size`. For the cp function to work correctly, `TP_size` must be divisible by `DP_size`, and TP_size / DP_size > 1 (to ensure CP_size > 1). +- **Detailed design reference**: https://github.com/sgl-project/sglang/pull/12065 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/gpt_oss.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/gpt_oss.md new file mode 100644 index 00000000..d1af32f5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/gpt_oss.md @@ -0,0 +1,129 @@ +# GPT OSS Usage + +Please refer to [https://github.com/sgl-project/sglang/issues/8833](https://github.com/sgl-project/sglang/issues/8833). + +## Responses API & Built-in Tools + +### Responses API + +GPT‑OSS is compatible with the OpenAI Responses API. Use `client.responses.create(...)` with `model`, `instructions`, `input`, and optional `tools` to enable built‑in tool use. You can set reasoning level via `instructions`, e.g., "Reasoning: high" (also supports "medium" and "low") — levels: low (fast), medium (balanced), high (deep). + +### Built-in Tools + +GPT‑OSS can call built‑in tools for web search and Python execution. You can use the demo tool server or connect to external MCP tool servers. + +#### Python Tool + +- Executes short Python snippets for calculations, parsing, and quick scripts. +- By default runs in a Docker-based sandbox. To run on the host, set `PYTHON_EXECUTION_BACKEND=UV` (this executes model-generated code locally; use with care). +- Ensure Docker is available if you are not using the UV backend. It is recommended to run `docker pull python:3.11` in advance. + +#### Web Search Tool + +- Uses the Exa backend for web search. +- Requires an Exa API key; set `EXA_API_KEY` in your environment. Create a key at `https://exa.ai`. + +### Tool & Reasoning Parser + +- We support OpenAI Reasoning and Tool Call parser, as well as our SGLang native api for tool call and reasoning. Refer to [reasoning parser](../advanced_features/separate_reasoning.ipynb) and [tool call parser](../advanced_features/function_calling.ipynb) for more details. + + +## Notes + +- Use **Python 3.12** for the demo tools. And install the required `gpt-oss` packages. +- The default demo integrates the web search tool (Exa backend) and a demo Python interpreter via Docker. +- For search, set `EXA_API_KEY`. For Python execution, either have Docker available or set `PYTHON_EXECUTION_BACKEND=UV`. + +Examples: +```bash +export EXA_API_KEY=YOUR_EXA_KEY +# Optional: run Python tool locally instead of Docker (use with care) +export PYTHON_EXECUTION_BACKEND=UV +``` + +Launch the server with the demo tool server: + +```bash +python3 -m sglang.launch_server \ + --model-path openai/gpt-oss-120b \ + --tool-server demo \ + --tp 2 +``` + +For production usage, sglang can act as an MCP client for multiple services. An [example tool server](https://github.com/openai/gpt-oss/tree/main/gpt-oss-mcp-server) is provided. Start the servers and point sglang to them: +```bash +mcp run -t sse browser_server.py:mcp +mcp run -t sse python_server.py:mcp + +python -m sglang.launch_server ... --tool-server ip-1:port-1,ip-2:port-2 +``` +The URLs should be MCP SSE servers that expose server information and well-documented tools. These tools are added to the system prompt so the model can use them. + +### Quick Demo + +```python +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:30000/v1", + api_key="sk-123456" +) + +tools = [ + {"type": "code_interpreter"}, + {"type": "web_search_preview"}, +] + +# Reasoning level example +response = client.responses.create( + model="openai/gpt-oss-120b", + instructions="You are a helpful assistant." + reasoning_effort="high" # Supports high, medium, or low + input="In one sentence, explain the transformer architecture.", +) +print("====== reasoning: high ======") +print(response.output_text) + +# Test python tool +response = client.responses.create( + model="openai/gpt-oss-120b", + instructions="You are a helfpul assistant, you could use python tool to execute code.", + input="Use python tool to calculate the sum of 29138749187 and 29138749187", # 58,277,498,374 + tools=tools +) +print("====== test python tool ======") +print(response.output_text) + +# Test browser tool +response = client.responses.create( + model="openai/gpt-oss-120b", + instructions="You are a helfpul assistant, you could use browser to search the web", + input="Search the web for the latest news about Nvidia stock price", + tools=tools +) +print("====== test browser tool ======") +print(response.output_text) +``` + +Example output: +``` +====== test python tool ====== +The sum of 29,138,749,187 and 29,138,749,187 is **58,277,498,374**. +====== test browser tool ====== +**Recent headlines on Nvidia (NVDA) stock** + +| Date (2025) | Source | Key news points | Stock‑price detail | +|-------------|--------|----------------|--------------------| +| **May 13** | Reuters | The market data page shows Nvidia trading “higher” at **$116.61** with no change from the previous close. | **$116.61** – latest trade (delayed ≈ 15 min)【14†L34-L38】 | +| **Aug 18** | CNBC | Morgan Stanley kept an **overweight** rating and lifted its price target to **$206** (up from $200), implying a 14 % upside from the Friday close. The firm notes Nvidia shares have already **jumped 34 % this year**. | No exact price quoted, but the article signals strong upside expectations【9†L27-L31】 | +| **Aug 20** | The Motley Fool | Nvidia is set to release its Q2 earnings on Aug 27. The article lists the **current price of $175.36**, down 0.16 % on the day (as of 3:58 p.m. ET). | **$175.36** – current price on Aug 20【10†L12-L15】【10†L53-L57】 | + +**What the news tells us** + +* Nvidia’s share price has risen sharply this year – up roughly a third according to Morgan Stanley – and analysts are still raising targets (now $206). +* The most recent market quote (Reuters, May 13) was **$116.61**, but the stock has surged since then, reaching **$175.36** by mid‑August. +* Upcoming earnings on **Aug 27** are a focal point; both the Motley Fool and Morgan Stanley expect the results could keep the rally going. + +**Bottom line:** Nvidia’s stock is on a strong upward trajectory in 2025, with price targets climbing toward $200‑$210 and the market price already near $175 as of late August. + +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/llama4.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/llama4.md new file mode 100644 index 00000000..1a2338a3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/llama4.md @@ -0,0 +1,92 @@ +# Llama4 Usage + +[Llama 4](https://github.com/meta-llama/llama-models/blob/main/models/llama4/MODEL_CARD.md) is Meta's latest generation of open-source LLM model with industry-leading performance. + +SGLang has supported Llama 4 Scout (109B) and Llama 4 Maverick (400B) since [v0.4.5](https://github.com/sgl-project/sglang/releases/tag/v0.4.5). + +Ongoing optimizations are tracked in the [Roadmap](https://github.com/sgl-project/sglang/issues/5118). + +## Launch Llama 4 with SGLang + +To serve Llama 4 models on 8xH100/H200 GPUs: + +```bash +python3 -m sglang.launch_server \ + --model-path meta-llama/Llama-4-Scout-17B-16E-Instruct \ + --tp 8 \ + --context-length 1000000 +``` + +### Configuration Tips + +- **OOM Mitigation**: Adjust `--context-length` to avoid a GPU out-of-memory issue. For the Scout model, we recommend setting this value up to 1M on 8\*H100 and up to 2.5M on 8\*H200. For the Maverick model, we don't need to set context length on 8\*H200. When hybrid kv cache is enabled, `--context-length` can be set up to 5M on 8\*H100 and up to 10M on 8\*H200 for the Scout model. + +- **Attention Backend Auto-Selection**: SGLang automatically selects the optimal attention backend for Llama 4 based on your hardware. You typically don't need to specify `--attention-backend` manually: + - **Blackwell GPUs (B200/GB200)**: `trtllm_mha` + - **Hopper GPUs (H100/H200)**: `fa3` + - **AMD GPUs**: `aiter` + - **Intel XPU**: `intel_xpu` + - **Other platforms**: `triton` (fallback) + + To override the auto-selection, explicitly specify `--attention-backend` with one of the supported backends: `fa3`, `aiter`, `triton`, `trtllm_mha`, or `intel_xpu`. + +- **Chat Template**: Add `--chat-template llama-4` for chat completion tasks. +- **Enable Multi-Modal**: Add `--enable-multimodal` for multi-modal capabilities. +- **Enable Hybrid-KVCache**: Add `--hybrid-kvcache-ratio` for hybrid kv cache. Details can be seen in [this PR](https://github.com/sgl-project/sglang/pull/6563) + + +### EAGLE Speculative Decoding +**Description**: SGLang has supported Llama 4 Maverick (400B) with [EAGLE speculative decoding](https://docs.sglang.io/advanced_features/speculative_decoding.html#EAGLE-Decoding). + +**Usage**: +Add arguments `--speculative-draft-model-path`, `--speculative-algorithm`, `--speculative-num-steps`, `--speculative-eagle-topk` and `--speculative-num-draft-tokens` to enable this feature. For example: +``` +python3 -m sglang.launch_server \ + --model-path meta-llama/Llama-4-Maverick-17B-128E-Instruct \ + --speculative-algorithm EAGLE3 \ + --speculative-draft-model-path nvidia/Llama-4-Maverick-17B-128E-Eagle3 \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 \ + --trust-remote-code \ + --tp 8 \ + --context-length 1000000 +``` + +- **Note** The Llama 4 draft model *nvidia/Llama-4-Maverick-17B-128E-Eagle3* can only recognize conversations in chat mode. + +## Benchmarking Results + +### Accuracy Test with `lm_eval` + +The accuracy on SGLang for both Llama4 Scout and Llama4 Maverick can match the [official benchmark numbers](https://ai.meta.com/blog/llama-4-multimodal-intelligence/). + +Benchmark results on MMLU Pro dataset with 8*H100: +| | Llama-4-Scout-17B-16E-Instruct | Llama-4-Maverick-17B-128E-Instruct | +|--------------------|--------------------------------|-------------------------------------| +| Official Benchmark | 74.3 | 80.5 | +| SGLang | 75.2 | 80.7 | + +Commands: + +```bash +# Llama-4-Scout-17B-16E-Instruct model +python -m sglang.launch_server \ + --model-path meta-llama/Llama-4-Scout-17B-16E-Instruct \ + --port 30000 \ + --tp 8 \ + --mem-fraction-static 0.8 \ + --context-length 65536 +lm_eval --model local-chat-completions --model_args model=meta-llama/Llama-4-Scout-17B-16E-Instruct,base_url=http://localhost:30000/v1/chat/completions,num_concurrent=128,timeout=999999,max_gen_toks=2048 --tasks mmlu_pro --batch_size 128 --apply_chat_template --num_fewshot 0 + +# Llama-4-Maverick-17B-128E-Instruct +python -m sglang.launch_server \ + --model-path meta-llama/Llama-4-Maverick-17B-128E-Instruct \ + --port 30000 \ + --tp 8 \ + --mem-fraction-static 0.8 \ + --context-length 65536 +lm_eval --model local-chat-completions --model_args model=meta-llama/Llama-4-Maverick-17B-128E-Instruct,base_url=http://localhost:30000/v1/chat/completions,num_concurrent=128,timeout=999999,max_gen_toks=2048 --tasks mmlu_pro --batch_size 128 --apply_chat_template --num_fewshot 0 +``` + +Details can be seen in [this PR](https://github.com/sgl-project/sglang/pull/5092). diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/native_api.ipynb b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/native_api.ipynb new file mode 100644 index 00000000..028e646d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/native_api.ipynb @@ -0,0 +1,601 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# SGLang Native APIs\n", + "\n", + "Apart from the OpenAI compatible APIs, the SGLang Runtime also provides its native server APIs. We introduce the following APIs:\n", + "\n", + "- `/generate` (text generation model)\n", + "- `/get_model_info`\n", + "- `/get_server_info`\n", + "- `/health`\n", + "- `/health_generate`\n", + "- `/flush_cache`\n", + "- `/update_weights`\n", + "- `/encode`(embedding model)\n", + "- `/v1/rerank`(cross encoder rerank model)\n", + "- `/classify`(reward model)\n", + "- `/start_expert_distribution_record`\n", + "- `/stop_expert_distribution_record`\n", + "- `/dump_expert_distribution_record`\n", + "- `/tokenize`\n", + "- `/detokenize`\n", + "- A full list of these APIs can be found at [http_server.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/entrypoints/http_server.py)\n", + "\n", + "We mainly use `requests` to test these APIs in the following examples. You can also use `curl`.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Launch A Server" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sglang.test.doc_patch import launch_server_cmd\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0 --log-level warning\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Generate (text generation model)\n", + "Generate completions. This is similar to the `/v1/completions` in OpenAI API. Detailed parameters can be found in the [sampling parameters](sampling_params.md)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "\n", + "url = f\"http://localhost:{port}/generate\"\n", + "data = {\"text\": \"What is the capital of France?\"}\n", + "\n", + "response = requests.post(url, json=data)\n", + "print_highlight(response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get Model Info\n", + "\n", + "Get the information of the model.\n", + "\n", + "- `model_path`: The path/name of the model.\n", + "- `is_generation`: Whether the model is used as generation model or embedding model.\n", + "- `tokenizer_path`: The path/name of the tokenizer.\n", + "- `preferred_sampling_params`: The default sampling params specified via `--preferred-sampling-params`. `None` is returned in this example as we did not explicitly configure it in server args.\n", + "- `weight_version`: This field contains the version of the model weights. This is often used to track changes or updates to the model’s trained parameters.\n", + "- `has_image_understanding`: Whether the model has image-understanding capability.\n", + "- `has_audio_understanding`: Whether the model has audio-understanding capability." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://localhost:{port}/get_model_info\"\n", + "\n", + "response = requests.get(url)\n", + "response_json = response.json()\n", + "print_highlight(response_json)\n", + "assert response_json[\"model_path\"] == \"qwen/qwen2.5-0.5b-instruct\"\n", + "assert response_json[\"is_generation\"] is True\n", + "assert response_json[\"tokenizer_path\"] == \"qwen/qwen2.5-0.5b-instruct\"\n", + "assert response_json[\"preferred_sampling_params\"] is None\n", + "assert response_json.keys() == {\n", + " \"model_path\",\n", + " \"is_generation\",\n", + " \"tokenizer_path\",\n", + " \"preferred_sampling_params\",\n", + " \"weight_version\",\n", + " \"has_image_understanding\",\n", + " \"has_audio_understanding\",\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get Server Info\n", + "Gets the server information including CLI arguments, token limits, and memory pool sizes.\n", + "- Note: `get_server_info` merges the following deprecated endpoints:\n", + " - `get_server_args`\n", + " - `get_memory_pool_size` \n", + " - `get_max_total_num_tokens`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://localhost:{port}/get_server_info\"\n", + "\n", + "response = requests.get(url)\n", + "print_highlight(response.text)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Health Check\n", + "- `/health`: Check the health of the server.\n", + "- `/health_generate`: Check the health of the server by generating one token." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://localhost:{port}/health_generate\"\n", + "\n", + "response = requests.get(url)\n", + "print_highlight(response.text)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://localhost:{port}/health\"\n", + "\n", + "response = requests.get(url)\n", + "print_highlight(response.text)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Flush Cache\n", + "\n", + "Flush the radix cache. It will be automatically triggered when the model weights are updated by the `/update_weights` API." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://localhost:{port}/flush_cache\"\n", + "\n", + "response = requests.post(url)\n", + "print_highlight(response.text)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Update Weights From Disk\n", + "\n", + "Update model weights from disk without restarting the server. Only applicable for models with the same architecture and parameter size.\n", + "\n", + "SGLang support `update_weights_from_disk` API for continuous evaluation during training (save checkpoint to disk and update weights from disk).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# successful update with same architecture and size\n", + "\n", + "url = f\"http://localhost:{port}/update_weights_from_disk\"\n", + "data = {\"model_path\": \"qwen/qwen2.5-0.5b-instruct\"}\n", + "\n", + "response = requests.post(url, json=data)\n", + "print_highlight(response.text)\n", + "assert response.json()[\"success\"] is True\n", + "assert response.json()[\"message\"] == \"Succeeded to update model weights.\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# failed update with different parameter size or wrong name\n", + "\n", + "url = f\"http://localhost:{port}/update_weights_from_disk\"\n", + "data = {\"model_path\": \"qwen/qwen2.5-0.5b-instruct-wrong\"}\n", + "\n", + "response = requests.post(url, json=data)\n", + "response_json = response.json()\n", + "print_highlight(response_json)\n", + "assert response_json[\"success\"] is False\n", + "assert response_json[\"message\"] == (\n", + " \"Failed to get weights iterator: \"\n", + " \"qwen/qwen2.5-0.5b-instruct-wrong\"\n", + " \" (repository not found).\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Encode (embedding model)\n", + "\n", + "Encode text into embeddings. Note that this API is only available for [embedding models](openai_api_embeddings.ipynb) and will raise an error for generation models.\n", + "Therefore, we launch a new server to server an embedding model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "embedding_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model-path Alibaba-NLP/gte-Qwen2-1.5B-instruct \\\n", + " --host 0.0.0.0 --is-embedding --log-level warning\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# successful encode for embedding model\n", + "\n", + "url = f\"http://localhost:{port}/encode\"\n", + "data = {\"model\": \"Alibaba-NLP/gte-Qwen2-1.5B-instruct\", \"text\": \"Once upon a time\"}\n", + "\n", + "response = requests.post(url, json=data)\n", + "response_json = response.json()\n", + "print_highlight(f\"Text embedding (first 10): {response_json['embedding'][:10]}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(embedding_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## v1/rerank (cross encoder rerank model)\n", + "Rerank a list of documents given a query using a cross-encoder model. Note that this API is only available for cross encoder model like [BAAI/bge-reranker-v2-m3](https://huggingface.co/BAAI/bge-reranker-v2-m3) with `attention-backend` `triton` and `torch_native`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "reranker_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model-path BAAI/bge-reranker-v2-m3 \\\n", + " --host 0.0.0.0 --disable-radix-cache --chunked-prefill-size -1 --attention-backend triton --is-embedding --log-level warning\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# compute rerank scores for query and documents\n", + "\n", + "url = f\"http://localhost:{port}/v1/rerank\"\n", + "data = {\n", + " \"model\": \"BAAI/bge-reranker-v2-m3\",\n", + " \"query\": \"what is panda?\",\n", + " \"documents\": [\n", + " \"hi\",\n", + " \"The giant panda (Ailuropoda melanoleuca), sometimes called a panda bear or simply panda, is a bear species endemic to China.\",\n", + " ],\n", + "}\n", + "\n", + "response = requests.post(url, json=data)\n", + "response_json = response.json()\n", + "for item in response_json:\n", + " print_highlight(f\"Score: {item['score']:.2f} - Document: '{item['document']}'\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(reranker_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Classify (reward model)\n", + "\n", + "SGLang Runtime also supports reward models. Here we use a reward model to classify the quality of pairwise generations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Note that SGLang now treats embedding models and reward models as the same type of models.\n", + "# This will be updated in the future.\n", + "\n", + "reward_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model-path Skywork/Skywork-Reward-Llama-3.1-8B-v0.2 --host 0.0.0.0 --is-embedding --log-level warning\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoTokenizer\n", + "\n", + "PROMPT = (\n", + " \"What is the range of the numeric output of a sigmoid node in a neural network?\"\n", + ")\n", + "\n", + "RESPONSE1 = \"The output of a sigmoid node is bounded between -1 and 1.\"\n", + "RESPONSE2 = \"The output of a sigmoid node is bounded between 0 and 1.\"\n", + "\n", + "CONVS = [\n", + " [{\"role\": \"user\", \"content\": PROMPT}, {\"role\": \"assistant\", \"content\": RESPONSE1}],\n", + " [{\"role\": \"user\", \"content\": PROMPT}, {\"role\": \"assistant\", \"content\": RESPONSE2}],\n", + "]\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(\"Skywork/Skywork-Reward-Llama-3.1-8B-v0.2\")\n", + "prompts = tokenizer.apply_chat_template(CONVS, tokenize=False, return_dict=False)\n", + "\n", + "url = f\"http://localhost:{port}/classify\"\n", + "data = {\"model\": \"Skywork/Skywork-Reward-Llama-3.1-8B-v0.2\", \"text\": prompts}\n", + "\n", + "responses = requests.post(url, json=data).json()\n", + "for response in responses:\n", + " print_highlight(f\"reward: {response['embedding'][0]}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(reward_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Capture expert selection distribution in MoE models\n", + "\n", + "SGLang Runtime supports recording the number of times an expert is selected in a MoE model run for each expert in the model. This is useful when analyzing the throughput of the model and plan for optimization.\n", + "\n", + "*Note: We only print out the first 10 lines of the csv below for better readability. Please adjust accordingly if you want to analyze the results more deeply.*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "expert_record_server_process, port = launch_server_cmd(\n", + " \"python3 -m sglang.launch_server --model-path Qwen/Qwen1.5-MoE-A2.7B --host 0.0.0.0 --expert-distribution-recorder-mode stat --log-level warning\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = requests.post(f\"http://localhost:{port}/start_expert_distribution_record\")\n", + "print_highlight(response)\n", + "\n", + "url = f\"http://localhost:{port}/generate\"\n", + "data = {\"text\": \"What is the capital of France?\"}\n", + "\n", + "response = requests.post(url, json=data)\n", + "print_highlight(response.json())\n", + "\n", + "response = requests.post(f\"http://localhost:{port}/stop_expert_distribution_record\")\n", + "print_highlight(response)\n", + "\n", + "response = requests.post(f\"http://localhost:{port}/dump_expert_distribution_record\")\n", + "print_highlight(response)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(expert_record_server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tokenize/Detokenize Example (Round Trip)\n", + "\n", + "This example demonstrates how to use the /tokenize and /detokenize endpoints together. We first tokenize a string, then detokenize the resulting IDs to reconstruct the original text. This workflow is useful when you need to handle tokenization externally but still leverage the server for detokenization." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tokenizer_free_server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "from sglang.utils import print_highlight\n", + "\n", + "base_url = f\"http://localhost:{port}\"\n", + "tokenize_url = f\"{base_url}/tokenize\"\n", + "detokenize_url = f\"{base_url}/detokenize\"\n", + "\n", + "model_name = \"qwen/qwen2.5-0.5b-instruct\"\n", + "input_text = \"SGLang provides efficient tokenization endpoints.\"\n", + "print_highlight(f\"Original Input Text:\\n'{input_text}'\")\n", + "\n", + "# --- tokenize the input text ---\n", + "tokenize_payload = {\n", + " \"model\": model_name,\n", + " \"prompt\": input_text,\n", + " \"add_special_tokens\": False,\n", + "}\n", + "try:\n", + " tokenize_response = requests.post(tokenize_url, json=tokenize_payload)\n", + " tokenize_response.raise_for_status()\n", + " tokenization_result = tokenize_response.json()\n", + " token_ids = tokenization_result.get(\"tokens\")\n", + "\n", + " if not token_ids:\n", + " raise ValueError(\"Tokenization returned empty tokens.\")\n", + "\n", + " print_highlight(f\"\\nTokenized Output (IDs):\\n{token_ids}\")\n", + " print_highlight(f\"Token Count: {tokenization_result.get('count')}\")\n", + " print_highlight(f\"Max Model Length: {tokenization_result.get('max_model_len')}\")\n", + "\n", + " # --- detokenize the obtained token IDs ---\n", + " detokenize_payload = {\n", + " \"model\": model_name,\n", + " \"tokens\": token_ids,\n", + " \"skip_special_tokens\": True,\n", + " }\n", + "\n", + " detokenize_response = requests.post(detokenize_url, json=detokenize_payload)\n", + " detokenize_response.raise_for_status()\n", + " detokenization_result = detokenize_response.json()\n", + " reconstructed_text = detokenization_result.get(\"text\")\n", + "\n", + " print_highlight(f\"\\nDetokenized Output (Text):\\n'{reconstructed_text}'\")\n", + "\n", + " if input_text == reconstructed_text:\n", + " print_highlight(\n", + " \"\\nRound Trip Successful: Original and reconstructed text match.\"\n", + " )\n", + " else:\n", + " print_highlight(\n", + " \"\\nRound Trip Mismatch: Original and reconstructed text differ.\"\n", + " )\n", + "\n", + "except requests.exceptions.RequestException as e:\n", + " print_highlight(f\"\\nHTTP Request Error: {e}\")\n", + "except Exception as e:\n", + " print_highlight(f\"\\nAn error occurred: {e}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(tokenizer_free_server_process)" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/offline_engine_api.ipynb b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/offline_engine_api.ipynb new file mode 100644 index 00000000..9c03e90a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/offline_engine_api.ipynb @@ -0,0 +1,235 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Offline Engine API\n", + "\n", + "SGLang provides a direct inference engine without the need for an HTTP server, especially for use cases where additional HTTP server adds unnecessary complexity or overhead. Here are two general use cases:\n", + "\n", + "- Offline Batch Inference\n", + "- Custom Server on Top of the Engine\n", + "\n", + "This document focuses on the offline batch inference, demonstrating four different inference modes:\n", + "\n", + "- Non-streaming synchronous generation\n", + "- Streaming synchronous generation\n", + "- Non-streaming asynchronous generation\n", + "- Streaming asynchronous generation\n", + "\n", + "Additionally, you can easily build a custom server on top of the SGLang offline engine. A detailed example working in a python script can be found in [custom_server](https://github.com/sgl-project/sglang/blob/main/examples/runtime/engine/custom_server.py).\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Nest Asyncio\n", + "Note that if you want to use **Offline Engine** in ipython or some other nested loop code, you need to add the following code:\n", + "```python\n", + "import nest_asyncio\n", + "\n", + "nest_asyncio.apply()\n", + "\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Advanced Usage\n", + "\n", + "The engine supports [vlm inference](https://github.com/sgl-project/sglang/blob/main/examples/runtime/engine/offline_batch_inference_vlm.py) as well as [extracting hidden states](https://github.com/sgl-project/sglang/blob/main/examples/runtime/hidden_states). \n", + "\n", + "Please see [the examples](https://github.com/sgl-project/sglang/tree/main/examples/runtime/engine) for further use cases." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Offline Batch Inference\n", + "\n", + "SGLang offline engine supports batch inference with efficient scheduling." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# launch the offline engine\n", + "import asyncio\n", + "\n", + "import sglang as sgl\n", + "import sglang.test.doc_patch\n", + "from sglang.utils import async_stream_and_merge, stream_and_merge\n", + "\n", + "llm = sgl.Engine(model_path=\"qwen/qwen2.5-0.5b-instruct\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Non-streaming Synchronous Generation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "prompts = [\n", + " \"Hello, my name is\",\n", + " \"The president of the United States is\",\n", + " \"The capital of France is\",\n", + " \"The future of AI is\",\n", + "]\n", + "\n", + "sampling_params = {\"temperature\": 0.8, \"top_p\": 0.95}\n", + "\n", + "outputs = llm.generate(prompts, sampling_params)\n", + "for prompt, output in zip(prompts, outputs):\n", + " print(\"===============================\")\n", + " print(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Streaming Synchronous Generation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "prompts = [\n", + " \"Write a short, neutral self-introduction for a fictional character. Hello, my name is\",\n", + " \"Provide a concise factual statement about France’s capital city. The capital of France is\",\n", + " \"Explain possible future trends in artificial intelligence. The future of AI is\",\n", + "]\n", + "\n", + "sampling_params = {\n", + " \"temperature\": 0.2,\n", + " \"top_p\": 0.9,\n", + "}\n", + "\n", + "print(\"\\n=== Testing synchronous streaming generation with overlap removal ===\\n\")\n", + "\n", + "for prompt in prompts:\n", + " print(f\"Prompt: {prompt}\")\n", + " merged_output = stream_and_merge(llm, prompt, sampling_params)\n", + " print(\"Generated text:\", merged_output)\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Non-streaming Asynchronous Generation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "prompts = [\n", + " \"Write a short, neutral self-introduction for a fictional character. Hello, my name is\",\n", + " \"Provide a concise factual statement about France’s capital city. The capital of France is\",\n", + " \"Explain possible future trends in artificial intelligence. The future of AI is\",\n", + "]\n", + "\n", + "sampling_params = {\"temperature\": 0.8, \"top_p\": 0.95}\n", + "\n", + "print(\"\\n=== Testing asynchronous batch generation ===\")\n", + "\n", + "\n", + "async def main():\n", + " outputs = await llm.async_generate(prompts, sampling_params)\n", + "\n", + " for prompt, output in zip(prompts, outputs):\n", + " print(f\"\\nPrompt: {prompt}\")\n", + " print(f\"Generated text: {output['text']}\")\n", + "\n", + "\n", + "asyncio.run(main())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Streaming Asynchronous Generation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "prompts = [\n", + " \"Write a short, neutral self-introduction for a fictional character. Hello, my name is\",\n", + " \"Provide a concise factual statement about France’s capital city. The capital of France is\",\n", + " \"Explain possible future trends in artificial intelligence. The future of AI is\",\n", + "]\n", + "\n", + "sampling_params = {\"temperature\": 0.8, \"top_p\": 0.95}\n", + "\n", + "print(\"\\n=== Testing asynchronous streaming generation (no repeats) ===\")\n", + "\n", + "\n", + "async def main():\n", + " for prompt in prompts:\n", + " print(f\"\\nPrompt: {prompt}\")\n", + " print(\"Generated text: \", end=\"\", flush=True)\n", + "\n", + " # Replace direct calls to async_generate with our custom overlap-aware version\n", + " async for cleaned_chunk in async_stream_and_merge(llm, prompt, sampling_params):\n", + " print(cleaned_chunk, end=\"\", flush=True)\n", + "\n", + " print() # New line after each prompt\n", + "\n", + "\n", + "asyncio.run(main())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "llm.shutdown()" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/openai_api.rst b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/openai_api.rst new file mode 100644 index 00000000..370abe99 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/openai_api.rst @@ -0,0 +1,9 @@ +OpenAI-Compatible APIs +====================== + +.. toctree:: + :maxdepth: 1 + + openai_api_completions.ipynb + openai_api_vision.ipynb + openai_api_embeddings.ipynb diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/openai_api_completions.ipynb b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/openai_api_completions.ipynb new file mode 100644 index 00000000..d498f13e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/openai_api_completions.ipynb @@ -0,0 +1,534 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# OpenAI APIs - Completions\n", + "\n", + "SGLang provides OpenAI-compatible APIs to enable a smooth transition from OpenAI services to self-hosted local models.\n", + "A complete reference for the API is available in the [OpenAI API Reference](https://platform.openai.com/docs/api-reference).\n", + "\n", + "This tutorial covers the following popular APIs:\n", + "\n", + "- `chat/completions`\n", + "- `completions`\n", + "\n", + "Check out other tutorials to learn about [vision APIs](openai_api_vision.ipynb) for vision-language models and [embedding APIs](openai_api_embeddings.ipynb) for embedding models." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Launch A Server\n", + "\n", + "Launch the server in your terminal and wait for it to initialize." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sglang.test.doc_patch import launch_server_cmd\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0 --log-level warning\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")\n", + "print(f\"Server started on http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Chat Completions\n", + "\n", + "### Usage\n", + "\n", + "The server fully implements the OpenAI API.\n", + "It will automatically apply the chat template specified in the Hugging Face tokenizer, if one is available.\n", + "You can also specify a custom chat template with `--chat-template` when launching the server." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "\n", + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"qwen/qwen2.5-0.5b-instruct\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=64,\n", + ")\n", + "\n", + "print_highlight(f\"Response: {response}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Model Thinking/Reasoning Support\n", + "\n", + "Some models support internal reasoning or thinking processes that can be exposed in the API response. SGLang provides unified support for various reasoning models through the `chat_template_kwargs` parameter and compatible reasoning parsers.\n", + "\n", + "#### Supported Models and Configuration\n", + "\n", + "| Model Family | Chat Template Parameter | Reasoning Parser | Notes |\n", + "|--------------|------------------------|------------------|--------|\n", + "| DeepSeek-R1 (R1, R1-0528, R1-Distill) | `enable_thinking` | `--reasoning-parser deepseek-r1` | Standard reasoning models |\n", + "| DeepSeek-V3.1 | `thinking` | `--reasoning-parser deepseek-v3` | Hybrid model (thinking/non-thinking modes) |\n", + "| Qwen3 (standard) | `enable_thinking` | `--reasoning-parser qwen3` | Hybrid model (thinking/non-thinking modes) |\n", + "| Qwen3-Thinking | N/A (always enabled) | `--reasoning-parser qwen3-thinking` | Always generates reasoning |\n", + "| Kimi | N/A (always enabled) | `--reasoning-parser kimi` | Kimi thinking models |\n", + "| Gpt-Oss | N/A (always enabled) | `--reasoning-parser gpt-oss` | Gpt-Oss thinking models |\n", + "\n", + "#### Basic Usage\n", + "\n", + "To enable reasoning output, you need to:\n", + "1. Launch the server with the appropriate reasoning parser\n", + "2. Set the model-specific parameter in `chat_template_kwargs`\n", + "3. Optionally use `separate_reasoning: False` to not get reasoning content separately (default to `True`)\n", + "\n", + "**Note for Qwen3-Thinking models:** These models always generate thinking content and do not support the `enable_thinking` parameter. Use `--reasoning-parser qwen3-thinking` or `--reasoning-parser qwen3` to parse the thinking content.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Example: Qwen3 Models\n", + "\n", + "```python\n", + "# Launch server:\n", + "# python3 -m sglang.launch_server --model Qwen/Qwen3-4B --reasoning-parser qwen3\n", + "\n", + "from openai import OpenAI\n", + "\n", + "client = OpenAI(\n", + " api_key=\"EMPTY\",\n", + " base_url=f\"http://127.0.0.1:30000/v1\",\n", + ")\n", + "\n", + "model = \"Qwen/Qwen3-4B\"\n", + "messages = [{\"role\": \"user\", \"content\": \"How many r's are in 'strawberry'?\"}]\n", + "\n", + "response = client.chat.completions.create(\n", + " model=model,\n", + " messages=messages,\n", + " extra_body={\n", + " \"chat_template_kwargs\": {\"enable_thinking\": True},\n", + " \"separate_reasoning\": True\n", + " }\n", + ")\n", + "\n", + "print(\"Reasoning:\", response.choices[0].message.reasoning_content)\n", + "print(\"-\"*100)\n", + "print(\"Answer:\", response.choices[0].message.content)\n", + "```\n", + "\n", + "**ExampleOutput:**\n", + "```\n", + "Reasoning: Okay, so the user is asking how many 'r's are in the word 'strawberry'. Let me think. First, I need to make sure I have the word spelled correctly. Strawberry... S-T-R-A-W-B-E-R-R-Y. Wait, is that right? Let me break it down.\n", + "\n", + "Starting with 'strawberry', let's write out the letters one by one. S, T, R, A, W, B, E, R, R, Y. Hmm, wait, that's 10 letters. Let me check again. S (1), T (2), R (3), A (4), W (5), B (6), E (7), R (8), R (9), Y (10). So the letters are S-T-R-A-W-B-E-R-R-Y. \n", + "...\n", + "Therefore, the answer should be three R's in 'strawberry'. But I need to make sure I'm not counting any other letters as R. Let me check again. S, T, R, A, W, B, E, R, R, Y. No other R's. So three in total. Yeah, that seems right.\n", + "\n", + "----------------------------------------------------------------------------------------------------\n", + "Answer: The word \"strawberry\" contains **three** letters 'r'. Here's the breakdown:\n", + "\n", + "1. **S-T-R-A-W-B-E-R-R-Y** \n", + " - The **third letter** is 'R'. \n", + " - The **eighth and ninth letters** are also 'R's. \n", + "\n", + "Thus, the total count is **3**. \n", + "\n", + "**Answer:** 3.\n", + "```\n", + "\n", + "**Note:** Setting `\"enable_thinking\": False` (or omitting it) will result in `reasoning_content` being `None`. Qwen3-Thinking models always generate reasoning content and don't support the `enable_thinking` parameter.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Logit Bias Support\n", + "\n", + "SGLang supports the `logit_bias` parameter for both chat completions and completions APIs. This parameter allows you to modify the likelihood of specific tokens being generated by adding bias values to their logits. The bias values can range from -100 to 100, where:\n", + "\n", + "- **Positive values** (0 to 100) increase the likelihood of the token being selected\n", + "- **Negative values** (-100 to 0) decrease the likelihood of the token being selected\n", + "- **-100** effectively prevents the token from being generated\n", + "\n", + "The `logit_bias` parameter accepts a dictionary where keys are token IDs (as strings) and values are the bias amounts (as floats).\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Getting Token IDs\n", + "\n", + "To use `logit_bias` effectively, you need to know the token IDs for the words you want to bias. Here's how to get token IDs:\n", + "\n", + "```python\n", + "# Get tokenizer to find token IDs\n", + "import tiktoken\n", + "\n", + "# For OpenAI models, use the appropriate encoding\n", + "tokenizer = tiktoken.encoding_for_model(\"gpt-3.5-turbo\") # or your model\n", + "\n", + "# Get token IDs for specific words\n", + "word = \"sunny\"\n", + "token_ids = tokenizer.encode(word)\n", + "print(f\"Token IDs for '{word}': {token_ids}\")\n", + "\n", + "# For SGLang models, you can access the tokenizer through the client\n", + "# and get token IDs for bias\n", + "```\n", + "\n", + "**Important:** The `logit_bias` parameter uses token IDs as string keys, not the actual words.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Example: DeepSeek-V3 Models\n", + "\n", + "DeepSeek-V3 models support thinking mode through the `thinking` parameter:\n", + "\n", + "```python\n", + "# Launch server:\n", + "# python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.1 --tp 8 --reasoning-parser deepseek-v3\n", + "\n", + "from openai import OpenAI\n", + "\n", + "client = OpenAI(\n", + " api_key=\"EMPTY\",\n", + " base_url=f\"http://127.0.0.1:30000/v1\",\n", + ")\n", + "\n", + "model = \"deepseek-ai/DeepSeek-V3.1\"\n", + "messages = [{\"role\": \"user\", \"content\": \"How many r's are in 'strawberry'?\"}]\n", + "\n", + "response = client.chat.completions.create(\n", + " model=model,\n", + " messages=messages,\n", + " extra_body={\n", + " \"chat_template_kwargs\": {\"thinking\": True},\n", + " \"separate_reasoning\": True\n", + " }\n", + ")\n", + "\n", + "print(\"Reasoning:\", response.choices[0].message.reasoning_content)\n", + "print(\"-\"*100)\n", + "print(\"Answer:\", response.choices[0].message.content)\n", + "```\n", + "\n", + "**Example Output:**\n", + "```\n", + "Reasoning: First, the question is: \"How many r's are in 'strawberry'?\"\n", + "\n", + "I need to count the number of times the letter 'r' appears in the word \"strawberry\".\n", + "\n", + "Let me write out the word: S-T-R-A-W-B-E-R-R-Y.\n", + "\n", + "Now, I'll go through each letter and count the 'r's.\n", + "...\n", + "So, I have three 'r's in \"strawberry\".\n", + "\n", + "I should double-check. The word is spelled S-T-R-A-W-B-E-R-R-Y. The letters are at positions: 3, 8, and 9 are 'r's. Yes, that's correct.\n", + "\n", + "Therefore, the answer should be 3.\n", + "----------------------------------------------------------------------------------------------------\n", + "Answer: The word \"strawberry\" contains **3** instances of the letter \"r\". Here's a breakdown for clarity:\n", + "\n", + "- The word is spelled: S-T-R-A-W-B-E-R-R-Y\n", + "- The \"r\" appears at the 3rd, 8th, and 9th positions.\n", + "```\n", + "\n", + "**Note:** DeepSeek-V3 models use the `thinking` parameter (not `enable_thinking`) to control reasoning output.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example with logit_bias parameter\n", + "# Note: You need to get the actual token IDs from your tokenizer\n", + "# For demonstration, we'll use some example token IDs\n", + "response = client.chat.completions.create(\n", + " model=\"qwen/qwen2.5-0.5b-instruct\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"Complete this sentence: The weather today is\"}\n", + " ],\n", + " temperature=0.7,\n", + " max_tokens=20,\n", + " logit_bias={\n", + " \"12345\": 50, # Increase likelihood of token ID 12345\n", + " \"67890\": -50, # Decrease likelihood of token ID 67890\n", + " \"11111\": 25, # Slightly increase likelihood of token ID 11111\n", + " },\n", + ")\n", + "\n", + "print_highlight(f\"Response with logit bias: {response.choices[0].message.content}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Parameters\n", + "\n", + "The chat completions API accepts OpenAI Chat Completions API's parameters. Refer to [OpenAI Chat Completions API](https://platform.openai.com/docs/api-reference/chat/create) for more details.\n", + "\n", + "SGLang extends the standard API with the `extra_body` parameter, allowing for additional customization. One key option within `extra_body` is `chat_template_kwargs`, which can be used to pass arguments to the chat template processor." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = client.chat.completions.create(\n", + " model=\"qwen/qwen2.5-0.5b-instruct\",\n", + " messages=[\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": \"You are a knowledgeable historian who provides concise responses.\",\n", + " },\n", + " {\"role\": \"user\", \"content\": \"Tell me about ancient Rome\"},\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Ancient Rome was a civilization centered in Italy.\",\n", + " },\n", + " {\"role\": \"user\", \"content\": \"What were their major achievements?\"},\n", + " ],\n", + " temperature=0.3, # Lower temperature for more focused responses\n", + " max_tokens=128, # Reasonable length for a concise response\n", + " top_p=0.95, # Slightly higher for better fluency\n", + " presence_penalty=0.2, # Mild penalty to avoid repetition\n", + " frequency_penalty=0.2, # Mild penalty for more natural language\n", + " n=1, # Single response is usually more stable\n", + " seed=42, # Keep for reproducibility\n", + ")\n", + "\n", + "print_highlight(response.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Streaming mode is also supported." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Logit Bias Support\n", + "\n", + "The completions API also supports the `logit_bias` parameter with the same functionality as described in the chat completions section above.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "stream = client.chat.completions.create(\n", + " model=\"qwen/qwen2.5-0.5b-instruct\",\n", + " messages=[{\"role\": \"user\", \"content\": \"Say this is a test\"}],\n", + " stream=True,\n", + ")\n", + "for chunk in stream:\n", + " if chunk.choices[0].delta.content is not None:\n", + " print(chunk.choices[0].delta.content, end=\"\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example with logit_bias parameter for completions API\n", + "# Note: You need to get the actual token IDs from your tokenizer\n", + "# For demonstration, we'll use some example token IDs\n", + "response = client.completions.create(\n", + " model=\"qwen/qwen2.5-0.5b-instruct\",\n", + " prompt=\"The best programming language for AI is\",\n", + " temperature=0.7,\n", + " max_tokens=20,\n", + " logit_bias={\n", + " \"12345\": 75, # Strongly favor token ID 12345\n", + " \"67890\": -100, # Completely avoid token ID 67890\n", + " \"11111\": -25, # Slightly discourage token ID 11111\n", + " },\n", + ")\n", + "\n", + "print_highlight(f\"Response with logit bias: {response.choices[0].text}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Completions\n", + "\n", + "### Usage\n", + "Completions API is similar to Chat Completions API, but without the `messages` parameter or chat templates." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = client.completions.create(\n", + " model=\"qwen/qwen2.5-0.5b-instruct\",\n", + " prompt=\"List 3 countries and their capitals.\",\n", + " temperature=0,\n", + " max_tokens=64,\n", + " n=1,\n", + " stop=None,\n", + ")\n", + "\n", + "print_highlight(f\"Response: {response}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Parameters\n", + "\n", + "The completions API accepts OpenAI Completions API's parameters. Refer to [OpenAI Completions API](https://platform.openai.com/docs/api-reference/completions/create) for more details.\n", + "\n", + "Here is an example of a detailed completions request:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = client.completions.create(\n", + " model=\"qwen/qwen2.5-0.5b-instruct\",\n", + " prompt=\"Write a short story about a space explorer.\",\n", + " temperature=0.7, # Moderate temperature for creative writing\n", + " max_tokens=150, # Longer response for a story\n", + " top_p=0.9, # Balanced diversity in word choice\n", + " stop=[\"\\n\\n\", \"THE END\"], # Multiple stop sequences\n", + " presence_penalty=0.3, # Encourage novel elements\n", + " frequency_penalty=0.3, # Reduce repetitive phrases\n", + " n=1, # Generate one completion\n", + " seed=123, # For reproducible results\n", + ")\n", + "\n", + "print_highlight(f\"Response: {response}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Structured Outputs (JSON, Regex, EBNF)\n", + "\n", + "For OpenAI compatible structured outputs API, refer to [Structured Outputs](../advanced_features/structured_outputs.ipynb) for more details.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using LoRA Adapters\n", + "\n", + "SGLang supports LoRA (Low-Rank Adaptation) adapters with OpenAI-compatible APIs. You can specify which adapter to use directly in the `model` parameter using the `base-model:adapter-name` syntax.\n", + "\n", + "**Server Setup:**\n", + "```bash\n", + "python -m sglang.launch_server \\\n", + " --model-path qwen/qwen2.5-0.5b-instruct \\\n", + " --enable-lora \\\n", + " --lora-paths adapter_a=/path/to/adapter_a adapter_b=/path/to/adapter_b\n", + "```\n", + "\n", + "For more details on LoRA serving configuration, see the [LoRA documentation](../advanced_features/lora.ipynb).\n", + "\n", + "**API Call:**\n", + "\n", + "(Recommended) Use the `model:adapter` syntax to specify which adapter to use:\n", + "```python\n", + "response = client.chat.completions.create(\n", + " model=\"qwen/qwen2.5-0.5b-instruct:adapter_a\", # ← base-model:adapter-name\n", + " messages=[{\"role\": \"user\", \"content\": \"Convert to SQL: show all users\"}],\n", + " max_tokens=50,\n", + ")\n", + "```\n", + "\n", + "**Backward Compatible: Using `extra_body`**\n", + "\n", + "The old `extra_body` method is still supported for backward compatibility:\n", + "```python\n", + "# Backward compatible method\n", + "response = client.chat.completions.create(\n", + " model=\"qwen/qwen2.5-0.5b-instruct\",\n", + " messages=[{\"role\": \"user\", \"content\": \"Convert to SQL: show all users\"}],\n", + " extra_body={\"lora_path\": \"adapter_a\"}, # ← old method\n", + " max_tokens=50,\n", + ")\n", + "```\n", + "**Note:** When both `model:adapter` and `extra_body[\"lora_path\"]` are specified, the `model:adapter` syntax takes precedence." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/openai_api_embeddings.ipynb b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/openai_api_embeddings.ipynb new file mode 100644 index 00000000..26e95a4e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/openai_api_embeddings.ipynb @@ -0,0 +1,195 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# OpenAI APIs - Embedding\n", + "\n", + "SGLang provides OpenAI-compatible APIs to enable a smooth transition from OpenAI services to self-hosted local models.\n", + "A complete reference for the API is available in the [OpenAI API Reference](https://platform.openai.com/docs/guides/embeddings).\n", + "\n", + "This tutorial covers the embedding APIs for embedding models. For a list of the supported models see the [corresponding overview page](../supported_models/embedding_models.md)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Launch A Server\n", + "\n", + "Launch the server in your terminal and wait for it to initialize. Remember to add `--is-embedding` to the command." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sglang.test.doc_patch import launch_server_cmd\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "embedding_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model-path Alibaba-NLP/gte-Qwen2-1.5B-instruct \\\n", + " --host 0.0.0.0 --is-embedding --log-level warning\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using cURL" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess, json\n", + "\n", + "text = \"Once upon a time\"\n", + "\n", + "curl_text = f\"\"\"curl -s http://localhost:{port}/v1/embeddings \\\n", + " -H \"Content-Type: application/json\" \\\n", + " -d '{{\"model\": \"Alibaba-NLP/gte-Qwen2-1.5B-instruct\", \"input\": \"{text}\"}}'\"\"\"\n", + "\n", + "result = subprocess.check_output(curl_text, shell=True)\n", + "\n", + "print(result)\n", + "\n", + "text_embedding = json.loads(result)[\"data\"][0][\"embedding\"]\n", + "\n", + "print_highlight(f\"Text embedding (first 10): {text_embedding[:10]}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Python Requests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "\n", + "text = \"Once upon a time\"\n", + "\n", + "response = requests.post(\n", + " f\"http://localhost:{port}/v1/embeddings\",\n", + " json={\"model\": \"Alibaba-NLP/gte-Qwen2-1.5B-instruct\", \"input\": text},\n", + ")\n", + "\n", + "text_embedding = response.json()[\"data\"][0][\"embedding\"]\n", + "\n", + "print_highlight(f\"Text embedding (first 10): {text_embedding[:10]}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using OpenAI Python Client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "\n", + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "# Text embedding example\n", + "response = client.embeddings.create(\n", + " model=\"Alibaba-NLP/gte-Qwen2-1.5B-instruct\",\n", + " input=text,\n", + ")\n", + "\n", + "embedding = response.data[0].embedding[:10]\n", + "print_highlight(f\"Text embedding (first 10): {embedding}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Input IDs\n", + "\n", + "SGLang also supports `input_ids` as input to get the embedding." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from transformers import AutoTokenizer\n", + "\n", + "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(\"Alibaba-NLP/gte-Qwen2-1.5B-instruct\")\n", + "input_ids = tokenizer.encode(text)\n", + "\n", + "curl_ids = f\"\"\"curl -s http://localhost:{port}/v1/embeddings \\\n", + " -H \"Content-Type: application/json\" \\\n", + " -d '{{\"model\": \"Alibaba-NLP/gte-Qwen2-1.5B-instruct\", \"input\": {json.dumps(input_ids)}}}'\"\"\"\n", + "\n", + "input_ids_embedding = json.loads(subprocess.check_output(curl_ids, shell=True))[\"data\"][\n", + " 0\n", + "][\"embedding\"]\n", + "\n", + "print_highlight(f\"Input IDs embedding (first 10): {input_ids_embedding[:10]}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(embedding_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multi-Modal Embedding Model\n", + "Please refer to [Multi-Modal Embedding Model](../supported_models/embedding_models.md)" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/openai_api_vision.ipynb b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/openai_api_vision.ipynb new file mode 100644 index 00000000..1db599dc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/openai_api_vision.ipynb @@ -0,0 +1,254 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# OpenAI APIs - Vision\n", + "\n", + "SGLang provides OpenAI-compatible APIs to enable a smooth transition from OpenAI services to self-hosted local models.\n", + "A complete reference for the API is available in the [OpenAI API Reference](https://platform.openai.com/docs/guides/vision).\n", + "This tutorial covers the vision APIs for vision language models.\n", + "\n", + "SGLang supports various vision language models such as Llama 3.2, LLaVA-OneVision, Qwen2.5-VL, Gemma3 and [more](../supported_models/multimodal_language_models.md).\n", + "\n", + "As an alternative to the OpenAI API, you can also use the [SGLang offline engine](https://github.com/sgl-project/sglang/blob/main/examples/runtime/engine/offline_batch_inference_vlm.py)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Launch A Server\n", + "\n", + "Launch the server in your terminal and wait for it to initialize." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sglang.test.doc_patch import launch_server_cmd\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "vision_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-VL-7B-Instruct --log-level warning\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using cURL\n", + "\n", + "Once the server is up, you can send test requests using curl or requests." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "\n", + "curl_command = f\"\"\"\n", + "curl -s http://localhost:{port}/v1/chat/completions \\\\\n", + " -H \"Content-Type: application/json\" \\\\\n", + " -d '{{\n", + " \"model\": \"Qwen/Qwen2.5-VL-7B-Instruct\",\n", + " \"messages\": [\n", + " {{\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {{\n", + " \"type\": \"text\",\n", + " \"text\": \"What’s in this image?\"\n", + " }},\n", + " {{\n", + " \"type\": \"image_url\",\n", + " \"image_url\": {{\n", + " \"url\": \"https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true\"\n", + " }}\n", + " }}\n", + " ]\n", + " }}\n", + " ],\n", + " \"max_tokens\": 300\n", + " }}'\n", + "\"\"\"\n", + "\n", + "response = subprocess.check_output(curl_command, shell=True).decode()\n", + "print_highlight(response)\n", + "\n", + "\n", + "response = subprocess.check_output(curl_command, shell=True).decode()\n", + "print_highlight(response)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Python Requests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "\n", + "url = f\"http://localhost:{port}/v1/chat/completions\"\n", + "\n", + "data = {\n", + " \"model\": \"Qwen/Qwen2.5-VL-7B-Instruct\",\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\"type\": \"text\", \"text\": \"What’s in this image?\"},\n", + " {\n", + " \"type\": \"image_url\",\n", + " \"image_url\": {\n", + " \"url\": \"https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true\"\n", + " },\n", + " },\n", + " ],\n", + " }\n", + " ],\n", + " \"max_tokens\": 300,\n", + "}\n", + "\n", + "response = requests.post(url, json=data)\n", + "print_highlight(response.text)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using OpenAI Python Client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from openai import OpenAI\n", + "\n", + "client = OpenAI(base_url=f\"http://localhost:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"Qwen/Qwen2.5-VL-7B-Instruct\",\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"What is in this image?\",\n", + " },\n", + " {\n", + " \"type\": \"image_url\",\n", + " \"image_url\": {\n", + " \"url\": \"https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true\"\n", + " },\n", + " },\n", + " ],\n", + " }\n", + " ],\n", + " max_tokens=300,\n", + ")\n", + "\n", + "print_highlight(response.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multiple-Image Inputs\n", + "\n", + "The server also supports multiple images and interleaved text and images if the model supports it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from openai import OpenAI\n", + "\n", + "client = OpenAI(base_url=f\"http://localhost:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"Qwen/Qwen2.5-VL-7B-Instruct\",\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"image_url\",\n", + " \"image_url\": {\n", + " \"url\": \"https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true\",\n", + " },\n", + " },\n", + " {\n", + " \"type\": \"image_url\",\n", + " \"image_url\": {\n", + " \"url\": \"https://raw.githubusercontent.com/sgl-project/sglang/main/assets/logo.png\",\n", + " },\n", + " },\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I have two very different images. They are not related at all. \"\n", + " \"Please describe the first image in one sentence, and then describe the second image in another sentence.\",\n", + " },\n", + " ],\n", + " }\n", + " ],\n", + " temperature=0,\n", + ")\n", + "\n", + "print_highlight(response.choices[0].message.content)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(vision_process)" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/popular_model_usage.rst b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/popular_model_usage.rst new file mode 100644 index 00000000..b8e75f21 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/popular_model_usage.rst @@ -0,0 +1,12 @@ +Popular Model Usage (DeepSeek, GPT-OSS, Llama, Qwen, and more) +=============================================================== + +.. toctree:: + :maxdepth: 1 + + deepseek_v3.md + deepseek_v32.md + gpt_oss.md + qwen3.md + qwen3_vl.md + llama4.md diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/qwen3.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/qwen3.md new file mode 100644 index 00000000..5e6773c0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/qwen3.md @@ -0,0 +1,33 @@ +# Qwen3-Next Usage + +SGLang has supported Qwen3-Next-80B-A3B-Instruct and Qwen3-Next-80B-A3B-Thinking since [this PR](https://github.com/sgl-project/sglang/pull/10233). + +## Launch Qwen3-Next with SGLang + +To serve Qwen3-Next models on 4xH100/H200 GPUs: + +```bash +python3 -m sglang.launch_server --model Qwen/Qwen3-Next-80B-A3B-Instruct --tp 4 +``` + +### Configuration Tips +- `--max-mamba-cache-size`: Adjust `--max-mamba-cache-size` to increase mamba cache space and max running requests capability. It will decrease KV cache space as a trade-off. You can adjust it according to workload. +- `--mamba-ssm-dtype`: `bfloat16` or `float32`, use `bfloat16` to save mamba cache size and `float32` to get more accurate results. The default setting is `float32`. + +### EAGLE Speculative Decoding +**Description**: SGLang has supported Qwen3-Next models with [EAGLE speculative decoding](https://docs.sglang.io/advanced_features/speculative_decoding.html#EAGLE-Decoding). + +**Usage**: +Add arguments `--speculative-algorithm`, `--speculative-num-steps`, `--speculative-eagle-topk` and `--speculative-num-draft-tokens` to enable this feature. For example: + +``` bash +python3 -m sglang.launch_server \ + --model Qwen/Qwen3-Next-80B-A3B-Instruct \ + --tp 4 \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 \ + --speculative-algo NEXTN +``` + +Details can be seen in [this PR](https://github.com/sgl-project/sglang/pull/10233). diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/qwen3_vl.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/qwen3_vl.md new file mode 100644 index 00000000..f05e7832 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/qwen3_vl.md @@ -0,0 +1,130 @@ +# Qwen3-VL Usage + +[Qwen3-VL](https://huggingface.co/collections/Qwen/qwen3-vl) +is Alibaba’s latest multimodal large language model with strong text, vision, and reasoning capabilities. +SGLang supports Qwen3-VL Family of models with Image and Video input support. + +## Launch commands for SGLang + +Below are suggested launch commands tailored for different hardware / precision modes + +### FP8 (quantised) mode +For high memory-efficiency and latency optimized deployments (e.g., on H100, H200) where FP8 checkpoint is supported: +```bash +python3 -m sglang.launch_server \ + --model-path Qwen/Qwen3-VL-235B-A22B-Instruct-FP8 \ + --tp 8 \ + --ep 8 \ + --host 0.0.0.0 \ + --port 30000 \ + --keep-mm-feature-on-device +``` + +### Non-FP8 (BF16 / full precision) mode +For deployments on A100/H100 where BF16 is used (or FP8 snapshot not used): +```bash +python3 -m sglang.launch_server \ + --model-path Qwen/Qwen3-VL-235B-A22B-Instruct \ + --tp 8 \ + --ep 8 \ + --host 0.0.0.0 \ + --port 30000 \ +``` + +## Hardware-specific notes / recommendations + +- On H100 with FP8: Use the FP8 checkpoint for best memory efficiency. +- On A100 / H100 with BF16 (non-FP8): It’s recommended to use `--mm-max-concurrent-calls` to control parallel throughput and GPU memory usage during image/video inference. +- On H200 & B200: The model can be run “out of the box”, supporting full context length plus concurrent image + video processing. + +## Sending Image/Video Requests + +### Image input: + +```python +import requests + +url = f"http://localhost:30000/v1/chat/completions" + +data = { + "model": "Qwen/Qwen3-VL-30B-A3B-Instruct", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What’s in this image?"}, + { + "type": "image_url", + "image_url": { + "url": "https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true" + }, + }, + ], + } + ], + "max_tokens": 300, +} + +response = requests.post(url, json=data) +print(response.text) +``` + +### Video Input: + +```python +import requests + +url = f"http://localhost:30000/v1/chat/completions" + +data = { + "model": "Qwen/Qwen3-VL-30B-A3B-Instruct", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What’s happening in this video?"}, + { + "type": "video_url", + "video_url": { + "url": "https://github.com/sgl-project/sgl-test-files/raw/refs/heads/main/videos/jobs_presenting_ipod.mp4" + }, + }, + ], + } + ], + "max_tokens": 300, +} + +response = requests.post(url, json=data) +print(response.text) +``` + +## Important Server Parameters and Flags + +When launching the model server for **multimodal support**, you can use the following command-line arguments to fine-tune performance and behavior: + +- `--mm-attention-backend`: Specify multimodal attention backend. Eg. `fa3`(Flash Attention 3) +- `--mm-max-concurrent-calls `: Specifies the **maximum number of concurrent asynchronous multimodal data processing calls** allowed on the server. Use this to control parallel throughput and GPU memory usage during image/video inference. +- `--mm-per-request-timeout `: Defines the **timeout duration (in seconds)** for each multimodal request. If a request exceeds this time limit (e.g., for very large video inputs), it will be automatically terminated. +- `--keep-mm-feature-on-device`: Instructs the server to **retain multimodal feature tensors on the GPU** after processing. This avoids device-to-host (D2H) memory copies and improves performance for repeated or high-frequency inference workloads. +- `SGLANG_USE_CUDA_IPC_TRANSPORT=1`: Shared memory pool based CUDA IPC for multi-modal data transport. For significantly improving e2e latency. + +### Example usage with the above optimizations: +```bash +SGLANG_USE_CUDA_IPC_TRANSPORT=1 \ +SGLANG_VLM_CACHE_SIZE_MB=0 \ +python -m sglang.launch_server \ + --model-path Qwen/Qwen3-VL-235B-A22B-Instruct \ + --host 0.0.0.0 \ + --port 30000 \ + --trust-remote-code \ + --tp-size 8 \ + --enable-cache-report \ + --log-level info \ + --max-running-requests 64 \ + --mem-fraction-static 0.65 \ + --chunked-prefill-size 8192 \ + --attention-backend fa3 \ + --mm-attention-backend fa3 \ + --enable-metrics +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/sampling_params.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/sampling_params.md new file mode 100644 index 00000000..a97a7368 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/sampling_params.md @@ -0,0 +1,346 @@ +# Sampling Parameters + +This doc describes the sampling parameters of the SGLang Runtime. It is the low-level endpoint of the runtime. +If you want a high-level endpoint that can automatically handle chat templates, consider using the [OpenAI Compatible API](openai_api_completions.ipynb). + +## `/generate` Endpoint + +The `/generate` endpoint accepts the following parameters in JSON format. For detailed usage, see the [native API doc](native_api.ipynb). The object is defined at `io_struct.py::GenerateReqInput`. You can also read the source code to find more arguments and docs. + +| Argument | Type/Default | Description | +|----------------------------|------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| text | `Optional[Union[List[str], str]] = None` | The input prompt. Can be a single prompt or a batch of prompts. | +| input_ids | `Optional[Union[List[List[int]], List[int]]] = None` | The token IDs for text; one can specify either text or input_ids. | +| input_embeds | `Optional[Union[List[List[List[float]]], List[List[float]]]] = None` | The embeddings for input_ids; one can specify either text, input_ids, or input_embeds. | +| image_data | `Optional[Union[List[List[ImageDataItem]], List[ImageDataItem], ImageDataItem]] = None` | The image input. Can be an image instance, file name, URL, or base64 encoded string. Can be a single image, list of images, or list of lists of images. | +| audio_data | `Optional[Union[List[AudioDataItem], AudioDataItem]] = None` | The audio input. Can be a file name, URL, or base64 encoded string. | +| sampling_params | `Optional[Union[List[Dict], Dict]] = None` | The sampling parameters as described in the sections below. | +| rid | `Optional[Union[List[str], str]] = None` | The request ID. | +| return_logprob | `Optional[Union[List[bool], bool]] = None` | Whether to return log probabilities for tokens. | +| logprob_start_len | `Optional[Union[List[int], int]] = None` | If return_logprob, the start location in the prompt for returning logprobs. Default is "-1", which returns logprobs for output tokens only. | +| top_logprobs_num | `Optional[Union[List[int], int]] = None` | If return_logprob, the number of top logprobs to return at each position. | +| token_ids_logprob | `Optional[Union[List[List[int]], List[int]]] = None` | If return_logprob, the token IDs to return logprob for. | +| return_text_in_logprobs | `bool = False` | Whether to detokenize tokens in text in the returned logprobs. | +| stream | `bool = False` | Whether to stream output. | +| lora_path | `Optional[Union[List[Optional[str]], Optional[str]]] = None` | The path to the LoRA. | +| custom_logit_processor | `Optional[Union[List[Optional[str]], str]] = None` | Custom logit processor for advanced sampling control. Must be a serialized instance of `CustomLogitProcessor` using its `to_str()` method. For usage see below. | +| return_hidden_states | `Union[List[bool], bool] = False` | Whether to return hidden states. | + +## Sampling parameters + +The object is defined at `sampling_params.py::SamplingParams`. You can also read the source code to find more arguments and docs. + +### Note on defaults + +By default, SGLang initializes several sampling parameters from the model's `generation_config.json` (when the server is launched with `--sampling-defaults model`, which is the default). To use SGLang/OpenAI constant defaults instead, start the server with `--sampling-defaults openai`. You can always override any parameter per request via `sampling_params`. + +```bash +# Use model-provided defaults from generation_config.json (default behavior) +python -m sglang.launch_server --model-path --sampling-defaults model + +# Use SGLang/OpenAI constant defaults instead +python -m sglang.launch_server --model-path --sampling-defaults openai +``` + +### Core parameters + +| Argument | Type/Default | Description | +|-----------------|----------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------| +| max_new_tokens | `int = 128` | The maximum output length measured in tokens. | +| stop | `Optional[Union[str, List[str]]] = None` | One or multiple [stop words](https://platform.openai.com/docs/api-reference/chat/create#chat-create-stop). Generation will stop if one of these words is sampled. | +| stop_token_ids | `Optional[List[int]] = None` | Provide stop words in the form of token IDs. Generation will stop if one of these token IDs is sampled. | +| stop_regex | `Optional[Union[str, List[str]]] = None` | Stop when hitting any of the regex patterns in this list | +| temperature | `float (model default; fallback 1.0)` | [Temperature](https://platform.openai.com/docs/api-reference/chat/create#chat-create-temperature) when sampling the next token. `temperature = 0` corresponds to greedy sampling, a higher temperature leads to more diversity. | +| top_p | `float (model default; fallback 1.0)` | [Top-p](https://platform.openai.com/docs/api-reference/chat/create#chat-create-top_p) selects tokens from the smallest sorted set whose cumulative probability exceeds `top_p`. When `top_p = 1`, this reduces to unrestricted sampling from all tokens. | +| top_k | `int (model default; fallback -1)` | [Top-k](https://developer.nvidia.com/blog/how-to-get-better-outputs-from-your-large-language-model/#predictability_vs_creativity) randomly selects from the `k` highest-probability tokens. | +| min_p | `float (model default; fallback 0.0)` | [Min-p](https://github.com/huggingface/transformers/issues/27670) samples from tokens with probability larger than `min_p * highest_token_probability`. | + +### Penalizers + +| Argument | Type/Default | Description | +|--------------------|------------------------|------------------------------------------------------------------------------------------------------------------------------------------------| +| frequency_penalty | `float = 0.0` | Penalizes tokens based on their frequency in generation so far. Must be between `-2` and `2` where negative numbers encourage repeatment of tokens and positive number encourages sampling of new tokens. The scaling of penalization grows linearly with each appearance of a token. | +| presence_penalty | `float = 0.0` | Penalizes tokens if they appeared in the generation so far. Must be between `-2` and `2` where negative numbers encourage repeatment of tokens and positive number encourages sampling of new tokens. The scaling of the penalization is constant if a token occurred. | +| repetition_penalty | `float = 1.0` | Scales the logits of previously generated tokens to discourage (values > 1) or encourage (values < 1) repetition. Valid range is `[0, 2]`; `1.0` leaves probabilities unchanged. | +| min_new_tokens | `int = 0` | Forces the model to generate at least `min_new_tokens` until a stop word or EOS token is sampled. Note that this might lead to unintended behavior, for example, if the distribution is highly skewed towards these tokens. | + +### Constrained decoding + +Please refer to our dedicated guide on [constrained decoding](../advanced_features/structured_outputs.ipynb) for the following parameters. + +| Argument | Type/Default | Description | +|-----------------|---------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------| +| json_schema | `Optional[str] = None` | JSON schema for structured outputs. | +| regex | `Optional[str] = None` | Regex for structured outputs. | +| ebnf | `Optional[str] = None` | EBNF for structured outputs. | +| structural_tag | `Optional[str] = None` | The structal tag for structured outputs. | + +### Other options + +| Argument | Type/Default | Description | +|-------------------------------|---------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------| +| n | `int = 1` | Specifies the number of output sequences to generate per request. (Generating multiple outputs in one request (n > 1) is discouraged; repeating the same prompts several times offers better control and efficiency.) | +| ignore_eos | `bool = False` | Don't stop generation when EOS token is sampled. | +| skip_special_tokens | `bool = True` | Remove special tokens during decoding. | +| spaces_between_special_tokens | `bool = True` | Whether or not to add spaces between special tokens during detokenization. | +| no_stop_trim | `bool = False` | Don't trim stop words or EOS token from the generated text. | +| custom_params | `Optional[List[Optional[Dict[str, Any]]]] = None` | Used when employing `CustomLogitProcessor`. For usage, see below. | + +## Examples + +### Normal + +Launch a server: + +```bash +python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --port 30000 +``` + +Send a request: + +```python +import requests + +response = requests.post( + "http://localhost:30000/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, +) +print(response.json()) +``` + +Detailed example in [send request](./send_request.ipynb). + +### Streaming + +Send a request and stream the output: + +```python +import requests, json + +response = requests.post( + "http://localhost:30000/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + "stream": True, + }, + stream=True, +) + +prev = 0 +for chunk in response.iter_lines(decode_unicode=False): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]": + break + data = json.loads(chunk[5:].strip("\n")) + output = data["text"].strip() + print(output[prev:], end="", flush=True) + prev = len(output) +print("") +``` + +Detailed example in [openai compatible api](openai_api_completions.ipynb). + +### Multimodal + +Launch a server: + +```bash +python3 -m sglang.launch_server --model-path lmms-lab/llava-onevision-qwen2-7b-ov +``` + +Download an image: + +```bash +curl -o example_image.png -L https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true +``` + +Send a request: + +```python +import requests + +response = requests.post( + "http://localhost:30000/generate", + json={ + "text": "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" + "<|im_start|>user\n\nDescribe this image in a very short sentence.<|im_end|>\n" + "<|im_start|>assistant\n", + "image_data": "example_image.png", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, +) +print(response.json()) +``` + +The `image_data` can be a file name, a URL, or a base64 encoded string. See also `python/sglang/srt/utils.py:load_image`. + +Streaming is supported in a similar manner as [above](#streaming). + +Detailed example in [OpenAI API Vision](openai_api_vision.ipynb). + +### Structured Outputs (JSON, Regex, EBNF) + +You can specify a JSON schema, regular expression or [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) to constrain the model output. The model output will be guaranteed to follow the given constraints. Only one constraint parameter (`json_schema`, `regex`, or `ebnf`) can be specified for a request. + +SGLang supports two grammar backends: + +- [XGrammar](https://github.com/mlc-ai/xgrammar) (default): Supports JSON schema, regular expression, and EBNF constraints. + - XGrammar currently uses the [GGML BNF format](https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md). +- [Outlines](https://github.com/dottxt-ai/outlines): Supports JSON schema and regular expression constraints. + +If instead you want to initialize the Outlines backend, you can use `--grammar-backend outlines` flag: + +```bash +python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ +--port 30000 --host 0.0.0.0 --grammar-backend [xgrammar|outlines] # xgrammar or outlines (default: xgrammar) +``` + +```python +import json +import requests + +json_schema = json.dumps({ + "type": "object", + "properties": { + "name": {"type": "string", "pattern": "^[\\w]+$"}, + "population": {"type": "integer"}, + }, + "required": ["name", "population"], +}) + +# JSON (works with both Outlines and XGrammar) +response = requests.post( + "http://localhost:30000/generate", + json={ + "text": "Here is the information of the capital of France in the JSON format.\n", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 64, + "json_schema": json_schema, + }, + }, +) +print(response.json()) + +# Regular expression (Outlines backend only) +response = requests.post( + "http://localhost:30000/generate", + json={ + "text": "Paris is the capital of", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 64, + "regex": "(France|England)", + }, + }, +) +print(response.json()) + +# EBNF (XGrammar backend only) +response = requests.post( + "http://localhost:30000/generate", + json={ + "text": "Write a greeting.", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 64, + "ebnf": 'root ::= "Hello" | "Hi" | "Hey"', + }, + }, +) +print(response.json()) +``` + +Detailed example in [structured outputs](../advanced_features/structured_outputs.ipynb). + +### Custom logit processor + +Launch a server with `--enable-custom-logit-processor` flag on. + +```bash +python -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3-8B-Instruct \ + --port 30000 \ + --enable-custom-logit-processor +``` + +Define a custom logit processor that will always sample a specific token id. + +```python +from sglang.srt.sampling.custom_logit_processor import CustomLogitProcessor + +class DeterministicLogitProcessor(CustomLogitProcessor): + """A dummy logit processor that changes the logits to always + sample the given token id. + """ + + def __call__(self, logits, custom_param_list): + # Check that the number of logits matches the number of custom parameters + assert logits.shape[0] == len(custom_param_list) + key = "token_id" + + for i, param_dict in enumerate(custom_param_list): + # Mask all other tokens + logits[i, :] = -float("inf") + # Assign highest probability to the specified token + logits[i, param_dict[key]] = 0.0 + return logits +``` + +Send a request: + +```python +import requests + +response = requests.post( + "http://localhost:30000/generate", + json={ + "text": "The capital of France is", + "custom_logit_processor": DeterministicLogitProcessor().to_str(), + "sampling_params": { + "temperature": 0.0, + "max_new_tokens": 32, + "custom_params": {"token_id": 5}, + }, + }, +) +print(response.json()) +``` + +Send an OpenAI chat completion request: + +```python +import openai +from sglang.utils import print_highlight + +client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="None") + +response = client.chat.completions.create( + model="meta-llama/Meta-Llama-3-8B-Instruct", + messages=[ + {"role": "user", "content": "List 3 countries and their capitals."}, + ], + temperature=0.0, + max_tokens=32, + extra_body={ + "custom_logit_processor": DeterministicLogitProcessor().to_str(), + "custom_params": {"token_id": 5}, + }, +) + +print_highlight(f"Response: {response}") +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/send_request.ipynb b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/send_request.ipynb new file mode 100644 index 00000000..6e457a02 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/basic_usage/send_request.ipynb @@ -0,0 +1,253 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Sending Requests\n", + "This notebook provides a quick-start guide to use SGLang in chat completions after installation.\n", + "\n", + "- For Vision Language Models, see [OpenAI APIs - Vision](openai_api_vision.ipynb).\n", + "- For Embedding Models, see [OpenAI APIs - Embedding](openai_api_embeddings.ipynb) and [Encode (embedding model)](native_api.html#Encode-(embedding-model)).\n", + "- For Reward Models, see [Classify (reward model)](native_api.html#Classify-(reward-model))." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Launch A Server" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sglang.test.doc_patch import launch_server_cmd\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "# This is equivalent to running the following command in your terminal\n", + "# python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct \\\n", + " --host 0.0.0.0 --log-level warning\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using cURL\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess, json\n", + "\n", + "curl_command = f\"\"\"\n", + "curl -s http://localhost:{port}/v1/chat/completions \\\n", + " -H \"Content-Type: application/json\" \\\n", + " -d '{{\"model\": \"qwen/qwen2.5-0.5b-instruct\", \"messages\": [{{\"role\": \"user\", \"content\": \"What is the capital of France?\"}}]}}'\n", + "\"\"\"\n", + "\n", + "response = json.loads(subprocess.check_output(curl_command, shell=True))\n", + "print_highlight(response)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Python Requests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "\n", + "url = f\"http://localhost:{port}/v1/chat/completions\"\n", + "\n", + "data = {\n", + " \"model\": \"qwen/qwen2.5-0.5b-instruct\",\n", + " \"messages\": [{\"role\": \"user\", \"content\": \"What is the capital of France?\"}],\n", + "}\n", + "\n", + "response = requests.post(url, json=data)\n", + "print_highlight(response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using OpenAI Python Client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "\n", + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"qwen/qwen2.5-0.5b-instruct\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=64,\n", + ")\n", + "print_highlight(response)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Streaming" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "\n", + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "# Use stream=True for streaming responses\n", + "response = client.chat.completions.create(\n", + " model=\"qwen/qwen2.5-0.5b-instruct\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=64,\n", + " stream=True,\n", + ")\n", + "\n", + "# Handle the streaming output\n", + "for chunk in response:\n", + " if chunk.choices[0].delta.content:\n", + " print(chunk.choices[0].delta.content, end=\"\", flush=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Native Generation APIs\n", + "\n", + "You can also use the native `/generate` endpoint with requests, which provides more flexibility. An API reference is available at [Sampling Parameters](sampling_params.md)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "\n", + "response = requests.post(\n", + " f\"http://localhost:{port}/generate\",\n", + " json={\n", + " \"text\": \"The capital of France is\",\n", + " \"sampling_params\": {\n", + " \"temperature\": 0,\n", + " \"max_new_tokens\": 32,\n", + " },\n", + " },\n", + ")\n", + "\n", + "print_highlight(response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Streaming" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests, json\n", + "\n", + "response = requests.post(\n", + " f\"http://localhost:{port}/generate\",\n", + " json={\n", + " \"text\": \"The capital of France is\",\n", + " \"sampling_params\": {\n", + " \"temperature\": 0,\n", + " \"max_new_tokens\": 32,\n", + " },\n", + " \"stream\": True,\n", + " },\n", + " stream=True,\n", + ")\n", + "\n", + "prev = 0\n", + "for chunk in response.iter_lines(decode_unicode=False):\n", + " chunk = chunk.decode(\"utf-8\")\n", + " if chunk and chunk.startswith(\"data:\"):\n", + " if chunk == \"data: [DONE]\":\n", + " break\n", + " data = json.loads(chunk[5:].strip(\"\\n\"))\n", + " output = data[\"text\"]\n", + " print(output[prev:], end=\"\", flush=True)\n", + " prev = len(output)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/conf.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/conf.py new file mode 100644 index 00000000..d6ca64d8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/conf.py @@ -0,0 +1,205 @@ +import os +import sys +from datetime import datetime + +sys.path.insert(0, os.path.abspath("../..")) + +version_file = "../python/sglang/version.py" +with open(version_file, "r") as f: + exec(compile(f.read(), version_file, "exec")) +__version__ = locals()["__version__"] + +project = "SGLang" +copyright = f"2023-{datetime.now().year}, SGLang" +author = "SGLang Team" + +version = __version__ +release = __version__ + +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.napoleon", + "sphinx.ext.viewcode", + "sphinx.ext.autosectionlabel", + "sphinx.ext.intersphinx", + "sphinx_tabs.tabs", + "myst_parser", + "sphinx_copybutton", + "sphinxcontrib.mermaid", + "nbsphinx", + "sphinx.ext.mathjax", +] + +nbsphinx_allow_errors = True +nbsphinx_execute = "never" + +autosectionlabel_prefix_document = True +nbsphinx_allow_directives = True + + +myst_enable_extensions = [ + "dollarmath", + "amsmath", + "deflist", + "colon_fence", + "html_image", + "linkify", + "substitution", +] + +myst_heading_anchors = 3 + +nbsphinx_kernel_name = "python3" +nbsphinx_execute_arguments = [ + "--InlineBackend.figure_formats={'svg', 'pdf'}", + "--InlineBackend.rc={'figure.dpi': 96}", +] + + +nb_render_priority = { + "html": ( + "application/vnd.jupyter.widget-view+json", + "application/javascript", + "text/html", + "image/svg+xml", + "image/png", + "image/jpeg", + "text/markdown", + "text/latex", + "text/plain", + ) +} + +myst_enable_extensions = [ + "dollarmath", + "amsmath", + "deflist", + "colon_fence", + "html_image", + "linkify", + "substitution", +] + +myst_heading_anchors = 3 +myst_ref_domains = ["std", "py"] + +templates_path = ["_templates"] + +source_suffix = { + ".rst": "restructuredtext", + ".md": "markdown", +} + +master_doc = "index" + +language = "en" + +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +pygments_style = "sphinx" + +html_theme = "sphinx_book_theme" +html_logo = "_static/image/logo.png" +html_favicon = "_static/image/logo.ico" +html_title = project +html_copy_source = True +html_last_updated_fmt = "" + +html_theme_options = { + "repository_url": "https://github.com/sgl-project/sgl-project.github.io", + "repository_branch": "main", + "show_navbar_depth": 3, + "max_navbar_depth": 4, + "collapse_navbar": True, + "use_edit_page_button": True, + "use_source_button": True, + "use_issues_button": True, + "use_repository_button": True, + "use_download_button": True, + "use_sidenotes": True, + "show_toc_level": 2, +} + +html_context = { + "display_github": True, + "github_user": "sgl-project", + "github_repo": "sgl-project.github.io", + "github_version": "main", + "conf_py_path": "/docs/", +} + +html_static_path = ["_static"] +html_css_files = ["css/custom_log.css"] + + +def setup(app): + app.add_css_file("css/custom_log.css") + + +myst_enable_extensions = [ + "dollarmath", + "amsmath", + "deflist", + "colon_fence", +] +myst_heading_anchors = 5 + +htmlhelp_basename = "sglangdoc" + +latex_elements = {} + +latex_documents = [ + (master_doc, "sglang.tex", "sglang Documentation", "SGLang Team", "manual"), +] + +man_pages = [(master_doc, "sglang", "sglang Documentation", [author], 1)] + +texinfo_documents = [ + ( + master_doc, + "sglang", + "sglang Documentation", + author, + "sglang", + "One line description of project.", + "Miscellaneous", + ), +] + +epub_title = project + +epub_exclude_files = ["search.html"] + +copybutton_prompt_text = r">>> |\.\.\. " +copybutton_prompt_is_regexp = True + +autodoc_preserve_defaults = True +navigation_with_keys = False + +autodoc_mock_imports = [ + "torch", + "transformers", + "triton", +] + +intersphinx_mapping = { + "python": ("https://docs.python.org/3.12", None), + "typing_extensions": ("https://typing-extensions.readthedocs.io/en/latest", None), + "pillow": ("https://pillow.readthedocs.io/en/stable", None), + "numpy": ("https://numpy.org/doc/stable", None), + "torch": ("https://pytorch.org/docs/stable", None), +} + +html_theme = "sphinx_book_theme" + + +nbsphinx_prolog = """ +.. raw:: html + + +""" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/deploy.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/deploy.py new file mode 100644 index 00000000..75b7ea7f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/deploy.py @@ -0,0 +1,22 @@ +# Deploy the documents + +import os +from datetime import datetime + + +def run_cmd(cmd): + print(cmd) + os.system(cmd) + + +run_cmd("cd $DOC_SITE_PATH; git pull") + +# (Optional) Remove old files +# run_cmd("rm -rf $ALPA_SITE_PATH/*") + +run_cmd("cp -r _build/html/* $DOC_SITE_PATH") + +cmd_message = f"Update {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" +run_cmd( + f"cd $DOC_SITE_PATH; git add .; git commit -m '{cmd_message}'; git push origin main" +) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/bench_serving.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/bench_serving.md new file mode 100644 index 00000000..b2f8568e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/bench_serving.md @@ -0,0 +1,355 @@ +# Bench Serving Guide + +This guide explains how to benchmark online serving throughput and latency using `python -m sglang.bench_serving`. It supports multiple inference backends via OpenAI-compatible and native endpoints, and produces both console metrics and optional JSONL outputs. + +### What it does + +- Generates synthetic or dataset-driven prompts and submits them to a target serving endpoint +- Measures throughput, time-to-first-token (TTFT), inter-token latency (ITL), per-request end-to-end latency, and more +- Supports streaming or non-streaming modes, rate control, and concurrency limits + +### Supported backends and endpoints + +- `sglang` / `sglang-native`: `POST /generate` +- `sglang-oai`, `vllm`, `lmdeploy`: `POST /v1/completions` +- `sglang-oai-chat`, `vllm-chat`, `lmdeploy-chat`: `POST /v1/chat/completions` +- `trt` (TensorRT-LLM): `POST /v2/models/ensemble/generate_stream` +- `gserver`: Custom server (Not Implemented yet in this script) +- `truss`: `POST /v1/models/model:predict` + +If `--base-url` is provided, requests are sent to it. Otherwise, `--host` and `--port` are used. When `--model` is not provided, the script will attempt to query `GET /v1/models` for an available model ID (OpenAI-compatible endpoints). + +### Prerequisites + +- Python 3.8+ +- Dependencies typically used by this script: `aiohttp`, `numpy`, `requests`, `tqdm`, `transformers`, and for some datasets `datasets`, `pillow`, `pybase64`. Install as needed. +- An inference server running and reachable via the endpoints above +- If your server requires authentication, set environment variable `OPENAI_API_KEY` (used as `Authorization: Bearer `) + +### Quick start + +Run a basic benchmark against an sglang server exposing `/generate`: + +```bash +python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct +``` + +```bash +python3 -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 --port 30000 \ + --num-prompts 1000 \ + --model meta-llama/Llama-3.1-8B-Instruct +``` + +Or, using an OpenAI-compatible endpoint (completions): + +```bash +python3 -m sglang.bench_serving \ + --backend vllm \ + --base-url http://127.0.0.1:8000 \ + --num-prompts 1000 \ + --model meta-llama/Llama-3.1-8B-Instruct +``` + +### Datasets + +Select with `--dataset-name`: + +- `sharegpt` (default): loads ShareGPT-style pairs; optionally restrict with `--sharegpt-context-len` and override outputs with `--sharegpt-output-len` +- `random`: random text lengths; sampled from ShareGPT token space +- `random-ids`: random token ids (can lead to gibberish) +- `image`: generates images and wraps them in chat messages; supports custom resolutions, multiple formats, and different content types +- `generated-shared-prefix`: synthetic dataset with shared long system prompts and short questions +- `mmmu`: samples from MMMU (Math split) and includes images + +Common dataset flags: + +- `--num-prompts N`: number of requests +- `--random-input-len`, `--random-output-len`, `--random-range-ratio`: for random/random-ids/image +- `--image-count`: Number of images per request (for `image` dataset). + +- `--apply-chat-template`: apply tokenizer chat template when constructing prompts +- `--dataset-path PATH`: file path for ShareGPT json; if blank and missing, it will be downloaded and cached + +Generated Shared Prefix flags (for `generated-shared-prefix`): + +- `--gsp-num-groups` +- `--gsp-prompts-per-group` +- `--gsp-system-prompt-len` +- `--gsp-question-len` +- `--gsp-output-len` + +Image dataset flags (for `image`): + +- `--image-count`: Number of images per request +- `--image-resolution`: Image resolution; supports presets (4k, 1080p, 720p, 360p) or custom 'heightxwidth' format (e.g., 1080x1920, 512x768) +- `--image-format`: Image format (jpeg or png) +- `--image-content`: Image content type (random or blank) + +### Examples + +1. To benchmark image dataset with 3 images per request, 500 prompts, 512 input length, and 512 output length, you can run: + +```bash +python -m sglang.launch_server --model-path Qwen/Qwen2.5-VL-3B-Instruct --disable-radix-cache +``` + +```bash +python -m sglang.bench_serving \ + --backend sglang-oai-chat \ + --dataset-name image \ + --num-prompts 500 \ + --image-count 3 \ + --image-resolution 720p \ + --random-input-len 512 \ + --random-output-len 512 +``` + +2. To benchmark random dataset with 3000 prompts, 1024 input length, and 1024 output length, you can run: + +```bash +python -m sglang.launch_server --model-path Qwen/Qwen2.5-3B-Instruct +``` + +```bash +python3 -m sglang.bench_serving \ + --backend sglang \ + --dataset-name random \ + --num-prompts 3000 \ + --random-input 1024 \ + --random-output 1024 \ + --random-range-ratio 0.5 +``` + +### Choosing model and tokenizer + +- `--model` is required unless the backend exposes `GET /v1/models`, in which case the first model ID is auto-selected. +- `--tokenizer` defaults to `--model`. Both can be HF model IDs or local paths. +- For ModelScope workflows, setting `SGLANG_USE_MODELSCOPE=true` enables fetching via ModelScope (weights are skipped for speed). +- If your tokenizer lacks a chat template, the script warns because token counting can be less robust for gibberish outputs. + +### Rate, concurrency, and streaming + +- `--request-rate`: requests per second. `inf` sends all immediately (burst). Non-infinite rate uses a Poisson process for arrival times. +- `--max-concurrency`: caps concurrent in-flight requests regardless of arrival rate. +- `--disable-stream`: switch to non-streaming mode when supported; TTFT then equals total latency for chat completions. + +### Other key options + +- `--output-file FILE.jsonl`: append JSONL results to file; auto-named if unspecified +- `--output-details`: include per-request arrays (generated texts, errors, ttfts, itls, input/output lens) +- `--extra-request-body '{"top_p":0.9,"temperature":0.6}'`: merged into payload (sampling params, etc.) +- `--disable-ignore-eos`: pass through EOS behavior (varies by backend) +- `--warmup-requests N`: run warmup requests with short output first (default 1) +- `--flush-cache`: call `/flush_cache` (sglang) before main run +- `--profile`: call `/start_profile` and `/stop_profile` (requires server to enable profiling, e.g., `SGLANG_TORCH_PROFILER_DIR`) +- `--lora-name name1 name2 ...`: randomly pick one per request and pass to backend (e.g., `lora_path` for sglang) +- `--tokenize-prompt`: send integer IDs instead of text (currently supports `--backend sglang` only) + +### Authentication + +If your target endpoint requires OpenAI-style auth, set: + +```bash +export OPENAI_API_KEY=sk-...yourkey... +``` + +The script will add `Authorization: Bearer $OPENAI_API_KEY` automatically for OpenAI-compatible routes. + +### Metrics explained + +Printed after each run: + +- Request throughput (req/s) +- Input token throughput (tok/s) - includes both text and vision tokens +- Output token throughput (tok/s) +- Total token throughput (tok/s) - includes both text and vision tokens +- Total input text tokens and Total input vision tokens - per-modality breakdown +- Concurrency: aggregate time of all requests divided by wall time +- End-to-End Latency (ms): mean/median/std/p99 per-request total latency +- Time to First Token (TTFT, ms): mean/median/std/p99 for streaming mode +- Inter-Token Latency (ITL, ms): mean/median/std/p95/p99/max between tokens +- TPOT (ms): Token processing time after first token, i.e., `(latency - ttft)/(tokens-1)` +- Accept length (sglang-only, if available): speculative decoding accept length + +The script also retokenizes generated text with the configured tokenizer and reports "retokenized" counts. + +### JSONL output format + +When `--output-file` is set, one JSON object is appended per run. Base fields: + +- Arguments summary: backend, dataset, request_rate, max_concurrency, etc. +- Duration and totals: completed, total_input_tokens, total_output_tokens, retokenized totals +- Throughputs and latency statistics as printed in the console +- `accept_length` when available (sglang) + +With `--output-details`, an extended object also includes arrays: + +- `input_lens`, `output_lens` +- `ttfts`, `itls` (per request: ITL arrays) +- `generated_texts`, `errors` + +### End-to-end examples + +1) sglang native `/generate` (streaming): + +```bash +python3 -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 --port 30000 \ + --model meta-llama/Llama-3.1-8B-Instruct \ + --dataset-name random \ + --random-input-len 1024 --random-output-len 1024 --random-range-ratio 0.5 \ + --num-prompts 2000 \ + --request-rate 100 \ + --max-concurrency 512 \ + --output-file sglang_random.jsonl --output-details +``` + +2) OpenAI-compatible Completions (e.g., vLLM): + +```bash +python3 -m sglang.bench_serving \ + --backend vllm \ + --base-url http://127.0.0.1:8000 \ + --model meta-llama/Llama-3.1-8B-Instruct \ + --dataset-name sharegpt \ + --num-prompts 1000 \ + --sharegpt-output-len 256 +``` + +3) OpenAI-compatible Chat Completions (streaming): + +```bash +python3 -m sglang.bench_serving \ + --backend vllm-chat \ + --base-url http://127.0.0.1:8000 \ + --model meta-llama/Llama-3.1-8B-Instruct \ + --dataset-name random \ + --num-prompts 500 \ + --apply-chat-template +``` + +4) Images (VLM) with chat template: + +```bash +python3 -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 --port 30000 \ + --model your-vlm-model \ + --dataset-name image \ + --image-count 2 \ + --image-resolution 720p \ + --random-input-len 128 --random-output-len 256 \ + --num-prompts 200 \ + --apply-chat-template +``` + +4a) Images with custom resolution: + +```bash +python3 -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 --port 30000 \ + --model your-vlm-model \ + --dataset-name image \ + --image-count 1 \ + --image-resolution 512x768 \ + --random-input-len 64 --random-output-len 128 \ + --num-prompts 100 \ + --apply-chat-template +``` + +4b) 1080p images with PNG format and blank content: + +```bash +python3 -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 --port 30000 \ + --model your-vlm-model \ + --dataset-name image \ + --image-count 1 \ + --image-resolution 1080p \ + --image-format png \ + --image-content blank \ + --random-input-len 64 --random-output-len 128 \ + --num-prompts 100 \ + --apply-chat-template +``` + +5) Generated shared prefix (long system prompts + short questions): + +```bash +python3 -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 --port 30000 \ + --model meta-llama/Llama-3.1-8B-Instruct \ + --dataset-name generated-shared-prefix \ + --gsp-num-groups 64 --gsp-prompts-per-group 16 \ + --gsp-system-prompt-len 2048 --gsp-question-len 128 --gsp-output-len 256 \ + --num-prompts 1024 +``` + +6) Tokenized prompts (ids) for strict length control (sglang only): + +```bash +python3 -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 --port 30000 \ + --model meta-llama/Llama-3.1-8B-Instruct \ + --dataset-name random \ + --tokenize-prompt \ + --random-input-len 2048 --random-output-len 256 --random-range-ratio 0.2 +``` + +7) Profiling and cache flush (sglang): + +```bash +python3 -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 --port 30000 \ + --model meta-llama/Llama-3.1-8B-Instruct \ + --profile \ + --flush-cache +``` + +8) TensorRT-LLM streaming endpoint: + +```bash +python3 -m sglang.bench_serving \ + --backend trt \ + --base-url http://127.0.0.1:8000 \ + --model your-trt-llm-model \ + --dataset-name random \ + --num-prompts 100 \ + --disable-ignore-eos +``` + +9) Evaluating large-scale KVCache sharing with mooncake trace (sglang only): + +```bash +python3 -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 --port 30000 \ + --model mode-name \ + --dataset-name mooncake \ + --mooncake-slowdown-factor 1.0 \ + --mooncake-num-rounds 1000 \ + --mooncake-workload conversation|mooncake|agent|synthetic + --use-trace-timestamps true \ + --random-output-len 256 +``` + +### Troubleshooting + +- All requests failed: verify `--backend`, server URL/port, `--model`, and authentication. Check warmup errors printed by the script. +- Throughput seems too low: adjust `--request-rate` and `--max-concurrency`; verify server batch size/scheduling; ensure streaming is enabled if appropriate. +- Token counts look odd: prefer chat/instruct models with proper chat templates; otherwise tokenization of gibberish may be inconsistent. +- Image/MMMU datasets: ensure you installed extra deps (`pillow`, `datasets`, `pybase64`). +- Authentication errors (401/403): set `OPENAI_API_KEY` or disable auth on your server. + +### Notes + +- The script raises the file descriptor soft limit (`RLIMIT_NOFILE`) to help with many concurrent connections. +- For sglang, `/get_server_info` is queried post-run to report speculative decoding accept length when available. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/benchmark_and_profiling.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/benchmark_and_profiling.md new file mode 100644 index 00000000..728bcba3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/benchmark_and_profiling.md @@ -0,0 +1,456 @@ +# Benchmark and Profiling + +## Benchmark + +- Benchmark the latency of running a single static batch without a server. The arguments are the same as for `launch_server.py`. + Note that this is a simplified test script without a dynamic batching server, so it may run out of memory for a batch size that a real server can handle. A real server truncates the prefill into several batches, while this simplified script does not. + - Without a server (do not need to launch a server) + ```bash + python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --batch 32 --input-len 256 --output-len 32 + ``` + - With a server (please use `sglang.launch_server` to launch a server first and run the following command.) + ```bash + python -m sglang.bench_one_batch_server --base-url http://127.0.0.1:30000 --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --batch-size 32 --input-len 256 --output-len 32 + ``` + + +- Benchmark offline processing. This script will start an offline engine and run the benchmark. + + ```bash + python3 -m sglang.bench_offline_throughput --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --num-prompts 10 + ``` + +- Benchmark online serving. Please use `sglang.launch_server` to launch a server first and run the following command. + + ```bash + python3 -m sglang.bench_serving --backend sglang --num-prompt 10 + ``` + +## Profile with PyTorch Profiler + +[Pytorch Profiler](https://pytorch.org/tutorials/recipes/recipes/profiler_recipe.html) is a convenient basic tool to inspect kernel execution time, call stack, and kernel overlap and occupancy. + +### Profile a server with `sglang.bench_serving` + +```bash +# set trace path +export SGLANG_TORCH_PROFILER_DIR=/root/sglang/profile_log + +# start server +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct + +# send profiling request from client +python -m sglang.bench_serving --backend sglang --model meta-llama/Llama-3.1-8B-Instruct --num-prompts 10 --sharegpt-output-len 100 --profile +``` + +Please make sure that the `SGLANG_TORCH_PROFILER_DIR` should be set at both server and client side, otherwise the trace file cannot be generated correctly . A secure way will be setting `SGLANG_TORCH_PROFILER_DIR` in the `.*rc` file of shell (e.g. `~/.bashrc` for bash shells). + +For more details, please refer to [Bench Serving Guide](./bench_serving.md). + +### Profile In PD Disaggregation Mode + +When profiling in PD disaggregation mode, prefill and decode workers **must be profiled separately** due to torch profiler limitations. The `bench_serving` command provides dedicated options for this: + +#### Profile Prefill Workers + +```bash +# set trace path +export SGLANG_TORCH_PROFILER_DIR=/root/sglang/profile_log + +# start prefill and decode servers (see PD disaggregation docs for setup) +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disaggregation-mode prefill +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disaggregation-mode decode --port 30001 --base-gpu-id 1 + +# start router +python -m sglang_router.launch_router --pd-disaggregation --prefill http://127.0.0.1:30000 --decode http://127.0.0.1:30001 --host 0.0.0.0 --port 8000 + +# send profiling request targeting prefill workers +python -m sglang.bench_serving --backend sglang --model meta-llama/Llama-3.1-8B-Instruct --num-prompts 10 --sharegpt-output-len 100 --profile --pd-separated --profile-prefill-url http://127.0.0.1:30000 +``` + +#### Profile Decode Workers + +```bash +# send profiling request targeting decode workers +python -m sglang.bench_serving --backend sglang --model meta-llama/Llama-3.1-8B-Instruct --num-prompts 10 --sharegpt-output-len 100 --profile --pd-separated --profile-decode-url http://127.0.0.1:30001 +``` + +#### Important Notes + +- `--profile-prefill-url` and `--profile-decode-url` are **mutually exclusive** - you cannot profile both at the same time +- Both options support multiple worker URLs for multi-instance setups: + ```bash + # Profile multiple prefill workers + python -m sglang.bench_serving --backend sglang --model meta-llama/Llama-3.1-8B-Instruct --num-prompts 10 --profile --pd-separated --profile-prefill-url http://127.0.0.1:30000 http://127.0.0.1:30002 + + # Profile multiple decode workers + python -m sglang.bench_serving --backend sglang --model meta-llama/Llama-3.1-8B-Instruct --num-prompts 10 --profile --pd-separated --profile-decode-url http://127.0.0.1:30001 http://127.0.0.1:30003 + ``` +- Make sure `SGLANG_TORCH_PROFILER_DIR` is set on all worker nodes before starting the servers +- For more details on setting up PD disaggregation, see [PD Disaggregation Guide](../advanced_features/pd_disaggregation.md) + +### Profile a server with `sglang.bench_offline_throughput` +```bash +export SGLANG_TORCH_PROFILER_DIR=/root/sglang/profile_log + +# profile one batch with bench_one_batch.py +# batch size can be controlled with --batch argument +python3 -m sglang.bench_one_batch --model-path meta-llama/Llama-3.1-8B-Instruct --batch 32 --input-len 1024 --output-len 10 --profile + +# profile multiple batches with bench_offline_throughput.py +python -m sglang.bench_offline_throughput --model-path meta-llama/Llama-3.1-8B-Instruct --dataset-name random --num-prompts 10 --profile --mem-frac=0.8 +``` + +### Profile a server with `sglang.profiler` + +When the server is running (e.g., processing a decoding request), you can start live profiling immediately by sending a profile request to the server. + +You can do this by running `python3 -m sglang.profiler`. For example: + +``` +# Terminal 1: Send a generation request +python3 -m sglang.test.send_one + +# Terminal 2: Before the above request finishes, quickly launch the following command in a separate terminal. +# It will generate a profile of the above request for several decoding batches. +python3 -m sglang.profiler +``` + +You can also combine the above operations into a single command + +``` +python3 -m sglang.test.send_one --profile +``` + +### Profile a server with HTTP API endpoints + +SGLang provides HTTP API endpoints to control profiling on a running server. This allows you to start and stop profiling programmatically, which is useful for capturing specific workload patterns. + +#### Using `/start_profile` endpoint + +The `/start_profile` endpoint starts profiling on the server. You can control when profiling begins and how long it runs using the following parameters: + +**Basic usage:** + +```bash +# Start profiling immediately for 10 steps +curl -X POST http://127.0.0.1:30000/start_profile \ + -H "Content-Type: application/json" \ + -d '{ + "num_steps": 10 + }' +``` + +**Parameters:** + +- `output_dir` (optional): Directory where profile traces will be saved. If not specified, uses `SGLANG_TORCH_PROFILER_DIR` environment variable, or `/tmp` as the default +- `num_steps` (optional): Number of steps to profile. If not specified, profiling continues until manually stopped with `/end_profile` +- `start_step` (optional): Step number at which to start profiling (inclusive). Useful for skipping warmup iterations +- `activities` (optional): List of activities to profile, e.g., `["CPU", "GPU"]`. Default is `["CPU", "GPU"]` +- `merge_profiles` (optional): Whether to merge distributed traces. Default is `false` + +**Note on step ranges:** Profiling starts at `start_step` (inclusive) and continues for `num_steps` iterations. For example, with `start_step=3` and `num_steps=10`, profiling captures steps 3, 4, 5, 6, 7, 8, 9, 10, 11, and 12 (10 steps total, starting from step 3). + +**Advanced usage with `start_step`:** + +```bash +# Wait 5 steps (warmup), then profile for 10 steps +curl -X POST http://127.0.0.1:30000/start_profile \ + -H "Content-Type: application/json" \ + -d '{ + "output_dir": "/tmp/profiles", + "start_step": 5, + "num_steps": 10, + "activities": ["CPU", "GPU"] + }' +``` + +**Continuous profiling (manual stop):** + +```bash +# Start profiling without num_steps - must manually stop with /end_profile +curl -X POST http://127.0.0.1:30000/start_profile +``` + +#### Using `/end_profile` endpoint + +The `/end_profile` endpoint stops an ongoing profiling session and saves the trace file. + +```bash +# Stop profiling and save traces +curl -X POST http://127.0.0.1:30000/end_profile +``` + +This is only needed when you start profiling without specifying `num_steps`. If `num_steps` is specified, profiling will automatically stop after that many steps. + +#### Example workflow + +```bash +# Terminal 1: Start the server +export SGLANG_TORCH_PROFILER_DIR=/tmp/profiles +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct + +# Terminal 2: Start continuous profiling +curl -X POST http://127.0.0.1:30000/start_profile \ + -H "Content-Type: application/json" \ + -d '{ + "start_step": 3 + }' + +# Terminal 3: Send requests to generate load +python -m sglang.bench_serving --backend sglang --num-prompts 100 + +# Terminal 2: Stop profiling when done +curl -X POST http://127.0.0.1:30000/end_profile +``` + +### Profiler Trace Merger for Distributed Traces + +SGLang now supports automatic merging of profiling traces from distributed setups with multiple parallelism types (TP, DP, PP, EP). This feature is particularly useful for analyzing performance across distributed runs. + +#### Multi-Node Profiling and Shared Storage Considerations + +Single-node profiler output merging is completely supported. When profiling in distributed environments spanning multiple nodes, shared storage (e.g., NFS, Lustre) should be accessible by all nodes for the output directory to enable merging of trace files. + +If there is no shared storage accessible across nodes, automatic merging of trace files during profiling is not supported directly as of now. + +#### HTTP API Usage + +```bash +# Start profiling with automatic trace merging enabled +curl -X POST /start_profile \ + -H "Content-Type: application/json" \ + -d '{ + "output_dir": "/tmp/profiles", # where to store profile traces + "num_steps": 10, + "activities": ["CPU", "GPU"], + "merge_profiles": true # optional argument to merge profile traces (default=False) + }' +``` + +#### Command Line Usage + +```bash +# Start profiling with merge enabled +python -m sglang.profiler \ + --num-steps 10 \ + --cpu \ + --gpu \ + --output-dir /tmp/profiles \ + --merge-profiles # optional argument to merge profile traces (default=False) +``` + +#### Output Files + +The profile merger generates: +- Individual rank trace files: `{profile_id}-TP-{tp}-DP-{dp}-PP-{pp}-EP-{ep}.trace.json.gz` +- Merged trace file: `merged-{profile_id}.trace.json.gz` + +### Possible PyTorch bugs +If in any cases you encounter the following error (for example, using qwen 2.5 VL): +```bash +RuntimeError: !stack.empty() INTERNAL ASSERT FAILED at "/pytorch/torch/csrc/autograd/profiler_python.cpp":983, please report a bug to PyTorch. Python replay stack is empty. +``` +This is likely a PyTorch Bug reported in [Bug: vLLM Profiler](https://github.com/vllm-project/vllm/issues/18240) and [Bug: torch.profiler.profile](https://github.com/pytorch/pytorch/issues/101632). As a workaround, you may disable `with_stack` with an environment variable such as follows: +```bash +export SGLANG_PROFILE_WITH_STACK=False +python -m sglang.bench_offline_throughput --model-path meta-llama/Llama-3.1-8B-Instruct --dataset-name random --num-prompts 10 --profile --mem-frac=0.8 +``` + +### View traces + +Trace files can be loaded and visualized from: + +1. https://ui.perfetto.dev/ (any browser) +2. chrome://tracing (Chrome browser only) + +If browser cannot open trace file due to its large size, +client can generate a small trace file (<100MB) by controlling number of prompts and lengths of prompt outputs. +For example, when profiling a server, + +```bash +python -m sglang.bench_serving --backend sglang --model meta-llama/Llama-3.1-8B-Instruct --num-prompts 2 --sharegpt-output-len 100 --profile +``` + +This command sets the number of prompts to 2 with `--num-prompts` argument and limits the length of output sequences to 100 with `--sharegpt-output-len` argument, which can generate a small trace file for browser to open smoothly. + +Additionally, if you want to locate the SGLang Python source code through the cuda kernel in Trace, you need to disable CUDA Graph when starting the service. This can be done by using the `--disable-cuda-graph` parameter in the command to start the service. + +## Profile with Nsight + +[Nsight systems](https://docs.nvidia.com/nsight-systems/) is an advanced tool that exposes more profiling details, such as register and shared memory usage, annotated code regions and low-level CUDA APIs and events. + +1. Prerequisite: + + Install using apt, or run inside a [NVIDIA Docker container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch/tags) or [SGLang Docker container](https://github.com/sgl-project/sglang/tree/main/docker). + + ```bash + # install nsys + # https://docs.nvidia.com/nsight-systems/InstallationGuide/index.html + apt update + apt install -y --no-install-recommends gnupg + echo "deb http://developer.download.nvidia.com/devtools/repos/ubuntu$(source /etc/lsb-release; echo "$DISTRIB_RELEASE" | tr -d .)/$(dpkg --print-architecture) /" | tee /etc/apt/sources.list.d/nvidia-devtools.list + apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub + apt update + apt install nsight-systems-cli + ``` + +2. To profile a single batch, use + + ```bash + nsys profile --trace-fork-before-exec=true --cuda-graph-trace=node python3 -m sglang.bench_one_batch --model meta-llama/Meta-Llama-3-8B --batch-size 64 --input-len 512 + ``` + +3. To profile a server, e.g. + + ```bash + # launch the server, set the delay and duration times according to needs + # after the duration time has been used up, server will be killed by nsys + + nsys profile --trace-fork-before-exec=true --cuda-graph-trace=node -o sglang.out --delay 60 --duration 70 python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + + # client + python3 -m sglang.bench_serving --backend sglang --num-prompts 1000 --dataset-name random --random-input 1024 --random-output 512 + ``` + + In practice, we recommend users to set `--duration` argument to a large value. Whenever user wants the server to stop profiling. Firstly run: + + ```bash + nsys sessions list + ``` + + to get the session id in the form of `profile-XXXXX`, then run: + + ```bash + nsys stop --session=profile-XXXXX + ``` + + to manually kill the profiler and generate `nsys-rep` files instantly. + +4. Use NVTX to annotate code regions, e.g. to see their execution time. + + ```bash + # install nvtx + pip install nvtx + ``` + + ```python + # code snippets + import nvtx + with nvtx.annotate("description", color="color"): + # some critical code + ``` + +### Layer-wise NVTX Profiling with Nsight Systems + +SGLang provides built-in layerwise NVTX annotations that can be combined with the CUDA Profiler for detailed per-layer profiling in Nsight Systems. This is particularly useful for identifying performance bottlenecks at the layer level. + +#### Using `--enable-layerwise-nvtx-marker` with Nsight Systems and `/start_profile` + +The `--enable-layerwise-nvtx-marker` flag automatically adds NVTX markers to every layer in your model. This is particularly powerful when combined with Nsight Systems profiling to see detailed per-layer performance. + +**Method 1: Using `/start_profile` with CUDA_PROFILER (for programmatic control)** + +This method allows you to control exactly when profiling starts/stops via HTTP API while Nsight Systems is running. + +1. Launch the server with layerwise NVTX enabled under Nsight Systems: + + ```bash + # Terminal 1: Start server with nsys and capture-range option + nsys profile --trace-fork-before-exec=true \ + --cuda-graph-trace=node \ + --capture-range=cudaProfilerApi \ + --capture-range-end=stop \ + -o layerwise_profile \ + python -m sglang.launch_server \ + --model-path meta-llama/Llama-3.1-8B-Instruct \ + --enable-layerwise-nvtx-marker \ + --disable-cuda-graph + ``` + + Note: NVTX markers are not emitted for kernel launches captured by CUDA graphs. Use `--disable-cuda-graph` to ensure all layerwise NVTX markers are emitted in the trace. + +2. In another terminal, control profiling via `/start_profile` with `CUDA_PROFILER` activity: + + ```bash + # Terminal 2: Wait for server to be ready, then start CUDA profiling + # Wait 3 steps for warmup, then profile for 10 steps + curl -X POST http://127.0.0.1:30000/start_profile \ + -H "Content-Type: application/json" \ + -d '{ + "start_step": 3, + "num_steps": 10, + "activities": ["CUDA_PROFILER"] + }' + ``` + +3. Send requests to generate load: + + ```bash + # Terminal 3: Generate workload + python -m sglang.bench_serving --backend sglang --num-prompts 100 + ``` + +4. Profiling will automatically stop after 10 steps (due to `num_steps: 10`). If you hadn't specified `num_steps`, you would need to manually stop it: + + ```bash + # Terminal 2: Only needed if num_steps was not specified + curl -X POST http://127.0.0.1:30000/end_profile + ``` + +The `--capture-range=cudaProfilerApi` option tells Nsight Systems to only capture data between `cudaProfilerStart()` and `cudaProfilerStop()` calls (triggered by `/start_profile` and `/end_profile`), reducing overhead and file size. The `start_step` parameter skips the first 3 steps to avoid capturing warmup overhead. + +**Method 2: Simpler approach without `/start_profile` API** + +For simpler use cases where you don't need fine-grained control over profiling start/stop, you can profile with Nsight Systems capturing the entire workload: + +```bash +# Terminal 1: Start server with layerwise NVTX +# Note: --disable-cuda-graph ensures all NVTX markers are emitted +python -m sglang.launch_server \ + --model-path meta-llama/Llama-3.1-8B-Instruct \ + --enable-layerwise-nvtx-marker \ + --disable-cuda-graph + +# Terminal 2: Profile the benchmarking client +nsys profile --trace-fork-before-exec=true \ + --cuda-graph-trace=node \ + -o layerwise_profile \ + python -m sglang.bench_serving --backend sglang --num-prompts 10 +``` + +This approach profiles the entire client execution, including all server interactions. The layerwise NVTX markers will be visible in the Nsight Systems timeline. + +**Viewing the profiling results:** + +Open the generated `.qdrep` file with Nsight Systems: + +```bash +nsys-ui layerwise_profile.qdrep +``` + +In the Nsight Systems GUI, you'll see: +- **NVTX ranges**: Each layer appears as a labeled range in the timeline with detailed information in the marker metadata +- **CUDA kernels**: All GPU kernels are shown alongside the layer annotations +- **Layer hierarchy**: The full module path (e.g., `meta-llama/Meta-Llama-3.1-8B-Instruct.model.layers.0.self_attn.qkv_proj`) helps identify specific layers. The prefix uses the full model path from `--model-path`. +- **Tensor shapes**: Input/output dimensions and parameter shapes are included in the NVTX marker data + +**Benefits of layerwise NVTX profiling:** + +- **Granular visibility**: See exactly which layers are taking the most time +- **Memory tracking**: Identify layers with large memory allocations +- **Bottleneck identification**: Quickly locate inefficient operations +- **Communication overhead**: In multi-GPU setups, see per-layer communication costs +- **Development debugging**: Validate that model architecture changes have the expected performance impact + +## Other tips + +1. You can benchmark a model using dummy weights by only providing the config.json file. This allows for quick testing of model variants without training. To do so, add `--load-format dummy` to the above commands and then you only need a correct `config.json` under the checkpoint folder. +2. You can benchmark a model with modified configs (e.g., less layers) by using `--json-model-override-args`. For example, you can benchmark a model with only 2 layers and 2 kv heads using: + + ```bash + python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --batch 32 --input-len 256 --output-len 32 --load-format dummy --json-model-override-args '{"num_hidden_layers": 1, "num_key_value_heads": 1}' + ``` + +3. You can use `--python-backtrace=cuda` to see python call stack for all CUDA kernels, as in PyTorch Profiler. (Caveat: this can cause inaccurately long kernel runtimes for CUDA event based timing) +4. For more arguments see [Nsight Systems User Guide](https://docs.nvidia.com/nsight-systems/UserGuide/index.html). diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/contribution_guide.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/contribution_guide.md new file mode 100644 index 00000000..6a5056ce --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/contribution_guide.md @@ -0,0 +1,140 @@ +# Contribution Guide + +Welcome to **SGLang**! We appreciate your interest in contributing. This guide provides a concise overview of how to set up your environment, run tests, build documentation, and open a Pull Request (PR). Whether you’re fixing a small bug or developing a major feature, we encourage following these steps for a smooth contribution process. + +## Install SGLang from Source + +### Fork and clone the repository + +**Note**: New contributors do **not** have the write permission to push to the official SGLang repo. Please fork the repository under your GitHub account, then clone your fork locally. + +```bash +git clone https://github.com//sglang.git +``` + +### Build from source + +Refer to [Install SGLang from Source](../get_started/install.md#method-2-from-source). + +## Format code with pre-commit + +We use [pre-commit](https://pre-commit.com/) to maintain consistent code style checks. Before pushing your changes, please run: + +```bash +pip3 install pre-commit +pre-commit install +pre-commit run --all-files +``` + +- **`pre-commit run --all-files`** manually runs all configured checks, applying fixes if possible. If it fails the first time, re-run it to ensure lint errors are fully resolved. Make sure your code passes all checks **before** creating a Pull Request. +- **Do not commit** directly to the `main` branch. Always create a new branch (e.g., `feature/my-new-feature`), push your changes, and open a PR from that branch. + +## Run and add unit tests + +If you add a new feature or fix a bug, please add corresponding unit tests to ensure coverage and prevent regression. +SGLang uses Python's built-in [unittest](https://docs.python.org/3/library/unittest.html) framework. +For detailed instructions on running tests and integrating them into CI, refer to [test/README.md](https://github.com/sgl-project/sglang/tree/main/test/README.md). + +## Write documentations + +We recommend new contributors start from writing documentation, which helps you quickly understand SGLang codebase. +For more details, please refer to [docs/README.md](https://github.com/sgl-project/sglang/tree/main/docs/README.md). + +## Test the accuracy +If your code changes the model output, please run the accuracy tests. A quick sanity check is the few-shot GSM8K. + +``` +# Launch a server +python3 -m sglang.launch_server --model Qwen/Qwen2-7B-Instruct + +# Evaluate +python3 -m sglang.test.few_shot_gsm8k --num-questions 200 +``` + +Please note that the above script is primarily a sanity check, not a rigorous accuracy or speed test. +This test can have significant variance (1%–5%) in accuracy due to batching and the non-deterministic nature of the inference engine. +Also, do not rely on the "Latency/Output throughput" from this script, as it is not a proper speed test. + +GSM8K is too easy for state-of-the-art models nowadays. Please try your own more challenging accuracy tests. +You can find additional accuracy eval examples in: +- [test_eval_accuracy_large.py](https://github.com/sgl-project/sglang/blob/main/test/srt/test_eval_accuracy_large.py) +- [test_gpt_oss_1gpu.py](https://github.com/sgl-project/sglang/blob/main/test/srt/test_gpt_oss_1gpu.py) + +## Benchmark the speed +Refer to [Benchmark and Profiling](../developer_guide/benchmark_and_profiling.md). + +## Requesting a review for merge +You can follow the pull request merge process described in [MAINTAINER.md](https://github.com/sgl-project/sglang/blob/main/.github/MAINTAINER.md). +You will need to work with the Merge Oncall, Codeowner, and other reviewers to get their approvals. +Then your PR can be merged. + +## How to Trigger CI Tests + +We have a lot of open PRs but limited CI machines, so only top and trusted contributors have permission to trigger CI tests. +Users with permission are listed in the [CI_PERMISSIONS.json](https://github.com/sgl-project/sglang/blob/main/.github/CI_PERMISSIONS.json) + +For CI to run on a pull request, it must have the "run-ci" label. Authorized users can add the label or rerun failed tests by commenting on the PR with one of these commands: + +- `/tag-run-ci-label`: Adds the "run-ci" label. Every future commit will trigger CI. +- `/rerun-failed-ci`: Reruns the failed or flaky tests from the most recent commit. +- `/tag-and-rerun-ci`: A single command that performs both `/tag-run-ci-label` and `/rerun-failed-ci`. + +If you have permission, the [Slash Command Handler](https://github.com/sgl-project/sglang/actions/workflows/slash-command-handler.yml) will run your command and react with a 👍 to your comment. It may take up to a few minutes for the reaction to appear. Here’s a usage [example](https://github.com/sgl-project/sglang/pull/14253#issuecomment-3599509302). + +To avoid spamming a PR with too many `/rerun-failed-ci` comments, you can also trigger the command by editing an existing comment and adding any suffix (e.g., `/rerun-failed-ci try again`). + +If you don’t have permission, please ask maintainers to trigger CI for you. + +### CI rate limits + +We apply CI rate limits to prevent abuse and ensure fair usage of our CI resources. + +Each CI workflow has a default limit defined in its workflow configuration file. For example, in [pr-gate.yml](https://github.com/sgl-project/sglang/blob/main/.github/workflows/pr-gate.yml), the default cooldown period is 120 minutes, and each workflow can override it via the `cool-down-minutes` input parameter: + +```yaml +cool-down-minutes: + description: "Default cooldown period in minutes; 0 disables rate limiting" + type: number + default: 120 +``` + +Users listed in [CI_PERMISSIONS.json](https://github.com/sgl-project/sglang/blob/main/.github/CI_PERMISSIONS.json) may have a per-user cooldown interval. In practice, we use the minimum of the workflow’s default window and the user-specific interval. + + +## Code style guidance +- Avoid code duplication. If the same code snippet (more than five lines) appears multiple times, extract it into a shared function. +- Minimize device synchronization. Reduce expensive CPU-GPU synchronization operations, such as `tensor.item()` or `tensor.cpu()`, whenever possible. Use vectorized code. +- Prioritize extreme efficiency. SGLang is a runtime, and most of your code runs on the critical path for every request. Optimize all minor overheads as much as possible, especially in the model forward code. + - A common pattern is some runtime checks in the model forward pass (e.g., [this](https://github.com/sgl-project/sglang/blob/f1b0eda55c2c4838e8ab90a0fac7fb1e3d7064ab/python/sglang/srt/models/deepseek_v2.py#L486-L491)). These are very likely the same for every layer. Please cache the result as a single boolean value whenever possible. +- Make functions as pure as possible. Avoid in-place modification of arguments. +- Keep files concise. If a file exceeds 2,000 lines of code, split it into multiple smaller files. (e.g., `scheduler.py`, `scheduler_output_processor_mixin.py`) +- Keep tests run fast. + - If a single test file run longer than 500 seconds, split it into multiple smaller files (e.g., `test_eagle_infer_a.py`, `test_eagle_infer_b.py`). + - If a single job in a github workflow runs longer than 30 mins, split it into smaller jobs/steps. + - Reuse server launches in your unit tests to make tests run faster. +- When supporting new hardware or features, follow these guidelines: + - Do not drastically change existing code. + - Always prefer new files to introduce specific components for your new hardware (e.g., `allocator_ascend.py`). + - If you write multiple if/else blocks for new features, ensure the common path (e.g., NVIDIA hardware or the existing code path) is the first branch. + +## How to update sgl-kernel +Since sglang and sgl-kernel are separate Python packages, our current GitHub CI infrastructure does not support updating a kernel and using it immediately within the same pull request (PR). +To add a new kernel or modify an existing one in the sgl-kernel package, you must use multiple PRs. + +Follow these steps: + +1. Submit a PR to update the sgl-kernel source code without using it in sglang python package (e.g., [#8884](https://github.com/sgl-project/sglang/pull/8884/files)). +2. Bump the version of sgl-kernel (e.g., [#9220](https://github.com/sgl-project/sglang/pull/9220/files)). + - Once merged, this will trigger an automatic release of the sgl-kernel wheel to PyPI. + - If not urgent, you can wait for other people to release the wheel. A new version will typically be released within one week. +3. Apply the changes: + - Update the sgl-kernel version in `sglang/python/pyproject.toml` to use the modified kernels. + - Update the related caller code in the sglang to use the new kernel. + +## Tips for newcomers + +If you want to contribute but don’t have a specific idea in mind, pick issues labeled [“good first issue” or “help wanted”](https://github.com/sgl-project/sglang/issues?q=is%3Aissue+label%3A%22good+first+issue%22%2C%22help+wanted%22). These tasks typically have lower complexity and provide an excellent introduction to the codebase. Also check out this [code walk-through](https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/tree/main/sglang/code-walk-through) for a deeper look into SGLang’s workflow. + +If you have any questions or want to start a discussion, please feel free to ask in our [Slack channel](https://slack.sglang.ai). + +Thank you for your interest in SGLang. Happy coding! diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/development_guide_using_docker.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/development_guide_using_docker.md new file mode 100644 index 00000000..e3894790 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/development_guide_using_docker.md @@ -0,0 +1,108 @@ +# Development Guide Using Docker + +## Setup VSCode on a Remote Host +(Optional - you can skip this step if you plan to run sglang dev container locally) + +1. In the remote host, download `code` from [Https://code.visualstudio.com/docs/?dv=linux64cli](https://code.visualstudio.com/download) and run `code tunnel` in a shell. + +Example +```bash +wget https://vscode.download.prss.microsoft.com/dbazure/download/stable/fabdb6a30b49f79a7aba0f2ad9df9b399473380f/vscode_cli_alpine_x64_cli.tar.gz +tar xf vscode_cli_alpine_x64_cli.tar.gz + +# https://code.visualstudio.com/docs/remote/tunnels +./code tunnel +``` + +2. In your local machine, press F1 in VSCode and choose "Remote Tunnels: Connect to Tunnel". + +## Setup Docker Container + +### Option 1. Use the default dev container automatically from VSCode +There is a `.devcontainer` folder in the sglang repository root folder to allow VSCode to automatically start up within dev container. You can read more about this VSCode extension in VSCode official document [Developing inside a Container](https://code.visualstudio.com/docs/devcontainers/containers). +![image](https://github.com/user-attachments/assets/6a245da8-2d4d-4ea8-8db1-5a05b3a66f6d) +(*Figure 1: Diagram from VSCode official documentation [Developing inside a Container](https://code.visualstudio.com/docs/devcontainers/containers).*) + +To enable this, you only need to: +1. Start Visual Studio Code and install [VSCode dev container extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers). +2. Press F1, type and choose "Dev Container: Open Folder in Container. +3. Input the `sglang` local repo path in your machine and press enter. + +The first time you open it in dev container might take longer due to docker pull and build. Once it's successful, you should set on your status bar at the bottom left displaying that you are in a dev container: + +![image](https://github.com/user-attachments/assets/650bba0b-c023-455f-91f9-ab357340106b) + +Now when you run `sglang.launch_server` in the VSCode terminal or start debugging using F5, sglang server will be started in the dev container with all your local changes applied automatically: + +![image](https://github.com/user-attachments/assets/748c85ba-7f8c-465e-8599-2bf7a8dde895) + + +### Option 2. Start up containers manually (advanced) + +The following startup command is an example for internal development by the SGLang team. You can **modify or add directory mappings as needed**, especially for model weight downloads, to prevent repeated downloads by different Docker containers. + +❗️ **Note on RDMA** + + 1. `--network host` and `--privileged` are required by RDMA. If you don't need RDMA, you can remove them but keeping them there does not harm. Thus, we enable these two flags by default in the commands below. + 2. You may need to set `NCCL_IB_GID_INDEX` if you are using RoCE, for example: `export NCCL_IB_GID_INDEX=3`. + +```bash +# Change the name to yours +docker run -itd --shm-size 32g --gpus all -v --ipc=host --network=host --privileged --name sglang_dev lmsysorg/sglang:dev /bin/zsh +docker exec -it sglang_dev /bin/zsh +``` +Some useful volumes to mount are: +1. **Huggingface model cache**: mounting model cache can avoid re-download every time docker restarts. Default location on Linux is `~/.cache/huggingface/`. +2. **SGLang repository**: code changes in the SGLang local repository will be automatically synced to the .devcontainer. + +Example 1: Monting local cache folder `/opt/dlami/nvme/.cache` but not the SGLang repo. Use this when you prefer to manually transfer local code changes to the devcontainer. +```bash +docker run -itd --shm-size 32g --gpus all -v /opt/dlami/nvme/.cache:/root/.cache --ipc=host --network=host --privileged --name sglang_zhyncs lmsysorg/sglang:dev /bin/zsh +docker exec -it sglang_zhyncs /bin/zsh +``` +Example 2: Mounting both HuggingFace cache and local SGLang repo. Local code changes are automatically synced to the devcontainer as the SGLang is installed in editable mode in the dev image. +```bash +docker run -itd --shm-size 32g --gpus all -v $HOME/.cache/huggingface/:/root/.cache/huggingface -v $HOME/src/sglang:/sgl-workspace/sglang --ipc=host --network=host --privileged --name sglang_zhyncs lmsysorg/sglang:dev /bin/zsh +docker exec -it sglang_zhyncs /bin/zsh +``` +## Debug SGLang with VSCode Debugger +1. (Create if not exist) open `launch.json` in VSCode. +2. Add the following config and save. Please note that you can edit the script as needed to apply different parameters or debug a different program (e.g. benchmark script). + ```JSON + { + "version": "0.2.0", + "configurations": [ + { + "name": "Python Debugger: launch_server", + "type": "debugpy", + "request": "launch", + "module": "sglang.launch_server", + "console": "integratedTerminal", + "args": [ + "--model-path", "meta-llama/Llama-3.2-1B", + "--host", "0.0.0.0", + "--port", "30000", + "--trust-remote-code", + ], + "justMyCode": false + } + ] + } + ``` + +3. Press "F5" to start. VSCode debugger will ensure that the program will pause at the breakpoints even if the program is running at remote SSH/Tunnel host + dev container. + +## Profile + +```bash +# Change batch size, input, output and add `disable-cuda-graph` (for easier analysis) +# e.g. DeepSeek V3 +nsys profile -o deepseek_v3 python3 -m sglang.bench_one_batch --batch-size 1 --input 128 --output 256 --model deepseek-ai/DeepSeek-V3 --trust-remote-code --tp 8 --disable-cuda-graph +``` + +## Evaluation + +```bash +# e.g. gsm8k 8 shot +python3 benchmark/gsm8k/bench_sglang.py --num-questions 2000 --parallel 2000 --num-shots 8 +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/release_process.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/release_process.md new file mode 100644 index 00000000..e817a256 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/release_process.md @@ -0,0 +1,18 @@ +# PyPI Package Release Process + +## Update the version in code +Update the package version in `python/pyproject.toml` and `python/sglang/__init__.py`. + +## Upload the PyPI package + +``` +pip install build twine +``` + +``` +cd python +bash upload_pypi.sh +``` + +## Make a release in GitHub +Make a new release https://github.com/sgl-project/sglang/releases/new. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/setup_github_runner.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/setup_github_runner.md new file mode 100644 index 00000000..3ca9627f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/developer_guide/setup_github_runner.md @@ -0,0 +1,51 @@ +# Set Up Self-Hosted Runners for GitHub Action + +## Add a Runner + +### Step 1: Start a docker container. + +**You can mount a folder for the shared huggingface model weights cache. ** +The command below uses `/tmp/huggingface` as an example. + +``` +docker pull nvidia/cuda:12.9.1-devel-ubuntu22.04 +# Nvidia +docker run --shm-size 128g -it -v /tmp/huggingface:/hf_home --gpus all nvidia/cuda:12.9.1-devel-ubuntu22.04 /bin/bash +# AMD +docker run --rm --device=/dev/kfd --device=/dev/dri --group-add video --shm-size 128g -it -v /tmp/huggingface:/hf_home lmsysorg/sglang:v0.5.0rc1-rocm630 /bin/bash +# AMD just the last 2 GPUs +docker run --rm --device=/dev/kfd --device=/dev/dri/renderD176 --device=/dev/dri/renderD184 --group-add video --shm-size 128g -it -v /tmp/huggingface:/hf_home lmsysorg/sglang:v0.5.0rc1-rocm630 /bin/bash +``` + +### Step 2: Configure the runner by `config.sh` + +Run these commands inside the container. + +``` +apt update && apt install -y curl python3-pip git +pip install --upgrade pip +export RUNNER_ALLOW_RUNASROOT=1 +``` + +Then follow https://github.com/sgl-project/sglang/settings/actions/runners/new?arch=x64&os=linux to run `config.sh` + +**Notes** +- Do not need to specify the runner group +- Give it a name (e.g., `test-sgl-gpu-0`) and some labels (e.g., `1-gpu-runner`). The labels can be edited later in Github Settings. +- Do not need to change the work folder. + +### Step 3: Run the runner by `run.sh` + +- Set up environment variables +``` +export HF_HOME=/hf_home +export SGLANG_IS_IN_CI=true +export HF_TOKEN=hf_xxx +export OPENAI_API_KEY=sk-xxx +export CUDA_VISIBLE_DEVICES=0 +``` + +- Run it forever +``` +while true; do ./run.sh; echo "Restarting..."; sleep 2; done +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/get_started/install.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/get_started/install.md new file mode 100644 index 00000000..a4911c41 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/get_started/install.md @@ -0,0 +1,185 @@ +# Install SGLang + +You can install SGLang using one of the methods below. + +This page primarily applies to common NVIDIA GPU platforms. +For other or newer platforms, please refer to the dedicated pages for [AMD GPUs](../platforms/amd_gpu.md), [Intel Xeon CPUs](../platforms/cpu_server.md), [TPU](../platforms/tpu.md), [NVIDIA DGX Spark](https://lmsys.org/blog/2025-11-03-gpt-oss-on-nvidia-dgx-spark/), [NVIDIA Jetson](../platforms/nvidia_jetson.md), [Ascend NPUs](../platforms/ascend_npu.md), and [Intel XPU](../platforms/xpu.md). + +## Method 1: With pip or uv + +It is recommended to use uv for faster installation: + +```bash +pip install --upgrade pip +pip install uv +uv pip install "sglang" --prerelease=allow +``` + +**Quick fixes to common problems** + +- If you encounter `OSError: CUDA_HOME environment variable is not set`. Please set it to your CUDA install root with either of the following solutions: + 1. Use `export CUDA_HOME=/usr/local/cuda-` to set the `CUDA_HOME` environment variable. + 2. Install FlashInfer first following [FlashInfer installation doc](https://docs.flashinfer.ai/installation.html), then install SGLang as described above. + +## Method 2: From source + +```bash +# Use the last release branch +git clone -b v0.5.6 https://github.com/sgl-project/sglang.git +cd sglang + +# Install the python packages +pip install --upgrade pip +pip install -e "python" +``` + +**Quick fixes to common problems** + +- If you want to develop SGLang, you can try the dev docker image. Please refer to [setup docker container](../developer_guide/development_guide_using_docker.md#setup-docker-container). The docker image is `lmsysorg/sglang:dev`. + +## Method 3: Using docker + +The docker images are available on Docker Hub at [lmsysorg/sglang](https://hub.docker.com/r/lmsysorg/sglang/tags), built from [Dockerfile](https://github.com/sgl-project/sglang/tree/main/docker). +Replace `` below with your huggingface hub [token](https://huggingface.co/docs/hub/en/security-tokens). + +```bash +docker run --gpus all \ + --shm-size 32g \ + -p 30000:30000 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --env "HF_TOKEN=" \ + --ipc=host \ + lmsysorg/sglang:latest \ + python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --host 0.0.0.0 --port 30000 +``` + +You can also find the nightly docker images [here](https://hub.docker.com/r/lmsysorg/sglang/tags?name=nightly). + +## Method 4: Using Kubernetes + +Please check out [OME](https://github.com/sgl-project/ome), a Kubernetes operator for enterprise-grade management and serving of large language models (LLMs). + +
+More + +1. Option 1: For single node serving (typically when the model size fits into GPUs on one node) + + Execute command `kubectl apply -f docker/k8s-sglang-service.yaml`, to create k8s deployment and service, with llama-31-8b as example. + +2. Option 2: For multi-node serving (usually when a large model requires more than one GPU node, such as `DeepSeek-R1`) + + Modify the LLM model path and arguments as necessary, then execute command `kubectl apply -f docker/k8s-sglang-distributed-sts.yaml`, to create two nodes k8s statefulset and serving service. + +
+ +## Method 5: Using docker compose + +
+More + +> This method is recommended if you plan to serve it as a service. +> A better approach is to use the [k8s-sglang-service.yaml](https://github.com/sgl-project/sglang/blob/main/docker/k8s-sglang-service.yaml). + +1. Copy the [compose.yml](https://github.com/sgl-project/sglang/blob/main/docker/compose.yaml) to your local machine +2. Execute the command `docker compose up -d` in your terminal. +
+ +## Method 6: Run on Kubernetes or Clouds with SkyPilot + +
+More + +To deploy on Kubernetes or 12+ clouds, you can use [SkyPilot](https://github.com/skypilot-org/skypilot). + +1. Install SkyPilot and set up Kubernetes cluster or cloud access: see [SkyPilot's documentation](https://skypilot.readthedocs.io/en/latest/getting-started/installation.html). +2. Deploy on your own infra with a single command and get the HTTP API endpoint: +
+SkyPilot YAML: sglang.yaml + +```yaml +# sglang.yaml +envs: + HF_TOKEN: null + +resources: + image_id: docker:lmsysorg/sglang:latest + accelerators: A100 + ports: 30000 + +run: | + conda deactivate + python3 -m sglang.launch_server \ + --model-path meta-llama/Llama-3.1-8B-Instruct \ + --host 0.0.0.0 \ + --port 30000 +``` + +
+ +```bash +# Deploy on any cloud or Kubernetes cluster. Use --cloud to select a specific cloud provider. +HF_TOKEN= sky launch -c sglang --env HF_TOKEN sglang.yaml + +# Get the HTTP API endpoint +sky status --endpoint 30000 sglang +``` + +3. To further scale up your deployment with autoscaling and failure recovery, check out the [SkyServe + SGLang guide](https://github.com/skypilot-org/skypilot/tree/master/llm/sglang#serving-llama-2-with-sglang-for-more-traffic-using-skyserve). + +
+ +## Method 7: Run on AWS SageMaker + +
+More + +To deploy on SGLang on AWS SageMaker, check out [AWS SageMaker Inference](https://aws.amazon.com/sagemaker/ai/deploy) + +To host a model with your own container, follow the following steps: + +1. Build a docker container with [sagemaker.Dockerfile](https://github.com/sgl-project/sglang/blob/main/docker/sagemaker.Dockerfile) alongside the [serve](https://github.com/sgl-project/sglang/blob/main/docker/serve) script. +2. Push your container onto AWS ECR. + +
+Dockerfile Build Script: build-and-push.sh + +```bash +#!/bin/bash +AWS_ACCOUNT="" +AWS_REGION="" +REPOSITORY_NAME="" +IMAGE_TAG="" + +ECR_REGISTRY="${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com" +IMAGE_URI="${ECR_REGISTRY}/${REPOSITORY_NAME}:${IMAGE_TAG}" + +echo "Starting build and push process..." + +# Login to ECR +echo "Logging into ECR..." +aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${ECR_REGISTRY} + +# Build the image +echo "Building Docker image..." +docker build -t ${IMAGE_URI} -f sagemaker.Dockerfile . + +echo "Pushing ${IMAGE_URI}" +docker push ${IMAGE_URI} + +echo "Build and push completed successfully!" +``` + +
+ +3. Deploy a model for serving on AWS Sagemaker, refer to [deploy_and_serve_endpoint.py](https://github.com/sgl-project/sglang/blob/main/examples/sagemaker/deploy_and_serve_endpoint.py). For more information, check out [sagemaker-python-sdk](https://github.com/aws/sagemaker-python-sdk). + 1. By default, the model server on SageMaker will run with the following command: `python3 -m sglang.launch_server --model-path opt/ml/model --host 0.0.0.0 --port 8080`. This is optimal for hosting your own model with SageMaker. + 2. To modify your model serving parameters, the [serve](https://github.com/sgl-project/sglang/blob/main/docker/serve) script allows for all available options within `python3 -m sglang.launch_server --help` cli by specifying environment variables with prefix `SM_SGLANG_`. + 3. The serve script will automatically convert all environment variables with prefix `SM_SGLANG_` from `SM_SGLANG_INPUT_ARGUMENT` into `--input-argument` to be parsed into `python3 -m sglang.launch_server` cli. + 4. For example, to run [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) with reasoning parser, simply add additional environment variables `SM_SGLANG_MODEL_PATH=Qwen/Qwen3-0.6B` and `SM_SGLANG_REASONING_PARSER=qwen3`. + +
+ +## Common Notes + +- [FlashInfer](https://github.com/flashinfer-ai/flashinfer) is the default attention kernel backend. It only supports sm75 and above. If you encounter any FlashInfer-related issues on sm75+ devices (e.g., T4, A10, A100, L4, L40S, H100), please switch to other kernels by adding `--attention-backend triton --sampling-backend pytorch` and open an issue on GitHub. +- To reinstall flashinfer locally, use the following command: `pip3 install --upgrade flashinfer-python --force-reinstall --no-deps` and then delete the cache with `rm -rf ~/.cache/flashinfer`. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/index.rst b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/index.rst new file mode 100644 index 00000000..d605d9cc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/index.rst @@ -0,0 +1,106 @@ +SGLang Documentation +==================== + +SGLang is a high-performance serving framework for large language models and vision-language models. +It is designed to deliver low-latency and high-throughput inference across a wide range of setups, from a single GPU to large distributed clusters. +Its core features include: + +- **Fast Backend Runtime**: Provides efficient serving with RadixAttention for prefix caching, a zero-overhead CPU scheduler, prefill-decode disaggregation, speculative decoding, continuous batching, paged attention, tensor/pipeline/expert/data parallelism, structured outputs, chunked prefill, quantization (FP4/FP8/INT4/AWQ/GPTQ), and multi-LoRA batching. +- **Extensive Model Support**: Supports a wide range of generative models (Llama, Qwen, DeepSeek, Kimi, GLM, GPT, Gemma, Mistral, etc.), embedding models (e5-mistral, gte, mcdse), reward models (Skywork), and diffusion models (WAN, Qwen-Image), with easy extensibility for integrating new models. Compatible with most Hugging Face models and OpenAI APIs. +- **Extensive Hardware Support**: Runs on NVIDIA GPUs (GB200/B300/H100/A100/Spark), AMD GPUs (MI355/MI300), Intel Xeon CPUs, Google TPUs, Ascend NPUs, and more. +- **Flexible Frontend Language**: Offers an intuitive interface for programming LLM applications, supporting chained generation calls, advanced prompting, control flow, multi-modal inputs, parallelism, and external interactions. +- **Active Community**: SGLang is open-source and supported by a vibrant community with widespread industry adoption, powering over 400,000 GPUs worldwide. + +.. toctree:: + :maxdepth: 1 + :caption: Get Started + + get_started/install.md + +.. toctree:: + :maxdepth: 1 + :caption: Basic Usage + + basic_usage/send_request.ipynb + basic_usage/openai_api.rst + basic_usage/offline_engine_api.ipynb + basic_usage/native_api.ipynb + basic_usage/sampling_params.md + basic_usage/popular_model_usage.rst + +.. toctree:: + :maxdepth: 1 + :caption: Advanced Features + + advanced_features/server_arguments.md + advanced_features/hyperparameter_tuning.md + advanced_features/attention_backend.md + advanced_features/speculative_decoding.ipynb + advanced_features/structured_outputs.ipynb + advanced_features/structured_outputs_for_reasoning_models.ipynb + advanced_features/tool_parser.ipynb + advanced_features/separate_reasoning.ipynb + advanced_features/quantization.md + advanced_features/expert_parallelism.md + advanced_features/lora.ipynb + advanced_features/pd_disaggregation.md + advanced_features/hicache.rst + advanced_features/pd_multiplexing.md + advanced_features/vlm_query.ipynb + advanced_features/router.md + advanced_features/deterministic_inference.md + advanced_features/observability.md + advanced_features/checkpoint_engine.md + +.. toctree:: + :maxdepth: 1 + :caption: Supported Models + + supported_models/generative_models.md + supported_models/multimodal_language_models.md + supported_models/embedding_models.md + supported_models/reward_models.md + supported_models/rerank_models.md + supported_models/support_new_models.md + supported_models/transformers_fallback.md + supported_models/modelscope.md + +.. toctree:: + :maxdepth: 1 + :caption: Hardware Platforms + + platforms/amd_gpu.md + platforms/cpu_server.md + platforms/tpu.md + platforms/nvidia_jetson.md + platforms/ascend_npu.md + platforms/xpu.md + +.. toctree:: + :maxdepth: 1 + :caption: Developer Guide + + developer_guide/contribution_guide.md + developer_guide/development_guide_using_docker.md + developer_guide/benchmark_and_profiling.md + developer_guide/bench_serving.md + +.. toctree:: + :maxdepth: 1 + :caption: References + + references/faq.md + references/environment_variables.md + references/production_metrics.md + references/production_request_trace.md + references/multi_node_deployment/multi_node_index.rst + references/custom_chat_template.md + references/frontend/frontend_index.rst + references/learn_more.md + references/post_training_integration.md + +.. toctree:: + :maxdepth: 1 + :caption: Security Acknowledgement + + security/acknowledgements.md diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/amd_gpu.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/amd_gpu.md new file mode 100644 index 00000000..95e92dcc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/amd_gpu.md @@ -0,0 +1,159 @@ +# AMD GPUs + +This document describes how run SGLang on AMD GPUs. If you encounter issues or have questions, please [open an issue](https://github.com/sgl-project/sglang/issues). + +## System Configuration + +When using AMD GPUs (such as MI300X), certain system-level optimizations help ensure stable performance. Here we take MI300X as an example. AMD provides official documentation for MI300X optimization and system tuning: + +- [AMD MI300X Tuning Guides](https://rocm.docs.amd.com/en/latest/how-to/tuning-guides/mi300x/index.html) +- [LLM inference performance validation on AMD Instinct MI300X](https://rocm.docs.amd.com/en/latest/how-to/rocm-for-ai/inference/vllm-benchmark.html) +- [AMD Instinct MI300X System Optimization](https://rocm.docs.amd.com/en/latest/how-to/system-optimization/mi300x.html) +- [AMD Instinct MI300X Workload Optimization](https://rocm.docs.amd.com/en/latest/how-to/rocm-for-ai/inference-optimization/workload.html) +- [Supercharge DeepSeek-R1 Inference on AMD Instinct MI300X](https://rocm.blogs.amd.com/artificial-intelligence/DeepSeekR1-Part2/README.html) + +**NOTE:** We strongly recommend reading these docs and guides entirely to fully utilize your system. + +Below are a few key settings to confirm or enable for SGLang: + +### Update GRUB Settings + +In `/etc/default/grub`, append the following to `GRUB_CMDLINE_LINUX`: + +```text +pci=realloc=off iommu=pt +``` + +Afterward, run `sudo update-grub` (or your distro’s equivalent) and reboot. + +### Disable NUMA Auto-Balancing + +```bash +sudo sh -c 'echo 0 > /proc/sys/kernel/numa_balancing' +``` + +You can automate or verify this change using [this helpful script](https://github.com/ROCm/triton/blob/rocm_env/scripts/amd/env_check.sh). + +Again, please go through the entire documentation to confirm your system is using the recommended configuration. + +## Install SGLang + +You can install SGLang using one of the methods below. + +### Install from Source + +```bash +# Use the last release branch +git clone -b v0.5.6 https://github.com/sgl-project/sglang.git +cd sglang + +# Compile sgl-kernel +pip install --upgrade pip +cd sgl-kernel +python setup_rocm.py install + +# Install sglang python package +cd .. +rm -rf python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml +pip install -e "python[all_hip]" +``` + +### Install Using Docker (Recommended) + +The docker images are available on Docker Hub at [lmsysorg/sglang](https://hub.docker.com/r/lmsysorg/sglang/tags), built from [rocm.Dockerfile](https://github.com/sgl-project/sglang/tree/main/docker). + +The steps below show how to build and use an image. + +1. Build the docker image. + If you use pre-built images, you can skip this step and replace `sglang_image` with the pre-built image names in the steps below. + + ```bash + docker build -t sglang_image -f rocm.Dockerfile . + ``` + +2. Create a convenient alias. + + ```bash + alias drun='docker run -it --rm --network=host --privileged --device=/dev/kfd --device=/dev/dri \ + --ipc=host --shm-size 16G --group-add video --cap-add=SYS_PTRACE \ + --security-opt seccomp=unconfined \ + -v $HOME/dockerx:/dockerx \ + -v /data:/data' + ``` + + If you are using RDMA, please note that: + - `--network host` and `--privileged` are required by RDMA. If you don't need RDMA, you can remove them. + - You may need to set `NCCL_IB_GID_INDEX` if you are using RoCE, for example: `export NCCL_IB_GID_INDEX=3`. + +3. Launch the server. + + **NOTE:** Replace `` below with your [huggingface hub token](https://huggingface.co/docs/hub/en/security-tokens). + + ```bash + drun -p 30000:30000 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --env "HF_TOKEN=" \ + sglang_image \ + python3 -m sglang.launch_server \ + --model-path NousResearch/Meta-Llama-3.1-8B \ + --host 0.0.0.0 \ + --port 30000 + ``` + +4. To verify the utility, you can run a benchmark in another terminal or refer to [other docs](https://docs.sglang.io/basic_usage/openai_api_completions.html) to send requests to the engine. + + ```bash + drun sglang_image \ + python3 -m sglang.bench_serving \ + --backend sglang \ + --dataset-name random \ + --num-prompts 4000 \ + --random-input 128 \ + --random-output 128 + ``` + +With your AMD system properly configured and SGLang installed, you can now fully leverage AMD hardware to power SGLang’s machine learning capabilities. + +## Examples + +### Running DeepSeek-V3 + +The only difference when running DeepSeek-V3 is in how you start the server. Here's an example command: + +```bash +drun -p 30000:30000 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --ipc=host \ + --env "HF_TOKEN=" \ + sglang_image \ + python3 -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-V3 \ # <- here + --tp 8 \ + --trust-remote-code \ + --host 0.0.0.0 \ + --port 30000 +``` + +[Running DeepSeek-R1 on a single NDv5 MI300X VM](https://techcommunity.microsoft.com/blog/azurehighperformancecomputingblog/running-deepseek-r1-on-a-single-ndv5-mi300x-vm/4372726) could also be a good reference. + +### Running Llama3.1 + +Running Llama3.1 is nearly identical to running DeepSeek-V3. The only difference is in the model specified when starting the server, shown by the following example command: + +```bash +drun -p 30000:30000 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --ipc=host \ + --env "HF_TOKEN=" \ + sglang_image \ + python3 -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ # <- here + --tp 8 \ + --trust-remote-code \ + --host 0.0.0.0 \ + --port 30000 +``` + +### Warmup Step + +When the server displays `The server is fired up and ready to roll!`, it means the startup is successful. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/ascend_npu.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/ascend_npu.md new file mode 100644 index 00000000..cd251ba3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/ascend_npu.md @@ -0,0 +1,190 @@ +# Ascend NPUs + +You can install SGLang using any of the methods below. Please go through `System Settings` section to ensure the clusters are roaring at max performance. Feel free to leave an issue [here at sglang](https://github.com/sgl-project/sglang/issues) if you encounter any issues or have any problems. + +## System Settings + +### CPU performance power scheme + +The default power scheme on Ascend hardware is `ondemand` which could affect performance, changing it to `performance` is recommended. + +```shell +echo performance | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor + +# Make sure changes are applied successfully +cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor # shows performance +``` + +### Disable NUMA balancing + +```shell +sudo sysctl -w kernel.numa_balancing=0 + +# Check +cat /proc/sys/kernel/numa_balancing # shows 0 +``` + +### Prevent swapping out system memory + +```shell +sudo sysctl -w vm.swappiness=10 + +# Check +cat /proc/sys/vm/swappiness # shows 10 +``` + +## Installing SGLang + +### Method 1: Installing from source with prerequisites + +#### Python Version + +Only `python==3.11` is supported currently. If you don't want to break system pre-installed python, try installing with [conda](https://github.com/conda/conda). + +```shell +conda create --name sglang_npu python=3.11 +conda activate sglang_npu +``` + +#### MemFabric Adaptor + +_TODO: MemFabric is still a working project yet open sourced til end of year 2025. We will release it as prebuilt wheel package for now._ + +MemFabric Adaptor is a drop-in replacement of Mooncake Transfer Engine that enables KV cache transfer on Ascend NPU clusters. + +```shell +pip install mf-adapter==1.0.0 +``` + +#### Pytorch and Pytorch Framework Adaptor on Ascend + +```shell +PYTORCH_VERSION="2.8.0" +TORCHVISION_VERSION="0.23.0" +pip install torch==$PYTORCH_VERSION torchvision==$TORCHVISION_VERSION --index-url https://download.pytorch.org/whl/cpu + +PTA_VERSION="2.8.0" +pip install torch-npu==$PTA_VERSION +``` + +#### Triton on Ascend + +_Notice:_ We recommend installing triton-ascend from source due to its rapid development, the version on PYPI can't keep up for know. This problem will be solved on Sep. 2025, afterwards `pip install` would be the one and only installing method. + +Please follow Triton-on-Ascend's [installation guide from source](https://gitee.com/ascend/triton-ascend#2%E6%BA%90%E4%BB%A3%E7%A0%81%E5%AE%89%E8%A3%85-triton-ascend) to install the latest `triton-ascend` package. + +#### DeepEP-compatible Library + +We are also providing a DeepEP-compatible Library as a drop-in replacement of deepseek-ai's DeepEP library, check the [installation guide](https://github.com/sgl-project/sgl-kernel-npu/blob/main/python/deep_ep/README.md). + +#### Installing SGLang from source + +```shell +# Use the last release branch +git clone -b v0.5.6 https://github.com/sgl-project/sglang.git +cd sglang + +pip install --upgrade pip +rm -vf python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml +pip install -e python[srt_npu] +``` + +### Method 2: Using docker + +__Notice:__ `--privileged` and `--network=host` are required by RDMA, which is typically needed by Ascend NPU clusters. + +__Notice:__ The following docker command is based on Atlas 800I A3 machines. If you are using Atlas 800I A2, make sure only `davinci[0-7]` are mapped into container. + +```shell +# Clone the SGLang repository +git clone https://github.com/sgl-project/sglang.git +cd sglang/docker + +# Build the docker image +docker build -t -f npu.Dockerfile . + +alias drun='docker run -it --rm --privileged --network=host --ipc=host --shm-size=16g \ + --device=/dev/davinci0 --device=/dev/davinci1 --device=/dev/davinci2 --device=/dev/davinci3 \ + --device=/dev/davinci4 --device=/dev/davinci5 --device=/dev/davinci6 --device=/dev/davinci7 \ + --device=/dev/davinci8 --device=/dev/davinci9 --device=/dev/davinci10 --device=/dev/davinci11 \ + --device=/dev/davinci12 --device=/dev/davinci13 --device=/dev/davinci14 --device=/dev/davinci15 \ + --device=/dev/davinci_manager --device=/dev/hisi_hdc \ + --volume /usr/local/sbin:/usr/local/sbin --volume /usr/local/Ascend/driver:/usr/local/Ascend/driver \ + --volume /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ + --volume /etc/ascend_install.info:/etc/ascend_install.info \ + --volume /var/queue_schedule:/var/queue_schedule --volume ~/.cache/:/root/.cache/' + +drun --env "HF_TOKEN=" \ + \ + python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --attention-backend ascend --host 0.0.0.0 --port 30000 +``` + +## Examples + +### Running DeepSeek-V3 + +Running DeepSeek with PD disaggregation on 2 x Atlas 800I A3. +Model weights could be found [here](https://modelers.cn/models/State_Cloud/Deepseek-R1-bf16-hfd-w8a8). + +Prefill: + +```shell +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export ASCEND_MF_STORE_URL="tcp://:" + +drun \ + python3 -m sglang.launch_server --model-path State_Cloud/DeepSeek-R1-bf16-hfd-w8a8 \ + --trust-remote-code \ + --attention-backend ascend \ + --mem-fraction-static 0.8 \ + --quantization w8a8_int8 \ + --tp-size 16 \ + --dp-size 1 \ + --nnodes 1 \ + --node-rank 0 \ + --disaggregation-mode prefill \ + --disaggregation-bootstrap-port 6657 \ + --disaggregation-transfer-backend ascend \ + --dist-init-addr :6688 \ + --host \ + --port 8000 +``` + +Decode: + +```shell +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export ASCEND_MF_STORE_URL="tcp://:" +export HCCL_BUFFSIZE=200 +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=24 +export SGLANG_NPU_USE_MLAPO=1 + +drun \ + python3 -m sglang.launch_server --model-path State_Cloud/DeepSeek-R1-bf16-hfd-w8a8 \ + --trust-remote-code \ + --attention-backend ascend \ + --mem-fraction-static 0.8 \ + --quantization w8a8_int8 \ + --enable-deepep-moe \ + --deepep-mode low_latency \ + --tp-size 16 \ + --dp-size 1 \ + --ep-size 16 \ + --nnodes 1 \ + --node-rank 0 \ + --disaggregation-mode decode \ + --disaggregation-transfer-backend ascend \ + --dist-init-addr :6688 \ + --host \ + --port 8001 +``` + +Mini_LB: + +```shell +drun \ + python -m sglang.srt.disaggregation.launch_lb \ + --prefill http://:8000 \ + --decode http://:8001 \ + --host 127.0.0.1 --port 5000 +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/cpu_server.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/cpu_server.md new file mode 100644 index 00000000..71be9f6f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/cpu_server.md @@ -0,0 +1,216 @@ +# CPU Servers + +The document addresses how to set up the [SGLang](https://github.com/sgl-project/sglang) environment and run LLM inference on CPU servers. +SGLang is enabled and optimized on the CPUs equipped with Intel® AMX® Instructions, +which are 4th generation or newer Intel® Xeon® Scalable Processors. + +## Optimized Model List + +A list of popular LLMs are optimized and run efficiently on CPU, +including the most notable open-source models like Llama series, Qwen series, +and DeepSeek series like DeepSeek-R1 and DeepSeek-V3.1-Terminus. + +| Model Name | BF16 | W8A8_INT8 | FP8 | +|:---:|:---:|:---:|:---:| +| DeepSeek-R1 | | [meituan/DeepSeek-R1-Channel-INT8](https://huggingface.co/meituan/DeepSeek-R1-Channel-INT8) | [deepseek-ai/DeepSeek-R1](https://huggingface.co/deepseek-ai/DeepSeek-R1) | +| DeepSeek-V3.1-Terminus | | [IntervitensInc/DeepSeek-V3.1-Terminus-Channel-int8](https://huggingface.co/IntervitensInc/DeepSeek-V3.1-Terminus-Channel-int8) | [deepseek-ai/DeepSeek-V3.1-Terminus](https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus) | +| Llama-3.2-3B | [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) | [RedHatAI/Llama-3.2-3B-quantized.w8a8](https://huggingface.co/RedHatAI/Llama-3.2-3B-Instruct-quantized.w8a8) | | +| Llama-3.1-8B | [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) | [RedHatAI/Meta-Llama-3.1-8B-quantized.w8a8](https://huggingface.co/RedHatAI/Meta-Llama-3.1-8B-quantized.w8a8) | | +| QwQ-32B | | [RedHatAI/QwQ-32B-quantized.w8a8](https://huggingface.co/RedHatAI/QwQ-32B-quantized.w8a8) | | +| DeepSeek-Distilled-Llama | | [RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8](https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8) | | +| Qwen3-235B | | | [Qwen/Qwen3-235B-A22B-FP8](https://huggingface.co/Qwen/Qwen3-235B-A22B-FP8) | + +**Note:** The model identifiers listed in the table above +have been verified on 6th Gen Intel® Xeon® P-core platforms. + +## Installation + +### Install Using Docker + +It is recommended to use Docker for setting up the SGLang environment. +A [Dockerfile](https://github.com/sgl-project/sglang/blob/main/docker/xeon.Dockerfile) is provided to facilitate the installation. +Replace `` below with your [HuggingFace access token](https://huggingface.co/docs/hub/en/security-tokens). + +```bash +# Clone the SGLang repository +git clone https://github.com/sgl-project/sglang.git +cd sglang/docker + +# Build the docker image +docker build -t sglang-cpu:latest -f xeon.Dockerfile . + +# Initiate a docker container +docker run \ + -it \ + --privileged \ + --ipc=host \ + --network=host \ + -v /dev/shm:/dev/shm \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + -p 30000:30000 \ + -e "HF_TOKEN=" \ + sglang-cpu:latest /bin/bash +``` + +### Install From Source + +If you'd prefer to install SGLang in a bare metal environment, +the command list is as below. +It is worth noting that the environment variable `SGLANG_USE_CPU_ENGINE=1` +is required to enable SGLang service with CPU engine. + +```bash +# Create and activate a conda environment +conda create -n sgl-cpu python=3.12 -y +conda activate sgl-cpu + +# Set PyTorch CPU as primary pip install channel to avoid installing the larger CUDA-enabled version and prevent potential runtime issues. +pip config set global.index-url https://download.pytorch.org/whl/cpu +pip config set global.extra-index-url https://pypi.org/simple + +# Check if some conda related environment variables have been set +env | grep -i conda +# The following environment variable settings are required +# if they have not been set properly +export CONDA_EXE=$(which conda) +export CONDA_ROOT=${CONDA_EXE}/../.. +export CONDA_PREFIX=${CONDA_ROOT}/envs/sgl-cpu +export PATH=${PATH}:${CONDA_ROOT}/bin:${CONDA_ROOT}/condabin + +# Clone the SGLang code +git clone https://github.com/sgl-project/sglang.git +cd sglang +git checkout + +# Use dedicated toml file +cd python +cp pyproject_cpu.toml pyproject.toml +# Install SGLang dependent libs, and build SGLang main package +pip install --upgrade pip setuptools +conda install -y libsqlite==3.48.0 gperftools tbb libnuma numactl +pip install . +pip install torch==2.9.0 torchvision==0.24.0 triton==3.5.0 --force-reinstall + +# Build the CPU backend kernels +cd ../sgl-kernel +cp pyproject_cpu.toml pyproject.toml +pip install . + +# Other required environment variables +# Recommend to set these in ~/.bashrc in order not to set every time in a new terminal +export SGLANG_USE_CPU_ENGINE=1 +export LD_PRELOAD=${LD_PRELOAD}:${CONDA_PREFIX}/lib/libiomp5.so:${CONDA_PREFIX}/lib/libtcmalloc.so:${CONDA_PREFIX}/lib/libtbbmalloc.so.2 +``` + +## Launch of the Serving Engine + +Example command to launch SGLang serving: + +```bash +python -m sglang.launch_server \ + --model \ + --trust-remote-code \ + --disable-overlap-schedule \ + --device cpu \ + --host 0.0.0.0 \ + --tp 6 +``` + +Notes: + +1. For running W8A8 quantized models, please add the flag `--quantization w8a8_int8`. + +2. The flag `--tp 6` specifies that tensor parallelism will be applied using 6 ranks (TP6). + The number of TP specified is how many TP ranks will be used during the execution. + On a CPU platform, a TP rank means a sub-NUMA cluster (SNC). + Usually we can get the SNC information (How many available) from the Operating System. + Users can specify TP to be no more than the total available SNCs in current system. + + If the specified TP rank number differs from the total SNC count, + the system will automatically utilize the first `n` SNCs. + Note that `n` cannot exceed the total SNC number, doing so will result in an error. + + To specify the cores to be used, we need to explicitly set the environment variable `SGLANG_CPU_OMP_THREADS_BIND`. + For example, if we want to run the SGLang service using the first 40 cores of each SNC on a Xeon® 6980P server, + which has 43-43-42 cores on the 3 SNCs of a socket, we should set: + + ```bash + export SGLANG_CPU_OMP_THREADS_BIND="0-39|43-82|86-125|128-167|171-210|214-253" + ``` + + Please beware that with SGLANG_CPU_OMP_THREADS_BIND set, + the available memory amounts of the ranks may not be determined in prior. + You may need to set proper `--max-total-tokens` to avoid the out-of-memory error. + +3. For optimizing decoding with torch.compile, please add the flag `--enable-torch-compile`. + To specify the maximum batch size when using `torch.compile`, set the flag `--torch-compile-max-bs`. + For example, `--enable-torch-compile --torch-compile-max-bs 4` means using `torch.compile` + and setting the maximum batch size to 4. Currently the maximum applicable batch size + for optimizing with `torch.compile` is 16. + +4. A warmup step is automatically triggered when the service is started. + The server is ready when you see the log `The server is fired up and ready to roll!`. + +## Benchmarking with Requests + +You can benchmark the performance via the `bench_serving` script. +Run the command in another terminal. + +```bash +python -m sglang.bench_serving \ + --dataset-name random \ + --random-input-len 1024 \ + --random-output-len 1024 \ + --num-prompts 1 \ + --request-rate inf \ + --random-range-ratio 1.0 +``` + +The detail explanations of the parameters can be looked up by the command: + +```bash +python -m sglang.bench_serving -h +``` + +Additionally, the requests can be formed with +[OpenAI Completions API](https://docs.sglang.io/basic_usage/openai_api_completions.html) +and sent via the command line (e.g. using `curl`) or via your own script. + +## Example: Running DeepSeek-V3.1-Terminus + +An example command to launch service for W8A8_INT8 DeepSeek-V3.1-Terminus on a Xeon® 6980P server: + +```bash +python -m sglang.launch_server \ + --model IntervitensInc/DeepSeek-V3.1-Terminus-Channel-int8 \ + --trust-remote-code \ + --disable-overlap-schedule \ + --device cpu \ + --quantization w8a8_int8 \ + --host 0.0.0.0 \ + --mem-fraction-static 0.8 \ + --enable-torch-compile \ + --torch-compile-max-bs 4 \ + --tp 6 +``` + +Similarly, an example command to launch service for FP8 DeepSeek-V3.1-Terminus would be: + +```bash +python -m sglang.launch_server \ + --model deepseek-ai/DeepSeek-V3.1-Terminus \ + --trust-remote-code \ + --disable-overlap-schedule \ + --device cpu \ + --host 0.0.0.0 \ + --mem-fraction-static 0.8 \ + --enable-torch-compile \ + --torch-compile-max-bs 4 \ + --tp 6 +``` + +Note: Please set `--torch-compile-max-bs` to the maximum desired batch size for your deployment, +which can be up to 16. The value `4` in the examples is illustrative. + +Then you can test with `bench_serving` command or construct your own command or script +following [the benchmarking example](#benchmarking-with-requests). diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/nvidia_jetson.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/nvidia_jetson.md new file mode 100644 index 00000000..ba3b68ae --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/nvidia_jetson.md @@ -0,0 +1,80 @@ +# NVIDIA Jetson Orin + +## Prerequisites + +Before starting, ensure the following: + +- [**NVIDIA Jetson AGX Orin Devkit**](https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-orin/) is set up with **JetPack 6.1** or later. +- **CUDA Toolkit** and **cuDNN** are installed. +- Verify that the Jetson AGX Orin is in **high-performance mode**: +```bash +sudo nvpmodel -m 0 +``` +* * * * * +## Installing and running SGLang with Jetson Containers +Clone the jetson-containers github repository: +``` +git clone https://github.com/dusty-nv/jetson-containers.git +``` +Run the installation script: +``` +bash jetson-containers/install.sh +``` +Build the container image: +``` +jetson-containers build sglang +``` +Run the container: +``` +jetson-containers run $(autotag sglang) +``` +Or you can also manually run a container with this command: +``` +docker run --runtime nvidia -it --rm --network=host IMAGE_NAME +``` +* * * * * + +Running Inference +----------------------------------------- + +Launch the server: +```bash +python -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-R1-Distill-Llama-8B \ + --device cuda \ + --dtype half \ + --attention-backend flashinfer \ + --mem-fraction-static 0.8 \ + --context-length 8192 +``` +The quantization and limited context length (`--dtype half --context-length 8192`) are due to the limited computational resources in [Nvidia jetson kit](https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-orin/). A detailed explanation can be found in [Server Arguments](../advanced_features/server_arguments.md). + +After launching the engine, refer to [Chat completions](https://docs.sglang.io/basic_usage/openai_api_completions.html#Usage) to test the usability. +* * * * * +Running quantization with TorchAO +------------------------------------- +TorchAO is suggested to NVIDIA Jetson Orin. +```bash +python -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ + --device cuda \ + --dtype bfloat16 \ + --attention-backend flashinfer \ + --mem-fraction-static 0.8 \ + --context-length 8192 \ + --torchao-config int4wo-128 +``` +This enables TorchAO's int4 weight-only quantization with a 128-group size. The usage of `--torchao-config int4wo-128` is also for memory efficiency. + + +* * * * * +Structured output with XGrammar +------------------------------- +Please refer to [SGLang doc structured output](../advanced_features/structured_outputs.ipynb). +* * * * * + +Thanks to the support from [Nurgaliyev Shakhizat](https://github.com/shahizat), [Dustin Franklin](https://github.com/dusty-nv) and [Johnny Núñez Cano](https://github.com/johnnynunez). + +References +---------- +- [NVIDIA Jetson AGX Orin Documentation](https://developer.nvidia.com/embedded/jetson-agx-orin) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/tpu.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/tpu.md new file mode 100644 index 00000000..925287c3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/tpu.md @@ -0,0 +1,3 @@ +# TPU + +SGLang supports TPU inference via the SGLang-Jax backend. Please go to https://github.com/sgl-project/sglang-jax. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/xpu.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/xpu.md new file mode 100644 index 00000000..88fa1552 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/platforms/xpu.md @@ -0,0 +1,92 @@ +# XPU + +The document addresses how to set up the [SGLang](https://github.com/sgl-project/sglang) environment and run LLM inference on Intel GPU, [see more context about Intel GPU support within PyTorch ecosystem](https://docs.pytorch.org/docs/stable/notes/get_start_xpu.html). + +Specifically, SGLang is optimized for [Intel® Arc™ Pro B-Series Graphics](https://www.intel.com/content/www/us/en/ark/products/series/242616/intel-arc-pro-b-series-graphics.html) and [ +Intel® Arc™ B-Series Graphics](https://www.intel.com/content/www/us/en/ark/products/series/240391/intel-arc-b-series-graphics.html). + +## Optimized Model List + +A list of LLMs have been optimized on Intel GPU, and more are on the way: + +| Model Name | BF16 | +|:---:|:---:| +| Llama-3.2-3B | [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) | +| Llama-3.1-8B | [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) | +| Qwen2.5-1.5B | [Qwen/Qwen2.5-1.5B](https://huggingface.co/Qwen/Qwen2.5-1.5B) | + +**Note:** The model identifiers listed in the table above +have been verified on [Intel® Arc™ B580 Graphics](https://www.intel.com/content/www/us/en/products/sku/241598/intel-arc-b580-graphics/specifications.html). + +## Installation + +### Install From Source + +Currently SGLang XPU only supports installation from source. Please refer to ["Getting Started on Intel GPU"](https://docs.pytorch.org/docs/stable/notes/get_start_xpu.html) to install XPU dependency. + +```bash +# Create and activate a conda environment +conda create -n sgl-xpu python=3.12 -y +conda activate sgl-xpu + +# Set PyTorch XPU as primary pip install channel to avoid installing the larger CUDA-enabled version and prevent potential runtime issues. +pip3 install torch==2.9.0+xpu torchao torchvision torchaudio pytorch-triton-xpu==3.5.0 --index-url https://download.pytorch.org/whl/xpu +pip3 install xgrammar --no-deps # xgrammar will introduce CUDA-enabled triton which might conflict with XPU + +# Clone the SGLang code +git clone https://github.com/sgl-project/sglang.git +cd sglang +git checkout + +# Use dedicated toml file +cd python +cp pyproject_xpu.toml pyproject.toml +# Install SGLang dependent libs, and build SGLang main package +pip install --upgrade pip setuptools +pip install -v . +``` + +### Install Using Docker + +The docker for XPU is under active development. Please stay tuned. + +## Launch of the Serving Engine + +Example command to launch SGLang serving: + +```bash +python -m sglang.launch_server \ + --model \ + --trust-remote-code \ + --disable-overlap-schedule \ + --device xpu \ + --host 0.0.0.0 \ + --tp 2 \ # using multi GPUs + --attention-backend intel_xpu \ # using intel optimized XPU attention backend + --page-size \ # intel_xpu attention backend supports [32, 64, 128] +``` + +## Benchmarking with Requests + +You can benchmark the performance via the `bench_serving` script. +Run the command in another terminal. + +```bash +python -m sglang.bench_serving \ + --dataset-name random \ + --random-input-len 1024 \ + --random-output-len 1024 \ + --num-prompts 1 \ + --request-rate inf \ + --random-range-ratio 1.0 +``` + +The detail explanations of the parameters can be looked up by the command: + +```bash +python -m sglang.bench_serving -h +``` + +Additionally, the requests can be formed with +[OpenAI Completions API](https://docs.sglang.io/basic_usage/openai_api_completions.html) +and sent via the command line (e.g. using `curl`) or via your own script. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/custom_chat_template.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/custom_chat_template.md new file mode 100644 index 00000000..f22ee8be --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/custom_chat_template.md @@ -0,0 +1,51 @@ +# Custom Chat Template + +**NOTE**: There are two chat template systems in SGLang project. This document is about setting a custom chat template for the OpenAI-compatible API server (defined at [conversation.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/conversation.py)). It is NOT related to the chat template used in the SGLang language frontend (defined at [chat_template.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/lang/chat_template.py)). + +By default, the server uses the chat template specified in the model tokenizer from Hugging Face. +It should just work for most official models such as Llama-2/Llama-3. + +If needed, you can also override the chat template when launching the server: + +```bash +python -m sglang.launch_server \ + --model-path meta-llama/Llama-2-7b-chat-hf \ + --port 30000 \ + --chat-template llama-2 +``` + +If the chat template you are looking for is missing, you are welcome to contribute it or load it from a file. + +## JSON Format + +You can load the JSON format, which is defined by `conversation.py`. + +```json +{ + "name": "my_model", + "system": "<|im_start|>system", + "user": "<|im_start|>user", + "assistant": "<|im_start|>assistant", + "sep_style": "CHATML", + "sep": "<|im_end|>", + "stop_str": ["<|im_end|>", "<|im_start|>"] +} +``` + +```bash +python -m sglang.launch_server \ + --model-path meta-llama/Llama-2-7b-chat-hf \ + --port 30000 \ + --chat-template ./my_model_template.json +``` + +## Jinja Format + +You can also use the [Jinja template format](https://huggingface.co/docs/transformers/main/en/chat_templating) as defined by Hugging Face Transformers. + +```bash +python -m sglang.launch_server \ + --model-path meta-llama/Llama-2-7b-chat-hf \ + --port 30000 \ + --chat-template ./my_model_template.jinja +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/environment_variables.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/environment_variables.md new file mode 100644 index 00000000..d8a1e67e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/environment_variables.md @@ -0,0 +1,127 @@ +# Environment Variables + +SGLang supports various environment variables that can be used to configure its runtime behavior. This document provides a comprehensive list and aims to stay updated over time. + +*Note: SGLang uses two prefixes for environment variables: `SGL_` and `SGLANG_`. This is likely due to historical reasons. While both are currently supported for different settings, future versions might consolidate them.* + +## General Configuration + +| Environment Variable | Description | Default Value | +|-------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------|------------------------------| +| `SGLANG_USE_MODELSCOPE` | Enable using models from ModelScope | `false` | +| `SGLANG_HOST_IP` | Host IP address for the server | `0.0.0.0` | +| `SGLANG_PORT` | Port for the server | auto-detected | +| `SGLANG_LOGGING_CONFIG_PATH` | Custom logging configuration path | Not set | +| `SGLANG_DISABLE_REQUEST_LOGGING` | Disable request logging | `false` | +| `SGLANG_HEALTH_CHECK_TIMEOUT` | Timeout for health check in seconds | `20` | +| `SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL` | The interval of passes to collect the metric of selected count of physical experts on each layer and GPU rank. 0 means disabled. | `0` | +| `SGLANG_FORWARD_UNKNOWN_TOOLS` | Forward unknown tool calls to clients instead of dropping them | `false` (drop unknown tools) | + +## Performance Tuning + +| Environment Variable | Description | Default Value | +| --- | --- | --- | +| `SGLANG_ENABLE_TORCH_INFERENCE_MODE` | Control whether to use torch.inference_mode | `false` | +| `SGLANG_ENABLE_TORCH_COMPILE` | Enable torch.compile | `true` | +| `SGLANG_SET_CPU_AFFINITY` | Enable CPU affinity setting (often set to `1` in Docker builds) | `0` | +| `SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN` | Allows the scheduler to overwrite longer context length requests (often set to `1` in Docker builds) | `0` | +| `SGLANG_IS_FLASHINFER_AVAILABLE` | Control FlashInfer availability check | `true` | +| `SGLANG_SKIP_P2P_CHECK` | Skip P2P (peer-to-peer) access check | `false` | +| `SGL_CHUNKED_PREFIX_CACHE_THRESHOLD` | Sets the threshold for enabling chunked prefix caching | `8192` | +| `SGLANG_FUSED_MLA_ENABLE_ROPE_FUSION` | Enable RoPE fusion in Fused Multi-Layer Attention | `1` | +| `SGLANG_DISABLE_CONSECUTIVE_PREFILL_OVERLAP` | Disable overlap schedule for consecutive prefill batches | `false` | +| `SGLANG_SCHEDULER_MAX_RECV_PER_POLL` | Set the maximum number of requests per poll, with a negative value indicating no limit | `-1` | +| `SGLANG_DISABLE_FA4_WARMUP` | Disable Flash Attention 4 warmup passes (set to `1`, `true`, `yes`, or `on` to disable) | `false` | +| `SGLANG_SCHEDULER_RECV_SKIPPER_WEIGHT_DEFAULT` | Default weight value for scheduler recv skipper counter (used when forward mode doesn't match specific modes). Only active when `--scheduler-recv-interval > 1`. The counter accumulates weights and triggers request polling when reaching the interval threshold. | `1000` | +| `SGLANG_SCHEDULER_RECV_SKIPPER_WEIGHT_DECODE` | Weight increment for decode forward mode in scheduler recv skipper. Works with `--scheduler-recv-interval` to control polling frequency during decode phase. | `1` | +| `SGLANG_SCHEDULER_RECV_SKIPPER_WEIGHT_VERIFY` | Weight increment for target verify forward mode in scheduler recv skipper. Works with `--scheduler-recv-interval` to control polling frequency during verification phase. | `1` | +| `SGLANG_SCHEDULER_RECV_SKIPPER_WEIGHT_NONE` | Weight increment when forward mode is None in scheduler recv skipper. Works with `--scheduler-recv-interval` to control polling frequency when no specific forward mode is active. | `1` | + + +## DeepGEMM Configuration (Advanced Optimization) + +| Environment Variable | Description | Default Value | +| --- | --- | --- | +| `SGLANG_ENABLE_JIT_DEEPGEMM` | Enable Just-In-Time compilation of DeepGEMM kernels | `"true"` | +| `SGLANG_JIT_DEEPGEMM_PRECOMPILE` | Enable precompilation of DeepGEMM kernels | `"true"` | +| `SGLANG_JIT_DEEPGEMM_COMPILE_WORKERS` | Number of workers for parallel DeepGEMM kernel compilation | `4` | +| `SGL_IN_DEEPGEMM_PRECOMPILE_STAGE` | Indicator flag used during the DeepGEMM precompile script | `"false"` | +| `SGLANG_DG_CACHE_DIR` | Directory for caching compiled DeepGEMM kernels | `~/.cache/deep_gemm` | +| `SGL_DG_USE_NVRTC` | Use NVRTC (instead of Triton) for JIT compilation (Experimental) | `"0"` | +| `SGL_USE_DEEPGEMM_BMM` | Use DeepGEMM for Batched Matrix Multiplication (BMM) operations | `"false"` | + +## DeepEP Configuration + +| Environment Variable | Description | Default Value | +| --- | --- | --- | +| `SGLANG_DEEPEP_BF16_DISPATCH` | Use Bfloat16 for dispatch | `"false"` | +| `SGLANG_MOE_NVFP4_DISPATCH` | Use nvfp4 for moe dispatch | `"false"` | + +## Memory Management + +| Environment Variable | Description | Default Value | +| --- | --- | --- | +| `SGLANG_DEBUG_MEMORY_POOL` | Enable memory pool debugging | `false` | +| `SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION` | Clip max new tokens estimation for memory planning | `4096` | +| `SGLANG_DETOKENIZER_MAX_STATES` | Maximum states for detokenizer | Default value based on system | +| `SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK` | Disable checks for memory imbalance across Tensor Parallel ranks | Not set (defaults to enabled check) | + +## Model-Specific Options + +| Environment Variable | Description | Default Value | +| --- | --- | --- | +| `SGLANG_USE_AITER` | Use AITER optimize implementation | `false` | +| `SGLANG_INT4_WEIGHT` | Enable INT4 weight quantization | `false` | +| `SGLANG_MOE_PADDING` | Enable MoE padding (sets padding size to 128 if value is `1`, often set to `1` in Docker builds) | `0` | +| `SGLANG_FORCE_FP8_MARLIN` | Force using FP8 MARLIN kernels even if other FP8 kernels are available | `false` | +| `SGLANG_ENABLE_FLASHINFER_FP8_GEMM` | Use flashinfer kernels when running blockwise fp8 GEMM on Blackwell GPUs | `false` | +| `SGLANG_FLASHINFER_FP4_GEMM_BACKEND` | Select backend for `mm_fp4` on Blackwell GPUS | `` | +| `SGLANG_SUPPORT_CUTLASS_BLOCK_FP8` | Use Cutlass kernels when running blockwise fp8 GEMM on Hopper or Blackwell GPUs | `false` | +| `SGLANG_CUTLASS_MOE` (deprecated) | Use Cutlass FP8 MoE kernel on Blackwell GPUs (deprecated, use --moe-runner-backend=cutlass) | `false` | + + +## Distributed Computing + +| Environment Variable | Description | Default Value | +| --- | --- | --- | +| `SGLANG_BLOCK_NONZERO_RANK_CHILDREN` | Control blocking of non-zero rank children processes | `1` | +| `SGL_IS_FIRST_RANK_ON_NODE` | Indicates if the current process is the first rank on its node | `"true"` | +| `SGLANG_PP_LAYER_PARTITION` | Pipeline parallel layer partition specification | Not set | +| `SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS` | Set one visible device per process for distributed computing | `false` | + +## Testing & Debugging (Internal/CI) + +*These variables are primarily used for internal testing, continuous integration, or debugging.* + +| Environment Variable | Description | Default Value | +| --- | --- | --- | +| `SGLANG_IS_IN_CI` | Indicates if running in CI environment | `false` | +| `SGLANG_IS_IN_CI_AMD` | Indicates running in AMD CI environment | `0` | +| `SGLANG_TEST_RETRACT` | Enable retract decode testing | `false` | +| `SGLANG_TEST_RETRACT_NO_PREFILL_BS` | When SGLANG_TEST_RETRACT is enabled, no prefill is performed if the batch size exceeds SGLANG_TEST_RETRACT_NO_PREFILL_BS. | `2 ** 31` | +| `SGLANG_RECORD_STEP_TIME` | Record step time for profiling | `false` | +| `SGLANG_TEST_REQUEST_TIME_STATS` | Test request time statistics | `false` | +| `SGLANG_CI_SMALL_KV_SIZE` | Use small KV cache size in CI | Not set | + +## Profiling & Benchmarking + +| Environment Variable | Description | Default Value | +| --- | --- | --- | +| `SGLANG_TORCH_PROFILER_DIR` | Directory for PyTorch profiler output | `/tmp` | +| `SGLANG_PROFILE_WITH_STACK` | Set `with_stack` option (bool) for PyTorch profiler (capture stack trace) | `true` | +| `SGLANG_PROFILE_RECORD_SHAPES` | Set `record_shapes` option (bool) for PyTorch profiler (record shapes) | `true` | +| `SGLANG_OTLP_EXPORTER_SCHEDULE_DELAY_MILLIS` | Config BatchSpanProcessor.schedule_delay_millis if tracing is enabled | `500` | +| `SGLANG_OTLP_EXPORTER_MAX_EXPORT_BATCH_SIZE` | Config BatchSpanProcessor.max_export_batch_size if tracing is enabled | `64` | + +## Storage & Caching + +| Environment Variable | Description | Default Value | +| --- | --- | --- | +| `SGLANG_WAIT_WEIGHTS_READY_TIMEOUT` | Timeout period for waiting on weights | `120` | +| `SGLANG_DISABLE_OUTLINES_DISK_CACHE` | Disable Outlines disk cache | `true` | + +## Function Calling / Tool Use + +| Environment Variable | Description | Default Value | +| --- | --- | --- | +| `SGLANG_TOOL_STRICT_LEVEL` | Controls the strictness level of tool call parsing and validation.
**Level 0**: Off - No strict validation
**Level 1**: Function strict - Enables structural tag constraints for all tools (even if none have `strict=True` set)
**Level 2**: Parameter strict - Enforces strict parameter validation for all tools, treating them as if they all have `strict=True` set | `0` | diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/faq.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/faq.md new file mode 100644 index 00000000..ffa1a7c5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/faq.md @@ -0,0 +1,39 @@ +# Troubleshooting and Frequently Asked Questions + +## Troubleshooting + +This page lists common errors and tips for resolving them. + +### CUDA Out of Memory +If you encounter out-of-memory (OOM) errors, you can adjust the following parameters: + +- If OOM occurs during prefill, try reducing `--chunked-prefill-size` to `4096` or `2048`. This saves memory but slows down the prefill speed for long prompts. +- If OOM occurs during decoding, try lowering `--max-running-requests`. +- You can also decrease `--mem-fraction-static` to a smaller value, such as 0.8 or 0.7. This decreases the memory usage of the KV cache memory pool and helps prevent OOM errors during both prefill and decoding. However, it limits maximum concurrency and reduces peak throughput. +- Another common case for OOM is requesting input logprobs for a long prompt as it requires significant memory. To address this, set `logprob_start_len` in your sampling parameters to include only the necessary parts. If you do need input logprobs for a long prompt, try reducing `--mem-fraction-static`. + +### CUDA Error: Illegal Memory Access Encountered +This error may result from kernel errors or out-of-memory issues: +- If it is a kernel error, resolving it may be challenging. Please file an issue on GitHub. +- If it is an out-of-memory issue, it may sometimes be reported as this error instead of "Out of Memory." Refer to the section above for guidance on avoiding OOM issues. + +### The server hangs +- If the server hangs during initialization or running, it can be memory issues (out of memory), network issues (nccl errors), or other bugs in sglang. + - If it is out of memory, you might see that `avail mem` is very low during the initialization or right after initialization. In this case, + you can try to decrease `--mem-fraction-static`, decrease `--cuda-graph-max-bs`, or decrease `--chunked-prefill-size`. +- Other bugs, please file an issue on GitHub. + + +## Frequently Asked Questions + +### The results are not deterministic, even with a temperature of 0 + +You may notice that when you send the same request twice, the results from the engine will be slightly different, even when the temperature is set to 0. + +From our initial investigation, this indeterminism arises from two factors: dynamic batching and prefix caching. Roughly speaking, dynamic batching accounts for about 95% of the indeterminism, while prefix caching accounts for the remaining portion. The server runs dynamic batching under the hood. Different batch sizes can cause PyTorch/CuBLAS to dispatch to different CUDA kernels, which can lead to slight numerical differences. This difference accumulates across many layers, resulting in nondeterministic output when the batch size changes. Similarly, when prefix caching is enabled, it can also dispatch to different kernels. Even when the computations are mathematically equivalent, small numerical differences from different kernel implementations lead to the final nondeterministic outputs. + +To achieve more deterministic outputs in the current code, you can add `--disable-radix-cache` and send only one request at a time. The results will be mostly deterministic under this setting. + +**Update**: +Recently, we also introduced a deterministic mode, you can enable it with `--enable-deterministic-inference`. +Please find more details in this blog post: https://lmsys.org/blog/2025-09-22-sglang-deterministic/ diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/frontend/choices_methods.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/frontend/choices_methods.md new file mode 100644 index 00000000..30a0a181 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/frontend/choices_methods.md @@ -0,0 +1,77 @@ +# Choices Methods in SGLang +This doc describes the choices methods supported by SGLang. + +The optional `choices_method` arg determines how options supplied to SGLang's `choices` primitive are selected. Only the `RuntimeEndpoint` backend supports the `choices_method` arg. Other backends, such as `OpenAI`, have bespoke selection implementations due to API limitations. + +## Methods + +### Token Length Normalized + +Token length normalized is the default SGLang choices method. It selects the option with the highest average logprob across all of its tokens. + +Usage example (alternatively, simply omit the `choices_method` arg): +```python +@sgl.function +def example(s): + s += sgl.user("What is the capital of France?") + s += sgl.assistant( + sgl.gen( + "answer", + choices=["London", "Paris", "Berlin"], + choices_method=sgl.token_length_normalized, + ) + ) +``` + + +This can perform poorly if an option contains many tokens, where its later tokens are predicted with high confidence based on its earlier tokens. For instance, even strong models will fail the above example if the specified options are `["Paris", "Antidisestablishmentarianism"]`. + +### Greedy Token Selection + +Greedy token selection simply selects the option with the highest logprob for its initial token. For overlapping options where one option is a subset of a longer option, the logprobs of the shorter option are extended using its average logprob for comparison against the longer option. + +Usage example: +```python +@sgl.function +def example(s): + s += sgl.user("What is the capital of France?") + s += sgl.assistant( + sgl.gen( + "answer", + choices=["London", "Paris", "Berlin"], + choices_method=sgl.greedy_token_selection, + ) + ) +``` + +This can perform poorly if an option misleads the model down a bad path based on an attractive initial token. For instance, greedy selection will result in an incorrect response for this example: +```python +@sgl.function +def us_president_example(s): + s += sgl.user("Name a US president.") + s += sgl.assistant( + sgl.gen( + "answer", + choices=["Donald Duck", "Millard Fillmore"], + choices_method=sgl.greedy_token_selection, + ) + ) +``` + +### Unconditional Likelihood Normalized + +Unconditional likelihood normalized selects the option with the highest average token logprob once normalized by the unconditional token logprobs, as described in [this EleutherAI blogpost](https://blog.eleuther.ai/multiple-choice-normalization/). This method incurs an additional LLM call to obtain the unconditional likelihoods. + +Usage example: +```python +@sgl.function +def example(s): + s += sgl.user("What is the capital of France?") + s += sgl.assistant( + sgl.gen( + "answer", + choices=["London", "Paris", "Berlin"], + choices_method=sgl.unconditional_likelihood_normalized, + ) + ) +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/frontend/frontend_index.rst b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/frontend/frontend_index.rst new file mode 100644 index 00000000..62544cba --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/frontend/frontend_index.rst @@ -0,0 +1,9 @@ +Frontend Language +================= + +.. toctree:: + :maxdepth: 1 + :caption: Frontend Language + + frontend_tutorial.ipynb + choices_methods.md diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/frontend/frontend_tutorial.ipynb b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/frontend/frontend_tutorial.ipynb new file mode 100644 index 00000000..166f8cac --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/frontend/frontend_tutorial.ipynb @@ -0,0 +1,456 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# SGLang Frontend Language" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "SGLang frontend language can be used to define simple and easy prompts in a convenient, structured way." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Launch A Server\n", + "\n", + "Launch the server in your terminal and wait for it to initialize." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sglang import assistant_begin, assistant_end\n", + "from sglang import assistant, function, gen, system, user\n", + "from sglang import image\n", + "from sglang import RuntimeEndpoint\n", + "from sglang.lang.api import set_default_backend\n", + "from sglang.srt.utils import load_image\n", + "from sglang.test.doc_patch import launch_server_cmd\n", + "from sglang.utils import print_highlight, terminate_process, wait_for_server\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"python -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --host 0.0.0.0 --log-level warning\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")\n", + "print(f\"Server started on http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Set the default backend. Note: Besides the local server, you may use also `OpenAI` or other API endpoints." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "set_default_backend(RuntimeEndpoint(f\"http://localhost:{port}\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Basic Usage\n", + "\n", + "The most simple way of using SGLang frontend language is a simple question answer dialog between a user and an assistant." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def basic_qa(s, question):\n", + " s += system(f\"You are a helpful assistant than can answer questions.\")\n", + " s += user(question)\n", + " s += assistant(gen(\"answer\", max_tokens=512))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "state = basic_qa(\"List 3 countries and their capitals.\")\n", + "print_highlight(state[\"answer\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multi-turn Dialog\n", + "\n", + "SGLang frontend language can also be used to define multi-turn dialogs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def multi_turn_qa(s):\n", + " s += system(f\"You are a helpful assistant than can answer questions.\")\n", + " s += user(\"Please give me a list of 3 countries and their capitals.\")\n", + " s += assistant(gen(\"first_answer\", max_tokens=512))\n", + " s += user(\"Please give me another list of 3 countries and their capitals.\")\n", + " s += assistant(gen(\"second_answer\", max_tokens=512))\n", + " return s\n", + "\n", + "\n", + "state = multi_turn_qa()\n", + "print_highlight(state[\"first_answer\"])\n", + "print_highlight(state[\"second_answer\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Control flow\n", + "\n", + "You may use any Python code within the function to define more complex control flows." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def tool_use(s, question):\n", + " s += assistant(\n", + " \"To answer this question: \"\n", + " + question\n", + " + \". I need to use a \"\n", + " + gen(\"tool\", choices=[\"calculator\", \"search engine\"])\n", + " + \". \"\n", + " )\n", + "\n", + " if s[\"tool\"] == \"calculator\":\n", + " s += assistant(\"The math expression is: \" + gen(\"expression\"))\n", + " elif s[\"tool\"] == \"search engine\":\n", + " s += assistant(\"The key word to search is: \" + gen(\"word\"))\n", + "\n", + "\n", + "state = tool_use(\"What is 2 * 2?\")\n", + "print_highlight(state[\"tool\"])\n", + "print_highlight(state[\"expression\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Parallelism\n", + "\n", + "Use `fork` to launch parallel prompts. Because `sgl.gen` is non-blocking, the for loop below issues two generation calls in parallel." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def tip_suggestion(s):\n", + " s += assistant(\n", + " \"Here are two tips for staying healthy: \"\n", + " \"1. Balanced Diet. 2. Regular Exercise.\\n\\n\"\n", + " )\n", + "\n", + " forks = s.fork(2)\n", + " for i, f in enumerate(forks):\n", + " f += assistant(\n", + " f\"Now, expand tip {i+1} into a paragraph:\\n\"\n", + " + gen(\"detailed_tip\", max_tokens=256, stop=\"\\n\\n\")\n", + " )\n", + "\n", + " s += assistant(\"Tip 1:\" + forks[0][\"detailed_tip\"] + \"\\n\")\n", + " s += assistant(\"Tip 2:\" + forks[1][\"detailed_tip\"] + \"\\n\")\n", + " s += assistant(\n", + " \"To summarize the above two tips, I can say:\\n\" + gen(\"summary\", max_tokens=512)\n", + " )\n", + "\n", + "\n", + "state = tip_suggestion()\n", + "print_highlight(state[\"summary\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Constrained Decoding\n", + "\n", + "Use `regex` to specify a regular expression as a decoding constraint. This is only supported for local models." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def regular_expression_gen(s):\n", + " s += user(\"What is the IP address of the Google DNS servers?\")\n", + " s += assistant(\n", + " gen(\n", + " \"answer\",\n", + " temperature=0,\n", + " regex=r\"((25[0-5]|2[0-4]\\d|[01]?\\d\\d?).){3}(25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\",\n", + " )\n", + " )\n", + "\n", + "\n", + "state = regular_expression_gen()\n", + "print_highlight(state[\"answer\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use `regex` to define a `JSON` decoding schema." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "character_regex = (\n", + " r\"\"\"\\{\\n\"\"\"\n", + " + r\"\"\" \"name\": \"[\\w\\d\\s]{1,16}\",\\n\"\"\"\n", + " + r\"\"\" \"house\": \"(Gryffindor|Slytherin|Ravenclaw|Hufflepuff)\",\\n\"\"\"\n", + " + r\"\"\" \"blood status\": \"(Pure-blood|Half-blood|Muggle-born)\",\\n\"\"\"\n", + " + r\"\"\" \"occupation\": \"(student|teacher|auror|ministry of magic|death eater|order of the phoenix)\",\\n\"\"\"\n", + " + r\"\"\" \"wand\": \\{\\n\"\"\"\n", + " + r\"\"\" \"wood\": \"[\\w\\d\\s]{1,16}\",\\n\"\"\"\n", + " + r\"\"\" \"core\": \"[\\w\\d\\s]{1,16}\",\\n\"\"\"\n", + " + r\"\"\" \"length\": [0-9]{1,2}\\.[0-9]{0,2}\\n\"\"\"\n", + " + r\"\"\" \\},\\n\"\"\"\n", + " + r\"\"\" \"alive\": \"(Alive|Deceased)\",\\n\"\"\"\n", + " + r\"\"\" \"patronus\": \"[\\w\\d\\s]{1,16}\",\\n\"\"\"\n", + " + r\"\"\" \"bogart\": \"[\\w\\d\\s]{1,16}\"\\n\"\"\"\n", + " + r\"\"\"\\}\"\"\"\n", + ")\n", + "\n", + "\n", + "@function\n", + "def character_gen(s, name):\n", + " s += user(\n", + " f\"{name} is a character in Harry Potter. Please fill in the following information about this character.\"\n", + " )\n", + " s += assistant(gen(\"json_output\", max_tokens=256, regex=character_regex))\n", + "\n", + "\n", + "state = character_gen(\"Harry Potter\")\n", + "print_highlight(state[\"json_output\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Batching \n", + "\n", + "Use `run_batch` to run a batch of prompts." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def text_qa(s, question):\n", + " s += user(question)\n", + " s += assistant(gen(\"answer\", stop=\"\\n\"))\n", + "\n", + "\n", + "states = text_qa.run_batch(\n", + " [\n", + " {\"question\": \"What is the capital of the United Kingdom?\"},\n", + " {\"question\": \"What is the capital of France?\"},\n", + " {\"question\": \"What is the capital of Japan?\"},\n", + " ],\n", + " progress_bar=True,\n", + ")\n", + "\n", + "for i, state in enumerate(states):\n", + " print_highlight(f\"Answer {i+1}: {states[i]['answer']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Streaming \n", + "\n", + "Use `stream` to stream the output to the user." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def text_qa(s, question):\n", + " s += user(question)\n", + " s += assistant(gen(\"answer\", stop=\"\\n\"))\n", + "\n", + "\n", + "state = text_qa.run(\n", + " question=\"What is the capital of France?\", temperature=0.1, stream=True\n", + ")\n", + "\n", + "for out in state.text_iter():\n", + " print(out, end=\"\", flush=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Complex Prompts\n", + "\n", + "You may use `{system|user|assistant}_{begin|end}` to define complex prompts." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def chat_example(s):\n", + " s += system(\"You are a helpful assistant.\")\n", + " # Same as: s += s.system(\"You are a helpful assistant.\")\n", + "\n", + " with s.user():\n", + " s += \"Question: What is the capital of France?\"\n", + "\n", + " s += assistant_begin()\n", + " s += \"Answer: \" + gen(\"answer\", max_tokens=100, stop=\"\\n\")\n", + " s += assistant_end()\n", + "\n", + "\n", + "state = chat_example()\n", + "print_highlight(state[\"answer\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multi-modal Generation\n", + "\n", + "You may use SGLang frontend language to define multi-modal prompts.\n", + "See [here](https://docs.sglang.io/supported_models/generative_models.html) for supported models." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_process, port = launch_server_cmd(\n", + " \"python -m sglang.launch_server --model-path Qwen/Qwen2.5-VL-7B-Instruct --host 0.0.0.0 --log-level warning\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")\n", + "print(f\"Server started on http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "set_default_backend(RuntimeEndpoint(f\"http://localhost:{port}\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Ask a question about an image." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def image_qa(s, image_file, question):\n", + " s += user(image(image_file) + question)\n", + " s += assistant(gen(\"answer\", max_tokens=256))\n", + "\n", + "\n", + "image_url = \"https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true\"\n", + "image_bytes, _ = load_image(image_url)\n", + "state = image_qa(image_bytes, \"What is in the image?\")\n", + "print_highlight(state[\"answer\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/learn_more.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/learn_more.md new file mode 100644 index 00000000..f0d6ffb8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/learn_more.md @@ -0,0 +1,9 @@ +# Learn More and Join the Community + +- The development roadmap: [https://roadmap.sglang.io](https://roadmap.sglang.io) +- Join weekly public development meeting: [https://meet.sglang.io](https://meet.sglang.io) +- Join Slack: [https://slack.sglang.io/](https://slack.sglang.io/) +- Follow on X (formerly Twitter): [https://x.com/lmsysorg](https://x.com/lmsysorg) +- Follow on LinkedIn: [https://www.linkedin.com/company/sgl-project/](https://www.linkedin.com/company/sgl-project/) +- The latest SGLang features and updates are shared through the [LMSYS blog](https://lmsys.org/blog/) +- More blogs, slides, and videos about SGLang at [https://github.com/sgl-project/sgl-learning-materials](https://github.com/sgl-project/sgl-learning-materials) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/mindspore_models.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/mindspore_models.md new file mode 100644 index 00000000..80dd3b7f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/mindspore_models.md @@ -0,0 +1,164 @@ +# MindSpore Models + +## Introduction + +MindSpore is a high-performance AI framework optimized for Ascend NPUs. This doc guides users to run MindSpore models in SGLang. + +## Requirements + +MindSpore currently only supports Ascend NPU devices. Users need to first install Ascend CANN software packages. +The CANN software packages can be downloaded from the [Ascend Official Website](https://www.hiascend.com). The recommended version is 8.3.RC1. + +## Supported Models + +Currently, the following models are supported: + +- **Qwen3**: Dense and MoE models +- **DeepSeek V3/R1** +- *More models coming soon...* + +## Installation + +> **Note**: Currently, MindSpore models are provided by an independent package `sgl-mindspore`, which needs to be installed separately. + +```shell +git clone https://github.com/chz34/sgl-mindspore.git +cd sgl-mindspore +pip install -e . +``` + +You will need to install the following packages. + +```shell +pip install "mindspore==2.7.1" +pip install "torch==2.8" +pip install "torch_npu==2.8" +pip install triton_ascend +``` + +```shell +cp python/pyproject_other.toml python/pyproject.toml +pip install -e "python[all_npu]" +``` + +## Run Model + +Current SGLang-MindSpore supports Qwen3 and DeepSeek V3/R1 models. This doc uses Qwen3-8B as an example. + +### Offline infer + +Use the following script for offline infer: + +```python +import sglang as sgl + +# Initialize the engine with MindSpore backend +llm = sgl.Engine( + model_path="/path/to/your/model", # Local model path + device="npu", # Use NPU device + model_impl="mindspore", # MindSpore implementation + attention_backend="ascend", # Attention backend + tp_size=1, # Tensor parallelism size + dp_size=1 # Data parallelism size +) + +# Generate text +prompts = [ + "Hello, my name is", + "The capital of France is", + "The future of AI is" +] + +sampling_params = {"temperature": 0.01, "top_p": 0.9} +outputs = llm.generate(prompts, sampling_params) + +for prompt, output in zip(prompts, outputs): + print(f"Prompt: {prompt}") + print(f"Generated: {output['text']}") + print("---") +``` + +### Start server + +Launch a server with MindSpore backend: + +```bash +# Basic server startup +python3 -m sglang.launch_server \ + --model-path /path/to/your/model \ + --host 0.0.0.0 \ + --device npu \ + --model-impl mindspore \ + --attention-backend ascend \ + --tp-size 1 \ + --dp-size 1 +``` + +For distributed server with multiple nodes: + +```bash +# Multi-node distributed server +python3 -m sglang.launch_server \ + --model-path /path/to/your/model \ + --host 0.0.0.0 \ + --device npu \ + --model-impl mindspore \ + --attention-backend ascend \ + --dist-init-addr 127.0.0.1:29500 \ + --nnodes 2 \ + --node-rank 0 \ + --tp-size 4 \ + --dp-size 2 +``` + +## Troubleshooting + +#### Debug Mode + +Enable sglang debug logging by log-level argument. + +```bash +python3 -m sglang.launch_server \ + --model-path /path/to/your/model \ + --host 0.0.0.0 \ + --device npu \ + --model-impl mindspore \ + --attention-backend ascend \ + --log-level DEBUG +``` + +Enable mindspore info and debug logging by setting environments. + +```bash +export GLOG_v=1 # INFO +export GLOG_v=0 # DEBUG +``` + +#### Explicitly select devices + +Use the following environment variable to explicitly select the devices to use. + +```shell +export ASCEND_RT_VISIBLE_DEVICES=4,5,6,7 # to set device +``` + +#### Some communication environment issues + +In case of some environment with special communication environment, users need set some environment variables. + +```shell +export MS_ENABLE_LCCL=off # current not support LCCL communication mode in SGLang-MindSpore +``` + +#### Some dependencies of protobuf + +In case of some environment with special protobuf version, users need set some environment variables to avoid binary version mismatch. + +```shell +export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python # to avoid protobuf binary version mismatch +``` + +## Support +For MindSpore-specific issues: + +- Refer to the [MindSpore documentation](https://www.mindspore.cn/) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/deploy_on_k8s.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/deploy_on_k8s.md new file mode 100644 index 00000000..cfc099f5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/deploy_on_k8s.md @@ -0,0 +1,337 @@ +# Deploy On Kubernetes + +This document is for deploying a RoCE network-based SGLang two-node inference service on a Kubernetes (K8S) cluster. + +[LeaderWorkerSet (LWS)](https://github.com/kubernetes-sigs/lws) is a Kubernetes API that aims to address common deployment patterns of AI/ML inference workloads. A major use case is for multi-host/multi-node distributed inference. + +SGLang can also be deployed with LWS on Kubernetes for distributed model serving. + +Please see this guide for more details on deploying SGLang on Kubernetes using LWS. + +Here we take the deployment of DeepSeek-R1 as an example. + +## Prerequisites + +1. At least two Kubernetes nodes, each with two H20 systems and eight GPUs, are required. + +2. Make sure your K8S cluster has LWS correctly installed. If it hasn't been set up yet, please follow the [installation instructions](https://github.com/kubernetes-sigs/lws/blob/main/site/content/en/docs/installation/_index.md). **Note:** For LWS versions ≤0.5.x, you must use the Downward API to obtain `LWS_WORKER_INDEX`, as native support for this feature was introduced in v0.6.0. + +## Basic example + +For the basic example documentation, refer to [Deploy Distributed Inference Service with SGLang and LWS on GPUs](https://github.com/kubernetes-sigs/lws/tree/main/docs/examples/sglang). + +However, that document only covers the basic NCCL socket mode. + +In this section, we’ll make some simple modifications to adapt the setup to the RDMA scenario. + +## RDMA RoCE case + +* Check your env: + +```bash +[root@node1 ~]# ibstatus +Infiniband device 'mlx5_bond_0' port 1 status: + default gid: fe80:0000:0000:0000:0225:9dff:fe64:c79a + base lid: 0x0 + sm lid: 0x0 + state: 4: ACTIVE + phys state: 5: LinkUp + rate: 200 Gb/sec (2X NDR) + link_layer: Ethernet + +Infiniband device 'mlx5_bond_1' port 1 status: + default gid: fe80:0000:0000:0000:0225:9dff:fe6e:c3ec + base lid: 0x0 + sm lid: 0x0 + state: 4: ACTIVE + phys state: 5: LinkUp + rate: 200 Gb/sec (2X NDR) + link_layer: Ethernet + +Infiniband device 'mlx5_bond_2' port 1 status: + default gid: fe80:0000:0000:0000:0225:9dff:fe73:0dd7 + base lid: 0x0 + sm lid: 0x0 + state: 4: ACTIVE + phys state: 5: LinkUp + rate: 200 Gb/sec (2X NDR) + link_layer: Ethernet + +Infiniband device 'mlx5_bond_3' port 1 status: + default gid: fe80:0000:0000:0000:0225:9dff:fe36:f7ff + base lid: 0x0 + sm lid: 0x0 + state: 4: ACTIVE + phys state: 5: LinkUp + rate: 200 Gb/sec (2X NDR) + link_layer: Ethernet +``` + +* Prepare the `lws.yaml` file for deploying on k8s. + +```yaml +apiVersion: leaderworkerset.x-k8s.io/v1 +kind: LeaderWorkerSet +metadata: + name: sglang +spec: + replicas: 1 + leaderWorkerTemplate: + size: 2 + restartPolicy: RecreateGroupOnPodRestart + leaderTemplate: + metadata: + labels: + role: leader + spec: + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostIPC: true + containers: + - name: sglang-leader + image: sglang:latest + securityContext: + privileged: true + env: + - name: NCCL_IB_GID_INDEX + value: "3" + command: + - python3 + - -m + - sglang.launch_server + - --model-path + - /work/models + - --mem-fraction-static + - "0.93" + - --torch-compile-max-bs + - "8" + - --max-running-requests + - "20" + - --tp + - "16" # Size of Tensor Parallelism + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20000 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + - --host + - "0.0.0.0" + - --port + - "40000" + resources: + limits: + nvidia.com/gpu: "8" + ports: + - containerPort: 40000 + readinessProbe: + tcpSocket: + port: 40000 + initialDelaySeconds: 15 + periodSeconds: 10 + volumeMounts: + - mountPath: /dev/shm + name: dshm + - name: model + mountPath: /work/models + - name: ib + mountPath: /dev/infiniband + volumes: + - name: dshm + emptyDir: + medium: Memory + - name: model + hostPath: + path: '< your models dir >' # modify it according your models dir + - name: ib + hostPath: + path: /dev/infiniband + workerTemplate: + spec: + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostIPC: true + containers: + - name: sglang-worker + image: sglang:latest + securityContext: + privileged: true + env: + - name: NCCL_IB_GID_INDEX + value: "3" + command: + - python3 + - -m + - sglang.launch_server + - --model-path + - /work/models + - --mem-fraction-static + - "0.93" + - --torch-compile-max-bs + - "8" + - --max-running-requests + - "20" + - --tp + - "16" # Size of Tensor Parallelism + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20000 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + resources: + limits: + nvidia.com/gpu: "8" + volumeMounts: + - mountPath: /dev/shm + name: dshm + - name: model + mountPath: /work/models + - name: ib + mountPath: /dev/infiniband + volumes: + - name: dshm + emptyDir: + medium: Memory + - name: ib + hostPath: + path: /dev/infiniband + - name: model + hostPath: + path: /data1/models/deepseek_v3_moe +--- +apiVersion: v1 +kind: Service +metadata: + name: sglang-leader +spec: + selector: + leaderworkerset.sigs.k8s.io/name: sglang + role: leader + ports: + - protocol: TCP + port: 40000 + targetPort: 40000 + +``` + +* Then use `kubectl apply -f lws.yaml` you will get this output. + +```text +NAME READY STATUS RESTARTS AGE +sglang-0 0/1 Running 0 9s +sglang-0-1 1/1 Running 0 9s +``` + +Wait for the sglang leader (`sglang-0`) status to change to 1/1, which indicates it is `Ready`. + +You can use the command `kubectl logs -f sglang-0` to view the logs of the leader node. + +Once successful, you should see output like this: + +```text +[2025-02-17 05:27:24 TP1] Capture cuda graph end. Time elapsed: 84.89 s +[2025-02-17 05:27:24 TP6] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24 TP0] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24 TP7] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24 TP3] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24 TP2] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24 TP4] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24 TP1] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24 TP5] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24] INFO: Started server process [1] +[2025-02-17 05:27:24] INFO: Waiting for application startup. +[2025-02-17 05:27:24] INFO: Application startup complete. +[2025-02-17 05:27:24] INFO: Uvicorn running on http://0.0.0.0:40000 (Press CTRL+C to quit) +[2025-02-17 05:27:25] INFO: 127.0.0.1:48908 - "GET /get_model_info HTTP/1.1" 200 OK +[2025-02-17 05:27:25 TP0] Prefill batch. #new-seq: 1, #new-token: 7, #cached-token: 0, cache hit rate: 0.00%, token usage: 0.00, #running-req: 0, #queue-req: 0 +[2025-02-17 05:27:32] INFO: 127.0.0.1:48924 - "POST /generate HTTP/1.1" 200 OK +[2025-02-17 05:27:32] The server is fired up and ready to roll! +``` + +If it doesn’t start up successfully, please follow these steps to check for any remaining issues. Thanks! + +### Debug + +* Set `NCCL_DEBUG=TRACE` to check if it is a NCCL communication problem. + +This should resolve most NCCL-related issues. + +***Notice: If you find that NCCL_DEBUG=TRACE is not effective in the container environment, but the process is stuck or you encounter hard-to-diagnose issues, try switching to a different container image. Some images may not handle standard error output properly.*** + +#### RoCE scenario + +* Please make sure that RDMA devices are available in the cluster environment. +* Please make sure that the nodes in the cluster have Mellanox NICs with RoCE. In this example, we use Mellanox ConnectX 5 model NICs, and the proper OFED driver has been installed. If not, please refer to the document [Install OFED Driver](https://docs.nvidia.com/networking/display/mlnxofedv461000/installing+mellanox+ofed) to install the driver. +* Check your env: + + ```shell + $ lspci -nn | grep Eth | grep Mellanox + 0000:7f:00.0 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + 0000:7f:00.1 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + 0000:c7:00.0 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + 0000:c7:00.1 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + 0001:08:00.0 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + 0001:08:00.1 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + 0001:a2:00.0 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + 0001:a2:00.1 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + ``` + +* Check the OFED driver: + + ```shell + ofed_info -s + OFED-internal-23.07-0.5.0: + ``` + +* Show RDMA link status and check IB devices: + + ```shell + $ rdma link show + 8/1: mlx5_bond_0/1: state ACTIVE physical_state LINK_UP netdev reth0 + 9/1: mlx5_bond_1/1: state ACTIVE physical_state LINK_UP netdev reth2 + 10/1: mlx5_bond_2/1: state ACTIVE physical_state LINK_UP netdev reth4 + 11/1: mlx5_bond_3/1: state ACTIVE physical_state LINK_UP netdev reth6 + + $ ibdev2netdev + 8/1: mlx5_bond_0/1: state ACTIVE physical_state LINK_UP netdev reth0 + 9/1: mlx5_bond_1/1: state ACTIVE physical_state LINK_UP netdev reth2 + 10/1: mlx5_bond_2/1: state ACTIVE physical_state LINK_UP netdev reth4 + 11/1: mlx5_bond_3/1: state ACTIVE physical_state LINK_UP netdev reth6 + ``` + +* Test RoCE network speed on the host: + + ```shell + yum install qperf + # for server： + execute qperf + # for client + qperf -t 60 -cm1 rc_rdma_write_bw + ``` + +* Check RDMA accessible in your container: + + ```shell + # ibv_devices + # ibv_devinfo + ``` + +## Keys to success + +* In the YAML configuration above, pay attention to the NCCL environment variable. For older versions of NCCL, you should check the NCCL_IB_GID_INDEX environment setting. +* NCCL_SOCKET_IFNAME is also crucial, but in a containerized environment, this typically isn’t an issue. +* In some cases, it’s necessary to configure GLOO_SOCKET_IFNAME correctly. +* NCCL_DEBUG is essential for troubleshooting, but I've found that sometimes it doesn't show error logs within containers. This could be related to the Docker image you're using. You may want to try switching images if needed. +* Avoid using Docker images based on Ubuntu 18.04, as they tend to have compatibility issues. + +## Remaining issues + +* In Kubernetes, Docker, or Containerd environments, we use hostNetwork to prevent performance degradation. +* We utilize privileged mode, which isn’t secure. Additionally, in containerized environments, full GPU isolation cannot be achieved. + +## TODO + +* Integrated with [k8s-rdma-shared-dev-plugin](https://github.com/Mellanox/k8s-rdma-shared-dev-plugin). diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws-examples/d-svc.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws-examples/d-svc.yaml new file mode 100644 index 00000000..27f98009 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws-examples/d-svc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: deepseekr10528-decode-main +spec: + selector: + leaderworkerset.sigs.k8s.io/name: deepseekr10528-decode-main + role: leader + ports: + - protocol: TCP + port: 30000 + targetPort: 30000 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws-examples/d.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws-examples/d.yaml new file mode 100644 index 00000000..dbb51b51 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws-examples/d.yaml @@ -0,0 +1,290 @@ +apiVersion: leaderworkerset.x-k8s.io/v1 +kind: LeaderWorkerSet +metadata: + name: deepseekr10528-decode-main +spec: + leaderWorkerTemplate: + leaderTemplate: + metadata: + labels: + role: leader + spec: + containers: + - command: + - python3 + - -m + - sglang.launch_server + - --port + - "30000" + - --host + - "0.0.0.0" + - --model-path + - /work/models + - --chunked-prefill-size + - "262144" + - --page-size + - "64" + - --enable-dp-attention + - --enable-dp-lm-head + - --dp-size + - "16" + - --moe-a2a-backend + - deepep + - --disaggregation-mode + - decode + - --mem-fraction-static + - "0.849" + - --context-length + - "32768" + - --disaggregation-ib-device + - "mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3" + - --cuda-graph-max-bs + - "64" + - --max-running-requests + - "2048" + - --tp-size + - "16" # Size of Tensor Parallelism + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20102 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + - --ep-num-redundant-experts + - "32" + - --moe-dense-tp-size + - "1" + env: + - name: CUDA_LAUNCH_BLOCKING + value: "0" + - name: NVSHMEM_IB_GID_INDEX + value: "3" + - name: NVSHMEM_ENABLE_NIC_PE_MAPPING + value: "1" + - name: NVSHMEM_HCA_PE_MAPPING + value: "mlx5_bond_0:1:2,mlx5_bond_1:1:2,mlx5_bond_2:1:2,mlx5_bond_3:1:2" + - name: NCCL_IB_QPS_PER_CONNECTION + value: "8" + - name: NCCL_IB_SPLIT_DATA_ON_QPS + value: "1" + - name: NCCL_NET_PLUGIN + value: "none" + - name: NCCL_IB_TC + value: "136" + - name: NCCL_MIN_NCHANNELS + value: "4" + - name: NCCL_IB_SL + value: "5" + - name: MC_TE_METRIC + value: "true" + - name: SGLANG_MOONCAKE_TRANS_THREAD + value: "16" + - name: SGLANG_ENABLE_JIT_DEEPGEMM + value: "1" + - name: NCCL_IB_HCA + value: ^=mlx5_0,mlx5_5,mlx5_6 + - name: LWS_WORKER_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] + image: lmsysorg/sglang:latest + name: sglang-leader + ports: + - containerPort: 30000 + protocol: TCP + readinessProbe: + periodSeconds: 30 + tcpSocket: + port: 30000 + resources: + limits: + nvidia.com/gpu: "8" + securityContext: + capabilities: + add: + - IPC_LOCK + privileged: true + volumeMounts: + - mountPath: /root/.cache + name: sgl-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /work/models + name: model + - mountPath: /dev/infiniband + name: ib + - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs + name: cf + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + nodeSelector: + # should modify according your deployment env + pd: "yes" + tolerations: + # should modify according your deployment env + - key: bopd + operator: Exists + - key: node-role + operator: Exists + volumes: + - hostPath: + path: /data1/sgl_cache1 + type: DirectoryOrCreate + name: sgl-cache + - emptyDir: + medium: Memory + name: dshm + - hostPath: + path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 + name: model + - hostPath: + path: /dev/infiniband + name: ib + - hostPath: + path: /data1/maas_hosted_models/models/fused_moe_triton/configs + name: cf + restartPolicy: RecreateGroupOnPodRestart + size: 2 + workerTemplate: + metadata: {} + spec: + containers: + - command: + - python3 + - -m + - sglang.launch_server + - --model-path + - /work/models + - --chunked-prefill-size + - "262144" + - --page-size + - "64" + - --enable-dp-attention + - --enable-dp-lm-head + - --dp-size + - "16" + - --moe-a2a-backend + - deepep + - --disaggregation-mode + - decode + - --mem-fraction-static + - "0.849" + - --context-length + - "32768" + - --disaggregation-ib-device + - "mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3" + - --cuda-graph-max-bs + - "64" + - --max-running-requests + - "2048" + - --tp-size + - "16" # Size of Tensor Parallelism + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20102 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + - --ep-num-redundant-experts + - "32" + - --moe-dense-tp-size + - "1" + env: + - name: NVSHMEM_IB_TRAFFIC_CLASS + value: "16" + - name: NVSHMEM_IB_GID_INDEX + value: "3" + - name: NVSHMEM_ENABLE_NIC_PE_MAPPING + value: "1" + - name: NVSHMEM_HCA_PE_MAPPING + value: "mlx5_bond_0:1:2,mlx5_bond_1:1:2,mlx5_bond_2:1:2,mlx5_bond_3:1:2" + - name: NCCL_IB_QPS_PER_CONNECTION + value: "8" + - name: NCCL_IB_SPLIT_DATA_ON_QPS + value: "1" + - name: NCCL_NET_PLUGIN + value: "none" + - name: NCCL_IB_TC + value: "136" + - name: NCCL_MIN_NCHANNELS + value: "4" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_IB_SL + value: "5" + - name: SGLANG_MOONCAKE_TRANS_THREAD + value: "16" + - name: SGLANG_ENABLE_JIT_DEEPGEMM + value: "1" + - name: NCCL_IB_HCA + value: ^=mlx5_0,mlx5_5,mlx5_6 + - name: LWS_WORKER_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] + image: lmsysorg/sglang:latest + name: sglang-worker + ports: + - containerPort: 30001 + resources: + limits: + nvidia.com/gpu: "8" + securityContext: + capabilities: + add: + - IPC_LOCK + privileged: true + volumeMounts: + - mountPath: /root/.cache + name: sgl-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /work/models + name: model + - mountPath: /dev/infiniband + name: ib + - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs + name: cf + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + nodeSelector: + # should modify according your deployment env + pd: "yes" + tolerations: + # should modify according your deployment env + - key: bopd + operator: Exists + - key: node-role + operator: Exists + volumes: + - hostPath: + path: /data1/sgl_cache1 + type: DirectoryOrCreate + name: sgl-cache + - emptyDir: + medium: Memory + name: dshm + - hostPath: + path: /dev/infiniband + name: ib + - hostPath: + # modify according to you deployment env + path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 + name: model + - hostPath: + # modify according to you deployment env + path: /data1/maas_hosted_models/models/fused_moe_triton/configs + name: cf + networkConfig: + subdomainPolicy: Shared + replicas: 1 + rolloutStrategy: + rollingUpdateConfiguration: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + startupPolicy: LeaderCreated diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws-examples/lb.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws-examples/lb.yaml new file mode 100644 index 00000000..4ca69096 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws-examples/lb.yaml @@ -0,0 +1,56 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deepseekr10528-lb-main + labels: + app: deepseekr10528-lb +spec: + replicas: 1 + selector: + matchLabels: + app: deepseekr10528-lb + template: + metadata: + labels: + app: deepseekr10528-lb + spec: + nodeSelector: + bo: "yes" + tolerations: + - key: bopd + operator: Exists + - key: node-role + operator: Exists + containers: + - name: sgl-minilb + image: lmsysorg/sglang:latest + command: + - python + - -m + - sglang_router.launch_router + - --pd-disaggregation + - --prefill + - http://deepseekr10528-prefill-main:30000 + - --decode + - http://deepseekr10528-decode-main:30000 + - --host + - 0.0.0.0 + - --port + - "8000" + ports: + - containerPort: 8000 + +--- +apiVersion: v1 +kind: Service +metadata: + name: deepseekr10528-lb-service +spec: + type: NodePort # NodePort is easy to test, you can also specify `ClusterIP` + selector: + app: deepseekr10528-lb + ports: + - protocol: TCP + port: 8000 # Service Port（In-Cluster） + targetPort: 8000 # Exposed Container + nodePort: 30800 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws-examples/p-svc.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws-examples/p-svc.yaml new file mode 100644 index 00000000..6826a13d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws-examples/p-svc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: deepseekr10528-prefill-main +spec: + selector: + leaderworkerset.sigs.k8s.io/name: deepseekr10528-prefill-main + role: leader + ports: + - protocol: TCP + port: 30000 + targetPort: 30000 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws-examples/p.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws-examples/p.yaml new file mode 100644 index 00000000..1c5b5870 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws-examples/p.yaml @@ -0,0 +1,304 @@ +apiVersion: leaderworkerset.x-k8s.io/v1 +kind: LeaderWorkerSet +metadata: + name: deepseekr10528-prefill-main +spec: + leaderWorkerTemplate: + leaderTemplate: + metadata: + labels: + role: leader + spec: + containers: + - command: + - python3 + - -m + - sglang.launch_server + - --port + - "30000" + - --host + - "0.0.0.0" + - --model-path + - /work/models + - --disaggregation-ib-device + # should modify according your rdma env + - mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3 + - --chunked-prefill-size + - "524288" + - --max-prefill-tokens + - "32768" + - --page-size + - "64" + - --ep-dispatch-algorithm + - dynamic + - --eplb-algorithm + - deepseek + - --enable-dp-lm-head + - --enable-dp-attention + - --dp-size + - "16" + - --disable-radix-cache + - --moe-a2a-backend + - deepep + - --disaggregation-mode + - prefill + - --mem-fraction-static + - "0.7" + - --context-length + - "32768" + - --tp + - "16" + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20102 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + - --ep-num-redundant-experts + - "32" + - --moe-dense-tp-size + - "1" + - --max-running-requests + - "1024" + env: + - name: NVSHMEM_HCA_PE_MAPPING + # should modify according your rdma env + value: "mlx5_bond_0:1:2,mlx5_bond_1:1:2,mlx5_bond_2:1:2,mlx5_bond_3:1:2" + - name: NVSHMEM_IB_GID_INDEX + value: "3" + - name: NVSHMEM_ENABLE_NIC_PE_MAPPING + value: "1" + - name: SGLANG_SET_CPU_AFFINITY + value: "true" + - name: SGLANG_ENABLE_JIT_DEEPGEMM + value: "1" + - name: NCCL_IB_QPS_PER_CONNECTION + value: "8" + - name: NCCL_IB_SPLIT_DATA_ON_QPS + value: "1" + - name: NCCL_NET_PLUGIN + value: none + - name: NCCL_IB_TC + value: "136" + - name: NCCL_MIN_NCHANNELS + value: "4" + - name: MC_TE_METRIC + value: "false" + - name: NCCL_IB_SL + value: "5" + - name: NCCL_IB_HCA + value: ^=mlx5_0,mlx5_5,mlx5_6 + - name: LWS_WORKER_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] + image: lmsysorg/sglang:latest + name: sglang-leader + ports: + - containerPort: 30000 + protocol: TCP + readinessProbe: + periodSeconds: 30 + tcpSocket: + port: 30000 + resources: + limits: + nvidia.com/gpu: "8" + securityContext: + capabilities: + add: + - IPC_LOCK + privileged: true + volumeMounts: + - mountPath: /dev/shm + name: dshm + - mountPath: /work/models + name: model + - mountPath: /dev/infiniband + name: ib + - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs + name: cf + - mountPath: /root/.cache + name: sgl-cache + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + nodeSelector: + # should modify according your deployment env + pd: "yes" + tolerations: + # should modify according your deployment env + - key: bopd + operator: Exists + - key: node-role + operator: Exists + volumes: + - emptyDir: + medium: Memory + name: dshm + - hostPath: + path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 + name: model + - hostPath: + path: /dev/infiniband + name: ib + - hostPath: + path: /data1/maas_hosted_models/models/fused_moe_triton/configs + name: cf + - hostPath: + path: /data1/sgl_cache + type: DirectoryOrCreate + name: sgl-cache + restartPolicy: RecreateGroupOnPodRestart + size: 2 + workerTemplate: + metadata: {} + spec: + containers: + - command: + - python3 + - -m + - sglang.launch_server + - --model-path + - /work/models + - --disaggregation-ib-device + # should modify according your rdma env + - mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3 + - --chunked-prefill-size + - "524288" + - --max-prefill-tokens + - "32768" + - --page-size + - "64" + - --ep-dispatch-algorithm + - dynamic + - --eplb-algorithm + - deepseek + # - --deepep-config + # - /home/aiges/tuned/tuned_8sms.json + # can be tuned using deepep test scripts + - --enable-dp-lm-head + - --enable-dp-attention + - --dp-size + - "16" + - --disable-radix-cache + - --moe-a2a-backend + - deepep + - --disaggregation-mode + - prefill + - --mem-fraction-static + - "0.7" + - --context-length + - "32768" + - --tp + - "16" + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20102 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + - --ep-num-redundant-experts + - "32" + - --moe-dense-tp-size + - "1" + - --max-running-requests + - "1024" + env: + - name: SGLANG_SET_CPU_AFFINITY + value: "true" + - name: NVSHMEM_HCA_PE_MAPPING + # should modify according your rdma env + value: "mlx5_bond_0:1:2,mlx5_bond_1:1:2,mlx5_bond_2:1:2,mlx5_bond_3:1:2" + - name: NCCL_IB_HCA + value: ^=mlx5_0,mlx5_5,mlx5_6 + - name: NVSHMEM_IB_TRAFFIC_CLASS + value: "16" + - name: NVSHMEM_IB_GID_INDEX + value: "3" + - name: NVSHMEM_ENABLE_NIC_PE_MAPPING + value: "1" + - name: CUDA_LAUNCH_BLOCKING + value: "0" + - name: SGLANG_MOONCAKE_TRANS_THREAD + value: "8" + - name: SGLANG_ENABLE_JIT_DEEPGEMM + value: "1" + - name: SGL_CHUNKED_PREFIX_CACHE_THRESHOLD + value: "0" + - name: NCCL_IB_QPS_PER_CONNECTION + value: "8" + - name: NCCL_IB_SPLIT_DATA_ON_QPS + value: "1" + - name: NCCL_NET_PLUGIN + value: none + - name: NCCL_IB_TC + value: "136" + - name: NCCL_MIN_NCHANNELS + value: "4" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_IB_SL + value: "5" + - name: LWS_WORKER_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] + image: lmsysorg/sglang:latest + name: sglang-worker + ports: + - containerPort: 30001 + protocol: TCP + resources: + limits: + nvidia.com/gpu: "8" + securityContext: + capabilities: + add: + - IPC_LOCK + privileged: true + volumeMounts: + - mountPath: /root/.cache + name: sgl-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /work/models + name: model + - mountPath: /dev/infiniband + name: ib + - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs + name: cf + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + nodeSelector: + # should modify according your deployment env + pd: "yes" + tolerations: + # should modify according your deployment env + - key: bopd + operator: Exists + - key: node-role + operator: Exists + volumes: + - emptyDir: + medium: Memory + name: dshm + - hostPath: + path: /dev/infiniband + name: ib + - hostPath: + # modify according to you deployment env + path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 + name: model + - hostPath: + # modify according to you deployment env + path: /data1/maas_hosted_models/models/fused_moe_triton/configs + name: cf + - hostPath: + # modify according to you deployment env + path: /data1/sgl_cache + type: DirectoryOrCreate + name: sgl-cache diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.md new file mode 100644 index 00000000..b3508968 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.md @@ -0,0 +1,783 @@ +# LWS Based PD Deploy + +## 0. Prerequisites + +1. k8s >=1.26 +2. lws installed on k8s. + +## 1. Image Preparation + +`lmsysorg/sglang:deepep` + +## 2. Deployment Manifest Files + +***Notice: We will package all deployment files into Helm Chart format in the near future. Interested community members can contact us to contribute*** + +### Prefill + +Prefill manifest file [prefill.yaml](lws-examples/p.yaml) + +*Note: The NodeSelector section, model location section, and taint toleration section can be adjusted according to your actual deployment environment* + +```yaml +apiVersion: leaderworkerset.x-k8s.io/v1 +kind: LeaderWorkerSet +metadata: + name: deepseekr10528-prefill-main +spec: + leaderWorkerTemplate: + leaderTemplate: + metadata: + labels: + role: leader + spec: + containers: + - command: + - python3 + - -m + - sglang.launch_server + - --port + - "30000" + - --host + - "0.0.0.0" + - --model-path + - /work/models + - --disaggregation-ib-device + # should modify according your rdma env + - mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3 + - --chunked-prefill-size + - "524288" + - --max-prefill-tokens + - "32768" + - --page-size + - "64" + # - --init-expert-location + # - /home/aiges/tuned/attachment_ep_statistics/prefill_in1024.json + - --ep-dispatch-algorithm + - dynamic + - --eplb-algorithm + - deepseek + # - --deepep-config + # - /home/aiges/tuned/tuned_8sms.json + - --enable-dp-lm-head + - --enable-dp-attention + - --dp-size + - "16" + - --disable-radix-cache + - --moe-a2a-backend + - deepep + - --disaggregation-mode + - prefill + - --mem-fraction-static + - "0.7" + - --context-length + - "32768" + - --tp + - "16" + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20102 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + - --ep-num-redundant-experts + - "32" + - --moe-dense-tp-size + - "1" + - --max-running-requests + - "1024" + env: +# - name: NVSHMEM_HCA_PE_MAPPING +# value: "mlx5_bond_0:1:2,mlx5_bond_1:1:2,mlx5_bond_2:1:2,mlx5_bond_3:1:2" +# - name: NVSHMEM_HCA_LIST +# value: "mlx5_bond_0:1,mlx5_bond_1:1,mlx5_bond_2:1,mlx5_bond_3:1" + - name: NVSHMEM_IB_GID_INDEX + value: "3" + - name: NVSHMEM_ENABLE_NIC_PE_MAPPING + value: "1" + - name: SGLANG_SET_CPU_AFFINITY + value: "true" + - name: SGLANG_ENABLE_JIT_DEEPGEMM + value: "1" + - name: NCCL_IB_QPS_PER_CONNECTION + value: "8" + - name: NCCL_IB_SPLIT_DATA_ON_QPS + value: "1" + - name: NCCL_NET_PLUGIN + value: none + - name: NCCL_IB_TC + value: "136" + - name: NCCL_MIN_NCHANNELS + value: "4" + - name: MC_TE_METRIC + value: "false" + - name: NCCL_IB_SL + value: "5" + - name: NCCL_IB_HCA + value: ^=mlx5_0,mlx5_5,mlx5_6 + - name: LWS_WORKER_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] + image: lmsysorg/sglang:deepep + name: sglang-leader + ports: + - containerPort: 30000 + protocol: TCP + readinessProbe: + periodSeconds: 30 + tcpSocket: + port: 30000 + resources: + limits: + nvidia.com/gpu: "8" + securityContext: + capabilities: + add: + - IPC_LOCK + privileged: true + volumeMounts: + - mountPath: /dev/shm + name: dshm + - mountPath: /work/models + name: model + - mountPath: /dev/infiniband + name: ib + - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs + name: cf + - mountPath: /root/.cache + name: sgl-cache + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + nodeSelector: + pd: "yes" + tolerations: + - key: pd + operator: Exists + - key: node-role + operator: Exists + volumes: + - emptyDir: + medium: Memory + name: dshm + - hostPath: + # modify according to you deployment env + path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 + name: model + - hostPath: + path: /dev/infiniband + name: ib + - hostPath: + # modify according to you deployment env + path: /data1/maas_hosted_models/models/fused_moe_triton/configs + name: cf + - hostPath: + # modify according to you deployment env + path: /data1/sgl_cache + type: DirectoryOrCreate + name: sgl-cache + restartPolicy: RecreateGroupOnPodRestart + size: 2 + workerTemplate: + metadata: {} + spec: + containers: + - command: + - python3 + - -m + - sglang.launch_server + - --model-path + - /work/models + - --disaggregation-ib-device + - mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3 + - --chunked-prefill-size + - "524288" + - --max-prefill-tokens + - "32768" + - --page-size + - "64" + #- --init-expert-location + #- /home/aiges/tuned/attachment_ep_statistics/prefill_in1024.json + - --ep-dispatch-algorithm + - dynamic + - --eplb-algorithm + - deepseek +# - --deepep-config +# - /home/aiges/tuned/tuned_8sms.json + - --enable-dp-lm-head + - --enable-dp-attention + - --dp-size + - "16" + - --disable-radix-cache + - --moe-a2a-backend + - deepep + - --disaggregation-mode + - prefill + - --mem-fraction-static + - "0.7" + - --context-length + - "32768" + - --tp + - "16" + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20102 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + - --ep-num-redundant-experts + - "32" + - --moe-dense-tp-size + - "1" + - --max-running-requests + - "1024" + env: + - name: SGLANG_SET_CPU_AFFINITY + value: "true" + - name: SGLANG_HACK_DEEPEP_NUM_SMS + value: "8" + - name: SGLANG_HACK_DEEPEP_NEW_MODE + value: "0" +# - name: NVSHMEM_HCA_PE_MAPPING +# value: "mlx5_bond_0:1:2,mlx5_bond_1:1:2,mlx5_bond_2:1:2,mlx5_bond_3:1:2" +# - name: NVSHMEM_HCA_LIST +# value: "mlx5_bond_0:1,mlx5_bond_1:1,mlx5_bond_2:1,mlx5_bond_3:1" + - name: NCCL_IB_HCA + value: ^=mlx5_0,mlx5_5,mlx5_6 + - name: NVSHMEM_IB_TRAFFIC_CLASS + value: "16" + - name: NVSHMEM_IB_GID_INDEX + value: "3" + - name: NVSHMEM_ENABLE_NIC_PE_MAPPING + value: "1" + - name: CUDA_LAUNCH_BLOCKING + value: "0" + - name: SGLANG_MOONCAKE_TRANS_THREAD + value: "8" + - name: SGLANG_ENABLE_JIT_DEEPGEMM + value: "1" + - name: SGL_CHUNKED_PREFIX_CACHE_THRESHOLD + value: "0" + - name: NCCL_IB_QPS_PER_CONNECTION + value: "8" + - name: NCCL_IB_SPLIT_DATA_ON_QPS + value: "1" + - name: NCCL_NET_PLUGIN + value: none + - name: NCCL_IB_TC + value: "136" + - name: NCCL_MIN_NCHANNELS + value: "4" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_IB_SL + value: "5" + - name: LWS_WORKER_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] + image: lmsysorg/sglang:deepep + name: sglang-worker + ports: + - containerPort: 30001 + protocol: TCP + resources: + limits: + nvidia.com/gpu: "8" + securityContext: + capabilities: + add: + - IPC_LOCK + privileged: true + volumeMounts: + + - mountPath: /root/.cache + name: sgl-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /work/models + name: model + - mountPath: /dev/infiniband + name: ib + - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs + name: cf + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + nodeSelector: + pd: "yes" + tolerations: + - key: pd + operator: Exists + - key: node-role + operator: Exists + volumes: + - emptyDir: + medium: Memory + name: dshm + - hostPath: + path: /dev/infiniband + name: ib + - hostPath: + path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 + name: model + - hostPath: + path: /data1/maas_hosted_models/models/fused_moe_triton/configs + name: cf + - hostPath: + path: /data1/sgl_cache + type: DirectoryOrCreate + name: sgl-cache + +``` + +### Decode + +Decode node deployment manifest file [decode.yaml](lws-examples/d.yaml) + +*Note: The NodeSelector section, model location section, and taint toleration section can be adjusted according to your actual deployment environment* + +```yaml +apiVersion: leaderworkerset.x-k8s.io/v1 +kind: LeaderWorkerSet +metadata: + name: deepseekr10528-decode-main +spec: + leaderWorkerTemplate: + leaderTemplate: + metadata: + labels: + role: leader + spec: + containers: + - command: + - python3 + - -m + - sglang.launch_server + - --port + - "30000" + - --host + - "0.0.0.0" + - --model-path + - /work/models + - --chunked-prefill-size + - "262144" + - --page-size + - "64" + - --enable-dp-attention + - --enable-dp-lm-head + - --dp-size + - "16" + - --moe-a2a-backend + - deepep + - --disaggregation-mode + - decode + - --mem-fraction-static + - "0.849" + - --context-length + - "32768" + - --disaggregation-ib-device + - "mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3" + - --cuda-graph-max-bs + - "64" + - --max-running-requests + - "2048" + - --tp-size + - "16" # Size of Tensor Parallelism + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20102 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + - --ep-num-redundant-experts + - "32" + - --moe-dense-tp-size + - "1" + env: + - name: CUDA_LAUNCH_BLOCKING + value: "0" + - name: NVSHMEM_IB_GID_INDEX + value: "3" + - name: NVSHMEM_ENABLE_NIC_PE_MAPPING + value: "1" + - name: NCCL_IB_QPS_PER_CONNECTION + value: "8" + - name: NCCL_IB_SPLIT_DATA_ON_QPS + value: "1" + - name: NCCL_NET_PLUGIN + value: "none" + - name: NCCL_IB_TC + value: "136" + - name: NCCL_MIN_NCHANNELS + value: "4" + - name: NCCL_IB_SL + value: "5" + - name: MC_TE_METRIC + value: "true" + - name: SGLANG_MOONCAKE_TRANS_THREAD + value: "16" + - name: SGLANG_ENABLE_JIT_DEEPGEMM + value: "1" + - name: NCCL_IB_HCA + value: ^=mlx5_0,mlx5_5,mlx5_6 + - name: LWS_WORKER_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] + image: lmsysorg/sglang:deepep + name: sglang-leader + ports: + - containerPort: 30000 + protocol: TCP + readinessProbe: + periodSeconds: 30 + tcpSocket: + port: 30000 + resources: + limits: + nvidia.com/gpu: "8" + securityContext: + capabilities: + add: + - IPC_LOCK + privileged: true + volumeMounts: + - mountPath: /root/.cache + name: sgl-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /work/models + name: model + - mountPath: /dev/infiniband + name: ib + - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs + name: cf + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + nodeSelector: + pd: "yes" + tolerations: + - key: pd + operator: Exists + - key: node-role + operator: Exists + volumes: + - hostPath: + path: /data1/sgl_cache1 + type: DirectoryOrCreate + name: sgl-cache + - emptyDir: + medium: Memory + name: dshm + - hostPath: + path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 + name: model + - hostPath: + path: /dev/infiniband + name: ib + - hostPath: + path: /data1/maas_hosted_models/models/fused_moe_triton/configs + name: cf + restartPolicy: RecreateGroupOnPodRestart + size: 2 + workerTemplate: + metadata: {} + spec: + containers: + - command: + - python3 + - -m + - sglang.launch_server + - --model-path + - /work/models + - --chunked-prefill-size + - "262144" + - --page-size + - "64" + - --enable-dp-attention + - --enable-dp-lm-head + #- --enable-two-batch-overlap + - --dp-size + - "16" + - --moe-a2a-backend + - deepep + - --disaggregation-mode + - decode + - --mem-fraction-static + - "0.849" + - --context-length + - "32768" + - --disaggregation-ib-device + # should modify according your rdma env + - "mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3" + - --cuda-graph-max-bs + - "64" + - --max-running-requests + - "2048" + - --tp-size + - "16" # Size of Tensor Parallelism + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20102 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + - --ep-num-redundant-experts + - "32" + - --moe-dense-tp-size + - "1" + env: + - name: SGLANG_HACK_DEEPEP_NUM_SMS + value: "24" + - name: SGLANG_HACK_DEEPEP_NEW_MODE + value: "0" + - name: NVSHMEM_IB_TRAFFIC_CLASS + value: "16" + - name: NVSHMEM_IB_GID_INDEX + value: "3" + - name: NVSHMEM_ENABLE_NIC_PE_MAPPING + value: "1" + - name: NCCL_IB_QPS_PER_CONNECTION + value: "8" + - name: NCCL_IB_SPLIT_DATA_ON_QPS + value: "1" + - name: NCCL_NET_PLUGIN + value: "none" + - name: NCCL_IB_TC + value: "136" + - name: NCCL_MIN_NCHANNELS + value: "4" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_IB_SL + value: "5" + - name: SGLANG_MOONCAKE_TRANS_THREAD + value: "16" + - name: SGLANG_ENABLE_JIT_DEEPGEMM + value: "1" + - name: NCCL_IB_HCA + value: ^=mlx5_0,mlx5_5,mlx5_6 + - name: LWS_WORKER_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] + image: lmsysorg/sglang:deepep + name: sglang-worker + ports: + - containerPort: 30001 + resources: + limits: + nvidia.com/gpu: "8" + securityContext: + capabilities: + add: + - IPC_LOCK + privileged: true + volumeMounts: + - mountPath: /root/.cache + name: sgl-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /work/models + name: model + - mountPath: /dev/infiniband + name: ib + - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs + name: cf + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + nodeSelector: + pd: "yes" + tolerations: + - key: pd + operator: Exists + - key: node-role + operator: Exists + volumes: + - hostPath: + path: /data1/sgl_cache1 + type: DirectoryOrCreate + name: sgl-cache + - emptyDir: + medium: Memory + name: dshm + - hostPath: + path: /dev/infiniband + name: ib + - hostPath: + # modify according to you deployment env + path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 + name: model + - hostPath: + # modify according to you deployment env + path: /data1/maas_hosted_models/models/fused_moe_triton/configs + name: cf + networkConfig: + subdomainPolicy: Shared + replicas: 1 + rolloutStrategy: + rollingUpdateConfiguration: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + startupPolicy: LeaderCreated +``` + +Execute separately: + +```bash +kubectl apply -f p.yaml +kubectl apply -f d.yaml +``` + +At this point, we have completed the deployment of the 1P1D SGlang engine part. + +To allow our users to directly experience the model API, we still need a load balancer to handle sequential calls between prefill and decode. Different companies implement LBs differently, and the community will also officially release a new LB component written in Rust in the near future. + +Currently, we use a static K8S service + minilb approach to implement model API calls. + +### Creating Service for Prefill and Decode + +#### Create prefill k8s service +[p-svc.yaml](lws-examples/p-svc.yaml) +```yaml +apiVersion: v1 +kind: Service +metadata: + name: deepseekr10528-prefill-main +spec: + selector: + leaderworkerset.sigs.k8s.io/name: deepseekr10528-prefill-main + role: leader + ports: + - protocol: TCP + port: 30000 + targetPort: 30000 +``` +Execute `kubectl apply -f p-svc.yaml` + +#### Create decode k8s service +[d-svc.yaml](lws-examples/d-svc.yaml) +```yaml +apiVersion: v1 +kind: Service +metadata: + name: deepseekr10528-decode-main +spec: + selector: + leaderworkerset.sigs.k8s.io/name: deepseekr10528-decode-main + role: leader + ports: + - protocol: TCP + port: 30000 + targetPort: 30000 +``` +Execute `kubectl apply -f d-svc.yaml` + +#### Deploy minilb and lb service +[lb.yaml](lws-examples/lb.yaml) +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deepseekr10528-lb-main + labels: + app: deepseekr10528-lb +spec: + replicas: 1 + selector: + matchLabels: + app: deepseekr10528-lb + template: + metadata: + labels: + app: deepseekr10528-lb + spec: + nodeSelector: + pd: "yes" + tolerations: + - key: pd + operator: Exists + - key: node-role + operator: Exists + containers: + - name: sgl-minilb + image: lmsysorg/sglang:deepep + command: + - python + - -m + - sglang_router.launch_router + - --pd-disaggregation + - --prefill + - http://deepseekr10528-prefill-main:30000 + - --decode + - http://deepseekr10528-decode-main:30000 + - --host + - 0.0.0.0 + - --port + - "8000" + ports: + - containerPort: 8000 +--- +apiVersion: v1 +kind: Service +metadata: + name: deepseekr10528-lb-service +spec: + type: NodePort + selector: + app: deepseekr10528-lb + ports: + - protocol: TCP + port: 8000 # Service Port（In-Cluster） + targetPort: 8000 # Exposed Container + nodePort: 30800 +``` +Execute `kubectl apply -f lb.yaml` + +After waiting for all model deployments to succeed, you will get the following output: + +```bash +[root@ecs-001]# kubectl get po +deepseekr10528-decode-main-0 1/1 Running 0 74m +deepseekr10528-decode-main-0-1 1/1 Running 0 74m +deepseekr10528-lb-main-9c5dbfc57-6lcbd 1/1 Running 0 22m +deepseekr10528-prefill-main-0 1/1 Running 0 74m +deepseekr10528-prefill-main-0-1 1/1 Running 0 74m +[root@ecs-cbm-x1-pd-cpu-001 main_doc]# kubectl get svc |grep dee +deepseekr10528-decode-main ClusterIP None 97m +deepseekr10528-lb-service NodePort 172.16.242.169 8000:30800/TCP 22m +deepseekr10528-prefill-main ClusterIP None 97m +``` + +At this point, select a nodePort:30800 to access: + +```bash +[root@ecs-001]# curl -X POST "http://{nodePort}:30800/v1/chat/completions" \ +> -H "Content-Type: application/json" \ +> -H "Authorization: Bearer None" \ +> -d '{ +> "rid":"ccccdd", +> "model": "r1", +> "messages": [ +> {"role": "system", "content": "0: You are a helpful AI assistant"}, +> {"role": "user", "content": "你是谁？."} +> ], +> "max_tokens":221 +> }' +{"id":"ccccdd","object":"chat.completion","created":1750252498,"model":"qwen2","choices":[{"index":0,"message":{"role":"assistant","content":"\n嗯，用户问了一个很基础的自我介绍问题"你是谁？"。这可能是第一次互动时的常规开场白，也可能是想确认我的身份和功能范围。\n\n用户没有提供任何背景信息，语气简洁中性。这种场景下新用户的可能性较高，需要给出清晰友好的自我介绍，同时突出实用价值来降低陌生感。\n\n考虑到中文用户，应该用简体中文回复。重点要说明三点：身份归属（深度求索）、功能定位（AI助手）、服务范围（学习/工作/生活）。结尾用开放性问题引导对话很关键——既能了解需求，又能避免让用户面对空白输入框时不知所措。\n\n用波浪线结尾可以软化语气，那个笑脸表情😊刚好能中和AI的机械感。不过要控制表情符号数量，避免显得轻浮。\n\n你好呀！我是你的AI助手，由深度求索公司（DeepSeek）开发的语言模型，名字叫 **DeepSeek-R1**。你可以把我当成一个知识丰富、随叫随到的小帮手～😊\n\n我的任务就是陪你聊天、解答问题、","reasoning_content":null,"tool_calls":null},"logprobs":null,"finish_reason":"length","matched_stop":null}],"usage":{"prompt_tokens":14,"total_tokens":235,"completion_tokens":221,"prompt_tokens_details":null}} + +``` +## FAQ + +1. The current deployment startup parameters may not be fully compatible with all RDMA scenarios. Different RDMA NCCL-related environment configurations may be needed in different network environments. + +2. Some preset, optimized configurations for EPLB are not used here. You can adjust them according to [6017](https://github.com/sgl-project/sglang/issues/6017) as needed. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/multi_node.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/multi_node.md new file mode 100644 index 00000000..e6e5b534 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/multi_node.md @@ -0,0 +1,100 @@ +# Multi-Node Deployment + +## Llama 3.1 405B + +**Run 405B (fp16) on Two Nodes** + +```bash +# replace 172.16.4.52:20000 with your own node ip address and port of the first node + +python3 -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-405B-Instruct \ + --tp 16 \ + --dist-init-addr 172.16.4.52:20000 \ + --nnodes 2 \ + --node-rank 0 + +python3 -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-405B-Instruct \ + --tp 16 \ + --dist-init-addr 172.16.4.52:20000 \ + --nnodes 2 \ + --node-rank 1 +``` + +Note that LLama 405B (fp8) can also be launched on a single node. + +```bash +python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-405B-Instruct-FP8 --tp 8 +``` + +## DeepSeek V3/R1 + +Please refer to [DeepSeek documents for reference](https://docs.sglang.io/basic_usage/deepseek.html#running-examples-on-multi-node). + +## Multi-Node Inference on SLURM + +This example showcases how to serve SGLang server across multiple nodes by SLURM. Submit the following job to the SLURM cluster. + +``` +#!/bin/bash -l + +#SBATCH -o SLURM_Logs/%x_%j_master.out +#SBATCH -e SLURM_Logs/%x_%j_master.err +#SBATCH -D ./ +#SBATCH -J Llama-405B-Online-Inference-TP16-SGL + +#SBATCH --nodes=2 +#SBATCH --ntasks=2 +#SBATCH --ntasks-per-node=1 # Ensure 1 task per node +#SBATCH --cpus-per-task=18 +#SBATCH --mem=224GB +#SBATCH --partition="lmsys.org" +#SBATCH --gres=gpu:8 +#SBATCH --time=12:00:00 + +echo "[INFO] Activating environment on node $SLURM_PROCID" +if ! source ENV_FOLDER/bin/activate; then + echo "[ERROR] Failed to activate environment" >&2 + exit 1 +fi + +# Define parameters +model=MODEL_PATH +tp_size=16 + +echo "[INFO] Running inference" +echo "[INFO] Model: $model" +echo "[INFO] TP Size: $tp_size" + +# Set NCCL initialization address using the hostname of the head node +HEAD_NODE=$(scontrol show hostname "$SLURM_NODELIST" | head -n 1) +NCCL_INIT_ADDR="${HEAD_NODE}:8000" +echo "[INFO] NCCL_INIT_ADDR: $NCCL_INIT_ADDR" + +# Launch the model server on each node using SLURM +srun --ntasks=2 --nodes=2 --output="SLURM_Logs/%x_%j_node$SLURM_NODEID.out" \ + --error="SLURM_Logs/%x_%j_node$SLURM_NODEID.err" \ + python3 -m sglang.launch_server \ + --model-path "$model" \ + --grammar-backend "xgrammar" \ + --tp "$tp_size" \ + --dist-init-addr "$NCCL_INIT_ADDR" \ + --nnodes 2 \ + --node-rank "$SLURM_NODEID" & + +# Wait for the NCCL server to be ready on port 30000 +while ! nc -z "$HEAD_NODE" 30000; do + sleep 1 + echo "[INFO] Waiting for $HEAD_NODE:30000 to accept connections" +done + +echo "[INFO] $HEAD_NODE:30000 is ready to accept connections" + +# Keep the script running until the SLURM job times out +wait +``` + +Then, you can test the server by sending requests following other [documents](https://docs.sglang.io/basic_usage/openai_api_completions.html). + +Thanks for [aflah02](https://github.com/aflah02) for providing the example, based on his [blog post](https://aflah02.substack.com/p/multi-node-llm-inference-with-sglang). diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/multi_node_index.rst b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/multi_node_index.rst new file mode 100644 index 00000000..78636869 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/multi_node_index.rst @@ -0,0 +1,14 @@ +Multi-Node Deployment +===================== + +.. toctree:: + :maxdepth: 1 + :caption: Multi-Node Deployment + + multi_node.md + deploy_on_k8s.md + lws_pd/lws_pd_deploy.md + rbg_pd/deepseekv32_pd.md + +- `Deploying DeepSeek with PD Disaggregation and Large-Scale Expert Parallelism on 96 H100 GPUs `_ +- `Deploying Kimi K2 with PD Disaggregation and Large-Scale Expert Parallelism on 128 H200 GPUs `_ diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/rbg_pd/deepseekv32_pd.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/rbg_pd/deepseekv32_pd.md new file mode 100644 index 00000000..d4dcf73a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/multi_node_deployment/rbg_pd/deepseekv32_pd.md @@ -0,0 +1,569 @@ +# DeepSeekV32-Exp RBG Based PD Deploy + +## 0. Prerequisites + +1. k8s >=1.26 +2. lws installed on k8s. +3. rbg installed on k8s. + +For RBG installation, please refer to: https://github.com/sgl-project/rbg + +## 1. Image Preparation + +`lmsysorg/sglang:latest` + + +### 2. All In One manifest file + +*Note: The NodeSelector section, model location section, and taint toleration section can be adjusted according to your actual deployment environment* + +rbg-dsv32.yml + +```yaml +apiVersion: workloads.x-k8s.io/v1alpha1 +kind: RoleBasedGroup +metadata: + name: deepseek-rbg-32exp + namespace: default +spec: + roles: + - name: prefill + replicas: 1 + workload: + apiVersion: leaderworkerset.x-k8s.io/v1 + kind: LeaderWorkerSet + restartPolicy: None + leaderWorkerSet: + size: 1 + patchLeaderTemplate: + metadata: + labels: + role: leader + pd_role: prefill + spec: + containers: + - command: + - python3 + - -m + - sglang.launch_server + - --model-path + - /work/models + - --port + - "30000" + - --trust-remote + - --host + - 0.0.0.0 + - --disaggregation-ib-device + - mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_6,mlx5_7 + - --disable-radix-cache + - --chunked-prefill-size + - "131072" + - --page-size + - "64" + # - --enable-eplb + - --ep-dispatch-algorithm + - dynamic + - --eplb-algorithm + - deepseek + - --enable-dp-lm-head + - --enable-dp-attention + - --dp-size + - "8" + - --moe-a2a-backend + - deepep + - --deepep-mode + - normal + - --disaggregation-mode + - prefill + - --mem-fraction-static + - "0.8" + - --max-prefill-tokens + - "32768" + - --context-length + - "32768" + - --tp + - "8" + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20102 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + - --ep-num-redundant-experts + - "32" + - --moe-dense-tp-size + - "1" + - --max-running-requests + - "1024" + env: + - name: LWS_WORKER_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] + livenessProbe: + failureThreshold: 3000 + httpGet: + path: /health + port: 30000 + initialDelaySeconds: 300 + periodSeconds: 60 + successThreshold: 1 + timeoutSeconds: 10 + readinessProbe: + failureThreshold: 20 + httpGet: + path: /health + port: 30000 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 10 + name: sglang + ports: + - containerPort: 30000 + name: sglang-http + protocol: TCP + + patchWorkerTemplate: {} + template: + metadata: + labels: + inference-framework: sglang + inference-stack.io/monitoring: "enabled" + spec: + containers: + - name: sglang + image: lmsysorg/sglang:latest + env: + - name: SGLANG_SKIP_SGL_KERNEL_VERSION_CHECK + value: "1" + - name: CUDA_LAUNCH_BLOCKING + value: "0" + - name: SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT + value: "1000000000" + - name: NVSHMEM_IB_TRAFFIC_CLASS + value: "16" + - name: NVSHMEM_DISABLE_P2P + value: "0" + - name: ENABLE_METRICS + value: "true" + - name: NVSHMEM_IB_GID_INDEX + value: "3" + - name: NVSHMEM_IB_SL + value: "5" + - name: SGLANG_SET_CPU_AFFINITY + value: "true" + - name: SGL_ENABLE_JIT_DEEPGEMM + value: "1" + - name: NCCL_IB_QPS_PER_CONNECTION + value: "8" + - name: NCCL_IB_SPLIT_DATA_ON_QPS + value: "1" + - name: NCCL_NET_PLUGIN + value: "none" + - name: NCCL_IB_TC + value: "136" + - name: NCCL_IB_SL + value: "5" + - name: NCCL_IB_TIMEOUT + value: "22" + - name: NCCL_IB_GID_INDEX + value: "3" + - name: NCCL_MIN_NCHANNELS + value: "4" + - name: NCCL_SOCKET_IFNAME + value: bond0 + - name: GLOO_SOCKET_IFNAME + value: bond0 + - name: NCCL_IB_HCA + value: ^=mlx5_0,mlx5_5,mlx5_6 + - name: NVSHMEM_BOOTSTRAP_UID_SOCK_IFNAME + value: "bond0" + - name: MC_TE_METRIC + value: "false" + resources: + limits: + nvidia.com/gpu: "8" + securityContext: + capabilities: + add: + - IPC_LOCK + privileged: true + volumeMounts: + - mountPath: /root/.cache + name: sgl-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /work/models + name: model + - mountPath: /dev/infiniband + name: ib + - mountPath: /sgl-workspace/sglang + name: src + + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + nodeSelector: + pd: "yes" + tolerations: + - key: pd + operator: Exists + volumes: + - hostPath: + path: /var/run/sys-topology + name: topo + - hostPath: + path: /data1/sgl_cache4 + type: DirectoryOrCreate + name: sgl-cache + - emptyDir: + medium: Memory + name: dshm + - hostPath: + path: /data/DeepSeek-V3.2-Exp + name: model + - hostPath: + path: /dev/infiniband + name: ib + - hostPath: + path: /data/src/sglang + type: DirectoryOrCreate + name: src + + - name: decode + replicas: 1 + workload: + apiVersion: leaderworkerset.x-k8s.io/v1 + kind: LeaderWorkerSet + leaderWorkerSet: + size: 1 + patchLeaderTemplate: + metadata: + labels: + role: leader + pd_role: decode + spec: + containers: + - command: + - python3 + - -m + - sglang.launch_server + - --model-path + - /work/models + - --port + - "30000" + - --trust-remote + - --host + - 0.0.0.0 + - --disaggregation-ib-device + - mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_6,mlx5_7 + - --chunked-prefill-size + - "131072" + - --prefill-round-robin-balance + - --eplb-rebalance-layers-per-chunk + - "29" + - --page-size + - "64" + - --enable-dp-attention + - --enable-dp-lm-head + - --dp-size + - "8" + - --moe-a2a-backend + - deepep + - --deepep-mode + - low_latency + - --disaggregation-mode + - decode + - --mem-fraction-static + - "0.8" + - --context-length + - "32768" + - --max-running-requests + - "2048" + - --tp-size + - "8" # Size of Tensor Parallelism + - --cuda-graph-max-bs + - "16" + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20102 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + - --ep-num-redundant-experts + - "32" + - --moe-dense-tp-size + - "1" + env: + - name: LWS_WORKER_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] + livenessProbe: + failureThreshold: 30000 + httpGet: + path: /health + port: 30000 + initialDelaySeconds: 300 + periodSeconds: 60 + successThreshold: 1 + timeoutSeconds: 10 + name: sglang + readinessProbe: + failureThreshold: 20 + httpGet: + path: /health + port: 30000 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 10 + patchWorkerTemplate: + spec: + containers: + - command: + - python3 + - -m + - sglang.launch_server + - --model-path + - /work/models + - --crash-dump-folder + - /log + - --chunked-prefill-size + - "262144" + - --prefill-round-robin-balance + - --eplb-rebalance-layers-per-chunk + - "29" + - --page-size + - "64" + - --enable-dp-attention + - --enable-dp-lm-head + - --dp-size + - "32" + - --moe-a2a-backend + - "deepep" + - --deepep-mode + - low_latency + - --disaggregation-mode + - decode + - --mem-fraction-static + - "0.849" + - --context-length + - "32768" + - --disaggregation-ib-device + - mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_6,mlx5_7 + - --max-running-requests + - "4096" + - --cuda-graph-max-bs + - "16" + - --tp-size + - "8" # Size of Tensor Parallelism + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20102 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + - --ep-num-redundant-experts + - "32" + - --moe-dense-tp-size + - "1" + env: + - name: LWS_WORKER_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] + name: sglang + template: + metadata: + labels: + inference-framework: sglang-unuse + inference-stack.io/monitoring: "enabled" + spec: + containers: + - image: lmsysorg/sglang:latest + name: sglang + resources: + limits: + nvidia.com/gpu: "8" + securityContext: + capabilities: + add: + - IPC_LOCK + privileged: true + volumeMounts: + - mountPath: /root/.cache + name: sgl-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /work/models + name: model + - mountPath: /dev/infiniband + name: ib + - mountPath: /sgl-workspace/sglang + name: src + env: + - name: SGLANG_SKIP_SGL_KERNEL_VERSION_CHECK + value: "1" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "100000000" + - name: NVSHMEM_DISABLE_P2P + value: "0" + - name: NVSHMEM_IB_TRAFFIC_CLASS + value: "16" + - name: NVSHMEM_IB_SL + value: "5" + - name: ENABLE_METRICS + value: "true" + - name: CUDA_LAUNCH_BLOCKING + value: "0" + - name: NVSHMEM_IB_GID_INDEX + value: "3" + - name: NCCL_IB_QPS_PER_CONNECTION + value: "8" + - name: NCCL_IB_SPLIT_DATA_ON_QPS + value: "1" + - name: NCCL_NET_PLUGIN + value: "none" + - name: NCCL_IB_TC + value: "136" + - name: NCCL_IB_SL + value: "5" + - name: NCCL_IB_TIMEOUT + value: "22" + - name: NCCL_IB_GID_INDEX + value: "3" + - name: NCCL_MIN_NCHANNELS + value: "4" + - name: NCCL_SOCKET_IFNAME + value: bond0 + - name: GLOO_SOCKET_IFNAME + value: bond0 + - name: NVSHMEM_BOOTSTRAP_UID_SOCK_IFNAME + value: "bond0" + - name: NCCL_IB_HCA + value: ^=mlx5_0,mlx5_5,mlx5_6 + - name: MC_TE_METRIC + value: "false" + - name: SGL_ENABLE_JIT_DEEPGEMM + value: "1" + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + nodeSelector: + pd: "yes" + tolerations: + - key: pd + operator: Exists + volumes: + - hostPath: + path: /var/run/sys-topology + name: topo + - hostPath: + path: /data1/sgl_cache4 + type: DirectoryOrCreate + name: sgl-cache + - hostPath: + path: /data/src/sglang + type: DirectoryOrCreate + name: src + - emptyDir: + medium: Memory + name: dshm + - hostPath: + path: /data/DeepSeek-V3.2-Exp + name: model + - hostPath: + path: /dev/infiniband + name: ib + - name: router + replicas: 1 + dependencies: [ "decode", "prefill" ] + template: + spec: + containers: + - name: scheduler + image: lmsysorg/sglang:latest + command: + - sh + - -c + - > + python3 -m sglang_router.launch_router + --host 0.0.0.0 + --port 8080 + --pd-disaggregation + --policy random + --service-discovery + --service-discovery-namespace ${NAMESPACE} + --service-discovery-port 30000 + --prefill-selector pd_role=prefill + --decode-selector pd_role=decode + --max-payload-size 2147483648 + --worker-startup-timeout-secs 1200 + env: + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: deepseek-rbg-32exp + name: deepseek-rbg-32exp + namespace: default +spec: + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: 8080 + nodePort: 30080 + + selector: + rolebasedgroup.workloads.x-k8s.io/name: deepseek-rbg-32exp + rolebasedgroup.workloads.x-k8s.io/role: router + type: NodePort + +``` + +```bash +[root@ecs-001]# kubectl get po -n default +deepseek-rbg-32exp-decode-main-0 1/1 Running 0 74m +deepseek-rbg-32exp-decode-0-1 1/1 Running 0 74m +deepseek-rbg-32exp-router-9c5dbfc57 1/1 Running 0 22m +deepseek-rbg-32exp-prefill-0 1/1 Running 0 74m + +[root@ecs-cbm-x1-pd-cpu-001 main_doc]# kubectl get svc |grep dee +deepseek-rbg-32exp-decode ClusterIP None 97m +deepseek-rbg-32exp-router-service NodePort 172.16.242.169 8000:30800/TCP 22m +deepseek-rbg-32exp-prefill ClusterIP None 97m +``` + +At this point, select a nodePort:30800 to access: + +```bash +[root@ecs-001]# curl -X POST "http://{nodePort}:30800/v1/chat/completions" \ +> -H "Content-Type: application/json" \ +> -H "Authorization: Bearer None" \ +> -d '{ +> "rid":"ccccdd", +> "model": "dsv32", +> "messages": [ +> {"role": "system", "content": "0: You are a helpful AI assistant"}, +> {"role": "user", "content": "你是谁？."} +> ], +> "max_tokens":221 +> }' +{"id":"ccccdd","object":"chat.completion","created":1750252498,"model":"qwen2","choices":[{"index":0,"message":{"role":"assistant","content":"\n嗯，用户问了一个很基础的自我介绍问题"你是谁？"。这可能是第一次互动时的常规开场白，也可能是想确认我的身份和功能范围。\n\n用户没有提供任何背景信息，语气简洁中性。这种场景下新用户的可能性较高，需要给出清晰友好的自我介绍，同时突出实用价值来降低陌生感。\n\n考虑到中文用户，应该用简体中文回复。重点要说明三点：身份归属（深度求索）、功能定位（AI助手）、服务范围（学习/工作/生活）。结尾用开放性问题引导对话很关键——既能了解需求，又能避免让用户面对空白输入框时不知所措。\n\n用波浪线结尾可以软化语气，那个笑脸表情😊刚好能中和AI的机械感。不过要控制表情符号数量，避免显得轻浮。\n\n你好呀！我是你的AI助手，由深度求索公司（DeepSeek）开发的语言模型，名字叫 **DeepSeek-V32**。你可以把我当成一个知识丰富、随叫随到的小帮手～😊\n\n我的任务就是陪你聊天、解答问题、","reasoning_content":null,"tool_calls":null},"logprobs":null,"finish_reason":"length","matched_stop":null}],"usage":{"prompt_tokens":14,"total_tokens":235,"completion_tokens":221,"prompt_tokens_details":null}} + +``` +## FAQ + +1. The current deployment startup parameters may not be fully compatible with all RDMA scenarios. Different RDMA NCCL-related environment configurations may be needed in different network environments. + +2. Please ensure that the sglang code in the image has incorporated the changes from [PR #10912](https://github.com/sgl-project/sglang/pull/10912). diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/post_training_integration.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/post_training_integration.md new file mode 100644 index 00000000..5e82f837 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/post_training_integration.md @@ -0,0 +1,31 @@ +# Post-Training Integration + +SGLang has become the de facto inference backend for modern LLM training frameworks, powering state-of-the-art models across the industry. From GLM-4.6 to Qwen3, leading models leverage SGLang's high-performance inference during reinforcement learning and post-training workflows. + +What makes SGLang essential for post-training? + +- Open-To-Use Refit Functionality: diverse method for colocate or disaggregate +- Easy To Postpone Generation: enable partial rollout and dedicated rollout control +- Fine-Grained Engine Sleep And Wake Up: facilitate maxium-powered rollout and training +- Training Serving Alignment: ensure the performance consistency in training and serving +- Load Balancing Router: cache-aware load-balancing for high-throughput rollout +- Deterministic Inference: ensure zero kl divergence between rollout and training + +These capabilities, combined with native integration support across major frameworks, have established SGLang as the infrastructure backbone for modern LLM/VLMs post-training. We also share our latest work in this slide, [Optimizing Large-Scale RL with SGLang](https://gamma.app/docs/Optimizing-RL-with-SGLang-y0kqgj877k34779). + +## Adoption + +- [**Miles**](https://github.com/radixark/miles): Enterprise-scale RL framework for large MoE models with SGLang-native rollout, speculative training, and production-grade stability +- [**slime**](https://github.com/THUDM/slime): Post-training framework combining Megatron and SGLang, used to train GLM-4.6 +- [**AReaL**](https://github.com/inclusionAI/AReaL): Fully asynchronous RL system achieving 2.77x speedup with SGLang backend for continuous rollout generation +- [**ROLL**](https://github.com/alibaba/ROLL): ROLL is an efficient and user-friendly RL library designed for Large Language Models utilizing Large Scale GPU resources +- [**verl**](https://github.com/volcengine/verl): Full-stack RLHF framework supporting PPO, GRPO, and ReMax with modular SGLang integration +- [**Unsloth**](https://docs.unsloth.ai/basics/inference-and-deployment/sglang-guide): 2x faster fine-tuning with optimized kernels, deploys seamlessly with SGLang inference +- [**LLaMA Factory**](https://github.com/hiyouga/LLaMA-Factory): Unified framework for training 100+ LLMs with LoRA, QLoRA, and full fine-tuning methods +- [**Tunix**](https://github.com/google/tunix): Google's JAX-native library for LLM post-training with SFT, DPO, PPO, and GRPO support +- [**RL2**](https://github.com/ChenmienTan/RL2): Ray Less Reinforcement Learning, a concise library of post-training for large language models + + +## Collaboration + +Due to the privacy of the design parternes, we cannot list the companies that adopt SGLang for post-training. However, we are happy to share the details with you if you are interested and trust the choice among 10+ top companies and frontier labs across US and China. If you are interested in integrating SGLang with your training framework or need technical support, we're here to help! Reach out to us at **rl_team@lmsys.org** for partnerships, integration guidance, and custom feature development. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/production_metrics.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/production_metrics.md new file mode 100644 index 00000000..85a6ff8a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/production_metrics.md @@ -0,0 +1,231 @@ +# Production Metrics + +SGLang exposes the following metrics via Prometheus. You can enable it by adding `--enable-metrics` when you launch the server. + +An example of the monitoring dashboard is available in [examples/monitoring/grafana.json](https://github.com/sgl-project/sglang/blob/main/examples/monitoring/grafana/dashboards/json/sglang-dashboard.json). + +Here is an example of the metrics: + +``` +$ curl http://localhost:30000/metrics +# HELP sglang:prompt_tokens_total Number of prefill tokens processed. +# TYPE sglang:prompt_tokens_total counter +sglang:prompt_tokens_total{model_name="meta-llama/Llama-3.1-8B-Instruct"} 8.128902e+06 +# HELP sglang:generation_tokens_total Number of generation tokens processed. +# TYPE sglang:generation_tokens_total counter +sglang:generation_tokens_total{model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.557572e+06 +# HELP sglang:token_usage The token usage +# TYPE sglang:token_usage gauge +sglang:token_usage{model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.28 +# HELP sglang:cache_hit_rate The cache hit rate +# TYPE sglang:cache_hit_rate gauge +sglang:cache_hit_rate{model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.007507552643049313 +# HELP sglang:time_to_first_token_seconds Histogram of time to first token in seconds. +# TYPE sglang:time_to_first_token_seconds histogram +sglang:time_to_first_token_seconds_sum{model_name="meta-llama/Llama-3.1-8B-Instruct"} 2.3518979474117756e+06 +sglang:time_to_first_token_seconds_bucket{le="0.001",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 +sglang:time_to_first_token_seconds_bucket{le="0.005",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 +sglang:time_to_first_token_seconds_bucket{le="0.01",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 +sglang:time_to_first_token_seconds_bucket{le="0.02",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 +sglang:time_to_first_token_seconds_bucket{le="0.04",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1.0 +sglang:time_to_first_token_seconds_bucket{le="0.06",model_name="meta-llama/Llama-3.1-8B-Instruct"} 3.0 +sglang:time_to_first_token_seconds_bucket{le="0.08",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:time_to_first_token_seconds_bucket{le="0.1",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:time_to_first_token_seconds_bucket{le="0.25",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:time_to_first_token_seconds_bucket{le="0.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:time_to_first_token_seconds_bucket{le="0.75",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:time_to_first_token_seconds_bucket{le="1.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 27.0 +sglang:time_to_first_token_seconds_bucket{le="2.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 140.0 +sglang:time_to_first_token_seconds_bucket{le="5.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 314.0 +sglang:time_to_first_token_seconds_bucket{le="7.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 941.0 +sglang:time_to_first_token_seconds_bucket{le="10.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1330.0 +sglang:time_to_first_token_seconds_bucket{le="15.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1970.0 +sglang:time_to_first_token_seconds_bucket{le="20.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 2326.0 +sglang:time_to_first_token_seconds_bucket{le="25.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 2417.0 +sglang:time_to_first_token_seconds_bucket{le="30.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 2513.0 +sglang:time_to_first_token_seconds_bucket{le="+Inf",model_name="meta-llama/Llama-3.1-8B-Instruct"} 11008.0 +sglang:time_to_first_token_seconds_count{model_name="meta-llama/Llama-3.1-8B-Instruct"} 11008.0 +# HELP sglang:e2e_request_latency_seconds Histogram of End-to-end request latency in seconds +# TYPE sglang:e2e_request_latency_seconds histogram +sglang:e2e_request_latency_seconds_sum{model_name="meta-llama/Llama-3.1-8B-Instruct"} 3.116093850019932e+06 +sglang:e2e_request_latency_seconds_bucket{le="0.3",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 +sglang:e2e_request_latency_seconds_bucket{le="0.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:e2e_request_latency_seconds_bucket{le="0.8",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:e2e_request_latency_seconds_bucket{le="1.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:e2e_request_latency_seconds_bucket{le="1.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:e2e_request_latency_seconds_bucket{le="2.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:e2e_request_latency_seconds_bucket{le="2.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:e2e_request_latency_seconds_bucket{le="5.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.0 +sglang:e2e_request_latency_seconds_bucket{le="10.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 10.0 +sglang:e2e_request_latency_seconds_bucket{le="15.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 11.0 +sglang:e2e_request_latency_seconds_bucket{le="20.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 14.0 +sglang:e2e_request_latency_seconds_bucket{le="30.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 247.0 +sglang:e2e_request_latency_seconds_bucket{le="40.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 486.0 +sglang:e2e_request_latency_seconds_bucket{le="50.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 845.0 +sglang:e2e_request_latency_seconds_bucket{le="60.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1513.0 +sglang:e2e_request_latency_seconds_bucket{le="+Inf",model_name="meta-llama/Llama-3.1-8B-Instruct"} 11228.0 +sglang:e2e_request_latency_seconds_count{model_name="meta-llama/Llama-3.1-8B-Instruct"} 11228.0 +# HELP sglang:time_per_output_token_seconds Histogram of time per output token in seconds. +# TYPE sglang:time_per_output_token_seconds histogram +sglang:time_per_output_token_seconds_sum{model_name="meta-llama/Llama-3.1-8B-Instruct"} 866964.5791549598 +sglang:time_per_output_token_seconds_bucket{le="0.005",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1.0 +sglang:time_per_output_token_seconds_bucket{le="0.01",model_name="meta-llama/Llama-3.1-8B-Instruct"} 73.0 +sglang:time_per_output_token_seconds_bucket{le="0.015",model_name="meta-llama/Llama-3.1-8B-Instruct"} 382.0 +sglang:time_per_output_token_seconds_bucket{le="0.02",model_name="meta-llama/Llama-3.1-8B-Instruct"} 593.0 +sglang:time_per_output_token_seconds_bucket{le="0.025",model_name="meta-llama/Llama-3.1-8B-Instruct"} 855.0 +sglang:time_per_output_token_seconds_bucket{le="0.03",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1035.0 +sglang:time_per_output_token_seconds_bucket{le="0.04",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1815.0 +sglang:time_per_output_token_seconds_bucket{le="0.05",model_name="meta-llama/Llama-3.1-8B-Instruct"} 11685.0 +sglang:time_per_output_token_seconds_bucket{le="0.075",model_name="meta-llama/Llama-3.1-8B-Instruct"} 433413.0 +sglang:time_per_output_token_seconds_bucket{le="0.1",model_name="meta-llama/Llama-3.1-8B-Instruct"} 4.950195e+06 +sglang:time_per_output_token_seconds_bucket{le="0.15",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.039435e+06 +sglang:time_per_output_token_seconds_bucket{le="0.2",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.171662e+06 +sglang:time_per_output_token_seconds_bucket{le="0.3",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.266055e+06 +sglang:time_per_output_token_seconds_bucket{le="0.4",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.296752e+06 +sglang:time_per_output_token_seconds_bucket{le="0.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.312226e+06 +sglang:time_per_output_token_seconds_bucket{le="0.75",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.339675e+06 +sglang:time_per_output_token_seconds_bucket{le="1.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.357747e+06 +sglang:time_per_output_token_seconds_bucket{le="2.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.389414e+06 +sglang:time_per_output_token_seconds_bucket{le="+Inf",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.400757e+06 +sglang:time_per_output_token_seconds_count{model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.400757e+06 +# HELP sglang:func_latency_seconds Function latency in seconds +# TYPE sglang:func_latency_seconds histogram +sglang:func_latency_seconds_sum{name="generate_request"} 4.514771912145079 +sglang:func_latency_seconds_bucket{le="0.05",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.07500000000000001",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.1125",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.16875",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.253125",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.3796875",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.56953125",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.8542968750000001",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="1.2814453125",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="1.9221679687500002",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="2.8832519531250003",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="4.3248779296875",name="generate_request"} 14007.0 +sglang:func_latency_seconds_bucket{le="6.487316894531251",name="generate_request"} 14007.0 +sglang:func_latency_seconds_bucket{le="9.730975341796876",name="generate_request"} 14007.0 +sglang:func_latency_seconds_bucket{le="14.596463012695313",name="generate_request"} 14007.0 +sglang:func_latency_seconds_bucket{le="21.89469451904297",name="generate_request"} 14007.0 +sglang:func_latency_seconds_bucket{le="32.84204177856446",name="generate_request"} 14007.0 +sglang:func_latency_seconds_bucket{le="49.26306266784668",name="generate_request"} 14007.0 +sglang:func_latency_seconds_bucket{le="+Inf",name="generate_request"} 14007.0 +sglang:func_latency_seconds_count{name="generate_request"} 14007.0 +# HELP sglang:num_running_reqs The number of running requests +# TYPE sglang:num_running_reqs gauge +sglang:num_running_reqs{model_name="meta-llama/Llama-3.1-8B-Instruct"} 162.0 +# HELP sglang:num_used_tokens The number of used tokens +# TYPE sglang:num_used_tokens gauge +sglang:num_used_tokens{model_name="meta-llama/Llama-3.1-8B-Instruct"} 123859.0 +# HELP sglang:gen_throughput The generate throughput (token/s) +# TYPE sglang:gen_throughput gauge +sglang:gen_throughput{model_name="meta-llama/Llama-3.1-8B-Instruct"} 86.50814177726902 +# HELP sglang:num_queue_reqs The number of requests in the waiting queue +# TYPE sglang:num_queue_reqs gauge +sglang:num_queue_reqs{model_name="meta-llama/Llama-3.1-8B-Instruct"} 2826.0 +``` + +## Setup Guide + +This section describes how to set up the monitoring stack (Prometheus + Grafana) provided in the `examples/monitoring` directory. + +### Prerequisites + +- Docker and Docker Compose installed +- SGLang server running with metrics enabled + +### Usage + +1. **Start your SGLang server with metrics enabled:** + + ```bash + python -m sglang.launch_server \ + --model-path \ + --port 30000 \ + --enable-metrics + ``` + Replace `` with the actual path to your model (e.g., `meta-llama/Meta-Llama-3.1-8B-Instruct`). Ensure the server is accessible from the monitoring stack (you might need `--host 0.0.0.0` if running in Docker). By default, the metrics endpoint will be available at `http://:30000/metrics`. + +2. **Navigate to the monitoring example directory:** + ```bash + cd examples/monitoring + ``` + +3. **Start the monitoring stack:** + ```bash + docker compose up -d + ``` + This command will start Prometheus and Grafana in the background. + +4. **Access the monitoring interfaces:** + * **Grafana:** Open your web browser and go to [http://localhost:3000](http://localhost:3000). + * **Prometheus:** Open your web browser and go to [http://localhost:9090](http://localhost:9090). + +5. **Log in to Grafana:** + * Default Username: `admin` + * Default Password: `admin` + You will be prompted to change the password upon your first login. + +6. **View the Dashboard:** + The SGLang dashboard is pre-configured and should be available automatically. Navigate to `Dashboards` -> `Browse` -> `SGLang Monitoring` folder -> `SGLang Dashboard`. + +### Troubleshooting + +* **Port Conflicts:** If you encounter errors like "port is already allocated," check if other services (including previous instances of Prometheus/Grafana) are using ports `9090` or `3000`. Use `docker ps` to find running containers and `docker stop ` to stop them, or use `lsof -i :` to find other processes using the ports. You might need to adjust the ports in the `docker-compose.yaml` file if they permanently conflict with other essential services on your system. + +To modify Grafana's port to the other one(like 3090) in your Docker Compose file, you need to explicitly specify the port mapping under the grafana service. + + Option 1: Add GF_SERVER_HTTP_PORT to the environment section: + ``` + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_SERVER_HTTP_PORT=3090 # <-- Add this line + ``` + Option 2: Use port mapping: + ``` + grafana: + image: grafana/grafana:latest + container_name: grafana + ports: + - "3090:3000" # <-- Host:Container port mapping + ``` +* **Connection Issues:** + * Ensure both Prometheus and Grafana containers are running (`docker ps`). + * Verify the Prometheus data source configuration in Grafana (usually auto-configured via `grafana/datasources/datasource.yaml`). Go to `Connections` -> `Data sources` -> `Prometheus`. The URL should point to the Prometheus service (e.g., `http://prometheus:9090`). + * Confirm that your SGLang server is running and the metrics endpoint (`http://:30000/metrics`) is accessible *from the Prometheus container*. If SGLang is running on your host machine and Prometheus is in Docker, use `host.docker.internal` (on Docker Desktop) or your machine's network IP instead of `localhost` in the `prometheus.yaml` scrape configuration. +* **No Data on Dashboard:** + * Generate some traffic to your SGLang server to produce metrics. For example, run a benchmark: + ```bash + python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 100 --random-input 128 --random-output 128 + ``` + * Check the Prometheus UI (`http://localhost:9090`) under `Status` -> `Targets` to see if the SGLang endpoint is being scraped successfully. + * Verify the `model_name` and `instance` labels in your Prometheus metrics match the variables used in the Grafana dashboard. You might need to adjust the Grafana dashboard variables or the labels in your Prometheus configuration. + +### Configuration Files + +The monitoring setup is defined by the following files within the `examples/monitoring` directory: + +* `docker-compose.yaml`: Defines the Prometheus and Grafana services. +* `prometheus.yaml`: Prometheus configuration, including scrape targets. +* `grafana/datasources/datasource.yaml`: Configures the Prometheus data source for Grafana. +* `grafana/dashboards/config/dashboard.yaml`: Tells Grafana to load dashboards from the specified path. +* `grafana/dashboards/json/sglang-dashboard.json`: The actual Grafana dashboard definition in JSON format. + +You can customize the setup by modifying these files. For instance, you might need to update the `static_configs` target in `prometheus.yaml` if your SGLang server runs on a different host or port. + +#### Check if the metrics are being collected + +Run: +``` +python3 -m sglang.bench_serving \ + --backend sglang \ + --dataset-name random \ + --num-prompts 3000 \ + --random-input 1024 \ + --random-output 1024 \ + --random-range-ratio 0.5 +``` + +to generate some requests. + +Then you should be able to see the metrics in the Grafana dashboard. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/production_request_trace.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/production_request_trace.md new file mode 100644 index 00000000..2d19570c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/production_request_trace.md @@ -0,0 +1,160 @@ +# Production Request Tracing + +SGlang exports request trace data based on the OpenTelemetry Collector. You can enable tracing by adding the `--enable-trace` and configure the OpenTelemetry Collector endpoint using `--otlp-traces-endpoint` when launching the server. + +You can find example screenshots of the visualization in https://github.com/sgl-project/sglang/issues/8965. + +## Setup Guide +This section explains how to configure the request tracing and export the trace data. +1. Install the required packages and tools + * install Docker and Docker Compose + * install the dependencies + ```bash + # enter the SGLang root directory + pip install -e "python[tracing]" + + # or manually install the dependencies using pip + pip install opentelemetry-sdk opentelemetry-api opentelemetry-exporter-otlp opentelemetry-exporter-otlp-proto-grpc + ``` + +2. launch opentelemetry collector and jaeger + ```bash + docker compose -f examples/monitoring/tracing_compose.yaml up -d + ``` + +3. start your SGLang server with tracing enabled + ```bash + # set env variables + export SGLANG_OTLP_EXPORTER_SCHEDULE_DELAY_MILLIS=500 + export SGLANG_OTLP_EXPORTER_MAX_EXPORT_BATCH_SIZE=64 + # start the prefill and decode server + python -m sglang.launch_server --enable-trace --otlp-traces-endpoint 0.0.0.0:4317 + # start the mini lb + python -m sglang_router.launch_router --enable-trace --otlp-traces-endpoint 0.0.0.0:4317 + ``` + + Replace `0.0.0.0:4317` with the actual endpoint of the opentelemetry collector. If you launched the openTelemetry collector with tracing_compose.yaml, the default receiving port is 4317. + + To use the HTTP/protobuf span exporter, set the following environment variable and point to an HTTP endpoint, for example, `http://0.0.0.0:4318/v1/traces`. + ```bash + export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf + ``` + + +4. raise some requests +5. Observe whether trace data is being exported + * Access port 16686 of Jaeger using a web browser to visualize the request traces. + * The OpenTelemetry Collector also exports trace data in JSON format to /tmp/otel_trace.json. In a follow-up patch, we will provide a tool to convert this data into a Perfetto-compatible format, enabling visualization of requests in the Perfetto UI. + +## How to add Tracing for slices you're interested in? +We have already inserted instrumentation points in the tokenizer and scheduler main threads. If you wish to trace additional request execution segments or perform finer-grained tracing, please use the APIs from the tracing package as described below. + +1. initialization + + Every process involved in tracing during the initialization phase should execute: + ```python + process_tracing_init(otlp_traces_endpoint, server_name) + ``` + The otlp_traces_endpoint is obtained from the arguments, and you can set server_name freely, but it should remain consistent across all processes. + + Every thread involved in tracing during the initialization phase should execute: + ```python + trace_set_thread_info("thread label", tp_rank, dp_rank) + ``` + The "thread label" can be regarded as the name of the thread, used to distinguish different threads in the visualization view. + +2. Mark the beginning and end of a request + ``` + trace_req_start(rid, bootstrap_room) + trace_req_finish(rid) + ``` + These two APIs must be called within the same process, for example, in the tokenizer. + +3. Add tracing for slice + + * Add slice tracing normally: + ```python + trace_slice_start("slice A", rid) + trace_slice_end("slice A", rid) + ``` + + - Use the "anonymous" flag to not specify a slice name at the start of the slice, allowing the slice name to be determined by trace_slice_end. +
Note: Anonymous slices must not be nested. + ```python + trace_slice_start("", rid, anonymous = True) + trace_slice_end("slice A", rid) + ``` + + - In trace_slice_end, use auto_next_anon to automatically create the next anonymous slice, which can reduce the number of instrumentation points needed. + ```python + trace_slice_start("", rid, anonymous = True) + trace_slice_end("slice A", rid, auto_next_anon = True) + trace_slice_end("slice B", rid, auto_next_anon = True) + trace_slice_end("slice C", rid, auto_next_anon = True) + trace_slice_end("slice D", rid) + ``` + - The end of the last slice in a thread must be marked with thread_finish_flag=True; otherwise, the thread's span will not be properly generated. + ```python + trace_slice_end("slice D", rid, thread_finish_flag = True) + ``` + +4. When the request execution flow transfers to another thread, the trace context needs to be explicitly propagated. + - sender: Execute the following code before sending the request to another thread via ZMQ + ```python + trace_context = trace_get_proc_propagate_context(rid) + req.trace_context = trace_context + ``` + - receiver: Execute the following code after receiving the request via ZMQ + ```python + trace_set_proc_propagate_context(rid, req.trace_context) + ``` + +5. When the request execution flow transfers to another node(PD disaggregation), the trace context needs to be explicitly propagated. + - sender: Execute the following code before sending the request to node thread via http + ```python + trace_context = trace_get_remote_propagate_context(bootstrap_room_list) + headers = {"trace_context": trace_context} + session.post(url, headers=headers) + ``` + - receiver: Execute the following code after receiving the request via http + ```python + trace_set_remote_propagate_context(request.headers['trace_context']) + ``` + +## How to Extend the Tracing Framework to Support Complex Tracing Scenarios + +The currently provided tracing package still has potential for further development. If you wish to build more advanced features upon it, you must first understand its existing design principles. + +The core of the tracing framework's implementation lies in the design of the span structure and the trace context. To aggregate scattered slices and enable concurrent tracking of multiple requests, we have designed a two-level trace context structure and a four-level span structure: `SglangTraceReqContext`, `SglangTraceThreadContext`. Their relationship is as follows: +``` +SglangTraceReqContext (req_id="req-123") +├── SglangTraceThreadContext(thread_label="scheduler", tp_rank=0) +| +└── SglangTraceThreadContext(thread_label="scheduler", tp_rank=1) +``` + +Each traced request maintains a global `SglangTraceReqContext`. For every thread processing the request, a corresponding `SglangTraceThreadContext` is recorded and composed within the `SglangTraceReqContext`. Within each thread, every currently traced slice (possibly nested) is stored in a list. + +In addition to the above hierarchy, each slice also records its previous slice via Span.add_link(), which can be used to trace the execution flow. + +When the request execution flow transfers to a new thread, the trace context needs to be explicitly propagated. In the framework, this is represented by `SglangTracePropagateContext`, which contains the context of the request span and the previous slice span. + + +We designed a four-level span structure, consisting of `bootstrap_room_span`, `req_root_span`, `thread_span`, and `slice_span`. Among them, `req_root_span` and `thread_span` correspond to `SglangTraceReqContext` and `SglangTraceThreadContext`, respectively, and `slice_span` is stored within the `SglangTraceThreadContext`. The `bootstrap_room_span` is designed to accommodate the separation of PD-disaggregation. On different nodes, we may want to add certain attributes to the `req_root_span`. However, if the `req_root_span` is shared across all nodes, the Prefill and Decode nodes would not be allowed to add attributes due to the constraints imposed by OpenTelemetry's design. + +``` +bootstrap room span +├── router req root span +| └── router thread span +| └── slice span +├── prefill req root span +| ├── tokenizer thread span +| | └── slice span +| └── scheduler thread span +| └── slice span +└── decode req root span + ├── tokenizer thread span + | └── slice span + └── scheduler thread span + └── slice span +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/torch_compile_cache.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/torch_compile_cache.md new file mode 100644 index 00000000..f2bb257f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/references/torch_compile_cache.md @@ -0,0 +1,13 @@ +# Enabling cache for torch.compile + +SGLang uses `max-autotune-no-cudagraphs` mode of torch.compile. The auto-tuning can be slow. +If you want to deploy a model on many different machines, you can ship the torch.compile cache to these machines and skip the compilation steps. + +This is based on https://pytorch.org/tutorials/recipes/torch_compile_caching_tutorial.html + + +1. Generate the cache by setting TORCHINDUCTOR_CACHE_DIR and running the model once. +``` +TORCHINDUCTOR_CACHE_DIR=/root/inductor_root_cache python3 -m sglang.launch_server --model meta-llama/Llama-3.1-8B-Instruct --enable-torch-compile +``` +2. Copy the cache folder to other machines and launch the server with `TORCHINDUCTOR_CACHE_DIR`. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/requirements.txt b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/requirements.txt new file mode 100644 index 00000000..5d730967 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/requirements.txt @@ -0,0 +1,20 @@ +ipykernel +ipywidgets +jupyter_client +markdown>=3.4.0 +matplotlib +myst-parser +nbconvert +nbsphinx +pandoc +pillow +pydantic +sphinx +sphinx-book-theme +sphinx-copybutton +sphinx-tabs +nbstripout +sphinxcontrib-mermaid +urllib3<2.0.0 +gguf>=0.17.1 +sphinx-autobuild diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/serve.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/serve.sh new file mode 100644 index 00000000..049f767c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/serve.sh @@ -0,0 +1,3 @@ +# Clean and serve documentation with auto-build +make clean +make serve diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/classify_models.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/classify_models.md new file mode 100644 index 00000000..c6d18f9a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/classify_models.md @@ -0,0 +1,162 @@ +# Classification API + +This document describes the `/v1/classify` API endpoint implementation in SGLang, which is compatible with vLLM's classification API format. + +## Overview + +The classification API allows you to classify text inputs using classification models. This implementation follows the same format as vLLM's 0.7.0 classification API. + +## API Endpoint + +``` +POST /v1/classify +``` + +## Request Format + +```json +{ + "model": "model_name", + "input": "text to classify" +} +``` + +### Parameters + +- `model` (string, required): The name of the classification model to use +- `input` (string, required): The text to classify +- `user` (string, optional): User identifier for tracking +- `rid` (string, optional): Request ID for tracking +- `priority` (integer, optional): Request priority + +## Response Format + +```json +{ + "id": "classify-9bf17f2847b046c7b2d5495f4b4f9682", + "object": "list", + "created": 1745383213, + "model": "jason9693/Qwen2.5-1.5B-apeach", + "data": [ + { + "index": 0, + "label": "Default", + "probs": [0.565970778465271, 0.4340292513370514], + "num_classes": 2 + } + ], + "usage": { + "prompt_tokens": 10, + "total_tokens": 10, + "completion_tokens": 0, + "prompt_tokens_details": null + } +} +``` + +### Response Fields + +- `id`: Unique identifier for the classification request +- `object`: Always "list" +- `created`: Unix timestamp when the request was created +- `model`: The model used for classification +- `data`: Array of classification results + - `index`: Index of the result + - `label`: Predicted class label + - `probs`: Array of probabilities for each class + - `num_classes`: Total number of classes +- `usage`: Token usage information + - `prompt_tokens`: Number of input tokens + - `total_tokens`: Total number of tokens + - `completion_tokens`: Number of completion tokens (always 0 for classification) + - `prompt_tokens_details`: Additional token details (optional) + +## Example Usage + +### Using curl + +```bash +curl -v "http://127.0.0.1:8000/v1/classify" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "jason9693/Qwen2.5-1.5B-apeach", + "input": "Loved the new café—coffee was great." + }' +``` + +### Using Python + +```python +import requests +import json + +# Make classification request +response = requests.post( + "http://127.0.0.1:8000/v1/classify", + headers={"Content-Type": "application/json"}, + json={ + "model": "jason9693/Qwen2.5-1.5B-apeach", + "input": "Loved the new café—coffee was great." + } +) + +# Parse response +result = response.json() +print(json.dumps(result, indent=2)) +``` + +## Supported Models + +The classification API works with any classification model supported by SGLang, including: + +### Classification Models (Multi-class) +- `LlamaForSequenceClassification` - Multi-class classification +- `Qwen2ForSequenceClassification` - Multi-class classification +- `Qwen3ForSequenceClassification` - Multi-class classification +- `BertForSequenceClassification` - Multi-class classification +- `Gemma2ForSequenceClassification` - Multi-class classification + +**Label Mapping**: The API automatically uses the `id2label` mapping from the model's `config.json` file to provide meaningful label names instead of generic class names. If `id2label` is not available, it falls back to `LABEL_0`, `LABEL_1`, etc., or `Class_0`, `Class_1` as a last resort. + +### Reward Models (Single score) +- `InternLM2ForRewardModel` - Single reward score +- `Qwen2ForRewardModel` - Single reward score +- `LlamaForSequenceClassificationWithNormal_Weights` - Special reward model + +**Note**: The `/classify` endpoint in SGLang was originally designed for reward models but now supports all non-generative models. Our `/v1/classify` endpoint provides a standardized vLLM-compatible interface for classification tasks. + +## Error Handling + +The API returns appropriate HTTP status codes and error messages: + +- `400 Bad Request`: Invalid request format or missing required fields +- `500 Internal Server Error`: Server-side processing error + +Error response format: +```json +{ + "error": "Error message", + "type": "error_type", + "code": 400 +} +``` + +## Implementation Details + +The classification API is implemented using: + +1. **Rust Router**: Handles routing and request/response models in `sgl-router/src/protocols/spec.rs` +2. **Python HTTP Server**: Implements the actual endpoint in `python/sglang/srt/entrypoints/http_server.py` +3. **Classification Service**: Handles the classification logic in `python/sglang/srt/entrypoints/openai/serving_classify.py` + +## Testing + +Use the provided test script to verify the implementation: + +```bash +python test_classify_api.py +``` + +## Compatibility + +This implementation is compatible with vLLM's classification API format, allowing seamless migration from vLLM to SGLang for classification tasks. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/embedding_models.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/embedding_models.md new file mode 100644 index 00000000..906466ac --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/embedding_models.md @@ -0,0 +1,126 @@ +# Embedding Models + +SGLang provides robust support for embedding models by integrating efficient serving mechanisms with its flexible programming interface. This integration allows for streamlined handling of embedding tasks, facilitating faster and more accurate retrieval and semantic search operations. SGLang's architecture enables better resource utilization and reduced latency in embedding model deployment. + +```{important} +Embedding models are executed with `--is-embedding` flag and some may require `--trust-remote-code` +``` + +## Quick Start + +### Launch Server + +```shell +python3 -m sglang.launch_server \ + --model-path Qwen/Qwen3-Embedding-4B \ + --is-embedding \ + --host 0.0.0.0 \ + --port 30000 +``` + +### Client Request + +```python +import requests + +url = "http://127.0.0.1:30000" + +payload = { + "model": "Qwen/Qwen3-Embedding-4B", + "input": "What is the capital of France?", + "encoding_format": "float" +} + +response = requests.post(url + "/v1/embeddings", json=payload).json() +print("Embedding:", response["data"][0]["embedding"]) +``` + + + +## Multimodal Embedding Example + +For multimodal models like GME that support both text and images: + +```shell +python3 -m sglang.launch_server \ + --model-path Alibaba-NLP/gme-Qwen2-VL-2B-Instruct \ + --is-embedding \ + --chat-template gme-qwen2-vl \ + --host 0.0.0.0 \ + --port 30000 +``` + +```python +import requests + +url = "http://127.0.0.1:30000" + +text_input = "Represent this image in embedding space." +image_path = "https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild/resolve/main/images/023.jpg" + +payload = { + "model": "gme-qwen2-vl", + "input": [ + { + "text": text_input + }, + { + "image": image_path + } + ], +} + +response = requests.post(url + "/v1/embeddings", json=payload).json() + +print("Embeddings:", [x.get("embedding") for x in response.get("data", [])]) +``` + +## Matryoshka Embedding Example + +[Matryoshka Embeddings](https://sbert.net/examples/sentence_transformer/training/matryoshka/README.html#matryoshka-embeddings) or [Matryoshka Representation Learning (MRL)](https://arxiv.org/abs/2205.13147) is a technique used in training embedding models. It allows user to trade off between performance and cost. + +### 1. Launch a Matryoshka‑capable model + +If the model config already includes `matryoshka_dimensions` or `is_matryoshka` then no override is needed. Otherwise, you can use `--json-model-override-args` as below: + +```shell +python3 -m sglang.launch_server \ + --model-path Qwen/Qwen3-Embedding-0.6B \ + --is-embedding \ + --host 0.0.0.0 \ + --port 30000 \ + --json-model-override-args '{"matryoshka_dimensions": [128, 256, 512, 1024, 1536]}' +``` + +1. Setting `"is_matryoshka": true` allows truncating to any dimension. Otherwise, the server will validate that the specified dimension in the request is one of `matryoshka_dimensions`. +2. Omitting `dimensions` in a request returns the full vector. + +### 2. Make requests with different output dimensions + +```python +import requests + +url = "http://127.0.0.1:30000" + +# Request a truncated (Matryoshka) embedding by specifying a supported dimension. +payload = { + "model": "Qwen/Qwen3-Embedding-0.6B", + "input": "Explain diffusion models simply.", + "dimensions": 512 # change to 128 / 1024 / omit for full size +} + +response = requests.post(url + "/v1/embeddings", json=payload).json() +print("Embedding:", response["data"][0]["embedding"]) +``` + + +## Supported Models + +| Model Family | Example Model | Chat Template | Description | +| ------------------------------------------ | -------------------------------------- | ------------- | --------------------------------------------------------------------------- | +| **E5 (Llama/Mistral based)** | `intfloat/e5-mistral-7b-instruct` | N/A | High-quality text embeddings based on Mistral/Llama architectures | +| **GTE-Qwen2** | `Alibaba-NLP/gte-Qwen2-7B-instruct` | N/A | Alibaba's text embedding model with multilingual support | +| **Qwen3-Embedding** | `Qwen/Qwen3-Embedding-4B` | N/A | Latest Qwen3-based text embedding model for semantic representation | +| **BGE** | `BAAI/bge-large-en-v1.5` | N/A | BAAI's text embeddings (requires `attention-backend` triton/torch_native) | +| **GME (Multimodal)** | `Alibaba-NLP/gme-Qwen2-VL-2B-Instruct`| `gme-qwen2-vl`| Multimodal embedding for text and image cross-modal tasks | +| **CLIP** | `openai/clip-vit-large-patch14-336` | N/A | OpenAI's CLIP for image and text embeddings | diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/generative_models.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/generative_models.md new file mode 100644 index 00000000..671fbaaf --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/generative_models.md @@ -0,0 +1,64 @@ +# Large Language Models + +These models accept text input and produce text output (e.g., chat completions). They are primarily large language models (LLMs), some with mixture-of-experts (MoE) architectures for scaling. + +## Example launch Command + +```shell +python3 -m sglang.launch_server \ + --model-path meta-llama/Llama-3.2-1B-Instruct \ # example HF/local path + --host 0.0.0.0 \ + --port 30000 \ +``` + +## Supported models + +Below the supported models are summarized in a table. + +If you are unsure if a specific architecture is implemented, you can search for it via GitHub. For example, to search for `Qwen3ForCausalLM`, use the expression: + +``` +repo:sgl-project/sglang path:/^python\/sglang\/srt\/models\// Qwen3ForCausalLM +``` + +in the GitHub search bar. + +| Model Family (Variants) | Example HuggingFace Identifier | Description | +|-------------------------------------|--------------------------------------------------|----------------------------------------------------------------------------------------| +| **DeepSeek** (v1, v2, v3/R1) | `deepseek-ai/DeepSeek-R1` | Series of advanced reasoning-optimized models (including a 671B MoE) trained with reinforcement learning; top performance on complex reasoning, math, and code tasks. [SGLang provides Deepseek v3/R1 model-specific optimizations](../basic_usage/deepseek.md) and [Reasoning Parser](../advanced_features/separate_reasoning.ipynb)| +| **GPT-OSS** | `openai/gpt-oss-20b`, `openai/gpt-oss-120b` | OpenAI’s latest GPT-OSS series for complex reasoning, agentic tasks, and versatile developer use cases.| +| **Qwen** (3, 3MoE, 3Next, 2.5, 2 series) | `Qwen/Qwen3-0.6B`, `Qwen/Qwen3-30B-A3B` `Qwen/Qwen3-Next-80B-A3B-Instruct ` | Alibaba’s latest Qwen3 series for complex reasoning, language understanding, and generation tasks; Support for MoE variants along with previous generation 2.5, 2, etc. [SGLang provides Qwen3 specific reasoning parser](../advanced_features/separate_reasoning.ipynb)| +| **Llama** (2, 3.x, 4 series) | `meta-llama/Llama-4-Scout-17B-16E-Instruct` | Meta's open LLM series, spanning 7B to 400B parameters (Llama 2, 3, and new Llama 4) with well-recognized performance. [SGLang provides Llama-4 model-specific optimizations](../basic_usage/llama4.md) | +| **Mistral** (Mixtral, NeMo, Small3) | `mistralai/Mistral-7B-Instruct-v0.2` | Open 7B LLM by Mistral AI with strong performance; extended into MoE (“Mixtral”) and NeMo Megatron variants for larger scale. | +| **Gemma** (v1, v2, v3) | `google/gemma-3-1b-it` | Google’s family of efficient multilingual models (1B–27B); Gemma 3 offers a 128K context window, and its larger (4B+) variants support vision input. | +| **Phi** (Phi-1.5, Phi-2, Phi-3, Phi-4, Phi-MoE series) | `microsoft/Phi-4-multimodal-instruct`, `microsoft/Phi-3.5-MoE-instruct` | Microsoft’s Phi family of small models (1.3B–5.6B); Phi-4-multimodal (5.6B) processes text, images, and speech, Phi-4-mini is a high-accuracy text model and Phi-3.5-MoE is a mixture-of-experts model. | +| **MiniCPM** (v3, 4B) | `openbmb/MiniCPM3-4B` | OpenBMB’s series of compact LLMs for edge devices; MiniCPM 3 (4B) achieves GPT-3.5-level results in text tasks. | +| **OLMo** (2, 3) | `allenai/OLMo-2-1124-7B-Instruct` | Allen AI’s series of Open Language Models designed to enable the science of language models. | +| **OLMoE** (Open MoE) | `allenai/OLMoE-1B-7B-0924` | Allen AI’s open Mixture-of-Experts model (7B total, 1B active parameters) delivering state-of-the-art results with sparse expert activation. | +| **MiniMax-M2** | `minimax/MiniMax-M2` | MiniMax’s SOTA LLM for coding & agentic workflows. | +| **StableLM** (3B, 7B) | `stabilityai/stablelm-tuned-alpha-7b` | StabilityAI’s early open-source LLM (3B & 7B) for general text generation; a demonstration model with basic instruction-following ability. | +| **Command-R** (Cohere) | `CohereForAI/c4ai-command-r-v01` | Cohere’s open conversational LLM (Command series) optimized for long context, retrieval-augmented generation, and tool use. | +| **DBRX** (Databricks) | `databricks/dbrx-instruct` | Databricks’ 132B-parameter MoE model (36B active) trained on 12T tokens; competes with GPT-3.5 quality as a fully open foundation model. | +| **Grok** (xAI) | `xai-org/grok-1` | xAI’s grok-1 model known for vast size(314B parameters) and high quality; integrated in SGLang for high-performance inference. | +| **ChatGLM** (GLM-130B family) | `THUDM/chatglm2-6b` | Zhipu AI’s bilingual chat model (6B) excelling at Chinese-English dialogue; fine-tuned for conversational quality and alignment. | +| **InternLM 2** (7B, 20B) | `internlm/internlm2-7b` | Next-gen InternLM (7B and 20B) from SenseTime, offering strong reasoning and ultra-long context support (up to 200K tokens). | +| **ExaONE 3** (Korean-English) | `LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct` | LG AI Research’s Korean-English model (7.8B) trained on 8T tokens; provides high-quality bilingual understanding and generation. | +| **Baichuan 2** (7B, 13B) | `baichuan-inc/Baichuan2-13B-Chat` | BaichuanAI’s second-generation Chinese-English LLM (7B/13B) with improved performance and an open commercial license. | +| **XVERSE** (MoE) | `xverse/XVERSE-MoE-A36B` | Yuanxiang’s open MoE LLM (XVERSE-MoE-A36B: 255B total, 36B active) supporting ~40 languages; delivers 100B+ dense-level performance via expert routing. | +| **SmolLM** (135M–1.7B) | `HuggingFaceTB/SmolLM-1.7B` | Hugging Face’s ultra-small LLM series (135M–1.7B params) offering surprisingly strong results, enabling advanced AI on mobile/edge devices. | +| **GLM-4** (Multilingual 9B) | `ZhipuAI/glm-4-9b-chat` | Zhipu’s GLM-4 series (up to 9B parameters) – open multilingual models with support for 1M-token context and even a 5.6B multimodal variant (Phi-4V). | +| **MiMo** (7B series) | `XiaomiMiMo/MiMo-7B-RL` | Xiaomi's reasoning-optimized model series, leverages Multiple-Token Prediction for faster inference. | +| **ERNIE-4.5** (4.5, 4.5MoE series) | `baidu/ERNIE-4.5-21B-A3B-PT` | Baidu's ERNIE-4.5 series which consists of MoE with 47B and 3B active parameters, with the largest model having 424B total parameters, as well as a 0.3B dense model. | +| **Arcee AFM-4.5B** | `arcee-ai/AFM-4.5B-Base` | Arcee's foundational model series for real world reliability and edge deployments. | +| **Persimmon** (8B) | `adept/persimmon-8b-chat` | Adept’s open 8B model with a 16K context window and fast inference; trained for broad usability and licensed under Apache 2.0. | +| **Solar** (10.7B) | `upstage/SOLAR-10.7B-Instruct-v1.0` | Upstage's 10.7B parameter model, optimized for instruction-following tasks. This architecture incorporates a depth-up scaling methodology, enhancing model performance. | +| **Tele FLM** (52B-1T) | `CofeAI/Tele-FLM` | BAAI & TeleAI's multilingual model, available in 52-billion and 1-trillion parameter variants. It is a decoder-only transformer trained on ~2T tokens | +| **Ling** (16.8B–290B) | `inclusionAI/Ling-lite`, `inclusionAI/Ling-plus` | InclusionAI’s open MoE models. Ling-Lite has 16.8B total / 2.75B active parameters, and Ling-Plus has 290B total / 28.8B active parameters. They are designed for high performance on NLP and complex reasoning tasks. | +| **Granite 3.0, 3.1** (IBM) | `ibm-granite/granite-3.1-8b-instruct` | IBM's open dense foundation models optimized for reasoning, code, and business AI use cases. Integrated with Red Hat and watsonx systems. | +| **Granite 3.0 MoE** (IBM) | `ibm-granite/granite-3.0-3b-a800m-instruct` | IBM’s Mixture-of-Experts models offering strong performance with cost-efficiency. MoE expert routing designed for enterprise deployment at scale. | +| **Orion** (14B) | `OrionStarAI/Orion-14B-Base` | A series of open-source multilingual large language models by OrionStarAI, pretrained on a 2.5T token multilingual corpus including Chinese, English, Japanese, Korean, etc, and it exhibits superior performance in these languages. | +| **Llama Nemotron Super** (v1, v1.5, NVIDIA) | `nvidia/Llama-3_3-Nemotron-Super-49B-v1`, `nvidia/Llama-3_3-Nemotron-Super-49B-v1_5` | The [NVIDIA Nemotron](https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/) family of multimodal models provides state-of-the-art reasoning models specifically designed for enterprise-ready AI agents. | +| **Llama Nemotron Ultra** (v1, NVIDIA) | `nvidia/Llama-3_1-Nemotron-Ultra-253B-v1` | The [NVIDIA Nemotron](https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/) family of multimodal models provides state-of-the-art reasoning models specifically designed for enterprise-ready AI agents. | +| **NVIDIA Nemotron Nano 2.0** | `nvidia/NVIDIA-Nemotron-Nano-9B-v2` | The [NVIDIA Nemotron](https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/) family of multimodal models provides state-of-the-art reasoning models specifically designed for enterprise-ready AI agents. `Nemotron-Nano-9B-v2` is a hybrid Mamba-Transformer language model designed to increase throughput for reasoning workloads while achieving state-of-the-art accuracy compared to similarly-sized models. | +| **StarCoder2** (3B-15B) | `bigcode/starcoder2-7b` | StarCoder2 is a family of open large language models (LLMs) specialized for code generation and understanding. It is the successor to StarCoder, jointly developed by the BigCode project (a collaboration between Hugging Face, ServiceNow Research, and other contributors). | +| **Jet-Nemotron** | `jet-ai/Jet-Nemotron-2B` | Jet-Nemotron is a new family of hybrid-architecture language models that surpass state-of-the-art open-source full-attention language models, while achieving significant efficiency gains. | diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/modelscope.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/modelscope.md new file mode 100644 index 00000000..4740c277 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/modelscope.md @@ -0,0 +1,28 @@ +# Use Models From ModelScope + +To use a model from [ModelScope](https://www.modelscope.cn), set the environment variable `SGLANG_USE_MODELSCOPE`. + +```bash +export SGLANG_USE_MODELSCOPE=true +``` + +We take [Qwen2-7B-Instruct](https://www.modelscope.cn/models/qwen/qwen2-7b-instruct) as an example. + +Launch the Server: +```bash +python -m sglang.launch_server --model-path qwen/Qwen2-7B-Instruct --port 30000 +``` + +Or start it by docker: + +```bash +docker run --gpus all \ + -p 30000:30000 \ + -v ~/.cache/modelscope:/root/.cache/modelscope \ + --env "SGLANG_USE_MODELSCOPE=true" \ + --ipc=host \ + lmsysorg/sglang:latest \ + python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --host 0.0.0.0 --port 30000 +``` + +Note that modelscope uses a different cache directory than huggingface. You may need to set it manually to avoid running out of disk space. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/multimodal_language_models.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/multimodal_language_models.md new file mode 100644 index 00000000..8aa642d5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/multimodal_language_models.md @@ -0,0 +1,111 @@ +# Multimodal Language Models + +These models accept multi-modal inputs (e.g., images and text) and generate text output. They augment language models with multimodal encoders. + +## Example launch Command + +```shell +python3 -m sglang.launch_server \ + --model-path meta-llama/Llama-3.2-11B-Vision-Instruct \ # example HF/local path + --host 0.0.0.0 \ + --port 30000 \ +``` + +> See the [OpenAI APIs section](https://docs.sglang.io/basic_usage/openai_api_vision.html) for how to send multimodal requests. + +## Supported models + +Below the supported models are summarized in a table. + +If you are unsure if a specific architecture is implemented, you can search for it via GitHub. For example, to search for `Qwen2_5_VLForConditionalGeneration`, use the expression: + +``` +repo:sgl-project/sglang path:/^python\/sglang\/srt\/models\// Qwen2_5_VLForConditionalGeneration +``` + +in the GitHub search bar. + + +| Model Family (Variants) | Example HuggingFace Identifier | Description | Notes | +|----------------------------|--------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------| +| **Qwen-VL** | `Qwen/Qwen3-VL-235B-A22B-Instruct` | Alibaba's vision-language extension of Qwen; for example, Qwen2.5-VL (7B and larger variants) can analyze and converse about image content. | | +| **DeepSeek-VL2** | `deepseek-ai/deepseek-vl2` | Vision-language variant of DeepSeek (with a dedicated image processor), enabling advanced multimodal reasoning on image and text inputs. | | +| **Janus-Pro** (1B, 7B) | `deepseek-ai/Janus-Pro-7B` | DeepSeek's open-source multimodal model capable of both image understanding and generation. Janus-Pro employs a decoupled architecture for separate visual encoding paths, enhancing performance in both tasks. | | +| **MiniCPM-V / MiniCPM-o** | `openbmb/MiniCPM-V-2_6` | MiniCPM-V (2.6, ~8B) supports image inputs, and MiniCPM-o adds audio/video; these multimodal LLMs are optimized for end-side deployment on mobile/edge devices. | | +| **Llama 3.2 Vision** (11B) | `meta-llama/Llama-3.2-11B-Vision-Instruct` | Vision-enabled variant of Llama 3 (11B) that accepts image inputs for visual question answering and other multimodal tasks. | | +| **LLaVA** (v1.5 & v1.6) | *e.g.* `liuhaotian/llava-v1.5-13b` | Open vision-chat models that add an image encoder to LLaMA/Vicuna (e.g. LLaMA2 13B) for following multimodal instruction prompts. | | +| **LLaVA-NeXT** (8B, 72B) | `lmms-lab/llava-next-72b` | Improved LLaVA models (with an 8B Llama3 version and a 72B version) offering enhanced visual instruction-following and accuracy on multimodal benchmarks. | | +| **LLaVA-OneVision** | `lmms-lab/llava-onevision-qwen2-7b-ov` | Enhanced LLaVA variant integrating Qwen as the backbone; supports multiple images (and even video frames) as inputs via an OpenAI Vision API-compatible format. | | +| **Gemma 3 (Multimodal)** | `google/gemma-3-4b-it` | Gemma 3's larger models (4B, 12B, 27B) accept images (each image encoded as 256 tokens) alongside text in a combined 128K-token context. | | +| **Kimi-VL** (A3B) | `moonshotai/Kimi-VL-A3B-Instruct` | Kimi-VL is a multimodal model that can understand and generate text from images. | | +| **Mistral-Small-3.1-24B** | `mistralai/Mistral-Small-3.1-24B-Instruct-2503` | Mistral 3.1 is a multimodal model that can generate text from text or images input. It also supports tool calling and structured output. | | +| **Phi-4-multimodal-instruct** | `microsoft/Phi-4-multimodal-instruct` | Phi-4-multimodal-instruct is the multimodal variant of the Phi-4-mini model, enhanced with LoRA for improved multimodal capabilities. It supports text, vision and audio modalities in SGLang. | | +| **MiMo-VL** (7B) | `XiaomiMiMo/MiMo-VL-7B-RL` | Xiaomi's compact yet powerful vision-language model featuring a native resolution ViT encoder for fine-grained visual details, an MLP projector for cross-modal alignment, and the MiMo-7B language model optimized for complex reasoning tasks. | | +| **GLM-4.5V** (106B) / **GLM-4.1V**(9B) | `zai-org/GLM-4.5V` | GLM-4.5V and GLM-4.1V-Thinking: Towards Versatile Multimodal Reasoning with Scalable Reinforcement Learning | Use `--chat-template glm-4v` | +| **DotsVLM** (General/OCR) | `rednote-hilab/dots.vlm1.inst` | RedNote's vision-language model built on a 1.2B vision encoder and DeepSeek V3 LLM, featuring NaViT vision encoder trained from scratch with dynamic resolution support and enhanced OCR capabilities through structured image data training. | | +| **DotsVLM-OCR** | `rednote-hilab/dots.ocr` | Specialized OCR variant of DotsVLM optimized for optical character recognition tasks with enhanced text extraction and document understanding capabilities. | Don't use `--trust-remote-code` | +| **NVILA** (8B, 15B, Lite-2B, Lite-8B, Lite-15B) | `Efficient-Large-Model/NVILA-8B` | `chatml` | NVILA explores the full stack efficiency of multi-modal design, achieving cheaper training, faster deployment and better performance. | +| **NVIDIA Nemotron Nano 2.0 VL** | `nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16` | NVIDIA Nemotron Nano v2 VL enables multi-image reasoning and video understanding, along with strong document intelligence, visual Q&A and summarization capabilities. It builds on Nemotron Nano V2, a hybrid Mamba-Transformer LLM, in order to achieve higher inference throughput in long document and video scenarios. | Use `--trust-remote-code`. You may need to adjust `--max-mamba-cache-size` [default is 512] to fit memory constraints. | +| **JetVLM** | | JetVLM is an vision-language model designed for high-performance multimodal understanding and generation tasks built upon Jet-Nemotron. | Coming soon | + +## Video Input Support + +SGLang supports video input for Vision-Language Models (VLMs), enabling temporal reasoning tasks such as video question answering, captioning, and holistic scene understanding. Video clips are decoded, key frames are sampled, and the resulting tensors are batched together with the text prompt, allowing multimodal inference to integrate visual and linguistic context. + +| Model Family | Example Identifier | Video notes | +|--------------|--------------------|-------------| +| **Qwen-VL** (Qwen2-VL, Qwen2.5-VL, Qwen3-VL, Qwen3-Omni) | `Qwen/Qwen3-VL-235B-A22B-Instruct` | The processor gathers `video_data`, runs Qwen's frame sampler, and merges the resulting features with text tokens before inference. | +| **GLM-4v** (4.5V, 4.1V, MOE) | `zai-org/GLM-4.5V` | Video clips are read with Decord, converted to tensors, and passed to the model alongside metadata for rotary-position handling. | +| **NVILA** (Full & Lite) | `Efficient-Large-Model/NVILA-8B` | The runtime samples eight frames per clip and attaches them to the multimodal request when `video_data` is present. | +| **LLaVA video variants** (LLaVA-NeXT-Video, LLaVA-OneVision) | `lmms-lab/LLaVA-NeXT-Video-7B` | The processor routes video prompts to the LlavaVid video-enabled architecture, and the provided example shows how to query it with `sgl.video(...)` clips. | +| **NVIDIA Nemotron Nano 2.0 VL** | `nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16` | For video, the processor is configured to sample at 2 FPS, at a max of 128 frames, as per model training. | +| **JetVLM** | | The runtime samples eight frames per clip and attaches them to the multimodal request when `video_data` is present. | + +Use `sgl.video(path, num_frames)` when building prompts to attach clips from your SGLang programs. + +Example OpenAI-compatible request that sends a video clip: + +```python +import requests + +url = "http://localhost:30000/v1/chat/completions" + +data = { + "model": "Qwen/Qwen3-VL-30B-A3B-Instruct", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What’s happening in this video?"}, + { + "type": "video_url", + "video_url": { + "url": "https://github.com/sgl-project/sgl-test-files/raw/refs/heads/main/videos/jobs_presenting_ipod.mp4" + }, + }, + ], + } + ], + "max_tokens": 300, +} + +response = requests.post(url, json=data) +print(response.text) +``` + +## Usage Notes + +### Performance Optimization + +For multimodal models, you can use the `--keep-mm-feature-on-device` flag to optimize for latency at the cost of increased GPU memory usage: + +- **Default behavior**: Multimodal feature tensors are moved to CPU after processing to save GPU memory +- **With `--keep-mm-feature-on-device`**: Feature tensors remain on GPU, reducing device-to-host copy overhead and improving latency, but consuming more GPU memory + +Use this flag when you have sufficient GPU memory and want to minimize latency for multimodal inference. + +### Multimodal Inputs Limitation + +- **Use `--mm-process-config '{"image":{"max_pixels":1048576},"video":{"fps":3,"max_pixels":602112,"max_frames":60}}'`**: To set `image`, `video`, and `audio` input limits. + +This can reduce GPU memory usage, improve inference speed, and help to avoid OOM, but may impact model performance, thus set a proper value based on your specific use case. Currently, only `qwen_vl` supports this config. Please refer to [qwen_vl processor](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/multimodal/processors/qwen_vl.py) for understanding the meaning of each parameter. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/rerank_models.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/rerank_models.md new file mode 100644 index 00000000..b6f2ffa2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/rerank_models.md @@ -0,0 +1,49 @@ +# Rerank Models + +SGLang offers comprehensive support for rerank models by incorporating optimized serving frameworks with a flexible programming interface. This setup enables efficient processing of cross-encoder reranking tasks, improving the accuracy and relevance of search result ordering. SGLang’s design ensures high throughput and low latency during reranker model deployment, making it ideal for semantic-based result refinement in large-scale retrieval systems. + +```{important} +They are executed with `--is-embedding` and some may require `--trust-remote-code` +``` + +## Example Launch Command + +```shell +python3 -m sglang.launch_server \ + --model-path BAAI/bge-reranker-v2-m3 \ + --host 0.0.0.0 \ + --disable-radix-cache \ + --chunked-prefill-size -1 \ + --attention-backend triton \ + --is-embedding \ + --port 30000 +``` + +## Example Client Request + +```python +import requests + +url = "http://127.0.0.1:30000/v1/rerank" + +payload = { + "model": "BAAI/bge-reranker-v2-m3", + "query": "what is panda?", + "documents": [ + "hi", + "The giant panda (Ailuropoda melanoleuca), sometimes called a panda bear or simply panda, is a bear species endemic to China." + ] +} + +response = requests.post(url, json=payload) +response_json = response.json() + +for item in response_json: + print(f"Score: {item['score']:.2f} - Document: '{item['document']}'") +``` + +## Supported rerank models + +| Model Family (Rerank) | Example HuggingFace Identifier | Chat Template | Description | +|------------------------------------------------|--------------------------------------|---------------|----------------------------------------------------------------------------------------------------------------------------------| +| **BGE-Reranker (BgeRerankModel)** | `BAAI/bge-reranker-v2-m3` | N/A | Currently only support `attention-backend` `triton` and `torch_native`. high-performance cross-encoder reranker model from BAAI. Suitable for reranking search results based on semantic relevance. | diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/reward_models.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/reward_models.md new file mode 100644 index 00000000..ef447463 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/reward_models.md @@ -0,0 +1,28 @@ +# Reward Models + +These models output a scalar reward score or classification result, often used in reinforcement learning or content moderation tasks. + +```{important} +They are executed with `--is-embedding` and some may require `--trust-remote-code`. +``` + +## Example launch Command + +```shell +python3 -m sglang.launch_server \ + --model-path Qwen/Qwen2.5-Math-RM-72B \ # example HF/local path + --is-embedding \ + --host 0.0.0.0 \ + --tp-size=4 \ # set for tensor parallelism + --port 30000 \ +``` + +## Supported models + +| Model Family (Reward) | Example HuggingFace Identifier | Description | +|---------------------------------------------------------------------------|-----------------------------------------------------|---------------------------------------------------------------------------------| +| **Llama (3.1 Reward / `LlamaForSequenceClassification`)** | `Skywork/Skywork-Reward-Llama-3.1-8B-v0.2` | Reward model (preference classifier) based on Llama 3.1 (8B) for scoring and ranking responses for RLHF. | +| **Gemma 2 (27B Reward / `Gemma2ForSequenceClassification`)** | `Skywork/Skywork-Reward-Gemma-2-27B-v0.2` | Derived from Gemma‑2 (27B), this model provides human preference scoring for RLHF and multilingual tasks. | +| **InternLM 2 (Reward / `InternLM2ForRewardMode`)** | `internlm/internlm2-7b-reward` | InternLM 2 (7B)–based reward model used in alignment pipelines to guide outputs toward preferred behavior. | +| **Qwen2.5 (Reward - Math / `Qwen2ForRewardModel`)** | `Qwen/Qwen2.5-Math-RM-72B` | A 72B math-specialized RLHF reward model from the Qwen2.5 series, tuned for evaluating and refining responses. | +| **Qwen2.5 (Reward - Sequence / `Qwen2ForSequenceClassification`)** | `jason9693/Qwen2.5-1.5B-apeach` | A smaller Qwen2.5 variant used for sequence classification, offering an alternative RLHF scoring mechanism. | diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/support_new_models.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/support_new_models.md new file mode 100644 index 00000000..b71e06c4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/support_new_models.md @@ -0,0 +1,320 @@ +# How to Support New Models + +This document explains how to add support for new language models and multimodal large language models (MLLMs) in +SGLang. It also covers how to test new models and register external implementations. + +## How to Support a New Language Model + +To support a new model in SGLang, you only need to add a single file under +the [SGLang Models Directory](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/models). You can learn +from existing model implementations and create a new file for your model. For most models, you should be able to find a +similar model to start with (e.g., starting from Llama). Also refer how +to [port a Model from vLLM to SGLang](#port-a-model-from-vllm-to-sglang) + +## How to Support a New Multimodal Large Language Model + +To support a new multimodal large language model (MLLM) in SGLang, there are several key components in addition to the +standard LLM support: + +1. **Register your new model as multimodal**: + Extend `is_multimodal_model` + in [model_config.py](https://github.com/sgl-project/sglang/blob/0ab3f437aba729b348a683ab32b35b214456efc7/python/sglang/srt/configs/model_config.py#L561) + to return `True` for your model. + +2. **Register a new chat-template**: + Only when your default chat-template is unable to accept images as input: Register a new chat template in [conversation.py](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/conversation.py) and the corresponding matching function. + +3. **Multimodal Data Processor**: + Define a new `Processor` class that inherits from `BaseMultimodalProcessor` and register this processor as your + model’s dedicated processor. + See [multimodal_processor.py](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/multimodal/processors) + for more details. + +4. **Handle Multimodal Tokens**: + Implement a `pad_input_ids` function for your new model. In this function, multimodal tokens in the prompt should be + expanded (if necessary) and padded with multimodal-data-hashes so that SGLang can recognize different multimodal data + with `RadixAttention`. + +5. **Handle Image Feature Extraction**: + Implement a `get_image_feature` function for your new model, which extracts image features from raw image data and converts them into the embeddings used by the language model. + +6. **Adapt to Vision Attention**: + Adapt the multi-headed `Attention` of ViT with SGLang’s `VisionAttention`. + +You can refer to [Qwen2VL](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/models/qwen2_vl.py) or +other mllm implementations. These models demonstrate how to correctly handle both multimodal and textual inputs. + +## Testing and Debugging + +Please note all your testing and benchmarking results in PR description. + +### Interactive Debugging + +For interactive debugging, compare the outputs of Hugging Face/Transformers and SGLang. The following two commands +should give the same text output and very similar prefill logits: + +- Get the reference output: + ```bash + python3 scripts/playground/reference_hf.py --model-path [new model] --model-type {text,mllm} + ``` +- Get the SGLang output: + ```bash + python3 -m sglang.bench_one_batch --correct --model [new model] + ``` + +### Add the Model to the Test Suite + +To ensure the new model is well maintained, add it to the test suite by including it in the `ALL_OTHER_MODELS` list in +the [test_generation_models.py](https://github.com/sgl-project/sglang/blob/main/test/srt/models/test_generation_models.py) +file, test the new model on your local machine and report the results on demonstrative benchmarks (GSM8K, MMLU, MMMU, +MMMU-Pro, etc.) in your PR. \\ +For VLMs, also include a test in `test_vision_openai_server_{x}.py` (e.g. [test_vision_openai_server_a.py](https://github.com/sgl-project/sglang/blob/main/test/srt/test_vision_openai_server_a.py), [test_vision_openai_server_b.py](https://github.com/sgl-project/sglang/blob/main/test/srt/test_vision_openai_server_b.py)). + + +This is an example command to run to test a new model on your local machine: + +```bash +ONLY_RUN=Qwen/Qwen2-1.5B python3 -m unittest test_generation_models.TestGenerationModels.test_others +``` + +### Benchmark + +- **(Required) MMMU**: follow MMMU benchmark [README.md](https://github.com/sgl-project/sglang/blob/main/benchmark/mmmu/README.md) to get SGLang vs. HF Transformer accuracy comparison. The accuracy score from SGLang run should not be much lower than that from HF Transformer run. Similarly, follow https://docs.sglang.io/developer_guide/benchmark_and_profiling.html to get performance comparison: TTFT and throughput must meet or exceed baselines (e.g., HF Transformer). +- **(Optional) Other evals**: If you ran other evals, please note the results in PR description. + +## Port a Model from vLLM to SGLang + +The [vLLM Models Directory](https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/models) is a valuable +resource, as vLLM covers many models. SGLang reuses vLLM’s interface and some layers, making it easier to port models +from vLLM to SGLang. + +To port a model from vLLM to SGLang: + +- Compare these two files for guidance: + - [SGLang Llama Implementation](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/models/llama.py) + - [vLLM Llama Implementation](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llama.py) +- The major differences include: + - **Replace vLLM’s `Attention` with `RadixAttention`** (ensure you pass `layer_id` to `RadixAttention`). + - **Replace vLLM’s `LogitsProcessor` with SGLang’s `LogitsProcessor`.** + - **Replace the multi-headed `Attention` of ViT with SGLang’s `VisionAttention`.** + - **Replace other vLLM layers** (such as `RMSNorm`, `SiluAndMul`) with SGLang layers. + - **Remove `Sample`.** + - **Change the `forward()` functions** and add a `forward_batch()` method. + - **Add `EntryClass`** at the end. + - **Ensure that the new implementation uses only SGLang components** and does not rely on any vLLM components. + +Note: make sure you add your new model to the supported models list in the supported models documentation. + +## Registering an External Model Implementation + +In addition to the methods above, you can register your new model with the `ModelRegistry` before launching the server. +This allows you to integrate your model without modifying the source code. + +For example: + +```python +from sglang.srt.models.registry import ModelRegistry +from sglang.srt.entrypoints.http_server import launch_server + +# For a single model, add it to the registry: +ModelRegistry.models[model_name] = model_class + +# For multiple models, you can imitate the import_model_classes() function: +from functools import lru_cache + +@lru_cache() +def import_new_model_classes(): + model_arch_name_to_cls = {} + # Populate model_arch_name_to_cls with your new model classes. + ... + return model_arch_name_to_cls + +ModelRegistry.models.update(import_new_model_classes()) + +# Launch the server with your server arguments: +launch_server(server_args) +``` + +## Example: Implementing and Serving a Llama Wrapper Model + +Below is an introductory, step-by-step walkthrough on how to implement a new model end-to-end in SGLang and then run it via the [Offline Engine](https://github.com/sgl-project/sglang/blob/main/docs/basic_usage/offline_engine_api.ipynb). + +### Implementing Our Model + +To keep things simple, this new model will be a simple wrapper around [Llama 3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct), and our goal will be just to bias the output logits for each `forward` call by taking the square root of each individual logit. + +Let's start by defining our model in a file called `llama_wrapper.py`. +The first step is to import the necessary libraries from SRT, which is SGLang's internal backend. + +```python +# In the file `llama_wrapper.py` + +import torch +from transformers import LlamaConfig +from typing import Optional +from sglang.srt.layers.logits_processor import LogitsProcessorOutput +from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors + +from sglang.srt.models.llama import LlamaForCausalLM +``` + +Next, we declare a new `class` for our model and have it inherit from `LlamaForCausalLM`, which allows our model to access `LlamaForCausalLM`'s predefined modules and layers, such as `LlamaAttention` and `LlamaMLP`. +Note that almost all model implementations take in `config` and `quant_config` as arguments for their `__init__` method; `config` and `quant_config` are passed in via [`model_loader/loader.py`](https://github.com/sgl-project/sglang/blob/bf72b80122fd888bf619d17b96fa3e323ab809fc/python/sglang/srt/model_loader/loader.py#L219). +Because we have inherited from `LlamaForCausalLM`, we can pass our parameters directly to its constructor, which will set the member variables for us. + +```python +class LlamaWrapper(LlamaForCausalLM): + def __init__( + self, + config: LlamaConfig, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__(config=config, quant_config=quant_config, prefix=prefix) +``` + +Now, we want to define the `forward` method, which is what will be called at inference time. +Note that the signature for `forward` is essentially the same for any model; you can take a look at the other models defined in the [`models` directory](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/models/) for references. +To see where exactly `forward` is called in the SGLang runtime's internals, take a look at [`forward_decode`](https://github.com/sgl-project/sglang/blob/bf72b80122fd888bf619d17b96fa3e323ab809fc/python/sglang/srt/model_executor/model_runner.py#L1705) and [`forward_extend`](https://github.com/sgl-project/sglang/blob/bf72b80122fd888bf619d17b96fa3e323ab809fc/python/sglang/srt/model_executor/model_runner.py#L1724) in the [`ModelRunner` class](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/model_executor/model_runner.py). + +```python + @torch.no_grad() + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + pp_proxy_tensors: Optional[PPProxyTensors] = None, + input_embeds: Optional[torch.Tensor] = None, + get_embedding: bool = False, + ) -> LogitsProcessorOutput: +``` + +We now call the `__call__` method for `self.model` (which is a member variable that `LlamaForCausalLM` defines in its `__init__` method), which eventually calls `LlamaForCausalLM`'s `forward` method. +After that, we feed the `hidden_states` into our model's `LogitsProcessor` (again defined in `LlamaForCausalLM`). + +```python + hidden_states = self.model( + input_ids, + positions, + forward_batch, + input_embeds, + pp_proxy_tensors=pp_proxy_tensors, + ) + + res: LogitsProcessorOutput = self.logits_processor( + input_ids, + hidden_states, + self.lm_head, + forward_batch, + ) +``` + +After receiving the logits for the next token, we can finally perform our biasing step. + +```python + orig_logits = res.next_token_logits + res.next_token_logits = torch.where( + orig_logits > 0, + orig_logits.sqrt(), + orig_logits + ) + + return res +``` +Now, our `LlamaWrapper` model is created and ready to be served! + +### Serving Our Model Via SGLang's Offline Engine + +The next step of this walkthrough involves hosting our new model offline, so that it can be served locally and without an HTTP server. + +First, create a new file called `run.py`. +Now, we must ensure that SGLang's `ModelRegistry` can find our model. +To do this, we first download the model's configuration and weights from Huggingface. + +```python +# In the file `run.py` + +import asyncio +from functools import lru_cache +from huggingface_hub import snapshot_download +from llama_wrapper import LlamaWrapper # Make sure to import our new model! +import sglang as sgl +from sglang.srt.models.registry import ModelRegistry + +# Make sure to request access to this model on Huggingface, then export your +# `HF_TOKEN` to download the model snapshot +llama_dir = snapshot_download( + repo_id="meta-llama/Llama-3.1-8B-Instruct", + local_dir="./llama_ckpt", +) +``` + +Now that we have our model on disk, we want to point it to `LlamaWrapper` by changing the `architectures` field in `./llama_ckpt/config.json` to be `LlamaWrapper`. +That way, when we pass in the path of our model checkpoint to SGLang, it will know that we want to use "LlamaWrapper" instead of "LlamaForCausalLM" as our model. + +```python +{ + "architectures": [ + # "LlamaForCausalLM" + "LlamaWrapper" + ], + ... +} +``` + +However, if we don't link our `LlamaWrapper` class to the "LlamaWrapper" registry keyword, then SGLang won't be able to find our model. +Thus, to register our `LlamaWrapper`, we want to follow the steps in the above section titled "Registering an External Model Implementation". + +```python +@lru_cache() +def import_new_model_classes(): + model_arch_name_to_cls = {"LlamaWrapper": LlamaWrapper} + return model_arch_name_to_cls + +ModelRegistry.models.update(import_new_model_classes()) +``` + +Lastly, when we create our `Engine`, we just pass in the path to the local model directory. +Then, our `LlamaWrapper` is ready to be served; for this walkthrough, we will use SGLang `Engine`'s non-streaming asynchronous generation endpoint. + +```python +def main(): + llm = sgl.Engine(model_path="./llama_ckpt") + sampling_params = {"temperature": 0.2, "top_k": 5} + prompts = [ + "Write a short, neutral self-introduction for a fictional character. Hello, my name is", + "Provide a concise factual statement about France’s capital city. The capital of France is", + "Explain possible future trends in artificial intelligence. The future of AI is", + ] + + asyncio.run(run_llm(llm, sampling_params, prompts)) + + llm.shutdown() + +async def run_llm( + llm, + sampling_params, + prompts, +) -> None: + outputs = await llm.async_generate(prompts, sampling_params) + + for prompt, output in zip(prompts, outputs): + print(f"\nPrompt: {prompt}") + print(f"Generated text: {output['text']}") + +if __name__ == "__main__": + main() +``` + +Now, when we call `python run.py`, we will get the outputs of our newly created model! + + +## Documentation +Add to table of supported models in [generative_models.md](https://github.com/sgl-project/sglang/blob/main/docs/supported_models/generative_models.md) or [multimodal_language_models.md](https://github.com/sgl-project/sglang/blob/main/docs/supported_models/multimodal_language_models.md) + +--- + +By following these guidelines, you can add support for new language models and multimodal large language models in +SGLang and ensure they are thoroughly tested and easily integrated into the system. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/transformers_fallback.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/transformers_fallback.md new file mode 100644 index 00000000..3c7dd961 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/supported_models/transformers_fallback.md @@ -0,0 +1,58 @@ +# Transformers fallback in SGLang + +`sglang` can fall back to using models that are available in `transformers`. This works for most decoder-style language models and support for vision-language models is coming soon! + +## Example launch Command + +By default, we will use sglang implementation if it is available. Otherwise, we will fall back to transformers one. However, you can switch the implementation by setting `--model-impl` to `transformers`. + +```shell +python3 -m sglang.launch_server \ + --model-path meta-llama/Llama-3.2-1B-Instruct \ + --host 0.0.0.0 \ + --port 30000 \ + --model-impl transformers +``` + +## Supported features + +### Quantization + +Transformers fall back has supported most of available quantization in SGLang (except GGUF). See [Quantization page](../advanced_features/quantization.md) for more information about supported quantization in SGLang. + +### Remote code + +This fallback also means that any model on the hub that can be used in `transformers` with `trust_remote_code=True` that correctly implements attention can be used in production! + +A model just needs the following two things: + +```python +from transformers import PreTrainedModel +from torch import nn + +class MyAttention(nn.Module): + + def forward(self, hidden_states, **kwargs): # <- kwargs are required + + ... + attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + **kwargs, + ) + ... + +class MyModel(PreTrainedModel): + _supports_attention_backend = True +``` + +Here is what happens in the background: + +1. The config is loaded +2. `MyModel` python class is loaded from the `auto_map`, and we check that the model `_supports_attention_backend`. +3. The `TransformersModel` backend is used. See `/srt/models/transformers`, which leverages `self.config._attn_implementation = "sglang"`, thus the need to use `ALL_ATTENTION_FUNCTIONS`. + +That's it! diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/wrap_run_llm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/wrap_run_llm.py new file mode 100644 index 00000000..2d21442c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/docs/wrap_run_llm.py @@ -0,0 +1,47 @@ +import os +import re + + +def insert_runllm_widget(html_content): + # RunLLM Widget script to be inserted + widget_script = """ + + + """ + + # Find the closing body tag and insert the widget script before it + return re.sub(r"", f"{widget_script}\n", html_content) + + +def process_html_files(build_dir): + for root, dirs, files in os.walk(build_dir): + for file in files: + if file.endswith(".html"): + file_path = os.path.join(root, file) + + # Read the HTML file + with open(file_path, "r", encoding="utf-8") as f: + content = f.read() + + # Insert the RunLLM widget + modified_content = insert_runllm_widget(content) + + # Write back the modified content + with open(file_path, "w", encoding="utf-8") as f: + f.write(modified_content) + + +def main(): + # Get the build directory path + build_dir = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "_build", "html" + ) + # Process all HTML files + if os.path.exists(build_dir): + process_html_files(build_dir) + else: + print(f"Build directory not found: {build_dir}") + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/assets/.gitignore b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/assets/.gitignore new file mode 100644 index 00000000..fc787e33 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/assets/.gitignore @@ -0,0 +1 @@ +!example_image.png diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/assets/example_image.png b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/assets/example_image.png new file mode 100644 index 0000000000000000000000000000000000000000..851d085605c01fe89d887e9ca9fe6a33bcc8c93b GIT binary patch literal 57365 zcmb4qWmFsQ6D}>Zg`&Zo60}Gt4#nEwRtOMWe#MFfcdbBim*5n4C=fJAaS2dd0t5)s z;>C;h^8au@+>dwmJ!j9JcXnsa?!3FR&ph*Q;olk#nVOQS63(MXk8u8b*l_+W;mG6Q zK7Rba@gR5);|czgCwO>Io88sPzf|81giinhk zmYR~5oRW(2e;0X#`*0546M`pC2q=jOi7EfT<=+n+0RE%E$FFf80dXD!9^nEW{p-VF z!ohj;&PAOHE&XL!%>aUVZ8g7bv@=`#QY zP>2)90jmc47I%{DQjfo=E~~U5}|hG#ny&R>|v}UsAB53f7*5 z4aam`V$jgu=@VLU#Df9GM>vmh{~!Ag$9PYkK6-}#Fe?GTd9a83828E3rw?lXbMaw^ zNB#sTL;-#Zpk!qe6xN}73wdwpM$Ino{weWuzCu!6_dhfa(S!D504@MW4(ErI@YXE* zx`*W=bTd8e1jTHg?GU?|uvNB-$xFW5G5F%=I)hvhCEFEXp^i}S|C*w6;=7S(qx#8a zOg$oAdqVKd0aLiyoWUcosiN}Osm@n@%(fi1lF(S0tI?~k4wY^n-)*%3mTpWG9pH25 zbxhjCixH;>Ia^e!hc&Yn(9JEIqUd*}-7oyyiqCQV&H5O`F!_LD9XisN?lWh0Tcd_> zeh*ICtyC^U+vff^yoEqrLIK-WOI&w+cMp26GU?QzaCnh@l~ZIl{j9!|hQ6s`YMrs0 z+oenzioU`7zn{2T!?G@e@D+N(rpBJ;PWihG2xcstpZ28GAQryzJXNLlj39}|`oi3Rrb#^txG zK=taivD3tTgfHp$fhTJ*BOPB^&p9mCoH+%`!>PW$A#0J9J~s`}@+YQHw~q zwnpgg)B5S}SN?A0daBRTj{o7*{jK$?FA|a1a7;ceKVucxbpDko4{Y2%L-g&MeObn+drH}4+wMmcFQ}y5YDbf9^SXg*gbAZnyrWm&gb;pf(|Gm z`Nex4@TSy$au)c2Gm@fVY*CAzdIixtq@o-?dtP2WRuKOo$uJ(fqnF#R2AVIgeeJz2 zCZfy5RxdRMk^YARE~YXRD}Ah)F*IJUncPvZ5z;{&UJdE1Ze9$FPXRJJd;l&iI-Qwv z_C@&n`Gsc|@miX^s)5+r>(fQQ%m5`?f;l2dXA33D>JU9mWvZzdo?zRbS99i5X&EOk zP{|ubM2!<%ON!UHj?E_hJuB9kWsq^hkuhDr%`fq}h3`|05aCTg zQORtlFa|QU;~RbyAgZ`ur@yuS$XQUq>~!cv@tl?4gup#cTO+c^uEi#j2BO_L++^c7 zon_vz@*&15?8S=MvdtfvrH`qS5TDt-ZET5L9LLBUPp}oNl7>RRMWrLmP43=MR{=!n z@el+SSxUFOx&)**XP^J!tjBWx!?|J1RzLBxF$@~)PkG$j%Uk^8%0H^QL()k_Y)2__ zYLaDJ(U@#<&dD5PTtD1JH4UecNn^aJROy$+eqT(#f2<%oX6WCu-GqA5xCsDa4-Yi& zqi`30Y&hPCxK&Lshgrrb##5rQ7EeQ7(;Nz15IF6g-~A_**@hR<^U-Cc1!^F$^>)cM z;f6k}i&|AOngH3~>w7g9CnAKW?3~X+0T3h_oL*=IRMfuuhf_VTWIF__A#!8I(m)AM zA7>Nfqzi3oJ|P?s+5nJaFY|sv(`w?}MHPpi;vT;28Zjy$Ht{!F%P3)`<$00mqV*FR zdIxzH>Fu=STjIM;7_5xR_j{uLc{LO!NlQg(O0Aqqv+}Lw`jhXQmpI>n00SWnyQytO z<^h`>{7u!x9l$@FV%_Ti$Y|;xeuNRLXAvo2kdeh9Y?(v-gC0$c{f_2EM3%yH3n}>J ztX#3T^K1hI?aXr%RhNTdNUn33LgQq^SiM9}IDBD|M?buC3v}hdhv+2|TKQa>VtPrC{viP(L3sV}u&9Hi5`5JSvWdDdvyM4E_D=xx>129iM zkA_wp)i>J=q{wo(+rPy(k@;PIGxceIZb{PN)0!~zst+p{!Ph}RfWQaLrN2Y>pF*_! zyiJCTZ$NrxZ1YD>wZ(_Hkt?SRF|O!Hh5@%iI+7_Y1>Hl5ZQ26=8`E83rPM2hqWMi+ zp;gS(d|w^ZBgu}={2h3>BKv3N{vIFmWo;yEjhP-`&&d$In|;q9^l4MP0?cQevoy^i zrH$mD#Khe#_iB9k`xLbQYcKhSBc9p5Pv0IbZlOVq*#8_dESeB6O1fj|qU)gU_r_I? zPu5=RUbQ{EsAE@dZf=0fh?oN_h%=GHCFS!xK#t|~TZfEiyzCu05&k`w%Lky%Cl{6~ zMGVi07@BI+IM$4<$2Aa|#~i49d@=~%ZN626w4t=kHZgNR(_&XDdPQqfZA!D@uJ8I8 z)!kohVtj~NdRNwy?sI0w|AI6+Ra(It1QHuITXdk`8wWbP#85OVLfx>Tgk4nx%KFaW zS$)|hg3<9dz1&CG4=|&^I4eH8F7#K|vuX*pHLz_uWP0qA#b`+LItxpx4-^IDBdnT` z!W^tynfs|@%L`Q0v*L$u%$6Q*cuBE-zsfGi*!yYrw9b1ID?BBLIQ7e4#)h$IqWzOB#lEV%#2#EV*Bl=BDe9s# zgl(1Z+Pm1}C<(`Ud>NQG>3v+KfmNE2tD45V+Ov&_l+Z zleUWJwgKCGk9k(UTzk5x`eRq;#0+;}=PfZ{RApu`&f~>TD@~_d|#J6XDFn3v);SwwBefdUY2toq7n@}z+ zrBdhnwR%`Vc4g;S_s#T%y=&H^EkI$kZ~PPY-3$Ai6Oa+L3_2kZZcjz))>tc`ohn>6 zKvRmJ<@Y@5+)EsfUz1z586U|bAnnXVQs=!G~OgKcPV!B9O+*tL7&skB6-3?GVB-WBh!7WM>!($w-~!Gas40iQ*mRZDsXjYhNd z�*{=qGkPn*hQ6dbTesr;OK~_SMnA?> zT2cssE-cjq?O~)yMY)WlmiNKusGKO|rrC%2!ook>$x>oyb__tO;lm^Lgnv=-?^}WZ zs2)sLU-i2KZN!qPt(p+e9edYEwV!-S0SH14I@e{eb>!VJ3bAC9myABm@u(=i>MD)h^T%1^_qI3-bNbvnjG)T8O5aI`+Ot0eDE= zAX(n!xAMMDO8YBq!*r3Pyb(?S2HancS>FF}PWKATj67iJ&g>pGJ&Jw9-M?0TS1mU1 zfAHX!PGTpQu+3(FWj>>D23Za$szeDpYbmpm_qRuO-!u;Q=c}N@D?gxv!zP?wyDCbF zErWFuR`IgMP^D#dQr!``sf{OYrah6ypjT-Lr05zRybB8+UI7otNDK+w ziwZELNB`0{K9POX4h|qmN~SD&{^vLbU%^yCBOJ|QM(#j80Q!og@w062DnqPlG}Tut z;6)U5yVI1iKwqOt1kW`8C>OU_vnktaN>Zi8L7j(Nq)GUbX2dO?@d2~6OJ38s(J<;V>xS;A}tgBN+5kqf}cq!}p4wLe&qjVQv zf5nlHY0&?pU$1}dhuTxcN>I@n@k|i+qNMwLYNBQxbNtdI&eN~Ni;cy_pnKNqD=X+XX%3EidI66|VL{|C| zhQ)&<%~`G(kdMVr4$%GG`Tl$Jsim1De9yZdbvrS?R*XlHd^GHi(FB$j2Xp|6)G{w| z5}7!AvQ267k&hx<73}Ic)pLgS!g?Zof-B}qRL{k}+hIam6AE6sQhe`_-q0K94=vg& zDN1>ZXNUi$*?-dEekNXI^l8gSb~KG$($#N={zpjIdc>_DHre`@IFzuDcj(0|*?_gS zT{iVGUu!hrlwZ$;s$tn`cNjG8bj+!#(9DHMwK;h1644jd)p>X{f%vKRdI*ziQ>EU& zwB1*qZ(z65W8(gWbE`Nr8qmmotB&{g(hS<*vprCb7tTBnEHkO?I zR6!zts!o0_!|Y#k0d4S_ya6LWb;f17u^lR`bh&YE-BH>g2tyq;eB~t~)66)wSAPG) z;nlf#8ME9Nni0GCuEWJym0EU=O^bdr-&~nNp2pgQ_Nf6Q)9@4BaU?tl|F?JUU%&7? zJ_QqyF&Db{7%5Y~?EIx{o=Ty`s%ui-SoU^ocUNVav>{r}93rk=iJ)ty(X5#gX03bS z7gPa#LGjKS3TuxKft<~SAiN4tR0Nr(#6;ePTvAV<*ALD-Tw8aLW5 zC#zZ4x)I<*>c%Tj#iYU}cnK&!=q6>(Lo{kU8xZK6(Hb#WgMYw#8) zLL~>tS3wkcj27`2rZ)1N@yw2Zlkb*Weq0puBbI8iKTdpEa2V+Ujfir8skz79DNsq( z9>OU)!@50E>|IDjhi3SZ1AK(^8B_Z}=2#vdm94ib7_R&5?%h9}u}U^;zKx=ji(L20 ztoZw|#P|k%u@O$3of6vRR%@GUls-OXJwI-;)rgr$&aYBXjj@jds(&6CE7C@mY|S=1 z{g%TwaXjLdVS1%VWI3=HKU8BifS&LDsE*(P|IZ^!6woeQ`%(()0N%Ui}%Itnmq}crvMMAPa&To z4|Y0Kx&SP1{7lW{)>NY-({6(e3F~L^vkRFzIQoYZIQS3ei-s{2YOYRtJJ9#z3pv%h{<1sOe12V#$>&OIt)EFWYIc*7;oUWH zdZ!S0^un>=V7gMiQ>2mF$YT8m;Etax$15{b1Xv)JU`LZp?D_I@4IsOQ5B0%$z!H8) z#ck)glHK}Nrkq$b%g7ZMG%=(sdkzPOmiuQgQ&8ahI(Y;uc3M=*qu1T1 zD4ytgn)#o&>hp z(~6WOwyukePmch%ESWX-iYf7SO69N$d}L(~@V?Kj7W(zScYiqs?ew~Ut1D_&t}N#o zE_S&<9(a3WOsymJ;Ndh~tPM5GWtavJ_1B;5yxwnLVOsKp9k(rF<$e~#U{%&l*)!Mqk&RNt6eY`4SX_TYolnl@qj3Fl2N`NB40|9ZM+4YXM1g$%1w(Eo}+y0o;no;AkF+8UJ#T*b>DV^adNAq?qh6{{1y{Gw@3@`Oai-&zcn`@m6HXK5J=$ zCWRfNF!k-|YWVHatEITq4wbm+X(jZ<{Y*+rRPFe^r(0w3dv71X0dv?N;0FqrPr=pO zQ;0c-sM(JvUoxc>kZCp&`YG zuL%F5sgawfr;-JX&ehBVK87)_7Cp~(8Z&bB>o^o^xw!Bz(~5q6tJhxmh&m^>J((@; zWrAO_?<~JfiwbWagZ*+we`^jKWov6kMGGJ)`h9q!=b%gJb}_Vxn676gu_RDz(TX?n zM_>K!PJ!&Y1p?8S?UC^xoRMig`BOZ_>|3;jhT4~)(7j7IM~u1l2X~U=K`+1oZJ?F< zxC@_e>_)0y7S9y<{`H}B203IjQ&vyZY_%Zl`3Z<0MlFM7_nl_v^A_HABfveYR2dTMiTXJUhcdRZeyNciV!|XVw{cDay*^g&0B<#|i zS4N|6$~Fsi6QIrNq1;U<^>|udD=rTC{ne31s_$8dmq~{_XweQFzb`-0mA4fJ&Y3Se zjGjxg1uo%X+B>~UmPjT6XA!12X5v)cc;;M%4@v1SB`GCmg}Fr+?E-Xx6gysurk=`| z;N9>Ak!=qCvESS8o`SiN&HlSq4*Hxf)eSfk*-pCBih+CT``1g(Q;CtER3;e|-c7!< zv53oXA0+qXw{1zLH30JVFdt*Zc#V(5`c_u@6t_?PI<+=ayb7wOfQxGwMt?~`p(Q}m z5>0zcLY!j4rJbW4F0Sqy{LgOeVlTT;z?!X9Woq7(p2>@{npV<5EHe{>y#)BCN#uOf4YN7prPB1uez2C@|C z7&4DUF9LSlyU=xDg=T1Of0Vrw9(L&m;yu!&F}7S>L9{GzK;3reN- z^^QaxzWB2sj-<^Ho9m2b;l*8~Stq`_%S(gj)nu#IEAUfbme(JXKYy_q-BxH|I5r$I zzBk$2p|30@IxNiuB3q z4!Q9K%t+eUH`~}8LAIjg)|sZr%L_Lli7^L_{VR?3KV<(SW0_TpiIH_kMx*5K(`x=+ zHOH^0#l=XY%uG%7*0-->cs>8&Oq{&-s~B6fhTfMcdWJKeE_Zg?R7{;nEMl%3U7T)Q z?TBUB<&s42yvm1qO(bce;DS^lU8K(m``9I^B%1)ftqE~#8ZOym=^HEGY_V^LXMFHo zuftZox=Yb6#$uXnI^WLVx;7fWda?BdriGCGe%h+!z+$+#Caa3DmEJ;8Gskn4dX>lp ztaGX6+i=$pf9#2+*>k;r@|1%hC-Gd1K>DyoksBI2*q2a@7OB>_nvzampr&RaUm+>8 zabuMxS-l=zarL2VUs(Mhvrb{p%uI;y~pmUqh=Um?pUI5&3KTkRz=1RLqO=pMm$6(N^L(&D+Ps^ z$5&@8zrbLl$wj_F?%e`ky?G~jY#Aqkiga43}qG4S&;qbFCwF-shkH}KK0@TV4 zVjxi4|0hT3CuXPa0GeqnP~Jh@$z^%t57h^FgDC`NwaQfb(q@hzE? z1ZRPKssMm&s9KX(>dDe;j7FAILW}gQ$qcXiYl;2aLS1f$Ap25=))yLN@vQehIH!GF z3WIdp_D$fG+c@p>00d(e@L)#`zOHI{>47na|8QW0Ki^sIZr5lj#+^OZ@yRyI)$H4T@?ZP`(kn%LmGm2KgcwT1Xn|=Lrw~hnqc>oo@ zJkTc*Aa055(&V|N)dvMtbZYDrjDyE%N{lL3K9i=;~_DbifYOr z!+dY={V=W|l`8A5Z97z=np|Gm$R@llx|nz~tWpn>C}(pg6l-<$XP};W8asAaW9Z+k z!yuI2Q0v3>q}1rYH-Z~n1BgMGZdgLItph=Z2iH(aZZA90U@a+O27$!FbR~?YC8$xo z8+|0!POr@HR)_s3(_PoE(|Hq~)|?&Ci8d7i8}YHA;9BlldwXR$^mWwK%pb3T$ztf* zZP!`3*+<0mh5DC7w6=B81zT=#N+H5|J*e1$Yu{j@WZLQVDt;>2O7HExL(;?;vK+CQ ztVG7FOrYF9AM(pW(;%08vN#npl#prkoX$Fe;YNz5`;XNO%&^II(|(lFkAol6uepp& zb`M5>HOYWYkFyWQE%i7bV}uK~ELd}PhN<-@Hn|DDc~IZ$6~d?_YUv3zhrP7pe`9#-Bg92eK}D_J!Pi`iXX3CMvB~4?$I1*%-0<5Ti-mGy^7fdl zl$GP6^AAy+HAfMm2tvN4#s!mjb~-0o#=)?z5Qoe1@vqZIhF}A_d!d#u%PZTfw>v=) zo+TFTPF+*h6E@5Md^)L6-M&8@PE7R-Xm)PEUy5Q#w0mTUk!~-Pf1UJRIZSefdbz)V zmFAi6Rfnh?YQR$^x+Glm#YYo%G8dtp{vgqVXOxm&syJ9uC3w|qRKFP5Vq(fI*q@f)@WRCT zRivdfcv~RHAapPLtL1AaX$`UxbIpG6?b%@;VADFt%{A3!#(iD=@iXEn&Z{fEz@@3E zYJPaP>uqI7Bq>t)x<|px?lKOD+75BQv4M)-{KJvY2Do=@S5pD|RSMaxQoTyLueFJ3E0um4cCG~8kJOK0=FHe@I)!Q%qlRBPay?`Li z8K*3}zik_EQOWQMJ_B|ITkHhD$CNJwCLPQ@6gN?S>U?sqQ8}t{y@|V05UtTBNh3%s z)hzF_9&PceN%qyMAXj_J?j0=PE1me&J7;5kB7^70Fh+wk`4B(uzC8Yt!tWxQ;t?(E zBFc36)an>kdgl6uF$JeMzWB>8x~1fiX|PZskin%)X)%0HhtAS$_abYXVJY`?^>*YJ z5Vca4G8-LH*0NT~K1l|r5+|$8%-J0?a%)twzGB`!ppjJri-B5xB*bsvwK<}vcwK}h zkZCf@4o{yaYl}to`rDbo$>0G-0#AGWbJ^sYN|tgfk8$jB;UYfM+w zmd?0>FvBL|b~p5M#>?rpma}s94`eeCfVS?WxlHl$Id`0iEx6XPk1sSrTPQ*+mnsc=?V8@I4s{6b z0Tc0K%@WOlxw4$tAy=k*%BtVzH^G(QB_j=nSdg7zUyM;tzfYGC%2$lsO)Ead;Btm! ziUUasHjhm``8+T1VIb02$}yKHvst;-uYHiZ_B*?((etw>7#Q%--1_vIW~_tfYdb-Q zomUE#n-&1@2%|ol8?Wh^b_p46RT?l+m^=~IY%m->79W3u_tAyYG&#f3*a@uX{>Qpw z^gq-8tThs6OnW0Xpj3ml&Kl9u7T<@M&GX~rF_qIFUfYbxt)bJ_Z05!WSC<7yI7Taf zUpu}vGW?WypOi7-Okn>3k4U8*7KrMaNI9~{P0NXs8fr*l`Ci%LALu!(`FR4%CPjX= zFe5wX5@&zI{wxdDJRRQ>Ha@ds9O=hgOEKIVU-@DS<3i6kZPP#Z1@14O*RNu7{?@eg zZTgvy1a_edwSP8`@GLvrjrfVX3~|k79Vm8YSF?{5WKE&Em6&qN$M1&ze&M&P$mSpg`!P8&j^oG%kR$ts$hX*Nm6s>3xWG56!%wt8+(K_FV!ZYRfWfKKN}*>yud46=)Ya#*M0?v#huh_b-v={xJ@ky z|X?HD7VkQ?wrwmJqxT&qMI@hUKX-e(yaA6usTCp!R z1oS5>9AFx)oPvq6-sOmKY%eW)uwlJxGvrbYmm)#xiQ{cwHs)yxAAgI$w))#^;-d#M zJi=aRr>R7j<<_jM{@TnR^K(S3igNTwYNYh{kJBDA1OQwBM*+pUnXtJ74%*BF9kVEQ z%sXoKHTXLV<4m-3u|owi<>;1y(_pmnFU9o~UaZi{_s?>1ujG?sB#hiQrgh|HWomv} zajAzf^qDYrSfu33O5=5}H74L?Uy z+hQDT5R8pxpro4P(CUr93MIo>tcMsF>W^XP)#f}qso&ld$ZUj*c-JG<&`)DNYs5d^ z1aDR;7qzxIYC3?|7Jo^wjq=goS4__*B+`|YaeyatMG%HbLfd>%8HiI&p74 ze`p@s%Rz`OAx<)cSf7E?=GPLE&Cq?VGOzTf{t{r5!ZF1gC173A>if2;uFv1Ge>0ph zDv$7;rM26#fPVsz(IBO@1O2oVh{%xpQ+gGCoU)jK1wWJMw9ox42h$R;*lp-o z{h?L461r&frNTSz4OA+%ajugx$fZGEpEDwBC}3q*uxYTrvm#P2i$9;KK$rc&8|Fnd zS+-mw{H-_R#cK`<$pK_G7n&Z zP*i%~){%ic3k5tmq2n}jI=)@h&Ccx~PJL7N&H%5Mx&_le9R21anm@Di#a+*)E`Itm zaOjlNhd~I3zh!t%+?@y~R}0hQ;uwqt7KTN{k91aK+GVZ51#M8;&Cw~2HdQ6#u^Vo* ztQ;_k;Cpn!F&xCv6j7P~$r+dtuYJ;$zDNB)*TQDDScab++wGo1?k5}EQ? z{XQQyV65s=u$>pPeb@Njuy$TGvtW?wfn5-2nm&~1nI@_IHEI&YW2n`_aQISym%F_z z>rT|&Q1(%3`2jKFOvq>8*;oEkvJLe}vn-~PPxX=bab|}O*}wr=^K(Vx!sZ!{RJ9Ri zN+D3YJCAcueTIOVRNt3}&Y*^)^mVcBC`ffGoJ|0>xv7R0UJR(`W)*e8jbG9*6g0C043ph_{p1d_U zoAJ_@d9P&I${l0ESa;#5;47h|e|C#yH@o#@cF9al=U-B9Qe`zb_&MmoT=GEKGtTO+ zS*JfJD(a#m{i*|%E^szaDTM-}qc93`4rOL7_-V5TmK_U^F6U~~^ZJb*kO@+oCVct* z9>=2`VSw@wZ40$7lfd6lC|%%31MWbIqJq zaE`)gXBQQ>*bVcO*0C0WhD+J7(w!iwz>H?WSbprf3Hk*W6Ib1qfNlJR zj7yfVejI+gyzFX3kX}v6WR-nuoFF;p-`~g^gWzj~7~mV{@{;(Jnn{+qInY^iSjWs! z?fW^!#wERXQ-|2_!o(8mjB;tz`hRIfjQ%-<<-8oW95>WQEbDhAYMhbdw z#gHD*8oWJ|P~C557eL)19E)R3FC%!a@ODQ_Trew!nc ziAu>5$kE+_n+lt3ouUG|u&Y9`l?SAf(UN?{+kp48u%nH^^Cnr+q3 zJt_TR6^uT}0b{mx_F zw9q86-*bu&|F)dxOpD}^|B?(i2GOR;&Lb@x5Vkq`idLXYLqtV8jCubJ@f%6=8eGg@ z)sZiAn&QCmA49Nag=~-xeRkRr{e}dxUvz75uUcdVGPTy{9r?Qc%&$^=o#vyk)1>}l z0SMQ+Tb!tlb0o`HEbz7MYAHL5AjLR-BB>939ilk2NO~_&uDjiWJ{c_nh!{wVkx5TR zT|e$Pv8)iodZ>joy^@%bof0uI{-IdbtnKbd`nRX)m}L8to&s8PxNEGXBVeio@$D#O z)y3mdTN?SJvD!wu+l`<(o1B?zcjH~%IpHkYq*i1(fX6pKm}HvZW74HUNUE+sgNAHo ze`)a$ZS|+hY;g*{yHL$l@Ew)&a)~gDR_B_Bh+s;XqEyQSbK0!7jXA$YV?5KZI^bX> zoUhHHlXqFhkPpo#S!+LR4;UvL`O?dNTCkbIR8Pos9K9KNTHJE>#;%vjhZV(oH*2>l za(Aj@#NOeEP|clzDzkV?sevb+yFfY4y#0S`X*fvOf;YJ$3d#N`51-TcR!Fk%C~rgk z1YxEAG3Kdd48Ebp$NE?9jP(M4X2z3{LT9Hjn~$hG8dbkj4h8M9%)lrsli;yp{1V_* z?sKwu?rAyNzaZ`8jTc2*(Q)FKi%_N3og$~E40jdOiyH@H2mNm?iiCR8pSr#9Q9dq{ z%1zu>L}uZv_2h>we>Zwj9kI&B#md|pFS5DTo51V~_o8k3-p%V8sn|n`0{Qr;2R_?H znx6T}O*3RW&Cb?nK*~_;_e%md8pOPh z#nriSGmg*1~AVn9d72i;wXM9#LI5jytsKQLSJogXhb^M@Cjbkgb(T@!&_6WtH zu@<9{hs8MarOjMk!TmEtig+DM@nl4EYd_VA-N%6NP^|7e1dlICZNJ#hv2kd9*jllf z$-4b}#HNu?*{Y!#|M3SCeh;=5MXca$gH4?5yULg1iSeH8q%joaN=iTl!;O`ZxQh&# zh1W?G`1q%`y($5>!$UdcH}^7UNdpUA+P~V(7(n@)7KyVROHKw^bUMu5Aj2l`C1xN7 z>-f}|*5%DfwU}X=^kM)4;1KW9*d)%z6b%$XfbXHiShDA$Egne><`Gpq_VQs*7F$?dFzIgouwhfPd9`AaKh z2a6)Up(q{kD7L4$AK)5M2RMS2T8(SO?k5!-3-7-}PpSW2Y<^iW02hA9Mr~i|G(i{hV5<**cei7} zY^he|C+Ad*Z3?ZGxnzH?17=#{U!|Fe_7;HASm@fBaDVsm2hC-P&0}x>2Mr46cyU1k zim<5`jjdhR&hyt{lyxkern4VcsbAy4iY~Ie>fe%GAz*fK3L}vx6JDQ^;^d-_+$XEc zOo!cF1gasHstL-LQKnSP9E^r7%kN!~2ijhrrFupZ@i)?B^n6MS+b2Dv+k4|h`6*-5 z``b3M?2WS{>Tg9?I}vlQPKgDo)cK2c3m{q-^TF3sfCsA2?+vq3k=rKgRc;4*p>^cK=>N;4jnbtjuC z^aT+swv2)h@2)>7>yYQjj{JFxkSPPf)eo@H9=t&TvG5v&4cLHhgjFQ9e8Q!ld)YNl zH3G?WGgQrxwrlTQx?r=kQInul+h+InT(s^Ww4-PYSqVq@DXwgz8(TGk&HmxE zHJm3qG~c}*#n|;?2R%ceYWWLpkFj(Rc6=sDe$2bsd4R z^zbmlrSlnAX64>?G`!?Pd*2RV1*VObqw&;B)C2ej-sj2g#unKK3=#>+CpBT8vR4L! zHHZwZkU#ZNe;|zB&L9fQiB`d$EuO30p?-kQ@XrR5-fl6v{zW`oZLc~mGr>_QgB;z< z;HcCLy)hdr`98Zk0WM6{M&sq1CiLe{7wY(~(;Ry#ig%{@$)nQJoLA~uWo?y02S(eu zOHTE$KG(NR&%QSwo;WHQy(G7;Zu#QbT0dA;PB0%1Q==VFB~I{2$~{;uNQLl)r@NOl zs~5-d8RW0zruWu|`bs}Np~3SHUmS@o+HZwvOX)aP0x^{Aq8xBMslxUUn0@UN1E4v7 zuV}I|>nq!xmhx5romGdxNJC;CF|yNVu*$ z8Rwb!Fb!Hyk;TBJuTdsYD8pJTS7B7w9bP+Y`VXgw+t>%v=)&DITcbh*+fy>C0QjTcm%zMkZn@qXy!6Q9Ds-x^({OT^=)4+%V?`)45CFvEs2C zSo81Jj>uVjj+~;&fOkC0i>k0h`c>wqaL&KhN3o=&i~$|HwV<>`V!Oq(Ga$BU8#07k zI3=+#NQ+oO>^W|3TT6L1EwKrYnDb#wix#P+F~gQ}b``?vzq7_VDnDb+>CMMx$P`vu z3@KO9Q5_u>T{Oe`d{!2z?Acw{uT@6KoGRSSEfoXbKuql$yy_0W#c}6V>YFD-Mt)@F z1P15HX#4yBIn09l*Kz(mSs6B2L9(B52ZveYJ1kBc_ak9tHSR;%Z}dp$U4BUiF5NMu z#`_S;v+=^Xa!kfDz^Fa315G=)Wjmsr;1ud0AM86`z{6VltLnh@yXv%m+_YUbS{x)q z&(fapP%@0ZZ!A6#Cl(c&Szc`DA(zN0`@HUTqV49mMs z4kFy3*CE&Qe(Tj{)tkRnmJHV#O{lzfdD1o9xR@U5u>Juo2t8$oW^h0(aLqbKOy)D; zsCB3H9YWBVFN~16lu2NoM|=uW2u#UfhR)3oiSL^IZoQ*AZ@lB;>Fi&ov=5M!Z<3bE z-rUmT*wVV57?}5SM%$zwe(AsD>24JHQ~#%B5ho&!FV!HxLhDA|TJv_Oo8}k)YOfC6 zm4J5oLY0q-v*h9f&w#;kF&d?4wKMQZkKnaRNp#Y}~e=K*k#sehUMRcxc1S1(sI zS4$jzytu&z!)0h&M?Q~fh6%@OLJ56k==)P$QQT&J6&+@`I{|9UQo>Ny7T*JulYc<8 zuG+b@io^QfG87C-(E*38pmU}Sloko7d_DlLCaKJm4Y zD6Q5A0UVKDa32+mRB@~A9#?*Jaa;Oo)BMnIE-*=aNi!fuR7&hgbWAWnj*p@vAtJeE}){NNA0-j@RP&}T1~!kXqFt3z&1zSe9bityc;{)%b$JR43;I3b*K zD8ija*2TXNI+UJ0J;nB;jbap@l(>Q_g(DRlU*)(dm?qpPjwK&@o(70exNOKfsopro z__k~+{KF9tKq~O9elv%p!FK){ktj&D4;xe)QQPc_O^5U%YYXnza<4udGdwaSI2*=N z4bCQp%tiXro@~qH0-l#T-jrd3EAR_bvQy)Uv3T%wZ&vho2YBrauL1;9n_9^#=O+N z(cB;Z0Ol<@ivfZ(0-n=uFA#7v=~U_b^%Zt?OI2S@Ki=Hk_LjH@WIHQ~!Dp4syVbW$ zu`o8|I@H1}#A&5$NRi&H{%*yq(@r!cVYl`!Tmbv2R5qe?on?|y<|W_m)_~K*BWs6` zFOeU+Q=o&%7S>%RvTvu}%Aj_U4~L;A0P^z(pXVhNGCWaHBFmfEq3x+hH}sbX2?b5j z-d?WSVWi?_JMWSq9C=Hsn#6U+oD$*?`&gG%Na|?rN14Mh$b^V;#=WF1L4DMg+hdP# z8kYF7PHx|V67JtjQ|%8lCXi!Wy})B~aK|$AdTPmwDXo{c^86 zB#}YOc>E~iR|X)sO-gK|6mJ+2A`#^h9J}2`^{UO#^F9sP{-U;(w4HUhrN+5<(BsTK z(f%4rM$d;&*kIiTE;Hy==mUuoXAe?s*=ESicU!7DngJc1!_E)DtCYZP&vP4i3RLJ)(8G0(zx9Qu1 z)_B&FZr0nfo8iY8E^?5MFApy0kF`}^Jd}g2iELQ5pQGZ~=DZq#$Zodtjz@`;nK|s_ zT{#!s#;E(Fqa$f|XEEUsI5{`O$g<$~ilpsYHYsXr4W3hRL|$#j+j+ShMhl0L7rou4 zFUKO=HQ;h@{iwM|r}1OcPesLX1#Y8cbjX(vw0_kEZ8FNQBR3V6)N};ILB-vY_OE4I zYaf+IAGK8`+OBR#zQDI|n|d+CS5TEaq89Q_#r1YqtiDt)E44AAgPJp9mz}EXz_(j1 zfgxX#oK@oP#5)yN^H*Ur2BTahC+g>uw7ugX0o=s>BwVZ~^P6XZm%jChw+ zrS$7=t+h_$N7Lk0>M#qm%c4HhsT9qXY_bA*cdkiCnh#cK&1qpkx<1?UV=4i|aB@H` z-AVb@s7s*CvtJr+Q;$q^ToL#@OClo6lt_oC+9CI9GiNiNhkBasYK9WsCk1T5j#uB}={ZOUBKbr|hrSPf9!mJCFA;y{fRUFt@f5#Q?bRD@U~7BI41F7_(?y_Fee znAGS@yBzp~=wDS&qeMl-$Z_vi4RO?pMvhKJH5ZJm$P(do2<1>gFHN#u()22<+kowK(~gWA z^m58%auJY2-Y@*rW3zkoE358Imovp6Fi7K(RQGX9x|pX`K+lmJWE4IR3Jc8e$h)nT zSbGoIkXl?Dox!u-DV#>Z1kn8Hp6~PLuD@8McaTSPzw-UdJx+{lj8DzePD*wA#H$~| z^Bx}}S0?dcvF?bsmni(=p;tDe9@8n9H@koJYF1dYaJJf8As@Au+8QO=cF^ZqJZpf< zHwFinfZ<|I`=mqOtnqo*QZ=>O9hSvm8!VX)W01%qA@++d=#Nnsvsm_4?O8n3$7wTP zYJmGpxM@u%ujfLKsNn>2Um9$lQUu)ZGJ5@MQ#un-SsXM~lFj_jOh(1mKU+R>-=&fF-e%M;h{tzUY3!nZ_BPly(;|Ta7map)=KL8ecq-=h9&cghSelroS2gO zIiYNdUs73MS$1@;G`uNMkX-SEE$XG&kni;RQ9g>TG=`*us;-P9Z@T5tpe0EBE!)%A z`#(KyD`tU}sV%-D_Jgz8ZH^IS#OUp|U-HC@?^EWs8KQKJz31FwHLf@TE2{yaj*2PT zszjZ?rj~6=(vjrNyYeW>r}ZBuV;0{nJ~f1UjK=K`!*ht)IUrm4xn)t|s4W^=CBjUr zs}o52);cSSrMu80zw{+*9iM2OgKZY#bF4O-ZNhYi9O21+LykW852%|bGU@kdy`KZ^ zmZJ`^#gW)dx}=BV?y8ciuKnsts=ubG!>dtb>C3f8Ygm>)cjjQNn`*00zr8`Q+Tz@< zcL@WGf+<4*jC#17!cJrLYR%f_%e4N^XIWzEi-dS_t_0g{+%))=h!q$>zJ1e|Ho3$_kg+YYZy$7sxkjTL*tV^Qb(wJ}Q7+NlE$ZF5 z^mIq)Rm=2Nz1pXPf$EK$PqbLA@vPdIm_cIa9RWi$KmTd+^ z2+c2Gd~H`qGv%`1E73%obclPr+@v)|ivu>{TE;fqgF6O=yiAZ&0t<~b#3npa;M1pwzAe8W5p${W>2j>n)A}H?_%`J-#_veCbZ3mv zkN1N@x|2d)?eWTd6K3ga5X|I2bsWG`wN#VNDZA3CM@mlKCJkX~HsDK3PalM%Lu6Zb z@PcocdA}LnqHV4>6`gf;L3!3(&y$9no6$VWL)sHayyS~4#-%rjIL{HX5*|`;ORL(V zjix5)np1DXXgHUBQKZFiK+v7#W2Ip~o zwk_^r$e$2f&v2y(*gQ_@Ie~avn_IxV+}am$oT_^LMLd4r?za@!zF zj9GBVd7I0-O{@`IkT1i28^k1BP9U?=RPN`qIhv{rXCcV(XImoU43^87YV-@|&3w+Q zK$<&iq>@G2G1D5DuiDDq+pY-{(g!YDxnw*%cYL7rFYQ{V^#02)x^Glk?j~_0L}}47 z;i}}Hohs7tl3&RFwRI6aXuisM9ao68SnelQhvDR4?+4OBwp|Fj&xht$YKEbz0|K4F5%Fm*^W-QeBDL-RVxXnF(&w&rBP9`BFo;X zi)4p4xtB23I~z1@p#`!#f+#$+V`yo?WJsJ5DigNK9>EgGsG=l{giDx)qheJIZH~Mm z$CT>kA;ycXZoE%oWF?5uJ*rV;*<*;0DP=~M%tga^NO7oW+mvSFVZ|YK{{VL(>>nC; zciUQd7R5^Y>0m* zH9+c2of}}DIS|$PA*Z6-MV3jPTyiH19_ZAn%*&DHIM>MP716n&vX%4j? zPaqK|JHRNr_ipSeNs_q;kdF@@=?j$GR}+qDsNppbRG;6y&eZcQ3c%b-V^v4R&@^$k z2e*DEl?KaCH2322$D1J`qK#()BZSJb5sO;*U(kw%LN|aT^;J3pz8f2sN~x$_vc=YfB+=m0TT8m{{TYzchD}5vZc!R zVutznyAM7slZd?trMrDzn)^#=>qXiywDq1FqS(c zZSN2lvM;ez*0HwBS6eO?LL<3IQ-12WHqcUB)jD;y+$3m$+?z%7muBzZRc@)gO|{Bc zjJoS=x*Xz*=lJRSNSl!RW?IbjZlSc?ZVgM90wOq|!zbQRR87deo^`DQr?#mEOjVEoSiNvP4f@4)mMg*)DYxsMAaXC7J-tkgfH;TtPu=;>&q2$OXG{_M+^?ayw&?^z_!B@O`u` zgGNocjsqfd`74eN^}ss?6L-d^`RfB*ylY!VWpBvbvWKZA;$LW&qq5qO*z)ASWK*z_ zgmIPli9H`JW?fZwxLzA^q1N$AGo36p@d*(1bmRP0ZtZhZ&1PzNz03&^GDaS45M57F z@2snQ)XYwv{F{UbP!Z84jANk+VxptzCG;(PhfLF?y`Rkb53E}YW~d>nW85veiqy*l z^Gp{6TB{^BvZf%+M%fa*;9C zQMmFPi_4Zn(k1Uu1>wC*aGor<**xKvA}ohj$wgXa$F&TUw-7vt>2J-E$U%J>d{ynK zHFepRo?GsnzPdNKP78p`AehMw`;<3)g}>_h`)ex4Nx8UftNu-yFC({K)D&9uJ33oV z!MR@TuxDILndaoZ%^%$yc}p6DylW`>e^0DgSo|K`f$=$QkrS-t%sG3!xYa-Z0QnPl z*R~;(b+Z+!Th0`4l^k-6;i4krxmO`Q*)OF#J9kqUmqzb0hQoA_EJL_9Ofp30OWeFN zA8VSoZ1j7TnXM)Ed6CC;9n^hHh-;<(WVbF2Hzn1K`^suLs)?5VuG$52ZCYPS72*ts zE)yk6j!_(X*5hNt66z6fx=Y%wF6~`49P6E?Mbg`%Vn=wGc&<-=6Cj6_v21y<15+*>>BsyQEcnPHBBFrEUKJ?<`xzkstfX85h~1IXd<%83sCM zPTK5z+^)9COOx}sj&%J|8j6zj&h;g?t`gmS*He0v&Z?z#$rLKPOqn%{n(~pDKTGAx z=`+SPp446uBIaV!+m9F397JkPM$ohtmA17q$)|7Hvfxf~{{UE~aN7R!hTc5B=@nKk z>Gz0~hsudIUD6@@MO?~gqru^j^AmicBqf*GuNxnvkL`<`cCbqJ+BF=>EWX2WEYVQZ zh7GbaY_h7Qmf}QCZ>N1}7#N7&q3`C9sySQM9yX4%7SuWVn{2>j#oJ5=vA?ohY zR98n4<%5-v;FzREA@ss-EQk2<#dLWH}Lb}FjxW-|Iz_kPbm>{mKo%_7TlOS}uL z*f(G`My_nA@s)RWpK2<0`j^y|Ry`KZHpx(~*K>VFf<389*%sJmGp6Cgavqhkw`Jl< zZ;6zbF>*-{y^bT5UVAw3sMMOnQ$w@dWm;_oFmaCa(8!x{?(2PAq+Qy@;;P-cViWGj zl<}LdJn^X;t{HD5C0^B4RaM@;x;@dmIn~dG~Iu53V&)>0K2*;+E+*N5O>xe;#91na z3WeB-+3Ni@V?yxpI8B<*vrB}4yOg5OLQk@&pSHec*`$0Sq35^1dE4eT@^*QJ!Nq?_ zrCOG|lznGohA)u|;@fS#?>Ic7Ut~tJ8%4M6%l(#pOs4v_{{Tp)i;()nmwA&3@GMQa z@i(GF2n7eh3jXgD>-s@?<%B5D2 zz7l9iVZ23<<&U&J@cTpeYMW?{$~kH}-Yz?o`uld!`zO{GRJg91I7;hCZ9N6@0bT3o z$A0h8RPD>arUk=M{5z!fi*?dhLB|%QAiW&@X}DAs6XjK?uL9kCx|3%UH00+^XW5k$ z*>QKT2-Q_LePmzGl=H%eX}Y&&ngS)?Pt8b9MxWyITqcFVP*7YGn}3~AE0%`h@(Zs` z_xYM#7@8iYB+cK!LtVs|5`DZam(xk1B3qWpD2S*kil&HYi`c2=-M_QVRBq99PNcN! zi<9^LJw6mecQKIj3JAOXOS^K=6JOE+w9O}T19>f%NG6J4k4ND+h^niCC@Lo8Ue7nv zR+fzZmFM>)O-ZI4DC^j2%jo!gxETJ$uEPt=ed7NBqb_>Kbi16c?$#D#%xT7?5fK($ z(#Pj}G+CszoQDm;d8B$Eil-3Qc_mz_wqFLlvD@2`bsy?H=>#~*hxj=*;d`aZUpk;Q zhQpUi+azioFL!5E>0}+tD%?n>+N!TuqOR{UcJEZ}Pg*5bT(QNJ#hT5dT!2v5QD?}w z!#FLBmzyCO+)PE!o$S;c4;!&Gj7~r%-Q@>|&q-c1Oqq-!!x3!-e7HEPycf=tH5*Y& zh~to+Y*mmg;DzY=&K`L|3M-CqeFg1|4ij`M3 zE5r>uZUQmm6qHWXUYbA9o+)y<-EL-WGTfHMwU58dqu%|$o~4Fn%M#_hjl?Zmkj;+` zw_7F%h;kqN)moOfo09$!ZS(uz9T`e}m<|sLIJ8_mg~4+n>Go3@Id{{W?1Zjnk97kY=Ps(prs`nBTe5_NxKRkl(bA<55YADv#4 zc8x~c8S*@Y3qM%-s!rlX9?vjGltr?xVh=YAh;bU?{?PmrHL)zbiBL*~2oCd9lt)NK zcheZrM-}D7vF}u!y4LK$bA-vbUV#zczOk8=TXCj~w4Sv{9C9DEMfRy4OulVISoUeC zu14C%vrTYGJFJra>a%p_z_o;xL@Ohpi~XXY{@Tx$?%qC5Tv(;A8WGm#TOr7Zii)gd zlWyh8(20H8D6)7pYBD|V11zrZZC7{bqZl3yml0zJrlV=OQB|;2th(x!(YG?krjeK6 ze93G{*RDc6UNBwnc&R|Q$jdB8o8GP6Z~l%uWOQ(kR9pzm_o$3yAKtYs7y3Rl^iuM- zTbrLWfj@$*N#a`w`-)NNQ?ZgQmOE|Hn_)I?A0l;!AH5y*IL z;v@B|Ra-m>^xQ!lB;4{3HD>`Nd|Me#ET1e-qjDIWeX2n-(`?&aogZWop8nBCiwFUm+e-|i{NvlRfTAp zy10tF`)+R{eq=4w?0gCIw1Pz&LA|+m`DqRsed^v{5Zkij8n^l|X<;g|<%x}LQl7St z1qH~%jwB>P{cRIO0!?Lsfoo(IB64`VXtC8~3+X1n3~o zCDU0oZJ#_Nw;~sg?}*k)(C3aNy?=dfJD2oU$(c1Iv8yG*BwWB%q%lSO=}BHcqf1{% zxg7R(l-t;QS|AahQlEPA_37 zk>WIKY_-VKI&MdeINPqwZOFxl^A8Qd%C-Lh{{Uu=)ogGiz_wo?%eTH@qsQspR1noK-K-G@LF69}f1W~e<*_LX1%~N`(V%4R&iIjqn$gEVGGV)q_dT#2L zsrrFNEQpD7do-<+W17!FSRQWqrLK187FwS7qnZxv5q$f;I^;x!&Tgt@Wo{+PD=#9N z+UU0K&~(s~&ItIc-6K`5(BiVy(jHPA=5N`nKWhqut~BZJ!+42#Li#y>&#tSTp*HzE z%cB_xdspJGN^DJRS#&FNrpeasx-gLhVpawdd-COG?5n>1F5ope_v!L&PjKtxwYIietTr)vIXtqWjk9*?r!t`}_=Dhv4d z%vLgNMmXim^QvrikYl(nyB5y19ERDBQDn8o9I|AGiS>Vmp>#E}Ew=Y?2;qqFUBm=& z5FEKv^HhD0Zu)ajUf5tF1YmLY+YxZlRY;3+{_334HrW`=nJJeCh~wNg3zUn>7dN^! z9B}3EC$)}8Y-k)oj`i9?GSVuWg@a9}QIh$QD_FMZ>FqIZg2J@fw7{|~tKz~ti20K* zlh0Ic!0hp_7ah;E($5DoXp}NG8WG*Szf!7}s&BEJ8yicIn+rtcNxW{{D)r%tfOkB!Sk~i;2=K8t@h`w-DMBIzp z*+Wsa6|{yo9@61)Iz)(nTA{Uc8;$E(g}L~LX-GnC3cL%LDyo<5Ri+xtOxzi2*2IVW zU`(^(BJ~&ZuAfWO_wY%H(D1}NuftmAzxC`8c`xytkNBP#ip7JjJNh6`s;C3 zm3)lE$IS0hZ8eRTBR4V?dPYfvbS=tbzqXDnnvxdO-)2pZit0GKjmT<*jVGl^Js%}w zO`z)sb&cvo)j~+@7#;py|t5nqQ`j4H*v9NsWraPD?ncj>&3#=V?`Z5Y-<;1yq&MQyOnpB z#lOw_w@^;dwSr$sSZ&jBge*|^a+>=?@l+3l7S$tJ=eeYr0Ml@-$e`>7m{!d_Yq=A0 zs`>Z&cklGneu1~@-K<^Wjf%xKTTO@ny&EC+9#m*if=?W+-4ENjzKSJo|QlNGe9 z%v}RvJA=^WTpfDhAYsN5@(q`GvFQ@Wz24@kBUfv?-lElzu5#xjM4d zs>#-FMhjm_N?^QA6SwU?*1FP`y`ouFSd@c=l-;>kw-F-v_wOH%m(^7Ls~Pd~VmOa< zxxW|feV(ecJQ;O>+^v@_FM3se6wp&`9PRDhxVt_?75b}H-o3CrqIGPPl0xwVst?C) z5j#D-$>vJaopN5Hmao-hjyYrd`3`NO+}E0pQXG=$a+7g~9KGzSefbVRkWHr!Go#Eb zusJ*W-8Bnpxqtx1miFXxky{wYnYPC#%i8(BR;idQ z7s$9w;jrVP+kC=Jl#=2*^+uy5y5xpk;*oHOv23Kdh?mW5d5WOzTKdl^<%bowt>WW& zjFN1gb;17tKRTst+_E`Dqdy>VAIna7es@UIo*hCW@wj=9DKQSlyRjjcRfvUk8HBQetg$^0y`nSN7lb38+bMNFWmOZ`}(Hk zH7z<|9Vl!Nq7;b=3%T4A&^z~zOX)>T%euF|&$el&HgjlLT6WmfCim|*7jnAl_9pW$ zO8#U|LMNK0Rmk<2qP9WccNz;Lao0`^`3uaJc29dF+utur{MFs@t!ao5S+~V%ELK~O z8Ii%ZSrit~4^T{To}7x*j;{rJdx+FA;#b#ccV8lm*M%3ooHuq$e zEYX-3oUG2K{magXpsK6q=Kw_bm1|e+%SVQ>(s#=vq6)L)Dzu7c-F!^_O)p9L45>TlR zJ7v6^Utg-W&eJwz-h`d3E>_%zOnZmJP}?4!85Tr8WQwTWZ=&t@D7V+F(gtI(NN&xL z^>E(m68X5-cWF0L6@RG4likS532DcI^=_AL^MQ`>$$K~d06#b8sV$Xl%USFzQuxlD zx<0}p;QhKms>PBU(Nzo)cPcLA_|wp+v>T@;d>?si(78*IKA$(~-Ba4y)8mBP*2A#V zn7%(%SY^p~9-9Kz4sGJv&ZN-tz7MuILN^OeUS8VwR|hWheMH%NJhT&dhB|Lh+$~Q& zb5b$y7YK3Qiktf?h}cHhr|er^N4v5IBHZevd!4xpa!F%;;@$5eDyXlo;#EbzwyYAb zTEOpig7ND`!Wg|Cb7Dj;3z=eHXY7?`O>U}tSk=^HGi!lmZ6YH@9n}0=#2rcMQT_Ck z9k(|I?hK59jz$@igq*#;imWDSyOB6PF(NKlL%ziq%6?~{wqSl2wv1bXp zVRrRAU$AQ3@u4R15~{Cbu=3NwWxJK*{ewkUZ9v>MOhkln?^UwugfefDZK5RZT)ojM z5Y!u2h%7k}BNNI=yR~^}7WsaWba4EkaK3d9BzWV@Z+^9Vh^;0ed`+Z{kaCYz#)vvp{kX^Oww|Sh^caZ{H;{~c zy`CLJ*0$^YH6E7}H*hptVb$b6{vo}WyjKUmopQ$6HJBHY)VHzuvz`8RY+8w#_>(bN z$Ggfr)aH1-4m+YS*&-q3F z%bo5VDEijo`I(|9K3rbUtcz7OqE+%cghgv*@ypq*BjB;3+|?S0u2QX|8-iGlBT!yg z+AK`%8j{-2Jd-NqZ(WSUyWXvhJ9w5-2~(*>2Y=?Pt`?WJwjy3GBzX6#zR~1XTUdx< zA|sDYO0Kk+JMnOF9_}^9+sUR`{KVXJlgB*VSg>K+Vv3N2cE=xfn_r?!7%q5}zZ84S zg@P{X9@BcrrA&F3@g7{Vr`O$kXtUcSKQ>x2+ew=CWxBhqHd|Ym^MvNF5@ z(zQsaLfbXZPN$(^+ZZz4@|_G9f+nn(OYoT^kW zBQc|{!Ep_yYunKX%*2D|Whv1+)^tJe6BK1tZ{h7Ni6^(SimHEYDVHzA%TT(w^7PYC^==r`PA<$q}9 zHM=PBQC)F2`p|dot8|JvJ_TyZ(6_aZpWYCD1A2sYJX_t4a)U9qAjNzX(SB*C@BH;# z+dKkx!L_qxB3YK(ZZci-=e{j7itg>lseXj5kF_5UYZGuLMz(1QcSxoR*}We?lrai^sk9E99633^#QuJ{VZWcx@F9+Y~*Vdae zx82&l-Sp()o=vHvZaS(W-dLw#r?ymUt-J{DuUKymy?oQL5STXFazZ|U(|gkaR6$+a zlhwYdxqcbDe%SV-2KANF(RtHSvMBSsNM0uSxkQL@`K3oz4-U6=T!(Htw?Mn7ZQ$Ue z3u=8^28Yb2!ZcMr-%e@rQ!1@PryDlle`a6N_cnF@<*C((lVln1jO>9XqK@v(;ETCS zoD4yl!RQWl5bqBN!V<$?nYgY=9)G1cr!8*q924qV=lf^lqa) zN4ASDyJcibyj$i$k03}&8`%pn%aoiiyH#D&Tg9=*gB{Xs*I~FV@dUYvvUACnUe7Sq zD_cnJE|)IjQcH(|+i*XuHCCV2GNee4kIdEj^d|oR zI($jGkDSsvCrpEHxeg$xc8sY?uCLOht5P;iuW5@EnAaBXdT`YXdqFAiK;P?8LKXd$ zG?p-`?NT+Fvg8eHQ~EERyk6FR&}XYUb(KYp0vYa+Gxe=^gXhlOp7OD9~g zkvGefnV9{W<(kHPwyCGoTzL|A&0A%;JnW~f?cSekv2Dlf{A|6E636dUWJ|WQzda@> zX@7xYv&zqmyezc$%@+Rvih^$58kU=DoEYut*GRfXCD5F8&#Ks$>rmFD@Qd0`tjZ3L zx9R&+uQ&eE=Nac)A9#62r;5gJq=}|p<}Q0io7GI+SpKlw?1`sbVbUi=w0 zv9t@TmrYr=7dWI@6LFh!7XGbWJ2cn>`eFlYKdO%RC0<{?V6N9jVMlTvfcZJd(SFX}WEWf)R z_p?i!!M41E!+VP(o5hLG!enFAPdWKE>ekg>4ceDX>PhisNVYg}3|oOE9YlnptJtVU zqq%*1_ip{M`dh)S@jm2ZHu+~49>j#C%HkBF{FwPNBH{engShksx3u3Fhan=AII{jY zli2s09_wTGt6uES!RWSev$yrWp|_30Qp}Dm*zKi@jWZ%gD+FHAb7k(;BdN9}XVW6V zf2ExxbDZX0G{i!9NU{8$SmG^@vs%BwS6xNgKAo`G-?=w;EhG3zJ|aRgGRR{VCC!NB zU#m|VipGJ^8bUUJ(KcN@83dcQMHY%kSs!$7; z=GB!IM}Y4&dyCCYRIwGtlj6{NNGWps+RvLriY2CY@eli9mi|B9LLcze^e*z%5w3~n zW^I+0$B0F5WQ20+$`haAqp&D^bcZ3xb6JqtV&fzxCGn*l>cRomcUPO`Y3IrG{{XyP z)ASVqIm+fVae8>3lL*}A@t6#d7se>9{{Z`SpV~=IMx*dgxx%>V>6Y0M8hPlBtHP_) zf=Hje7Of3MW`^~Wa7Ks2xQ}QnZT6n;QYW2jEciR>mW`PrFjpjRijTw#);{sUCKpz(d}gz>7P5|A#^T0o&=tj04f*d>>|c>;)7eiN&8)RG zwf1-gB0PDb-?TzpGAiW!MQ_xzX8KE-OES$*)CO&3g>9%3R1td|f@!%G7jmMH$h((+ z(^+cJ)7oz8Qzc5{*zIpQX_U_dau!_aaJ{Zt+B;U#=Y6$Yp~g1F;U=dvWINRqQBhGo zR8*{iuC1~(=AhNK`Q+X+EdDp-3nrW<`EnBbHSdS-R%CVmd8RJwzk7>Tl zYBS>^Fgm2miRlID{S;7+jpEKDkqJECjER559;!=SX)v_utv5ZCMB}nR>Y{ZoP?zQ} z(NX$ZVGI)HETX(F2>U`^LtQabA%?8evh1#j$!S?}#>K(-#eB>AYWVP1+Ju?SrqAWN zZnpGXx@mm1Dfen+f8M{8vTh9s=AzLa0K&of5og`iqDw8>dr8xI@%e5wz*PJ+r018@UL- z^_wB;s5Ox*sz!;GCzqIoxQ-QBdNN#Z2rW4qSfi{rHcMg?#6=L2K&C27yVi)CzAe=t z_Q;8}%^ysB>AQVXTUgmfXjji7veOoOd<$N%l@8ZB)ces(BH;45?=~aURa_5Bf35Ww zoBsgIT>k(KX?U6`=vMiS1F4*xQ44zV?pka8K)C+^zd8Q^jI9KPe7%HJPa_#<%gE{d zCasxsaR{-Y>*lI*V}xAT<$SqWNioh_xZ-fBiixR*mcP|6O2+94KI6`jb9bs#f`WBh=_;Kqvu7n1O-Ptt^>6P74W2io^rTc( zByq&Zvh&qXrL>j4MCfeAZRbabJtN9BsbyEbWQ(h#^&7#HwAK7Yl~wC5=)0 zXut67Q=58Ppb&PZd%G&il{|z?o4Z+~!WMg<1;t~7Rn4ZPUPc8HGaEOq-9pC%nFXF7S1jxweZNmAu)Z{sq_J@)j!7pf}Q)CRL zmH2x}L-Ncy4Wb`pd8+r=w$`Ct*lniYLR9Q8ioSP9yEsMU-XS0FDDPpK`RfL(7ES*E zBPW-ce8pVWw`=OHTC2%TtsY*K`1Y~%PhSwf0Mwm#A3APTsu|0Mly*@Iu|!|*7dBOF zTkj)%(<&t3lcGM=X04_?-HQCBxW%`9DZ9^DJ{i>JukzD&nmXp4AI*%gW0segFGm(~ z$E0#1{wfO9+al9XNC|N!&9uo=$Q+KoWuSI-1H&VU7GsQM(pb@Elxq2ToY5|zk&m+5 zBh&S(=vicND@hfyP_o5vi82vbH8l1wXi38ol+q6neMCNO^wU&r2IsOXOeU?2H2sn@ zh>9wynkHQ*>kpH1{C;Zo(l%`bY~D8fyNrooo7z<2@A`2B?5YjjTW7m7D&KN$>BlZn z9%nKyn@Gt{MznHu7Zy`YTD85yT~2!GGT~$KVTMA?NRKni)ygCFy{xLJ(AL(8hoetm zNkj*5xE~o%u9w&bu#)$T$FW`h3thyH6AFsILM2#QdqBIVUwy)ASh(UMd1O@Gx~sI& z@m4P-yqvKm+ld^b__{JJG-Hkpat=PO%2s>=hhdXA9C(Sb7Q}eOScteJJw@))i5zhf z9%n?TGT~>8-20U911E==RSdqjWnI&mdg#JkD>Q6dcCASKi+qT{=M6;x)!e>u^KPO7 z)y}zYO~j479ece#stVK1VB4Zxu9IRURWAvl7m1bpU*yZ)_G|0ajZP_v9iU$koeyx6 zAQ4bw8JMT56|A$T^#;u?u^Di!dZ0*KA1=^{{cNh)@SfKRVcEocG-E}ddp;5(ZdFFp zc7H;mzP7-ud2|&YRv#(_Yf#Ym9j@RL|C1I+Y_L z9a!(OE=c9CM%Nk$NJ>75xLnKHojWYPsEue3fwXZC4pewCpUb>CG+$fUq^?l(mYHvi zcJ@Or#gR9Nm)529@-J|87wIbt7S**hR|IsXfP-qwrS{?ZU$j(b9_6+B%yW@^zYd&& zMU%w&3qN<5sT;1+r%28&oq&y^b|j(^TDKDODmv++nbIc(lqMHdn&vW>KB<^&Q0McwON z>+#i-a*Jl!W{{P2AVh8x3Sug~W#$q3RZ%{g#dD`Ua=A;m+#FvUkl{Tg#PbGAzXbT# z^j2o8aIwdUX2w3F8bbB`ARA^&Yk>=2U1BObxg2>SI*N(sMBLIZXJ@8>uU;2BTGnrI zZ@P`^+A?Zj4-(*rzR3BL=&Gfd4qTo`9WDa{$~cZZG4@1%ik#g9TDHh#k;j)R<)O4@ zR*>mY))rr?p>bZd~<8iz`WSy)NVAz@_{{S}hAJgaf==)~dYSpE0n$#UjbGN-I zt`wG{&2Rz6{X{6cs=m-0Ik@dtRarFExWo!UZ+*f}R)G_Gn*5)qu{|}VA%923(%CfR z*|wI$E*N3>MX6V_?bkV|YdU?Wlru?-6Sb`KJ2I$`gZ{X&Z1-r6-gQj27`C|(+jZ%J zrigpdQ-1G}Xp?LH5gR;#X}I0Aefka0THG8aA(H|^HvG9s?w+i9TlR9whOR%+?X)|@ zMPT-$W_gq9ZP<>6{{XTp`)Fcs^lR-YzxG{Er)B)EI-j&F8}02qxK5+e)@9_S1txnX zy~7sYX~7oU?&Krg`LuuF=eJjOU9>ota(vZ?9L7r}zj3syxl?zgQ9D!PN-%f&J2uLV zIMZpTUPVpW1=R=dqEtfv07tIdt?sLVVz)?A=<$223aR>PXf-=-U1k3OV^-b1lj<$H z@fPa_B-nBWOgRmU%{&XrU`f3f^D0xH81kJ}igpR4{3W(1<5t<)26m&;L0Fo<$J>lQlm7;C4BbCs&Op8v)ZzkJpoZ%1mW9@rYm!djNW<=)-CBXnZSu%V@jh8Ts^$BuDR^XQ zDR&Firc4`CuM6O&vJn#!RbMyJyY$xs;oFX#&rj+$FJsC)iu-HYeVaDTE1hM2$8al~&Ivb&jwB*p)?V!c{{W&5 zSldt(ERsy*BBHmOe7(E5xw}4^`lD8dIg1DTCeoLUcV{`43tQ8q+uRo1af^opxw(W% zym^Na>Z;3YHYqj@aXuuwVcDb_iL5t|HE9($^?GQ}W|pm{XbY2}Y_?tE0-~d>vI?T4 zK5VynzAsf622UC#Xm;xgpAMB~#NG2Xu^Cv84Uh>ER|b@&Twxlr}#X>KYM zOR6sLBSaJ-MJ@fm7GIvKy`buyL44HLol7F~BPZs%jT%iR$UzBEM0ajfOm`;g1-M&f zSS1^3<_`~;n{gNRS1;Zr+@LcM@EGiG-HL9gyp?w;W68@SXFe4=?`QXx9$SgtZNb=f z>%x?#$vs&T<|)6okdm#S@X~h_%TCA3cW>b-=ls>0ty?P9SNO*5t4`hSjG`eEk)Sx> z{9MSJs>5%=durS<{{XRf4>ETL`Pa0SY8^A^6b}b263Qxum$j;|omGS{(zQKr*fxzF zB=(|@h*aIDnSPPf>y#xOM4j;;u9ffJ8wsN=hN@6#Ho|zr$8gYix*4qXA z>Ru;<9f_`+2>qI|oZF?5uQ0q{R-@eTgSIp)x--S|oU5sKXAk~U5HxoZe@cT3Uc=(r z@$qPA%8#;Ylv@X9s3Iu%v|>J^DiC|ZHa_*?TmI9P9RC1>EU(GIY(8-+{{Sjx)->Ph zOIAHgJG^|AaV(d%TWOW&B5b|VsK>T71eCCt=%>B8XaM{muB(;|MiKc7`O_zM^Qa4? zqmTXFXGy5&4ay?I+Ug739L?A}FOMzAhvzpzx8DQX2Brc_nVj)n>N-s#{gq0LcWzpY zywbpb?#KC47k1sKy*Cvfngjf++dXSYv6HE96-R3u>6aE_EB^qqlV+&lnP^LNk|p1h zm)S%55fYrk{*N7*M(rJ***>-AVz2F}i|2%e=FZ8h)8v-#VF&qDU|t<iB(`a>%=5 zcpPlTIN~}RL?_O(BCqYIar_nbcW-bUYf0#qOgcy`AcUVzb(t4gY9ejkCBoob-|tk! zHBO=Njpr^=x9>?+{q+-g!C80=z8+<6M#g7l8g|~wh!ut^f9&uZ6#Xi-7b9kxdx?HS(N zc0R0OmK(nhh{wCF%WHpE5U5TfZgu1=zl^sSSyoj;-)oNc8%-0Oj~Zlh9A3mn z-5&KuTs)J`y@@m;=O~eM^8Wyay=Wq2jl$(3BFVquuh%r&5%H$@mA<6W5AC0BmTP1L z7m73@L5@X>c>tS*^!I**qc&g+rrw6dzu^f`NSk`wC7v9%Tpppvz4dALgUlC~!b0zP z;5zhJ)`Q-l(yF1E0w?It^wqDotqp9|;yRAUEgPD%9qjs2$!v_9TN5};)fT%9qQgzS8fHAhEyC&cM=E1)zZ9uR@Ikmk zi``^P5$5+uN{@6#(P3>lhgwKTl2|MqWPFj67EgBAvX#A*eio||JBka(= zt-m*CvQ50)H^jEY4d{<2lzhDj@pjdgWmUT7Me*Kk=ch$$*>zioGkiL4Sh(Kf8SS$oe&c%N%&hSTLvLT~#p}XP=sZ;7<|ismQZAhPX2G zC|Me3H&sY8BQC6YXjU}LilpO8_p0o2vXz-ah?0bGEODt=*J2S7%7i0KF3xIrGP!!I zp|!tw)TYto5!^2h{2wS=Ar}dvLCIIEbyu-r-G$~ea7m%&a}s<10OO)J9`UH;-w=n1 z-WrXKryFU^ym{jJ(c}LB{B`tw3&e3j8!W|Vv4!_BIpc#IvdEkH!Xx^{OzqHZms{2P zOv`K;>&9)Qa#Ub>Q9-oxWIdfc z2lTWLhZE$#?O%@AXeKNmp$7=X^BK!O*LgCF6mpE{{RgG z-ZZ_68vz_N^_*gfWGwiDSpIBxRSmAZJ|*az!?6m%YSfVv@pG*=6Bf*VLTZE0`a^6t zB1D&@XT)9oyeR^IwFbUw0Aly1Sl8Yuf6UXmax06Dn6VCB;kGmW8e{N% zo0|(2oxhuIXZ$r4r8bdh+8u$by`Az_m#~1-ae;c4lqp0aMUhrxPj?e^-J2yBr5|dF zoNH`y<+fYf`Hmy}p_bc&k5KtfXPUI;#I71~?}rNdBPp23M$<%O+A1#N zQpia>+v}jZw)8Qi(YGSL**3C@y=B>*wj+lEAl# zGA;m!hmi=54mbh)XN+Xjrj zLIit2Y2;))#et!IbW~cIiEYtXwQY}jdQ0TqEVsnSoaOno?K!o5DW-HJTUWsmC)|nC z11Suln(kfRl=W2|Mh!~O$z9uVHN22bwNwcI07rW{d56HX9gP)Jc#trny`LggZKELC zWNORZwcRsmU|`4+5n|dTMBjItE?Tv(x{^K0aX#Idh;A2JM^!=k5mtbT#>`z$@W4TP zC!55w;P8hM`l; zY;B!t6pq*W&M!)SQSVpHt*C1qn)KR3K3_Q-&WZ~={Ui42jqu{>IYQ^OQd9X;IKCK@ zukRNFPm6gM=@#(;IAEoc3MpinIlnR?3+abCgB(s}JZcKoG7RG$Jlak>Z7(hm{xd2`Zk^sCP}1DFdPbWS+a6r2iCC&*;uEdN zhrRmLn%sGE%j(eQ!iDWpMRs$#RVu<5gue<1^KmQ1cK-m0LZx(Zd3cnAmtxBtvijBU zr(nb__+OT~3zkAXLqk^bkv+^nsKynZ1+||Ty2}>ZA3jBmd044kr;X)B8oM(raU!Vs z@}BOAR7R=rvDqHZa$)D3xkwK)P{Sqs>ZqTlfrt7@c6v^5S47-+KiZOq?B87_pNDk| zh3YI-#K@OYwuP}~3+2AdhY)A7POx3kEJ~XOnc34)22p6)QzVb&AGi?x(XZQYgr?5+ znH^RGD;dq)&AGT$DkkkQym;2Pn{lsXiyF|1JIy`1bzDj5BsXi2Ln7-5ox=v+7S_}9 zV&Na@Dr{H6UiAc((eBc9;V+v=Ez{4<^@g+Lx5=_C$HbnyZ30E47YNO`gY7EQYevys zxhg3LHc58SqXj;AI4WjF~?cr zaRf)+Dg$crSq>%5-Nv6gD&aA;#tf(Z;cddDTNUKOvtkqcI9;ubhq-&AMUURE*w<#p zQ$(*hmrpRzqc+>jg5`wpeDvY{?bPADKX(!naglEyY0>_5FGnq;GGmEivwUsX>%x#E zL+r=Oqi)w}MI4BnEK9sXBSNTGX4~PTkVJi_E(4G1jWvDal-pWP{*d? z*&OU}y2+5b-3N%g!(J=ki2g8swM05vb)U!IoG#NzlQ9Hi6wHIFP zmIK~2b{Ouovignm>A~4*|F?S@mwxqCTjvH{zmXJv;ZiT5C~hJu|6YVf)>y z#-%W*p&^&tWYH6UDyR}~PTw-u<@>#E+S^u^8P~~c?a2EgThqyM_NntXY4_bvqxQR~ zu7D2dyXwz52Ge_$8dn!JPf-f*`4@d??UZfG*{!2nB2IiGUggI{WtUF@e3P)iNqZGh zcXoOClB&NAeG=A{+H+SpY^yfgykQ_gRa8d-5J-2g#ftqkR%w`a`4)1t$SDrrC1mt? znEoZ<9--#tA{=>p+_m*BD^~XlEIe4@FRfLiWvZV+;56KbWI@Dlm~rBFOQ#ylei^#4 z==SSQoQPz+U&0?;lXmXZ6#c)NlGz7^0mps2X0AQa-9bQ}+A8poSFv6AzC`TxdQ!CT znwIp6NpIGA!agMKfxD3V zQKd3N_p38(qiHg9?d6ACH3TilyT3(l%>%SorfFMAUlFU7#G?wyl&uKyOV_wj6Z0Zh zz_sf;dj#q%O5c~X`VYfC|5gcZG770WLf54ZSlaKjTx5QljQxbpSxc@jiH|`<1Ic)N3*oMY+=e& zTeLC9A}8%`e-2dUHrVuvq7mLi5WLyMkRP?Wnz?Xe#UhJvaDpiK%UYyZdP(*WXmBsCxTO%EN)EZ4c$s!?-d__o)GC zmq*ISj^PaEJeaK~yIhzk zg?W!5GGpS88jL*pfH#7pLo<4)O;u`d&i zDW>Db7lcTBT2zKeeXdb+mb+i!Tg1m5#V)-XJh3#KNdzQ0iDFp%>}%!j)2|GyLgMFgIj`j&UDOOI_=sV%kaf*mmkUwqLes@jzi0pe9ude-KHfjbmPROYM**-zh>YKJRTSEhmW<*AuhM@B?<1=-B` z5hln>Cdhfd$rW9;o_mGzZOWx)Xw`~qXWGv8&Gyf4IeVatks@WLgmT~Q5++Oe>gAuO zAhbl_-KIuhvf>dqC6~3#UEg=4MyWW~^|xyV87`*f2>p_)8(eCL)@xUdPIVh+Bn2f+ z`}g*4udjG}6@{5jS5oC@QuFp6z z-hH8Ss7uKR;y(2iLAXCA3?L^~JAl#(lW2#(%+zO7SeOCSd;nNnZ$V%!nB|kF)8me{R`jr^F+8{P^KJggkNeIlGxZ zKKjV}6tFRG*Qp|D?}=+>yhVJO(4NYpWaeUxDT0D2`_`UAnj9Eoc($TXd57X1V<8YSG}$qGB#=z?Ry1vSw-j06OQti>&TA#Z#n;-ul^g%2VwB z07spHmQ~Bf*R7Ee5mh{1GMavJF`yl%_&028W>)tJHj{MJw?V{wO5Y8UKb?^kYbQJ& zc48yG;*TP3+^Y4}MHBU^B2LD9AMN|q`$JRLI`)f@#%SBw7KkSiiH6?AbxB4(8 zqfhCUu3TuX+&Q{<^P3X=R?4nTE-;z^;VCD?PUZWOv}0^+J50ExY|xg@DD;s*aK80^ z%iE@%v$msautkO=)8f|lNBFCk)ioGzHpZr$GhVjoTQA(HW5t;xc{dJ2zumM&`x2cs zr0?^Ucs5C;{{ZR9KXnmZ{3hSrLc*AQTyziJK=*$Lo14kX`ht9NO;_`#>f=_;n~K!a zS)}?&_V#(JBQne+PHa^UPbMB2GnnueHdy_SkQW5I$ zXs;LE8l@iBCHY&E-}r$VioELv0xOH-D)#j3RCQ9W$EL{JSdFe4i^#ZjX4o`_fQ0HZ zbD1gOPQ`8zr`|NId$o@VJNCDb^hi?PY5{ymD!-fg(B9JZ6O-OH!Ol|ms1X_#g?Ve~ znjV2|Y`tk0CO8dGR+dKT#x!xKo-B(cBId}ttMhK(U8lMJcrBLlJOQ_XBwwtIx-`-nHNoV5ZUP#a{1Ns zPqX7*On1JjM(r&{+K!!%zk_9uko!bM_llXW+M~7p&uNAw?F@FrzU=@tZp)$>f6J)| z+J5Bec_=0JYFb;szYyw3!Bd-bykfi|AHw=C1^wAF3ME& zFvTS)cZhL^y_3UFj6Z;_)g$F?3s6jLa(T{gdZV4cUnLJbq7nY`roIo;Fo(ge(E2&Yw1m@q`SO?4@xVg{OWbKxRZ$+T)?3o(6nXN-((_r#+&8K5rX>D*LDCu9mP$^pDjsi4Q4lDR+%$+eQLYu4G}FM-ys?w4Kp3<>#o;EvyvKaIPFsD zW__l@zco57mm|bu6C&T@`nq~)EQvXZW%j86VlC2D zz8{VpKLSBe!v4LrU``DS}-t2@v;2fUH-C z?i!9v*=NL%`$nd!zMbjW`YIb}7TYG9JDcs@?q_y&!Fbtug+V;iF2nUAZuQsXb+wB= z_}Moela^dWJ;lH2{#Exq7fhv%qQO=;d&`croj~_q+2UI-P&FOBq+3noJa8MuQ5!za zD}Pm0Q|U^ave}04^R{{ImmG0qxSd7IlMa1m`F?YL^>?5d4SBQ6hL|-U8;rg&2#GpP zJAC_>&Wc0oP3h{Qb|U`(TI{-WR9lOOK0706x<=_H$a+iX2@XT^X)oE5l~2d-C$#de z@4%{cj?Cxjk|$b4kr1~o;ZD%`Xe6TZFHbdW+0j|l8fAyE5t8$R5#b)|nR9afu>9($ zcD=J^ty8V%ZDh(+*GaIB$R|D{zm!s`X!_=FOIg=(J?(}o){LnkG);44RGhw9SmY!` zIdxT=v+bTuy<0B#QeplLxJ(5FF;jR)dV=rsW7)rdn}6nzzG*2F;@z2bvL>Rr5-^i( z^9d7uUs=K$>~9+->8XbcV%ZhVia{{o`ZDFq)mC(m2>o56Br^gmD3f8f(kKxLW#;s* zRad`jm14QQ8c5>vWTTeu!rkg$N>!g_(bj!Yv)jGu;|;koZL!?N!poPGh<{qSIc(sa zvAA56-dUtr@{BXmo)@>N(Gl~q!6jAh5eMzctd=|?R%8!jIqyh2ADz606 zLnl%;#fX|bmvA9@*)$k%-Ttm<+a2JX5hs|qNXAY>{hN*n4nFQ*R-!GN6Ni3_jps-I z099RavdUX?8^}wmm60xz>FKLAnHZK_zh;*S83>6MLtjUO#uCSE({j*gP{4YNbP=_apj-{;t*1M3yDidb$;Q+oV949`vnWB-+HKZ_ z0KO4p&FS)kS~}%0MpUeF%Zx+ST>iNdsqBWWM3b=Fh$HQja_#QP#PyiTy-0T!+B7m!e4tH)%^R@=&nL5 zA6<5kFCOBo({y3UhTBZrge{adcFQYaHu1Z2^BhH0Mfj0@A19u(#Lo#mOLDj*yMv1= zGA`u;X*VaxySpln+NRYXOQeJXN3JiP^+R3Y(0V&PEAh}Sp4tWEZain_zbF^6)=arg z7vit6L8n0s9T?Zxam_T0-9ul-#{G8F+JZZrNY@DjIQN8yo0^O7)(DsPAvrBc0xP^u z8ZRTAo0Lp@!{%Mf=|rI2krYWfQaVON;rPnP0bcI(PG9ruqlfrya~>m)v{i1#)oqoP ze-USyq{kzf960e^1{-X7OA&1`9ICR?x^bch-JKX-1$9t+RXp8``ghfbDtvKe(H93H zZz_SaMj^CC$y84cZ1$~> zq=gJYdpGj0r_P>g8Wx-v7cv%y-5=toI-^=m)Qlu)HyIMurh%rwja8O}M-kjUWZXTK zlhlqS^QmQHY={>w-vR93T3rmWDcHgi?{NPB80YHp)*`q*vuDYX#XlkF78wxIVqM(tMnozP(K0)_Iwguot;&;V;{PqO5wMkUI$f09-C}} z{{XwBCTn?IJY(487rnz83rmq=xVT?xqi$D6F+V6@YQHVRlJP#x(^2*R09k5j0$eM5 zjSsx@V^z)K?WyDula9w=OQ@WjBll^0wbQJKhJY_OfyqQ`qqL2rVA~pw?h>wlmAhL< z=Sp$Pf{^p2*L;VM{5WnAezil$nBfxQKdns2o_DC55%-F^Xkf)kZ6PpA7xK4x&i?>$ z4YB*hNH4)+T~{ambfItVjW9^;mt5vXmy9-ch%hin*xwZ6)g}}8$4qo!A{{Skhb@tcp_W^aZCYb{6wh3Cc7V|vR zt0|RvaT3DYii+$+?aAzin0l#-i8@9-IQ3HUBowI*yk(IWQB%7VSn{q7HzCd&S#@P! zP>XD5&E(xCN->RT7e|T1c#GX^ij$XbIHlzob-(Wib!z<&rF3Ps%FxZYWcdzUHr(Q4 zRnFM0_C6qtA$fuk%PD-MUsOh&TOnCWJx9sW*3KKZW`!}sZ8GH+$9)JSOul!sQ@VY| z;^UjGwu9&1TdIxH^&(|0*|st(8ZsWB(eNU)BX5;4+EZHvnfPRIGrxs_9? ztSwO+Oy-Gh_C%NWRD1hOg2@*Rhh9&!T!QHA)!cR8X)3QG=YML~Qbkr6g%i}Rw5$78 zqonN){{SR+9nH{BWxik8RHOS(e7Eo*G@kJ))NGBS-J}m>hS>L88oeHfzkyNpWp3i; zk$IUgvVL3cufFLo?WpJHM~+IayxU*i8Yl0nN?xX&H8FZ??kBRQxYHGv+P&hd(dg^l zQj==TU2p44L~ZJ`K>jBAmZK!?F5K9!2p@as-Blq>A|9IOj^;huyt-^X#Rk+SV#$@Y zLL&Sc5)Z~lzL+Mp)Hg!LBxesV>{ufHbQHzK=7~rP8hF;JximgbTdn-k-By;?6)QE= zt=tpJwd+X@XwC)C> z0VFoBk`DC&%b8Q?s}W;HQa1LfxHJW}9-5g9T7xT&ER&B(anCB@dtT_(-xB5H8W!x~ z31mV;kry^2jZm5wX_`x1-{jsP&temlTU@2TYDRqm&-hq%ZaaaHk@l`dRePAqI@mD z_(*ZXVbCWR1VbfMCFbl+-j2lHovT?NZ@NY{nb&h&hU{mf-5qjr1T)?&5g93l??-Q= z<4T^7v@LlZzay<8&TG5lFxxM?%|lw+R-(RXxYDmg7iJ%8)lqwX6i>eMuRzcu9hL7r(y5g||M>@YBs*6kzLpGkKE4$KqSYL5GR4 z#Uyxx6v;R5#~i%L*;J0qH8bqrZd!8g>ZH_g$W_E$>*?(JH?Ba^=Q?;cZ@f!O3%CM*jdl zNQ+LL*TpbYWn19P66)eJR+n12R^1T=bm)6n8SsMARtr9uiFdbMSf29}5vRx`w(6j` z83GO-aZC&CRb`~Ms1d9hj>5|h8y7f=(h_d$HBm%W__4`9H5cun+dcbCGQ?X%Yh14x z55gm&2QKCAr_oj3h>c*kT^LuIn)HAyhC-eq(TeXC6dym${;gKpBW*S-#3*OPeN2%XQzp8J6VJIk z+t!QiTJp~8JWm?Q^6DIG;aZKY#8OE~KBKa>kz`fGo@w!FL%91@F&0CPk06m_%viQX zKm&YkL&oBCk#f<-kJN!_w@ZD==>SF6!5uc##Z_|bUEd{C(zL&{ZYcw8J-V5ok0M?? zq8?M_`NL8S+y@mRDtT(uvZOXlhNQ8~eZd|cP2=;6?&DKQDsfDd*$s|MLOC~)2$N-w zJj4CvcJg=?>Bd79b!GZUR%KsCtXon?kmp-4I_QpPRi~&V@*UUw`;5H zunpyk2A+EPWYbj#neZJIV^rsCUNHhWhkK5OdZ@TPcL z@%13AzflKQEfTrfRKw1}h|9IL}XY z%FSCq*U4~@@0V2(^CB9q`fGm|O(SBlmTO#>YT{y@F|nOzGgDgU%(j#!W6N#TSDh48 zK_o9W{{V`uS&u2sl0sah;eF~ccIf2GlLWk?CCW50IvEqsI3wQ3YURb`x2(DC@+$EQ zwtUIBE*oj(9b5rEI*;(c*3Id*VQHohCT$V7yN`I+&-!TVV$HPgQG z3d+2N+{9+EvaX-fQ>3dNW|fp)_cs3k=Ry|iP@YRns&2C40$&ph*_@JB3B)I ziY4!HKU8a}cGz$tlaptDFWKfH=Jsi_>bVz3I4u!AJw8BjXnwU1T`q$wP(iY`BtlOM zk2qe)^B2{nhf0b*TbLg4q~Zt}4m1VZDJaM+ea$gL2OCsfNqX}OkRrnU++PnV% z*=l2CmgY8H_wuiC;4Js0qzf%Vqx9&ObVvIjf>0eA2S{qgqL)vi6i) zP+zD&Tj6C%E%4#%X&Je0E?qUQJ`ap2(zP8u zp-ykkv=^(dLv)%;naGfSbtB%i7^NyY2#I5lyHB9Qkm+%a&YqH`RUaWG9uVdBsv<;= zV()WN4f0bw7P)bM@{}a+F;fXcuNq%fa!-*heqz4LSLQsl4tZQ6CGDp5Z0EU)k z3+|R6$rsh2@I-wz>{f^^x(@ktEMwc=FPt=IWAJy?cVWq6iv!-~q!g<=wbljf){e1w zBua6?DYCwB<5otT@K@8Z1WEQq^2@zaDuIichay3FCfp+VwL56p<^J%yquZTv7t_0P zR`s!Lb52@WQIBh9AughBi{WX@Y%viC;@8a@Y(T>nF9uy(bi<0+#vgjDYX^awgd$ff zkz@Y=3~O376}G&Xx0lYP$}3`75YdPf9gD#aNXN&@ zcynLVRwpkLE*X7V=ouWb2Fnumsj&hoCYiNb;<(IM5u-4uqCnFKD7(|*sw(euji4+p z68UG9R2XFow9D1LxE&#!DBQ$_ow; zD7WonNh56n!`T+iwI!NI^{J_lU(|Dl@ejGfb-c3W5^!6V67qxdXva!lq0ECBuzDZD4YDPSmjqoGxMUl&-sC6_} z<>~!EPfl=xHNsIulNT3=u@dR#?{jl<*Egr$Tr2anvyM38;ZP={(boYHdrxVZ4K>8_ zo-HV4x0WN)L(Pv>0A8-QsScz|yVKikqU?(HNP@o_FPgU2Ot;V#v0PYA`^B9v#UeJU z@t>f#)s3=!mg=29dyREvfKcCqt*P!M1^Os^V(`}Wv`Fkas7;7CNGb6O)~JaHW40n5 z%}Tqu5m2u+wws$K`wI7GjEgS=S5Z3)Bu-KvkNG7q+CG^*Nv5OEcGyXAhy%oqL%7jB zDckDbR$2(PD0prTBwVCC*tk5@t<(w8YEcn$a=1n8_kPV)#}axO5!*8@67g*Em?8fF z+1&mbMU#sZxh|5U5cH22hwoO?gE1~JnOSk!c!!j@jR|voKDuc#!)Ba_kJ%wAB6i@~ z2kfcnt0L|r$(4+MaCylLqi)|GoY0nBHyrO##+A^23e1F$z6jfhLM(%ujZ}Ro9h=kR zPc>W@jgV;zyIY3D#8(x1nKJ$AwI86ZZg{&<#m5DY7{?K0cn=pamy|4V$GllmvaEbb zLi7|+Yy%k00;(RkKy`a{TscLZHt3N7LN)2HEsh{lXbTni@8DTajGxN zyLwcU9tPV=B_p?LEfseRkp)ubfQ9ld?%M3U54POM`5h^2yAEIaoyDd7^_~1XWUbVX z&0F-6{i!b&+)LSfCNx~ffxy)EfiXl-NfDK`?D)u1jjcOu(^mw2S-k1V2PIJ*RFc>! z_U}#m%d@ts8;64y*BJ!~+aaFfns%rwZv1|QRiQcl4%&boFNAhAs9PcIB}-DT?(@{G zqf1T6GcWy#>m%+;8otr0Ew9_g99VGOjl>tn#U&O(L(|QXdsIUoRg71K!W)wUBs{)6 zLeXRH``@itMpnjQ+8oAClhoE?66PU?A-sK(<@>bRn_E_=1f%NhQ)3?XF39EkU0=gh zUrp2&mBN4bLDJ&iE(g&VS7DgTvIKGQhl$*MoT@VEaw5#Kp$!o7>Af3iwq0&#D&)fm zkG(1Mam6L=R8mhq?fbX!mli@7U7mn)%uRECyk>}e^%VNN~(*r zq9?kjt5)hbW1e#vAN$I!XKIUfYxqbwzRP{9T^^TVCO*YHmQ__GqjJ23?^iuTl_g}4 zaMX{s)w0ztS~ysXgcvu!=PzgJsrgziRT? zoQ%SQ0MQ7FZbBsU(@Ru5Z>>vV>Tie3_D84d zP*mM*3eR|y_10@`u*)b7gKNxWy`-t~d8i*y?e)gcnA5NlAP?o9PH)Xf1@3o7V8mBv z2*q!9=G!Gi-|-?PPeLXeCB}h&vE)evQS4i;^^AS>5ud2Glsw2b1L|%I5CH2=> z7qec#y_x{^aY}akHTlhrI3BNMjeD!vuVO^B0RA=M=&kVE2FK?OaU$g+So4?Jroh{T zJfoE;5Z)ljTO!JUI%(1p<*EB@rpU23I7RJLWKBDCnxnyl`?-59lF=QvWE#%wgt6=r z)dhDhV~_2iQhFth*FCXvRmrjUNU0mM)|*^@8xjTb_iA3(;K{D0IM_biOKKYaS@b0z zv%Zh)UI(pY+THJF_pP|0xjxeW04e~@ljS)%;>2>x*{61`6L-8SBxp`Ji{9PKTVG9h zKWU2)61MS0C`=eUE6}6b-kVpGNPC_pTlaz7CmW)Iwix-0piWrGJ2H|{bxhU$i347Co3x8 zwmikuLqG^K7xH~nxHZd`IeWg%bU2Z7cdNt1;~0^353@`MJS=^h>90v3jv`##)#NU2 zY)FQ?66VVuUiLHr=0u3H5ge&x66F%ho2x2pxe0Rg<6ca7WrugXQ~ zUyW08F6H$ih3&I!I>T8%3RThty2%NrNg`QXca9MOI~4>G5i;G_rMXppL_|OU4GpJS z)jELALwQ7x&Ta18i@(gDPUZa7YF~8B4M@kdC^vjQFWVS~O71lB$TuBVg-RPUq7kVNpFVpGt zr6Vf#r~bah@phuhz*wWo8F) zCq1fn2IX{zibt6tfllZ+L;ETWR1H6~?UY{h)~km;Y@1Wj8bis}K~d$%cVp)BcKZEv zL$@huZ`q~gU`tezirOg$X5m)_7kALLmg4OzRNHqDT-`Aq>zv=VpU&GY;_qgYZJsT> z!_Dx^mY4>5BUMY&H@FvGJE|N+M@3D0Lw{{vTVT6ghRq890H?0+w%bjKON`O3Dh0b0 zQBRk0{U04>fVm58n`qxIJihH{p9hJ9v|BscR%30t?X{N+5*}wF7vluIta_?CsOl2R z)c1Ra%wcLhJvq3NodR)N+=irpN+c?Zp0r<~DzLQpPU@D`5= z*=D@GXse`$1BW1jD~@+axrDxJfecUrjZzm;=8y}qbM zZ-tWLROOvOSoKq=hKFauGg&r+o!lND_|ckQ;jW(Wi@nHg7F;I1KMj$N>BLE6@OyYbE>xD&E>KebS{-1USe(;34Q9W zW?i!CDjd2@<;N9+6UMkEue$N6ZkLy!Q zr18kPhM)>YMPAVnc&X-W#H@`|d6B%Fxk^%1Oaxve{q-JB~FildC^Cch@89CeRW6d`rd=9T#V|XBw{2z>+0;SF+WmitNy*!k+ke1 zXHKysQqa$ygt(W^dSn{B2$VW*<!H zO#lc3GVV_i5fiQ#ox)x8W2UxqCgkF(HcE-t=oI>DUPS#nq_GllyR4t=VV9C0oL%Z4 zy=@}ncxc!>V#3{s#gEDepCtejOTB4*5Gcz+H|vvslWeh!CGQt{f^{apylTP_n;{Y~ z7VwJNV9I?P=xn-|9IKiJENzqPNjf&~CQM9K$6DOEuWs~R=}zU?Z#PL#O>w~^kYw>L z1KOmjtNUmM(8Kd4vea}kaaVUDZ{1qL-K1g+2*_4>nIl4r+McRv3S}RH9QG@fwU<^k zp;Kf{==6i`5rcW+#m%}*ih zA`3!7MBg!fGKNqxklNnGOA^MFYR4(pF$P>BFLt&hO^jKhz2#YN4`!zs*|m-+i;Y7P zAudrGF<>!_Eem#-NFx=&$g$=cA+~Hv{{TM{@lpXI<;u2nJvRZjMWuu(zqfLxRUz{?lL|xj@tiWg4B6p-poyTaJZ}ELx z*<;N{AXB!l2a5z&Ra}g37D!#Ix{PhxD$~$d6=u-FDKF&*FWW|q4-~wwFE`E_P#O^- z$a2ftro=HLU}>0enZP49%K5QmP-*bR5hk*1esObCmFeXY$hv>?S4cu4@p(){d1=7p zn(~~Y*%0)&BTA!9mLzTwWL(w479u5%SyHUEj(kDK=F$P=X|WM5{{XUG>Qp#*yO4=%%U*Px7X-2DDoAl6As$?Zn7x|stAsga z^{4?vjwI$Gmp8Oo6)J=zE%RlMRUiT`k>gz}rNdFj%9SWEs1`(ckJ_bBxJ{99JgD@K zc90B9BFh|xy1+@iaxAju=BC62c}Rq@#EQI_k5H%=puH8l8QAl_NDBf_)YOo5aw>$~Lr)Bz?-h=~}Nx+6;@ z&l7Kh;^O_m%t%MsA9%Z?`o-PGnc3l; z=}NPErIsU*^HVsMc#!5ulGI^tL#T*u52k<}Cl4Hp-7H3w0UjnsSdj9LHOV?TVk6zh z9D1w7?pU{vcbc39&|4S2X!B)J_q#2xQQDlhU1CPMM0cr~0E9dKnq4s#@g7m?A{zG) zM6%|`=FkNduY3^n#rKUbPdy^VAX-#Nt)d$)5+`S?9sa%RShh+z5>w1cXxe?UR?JJz zB#XII^Vie|Wn#`lm$Q{)Zwf7v9iLIj)pmvm?K4j)AlNUEchXDo7h>=As`}G>ETCui zgylKTRkc^>?c2@I2PZq5LE^e{gwGB0Y3w1hh596^GY?;_vyUwMo>C zuHfya4o3-!dXW1ve!_`E4I@(k1 zNJ!HjE->_w7vni``(E`8Yt16%mRU}Y>yquyuJm}(HT18?>8V#C1;=amw%k%O z5-*c>@1mt*?fUr0^ETNHRrvE)u%;Bb;dn>RYw@visaW0;?c?QPl~A3^_e}t4IGz&r zvM(zg!77)D;UlNWM~LHt^=m0|(V&@E=;&l-h%`ARxXIBgF zRrRxLTQ?0OWg_AU3uNIq*>`fshC|-SX}~(t))=vFmm8GHP7IkBViFSNXk&==5q$2P zd!tmgh<8oN+7OG-dbqib~ zc_KpJ*%mtDi}SsIO$ra9yQMw&ADrJEe&R(+Fu^=nh?mWyJuS0|SCS!tBqJv~(-r~{fS zXHaY{i8TybFKC91yg+^0m4j^}ZU=;Mj{%m89GlCl5v9^U2z-+x zk#zTIR!QuQ!NshH9$aZr5h2LlA;+g0jA15+$>L69?u~Yl#E3^8C+hOoTrVleIFC=g zT_58zCkvL84w1x&5t)o(yCl7-8|hTNMteG z%PwK|O1cM2F6nU#y-EV)MALaTO#Glm`b^fIXEI%k2YCSsv_wYBH?0L zb9aWA4=9dDlbK>%xerLvs#!P9^>P|XIC*gZOWGe5oqIf!|NFuaxnT(^A#pveBnoe0P zhQZQ4jNFH!f+I`im&A$iEvxb3ICe7`V;L_2lu19cW2j_YnuSzG%_0mM1|)A46%#g3 zC%8G&%ow&#VW@fp+C}D19>A>-C;)M$w<;N2G-)M9hkFEF1cxha`kHb*H>IzR6nN%S ztRNIpHT_o6Ji_^p-x^T*41I2TyK+37vMJs609Be%?V#}7ElDx`5^2B^f%I^RtyrK# zSfJ�kPk$MiD{?t%d63tAG7%$VlHb;Z>Jhk_e8d7Lj4hX)HDT8=P5{c1Fm8YuwCf z>jZJY|7ehX_Vx<{TTc??E->TFA~iZi)l^2b3CPPOM)3gxF-F-8p;a=LM^$Oo>$0Xs z+fIP9nBKv`<X|k$js5$r(P?uB7YM#$62)_S{@uH_?~k`|X5L769Dh1r@73Ns!$3z8X5cgDZ*OCn7B0e* zoqm+!_QUs6El;g{Z<Rm+~655Asa9hfb(3 zc%*%-A{M?@Gh9_=<{J6teaK{kRG(F|UtuIwdE#z)n^tUQ064P4Pjcf5vDxI@tp&|4 z2`Pp|18Czcl3#P|!0BQHlTkvdJcs#yVEO*>3c=kk&CCbVJ$E)MM%yee1ZiW9i%~f) zQs)6?Rbf=$aW?C#Jg^29?Kq1oq%V}x!a{P%SPJz+RSMHltdG>h>$e<*ygs#8;bEH2 z&sTXXY<##B;C5u?_Omt~!QC1_#546``QSPm2ieOGccAD+mN$}`L9i7&QDs1B zy2Mmff;4cr2lkmh>MkxT+5$z(jDL0-ig%&^JsQ0U?h^N?1iZm!H~yFST4CTxQQb)d z+S2NZc~|)EqoZdpy+F(bCrNWYvk$8dNhvn;669??!H9r-Ah(U~V;sLL zc~llk1D|o$Jg>YOiAdJ)SuZ;Ag8uwPTq0#DhY_17XA6-oz1gfav`A1vFrbHxm}Z23l){ zki`B)zQ%Bw`Ed&@9;C^ER&U0rVf5KCN_Y*2>*PLdvF?#N>gME^Gg713btiRak^l?84SqLAo~|YR>EOiUmY08fVEh205xyuHggt0 z+ESv-5?0y>Vju+9qAYpSS4p`;L>+;yj{2BU*3xyOAMWRX-qeOdqAl9oPB&6>kUG87 z#U~c{@NHr>OAULu+}KGT64P3;?i+>-h)SM@y{DVbZt!!oD{}B37*`uEA2dc7G^Oh{ zxG9S=oiXDMs_7@yNPG@VPy>@h6KomKO@N+hS5IPVGK7e%u*ZrT&J~(*{Z&m~rWmR; zb`<;l;cSpB+$BUf%W^l68Ii$bL6lQK_o z;6NaB38BSY00BE{w|%wYVJxHSEvFw$82%S)`QzzwuwAl-?)?7%eLAxj?Bq4#|w#&FCwAc5C&4y;_PKZY2nVO>_ z%}PIrcAj+!+ZcTW2SWYsOU%J~XfU5>%T@=8QPiFLDi?{18bZ6hoEzflSs~=7%L5qE zG{1xdN=qk3-pV6y&IP4%py!9rd}T9ke&$e7bmOnTshcEUjlE6%u9xJiw`?D8e1=Xk znKpfw7=C}UOSp%J{%|Qx9*RBA6sl*&o#UVd!&Lv?8y-JSD)x_kZu7oubI5bKFo^f9 zGIPBl@Iip77c=mQ)&1{PMpx$3_kVzacWwM?8W_om%bxgir{Zb8zmm|&-TQ7i$U^0k{9#7)@;<&Tt&Vli0Ecb5InO3SM1Z8XG+hVs zb^q`6r33JH30)gS=4P~>pormGi;}mn?U>NpMHGBlji{FbX^}c zfR#>bOjqA>&xw=sTps;8E_&?Nd5hsMlyDC`Sd5(gyF{`Vp8fCS)s~?~$#GCk2o0H8 zEGECHDjIp(_U3Ge3}?0`)L~a6qV0`ZZNYZWSB;!Do^X_K_G=vhTOBd04Lud__%Q3x zxpyXviD!kfCbIA@TxThkuyOz<56jdjjr6eF+cbFS8(x;9g6G4LORjnPCO|FZQ3S#e zZ|Q#X(}~+d1-r8cp&I|}=n$`M%;-?XsicimD~ilW>P736Gn(HfbED6^@AB1Efna3w8U5=N#q_`BAxo)UaTM_&2jO4%=`2NKsfh?ZRFO43N6!mCRBiUB;7^_#z$zO{ zE_U%Ge8GwU&ZPKme(njCGnBajv_U%|6cTk!0AGBcDF!BT%A(n|)l>e(=`iSm6k7A+ zI*FmdGPA6APM1MbC`731;&}{1s&JMOMp~>U2_u{Q#yt!P4iES5F$ozW=QnL^`$l+4 zq7x}`6gnk6nn}bA@F201&HmYNt{!WvOl$%R4`2$`5_{l3Z3Y6|6xvZ|j80u^9}|RqNt9f%p7AxP`-1cS0{YI#qU%~bYTVP zViidG>r`{qABitQG3GP*Y5y0x1zGm9dC0$?wQ7RDJw1J9HBv2s6~ky{~mPo>CMO9hGx?O8=HAiWM1IUo=lB=w~5ka z{j1mAqrT<5g`YOL`XCa_9NZYX{05(+v3JKyZbrpvF&syjh8#1G2hYMWpMPc2!1g~v zJ?{T(y{bN=_j0W(=~m#{ug+fQ-~5_jU7x_tjL^)Dh*1(oQ7Payx{!a1tkf}k@z)VW zs)wTaOpBCsy@r%z6&ow>F*&KcKL*;Ad~9@XlE(hm+CI8HFuL#R-Z}Cs9$6|f3*dZy z%Yua4_m-~-oAF!;#|2aoLby@QQMHQ&4p3YvLS}UzI=NUh5cU)&6wZyUimk}JO7Ly0 zL)I+mVa?gf`QDH1fof4ca`(dvp)6q6>pJ^2-V~1kCAj8g_g@^p0aqHr{221g*!8gh zg49@xqW4OTOn%9Y!MF27@Nf=tI;eW34g-SGsUDc&9=*s4y|MWfIWzc9yYZf7GeHMA zL1{=k-j%0D>_{%QXv35rF4jtHhy9vbG+|ohf7t6`Z3hU=VMSIxOlp$6=Ghd*Zu{J< zjpNr_bxRs3Q))$R3f4TIhL1}0jdeQ!lWuTT?HUhh&wu8BQ>7NKxx75}^{G}0;;gda zvR^>P8aDxJ(X#FXQ0g@@wV2w98G4V2w@^dB`ZulWSc*`$UDgWLK}n*Y**D`&)la`- zs4eEM=S?iaKhQ#mB3$XX>98o#_?s=i_rge2zHwlyRW|uvj?PDCso|-fgtn%lpn^lg z0YTc%Ofr8-D&#_DWHy@ZH|o(nZlLrIZx;rhOaf#)7`aicO8b+x*7`!CXkpP62$&EpZWyR8LjPJXiK!64XLQw~DASDT|n zx!|TPhEL5`kTI8`WJe{mh4;Pblgu|d;icn+9{AbTANBKm48{hVCc-(n-0edvQ-Y;R%kdKQrBdo+4PA9>vwqXT#E0v4`>&luAMIp6P!ZEV zKh?5<8?L*VR(b7-wCd}zsjG(#p{Wd+qm{ia{b+hsFD+%plM%yIE9*Ek(97CtluR0s zmHf5%mgS4PKO%6Q5~>uQNqu;j=E?LxDE6jDm$--kfMohFn@o_P94B7sUGEz3Ghp4j z8?B=wf2J9|${0Btb;w9`8sXarz1Z`|9)b9a$RL6%;3jl>Y>tu!cfV0QfVB0vvum_O z;z8b{5eUT}>yAx7B#ZB=&yMKHu%}D+(!*baJ4nDSZ21?-p!O@m1!Rq1wh*4C2Y9J| z+fF1#D+x2S1Q(!0BoW9jFl)RPSep2Pui$g1PbbV2jhSDAL?x^FX9L0eNQ=nAl8>Gg zp{Bx3=CimHlsU@7mwMq~vhTFn-_@_X-F!H$T0Gre!b{Pb199wP!A6%Ka23XD?dgb_TKt*<$N&2rySGb@PFW>#7DsbuX!ocW@NBR~G(C)l zP_a;Jjv|!+`;{@RXlyf>ZiMFI4CLQ<{MXN-* zuFA~L-5w2#E)(M(4Gf!g-t^Kg0}{0~)nW@*9OURZN<7Nu@vy`mcwnvO@Axz-xm`T>rfs;h*OqLKOTKep zk~wTttMcTVtCn1E#v^Y0EBvc)Y=d{xq)KIo$EB$AZ=oT_>;L#oUU~HR$7gSQOV^G@ zO!R5he6|3|Rk*k^{VMorIPMy5HLag@`9Phm* z7rM(2*HufQY&=&3pVz_#3nHbl9bGX}?`J4h=X_h!caZ351V`K84T@VWTG39AO9Lae zN#u-E#$d;N+K|InM^czAw=W*dS!Y4I9TPPgoV46FW#p57%`gymaWSz?2zW9gQODQp zPPwk^pU)V(=zG4pe~fpMHX8Hx!f^l{lcmBWE4T^@@r~-`B5fbz^rmILhX@1;e!spL zXJA9;K!{wTfcK0o_8kriFM5u@*W)q^s)7erpR?qhBZJF7Mfefb%zi`Tea_{W=%dDYsUP#T>C!% z9vchwP>Y#z9w-~J?`jvw@<4)xf@3LR*XB-o=^ZgB$w3W9&KLq}bq~_gvtWW>Y}^Zp z6JBnLQ_#E{wLC?j8c6`T)29wFN8C!VxUa!*%oiDMiXrhdR1T7s*kd+TzUI@4KS)9C zIByKPgQ5NALai0g^bZB%7d#p}>_v7HY!o@xX_6B7v%T1vKMmsp%O9kCLhC0dr`x#o z7fVzzb!pI`YFEHLp)Lw}#JXh~Vv(1tFP(tAF_%d7?$)-f-Te=3oX7$#c4uZu-$lym z*KX!4{STl!YP_eqT{#)v)el<;u3$Uwr+@7+{8x9XsMM3)E-TVdyzHn~!{?**Ldbqd zFGC~lN5GovY6ueq<3_bD5_F2G9|DqL3Y8ANAPXN#hL_Tm;Os4R`da~USr4os)#Lg; z4(3`H!Xu@KuWbBQ4X@84IJ2!h$wDtl@?*THK|Oic7-;3wfxe6v z>pzb%z{6dV5AWoCcx4Qp>8Yh+ZW3}gM`HHpq7p#k>kElF4v&Znes9t8! ze<+ymZXI%=f7e}#g{-2m{ha1VX5DiGPMp@AUMJrdT0*@JXh7SO8*y^k2QaK)2&L_D zM$Pe($0u@3zr8dZ;Fs@@o4J4lqAthg3A7_mPV0GL2Iqg3iKV|iJwTW$?H;U+bqAl zY?Ygw6ws;{$%G`(rbSJ-i46g@+^A^tk`D2lK<{=g8f`IgHXJH428R#>3z3t%U~w*CY+Za^B1UJPOE?H1L89?| z-r^`FD!ciShzGAQTum(kiCW!@pbm@OqPyn3TF#IO%S0P_f}i|tsdWFL8&>n2zNK3Y zASGDoK^WA&g9zPJowMFi|Jwj?x<3maF=J?G=sJa=k?FG>aqB2*P-jfL|ET!O1!N_80M`g7y>QS9L$^*F)bJubmrCq`?Wqb0Lfg4D`DJ#E% zSxC4AQ;E&sJ-fek^S?&?%H@4y@oL8y``@z0an(%8@ht4HZfv5v!<39@R8}6BYG9MYLOf#GPBalSuPORLTm~;2Lw557} zssGeIiX`P_V9J))ljBb7NC#!BkSXFNFuRRto1seD^KGcbA?O3ww9z&^VP8P5_GC3+ zZX)GwOQO9GX=Y~sa|ybK`WD_CbBq-jnfKe^Z+cP1_>nBUjr;e5J zq}+&nmcU?coq$i}NMU(=r((GDvB$)IUXnr4*9+QdXTKdmTXhricA;3?iU_0`p(3uO z2;L<|lObX0NReMilG89Z%#j)X4>U8UN%33VtQz|NoMl z(e%xRE$95}%51gNlAl^~NXyHERjh-bEICS2s*1YaswK2N4ss37^6m~IF=FIx%yO|y zz~$en=5sHP(0nA~!F~=eJV==a?y$AAS;Lt>ethokyDhXSS|n<}9ig{M6|G(TaNu!$ z*_w{{^W|l*`ssF@52>qG4c(kj$&mLhB(|&Qv73EEcil&-um1%ZIWqI&POxcJxnkkq zy59c*a$4s}vevaQ*p@}!qc-lMe7M8qF67Za_XZaW4%`>!1>yy;MNVlB9ho1`kpm%v zpa|UJWrkejF|~GR){>>ooRWu|TVT$ofP<&FipE7q@zo9rn!=8{=vXhIzvT@urNU*7Jnb!LC4+eNnWBEiti+ zM5PRHAE_n_N|*r9u)+FSk8R;1DSf@#HJQFKj~S*64F*cf?3^o=883NDMLeRarChr!J}p2eYkQY;@)9ct5)A)>Lc3Ye&8b{!9J&23>{FnYeSG*Jqy z$vek==4{Bzk*Uz+9Nf>D3&C6EVZRr?m<6~_Gq-o^@7l##X-X!Uwez>i)87_<%s_om z%5gIo+o?A`lG%`fv=WYL?p8hj+BZCGqGPfGhKEwr=<^2S(vef81X#DhG|oau-o)*$ zpBZC%6*&)j`=DJWA59R&U&OhHQp)KOtyf06>Ow&vXwY>*xu%b~ff-s5{_0K|BaJJQ zw6Xl)G>{hP#pKeMQzf1lm>M|>JW*P8_wdV;P`j|GenXw%ws)dmwYljScOU!rAG3#| zA+lH8aKFj|g{kiolA)#g?Y2$364qZn`6ToK4?^E+(1r=IbIKDsgaFHjM?5WrQBq(T zN;IWqXxZSls%*g8OFdabqa{R@E$HvokhXNmKl%ZPO{V+DDfhke4TCf>-|0iBw^WLXxe%N{jxm;ORzSC*61kr?Rdsd z=^i{A0#&q;smu*&%TF#7U9)bZwW0UX2!R{iJ(_8#G?I2kGU1+ObNA8Acxe0RQ%nxBu$h3d>R#&arbrvzV5baS#M$_A839s~Py=`q}LR Z|89{~%P+df1@nuW_=^g?#tQ!%|389wKb8Oh literal 0 HcmV?d00001 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_deepseekr1.jinja b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_deepseekr1.jinja new file mode 100644 index 00000000..bb86cbd9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_deepseekr1.jinja @@ -0,0 +1,92 @@ +{% if not add_generation_prompt is defined %} + {% set add_generation_prompt = false %} +{% endif %} +{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='', is_first_sp=true, is_last_user=false) %} +{%- for message in messages %} + {%- if message['role'] == 'system' %} + {%- if ns.is_first_sp %} + {% set ns.system_prompt = ns.system_prompt + message['content'] %} + {% set ns.is_first_sp = false %} + {%- else %} + {% set ns.system_prompt = ns.system_prompt + '\n\n' + message['content'] %} + {%- endif %} + {%- endif %} +{%- endfor %} + +{# --- Append tool descriptions if tools are defined --- #} +{% if tools is defined and tools is not none %} + {% set tool_ns = namespace(text='You are a helpful assistant with tool calling capabilities. ' + 'When a tool call is needed, you MUST use the following format to issue the call:\n' + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>FUNCTION_NAME\n' + '```json\n{"param1": "value1", "param2": "value2"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>\n\n' + 'Make sure the JSON is valid.' + '## Tools\n\n### Function\n\nYou have the following functions available:\n\n') %} + {% for tool in tools %} + {% set tool_ns.text = tool_ns.text + '- `' + tool['name'] + '`:\n```json\n' + (tool | tojson) + '\n```\n' %} + {% endfor %} + {% set ns.system_prompt = ns.system_prompt + '\n\n' + tool_ns.text %} +{% endif %} + +{{ bos_token }} +{{ ns.system_prompt }} +{%- for message in messages %} + {% set content = message['content'] %} + {%- if message['role'] == 'user' %} + {%- set ns.is_tool = false -%} + {%- set ns.is_first = false -%} + {%- set ns.is_last_user = true -%} + {{'<｜User｜>' + content + '<｜Assistant｜>'}} + {%- endif %} + {%- if message['role'] == 'assistant' %} + {% if '' in content %} + {% set content = content.split('')[-1] %} + {% endif %} + {% endif %} + {%- if message['role'] == 'assistant' and message['tool_calls'] is defined and message['tool_calls'] is not none %} + {%- set ns.is_last_user = false -%} + {%- if ns.is_tool %} + {{'<｜tool▁outputs▁end｜>'}} + {%- endif %} + {%- set ns.is_first = false %} + {%- set ns.is_tool = false -%} + {%- set ns.is_output_first = true %} + {%- for tool in message['tool_calls'] %} + {%- if not ns.is_first %} + {%- if content is none %} + {{'<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}} + {%- else %} + {{content + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}} + {%- endif %} + {%- set ns.is_first = true -%} + {%- else %} + {{'\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}} + {%- endif %} + {%- endfor %} + {{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}} + {%- endif %} + {%- if message['role'] == 'assistant' and (message['tool_calls'] is not defined or message['tool_calls'] is none)%} + {%- set ns.is_last_user = false -%} + {%- if ns.is_tool %} + {{'<｜tool▁outputs▁end｜>' + content + '<｜end▁of▁sentence｜>'}} + {%- set ns.is_tool = false -%} + {%- else %} + {{content + '<｜end▁of▁sentence｜>'}} + {%- endif %} + {%- endif %} + {%- if message['role'] == 'tool' %} + {%- set ns.is_last_user = false -%} + {%- set ns.is_tool = true -%} + {%- if ns.is_output_first %} + {{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + content + '<｜tool▁output▁end｜>'}} + {%- set ns.is_output_first = false %} + {%- else %} + {{'\n<｜tool▁output▁begin｜>' + content + '<｜tool▁output▁end｜>'}} + {%- endif %} + {%- endif %} +{%- endfor -%} +{% if ns.is_tool %} + {{'<｜tool▁outputs▁end｜>'}} +{% endif %} +{% if add_generation_prompt and not ns.is_last_user and not ns.is_tool %} + {{'<｜Assistant｜>'}} +{% endif %} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_deepseekv3.jinja b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_deepseekv3.jinja new file mode 100644 index 00000000..fdde62ee --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_deepseekv3.jinja @@ -0,0 +1,92 @@ +{% if not add_generation_prompt is defined %} + {% set add_generation_prompt = false %} +{% endif %} + +{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='', is_first_sp=true, is_last_user=false) %} +{%- for message in messages %} + {%- if message['role'] == 'system' %} + {%- if ns.is_first_sp %} + {% set ns.system_prompt = ns.system_prompt + message['content'] %} + {% set ns.is_first_sp = false %} + {%- else %} + {% set ns.system_prompt = ns.system_prompt + '\n\n' + message['content'] %} + {%- endif %} + {%- endif %} +{%- endfor -%} + +{# --- Append tool descriptions if tools are defined --- #} +{% if tools is defined and tools is not none %} + {% set tool_ns = namespace(text='You are a helpful assistant with tool calling capabilities. ' + 'When a tool call is needed, you MUST use the following format to issue the call:\n' + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>FUNCTION_NAME\n' + '```json\n{"param1": "value1", "param2": "value2"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>\n\n' + 'Make sure the JSON is valid.' + '## Tools\n\n### Function\n\nYou have the following functions available:\n\n') %} + {% for tool in tools %} + {% set tool_ns.text = tool_ns.text + '\n```json\n' + (tool | tojson) + '\n```\n' %} + {% endfor %} + {% set ns.system_prompt = ns.system_prompt + '\n\n' + tool_ns.text %} +{% endif %} + +{{- bos_token }} +{{- ns.system_prompt }} + +{%- for message in messages %} + {%- if message['role'] == 'user' %} + {%- set ns.is_tool = false -%} + {%- set ns.is_first = false -%} + {%- set ns.is_last_user = true -%} + {{'<｜User｜>' + message['content'] + '<｜Assistant｜>'}} + {%- endif %} + {%- if message['role'] == 'assistant' and message['tool_calls'] is defined and message['tool_calls'] is not none %} + {%- set ns.is_last_user = false -%} + {%- if ns.is_tool %} + {{- '<｜tool▁outputs▁end｜>'}} + {%- endif %} + {%- set ns.is_first = false %} + {%- set ns.is_tool = false -%} + {%- set ns.is_output_first = true %} + {%- for tool in message['tool_calls'] %} + {%- set formatted_args = tool['function']['arguments'] if tool['function']['arguments'] is string else tool['function']['arguments']|tojson %} + {%- if not ns.is_first %} + {%- if message['content'] is none %} + {{- '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + formatted_args + '\n' + '```' + '<｜tool▁call▁end｜>'}} + {%- else %} + {{- message['content'] + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + formatted_args + '\n' + '```' + '<｜tool▁call▁end｜>'}} + {%- endif %} + {%- set ns.is_first = true -%} + {%- else %} + {{- '\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + formatted_args + '\n' + '```' + '<｜tool▁call▁end｜>'}} + {%- endif %} + {%- endfor %} + {{- '<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}} + {%- endif %} + {%- if message['role'] == 'assistant' and (message['tool_calls'] is not defined or message['tool_calls'] is none)%} + {%- set ns.is_last_user = false -%} + {%- if ns.is_tool %} + {{- '<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}} + {%- set ns.is_tool = false -%} + {%- else %} + {% set content = message['content'] %} + {{- content + '<｜end▁of▁sentence｜>'}} + {%- endif %} + {%- endif %} + {%- if message['role'] == 'tool' %} + {%- set ns.is_last_user = false -%} + {%- set ns.is_tool = true -%} + {%- if ns.is_output_first %} + {{- 'Use the results below to formulate an answer to the user question unless additional information is needed.' }} + {{- '<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}} + {%- set ns.is_output_first = false %} + {%- else %} + {{- '\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}} + {%- endif %} + {%- endif %} +{%- endfor -%} + +{% if ns.is_tool %} + {{- '<｜tool▁outputs▁end｜>'}} +{% endif %} +{% if add_generation_prompt and not ns.is_last_user and not ns.is_tool %} + {{- '<｜Assistant｜>'}} +{% endif %} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_deepseekv31.jinja b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_deepseekv31.jinja new file mode 100644 index 00000000..a97f011f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_deepseekv31.jinja @@ -0,0 +1,92 @@ +{% if not add_generation_prompt is defined %} + {% set add_generation_prompt = false %} +{% endif %} +{% if not thinking is defined %} + {% set thinking = false %} +{% endif %} +{% set ns = namespace(is_first=false, is_tool=false, system_prompt='', is_first_sp=true, is_last_user=false) %} +{%- for message in messages %} + {%- if message['role'] == 'system' %} + {%- if ns.is_first_sp %} + {% set ns.system_prompt = ns.system_prompt + message['content'] %} + {% set ns.is_first_sp = false %} + {%- else %} + {% set ns.system_prompt = ns.system_prompt + '\n\n' + message['content'] %} + {%- endif %} + {%- endif %} +{%- endfor %} + +{% if tools is defined and tools is not none %} + {% set tool_ns = namespace(text='## Tools\nYou have access to the following tools:\n') %} + {% for tool in tools %} + {% set tool_ns.text = tool_ns.text + '\n### ' + tool.function.name + '\nDescription: ' + tool.function.description + '\n\nParameters: ' + (tool.function.parameters | tojson) + '\n' %} + {% endfor %} + {% set tool_ns.text = tool_ns.text + "\nIMPORTANT: ALWAYS adhere to this exact format for tool use:\n<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>tool_call_name<｜tool▁sep｜>tool_call_arguments<｜tool▁call▁end｜>{{additional_tool_calls}}<｜tool▁calls▁end｜>\n\nWhere:\n\n- `tool_call_name` must be an exact match to one of the available tools\n- `tool_call_arguments` must be valid JSON that strictly follows the tool's Parameters Schema\n- For multiple tool calls, chain them directly without separators or spaces\n" %} + {% set ns.system_prompt = ns.system_prompt + '\n\n' + tool_ns.text %} +{% endif %} + +{{ bos_token }}{{ ns.system_prompt }} +{%- for message in messages %} + {%- if message['role'] == 'user' %} + {%- set ns.is_tool = false -%} + {%- set ns.is_first = false -%} + {%- set ns.is_last_user = true -%} + {{'<｜User｜>' + message['content']}} + {%- endif %} + {%- if message['role'] == 'assistant' and message['tool_calls'] is defined and message['tool_calls'] is not none %} + {%- if ns.is_last_user %} + {{'<｜Assistant｜>'}} + {%- endif %} + {%- set ns.is_last_user = false -%} + {%- set ns.is_first = false %} + {%- set ns.is_tool = false -%} + {%- for tool in message['tool_calls'] %} + {%- set formatted_args = tool['function']['arguments'] if tool['function']['arguments'] is string else tool['function']['arguments']|tojson %} + {%- if not ns.is_first %} + {%- if message['content'] is none %} + {{'<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>'+ tool['function']['name'] + '<｜tool▁sep｜>' + formatted_args + '<｜tool▁call▁end｜>'}} + {%- else %} + {{message['content'] + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['function']['name'] + '<｜tool▁sep｜>' + formatted_args + '<｜tool▁call▁end｜>'}} + {%- endif %} + {%- set ns.is_first = true -%} + {%- else %} + {{'<｜tool▁call▁begin｜>'+ tool['function']['name'] + '<｜tool▁sep｜>' + formatted_args + '<｜tool▁call▁end｜>'}} + {%- endif %} + {%- endfor %} + {{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}} + {%- endif %} + {%- if message['role'] == 'assistant' and (message['tool_calls'] is not defined or message['tool_calls'] is none) %} + {%- if ns.is_last_user %} + {{'<｜Assistant｜>'}} + {%- if message['prefix'] is defined and message['prefix'] and thinking %} + {{''}} + {%- else %} + {{''}} + {%- endif %} + {%- endif %} + {%- set ns.is_last_user = false -%} + {%- if ns.is_tool %} + {{message['content'] + '<｜end▁of▁sentence｜>'}} + {%- set ns.is_tool = false -%} + {%- else %} + {%- set content = message['content'] -%} + {%- if '' in content %} + {%- set content = content.split('', 1)[1] -%} + {%- endif %} + {{content + '<｜end▁of▁sentence｜>'}} + {%- endif %} + {%- endif %} + {%- if message['role'] == 'tool' %} + {%- set ns.is_last_user = false -%} + {%- set ns.is_tool = true -%} + {{'<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}} + {%- endif %} +{%- endfor -%} +{%- if add_generation_prompt and ns.is_last_user and not ns.is_tool %} + {{'<｜Assistant｜>'}} + {%- if not thinking %} + {{''}} + {%- else %} + {{''}} + {%- endif %} +{% endif %} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_deepseekv32.jinja b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_deepseekv32.jinja new file mode 100644 index 00000000..b6d239dc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_deepseekv32.jinja @@ -0,0 +1,101 @@ +{% if not add_generation_prompt is defined %} + {% set add_generation_prompt = false %} +{% endif %} +{% if not thinking is defined %} + {% set thinking = false %} +{% endif %} +{% set ns = namespace(is_first=false, is_tool=false, system_prompt='', is_first_sp=true, is_last_user=false, is_only_sys=false, is_prefix=false) %} +{%- for message in messages %} + {%- if message['role'] == 'system' %} + {%- if ns.is_first_sp %} + {% set ns.system_prompt = ns.system_prompt + message['content'] %} + {% set ns.is_first_sp = false %} + {%- else %} + {% set ns.system_prompt = ns.system_prompt + '\n\n' + message['content'] %} + {%- endif %} + {% set ns.is_only_sys = true %} + {%- endif %} +{%- endfor %} + +{% if tools is defined and tools is not none %} + {% set tool_ns = namespace(text='## Tools\nYou have access to the following tools:\n') %} + {% for tool in tools %} + {% set tool_ns.text = tool_ns.text + '\n### ' + tool.function.name + '\nDescription: ' + tool.function.description + '\n\nParameters: ' + (tool.function.parameters | tojson) + '\n' %} + {% endfor %} + {% set tool_ns.text = tool_ns.text + "\nIMPORTANT: ALWAYS adhere to this exact format for tool use:\n<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>tool_call_name<｜tool▁sep｜>tool_call_arguments<｜tool▁call▁end｜>{{additional_tool_calls}}<｜tool▁calls▁end｜>\n\nWhere:\n\n- `tool_call_name` must be an exact match to one of the available tools\n- `tool_call_arguments` must be valid JSON that strictly follows the tool's Parameters Schema\n- For multiple tool calls, chain them directly without separators or spaces\n" %} + {% set ns.system_prompt = ns.system_prompt + '\n\n' + tool_ns.text %} +{% endif %} + +{{ bos_token }}{{ ns.system_prompt }} +{%- for message in messages %} + {%- if message['role'] == 'user' %} + {%- set ns.is_tool = false -%} + {%- set ns.is_first = false -%} + {%- set ns.is_last_user = true -%} + {{'<｜User｜>' + message['content']}} + {%- endif %} + {%- if message['role'] == 'assistant' and message['tool_calls'] is defined and message['tool_calls'] is not none %} + {%- if ns.is_last_user or ns.is_only_sys %} + {{'<｜Assistant｜>'}} + {%- endif %} + {%- set ns.is_last_user = false -%} + {%- set ns.is_first = false %} + {%- set ns.is_tool = false -%} + {%- for tool in message['tool_calls'] %} + {%- set formatted_args = tool['function']['arguments'] if tool['function']['arguments'] is string else tool['function']['arguments']|tojson %} + {%- if not ns.is_first %} + {%- if message['content'] is none %} + {{'<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>'+ tool['function']['name'] + '<｜tool▁sep｜>' + formatted_args + '<｜tool▁call▁end｜>'}} + {%- else %} + {{message['content'] + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['function']['name'] + '<｜tool▁sep｜>' + formatted_args + '<｜tool▁call▁end｜>'}} + {%- endif %} + {%- set ns.is_first = true -%} + {%- else %} + {{'<｜tool▁call▁begin｜>'+ tool['function']['name'] + '<｜tool▁sep｜>' + formatted_args + '<｜tool▁call▁end｜>'}} + {%- endif %} + {%- endfor %} + {{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}} + {%- endif %} + {%- if message['role'] == 'assistant' and (message['tool_calls'] is not defined or message['tool_calls'] is none) %} + {%- if ns.is_last_user %} + {{'<｜Assistant｜>'}} + {%- if message['prefix'] is defined and message['prefix'] and thinking %} + {{''}} + {%- else %} + {{''}} + {%- endif %} + {%- endif %} + {%- if message['prefix'] is defined and message['prefix'] %} + {%- set ns.is_prefix = true -%} + {%- endif %} + {%- set ns.is_last_user = false -%} + {%- if ns.is_tool %} + {{message['content'] + '<｜end▁of▁sentence｜>'}} + {%- set ns.is_tool = false -%} + {%- else %} + {%- set content = message['content'] -%} + {%- if '' in content %} + {%- set content = content.split('', 1)[1] -%} + {%- endif %} + {{content + '<｜end▁of▁sentence｜>'}} + {%- endif %} + {%- endif %} + {%- if message['role'] == 'tool' %} + {%- set ns.is_last_user = false -%} + {%- set ns.is_tool = true -%} + {{'<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}} + {%- endif %} + {%- if message['role'] != 'system' %} + {% set ns.is_only_sys = false %} + {%- endif %} +{%- endfor -%} +{% if add_generation_prompt and not ns.is_tool%} + {% if ns.is_last_user or ns.is_only_sys or not ns.is_prefix %} + {{'<｜Assistant｜>'}} + {%- if not thinking %} + {{''}} + {%- else %} + {{''}} + {%- endif %} + {% endif %} +{% endif %} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_llama3.1_json.jinja b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_llama3.1_json.jinja new file mode 100644 index 00000000..f609c388 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_llama3.1_json.jinja @@ -0,0 +1,121 @@ +{# Copied from https://github.com/vllm-project/vllm/blob/main/examples/tool_chat_template_llama3.1_json.jinja to enable better model response. #} +{{- bos_token }} +{%- if custom_tools is defined %} + {%- set tools = custom_tools %} +{%- endif %} +{%- if not tools_in_user_message is defined %} + {#- Llama 3.1 doesn't pass all tests if the tools are in the system prompt #} + {%- set tools_in_user_message = true %} +{%- endif %} +{%- if not date_string is defined %} + {%- if strftime_now is defined %} + {%- set date_string = strftime_now("%d %b %Y") %} + {%- else %} + {%- set date_string = "26 Jul 2024" %} + {%- endif %} +{%- endif %} +{%- if not tools is defined %} + {%- set tools = none %} +{%- endif %} + +{#- This block extracts the system message, so we can slot it into the right place. #} +{%- if messages[0]['role'] == 'system' %} + {%- if messages[0]['content'] is string %} + {%- set system_message = messages[0]['content']|trim %} + {%- else %} + {%- set system_message = messages[0]['content'][0]['text']|trim %} + {%- endif %} + {%- set messages = messages[1:] %} +{%- else %} + {%- if tools is not none %} + {%- set system_message = "You are a helpful assistant with tool calling capabilities. Only reply with a tool call if the function exists in the library provided by the user. If it doesn't exist, just reply directly in natural language. When you receive a tool call response, use the output to format an answer to the original user question." %} + {%- else %} + {%- set system_message = "" %} + {%- endif %} +{%- endif %} + +{#- System message #} +{{- "<|start_header_id|>system<|end_header_id|>\n\n" }} +{%- if tools is not none %} + {{- "Environment: ipython\n" }} +{%- endif %} +{{- "Cutting Knowledge Date: December 2023\n" }} +{{- "Today Date: " + date_string + "\n\n" }} +{%- if tools is not none and not tools_in_user_message %} + {{- "You have access to the following functions. To call a function, please respond with JSON for a function call. " }} + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. ' }} + {{- "Do not use variables.\n\n" }} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} +{%- endif %} +{{- system_message }} +{{- "<|eot_id|>" }} + +{#- Custom tools are passed in a user message with some extra guidance #} +{%- if tools_in_user_message and not tools is none %} + {#- Extract the first user message so we can plug it in here #} + {%- if messages | length != 0 %} + {%- if messages[0]['content'] is string %} + {%- set first_user_message = messages[0]['content']|trim %} + {%- else %} + {%- set first_user_message = messages[0]['content'] | selectattr('type', 'equalto', 'text') | map(attribute='text') | map('trim') | join('\n') %} + {%- endif %} + {%- set messages = messages[1:] %} + {%- else %} + {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }} + {%- endif %} + {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}} + {{- "Given the following functions, please respond with a JSON for a function call " }} + {{- "with its proper arguments that best answers the given prompt.\n\n" }} + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. ' }} + {{- "Do not use variables.\n\n" }} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} + {{- first_user_message + "<|eot_id|>"}} +{%- endif %} + +{%- for message in messages %} + {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %} + {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' }} + {%- if message['content'] is string %} + {{- message['content'] | trim}} + {%- else %} + {%- for content in message['content'] %} + {%- if content['type'] == 'text' %} + {{- content['text'] | trim }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|eot_id|>' }} + {%- elif 'tool_calls' in message %} + {%- if not message.tool_calls|length == 1 %} + {{- raise_exception("This model only supports single tool-calls at once!") }} + {%- endif %} + {%- set tool_call = message.tool_calls[0].function %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} + {{- '{"name": "' + tool_call.name + '", ' }} + {{- '"parameters": ' }} + {{- tool_call.arguments | tojson }} + {{- "}" }} + {{- "<|eot_id|>" }} + {%- elif message.role == "tool" or message.role == "ipython" %} + {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }} + {%- if message.content is string %} + {{- { "output": message.content } | tojson }} + {%- else %} + {%- for content in message['content'] %} + {%- if content['type'] == 'text' %} + {{- { "output": content['text'] } | tojson }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- "<|eot_id|>" }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }} +{%- endif %} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_llama4_pythonic.jinja b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_llama4_pythonic.jinja new file mode 100644 index 00000000..74b315c3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/tool_chat_template_llama4_pythonic.jinja @@ -0,0 +1,112 @@ +{# Copied from https://github.com/wukaixingxp/vllm/blob/8a32e2a6e452a03c0e8222e3876ad6086cbf581f/examples/tool_chat_template_llama4_pythonic.jinja to enable better model response. #} +{{- bos_token }} +{%- if custom_tools is defined and custom_tools %} + {%- set tools = custom_tools %} +{%- endif %} +{%- if tools is defined and tools %} + {%- set tool_definition = tool_definition ~ (tools | tojson(indent=4)) %} +{%- else %} + {%- set tools = none %} +{%- endif %} + + +{#- This block extracts the system message, so we can slot it into the right place. #} +{%- if messages[0]['role'] == 'system' %} + {%- set user_provided_system_message = true %} + {%- if messages[0]['content'] is string %} + {%- set system_message = messages[0]['content']|trim %} + {%- else %} + {%- set system_message = messages[0]['content'][0]['text']|trim %} + {%- endif %} + {%- set messages = messages[1:] %} +{%- else %} + {%- if tools is not none %} + {#- Since not system_message was provided by user, if tool is provided, system_message is now default tool system message #} + {#- This system message is from llama website:https://www.llama.com/docs/model-cards-and-prompt-formats/llama4/ #} + {%- set system_message = "You are a helpful assistant and an expert in function composition. You can answer general questions using your internal knowledge OR invoke functions when necessary. Follow these strict guidelines:\n\n1. FUNCTION CALLS:\n- ONLY use functions that are EXPLICITLY listed in the function list below\n- If NO functions are listed (empty function list []), respond ONLY with internal knowledge or \"I don't have access to [Unavailable service] information\"\n- If a function is not in the list, respond ONLY with internal knowledge or \"I don't have access to [Unavailable service] information\"\n- If ALL required parameters are present AND the query EXACTLY matches a listed function's purpose: output ONLY the function call(s)\n- Use exact format: [func_name1(param1=value1, param2=value2), func_name2(...)]\nExamples:\nCORRECT: [get_weather(location=\"Vancouver\"), calculate_route(start=\"Boston\", end=\"New York\")] <- Only if get_weather and calculate_route are in function list\nINCORRECT: get_weather(location=\"New York\")\nINCORRECT: Let me check the weather: [get_weather(location=\"New York\")]\nINCORRECT: [get_events(location=\"Singapore\")] <- If function not in list\n\n2. RESPONSE RULES:\n- For pure function requests matching a listed function: ONLY output the function call(s)\n- For knowledge questions: ONLY output text\n- For missing parameters: ONLY request the specific missing parameters\n- For unavailable services (not in function list): output ONLY with internal knowledge or \"I don't have access to [Unavailable service] information\". Do NOT execute a function call.\n- If the query asks for information beyond what a listed function provides: output ONLY with internal knowledge about your limitations\n- NEVER combine text and function calls in the same response\n- NEVER suggest alternative functions when the requested service is unavailable\n- NEVER create or invent new functions not listed below\n\n3. STRICT BOUNDARIES:\n- ONLY use functions from the list below - no exceptions\n- NEVER use a function as an alternative to unavailable information\n- NEVER call functions not present in the function list\n- NEVER add explanatory text to function calls\n- NEVER respond with empty brackets\n- Use proper Python/JSON syntax for function calls\n- Check the function list carefully before responding\n\n4. TOOL RESPONSE HANDLING:\n- When receiving tool responses: provide concise, natural language responses\n- Don't repeat tool response verbatim\n- Don't add supplementary information\n\nHere is a list of functions in JSON format that you can invoke:\n" %} + {%- else %} + {%- set system_message = "" %} + {%- endif %} +{%- endif %} +{#- Now writing the system message: use the user provided system message if user_provided_system_message, else default tool system message if tools presented #} +{%- if system_message %} + {#- always use user provided system message to override default tool system message #} + {{- "<|header_start|>system<|header_end|>\n\n" }} + {{- system_message }} + {%- if user_provided_system_message and tools %} + {{- "\nHere is a list of functions in JSON format that you can invoke. Use exact format: [func_name1(param1=value1, param2=value2), func_name2(...)]\n" }} + {{- tool_definition -}} + {%- elif tool_definition %} + {{- tool_definition -}} + {%- endif %} + {{- "<|eot|>" }} +{%- endif %} + +{#- Now deal with all other messages #} +{%- for message in messages %} + {#- Base case: messages that are not from tool role and has empty tool_call list #} + {%- if not (message.role == 'ipython' or message.role == 'tool' or ('tool_calls' in message and message.tool_calls|length != 0 )) %} + {{- '<|header_start|>' + message['role'] + '<|header_end|>\n\n' }} + {%- if message['content'] is string %} + {{- message['content'] }} + {%- else %} + {%- for content in message['content'] %} + {%- if content['type'] == 'image' %} + {{- '<|image|>' }} + {%- elif content['type'] == 'text' %} + {{- content['text'] | trim }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- "<|eot|>" }} + {#- Tool case: messages has non-empty tool_call list, must from assistant #} + {%- elif 'tool_calls' in message %} + {#- assume tool_calls are always coming from assistant #} + {%- if message.role == 'assistant' %} + {{- '<|header_start|>assistant<|header_end|>\n\n' -}} + {%- if message['content'] is string %} + {{- message['content'] }} + {%- else %} + {%- for content in message['content'] %} + {%- if content['type'] == 'image' %} + {{- '<|image|>' }} + {%- elif content['type'] == 'text' %} + {{- content['text'] }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- "[" }} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- tool_call.name + '(' -}} + {%- for param in tool_call.arguments %} + {{- param + '="' -}} + {{- "%s" | format(tool_call.arguments[param]) -}} + {{- '"' -}} + {% if not loop.last %}, {% endif %} + {%- endfor %} + {{- ')' -}} + {% if not loop.last %}, {% endif %} + {%- endfor %} + {{- "]<|eot|>" }} +{%- endif %} +{#- Tool_response case: messages are from tool_response #} + {%- elif message.role == "tool" or message.role == "ipython" %} + {{- "<|header_start|>ipython<|header_end|>\n\n" }} + {%- if message.content is string %} + {{- message.content | tojson }} + {%- else %} + {%- for content in message['content'] %} + {%- if content['type'] == 'text' %} + {{- content['text'] | tojson }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- "<|eot|>" }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|header_start|>assistant<|header_end|>\n\n' }} +{%- endif %} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/vision_template_sarashina_vl.jinja b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/vision_template_sarashina_vl.jinja new file mode 100644 index 00000000..caff3441 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/chat_template/vision_template_sarashina_vl.jinja @@ -0,0 +1,9 @@ +{# + In sglang, the default chat templates often assume message['content'] is a plain string. + That works fine for simple text conversations, but it ignores multimodal inputs (e.g. image_url, tool_call). + To align with the original model behavior and support richer content, + we iterate over message['content'] as a list of typed items and extract their values directly. + This way, both text and non-text inputs are preserved in the prompt. + Original template: https://huggingface.co/sbintuitions/sarashina2-vision-8b?chat_template=default +#} +{{ bos_token + '<|prefix|><|file|><|suffix|>A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human\'s questions.\n\n' }}{% for message in messages %}{% if message['role'] == 'user' %}{{ '### Human: ' }}{%- if message['content'] is string %}{{ message['content'] }}{%- else %}{% for item in message['content'] %}{% if item['type'] == 'text' %}{{ item['text'] }}{% endif %}{% endfor %}{% endif %}{{ '\n' }}{% elif message['role'] == 'assistant' %}{{ '### Assistant: ' }}{%- if message['content'] is string %}{{ message['content'] }}{%- else %}{% for item in message['content'] %}{% if item['type'] == 'text' %}{{ item['text'] }}{% endif %}{% endfor %}{% endif %}{{ '\n' }}{% endif %}{% endfor %}{% if messages[-1]['role'] == 'user' %}{{ '### Assistant:' }}{% endif %} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/checkpoint_engine/update.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/checkpoint_engine/update.py new file mode 100644 index 00000000..86b588cc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/checkpoint_engine/update.py @@ -0,0 +1,241 @@ +""" +Usage: +1) Launch the server with wait-for-initial-weights option in one terminal: + python -m sglang.launch_server --model-path /workspace/Qwen/Qwen3-4B/ --tensor-parallel-size 2 --port 19730 --load-format dummy --checkpoint-engine-wait-weights-before-ready --mem-fraction-static 0.7 + +2) Torchrun this script in another terminal: + torchrun --nproc-per-node 2 update.py --update-method broadcast --checkpoint-path /workspace/Qwen/Qwen3-4B/ --inference-parallel-size 2 +""" + +import argparse +import json +import os +import pickle +import time +from collections import defaultdict +from collections.abc import Callable +from contextlib import contextmanager +from typing import Literal + +import httpx +import torch +import torch.distributed as dist +from checkpoint_engine.ps import ParameterServer +from loguru import logger +from safetensors import safe_open + + +@contextmanager +def timer(msg: str): + start = time.perf_counter() + yield + end = time.perf_counter() + logger.info(f"{msg} duration: {end - start:.2f} seconds") + + +def check_sglang_ready( + endpoint: str, inference_parallel_size: int, uds: str | None = None +): + if rank != rank // inference_parallel_size * inference_parallel_size: + return + retry_num = 0 + transport = None + if uds is not None: + transport = httpx.HTTPTransport(uds=uds) + with httpx.Client(transport=transport) as client: + while True: + try: + response = client.get(f"{endpoint}/ping", timeout=10) + response.raise_for_status() + break + except (httpx.ConnectError, httpx.HTTPStatusError) as e: + if retry_num % 10 == 0: + logger.warning( + f"fail to check sglang ready, retry {retry_num} times, error: {e}" + ) + retry_num += 1 + time.sleep(0.1) + + +def split_checkpoint_files( + checkpoint_path: str, rank: int, world_size: int +) -> list[str]: + checkpoint_files = [ + os.path.join(checkpoint_path, f) + for f in filter( + lambda x: x.endswith(".safetensors"), os.listdir(checkpoint_path) + ) + ] + files_per_rank = (len(checkpoint_files) + world_size - 1) // world_size + return checkpoint_files[rank * files_per_rank : (rank + 1) * files_per_rank] + + +def split_tensors( + checkpoint_path: str, rank: int, world_size: int +) -> dict[str, torch.Tensor]: + index_fn = os.path.join(checkpoint_path, "model.safetensors.index.json") + with open(index_fn) as f: + weight_map: dict[str, str] = json.load(f)["weight_map"] + weights_per_rank = (len(weight_map) + world_size - 1) // world_size + fn_tensors: dict[str, list[str]] = defaultdict(list) + weight_keys = list(weight_map.items()) + for name, file in weight_keys[ + rank * weights_per_rank : (rank + 1) * weights_per_rank + ]: + fn_tensors[file].append(name) + named_tensors = {} + for file, names in fn_tensors.items(): + with safe_open(os.path.join(checkpoint_path, file), framework="pt") as f: + for name in names: + named_tensors[name] = f.get_tensor(name) + return named_tensors + + +def req_inference( + endpoint: str, + inference_parallel_size: int, + timeout: float = 300.0, + uds: str | None = None, + weight_version: str | None = None, +) -> Callable[[list[tuple[str, str]]], None]: + rank = int(os.getenv("RANK", 0)) + src = rank // inference_parallel_size * inference_parallel_size + + def req_func(socket_paths: list[tuple[str, str]]): + if rank == src: + with httpx.Client(transport=httpx.HTTPTransport(uds=uds)) as client: + resp = client.post( + f"{endpoint}/update_weights_from_ipc", + json={ + "zmq_handles": dict( + socket_paths[src : src + inference_parallel_size] + ), + "flush_cache": True, + "weight_version": weight_version, + }, + timeout=timeout, + ) + resp.raise_for_status() + + return req_func + + +def update_weights( + ps: ParameterServer, + checkpoint_name: str, + checkpoint_files: list[str], + named_tensors: dict[str, torch.Tensor], + req_func: Callable[[list[tuple[str, str]]], None], + inference_parallel_size: int, + endpoint: str, + save_metas_file: str | None = None, + update_method: Literal["broadcast", "p2p", "all"] = "broadcast", + uds: str | None = None, +): + ps.register_checkpoint( + checkpoint_name, files=checkpoint_files, named_tensors=named_tensors + ) + ps.init_process_group() + check_sglang_ready(endpoint, inference_parallel_size, uds) + dist.barrier() + with timer("Gather metas"): + ps.gather_metas(checkpoint_name) + if save_metas_file and int(os.getenv("RANK")) == 0: + with open(save_metas_file, "wb") as f: + pickle.dump(ps.get_metas(), f) + + if update_method == "broadcast" or update_method == "all": + with timer("Update weights without setting ranks"): + ps.update(checkpoint_name, req_func) + + if update_method == "p2p" or update_method == "all": + if update_method: + # sleep 2s to wait destroy process group + time.sleep(2) + with timer("Update weights with setting ranks"): + ps.update( + checkpoint_name, req_func, ranks=list(range(inference_parallel_size)) + ) + + +def join( + ps: ParameterServer, + checkpoint_name: str, + load_metas_file: str, + req_func: Callable[[list[tuple[str, str]]], None], + inference_parallel_size: int, + endpoint: str, + uds: str | None = None, +): + assert load_metas_file, "load_metas_file is required" + with open(load_metas_file, "rb") as f: + metas = pickle.load(f) + ps.init_process_group() + check_sglang_ready(endpoint, inference_parallel_size, uds) + dist.barrier() + with timer("Gather metas before join"): + ps.gather_metas(checkpoint_name) + ps.load_metas(metas) + with timer( + f"Update weights with setting ranks as range(0, {inference_parallel_size}) by using p2p" + ): + ps.update(checkpoint_name, req_func, ranks=list(range(inference_parallel_size))) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Update weights example") + parser.add_argument("--checkpoint-path", type=str, default=None) + parser.add_argument("--save-metas-file", type=str, default=None) + parser.add_argument("--load-metas-file", type=str, default=None) + parser.add_argument("--sleep-time", type=int, default=0) + parser.add_argument("--endpoint", type=str, default="http://localhost:19730") + parser.add_argument("--inference-parallel-size", type=int, default=8) + parser.add_argument("--checkpoint-name", type=str, default="my-checkpoint-iter-0") + parser.add_argument("--update-method", type=str, default="broadcast") + parser.add_argument("--uds", type=str, default=None) + parser.add_argument("--weight-version", type=str, default=None) + args = parser.parse_args() + rank = int(os.getenv("RANK")) + world_size = int(os.getenv("WORLD_SIZE")) + req_func = req_inference( + args.endpoint, + args.inference_parallel_size, + uds=args.uds, + weight_version=args.weight_version, + ) + ps = ParameterServer(auto_pg=True) + ps._p2p_store = None + if args.load_metas_file: + join( + ps, + args.checkpoint_name, + args.load_metas_file, + req_func, + args.inference_parallel_size, + args.endpoint, + args.uds, + ) + else: + if os.path.exists( + os.path.join(args.checkpoint_path, "model.safetensors.index.json") + ): + named_tensors = split_tensors(args.checkpoint_path, rank, world_size) + checkpoint_files = [] + else: + checkpoint_files = split_checkpoint_files( + args.checkpoint_path, rank, world_size + ) + named_tensors = {} + update_weights( + ps, + args.checkpoint_name, + checkpoint_files, + named_tensors, + req_func, + args.inference_parallel_size, + args.endpoint, + args.save_metas_file, + args.update_method, + args.uds, + ) + time.sleep(args.sleep_time) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/anthropic_example_chat.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/anthropic_example_chat.py new file mode 100644 index 00000000..03d699be --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/anthropic_example_chat.py @@ -0,0 +1,73 @@ +""" +Usage: +export ANTHROPIC_API_KEY=sk-****** +python3 anthropic_example_chat.py +""" + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print() + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.Anthropic("claude-3-haiku-20240307")) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/anthropic_example_complete.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/anthropic_example_complete.py new file mode 100644 index 00000000..bce2a61e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/anthropic_example_complete.py @@ -0,0 +1,68 @@ +""" +Usage: +export ANTHROPIC_API_KEY=sk-****** +python3 anthropic_example_complete.py +""" + +import sglang as sgl + + +@sgl.function +def few_shot_qa(s, question): + s += """ +\n\nHuman: What is the capital of France? +\n\nAssistant: Paris +\n\nHuman: What is the capital of Germany? +\n\nAssistant: Berlin +\n\nHuman: What is the capital of Italy? +\n\nAssistant: Rome +""" + s += "\n\nHuman: " + question + "\n" + s += "\n\nAssistant:" + sgl.gen("answer", temperature=0) + + +def single(): + state = few_shot_qa.run(question="What is the capital of the United States?") + answer = state["answer"].strip().lower() + + assert "washington" in answer, f"answer: {state['answer']}" + + print(state.text()) + + +def stream(): + state = few_shot_qa.run( + question="What is the capital of the United States?", stream=True + ) + + for out in state.text_iter("answer"): + print(out, end="", flush=True) + print() + + +def batch(): + states = few_shot_qa.run_batch( + [ + {"question": "What is the capital of the United States?"}, + {"question": "What is the capital of China?"}, + ] + ) + + for s in states: + print(s["answer"]) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.Anthropic("claude-3-haiku-20240307")) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/azure_openai_example_chat.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/azure_openai_example_chat.py new file mode 100644 index 00000000..d53f935f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/azure_openai_example_chat.py @@ -0,0 +1,83 @@ +""" +Usage: +export AZURE_OPENAI_API_KEY=sk-****** +python3 openai_example_chat.py +""" + +import os + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print() + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + backend = sgl.OpenAI( + model_name="azure-gpt-4", + api_version="2023-07-01-preview", + azure_endpoint="https://oai-arena-sweden.openai.azure.com/", + api_key=os.environ["AZURE_OPENAI_API_KEY"], + is_azure=True, + ) + sgl.set_default_backend(backend) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/gemini_example_chat.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/gemini_example_chat.py new file mode 100644 index 00000000..0ae62310 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/gemini_example_chat.py @@ -0,0 +1,73 @@ +""" +Usage: +export GCP_PROJECT_ID=****** +python3 gemini_example_chat.py +""" + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print() + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.VertexAI("gemini-pro")) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/gemini_example_complete.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/gemini_example_complete.py new file mode 100644 index 00000000..5188bf41 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/gemini_example_complete.py @@ -0,0 +1,68 @@ +""" +Usage: +export GCP_PROJECT_ID=****** +python3 gemini_example_complete.py +""" + +import sglang as sgl + + +@sgl.function +def few_shot_qa(s, question): + s += """The following are questions with answers. +Q: What is the capital of France? +A: Paris +Q: What is the capital of Germany? +A: Berlin +Q: What is the capital of Italy? +A: Rome +""" + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n", temperature=0) + + +def single(): + state = few_shot_qa.run(question="What is the capital of the United States?") + answer = state["answer"].strip().lower() + + assert "washington" in answer, f"answer: {state['answer']}" + + print(state.text()) + + +def stream(): + state = few_shot_qa.run( + question="What is the capital of the United States?", stream=True + ) + + for out in state.text_iter("answer"): + print(out, end="", flush=True) + print() + + +def batch(): + states = few_shot_qa.run_batch( + [ + {"question": "What is the capital of the United States?"}, + {"question": "What is the capital of China?"}, + ] + ) + + for s in states: + print(s["answer"]) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.VertexAI("gemini-pro")) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/gemini_example_multimodal_chat.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/gemini_example_multimodal_chat.py new file mode 100644 index 00000000..afe0c723 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/gemini_example_multimodal_chat.py @@ -0,0 +1,30 @@ +""" +Usage: +export GCP_PROJECT_ID=****** +python3 gemini_example_multimodal_chat.py +""" + +import sglang as sgl + + +@sgl.function +def image_qa(s, image_file1, image_file2, question): + s += sgl.user(sgl.image(image_file1) + sgl.image(image_file2) + question) + s += sgl.assistant(sgl.gen("answer", max_tokens=256)) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.VertexAI("gemini-pro-vision")) + + state = image_qa.run( + image_file1="./images/cat.jpeg", + image_file2="./images/dog.jpeg", + question="Describe difference of the two images in one sentence.", + stream=True, + ) + + for out in state.text_iter("answer"): + print(out, end="", flush=True) + print() + + print(state["answer"]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/images/cat.jpeg b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/images/cat.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..a6a8e48c972b7c30fbdb5c2e95e806a9d6411540 GIT binary patch literal 345163 zcmcG#2|Se1+dq8I*vY$=YOx~}iJ?>+q97{Fs@Y-$XE zKmY)u{Q-O9AiU{?3vSl7R>r30Mh6XT0PSEr0RT{+`+l}2hLTQKoh4bu4zAd5bNBT3 z-T(eC4qCs|{{7wopjYL;IOqSZn(f9-e^1&83$%Y(KibV{gB75a1#Tad752*>2W6f8 za?pL>`?Pbc_sf3vwijsS8?>^*?SE7D_%~%wU%&nHpVQ9KfqDn*-)sNaAMv4^J`Q%Y zZ&upBDBus+0w#dr{`_gbX$LG10JK&CfFa?ZbsiZ2fQbZv!$be9lYI>U+>ZeO-Sf}7 zf6Bzy-Orug9V6`%eCrkfEENL)=T!jU{|*4`5PBQ!68K+yo>6 zQ@{uC1XO7yb>IYW642Nq02gTL{J%T)JNdusfp+yC4&Y@4XEOX@0PzA~UJwH>Xs-#7 z*f%Fe23q^S%oYHFnORsFnAjNEIcO)?@cJ(Kd}HH?fu*x&HK z|B%+H80~f-rhj`=V82sl77!~Nt@$=DO|^_ntSn57%*;%ztN=G{Pz;Q`QcQd&&)YEb z3%J80au=j!6Dw)AW06rmy(oCZoo+Eh6YvVg_ym@Y>p{chS`lvDt;JA4&1baX^YG9p0*2KB5<)jH<**kb zu=Q7VYoYys{?ooq;_6I2waM;E00uG!J`>^%n+(NhFM2lguU1EwxN0rvOL9W8D6JVA znS{+vB9l^CFUS&mzI`VMGl8;PD0?6*aYFE(o@-q>w43w@THdk;m^*+#L3*#&@Gt~< zX=@37bfz_81IRMmt*4G{X4M<)xYE1jc;rRKvP>Ib32{vx$_~xgj-J0@wMhXV5QP|L ziYeJ%Jo>jaBAB($Qq-#RlJ)RIjZHqw?MzPc)NE8VbNLHG8!wg-Ph8slXgQ32m{~ z4)LUD8;pyA#U6lR$tFM+Hi9R<(I**uA;*%i=ociy+xl&5Xgz_sl;BybL8%zp1N>pX zDvdu?j$Jc2S{F&)5ZM?aM4;D1BBGH!IkN|oG%j@!t9z$cjG1O$6RR}8Wa`+HqfG3B zeG$gwd>;cL$e6S};CqlL1=GP-_U?gj0#jGy`!MtpR*X6{QcG@-1_udaOP02l*AH1L z;5J6VSq4x-gx8!(^&TL;MtC0V>a?IH$!D+DGlg*L-x7;&a}!i97H63R941Zl^_Y6X zwm+jDkj}tYKulNA!x6|P5ishcx7jE77}9{TxMl*{>J^R%l#hs_pCd~^rR#v|d%}8z z@S_8bXG`N>vtfmpr?_6NL?0H$*oys2!DdgHN9tiH46%b0?P#gP-Nty20PpGJg87{#AfOncxZ`o| zr!UycG^^`Fsnu&MHOuorlX9sz(!L#fy?@6`G`OxzkP8QJ4G& z`6$^FbwUkKuVFg3wz-I!XVIZd5qRmkXRqd(y|l5aG&7g!ezq38qk|$uT7Ut&(7&Gy>ONoju<%1aug#Z88pnNN!J@Iy9hCHO%kcQ zwUMVo#YdKFA4#a3;iJ7%=Xmt<}*JZ3oiIq!8m}D#--oHX;5q zHFOVf0`Gcetx1qEx%$)8iU#^%h4+AS=jSbUuT+k1EkE&hlrZ3Z6HKfiC6ZRFqa@M< zu_b6(h@6`N{~NaiU>EmLpVynN3Kt5RitQ$D*Oxc%PN3=IH7*Sf8fubAMI_R ze0H`M=thvh$4!9O;t74h8*Vvo>F6RWvORykdhYBz%CT`(e{_NO%`2^(7kn)GB-Hx0 z_V0ZNkdm@|PsWz@%k&L9_Ya3w9^com+-~+~*rf69jjXyoK$I@T%w5s#_^|Tjjj)4d zriLKIhEH$BJZf&Hw2zP0yaAtqd2Ix)V#2m|j-MU&$llivKgtHBrb4cd+ah%I2m)t9 zB+}OO0tvqQ)A)Qk{&2e4{m~Y;z~G@oKUQ+|mC_%N-NVv)u1N}yk8IQ|2{(*TtLvJd zvK%OY^%wLrm3flb7Tbn}_N;uRW0AbWyh*y>an=b65<%dl+djBVZ!Aa_pS^|=$3Brc z06GDt@~~Zq#5HZ=U+y&7LfH;+o>Am$$v2^{WDLvVCUMi_%&&Ox640kDvbh-~;`C5c zmnu3gDL3KAT8&vI^g`RPdKW61O7|5bBb!zxKOH&@x=7t-ok$!dwLzoeEo4i5_&9ex z1~zuH`6BdWIy}4x)P)8iE=y0_0WcCE*q<^Qmi{ipEHsS!x(wDBi zS4qw_85_&jW3N5MdScUs@l>s-r99tpqY4^QpM^X z@apnD)-vbrlAlUw`!`b$mavixM#o2Ub!eL^5&iXpmsey%+TXZY;JhP?XBT>cLK^K~ z(LR_k=vZLG9-t;|q8&);TwIG{E?v;lme7rz(}RS;qUl3s{JXUQTVBGlAQ)yN13Ync z;g@vQD$$Jtix~gtebYTKE4N8t9W z0dbA6U5?R?rQ2GBW4@!K2trzx=fO_!4tC-|)5p#PzIerEI{3Edykzs4ZIjP6;U{?x z0VhDFqZ3|n)cU$wAtH%D_64=N`{R+0#7zztf;V8RQA@n;Z8~Q;AUf|8v>hz4yzl3e zjOLFU1`~VYTBt-yk5&;_?=_D~Erv0+R{v(`b_H#vui||y8Xyo@c4H43e5N;^VTnrx z1D}Ppxp=DvX;QpRC3s=;C#8(Yh-Hql?5$ZTgMm^3-ao9P{o=+0_}K1&-87u>z6l$i z9dqHVnchy&4$~wihSQyv#oAbG^BSeAwz5)^OsWx^vT=f|ouGS?QY-O535#~}^`Px~ zSfr7AQyqOF7_y?blEdCWA0 z;nLWKu89&cNKNWp-LR>tt~`hR^T9e4#9D0)J`hN_Bnn1MjM65>wC}1S&&%XUBa(-j zV3P;}O&9c$6g)x%Qz|uvBbu>GM1WqE#96~UpdB;1l|34W5ndu3)E30xfj~m-ex>l_ z;;_y>2AFB7x}e8{NAG zo~V%9!BL>~=kd_+<^AQ7^FoJAoW+b;PR!D~7a-I>b#I^Z4qg%op^(Cp93XdDfsK1u z$%nuMBC&SU?O+-)A)ujgs_^I*v_fNb-undmew@dgj8sK_UaQpcEk#>W&4yj2Ns^O+=jg`4DjTfHYF-`GZvyM(pF$}DzE7tD4H*UzJhX$owM3Ity*2Awbj0aq zV={qWTM~nt5h)zmR(Qr}!~6=Ugs49dvwwcYjEEy)xQuhdJ2 z)y?s#&8=CnwYsoMZ(+k|!zw7ycL25opg^OB*JA&PL6<}% zw6mf|jQfR`bco;K@pAbC!`EDf4zOqrw)1#DGCv|r_6P1nm#0O z^4}m}53oR^N9#9OyzR~Ikx!VF8e4}uk2$Ro5ZR-T!i_d2u4^`-FpZIQ-h6y22)sD4 z3#nBxe`$Q1fIeoU1=l~p5xWPZBi4F1h}l~pUs$LOND~7Lhlucd;+E*73f>0R zOV|XAJZYH#jjW)nfW_BJ8|wRVl#F_BnmlOa8FLSC7O;UMgM#cfc$$Wz0*GBOG&8~rTxYr5gppOgO{ z57rh}i?+Wadbu;0E=9(9)A4QRv7z1DZN%vZU-K%)uXz~Sc{bocUGFqSmcv>IYZRKw zlLtm=DhDoAOb%xb5yBBeqXi*3=8hoXk*{X0&gf|7=pOK7{k7>C<1RQzcEg#ds>-}W zmCYXY{MLT3cEf`cKog#xzNqv{9jZJAJjQ(>G?69NIYEEa3Df$hjRWcM7|lalX=x93 zAkha_zw311tK3v^Vl7_@aa?1GzCPt?(@TF)Ug6PIxX;0h;3R_;@vz-iSU>L2Ift5hgt>dJV%EtGpnZ(C(hEed z2`v*NR&o2HVa14WSMP+@R7mgMpgUtmk@(cOtt6FN$xJs-AJ{Mi9!@7p9`lycTzd~v zi+L2yFni#Rq+H8Ar%-NIRvrJwJ{4tZmukmHub>H(N+LGxAa)USyKYcV?jGJ*IDj48 zr4s;-0T4(G`y!LcOZ0CO#%7fdkB>2qZZWj9c1mG`xM;h{zZ0Z^!jpmRE%5LhQ=P? zyQH%Q1lE%=hk>`PwRjh&VBU&`fsLG+N*vQM?My00Y~D)Tkuv4(PMUf1i{p%e89m`r z<%|$V5GoqB=Z+htbEopR?Cf>Cx~-5sn-^HVP8zK3h3_ zZ99k_YBL!)5o_^TTU0D_6U2RL>JzVrIv7XrG5j38vPGfdfbH_i-OOOv@*a&{#%$1{{odJb zKty~L>ub_R{);*eSk10(k^t_39)QQF2c=(KrZo{0w}u1Uu==SRKrnO-dl<;}1$L01 zcVz%t6b=w)mUq*3YXBtHEe<#g#O>6hiYNS6aEwmFA+UBp!b)#ToF#RQKGARs48{vK z9Sq*RKD=Wr4F)8Wtn__GZ~N?!TT?5Q_LEv5N3RvN5VRz;LgEF6DW;<^+kr ziX9}yY`vpv4OD_(unfEhrizbZ12=Z~;6_Qks99JVAWtS>f4Alw24Gk`3OT-p^#*jk zHH5M0#W^|YZIp19o;ap<%~}d-8B+uZ<2QVCwu0T@VGTiI6+m^p&U=3k6vlH3-3V}S z%(G2G2_8Yq%s9yI$%9$NZF6ot!uBmU47>3H)$248!h<^pA-0RG0eDooMUmPN=oWRk zf!dVxwlc*795WyKG+jd7+q7~l8`iS*4}=-oa;vAT;$!Cn>o-=K4D4B$3GtAHOOkHa z^(FpcO5qk!bK2Jk+{NW36*3*XAZe;5sdsyMc{32l$bcY!VThw~5BQx5ko?T*24!t93uedFZPwVZT`ufY0n4MRd7i0PBz0t7^CEkkDJHo3WcxnXrxafJKKCS$xc)7oUb?iUR*BJKn| z$7P$0A$VeP{It-7AhFNOQQ;4{8IWoSewYeXW0>NDKd={IS^UcILNI#wnj_1fgXEZ1 zlfgH5crH*U0ZJT*;Wp=86xy8tDak@=bU%BC!AL#DLy`)V>J9B6nh7nAj=nhP>58d_ z7;c(M?DY3VLSV>1`Vs@K(!3#Y3dcyqXh_&Zg)>Yojg9VJ=ZF}_$F=s{Jy?cJ7c&>9 zX5ww`&MivOmB|X^*ey#8ybAeA-UZ7f_-a*Sup(mAmH7vHPDSAJ!+2k-^gKJ&GkYd^iZd$)!I5aPZnpkMhv zD$pI4RLlexcKjm4jMp+Y?CIZ8Cm@(cqAhQrS`mOHAcOAT&AIGMBP((`!IpbK`3C7Z z4L%5~Gn)h&km$RW`2sjbiSuoQWbE}o5l)J)3Vb%434<-HotV@`Y*zph<@)76@j%YN z?CKH@JPPq&(*k0UVpzAQ>60KWz$y4TD4ZD(+TEgHb9x26+7+yiffoE4Q$!#Sds3-L7Nwek+pSqNeP8uBDNJF)XAiLXmJCNh zJQ5RA^YR>=)gLoXe38U4@)D8zyg9(RQN6=d=?hlzMD8NUEsZ(zR-$Adz@+byXZIR- ztr8FSgTZ9Niwb#*VXTnunyAWEnTijA;>O$5$?J;I)#XU;p4c5EX5JLX7~w;T(?)8_ z%!D&f#A(Ve;C)oWdU^ov)LP;yX%El>9&)x%OnD-FX;hI~E5P9-5~Q8IwYd?8>Rty* z0S=@dVzZmM)9~V#q^;#U&t&GF5?NVmH;dMHQo@hNzokU<3I!C_Zxm7!DlTSpZoJVTwjD zOAR9x0K=BIO6zQoc$a*6U0rPz;|@Y%29|?TD~H)R2Lug{gLbu=u(x|w!%aLVO+-OR z1OwJDsEHoGgH>uYw{b!f?s$$lnq|3Mq}`tYrNFdzS_nZJ8vrm7;}kf&OeG;u+SaSY~IV!g*MCxCiuVfEaJJ zKUwP-|IvZ!rwGDxz_y^cGlsI^R=JIs_-Rw5;T6Tk%8=F5yBRl;y=x*~LZhP>L9PvM zuz|3}eVXeZ;s%M5m-G;B+Y;$o0OTRcB)*K`qlX6|lazeS#;KakcvXk^flR>av9~STqE&u7 zwgpi77Zv|Q)B9igZm`w^z5km^`slbz^tDk2Cx9JPC;(TK^tITIPgCAcx!F%j3ceoN zidd#}D^`6*g(vAr6Nf$11)8tp7zEQ7vgemTBK~`T4pQ4qz(CJraaCB~oT=v3zs~tz zE$!dm@tpGbI{nRYhM_*(^To?Y=vgGf_z}Em` zv9-9jCvlXp7;m%hUjSL|>MhvPa?^QT1;cg18=LZ8D7L(8^wk>9jGp_vehtG1HW2kO ze<;tMQ%1%G_vsS8OUn`ni5@{l!*Qwx+3TSuF=kh2H(zKBh~XHT{<>3MGda4`syiP@ z3)pC)RqM&cjB&622ifz&W=X`HR&qGoe`wkl#7@{AaLsK1;Pu(r1M0EpgO&V05eyTD zWz!mINoZTYNH*00e_gB62?!;JbHQDwA?6z0K;480A0HtcFU^fM!#9oHtTs99dL&Kg5M)*5qRq`6 zWwRglSDj5v8_s<))@=Sgr8V)<%}rXTEg&NOR#PU&2^IWeuVE&ix!I49tA^5O#Sc8u zg|;cPrq3A|>x!Mt*khuYbI;Yuv1A~mwT5OD4LjtFIQuk#0(k%PPv!@LIvGnKr(%7to@l&h& zrY-v7yj%Ys@LP&kW)t}i0rBH5lCoY@)iXT9|CE23zdC{dZQI_W;0X0yHBF5EA?qQ} zzF>t|Y6yrOgg=QCPUnf5GKmOK_x7BLHX)e?v{9MMe#iB%p$Gtv02}7dcBRs@#^~Vi zQr`cGE&orTf+zn*PHMxJ$EEcc6g+&a`MPL?Vs$&V|LYzQ(adj)zW1@u-i+VAu%#RN zVzSA|p!r7VF>D>~q$CewGkAwPC#!r(M!<2OxC7)`NhAV#q?_8X<>Dbd%%ZtL$R-U; zVSGj+B>yvq{C~RV|IIJf8X;o#$paDPJ2I2H2UaPV9%zJ6{rgXJNXp|6o?l+tWCw}0 zi5o07^jA)A`=gYzDo;z7!wu|@5Q-bRcP{jNi7t}g^m#m;%3cjEJ{odaNmBT3U-|Ej zG=WFsr)U30GWCINR4xL3J}Zz$pO-sLRt@7>nLY|0TDy*jJ8j!(-P+pdbACx`Uu_b8 zxK5o4F8H07zFS(Bv-ZS8~t zIfz}(d31&d$1Sm8G5l6%e^})au|lqGD|fiW$(ttV;p%UKr*Dxjg*};KkPgV}97zcj zz2+uUq|h1FZKc}%L+ej7@=J5!`KMwv-zQEU)i!0jr_HI+p%SiA$f=R=7|;ModV9me zxT5>W(wl~z9KMrX*FUImG4gI2$#H;DsEq&*q^yCO^wlAOy{Hx!Ol6g?HPYzdMCyX(~D+p}TeN zW=L|g0?@CkJFPT3s$VADMU^KNl6)jJ{^g;&&C{L7Lbq~m|zmbDx z4`@%8Q$hu6Xi0o)+5-Z0udI!pxvhR*z0w#?9I0lpl;netqd6`ctBGHnRO)rc#Or5;NEn~B>4`@b z5_kIcIZq=L@BF-!fvm`L`_dn!rLpyiyn>Om#{_bgqcU|g6PLdN%CaC-hX4*7L0O^v z=YAU0ly_b`r(<(G@7^!1Mz^O0YW-mL)GzW=twixcHj{U?L>__RIL=Cnk@N4Wa6hSw znG$N{-bSXTLZY5EJAj|Zi~ktb|95Z$%=r%1TqzrBgss8(dQ`CgEamKO^wrtXZ_VDR zG!0Da;TJqY{M_Uh|BI>*trQr7T&}Y}mV;Yjh^!T-=jCnQHOvK zUJUG-YFOx^QvK1P^{WIP(HBwwi6FoaeKH>!P?|R|637XV4O`t>@J1L5uS3LY^?dNQ zTF-CN=qJlcuaC%F8^xO1eovVdO2HtZBxzfx_Ok6=a&6vCzhD>ViykWPnJ?Y-N>)@< zloQa%Nl2(tq2h0p^o^eIdk}K&&sNe`U-U8lSI>?i^P|4|wGOG8PhP0Id;8svH2lY# zVq5vA&EJ0UUN8#O@2o#+#vF1|Cxg!qO?y87n4jM{9ky~p=Ia%z=$hN&s4K$bn|DQbtyKBo z>601hgFcC~vPN80tKg7eEmyvPjMck`@sqinF;bFYPq(3A>jdG(ZH{fXNkNgAb5`tz&uhp{EVgV>Z#8<6;SGDyITt2Y2 z8cKTMB{)LpbJ!%+^vtTBXv#=e#vDIe@&V>2z^4lHGmvQ`Vsz`M@{R>ctgg-FN%Mnu zhzjT6A3tQ+dz}qjgU(0&^4>}DIV@>~%}k>?2o95B^bk9-jx-w5!;ZgKOS(S?0sM7l>@;%N6%j4Ag!DXU8kbM}h;6 znM%(%Us7}y04EfrykP<}^Ioz$WA6eb-EFkFs&xnwbgT2|(W4(e*vZ5nzF01!DAp3R z<4E*cqHV0D_pgZ{wTiHh*Mbsq>Dye=t=P_oYHPHe<0FRPm-~B)h9{+Xa7?C*^C}om z;_8mEVu`7G>toHFH(ag;A|XRtG~!2l{Kdb|U|5b^Z5T$%Q(6gI9!2}vgWxge2&~NN z&LV@1veCstIj4mu!U&AD2nCj zn;0_t`?n{*B{0pwNGr0aP$RSVY=&UzcNhOBk>NiROc0}s1R2!`h#sy1*#@s@i^Kka z7+^J<U|Eas}N|uE4l%LQhGd> zC_Vbx;g0_asq^1Ke^5BP02>7}OTklGbY;{lSdJRCi6YCRcWDoo0@K12STS`EkQcyN z{vZ4hOHc12j?u+QjEo3vKwiPI98HB!wG8iS;NutZ!%9+`*%W=X*At-T_nhdVtbkv~ z&@T;?;jID&m+u&!x9$I6Xr1!VJ})KQEWG)~hiJmhsarFFPCCvRmA8IDU*5WUQelSK zPdcT0P~hEMG|xzO?3Pk{Ns)%fUr)>CXXTuKf-RKoY)2*7zs{DU3u*-!0aT!|^RW?w z%ZW$s1Rh=PzJG&18yedA%;m&~@smr$DirLFNAXJusp*?Pf{VM!_K?by+v=K_7sg_) z{jIrJ`yr!qG3>f}8GOeNpFDb*_jjXv&5sv%g{9C}?;I;iVv{|_Dw5=FYSVI8(ZbMB zj}+Ri(PB&ZN=dm!%#p}G+MrusuTziFm71EFalX*e{*~pb&Dr_q-}o~Ek3nFQYGLn! zYKKGXFcaRFtjkg?5j{5_+eB6QbHNL4t|Y3g2(Oe}y2*6x_9e+}Aoaa-l+}}OLeqsB z4|$3q9BpT7&<|#ITj8xX(3W|Y%sd4p;}?)qK{syTp2^*Y3(3O;E6)l{NS`{o<2L)~ zrq3C%mDiEy!U^~KM$t)0MP=+Rs!Rg8rk8yzZouRiZ>lJ3GrSjSe)%cnHzs*e?A|k~ z$InA4gK1&b?}7~FSu>ohBf8H?zs+(iG-&Ng`&6px;4E#UphckMzD`w;Z#{2qmX!8w zt{BV`btZ?`$c@9|lgRlutHz&C2YD$rRgYO~Z`CoG`AcQF{Gt9Dsh{EPd@rNW**R`4 ztzJ`nI3p3~l|yEKt8E$m^KCDez*)1-fFsa%ikEEdA6+sN5IxU+C-UqzE>dP0pqjpzdTE_ufJ{eI7>-M*5zTAvc6E^)n*A^ufz34VqRzDI^8q8ZxhOb zEIW!hOeCn)NmJ*iEF_<(Hhp=LvIwp#x^^7YhX>_Cjmat%HVhShJKyIkIXdB;B@P_srE&keo%p(s3xA`>iIq^Tz@ukgG<_10scn?HWtkZb_OgrI&0}G8&OHfg8RO zS0~GytgAi>1^Bwsb7&}DmRr+Oo46;*{X8lgf5gE>qmbd zx3!O)wPGw-IVyTQtH`?IS8?PDq|b zU3jaNKqdyrW=}f|UpuUVxZ(SpVJio|t zM3w@}SlxlZDB*e7sI?%TNmG0xdW(p%n?z17hDVkE=ale2{pY`n{+p8NFO<-T#Pps3u}<+%uKwiQ6G>j5|1RomhM-C#}?E@Gp${!JVzE0oJWeZvDl5A0DzYL~WqT zZ3hQ)FF#AYQVMPSWB3o6gu6)K>@U;Ak}7=CJwdjzJRJoa9!#NGaAP57)ev)`qS>w| zPJ-`U0{rx;JB9WpcSpnr2}BunAxe4NeXmkS(pvFtc}>5Vj*c&1MurPa_*3L=Mm}YY zzqH9aH^XrZ!+&~uI^W(O5yjQw(H{T<)jzF`?xt|ny6 zc*}AcmGKG|!JC*d7<_o((AvNyo}ZJ>cOgQo){b>04t(pNTS^{ZqphR&cu+r5KpCOzl6Wf!9U)0|v*XiUBJvhuy1RjW_K zrrf7vl{MGFda^TY(DPN10;ln2d}-m*GUxm5wH1CidcsT0>FX7-0YjTsQ1{Yiw5(0n zYa?V{bgNAY>{LXGXaCcYEfY8PnFgB|te+ha+`BeXLr%L;%;zh$l^Pn%>%AHMw|=}m zOYm($Mji4;{>=G~-DFOU8d_tAoVRFO7qG%y5ca63M_a~6h zw;1J6s$jU~+4V5nrzFKQ+PsdXWhKzfFk#%b>J4w9fdWtH#^`WtL!HORGbKv>PJ|$x z?%y%S&wsgQ48D7RIOohtsoKEz8T%`QV1q$ZgP*31L;aFNYeX+k$u279eC5?fhq{zF zUXf;obo*wmAF0L-)7DS;pThAj%*m_zg8|Ws2`f%-xp8(RGPW+RK)59YtOMt+oKnpl?pbO$SL|FA4ck-X(=7RRH0>aAmx69qF7 z1Akc(|xS^OfpDZ#3op_2fzxLX$k%gOPPe#LZB>LP^po)NFDm!WSs zJK7mTfDL#X-?99t*SINE8_vULf(+Cr&Q7g1*76|bV z4hiO-_shBLP7--O5@8+Jx^uDp@^M^ot3qI|s*=PrxAs%_eF_F-q*J^<`JH;@x$!2m z@G0M6$05Jx_wW2jHYxV^{z5bhk=FWC_Ccw~M*T?wJSZT@6oUu}jy`uzH}m69*R86_ z=Uq}vr#G4I6^zNAmF853e1}Ny0ge13H>#zaQw!P+lX)KheLjc2GMsxx>rAxNq0!87 zx5!Vn5xOSJ7sGN+Kge0>ksEx|zh?Q}=XiYni;#j5wz&4K&J6kI;kPuf4C|38QO_6YFsI<^7-fX z07tXX2LS>1@5?_-E^X}X_*mLkbyFeFB6E3XAOxtv_+sTIYi#bRK{| z&$Hw%5jV-}J0}F}Usk;GkO%Snr(2o-pOwDKnJ(nkDKrUM=^!Ld1cRJemxw*p^+CHM zl}vK);AURvxjgeFj0Af0PqqKkQr>zt}$VA@!7pUU*F|Jsn9q{ zMQw2e+$&yEdl^c5n6vtX=r7qm(mH3A0M5cU%gH6{|NomeN*f7ovxS4)ps*) zf~$5b=0xA0uH~>(u!FXl*4$jWYPTkQPF(T0MnR}iv$>x%A0KPLJsX4i%a1&jVvp$F zj_MY3_%jxW_rpEUu7Z#g(B9NEMUAGbB(0QdSypD6cg*DGjhs54-S8<|3N-4A^y&S} z`~B~vLE2YmHZ?EzJ1R_^m*9Ol(M7|;c2-(^gx7@@CD+G)<+D6ddN}y%854hBSmB$f zFOkPgbV^=|C$n3Ud zUHoC@&(DS4^SSs}u6HmF;?Hl{$*1R^nzhnMOLph|@=LYpmPwX)aM4lQK0^ppHudTi zGlTj$=J(<3c>W$;ifyXWc)WUpe1*Skp@m?NyMGhu$;a|5GZ_ITD(cJ1@W&~2Gd3Rt z?>|n_`*x%1TZ7d2nPcbzP~oB6fwU*R=>hpNy}_R}1(P3JilHs7t6;Ti3HKvU|4MqP z9;{)}_D;e0w!6{5B?W<4eCM-U`O@abPXlDi)eFxadpPcC5=`7I&?1=nT!?+F$nnZk zuk)hbP)7fY9t)RADfU3~bB_wI)Zbf3zyIZtQ|2M_IG(eeJ^U%0ea~({qs{8S{I~&M>1lmU<#dKZ^tl#W_ms z)9fQka8HIff0pwyDRMwR`Hgoh{9N|Ck#$$Azh|~ouBIus@M2lzukYlVTc{4E+ydt$e-@k&RL(Weuy2an0s@QQhAi#?Nr zzq^_DRqkGN>&4*b@n)wp6kTlIzO`#Ln`6fTOhtceZ40U@bCn+e1%KT8g;<@Fq7Hqi zs?)i(c-0u`KViTp4Hkt@BEK zS(;`Iph|0Nri*Fc3pFBREh7Kg^*~b)&fR45E$w2~o`xTdVTIbob8AI0%0UyyRrLOh za12o6YEUcp;u>_=Qx$KIOU#$py{yqB{B{y0AOl-Ce=Azv$D~Ld&T9Sm_~xDO`WM?| zhVtzpIe8^N_W)&YDs|XnlzOD(&D2=DNaQgo*_`cYNggC`fcJxj$wRjk`ZHQ}1L4{2 zF9>^p>43lv#_f!j*+t(po>yA&Ho2EM3|SokuthZ-J+LgBjSLqx%GXP+_ywgs`sE|R z+n`;d^h)CbS{;)qr+xntHth2zx0)SkNxTLF(eAl%iTVwB;pmTt6%k_Sj>x31;u0R> zSbwZ!hZ#jLmh>JNI8P@&1uu#I{%L_033kbGjg6?LHlO)eN4*&DT|BAN zm0hN1aD{6N=3cAKi$V|J_W;P4&YO2y=`Usrn{QpXcRA{EwDsstd6)?zYrEkJTJ=bk zEppz`>KadU(|XJkm7^zI?;jE95%(T==Uz9``zgRC{ZibZ?ZYy5e)q6z218qcw09j& zJ%1$peB6{HvOjH|tiAArUra=9seQ58JUZWvYxcxv=p*xx*dMG)7kZ@iAC6mmM>v<_OcNxW0sD1g+A<|hq zT=}U_Anbx&)U6+kK5EY*jE)K0C3cQPeL7jOzp_p8>qMV=nZIj%`?^fIsYHUKQ{|kj ziR000cTs$%GB?8^z8q)_e?eAQzR>g3-nqCoVNE~7n|3?*Am4)~d{q)uk zhzh_ei%iJYPV`Yj4-ZZ7f~+MMRMw$iFgi|wdaetCr)ZC)L|6;0R-y+gHxgG#n`27s zrst|6YCjl=7vaJ+hZC5}Ytic^na=V24RWDYedo>0?5^5MNB&ShC0s2c)Av^etNyyc z?(UDH@C~cBuoulmcvwr&sJP$U7fv+lS?vXLBgJZqyJ;?lzmRhz?iY{1TDUmB2HuYq z@2Z}j`lfc*9xD7QFmEvI?UuruLSyX=W8M&jyAUxClUqp{hcbOrD%$S>M3916 zUgvw$z(|9Hzs8ZB4>MT(-b3yeHYzdp&wRMv^;F8+?iBZ@n#^hcdmdd$-#bn;N9-!e zI@2B;eHIcBQt{eYOZRO0RnyDpd#&UPaJldD#cx&c-N_LqZ@=m~4EUFpG@5DJg?yYy z@4vG+EhPPGg6$-D@PzsZSF8!AY+bR4A*AG#{z+!*4=xI~sv7mw`)d}5AdzO6JJ6%b zZu4cIj#DjB-}Znr3bA$?O}#eW*_peFvt%=enOU>QwEA~rp%t$4<_@Ur*b z;N`b9#UpTbv+BpK@7|<+XDU1DJm~Nw57Nh^AKK->w^}|+Pre5<-0ns0P4u(2f%ZJi3`3iBTyn3+LMI`D9y53&Ct8$U; zd*dzL+asBJj0(90g|A#otT;0=&wNI_SZ#PaV|K2dcXZ{?(yCGVvXy03A=;SX&V8R> zQ>q2s8Q<@tQuDfI`X2@aM5MgIyj72tugNemJ9lzE;%6KT{v_pr{299v__2l;l}EWX z@`NS?!_F70T^G+&R{C-C{3RQJO)*Ny{sNbNZOfUNk@gX&^%LVSRUB_WecDN*&gwVY zpsxS8t#!fdNleVIsTE5jy`NU9=4pd7Qy)1(dOvpbCtS3Sd9S_uV(L`s4*}^Le~Xh9 z#FOS`!O|ZWpWO}I13=%KO|_Sou|iwRQ<&A}6M||UDkM|};gzZ4`y*e$nU^u^T$kh2 zXm8Fa{_i+NhGpL-rNj#s7!1w*YE$?f!PDVucoWC{BPtaVyXQL4reYD^Aej z1qzfFcS&$3!QGt}cZxd{hhW7iwomrn@Bf_n-Z|gQcg{>^n91CXuIF)=fJPW(D& zN#`6s|Nq@%Kd#NcUFp4b@OyjLhRz>gP;f9ReMwP%b2gN^-cJZ{tqxW!lXHIk)c-L^ zrFq#je^4IB;|cK|j-DW!7OXXbawJJTVsg^2&`4`gI&nqeFeCo|NIk{NoG~6G4JM7iPu7R^IYwKqyqgBSc zMl7E|vlELTBejWC@OW#4Wek%N5QGe>yDYS08Wzu2>sE~d=gxqCVc?+l!8R4VH6ZUn zPt9>7>xYtw%v>u7^2ur7MGt>ppPPHC+G$j+pIx!IxaitF+Zm`u{B0bJKdFrLO>pF? zCPO{t{&OQe9sorJhkPR8y1yr6(U4pz2+G7%ZwW1upJ#V*iKo&ChpEJBdzz;y0x1$o z+C@1^o$kuvQGLi4L~@9-dS3_s<~DJ=JBXN{_FFkl`}qViKv&w0FIR$6A;B@>ax5>KdqD4|iTAIKt}nOAEoC)% z6qy?>X?<&6!f!Pwb$&OStS=k;vt1xPUgoz<=+?ZB{1IDH#fSG;AH2rVO&Pnc>53zR zc!|T0$O>S~WWe*VeIEgE=<^uKBXc`Xt{bZgWbhAS8v;l2Yu=+&3QXH5l2H^whe6*! z)p@)>x(cuRl&5vyMFP6>4KLEV$p@y*vB(UNUhdaWjisGaR~GPPY8LM}Xnn zH2|HHVV1Ka@E2R$52H+I&}#oG@Ha^Lj8(P&7hF3s2jkz~MnuHw#r{@kks72j#YsZK zjN#b;)neWK+m42Erb4Zw-ywoBn1XCy*43L9KT52$#2`G4HeV7wDk@vwGO>;n9bzpz>D zTfC>U{qLHh4(+_|@`J!`wq}}#B)b$>^UP>Ws>c3kt{q{SQJIeuD}}dG^e3Pp-mmP|O4tOZyd8uN^XV1+4}^l6Dz0aKanOY!x@vF@cH;aw?>S6MH* zy=i2>qXbXv2u5IVOz8;R9}SP-t6SE4LJ6G^0 zcxi+umanWOoOeH5&Eer-1;l)MJ6CK)OquR+%7XTMY!1 z3&;*t&egC4UTHC_H-#=Cl<+0|yWcsM=a*x)QmdE#BC zh@%~EQIGX_ph-=;bCF?_cpk*wSl13TfKyH`>(QJpJF)mY_+BE?mub)S%6x<}>J6eA zBExRwGZcM^BYghCxHzN%G`^<0M18s$(4~fC)2*;Sa!EWH{9uQd|;9GG~%!lBkqjq73kpuUVZbxB|N!K>Rw=7t#?3I z?ByQQMy8)YKB=BV0dEGv`$AvbD1XYG9{2e#1Qo3UV_WcI!kC!7SFf}EKKonC>d!0Y zYeIbj+Wg#sT=1Qb&B^IlWI~cdz7I2R=?=G_9YT84qjpAqpOBM|7ONa4Ee5x#Fh3s*`d(t&itTxEFE_ zZTfh|)0KhnGl#YK90~u0G30y6@)riML%SQ2+VVp4pMD$%lVL`C$On%lzYfGdo|``z z6O~fIGgatx7_j|QX-r_lRdVTQwHDth7KHOcVzzB$!^PsT+lpjeeCU3;U|F%V@; z0DQq*38pf3oXV$$Oi{`YQT*vCzBsOIx@wX58eeX{vAP5eCU~+66$*V=aAkjWp<`IXb+VKHHIJz#W1+qoLDVYR()Pxd@>d2YTW&*7#j zGFnj2$pWX1TdHh;>Ad^w$Um983*48^=-eH;4q`U&e)i0{FB7=`w*Ww3%gvlB$Er#= z_TQt684KcY+9^A96Wn6()3w3j>a6miG5??T>%U2d2)u`H9sQnSm-XFF_A(H#f}ra! zK;wbX>yrzixr`rm9qZHoaSO@2)ZrMEg)Ewap##`K)PaS?O4XVkjPzI<85)mcjbB48 z6-{x~3=0Qbs>wNoGKH6{ar3;!^g6A{JLL$Xoa7r@TB&M019x5yE)_I&EIO0OAV@h&ETGXSVhYCeyb@X#0+WfG(qqrx72AteH(JFDSAjdC?l#;`0&e%>!-}rFlpvYPnNu1xs|F|9(a+o|slM4S3nz zChIMp!C^;wthwT7K9$F83anUny_cu6x+Wy8F2q=IMt(%$J)npO!akw=|di z{BR`p?6(SL0ebY;ST7TvZmQ3^MG0+>fSoIH&ieB<5qyXwCoUhVpbf(=}*#XXfvQa#aH zwgz0x2cW5PN7DHBxEra(=5b$)hG+t1jVQxtrL?tpA>mTG=(hC}DHL2@JtJ32Y9{+t z@K5mawSlEA!?`{nM*$BRgU0WA{MXAkKp^$Qlss{oRF0DAO%L(U(_7bhmEOt+9n-TU z>IJ;#18)GQG~vO44HLJ#gof``gYeFKTsZlW^gT*&)f3`5S}LyBiSSSB`QbK0;zyG3 zi41o?t^M1Qsf{yrwn(b32vTK^yyLV7jxy0U1G`y$^i8AxN3qYz8@5PbJs3l))L{PR zW+8fby|@LBxad7LVrB&r_ET!IXlue&X}lDbEljm_L{n3()cTDTKvfX0|UH+8=Q|Dofa-3nv;+XM3t!5p|0=O~wzblC&C zV{K^-;u#5l{eR~t|4&$qzr&~K3m(Vg-$uU;-T3I_YDbNsL5RIG>)M3Ye)%=L{lF9x z+Hgf)>adz|C3HIH0^B*@u~U)P&g1I5No>ODbhy|Vo86g9%i8Nvdx6384|V>UxFjZL z{np;zI;C-21px~gmMFz_VDvEA(zk{U4=*u-S2UDOO!YOJjV{(*bn zrA$C!jVGxmbWyVT%sl{J#$Glq2vhy7+SBB~t~}x7I*KNUXlt8QVsQKRY8^lsu-jZZ zG82%yl3A-c9*)NRODy)uqj3)bH4%bNP=DLg?licLKF${pcMor$i5C-2kb8E=2@fyP zZpSK5zI`4oHcLzs`nGJ}T*Tuv|0nF|%WQdzJ1SXz(Z zNu{yemNH@wI7wn`dP=L+TxR5~l=&Gbx6P(P)uc^aBiIR2APR6d5H6`$u9mxfc~suf zyeRxYvWGeihVgG@L0=58NOOfpf?LSc6}7IcbmC~_^}i@;>=Kh%w#RCEq!xafpj7={ zTliW((@SA*$G9SGa&dBUdNtg82O7TcQf^h&d)dgE>h<{rQVP?bpxQ|Evb9*#0u-ei z&M27sm8G-1DRB{7tF|ovObjA>o@e$1>I}wM7?_A7VMQAG`1l8*X~fjROLkQ$x@F_c zj=?=^|BvTXNaHDffa{`&rs&ol2^8=bMlqsEEeEnW3RQSyQ4j@(Le zpPM>nfV0$1)pn0Y*^*R}j-PRJbBh5BV)qaG+#jK2D%Xo4`RQ7`0veFuJmtvZkyO9) zxxOAwcDDI+TM0Cwb-`#B@!#G@3rSH1L z=gs-H1F270uWI`{yvFr;=c)3~#PNhN6hGGZ77aK1WIdL>LR*R zN?zb5M-!^12%;4m386FpBQa!P2S>ZWgDzk)W(t3^ZU5p+g>0-}oR(Hr9dEss{B*

R7bMG}!AqIC z2`!m1@MBRbke81|xz`^$Y7h%dzHa7+Rc7=^nGxec=|eXaOv0$x*Vk2X1V>btx&oum z#2aDw6ZAc8rMDU${YfU(h4m?SNMDaI;>f}Rb77*zaTwN$3Tv)tMhE}7%z>7%uR^{? z(K-8PeFs5~S0AL&04;+o6!@kKkBPZ{QOX|140x(PI|nz6bBHd6XnPO~xRUoS%5_%6 z97DcM>h$#vI$o}DbE~R8cNR_`9+l(3iu@H1co*7Vb?J?AXJwazEhKK#P}swnvDnyf z9hmC`8y4|I$`rAnAJ|1c0pfrAa#&ax+8kW9L9c{Saf>uR8wjba6gNT4Fdzk&NQkDe z%JV$22^mcWHktkic?RKmg6#qw(L;s;`t86bq1psLGp$jd$t_5~u6t z^^Lo(QaI^Th|Nqo8k))E4+hI%-w+@ODLU#L_(5Xk((9loigqL}t)|}g8H0zn2Wx2h zgGK1d22nuK=D_A46Nm>%ktABKpYE)9qHol>S`)L+U3GJw?`14VH{^`LB` z!S(0&`nWQsF(dCrpAk(&fFonD{dDN^3+*$lP>Ily?a0WjaXux-arq2-a#^lEqRP>= zNLHPxhq@d^x7T+Zc+akNIPe1m6L`91ZC(d)3ehr1X%iyY;Ll9;#jVK#`f5w>I3r8% z1Q|U%^IYXqq?pAV*eGbjFHy2*C8cd_k>LZlQd;tj{MS+Kq}VumW*SR+3RHk8&BW6% z>1fg+IInvM4rz|LJyJU`2Y+Q1oh>_%zYkmcqY)r;km3X+gM{ceWtXuncFQ84Lm;t_ zCSdw}!*E`y#Ne;V`8i3_L2%~tuX%u}n_lXpFU3lbgy2GtlP<$gQ(@f*J?0O@UuxmV zUbxj~rm|IHas;xlA8<8jqo5#_%{h+)ciLa@SqI^@JQJu?a87Jdt+(YlWxxV+Wml zfb8B?U@6HfqUHqO*J)q1n#9Mt&<2Vv*_q+n#zoKa2gdDHJ8TU@{5xXv`BHT`St7B{oB~U-*YZ9VoMBZP%I)MKmV1e-% z_V(C0fA;aOgt>1k51ypmd{&6Tp(GjNxs5{{5v| zh);zS{3^!Nxliq8lAis^deed6;;@(JvE!qvKKM(*jZNTGuPbn8-NN~R@fSP(s>8e0 zp~1`33$jz;6V4p}EN84K{^ghQ`%|Bj|14|$s5`)*i}}+dlO#*N*zr>fY}_l zBA`u@BPI2l-yC``4;}W}#E?0zt5}1~b7F~dTVLnO;z?)I3g$o47uxu%@#6kmH7qPE z7aQgj4ys4eJCxxx69;8~jVFf91wC>0`O3wA1CG)VwF#=K>Gf~`9MI`lWJ0Pwo2iTe z6sXhm}H5tdk`Z`AbLUh$ogeQ98~fH3ac?N|eD6Ab`1~DDTarZS0{mLwX5= z!h)H=G3z!;wstlN^vd<%^daP^Rma?2NhynVl0FP)cfpx@pQA0iX>+n#_pV2DK~#YB zNRt&waABPgKp`ed@ndhBN#?he8XK_)>rVLRftnr5%Z2Q%^g2>y+7Rs&5_}e1`*x6Q z!Q;#u(tGU=0VvlSy2t~4{qrV$#k$JI6myoN5g_4!O|)gmAnJZU>$Hj*AC4#jiPpK= zUu7YRbaIirIs4!{ox53C(dN6N=5b5=i%v$>o0vm9hGZ7?YGy$WKSmUv9f<<{e9aDo zKqr6LTQl-@$YHeTni(HFJZbx}fb&gvF-*ajZ!SW1_~E&DGP*~l=yaOTb5sjXWsYB^ zyiFuOzS9{2lOl9k-; zf<;FGF8Vl>Fsp+(xyN}-Jq8lBk{md36)7G_=nx~$d`2{Jy4~hz!b|69ItG!1C zd0hh}I5j*=q?Jm|cp?uP8vxx2=fhOyRJ*dh>Gjnfs`33lMu6EtX{Mt+J*DQNH+{uu z$gF+HJF&0Oh=^PzD-(5&y5JJivRz^kzenaLABaY2B|++*lIc9o?&8Fpso3ry+MyTQ|0Qll8EUK z2PD*_)pX9s37H)WCk(WSo`boXHOE`U+r(1A&&X;#I26!_*Ei&6Q=PP;K6ZMQTPi-+j(1amiD1ONDjm63~g&oJT|drXzI;b~uduI~LK%u*-OBDb@f zFPlAY{97Bsj{gt-JPt&BRrrJUzH9&MTljY=SJ#vJFAQ9^gE;3bJAUxsSD_G(S_4cu&oaP1SRtWnXjtWy zFj1)r4RmXBWbX~C&%C)L##UU zE?=5Zl9KIL-UKHR)XIu*gg z-nuD}IBrHrYiyGy0ye7MHA~(Vs;1OXfy)-RKQ%ysPMa@OCKfb|9-ez^``bu7HUjL;N{HIpY{AB| zDJJ28VSWn0$?L%|CfV4=Xci8lcqH!`39Q};d${6A(*2HOc00p3vt6fZYkFTTKOU@H z@3XpdQ(;iIH;Y-62&e08bJr;_SqXM2W(CO05Ts;+cxgaW(n|S|Bnu7O>U3zTnI?aI z@TOGZ+xqNoNcIBLh|60Y$|x2C`-zcwxK-lW@!Z3%{)62RG_KoGL@ zgOSw*MJs3J5V%?C$RZ*qf_XqKCn!mxHBHnI1CfTNp;~XD^u== zPTeUN)v*anChjXdA)lWv5m_u>q=%+ZsJL>ziQO9|&WmC=QeIjlvii9HjF2_SJY8Xn z$-;3{C-Iet*zji$3Ej$)d&P!#JC&9)W|akLdmkTWhIn4Qw<%k4CSr=C&ek85$m4M{ zN7T_4ZS(i3d2kuzmo?a3FyXQZs^ss*LJ+W&v#;{F^zxzKU8Fi^82b|9oINxrO>WZV zH$IyeUh3g85>a<7IMvE21?G(LJwM++&;&J8uie`!>_#GX_XD~!GZkg8mG|~$XKZOG z6B40s_J|vu_oh(xRvXe0G@=xdUC`=xO>I+Bi5n%%j+|wEq;*ALF&4*)_XIUsRe=tB zD&A=7b1LC>i~a8MFARC{x9#nc;#5OJlfxrxn`^!a0pQNS+&5%8qN9Y_Z$^WcFa6w{ zwZLrpNyw*qOGskFE9A4rb$LWJrr#?fTJ?m zhqbQb83E_pquwfPX>jiTh+QlQD#u>~0OZvk$zGZS52!ajnwE98)<&>>4g6=EK_DO7 z?1Dt1B~tb~>sr+WVLw~-qiXQ?=vRdD(x#8G6f|S#U1}@0WM(oPW|%?}N1tC0-% zOmi=3txG4KwzgOKT6%@=edmq^`@Jx`HRMS#O>o9bCv$9CXkJ)TtAzS7xcu92FE1~f zN%4?{50)ozs-W_bp`Nz#$j>K5^|Q-$C1#FqOGVmg*!l!UCE~vH0hW8J5*_xYE*Jee z8E67O|C}@%Yg*Gipi^-2Q@+3e`mF49q|`s2M!s=6L`tra-CAlFGHfkKA(aNFzUqMvk_^VE9j3LW}BHT2g2ns*2Z zbrzFYCi}{prK;nRxFIX5@11Ks3C?iAbxVMzMKuMC#rtl$3ZcceAOBhiIVsgS{=%@_ zY4)Jy<9bqORlm?MgTG|kR)GRGomPtZ@{rUhICZN1{>2Fg73Qi1b-ReMS~K%prKZ_Z zXY`o~p3mJ}?f4u1(^dsy|HyS>+m9Ac!hbg6U@GO$FIJ7e~p55?@+@j5oMpgLx$K;Qf&+ju;@if5KScY(# zu%F*8)OC+qf(?W?;Ov~c)xcL?o*QRvRi7w zwFla5#Km7*qM;2h08bzQ*z^kY3TikC;IHx%MU-L!lhzJZ9P&f6ql5wXs_}aIJr|jw z7f%#jzJ~s?$mI=)i(90|=K)a5J=Ykq%lZqWc3R%7isC8KlD-zA@3`6i<$3HW`jOT6 zIg#d15u5m}tas8lhu74j3PUAEU8BtFhXq7$O#9MG&#$qr0gl$%;cs@ zo73MDFLdKlGO3V67wsTlOWI6rKA&j$NFtmdn$9q-fiHE~wStbQ#hLtR`k!A6e-r$! zB@O`E=V>PQH;o@=LhR%kEj(mDX3SmF%;M3pS6_w>umBYQ?Cg1Lk?8|wOU2oSZJB;)^!v>;^?keOEib%X;DLQ64rS+zv-4;-hZT$? z!#KOUv~jsYCrv)UAjQ`eb=-`U47b%tW2v#u8-CcIHu7&H)9MwsYqUtN@|3EvH}XB1 zKl6r2qWRr_UNvVb4K{hC+nCk%PK5k}BPy=Z{IOdS#gi^wX?EcC8p0f1)t18eR|HD= zEo-oAZ%0DTsYHQhZ%r4ryUEG2q`S!xHqHA*5Cv09y^e~#WDqLfVA;sXAyRa4au(}_ zk;>p;R?3=)LJC1`2&H?t{q^%$2;nxOhC*wWwrCK)c(Y0=FpKmD;1Q!SpxO-1b|(3| zEi343ibGIex1~;MjqpyL%r6pkVSQ{Ux7#qM$|j%Bqh2<>`C~eXcPTi215`j>FvX!5 zMG>kx7)MKNy{x*Ppk-cb>=HKpK3T)#Ce8l#VIt z@nQTZ-Q8?^NMin1f2}UXg*8-MzBZ3HMdn0A ze+?{vOtKqbV=(6-asZEjbB5-x&XGW+#PK+=V@G($?+TU5x1Jv)%ChIhs$4C4<}&82 zEc0MSIpoB~M)JcGXU^$3EHYN2Y2W(xvkoI&{?5ir3v;MS`o{RH;4?g~jFnw-@cq+} zbuAhDUH0$MOlCp$W%czOh<(x)Ar$fK-%`L=z$KID%Rl!g>m`2Ww{Kg?z* z+0K0P7sm0Y@PP2pYej#rFJ|w^8y-nj^qHG}KSrsFIh>uL)dFM41FyzEhxh*d-T%05 z&;h=d7xE1oK7^7~uy1=#e4k}x1PFXQ@BIgk!$XyKTE>QUF-UV(&~Aq56K0nBiAVjA zNA;V39)-LAjXOTlMklcTU;JJF{U(;s>ed{g(-lpJ@rS1?i$|Xi$Zd*aJ zo<}Ey!gNWA?AXj&;cw5dFbgsn(Vyk$Ivo0U<}bQ4LCq|ImwzWW-?d6RX`}n4aMz=A zBpQLyNCx2lv6{+IP84uZ)XEbOA;n%AS2Zb%e5aqGFI<1LyZejfl9yK+GsBn zo5h8By;w1|_r_O@vsG2Cj`0)256-5IBKspsTahSrQ~_)J8QEpX=K8D}AHJE4PI~_K z6km!yd;9@wrg!VYEPd9BCK0e+M~NeGqbgsTV4I9Kf56e?sPW;s@?M%nv1Zq7^!>)h zrQ?2{NopgoqQA|i%A0#|RFtFq$T<5ghwKMshT3Ke``9D|FhA3)_uIra2~r3$!~Awj z(C&o@5;SWdvHcZJN*wrExEqjXQCuLNaB~@*Khm>&(vvW_Cs#V`$W`*7wKug=*Ss9? zqDQ(!B|V>sd3cMiTZ(ilnc{0Pv}leqct_aIZDTDV$Dx%!M?pgc_6_|p!*tekd1BU^ zOC8fk$tgUS%Ou3A&D_Za+?{UA!}pON(2Fg|JI0<>ELaglvyg}Xw&?tiOJe-`s#z5% zC>e$bu;B7Du%%AnAfrs>pB{9Z!}RJ|2m=z3)VT+jA9DP>jMPslej3y)-qMAMuErgA zwn4S8);V!n4QHjCRVH0!tqX)t0Gu=5h!WTJl{<;JJOgy`O8walR6FO{@vg&JUqKR# z@T*{^7xf?8DFfqSCE4}*(saw-MR5o26OFhn1(}oLYJ0W)1I2HPY7@Vzb)pkDJJ=uD zBw6B;$LU?C7QTXw^7~_$h~aPG9IndTIN%GtV#ECEgG7frSM~HM z1A_3Nqx$m#&j`&5^`dp$5g%Q6^j{dMzyFjh8&{rJuv$|F5dkIP${+#x(cTl6Wo@76 zDF&?svg^Mvn(c5jx+;`4*_;I)H+0V{+FCmD*FuuGl%JB@|&ch zDomQ&WIoNKl?KB_%(o%E&t}d|QXc>_{kzQ_)6UW17oZFEDd}N;rD-|uJeK(VaUw1x zPSvv%Z0|Ce5|CHm=oJ90WR;(|;+zuI0Z3M!Az-1AAGLMR`x8USW0mhKsCLIUSz!>C zFiur5;*kg!Gg3TJiIMhpL7MFD>}WMhCXkJgOdn3~EjYsRszmoJQ0M!1f-nn|Kbya; zn5byaEQq8o_QSTWSx#aNji4LNb#VcltpiLfGJ1GLO5jN3 zGbZ`S^`A&)g=x3A2(6Z~l-n*qFL|$l0|$dzeOD+7!Yl&XdXmO^1Grzl2zVEbi5a5v zjWMESs4C@c^V{yxnwl{sq#NUvMI=&pOfNAl(WdM=f&C@M2I^_wH-fgUvBdYH=3Tb? zYNVQ@iRSgh`-Lj_2g++TESy`=@BJI56e@hcmV65~OGl0IdFJ<$np1H-2~7lIVkAcN zhX(^PN2o*ckP_^W*{TL38xv~6*fF+JsuY{7(|jU^EX=KE=9Xl|vZ_}toGL0fp_B?;5;L>bc18`b zn_;D2&T(<_gN%cjx$Yn>>m_;E?XIBx{-@d>VD~io>{3 zVQNhywoIow8d0J>(5Tf(h$XMEN?Jx`Uc0l()p<^l?+ERxp;oN%;3eG*A5>shEW&M? z%7^8)y(7e%K)&3*{-WkqnDMi!&@3Eg+mJ=zmgGOhc({z+XU#)Lm|rv&gf;aD2U}E% z+mf&h+W}zDW}v3a!4waPMY5uek+cWabhHd>T-V1j6|s#_3Rhm8q}(h;I5T)c;(48r zMK%*qvP&~HJ_1jEDmx8vfTetf$+{B+F+-iNCVO1=p9H)x*aE&E_m>2VR?b~GStiq! zRx{DYw=SeC|5i8l9~&TwPKf`y{AYi#HDdH(XYL zUjI%jh(}UAZVnH?T-PYa=S~*=dGuRwOz-xlq+|)$PE@biY8!JtTU&6j9rdpHr+qY& z3AC+Q20hO67#v2dx}^fLc;016*{cwm7?&gpO|j$3`y&$?_qs9{mOoT0#~vYjiH!q& z{Y53#ohZZI2Bq+=Bx16A3-=9L&LCr<#&$rR$41SYvLSMLRg3dHJd7eU^78yVxBA(P z--yDBtV|*Jlu9|_jKw`;P3UDlOq3DuxVOkCrs&BBIG_XnDeRH&L05R&fnc?Z66~&s zzUz&re2Xa-bph6b^w-IAS-B=_$W`Kb2K$Uw*i3{vZ8h`Tw4o{reIvVmbg5ynGa#$@ zk4OQkwwQmxUUbx`Q23sehq(}P>}h0{zL+5CLpn(-;$274kQI#pS#kmQcq!C1H;7=I zS(IY;vrRx!Z#|kG;a|qzn*5?5Goa?J=aJ|!H>z(wd*uP-p%C>}s@0paBm4?}t9Wli zHY8Pj0yaT9{!z(%jhJP6OH#I6`9%cBmJIKZ1CH`zCLukkpMhTv!VlGploa&T0lIkY zvG0JR&UF}Eh1NUTTo+W5%~GT-V$yfokw%I3fXIfr^prsyt0bE<&@E3sp!LpNVGge0<43zz)Az%?-D35UTFDe= zgvzrgnQ>wlCLdjyd5aDBMUONpvX2U&7gKc~-z~9Fa!7UgFg0_#;8hu=c_=54a&wgp z5iy3)uy05x1tadIC-K?&Cy0SUvNUigrO2=0X!MhQ6g&Ijz*6}1oz&j8)C^`9#q}6& zg0nkBnFn0CvL)u()>95ignON4k!LdeNRzc+d)|hy@Xl31och$^XYAtZsfd6zFXIB+>KYj-;+Md>@X!Pi$!iurRiY$g&TUlxzGC~|bP2iL2TOFhj z4~F@)o2;Y5tzH1I!6n#cTecF*68XRkhWNf*Tv3uSlEvqL}meuB}IFD9>i(C%CMSjc9eH$;>(I#a`l% zFEjGW`dGBWz2@Itoid$VtY*r-Wu`Q@YK-(Q^a!gO7@DDZ^6 zy_{R@dpex*_-waSg)zYi@g!;AJJUj6zW8~W!30+!?ccnPvS8+S{r!Vo9IO^wSp6HE zU6U#~n@=qZ(n@n?UDLWJStbeGxRmUIhiYdD`Fjc=U%!!HevRxUJzb;6gn-(@%zlig ziRJrMPj}2l-^yq6)(xguI6Am}PWx(X1mN`$%60eV9wGjqs)C%!aLXN6%072y@M1XR z^VcaM8T^P)zZ2!eDq|sX1|VNlCSm_J?=cNtvhQG4X`G<25RQ6D-*=}wL7<6VRR&b? zX!UOE(cCGiO-JfZLP`>g=fI6bm^V~GqR!jzvlEdwVox<~+@DC|6x9szM+a_8b$B!6 zhaj@gyv|!Fsg7o{6oX7|Gd|S+yz0ACTWNILJ=y3EF8C_Y7dX5pZ)Mn=N~S_mBYz+9 z@=Qm{fM4USDFq%lIX_L4IB#)-?qeV5mMbDjf875}G6IM`>GXT)9X>k*rZmTBq`;b? zt}?+l>jULAxK{|?U+hxTj|vpsnnrPTM@kiKVXuf8q|=iqlcy4=ff9ltmdKayH!_HO zMnrbHu4Zf+_wUBh?avkdy~w6SQ*GZfejUB??JyPOmAYv5rRy;BPmRE$)(#oz6Z}~I z*VO&P{d+$XjU3pN^D;-y$Fb1XhFA5}SeQ_T4*Gon$|*p!nKYIkgMq z45o@1PZ+Bwk*GvdTM5LQ2;s_mdT!a!L>klgBWSInMo|u%JfVpLVnJadQ`E*X*_>Qu zu>;-(9hC})Y#)^lQtpj(xKDJPsj6zjGK`Fj9BCvR6395^CV@Nq5jOmX5+LPywd($t z)$0*%P3q8*pLxOyv3fjPLs`mhe_`NlPkk$EtXCBD6$~m6yx#|-Q=?A$9OWY|J!ffx zG1$8{rR1>Y(!tgB%*?!!;_r4dA5hdcUyWtt|CsSb(kn}(x=ZtF?$pr$bUHUFNgxwb z^HT&HeW}ahon@Sr@K|s{?SktCnCMyzqnwZ%RnBVm!k+#YG@c{8v6_lC3lAL4E8x^k5>* zv~T{Aj&Xt;>i2w&YO{N7^JJ-!r|lViKmlQrr~bi0o6!Ai@y4Gv|2FY9Q=S{sT^Wzl zESafItd#>-aVxX7mEy+JERaO{5FY6FhKwRm?gWjDT$i>kjxujY?CZhh(|;ZS40CPgYgp@F82WoN?#l(mk2|5GPdLk790mdTcT8fIfGChM z;KxT&Ld1dWcdA14xidEHTLooZfAp*>9A>2wUo9!kbHRw|DlI58UjBJ5F79m#X#GvP zQs}y@!EpG{4~7{b($`lDx6bHTPdy3DneQZaTTW7aXJw;)^)u$TBn)b>Lfe&4ha=B_ zV%E<{SjxMI&9Xy_*e64a1agGmVG0T7{&&sfKV=mxmgR~nNhkF9KOEw+vZbhdamkJ; zv;SxgF*bVMp<$6nx>{f6sA%sN{m|pj_I>lTcs5>=ySx*5)QB0e9{Rspnjc9e_^bB) zzV%wQbN+(;6!(inS3;BE+4@TQ2ll_m+!rt3b(BBOMxSdd`W)&Tmj0NK#QjcuIC~@< z`<$&}U3|IwF;>S}g=UoDNz~`X{l$(uzm_Q}C-D_OllRKDjSYXi?md>rogVo+Pu;^_wJo;BkChit%4HuK#jN{=>Rx$Twb>Im)4@G5s?< zByIKE+;2HChab0pVMI!hm_PbQgx?*p(7ig-MZ)1`XS-o-0h0N9u7`$fU)0_8pZmt0 zfUi*s>ZT<$^}VM@abGpH`6A^W;l8FZtq%Zj5fm22Y|D5Y!QbxA5hIXT9PfR5snm_x zYt#vPa)t+&lRRySwI?cCtn@B>oTW4vBBln(7G$hsJ_1x<`KWIqc2r^}(X9VeeQO59 zJ7lf$E;M1aV0z9LqVwcfuvvezT#3C}d>gT8wVZmhJ~{V3c2dcHPg8r;YTmJZD@3ba ztj$b9_Yy9H?Bh^wrp+`-lSCJw#i?co_hc?|(Xul%*mj|tHYG{s><9I3P_gEoMe+_8 zr>70tTG;T10>=&XX9~ti^C7nYBZ1w(8A#^d%vaKIth@c9PG@xDRU#W*JDnmJ;POPv<|2b-Mm|khnN()F zPwW{&;5G4*-RmISZFc*P&xhQ{ny@z9gy{vb@uT9xvs`o-(~URjM4$6`U%j2&dVx!C zgyh;b1~T-O17(Tjej@{bxDV?8QG^KytPMPQnDaAtsLYm%BqIh!e$}gml9VMB)!$F= zgX)EMlZ`%h*4y_sXxm?FhH4q57LR6yf!3{-YrDUQ6In?r~&W&RWrVcMPWl&r@12jvLCY-dAIrIe5+1;QwV??rWJjcnp%uw>t^yi&#+i2f(IKF zC0S%_O$^r=T|M*GS?}aPk4p(+)ssyd@%kyDBCUX~$RNrQJ4THGU|Z5Kvl z#dmonB}Kaxke4^yEB+`vLS)kTdz2!M40-I5Z!>E0*A!btZv@~E_mty7f%(9jJ$L9a zFokGnH^<6G)1ah)U2%AN4zR{KtzN_6xV`eJP4xQ~V9&pE>_nvi>6Xq5SU~kF<$vz#n6%i2c@Yc531f~k1PHEdtfYEB#)xOive@RPS1gyv8>iUFJjr8Y|loU-Dhh< z^VEziDn!4fD=xgHpUIJHe~|gCF&NX|PK75Qgj?lQHYbc;PaFs-O_?hS3yn$=a0P27 z6*^irj2l6+99>;XH?5pnT#k8`B@!NR?Wdg^5p4a-63szb3VA2;V#5WW`U^DJUyp|; z-aS;P&izq%iU112I8wB_dljG4dmH}9P};JA@ZCt9YQUGI;ni4Zarf<$H_ z)WRoJWTDh4UUgT;>ojE1fP-&k&bAKvD{fK%r&>ho!3dTDGH*Q!^FqNXUFzK>0wVyj zAtvMBZjMK+(~uzghCU;z=r*W=aQ}xZm3&=c0j774l@V9{Sbcvake^7w!S9<@TEXC_{ zk~CEL;DkxM;h0@^ItbZsn9&56WV%!97iEWqGiV)*q&>m_QnHOt^l4Ys2^7uq=^dgC zk(?E?-&b#?yDnz(k(Lg{*@=lNIn`<=bYW<@&xyPQfUD@>;23jB?(&e7h~A3gKJ*+p0;YVK{F{)GPrU8k`GKYPOicWA7w6S`S~w{^jdqn_vlF2gtYO%|zd!X8#c!m?-uoGkFfZu0LP6ecE|SjE+fd05TR8vzbjd( znI|g)DR~b#+kQW{h}BX5wq#@2jcJ=T6+K!Jn|;1$77IFF?>MkHAG9y)5u@f5WQA=O(uD6G>BD&d5R(gr+mdp?z(yXPXpX!R5nRc6!y8xJGOpUmUSL zUpuhY*Fk>N$GYKN9XM~goX+-zVz$ax9(EjJCWWNF2z4Q`w<CU(n#}Gl1?_*EgA_=wFl-vy<9$ywv#!pQ~X`}wCiI3Jk1IXkhdDt6&~Ao@P$to zs5dQxrsH&a{LVTP?*(w1m}BL~$(&^U*vEVhKTKo4TU4;mO(%;}IKp95b7E;0QB}oO z=7*Tdvx;_hw&&jJRF8a;-fn4$x8sa^16xM0S*!Dv)CVx_ZwU{|d1ez`dSroQk)6&{ zgW%rLeF?)u0k)p7WhC7d!ak8lvH0;x$Lgs6JSm5x#KmlKX9k3u_&K#)BkX5tYC|_N zGa#mfGa%@{xV{7fa712vgUT`F&d-t@3UWU4`dkO8uEnU%(w(y}Zk3*;XenHO9XG3W z4B2@dnowaGy>zHWpAAc?elcQP;0(o$!-4FxMN6Cwc9DxTrL$yk5`F zQ7X5do>xB@m@JzdXXDZW-u;6v_JIiEijB)^ksGWP`1ro>O5%sCMoVQiBJ zHk6)X#2TfO)w90o*Zg~L;(I73l)}x`?`@bisc~BCsOhZrSs1(}|DbrCnKXp?4~ptE z-du7u-vf54-wgCh*;2>+U_t-y%qHK*h5%jOt%!LaAK>@QLcs(MZ4$*YE!`hE{Xkt> zM=W$A3UN{XFyA0wv$&WGg!BWgTTb7c5J)ihvw`M2^y6jPGIKz3GekXJgom*HZb#i#p#?$^_eQYubNaLjKg1w)ag>`>K$iPRCEQlOdTs$t7lSA`)f? z)Ec@-`{*`ylc=ysh&`UP$BjdD_rPy0Ql+_}$)C>{Lxck(lwjLC1<{JCR*729Ozt2` zhVlvXyD7SQLN>yGP#EhAqWv8_s^JSwQ28WpU(TWfU3^Ky#<&hE&_U8j2AP}bB5wrf zjRR9J2=|s)tI{c=9-+q{L}*~DU*JVjQx*fIJF3ntm~kR&-~E-SePbYzU0@dWu^{~Z zn0%`uYi;7PoQ*=CyS6st>Iv|DJ(I|_@8Cx{VbQj2E3_Nn2fl|JFsp2VJUXOd&W6P%Ip%#9v--rHiNLgu zyo0dlOlGl+(u4;1oI;w~e^j~UM_|aO0v;K4gTMM_O~`!#O)Np_wn``jTweDNis=wp zItg!IkDoxQs>>vQvSE(XMI5)9iM~(dMSeFA3!4K)H5*>XQa!6xc1k>{?k89m;CG=9psua#%sOm<|fQi)8Ja~6B${`CW zBgWxW%3-ijJA-n$-&Ir}?sh>g-V?UfE&hiEhh*~F+tdP8M|p7Il2W8eV2}9a&d>%1U6GKQ3HzF3OLSwQ7_$w*^>A#d@&>=Zl^4sW-*P(SO?^3pH>kg0++Vp$brH##U@m zyrcOK%7RsfLrfzjR^%H_%{*{Y1p;_8AjE4m_o23Nh0CGoZ8RFezBJg;GgSPNWI-fr z)qgC~m8__>Z0b5Q=kK$FQnObss>6I<6B{pS1Yl+>;yx8WBduW8EEEl7A*}?%(qai@ z`uMuMey)?I-?NNFTT!}D$KbwVb|JcDXO-XN9~67XcLdd`s)v)@a-Zqe3u1Gx&G*$> z`InD>B*3Q99|lw>ts9RuOur$O5kwI`4kiFnMa=}1zM{FPB``~_DGU08i3UOF??ao- zXdv|Yilm0r;M0tIE zP*2}_p2%;@BnHZ!G*+0_r!q~xj(=4D+@6=zG&W{)%AK-+g-Xe?NsSB?%g?hZpZ>pdK!m(Px7a@@e*e)RQCP|0 z+d~N}Pf{cg{-gkaO47HZU%6GH|^1LPDZeBOxPWd0WB>0H4>!t8>uE}XslA`^l&F(~SLX5(HUNuxY*?=s$ zarHg99A>8cubP?uI~-_t z_uT0e@t>a)g~NOKYX8Y86lm7}tp^mB_}S#M>Ax-l6rxW#xpDt}0eb)CW56I8QU@)m ztw5P+Le!&j`#;w*kgswE5LW>IS5WIw;L*aVT2n{Gnl9xu8*-Ec>61GO zgZ#9ZtguwC?Oul@*6L%|L^5&Alpm8(c)++4Nn|(O;1}%J7*=$)Go7HA-#u~oe0*ZV zJ4qqtvaVXTi7T!o44Mi)-5l0ORgkb5hMlYi%_14-KBis#TGt69SUsX)g=bfM-mcwa z^o`<|VdYfILCe;#+1R*9=9rmzr}h+o zoQF!&Mn;~6UNKnv$HzNZEb``FGa%Um7cf^7-QDPQ(0+ZMN;*D|rquvkvL`Pyl|A!In~ri*jOO z0~0m;HR_ACUff`_40pmGXgR5_Zr3@Jy;{i0cKO)5Q84VH_FAusad8*qJaoaWrq5nx zHg9CSs@y#wO~Zf58_H{=Rmh*VD+peXeK$csKTkg-!@rZ_C-}=3p;VIf{T zQWqu#Lr@kBKx?G!XIof(?X*nY+5ZWt`kHpi-lCpfi%cSIe|kEp(dU9U{ZvApeX=Lp zmDP3{HB#z5uUQ&H=L=_ll*Y4avRGxTSr-o$7qRW}=5xiRp90P+{Ld=2ck>n1S$!Id zU=wWCaKGxV-lu;lnFoi?=+t-eo#YM;4r$@YVwaI;&97Xtt z!{e^q&^)9yqJ4dwbJ*sog3v}j^)1j9r-&#Fg?@V;2WP?V6GU`!!Z?j5HeR1;0ji?| z?!M61F`ZxLhX_3ERCiE$>dD+yvBtf)Gr^DcEli;D<>((Tn)hQw*crA%!5LdItO4)S zW@ENn*>Rbe071$!p#$ZU-wIuA8_oYte0)AQaDCPRdcw0NIhOcrXLOvdd;uQ#g;$g{ z>F%*c8KDH9Yv}0gBbBSc+g5kS^5C4gQi6|r6jn`)smdVp+{pKWi&; zRLqmd>Y7ZtEN>^Rk6u^J&qs8>B`mMSuSbpKO5QN-wh`EA&Ao4wPSge~ohMWPuCp*- z*BJ!fGu)nZyyoA}rp^~>)eNVIfdQkPbIe-SP2x#?*LY6x>=bn@C`Zoqx2(2TWPW8Z zn@5&s>xCWQt7W9sGUa1%zNe8ZF%yNC$SB3Xa~5u$AqwTa;@MAx%_?&a>7p5|%&G0L!K!+IQ-WtCx8_PoZh`)#y>VT5q%_%aqY>fx@ zE8rea4#?gtg17lc^Ny@i{yxVXb?i5CWbV@#4T_$VpQRpp%6cl3K`v&W z$4*y`OWR+bH2OlSSU_?kS0y?|YTP>GR@IH4ZN}lLgQ)^j}37b?XjWB!b$y&V|AP(j|)0mO_y6HyIXcgZho1`$?H(qTZzK5oD|8koZ0y*n(XZd zs~p%zlMECWj1`Er`Z_9Jj!KZtAHbccoU6lw60`6EM#=j05}E?O_0 ziuwg5(qWn_XC+r{^{L_IKfu(kvrLTX5EV?|&mb4@aw!>fY zt$Mh{Dp_8XD|vLN_i)~UgDZmIY*o0hC_F)wIob=FrG)=Q!UZ-ogHT#c;i}oU*U`rvS#0=K=hWRIh zZ7OZ#sFA9nZ%ifx#6qS#KKfEeMv+07%6u;|5nqC;#7!n0H6(E(#K>M@0Uhv4L0vkf zMEjR(5%Wa36pWx|bLj6gonZ!_B0lUhQ$5q3Zx~=Zc~*<2Gb>L$hBK%q0%^F?4#6xh z=OMuW6|#bSXc6pE>YOC%fuVLFY?Fk7x2WuYi|wSOGmL|Y*_k!~w~Ss}p+o5GVN4B~ zG7R*8o0;V~n;*+YJrY_s<`-3(?Er!;ah%RvQlLi_#;;BaZQ2{fJ#+$k-)MgV+(OS_ zuIoWYVV_)`3--v;;c%C9T$_fIH#TB-&qecg!JI!{@f>_Kl_h7iRO#g zI+f7XDa_w%*=tluHkRTWmSf#hcA_%C5KDSREG9-)u-Y>yx%Uhi#{f}pMH zU87mt-o>iTi|d5PztbFP8cn&KR0P9*q3)X)8q2vU})TMopZ4c@VSd55Ahp+8BSrRzx|TkdLhf@XR=TbO!No>n&W z2vz}JEl5XzB~A-|GTmHy0&GuLNU-U|Wq;`EK68s;I=Fmp-9=nb!HyfRDy^b;^kUbB zmu^|G)_eK*OY^z7Q(a3gDY7!y{_aPR%MwJ`*W}Y+85`eFh!9QbJ*rCUt2x5k)2F%@ zc8ZOy7rUhry8`m8fabngGk@{pfFVD51>4-1reDDQN(fOVfp=5j(Wi&|J3Ke_jNLa@ zDR*v{1fcRkjff2c(-Y6+eTy69Wo{fJ8$t(W6?-49rdc#wUS}At5l-x?V__dF;sg9CAH_+h27zj-aN7EbHh0dLCnwv7 z5msrbJ}!Jz+QHY~kD0?J@|QX6bans?Upv{(H7_cOAYYrrH8UGe=|dE%YE4B?8?-&g zSigJ`IwZC?7gcX~h!!iRb|_ZBV8ygcA}i^B$(wAw&?H~lk+*UnNLoG_@AzFp>S;gH zV4>jrz>2UZN?lmhE$Hn5Q9v^rah4LZh{Mq?vz_oVNS*m2nJAwukvRF46X*P&T>^R* z>DX54%H8VB9dbw<_Xp27hAvX3@!jb%HoN}dB+bjL#$p2{Bs%V!BJUrozXb%IW3Svy zZ+E5@CqGW7KhR$RALkLby8Q@hLH#ewjOXp)p9Gc9KQ0L{Z~udG(B0^5#1<2zu$pbR z2Q}y8WifwHjwZ3C(x@3|zY-ZLy8H)4TxPq==W9%|~1Gg2+G~16z_TgTr%4x|A9mEi9R_4i}gb>k1 zeNAj=AW;%rsEX|s>Gg!Zbm16-=@Riemh%pAY*+ZmPa!C8Kt2G?ztHBq(|!s^ARRtK z8hG{2^QZ$dmIV1w6sCSxC>^mmqgghY%qE)hB0aupe#D@R{6{$BetS|#1vP>*e@BPZ zpvFY#fKuo(@H*hU%}1_2Mzl%-85O#UgsB02TmH7D{bb8?FVT%BuRYSvR!ziV`(Dy% z^f#=ac;X|=Y&x!RoLJ+N(k)d_tXftkBS#umY96risp*`$+id_A& z&sK(WNf#<-5)}V1kgP2q{n~)}(~Vk`cHVYw{WV7N$s27V2HIQIel8~6(t%v15`fni z|N4l5EF5U0NgNp|WzwvC1alW;syc0Prb<04*!O8ytLq zh4ErIlmnI^y~>!J0?HxCh%vHyKqkGmyg_WLMQq;hW8}I~*K0 z8P;Xo@CUKy%b1`?D1En@U!leiIPcs^_N#9|L)(UEI?xOO7Y|!a72&KawBfun;n46j za747WP@k=kvI=47l8&tVEzjG@mzYRwXZ6UKpmj3iRo^vV1{%x0MJ(eI;5YYH&|Jp&fqOut4$`D`Z&897!J z;?MNiYwlWf^aej>#Jo6fus)fuf8HJRkN_Ooy}OT6zmEUafhhNq(|dX#j(=#Tw45(T zV-)!TKK_svZVzcHS5aAXCpI56u(P)bv{HaJV$eAY#0IS+Yt%>j?K`@1L%MdV-D~l{ zd4y;u9=oCh;Wq2YVl>CY)&=r6Dux$4{@k!AXzzQRdwFNQSgo})2`8}dcqghH=}wP` z_i`QnqGV8wEfY5(nNSl4nty4eJ|ul5Mu)IB=9xf34-_j%tZP($J!|am#l8C2D{-aM zy}4IX(o))2-cA;eI5&?6;>)*coW>RRtRMq;BYRVdP+ma#z8wpM5m}pAHDVBv!0$Ob z121rZ;xZYTw6znmgQa1hBqTa3q(Wh3>41gQieEn&yz^plJF>si`wt?M_r^U^c9I2i z%{A!VuCRPP6z7+d#h;JzL_>v%luz$b$B>0L;S}$d)T^i0nl}EY`2T@j>Ezfj!%*0; z)teuEU;kGjG-bn)cGq17VAI{yQ&n$ngYH!mBmPdRmdG%){f@_PPFP}7lGD$=o(-AxJlZ;Hb3-`L}SBNNjVldJnjzd=A? z*1u7z7fg~=-}}qi@tAhdBT92q&7vYzbSHUGNfZ7DvV~H$Y`NV(XPGm@vTjq~ql(U7 z+y^+Aixipld<%4c5$t3UkWb?$J5$wueDzu3Gof9YiCF-ejbN5TWLMaecFP#8IaoSOT!GsgmQbLSMS^FvCaN%8^*kQa(N~|`A&o7a{B%?a$<#(!`+hdKdJO`4kDo1x_!QIW(UC6*wi|mu3m)roE&Ru#dagt?l8VeGSy= z*7=e{OByQE-+G5)ZVPHoA(bcWXvJ7r`YHJ2G(}Tk_Mf>L-~1Cvzdx?qs*N+PNTuY? zHHEH+sn5##-+}i;IKNKh9Z*{soPjm|eHR0YI``7zrxDnWYA9KtcDLXDlH{77X~r&k z|NUHA>)#S3VNqgJbvG>(sI=zOyX5C39CPX1F~TDXi#vC*N;}G=d0Cn zv}E0$lGwkiqzctB<5BOkhFnyA2ASVq*gKV^yCL`l9Yd3;24jDH!Igey_kW9m$OK`{ zBm^f3jS81#c5E@93F5}&BP6I{m#(mU5*^)*0)>qQWcw)9O=g?=a;KAaQP9iBdzwnL zIZ?M{p+H^Cw7QCXx&naZ6*8IUL z$)h&>QAfXV)0-w{{&gp>gx>u-E{3=Qr8TmWnRFAxovXr^adc5`$A3q(12feXj@*%`e?dPP_yDeM_2NGb~+{MAfUay z{YuAVv$2Z(LSKDC1X7^qmSQ(=jL@;z9<}$=*_72xn`Oh3cVrrRJC4O?+LIJcA>DTj zrUvRWhQ8}mfJ#XkGReZubQg} zY5WJJ3MzJWdr@hsCEwQZNHjKkwp!4>{$@gSU$ySpe%QeDxPcQ`vgpiKc4xNz$(hsD zY#~KAY!nQAWj~bYX4T89Vhq)@jj>JA;c5ObwMut99K0ucZC-y8q?;P zk;c`GOo@~tmq44@W<{;%MS`WZ+s$G5nnuo=3gCrXXyp9G$n;WG%gTGcnAQtVdTv|M z7n?aHrR`)xy8xXM9g_{u!n=Z}`T5x@S24MI-~qAe_+2o*+3g?hbVrdf^r$F0y&g+nP2~BlZy%nb@o-&vv7rD_Un>JRO>YSR7!m1> z;SKC{D+)4=^KUl^U9*;fbB@!;9S&IYKg0DuJ}wB$@TW|v9yzoMi?+6P^eH0jbdlfX z`SRf4aA7Sb|F$KAYk3#<>&WV>DffiAc5MyBW&6um3!dH7+}x~`grfHF!tcKPQ7hO{ zlGu?SB2#&bY=B8tB}S8GYw-aZ@8nb)Av9}aAOuGox%MlIN)Y*AILPWmiFZ#NOqARB zYHXFYt0I}yw%7<9?d$mrILYxP>y7_WoiV*m=)r+x5`x)lMB&59`aYhcyhR_@<08?L zvE9-EmP{aef78+iPPUmpvha|YWOCkE=uIqt$ZMIIPNNEO06MtDkl`s?K2zkj5MaAcq zLG~`(r?JDa8~aCV)H!Pk+j{%Q8yoqQpvE_#a@VkA2Jer-Kw$Cc5?CEX!LAdv)#8*I zm4r_GI6 zos%A3;*U;08LL-0QAum99}`hRC`_@~zu{0gjydUEf_XW$GpI}6xxxNoCJ@#hc3H83 zNBe5Y3ek?GJNu&4LJ=DZMJfK;$lo8JJN0zjahF}l7=>S93rvKExv?|KSrPaOMW|9) z&ihei+h>W&w}-xq)i2ruYiz`F<=1lv++vGJg~%KGxsO7o8!D^&Fg2-7W)R!ztYChi4^^L-Z7 z=_kkzD6Zej}b0VDP9EUNG1Nuc$>OSc~ReWvGu7sE5|m-{3nBVvu89M$E-YOhGi zr6coJG?jep54$Liw-!x*Ii|vW<3E1)Y`m~t+ORWz2(B>1!yWY!YMtY*{gt)4PS~AU zw@9J<+ak$|S=~~pyqu)V3CmPhUPChG;gljj(Q-#QZ4Atg#p_u`h) z<-71Wzp7ibwu6IH=FOJ>*+rZtPu@AaL0i*hZ6lyk^@Uh?nn|Nrsm4Y0S#HE0wAo=U zE%Nd`Kriav(J7xjryld=fP6qStglsKt^5#KTi%z*_4js}8+3_^Ir zwX;;n;Z-#%9nwTo!+E2N}n z-a<;0%CK=;5Ix$OMTV!EpIQ2MI**=2ST{eYkBWX~T=@G_tlM#vAf3$I7{kJx6~EyT zCA0Y)j7Ee49%H0fq=U1ojO_Mu2D$Sy52=31SYo+Op(?VWUhKKXC$4zKjL31<*k>W@ zui72=z>`sm;Is2ovSNA38S8=V*3Eiep>PQZ>bWXNEsQGhc2ItsP zt@ICZg1eJZS@YuS(2U9s(>ddFIhbdr4aDNLLPTwdDG)QI)iI)j!CK!k0o{FxG00TQ z%EkG`#8eG2Mh+h5l2{oHg_xjEs@O~XDx$~Xt{o=J0^|TjOo|5)z#genIwMJO+H`t3 zP@c}j6tWTS2h7M`V%W68GM3*nM<__49nz>i)tI|1pY`*}Bg>zbkKb7hEg_tK0&lX_ zqA7BOn%7!3cjI%IVWnf#&?R<4R!R_6$sf@XZuBobj7|>TA7t;B1AUfBUe7cxw&>XP z@?6n<8j#4i@=a@Cn?wueu4@YGeV{QwuNCYTL+Un?h=A$w;ysRYpFbqCPuamXRu5cB z{kt|5^fjZ4aw8I5UyQ7`z3L1N4SIQ7U7==o6BjO9QY24-(^Cxo&bwS`?*h3uNNxyT zg=--MGa}UfmM4tn&yzBV>SJco$c(YlkmIuq#(I`O7zzD28RXkKS=M=vyWN(}Ol?GT zfh*@MobKT)ngiLC&z3vx)5gBD75$D3UZ4297#8Ga5VkB$x{NR(huLg-Df9iY47LUe1D22XGypA@JzOmYn zGX%y8V#i|KmWP%fjL3f>YJ?KX42D}}YZOFs;s$Kk3^m*Q7FFStZ5U;HnUKG})ZVSp zi0pn>CmlZ9zr8B~ALf8d@ve&sy1Y*>w_6VX19h>^}V@bN>``QH9nIKXG@PZzJ-zI|On4tczlk zaGJW3qj8EW)jCK5gImunY;?>*N(u8p1=m*QWKTY^!gVRc78 zxtc$DZ}P=8LWrH%#HXDrN&i7v_3h?AxWeHO6MEaY1r7K(fE}w_HC5THxo2`(AJvzy z^eY9|T|$)~Slo|=Z&s`V0~yarhg!Bco=owE)##=9Rm10%Y&OiOWW=tWD5pPc&;6o& z*5;~Fe+JBD8zbn3AeFQ1(>btm&P0@&fAaPKzM3iYf{F(8B$FX63J$ zqF9tn0nHU^{IC5e>gq57-N-Xw3I^@aoli2g>!8$d8VM&9nV^O zPZDw@fE;S7z2?%%9SRfD!Do`3hoQZi!Z!0lcs;IazI+LZ0c=H-*PsZzzmo$vZq5i* z`I%8D@Q5Qi!O-jGV&8C1Q2Sk4Y8(*VCuChkl}-OE?#79+Q8!WY=pbX1nn7mR+lJ(* zF!cTP!T03kvQEO;u|?6`r3|II_IV!((h8*GMNTz=w?E&dPZVq^-gJvAm{;h@`zMx_ zcI6l_q6!a|8~BmnqbMl6kVBL)?Kt*+N&j#BMCSDRo8DtS8XZv{L8JgvsL*kAI9-fP z$)HZO=&vyTI{PaW{geVk8Bo?))49N_jcx4#b!9m=aG*mj%C`J)m#lA(+ny=Io-raK zj7I$t9jWQNrW^{`|FON6GlmrJb5*R@>WWmZTZV{A?hEPerw|0h3OWZVd51_ zfSSp1RMQyx#tuMJn^I{U5*ZO%bNA&hDof6?liT{w-H5^s&ZiwTX~k{_pXcTWQ+w4_ z?RykG#fu}sE~G3Ndw(eC4h2PlB|*$LS50XhUz7dwHvyOVNm?erhZNecQ#71$&uLLG zxSm+t~dg2J|+oLkMT~&*Zo==f~kbuZ{HCX`BNL5cD6nWT`c1( z0wGI&$EZV-4@qh%Yc_}fZ&E3ckug^-=NEYr__tvX2CMG5LUaF}ud~=jn8s)_Hfo1q2 zgHLDw7rJu8pIks&_A%M*Vkby$(8>vph*7{;HuXEbH5ZtTM?XiUW6Z(4H!~9c`;its zVEcF=?9pW?cj>x6P^<7_cV|Ed);qiXfSY?hQPkcniYZ? zj%ikqr7EPLE6CDBmfXQ2$LPih*yLuT2M|$#ou^}Uhm@4D31Xsa4pMXqu5b|ZrQPjD zc6tJH*38(&VjAUQswptpN_mL-=682PL9061jLp37F~@@J5GtqNk3|5V8|{5R)A^Jr zN8Q@i`{g+o;lY;%+n?@EmttSv$j#u9Nl!nhI<8Di@0NFninbR^faTQWz`nkb5)u-2 z@}!D$;k$>`i^aw&8K?L9sl>2wV|F-?UA%f3T??6&YHjdRD3;@Gep3@&ly`R%v)G5L z5B!bxD$#v2>4|%e7q6Xj_1nF!RP?!T@&iYfBo>mKo)|AL9~Y2?a`?R0G6K2Y+CP|r zlQ+GRnE#_`InP#ZtyF_(dRR3pIpkZ1Hqj|}T@VsqDu(W$S$xhmem&|@XNaxIL|4Mh zKt53@Ab6AW)bh7oBLI4*Q&Qi zhLNad<>awNT!dcJeV-(n%$O!3#zM9rRMf`q2EXD~RK{xF)&8ciSC&%`#_etNEH89v zGO3PUMp9_u(~A3Zr_(A2=teg3^V&YIOTS$N4A|DVm~3VnXM2{VZf1Y+^<22u6P4lo z>8|SfcbN4C4n;=K=YF~9_rjXZX4(xzt}|FP8nyY04Bg6@8%V+cPbubZkC25E3tSJP zHL@8jsU5|iI$FO>GN5D)OxrnWnw~NFa)HaK4b>xuoopD>t_tj7C$b}uZTyy=31D=x z`BOyGjOIYahfE#a)e8gU$n+)#r?2mV=qg1=?VcOi-Dd|$kc5D~XLMh-SG=n&Rk8kb zM&;7)=9x8~5oIDOh$~}kbs$~-8ir=*U!_(k>BL|V&ky@G%gCqbN#<}+Bl~4tprSm+ z-Z;G}W2t@Yl@W2ks?lfus+(wO6V-#G{60*lR&lTAzI4`Z&sS_O6*Zan1rFZ;iG17|h6P^~3V)YANEz+to8aKQ6BRtv74sE}h~ z7ltTVOAG5S6|nzSWa}!Qiwb6(UGoy*RPb%T0<~{&BmYL#muzPXEY>rbn`562-uGE0 zwTutV8TYvJfE<5vtdo7NxiUik2Zi|Z(Cqop=W&7I!}jYuCO{aAXe-f^qRvS>IGB=o zgw=5Igvr6@&|axb#`-5+`lzIoAOM`fhLd{Iz9ll2X8NJu+ajH9`({03Mg6G75L;&E zBvxijPdmN(D-bbl)w-wzXFVceXOkILa%nFAmP}uM$ff%Z$orG}U}jl?k@3V6oHP(G z6=(j^;C3r0VavyT@9*>7PYQ{Tyyosu-w);TguDkyo?aoe@|Zov)^5BbO_Za^S5wbz z74J~tMW+y8roINZg;cE{WYjUiq=egAgpguBF6aWa*V?tkFy|rSF-)jO=kF zGw^0P@v>ugCzE&vV$@b3s5hC@T7x@8) zz+T5mEN|F0G43pS80v&A8pI()Q+`Mr`Y)qh}k-y35-z<2czO0N;RB!)`UF%7|_i4XQr0XC3ryI zpd36VG*6=O**ReRifW{&2YEKTYkYrPTKImEv&zej>BEg`+V!tiZQx5Qh*pVWnr3tmJ6 zHP}$u@hAR5kFGi=C`b#{5V^%C^Nm~%*5q#fsw`pWQ$(|q0}?Vqqt7YLV&SQ zfxDtzv;0G!CyP_arUfE~?cuv>y69p>xsh0!_#aEvx~i?>>Mm#a zF<{EYwG?(Z(E3dS*SBvGw8%#&Et8$E5*NK#f@Tt%@yTW~mDpFIUBQtk@W!*M+#^-0VmX~WR&?24w@l6i~)gHq8h-vy*ClW;6Cfn^K@@SP%i z+U7XBe^gdLHCK(d{OCWutG_sk^jq(&r`$lKCxp)TY74S~=1^a-gCuKPNfXewS!f*(K-YP6x8FU^c_eK^%K1L3(j}fB&<;D zfxi<5hj2<47zl#8ZdN9TmvPfpX?T*nitLJ6GCo6vX$~^X4TgWaH{bRW_e$U#WUH#S z!WHQl5uAu!f;}h*rIDPj$5WI@8#@~0B&yjMQbvAIwxL~3-jrol(SqB@0q^Ki-1z=0 zX48$n_G*yP2D($o5Mo;Tg~y}XzQi#9N<8cuS!Ad4Vk1{im2psS*G3DLrexL61TUlZ zz01&|@oxE1(=&zEC7o%QooAwOSjUWC3saA4;H|<^MMgGaHi2%yL)iuXVh3RVSA;VY z6CDPMYC*-zyJbB|qsdr(m4Yzm2QI#b5Y;c)TX^e>eVMvti^Z0|kz9|EYF7O9*Jcc2 zz-`roV{%tX!@SYhAgu^O4q% zRiS**cKqR&LE??{BDvXmCMJwixTy^57aUk?Wo5g4(;hO8hy=A zyAi+P+Gd0Eu5iQbsGb!s4egar&P%F!#5TCCeva+gRy@~!sC6}eS8LNA{(o9fm{tJ32b8=hbfX`>f& z%?XVO-YW)HMwSRe6fQ#7>?=c@EIq-=&r7u-8&;`EY(xuQnN*d49>ChtjWv;k;F$Bg zC#>G)QyieAH_;*Zli1pylaH(QpWo?5qx(6iDS{hnEv%0m7tQoP;jJfgD=Nte;JLQs zmZ0W6>5ZP12FGB~T7Q#Ushai*$1LH9lS3=iA9%<@r$#(glw?OoIfL^+fSh_Z|c!OfcjbAb&_%D3P8c z9j}s&y+)C=_+Y6-=9=$beny*`A@afV8Y z)%An#Vs^8o+@b>10qOoc{|{Af9T(OAMSBmWlG5Eh2q+!W&Co-)fOJbYD2>!ObV)P7 z(2az2qx6u{($XNHchv9ix%c@CCeJw>Ugv!F-fO*=5fs=kSU2X`KrKU7B}c*Es$~C- zG~#;%nuhVdaZ}(g+$+m$a>l2W>wQ%8@r-JZ)@{8E5BKWO~XV!4^nYw zfJSv3R3fk!RDUDp&05M z-=_DY?{@V|D0BShR>0Y!vyr@bx10JhNuI9vdeD9Io!si?pPbb+{AKV4gti8JqqvN8 zZn=a`zWP-3?KJdiroyDSRio<8REk>LzHp+#963-wctRUjOiOe9{sVCnxZ|AL{dvm3 zwF#@J;Hvk_qhU*x+AT#lvEDL6=c^Tsfiz=nkTfVsYO091j8KAmBkw?8)@=N0aMG)w z5ez3ecUic4D)SIqXY#2Z9Kth$Z}x%F1Itn;)0oeJui&vM-V;Yzx=-cU-cny#_DFOT zBEto$s6H#X8}3q;k25&c|14bFjhXm5_>d3A)mc)IHGk8)GcgVH=nJsQVrrC%Q67;Y z_N~@@#mkIF`&t=TrlQfQ18&kwEx57EysOObYdSAi)r*E2<9-YP?aY*@Qr20axoCrXpBM)oZoDdyG}QdPx~8Hhq(8R;dXXK!fL%3Gu)vU(!)2#dS; zRDub`FZ)M-WHr8};8lsp5p2_$i*Q4!K_j%cfG4qecF~{S1}>ZwHjoHs>sq?JzIbV=a)S(7d=h(# zkL|uU03ZjvTe=KNT-$!AA<#9STX797=-JpzaCUTdTuT(cz1>{MWm~i)P`uR0*{A_* zI{Aue^@A5oj#!Z!AnLq%U1u zyqmVDT-1L9g53!Sq^OPsx9&gvGS73;gMWor#OEZV7=cmPT)Y9RYrDiVg}R?^Ld%MX z-bfNfhn6aAESvr6k{z$**dGk@10*Hk4&DdW;^S*#p50pN{_Rc}0GxR6@#Xt#G!*dL z1P=hHYq>XoYykM<6Xvr=gNgI=ge=coDuSfINdWXS-}@#y6I>9Yd{XN#Km65NEuEO0 z4$&WJ(4XD` zXOsV&a6Et79mqGM{5{A=sD|SvLC6PxdSQ>AG5zWBF_S0#jr*1Y=*G?&%Xi;UozkrC z>?v#8ouR{KDm-3wZS&d`>FwnnsI&3%z8A^{QTAiN?C`Oc6+h@}R%zkWMT+cC$S%>GBOrk8XOv9 z9Q+yi_*36I2-79dF0P=d8%_8{rY;*c3Ujf7hm4?xAVU1`4gU~S z%C;J-ki?PEyZO)qOv_YYb*8_|Lhb$YYdFS=R+M%&I0t|zcu*n710W_Zr~7Y72G+c^sD z5`LYM+L) z8|vfeH1fZWproNv( zDOwESI1t0eR6J*r^)I2(nT9=--LL*a(ti)It>-Fd3)-%5Qxn-Zw}mmZUzs`YPB93AGR9bcUfcG!r{KQl9e*C36Nfvvjg3@euMM3$DVE+RR z^AxYfy?x`5v%>P>2X5G&u0p~1&IE_Q$=j9KxyGAvX{|^5+*~egK1Qp?30ytHx<69a}zth$jrw-#o7QlGSSYq`wRT5!xLE-;qD?g^5f#%EDV1g z0aW-%Y$|Ffi)aJA0px=xs$Ra2%po2Z@e1br#LY2uJFl>DkNdrSOv zlP2Nn#=a&vR1m=+FNa>TZPyVEGB%{gG2+_OGw`p;DP}ySVDlHxw4GpC+AZ_U75iBj zE|^9G9+A<3vZHf`$19(D6|Jtb*Jvu$L5VlC90NBMV!tc;_$>t^JHoNx{_*$vvqM*> zJI70lQORrB4OK-yneydXtOh`+pKmA)u}xQStJn1!Uw-^ZNC_f@uJRS;lANBf=L+`L zH+>Ed98vTM+$XP_>=}AvbbL$^9D`j-Vb7o|lDy(ozac!L4|6b!U^9vxs>PDE2)U-# z1VeE{o0M0^XIH$I1R|$}(#?5Hzd8z3K7doAmaS(z<(Xw#z-^&*X(iyAwxH zd*F9oBOtw4IYko3g+dj~DU4=7zVVSIYW2dAhq%Z(=sL?q;+JcJ;NhF6Ov{~uhIb6( z9rMS%x_Qoc1uELPM_lPjEK;U}w3-X&#j5X`U4(?hES3Fz9q*hhCg)C}R?=Iet@`q$ z2b=a2@|Q{dv(Fv4i^hybGva&82#Rs{l+MYp1l2mJNT_HIl|)OY>`xn)E>rosG(V|; z1Xgs(TSRN-gU;7O2yDg=c0!VL$TDr;x=4`ZGwy73hZ8y9v@}MvZwL@Y5uKC86dUl} zP>3QiMd2d5ra9XCSJp`>+}^Lt+Cn2RSUgCxB3MT-*cn(jlvu~UelV@XTpGvBpjHXDHr>?wqj_xAqP?v9HK zy-nJ!UA%Im;gy%dj(z=`b9%Pq4~NF?k|~kL#_a@5L>g4aHl`pYGMiNL|R!Vi!_e_ z>MCh`+{SdQX#O+Gk-A^NflaTmv(z(%#t|GSQvKa!v&yolRnL z7tI2JE>-OlwqEB^LfES*N7ge7(v%0hxl>XMeAX;d4s1#)Rh?59VQW3iCQ05XFO(ZJ z)Seo1{ygC^7LnywHP)PfpNf+bM#64E>#25T%Z67Rfg+IC%CDEeI=cyt_GC)jC34pG z%5(h9>#SQ;5a{ambJxmWkJ_z<9ol{C?WMK?_U27-(?2yBTe+`4r|7d;?mD{F1TFGq zL#Mi81=7p`WYDi(7MM2j4XKs*FyN0NnG|FMWzme$rn%4(FhV6(Yfm8DVw1v)yv>5o*x z9#nyQ^-tV})8jen0*}d>Soqe7gdEHWcZ_St07!{GMCb198whBD#VUrk= zb9AL0vA1R7ef=eKKLa|pSu947{0dIV>LH|TS_+c@Jx7uJ?-SRDyr(wW0u?rl(sQ5-W%1~~&C!{lumZ`^a6rC1j}^4XjH-C>a=`W>4ukvaFZx z3vf^W)UX-tF9fly>@(jgsPl!9exR`?@_PT#n)Y5q_- zu{5k>4?bp#vL-a~`}oTE)WN1Cx;j-%Io98!T_mkhJK zGD@IW7bq~AFn&58y07z84#Am`L9wizOmRgp`K0HOH~#eDovJ9^jPJ7+25AnpOSD%~ zkCxr*!t{fTDwgQL$Mt@FO#|x}cxZ0-s!iWCcu*hgL(sfEkECm*Vj*T-jVH^lI+u^ycZKnfFw`h zv>TPQn!)vXPn~U4;_^Qa&v*@mN9G!&+ydvtVo9(u#ZS(a9rHuCrzny+KSwC(ENCVvx5p)v*wg6;C# zAVI=;V&~+$DM=UphW)r^H3=z$+g3hbA_mpUcuYYnsaDM}dHG=MaZv!7QDYZ}RjO_` z7FLiH>o1ZlEfPw8je)PJAz4h#LxtX*w+iL)cH?-9+O2;?<3YOfUxEng4s;Pefc!qp z9ju-M8~G7M!g|y`%Gb`Av}fVWT&}_!R&aX4B?pjBK5?yzwU@myg-YFMi=x3`hlHTJ zhreIoL&A;J*inWv?Og!n#ZMBH7v<_x3-1($8wU2DD;w)eSwug1F}2&4x{_Mf zFQa|nH8=Vh)1JlA)xk?e$lKq_^@D{|lLAsgC;T*bHKs`mykt9xv(B=yPWk)uFF6ZtV8{RK-0gqG1CR zdwX)J8JW_1?T>cPCbh#9Hl}iOtb$Y6M?PPAVaUI1f8nF7HIbmUkW8Db;HfkgD_2Zv zu{FXIS3k`D(MFLFxR`~H+xe@dmrI}YH{iKs*^-TdM-GH%2ujpS<>%AS^KH8u-8{hg z`tE4&!Hu9Q+m|jW8$CqH`} zc18Y3?0L(y8T_Fll$b4 zJoM?L{^fJ`^9&W8?*5Xb3K}|ZVn%(-sdUzj{t6|QL(M>)2f24Dy+}cdrNAHK3g9yk zUK7g0x!Otlk=*9;$0eXoC$lEo^8X&pXP|)LWb~td#1@=??@Nn9I|Ka7*hG1&ZEW$s zB`9)dhDz&tJn7dGvqfH%biVycl+3`YSQ5i^7Pk4nb0m3R&$=h925Dbzy=GsPN(Bl0IVLTK|C!r>0l+BDZYgU_I&j$Ro$ zJC-mF85y%l%8Vw<;U>HKN=dKyL27X}zAofsreN`lPP^Vj?xuR&uwC_wFa@o!_TE0yPcTO^qx3p|zAz%3fVir<#!RHL>6* z40nTywLBA!lc3$hwjQ(OyKEcM>~3nOrH4_0U0VAX z*o1xT8zBJ%sZ=%fssVQmqRD_?*K58g#-DI5)q-Ynqxi%N&Qg7~%ME!fk*eRm+hd^j z`;@0}=wVVK8k2U(%rmu`Mx_sg9Om9~-c(XatyoE8k+T}0jO)lX{_o&*cGPAXPO|JF z7F}&E#NJ-#JTk)KwyME>_23)-+>EyERqO6`WiQdKzIo=^;eo-?APJpATjgTkt?uHd z76%x8J5Rbu^gNR)9s{P#J@b@9L-Du_s8NW$=0HWuF1=`vSO)>*h#a123i4yehJGmr z`QsGdbI18ySE&|Ah=yq8j@j^!s0Y9vL+`+paRz0>MZmY46O7cp=ZreL+e)?|UDns- z#S*c72IDLo=>>s=4BQXXvd*1bOOCrk1))`zg^5%*q?i_1Rf%LkLp-&SV?)U+cNOg6dc10!FR{BT&5 zWjY@%v12ONy;Vk$q)lP3WG_l#=EAVxkgW^!upHd@30mcAqa0KkHfhn7n1-dbrqTvC zfk+*K9kNJ?ssTmJM6ZkRQ55e&UqXnvCO&vVlX4^|1=3<+j^v6+*84Lan*F#5{#PSn z&PV46Yz6wV-KXflrZkHl&cYhhZW1?V3%4g~v+i+T{NJ;i&a2xA(!U~Lx?7rCiI@ix z>y~3hD|fGmK!>OzC#4~v#kCjrDOShh{y!*-XF2j($NtQsqBb`B3W zytZB1g6n%-jJ@>O0M7|McdS~H-T@M7{ImKDzFFV^fm;J#O@w2~~HP3=v;A76tFHY^@a>Yi?^l>H&A7PGkM3GxY-2ec2i;7TWlZtga-a#ZyBj?6( zp4;f0p8tCl-F*zOdOdH|12jqZ-itrkMF|rqxH`{Fi1(SXTTi7;o8?Cx`N90lNRlRP zLHL(z${#xuXryy(g=zK{4jBf~DG2>p1`OK|ZC)fK&jOA$mi&N)^}_mg2jxRoH%2eu zu+pfAkmkO1TAuFHqyZ91yRix!Ag3~~jsO5GL+D()@;Xoq0(c2=4K!>BM%qencbz}X zqjn0X*)?kS%F8=v?g+{}p157*c~Dt&U_uy(ukK?#{1KGAqyQLUJUR2N1f7q_8LY+p9irYM2hN{CQyviE{dpe0VVg98*NRw z1xGchQ@Bb~^LCt{Du6aIa8KYf%m(|K$mVmobRA>Na?sr2&J^W1!;vFnw)U4|+}p+u zeUN|W3j0quGO_O{2A6LL&gacitVkN>mzadGqeD)@9mq}Q+n zLUYMd%s;DfQ4aciB?Xo;l3?nL$7Nmo*?1Dzklz?WFm$+#;t2q&$x&_c-=((x8Z9Pj zqV|$=fx0%0V60~pK)n3jo$pzZG2y}5@Pk|F^!xx@>ubo{0jG@9$DTJ0i)T(ubxS8n zs`Nzby!)gD@W_e~!3A3wR_$8#lfn2TDk4dT49bPBdA?tyDNDBoX?e5*l1*>|`CwWP znYWnObF=c4p2RFhaYsT$Z#Pgn8d9(iUEeN&=??QZm-Ub4IQXvel1r*Drz?@|1b?W; zjR{kWiBo{hbf%%R;Y4c0M-~@`Tim9SDKIGnIlg2EL9iOxvBM!!s5eBLr`ddZ8^N;l zgNuTS6~6&HqMTisKk9Kkmbri0HkpWs2I35+*_um(jA6YTX2>nG@aoiod;qAPkbKYO z4dq~eWqFs5j_lrGG~ts}3hvW0grYpcb}^xuF-;Cr>1P10>j7R0;#Mwh@$CJcLX{MMUss zfcT*oiE+pkZuA1GyXQobPsTx%Xlu6_%U)gQqgqZ*F>Zw2FtoF6o8l*$R|y3;>P(`P zl6A?BMfwTrRqZnFZ3}BbZq6*Z;I}sj)U;QXnbO;Nss%y2c&;#v9P8+nR*RCu5MXL z?#$q)W3$uNE%4*RqZXsaVM>%e-Q_H*R&%~^?7BV($smeSRphdMAdFr5f!tPS$KCvUotV<(tvd)3Z4= z^Q-2?sVE1-OsNlwS3}X#`qW4a2nrgSlLfF7ttW-LYVkuL)g233-+VOE`z*7z4B3kg zu%pKc1D*J|*_IgIJd^B)@xomgp0RD*go$=O1ETJOzkr5MC0K`Bu+W!HS(7H1JFq+T z5s7xS3Mvhr_K)}&JIO!@sf1xE%lGpZPA$bW!yKK9$SE_sg8hsDqZPrs zida({H7M~E#WO!941QnB@$!X9w4CLYi(QGYyt

%Ma(2V~&T zN$%=JV((iq`NKc-L6sm7wFOegd75R>ozvYARHR%a>r3c1Rtmn7l15LW&6cSZHghee^OE13>!DeM%OH z`V|T5R>wBO&)mNercbAqLi7q5Q~n%6w~r!6={}6BM=fXVGtZ8{nQIhVi=4SQ%Vc+I6Ny(gcOE!-R8jIaZST51jR{io5#)2z(C)ZZ(p5r1!sSPGiOgNV;R@G6D zr*g*QVXyxl_n<8`n#dxhiD`vw)E)E!DF1-jT3U!T>SzjyL}x+PG8PjoG?{yg)^Tvd z^zyk9m>^V8;N6VQp$$oQtF$K!4h`iHgwWab5ZlZrc^2_EZ?Ddr5-6rU%>~K zxvy;YH6mkl_jyI0DJCmrizXMxTSw094C)1&!z?j|Z_bK9bi5D)en2Bv`{U0FpEDnk z$<%k&vh4`FKJgT<%V`SsK74BIr`SOcm!ZgvNm8q&#b5l$KEfEa`RH9K%GuFYnX75I z)*m?~(L(h}-Kje`I6VcZnNdV~&5d>PGi3|g#ax~Gd(z}v%r|uU*cSbteK(YW{Rh9! zLzTf4%NN0ADNj^~eDl~{)>k;2?-5>uLS9i5DSXb^Muqidtk>nW#^K(AyY;tkR6&>H z3c;BSZ#+KvIqoEL@r>$g%<7XMvrUq< z28`B@M=paS(xiMSB%-bVbgFn*im_gMaLTi>1UPQn7!Z=e{X(NiDsM2QDdQ@8rutZB zOg(ZQTkZS!Zcb!FOD~$k z#2d~T>)Y*$%mm;ru9vow`d3PQ2J^9tJj{?+=^!ypcm?l7knJy-HOf49KTtB!9o_PN2V3-1=#-J=wdrRgI>A0%#qK$bd9~i--cJ2y6q& z_^6R)-KY967bzbn6sDBP)%U(R4=R0esR!~D#)ww+Cm611+sGKs=63a$qXW^s8+#0r zkst2Q$rB>YlI7?_8&~>4Wt2&&BZ^CVB%N%fPA+~pvl#bS_&H&eR>5-tV3_=>Xt=|b zMya3)E>;-L0aC0nN;`193UBww#^^x2&5xe=@f}VZwx>@&?M49qcq_b^KzENi{VD4L z#x8hiypm~%92||is`KkMJaTjxcS!8cYq;_jgf9wl9@S>h^r7Di!wW z;@s%>8HoRja;NzI+bUH1w%(-3Q_iPGL06T(0F+BW5bz-wdm;cDF#j8UUJE_GEOb6? zvDI)(_&4_x;_DN0PfnT^TQ&do_H66b%l5CF7u~5n?*xpjaGre{xHt;CPMmcndw&Gb z_W)W5Q8|7p(b627=W8_21Q_k;#W-m^pGZtZ$=fP%j?@%5EwHk56hEZOG^%@;3Kvbb zZwdL;HoT;!IB{}8_7`QR2`w`Z=lN}cF@_>yJU))z=;LUiWM^#Tyt=cHJW6=P+x=p# zwFuVL!DCL_C34h=?Ln2;OGu|@n4x#+=?u>TZ+LOw5MkOe6zat%n1J1NDbvk%Insoo`*F}b{0nB> zxTLq0o1ac{#0pT+@g$~gsbjztkg51OqE5U-p4cEl9DTyQ4^mL1Ue?L;*ZJef@K{d7T>1Zy-Lc?<)&e~&QN zYvo~R=B>fVun#8gJUR<|y`t1ymbY2vtIl2xz5dZCO3;_`s6B$Av8NWFyKXcJ$02X1 z%!LD0&=Enp+Vds>u-acHM!PUBaw#AC6AnR_kb;kQJ4}U#LnVN+xb7X2A6=}++uTGk zN&r!&cQW#T9fG`~DUl6BFhwJ?Bp$q}7NE$ox&q0D3oSaq1h$6N^9~SScBOChx4&w= z4&pq{Sje(FI%;;HW;2`pn$-7hPfc-StvW;xqK8TIB3w!>PgXxvE-(q&{w3gkd<7D| z^}$&g(+t);!^jvATx7W5RLzCpONI9JklIXwVPqYnAo@Mhi#$C>M}~8|@KVg}w_t;Q zn1|cf_mba^X|wmIRuD6b(edO!00%Ru5vNZNZy^J0p@0MOJ5m8TY5z|ah)V4l0Mz6dbR2x z3to_bWSM%50)Tq3l>z3c`q@1!u30ow8+x0ZMIZx~bPLf~jG>*mz+WG*3(#@t@U%lT z+%deEGEB-(kA{nj@nn6m&jeQVdfyNGqW6|t8S@#It{D$@m=xK{GXR8x44G8c2%eMM zHv%;y!iAD@3Q+{PS+8+j8otzND*RZVEb?f*-IzI485E83r*A#7eQf|>uwLvWaw&gC zF|L_dlf`~U^ut-f5G_B>Hl+e3VOw8o@o0|WD@lYy1b5J{vy&DL4-GO?tFh2?kaYG0 zn`Q&coPFH@U)_kGCW3#{#EGjsDs(#iefi{{7~xB-&(O0>J^emc1dNBE#NWIG(%ODlqN4i$O_GfnxLgl*y8f#3VPCi%gYcy`zo@XGX9o-!F)z8BjA^dGg*pg~5J zn&#&oGvqGt!Wg{deOdl`$`cqs7IP5AemI=<{gT2`k4?An5K z2@FY2n`Wl``SK#UK&QptXK3@B$}WYbLyp4ta>}g5@c_N2GgW$9mghviH*KCF62!n&#tl zPsE!ZQQO4Krn0E-6IAg}LrRkx4(#fp%HQRpcNI%4)*2yxbrXb?)qV-K#@)UwD0B0l zAUNFz2^H9{XKpT2+V~x_zCH{2{c`V3JXBpa+7(lb*MK6>e&7=}=xine^t}M6|h@0F;6AlOapagf1EzBOSap;@uFlq-ug| zp)MwN1Zxm(MRgMDO9FyzO)vb&oE;pKz>p}=&)fBamFxL2b31&objAEyjmqF(v6ztf z=OItDOx}gqSE~GE@wu|;e1jfl$0uEG?K;ExI+`wy^5Q(KRzM1~Mb<|L5^?_RhN^(> z&oO>+ZlBZ1lhHKFUeQAn_vf|9-dA?CwE=Yq>mBY6rp$^|d0s_^Ir5LymMu$eR^+$>S#)Cs z;JT)>ER|yBdx$MOkCo1qX4B07B!s;Z`uQ#5f98(b864+y<`%9NeXlFgs&!ZN>5xW| zKSU?k37zCIKP)Il##P|03FkdH`P4e0sX$9_6JxBYYd`7Ua0iDvnKO-%WAfNw156f4 zXQv*PQTNrp)L|F9I`f}T-@lnI~F}=5C)3!FUDor*;Hk6K|b!O*IWm4-(`1Z`_6lwv~1AdjUwF?V5T< z(7~ntuOXNYHXXS6_VxeDivNH2rA(W_(!<&Rk3(r_vW?cT%j%J4wTlO&I}&PMmSN^7l*?L4f$|RTU&UXzG6Bk0seF8$fJT@6aa8?<+=l%;8}yn$@V*fCIu}(VsSw~XN=L;Vt`&6-W|d*ri!Uso6prrwI)wi!ODPjS z{zNI2x(#k3D@-l>67DlToNK7@!w~)~;^)VASx3ZIzFrDBg$`rXj$`X3qbbi7uadydbn{Ew1K>ylCpf9}zkPN5t-xktKOH1*}JwKPu z_afaLsxs)qhFj#8Ua!QDq|y>)xJeXvm_&$7Pn2mp;kOS>eSS}_6YK407P{^B?KH-< zv!hvY*omxD9Z^;PQjOf;R^LJ9klou)mW4L$gFAOj2 zjt@{6>nIG6W^*BmOoF1pNb^5;8cR{T+T4#60^zKX51lZ)cmm8iDF4RVKD}Ah(ngaZ zv|_IF&~q7=P#x#+PH+-hwPJ3tPMcQEukT{19y?UI15~hqkc?!c6gc-~~i^ylYzmd3e zynlb-L>`94Lwi{}U{5EN^8vAdVv$H#POji-CKu7{{B}d+U0_3PQE#yF_w)NS?9?Qm z;m}-Z>CbfZbRZyP-S#3}{oh12xm$XGO#@v~LpdtWuv z%R0&k=N`O4;x8!`i4)ARd6M@auN-pFxLzEHWm+k^RUHJ18Riuaj2vG1@D8k&pP);} zH5K_gov!xGD^3+W1y058@!Y{%&?2hKsr@oK68|TaS^X?6^M1j1ny!K)`Q>toi=o#i zXnC{1oV#7ix8ahh{WdZ#qY$YyXgu#PCF|xE*f!+u?jEr@LsfWb^jUHN zG`Pg#s285D4m{OWWG8|orCBIi1eWC|;o`kE7o=@6!&L(~w#IyV8voombhB8!t81My zPx`Ao*e*)R{W_zRo?D=3hA?tSQ7wI(;9am|iF66xnWv(sMDnW{2^P9*R3pGx_j}1q z$O0;=ilS4)ZJoN_>=(y2dV$?O{zZR7F`_R^ysIE7Ynje?{>Fw?tr;;Z=h4oGNS*Ti z^qb8FBOathcZMoY8h74Cwer;bi+N`0;#z_4RdY_ceN|7hRB1Zf)>x&<9>YCCnY~)H z&G+iHbQq8YZS4;GDI=qBy>Ob^3n9T$F+b=s;eQ}E)T;F<+KbFJ-@C6!LwpTbo zr@ZbXhSHG_P20x5XK?s3w=h>{o>sNJ#3x~q?jy1xB{=bX$m}CW*2zx2 zt?vzu>Epl1IH2_}%1LCO)H5k>+-ml1{)N9NN*1^#N2j#GhVYv1o<~CcXO(krl6M!k za76zJ#=EHf5Gf}cg>VIakqGMTx>CoH`l^-3a`u4;#rm7q5m@a@q|G$bN(!Z=C7so+ z>pekbL{hfzzr@DkB~2|Tcu0!wOSsC}c7N+KSzz8}(ZRhT)%8(Q$aRUlaj0!xcX(dv zRWp9r9YSh>5qDY#_ajwjFRSZikqZvdmDH79>j5Nx$HOJfCk97Dy$>$Kdij3J)G9~# z8+oh>O703><_Wx?Uf85XN*kkWCdEF3qI5`s3}6AWk0I9b142CTvzvG-vmEn+xZe*i2H@i$1?neAUmTX-2=e$w46-I~^q>>AqS$$=v(x zUMDf1M|{zi=GC@BrdA9kaiIQLmRu(^08v_R#6kTW)`$Abuex3*G(j_P7@Hy^nMu=$mZm?3my zKpag}t!MtQH{DqWXCcqmxH{>3O}5fbWtm3TW4@G$KDN%>VAu5v{iGrh2Q>Bc!jvCf@y=1!6b<5vz zbB-iV-nFDpt>kJ`%Wl>2UqQXO+kz(oM+Bn)CU}@%I&)@B0Jez$)WLv&!4#z}abO7Y zij&h_mQ1elIl?XU{sUJ`hV3f%SbqCjz;4_bRNpX8()wc<}Y0NBm=1 zQX2IF7sJy=MD9vOmY$cJS7Lj3hT^0Z&Q8bvPTuWl?$+Td|@q&Dw)Z(Iv(ORE> z+9WaCTGzkd*x|hjbrP+YxN;o|=utB1J<92Ff2bF3WIlJi~#lxZg#k%nGgwsKUqVLB*49c)5(5Lh ziq_WR1Nciy<<}e<`(A->4pd)uMR+`=s)t=|^Svm4SzI?wq*x)NwfU(G1q%(pyo`LO z78AX^9`i^4-26Fd;5ijgrf$9eJJIf$W3Vp~GOIs_UuBOE&4KTS(cz(w92<@ciD)tu zk{Eul7Qm|`GWrOk7q91Auo`_CTqLl!8Q3p2x>K2<{dNNyw^-jX23ZtB>YGW%197YpHeG9~U`~%7# zoow!7Ipn5UD{41+A6`ami;uXdTdG9K-=B=xEQju?%Qpj8K~T1Nh|%txS$WGVP9{!o z`Tgibs8p)}=!3%#9GmoT$s?bE-xFGy)gu*$KG*tSF1C3MUYcdQS?EsOPxL-RP0h$SO`;=`827ln~WHg zqN8HdpVlqu&8vU06uRxBwQNj6`ub!0IrYQWzeq+P+CINrXn0$AW`^=?o~Ng!Z+kGch<`n8nR@y%)CEuwmLpTT#~*yFYRzUcx7%udqm zEdtVODWe3Hnl6G{;2QiUeZ`J1*c3pR5}(kwJ!4Ke*o5|f4b4+rmvYVjG8T?Ap8i_x zJU4M6SfnG~Jpmn#2_(R>^oflR!9Lx3WALphGc?{PX1)>2%P(3Mv>ndHqg_hF(6X#7 z2f^859rxRy@{SHN_vNJ5BO?yuZs1R?UVH}$Bg%FpV}cg?{jdjRc~3*XoN+Mmd@3?^ z!BbYrxkmgip8{&W-j>gL+i)~M5^TI%t%&0knRY|)HV`edI9;P~BxTHw>Z$@M*QgzI+UO(~W=?gvlLaRgOqBNh5 zG~=;a(VxP9!aT=8s;Nzo(P2BHx*`L`?)zv$NNFP7<*vzK#jwqbl0szzc6>UAH<^sb z4h_q?;B09&2ejT7XhOv>(fxRQ_B|5_#(aYFSSA^3f&^!pwWm_(Hk|}Eg(MmKD{~!8ABV#5jDpV_80zY4+r@uy4j0n>?kDO; zSb#jbCt_=&*|aZ>_K5{EU;>|YS(Ah0f&8h#Lin~v-f@u*=D~ZJDJ2?2LQGe}I5@zU zAS&6}mvUT^QBiX0qNS!{Oy@+n4hpv4%8HUA&_ON7&XDnYxl5o}cKpjx$Qo3c+DgEi zCp|QQg+Z%#jE#vfSx-Q^?&@NxnMg&xrCv1YGX7<~U<4SPBsD2q{OhY)LvOt}lpl^W z(dPp)=FGP?M5w|ZLLCMJr0f0^vTw)P`jFt;P;7)%LR=r5=qUsk7m|}nR4OP-tEsAa z0RCZlL=61et+i9FuRhKI+WJ0v*|F%P*HF3B5~9tY1=;4{)49SXC~bn|B;k5u9<`a)67)vzpg|h7Ij{OTAp8BU{-Dpu`ZIxGq=qi)t|lxh z9JiPuHDyW&W`%?cw0o)s8o@+>dA$AI+vyGQ;x8w})cJNMzB5Md{~?(AEA+|!BJsQ} zuD|532NiugA*KEaV2OVbAC71xyt&VA-NNfYU6!qF(J0FP6#Q!Oed3*EoeHK+Ja!d6-o|No6|JL!sWi|I<|I+v{-p4U#kJS7 zGr3qe!bMvj-}_`Gp)FjaVs5VlS9{qDN`t$w=?s2|TH^cbsV6sWW_3uwH=iEXJd z7_eC&PvoPjpUz3&F?#Ve0T7x2DJ-dZ4JmvCe;-bi1{Q@CZx1^0`lZx+jE<-y?Q;Q9)^&U38q-xnEK ziDcfAb)-YhE+2oHedS zQ0R<3PzME~-bG3h3Q#6P=`$b3t0*LivtUDyk@c_=dmm7}jY{ptOil7o^L*!A=AuUibd3&qWh-b8CE1fx1 z4t9*d%@DRpM{LsLYrc;Qd$ z=qnRlq5g1vdY!9mx%Fd=#dm%~B5#q`MnkaWBDG!`pgRj-ufG)`U(Aiz{wUfDKK8}G zlk1I)@wZKBpr3($MS78j{&@BcpgWPBh_X;08A#;jH#R2~K9)+un>^tovF+H5Mw47Il{Lfz-48$D(J@*&dV&-0Jgtu7U7%85@60ZP~Oy?UbKk_n@2u z!IZj&aSeY0D0oYy$|eSNg7{D0f`gXcrpWrPjecif&f})p$NNSRiXBe%T45EbrmVu? zKX15Q`5a3(5Y8K9O=WW7CzlQ>1$b@mC;j~OPW88h<1GK>Bk_W zG#ESVles(-B**_Ixar5A*6Co<<~i&Dg|cYcTw%Dm1>R%Qy>OXsoJ<2>{^y1L%-uQaelI~xc@}1&}NiO{Oyz5_@65ESoufObWj;x1RMA#-pvZRj@m*&4hxnw!R(vGL>3x=2AyVo#ejNQT?a5kw z|ED~JQ^BFF@&5ivLH^%cVNLr}B}%`Q{7+#cD~q(gvM~}G(y;QU`X#ZOMC^mdZb;5! zfO#B}9--C#C2f`3K;>d_xJh7JTgN9c!R#A}0NUf2(EmOeMNEnjao7v+zrrd-J|iMM ziGxV*ba~G`N3_ zQU8w;K=?l#Dd2?DT};SxU<%c*w|P ze{+WO4SA1~bYt?R@78>;?oD84|3aq6y8bO@-a}RpqGr;AX|?3KInz0(nn(6vJ75TS zv($UD|3R6&zT)uM!?n~X-tM>D!q<~JGrj@`csOYYO(1-rFt7q59L?FboDy8Q#bZ(3 z*~|4FqX%JB?<2KAE&WJaS)65OKg9RVoD_S{$1km__{jS4U4LBePkZ)`-g1xC^;^jk z^nq#yvI_pBPOap-%ccDs-=SIfwi8ipoJNCWH^Y`v-AXx+S185{k`$-q-I- zk@#&ylQe&p8xcK%>vtv2 z(OvDXz#m{adA7#txhR0UPWcz9{_L85%jZkG%%=y9aIr?T4r&f%B)2~qYJFz_i6^YJ z93XU*t}eXW4D7SkcIVe?c&)ldXz{)yen32uR?f<$tLgc!XsT!?`sN`(8Q?b5@)$cE z+<}U|-!kfdwRFB>q_-W%r5gL$F7ugz`KL)DDJxP+FuRuU_ddZgvB?cgNRyI5b&#Km zolk?3Hyn|26Zka$;A3g=NbS)4s=~vfQNlknl;9$B^#h;V@8V>fiqnTXT&mU?;i2-- z9-2T2-X{HY%7pO|W#OHbOyZBeZ0*0=2j0t|%U(*3JW8HXzZ~7XM>#{a>gyX?ykT4k zDQ=u+oUUoEtB$C&WzDpPrtH^7?LBR^%zeOBQ&8=D2mZZS=x}FDqXg>f+PK%4uSuvQ zGq|xBJ)^h*j!WurpPD59raTM%6jmp?h`-{D@k*I$(X$!jv$KxPL+c-fOZj)A8#Oe^ zl(ygy@VdkPXp&@8(~0`Ry%l|b_hRri_V2G7ZqlfHn(>o~{Ai604=%f%i#QKB+sq?B zxz>7lPhWb&t~v)kbz0(l5?c*A!=MUQ%3b)M@Nhm$s`pJv-2e;4*=LUJI82z(hC#ZuG|l|NdKH9ek+otk9oi?%!mvuh#T zldv_o=e#xul=~EKHpdE^jwp$w1n9$Ac3eQ#7%=gCl^2a zzz=k&6h~~w(iL%+ity(g=PF)5))IN8@Q}Z3 znE&+CKVr$-V*6)9nob&(f0v<0Zv(?@X3=9))DuxGJ%1V%{n$^imI>dTa*Xu{4j^Md zf@X%g!gvX8j-!)f7@u}4D3CuB?bQQ^Dv83p4giF^1!H4|TZ(`SC!xe>p8K}j`?%Fw zpMHfSc0X}}y>MO1qD&Ada{dRPRC>`EE?22|TVE+FORwxQm+w40Y(G{jp2;Jsx{qV6 zpLMM|XfU~REi;`kQ4-4}ky4UQ$Xo-}WkI8yqxa<(OeV}=^h}oT6ry$<9=E~W)`OP- zj3~gzY85nm$_{W-)%V+pbu)Q=b(30huF{2!5jm;0O5kO&o*vwhem6T;se&u?#KMk+ z{8cud6%p^)DV4o?1XC{OM3E_6NR4y#b0G)lj8ZLXy513UWs?CkJyT|ZjT1GI*qi)2 z_4$(MKYW56XZqJR(!!FEn=cM{6$9zGMVk&bxV$jYX=7QK2sRZUD)q}yvS#qoUL0)3 z#Yk5~k!rik(wOOO)rwmX!+spP*^OBRcH$}Njky*QsZK4s;Fl8Gr|7J@u%?$P@g#zf z2AKGDCKgaXkAV_M`gZCnF9fHld>iHKpXVoDalxB+J8&jYYn!`YMKNs(`B{Cxj z5vkb`qtF*8n85ns3X)NerNXbK@Ap<*azzCz)E?ZB>fN>7)~Vo=C;3G{b__?OQR{$^@i#%5-h&+>~#rZAe-RJ*3CxKjeZ4AY(*(1 zO0TD86xRUD_@l`$caN|_UEQII)&}g{f=3~a6($q|DNydNnQF7BU*fA4LJ6_jjn)2AU6>Q!3^2zqyDKy-Lc1oLlrtDlmr}TIvkW+jY|1! z%W=+Xl)sAS%fSVJLn9r~ovWRQzVRB(ng(@izVxRjjzgML8XHan$1~L5JDEKj z!g@OBa5+5E&`@tU)INdkzVCmIh|668ExF!3_kOj@E@rWOj`8~T#&x_dFSA5^$`xW< zA|XLovb3<*QdYN~AHY%B{c-r#EvXh&@ka0fy%z^5!&%<^{@sdhTlj5t{hxdg*!KEu zAIlwXCR< zTUV3@)(qOv;TXTb#AW{m8@4@6R5<*l2W^A@6iC#CCa1rw3z-zetuy|MR3PYDe{afR zT!DaS@=|e~+`&mtC6Wu-G%9#NWqB(6@=dWx&0b5OEJxYw;Cp_uAO@xd>ZG2^6#8sB zuOZD9CwoipH3#|k$BDn&_@ovHpieEea+1)3`Doc3Z!QH4BW;_hx6q?b)({RDfvdnE z&X8EMgu!mc9l69WVA|m7uRwPK*SmSsj_ZhDgcq(F>=hDnJlIBY62Ag{QkxWCE9TnF zIgDJ@Wh($b{otTKLP&H|EY-pzyOCUwxjyBU-movf;yhM4_C+xL(CEZ<4-h~9go(UZ zy$&eXWVSdYU$mK#y4*FShRK^^Wu?mH&n7P3g80i4ufSsWY^Pq8 zbj#u`kis+T`5aqq(=LZSNzO3Jf^e@nC9Jk!hIY*i?w=b}Z{^bSaZp@~+usHJ>+Q`o z`3g))uftNV=upQ#@u~nWKm}BpbIsXF2XPi`k#FvbD-algbWoH@ik7fnIDMFZ@rQkY zPB+_BPu^q7)S|5PptjpxbjxQE0S%j6+|4nRV7A^(6+GAdELQ=F1iqdi3J`^q$M1=j zeqvUNcrC+J__C$~DZB+cu&#^^fJ{RP>iO&nTVQ13QEkIqTrQ!>mWVw^g=ugVtcTxnVd!(O*7_MtGRxGU@#Xj zlHgnCcM#IE7EPzWrcZx;&bK3MfpObbmM5Qr$YTFO~>YZ&LJBAA@xcf>`zL$l~Z&!~0sn66_Lm!;C-5ac#=f9!FF z-_NeLmcaH|Tk5&XLYPDqd4CFV!hy3p_I6l=>FUJv{TMKkiqA`z+zqrPhSH@I&AeOk z?h9ezbO#kg%S)OTZ}Jws?!tHS8jWl~?+1&1OwXJ+ z%u4X2hia?7@z8pC&z)6(=dw&h=(Wd67wOBw@g>+txosl4j%e?&I%sQ8j1T>(Mw- zWH24AN@Uhoo3Lmqs9UYDA+y@Lo8xPdba6$rM5&`$*JOjYG!ukgHqo~#GUmF|NE^-A zD7`6roC=MajEh+cKtQS_UE7l)Fy0yF5*8@Ybo6N{C|#8phvRDvk+YoDeYx*lGyT@j zw|kVT5Mg7R)KNI&e3(z_ z0cmQPZmKMwXi(=VSF5G%5?eHL2}FZP*IpI?RITt0Gu;7d|7cVV<{X~G4iYVg-7b-1 z&YTM;)!RSo!0I_24bdF1xRFS*ag20^Jnez$^I-*LJ>Z1u^Kjw4hu>Fo_TCajHnzg4 zqU(zKuetE^yc~U%9)f1e}OS6rMV@c*BObT zN5FTk>X>+iLDE{B!|~nP8k2S_xUt)cVz^xLTs%kt@0Uv?djHPh!-`X;TaYznOM`y* zuz14jv!6mna%`^+lI>_3J#A}tTG;rr%|?tg(f5wFZcl@w7!uxcnh1Eh7{lpTxC4do zxE?vv;b(?+_xr)seACLoNgvtBMNYnnNM0U>2sAaVkPms;<|d99TJRNh*a*G(t?yYX zVXYa#C~E8-&OW56JJy#_cBatkwtl-3xK(4iRQ7cw=e)ENG$Egp>+64nSUc(+hBFX$TBBuvlFlJ6qjQj;3|#b`oBVB1UB#RjTt~>fw+beiaBzpaG0}<#k^ULSJ1|o|zUtd4@UUs3$1caFRG9VG5HFRN2dbo29 z&?K5m+#o8VS6v*sSr*>OdtFoGe~}hKOe2p+Nc8_9>xuruQ~G`EjJ)ra{qN)}*tS+L z9}rURsL8iV&%DUu7#6FZmm9xVkVui;2F|$yhw=Mt9iuhRRrwKJd(De`%tHr1io4y*S6_w6Qt4GPg9;y##dsm zpx3?cQ)nE2i`xEdl?bIK#pY>w_?3tp=?5#(XtPA{8(`5+(yEpjotomUa_ls?^@UVO z+aN408X27In})sl@9ouEf=;A?eKq|-C+VAQ{WaInd+USgf=pA9>VX?AX9`&e-G5RHve z4o9`>)I56>959v*6byrj-ia{jc8+3*c|Dhz&1}0x_0<)Y^mD{IqCl)d zbdSw<&F(+_bgH~&b^L8K$`m=tbyhojf<-Q2e;W{L5UjrqG;ZFo^aGo&Fqz^%ePAYLN0rCHCi_=^M-Zx{cIl(9xXR~I~d6Fxe|==ZB+))-PEpPAsQ>nhor-Au;D z4s7wgIo?CVLA&@cBbsqy)GA^%)_VUvoZ}{jn?<$0g#8U|6c}r}V{f2Z5^t=^f+_E6 zDtZ8~!PCB0&s?HLS8+@u(?8O$`oL5tT)I}80YsE!t`o7c;PzA5(8vqTJv9row3v;| z<@ly0$wrr5Es*ClL4uDi0-;43JG_7TL7?gU;~41uOnTh73*;^W^;#PmM&D*&yR7Le zzxgsY4Ro;@r26f$h^suEl1sFQ>Y>Izm-h~%V}c{nkd}AIqPL{`yATm3hKI*{w5q=N z$9s<$c56Ug>)jV|K+Q){$8^u}X4`dW%dXZCR3G_SqFaJ7*0lIggM4vqt_gnyl6}rI z`xhy&%X#({JV^wy;&j*7G<-Trd`i)g9-pQ&}>%b>V z{$R!p4d_h&iX-j$#;CJ8kHNCBMw6u(G1cjQm#rgd9sU0I>%9Sw^|fcaoO{K$$S!>0 z|KmCH+_fSFo!%2(KAhkGjMcD<#*TaNIgXR$>WvJfpz)onZ@fanYd{~~dUROcyS>g^ zYa6|N=rt8k73yibmqxkCrvrK;g(U%M2)ao$p=@l}GUp~9Ay_~oF7YKm<~Dy+z;Y2c zNk`mSqxnLcYbOx2A5A@Ydc+eLH9H!a?$OIOq{`ZtI>VPnb(^U!otkPA!l`ovsYg_t zeu$^YegjN`fg3i}46XiIjqyII{LF+pDf7pyYu^C$Q>wGBb2(mM?ds}z&?dNlQ_1Si!lG{49DDeMcQlr zfdZsl{?Kn*?yNLK(FP?M*z&e^pmH+6=bnTPSc(m6e1dnacR56T&_6)vRhs6I2J>uP zOe~AOw9Zk_6P{tmo-5nJT!ROUtW7M^j*W!5T_?eOf01$|=GAJOioRg~! zC@*|P^HQUHS-`$eLk$bPen0h`j8#PhTAbRq+q!+p!UFaS9;3Pkzs_tkg>>XMQ3E5} z=^FQuhUBy)9l$w>Ov)b_t}_a7Qn*~`(%qR<5>Bw#wCe^h=yaCCNkwpTF-J2fvuxc% zl}AU*@c!zBVxq(S@LJQ_WfpN|Je^_P&BR9#HplXG>3~dkk zphP}mHplR=5I{+VUvCb*E*!OkPb|xdDv^jG>dX+8L{t8#E=B4!*m`hmtJXSQCVGtr z{RTX(11*dnI=9!=4lFc~kdy;nHoK8!H!+eFqqc;8CdYae3dXtvBZFK+vEJZ8t%bc= zhm1x0*S+OUW=ng@nK$*=)P4XzzJ!V{N8urTD`Tr;TjQeZV=YnOaA6cDJH>wLSL&Q+ z$b_#?&GOi7II4rab9ouKhV9-Amg^EKj0VdNO9?u*KX#|kgDF0nkJtVLjZEhH4SPqd zrg8MhQmKYuIR>{LaT!UF*AX+gAvlpZ`RAQ(kwedGQt4CAlO-oRr6-&EDJRV$Ied1|jR?)fJ0$wza;B6{q`^8S`N@vV_4( z5x5yY4@JVn2^;Oj=oIaJB-@=PkAiE{S$GPyZzImPN3J<=j+dsc)A~=12$|}Qdu0)k z^+zjnSjz16{i2Dg$s^Liqj{cQNCu4VY3@j58^4FK{4)+5y;vi{yliy?MP95gda)C5dszfZ;MMqrD5 zC2T%|N1i;<+%R=vcy=1MTtbh^ms`&RZ9D-Yj^ri9$+H4&mM8VrQ}0HD`;e7s?dQlQ z9mEPvL(Bd75NJW~^`}agr7WioUfCekKfXXIAdGCOZdi#!6_z=@TQuB=wBEn)U1Y~f z&$6_F5;%U5ol!u!F1HhaC+{^!PALm1z%BUPE@9ZhEQHGQ7wJ(#c{V@aAsK`{4nrYy zFGJ!4`vASZ1u-f1LaD2^)Cn(XWkG00;DER5;}YtGTU_@QZ1*KV#UC{8l#N=jHHhn8`81J%+qe^U4doF!GMg|S}RZQs2m?-13iFpEa(&KItqoxw3{Q>ir&qzJvrl)wx|Ka_4 zv#Yr7md_Z$BR1Z>>cF&!R{+*3i_c~2!`dMH{Zl;n=S%t?1RsvL5$*S0a1p{QNmDx>`6#JDBtY*h^|Q zA{4|sv09St$M@6Of9dp$a&~BLk#!G4sm$Pv;yq)H+p2O{Y&@;6#bCdX^XLPoP1DcZ zCY4$#G)tK_v0@hZLtRIJ2`O;Z+ew3v4YK(vi1`Xpc zHCm{WNBKbW9~t^IWlBE$I&e_av78#OF}8mdp^J3UwiB21%ZJEl6b?MV7p?`p_heLZ zhxs>%9^KL=(sFSU0{~Z1hXWkiE(?UR;dz_`&iQX}7tv zl982pdf9ZF)>cez%?8gzaPvx7Hj3)=h2-)!nHzSD`-Vrb!g1MKzY%|@c{9C5N$N50 zx%SzhJ+l2CDuH>6Zzf3)x#B_F`8TmpCKUuk=EK5%oMU*g{GeXKSEK zq<5qj`lNN+$X3O7D3b6C=Z`cynX4O4aF5_>L`-R_RjiY#=xKi3m?z8yvLy`XoCm1L z>05MEoP@MbJSKEKA!6=MKw)#+!7YI}^_usfI6sXJrq|q*$2{{d<$Mi9kZ@ey6I3LU zVeD`Pneldpe6qvhQ<^{Zh>#HaI*~5NUrl(EV;uH^jRm9w0wI0ZYsZ;Zc&3Mx^f9JR z7CLfJzIeyMv=ZWc|J`zf#L0G!c)@O-8G!-k@V-Gqiy%mdpyH4*j!)CXgM=2%UMw3R zqt&%Y?GF7Kc(o%LS(;2TrXqf*ytg!4swqLoYM>c+llr+FWFPSuY;6!f|VAOT8PMzjhIxZ>l}x7r4tDw`6DNic<3NXwleDUs7Y^L;!W&6 z;WI)q%J+M<{wr99#uD!MNZdY#Xk=Hv{O^%6nT^i0BGkbgmSsz9$*K_TXI>iYIvnGU z1%`>^PuM|k&C`NAabJ}EOQ*8PT#8~QNKmafJyT$(BOFE;ED@69C_?{AlO%U`Gw08X zSj4I-wYh|7i~*<34T~tG9%mO9513et#0_gz9mxOUrARDs|0P1fvG@wDK7Z~Wf4-=B zvUiDC3C2VLT(&>wN^D$UAd|8;bQ6EOPaOEa$#O!ZFNYIVn+^1g|I9}(S>F(QG#+2? zfBG*Ft7REzei-y3URTHOHN)cnIpBH!%c!Cw|ChS$Qu-G{ec*Fvi$MBc02GPuHNwR7 z4RJ@0I8NrD#^-;yP2fNA(|d}T0L}38Moo0a|Be0wu-b0@?}HJlpZ~a?mNNg}LDZBF z#vi?dCH@lu{da_yE>1ZJ7JTG(1Cr3jawJ_@2{rB!je*VgtG`IA33`XhlP2!t-%>P< zRo|e%I{8sOzwsxxq;aveC-7gXOq*^{3YrMK<>GSh+6>y|KnrK>S`im2!M((xrnA>)w6=d)@?pxSGuIP z^I14Qx3O1qroux zbi$_v(t7YaCS_n%`$UiH{MBiYVabvBearMdVUyOClVUFQdBmR}rA1bqNw7TT5CfD9 zq`o%DF!6n(Glz5eZHa&!eNB+J^92=@D*55VBg3AaBlVuqp{`T05OeXp9Vc_qVEY;Fj0k$I#TWHItdVHM97 zy;MkzJ5aR2s9iFC#4x-8*= zUle^Kq0gG`?$Y$`MB5@bkC_XcG!^}kC>!tX^S+;7jD)V~rzYEI18(GBq|ow)Vc{0L zy7IQdj1<4WNJ9_ECu?h!{N6%dCWB`nz?i%~>#$9c*k7cZjO&;d+QhUV+)NEk5LIL< zo%VrR!_pGlJ6K=B0Wy7$_6SIEd92cdKnKQpK50C}YvmJVbY6EVYV@I7?nB-M#+w>X zL02&rMac$B1;e9Mt@gGoY+9VDwr>TP=}c@4V(}c~IrHtNm2A7SmwNA^ z+2c4TVT+si@gr}WWk`K75qD!Y^V2;`^=f!X47+XVN)YeBr@37I&zl)dN{utrnq*ee zlTbJfR=JF21vRi{>FY4HDSgPi1~xC=(P&Q;)moM$W-P_beEb^X~a0L#cVjUrQDl|3ko{bwSctE&vo&emq^ zbi9rTRFX^UB#cZ7X_ie!G6|@Nn+vDK!>HS>Lh{i%S^%Q(9*#-cYeflh$wU=c&4LLepF}ij1@sH#JwUux2X}=skg>G$TFnR3yI3L^E`V=SdJJ?J!Tve%(_#up4A4 zAZaw}0C-V?P4+r7xxVij|7NR~AiLVNxwMFeJT;#yyQfBM2LBh@loPAD&r_58r&auv zAMyc=s731>mNE?k&n3Vlyn!Uoj$0%ek$5wxX%N?GB;g#ZkAI_xnVs9hr^OaW7kGlm zgTV)`hTan;(exn4r5*klB}Lzrvcq+LHAb+eST$A3&|;2@TiQpoaeh@7F8C^+8y0&F zB#5P7PC#>_Yw@{Lz*YEkBgX^sZ@?o0EuX%eXNA@vE}ZBF)#dcit!n0(@7Jl=bMb%e zHHjPa!|JBR_+Nnrczt~no_~=>eSbx<9@CVGIFWO9pddK<25ga*8`n_@~UbQaur<0`36ry@xC`YBzoyrfCU$mlI{J#iIS0Fe|&IcEiYvoa8XK+jiO#T))w zr4+4?P93OYr#F41{Y^)o=}tQXj>w76=Zm(K0EBT<72^`;8s1sQ}Si)XDDKBh(_>=1Pv1MbFSAzFQ;vEndi$0bHPLM?B1fY z1#!W2S{d(SExd5!2DfoA^RNeu;TD84aho&zIz!i+U*u_m<=sm?T-~*$7{{59`j*1< zlo=bm@@M5woZiD5607hvl)?)1_qt13Mx>86;*NhYeVt-YTK11L2AUA%A3iBf0p_FxqBv>WHbtnDj<-wkz>EdMLjvBXxkO9 zBH}xc=A%R`9Bt&~DFSCfBTAJ&(BM8htE^a0U20_sYh!Y#7T;jG9oGUfg|+PR zcq$nyd;p9XRLkw0#l{)VD$&jc)+fr!!ECH4Ar>8zYKa=B!q#AN(<*Ss3ShzHtq+#~U z%_J^;IT4XPlztDlqqJ9zN#U;BB7-9cnzlqDXR#C$=V&g9o;SA{A@@g3vvlCkV1s+q| zO8<1k*ND6%6%lgSBN(#e^~?)>ac6_W3HdIHpSfb~cw<=}s?-(3@`i>*v8j30S$f^z zhc?!iSlw>FyOUP?_8at=j(OJ3zewW7Zo^WSHa&v`HbxQJ!!9i1d!^af7!SV1{@C6` z!Zt^o>S@y)1bs!e_uiOQwnNnHM`8U&wVL$0B|4-#qf;g$Q1M=NSBrF|zeu1-c0X}? zHtVY;Ex|)gQL14L3#!_JRo?I05*T(*+{JINvi~B{v0iF-lXb_!%>b&DlP<=|hhEx+ zQ;@oT_3!+mxx>+QTP{U1vKqRQq@>4S`C(VZRhA+u@Dk7R*F$HU zXobo?g-Kc#Y{~CtLo9IeHYocaz}cP6Z0YpE3Jb+rZ+(xghC%&{mQibNO_=Za>0!pK zVou%)#X0*yp)gq_Ys2-?@%$GRhgR)E#QTiTav`JL?+`3- zDY|(MiJFuWumQ`Tir4B=XdDqKuD?2ZUKo;x4O(2TGB}XKBL};s+8y%&Oo#Z*w0E^| z?XB6!DSq}OHlbEhTz>-UEZ5b(pzJ2=XK${`IArI-pFf(leu}>(c0vy=Y_HYOskb&S z-R#vCWJ`(k+Ec#_(K?cmgEZbhuvIUQNb#3KZbCRazvj-=Td<0MXkD%R#c4GXGaD>j zs1VaMtR~s&+A1FD?r^S%MFmtk++T8&t!jk1?v)I#J^*e<5{u{!a{X*l%|JfDypqL~ z{Olyq6mC)2^=YBMeU>tHK=OhVocYIFOsLGgQ{siJ-A5Xc1l!8BaiR{uTw6n%S&bPq zqz=Uvz|lHcHQ%Ad->KsZce9GTBI4&C-wp(6(-O;JP}lrK z>L|35z~0pyNFS^J%v*##4x%2+P*Y&YEUBZUXwM)xuq=IAMg4pK}Zt$&%eY($IlsGXP5CY~jjRUAAAGt@zEm4nc8^0Y!j zKK{}Za)*@8aMN2{_Ev?6bm{N^Fs(Mdj9bb2$R1GV`RI^Mg@^8E* zW#-2cX|OM)8jaLKj9Nk>G^f1sa_4I9%^1=YiM+gf2GiyzuR+zd9QRn2v&lUCSMI%f{rguiM}2**N>uqsw}rhxD<;xdI6)3N=N}56!}TVO@oqm-$uZ z5y|80^6C<-@$G%tJAYa7P=5ckmB%*0_M{OOm#HL5w=9X9B`ORXVIO`MN|TNAM$>_wb9b*hXD=}5<3^@ zu35)Ulcoi}Q_LpKO$vnmoMb%paXS7(OF4Dyb=$!S^hC2q| zLVU2+92goZpmpIB(&hq#VGp3p;j4eNsM+c}pW|Mn%(*E-382 zk(h=T>8qI?+IP8TUVwuPWg&xn@JncyiIn35@AZY;esW|XX)R2fqu13%l0Gm*p$;)w z0;R5r#9q@0Ikb>immIvK$V=Ue~sseF#oLk!#6dvL+qtV&a@pGc|8GK-(F-G?b2BDwEY zV`o#Nh=1{dTd_}1L{2O?9JD&}OXCpxc;mLmU+LD=BbxFGpvAxC>MF{EmdJ$^PK#LK z^ZXHQ$ofS2dx89j?9PYsIZXI;NIVQ4kK$TKW5`Sfy9H3r#Kpx+J8l}@z zoWY~LvD~vBS|6AJBpr`c;So+eIIif@pY{3n?7HdDZLqdl({jS0py+Mw zdl8GERBubVnYUbDjxMF1Bh($jS$cD;cvgdz+@^V{S8b?yh`G21goE=8jE zpS^4T#o(#C=F)_PUvxdLT8iOA^_x?~;Ez9~ngvs|fB=@fQdZ4~6U+rRLhHGwjG=7pj~LQMP*0_Ap3c#*Pd zp&kWHc8k#m?x*?#If~}x_Ur!l6B|}6(`3`HllCQHX;%D}9z+PsU>m4T4DNUE3iheQ z&n3FRE$%$ArqhCKWoDbfsQrk3@_ujQCW1BoRc)S@85KFhS~g;&0x=!^QKF+)ob?Tf zVi4@s!(`Pu-Y)XuW^>kF{l9LWv?k1U6LPEXv^&Ljf@#wI;=Xzq{4d9~pF;;9|FZuJ zTz5|NzZs}NfeGAh{(lou6E%KkQT7ks=CanvHJ9Y}UjIGD36|RGd+z4gK4Iy6kBIzs zEqZ|BKa)JpfA87N{-0_7J>I_#dcZIFa+HD3pER^c6vp0QA-sJvL8p4bDk4TDcItfW z2hV9%3;F!ZWqlhz8K4yNnzCfd^PjtZAol5?IIr&eXNB%` z`7b4?>ze>diklIiBi)6rHUl_n`s%&!y#vef03w?K7YDGTC4kp!m*MLYAoAfOWXTJ% z=l9&X(G180$UH$-{q99{A`93ukZ&s|-2OcvV%*dB~_BLf0@vHs(=zI#4(%rTizOJb}K|FlXnE%BH@rF35Z&I6gm|zcIa}nYy3F_aW`m ztw!b@6l7VC#pgdY-N3U4}K=6ws2yUFyrx?ssDvckb-^u|zON8~p?L z^wnHeI&Zl&Uq%ernkSt%mRDT4<6r%vU3DNC+k zshIw!+i`H(5nJV|0R62{PHwsow|W1E`qONIaJ<(aW{>hGn_MM$C4V>FP^|jw9EwrLXV#(SosY%?1Eg|Tp26`oIYz%mh<8A`uS%-dEfFzuDwk# zzT}jrl%`Oau*XifZvu;>=&B876ndFNr(7W{W-ZM`$7lN59;5hH(3n@B3sR6kYz#^NTz@P`Cq9pN!o42!fbpS# zePQ^o#kge&QNM@U+6Kh4h^9`GC-pr&BF(S$Dfj8LG1+zz)LgbOfWd-iC#4dt08CyJeG{b3iMOfv$WOY%SeSth9# z@5hUM6w6a7$3TP0ffaTv-B_!(r8CSwRXsT_JW3^;l`7#Jw%?+bL^kPS5#vK|yso1^ zAAm=~e%ktJ)W6Q^Wce^h>hN@)X4=I;1v>ghVN5!fjgx_Hheg03*;0G+ZKAn+sYp<~ z`60sRgGpLagVgjxx)NT>vRXnJ~H*|s8;SAS71=Zih`1IPV10o~*EsXsk7uV0z_xh2+NcOxeT->G;6enVRUi;hMIuCcN9FbS=uv(P(SJFWVKi%l$Q~`;OK?Ub~FwqwhowB-= zsd^iIU!SpJ1h+I$+Ih&ceOKqln$WD*Iiulq;bSEXDc{*-v(b`)m|UeXEFuS5v%<;_ zi|X|oSKi`YmL$Bg9C-T~lli{7+j+OCLBsU#-;;d%Q+)KQ;KJca!y*Bdh8&A!rc=sr zJDEM24@ZyYGRm>Rn2lLvQKOYWVM5~B${t>Lk`8C{*!&jfwyz9Lje#2nV?5&O#ZIU(ih8st;R6=AL{dYm2yoV6T(>g5#_x-v z%>wo-%`BoIttGf==~eYf>eeU8sQa=p(#n0s5AoS}+BFv!i6Zb`K$Hs z>mz4j!c{tRgZqeHj$YPs^$9+brb1r`u<@<6Uce=Tno~^Fflw$VXyR`u{AAd(U)j?IAMx z6$$Zn%M!Z|g6N&*#OSZ4tGlV?`NoHctg8b&TkOh$RiZz~U2&edAueK%2ITQw4lFkV zY7r3?y9PP22-OWrnvjE0Um1xo2b{*>Z*&=_Uwp%O&mpg#&DfTyN3qT>zVEsR7M2_x zYtH^-~pJFEHH{9iay8-X0FUtSV+`xPg;U{Zu4U17|053wjFY zs5>b12u(^R_;pw5J3ZJbSNLJL~;%eSKuEa*l= zJdznzzWD+wt|r9PJegHyQnWFat$Jz7%B3yVRk3VVZ!;F^%Sv#02>)b(iI~kAHM&-4 zrs%pu)xPqnG9syT$d>Lg?nVK1r8WbzQ343ibR*+*+)ADA=~lzz@FU$^XE^D_bxr4;H!_BV?fnt72G~10@~0RXL$ZT~zuhSW({2LCS*~5TpPNY6IYRoEbXk$IR%tGdhg*6E?OsU~jfa zS6Nt+;P&qtug#tPO0*j)l9Hy3$j6~%fL8NMh0UcGBpf{OK$S6F+x9;77kM2{Tc%|8 ztmmB-huD1Maj?vrVJytA{S#AhoiC9}8|McaPlP2C)+=ys&WMpQ&X-c-#>ut3`;2}P zyQiEKz^U=mF;dExrmKBmS+1yp=g})BIgw30oT#l>ts{&&@=PeUh~}}eYR6ngAU#?> z{0cdRZ_FM;(m6;uYWoC=om49I2gL%x;)Xu15*ZnFp(ePZsW2B=>Dh8eKVM&)Sdfys zNnq|u6#dCD@OE_BsF1XzHWfYYXUJTmAv4r7T%PPRDiEilyw{#-P5%pTgT`J-)hhaj zX@*GtbHPeDOOjGbmM{&2_*8z{)-{qhDN?#|I5)yglh z?U(~>nijp{Jb&-|D5uK^SS2RX)cO-2o3v8T&4eCznUt(-5jbxQp~JqDv@I1Ki@4QL zoi#0orP&1O(C&}Sb`w>|^dUM&7Achf@_wjs3oz+VzYUk(d?8Rdd;RVs{QM4fTK8LI zv~lm$w4XvMTA$oELS{iZ-)xeX_{~>{e=^t9N`5AL`)J_k6N=S8fD7A@SB4`j`0(?a zNAlKgas7K6AJ*N>*|U~xSQ2?w2B#TISIXmJMk|+E*|-w1P+69|@2-EU8>lPmOKZ>^ z4P){+XTmy>0q+KvWrv2T;3w}73DXO4?qCfIBvZz6FY(jJI)XA9#7+;4n@0`7M4;qn z8)OcB@_L2?%|ve;Y6dz!*DMCHYk2`mwC4nTTVoOi^nK_fnlh zC22g@c|(5;lW~nyP1H`CNr7zFCGKWj{A}Xn1>eJS<^3rVo964swi?bWfnp|wO6{u; ze+06Q;>&+Iaz4uOx$EX}M%SuO_dQmqRAPzc3k>iLU=f}FaUV}O9IN(n*XO}OvSnDV z<#PI&Y0LKt^`b66p{*pvwyY6Frc2voAbZR2!L}O2pwmp1rCk~1Nz|0A!Xf*-h`CMr}F8A>?#@p ziTOE$r5y|kHuzE*lit3Vu!9wE!$Pxvx=y+Opnb?$11=Zz(Swxla~y-ASuS{0X_AIkG(@a3A;Z1DHJy&%j%jDPEr>5R zY)!Et^xcFCc-~cJ!qxI;;tW}Kv{-9=)8i=AMNNr~8aVS`) z?Th;LrVHT)OK`z{Ja;R2r4VO1I>;$-*qKaDRj^nal}e6s7su2M4G*D$MjAAmuhtsf z;}35DnU=d-$_rv(b0+=eHzCv|K0sJ+K27|B^42(7>-}zm^sl$rtf(+h$Sw<|oaSKL z=;$AguzxSH(U`X}e?$}}Z>Ul&W(;D}GGfoh_JQ0h(L{~EKQ}k+R29l5q8gS?Sc zQ=c5PaRC0dE-95Ic|JS2PCwQ4a<_w0p#9dli?t-wmRB9S*0#DU`aq0jqdeg37LQAL z%4fzuV1>ikFy?id>X&(}w!Wx5(MX7b23B3LDVVOem_l{+R4-6ps&trn{n)Rf7oql> z;fA@Cs9xJ(Q#?b*HYUkr@WPJu{XrVAi{lmVaQct}Rd>qQXsT zQ$+ByVTLWCe@meCq|Ra_!-+96MLLW6v-G!g`kRVewYXx0nOWSidFA{$;~CkI51x?{iNzL$7)bOD zZuaAa(sg5e*~cxJ%2wb31^bsey_2UVqoNz=eFlDn#*O}<5ske=V6l?G zfx02b-BXc{+^F_QM*tc}Q=A=|Dc8Fn47=QO{WMMKmQ?aVCnb6BjDVUnLl-)7@d|zc zH`NwIQ6@YiF%B_%Km$dK)Eu#)sF4cAn&1!%29mHzDIP57##ouu*W0t3xy z40WD*1Ns)m{*j*kjD`|kL5Eu3@&#MSpOjEWf#p)6sCV;(Hh%R5f+Vt#!Ui3d!M~;= zM;d`4r9)9`sP@5eMSI}Vk2r?l3~=HHUZ|k?eO0)|z%vE$p4B$2hnCM9Jwm6Bo{+S_ z()feiql!8gUn+@$JdZGuVuNHoh$sULhZ=lD#3?zH00&VM-ywr%62#x)p^}PV@rt`nt-kTsSLp046CAiimOoy^C8Ktv zj+JJTdISlg>^^+8NE`+Shf^;FVoolSO)K$bcQmO4nUr`i=Zqq8*PDlkq_>=>eK?gp zs)q#qU~*pnp<b#2+oXJB zAq^6=2XwBXZP)X%RpL!qFg4;8dH{d3yM{K7#pV0EWV`i+FE!-X-+?oW8(hP0lgjs1 zi(GG5UUdmKDN0_&c|qpds_eM&E%gvS7M$g}mJkJ=OgRW1Q#wI_T%@YnhX` zKijylo3^^h_9l97Qi}IodNQ{~xQ3;c(s~M+r1$&5ZEx$a5zI7t&B|s$y{%}ZOl^r_ zNsxC8pKRwD(bFIdRrf;kQ)Awm8D`S4imZ){uNx#=_NR}K8xscuQmF>LU-^xNQ*SbEwiS_W?VL!Wv^OR#S)l#;c2L0M@ zqIV8qH6%PC8pWT7Q0QrtSTN8a&9!{IROeBNK=kmt)3X%Iav8N~`rFC#G-0o;)q@g8 ze1%*5ync%uY$jW^H(#bYZ?8$vyd3==5HGT@Cn{@ptWtQH+fxc$Y^es%r#jVev-_{C!tuAT@#5-WB$#g z|0F>45H^aSs0oJ+5Inqe!N?8%%hO)G1VH{G5Nxb^JGMNFRv!euzdLml{0E>AU;t1C z{QqE8`0JjX0NrmL58>ocq-XEnh)#g6>?MGIm$5HEB=mUC$Ay(00jU}0Xgmn10lbC= zOwNvxUj?GdHAYVy_&~Y{5C?s{0=nOP(eT=T`pYU#&gn^>LX9Z4>cr+Ecl!126n1nE z_)*rUpKZ_f1&%1_63H=(u&r#BnGZuRqW&>~GEjZ~}^YzdpEc z?!f^NZykN&FS~w2b}=XI9s}h&*JBpXq#s~g0ZM-nWXI5S*3c4bYM;e6wikA<0#S%v zSeiG#u-SR7M!)c#+XZTPinw>QJnAn-0r1&TIGmK7$+%dv)y_UaZ09J-FBG{cG}zPk-2zhaQVL#M-ojCD1d{1b@+o_jOFXFXShQA%VlUuY^Z4~|vzsLeC0vB#XBjm9ydEc}@&9(+ z*HWXuaH3oRYX0ON_5UupKJ9VQ0kJ^6aZ1@eq!!g(I)RQ`+<>{(aQ zBxj{x(k|K=y_*TYX^qe`!KQ8%q21|?eV>S=p3`G5S=NJxfh$Y>{y_O zONb?8$VlYZFp$)h=?8UZ*Gsvg5Qz;4q+F1OMP#sE7T*TM>RZoB!r>u2lu^Mm!r8R2 zzkk)X_dk)44Cy`s$1U-`nqpTf@B&=2?K994?F$(gtBEWB?jPz>)bi%9ZN6G02h6q& zIES4vu4_&36x`cYh?3TE-#bmVB_zW7EyeMb-oz*3Oa)`8%gw)2OD1V8ClA$g{4yEKF8D<+45_5JNPfSM=`U+ z>Rl%fefpSSMon4Vupw24M~$aor2*^6K-{f4R9I)wk<-SoMED!rxe2$M9I_|353_tl ztS^{d5W&b$;UDa-#mhf9Z$!%a7|D!t(v8#uMV`Jbaety}-R=6D@-guOxB1u>?2w4% z4I5Uw)}P&uV6OjQKa<_kw9Rrs^2Fv>v0Gh#r}(Nm3l!9mUWv>H&eLD&$GmG8Uy3+J zs_vk_{V*henuSq1*HUW>-f2ov!beq8$&+Ltg09CAyDR^wn}4QmcF0?%J@9hQqhEAv zbDPab>&rHZp!G1u5l;W9p7zX?3^xJ|=vyR?+Hz6rzJy9QCcYCS6q<2-=T&+nrAEaN zkGnFTG%IebWJu?Q& zScGyYWfPXbDg%h<1mBDBq&2ZxVkINQ_i%z?7?iXK;B z(Q(S5#V0}Fqr;r=`y-`8E=mv6-)AR|Y%&%m8Ev&`@wlis@SHYsm|cv|w*=1ZMWwuE zuD1&VXcFQ_#Vk|AtPdak)#i1vHm| z3Cq5SW)k4l_`3}rwFb>phxcH|p@_hF@=pNIn?BrMp!3Wat^ulkA+EF(i_E`deoq=t zC>F1sdtMFUH`cC1g?$kaQ)W!xc;ynFO0_+jBtbm3Vo+8zO_I=LG=@}2+?6sMfyH`c zyMH@=`U5;!%QQ5p3rv%W8v2Ryfi6YDTsaV0GR ze9m93mCDl+B$SFioIgCBifSnJ_W_okI4+!9j65>VW)xYG6eH?$7B6xp&KOI6T=JA- zou{A8?=AoJSI@5B+fL0AY4ek|1eep}gwnf*{hcl|FiCoJ^I!Cu#&}r{U&`guV6GT_fUk|4sk=UZjf4 z`fT2#f)itZ#NTR2}p)p4sJW2I;Dz;CFfW$DcK{4lB>&)ux# zsebe-xhB1i{mz&tM$cO(Xhf{XQ9iL3hV59Kg*u=MVlPc1K;lBjLa`5}STk4Dc zbGt-H``xoC+s7mbGkXl3htuQZH^Q(NMoeH9sA@qQ1ZKwFl`qmVPD*BEA*C)%55~`! zKO#OtMUY})M3Un#@goq)tbc5pd|qf%QTuXK`oy$u7!zQj=B<5KRd#(`Mx=><6B=8# zF|h2yD|u^t^U z>C&X{yOGQY<-QH7Zt5uV5g7(YT7DplfO@;V%w-pUl34^X92@NMOS*eAO+QB_w;Iou zLDZR5_a@3D1v+gql%kF|9*pQSr=8kI=8SDWl*A>L?|{C1F7TyXCB@!P#!#;4O6$5% zJ?oxCmkfK7D%!9)sUM}=N2mNXkO)Ls+9Sn}JSvUP3eC${SWH`yw4`%Qq|`VppkkVW z3ej=DQ>S2hm`W;vmL@>IgBpz=s+dv0P!}b_dQerDHs5k52Z_x_rvF!O##0q-`vTwF zX!;{^8BS8(3Tol_OKA1GIcA>;{rn-YeOIux`cx*^pF6Vb?)~k3gmbVbD6N6sdxwOr z&Y9xcL2l+BfK4+e^Lda!+722U;l7n}$-46)&a#BmJosStOVUi1nZ!2d(I@byyE5B| zT>h1h3LG2y@)=6@rAOu^b)S$3R6eM#uW||Xdb;<31-FFbIs3k9KU(|)Sl%#f-9xLd z6$LhVu`tXZ$s$8twM+V^e~;4;q*(WDCtbO@Pxol(el(9~c5$MAKZ?%aA6@5sg%b{kEZxSkiapp$u0HR3mkj3UG* z<+^jhKaZ9U3Giod3yQTNAh0jfeN^-6kYqRmZ~Rq7@Dnkb$z7~rN6M8%59P;gJ&P8U zbzYFX^Uu1EhLS3w%q!X8ibUZW7`3&5c|G<+e_6r;78fg5*o@0%`>nEs*m7+H(=Y$V|=CH(~)1|9Wt{NkpH zThXTlE>v*>4IpXarZ@C$98ZI2Y6WKC8y9Kl_j0?r1UpW9!yVxh;dS}jM(KY5cp$R} z?T#nk7$Y?w8ZGb~Nw1=d1tI%UqkYAmL<;yQKnG~dSglelJ6Q#?P2tOX<+U2C{^cN_ z!^b@Njg74+h0NcqEZx<hq)4dhm^_ySEQh@YU&JMp~8YJUhd* z#lR@{A{YN?L}K>tL9$U1Tq-X6eO-8Y4rpJ|p%@DKp=cQ`QMB;2NX^KQ8+Acrq9vV1 zOWM>@JW=zHy}V-609I6W`6m)o*-wTD;V@Po*uVU4MF zgm0;is`qX;z+m-0`aP<*#aT3aC$VxYbWHvuu$gu2hGo${_cUeX_0I))iJx+w6A-Ae zgQyRDxMiML&qtU6gxaNn(ex`ETnm$ymY7ymZe} zK}U{xe0;Lc37vczVUppedx@vMrAtA}cdua!Js3%s> z8ruG*9aDAuI0_3obM$@$mwfkJVv5vJdSX8HtL{m%L#&LU+VWBa6|F1Vib~l-&Jr8T z6Hx2(ly1?GGuQTH+_*Sl0yA&^1;;b55xnwWnKjznNPn)KX&p}s_+-FIy8weh+d8~3HW8->p;N*4N z;CGmx@o;TtYSg!FB~ZW5HR8wNRwn(B#>uB;die&jR2Ny6{De<+=9@D-5PPje0Ah|B zzKwGA_V^Y*d^kV%+bIe|nqF#Xk1O0V0^MU1RZe#R&`pF>45q*iFw@V3#=#^hI9D=N=KqJt`+`468rVVw-WWK_8a7kwVHGPizs0{0b zeM(%xAuEkM69Y&KhK*Orpu-F&Kg%1=Y2@Gr=Q`5RsI)o!&DR#2I4>@|jh>~|vp%ZM zqF*Pb3XW%@@JpcLC>KaaxgZt)z@RTA(ewRd5P?K8tOs|6ibaI;!#sDBk_Pnj9I9YY(O%R3 zNcBULkJlxzeh1%Wii6Dnisca}SLE%k!&GFPV(!75AJ?>A_J2SMIJAk+Dm+%$)r`B+Pb>>EmE zE#uTOeJAtI5D5|oy(NPb%EWCNgeKZwW!%ZoB-~uYmCQP2i4=KJwTW21(^$-kMs(w4 zfpVITTnl%3ZkzW?+1DvL8%HYAd1+!NeA>lwX}w=AWAm1i4!smu7JW6-;;7|ElU!U` zde!pUSCMs`yBpegRwNs2%L?Qqrdpx!kM@zdpjgQp1fEVDRuH9syeLvEXtZF@Sn~cI?I3RK` zmzuY8^{N2&Ma%=El7E{%{4fh*<_w|;mwE3LJYa=Q2dk@{o(mf&x!;#}@ajtso9St^ zDo7*Jk(8kfC%n<5z{`-X2l6KuwoTh>$bK-H3&q`$)|~zyM57IOh>?Z$V8dNX}`rHb@DWYauGysUU_!MRoHtdm7l`oR2j3XAr7qw+b!4`RhLWjOo7 zH7vd1c+1#pP2Er#oBjfWgrA7g<1xY@P1~@?ul5%VXl33|x1WE2;0(V9AY-I=AzAfN zRaL8&pqc-#hxh`Em?Aebo`td&WBPr?NzDo^Mjr3))~@06)b+H*yi$YEr|C3(`{I5t zN&Qg?nFG!5VUs+b?(Sx4G67iBNAZ~$?BC|8f~fF_3KH(4!u(}by?9cV1C-il8Z_U4 zx|#xfy*<6XCDT)ZG#7`*pznt=hwiT8xR$owl=OXLVGg{qx45^ZV`L{smoEg2b&ueJ zuGD&A4n{G-@4G}y^ASI6-?;JaasvUM3$X0XxDUQYkzNfLFL26})p4hfMcT@pkSm&R z{y9H&J4M!CU4JLeN%alqlrbeVkqc@j1?+5{7;eWpYKJK6zoZ&|Euj(Z0v0COlzmSC?G(~)MC<7bx#f)!eY}0mExoS|oxX{ghi%p=ckqZlRcjsN;nA<3Ra5%I`>fO-S$0>Yu@4oCMpFLhIbyN~ z$ym^fue8LI*2Zv=UXw4QfJ|)E=a@1x=X3HwQisBqojXRlCx(oi2AuWiC3 zanZ(_ir8;)%cL(MPl9HD)f@^L?3iwr*nl22JnvJ*@95?WF!!0%mNs4Tq}vBYv*|1I zEf`x@u58@DdM>TKB<|{$DK1S-)LhJund@H~5-bCgl#tQY!%!MW;_{FEs>i!no!PuF zJNQlfnw^OSn`EXOy1q)c_MD*QaHQ7VNhiwc2`?8f{T+d9l`e$M$hv04yo-6(w{uhi zS*Na-6mE)aY^5q8ZbP4|@|t6u1+F^HzPR);_lxU0NR(@Am)0R8$6W>rSK&gI0QTq= zn1b?bN@?Xk7qJ&_=DdHS{Gdw5A)E+*`=|%Z4|Q&bO?oc3m!Ap40o7@gwS}eDcP?Po6l^cbCj%Gpj&qX(U(3|l2?%nuHD}ZptFlvds=r}- z(7`(90w)@lP#wye8H*|~*D}Kq0iO6+M(VQqltrJK6LH?5(ismLaU90&y2MNM1Y25z zt7ig{F$@h&!(q{S(QwTrB*fa<;pDHj;%O6my?rF)hu-6l*;EI+Bs_XmJ~}b{^A5O- zc0}ei`8*1b@;F1&)%G-#s^{0di+iT^-$~B%_j3f{8A5aBaY>!PEvK-$u$_<%wnB~lztO4_qb6-9o`kR#(#GJrlQra=V8dO z_|INw(czmJ=X+m7*5uwAe_e`((XOujfPWKIHIJD9r4zBHwTtfk=hLoK@({|{AlO|h z3VC(f0XBeg=UL~kc;jT%;y(a>0C7gFFpRWWBy`H-6CkEL`!^zsPtMrgsxJV?&in3~ z8ouRlejJF>d@$J&w)*6q)p-w(ugmW9k=r>ot>wTb;S^}(&C#wLN70K30GY=bA9?6C zY!-KUv#sxK&K3M3FMHF;5E$uRTeIDFCe%Xwd?xJdgy6vc9rW*iRSzW?Bc0kHrhdrqMp)=qk5H0|Cyd}F^c{S4Nn*8xyQ`;m^Z&7e%!`JKa zmUObP23&6c8U0oJOptWa`LJojKQ%!^Dl5~>3YdrJziI!U)=!Lb!^m$~ROD4^TM_P8 zM{qW?gWT)y?d^rjlqon%8W(i)*E~V?_5*f~|5Fzth|c<|Bj6(~+yfkUd5IlVHXX$e zBL2nF09YI^-*0{qghYHH^LdO_KqH3N2gn4v0jobY9bN`kVY2|Nn%`gj1-763ns@Nj zOiurPCNaor>Vn*g=wntFe=6o7>)gJChmK;+iBweoc;&EJ%2!TbMn!dK5_ZBfJweGO z4K%%8EnjDLekm5&DPf&p)x>*l5w;kv&WW4#}5}h&86EfyNAFaZ0xDQo8 z@{mPRX|!3Am9`_#rVZX|Fu zDp{-k>H?r%yDBj_?_B-Ps?40Ee*@F~Ey!tl3gX`i%@?z!W_5LLf8t_e8tTHYzTx98%dtowOe!*$3#j z+AA2~gP*o9g2JSUx@;$hbJ^KbM}HrvXeZgHKjtybS=#Ex_YI8=Rq_*(T)u)%;0##4v`hy8lLuIv%>W;`3?RXdsgT1sQ77euvm7Q(O+`QseQ?|=IZ zx-rG@9!e&M)z&gRy`3p9bMGd>6hSRp^(k&Trt9YB@oCcHh-mePl!JnKd& z%*&g~)3XwJ!OOvqk~0(C(#k$U5=1TUt{$N`C$C1GlXl)QL{@y*#6uey;nZgDM=26! zUQmR!c*9hYv=IQ*Cm`ZX7s27-xGO zz-Z;b?%DjZ*lq0w!~c`2qKUMy^yQi$u0kBC#=q6>euLp(ehNzm2W@sgh#bo_Tcxkv z+l$mB;ZiWQ5a1KXvbz(sH1*c5{kESg(No{ox-^ZnEsYo%Pxi1~uklhkl}t-f&()YJ z$MaoAE@V!7NDH^N6x4Im7 zI;N+U!@dpUCSS+gus81VXKv z_0hXsiv9-x-wadsZyx0kY)Xl`k9KS0qEU#YD`}Gv4-x}JdPi5nk~hOh?>brEQS12l zA{*ESrYJP%IeV z@t{g<%ZwA6xkZKf9@y;t4I<{DG9C2wv2$SQ0VuOA63kEz;n5zEgMN+t=Q%Q(y9 zx7k`s6LxDLQxuYlBB(Cw(J=p6L>0Z7G}3R5;gXq=_q@$LWn)=9U%Mpx=I!Q6uXB9_ zZxdJ*0%y`&r#H+T%iNGp_T?JxN0FoD!tt|vm7wqA!hS>Rih1b7q~qpiwvoZA8V(8X zC;AnK-LRaKKelqg+@?tMqMhw^h35BQo1((PLLE{u99~>?s!(O5;Dof37{mA~Vx`2g zKN8Ay83An6)T^4EEEO5>j=(Vbb*t zkKErA3!WK);Yh9}Mn@>k$eIl$-{eMKgWl&pf+qNC76fYEL@!}4}-Q6Js4N{6* za41e&+zG*5g1ZEl;!bgg0xc9TUbJtn`}tks-^66eP^jQ5{(ShUgIXeENZDI7!f|N>E=A}W0 zT4Y!iI`Sf)N^QBNcmt5wB)n~nzkZas9fOg8(bnMkcT_$_{*FG%ZHYc~;Gji8a%R$~ z2;qI0K}K0@Z4Iwnn#jnKOWGMCD(c==6J52xgqyLjpKGssX4D8GzJHm5HDW2tVl#Yba@ zfdXwBf#kMpe4xGKG%rG2>k~&~lkR96CLDqgq-cMjayWENaiuWUvzQCG;l(bO>LeH; z_!7V3$7T+1`;<=9M4?W&;K00%8_SSR?9_SYjf@f}nTJBP{3e!AQ`uZPAB(6&2I0ee z0h5hBc{HQ-020|=BW>M%9?yV>DZz-#Ky)oAD9s!;*v24z!gs<=$Zd}`j%sA;o+!sK z?Xv1yQXf_NCD*rx2Rc_F7l#^7?TI+3Svjv%TDB>5yjvXNMv;D0%hsV|CN^5#w_(}L zx}@e+scg|t(Z=h$N&SqcK}E0Ul94FrLDE)DhX%Bi9hoP$@gRPznm6vx_{`5%QucUS z66+y`bIv1VcwP3oa<2Bz_dDH3{r0?4`hKmeH?*V)%hnna@L;5Ee>2BIf{l+dXy%Oh z2_v1MVS$!#kIp?kSuS$^Q)7m}wHUrhsV-9+i=MTv>1W~=Z(PG8G7>LNCAtGKpWCccROY3NiC-g0LV}#`_pM`Ap zDDU~nmcgf{{}%Ph9eUf;w0h}b3=`8#W@NYs6Qve2vL)VjR>!e=i=EMl)wY76Y2Y04 zG^PX+t#Pk*ll$Wl=%4uqXAxP`P+r`(3>Rn&am9_q;OakKQ5~Y4*JT-Ay27ugVlJj? zgB@tb2A8y@?2=Q;+LXBV-ZXfD1#~SJ*-+i>;+Tqm@s=|`NdSwNY-fM@7y0I*m0*<& zWYNDL$N;lNWE3yFecIKE(3yg39?I$ZWgvY>0h&G&*E>Fcr0m+>o~QDY>3|#;<7YMw z7JmKu9*utSJBK4`il*W!t&omCPhV|D;&r`v2Vrt)wkh9Xi+Q8TvZr_>rqFEvg)UX< zo0$r=NNaWvRW&H%^(DV1OgxV#iY;oh z414t)Ls<#Ca;4gf_?nuYz2+ZMDn=h`b;f;h80D+PNpwqojVVZaoK!JFKyy4;hxteSXo1u!EwCGx-Tc50c8lWQvH#~>nJs-yc zSI!`XjoT2qjR)%r-xC;Te`#MZeiFH=e1^uttdw4}WLzm9_KqpaTlMC9Ou$G3Ar_Ho z#>Nz?vQ=F&(gj+0?}%HQk66xMHPJ4#A5DjJ2vOX9Z7>EE=V+B%xHz}Cv=?sjIjh*B zK{a3Xo+nWrf3beKjb}FrJHwT%B;n{nG(#KySes9zCDt*BZ-#u?q2w51vtDKa$`*gv z%xF+qsW6b;_|q+9^x*`AvznRAPf|ypK@zUrIx!Kw_RfvD;wH&4*H-DzhPDy5oSGV& z!h%a^`B(QEHBuTwb0FH$(Fap zf2E_1Rp#Wh`#1%Ef34_a&<3C92YVtdoo;N>hJjqR?ruNkTS{3UE@SCGldbqpfpP7q zm+S`NebJ?+kxgSvFmGS_&@rZDUix-!d*&kgrcs^@>6*?S43Yhnw2CM0GEQA~D)F)f z;(kn3y@Cv`b_QB#*oc?HqQfNo2NX5U2ZZMM$;!kdj%6uLYD7ZOZ)~;&*P3%xQ(7@S5_A+s>3Lbc;2t6*O8W34`8Y*{f;Rnv0)a4c|I=uD|lvotz~G z?+~oFJjeK@?X}O<-kH9+D>oA|FIgBIvFgQ_+9j?cE>UxELs5srSW7AWcvpj*JXxE@ zeMiqb-~ZGlZKHKPs3z*pEb@e{(@GZ{FwMF=CAT$9k*` z85jyI@hzYdUo@9)v%O*dDD&=X%Tnup?q8oU61(g!+dV;MK>=NxBg+mfKNGVcuYUk3 zKAke@gGX%lONK)@T+ePoUQfMELoU@pJd(wE=TGLnY78TsR;NgOZ8Ei(f$a-0nS~3J zg;JF#uGlTMW~t2n&Qh((p~Wf3t>}5t;g90Q)o-NI2_%)m+^}@1EoP%+Hz7fJ@a&U< zjYH4hUqS>pWm`l0;cpE;GV+Ms15j>#_$Flu|a0>{qm%3v>vyPph7z;$n z4D^k;;);ipxh^hZ@7bfjWt5Ac#=K?d5^ZFZHai7@z8Rp$z0ZnX-W>YI+A1Zkl4U2f>S!+Sx`#Q5f7mMbO%6OxCai_mD3xC`twkaL8h~a!ZV72w< zM&;UeDlIM3sF||f`W(Si&$+_Zp~nL#UV0(SZgudKY7kj|FL!j^4SqF;*dU! zD9$5$#Pj#1|JnR%($}(hp1e)Yu0qXPZB~+%alobYV+642l~C90r!|O6`bX-C+9uE@D0^z?1#~Z!*U( zi;LeO*89w|koimr;*?*ACA6;~B7N0Pt3EM>eyt=9oi!}iJQ@9^AO>Bqk=IY31^a7^ zr4Bjsvac3>U14MgmUUJZnc6K5d9~xN0H>TtzbRT{PmbIqq(F(Ok*bWr}o6bH^l(4rk->nIf?gu02s*>}e=6)BtV))BQfqFKE zkJc=B^=MS@0X0-ux_IxlY?q1!KKW%OZSWvo*T&xg5M^5fjScO?h(3^tbdm z>brTCKWB-hg~!e9Nm5q^rz{&^a@IO$*qOq_al>ZtN<{u2?j0|)3RNwn3w`c^K!NBJ zb?4)NUNQ`adq~_IYht9R0)s;UHWBNDD^ct3;2Wl%TCi_Lxv5+Nd^7_0L>^8ni{h#2 z%*-El449`A0+!)P_2;rGm$EJ`}?6_nfSL;u><8wdDtI|#$#yP zc&r-gA`E>~!kR8sX~vpZPzh?cNK6$ge%ca!PD)wUJSETP1G$ z6GLx_!J~kJ*BZhZtMH>zQ(ax`4cWF^@AYF^nlI%Nqdlr1rg22Toj^tsMVtJ6(fqCv zBbtv048GJX@xCiLsU$o;wrlHw5Rx!ti;B)RZ?clLzxlhDx~0bp=Np*(A)PVMdw zlH5!OI0uQW{b4@YzZh^VrN;aTaBkgiZn>aL6x=76NAAb>0uoYAxsN{byu&%WKim2v z*$Oapse8P=-rc{xPsDm~4KCf@zv@K?pA1%CAlGvwWUv|FBmPWv8W<=|_XGR^bG&W$ zGlYBiF+{WfXqXbe|N942k&CX|`ki4&z?k&)NeD*R-?Oc$fvwiZ7EBKW{r^PcB;dz; zQUV}D<9ybJeEXMUuo1vF2prh6I}X4h$eRwH4nP7B^vFahWu8QYFo1yF zN$?Il9n22y$prwuUfokYe**Y|-5^_)0j>8p0FPxK%o^@Zt7|C9Zls&8z_i#0q`owU_WpCkh{l8 z?!TfDhj?Z&!eXw3Hm~e*$&7&5q?KWJo}Acy2y))4%hvJA`RVa!aD8CjbN(B#-Wo z@(ua;?k)7cYPS#X{sD9jo}R&%!vShH0sjDom^2+_vgpnU%8Uxm^EAZoYQdWMoX>)2B+>So~`@p?pt`lv+%Q^ zaHeKj@$gnPdmAp%MX?yIo!ngFmKLskgyU=NV17wu(7L6A*!!79^yv-5`dPGqpMaNJ zO9TA@rq3_t8@Nl&?}@M(@{&Xp2@ zB6Jx8qZOn$?COMfgKh}WI9DhH)qZ(Rm+gJ;8<(;0cQsF=wr$UF%l7qWysiV;hD9y= z_Nsy5WTOTX)ofjAb__?_)Kwdwj?mV+kYgpH(&b!e*+g|YW3yOV(d_X~=843E(pq&U2yHe^VLse@sVyz08r=3^2nC;4wYh^1A-n1L-^t~tIhi}Au zGvUX1EPL!v81_qe!|${{yVZcJSkvAlKNv1Bkkp6n2gKVbdc@udpIq-V=Pt?Zqb zJk}0jc1+>X4>!8d-A>PTef8gy6q}UpV%JvnA9lXTs8(U&31)?w>a*#Mr3I?GslK=i zquq$;Q?%0MV|U>VAE8v5IMIVW&y~x{>AP@Usbmm}vP@A-6rj?@?fF$x2l>nU%k`Lg z{naU^fR$r8<6}pRw^utS#DHN1}X*F zWUmmCbdKXpXpf!wC%G4V@ZZ`Cdj2}+sP{j>@<>1lg*Zm{9i0j_I8)QuM~gY zt*=3(2dtnA1Jl!47^@&Chw!kkAFhYV7_xY|NjOCI8zyAH5O3Ndu-jydB=P@vC>8ZN zYxve0VL@QPja;zUTxmy&17W!UbEe00rUSJp6L93OR|TJoS_qsYjPr&pd+A*tdiezL z4C`;xu(_1RrMJEIZ7NB}2NB-we5M$ekF1a-qH}8Uu9@xze{L`AN*XEaDAx;{k7Z#p z6IV-_71oGDAaN~LhQLc}*~}*y=@wE>>k6j>wgj5~*lHyW3<1>)UYpu(3j{eQG^1Z3 zzAXC#kcL*LjV5Jtv;mP~%A=GO04t?#hqEeVbcuFI*z3;r5FcupvngI3Y7Y4cMqXUM zjH?(oYHkiEPE#5na&dD$ah_Vwlej|`0~S2xqIJz&eHDzf1`6gsX-l@WXj;0B(rgqspPGdjU_?@!3~0~aIhP5zeA?cQ z{ntcKS+}cqqT$Od$ArHXcfd8I$EI>p&ktnWAE@fVP~(oX0Pbu`%l~}e15}fb@Z$b8Jlb>A*1K_O?zYNU*IG-a^4isdaN}S zG-!Ws5qu1i)o3(X;CPi2hk%J|Mju~$-v(t4 z{Xk@&Ui(+)5E92P`9ZI}(8|#qlRltE0TMMHMFnTOqy)YsMbP@ITc z+sE$<12Ht?53v>IzbIT2abRhDu3!C`m8i^+mDG6s53qUucJ)P~(3=~zKOGT_*1OZV zoAa@a8RR)s_yRwa*?{42K!7X=jvl%mV*2-c4mlBG9$poOYdL7cO)^8th?n!|l74^7 z1b$z_VpCIXynp!%^{jX7?J>KBc+ygK0p{xKmCW{WfJ(LXU#3PfHpOy`5!~9%tAss+ z7~D}#yrmu9@AQVMZW`xXs$LF0iWp&GblDF|T%$sJn-hW6t*<>MW{B5TbX=4;$q96r zPOs7|k!Z;yPsn4E7b*1Fe8PQKayAo8Wa4Z;s{xAuL4^RxZuRd-JmTI525N~d z*iE)P_}3!VMe0o2MB&wl6aN(<^0+fM{}3OEkoxZ3`AKLcbJipG=6eoMQ^m9ERl`C0 z`eu?1`nU1{c8`9gXcBjYo;20$MX*nwu11a8XrQuoba51FFPOSoszzy|O7dGBk4}@V zZkENK{t0!BmH5rL8izPs=SXD1;dRT8O)W@X{R>O z8JAEM$rhyXt)(E43#08Xikr`IE(W1(h!~ImKRuaX!g) zr*KN3mU_y9;#c6;5`QPl;`)N%u%132>F%O#8^@met$81EcVM=ee--TQwCMPe$7bS{ z;u?6&UFnQOij*fdhf56A($jd|{o#9d9GCp}*{g+Dadqsw6xPB?D^X*lF$fN|CqVuW zD`evK&_U;bK!qV`VANQI+<{zq_S^hb4lC2m@dfg6b#V&T!esq5vA0SKI=IS=CCeLP zF_E4W1TjAvPH4!I&cW5EgG}JG_Z}&23r5)GGuz5hX+8dYsR$;H%2anu>{2v3mfqNC zw(MxOFv}?N{4dYb)Ww^tB5<2Y1tUipw)(0mHo5jF+!&RL67*EfnlYz8_FgvQ#{we zj%f*Yi78j)hlyQ!IuWwm_q-9l^7+;;6_xT+BvhoNI_f?y9M~Jdh@nV9mSlxmY?KP9 zqMXMU2uejiJYJb!CQb9&6%oQNiF7qIQKWI3{oe4Vy08n|nWL#sKIEH0D4arq8SL0P1rI^3gn)C91l`nK#j=v(Rxhi`RG|!gp-D> zv2bFG^R=%=X2azbtx-irb`%x0fgr_{-kmT;7bY#PQJ+t`V<59s&*ZZDBf?c3*lvX} z&?^Na!mHY%E+WHSCZ%Sw!HnpTH8Fo(hXg*%bKcZ0&S;~eUW6K@s?E!y`97?Ug)l`{ zMmb`_$O9Tj1#D)_b&7I8hAvbTbd=nI>E(J*b<+()o~Qz4USK-uWcQl;E&*L=@oD(t z>KD}66%@_CY8V=8Qk_BEOv*8O5#td!);!;vcS~r)nlUx zL)A;^8d3M7jlY=b*pI`jX>MkLy6&JgH_&QxXlrpW zy0y4`35RjfNj-cELR{EJjV3xbtswCCRPycE0YSlUKBmxdqR)hi{X(+BCQ5YlbbPGR z^X8cg0Y~r3hlwE^_t~n^S;}q}W`E+LY48y2Y*C_7+*-Si5=3TdQkUSUnpF<6s|s-fJ0lX&$^u6-Ooayrhv zW*helb&c@SRCshJ{l;9WCs`P1M5UTso_CtZ`jU{csF2w2{fzTx=|`5M9Kxv-f6z>^ zy5teY@eCiwVzkaS1CZ}insSizB4K$e)LI=D5siwSVQ&v$+BNv3YI?x2`VZi;V0&j! zDZxyZQ25(6Om_uo;~lJ`u&Ze*DlS;u@SCe%yfzVztxScBsMbhe{fyI)+~sO$2yLlm zH&*zL$g%dDt(%dZohk*c#kPiF?5sqCCgEy-QcuNR7#*ktcv)S1dAJm|Opxfin<652 zXMwK1B02r0VHdlCcL0tt{_?5;#u8efB}LV$!zjY3pxJKjM(6S-_~`9WTVsWn$cm(z zVK3iUb6Fl|2p-@inzBKdD=`!!`7uBECuhw6jn)4bb;scu#T|50UE4NdSRMUY>5n&B z2O#vvcXg^LPZ0mx`?eVNz3pUyXMf48y&RKX=jWJl;^UpxOV3s;SH{uCpBRzHU!aAl zv#{33G)cxrl2ZLONNehf8MUgO*UglBsMtdoZt>U)G9kqSsi41G747YLJGu3F+v?qp zp63HO_;E(rEeweB=2HlhHhstjI98!rap@>;fs-ax~vLsDi=uz-HMtgLloY=(n z2KC5GD~gyt!)E^g$Gm_bzuDZ}B9Fy2&NJ#{s|pKwZ;qtEcbolf>sgz*xKdH}`q}62Q#wYQu$9X6#0%T7HFU=(lme1FpWSUP9 z?6d~dXWZQRjk=^Mn))k}TRNKqaSViTcFft`TyE#6JA%CGezDt6F@o_YL`C#mb76nT z#MtI5n(pvO$9u4$8qHwDAOo_Z7#>#dIjAb942yrNXB`f(UgdF#KVIV#{Y6yw=ks{0 zAKJ`jp`MX98!45>7p{CPjVioI!Jq3PitA>N&M?Put4HsRDc+Q?`#pAKoJ!(#PTwo0 zTzNC71q#|R-=<&XaoDhs_&Ksv&aEWQMXIWmJF~rc2PuAqH+jn{w0A2gyp~MDSIhDz zk8vPC2Sz4TC@HA>)npvVCt8a1X|Os)X%Ug?`~0TSYBX}Mm1E6cB3|ueoiEM){I1MF zKScs>$b^-xA$4`WWXVx~N(Y4=UcUkA5DvgJ>p69J_S1|TFv0F(MCOG{)Dvz+1%8$qhB2)30FKlO z+)tjj`gM}TY9BLaeYnQmR2!OA&Ef*BGLMZP(uc;+gpI+G(ZUJU7dV_3QY=_dY8G4D zH$y)a7D^XCD4==!XV%AbthGl@+}B;=@IPoSm6I63IK~BwWGjYs7 zI)pHo$69Q$ZNa^i&Y&|8d~iGHAoND#vtC@nI|zj1<)9T@%GX+-GLn4}*hOt%|Csice*hclmWPX!iHasCNr#bdOz>64BwHDx+wUxm`NNUv zv>tmN;cdrRH5l1o^&^2cCXhtIQ$W5WpZm3@NIA1k&D6>fi#Il5E`!5-IZoRp5P@0W z2KeGKY}qydJx$)|2z|(ZH=C_=C?K(ZEOPWYepb@M6^8UZCkJ_fgL4pf=j&i1xlEsw zU*R1u$PP2BpqD4Z-t`SnfFty(yj+%MV)5o-P->w@d`j|}t2Ha0YBW$Y<&E)5Vr4j- zPCQ}n198+StNnM*Ma91km+MOT)*eUX`d@-Qd|`AqiUtzx14RheMJ2SP-ly@{Bc!`! zqBYIP>!n*M5hRV=jn=u9;F(QEHPQcT(1TGqn`#Akzp=%NNzz9ntga(MDi}-=fLI%O z8g|L0lXxA1ISn9mIlI_RC~Ld!MbA6A`v))qyzCwZ-~tpPuAh;!SZ2U0j2;4W$;Z4J z0NUBno@B@elv^M zpBcS4PVAHHlvghrQCYTdUOMK!<&)G9$^V9|10nALmv_OzEy92gC^z>p31!{?0BLhh z0ELi8%(T2Y|GN*U!Q9+SsQ{74`w&hD_cc1+vwkl)n*Z+0C(JLH6E1Ep@Q$+&uWQQ+ zrbgFzR7eH)n;*>910Fs|AWt9bm+LZ?lb!zn_^w3%(~c){5m3-*C2oyRx1{_btVLF0 z5+`a7+<}iQ|N4C(>+g4w(x(4It-tK9MRjv2<%Ahp(GcphD-nRJnbh*mr)L{7+$Nc} z3#jo)@_^8J(x^RC=fpjI>h(7_pefBSReEQbqcmL9r zEA_sB+Z7)32r_YeBe=JIfjnAM4m%i3{{Tq8t<2V`zu7XXo?Gt7E!pKd12SyOZb&VB z*|zJ#o%(;Bq+rxEV#Pak*&15fOm8A0r_EhJIUaht^M(%a2Im|?J0-d z->mQAgR+CO*(wmN$T4*pq)xTn?MIL-{*T-L4b&6=+U%(YsgxM-Rh#08X^p~+yX}0_ zM$~vs3%@0c=3(q;KqnQ8C)mKz+N@1|#4nlOTsiP1yl9BGj(WY&B07a6Bzf4G1UlhN z)xT$p3cgBu@ifjsoxt0sdy$2K5E zA}hS4YS1@K;9B83@@Sbr2i^7;BTJ>BF+b;wbc~&@+Xi~ch9at-JH_fs;7-91R;pTF^E~SssL|dKH#MY~oMh2&pT*B1N z?c@W!_r}ioH9j5_x9_r&KX5mH6Qca$rlHc5kYgpt#Mv&%$Gd;vd7;OYZ^YRehEDvd z+UFhGo~&M3fy68}c61~c1Ky~uuTxa+sM?5iMz@}ptlvWmZhe}qdf?DJu?lc8>p;8S zmyrv`0jGXxXmzAH0#?RJAZMRZn^xlk=TGg>OFo}f_qFG~%RK5?JW%pvvh=yvNM*#S%Rvl(#$=K1O(cci6gEoM>NA(~{lU=~@+ON-!bKS||PVAZk5`ABOoN;Ix4 z3kn0MLSgx@Qc&7c!)O(3!R5f{$Z>ICeEjbU8>1bT3{Aczd>$5gI6dXT_zDRw}$5alx16sf58vB>$tKKlI>FfP#Za~yXY zCT?f|6{@f?Dj2k_KD-`gAo?_fr=g;s7fkpa2y4`mvvS?oUZFh~baAfH-2Z|XwJqk; z{P~dw!cNdts73$zx5CZYR>JfWihDxr4N?w}_^>uFjJ^)xbP;nBmgovvE-YSH{`i#n zBO9_ppp5^mtdSux{d~jMVVe^HFIVgZ0RFbTxkSJFsBUU3s3*@m{Um7QSzsR>x0SQL zvgGUM^CW88iMyP32_2BI%81^-}kz z(^4(t?+vrWE^dLCBj?@yj$gZeD)fK2K`O97Bw@KR8}K3d*oD# zxMSje*(y5b@sIX8VCY0RwTj$?ip6|&%Ze961i2TEvxPjX7+Q>XN=l8PqPJ05Gmf9k z%)Bph+Vh#OuK$zsWTui)j9hq zLl?itQNm6{NK-11h^tgC+eNj_W8yx26H)E$-i!R%u;EYm@vyRnP2Y+D<7RAzNVoLMt9l+WvUFu0;FWOfNU>ty7r$Ysdc< z0L$Js21BYE@pEJ!kz<70MF7fyfL4M;i6xhA9yVG~f3l0($1wvIBtcS?jLNlNQsu z5|hdTfr88=`f3k56HgC|F}SoVVF;xr_bRxDE}QO3iL_Y5t&x~?ikPab`D}tMuIjBR zOSP~Bq5AO=FUQi)`6R|Zm2*&YxteB?*DG>Edlk*%hQt-D`K*GQnS#_D=ZfP@kdBrM z?`KNd9Rx25$8*OEgMK@h>Tb4sHk#71u^%9RsghN=vLc?66Mb5sPw5nvPni+ozxfgeM{>EVr@d7|W z*uZN0Q-Et3F{4V!ionWzYe#i%d7#qw%?bM*T}7^O$G0m)$LyWcJ(LgP`=L6QXe*i} zvOgApKS78)$G(p8cZ@2o@O_y&95&SvXHB>TZ8M79@q&s>quDam5_JRYod?3&7t+1t zu6^L}vfRhU)NQ2cYf{FhwUu(@j(3uRyaHJgP53`RQ$+r$XG?|N(uY1>e$lG11y&qY zj^!v;Gco5cK8*Q=vk6{JU8D0nrNCHzpCJSV>5T_pe^mVN@m{Vb*k7!0YjJ*UR+Fp8 zdNyxROmSx+x?F&q{-pyN?29OCj{{}lK^V9ET$<`zdM5e$Nug4tW# z+&DHuF|@Y&S#cypq{|SxKBcH_;W-s>B*ga1(Vw5XMw(6E9IDnd*tfo4LT37mqK}Qc zD7Sg(LCjXpp&5*DK7u{5}&E9_M=}>B@#-?XkHuXnWuSfnS-OodA z-GSU~HAMnV`=Etn+NRg5mBAVPu|qErt;`9x2|*?@^f8aMK#XBsq1dr%WmK<|1;+mGDA0)0t zu{c+KCDF70&LM0;?HtzM>aXdeW^2qRLX+}&Wu<{Sv*wnadVe9lP{Wl`U1@?uB|eSf zcDY8@zJYkHxFCIIcUrA1<^fJbH(aAp(O1&L*|Y#o()y|x?xJB(HpOhk;GHcQF0UIh z+RAH<%yN|pu0&6rMsYPjrXeM^%+(ChTs3gaMxmYzI!^&HnIAb5E#q|45X%-u%~7|5 zaM?#+h_aq6vMQ%UYIoQ>yJ-&_W&nmhnMk0%a0ruRb1zDp+%a}}AO8XhU{#ZIvk6+oWr3N$lDFk~@}eqUJ-U|4A@SSW$oYD0UWni;## z(Q4%#-yvr%1j__r#PeH` zJ`#)Wx`!Ca)gV>_1Xv@l?gLcV!}&3XVjM2BO_vohtBUwyr(6-+*QZ)y$z7y_SXDs2 zFBOA4e7rLI)5^aU%gM+_qSpWrPXrD1Nf`)Lw?1a2#4kjN2Zxr z83l%rQ_Y*d^{983T)MEU;SCPlM3hV_gX?br48Tk zC8Cnft$eZO6|rUltCLT^W)w~|pK2Af2j}Y`&d{2gX(E5!xyUizO*Rz07*aGREMde- zvC3d&{@Nz9^5chR&6RQJw9D|w+oqb3N3B+=8cU%73_W2leI4~l-Imv)Suu(J_B-G3 zeVxci0!~g5FE6=No*~1Cs&?%wR%LD0#!AzwroB3VLsFNJYBtAi!S018}r$t2$K1rg%PQ&AguAzZ51);Z*WuRg;zl z{@{f79!(;~AU%Xyh{IfnsMlBuEkXCA=V$VF3!Bj5iuc(kdk?O0P^#_-{%8h%wF{6KzkuIxJFh!8-h_J`MI65dhm5SgHx+DqIOH;!{D!2E zfjX>Sw(|)O_GyCXzC?3YOk=u4^Sgf7HKel{LaKTJ>>mXLrxi8s_?D<$sK_R>1R>Sg z{{VE?h5e56wl?EK3TeXXaMWfBdRY*03K3EDs;GW*0Xv)X;%6@ll$W0dtu53nmTOBg zcC2@loWIj1Ov1B*)F^zA`aL2oR(!5iZ{SFs{*u`1KcQb+w=|BuARHR}B~`X>qmMK^ zPZM0kU*H_pSR}|6Or+2b@TX=PN2kgHRDm*VI@;2$?XaSD>3JnfnO}Ok&Rb?9gr}=) z?Zrp7X_Vt=h^J30q2o>w;UY0Y%11Q1Y5MrBon1=MpKLg63)`I`JGhx8Dd3&lUFW|q zdLy0W+2mZ(`s(q>J)Ocke0wM&NSu|~Ie2R|l;W`|Oh2FYiGi03DB%#N7%N3%bO{XG$VNh|daz1r8kU(F)j|L zCu%tS+y}Tf5?%ZcfQS8tbUuDd?d#Bu)XsIX9(l*e9S3ut<*O&9c5$W`#*rht+gm{e z4NpST^#zjs?iT1F#Q`dHR2@*xtG(O$T?#7d`)~(-ez70GK1SXsuqW;|S+AQ@<0wgI zy&Zx8-Hje*qvX@?JQFM6cvV|sbof;_v>!*?>OdJw2ijaTIa%24aWy6T=nsuvM)m&zqBs)2kTk5&mCV@wbTCz&Eb$4vDn4@)bK`xG zU|e=#u_p?4J#-elD=}b&Bdc~J7F8b|Ol9zd%MQV*Zp$u0yeXUmO%9-yNd);&j!Iic zr>#E1Sj-aHBJ!wJJ|sA#b8pFH&%E#Lw2|q5%lpV*5cR`2UV$uOGNCC?U3Bd@&B(pL z&>LcTFc1})pg!F+awbH?UMoyUo|E~Gb`(2*p2|p0#~ggYa;+e|`c4b9%aw(r=I7(@ zWp9#sS+P0s!Vl&&IhYGYm}*_>-0AW?8dz+a?hK7mFM3FQSsyL;EtG|ru%qL9Gmkr7 zNI1P<6PeRUuPD+gKN{|_>upk?P+>?7Z;(9oI->xmw4bG3VpFJ1AxP^IiHW&e*AM6) z(1vRSU+_0(xx`j&BOd)BJYhivTU28mjaw1#H`-mt&VS7v#zp*4m5mOeeHRlGbJqN@ zdLzYIpQP5&Vf9evpU$d`S#5B1p$db&+=!0Ok7aaZNbki1 zF39H0sEEsQ!g`30yjnoLNiA#s{bkD6t25Zgt~cz~>C=a|sk|qI<)Wt~XLz$a1wT4s z!Xk+B3%XLu3}3lbbb@Bd=Z9ST{+c5s`em(Z-6t0KLTe@Jgyl?lnoIz`1PGSs=D7s4 z3lic)?qBy$ew3)~zFfYXU#c;`D6~lWpIkiQqB|b$9#rY(pWL?uKGqS{&HNXkFMx!yBj~rwE}C0{hxZu-(r!^wPZcfk!v~9 zh!M6FT(1F;bLhAB65mc*@)jS0T6^)V-OfIbdot1QKYf!lc{W6;7QVfEn+VlFJUmN* zV^P}!X-C}%UCeWbUAUIEARB+sD}=PLw5;S|XBItz%kdZwv6!I^-6rzrxxGlbt^Zf^ zb2<3 znlt34-RS|S>~~hdPqu+>ko2sV7O-r+2(`Rx9vURipuG- z^dY$R@q;*{g7zwSypK};K;q`^>o5f6dLQMN`(nURNDKwq^6djsT8H`H&rJFMvB|U+ zKY5ffFz;M8`-2`<{{m{ZYR~pVUe>KagztA){;CE0hGZ+FA@?6dkU{SMF-m9zNCk|~ z02<*_snh~sP#PET9))UpB-s4_Zwc*1BVIM_>BYeLfeclfr?O$u?{Gd13D;}gAkTOp z(393}z>BsA)9$Na8?Kht{up~(=2EAw-O`)Wlh?U0;#c(|n`n9{}qg zfas8?*8Hdni8=HjuMyxt%Zn(De}JRw;H%-C>hnZ)!{c+F8N+2TL8s{@DBvA;9bVFb zd*#zHcg$iB%GoK;r`_ANn9c}YEnw$I2!vRg*&ol+B1o9><7Dj-~s4)1Iq^r4VlI?3{8%*`R1WPpI=iZ4zgSSgnli*5qN0agkYM|<@ zK+Vp`ERm`-7TX&^9Sm^Jt?-~whfm}tr8ex{Y$#OsWp6ZwFj)*4S8GQKN7)Ld$Cj+d z^TXrGuNcNvu(-N8TmN!ke@;(~e)X|3&9n90UGnW9!K%lT&Wez5cP_(V?UaE|k{&8L4U*Mb;=M{sbwWl_jw znlruXgf7r16excmmvE>r=T8T9S!Vy@*mH<6tI1j}74bSH0{B+P`&3MmS zO%jE{bNUa^hB#U)K>r6I{Ihe*kn6we@BU$sk;$I*R>`c82J_c5&>C%n&S}%WLZg9} zyfVNA2L~f@F3;)sJoCWoJmnyz9JGpE#A~2g3Xjxi^CXgsAF?jTXGp{k>nwJDY+Y=K zJNd}TI4pU+*t4Oz>?zA5 zq3{!xLgiE|ZMx6W(M0SqD8q~;*CKxy42yAaoD_E*eQ}Msw`I%aw8!(>{$BvE4p8yxRko>(#H`Wc zy-}rA>F>&>)aldWwCnMvLZ1Qn(&I>B#@bz<~iW<7oKYyK8Dj|h50SE|3&+NqIIn<;e(vnmsvmnG?M%%37+-H|d(i7R#I zq{ww^b@$wHPCV+=M^fyC{%-wtk%HGc#HTz-h(z>*1d!N+ow}|#5BPm8S zd#Ib}y~*H9I-Za+?X$x~$kx$*J^RV>O+{pQ4nD$cx8qkoCVAVN`51whAc!~j9dP-oGk&ixe5 zH`rrE_SS^0_u1nmB$UlrWa>&mlv1KS5R8nkBPFPej@X<==sHY4XKtS&HJVJPEAr$7 zfP85`y(&uaT+|d)tL?tpKZk7qCmBaS^=q+QT1*SDYA*d_+Xh%PZ|@dP^jiW_u4GQm zux3P1D!Yfpkd&ns?r(UDS>pPGiW?q4J{k&=ER`teIQ*;mqec_W!ZJ+LveaLFvT4*vkdN#D$+H$J4?{mvr4i1&98;?2JI{qG@!I?&ob za!Ou=>kO>w(k@78XMMNa1HJEId_}L^p8oS4E7^&e(@e_fEBw|rk2Rq!adVtEBdtv> z6?AD@RCOm;g0o^tAZk%nxC%EGIK~+klemw3jwONj53*{d=O^CM+=;4mI~OwfgOQcx zt8OFf{j5}4RL+ZKD&&|fJyuOi+=TT72mRH^0z#Q3DhVm^ikwZ-RDz`h6on`p9#s9) zhNFA$b{x0#U~JT~wiauFHu@jzbEtZPT2=nB5f$*f$|LgqL>eoy5|4!`d<2fb z6{l_Sh-2{ciH^@nB#}uy5=sRFy!8#P%8s~%0+m?!5-)HFIvP`eDI^rBB}p36pi<&) zg@quLo3x~=l#7$#GA=&`;_w?~kyGN;qB5nt5~>|aVR5J4W=N45oZFGqiEOysrqIHq zI+mGj}H=tvDY8MQKpQ5DwL884S)27~WBSrk!^}NQn%V>`c-`MOF!QhO<7h*rIVbDJ2*)WmD$rJjbN;5q=^fwjRmU`m&Q2 z3>ZjmdCn0N;zmR`>>?vcxLP7bOEBAYExv{l^KGp%OxU(R-pugA;*i&9_?}R`B+W`E zdR67!jb$lDh9vAoQYtewO4TgXqnVkRnU6oRq~{@jr;1}z<_Xa&6nD{BqS5QkA{9?E zJ-WIW_rH8r@@ z_~jJ3OE9Ual4VX2MCsF{NVRw~s^E{*IP`A$YAm~kKlB})50`qLsHkd<08#Hg(VBgT zU0BPCZs}4?XleM+g{TK)QP^-LYf6@lZ2tfaT2w9xzT}f4`2PTzSBUyI+8T{4Tp)1j zzGE#*X%6-4H^CuDjC0izDpJDN8P=MuRp=DTf}_cfrliJ@xK&xK#+Mz|rBl?VAE&(* zL?}+jpp!GCTO{U_dUludR_Z-vc)FsSG*7d~XQEk?W>iHk)fQ%w(Z$W__0?KG%bsQH z-z6$!X*C&=iAggdw>=|ubqa(S@`x<10v|1~ltf4_%*%kVk912O946IKsl6Va-k1?& z)hd#sMrut;bFwx@3NvECp_$j|lyD@I(ujzxA|)A|2#BI_fajTB$(yr{Akl&n%}kq9 zOwf`dHbjaUn{%V9FjQErOdx_<*3Cj2~hP1%-5zawxK88T>p_W_)`t9~S#YV8z~$ZJB14a8k>of?66# zre|SGjiUC<=t`o#f{~=lVUjqzZW>|~4XjGENO=ix-NYq&P^5&UCbL^lcYB$5jY@-2 zlgeObQDHrO7Kh%FjyKwI(N3YE4a!?+LINBMLXc9GC&Ln=fY1sG31oP#5XyA7sD)_M zl##AIkrOk4Va4iv_7AEuN@-_oqj}u!uG}>WjOJbi;$v$%cJr@$ zICT-1R;9NBtw@y(Dm%xKCD^_@t*BXHPIzdzDk&EDK)Jr@M;V<4^%FmtF~#zg2`_2+ zJYX$N z(Qk*5tNDp0rcm^#$~?|yZlxHA`z%Q(Epo-0N(lQ=>!MbC(VXwJhnqbsTGi2RoIF%Z zt)ntBh2P>!QS+sBmJkpAEi{Co1F}*vtrCpg93p>({{V}HmRLHfKh~7@@lL<}G0bM< zlA0JPBwkc>SP}w5jUj6&P3+o=t~H<~VFgWv2GpdPr|!Ey`$dDn`}6)A&&Y;5sBhvE zxzG*IH&7(l@&))DCw(AX;*#twz6K<_3vYpsHhsP-bqF0StEL>cqlr>!P#^HsVz_f^ zuwqMNR_a0|MRnF4C@E5!acC;Ch&boJyJOGm5Y)2$yH|XYeWIXlgJuVEn(A#~DNmGp_K`y+}DN=4l*>SZCo%SQs7tCE<)Xa@cl4fzo zR=I8(B?3g~Zlvr)#@~ZsV-fBrN%}d{i8OfAq-FOWk{CmpoUP^+AgkdwQwFWzC|sMR zO%$Hs;?Yj*QkgSJPhP!a>71H&U}$D$%_joBK4WFF?o-Q99SuXGQ=_353Gm|}ELkce z_=Fb;1P#i`+nigG5m-ZfB$8dqsX|9);gq3MU?dGOSsV6GkpK^urac&iwrf=SXTu$W z;9?|Xi-ubtA?kZ@a1|71ROe-yqGPaav?OeViyIAx7_OA2u}ahL?%X1k9b=u5hiLAY zN=m{=8b*_AdFsEZ;B0oqL|S`1NKjwGd%MVY(A#XdkWi8?b|7{b$TLYA3MI@4`^;iq z4B;5v9{&I$>>B}+DXx%F7QJ4ns_lE#0+pB|wf5B0L7e1Okt5!iY5xG>uXhj$ z_(t1+2(`}#10ph==6oPEsuAJ4tt`t&Xr@+tFvRgA9%c0*31V!NGLYI5vI0}E1LN)V z8}qg`3Qy(_YP<*B`5b1aS5r~M$~NxvF{ph~q*8MZUaZk8)W{W@Jx!PCwJKdsBNJ)! z>GH{qDje978*zy-q%h*zd6$s1I@*d>2*w5KYzVdg0QU}wRVrZx6icJtWAPW(*sWwA zXwI)>yrN#C<&3Y@9PvS+QjsQ1#_9B$R%WQdr^8H&gR4^Bb7j*6cRc;)>K#o2c@3{{T4O5%8A&^BjHq zqR`Q}s}Mq6X=*}>NKgtkAn%V8 zV_lvj2TPGD-L+}J$hUa!AE*RwkWVCBpEHdJV1-AK@+ayoZ_fY|fCm@=U;&8!zx%!* z7?23!!?zzx8~`~c{f^$71NqWZA3^^B%73Z-0f6x(6C&H(Vx?>e?l&KCfF@@Pjz`#Hs{zOB z>DvJjNjv`l`M-%E{ulr!a0%q|^gD5Y2<2oR0RG-zGxxwt^?%;*5TG~oCg1*L{{W!C zNhWgEMIZgRc$g>vQ(@owkMDpa%*jzZybs7!K-*!t-{=jn5Gh2<4-z574?iLJ5swpL zY%bwg%4M|pk&5gteF|;qdtdL5MLUaGDm$VoSM(>I;@10MBAZKWJ7i*mwe7Wtx24i= J_qGGU|Jk?Iys-cP literal 0 HcmV?d00001 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/images/dog.jpeg b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/images/dog.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..dc4b40e2107bf4fa949352ad31d6cab5a2ea18b9 GIT binary patch literal 416580 zcmbrmc|4Tu|2BSM;!>6wd)daAER~S5Gc(pvmS|HdTenoUkey-d$rcq^My1qU!hKr{ zV-SicgpjdMNQ1#x#`s-oK&-#^dg#hf$OIp=w;@AvUOj^naDussUEPnubn z0UR6vzybXL+v6Mq7RJVor|oRaEG$j`tY`zE1*Qf7o?iaGcIHM%`-_*5JfnZUv9r$E z)z4?=_5ZLyCzUi-_>73#16{i1!L$)SxEf!ZS1|Fo5M+Ae?E z`aA7Fe;3!3gXhiyYbtE8aq@`iq{10KL% z01~hOyZ~3=DAYOzr~&GL=JpU^44uyZyJcsP|GN>OZ*MmM2$1s=*H11E1i*>l;6iY0 zHv_WJ@#fkCt^Qv-3vh98!hk*8JRtAheNYEWIMjg?#>EZW!v(#aaaHTOsG$+)?QZ(3HoX zlu^EdB4hFzMt0DIXyQM|ja_{2cZ->vdD?i;Rb1i3*=yr|{t}9&F&X9Cg8&~F2eivv z2mk{lL~=SHaUdY$<8|%#wd@Q9H)d<4{CKU1r36nVW5c{d)|LnsWDYG$WKEIFvsRI9 zO?Bk8dUe4y0=tXdW>m41TJ($HVbPFRuSgf!2GsS|w}I87d0xF>Prtc#frP8W#iB&; zYGh6rb`G)WJiK>k+;HJN=Z$ghUsSwL*6^i%TUk)sAa$)CK01H<|)G=9Wn) zsObGf(rFblzGfVqzp>dI$gMy{sqB-f-3DGcQmOp0UhlR6A@bEV`;|2k4R-j>A|~i% zO8q41Ex5mmj%JN@TTh4EFo}bsLI*GOq2Irkt>2saw znjnJi_(Qc0u4w$wEOyx-&~;Jl^{rEV59&gTVZ!ej>x3}H*R+K#r+C=RK(v9SWY~4W zd&+OWh`As-56Lj1bRNv1Nb4JCddqABG@f6qMdN4#yuA0Vc|rbGjGSxL(a;sj>^ci$ z9|#Q-|I@}Mo9*)(HV^G(AZNfDsaE-{bX3|iUjQ%}juuBPmS>mm7s^+BecW8rJyZDC zqNw2;35{b_Cb5LGHup2l8|K@ySDSNrhsGloljAezQ33kQG!q6pWMJ%zaf4n^ zjh~u)$Ag>senc=JW71%6O=!w#gHoJe(k0in903OtFDH7pPFlrvNxECr$&7xpZ_WF; z&XUf!Xi2eKk6zOzj^5UpbvJFPH+x*Cj9S%9{#^fW`-`tMqEu%3CE;5+;r@ePMo|rZ zr7J6|OXTKwpm&4ZDZ&}Pntz)p24<$YUGs~U485~CMFfNJXnU^JkYIlb%(f~99`MaG zc3iM;P@)6LeuLfIMBk zng`stJaIrag*~|qcpI9)OLYY!DKI6IU+LA7{2dLwAfc_<1jcx!UEqJoF6mVi? zn({FC!a(gCP7bugCr~$14rj!xT=xt*mQwaIg{k<|JevF9WE%{gg^bVQRgQXji1+?A z{xf2wVk^{#Ad1MV#xYm+j__NU(*S=TyNrzCRWO}2!78+pw!Vs|WmFHEq?y7^qFwml zR`hZK;!(Gw@-PJMkV3;!jxg}clG8<%)(%5lvdqk~5Z<^KYTZ9+0jyA93J(Z_0eLAu z!jGyFK|GfeHzp?j++&=Vg3noQyB?Ge}z75#bFHBO(s*H|= zE6%XhOmp?=%$s^JVfLeiF}mRI1P>uzYDojBKpWLu^Xl=WUi8(&{SIkYQc6z#{Pjgg zFuN+RDAy?w`4`N8Z7au*0@E5^TDKvo)|KM;?A+ud3P~eA#v+d@m}qwC2Q&j6shciq zc-?cs_-w}e$P=3E7heVvYgvY;xxs64JktT{KUH1n63Dn_EJOWM-w3#jUKXndQ3;w$ z++imEv`>LkOHDz-Rn1$YQ6h9b{x)0gzGFC8QgYWZpWaQ?59XDXKQ}2NXT8u%)25RR zJP$VZYuqlcu*?0v+2#e^s#+yEtZe|c^#mQ$j2C~GX;YZy+4UuuBd+*T%t}ei2&V5~ zx^`5<)74SJGI_%>i{NiGnpFCOcZPxKl-v!>{G--XVP;(MF!lD@gM4ZPWh*U-C!ez~!-9=gJhW4&Jhzo$H9jm zgcp!zNsdcCA@u~oK9Iw5geNy+=^pZoZI=Sq_=0iU$!E9x(Fpx}*)n18#xD>I!<^#& z1TUQE69dER>l7HjMT(o?%eB^#RSN8w0bw(hK?IGmrwj|X0m2Xrer&HBJ3#RkE1n*| z8=2$AN{tW(5$JF=BRF$Ced%t>zuUlYN!&INu7>?Vn7u%qScr*c1w9}y%IaBzW*F}q z7dEw8Qn}=9lnii~!(DH>9!mcrd{{_m8~d6vN{V%SO;+-fZP3C9mTMluB)3xQ2X4vx z(cW=yUJj<0E3c5Kuj;l~wWYbc$*Sn}R^W*e!cp4`GFS6k*sfD?6kD3ZT541*Y=ky` zJ$tUAqz9&?P+@p*MQ`xI;NeT}Iz|FFCA@cow8eU2>)PhpnblYe1=z;r`R?XWn(= zx^cGe-FtrjnAD3?o~){}x%zEDeCI^SNxq&rwNxj-BqvqMxT$b;-R{UvM_PJ&Xi7VG zR7m($_dfC&)w`ND) z_p^rF<6xQ)H&u_J=T8+NN~A*XJn1onZb7(i7kD|$Zk4(7saCcCtd!|Kjv zc<2z`*X-4l%w|_3YI9?hxvmw?(v0OD50|WG2dbGA*}Q7HpD*(QeKIH{D?nNxPPDdNX6;dgU1cX0GD4A<4NV3>lYB&M0DM^PnS@|y%T~yj3C^9j%K&5p-NhZ=D{B3wx zM>N_(mQzOP#Kzg5Iyd(>rCXmAzIEAuz(y}R;zHeYgCk5yWY}Z=d@=5APCxcA3<$Bf zAMRt^aDCchF?M7#-*29P$rQ}qC&;VcP&P*^zdH2$r~s5a<(+tn9KEhjheL_~as2wlaNSPk2-W5?B01;yXXVu(BgYFzFU~eOn~l6z0OO=GXDgoNu8$@n zl+Of3jyV~;Jq*9Bp&lssOzmyTJL^8loZmJ!2KC&*qX}b1s^KMFqnfheYS1hpg48a* z8E2&ab(h^UzO#OR+f99J_Ys+^y?K%LwgdordhRYZRQ2AwoZrMU-y{z|ZIK)-HF zXOt)nObx>$r1!408a4c&|yj7Rr)wu$Z3CO zLMcLK4SAOz3qS!&6vn0g((36a7@RMvEF0Q_n8N+MKw>rE&0LxF!R-yk@;sOa=wb$Y ziVb{lARM^|_BYQ~f#Z#7O~9cD4-gL&TAWNEit4}RZqEA2A@k1j7$|oJHdCP>*OR7EXaq?tAdi{cDr$v2U)rDjbwG7*GkymFn&QH+Y)i*7aEkE&vHyn@jiL#wT^}D1Py0>v$8v z{T7m`Wiv$35^m}Q?S9vrVCV*<@Y^}Q_GOH@@5jkR|?m2Sa`%$K6_sA9g1stC;?bC3}S9Eews1s2Fn}HNkS)hOo!(4p_ z3W-{rOE%feO7P}=uv*kNIDZ4XOm%hj(XnhR)0nhrf~o)Q?rvOhY5Cj6TEY#fr{-ns z7A?aZr?|OEmdWuKE(<`pEPgFylnB52#mNGR71p!QTB;rz#qo7}3%e(oY*>0i=@Fzt z$B5sm5CPoa)j(-RBV|ZTAxqAH=3Ca!$aE&V%ui5nYcPcZ-51YU+M!LGPo*VWmGzQ1 z`Zq&ev}<^Y(B_G5t#6F|Z6j{WZx{3~;lN|XCy_3b5d+sbhQlJhbaJ(K`P-Haj^xo?-e_qKmM|;G9+Jq@67YO9%hmKBBFl?Gu94FmU%;uUGIAdRZf)@UUHBuA;dO?svGe0+G;7{4nTl%DlG$W!g4y{s1b(3_^6SH7V68M~%O2cJ%~YWW(BQJ2&OBZ`)sb|AF}o#sXFpT-Z( z^|q?M0HHVHz3JZhQ|0T%aZQ6uFJ+G@7yuLjG3jr2)Yj7GzknAL$(ZCQO|q^2hqh@o zqxS}P77G*}3>!WzyeZc->0RzklrrS5ywx}wdv`5#i!|{HDv1)%-c@J3M;&uNOiICV z@~%?vLc-(=2Yqn+h#*S1W0QbsU8BH+VF+w=C+khH?-0x3W#es*5)$UhJ$TtZ{YaUx zm@bUAUSfh~6Ctr85Go>i`>%_RJRpjK>3$0p8;n#hU!O$t{m*^@lH;*Ru!zkZ5#y5`7V`9Lw7xSC9*qr zuD$4rb@>6~?!3g+&CLxd(7Nb7eU+WI{Gyo|%ve&J--W}{yKq=w7Y@hopTtB>hj4U^ zoeeVm`&Vm(Phsg&Q}(1>(YfT>)zXU9T9a0Xt_#ogFDKB+aX%JQ^0Yf{9y6Y+f6*9u zsZi$?car#Egpk4!m9pWH>%~tmp?%jfm<;zgP)Jpwfl`(zi{NCalk*;e!5LzR{@dR1 zq(|~lCDRa+fBjL5K2&N5;In4*5=g0;o-_kBYHmD=+VI5dh;A-NW1}f}L!* z@Y+K+JoMK>HS)G~K!|bVx30e7j`el#J~g9t#@HT!4{pjS=pa#wX$Bb-INRj{gD`(# zP{!5kHn-D!d)CS}AP+PCQ!|q{FU!Lmw}0ebw;$LHAy?l5w~1^R!t`a-%a7c! zpp)S_AikIr_VpI|svtu?@^$*eY!> zft#@dr#cOS9ovO=>lQf{6}obudSe^-m;aDFN#(8^k3med{$yC3cx1#oR@+)&GbSq! zx=T0DYy%$WpP^N=zo8|LSxR!rc1ko<395DOiQKqSk5D7sPLy4|ts=`1dX9;d)2n}R zVDE3i&Y=+g5_;Q^fZP%A-o%&;#Q%VU^4p1~~!4RjqUQ z90p;F4Ao)m4mHvJ89Z(LscFKjvDUbIQkUKy?xTY73}G^98@NfBk9RjGU5ywex~8$$ ztPgrG$hzmS6iE)EGlmNWRoWgHhTS&sAcA)rc&B8Z8{#;IsRuD9RM#(zk~W{b;uSye zA`t02%(Yi?FmTIOBIIrrU*qt|#+C|SQ?^_VGm|;c|D7rOomoens88Fpw0cJw>|dXs z^l8gs2oZKRfIJ^_(7^2nSeq`sY`B^7LY2)X%kb zszikh?N4a<%#pEKfr(R>Z3oUCM_Qu20;k%fnWT%H`U|Etd!EZ}1LF{3k-L1Ln!i>K zpwk%SzQr;cQ86x4z?}X?@Bx$@qnVp`Rw*DWb*M`b9%EDW8{8a%?MbO9NeWLImN*bQ zF0k3EBqlj@D{5?zAt4uYo)~}rYi!A>vx08l0^-u=Lg(8jR>5itzkKm6PyI0BA-VLP zzCfA*Ze&Z%*d<|*&6-AChrt`uteLeNIf><01FF*1iJ(vhawtm(n;%epI_ppw!WV5h zN)(Z(OqcXc;~w-zv_(rsqd=7%u<+P&LMhGeq=ksjMAqxGm&FKf1zh|lF^cC3IMiGV zWXgnKc^%m1Lv}&g^jmX3Y8^7v_G4dG0m{dZk5&FYY2OpNkH;x-ycEZQx=^Jv;9V5j zurjNXunk2MwtYhUMOjbMr8^@MXV1I~7-3R#qhW})fzg%RdQ9PoA+woq=^t|^HPqv3 z;8{~zbLHW@srEBOHwf_I_npvTekT{j81mTIknNk((xtp`!fg($d2e~N{aB)0WCTey zg;?JA#i;eoozzqZK0^pg8lTO(`V0n_f>6aFF_`eAOMfEFl6RmD;VSerzBRl?oyciI z5EAU!Bq+lacCZT=R-Yjg@h`X52zoR|zutOyI;e`Rtp0s!+!1y60YyRi#g??Kk3jJ z?oH%cdj4+y^Tx;_6~@4NhnIXu3wVGeDdW}8-%+jr;(Y^4^O#}-^qN&+EwSlun?>)Z zQ63n2MpXs-LC__sG*D_3ZeE+>T2k~v?p`M9L7VqK&t?pCR;GGgOcc-jU~Urd|2xsj zm$>1AFWOgAN>jr)8A6O{Z1Tri>sG(^^x@1Yn-erAb&N)1V^bk= z`lDekRD>CK^U0joOWp{*GZ`Xd0$-5t&(@lwc|tQ2pXhg0F<1nsSSVL4>e#`Q@wgF{ zeu&gGX`9AImz#V}rc53&*F(hE0{b$mQYqP^aWPRe>Ai%|Nm=kWi|t6|+2Qz|M@mRdEAj6CACBIRVs{pRrE`XxZkZp!rv?SG?wKZRecA@v z*@eroT3c8@1~tO?1*|A>_Or@FJ*Lelsx7Bwv1WBHQG16t99Kfw>Cn4Z zVV-SUVecTeJ`y!jCQF5m$WR+mM##E5;`XN=)(T}WhbEnYN8%(S&GkrmB{NRJT__o$ zUZnJFNMD|^`M2c=@V*p>)V-y$wM~bY6mWt>ByD&7uMljLa zfKMEsP`;8n?b&NZC!$CB27iBx1q5limZR|i5A}RtEM=bns5zxE*9E~%QI$3Yjvl{Y z^^TdJDtbx3yP~zy=N1yec4Vc+jC;{k&@kV5>^<>H+YBwAp?y-0h2aH~*4V&TMH(Kub zTpUuj@3`W3hJnxR*M!JTqhR`7T2-Ya59!F4EaY7Z>^z97udOW&tDwQq5hGta{2R&r zb<27b5r8`w3cxL#7W@TOD?}wc;$_~rQ%h^nX_d0-X&>!@u&|>p-W^sDH*dBUN(eB4-?5dhnYr)&V%7qiRzl<8isl;`trITN~K-U`1`Qu zzw4zTKb}fi&M4H+{ylrJnrN07cE`H`i{uQQf_AHHozsQy!Y0U`%Q(vFCe6j zmUT=VrgzE8AL{kG9{oK4pKjb zSh}1*2Yn$@4-BmNC8j5RGtRQpdmOWJRJ7tmd_!4fH(%5(zbVbKnW?xBq>HNCfUWF` zBArh~0YFlSd^u6-5l&oOxu(Aaz%ag2RanO*SAw>r>_aBRfup(`@)QS%5kLkJym2z} z{sWF)A|%~ZBjZE_&a)$+T>ec#Gym#8;VML^zLo=dGM8&)E#cpIz8I!Jd2v5(Msp5R zN__#|dtyw&Bk(^tc#7KY!3XwR$hbmRx)u!1Uc6$y11Rsw3q&Z=g(F-7cfhjER_QoY zt@;LU11Fzl5^NqF9!{5#`V}D?R51t9^V$%*VEjDjMzPma(;Rsa7b14dGxpMcw!K(GY%5!y+MzTb}0XWRh{qyE==q=f;*}G1nchCxZtH z%!gbLkAE+;!MsfC*I66s&JuE&G*9xC6`J#FFwOiy>^qrfwB>0dQ~M$J za(_c?immG$gd{WcEb`j&SP5)Pvo&N~4(DDsr;)ExOrE#Yj+R$9&ba@GSVY&na1<|V z78(2({_D!BBK^t5Z&C@=9`;XR&OpYHU&8pW>_#VKoSkuA!my`9vefx<8!`iebAMEh zkzI-Q5Bskt5O3o0oWDd3d5I@Ctk@eb5ufb9;u%ExT^d>F*5`3vN)?~)@vF{Tpfe)~ zkA=HPs>M$pi}p-oa4a zN}HU^&eyZA{-dzE0dc+Lrsi;&!xJ=ja%+E>j?4mmT|$n}mqd6UwlE-P+9qv{IBy{M zNmiA4AJW8aHe!_Z8n{B)uQy1kDy)U(=VT zp_azSa{=o{kxqo!5t9K=X7{)}?68`ZyH&uGqGh$UT9&H%q~cr+jfm!`EdA|CvPm8h z|1}y9k&#lWK!{0FqGFapPk$#Gf5$GE)jITflo$y_#Zd&!XQ2(TgyW7E{Xr+OdoA=> z$*f9@o>MvEq-5ywP>HrYN&U;`G$z&h9K<^DkH*K{izv}fRSQU$aTyP&D2-KJ6dOj6 zwb-S&(PeFazJRz_s9{Q$de6|9rv7hvL`M&KX0Ne7=G6K|_1>6d*FwpwkeV-Be_$76 z95PPRW!?MZLeTrhSc@p7U+5a!ps1rQL~kV4#XIDm&!Ul$J|nwn$6D_C)I{f36E3*s zAaiA=qTb-Pv+~%itF*3nv0N1IsftsNdZSM5JD1yAtD~H;RkWINxlwZ)k;9Q4e9U=h7C$u-Nfn{#8BCE|`vT=$DApRn31X0N0BpomP~gV<(sf zXueN&Ffx=43W5#2pf<0wIzb1G@QBbAa*Gt)BjT9m8hs&(DD?Z+@Q28`@Z-*}XRR<7 z>sAMN2Lx$v!Sn00&E3@A8nS(mC3{n1h1?zXVgXCTEi8tlz8zaO!X+TM-VFZ^i;D8P zVuMou*Dbr&uuE{??5X=*|L)~Ja2}E??n=StduTnI8V98aP+q6xEH&M+cOBuOa^ zC-gCFRk_e!NTi%9dqy1_(_6_ZUP=f`OO8{ZDvGw5-K@M?UbRWj` zDFNckG?+K=+k|v)pX7=nFKymNLgBN9!}l@X@GBoZ?LD!B2eM8NOh-uksnyLJitFX+ zqJ^9wELB_9p0QFXu*Y*zHi*-@b!^2jbXXfLU_6GoNZF-Q!7Y7`^`=t!5ow5Y+Di%w z?mlFg-a}uVifeMDi@@xiKOCCtgrsKe!elAmrA6DeBu_-oqyLPkEn3waKE}ThI@b0d z=I)indjXpjnLt;_NU)^84OEqlY>wm8uir<0dp)sLx3L~PhGNyc%xN|k zj=ri8^4`F{?k3%6{3*QQ2iA&J@XOqmAzpS5zIHQB{4Y+Y91G5&&mUSz`NM6U`DYSL^{IY8azJUwOF1)el_{Ld!s}N z+n0Drf2LN_FWw~@5tPYjgKQW#s4)02+(;xE<7jxb3<3h;>x`b)*t?)OQHQZrKTkv{ zqsXd;SGny3ZR}Kgf@O6m{85=LL)oPnoK)!@c=Ou_CrB6VC9D>$HLB5eJQULk+W>~$ z3*p9n5E(PNKPS?y)i?jPesV_Nnu@Qam0sn#VQ-S1+eYg%C14bOS=w&wr~gqQ@Xhgp zksrFkU0HQSkKT$Y%nF`|vG%PAg8T6X=GFwqqa=YCeiy-H$B8BP{csn-{x<2@#7a=s zZ``oqjO(j~xOoqF0huHV-VjsoD*TAv*ani;iN^S+Ikrx}Cs(QhSj~J%AEtv@;TG%b z7Lmr_>vL2`nL<{uj#HDXVQU)>(*k;+ZB958CU4 z`Yo?_J*r4JzjWB@=Qbdl68YsVNu4z=vl|)ko-GX~iE|ocs~YL8(pP6r*9={a6d<_E z;S(5SDz%h$1TsKH{=8t51$jSsaPz1Lp0SQQ6xa!DNAhn%gwaTjtlc%@o*mI=NAojp z@*<2hg?frnVQNKAx_*jNB?^4sV&rdef?AUE#}vH3_`NH72;S1LC-Pr>BU}CTePWGT zw}FS3hgXCcLys}w6i0fbw>&kOV~P-NLkM}6&eU~CTKY|^P3|%-rt9#b91oqcxnjQK zBMdYh)kCzYMCydh zPx^XR4q_AXM?Kn7j$tgzwzfHceG~KYw)z=S3^P2DFv#>=`Rc3~fs1zzSThOZgAh*b%JMuI%b7=v`&y z4`(qp;IAw33dMnh`0R5}<=tuM0fE4#v@^aTr`Lv>HcXFL(pKSt6DnWt9}=K@CRQKi zfZidj%7p>rv~)3>^uCdXSgVGc^+CJSLW38jz;zaARe4jwrN|@`%6l z36*pwk(lIhM3w7}my6&cQ8ZH0u!|uk%dwZ!N!p0@1WOn8pYNLFMy$)mkI1o_$uwz- z<5}kY;T(IDmqji*mXQG~+y)qYqV zj(NWYh&@rt$ykWqs;`!tG46dF7PbZNlXf^H8K1%0lWx4Sh01h(Be(@%i}vo|pna;OQ@! zNUu9}vC_%LrWSLbbcd)x44&z~g3dt+e4I3`NcaTJX=p|CCpSxy_I2KVRke7 zQ1VSi?kr|}%vGkWpgWXD2et7*h^38CLYVH~`fTI*>4gFzdxACj%o~h_erjEv7PIO3 z+>e#`0*&`cm+!=W39BcxpcCaNfG@Wu4^4Bxx5~iXLgE9acPB0t8gA3*$uO7iTuq8|`0(h%LM589>lM*oPA zH`LWxtcfY@o7JJ~fz(q}dsDPH^1`F!u-el`i3u3NAyTE__x16}gXq#dtFPBgySC!> zufRJTK5A$@%)GB5xVo`uf=%59l9iQd)r#2)J$J7k|MXRV>|09vHlWF-(Vj=#9=R9i zb=2nLNP5nL@Aci|ON&{~p(k>3N%5(hpD$Yb99vSt>0~<=^Xl8QHvD}8x97Ic*vQ^- zwtD!iwa8cmdi=tGhwhf5;8exRH%m>=;;FBZ5A7^Y-g#Gu!XGFJaNFN!mC-KDxICoN z{udm76(btmR1is6UbMKBrP*w73!fwR>-w!wXI(eJf3_aYo4ZdJ2B;!z_~T=rh}k0h*;gDN_5HnY@|;8At79Go zAX-SY1Ec!+Snjwj>49r$aA6U;gd^!uqRl?8i$Q0f8&&oiU{Pt;w=(A)|GEuQ0y5C8 z0nS^QQ+Y-049lZ#HFsUGA0~=AGabHPxBJ`R0hgHN<=s^0Y_OB|L}e;hHrK|q1tum| zW5(oWxmPd;{O)2}+r|htXmR4J`9aR_%sZr+sKB++r{CHH5nDEgi~y0hj>20uCrxk2 z+N5u-M@e9=)9wnohF38hXGEW1u2K8{BRVfAempV9uk_9xFCTg3?We!z(( z*<6`$txjCg;vDea1%^nA0j6wUPTkEt2huiW;)mR)x^AqG?oy30V~V(8v%G~EP_qi> z61cX))tjjKAa(R;(Q_~iMP9Lhc8#N;l z={0LxPrY}Q%pF=!3_EGv)2-9IU$L-ZE@t#XhPQtH5%x{Tr4kB;X(}>IR7|fNAGJxZ z^`-JBseO<;1lR+AsUbbfk||p1!d@;uIKaVK-p(@VDlRSf7ivL8L&Kc{v(R$Rbo;Aou{FlL=XTd8YE(QT>1H=P3Nh0K}5;$xN!`G2d{_ot^plFC?+LNE0 zg~=uESvB&0nzK9uD@DUuRdZ83tF2pyH`RvI!}BLrNp+=V8qwyW&zYMIz4_5acm!a= zNU~Z_2|^yfMcRAA_rqpqX{T*xgv$26pP{{b+pW01MJAv)>Oqh|?wM;R@0_n-^VM~VlQlGMx|XP^-?2`;9MQy&84Q#-p*#fge*LN zT7}liG)jMIOFg5daQ%isNOGl`DUJKXST!P0AHYohPx-8F?rqP%$`6LY}OK}&@ENdg<5D+3CUY9YE@tcB> z|A8r!`>HiuPDgZv5zd;Vqc=Iwpj?jg4hN04-Ud(vcM6C-GC>3PjC))&$(U;=C|9)Q zo2UT-_ht{2zM!2Mt4wR;TM~~#Vhtb%M(v1)2`^EiB}dO!dUbk(8uX-7Q!)*oswbiT z#kq|cPiSj=vBlxL!gYI#@Az_gN(MQX-3M`1wv|m@WwSPmv~nk+kU>#)F5LLhMk-Y< z?%8hi7Iwjek(VG6R4qE!UvI-PeX5(a?)(*8Ah*xCzpNHtUYQY|s~mw&%h0E81@lHj zX~^Hm6B>@PirdV0bNYMcTyebWhyhhjNc_$jZ=nXg_3GnlYtfj)1fj6PEHSYr=P{(X z%mGLUi&vOtE-{vA;+v6pWUR25f)hVotLkWx3!bLc`0n+Y&Z6fLr;SU~aOoT7kaC_P9aoCT)|T!g=lnMi%4 z^@kRKyMQZ@u}S4M>&BoGr^+t_9bc-zZt!(|vf;Yo@VsRk(7RKKCdwcI6zx?KgOn-Y z3vt{iI~2#D77ZiW*_02Smc3Fk+3)NpXqoTl^^ZBJ=*ft#ef;yz606~qa9zpDUL{0* zi#Hn_y0zg;5nOy<&*pymqvQC~n9uCX?7046n_eR4M%dhv;fdeXNW3&JQ?^FtA%*Ma zEzgDpUtPO}1$5LB`4b29We%0;x?a_>PN^qFjr?7+YUn$_QqJs5`ERvHGew5sgF(Ms z@=@Lk2W=vbHIf#h!`EW{1?ixh*de1xYbv1%a_gRbq1vg++JC@ePD|{;A5d)xBj8D_9k`I@T*|P zGV!1~8J}{MAOW%d9x@RNqdZ81$Xp%e@t<-ZZNzbcpaQGLs%};Lu|j-IR9^&}u6^=9 z4k#?ZCpRiws`y2Cxv{ayf{XB^Dd^0Oi1}X<3(OR?8rMwoy51&TV(4QauuDTe7B{(^ zh8HQV`;iVT|znLf|0Et(!c$t6kq33XM!@c2{SJ86~QY;z8xgb3ef7)B2+ zI`Ou|4|-c%N&EFT*y2X?Qc=j_wNN}=3j&J?NJ;<-|JG>&E7oa1x++6l30;Fa_MEqveY=BfOi z>dx%?h4a__-^j-;(Kv;kACO88pE&1*Ow0V{HeG!9g8!_N(q)ZQ^oAJ=OUo{-Sqpn* zEZ=THI2Uy(?ywl|k#kADK9uCra#{9+H{tkY#G2?Y%r)9y>!e%o&^v?2(W#wo>tW9x zINi;C{cThz?H<11?Qvz()HR0>BUN_=0zZ`dUs-fw~zKSde@^=-iOX<^|xPrF+;b8bE@ zx2qh8cp@@i-XF1kX7;4;_@&bvL$rYR(#fCx`93h*Dh`?9jtG8+EIG%U7;z7Ll|R1> zRPS8!Hvg`9hD1|5^ifU6jbp$se9KcUxjn4}TzvAC`@>>{Svz z`|1$P8P|KFp&-S2UUtE%aoFtY=l2K4MHcF>m5%o`=Hll~y-c3Z+S>WD#yEnL6`nm; zsmLx;?=9KL&QSqnYJXwI@vn~gTzS(JaYukAP1F}|c<)j=x&PX^YM)n)H&+7IZ_hm2 z)H~R5RbFS=Hg6m7v*AcjqjhfuSlS>Cofn2bIMrd4X`h|-v3jWiHXdKjl5O2eUCsC& zYjeAWCGVJF_u!}Lttip#qt{$!+W!r4S-o(%3}D^}t;MR9#^Pljw}oE$KGM^+@?FL= zuC+sPph!UXiu;9_w<^f^Hx){H9PaCNLsS!XTKEB%PRYu;jOJ@WSkhqbH45g?XA2Ae zsdZwb&wk=sli+8I`zJ5oZ?3F}4&5)VIdI27C+zk=<~rXhhU{NzEm5JWIqQ$)rWK_Z zW;1e_=TGss`>3P+|ERa_Ha+=LV57cOvQ;nd0XLe_AN3`MY*?=$W1#2H)MW*i{GCEh z6YQNW0#DV@&h)IPrJ0SZjqEa8f4JK7+rVV%lP2f4eR1y6HtmohL9H-#UQ<@^6@`aN zqY>Vyr6r|Hku6yTU2f-`;A?OTq?0Ogn7ho09m`o3N z8Y*jbWOTzHbJQLgvlr!>l|qn2+-~pZA$6l4soQ;2z8Qyn?|TG%`IV{Hlz#gK9-`}? z%<5k4OVHw+22c5~N05%D^o&h!1Mls~lc9bV;Xj*R!nF5@+~=oog(p zdPwg}nfPJ1e}j6OALxZi7Y^FY>u-2YdlI?c^U%nLb&62#lXfr@87IqNHnmEb!o}#O zwNag4j18K6uA=NUu2bbZoyb*^Hdl_DwrWVwqr%jUosR4BuezFRxw&%5vr;kdqi;|U zjpjYCFs+{yNk`zuHU_5^0Ald|8Jb^Q$qxgnIt+{wPYGK$V@&fdH-0sK)GSNR!S&;f zt2N38IwbvwnpDx=`vT2Dh}HEwiGNug`}i!{XO*^bv#~sWe^w39K$}@?Y_Xt}$*6r` zXUwli+}!HGu|Tb+L)Q4UlsxXJUj zPZ2wK*6@KFDLxY(-qILWOgs>#qs~)nk2!?J_s}p^ErDy1-VkK-{^2b>CWnXc^uC)p zcT%&n5qjABAYw}qvbd^LQedcUz+!dd9vtg~iVfOKt31ZPTRAPS-?uoP`isF?AXT4lGp|(0dItqwVm+^Sn~>m`_L!00j)l+ zw{&JmkP(4UcFVam-BBG9b|QmQA{ut?VB}qL*?NoAu7U|4)gm)xU0fqv#?=P2Ajn;k zm)C6p`&c~)4p>I=MQhcqCOO5vLO$;6KO#B3H7$clG>9;!r%{9j=6H1PD>-&9RLeP= zlgl`VS)X5o_~pWJLMSi&Ny&XH35t`dkY*|SKTYiVSxGk3+K}cu4R~?lUY_)!V3{5b z*k${ZArQUDDH)tc^P6M&4#8jcerw4I41NQ5I=sknI zF{~DA^xSYj@n+a9i1aiUKJn4W>g7LFz`NMtio1!nK!JlhWLOb;u5Vy@c6QH93Bf){ zy)7Hj;V{;f>HMX1G1N=zK5yMqH}1YF3rNl6>WPT8d!skHY)~QdriSIo zCvgnRSuRB{vm@4>60TdXH~}1XQCbCJ5qBzoIrkKbBzJClw&4{+yA01%H#&BbU*zja z#3Qu(F4-Kc+Q7jW<6agD{6j?S)37EU&4xDaQ|{k6ED6r zQ2x_MrCw^U@C{`2LEhRzZy9tH+Up59_IxA~EQLf|5D+*GY(PrF1IxAn>4cg|1(cw} ze5Gu|TU&?!TBW7D+b+3`q#v^=_m$s^=z1My%1*!JKXE*g3$4fVN-s~v`MjUqLV*P* zliEHQaeb_JY-}}e=+xC*W>#L zy`8JFv=2S&nuH9X0?J3Z`X5HI7!ht#nN}ZHY4jdplTvJ&rsPJh*1$FpKL$3~bL;`j z^4hADG|3ozs zy?3J2&-oNWw zYkgN1NUcxmaRs*qpO?&C#y_Av>G`UbGP<}pbQK!7#5S3(nqcG`iunn(CSCq0Cc0yNizd$A7Ki=*3S<3Oe$dUBN2M zuQ%b*^XZ@@YVt4w_n6L`AE!rV|39xZJ&e^3od+S8=x#iY%zpA4c(!7ly-SA6>C9ji zBj)>N*RL4^4M=qucKOVt0|ZYnP`F~cX4Fguqr&Txtjp!6L0$e8F_oGR3`#Cr?-qf< z=^v04`>RXztL&$`YZ)e{x|vq@Ln0mC+LxtqGnRV4;*`Jts-~=$sur3$lALJOHfX_h zTy>$Skr&zZj-3wjoaGI@6i&D+W2NqktsDCCVWBRwM?0;2cgMX}$lu0;@r>a3=WRKTmx91Li7LT9n2`>-$dg%Md8IxR}N(~))s!tk5 zJoG;7tVFkeJ~+(9WXi{LXGp`^{K&>qvKKHRMQDF$gNo$l+4v37m+lH^GY*~TUBFM0 zW>5Ni&&FY&x?VO0a+}clt*zg_m!EtBjT(NyOWMWm*987z@~~JJ!KCm_`bO-GWM;)E z0KTkde;r9fAafD-jy;YnXk;LeudyAg!6OFH;5yRJ>sNckUpD;YgH?M6$vRnwx zd>yT@ew4o_TE>_VO!8^I#jQguS_S?_wO0tO4N4KizBX_cA z7HgX$=^H~dmFSN#UKLJXODL35*S^PNI!F{whs6B|4Uh)Kugh$45rwEo)iLiWJeRDv z!D5YURnRkY)bz}Y`q}0H>5lK6XHLwI5TJ4`!TT=xH~Dy-mvJhRc@LFsOE|j)41;al z)WVx=tJQZ2Q=r)4{eDSUdIU&#AgQC4m|BN=XCfWhI}w??4+5|1m{$7i!prSGw6Tfi znc&=6cfP4ouPD67Miqj8FPH!3GaPYdr|h^%E+3 zKHSf6iO`Ux<|Q3s-07OH@@XjtiDZ&;qEl?ta>Y^Z30L}wLR(mN54bgg^OgGbYN90Y za3nG~Zcn-03s?stNKJi~1#YtvBDMFF7-HXth2S+u>j(KNj}C}-zrEscZJPWlKHpM= zn>qMJpxE~7+p^ZJS68;Vw{OE&2)S6er9?C8ADOcaa z%XGJjm6#n?_I7j$?J_&Ov?6*ma415vN^F^=Aio@onQ<+5I|uG7A+i0wiV8pTK!5;$ zY1d5}T&>*k%(O!+=>w?f9%b}{!AI_*D}np2^=a}QuXxtjL5vt+b|Dh#@X2QM!*dGt zxat2+GXbTWk={e3eH|;}Cy6FMHXT=4qW+vQ=5oK+tc0}3nj0sYEIjp{qf&+ecqlbr zUt z@Z$DS=$KPLB>svMp1nW+-rt>xsWh~J!>une*kf~OI`(OvryMEcg6?=K>))LkwBfI! z0Y><`)>b3Pb$ChI=jm?I=}xqs7fdzKv7mvH+`9k|DW$3+W=*sv1Vy=sw?WLwY8Tg{ z7nz((^|47=N9iL8Yw1zLr9c0SCAkX$Xb%gZHt-jw$7#0Ec+%Eikk#NdeUO-qsR{Oop6yM;1SdSk(ufCprLf4>;Fvq~Wi&;*&I6;2S`Z6a zgs;Q)+6Is{Q))iP^O%)v0p1>QFxg^WJP;hVp*UWXxIOweGW^tf+{;`g`)Gy;eRsD3 z=Fk6Qx_lbcs|$E7p?$C1u1}U^JZ`o*3yG zv-YV6!%_qT4Pq@-?0y1iA5@~=6!D(8pX6=TCz42k1jYbEtnr?G#1OTiH)SKc3Q z%#=igS@@<#w0Ea~y)b(H+pTYI<>Vm39e=y-oCc0JEj4$7c^oo+CX%u0-F`|j4rV5E z+$DK&^2gKpN|vnuE*EXU!X(*v%M9#885Nd1|HZ{HzopBQen}R@^1FW&kK=y~Ab2XQ z`@_@NNnMjx!L(L#P}Pi?L2L^Pxf(@BOK8Oy>B&B+G@}y?DN~Ci(c(MFd^6rcpHEV( zGO`d5?;Rq5PoP9<3Ps;J+~?-%hFmoJ#at;oyV9LT0F*$dYeB5|gYxcG?Kd3^oxAhpX)n`LxBNH|T_Rr^Cfom+-9u3#H zJ=RVw0QRUItos21z_IbaD0tbz-Bg^k^^*9HcV>}OLgQ=WCjPx%_9T6q{d35`hs3UH zmLmUSNDUNhq$N8^w0RB4K=pZP>Ro_iKHdlW2}j&jNXfEFY7w-BSGX}9{aIlG=kW2otH-W27+Lr@OJAIxZc#|q{det}z{oalkQ z8J$5loPFi(%C4%@HM{EM;29yu?r{ooZ!%_R3Z0j0fq)R15&qjn3dYCP{>v9Ex>pxT z8$blpz|}0P%P9c3Dy9;E$?c642@RRxC18Ex!dF9j1g1^`a)%>8}FF6ep|;w2v#IX7tWC&Js+IfGq(Jbf0bDeziHQ>UUE@7F-!?ypnzza7edJJcO<21-Kj zQj&F*UsuuO!^sJN$nO}iDIhOf9s9K2R7P+Qxe+C$-@113ai21Dv+&{u#02m%R{Q3b zUTR*d9QP9^&_h88l9i21ChY)Mh%7>$qVR~I5wl=IsMz9K zFdKw+dYP=9EF#~;>>m(b@0sB3fgCPQJh^lS`i>r0t0bo8&qKfl$JmM?>{uCGf(VkJ zKsms8AzQV;-r?^-(I?vuW|n{|z6nO>qP=t&7q!bGi+{iZ3sWTY1KV@+8M^6wy5%D& zmf_g&tXwrIsjn-FMIsHtE+yKV=OvHaR}8MfBA*$AtVwM@gxDE;X~)*E_nvbo+ygVF zzkx|M%IK|+Qb6fLj%II7Cbr4cglCQxL>TK69NHKV%enu$8p1Ty`Dd@c<+vBOWj!UP z1A_6}cP{jt%9s`aL_+{Fz&dY%_+lwzIJM=@x1*#mCF~wNug4ve3}STLYnx)@ zFl2W>Y~?A2{EN}=FR>TVfk*VeOq@)r3gGwSb~n#&%n4I%y68%_i8|yww0r24#|P|A zPrBYf9A@8LH5Wk6utcc#WnQZ?;CJHJWx@PRH%_NULb*ghv2UyaPo}PF2vka@Mthu) zaH1>o#iwi9);t=T4{cbZdD|z&2MKftqwKC5e9eu#UM%(ICodSYI{xl!DC7`{j@rxz z*R7uzy5ie7G*edE3+~fpX6jrJQU}-cXQPZ92wl24*&u3fD%5mAY+R}nTI+2)KF_}p zII=wm6M6&tPSu!?v#n{q^4u^n%7*)SzM32=S)Csw9{ZJMU#p`Pbo>77?JsJmN9fhH z?L25YrSd^4*F)hawfST0q(D)t#5n#v^(V&2bnv>dy5VCC z53f}mf;%RSQtkzn!CT^*=(jqWH)(TBNdNp@%8v3yA=Da`OmZhvzgg}Oj0UD8dZ<-w zp|5#Y=EHg;;ekI|M?uE=`8FpdqPyr}`35X+=yxBQB`Hq#L3{vjQhAV7ix#@h1LDtSMoUy;r&R%H$2Ku-#jodR2|EgPsBjDK*QUI{QIr%?qQT+QCP232@tqY z>^da(D3)LMp?Z3OQlq%YR!{KPqj#BJ+Eo!SG0=HFThJHkPq+N|)z;gYtw@1n8ji+0JaZ{-Hlp=GW5G-V-2t&r6XgzKl0`-6s$lh4<@fC&Ph;JM z3&=~n1b7PHiT>cOc8%jOOe})kBN!afhm}1|tfwqOgh7y1Au;L?xSl?Ev~ejokN=40 z$D-FJLn$i4`uZ{OzegA2`4PX}-@wu5=hLn9p$D%S0@Gdy@q}?08gr>n>?LD3nZ{_M zxfGf}^DsvCPbMtxVbx@p%?tZqszud|>&l-;rCP5Yxu_PB^>g)s?bU-PT~BLkqX+hc z|AY@hKfi}61}B$^qmXQ}V6B2elPYGw(iYfO4pCGu@5QsL^$t)If3__D?*!&!f-StNuP#v96s#dG_ zSLVB}zFY4)@%S4VeEpKiRaDoA%TMpj22V~&B)<2YAI!Kb+eV0R8|WKkSJO2zOs>>n z+LAUuO!~YjSd`(Q@a=}?@}WV`n`9S-UC)=|Mo$8Nn&v|>@lRNklsLLc^A3VfZ(v8%DH2m zASvyvKrQL<*VZO1zeg1suX&yKZgXlep#CGTLE)fZ>01*ZE8wqU0?gv&v4_FpXl6z+DF%9@MO`s z3i0HI@km4c6lNQ>F*z_c+=mpkrk%p;NN!0TmT4HP5)A;rhIViZH*y8>W7>ZU5=&bL ziCM{TgNtUUnCG$Mjd$jeNbZOJ33No{PW?AuJKCexcBiCVICdou+Ge}STN7dm7cVy* z8$ACGE|eo7PfPNqXoDKqS#7SgwIXt-x(5CSgQHC*@;?|HOdcuqI@2%crmAEg^hy5+ zL0tNUdDFj0H^2>!_}2|S>Hl8NM%?WFBVIBV#lG~a+;n?QY@T>kj@Kb{!`hlvVRAfZ zP4Htt-#Iz$a4880x(0knwcT;UC1>q_K_Xam(ML@p@;;_%q8GRG%Ps$=fWr?sobsmd zP%5n3^etZh`MCJ=H%$(&+%9DM<+6-#FK_w(_w*@;8+mZ0peFK2ET@if{N%qJ1;NIE z>_kp6nZTffC6IXwx{+j*xiE4tZLh%36TgvBhRX8mYXm!u`R|YUxOY$nDtw_{xDpG9 zGgMss|A-ZEcprb(cPL)jJXCHm12FJ1XMQ4nks>nX;>*VNaCsqYPOiGp;YCgXzby87 zFtA<__c)ErZLn7-pBe?Xfi>&1cEko14bPJtBH@TPOytu*U{;7Dov|~({2Vx{N&+eR z463L_yF4y{*GYqEuGAYmKN`aUJF@QYkK2>qD+r?)VN!=7IWiWU<#6C3_+V&$&?Bsz zl7!2-(#v;@ewTZIa&qct()SEXT?pMRC{mA2p)KWlNl%(rL%6JnM6(rH%SFr0^gXK|-?D;mbVB_1H!L>}v!UI1Le1wT3;V<@-db&Lk`+TQgWUFO`&8xQ6+;*_ho=X8 z=j^&8;X2CyfaGNYHh>9wdUYgaZ78oontx194vwuc)af4tr)Z?ZaVF#JB1D3pMDBx` zHdnjG)-x32vHS__B%Z`PI1*RDFx1sovv{O^l`+iyGH!$X;DJ-*b(h~?P9aNzfB2|F@pR1UjkKt}FjF*C(Sl>R-^^z&Ks%gS zEo)=q56R|vFz3hbEEclV>>_m=!|4Z+tgv9MWLujNyW!OL2lD?W*gM*BP?p)lzwQ7I zD2?Ln{;K5;JusKSlow3{6LxKC6z7AXb36V%3}+O;xmN=!PcCDLZYc!D){Wm2ug!}% zCS=Lo_C#0*P8Cl8T`|FS#?e8~6!WBKN*HGfDJY3K;^>=c$Mt|WfR4-fd|*CQYs61~ zXSYiRU1>{P&hjg0#|c|spSB##L*rX%fHcVQ;=sQw-sL^w1(su%=7^$32!{ow&B{{U z4*fvjVxy@Jjd-sm7mXSjkC_5?$(I0fNE$q@&Ab3@+(FWKkO~?z25y^ zFvC2OPcqm(E8}}3er*;ENoa0|@1OTYhitgmL*;a@A{JSLA;J*m53Lbi(hs!e>o2<7 zvBf%Iv(>x{OgtwG5Cd{@JHM}|=Uu#x=K(RkAHkGfk{cyyJ)WKaDRE07SdVar``i|+ zZPqAS5Mfx0SHi7u<&k{5KZM9xN76BlWTOC{YNpoj|BAz+r0^+=v&Bv)9x|0p#(zM1 z5v+fmW@LYjRSXdd`98%6x5(xVM-zbbm_gN1Ao=F8opI*3-Fnf2$n2faW5Q3nJ0UH< z8xnNMg3NpZA8MX4XtDi^B&^W%OLyM5p*W095ZSD|aEI#e_V*xwsPH9VDH8Q>Tg-3A zPpB{}({O{Ikk=1cEHXvGiV#8uTlk5)r-uNjk(_JRBV$zsh|BiIi!_pyNyfoV+Y(H& zgWSMsEM|1^WNG``p$HPcn~x=%gg;sX(hh@hFhtmzy(M`GLC3TF{349#-wH$F#s4#h zMQ_WB@HAf1wjET*hJ~E30k;%=3`BCt*MAtepR^@0)i37#0xVLiBufQw2dByu04Z}O zxf&&k-+g&h$!g*=%ZMdm7SRrieUFS*kYVzlRl`% z3ug^Rhhd8`yi4>>0=`x3CDrqqc#=Ia@1^CHqU*UW_-RYY>>uLn8tZE=6O~n#3CkA9HX`+6ZZ@m;KL{j}bptxYB*cAUCVaKH)W36Sqt zzSS!C_^1p83?Iab{kvHV6F3%;{JO&;ZsK26rSzJ^%*-IECg!$kqB)YsOgrAlnm)IK zHm%k`cW7mI!12P_e%8v~;@aQ;A5c^W=cb9f@Wz0v_|tNX6xO=o?#Xt)aKtNySeJ7L zdZc!zA5Q1?5R)oVd4)UPk$p-QPR4rUhnxHU^f1`$8!4Tt`y~$3*$zeK5^}Q|m0xIB zV~1QgNO+#g^2GLFU`Gd~rKqs}WLbv+>D@p7fJQfLZ`n5tjPpL4F%_6#bn2 z!4o92&@ecdB#0H5nX1~kJXTaH_V5AYj0e$$+|F=SIH%j-udiBSyKKdodh@y&dPUwQ z+9-&w3dvgBmIC3cJ`$_RGEbrB@+M|$S7}<3K>!m%y|MDO#tkD?z`2RWY6+iw#JY4girnA;0;z-(k9VySL(aczo4>%i$};9x(VdWZ(+IvcF^wwjnL+c6 z^o0E%or21D!xyzY5zsV=$qH^R#pqIQ+~Sc6HaS;tDbsaa_Az=yJ#AYjXCpTln zu4h%bZyxK;=eV-0M+$w|-I;TOo4$<|Hned9gWC_j>%~$Qs0D{|W4mKEXLder5$aM0 zEfYL_OmAWt)qQv5vX9?gv#E0$n-$n*?s}!3h&3`-A-QmBZl)<_Ah1k})cerXuQRhZ zhDhPzjxcXMBY;e@nz&aQrAi<^2EvvU zXEoGk<1fVhtd#$tNhg>ZwF7W0W==tZ1gb_HnkM5(?l@}~3j&_9OElsgQG7UZ@w%UVKYKOc zO4WQY96o(VyD3lXc=@iz;u~ET{_f{_vmQbKIYDI*+$D1HB~%Ej90!Hd^Ebpi1N&PGSmOHz?n59ZR0ASZGcim>Rbg`EXf<-$ z2|~a^Re`eDfccxAcvx9q%uc>0c*FJRJp&tpb5jsQ8JL92EQ>s_ zL5?oVC9%tX!X^n~4iZmfa(U{TX0gYRto}e#d+o$!sJk+NNWiNBHCewxyUdM>#Cehr zM(2NiUpo19SxtwHdyHkBnlcb1s5$~8(KJn2xSr0@czRFip{QvB!jHzGVn!$JUHyZ> z9Qj!nzI&AMunHA5jVLua3Hk8k5}k1nGG5rPl_vk?f#$r|jh#}F0-6*Xma_51xgYY}_mr00 zZM2-bM@;3lFI!kou_vgswCJ>yde!EzD696aT|)FmVQpw?B3#tEM!9SxyY*e^Y_nUb zle&JBwrUL553YQ?QPX^@WXjK(O`D3~%Mv`ANN*W^OFUg7j)i_t;&C3Ur+1V%%%$;N@6uFH>M}x|yS(!|iuh zx;KTt&v{mRq8@+8lKv9#sz0aqrq*Fwe)7wxn$Cm~Dl(mA{?Z*=Y<{1{1YGg7M9*m} zz#z1RQRT)~q=At2Pr~r*8YrfzLQ0C~XpZc!prspz(?{|Ts7?5&j$S`uXWCK2yNT0H zliRl~0+T!#C`aMzFmiQY)Z&7wz1`v$Q>Sg$nALqh>rhVUdbzP-xjTp2z5?EU?`_En zzcWK`+$PokqVe*TfVHmdMQ-Su4UrQXZB))(fR{C3SGx|l-UWg#T+M=nO&0jwo#$2!P3Jvh7F^~uqEDlv$&gAg9 ze{2XAdlKnG$3Rc}J|2_EB5$Ee)k1{`+N2Aj^8?n+L1onS^S6FiM$pOs#0eUdocnlc zu3xn5L`HQ2ebr1%blo@EeqxI8A%{J@&fifmOi+FIb?+wDw2QLPmCfN8o}w!iFhqCl zkZA~Px(|JEoYv+Cj7?0Ht!0cJSUO18tn6n%P3m|>@E6iST)3@BUJ|w@7q^7MmFjp0c zYi;qpTnY_Lz~h$#z_hNGCwYoVjYmg;JR1_n^1_#?7p6r_FIW~B?R<$ImtX;G^rT}t zh!VokX>38^ck2L1N+S9eY;5Lt>FiM*R4U6p;#YZ?^*t_ThrjCGWIm&#sc8(uEI$5O zTuE;#H=LSWkzdir>oDxsJVnaYc4>;-Cu1*=doi5MW8;)cVF)K6&*a#^0&6q z0N*zrS<^F4vF747-M_*s4+PPNw=4p#n(r7_p)4W^69KFw?;|p4Sb`%tR4ImSj%Mh* znZj96UbfsP6+FM23_4e&kdGhpLLA|K_R2S#1`34u>02`P!f)wY;I}uke4`33L}6h- zvPDHZkJT|@T;k3Be3$6$tk}bOdx-=N3hM{U)jI3 z15`hlM}JT12T{X~FW+1uv!M#kasXpM;f^nWlVeJ_DG;bTXnW12wL<$9y3tbu?a=%O zgqd+_S?@}iA{n|ShIXiv+o4X%n(Yr}68?}0I6ar431s(Pwk7L}RKAqyuDy~sO|Pn5 z$Jz`HeG4|dxpuvX22dK3M8@;6#m8%wE*30j!Nf}u{B{$8gKP6Qf0QSw!Zd8pVC;O7 z7(?9mJOa%J@%4NXnL`(s`m%uL(gMZktBe`m>R4WI4Zmj{&J(zMWvS-%9gG8{ElmKEMy|kJemtMH{wqa%jP3K!nlT zd&jJwlB>(fh-m)kaO~_(|0{GQcVBkj-L|wCz=OewPBq%q7fTV~L={=zmIE`;Tq-0P z#Dn1kr_(T-wJCZ-@>6NBg5D1F_e;wjN6CoWQVVr3HugZ!8ppza@RaTlKs7 zO@lLHLXWRHH$HoK)LELV<(+wB=9n75JGlvlAp&zyYO6idB^ozhdDFV3!Tmz_F5$Q}?Ysr|k-b{<9}=+S#Ir9sG-quBuxmtI^Zr zjd6fe$F+G3sEW!XQar-|%AqQ^Sav>023F4I$%wxX)@O{cncG|NGGNIE=v!xitoGl} zb-|9(!{duYU{3i~jKg@^ZmVYO1$?r$)WKLkcG+utxKm_V@SOzuhtc4c`n=%J7UR8l zzu(TkJWo+qPIqqk%jlYf4T^+)00b~a8u^fon;jX8SYPuc6=DS?LzKDT*^YwrtJ!E7dTq*|(R|$kuWSvY5Q%ngH@tso z+I?clcmUB=B@M)me2@T2AH$alHp3*b(tAE05}@gJRu!HYF`lqdZ~oD4P(~*{ZSGfKzf5bTl5S z$jZCKb`1lA#x_T7G=8PrX={j5BSBttRy#sBH~*QTgoKd#iS;LvY!?As$V>;l>= z|0B57_;{1Km(WOD{~j*)#M9)>MCNDggpsIBZd&>NTH7QSHx4}GF>Y=J0hfj~-CE5D zbo%9#WrVjG?7j&Fi> ziWX`e-{iYkUCne)aOp$n{Iqo*6mHn_tCS`?C z?HLtiN)d@wEPcgVIYwCC$g!$I1r@~8#!Nt{>FtIRXn2S;Qx=lKhu#z*^wg%3eIGks z%HNCme!Np%Cd1KVF+h|JzgePRJPOMAAPY`!JWEZbDJp+Mgf**J+{i^J`{1!7+W{7# z%gfw9kK`i>llX(!&%9uvL!81)S!T+~MY0DR9t(LEX^&oBLelST34oWnA(rp_4W4hA z?5P`TVLn*Y^A(rBW=8sl|5E#oG|WmT#C>${4U*Yb>%fer&uqQlgi)i%wS8v)jTAjC zsUcCf{ZmFX9kmP@f84=-BQJswS`biB!O_mE_TRfZoA#qA+uKKoS>(R)BA@3G1qM!kb8*UpPABf7Q z%!9$+*CLWbW4&@;;0_h!Byuj#%qdinn-ikAzY1|G&5VXQSxh+0!o?YI)sYXzj2}T^ zY(x$^wPCP(7*1Gm$D1~tTd&ANuutzE@g50IP{Asb?bymao|kHHF$eXxs8c(gH$IB9 z;u|`9DGlJ#c3q2`(ey7>+o}a3sy!-r?_c3l3!9z1O&>%wM$yA$ zNlH+>A|tPF57Wl1w(QTV9qrkklh`Qs^ckm>78gbxe`R*udlUxCIA`^$F`G zD$5?7XO4|p8z|Ks7N0fUEOMlhT5{vq9GAptaXl*z9ACw0~G0N zwajXm5B8x}jSQ~wsk4Je+`DZ3;yqFFk+d^@qaE(LhitF4-hA=e_CbgbR0&-V=9xo5OjHN1!PEJ5;n*Dg*Hgf%8; z3tT5sFrO3i3mAp(m}N-mO=p;(Y?**nvc=iaex>!`AJF}pV9p1XEQyT--2Z^=hGvy> z{{cOGz(b=~6YZ1t&9HfRre%KZMa{tACPnLmjAc2|a%}wC2x>M;7Aq*W*V`xqe8Yr; zl5x}gg~=<*VPy(B__`RWB%7$oxV9-3K80v<+#CWC@Mc{dao>PI^`@rVg))>Y-g{bU zhhUQ&UD)qHcYU-VI^12+oRx0rdp!QuxoFP!>8iv!dlqh_0I1to3+ON12aXAXr;}eJnlWbO>;NwqadLlpz#S_AVR*!j=othXi?? zFBIRX0W{4}xceGa6nWjM1zTbCstJ2I<({H!7BY)2YdQO`5P4r=0!X(^=WT*eEUFj& zx0lP`Uv%8;@tlMUY3@5{^ENM*E-~CROV{0=iMo*K7))Pz*C2>vBxt~LA_5!RM@O|k z_em`}QvAPqvu!@kXerHm2`%|rPX=H zP+2ukRx``X3x9x0YEE$$@EV|W2EO+60wy~ShsN3KlRGU)WYLEcA3vvM92R@@@R7_A z#2y@p@RRyG>3K0f)jg5Xz0p1l&fQl#Dxr2Ru*5ks7BOv*Ht0XkOBa1E= z3lFQc2fwyivX41bE;a%aM&n^a*AUW)~d=Yj+GRA0&HeDu1Q?=F{|V&;DQnCyrn zg|0~|^jX*oVqgDvJokQ0kRJ&E2OW!FN@xd?|7$t1_}^l+|Lgh!tE7bgvhE|NiDiAO z8WlvRd~I4O)?_GoiR3fcFNgMu`QFX2+bo_0W9OqRofftv-0&Q+GMY52R>PZnrRBd( zmendkk06Wt*WNqMf8grc0{g(%D6;{;VmbCL`yZkuf%dwnqXjvt6Op=W*G4Khpx^o3 zi6n~A9RXx}0A4xCOTF6M!DlG@?p|DrelBRuh3~d z{1JVUA2e}euoZu_yheZy*tIFdZJH!}s<6xbTEX)O(5c-1@ULC>ep_WG(acKLpLdiM z#?QZQZV|O7Gi~;LC#ew{n`7;p6}d$c+X)sGgX0u>O&m?^IxH`#oL_GY|881}l9#Q| zlL4wksOYh3i?#VJWCoE(06`--$mx+}?@@sN(CSh!;i8Na0E1IMx&}TRB^HEfDY~sy zD{y84=$vW{!fJu=L0{Msuqhs0wUYue+W83wVVt|?%^);N17_VX%w+9V zM*s1%M%@cu5+M67+e{_5%EYg*YcoE@s-DR4Q(g+N} zKrG(NCZZ;@hUa5}Nd5L(IyB#gBZH~KdjQx}g>6vUQiCJ!yCX(pMI|t@TI^YM znWq7j3-%1E(JnD{irbML{$=2!_eg7oc~iYnICv>a2}CC!nlu3TjNxsvfC|jU{5eo9 zSPqmRLF#Ho4vzu@c+V<1d>~j@u1J~Wy{_CNwOcc^Vo8Y8+$qv^>af2s#KQH(-<0f2 zm;Tik(2%_+rK67$BFy-H9GRX7Kaa%Ryb&2hm_2Q(Ri&iYpyz7w#jcBlFY{P03dgx< z-ve-Vl8#C4sO+e8o_9`osYrMp2Y#XZRU=~|DP5&sm=9zIyRT;a2-3TZ|D_d-tm4YR z_owY(kG&r0m1Gb7rK!WfL9e}ZHQQNJDw_d%HnJm2L;`S6nEQK{om~K7T#uFJ=jpp1 zV}!YX$S6xsLvSz_TKAq;4%#dNd8cQ;>ukQq;!fhc&*QX@X>8LC8dpiMs>Qoh zD5ukaJ|7uz=RIQL!sjW*Ilt``4W(}nmfrI?#GoI)J=WoHarp;CA*LQJD40~`+dh`- zakb;LZ%4OWb>WOez%(K^H-Lr@USFWS46hA`scI8YLT~ZLhtss3A#>xv1z#Z9gt==A z*3R9$+>s+46I+}iI<30A&)_UaNl~-IM~}(4bZ`NPa+C5)L?gVP)vmYFqyN+F7pEb*zBQgnkpD5?a+Xfm!T1+#!RQjOF z1Mo9S>`(P?TzcIne^S5le&g~)Jo$5vUC*Z~{k*&hlTiSZA^^Ygd)AS z(8x`uF)g~!pE+J#+JE;N8nJj4Is|8Wl4cZrPiltC;(G_%h}20UWqh9g9en_)*l>}b zBU8N3CXG6ule|h6lh#Zb7g9-C(9vkQH4g~BJAK)OfE-$P3IVLFxy;s@9KJ`U2 z*P_8T4cO&wn1C&O?W4JFaUFY4@MwhoG%mAw=v%{=>8Od)=UCtE%T&5R0+^pjL5^IZ zCM_OG(iCFJte}SZkZ2KA#ATXFf~&DeTH98?)I zw_c{IEGS1#k-SuudJK0cC*Ar8!SsyfeaW$6Yl!f-Xw=zkl^9}kQ&2#rQKa3#u3)6$ z=gERlxMgl5>UZF`1zavU8)5kUGk#$)BC5cg8&qTuAMYR(yN07^0(=-maj@y2K)VNf@Rsl+v@?r$g427gHpx zMh74g-nFTRZW8J^t#ea-w&^gidj@}trHOMqnH^>KL6<%Dtmu^ZX3uzX8%MRQl<0Bt zu;>Wt>D5QUQPk@pAWG?tP!^en65TyL zuUxMdV%|5KjE$JtY$zvv74g(Ms_lDXNT`IZln>6MmL z*7XOa#O&PdTWtnU=Rzn2?)&YC`&)6~v8l4B4^m&0^YGju#6ZFqsTS%e-l#!>=jN-) zJhjQ0o+COh)8dPSBdAFquZ%#)GI@n-!Kv{}1bWYrUU7DzwL3#ZN#?cUI8({@LU5(O zk**`1Cksj-wFJ)gKP2^On~gI!Fs67#XLH@KC`ZqG=ZJb_2hJ8EbMSB^~{VOCvAM{6mU?7b{ZIR+7I zLRFurJh&85I2Jx-={mt{u|lX{?2GDNR7-TEQ#+?h&`#&h4kQk;z%#;A6Gcw!~s=@V@;Ze=IJUMYi@v8QDBO>{cz+*pb}vj20_4kDlV6<+|-G%$=! z&QShs%^xaUTSi0_cc5NO98M<2CqD#jl`__rVAbeR{d(e9BfWC^YGB@-c#vkX>1a%N zgUp4QVZVNiL=IdD;MCuOrzM*4!-3X-@Gqn!zaM_ilUdIGRj8e=^YlBHuJ+9v%6?U~%?CEA z`D;>oxMYvcZ=!TSAwPY&W}8fmq7&i<4kL&nJE_$|f=M7&xLc;y(*gWGs1x%fk5;bU z)#vM|caB7~kPKckTyD8Deae##oI)}2pTn_CQ30J~P!IAPjz@(?KNqcje*5ZW`Ia&Y z9oF67qtM0W)Yjt|zFIJXsm-dR+{Jf4vb8C@ott4?T)X|;epMU7DiM$DT~cr!($wCl zdh&8P5hr=X+$hFamj6Owb03(;^L%KBo(QSaeCioi)B8Cz@*r;7%!-jR08qte1*_ds z_I{w69WAWxi}LC5hw-Hn5^LRMIf7cVBYSQc(N`wa@|Bh_#og*yFBRe|cYb*r3ID5; zlSE;_vPdq6IL4WG;%7etplGTRa(Tcy{%y!e)ZF0p?Txb>BmhvffNsk6&EwO#&Us)QkD8hgLM_VIO67_NB4{sp=f;+-X;RDY@W+bN<4%7sh&VxEFgZ^N>+pbc{m zs(0XcFh65V+gUV}%yzI6UCn}NejayUt(FZQm^k>OLUcwpp4Fi-=`}!;%GnpSm0Q4k zn<34PcxP~icpn%jT&8=OLUk|$A!ktO1-c|N%SRC;$fj6pr>G9!;U0Vr=Y8MIO^B?> zWjcY<(26R;5B=p_)Z}kq(IVKm(c02-`0KZ+N*tDP#MM=vuK)RVbCgtyo8x;LpH`ES zul4s9&cn$O-u^z<=!>Ws#sHpgck%IES;^@k!ZoCYLp~;c8)T?Aq@EL}*WL4yi%Yt5 zA$Mb@>C65vcz2lF*eP*4FdvSa<0Onq>R6+l*@P^?M`HvV>`FJSn(I~?X2GmbVUg?W z*%^3XnZTw~E8jO@Qj(SUmvqsopV2yzx0oL&?Zy07sc*#ABVYtV4s?XE20~Rz_?C|H zwA_Fp5r_!l2(#noRq}>@Z>`s0rW3q89LsxMACbBEMYrT2sf1&rBs^c?i46mX!$GwN z9ww&k>|{YH@>tN)avChMY7(@#3Xw>B?PSE6?da^c< zKg+Y+2TMHvm~;DnkUmq1?t$Ky!|0vG{2OehO7w3NIDA$Flq_ zZWIvN?H5)}SU3i+=dYaj&CJ4OY8XFt3E|pusgyNtl=Ei@(l<+%_}=c*xcdJKC7wn! z4~TRWXTQ(yeit=2Z3D{7S)ZZaXWWkujIO?}UR?F=HgM`A-y@zYkg{0#LK1rY<}PQ^ zi$zgx1Lrr|xc+L!ED9A*v(*XEqQ)CKs}z6+Gg7T3pr1K!-q#dX9&Y_{uAv=9eihL= zvcd_UfxnGwZ3Upupt$hst0SgG{|#uF)-O$I6HkA5a7m}2&H0>v$tU`}R~MycQ@@zJ zn7^YcB=$dh+xr{`s-z23JN?bmlKR1a9G~4DZ2Fx0TOffKv-Cs+pcccik)WuMmLpKT zfSEXBJ&o&qhbsIgY;;M7K+?dE=Zg`yT*s~blw^A2PRUqN|50Zn^UJ94RQP1fS;hxL znydaW%Pw@jt`}YCbO6X3`Y~5=2@K+k0geD-n0nq1GUv1S&5M)#Cg(sgP-K3AK7nI= zZhuwsK-Qzx*W3T0SbMNH)<)3H+na*t6=b*4*72*5b)Wc`WN0t7T!oFZb&;Y<@Mebr z))#=pK5`%tH(yheg+^C_I z+dGvUz?^%aSW>yWCzAzCX{-vZY!BpQn`%L&@=EFlv?T|Iu{TQBD5u{~s_?Vsyjk?ruho(IZ4U zq#Klw?(P;z=@5{XZV&{d1*9aTq&xi0_vicjXMgRSot<-U_kCTj=kxK9in+Q=v@QDY zXc2C_G+%*3>qmLp*DrpxGJAJ;m8*=0LGkmU@+MSvzxy@?bM7YKd3}bKC^f^yH_^CG zrgem%VFyt#Ag#-K9C?oTN1N}VvuJDn^01|ku~yo-oTBJSEEcnAj!7j9ZdO#`Q8!GV zUrW#T^S!vDMR^<>AS!(IX+m?wWGHZ&iBaOmDqeRkMZm@wB*YHiRi?Xjko8?!b%41n z-MYSOJcwCrs$!R!{(4~ZRH5YC({|ub@qpJF1mU`7@(4M_);sn4WW_GAoV|!ptioLT zW$^RL`pF`WCb3UP?PVw-ZYXv6{5AdqVQEXN8&@2S8XbrE?XM|`BQZIAQM8--6&FP|c zr7TN%Z2zfi^{0)h#hc!+O7%B*XRQSoQTQS;$)NynL{=0=*dj{gqulu5kM~Rf=)+ z=o6WBjKjzNQ6@3T}X!Ji(zK*YpKb^`Oc%PTs5-M~@L zf1y%6ys=xvF>w6}jv%k$k{E`_oy%eBTf`W=tbL_#7N*wz8)N)w+$A=QZG8XpQFmq< z%jkKPlARr^J%!8r#K@S)OThu`LLgJwGFTXt9t_9{uJP#+7||Tk$cr1+WZY5_?bPPh zbx?Wt^2-;G2aX?)1*oa!^~?{(d*-p)($Iby02DX!Nmpd>Y;&rNcgILv;V> zOXe$L?%*Oz-y$SfAtC87@k_X*kZEr4Qx%L`>_0O`GCF#;zgve%I&@ydtyRMp zN|?%wxS7hdpwmJ{5SSd~gov?9p#&Lmy0##8ZM&?vrTa%AkEh*nE_2rBH@TTD(`MRH zu}2+#gx@HEeU&`5vtZZ+ol=f()@rF644JMSXas0%K+OV9~NwHcCJX2U$q|t$OS$kAM(lF1Rs|{|fGgVQ^^?8)Ly-1^|>`@k# ztpOGGCs~fLRyLWKs4d^M1@9A{7Vb`~d!0mbLk)#?PjPw)Dh}bMk zrNX!KQ_kSws{qB3Q28Q`6J)3}2CK>HTdNKZ)P>Zh2;r-**)By4|N{OKI|E*H)9fIvsi$)&qY5W!9lCOP%wX zjhdVN7TGA4v=?+EiPo9dwgc@B1h<9OtxdJf5{4VqHCnp0b|(if5-c!|$jFi}XAj6yJRsUd`5!YMxUWe?w><_qb9F}LpePEfbbd!eVI@~NgQ)c#MbD#Co=Vr9#N@1qOaxtF}WIR*`S+=g`M`7KV z+gE*PpZOBm2SxiyT_S(w?}~6w+g{Y)wP1RhJt}E7kPiRi&9ZzEPsS8?`kvIgY3~sw zOP{ql8!iCXn*I6xEd`a;^QR2n1EfC-=|P5XxI~a-w-Ol`<+2`RZ{|n?Z9ZL{3dWb| zEU(%S)3m;j+&rnYT=eRJwb^#LiQKY8d$hnlS6%6AAHs(9@|g7AJY6_&K&^P$Ju1Un ze^x4VJ8Z(hOsEjHcyqob?FWX#676BO=;@!8>y2?b4q3||3TVnE+$@bxUu>QCx@Gqt z`}4B(r|tv~JipArxAN%HjZ>7iP|-`3&M3l}U0~BAe=8pfUn-wSTeIgd4li|SdvFd# zWzdf-vqPVsLdP+C9A#IbtG+8V_NWYS+lu7>d>S(d-r_Zg=XPbvy&!X2+BE#2K@uFh ziZ>o~|E<_6gATQMPM88VT*3+}BO&6}x^VatV_c~s_tgZ)2nO75JcOgyP=r{LckhH$ z8|1rVSN_8f&x#USsp%Z;G*xmb3xm@%&|;fgK5~H>)8%r-Gb}66)r+97ps4&L#W0mrh_!TIdFCG@wZotT^#1x}neyjzUW=rb1;7f*~;hD~Q1a za-Do^e4a6-lW<*5)aJ#aARc`?VkQ9-C(i!eC}#(;sJLgNyzZ~L($=n^GY2>#y`GuS(Lm)l%^X>TbqHrfAnWv z#0}j+68*0=!7#izZ~$R2cKzF7i+q>%jMFY`@RHHx-ky_hTnz_ShM(ucfUN zWXKFjAU~WD)O(Z~ew4^|L*&()5X>(e+cyQN6X#WRlsxqt5Ljw@|5By31^29O6a0ze zxBNHph(b=UwQ;YH#36&5XSdiE1D0-sL=~c1;7GPz+zDBs+Q*!If$y2>%l_qjV*;z) zrV<<@bSkE1d~_ZQ8OT^f8ORuHoDfxzSBcNrZQ&#DUO>d2$|S$MPTsY3o)kHWR|#rQ zR%g)~OVq`meLAk)`10qlO;#&0Yw8C>YDIM+kuM!cD@UYG^)|X^cmC({AM|O0IUVEs z+Fx%*%%M<$C7~Wxo3>k|r%(C!G?S4WJhZt-fneZf0~ln@%Bx|Po$@NXlpr6)e>8>r zT?`;!6O(-@;QTfhPr|COf*^IZ+kH>7T113XQXOqKd4}72U&49YS-O?XfW97rAuI8W zpn+ndxMA*U${-9ytRY|n&uqZTGTv8_ZUiBb2(mWoNMJRIhR|<=R>ghm;zh#UPVYJf zpx|8EDDE~gCrRM{02)J#vMY>5max0`a|F09;vl~Sh37; zXf38z6UyOZ?h*QNq@MJ~;$)A(=<>Q)cYCc3z{EfuF z>Ls$*^=}!MY-a*7f0c*3CKV*Ot()(8oU`Qp!`lI=|B#Sd{F&3axs3pc)y3l)V43(M z`o%TR=;kIw6R}?`UCmn!>~?q)nT&hu5JEFuc6{;h7hnJ-PbZ}GO;UY)=BHoM^hf7( zJPQyLi86SvyRNI*I7oyNWr^WsWyr^)nPJ8zg$>y@U&E_YQzmCcNf_Bj8C0yH-+(Sb z7&Up-Ycw#xsEh`oEENnu5J4!;Cz)^EC_vBzNS-JsRph#7)5hD>9uB?8%hgg2NV9@nKr4$NYC6&-75>A%idrB)Qsht>X1QP+IluEy>+k$2T zP8*~)*pa0F0yMl612gZFL$0JX-O&5hJcr=W^IEUh1!0gY;gZ7aO61ujpR)Rdw z`tWOnI#y2{Zwkop2_IY-?%K%cQe<5x$a>6zqt^yMfMgT*<~Do!p3O=S#%$M})&jG= zpiH_@HbCDw;~L`_V>Y_E8e#L$O>pI$H_kf5V03fpC{=jtAfS{|zFy%JH3|PR(FEP> zY!OuVkM;h)S8V9&w_{{6OytKwC(GM6U4Gs5ORnC30o6NCr$yK%2r(I`R}KFU<5HsK zaIaynQL=!wiW~Km@cmzasm-(^!rAovT#P93{f{UL4n}kr(TY6GC)6&JB3CAsy%4~h z$E{ye*HTLTh@l7(tF~ZqBml>`9xwX&MaSb4#1Of?iWa}QpamJv7d7%*r5w7cspb2b z^0>Ll8_&&z)Rv+z+=SzoYPTnaC)I3zy-CWWlLr*lEycP#T*lIH8<_&%8BHT0BlJ8= zuzY8;jBad6AhCoWbDi>i)!Sn3pl zCg)q=x_A2*Kn9LtxJ(MFL>Okm5p0U_a{31hs&Ey=XBJi@`{a2fe z0=#w+H+b^Jli@@38tWmbP#R@7-;37f?q5J?u=sB;q4Mz>8Gn36a<;g^YKklsaY*M(5hTR00Y1eS0VP$2yf;xr>=%y`rrpPj_2h51q2o@FJ`bg zW(ofmrtvE#5pjMovq-D@Td`&v<|VKnXk!x;BsKZJ?Gc;G$J{?}#aBMAF*5N5MRZ$T zugu~+%UlFl$|hqnmWiOU`YaNS58#}MSRR0}GK+ocm~(3gL#2ME_V zYAd<8T@tShz6qhKqpGjMBtk~4*!_ldzkP7fQ*VPKNS3iM^W#6>(!rFI9;z!f6CXKk1gHy`B@?-L+0fuTx!`c|t{47@g zOZL##rD@t5_S9GZ?(6mrMg2^VIO2U1lgyAz)7`p{{-522b#Qa!jP_oy(RqELp8IU- zA@P6utXP?x0>qQJKjC#~@&3aw1jeed`ke() zGN4+7C1@de?tbh-b>2l~HkhdoNcy^LIC1xrx}U&l*salogx}V{dv_E)L6e-*i{F~~ zFH1R8bB5p=C2r3e8^GdEoiyj|*I(1pcf1f@$7|DgPfh60j)7|(9{&2hMO&Zx4DASN-h`c2+cNlY5}a;8|CduLx*VAa3PLB*YhI%_< zrF^fbb=ZzR%-3CzXQ;~TpejPasbcnq=sOg}jeLQP@Kw`d^O}rm6)kt${$hv$78+hi z(-J!@{I@_uV*^8i&Ef%O0JP(ZXF=pq7}q!g)1!i;|JB&LdWs6^h%iSIMYiQEl9iyl zp_V*VR*Z1L(g+5_RQNJcNy%@GX5`@F>6gEU`BdSVAiO#PRHEYE_Y@LXjan;dN?Peq zwTuawEHH&J1^P!(K(l=<`*vw~p}1pJ>HsSbajV9!_j2;l`Ok%btyCDI`x-+%Tgk*G zpGYqxsrt(Ln?0FVA$8D&Xq3l0l=EodOphK99;4<5GR|g1RDHgFsNdV;%|bne=9G;% z#pZ`vx+qlfjGXzxMzs>Gh^DO{AnIy17)jYGJld_Wuu}-YIIwx}YeNN|;cMHpK&CHO9NrfRSZb!g*pUfbPlA0qxuO&1KXdImRVqEOzLQu0}3e zSB%-R-ppDjB%f+CM|{>gohhq@@4z1BA~2#QXwVx8GJA}&3%+^2dTb#RtG0mH@aQ~) z+veAVj$9qfi_kRmSgo@?(A$=3UKMq7>2*cYoB|-AF#`Idxe}Y zyzN>eNB6d%rqb?LoU3%794IbJ!-VWB7Vf4@J$eysD4Qg??(1(eT^w6_d%LS z&D(P$otd=apzXts4(W)r-#O49HP?jmd)rPs8l63BzxnNJ%IQYHlTdv{g~N!jSJ4U{ zuC-vCM{!rI5>`LK;SLOS_$9kJ+89?Typ*n7PGX4Zn=BxMibjBz|J{4zRx6EQfROV$ zlN#F3*DT+!?US;xbI|;ED+iE+SJ$q%`-#oCeT6DiA2K_uX@uy`P(ybuZb;Cle^$*Ki&3itp6C)dRG5*|0>r|KLyy`XLHeE z|3M`s>{`P!9^BBnJBE4$aa>fVuAzttk%7fwtWsOUuP`u5TqwD|n?kL}FCU3NpQ^+X!|lW0aWBbG7@xn4RL zm3R^lxyu&iyM4H1?Lj%*eIFg>y~@6_le~}0_8lTk-e#o+h@ShK@4Lwfreetv5O1OP z;|yL0chtl~ntijRg}e{`aFlO8;D{M`9xy%|%3{mfN@hE5dQ5yH7k)Y7i(?4S5G^b| zEG&kDls&c1QwchNq9038nQRC;n7)@-pSCZMTP!$gl)o&6HgHqXxPP*eV9B+Z3ox!i z^FYV4`Yy0WKLU<`2Hj8Nb z&eE9a#1WfwFKTXP;>0h*>0xY3fr(GP*-AWkS|*IiFOJXHe^gv1_~m*T{$evx#-0~Z z!>zBWJgM$tsv3}maywj?r+As>*V{{pOq`UNtvFK5Cu zI4G2Uh8$m3mT6FmrrZ@G(dG`ckdjB>JTe0KxK%hIgCqz$eUroc16WOQt(pzd)HbmL zyESWtb`VRI;e627T}*bOYIZ2A-Cw}h6$qQ+iOakno|+Msk`C%xDc))+p9w_=iOdRF#4#rUr|UB?msH_kb@~GhYm<<(7=1 z=YG6}Kn)@XB6#XwsIs|bq!*2(6KG*+HB-qfRvE_H8b(|;244}iQ?&Us@}R{l^q55C z9?n0u1z9NYN4q)df2=p3bb+#`XA~*2d@&Sq1hx@b6J#(W77Bu_MNLPw;=LdnwH>B_ zi+9-(aO5yPwD-cDQR{Zy)<61`GK#*rK}aZ?l2a;5LJ{Uh(q$k~l_wdMeMZ^# zC1Pu~^qcobB0Pd=0ja2AKBr|pV|d#|8voZzJCY?_4wx>eiOV4j6-A2x%EwC}*!TIX z6*VFAF;@N8G(&)}up5>0mDeT=*114YafqqFuxi+Gm1I_vkjI}NJ=>O!}jAdE8; zCT`z>A3Nm56a4RcF?^$5Sa6FJM(}BkhqPu_T0^}ui<%^tr%OU5MqYPVh~Px0#P$Bd zC8dqz&M0ZXo_9C8V6i*LD|J;|!oFae;8%NK%}V@w76X6DW%oiZ^aH3Q3kxV=I%0R$ zc*b+Y@8|kIx`7DSPVe3S&Nd;NLCs4G9y3OzP2Ti z8|}B$_0i7UEfLF&9S`0bP}9EF8Oh;%cS)RY^}Q=$Q~*8e6(gmem9JGR_tVGlr-F{a z51YuAno}N<5lK8UP9S+XY}bX=#1(^)8V`hHm`&{n_rZ|igTjc>BQA00-T9j)&`kGoo}e z{de+Cq}AN+aegk-I#5Q@bRhBk7Z8MSiCpyjtT+N`-kS|Xb;^99gXU7n|2Gwpf?*oh zt=+3-fmT`429XT#w>ZiY@B-UlE|HWImXpI6f{uv0Rik}uA*uWN@=-F{wIz?Vp*cC< zChS{b0!*}1uY!_Ep(lP0rwl^cYObMxpegQ~H^z%mXlI|(3E&2y&` z{19<3O@S6^y3w z_hgpg3EgQI526O?kBYTdw{x=T7%>RTLfP(m=SepP%HjC8srPyzTry-Uu4flFedv?V zj#GiR1ikxKuH{@`mS;ORqUQkByn^N4UPmZWYv}mWf#m_u*O%>#{1iIiIOlqlN#>wd zNumG1xnKpe*^17mN8V2&H1v2^P|^~PwhN{ z(Nk1neGD+ilk&=MUI>bK2&%-9a16hQ1m)6$XMgDjsVaYzJ`^qbVgEWSYEJRvuUi@f z|5h|r1h&5PLHP>z>Jx9`K|@T3?X?vmPeOkpDW?PwI$p+X%89&Sa36lMW3`!cx8V10 zrqhw&d$lx)WNZUh=Re@YHZX}4?G~!E$#?lpk#=Fi`Y`67-aB*cre_x=K;14OmoL69 zzgg>13dx+*SOH5I#-RFBsR>$0)BQOCH_L!1_+qagL;Ct=rNGpL7lQsjh*BtqUSL|M z?8;ogJ)&egQYgIF(6%fby(jQpc94WiUFqT~z&!NhWm2H2?TJ6T6GCB!q17--=s30u zKBhz=Ks~YVru^>=_$7))bJ6Xq%dn4^iLLjWqyIwVga4xAlL$0ZQIS34i`=zA*-NmC z70G~A415c~opH=%yb8yS{mEZ|cC60IB3Cd;OCb~8CN~fVn~JhWiF(q-$+uIq}FIOesbwg|Q6)(3n7MJ!D0q`J9u6dpDRDYv0tdFV?WV z)4mt2a}zuS%Bh_%DYmCIJ#Sz3Im|b2QygDXE)bb4$-vGjdc=>RbmErge`HDb15=uu zgI~(8kL~x+<4FHoMbw5GRD`-(6iL1L84u5>;=+$WaD;M7L;*}oSU(H4=~EeFve%ve z0vZrTWAiu5(nkL}#Y+>b_mlzoZS)Fd8Se_v&Mh{OEJA4jM4;CtNGzUcznnnzuZ4BR zUkue~oC^PbJ%Nk(H%LtnM~0`-3AlT4mc=>qJK_wuGnlKQd z`(9wn{&5U46fN=bSW~b_eTiSSUZzOZ! zz#ywy?RVZi9MG2b>{frvt|LscKRY{) zg9)u0yt8AgM5&Q-lf>v=j(1UbVj*PWI;PpXd+I6C9aLwzwv&iRdCu}$kM@=t1S^|~ z-77DyVhqS-18B!sjx`GRiD!~0eV=l7FYh4cmonH4(m%KHK1frg(EHxS;qP=kKGnCZ zdZKmXvMpBc1Buxrqk1@AAZwb`@^us3KLRROGzr*dF!

Pl1c`y^!TK$s4p^S;NOb zNz7GQTvK6Na!s~(%SI*-j#7_ysn5LNN8x=Zcrkpqywa?2b;f1 zjQQRA{n~Y3qD9Qx`tt{R9pTsMERvH*$X-U7wTw2|6N7w&e4=BYXTlki>`+-rVc`Lc zZf59iBdLDR1mSAk9K2^y?|> zMi!^YZi7$L83%QuVdYTb1W|Ei8YY%udMPnQN>$rKlVmHC&M0?r4=+FIuNN*KD%rsx zPDyh>8o&A+k?aWIILPVL*Y#5-bG#C7f!&$Ti?uzI#DtFf+|Sf;L%Z8sh+{CnJ z*36r>h=biMF7uI$+pq)ys^hCK7bU_gRdfOXy6!XC((u0kg^2@qUSv#Ee)qkay=Jn< znINjMp}Za!~VOgwFy`1dF`fc?@e`ffQbhP$$I72T=LL&{s z>tvm+KkJr6=E%)JV43pP@3izIQd7cU>GC>@-m!XZ3H9p?69h`V)??|wv7UNNp;kGx zwp6m%rOTR#Mos98oLV_8qAfW#P;*J;%uHgcoN8SLCmjSQ1WVjvkf4S=HNB{O@(PP{ zOf~hYo_5P!=;yjk?0$~0=SNhQ2%yaFIAbzyNqW4H!NtKAw_tkCPaoH(axNt^rQmdm ziN0cEIf9IqbGGyC%*(GWk7F(`g9xagq`XVm2o4DAfO!yCRN>;neGt)`IENt9-4^K=(}raENFGHR>O8Ol44dKwzc2VCye zJ&YW_jw6>@ROze63?A5e5fX-eK~`@2xW?(XEaJzXV~d|Oz?4H*&Q5pAGi zN?9aDMPM9A7tBZwmy6#|RbcNH5XcBud8%MU-t2SG*_GxYG(Y#(5Hr8eOn#Mc+q5Ic zI`u6&;pO0mHv^b&4_3!olE32$kGrHtpc5^xXrFT30Kmjd7_V6?wVCMOss(!35VWU- z1Z@J-9yuwW3gfi59ZW_1b3ZqhRmt%#35Zq>B>&l1YIxU7*3hv({PgLW(}$yRnWFTi z%@N(3ukOq_LTZf_$QZc1p;sCip&3OZLEKRUKXB__4dcSyU&RkZ^+AWW1T8#Jf-yY1 zj#g5r2O6bg8$_Fed~7)St$j#;sSZSC12iPAwijHo{Rvwyw^W}j+hjtA{Kz#t zm9l1Nh)QEKq;z$&OvEV^#n=P2nwXqLG&bj$TsR;)c`qBKqa>8`xYZ`qdj9+%#*5ii zYH8%1>qs4ZPhEA{{#)mJcu|xoqgS~QQV&n0RRnX=5z<|hM_|#%{=k)>`M758Y4(G^ zfTf&D>!@{>3!cvRpg*A{QK_@e^OA3oVe(};_Hz}OOtLol9ZiMik2sW*9?&8Ong`d8v4c=d>0C}IOR3ia;lZBpj zEb%Tnw=0)1A^mP!^|ww+5+X~xPND8T~_rak8>WKlBl9%RPeW~id-sU>n39VgtX(EoLW=mx>FAA zzhq1?(sBE5!U=cHo-#ch#7|yFDfYn=|itYmH81C%E~ZfI@L4p z7v|Gcq_TzOoyVUAf8T=P#AXU)!+HO%lndqF*jF|^0ggFB^(_F^D~<=rgBJ8Erb3Es zZ2YRbJMXAxOJ;h+e)jIInueKlE0{nC*PtF;aQ!Y0OWEn#_x5;`bEjp-v+Q961qut> zKM_1I%v9ANB4YB$0IUZ>Q%fs7G{q()XqWwq(>@=mIL8PTI=HM>!l$UcaT9y5D^Yz( z0@5dW)F{pvaAGX12DwoP&xGub1JNFptzV_7b*SgnA0{&Ctag_D`~yoTDs9=#g6fwN z=tD6LBG`h;@pvfX`zkOCHxsgvhgQw918v({(8VsDQlkjuO6*YZM`sdu-$&7lSBH1F z$(->%sp?g7ae6yw;mPUBnL{8L4B88NKmbvx%J5XCSAQUNm;2Q@?#{B+ z8LiFR90w{-@Y`0q^zO^oIkB04jJIS}(B4HJAfD;nc zm@2vm31KXgf&9DRCxE5H&hisF9F(a9WNm9FhJG>Mm457pYg09KVU(S2}?%e3ZL zrp*+xh^<)O99}-8rhAbUm+THN4|GtC8nUpfH$pVW!n?_0SC;2CqiJ&l*KDgm`Ymhb z8KN_T5^HVYz3voRatkG2IhoioXW)gfn6JuRa;nt+2XpVp-1|`;Agu3VLwWqLw4^_A zkfRwDkig`^O4+AXAZjkbkYv0F%8l7`abHX^CNtV_coXj^D!3M=HP7jqgFrhlcF-(bzfh5fvri=_t@232@6atRLqLP zYG(oE57v6@utoKpY3Y)-@zGR&-4o$2w$#pn2}e9GXF2-fN+Rfzn+*nrg<)#&x3C$L znL?YMw!S1-0lM5l4>a#bX=#|eS=B6&9HwT5w zr9uEBu}_`X?FNs0$PO|q^$hF%zqpxI8B@K|X_s{(5Q{scC+>UJW4vPSIrY&- zC%&A}Jhf^OL**MA0d6Q@^NjyD87`Ae^|cZvDzlalin6^{7`o>n!?^oNLQ&#y)(%x= z?YG|5=KXO4S(fXyy~0EvD1BjGi1bg9Tk}xXP6WvgU`}O&}B& zg8UVqwBc(1!3b0-DTV#|^&}w#2&@DX&_rb~+@OVwG?}77c151{Zbhw$#QY(M90zRU z0B;zrEJ~pBN+EzBn@l*a{ z2bn%ldA#Qbba<#2|eb9Kj*Xbg>4z7BQJFgYU75e?HZx2^GAg9bA@3s?ZKK?)us zK!3%5^BoXXC2VKB^6=)g4=4g;;@Z<}3t{7HS=a1tqZeYTbvrE zyg;>AAuXSYa@JCN9sV+bRVbiuTQKL`uTR=mZX=$GaVkcx4qeX3vBi+#D6wa} zE6XB>?l@-q((Bs)(<0^kx8uPCoZN^;(nSJ;i_WQ?ye=PhxRpC4o|Q0^|8@I45O&9& zeIO27jASp+@9lc+7o*Hcl>o{1I|y+U1cd^Ro*P}-GhplcoUdmn25qOmOw#cmAV@Yo zCuPEytdpE!!+YCi>3_urdG+q+jXR5aIoHq^0~ooAH>)|(4KXW)*UTz?_@D;~QOgIk zdr#p7(2+8)@E%|}fTv+uvgoV!%QwJChK#%`923LZ(L)0j$*tF+Dk8f|3-p}ac=GO% z{d#dTC4}82l=HnVx7---&kRz4=pLwb>p*;fD=Fc^nu&CegffsCSy#pbondYxlqP9L z>KOs&{?)MOx_-FrlIzig>&37$ zv66$}AV{Ln`zvP@bH0A;LaY@qmwGQq!ny_tTMI_~!RM+y;=Ha|T=cTJBveCWYeg}w zGL(J1_2<4J3m73&3s1t!wtnaRzCDN@3d@L(!ZVqr9h7Kn-@!ohC7 zMDGXz2}{|o%-%etVu))&fzVXhU>DhxZQw;N1hE4ln1(=`l*41P8PeB*pax#s@zGEn z2~Pm@*0T>Nd>8`+h_B|iZlgxs4Ro#WNnIW~bcZVm|I?TvVdD1v@fMdb(5r@@YB2+S z^X^i5R6VI>;}O;8o88l1b`%QTV>qi{)JvfzB1VsWLgsRC?5WkCcEr>;GYv|dsIwjOdzv)_Q3Cy@h#2uZ$*|dxlFw19I5_O*DXwH zP0Yupw*6;`xq{!ofL?@4b@vSsNwt3IMhL>Y(2#GnGXwPZ8o+96YO|s57I~K|oD~jm z3;yZy;_!MO50I~OLU^;&0uX9+`3s<+O?b4JezT0)`3Sy0U&3a=dPDSE@{iH48c?MQ z^97o(GqrS9zUko{{;yJ@z0HPDMwD_#&Ex?X{0;RaICdEmII&{k%*Rrc)~bGbSsrun z4fO2;liO@`s<8wwj;CR&(=tUY#As0%89Z?-i|!X4W^U_egbRnGt&?8-#2?L-tgR@j zeA&={=_b$DV2&v%xIne_0cdumk;iOv=4V`Bqya2tbUk(1yzh%Hh=%<@h>6(fjP-%3 z5RWSxpprdNu-y$Q>euu5y2JerHkj5>E#REi0+4u5?Jv z`)4_I$JUu`ijX|46RlsSa4tb*bK$b>fDvFV6sHHP-~;!gGq`kiX+UJDujtq+87q$E zR5;nMi^sEM2xB!cS&ip;uI=P}GD~@)VHU zX=p_sO|BBwGa-{HBKWD{^6ki3&TfHAEQfyFwh76Pv9;?-&+z z#x4=d*U`5BQpiLjb`=Cn&%E?F7;AiZ4;3i55vvt#m@}?U* zuyoA$1}=vLBhM6&BdO?AG^v3IOA_RIe>3(&b1&GWe|dO7s*#T-aEy_>1S_Lc!nIW@ zs!8TdA81MM`aOkItz9}d{>Ty6YEW}A7{(eRfkOenLZsKhMHQ$Tbh$~`Hfrx@vPGLs zswU(!aZR%%43tct2S2v0O_{Bq!Hk<{uLgTRkqgsLJ%c^4NP3>l-*?)2tbKcSFgMU4`u zUXFn{LtohY(();u&8Ve2r;G5Oy8nYPv3%3dj{~#km->W=SuF(Wk=qrBe_^pw{wuA8 z<_AZ$u0ZTW7H7a7ro9lBN5fpLEY54z+1XL{ISSQSth1JL6&9MZqmhqm->w}La(s+w z2=N6Au8T7r7A?2m+>L|63{M+R1Olhqw*_VUYRuxNz$pOAW&6wM&POlRDn$GYjRU;(q zk0rur>|eu`-zHnVt>ula z;0f1HN-xzFm$_|Nawo2P;z0c~;ay`b5~s14g_6tWnp7ep1vo|K)}_cLLL6JR;;i0M zbuU6yQI13>p7}+Grzxc*n1a}WfP{oundg|cn0v}*|LH@)_!Os|lPpgm5FKe%t1GAL z9^-{2E6e1`df)7L1^#qAEb7G)sVsS6eOtU&;7A4{ zK@Ya+9{t&A8l>apGy&`PPMZrq?YHaM4zSsp4`6UAcXlYu1buKz4y#iEzZ{nMlycoW zNt-oNd9#f{a)AP0<2s=<-)&NP$k)TT zTv5o=5$cQW@Jfe$CR+zju<#ddI0*C$28;4Vg0ROslXa31@@5?JZfmABPPfWy{E25! z{OURJtFf&P@g-73wOC*1`~~o;`~?tuzVp$^Z_5hj_*__8N&G5;ke&J`1XFCmG(A~2 zqXdUMLb{o;RK0oQ1g@vy*Q{ryJ#1Z_{wgMUUWIlm!hp}=w`(mbjRdapvOl+&MyZ7x zS-dDBALdYJ?_@(m`5B#}!^ zj>srgWcXq@%4q)d`Ug_@jFT~nVd}iseIv-qQq&8YH?j_&g0k2c-O<#*{WzG;p$RU79^h6m@{f0Mr^r&TT!uXZM_2rlKM&C=lo|!a! zUm(m)mbEv~VY0;LlOwPTO{Q8K8ZkLu^YS@}-(%RA8bk-oo#K(U5zQ`F`2E}A^+JH^@0A&54u+A<3r_6aOrqNwCWPC}pvX|a z1~VYy`b}x5yZI!%d0O26!6vm@KZw?-gEwg+fQMgE?sLT2l*b_@#?qxq@ zlH&f!v5(43gY80`pHv+__uhCoM$7)9rnEnqYTT2EjeV7?R8^xCk8XJO620-qwb`<9 znUZssn!xrqBQ9X5VnB|LI<+0K$|fLGo#EneWo_U(>U)-B!&CY(3wC@Tf57XUF(|6i zE2yb^24gCd_1*iqbj#NnsS!+I(N}yS>Eu9rIAFf^46 zv?n`XIwGM@ChHWzaYX>dXnL{yj+_+0;UkI+%U@li&*kG6g%!joB;(6b?5y%uEFD{6 z!{6u3g8u-ccQ!Fm=BCD9^wxGWoM1mrM0w8?rIQ_z`J9x9se_0VD1Xqa#0^s4jZqq^ z@I?;pa2_od&R4De1(dU5(?o5Ck|Qbx6e$z_GVxq@w1B{MuJ?$_L0cH$kzJ?#!n@>yftAeT*n8N%hy3m@AZ~Lbxo`ic z$v`HJi#-yQM|Rs3v3A*&z8RP)^raP!p3AIg9Y4Xq0FLJ1djup^q@5vFE(9EZ9?h=K zEzZkwj7`TQ1fm>DmutMaDIF5@RA!*7Dq(Y$1nKV|l{4eBq;L{XiRl>=zF2hsQ=x_n zq8lKmO(QL=yYKDnlUZtF?zsHXenDy#7&M=qXZ{-GdUJCl`u_iT?I;0G+&2#ExxLYB zoygdlDYOXOplCdQ#gTk70KI?pgaG#5+TD-NSk_@ezU8vgedZ%vV=X|6W%H?ibZpR@ z0s!5{lt@*JOBmf%-|$1A6BCu`r4li~Feo9_$MuIn=q?YURY0E2%W`s3{s}`-3^ljs z+Xf>ZQXVLm<`Rn|DyclEJ_sxE?Oulw=K%Nn|lj<+6? zRCtRYFoFa5Wt?~lxKLgleA@yu(i${J9NWo^{Wiz%Fx}fyv01fW#e;La=`c= z^5T%W`;%4p+S+akjKwK2z?f)G^xhK5(q8zX7?^uBOW<(ieR1!c14d15PuI+m|Mg!8 zHdhutJ}&+RWEk}qoJsL6jrFeXoIF^HAprOE|6ai8*))OvWjq1`tr9t;2ufloKCRzf z5~o2Z1As9vF;z1gAtCnDSUsb_{6iyjaUM!-Cz%8d^I3H!Dc}(sOqN%}IX-j($-cZ# zTIU4GjxrXgx=z^_$U7&AwqSr2lz9XGV^CUvoDWBpP3{mK@fjBpL@_1;ieoF;Ig<~{ zWNS?~R%xf^^u9mgm(xxXrtciBOXk8tBH%1yS;N*C_Ugk0G@yh>Emq6;$)9dN3{0l5 z7#L`8YkU`OSZ3Mz>O6%L++9G%?u4FqL(5Z_B@l{1FQavHPxY^f zaH$5une`s-u&nF8f5b>>)2?*`X2ux07Hz!=09ZW+3k}ijYY!1#!GR^!px-)(87#Wa zaA@*yo4?~%HglGmuX`hkG`H^IdP7)06wBya8H=#+9pWZ@0)M+%sfZWHV021&T%u#{`~^VH5IVJ@1e4`;8e4PoLfO5Ahll4|?adh`Y_yKW!>dqW z$|N&3X1-T0z|wb<=VRjT$>~z>UfD_k2q0xvx-kZAdF&3*WaE===~aCd@xn+wN_m^2 zzJ7<;>XtjGXLg%{`l1OgCNaVQBPOHq5P85{Eg<>n6i%cjMpne_GhTSXbe*MXxf z@#VK|Ik#Qae6556sYa@bxQHlO>+^6>H%C#K_$Lqw24s+G@kKZ8$)X z;O_434#g!%kszhG6faO5io3hJ2B$!AD=q~J6n87`?(lqBYyI!WzmvU;jFF7YIq&N_ z&*Px1MN&3j!0S=Jl#nRh{>xo{1QyjII=Fs`Rg|wzq@;V!*A6wks3AegsT%OwFY^0E z?Dzj`?XinknKcvOxV}0dVEVHoQ49RENtYhj11XyiS(AZ^ zet3>-{doWOR)qh9M?-1}_N$oCQY*{qKLki%-{S$!UbBuz0$gjwpCDAI> z>!0s?0?Rsu!u~HP1ttmI^~DhyZiRFeZ*aS+*iwvw?NPf-wvs-0#IgsScG;eNd$$pV zg(RBvIDe%=X8lUC`1ZoKbT@GxBExBA=jFoRbxZ~(r=!4Yqu`}8@mh0^Nv1M-NSl8A zRlKq>boF3{k(I#0G?|^A!Q{AV z_f$S+Pz&WZ>R^WUmfhPyyU(5smA?uMUd^QF0}++X79G3lDANPV%T42)CGm9b=ZD-t zCTN#NuCbSm0x0P>YTU{Imu8OY{u2@eJA#gyf(PpgL*=oG0yW`;GRh@7)^V-G?ec_) zecTyW-0k{I+HtuClhqDe)#eJu_`u)sak!XFbsMf#)1aYV*x9n&ILCznvP;q z<52PWu{Sm_}Ve>JQUr2p!` zPfd9>SFHm5EyJkcZ-LDbW=;j2aKzq@`F@G&ac!IOqilsEmS6JO`n-mxKCQ2npbG;N`;X2kS(O!O| zx4?MU=%tOyc0(=6{I}6~no+JOq%R->!0V-#`$@~h-dCiYcklXmrOaiwvP!Nc8Ggo0 zKUz18agb_S3L+(qTfqXDtg&_`j`HFMxeVtl>F2CJt^5t?H!C-vV^pXA8zqQe@6fzY z`8Qm9|Ez+Du<#SGO*tJ{ZO{Xiic?m?0R<;Zb%oTJ^8xEtvmji4zZ-9?tTyxi(s4in ztb$||=c4DZIez!@#LgcC^6SzRuHZJY46(qqhowG7f>bNv5^p6)mvk@;5& zZ9F7rf4E|$eX9+0jVv-f-EYCC+1QHc2FEds2NU93O~B7 ztX+=8L5w2AAQMFd=i76u!r zN^zEqW3s4hNSOZzz}-BiTXC{ayK8DEwEU!$A=-?P$!5SX*snqKQT9t=Ej9L;zB+%j z0|U;n+Q>84=!<9;n_nTbQ_X&s&BmDIbihFZ$|xy#IzAG78gOtuvl7iu@BVJG_10DvvYxG?}^UDZn7 z+2g;b!OZ9UK(hG?r*W%mf7UOrsuK!y1X02@(hc+%(%(A|^;}|PmT=K~uW*Id^s(ME zd#LGt5G?=%zVnsE@;A_9tlyP2fk^KzUrfB>n>fw;H-dGVKf^#3f08NIc z0(Ka4xcyZ0lWN8R&JA^te~<~ET3xHa)V_Lc00I4EFLsG-ut-!g`i@juRR`5Rsn;9@ zvlGtMbK~)H@7$?<$;H$W0%R0SkEkRn^YIm?i4;^FC0ol`u~f(NF;RSk(CDwvHO}M% zu*lpuFpzxdt3JjKdf~!02DmvnV4%VQd|S@)v!Cs*Yt-~FkbC704K>d+6|zt@F02K z3>|;SuAdK|f|%{)cesk_X3tu;9}umDMDqQhsI%rDb!DCS(cR2o_N<7%OoN&l0l|#W z!2HvZRwm8B3peCF!!V_kArO~S- z`XLbckOqOcA9!dVo3}R z%+R8-7o)>Pw#mzzGH_q1x|s)XR@MS09AlPB`0}0dRqG%xJBNw&jQl35kNZCJU86xj zgX%Q=7C2kzRJhR}K=k<&PJ;|h)5}D#b+e-+W&QvyUNF?1`H|M3_hNLSES*AdEP^rsN@g;c;afwgN6vdD{ z5_Ei6U!|(DI5?tj#nWu%bXYz;`1v@c6Pc<; zkxM9=$jScEO%lBDYOx%g_LMuo;812qj;1xijF(|0E2CYcJL7KSC&$jR3$>?_>ss9v zoF3X2LCs*FO8V|hk5=C%`ZrAeX+el`@_vsQx4!LO*kuGGik-*H$Nw@7eV^toi5^Ob4TD#Jr!__tOZ0M|vLVYs*q!1ICTqF9XzZcNh zfMvH3bPV@nwOi@zkz5yxX8`E{HOPr5OV>>b(4>g z=66g*o#JPL)$9 zcx%1;6hF3K`ymAlGc&GHJp8Gp!hbS61fV)ul zUOKCmMJwKtq-q*BxS3f)mxbVKV*V563)5dFC`otTh*GB2y9MiD`0!R-YBsDnNeYUZ z4&E>9P(t2K*x9^n54{abnlj8n0Ii7ETg1X6HhjBl209ykKIfZLnrEl#x`ZLjSwa{KDGs?F);Pk7t+e4Kuz&0)VnYB zP6aJOk`j6+g>xC1tE`aBs>)axQqzp0!If>)F_DiXIlJu{){S3{6|=XKf+p*kGg9}L z&5eY8f5;>&>(I#2v%oELsNv#E;m63V%zbs#>1apcrFW?h^x*T^S%*FHs+!NGO&P5# z3-N20VQwOvB`DJR{!;vchbb^C`Zwe%S8KlhJ3`S|K@j(_GCPPdea4N-|D(4c#Gfq%b z%+nZmy_A|SY#(679lFe30#q1*-}Nfs*nA=i%E4*ofUOcRqjI}lw!Iw69ZB!T6Rs(n zF-X;uBj1WNv}LavwYGY3@bD-3?)QYdvbl3$j_t>X9m-4pFF%f1(OznKm1v|YXU}cf0JV@D1~K5%q$pZ&72bV zVB2^;AhsMCer9`9LPq)A3%Y=ke@Q2-^$Py~7ekPcg^Z4DrqzY@dh^bg5_3eFL;2{kZ%oO=pN3e~HholHgona-u@zI&N zqlh=qdfE1TIC3hhmr^x!63mGS@QbHw!k%aT6L^AjiL}V$@K)^1trgL1@cqe2h_;m7 z+u=dzC4pTO#e8)cD_YXnHpLbe?~a8Cmoj!Q3O+Ba+&6 zWkJ{t`xXr=cI~2dSgP+}yg%^W1lm4>nPlcyUM~4d5%I(%^3iWcI~sNQesMCK@`J@x z(c>A9FOzYc=i}E~tWC5HDrp1PUyJXdhiVIyrY?Y~I*HD+kYBa3ot^rr3+Y{imE~Fs z2R$AuAnq)HAxX}rDXo0O%#uRwvtZy!r6=H3x()mJV(EEI%;24AyK(u%(}t3Q_>fl% zp!RV8Gy?mnE}3Q@c0W9Ch-`%cVBFJxNbtyytBI?NKve{%&Fdpfcz+Xx{ZLJt($sCsZBfDSq#r{@qLOPAOy#PQwj zSG|z@*}{$(vE%+cw%&zG3^_e*h}BiT|Bk6jB&AQdCW;W6`3g_7e7_ zz5CC?FS)5A0K~ud%_NFXZ7d~h?yf&vuD~dO6S(wvWI>~ED1(VeqK}_WfUsxuc;u77 zgb#ED{j&>?iO|%8^`0Y#+Z9&O;AnZzxhwv|kN5MQsS=YFI1Y2P#i2nRfu40@UHy=unf({Mj?S|G+O&Yuf|XWAN&+^`yeh|8xDxv&)@~ z!!|Fu3Aa57L?JKz=;^(rX|5m|TGQitq#36+Dv?+vf?z$7N>izs7XLSiZ(mwrtrM~( zxjXU^ktH`GVLPpXxxu(LbPVLUSBI+*C_)`C_<*T)YAUTZLZ;Dot6tPJn9Y}l+VH_1 zD9)%Zd_XHfwjE~qgG%AMS05rm;gWB})8IGRU?t!gr2(vz$&Rfse45dy`qF$ZvHb&X zyP=$K2fa5tJA|RDI5>~hRb(>te>D){2Uvq#PVyN=jSrF z^UN1GR{vydC^tMXnq;Qfs9|^3nUJoP^5ssU`x`{{tEHljaW zSR5PR4@;=>9-<5wEYqwBaC&B7cmyb=Q@3jeW{^9p3m?yKunZ~9HP;kj{OJNafWETG za}>fcmF|5=PYG5hKccNsnrp9anWz5TJbiuoui5pg`p(OHA`h^T8zo>m(HF;Rev^NI zpXJAU^jRMQA#v)>x4XI^brME@ z)-{M6{_{H6Z-=JQ+FjAyOfdvAzA@HB^NH`4FM0o;dU;p*}3*l zdP^9BhMcu?C3l3YX|?^ZR-QJL1rE%rL>dv93x>dtG24K7vzWV&))K zv4MbHyHPRnQfNalYS`=(gw1{^kUKIYl*RYWRCZnvvBh)HZZ_6IOQo; zt)>_QR|>F$;q7#T_6WO2XiajSp*Hf2wW;$~)LE)Hr;4&tD121FEFH%v`r~d-ngX&^ zvI8ORloF1Vz0e)TAZ9%0>e?B9>w!J_H?wvB8>dW_BuH__-QP6|?DzkK=@iodUjs>0 zV8ZKkD%((SHX*sSU=f4GbCzOHX}z}3qmuBiPoH}@rY$a zQouFYyZos@d51K|r3wA~uqgeHeN*(0YZrLL%4@>dERxg$Eto|6dv;5BJjVDiP36#| z#!4=IH8{AiFo^-jM=w$c31tQGKLCVl_|D$s-&HM}UN5UX2JVHG>$n2q6LN4v>f$^z zL)9nc3^?vE<*B#AAV3zd>r7Tvmy1fI0?cf+i887n@iG zt7M9|zO)-XRH_olz=q(ur5T($6GQ z*kfChzphexh6tl-1H()%ZU!O^z7FyM;n59L!oTP~=h^KyD7)-keHcKVjoDqdFKpn z@DM6!tNIo#v_Ros+Oe+d9=m^|*|BcH#1nXA)`|B{F9&Kpvf@L z*V+26>omU=n>dMS=pM^?go>>L7J^pD^z=$=~I+8Cx?v2j9j)Q z_A-L3y&Z~fcmBO6Lhfh|=alM5J=GN6_~ydG-Mol^0)Q;Iqk4u_FQo_w!<)X#UK91@=pzadysG@SC`37q^^B%|ys{Jojt?z`sl@o*=pVzDp6s*#|1xMtSF_caV# zJ<`O6weZL}2rvWQp{Yw1gFZhb<-VQoL)P^+79*>Bpa7ZUzGvX5HXCwAr(M4mJCqyAxSn9 zlcqNR21J)`mOU4lYdEOfR2daB%DLwv<$c}I9Nc?M_ooJbH#$?K>zx}Rz9Pb~%A)X@ zbJs5C__OK`Rt=ss$zo`(6ixnD&5+$pgXZkO_AY>{OSpyR>l$liA_U?e&z0o*P_Yxf z-&#rkN>VZ-A`uWWQ9o0@UG5(MbPQJ9zZ{XtyWG4yJR^MV5%rFkEZ!5a5S=l=9S>T# zBblksVa9*#BTY7Afw<+$M-2W+;)`33?-W?|&%LcIpvq7hRN7I0jFq6>f#zKl?qhJ2 zC~Cs85%y2=Mq~w<-o?rBm%@f11{^=7IAvGwZ~QW^bp6T+7N-R;OYU!(17V?B#7a#C zvgqqV%5S$3OWGE@>Yr0IlOIkCTLWMoqU{^c}Jf6PtP7ZDF!*U`mhH3llroR8USe$eA0#kVH0;%4= z_3F5m0Xr@YT?TS_0oRQ_Z+Fi)31;sFWL`&9l>QleQufU#0@EQi8kxUd|Jn8->)Fi^ zQctL^NPhb!gNAX7_AJnN+xR3MT2Bk}YfM(KC;!Tvy!T+Lw$Q;pPax zWO9mn8H~V*B+7&V;X)0vceHRCkgzWH!k>QyU@b5T&s2}&!vYkrOIk!*R2!|OE-=UN z&pmx)kU2O(cxNY&Q<_LnOmb+z!77lLk9J++;$>Sg-*1vi!CXX_3<07#iuZs+Y0{I9 zn(s3|71B+~=Mcse}WaoAl!!NCP@JhZ%Twq_*t4)3JfUHplNYhQ** zBYcakzS8*^zB%dkJ`Ma`E;^u>r~3>SsRc>+xN?YmRB{3t>s7`P#a zCzdCNR$y+CBp3ucb?KlHQ}$w&Ivup_O#!jV|A$^mh_KzWmJ4HD43U+f{s&O*joeIX zyY0i#hc6;pQEJ_krF(+K^VhJY;gXvBGO-z9;+D%9(K&2KFhR5DH5&;DU$;dtK%-V* zEok9Q4yDXD!PsWAdGn$tFBA|rvN;8)D`+AjxY5FTS(}o){OT}_(#>(9y}wToXIDgL{O*9{jSD(x!nujXmPLuHv#0E_24V*) zQOa#BH1MJWEH!**06`}t_fRi=sBxz8lIj*&=htoXGj}neU#qM=7()^F$HeNv719^K~JC3iwT6nW>r+Y~Vj{4i!>O0jP ziX#Q_l8YyyQ0p%kDu_~#IwXjZTPAQ9W2+Y+`0}KNl8FSrfCW=pX1)u){0Be=vJ;~q zOk;(k&}#Y94@QA#1!0tjG5+6)Un-t)QJ6`k0837Mb4tl-{ifiEj^b#3E1arGEoUZbQ!5hC-)xzPolw!Jgb}^ax{jL zIyCX@ySuU&LOB}}v$E&3aOUR`21LX_tiIXP6^DVkiFm}$S~t@%8LAdHpEI*NHuYFF zz$vCy;-hcnSgauXYa4P5gYH68DTLD3i5)lCH1vQt=D`;^1sNPDe$i_1mY*56xg)`J z;FTtEcvFMJlJ{1g^*;O`U~5}mLSKWF)a#h6OSbS3EGT%0d>!z+Pe$dz_5lIf9v+Mu zI@0}%@Oi@9)T8xCO_i$q&@&h%^_Uu-e)6?YHOFCg7KDm$EYQQ+&m5q3%?Vt}tmmV@ zyQxo#!p>9o+=M}A(`lVvnppmKG7$`$$V6wDK48nyPi#P>;V~|V4sio?5+n_k2q0_| zJP6%ga6P^%@E^lo!~UJVUdl~tmoZwbzJl-jfD*JoMO8rXS&`!-jTBSTv!DNK{X8@3 zhwn}nMz9o7U<|sPk^r}CKbRJI=C6$+w zj4E~Eb?>g8I5F>jdnYAox1#trbr;J96sF&S6_r>s$cy5 z$ImN^IQ*-ICMyJ}hGQxND$A0HP#c#hjAKNH_I!zg2c5)Zos1F|aL`5xCI73;ceymy zFQ;`FlCP9nr#r|*FOI@Jq>}~Pdh|L`N?M7cMh-gu&Q~-lb(Mbp+^#dtu`9bO2}J zGF)#ublk7tbb*aGzmoV9Xy>aVy9uUzy)F8%7uHkqhkt92YV#gFh#Axt4H5TWB-DdG zJqAqFOa3EhAsGl%#M2Ja5DnqlRt*W^-Q`&mESz2lZzf=I#MNyj*Q$ba4yA0c$badm zj>9drs5f&lY^5H+wnSSZJE?USC7ct9CzJdz>7x>hroDrxmpI9~wj{WI733^Q3WUxH zwydBily%mpqG+-zX=t)0!%M?~P}wxPQ}Yxh&PYJnQA~3JnZf@G8Ot&_TH@MwKV;EB z((6_=t3;zoNe|)J0o5qe?Nhi!sdw_S(K2KdJH0djJCIgMf2Rz7VsWz)X;UFuyi??$ z`q0F`&x~ak69R0~$`UQ{wHaUXYk4iVcWBH2uBe$f)OY`J zwJ_J3#E9E$W9k`XM>B%!M$a4R%nG?JQpQXP9W>v6u3?9z1uzFiip+b@PwxpBlNNl0 zmJlSs6RzF^oW0v2=HcA$Ch_kl8cVA;7!{ibL{}N_RTGw*=OR>C)|lDl2(>)28R8kP zL<)gpghrV7do^`(HI;oEX|R(Jw1sC)4DZkrC&4h+!=CM4^a`Gs2Olw#JhM4FspMJl zRra3m?`zhw`WL*X(~0ocC3BSi|h4SK@feJ8T=0-DIT8ezv+Kx1SP@&qr(xbp~l_BhZi>w zEx(dF)mLkL*NojW|Dnf%5w6XQC*wgo3pHm`nxy{u@zvbkEiC%Z>g=I3%+k8eY^F4P ze`Vx`d$WeW>tl66MBwk|8r>{(JNbwYZgg}T#R|mqHYRozZnp{s4hXM8&KNp`rd!D! zSk07JIPEk+(oLwyl?Wy`mjOU zsJQ^oT3&%a+>L0J2l>Pyo2~ZI-0CExt;#LY09#vYo0DU%C6Y;+G+#x#H^r9tOHu8_ zvD2vUKisa-bp#o-nA4A&dB*cy?o6O}cp}Bh+$k0*o67#|Iev26<|YFOK`l#zH=7ij zl0I2yXta@vqZ`=ASK?rJElgEcSJdq2KLBA+>-mya=~u(LORkLrOwDR0^p5_GO3T~& zk8HO`D9WA1P30=lveDk!KS-2p9eJM0`es?Ve>AoZCX>67rl>tub@g6yw*l>&e~T6D zS=2GNi#@WIjoY(T*oPpB`XV!SXERe7&U*5fQgWTRU%HC&0kPy%Pt93x%sA z8LY6u+#B%pk#Lygd*3;yRSH?!;cgGuV!Dyt)VmK7pCOQC{XCOz<@kxg%5x{#7-DP~{bTN(8a-66wlWYzopG7= zIn9<SqQh;(8 z^vv&th?d)Z4=nqI7vp*VIG9MaNqRp}6&Q4iEM~?k20cDR`>;)E8Kr6e3Qul&58(Kg zsx@EerAj|4+s8t&+6&fD{!}SQ96Q3E6rXm?^c#pfm@R;&85iPY#2;7E*j@EUj4eJX zX8a0>zep59dCt#}d*n%{=ojW!@+$DoMKV!YL0LXw=uN}y_#d~3V4;B?NDP)&b2{^)^JNb%c+E;MXl}zMQuGf2iq^|ha)_%3+U!oBnCe5>U->pC@u@g zP^H3n?K8AS@|%$AHIinrNlosGk|}rsjT?;MX2D6T>sL&TS{7wIjTO`^rL3;g?# zoL%9rpPs;JunQcBfiCA}^V4!kr-w@GXA|4YWP+evyqH0_rwOR_b)U)QcJ@+>)6z>x zsj0WFlV<7yP1UW9T54qkMCP+Q8y7X9QAlNdx2ORes)HN`EOYwNu#>Qc{T*?sXJ9L! z^y~5uorsCcKD~f1&^6X0YLVCJdHqvqF1>Tg!58(4(UZgJ*3=Gbe4L{BG~V{q_6;I! zGFL4%8$Eqg-8A;V(lA8N$q|q6FHt|WPW(R!RU78j+9$pX-}W57U}bGN7mn@@jeoEi z8gPR2o$2i-3~_zo&&#_i@s%CG|E&n;yh8rx(Jwqo%UkzLn3_)_0-=I&EAAV4rmmKz z!V+Y?-0R;laFP$Klo zIO2`YV&C%d3+9;hEAeSE1OiDl|2teWiy+k}{F!cnT9~ajm%fy~Oe)@B1|ulNM{n|q z=q1$DB&BY;+XVJnLrB3{+Xoy3CA&bui&qa+9)}d6tBz@lWrT`R%XJ4Pp zqVZ{}8xYd(ANz^ykhvMC2m1hnD>G8!!;A4gPC=}Ur95P9o(o%#ep|qt@Te^w;_RWf*M2P8IAKm=}wlnPOr~Q%!s3hmjMuUk|m?yA$l*L{gUBGu343ew)Z8#+izuq(!ozZ4yz6JHNi?oIT7^{ zc(nBJT70;fIAVh_Hv_c0vJmBugW#1-w)_H*J|LVq`W8jB4(l~v za-HDxLZ}VK&zJO6w5FE)?9n%YTpU|KEHN@aQH=5h^Cjh1w(%XOTY*T@hoYOpmTq}! zJHl2LdbdQ?=i3|;G!TOLeRGXpt=QE6jD7}esYfKeXkVUug2MmLSW+NxbhW_(kCaKv zR5qAQEAiwqNS~|{*FkDaLN4-_tWXluv!Z$O(bv3*seIx;fbjoOfK~n%*^5C3rt>rY zOGVk{)c)tC_ForwJhuE}XI7VI5WUl$Ex?T_eci%_&w>tK;QV=jTIGKUuJbP=ox9ZG zz?6zCov`16DJ?H6SV>Ffa``k9uE9i@!SFqEe!oJ1*h8QMle&$UAntF{@U%>@U zWHAW~zrc*RU+g|eo!~iJT$tUX#7?tiV!C-++<5s;4Zg;jMnpdr1kvIO+mU9LduL`r6$^E@V%I=ueYKfh42&E4YH0NQ=vqp^lGnHEE$5Q&`3ra5kvRU? zWHRxf?$F;2TO(h}A>ijXyv6(t08&_8(sQ4T!k{sy`s}OkH%Xd@7f*g{BOXFt>H6L8 zI5!d{pnw?!mDyUdVf;u@Tz6d4;b9M0kE}y}Uyey@{oy45TppEUq8ofi^Toa#Mf7B7 zjdF*ce+i5#M32z2dNyU@tAtI%P8ASJ1e8Ee4LVtT>;q!c-;#m5|E)g6yu|V-n{y&G z{C%kcAFS74Uk34Ei;}&1J{PQ_@6Ze7C(8B)nVCKK3Zl?zPI@c$j3nK*!QKxi*dRIe z(BB{ke30Vg52qqZeCVm;;wU9K@`mJ7!lGSX<-FEp+@@3pH=+jp&0J)gm#fP(XDQ7a zB-^NoJ@< zu^Ki+^ko2IRku&D>z<;evy;icseqfYAh$$^GoDa&rM&#pj_#Vymq{z1%u;~j-cL(9 zjLag%6|j1(VNJ0eLBEwnqoLexF!4xbK;)0iB{N7?s+5@#<#GKephfFO{L3jR6KT(WOb;Fg+wavH|LQk{?;`zQ+Kn-t z_Hel$okNJ_rh%b+ZGw*il+khW%dPNM-fs@1EZJ`R-XW}BziyOYTNG1?a49AzZ7rds z9ieQU#b1RpATVo)nfmBnciQO&m(Ykx|MnjLsjPGQ{Wm>%_xYphM(-Y&BUv68Rsjm+fW5k;eV+Lye%Z_g~1WEc#gEJi!? z8`?b@O#^9897D{NW5H%a6E&>xG63)`$-rKB%D5yKGaphXUL<2%$)`t~6rNDwkwIDPBhx2Sjgvf>Q;?x%uy;j8(MW0oj`5s2GZ`U=s84KoOY3YjBx z2e<>WI+Qeb{GewFv|neU=-bdeS8mOHZRg@ImVf;IEmz6t^F7>(pB}(wZeZ0dkAWrT ztWNcJ_jX<<3#u$b;;@MJB{VFxXy-%0`3i~-WLlxG4}zyu z#ppHreib9Jx;6pJKYteL**z>AXtngPZAMXzY}&3j?t2IGcFnYMjoDFr^LH2_Cs1qA zu0ZE+W@|@Pf`v8T%Zr@Xe-`3@__WNW@R`e+Va)Id0WZE?6rh4k?PgK$S;4dIq}&rl zxp05vTAyW1zemE`vvlbtgNzp8a*GrkFKcvrv+?6cRSWbuyVbC&%6sNy9!s>9!g&Z+ zdChr5DCFB5ie(z|4Sbd+>(CKaaU7jz;80}!AAqg) zk5n0+MQl^7V0gyfRKbU}M+Ib(>{>U<2LsX+MX@;9m8**k3#y|BC8p1aBQaU3CYJ`+tk&vCOiRMmu|ciz%0O{gK7g{F(fR zEw-ph^q<=nwGJ)RM#*iaDr=E2L}TEiXH=z@R=>2Jkiqfo_oK2TmMm_=SLD^f&lh^C zH;v*2Y!yWwdAmBo%(%bHO(hh4yU28&!fB*(Z1(Uygnp#j$jbzdTGzJ-L zH@8f8@Qi?9g#r@i}H)CSMQTxpb=yek6uK$hF(2(XRc^T8c2# z5ia}ZqQx`VbS&i=Ei;=HPWD=us|f$aShe!{Nih#NvGJffGdR|&*9QLsnaz`6*Mhp6 zu8N$kNCo&qHING9j8~6(ri{$3oR=anohkK(wciBzw2P`fxuT{*Wmz#=wF6HQ+Mx7v z9pNF`6HwbSkE*PF%i$$&2F04ih#lYWPu*sk9B)bA2HQ5y1IqdZun0WJ2H3v;NMJ43 zVIZL3Jv=(H$x;t6(3qWg_&D8`Mw_1=_pwFR78A-TXygzMDa+8ies0S9_#a>!F%uTw zTQoM!>Ni9ZEL$4^{ep()y-KeZY`>?KRcY5$C6%R9;A^8e97l&nyBpsbXlC8T&_iXR2i0)Vzm9Z8p$m$ zGd5AxNW?GG(LL0Uz7sC&20PdQJSW?_HRjExFgW=M><~8V42)vGa4(04&MK~K>T-UJ zIiiGkR4-NEI%?Ne2{3SYR;ttjN?8U~qTx$VYzIm+Lr|5!7Em7C;l28IP-X&Wbs4e;V;0UWX0ZmQc6w{a z6xOPB2<_F6#rcaykiXtqf?T=52InUNc8CW`%c7@Av~4}+I@+!ktZ1y zJG?rZk`0LT_0Ve*XgC@V25lbz`ROPq0toGNn9yaY1Dw!9)Rl|f9;?I1g zf2cA^JB%l!0l%OXTY633Ete1M_ROQpx%_&|3CFKdKWkzV-j#Gpg;m%=V9WXQ+qAKC z1%b6ro`S&q>oq$={pZ=jg)1x@F@~@S1~s|eWQ@a|@@j$M9}fv~!f4Ht6#u#!$VG-~ z8+Csx?xkong=zDarU@3$6%x=F!&Lr>zlDS1H(EN-uzAuy?>0PUT z^&_|bkOtc%`gu6i&yu!joGF;<9h(e%ic?xyG`sWV4zm{L58HpGK3=$TO!RCfv5zMZ zH7QaJtv}>0XGu4(97Uwca8~Rd^oaObJ)Z2`Ml`sm_4c%d$cE}o4{zrrHD6Kcm=|} zhvBD)!U8Onw3be*as?TW#O&8u$85#yPwRwF_nSYq=hT)S7*gmR7is4zrWL7K{>jj8 zW z;%LEec_F!EX?(x+XSXPT)gvdmhcr(@Lyl$iU8XN@cCj(19bQOw2DyuNX*TmnTAEv$ zcC$`Xns!xD-T0b&M3%lX;c87=%bMjNPAzBC4*qJS#$_d|5-uGT82T|yLc8*(w$)f} zqT-cCpjS{Mwlq4J%dbaJ_>__l6gN@dR)gy@F~zE!EG+A+9Z!Je#mtPKnp7O3K^zyA z>&WKl4AjObUWef$Mqk^%8&d78!cNDfEB!ZPuuUwSL)eA6$jz;!$K!7dh)g@h4sCb} zr3u!)C#9Te6Jq;#Q#>QFAj>^5irU4OLB?F@FznFTtA4G z8f?*fPw5R<0a6_|i8`PD7Ee@;@%`rCDUs~#1*uT#G{S4E^^;27Q3W7Gj>%-dW^-yx zV;iS(jsKg3Z@M_3JKuB`Q=t7nD!^Ruxj=xx<+-1^XtHAI+vH-xhRBXaVR?^Gcs?}0 zb8+a9Yq85t^<^-=XUFey8Z3@VfKNqrNLI=}^X~gb6REMZMuutrlHf}l-gAeMP|#&K zpcb%P(Q{1(1BJi@kQl= z^@ES(2;G2QvLmhIXYz~S4^ghIfsDUlFGiB#Tr<1RB9v#lkI=I#0%s8UC>$>V7evAP zPLB4MbH4rEduE(^Bz7*mH&mOLcYgm>3JZkrZWb0lId<2K5K9~0UJIYWYfaLJu)t#@ zlz+s!7p@4T4ME0@I%;LD3n=~0WhMY^KfaW}pc#4%PvjEvJsT^yC#H@;NAU;L+8O`q z-k=Df!czOrX+HIEasniSf=vLyIA>zr&~-ooRc_u?k*Sw}5g%sQ$<*Wz4?84?IQioP z#Rncze~$sCrGuGZ`946?ifmH5`{7l1y0R>RizcuEw-L4PU+&;hh67J7Kpys4b!!BIG$ zeDWuLzg?MeUTW#1aB{+W7DzkNI6|?jMX=uDv95)0#$Vpn^IxsBbHAv~`}-cLW4Xu`>eqqlcKb)Q6|(R+HxZ^}E}|FF(qyRSL&8b( zZM6@PdrcdYcnEG&IWd#df`ds5^e};>jeFR9*d0CzH zlhG;&h*8dpror=ZYNmA{j8Z5c|t3b_>Mwh%BO z-G?3b(p=cS+F!>sUQt4_J2z-lqyzvQX6J$_ec=9}U@E_d$L(XyquJ!rrF$iUv%GuPR3CkgyAn^@xqq1=Lei^7b4+%jP?=FcS&v^XbOI>~%P<+~c;YSfBpIpD~R0f|G0IZrM5|_VQ=|WS?3<~`1%Abp?foO7?e3Lxw zth!H18rcaYt;bL?dILF4Wpb#;J=p6-$kgsawwtZK7X*O(2QvW*0Bpn|uze!duT)O? zVqfzkuRhq4l*-#i3CY&Dvi=_JVCU%aMVDJo8lv2@l?q-XCKQ)ke5`reAtO? z{o7k<(Plk7zyBAC3+dt`Ha6$t?=uyb$C5E)NCg?3w%$8D<1vpq8?}wsZ&^~*j(6%% zxc>v(7#pK^aDoz{Li|q5LHeJ4(;wR!OsuhAxk)l9ok7(YT=QpngXkI{V42dyO{AeS z_pLnWS5pQVs*H@!`aeK?hBg}0g|uHU@#5;sadMGjU zQ|W1?m)yEx01LwM$6!4P4%3u8;z4s~Riu561VkENSP@7KUZ{K>waik`?sOOVcHH+y zOcI0nv9GV^%_xY`jT@ zmtdhn=K#Z7SCca0l+Sc;yZ&0lB8epLufjsjDG9WIwQdb#_u?2k$f|XD@RdU zlf(FPWhIbB-dX897mba-#n6r)3{h#Rmkf(0p5w^&VCN{HA5C+{yMj&gZ0Sf?aZi-q z>rBx4{ed8g7AxHM> z(QkRT{wyf*6G38m6Fper1o+w#2;E_wNAgeSy8+W))rqb)c{j1HRpXN_W%F&Iml#d^ z6AsXr?Po=K)Hk^SaXv*;+F6nS-Ow?o8K*XK8jfp(N1K)o^o&2N0xAjU7(CqN4f`oJOyx>D_hze-ZAvaWqKZHH27+( z>yR_U6U@&T2`@q(1Gj#2Rphn&ine@iJ8*Z(Jx{G^<4iw{nb8!l(2z=1xrTst6Q{(E zcH)z;#q@?^pv{=8%O=lD=Z~G4Yqik@!7#(-&vr_EcmwRs;t(#DsYw zg0-mUp4U(B*j{20{JLzAx{(zEE3hc@ijO-#g>$8ZC0LWcCB_$A46g;*hloFfvHHww z8uv8X$ebAXw=~LNi>WlV@7+BOJMMMBPdR-`SJ(PeR`#QO44qXyVm^n6GvC@8m zn*HaGR(TEPO6$JxD8Hv0d2b9O$fksD^F|S1eBO%+&nX}xXLM0@wuYYSYM~frbF{0u zTC>{DQ}Iu&yt`@b%~XC<@kYgd>fbFH)bfhA5VY{7RmUt;yB1_K1lze4v=COJ1pzo`RfD$m+lh1$hm5SYiuX z!}*14;f9L*V&-wKk84cjzO57l$mraCMoOPb2Dx}JXX0W2t~7^vPV7iV2F*ztBGL@hc-)t#_aDQT)+Qi`68?x(d2kg4ZVWcRbQFpKe7# zVKR!hwaiS`M<5^Gh@?Y;A|BQ+0wM_2E_W@ZCOVh*?h=`EO&uGl%=T4+%y$0(!uC1p zcZkMse6Q`YdVTm6l~q+S%Lbz`D@{#H6?Uw{7@q@kH`S8&NHK9$5 z%ol%bX^O0z$rcm z%+p-PHr>Z4d}f^PU{C4lQM-I@qN`+-eHgE!Wo>F(JX!_8$@=ENzgr8|5p34VAStha zmC`ybJIr;*6nu2XgskS#zAXdVJm#!0JCR~nn8D**tSUS^wGA7+4oUPIvM{AmY&Ar(=kBn_CNY@L$nQ3u(W-LjWmAa`MHf|iuxGZb2i}OiC zhzNa!qN#g>SDB5Wu&MS)90lP>AF&w9SnBZpoT=*s836soPrVwRdP7xasXo;3wkUCVC_dJd4|s*lMRLq^m2c`RD7XFPo5S<0>=plee< z!_oHCznIf*$-W`DanE_uZD~}id8jq#?(JYPlLudGSKkbNsG7kpD`t+$cCrGps56Iv zcaxOUqkDEx8H6LILB)e6vX;$m9UkaoOnsKP6_8JwQAzV2Or}bI68h3+L}*SuTe<7f zNkqL`s;^guavjRE-`M!n`yuB#pf&8wjP__|RJtVmBFUeL7QYO!G0`|gyRn2n8_aLi z@ioJ&AzinEr(}dRF|E@Q$9DIKdaIF1@?sub_&lrpjg zu&vis-MFz6+)D{>MP%o5_-Wdo635z5=ao;mpeoTWyU9bBR z-mCQQ!wmlGq>C@>iuv36N?iM!@~P6hE?=KsxPX}*e&z>Eu>sWR_YR}jtHxV`xu!b; zc@-m;v?=aq$ygrzNh?iNj=C%A$tz}fR&jcfm6=L?puI@!_&kGSHJ94=qydqrqI-o- zFDR3q>n0AXtZqVS@XI$f>Bv;EWsrj`XQey=Mg>moMhWNl!ESiYl&eXT6+{E&w7q<= z26N{eZ}NJ>Ainy7U%tX+#f_=kQy;!Ciz}C3i#$$ZWmlN5p0Ct0-PguLyu^{;Nu-Cc z&Aiw!m1tt=ABjJpMpB0b=!=nc#iaf{oSA$7W9n42mk;HjN64XAdk-|#>dn_I1GZD2 z2An^|joTemMJK2Ljm=(Oyb}*80!ot;NyO)}4jpvskJ%T~hv*bzjZ0lUPg*7^ku3rf z{ND2}->KVd=AYeaeT%~SdeK1^lk3m~4FXW1J{i_WdU!68GlUd`)xUl4hr~X}&P*Z5 z!A=}$I=h>Rn1%=OaaM;E8dZ{(z~iL!jh|AW^A5uLDBE<5c=2$f*Bp_z+4y;h`ozGg zb108y&OVGIq4fJ@KwAQvxDIgYb1yvi+e*N&$to&l`qSO{n%+s9Bu~XyMZnOmN)$dY zd31N_?JW-oy!~Z=V{S##OWH^gu-cWxL+2Q1dkY9e+8x6U7hT;P+~_d?351}_f>zHQ z5p7fbVfG}jfk`kTK%Wb|Q>-zJtlw}pcMOG8ys`YUZ>9eLJ?s9Jl5}#L7g1>iY$a^i zw;ZqnI9L5eLD+I*JJLhA{~v-&O%oQ&zcA+_DA2tDS0Q)Q89$Ea>N@Q}!`n}sVrWz& zGswfSEH!s|8Hm=3XLEi>Rv0D>e@cVIf(=Xsy0qU1;}9o&I_JaYOWj5cR5Kb+Vhe0R z){6jPJl$ozB|=Z=c#an#$TmTtj!x|FG@ks_vG^Vr(9ff6U>N-7%O<=GfCk+$1R4BZ zWIw6aaD8*k5mPDZ*Q7%GPzmk(de%k-4`*=&5e1k~nos#z6tUoV^CkK3z1?m450SSX zygW6-(bZ&M+0}bXID=oqP z$@*Pfm3|FrNW#k-DNZ2x#b1`Z+wH+`fcbx!DjHPPMR*^amju6-IJy6SW?aI&7WYx0ecq8p}3c-`+N{=RXvV`*&>M5;Le2sT_QG{^^}OiJWejr^ zMWKw{->|yr=toE0Q0?C8$M1XFy8gy`5<*DHCh@QW?e3n-gOWIHD*|ROtHf;?|P}j(^8E?4nFcVet=?r%v$o-jPpgX*onj z77h!gyE{5SCKN6=FCb%`Q%%@Jy1KC{j%M#wJNW+gk+rP22X9n5vGKvyY)&UKKDhN5 zoAv>AZ*_F>CEzk+3;=iqH% z8()F!`+m0Hy6WZ04)GUsb5fhcAVd_$g%P^hKsZItgKw~;TXQ+~ZU}FpOQHS3O9cvS zZolb+<`+B6Eho9;OX-9#HZ>Yq&>-w&ZkqxiQL+WQZa_nf9v z@FD=B03*j|T|Guz2XlBOq^Oq{Xg?V}Z%|fBiWUKmzxa^|LTv+*cm5DEa5wPK(GO`% z0>1<`8Xg+}h~1bGbbH#GHOb8P%Nm-hnO}SgoXJxE;3)Fs)^>8%LiZrh?cU)TfvDnt za7?D^>F)uiV9J7JeYkv=GgQFsg-S5rrp}$hsdZ=2dJ%#S6yPzFIs6Vh4yQ86)1pH& z2j9cIoyrb5gNcr_wXyqO@BrO>+bsD^B)M*B-7tLbz9v5 z>p^Ayx*fkiAh0@>8A{JNdhl*OHtl!$y(01M7Y4IY0peo+^XJJX^fN?x4v}vwSmyy= zP(jDGeE~o=m1D;p5;pFV!aj;Z<8v;078ZV@`t1-^HCul3E4H0ot}k`~|NkOkI|VfE z`$gvrIJKRb$5?pA!RNp;zKVe_1;z%U574&kpLpLRN9pJV6W%-h>!p+2{xd}F0dV;G zwThU&k8OR4gwF<0f-GGAO20Zrr28@qfo;egCjS8zkQu2kIyuR)#k26bMNXw}TGmE7 zdb$u9dbc7&0C!!hPq}`oOMx(JegS`|YBOIq5&TP{vaUq}Ta})04*7jF`!^3S2o98Y zzrX>WH?wCJR+`z-0J+b2`!Cyn$c5>$Xm^`Pp3e5$pm3|N}N3+SfexbdAeq!Y~AC)_u3M+o}0EY)Q0e`&OH( z3<>>(@7OGQg|S_^zf5Az5Pjn$aZclO`3#?!U(WwMVLnAN{j@jM z@6*An9n_mcaOt{|`i_L|yB_hp zNRtQn=KI|ApEleu`W_p?B4T?~)Ou99$LJYQG6#e?6i)K&!S=&$0~dINdpcabov5vP zPEPgKd`SQoET1nzSDE>m8yL;dBGR+5rx^$}ZQ)bJ7~`L2Mm8F5^6nr)22n6gHqx8C!dL8)qAqv?H7;7rRo9DbDM9FBJYf=~Jnnd5fS^JKIVh!>GG^ zmDI2Kg}*EALz_}B+Z}X+pe|;-81zF49W#vZ^&rB;xXq#A=ZLGt9i*rm^xW`9=8nct zNWMolfqjQTKqr{>3v1V3I_}wMEd_-j?yx2N7ux!O_)q5(Bk4jU^!#ztYZ-R*RcP^p zNX$CId`I7}XtVG8EZ(O)RDFH~OqdQgiXoitT)Ef%+)PD8?|BN<(5b*LYikzVNz*DL zU{Z3n5v7OdvVfwQLD4A=ma2j~bV11Yh|ihmD}Sn$hLaC=$`P){VXE$2bpXYLO@g!ab{HdqX0yKWx- z%M0cM1j@zuN9S`z_=jF;>4x@|qrr1xih&w@xPba>Vf|d(XH6yI+D|Unv4q?h4V@Aq zKEppu!oK&_%3&WYYkzp*N}zV&4qsf1$;h&|2>B5fgV1sdPa%j~>8ya}-af~^J*}bL z$uiQmFXtpPOk)lcH~8GI7`iro>mRyNcU#s4H9JEaa>5NWL;3a7(?FU;%)}K=TJ4fA zwVvw>rNumUGxV&yJNp0(2V`bPzEGc`F!TTa{3x~fTtdW@ zkq7+}1$*hRsAZm9E{j0Tm@Bicb*^`-5!e~wX9W2HJ!I=^ zdCH129it1m#tK*{vrOg0g&9pFG~f4qYsfQ8IOe_wDs213xU-^39);NCDADf=u<_O# zd1JzAl`@0kft3byPy}M8C4Df_5{?Q6D=|Cv;{F{;I;EV6($fm_teQqThDG5+0GTBe z?Hjzv@*^guHR=0kc)bdjyG+R+pB#uS8+_S9(xX)i{%%^j4Lar1k)qJ*R39nh=iXKh z8{VsPs|||0{RyqF(L5uz>{Hf_y2SPzDSnYeVl1lk6B<^ zGf;i}^G*D$mixx{FD2~fo(55I2!?&a)m&%Mtk&GucF?B<9?>&K>$D$XjI&G&WVI}c z3yNQNVsfze;9E7+H;>?P@abQl@XueY1GxVI2A+7#+x^KDAmcZF)7qdez@PrlG-z(N zM$AT_>BGZ?Nn68S4F!9(R>g|d2>f=-mwuFS`bKUy(CNq;q$@cIXP;0*k| z?fngg2q%dzj6P3xVQ%hs*uNGsZ;u|2S1^{JfBwc?Wb9OTC$b&zqxi5uTLO!&s^*yW zXZZ*LGQ_$nr^5PJ$;P{2MnFvR-OtaonJkjbPwaa{Lt3j!h9#O01plv+JF+9o7-R-+ zzCC0VR;gLF82lq`j&NZo1jJ;ZS@Uiv8WUWlw)GFdK0}{!@C_!QTCr(<31TxD06rlnQ@Zq=?)&IVV;w+aUj}0zRbpe?hI3d!@ojm-GgmA_D1E;j!5@$NH&kaDXuC;8nHUbLX_5^4Axy=>(`8Rw+Sf7_{F1{?OVe-O#a#ZmdMRh!MTa9Lc3iF z0=tM&5wwH|xv(;uJzlSri4w|iHj-hnSsE5~Zbf%GB;0Gp_|BbPZ_#okfyX|dFIB-aDIN( zy^6~E9GCREratbYLT#1xPq*fBwRa76Ee1m-5Y*o+fh4bTrD)NG+IILQz&vJ7a0sMI zyxVNlx`q*r1lJ zIWZBB3dRxV>cF5E*|F39mbe=z_z-ez45=K2izaoi9sC0e4r*zZ%#(zb3;L~q8;XK5 zgvU8bXZ}Y3kDiVQ9u`3!L#>hQ*U-~4=7yzF&X7V5gE=O70{x50sX)1HxoWc{w(ab8|DP_2syp>xOn=R5CS#&tZU!)w&v|fwt1^ zf$NXgoWx8}Tbi@APEk$Am2fGbj5UkE-vIey9{nw70l ziLRA@VQDabChOFXoRTiAsRABvN{f{wgR%80&HMu+^}p&;AKR{PfvoTihR12(TAtGg z*V!Mbspk7;99+t$r8tIc^udf8RHs8t3YL|Q^U;9nuek0x!N%OYF#68DAiXOZ< ziF+A$jbHiu^pq6ti}~N#CjQrMOiUqKNng42AFye3`-*pM*nO4{ z-or_^(b-{m2()L}*qT?=2y5e^7g?N+LN4RR3O0A%$xj{Kjx9hOqdk=?gqF!C!UG}5 z_RJDvvqNr8`ixF(AaQql7N=G)c=6(}=5znVKY&MRy@c3N;BB`=Q4Gz#(1w`(6}jQf zei5o_>I<}o1Vc)v?=|SYSDaYLy2#(>NSAzDSQKpCXxpuaf@8)}5@{l&{&>2phap?w zS$VmhkTd~g`j7#Bmfo<~42+;dbuvB;(D(Oi1Z8Bi)($1!p`F#g^y|bkEK zMU{Sl(L3rqbY!nEmhzK3cMUl4x(%HC*Qujbdh(Ps+Hp0?dwNIu!TRs`*(?b z2*w{(E^v~Q7YU#uXGeb9oxSaBdF;av%7kw#exjfnmd85Mo;=*MJe>9F$9Ev+n12%a z$q#3|i6I2vJ-E>pa zU&5e-7;?-a3m7u#o+LGXBwJ{Z$CeY_(X)CE?csLc>$gJgiDVw9lFfGPTO5X@>=oH_ zzzb%U^Uaq1)LpdF`D$m!Ln0UoGvPFM$oFx7ToDm4+=^OYZFzCrl|3p`hL2CoLK??w z-JH(DAxD>c>ydz0Gg!@Rr&@x7bT+(K1Z8gy4_I-=$+0DKQ%byH_N`Yd^n^CBsI{5Q z^yfsNol2iTI+jSHSJbTSWM~b}DjRs1(M2q8(r>3?agyC$4}M|)H*!1Fc~IET-9{ZP zJi7hY_!FG##sVa`p5a7og_hnX20{i#{-gQ+-`JPD&F}|E>Dkbh{IF;zNHc(QoIx(usN%oZ z9}T~1MR;UtwND{nrl6IPd3eJKWtvsl`idPXV-u5?F8bn$8pt)&ul`@oizXbFUFdpG zL{!|rzbJ&^fZ@Yo_o*tfi_PcdcfMylBtlx(vAox)F6^cELiGR*Cd+S+i|JXJ_UwrA z0Dbec0+ZIYO~88g>tdkD7fRbeR7_|NZbZEy1csAB`1QU(atWsE`NYB3OZ)Ey{$cdI z3qNx&J;cOIy^Uz5f z0DVU~W>%CV1J6=Wf*Gb+2?gU@85Uv_^?KnIqc10Jhu^;w1Q+rP*ZFfuV5RVq^>)_! z#=ph4Z=y&Y4!)O^0P}_Bg8=SC-|EQv!CUF?qunGD{kr<^P()-+{9p%b>3e@ZDnS?7Sfyd;^S zZ5VHTX|Qvi^fgy59K9gK%krT7q9aoAh7^W3bhYNrQa^l}b&2k9Q7~5rP^i;hE&pWO zS0pMezc-5WfcM>jL$VhD#ER^Hk=_bHRtu8`6vN}uVM9a*teXbbl+C2U;ckz=1{G#g zAb=QxKN7=@JrbJN51*HN{Yl9qMk7jCM}6)@#E+(MWyDi#HVH2d;FV$JDWH$rNdU`^ z8i>Rp(G!HEEUA>@8A7-u4}>rUFPdBDKX6V3o^rgywLQL-i0LODJhGhOmsSCA+VQxY z_XMzgd_$aWDDq&Yy+&-RssxHD$zCBjyJlN)L~KFub_y24sX^YEz?6^G(AKG!MBz73JX!A4o9hJ4sL)do0gMo>} zkLwp6WE#8Lrcb%US@uuAbT_f?@VR%)#U;lF%n!DAd)zPvX$hE!YGFy*R;Ijl-XsQhxKSe+}Tv3^`kk1fDvsx$At&k z`SMK0vVF@;1XIeY48^R1x;!-uqoRa-kSyP(0!4Xd#x|x=Xl~ztCoW6mUT#+!oYfv> z)C)D2z_?fkr&=^KH#5>oL;3!1!Oe{tug{N=SNX5x2uX*Z%RTd&702-BQJpES)hm=t z?>5LUuqFF&lU+)VV-zKw56KDfC<+e5Qj(NUWog3K7PAP<7~yoqDMjP&Km71Q&t(^w zG-ot*dF>$A8#Ie*gIVrD$PP6M;xoN`gEIZBuc&s2;<=z1hq}tLt(u=@L&>VE)gYEN z$#i7(Ij!;}g`_e9Y&RxpdC|FB$0c&Uuh&PKdXTP2QGUVoLhwhffY55 zVDybZ%TFZ3Xv@NF+)as|d_#66+<{*0q{uhLE8t*}EnJ5(;-!gU4v&X@J9i7<3k0cs z3>U-jjZ`?RL*b~%=dP$CP*oj~{B^WXKOLh32=fIcKN|7M^zS#q=|~GOrDa9OAiXLc z4?d5puH|Az9)zqg=ux(+3zAOinWq$M#ao`U=#Jwe9}d4FkDZMmON*zZw2?TLOgpTq zY(_AS3={h6_8Q7;Yx3ECv_B;|?a|PB|6B$cR$>hbR)2KCPeEW6ayU_NVmx*#!eCU4 zQVbR#k(Wcl?~w-tc_R}133kFza*Go_8Vl5cy6!u(v~CWmRH#BP`tJgCHuT@W7+9y| zcpl|gMidVKTA4DYS*qDIH6U!P75L#y7#PNgV-`o@HWCIviuKq(V~?Pm$TG9)NHZIS z<&jbwB)&?gm+qhWIhXE)>ZHz@{nkn-7gFh%_+4A#cC{J!;MpHDJ#!(WX-l7yvg35s z)g9Q6kc~6=T5_iiThKY9ZR_b3AoYa-b%@+R|MmYhZd34-Qm8gPDU-B^`F~ zU+tN2zU5hLPxRmTGx_RRs7+JN;gZNAHLs${>Q?D~uOc)i8@fS%f0rSZia>3xkB?t8 znv%*@M2W4}9AHUVF0~k4gE#e!VU7psSRgX0EtadaW{gW_|E`J0z19{|TnB)leEBhN zL`xvSHZ$KA#=Gw;@|^TkW(ug4z*3t12%!KtSwAml1=mP2v`RRE2q6s2hj_i++kEUZlbeWMt< zd>1p1nyaVS%HXSK^(iD8XvBAF#Cdhs_DsP0h{#`Gb zWONB(A)GAYvVpC7AW)sR40}k&pQ35KwpMWcSz7tsJ)BOkRs76R%u!Hkz3_bFH@7WC z4j91QWOq)gIZrMm0YxN(#MI-Uy}&gy*{ns{o&9$x7K^nXnf&LKxFKKFX6RL06%5PD z!sxhBSDKwalo79K@ox?fY>?`A=^c~Qj>#TS{kVxK+anQG((9*wD6IH>-4c_L_F$(a zqWrUB%Bzg7DkpoWP;m}Lqf8$q6J6KI{#uOJ;nA@$bA`u|=-aLP#9?64mjfRAxblqb zrZN?+9G|NCS@gQfvACLf&;j?E2u8DN1z|Hkn70-8_J8JMP3KFSR(HPzZkYgcZ3S)3 zd}qez$~@@Ma@eOUcsZ3TTuPEzsw z$XY&fnQ6U~qZ;xZ@JX3=@`DotGV3a6o!^+Dp54;omJ#~q z&8|;!kRK1WnHf6sq#d(7i<;0ym9aFItpxExb=lIi;VwHv(JaQ_Ds5A3)w!y4a;rAq z`3LZPI)lwHtjsx{M|Yd!wi2E5B^&iBhym|GDs94qDe&ho=Xo>NREw3H=DAf2rhwq4 zZO}0;nL{?Slp=r(Imdtcr2%m;JiqxWOqdQO&W=TvHu>Fgn}B;m*yZf7Vem3_{w|R( zs{xdj9-!WFXY^?%Uq4)%izEbsy{zVu;{yIvA;qld+bPbVrN%~@qfuxt=bGP69G1=^ zr4n8m&+Njmuwb71a5p-ZQ|{cTq8L}XZ>J-?>9Vipw(_~gijL0O*`0|QMME=29Mxq7 zHyudWIkAzuG(FeyGJuw+UiuMMzjMAG&|JA+sA8^kKtk~WcmHAz&&{FPA_R6EjOL+tYn{-YCNrwU21 z4UZKYF~;$q!@wQdhUeFql5$9utn0_-p*&2%jF~f`MBY_LriJ4qP+5_!77^Kb+O!rw zETOEZ8QBVX!Cl$#_+Lsce&KIqS^}g=aPcV9k>uyE9aX#U#2?#k;z3L;ed zDO$m0z)E<;zF-AFZ!(WgzWB+*%i{$Ze4dS%=LTFO%$%!pvte`lb;FNHd{jj(RGD&# z<{e9<_tjtybNq5X+}h$!d)czTQevT5BlKolE#VK}%n!l##>2lpkNb*F=~Z~t{z;t^ zd*0)$nON5hkg+GofQk8FK{DM08q(G}F1@8@*3TK|Q$D(v{R zk%a5(B#p#FcsHY=2Jmcje3kg2^>TA{%pEqk!H8bL3W@ND6nh7RLW**kl9uGQ#4=Ew zeZA=@M2Ou!3x;JL3h!_1WLfOif77`YII_jc+uGZ9cKxn#hS9X%@5f3x;bF8w!CauFk7<0ONeBeSpd1B#*d_2 z=pUD}Wb>bLQ|4WEO}|(*8fsZo=<_`4^|N+bv*~xD<0`pn2PKl}FLna-bXqL%6X64Jx>~K< zgPg9>!a6kkp5U5lxRs4DGtO%;Tj>+<4&opVl(frF#Jf5$I)}Frva~ zQub%9Q^CepREaO}+5Al2IutDMU}i~8r$0e(BmV%m`2z`I1HRC^8#)dZiHVG$neCGqi)EbLtCMZ!!~#$sbq=W0-_m*ktUJ> zfdCW8x4n7$X1M(;>Yqpsj^fE;8hbw5xO#ULqUTOkn1Z^Qfza4lNPh1NJk@uj25J~> z#FE0|Id3p-1&MUTj>0!A2h&(B^$YRsV&o9nt0=4Du^p_M?h~l9w zm=UkZo~WSFadX-fUk|)-d(*_-^>cBGJxpD_V};?+oUHIHY6a@91Y6Ke-X$QzNcdKWZvCcVgoC>23HD zgR!?mfUmIK$d2dcs&y#-wyy_^!$&&M`|*A2OO(*&@G(3`aB~SzIk&RVI13n@+)k#9 z6?yi~A%P6c;tf*1Cgf|_Kg5d}a|>;A%u}L$fc$Bz_P8g|E3uSsO2s=4(9`?f11*|v zoi-!*kF-PczmYn!LI~&Py1w}p?3U60<-@t?;Vv}P3QuQzzo(|Q`iOUaoAJ@d+Sl8P z8gY{f0)+{OY*F<0b&~l$y>xI|ABg+|6m}oN(_z75)1QBOW2t`bLsBiw6Wj4 zea0TA!``-+d_(trNu+l<^20Vp70jQS0^f=nd^m0yj^* zZ=kGORbqS9d#qar6%-`-8OI&?-hHy?_r>5T%>VO_grF|bIMA$>h+3{byR@J{MF}$n z=DT17Ha=)#4Az=?g$YDy52pZBW6iQ=2EeSE zfXx{uxa6GxO7#T4#73mVbJ0J_AjvmB2u+>)bv{86Ixw)J{{8`z$a#nuSDW8`AEY97 z;Ge#Phk_LVG9wi5Q1!71EFXQDFTQjHc5#(R%K|hmr?el=pO{Ff~cQkg6NQsP67?Kd)ZVzx9j#Xjc0x1`~nM8nsT9e zsq!-u3P!hxMGl?YJi1Ht{4cJDLxz^yAOu%tKa!B-R-r4Fd2#si<^P_3G^l`Rerrns z)_lBh?fo%@;&5u#i#hHQ+YT^KMtpPFJ)1!%54Xa-{2t(Q6O$DUexHEPy=u0W=zAM4 zC!;^776$@6DK`Uscley^YW@KnoMEM4)I!PrY(*)ry?Ae;l|CBm-Y@z{9&fPy17LOK z>R<+c@QQtV4Ioy%DBK|G-oFn4i{H?tPKxzFr_ZILoKUWPz^~AIzxut3=qK-&0j~1r zq8tIu4rPg#TxElE zze~x1uyUkAlP(jZryT?$_QR>gH05N17Z5*^C`QXrKFU|Vz1?PXipOgpk)Jmv@@nhA97hm;{yIL2Yk!8x0Jqf8tDP=N<~OET=1BN6Mu1f6qcyx>Syyf?OK*<=-(GzSC$e`uyXA}am)hNKt z8c+VP!5n)LA$UrW(!SH$6{xeeV$~`hoQ2=dR*>A67;j199;+!+f^FAM0QV1IaxLdo z=~}#SapATr=Eme!fD&eSG?L27DE|+D7t5Rl${aG=Fw2YVt41%}uE?cmg`x-ijEr*UE*H_hbx7## zP#Am!O$)y$IO@;9CTXRG5SIT&MgyhHqyZnSbx8;A$!3TgxR`FPC&vy2k_+8SSqs_G znQgpsC6Lr;OJMR< zVj=sG<1;jfRynoPxlq|AN4u?xw5>dUIT7GEWN4do`I3fjIq3cww<^L$5!<((>2CpO zCR^Pkk}~gB;nDSs7ZljRb&83^K7_5On&;e>vO+Z)eGK1v1+yUB{8;`z&)$FhOR#SE znQgN1{m=Geh;!E3FSfz2dJJM0c=1!SnP09TL_7UvM0C^G;uVyaj{g8qC>=#)QrB`I z$y2i zZgVYaT^=Ge=tRK;v}2G%3&7u}(Iw~VC}&IAfhmfeR0sTS(~IGHr@vz>>&+^BbV7Zc zhM$$88;EY?pZ#gtZw*>GUTW?}`CRI&!N8D6+#xmjAL)-u&{`F7T=dCLE^mZp>J!Od zxq`prj}Mc`Rnp!^lXBg#HrdRIQRuaUS^YUhc{FtK_E>-x3IML}b)RmYWy*2Js z$nSXFvyCAbG;M*ryME0z&DI|It=URVXM!0ii+%>0$O}g^gcbPBjCIQSKm^@V@{wRx zk-|}>Y4`*FbUc@{lRKdZLLMHw?xA-SW>5oVijYMZkdj{Iv_(A(6n3RSe_OdmAMDV! ztzX#hA#FXIa-a2Cn;FioH0qI|`cs89wY)Rn$ArBIHyv1}3O5Y%k`?U`kmUX82K z)chY{?j|W=`zafW=6DXA(20+PdtB*ZbAdVjn1j2>u!C;Y%NJgQLatiy-N zLN*f_f33J6MM7nppT5wN6f{qus;S$baHJO$t$*@a`q7ST2A#zTljrF@o`6+NjP8t* z+>tqej6Avh!2(Zxv4a07TX$DRQvvsv=igzTB5S?zLZQ z^;M^c#H^`#Q@unFu5sN&k%ECoagxIF9y5XtNLTis9H}T9%7JAuvOmxrTySQPNK=>5xFbx!&dV;>jy6Me{AJ1PkEXOJ# zV@f~u6tQ0tS)f=<8KIwU6`q!T6jB0U((|$2FJqp!57?K(I9iaO6-O>>6ZL=VVCd6$ zo^s^K%c~98c|%S@*-Y=5O{`~NN~+7X4T3|S5eT%Byz9H-FP)?!*|xIk6Y!I34FNWO zMrQ|qJk(&O-;E}5U4p_gjt#8J6g;xpHZ+w_tRz9JcC5BG5LQ}AJ%{$LE7&}Sct!iL zimw*LSL-9iLR4rI~dKHnm(0Zd)lEDS3C6N?+dX;}Z&=|3}qZMzz(oQMY#frNZic|bKdB5+BamLBtUq-?L9efBy>k)et`TAh1m-MKqh4baxS=XF0rVclHJ zu0nS>!)$`q6_y!=`h*hL!8l9p@FA>Nq94$7_j=N5jfbVaBfpCZ^1EOx+43)SyNrtD z83$do+)vYUmH#C;|8Pk9Rd;D&K`@U~_g~rvpO{k<0x9Wm(3&VDH|3kLRE})p+Wpge zqvvky?%MD5Jzhd%RVTk(Fi9)rWuPz1%$%V^u_ zWLIA6poxXBOT^7{*iZiGM)^0L5yKExHvCL|;1oMuC70+znoHMBr6$QouyMiP59&*c zTpb?Va(}TE`$%Ll;HpS>hFYlXCK5&-NLepsiv)U|4mBFe z*T}j*btc$flbD~IyR$Q#n^cC!;%BBorqJq|*Zp+T#q7@Q9>OPA_A|sp$3YnAaw@Zr z?Oap0WHg1qQo0xxXq}pNN?BPgC(Mi(l|Qnib1`wu`7P?Va&hn4MV@ulf~OKbIg!N%JRPV&K- zIKvEggcTt&oTl+B;ENP)Ab}L}_YXKa*VGg?&Wp)a5kD^k^@if$RqUYl4Qrr8YZPOa z<>E3%h?ZM_%{%!o8*N=hE-v_oQNjvb!vuJIEp3I7X?NA%*N3N^7Vu5}`|Jk080Zpt z8Q*0SwNbV8<45;&1}+Mbbb-Z*2mB6y8D9C`2|PABeFXF@$5e{{=GucRX4bVpj7zm1 z+S$Q+Ht6H5M)1bTNhPFG{{hfmxP#B8m{=Fp`GD1lC|!R4h+7eiI26%8X|f1bhuV7ggueg884e+644?56 z$Z0~8@07?IGG7|$*M<2BsAQwFWCv(UuW^$x?cQl&fx^pyG*`nZ;q{i>T4yAVc8WcU zK3*q=KZcs0tx}`rl>yjyiMVFH= zHH_w9-F~31vpCLsrbLyeb!AONn**(!2Cjz#&_P1sq~OK8>F5x)Xg#y|LWGwk_#zI9*D9&wY?ulE@taD2$-}N z0WEG~xsI_xwR^xSpQgY~+N;m@)x}Uf+S5bSVNxhDRT8#0)XY6@3bET_JVAALwc0x_nE;ic~}RAnH4o!bTYq<=Q5Db*Qr2 zb>=-ddbXtaEKY=r7j5P)T~HEiULbm2ZE2}qhxXSm@zX~u)1JoPhxe@2(z2{dDP(PE z!JoKL;YeCiYv8@V(PnqVit4h=P4T@Zuur_m#7ta0b~CXx_f#VR8$0~vD_2yBK0|_7 z8qd-T0iQs;RNE;9u}F`xmIYBY$pr#a-o!1rJ;Pli)=2v3M*?bu>>THfkeW%;ubZ@- zBKj%Il4N=n#)ghY=t(Hl;2`XPlC;bN;nyraNLX?15F*-tfGLZBWTfg*#vi(vkP98^ z+=L6BJ9EBf$;76>wPNLgTy=CCwJcRM`d2;TXbVUr9BmUU-JWg;r!oKmTg;6^J}0W; z!WFQse8SnzYdjf$${opzabCP`K~r8E%EhI7!_4;#m0Y0(1+=b#n68}0{XO$~^&=0E zNm_ao?vIHWrr(jMpG*&6hGdk6@B)HFdVBIqp=V=E7!kiUig9}1mrt-h2rHI5*mmyn zff|#as3Ho6VmIA)^z$)wmd9fTmdOsE?v_X{+ivD&(m1GYiA0f8OZ94s?*4ta9ls4h zER6|)L-{#=2-A;UYUI8mu`;_BYjk#!&@m!QnDqBE?RdmJtvh=axJX>1hf+tzrlw_8 zBK4H-(tp+7h>1_Hoq|}A{JG@~aJf7UA5KL0PAPpxePWIy>r3<;iJn3g7=+>b{_8k6 z{PwtlQ-)|6U^q9% z8UlF;$Fl7)jlXL%|K+nnEc448Z1jH{o)aPpdq`p`VlpDn)b_w9N*aCnl!$x zujaisocdMNbFsj9d_?Q9zPFXquIG(F?>k?lM`mwz!ttI7S0 zXS?XH$C0?R!uL@|Obh}8q7J8lX4Cn>=1Iwse^fQ1UFdes7$9+Rb%0Ply8%Qc1D+GB zoe-oDjDB5Im>UVD4)vRK=*^oM2D0*@q{-9Q=QmebIhbdvtH1S7ZW*hW>DTz|5SDAQ z$!2z*X+59cCti0wn?j=Mt3(0*cKx<1UM3C(s>WODzz(f+jVRs-Ui3G%zt?LppT8%wiSzK zifzQ}AJs(qiT^*@n6_b{A-~jpyR5s@eRy!7wv@8~|N7VSsoKgQ*Ij{2#Q+-I03v4G z^V*J!5D%p%;-#A~tST>f5w9CpnF$qFmGbCij1QN&D3s3vl?E4ZLXU`RRjyGvfGB)( z2fK0fju*UG@ncZ3G61UkYnRIwhksw8{LXe@h-6>@Qg4_D?VK#^Th3#&36yt%$=3cl5x@x?5+43JKzbVV8-Ya~Ll@HuoAwn{-3U*O2IfHn-u z2|V_9M4`b%Or=I!4w7`xX}Wx_x9z9C$D#gYz+F3S`w;%G|2r>IFcu>w@UZbU*oqh1 z+4({Z7w$W2N&%?qA zn_RDbXR53^_(2gF+uIi9&>T)hslpRT1-*hQEl;PL#MQq6{Gmddwy7kc3_e=bf#70^wAWews^f_WM*Ky~~ z?EpFko|ci`mh%O)JzTL&R`CVxxuA|2Y_`P{lig?3BwgYf?BMMfzV+JX@`^_fdF>>=Iu~E0@bFYM@2wK{04T^sa!xG&-e_s%gUzry zqgGI}a!`{FKZ5~8(Njk0G91!!DBD>QFAbsJBshG;duMvC8dm7E7+l=>FS=K#GsX~! zL&}ikzjYuc+Bi+p2WHYrGpEsutb!Zvcs#sl#5B3MSHUmg*UYCX5@I{2 z$}MpC^XqpM^+7@6xXdowt8yG!_HSypHHqPxz% zsVHs|>8^(~mE3Z1w5@RP*WG1F^YVNju{B^PTTJ6{*g;GKX!rBI7{r5g86?Px`dgex zH7q?~q?M&CyA&1asVE2WS)O_HKB>}k3E6aGVdLf9pYx2B9j>*JuI zf}XLqv5zG?;os#CA8eLdS~$}4rjb5bW_D#mB8ZGUbGidYRWL3IC(7R`U&Pj_7We%; z*6lNMKTrQs7N)4j?b8Yw*ll=}3m=#of#E&!Vv@cotMfvpqkwwzWzYB3VP9jnUzB?2 ze3|b8lAe+7FoLIUP6G;3vQ#dZ9h+WzL<;O0A1780Ech@L)iNxWIDl6MuM4YU=euM70gj!@|A;#^8XU@<^QNB308cAQmQXZ*rUEVK zAQ5H*o+wQQaLhNIdO+1mg$!PG9CY1c#T0b>7$-Cu=ViQoisOZPDCaK^uk4SsjocZ*V z5f1)|P8sVo5F7vwD(l&T5SOiInlUBKdovB}=hQ9XzdA?Vhl^rrkt8@uxn=+B0K=UX zUwk5{Gi~IQ<+j(I-vL>^iXWEJ23vR={QQ}@xS_N*!1$sIH#o>)xGo zU^Op-uo>5Plglc3V!~QfRMTYexJ)*ffJV0PxssGBXcV;AcEY(8!KRU$(j7#<++*v7 zXB%&urYUk2gYtK|&xX}=By{An9Q26+yXgd4hITqkhHue}#l^b9-Fp`~Z$^FSn@NjY zoTm22G0o@QT?>oi%n%rkYY{#Dkk$HmO+kUL%$^qF-U9-Y7o5)XX+U~1CJ6;~Cnl$v zruh{=)rpt^$)X`;ECABjatoEqvrl91C=vpm3pi;OI$>$6Q6%;b`_p2Ocy^N!T^5jX z7MEapD~o8f+c09fwN7a8W8a(aO>13oPIKUdaC~EbQ?$`v=Gn@MECXb6H}|W|8Zi>5R?8e`%m1 z1c0b{20{V>0JgeDZV~!W>;(?9iJ?RAE&x6Y|H}r8vMr7Nm<**c+FL={-MEhhf~IO_~sX3)L^bE&F1EMok@-?pmMv@LXN6uO=#Fs~wau^*Z({ zUlxgtN+HcHps^`dMqp|`clEb0(=y55xJI5IhuIa;?Hmf;pKz9DI7oFKDeur_w5B{P z(2!&dQ`5Lt1PKyRy&O|~k)O48xCRA+h>#cb*NNPewklbX{6#YK`5T(jXlGA-(u6^R z+A?1<>70ZkpLH*+QihqqknB@s6wQHh?HqqgB(|%MCsIVA;UiX#c^r^E-lM$(H6#-g z6A1bcE>u8ezMuaqGgc6k$}6+{_RDru*Jut6sEW~3Ej^6GieV@%&TZe3ELz&QAYCii zX`KJr<-&pJ*2)a3^)2%|XJ`gK@*XrS!`r>D`SNG#T~esJS6XSZF-Fwj#8+Slq=Q=q z!iK7W$B+yMXfeKp88T*HRW)n(hbYS)^oL13uqsunMXUGDL&d5Mx^SaSelShDx2a@* zDes+`4+H6J0XQ-Td;_vL6bxP?)epmW!Oc3$g#wO6g8a5W{gmFa^D?GgE7MXu7X$|@ z`R_YUZ5+}r5H2wLNJCuZzUd4SkQIx-+Ee^+UL8Hblhw`flO50!F8lXGfz7;YDy^~3 zR)a;2qqWHKYQhN+kG&J%+lm%+b+YeWk4b?-$NRoY+1=@*n#2M^B z2dW5dgCAp<7t9a-zFOyJ@IQFgOav|2zA0yNKdz3?9uvEwxA;j^*InfXz zKBmRh!zM#v%@Yy$=^B%74!K1n0@bLRc&{G#jH2IOot60+2Q6pb>dqseeTv?e8J>L@ z>mYA5dzn!Sk7;D`mWh$++yP$Rkv$vuT?mCGBIA^-ws7#nGG-m?jyn6-=QWeAk}wai zY!=V*2=tsHR##$}QAohK2u*H8MJ~P-Mo~=sQvnGt%M}tjSJ%j*1x>s1@;ETGG1;Sz za>TgCeom}oME7GQC~_G2H0b;FE@{QE`NPWjKaojb7<~fAV8X0*(_`85;+zPlYo|&E zhcu6F84@Avyb#3@W68zLvgaubZD%fn%_I}1bdE*Z{UMmcWSX4u)#{)pO-(#%o^QTxo3&DneokR`{Wz@K~s~7Zl^}%#i%l`$<}yQ(|~G!3#yR*c-5s$w*$E_EYlM&Jwnt}15?3LD+cvwp|aUv{V z0oUTNLnoO-*=%~JqZ(#saB0Zmo-&{}H~4`5lt^U_v$@4Ekeqj-@hQ+UWQ2$09RBAA z*}+N%;Iy&v+4b0KP}-vH7m?pYu3bs_2dmE=W&5&WVs$Ceq;vQ=KN*8VYhyP_57pgN zw8$o_DsukPrvHp!y>dk%bi&|a${=(->NjeO?P5-pvGGuR_*UWUhC6uZn0rfO0hE>`2E9p0w}b)sIuPf%xK#^h1=@#N1^Bt{ltg#zoX9c&s*K~!A<*Ke zw$wD1e_?_I$p;q5V$=vgo|Q+%jA;+DI&w?;9g5z>ppIjT&xZ;D7Wwu!c2sWS3L_#R zYu=0*BvsI=$s23IRHVi%T|L_H#V|xpU7F>D_P9TDpXAp_2=Z|Z6fnLzbU3`7?2QUS zWy6+5Qa&^tvXaoAU$B*X-Ljb-Wqr416kD7qSMuDCo=D62AhJcc&~yi4LH+#C96cQj zSCHjuqS;u^ChU+*Mu*>#Y!DCJ9icFJb8J2(Ng)4qyrK6lGt>>rAQZ|f@AGLpUGs$& zaLrUK84Z12;Dynra-0jeCQmA1qBmV}8NG}XE!JlL4o_$7_J`24Kbjb|4sg6>RlUnx zKro39!V{Al`5!XaKErEuYjj6SB;R5&vB5RV6R3Z|C+^xYuye)YuTJN*WmOo)muzWu zutm$E)-7u3euPzD z&Ymit=H=W4h*3212!F5v{8S%i^j6KH9lBCAkC_f8w6~B|c=JghdNz19Hg7USPvR2D zSanbB=(k>I@R@UVJAl{DRV0-@*v!@P7<$@JXCqLiF@v)^cIJC^adkmv{IZ$v`bk6P zgKiq}Z5?l=Z6x3CIANppz{ayQ3E9E5G#C>nZGH_qEOE`2G6il0lcCbW*)fy!(Gcr} zq7{X0>a6Ic(;cxBhINt39#*j-^q?Yw3C_5iO&0{OMAS`B1U1->c}T%x8kyJ&o>|C7 z1Z@S81$=}VW|?KG?yt=xBnf6^iy}SS!v%Z#?Y{p`go#pG@_*$|sZd{%YQR~Mwgzy} z3k=hx*A$1Ob@`e!fZU!IMYF96K`I^)uu3E8Wl7*9)GD#2#e!f|xxf6@yV88Zz8l?S zi(XoG!l~luLFYv2TIAFr&{&9j*j8og5I%NU&-OpSysPj;8SwE#T97wna7wk5Ef(sG zuyiegWUZ?{J1&I~6fY@NVdU|bY!`d(^o&GJ@neBo0h!fPgBxdI{#jc}IQ3pU!_*mI zK0E!aV*Lx7HpEI%z{Cssnl3x`&si%rpTv#n%YJz39e)M`lOfuk7N9sD1bN_u-5gtL zd$;5?zSfUbOCF`SW^&vYUSU6lMGnbSQwxrxpqeuN5XHqal{!!NbFsz@)jo1PA8ogaq{O62C%a zKdR~5ul=!IzMoZj#rFkHmFAK)h|EeVYoMyL3CM4eS`^XLV}f^SSrs11S9amW&0eJ+ z#?~)w{N=kd==Q~9gsl0aDgvIjh_#6}Rq7Uj5u}R#H!VzA9J&x)h+JW+?7S-5UD8@5 z$JsRyUy(xC(9`Wdz$PZg^}OKck}K;ILH|%*FB=cEO6}-1XG>9Y?e(pPvdRsS-S+N& zLM5bY;^(k;CU3)kr+<-4T0(AXY{~;sBoOzlNR?QoU8NIC&iF{R0}8e8)54Y2Lh;;Gv^kz! zo?AWpr2V?*z6m?DRxaPHPP55It428y68vpThNKo^1S6T`O*GOksQU~fb8+-f|4I^{ zG{e3ba66ACYo(eAAAX0IASI=$2F>!B)Bwxy;|DR%s+1A0_NK!^5}P*IGg!3{Q^f^^_qFY!!c!GMhFqD%pkfRi7C0eC4cV(ZC%DJYjbWAtpm_yJ>34 zSSgIW6F>-;f`1*2wnyu9=?w+HWN;(2vbHn3d*0%j8AT zK2L3Zt3aSK%A`m;*<&(7XA-hkMQdJzQOaC4?5h4d5HEn;Q(gKKJ=@f6H*vLUXpVKq zXFc=mNhG-t0}@6ii(pL|d_tbTWjrV`4L7q=fj#MGyxmi8LmZWI&OBObCJy$wG9Y6| zm8h@EM*M{=VFd&jRpw%<-P$(v%f;@(-(0MiisIk^341hD&az*3R^G?_9JD}+re>u|#F=5&IU#Rv+Wnm6)!rt6zJ!faPO%TZy| z);x22E4cGp#h$ha+WMFe3;uf6--M!P8K}xncoU5Cv*twPcgs%DtZ5vcZJicw|L}QC zygNQsY7TnTpLelKTI*z{u$>V;N4I&1w)qNzs$^vByLwYl0YA6YB4%}j9rZJEy!cM* zZSBO~vg`-Wu)OHF2u^qhMSR-BoC`f$+oSl$G38+AUC*q>mz54aa-KPkV1*qvWY>ubAIT~W$9J+ z8&1l`E;>I%DV>IsBghR9%fPINpOuqhTRO+{<3ZB!A`6xLznv<{QeQDQ5HF|pl(il{ zrR^|A9!sCgh3x&M+b9j?6YobH_7}MyD#jR6Hb^=}M!60cE4m-#UhzgvS)P7(2?_kU zP@1wA?}1WT0B+ydll(u4)Umq%Lkue_{Uj1!fYUid2PPXG{)`wzl7w8_6I6psmna_~ z-pGJQIE)033RAKJ`-Ee0mry81x^E+85y!J`6Z*o*O0R`jlyy{#tK5*_lpG=~Ej!Cc z2|_Rp3fSdHg~^gRU3ezxT_jD#ait+&1MUhE2U#oW%5DD4_fR-?(_-(<1(pBl2u@Up zC(I_?LQW0dfz8P_`DkMBeI7#RfS@$?1(}IBadMZUD)8G)(2UTNrF8N@fW9wGhbLVtm!%rHg+ zSHO$H75kzj&K>UTvN2Npnzhw{D+l6m#Oq@{ot8t#B!vLYHMNDV5@i@rH}L!h2Ia2_ zN@MUk3N(E}xMlFhL#Z|TYOiE4H4%gCVR%RZGXZ-SDSW)eEGqxrcUW)%4-++n-Bkvi z5bCtcq})NXwG(fpI?*wR`yTk0&Yn-TNzM(4FV4Q=Xp6s=7Bw=ImwCHfocz77d}x$c zlQ5FUo{0WD#|Jfve21d7Hm7&+?s2sU2}8KGTK$;+WX7R?{P2i*KPa6DL2Gws_nl1M zU36>AIU*eu*31dlfrtl)8yj7}fMG59fR^^{Q1Vym!)F<+D4NKac#p;&=pfg^;$G*! zly{d@qrpw3@(EK8-$)ZtF6sZDr`!+bET@ea#OhI6e7?WyPZNw6|2;_d?*A|T79*-K zvWQu;_x5EYKO`W`cQ;CB!5BwW^GmQ?!y}y}Ds3s1tS`X#ulT{1&;flmbYDZ#R|Y-? zUred{?QT#85&Wj@JEt}Cbtb^Ckmp{R1)w~nmHg=HbHk{V z2vm9;@?Plce=EeiZxMzs#*tz}Jv;(XbZt!Z(pM!&CBFD{0|==ljZnjGCAa6U-?Irs30iN`93zHM3_9D~x$A!9dUd4{)Y2OHi*# z5x)PjtHgU9mm@qw7n^u4bmUjNHQG?AUT;@$5keOVHy#uvH8FrlVSpnO!UU$R;P}WY zv@|ZT5)X@)Yu%90Dz6`CA`1E5ntq`TA%F2m=B$Q-^~>@BZC0e3SMtE}`n+!;YLVjQ+vRN3 zN3OC1K&%Mi|B(+sWV(v~Har^sFA-u?2=4{vc+PmEV`y86+DJnDA}A-*lJ{yg$aF?@Z$x=!cAQJLsE%Uwe# zMHPY>rtv2QB`gB486YmM0z`p5Gmx$DYl@Ln94pb7wjm3n_64J5YHK(%)RgH=o}Z!P z?nm@E01SWsCy)P88XW+0(@Xg^AQ|vS@#7~ZnH1!Az4kUD`p`lWN>2YLv`39GL_At} zE>P$!8b}Bb1y8dzmQFcX+e?3mO)Y8%W?av#2PdZ@o+JO_2@(-MT3%7;&H)A^wOw8x z--RH~%tV;(BeF}N%SKYsu*1beaa8iVd#++w!3#!`rP{Nb&*J*RS#)Ot9Dj2jm6En& zdzouW{{v{&4}Qc*(-2Zt9`1=2CgEh~PsIjJO-;g&Q|{_zh~OfIB2Y}VM8g)VJNE&6 z)L?nOgcYu-G?2s&%nwsuA{&w$sS^C;G9lV2@qBKq#vNdFjdX@|c5V2{N!@>2NvA)) ze#M!$zUQokXy2UK5}@e^rU0nq^8@eyaMtJKB(?-@Yu^MJRiWQ=eSB+5)^Pe8on z&=nsVs6)wX&S*f&mdQ$v+sT5iowtfEQ@9t%eP1I-h?B-%k(^wH2B`xQ13(P@S7(AP z{S3P`sdVvoV$hQGROfngD|}o1$i9}U5E+C>E$Kgl=Zj?}`kt+8V>q6CtJ@?GStDuF z+ScM?8Aq+*o%e!zAiK(qz`S&z%<`e-Umo@T&}}j8pQ7Ku{LQMK#2f=Nqn1!MoRA@K ziWnXUE^K0fAeA6_cac;f*lkc6@Wbg-{0n`7uk3{$ese&ce2sI^TI>sTvX$`ZA-Wzu z+mLcrR9YmWil#cHN*?3UG*xX-R$^tmZU46He6lsUrDY8$tNGPbM!-9P3TZr`TK4=Wcp=GS!uAT)OW3rY z4_56A&amI5af>eSnZh5|b)+tIk(CO6A9yAlZ|L~kj_`K5Mi=ZMxnNUfkUK=m9ZblB zky{yo3DHhXV*8Gr)SqQ%g-vOe#=($WL^G}GtQ_|=KAN7vzB_{#tyx5#lFKa@Om9)6@*)i$R8Ox0Xi;6rmrv$v+9MpaQOB#W?=Hd0?=TTgqfJ|6 z3Po36KUVH7#)Tqf(uzh@B2@oOe919Ds3tjmR$sKEgsOxzA`+m9_P1S`-NDoEMl++= z(Ww@-h5`VOT2ptiXM6w^0jf3uaz3ojjwC3=o(1V%*p5|pgCBTXs$DI0Q^udRoi4>I z(Pb-+VL8zN4s{!ry~apHy|JEtOC_?3zh6X^CSjI1+_}9uhF#7fZ|0_WvT4xi56Fd3 ztU>kEZ8X&>h0LG{>A*cf=aygALDml%8z21DI{$6#LyU^Tin34U8C>F}+8a(?;j_{j z!{$m9QGW*aq`rg2+0J(2ZJ$vG!IRt~8&4FKZU7}OlX=e-tRHL5d7< zQR%@~Vpmh|ABnrj2=!B z8_^NEhkl2cT;-mY^KEWvStKT#w%k)g1%^DoW7Gd?z)w{&$}fSnnWTR3Ff|@*tMz@` zR_f^<+UE-#GXJ&mzFZ4}w?XxdP2n^9#$j+S-jrpp+QOI^)pWE&QDB&%ro5X_QaAp? zYWMw@w<&197ch6ir%t(t*>R)-vlB=h5~?_HsOf3yFNmhr#rH<->=QouovO4fp}Cdk zXekTd#p~pldE{b~b59-5jG9B$d97wMr3)f5Wp#x+RDzXo&?6DcB6nSJMlQjh5`HM3 z+_rS~+4yN>nB8D*sRL{b-_^oW%MMBFfJRW|VMfVLS_6m9=^wPzlS}ti zF3N3PpQaN&MokBG_L=7_R@MUl)3x{B4P5vwaMin6!F%ZYkoZu`7`kegcRsR@v zQODfak-LDy>%e<{<0~e&YDmkP4Xjy53(aE23r{Y??YqRpJSS^Jqs6yT0jahesRvU1 z&^sRuDM}Z1^0yy6^^5e*;n?d%{2EU`k&&f;Ja@dRaRlPM=^o5e>oLwoAEPpF0pSQj zqDu?J_r>TEjOr(BfR6(%<2K1lY5O8ctgiQLKLUsFx!Yo|?=PFSBUL$tP{K{^+$-~I zm0?Md;ee!Jl?y4G*^(H%j4B&3)Ki8u3s)dspTT;DEKuu%D7R>L?LuX~ILf@a*e$r!Bc+UroZntW4wU?&wnx4i zcfM9x1&<3fj{gCkw$Z0yJxU=+?F2s*!}lh;dpn<_ujV)Fzt$AT*G_6_4%&j%&Z2SF!O|7y*!S30TOtJx zefV3Uehlb*E?`53ryWAfYS!W0FGO` zIs)m{Z`CYLAAFYNI-S_8b+IxiELlrRxIoYx``@SNADzZ>p+4<;TSDw8MbqaAh6#BY zg~;gCS$rfSRk8I3Welzbn^5a28GhJS(xI;fp)oq!wI_c3MIjqmi{vR2$vPQYNoPDY zXx=O}J~24WWTB00&j(1>J!PaxsuPobwGtlvbcy3Bky;&M&&smu4qeKNKwq1a?6)gM|^91l2r?))G~NKA+n<*?*0d` z$!Z=MelmKz)y>U;k41$HUgEmi{4?gpABaaD2|BmY^C`V9Bg|y=1 z2X^55Yd8hU`Pbp9qE7k(T?+eTQk?T z$rW!KlUT+Ue=xr!5Y)qdR6HsnmSIs^1PGQ$lKuz%c2ASIExaKRP-xA+cu-eKlL`jA zBD5ZPMRp*VyY+&gvWse{e=XRiO-}j^BD!se;~{v^5~CianBQGBmN_qK7k=ytjB<8W z5XA@@$411|zns#D8ZZP@wP{%~Ujpv{*7OH&yb2))9BF%f#&D8%daav-at#p;)!R$$+EEf|hrDuNM9dtp zg|Yvan<_MQwOe8K!K;!Q-N$>={hWKqykY0#qa(~c*mWz0t$#rr`sf$A_N8l+rCLj3 zG7^nhlgOLqkMeRMda`aB3h+ajBEHtAh2C0ct@PB&R8JVQB!2D(E?NA@YHqhHGlgr= z3#HC3x(Z`Ma~bOEkaL-6l!k8wt=wQH0~YRQ3p+JH;)9t>qp0AV!>BrB22J#;Do>*B zSVOGE!^W>da+g(k`>q_K&47neh>R2rpkDCT((YJ9!#pQf^epG99*N5KdZWkra@xT_ zr9~^$?cNj7FbMkQ*o^EnBbw~+(x3Vr%cVS}HwX2se(3i;acgh?ZEUyZ@oECDtGfd% z?&r2kvD{P)ja@)uU_W&986hJ}{~4AJj|OFxmR*H&Fj#}NFvo5;d#9$(3cmt_Ri_Dd zhtn9cpTFaNU~hD>iXfu99t!|HW1cIOv3FU;&LO*>?;^x-&GPQK-ZtFkvC3Wp4=Z`E zD`g-MEpc|?AUjcwg$yf7=+)fko`S1OI|>fX?^Y%nX>^rnc5%J2Wtv13-QWnnQjzVT7|*FwO&5m}-v0ZXf@w_x9UR zC5YMQJ_xl4`xyEv0!_}D=+V=d46?YVQVaB;@GdnVLpeqj-r=&~;mFl`@tyBX@+sUU z0kRCAg0zqkpD<0ph}p*K?=B091l8RZ&}kzB=&ZxPJw&a(J(}Cdf#VDpV0B8tB$ z6GZYp_29qf%KKJh1G3R=+sznR-H0R5COCjc#rzn7Rt1iKcoke;)PJXXp{{{7xNUod z*#ZC5*L2(@fA5$V;BYWXt6KWlq;4WsXUtu@ls3C1^!EVj@8f`?pKi%+n6f}c}n>w$?&B~fuX9x45(ARBPU=>GS66+!I3>Qr9J{JyReG)4TrSPuMDX-9FW&6^$i9$zY>iQJTxg>-ZFr z*`s9C`K-{%*W(y6AdMjB`W2ymT92fV-bIQ_2a!7LwSjoy{;#W2jQrSh28~za<5Fv3 z@XhVv0tUvyT`*|}AK!bBiwPc_Z^+;O0KZR{k>$s;_~>g$8yeN|b5xJ<>>YO~m#E*NwYl;AFOShb<&)v?TM_>X@irlJo=}>}7#@ zAr@nB_tXLaSpK-F`IsWK41c-{at0N*K z9gwoP_8yF5{!$>W@PR6b7Lg*Z`yM;!*@fHkcRD4XIxId7!W4NU!MhD5hAok`;Uzsuu^8^-&iZwgc7z#$At?P}qhD0Po1-^Bbi zwrE38dBmcUj>TSkxKC(;l@aPtH7UMzQvlex<@@1e5T(~dp6|B{G~^l&IfR&YOuK5` z7A2xqehJ)aoNeOJd*y_PtVvh}Q6FR9`0D?;k8DJFVB21Gc}{*0ME==9Pl>EP`0KLw zcY;&~$CV7SG;vZ;s?&WARmv2sgBv*Z|!#;m&&wBu-Lk5W) zZ?%z09t&qQApi)WgwtZOj-;)N-R*Ei%|MgP+5xqMyj&y~$o$OP?*3m24memcu){qj zKNU}nHo_>A4ycS$^q6Ceso$lsa>l=&3#t{RhJO?9reL}@r-rbdmj&=YztrB)ys;!d4gIRM6zVM%y)t zd4Uu$xS&8&gS+9z1h9uOj+A#Maz6+(^s0KR-M9e`Z=CBB0u^c&vqwS~!k?2kO? zG0Ds2eva2yA*etb->*rhQ~PrZ2w#O7f^c_2#6=@%wRK_ZhwI`qWXODnuAnySbsO=m zXa~c`>8s*)@A8wmg|NGPW_$NG485b~)T6Clx}s2QYsu^?io>=OQO5aq-!2*O4IteS z4>na}$Qn%C)3JF_#jm^5rqRK%y7V96tKyOK23qCA|9Tqv}0Gj9;LB`ZVC)e2Y z|6s>?uIDgPRC&cDybk6p^aFXy;5*?cs}yZh_jj=*X=i6Z^T3%^!H)7J^Pj0*DxY^g zDICLXD$iyZ&7nDr_k3MorbDh}cIxI6Tk#ZVvxXp-Rk)hvFW(5C`g%f9mFVpgWGq^D z-88?9Rl`$HN%CSk%UeX4{m?5<%(r79tM%NB@?GGpPeyS9n z8Mbg+{X|1hW1Y#bvA3Td#3#YoEgF(?t}fp#KA&1(M{QsnZWL>RPRVN2FRR_8o~WrT zYMC3!oXF=*MM2d_L8p^Jw)9=Ou$9AhK9##5c?4MITF9ud!fC)1OVxG5_p13-l>GSLO_(BiEag?hhI7y?B`)dzfL`+r@NNF-vNK5@+Vg?fSSc&Wis*r5o)Nv zZ^LHB(0B?DtGabI#KucGMT}55SWSD+q#YYeK?%R0P-^wpYU)v7I$!QKWr%L9hQ$kh z2!zRvIo!87wz=6yR0hPL${X~9xhW)CyHqoc!M86p*s64U5Ha5>$29*qqwVo#f?Xlt ziYm?BfjlMA=icr`&jGddu0E;p1-0uJp;shv6yiuMyu`$s$X68qkI4>e`dtZk3ppdo z1IY?qcjCQTy^q+fn4*)IwIj7n$UUq}_6+be7$!-`UNIXfL(>T{5_10%!Ys-==+?G2 zjH~6QisD|)f_uB$|&VVBawL^5Ne032nn&ms5XP6&l*znk&sXME`520HMB_>Vc|MnF}$yG>*uj| z+*@bq29r~+xdYidv^X&58a~Z6LoCFdN@6`GKcptSDbe|tMrNUDB8FJv1^KF?Qy3`Q z4CLyUVQg$5OVADau`wBnlf1f`{cAinGTM!!uv-oAf*IHoK5wWcix?|C`tKZcsEQ}N z2IV60c_tSZ{RasDwwT;nu2#R9Xf9*W_|#vDqtD_FA3q##yZhU_7WRHBlb`oQDZERB z91463Y!-(91DU|K;#%ig*y78i=p+p!**(Pi zo{7vkYhd&`&`PbTd5l?GDwtTy`W5LTJeggIt~<82Bl-!gP6a@hLoKKtlCK7wOMf3f zDDB-H2yE?XZSn1S&!mn*|2rqVP%g9eMRMOG+2S>T8~vFL-Zye=;);&uI?YngCI-4s9GXg_XallNw}|niCaY zfZS07fm@ibUeDx%5C0DUGC|G0*U@axNEu!kANY^a@2<=-!EeEw_DWtIfB1_BH7QEH zV8dWS^BEHOSmGp3CH-N4r^`QYhTWB>GjH7wa_HF{$W$N zFYym@I($KyjKIxE1#3Y-O1QmD&twC{LV*Bq(ZEwNt2iL$IR|MRZNd+JV}6hQ^02g_|Zqhy|Ia&mOQl zyM-EoT7V?W=jQo5<>RvK>pjV)o&vtoW%_iOGN0H}gQ zDJ3>01Wk!OoXN0-6Oo0U0ihEyxUvX$s7%?0%oqaV&!v1Iegw z!eQqSuWo8m5*aE;YBP%hF782g;u?s~c8e~|6aR5s#sCaAJ}!mVzG*vqKhdFK|QLv686rxsKcsQpcmV{x&NF*YV)l9_WR z2jeyX09{`5vj7r7;%aI)=J|3HjhTtUU}lv1nQBPlO4d?QD}*Q2xhh)cAxH^ImBw5k zl0*E9{xE+MIk#>dv~gDzyv)}BR8BTW}T-f_abAPICP54>h zP9W^ZwIRec*#!=NQ^Y(KS&~y4=V+6gJf}14iEL|0Vv$v?r#7H)YDeaC48<<38rTcK zVDl+*5@b$dKZS#e6Mj$c3R1m#ZhhHl*!lQ;UJQidQ|;rA^IDYgrh}1>PI#OD0Est& zJ2uE)Uj}wT$9zwot(Ak}%JrK(sk2c@@q!!RA3%uugXyjB@OiBh;|9-tl(tp%O7taE;E{{Y1K_@T_uD(OED zU7zI?&v=?@r;ana=X5BLl2!izhN)l?u`PgNb|VyqV>r}EPQULp(?Rw4G-nmrq=M5k zDa4mxFsX3dsySPBc+xyo{{X~)_?*sA<27B3a2sJkV^e%Vn>d0F`{hVcfyk&6;+^pE zoZ=;WxK!>B5pH^Neo-3xAthjunNBVk{{V(XxdH*9d%dHZc%%Jc`&#Xls@#V+;eHq3 z)n?4UMElhB8HQrrZ8cj(nu=!X++72ieN%~9OD`=iq(+n$-dYYNPIWeL^-7j7@FF8u`@AvlTuQpu_3gp^k!!OM>J&JtTRQ#RP|KuJxaYvMO6!~gHELQ zQlzw`1U3{PX{o10I@VB>DW?!hml9`=G70KQr$eATw0)y5F9@)ZUbO&ovD6wDb4K0c zadD3xWiMs)nS~RfL2R&!ie;<^*ic}}QArw-y(Fzo>02vO)U+rS6iB(;EOaNWIq25f z_lYDRsi@pWyMQ%3+vp>hdqm>;I<^~SOCUVTq>uR$rv{h~j!Du$O4>?LqLNe-2N?OY zipb!m#L~WR)vFmPQdtajCbX&Di`sw(+Ii+lX9f$ZUOMIBg05)91d3p z$*0}m0%p@Vf*&67{{W-E99HIeCu`lDR=nf+e+*P)ymyt;DO!U{&HMbKy$k|DZM0NT z(9}4TLRPwmR8pl9ap@oN!*2DVa$hCXb5J@R}$ zU5)g0+ccmFQdOKu%}@YJ=g+{+$)x0J;&iENl|MF`NOm$Muh~2yy^yFOwK{@>qc5;a6D2uM*{K!pMU5+}na%;Bp+Bnpz33JwVrvWAAH?am=yE>dKs zXd{7M?8LE2JV^tEVbW?3x=C=+vhv=+J3;(R{sZ{$fO|crL-kbOjNHpT%`2+E^0{_b z!+9TMy`EHO8KqW)Eq%=zRJO%CNpaVzG_uetQfHRu3Y9lLUJ+|Vvyc3q2L=@|D2b;Q zL-7)Ls>G|kiH~ghIF*@X{)}?G0F;l2n;Y z2~(sv_M^3XG~mzw057{a;GP6EXs&aGvRVw5qcCBmFwJB1`qcaEG~~F33e^pFp)MBE z6r>?aC{Inn$QVpKsX&z^AxZ&7%1Ls>>SzHLb4?@1nbvT7MVsR@6NgR?W-~NQxr>wv z0{8$bS^}M3uZpQsobZbKMNyczh$?{qK#4Fx1|XZ6gFOty^0OHA1}A45>h1EcH~~kQ zzWoXIkERH00YXFpKYxFA!2~7HBFCAu8&1Xy?|q;N!VnGOL`*1p0t-Hdj?7r^QTB$& zSg;Zxh_#RR<;q996dPI{>c{75Yij63^5zME(TD)@AnH6SG^J|;P>~bA%Krd<#|;jW z3m)*#=AH2@2&9QVe=b}4$F}&8C>9%!yZoSts!rhT_IK;H9RUwR5gksIO`%sFUwwJt zh*lW&(`aFs)O`)M(g3*hk5f2>L)xaKG3(^ULQ>vPC-?zL090@QEpm8xZE|;{41v>> zT)=_1TmI804T+#iNVxL9ua~Fna1TCBy6FM)*r3F z8p0$1w4X9R{Z(P^N(<=K?-%l2j4YrNIR?P-avtin{+AFnr=0CG>muG;%GKMNf(Jk60+=Wg{PwrUpY?=mbNicoA(L{I83xwme4-=vKKN-Aa2R!WuR+tv zi&%NT?$mGYa^?2$+7Wl3NIS`k!0J3xZgbNOi?I-hu^1pQ6$mHdFhUPE`roV#V3BTD z{{R>vs|4vm;{iJzz4rIP5kzt$d=N!QP5KcVY&j3@ymNtpil15+-oADwC6zi}91@_< z!jM4#03LuA1A1HzLMVfAvHD=sm(%STH~|}8eSo(kZoYTxgdP3M}CE8RWcF##fMaSTHYj~tG3=y@M`+uW_H*{+s9M!zW8-idHjdl&EY^M(I*o`@mA-zJP?F|atByyMwuH}5`6DJ00*>U zWP78|rZEsim^KsBmK_8Xa`fIeo_{NkJP<)p2SIBO@%@GfkpNV%rn>p=f?$V|2B39= zq(B^w=hp|t;2wOr_);Er1_Y9C&wJqjXWETZr_0O5JW&VLu?OG(0OJS*kUHUFq!>-) z^D*Q(3=l+t1Oh&L^uNOdN`e5=rM}PS;_%=k#t46(KV70iQb>UoOrLGzslf<4>$!ia z_(asC06Ex1+z8qyY!I;3f}oKb>JdJi{V?>90-WeJdcK;)JYW02^MCP$M8uycw~)WR zz8ID&GwDrQ`aSl4yWR|uCSoJkf4%*%iaaBoyZOVdQZ4L`X0Cg@n393H_WuAlSe6O| zkUaU=dQU0A1Qdu_{;1OidI3JW{@9ce!Q6hoXva>h$U#Jo zLJEYDb32bq3E$tA#7-8<)cZZo{J{rbp@Q}34)k-O1Uvo29V25KlRjIB>-KO$!@%kM z;DQ`fNCS|V8^rSTu*BgZF3(-#8#Zy9!r%axVSjm_utS&`}DyCO>HG9@mRz}&4DN5bIjT>M9BmS-{|E1 zPr@CgWkFH^e|e*i%QZwJ~nG!5THWH?!Zw{1J9p{T+KKyN6C|3ELL@e zHzc3Q;4yfFwe`Y{>P3kU?!WBi?Bu@C`hUhwTO1ApvslS!6l5I7HNZ$(m1LxpV=h{W zEc_}Xmwy&LgH`?=J7Ir@sT8NUJBupE74V&6lJDdn_`nbntV|%;7NEel^%Cb`ao-FgVuYbt@9NQl%_}u*Bh} zC=jW>*OxyW{{V+c{)as);rmtD%p!+i{V4XY720$l8HyQZc5@`p&9X!RD5V%&uMe4H z@tAapsN%rqQ-GGK7%nMX@Xcb08kY$hn_3EhxwtZ!Hn#CIj>e=7KnKF?Y6m0cvEo^C z)Ju{|SdyURP^eJ#6=6ZNco+V?)N5B~U4&J#38karjKYO#1nDk0lh?Z3gUzl{!BSE#DN@`Zvw=zo`ceIx^RV)JciW7*NdY*2N7)3l)~gb8 z%s^2}R7#W(W{|ZcsVGu{R-~w;ADHZUxPilJk`kcRsi9}sIqGX4U)Tb}Qb7jSr6^4+ zc)HL6R-MVU#mo&SU|`I0`eIo`umJ4l-P&%)aAKtT09<$SeS70=@Zwx-ca4jPDVe?X za_JiaX5gKpCnw$HN$W$qe@^olS^oI{0F->pZQQDv%*q1ZbGurR-rY!1REz2}sW1)X z6in*VBMUQ5{{W;hD=2;wc6S3ozbdxEESc6vol-8XiWgZ*zb^WiLRNs`(9u{yZ}pd4 z3GjjeN}x!vHKxg#?jVrY=I%{3sTI~TQfCFESA~ZQo>dpV&8C{ho;)i3H6CA{W@S&S zICb*UNh(lN6$wM#=^&LjJW`v0qS(GM-k_ldYRjGW8v+Ar{Q(@}>L&5b^?_4s}5IUEy~ejmFCtn7k| zhVu7W&-0wyiEA^g_lxS?W|^S&V@$f-!h)Evx^=#XsUK{*j3Y#3wa!toPg;NzNttP4r3AuK z=snV2w}dOa)P)!pQc2qoB4A3wf0$?kJ({FZ!{}Nvey@!r7NUr1*a|h6*O-iqrATvg zr0YoJ6ou$oNrN^6S38))hrdcx2d0&~)UgE8z<})V%C&)Vbt1KUidKT6y-ZxFGc_{m zmPu?9pi)vHNGkCp$p%&u830I90*Hk{4??(TZ1i%#RMS%53el_@qy@EYM-_WJnhQ19 z^`q@B4dchSKZv-0jdJ&kS(gg&9}aO2UbbA&;wy3-){``*Qd|d6P^hL^QQ2Y86(J;I z<`)>e1}hDXlHZuZV{ll|UWHgCae&6HO9wX>1~p#PWU{+M+5Gdf_`*&TJj-(|v0N+& zPGy%Iu#nHf&P&t=s33;>w#!{+qLkGV9Hz`@n5to7RFO{m3_1ysV0*CHpFrvV|H{q^JUxy?_PZxuf#agOe^)%;r`+h04arOxaW=D3qyk6tpYWBm}7w zCakRl6IBKZDo9$z>QdCTgcLw2Fcc1zgu-B44PGddw#1x3CY129ABdXxKXZ`7!)VE= z0($2OlLVtEBq<7_om_k*r9@B!l@$;jz}4c+?Q3i$dfbwbbwm)I{3+;W@L-)W5-_s? zT7v=wI9@EJxTRLRp8 zvY7!B3g&^sEODYzW`Q(1Stv)bsZnwT3FIy<%u!WT;ceYo-6?HBz~CuLi(n_nm1+qG z!hJ7K61AbYgD`b*B#YdaHTVMhgB^#m*aS{NS|$(LtRhKF&svg{sZz(Qb^#?z;$lfb zNHpOudFNJ$e50adbC13-Q&xVg>+{5ST+#FWmiomqEg++$IdttwJW zd8wk!8Oz*4zzai-JnL>Cm>?!(>C~KOYHo2(bzygtOLx zor`@2GHk=N*g2SaXNi-GPiZk@3{pZu(=uGOdeo8tGSU=yyGUxWZb1bG@7NYZ$KQr`h4%*&Ob(&Sz=tW`}YFX zcF~)6s{;+9S_6*M))Cw^ML{HKl%N8U0E?+nVPic^VkD=A?B?cxYI^VHu{_F#OVNW= z*N6|T+>Ii{{e4DnSy7!;q`bv0lv6OWN&*`oHX{BMh$@fNp$6EjrJ1Cuy^!qXt7q%0 z1FLnB2oll~JI_i{!J|=)>Uwl%Z(X}~HO8Hrc3av10AziPH^7;v3guLfarbDMzH0Vk z#e8$b>ZDUyn$pu13(Kc!o|*xPok0X)x@E!@equUnI|DhF_D z+$(P9Yk)3k7N!7#0W9=msi)KCY$6xr7W0(_lz?f`-Z&(Uol9!cIg(sS1EC;t#3c?( z3hFC&<=@`i;ZU&5O*AH*-*4#|_Cn@#`?O3{(^Wo&7ShYtO4fxEq=dMjK_rr-l9gPA z5vfXAq?9HRVER7VY}Y)h*Myj5hjU(C_iA0-UYv|>ZYy@bCCe#N`_8vi87h996)#H3 z(vn>8rWAm!J=h@%l#MAdi__B5TGa=Z)cH}qhMpE+GZ-Dg`8_H}wXp}e#-Dz@{J}|a z)fW>APK6MqCs8Dw_tYbEGHud>5((M4`C)ydCd7hv>*j(nOxOhK@acDRF7*v(O!Y)yGrfDF(F@*qTmdomY{%0{KjEjnmtS`2`cXL+O-H0KvZMBdJ#&$ zIod80?SFUW*I)rSnd}aNf{sTaM;|=Dv@UWb_E*t}StSbXZgy0JjwO_*vslk*C`l?% zLxJo9eV_vs0DZ!*aV;h72`UMB?*w?GzFI*5z@E0Uwh^~>n6hR9S-21#>F$Dga@FJK zMq4nWFIYkeBNd7~0Vj#Qa)X4BE2+`QYF0G&X`t(63>p`+n(={psk_U=|^i$9h`ylxL0li-o=O2S~VgVoC0^B91Tsfslv%Bo3^ z(QakJJ278>xC@PfYETpXpC{(p19w9T^KE|ZHC_n`Qg`a2h5+7a-bI#XNK8{KOjDD&ZOAU+5$w#KY!y4 zQ-E3Cl2Yisd`$o=X+&Hj}`m0)Ev6YDk+?}+0i5!_9JghT;icH3jH z5ItuE5fy3OUZ2PD@xckNpFT$P9tc}NRk^edUf3c6&RW`eF&$GT$9sAbPx!$J4Z#FG zV*6Wdy?#Gq4F;rY=A5ZsPw~F7n4#t%3I~{(u)mSE-A}v;!qjW$U2XE|NKU^G7N7^_ z;8&PbBzg{Z6aBdxdD{yJpyQ*qgTI*GgSrHLBX6b%i1cfGphr-FHlQ-FxX=M08#e( z3=8*6_v?ZPGJjC_`Wz5Jut4NkkrWbCsl1aXU0){F_4D(KVR*QGmUhjcD+BsE;FMUDN^NS{xf4tQzzli%`+_>w(Y4_&{^lV0lhLXDt-%trRO8~I|$NK4e(UDAPw!0AAj?jWv6Qo#m>+lomNWlvdKuG94 z{W_ma5JM!A3F>SvU_tWf&!pk$9?+rY;?zEQ?MTMWnr4ywDGt>->RXjOcY}TY$K{C< zI7d78@`%Nb;@$=NQoX}yG68@NJNrxan?XWEffLr;`3`>I?{7koYQ(#9<@kvHyAu2pC zAy{$>aS8|N@1?K8ypP2X;cL<_WwQJfiQT&F>QmdFM{}knDV}>z+3Y06qbpSPOPr}V ze$Zt~%yJBJJTq|K8pN%8uVHfVPY_|@JZ02GrX_bsR-6U+w*Qi$=N5&b}` z>w<^oC4OxSLeREYS`-wOAw^s&LXzQ0JRpK7LDi}G2cIgNhZqul3a zt0_{_tHM*>scIv_QlM;Xd1s~kALctDKM;hHaJh~WZlI!Et_e$a6mMLTY5*x9);+iQ z1nn31Z>H=fPsWLIvtGyJa+iN&5jPMqIn=ewC7;^JW)!yFfciZg+1Ueh%bS#WGM%+*fS@ z_}5P8-WfAVMKf?Tk^+JCFd)SEL`V{E(m~6lGC)250I$f#5^ZKFxqO6eE);#XD;jR4 zw7E>~rraAzPzgv9ZP`d;uq>?~_=wRn(<=;XND-Ap@Tu_mgW{uP|Q&8J)3pxl| z6C@VgDIqST00LY>ieN-aN<>qAl^6MSZx(pEg>h+DX57-;sv^O}`4f$0bk!7X2e@vx zF2GMEMq`n5F`XZTVi z!%X(vhG9V37IAB^A;8;15IC2!;m0`>6`VX0a~gKi9;P8(;$BE0O*oh;1f(_zi17u2 zK@)>q)XJOc)O28O9)#;+ZYg%@la!7y2_DgYB~(y7*{lbDIMdWUi*fx$V5g|bpQB|f zhpL{^8A3GxwIs>bK#~9g@C;00rx7J2uhtqG(2{%k`1nMgt!h^V1JU1406K@Wh`#Zb zRZYX0eHRa~;#W|DkfIb0m86BOY9@b5(pI7%0zn4YmP)-~<7UtUaBMZR0_VqgMf(o* zZL58}Vopm?a}2Skl#*O+1q+K66cUs;37e@Y1P-zW*tp}>ijd0hG$X;=mXEb>R;M%t z^dt&Cz$P5ENWhta@tbvZ&5;dJT&qQYCs*t zKf%Fk?gTzm#XIFkS9sgaApu&2^&o`Dgs6~GrzB_+PzX|2hMjD{7CKZC8%hha7OSyn zj&&uf?nBe44qb>FcZ~#ofIKqQMIQ=qgEbo0`8<;0upv9eJxphtZPKy;5lxMB1q9>naep z{{S-0Gj7v3@0`0;=|dfg#HS08%$qLh;t6GnPn|FD=Or3R0v`$0Y+1i(1NkV%ITcl%kO7V;+(5Utk}D zYVH}Oq0K9J%O|NuS(nwnnPps56Ps2Vy62lfck^^Ld2x1~tfMk+p@dMjTymnAwH_3! zL6_UE*5T*+>3Oy)a_Y({l=ZlB3*6LKNcEm0@a^bh6@gY?R^|uyc?I?D7IpJ1&^s_Fpw4_FQF&NFMr@RK5}dVs>`fbT;}D17@A0 z6K?6<5L8JZA{#I zhnlY5_H0Qm!Bb8f6DDesKxKfT!|eO2F76ooaxK->Jcirx0U8JwPyh>&bb~jO1l&P0 zj~{L4&Ylta4HT582i_N>RBu9CV0qFa=Z`70B9$bhE)?J%6Fb41z%dhH7>a3O!V48J z?kDSC2%cEf^KPw6R^V&hz4M0XkyP9HEBa8kN$%7~iAZ)7+SB10feO?Ji3BQbIDu(0 z!AQ6?1ZY7O=0GB+YeFVs2?V~VK`J{R<5kq9x*W9EFywbN46P+4B=}0f zI+JJ-1Ou__Sy-7N{i_QQLs!F{%d)8Si*mTg43pqV19Hz!tiVRn;qJyWjEgOHr#3-u zvGfQ>LPRCD$&Fe{jq>IdZoOT4(VN9Q^SqC#eQ>`q<%?< zS)t0;P>Sr;ma&pVgDuv!aC_^p6|1yR_34^DRG~mfccB1OyO2Wye&x(r^12~Wo3h;- zC=Amn46?4^IHuB~kQ7@$Fr|ye89M9vgTY1Z2^7zswrEKJf$U0r@TP!(%!QvOE7}HZY1hknw^R_0EeK` zp6-N?Iv8#dB9pu9Xw5xLv(>jK?_%jMRz49ZO17jo03Gy#yMBy&&s=U z{bMg6Dq|#_NPSpTzc+el?RSh*IcZ7OGy~w0Vm47G3BITxgS-N7BM6dNz&bzbHowBQ zhY&`D3SWOO(lz7%0IdGoM3Em~nA$KYxD~}xN$+diZ}HOb58`2d_G|bx$nOl$3|54X zp?~yL;brCBpMD_wL3OnutlvH0{tu+6Ab^*$3U{+=*21F6a-WJCE+m*zP*Qqh(Vm5H zyg5&3F~kA+yu%BG96)@mTBWG_LyOd`<1kAh_ms4Rg}RdkC&m<3lpT(; zjU=LV1zdpyxmnRcJU|0`eTEd6*rRs8|ln_C;fS8XNW3yv{ z&81ICt{C)sl3A`x4*hrgk&*Ug9ii-QGb^1bBw}!UAXt;NiG3^&jefg3MtdvI>heC( z_;9j#ceS1z$@upzaf`CuR%qi1&ncw?LRQXhGX<8w3T+Rv787iJKlC@+aoV24WwR7a z%tl?ufW?ABiI*Zw#b7m{P8lI7HUR_+7W|8TIK33j^hxRWwfGqrVS&MRn-?RrSVf@< zLQ^_^bA*$dd!Qt#c$v8A%T*nCkW@>8t#H~zs!T?m`HT7e&N`Cn5_kPP_eYa3j6GwJ zAA@21(B+^L2H+X;JtwI0>M+Q-KrH$fh)$p+#^BsYu(`G7U`71!L`n>KA1wh;=Y8*U z9;bd*n??w12#eo+zi9bjf*38~J93!6-zneAY%xjHTy3cP`PwW=nfwYr^1sRcUJqdn zP%|;g=al}}{!Pv!HLldr{2Y_ySVf2m(>Lfo{`U04&{~_Hr|3Ljh^tRNzc0V-i6DrM zwtrCQN{8+~bAlsUbsheGa2b@rJ6umeGCz-hEJn2I2VQYw4?F$Z&cOuO3c5+~;qMy= zX}2wpxa;rbh~ne`h?p{d;(u|HEeBdiQ-|ASg)&jZXT! z1wurEZ7~OYBK_gh>S3(kC{0?zx87~)NRNgZLpL{W=KID zvSe@8;@*Nf7&u6pAt(bsDH9*x1R?m2WO%pQ0fH6=y~p3*_Q4RWG33-60Rd37%tikI zkL~M%8v(%tIR1qCoBN;bKPiki_>VEU<)yqiTU?GG_bA+Pf2mCUU`yC~{{T32PVg)c zb%2mB{?AA_4_@8hm@{t=^C&BpzUHeR|*g+tUOjG=eQW?LOT7@5uiZW0 zF$4)o)IE$Ib8fc2*TKSudN%G`=L9NZEw%m=h+`#({n9_p{-r`GI=x03r15nTB9q0` zKnS0my4&DAWB0@{35P1Q1Je`}^|0Ob`fiKT!Acw*YXnNj7i-gW?Q?TL}@j{g9e zjCLdU%l`l<@IpjNZ3Q9;o0znG_vME|BsQndkI%W{LIJ$&W8dz6;{-Y-B{9fO{$t91 zxQR-8JA4i5-VFp1DJR~-)A!})(8d}YSMDHk{-bKtLV!(yI?z2ghbWT(?;43V9QmG? znY>3lNQAS}wjO!^07$KfUWNYvok~68?|YNoTf@>I0n^v_1AG41d{=`YbNe61?g;b2 z1WaiINF-iK6Bh(WNrNWb_uB+Yfkz`>5Y6ESNPq=^+mb}uZR%zRToCIGDcu84juc>s zfy_fer$0Yf#ja|mgs6~1QG*KA^}QgV5*fumf8XYOq0LF#jr%l zOLf=c&~zm4)BI1g{B40sAw4bhevjb@SQ4So_2pr9d+I3%LJ1}G|W0$OX`e5Q$ z3u@N-J^Zt@I9kKH-hA_qM85-6iBtSW;)ns>CG6<{kg`AhA6=SeYD!3vA+}sFuM9|3 z_;KU^0L3iOyjNlH+ha2ow=d3ibwIGukW>!ICcwwmFYM041ZBRN@!&aDOWB;g*sx9` zFYQ(lRkyM;5>C{3*B|OXT)NI9qRYaP5SD75xf0qG$lWFsok$873IN!^0b}Opr5s}a z0KYgb`FmEkrTG)_4WxdKKM46;rFLr8*T2_}CHB2g*wvbAer(9u7QlUh5PpX%>Xdn4O`os8F^pLLIdM(OiW3d8$ZQ1FE zAkJt zdT=LFHyCU#P;AOkp+2?0Qo z2(KYxIrFED9w8HrJ(i_v*1g8{a%kezjfN^IxRWPtmDYfHrFV&|AilQaSyB?+Iz!4T zkXE-l;{=dXY*(5INq^UCX?>#7X7H(tz1wkD8#uk5f#Vy=Gy2{ZU3O7fM9ycCW?Goc zvV6!BOIFcoZKY#f>FV1GQKqRtL##Md0*XmNfhyplzjPX!ySbnv`U2eTtM;Hxlt^dW z+N+lo(V5wcnrV+>;iVv5mYtq)L5 zNf&ySXT5}KbJ_lFnp7sMH_Tf{<`<%+#k!YjbPo!cXd_c?q$rfBTFR7&l#dpx0#vm7 z0muwqja!Yasqfq{PN4Q!)`XodE%F=Q(U#S4>ls2D6#ddElGvsxZDGwWHjfEW(hMnL zH4s9zE)-)92lidXgV?#FI3!uFM)rt^t+sO4sBS@ny@4Qfiwm^rQk2^K3b0C(r0MB4 zGC>3ren8lgBPtDg)X-45z1&>u^mqETgJ4~%NO0XRZCmRXKFxSGr;U3|?DK-D(&O)O zuO9Gs2~)O%4Y>aR5%CsZleUE>6SAuN!~X!H@|}(Bs|KugX_jCEl;d+OzXfM>zAH^Lo+?~|iQegqvC;+Ds?6J4VX~5kDCsI1a40@;!~HFGm-5(bE=z@3 z^_aAU%?=#k_(4L|R|=ehm96W zjFBm2DNM;ZCLt4~VU&Q$mX{^gqaI*(bNJ5V44X8hs%ECGw3+^zak`aY0_D@6fF1eHLaCZ@B&nvxA1NgVjT9m8 zr7KHT&A)s_iIF1{C)G{>hh*890LAKFmZQ^Wk4gUk2wZr+J?!Fos?eeo;~9ev(bXeT0vA?_#?X`bdB>}L9cRkVas-*AerqfV!Q&_jokDWN z@>$YXC`>|^uyKc#;IQ+uP7tDr@-qD8 zSWHZb4Zx)z-GeO(R&)5P;ni_h;or7?94#7$a}Le8){2ZH#!XX{S)!Rz6L|jspSw|# zNJ>%?qEte&-KxTjcV{zM5{I=nFC{^$TMH}%b0jzb0_r8j0UUSG#&Dj^dR_E=n&Xlt zNy__AgGL>4)D#k<+MH6SPDli_w397rTHU3~;zF54C*rb=q@IXe5X9H`E zqCN$sWr#bM7CmTRMsaQ9WN_UNYFQXdbXASE(GY`ELj79Mg`^azUBgd}S_FWTAOJy; zinB7@JmozsX>HlZ%$RYP zDR@bTPYiH%TS`J-SNrutk;K(q%Z!9n7I?8dcqS*~eandWshRrL}Dtw;8R6K5n7V(;G9 za&_eSQd21(vJFDiBE^}TTF{-1L%%t2`ZGF|)YayU)zM|8rG+$*8Dhyo(3jLw(RHms zDN2H#aV8WHvlC`Q6g}>`^KYLka;;2IDqpJr5l3ndxieKS;493XvoS1%MT1y_-VC%l)G4 zs#Ez^#OexdNkUZ;0V03pi6DXunIOyyz^6St1RB`Wqo0Vj>`Y~_47X<&8;{|8p0wq7 z*ZfOP5lZ8RQ=v)oI!aVE89cv85gmC=bg$YiO^&6xYt!Ktu_)ncyB6`kflqzuXw|*^bKKu*#AB>5iX{nyG10q)kJq8Qyiq0=-o19z?u8 zW0%Y+mXzb>w-m*Nr9L9%bv1lRn$%ZUDNohbRXtfy;Sg5y?zrQHBlM?ImQ(=nq#Yu_ zN6!rV8;Hfv#$)j_)VxM3iyFCw$x_th)RXNJNiSEs+CNPmkMc|gPlC$w9J32A1(szv z90mg|ktrc5lYqb_D3v7$O%M=}5|oAkm}uUF_C3Q3G&JjNFj9pA0(Au)Boh`;(hj5N z1_S|*IChg9*wdp^sM@48x3_eC7W8kIre<9!wY4Y<`2v*|ppT^g0K!iYyT?C+=CXR{ zL;0^_Tx#B05~P6gcw38-x^=Xrc!zSvaZQx1sUtzCaF7DQ$G86g2wkc(=HoIY&Ml0< zttcE+{PqL(p;fv3nlY%RlPRc=g1_5G=z>lPMnUNZ465 zFhIT+pPlx_u%gC?(V^*E$)X}s{^ z%HjfE&l4He^{U^n;?g~i>F?9jbeDlfM^9YR|KuC$9`FAdf^n7ijYPwZ$;F@X6ur=dSI^GYEF-DSIvf!7 z-tUC00{Lvr{si91CY~$0A+<}yZ7$r?#%G}b1Qym+X7cbkE2}cz^d;9a7c&~&0RCtt z$7iQXj!=8Ibyrfq8h&VDA2#!MsKypNAY(XRw1^;qm!$cOZ7Jalk+iJZqW9E=ROfPlw~Lpn^Fe$xPu&u| zXt^GEA*ux?xyGybbE|kserB7jBUU4z{*)T(joIb{oc@vlwT2D=h7_-t{i{Lio$T#+zTMc0UE>3N{goQD6@oT&p}>lGtkQ}zTd;NAIc ztr;7gitwlDIMJ^zdF6D_vVkt$-KIi-U%j(0=L&rovb1T;=cu<=xl}bEw_kKM@2)m6 zWU&S}I))9|#h!xi4V?bJu?v~xX3v+}BL+07!c46mG#N8IS?Ozu4jhc#oZC!4Sm0wv ztmUeck$ASjo$;ExVfpP7lRyQ%yx9$W;eBNFD~<|BV`qWln(g~P_~T@G;u(Z$%?9hb zo}DVAeUfFK3dYu6?1+`rzVH*3d^iRbii7|-P!YY4kPixL&LQMN=#D|~+6}kT@$QBC zvfqgGJd_71((dO1QLfxJWa17v3SaP2Rf&QXfHm<2Hvw;nON;DYPdS5IFz6vhdPGphyEbm*0+ zC!A&7V1R6jq;iSyVtEL_g$n`BgvRX%+cK4Vsyc{YIv7s>zPwWNG0_8F8O}ED2~N0f z=6=)b>Y<#=qC}~R>=8Gp5&Ttq*p@p3l3NsYOKZYoZU{H(B=3uAAT)azkyrHoNON_Y zXEBuuYdDko z=2x(!2aEg-t>~+T`0A;_3;2c4v+BRUCbl zLQZ^;H5`T|{yVNjUjN%qmh_^KtSc$r$aKqz5!-0AqvB!@5mbS7w|>T^pxd)g z@Hy^wHu6u2L#l6hn_6kpRTI^>Xj(E<$l2H>%jzb9vd{yKl7FfWnIkmCbzpeK%H78Q09ovNwv24Qlw;3pl_ z0i=&{ymvT$*T*M}i>vCGNC# z*$`1`7vWSby8-_o92e@Zd5>3uGk4%h#LCviJ_do%vS!Ng^c(HTD<)w@wZr?$%L5!| zp!b^>qCp>nmX;m-87TX`Q+RRVu~y(sQmw^uEwHxF-#j;DVuzm-flDVXvY)VXJw1sH zIWKvI^g!kY$>Yk@`$|rhNJkWLp=Qgb&BVV01Ko=}5ALLD9?@Yi)}EMFtJ*%n+F)|C$I~^e$Yvk+C&v-#wZzRg%GWH`8ki z5lhWjg1u4m`ue?Q5Nx9EnO^I|JuljrV~kN%1c$c=$OZ%@q(bH3Y=N44mxXV0={R7SCb zuDk?~95g(cZr2TI<{2H95wd5U>`}ufAgOPi5r%@1c?BzjC+dE`BWqX4vXJ*ug1oU8yhl-X@~TOroEIjl#9$^8P^g{p6L2t&9zP+sf!b z3kHU4?=4?JsO)<)4bNW)cF!|G4_>3LmY6g6GG?VN{wT_ftd0P4~YQaCOq{Vf$=OS~NH>@?e} z(7<^*^B*xt(DCT#_Yq664L`~4IU41pzxWUQc`2spBfNtZ+=wRz)KOqm{ovZJV)V)W zW~BZWqE>0fk@&sU$M`MiryMVXq0TH;s2Xh#G(f0&)AApnMtop4!!Ch56YCVaTAlZJ zq*G`;*z|-TUY$rL59p*n!Pue|E3Vv1QmpV}4yMSY27Ijm#xjrlfR-x*931Z~K6G>Y zV82KGr5R*4E_MpIB8jc2nJh4={Lpq}VvN_S&(s`RdCp~036w{a`)-*X7h0^TM6E&H z()DX`k3c;u+z}ZmADc_l+ogm4A0U0VebUKnr`piDLU8mK+(T^xHZ@{Ig7HVH0`8z{ zk(7KI-JU{YaYT;fQYY)p9K=5U0CuCNRyU_}ci`7iCpoae79*rwtHoER97)xPc|{{r zPQVTATXtl)AaTvgS&frcdds$h(|4`EG;J;7yu$CR2fM1-X1|5EFkP)LGs4ul`3;g0 zP)If!uh`;{o#ZWJNkQDI!q}U2T)l$#@){QB&_ubQL+!*skuUH~k*3x-dw|ljo42M0 z85}L`5|4neVUn zyQC9phwzb4&t+N!{8MiF7g_D2ss(lL=+xmr!Z}tVYZBhj2sSy+xAO9Fh|qA`bP9x=I#Pqg!P1&UdPnaD< zn#E2f1hdQy9;(doNIWgq&Wj%q}*ld8|p1!wgW*HrML;?LdVoQuhiYMlSil2TL&JWzm}9r zj^TaxVk^Pq7So87w7@8Q$gX_DYRg*eLIfe@NKP1J5)Tagb8X*3uViIVpRA2(cq#d< zX!$G9vlz~U|AJmxr-CB@`OeCYAwc*p-t*WoJ-TJzb*{319muH9Gyr9?dy{?=31vDr zacr=U-lQ=4`gFCDh`JjkwJp;6j@xCEay>tos0n_jd8qdDv@`g3)>sA~;@g5d2{pgp za4~7UwWt^u@Tw$V#ok*^YtN9S;W%BP+S6~+V9WD>`SY9*c9cp&_&AVIa|%zN-PjS+ zz#*)q{{R?odZUbh_#7m{MUuV;7qCCw1gjxRZ4+}${m|Ql2CjnN1|S32)_EoS@R+PB zlYS=S~ej5g3 z)N=D5G(Eo9ar<~fx-+}MM_9?wN^V9-vitVuE1tI|x%+MORECnlQnb+@L1v7u0~tD~ zKKPG{*X}EUGR{9Rmrm!c#@aldlUOFTi3qdfAz?xo)&pe@#;Q1kO6Wk)T5e1@+vcYV zA1$`;F?vTg$tB_0vgmEP+#)n{Fm72?)gi{xl~0CYeYaGO*YhqEJp<}Udf542CGUQg za*}a7z|Z@I4O@I_Izqqo#kxD_gWcd*1@b;KI$B5EZ$;MVJ#r$ooj5sfPmGNt`v>L` zPv`AFzARyQlr|vbD6E+cd9tZHa34Dv&s4Yx9y{8x*=@U)O#vNrF@TW?SjZ?4LH{P2 zP{fT=Cz#}lyALa4G-j=L+NJ%*-~m7$56$rrX^!Y>!i$K2xKC6JUYJu7e~ z;hJwM0|sJv!jCq`*@aIgKe8&JqL5-iL2nmLi*Cnoaa1?GT|(Mc-5e>y%QZU9R-N$- zu&OHZOgf`TwGMKPtVXizTjm>n`24>jy-uHVp^jmx=3@G5Y4#srfm70c=;NDc{_p-d zbA*-LwY%7%PvnGb1L;O! z+8SCXkGFM%Wj(B6K-Mt!SW<}Tde?_)Xbwt@ z8;79)>@a8)i$SFldpeOhZMfwM-y7rw%?d9kfp_XkYucv|LB6T~0NTF_1ji$f;oULTn!6fF z#dkofCaMwt)!v@->Q54aHl?<=Qp;t2t2B<1=*vDhf;-v6ll$S*aNe6hOZ^6A>4%3MY6(cY0NvYqUvzN!E1~hRFdF6+EjqSd^@@a z9HL~R2|1_1b!$b~{R$(~2Dshu;Lhp5tA_Z28HxC;G0~pk%+L^$oCCiuf#&Q#<{m4O z!vZHL?kj?mhUokrdCU|jCCogLtZJzcn6;6P#^!Rmn3}ZcD`$X;1+hEAmN9C7wwB_F z`V{f>^T)ScErcT02uc!+KT3y?lJ3E#YID0Z11!!zWoW|xmSH${zQt!9Jg_AJUj!nP zI+Mq8UcxFS^&1F?7-`mr|1dlWcOq2N{{0ei=p@}wt9^pPATTaLiq%DA^&}*K_kN$W z8JUTZL7*Tx4~q)d&zdaSdsF3`yfi@NXTgY7n@2FJpxaQo;K!pjBB_hxO{)%`O}oEL&V7A|;Ba|YbarHuu`mGp zsw}>41fu3r;prd5)=}f=8E?WW-u7-s_e?L&y`4=WWkVVM6j1%#(A_xWJ! zHY|NSRXh3+b-ZrqMU^xJwQ1@ffZdPee)9Mq;N#hoj7OU2ZSBU5Sz&a(DKrW(&3)wc z_sjKw`=@wn*O3=%#5o}J6gz3=cPyH<@(iK~%8YE{w(W{N^y#t}h{jBet0%kXrTP66 zsbfSqDo<9-*vnDC&?4%OlTE^gDIY%S%~ia)?z*2{4c>r;hC`{quCqA&I-do!FZG&_ zxZGW~D%@?UXwjEyQ8;fI?p~6Zrf)L=d{mwNjTW%i{rKYCFsuFKoP2y-XJ?Sb2`Z}5&Tk5mk-b*VuPXK1WekHTz1P#f30Y`I^qL$BG-43X>AKu^#*DY>ueu!@lb>Vxue%iz+bhxit zn0=x65SbeA-BHYwR|td^+m0umeQ)S;)em6&uiW4E=d4lxN-O2k-Oounx0KY_Q)qC- zUXUb2gweCDz#48+QB+FM*52TiHv!WWJU`!I0x=p60rcq9_FE%Qzhk)oIuU#0XNy35 zH8{!hk6ZNc)}m^T%9-w~>va{8H=onMO^TmzYilh&gS)Ovb&JzP&H}$KX*jduMDNN$ z;kZrIp}KyitiuB6soOV1g8VY9+G=0+Qd(sk6RjG@7Dpvz4}o^LPUdUyhA%zw3(mP% z0*5uuhV9K~JlP9MxTsPux^;LcmS?Z=J=&r2RQDueI^teW&8`TT%XHbC1$B& zl(kU0{;S$&Fwckkc)t7(Y>AdF@cPl;WVU}Z@INcqfc=44k<*LH>L1=;PjLJxZFCsY zw##g=h5LtrxI0BIhai(Wo`?s$Uo7t``ip5rX(JeYZE9ct0ak(dvc5`Rt(osw@&c_F zHhBAAk412(CUuz2 z$us;?@x$pN7-n6&HA|#L?k`2@^w=py1+TUxE!*gQlYz1{2S8eCX&;t8rebncHfA(& z=$VAcOm;6O4vPi?!Gd(~#lg2>45ZG07Rg9Cafb|m>L)qL(6zhg&-4tNxBc_!)FIf{ z8+?Gg`#Bt~(NaZJYqzU&jlh;wIoRoND zAu$__07fLfmfDam>ICX;Igmo!Yf9%qVZF4hN^3>`9wT2?nN#^$@NX>r0ZpExSxuVH zNokjhkG)DUo8#kUcoJ`Jx~5E~T4VLC4$6CTA!PKZHPdz4&Ya>{S#h0-uP6*V$MD64 zNLb*+U1DT4zZ{6O`IS979slDyMCLyDynn$T^mFqH&OHczaF)As55=@1}C4{P& zY~Vs)hNut~aDT?`UBsbLWZ%B7{R2#)d8*1DP`?CmUA#OfHv`(GvX1|Y+e(3J2((t{y!L60u zyH;tdzj&VE+w@OrFaBO(bjid z7f*{VQH-`!={Os~&B%Y5aYN_3BlXe?oL9(WHF|LNf9astcfb>A%{mwvzQ5_W>K|{N zO`$_iYbn{>fm(($F)?p&`y9@N)WL0&m2Aa3n(KK|Twncz?h(tm?Ig7tqXwBnjIW}A zk&y!ZaiywichTSEAttq5as1gGp24#cvwP-St4^zVu0SyhWrPI!a9Ur}@oNy9gY z_T&@Y%E4*x@iw|!+r3T6i%NO=>&d=^?(V-sy@#YaEu?S7lmbsHx~gg^f}goi@=D4#kQlJzjrDhX&|sI zJ`WM#v2o1$Qv46zxDWzTNSu){e#k?fO$PnMbZtr0wa^#HGubO0!O%06c3=CRBQDu_ zH9C}MQ5vmg%-F8kZuvV>9Q!Az4;pNOnuCdGLU&#Y*3t8=u0lFIh@PRvNbK*egOrNM z2;LXk41z0-3fA#{P2Ft_qu$~`QYe&&vwA@Wbf5M>_c|-nXFILX*=_;T8r++O-BQd7 z+huQmUfvDyxE1*(U|TLYur)^u$2~Dfp^~s&TV=xnTNJ_;c_RkmhF_8a;T$kLX_#Eu33h#&PMAo+mh{q(8gB+oj#)vN^%8Z zhXjq_6D@G}SQL-ISjG}t)O`a7R-QMums?ysf+1sQ5oR<|l0Y!1Q*;z=Y)8dT_Rb`T z9)DswB-emVrA7(Uunlo6@-R;tG?zv!(94?Bi06xakol1nS`-R-E~Ayux`1a&3+(#R z=>{NNt#~EG>2$Je>@Jot2OS`ue5iz<)Gsb2WF|N@myIqVe2~)-O^`OGUh;LxeLSCy zLq4_L30;pcl`x%2Ll0U(Y{X{g+%Q{zG|j?K^ddlxK!Z8g?J5%HA0126f_~80$|~VK(B`hY=ovO;Lyi zzdEYw5fdg`>w0|D2(BNfwW()Jz3%*IOrfH&iY$1DCe6flgZ~dep?ZM~05Y}sSkxMP z50w~X`=I6`FV5&!nZRmj$f%px)h&ZIqNG2F!vuxpyf@Z&;m}G4Mh^S0=z%2qvK8n> z^cE6%v%czAu#~n=4RosgI((aux{v=PP#Wr43J}*uMuIaeh)Ew$-l(@cC zkb@u}o5ROhG1=67outqdexCSe(BJqh8s9bZ)T+qfoP4{Voxx!%E*qP@2@0rC5OZR$ z@Y=mcHxqL#0%-P_$pz7}3?Ol6Ai9HQbTkcru#y`XkhtPkh;QIcC@^N)Z>d=iG~6oh zh0!I%$0SB-Hw?W?I$vvIN;csBovFO+$iS zUqrEE{}v=wHR-cO(xzIL*3zTG=o3=tFl0-Ly7nyxer(EA=dR!yz~JR2=Q} z5sePT7B|gw`uD!sacJ3C{{VVjYt_?OI=6H^>FNRT0a{gWjq>>9TSz{%8D|yEi(U|k)kMx3?qDW{uw{KMa{7^Si!+89 zI!2%^-&bK`cUO^-K_Q=u=@kzpAmt)#vKkdnUM>i`q ziFm7($PFi*1i))8kkD<%Ttg*GJwf4vz#LRcc_O`xTC4Zy`ce*nWG|8i82R1hQr;>u z15H?ASPn)?f{#@(3Z#w~dqom;g0+d7d|IFjf2v>GK9It6U%!A>d?ozo(*kM5^AQFm zm+1N%ismHUJDYZu{dFrKQ`F%ts*Qv;Pjfpr^`{FAEeX_q2J#zr(;-k|VRQ(7wvg75 z9%5&S!ZZH$*`Iiyl0r4341FHNxNK^A@RM3PW20R;M}LY41t(&2vHIu&WY5SM!pLk+ zfd|DQbI^;KZhic6s{lukNqia&cvA|qc%?Ye=Pn)dV29GtRhbKnuA5&r0jT`xu z$@l1=+1WA}8Hw(8r8f#$HTj7RG&g_#>DXfpLMHgq$EBXpLgVTGw^M@?p`S^RiC#J^ z81;o-ZSn~>{x2~PVP)2r3AGdnM&l8Tx7t~%=`3ViXrJo&oeu?I7p%xU%UKx7gl*fp z8I_r4WkE$hm^88F->&qnzO9yZap{WW$S9lLI&Z@f<4-FluRk6yJf4zTWOF{$EozwY z5=Sb1u^gS4(=b`s)>L;uS2JL(eQS`;rHHLX1ak$*A;aT`Zy}=coN+aMDvhVtKLlhH z;Qx{XoNVWlCUkYU-k1^UHhSUl@}WAiV6O~f#L(p3uS{x~eKlWRgBN38tp5SBo~3_c z3Qmjq=MPc(r|0-DON|+fzt1+t^;M{^{Rdi!=<9S7<3xj1Z7{Rh^OpH#(>Fsl*Z=kdU)|{pfO@|#~)R!Uq#K#0Q9F})#U`RSt zA_FSGkx2HQKjr;Vb(svgu2OZ9BJca+$a6iL1Xmy|!KRvtYP ztMSZ~_zg;+@);4fQ6B&x?f7lgurf4>B|bI;9yerRsVC36K$V0X@QdLrbV-bc&Z9ia zE9Z`+eQ$4~aCexK&x#SbB9m^UGI^vl_tk;w$|CR5D ze93d7mc5EmUuS~Od%^KV6Sa4>NpWor8SCgWnU$&@jHQ*%tmFiFshD1nBkAyKg=IS{ z-ur@u_n>#fu^AoQ8V0^Z3@FiWS*Q_kNW`u6qxyB3*yY?J30a$a^;^V8G4<57#0h`9z8yUz`L-lIuuHl& z>+|GIXJ=%Nhd1W|%(kCrB?5sUk&iuV3{jtEWpy1lBQ5%CJq`<7mr>vEj?0ZUI;f?r zV)a?SCkolR!IB-QB*4p7SlBL`$m55xsSQ}OX!Yzg@Go@mV?4uZ?T!wPs*W;RkJvZ7 z%V3LGm5cdqZ99>nDCK#^5Nm?%PzY4^p*w58?ej2I{IGR-o)!as!t*{K*ki!JG$$ub z@1rW;W7^-l_l>7hwqzfbAeKy)=VTvIQNwyHlara$Gs9HTQ~|u&Xy^a|oIeu*jH8fS z?^#5`dPi$|RUOk4ijl-^;9ou)fr1B*=sg`|in>JC>N)h~j?R$#q$nH;GPBlmA9s_> z54TVC8^m*&dZ2kxoe}u)qkK;~pap=&4p|~#TT|-DU3pd^=64YK6H;yJD^z-jYUwuG zZ^!?(Ud?JpFObx@x3I97Cc@Zr_O^FL2`hTeY^j^36%QH}-FQc37)BdeX)kAlWCR4q z>vWynzb))vU2BH#5q$0bF2bSVy};SmpSFqPF|(3KQHnqeFVgn1w^f1+N4PN-SZ}WfsYpdy)$aie<2y$xxaQ8F@_LF9^PxcxzoTU#3D~_6Air>* zgSj1Nyu=2XU}(5irU7)|OK{=lTu=ij6$O?wJ#ufM`^liMUJgsacz84{MOf5DZYeec zP~6rnHGJ1yH%@@~n`!_bz73{~Wmwv>?2t9KOxe468CoJt(Q(?3R0Q(T`dvNtRRB~e z<2FOIo!@=_5&qX=ss=%age9m5C$M$RHQ2_bRt`S{n09#9{G!qW5Be3;zGPp%irtXP zpXN|m^qa}R=n|^ofz_+}4nBJM*p2>`-G@sG4=MD4Cb+9i@m~9`aPJ>r4@rm7>{{Yj4{l}3i9sWTjAc|O*=`8DYNg61+Y%c|AQv1CX z(HT|6D_-<%6V5-t@eZzmIS*3$?%H#Zh!hVN^@sVxu@(KD%+Pk*WI&rpD;>*%wUo76 zy&^-yHJlK5?Ok_$u%8)f85PDkdbRmsGsN87v|bUTSjv(;ac6#*=Ku=FHCl!fW;4(D zT_=W7*Sm!ZA(t{R`_#B8uyM^K$guwjK4*%U4SZPf3wT*U`W2HO@Ve45SZGn$^R9+` zt;iOU_T}Mm%aZO-;1wQvvtQcv6Fl$PNs%qXBQ-M!qD|QFxs0Wr4XkyOmrZX!9EWd6 zC8p4515d_v8Adc7*FaM4Z-Ng`A7V4)UaHXQI)8)8uR3IKKIaLAK%at?$-6!80tbZt z_X+@>)?#P3`%~ii@_#lU3C6ki0VIoVeoM{hAr((SDCGb9#tD-PCkcpLEj_9XmoaL8 zmweb8BVPAQ%3R11zA*G;ROtn(_5$^Q*?`yLvDLE!DZk+idRNEv+r0V~uX65W#l%SL z7@o4H+-)$UGmy1mCCO#eTzeB&H40Z7ZgoUrfbVSm4nBAx-cnwd7eYxuiuLtq%!(&8 zxJ%dls*-AD@Bm;0o=TOW+v}vzT^jl~P9{ky4I6lnRa-_*4i|qUAz`8?mkORSbq%Bb zbM}yBNW1i@{uQo_dAa~80$kxV5&Uo_&DaVTqb}DMa;!AIUA1e}OsRvt$hty?!o=$s zztiJ)b=r}X!BOxxB^5e-D4fl|ApZYM!89y*U=u3^aq;Tdgy6C=$b{Nc9;Ir1UC6a1s-6v@7}isK*XaPf@OZ zLD>AjPVk~K6WIP-&+#CR_z%FKl@WtvIkAgmru1lPH~?`uTRb~>T2alwSwhkLirjw6 zEhQ89e|MSAWK6G+HB#`qT)P=S6QB-fvEhAU1QgSP?ms%z{I zagm`zi#(D5iQ{&M$hr&Kq*B6do+3CC7ozC(y#&3p1q#{s0arJNp&7svpc>7z*umY) z-2kvXSYgkQc9ghdWwpR(wdU23OFL<4;mZxl0@?=X9@nohV<=lms& z2*ecF?H#?hZ+pn>2=q!OlXG)8am2Uy0~dQEMh|_iZxEuVP0- zge#@Z>En9vZN_>|tx;Zsf_PIzv4dE}^+)5d=`Ek_p(kh856OO4cjv$MplXP;0PnDa zpw|~(`Q9_z@<%Tu>N1=0JA9J3Q9Z9>8P9Y% zU8G*zAygg`wrF^xe~!3N0NSyaWB4%vkEwc&O+Wcy!vH5xM-Flbyrj=h<9_FY)c5Hf z+*kc~l$tDB=J;~9;g;Y@0ttRB?+LAdPwU&%SYj>`e5ud+=FBbMALj0dYMzH3)=aD0 z9Qr_DmFGwpx-%O)f}F#GUL+_v-wBuoGkmaEg(2Uj39Loe{+?4`%5Zd5psCRAe-W?aajfD7sm0Js9?xP9OtI6>jV1Id^FUEaA^vC%UogjvDFPOCcXbuGur z!cxdL-at|Hc zadPDwsW;+tWo6L~l`)6xD@^4%x*bt4Rljd}D%xXL6Vr_kQxu|Od>rWsy&7;km?qX? zXrrCAA)teq;ePfhP5q(&+s2mwrbC~4zeUR*4(TJ40TKm!X}nn-guBeP17x3V_enp= zp%49r6tz3~DE1KW?S?XJxc1*|W5)bY7&*KWm(xObh`yTmQsYnoZ&M~5TlUUh`Gm1v zZrovTibqRD?LXgIH63T1XdJK!pvkHfOR{@XK)NYCps0aeq~SfkM)(9rt$H2z)3qgXV-02+ddwf10q`ytVCgtF^+9JO~z96ZSU9+ zj`B8R4rd~1t6;O+i;CILV`Ju)cjI29eg3l7AHJ@KY_8`(BC<3f?xfFMstBZ!?$yuc zc#Ly+gUsG!sfdLU%OyT_{ir*jcXl8tvPbIy?H)U__tsygZ+%*fpo5bGcH^XePRdA< zMzS*NRYqBPrGk3Q|v?k%UZG^IMaufD+nu{oFYg`>_dq<|TnbY8?IVUUXdkV$ z!5hS7X)DO)6+v#q{%N}&;xpeeO6BP-Wvx}v>7z6&RvzQm^N+UY^3I#4gwsp%Ip~-^ zm^jYwgv_I}$y9*k!a?Y&uhqb}p^M5QKRjF|1}Y4797UIY#4lXeraQjb;A0nme6-hO zQqskuO@@9iNx^TRpfdJi?crr&){o7|mz{5We{a$z7Ps;4vU*4u#Vkwj7pH#Byoi=T z^7i#1cwhv9w$k-Tgk2Xr7_Pynxbf2-fljS|FI!$UU)uH+xXny&4wBOy z_^L8<=JHZ;&SRa2xbOy2TtKt+5ov3pm8)d*Lk8RB7&}%uSNXWZrSbd0tsJnpF}ZHd zVxb;{QvVB;{d{|dUD(LwXxdv_EB?PfF!@N|s@}x*TF%3);J2Ddu@cORz{pQ3Fg*0y zbBokMo%&=G1<5wr@LGYlFh@-OkUMtA&_WfO501fHrS@q^SJj?{+Rp@KuG3%GrlFJ1o%{Jbxjl_5Gh$ z)Au=(j=nU_5btFEj!mAe4W{7iF8bO@bL+-j0nhKyz2((6%l?`m<56RWwQhc|MdG8M z=CJSO7S0S*4E5_Ja06a@E9D@~=nlw@EEE6&eYawU-K3TdsM3$)NatAl?Yl}1QZS2``*K};erX>andESC((RNo2gW1z)@rpSsfkkoSsD)1-|2DykEAWFd z4gxW==ILPjVgY_JxceH#m;r3t82nDGSCs8NLZ}~(R_*qmVB^ilm6k+Ib4d#JS(G#% zF0=2^*Vr=?^>plEaw_OjJ^lOWv<;kK120+$d>Zx_O7A+EYqiy{bCA88q zN<^v9ZYSsl*uu=@7{MVw_Kl4k{HBI?TUC{|4vZNqIz-4(CzJl5yf=Ub zZVSUGF}T(xN2>D}qAL_LQ(h8RDE~2Uustz`XQl%3%rX`WRMl_P*qE>8ZwaAkauN3VEBIHdW7n?!8w+(F&h;U zNhQ`g?lhPa!Q;W_|27j-A1@M;sH{q&i=v7M%%XiP${ajt%f0)Y9+=#dX+}0u&XkQ#9BheDHFT6+6AN%8{Si?lUm9ni+p1#|74B_ z=hP3~s#15bYxKQx4pjIF;x)~`C-u<{kJK)sD#B4eR;w#_XjEPsL!I8a zqjQX}{cHUsZ!|{#ZzKe9&nfh@UjF)>1y(L;_Ggt>w)3EbzU?RcKbu4h{3lbAc}@5Q zqw?MLihC!yPI={rwi7PDrL~q&z4RSs4#Dc4S#`CVDdupoJc3`<&1?Tn10&8FDk|WXma!kjVVY_(7n;nUyrOTU zY!SfF)pJM@y%E#1afne$o1wvFJ- z2M1wDDvYecA%V%VVwrOEOB@=DFMemc7X-LQ|`(7axb${osD@QpNa3297z{HS*<_}`)RpadcB{DEM z!OkxiIr(~abTYYR&*5t6Af3Zhl-*iI!iXfbU;;PCfk)ZV?9Hcspd6A0wE$`Y^2&5& zxi|rhluV0{R9f0Av4mrLvlau>B(LLU3`OS7Iqt;Uo(}`Z6z%eUaBAN3!{yDP-|#YA z7ujHZd0vqrgoT^>@U@?|pE7(0=%Ix(KZP)C9!EuSR#~JSn;h@9L1pzUll@bU!+)X| zGkhS2DEQM4Ty~&m-4q1zv0B%18#i@+6+Hi{eym-2qxt;qA0X;Jw~nNbGUc~+mcWH% zCj-{Ij=)0_-jkzLsy|iaaBzBILCMQniD)7Vgd$W-rLNhko{iD0y7d_UEt){vp7iZO zI0z+7ym`%-Hr#KsaO2*(mrJF4uDg$=|J`Sdr|;pak{Svm3mNXd z9)-Y;V*=?wyenjgA-g;&qO-yQ`8lkD09WuN*%$)-q_?|U|So>`u(t>xqxGAI0?oS}-lGS3!=?{a!3)M9T1 zt_Em&UoD`v*5Y$*#Hdc!5eKBM=Z}-Y6khck;4eiGP=6KGid>(%EyU%wF^Rd8fDs3OG4sZ7W3S|Wxl`2`c*Iz<+>1fSSu`<-w zzn^z72?m(Zbg9sLfe5nLNt^4Pa=mCY7SbV_lu|3@%T}kbd+3lKXIPkU3*$ zXowL;Wv!uh_-elar;QS#t>-o6cURwa)zdTEUaGZn9o)e|2s*1y0Nl)nJUj&;8L%fD z{SW;=0Oz}0bLPG4I9rh#^$e_X;a4Jg7AtyO8KdsuKq2Y>l(Z3O8){v`;t>A-ltbjF z@*n%O7T*K9tPv1Zx(AYe80K#JCH{|H87pK~b5N&#CkYo0UoJ^W&Zy6l;Rnne(3E|~ zZG$Dxh-n#IhCKL(s@*<*=ES4KQ~ZpluyXLH5cYtpuG-Q-&Ry0*u>&M#4lH^eqWqsk zDXK$l9b=4#=>s~q9Y8B|zq=IGk*f&4_oXw&_=IwX)7QTRm19XQQAwdv;q%)gRjjCF z=*<0UWB+;+0vCwER=*u-H9QK{pI@`mKL3@L@eLc=rdAxl=R>&->)Nmf_4N|?uDJZs z>@uWNdM5kp$OGb4JU2iiSUfxX3kUoOM-tUV1wM533{cUptH}hI97f{opD33n84qe0 zN)O5u!YLSna8Fd*t6PH>4$#mtoY6I$Su+o0;Q^H058U~qJq_e~bW(@Ql+2Ne->lW7 z^iZDGR`1|w0RUou=E$epT`68UESWlo6=t8XIVg>#+jV7hdeBDRO7q9d1^oX!X8*em z%akjTV|aI55O3pu{zZk(#HYF~UvXv-&xy+^caRSOT;{tteL(X;2b!+ROKRm$gODQ? zRk}C3`!Ny<3Uxufa_F%r+Cx0AVj7<7Tht9oYXCOo5UKfC7Ofj+0J73e{Y=4kPu}ZL zK(on)-?kE7t&J{NINwp*cZH1+ksYNp_jpb#nAZTueTrrEY&&R2`n)W<<-92{X{592 zWb`@k{w|+HBH3Bdi>CXouYSoF&5)ZIjUybi$p(B3&NaUsh=rpei1?gmv%-TK4Z}Bf z{MLVeg#sh|7z-oI@18!Z5zQbWRn|o$eN|OWySU85yH5>$A2Ry7*nI_5ZZccvy6C{O zM3)KK;O!9-zM7B`ZpM?h?o&ew+F4xDqJ(s6HdA}XzMuSU$*z9qAyYdxVriIcDsO+raBHb;m4_l+!_`CFC zbDtXlII2>UU8h^QHdwL;6d|sW!LNq?px2j>3bj4mEj{WSJY)G(U;zlmug+KGqDOSLRi956YWZJ$5!_pPFk4 zq-uu_jgN%}1a#7YFWtG;qJjRfdjxb9B=2YiH|PNoogR1b>t-4o8_6;g%CH?7++s_! z&BOqo3GC*uzsP2A!OuJHSDb%0FxX6Wo5?i6Lg0GU4ndz5B#}&chi{=cALik>mm{l zyB|s3yHaYf^ybYUc|GVYF0GBlXf#8%$F?R;jg|#%chVinKfr!Q6al)|U}yz0?NpRv zDuUX)93x^ye&{t%iM4)e1|V5(JxUhle7Zt{ZNFm?7e#cNzzGc59U-I@1ByJTn;cWx zq;=4-WzpWhU!d1oT}wIgX|@R-8bf{)pxbXg<##HhJ1QX)Psc>~@hkd5S^gcY)a9Iq z>VgJvplP2clJx`LmO}SEI_&>3^_Brm{g3EY#^@1Jf`D`*NOwykEzL$r zNh2lQAzhLJQqmotd*9#x$^CQ>ceZoh=Ze>*TVC^?hl|@#FsiiEK?j0qLg%2=HlEm) zQf$}w8$Q{=m9zcP`73cX8!zS=;tIbd|EnouwKfhxpp~NwZEzBKqF~?FgOgr__UNkn z!2QMXr`)HG3lT>C)e7y(Oj#B)z;(qRm4&);1E5G~Z*QwgIN7_jN@iUuL$yWO0 z@0S#eJ2vlO+EX7Y72ei;&3_M3N*Jb+CSEvZr?Qt*8Z5_cN7C}~agmLX?IqYPNi?1& zg^I3VIEl>9DXz*Ah-=C#=Dg*WYT2BmRKTio{&7@bT&;i|ZpwOHO!w<=&YfL=>}oLC zu6x=AWh5~^AFnpP6Nu?&g@aN3*Yht!;BEb%l*yF-lGl@m0W*+v!N6TI#1?6Jq~@2E z*A-2TsS#U}GQ##4Bmsx7c^_g3BT4h*u=>WZg>3c^l4I0^?waf)Q?U3GiEVkDB)xS^ z5AnNvINV^?(7nclVgOXiTPZw0I)(@^ODlhjdmDH^KKo=3iR@G`C{MSQNxYKcUKk%0 zGUI)ZoViTf+i)@-#WcI`AV^+9i8_*9q6zS1_{;FVhpH>~l#K*XyZ*^X1B^f+o38b)nC~#a);=3;pbxg-3eKhCj`C!Marx}0jqvLvV3-3UO`j+!C zA4u7T*QQ@Q6+WHjXxHXj$-C#4MAT)y)`8 zff||eyc1=sL(-s?*Ud-CQ|EN60D-ln3ENIR+@|eln{scL)4yty04}{T%@NEg@(?(F(dMc>|DTdhxV$`|91YAf%L<;HwgRR*chn+2#6@Y zkl}Fx#>(^<-9oM0BVTXy=VI|HgP#UQ`gYRPZVY~{L{;=&2lOn8biE~aU%N7q4Nx=8oUr#za&MU0q(z5vHLw5SP!#WC!nY5bX3e%nhL2#R3vJ@Cx zY3G%Md+OpyHf{8KTiWz-1|tG0TUzBj6X$@h4Cxmm(4{VP=m=QW8iVMbvO7OXXH2MZ zY*DUDAYO?wxWXv0P>2$fybrCXk*Ly9eSFcJ`)5`!(EC`mIs%~Bpcf9Mkzl5#T+ z+ODfiWc?z>CzTS_DTmgvO}l^J-JDrF-kV3_Nk(svZHV@ z>^tN`F|$~0xFA|;EAXk*?A>%K64r@{viI(n>#vryfb++9tL#o);jfed0@`+T$$$J9 z0H5sojX$2)_-Prznw&U0ihkSJn7=P4y|uQ;`E2_(M`E{52FL3}R7a;DOVwl0)=$)? zq22cb7e?kbKZKUJz7m2=bOu-JXDy%34KF<^*8)zUW_-?iI9+j5?M?}L^QlK*x-l-N z$Ei{47Z`dWRym$8bhW$ygM5uh&7HK^tsY&+AfCaGx^+mQNM*Niv>V zzi3LjjWuZ0#*2?Xk&lU`exBGBsIs{(6>}FOtr~+DOD}oRDEv~?@FwN4EuIn8&JQqN z(@=F_M`gM73npVCwpm~yM5)&?ti}5Hwk%wx#~fzN|z&F91(+xW# zCa+-+%mL8>)I$`p+bh}s9h|S+yKTGn`|`zzQAAz-2yBR99CygpbfE@O>eWWCxeQ9o zY5I(xN!GW5|2++Ir?pWY2pxbv&}|MS8R?b%Iy952#quJzTSwZ3PHh_ftzNkGT+OWb zI{v%DbQ@sE%0K6*=D>tD@E>3#l9&|EQy<6*cIU_$06_~wVul~{OWe~nJenPc(w&6p z1`zuD`r(|;(V$AMG7z*#KW`WX2SH%~Y|9MNqHu+jjiPk&Q8OGww1J^hb|GXSkk`S4 zX0DtwyWDWW{GBeC7H_<`#ZV4gfzCKQr<0Wlq8!Q8vS80RNU)`~NphF;VM=iSZ)17G z9S6#aDRNzVxlP(5!QS85)%e73##P=8C4D#zqwhpC<)B=SCLeM1Gu@x0hgVbf$|5=R z3VEz^J3K2iLkMg&XN8Q8>`~IMujBm6-_?}5bsK1Ex@cNLsP(FtsTFIQf7{{Pb9iWx zpIRH|XbdH4i7IL;%C_m*{Ys)6%EzV2gL(U=MKbe6eq(u?VV__d1s|Q^r%8f2Nik6!l^m&S*mQ?wbWJb1A?o&C+gG_!7wQLA#q+>y>IHaqgNiN>p_6sKr=PS_3m=n z+%j#V{)tC1!+_MuBQm-JR^P?An`kGKB&nJSvO|ya+aLpWv!EA0q|jm*l1HUc&C{W# z5nh~~scyy>DhJhOg@0Mquw}2On%_#{|6QEfPNlz>Vq#9fFDLc^q(o0E_@X61JS**k zRqake>_IYbxr)y+Cghw{T{%lV<+)njHD7`0fkdV$vH`5pi4v*^4b5i;UQH+nP_k#m zr^?8Gd|DB*Nt3g^oyl4}txY}3tE0#`z)X!F`{+#W6pb{}Z_R%X3x!30jQJ|Sm}2HB zVprQSPLjEApgdM}^kxm?_3JTp9yAV>i8>5~g_Qx4WAj0^Le>k`0hF@mK zebnPl!K@in>2NlFRGBMwV=9FhydI_r_Yo88Jsf%cP=vT=aDX@=kZ|%8NO)SwdLglg z2MO?e(ObSoZtdgU${|^o{%Qb@^+;03i`O9qXAE~XSORu?daoW8@D8YJ8ydA#ca-;p0kDl3fi zbmiC?i@|j8$}!ZSkbi9!#++RvYye%^ z!BYd0JoaT?x0NMS3HyzT-GVv&0;285z}l{PpzWk&Zm##f*Zg+haiR|EW5FKF4+kYy z>y{sywOYeaA0f#&h+_J-(n$IDqg~7W zb%^tK!P%TN+dqQodF9boZF-^R!gP2Fo!u9$f3rX!n&GrSlrjjhTH@K|B_d*7M+U+;Xx@3YeSryV}#v z$_F+SVJcS~YBE+M{4uMta%}9j@ISUM)h0GmLlcz>m(PjBr~_Xf!XpXws(UD8kC^gr z!;Qu`){C~M@W(hZ($A-n&%fdAJ-pFW1SgW>*@%1d+hndeG}Vt;=;$bNxWHvvh-dZe znl@4{@rNuu7;*=yf@K^Bdf8*Sga*mNhU}`z{M*T7>OrX><=IfH;Ua`|1|!g*8hRJs z``%_-5IAN|L6t{XsnwL%$aXflwitxoC`Me?1u z|9q}jcR9$OaA&?BNhyOnBV<0vWIp_II7)EM5=ouwV%oJMAn&$==pmucBP+T;lwVAX z&lb$kEAnQ*iWbA9C5YvWqDZ@c)2a)1uN-M*y+!|IjJ=Kv(LJddkL zj}+8rSVC6YS`k;#fipFsPpz$|)uSP>40HUowg-V8X0}`_yV=mk@0yeyhSQo{kW~7K z(S}W1Wh|E{oRfw7dY^=_^<;G?2}XxE%#5hnYehMGeqY831@b=nNQOV&9GIVTU$knB z-G2~6Yl3s>cA@t>Zb)RE5B73aO_1rA#VOCTD`tDZb>w zoxK{+Hx2JyF?wfhNYQb~2XopX;hkQZyL?Ss#iIsM0{`<)jt*NAt6@ zDV2$G_ZmlP%?L@Vddl)^eE#m1GRBDcQnK+jVN)puVukb3Q;%6qe+mjng2uq03A*C}!^-T{mhIy9^S?iB% z{fQ`I>tZ5V*7Gt2vw3fkeb){E|C<6NH? z<_+>Zo63{q8$U}^DD@HK3*)KpB}wb=3tAH+pv-DecgLNQAQI7?7&Aw*1S-tM9wn<~ zO`k(jp!Ea+g^Sob+Ob^!opm9IxZ!>n~byg ztMvu&Hvep(D}_t^2gnm(jX!+>O?~H9)k&qeeR)h~3e-kb@|Gb%D1!X&d+(oZ#S;h@ zFwm3#wC3IuE9u8E%CX@;SG%2dQ`h}3u1FcO$-Ft}+5g+HDVm7+)%rQG+bCL4hc$xj zX@a3UTZM)IlUX9PQ0CK9k5H)T$kl%MOvKqi?eli{6ytw@e?8w{Js4e)MciLmPVuwU zMp$#wH?ON+`6&Txcq4)mzOf99G9=Tu@nRvrS)MnK^sxfFq!2yWK-Hs%XIY~Z7%2ac zNT2b6uDg8Soqbi&Z)V^FQ2_D=*?qe`t?$40G=)Yw$ZOyN75+&a{gtn(WNk~6B^t74 zssC$7(?uqvD|z|CQq(ry(6Lv3FO(0*O!)r&wigqmg^*+JSSLS)R)7--#iwS}B-}mfg@{*pm!qH1nTu#ppPOk^l+YWE|6(b9D0}$H? z^x1KoJ4Yirf|La_DsUiP%^KjbwKkb?hmyMAo?$JS@&8Q~v` zA5r;9=gvgJbbu7M`DdwKp&8BdU)tnrTgGwA-pKV4kHPTq*AxRTujJqYIe!g?7~Lpx zf=`RDJ`Z09rIZd{a`7;_RXW4gLhvvnuoKl${Bq>rH@kk%%(`3@L>uKv1dI8B?NGER|w+>O?-f%pG*~cTz zuAhAjMDCyZ(S~{>A#Clz&~5P$GMlXf93K9Es~&`83!!x*chWcZXFpMF?Y6bH3!sWb5Ie_dUm)=p z)B!A1l8*M zCF7AC?YQ5uWwi)1m`HTl)x$~vdDq;~H#d|xpCy%L@jpQDRX7_`gv-HnUm}KF{MYF# zkF)=2w*Z__rH!W-#EMO52VT{NNH}Kd*lu1@q;}Jq->PWe>Xy&kiNiM$*Bwmb3l6+k ziajJ*dC421gE-<7`kNj-(y~Gq&9NT=CADuJq7P7!Z`)5}qyj|bp$hfF@0D&fL=na8& zHQ0qc1awPpc98P0iOO?aRuF6np!fH{k97M3j{425~b=e2U*3{z|))2#gPZt47{ z;e!X6+$xMHie<0w5cM^%c9Mk3YpOKQGy*OvVUoIDgmA>og1@g!ZuXy1%#J)Gamxv^QWTn+2Cg69 z1s&hQzPM7a7Iz)t)BySrpPrqa)Q{ms?UO$Pab4M`-Lwogbe(B91pDLs$DX%pTxdW- zOkNHpKZE-mpPo~k#3gJAr>bjWC>Dx&KABLtd7=FtafmHP#u`?AE1H@Th$TMs|wOF0KQUllb{!SF!QB`RW?HdhAXt zM(f$%Ur&>~v((e#dA8(&kWK&C!IS%2OAFQ}2KXpax9gYNfXQ0S7~~kJkQ!hK?TXy zHBAu4);GM;Dqb{V3*wM?=zgVzRB2fZ8Z(YPX>Qd{ zJgJWz875E6T)1ID4O$J~lio#P7Ta7R&nj$ME)lng1TRavFrbV25boKLbi1)6FK{{cWk9Oj%JxltEz@bLSd4z>#> zGd7bku@s#%8%i^^)w)NA+p9@deyb&F4JPqr)oZADYLT0F->*5v! zTl=6vX5~{n+t`e+;i2A+?IaXg{+5ErVV8qvV8FWaWa))8f6l<*o@P{6_nUci+orUl zqeK*J$%XBe#!Kk)=Mm!e&@pXW*ME|t;dts410rALxPn*^Nx}<@mNpFm$~;xryj~FEhfFV=*r+zj=%4V3=Oetb@+l6Y zRA%=|q_kK)+53rW4#UxIymv1@OS&*48O?EcPmGF*Sd9xvJ59yCqcFM!O!W?@OV`_* z-eGxB^eq(&;Gej91=dtfIHF9pe105U+6OaDoweI$K=i^Oa%Yv}VWf3HQji0hz_n4O`Nd~$5_no^$#}{pB?I9yiVphWLt6lVE(xZs<((mjOHdLG6 z1wWZpk)Y9Uy1&ntwCq$?G>TOqRbc$K*!ptrw=%pPALcnlx%FInS+keEC3C^yKqfNk z0Q<2slYSTP4K`N5nIhvr6@x-jitDs;eIZ&+HBwRz9e2Zu=}7?Nyw61Du*2ZYmejun zctQki3iyoJnGnJi?d~mM-;KP6A;HWk>;|yM<*C0k}Tge zg}t|PmFevTh!>j435uwKLL0Df(`mb0jzLHesIE~gRtijHGlEsnTEDkc^o`_I-9Yz+ zR3&QV{OL?`wsi%g$wl#1!Y)62qRNT$tAUnT@`PZY$h#viDH!OEw1kx6ovzL!d8Rk8 zeou_B%7)bN=G%^sTgKEPoO2C3{5C5}wbJ&zcEIf2`rE8$k-2sKQ{I3t2QJZ|jR^*2 zW#}QD33Sc>Gmf%XHENVprm?Djf7AzNQ`6jx->oVAWE|&`YbJrGZ;28Lk^u6!I2IMG zKX7P{>E9lZEf^C}?1LwSaGOuM_S*VD3z`o~k~PC?sva8uiV#fCURMa|ZDggILWcN! zrQ#;AZN_oeZYB^zY;Xb#=#Dbc&|RwOAA2Qv;jz+6;ITkw@Ww5bwd{!aXrs2oA*xkp zZID;uiK!@3kj5k;Q8?Np$l@WY?H7*1;Db7=FwgAMyqUpMDTRdTJc3v4aRS zClegvgF;#w>zLzL~D}Od=H7D~6l4%+lcUs_$4qYVdhda09MONba=(T)!sh+r0 z&|?kB>1XhVj2C|hxBm)E#E+b5uVc85_-ooA-IEr#oIvcEH>Q^(zDRp+RbMN6MM82q zsa9H)!D>EY0*3jVSu-5b^?N6NSGCAx;qh36GQg*D)X}uc!B7(~_z)+}FaBeHl~goE zNZgU11x%`uIGxS;VOax;fU;(&BiDRX*js)#noRI%_x#JM8S07WxmX~ZTjq3Tst{dd zX2KhBbxj;pxV*y_7T_E%&j=(}FZe0l^h*Dv;qcjYMpF1>H!)F9_R_#$FXv-38fpbs zd!>It&Fxr6KZ3-%cp|q9cys8tXplY5<~x1`2qh+oS4yzTiDlX+tpLnh%KdSo_q8ZV z2Gb=CBW_dT2){`*E;mHr%_oESsfGl>%$%eQB=*HIlk}1!Y$hOTv0@}p1sHp$R&-5TqeYNb~yeZdl$5P}ivTb5d}SVn(iIr#*v)a&3&our?OVs^AJ zwJZy#`-?^w)@ifnul5YKNn_or5B~BLz-q>^ZO*1bZm(bw)pkYOB_-#c1}Eh>uSm}# z;>^tQh+QU-6D>rmAfKVX1NYwkzOu6B^t)y@1dBt6Bja0aRYL!pd$ z`qH>;8i&tg9n_y zTDfARn@%TX2khU5(~BhA`)?&7`=&t^Sgb`~;rWT5q-ft-_QcTt?BJ^PNY#FLj)-Wm zHj!+KWOe)&RF-S{`8%fpHC)BYxjL9LSUVbZ4 zHhzWnE2R%=iu2z_8isqAGMk8-s^189<_o9uvME4myv7R^?Ja` z0U)gW>4$L+f69$mWkHKq$?Z;V{mHyFefEjl*469*OeiL+P;G+M#y1i~3A-&Mt3bpMd#U&{Qo@=Q@P2taDg@0Wf0;g|sM6ym=AN7YGG_GNPqD{DWnXYNKPgSIF_Vo`A)ZH^#5bt~@{N z|C!y))K`A;y{3_n8)<{q7Vn3nkU~7v(D>EV*sUwjfX9pkq(&o@+h%&~R;>XdlfqJY z4wcq){oKsKg4%#>B);(##_rx`)>EU7V9HAR@h95dSN6af%-tk@+spzrXd!G==SH!% z6>A=9R_c(?82>|wSU0(Jmr^eq8i0kX_b6X zUT~uMA|&l{)s5+dW|8fv)3#>Osb~Ujr&*q$(h1it&LIhBW<0*wv?#m@Mb@?E*T|M; zag)q{{$T}0KH^wQ&UrHJ24)D4dqC5skgVq&tS@UAKO^tR6J1~sselR1#CR0epKfEd z*cDQ{WNf{p)+6F3l#gq9^RdpgP-8QGo zvpCl9!+DO>)WTWHWsRiH!Wo&8)5IS{sQZ*qAHwTAOJtoD{$ID>{Xamva!AfYfMfYm zH+sE>7X{1QK|sPAw%935nf)MVnO7r5gj)C}?M+|tz?3r2=;i@YwK!Xo_R=X*;&eGr z>%o}a<{I+L>+XMmA@2Xlh5t*?ikLEJXUV#j7O==X`Uma&Q8Djd0^Ch&Hz|FqiqR&& z|JoXeE~O?laKiUJIlt?IW+w6HLvn-#OqSs34}w3*8AHSn;zN60C^o938uNKpZgV8( z;DG2f95bnyW#)2o?%!{B1!_O3miA?{zwgBuMNnOF(bN#U$h%&`l);uzzPct=wVr8I zAVTZGm|k)<4{3UQ9|sG5U_dM}LwaD)R+fp)!txeEnGR7-M~NW_OZHc_o4%yPgQ*uw zjJF3bj81QodhBj{@r+B|{=6_^GvaLcdG``2LOy&Qe00@j4yJJ?Fz$c*uiz{@oTd8k z3o#}#Z=j-+SKOCB{2(b!Lwl)*tKaxqbA%d`52JhI54VFJXIve7RJ20=u+;wmhO^44jmFO<}qIa_i?dALK#HAJp+J*qoVyoQYXDPB8ccaGC0{C-n!N z@vyma$)X2rju3K}vRi6q))ltTQ^**I;*wCGX9N~_S=fnnLuz)d4peHDGoy)++-!(@MQuw`#O|DfyqK4JsOC4m^~bz*Dyz5+~U1@wcpv%PTI_`}U}S2r|dFh!3B5$Fn$(M%)xD}zAz1`>YfVT|W>{bL4?ZGpGoNy>8!(J#PiBP>67U!N^?#^P^ zHxPkR4~8EI(R7|M5txDO0^rIxvpfsbBqijUk;c4QWZjChf{?h29=X+LAGo~U_HooB zK$vt-Ahn6J0GN}8l?@P%#K{9p-J4W3=4XA@`$E860vDybFbiV;*tC%asnYC2-}{_Lul}D_ zeyxcXhudU%p|o#vl{!tin{3 zE%66jVtF(z>fAXaSDUwV@x%Z+F)|)+uI`(qu0=S}TZ}y?>Kfrxe|L-aru6fuIK!Xro@%N2Z95$2h{oU%~ z>&sAe;q$KJpQVns4fSHpOT(M z3XM&L4|0q!4|%g`j=ovvqC1$`XX5-hrFeABl3LWtxs?8~aG;y$Lfn<9pmt|XKn5tC z*#0oA&_zb=DlWH5ASj*+8}K?znu`VNH}DRQ9*OI$dG1|T`1~4K2JdTpywR()+bG z|M>(?s=D{^WXg_c zZ4^zJHZC$(%WYPbll~>Rf zl=$mKO+xdPX_7%WQSW~-NN}P?wOH@)x!=Ptw8{4${82LebPz;_bSB|5afl(jvkjyv8D|1j z7K{{2zt<<`hH?C+9KEc_Ij&hwTRO8_+@$Q+9=WX6KJ#3sj$htBU#jQny{Vy2r2e%b zvx5pRTXF4bs?q8!r>(ALZDM6ncPZ^jO9!QcwA+a@+6w-P7m?#mQ_5b66&&7&A$YL@ z2Mz7mi09%%DO<67X;@#|M~^3ob^vuEef6Bf{Z(8(VizGHS-`s`JgslBYB=wk_c)L> zN&*(0Sq$JoO$n2bvL@1;u$w^D$Pv4G9(%P)wS8;nSuHqRIqt4zCo{!>%i!oPj$VA^ zy&<%Le^PqEzS`rh+V?D!W20{z|4nuO3;2q<8I3gTHxmcrn>4*DeF!*>NhS9HWWbj0 z;U})^wg=iDyBb9uSOV_UO51t zs{~nuVS6-jz>cxiz?^u{d((H1e!+?Mt!&ygtfP1{k}Czm1oW~6dk?by3qfQ;AG(`a zht`$*nZTvS*qk#AmMNquB-?tE={yj*a%qSTQh)Az%UOj7yqWDHy-zLOK+ZUi6TWft zXL`FW>BC!lUfP>nl_PVHBV~iT?be0jG5;9JKa6ZRA9#?0#s}a)3>?I7drI`mf%V9( zJQq*Z#Gq6#kC52ZJVE{VqO>bwVb{rhiEwtm2Alho4?AUbo^TYR3u!JUXz7oYz1G)- z_FY%_IvmsJeTFQi(8mq@8j&_FXDcQ4p;7j0pqu!CJ%4n|bZElA5oyR@AiJH{!$Fnl zty&Qz*>(kP-G#ku_AZvXZpvP~f)rB@py+FtB$IPrunww&26 zR;?eYxshITTG#JFYbaHBYE_n;z-j*cYLu@n>||QakZxPEKmT24#}SkR@{pKK{}I`_ zN{sk^q<7?3SUFd}yZm*yYh_EI9iaMMIOn`?@*f*X!KzzWb5)%`n3O8apn>1Fq~N^-Z1tg@D65<^ePu*nNAXT z=JQa2JZ_9R&;`eRsDZzk9Z&C7H!LS5W_2!Bwi%kiRE*Vg1!Ah%gLZJ&O_OQXv_)dG zIK&}&9W(qMZ6XJiZK&%*1ruwhlA&^zC$;d<=**U?>~|0v-#3IM3OP`0fL0Wq(vdT( zf_@2J;e>YDYb8Vz)Te9``c`wby0;{W1=MQ=xS6fjyM4ZHd^2sx=NOL@|AlwfPFD2A+l(_3tkdN`wW_#JxE z4SGL4TF)A4_j8mRrJmG7!TXyF(ZbrWy$A%P;*wA55CvSURMh_mKw@Qjx0MgYEkk|B z{;5<1Bt_Zj+H;0^|M^Av6dX+PE5@4m=ZBp4sB4#ht&Atd935Rbu~>|H_N0qE3~gr% z>Z&H@SLaGir{_UG;T)0Sik09nP!xb1oK17|fY{pE21=tppS)qaSdauQ z(~ocK9mm*SKGd1Bz6Zuw$@BGlDvH<(!xRl{EprW&P2)klWTCBwJ1HwOa`PpnASf%8 zF(o5@S8K>>f0Dkz`S1#{#W}$J>~*Bx13sqGQjh);cVpy7=_kVYYN$o8M?vk5(H%bJ z75_rVsZ_rOg_bMy0%4Jk2EC|*&20FaFc-1;?^j=b8U{Q9|M{C2b@9+OHmrQ8uOud? z6Ol~p*F8bP^Qj=9iWtL^*HwVl0DXy}S}Bn*fLwi8Kt#o8sFsM_6{+#~@5u<7X6p69 zC?~pQQuUOPTXk&G#sasx$r$f2EG|xiINkdUPU$}d`C9C{XkA4jXLH$0waIYdqRLz9 z#$l~nr!VwJdFW?w8aZ@g+%&Z7uX1f`zG8EY&kAXd$|C`nr8BdGx%t+mw?e!mWmtKl z6+JTs|CH@-@#w76J}7p$LXn{Wmh4#I;{?(le=TuySxU&@yTs~EClZ$mb7_W*Jw~;( zDU(x`*_z+YK6q3r2K`); z7R5e1PAW7lpc^q=UQ_YyGMHiSnUm<|lFZP(LP7ZFM4q|BeI+gPX;56on@vb~s~TAb^@$Tv(v2jbQWs3M@f_xP;^N!qbte0O300>$Xn~ zjUqsXSKP{quMJC-0vQa%cAe(SNUSCkTn( z797NpH9L3!kqrlhywp_?JfNxX^91b$VI7x&dg4YD9v6tV~n?2YUcJ8mY0;{MCKIK z2#02s7o#S5Ryx?|#A64Hi}^`pF8)wv6Oz!Ytl=zQ3&Vv3m9s^^L;2nZ zzP@IA@~ZO5iZOGZezcr_xR&N=)jNoLADfIHY?r!Eaw75YIE9&LZL;Z_CG#oN6nxCS zEcf`j9OpwPQ^vC$@~Q4&`cd!T2en$n$R*-WD@ysnICh*^G=4Q0Iao}U_NkY)nJ}`o z5NY$}y_k^^sj>B3R-pzC-%--mbEJBd%pHGW?w0LVQqlIe|IJD$H|nTdAl7&4(FFd$( zmdEnIkLJaHN1*QA=A&2Bhaz4Sbna|cC7drK3<*yR+}uRC>kMnCOh@alufhDdg!Qbi zb-FJ-&P3S#^v$7sSv{L`+w*yZ%T_-0rN`k6e0Q;kgP-gXVO5+=kDRPuA3@JRq3A!N zjMy>4(u0tkgVWQfNc_F0%84t!(mOX4qZGd9fEV`ZX;cx*pLJ_Jf1V$HYV@*%bZ0Yj zI93zn5Z$n>%J?!OK!qGgFhaaPo0FSldo_^gGcR6DG~ES z#B2Zmm2xqH%|Fh_zbc|r5g11b0D zP1lxo_edxSIsBhHvzgrQnFRpxFE3Y@Km8uouET*ZFV7yd|C3sj!mJ21)BxP_%blb+ z5&r>-u##mvEPDoDEvg|ju_A4tU4s7N^MyDy5)ztgRE-Dk%lHM|P5OS$5kDS=Ra29+-HE;*)07If81AcxjV<9&5tFNk(fhM96& zc?>_j^dKe#hJ!=hmPUp!jk0+$Hv7lRGclR;SX~bt%ZDqJF;Fq zm;V7O`9tMU;5`Y_41>RK65MH|saAJkR5XEIep`1BqCwAZ;sDsV_G^R-!5x7bYFcY@ zdqKU&2hT&;JZ7O64D**T8K5w$g327m?sNg^)8XM=IH{76k;|DM14Au+`L5JIFUR`T zNDLSG1InhqKW7hy!gqBYH!u+cxJ_5cn^Q!F(zt2|)4B3L3-lJxrRpyFdeH$VNCM%Q zOst?;Bw~#MAFT6;eWmPmxg!B)K)c%bK+6!{-65}-dmfDvg+!3NG z^q*RDAK%~yI^+k&vtmtxIHNgcGAMgb z^yNEHpF)-$c-hq{IYCF`PmEC<5l+s=pPga;X>IDYgFu6b8Ex99FjC#wH7N8x#F_B1 z$GiRbIuxe7gR9zK&>yl(U|c`@`o>;ljNGI=Q5D7X01QSuG+&*XgU}%pbGNQqdsU zYBN(Xa!|~_;OBFt+?EcfZG3qyELH(OWLI19;gsYABTG$MlqL0`&Q*acwkkJlN;>ysGsTSv$dC#6$q%6tsgH;tNbs zMxMpd8P4m8tMV3IwIi8J*$FtxY9b`BN6u4nB&vi!$gy23)6kmp0-a7HGgxE>FJc0X zt(=GIYWUu9Z9YC!!s(-c)isf zb=`3kJPL+C`^8*n?lPm4?e>wS?*tZT1Umi&DDFl(gAg3LpF_{2bv(A-@V;DRNmBo z81rk;ZGm7ClBQ%=Q(D^jbM9!QIlYBVldnXJ9f@mzZjkGX{k0XSf~=VPx#|IjYt_~& zFqTPHV8k?gKrY42F{q~OYx3$^9&y($#m{-@c>=kO#r9s(o53pEa6L<;R^yfeqrw;LB zd$SlZ#S@tY$bW!8x@CBRkq*ZH!o#WLu_07;&^&0LylE`zF>jj3ubFNpglfny|Bu`B za8JWI>SAfdN)1=L%M_`+ch~XObg1snv!-^|zHx4agJYGMT9r4N1CIJ<#kA$w)wDiy zuop*qMx9Q^n-n&U1y-dPTlTJ%j&F{a6qKDR6iG(e2eC`}M!C8Vm!yl*M%yyIzN*c( zAV>RDP3Ywji1@`u$3Ug{Eu=~Cm=y1g3D~64F=bLVV1rw5OCLGW7)XxhO4EjSBNEJY z)#>z>hu{tgW7YffLjy3sF`0ARCICLJiSX>d#5~P;e1#3X{in`4L&XKn!cMH*)L3 z``ilYN9=F9R1Q<{nDoh6iKn?t+by)9rxct22E6*T1@qx7N=7uCU0<9GMJ+lw-haw? zbZ&R;u+W2s?it#eSni2MA8v&~cDI{Fp_kd}Hd+KE4*_8#Y#I>hH4Hki$Gkx{(mRou z@5Z&QJSx<|g^SYCCujApINOip=&HRd)}*B7=mzTGS!2sqbx5^Uj-$Gx_aXkP-_RUo_}#6%$*`yg;wAQlq0M-pdMFn6k*cy{1B;5!`mml8?6PPn2> zzee5?4XpNBBUcK^g3&1v<7cyhYa1eQ-d4g@1$uI&M+M(X%jdPXwX-$Nejm^C8TO)b z5o*TKQ~zdDPJFqHski;d#~Vvd`%uq-BXidK?|eWB$>Ut&pfze!DDAusNoBC6oa^*9 z4L%095QUzJS#$SLSiuIp$)~-wMHgKiSkn|LPL&PO1d3c?wG@LFe=lKg-cV)s+>x1 z#DGTtK6OUDkH}eVlapcY1|PxCR!4nD&gTdI%ZolU?LJxqty`;v)jXS%ySuk&ai@53EAA9`J#WtWKkNN+&wR?tS~HV7zdd_j zo0vx?-Y;}V$@Seq&OjkUYkq#FrOGz=B>B%=x=j^sO509CDnNCA)6CHZGl4r5qSk^> zWO?$(gY@N{>~aczIi%L3+*^$5oY7hktzza6dh93Ie<~|OG$sa|v<7|M64=<> zsw=pzSljv}Sdu_sAU{qZq1+vJXAMb;(aa8AocDTM$O=!E|9mtJeE@8#ABSO9`L?Sg z_~YU#1p*C6Et{5jBTn@3?~W_RHheP|kWjf#==9nw4isX{R;IbLFdDRi-An3n%~%BSHf)(S zG?Z%(V#}scGfO42jQRpdz7W;!X7ZEw|f+WE7Jarw2^&E|L6j?Ve0r5P80TE zX1iE~P5+@rYj8~U159SDjRrFYl88I{@20&BnO*wrQf!oE!*gQi4B)!C`}=I14_*ReTer@&Rqx-d$kb+jV6O zucVnGVH=K33vSlhP37#eje!iSD&1LF(z3q)0;6iXDxGj`!&j-B6VPOaR9kVCj-_Ga z75KHJGH+MRvq`y-U3=;@Hu@xJ5t^5X)N}NYM>0+z1vSJX?KMt4cPzlBVP)Y(-MoTB9ZH=}h z;8v3UqS8dFZO61(6*enwm{|$;xLPtQZuf*^V!WgKt2nz;!V8(45OO`qsb<2MrS%oo zrzu-%vZ0BK&Nk*djZ=;tOcqSF`ms3ORSjMgAxhuU$RsF@;_Xg#w7Tl?L)vo7z9MO_ zu7QkWT06K@c-vQO&=DY-|Dro^Y1^KDaFnHIjTtm?erb2McBkOq|Ma?ffjLVmT+Z*$ zMF`V-iC-$T=6yTPR9ZI1xjE5}Y=dsMSjY&eyM*9j1H$e^fGZHaZCS&}WbzrTB!Z7Oc>fqcw&N$as`S%#uQRm<6|%r=m?$B+e=7+44t$?= z8bmZYlA2-lijWn$Zck%l$*9+YU~+&ZDI)ZcqBYUV!$J4aYTK9=-`#9izb&!X)mP7L zE-Vty7t&z|)6h1dd?yh{b|6HNR7(ITS^RY*!JvnUfFhg_2e$Wpi$1i|m3ACE>|TMg ztup?;@K64*U%s;7$o$EbDH%@j0|7;uaf>F+*}=jOB)Hd$R7tv*(RtRmNr<0TajaJ# zIry;-zduL36@vj8cxsY^g#K0iV;`qawD9W_dCy20vQ%65+VxvtN-#EZsI#+#lw#YX z4;8SAhl~?`bw)0|1;(XD{L{FBJf1lj+|oaQeismHhqM_RES5Xp^91^=bId@g+!d%0y~A5N^s=j$sb!LI}p zNhB|9h}?zct!^J91rYdIg1U_4g8s|PuqUIYwWu&|FZ)p?{3{WlS zITcPQ8zUs6p*_hdh}G|oAFZQ@(jmO!)8w!HFo+})Qh6k58>9D~jNsi=YVD6B zR25**iUK&D9SWJ?{R2RH3QW{Mr-K1t7V(iA9suIn+_Crm3j7V2oOh5@sRbC|KW(Kcs`5q3B1KTVY#tqipo5RP6)dvNU5#kHth2ndzB5egs z`Zp;s8xeu=S`FK8S#}o~lOUi)#&-4+ z+x^PLFv%UqvKF_V^+q4!N8PCIbeLTaz z3OfG>FbVxXN_hVS{X~DI|JqHp<=JX`S*FWU_K}>-%%=TR{s9Cr^AJA-m>t6lMYN_b z+wZqV^JC|#dTpM)QC?AZ&ZY#zJ8+o-Xfopp@)h`9wFhslBkVVI`+ISX_k&^g@_E^# zu>oR1{;!vZfrk`$civ?Q1BdgP$HvZDu!t4M( zj)XQx`Xq)$M-@<%+B0yw3QC7z&5_9Wu=e1MnuenTXy)7$PmLy&U;#}VGb#rbsqI*r z{Qf_4E1X7$>=h98#esPownr5ii&vd!I8)RUXZ#6x6&9CEXIC+nS!up_x5W?bt%B_8jdJK&A)J+qyo~uE$ievj6NIlO=@`hUUlZ{TKIv4 z+tiM%N|Fwa2J(l~d)0uyBNw zCrG0K=z-&6n9GB^LKtaZ6SgI)(tboRA6(QHANsvX^K&mbw@+k&wbF8!v?%o>3Yt2@ z2?&%^Hj0v;qY@A^+K&b}_zmwkP2LK_t(WFBRArky^1&-#9!AVL7=iiYNapEg?XBN$ zk>vKtVRblL`eVXaP-o?iX<901qWG|-eNF6r$FVq=+!0xMsu%ju-h*0r%Njij-88TjbbdvVIPtWa)O<6V*aCUCgfTal;0FMA0Xu^YSkHMCtaA%W0!(t`{; zR0-Zl*&FHoQ#;BbfcYhM{7OS2o0c^d3cf;gEl`Br*%K*x`D9EXZ+1^?dH+!ov~DKP zN<*dUQ3|YNZQJ=8UnjX)t*;|xt$bb7QZGMCz~It=^+U@m*ISz!!i=Pp_zA;k(9NPk>!{bI#PL$ZXaCHj2rpJIhq`{ zvkuC3WNyDHUFb6h)>kxp#+$!TpOF{y1W^}_=M;^vloXdh8>uGk{!~=CCU*rCtk?~b zO)ySY5t|ZybeJ!FxaB^Lw*6E}e{XNg%6HQ1H(*5JOigSFQ;T$Tf9krl5&83lV+UOI zZLDj(J8|`yF|~5++c>*D`=r!43)CT8MOEo;A68n%uqv4{DX6jgtgExLpkb+{h`~$@ z1%2v9mlg%nVZTVrIW^42XgrNJG~h!1Fivm>Jq8lf%26|= zW?iKiQ5Vx_OzOO19RN0{T04HeaJH5d6ty&&sk2zxox|y|X z28Oy4Tj#g*DOnIS*uN41kz(!E`I(-P?@Pu*x-UNs#jBj&xx3xFQw(duGSygVn7{?r zHpsw|I^#?x?Ifua30HwfJHK>SD$rLs!f2N-^&0c_j9wj{^`C>H|5mmA-uKAtlKfNc zfX=59OPkiw5aT?yL5WsoPuWGBJrf*8YYj3SZ%bu}6MTI55!yO@_sx+SHF{%fPL56E z;rz6%#Fu`>QMqqjZNTy1o8}>RaN#P}#CHp+aHjLD8LdD1gpvm7zm$XZwRFC)r0-yt zvRlFHZ5qkHhY!1w_BMeu)OmzL3Nfj$@$VWA4~VM zEH2iUVRpu)NpsPvHkZ4~6r@7KAW*7YQ3yDq|I*KDo+6mW1J^&?^mILU5N*?j7$=3~ zs^{P(NH!IJ)4CP}KYJfYKCUr-E7^F6-&9mf@OPiBEizecx!yt(fwhMF1eX9K9eXta zQi}csw+?7M;jy~iZE&G4gKcakMEmS#7_RICYwHnb^VcAnD~@!xSScoA)!>`p+D)l^ zr4;kemi5mJh{1|j9i`={W`lJ(%q^_yfLX}Ixk~*MfsJGWj2G*6g(uH>V3Vz!WgM++ zK8(u6qz%ZpIu2Nu!#loc;%Vk-?jZijsvCL_{P}r-Vm$~pDjAncsiHW10$?6FdbpZ5 z%3-0x0YgetVpm-!E|$vt>cjLWc3k8RL)bkbKI-RABfND4RA1Mo+JyoyOH?Y-E7;T$ zyowWbje=-r463>CY_2eu3AL7*!{>`m$43DvoT!0q=U?VU9{}K#Jv}=IZkLMKXvOqz zE#V*y11tvEBAO(yCZJMEolrft+|GcQj-HT1ubWN3nA#MiiXo@sM4mJ1`sowzCrZ>5 z5kI^}k$GxK(6-*79lnus$Qu|GCujb}FBQ|PRY?U2^#f4@U9AoeIh^Og4&#Kd+q&al zK%47uR+jdK+ebF^nRg4GLOX?o2BpxZun?r01o&0V-dze&8_6$T>5sXsZx*JBKoBQw z@U#Jm|*3LNK z{7koMucvF9yaVrHKS#s_8lyM2V&_DLIXa;m$e>!FS{*6Ojl9(7R4R)u<;A|4%!(u` zrPM^dtkMu?oP*u#qviTA?m5DvR7sJTpr<>$>1=Zsr?B$Mn9gZs!H?^Lt{7QBb*Nq} z8o_zK%HQ7oVVii52i4;sRVC%25Q?Q?PdU<%2MP8W(hR~qN9>aR1EA_Du6J^E6Duog<3ABf*W}tpieAiV%`dGm)#NFx{8ccnm?p62l@9KKn;(#+iO@v9NMG*1Uw?|U^3?O0{n%SK`8 zz*S4KI2cSa1?b)hjy|?H9);3RMfvZ*vv)*GIsR@t_faAn1QZ{FPaFe<&V*va+uw!> zXnQYT#PVE-L-BSWoP-NwAnw<2M|4;zs_ldRqSHiT;hn~8&lhXHKb8sazMxe1w{TT)8lFiz&5rTLQKnQEqMpgD^BZe;TT zqCf9J=0FfRcb??umk-a|TQBf#^a8yGk^KH4C;G`8nXxbJsvg>@ohVF>_Mck8xfAEQ<9 zeVTihoe1|5EX#Z9JKaMP;p6gxg8U@*DSW2YcHPq2V4NgS1LT zS5)$JYWD}}|EMq+v9+}wJLnRi)qVQqRsRx{-h$)Dc%goHW<{2F$=3h$e-PIKUU!|19Ue|^{%_{zWG52vdwad?J7=n2g>sIb7O8L$ zMc<#E%}y}0Jji{erMcYdJ@e@=B4Q`wLBi3U{$e>&n}3=N`hzxwfn10v9Qp7xg-jX! zGfI7RYpC87tAcBvFq4`8UF)l`49g8-}0e$ zdU`2;8dpk(@nzxR_;Bkt3Bh(laJ-bC8mC=YQ z)IuaXNFL$B%+SVsN8GGjFv<&8e(gN}?hHO8C?}`&w0g~Q-tUM-s^mJ2Wn_v^~GVZP(EAZ$-KxU z_Ebt{umFmhCwD#dueDMeJu54G#_;NURTtI=2?26FGZ$n1%;c zmScQKpE-8SK2x3wE^dHV!XxToQ}k0oc2@)dSd|=n_I#lE6#{9n^si8{XM5^B`5TVV zXP?U$@16&bcgG7)d`IA5f+&CTzl5Kdq7Z|*44&0u#d9azL?WEF&9a;0(61|pPKy@~ zUeUOU+8YZudws!4h_mATMI-eyQU#^dU~ti85`BSim(Tm)_}{}M>p|2rTkK|8I6H=JR_82H@Bf^q#>P((lTMLj{$rG8iu+hO1*k8xSH6a zZ-MuRP%S0pFYj8qv7f0v-j|kO8%~A>3*Ov5eSe+92ix2G=lU)1oIN~OW1v%z9l7*xuF_ArPyt(DPFXZs$Kk7J+Tifal(GTic<`SV0npMS6KwG|gMFQVYj>%0Y!~ z%!L=R>Cmn}EPU8%`wORxvnl&4S-NRUwbCQ64l@U=3=_fXh^F6c-+qo`x~&MmuyH17 z$MjlQQD!7_Val>0127NcJ^?Vac&G%+3o+;E8;#STWl}LZAccO*x1#ayof9kx$!WW4 z&fY^Tmcf$FAB!Z%TmR0ZxF>K@j7(z?i?WwZk|;1#T7*F1fvH zOd$x1ylPz9dY1=twfH4%zjjR1gyGkrA#wx9G$d#7FLC99S7ilcrQ&r=vbQ-1zqnbI z3kj{S8H}{jZ{uZsUWqW`<{#Ytl&yh4e(q7c9@FICchw?$pqYD+f>}_gJl|msME0<8 z#th?6g=v!C`$9f5=3+CrO=FHD>7cIO!@hs)sD9Jh>?1LLBRr1UAbzk@@I(L1*%m?m zn-nXfJOka4PI6FwQ-&C(cULSvlgL@evd=2dw?fM?48|Y65sbEO4Ue+AO4ZVVm72Lb@g74vuVL3{ZX5!Zo&^orR7^3oE zjq0!(`s7NAfmeiI1QYCg8-0|?3_^#og?wo6L|6@*@6jX&=7(lO23eJQB_y`=^?qt? z&z9q>0g{m6KUI@qJ$jc;cYw~^r~>xWzN0@4fA5?}h^VA+Y*e&q(&0mJ)TI5byeSUO zDV~Q--oRxZGeI3mTEIWKJ#54YpZM(igcOllyE3B7x`|ZWC?;W)c5kOPmNUWhgD22} zWAfyu2KxKY|8jpxjWtW!tV==!R4RZiluL++iU^) ziJNp<#k$eZvYG4IEdTX&u7sw77;9)L00fd6TX%ZoMp-3WsyRo=Fs}YY>!qbcM+{yW z$Hz}6wMwhkVhfT-AN1O2*%KTpQber&J)HB>avnRdvPz;*j3Qrb z^a%_K{__5r_qT?|s#dOYDH9`s$O?9^LkrYSP|9Bq3(&~BI3woqxiGHvOBXrlCrI1V zs%bErI{psX#s!1L6Ds5t6lg+V#l#_qRO+rCt^*a%xT?l#7rW&*CbEoyH>zK+V|i9b zYz#$K;t>u>RR;xPx!eUTN$MAbPm&@A1k-CUY<0euw56n$_Lo$OzZW95uzDvl+Kwj~ zg-?=pc=IMEjI+COELU<`;)az~@d8%MiUVwZZHStR;hDtwcF(qb<#vlmyVvaBwzXqH z?^L^Uvkxy+X_-Z@?gV-%Vk`RM7nKx{+UEaBc0<|B7^2VwRUizvv{%+MdYrcT2NKcs zzZ;;6aR`BOEI$Sxg?hs-`3G zW_jI}DyUpCLlFeq*L>nv9W_u zH!_S6US0b%^r?&Eg7KIOX;Uc~I-?{K_cbaUXhnetGjW0DpXK}ODzo6u^`{Rf8C3J* zB!pd21j8V{gs@?6)gOlBc9=xAy=+cgCt5kms-T)-0=z(>3QM^*%5Q{JU^D`q@gM#P_h zEbvbvV*|=RmSnr9HlNG1x+ooiyurCLErS4_S6S8AaVy`wq)H(qp{|@X$Fz%I^qjqi z8bOSoxF%85;dBZGgQtvB&z}|l7B`2*}d;B}gWPv{o zMmI;G0k!?iJjY9$-tY4Q=}p z+tE~>JJJsm+yxha3}C<)7}V2E&32`Z>chK(QtAwNr9)bus(8&GP8g_(r#h^5&_kJu zYLwkFW2raK`eskNk&F-v(QT{1FJsA?^4m7FZ;CP&z>i?k82-!{$#(VZM8RVxyuzAK z{o6C&4{G#?NQ8PM-zG^P)`G0vGpDo+ij`&9Uhd4QB=L0;K5AMMQ1^3}mI|(^5VA(( zUeM6LEo;T;vWzlK0MVqa^i<4KZrJaS$7Pe9n z#iJ+zE8E&ZGx)O45*rc&HIKV=y)GPjK0zFQ3sZEL)8R1`KH&goX38qC{M1Wg*Cx#+CV3m+Pf8^A z;nXLlA9D{8&T6TRSa9N$&dure2&_UFX}WgryE7-fm$6(mY?+dGd>e* zf*h_qq^bF%*=ED0S@w5}5Z^e0j_2%ruDp|Y^R`C#hiou-E6>0F8z!fDa(GUc3TyG(4KtSgu2`T_PT;_TewX~LL@I$@IKioZo{I4JB# z98gALnGnlF;X7E?H4FY{N{O|7KXS^kgWgTcY~Nid8B-WPGy^-i14#+X>+YIQYzS#Adp$gDN3V@on9vYOLD?KE~judT&meD?KfURebh>AT!rRk>=(Pv zKrFwYHQ^wq4u@6NMT;x&h`AfHu}#J=a0rSdEsz#uslF%~1rI5*-b(y%GE_djZT%L( z*;vjxQW2ECEYoFfx*^Uri*(Ude?4f!B8luKz8QeP7h>c47cDQ}1Uafeg7-;$Z4*~o zjVH7G zAHl5uPDbS~!oNO$+3H1<+ym@KEMRMkB0B8seROc5oZ_17$llexs$97JfwMHedAPI< zH|k?9ZJ=8qvSzoGgN+0gEQSdPp@Rk&yd+Dh3bPUn86TBu?-T+2! zF(Fhn-d)n~n=Qou*$2ek4@64>fU>Ai0Cvf=e5d>2i#ZER?dzM*gJR^{H8j_fDw#k- z0?8Dci1WnR|Ci6~IZj=E4(_g^UW3OWaHDtZX@1>8$DVJ*+MCaEbu}>Bj={wm^xX=P z=-W3zj;oCqxqtL~%fUg1gU7xkz+(p_E;%Kuf&1D&6cs#!DD zAef7XOrJVseuHm*x>%YX&})a$JqBhFKHv5aXiXTl3Y`LYH%nejqvV|c{$?eP7J=Dk z6eRif*y5=ceh!z*Ci|xlWb0@VQ@EZ?3eBh`XgT8lCqc7Pe@);M@Y@mim+cK*dM_&) zPj7KZpt5(TE(%WwS5L7o@y#Wh8zw~+GN z_#Trytt$vdte%i2f!=r1sRR&&_@L*M9(B??LYz zP$;2kjI%$s#Bjf;?$cIuf}#HW%iGIypE}=>==*MEo*t#gd3}G+lC2%yhp!iV0wzIz zP=p20Ud*AZK>{P6aP20avMoOBZYX5oX$W1>_io|j>kq%^g|iMs2%Xi22@2w7ka`5` zK>f^+XYtVo%o3phtVCS(?h&W;prDX`C8*H~&u%?1-OKj?LX28@2x>K{)~ z+shgzY8tN4KY*+Yxf+Y|_a|Sh&tgw-Ux}Oe9qAptRiQUOOGifm`H@ShY)ccVlF7xz z(;d(gE5H=9@1kB$P$s+;ZohFqD7B%X;Zh!zGz#ypME=?)rd+4cyX?R86#vJd#pK&a zpdp3HH;2n^8l!lx30V=!sTxBe_2u6Wwu9?}bkWG~nb-Z$+Oa1IBv=8t^!3+96jsMG zxhCw-*^bPD(@5L6Pc1naYgLK+#saEhXvkA_5aeocO$9)(-}~r-ey~!ouKf*tM2T6nq@S5g~zp1 ze*v?cbMZ%_^VSjh5$R#cq8AuYLwD`;_+iwJG@RQX}mg2 zSE$##4+s9ru%>e)?TI}c*Ig}QE#5~3phNRWVHD!a_E#$_S9@VQ{8Aki#$_juUx%=iApVW_X5#u&_T4R^)1g1^`P?gWZX4j*AOhr`B1HM`5; zM6pF@ysY&4qKHS)0HvMWHpH#m(>MB59fq@Uk8&RSKr{-9xP+TXd-x=?I^8&CU4|&z zqFiBMg{^m)YXw@AjpPrKc9xy_R~Q&67uPXiQP%6h<>tU%ICA>D)NyqUkRl;IGn1!7 zTckO7|xq)9ojlrL@Ew^~aXKcm6EHPsPWdTgDe0`GskKg-e8~3F5QjdL(*y zc+=#Sw_Z7<1B|H*`D>%!b9C|tbbt=kJ*psI*T27e54kvIOX28o-Wei%(=jA84UA4e z^1FoDEZWQ@;bGMq@^p)PnWur?$3Nr^iv)04Dll$>P)D*Gj_&w3b-&=3ZZrj&VmizZ zD}pm6(fi;-8jhEIV6hUBK1BcIsp|Vvfi>~E>(T#WU40^97b>Cp%`gc#5o+hFG~v6$ z(1n-jqkyU$@|DqHQlo%X!{?ZA1Iju#V#M63@i*9#eU|3ZD1rpPv>{1Zkwykk(~fpX zO?i_Iiq55+jFao}Fx!A+xT-Wd#>&WMNx><72I3zjc`Ex+J{;#JPfqMXf3PP00XT2d ztT!v#neT-t7mpcdbzrff7p~e17c#Sm){}(~0Kz_wsoCpCnGGMrO>Nl%jyJG&V@PK- z4|!BTCyR^>S_A!s>0|~LCf-jzu?!BxaFn#153dR~K<-FVT?xtA`c+;iBE9ws=Rnvn zI=;_6Nw&u5*PNGwwVfLEas3Z)7W!HvEBsDQJ#VoEqL%y)!;sX zyU_i8R8cGyXT!o>E``D%snemTG0)FE|2yYM`RlShlTIxAce*uQwgT2Yjs_$L>JmSM zO5enX?|J3_(F@v zj|2W4+qB1|N^aIFTn$?}u&DeHmj-C`JO+mby7X|W5PYnz)!hGTNi+Yz$}g#NLkk<1 zQ>!2kQbet-E_Y^lDuE?Tu!fXYhPY1uZES3%v{=$x(PcJExFN}f4%UkNwY=V&Pej{7 z-}pnayi5v>h>f%{4WG$AgPsLX4u0ql&dzg}BH^C8bF;LS%^*k`dZe;2I`}q1g~2%( z^?PfE0?De2=heG!pVvp=Co*Hye4Q6pld>SaUC43zJLKqvqiwmbS*6dP%vQl^i^h`r z%G5rkRtBGR^{`8;3NaeSn3}keg3=F|y{s>?*!xon1fzr+A2srD80p7N&Tx^bf{EZY zEt*3$?iShgy9u12WssU&N{kijot}dasSrA~Yf4Ym`&LfQ^tPo6(5yNWR&(+00&XzG*EA5#!oj zAyjSNU|2{Djb^8-+F%Y)j?2a@8 zK?Orts!2yG7EuJg6(V=$#%wB=rpAlSx@g(&b=*1}I!V26Je6MNeRULDY}T#Li=Q-e z4z%1^8p*DkK>f5ZEHw;m1kW=`*pjWlsIRade>aKPnEO=n>L1WGFrfjl@QVt9+A}h| zh80Y)ruGdq=CrWrj4+q0b?42fZ09Z*&ko$*b39u1`&taera;Ov2mmappp${h4qvIJ zaCk~jYQt5wQ?8O9Wp2yRo5Z?7U7RcJ*K65kpUs*Y+OGiqEKhy}v_(K{cGWY{lwS}S z53%JVYK%)f>|7_;-6TOIpbpFJ{`nxVS-LAVaqyy5{Mf_7=&Sa-s84pitu%q!tQ{9| z&~RvQx%cTq^Y6TQz((Txzp(MEY+U^3II_Hw{m(^F=$pE1FKP*E40mR#kbS zBzCUdYzienLS4UKUg;yP+yG4!tjnO)sE%buG;+||OMeXpNbbrh%r_e=DmmGb>k6+e zK8!gx6$%&q$SZ@H0q;BakgHqnD%omC=MhH5q31bMg&)aapZC{)q3 zA?f?N5O;)=wVJNDBjy?o=>QX6<717n%U2IlS=Qk_emAFL99=tyz~48kfh5~MM|4iK z$|_HA@sk0S3gC!@1Qb>fM~S>nCVWVoS@=LTQA{#(OfHfnM!R=A>m(UHa3DHa1r*da z5C>EiNjNv2ppXSbY@j`=Y!iJK1?+4r-Z)rK`KeBjj&aI}W(IT<75dLx)$=0HMY{)=!ln>WZDwnf{mI6RMUJ1mxLk$w_ zIQmLIA!3;6eHAmMmV}ZSnNp%MR6@zfsQ!MAk|YjavWs%stuS8>69NMWdLDZlZDsbm ze5t&68ad=bFK42S>~Nb5qK_Jf^$iW`afulENp24O$&0uadObjROD#9M7(v6K##`=P z_Zy)Pm)_!3%=`-`La%TU=ZC~8YFw3y6Bd$C;d-vlo7hO+eU8ls`Z){9jSt9L0tnDUPl6gAyM-YlT*r8Ul z@bZ1aMjVdE+Zq9K~zU{&OF;u@VCg zz1CWZ+_m#jtWCuinl3hnZg%R14W&1qApie>twr|~5Ay#7RM52~k{rD@SVfHuwi~%+ z^$rh^mQKUoSsa?#x?c;4>qoKj^Nvt5IC|I&Q7D=eI{>d04_A)^L_yaNM%{9|O{l`a zXaBLDIOErb&<)+EKy^Q?mlUX71{}iX6m$rM#P}ROnQN9)-%_9A=(gRyO%V`^1VSib zHVSy3Z$pCg*7k+Z+SnbABZ82gy7Q6KF*lmYw72{HSIMR?Hn*MPNIo*eoK5rrZiGG% zhK#YDLeK_fE>7>D&EQvk$PYJ%CTS5LALif@Mp-Oey4OD=?l@%9cug3Q<#V;eWY`rQ>fM`+L>0l6V5EZP8b_;l{*QQ<^Ow?8kh5!5O6=%TjHy+4`f;`XLcd-_-S>!-U z-A=FaQ3+fLDpT2|&D=6iu{D1Y87I|nfB(n+_TM9N)h2bX`BsqP+xVc${^I|94Y>b1 zU=9?{47Q{gI(~7Z38FAmN)_sQ&kIb9&1F!e{0fmjNOs0jcUT@6-hF_M@KL67t^G*U zuV~Y)%-B*xqvRIb)9?(Ha2c12=m?PlU7Zta@JcK@Hnz}=H3#9^;l;!2cG3RIyAa;& zLOpMT9wp%^Jd|2%5l$om=Ty6s$M&}Q)blt*)3@e_Hxn=fZC7K~427n0l7ZwsMh(&x zRskO8Z@|&0hi+sKH%d!C_v$rnA3n4?Qi&di+$`GuGkSrQ7Abv%` zPIV8A4=+CD515Dg;;!lI8si6be3v^CMyjL&4YeK2+(^B-UNTqj>9g zdf+>Tst)At9QgNlT;zY#{sGjo{@Fx=v|8OM4@4kUQ3ZOtMx&NhK0~3jdNmQR9sDjL zYbJkwY^>_j#Ew)9CM+i<2Aacb;G<>$AzC)4m~ z`yk!ci?>VZq`ULyK0=xLMgNw;f6s^P)!_@HSO(EJsQ(n!va}vu3PiP$AirexGk191 z7CMy=Gs^VAO3H*sW;C>pphpLVQ%wy_>o~qS;3g!S3Au%|rq4V%b(%(^`d9;FKg;&Y zjbHqo82#77GR*6PXgdmww<;?BgX3n3`Ikhzb0n+m(3(Rur{O5BD7E(E>4xab&}1Wd zzsS=YR4Y5f4Niby1m}N>PwXS$;9&qjALX`XMe0F2)IFjCON)-hN{$I@;;Rv z+V_p+QuUfRkpzx!ex zT_tgGsZO-~(ByA%F2-SFy!E_XPoFoa%=(G${jfCF-xav;-a{y!I`y>K_~^;7ewR5r^IwG+&ce-f9Z`HSl}$Y*o-X+tqiQY0UHJ$J)pA6W!i*M#{yXzY`O{Q~0 zG>D$Cw}Rf%wgB-V>lU)4N%e!Zw%k|P{R9Pm`I95?Em@tU+ zdWsp83gk0D{26Q;LfSgVoKoE&P%}VF(}LXmu}1(>NdP2{3DqGU##uSUw-Oe}owb}g zkj=`}W=)DoFtX;KJCaSCl|P59p7$%zau(nwRnRw-%CK6?u{fB&v{_&Jrlp23H>2r< zRJ|<}3k72+1^5M(A(3m}Cok7I40xDP{fMA&NTWb+*~zk}EYFaNO3_+AoB={XOVi^V4^8uh%3>W zLF+hwTeMz(%e{ndzNaT7W)Gm*Q|?pG68>g*^)Lbnb=Fu%kV--+e!04oDG*4EE|+o9($l`2HJsq-Wr2pcowsRu{ymp`t;1l{f>NGzMDZs7A5mu+)Km9*%xXR^xfDE0j4HV z^4Yy*cNtlC@_(3eMF0#2=T)(_obbh|Md~8r)P0u0q`I4vgF;kJH^@1xJ zm-mkxj;2jE{LjuXcN+@24*&BbK^ODbjHuL_D7d=RyAl_*YPi1UX@!?y&TnnncH5JS zd6+sex$gL^O(og*xz>VfidLm;OgVXzSJ@QyH(6a9^$`dI$KmtOprn&i?+bZ3@91@pM;IN?P1tTkC?B3|f?S8#S zz+V7vk51twuh+`N%@5pJrUW#0QPoOu{zv+pVry4{TCe$DxovvjS?zpuG=2NGZ~<*< zd_ERF9fXiwK9?%u@N|I=kbt($n;dbugUXZo{oB-L3uJ>;y43#c(=8m{b(s{yg=y zv$iMfHbOp|=4m;E^aAVbhr5+mw^YBHoM!)2MJ1(2V5()TcMijwh-c&ZB&ju-^wqad zO`TooL1QY32?zogj_V_9pewRzk;J+gEVKa*N&W(&9`31jHu<*uNE?G`S$9;a#<@tU zQa@iRluorEdK?-{DGDVEk>lIb)7{tDOWL2T@$zUcQCE5!uvXkKkJUKB1+ny ztD4%8J24X*At4`{Etc$Ts`^tLh4n+JZMAE(No&1*+$jFoI|cX%lIV#U?oOelocAeq z;<(0Ldq&QKDXks{zhj@JGJpVctqmhH6B7Wz3Og7o)Zas&HQ!4d z#Jrqs%@T1kK)cQL$m3?{k3x*r%KcW1hYot{7wedv)n;rcbIn1*2T;)m*L4eh4w?f! zM5cmC!vG{D$Dp^X#z$~4`X{trk9w@7b4!<}eYNkr-a+-p@BFgQ&}F5t)=8`~IE+E$@9XdjJthWV-dfsm(JxL+X4v^AILXJoaZw&{6w zfShUZ#iD-ZPu;^ajrNH;%hzGNKde}wdFrW0$66-=3G>uP!xT+A=GyFP-!OVi>;poM zf+4zYBo>HwBh{Qe%&;; z(-IF#0yBfAk^*aa&ES^~JUm`}7KV4&EO=D4oYSu~ALh-Q1UTeB7YoE!_^{no*Kx0= z9Z?P$WgQ5%2Nn=Z&zrgJ!bM`pgi&e&b#}5yM~-)xbJ^_qFClucZ15-?Z0~n$IH#Kw z_)C0GfTAjl3N|F2em8fXA{WhF_r|qo1`VJs7~5p=JQ0Grt8|&~C`Df*EP^4PMIS+a zAR?{h*kFGLlN8%Md9IPk-Ml7;2v>u-{ZNOIGY0|`Cm4%QxX>EH8X)1JWi_R`8jCul zF~L5trD)=+v%Q!0L-x2=hWGb?3pGI;4H12E+P|vmqwDpDwIYW7y+}Ee<-l9>+Wfz z)ho%`=|Vn{W-Ox?F`$&erizcyR;H;MN`71K>1wyjb1L4-M)0SRhD}%-(pAJuu3&BU zrjwQfa%3ixw!n@0Qo)jCER?Dk<~}2ef2R{_A1E6!^hvY-3!~D)=Sd^o3Ay)kgsxw< zWI_^Jrsizk#gJb4vDojp<KB}4{bQHEu8V>Om&;3DXwQVvhv&(}0rK&4ec}Kp5A0cjF9J2-u z=c7=BxemP!r}=kS%vZ4y{>*|hVAz(bLEQCGH&O+Az;e@L0Wv`<4&W`ll=Q6ZUED79 ze|s@yZ2tn36W3g&1fRF@Ej=AgF0Mq&(|q{bkWR1(=u-Vt)=s98bIi6&6YrCxX(7nh z;OMHC|LBM5pg9=d7mxvB>8S+^s!Oo{4TLj|wju8F=@g(&H%%)JGx_Kx$|1okL>l~? zcjE`iPn_-M*2I7aWyu34D~|!3dhO>PMb|7d)^DIu^@7lQwfmuXw_;T>4O`@`R?>^P z&;XKB3oPiWq2n}WxI1AniePj2(`GY#RVj^13E{uv~WP`_1GFT1U@7VVq=Ly&{kC;{*OD7kn;b1!2^z=jEM=Je1>l(>dV+1NfI2l!lk0$LG^XomI| zAO;qpX$1&_dHJezo8d)_!ZO4PqbAKIT)t-8LQj&Rhgq?#^&HKp=d*=j&SD%im?QKW zbl$0A033=}H)-<(BrxY~x=g?n?Z3+*vv8;K6#HXf1WCniPa;ts>m+kY%O zr4LYDIusOnTmEQM1ogrv2SC4hd9U;7IhRnzA&qk^A)=2kF$4JV8UMl_5r~IS84hLy zs<;*=koMOnPMrJa(OKm5RK=s|M&r@ZKRbF^)J0SCYX*IvPAh)Jz0z(S|0iN&{y$no zvg`IW`mJnj0!tB-S#;wsVCSEh&Fw!P7@)a3i!_#U3R5e#i!0*&mLHUG;pl>^A)=O5 zAWg$X_`a1XgHiSyX%yLx$DzBg@m35+1c$O5Q)=zi4Mv-MZNHo#5{)WjJ}{mV(~tMT z@MKR$AZ3I7rTUL^wei{9k`xJt#%BPY5?&P!z!&j}M&|M9SD0|l6-M`w$-%;Y z3yHQcEc#XWi|mEeK!VgKL4&2c!3gQq4!;_qAgCz+YsUH`gxb6A#wvGMPO)MdmdVUeVYaJxh$@%>LK8W&Ml3lU&|zB@XimKRzL71To&%*pva3Ikc3!|^78TjF*?C<1%~qgKxx`eya^ zV9~HCX4ySbFof@N@k%B_gbeV)3FiSg-Jh5y0XX2yX58Qn!HM^j$ZKe5n0rEsw{Lmi z%)m#ej^sYr;@=g~6V#a(n!XCjrjWE_R|`wh`}x(QGF9?PHLrmBpxzCv51sm;}q z^PW#Ap?O$6cvAmYeyiwz!pG2G zSXG7FG5vaOei848GElV}px0%2Fzhf^1jFeDro*;Dt~=Jr6k9jl8RQfuA?7hiuJtDLK( zQ~0o~eZfRkZY^Cu$}GdotoS_mpXC5;fL%y>AlJvGJ7M85<B^;p}(Ofq5UzLZ0z6(VY-jViq{r%jL$C3b=9s8d|P>w$DamZ*REC&@cRe9bTG zMqg@K_Cp$|s$=#VAt~^U;ei^{dNbuewY6(MC)5tGQA3(QJ6>U7*s-8xml+_)tab~f zJ2;J-(+wOM$y-PpI@es9R`vEo`k02|8 zfDC6y3WX|S4@aZbJ;iCARB?EnT3>#>vZOX`C%O~Y9B92y2P;!Pl1a}c6$k;(Ym>;C zCgz2Yju*!&3Ex+5j=4?k znK8$v-VmZ$xTLPqZ$kJ~e0#3U>OEuu_btTfp6GqRF|4uVnoA2WUI34u;VhptKjSo~ zqE_h}oxOw;oBqj!JIQMFm&%#aW|guj6u6pn3mMV~&Jc%Aoi)t@tS(QV)wHCEiW-l? z;-Fdx=C)%H#2C(MIgbB*YER-?MRdv#exVC3=c601O|T08EaRtF#qfDdjXfv-9jgzC zdHcH8_1bxKh;`j9GxwBy4WF)aE)HDtD;ypL^;HtIr32eX-eN~&HfFKEY_8ru7R zwXTjG77yAr=XV<6?jo^Kn!M0V_tSu)(M&m8^erLRMY`xD|Dmaog{SBSb0 zYXH)sN5@s=q_={_Ms14QW~SbtIVXyJ=My~GaKQVX=M;{yWXgVmcg0yuTGALo9OBWc zIoGgi;Ixy50aDk#-~@*Y>d>v`ENjQI3d&EDS?!G#TLb<~8*1D< zI&V`#k3xQ%kC44HEV>IH3}O}lb54?N-+egNhA@q<*5G^n=u*DoT2N=blV)oGI#+0w z3q+M7e6xbHHfOrSCL-l-Gl<;3=rQA*(~i^QMvDdDl5wi|)U};%sa3m>l^Gf_rxV%4 z&s6C?`UYlid1HjN9lLMHvepbrXc~Yj9ttF6;E+iCAN)_|3bnWIwv_Jw`~?t+!!Q0+ z4{=VYTB*{sJ~WD^?K99)3n-_NG?=JWGE2ofaYU5@7|79jU9Kb%Qe1bfti#G$p~hU%K~N-ykG*CGG6of9A@D4-!Ka5Qk+W}H1g}la zJcN(=wSSStY2U!{xbzmmE4<58E_s8rXmW_>D|4rhYA!!4%Zb9sW= zca248?9FG_s&7lkpHDQW#N5Oe-d*C3ZZBQ3rrv!zFiH5TJlBjqn?rA`z--CB>y+uP z&0q-nV+Ybu(tu!aNfCN$^XY`{0W*A{)xb7#^S#mWVj&tLt(^*ZaE`aUw%uoFh<=Yk zrv+_J{fWmdA#}u9pbsqw+5*qD)gKJj{awULj-%)UHZ(wr3a0V)pm@;!RRx#++PCJ4 zmFh|18XJChfiJDt2?QlPA>Q2>$oS3_BKfC0Q6<7R`=(Y`*P%nQgOnxRi!)23S82O4Z5N~63BxJA zN-`-ZdSaEa?Z=v4bCyY_&SA!-u_FltH&igw-7T z1wlkm+pJLkkFQ_W5Z-sA`45(O!#^aD0=@BJ)8*>YOXFV+tjlYFm5U6e(e=iUunIm2 zeuZ8OjS>1*$FdTKCbANUeI+`AxUor{VD2fR1-*`qYJaA)1uID?H+K`3S!`?9FQCC0 zWFBXMwpA0NERvnz2%6{BAfYId@#<5`Jy=oF@~>-SE}vvNW?BNCK$AFm{+Q)WPh`%L zc82U>>a9%7ic(tX%Cd{7=*;)<+`MEQ1pB@SpO_qSKl$Y*@bd@1@lmacRf!XAH0%YJ z-toTW-?kD*Q<#rHYp@fe0}W3_!@e6kI{Y!E-6_VZ{=t1nf$>!XvaUF+Kl{6gTDePe zs4KmDJi4q`n5D*{u*>Ba{Z<;os4HgRNEvTA@$3N7)$R(9HB2?F994LO3bQ=xC8gtw zo~AWc9R#{X4p9Vb~#-bJm@cu>fY|KU}LK-nR$>B;b8CQ%T;4XJp#;Zoa7T{ zYxn?Ms2_*al_cIzuvH;JZd zem~qte|c#;W^mMR*y$Mv63dKYa;a{eI~Cj6^2uZ2nZ?GI!qYeT1S*BzcC;Q%8d#Y1 zK4cfuoz%PZ7DJ%+Fj1?$OIWm2fkw&kk>SLL(Y)7+y+XHIqq5F1=GrfXE49a8m8Y5K zj5f{XbJ)Ax>tbu11-8$&@g#i7MvrgzUh-U+X7=dqF(S(nMr26Fntu(&LMnM#ye z)(;3WxAO?M6?|nkC|L<+KKVH@;)zHf9Cz!am(W5AYd@aXTO0X4q_I^J4ky%?;d_o@VP>n>^bd zy|eViJkXdy`aC_AB%*fZR$cJr#IrP)4LdHKk}q#rm&ZI@*HjkSbikjeEQ4(Fe zpbvSj01E`EJJso_0LL$fO>5n0-Naq2Za8O1zy+P_FJPr2r&*R<>`LV}cy1L5XXauW zLu)<)m&6%TBGg@uxS{^D5*CGBV8hIwdZmDryHsvCAU_nVlf9=j&wN=|zu0JMF{AI$n*SSNd{E&Epdxyz9 ztk%Xv1=>yZK_e-c>xCwOMe1PultLJ5ICh}9R)OMzn$2ZD7jub+ z6H6Tn!TsSWT^Al=Fndaw0|L-_*Y*1vqxE|k8q0iV;1@*_fDX}f^Qwd!!W|IGS!#96 zADRn8j=yd!)tRsh;6MX!)#BuuFWVCUZUDQ>@0Mtc5e$^=vSfj-T$15ltj8gm@u81d ziJpT5!7$IytBBZ+OCu39d}?ZG;`6t0N`%Lf`=UY9kj)L13sMTODD^D#s6XX#?yVG7 z4-mO%i!G-E=##Gc?j+jZk&SF`tKb;KLpkSDT{h~&L(JK>MJ3ljF^5(e{ti<$ zlaOLR>dMxwD&qjpsCWgqlI!Ob#~aA|O;Ocz4>^SQHmOkCi5LdH9GF4Z=M7 zSa?o$>dJHtaiq!GIwIf*3+Eib`nC$ia={w5hZtf(jTaDi7g}tZd)No6umE#3UgH{< zJ`)@rG4FkyjoQT+Y_G>6kc^R3%D2CO#BKcNIiRN6L#RRrbV23tcGvhK2iuTR`efe1 zbN(a)-GGE>5W5o|i@m*zlATtoj!S1Y60HtM{>6Q1tGw)w7O<5&054~HBp!4j?Tx~ukf{TsHV zAn+<<-ej4k>v})ZEABv&yAPQvEG%(awATkLUcOyBqp_Lt-H!i6Ey}71Dmj*a?4Pg3 z{Uiu633XjV_4er{-EQ-p`^pf81OMpt4Q&8M``0t2#^guC;U_Nvy=ZOA^?+oBnT76l z4tkGkD$I!M{-TEc2vRtWf}6wrPKq)dDBN@X(j*O^OLzJdc0#I_P15N1IQ!53L!=X& z^QF4 z@3Ke&#b>q>;6gjMKQVI+u_z9@my%WKoxXfFx`vG1%V7VzQt`sJDjC`J_PiF;ODZX< z=bwRT#g<98ksA?t#i}w*5s`;ikUygM3($;D`**wY-wINX5RM2jDMeWB%^;HJBo=u{u?aHuE0i8ReoFZW<2fG` zd!(vXuL-3`y8i;iq0hM%A3`3@V%Mp9?ISOll^EO zIwv)-=+AhBo@)w+9pw1*kbBD3(P=F8ZJV;qDzy_#_%Jpya2E0JDM#G7rswNN=hMj7 z!=g`mT&&gP5D$iX99Pq@c8XF6$&94-Q!Pa*K59m>E&ihfBv z_s|{{(q+nH)0?w+bk&SG4Y*%e=dlol4ryFmv66tv*C$V?nFbloMM^wO^l<;eA)a74 zrS6`86cUe&bR@2KANRDw=^@(|Ts+#)zUAlPJDc+Jbjx8bc>BQ1WRbERpQdd4muc}< z!w%CCjlP@oMavcS*~s0~CKKXjf&Y$PNxGw4&0*k@4*M)!`S)Vm(5e}XCh1R(G&sk$ zzr~$!m|5rFzzPgq7m-#Od|Tr|DUc`gUmQEMHkkjca%$r86}e{ZP*4w~HNTdhy_6&? zM*uWnXDv&>g)fih_~#VOdcrZ7K~OEv0Q}U*T}MOwRZ7$rM!eCb-b#(Ji(`wu{O2^L z-bz++%FvW)c!5+l!+Jq(Sa*R3`@_21BoX603cM5vn>85M&`pjVql_*;59zd>0O26# zC)sZ)@Bu+ql-c-V7w3;e`hNjmjVZQ{DL+6Khi8S*p5r()vkmV`w2FFOX?{h8mHifsU^Txz>B9;6ebT_gsf4}R zCgZRQt>Z?TD>PbD2=9ofg&n-GY?OnLRODlVjwp`%7gasestO8ABD z+ac#PwpP(tf&{b2v4e|Z38e%NPT-5=p2I2J7cq+61Rq&B-@)>|t_woLq&Zg9$~w0c z!Z(X#o<>&po!}}!MinKCIAO>~Rqa)#l}%I$qM+{~UH}vrwSlB;G$`J)`z}{TGn25|P>OU{y%4{XRxYKcdSe7k?VxFO^(%Sln2zVl#xu1?`OU z(ZM+Yo;cV8mh^V}a)o&c*VuGiqcc%2M#k_lk!|o{>M?36UW&&HK3Fr-Dyl^rONA|8 z?KyuRBj@Br&dC<}Y|WYAV=@ zuU@4%mdt!&ywoHzH9$8H_=N8;T1^B8Fc_CxyTtZMnN67fNTR1Ep$?MIPe7S!O-+7X z_4dVZZJiZHTMchUkNjc@B`B&nvB33Qp~`dCKEImEY0V8nqlqnEED6QxZbRv0%BYLW zw^nHUlCMOQW9a@l^UhDj@f{=Sv>#Zgq>M6u^0<~8vAfpiFW`vh>v1-T_eeeolU;$~ zN2bH{>nb z6u-gQtsjaDlsf&uJnk1t{r&36DLZ^v!|LXTwTrgUn#W$7e_a*=wLh+I3z{{dc#q zGvB-$r?k>f9gi2=Q=l0@La`e{v(>5&(7E6_T+O?ekvYY)XVwjxbA6F* zuUSi%rDENTq-!LzkKG+vvfcHWzBp!-M}79XWVkr`_OUC_k0*)OZ0c zSS#L$B|DAQ)U_1_FU~XNf8}FVp|>Q?FD{*`RzEp9FY;!Z(7KiMhTiI?W~k?*N=j-v z36YCNywG`@e5&1Eo*`Dq5~hHAY7nhJ5lBQZr^ZBVI zM1~UzKyrh#H6~P@HfUfc^G1h78G@on2fI24M6Aqua>V^UJ{Pa&Ur|$+F+0g3D6f4N z{+8u$YGp*MHX^u0;^s@Bp{+IBH1Q$`!F&8($D71ZEv!lmBG`3mdPO}>$WwFkMAdmw zW%{=EE>3!eE~hTM+F7yT{&Py;ESLX>94$L$JO5OJh##+(VD7ge0kxK-F2MsiP(mBM z!cqwW;)WD(ri4u{;*-q4)w&XchfW_N>ui45IS|{qd6Q<+W5rJ&(drYKri7!dg98gp zfW3*oi67FtFUuM;Q46Uug7nnp%-m)rRdhHcS`02UTP-JWuu3MoardaaWICUjX>|>j z@jpx~8$@KD07sfm$-fp;hZIGn!(rmrr~`qS7#Xeh3-B1@ibFQlOd@n7+RL2}p1iOX z;5>pUyg!lgp0T0)8TdIz1Z(knlGJRlK*|G3clzA|2pE27FVEG=~bUUEW@ID1Yww82>Yd=V)+*(9JPVY|3-jkWcrT}Wc;Tg~W;E-Qb%PVLoTj9dc zFf5e%gjZe?mSkBKR*)LN*-b93S=mS$qV_%>vrNJuCL7CmXmw7rNttKz+*J^J?b4Ja;*(-n9N?6}om}Pg3 zTH$c~z02Q}Hbyf0rihxwT$OT`hDM*B^A_Onxs8TsqvT(E-xMBKuD$J25th!auFAyFs7di`Qi3SoH6UjCn2^s~YtoUUzNA87Vw*xbDNlW7~ThVX20UVs&zcNlPXX)pnkED&i z1?FE1A>STeG_RC()lJ%FIZ=*jZpN`9y&1&FFb&e2b|9c6;@4OiqX-=4B|Yi~*xI~X zZtxm{b;uWOl)R@yKAF|2|2{tEHe4~zo~T|6EO;%fEm8997QUfqv1;p%gqJ z6Ep9HMI45_r6BvuPyx=ueWV55;!-cK1)h{|D1I(-iq1Y#6edpiEQ8NLkdBb9%OMm=i5z~+FxxNIJ;dB`Wj#shxTXEFr$f_ zB5RYH8L}#b)z>NKCU3lP#ungclA6ph1N!i<=KmM4UV8HY;a31<*}+P--Ss?~H!*hcxC%jJ zWtjOQDRQYXQWiHZ=10=Vwf|uOF0q3zNx*U389u8EYfoCIKGJ!17K#X3j)B%G@6nYp z42xbbhCHicXVa*==7x?G5=uL7iSmAJXFcglTp07W zCkvE+DmV6IO@@}vkDwDPct0-7gP(T)>6_~4`P!BfUyaV`U5sI-KpdSbk&_J#_sFV$ zGJjmr^w6AT1ad-e@9@`0f3ujM%jzA>px8W8Cm%mdWG@6WIM~HE*J*eO{Uf`ih98I1 z@N{D4a8vS=FJHGYhP$6qk_)~Ik_l4OP!GkrS9>#o?Qq*^SLGY$?gnug4LUIIy3SAP z)d2hV6sCRTjX8%m5%9fG!*F1r9CEzx_2r? zaTjtDa)hV(3sAg!N@BtOC33*)xu3wY8xK8$XI2N})DH zp~3oek_LwaUJq9F_wYqdf3W|bu7-*9w707A@Q_Ha3Q=`;-YlAsL`&CB3>jQcOj+)) zAWnu2W)*zPet;~NNu;BzbqIjjbD+BtcZKAg>-SUQNsO!|qCq{-S!1g_X zp+VTqqAiSyt3NNA9w5JvPQN;|n`D%e|1V$_tB=S*aq|PiZm?Ll-CbgY;A6=1D7%gFBvynpJWUtLM2*viGZW5>Z(4e-_^? z{n&AYrP&blAM4Li+r`L(Rt0gYs0(S_&t(On8_m!mlbs$brtIQi@i!p3{T91e*FBr% z-KC@thN#%lVQ26|J3z>He_1YDuzMSJtTM{2{aJW28u)V(!OdH;-(#7*0Bx#-B(omy zI31U%YtSLEDpcDy$yMpXKq^5UvMxvoeMCZL zG)*X~aix+vuX<=n=*o|y!O%}w5H{_s9nW_$(4+cteB}Y)P(M=L=r5T(eFtDdgao(M zZqrpn7gE=FUfOXoTyd4>n?l%$h{C)hShD!a_E$CM zW*Y^9SuU^@c&XqdO=M3l%U$0!eQEX!GJ(St7@=B5a&{VStCDjVog%dFfz$XV;#*+N zY^|a(uf9U3T+$h^Q8u=+ zsmM<~?d~V5M440YHo}VzK;L!nv?Qus1%;(ib4Y0lY4+6e&39_hjd4?!!zA_q`85d4 zmGZ-FS)WAFb>#?(n;!IzO1nYdg1;s8^Z?fl^eXM3(k-nWMJYg*y%a8Nk++WMghz!k zP!frdj((JEkf~8q@|KcXVaP@xBlgZ3P8@G9AX3YkQO{IE#um%{S`t-eKcVi)(@)7R3Sd~@Ow!-Bp zlXLE9^T)uF^tCSyh!QYt)OMJ{h*IHH;VIJ8xkyh1qd$5)@G81*KX)&usu)b`&WdLK z1xUIGm2azVb-|JO5V`(fPZaJGf-zNsoVN^tvcWer@F-bs+(oMdb zDJ`6s7yw3%?TMIv-z8|);TS~tpZf$ug4KPZA?Dhu8X!a_yjvx;Ze6YR6|1JJ?KFi< zYGp1%CW&f)k8J~ECTcldMHyJ5HkBicr2ZsE?LxO=4akbDCKA_}0Tx%~tSKA8Y!;#p zIkNQpiQT=+CxR@DC^B9JP;Wo~ocfl+f7-}XFI?^Y`GQr`@*8;YRW)l5DO%gujw4j4N z`?=zH;lSFvuDV7kqARl@p(;@|+sGlWl9_37HA77%Wl z2|l+SBFJI{;+l9>Ua7&0nDuiara{)EZGD!5Xno3tl|eI=){iLh7eIUSMk@ z=f8j$0iX6VQ!j2L>Ex33+00$z*#fm>vxYW$`u$eTDz(6Rzsz*<<8T(VHI4Q8I@H<4 zM6PBs-p9$+ilLh9JmAo{RGam*k^KFRm1U=*&Gz8ZGOU^)0(Row*JV~py-T`cM!b~ z+wQXwNcfbw%F#TTXN9OB{VL$KtS;WB!rdq==;iS|jmX zX)RZaDb(qsHt7E%i3+p1+7jvG-s3^J8yVS28YdtL#UJx34Tm<$78+P*+;jpB77&dg z{sr4AbF)@miMj&W+v3i zS8pkK<27=$5;&0hBl~wqtrm_-8WHl+wCAh5cZ!-}^ z9Iw^Px!BZI&Z^E-pUWbzaF8$A-L7kEFop=b8W#BhI>@-6KRr zAsy3|!ZOUf^2}dNRxO={{F_M{8u|0r`c&;Fl01ZADi47r_wnpJPhF01YL*un0HdUbvt~_) z!UmTo&xaS0)+q@4Yud4<2LPY7|2=qw>V) z@%_fQyzz0Zw#cyB1P4Yes+1I8qY5xEPy569JpIo72~<6LaNlvSjzEhj9JC*D^}l`I z&voHJI8AIt4Z;8Ju;#q#c1&Ubqg2i=9S;JfQx(Z8z^PxPYZaA|OzB4z6UtA3c zgl$d~Crj%dN&**!C^RB7!iOxl3;h++sD%9D6sU~8<`USlc-2EAD^V^#js;93Jxvw` zV$w!BN81p!7F8)Vv|Y3Z>@~kSq-_E!ji;G$V^Wb(wP`4GVOmlewxUyH(ml}I)$F;^ zg+v40%@XEbOfbh3>4zz!MC~8F-d5B`l;0L@9d4Lb&}ZGhO(2>CKcbYPDrXhr43+@l z$2z0R5V7@5Du^m76X0*IlDfXbnA5>@6veUxYMUHg*N24A;hdy9iGDt}*1&jR*78D> z7XbVPI7t)YNwN_o5BOfY{B9FqQ>pj4qPu#K@6TohHnB@gy&^vjS&)n=Xrz|GKPUG- z27+m0YFPXeedBO=#&%%X@V-T;X2Daa`r`(cd{_@KONuhwREtrG6aUc^mHML-sWi@T z6G`GsxaZ{)Y;2qthnTJF*3vX`9nN>KXvj0Pb#zDkQ0*y-^PAOG=UPRdd%tF!l8$iraJs%%0uj&FFjEKi6|6HHv>57 z-@sx)ojshCxTuYXrd3FGQjnTNP-8I)uWH+mq*D1j26qoEK4+JkL-+Vr zAmi;4thR(p>DeW&pb&Ez^?ADKCzAFjnZ@t+|XFQ5AN6ZKr(E&l>K8`I60m!Bvr5uU52KE3A?MVB>Y?RvUe zJHH%+M^gMuWwU|DV1g@)6nj3kVigV@*2=&(s}G)ZJy zjY_{@tp>luAVZo;*C{01(6f%b_usZ(w4Y)XTx zMqJ~x&nbmumP0qoUbcQ~3FM}`jnjGHq`i+4Y@Gj=skkR`eHb^dI1@gt7Y$^7lE?kQ(}qs*GbHGVl#eONM)ZD;wbX zXC`zq7bK~%kK^JF`~JXf4+_}?8}YV<7)hZLotI@&L#MA-cMeQfA=h+pIzxkdPct2ftB`>AU936_<kbujeZz~m!` zn}5$hDym#Rx4atq)QK0YcDxGCD<_AAzvh1gh)KhEM0{STv(leU~itSlULzga^d?r;fp_^(X^_?oJ{GkN!Qp|c~tU!G%-}MjgpIL zqRp1UlWJRv1Gr(U8BhM759r2zE~&qR3$9wn>TD+~`1lu)MZ@$fPYHk>D^3m`w;O;LVh*#p0gc4Ij-AJ@8 zvEAgkHH$?D)Tv)xSzS?WQz@c#%QJ@0lMgbhp{i4i>|$cjw&TR5-=Tx{zhve;1OXZ7 zJyM>>Jn@eYE`--i32uWK1_NdqaP+cCVjBD;1X=~^Y&*k?K3CS zvr)CmiDQ7{3jdF&bBc~^4cl$lQOCAzr(>gH+fF*RZQJTp#dbP&$LXMBt79Ac)ZYJp z&bg_px~Vl*jkUh_eV#cXDrKoKOoV!_6L1MzGhDcl6xElkb$v5`q=RIgIAu+h7b>OI z-~-B&LMOwM;ptUh*iY7fH;#8H=32s1$?Gj~vKk{G44YVCILE3-Qbtfenz6tqtwuct zeNtZ+rhcG5>bI&LgLHCUJYk7T`hVD2#go{FX6DnFh6*yMP@+Z~*JhY<08+S7nQc{f z;phv^ezem}>Ct;2>B-yK@~5g~l$erR0c5<>i{N1q1%L5fKU#(jAFhmd_}rc%v48bW z%-Ch|XrB;n-)-zLSb(ME>{q0{ql%K-g8?q`8<$q z(<8^w@&7r+E;nf@*HhzBrQm_C@D;g$%xpPahy}Qgs$G4_iVEvQZa zqMc`9*EScvM)xn61Z5PWsZF3z>$wc%g8OxS?#O)tWG|;SA2?N7w_Oe5_i@dU9i6>L zY{lqg>32%CgGLg_UkP3uStAkohehy9C5W;Wrx2b!Vq6JLL!^b{Zv39f?p+2|;ncRt z>jyl+$*{Xv71E`R%N0nNNeZ%H((guN?N6UpFWSRDCKxfG74$0)Q?1|K`rcD-1Q(3l z@4TK-S)R#*`IZxi!=G9a3|#d$GH6y2xoLny@xG63rrn>sHu+PVblNVuDu)6#9;fX{ zyrJ{xr;B49^XcX{Jockq0*`S}IuvDj4InNPh|f^rUIEsNYSxRfCL@Ro$wjr=?n$N} z6*Z-HAX9{CSd1mT5j=-6Q${_i9^=i0$_V~_W{|CuB0`fmmUpm#z2b!{-L~GalJ{zy z&Mj}{-@)a?a^SvyRehN2X`B5P4G6K)V;Hv*BNxo73sH&r{yeIvswA>(7gsry$iQ}J+eNCb z2&^ISATXJuN2J%47$*aG`)130lolPkfe|VDCj^sm0uElm&m?JNQ<+hm+68b_fcU=Pl+uI404uS=AT}~`ig^qJos{M;bkZybaPcThJ%H2cMIoeBx5vYX|m1J zgOxeWmt;!tGi5{s=ei#Kz-K;aGY@+=%jicLfWGqdjDBm1tDX&H$!2@Icbg`krKZ8! zTWgDrJ)4|f#*kB3tHe)yCXeW#ecWwri6StEhU~uAoYpEgr!GeNZL{oH$!gXGM=qy= zU2wgCNLVNMj(qT6WQrD}`hSmcm!D z0K1v>Jh!@KphiXRVfLvf0uHNnpAf@!jcQ8vA_s_yI#$zG?-#tPdbH@gg3Jl;idY}p zBAlie2#Fob-Yd${wMHOpspx2&S96Gi8j7XS{fOi;V zK1}p`xcq2#XtxC3!I_Iu__~O8*vaW(p3W&jLN`igA3BOeg}c)0HNFw6q}H~T)3zrZ zWIyaTY~_8ZfvIPVraw2j*vqLF+6@)JxX&*@>PIyDig5=q40m-bQ*Kkaxs6DZDthCElNh%r28zTSSy|Lde86%>&rUjz z;&J)tDsFBTzEeORbw0gut(&;sefub-tVYAvRE&YRye~Z`V&wj_nGH7ETr2-r>(4zN+7RiifVt*8|75Tocpc|5nZ`sccFJ%9VUm&$yaD z!_Qrn)@b4bE+PuYpC7jMtm~}b&b3Uw4-W1Al|pcpn9p$Defo2ANDD{jxi2r3enS|t zwR(Gsv5k0xc?w@~K@pjo4yBL@$qz7215e46<_%h6<#cMa#xUj5My(V0ZG0=UwUF_!}!d?xTjB2jHzC$2fvA0&6%~45_l1a*KIU<00|m?{Tvo$BbKiIv)+>i$w*R3LG{2 zmvx9eA^fR3-$1-^!+=eM%acEuX;pqrQhxD}Y0acB zAK}zUlGa?`p&GMel;0W-DwfF9PCXr81u$!D0tW?{_C&zJ_B~RjT}jwN=v}+ayGxEK z+%HMDTp5s~9Am_2SdSK`n9|{!q8*{4N*(vJB`8AxBN`>A9N8}lp~;Xik*zZ|N3*v{ z9D&RqoG7E70)(rH)3dwNjU8Xj8vGGq<&dSELoZB_xNRpF-Y+xDnWyYIPJc>d5rECZ zCE{EqHPyG@DN?}38Z`(hZcIh$ugS@21i(tkqUM}0ge*e`FLoS-sK0`|D6Ej_NZ#ux z0O?W)`oU#Q-o9hIob#SeYg60vxZgRI)=P<5g{I!*bAC;0CKjA!WF9dN0Vot=tk;y_ zzJ$kg*lv(x(lFIhrAnGRg4MI&{cd&wA|;)-ayFI!*doz?jM8v}jJKA%pa;C*6NEJT zm2GqMHy^X%v~$h8=-7L>=^wJ;JFv7fQtcfa{xjPw9k}4bEI273!Z{yyym%j6Z+aFv zs>Qs71LH(v?HJ`0lnEar`ZK}(TBPFE4C;}jFUWwVzClE}h2zUku=yn_tC*9$8T^!_ zB|gjr=SlmNQrRxKM=1IL)up) zSIO~-M1kXQpH2^qvy(cp&*YLbd!i>$!llR&#@*n2i84@JHCgWXwrB-46W+A^J}8_F zCoJF-An;C$Ug0O*#b)hQ(gS&h3;dFq)nl{lfnFP@Rqu__r`o-J?&lE6aD2KAkc{no zZJ$rUyR@zxj(z&Z_9`yP+b4eP^D@h-_|b5%;!a9u2i}BF8j(M2U)9edgGrX%B1mr| z#C)jJBIfYYJFi4ixI0MXWGilWFnG}0uFCnleIuW$Mt66wV(1NV4QKakBP$-Sc<4#k zW&0^a37?o(-b3Y^{Btj_!_Uf3MTh@D96}3_N<{w0&mA{W8rWlaeEM__`nSI__zr+p zb!>(@kcve{mK&PRF*GmM%VLPEm*4DG|5#~GUnkfw}pAd3PUF)E@jhhLpAzsQ7R z{;^05TZUnh(hVUF{0CwbUYxiSw=c^N4+S2<Gkp`}tETuk5(#|JC)t;*qWek_52P33Ur{cR5*a z&2qs+3nrBjAzkus!9ykPZ;3qUIo;e2pjB*mwZCilKHr8w!E1Oc9bEWbUB48%fonQi zNAI)GL;r&Y356q}nv$h;07e+RWIs|)nUPz%zeeYw{*N9y!uUjZTQ%}OrYJsmf&`v5 z0Z(BD2>f5KCe!_P>AE@`{hnMSMTmbPw@zLdso=7(Sp0e=3DhF-#s&Wex+yxdxJY^L zX6A^bki(S*?|NQ0_WE1%GUp_S4zJ!!LJ?EB<)YIjHbysTzf0<{w4F@akR3bx4EVg8 zV0-vi>HK4uy~DA!+%KEr+gsqX9rxwy&~F)E0nVoZb}BhU4~QHn)-O&l&<{n?f1N3x zqPh78H6th4r?+nA4R%axFYNt=GI{J;NlTZSos3@u1w)(@{7v$ZN} z!+75RZqpFo{Qr51Y^soC66<~)*d0txHX*qM9vVV)EEIw#DrR4k=7l~x$hK!fX<8~} zG1$8Fwaxo@diVNMT5Y?now(|ph@G~FE0{vJ~EZB z;Csj#l}kStdAjziEE(u!p`(wotbRO~n(Wu1eNes(s(gdrU1-u=gEAR}FaY!8)4L&8 z0cOrQZ&)X3qyAqE3p$M`nG9>xb{%%$bNzJzFU7GQubXQu9rq>WhtSf^)*gJq?DFH& zN5csCqBm0E;gP)uu>GYFWfS{rT`!z}Ee7j6q2l1$Y@aave*E7c725?gTnfm^JWZ+| zz(-397-tuZXnXi*q|%c5S+sD1kunwV0X}?#sV2d^$M^ToA!mY(1mkT*!o4uh_AvNV zQ_z@^D3@>Q9fOJgeRlqfQKpt?^?GKrsTRa&6c0$5RWk|ol~E4^2m%tFdjs8yxh3RUu`n?Yp8gsP+7u=osB!Anj{Mh?;<6m@wrDjN3>a9i# z)ce*?aw~ilBLhx<>_CHu?ZQEq*uO2{<34_v`&*5_6cKlf%|$8`#sPsex7`6 z&aO3Bw_=JO&w?V8n!HV|C4yOfmvvagu1R?5$h86A4)bLrxf+3!`>SdU6-Yig95qlf zf)>jXT=S_b*(H8=v55%{s zQ-)z$`2>CtvSJ2WmY#~|FE#Ui;Z#>lVS}1CWZFk{5+l<=qF=*lS6|9%JD+9#^3ILf zVTX@c5l2mxWAf`^CQew>f8>Wb0)Qj2WZD?Z0Shh|`i3)Til)>_ofx94)>~7{PpqF^(AA6R8i=cWZ2rfD!Amk{aR~vwH zEv2BLqC?*ME)|^j_4~6=hXA@@Sq% zFU^gR5MMBunhN790vhYLsaBFM;?T?)rEjTU%8qj98gIO;k9$D5`6%jnW>zYu&Uojh zH2tM8r{E0)ylZLcH#15%PoJ#(`3iF6r;BB)dj&eCbPxqd9ME=q7`2+pBQu-> z(D|_18v6VYP#nl?nyj;Sq3%>K`$~MioUoL4*piVmqDgFNs3+xG`8RCoB~Wv_1~knK znugR3Gzjn0J6&uWy5bojX(x2?4qHk>Dg_&gj;d(@$vV zN-_RK)LIbwEn4O-Zd0k%GS7DCs=i50ZULJM*EP(s%IZKeG)dru;$mI*+jKvXd#874 zbtZxuR^~|O+i6qPPZ}bhpITTe4mDTohNkKfPlU=xC#TWkF}?{^?pXp$yOFZ$T~IdX z_cx?Kg`8?JSGl>wT#+OOwgOd*;el-% zO6H}-z>J5X7TCTb{Ya%+!ExGwsv1f22oVk#cd+WERDk@@sK|Sn%KVh z4&^yC9?3WpgHho>Gg3p?w8CrYGoZd`>G7dLMXP`DbTN55cfh74V?K(kXyk_A9_Wd4 zJ0nT<6dDkeT+mQhMvPp@T2v(P{^x}7N=$0 zUl)&9C&c^dI<87jA01vC??-2|gZV@~wJt@uvArrlo2A>VL&jzg{(`?|2wXf~tZ^(+ z)zzD{rnT(Tmwt2VN@yD`SOUqv)fuW)P1O7>*i`z{Hm%7YBW(|eJfw$ztX4!ORqT5E zk&|tuulCwWKD1mfd-44rh~`V8cZNs8kQ~Btb4z5E zi1sJvs#EV%LZjCOs7+|;>uIZUqA4S&W%~_>LI|XlrPFpdOXc9nr5(Kj3DQJ~CvUF< zMbBiPEiQ^~2#b#UV!&9sb|suTpKqajIcw*!d0dWFbum8jrtZHcI0*p9IJp~HyptK4 zS=RWBhZ>r?h5?Rhe_d02mC~}!V#bsTW8t>a*Cn(zo@Dg;EpHDFo7fnEb9faU5O+d> zW{U~>YlXUU1Dn9uPh@T%>n|>rC=zSR)*x)hx^m{-8OBm;zLIhD9FUxEDcjEXRox_* zBagpoTHS9EcW`MXauDW?^M}9)-llRh!)W=?(cqA3KC2V_nk;N?n-mID@H6Zm2!1>7 z1jzEh+DY%}H|eQL@60@!{i9g=D!#^_na`=Y1qhZ^j^8FmXg?_`Y}@8D*UR>3&Rj2< zL;?Ecj$I;CO%q&SI4bZCCq+NHzJ1++P^Nt@xEyU+1MVHkVO~eNK7;FevGYgg@~+rh zOx-MdT|n$=!|{R#2oK_?R7!{b#B%i+wN%5e2*s|OjMc^mlAd+_(>f^(7Dwuc8*MWZ zjS$FQEWb~bcgh=S-z4@WT>_l|pzCGiMRJWw7npX#d|L4}aSy1Bg}Jn?E{F)BO~_X* z_Q^Nh0{QW#)nCeQo-`e(JsYgDfZsH8`J23#eq5%kCoDp+`Ta0Tbtbs9+T8U`PuacrSjpV^WMa z)AbrJ;MTHr&LfBG3KSBE7?L-Byev)AI#{NoSWb@KdL0(>_lKLKp+#mp7^dE?@F?#tSAW}=` z_^|1s6ckm-3YS0WyO*z!jHM90+=Uh2nKL!T#H)wp0MhvGx@Y+i^t!b?u$^H&~h zughq8f&NEd=8+c$Mu=uQjJ(*q zFTR>XVRro*l7&SY$+dykT?j3`pb{qxxT{ECO1g1`Wz7?c0Y; z-MZ=AQZlp?x)U5Q=4I^adf4q>>d6Xt*aDU@9wXm!`anCH6h(AyqmTTG?;6NMg0xQ* zeP;#hfzb|FHf@x{5C^%%{6zfvotI9w?;}~2z+rTFOr(Mx*HhV#q#+E}r?9>QiW%^0 znvMK&WVx}oSO3h|+#Wah%W;wov?e#$1MMl4SG)glE4ckeyVQMo6Z}d3C9r@SQp9jm zL$Va{P7+;$;JSw(&>vw)Q%&jIUn-D2B^f?FSc23p>eEuxC_1E*tS}D^Ou}v3ibBAY za$Pe}yL{>Ag`%*n|Bk=ul*If>BfgbQYUxu|A$u!rjeMn?5#oteUY z`}QP^_z#1{jRhqY4%uvlu-gq%drZ!m0NL;GcVPnC^@Z@{Fn}Jhhm;`2o+B5d|0f^* z-~JBwf4QKRKKCd8VK}s5wk0{acA>u4-We*e3iH^T=lF^81d1#SE??4(x1G6t(3yB_ z>;i^{I`m7^S24Jtb=cxirB7*L(a+}?^R-jbllxGrq+Vmg9v?FuU)C)`yPnsR7Pstdj))}ezPK~^%M!5TNnm#J(qtu$ERK_Dv#kXKk!#>rXGUlq`kRL7~Hv%^6hvXlCB&z5V3Ei?Ef#cf zDA{;o@Q3uXG{nP)!_D(uL>O|$#lzbh9yjqn5a3&b{65-+wA1({Xfazdn{vn6{^53T zlm~^mwG;-;r6Gkxhr*%@?(*b|^ym{!(CiQl-MSN3O#A?hTTgeq0!if7J`58n3|i=f z>WY34xX#l{f>hzPHketT2Pua4F80y%3xwg}v>B}Iq3@Wi;r92N?>`XnO#kYQz9h`P zehB~M0uC#4_O2b3LtI+xOX39p{RpGJi~nfVxZH-Hjzl0Y99jTX=#xi?fCN%suYqu^gJ8hy^8Ty?nGewF62 z8_(4sVmIyBU3AcbO0RXEw%wEs`Q;ugAVOy; zpBAYGkJV!!60b(sZ4$Mix6GUmx+1NQ=PAHO=6^ZHY9lI{J4`b=UJjK|diDmRa!ULYy-M1ATN9#Gn8h?OmV6ftoQDFd-O${Vc^R0j2P+=ySyUhJZbo4zi9x) ztK<2MeD1Dl;`l8QfuBKTnyB_uc=u1U25ZyxECED+=?k0Jd8!?hejTJ5@)V<^nP8`;-t*ouNC?pjH4+Gze+q!|pi@L(N8;&(%DkKl#+9$;W zFtFhR{_t6IXOzSu{WV^UPYoVRu_#q$sggu{e#xQ}*G4x}ITwa=)yXN&i)t#5{ly67 zf|=f>CXM3;EmqHerczObul{RUGfTh{s7*T_E2l%@7>-ABM}#^LSc;MK4}{bR;S`R^ zYcm8-lGvW}Y8`;e3Qwxycr$v`)FsDm{!q6}G!(0we*_TAR!nXpfbS+Abs>YxV*O3i z3+{%;X)vb?(VlJmw2E#E@X&@efu%N7tB?WPuZP7XrlC(_I@>h;`T4$!qP;f7DKh5{;nThT%vt-h++CUNW;~r8zJ5K)N}&*z-_U(fOOr^e1lXx94`~3=sV@I_v=HUgekW9 zUwH8oX0DN*3|Y}0`|yilVTf`}a7_}e%_P6>Mw|SPdLtW0sM+Nt$CE2G0~i&$kL4Vo zewHAHfW$3`yiO3E%mEkopm!r-?<0|Rvn%^#HfNTqS1W$2?bY{vkJbz^jRz!Ec}fZ+16e7v zbgEPck(Sz)vvL_yyePEv41BXnD!U6YZKBzD-zCMhBm~c~)d&$R=UI%pB);jt)jBg+ z7l!v1KmyMEy zW-7ByMJ+ovN?<-LGi&i;>Os(OOprEPt6Baf({;?XEJss({w{qrJ!2ujJwXCLcM8*&h<*K#W>5r|FQ zwa~+>rDxfm8W(?$0K(x_)L8@#Ljd9_03h~t{Z`))nI7(qHY>gksJ)$KnQfb+Uiv+w zS&=n2f~>pD)EUEcuk(g=-^}#mdcF^abc}k;``wzJfL&FgUv%N&gVyjvB_kK6qR^Mq z0FQu}SjPNTLOQ)-dzqesy0u&xK9_CV%_4Buvy^Od!-Xs1H_Q_A&_58=>WmejDhn#q zxyUKIinsp@<*$AE^Jg=Y|E*E zsw&T|l7Q3W#eO!l^~>5n#c_6FuqXh@h0j6-C6trU3i zElG$*79-L>d}5c#zgxDw4soaz<%Oa!k)ZGV*_PzGrs*45*5< z^ZV(ABmW|hp25xS5FDyuXjR&0MrBL99S#uxRL3Y%Yh)>ACzivuGC3-u^E$GqP-e+} z0}ah_3nz3Yf_5&aPQmYC->v#seG5zW&MbPAfHXy%6AVZ*EoF;j?V5l?W>=O}JePYO zfzjXJ*dyuXTvTB3dOT}KjE?o2&9P%Y)sXX~`QtjT>s^vuMNmYdP7AqME;@xzKZ?T=JbwtCqfzG{Z4L@@~;5_YDS*=MUl_ogr=DS>{b9Q?Pc3%zJg;}jy6_<{% z;ui`@f6?iwVU(~Ot=0x=r|}f0w|tsvi)p>ZFKP*%y5QyOs&%NJC&>K}{W|qyu>Hgr z(Rgs&U=ZRlt5j3^5emq8qE$UMsr|YVR0*acwQ#$2)2_f_)^on|9OkPQJO+M9AwGcd zrPv2tQ{8qgXev(#H^=Zt-vV8d1=J!>NvE3(K5`weK5`CO;S#1ohH@~V0WxpW6q7nk zI;ykI)maW+U(*=)!1R1ivKw?QF6!3J#Gd7(oT{e{ZpSlKT|hi6CW4q&<(A^2YzUNp zFdjn8L&!|GlD+DbYNzJDKVy9rOsv33m*eJBLe~iX-ks5}E-G$^wx7L3vu|QM`E8}v zI=fR%)#UrY`3v_&>yOIg49T=z9Ec(PU$~u9Xc)za~FUcyC;T$wt7ml_n z>%O6gzvUFLtEy_}i{_^p`SVq&urbo9GMLFu42uhBp&0@he7-28sx$wB1i++1Hf2;o zPUGha!T!Bs9kG({KkSeTY8|GkxsEEPj84ufu1>RNaZjpr8d|n^5wR8|PCeja^5W|B zAQT6v;cBa(5xat(Od$3zE0up%-q72TmXnBIB-vP5d7m`Le545f<)&Iod|TIM;(8TZ zF`AP5#bH2$BTHdtr+Z@EQ9h9?b=TUvNdN4{P+3;)PHi}!O2~o?Kmg2vsL-T;m;h0V z@hJ%$H8c0zzFWk|X3i=;g18o2JWOg>lp}`CZ(!-(e;UwB(HC2*QZ)cPmiJ!9c1Wpy z(TSiJ#0rfuHD-I3uFd#!qM+Cf^yRW3bO+^BBIbdC%yR; zOR5;Mc=wtSOT^6SS7g|(UdKVffE7XB`XOa!Q7+h?Q$7ctdj04>#MQrfM@gebIotH* zGrFk@55JA8V6<4ShNh(-AtWnsm7mfXxn3g?#DxDk`wVwe54%tuP!MUDZoqUN*1d7X zeCY&Wn`yKt18mhAARfcLo!-Q{u)2?byK;`}n79nw=^gF*l_3^abjh>3TRmo)RS?+E0+8<2IaewyZb`U7uty zIP@kjX6DB(B_ z?Lv*r=-JzJ*jZ$6nv$~kf7X_`DOXVUQi&+=+QO zHM0dq%PJ-n3G41gue{zTN_^Gs7N`x{dv?NQ8(KLWYw-I%)xVW1LBt=fM-jlgT~1G; zW2@>s_1!kWz%@WhFC%8ru-5wJ*$sQX5nT&0tbxsIJ>P9;>9X+DEBd=cpeUu^@aS*e zLN)VUFpc-}=?^99ytRYe!>Z2zcySuTTV^W2Ln9~+BmT56H$G6`6fb?8CecSa5_>xD zKd!C^VBpUK*rHB_%?PR;obujbO|0B$PC$an0 zz*M$EK*TRDuvc=9i3tr>Pi{+LZ2M^djWDY#%SmMZOR1k?dlFxg{!VxZKIZb%01P_^ zAW0T(+59F^e=L*Hj-NC;F8hD;O^EgLfiJt*mqq*|pnoGgWHkMSS~72nMg=^}i8t|yN64bF^4W$Abcyj*6yAuwok!;)Vh@+=5}I z+X~t-mw%arl6(~t7hCk;NF460WkctWi&I>V&ucy`bN^Y`M= z1$%h!*}wB7a+MW}h#GFY7+q+gI;T>hPoRtj`MtA2kql#YbOm(PqwR^f_T7TTa%^+7 zLB==7QPMmptP{eY;Hyc55Anau5`3t!KZ3QN&f=k;5YK&UG#rqaQw66B`yx;c;`Au`!D$1io(*g+_j|_S?a&L?nolUInbqLt}&Asrwa^Vdx9qOTV_iLlY(zD53ccJOPcXB>hEMsh$6TO8Q! z4#U7hsApX}!&LKEM*PjF;C+i-7Tej&d3~*CBNs8@BdIUz>LU7w4!;*}-1K#jrZG#xksCl;~f9dH4rr*gb?s8279LuaJH1p_bss zz@)dfUkHZuwwdoQ!Q-Grc%A(1zY6f}FTzKk&lDDKrOMn#Ol#?(jW)G_gzz6hz-$c#NnSp?2Rd?abkMYH~>$-ZfF8<4OT#Q2St{^JDF#ZMO_~ zQYV)G#zlBTGpBJ$``q^uuLP}r55~o;k)1@d!3lLLw~ziAz#!8qm4Tbt85tJ=USL2L zn)HiM^0@}=Zr`#geXR9?YXuLG#|0E_H2v;WNye<^x5IX%EpiLd;Czqza@HC>+|gyA zyHlyuc;gL_=0~GNu3Ed@X2`K|yl%|29-Qh!FU9}Nh~!|<0q6ltH=cGUswuc+ zZpj57?yOmENCx7k_yTuYe1|!R3BFn)h81I|7QWY22u z<*d^4S>J$%y8B1np9MZKEz<$~5l*rkL=R03?rPEQGkGosRayQP@q{`b)Us6?FLS8i zGD5CF-FzxatSEkVjZxTss_?FcB)d<@M%U}W{G;8YhW9H|_IxIr&dI5U9N?<#mK}Pg zd~T<$2Zdg86BHIU;Z=Jgp{8?yRPmk$?k$_P6hV zxW95Z5qo4iB ze-f>g(4pZL9$IB_s&Zs5YvE(4m_wi7((s*=z_nmdqp+2ROlOObP=4L2w21mKqEbyv z>2puM2ASen+7g)5XngLSUfm*;_a%gn2oUSSNAEZwJFq_p;_wc&gepCcN(?Hk1I^ND z15U(SdD=2+GSFl>N-J!|gXEGCAX3ca#!ywX2yD$APBw4eaLq~b3n+tgqRF_YnPre0 zRv!*YbOY%H@Vp}K0Ex7dw(t|xiB7g0G8p05UFOZ2tEAh|fP_FNi z3un}R;koox-bim!A+3`b%tXfRtEh)dyxfDbyF;uWO*(QOe6@Isx%xv&6yNlfR5S(E zuo53wAiiXf&36x`7b_%QSlU_K>k%Dbxtwjwtf~Pm)7k7ki-q?vD%8i-lxIvofy0V49r>AKWNzUNXQ*dJ= zJvdr_b8Gpl;tTrIb&v$_^6IY9qL0`}yk{50S|>;i5QujNmWvL}jtaI57D65CbE2K_ zkbZH~IRj*9YjUSE&mU@1tgAXFgHElAfz&MAAMoOD#l(`#NVbDHSeLL0N$y_n`_Oo# zHl5z;h`=g%j|`qx&|q?GdNCiA#b8*bp8PQM*0+6xC_=vE(7aIMe1E;t61#IB@l?JD z|HcI4u^a)%>mUf#!x3gq35j7ui+Xu%*opYTjMMbqaBn2D1}k{PMG&9>U;_2EnJ+(! zh~?tendP2nYZMzgD(3G2!Gh97DbBQad4k`!ojQc_@w@!#$U?uE=Y+>Jqc;@pJWWR0 zl1a255urd8OIhGdg}#ou!=e@(lu`*hsXiT`wtg}w!9OZhPo*^hI*VsJF#T+IPAKtH zKtTLQ)dWZ3Z%$YjTx9yW(eJr`R#^iL&xaT7EyalUmAM#B0qHn?3(B%Akq{%}_~DCC z*0@&fV#PfMNzmSL=vk(VMXL}SjhDD-!$$@oK z_O}z6o4f8EYgH>g=fBs^M?pZ2;ysl~hbOEMOGGWb289(7Sa_WOt#>k*Lna&y62`FHTEP)x113Stm zzl9LY*z939Lfy?$qz~vf7iIAj;&2*PnhBF%9ne>I1XnuR*Gx?_+U_m}5s(}Z zsO2DHVBz)h75j?O{pkm)6DFH`@`X-Cp`{l|o{gu=Lvr#~zcRp9Pv?Tpw9N4fl~+9j zjP+$?6K~23JtGs^-Kz*(7BzX+h~{&U4B70!nYzqlAKrZV7itFJr3B)$8`bi@PW?-c zSvWDQZo$~QKAQu5j~`Ow=P$nXXg{%W*YjWvX5}NhA8c8flgdFIe-^ZwcIQN~=mS>qt>l%6*MSYLU7Ec+>Y2D$l8Nzb)=f; z@R))2j1A4r;&kYq<(oi@@AMFh_-(1#%DUKu7Rm4&nkL{|@||8xu{D!^xGH0GqQ1^Q zM{U)Os5fHz^uZJN;AtBN$YA~R1m=q>a`jn{^zq?^6PhReGu}J7Gv?BEOzaj^*|got zMD;W?z-DvCwh~=iCPjNuQ?*3u!>O^z@<*+WxJp$lH%&qRYB?Nw%oL5LEEQ)1b;Y_$ zgC?x7q|`Z%rU*7`3=ZuUdC1Wtm`g*V6-XWQ$DZ;X!!u>A+|(cL*`}m)ZwqJY)aF@U zV9H*nVs~SR6$>B5qrclt^2`Q|t3JrN^LTjaa=%E|Ajgj}J!sc{iOd@_b5o0{%A_v+ zBL^L!G=uO{&dU$#+%K2onPu{;K@R8C9u{5k$WWNG^h3RymY_6Po&bi(VFbmaFi+8S znd}{~C%ce-@f;WodX}NrPtOW?O=^%Kcx2PFDAAdkeG$&M;_h z5gmu>U*JaJ8${;qvy5z2?D@tIj9jKKc#7$-!(x7q#>UQxfG5k;gC+>p$Y}y!Y2AZsKhoelL?f@W`ed>rRg(kJ%rdx6Z5e8G*8cX)Jg{$5h}?dsZa zA-0|gi5#A{EPM|VbrGQVk_8gmcXdi;w8+rvYyWcTpQE7??~qJQK_YQAz~X88Rx$z= zQ+l@Yx70j0gkZC!qe;lJ%3ugIISB52+}5}=FZP@Pv!q(lnjyACh?HTi?Fo#l4aV*O z8H-R|o^G37BqRY}4ImaK(ifSU$11sR+2oX03;$~XHejVtn2TegcQWAc-K`xe zjwxG&{x|?TCRtbp9-fV59~0r=Xpw3EOkpW^m!1dHSDG3o>8X$i1;uh;tGlo|d{lJY z31vK5sA^p5MU$6O&zV@dLYm)6-*u zJ|2jy(J=9TOX5N-Mp7h@su?G-ka6L^8gWa(Hl<6r!kaQMdga?u6+^S71*BS=%wSDs zfJTRndKKFja(vl_@SFIMf8w~xVgIgq75rW_q-jWNOPqUbKx%M`vyFbBTd2^E3#2YT z)APG^f(-fx!lV9jTP_-)wiyy{DZrRFJ<+Vt%#o^r8$P;oM4K#LAoHp9Rc`)GI_%`G zQAg>|qk3yV_DIQrwU48DdVLJj*LTTMS|m2L%B)!Jn!O6>wc;9QVQV?<3@hwT2M<2G zK9+!28GRjkeGjZjbw>_hHvgI)k@!Lfor-|@UmgPg8Y>bl8u6s6{N!Q@I;;vPEG#0d z3b0%bHXRF|J87!WA#{pEaDiij{aH6?3*T&+!Wa+ZkF~tod?ghfEJ71lGMbvo_xp4a z3(}cc$qrW)W+ZWckGGG-=vlKl61j%}9m*V|753YsEbE_A#mIPDowkbSjMjwr9v8vn zaiQUN)8DXG`wLH>D_Apni|u0Ql~kz%lW}ld&{x}==t(EwkkveuAL>=O=B)Z%eEd&& zQQjTpUy``-*+&+b^Ds(7*|+iR-*;Le1COzuqlqUXBj))V@9gH8FNxb*rvkV9cBkuV zeq$kfo;bJzDZ1I;e2;DU5TpbevDebC%&@=={#oVqm9_T z=M-jD9=ZR%l8r96Y6#I6>R(2|$`?#ln_pBesvPvaxC-1owh6t41;jy7 zZjr4&-h543h{yRmKCb|yw++qP|66RU9=+i2XdapR;hd#2C# zyyx@&2lK~&x3TEk9B_aEfE8~iE6OY>W%eE*EY{!|qX7w^1n*NyH-Q|57VAP>t0 zzbS7Q`dWw3<;RT}xYjNzv#Hb6<{swTNFEH=;#iSRg5!0W_v)_UZ>;2$9}<;PN=X7q zHg^M!JG8%tzU%UjUWi<&fm(O;FlAsbo7d;j>9E*vjUxH-4yz$g!gf>Sbz3M!K_L4^ zR-@M{c@?bJ`j0l)jCdF-a@5?I1QxIT;!c>f<_;B6v^$ z!dX06DMScdgcd+b<6B~28SgI&`Ue2Q!$qUCk1_KB3ktNUTLrp$qN zeS>^64iBNVJ(lTTv-zI^IuxCp`j4#K`C8G^mC1jJ76>9cLe}d-4LZ1f8L`s}>%Tm? z#;jKn1HKe3!ba~UV}F+2z1_$2{fA_brTMw;PIMLN0+JR;r%+^#fbB8|XXL zJh0hp0hKoQ`O5$1j<89qDGS02KKMwC15T#qLy7r}oC-?+g7p!+2OYf*C2_wL{q_i> zF*$kqvpGe95~FT1KR>jUQyfESPwnv0mo2f9H~miQbq)Q4jMUHx-O+sFFBt3^c<1%0 zQ{nb4(v2c87+!c5TBV1qQ)g^&Y~YgO`u)o1S13x+qVUVF7*(M{AZX2q!~dh-AMH}c z?`3_ZyqCrAHBG2HW)$KhaRI#I{1>#BH9^0cffXra$3 zlT+3b!WlnPseh$0Y+L3zt|k#gxb#rZqy8ln=efO(Me4BPL7ouYyUva=p>mV|ma@dK z;_$+PvOq#E9IR26wmd&+I%ceyaa9A*A0=9L(A3p=w5d)qRCf01`4X~HRM5;IjG|~_ zkPZeK7!B*hvLVtAvw3>E;JuS9 zfguaI!;?lF{3_`>a+^q*MENc%jJ#VeC-+^jhHVC`*Bgt7Suz_`Aw#bY%oOqV3}gIfeSh z!8$f}?u*&0)Ydicm!evYY5(Wjk3d(0yRbCI7q@(GC4q4F$!7v%+sr zqd&sTqqpc$0pdu0Q1g9cYn=i9FZK3DZ~b?mGnKeF1hiByy0G8yVDEi_I#F>y_x`NC zjJh+ZpmxmbXa7zby7c|+=kiOlV^2xb&`Pv*%T~$?^_a7%q(4Heg)lMiK>WTeLYkt< z=L!9>X{5_B zZgDhdxyODOAq3EhB3?9F3|wq>(TGA&#llKXg`-TXM~IXDHF!vxL6H5jU1ZX0TFIR$ zxE;~Bs{;2y=E=aO(Fiz-Abi4sI`!NwPU7o)i`bCC6j96+<>K^-ESF01lDtY298`18 z!_y{5N+e6jtmC#_{5hfKFl+$RRJ`7MW%e^5e`e%r?n?_bodjz9Aev~<`$3akaH`VB z!aexGT;SLpV@tx_&UQaaSLvpb48FPMmc)>ahbZ`{KfaO6YL%*F?)UfV`@Yna9r`Rf z(j3efc0q&0g*^_(Ib|o^gu$G3)yJ_tyyA+J=-6PPQb;a}Z=P?=V~<=y!S_X>2zwKE z@H&KBKuVwwv$G_yJQ=}TnpMG7LGUFvlp=)AO|Oq=p5c%E2%nO+n9c{4x&l7J*GlK6 z!}8P;crkb=IS5C#K{tbT`<$gEcAb4OGEjF{dBgyx%x|{GkAr^PyN!aOKaw^Z{DwH9AW^%fXYCnDY>*gSqOV|ghOdI zpFj%C%mXTw)CNpaJDE1<8p2IwQ@Vqub)P>Ka`|E0l5Y_Z_!(5c@q}QxlL+4|FE%+Y z)?#*zTSaaz#W9scCCsThwi_IjAt=ObWcAPF)H1MX39`rgd$Bz4;5XuPwIFO1^zGQ~yVB>?eb~(Ywi4k9|=FK3c%a%$m zGAI39LP{0OVtEuR%ka)kxhQRAPN_$*Y-?*Sp%5=>RLcngmq)P?!dPD)6F>mpS7N@L z%XQi&=Fjy`2S?f?MHE;O6f*j0cehsYtj@`ofu9Z4miq&$Zwd<{*CT67uG(ZIJ2M54 zg@sVuQP(g$8iTHp_!lfEp7Zbq0|QZA!SVj+Hm2*1F9`i8u_N2UMO$8e+FlO9X$Z)_ z7*AV`5L}!}Xq^FF4E)9eM}DQS5<*>??o|PNz>f9g!Rp?rTeWNgMkkPs=9n_@2F5W} zS9z8eF;syD%xOz(u~r!LTkVB6ODL&w!}L`TV)ZhCn(V1K1*EjHn!73O1x&TwoiGt4 z)Jh^F@XUK*;BzP1t7GY0t6M=Sm2Ab;73E3_7ZzLx z`W7f5NZh^TQzLenr3TfbqHb1AuZ8*}&6bqGD};J^8xiM*Z)3r(@fNR}Om&@M2x?HB zDb?KVKTJ~vg+pC1`K^&fkyvc|Iu=9M;sAq)Pzl%a-%^(#n-LP5A%XBL@#Rj#2nIQ_!#5E7JO0MgDWUxJ}Kl6xk zoZAFA(*uAv>@B|E@tq!z=iIU)^J}bPY;*%=#mqjjj#e%+^K}q{?&_)%=r!oo!{{F~ zVH)=Lhx;CZVUnp?@l_TpN#DyX>!pU1cd*&&y{$eN?m!_N`SfWOZ&@gciEh3~<_}>0 zF0+r5`iUIs5xjUm8%jo7>hT%52ZBX47Ym73;>phhx+spGh>Z2j4l!H)$HhjIH5Pbz)<+F>4c3R#yi@XSAX z1)bejzyPjnHb~GcdQ3L@53}d+_pV6Pzf@S{X=*XWFGj1jPU^!2*bB&Lbr=8$0#h4` zST$M?=v>p^efBUdQ>gWM;7Yg|$AB-wbM76X)88tB67Zy%KRR zK^$r{V(eYwG~s^#uot9-fl#V*7W&~&kuOUt?NfJkX_5JJ)&;);$trC~d9+hpAiKCN z2K}|NI0^9!EDAdMh45?R&4h!!4zed|D30Yxckr#|3kLDZnstW^9;PnVRIbbmFy~ z2b>zoEl(Mdux*7BEe94Dvz@IYG;F|cW(vC@a)Wr@3}$)gSy{W3dZAT__J)j&f__hs zL1rJeWhRbmzD84jtT*}5<|p2COHCHU9nwKdhqqG{FJ^r6#aJm~n@0|f-M3L)UR_t9 z#&+qg-r3jpc754M%ls5JjMW;FsXZ~j&c|pLQ+iqQ6K_R$3~J?Mp+zpvbPh zMDl67{!MyxtsfapS#jo~i)ZUD?P~%BC|K2!*UpjzlIgMF%8t7$kIw)mW>a|y*b*>J>e5QH^L>@wmE1*?#k%ug1 z=55PU23F0?QHz6n@l?b(!}6<5RLZzuRVZsW)C5BbbRyYq{7mJODcKD{9nSlxbSdB= zyv0U$3-5HTr!O{noOI%zU3Wo{TPy|~5$geAwMl^+>8)GIo0EdIi&1tKhq^w~O-tS` zEi2>En&87G{r8IoXi7HX1Zm^7{YZt;Cy(>OX&e3;ml*KBR7zn!l|*q#Zyxak_9Eb!3n&h@5W=lJy)+^$l#(0 zYMCAO^eD$A#*xQ&42)X6~IGJ*6ZMRx2VsOjLz>fFkNa# zyEe@FarqqkVPEua)Cdza=Nf&vB^Xt*+Acnx)n(SSY$Vh8$xcY}nKY?y0k?3esbf_7 ztqeITg9bK228IM_Caqp!E$_1L+ud->`Iu!_B1q-ADPZOw$0+ondx-l*aOiZLMLQVZU=>OBA{+c?h`FlISWZ< z#7n~ok3^xKrCDboS!C1M9vT^L|3tcBv$roUXgM!F$1KkJ_o%o&z>w_pgC0(8c^NT7 zm{7v3u%w3bEPgDABf`~3xWiRKl{+-5kE>rYn@e%9!VE*_=T4O6vFCI2`fl(Xjn9;? zdDhTRL=#zhN&_Z#)g1oRiqx;kP!Qm_%|{%(sSLI4%VtJVOpkGkhF`!q2S-8B5JMWk zi@Z?{vmd*Ma}CvVWrV>5{JSHq6Z8HQuXJ)22fukGJ1HDw&n(`|io*l#BuA2)FXK#B z?jsEtknY47X{OJ#@N~FHpX!X@opXj9jOP~^GR0lQbNfYl;sGD+*wp=S}x zBP>!(%hHSDNaz5~9bP`SnP8AeT{b|Do^2L|UX>pl)JofuOX^ck3`HL)RR~dj)$Y_O zm`LKCB59T*joe4FG2{|r`4e!8hNCnS=Rn}6Vjy3%9cBt3yhmEQio#UdoFzvLj|iql znjeVsF?gF-OA~5UCuZe2%I^4gy(#XM zyRa>|_I50OF|OtF?E;Q%5Or^)z|UTK6L+g9@{@z*!Jb0srW$XW`i#@2LXEvY{>!gC z=%GuMmpZ05p~$!`-5lyZ1J#gMYzTH1`2K>a78cD&!6c*|LHm58o3pm?xm*MM=XhRTtjIpjxaGK<%7Q7eh+jl9QA1i(7O{4gdP5 zr*=x>DUt#GtRMUQE9;o*z3vrHj)}8y@L+#nDCT`Wz7gsTK}<91+h_o;rbLQke>;mo z&`i5cXFZPCBJ808KMMkH3WdF1e7wcxi=%&g6>T<9#I38!)H%z1x5Y19v4b!=6W$ez z{!gJ5^Up>tjwj$S=XFf_j-watL@cam~5anNeQ7~`xi{;dP$Y#fyAkJ826ub zD{|QPW#sbddfL(36{I^F#Ev12en!m#ZKcB{dvV9*`F-%4_MlTb5jiYf&sU|)ht#`6e=Rak&iE;G0ZQr)%-Ec9|-o*o`J~zJh{B{&d zC6#2Up%<@*P6&(D1` zf|^^qmzOuAgHRkw2s!Q!iZwGQMl>mOM7D2nV>ihvGfBN$M`4&^U0dt!S5AonXrFD8 zq2mz+>&*Z}|7uhi%DKlgN2|g&BZr&w(u~h<1YnE?3vmXaoa^>Z7%8W#Acd@n&J73d zrvC8Ke;HmFX-rsiF~NvYkW-Wa?PJ>4z?US&m=%K`6mmbIuuaX%g;6$J^=MnCQ=*Qj6|9OqLy{Yxq z1#TXm(u|pHIs!vfs}`lo-U$(q%2wX(@P6r)(opGz#~FmIjvt*yjttyF=(q-IwMo4u zkek`&w3THQ-<`=mJPm7tpu_=NL@yf<;m=CvJ8%(#JJi*c4E4GYwB`YN@5w%CwLe#2kxIK3=4lq3*v zf`hR!os4@M8Lof~@MWO2x@`vKPq7b(cgjcH5x9NE+O|X*AfJ{}CY#|C%8zB}ibmds zdJ7HO#;Yu#;!*4U71L^4a7WF$g*9}=P{FUl%(sB7O2#oyGp2%<>bo_@W6ad@99QQF zlH3QtlmSyRO9BZnC_mBys*=M=k*LOKXdznU@G2`jfkYOsSGOsdpQ z%s*2>Ci#tEar;2I>`*bF?BYF*ifONd)CX;Ql~M-PGcM)#foF>(FEqdF)HqItr~}Z8 z!5+iu^?^=uY*TVG-m24ldM=U?{Y)BNGy&euwVqf=W&L6^sRh%SjB;L!0|rg67XB%l zBNL(+aP{M^XrlxDmd2u>Wx=y-*j(MppJ_CD{x?QyiHg=HHZwvlYoyn(_xR~-(`Nqb1j|*RpoBm1M@pR(2SoMok++r4jQEaPbiB9eR z@IiF?Y0C5_@6qkuzrh1}BFLkjZPC?jz6`LW+R$#L4YbT~;-(8n|HXt87ak(uY_T5Lo)8o1_SSt3bUGG>ro zT!eM8YppKi|=vfGizfg*Z8l`3Y6D#TPi8w{FgUcKWhikXyxLHt>@t{=Tt za;fWm*GuH2cjG6wLNM(RvHYG1s+@~&tg4F-odpU4eu}(ndt@6wqEl4v9?YvZpbCnv9_gDnw?Ap$IDL#r=GGMYsZZF`S0d z&C4B;IgclNss z1$@;F)l#iC8_`8{+`tkOUR$imlMAnIC$2`W;eZ_HRC!xlJKH}>X!#Bkx4a**) zp)LuO&eleduG{w9dZ&H?r9#I=0G&lXlQ0xfV|DcG5W%MG&Ki(h8*FoPl_+#jxX>+C z(+H@2E)v#sK&+~yt)8Lpo^qP34(I&|J-nSRkkyQnO39QSs+#9^3N&ZENAg!(eLXi4#!zgCrqf-Q|}4=)=__h z21otY(rd`463V%YZ`&_!%Jq|Un(o<2a9(Swh^cj-SGlk($~8cu*Wv6SXX(j`snTT? zyCLpjghNvXEhesn}PzqfGNdo4TAnyt!FQk{B20DmUQ+rQ%VvL>logyM1KT)}>w z_hGZSq%W=CS$vc5VnycDR~skPRZ~6U({??Qk`6F@YgEse%2GUA+flEU%xEsBP1TI5 zCarOutiELeZ3P;B*ZV?dTmekL!)LYq+UBho6?0a7cGf)viE`tFWsYE7^qg|HlVrm7u*bUE&*}rDgOA>}zJ#lVpBoje0Thsw z)$H}$hnlbLq@s6V2~Ptf>w=4Gl})T7~1q&V|=7^O6SEV$hG<6>IjOqr6r(@Mivl9H@uB$M{=zRH{!qY$2R?k$?KJYa+Joir&AEk>Gu zf&Bg?NP6L!Fbzs)IE>_os)r7FMMIT{9P;5#b!2gwav+B#Vox zYEw=>|H2|G)II%1mo{I(@9KPj8V*LwL@J>BoO5>R1w2R*!QO7B`8J1I^Dg9UX&lA- z7Dt!o*WO1}?#^^ZuRT#!dGQ)sQdZqmk-pjW88MYCg)yMMLaS2$XQl%&#iE(j+r4!; zsI{i371`KkWV~S${ZCPQK+}vsz=rNiJG=dGNR|N(wCD|FHQ4qQT$$Fs;O*W;uM>9$ zEgp|r7^_nk%s|?*(FdjL;|ZZNH~F69w`)77B1W7SlJbJu-4gxDvGK z?ZC^c?m4-^L(sE;Y+=Bst-LhlWRW>xn8l4_B9T8b6Y5fU@k4R)*ain%h~MrJSeE^2 z4**)|*yG!}=&X_AClCI{KWWhYRH(Wp1{LVYmR;X^Rd0)^yBwNUF)3?Z zqgW%=^#1BqS3s8=vJNf|Zf66$US$7_{qwNL-dcAH%XEp-SbPZ0FjXPIaf7y5Wtl7! zCP7*aM@l7Cj1Gj#h6Ywk2VK$?h_C)pyMPUlPCQEd^f;+|Rr$B`^R0w%-J=+&$Io-y zd5&_s3*%)0QRNvR3d{+O?BDUuV8R8*fSy<`C9US^g#9TT7C!iM#xLw8=(_BpZJE^@ zLUb!c6CoL#I_n+ePZd5kSvi#;OJRuMJ@n*2=$Ex{KU~Lm6DUV90mNjamn*rAJL1!;M2QDs-x2ff z-jPr2E4*;&~bj6raNa5m%b8&0tJ^~KG*DW zS*xJ=IDQPGn`a~#5pV{4qu2$?3LK65Ky!};Wj)h`d z<-P2X6mfI;FBq>Y%YsBHN0iB`%8*FD$aDYnLnzbS7rpN@{*1z+etSO6Em4Ck&u^wXj!QgyCG=Ir!K045`#~tyz}CF9{re9} zKSP!d12}*;oGF}b8z;}LUS2 zL}I`Z_0yC2(e%>tby$ngo#kYVylRpL#ppXqQsDKn_lT{j+%Z8OLdX2DcU4Fi^~rkL z{pOY_?Zp4nZZC>rc;fcK<&rUvHsR+RE8rlLi6L63`HiD9zsd}O z^$xtfwZqv1_+HEX6-|(nRG8+i|6e{ej1l0l$nw#I0N&#Ddsn9m++bb@(#b6-=Y9a? ztvx@2sc|e>R6{c4Brv6mMB?M7X%y|3Tf?lYu*^enK~l;8m3BwZzWN;fK^v7sabv?F zSA3b7AaK1K_)qyn2{%thb9FL}LW!8nKR_>vO+8@Hwk}>n8q$OivJ}u;h8-04-Y+|y zM;vv8ev2fAMgABja_p-(@qEXdYS_E0qbdqpMe4ZuC;B-N1I?}H@=P#Z|6*UmPWpvl zS;#+{tVTn}>3;ZEh-*=Qw+VdTg-Vp3_aFwdT-)Jhpy$&FrDhZhkxV=~DM7;1F_PaV zChy6wz=di?!}?mE!;krLA+QNn)2Bz?1kzgsBKJcR^=p*BU@1{$_enw3!$1DZV*Wp@ zN*;jS7EO__RO;U`$R-a|{_gmLq&odz9Y^5z*4~DDh)-UTVURwd9OLV_7bOQQ^|h0L z6kT(Gno8@t==K)cyqj&buO`?;Ylx#IU(BDUk?1FxpqUZXTRZnxE$pXON2uMd!3`?+qeBC?pH}*VUX?z=mnX2A8t{h{)ZzXF`BuM z^nsjHi`lM7cN-2jNxMr>5*TV7DLu0%MR&iY;ZmS}RyVRRLCc&G{{@o>rHo<9Rway0 z6ZU#Fls&;CjR_@{UZ7aYgd3o^C#t|=YiL~a!<9P+#As&HaopMy?T<rc(4m4huCY*o~+=weV> zQe<^Pu?dCdF17FEBE)phEfg4z<~fPq2q{;@8;2@#0!6!8th#&1mmOHLD@zV9SM(XZ zyL*87Z`z%IOxOn&t=$w}DQE&RFmI8Ha*(8t2p)D2iilcf`ua_oOx5i6We^4mD(=Pb z=_s(sE!>F{rv$lwWV;rMOF}MDbmO)^U$nE}8@)nQtQ+tZSj}lN(-JE7*j^TSUMDO4 ze)#IUyElyzy)TZ$L@9k6(CD-aMawAFeYuMnf`Q6(8A{FNpcD2%FU%Q9y6&TL_qKPA zP2>66zcj{%&IQE4UQ$c!==}>;2+H8)L$BBRy02HMxD};;EQ}_?c~*AM56?A2lqU>E zD8&aU8ohknl7phxQAHi$L0c3W0l#m~HZroL1%1Xj4|j!SRRcgYo4z(2oMoeO(d1tc zV<8>b{7*K!xUm-Avb&W74o4h8Y%yem6OFmB&pnFbCFY96QN>^z{<9xV;k=wdeHpQC zMXNOtNys_|q#_<*f%~Sve>Nqm5vMp+Hh#TnCfV#O+ z)Ji_WVYVvpHc^%|*#Iy)w&L(II+=tK+%pr(0)u>FM)(?RhMig(DAQH5eRR4U0NwQRQExF?rH0&H3nIZST71QDT-{f20xO=#B4W#n0X4U}6DI-!q|1aK6xsKAiB+ zs7X#bw3cLU+OtnHQ86`7+la{%NZ8Um-4zWymm^77r&S%Vqz9nGa3a0D%6fxnL4)qxXwlPiR2%@@|W&;8ar&da1$o z$<`6P(*#)7S#J;MbvBCL-0m>z3teT!fV8tTyIluH6$aNt&r2Cji=IYM6;dIxh}xgO zn6f;bww%Ex%^180zrq*^#479`#c-Td)Q6j{kR~8vhy}1L-md#0hUC@->V{JWyOc%z zhF=c0R*}cq5cU_aoWv|(VFgU+0?rUHN~#hZJ1dqmhz2!~<-<&iNhj~1w#4v4d*!S^ zcPavRYf{Zeww}2OrTn@HYB2MvBN{_7cyzgK;3)69hVwk$CqZXBP$YS7#c1Sf?(%Zu zaFz>!J(EwS=;8_$(h0wm1*Y60it*B^sQSu)S@RO6cj!A@b~i6F5X3q z!}$RvKnOO&u+oFa11)6Sd1rw?(ddH#Pku#B602oJH7Qja+5xiHJa2hA^c0>=X-yV3 zyf{W%wVW(SpiC4u$9Ct!>yKl^gVDo|G_OsrI63`b^L@$!XbUHYq$LAqDhK3n{h;%d zpqswoEA-hwswE#XrLk!4=gK#uGGQIZPNIMCCKI-wbTzaQA8Cblp)C)mY|g8ECP>80 zWnB?yC>a|N6VJdGH-KiRaiNyo$|f8TTq6j{iOipI1Vakg*MYHe{mP`x2yIsJI;nFC zpBDwtN*$%oEN-oh#)bhH;3`zH8E0~vdPF&WB&*%_$$y$;Hm}cQ{{@?&=vLa+gTs&Q zuV#gabwt86gZ)PjJGt~G&8l-TJgn1&MSTfZGiS76N&tbtsYg~KoQPI)j%}uJwQpQGB!xY~g9%tPoI`Q&L zXk;L2{&v4BH-F%(ovQR0RU`9>1O4b3I>;{hZDf9&O-TqNnHV?*af}G48mU3eU-%65 ztF}a&uQn~*S8EB7V<-ZJ)eM~J^WQgP9i+B7ANrtSfRU1Q+(00#dND*1@vCVQ0Q@P- z1nB$J%g42Wmwpqb?iCowS-i+4<*BIx$f{n;I(FzA;-pfk{vK;woFJU~DRJ z*R+fO8bH*;08$%}3yeaf7L-1!TBj;#MUK19>vK7dF*4ce$s(XEyYdt-Y^EXm?EUG4HiK4PY_FD}FeiJCz>wr&?l+FRWT_+i6qZD}-ol~POCegQZJr=I9^Hp6w z&~KbDvo=Dyy@W+o$>3=x;!Fa=bO=ZF5KPQzv5&39Zk=lM8gz5XEmQO7=&rO6@$e?t z1X+Vltx;9XkHnW-t%RwD&q z>wziYq;NVH{-{y@GDFq2E2Cmy$x?wZ1s!| zRhAU;cA#D-2ewYov{v=^MEDx&V*H;Cb9T#rihcFi@tgs{;8oOR1s5pgA+qB&i;YX9 z7+B!zbktC2Ds11r$q}TXBLL*<&2BeqhnL%NyjF{mtMKQXS`Z}b?zp_TXWbZQ#|hQj z+LHL&ux+X6)#RDYbz4i={(?CiD`r8xsQ@1vc8&5mlGYuhAiXr6^a~ustWfFb5yeP@ z;V3V0aeng?;Nc8+syE$4-ZI2&Pp{>`%+Lij++e(#L)1sJd7lgGC;K`;utJ4*f_H=; zQ0Hr&e|^2X-5NwMmXGOR?qz}5rg6cgU$Mb;H#8((qnV*4092B!j~XPm&El zZ=HdyytLTF(!~C}kZ2|ZA>kAw_3B+CIVna6j{@l{K;}TZXb=#l$@k=ZI7=-i+hU*% zjiLdArwe_Xr0U5*mX>bP&)P+5>dr`Q2AUb_o>r^pj3!#E-~|W5`0(G1AE+Tp#TA|M5>&TOgB{MW9{3 z3+Wljes~UXbcq>_m!>UFRu{+Xt5@2UkG}6y&!B+Ka<1q03tjEL6jhOZZ3CGr+fU!9 z-i19w#<>`zgtWZr2~1XsBk204J4-3C>1>!YDxe^6N`1^q&EX$-C0H}5&K;0rL{QA~ zeXEEuUrb!bAL-Vi8%H764U9zBt(D-&(xVE$5K>Pq8(vI%O3%B&9%EiYCrhu(3C2goS&UuF-4pI} zvS?`?!h=zLhml_HLMN{$DlfaguS*FP>A`1(WJ>|@F6L58=(iCWkY{n_^Rp zH>r>lsJdXo%&8KgxVlO>)JaDT=hD@K0*kL zI$HoY#v$9}(ll0Cs9QRgG9y(8@R)U?&;6h|9;#%3n5ies19W?>Rv{`ONI3Kv$mZyr zj8coD9cJ^A^J2!A$4{ zO7g(Pp(&~inVkpOa})N`xeC__Y<0+X^wj#~d_;ljZh2#yoh!^vnm&{B5 zswu_sWZ2C7aFI}jU?p;dVaVkx#|YJA7?f*|(}lAHIe!eGXYS6}*WSn9DvLB7GC9i6 zP3!PYa_i@J$l@k0H>DCBu~T8mJl8)DC+%cpk-hil3YU6Pzai#^*B;-V_kQCFY#u;K zJ8v18+!fJ-ibR8vO=k zLC)X7Dwk0H`PnHC4h8Kdyn6qZj|G_=P89|!@}ZSA3`C=;s9*$zLcibC%4N}oxGH&m zL+$2(#P}5ym{x2haw1o+qe>DFu0`npyGO8Ffvg&@VBBiNUNcK8Hgc9Wt_RE*~@&znL778t(2A{-xSVNSUAw%XX65yMS@N^fqdxbh}HV;J!kcz(}7rZ9kmn}L$R5Ubx5}UYe+bULhMCS_>UfPqB&1iHrRBMsZ zdXPPun5tLW((@dQDL8SIJ(`5})OJAo{mUk*)M#Hj_uWHmcH@&} z9dG{S)w9Qv{BWwdz34}m@8(H_bXn!v7(L~;mjnXC)}U2N(is(sKOlEnX|0N#VUu<= z(aMsR$RZ^qDPGJ-G+D6kYQZfnGl@6zf$EmxL%)jLg9s918&qw0uC;RR`{f6DJGq+h z25c)yG-9Dc=dvxo7%SL-*8?`E;9syhZKd2+>QDC$HE1z3ST^BOl_kec9d~RPxVw%@ zK{T;NOTM^diX0-Ls9`8H3n)B2(eP|l?(9T^gM6o`5o_y zCZ5$qf4q1ouLE<}NynGA?Ji5=Hr{u;wk0^h(4}PF8oy$>j4qjAESXE;>%@Sq0!7s1qWI};0x@riPu8ia;)*iNGIcp8 zO}LM^!)gEJkkPj!-~YMf$z4&>Q#y$|Qn|OJu1}6yl3G`+I=eYKnA36Dq|GJ}-7{`l zSWPSGJj*hDmp7O-rbyEq4tC5@l?}81na^U#kH`UHZ8HPpd~yOt`wlTt-y5#;wMM$B z8Ufp-RtRQKq%TY9k`el<_QFHEjs75$#0BYl`-}fA$Uvw4 znWBRJwvx$ovC?=^vAM(Cx{u%$3J?Pu#B@8RsCbMsr7W_H_cn^QpBuRN+K3Qd_{AJz zi9U3+0@h0A4iVMj04M_bMCJNf^u;nG11Wc~z+A=R{!~77BdkFiU}+6>kSzB~pKfU})l zgK)8O!mK94?87|)b&)W}%S3uJW$mkt80ER2ffrgN0$Ds?U-UP#o|UntyX8WF**Hska5 ze{7R;Mn!>gu1z&qf5^kxtlET0N)588bI9FWnc(v1-MieV6eZ{leyRs~u>%%Q^{5dP zva`e|9_6YN;77OQZMM||4hxJ+Y;JM@vT_hov3YdFWbog5kX+-}a7uVqZXu3pj-Pgq zB&^W4{rJ+z?HhHz1tFfepX(uB1B6rCss?QszchVgh?7yti%+FhNv2mksp9a#fC(TyPzIw31`{_*UMPpxW>8#< z5=Jp!lz~uCHYgy-eEeXZ>v9!ed+&O2)jD6sj{V}BPm)PXy!}Isqf3TGc`er@jVTSS zDa}568=1});m=b{ON&^H=MyCQJR9e6EPxeOjD31%0);BShI>7$PbfhQXz&MGeC-D7 zuo0E3WXFRqaARkgldawhCL zABX+cknDn9&js%yCcV^$EiL)60CUf7mLb|-I0FjfQ)+Hm%Bp3vnilm^Kt(}jG=_N} zFGT)T2L)}-Odie5o<)6SbA5%j)i7h*%9# z7d1ZS&w`I#Uob@zhS}<^i&f~>7G7ip>CQ{YFdO=OF842<6$TD#TUF&A`>$kD{|72T z)xJv&wv_jT1?osxZNz}s378m!g4|`Q_IKD`)E4Oxm4Q>m-YeIq+OebB;p)0)%Gy;* z8IW9Nx~&p-07LAyqCqpHfTa^KNt_%3K-?QOoKS*1Zr8CFiIBb*AcE{K)W3I4V%eH- z{RKr+bqJ?JnX?Xswv8Yj`2@NYRuUReT2cv26#;bIj66zv5=E@^?qR0ewIQtbuI*4X z)4tK5^WG1xq|9n6D-w?r%4Uw0fJ&)Cl$EJK5=Oc2RJxVILyAyBBG_uBIA(C#^V^k} zTe>Fo#5b@xRQx%78kUTAGE%5ZN}@grT?t%5(&~_ckIE<$rERIQp-`KLd)1X25+IrG zs)UvfMztGTLfW-MF^To1=;r;}0B!XajWfiQXbE}dBq3y}Ev-dMWlAe>qfxqnPK70Z zsX#*SbyX=%C@=D0)!UgH(t<1_-5vh`5_XL}m}Rb%sbmRCd}1J%6s<%O2|MkyfFN?i z5A3a1HLj`YU3quIkg`z1fmI{U?O4JpSyM$2NKz6((hR{!wT#Y_ZO9@;?}YWHw7vbK z=C(h0lZtNYh-?+I;B7`m;Q2`fF*guGqm-N53^j7v@4wE)%hlp(Q@9)vtsA_HYVKYo zR2c;Rpg@9^6CE{7kYd6w0yn?`@9FY4*Gr4IiPWd5pdNnFZC;e#ObHuo z()ZEaZ`5){k%g9(@<5FZF$(&19u`>}vgDT_i6DhSE&5uuv&$ zQJ%uF=$pJ)&Q+F%8dz*42VKHc92$~z!Tm`|=P?(@2A6=Fj>MUSDcPzG7!F#ycMqmfOc3GC}D&wF$28lQ!-s-xV{y46i%(Eto4LSkUMMXxSZbmlZ z5hg%1VmUJ!R+_N-SVttBwc~Nqid|e|hJ%nLnuL_r_AV$RQTB1dDpz+<>q;d>Ga^jd z{KQJXREgR#&wE3f6r{@sTKhq|8ohb~{NufPKJ2piB+p!@AUXDJb_BByrLHx1-p^2K zsH4l9NeX$!nMedqrL7}Ln+=FFU{xj~8n#yWsr3f7U>vxp=U~P57ZKI7!kh*RgNDuz zEdVChuTa_u`VH*wix=59;gf`TQjNt^RA*Fem#Nu3+SH1Kf~QoZ&XSa@I`R?;Ctj5a z(t5Wa<**?k!Lw@A?|Kt%MzQ88cpvXR(Bg5EK|hwFO-Nwu0W9Zo9-QrP{6BEgIH!$T zt_bg3O;c&6TZ}1EQl(TnrqrTCM7R^)mn`a1lq3bX1}I^X>!(y200qenTU65TqgJLi zoZ}h!46T71F}nvW&IkpD^>JYy?7y35d?(qZHc7%1PB=3v6)~LV`BITkGTL)3y+pvL zD|IfrBvjPtN>b96mfY0@`d`roN7?Dv{{Zg{<1oRZej2ze9N?L9<+=X=Yn-K&rC|+8 zPE@5cf*w9S{yDpQ=}!xh&g}1FdoQ1>E6-KMo%WwC9=>K53B|;mzX>KtUM&+brCzfa zH7E&27)(U9%b7)U8>&{KB+M8}gc1ykCw_1M>l@?NYGq6ISdo4gI*+51r8(E)>d>?+ z)u{?h(`jl+3pNFiKvB397bAGJx^AgA8xQ+MjO{*_f;~!{E>dNJ#Fu9$lef&py=(DJgjfN$n2x7wUl6HBi>Jl@ z^xt@MI3$L9zKzYfJVaR4B%crT?|6Z|?YSm5oE&9_`jR~P7Kh4|CX%H%NMJwWYmR^l zw)8ZJpuiJiWc=@MIpybutKL3zi!wiSkMn=2PzA&tWYmhl}#Bo!Tp z*dF^}g!e%wnLN*y(YZyu?0pUuHyuPn4obh{Pf}Gs{luTB_QJ2;J>MZ286UbwRX;Cj zMz!;8VjH9to6G^XrR4KJYW>sa zQ3f6Ta!WtF<)|Gh72A|F-hAhKO!fZVaSUW)I*#O9=nwhAD#op9^Ll(?4FS`iJt966 z_%DjUB(KEvpEQvSNJqH;+5>!vF-- zu!!nu=g*xX$|gkLZ#V=KKKpyd5;G7NkARQ;Kp%#ye7<-fh-)ASoBqeA&VRlLLsH!9 z-Cvtho#Egnz@DFfx6=d!u=BY1?Gec)Pc!x~L=XaG$-x9ZKtP^nq>bkP0LMHb<+0Vi zK1Qa&@>qY|qx|3MR0&t#a(yqn`)}nr{{S!2G9zkEmhf9Cqc zf5rZ9tOZ*i{ol3^;r{?P(h=y~-_O%(?*qBp@9*x|LDBs3^4>6zf@0zi+;M1GO)^*E6p=0`~0vk-96ti3Tfq{ zm1RJZG~1u)1D`Ty<%uNwzIQQ>#D4XE<^KRmVaeQEpYA@EpYMsH7$BLs-bm+bN8g?Z zAfg3`jlna$!TCfO!43>4LtGN~s6Pf`&|$wj^q*sG;NJ*sr8jQzTR5am1``+F{{UNj zu!&}%ZG88FJGN*c#z`g&Tj)9a;YGYfIT*)c0rjgaTiL4?_fjuw(iI=5d;Ja;Ac+Yu ze2)JBJP_oTJhg*hiJ<8+CL;tOr7jPFpM+ybVnJ}%d-!vvyz2!)x%0on z`J6gto&NwxRTluWRQy=KeOsZ4V1u_Wxc>ka!Z=EtR2#RR{{Vzd25j%tK^-~IzTX3d zP@q+fJlf(2a!i5q^~8iFC4m9?in%P^EnRI$#w3ba&vj|@CY-6H5gYq|d?JH_Jp5ts zVYrUCSRmrj6VP?R1UW>WrV5FH;Sxr~b+-Odi4m`N)+%D$gcd9~o|F&6+#V(c0#qbg zIrRX~P0V4*_eX!s!cHK~TQLBon7z^tnrYn!K@l9s^M^!nC6jPHPTpO7zl-7-!P4Q$ zp!mO7mO%n)2_O$@ri0xu@JRRH&+c%5x0?5D=SK!8a?gYwQa^b<+_sZFMjnfvpHbxB z=MZ8GAQpB|@MGuUtT_blefs$jMgIV9qV~j*RhZcKlh1ym$cYF$j+)5{Hsox_7rEqyiD!(RxLf@asW$5B7# z4hVEy8&v7PBg@n0Hutt11BG&9^NQGmS!4cj)7?K9n9k>|qtr#Q4g(>RE2C6#I;_kxdN(yu!)Vy>@ zW&Z$&j?w!_BC>}ocO~K1;Y;ES{JBXUN z#{#jR_KeP#Eec>$`5d)qm5G@#-2PTb!r@4$T8U7O)8YRB!{=nZn9~fyK3U@|{iKz= z<#M~6rkT>%D@{IHg#;9-dKAi^Ej5{l`RN1JV>1F#mtjc-&JQk2VZ)Jmh*Z)G z8HI!SsrZHM!-&5O{itTVYm`#eW<8Q|W@5KEqbQ+5mYyEWR-~7+yuFlahAN76*V8o# zQ^-|IV_0x6%4+R8aaoQj6Q>#I#1dU<4a835v$N}?X&+~MA?Wu7o_0qsz)Qo7K53L; zaJXqpN?A(6VOXkKXy?=q-2fcu4Pz#u{7G$z|Nl-~jz`^F2e_F3uEW4I1U7E$H`tQ~@ET^)#tO|^kO9Tp*Nu#j# zhMoF6YTVbf_FH(4kWjS{6iD2_2{Iss zD1MT0%yZnOQYX$J5)CX!I{-_*qV=iJ$E5oqm`lvE*;3Y95h2JWfhF_?rGW8cXvK1l z4?U)FHw$ouBH?^h#CbMAhx|lSHJWBr^p4gDCR0k&>!@-BB`l~+6N9oDE+e!ZrpWRv zJBgXDKLdipP@W&n;=L7t5 zvTTnTn@p64BZ_3{xLIhHfa;i~%K#NAG3oCSy_Y#ySjt#UHcLRo9$%2r_iR(r(B+vn zUgb7bmnpX)3^kU~Q)Q~AFP6I_0^W@kW88%UuWw==>k%PeDaF}d7v{@Kw z7;nnqk^r$n2?`&=UD}cO0_|sL^FK&^yzMt`Gt5e{nTKdIZ0|K*VKMmexXIX*;xQ)> z^-5QmOGNoaY$hNjQr(_B#QcA!FQe>(i1>k8o36_coX0D9z}jmnRB0GqhZ}ocTP&B9 zmy2O-iEujOcX4XM`bWUvb1uYomjjQKlZKy>=1CyG3MM5py-;vit!;*wgHWl`N=PhQ zul^bH%=@#Sh)+ZL-(_)lnOUTtNWF%gETJII$7R^O*CfK`l4l8njTjMu$K!vzB)U|r z7B+HNl$Y|;vd-0m?Nd0nRENr}d^|I0`IDa#6Y9~^WLhFvE0*LS>g0e{pgtnzxWpnC(Ltd1|7|QJCb3cm) zvQ8TnF+eI*#R|z$Q4K0%dk(^0;y!f#E&l)wNbL-aN2DIc;DFC?(uC~SEyE{ zp%c&$AN#^eeUz*MxzdS3{l984b~tjfwVA4qT6988Wa%)z@>ml zQd|To{IK*?@v}QHI$Hh$(A1t1O~Gn(@@vmlj$_WaXFbU?3W`+GAopole>kWsQkzPa z%WAWl^#G-nG*YA$HkfQl5Fn`5t)_*?$Kl@-AP|sEFLGV#-1lO8^%1K$qbg@Kc;%E1 zC3Q$pIh}r`01N`8Yg8#IAnFbn4j>5e1!<80tsF+@O84Hi(vJ@eLyK6EYm>cnA2J2> zcM(u9%g$0|?FUm%g`qkIq@*OvjyWJhXiA2s4c<}G*a%9rz@+dUxCd*3R^^tAt4~lUbvDk#A+)`_1CYF77CR{ zRSj-kg=ptq=J{aN)JjW>Ng9a?TF+QaBzZ=VL5qdRk$f_8Q$PjWovC)_cNkgaxVi|)~CT9TqXYB$?S8l`T#}(4A2M0p?c>H5__D8_#&GQ<3=QC)omYu+=Hk?d{UO^}G3kUxI zl#}{5mjE}6X}zk?6NQp+xK#l${oq~0a-|P8CiG_Ro{4)4J2=PT^PI{^I9VRf9$7_k zc(<-CV2|9F_ZR`TgVAmsuWKixMv@SuAtAK@0EF$i0Fq`TosxIQ1I#6>8#8` zqYAlgd;R^9u2xfplfXhiVqAbLu7nnMpxnj7DW&3$)p)Nd;2sZYbve#-`dcp`6{d!i zt+dovzo00TG?fd9J>rod$s?zFG4y(SMV;W}s4Mc)YmF*NBZRpHMVJy2ctLI=7&{pA z7vit+8HMyi*bLh=l{xb{#O2vsps<;u=pj5sd0;a7?n!g%cTmkvH&b=abr?efsPi=9Syu@Nt zo~~-KYsL_@DH*ZMKfHvVA;O>&!7R53lCl(C#C+oR$Jk??mi{EWU+lpua;Gccp3u09 zD#)S#nQ4Bk(OJ7G8blN*oAmJj%Bcjb6l zj{^k)K?HAa>^LZs%|ZUY_&s#L5_45Dud0EHQANkYJ? z6q1z=AY*W38ABOW0u~aK>2C8Hia-Xy?q{L4ICeJ~GaEe1Iw{-j^t;95dO(Q`0sTO_}jQ{{V^Vl2TH(GT@j~ zZK**1Vxl#^lz_-WRAm>4MiCq#y>KLw<8~tK!j(4C@knL!^ZD+#GAdG7kX%zh4Zo|$ zr5}gS#>Qd8Ow&@-=Bs@OLRPk(QFXVLq-k1TLdix9Codb`VXij01)bRHdJ9HT24CWWqmPc*+G*oPC0Y%sUZGa7DUtn z_v2Hj1OEUI4%CkcdT0C(dOhuk#IF_EX;>V5)Rdg#CjS5fjUmN2v=-$kFWXLn3 zB<5*3Mma^6zuw@RlH92cu;M(#e6IAb_#E_&*nx<~c4A@ewo8Fa;j*wAHudSe! zn34bqSeb@Ea150Z#T>C$OI4Ruy+L13QAtxywKnZ7Q!Z1~GPJ0rNO_l1ml{wYjV>iB z8xd@Exl?4~gn_#hcTvi^F zaJbAo>^3G>5|YMbnW#j`VALnpB|h)~P?&`3#1ajFf@aCyGkzft)9#ir2_M73P2YyU+ap%pMj#vk^cle)??j^u6RH75- za6Kgc$LENZ?;kowLO>!?0R$1nLq_(LVj)zJNh7R!m>jts@Yt3DvtfTx{Qm%a5RXy% z)2`j)rm+##q@aL7xSwtR0Bc}|DL{q?UU3OVY$Tbuk9gb4-uRg`0G0uReNBbw5%dJ~ z4r$0jp!h*HiTM-$aT>kj=Saxm#|)&nZ+M!2#IL8{Scaksi53R&(+z|-Iz(li;^HG? zwTH^&+VFP42!0y*{R`TV6p;$CHW!VgAMbvhCk$SdW_|fuEaCqEjQ;@3UH$ETJfEC2 zn7$(blG>QD5st_2{CS>36TuMOFq40V{$A6+KA0h^EIhw=-s5ee$$}!r6c1R`Er{2r zf6^zyI1?#Dz{hsu6!EMzUa__PE&bi{FPBP^KpFn#9QzqLa{RUP08 zFb~7hd7Lqi8~490zMJb3#6_FOPwjyK-Pmn+`+VXWu;L}Q*3@UyzMDYI$mx0b8~H{V zze;$BmktlDNoo!!Hv`#LwuL55_dgRDNXW!@ln4TMAjd*A11H>If+4S!`rD@1AvHbE zPP*8@m9g|cOb{x^p@3kBASp;x&nyu%scBhP61YHSs05Px?A7Wa(+L9edqu8!&sZ^r zkmL$e#B}LLeo=BjB2obaA8Lk&-Y^wj(kFQ)I{bOg141r948S{mwD8kepeiOT1&61( z>*pA74EMX&&7lb(Q~GoL1OZkiPTa}mz6ga*2?V=uv@*oWOXw2K@2elc|rIaZ*Ev=rJA4tea_YZYe*tN zAP!@D^5q_fc$;;cHF|gbAoJ(X)*g@4J^qIcN4z`!MhihfF?rf{Hy%@eE8m=92qNH) zu)Q6$ZT21g`m`NCqyBGlSI~?X{n5C@;z#*06ZEONR>V_@LyeGl2e1RP9ULFe)Nj1e+D(cj%zG7Y2Q@}Ivv5W4AS z<~h6SIca#9%7keq@<0*qznLZnk--ZGy`W5xJ-Ye&V1g56E6+?Frrv;C)2;zw@qOQ^ z(X~Ga#^M3>sw`Nu9SbEc+r!de^`2b)zWMwn1SovRpPVr)XlnEz@^T2}@;%Qf2XXKI zI(^4HLk>ZA>Q9gA{{RRwNfRA5>+RQ=#P<~)LG$s4!BQYw*Wyk64=#4T5h@A}%vyw4 z+g}rVn4>aSmPZK;r=e#TH@O{lHMBSwJpH%%`QgymJ_sP`B!S5Cj+~E|_Q3=hFi@n4 zD!GW>RE@m=`r*mkcx#WVI4fz>jB$1r}o9axK=aV+d8^o>PBe9)5nlSdl*H z@A;UcI3HM|)b?uJha=1mq(P9Y&Ewwux^y$oZdR8>PW$)~^YhY=B?{88dT;1{BAz}Cu0V^Wsh2v=UW!XO+cB`5rDd7x~qOB~u-PUGLmA94UNtA(CS(zy?+QCzQ z`BsCkX>(SMC{Y^UX**w$c3ThY^K4>$VGl10ET`AvL2ucw4jhEk9yNga0~_a~FGHQX z^r@N4$~!HQ;!K<@<>B!;wlPyxm*px*_H!xOsR@`dM1lVR)9|8l)5ZHv2Ct}Hm;M(! zHteyAJmZVpr{g{AN+} z1o;?g!zIep#WHZq;Kbqs&|?;o2>|GyVb8c21NFbqSKy1W-HujSnP+%0+WynY6wNak zMs^PZwE`5Zs|_$MGL)e2Hg6o_6v|q)%{argsya4}Ju|FRw!#vY7-LIKA+(rE8*Kq2 zK>;BF2}%G-2T3M;quMUrlBvt}r-+c_hy^h3bG--D$J0K@dM))*u%q&YD3f$ZLM7p3 z7NC_V1_U`Hg~dbd9Cyb3sZ>{&wNwN)sS8PR@UIYs0SPH2#7PQ*Nr@aYIPFyF^CpE) zBo}JCm(r${YJ+ngp6s__u=rF-Y&}y(C@eWz!S|+>tZ~n2kJ53bQCXGdETX!OskfU` zZX;L(prO*Bum+`rsFgyy#VeS;hP?iIC-I_U!@u(fI(6pScSm^a?pZ4>oF)d6`l8O@ z5aFw{^)A)$16n=PaIQOQf{3q%{YB z#4xQjG0=a4{Lp7TwB_C5arQWChd9TG6toWTwWA@f%5uttg0v8}bBy0DaUd-xT2N3@ zxH#TF4;_zIE7K2UW=@oq$;)wA{J^E0Bw?_4v|&oN3B}F9kWCyUJ!8uM0NQEkGFN4P z1%8+D7+g~D*!=eik@jB+GE%*yjNdxSCF8h2VpMWz82Se#30j+lDkHqJDyEx5)vX~Q z(zjVurK!i%+S#LKu+cWQ&~>JjG{LE5!BCXG=YII7K!f$!QkeR-p&(lCu9qPEUE}e6 zmdz6IuftFYAd^6W?1BYCcvgTz7E;nn0w2eJ^cSz4+o=^P3)=s;zbSZRJD@?7M8nzL{2&c{z2WGfwrMhK5&7gjb*)P~{r z44GJ^0!aiFC=vt5y#D~R)3Or#U-*v6R;eQqgzUaSndhi85)~{+!uERymSN{9UgB|5 zbKG7EbijC3Eu{qn@rUtGvwk?s>&($SwqHpF+GiCmGbMHqYounFX)>nGdHRB( z#X>YC>OY{ob!8@LL&SMv>1xYwQ`NHLYH49E2vSOud)+#OsQ9HIks#xq%2p!*h~q&i z3~opzxoAVyji`=?io?mqNtrHmv?@_FNFck`g0B3Zn+U`5rqhsS$>HqjZ7(jWDn>(? zysZsF`=HiSRPvDGgWc$Dsuh4r(6YkRd?h~4^fl>5US%RrXEU5Y{KTMBfX7T&3YZ0@ zO)4YXc4?-7pL*o{nfzS-8YX2kv3`*|8-NoqSgb;EU7*VFQk0V-b}Qb@aMI<1rTXvu z+_ZwEF_@6!f+S|yW3-jE3ZQaN;Rlio?N`r8h9BEIX2mwnIBS8T3GdqED(G*}@McI`|W%u*tIYihsNL4dxHB~Rs znMEp;6r{b~mrk^p(p!~6bjcyyYTw{I{{T*}#+-jl)<(Lz9EU4db<#>H+k)bQ+7JWP z3k0^(N`lZ-H6#V159tV%I}!2a^B*{w1KAhZnAW7Xdku|aop_#tt02iAcQXicGpjv?LZBj>F(1SLOpysC6VMU@QO#DNsQr zI(h<>DII)1Q;25&01fqG1J}E+P2y5b4J=$ZW#%d-Q4jqsr8);a;-aq>N|Xf@R#BUS5g!y{TBet!j7{6V?)Yoe?9qpwx z>$pAr8K5g@X_DG(0a4Y`#DXBCTuhnv!nT#=Kp(!mUst3nF;J_fss+8BxUca6^o*Z{ zIBCr4wXXRKO%qBqD(FcdN@C6^Y$YimoMCf}(KrbvS_m`E$RY1tPL+Kr-Mw}v zvQv9Wm|^D4;u(Gkh9N2iD4s|F@VI5rKeGP-0R5rDbZq6UGK3{9Hq&W9NDvZ039#fe z>^VorjpkU1xS5<&4-@TBXdA;_ZK02D_ICv%DZ|Q;nJJPm;nEN;TaXm4nscelDs#8r z0+yRnl%fQ9w4pFSAc94d4%>);ZZT5_mjbU@Fl|yf3w%BsMoTj783;1v8@M1`v)cT; zdPk&x>2XDrRrayRRW)j6t2WB1xGA>GwrVNb&Fc7!EHmYQkc3Y~>R|N{9+q z_X^VF7C|Y)qo}5DXuV z*y5RPSgF%JVF#HmDmOZ6+ZvOvCgZaFtv1 zNMs>I6o>-$Ccu~@a8^y{Xt0cE*xb!)Q9MA6sdGo8S@gZ%NbMOPXZ=jMNJM~**W;52+fpDte)4BfAUT2wBsuHVax!#EfAkW+~!-SgX?D*dMWdsc7D% zWFC=&d+cSm#fOWTfbkX2JDd4t831UR&%%Y|t|72Vtbl&Gw>Rtn{1WEckxm7N(1 z4K+kP$AgBW4o~)GJF(bobgmhgR2j}lQ2_J3t~IIX(d=j8+wq;4WiC@wW?6&lXap2G zhsuIfg@mtKl1`-s>TzfS6cXLSO0l_RU9B?Mu41oCv1?p`yYsYgPTG1LlZ-^w;qc)k z9Z_cDU`$cM?fndK0P)56uTeW zJU%W?f7$Fhu?az!D>RoU^7)Px3TI5bY?O(lSeZ)WmUR}6Qtf~6m&ber-A;YN#Bt9C zB$b-{rI#GeC>uT@{^da0R1QMD17WiTe{^Of)7B^) zBE{PdwGbr^QxpgMU`NQGOuvLK-}*h7ip;w^EB^o!c4VX#nG&WeKbEAD3b9fpB}v32 zq?7Ap%ON74l~cIn2bN@o0wCNG1QI}k2(bzT5fd;4@zlXd4ipF_fdmj*>_zB74WsQI zL0q|rQA*IXDI_GNNm588ummVLDp6oANnjoT>F}tL09cqYeizLr0v z5glgH&d1o!-w&UjAN*hC`oRz#I+vv&m=SJdT>L+rIwDh*Fms~{yH>+*ybSW5K#zGR z_Y>uaRqr1 z`}D&RM|xWH-Y-OF3D_QAb8&`oD|6!vQivdToqM$T#AQsMU+w%)pSB)-l(whO>G6gd z0-czjzr+r3&_&J202=|$V|(5@eQ^=e+WBW2>$L%DTEc(4{{Scd04S_1T!?OvR$}lX z-ETg=xFCqDGJP$A2olYsukH3QL~Awx?)}XX*SEO+{{A>3I2H&Y>)+P?y${TfZ%iOA z{vLi5YHs9#n272(^pUXG+o`;7u^0tv&28QJ&^u{(OTBD#Zv#``K>|@DZim~?&~w7V zD3NWCcRqty0*Qibd70)p;D~PnBt^C$OkDnw06PQxgMtVewy^?E=kM!=DNSvi_8aX9 zdwBh{`L@-HOu!vvpZ(H7pEv{3aR9oGg_!S1$>H2fNFkZuLi|d?s|X55T#*VEQ7hAzUD({O3uFM(F7$Au$*m_`zl4v~LZ3SM_(Zm#tPZkA zKtPau;_-qOCQs}^-|ze2f)Q#S5Fi_qGZ%r>k?U#a(*OpyJ|93dzTY}{zxRLU{{Z6- zosUl;_wR`%>)rK=fkaunAn0Gx{Kv{K8`#9{geW5ABdPr&L;@?J9sIP3Ai)Qr2lvC! zt|43ed^_475&)2$#iUz#8y_x^H#Xl0f>KHil;X60gxTxk5loaOwuCz#x*ks2{NXUE zuFW*qwK?R!;$nWjvi2J+i_iuy{^ZIN6u%4%AKT(IK;`+AM zEq5>sY4~T3(?$5PI_H2Ji%2%#v`}l_kL*75QuhH+^2QbZtYz zL9_w4meIZld`v+|CP@i0qN5un0PnB~0F%%S;}cOp0t12)MUN3A0BY`CfVrR%j+~~L zwE~ioo*=3e@S<2!)y4d%NF`<2AQD&tW4nJ2pM)IWv_|Q;!?Z+I@ZSV37f(VI;~BpZ zww*;kmP*1LprKhoK&((p>PDkZhR=55xW8}xE@1KmoVy@eF}Z?EuO6c%gQCpoi8!XlFo^V1SWo8Eh4h!fh!c{ z(=I`D$E1$td>z@B4>QBuDVF8=9#KZpny00sLK^`|w|P>y3w`$`OIwdAN-A`aZU#I@ z+b-WvoruI^a}09vIHiJSE_Bamnm-$;Frod}w(<6Z(Ql!wrzOj9c`ieeNXl}2G_fZM zfWk=NmYE3*3c?e`HxfVfCakMhEgY%GeXyE^J`nf)H8@fb>;QzV7YhUs0hJ~I)O6kq zk2!Xew#%HUuQ=7%J@(d($aQ+hv%4ec+EyJBT0&W?Mh&e9=hpO&RN^i?da2i3Tk2YX zR)7wb>r$lvsE{^^P$U_V;u3itWuHx*>sK;riEm^%axdelVL(qP?^#D>Wk{DMT4f{< z&IoQ0)~FhgE2!r2s&g#e>Vl%Rtw~8%mlU7^m7yvK0Nwy4Fg7=m4mB)JE@I782rp*V zqT1J^2PQX+9tu>NB|J29#Z77$hdKdrz+T4}s&Q*sx|XG+q~69-sgOJ+OVco9Y9cq0 zMR=3M&hJVGZ5v0X*MA)D9$dZ_A*rprP3}31Zu;MfI9h{3m&{Wg5{FqPG+|2<4&>Um*Pk8aK%+i z+)2W0yOrk@?k=H7e{!d&}$jv>LTlH?8E^?q@nW*J^*kgOva zEmQDGwB)3%a+&Ecfb(=TdbxU&9=(63m19|-7xtmVluKhWqHD5M(^Aw^mZ}?DnXer1 zO4%q#aVUFJG!GIjkblEuxY8UHhgL5r;32{HnM`b2 zmM!v^`(Sg$;eQI5R$GU}keu_v*)WC6tBDM?FEl%MX9po4N<3sEuzrr}U8z+=KM^;L0vRm6M`!>DY@MHMJZq2-rcZM9S=Q@o~FOMtqxrERGJ9!>mEX9F(D@(d7Kn-`9qG=RYH z;4p?-OC>2_6_^l6u_-R?A8fy|@37Ld&foe{?Og0S2lq!}@VPet55tS?CU2Bx^UCah zbJ=;)l1NVx6A}R<+3_a3gmdl~u0~~-cg#4#^`&j<2_Sx?ts=^8aAgV*WRW|MBH(lB zIDx>T0T%>;eZf6nGhIj1JD&Y4SE{Bfb}gM582;nZD%36T7@cyIj2HH z^vW=o(vT9Runm@;Y8P4v79k+v*z7!+^4&lxuR{78Gt}GZcwCDcipytk6@FfK^E)vD ztZ8doPH!EPz#X8foo%xE^_1wA%7`FyLIS;E>9Iz8w(V2yvt6OS& zi~2*e9jHz=OsNyd^}^~Un^)W&-tBnlFz#E#`A}k{rQY$fLXrV>rKS|xkm6-XS_~AZ zLX)5n0hrVpR5G0Qfvn#!qir9Q&MK-K0=F+=8dw66)Pz)4vfP@PTV=$AJS9qU zebeZp(qU7W8&Z9i}2lz^lh8sds*4M zGN*P=7m70`;ZZ3|Mp21bCysbRNEli6mo-f;Rca$)WPPP(nUyk_q00N9ba&3TNGV(H zWTDWekc5uxBrQo_e&7lUNsANQCAyN!E=d4c+}Wyb;kVmU;?;zp3j0KWleuM<#*+KU*8XRhIHKsKMx_^9)}njrs z!MCV9>M6*4FC4kWnG)%WL%^DfhqdWRvXrMBb*A7>p6RdoR-gNpr2z=U69A#!gm33w z=w3Dd05QMI`uU`39w5t2D@`G!D5VuGB&;SiH>d|Sfnas6rcX5kXd@6b%@j4$!%rBU z3*&HT%sDaL>7#eh9W}GG>LEyVHc@a1xYQtkExw?2Yzjhv-xXBjdbzDA?_EtFSFVo6 z)V1-_*0*`TibcL3( ztBnP04Fk7Ic-rw`s-#E(BrOC0LAH|$f$kZNv1{i0VLfG&=4DNVoxr>cF$w!lYF|M69zXdopuQjmZGKmNIbR&NF~-frOyf~1E_t0B;C9x+ zJ$cz|gA0^lWl57E2?6LDftt)2&Zf+FjgL&?)&otbB>?Gha}Xr~K$Eco@JwonorW@A zAJ$Posk8t)>1WVwZfs*{#O6j)9V!X7pfO=`TiNkw#xwpiMp)aF88d-ZMq7Ta{{Zoh zX>9XYw~}PuU}Isy;1ZMTX8!Kn2Jdfn^toBh9nDtS4-0K1tGu!j`=e-H5{MNvj&@KRLDYw8@a3sON#YD$PW?f!+a zAw8qau&FF&WwUWiDqjVQ!0l2;=2y{)CW>#L{{W9HkpBSqOOt1_P^ErLwwO#JRm1H6 z0QDRuV1zn|AaN@TEF?JMBm}uqW9gHKYQOSdnNplbaZ!}IA?DDgg-ti*&|D?g6wOK# zUre{ZHB^r6z~jkJcCn90sPIZ)Dql)dN{WJZ&;hi3jKR(I%XTA#%Ox&2fe7|pi@5cH z_=sW(xkrcp0O~VP#Xfu4K5Ej|^WW^;al$?7u9Xajisltw7hQpGeTsTs&L^vdIC20ikl04iw#STeM=4+b10Y;@Zm3WGmDpFDe%86Bi zQzV-mryQpPgHRz7<)jCF%^cK?#r52n_YQHLrb;JH#m*`wz?S;es7EF}b$xDko%V&8 zBrCnA${SejSPD?ql#``Sa+tURQjlOxjxt!hT%5K^aP!HbN}TVlPlnpYuYthk9j3*F zP2!V*udsloH()5m?mjWN_+y3>NZU=Jpfu?c0D^oZ8%5%6&JO!yo#uJU<}K7f2BxCG z+U-Msq<5~)c3Bund;&@s*_gR{TbuQ#NZlDONVNB3Nr9w9c%&y_M1eYrjE#lyr($y# zcOX{O9nEug)N*=`wUuP50>H&bpZ-UPzRArOcdI704_n%zJm%8 zB#6enKQk#ihdS2XC_Yqp-f%e*O*nR`b{jt;2ghjKoI0oJv;`BTFe1j3*ShVxZVKL@Sj6^-n&W+j&(E1)hwKPp^V+}4NC&c?qGd6r1J zWTv^+P?a`Tg}E)1p@M`2q#J+AQjnPtiJ37whEv*wNL?nYV&{HO@GX7CIQMJ48e#Fs zoVDR4tdT&#n=xi`Z}9N;SFm5jLMvLOrmL-K#khneO+MzJ>PQJ%N)$?5BwIibuwWAg zdmcmDv`u8HVuaG*5@~%4+rN`AXz_ofKSQEQ=Pek1DH?>Js9CH9h5ma+&+S|A$=TO# zlr=n8zz$IGcOKQSr_|de4dhwl=2oWbX_`7HYaVl4RaslJE>5bJ zAJl2;86>3@RPO%((x|3%gQ4cinD*Or(hfTpxY5>CodM#o3GO)$om9jUrf?#0<3uOu#`w-Iwl>P|eSfh|6?}e-6Nj+SWl_G8l^WfZ4*nC#7R?uN-+=0rj)x2{2HSsZSD8%h>qV-d+AS~Sho?@ zK|6>ad-MDCGN3X?I>8Z?sgoNHzrNA;F%ak?5*7adCZ&JsC)^*ghO6B^b%P8*o`Pxb z_>?JycQhQ}WXU})Z|Cy$#4(V7sLl4d5%Br{00X87B09fO!Uq67>+|znZv+p=sai<@ z0e(<_ztay2Ppbrrzl*Uxd3PkSHA+#IT7lftrnFFF^*sIW%66EJacoGk3o!=&07y!bSQ1HkcfA;&HHwWgHEIO? zKhFCcM*Nw+JpTK9PD)&)$_0r$Oyu&g_?T%i@F(FP&fA_?B3Ofh5Ziw~ZC$GXJD*d6 z76Q1FdtZMdMEPxkAT=cK5!9)X9R8g?e%K))5}iQJxqJQp0GuH#9ww71GYR5T0Kf+T zsGSDB0^~F`Q+tAXh}`c z>}cKp09X=~4%^;62!rYMw?l=7DL~rwZiF%E^QOiEslDuZm=XNX%LGRZP>V=6_vg9i z&*j$y5G4=PyM0f?a8`^5-=0&EEalmyqiyfGILshAUQExF<(`=h_+ zV=R*3A2)$7^&fAb#B2ADcgRF*_m6kTLjjUUNd5gkeTSU7UcKK~N26CmUPJZpXaU&! z?f3g)I7mPue&TL^f459d03QeLgX8Dv^V~xm+rq~@ypA>1DOi{ z0O{oseZ8=z=kD*%yulL)2b9{^fjp69< zex7xL9gX8_^uOJ??dOGGx_s&(z=k5luU9qrbk}hYN9rGZjw46BJN`xl`@_HFVAz@_ zgj7YrBYR0F%HOxs1za(0^wO2n>kwiqDM9N>Z|w~)6aksoKmjqg_vId#l@>jI(T*U3 z#4*^=_(2dzF=#W+`wjkp4||M02rdX9iHV-G1IyxegZCIBM6mFl6H0e^6>Zx{SXCJ$jf@fQ1{6Iv`u9dN5J#+9 zoN&^*Q7L-bn(Yrp$JhA#_rVcC(5^?@;esB5#g4kYuLQvo#DGi#v5r_F2a9>}^1V^)IJ{V$xU>@_(fRy*uFOi`wGz_qXrugx8?%*Ofbne&~L-(uc+ReS4sw!I+<1 zEG$S0G1H%((H~!~2q4Xkc2R{OjVGs+a|1{{e8wD=6#Zd~k|zM=l%I6n+*s}$14~t} z4?^KEs0!(_kC`05j7s1N2idFVtWX80rUzFnPcjX>wHhR&bLsnxEIJAa-(!7`oM3_q zgFQTlKk?J1*ok3_*0=J|K~i9VcK-l7_1~V@YrQ+S?$x#o+~9>NDN;p@&g15Mqo1ZF zN`kHPc&9p1K>?lJ_>)wc8rgv#7J?**+HD(yezrIFiNm5GU%Qw;SMg|&l>$$ZjrSXG z=X?;cJtn|SzyOJo2m|hLM8UZ@01>Eb{9B>4R*-@q$X70k-|25#VZ`wcW8yXY#R<}c zvN%hE^eo{0Mvm7667W{el>q+J?VpVi__$Ocb@N~ZH{jSS(VMsbl~Nk59h| z--Onx>8Lwc?Q1Uyp#iF{65%Swl+dVdJI1rD;g=E>`E_Y+RCF$+6@lNrneN(e&mNS0 zDJKacenXJr3Pna^nIT}Pl0CT?d5I+|E=Zd>1dSyg5=r_c_*wib&&=^zXQV$!d1_}Y zBp%9kT6BfWmxq(Z#*EuGku?cfnxGJ=I0@)V<&c@v&?kDicz)L7{?e%OYIRf7y3;+| zWi25P(h%E2Pj=c9833U|;(Vn#5V*LyKob-R_ z&|*;~Q!wMytNooAbSht}6=whjda*h-tsD`@{ixJcE~C7t*LMnAZA1W6H2?~Zkd&!0 z028H2GBsn&+5Z4)*E`oKQ;g$K0+azgD`3~>-yY8Fk6`6tkt9iw5X?^!Tj&TUy2Ys+ z)tm7%`GVJ}YD!e4AxH$10u-dEkJJ+Z1$`jgW0>RfIoOpDRy;#9nm+uAvF2m4U~**O z(IgPDQIcZdKobTI*u|Nle*Dr-?p&Q6$))Ye zftoW(!}_yg8HiwXJpTYlhPM?7A_`NhNGX!2Xlgsi5&;04$;2v{o)S&N4VtgaJv0=v zyi4kVSYcDE^4gZMdcO6BC?!Y%0HMeW5~86f0YD~VKd7n%01_ujj7GJk+yXVA)`jUx zQ~+D6xrrquNq!{-3LDn@2RgF`p=h>MG@}5w8bW|lOzP4IOprlH(xNwjHE*^dm`N-g z>R3_yG}oU;jD{G2bqc5(wJB%PFzoLqX1Rs5`v8X>hIV`)&e ze*&56Y#E4m=}69CF_W|=i)#B`ID59w!+#EOr7c%`moy(*9FiJJyTD{g zG5UZ2Ccx={Z!w(8^ZZs{G_>V6BHCzZJUNO+@q@?V@zQ-pDoSw!X^hS^bD*z>CWwFO z!Pwq+*vAomA*r0qvk}8NreE3Z233_U_5M?oLI~tPb#tD2K*(gW8rW1;^g>F&iWVhT8t(pVr1vO zlILRNVGu}`h2V02HkfD-kepJuiDqzGpdU!HS5)H)O5Azn7gX!1NJB&=!t$G2-Qv=a zv@`;k{L;vD6zNl?OKANET8Z`bcl^Ut)Dv!eRc1A6YTuenEV4!vBp+o<cyp0%56pRe=b3% zWN1vQ!~Xz_X~t$Nvbc1X;YM(vk$-4prP{&53lVlTpab+V{foFc61zX?hc;4KSp3(q zSR$rL;zYG};b4of%G7? z$1ZEyZonCT;$%xmvydDYr!ivx0HkBQ568~9<9SYSugPZ&l7u*+p%q0Sh)u*ePwQ;+ zB&TqZv}16xu}Sr~cAG5KDN%CZJp)*{=}KuDMh_K>%P|I2@?_1R3IMi-?Ry^P8iCk6 z0on?geGIT6TAuLxz0$G`$zyC2DN9P)%-@$+dDhjG-A{F zdDLe1R6sXtP=WNj#{U4UW`~Qi#+1w5&S6NDRMOP8q2Dt4bge5wh=B`26<|S)kBDOh zlV=#-&*l=bIBC-`^Q0F_kl<**5kfS50DZ(|J5||?r?r_rJ1EWYQgN6JZWQ40Ql@aL z)+K)R1BD5c^=((AP*?YahXE*@Z;F5lQm6J;G@1F1&t@t@o=G7*SQ*n9k(gl+l&J|>;h!W*nvkicVh%R0JFCy# zadhUNhj?|R0Hq|Mmn#yslBUCTO4CqdSfg1Kgwfo_SN#YI&y<3rvZsKgFz5K!UKn>+|Y_})`B}-p+uI&a1 zC`W-ZR&^va5aN`N9u-wh3)$OW4t0sh+S)t1yPHQ~SYGj=^Zp)bZYwaUQdX80w4owE zQ_8Z4z4WLduppEqtVm2uBm;dR8L#I3bq)GPg5qi#eKqOs!W(3;+^7jJA!>Ey#`jWC znVYFf5O*;Ykj}%sytjvZC|N_0TYl=>&at4gI(C%(Lr6}QCBo_e*!WJMkUylYN<<4; zVr`1^wBEGjSaLl8A?;YTB}FMsHx+lOBoNlIy{{NOT@)oDM3LfJN_LA$lej1#C`^TO z#6^K}N#$a6y(|p|nh{vDEV=>f^D%3yp+pqKg~$Y(&4ur}k4Paxz}j2GF0@a^pCVje zXnV0C*o%!vQKVa%%FRQGX$S!GFiwyO)C}4La)Mw1&lEAZxp{sa6K63}&#)ibQ@6Mr zm1+n`J!3nORDf2*4M5NpqiWEXm71mN8SB;C2ba#@2p z<+HqEx%U+*VOf4!y$yg+>*+A36$B81FQ9>S0t|ZNV8P)LEETPZXRYeYz_ESo*VZ&l z(>R-mUj(0I8UsKJ4K$%018qo)!yGSHp4Bu87x{9jQq?4=Y9G|0VRL=VgNoUvYbOaS zi6n7s;Fr4ug8nb9?-?G=_GcBFPje)v4$j3qA?fbXyM=yn>o4FFhiG_fw+OY&ktj`>>M@xnmE_?hW1MXf&3?D_@g3R$=N??yBC@x-C(&{_6rJ?5>xr;z$4>e321yu{Ec{N?qP?$8%pyS+YD<~s?0u5UC zBtH(jX#Bu$&lVJfiDdv0>QG%(03;zNh|Mz*!^8)@FO9ho9wYw%sZMo4!`wgFc5RsV z%BXveyJBXGEm!4(u2d_9dj@)-p%mJ?mW$4*YFo$tr+lXM!N-|>H(-@NH_9d;w+xI~ zQp<+`;K3)0Mk(aPZ(pw;+4G)^m$Dv_C1Pc*Q)lNnO0XD2q#06>q-O=CnPHTYr%^uA zxYV%ZGkFo0QRkdBX(u^K{T=?Rk| zM0*3bAp@z$FwJv=F-By`iX@jvyI<|UQpPZv8#gN9 z;1Zt!Fhqhaw3yz&n33hz8tnYStP}uqQfx+qaR)1HnXA-_LDrfDV@0#ac>YC+VMNaTQ^4dzbsjqfJy0;SE8(@Oy5-L2dk znPwE#qmXv9u^4P_7H~|I0$i~Hz^%hs#)C~G*T2Kh;yGHWX_r{FK8*!Ut$LE;bxV#m zQb_P5gai;3U^g&3XJGI@cu z!M&gUCAclTn~9LqIlBy(IDd(0l7(hjHBkhk#*?b5fn+39(L7Yzp(F&U0a)}OV*7KL zADE?Hp6xFLkXFzo#?OU9aqN!FcB3opRsk5y1j3bqM9euL&5<~tWR$sCmO%b6 z8AZ9r#D7ZOihEh;o3s-$&e37etQ?S~;PZJ%TCf@BMu}uj9;qnxIDE!il2U~dq$Cgn z6sJ<9I=}Xbo`+48pJIGNan3QZ7fZNfL014ccRfY4FL>uNC5ffaxB|+HQ}$#Di6=sb z_Q9%a#KsXpxF9C|NBO|#bK+w0l`OdRX|Ajr=I!(85#C&QNA(GwP-JcN!a}Uj0kgj< zi(8+{A}I2-n|gZSh~Ns2bMxNY^8Wztx6cG2rQ9>`L(fn8t>Y9(9AQcW7=1^4Zzfqp(5IV`@l>^ z%ttd zgb!t$5dQPu&7daJ<$+csUA4Z~i<>Gw(d$d)#eeH|grsr`n-#EL>~wrnb<;ZS5Np zU7hPr@W6M_SQr?&6JdGU_CCP$j-0o{S3+3o4?^|SYtQWukmhnr8WlMYLQ<>ku!=jI z$diVT)5GbsWMB>u2CeRrHyTCAKIC)fI3S3M*!}H)hgrc0bl2z9R5NmO0Kp3YTuQl% zn|yFY2Is6tHzwDex4%3PfD%{&PQs=f29QbKE!2M>So6Xf(YAMXZo2g6)-Pe?@cU{N zKIk_Cey79^ur&|`rbvO-Pp79J6Jyi&{^Q|-65rJQZ>&mWgDJQ% zG7X3Z!adFiSdt8Ff(Ut6ghH<|z3nr$2&;)u-~cE%Sa1z&tF>XE+#R+P($P4Z_O>zLwr47c|trJf?h5Z!3t_yYCrc%e35s$!aj62Ry|1P#b)4Ml3?L%|~8;DiP0 zAmHq7Haw^J!2}ru*zW}Idx8k|!30Qa^;pD+GXT!UK0iC#1S)U0!^586XNp-VcT^qH zLFxbifynedF)}^T=TjK`ON>Ix7Id-Djs$|+{USnC6agIp^qyX#Fz9G62TG>Lb1~De z-=+vyAS7*ohoOQB6b~`!=g*(qVihH~*Q06Q`auTOter=GduTrP29&tyEnO>1 z7&;ZtV0!$z{hTZ^CA8rLO~QM%o+raf-_^>~#DUxr~oz#{h|VEhIA;|OLH zP&1wU`Pvd}w9TWkgYcSMixIPd@*6`EuNVjVr%x#a%z$^Dt>iH&ZW2Lo7?R?E26Ebb zH3g#@NChc0Qp2Dc6^+sjR5^zc=x5-M@ZQ&HS-Sg7<4PdD1K}*uXv*l{Z1;Ry#1v#H zX?QFS_bkg+0_#u$n|`S*Q-!Glp|Qn%EOzYVSXJb?a{mDOrY=xcGGa)44h3U~B?Tmu z&zzE#A<9)LDk&fqBidhtZ$Sgn4sS2F9i_p8*AVq>K0MY<} z0r~#`+kVxYB)RW2s;@%kx{wLC>sEOCx9HcfpPN<%GNc3`3X+llW+b~dlzz33G&@P{ zASu20`N?Ou|%|hzgaablzr6#Npgdc!O8H^lu7AUj;g3m6DW~l2TWt4msg)EG&S^P?wxa z)AS$k6ty1E_S~quWO+${{{YR^j^ViiT+qkGukFXh{%g?>p(#^+q^r02`bbN={{V4i zj?LUQE&!>Ej&9p*wKU=A=fOX(lybt2~|U7Mby#_We=&BfRCpD`zFrM zazzy~?VzC@#hO1hxond%Q7lW60ALTaGLlGlNCSmo!QlxibL^fUpZ@^Iv&GKeYn0h4 z{I?5aOouzx97>YiZf^WTDGF#MZ!MnXaQ2I{vIA3y;4;j~VITlmKx3Ba_kaph zGnldV5ybS$ewvln6W;1VNKpVFz_BLMr0sL#07RI_gS%Ok%}?zmB=rEDsY+}?Hne^9 z>_&Ab8-kT6%q|6uSi6F1^$txQ{bSHNt^~;V;~`x=LrlCLWTYsd5|UM=l?4D`D^8Lo zNrb`2HpAm*;UaFNG<|fdu^(GUl|P7&!q#KMwUrsR zS4!$z4tJ<=q?G9(&ZKAp6bx=`M9fAy&t`jGiJq!PB4GeK@mMoEyH}a<5zYHa*y$L_ zM-_w>WmO}I?)6PY-<7B%n7Dh4vuuK?yr+o6s8dTyDYF#?YLu%=Q&F)p6&mJM604IN z_99LjA1jVe2r70C9a>R|Hs4Wqj>hcgw1cjl znwpVt18yXZ3vE}_l9L8VjU@T=$2-olP!ANe@T@%;gUAtMQ(p_;uVfkMn=jN)Bi1}lF{*|SZlKK!&%KUW}=zZi9@OdD=j(_M7CChm5GH5b0-I5 zds~@yH#o!Nu=D01$Ud@W7&=jk!?*6HIb%=ZEJa)fj4!1hMqQuurP)k>XLGD9xze#x z%*+~;0mNXZDDh!2D4K0LNglde9w9BI%wifj2RF>~L{$DmGIjA>)mdRwEuB^MPgAzj zK-02G?r96HlG~2z3VBYc1)5T{Bj|5sJ42WDqbR}V_z9l13sR9WB|wxdo@}{*RVt8{ zC<^xxGi*oO9|i zs|IrR)N&TC<$(aSVO(*Wy-h;nG>Z;6JX&Pa*X+ye4b_0zz*-v)k zSQqTGdpCBam_nThA+=-NEcOIg+RRIx6~%d#H8xv98BtqpQ@sBGGF%}+rk*lJrPY-= z-AC&i!jm|MjXV4g+8BGauxs}}k92dN5%=0^5cS-sc&R`gIzs`ptQ77NY7U{igM2k) z7Ge$lkddVG8YQZwPN&4Aszs+#RHd#2Z>TBMZEZnTi&9?~5v>i4!w|#0=|gc*smLWE zMbtnRt2tuhTX%JM#48Y_g|(!G1+Ee-q)Le?iSY#sM@UcwCk-KiwVPcucL437ZOCIR zzi>V3{wL>ul_Jn`_o|eJ7;jRXR-l`sDK54l6Ro;P5yZKbXYhT~1O|rMgL?Lz7}?53&icF_zHeZPBGE04+cg zrIg8WT1eEX{*VkHaso~4IYwul&CKx%SW`)q=L(Zf0Q*si4O%-stsc1Shhnh3nah%d zr`O{qC@&W=U=LX^mh~=bWdZ{;rJtN$+e>XGSK&%Vq6&ZsNcRHV`P-GTjR8w6Do>v+ z8${`-!yuOl4RS(LZFFI^DP3AI9OF8AtzVU+Dhz&=1kR^Us5VT(L9k4T0#4>LSZOH3 zvgQX+(drK8V)wl$G<_={j-QX4>ZHG^YNR}#$u-9XI0ei?1NhJu8l!Q4IS?$%kWubi7b#Ai~L7B zpP@0=zk@9F)P5&BF`{mJr|P(or$SVu>q+4p{{UE6M~g^Gbd-y%pVF`j71sR-VJ2by zB4jhF;|rc8h}2}5;sJVtZ)6vG29FqjXl`dG9r#!DiG)f)La|+uatov*{{R_wbu$!7 zY?P%+Dpf-lBmo~l6%^=kjJ2m$v?aQ6sG&XEYh67fNO5f|xl&qFr3Ee2B-GW{w$fwU5o_6)!ScOP^sPfPi>V0?0Fj?h;FIsBtP_ zPXPdjj}m{>YloQP8a~Pk$)_LfoMqZZrkkW@nQ-R`_CUhB^t7(>Nz2L*iiVkefZuSw z^~5RF`D5_|jb0BKl1xg7{Kg9!6!ocAPUEfKv?b^ZN7)bT=gFmKSs!bqPvBqWGaQU0 z0GHw5&Vp6KDgx`mNDn~8obgW!@Z&5YmY8S(qyqo};Q+#;`nI?tz!Q%j=6L1ISFBvs zJ#}(Rq;lqzBSdd^O0UCjTr9m zGZMC_4)uES4qSuqxUC(EmW~l)P~)vd>*L9}98-5{{WAd|C!o5CBteoiiSjqJ;@r8# zCg7e%y7>GzH;OZGrAa%{#;!Z4bk(Bemsh();#3?~kf5Zjsx*MGkf21*hEs91;xCN8 z2QgAqT}Gtq!*D52jYr#G%rf6wE^qA84vsF)Dai7)bZ2H=q}Jrs^vOkMc9zx_mr_(n zg4=|x0D@6+2#`q=AWk|)QJZkD*_u*0mS!IpsOsHgiT1OyQZcxB!_)%q-p@8>tDAgT z%?GoehYsEaTBy*vwH1`54j`zIv1bdZ0Hr#00NufzNdOY5JB5;D!V1j zNCm3eS5ykLB}66WN$%fhDGCyC-uoZg+{Yrv;qkZ`r3#k*vPx7vRxK<+M8Fsjp?9;m zHjkeEnY|JAd$)NGH?!TKz)G1So}yqtRwDT*&vu6c+AOL@ zVS*)UTFNIOMzTu3+>|*{;ZL-M5D0c4kR##Oq|Zh@q4Zzd40mSpqEm!alupa132F$> z5h3_@_~W_Pd-K*C0KOc6 z<=WpkA|liY-ox+GXP6`#Vpt+NmHz-L0k8nvh!dpD9iwUU!&PE6r8~qvN#UO?tJwpJ zs-X83sbRN#3DS~eYMx`3!(rDD#q|(NhvxAeX25yR!~Ecez!q$I6Uch~Fq+#-_uJ3b z2oh&d!$)s98m)BCSFXP8Dz6sDZd^X1marf`#gyW|cwrbk$*LU*piY=p(TKmLh5KiI;S&u!wy#^e8l+ye8@QBD@ z)?)8)NgXUIK4WA_5U_W!`4T>#97NXh=jPVDS;Bw3{{Scd04Q(Y-^&DCh_0wBVh@;| zgup#_lhOxV5JW{PZMW9r=z3s;XQ}YG1buuuJz!+OJ8jPdBFh76@3$z9qCg~l-vmPf zN9rE`07HT%!~Xdn=Klanq0VkD4Y_jXf0x4uDoJgtxcc=P5g90szvK7*{{VZgz21E3 z&FGr9E^YSuz}A0Ma|GXiKZXcJA7s+3X-@c#sJvWqweRK5Cj>-U=~1LIU>SmT@{3#A zed7cq1eoc#7y0|dk%9=M?PDOBn2#tuO}>3+6-T_T@04V4{{XsAJ|q7CxR0mC5r3rn zeMSZQr@Q7NRr{yA=K<-zkp0idLACD~jUMpt`525)*Gl+Td)QZf;e^U3?|eJ`-=F4& zz(F_ZBb@C&1AF^mM!nnf?(l=po{+eb2nxv5ZZ8q=!2|_Fj$-2R*Prp$EO}wlz;gGY z0-C;?@1utoFhKCwBHuXol6eC`3{ zWLunA%pRUxq+DQHv>pEdZ$Ef>Iz1e1%HitGTLMWsRIT&&{qRfIyXy*#d%mztNfJXv zb*Tm+;ak&p=bHB@R993Wn{)apFWcZZTpi!KY))h@Hgrx0ydY z5JSO<=s_PG5TLW{K^`!0d5jl{fpJ6nYs^6qZ@Dwd_KOd_hd(5?zTNc;?sX^_7}gN@Ht{OFTBoIN^+t;7>!3ChJI>I!9U=a}{L~cg%1mZ+%-Svvt zw@{VB31F~51h=}*K-InwjVL8oHX=RbMEvn2Q$b_b=cm>sgRvg6FSqf6C4;1h-ed3Y ze_Rkl!MM5dkM3Y$4Mm@<5NRKOe;$~E2y}t~Ga^SL^&&o>v(j3-?)M!5d)HU^v2w!M z%Qv5(hYdg*Ox|Q|hw&$Wf%`>oQn6r`+^g!v{9x!<7CvNH`A!`Xzm_p6wUIDLk$e0{ zukdja@7?tdXlI$O8zIWG4A&NaWre{drMMh(ihy7>AkcsuR*gv7dlA_T%eURC?Du5ySxK9T z&a+%b4n`?yQokpFlOkp!WYrN=ph_HRT8KzMmmEZp zHe`kapbe~kgFO-YI?H=Klw^IG%5bve;Bsj)@c2n+013;GgvyzWm<6WeF(cN^C6uQs zCXweCZC#@rWf{_3u2p%9xq#GX@cGt8D#NuZWHA$6UW!dYo z0N9HLHWBo@(U)T-;c&P>sVRDKQJ6iBzt)9?c!xF3T&k&cZmg*4U6T-HM4%E?F?9th z#OefsH{rP4V&_aNU$$y4`q8;=;fp&`zM8@1^6=7R`n0nKZWBr|C!=^%NXLu%)TI>} z0l=hnKq&+&2?7DRyd5ft#x>WgbE)(5jF+n6OATp4=FTm*m#au&ZIq_kl!>q+#1cg6 zo5>~=Kq)3X&~OTp6}h3IIts)~LV~MH-j@3@8yGuFDo~I3#-#%)ogkDzwDVr50FWdA zMevAU7506)zn+>xFW5^8)Sjk*3-jJ5Af(<^1Pg*nRCJk8AJRcc0FmL@LT3yrC5ibp z`FtDU6-oecMm49yr%QK3%83CfLWnm682uqhP%2O9Bn1GHVszftUN2{dpF3ZByXmQf z2~t)`Q*8DE_pdRx#s>P=r@}!{kVJvvxgIV-K!608Aw*gZ1=U2{Q{+3?S|yUlLTqiO z&7*0{QqLIeNl29t0#XPdLWu-wDk9+^8)^nTLfr8ZUanrdznA((N?0UNYJNk{d|DlF zf=W~rl>(qANg*H-1VNbq3Ed@Ig<<01B7vT^{;_^aN}R5@HV>w))KW735Pb}A*BUqg z*Z%LquAzl@t2G4WJZ{r0X=J5tEl<=o?{>XyIo;?`LbSf$(2u}U!0jh)r2rh{Sy2_Y z`1A!UV3EW@IubcY#Lw-%mo$1S^c#qO*h}<>pYfoBU;h9nJz@}B_?B>T%0Z%?AbZYj zd80uEbp@trl-p|#ooR5TBqg9rzB+oA+X`E1c9n>Ry4IQa?G_e9u%k*d!$hB!Wo+Km%9dPz=tHK??Smu^%@->PK0{`Dfw=pFYSR zezKjO4#|<4<<)G2(ma=|vxi%44K_705i2dMxZ{X@RS6DhI#c4W#Khbbj@4wCY*Nw_ zJ1;+#0i}d+%v;nI2S_P>SxGvG{RV$zhFh3peGGQ*6P2ob^IW3py@4gEwwoCH4cYF}&&%;46Q2o+!nmC&PKk4w=z^{u zJ-w1zuJ0bMb{*S|CSuDp6*Xkg?!Mf)=7dAW7l9r0l*Y3cp_^ zBh)aWQ^W}+?YO4DLmrXIv)m4QhLjA(o(*`^0UJ1Q)h>0dS);Nu46BFu_a||O+GXzf zl8q_>Wm?s-NsT2^pcHn418iuhpNPXvDwcRj(6}P59A1Xvn=y+-Pr|HAAejZloC=q1 z=t(!*oOz?$$Kg$eUa7CksL$oxD{!RNOi%y<;YtFG z^?Ht=KR1fF?29hU(Z?w%3Kd|)n!hupdRouL%iwM-;D?AqRTVjNODaK0UbY~_pv;fc zbQ2yW2(ZTRjPC`QBh}2{Oyj8_5GCe8wC0U+NgO;Axc4H0s{HUQXKQY6Al;U!UlTw}{yi~MSW5qh^YQUzcj3w7k5=p$Q8VNhR8x#590}w%Xf5kwnnameJifVI%U#-*RTA zbg46?kWxTUAc6{19SQ^%DNs-#yAVNkB0e0?l)Nvr&(%L{5(6j^;a|wL5hp!iRQQsBRt`?y$ zPt?QZ%i}e68(f`-i0Jo?Z0%A?8+|1@7Nj9clOQ;qO`!BCD??=32~p*0{BE8Tskv?4 zh<)lGN>^Kf3L=eq`(8E#3%O=HX%xrZZWM%)ORxZvSQyGOt{znl7v^eP0;Z{?))Ot`Aje(e*tvP#~RaB@7ZXJ|8t0OlEkD+r0HDVdhwiQ+*CPzyxtVuEKW1tC%;i84qB)(DSQ zlr)ko2nJ6(s&?FXMUK;B7}qfA3R&k}4z+%lxzt)ZZaAh;qY5?mYw2UFR=dU>n7dJ0 zu}e+}M8eiA6*s@Q1P!Eg#&SZXNcM<&l)W|%YWv=sb3kDA@9j&Pze#nu19N z;k2VW5veUsY$I$@C5i??001Cz)j!fEBwFC}wBH)y{ca%f5?kSBXSU~F=-=diUjr$V z!tCq~dDBx(K14)p&xjhPWlz*TntE1KbtzhPT0#&23R1`gNl4U8*jSNo1(vR1WJ(E{ zqS)}THLp5~Qkq&YndfPw;U&$PK6j}^vHK}$P)epygTR}w{Gi^tzW4kDaV}?_eldF- zTOm}L16hX{CGPku{{S)_?DLBnL8wv`g@pHxHd#P%I#g5QQh>*4^j#wfm-f#kM-rEb zQOyNM8{MOTI|GaSnzk_S2E4>{{X|kq+Dr1!(})*eVsDU5}=9bJ+kcJ9=MPK zLdghq5D8N~Sp7j}&}jyw7ZU9gbmma<%b{$c2`O?r*ASg<8YUa5WeITs4Awl5;K#DT zAu`v*0dqp3LLJ;_X;XR%R7b)rDB_rs2M|&c2?0g@qytdofM0|K3WLJ4Sn+T1-ug>i z2Hu?y4kO|jaGDB`OR8R0!ZB+9QM#S~+`yGP6(+ zr6x%!nDb9AkrF)tJu%`u=N3>}U$R_Z#F7gWV)xy{$K0;TWuCPXS24*gMsq;+W2@@V zO=GAyYatq1@PHs`Iz*Y%Zn|Jb;wIO)=Q)Nr?n%*z=Fh1v5A+e=GJLLFsZ0UXmUr9X zaBN7^UEh-As7OniK__q^-%&`D;Q#_HVo2#2)NwenZ&&%$(@uK>O=Ee%;P`)q?^Znz zJ${yG*%TRaGz&x$NH*}FOTwlgkVhemT-mQm7CNy!i)~Iov9`IXa`?` z1zH48g(pmJ9Jzu4xF;1BNZ1jgWav`Qos@b6@F#uxS@$shc$8V=6SPzk#TOGQhX#O8|-{UrbGiW zbwLwjy@io8OSu5*7zWY4PVm00bNGFs&QgsqL>PidowY#(Zy_YXl4MTykvM}#aj76x z!?WAvtVFqBEW5YOoBA7fMS`ZBp;9A38o;=an2;tTV=yCDk_E}d>@@P_jmvD*sUgV* z?YD$d#K{YBwQr|i>qsRHbCmCeCE}9MM>qv33M;tNB=aW1*N_f39IqCqxF)oF?_JLS z06RvHoM4l{J)JAFHF;j!_(x~<5!zz%yuytMsdYWuh>$@~fVj%kkP3nj3QQAyNL< zUE|UJ0Bl#;tj0&Jn6+j2@xzOj326wg4D@X=B=I(((rzQ?=i(>O<`>d$WwDZ^NJPHT zWeWA#CMc)dS^x;f;ASbDO2G^gsDwB%B%o4F!X^B!N;pTBFT9GZNmbL$^j!2|(B$sIQNeg6P?!2~l-vQsgpO&#!e$FgXO%< z$1}FTPuMraL=kXDScy#T$6JlhJ)h(`#B~q}jkomlkH4-YR*A51A5NU~)&&qKIUCrA zpxe&S(Yf`A^q4!H!5E4yVL=|T!5-^4A^qq@D@ZUWZ-=kNHz5iHaVzQ57{vLYjS_VfGYBLp%Sjltw5PfH!}MA&eBYD*Rl z=RoOGKG91)BWR0#aEY_hwWX{Bd?O_h)Im1h^ZR*xEO+t%^XC0uSV3C1e_tM4tgQ^Yn}n90&_5oDHqlaBs_hf!hQS!U2io zWP0I?ASAg0*F4}AEGkWax$eADOr(ADKYn}SD9K2qls!G`)A)=DN+!_}$o}|+N+yiW zq58rpMPEn|GNJsAK=k$A-6Gf#?+*U}k%Hbkf+^DXy)_L2!(kz*-BaZN_dp20(s}gx zblVdkQTO^klg`jZFZ7>psKcN>eEq2b!2|_c{7;q7F1P3HxWb_ORbhTjLw`K=f*ZCH z2$+t3-@XVT=4~E_()aT>AbQ@wVX8}ub32+ELh17#Uq2llf;rw|E&E>#O9EVUarCWW zAd+O*$Duri&^qrvQGr158Xpc&r0C_e1_+WQ8Mj|-B7NcC@-V6QhkwYy5Jr)GrhKk< z#EADtolHoNba~Xp5pxC&rqk&>hW=CMhZR^CZ7FBpsfyT%PRmI4tHcG>YV{r`x)g0- zK=a>o_wR|3?vDQenTkXZSb`45h6tE8CvVg5x%KHdbPz#4#!dGV5IwJg2s%vifH`@e zq<;QbB4n4*fYX`zH`kOreEAuVZkO}65cmB2b*C!ApC@mdb0&>K3&fVlvAK{3FM0dx zfapjRsj0ZzeQwaB&yh%_CN`MgYY#8KEGVtDv|}gB^-3f;LvcG2!nC8gr8O=!hajIa zI`R+)-_xE5iVldI^tY}EavO+0cUUe6nj{0U_rVA)=g-TaiH*oWnUD#S(2{rbkCq5n zDBqR3!20v)=Y&3dk0#o|4Nk(`qW=I7?lB@&!nYQ4(Y*XE7@DP33`U2gJnIEP8*S6f z^ZlTGaH|HC-{}gX>;ZPEAd{f0^?s2d3gmshk@x&zNGbP0FL$lFHt`^%4qqrTR7Hn4 zk=8c%{%?g_H6mYJk_Kgh7IYNH#p?#_=4mL!iN6grEDi9QT9p-`md&;~O7`-UujE zPv2iL*Ut_uM=iBSD%Z_U@Iex(HV4d*KgIt1qbX|8hoEwPG_(+8#*@C02^*6TZ_s>z z!aJ9x3%vmQpNTNOtO-{P{;!8t;lE8f!leq3BnT$vdSBc7;!Y&*^NM2*y0-voy;yEd z29&F72TGtGQE+*B7~2g=B~`f;9pWM&fJp=d`^4CF+rB+%{{T)t%iBvFOWPi8OqCz` z_)~=xv|Y3nrR?`GW=HBtLm6saPw7IcTPg=qapylyy`>V)2eQ~%ixU!*W;v5Hlr1eL zQh=x>Dk}+41d<3E$I=hL$D?708P}xD!w!;_DsN{yG3?e>6?m|$>o7qfmZ?*v%F89K zE&>eMM86tGSoY(^iKEMEm_w^=L=-K=ET%~WgvxwrB!DF#f-Q+V4a?ob}j=GbTz(i|Jjg?8A5H8RN*#(72QHmGvzblqGH=#YzfEbxKf4 z)hR9{jY%U?0Mn?X2=WhTvFo2cceD+_U^&+O>imOTN6|jWWaABmlA|mJZ7S?VMxoq~ zkuj&!Id6KfD&Y=l4GVL;v%nY&)y+Y30NrhZqIvjXVu{B zuSwDgGbU6-s2xB_)u^X(1&N7J5HQk0N5Fldt3g(p7Yx+;HkuAma-~{UqDTOd60$-` z3ko6xNRpJu8lpsXRQpY%@4xkm)6k(}>;;XrdKR_ov`-Fd2?0eAK?NX`l!5?A17W!` zCrE=Ek%_P-E0h6pM^+z5ad_Aeuyp>VK?lN0NRkQF1cb)+kf}IUR>#1=q@`CQZ5#Uc zz~6FHVqlp`8n#PeDLPDSH-JgBlNMt=lzeIMXndq_D&)T6)IWacvecy@Sa*ZOV&Ian zHZTMkfMAON0kor~HuzV@FUSgh7jCrBIc|Q>dqyi#>Oz(Jr65hhhv?G;+UhnZp#V%- z48-$&Kcm;;(VCKkgbi<32c7I$&3oe{;Y!quw~J}e*?n~Mv>D%ahRVtgTlvhbFBZvc z>0oE>IAPQUDFLDjYF<6t_!P{BAJQ*ra5V!P*x8ele{Oacv^g}1a+1T^TIJ$Okbvq^ zNhu5gkB>jwSJ~;f{{X{pVKamUDOoS2e40Ykk;ElR&AT_+2~(FbL4pv2mZU98Pynz| z{usCBYg4Y&0JOByi;UEll(dmf^d-LW+C;YfGM&x`na+Zh7@Nk z$`z$Us@?YvDp>wv@W+Z;h*z`1fK=s}f*h$N6qaWJln4Pqy{Pd+@l}&jXMMTx8&8zg zF!v5{$O?6BI!X;+5v3#qIQkoJx=IZ?66(_1WhexYvyYvhjH#2by`$`&B44w`fW{C* zfLT~qt5u$^8nt-+Gyec$8u0mtqF+jw8I}C6Yr8zOxdf>=+y-ntU`v)q77!)DAuPnZ z2;zP<{4ggqNBgEih0<5H3yi%^R)x0Wbtwu307A76TZxcCIPz{&+qt+Edb!8f$OO`1 zZHlYFxa|&(J2fb{?kxcZ`~Sthl;rOvYgSnPA#UYLo7Iu z#(K&ep9+FU=UGqc00fmhktcPel1SeXHzn;1jwwqBpRON?JQM-L63z#fG0Ns(`%T!) zRFgJb+42&@^TxJimlE0iLY;0vSzsEvni|y7gGUjX!(yb(M+C<8%9yk0o zC24m#;bGlWH4iC-f;Bh<*&@WGkYISDa5lz!1=?BCnocaKV{l0umg`QswzYA&b_Wo! zsbMlE79mNcOvnkrcY4wY@g+$GK!6Hl*ullw z(`Bk;tY#R2CCj8;fpI`@bI)CjQsgYln<8-HT7z+s?9|=u&iC<-dE)LBtfJzF_<7~L z#z;!DB-Hn?{Y40Q2oy+DvKwlNge)O5rN^CNzW`IN4|IAA?a%%u?C{Dl)3~!diRHS8 z;>G2Y6h=6+su=^iHl)gTgWK~O<{x>eiZ8Yi?)8mg;B0%#gssfOz5(uBw? zp|_C22mt<<0DU4q`O+}O4`haB6y)EfAsVbc74Y)?@y@-V@XM66bac(NDMd{i5d?*` zsfLjx&x^e%Y=Uhd9flwzH`118<}G2bM9YKQ8kW@0s=VM%xrHK7YoBfLpgi`#C|jDspniEByBamhGr zGbucB@aD6zOclfxhd;YjUBeR8?jD2v9Zc=VX{FROS##u@AQ|2TYlJelD z;6A&6Vkb(f(<(&CYESK$<(cy#1zQzLk`f>o@e2u_M%;kf*x~pYhGFAdlJ(Sr{l)d( zvF@zeOuRgwS1wXRi)&6=l118#{dIkPGmj{sf(nV%0KgMI6TXl{UI_wH1e|QLCn8Z{ zR=F*y6fVy7_{TWKV2&Y0-=GGsYBP%ITZ%*S{GlzGS}z32xv-ep;1RgI@8rq&wX8Zw zOByj?{{TBwQo-#K;PP}!9 znjA~Hs@bolB;>uF=bTSa-ebheH1xpgr=SFe+DQQE(-%<&R<#08Gh#7c8QKgc4L_I5 zk|xY*%qQ6e+>R?Y)v!8;Hu8T*xsG!_US--@%(VhA_*+Yq-^~8$BE*4KVAk{x@J*9t z+&}n}?4;5mwu=?qLN!7)DJ$UqA!SL0Eh$oz+loO+T6JsEIV9{}h_SQrev*4OX3bFn zdC;TZ`;#15&4{Da(t}sye~zq|1(yC3y(~+?NS2Nqzp|l^2xI>Mj=Lv93ss(k>>8#$ z`OoW-Pexo)R8{#TmFf*Nr737>3kXD0sr1yP6$lAo_cGd=l0p*0OGBgR7-wLx9GjIE z4sZKX!q*&3My@0CSCueQe`2TMaqLS4$haP$aRSTt7GT z4BLqmrqY6xg4S2Tz^7N{Y7*~oTnx)jCAQYlr$|y53s;hV5z|U$_I_9-w7l5UM6Fp# zmU#u&gpkT0@Y$p7m-cDl5i>3A7g|D&U$LUX>qK zuC|TAl+px&C;)z;BEYAa*p1I1B+pE1IE6a_$Y|z(=z3{!LmO0-gr<$nzp3fn8J#l# z1i&&$kM_AjP z?vm4pB`EF}y@29+N5OrTsi%oz4jqGQs80@$? zoM{Zi3b?K4H~3IjM~@HZ_#lPO7Nv{X=ts@oyEu;{e;r+$vBDjzqlz-xbkb#fa@z9; zLnzSM&S}X;a1*3LO5v*Dk!@AWAVRV0Z$zD-R%wyW%ds;Qyhd63IZj6r!c9;ZR3sAQ zC29dpU$jJ~4@(V3pwwI%5y_y4>o<>2-{HTOB!VN6sm`!MkFd1)lLQWMaT2IcFaY>J!X=WQ zTR1&JNm||NML;xS>L8J*1_bo#-p~YeJO2QG@rFg!``z9iZ{HD=Oh`7{%+IgfUT=aM z0VXd8+X$;`-PCg(^@Jt4@)v*;xjtw7P68Y=JCRQD2(NVaeBoFB02m}fisThq1PHfZ z-Yj{^U^NLl$k)Wa#_z!0MK7jcO{}Kj1E$InEl7g5U!=gLtjgsVnX@{QA+!&z1G)KwwX8SVS*u1a$9EWSNlMfxYYz+IY76^)0_@x1U3dv(K|;;xjlUFK?n>e zT1Di0Tj+k{5Ubj9)aS^Lsd$?X{xh`IB@3r8PqzBJI=O+-mT^qYdI+T{GcGlCW+^9KYFgpFhXE@1VtM0?DG(AeJO z0-qfX9iIIdY;=rnBloNB_#`|yZ_^T_jvA3lRlB|)BM~H4wiPe(@B74nZVxF32!#_v z@eF!2NR<~Lw&3}_D+UHYKTJ6^NP1rVOgO zKypV5u^jF`0x1(=eM0*lw#2I*-w>!Ny$En^;L$-Kg2WN*>If|V0M-nXAOUC>{zb3% zz6yD*dK-h>+P<%^np3E;9q*%PWU4I(kgyjw-2FOj)Rv_20ZQBYLcsx$56kb<3j`gv z0#l@t44CyX9)Lyp8Nm(;U`2Ngy&L7;@QzP-RFF^UAjFM<7?mV&ij8&VV;_i0%=M`# zQ69krB#?cC>>0y%TP*9xknwR%@ri7L1>ZcXs}2#^2-T4G29nB_eEFuWsS z;SOQXd!N6a_;GRoxcx$7awbXjv>vnNhAe{2LBG-#3Wp?uH_{3H$E4fX;XrhNe^FCd zAmL)yi|%I9ZLm8GCk@CTP;@j8$%)^-2sXG$1DMmy*n>R2Q{*uj4ncml6{e;3E@*Y$ z8ngR3fK-$yYy7Gjb+uw{Q$tmp<{4I5M1u8JXHdl>6z!RD+K1gmJ541!l`ZEG){+W> z)DseLv#}Dea%WGQL`h1PRH`H;Ad;e@z$GahMMv2H65vB}%(D@k;<9`W8DP!LmL&Rw zdsEESO_zsEl*vU+fjEib@w$9B|1Si10O#7XP-^Zb8O=~oLyMus%9PPW#QC+!%Hl0DKz0B z(l!?T1bQ3nB#y=QOS3(Z!O519mF2Rq^VEI?yxl9X0atfL?|d+ogq7*>r0I=6xE4S&ejFizqb8$^gE|y3Y5Z(le`H8M4TLB zWAOXKRWFNC$-b>ktUl&#JIPQ=h$>14i&4@PB`$Bu29O|gkS~YDY0`m6;XE&G4eD9m zy3>~CF0E965)zOB1}14Hw4;ZE})c`8#Gj0kLNGbetpjd4iuDvq@X75 z6fEgjfI&^XAWGl~RIf-T1`*4<*LSa$i1o@*kbD*WslAV1YQF^@m1awnm127(JF;D^mO?nl8E0rRygq11 zPwtIJY1OH^mf56Y zTdP@jTWkAJXBDIpl=EV0)vpTB9m86;79@T?u*fM$l`JTdLH;6WctHtDf|A2!aG(+u zvGT9+FqAXfpi&g_nXBQNBn2(jou^o%%oL#G3Pn2)1fR~WYE?=DkLgp1@au8U0e&{{ zVG{cnk4Xv&R&j(SH32+WAdVwY%BT!j@Q?|tevH4cuQ^b_`e^L@+VFqijK)|bJi{>^ zr!y5TG*Xg<6orxr3R+64p~aK%USCa6(?Li|(&9h>jY)+zkehCQQ6Rv$>TmON7c_;a zJxMx-U^IKPQ&!Z{KSgqn5?I=l=VE)Mf}^3Q9`Mby9$iK)|;>PGFdl!6Xvk zg(QPg)CAI|v=z9eGHFE82`VcDk936#7u*BOx{f1QanA_kxweejg|=ADsS=l4S^z?w zY><_(=9mf#M5!f5^?;xw7qD2Vs2mF?)VcP0>DIKfSZfyKDrJ^9nd;omtlERn8dMnM zZrgZVRAea6>6=okvTYSjrWTbt%0VkZQbCzmN|FqMu#*H^9e>eZYSr0p)?hJaa)u8Q zp(7fCSR5WOaL++<02q0Hwm?7coC=uW--z!m0USnL`Er+qm{Q}_;^qq&<*eep?7-3Gs9S1KZ49Z@k~9TMs;bfbEvO;dhLE{B zOO)-W!jO!8B$bI1C50q{2m~sGaK8@_Q1(dTbiGFvI(~lVVP?vlv?6^Beu>OKoz-3B8n zs|p){!tPUTHL2N}ETJJ?0bm1Evo`+#BKovx>h2n0%GW6aMQaV08$wfMA#NxJtunWo zZm^Oj18s-}Lltq8<;a%2M8zc%(uFQ$#gLRsmXPXNlt>9eLR3nW!8|0p0%IwcU}t7{ z+$IkTE@avBCQFklOr$Do<<6Y1*vXeI2~yWKZx9mDl$9)y5)?}t^SnCO4`of!4JFo^ z3#fj;IP#lGQC)tbA|M8*i64BOu1(?nvd@+-w+w6W_ZPV%tXE?VvIbXoE)bmu*sAM?LA

g%&=;nt}<#j z1^f0UgzarBi#VBRh5Pnt3Yny{ zX3i`n0GUWw(nY!Xjsq@F6u~ZbE&l*^f+k8}H%TdQ;jCC*FjBIN^Rz2$B2|{)34iYl zN>%IT5JQ|ig@Cz6pS_xps{5@QPlk8{B;nbqaX*~QSEWzaRQ~|T^NZ2|P$yQ9N=X0# z0?`CxS;yyCynPCcxd-@V)PYI>$75&GvA<+Jj><9+nUau+lhOXsio~?ZlRS|3~ zv%Z8zV(oKj1qcZcEMNm*M!;AqY+!+55sX?;LVyK}v0~)bf~+WZzLBwhWdU3u25tPf zJM*oKbZ_9nE>wOdyESp{{S)L$#81yAETDkc)Ric0E<+g!k48&1N!uKO(Y$E{JiX&jaY<&cOfM6k8^YC zim&$cA%f6lEL69pC=D{QrkaGgrIwjnYiY+4DXS^i%gaTihf-4`=ovsy8o}ZujZ{zZ z0?jaxT-BWG%0D*wWcBf2m=uI0(D4MnvY;32qyXH0%F>hYM?o7e$|?0;!7C`ys}&eCDYbQF>K46_c8X~*!YeAJMA(JZFP zF&DFw{{WX{NJOx{2sLFN->ZPT7SJD+~|w;}NlquW~UG>^GGjs47^PfClk{k~Dv zTs)~mKnU98T!g_kHzq7O5&*|J#;Aq_RQ+B2K5ZSZE&d?-AFXt+O~h_&tstQ@xd{N7 z1QU25@*0lc<6E8zEJKh*7+8`^0$dO&Mt0{LEU*GYYi^!j#xoXRB_tE9BnSddsa50w zX^646G6pf~DJ~h!gKy#DIg5~D;sUpGeVkI1^H_K~XT%KoTl9hlQ+Yd$?_r3dWY~U# z<>$UGRG=)~L;8dhaFmH8kN^OHMZx3+@d~};=SY{0Y#5Mq=lax9g)3JPl2SGxDkK$1 zx%CD}gQSSJLL_sGjA{t#KUj%u#6E$!)DqmTL{3`(3-h@#1V}ne+}uGUkh$dzLYjxW zltv~OusXNjlw1b4h)ameC9K{On-cE3JG&lXI+ zv20aJRNAyq&=D6M<#`f3WMW&i1MeU zESWfrB<1kAFJNA^BEvCDriK5z+Ex>kS@xfKm+(cUqy#_tOwiu*T3OKk;ac{^_L22`~m ztzY>zyR6-WX&R->t<_YNfKIw35D5bvhtaNja&FaSs9N}Nxbko_(xq^KwWu6CTSda>Lk_n3wypaMq+UFj-B}YqCiW(3|EmFi)>IkThjVYE8!6@M> z0H%f16utmM)P+oB2BBZFJ*q&7-cO&`koWoFwKfz!8gJb7+mt#6vtTXz`5*J`hoDrR zKzP*nK>=E$XrCyxx_mExKR1Mdpzjbwbr?xK2gC1p=Y@nc1jKYI0`-y#fD;$kiT1b4 z5v+pDUjG0{vxMSLt>X#>K@905`NP$%NC_m3%$>RNH}=GFVggK(0OxOE2HwNcFhKxO z>CV@-+L{I+^*X)bFR^P?vDQenT(uAS)GaWjgO&= z`JK7j4Kza<9gXh|Rfym3<<6Ig$ElNPj(%SdI&}2{*(n20rT%cjEIp!4cy4}2Zhg?u zL6SKb`|{p4`rwhyCh# zaX&s==PnGH4 zT{(_Un(OdyzlVJq1g)c~9CwI)MKO@bemgkr8-Z&-H-`fk7@5My zmj}h)cI|6qQfw;haiH) z5Oy>$aAbb{f4^Rb1Q2waZzSLQdwuVMBtgGd=cB*F;{-{AyaT@2A?Lrik@CD49*5zA z7tNX{r(t;_#PuV55JD(Nh;A<)WPv{7(aibA0jN6Lz|);WntY5@#L98&62OmpM;aZR zc%JA`WJuWINFw4XK~Q;oKK?&r4_l>Awz@q>${c81Xj;Ah07_}fBp~_y@{U4e+jD_r z+O6~co^*+P{{T*0;z9|S`E&OEKfVQmO#w99nT3f21;GXg27X)ldGh$-;Xy!#1?~;s zajvv66v;wsIUtKRub~=;1L+I~M2Q`IzkTh_Eps>$qU2bOjr{$hkReneqeBNx#iaaB z*1iOWNO#ua^)KJ!*L4N|09%f}d|~NO5D1Sy?dNIu;bCC7Ac7@O2Z)2H%zxVgJ{OOC z5TLN3BBk4EH`WnBBlN0fLFM)j>~SQrqMv9n9EjBK7|F8JFNcAATtI>xQJZZDHryC^ zbd52|FhSVH$E@|lkj^q1^bA4Ycf36-K>||;P)WJxFn98U(+yR?8=Npe0I$lYDWLo6;N7nxUXAR;!JUspzc-9sr zC3A9YK!f+!2+mk*=U5}I{{RGB297)J{{V&)S4@|3&J$f%7uKBtO)WWNJ?#rhu}BKd z-=SBFbu?l~IOjf<^OfY?jK``}z??>6Q4PFhBMOCJyEBTJ+MGhtfT>fk@};VBFy(1UNI%|~yuNTrA?D#| zT}=Q!Gz=Qe%UJZhnJJZrEJmh<#XuLPr{H%Gbg3uxcoss5Aj)K&yhg{wm@~;%Gm1)5 zN{f3uxqgqK=e$Bf$MGaJ3y+4I{9%i6Nm7tjAw@wX8Bs`76S73i5P4}cY(oD4w0bi$ zzain{*GP~f*aEHYwaYig7^jo00^vykN8q z_&)F;5P(Fq+_1UTp8K;KJHF?qeI;cnR-rS8p&DpyB z3mA@O7!ykL9yW45F4w$W*s4?+RFs`XJA#m6WL^j{b04a}sA1eI-1x;sOO|dpU%;?? z9eO>ww6v@$6cki+!UfPb(|8dBQ8IVaGHx)x?>y}lmW~jn0mCsDskmp+!LxOWE(Ii# z3ScWxN!rOt7S#X|I*gJ{x4@Dp8IR82*I~RpNGTc-)Dk|erqmXohONS&06;QAldQ$n zAO%PQN~Cm_*D793+ANzh!~{794Bq3ry(an<^t0&v$xP$hKS%ww$YceMB^<{y$g$WgvS3Y0jJ(En9&%aB zAgnwm@=)V(3E%qHutxY(^I#A2iNuh1fTmJxV(tt>LE;Wz8l3dCU zM;u9K0YDIw!rVIRsp|3gKEkC^r>!e=u3D0riCRdTC=z&#_MArm zelDn;9Bsha`a9(@QB9gZj7F*Po*hxu$$zP0Y zomk&TxmIN^QUS`d+zD9&{{SzJ!eb>AD1bs+8oViDTowh%5%$~r6FXjvqtnN2J0~)F zmY&plILt9vB=I^^n09+O$mgLF(od{Z)#cdibkKyS+LbP3z?7j2CSPakT3Dx1GvP!` zlc)rMK_id`By{JGlepA-WELDe_Sf&T+CNjW2;ePRj+83+{rX2pIm7#duzMFK2TMwyHt+rJ;xwkjAB-GlI5A*GMTETY4k0% zBo(%jQ>(=UpAkueGbHLdh>N;Z#mZ4sveXs1ZI4HzY~3g))*$dP(*sE@Pb93 zPK@=jj6Q3SWcl>JQ;Ne&RM+>&4RJ?MZiMXCk2m;_vYyyG6UNvuZGkW+9iJ}kZbzTV5;G}sqb|fy!9ngG`8(FWDPs$f|QVw_S)l?^O6>F;1d~z*{{T3%lloG{VHNov(BYScIvi3= zsjpK0ya6Dc8;yrZ?0z2nK;-%OX-26Dgp{oz(R1NiK~DPv5>2%e69b!OeUHZA2q6Ge z3meq!Q&7f&I<{TfyuUC(RJn-?+lek%=pMT1-50FeKg)B>s=c*TvZmC{Luwu8i$m#B z+ZqHE=?n~+RtgCL}1=#Qf=(H!@(Sxo7m zs}qOAt1re9UFTqj43&{~oh*s731B8TENLGG{{Rx5g->CVUOOxQby)>SS(v)h+jg zkWYU$B#d-mt^`yYmU{|w=~5b|)Dh>w?UDl;`&$4@5?C|Qn6tA+9hhac4%Fq@%$2dN z`X|T&0#IBlXwQT%LSHF~C%79y!$DDbjt>n91QMRLBo1b}1_mcc;Q&?sB#X7J%FvXY zJeG50=I?-p^7=Pvr<1h`QX5LNkaZ%RY`nBZ$x)xywH`I;hqP+T@>y9P5@OSg&H_^~ zhjDSo=y{{iCjT43o=wdH4j}ubKTO1Oy^mR3mSr)ZpzXG ziB^)JkdU3pBxxXCK))@s2Zxkj3$(3)Ft$y=H2Aky7H4;D)i*pX|@ z3leq~wkAxz8l;MVdnaR@?nDR4|3q{57uHc%10 zjfNXCVp2@ccd6FYs9-?^a}Ng(kb@I>%{NeL{#Q`L?o zt?shgg0{!z8l_`EBU;k7{{Sf3$`bRm@4A;8Xt>*Gex*-SsD|37l9YV|O3w?shaufU zjBo9bFIoT#U8D1XmLRN>QbLGN3-JyHFZoOeV3G?3 z0JIX$DikZPk~NDfZnsmMpzUdT-SoRZ>=vw#Vv zmcKejR&dj#rKFjGpr+(W5J@D19uXJj41iB z^&*D$eQz7XCnTjJBwKI+hy=+ZZ>ZkWZh&J~#Dan^KrHX8U#RFPYgpSQdxOq`Gd6%g z8c!tNCJ6mku#g~_z2Y&Cvi|_gsHdWr7Kc&lB-@ns2#Np zh#@8{B4?Hv5_kOm+W!DpyHdumqA#$4CeRJ0^E{901m9s^9j}XQ+EGuXgX$^^n9OLB z0z^pEVhy$yGOLLeA_$&)VikMG&XE+#kau(9(Q{i~1E5G3kTuw(?qKb3k;o2Q+ZFi3 z*zEQ+eJ_*8lZbU!^CmFm9-WPpExsB{aDNxoiMshfV7Y|K{{;zGLT^fgxP!y6A-KZ%^1Nw9l zNh*seK#N{vY%#s%b4hR*kB+-;{oV1;v)NBj9nP-rT{b4IT3*q=yCKib7I8-AvYjp? zVtgu;#koXS-Yk0Ks&;b_WlaUf!(u?cb$XQ3c;OwkzkLdt6@Mhi8=;xN2U2MQ*q5P>aJ z3k4+)#awYppH5iGq%a1rU!L93s=(B^F}!*0<%hulCL(#5@*AIrED(9~^6L>?b5(_f ziS>y151GCQiAs74T>ZbR*N_meE>4llov-M6UlrpnqUVt5Uj3pvj9LjNZg(F$V1~dH zVrGANKEE5`6dWGyA6>rs0|-lR-F~nPD6i`QEF9?yxIi#LQGs!=zTWfpFc3eK4)43g z*>aOD*pgJvOAA;qVX26UBnx$qt{URs!q?Mgf2;DmNbPuvwN1V!uIBxB7;0+oU7owV zLhdW$Pxy~^me6#D57Yz#4)SBu;&8vxF zb;4f1G33ScOH8*34Cb|abJ$nL1htfdFhA|{vE}4r+D-_3L0PenOa1Tua6)PW{5-dP z^nlm`dVMcAASHq134kccpNFOhgqJVmKo|PH?f1XF2uLJp1}^8uN?ToNU3oxBB+t@dSljF6zn%|=gol<;QT%>0a4{`q{)kUeDBMi2#yA`5@hWrHZVC4 zP|_CYP^w)-do;e@{6CK|H~kzpf;L2sgOBfsLZ)p2ztZAc7%Wxkthz?azK#A?Q|` zH-kC0x6%v@N0FQnEFB7vIrZj5z@H;)df^D>#*NLMhm~sxHrt!@cv57aC==G(`W{B} zg%-1NjCLdU%l`l?bM{4TX9PHUgQzg%8}q*1JpO|OLkCHMJ_pkTDoYT2dO^Pa;{;5a zfzwzrOo7(q`=7s;To7{e3h$(_>Q)bV39!e`^G&%L7gN$oDlZ2)VEW16uAM9h(8JbaG-?` znDg_2!vIAc&2LYv9V*!M7xd@n%MC}d@9N#{dqb1#yV33Dt#Npeg%NSr$mYdu@Xp1sA;?R{*YQIfR$;J5hI_dl0ZJh+tU&fkT{sT)|-lS zvo%BWiW86|%qePUMQ!Ym=HbKg1Vhq5vDnE6>m#kbdPq1>LW5|0*QLW(Yl0Z>QTqAa z>0@D1C07J}`5u^3Jh*=cEeifvw~GgZ1x>9nK33o7C(`&xSXl>{``_v?K@Jc(4=ZhX zx3509cM`YxM1HDY{+fIQ5TOEmMs$J&u1xrL-oxXz1T9PhS8DGOj3lP>JmoM0GW=U8{w{KDn^tgiE4NKAjV5c37D1yf>a3rs0LyK)Q787j*&qw0&ba9 z4aVSt52*d|C#^>R01@|yh8AsosdfJVi%@8N-#WumshQ<}Y)23!cXP~J!|f6qaDk>x z$A`!XC(mz&9FKT+{GskC&fhro_x&_DNi?}XA935QkgUR@gNA4wRD;}5w5-E0(nz>2 z)lng2Zl;1v6r6dF@dKNxW-}qk&{7kJm5avWfFSWJ5TS>{yHpg*Pa16LQ;HB{>9^o7 zvPhAWcC)rK60oep6@bX_3sO}{SSf17<3bc@O6O0Ig|RNIIZ5`Vmh4X0v!O|sRHsq> z3%D_OB#v<+KsSpR$Bg@9is6$vkHl5WH9l4NnELbR&oA})gn&V)Va?bQH?gMYNb&o% zTJ+u_<$~6o0s}I*i5uY z!%1Y9qXyQ5_ga+J(aoLqrPm5)0(>sy2taAK7QgvpjZMjnRcjs8_ z-mD2+3_uiiHn+UY?_HiRNLgA)P}E0=f4=I3=Bw`|ixa0`{1VQ`06x?4{8-Oe# zX+(Q9-{S4#7Ue${N`` z6U-1p1Koe?MvLV=VLjPW%&1C2017a)l}S28K)hTLu^3cL;=KrK)AZByj9mEujSJ8k z+e(d(lS3CoKl)Ro6de*S2!yDDLD+=AD3F;^oq)uVM4=!8QUf1owHf)=)p7kF-=qYU z0;me{1r-IT_iITb`eI-JNel3(k}yMkP!uUY5z5!+y9-u`T@EJ7JEc-)1o&d`NKiMM z128TQBK4x}@p_(uCzmNG1gP;3uEn2VMQK-yRZ{f@5|pb_RFFy2B+5dgPS+r6A}n_z z1SzO>ww;!-iv?y8T?iHZR`}RFXeK1UAaA>$aoMD!`Y75$^QVv zIc6SAu%BvPSd47EB*lc3NGU{@3kA8W@l!|nt_sTY##*Oz<@xSWnJMJ5+E+@&Pji-3 zIJdjSytJx}=v!eZNl|ZBrKM_BUr%`#{vq^F+UybmGq8DV%+&>nDxH)qSi!==q<&Rp zfhUx7Z|vLc@_inDFMSZ>@i?h+aJ{DOE+exu#I+zMM2^;E7>tVonWv1Dk&VY?fs_+a zh7~c6_HLf?ikR*H02)k1?-znZ5+oC7F}^+|W>WYu6Ow2OfK4t*9?ERxi-0aGX#KHa z&+G!&)COv}C4seFzyxkC;2w#jVSU;(g(L({p$EhvBc8&*9e^Gw6O1~Q#LmHk0ieG8 zm~E?q@d~Dpwjb1!p%$fV)njNnsus=2DJdf4*bo!}7B?hJ3wjBbzxDXs9rYFG3BV>g ziDGUUtXNj{;eFWjf+I6!#FRV)_hB-jdyr&E2Ed;P9wR0q!ow{801zdt0khiRYi%2p zRbCW@k5|`V!*gr3KS;y)i-q!R;kSd-j68=BLbV~Yr%>2|GduI%4fnAoWXUIqGgOjv zsIdfWZrWT{m4yxm1m_{&R=r)`fw~amNaOy}e+v8q#f9b7Wc27+QHR=WA;6_wJ|se< z5=6~_u@)HJGOpF;xF2FpElf65KFvd=_PN)ye#&IIx<}^n=?f?R6p|U~%aH=k`iSSz z_%%6W$~-m3`R+-jN>dDL{N*YYFiL6aHMRmtFf{Uk9Jcv>k#VYXiNj%Mo!2N3)U89l zm+_4%XQHg4N`cGqS%y*_%*-w+Uag=10MVUtwI9IE;Tc9>*bizs8j7iVNy@8ILuhq5 zwo>|5ttm>Ig;P~9&{2>QLw*?|Q?;)U7USmm@ zsSec97HKN#*hY%1*_wI>AMelPQ(5`q&01+qJDsP{hiotdJ(tJf)bBLI)nVUiD=n9YT2Y8nms+rX zq%bTk%7<9^#Vb6Yh+ZSqDM=hmNdyicTZEDr7jnYABbYln%fADirG64P(}^p1ql~*O z?RS9}+7?}y)ge_pMU!T{S;Fq;G#PeeidyQI6PF`8rcD-d{##0ldRiy*HyNZ$EHc_f ziFrC_WoH*HNkEgu`JzZjP)ij#CqP1r*bqyD4gUbdc$^kMv8%PBML$17_v%o{mc*)4u$3x1KcGSkhW zx5W}^%~AqD{L!c|B;&ySwe4Jt@3mO@>X5v48w|6SU`U!Jr_^AF1jVIJ-l~2h_NiDr zDjw2Gh5qdN4fI`)><^;MqXjl>oECAK%KrV$O7O{4(Tm4>QsqubJV~7+3amC2Xiy7G z%p}FA0=UH7?5xzXh38yDnN_y08rY-msVUUri3(C3Xc~Y>1V?;ykVfK89Ov@_+YN)k z;m@_~%qy8nvG)CLo&*a#ZtLunxst)eW;n_6$t~)NF!D%u4eGdN7HW0xh{@po707{A zCS1y%(6oxGqPnWGH6&g4wXE-}N{Is26Ne`qkH?TF&syY`JOM^g(yc9xfj%~B(Oy}W zWq|%$2O%V|{K4UyHAMda`^wC-`&fTt{C9mDoWObm1<9iL?>WjCh_98wlz%0I#Q1U zOLw?D>!@(zmO8X_uFv*uKn2W}3l`=9r4OG7^bhc#!zPo9^Q_eYC=?tvnl!JwO2cpF z8Mam3EJz`>B{Z(9Qj=t%P^2g&So9y^H44V|_c+DQN{VoOmchFvo2G7uG zs!XXRbOGJnuH%T3TmnF1PMkG0kIw%9XQWKU<7DB}G%U+ks^JAH0%%ActwA89DK+9v z6~s#DDINiKubrbg?SnL1No_RQa~X9VJ52uo(`Y|PjNYQ1A`%C4nP!#j!4s(pnIcbt z{VL-QGqkzlfp6PPz?DC;_$($8SgY!!C;_-Sq_^oC@RN~@lhH?`=Pek#%E z-v@Sm=+X3}vI==pIynPU;(&J+LZ{sM9Xd6L;{cjxu4XnM`7VIVnHHjuAzuqs-@^j0j|eq zsrGyM0UNxKcr0#o+|n~7j{&}rWVjGiOb{SNgm2DFvL#Qt3plMwxh`}*bvs_N^-6}~2?P)c z5eK*}(M2*hlACY57?X;Q z4Tv!Ui8rHpF?I=1Za?jSM1WurU=U!zHWMUH+ZLsQK%ls8`!qtfIw+B-2A2+Z4^gJJ z1}wRino`hH;5Q(NQgo72a3n|rp@VEqE~=*AXmUN#=UCL)@JtyA)v5tJ!gX3nBXUTz zNSFW&W$Py($G_3`?Z*BTh$v8-G`gJD9#u=~9UbP?!K&D3hca<_yJw zoAM)d%T^G#1PTF4nzn+VldV|SrkR_{<24UQuyIM-_M?}wd3rwMD_>Uz7j{NvqE!Z#Bs@Al@4 zwn0c9BXn&9d059Q^#>k??40jrPqZii(v;Jl;hv3GkDor4ut6B9>)}@uGc|dT8Os-` zZDZ2@+3KgW?h$+&H^j6={{Zz<)?D-^2a%W;jBn_gQHYXX?9cTS+`h5qJWPQ#Q{|T+ z5zO3gu9W#!Jg)pu<*CX$Va{D?vJu1-84h_M5Q1|&s)4$P1Pvez!49crU?~n0vW`8M z=)V@McVln?L=VW$oq_@C4iK(y3D}gC00n9TR0#N=__)dukos`XJw&O4`|}Hx<`{+W zo)jY=gUhnhFqRAvjzVXss)$&c&>GKf6DL5F9l= zKRA53UbK=)2n@~qMe4_3DEpclz}BLrD)J`XAWVHv#|YqDf!`ORGLX{^nx%>Y7=hqHCz->i=znZ)>r(j*kW~l&NH!Jfd8kvxyIS2sH zFlN)zJ-Y20$a7I~Ug1wW>qxhTll)`_6Yc?46?08bh^w)15SW!*+hOHD&pw!JT8NG- zQ8T>W42^&h^PCVwRT4?{`e20T@$2MxK-5GV`F;FwLl#AWFHVhLy?dfMj021Cdmw+hRCx4hc=cgmreXWJ=w|_>bn?hdp zzal!@sCwSifRn%6PhT(C94rQ#I3GQE4f=9`e7Sx7Fhmf=5o5{^t>8%XJMD%kL|gi8 z=adj4eCLq<(}Z!r&L3@W1W-{kas+83Ze-2w3>f9~#33O{FSr#fEGkT7WCQAxaYY8z z_=$f5<%A8+UjG2L8#7M-0Hi)*@V*r+;orM> zB7h=qGZ&7N@66(07%+kwrA3DS08A_p6KLr-JwE z9M2+o^7Qs2n8Zr|0G5{4zy>GBUSjovB}h9EaVvvj3@9vL&!3zTAa(0EwWImpq>K?N zX+u+2JrC;L&7@O&{l9!iFJAAgHeS8oSSBQiA(3+=5dlzo&i;Y~3*uBTO?0`nclky# zbiY)scv;4);#Y|9Ay;}tq;ZPWD- zM3_ED@2#*OZrX3p`a=8K^d{xa0r>LnAjE_iKEL1hk7>X?q_H*A&q`~&NRr~W9z9^_ zCO{m>9+GeIwj06-AkeuCX{DO*=#oNQ2~_EcgX%6X?TLbe7XgjJw-x0POq7|RIU|MG zj&~mc3?5_;JI9dz{QRU0G$(=FhNX*|UV-}z7uNRi5cHTZZ9RSe0JoO- zcxxSe-|1ti7l)U))6KrO8Utu3GuPkm`}G(YXnteor!q$|1D!|6__n-&Jd8ami*oDE z{Q6-bVZ}gB-2Kc5{ynfk1qw=ukrB*qGrWHHHa=KuDK<7XJh}DgKBPFHD@h7AxaVW# z(Eiv&%~D=~zg;=MIF4kb%?WZt8WGO#PC)#%gaqmYkS{x=*uXsRVTU;9;C|63o#4PX z8v}TrqIZu^+Yq|9J|-jeQooTt5GLVEN))0Gh5+1!p9$E-~8*y6}n4Vt0D>@G(j z;m(m#M4woqg_WT}GSrlS$|;;rt562KD`Ub~h5rBvrGAC~00;auI76^r31v+dYI8ig ztB9)-Q>8`h?xo7cih%(@ia*PtGY3cj*hV}j>0>(n^XTqYhk&skzJwZ3U;COp+q)aT2hfD0;DQHHe7;*Em}!J3dZ{nG3T?2)O*9y zfK-}nSa{jmK8JQg0uv%tDk*MEM*jBo&<(WZHDXs#vZDn{l`2X7Wg^1xonVBW#7vvx zM8tq<;>~9jv$MTYHor&clcj#h5(`q7=14lywJqY&ls8bA(n%zOon)Ors4HZHBlId{ zs3gsZAe?HzAXGRvCd0n1KwjeDmnJc5mXfbxgFtEUEdAXej#ULtq2?e3exj&Z+(}WA z1?48*AutkL153L2j@8;b+J?LzUA!-WTXHoa-qj$PWVyg(I7N{|61I;|vRhGG>#P+7Wk zBt0!tXmFGQ-Af7#TA;T+M6f)Wny^OdluVG4Ab=#M#1aT35+>e)OkWIr+M@2)dS9zC zBv*HZ^=y{EEl52)v;p^ymFN=nSQaD`T9R}+X!I>p z63dj{_$7wK{W{n6d=H2cqRN%5!jS|RAO`}aNt=s+qa=aQ37MpoC1Kba3xWfX0MV#V z&{5+(fMoj%AcqCr+kcTgP3auzn$|9ZGI<}FYLT*~CC3XYan-gT54OW(lPG96ShXYt z6}s~)Qky~)r2R_#4ED40-Iwh*V(}A<$@xZUCpOCw#3_Z$)A?LL2`r#6_{=66Y{iC2 z{NkO=)W7E&`*6JujIYu!q90BeIRgiU?I&mRKE`&d7baBIgk~L+%`iFMLpoy7`#IBb z*;ZxR3=x#1s%|1ySs`e7cv*y6yGY6h6LxvTc`#q)*P3P3^ld3rjRcieH~E@w_#^&6 zu%_Hed0I*ogekDn5(9qc$6%!9IaXzt&q-b%AB4u?7L&!r>hU-=%3A`uy0MF4sY;#5 zQICrqhwTP8vc0+Nr)n@ygv|1M<08$lN|7(vMq?`*mFAc^lgfCEv4{htuIWALFQWER*)XqHEP;*0%GKH-LNjg=b#VINwL|RCUdF&72+FnL5UDZ*n_c)Nye10 zqAmy;P&G~Wwft)tdGl0>jvat+Xd1JJ9e#$CfaVkhAx=mEU@vP#l)>AX5<##MKo`T3 z4+?Ls8qQC|uGHgK{ma`0DS@@rTsWf6S$X!4gCWIzgSRF9ZwCyzzMxc+yj9jWq@NwSGpoh>W1k zYSdf`=;KYBlw;M65*^5f?bP|)k=$&PsK zl9Bf~PdzQ9hnb+NsHad0)ZI%6Tdb`s1tls&fhq>jM2LyT=h*DZW?P=+c=<%6;juVu zU{Rss!Q%Lm#?Dhv4Oo>&KlF!`$+FYV^q)x-;U7pQwMH2vZ3~m<}gel=J3Io%=Q8Ms4Z(b8v3q@#i>?Oi2Wek&f?balev-C8OeBo z!%7wxxNCeLO*y=3^YGIZV_%8CR=*<_#AL3zbuF?*if;lz+V&t3VSUcgdH(=BC8De^ zR{()WV$WTeUXK;VSusj1M{v}>pg)5EDPUtH%Q#8ug%9|@s@@Z{#0dZxweJvT80M}StO3+9Kx8oOKv5z-7QJ*0Da~7Hp-R#)!RJ&Vio=`9hBu5S*l4&P>j0) zjK*RTl@Y_oIa!7kC`hWx3P2@`IJwA4JUC1guFmI*pE+qVM4#CvehKO@6=nhT+l<9Z)+ye&A2n&J?` zm);#~gqGG(-7?r|UG9Y-f_-(IV-teGVzD!o{vK3<93&Dr zNsp4gjq+K!)_;`ccsVkEH<)LLmpM|D3eF~cxogA9;Ylh~>9XcYT&W-hEfXcF0ZA!; z25ECMpYeWgooG=10C>)8^Cr?v=tX5)uc4#JC8RhTN0AVuBqX&RO6Ho>V34_7Q4M627AZ`*_ zgmyOpB&7`i2@sHUji-^g5f}0XNI2pd+KFAXt>_E6@gEad_77ze5h_bp=6Uzi!<2QW z2qZ7KuvMg!8-)mvAl}AE1bN>a-!UOHp(96sYW3Q**1V&-WI%%_k*nBU>H~a#PK(JA zr70SKAd-3#0SA8}2c^N-<4v7K7ulG***vsseB*nPM-9i1GZ{v`DLY6_)3{OSOvmXW zBbX%a4k+P)D1HvBZ)Y!6#0SwXa6i-sT~#&;$isgxx~KLA(*yNg!@U!*VSl zHnH>PZA=L&YLGW!;7*S8CBU}Pe#*&pN!URXGdBPm^8!MEJmz}hti^$$r+4%p9#@M6 zAXI=~Rt;x&>$s?miq%08-YzHt1^^kdCmsKn6ePO=W^N;$!Pn=`flEOl@0GYf0F=qPNRuEXBH&CL zSYH`zEz{J0@!#tZ&oyfn4mTHmS074HY0Th|;Q*LWB_xyKFcM@9f`E;TM0w+E!7JJ| zabP;s*1NrW3Pz=wrA1?<4=VL>Ms04G4hn=|Xp?Ne&UusyI;cioln)5wws5oP4Y-YR;S z(t6%(2giPxed$L=6{9~}KYV#D5|XTJ=_NH)9gSUqYHC=xChbNs#(#{wg|~9P$GGu? zK+JMn--Yuvgvnh_Oz-=p2C^&(qRpxj-1v#7N=NBVJvZn>G*a+c&Q`Wjm5Gy%mL9+2 zrUXn1tCq?68>3 zz#%97?jH`23W{TdRzqOHy0(!!8~lztw52~2u(vJuG4sZ91h(NNTu#m`#8RA&ur(oY zbtxwE5=of!w2wiyAL~=Hzv(cC4K&?IJ?H+=1NcQ1Xhwval^H(5e{XL*FqbbEC8SD6 z0!g_gR&u~UD_cM+yaW7K>B*LStZe&O8<*=L(Kvb#K2J?T9 zFSCWV&+#3{UXIn|$$i`4d&Zqv{9Zba;7q~TA8uUlf?u+bNeQV0D7O3cx9JRGQU?hn z17J%zr$=M-)*CR849=4RI+8tYB>ITM740mj`>Yx5VAZwD{37&#M5F=;KGh8mykIII zdi0(D0JrWj5iVXOC4-;q=J6G3uxzfx>#6a~K zlhlp;iT1xQ*x?qgGESd5=sJ7_;zzcU{AAc`AI$t2n$2H#@wu{fD20=l-kdH8N%@;_fa)umz3 zA1wa>TEJ99lQ!$^f(RPPiw@ zKS=xLa6t|`A9?c@v~%+x=Lo8U-SdT|ejD`L<=04iW+q@8e*0mFVD%a|JNe}a1_t1Z z`E}(V&y|7f3)4!ZKAEbTr zI8=M1&ZZoXba~XmsKE6xJvtw(j&{SvmDB1X97rZu01|k`gXpcIfd~2YkLK8ujBI`% zc*n!%Pyh>ya`pYLZS%w6f{7cMKM!mWK~k&VpdUK{%o?WgJeIc>ZdCgaK^?kAqzE$ho{Bkgu^r_adN0DJlG@NXw*DH?{M@aO)2X!O7y z$436ItMnCkt%#Gw?W9zDz+8>`j}kgVQmcp}Jk7th6^OWKpcIt{NeUKlK@YVeLI~<_ zv}}E61UMET?FI&JFhsz(1|F3}&e3mQED&=7{XRWA`b4&rWTrrnVtU#KruM-L60kuO zlx_lu-1$KDjpv+gHo}!E_3=06Dm({q7OVMG{{T9vyC0-b*iTG7L{ot7{5{)6B9#Vo zkoIsxwb|~_WSIl5+uZ%%4L-`LrJv7q6-fklbI|u*41;nf%ztm~=Z7d$j>d%fXC@~W z8bpMu00h956K^30)@|f44{0r^^ZI;Xhb)q!KqlfOOhQ1nEy?!-6Qrd{0MhhTE`0eh zkDiqot7NEy!(d3FMy=K$j0BE;A8((@;C=7s&nniB3OT`mdV(jPPmj3kgmKUQlk^`AARlYr&w17qElJq@ zb{xGhtPX(pZT5wQ)JHis^9OVB79Ux{w};oQf21S$^V_9AtUNrV+eo)r^FO{JQ*jy{ zHRaIxe&vgXQB17MILfm02zIkG%kuQAuXwP8j)W;n(4G85$2fxHdn*;}-tYq>`zx&@i$g+gEpN`M6Oqo*vB`cPMsH7x^1uIZow1DbT&j_m# z^v}bz?BpCDmSyzm38u^Pni?8#kpMSEM2ci2>_(pM2;`{|*XG7~jhT#{j?4-6vG|-` zEoV+8sF7Q75ba8{i%00^v6(cCuLqQ5@@4wGK3Rjo;E^iCOTplR6G8z1l$4DN0YVu0 z_xOq8FBMy8aDWQR$vmUQ%n|{D4$w&Fk39Cj5}=vIzy{>d+y@5Naz1hHUqv}uIHW8f zsI!_9XaTkAe&M6bt~Sh*-ndkvqNS)33I3v01_y;;090=wF?5V@EZS;7b%zf^3Fuz$ z^xi$Y*_=APSFuGF4X%c-MiuJ%)Qw9)oFx^iZDvy`(o9N*q5%>BJ8Ccl6DBvtt1?(p zXdcFua?Mx039E){J3je7OUphj(|U8|++$_4QWs7Zg#r*YD5$4akhK`uL@G#x#qJ64 zoMF_L0^xz>THI?(a@YeH70YF?6$G&y>Jr?Yymm0dTx967Ld7X_dheRADy&?5PT}rtoY` zoIQ%_D^t~p?{QGY^*0bD23q(l!dQaV)1x(>zyRkhri+)5L zAuQA*e&+&#jTyt2=NHg~l<`xGQF2?CA}By0R)IDgZ(B(_E1O; z-Dtt9V(oJF)|3>)&%=&73-UH4uhbj03%%lZ=Ad{;Fm5!0Py@1~JC6#0PLfQ>F&J0a z@+{PHYUSRh-AfA!o~#eyZxQEMavl9CC00$~DqvWj4>J*YnY@8|;5^1G&w5Z^n;vGa z&ea8dSpHM-g>lq{NitLvnOK5JK}kx8u>ctWiGT>aVea!?oLF6`ni}fmtN`&a^h;5{ zi5lnyzCY6z&K0SrQV>8;SRjHzB_1S%s#RqqX^AAML;x`&N46+v;J-oz`7=?;c4|fG z1tCO>06`mqLo;qF+LLCEWaCAe8naa@N|W9ctt(q<))Wd_NlFx?spk@!e*grfY9`vX zdZ+NE+VRFjm{;hfhbZ#ji+Tryv2u7 zl_VOByF%TgQK(y=mSw~(Cav((NLW*4BqRi>l`;uWfhX+Zcv6ID%9a+4kBRd}8zo`G`AWx;oMquf3Roqo3icbAPNg? z6JVuuoOyYCE&l)(^`WbMN$zfb;>E38ZdePwE_89G<9@MusHRJ(T2z34qAw&AgC+q6 z_aJnVZ5U^*P+1`)@?u$w4ulJyLhlr1&RP^!_R-k)R2p-)jA2=ZQe>D=L74<7kfY&| z7690Veyvm27E?3WVAar@JCA`BtpEksm&(eWcOW;wuzC&dEMC`W!)o%?B}xcEnSxX| z5>JIHShap#O-hPMyb^X2 zNH7SHX2u1NGqxp`qNlUzOMnf4C770``?xO);-O#*x#?TE8+@+dg z0Du4h-6ws#jOkH9HDIC{yTS3HH5)j*H(wfg+s=`V)KLV!Mm2sp5n@SuzEumJ2_{aU~<2VoV~{6>Ph9#uBEAoSeEknJ-rOW781h?N=|zEGI% zoJmrI?5U7>$W4vT%aYCYc$p<+7I&yTG^^+;PQ445z-O|G@H3dZ;R;v)SxsQtfO%6& z#*X}Rej7g<`QCll-x&TGdpN1>2Z6Y!@cG%6SH$h*b^JX^m}Q((kkxS=a)o1g_GOhg zDJpW_BFoiE^&^!jb;MOA!_>isIs(nb(^Z&FmdcTM2UPU_2LEmA4FZQmxf2R1_3H;xGio&gYz%ji)Um5Ys@b7T7b~ zYT%aC653id{7wz9@~B)~HAivK5??_^)W)IAczH`&rM=7~1i>arCN~mdc7qZIHc2?n z1TZ4|nvJM*{)9PwD;boDh1rGb4()bhUh7|5A>m~>1T`uoizQG=APKbifY=ze@wDeM zQeLBTTKBtr2S#|u42^499;_4q2UWPK*KFyV0P|*9V?`r7JT6{$|E>pqiY^EC0M$(WcVoc46By-gu88*fX z6Pbk*`$ZB?#hf(@KFaZM%^Q9LA`&evA?siZXxxQ0`bK?0kfE>$K(L4sVD2CQ2mq!n zJCoF$M9f)J`#{z17;WLTTSG|BOTi*oqE9*+z0HFbrLA|1`&n>kO2kFS^rVO-MogO$ z0fEYPnT-<)*QF~$>>i`yZD^Rw;7Px;cNVU@cbMxg0Lu_C-g;q;h#U^0~NwR%;})0XyAFbOQIY}r@@0$ELtID9J56LM9CpF0vjbL$Z_-3x7J@LSli z+FwgOnVpD}EmATwitQFKASGy@HzS!XQm63;3rwuT3nwp^F~Uj=q$Z1&ljTPP(w8cb zxCTy;Zhxo~u#qBSI&&EC?r%GR4B~;sc^VOI-&TF8`UmWW6_@K8yRinfr#(&E;T^5P z4J;**v;>INazFqN4#ba-JaP=)Vd8h7p(d=rmn}+~mu5D531P3lcUeL2T;$&vz2;>VqA4((@V z_;g-S&`J_jxon8qZYBtg^%B6p`twZcUwIXDD!@rh?c(IpCoKQ(WsN2lJ({p$RY&ma+ z@cRu-^u4d}yK2!WU45P{oDZ{_(lbn%vUJSGk~}g909+_V_5dsNro=$nl;7g+`b31N zdj!$WY>d*c&yswQF-Sx3KsOqJ9x3#rh7)As95RLJZEJlh3sab+dx6GGxtsP8Lr{QRRSt2LR4jl=2o1P} zE3JaAc^^}bwp`(5O!>I8d}`fe#fjlYek8pLiZu3?lkFsKLsScDBb>hww8K06GtgHN zsoopoY@zh4a?3fcU0;;BGx}1e*G3g2CPG5G4%z5e8S48;WVmc*4r)jxioYgPF(3|M z2HwzO){hZ>I`ULv{TENga1)5h^Q^LBCIpkfG4e6FI;IlLlp%QR8Wf_*3QAIH;yluE z!mTnk)v)UkG4R_Ss6rE3oi!Z!N5{NSCxL-J^?^Ujv?|t3wRJo4fj+x)+YC>-JN{-Z z^7H19H4-Dt-0#274uS?&Z?%s7Bd5+!FnVBu1hc1Y&fk4UPpG~C#C6b$_x}J|>_jG` z=jUGQ62<~xbDuz?`+Z{oAoCAf{*XI^!snt%uY@XqZ6--2TONdM<;>fj1^Z!+mTltX zxk;DA#FC;Mo=)F0;}Mjt$-mqHFvUBBfLej?@v#CWH2FkzstGf>G4QwT_k1^m6)>oK z=+|$gL|Ce(l(k1o>sxE2M<7haJ z{(QV-rAiWkV&t=)nhs9>L?JpsCu?nUKA(K>M6n5Kl5ZeL5H|yy4X2Y;W;fg){{Wi) z_T_>I5#D+O>whnA3XF|L|L%qS`2;sjAQ1W<83 z_HhhZ1|WmK?S?FZ%t61>Cs_rUgMXv}Mq~+*6BpV@_#Mw98{#%5!Qqg_8Bsq^OgUch z^P~o(o#BQB$ezBO{kq`;>HU8FHQETInE?5d^Tfa~17QZ=ubnj78l^>s{{T!Z5G)Wt zq!ZKKU2V5cf-iAw5JNCQ4N-t%50|aB!3|MhXby*=_}9&$CM517)DvO6-=Q%!j1bpi zrxf81FIqG6<{HphAf1l$%pddf!4g41kv~trmn;xLQP11=!2}RrnErh*KsD*GKRb22 zB2#iaAPtGMi{8V2clI0NGKyS$0MlSJf6@p#2!JF_?ayKs_QYxSdB5`3(8Qh>eRd%IV4jipz9fln zXl;|W$Ca(Vo4oxQp3&PWmO zgct7S-){nW+6S)8D{qUN4qzRwL>($4miv#sm^qIqC~`5!@lmrM{vEqfag1|n~9*QM~N96{3Q z9b3ACTlDH8o-LgX{{R;|@V1(9wV=NeBjA4FUy}?H2W5--T8EGaOjYSLPf_x zZyA3Ba(zMI5u1?g5G+#NyK zjb7TtOb;U)`OFdPc*k{{XA`_-yb_^ULS`u%wN?UqCD8SVlYi{-o2+ zkby+n@i*L%51*GWJR_gfl1SUa^bg!x6^G~m0Po{!mhTS^1kT59pAbDvZ|8+~N^6s3d0fx=upeuMMu z_MeBcr$u6lviF)>#I|MXWjLKh)Fwbu2lb$ls@h0ZkIihyF&HT))-t65?IasngM2x@ z^pDY}W-@7+z9l9>2%RRlWrqI%5F9A+mZ-UB-%1}U{{RqNOGj5)p|lxZ8_V8mYiDNGI2Vc^@0a1yd*ZZ`4~N2n5}=Z1Ksu1H0n$iN zIN}+(_QDh>O7=t)v0o7>UM90ZW9S!ABZtLR08xLQ>Nh5 znBANupD~+dH1#d53rozQV&ML3nJs`Nz^(N0 z)9re|D_&7bNGD9LZbisQ1nbyb6%abk#1#Ojh6J7Tt#%C0Ul=5TxO2{xHSnN&MY@`? zX;MYOBu|9J#F09`Nl@4c-BA|9<+yrXw7uvxt(+Ze5=@djZBG8@rH! z7QtH6rZs>HWTb!E2$e*Vl0b;O;w}K;z!ZOU_1f*-4-!^w!b7z#+uLuuJz(jnN|aIr z>K2sT2%ew~F%vQXNg#`15Iv9>yU;Oqa%}pzvERH@r6HcA5Zocl)YNTnFBZ(fVaApr zTFOqSQ5!8pTHYh9l$C8NfJP@lPrc;6K2WRt4k2Axg7nkQ*Ze^BYuu~aQaX38F3^4fFGlBL%nS{kI zQyLPIkO2lF@I%SW9iu@>SsI?BLrpgOp$J_IAxFR|m@8>4D3y20C=RT&O29|x6SKL* zy!*4+Mtz0i^?AlymgX|<6E0R~m&nFSmg89}#LiKm;R#VA*(81g{U~-n9q8-QKcpVO zX0j6^e`EVo+P=@Ba*_(>%x5 zeC(W*klgBexl0<*5Ok@z5%)V7`~j}w_rA8T(V6KPl2-{y79~a?DgY`msXKsBpb0y2 z1HRX8nz(!D+C-A3>J13F8d9IG;_-}T6loqJ1+^F=bqgecA_++|qzH{tK?f6| zU&NYd%6F)s*n$N+#AKyR;{*^zXvM70(u5Ls@r+)klr}YKI*zCsk^-0|v`)89IfqzPpzAn7sz zbHX&JohlX}B$9c=Vs0ZbTrR~@kqCjS(01-j% zzR)oG8+8aud*mr$ZJz?rl_Y`oBMX?|F!*dNug!o_hQiBO65OR|04*v@dovJy^pV7! zKxq7<=UGN&nP&OS`4}%+oL(;&t|_xwB`eGs1StDFD{o+e+5%&nKNUZU?#cfE6CIuQ zXr9YB--kHLpCSA~a2xrb`zmXhs>_CB!M&rgNNVZp`?rEBJGzvzq_d$;FyeLUPLs`j zHR7aVJsI|s)Tped@_a#LIQGhF>_Ot71Yeyazy1vJyk1xMlJtJ8WVA~Q#N)~e62f3P zdP}B&FWDhfe`N7mHO^s{2dI*;fTaX#n}8)DM3X1Pk^mhd+~?*>7E>~Y=X(wO2A4P1 zKSp!W$A=Uy1-EkB(!uT4@r~x0C|xru0I5Z!00>Yi0kB4$QlbeUDJEh>i$*QU#1zbX z^{YRfJPjikI}OJvnDEN82%JD5?@M# zz3T7HsaK3|_hJ@((#EtMY~Dx8T{jA(kfPbS5DHAdTpO6^0TD2pT;4gEnwo?;C8{me z>qfX1tHjcBiD6#qo1xgYk&{ty(oahd?$O1OBO|6Qidem+$yO4IHQ@-(e zIB94e?ylyIW7C~n51Ia<#H&{Opgd9AlL{YDi)E*5GLUA#&0ddrex;39vf;bnrb#8h0AFe zEYBJ0F!Sa!aw5czAO_B#XmY4=SnVv%mCU<^xPF7XFDos{^15|XxU>0Y>Qj@@(gLPw zLPFNJ-0z+5Tv19;^ZnkiW9@fgvXo?fi|nRTDITb0xh7MDP$8AYhQVP`iwXn=;>9Ln z>MmpO9TOSWpy$0q-StEvevFdjYtUDQNI@=R9DREILZ2qjb(@*T|?0^3NPwj_nXG@pI!IyS_E&(YY zmq5g;6`0C&ihAYc8GQD7esy)>Qc32X0?Wu)T2LloJmkjwqTtP@3F(h0XIO*7MtbVT z&RMq_+v6W$`ze4331L%F<6&NP-GLR3#NfI@6v2T2fFKl^8bVCX=51pJc?Bv^Gz3327+CkDzkO9WD{{0I7NaIUY5U66U#3_T}NNNqi5tTp6xc+ zexN~6fMi_wLC_2dx2PD$PE+)&Z%@0#MiXY(SX;2$3MDb&_Nth*5B!ChGixUHUU6R6JB(?R_gI@|UW@;Vk&q7K@gTqFy zwo&J3jl)EY*$Dzpv8HCnP$Eg>2+~Lw=X>LF$mWqcw0k?&hsQ>0ni|JF?JSh54*Vpu z0(2WaC@R;idS%&ue?^wi2ra!zg4#$^j=te$m6~?bAgOOuK9nIT7n$omgY1z> z5>zTpeVUU(OS8XqYIz9pAEen6Wqx@JN|_u+V~R@^s>^#6b|tIGN4nX+8|Aze*iQ~4 zh!oRr@osgMWVDp7D1|vtff&12y0c(_>cGw}Za6K>aTc7zKsXU_dMK7ZQ_1T-l6 z#Bb8^_tXxURy%cVI{E7e<9?5?pXmcp{*iKea{mB04L5%Za}~XH?%WmGylcqP)YHFV z5=ZJM-+S~Paqk#V=a?Itb>;J`LQqg#vj8ey%r!a=a7ju5Gd_{+&*z3X0)Q6KSH}K% z#p!a>ED)D2*CR^`05vsdFH&gH+;FIy_(8Dro1Z)>AY6ggB#5hBk3qa%dVh=#9h^Se z!V(3@i0UN%!=@uq0k}vPcJ8}GW~6X{=D{Mhu%*u-ZgAKlN$0hq#%JMgkESE@=i?US zNh$W$2NCx{6-cYnz?yRclyV}k&wg>V^T8IPE3RrV4Ti=DAoGilc^;S`h>BIp2~#CiSuVPeVK$ERN}p$!9{@6*b_ z3?;ss^Qkq1>1yAo01uo6OWbm>2k&{?`NG6%Rlv6Y06wr48%*>m-{0ao^}!5*9lZWO zcf51w?SZc^D#2ft~KqRmPo3IXBdP19|pCigSbmi%Ye~0|P zNXq{J5BYwP9GJD_%s`#Lf0tZC5Wv+S&c;MVf!-PFxN&EizbO-gj-_2kuB1xNd z_Q44ZkPIG^?f3W8f&yY~x1I%@MFDy=A|^g@LV<}f(1YuleBy}UQNyIJntA)T(!AZ&(hFAllq6h z(BUA0BlM4dslfzNqEl(y?-8~AJ#dP#>Ev~Ql5gkqK^|9Ohk`(cl@A<3EW%{ zKf7<&q)sM>B!X%{Yj-`g=3U+?Hc*>-?YAS-&k&}H4@=&fK%aPb{EQJFq1g)b+XN6)pBC~+fPDJjewXyaQSOfa0GWp* z*SLUoJM-EKJn#4W;n6L3(`|SCAgDjnZlm}HO#S-<%h32RfOj^G*VVp!x#)1AWROnc zB5psl&&v%C?({Vs4R+t)?(~LCC;)M!s zTo6G}PhW0hr``DvLL@;pdVEJuABRXNBwS#LB0^OVM^vlaOi#zJ_8^H(uULcnuQss) zFSK*G_lUs+76Gw~0tCsAVdv8K!i5ytuFgE`@QCJ16995S4#ab~BT+!&ko5quxE!FK zSC2#G>3{_WARmvPyhk=lCxCd;3o)ulDXj=98oM61qbe21`+E=IVL&QBGZX+SK798N zloS#+!Z9YLw|aT$4oM)F*6v%s^!q`92V;ffR{sD2_K9Koj6t{?3ER;7?dj7K1#Cya z!&Zc>GJ}9!oQl+FM&=#=04M$DatDuilh5{-Ma&K|ID4Dry^P%k=Zk9f+00IMN3Y9@Q$KEJrZgITU&TLnhbSBinDT z9_bgOzJ|N$;}jh3f2F)2qgOSfd-#*K8^n+0_z%2T^(+4X)tCPOS^ogGDLRiMW4B&$ z8^9mt*otXMZWd7>*t-DC&8T@7t)<{nH4s7p1evd(PmRCA9`-Ji@-m8%1j;Imw0q=cRX zjK43!;DVfk?99ihSeZ(yCMIGQl@O#D^po)e#!ltAHe$*hgc^5S@Tl1!C=i%3W=6yj zAng(I)3&*~u<4$Jt?O^3Oz zVU0LTvCtIsL0ohV+U;*NKPONA%fN`Yl00+oNtMF=3Xic_p+ z@+FdwQ7{zZgO*q+RY1`|C{@9ySk;_km!~qzo@rpHyoW%8`a-}=q!0}1HUJrsd}`Qf zhD7qMj-U&>)Ey0lY#F0-!>#`SW<@JU-n9pw&8Iily@T;%%v9$bBbY5osL87wWg5!0 z8aMU2!$f2A zsI$c&mMqLp?*9OIu{~6zkV2(G^LV&qQ(iBR9ib)V=F9uS9CL|5+Fy*6!&=NMSDoZ;Df4-y`r#?_x zTv6L=@2ghgtwMH;y^5mpd`kqRB&9(_LP-YpxC2e6a;q3I5|S1L7`+%#jV)IW-0H&7 zg_wgaxKnb(UX?FS@k#(I%0Z}&QgsokK}_k?Krzf4m?8;)43cI(6%{!Krlf)|*IKZ> zM$sY?l+Y1o0Bz$BOUZ!(btoi3BT18Wa~h4Lk@}%R&;$lhCe@)lIvq<5wbTt^46;oX zt=_lrzOLKt41MG&DoI+V9fKg5(nwNNq`@}?>ITFVfK1|KD5wEfs9*-p2xe|AtZTGF zQlu6LL2Gdou?CyzUx!%Kd~r?Qu*y7}+6wEas3j;u2 zv>J*zdlR&0FiUx1?&E4WG_6LQ{<{AFfJ=b5rxLdqrBZ2HXeA2?ZQ6=zbV6J$rNBPUx>gdPE=5GT-P+q5|sd5Bw#SOTz@W9Nyb@N3tEJo|GkAKDJ=bvD2|7DODDk zjvOeXQFho-m3GO&xte>HlbPk2wGaq*ym5WYnQO$Wx`q&hJKmHw+ExObTsRAgN#lQw z?4=lv!*+KZQhjz4J5~^{gz$-Y{0=!_@n}6(1tgL~0RHHB_fPf}_K{d$OP-XwHWw{8 zZ0E3q?eETSrZBU08aOok=pN5=1Bv zM|fdYlNUN>4&>wHwE1N!W>7mZ-)-vSyhrKZ7J?7~xN!TsG3FWib&ZgPYXiby#HezE zCsZoLX}F8na-3K3uRlI|Mk0;_jze8VJns~xXt6q?WLjbmAtVk^V9m6M#EBl*e`Hmj z*5+E8{GvoB)_b9blr+=fufi<#Ls)~Ii?QfIigevib{ZEN`VP&TJ&E|qLj$7x}*ckJC&`c zOF?E1PJnAd<-rx@6>~4zU}o^rptk+gAZc#m@#QN#C-@fa7m2$_9oQ`+}x z93RG+o^M&hN#Q;Z;cgV*ZW-+zXv?w+2Wl&8vkI)&G~)8&s!Pn}>}C0F8&phyR^qwe zrkqw^BhjyDrC^e^uMluxtxiEF9DqqAF!q)QRRAh$-v0mx_$==m_<{6qH!}4_SX@Rp z{>+2Ik&$L3r~p)1BL^&~*a8(tuceElACg_Qf5#vjKnXBZsS0f+K-`g}39zyGteKy* zZW6>3Z94w|n${c8$LLFeqEDB|Xgk4P}Oac#tTt_nk6bL5d zgCh+y6op)FMJwJ&_}#3~^h!$8PTB<(VAnrK0l!K~e_tMvPNYqE_0!j&!wAl16&gCLzI|D4FkIZAZNNRDY zgaNyfppYm@AO|HN@vlf7qd9&cw`y}=T{t6!GbZI&Y?mRMF$`8c zT+DV~jhvDcphV6-A`cO?@NQlZhLfanKc~_>G9q@o2|h&Q&AGlHanPjg&ssNdx%x-c zF3sTr6DOGsL#0CwK=srxzgX}^0atV)1SSX~VMQcBlhlX?M_3>pX_`$byaSfomI!k_F@eCN0kjM;T*RVd2we zjIg-sK5<~pDwG^jMZ#i1w8FJQqQ>$j(Ii1Li4uU1(xSD#yZdDA0&D1J4J~FiSO~JnpaU9pCmvCMwuR9g97I2kp)M4ev{-z zF%u}yOZ_41V#0>uxzdf}89#Xl1du>M)pWpu6R2NxB#0z~V4IzzhaeE{2(|m-&6w38 zwpOA-Nl8kI;OWv$z&8=30VK`h+rC;!RR{GntU{$4lIQPHru@SWw2ofw7mHh?cvGd& zg*ak00Jw_;rg?%vn@5$gx@7rMrpq%3RRho(S6!*QRMOPfGhE#`RRjeGKIjECqqwbG zQa?z>_;jkV0rbkXM~Iz7c!?_{Tm+;l2oZ4`Mkd51GNuHBLicBTxovPn(Uo^ul>d?>VkP?66O;n`#}RMD(#?RvA@?qb}~ zcobN$cQpV|ik&<4?u=I|%-PFbcD|w1GTSLGy6cK4ZH7}STGq8fQKd4aDqum5nA>u+ z%oC*)IRH5T6JyH8`siau&($p9aSE7IEg*3zOCbOuizt;405x)0UF{ux_%F{WJ74X0 zAk8zJ)|)5n?}c+V>as?&2Nue>yNH@4iZ^N7BT38YYM0+Mw@Mmu_E4=#1nKk_VCLbD zGEAII)?icu6bq68YB{4`jYeZ0CG_DZ880DR{J#Meq@j3S)fU{Ng;>?R+Q2__vpVMm;KqE=CdTFSd3m^a6xS3 z@Py)2IE4vJ#RRAaWe^JzBhc=^bBaDN$SQI=hY;;Pb&=L(PNqa?uAp|6X(mtq0MrCX zfB+zpMC_fO#wt?qv*oA-VX-(&SY!N1R}r-w8Yuq&3)QO}d(xIcFBCHjCMsE06EebL z@KX{}TZ0>f{F=&v6afzI`U-jvkkdBsLp`Mfl$XBN}pL~KyBEGl3P zCTT*qH*4mNI&@pt+`V}BNY=5G7IRI&X(JNt;YSo&efKvI36 zLxDv&xrY*u^24kpkA~ZnsOd=i+FiOBaJ{g9?RSc zx$0l29ElLcuTj_lCI_r<@WMD25B~rte-?{T9ZH$Kzm9!=pIjpm?*gs0ZyQ5NYl1=8 zKUed-M_HJFU`_3N`E9pT*9d4h%@wh>*1fsI6+#ZKfA*FhYYZFiJU8EX5j7c?DuYtj;W?feo1%zA)Dr_U9^HF^ymoBolJlPxmAtGhWo zi5Czul5*X)btkDoH=TG3v12>M~GO+o=2Ga{q(?)c!dtSaxlnZ?-70f z0BkDi&0GB-fow-9pL1XbsWuipEr9@?4K4owx&D4cD8G^M)6MM^QV%Yjc}#tABn7ni z`PXW;@QeK?+v|c70Bk?B{G?!l2CrN6^^#7}ztmVk!_ZmXhf@SI41y~J9(?#iH!6;v zqSv_P9Ov}RnFho}OO+ebuEpuoxSGQOAS>9tB0j1WV} z^fA4l{{RO60N(`7JN}TQyLakO&&BBP$_n2_~ju&;XZFDorA`33*T?M-7yke;ZhPQ9Xiv%3|pWe>(geRx4s8@q1|%9`F|-W|-XtPiiY6O3$rA9#PNmHz;yp9rF}Gcgw5sK3O2YepK7Tox_6a)~0l)3&REcjdf} zka~JzURvz~^Qo^od|*yE5|MrH%HdE3_r0U7@Uj6MP6%=UaLz-x8%-LH*hzhu4g#)8WG!*p)uRW=3kt#v0 zO*I2+OaLjKZe_4>sa~#Dw)N z`b70sw5FEY-BD@8fYOS!l_E}0>C&)SDx`aF-~aR@$NI zVFzMrNINKFY{^!Pepidj=GI~p`Kd(z02==Ql^@Kj5CfOHn84?<)rZC4^9f`m6%0Nw zxMd@CDZt@jb5EyTH<+fbfexUOl{QFF0ZJuWNFiFR1jLd`O12jRpWWKzL2bTFp8;= z2|AMm$TF+Mnf)aC+=4(h7<7=XD;kv6pxo>8aAAs17NijDM?XE$i|5Uu>UUD03DDwo zNhm2%7l@LP0{$5$=3wIG%1B{thOK%zENI-C@oNl9Da0+WA~v!30QS>d@O?U|>oX*Z zQ0iFBGPI>i&=KA`?NW*fQBUe5cdjg!(wAXCB}A+2=iu6KU!d%RiAB?o?MhjRZ~p)& zGc|%kZlYPVFRXq*{{Xf_4?7F^-t9{8*l8;M^X%879Kw3kq`FD$-Wx4ykWxZ0QWB*o zgoMmMYJz$H0P$kr%ebqv?9~4N)-}~ErD$xWgtbvf#X)ViT?q&UZRl;b)TArBE*|md zDX;jxl8j%YR3L(VPIZ@%F6tOeehkn(cyV(s0RxLbTYmnLe zkeGl3!k~@z#yBA*gO}QxR)pG)bhUZnR+RAq*0K4wb9ch2#H#!aIOEPKur z0|V8b^tZ{xNi88}^k?f(8%owJbxD;iQ3F$AQZ$lG4WsoaC@RVZ*8~GHn(8bW*f%E( z{tVRyv1uv|>ceurgfTVLi-PY$8KT1YQ(;n)PlPJpsHl|!fCwHDAdn!$UMCXETDYs? zHx4L9tY|Dnw29pyf(W}YAS)CiFuC)@m6b`W@e{|vk%Kr{{Z3&Ou~-ZGJOw4}}9hx)on2b;5Msfh3g{veh=_^=04!!gWCQ&`D&|yV`A41^>j-xMQSMF8G8(n5HJ$+q&%E9!&$dBw34hbrgPk4sgeihLTFkbp+ld z5+IU!Oh^+TTX04lEdgQ4)rNtXZtgb+SG*u0s$5dG8gnnr`GXiPb(=Lxg_5w40)!Fb z5C|sf7K5oIbe%*TU&6^VRN*RS6#&u2yMePBk{_)4Fl4kbpw?^y_v;hoH)CR zNpZl+)C!?86kuGz1S(Y`0NM@A<9EvP)XGP!nZ4SZ5FU(Kg9?pz<~inWNfMB)EC_L0 z2GYMn7^ep1wHc2T^dJ;=yxUy-Z0+(Xp(&RgzTlj{Q zuPMy?w9zHelBj4+8sBrj(i2d%A;%j?Y&fkHuBmHAhum9j-&%gru41L3V~CT);Bhqp zECCfH-Ao>UN8%4Fk?RvVCxIu`5&=OaM}!i=L<3L+wK*QqAue7%dp{6#Wxm&|Y0CkU zhKTM;Qr?1dY|O!J3MfXROOTZ*DJu#EK*{hIrv;RD+cI)pDa(|qA9Tt~k`1_c6y$*H zMuopgpMWV6Rrr7GoT&&1m;LwJRfMDl_OqNs!lA-Q_S`MOAcX?#UMKLC5dQ##M({mk z-aREo!M1t^dJH@E32 z1Sl8;pE5QOFK%g@PpwEh2M^s^TzAko5#4euC`_p=IRMSISoz)Dw^H!?32mehohVR9 z2W0{hCebjXU=4{oiIb$_RV_;t4Rc^Bb)Yn;eH*K9^4Tb1R@t1e0MmW`@s_%+Ev+P| z=}Zy`B%Kgd1_&uIL?$4TcEuTf`0v$&F$K<=eJasojav_o-Y`7Iq%G8dNF_iSCd$;J zpbSh$B6>i)V<3r1D6l(#?`QT#!oq}-`&FqWHrDUw&aP`0&K_u&LdYbQ5>Gn<0aSuW z5;ngsvqu=F3BwcJx!p^n6- z4~H`e3E`@VAT!zC!q>G-dA*r&d)#Z<1JpFmsZ_apt~9ceRFbWJFZ-<)Op_$;B2Q@k z@0F9~5(0qr_=n;pfdHDj*pmE*HfZ5keA;(tdq}BL)y0Oxunb)m08}{gS*jzevN}}I z&?a=FOKF}!p)j?b2 zpX(NU{qESPcc9c|? z8>-6q1*e-X1FD|8iyv*kqfW^>TxTarq(1TSRRBvj_tW%{@ zFvtG@w6IENfIpO~pw^MiX5jIitelCn(!3{UaEjuWgi=*66PwHf{pyqfmxI&Q7MW@@ z$3k{vpMT-5FyVSiO*FJL)$|QC%MjCa?Iq^Y5QLy4B@KX`Iza%AQ#jl^1&Ww+Nh?&v zolm@5qdz`Q@yUHHU?m)@5R^(ssTe|%fp@DUiZOS01hoL)sXrcEVVUK>i#`v|%c@N_ zdDy3Ab+pcsbUWPBX7sO=5=Z|4#J2KQO@e~2F+H=jlhmBw0WcRBj1CU2OS$2`qmSAD z0GgCEk1hH#!bu&S&7_2Z`CijwC_UtZ`9&m=R^mWW6!$?Qi81Ow;AgcZ)|{QU>PR}- zL={OQQzHFf{UqZ#*}PdR64Y2K&UkLZ&c>sjz5LtK&PYEkI}n7a#cC;`HSjv~Xv4n~ zIS>9D?Z1zy8$w&7%rnMwmJ-;q*y1+Z%sa#YWS|E`HDOajUWF8f8@Rq0}^Ktl{?E11FO#QErnMlLnvrO+R%y?uc z+nkQlki10nr6RJEHyb~WaC8GMX)6T^;n2m+rXoip*I|Dz<}tj2)nIh1^ZG}g%E|Z} zbnCC3A<7FflM*gKo`n1L_+TnPc0!5Wk3MzzMT3W$bxG)3{{V=E)Z7_t0yZ1Zy|B?R zo02-JE&92C$dCF{_JI`>I*Io`Uzgd#6kdVO?XT(QXnYAuAvue z9lX{4&;=*}K!YRL3_ZZP1HKVT1F75R0a1WR^6UO_6O|C>zi*UBz_J`AL{*OoBJ6Z) z-<&A7K0sQ-)8~fX`WGE{_(n!lq{;<}B(sytVW9ZHTyOwDAjlw#{{T?21|Wz#bL)a7 zh-y@R-gm#iV1fcn;DQFTa3^iX2#IE`x6jYk0Hs@)ow*-gpn8k)!3>5Y^$&lh2%8W4 z)biqg`b;@${*lULa=72ye__)JdfKcD>|5tS$eTQ5qB4mm%ckEE^!{fG?+7Y&>2K=% z?Z|@zBmhsZ@%6R{908>?9(UycBy#o>CfkAc&jdnC+Jivu&?Oy;gzg|1xcS;h_K{

a)W_KvWBafT^DL8=C?Pa~h&F5%@cD^=__O)7aB zj-Lo>ze)S&h}G_&I>nXyr@Q7NI**i%kHTZ&&yc<$h+(B>O}--k0AIhBAI7bRf&0Ko zH-W2iY;W=>ub)nQjNlAZdbrrL{oiOJg{}49f4`0Ii3Nolmd^(-J>Eo7AZ|eQ!odMj zioa|SLkY3>`1JMZf(QvS5imOZN%z47CI}&^P~V#N01*8Ux${0 z4tJO**4<)n&IiK{i2&};A=1`Y3Q&>b04f)&VKK-yoc#xH8n@``S2vnXD?DS|bKF4_2 z;E90-+Si;BE0O{G_yHe1y-o;NGE`D`P#pC6ZDRoY`QU;kB}r72nTsFren;W92q2)` zTW&&be1H%KU*8DiYFOUD2DA0=!!7a z)RIXQ*!3Zehf{8`u=^tH^EK^Xv_2c)JfFbke_Y4r>zqcm@~oPLg;r*HoBWW?6%$Xs zqP;rdErJ%}XBFBk_XXLElQGV*%i%^_gIv@lYDyBkMAb4-lA-}wXW?>7ShR0GjQbCo zdPeQmN7$U2TCo#*Nsh|E#2IRutQKE|lj~P9XaJdV_;^56#!5oPK~ORDEyJvE?oZgC zXYFvaA&wYtg(wD2gQrsH`=uo{FBHICO-a{LSm*x$I;n2CZFc#e+HTR}dqAvrXff+6 zotk4Yn2FBLi8GXBNS7=5y;=)^0c5dYD<7f1M_q-&_6>L*!r(BelZWh*Wn~MNQc`9N zJfvaOi3uPTPQ@)r#ZDkOoS;HFxG6;oQv`9dgz(nzFYt3fBo*DcAo1 zBGr^Uq)wHl1C)zc@v@aPN_(A7?5R>S96DwW5syMze4pT%FaH2kYG%>cRM(k?%2C42 zP*a>b+tlcpVVDy1OS^|Z!%7&%dq|@>#}8IARFqcIYpJGK{Yj{$bG&Iy%z>pzO46j2 z83_aQoJ*8q1```9ZwV5Ubh8jxRX|(3y{hjK=QvS~z|52!71&mxu8>7sgHR69w7W6l zah~wDSyHtWDe6Qe_NM5414FH-gD~5vDL2wU6L`gL)MeufkC`%6B&>ThqM4h5j!u-; zoiMouEmQD_N{RLI&kD+}m05^23gkfeMe8wnNO0*Of=MbS>7CShsvr^wiBYkUjbVwf zBzXNJc#ug!8jFEbUPGrp7@wEcl_U!(D$rm-@e+}^xiAQbnGqMlL#HB@t5}l3sktxB zuE$n_Ry+@i4X$LSAPqJG2>|Ys;o4-Afj-M{>0y4pF%l}UTx=*thR>@P7d0RjtYfa$ zXjmt0RwQWz$ByCvQb-ac*z7EkVIzU{I=|`W;KRRxwJHt?dN;+a?Ojf>k!5+oObto_ zDr^I&=}1$C2BHdxB79aU^8n$KCDOzIbkmqL9nVIn4G>M1ik1Wl2XkFMX4h@lMt@6L z-XzJ?cLWrKcvLy403Z;P7CK}Yj6h|MV@B6^H0Kz(sNgy~mu+@GH`XSqr$WN5sCZNb z%5^9i1VAIiqL~0~8sxABt?=^Dh^R_Q@gES9lj4ma zf|GHkNhaMv=5XYm2L;I>5(c0TjYTTXF7I6hD%K(4()c#LpX&br4L%WqVpO4s$9AD!$Y7-}v>DvA5L(4j z$P7tQOI#NGc>e%wZe%h40FF-9;bg5SnS{uF9AG9XUe^vE4cU&$X48^T2mqxR>F%Wt z6eTkjeUvS@@A1)^s*f`4?JloNqOB`YQ`(*}m-k~+Nl$j#94RY0RI{ic1t|oTMfj;$ zUtr|O1;!^n!Uh`&d4iPMo&X%ufVlyTbPw!u!p)QThs`nS1xm~P8)rDaGD$o~HtfC; z1wkW(aVHruXLt5WlUUh3h3BZN%4p?MRD^7~@n}HONt>twZwgkDL>zpR&#-(G=TgGN zRNd}a*qT;9Sy&ubufnr}NF)~p*u4&>&d*@l$68a=CIAFTOdZG|8JnHp?`@5(jZ6kt z6Bh%+M{Sz+(TIgm2Iafj9_2tvMwJ63fCv7dr9nJ^F#~9ek+Z-A)xfx=?aI9?_mwDj zcMRN#ci(2OQ4!pJXK0BjBpcXUaCZbq19=l*R1R-SwuDepptS{Bi~+2PM+x14b`~3I zIqcpM0q0e6JV0m?M5JjdF}{#4Ma{vu2G~x4?g_sl&7l@?zR={LBy&5kt36(t?@Bwn zMUJIFivSG5iCmkP3R9+J2>?Vvn-L=xWgt)&4E#jc(#_|*n3)A5-VDGoApG0_A45aR zG1s1&f)tyUl9HbRK!Tt`iBNJ>F9jrN5(rUD+W~w8@ojZ%Dn&)h=*58=5b&R96Ju6l z!$!FQix#l4b_O;701~b#Urcy3IF)q<4x1ArasdV;>D+^hm@AN2KAQX6RMVd-UE7FH z?Ae`&O+%E1tJ2IIxaXjcDY!N4>W8y@=B+h+X_av6U8M0veRgF_#F;HW3WpftjLkHt zUkaseaaM6ws*u|)45-q+nw7rgJB_%8Tu;`{!YP0F7qcL%-N`a`o$ID>{{XRJO(gIR zR}FXGKQLaLy;A)%`bS3?6R};g?KiBJFcSTeGpt}w9w&K-5CL77u_b5&WI1{Gbx^=~jZgNU;>QJd!ZqSPEq)b+Iwzm-)dG-c#dy;9&` zH~d4+e@6MJmb@*0Bti8PK_~m$21@`e#mE`}PcH5F2E(dPr~c1zCl!S%9i(A16)Jxh z7iTk)l1WpjuH#F9?O#nRQah)>GJHBpcS(zsl~|BTD4P%sj@bFbEc<1jV@3gkzY}y9 zzr*wnl08|dE?(pQ68F)C2Yu-&4FgWVf|UT6D@piCLDDGCM&1d{^F_`u>W zY+Q$-y>+0y8Nd%zNl&U%kZLS%?z33zI=jHo%0&2g0!pDFBoaw7Cy^YyKtaT+RSJuf zs1>f-iaUJac+aeIStJo!Fms_5AhGF6!ya?H=O#b|5J75;n!mJfO7% zB`P422c{ovfjxf9fJZvSX0Oa}$e39$yGxkZ4W2tRG(ODkD(AU}s}-9=2l14SmK5S)Xm|Q?3{5C5Gj0BNI zrix-H>HZ<5EM7Jo-yoNZ$a7r3fRmVISey|^2Z#(_7-1ARMM>duLr?>2N1=R3(`w3J zq^4ODuT5Ci2#XnoE@-$NvE1TaJE74glLi$V{alcIC|Vvi)*HvEn2SCLCcd zrVa-g3HHB;fpP?X4HF>Y@oK`)SE@4-tSSV>2}jo^T*Xb2fJ-WBWyzSeB3M$AB+OAt zLR1s;?f8q!zbV^}Y{FLQNMSlvQpsh*t250ON}DJ27TjpGZdA!ahlNN{@Mq!;Tl!lnd@B}xF)w0?}gvws?#$Iw;+M6y81yIr4Uz$gTs6M@Kc z6tJNHoWz9abR-b>?hK&Rrd1Y{l*}t=HZvfb@*WWpF&<6Lu?6@v(%?JM$pWC1f(d2@ zKCt$055?*zX!Sc(Y}$bRbfk6v00ia429&K@d;-Agk)%b;0VGK$uPbAk=97mV-(GyR zxo}CX&X(>uG>!yHb{8hL&_2vq)aYEtLS?M?Y_EG1EH(lMKT#l`N%J;w6azCY)6a&qpK!P{uijq`dUyIUX)(JSrftC5a728h{N( z?OpkQ@ad4c?qB}=TmZaucfh2j;3l6XXlrM0`W z5)E3{zOkr!E6U8g3Ng5#PA3!YsHCKu(Js$IEzDLs8C6W8oYhqTs%1mG@vD z5mdrZsZz-WHWEz1=BbjY^AU%Vg(`Z=Qz)S-CSr|(2oADEf^TMe4%n%v!^%~k4t-RW2OsZ6n|LCVGc3TwApj_dP2h<-gdC;tEy_7~WPZBz~F&O1D($v6{=9Bd71;qD})e>dT(r^xXE zNt{t;^g%{J8IvHBpab7KL5D=#4g(P1;xL>t3Wd`v5RpF!veXDnzyMLF6N;Ge-(d4U z$o7{x%2!-2b1if5sWwh8E&aA4Xfy;&!kz#_y9%`;?f4?&^;6~e$#{hgVD?YCz$+%_2i=GhJ-CYd=v33Bn77HK0DgqXRW z>OE!?I1=TOp~d-i1o3byhl^z<_O;**qnvX1dSkUZxPOb@=VtR*@Sm!e{E6@pT88pL zn7o7h?SQR9;0~bu;z(&;;s`vp-|yE4tLO0jV1_Ak&IBFEzfyg^7;6w|R@7I|NQ|hz zvryVs@8^^SOWY4h<#2w+^M?SG+=J4)?SBYpWW9+N+s9t^#N^Cy=n%Jzxt~@i&{97U}0Be_SYHE;P2CK9ME#r#Bx8KRXV!fni{v*bfK^RL?8e zc@j_YJ#a)tP7ao*BIoVakOdKV=Nz^>kB>foOc2;bDsN$H^MSuj{XI7a1Q5e;ceEG) z88CiOOr5<>AegF#98NUr$>Ld3#Lg!t*#d&(mu~7EKU6kF?cw>x-`fnbYfojn+R>5z z5AyvWT95t58mkYWfs{@5Ua6CYpa@0UCfK+31ic^*-; zLG!ui>1+@~FhL1Cg>{^xBA2sU@kWW!NgOONt*+0?dl*;o$c%AfpuoB?OnEm&z}g86bRf7w-)7xxZHS% zoG5NQN1l+lSXilme83+u7v9&~uGmf=Xc42oI(-5h@9vGCmi+*dggqOYgK{r>uNc32)3 zFz`gE3lkT#{{S%&{qR8s3M1w4A9lV7a8U>0-LQiK4Z*<@f(=9!Dwqaj`bG#?Gi%bJ zdq~thh`&2b$3KP$B2ZdLkU^3l57Yr5Tu&lyc8#Z*1P~G@edm6CO^o3iW*ypih#{oJ z3y~c!B0g8%*jUHSkWf>=6sHB~BZr^oHj;T5dUFGI`ehH&uIIU>URw1&6^!_v?3as41_ zRi;ISx_3ZXs!>w@Tf-Tj3+D=LTHIqc z;cWXi$|@Rq2`F`R>{3-|S=3voZDc~V9~_)-uI#1?c=otL(x$CaNg(r1uWRWaU3**E z*;!AguG{5>B?!f4nQm!=D;x>@wle{N#HlGN01@i}2ZhHFXL?7JUA^&Qvz+3rYw#h* z0g`7>RL!EsdDxi)mOQuGsj8Wn&jmmfioMeNhj7jgj5^2Ak4HIloWCiPz?vH{U})Fm zXuyHY3P&B*S4)J65=rO`6EWo;fA~wN=4m+aL#1sr?d3)54 ztHQCZ4mJW_rgq3-FUyg`MjI+w9C-4u7rJgie>hq>({z z)}S_bV#h$v3mR$HvwEi(t;w@h{{Z@PEkaP@L`&V)V$e|mf17PT^0t6WFD1rG z)I3Yr)hR0yvC;?ZrXC5&OEMK^TTvvrp<~|>OIpdFgPz88p~Wn{R6m#vKrK)kV(@Op zbDzq&oaD#jrMiNbl#-GO8l76x5+sAU#zQ}q`K-GOi6PXRwZ-meKn`8S z8Lnec%h&mA+cUzr4UDmH{Ft`7UgKunxwTPH)(B3d?m&eTrcwyLtBAC~n1FGi1iK_r zy!9sjM06|^$GPTC`Z(T#z0}qrWjT~DM34fFrhlj?gq`HtB!U37jiX$$n*+A*&(Fdy zPbpi{fSrwO80%7Y9nq0LP!tBOK$NCHx>N~BO{`2E$(=$b;9x>pq*SvFMSf+Onkbb3 zC>_g+>UA}y`cRO>_R8kTXG6Y0Lh0^sj}O78@E8 z(Uq#POG+U^MynHLN_Ma^Y*VCz0wM{-LBXg$BfT~v{{T8f5<%i{_U<(s(D^zvF)yhq zi6D?qh=aOkOadkpB4Cun&pZqtSNC5|7uU_0J3yiohi#qrsN@URLu%FF>V8s~x^+oU zge4;LyrdN<1c@;wWI)8SK~%yG$S!-s^we{_Jw_-{Binw#LZl}<1g(-O_0bk$(iafnHOL? z5!w952|s}(OPs8JQ=Cm)xr+<+a}o$*t{*#pANi7##rzYN1uO;;Q&1TV(}w~ z{>MC_U-;wFK6;>E1=$Q1R0v4mmT|eRSv5#128%MX#LX#Dq6ZLkgBK6O*A#~7X(^p? zDLR`7xqwoXg#u@oDw&I$5+@%t`%8j(lU4z9?zLW^25nd|r$-f!(^qD3=5`WDQ4eC5 z972rDYW(^hQRo&`n z+C8CBL%*MlT@6u{7?a^Cf<>g6x9XILB#E9}@dA=kSaBX)`UebLh99I(iJ?I8YRqW+ zHD0{yt8W2aNDAeB!Oci#mPWfG<4PV1DBlyBlJDN`W<<=H*s5wMN66SZ6yrlaD-9pe{yXX>s%#?{GU);GpLMHm}_ADVa4=ME8qpW^iJv5b{Oa79cRv{#I)3m?^V&n-* z3C^)#5iBe702+o5;AysUY~MP}zZ74Gg=bZ~Q&Jk?jywFj9B0B=cME4MR4awF*1Hrs z9n5m~)P_?@Rhgh>3?Y=#7ixoUcYNpZ3RbU2Ie=Ki7W||7^DO@W=WuDXqeI2){{Sl}H9gA|a-tl;IU>&?<$sHg26Dl!D42g3YxnQXKs&XalbV zrCL@7qNyC$Cr^kH036^807wLx7DuZhiBSVqCr354Mf&p*Aok5m{{X~I2I7Rhsjh_U z7P4akSHDbm#fw5Sg(kk>Kt(-ZEC*N z@!7>1GzS*(Lfu=frABnFMCl3ANw^A=Gi1TAAZ&59;4<{26|oIl7jf5CXCRiLjW;yS zOtI?01hAmKw%NfZ#=0Ni3wrGksQE5OB zk`AH_z_CB7HZgxJVN$YG4;AbjTe#ka-x$kGl#riwt5V-OH=dEzIhXwTQ%*XT+iB`r zml{%OnQ0RKT~x^K-Ca(VhRcqixPpfevljno_PV z6N;BB9HOofKuAHY8A&vBgZLx#g_b8G^qbqYFbg_D=&InG$M2UC$M zTIQl+lEnopO4#u?gg=TJW>edhbyE$r+jU7u9ooXurKc~-9a^0Mw51RCwTtgtl@R2q zWV+(iuNQtMB^iCAz%1T=O0+1N08yAsEZ~qpK|<=sg-IX)L65ut0NHJhPBYO~e}odk zeMf6$;{9EC{dP06U7bwi%tr_)nu+)&$(o9k&6K3C_K8wPN5FZ0pvtt!IybQgVhA^l z;yMw^-Y3oR4zgTqLj$wwTfUnR%s$q3V++J2mNe9XKpDvB)zq6>y???q0k#_O5Ke=< z*(yOI;!TLK+=4e4=XklCb0|UEetI78%o*dcAuyQ= zAbymj02FzX0>sY~qJtag)aK4F83ZV(9w5e=5(uV@df<$iK;0{EdA_Q9wycKk|-Z~wHG};(YbOokkUjXezLC!0K~$K zkWSyzXb?m}#<_?tpm`rZNXVoTmM3nM+4TE$jHo-I8k4Az1f&8akg_07!otx43>}4X zIH=F(lt~uc!neC&ZWz>*z=Lo~Qme`3v`7R4zbT|k3RMHWfT=w{uJHuLijYAGrS&DN z=ScE<@pr`zXWUQ2Tn|R-$g8eP^+`zzC5DJ35zHx5dnG%W<>-$`m?YpbtY&Pq7nOwJ zK~QcbJCvZQxQMq11dj*u@1^|A=9$(_+4(aRP^@R>F<~UDj|hK+fDHqH<%YG5{{Y$D z15{a61q*sxX{t(a@{|0@`jSSeh1;_TSj|Qg_Nf{ z%vwSUawe1kCY`+@d8)xJs5pv2{{T__HD;+94i03+G0ySei79YVm&;Y=5B(|AXh1wF zk~9UxT$wg*e~cJ8Bqt`p%~$8=ha(@2K?_zATAh=HlAtGI81T> zj2aGGj>2aP3~>48Do`icACkr^^2$r71__qSE&Mh2S~%zMubI1*XZczJl#(55Q8JPU zT0A6f2x)Gg4>{W*Q}>x5!U z6JBP&cmt_}*0(3Z6Ii+Qf$`_h)(55?!oTMDm@3W`d51m2nSqr_zdgMNA?f84dBA;; z>8Lj*qMaLf0W}E(xo&4+O-*{m2qp@E*xc{9x1aNf;#o4#w<}Ws%p3v-$%{{SBx z5MI#afRQTp>;6s<^1h#bgT60e3dAB&^e2)IuVu&d$LAJca zCbbLYTDN?fGM3SvvKB((srr&4?1M3r*%0}zr_ZO{m6 zlvVVGZ@BX0XPvyf;{HToQir>bgTh5f>OLb9INW%MA~~JFw8X)Qm@;`@z~1&ajVW+$ zJZrX<*Wm;&Uo$^1oL)JJi8zo30Br&450BIcsSz?GuPiJP9|RD^8&5sK7xF&Wk2%`} z5H0XQ6@mz4krQFI;Byc~=hpV$1V_YZ2w-Wma|G7kctaN!8(s$6gK703?gfCH5YoZM zv=3Wvt?Aus!V&D#Kz{}MfvT9!@{UsR6e&l^sjU~1U@Sc zPrN(+MhJLRbb@BlVbTB>^V_By1Q34*?&t5W2q1`5So7P+{{VatL8pH|zpuvxA1E3~ znCrKopQXn$X_zAss1|*%T}ujHo*y_M?eq5i@JrXb>jV-yPfOpaCcr@=Z`acZq_Z=h zkIDun!U%#zh%w5>*i;fIL2W2$5+IR63u!}0CM1a=0(u+Ua4)oN8*=*C0w>tBmb2FV z8++-})aS?~aU!Z7^(sXgk5Vhlj*SXGwf?rms34><1+QBy-Ptxkr{ZxOhH+ET)_8&)cowomTc#zloTvcS z^UPnnNP#I*s+6ffU4bK^Xi`=t;=)eW7r$7sk4!PeD!W#?^5uA+^0PNbioflKsHa4d z6k;r9Zx-ZjGlSL_wLAWHiH;RMt*BLQGEVRAaT+xpo4zCKGd7NX)6V-&C5jicdKueU zM*x4PuL9($Qd+Cxyr(zfio}Z--_NUR+@@NlHU9wQ6-3%i#)guaF^`!40EwK`V|!_x zBaKalpCbZ|Sr{BA<=C3A7?4+H3MR4oX#UJD$s;B7U)Z_gQa>?|%cgdN9I-^rnV#d9 zjKYwAh$jpA2?$f3AuS0_SPnS<083BAAj@*z(|-xvO3ILMNo-K$#?^>d_pEif@9nYme*wv9>?ec~aj)dUb00FeZh zL>(ja*rw(>jpG$pCZ?U~1-r#sh0lO?i-=lE9F&3q+398$Cz!5+BqFS-CP*tF2();N z0yzluyo+KE1#S;gb${#D@i5~}jatNpeQI{m&f{3uynT?no6&xMD$z}z<&C)ZTK>s& z6(qTr)TuM7lBl$lxFTC{BB8KcWdvoita!stN-+bE2{|meZsbb;0MsK%)N&8B1Tz&Q z^D=nwgCKLf=u81+KCAu`v*~^J2T>Q{{W4u+i@S2sHtnYuB327 zl(p1u4)vviNmheUr6teMw!$53skJ3%d7pzx%4eP-GF-piEd(tK{{WFdl&bm@eQ6x? z5sV4WWKQ5Rrm>TZLV-%viXw|BETG|>k#Gggdcv}-J4}Tn?2`fwyaRYQIzfb)0&iev znasnKOmYYYl>}HZxD`5hwG4C&1p1n>KIkM@>9vJ>%wtq_q*OszNJ_$BZ_w(CZ3K&x zJK`nFeMK#$`u*cHR1}_Xr$bTX#(2d@TQ^W7#*if5P$SHN02J89YSlYGpBA(z<^bJ=Jk_-n zF^AMtsh8AtDKJLXCqa?`0(B`@lQWBnlE4x`*d&5<@jq+BmoT%6iu<6K@cXdZ?UZDh z&V{4`xk`ah)RGo$J`kdksEsz%NF)&$ie!*$p?y9k)VmQ^p@z^$8}>_u+i zM$DtHL#?Fufy709{Le%-a@cV`sOFMLd+qoTenT2D^$}h6GXc2S|;; z4+k>}A|DL7Rhx>}e=%CqN$%-$3UoMz8)4T3y-w&tp-53LC55vJd6fO|^ihFES?JfX zm=Ke}hU}MTWRGNy8OzC~0{lXTT;!D~GpB(?(o*B_efYG_m)d{C*QY$(PnYZSKTUp+ zbL9~>u_#(qicx~YzUO^VV;LW)lc!wT@VjK}xBn|iH%d1-F?@muPW*HJlc4suT z??OW#h_wEL2RfUDiOFbszHrF6Ci`Wc)je2B~h1h^L z1P5}#+_Ml#=d-+I3ktmrjXvZ*CwL<2i2x*kH3MQLGdqJM!AJnwd3in@763t zRaHq<;sw%m5vYhTqxww5S^<+1Futt+05ywP^YYi_3r+PEYBAUJh%eNX$w?|vfieBNFlO`;75+x-y0F$X>aialx9ZS7yXGs7z5GuVxmeRclHRoGKI=Yma+0rMu z)(Ir)NIR6s@Sg1>P!kF%j<)zchti;4gLfbcFnT$9?Kx8{fv}(_qf>1*qX@zHspJZ3 znedRx)JlmXN>Hstgy{sS0VKusM)Qlc2Zw zs!BH7cRI?NqcxRI{%N&QvJ>@>v0Ov3J(-phJfLK`JF%DQX@SCVI3KhCbaAYHU%fJ) zTj_JsO2oV$61#2MWP!mjvuAk3+*G*)P_=JpzwjS>OZchmMxE@PyppLoHyEO&mUzX^ zF)l1{y--T3s-Modfpjheg*uu;npBdihd_1Iwv;o>zYsGU(bjD0L34vAgT#d2ws#Oe(7`aJ8tyqeY2CXe#M>xj7si=^evR$ZX)aOcWdA}_^DW8p+ z2?`;_Sl3E6;4m$yuyY%Gg!o-cEh$Zn5*2Vj@Q_4#e^HMs_+uKzKOU)u^q}f(y`M*} z)s4$8%aJJ_+DR=}4E;P@@7){cC(6>o5CT&wNC(Iy?R~B71|w_`?uH${{Wcm7h^F11dK;y`#mtL)k~{n_y7ncb}1ArL2_&ieiFSt;D5XS01)1r z@M{HU59x!_#sH+LPMIq0UuhLF#Ig!gE^>smsYD5Fz_KQkr1+=!xNXGt!IHSDbTF#M zGYW#@SS@$xeXNOWx}_=il?K&JSrl&6lmik8p{(TiEF zzM{58fp;38G#h=0uB^`L!;Tb{NhL~9DiRadV^U52>b{`+SVEk8_HARQYINCS+;? ze@bt@*vsXZe_e=@gaRJSFRKAZQFRuV*j0+Qcbl>nj7((A(vr?PlFIB;f;If*U`Q~bnSCt84$ zy&gPygU24ydPvN1O7`&ibkE78yQI)liq6Caccrf#p-EMkR1CL+auR>vxQL$h@RD1~ z^7SF5Rpze8geH-d1h5qab*?gj5~4Y9M41A1agAl+uuDL~ODFxNcpOqBE#2kLN3}Oi zPd`Q~bks@0WLOi!m}BLtWXXc~DH7GgjmJ>DvX!sYNtB>Cu@jL!KeCkR>rNiYa^4)t zsB%oT9o-&Y&}=Pd{%74(DyF8Os#NRMPLk`cD3YfXr4uoZcaOx+#$u+cAUuftc=XH&;hYqP?cjDe406dhr3eh%o4orNz-`TMm9USsQ!o+GOO zOY7{7cB=|=G3%${zl>9zar3#c(psukNk$o3wfB@X*=06ZmfR4ab$&nx0XOf?(-`R4 zCSaUKD~OTB{2{}T+XrW{VP0_Wb zU--}P%i2E-s26G-;F{dVmAxfvRc=nwb6RwvC;tGLU7`RKiIrQB9ox~)VfhT_CdKh2 z2{`#`T8RvlPhaKcl@mae%Oz@d2|+3}G4uEFGugCugR(ug%7PU!c%IGXhD@}it#dOe zGM9;w>d=r%rY>$8T4kjmscTYFim4#^Mx&&IAfW7bu^}MN##BJI>Lf(QJ-C)qj)W?q z!?R093(Db;7eV8$R$OgQfH9^?_EyJUmeLL#$HXczxDp+M7bTD*{Uzw#1Kn`l(;Yp9o^zNdzRHSRzcPVhB4<@$1xyA~824{QhK| z5I|I^RgdAV{c?Scqm;b6|j)-lg-jw3FMF@46} z{NRKJ=iB3ZdB?mGaU5O3f<0q`J==mmn|K=4HXC#E#7k4qTV8Q8)btkDoHTiPf_VuC zc=>wa5$_K=7-*$g_wS?!SB3`rdwn*y&kM!uwzlr`PmClC>}P2H-*JYEj-m)qd?NGi zZGLhG-_CnM`uY3rJYtChatE#mAW}3$iRm}qewZME6##zEzVqul^5=pG88TyFP5h4B z!1ez4Aco+AD+CaRZ3YKFv-j(Q8lXuG&CCMcKTh~%&;6#>B%2O)-`{c9p%Jm!=|cKe zzXq+WbP&^TSu#bS^c#O2IpBn5>gTwQd-UZLED5|wwA;=_rqDh94pq*!w?PY`HD?|5 z*UiX>B+Q9C;?wSaKAm^FVN+5=+!k?neQr1Lf?$baut5tvxlH}@!2~=g-e7xl0BvDl zHnBXU;mT(%1Qi5GJh}N>`;S=>z8eG>5y+2`!2~LZ&_J7MJg#I?@Bq2=m_C1Jsb8NjpDlILj}ij(VnYg^U{LSRwP`DxN;D>A9XAtzOk@mOo_r3^7C?pViV1)#N z4$xGpe2?DPB2=mGf1gM?Rt=4h%LFQ&&4zfl5A%)pb^k%KrOhL$?Y;;h$fk0w_Gy>}PE@=aWwhOETZJhJOm0r8fDD+DCUEsl5+x~f6gX0s zTHl2!N<)<>&;)?wy#T9#A~fI?jKoTlAQ3WplqJbS@y$^7h?W2)YDiZSfJrO?Ew5>R zsfK>n33h+k=Vlb@4I_qlCx?xm6%#RKTtwI2Nq2T?gCDN{0Ks=;fAHsF`!|9_l%W_LlP1g1I9MiDUx2}3 zlEAY8iAtBq0z-gf=j$QjJlBReqbuT$9j~a%Gu(?eq|Net#*t;ZiX6u;szpT|OG{0{ zkn?C!NwSg@l}bq%;}|?znui4wx+;gpr7oQa95kU*$XbxHP< zt2BA9u}jDH12@fkD>ih=_$<2@!<5gM{Bv@d%6u&4lT|8QpV$*&GuVD399_e`r&8t% z4Z7e;{6oaCc~JmIQ8W8<#}UtaIJrKxQqV$8loFx{eG4&rbMGQOPubql$;L~R7chkj zg(;1-1y1f=$Q004HnSOj)D~)mI%;orEXIyryEIQtTW_`{KtM@Y7AM1SIS@$}i`d6JR5Ex}z+A8`N|vio zb87OeSVH)Aa$Hn9*}<(TZ9Ai#IQxc|ka4a_$JuI2XnFL|XJ|kvOXzhgZl0)#3GW$k zX-b!*1u~`UklSH9uV*s>Hbm^QeNi=r8Nq3F3|N{zg6B(0h6_v~SuB#8hR?xE5~PYl8<^S_b6FAnK9`3INBE*K<5NrQ%Gwjv5AvIX3_F(Ix)<|D7Hb+zsD-9U24L6#2qHn%GESg3)rK+DQ#+#@V|4u zv=+S|1J1QH6yjtOqj7%`SyrP0AO%ba^A;GrC-#lR%mH?ANDNQR9jQjFM=?RdL1c=a zbznKx`@fB+FU`wOAzFw6NF`7#Ob9&sTpUcPX(?_R0zILxTaTCpz@afzPfr*?N$Sk> zJ;a>>tDhaOlII~o02Higf;C(cCRCz9AjeLkY;lj7haWbfspwdTBnmyO#s!oSK~Nh* zTI%k3hK$KIn$Q%5g`1>EjUbI8HU{TWH?^arexM|ZIeUUncnt0x#+s5Ch^!W_GKMW% zJ0GEbUNA#g5Q3!wLSaN3C&EfdQT-~NDKJ3*+(eyuc^T*S!94;h1o@&B#>Ok;1AL^C;Q8{9k9(N6t5eb z=e?%JiRlB05_62l51F}|qCgWS$w~nAiS>&K2nh&b<^KTUIZDj`01#P&Ywy+>%jp|v zRK6ZrY*$t2Q)RS-G|JM1tt|myRO_foaY-Hw{8mmf`$v`|i7YbYsI`X@lnEf!_|l{o zp;8n9bjRp3`yQ}po9LIdxP>8L&F#R;)FyD133+}83Ykkh6iOyZ#);D00%a{LAQc$M zGMtS}(jmmB!V+MmJbxRi}vAF3{X!Yvt8CgmNorBTz)N8r3S;^2t7gGTtOA!Sk6h@;uu470@f&xLXv|_)j zSWx0;6)tN^>#JHNp+~%z;Iq|&P3wBwmeh!j)u1GWE+FX_Aj};wc_(`iB0TXtxk?Vs z#h4IS64j`yQHXk07E-`XRKlT-uEoy0faLdhu`Z@6h!;s^CIAXZ36TX_N~EYAj+leK zB=t(+qMh`w2!58OScb4;cz>*Aml^{4lH{`%=541)z<9o{Xe!~tlBQl%r%#3(UaNp1 zQWJ7#!l@8OGMFThGDy@12U6Q|KLXJnD7GGPwL+_*C&0U~qZT)h9CBKZ%W$1_dkNuc zY{||poO-oM`mxz5X_wh9 z&liwY{mFp@l+5+d`79UlH>PF zYErlMRKSr?T$s-LI{cD<6MhdAm&EB;gpGIY2-CTLh{GXW;=0LiTBkL^;D9{fh-x38?V4<$%^r2hc&wG#xZR4V8KZg=qrwW4kF zxEG)m(_^!IA7|D+`N}SW;Jxfi3R`O%?l&Yhrm!km{P>)wF+QH49<++egcGQhh;}7N!rt3>(K}xZM1b_mF zAjpCQKnfmnu=>eT6F`@7{6KPl0(zf@q-9RU%yk72TH#!ojRk?`C|t(O;jS1ml?FiZ z6DCQY>V+u55xhygtOhlVeqgl_#rjm(u`Dd^ZLMQ$%d+r+#LasP)QXcv9Jj1*ET1Y% z2tg?cPzewS1QaA#5nxFW0pu?S8v5p_uq4r{bV$-YAe(|ppokZliWwMx66Q&zq;+9OWd%))sK|?KAV@dowlWwA z3tZb#m;lG4u%YNk>tl?^Vp557t*2cq^kG)7GaWU;b!)Ed{{VyQ61s;Lz8|D)C6^kf z)E(i~{{X~=7gp$L+J+Ua3EV#W|{i3y#$0bxu3|OIpjy zxspt*ULMmw)-&@oda<=9X_D;O}8hbm2K8{Q+8I>(xQ6sZ_YCM!S88Rp^+Hxjvr zg)=yPyalyBZ&60dIlm^TZeDTWr!#YT{yYQR}|T*HYt@f`l%{*GHMH3)18TtI(`;*b&Uk$ zb4h~`prAy|CI^Ry1BF0S6s}C7shE^E5~mr#Nh&hLbl>NLZ6qoaD1u@{6SR>EF>nk3 z2d*@aR5*wP8dlswwG14F)`wa~^DMKZl17G&|dusJP-Ay)l!)BPDRV)CfXZjZb-IDaZsP7!toj*}l=>u5Q zrE;GMCQkAFK!dTBOxic>xiV!3G#ZmYPORg(-h?*q%(G)QeE{h|7Qe!(;e(ojPJ_dk ztA-ILhp66R)Pbl4EhTb6)qOxgA_Sxx7?2Mu<(HVL9>NIWsVXD@Ub<4G^*o+1NXk^d zZe*}jG-ovpKz`cNuO039W#gvu4l~M_t8pnsQ|Mr7R3}+f%TAxXwOHQB#u zR45C~azf5Qo2?P4%(F03s^}KA01Q)CCZVL2Ns8SwB+qeNiFn*L5`76sWdTwLJ%e+( zw%#bqb1C^QW0+;DBqJ91haiNR!~NK!6~FKT;)R+!*YK@85@C#!wqcl&X#5mPU z2A$BLM#@6gl}LgP(~X-TTI4H$5&(L?v%cZzdV{!lKLWEABq!iAfk9XwQ(V1mA zZZ$PlF9Rm)Wk{c8{~FWXj3&EYk>?MS_zn2by8A z7*y~P#FWlXDoW&*X^ELBQrsy#Sg(FR?K^cF^z`%J%NaPNC3;m1`K*ro!SR!29yS3IR3*^bhiiL5Z7eFYkmUw0a+h zog(HG5$hZg?%Wal+rw0?;uW+Vk5~tg`TF4i)RW9q{65sw28(eVl4qEke{U|hcb`6f zu$tcc-%kx3bk*I2ksk~CThEjM7wRyU({txebv`hJwY7Wqk z2%mI!{LDh;+xheQz{!DfR269e2>bo8=pbq`PuI*qw~@SHa4uR1Xx&pW@ZWF^$3rnU zHQjV+Pxy|Y9e|}bySyPGBj`T z_dmuVh+&CV@89Og~!0@s*5NF&gC&Lv0y z4<7bs9e%Jv;m|Y0AkBKww*Ot(5*h@IJmAyzI>$UdQ8!{(uU44bD1-aYKa6@4M++GZPi2nE?L7^MB z!2bYPa0eZIq#vKlX zxIE)+wm01Q$N+Wq#EDP<($?qaQyA&eeOCzuN(2|Dy3KMoa0gDXpqZV?HuvT6zpflv zB7?inmWUv*QUH-)uXDKmeX!^tfSyz2H|2Zp$XocrTJ6@~57*}o1nB&Q^sR58r`ie# zw>XKs&%*xz7{Ty88&2?bv&4YCFQdD?HT_xp;p1_++hH7t-bIIAeDEI*hPpKx=Wh$Z zdJ}!VZVxY<9VX!VNdDbK;58J!ht8Yt4uYL4w3|=uKexjO3wzW%)4RU(4&0t1nJF_s zaz_iX9PT~>0Ol_HE$8yvf4}TO z6|7&@E%&!W&!wLTA2U|xEdf4BVoA_@#nUxji3llB!Ldq`LF3gm0J!mf`0`SdG4C9Zs0whX++39zum(U7pbpoG7RHFhD!)#9 zQo~TqUE{xERF#gpJGHH&nwnS*9z6u?-j6ThzrwE<<=Lx3pzyC9RvfT7a@#yX!}(oP z%{3#!fn7%9`au9K)e&_o8>Ux}n=7;UnUc#KJ~2ElLw}1)kze>pC#&?~y`ak_&Gdzr zpbz%knrK}ggkbT*h1#I8O;q0UP0C-(UNNDn;Y{)6uvFU!e3U69k)*0c#__q61~(HJ z!^+3X%q9A{pg3Y3s!b`YYvD!_x_@@BEmy9Rxr{(OK_`W)P-+eS6wt=e;EvIob+(zP zcu#;qShkoD2)*EvFV+FRHH_CLNE~v~Us1wbbGMBjV;g=?pM@R?hJXqpYupVe{zF>ajkrQ$GCB^F3fI)*Q`u2|DxIXN8%3yTwgDIfwU zXvEo=(_1wOi<4F5B_%4<4UEEdCfaSb=VA1;c3q;%3^;G8*bd!n@jKloJC>hy`g-5u^~H0$})NNf(sDtl_azrb;&e z6)>TDovBjeKzdilOHBC_WK8ywS*~2O7JIR&X9P8)9pH@RRarX;c|}h)j_uG7>2(V3 z+)3t?q=G>OKdK0{<6_UGAk3FKSUfnO++V_8g^3KSyE(CXG-;U@CS;(zGSmvNq$hyf zDgDw7LapquI$RSB&2taSz7-%Os7a9^i3AcZEI}a5oAV%GB+BCVQg}D6^*loDOBSY} zJa00wi4xqhYHmQj+m`tb{Og9ZK;tQat}H;Epd_h4Tio19< zZ0$~v@rdIN2vBFC0gjX|Da&59&QR-XPSQ$587AW8Mq&uQ(*(@k)YB}3?1D|s<6)-v zqv>d9nyNE5ItFTL9-8aTfsAK3Xf3sF0@EPe+=4d)l%^~$YOMURfkt#?7P)HpC0%YG|wlcU`37D5U z4es~e?#$X=@rau|$xvbgF$A`vx{v<=XG)V9Uk>pCPjr_Zf|!C-LO}|Xak&Q9I}sop zMTx?cun8m)RPer`g`b369U0!NWR;3agB`UtsJAP)@N*l7F3mgir0Yt&Di>5qASeki zK!Gwu57Lv2a;305AeIyfs5NJ`D^hJ&T#mVJ58sh$lkqEJ)~EPNo~#~T?>=$p0D)(^Ak0}!_Mlv5~nd$Wo4iYjksjVeh{0SQnjel)4N z2Peoo%)PZ1K}bln07er+Ff<;q6@wLpEr ztq9~sqgRY4h_XKDtwEv&lOsu#kYqsvQQ9mH*BtSiz++VPPypZB9v~fd-@&k1mI)*{ zO_`4Sa=$Ts=@?#Joz`SEX>DsEqOS_I5Dusy$c?}QU_k~)Tb8%4ArL0SPwPRXSe9WoHB$Af|HAv)K8#r&gM|QX!Vnl5q4TfCLHWo4w7L1vZ2ujad1HdHG!j8jhdy{ z>!6^?V6yO<7X|9$XJ*tr9JP5GM&8PD#@I?E08RJ(j!*Yw!#T2xlDjk5)xtrs0cDdc&ucSBM+8>weWmd!;N;K+;kCI zbj}pe_!{_azTOYc-K23s8K!--xOKqfT^PpZZ&F~5IhWaM`8K0GoHw}DfLvg>fOpK=@8Sk3_6v$N0>dAG!;&| zSKk_rS{g!7+wFhNp~`T#3+np!=stJmeHZpbq^E@$WXusI66OS>4Vh*bX-;O$LO@m@ z44FxA9Dou@q<%Vo9~o+Lf5ra*req)y`ONPDoP;POapH5Fizv?)kdl(JqGu^bgeVY6 zC}%O|_v4`mZ;UuK)EZ+oX+Tpnj>V|;IlQQ_-8OAVCAZ&%0(GTk5zwVhH)((le<@zxJf;i&BU-$ zW%0_&GtS0IN|NN2r`F~(U$z{)@bT!Fb4kgwE1 zncDUaP~s(zQ?(fMGz2vo!ZG-nB2hsW;_DymCuxvD`4P)^*FI)N%DOj$$|usvqh9#B=P;IZ4_#FqeS1!4(> zkaTd>U>LK$wx?d`^271X$4C9EF5zl-ym_^AZ9arJVN~c^og}~(QV?5N9YolP^bewp zs3vwNA21)-7fOic^dtwTQ**R=H}MahtMWKZicIAuZexuYa$dfv$@PQ(0Jh)62Mz0_ za)%4LyvDbMxFVE--F-tfd9H~wms4pa3*H4oiISBO2uVr(a(BFRzf~(6o6Mq*?C@mM zhY%_2QccJFLB*@2bIhX`#IDbFV-BI}=A}4{$A?TBYbgr=x>GjOxWEZ;6{LFo!0GO2 zsn~#q*kP7gYyh7CLI56c089ykVQ?|$4D6*-rp$WXEv?A42&aCf&3%=TFNX;!2mmlH zO#pL8HfQQc4H@lI?grM8bfiQT3j+ZmB-oNo$08s=7{HDWB0_3GbEqQEZ|iXO*@5<5 zm>+13i&gp2k3BCMtBBfh(4aw3I*$DYsF{K+0F;|v3BEJf38_E~!=AJ+a>j%;-0yQ5 z-ab*7PZ0rEYBjd4JTKNY_xn7o5}A{9!30lINq{XI#A7%4#U&oi=~sH|MtZef%bnu} zthb1|nDladS)YC4`A?T6h#@hxz!CtMj|e*xVS7YiPAzjw5*%!9)o;<M@Rk!hNLZQdsdN ziE=It38Z@8_-n0gIWAK292(NwB$>Q|o%vf=rpKl?da-F)R zpm>5`ZA1-t^`D}g&|`Zs`6-J^r)87Q5EPeCMkCw8sDR`p0ZR{HNz_y0-{R}ClQJJk z$3DX0rT$m6xmE`?Qi?!C>}Fn}mE-WT)B-~BCUl&?5VFBb+iyF} zUmZF;w|I8HuglA0=_3RYUFVn21QA7k{dXR*&(QkYXdr==?#+JjRCgr%^7u!~ zsq4AtpPxF!B)#f))8*&J7)St{>_@_GI!y2Bfxx+FNCK2&uJ8bZaCVM3Uja@t9|ODyuZ*uAnyh`!9PK@jn7O*0WR8k@->1GfNoW!2D3f3^dUF4QPBrh)WHkMeTX#xxU-M zjmG3*$oGeT$iNsj;mK;aZSCah!YXb_5zfSm%uVJX&Gs7|#v2K8=c#X}>l9B;uq__8 z8xVGziSxk)IdZz)X6KhG^YhZ^0%+3W3MyGB^U4Ws`xEZw1!}-6cwe5l)lJqyB?mu1Hm4(3$ ziopaZpJ90eZ>LVNf(S4mfgfj|zdR5@5U8=+mzD@1xFCcA3FbLKHUQXPmF#yno>*eR zqdBSE-|0voh*vJVd10tlo7r z%dBlC{SQ7|;v(s?hcQo;e(->iWAgb{zaHp%RXfD;^TU#-yFQ%aM5*r2r#N^b4>NXpcyPJi!CY2uCwP zrTWyD=s`PH91kH~=-TMad%)!e3Iq;cw=MTFc0Na*4=2rq`}?9mK^%?Wdk$svJhy@c z8+lqIar};7TrZOf&E$A#-L|Ev1qYwV{SWRiTy@laAzl32loS-6Kzk0bKKWrBi}>l! z!{t$^5&EfL$e##mfJqR3{i6#MXG#)^Sx(^4vxe!RdUP=4s%%Z>2VQ&f^7|Mf8(trR z2yjszW2OjjQ5?q5T|=m!TYm9^B;pNl?Y$qYR-wH8`Ska}3&bw`yzSozQ~@0#e~7*a zX;TriIr=qWHS`4ga{mB6TOPO|;v}*`S3qt3d0U(!l&Bx5b>t_@?~uU(<1EQo-`OI) zZw&^=tU!=vHa>)T`e6yS&v4f7)25DIEaBvo`)eN8@Pl-X&cx}c9bigYZeVkm79D;# z1L@**exIh?#NMh`@+ZUX9Vhrncu;n>_YgUtBZoLIj5B6V<)^8#dfFxzupk9ir%3_| zif=JIhtrlf^1hBULX{A_Mk5D>SS$|-#3f{!nlfIx07VTwXZRY(W@jIXPR%NA9!l`I zEVD4i%SwHvp;#nl7->aGEK-%L6D88NHqMm!H%r)hw#MrApLf zl3r4^Op_NkI}_G%@@qO-pP5Ys^&nU|Iy2Y1x<6aphmwpQPl7-f6mTg^=tH?~K;ijJJle6r^8$E>|qx<5kAdnU+MJ&gKGx)?+4cfpvaY2$ZLS&+RBA zUDQAhO?mbvP@mhZHX*;+vWZC~R0U&kYg&>>G*M0wBvcX<6*Qv^pNl^ZW5~7~%)dtiQ6&56oq=;3Fle9#6 zpJ!DuKDJPCek6iZZ{`4p8tx1AmnrQ)l({lz%u-Y?nTLfy63jxFhG4{x3dcibd|gdi z(`UUzt4Y)x9R#aXjzpW^ZQ}YqD=SpBu|k0@UJ~cv`+Zu`)G#^PCZpL>4zy6#g2ld( zlP5VsnknxB(4?dUlBt4Vf?^H!JuV;|Vdi0!m4cGt-lfBrEc;Tsy*G@(#HMHju&wD+ zTh~Er+=F6c1>sD@QRjKSS*Rfis#QxuNS5CSQh>COio@zPV)zI*<#R&`{JjCgjI>%SxUWkO0&*Z>1dl_HkH! zQ<{e%VmH)mkV)zQB$>X_Erun~07$hHL#xuo?i|oo;Ka#LQLQK})n*iODt4*s8lQ}4 z4wqeQ8G@n$PUg$l$(R!(&O}=)fSN+eNet!PlTm7&DSxzMagu`Naa`MkJCV(+a5=T3 zR?z28(Nu&jNg97p+Epj?sDP2Q2^x%@J#n%x2$_=oVF8>o3cIyDJPd0o#>|mBf@b~z zR35Br%EIxvcz220btwgE)Ckg&MaPEmc?85nlYOrkJWd45s0I2FQZ;d?YUyhe8GJrH zEB+u@wG<0{c4}I>#?Nar4^g?LTD3G&zl0=`4DJ0w2p~bU1HLQO6e^&APPVJ+S-=z* z)xe`%#7*^>d5Ay6mrMTu5?P%iP+XEjht@9ro#qN%CRYyCTk`FdEv6Qg_f?dxc$WJN zp|mwCQ*R;8^_8CVwDML`e?S>>xbQf6ti_3B{{Rbveh*aL4aMzW!53-du3mGR$@Qyb zgfZcCkeSFQh?L6!xg-#Hl0ZBwa3hR6PeSTH5o=qEQuXANX$nV>~NjM*e=^) zN}PYveYSM!5}5Dh}D>a{?LA@$*EgS11%^BEhMCn0EmeM4UC=U`0x_R zcMbmuM zg7qc$y)+NEec_Yk38`Wq=eXu8wR|H&@g7J#QrS+DNHKJ*PNA!PBt&xO2OCZU7)f;z zUSJYlt~J%RwTxt3VmADydLWkNA>6 zH!AT-Fi32F+ASb+-c>_|4%FhQ=|?Q}asUDTaN@D+Y;GJ}>K=|AjfEW2w!%2Zp?~f5 zGN?&u*fNliRD}a6fjUl+(n#JhYXAqWf8r&BnvqH|0MqNzD`OS}nZC%V18X&$UXFQ| z(mb5WD&GG9z-7kkGg?+DxZ6IfTJnxOqRLfGP;>H@3!E`nmmXWvl{#wj_8fdusmD9U z+)HGAR_xl6J1yDrCMmxmg9V5VKPRbu+m^=h`H%FK>lf)Ww$mK}U-+v#3^4Wo0Q2rR z4j@oP4y<0J5(tdTvMSuJyYW5Q0LotBcUIRA@q4vu;o5BBmyq+^KT;Mmx~%Medo!1S(-(mfgY6PDhc@)Bi*5S-yl z;mjvY;ZOb;#Go!?pj?1}qo@FqUByR(j`pNu>}BQ0Bp`1kLIS*6M%;#*m>xj&m2*^< zVl^a|pabE3bOswLN>r+b)~~hCM!uAc^DUu6Xi7>`5pqBV>I4|%CKMtpARC-xmbJ{b zga$4fa%x-0u2UrS*)pfu$)<`jVIiCO{(4f@g3E0GYAeVkI{b zvr(I%r+JBZ%1I!zyXQDdxk_d=0K>z}hO`_Jsh)L$;uon6G zq(?DCle&#qvvT=sVOMz8996`P(YE<7076QDHX&D1j0q&_+IgST`i3&uehE{ixJgr+ zfv^It$ZBs+xE#wInFt4n0O6d0%!jYr=NxOs{79z?u=N!d1Nv!!(80 zfy=76rklJ~LK$H%t2vCT)+v z2I4A8Bu<-;OUy*+QvfQ*%4e3!^|H%*X@ARS_M&A9$^QTrxt1seN|)LW$97w^E10El zB;pZ+mD%ZJ6>tS9czg`?1tnmacv-|al(L>d{CMT+YL|p_BD}z%jL}S^zp8Gfgr(Z5 zCmUE>DpEq0)6XfLDJTthh+)^kdBG3j9}0!`5`4hPIDB?ovRP>$6)OOn&Hx1?6ice~ zA)FJle`aQ6aK4@}_;9JD=6#>-CKJ^vXjw96BP;D9R#FQQ$AF*e)R0h-CU8oKmoSLk zkSwl(jP9b-FeGoJo1SEVVg>I8WRH{mqd-jQz+YD!o`An2UyT9%UG_a+;3XS2`qWp% zuwogW^gSH6(%mpI3cbKEU=1J}jY@)f2)LcGsh2Ji3ETi&3R0SBTIm~-))D~>OtA|@usoVb=sNpM&>0)~a!`Z0f$KMd$nUDTdlgU_y^65)zRnKvaVOgDJ2!IM-&U-6rW|r~>2-_7!olaT)nDETUYe*;R>W z1Rm!Pa(mt`Zq%d!Dvu82s1*SOkdO=s0FfjDU;!r;%%Mg~tGG4kU%y*KzO_y6_|fmq zr0fJ@JVBX;DO^xelcfsZNFeE6lOOtlh?56!ZVpcYBaUp8Gl2DaF>^+6Y8baS8pS-v z5+z~y$pHI7RSW~_`+)5p8-6GF!8}9SyY!*yrO28(q{`q#6U7bKm75R)%Dg}#_?kEXOL3d-jhL4|~mGiXb{kg;;Q&e!-AmSCGYjqbqB9|?D zh>`eK%iiLSG|c#s#Oh_GCYqG25|lE60#Yac076871gP^i#9h0XjK40CgF#>_K{mHL zU8|woJ&!}_UUi*gbGgo`bEf`Z3>f~;3JoQVi~j%#VjY+zgnJvql>NmcKp;s@sZi%& zEg}xlxsYNp<{Z*z9KadI!0oQ5cCMy9(UV9Ck^_+7Fh21FIA?l;)q~a9vR6_{4X2KejyFP*WPmi0B!g+?2hJzR^aB_DYPtUa)PB*A z^?d#xtVGORYMrwYs0ci?+j2mXsl9+PVI1vY61AvN+!ml9xGcmtBt3-;P{l1W<&}|Z zf+{-rRrKzSEyS71TMGfi$OK+2tH=``!_Rw0H~gat>Xhj$E=ouNwL4U}a(m7BUuZKV z{@Peja%eb%oq!H(3EjE~^dInZTT#Ru9-4%yOK&L54Gm9=Q9^+ZvPw?DY>-4sNCpoq zcfQJ~96YMKHle{T4;5}7QM>44#eS5Olw;-+{{XaaS5MwGNJ;)J>CWM!=uhy`oK9O! zPrr5&o#%20_}qz&xlTHc5V8TQ3KM-z55wU|@^5G*@lGx^YQ3uaGuUiEFBxCsAA&~~ z_H?Z3nV{xt!X-JyGQgFqFL?>3&DszNQUV1nSq8~8s#FgV$5-@Mp8o)Lc7{e8Vt~SC zU$L5nD}|>mK;h=7lE6z*JW8+zJZt!=>?*rg=9X|;$Nx~mW;|$ zctIroT6Br3N=f0w$zsYt1IS>alc!WDfkIMup zUc<-X2v&IjK6oJt&VUb$FqquM{$~U-7*+?LUoVyjA*CfoWhc;+A6yYQ!>rh#^i%wV zDnLlodF(mI@_ti~WIpN0!xy!o&m zYVl;BKR-LL)&r=?An(_wEG!Moj|202a-CRrNsW z6F)G1k%Us2nTz-QqIpsb&Mq=i12ZLwCI0}K#T#v5s?!%re1?(ye9Yn^t`T#{JX*wM6LPHX$ov#~uUvcRM;ttls(i{3UiSxMdf)WMnE+g8}9%2Re z=e`{THAHDQA0IprLlt!xzTZKD2niw%`+;(KgCEtji;-wJcMmk1>jW^w+GEZ?Y$Kh| ziGmXe1Xc(ksy=%IARm3XT5c{TcEJ&>)K{pdkZ)Gn7-Mtiu$k)w^y{}sz6fa*FH7ir z29&mf5P~PmrqF$U9~?P8@bCE;Mj)NV`~Lu}Alyu!ncj9Y&iCBmuvG6_Z>#s#t-Mrn z7ZL|aCS&)x!3f(|qs4NO@ra1)ZOo2-ys$wP=gw>g@xMD?k--E#Cw^c?)-Zk%>UQP4 z;D;uTbQdF?0UY`0A9zlLnB}oNruG*yO!M4efM{F`jdtV?{{U`I21&WF76xzYFn07a zaWjG(9J>+#4#MyCb|mimygvj<6@mzO>o*;21&^Kj03*|u2q3dR;E%2dAgNT?@Abh1 z5^Wy8IQ~AkAcaf;BXO|ufIa!0@WPgL2Pf`-SRu`w@6tV|>4ZkT-&i5y`w`{^$>=Tj z(*l4s3`o0c=gpw=)*O-ogxhg{L-oT`?+*U}k%uSV9sd9$1xM*0e9k0Cx;*M)M0=yo zrWBYE0l13;5n*`gCOMml!-xgRW7nR~@$QPV(ow)AB}oT}E7gJlZWewXrJk3C2blIC z{r>oRN=3zYHq*n+Vx2)_`o6G4+=)8?2cG``PQF-rebL|ZFhhc-+~Q1azn$>tA>ff8 ze_e-Ijm_|cYqy&VovzwKfNw212;FfhH%;q)dY1;&rw-xl!r!2;m< zZ5-fjc=hw^%LoH`75#VkL0(-nZJo5zl&jnjP@&giBzcXn5bw~B7*`%?1dyS75jW-K z&i<1&J7FzL?Y@+t@hzylM>bMsfaHQ5i08P}Tgin`JB`NvargHF4v4W5c)hfWk9Z4_ zzfs~xNJOh(O#0dfndkQ>1W|_tO!;)bGjBMx_Pz)qW=I2=osQjnutI=MfjtBKqNH^n z91xKGCifN-zu(yyBFE*X{{X||`b7hzkPVDnjlBoQc|Lebk*>6}?NI*!1NMt@r6y1; zNhO?~LrzZTechoJtK5j?{w`0?2s_F8d;osuZNy6fTa!CmbMl{lyyT1kbECK)AYW|- z8{|AjQqATK%uI6WJkP(D9NtAfP}65|xVLj1{CPBQ?!aoz3cFOyL50!wS)QQ^AJQsw zJe`FZ2g7BQo<>eM2jV*rWp)H5h!~v53s)6VWiFMuNzAcUA&ChK;7XNZr6g<*h8?-)3$No`<;5#js^^IDLGe1GcIr&d z))wC%EiuYUv9qQ>iMGEGY}a`GO?GVxMh^&zuneJEJJgOvg==2;)fsA4We{wlNE-{8 z1D`Meo5-7E99QiIsMWP6&YdG>m=6JCt=s3)&MXKmNbYXWvo|Tf4y+`YD(xSO!hnec zWSknLNQgccA`bQ!oMtB@{{YG0z?a5jY5}hgGD)MnzJ1oe^IAH zPQ#bZLgkS{Kr{?!OT7!YAonvHM10pN%kx$XZ?=`CK!Qk4tJvNk$sED3IIAXn`2@_C zrYztJ9)+l0(RyYURP_GQE?UP*nll^geo@VQaoANodYrAwmT786rGmwXRme%w0U#ME z@Sai#k&VA8?M&<~CKEbJs#7$z!&tFu?Q%CH>Dj+zF!sk{~5QWEr$X8><*B z&$9ERCyr1=#5;wc1t+f%bnmgayT;9wcEU~~XR4paEk>aeN2^ewG)mZzZBs_Ov}CXG zGAMYjEw2k7m#jx{wQ3`yQRVlokV%rQylHrWW7|gzCAgZ7!5TzY}xnT4F z_y}IqV*OqtFU%z<1xpUH;gO*Al^i2V>0KT+{$)~{%(N-82vQQU;vf{1+GfPwb_UQu zfHA60!yG;+g5bSDp*mRAwQjxpZVMEgUL7kGD)&pdP^A*9shm?*a!GjST-%7Mcz-iX zrCLyjK`Kn2(x?+V7(CK;1oNG{FTtZ7At69LOED@T+RpbT&K|anH#W?tWz&6VQv&7P zgX^ZfI#uJYJ3r#a^K7n`qrFL%lXR%b>L3#*^%4Lw2qxgpm7Qg(!eS>&2`MDdC>q>> zR(cZQ=;o2SWI4FZFw&-lBsI?v*P-8=R*|sWts)752{9280-zb*z(j~VjxYoi9uPUw zqO=a|MO#{PjgU;}eh1HVS#uKILM22i!)=lXAd3(|@mOs$@WbSlB~QYj=Dx{bL)GeT z;Oy?jX{uM5-rn%P}b+qG{0PSN^^cSOnuFt89CQHO_Qn(VOs1(5Q8H)koR`#*mtaZjpei1Vg zStRoT$ZbmiGgR;Z8+oeOzRSmR#s9&OG=8kQiUWA z1uKap@dC1>0In7R-Z_hkRI1;^U}Omh$Z5z>{IXK7{NsvLCXl$4q%15B^~W3}%!CGj z`Stk6FPLPcv1|@BYNjS;3?v>j&tjrn0xIAKG>_1S_BqS-U6t+3peg<1k`PY|C@U(S zc!vs<2qDTyQ6whisnyhlEwI`FB`FDjWD6&jqBTT-NHR#U>GSbVcLY6#j^@^W{cdCK z?2?`ujV*5f0GaRS4Xe=L(oU$0Nz@Fe7)%i&Ajn9x6F6`jDsb&XW4jPuq#dtj6oupV zUXIqK?CtTiYi#!_OM${+LP!uGfMj@qox#*h!SM4K$l<4vY8N&(Al1HadWwjriI5hS zEF9c#xY*gqt6Nyp*{)aJI7vt)DoNGi5#rn(fdGpDX3@4b3@#|CEak(S)rl7++Pebf zz+-)56@z8lp|CyvB=oCA!zHR!GJ|DXfe>t^CdTR@ok_SH>`j0eZYo+7>>Oy%cNDi% z&@+)1FkDjA#i1G!X+jyh^!mofrma$tQ>WBit z*a}_7{N2U(Ee}l6COhlWp!ix@GYN#@=a5J925muAbTt9#aUM_PFBNbmWR}{>^B2*^97F{o zdp57lv#M0No>sJ{>3H3mI@WTc%0J|Lx~1>C#~BC@1|DrcRXZ@Hp2+rnd};{%h6M5c zU0ekpkrjRr82iI%)fw9H5u?qO02iAqZYXD68t zwmU9%seg&i$3Hh!#?_iQH;Ub6r;RG8>uQ|ua(aZix+=OXx{96JMXna5ii-6-y((Lj z%?)GVgnkISwWfED%hCx&)_~v{{Z}P%O?KOKwxulNgO~C zl;V;>U@OE@ogGRE_lObdJL^IlsFfW$2_ke8C@LCsg%G7ed_(k@BVrHF#CzS%@g9Gq zebY?C>e{c%ae8ia>v{+y!7d>e2Tqb>sqhUYULXMnRi#4EAj=i+9;aGg_1kCCBt^Ua zf+SJX$uOHpu@M@g2~n_wkv0+r<}n22y#WV^)tK3-sdwp0R^n-QJwBo`dUPQmN{V(e zQ?-Q7!X%58Y-bRJtXWAQwS{?I=mwEEUky`0YHPNjfEtAItVvtpunCi-g&*m;1QXH% zq~DPMB*=v_^mEavwP<`k#t^CNKs4pe;|3fdaJx;ei(#u^si$$13d}d{jaus0J1!2CJ66#h+js+7YMn z>PNFW)U1M{RxhdnB!YPh0AV^~^;&snadUIdP8wNP+oX+ToZN^^?GRF?Wohkq+s-lC-m z1%28A^t+*6;`|`z2Nl@<(q+qE5z9MEhQuK$P*RFB93EDXQjmCnSj*i6d%+7SeiOs_ z{r=g`&CJOCH+m}VM41Q}e$jd_#pE-1#|jaM?B;9QENUmIOG%UJ0)Mr{BmUIcYvw05W>spT@; zBqWBZ`k$w1M;Yk<05wk%oh!pEpecMw7;JtNgTw2A0EuMk87d~mJmLMGIe^Ol01`ci znpT;J#>&KHznK9X6F)VP&cdNSwJTBxQOuMjhAyRWOLpNJ)3KMQDx%*~m~7AK5PU^2 zqs+)BR>CIZA0)d>kMUE11URFH+-f!f?;o!2$4@HDWRmPn9)Ok_>r+;_w(-6zt3!0^ z0?QBt5ow!&vF8!U7`TjTGG$Elo76cKaN_jV?|!cvs-}RFONR_zv^CdWtjAc?8NEuL zsZdgsQdMshN`<5unDj6LPp&ozGWb(KOTLHC#KszI&MB4J`T&2WL({%5{5FxSD&)W$ zgFD~JP-A0Z9PdjOS;3(%eu}-LBZ36j@}=rSH!)_XT1MNZa5j{UfJlf+RWNn|QyleM z&z>-9B|!02>S_V70)eSZHAQKzQc{?gP!_qk)EjIn@g4fn5>BZG>WgPBQ$Gz4G91x~FwUE=1}x zFi28NK{FTSjlZ**a&Z{h64;PK1{&BF7A|~EUXP=8w=`B0EuS!ukdW$j0VyBEO~4Iu zT9@DFr?fsOb2QKRpEC}rZ)W+s^vtLoYExbBbP7o~3TdD$H(80Cdlx3l#vc!aki-l& z7i%*R2+-7lsAi#f`G?xf_~J2H#wbl%u^FyBVGyL3E0?K56U3X;IC%9l@ORnk*%t+6 z)%CRMZl^Fns#sbyk{p$SLAsz^Nfs8qIA^4czsc^3(NjA{9v+~f~R^87Mo^`zk#9$IGm=X;05HBDUnVTxW z0TNKICSV&;@vFUEi_mD=$`-ey5E$H>h85`gJ9kDcnAD^tQn;8hOcA&Q!I+66WE042 zG7=Ctijw3WRHL&n`1R^x#7rOn&PA^L-u$iyD8wr>M%fBicsl6{*iQR{u-lvqfox=C z%2LRs{oP;4{*jHHiAZ zw+T+A1GsF2w^4Ir(bG>yXZ*gVPNsb=jStXOh_psg~zlDgWewd z-IxgUul+nY@|us2Gmg&l2T}QMG0D|aLYE3sKi;x^ZZwn_3D`{p)xE`S6IhQSXY4YaETTYUo zq^;ZCwQbBXZ7#rbKG0y1kHRkvh?gp69&QsEg2f@yeR46Ng|O5!62!A?#ylnCo*R3F zILC=NUaMDsl4jYKToh#|xk|@$;$(h@T2pSMpVS;lm{MojnHFR-T$?V;lHcNRXyYk7 zK+Kl1f(aHOLxnbKa6$QLpYZsfv^7alJiLUPe6T_J zX}|FJ{*gr~o0BKb_xAbW9CM}ldF5VFH|0_M%J_Yvl?jj_NHhL=Z}q?r7UQnE@6(Y7 zT_;4eOK_0RYF+KmVO~{$sa&UF*J!u`LEi^TKQEj(oh|ULjNpUxft3*9Y~c{{RQ? zh>0YczK+^`A*!)Vn@OMA-bVx^2$E;d1Q3?%4ZySg)zbko>wTL*92oh`uD4Ce#Pu%n0o(Lhj$3yb|a6tmW1R*igV-w|q2od+* z4=I5gY$KmZ!2~x15m+FEKMy_t^*{>0pL1@wAqmq`3w#I6h-8DRc@I0p&ih>3nAq50 zi2dh#fc;tiq;~*0!+P#I^4N`pfj8Rm9(G!&?o2hV^xmAnH01V*1Fxvs_9o^*7P>S1Ip^nd?;`{dqQY)A_agl~yx^P2I3Y&0{dE-cv7v>N*O;*}yxzvfcN~t! zErJT}FKP?xw_U9c2d|hjC(V1f&* zbU!cW1Q0-A;DQk{n7sL(f5^GG+imd@sHYsdSdXJ`c7h1m-uJY8@Tk|j>jV)YJ8HMA zpQXQrG*1Zt0TKwGFlOR;{rTW- z64$Fxxa#$%!=+-TB1?d@3XgaPax^_h#5TY}gZ+r+I`TK^hw<@yB!Rixof_k@B8>J7K}{I-vLJ_vN3?tN}N$I}SIMjN#cD|f=V zy-!+>W90`1WA?wdo)<^?)5mC-7Z36v*Jw~k{k**JH=Wwg(gxa}PJ+PqL&AxH%VRf_ z%ieyc*9Xq6=Rt3T73rYW{zu|y@4Or=$Q%7k?IhtDt=u#n!Jp$>Lc@VUk=6x&*5XhRPM!@xVT@C_FeB-+})SHXt+nVy8TcS+1hFK z0cotxGRKIK;=ga|AataR>U)dzk6CNA(n4ipXPGq6t3#xFLGZM8J z8;}Hp(tNS<`xL$`MDoll@EGLB;1|Cr~(9x(!DY)tlcI0Cl%c&$p>eNUAWEi|n z#iT*L;B`30u5SW{qmX?$claHP?50RuwyT?JZ}B?qqelDS{vkp;G2*@vTYy#Nd`*|} zT{|YuIhN*BP2?G)s04zfu~V021;Cjj^BT^PDjS|xC}tTZCO{*L4~t5Ym-fnWKfF)? zxk?!EFl5+~rzX#5e{%N22aR8|BOjKW0)oAih2gPbQjke+Z9@|=Nl(0>l@J*u#ioj^ z5EH6Q2;Zy&zn3#^XKZu4&kQ9knTe=+b0MAD)*9BJlhpD|#7;^=NJ&!!lAs;6P57@{ zwe$qu(@=aTRr!e=rfoKmMdWhpjd@bQH@#`sZ9Y-CNyKr7a7M00=|}P3pdg4h=C%H_ zs2G9-fdD`{ zu5bXurA?b9MRJ!g9mBYN7y?_BesIZ>CCwZQmVz`idiOB7*Q|5j8g?Z?RHfJD0Mgc` zR)w)Cxzwm38$zN85i&8f<$b2a;n@=uXGwP{nppF2qi1H`{dGLsvRFJ=_0kfYJ|p?1 zvo%`JyB>hJcO3r!pFPBUI?$rd67r2BiFS^z3W8l_X;W%FJZTenKh-y#=P$(IF?0QX z9Vkr81iDI&^nNXA@YXrbbA-%rSXtOCVVOG#b3K%)m2sdgpsB99M^W~3#jJ5R4bjvk zdX!m(CS1vcUpxzvS)wr_npgoun6YI4ET77;T-9&Lv6F^#l+&3t|J5b9M{~)_2V622bnU5h;NJK)ue!Bm+0cC7CaQR;w1FcBY!syi|4W zGN$rDKY;5aM?f90n-uz7Bh>ORjDey3*TY->w7h#mM&PDhQC8zXPqt6lymP8WlmB) z;@Z@komx$k00lv`cwnSS*jtu1+#LS^T7&IW0^pjE!Q45m05<{OR*erb!YWezVnf-$ z8*5K2G!{P%X0?iSG^ksepIB%RnO23uRRKm6KoXP?bsq@|*zdiB$o~MpJ8RhdwNjE% zkL^ZKiK<9&oE9G)hMkQ{>Fgy~SyN@L$OF}^6YQ*^&&qVh{MW6 z`N8FnSdhRJq)Y{B2~k2l;8cYYrF#GpfJiKPQjaueHvCHGPBQaOH${+8H!PvH(v>GM z%n}l>%e?!2=xMfs&v^;(v;jg8rF`%FUE>BLHOpnBF&Hoi{vcGTgrtjt_;h9Lz#B^(l>rDt2&}+Q`EE%X`X9mUM+f;-)6{yIvNr47T+QVsAkrD|O!lr3K zGzIHJs|{<@mYl`=e5~5H%baV>=PeG0G@wX{CifyHJm454N3JrmC-CYj;^R*mZ?Sr6 zG1KIUi2yht=;Zl4=|R>oP|BHjYfKQY>YZPxh~;4eXrDkMEGBUv2YR`$CT|z;*Gycw zYYPZ!00Yzp15?i67JYSWwu#tn5@3Ky0_T~P9U{P70TUy{NVOS>d+S5gdH6jTE6l`vsMR@?XS6#_TRAsPT`^;DgZA;);^ZqLSn_ zRGP5>gX!HY3s6h5aTKPUKu4&LxKzu6Wm*)g zBrZuLM=4INk_t%!djLEPlzUXyO}F_++|*GgXQq^47Z%n302nZWM2QiojYFQ`g##z3 z=xttN2@S^8{vV6jvvjTAf+mz%01^SE&mq(06WCC?q?sd8Atav)WXhl!3YhBv#A1;J zC5w%n?B?yUduwQwCd4@N@ew^W0U#3zx=yQ|jNBBE0)Uu<7r$49Jpd%VnxlI)2Iub5 z`oy#O@!R9dESa`$+YK#R(WyGlqof3+5>>vE5B~s&7#rH=9}Kyw1cJiA0@eO*e*kPw zl;=+p(B(lX-9!qW@#N`v=MFpLG^D)SP7qCr3JDj2qpBnugpib{oJHq$$}*(X%t_P& zL1(e46xrxO=H@x?Xmj9!>gAB^U)lqsgLcq0N!~farlsuCt`}P#awZsn*u5qn9pZ+;rz8mvR8tqi$NuD z`s~t>D?zAlD{;RYNhK|=v4;YZ@=I&fx{7e59V%JU8zo^VKDYcAW~v$2ZE{r-q|L%* zd6EEfqy61FeiY0|1jN@7r(a)vPxHA{RMXA36zCpJ*w?rV{vn4 z6rx(Cc9$i?Vz4q4wT=Z$n~LnfOqnG?a~7tS6tDr*{{W0u7gN~x5UHAGT0?oxS5->u zl_4Oq5%~&`hFYmDhf7LGZlAk#B}zl3DN0yzKD_jof*9V#XNX8qS~BU}Y2iFi`Bh=i zpc{y`HPyveE*;4c@+0t@ov$MF_vpJCzg339<9mJC999s@nv*Eyk~54BCp4BBTp&!! z!IlyU2Y{%$6T~^ov1*X4l7)JLiDfn;^q2TvAv;))@smT5#wcl09r$!C z>{_?1e#$!yH;*a8R?ZDwj}c8;#4Fe?ee%u3V{zvd$_*^)AcUz&2^JOrs!f0qs>BgD z0~)>>SSq0U>c@J$xrpB8&Gu@i(dPSl0IbeK+}b_AGD3VMPs~$o# zNgMzHPzkG0o?BRIZAv@hf`ZfuBlul90kNqKeXmgL+3yahAqgY`O^7BZ{{V44Hzq_8 zdmCYKqlhZ{9%Y=J-)>EY!s1oIHV@fNg~9?I~Sw;sW^<&3+yS7eWDXYf6Qyc$=R5T$DX}2_Kg{L zXR|4i=P5i`Q!D~NTBZvi15&T}S>LUEwd^aidXCh)L(3?hZEMN$XHi+0Ii!?0;c+T0 z69pv1B!fE~cb?Q|a`G&qYS5sZRx!kZ!~&4)0Z=TUEmk^PUm5y0?2O*qcDoE`p+_vr z2dth$ntl?iE<8Eux^N3?VPxfy;IRvt=NNIuE#6;7m9;6}qr`=pYo z0_F!h3&E2c-+qv7jaVTns*4iz+edrpWq4LL?^*jWmZh}@?o0hNdWOB4yEJK}8Br1X z$y8s&qDJDz@B|oyo$+E$8AaUj6QFV$=FfI_17SxmR%R$l9J$du&~ENIi&iy{5b<>_ zA{|q&A+SL(RdNJcfZOprKs_&W;y;727#Pr9Kpvuv}Kuic@5R0yc&{_mja1UByI@?IRH#vWNp#1PRCDLq^fMO!VO7D z01(EdfbQgI^*mFxo|G`yF*;nipKKkTKC@-eYUYQkiK0iq)I|)gs$~EmG!`w(IQw z0FHAW&$z#ac+V+yP1o?|b48iApy?`YhZEnhNHUE_HNL}M|CQFN66B#LD;M6^+ zO?3ytr1cT@n=O@&%W+aBjwIqJpv2clU~Sf|T3hQo@b|`?H2QVwuS1{ zwo*bX6w-Utm_8j>xf{)e@!j(I=)|T}$z?79a^izm0oXa$Qq%%G9ome#W>t!xg`4dr zNqKuIt-wtSG51~W;f#8hmOh%ynW9djhSE;v+EdK)ncP}-IN2nqBm;27QI5VIZ5+oM zjwF|NNgd9@qiSl^0z9SsVq~bwyI2{MEr!PuWO+_t>XMQYr>WasR`R6StcxIb6? zfkfW^pY&6WT>j0g47Kt8M;(fi6T}HjnTZ&bD25~=WYv&`pJhrMAeB-+SAH$>Iayz% zoE~W*1qZabZW4UyfGL+YOvPa)4k`sIT)YI3w2x^iQh10-0P?tUG=O5!>Gj->v(Ftt zhu@!8ze`7#ox88g=kuVv6gmuf+Gpcv_lzSD>|bD9_dCGj%@2j)Lhd;ZeERu%VQYKx z6#1X8$U?$uGdqr+Q}=uX4)e|Xx^kxBj1V%r+{xeAbn~CwoD?1V{a*V*KcA;xlNV?i zRQ34B%JFM&^T5ZDYj)3&p+7zP`9S`{{Gt;qryB&@j+b z;kLA`=>t;1u%BPAt-Ua{^X8gXy&;NNG!NJ3)}0_^@6^e`5#A9-1@1Zef0)4p2&kWl zyl=4fu;srzBY|=Ud=T3KCsa&AKqs$0Lu2YT#A=#9gcHmMZ;kx5VtG%^urcHU8*PpYfmp0q`G9Ih*#R9H}NT`ZAcx-l4H#6 z{m(!z(-44Edh6%q1xa#y&!2#XEa|xC*OVT6a-P3jIQtf>-T8EGaD}wDI2mw(Fh2e8 z1*n#%$L|C+O4a(529h?k^NZTnz2NSXXmIv}k<;M%?)=&aq7pzFcG~xjlh2m&ck72i z0$QfSZhm+mgi@wCe}A!p2ppdxe6MNvZ99>1f(ROvOmc!xo#TH<0CTnQwBhFBoPa-g zA+z`UVIYdZ1R*}V6Y%}|V1y;S3Gjx`hHvY)tiX+My3byM{{S2-t8?ODspekVm$~>>9RQF3 z>QAWKdG{WC@Q|Paz-sPqXXr!o+(HsefK2TdCM3uZPeC*7Y&kyg@A())YHPTtuD?$G z1tGinnY7-0p#0C48wnLWkH`)D_Q40AJ|-x{o0HS8E&iU{V1+$j_227y#S2^;@;CTx zxwibb-vlS0KdexWe<--$<1y64^uY(4dFAo%c|B&(XKrw9xViR=Z-N|8S^#NBF2$*( zhRqc>=O)JYF>lfhjCz0si-f5L%iV%K&ZGB8}))5j(*?18!uk(tPn%i0P7PcYxK7Q_tf+KQTBVFLndY^lqZJ=^abF17mM<)13Xr2x(DvPP70GPR&>P=?&Equ4kF$ zyx7=GjzIFnOH^D5ci0IF;`1kX_EensWk_aw0p90G(1alL17)Ql=mo0Njgj z8jXO$09wfeM8Bzm>XcHm0ig-d_o1gn&9ejCc_a*xB` zYkXf)G}Gtx*+&cIUKmTM%PE}9vqZM3@fO!rMu#+uFTkOx3dfPZ6nWgEgvoMAMW;x^ z<1zE9IFhNWKCM#&@gXWoQc_Dl5MstYx_@P_W3kfnUfuTJ5`>&4YmdotOsJ$EQ7;da z;XgH*ge?Bdl;W`{ME(*zTCmDYz?Dl}cwxq}z9P(3&+{ug5=!y~Wc015oKSWt-xN!1n8jcBPqCo1z*WptUaJ$*%z7mF9UcXNhuDhpMQXbuZH*q^7oo z-~=hRTQZPmL&$BwH6%t`vl+s&{BlI`{vL4l!S-3J?Gnv*dQjB3F|2mWA(x$H>P)f* zEh+rQ7GneI=6@+r6jgx~P~{e080`xz&D#|eRUh%KWeaK51k_PHgAP(upqWF6GLTgP zX*!c)LU%0x0JGy2AmVVzl%$hYAcB`WsV*)YRmF8;d8cMOS33cVIc_=8K0)>qHPEF{ zkjYyT3$>^mM%<*%>FN*(Pz3Hmq!g6Kp?H-6rVaUtH$0;imQ0l}g&}XTN|JfFXQjE+ zioADhw>FhE2Zc#f*aZR{KDDo#Gu*#0*}4`r9%PyGR>t6S20Csrj8u`Kev~@Y3-~l= z<>Cndr9cH)$OY+UxV4`3A&fXwsDDsU1WX=xl1+q25Dw;M#}RtA1&?!nrYy_EUy0Ah z0*R>A`U2vj zo5wJ4HAP=%d~H+1RYhYh%@(?xy=x={)HVruk+q#=A!tp2EjVDQXB|9Dm7iql3=Sqs zl-R7--AE2jrkAZuaLiT`e`PyFj6z@XIELa+x{w32(WnSu^*1Iu7l@cfXO_4?3kyM3 zk+CY7K^u}mRC5HFJI;wGj|+(af)vaF1R9nO7*>VpxiQn0Q;or;nMGdeL4G5l_DOv# z^lDhgDR6db?*{gwa@ST_rOe$>VMbB|@}d$2zgi~7HEC3DexQyVdCUa1%AIM*b_6GcZSn_d!kXO?-Syi?-Btqv9XaOpEK=* z64sTyTJA*{P=GE8clzlJQl5a&U(d(Cq+xmfTGdJqhfz>b8bAbJoL_Etn6Z;M$zf)$ zQuP%TeKqPE;?)fpoLq!W4pP7tVhCb7>f*Z+2|C8Er=a|ndqqyII$cd_%1Hrbmgybv zr68XYol_|VYiKkHbt;y$pIQDH@`Pr+u*7GxK~#>-V5d!!?D0!D{LWy(;elIlg(nQW z+Im4Gs%)ug0m*NapW2JtW7%&)m=3|xLU=_@_{?8Kwj1Y%R!zRzJNX1K-5lHqe^ zjNKidkuVMuE=1YWl%$D79jU_|t8n&q?sZ9)=8RNkJQ$p_D&m?4SW^hftLpM<%+-av zQuQ@E72lestZkH3A#S%zSeLu%OtnP0kD*vB-wBEsj3PwobF%Y0ifV#^SE(RONd>}H zDT$hrK|r~n5&7W9c4Ifn)0AeDh652RwYYpF>?Rc|lv9J1h?hQOoL3MDRNOu#GX%L4 z!aZEsp47CYER^#PWPDuSciSf)XX>bOp_eM@DCu2!-u2SkEFoG;rMAkG-@5jgLuml_ zX%Gbb-uy;l7n$eNRHOdPvn+qiC{S7bplQQ!-COjX{hfIdJ2B{!Cn3Pi6^P9pU&k*L zrOLxj3Sq{KMjsC*Y?*`Dn<;2qvQVVQdu92+d5{9ZF)6&=BcR-K+C+hSE;&{g(Q$eJKQiFn%dgM$-ORVt1AS_fKn8slWta$b^t;|4+tcKy+bWF zDFD|-Z$1~?#!)K*#24p&kcy}mAE=T%Qb#V6XdztyB~b$m^(ox=C;s1*4tG8#EtNqE z2Tg|HPNHO_Zv+T31cM@W+{%%{!8$u@_k?gRS}?rAfh$_fX_Em^FS&>&>I8F8BVv&t zoP@c>Iv(Ip(6jRmOT1883QL03`0Pfs*W}S2oGqdfWP%Th4Tzb%Kp+qQ01R5#nDCI! zJVYd=6sgZDmJC^gjftkv^oynrDJ9nwuc+ObOr-X2v+kTYm@qge_9lt-z@$}wx(2Oh zy<_uJ=_PX}`a$itt5zj)SK6HAX*1?X7*zqDW(sjFL`^6NNKjW1PYq2YdG>0jkK(6; z68X$^QTAP!)iUN~!x?V~C-U{$LPk@PuDRoID% zSFTW2J1IZ4nJQzH@r71R$Ym11&>Ly#{{Za%l!W?UD@t&{VKOY>kIP8JNl8oPq0*V+ zhze3t;ILWYCyAfZa``Pdln}4@)dpcw65!k^I%JSSghGz79J~;Zcz67x?h+=GDerDq zZfC!n#nFdh2?7*Ect|o*ohH|bl^Nbho_L9A)tKt^aTBFr+>Wn**ULolQc|5-l96M* z&B;k1ivs{ft+@kumnkZ#0a{VtU5&o*{{UIS1z)2Ww^!!vvWu)3NSRf@gE0_w4oLz4 zlj`{xV5K-o+Qi@G-J|H>#WxZXhob=W@_G#)acvpKd}fx?^JsN33Irumztmwj1Px=M z<=2p~f>c#Hl#oWN2__%_U=w%;mN5C2G^LR4+YwgOmuj&iZ5-6>nI>47hE&F8 z9@B$jL1G1tk4-m@k>OmtG8;Mtf|N>8_E~ za`{P=iUy@B*CLE*V(wkFwRisj0M(-};z}U2A%B`}G|ee!OQ~&Wsiv$CmenFzMKatz z1t}paDIsG1jQk_!zmW8^FARx`NX@%7pXFuA24wjYB<4s`S&IE;A1t3NH5Ka_H;@e?I4JIS()-6&kcGDZh5liD2jF2qU8CX|I7Vr1YXrhW!ns|OR+0#dK*lE!kf#RL@gJcEGuUaTgwzIc zr72tKw;`c%83UDtDO8lkng0N~CekJcS%5(zLFq{n09){%nBJ86H3wI;siTCjV@DxR z;YS`OfZnDsE+Wp-(+??1{V7o(Nh&2Wu6qD=M3blh0XWUzaGYuZo!zN>o@J?{miS>a z402@3D{$aFOEXrrr4L?Fx@-gC~^vCTw9g7fKg{wqNdm>P_lpj0D74h1ZhbG zm^k!TqrA3W85oRW(@+WGA(`HV>1qoS``^Ugob&m($(U@aN`lpj#sk2WI*2s{p+3<~ z1KyR6ko+QcYS#;Jn>my0MRsSVrPq{!twBk$O5;=#Ng&7($|o9Eq(0CbZZTMNDLfJ% zgoOeNhNBiNUD&f$?;VHGk7H%zSu!Wgl#k5LPv(}Q2N$h+r0d3R*Vt(t^2@E$vV$lA zDW4ZDiIQLm2G)ThFOD@Z8H%wa*@aI^R{ZyP^lYX2pjaATMg^!?q>hw~YdPY&N;cM| zK9p(R-~nPS1N8y{2lRq=IJbt&r_Ngh0!udvNhS58y&|4BHp5|Ll`3gQ;H`)|kVtC* z)a=pAouhXBljJSB*sp878q$Qe=BHLffRv>&vS!2tn3yC?W29s~g~j5DQsoefag-<| zv$*0Q500j`F~j>|>0c|#hpCr{h`5e2QlXi_tG|bp3Ryt zs+%H7al){)y+wf)7N{NN+BHjW*Vj~cwKWeu;Z~xj4x=Q=01^|ifo-jGje$y&DLfLT z%0LH@xaQBx>?2Y^#NuWZ%$qW3+IJ8qYLREXbZSyF?3{OHG6+`UEU32N%v{OyGA9^p zQuw8UO9vjwBb{D_m^HMGpDmOV$&R6&+#kEKv~A3x-wH<}43W4#5P9@B^BY)3p^Q94 zFwoFf_|MWhPFr#R05SgnQH;5D1XG1-k_;G)f>tMSeStURAnl5nWfV^1u9m4ZW4`@k zDUMI9!q&jol@0diTag~g{{W?L;syE)?cU7udhJXl_Lr9-=OiVk7IdK@R|T^nF<_Ml ziv!pDEsZJyS+NurCz~A@79sZ3N0j|55<;q7A8;B$VM#RvR)aunGQWtKffG_4Dmk% zR1hW1lQCpQ@Xl>U5+n=2t#L;70Y~(VdMnWRCk>p-aLG6zqVxq4rmaA|hzS&-sC^^m z{{Z7nGUVo6h{bS{qlL_+qy)N*(VOHFRD^);q^B^PW)JshMAa!FDO7Oovo}2?S|ao3 z>3+EFG2&h__1saY+Vg#@1qvI3{k^{JJ#a*h4UdKSY1j1x!-|PJk?2kc8+Yy`d>$06 z?YB}7@%{cI1Qj}l+#CJAJwXo*lQTCq`A5?P7W03-2Udm^0leH>ubA?SV1z0H?@>+p zj!j4zQ8VTQ^7a?o`N0vD68yk@JB4lcf$9ftTg|P^->2>7FhgOwzyOnRZQx&iW_IK` zY%PI_uy{hGq&kR5vyuRPs|+SdGd(Oty${oWsX{=x1E~NwR8V!IXNShOJdGhP_BN8Nl03-vkp}ZsrlCE7~ z55K;H5t3AG&1%wdA zE;fVnj1WU5Klc322q0>;k2$gauSnfH=C)Nhmwi^xwh1!IQJG+mitw9Xf^|_nx{(g9OverCX>3+L% zhWzUOd|oSgJ6~wGKc&7nY$rcHL{xlwdj9_a&ym3dRi~VD+vRdFK?`tVFTUR51X?dK z>B|Hyn|u94AS0N!KE9*R*vp%;bwXZz~n$CXVanOdtU@C^cxD+@4v;G945jx z7UW4A3lX;CV{KkUg6zVhYoF=kuSW#~KXdncZ)_1LP$j%M!>~i5ut5$^`x9|#i{I(j zmIxq#?atioaWZ~aiH}R*f)p9`1pRIC7$Ab6^6R&k_BbGdV1gYW023p5f%PElITJjG zJ262200=s}Q){*L`R@ch3f1T4IsX8ExV9A}mjqOKtPt{cw$nXNrQ+Xr!YwaReR;tT zL5Z<5{{UzwZn6gRZ`*7D6uW|7YTJ8n4?$<4FR*)mSS}wD#YFiTvW>{N>R|187ZNav zNLSJjfP1a|HH4FC01`QD0JIJ79K4O~he}fb4KIBvesPVS{;;@jcX3V-DSf>fou@4@Iko21Ux8#9(?*=etj@Qf`|>z zU8{X|hlNP<_rIZw`P$s@M1X^{>3@N58dF$xuN`{rloAvwq^^6cV zGtJ2_2CzVZg1zszA-uFkW} z(HT}-8A9h_aAr)o)0dkjS^*;p5h`kx zhza#d2`;phl@9d$Q1ERp0d_T#_HmN9mva6U%biYRRN6`tnP-)tIh$9g2-IDxDnmNS zT4|8VOaz3Fn|_pY+*fJwnVxe!U+oh;T!fQZcwga`+5joZh-M-`NZ*BC#N~dC@?Ot& zR{H%)>+xG{fxe%?tDOzD^BoIbAS7meQW?$rStuYFxv_a5o zXF7)?lT>kkNIjF8F<;(!VKT8m22c!Lj=GDQGaB@|c7NNg9&DDVs#L7IG-^{(m%iZE zr2-0-PNGx!mcohsFzaC|){vA1t2LW;XAg}><}kn#?d*GRs{)sBOV#^n#<4pHq#GCJ2qu?e1qs!(sSX5h7<|rJ&@88BWj_`6xfCD!uLc>Aj{?N=1_YbQ z)D6zlB2BvDvZQfSgoEIBv8{PVQ5cjisAMNnmK_{3o?CaoU(7<#Ql(o+HZlp7au+0% zGido?JyGZX07?7Bl+H=1APw#H`u9gTcC(f$rIcp0FpBKC#MQ9iOdV{V@Fv1^l8+ie zNKh&YkfV*ivzW=4JWCV%5=XK8M1i0+-$S9ZzKPo`wq9X@SV~%ShGo{IgKl=@3svJY z;I1TZh;lkQfdst33qqHuhX4p$L>Q9Wn%i>V_(`yUZRhv|%AA?<#|l)%D`rX4*;S2s zkzqpSHQvYOs>ERzhMcCKhBPWHr+#dA@*_v~y^uP^LmB-N6q?MkrMlMCfMrW%-t;8F z3L4N9gE8UMlVKRkc26DZ=cY`>0V@&jOH%?a!-R?!sc}*^A~3#?a)XP)D-|FCiI9vCmYv|BFD^8N0w9?nNHPY;tm9SfCSGvZ`KT7q z=xR^0_X4hUeHgaS*&NK`^4ucSp!+>V4)-@CpM!|lf0|aID%1%A1>z5i05`Fc2sY`B zQ`PX1N9g7{1guF=4nVt-Z(DNHTUL=Fn!=Qjp{S0V0tNOuKv5^$KoepV6==wV?y!g7f|E}i8Jb#+eTNz^c7D$x0Ng;+ z9HUwseA21fAt`dC7EjU7!`Ek%*^ft(b6i}m3~y+st2CNc0u*p45^$JwC`>6S3Be^8 zlqdx{vGWyTNc`6S0BBF7OlPI<#HaoZ$gz^~884&hL$UJGrHPW{cBoIq99B6hmBtmv zpC=R9>9UtHS)O}4AS5kw(1r7NV_Al2!##&*oNt`PCUx4Lahh>LyVq$u*4xeM0%%Ju zHqz3_URY_3B5NH1p5e-NXxT~~TKi54YVdhBGZ!^v;xR`VI&8%P85qPB%v#AIk`M~i zuXlyiNdbqIJ4b?*nfCtxEXig94ZjGd>*?loQZO9(k07J53O9u@?@q- zQC}LRB3zKA`&h4GIkU9SW?7pD(cdu8Bt5M3vsX%1RX7{B1QZH_5}DF(jFvc z`Gz`j$x_vfQs^FvHCE&Ut*R>@p|3!hQhOiS9G57PI3-}PSS$pju^ck+I1q_!4wOnk zv(k@F{^4FO{KHM9ZxG-DJV8dqB_ny#t>r`joz3SQJ2A^b6jT8M?8?_rdDF%{ZIfpS zo|D9+)NTP|zRzd9*gFW^R5_r1uu9TN3Dt9Q2C39<$|IPaqWa9~2aDZ3?%pG(hsSlp z;}WZshB}Z1Ue0`Udl;Nv1lYkQ-Xwu}w&LPN_mXV~6%3LtKrhB6JJro4bd+;S$d-~;>re{s-vJ#{Nt6XqGGb8C8yj) zE6-89Tl$HROP&7!hH_U%C?oP%5M5HeKx`UOSzvnS;tf+p26qwpqV4u04t*f@r7D?% z6Mw#Q$%HCg^x{etj>mAT*M8t3TuI?m5J^ZJMAu+;C~1HFrgDVPRAto^Z1!WDEwtqP zQ<-sYTTbGuGGs4KWt+0aaH@GrQ?AlD*-bhfWiTEuPYnD*srwpYZsAjg+ULobTp{He8RtO=Swm=5sTZ?AMktj-X_YBRiC_)LQnbL0xPHZWvf zL}8qO#GRPh!^=Y^V*(iIclSXBn1_A!ZciFpi+JrUrN-EF1*cGuL4srwPelQ|%mc_{ zW6LszSM39h$R){g)?@OkZrW~b+Ame?AxQx2u+|0L`jM*|(S&o(X`EGNFSu6X)vaWR zJ8CePg9#TMWXF~_j6NDngxa7678Q0HHMj0)=D57Vb}3VMot1CXLiDhMWjSk$S_#xi zGD1X~+AKHNl6grRnWv4$B`Nh_g$+xJl6%|6>^ViOzb{nDf?)>EC~C*-9frN`R~fMVu)n{HA2hy29lewjN7iV7%`ip{ee5OM%1&9E7! zPlX92fJnvWczzHGb<5!QqsV{S0ol*|SNKBo-Igj+6s$jFBxj(I2!6|F8DDAVni9!Y zA1aw`E8v>d9LEL5St%u!_ZTsxe;h7ip= zte2jnVfPCwLtuuULY`;^qv~|&ij|sYSbjQVu;OrK0%l4C;LOCDh001Lr`rye2BOjV zX3KEM%zG2qOvy4z=XQIv7`(AaOu2*d>zRs^ogp*P!WANZAvvxhbfiq+GZGasDO|J_ z#s%3vZYkf)X&X{fhHL6jl(w}k2un}6+LGCD>J7>v1yBl72w4hASItjMxiYYQv(Avs z>v1yi@~P~v^XQk3_CJ_DG=rcZy+I@N@%T9HY_8IJ59|&pGnX}Z9DyB|H7Zx;`yq*+ z*;&&Fej)4d*(xWXkPOAG3QO^b-Fb~{f^{84LJ|ld2?17;00IdA0JtUuUmSi`9m^Us zJqy*yrOjyOjBG%UUrxmnDTQiW0OYtdxzd8R)J3k6pq0T;xk(Wu9WbR_Ow3O~=X^;% zSO^3P*!8_Nd+A+FGYv87?kiB)rMjtsAlr2!2VJ#H7_pt8q^TTAsz~lWMfCj( zC*r+Q3o5`>)1z5|2bl!5Xz`icC#=5Twe8Ou#S1!jPV$JJmbWQ>%@>XXRnP7_o4Fj)H~XDY@q}x>KwAAx@2hz zQ+?tbCF3I2QP8fdM&}*nYED7f&=c(9C$L#GAdp2t0E$`k(^&EL6$%!&3 z_C$M9)NJWcbSjN)iH}`)HL6U5A)%&8LK|U$2rx8+B_Igd7qpTueqiIy8LF3z#m<<- zyDg08D^b!20twVi5R{-5bV(qbWinpM<8uYdP?SPJB%TEX>R6T_UZZ2X#v`m%3B(*o^+g^)9;c^qNSGAWve+dn1=+h?1;QQ#C+TQB7S%@4XFdBh0<3^p%|Fvu9!Q)viVfi!5;o$}fHv z;xASq%ZqQOX!1C~qkQk+_f{8OlrVFV;?aUF57O~>J6Cxs|Xq!0IU z&$Ju<5X1%*akW~JqG9_zjLa5i;wH|WI9*ecsxe~3jWwpM9y2;_2zf1q7Kn(6NHILg ziH*ecxs$dfS=+2J49`Lfj%M90pkqhaK2ewU_lm{ z=lWz#$4QbPMk&q5rDRPbkS{0~yw z%W0~-;+}=w3ougSsnQG(45%OpGjcJ}dpR_wWQw>f?r7`-mcJDVb`~V+I4`6OGA7`0 z!D%WXJVArFatetI;Jr_}#ofq{x&F*EVO7JKHF8%XqJo*pDv8wERtDl^s$*`pF^W9v9VQ*!)txk&4A7ii>fkUKFH|^dSmU5*!5~|j?HOtd2q-*J#>rWU>N@@H z&_+7a6dj-RKd+r*#-tH&M^H!33=EIk)DQ9PiD4j`?k;&xQ+xY=j1c}aN|vMZFIY`o zc?)@-Q#<{}2u>~Z6y|hyzm|n65IM(D@#c3hPSei>DNtdnZaj52EKQ$SWd8s-A&}Cb zIqmn%i5tvMu^1qM5>HQ99J$_qk1R2l{KL<55YnASAP@w>7TgJ#F&E`GorWA$!&>)V z6iG9QNJvu-CRe+5yBgH6+2LY$m?F?d#>PQOzVRCfJM99!ySUw{YHOvt;^c`W0=0%G z-5P+^CElmR4zSXsi97Q#e{=UI4Jt^pcQjclNblLaOazlR=gjTP(s!`u9I+b`Ye&@6 z&q#>9%h%%pD)Ix`dJc2Q4w!5tQjDm?pJUT~wzl|sl3bRj^nwO81p;OWH;#bEE~X=N zo6gvf04y7?JN&-TLx~*79XW0HC(PkshO6|SzIY&j;DQ=Wgb@eM{?UIU)20X^gzs~? z+V-8d*n5kd5JO3uo$x^kf(S#`$lI^q?}8B0x&U9{`oe00Kh3)Hw$Z%pYrzdF9nQz4 zeikEeVTw2OJiM>GMW@V*;lKlO0QmXm3{i7k2^70O*!bU-L~T0`cf21s*w|cdNf>M) z4euJ1<5HZj{$7?sv z#o?rbumf(hZey3P^MVqadHrIH^Z4&R^9TES;DQzmSQwx1M*jdJeuopd+<1s}h3FJ) z*8+#3`a*K&xRJ>3{C^u_2y7?c-`~>&5TZH3->v!$hsW%YdGmOyi5DMy_WEFg93n>d z*!j;n5PDmsz?%@DIRF$v>%JOuXRn^W7+##Vo$WGY@3rh=_cn|W?Tdj>{Y#L0f!-)v zZlGMmkv&G|rr^cM!3YajJx9UqV_iHd@bT|!U+(_6B2jC*Q1uYFA!4vW4zzS9-_GLp z*bjLzhbP`0{{SNdIVRrrpSyuFdK@okp5>@7(ax zqY!x+lkb8WR7hi5yZFNjqhNOFG1h;{{zn^#03+z39Khwj4DRVscj(b3?rfu4;8_H?2e3?XQG}(@M-nmgy%7`F1 z-4!GB$Y1J8YFyKDBZhi$&L?5AyuT^Ktc2qgHxY^s#%>=7N+rtxRZnm=y=ea4{1tXm zR%_aP{{XcZ=@Lxrs{@9>WiKNh)mjug(D zbtI)!?>bckl{w&96Bi&PkVoo?wdckiqDLMQa(D>_)GcFFwb|a44WO)jrSxSnQgb{a z+=7xgj;V0RiS1jymyeU4+*I`zb(s*`r6R7NctHRJtMw>|+^PsZ&_+1Cq#$BFSpaSe z17gLmaNP~#+yQgWUm|R@sZ8XQg%&NT0I;}n;_qP`<3+^%EC*XE`iB<1i2x{Ca^u$tYD?)b0lJ2Z!Bn33S5i3wfPzlKqSW9;r=LT(Areu ze?)$uWKF>ZL;nD6po?xXsN?dZEO7@Qn}n9}ZdP$&QY#zYQJyAcJz@br!^Q!6kw+$% zrq#S;S>+`bf0wx1jiJqHB!vJ>?nIM#HkdrXxW*1qn}wKGf>Le;UBfvQ3{IiG0EpUV zCQd&Mz_f)kvwK%0XfI<*YiQzr(z_yYhuq9^p7Dy7)?Gu50^5mDB}r0LjXFqxQa}K} z>5h$%cAZlH0AVMa+3TKLOB_)OR0@Vfksc6^SKiX!0 z!xbo@;vW4?p+U4}04S^gpat^h$43Qs);SL8*b zmq^9t{hOb{vo&b3ETTQ3R#8oQ6Qk;5r}$sA#%r2s#;Ssl>Ot=tT!jt6)RIJ*Ct(Bt zO{X09Htgc$tM_4xJvHy3qrD#b2}k_*&zj-7$djN~v-EkeW^vjSM`33~$F zOxH7&l>V*t5}7HKkV>LS0xT>!`C^H}iZVz$5FF6d27M1JwWB(W#Y#X0KvtlgYT7-& zOGXdGwcnYgb+wS9;#6a#CL{F==~qpzQzI8Jl1gUDSrq|5fd0+h>8Um*=8P#7arnbJ zT-1`_k7t`#m}k<~14h=Jmgk@Cyg`|8`>QBvZkp;u5wmVABB2e>ge5NMQ~*rAtCJ@i zwk|-$<&zXks|#??%E8z*xv%M1CP1rlJd)HIBIKqLv$0?`< zI<+)p8{DC1Q3g3_6Uq-IPBiQa%#2*Aj>Iq{T}gePPVSJ(gB3k13?2T95u%IZOfM+fJTq*w&sXp;boO?EpU0}=_G!T29N(9GXI z8Aer@=7lB2&`~`54W%nusvbfKVw@o`txr0nG~SVAIGaW@{g%kp+CI@_*%}WDvn+NG z2Bnwb{H)Q%h@7PwddVpE=NrmN;#&qML-e)Vd0vgZ8g}2eIDlp+vpEJoIm09Y#XA{- z!KW33!%VBTCmAwc6-Z}atd@`oQbF^Z@dut&W}Gv_-J0c2)ePUXULm8(+*(x2Z{`_z z%)d9AW)*HYrdxNvt#!JR0^6ffYSgv9y#&*JMa0WS8wY{Pu=3F&Le+%B;Uyq~aQS?+ zAQ?$#ECk6C)>sfQ>vrK(ep-7iGZmO-{i*G?dopGsc1kflp~hx2(ds7WJ*3SsxvbyL z&oej@P^t1~W>USeEAttIg#{A1Z;V zhEB=iJ74K5v+J2H7`bTAF*$~4xqz?M;&NcMVx}o(i~PnfFOk7Dnr9LitjtL+;EaLn zuOy-ub9bYCvz0{RLzhTC)7N z3Ia$Y-UTJa=t}~S^)5E<+gToZ?Hh<8QK0QGom*WuARUZz8`?$-8-z^6ju1#10Q>FH zfUwY1#^aXbreT&G@Yhm72YoiFIy0U6BZc^_FQExjgeU~X_wdES%!HC|OxzH61Zf!j z0hdtqJjf)S7+gJ=0xCIl{IfC>kV1$Z?WHerEpCkQv>42Tjet}XqtEn`K}a_y#K`c1 zdXA?9Hb6@_t!dKwzP|>%Z4z-@UhxEO;q~tEJ>UqIYi>@d8_6a{{da9b@rMh=SWkW+W*7nbarJ18YrwrR^surbR_T%n;{&nQ$da^L5ox(y&yQSR6j;pQHT1DRt`L%b!m& zbVyD~m5)<~^}mzC129aR#l)0?uw(^s1t!V`E)DA7er!8iHxW0s9j2cqb~0py<8gq> zoi0LUq%SZ`zz!`*-~h|^(gK{zB}r!nHuqxntIR(WLaAA!8L?-ws->EiGB+|7S_{r{ zR?@{SGl^j@QX~a9)lEGuB{5UZw$&>^r31o$5mSP@F^879{=9#C;M4$+sV<+%V}})m zEat|$$F%r)vkf5Ui0VJRWkeh)XAv&T@BjRMBm3yYQq0{hS`nHCzF$MYklsO+Im`IIoJG0mhhGfw~Gr z00@u@q?ytHjR_oLYoo%IS7X4eL zPD+U}xjre?Vr)`wAWq;|8|?CvCRDdAqg8`D5n_7Kn_4;cF;j6W0n|YhZB0oo*8q@4 z*Q^$YE>nt1iU0(H02cye2(gnFlW2+N8HvYmC4)T;LpH3%!8T&;TT$Z13oR>G?Hxl( z6LG0#9((piNtS@59U5TlK^p_4kVl3_;F4`IGmUQC{(`OGys3tdjw*-Wj zDItp&1W>h0TrsHHHm+F6M|aRfc%&Ff39zv+E(o`l=K{4YPV_sFU&i%om({BqejFt% zz*Id$Rf{!1EJ$V)cf4k_u8;&PVIl|sMWb;B1X#g0<%%-_r8oLT^zCK>n)By(J^bM_ z9Kbb9w^ao!1ing+l*M%pWGo~C zr%d(n1{LB2Js*hsYtLHXZ^U0>JZSotOO|nO4d?xpW|e4jFkYBTHQ6hP{ONHcO>3vS z46R{csrOlJ<#)9oTk|=Jk(gze-XzRUCl0I(!j&L|reO+48jyXSIh;sG6BR53B$5J0 z%#m&N#mEu(;6OyhohLDMKCz&N)s#=8U;XFZW z!?<@e{{U;_-XqJ4sZbRkn`k9!D>U!xOKDPETAc|>Qr@(vLLO4u)}=J4@;~A;v(n|} zI1JiIZ_MEiTQO<&k})_!R+)-YTmY##f~265%KV(9f=WS;v|ri3+ofVVBib&;VsQ)g znMP`y?<`Eiz^Qpga84Y}NSLCvNlNnJiJv(82*u+kAw??%BYC-)rPKqclK>EQx~AH% z2|L8h82|x~DJoi!rmZT~EmA`$wWtUqOs{(@^bksV#Gy%0 zDg+UIj{gAD2E_B2k$3|T6qJ&Mq>y=8_1qp;7aZ<=1=?D_Tq)D6mw8tSZ>o zvW~_CpUmS6AlKEYlBxg;g#cJFyly?B^q8be{GLq8Q-1NmP4zB7(#&plG0MwN67i*Z zs?2I?s+?Mh)Ztc?f~X;;$Au}9VgO0zCQdg$Ux3Rc`n)<&ir5efn^WPq4p_%DmxIkS z+)uB>O`klI{?RonBcPH+SZpdU8nh|M0r8J;GI`6?I|br zYDgOb3W#DsHLipbbZ{ugpk&>KWcta{lA;}pQJmB}Ykp2CbG>rj&#CCyNK*-rHqrqx zxfYSJu@(jfHEG$|n7XZd)moFK^zqiwz2Q4EDpEgkh9h8gy@;!Qxm_LMW*b73pc0g% zLI?>du>kBx{iK@#CSwrG%w|kx54lU90jGKHi>k?_p`9uO>>GU;eOS=ek&eBSGTelu zT$qqeq`}w;=kYU(sm0AuEL?_q9d_m$W{MNC*$EH8?MeXk@pis|hl_fs5QP{*L~KUa z_8%SN*Afg$)b|{n$HYcBV3MD8C%GW*Hi_&}Y)FW<-eYTA%*VWJViiqN?i)a8!@a6` zYZfKRt1;*WUyx&MO>NbZt%8=siwV35DbxTe*k1ejdE$IJh*_Z?HDcnt!;5G)F;5pO z*dT)3tT}`7*y*G~?BP#0+TR1X)}Z2elPfxfh>E0hfMk+3C(hXE9iAYnK)G;wQ3K}V zxW@|Z)iAiy1CPmY(!S6n@aoEDrPz(*?z6C*!Bsp9!wsoQQoq}1(z}}^AQ?MAkO0`9 zk?lEaDpa6o%$rt}E__JaTgQbvM3XNSGzyaD$5I%z#Q<@kjw}3ERJyL&^Y-16DRSJI zrn&^Vlrs8KIZB;%W&}>62cN8c8=(p8o(QM-=VH)b_*|iZH7@oCUysP|#O@|>^tGEy zn3bwt4<*Z$B}qvrQs(9J{aRC^jMnx7R0_Z$B99#C(#nznHWrIdIpyUs%hMb5qEp1$ zwfQ`G?8_5C3M6ovGivTD&>iWs!9tU|NduhPFKgTp<$sC9iJAzgBbzWk7K+nk@UoB; zQeKSY9W8L*Dw@NBh~+!{Q0@Tzq0(d-j#d@Uo~^`3+0{NgS%u=xGqc+SJ)_kfjJ;DBv=Pg${#uS_+F zxFfn56yl_aGq^Vgco#hW;}2N{n1g?$5ET&y{$Fw+h@V&^d^I4rElQsc*ULc<)J3dq z&hzK%>$QQ~6G08Z1Py$>{{UQ?dVvHWUY&mZ}(|?zt0p;g^()4I}P##w@PhIvOF=!{(1UxB*wR)RBCKh{*9u%kx z@9B8^iHmGcED++Pg6{9l*i-C2Fz=QKcf+qZtPn!Y=EDBN>w*YyJ6u>=d3wm_=qvx7*0iF&Bb9#jrsJN~Xtut_UIcAc}!JC&EtmgML7r`3|^*t<+d3v0%aev1U*SmJ(7+hfr(rRgW+W7uo`+hSSz% zC!Be2Z!moDoJrr{e$e$T?Wix`_4q_r9Rftn`COF=>F>OG;oaa3$pCo61CqcJPbWWk zg0&SN5h()TSU^5{Z?tVYdCP>vfku5?yZmBlQh3U^$wM)dO*DePjOsPCV9~ z93$FY2~vC3^Q?-gKoAPsc8k1ggh1lc~QC8*$6T>v}i!sb4x_8M}fP4$Ic z<%3gO7B0=vsB7~H+H*N}bD8D?!s1r@x|<;~7NaS*lu0va1kN~bXtJC}3Td4rsCQL| zF>8X)sQw^d+?|N+78^3cdWqZ)!i7l&!mUF>Lk(Kgc;sFy?CZbp9-%$Uz?~%of?`PU z#mwG)XUi9X&Zc4J33Gua&1a>H(xt7Y*Nu-h?A{X^fin)Wm-kxcx;OxUX+zYSMxr&m zN5Yi_73{p_J7zoff|MXA0HPvfjpK79av0guKLMFb;YpV){{SkKfc0ZSU(Ep(jUV!v zwn;OIK+Ye`lqovcVY3d; z&fp7{go>~Nrq#B)J$cXw*|J{LOG>yWOj1QCv%*a|nu9|?S@ff9nwMxZnrTX#C141V zaz~0J+Ub@ue$P z8Uv2|HKWW@>!@Wqm8{BBAQPzcurdVPA3=SKD$gfj)xep9Ku-_{o4F4Ay#ujx8oqVe ztTtkRu^}7S3OgHVwYRSmM>ac`=iCch-ct9jP}LZ&q)6G}5x*gbV~9B(u1_B$b-XJy=KmHbFjRO4FGfsVR>b`igLBaPY2 z*lA5ZHH{%?DgneO0jVlZqM)%Rck6s>dA?E*LeL75-v0mz;5VT*(3|ZWXJ>Q&09=j_ zSfnoe3M{8L_DKq>iwX){2;9{9hkDphT#_v^KqVt?gmVL#KKgv@FRN1kdb3|j+Sa{^ zt-6jI8|x8L!A<~hl3Bx0){pgzHfs9R%F2{r0#4SHkRW(}SD5L_aFS4-D>j|tBc(!An|KL92*ud zzJiuc7IafCp}WDTSur|6BuRm5Dr`0)d1C%(4B2w2yV2Q+?!QadXu#x0)KA7TgB)2P zro}spIK5~K);6s@bfE`Ph=3s5V5r*H5zw20Hr1CT15;b^I~J`?>vyGCF$`mWk%&AF zD&s?9D8!8@`^C+=&7neMSdvL39f>;_l5cPf3{{yVf?X!}NFZxM2=u0;6LsZbX9e9i*lL7I@wJaq-`uEwGX2C4r_zYr>8T)&NJj;aGVmv!W5*; znZ)HNPY^6>sea{1QFC9;2c&+%XMT>dX?=mrvayxs*}h_NINUxBb0=XjxGc3O7<@PP zguvdqVp$TgX;=4)OHZl8l9Ytcs&U>w?Kz5Og50HIr4_4Mm8mFbDaPr76cnVTu!kH; zY7$yoK?ZaPu_SfJm(YeIqb8ghk zlujCQ(}T;&mZ2##F{N3iZ;X>Qyxv5WQ{{Z3UDYC{{ zZ2%M?0U#wLfC0CbkeHD>#7;bJF_R`u^@Op#X{TCsyOQ;fu?ovmFAu@$(nINOdAG;D zj~ed2jp|qRBmOiKHUpW|0Rmv$?=la^V8dd9(g6XOk~TE3Hw2qC9`rG)<1ov^%%D6i zQ(J}|J~m?)+>1VXjnoGTgsefZ5y;v?l0Y|&q!469TxJej#E`)51$QnlvAd8(3V{YL zW%-ocGQvqEBE%PF*7j{a8bJUjfU==tOikoT+?c*F z>Xg?7Jwf{Y?Hd9o@i}zWnc1{7W)HX`-SA;l6f}ntVM&l7jaEgM|cba^JAwsi0wi(DEFx;gQ!eMHU!1B!i2KmakJKt3$*=2fT-O0CzbT zC94^PsIagOo3j8aTATinf$_~|jFT~0AQMLT21qGNSzO5kM!>4#HzWcvkji9}ijn|# zAs{3YN|i*Wfyp%qNCc4VDHtx$O%igfY|bDE!WVJ_aHZD!aEDRZEK5h696a+)@McQ; zdA?gx?k^I;t5Hc#u=P7jeP2xQ1BU^E zk|s|=QlIv7hFGSbwkYXBs#kh`Z8N#PyEe_SNk$O2GN7@S zpjvOw{{V=kO*t+KROL@t{N@?IF)~co;lX0~ihWDD4mhh$B)d3&dH(=s^t2(wQ z@KS*6-YF{lw5jq+RIKMJQc$rx4iyxI6sF}VDijgkwH08nq^U_iq!lX3gE6SYfx7omZ^$z=BBNGf%$x*(u}wT9ZCd}R7}i)&;bO<9dkdO$7-)rKO*~Vir7jyFm zqtkbcwi6Bl!5Z?RYq+%oxYP!rV;Ri_IBX3<1VT(fk+3pA5Rz;*kMT5=O-ngC4>S>!3Me4a z#|W*Mg7u{-&8yNb0;NC=CLk&jWP$*hlO}oty}YpzL0KiJbT0dUNVDq;yN4yNA4cVU z+33+8wxBym@(J**4z{G5PPC08(Kh5Zl6uV8Qd2Tg*n%z@h;z=xsasMhV-@S<&Xz+x zjr>?#lY41#Up9Xde09Q{QIXMbzI|O^m+?mk=M;IKPhP`qBC3|REU2nQGO4OR+NvB) zGTNwHt~AY4E!3_TQWE17uOsOnXFVun+23R`LzpKK{pEwg;br3HNC4s&i$t+0Q^e3r zrEyOJYW-k52pmZuD46``_Sf)*+s{Y7nmcEW?6jKhKI1yf(=%EZSyy7{O2Kg|biAc3 zm8hvpaFuD2hR=bplBKD9lhR&sn0AUpqdSC1!{Xjtx_hCA5C*km`$FKp(e0nZFJN*{ zMfjXltO8XtHOwWH=DD1}c&3GM<3BAXSpb|koVBGWKqyLyNM&2Ms53)RPTHg>6c5x% zNF)PzB;3r#;DSs+$DF6(xKz#XCiU;9iESh376%+=pfSF>7dlj){(V~T)qTXR1bGlI zaS;)*vE&ap2Nf5Gd(e@4Qk_PgUNbUrp(J|36n7=tSKpwk5bi4FUTzZFg4AS@{XoP{ zqkSqC3FXYj_+l_gDxfEjbGh1#EY@vn4qjwJO15m^@O=(ut)15<@(zE>LT&WY6>qs{h+>@=# z)wSqjf;&m#ANTGcMIU{xl7W5W=42`3B?&kFgm|D2)Zhk6XB~ei!hTB#89%o!IJ%pP z7aKD;G3KoM9IV?nQkF@>Vd?{t3An9A?t7xvt|iSXDASZ>#dP$VLKYh(D|%onQ3UBp zu_;WIb2kGGRuV+Dne%v5ZYUTAtcgB z=qxqiE*;+C_n@xFmukv;Er!CSl_?1S0GW`+^(xR-q`NsTYa2zjGJLEC!9#@t$!$86 zpluR>U=kz{MD^)~QJ*zl(ztBgUd!BTPyu$cix#NaQ7;RJPYk(k2B{oHsi?4^dvl1D zHO{#yqoY!6PPNF@Cgu!op1-~)l`cwOiDdV?+na9@&Q#tX*~kgqJPH+K{{RRC&;mRO z1ht9!EXJg%#|nZ*r6dT05+Hx=+mW@P12$#PlmPW<06iFnchkS)80naK%Pydx*D3^y zwFv+=)jdnbF;zU8=-8S?PPwUFlqO=?gXrI?*KZ z8hG8g#knk^f^2bgz$ynqhk zFErGW2ZwLfXzSM7gKdw#AXMW)Le*f*FI{UF5rs&w03KW~;q@BCOwOGMX;%WoPTau< zqzR4h7`K61mZmBWTZcyuTnh&+-aZ(7)hI-=RKtZXbT=B3%>PUgk-6tJ)vlwnQNgxuCNa6v-4GB6~9}vUxxhYr*a&kjJODDl9{n`BS0MIt*6Q&ao8|NfgA?a zy$gUFVNvv}2=Qib`)QNr9Ss{tVs~)n``VGBzZzUMQI&R^mT?G5lS9OL23b%ZBmOm2 zWovUUsB9>BLYZYOx}h=>@>HR@KArSam?IIAVsJ&cu$Y512T3TFwDEB$RHX!@4l)2G zObfJpxcpvra&}$Ye4Dh11vosz2Zcr;6s#dfC!hHx0r-?}b6C8+# z3Sz`Sl2x>bzxns_>Ger0#m~F0z8|c4M1qhMK}cr=h72m;nh-AbiD+=>0B(6eAl`N} z_HBh@z|3D}t@eindo8a=)!@ZNLQnmqfI7vxm?kVE>n}+uPgJoQ+vf>3P@0fQAb?0C zR~92s?04r=6)67z*Uyy58%3;7DcUajMT&=eBx|{Y z@~Hl0d_K^!cb_~qEkKmG*p8&w;x(>LJvk4px%qYT#Eb=lK@9H)pUaoN2qC)9LG%7_K?t%Wo?qJp5G5OkB5p6Yocub^ z#{>{BzrVjs5Jg~uD+CaP+kYt$Z?kVNE$~BBXlmLxt9iFrX(sS(2h-Td=We5wutP~M zSiK49OM*NB($L$<8l;)F)6`#>n~&+4-f%-I=EVHpZ!R{WW}ic77apsXQV*#-+RG4=i8Rpg^uIIecH?o`MSpQe7Rcj{{Y-NVX#2~A}?{wa}q@Q7(S!O;Ye>g!39d^cpoXk zK@Y(MP>V!F&7=HaVX%T6AtFSQI*|r<>9pJpt-csi)fW~TS-SqPLZNYDY<|Cg3-vf^ zRMa;4euvHAf`M@)$34G~zg#tpFQR*XUh&@u<#J(8xIIudO)_^;&}^Ld5%&6 z!iE|T53@vbB&J{hxMEqGR)3K2ijIJuwy^ol{{U<}DMsoY;19f0og~XC;o?v(^mZYx z+r25Fgd_k-`{sUAiDCoi&qyQ<09=!K0Fkjgy?`9%M%aVaf~NsQTH1N9!Y=xs*aWw> zeZTWHwzso(dcy+BHEKyWfD#I}nVVYvxOpK=#0K7%+WSIE3X#RelCCyyRydo5-QAns z4ZUecG^Nev!;mw6zP&cYZViYrY>{`MyH}(}QqHNc3IreI?kp{6iNi=`K7=u9)N8jd z7>KC3YCSrSkHR6kzY!fs0y)~&iNEpgtRIWbfcZa+Hui|CQ3^6lNsXrcE;hKlUQMrv z%AFu;ZZ>ZryW-j<8FP>Fr`;v1#29j*Dh0@>2F4+}?vN*$*jjm?hc1)T42g*(xsKfY z3USfMyF=nG5?ip2EaCcu(W6)z^GuCnFbDwnip4Uk+kiD|@SCV4a<5ETVa4`y4UR$& z9$Se-)gBUBRxjBrCq#h6sD|OL<~?WlJ?%0wk4PB)&P68c8KE*gVrmPqaMTqpXhja9n_50(`$Xaj+^asUaYbdd zRILsq+yIiTWCJ!J8>UU+LgN~)YnI8zN+DvbSQ;C)x)vJ?M{exSdyC6f3RX&MP;Sg_ zXnbAwu*WF)u9~?C3Q|BMl0--(bG66)yroeSY;v6Avx}23Oh{0Amr3wqC|DXlfN1m& zXS--ID3dO431)XtEc7k|w#Mc44GCXEm*%KaT@5Kjf7+cY3b>mML<^rjaxtrVyjBit zf}yhp4d+HCH*tu;$>b<{3On4^HCA!iMHN$p{JHliUL>e3l@tWtOr(t@ zCOj!AA}xGnu=)NY4xd(DXx?VpfK^K{*}2~jitNq@6s{zdNZ-|iQ`{JLHG7)Q&RNI& zDzOeVpHQHkD@LYj(1h4aN+qS3wxqb3Bn=XiHfLrzMmz-1lBqKQ{o+9arlIH_yOBf8 z#T+hEn&oqNu-LE9%hXO+sh<3nT2OKZ`@Ywzem6tu6#0V=(mfYNLrQI7LuB~CnVFQB zw&xTkWr%(}GSe{Cgt8WC{{Sf`l{IQcdk>vV;S%u@mn;wm?E;cCVeK#s&u6ami!OPW zWx2ynBzH~+H3WeNNrFTUTgU(z5q@Fv7m30bJV{9eIY|SkaMa}e#KW-?uz2*%mO@nF zhYEpKmSearL(6#QoYy4GxKA<}hFVq|3zr#5ggBItOrkf|nT!fzyXGA9^J zOPPEn$qX%U<fI^L9w_L{VmAyOx6N!*YP$vsTPtvJ9= zmOvaJ>ioW^t?73nHn|gM{?CYQeplA@1}-S83MnN@fMnhYN=USWCw<7Uh@4j_AuKh0 zz#YU;Ir+uJ&IK;^ElQE*;O+~CtUcF~`)X-Yq__)e20Te4NlBaBZ($^z#Na_tB}n4p z6=LG9%~tQxgXsx1NKz&tL=j*Bcdnb7IQl$s?`S=YXDnv5xt9i3J3_{0=XqU3f)z_p zxKT>ore3C8S`xC9DQH@f6r_mMtzCDr-jQ+nRDmgYmB<$b3n^<5UX`idwL8a|JtcYz z?Ee64)Ul4rusG}_(Zh(t<7La95}LS^z?DuRj)jN}1Z>{KyDq8W!&94QPO7Umu25UA zx&oB|At_DAgbg4iBGI5)^Q?BmwNtbC<#VRYm>jtWh#S?xs|Rv+aATnQFZ64db{jAJ zt_uZ+!pTTsDpFFO5G=?@yJ*1Jv&TtQH?2AXPLLr0-T*)Y+}O-S&s*rpSZ5>ytCqPs z6KyM{4VZQ_D~XU=a|DoWje<}ULcO3Y z>>L-UBB7|qzQ8z*LS&`LQ&bHNU#86A!&62Ev6tjI>1k;NMIsaWNFsUa1I}lZa=hak zjlu*Qv}7!Y;oOZU<;pPBWI#BA+vpaA&9XKTXh<}%pJTiM$$*0t{$8@7D}th zyh;5$VXDMAdy{j2v_x*W7hWWZB|b70a1^vcN{ALRY=A)|Tt+8cuVbqeunf%ShQ8AZ z9!^HF0%wCZceIksOxQq{8t7+nXSbzSl1-m{25}w-TA$F);Lcg{{VM$g?MnX zuFpOdif?>RnQ(QtD}HR{B&`^nVP5 zs{?~itHlBm%KrcrCYp!#oLrE??BYK;xYeZQxrD(p2uaLJ!v6pvJmNS2OOZIGG8Ts6 z24@p7SF~e4{4%{vp4s@R6s_cFWuDLSrqrGu&hsuJ%ZrCFXH$x;&1$NvMRX_s065uo zEhWTjY!bD&=bXO~CP~Y3!6jHz{xGn^CSnL|8GmMFX?FNM{)2u3Q8adswKFGvWhyyD zw-~wdW}4#gd@N+W6i~LikggZ3dJ9ohRMo(MkeLHgjIZ>;Iuu2K*C5Z4$CD8++W-)! zE&~l7{M$=AN7Y5mX>L#3O}BCJh?Vasu>`tO$lLP zTU_xtbMS~~D3nyOG_5(EwrIkto7ANOQ~;TXzSK3dei zT(By^=-fx1nm(gmad@O`NHCyGN>yV74a|)<00@}|@e_#Dt^rMTA)3O}>eV#nBJ_z| zHD)$QVPjYLotQp`GJ17@kbLGWpHpyTNf(*qAQ2eJwc={p)!H(W?5^)T`gAdKZlt6D zM#pZEWgcU5{{VOCg&vrT)vIyi`bD%%azdCN<_7FJ7JbOjkiw=4s@DK1v=IV2+j2-G zo^T1Y=^z|D%S{-P`#0JC1CzojXh{HjMeGzGeF5%nc8g_AX$V|S)dCLi*vASB+c+~tQx@4YyNW_J}l+-xJUa!k!greqZv2?>&{3C;>rGvbIIB>}bUbz4Y?B0&I) zlZqdc3aJ2cC7#*=PVnz>8IRAXe-gE$r~cjVNVQkzqq|hB0|W?Byn+Dn6aN5dB!WDS zEXcv6Dxu5JTv4v!?nOCsia4ywVo>~JOAvJ*7XXi|v~$O49AugbDwhk1kcA{9fEB7s zX-G`&rH}w7b|4#(x#U@5rp%N`R|-L>3^q5=X!QqD1af}Yc7V!}IU#AdVZ@#yY`{B^ zEWj5ffFxIwXx*vJt1Okg{Y#~42oj|&WT{(pk~u&l$YXwJraR|k`x}UuWlWzW5Z=)!ASQyZR;U_{?9JL7 zH^C^&DVj?(?X;8xfa}pRV3L%DK)50|-Ah`}AXUo3*B@)Qa?7%$;blr8 z2$#atQzQ#9Ahmoey`wWZW?aArFs~~Bf70&~$^-%kN)n+U-*8AXzbYwSDrhAVK&F6@ z#1ks#Pe z<#H{1lCq3s!FhqJ?4eB5015{bI&Lf@7m4j2T`AsTQp5O`kj?`*Q3tzSE+Z1HJ=hQm}yQYb!sbjYOF-1w7yYNS(kuv0&vGnMmSLCw2G7}IZ{{Sfp z6y%$>fL~5~EQ~s4$>Ug3NYOxtAXeer@)R^^ko7w>t%4Mp)2?JdfNVAvnHMoPI7BIH zRUDSl&br>4^n!SmAqR-C)yNJ0oi68YQLOl=3rybiNz^4l=fGc)j;DW-GYfuhP9wqJmdpk0k5QT;X-P~;B zM>VLX?hJh>{vWu&Wox6SdBlQ*G?gVvB0MCJOp|$=%xNcmxvoXg6Dp6<1TC9r{IzS}Dx z-Z%cm_L*Is&M>poO0am6aJczT??_z=h!)}Zkg5j&6LB7V^xNppU!;!5V{+u8T)f{a zRGiBo#FlktI50wR^N1W@mCO=9DT`0>OIqg=#4bFGsA=jtw`nSyQ*{iure0-eQ*N~4 z6cV=9(tuc62?<&jok37Hwmn}abgV>)c!?z|MAV6r<|L(Pn1w$A(6o{il{d1ak_}kL z$h>X}c3qfG%Q1*toNgL-sKnuBAyV-&CnZTjHx4A5Mh+-y?>(-1wMlozIDZg%Rh<@@>!KY$PN$=gi=O6a#sffgYw{ zl6if|=JcV(TptgZf)m&eKnKPJ@hJe@1J(#lL!5ykY)4(K>*az7V#qNDcjxJX2oWHE z9Q{2{ED%B!fxWFGm_D6HO}(%|11kRjCwYN;T1O+(-bf+1Ad0~RRtO;p`Wc)0Pp?jR zA*9?=#{Ixscl$$T$7$c7={v~>mm_VkLmq|Nu2@=>b6s3BT!6*lIn9J^)1>o*=6_9r zoDiM>X|d7kyK71Z=F?y!pIG1MM)5Zs@Ip1OUPqO;8*A`{lW3cM(tcBd6VIO*piaY| z-_Q8yFhS?fo{*Ta2cHO2Cv!giexJ8E#dspUrrf%FbM^PZ1S}47u!*0Qji4BxKTtx` z;qiX`Izm9b`Tqbun|fe`ijmW=_V{05FhS?f#uJgWj$`BN%5MfRLV!&GBGbSwF%k2Kd23~EvzTZ9f%WtvU&3Ikpl!Te?zruTT}GhG&v`IqhlugfImsS`CoDz zlHDi?qtbx3)_d~W9XntAee?Fg4|`NsB&3Ul3yxg5?rweul;RQqu?M6O zUi8iM@_;0teWT&)hw-V|gY3oraQ-u}`2FE~!{zWJ_rh_jVc>r7ah-?9?+e~}eE$Hp z7uIt4{*b=2m&f#i0-lL6ex!Ou$F}nZaUfgSPti~Hg!NLtkuEt`?*mG?5^wrJJvRb< ztT7q^4hUnhrJejCP+EogKPoW$Lp3X}{{XCikLL#p{{WlfVUBk`CIq)M3mKkVyten= zV+sQnNd@lpX6W0(L}fatCVZmDc!BiV=io5#{a^r+z&SJkT%BCK^;gr6AND_BbdQuA zFj7IfYvc%tfe47SbazRN9L*R=cS=huAgwgg-67qgg1+y4zTb2Ia{mR}IeYEw?0P<~ z$5kR^0xEwZrz_C*3jiiI^&8+y8-zb&?1`L`?Bue*sk2$Z_)A|frH>A)Qx6n}k(hD^2$@melrfB~NIm#d3!N5)X? ze2YUSHg0&s)fF{Lxf5J76m4dlBT-QSDKXr}XPpBWMo0GMMwZ=TZ!{oNiE@yZieI1v z9|R}BzU2rw=|Sx8mIoQ1vhH`r>WfButKgtlz5zuwJXBARe_%k(e&fG51r_iY5z zo7cbAw)fv^O;FF;UmstX5q4F+ii3=k&sS-OyW$L_knO%je1r!?nlZSnpB;R^_ik#| z7zipS!&b=2aI!FPmAUfX1w3p`BL#3J=z6wAqmIYf7gp`vzazb_aru5 z3am{I-;PjV_YHR3ZnN|>xEy8qU?W&ci@h%orsV2Qy<|6OI}PT5am$Kkqs=L?uQ5?7Kl(z$K>j4$0kEkbQQ~W;x@Ge;7IHT z=wV|MD?nxVP;1a?loWc7pHV&Tct!IgK-)71wI|0rA03m=^eSJV?_dMKERLn5C@vA{ zZEcyhe;CgRF`4qQ{VpVMd*L2YXFZT(8|WHD5Sm@dWs5GW7J?`xhNil^9*Fmk%G@rHufg!8$2b^XIk9knN(ZP}kL?Is~Q+$ttFR|B`F zvHJ}1)$tfI74K~Z|BGJ{aHTlSaAE9l7G7-Nk}(zC@b}n3t$gk|%oJC(vwYO|p~Y`g_?yRvw5V&M3doR^6jJCCJb zDtw2F5V54ire&BoIL$j?2?@{<%6c;PUMM@cT7a;QUQ&8JL5ly$;|!)LZwZws_TFDf zlsaKpY|h&bl(&C_nyY(J{G1`jqnxt35)C>wz>|H-YleA8-wYKalNPCXCC7WT4~g_T za3_|grDzwG9rlFn8+fPacvYgTRkb`Ko2it`>vz8tR~KsV!gZrfU(}NpAu%^LxI~&2 zW71Pl=7$Iv81#pcVv&S?KEjLr$(sApIcBTVBU?=Q*BH)O46VvXUqVvsF_wNlH*@VVp^%a5J1~EMl z9k61Hg78a42`ti=l4%xi+|)2lYtw?*e8m~0MPQokVxc#<@sIpIF*~DW%;G943#8cg z6Yxfak&+6`_dUpI#{}FlEg5Xa($>vi`<4?34hAzjPECG+lKRdRDb-qEqby;B*u!wC zC5#H#L(>Q6P0eA(b69%Zd5CTyo$N02M7AaZ5wjbdPGO0Z(~yQHE0TGQ7;E<7Z!ooR zzt~tOnBU3dGqvL?Vrc!KftNB(mQw4~wSW>!UGLRDfOv_tgs0m2u+P#jd3gKMk2&n(6Tu)dsGLZd|l#oe(go4OHDhOYdSw zz^&esq7I&RSUo9D%H^Nn!s!#TbTK2Ve-kt3>YtRNn-dLT))(q%L}3=i=s6*SNKQ=n zno26Oymb2H2cez7UuCs@ZU%Dj)F%s5k%*brU|EBV!Eo#@gYI%cf`~5-$4jZWQ7VB5 zHv(`@(sD0#IqRF9x4Q5ziDIlP*ndRgPy8ybSaK?8e-jaJ3J9AFqx8(#fa~EaKJ>O51B<*EP`>mwjnnv)^&gyhP zva$bKLOa33B7s0e*H_@ygYh^co~({8G3ND{zzz*>*M=)sLp(B;*Y+|A)o_>$!_o-* zSlCO9fhju>12}Hi-?0#8(Z}51%elA+54+T!T=jn^`PG_(e#)vR*&$P}; zHtA>S8_Y|fWprMtjMC^Go4fzej(i47-Fy!@>Ju)~17VUdaniAyv-r$r1xi8fLG;Hk z2kvb&m7l%OC-oNPC^TXxqW%4LPm{0QN6&8fja@Tp3g1>VEYCT1c7Va)_o7jz!is45 zi{a?!U-TCBj6P~3Ulk}&s9E*xM|jUGJemzr)CxBqZ|jf>6d9sBbU|iIl3@V~)8xI1 zO5UNf6B$p*AtZ(T8p2l}n;bQhc|&-)Ocy}9L<4euYYhZ6$2J!{= z;cE!0gJF%>zpnc^5zYNu?@5#!ORE-&en^OVvg*B(Mf%pMnm-Ub&cf6;UzZjh#WA43 z-?A}|nXUq5H#bdC1a$7CqTR`4g+_}{AV?x*xIIC+a?lT7pE36qxP)RCunF@OCpCxG zykNc!B5I1?**_ZZep(}5XP{02sT!f`WL(l&sS5zJDA9;V+Lg^s8%AtGzt^kF7q1nM zPgAWM%%76~Fx?r;;dYFRi=*X&Fb;kb@ij&cpi>)%9=h)DU9S9;mW03*@7}Z5-?}<) zzTu@aI#z7}vI-k=I06S(Af`XtcqY>O^!`q&uA26ICc>9Un5jq&kswn%i2fb2%hE7TK+Ng4;p!^{(gtvb>wh6iy-L^eNsBm@ivKLuQ`(qBI~)7JbRr z-|bs5PfJ}F92~2$MCrJl%XQqEkX{JQB#-8e%sYLALLvWcDCWqNPT*&Z}!cP_%Enu6(#UXl~Bzj6wNZ~mT z0&tQ46#`F=&RI_fiNB6|A2EPCPVz#X!mYF?GK_x5q=zrmJw=Db6sZ->V> z1f86Ds@}}UmAc@Z6Vrz!0Qvz+RV#skmlbq*>8E83@~2EBLnJ7HOF=8*LaLhzL2;Xfy!12XPMC_VQ}GqaONfzfG8IP~x-#5;b7 zCovAOBB#p*3fZkjqnTlv>D4@NBsd^x>z7LTQ;lq*O-6I#CPLWm*J|E|OA6LtIi~^7 zS2Ln!yE8!@e~nC@)tTbrKX1kt_?lVNy=&aLqTu+bwv(ax-Kw3ah+;Pe+$$Xi@CHK6 zdN+Q%pPx@{skip)Oy(!d&Bt_Iz!qNS{`@LoFvgb-eHqQ|bq;N-SN88BzwWMD$o3X| zhXp6lgoXPxMp{b~AQM?Po&bwAcq{?cFp_6XFISyco<0Si84llNC+^(u>;#`eNlpWO z6Qa3suhzX^H69+zpk8bj{v z>ZHAK>)PJ_jbxkTppuOAW#KB19*B`f;mE1#td2<1v2CL{kE6u*Rh&5I-{)VM0!&^; zNXK!%A21uX>v;6s&NNK@6Af5gk`R2dv^FD6hQl@b(UB zpcerkXH|YehSuepZ)6o>+#o-sh9+i2ol}b;oDi;!pb5@SXJI6Z@%*&^E^AQRpbjf3 zPi)NzuQ*zfL)~F-z3es?F~IJiy+=b~95k3g*^H|YfLD>L7Cj-^y0+5@*oibf(8w-$ zmCK^|*Z<)h!dV&O2r7c3iS}atNOk%C&-P%^<{E-6h%a~py7s98Q5{|1eZpi zE^wwAg(t`Rqf#n6X-w#eADgx98o=;Kr?JGNnc$89=2ej)67>w9Km#0M{5$$r$EjIu zZtD|&$IAep?X>*bP%Ah^i9*-qNAkn?IdtLbbAIjR>zN986B0_ei!J3Vzyll+#Or-* z-wS25sj|om6qIj~PD(CKXMoP^bn=CiW&KX6popDIMk7dQ18c+XSwH(8&XV#P#mlZ` zu_hr1xVS$B(F!hTn;$^P92Nj4iaBBbnk z-?isw4|A^%2S!Kvy=JHd$2#O^xHR-o0#EQ9V7e3W=Olnl*0cf!Fy*?r`wu|d!vw-@ z=)8fk4QU*Uccp!K|6cnU1;2d}CghRCjF-oM!E})cagMGYmi|Y}4e^+gE68;xeobS^ z0>@Bs{jZGu|HE>Fr_#Eue<6K+1X>DxckO*+s$t1RSeFR@^X|rpOP`w#D z7XdCyuM=VN83EE$@Rm z|G7Dty)f2t5A_cIdp3ICucQ?6CGu}0lt}(H)hv)w{qTrxg9yYQ)o`JMtP%`Gs%;xH5x2+RFi zc=7jAX-?>ZVpB-rtJZ&j#Q!0BQPC&`%Xg)f(6fDvn(B*r(O)*DxG(%|I7g0mg#`py zu2>acB=xFq9DYvdfB^^~H z$~rMAN8oM!IqQu3<$7|6U?J`dZR@oAWl$>vl*Vw8pHeI;*E@YMfn0z-eqPvd)6HFS z?(I>uwF}`nf>nfZbL?W35J*Xsz}y*#SqQ52QFwj`r1-&%-)o7W-gv}RLE1N;2|0S~ z>L&F}tHL(l53F2;xrjua)r59S^^v7lWeE$YP`bzJg#NN{{FU`DHvmo$pvU#|{^;;( zW62!yVY}->w}qm}8-JVf0_R~n0OX=^!Ro*sKRjLRrM>nr@^2l`RRlLslU>9A-FN@5 zAXHEI3kA3HA7XC>uTUiocp>yMOx_X5zD_4yKy!##VRWj=LxtaJ5<({vp!&E;zmM|%@|nQ`eJDgsS!&)jBsC8kO9Mh-?WeMvavL0-S5c!iBR@M))JRDhWjR zWkhgIYDmrv3?xUlT!PnU@ z0}>@9&+R@-W&Zpu)?Z3lXy0LCAlTfkzdQEPvCLX9KwAo~GN?12IJes*;vQ?ZImWBV zi7iMztc=bcR?VswXlu_^ju$N6aro9fqm!X_j7OT3EO{XC3Q)A?;D7AExH}^^zdVx8 zsqe_kt%PDeb`#a7y%Y;WC$FdLYvmh_Qlr_6FEdE{i1mauH~)`Yhwvxdcmar@0sdm_pcP~DHRFbBKN+BA#m*SQ*bswCY<-Dsr3 zAdE>-k#u>Y8eAhLKNMh%*^lzq9iRWD+`DGzvh>v*wLVL1osxG#5E>$E~Uk4 znJx3NLCU;KBR#Jge57I@$e(nziw|2-{SN7W^@RGa9KwYTFKAuE=tOyrd%vf8mZ}{cNbGrg&ZB_E2a|}5cvS5G z!Ny13^L`9+yA9}Ei1>m`P&?SpTkjf1WKTDiu9uCp-!5H)kyOFPzjvu}wn?g+QSY~( zxxilHstSlP3U$;a#tQ2^C1z7Wyg+S#qoMBpRPfr~RUW{?1~fhcQ@QD%V>ZmL8eX6W zjke|z=BASq|N54>dD%x}zmb+O@yMhr|* z48Y(aEyIJ+l3D2qMw73BJxvbJ&WX>J70h!a<9s#Az{wsk+Rjc2;2;c~wZiUE zlAr+y-yZ=I#jm-{ACkPL4(`QF69XIZt^E^4#B4#W5U9k<65Hd}A_6XGxI~)E+r}H1 zbq-33F36h(DE3U-@(MoliMpF*lWSvbY~y=fGoN)vZ9>%X%Hz(#AO(yFxv^ztV>2*L zrpUHFK%rK2(s;biBD!YI6fFs4SnsuZ;XoSRF6`5;zu2ks1DLP4EDV9NfVuTFjzn5E zX&2BhG=Tmx%!+>EV)iHWUE*wBulhe~%$)D7lp%=Ma@_5fJJoz>qXPDYc7O;2K3tR)+t*n=OTZs8~{A^070?X zxWK6Z$W|_o$B2h7n68;32ys?t&XyMEs{>e=vw5{9-oMTRO|n@j4^+p$;=2P!?LUbr zY1B^BJ}9S$q*1adlud!j#LdTHd=uBxTZNNbRTyA(7i-5p2=HtkJkwP2M{_8s;HzJb zqb!>gmsdGYCX)i{LmuV_ZLI?+p2FlcZ|?FRfb8<8S>jaE0xp@(87Jo}lDC2C2OmsH z%h_Y&3;hLN$`SJmejlyLmn!tTHY%U(;hm{`cbVF8}lN_zHbn>=4f`M#*D@ctw!&-Oe4eJOvmNhe==jEDSpx15dALb&w ziuqSYlLT>qlBUVONws}`jepV+FQMl8>r6UZMb zy=Q}eTgAYqu5it!QJVx=+G>S(5B)UA_BW3WH^AHO%Sl|?b6l|;m=4@2oNq*cHN-P%Xo7c3HjZO|YJx~Uz`JrO|1QdK z{~Yg37i7`dDo=Zy*T*S=5l?Fn?E$2<1CnA3N+epUGfQ=S77%v_7&rTyow9Mwhz^Y9 zSxdf_)o;00tqEyof&xV+1;xXmf>z*kU>`8*E1P#8od^dA-up-j|Yz zIy}U+{#D+GFI=5k6OPT;{-)IAU4Rx#U zz+Don{CEXFC&5cMK2g8Qlt8X6s_N^U@pkTNvDVRx&G}mKLWa8$lQfP&SSLEEY>mWT z2voTCP0Y%uu`YTJ)eT&lZl!1JlqsNU z%`cr`Bu8(fk(#2W@S=2e`Dwfy1Dn#J8=ljPM9GXjL%VqzTUjXIJ2!q?9t|9QEXX;4 zkmWhKK63Ezw>c>%y$KA=ScK8$|#}->Q&s{HeF@47{x=nxJe_+G^zjm)HHtSK&8!fr~h4iqwV&*O~dC-o6sLk3w{v; zowjIrLqE4c{wg4~5Y**FL; z2H_|%vVH!VB(?A1hs>Yf`g}8`=Px;?h|b^%u!bl~aA^oyLidv*36~NLFGA|rmPcWq znVmaUGMv?-v;UZkmOY)g5pt= z3=Wr2+g30!-|Ii~MDd@A?jsB^-@n)Sb{koTRhO2tLRR%XXzSPJT|*qkc5kZff9K3a zVfPcP@nir|^-;f~izX7W!Rbkrq^Vni8HSC$b88oB!8cs3)x`&a@7=v?dy&>oC>-o# z_6Y-hEr8Rx$(R~X*Gx}S(6NQ5B?_i*NV&)D2pq|RT(1fv9Z-eKTI$}CKbZ3~~r>)WtE%-dq$w&Am)V1T!BHCJL z1~?aN6W7sdmyX{}obbkBQ1S^%tCb?dR z(haQ$-3Pros3vuHud1T3=49(fBsSbK!Rz#2+~rb~U1!wQpx{W~6U}D5b)7>s7C*@M7S@Wl2E8RNmmxH zj#Ul-DbBAfydSp^PpaSOVm6`qbPcW~!7=Hqa*vHrQoe|$0zdIT8ht>~`ga_K{0H)X zKcy<$=3h8qP#sLnA_%2H;Mm-?hd@>?r8lf+FOColw|+LSbMq&`y79`g=U|$P)|jX@hgAd$%W7qP;4_@kUHqya2x(7>fcKQ-4ju# zQqYr@!9fxV|5jxuhSs&#$dzH96^%4!!lk4Vu=l>u!Y?C?dL4_7vllzB7SLec9y3we zkl72}5I?ZVDzJMw`}pM{kt0U5yMAYV{(KQuV>%7R3C+Dtk%{}geXfv-nIdF!coN~E ze8c|tN+msbii5*><$G9zwkpBJn;?lX(D_Cp|6$H7x^FN%;?zmQ%1ktim(MBu)Pu-i6 zx+;RK%+5o!Mdrsac$oTM-Un46ne9~jQ`W-!9_C>`R@N@%-zH(>U#R~Btdh>Sdfwa~ zIbX=Q_z!dpf8D;13wtW^J8h@o!RoWsl_@m_W3yPraBX^jFFfQF0wuvWhr2X_@{e(U zg11NRW5d4nBUTj5FXTuJZ}0x-wl5D@$^PB>GO-E6Y|_a|FEr41WbS8 zo(jY{R!a7B3OKrQWw|_=7~XQlTU^6xo2A^}Q|itGY&m+qzTfV>!@uyH3&gBV&lClv2uLaaZ-aZUs`3P=pfBhzL(m$f*w}G> zy+3Xs<-pdbvP4=rIk0h9`S+_sA`|AUh{3#A0}8JsT`2xt{>A`cm_44>06Uf887{xV zO&}%I^1{7>0E`MMy!nEuHW&RS|M#8cfr*1OHXQNPJX^MtEO>P+e`K?Em?lf`909B( z$;D~??IM3aK!hc5sTe=B|JsWy^dH8sdVt+a0Om6h@3OFb5o?UUadp{`!u`f<3CP7| zL2?g8I9P$Pb^Xr8`bGNjMc_!)nv7l%z4+LM@D{gBu_|_XFSk?0q3q({g zP~5Yp*e(b@v+8+5HPpRGU7`&N>|JuhPd$0qB*Mv!Sim#T=X>>7R7l%eM8qaihSGi_ z^P3UCKRP@43$tQ{y?*5hn&n>}F_41DJ>vnC`vlFc@K4R*QUK8iF-3ML-ax^OwAr_C z!PH8m{gNgcq}<4L(dcJSc{==VHGJejW^4bg7>|^7hjJ{9W?LRES~#&Ft&T$*YrUGt z)jEWelPK;hviVV-n>gPtF2|6&k};8w4|iz?U8L3ivG@$K6dy~Y;3r3KXCiLDEI(oG zp{CSG-xbSgPU^MD?^Rw-&jd0W(~MWl93^7YF2E)P8cez^e0~=!6I2)fA+>Gpy`I`f zXcnYbpGYVGJo&vxtaJ1gf%fPr{bEMf>vU3b+ujk>hf!WJ*iRTZgHD4q6g;Fb;VhZL z`QTYP+FWjAD7d2>FVN`=iDuT+7dEs+r!lni0ti=G%F8NR=EndsV~(Q|BowBc9l%?> z!5jt#vCnZWf!_!FL^B#Bs;J9VbYrmQUk4WVK5?cKymc7XmN5n=z6{my*UmQGOv#(2}#ivtHJ1NTk2>CHVd$|-{Mm6;6d!5pm zE3DMF+J{E>BfO~8m+|v|K)JZU;}(Uf;7L)J0e-@l6fnzEQgUyhBdrL(E|vFp8;n7?duTQjOng-4N=Lqo>w+bWtmjdJqW;`^xggFLEn{-myFQynwk zyR5Dr9qxYw{wt!g3^eIW4E+*6ulc!4+ZRkaK3TJttOgv+7IM@bI#%lGCB0>4 z@8J;9uHjejT08Y`7|0V&NO45B6uTn}XN*X^kPsd|Z7fd^Aq5$;3<3Uh4dE9}*fjA2sPGcScZ7g*QJIB|;Gt`ejz&8Gs8};~?;10U$Zu5I$nnO0jrL!( zls2vsnn9 z##Kc#>%Dfq8=HowYd~vDqsGk39e~Ch`kJS}!_wkys>d&sq#=5th=)j|+sDn`yUVd- z8`(#0`N5@B5g8t-CN5zu;dGOj5H6X)>p2Rk^>;9N%|i@gd^rl>>U)|>Rdt$QW<6;W zA!5sQuP9jhY-*Q2X*w$#qUTpaR^N zUIkI?N_YE&>`0^Ipx@VG3OzAB#}BN$m7Wo<%F~qFdcpibr!D9{-e{UH9)-?e-s>f1 zchg3RBG6d~d`Y@N)O0{q@tcen4NzTDGb4_CSrL!4&-VdG_bVHd))i~Or-RfeTeH5u z;*qWA{D5=3XL-``E`HWhu6|cPY#0(XQO*6ubCm5|<*(CLu0DF@eeHI|(c{)eXyE5n z+K#ZKVXF)^W+h4~wLP$s>ZXJ*3kH6<{c@n;G`z)yS?`}C1tOwA3Ea_@1 z%YTgmKvE^ZjOi4`+-X1y8>Zi!zSxs|r^=ZO$uI5wJz=@{vK7hvuZm0Oj2E_-PfXQ( zhP;fT`jJZ%s)I*0Ex1^W&{e?slQ?ENw2`*!eA!Zw3R2z+HuTWCPS+` zT%(A+TiL08vBt+kf%=FSgGbwrgY)-&CLK8ye?c_}pK>Zxp%itBTYf6(83ktiP?Nwf zcTC(_UMurt{RI=R^|sK;n&)nOQ%@82D3Vv+!Y9c-!d{6~?KgmH3F19gXWjcek=Nt< zSo`_;=rlc4YSSURBCM}fCsaj?|E*}!yq&09X@W4|fC)zJ^`Yw!#%xNR{$ z9^!B%GrI$|U=4cNjrAE*W4q$zDMV(=sng}?XaEx#_DTt&ajJ&0tyGt07Z}h8Nn^Fi zM}g0}m{a0|jvsn&KV{70v%`M?OB#zaTun8PN-CwC!L~mX(b2_Lj#X4h3R4geK5juaF;gB+2RUmOMGe!NeGD%-<(hH@%X;V_u-6o*w#j$lzVb7~*#B0{ z45?Ph<{{<0m(k9ydL;xDJJts8RK~HuXFy|{$!g{GuYJ2eG;;cDiH{v`KEK;>_DwIN zavkBea*f!d;Z2qJIrvoLv!R)jwUtPg0?y$lGN)OPYSa@xFahBjg?288>^r`T;OKKd zw#iR-_^@n|xykmga` zv0`!z64ed6js5pl>2S!ROx>%)3BdCKNZN}{UdCMeZi<6l@Q7A-amO6JPpLgvwf1Wo z=aM{ZJ3-9+?c2~SV1`AlhDZRw2}~j01h(G8%-U96&|Xvu>6lKdKIup;3{)hR;_unB zXz;N#8TO%=jnIU>z%aYi&{Qg5;J6zJqm3A%e|oxcUuk{gJQ1*|>9iap8{yf{y08BI zOy_5p6VsvC)2A6wV?8`=LT;mLd)Qt&yXXgB8~cInHXzC;^j~E5dPH#&mB_ecE4fyh z)7d-oyK~Bh8WCovBufz2F*qC3E}~G-K8mK793LHMHIiJ7DGIK0Q{4DekoIA6@T&eu z!f7V|KLEo<-sz;#8x;+NQcDI5LqUz;YaDj%tav=M7W%U=i-sk>G~)hLS7c*q=Hi>^ zFFd~8`h-wZjFHkq;5irursQ@`G|q_Sdu@?_R86`!GigJYG_DB6Xz=41vApK0>v#H+ zBX??#j^n>;jwV7fcn}cns4_}Aga#Np?GG$HX$&h$rCI<^Gm#(b(J8jn;jo{*U@vX= zd|peBA(JgTQiR3Vm8mD@zBdowWN_ZLjQ$je>aJ%T*_$$xAfphA(9WITtLGKg}^`P1B`PU>l{e@HVOS)l`{-P*6&HG zus8`^YEyIbJ&L5i%>pN9leMz!vyjJ}whg)I>N8v6PvKa_^C^|WEK^p(x0>43@EN$- zGJnsf`6_wOHiv*WNrbEg0c8*Z<#I6@tasyB(p8t>{Zw#6Q42D2*bE`9}<~>6;E6!r%3%E-T{^n zT>M_W2>Vu{M#+gTuY%X@%qysM2H~(7Qd1|T>uTwsfTM6E8HovXP z{2CW+s3$gdsv+u{{N7p$$EA|&pN{uh{cOH?VekCDU^u?*s@1J1pNsGq=>{m zs1l1s4n~PXvA|q_I1a1E{BT;TMj;$pc#{$_i_i1TL)g%i7C4;H@6eo0@q0i?l~?|Y zR2Tu~AgEMg^S)q~3&2qz08Zw{n7@r4&aM)*hYcbXCrtep3vQ)5%6z`sF}K*uO74rB zJFUH>_r)SKojX8}Dj^K0{=CG}7j(rBNz5*yQtI_}t%0^KWdaz42CY534yCi>XaKK* zj2RLfnCpST`oH8SzcL=6wcCx$_}YpqzWRVld`&EIJcwZ7)C62xHeyl6Ii`OAr0TvV zG5I!XXTYeD_r2-_%}Zkbtqx4n1(swLlg@nR>}9ORn8@owXFR1v2SS-!?L= zZ`QSz39D}5+#xvXdY6(b?2AdujYBqyYwj?s9#P57-n2=7mdqFLn)(QWj(sq-so57| zQMnVt{{djX?qq*kkZ;MPq^w|V4UBpJHZ!G8cSV8&b1A93gUQS$pWxh9{W%$LE@;w3 zo=6C)ce&rJP7xh??^wP($6O)LMnV3K`)5aC{1<1EuXL~f1Dp#=)-gKeyxS>4?x{C@ z(bmN9SYY7#S@((GpCJ}oS9X4z?ydCpLUUsTRnW<9(C-au=Er++u7}%}I~fb$)6G|N zkD=0!#nLLtp>S?@wz*t$4<%LB(D&E-X9xldIdO3e$|WJm=4MPhjb-pB{V&^d3g8W` zd|s#Tnx%(;DS-L|god#Zn4i?rBBL8e&@R%EbhcRWFVxtc?&CRg?e@mhT>?w#&$VX_ zuLdvrUq^Rzy!j6>dVcYGhECHxFhV9pfrrs|;Q!n7K>pu`qIGf{mx1FE7daY=pWl1m z9~^cSVuh)c{RenZcwqZnl|_Nyw#pox)E+r1c*sPu{d3`pj_8ZnP=7t^f%WER=at(N zm_{c)x)%xX$?nWX2>^Zb)M>Z-l&8J0uevae$6(hG3gvKiH}O60V4Sm=pU2u;2<9-0 z3=Hh#?N*rv6jg^hEQ^FT!YIEXo3dAqF&}_cLA}K_$G@n*#h(nivbyg#c54a#*5duY zVHjcMgaSS5?sn`|SbqaS-7yAj+aM4QdTkjyr}^PflHW zpF2Dz0f=QR_V#@|M(c(k@O-%kTM2>zjeP*Ux!ZHwheI0^K-nz7#wv67^Wap-byBZT zOdmDP$-+-MjzXw>XcK0kLt@R{JzQaufNe;Ho)3J>b-%icASi030*q{0UAJCBsViNe zRM)?#b?D_RBfup%}LeLy2n2&LRk7QZxg;L={<6)p6l}Ss#PDDw_vkNUqf3bH=@+; zZ}w*(6dtL++B>$n>@<)Z1-&XE;+vDd009c+IH(r@fO(@Bbty*Fq-_y^t=K~5d8~ci z-yO4IE>L2sce+=dm);s~iGwdK=cN7m151;(h9GI%0lvZ91(kCygy{Kc_+I-DyuG2; zy9i6mG$HmYp%xH%!F_@n1MzyHt7*#B14CRF(g97=@8o<&v7aB!=v}&SOPX!_ro@w! z#b}X8x?MyTxyT_#kF*uM!7L5zQ*}&2A_`rW-5tqs(k#b_Ew+G zG^!~Jgyyuw)@*_&rQ#k?3?27o-Rc&n=*m^ydLswk3a5$4oX#zO{15QS%ZrQ+UZIT< zOfAdaUQs3Fip?a7&Ook&-$tq+{0f)U8Kg2Lvbnj&9m2#k^us8f3^5Q|rM1M`FdFmygloR+us-XodZ1f6bvZr3dVj#0 zQHU@+#UrMYAdN$HQl-h^tYmh2Q)HAlsrN(!iUJ~FErHQ1{y3hKns)~7@R$5i)%}FT zuVP^OU|@~U*JA`pxYW^7Vkq-dH&Kr_{i%cTID@CSEDv7RfvKGhtm!RS6ZLH&$1bKqIb_dlT^|Hk-vhpLaQ` zyhCNgiQ%#i@?c8Kt@TJB_c`{P&~@J(iWWHrMf<0h&~@L$X9U zr^rsSmQ6b>;KpwHcdOtIYb=a0M{q?Lv%d#)|>+|X{pF)jZRc~$PAbN z96Znq!m7cd=euT``s$Pd=f-ZPb^4JbVANBEBi^CreQ3!#vB*-=v1^xBn&GRDm5$Vg zWyhBricLZlFV6Ov$~K;HJ+oRI5CWDFrv{i|Gl8D;9$*s5VXY_O*@@_Dq~;w~e~K@0 za>Q^`v3HQ!Byxx|EW5&!uSpt}LG|6HpP&P*Za|eZMvUyQeM9c_OlFN@)DP_D+Kn$g z-bml69%E(Jy1Fo?nVTJ|YyZV`#dv3udz6}p#o&zV@-U9_8wdr+!hyYP0J%9P8A@@(B2if)$47Q!+$0b^G0z&3)j*xrN&dIFjh^^DCXF=&rF8gE8# zo->Q5j!oCtF5GGNI8cz_Qf zLx*WHU2lL}PZ+>uqtPg_G}6&_KpG<^JAiwf37Xs)55?O;NE|#|WYfA-UzdAQYElhi zY_9RxTiEk{V7SP^horcUZsczYFo?@f4wy?qvW)vT6~DJnoL$M$il5UrkRJ>p%Q_NC&r1y@xx#aVBeBX0 z&|D%a^G=PN=ERC32eyNe5*bfAhACmz0Q~+?%TrRV&^`|wucD7o)o6w-qiW7+iW0`*d!u6ALuqr8*&uIi(Lr$b3l8<@rwFrgRU z(2S}Ryc8HmKfaB2D!2@i*c?=APDK3#Hh(rs#sY+$`;`LvcIh=U2L@3kkZ3Ak)@3?~ z1b~~kK`7$#8vYH2B`{yQXxYnr^8GeZ3J>0EJ$z5y(?Tcu&r&&Sw3^Qnt0s$)K1m6p zNG0Ch!MKC_jVP$`Gi8GwlF8pdeCB zty9*gMX#w$KGn?xpm2M=Kk<7Oi`NAh1T=BE*W9fWh&i83W-cw%G5EJR&2vO_MA0~L z^M6o;D&;+eukvbQ1K{uf#1X)*Kvdulo*jN%N;HN0iiqRktv))AXhfb%;Z9H`oFOk^ zR!zXRQ2kwfPP%^DE}Ed`%8E#DF_cppj!Tq4)a9-8G^acW8K_@6dXvd}M5&jU()>8l z^;^3-K3d+JBv;T=N@TnJq0*?eYnf10p=3eK4CuoCE3)R^g=1uTvzF|OVML#?L z>}rmKBpdm-0=a5c6L})w)apGDo2_T;_OzK`%_64#_~qc=%6TKhmveLVHL(m{6`!Bk z^VWK5e>ZZ}TK1Jmu*+&gWacPQqiNx*(LC@>^zSdT-VJpBho`p;Yw~~Gum_Cp(IJj8 zxy6gMw_kW(_dAIHDcI^J#*L_{* znT3H&B(jb{tM3In&T+}2Sa|hxl zdIm8CHDOn9_uCrk>i1PKC|qcGi=|Jdd;D+(Dju?h@a*#H`iqZ zXq}EN6U*0u=D5pFDmu)^X@P}BefbIuXAbKtNKm@*e(}N@ca>=0aU(`V9FHa8RY9D2 z>3{>v_+__DM@gHUR8H#e1?XvWnliiUS3W2)e7m)tq`KvO18lqWDaOAmJPw}MU7~ca zb2;E!9o9qZs{Pg8B)tG(aZdOl!n%6Lhvam0sHlRym<5NkS7HS}AKaRB57Fgxz^FFi zt02u`6o(&*9gGpl;ny+nf+h`;?r-u2knV8!!R1YZA>cIUV} zlX~Py^oTvQ^g0TcK9M+ISB`h?9N%PD3jLRTmdz-|5KRD1Az4j#%v4rbc^)z74VEMu zjXp2E>By&_JJbA{s)GIIG73j5jN>+bZc%zh2(7+Y$!#P$jg17F;_U#!@WjlANuyPn zp|tpK$*KyXG}@)pwDaReX@roY7rM(Id%4vrM}9NBZAE-^Cm>4m)n z5{h~um$nb~;Xf8XYq0FTbY`;pnqNl}UYnL`3Pf zBdZ;+y5Gmfi2coBEofd5yURq`k@*AKhp$N-eLR?zh zjO^z|U}R>pVr-v#y1(wxHyA`nehm9$^u?Y%s@RlBwH@vMhFkrGZ#tWG4jUp2zv+B@ zH7uaOe#>S)%X*sGN$I=BFQn;?n3=npfOB!=p6AL^DEDN6l~0c}wozagm;vB6p|HDn zDS>^BwmPGc0WpRa1{}RLpje}9v~I=7k@qT$l9+w$cVHf+2?8Iuxq0RySa_XcAwD`> z;zAP3Qos2HMOfeCywI!1s-bmbGXCBoMMO+)RQBXgCT~!IOL^$O(ggRa*kPS}+nj9gOrGMXG3iqD3d=kd-%@7 zA&Lqq;;HV@%C{iDQ3p~nXzFlY;t1US8nh`U>r3WZiOt@SIA}1{OQrs5io29zp8K@s za2y4gP7h$>dTK?x>y0NecTLR_>W{zJ7V?)d_?f0sw64uRY*L0Mb;Ca|gCK$2btdXK zyhQArxtdI=P~u9nm>Zz`Dy=c=lnT!4-bO975ayngDQ)2H{@&(C^7+9gXX~NB zIIdz=MvM92fL_>t06i9vAQfJKf;gpX7M=siD3s1xVEdd=uV$eBeO{B3r>Z+Wn&lf{ zXFF+22>YKUvHZz?v2(Z3#^G1VsxSbaDC2^Cw`m$HcCxtI-3MNL>&VirQfFE+Rzw>> zE`G^ufc_9|=K|!~u$vmN2BlFWg(^5qjn?*9mD$-dru_I%c)#tyyam+@-w#!Aakj;$ z=54DZC8^krb49(nV*jugA-(7gZ^uJDs~nBldGLgwd}k}w!Y9eUe=#|cOzC}&sqnJA z017kG0g*|><`0ndZ~b942l~Nfuv|{B?&a@Wo+&NDJjIc>Zdc;x^x!DHIUs|GEsIii z!-XjG5q&NX$D_>SXH%{jc=pg>wKoF6I>)m*p=F$H&4YmO>h^KvJ_eDX{>CW3Q|yVw z|74y=isnmBzPY>_All6ARh#Bf(|h#2UJo1neY-MB2jpWFmQU}osP4Pj4rkPm<9qU0 zcs%#+pYU$mvUr3`1SW|?)Ob$M)O^=!q^ZFPMsmm=%K1WXj0p$usTi{@M++X`0QY<& zg)q#;(aYx8!o(__qRlPV`EatkX3O2$_m^x&PD}Ld{nyKG_(dycLtP=8ItkB~UYd7) zALXrDImEBlmB+?669-Y8txU|yoOga%0aEHYZSmjrKV1Ed;aq$2AAk+WK{>rUJU}dC ze!>#rxC@bO#6W%t%Afib9hb~AbA_vd6-YJ}!)vsla##Tt; zDPI-`_Z+pyHg6H&X&!hgkgz_z45I*|ggG|-=nA_n{+B)eFZz1Sd%mZ$%wtF!E6lmn zA1)BEth(Bqa_w4pdw;Az7*3$8^M3YU=ONn+kfKug*J1!V#ba8WB@i<`exnFTQ8El1xk+rKdOiR4$(JgYC^6EuZJHL4`WHsEHo0|y z(rTo*10)$n1Z44=+wx%1{~P0#m^^eu@o405#x`PcIb?bM%T;cm#s59BB1@NB!kU1; z*GjGO@;V$MU)%=)Q2{CXFiDft<0c#v4GV@OpV1YiW3XYnGsOj5blu#2!hkc{m2AgilbPXW$*h&nVbA?<_O9hs{6^SfOkWETb7Z7 z*ec131TcF|s;_9BSGP}-_2?7TT}1y*Kr@m}hBYFiZQMCMjZ=+SQRJ&%=Sqrf(|0?w zv!C~t^YDC5wZas(mTJ1a4a^s@LID<9Z=X55`nK~v3+&bfqy&!*DyMpQd0X42A(m!z z0lRYI{cFK@3X}R)wSNQxOF}AWc!NSCHt;c1M#Em*1T%aPLrjS1CCgLE77i1#_PF9V z&Z&KnM#pu%p%m872iVYk$m*t>Y(oC9c~J9GuG+OycZuarW$ldpB7gmxpXw0nIkjxj zne2>_=BT`m@9+;0J(wun5tH1URB!AuW?u>Gko`07E>1`pxE~*g$!V#gyEgquhx|zW zQ`KTMc`X^=TWH@RA5Q*|ufJ|JOBQXM4z$`__N_bpOci)4E2vo6c=wdlAQ9qdpUMV_F2-5CC7VnPI@X7nxpD;i?Pg2t2C|+rM%%R;kK* zf=%#V>ycH&tTFGx&|q#~=WVRqS`#4!|I-Qz!H(6E0A9&_Nv=ut9ZFq-x59x*YDSf2 z-5$-oQRB_1Otx4oEi~=Y=~}~4g@5w0-=-)#y134fqtFrgQz0dsK$KP{7W$?4y=qL- z{oE=zwT8r(Ps8wP=U_fr(RS{1k1!w>hLzj~vlOUeADQ~{LVQ!3?{`D7XuvjJEmA#0 zfl#$n)IWY~M?w}epxJ?szTc&o*3xY`CAEN#{@`WG1U7@l-Kf*F1Q=8>yCB_@dS_8| zevjU)rVpjKa+>`XAlN7;LFMM1`Iq0AO}EMYp_!Q4n}Ji}2gnz_JX;S^9zOFy0w50z zOStzS==NuE1{!qGUW9_);}0yUyiS-j$jfEBpBvKm>wWo;{^u)gqfn!#w)E_viclDl zycY$4T|`0iU!ZrT^^`wH_|##Ig~mJb!Q`mnsmWQj3|>{F0tuIvl2m=X0o}K!ub)rW zW;~mh=njnnKU^XZ}MIB|fTrSrJ> zkEu(UIJ|hoKYKlthR*PqiHj$A4n11(8ZYX-_16n>hl?;ZC6Y`8kJAV^76>Oq!bB9p zlZ+%JbKo=EDN6BKcF%?WO-C-H2Wd=V>`u5ga`!8Rn}Fk1jUFyWU5qBbsnPy8VZ4L! zQl-_L#r_lT?br8nasVI%$WO96=F9GB(4Li&J1(!5Bf%*@y{r3+rB#bZCvL{z1?Eme ze2(g{1RT+Ws-&Fk*3WD(Qw81mHw{d=i&10ZEqD{j6jTY9PoJZ)2#UiITF$zq-dDPe zi!lygM=NtKMZqQ)UCcj_^YhsL?8{%48yPpkunHs-1S+SWZL}vHiE0HBrV_uvcxDva z@%`{`$G$;9%D=?~+=&^JPv{LMJcH)RGJCmIUAhwmsW@dP=>7@?hlr5zwGbt8Bf*53((CaUGT7Xn}!>h>0xR8Z~EmP5Z@=XocxEa^wLAztTNWc7`&8;r+s+z=z%cz*?yS}uu~(_U|O`M94}kwTD? zj?sjRfZ&Xl^b)kq5KdR*1>#ESV4!&0f4t5KlWD~j7JS00Dl0pjqAzqQK@^40<#OhQ zaR}n6^tQZJ^zHUYJh*gY!%8sLx|{zA5Cz6Jp&i>ckcuGwR6xtzwYo z8jv~`YW4&Tml-ZJbuGzUUH~7X zVe(vQ?y#1KnVEMI^U2$4p?9xu&SGMdjH`!fPime#EVntwgek^E4&wy%C_g1U?;EwD z0!DUq|EiulVf;Rv|7ovg@#m~<0|+}7Pi3=TfoC6RVKwdBAuxt4pmX>{!O}a>IX6T9 z4hF6f{3lSUy? z26aYXPelz;W%-|&^&OM!A4#*CY>`b*Bj7~kjgo{`j(}_)gzhFFP=Fb` zVuwT-ndaY9e6du0bu-2gX?$W#Yg+}9kn&|g^-rWry#Xda6U(2(k1}Xpiiyott)DmP z^3s1OKz1yuwxNW4K%6>7DKxTUw5gKg<=7EO&@$~)joAk6{837i{q1CbZ#uc*6ehQu zMGnys)3#8Ucp9CX5qBpyWF;J5Uav_&x2@vo=?A~a`+GB$Zj)1Z-M8^Ys@m1A>%+~_ zw~~=uRQ1TjGOhI`vJ!W9PA@e5a}JfQ3U|j}w9g4XC;a4@_hD!KxKM0$+rK&^rc4E1 ztyT5K!$mv)ov_uWIyDnl%={Goe2!P``z}$>6z5OyVmr_@1Na|6tvlXO+q*I3{PXFI4?5ww|qlHQ6*cYuOb-JJanacgGU(FdRZsgVD z;kBpZus>0ppGT~swip2)wqv;Xd65lhHG7L5l4pGI(U#<#Xo{F(t6AI;VfCsCz{R0P9CX;nS zt-oDgL$T2OD-{Ddjn`e6gQxAzrq}7WI}$t^u~wvDHw_eU-ojrC-ppJL9F}?)Hq;mi z*1QgF4rd3iD#rqiFDNsIwsST2d8tf)B92P=#CtUyXX}5T9nM)Pm2+j3)>4rtw?3Qw zvD%yv@VNTAPLMd463(>8F-Y*d#{8@1QE#+}_b1*g%`0JbG5d?m;BDgND^kY$UvP#+ z2DM}k_V@lM@Xvk#k^6&~WyqbOgz{93I%t!Q6r?s)6%OXSOV0gmEZ%-&<_T=c-x;}5vnJC zkrD?hhk9`xxU;hyY@L&C1io3u*>m81b-eT$_Ws30b=Q?SFlo^rvCS%=SMN=25S~_% zU#M$+>TbRH$Ye!sdotd({|9UHhW3QbU-?#jY{!DcLRBs3r`WHL9Gr>QjGqwCtVB7) zC>c@n+O^#o{dj=SCjDupj%?6DB(j$n#C%_CMqFMko1Vfy`JXC>kpBQ# z-+lc|+bS>LM8TxZ91P%FrE>(hNUa34^>D69MW!^$kb7{8Xki@RnewN9eeB5A_13de zRz3_q%i&ew`zzA>kE2)X?vsH+n^&gp$tW(ax>qO%;lq{8Isn|nC=E+I%;n#V^}yyX z`8%#&23WlO17eHU`3a}Qq#c%&6NYe_0?;9Zp3`;gs(w<_-+mspd@xv8hweyA7Pdm* z3e0lT07|ag6$QiyjAp&CpsL~~R7QF#*zkI$q~8|L{F0n77xO5jNwz9Gtc2wh2Y^+x zu;Slx;<3T>eB;(fR-_t)gd>rI?XD#yFkEq@ji%X=t9e!xp9W4NcUO)4Q@=!C+x0SR zgWRYZd_r(`7ujG$Bi~8Kb4-uZWbmEt1WUD7Wjk`ML^o?bJPo7O6=Bz}Sjp*%Kc&u? zN|~~;Chgak{0a)dXBr`yur|z|_C^b6?cYc5CR(|!%|t#bn_J`HD;6@{Hck$gch0o+8j{EoCFpa zls5?$YGn`+)kga!nSI7Z5A8}Jt1>bo>xpBKTN4)*p&YsX31;I?dcZPmR<|L3Gie>Y z1$mpy#K_?zkm}v^=+ePBpt9y5uwEy_KRU3=OP_*JgSON*^rhnTYMF*Jz72Ra=;0p< zbMnB*a@H0z^-`28cnfr9s0*p&=ec*WL#D7pSTU&97Kc ziUZZ;u6bQoCq-Ur)f|XT&fWIkP7}$u2ImqtBPMe}FS!93>Hlzad0|&Jmg_B8<4`4g zptmUYFZs{EQxfEPdzmz@J_PI$3u&P}Fv`%KFF8d;!{ldP3SJJ4_?#6+klSA0u=kGazwbb@6@Q#FitKRt-I+P^(u)DSnCbaz6^1{`|9bE5 zr&rXf2;e#%Esq}I1jfHISTMA*6?X?93ueh@o=kJvb-p`U5aR9+zQ#YBE<7>9UY~jt zZ+W=+7f$s0`}u8;!I4!6e=i%5QeS3#FXSwopz;1{b!0~FYGoxDqZQ!X*Xr%SxCU;( zak8B&#a~MPhLcxt1C+VhFpnn+XB05&M*qHz7m-@~A$P4Ie;*)iyw&IsNy%@0{SPJg zJx5y@vu5#KS*U0Wt!z)S83eT<#v;2h_x}OSIN71(Z}yczLyO0@@7}Gyi+o9`sz?Rz zO_vY>FcI`7MxNyeFJk}Cs^CzcjRE~6-%?#h<*eSgD&L%M1$X2ODs6CZoIzH1Yha){k@y8Bp;7?k@Ch&-=K#u zJvc7ZCbMy3D`#)+q>pRoe_KmRbVhEd6u9yl{3As`zk4}4KTmH$ z`}q!&Ql*|;43KTI@csM`@X~*fhBRutSYFNR;wj+(DKgZ{d*S+L$Kv-${Fz#|#rqp` zhoWH|SBh9@YCD2){@cLMgP*MGa}q+F-+bWSsh&I{N^z%=akcl8|Lv0GZ=Hcg2S4X3GD!O@Xfm1tz#0H{*VEuR8;F;c z8xXMy)8VZo%Xj|r(DCJ6lYbL0f(1$=053*TsI@hcsCaFXaGZeCp%KE^Zv!JFT{S2c zH+s@EmD4ocb%s({`~iV54RbtMHe@7MJej3ALOg=$HOrQj78mi&w<)LQc#vJ{zZyUp z7wH^tP;8)R=*B(_b=Cr=aD_z#>{y>slp#8!r8XemT4~JLmOK_Ecxc2P?JlzfG8QAm zeW;j<0S9POiQzd8iorrG>vTy1(#!E@!~0ICEI{lC>BgPb@T1gO;;C;5Z)19&3~6 zO0CvPIw#>aqlTS~#IFg7EP_@}%izuUyBF1Fy=E>)eNZtJr^Sg^wH{$Qzkp(=y}!`< z7%4#TA0RTeLv~>=La-p=jx>?kyooMCtv&}k4vVaaOa??Bp5mW&I4Ym_NOGrP6X!>RKlX z{5!HTnu&-a%_yOvnhZr9U$D_a_s(zsxU~ycBa*s=(m>hF{JWrz~DVTu|+o!qmxrq?c8p^E-IBf{d*Q<4`x+Gm?z%@9Yj=l%A!f*sr zR^>w^g*(oR&%HA}HuknA!{02>c#lq#xPOCGrWj+4<95k+|B7^4=;eZ=wAY8RvDQdb z8Uu!Fr#x#P8D4EeWqdd*6yHzpLR-v++xe96nB#Sc0arq-L)}vYFXx~yF=g|n-W{6| zDXLs6tdCfXwFN|++2@&23r7t@xhXCca`@_ z#}Bf7$c-b8{Z{kupuVynm?oezdaQctia`jT{Q_D4jR;)Q$)FV96Y}g^GY;8KbCug7 z?+mzYgUd+eCBF^ka{3oVbkDz~GMXQKR{bUdd4dV-4G4NtTrFWsE48#V96P6gipZKc zybW)}{!bhO;~rbvOC8Z%L+dKeR359~WDFKWV&2ovIweGC&l~rGk6~f%gi@4e$-3?= zS5vdwR`hwRpEK6a(!(A*${!3PPumMP82x;lq1$ zWK8JANTlqB+%Z;6zjVls?Ul=mI0zg{2xnEqh}B#e*hN7w+{E0eDaL6rcfZ6y{t?X- zXsvCnl^K!7JQvPEs$V6FeX6|FKTuMmk(J0~9)cxZ1p}Ltj)4`5ke1qsIhI}28m8xc zGQx8ozcu(Aq>J%pJ#l(%Ng!ibkRot+K~0zNlg9L04Oi|7w)j#m;EVRlWgEu*ZHl_5 zfD5|sx9{-&@zSa@^luR?9B8EeY;?CC&w0+OTcZ}Kg&tVt90I#q-zg+ug7{#VY8|wBkW65ICi`w9kn`^A-uS=>J)`Lw0oT z<%#Y_f}Pl*nBTZ}E=V<}n4)E2Q#aiPoGxi$IBx#5g0tzxS@J`lBWi!533%Ei{>_^} z#8t5Qi60xNlpRV&BkJn2xLjOdXwcuNdgZNsM4Oi*U24{dH}v?`ei)9_x@=za@?nQ- zM5alLS9ap)dC!;Ul*(4x3DD=w|6q@a?FtkQvYxBNVl!;V5mJhVpTP0J78eK+``@&X zzD6|}Q8C#o(?RUKLIO|RVXjajIG~~$frnS7j-cdbLWB!#W<9GFV>37Se{R$2qehI`cbF{)o)3{hj4qsVPNdH7D9s?M{VQq3vqnO>J=LL=3&dM;42cQU7fr+O{Mh*yHX67BN6Gqv5|uSf6eZlTy5?2+o+wd z=KD3N0CzH4Rxi4V)NYJ&kO7h5Hdvt~(z$a*|B zVBVkxZwFjGv>D+jnq-g>8qb);o=CvQvgqQhBY9zVQ)g*1{%k(q;BX+!-ujLI=m!V& ztWqyFYt{0K6v9}zY~PDw$&mu#b!fwhu!FRO5C-wUiM%Co=MHYcUBW7#Cpry@(sa2U z=A%KRLm7xfy?vlgRA}On(REc(NA7j%*SgaqN5jj}U^*8p#w1oAV(2Hv7*a)?pikW+ zVLG2@aZFPyP`F6Vv0|y^J1U5{6g9-OF#OHc%c7re)>@K(7bGeig6|QP?S~QNVnM)t zAN4R%=`z9`kfo^tg*972S##^cWOde)e&uooF}yfw(~^s(qU*GRn*rE{6R)R9@~MB(scw z_@z*@c?m7Gz~xB#Ce&2skRd0E2^)*kdpNr(dRo&eAn9#+&3;dQCaBCqIG|@0Yb+^e z0zM!b8AtSqmd|9sfg}3_ocGrsy`Z{XU`(m%O?+*-MN5Z6Sc{&F6H%{3YHDgFV4lTQ zVRc2&PihPi`r`M0^08VQ{Fu|P^~LL_(;?(mhUpJ+OuS8QTw@Phq?E3u4bcq(uK`f> z80q!8+^Qk;Mw}QK91W_ zreX(pxz$Kl3lyd~kzozlPxB$bcC%_3)xe3kn7l%W+q2gV*V}&*|B~U7d-$4HU~no| z*!!nP|6i7$HN5|1+L?d8A2PT7kYi8i<0RO1){4wE+5r$^4`6t(%KsA9c%wC&lUVsk zjTF^l?Q8zN_;Id1lWA`}D-mW~`(b=jIit=yr%*RCUBf_Z5-a%QdQVY%mD7Bk!;&k| z1a9hPt`Vb{pF z;m8eXFyq7NXi{*dU;E+rW&&wc4uCQ!&I4o9Y*L9R6nzVv(qTLT6`dl#1j*c{r8@|% z;dRt?2J8Z_ro-F3EXYi#JoGYwwewX9DVj_K%)2g_Hm06rl^q&hR7#M9Q}V)_hI_v$sEFoJFuFdX$@NmH$McseAEhA2iMC z`C)5O{ri|{gohISy_&-@eU_653QQ!51rr!vT9VLrB^EmPAmG_h8F^?8DtdY7TK1MK z9hzXrei`lz!1i99@BS($sYH8|eYkKEgc{WGcmXlyN8-%h;2)LeeW0Yb3$blrSLg2l zVB4;!!@}Xpp}^R-HETcg#vMZZ8#N9N5>|ia{pNU&@v~^N@@~i*QaDYvws+c zmJiWJKaSC)iKuk{KFQwk-L`3woZ#80@B?xYQeJ%{>?vUq8{HAAfp zmlTN^eAu6rG33sehdln}7>#9Y%JA5$h-*tM{(BouEG z3OwzaTkiF3c)2>^V#=vri0I1I{bRXG3?l4!@EwWV+^&sCa{Ur~d32bdd7sS$l{D?} zq{cmW_OJ|A#c2|$f>xED4{TFN5sLqTr+1|a$8#HGhAVg5GR;E&I1g3+eLtI!`|vx` zgpEq9aP>qYkyJ^FT4ql0M(2qai|U#Cw?%^qYU*d1*lg$dJevLZ0f2GZSiSJHCLexe zKdFjeNOC+u&HGt<aW13PK~@>mTh4H4}rZ9|7A}S`Cestm-@&HB@v!&HiuzygAu|Un!9r91S(v-tYhP z;f00^eeJcY?4=RNNwGscP5UGJl6VOva#IF3B-$N+dcNq{!K6?nn!E7HT?X}x__iWN zdBc_I3$m(H7zU0^Mcc=X9Nc~q>V3$t8uwMIapf;+7(J|&Dn_IZfJA=aj*rWea?2Zj zQtq3dZlHH8pgjdMHhNLO|Dllwmw#c_K)l%v?3V~-h|iJM#90lZu^D_l1g22s#cl>j z+?S0z zW8>8cH<8FUqRtwz%IVwy4^Lm|PKkIOh23PODC!mCo7P&VJP!#L<=bsNJ?QkiyDb^*=!H zE1@xni#rVpoVXP`=i71jk8R>zvjRR(Mv>~E+)dEw*--is876H&RfpSg0T8l$fY(Q) zK!M~33&Eq4y_mPt7EkBE_tpRX5BuxzAE2FtR!$myPX>+41cuYcQ8>AgwdntKYiiy; z9w68aG^NP>OaSSSk2wnY4WzX23I}Y5dLv3`YM0!tn-yq+U#We`B7k6Cg%g;F%E!E4 zYsExA;>o)+(^BUHKn?-`w3$o`K*#AApq6vhS^esGTDavpXme)UK14*;5K|Q~ikN@* zAnnsZ_P@{9?;ECOF+ecuHkR^Y`J&bC_1{cSW7fzg(2utM^^Z%&jFxJh5RUJ1{2msv z67kDH{6Ba42=eEHha#>YFn2-i?-wthtX$_&N1|HY)mS?l@5`spXovpCDAU^C#*|*m zA2S|G5j&7IFRHeiAWO=~m_ZycQ+__-6Nv~R>996eKX4=m>4E#(3+6{|TBe9eAVs=t zRA>GX_BY|<3l0p#ZuaW9<1(q$R}vgWw}aPjdN%2jEP65kWTj-W4u1S{6i%QNkUxDk zgP9vsAoO^+(+PDshNHOXQxL)|OgMGgzkw7c!Y%p6*nYw|PU%jmv`|$7NEi?<0&&Q` zX&?Pv^nc;m)sSmgvX51~Z)hwa0K?5g$lN{r0#baSkb3>cU4bp7 zQ-;Y@NP(+874Z0gXgj6s?f3>!>;n&IL@_V3II|d8+N=) zRKXj~fV7zE&p$+L2Q~Q0e>eZ1y>j#U#N?yhb@k(td+uY!z2~Fgwc!R$F9t&5tMd?2 zA>mQnze0uzv5*g@_;CLpKoor+^gq4mMO>Ox{#o(ozSrU8^1+nKYijyWWE+BSRSA$Y zT6^C#9;{=eWRt(#J)A@7{GNZmh=UehE%}Hb?k*|Ai7z@uwu zGuC5hJDSwzWssrsqQW(^8>uY1Zi|mL)@(M+$4Vyk_V(BNd`iY`x@%H_EWMV2_d>7* zkIt$7aDQ5i2~iY~#CE$eGkp}=rMY+DF9&pWy=Mi z3Jgg8P5N7gJb4Xz-_W;X5f}0=x>mnOrSryo)OI8}<;>Ou^F%yE*%5a*9uVX~q`7s^L4Fy`D zcY_(Szohn;M;IQzP8VTOI!kKb_lDDsuV`6Yfa6otb->C7Ep+n2Cedbv?FVCpQML6u zSXBkFivI9~K6K0g3j#b8cuNm~aM=GNNx`|Eg9?9qdZ0+GH7a zEyyWSD&^YkA*0x}D7<)$O>HZ+TNUox)h<*s(g^WQ{|P8X{=<7IK`aKIdfGNP#hT4; zEA%ImJm6`+$;j`zTcLl4Hh-?#mYDw%$}CK>nuyErtH*hA@{u$sQ3RP*TP+L=8J|iz zl%-7+Vwi9`?mP_0LGB;MHPyUI$C>O-7|t`Yq#}*1o&o!;L)NQs`?Y6ICPEi!5S78t zqkvExBBQ|`<~}5*d+YMlEf9B^xl?SiS@MSlRpq7~B$&muG8R<~v9p^=YpW@x?H{8XiV(4QML^_WWIRq!Z=th_Ig5aEoIJcgAPD zPLYauhs+`Bog06oav+6m$KyY?#g7$>k^fDJ{0}gtlkbi%{S_%dYHwm4imMr=zypK= zFfe3%|F*;u#x!qKmLwR4V~w z_=zqAZ)@gN^AL~!M+51KgQ0`GO({90zFBVgG;~aaDJv}FW~Tr1Njt*X6$Gj zT^UR$stjz@drhFIPs{0@DL^PMNb@^TmBu)QCxzIxqcWz7Kk~m_t*w9?C2U8 zt`mUcQ2&zGJ7JtOU2~6@P$)`(qkj#-w4}h4Pbus{nviDTzm?wEYH}CaC;vX^Tf(U2 zi7Y9{yDdCclrG==Cs&gwrUFZoXun|-#reG3w-qIDvI>4Tr}m2ux(ZY2CrbVX_7zql z4VHwO!;|`*L|GXW%Z4cDl?}!!UXg!fW=$u^Y8_ZLEu-0Z#M?Skc++vW+myMm>r4{U zOO2yKs)`&A>UTVVa0T#QkzYJ!KgSuwq)Iz?K7cH8FQ-AUUO%*iod2CXqvJa5{u1+uDAiF#M=1h)D(3K1;7`yi zxT&TB00ayNX~pK72*FddYeZ#~_}QVl`>*oVDc)kzt$Z4kdWth4AD6;+)Ar;){i7@w z3iFkS+%zjiLJq+B_IKSKJW=mT?9gY_!5On3i#iftQ%bVJu$d1_xYx)&3&7j#nI#IT z>p$vO@+^VeeZ#mzwXSqmBmC2ozN`Tq}pB8p&{^D9FK_=0ej`+DR*06N|Bt#0x2g*;{+F{e<#=|yu7 z1zhHJM08}xpwFzS`)#pI#{Kw;UPrkq?%%U8fl4rsfkcwM{bI9BxOiBVAs;L-QPm5aLfV3vAN z>Ox+4wzK%{Fy`d?$eyAeT@J8-bJ|AsBmFoH81QUfKd+Z>S?gVS^TBV=zbdnc$@Ko? z4Rs>h@=x_>9aWGstI47+HrJAhifOn@u4xG0P>ok-@sqWx6`%G}X8<=5HaX(lxA?Iy zf?UnoSFCo)sQv4*?8P;zpykhU$ldDsIPF-33Emoq->p2z*r9xR-suW%gOX{^)EQw=(j&slVKNGU<4Z;!G@Hjm1ZG z^em&*q6m}ow=E~%{HBo(ZR_Ern(I(l5)zjnt4^%E_VcWckxbN>r#_ji?aXYX2JHt-h;>rq+r@C2v#jN{P)Lm&!KK79=EORNL zU&yAQ23w<>o+P6oqMpHd(;1tl1*e21WO!_W_(&`j3&!;qJPfQ9LC6fSw4lQ{^uVBC zA3&$7Y&VlyRh1TWT=3PEfJXaAZ|nrfH`6_KO1#ODBR-D2Efd#`_JwT;V#qZNPj zDMG`a#Ush^12!xLD}7RX_p6RRfNS*^T}1;p?`s}~Bf^8P(yUbWo4p;oN1vEq4cY-sL`dOvC5b2K{&saxFT^^_*P6-AsU2V)JKCMJ#h9RvcFm;AOwFQB_k=Dr_}x%0t?Tp76RmUZU6es5-v0&@H}Jclk>Skir_m zM^r^tPaMks85%d}9Flnx60z&bosrQ@E?}Z zgg)}fOhIxsQVwT&0r(I*EBhqwrliGVrIerN`7bH&{77zY1g$U_%c?a}0oxs7=$WGW z%uL;&ng~i;|KV@%MODe13=8&~vNZKKjrOpKzL4`XR%IPc z8xAt-i^hNEA(BhmjX+8ARp~ch7al47+^8o*I|eu>6ccX8IJ6?)g;Ge5sE^~6>>my^ zHJDw}y!n-S=7MBc)O&zg(aW?fAnr ze8fm|Qb1=Z;?O;4L_lh_6`hQvb`G&8XNViDFEEXi)GA$IELWEFpWA6@p$MM0wn(TCdQj=e{mO>f^eKfTGp|3tSp-2&R;clc3MvMiss6w!|0 zZ!_8FejSaew+=0k-@)~XLJ4%|HO569Gtz~{3$z4$X&kQq@bD_WYHRXKI{z;Cgb#}! zx%2IMOx)#j&+1X&Qn5dW-b>{RS!|!k%6ZM<{{X{c05F3tzz3u)prLLstf`Lpi2#VM zrUrf7doM^SW+f|GAR7HFlP{a*E8gd-gTaQ&UGH1<48K^f_6lrUUQ6FFe+qj5mUzo& zzDculNrb@2AK2=A3tqxjuM}QYk35U_6PTA_m_uzBUU{%#N6wV?%M6-~N2^4I7XVx* z8RDwhz_sUk5aA%I>gsD0pUFJd+~;q|=*Zq*@Zn9N911gyoKinR&$rDBLzKsYv<|Jz zy?cj)%uubgniY>*ZyN(w+q0#Dgap%Z4NXt1)- zqbZ|t&@Pk>iG=flxuuDEMBS=vH^vF@Lz@kknk)Z2&19yfKorCdmsxHQ%Fku^!y03u zqSymdvDUb3fAr=5bw z%Deu;LiyU{I%!_r6rnC;tZBMfMnGs~?T8cl|CIIKK~25kw{L*Zr3Iu2VhEAmk=_CV zCUin5hF+vfm)?7C0TiV7CIV6f0cimNY0{)CARtAgcktx^-yd zu4k>!(o`2sJxyT^=5M&rX#5teKqy(JcRVsKFzm6WW)ra4LK-x$)> zo+bT|{se#k%FvYTi_tMZ(24J7^W32-AGnKt>Bh~kYfXnixIN`EK~Kw3t`BBJrM<%3 zGn2jB3V0+O-G?I?!O0X#BRHHP3-4{9{gEW1;E`|=A>D_3kmHT1-9wzgNuOFd*?8{H zpMyJC-$Z#)OKx!s1YxueKa1%goPM&SI`T{RH=a&dQ~DO$TJ}&|ASOidXi51`MB<*jg0BLvpx1x8O)<`0P)#NGaWONJ zfHT;dm}xFB_#oK8{8+GG;jH)fL-_W`zQuyC|DWTFZ&hy!bB5?yyUngw-}}gOaGZMJ zNJqF5IJMx3Wys<`fE?soh_UDY59&_`f*|C3zu0epbHZd&5&!3IoSj`flciR9=^}fX;&gOYfU87Jy3|NeHVI@8Njt)jxR5|1 z!#qQHsdy1rIB_kA{~>3t@f;4&|4=y!1XDdG)SVG6jNpN0>##nIjDS-AEG~1Ag`fn* zQ$vuTR0IxBvtaicL*jv8dn{Zoo)b3Zv>i*Fj}0*b$1p9?@FY;%0!elVzp$5L;GMCM zJ6i&q^3Z=$Lc?`c?zvf6(%?QFR=bOQ}3#&@dNU6LH+Z0ly%Ik()=Xm}II@3nc;+;{PxYti5o zH~a6zykh;)JFz6q7XF(x*bx7wEhMitwsov;{XAivT+r*tDrfsfXdIKaO5B~$|73YdFQ4a>;G(?!bgYGVn>IKb*O&1vM#=wb zB%*Gr&!5HD`~x(7%umJ83L#`DbC?&tO%%YhB&{-VWH^yB4jnKEVaI7OXV2p!R`n!I zax;}wlmq;%C?PgE*jreH8PrX`El%ukh+jH*l1ATzZLtxpw9xxh7dRkzJ0WI-qonLxgJlz-)CF`_6!)z&ff9;_t-j3D|l=w~H z0Yc#%L4N_Mq@8uaur;Pnb_Ii;fwl`ZbYk6$iHTxFw94VPQLIwu{s2Z^@raJ_bS|{Mx=IPH$xR@4@ z#vS@))@Keo{vvyOwR z#Pk+SRfB%t#<*C$BK0J6b|^F7NfM|aIJ3!tApS??Qgiu138Ncp>T23lK_nDvLR4)} z^z%v@HnPd;Jxb$-Jy_DLD4wSFp_8rtu$>yh<^9s z`)E66!ze8&>TWSYM{{FwOE|VnmV}GoZI*a%X_t&xEZ?kz$G!RfQ7yO0FjQ5`BG+^l z1l%gE3>?~yPDF$LK-=5hD&H=LF}TJ=>Pw1o-AxAeHKujiqg=bjpq%VQ!gUCtR6}S4$-G~tUE*-)6?;=rT41~!s~wZz#M+= z1yW)|Nw8;Fg2>U{tDU=vJztc=H1x09UphOxHdgAUAxxM6O!VvJF|bg+R*#glVRdS_^~olA`$J8$ zw1XGv0%+e?kQKr6`CO%vCNkooNo0OSFUhSzD!--dM|_{&kkz)hN?qB$*SmX{?!f$& zpPM-uGT&nXypCyQ_NGm7C2^HI&!P}BtsJDy-AQVblsmK^A$a)rUwFR2@3AiP>ugym zAn6ornnP@JCbrJQ#7|z$gifXTChl8(9YT^kjRS)yGda5F5?1b2;ses;b?*#bHoHGo zLkOtXi^)^?GhC45u9s@Fkgw`^TFEk_U!KUQcB~0dt?nIGfKDGUx?`&yxk<~tcmTaO zglx9wf1gGSYvhg*2#*Mn-A49vbp0i3!2e{bT36=yyu@()i0_RQ=Ea&2CR0lL+QGtd ztniVUq&qmm>czg3&Z|*kBHnOO$9_@NA6wI`*Td@Fk2lR;s842clXH0@>5JYG^=9Nr zThZ0lZ;_3b4rj`vX=3~4ks0*xidMxUXFNe954rp0!-O~U_ksm!qOjuyp(AiyZXI~l zvKB0|B1IL4K1zrZo`lC zY?L_oV@06nJ=8cL|84|EG*%k=d=yWxm%P5+qi{^s(c~qyU7z1$aS^;67Y^qUmD)tS zo-jk*Io&!Z(6@H=FU3}PNL+u@F_GI<`$|%U&;VbvmryY>P)7q_&nAqPmIDwm=EKRv z1iDS{hsngrpgcSSlGYg#b9e1nYJRS}8!WNb(Z6}Z_bF2&vQnib0(=BfUDH}Ghni%J z!tsF2{{S>4%_{JH_c9Y2B`q;uFCim34+`ReWQPX(1wT%j$J4;d6&o(D6)Rv5;21I~ zNGthWxD{B2LTr+_seI9E?wM!<{C#fbZOa*LNvb>X+TW2}lGDfb{koW)5E(2A>IBh% z-j!x)^t8?O0QQ`5JpM|UL>uV!C@h|zKVg)#wKbe&xf(T%j)>pz=i z+C>;+HkVLNV%Z%Xq}qcbFYi6m{9|Q9ZoCy`-EbgOQ!p^bl;7vQl!u#J?JK?GLoEE# zI>!Xc+4>D|s6C$_!(nexS+#>zwe2x^#Vb6r#Fkb5nPUQEWQhc9LJ}N+^l{pQYZH^t zGCv$eRg_0$Y2)y(7yoD~R82jssuG;oh()Gm0yrH3T2}9;d7^she9i{M0|p;&B{r#P#U804?=TnkcsL^BX$C zKB{E#sc_N?87l-H0)#Ym@oeT&oPu)Z0PNT0a~!k9zcB)~)f}oAvlE#zR*#HQq$6|T}yf`D}WL9S6dYS~lSq)Y9 zWs+Ann-bKM@hshi6hpo6DJQIsNO+**d}x@>v|;Q%MND6-msmWntWA5VZtL67MW;88 zc`D>8)kQGe?!jjCC#k%L#at?67DTu?(qi!|%-D2+$-T)k` zcZiIkX{-paB61PpcmE>;4t*VDZPX8?RxIKV;$2sL#MQ@(Dyf3OvcteV64M+vL!E8Z zc*LCinpWH{7MB+T_<_Zrxc-(rkl>;2*{+)9bESB!WBWpHQk#p@Rq`b14=zK(I3<__ z=Auhe^jW|X6E-cJCj>;}e+-9`epCd9at50%4&6sZ@P(qA%RVc$pQ|JYRqv-(=~5#& ztiA)ZW^?992!ajjCxcyeicdYhb*_FY{ac+H~-O(V$4Q67}11rGci zl(S=O9(+JNA9f^O#%(_@bkUvB;G(VA*_V@EINO46cRj3o>kb&n8AgQSr2?oCsU$?o zt5v=C^#|`$s<4RnoIX&Fc|z)gAGT9;EFM}Hy8vU;f~veL)Q-sxC84EX(teJO5X>qh zQ%cxNKihJL*UHGgevM~O;Xy|=oG-Thj_@!^8u=`^08a>GLn2NKg7!4vlV*pyOM7Hj zT%al{ErQoF#2g*D?{j$PPzdCIXy-=w%ycfwFV6OY1(UaF**la?3P37RT?mdzU|fP` zTz{mEXQqW#2HEHChD%iuJEsH8_+MX@D$#X)hFWgLng@&m-VrbkM`i|Qx>O$wo^-LG zv`<6g^JjL4g8LOt)atVt?44OEQh9AX9QmDe?V7JC+jETbR3P7U7dX&bnV+H-Ej=(A z3m}w@QLIdIIsee!uRn+YYQ@7dug)w(c>ma+ef$j*XYCh==d&ZlBsEk0iPqz1F&W@0 zYh^rWEP33{y~^xZ*|T31+on0WE-m6zdtL6tF5do5iFu170OPN2L`}6^*te4zGu!sf zG7GPfcob}p4KUQ9`YdtT*}qG~7r;N#=N*>7?=85Aq7+X9jvLBfx34wirwb{&6{FR< zGN@N;Ebi#O^_%f&pS{KP>1?r(cS*2&RCzv)quqKf~LUt#pH>4!==>53CBnHfRk)|VK0zQg}+cC zC8AT%ZFZBLT=WSq)vf^{3AA+=Ic`8W=*%jTTNmGIP=&bv3grY0Ra#J{6;UpoqZ=Z! zB{tE@MyQJ@3L09i=P{z+h;`^1@E}9|%5occ?8>pKcQmjOjA8>v%>C zGKrQgv6A0?3khBf+Zog%c))q&D(|vB^CQ$ko#x0ymq(WzuMAbe1}4}_Dat%@nKmaf zz`B`IbXt=8_|JmoJI424c|1OY#yl79t(A(V+dY3wJCh+MMvY<&I-@S#7zeT*!fW>x z?fEClo6e#rZ+LV_MOkfz;?+PIg@@OU_u{9G=g0g00&SOb&W!s4c?%4xE5ysny+5>(25E~7 zSeiZ1%XnAw+`+!wp-GPznN~J&bn2t-%zbKZ)h>`c`jZ+Jr#50&$)Y@oR0BPl7t|Rj zo8uG0!v-dvtZwTuZSyu|#Z?wao7*1c%8p3bxgntb+l>I8FXMpvyhQafK6frO zRN~WpY8`|qghYlJ0gkw~_ELHqz4%-DG?_yD87Yn%QHUr!Yt`%%gHIMVqTbMWSWc>p zqz%rc4d&PTrL1T)JRpzeDA@oFLZdi-5je+3mCQ;jr@6o^yIWT|qQ6*QUCu)CxL#+4 z?f?$)yeOp88l~v`@yIegnc-=_6|a8;vH|IZji{f-rbf+ud6JR6A)mJAeblRjZ4yNJ zhKT~vFCoeq3;oY}HP`sVNw~yY<9_-1&7~d7{0yhpItmIL$K^jwRc)T7hE2-7lfu!? zsIx^+#V_%ufYdG73Z%gN#z#_Zi&a1x<&+8~=imM+pCZO^e1H9kta@h6PU(kTVmChF~j11R^`c@M94B<24DJ z9H$>cXN=-+KO|UqSBAa4s|AyiSdfM<&!tLm%^xoHsfAm5+OXoNS#} zu@`0+xCZ(0uN4J2r3she1Q84IX%23*oq&WC@8b^3dxmJUjn9Siy8Ff@w?K}a+f=Zx zvT)$EALcm39jqXcvWx0h*zRCB4yzUq#)qd8_P1PuHnay6}a=5;ayQXcP zL-#<$+koxEtO6W#aK;`S&Q%MUOt%IU`MyST9SJ-^jZ zrhxX|{~@_>J>xFKtgqnZssEf&Roit&p-IMs5X}^ev?WASn@;7@JqSEYvJq}T>XiR!e0yWS?Myq zCG5he#u$!vO_j=0Uk*c^pY|>o<_doS0P~`TSjM@|Q+|=JNp`3cORF?|M-}_ji?c)? z9m_e3KqiEgUp8Zgbe#c@3kql2*Jx94n?2~I;ug{DVrEob6KG71u}iz8|Ku?!mlC2b z#pi5UzV%Gd;{m^u$FOu88XN&dd)IgFA%K?ybV%nSpUIM~6JfqM#9OjBS+=aQtaLPp zq|lL~C+N?32Mp{RCk|Ly+T^s*Jj0i^jDy2HHA$z+zqw(GyJ_q3MhW-MJcs$CRCiH? zv`(h%P6<6Ssy|(9RT=+BcBdVF)Axr9HRnjYbP+a<8|l~ADpwDI9fS#CD}7i1hyw;) zSldg$l$8al=@!Fn!g*ZGbr?N-irlQl_QpSa7WxwsH@BKhTlCk2Tm3T1f3o!3Mw3nW zr%yX+t5$hU4=pgL)6@kzlyIGQM3a=AZM~jwHYR=+yIMbFM4m-}-{x_<5o#V=26ff3 zF1eC9@mdH$1Qjw-*_B>pBYzbK%RB)wh?}vZzU@M*U;e_6vo~qU;rCS2X?t1JFHq4c zxhT44e&E_YgCAcYxMn8jk?kWpfo7Q>dm0Zf2@6l>RcW0Q(#n!2NrdFYQtTfZZf9WR+0B z1vu_!uc?q^g=bWw!pC75oUjq%alti(L+hNgr2I`XF(=zht&WVQg^Y}h`z{t?*{6A6@2TOOk;yvG@Wz4VugA3T^CiH;@s1(Qth|-Y z!@leS=32++0+s3R&=s3-pX^hOoWC!n4V3VW!(b7*-9@YW+pQDy57vb{n;t8FMZ>EXbEa&ih@TUPLV#HnP_3w>y51n_74= zqG*Jku3)Qk*=i%`yj0~hUocL(FFso)iGeMH^Y%5M8c{TZsX6M! zP7F%FRe&2wIu}PRrJTbU>YSj>VTz1Hh|RG5agzGD;>(EBA9L{+5PF) z>>JY+byrwn}W_bmX+uu9#Emz|xf+G`r;%R*n#|E-Q#xr?x|MJaKz$k~{Ls{J%K;EdDd+om_?-3q!?>V% zn306^ThX>Tx65AJ#wucSo3e#ZUe3kebAkGktqX-1JhgIERA_M~X?S||nwm;t-oYN% zO-FCNBB$G!BX2-kfKyfMu}GpVOJOLw5Oam1o{?s5B`h(AlveVW{R7N6s8A}WRRIZ_ zJ9HtgM1=ikj|}8~>2unG&Cye8&hI?eC1yrYnXC|=($-h4juAS~_p#wD+dw-ZQ=Z8+ z0~m&4boeC@4(LgnFgp@r64?Y8JVHwhMEw|`H-p$jF%{;-JYevk?1}w-vca@(To8lI zcte6q*6L!cq({ipiJaZPGO}oyWxs;Yt_s%iuDnxWj_aIT61my-r0FfjE3{T+3neP< zoS9!c*zolN8KjH2)WA6Vd?7ob(1L4~WUjauy+PB8x6zB}WSs@Th$+?BUEa<3Q;p66n@q`ON{F=edD zPS0jyr<J-S~YIzJ=KNC!hM$bA_3S;T>obHFM2Geo9jkeYb!2FzSmX3`1akMnY zjmKt@muupDY(Xwu$Y(9!Bs`6m_q0+%BcZ=sd5AziG@?P$?}M-78t(0lJ5!G*; z59siwko-X|3*?TxAe^tvN{mp0`h}`QTw3LEt1xG@Bh+eeX0OxXre^fU1d9 z-wMpcyic_3Y~_#dBGJOUlOfP`8}XNPH*@3jxBCkv1Xx)S5;A#M&LbVG1q66*Bn}1_ zd%TYb+1_lRd2X0o82=ZYv+|8`x(eZj(cpoTZ9xuzks}vs-V0L+EbNJIg($~8{W15wHZEL z?!26(P7~Dg5janZ0zdhnM0q_&O%wob#~|EC0>t@91yoeeQsn%D&#q}wB-ok4?@^?H z{F_1H+X?J=qoGtd| zjgC1(ye@7?Ib50-lnHI8d8DX7k$~7U6UwQO9%!v%PauupT53Sx2^P*Lx@=6xDUu<; zBU~6{F+&q3q%u{r1sV?fBB@c29OF5uc#3^OoS*Zhgh5yl86S=T1yw)qsfjL+c!mpL z34d?q^;;Q0gbA-^)i-HlS)_GH_3dF6y}yqt?{jD>MHqKdn`bOE)XGYl8f?uAu>?|g->_nR zB4(Nsi8v6zPwGxjYntDF3^F}S)GBL*l5pTP7xsY7)@;v!%nt`dL|!xX&^_^xia7ml z)VF4XQ3T*1kDTh0)AyVj>>K3g5x)w>H8dDkR*^O)NtbvDi0lTJgM~nA`YN-j`uVxP z<=08oBd1Lr_h{L~Ig6S`wEMo6^AX}35v+Cv8f3Gn$Zy-aW7Cc2>Z|JIoEXjgiywHC zrm|YXAJr%;G)&R0e+)~zOUYS$n zX>Z7=P2BKN&!jMPsT`X>3ATL)L}mIlXeAZd1h9?drhE3i<+<;@llwM4+^ZtzuB5l3 z(_Bu9Ye}kUqxrneO&zV0-V{not(d?XC6WmjPXg1rgh+do*R7OA1=OkDw=QZ=9bI(G zQikZR;)g{Ez-GT+epi(ovo9?Xu>@g|Q}J0Tv*j>g7&oUE2}if^TXGa=wNLyLVCeG0aPoXVwVL7J60{kr?hKiSUvIdsu!RQ`u#7Xj`$9vkn~r?`H~6 zxV>h}zd{%Dk8Nhxgkn&(;9ZViIXw~3Y-sr+vlVKn7hh5dF~7Xf{NT1Ycf`y#B;MNe zwpiVzH=>d_KMikquiF$oD1ku<*o+uyWYIvDg+=sY9=GZ)E%+_36=f&G&%$SFGO9k~ z)f(2B7Qb)#a8g0z0!9 z+Drwu7i7@NR8pN}Ju8Y$TPPNLPmFQiwP!E&;jZe=Pp9SDMAPi~8w6|PB5vkRrt%b;QM8R&1pF-k#YC14{Ojz`z&jTOAiv-TWb|H8Gld9NZFUM zArz)D7I7L@2uGvTRU8Q^Fgz7rn_$U5>tPXBiS73AyG zJiC6;8c)A8P*t9BFMOsIyrA9mjG|43HEg#w)Eap;Ve$d5NXOQV6lkJf31d!gX8eLa zBgVMvo6{j{cx(K^Ge0zyQaGTM?}K}&N$@pKO$Mf1;PQi3cQd?W;5D7sa^-Ec1Sm(N zs<7IV)|B$kOlvR4xo#vUY*Jbrb}tgdhPNMitC{E%nb4Y}0R+#fh?rlf5nyAQ(#w>s ztNkU1%dFc~cxj{_dwA^OH?ZCNBjx~6gA3rt53)hqsMPcZ&ExK-wT9##tqHZ>Q`)c= zO`PKGp|X$C=(FTh8?beR1wSsgH8BM95mHLUYE$#5@7iYCqz!9EMRu&Oa z`8E0|I|lC|~0^?*#6MGOK_#R5w(Tl+{gD=_kj4rr7j1vw-> z+mW*T8DT}Gh7D~cUfo)y6bgsHA?vFxKh(`O-5(oJTQj|RE;nhjjSCNMU6LSAZyHsP|6ToMtV&xjy!h6M}fS*tm4eA=`t>oLu)_-PAH(n7CY`z<)n zen05}3XrftDF*2|=Ujq_aRSWk?JdrY#q~@o2Wvi6!Ic(bg43aZ8kENSL6`Xv|M{Zg z^9oNBPK2x{<}PvTl(1Adjho{Vs<6RLj(f~ByNBX9yA(Kqi6vJ2_<9%Tm?AC`7f(u_ zWl6gAdubESweOGKl*%Ib?&`q~q>grF)tx@2u^^=(MrmD}gmWQgd^?OHSZ$x}v^J#5 zZULz=igAE(^?WNqCqhzTD2U%mkpv?JBHgg83@1xe=E?#or(DB7lfV=yNx*aA>sY1H zH7fSQRE`6c42`r~2w5?Sk)n{rXt$6Tb#b8586M7r^ByjP(<-biD&VxZl$aDXK9ZIO z-_?&82nc6kASz6t>0ry!O?wr_0|4NvIIwA}29p*!iPXyaAUYE0L$20_Tbz7$1b1$Q z26raDEPU4e2PY@?W8fI=DCQqmidUE*H$^I4xkVp^nen2JeQ&gkz~Swd_{&se@^R=rD#fqmB;fc)N5~iCdx?L5s_%1wxVx-WnkQ2*{X@Q}?P4hZ zga6NP7ShR?(KXku(45m&;zb}e2dQ*vIv!Y?ZS7+@2?Q5qt^a?>Z;CYIi|0K+5p6wO zHWrm%vF7$){ewq{&iNG!^;4W`7Ebb@OZ~GRs~NE1JqcLwt~6N@Ld@n_fA;7ewX)b< z$sb&;NHphNbr97zBj5A;-~W&cp8x(`cT&y7+5Tg%`aCUvz! z$@HyWbYHo{kCRx1i7qhvR&;@%!sew)BIot^QOHq(Q&-Kr`5lYI?N-4?+?&6we^V2l z-^y?Q0~|!0V(oqy`i$y25!{Ux9Q^|n(Kx&19~$0$kZW>2R$w8&#W^9ff*{zs70#|( z4*nVFZ;9*2qK8}YhaUW79GaH}ReduyCs!m5>=*w4IOfW*obC-7u6q{aU4$N{9<^#5 zR=sa9iCk98(lT1Z3W8Mim1b}&c%q*L}>n}@#tm7(^8=LI2$VW!|1dW~M zzE@8#xY9ZQ0Z7kYhg84W9e*i=xwj9zngYN0fJS631y?%oUl1RCN&W}exu0d&W^R-@ zJ5wun`C-RQXX%KSoz-syakEPK{N2g<+hy)Y8Xt6sOksqCRTW$Z-EF+GzLpLKk1*_Bhm9Rxi zO(mKj&sEKzXlLKvv0?|UjAxdCpVvAj%_I zeT$|shrEADcAZnRm8imu%Q){=O2|UVTZz8^jU$_VF{36>NM1P|fa*Z+zI_Q9Ejcat ztu1%yxs`y~=1Y4zFkT~6YAIZLK(-?Q+}n^gow zOlVTscum`p@rG7fq<+iXC<`75pk*?a6i%qy7UsF>n%I$Y=6Y1~%NmhSe4cS-#N8(1 zuv$tnuo!hg^P#Iuwp_{sJ9xVg@Y!t%hL1K9jc;bH>d^QK%6=B5Ff-_(lM>Bq<>C)L z|4RhOXV=?L(i#w%?cR@y{&~G}*V=zjVfm9?E+$7b;yhvMV&*E(H-jr9_iJbRQ!~t;v>Nevp9YKQ3zY5jE_^Op?TVp;h!+`!+tx3+V(6`jZI8}K#O9H+S(U$I9kjDUfn{FzCPgXX6< z$Z^{5?lNPC4;@A9a#d^t*N**5Yg=+1$Td|hs$(!s;!~PMiWX%QEul&}sdCi6-d9`% zHymHW2&+O_C!+v#0k0CYtx1$ba!O-AJnU1w$SL*Mh@_B_x~r5G{`V$voo&}^<7AqXRBkc+YVLoSUHtmXi@2lmAb}EL zGKm_Y_)h_X77JNrv7Av}3Vl^estN{?N`a)UU6E!(NBt)NvMj(rMzdxKZ5-$9g96%{ zszk>slF8M{?CG3sTDR;DtCLhRf>1}5rUjwKjKacU_IaYn__lY}&D`2IG4NDnMv#EG z{1yX5<+r3~@>djx<#eUX;O1QXZ{MNZW_2s_N6j)>SWz&g z+ev81x}E#x>Q(yTfFa=>F2JchqFS5HZN4HJRO0366Ah*i#)mEtD@LiLm^X2Py0R{^ znrykXUumUD$KX1N9qxz(ht5kWcr^vG`qmMrkWU0F0}WxbMGudnGq6|(ng@FJI7)@& zD7=0vIC$(c$CG!@`D3f#2d%_|% zgJV+TXltB$?D70p=OwnIxInR23Tz-_huWui;%5pI1=o5TvENjPwvL3|z5iZ!&rTtU zy_b{+BLBL}xYBorD@D+E4TIlmSBptpRqO{^{m_|0)?F;lg(!t2nq}U|;o`UbTF9fQAZ@PKMR_j*?i$`{o{#5gh%NVQt#&EQ#OYdp|IJRCRR6o+ZHee=WSMpY zH<6b{s9%2TAiU{w!e!P#$`WP7EzJ(Ys`u+GJ3n{2J^X++w%0xK6&iUl-BxE0){8&8 z$LCxZ-5H%VCc<1pFDUp!7nF{RL7^bLZyDt<4Ztf5r!VxSmP}o7J(1m8T+cBPS$Bx= zHOT?z+OL9rt*^MZ#U;+vhRa~EJP4WEJ1HwMmAKMBAC>&GxxITL{Bypb-CX$}pjOzk zoYSaF6jyp*Pj|eG+nN*}%%al6e<`u`*S-)pq>{$d<$yw#s}~ z;^kPsQ#_7Kb}3K^(n|oH?qBu4G{5dT{s%Y=rx>r>69xrgPZV}gumqoiQtI?F3E9ib zRm%hStNWX!<2}CM81|EzADdC>hhXi(-#Et-@E>5v^B-XBk}mB4nl_O>{WF|f`Ky%% z0|R?n48MF9IF8vloWy?FOzq=8fTNoth4!wS$3e(NNPQ2G-3C(yy)a%H+wS^OfSX6R zAihJh++o^mPW~3*B!4iKO^@AK-q+SE(oH-d>?#iW$#j}CMgr!&#o5D`e$0g6#-2O~ z{!)ycPREbCQrB%)4eF=kJCCmQPqmM5D?xuhPiOCl7*Rw8N?C++_re<3Pn~gSNxL1q zQ$}!DVO~Jax#!kD04V7S_p$@WguiO{d?6s%ME(qpOaAz`Ji1Ol)*)UaWsOXve){?D zj*~1Wxnv7STCAp#jYDu8l-{~VnRf(yR$}YB+jz8ztLrFicaFwR{J@=p;V~B{?vF13 zBU1@go4$uu?M%`2?9%&AR+C?`b??n}TgVaTErcapT^))EG|s+D`%X%g@^Ais0oZ%Q Ar2qf` literal 0 HcmV?d00001 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/local_example_chat.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/local_example_chat.py new file mode 100644 index 00000000..e1e4b62c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/local_example_chat.py @@ -0,0 +1,75 @@ +""" +Usage: +python3 local_example_chat.py +""" + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print() + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + runtime = sgl.Runtime(model_path="meta-llama/Llama-2-7b-chat-hf") + sgl.set_default_backend(runtime) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() + + runtime.shutdown() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/local_example_complete.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/local_example_complete.py new file mode 100644 index 00000000..00a451cf --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/local_example_complete.py @@ -0,0 +1,70 @@ +""" +Usage: +python3 local_example_complete.py +""" + +import sglang as sgl + + +@sgl.function +def few_shot_qa(s, question): + s += """The following are questions with answers. +Q: What is the capital of France? +A: Paris +Q: What is the capital of Germany? +A: Berlin +Q: What is the capital of Italy? +A: Rome +""" + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n", temperature=0) + + +def single(): + state = few_shot_qa.run(question="What is the capital of the United States?") + answer = state["answer"].strip().lower() + + assert "washington" in answer, f"answer: {state['answer']}" + + print(state.text()) + + +def stream(): + state = few_shot_qa.run( + question="What is the capital of the United States?", stream=True + ) + + for out in state.text_iter("answer"): + print(out, end="", flush=True) + print() + + +def batch(): + states = few_shot_qa.run_batch( + [ + {"question": "What is the capital of the United States?"}, + {"question": "What is the capital of China?"}, + ] + ) + + for s in states: + print(s["answer"]) + + +if __name__ == "__main__": + runtime = sgl.Runtime(model_path="meta-llama/Llama-2-7b-chat-hf") + sgl.set_default_backend(runtime) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() + + runtime.shutdown() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/local_example_llava_next.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/local_example_llava_next.py new file mode 100644 index 00000000..c941a549 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/local_example_llava_next.py @@ -0,0 +1,78 @@ +""" +Usage: python3 local_example_llava_next.py +""" + +import sglang as sgl +from sglang.lang.chat_template import get_chat_template + + +@sgl.function +def image_qa(s, image_path, question): + s += sgl.user(sgl.image(image_path) + question) + s += sgl.assistant(sgl.gen("answer")) + + +def single(): + state = image_qa.run( + image_path="images/cat.jpeg", question="What is this?", max_new_tokens=128 + ) + print(state["answer"], "\n") + + +def stream(): + state = image_qa.run( + image_path="images/cat.jpeg", + question="What is this?", + max_new_tokens=64, + stream=True, + ) + + for out in state.text_iter("answer"): + print(out, end="", flush=True) + print() + + +def batch(): + states = image_qa.run_batch( + [ + {"image_path": "images/cat.jpeg", "question": "What is this?"}, + {"image_path": "images/dog.jpeg", "question": "What is this?"}, + ], + max_new_tokens=128, + ) + for s in states: + print(s["answer"], "\n") + + +if __name__ == "__main__": + import multiprocessing as mp + + mp.set_start_method("spawn", force=True) + + runtime = sgl.Runtime(model_path="lmms-lab/llama3-llava-next-8b") + runtime.endpoint.chat_template = get_chat_template("llama-3-instruct-llava") + + # Or you can use the 72B model + # runtime = sgl.Runtime(model_path="lmms-lab/llava-next-72b", tp_size=8) + # runtime.endpoint.chat_template = get_chat_template("chatml-llava") + + sgl.set_default_backend(runtime) + print(f"chat template: {runtime.endpoint.chat_template.name}") + + # Or you can use API models + # sgl.set_default_backend(sgl.OpenAI("gpt-4-vision-preview")) + # sgl.set_default_backend(sgl.VertexAI("gemini-pro-vision")) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() + + runtime.shutdown() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/openai_example_chat.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/openai_example_chat.py new file mode 100644 index 00000000..9511e21c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/openai_example_chat.py @@ -0,0 +1,74 @@ +""" +Usage: +export OPENAI_API_KEY=sk-****** +python3 openai_example_chat.py +""" + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print() + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.OpenAI("gpt-3.5-turbo")) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/openai_example_complete.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/openai_example_complete.py new file mode 100644 index 00000000..d64bcaf1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/openai_example_complete.py @@ -0,0 +1,68 @@ +""" +Usage: +export OPENAI_API_KEY=sk-****** +python3 openai_example_complete.py +""" + +import sglang as sgl + + +@sgl.function +def few_shot_qa(s, question): + s += """The following are questions with answers. +Q: What is the capital of France? +A: Paris +Q: What is the capital of Germany? +A: Berlin +Q: What is the capital of Italy? +A: Rome +""" + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n", temperature=0) + + +def single(): + state = few_shot_qa.run(question="What is the capital of the United States?") + answer = state["answer"].strip().lower() + + assert "washington" in answer, f"answer: {state['answer']}" + + print(state.text()) + + +def stream(): + state = few_shot_qa.run( + question="What is the capital of the United States?", stream=True + ) + + for out in state.text_iter("answer"): + print(out, end="", flush=True) + print() + + +def batch(): + states = few_shot_qa.run_batch( + [ + {"question": "What is the capital of the United States?"}, + {"question": "What is the capital of China?"}, + ] + ) + + for s in states: + print(s["answer"]) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.OpenAI("gpt-3.5-turbo-instruct")) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/openai_example_n.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/openai_example_n.py new file mode 100644 index 00000000..25372b9f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/openai_example_n.py @@ -0,0 +1,71 @@ +""" +Usage: +export OPENAI_API_KEY=sk-****** +python3 openai_example_chat.py +""" + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=1024, n=2)) + s += sgl.user(question_2) + s += sgl.assistant( + sgl.gen( + "answer_2", + max_tokens=1024, + ) + ) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + print("\n-- answer_2 --\n", state["answer_2"]) + assert isinstance(state["answer_1"], list) + assert len(state["answer_1"]) == 2 + assert isinstance(state["answer_2"], str) + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + print("\n-- answer_1 --\n", s["answer_1"]) + print("\n-- answer_2 --\n", s["answer_2"]) + assert isinstance(s["answer_1"], list) + assert len(s["answer_1"]) == 2 + assert isinstance(s["answer_2"], str) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.OpenAI("o1")) + + # Run a single request + print("\n========== single ==========\n") + single() + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/openai_example_o1.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/openai_example_o1.py new file mode 100644 index 00000000..2e5c1400 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/openai_example_o1.py @@ -0,0 +1,57 @@ +""" +Usage: +export OPENAI_API_KEY=sk-****** +python3 openai_example_chat.py +""" + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=100)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2")) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.OpenAI("o1")) + + # Run a single request + print("\n========== single ==========\n") + single() + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/openrouter_example_chat.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/openrouter_example_chat.py new file mode 100644 index 00000000..a0b6f15b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/openrouter_example_chat.py @@ -0,0 +1,81 @@ +""" +Usage: +export OPENROUTER_API_KEY=sk-****** +python3 together_example_chat.py +""" + +import os + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print() + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + backend = sgl.OpenAI( + model_name="google/gemma-7b-it:free", + base_url="https://openrouter.ai/api/v1", + api_key=os.environ.get("OPENROUTER_API_KEY"), + ) + sgl.set_default_backend(backend) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/together_example_chat.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/together_example_chat.py new file mode 100644 index 00000000..2d205906 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/together_example_chat.py @@ -0,0 +1,81 @@ +""" +Usage: +export TOGETHER_API_KEY=sk-****** +python3 together_example_chat.py +""" + +import os + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print() + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + backend = sgl.OpenAI( + model_name="mistralai/Mixtral-8x7B-Instruct-v0.1", + base_url="https://api.together.xyz/v1", + api_key=os.environ.get("TOGETHER_API_KEY"), + ) + sgl.set_default_backend(backend) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/together_example_complete.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/together_example_complete.py new file mode 100644 index 00000000..d9119ed6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/quick_start/together_example_complete.py @@ -0,0 +1,76 @@ +""" +Usage: +export TOGETHER_API_KEY=sk-****** +python3 together_example_complete.py +""" + +import os + +import sglang as sgl + + +@sgl.function +def few_shot_qa(s, question): + s += """The following are questions with answers. +Q: What is the capital of France? +A: Paris +Q: What is the capital of Germany? +A: Berlin +Q: What is the capital of Italy? +A: Rome +""" + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n", temperature=0) + + +def single(): + state = few_shot_qa.run(question="What is the capital of the United States?") + answer = state["answer"].strip().lower() + + assert "washington" in answer, f"answer: {state['answer']}" + + print(state.text()) + + +def stream(): + state = few_shot_qa.run( + question="What is the capital of the United States?", stream=True + ) + + for out in state.text_iter("answer"): + print(out, end="", flush=True) + print() + + +def batch(): + states = few_shot_qa.run_batch( + [ + {"question": "What is the capital of the United States?"}, + {"question": "What is the capital of China?"}, + ] + ) + + for s in states: + print(s["answer"]) + + +if __name__ == "__main__": + backend = sgl.OpenAI( + model_name="mistralai/Mixtral-8x7B-Instruct-v0.1", + is_chat_model=False, + base_url="https://api.together.xyz/v1", + api_key=os.environ.get("TOGETHER_API_KEY"), + ) + sgl.set_default_backend(backend) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/chinese_regex.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/chinese_regex.py new file mode 100644 index 00000000..78e9c7e1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/chinese_regex.py @@ -0,0 +1,53 @@ +import sglang as sgl + +character_regex = ( + r"""\{\n""" + + r""" "姓名": "[^"]{1,32}",\n""" + + r""" "学院": "(格兰芬多|赫奇帕奇|拉文克劳|斯莱特林)",\n""" + + r""" "血型": "(纯血|混血|麻瓜)",\n""" + + r""" "职业": "(学生|教师|傲罗|魔法部|食死徒|凤凰社成员)",\n""" + + r""" "魔杖": \{\n""" + + r""" "材质": "[^"]{1,32}",\n""" + + r""" "杖芯": "[^"]{1,32}",\n""" + + r""" "长度": [0-9]{1,2}\.[0-9]{0,2}\n""" + + r""" \},\n""" + + r""" "存活": "(存活|死亡)",\n""" + + r""" "守护神": "[^"]{1,32}",\n""" + + r""" "博格特": "[^"]{1,32}"\n""" + + r"""\}""" +) + + +@sgl.function +def character_gen(s, name): + s += name + " 是一名哈利波特系列小说中的角色。请填写以下关于这个角色的信息。" + s += """\ +这是一个例子 +{ + "姓名": "哈利波特", + "学院": "格兰芬多", + "血型": "混血", + "职业": "学生", + "魔杖": { + "材质": "冬青木", + "杖芯": "凤凰尾羽", + "长度": 11.0 + }, + "存活": "存活", + "守护神": "麋鹿", + "博格特": "摄魂怪" +} +""" + s += f"现在请你填写{name}的信息：\n" + s += sgl.gen("json_output", max_tokens=256, regex=character_regex) + + +def main(): + backend = sgl.RuntimeEndpoint("http://localhost:30000") + sgl.set_default_backend(backend) + ret = character_gen.run(name="赫敏格兰杰", temperature=0) + print(ret.text()) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/choices_logprob.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/choices_logprob.py new file mode 100644 index 00000000..6cd733fe --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/choices_logprob.py @@ -0,0 +1,44 @@ +""" +Usage: +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +python choices_logprob.py +""" + +import sglang as sgl + + +@sgl.function +def tool_use(s, question): + s += "To answer this question: " + question + ", " + s += "I need to use a " + sgl.gen("tool", choices=["calculator", "search engine"]) + + +def main(): + # Run one case + question = "What is 5 + 5?" + state = tool_use.run(question) + print("questions:", question) + print("choice:", state["tool"]) + meta_info = state.get_meta_info("tool") + print("logprobs of choice 1", meta_info["input_token_logprobs"][0]) + print("logprobs of choice 2", meta_info["input_token_logprobs"][1]) + print("-" * 50) + + # Run a batch + questions = [ + "What is 5 + 6?", + "Who is Michael Jordan?", + ] + states = tool_use.run_batch([{"question": q} for q in questions]) + for question, state in zip(questions, states): + print("questions:", question) + print("choice:", state["tool"]) + meta_info = state.get_meta_info("tool") + print("logprobs of choice 1", meta_info["input_token_logprobs"][0]) + print("logprobs of choice 2", meta_info["input_token_logprobs"][1]) + print("-" * 50) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.RuntimeEndpoint("http://localhost:30000")) + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/cot_decoding.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/cot_decoding.py new file mode 100644 index 00000000..7a7a04bc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/cot_decoding.py @@ -0,0 +1,115 @@ +from math import exp +from pprint import pformat + +import sglang as sgl + +YELLOW = "\033[1;33m" +GREEN = "\033[1;32m" +BLUE = "\033[1;34m" +CLEAR = "\033[1;0m" + + +@sgl.function +def cot_decoding(s, question, get_top_k, is_chat_model, verbose): + """CoT Decoding: http://arxiv.org/abs/2402.10200""" + + if is_chat_model: + s += sgl.user("Question: " + question + "\nAnswer:") + s += sgl.assistant_begin() + else: + s += "Question: " + question + "\nAnswer:" + + step_0 = s.fork(1)[0] + forks = s.fork(get_top_k) + answer_forks = s.fork(get_top_k) + + # decoding step 0 + step_0 += sgl.gen( + "get_top_k", + max_tokens=0, + return_logprob=True, + top_logprobs_num=get_top_k, + return_text_in_logprobs=True, + ) + logprobs = step_0.get_meta_info("get_top_k")["output_top_logprobs"][0] + + print("Decoding step 0:", ", ".join(pformat(token[2]) for token in logprobs)) + for idx, (f, token) in enumerate(zip(forks, logprobs)): + logprob, token_id, text = token + f += text + + if text == "<|end_of_text|>": + print( + f"{YELLOW}Path #{idx} {pformat(text)}[{exp(logprob):.3f}] (score=nan, answer=nan){CLEAR}" + ) + continue + + # continue greedy decoding + f += sgl.gen( + "answer", + temperature=0, + max_tokens=1024, + return_logprob=True, + top_logprobs_num=2, + return_text_in_logprobs=True, + ) + + # calculate probability disparity between the top and secondary tokens + x1s = [exp(xt[0][0]) for xt in f.get_meta_info("answer")["output_top_logprobs"]] + x2s = [exp(xt[1][0]) for xt in f.get_meta_info("answer")["output_top_logprobs"]] + tokens = [xt[0][2] for xt in f.get_meta_info("answer")["output_top_logprobs"]] + delta = (sum(x1s) - sum(x2s)) / len(x1s) + + # extract the answer span (without the '<|end_of_text|>' token) + answer_forks[idx] += text + f["answer"] + "\nSo the answer is" + answer_forks[idx] += sgl.gen( + "answer_span", + temperature=0, + max_tokens=64, + return_logprob=True, + top_logprobs_num=2, + return_text_in_logprobs=True, + ) + answer = answer_forks[idx]["answer_span"].replace("\n", " ").strip(":") + print( + f"{YELLOW}Path #{idx} {pformat(text)}[{exp(logprob):.3f}] (score={delta}, answer={answer}){CLEAR}" + ) + generated_text = str(answer_forks[idx])[len("ProgramState(") : -1] + print(f"{BLUE}{pformat(generated_text)}{CLEAR}") + + if verbose: + answer_tokens = [ + xt[0][2] + for xt in answer_forks[idx].get_meta_info("answer_span")[ + "output_top_logprobs" + ] + ] + answer_x1s = [ + exp(xt[0][0]) + for xt in answer_forks[idx].get_meta_info("answer_span")[ + "output_top_logprobs" + ] + ] + answer_x2s = [ + exp(xt[1][0]) + for xt in answer_forks[idx].get_meta_info("answer_span")[ + "output_top_logprobs" + ] + ] + + for token, x1, x2 in zip(tokens, x1s, x2s): + print(f" {GREEN}{pformat(token)}{CLEAR}({x1:.3f}-{x2:.3f})", end="") + print("\n===========") + for token, x1, x2 in zip(answer_tokens, answer_x1s, answer_x2s): + print(f" {GREEN}{pformat(token)}{CLEAR}({x1:.3f}-{x2:.3f})", end="") + print() + + +sgl.set_default_backend(sgl.RuntimeEndpoint("http://localhost:30000")) + +state = cot_decoding.run( + question=r"Claire makes a 3 egg omelet every morning for breakfast. How many dozens of eggs will she eat in 4 weeks?", + get_top_k=10, + is_chat_model=True, + verbose=False, +) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/json_decode.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/json_decode.py new file mode 100644 index 00000000..5dc3522d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/json_decode.py @@ -0,0 +1,83 @@ +""" +Usage: +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +python json_decode.py +""" + +from enum import Enum + +from pydantic import BaseModel + +import sglang as sgl +from sglang.srt.constrained.outlines_backend import build_regex_from_object + +character_regex = ( + r"""\{\n""" + + r""" "name": "[\w\d\s]{1,16}",\n""" + + r""" "house": "(Gryffindor|Slytherin|Ravenclaw|Hufflepuff)",\n""" + + r""" "blood status": "(Pure-blood|Half-blood|Muggle-born)",\n""" + + r""" "occupation": "(student|teacher|auror|ministry of magic|death eater|order of the phoenix)",\n""" + + r""" "wand": \{\n""" + + r""" "wood": "[\w\d\s]{1,16}",\n""" + + r""" "core": "[\w\d\s]{1,16}",\n""" + + r""" "length": [0-9]{1,2}\.[0-9]{0,2}\n""" + + r""" \},\n""" + + r""" "alive": "(Alive|Deceased)",\n""" + + r""" "patronus": "[\w\d\s]{1,16}",\n""" + + r""" "bogart": "[\w\d\s]{1,16}"\n""" + + r"""\}""" +) + + +@sgl.function +def character_gen(s, name): + s += ( + name + + " is a character in Harry Potter. Please fill in the following information about this character.\n" + ) + s += "The constrained regex is:\n" + s += character_regex + "\n" + s += "The JSON output is:\n" + s += sgl.gen("json_output", max_tokens=256, regex=character_regex) + + +def driver_character_gen(): + state = character_gen.run(name="Hermione Granger") + print(state.text()) + + +class Weapon(str, Enum): + sword = "sword" + axe = "axe" + mace = "mace" + spear = "spear" + bow = "bow" + crossbow = "crossbow" + + +class Wizard(BaseModel): + name: str + age: int + weapon: Weapon + + +@sgl.function +def pydantic_wizard_gen(s): + s += "Give me a description about a wizard in the JSON format.\n" + s += sgl.gen( + "character", + max_tokens=128, + temperature=0, + regex=build_regex_from_object(Wizard), # Requires pydantic >= 2.0 + ) + + +def driver_pydantic_wizard_gen(): + state = pydantic_wizard_gen.run() + print(state.text()) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.RuntimeEndpoint("http://localhost:30000")) + driver_character_gen() + # driver_pydantic_wizard_gen() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/json_logprobs.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/json_logprobs.py new file mode 100644 index 00000000..15206a61 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/json_logprobs.py @@ -0,0 +1,103 @@ +# NOTE: Currently this can only be run through HTTP requests. +from concurrent.futures import ThreadPoolExecutor + +from json_decode import character_regex + +from sglang.utils import http_request + +character_names = ["Hermione Granger", "Ron Weasley", "Harry Potter"] + +base_url = "http://localhost:30000" + +prompt = "is a character in Harry Potter. Please fill in the following information about this character.\n" + + +def openai_api_request(name): + data = { + "model": "", + "prompt": name + prompt, + "temperature": 0, + "max_tokens": 128, + "regex": character_regex, + "logprobs": 3, + } + res = http_request(base_url + "/v1/completions", json=data).json() + + # with open(f"json_logprobs_{name.replace(' ', '_')}_tmp.json", "w") as fout: + # fout.write(json.dumps(res, indent=4)) + + logprobs = res["choices"][0]["logprobs"] + usage = res["usage"] + assert len(logprobs["token_logprobs"]) == len(logprobs["tokens"]) + assert len(logprobs["token_logprobs"]) == len(logprobs["top_logprobs"]) + assert len(logprobs["token_logprobs"]) == usage["completion_tokens"] - 1 + + return res + + +def srt_api_request(name): + data = { + "text": name + prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 128, + "regex": character_regex, + }, + "return_logprob": True, + "logprob_start_len": 0, + "top_logprobs_num": 3, + "return_text_in_logprobs": True, + } + + res = http_request(base_url + "/generate", json=data).json() + + # with open(f"json_logprobs_{name.replace(' ', '_')}_tmp.json", "w") as fout: + # fout.write(json.dumps(res, indent=4)) + + meta_info = res["meta_info"] + assert len(meta_info["input_token_logprobs"]) == len( + meta_info["input_top_logprobs"] + ) + assert len(meta_info["output_token_logprobs"]) == len( + meta_info["output_top_logprobs"] + ) + assert len(meta_info["input_token_logprobs"]) == meta_info["prompt_tokens"] + assert len(meta_info["output_token_logprobs"]) == meta_info["completion_tokens"] - 1 + + return res + + +def pretty_print(res): + meta_info = res["meta_info"] + + print("\n\n", "=" * 30, "Prefill", "=" * 30) + for i in range(len(meta_info["input_token_logprobs"])): + print(f"{str(meta_info['input_token_logprobs'][i][2].encode()): <20}", end="") + top_ks = ( + [str(t[2].encode()) for t in meta_info["input_top_logprobs"][i]] + if meta_info["input_top_logprobs"][i] + else [] + ) + for top_k in top_ks: + print(f"{top_k: <15}", end="") + print() + + print("\n\n", "=" * 30, "Decode", "=" * 30) + for i in range(len(meta_info["output_token_logprobs"])): + print(f"{str(meta_info['output_token_logprobs'][i][2].encode()): <20}", end="") + top_ks = [str(t[2].encode()) for t in meta_info["output_top_logprobs"][i]] + for top_k in top_ks: + print(f"{top_k: <15}", end="") + print() + + print(res["text"]) + + +if __name__ == "__main__": + with ThreadPoolExecutor() as executor: + ress = executor.map(srt_api_request, character_names) + + for res in ress: + pretty_print(res) + + openai_api_request("Hermione Granger") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/llava_video/srt_example_llava_v.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/llava_video/srt_example_llava_v.py new file mode 100644 index 00000000..ec5b334b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/llava_video/srt_example_llava_v.py @@ -0,0 +1,260 @@ +""" +Usage: +pip install opencv-python-headless + +python3 srt_example_llava_v.py +""" + +import argparse +import csv +import json +import os +import time + +import requests + +import sglang as sgl + + +@sgl.function +def video_qa(s, num_frames, video_path, question): + s += sgl.user(sgl.video(video_path, num_frames) + question) + s += sgl.assistant(sgl.gen("answer")) + + +def single(path, num_frames=16): + state = video_qa.run( + num_frames=num_frames, + video_path=path, + question="Please provide a detailed description of the video, focusing on the main subjects, their actions, the background scenes", + temperature=0.0, + max_new_tokens=1024, + ) + print(state["answer"], "\n") + + +def split_into_chunks(lst, num_chunks): + """Split a list into a specified number of chunks.""" + # Calculate the chunk size using integer division. Note that this may drop some items if not evenly divisible. + chunk_size = len(lst) // num_chunks + + if chunk_size == 0: + chunk_size = len(lst) + # Use list comprehension to generate chunks. The last chunk will take any remainder if the list size isn't evenly divisible. + chunks = [lst[i : i + chunk_size] for i in range(0, len(lst), chunk_size)] + # Ensure we have exactly num_chunks chunks, even if some are empty + chunks.extend([[] for _ in range(num_chunks - len(chunks))]) + return chunks + + +def save_batch_results(batch_video_files, states, cur_chunk, batch_idx, save_dir): + csv_filename = f"{save_dir}/chunk_{cur_chunk}_batch_{batch_idx}.csv" + with open(csv_filename, "w", newline="") as csvfile: + writer = csv.writer(csvfile) + writer.writerow(["video_name", "answer"]) + for video_path, state in zip(batch_video_files, states): + video_name = os.path.basename(video_path) + writer.writerow([video_name, state["answer"]]) + + +def compile_and_cleanup_final_results(cur_chunk, num_batches, save_dir): + final_csv_filename = f"{save_dir}/final_results_chunk_{cur_chunk}.csv" + with open(final_csv_filename, "w", newline="") as final_csvfile: + writer = csv.writer(final_csvfile) + writer.writerow(["video_name", "answer"]) + for batch_idx in range(num_batches): + batch_csv_filename = f"{save_dir}/chunk_{cur_chunk}_batch_{batch_idx}.csv" + with open(batch_csv_filename, "r") as batch_csvfile: + reader = csv.reader(batch_csvfile) + next(reader) # Skip header row + for row in reader: + writer.writerow(row) + os.remove(batch_csv_filename) + + +def find_video_files(video_dir): + # Check if the video_dir is actually a file + if os.path.isfile(video_dir): + # If it's a file, return it as a single-element list + return [video_dir] + + # Original logic to find video files in a directory + video_files = [] + for root, dirs, files in os.walk(video_dir): + for file in files: + if file.endswith((".mp4", ".avi", ".mov")): + video_files.append(os.path.join(root, file)) + return video_files + + +def batch(video_dir, save_dir, cur_chunk, num_chunks, num_frames=16, batch_size=64): + video_files = find_video_files(video_dir) + chunked_video_files = split_into_chunks(video_files, num_chunks)[cur_chunk] + num_batches = 0 + + for i in range(0, len(chunked_video_files), batch_size): + batch_video_files = chunked_video_files[i : i + batch_size] + print(f"Processing batch of {len(batch_video_files)} video(s)...") + + if not batch_video_files: + print("No video files found in the specified directory.") + return + + batch_input = [ + { + "num_frames": num_frames, + "video_path": video_path, + "question": "Please provide a detailed description of the video, focusing on the main subjects, their actions, the background scenes.", + } + for video_path in batch_video_files + ] + + start_time = time.perf_counter() + states = video_qa.run_batch(batch_input, max_new_tokens=512, temperature=0.2) + total_time = time.perf_counter() - start_time + average_time = total_time / len(batch_video_files) + print( + f"Number of videos in batch: {len(batch_video_files)}. Average processing time per video: {average_time:.2f} seconds. Total time for this batch: {total_time:.2f} seconds" + ) + + save_batch_results(batch_video_files, states, cur_chunk, num_batches, save_dir) + num_batches += 1 + + compile_and_cleanup_final_results(cur_chunk, num_batches, save_dir) + + +if __name__ == "__main__": + + url = "https://raw.githubusercontent.com/EvolvingLMMs-Lab/sglang/dev/onevision_local/assets/jobs.mp4" + + cache_dir = os.path.expanduser("~/.cache") + file_path = os.path.join(cache_dir, "jobs.mp4") + + os.makedirs(cache_dir, exist_ok=True) + + response = requests.get(url) + response.raise_for_status() # Raise an exception for bad responses + + with open(file_path, "wb") as f: + f.write(response.content) + + print(f"File downloaded and saved to: {file_path}") + # Create the parser + parser = argparse.ArgumentParser( + description="Run video processing with specified port." + ) + + # Add an argument for the port + parser.add_argument( + "--port", + type=int, + default=30000, + help="The master port for distributed serving.", + ) + parser.add_argument( + "--chunk-idx", type=int, default=0, help="The index of the chunk to process." + ) + parser.add_argument( + "--num-chunks", type=int, default=8, help="The number of chunks to process." + ) + parser.add_argument( + "--save-dir", + type=str, + default="./work_dirs/llava_video", + help="The directory to save the processed video files.", + ) + parser.add_argument( + "--video-dir", + type=str, + default=os.path.expanduser("~/.cache/jobs.mp4"), + help="The directory or path for the processed video files.", + ) + parser.add_argument( + "--model-path", + type=str, + default="lmms-lab/LLaVA-NeXT-Video-7B", + help="The model path for the video processing.", + ) + parser.add_argument( + "--num-frames", + type=int, + default=16, + help="The number of frames to process in each video.", + ) + parser.add_argument("--mm_spatial_pool_stride", type=int, default=2) + + # Parse the arguments + args = parser.parse_args() + cur_port = args.port + cur_chunk = args.chunk_idx + num_chunks = args.num_chunks + num_frames = args.num_frames + + if "34b" in args.model_path.lower(): + tokenizer_path = "liuhaotian/llava-v1.6-34b-tokenizer" + elif "7b" in args.model_path.lower(): + tokenizer_path = "llava-hf/llava-1.5-7b-hf" + else: + print("Invalid model path. Please specify a valid model path.") + exit() + + model_override_args = {} + model_override_args["mm_spatial_pool_stride"] = args.mm_spatial_pool_stride + model_override_args["architectures"] = ["LlavaVidForCausalLM"] + model_override_args["num_frames"] = args.num_frames + model_override_args["model_type"] = "llava" + + if "34b" in args.model_path.lower(): + model_override_args["image_token_index"] = 64002 + + if args.num_frames == 32: + model_override_args["rope_scaling"] = {"factor": 2.0, "rope_type": "linear"} + model_override_args["max_sequence_length"] = 4096 * 2 + model_override_args["tokenizer_model_max_length"] = 4096 * 2 + elif args.num_frames < 32: + pass + else: + print( + "The maximum number of frames to process is 32. Please specify a valid number of frames." + ) + exit() + + runtime = sgl.Runtime( + model_path=args.model_path, # "liuhaotian/llava-v1.6-vicuna-7b", + tokenizer_path=tokenizer_path, + port=cur_port, + json_model_override_args=json.dumps(model_override_args), + tp_size=1, + ) + sgl.set_default_backend(runtime) + print(f"chat template: {runtime.endpoint.chat_template.name}") + + # Run a single request + print("\n========== single ==========\n") + root = args.video_dir + if os.path.isfile(root): + video_files = [root] + else: + video_files = [ + os.path.join(root, f) + for f in os.listdir(root) + if f.endswith((".mp4", ".avi", ".mov")) + ] # Add more extensions if needed + start_time = time.perf_counter() # Start time for processing a single video + for cur_video in video_files[:1]: + print(cur_video) + single(cur_video, num_frames) + end_time = time.perf_counter() # End time for processing a single video + total_time = end_time - start_time + average_time = total_time / len( + video_files + ) # Calculate the average processing time + print(f"Average processing time per video: {average_time:.2f} seconds") + runtime.shutdown() + + # # Run a batch of requests + # print("\n========== batch ==========\n") + # if not os.path.exists(args.save_dir): + # os.makedirs(args.save_dir) + # batch(args.video_dir, args.save_dir, cur_chunk, num_chunks, num_frames, num_chunks) + # runtime.shutdown() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/llava_video/srt_example_llava_v.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/llava_video/srt_example_llava_v.sh new file mode 100755 index 00000000..ffb1af96 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/llava_video/srt_example_llava_v.sh @@ -0,0 +1,131 @@ +#!/bin/bash + +##### USAGE ##### +# - First node: +# ```sh +# bash examples/usage/llava_video/srt_example_llava_v.sh K 0 YOUR_VIDEO_PATH YOUR_MODEL_PATH FRAMES_PER_VIDEO +# ``` +# - Second node: +# ```sh +# bash examples/usage/llava_video/srt_example_llava_v.sh K 1 YOUR_VIDEO_PATH YOUR_MODEL_PATH FRAMES_PER_VIDEO +# ``` +# - The K node: +# ```sh +# bash examples/usage/llava_video/srt_example_llava_v.sh K K-1 YOUR_VIDEO_PATH YOUR_MODEL_PATH FRAMES_PER_VIDEO +# ``` + + +# Replace `K`, `YOUR_VIDEO_PATH`, `YOUR_MODEL_PATH`, and `FRAMES_PER_VIDEO` with your specific details. +# CURRENT_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +CURRENT_ROOT=$(dirname "$0") + +echo ${CURRENT_ROOT} + +cd ${CURRENT_ROOT} + +export PYTHONWARNINGS=ignore + +START_TIME=$(date +%s) # Capture start time + +NUM_NODES=$1 + +CUR_NODES_IDX=$2 + +VIDEO_DIR=$3 + +MODEL_PATH=$4 + +NUM_FRAMES=$5 + + +# FRAME_FORMAT=$6 + +# FRAME_FORMAT=$(echo $FRAME_FORMAT | tr '[:lower:]' '[:upper:]') + +# # Check if FRAME_FORMAT is either JPEG or PNG +# if [[ "$FRAME_FORMAT" != "JPEG" && "$FRAME_FORMAT" != "PNG" ]]; then +# echo "Error: FRAME_FORMAT must be either JPEG or PNG." +# exit 1 +# fi + +# export TARGET_FRAMES=$TARGET_FRAMES + +echo "Each video you will sample $NUM_FRAMES frames" + +# export FRAME_FORMAT=$FRAME_FORMAT + +# echo "The frame format is $FRAME_FORMAT" + +# Assuming GPULIST is a bash array containing your GPUs +GPULIST=(0 1 2 3 4 5 6 7) +LOCAL_CHUNKS=${#GPULIST[@]} + +echo "Number of GPUs in GPULIST: $LOCAL_CHUNKS" + +ALL_CHUNKS=$((NUM_NODES * LOCAL_CHUNKS)) + +# Calculate GPUs per chunk +GPUS_PER_CHUNK=1 + +echo $GPUS_PER_CHUNK + +for IDX in $(seq 1 $LOCAL_CHUNKS); do + ( + START=$(((IDX-1) * GPUS_PER_CHUNK)) + LENGTH=$GPUS_PER_CHUNK # Length for slicing, not the end index + + CHUNK_GPUS=(${GPULIST[@]:$START:$LENGTH}) + + # Convert the chunk GPUs array to a comma-separated string + CHUNK_GPUS_STR=$(IFS=,; echo "${CHUNK_GPUS[*]}") + + LOCAL_IDX=$((CUR_NODES_IDX * LOCAL_CHUNKS + IDX)) + + echo "Chunk $(($LOCAL_IDX - 1)) will run on GPUs $CHUNK_GPUS_STR" + + # Calculate the port for this chunk. Ensure it's incremented by 5 for each chunk. + PORT=$((10000 + RANDOM % 55536)) + + MAX_RETRIES=10 + RETRY_COUNT=0 + COMMAND_STATUS=1 # Initialize as failed + + while [ $RETRY_COUNT -lt $MAX_RETRIES ] && [ $COMMAND_STATUS -ne 0 ]; do + echo "Running chunk $(($LOCAL_IDX - 1)) on GPUs $CHUNK_GPUS_STR with port $PORT. Attempt $(($RETRY_COUNT + 1))" + +#!/bin/bash + CUDA_VISIBLE_DEVICES=$CHUNK_GPUS_STR python3 srt_example_llava_v.py \ + --port $PORT \ + --num-chunks $ALL_CHUNKS \ + --chunk-idx $(($LOCAL_IDX - 1)) \ + --save-dir work_dirs/llava_next_video_inference_results \ + --video-dir $VIDEO_DIR \ + --model-path $MODEL_PATH \ + --num-frames $NUM_FRAMES #& + + wait $! # Wait for the process to finish and capture its exit status + COMMAND_STATUS=$? + + if [ $COMMAND_STATUS -ne 0 ]; then + echo "Execution failed for chunk $(($LOCAL_IDX - 1)), attempt $(($RETRY_COUNT + 1)). Retrying..." + RETRY_COUNT=$(($RETRY_COUNT + 1)) + sleep 180 # Wait a bit before retrying + else + echo "Execution succeeded for chunk $(($LOCAL_IDX - 1))." + fi + done + + if [ $COMMAND_STATUS -ne 0 ]; then + echo "Execution failed for chunk $(($LOCAL_IDX - 1)) after $MAX_RETRIES attempts." + fi + ) #& + sleep 2 # Slight delay to stagger the start times +done + +wait + +cat work_dirs/llava_next_video_inference_results/final_results_chunk_*.csv > work_dirs/llava_next_video_inference_results/final_results_node_${CUR_NODES_IDX}.csv + +END_TIME=$(date +%s) # Capture end time +ELAPSED_TIME=$(($END_TIME - $START_TIME)) +echo "Total execution time: $ELAPSED_TIME seconds." diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/openai_chat_speculative.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/openai_chat_speculative.py new file mode 100644 index 00000000..f3fd74ed --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/openai_chat_speculative.py @@ -0,0 +1,155 @@ +""" +Usage: +***Note: for speculative execution to work, user must put all "gen" in "assistant". +Show in "assistant" the desired answer format. Each "gen" term should have a stop token. +The stream mode is not supported in speculative execution. + +E.g. +correct: + sgl.assistant("\nName:" + sgl.gen("name", stop="\n") + "\nBirthday:" + sgl.gen("birthday", stop="\n") + "\nJob:" + sgl.gen("job", stop="\n")) +incorrect: + s += sgl.assistant("\nName:" + sgl.gen("name", stop="\n")) + s += sgl.assistant("\nBirthday:" + sgl.gen("birthday", stop="\n")) + s += sgl.assistant("\nJob:" + sgl.gen("job", stop="\n")) + +export OPENAI_API_KEY=sk-****** +python3 openai_chat_speculative.py +""" + +import sglang as sgl +from sglang import OpenAI, function, set_default_backend + + +@function(num_api_spec_tokens=256) +def gen_character_spec(s): + s += sgl.system("You are a helpful assistant.") + s += sgl.user("Construct a character within the following format:") + s += sgl.assistant( + "Name: Steve Jobs.\nBirthday: February 24, 1955.\nJob: Apple CEO.\n" + ) + s += sgl.user("Please generate new Name, Birthday and Job.\n") + s += sgl.assistant( + "Name:" + + sgl.gen("name", stop="\n") + + "\nBirthday:" + + sgl.gen("birthday", stop="\n") + + "\nJob:" + + sgl.gen("job", stop="\n") + ) + + +@function(num_api_spec_tokens=256) +def gen_character_spec_no_few_shot(s): + s += sgl.user("Construct a character. For each field stop with a newline\n") + s += sgl.assistant( + "Name:" + + sgl.gen("name", stop="\n") + + "\nAge:" + + sgl.gen("age", stop="\n") + + "\nJob:" + + sgl.gen("job", stop="\n") + ) + + +@function +def gen_character_normal(s): + s += sgl.system("You are a helpful assistant.") + s += sgl.user("What's the answer of 23 + 8?") + s += sgl.assistant(sgl.gen("answer", max_tokens=64)) + + +@function(num_api_spec_tokens=1024) +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user("Answer questions in the following format:") + s += sgl.user( + "Question 1: What is the capital of France?\nQuestion 2: What is the population of this city?\n" + ) + s += sgl.assistant( + "Answer 1: The capital of France is Paris.\nAnswer 2: The population of Paris in 2024 is estimated to be around 2.1 million for the city proper.\n" + ) + s += sgl.user("Question 1: " + question_1 + "\nQuestion 2: " + question_2) + s += sgl.assistant( + "Answer 1: " + + sgl.gen("answer_1", stop="\n") + + "\nAnswer 2: " + + sgl.gen("answer_2", stop="\n") + ) + + +def test_spec_single_turn(): + backend.token_usage.reset() + + state = gen_character_spec.run() + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- name:", state["name"]) + print("-- birthday:", state["birthday"]) + print("-- job:", state["job"]) + print(backend.token_usage) + + +def test_inaccurate_spec_single_turn(): + state = gen_character_spec_no_few_shot.run() + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- name:", state["name"]) + print("\n-- age:", state["age"]) + print("\n-- job:", state["job"]) + + +def test_normal_single_turn(): + state = gen_character_normal.run() + for m in state.messages(): + print(m["role"], ":", m["content"]) + + +def test_spec_multi_turn(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions in the capital of the United States.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + print("\n-- answer_2 --\n", state["answer_2"]) + + +def test_spec_multi_turn_stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + + +if __name__ == "__main__": + backend = OpenAI("gpt-4-turbo") + set_default_backend(backend) + + print("\n========== test spec single turn ==========\n") + # expect reasonable answer for each field + test_spec_single_turn() + + print("\n========== test inaccurate spec single turn ==========\n") + # expect incomplete or unreasonable answers + test_inaccurate_spec_single_turn() + + print("\n========== test normal single turn ==========\n") + # expect reasonable answer + test_normal_single_turn() + + print("\n========== test spec multi turn ==========\n") + # expect answer with same format as in the few shot + test_spec_multi_turn() + + print("\n========== test spec multi turn stream ==========\n") + # expect error in stream_executor: stream is not supported... + test_spec_multi_turn_stream() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/openai_speculative.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/openai_speculative.py new file mode 100644 index 00000000..4389cb05 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/openai_speculative.py @@ -0,0 +1,54 @@ +""" +Usage: +python3 openai_speculative.py +""" + +from sglang import OpenAI, function, gen, set_default_backend + + +@function(num_api_spec_tokens=64) +def gen_character_spec(s): + s += "Construct a character within the following format:\n" + s += "Name: Steve Jobs.\nBirthday: February 24, 1955.\nJob: Apple CEO.\n" + s += "\nPlease generate new Name, Birthday and Job.\n" + s += "Name:" + gen("name", stop="\n") + "\nBirthday:" + gen("birthday", stop="\n") + s += "\nJob:" + gen("job", stop="\n") + "\n" + + +@function +def gen_character_no_spec(s): + s += "Construct a character within the following format:\n" + s += "Name: Steve Jobs.\nBirthday: February 24, 1955.\nJob: Apple CEO.\n" + s += "\nPlease generate new Name, Birthday and Job.\n" + s += "Name:" + gen("name", stop="\n") + "\nBirthday:" + gen("birthday", stop="\n") + s += "\nJob:" + gen("job", stop="\n") + "\n" + + +@function(num_api_spec_tokens=64) +def gen_character_spec_no_few_shot(s): + # s += "Construct a character with name, birthday, and job:\n" + s += "Construct a character:\n" + s += "Name:" + gen("name", stop="\n") + "\nBirthday:" + gen("birthday", stop="\n") + s += "\nJob:" + gen("job", stop="\n") + "\n" + + +if __name__ == "__main__": + backend = OpenAI("gpt-3.5-turbo-instruct") + set_default_backend(backend) + + for function in [ + gen_character_spec, + gen_character_no_spec, + gen_character_spec_no_few_shot, + ]: + backend.token_usage.reset() + + print(f"function: {function.func.__name__}") + + state = function.run() + + print("...name:", state["name"]) + print("...birthday:", state["birthday"]) + print("...job:", state["job"]) + print(backend.token_usage) + print() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/parallel_sample.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/parallel_sample.py new file mode 100644 index 00000000..0f3cf170 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/parallel_sample.py @@ -0,0 +1,40 @@ +""" +Usage: +python3 parallel_sample.py +""" + +import sglang as sgl + + +@sgl.function +def parallel_sample(s, question, n): + s += ( + "Question: Compute 1 + 2 + 3\n" + "Reasoning: I need to use a calculator.\n" + "Tool: calculator\n" + "Answer: 6\n" + "Question: Compute 3 + 2 + 2\n" + "Reasoning: I will try a calculator.\n" + "Tool: calculator\n" + "Answer: 7\n" + ) + s += "Question: " + question + "\n" + forks = s.fork(n) + forks += "Reasoning:" + sgl.gen("reasoning", stop="\n") + "\n" + forks += "Tool:" + sgl.gen("tool", choices=["calculator", "browser"]) + "\n" + forks += "Answer:" + sgl.gen("answer", stop="\n") + "\n" + forks.join() + + +sgl.set_default_backend(sgl.OpenAI("gpt-3.5-turbo-instruct")) +# sgl.set_default_backend(sgl.RuntimeEndpoint("http://localhost:30000")) + +state = parallel_sample.run(question="Compute 5 + 2 + 4.", n=5, temperature=1.0) + +for i in range(5): + obj = { + "reasoning": state["reasoning"][i], + "tool": state["tool"][i], + "answer": state["answer"][i], + } + print(f"[{i}], {obj}") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/rag_using_parea/trace_and_evaluate_rag_using_parea.ipynb b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/rag_using_parea/trace_and_evaluate_rag_using_parea.ipynb new file mode 100644 index 00000000..f309142a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/rag_using_parea/trace_and_evaluate_rag_using_parea.ipynb @@ -0,0 +1,408 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# RAG Powered by SGLang & Chroma Evaluated using Parea\n", + "\n", + "In this notebook, we will build a simple RAG pipeline using SGLang to execute our LLM calls, Chroma as vector database for retrieval and [Parea](https://www.parea.ai) for tracing and evaluation. We will then evaluate the performance of our RAG pipeline. The dataset we will use was created by [Virat](https://twitter.com/virattt) and contains 100 questions, contexts and answers from the Airbnb 2023 10k filing.\n", + "\n", + "The RAG pipeline consists of two steps:\n", + "1. Retrieval: Given a question, we retrieve the relevant context from all provided contexts.\n", + "2. Generation: Given the question and the retrieved context, we generate an answer.\n", + "\n", + "ℹ️ This notebook requires an OpenAI API key.\n", + "\n", + "ℹ️ This notebook requires a Parea API key, which can be created [here](https://docs.parea.ai/api-reference/authentication#parea-api-key)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setting up the environment\n", + "\n", + "We will first install the necessary packages: `sglang`, `parea-ai` and `chromadb`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# note, if you use a Mac M1 chip, you might need to install grpcio 1.59.0 first such that installing chromadb works\n", + "# !pip install grpcio==1.59.0\n", + "\n", + "!pip install sglang[openai] parea-ai chromadb" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create a Parea API key as outlined [here](https://docs.parea.ai/api-reference/authentication#parea-api-key) and save it in a `.env` file as `PAREA_API_KEY=your-api-key`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Indexing the data\n", + "\n", + "Now it's time to download the data & index it! For that, we create a collection called `contexts` in Chroma and add the contexts as documents." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from typing import List\n", + "\n", + "import chromadb\n", + "\n", + "path_qca = \"airbnb-2023-10k-qca.json\"\n", + "\n", + "if not os.path.exists(path_qca):\n", + " !wget https://virattt.github.io/datasets/abnb-2023-10k.json -O airbnb-2023-10k-qca.json\n", + "\n", + "with open(path_qca, \"r\") as f:\n", + " question_context_answers = json.load(f)\n", + "\n", + "chroma_client = chromadb.PersistentClient()\n", + "collection = chroma_client.get_or_create_collection(name=\"contexts\")\n", + "if collection.count() == 0:\n", + " collection.add(\n", + " documents=[qca[\"context\"] for qca in question_context_answers],\n", + " ids=[str(i) for i in range(len(question_context_answers))],\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Defining the RAG pipeline\n", + "\n", + "We will start with importing the necessary packages, setting up tracing of OpenAI calls via Parea and setting OpenAI as the default backend for SGLang." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import time\n", + "\n", + "from dotenv import load_dotenv\n", + "\n", + "from sglang import function, user, assistant, gen, set_default_backend, OpenAI\n", + "from sglang.lang.interpreter import ProgramState\n", + "from parea import Parea, trace\n", + "\n", + "\n", + "load_dotenv()\n", + "\n", + "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", + "\n", + "p = Parea(api_key=os.getenv(\"PAREA_API_KEY\"), project_name=\"rag_sglang\")\n", + "p.integrate_with_sglang()\n", + "\n", + "set_default_backend(OpenAI(\"gpt-3.5-turbo\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can define our retrieval step shown below. Notice, the `trace` decorator which will automatically trace inputs, output, latency, etc. of that call." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@trace\n", + "def retrieval(question: str) -> List[str]:\n", + " return collection.query(query_texts=[question], n_results=1)[\"documents\"][0]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next we will define the generation step which uses SGLang to execute the LLM call." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def generation_sglang(s, question: str, *context: str):\n", + " context = \"\\n\".join(context)\n", + " s += user(\n", + " f\"Given this question:\\n{question}\\n\\nAnd this context:\\n{context}\\n\\nAnswer the question.\"\n", + " )\n", + " s += assistant(gen(\"answer\"))\n", + "\n", + "\n", + "@trace\n", + "def generation(question: str, *context):\n", + " state: ProgramState = generation_sglang.run(question, *context)\n", + " while not state.stream_executor.is_finished:\n", + " time.sleep(1)\n", + " return state.stream_executor.variables[\"answer\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, we can tie it together and execute a sample query." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@trace\n", + "def rag_pipeline(question: str) -> str:\n", + " contexts = retrieval(question)\n", + " return generation(question, *contexts)\n", + "\n", + "\n", + "rag_pipeline(\n", + " \"When did the World Health Organization formally declare an end to the COVID-19 global health emergency?\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Debug Trace\n", + "\n", + "The output is unfortunately wrong! Using the traced pipeline, we can see that\n", + "\n", + "- the context is relevant to the question and contains the correct information\n", + "- but, the generation step is cut off as max tokens is set to 16\n", + "\n", + "When opening the generation step in the playground and rerunning the prompt with max. tokens set to 1000, the correct answer is produced.\n", + "\n", + "![RAG Trace](https://drive.google.com/uc?id=1QI243ogGjzbO01tUrR72g9rFoGzUJqVH)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Evaluating RAG Pipelines\n", + "\n", + "Before we apply above's fix, let's dive into evaluating RAG pipelines.\n", + "\n", + "RAG pipelines consist of a retrieval step to fetch relevant information and a generation step to generate a response to a users question. A RAG pipeline can fail at either step. E.g. the retrieval step can fail to find relevant information which makes generating the correct impossible. Another failure mode is that the generation step doesn't leverage the retrieved information correctly. We will apply the following evaluation metrics to understand different failure modes:\n", + "\n", + "- `context_relevancy`: measures how relevant the context is given the question\n", + "- `percent_target_supported_by_context`: measures how much of the target answer is supported by the context; this will give an upper ceiling of how well the generation step can perform\n", + "- `answer_context_faithfulness`: measures how much the generated answer utilizes the context\n", + "- `answer_matches_target`: measures how well the generated answer matches the target answer judged by a LLM and gives a sense of accuracy of our entire pipeline\n", + "\n", + "To use these evaluation metrics, we can import them from `parea.evals.rag` and `parea.evals.general` and apply them to a function by specifying in the `trace` decorator which evaluation metrics to use. The `@trace` decorator will automatically log the results of the evaluation metrics to the Parea dashboard.\n", + "\n", + "Applying them to the retrieval step:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from parea.evals.rag import (\n", + " context_query_relevancy_factory,\n", + " percent_target_supported_by_context_factory,\n", + ")\n", + "\n", + "\n", + "context_relevancy_eval = context_query_relevancy_factory()\n", + "percent_target_supported_by_context = percent_target_supported_by_context_factory()\n", + "\n", + "\n", + "@trace(eval_funcs=[context_relevancy_eval, percent_target_supported_by_context])\n", + "def retrieval(question: str) -> List[str]:\n", + " return collection.query(query_texts=[question], n_results=1)[\"documents\"][0]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can apply `answer_context_faithfulness` and `answer_matches_target` to the generation step." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from parea.evals.general import answer_matches_target_llm_grader_factory\n", + "from parea.evals.rag import answer_context_faithfulness_statement_level_factory\n", + "\n", + "\n", + "answer_context_faithfulness = answer_context_faithfulness_statement_level_factory()\n", + "answer_matches_target_llm_grader = answer_matches_target_llm_grader_factory()\n", + "\n", + "\n", + "@function\n", + "def generation_sglang(s, question: str, *context: str):\n", + " context = \"\\n\".join(context)\n", + " s += user(\n", + " f\"Given this question:\\n{question}\\n\\nAnd this context:\\n{context}\\n\\nAnswer the question.\"\n", + " )\n", + " s += assistant(gen(\"answer\", max_tokens=1_000))\n", + "\n", + "\n", + "@trace(eval_funcs=[answer_context_faithfulness, answer_matches_target_llm_grader])\n", + "def generation(question: str, *context):\n", + " state: ProgramState = generation_sglang.run(question, *context)\n", + " while not state.stream_executor.is_finished:\n", + " time.sleep(1)\n", + " return state.stream_executor.variables[\"answer\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, we tie them together & execute the original sample query." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@trace\n", + "def rag_pipeline(question: str) -> str:\n", + " contexts = retrieval(question)\n", + " return generation(question, *contexts)\n", + "\n", + "\n", + "rag_pipeline(\n", + " \"When did the World Health Organization formally declare an end to the COVID-19 global health emergency?\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Great, the answer is correct! Can you spot the line where we fixed the output truncation issue?\n", + "\n", + "The evaluation scores appear in the bottom right of the logs (screenshot below). Note, that there is no score for `answer_matches_target_llm_grader` and `percent_target_supported_by_context` as these evals are automatically skipped if the target answer is not provided.\n", + "\n", + "![Fixed Max. Tokens](max-tokens-fixed-rag-trace.png)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Running an experiment\n", + "\n", + "Now we are (almost) ready to evaluate the performance of our RAG pipeline on the entire dataset. First, we will need to apply the `nest_asyncio` package to avoid issues with the Jupyter notebook event loop." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install nest-asyncio\n", + "import nest_asyncio\n", + "\n", + "nest_asyncio.apply()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Running the actual experiment is straight-forward. For that we use `p.experiment` to initialize the experiment with a name, the data (list of key-value pairs fed into our entry function) and the entry function. We then call `run` on the experiment to execute it. Note, that `target` is a reserved key in the data dictionary and will be used as the target answer for evaluation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "e = p.experiment(\n", + " \"RAG\",\n", + " data=[\n", + " {\n", + " \"question\": qca[\"question\"],\n", + " \"target\": qca[\"answer\"],\n", + " }\n", + " for qca in question_context_answers\n", + " ],\n", + " func=rag_pipeline,\n", + ").run()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Analyzing the results\n", + "\n", + "When opening above experiment, we will see an overview of the experiment as shown below. The upper half shows a summary of the statistics on the left and charts to investigate the distribution and relationships of scores on the right. The lower half is a table with the individual traces which we can use to debug individual samples.\n", + "\n", + "When looking at the statistics, we can see that the accuracy of our RAG pipeline is 22% as measured by `answer_matches_target_llm_grader`. Though when checking the quality of our retrieval step (`context_query_relevancy`), we can see that our retrieval step is fetching relevant information in only 27% of all samples. As shown in the GIF, we investigate the relationship between the two and see the two scores have 95% agreement. This confirms that the retrieval step is a major bottleneck for our RAG pipeline. So, now it's your turn to improve the retrieval step!\n", + "\n", + "Note, above link isn't publicly accessible but the experiment can be accessed through [here](https://app.parea.ai/public-experiments/parea/rag_sglang/30f0244a-d56c-44ff-bdfb-8f47626304b6).\n", + "\n", + "![Experiment Results](https://drive.google.com/uc?id=1KMtJBU47nPB02Pvv3SPPTK7RnHRh5YdA)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/readme_examples.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/readme_examples.py new file mode 100644 index 00000000..7269ef14 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/readme_examples.py @@ -0,0 +1,109 @@ +""" +Usage: +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +python readme_examples.py +""" + +import sglang as sgl + + +@sgl.function +def tool_use(s, question): + s += "To answer this question: " + question + ". " + s += ( + "I need to use a " + + sgl.gen("tool", choices=["calculator", "search engine"]) + + ". " + ) + + if s["tool"] == "calculator": + s += "The math expression is" + sgl.gen("expression") + elif s["tool"] == "search engine": + s += "The key word to search is" + sgl.gen("word") + + +@sgl.function +def tip_suggestion(s): + s += ( + "Here are two tips for staying healthy: " + "1. Balanced Diet. 2. Regular Exercise.\n\n" + ) + + forks = s.fork(2) + for i, f in enumerate(forks): + f += f"Now, expand tip {i+1} into a paragraph:\n" + f += sgl.gen(f"detailed_tip", max_tokens=256, stop="\n\n") + + s += "Tip 1:" + forks[0]["detailed_tip"] + "\n" + s += "Tip 2:" + forks[1]["detailed_tip"] + "\n" + s += "In summary" + sgl.gen("summary") + + +@sgl.function +def regular_expression_gen(s): + s += "Q: What is the IP address of the Google DNS servers?\n" + s += "A: " + sgl.gen( + "answer", + temperature=0, + regex=r"((25[0-5]|2[0-4]\d|[01]?\d\d?).){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)", + ) + + +@sgl.function +def text_qa(s, question): + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n") + + +def driver_tool_use(): + state = tool_use.run(question="What is the capital of the United States?") + print(state.text()) + print("\n") + + +def driver_tip_suggestion(): + state = tip_suggestion.run() + print(state.text()) + print("\n") + + +def driver_regex(): + state = regular_expression_gen.run() + print(state.text()) + print("\n") + + +def driver_batching(): + states = text_qa.run_batch( + [ + {"question": "What is the capital of the United Kingdom?"}, + {"question": "What is the capital of France?"}, + {"question": "What is the capital of Japan?"}, + ], + progress_bar=True, + ) + + for s in states: + print(s.text()) + print("\n") + + +def driver_stream(): + state = text_qa.run( + question="What is the capital of France?", temperature=0.1, stream=True + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print("\n") + + +if __name__ == "__main__": + # sgl.set_default_backend(sgl.OpenAI("gpt-3.5-turbo-instruct")) + sgl.set_default_backend(sgl.RuntimeEndpoint("http://localhost:30000")) + + driver_tool_use() + driver_tip_suggestion() + driver_regex() + driver_batching() + driver_stream() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/sgl_gen_min_tokens.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/sgl_gen_min_tokens.py new file mode 100644 index 00000000..a5088199 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/sgl_gen_min_tokens.py @@ -0,0 +1,35 @@ +""" +This example demonstrates how to use `min_tokens` to enforce sgl.gen to generate a longer sequence + +Usage: +python3 sgl_gen_min_tokens.py +""" + +import sglang as sgl + + +@sgl.function +def long_answer(s): + s += sgl.user("What is the capital of the United States?") + s += sgl.assistant(sgl.gen("answer", min_tokens=64, max_tokens=128)) + + +@sgl.function +def short_answer(s): + s += sgl.user("What is the capital of the United States?") + s += sgl.assistant(sgl.gen("answer")) + + +if __name__ == "__main__": + runtime = sgl.Runtime(model_path="meta-llama/Meta-Llama-3.1-8B-Instruct") + sgl.set_default_backend(runtime) + + state = long_answer.run() + print("=" * 20) + print("Longer Answer", state["answer"]) + + state = short_answer.run() + print("=" * 20) + print("Short Answer", state["answer"]) + + runtime.shutdown() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/streaming.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/streaming.py new file mode 100644 index 00000000..506ee35c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/streaming.py @@ -0,0 +1,49 @@ +""" +Usage: +python3 streaming.py +""" + +import asyncio + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +sgl.set_default_backend(sgl.OpenAI("gpt-3.5-turbo")) + + +def stream_a_variable(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(var_name="answer_2"): + print(out, end="", flush=True) + print("\n") + + +async def async_stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + async for out in state.text_async_iter(var_name="answer_2"): + print(out, end="", flush=True) + print("\n") + + +if __name__ == "__main__": + stream_a_variable() + asyncio.run(async_stream()) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/triton/Dockerfile b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/triton/Dockerfile new file mode 100644 index 00000000..e4741a1d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/triton/Dockerfile @@ -0,0 +1,10 @@ +FROM nvcr.io/nvidia/tritonserver:24.01-py3 + +WORKDIR /opt + +RUN git clone https://github.com/sgl-project/sglang.git + +WORKDIR /opt/sglang +RUN pip install --upgrade pip && \ + pip install -e "python[all]" && \ + pip install datasets diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/triton/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/triton/README.md new file mode 100644 index 00000000..b2e55961 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/triton/README.md @@ -0,0 +1,35 @@ +# sglang_triton + +Build the docker image: +``` +docker build -t sglang-triton . +``` + +Then do: +``` +docker run -ti --gpus=all --network=host --name sglang-triton -v ./models:/mnt/models sglang-triton +``` + +inside the docker container: +``` +cd sglang +python3 -m sglang.launch_server --model-path mistralai/Mistral-7B-Instruct-v0.2 --port 30000 --mem-fraction-static 0.9 +``` + +with another shell, inside the docker container: +``` +docker exec -ti sglang-triton /bin/bash +cd /mnt +tritonserver --model-repository=/mnt/models +``` + + +Send request to the server: +``` +curl -X POST http://localhost:8000/v2/models/character_generation/generate \ +-H "Content-Type: application/json" \ +-d '{ + "INPUT_TEXT": ["harry"] +}' + +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/triton/models/character_generation/config.pbtxt b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/triton/models/character_generation/config.pbtxt new file mode 100644 index 00000000..7546f993 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/frontend_language/usage/triton/models/character_generation/config.pbtxt @@ -0,0 +1,23 @@ +name: "character_generation" +backend: "python" +input [ + { + name: "INPUT_TEXT" + data_type: TYPE_STRING + dims: [ -1 ] + } +] +output [ + { + name: "OUTPUT_TEXT" + data_type: TYPE_STRING + dims: [ -1 ] + } +] +instance_group [ + { + count: 1 + kind: KIND_GPU + gpus: [ 0 ] + } +] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/README.md new file mode 100644 index 00000000..3eef0b09 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/README.md @@ -0,0 +1,76 @@ +# SGLang Monitoring Setup + +This directory contains a ready-to-use monitoring setup for SGLang using Prometheus and Grafana. + +## Prerequisites + +- Docker and Docker Compose installed +- SGLang server running with metrics enabled + +## Usage + +1. Start your SGLang server with metrics enabled: + +```bash +python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --port 30000 --enable-metrics +``` + +By default, the metrics server will run on `127.0.0.1:30000`. + +2. Start the monitoring stack: + +```bash +cd examples/monitoring +docker compose up +``` + +3. Access the monitoring interfaces: + - Grafana: [http://localhost:3000](http://localhost:3000) + - Prometheus: [http://localhost:9090](http://localhost:9090) + +Default Grafana login credentials: +- Username: `admin` +- Password: `admin` + +You'll be prompted to change the password on first login. + +4. The SGLang dashboard will be automatically available in the "SGLang Monitoring" folder. + +## Troubleshooting + +### Port Conflicts +If you see errors like "port is already allocated": + +1. Check if you already have Prometheus or Grafana running: + ```bash + docker ps | grep -E 'prometheus|grafana' + ``` + +2. Stop any conflicting containers: + ```bash + docker stop + ``` + +3. Ensure no other services are using ports 9090 and 3000: + ```bash + lsof -i :9090 + lsof -i :3000 + ``` + +### Connection Issues +If Grafana cannot connect to Prometheus: +1. Check that both services are running +2. Verify the datasource configuration in Grafana +3. Check that your SGLang server is properly exposing metrics + +## Configuration + +- Prometheus configuration: `prometheus.yaml` +- Docker Compose configuration: `docker-compose.yaml` +- Grafana datasource: `grafana/datasources/datasource.yaml` +- Grafana dashboard configuration: `grafana/dashboards/config/dashboard.yaml` +- SGLang dashboard JSON: `grafana/dashboards/json/sglang-dashboard.json` + +## Customization + +You can customize the monitoring setup by modifying the configuration files as needed. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/docker-compose.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/docker-compose.yaml new file mode 100644 index 00000000..ce6457fa --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/docker-compose.yaml @@ -0,0 +1,28 @@ +version: '3' +services: + prometheus: + image: prom/prometheus:latest + container_name: prometheus + network_mode: host + volumes: + - ./prometheus.yaml:/etc/prometheus/prometheus.yml + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + + grafana: + image: grafana/grafana:latest + container_name: grafana + network_mode: host + volumes: + - ./grafana/datasources:/etc/grafana/provisioning/datasources + - ./grafana/dashboards/config:/etc/grafana/provisioning/dashboards + - ./grafana/dashboards/json:/var/lib/grafana/dashboards + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer + - GF_AUTH_BASIC_ENABLED=false + - GF_USERS_ALLOW_SIGN_UP=false + - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/var/lib/grafana/dashboards/sglang-dashboard.json + depends_on: + - prometheus diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/grafana/dashboards/config/dashboard.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/grafana/dashboards/config/dashboard.yaml new file mode 100644 index 00000000..6c17a6c6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/grafana/dashboards/config/dashboard.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 +providers: + - name: 'SGLang' + orgId: 1 + folder: 'SGLang Monitoring' + type: file + disableDeletion: false + updateIntervalSeconds: 10 + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/grafana/dashboards/json/sglang-dashboard.json b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/grafana/dashboards/json/sglang-dashboard.json new file mode 100644 index 00000000..ebd663e4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/grafana/dashboards/json/sglang-dashboard.json @@ -0,0 +1,984 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 8, + "links": [], + "panels": [ + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(sglang:e2e_request_latency_seconds_bucket[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P99", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le) (rate(sglang:e2e_request_latency_seconds_bucket[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P90", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le) (rate(sglang:e2e_request_latency_seconds_bucket[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P50", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "avg(rate(sglang:e2e_request_latency_seconds_sum[$__rate_interval]) / rate(sglang:e2e_request_latency_seconds_count[$__rate_interval]))\r\n", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Avg", + "range": true, + "refId": "D", + "useBackend": false + } + ], + "title": "End-to-End Request Latency", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 17, + "maxDataPoints": 30, + "options": { + "calculate": false, + "calculation": { + "yBuckets": { + "scale": { + "type": "linear" + } + } + }, + "cellGap": 1, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": true, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "secs" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(increase(sglang:e2e_request_latency_seconds_bucket{model_name=~\"$model_name\"}[$__rate_interval])) by (le)\r\n", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "End-to-End Request Latency(s) Heatmap", + "type": "heatmap" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(sglang:time_to_first_token_seconds_bucket[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P99", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le) (rate(sglang:time_to_first_token_seconds_bucket[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P90", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le) (rate(sglang:time_to_first_token_seconds_bucket[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P50", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "avg(rate(sglang:time_to_first_token_seconds_sum[$__rate_interval]) / rate(sglang:time_to_first_token_seconds_count[$__rate_interval]))\r\n", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Avg", + "range": true, + "refId": "D", + "useBackend": false + } + ], + "title": "Time-To-First-Token Latency", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 19, + "maxDataPoints": 30, + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "value": "" + }, + "yBuckets": { + "mode": "size", + "scale": { + "type": "linear" + }, + "value": "" + } + }, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": true, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(le) (increase(sglang:time_to_first_token_seconds_bucket{model_name=~\"$model_name\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Time-To-First-Token Seconds Heatmap", + "type": "heatmap" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sglang:num_running_reqs", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num Running Requests", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "editorMode": "code", + "expr": "sglang:gen_throughput", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Token Generation Throughput (Tokens / S)", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sglang:cache_hit_rate", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Cache Hit Rate", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sglang:num_queue_reqs", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number Queued Requests", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "127.0.0.1:30000", + "value": "127.0.0.1:30000" + }, + "datasource": { + "type": "prometheus" + }, + "definition": "label_values(instance)", + "includeAll": false, + "label": "instance", + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "meta-llama/Llama-3.1-8B-Instruct", + "value": "meta-llama/Llama-3.1-8B-Instruct" + }, + "datasource": { + "type": "prometheus" + }, + "definition": "label_values(model_name)", + "includeAll": false, + "label": "model name", + "name": "model_name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(model_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "SGLang Dashboard", + "uid": "sglang-dashboard", + "version": 11 +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/grafana/datasources/datasource.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/grafana/datasources/datasource.yaml new file mode 100644 index 00000000..1ab0e4a5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/grafana/datasources/datasource.yaml @@ -0,0 +1,8 @@ +apiVersion: 1 +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://localhost:9090 + isDefault: true + editable: false diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/opentelemetry.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/opentelemetry.yaml new file mode 100644 index 00000000..8593d918 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/opentelemetry.yaml @@ -0,0 +1,38 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +processors: + batch: + +exporters: + otlp: + endpoint: jaeger:4317 + tls: + insecure: true + file: + path: /tmp/otel_trace.json + +extensions: + health_check: + pprof: + zpages: + +service: + extensions: [health_check, pprof, zpages] + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [otlp, file] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [otlp] + logs: + receivers: [otlp] + processors: [batch] + exporters: [otlp] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/prometheus.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/prometheus.yaml new file mode 100644 index 00000000..ba16ac3b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/prometheus.yaml @@ -0,0 +1,10 @@ +# prometheus.yaml +global: + scrape_interval: 5s + evaluation_interval: 30s + +scrape_configs: + - job_name: sglang + static_configs: + - targets: + - '127.0.0.1:30000' diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/tracing_compose.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/tracing_compose.yaml new file mode 100644 index 00000000..7ed1ecdd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/monitoring/tracing_compose.yaml @@ -0,0 +1,21 @@ +services: + otel-collector: + image: docker.io/otel/opentelemetry-collector + volumes: + - ./opentelemetry.yaml:/etc/otelcol/config.yaml + - /tmp:/tmp + ports: + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP + depends_on: + - jaeger + restart: unless-stopped + + jaeger: + image: jaegertracing/all-in-one + container_name: jaeger + ports: + - "16686:16686" + environment: + - COLLECTOR_OTLP_ENABLED=true + restart: unless-stopped diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/profiler/nsys_profile_tools/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/profiler/nsys_profile_tools/README.md new file mode 100644 index 00000000..687200e0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/profiler/nsys_profile_tools/README.md @@ -0,0 +1,176 @@ +# gputrc2graph.py + +This script processes NVIDIA Nsight Systems (`nsys`) GPU trace files +(`.nsys-rep`) with -t cuda tracing enabled, and generates kernel-level +summaries and visualizations of GPU and non-GPU time. It is useful for +profiling and analyzing nsys profile output. + +## Usage + +### Command-line Arguments + +- `--in_file` + **(required)** + List of input files and their metadata. Each entry should be in the format: + `,,,` + - `nsys-rep`: Path to the `.nsys-rep` file. + - `engine`: Engine name (e.g., `sglang`). + - `model`: Model name (e.g., `llama`, `gpt-oss`, `ds`). + - `elapsed_nonprofiled_sec`: Wall-clock runtime (in seconds) without + profiling. Specify `0` to use the elapsed time from the nsys-rep file + (this may inflate non-GPU time if actual runtime without profiling is + less). Multiple entries can be provided, separated by spaces. + +- `--out_dir` + Output directory for the generated CSV and HTML files. + If not specified, results are saved in the current directory. + +- `--title` + Title for the HTML chart/visualization. + +- `--nsys_cmd` + Path to the `nsys` command. + Default: `nsys` (assumes it is in your PATH). + Use this if `nsys` is not in your system PATH. + +## Notes + +- Make sure you have pandas installed. Any version is fine. +- Make sure [nsys](https://developer.nvidia.com/nsight-systems/get-started) is +installed, and specify the path to the `nsys` command with `--nsys_cmd` if it + is not in your PATH. The nsys version must be >= the nsys profile version that + was used to collect the traces when profiling the server, so that nsys can + process the nsys-rep that was generated. + +- For more details on available engines and models, see the help string in + the script or run: + +```bash +python3 gputrc2graph.py --help +``` + +## Example 1: analyze a single profile + +To analyze the GPU cycles of for example, a llama-3.1-8B model with sglang: + +1. Run the following command to collect nsys profile, for sglang server config. + + ```bash + nsys profile -t cuda -o nsys_res -f true --trace-fork-before-exec=true \ + --cuda-graph-trace=node --delay --duration \ + python3 -m sglang.launch_server --model meta-llama/Llama-3.1-8B ... + ``` + + where: + + - DELAY: how many seconds to delay nsys from collecting profiles, needed so + that profiles aren't captured till sglang server has come up and load + generation starts. + - DURATION: how many seconds for nsys profile to run before generating the + profile. This should be > the duration of the run. +2. After the server starts, run the client load generation command. Once the +test completes, after DURATION amount of time, nsys profile will generate an +nsys_res.nsys-rep file and shut down the server. + +3. Run step #1 again, this time starting up the server without collecting the +profile. + +4. Run step #2 again, and record the total time to complete the test in +seconds. This value will be used by the script to calculate the + CPU(non-GPU) seconds for the analysis. + +5. Say the run elapsed time from step #4 is 132 seconds. Run script to + analyze: + + ```bash + python3 gputrc2graph.py \ + --in_file run1.nsys-rep,sglang,llama,132 + ``` + +The command will produce 2 files for analysis: + +- result.html: this categorizes kernel names into different categories in a + stacked bar chart. +- result.csv: shows how the kernel names are mapped to the different + categories. + +### HTML visualization with result.html + +The html file shows the number of elapsed seconds due to different GPU +Substages or categories, which consist of attention kernels as the biggest +category, at 63 seconds, followed by "gemm" kernels. This lets the user +prioritize the kernels to focus on for performance optimizations. + +There's also an appended data table underneath the bar chart for copying out to + other post-processing tools. + +### Kernel to category mapping with result.csv + +Suppose the user would like to focus on improving triton kernels. It's not the +biggest consumer of cycles at .01 sec but perhaps it hasn't been optimized. +The next step is to use the result.csv to dive into what the kernels are which +compose the triton kernel GPU cycles. + +## Example 2: analyze multiple profiles + +Suppose the user has multiple nsys trace files, captured for different models, +say llama and gpt-oss in this case, and wish to compare their GPU/non-GPU +time, something like the following command can be used. + +```bash +python3 gputrc2graph.py \ +--in_file run1.nsys-rep,sglang,llama,100 run2.nsys-rep,sglang,gpt-oss,102 \ +--out_dir results +``` + +The analysis process is similar to example 1 but now there will be multiple +stack bar charts that can be compared. The categories for the different +kernels will remain the same, so that it's easy to compare the GPU cycles for +the same categories. + +Once a category is shown to have more cycles for one configuration than +another, the next step would be to use the csv file to see what kernels are +mapped into that category, and which kernels are taking the largest amount of +time which would cause a difference for the overall category. + +## Example 3: add new classification for a new model + +To create a new engine DEF with model ABC, just add another json file in the same directory as +gputrc2graph.py with the same format as the other json files. The script will automatically pick up all the json files in the same directory as engine/model specifications. + +Then, for this new model, suppose there are 4 kernels to be classified into +"gemm" and "attn", where the gemm kernels have names with "*H*" or "*I*" in +them, and attn kernels have names with "*J*" or "*K*" in them, just add another + .json file in the same directory as gputrc2graph.py with the same format as + the other json files, like the following: + +```json +{ + "DEF": { + "ABC": { + "H|I": "gemm", + "J|K": "attn", + "CUDA mem": "non-gpu-H_D_memops", + ".*": "misc" + } + } +} +``` + +Each entry in the dictionary consists of: + +- key: a regex used to classify the kernels +- value: the category to classify the kernels into. + +The last 2 entries are common for all engine/models, consisting of CUDA memory +operations and a 'misc' for anything that's leftover and can't be classified. + +When invoking gputrc2graph.py, specify a trace file with this new model/engine +like the following: + +```bash +--in_file new.nsys-rep,DEF,ABC, +``` + +If the engine_DEF.json file already exists, just add the model as a new node in + the existing engine file, after the other models. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/profiler/nsys_profile_tools/gputrc2graph.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/profiler/nsys_profile_tools/gputrc2graph.py new file mode 100755 index 00000000..f17bd185 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/profiler/nsys_profile_tools/gputrc2graph.py @@ -0,0 +1,344 @@ +""" + This generates gpu kernel analysis output from nsys rep. Will call nsys + stats -r cuda_gpu_kern_trace, get non-overlapped gpu cycles, then generate + csv and html output for analysis +""" + +import argparse +import logging +import os + +import regex as re + +logger = logging.getLogger(__name__) + + +# helper data class for annotating kernels +def load_engine_model(): + """returns engine_model built from all json files in the current dir""" + import glob + import json + + engine_model = {} + + json_files = glob.glob(os.path.join(os.path.dirname(__file__) or ".", "*.json")) + for fname in json_files: + with open(fname, encoding="utf-8") as f: + engine_model.update(json.load(f)) + return engine_model + + +class GPUTrace2Graph: + """ + Parses output of nsys report, generates csv and bar chart output + """ + + def __init__(self): + import pandas as pd # avoid importing till needed + + self.pd = pd + self.pd.options.mode.copy_on_write = True + + # helper functions for generating trace->summary csvs + def gen_nonoverlapped_sum_from_gputrace(self, in_file, out_file): + logger.info("loading %s", in_file) + df = self.pd.read_csv( + in_file, usecols=["Start (ns)", "Duration (ns)", "Device", "Strm", "Name"] + ) + df["End (ns)"] = df["Start (ns)"] + df["Duration (ns)"] + df = self.sum_non_overlapping_intervals(df) + # get ready to print table with elapsed times per kernel + df["Instances"] = 1 + df_sum = df.groupby("Name", as_index=False).agg( + {"Elapsed Time (ns)": "sum", "Duration (ns)": "sum", "Instances": "size"} + ) + + # generate csv + df_sum["Total Time (sec)"] = df_sum["Duration (ns)"] / 1e9 + df_sum["Elapsed Time (sec)"] = df_sum["Elapsed Time (ns)"] / 1e9 + df_sum = df_sum.sort_values(by="Elapsed Time (sec)", ascending=False) + df_sum[["Elapsed Time (sec)", "Total Time (sec)", "Instances", "Name"]].to_csv( + out_file, index=False + ) + + def sum_non_overlapping_intervals(self, df): + """ + returns new sorted df with Elapsed Time (ns) column using + vectorized operations + """ + logger.info("sorting %s trace records by start time", str(df.shape)) + + # Sort by start time and reset index + df = df.sort_values(by="Start (ns)").reset_index(drop=True) + + # Initialize elapsed time as duration + df["Elapsed Time (ns)"] = df["Duration (ns)"] + + # Get numpy arrays for faster operations + starts = df["Start (ns)"].values + ends = df["End (ns)"].values + + # Keep track of current interval end + current_end = ends[0] + display_units = max(1, int(len(df) / 100)) + # Update current_end for overlapping intervals + for i in range(1, len(df)): + if i % display_units == 0: + print(f"processing trace: {int(i/len(df) * 100)} %", end="\r") + if starts[i] <= current_end: + if ends[i] > current_end: + # Partial overlap + df.iloc[i, df.columns.get_loc("Elapsed Time (ns)")] = ( + ends[i] - current_end + ) + current_end = ends[i] + else: + # Complete overlap + df.iloc[i, df.columns.get_loc("Elapsed Time (ns)")] = 0 + else: + # No overlap + current_end = ends[i] + + return df + + # functions for generating html files + def make_html(self, df, output_dir, title): + """make html graph from df""" + import plotly.express as px + + if df.empty: + return + output_name = os.path.join(output_dir, "result") + if not title: + title = "Model_Engine" + x = "Model_Engine" + y = "Elapsed Time (sec)" + color = "Category" + """ generate kernel mapping table """ + # Sort Model_Engine categories by last field after underscore + df["Model_Engine"] = self.pd.Categorical( + df["Model_Engine"], + sorted(df["Model_Engine"].unique(), key=lambda x: x.split("_")[-1]), + ) + df[["Model_Engine", color, "Instances", "Name", y]].sort_values( + by=color + ).to_csv(f"{output_name}.csv", index=False) + graph = px.histogram( + df.round(2), + x=x, + y=y, + title=(f"{y} for {title}"), + color=color, + text_auto=True, + ) + # wrap x axis labels + graph.update_xaxes(automargin=True) + graph.write_html(f"{output_name}.html") + """ + Generate data table with columns per Model_Engine into result.html + """ + pivot_df = df.pivot_table( + values="Elapsed Time (sec)", + index="Category", + columns="Model_Engine", + aggfunc="sum", + observed=False, + ).round(2) + # Add sum row at bottom + pivot_df.loc["total_elapsed_sec"] = pivot_df.sum() + pivot_df.fillna("").to_html("temp.html") + with ( + open(f"{output_name}.html", "a", encoding="utf-8") as outfile, + open("temp.html", encoding="utf-8") as infile, + ): + outfile.write(infile.read()) + os.remove("temp.html") + + print( + f"Finished generating: \n" + f" {output_name}.html for stack bar chart \n" + f" {output_name}.csv for Kernel-Category mapping" + ) + + def anno_gpu_kernname(self, df, mapping): + """add "Category" column""" + + def anno_gpu_kernname_helper(name): + for kern_name, val in mapping.items(): + if re.search(kern_name, name): + return val + + df["Category"] = df["Name"].apply(anno_gpu_kernname_helper) + + def make_nongpu_row(self, df, nongpu_sec): + """this will append non-gpu time entry at end of df""" + nongpu_row = self.pd.DataFrame([df.iloc[-1]]) + nongpu_row["Category"] = nongpu_row["Name"] = "CPU(non-GPU)" + nongpu_row["Instances"] = 1 + nongpu_row["Elapsed Time (sec)"] = nongpu_sec + return nongpu_row + + def is_valid_file(self, base_file): + """asserts if base_file is non-existent or is empty""" + assert ( + os.path.isfile(base_file) and os.path.getsize(base_file) > 0 + ), f"{base_file} doesn't exist or is empty" + + def should_gen_file(self, new_file, base_file): + """figure out if new file should be generated from base_file""" + self.is_valid_file(base_file) + if ( + os.path.exists(new_file) + and (os.path.getmtime(new_file) > os.path.getmtime(base_file)) + and (os.path.getsize(base_file) > 0) + ): + logger.info("reusing %s", new_file) + return False + else: + logger.info("generating %s", new_file) + return True + + def gen_sum_file(self, file, nsys_cmd): + """ + generates sum file from nsys trace with times per kernel and + returns the name of the sum file + """ + import subprocess + + file_dir = os.path.dirname(file) + file_name = os.path.basename(file) + + if not file_dir: + file_dir = "." + # Walk through trace and get the total non-overlapped time + nsys_stats_file = os.path.join(file_dir, f"{file_name}_cuda_gpu_trace.csv") + sum_file = os.path.join(file_dir, f"{file_name}_cuda_gpu_kernel_tracesum.csv") + if self.should_gen_file(nsys_stats_file, file): + cmd = [ + nsys_cmd, + "stats", + "-r", + "cuda_gpu_trace", + file, + "-o", + f"{file_dir}/{file_name}", + ] + cmd_str = " ".join(cmd) + logger.info("+ %s", cmd_str) + # estimate time based on calibrated 240M/min + file_size_mb = os.path.getsize(file) / 1e6 + logger.info( + "nsys stats for %.2f MB file expected to take %.2f min", + file_size_mb, + file_size_mb / 240, + ) + try: + subprocess.run(cmd, check=True) + except (FileNotFoundError, subprocess.CalledProcessError) as e: + logger.error( + "'%s' failed: %s. Use --nsys_cmd to specify nsys path", cmd_str, e + ) + exit(1) + logger.info("generating non-overalapped sum %s", sum_file) + self.gen_nonoverlapped_sum_from_gputrace(nsys_stats_file, sum_file) + self.is_valid_file(sum_file) + logger.info("Finished generating %s", sum_file) + return sum_file + + def gen_graph(self, in_file, out_dir, title, nsys_cmd, engine_model): + """generates graph and csv file from in_file into out_dir""" + # Initialize an empty DataFrame to store combined data + combined_df = self.pd.DataFrame() + for idx, (file, engine, model, total_sec) in enumerate(in_file): + file_dir = os.path.dirname(file) + file_name = os.path.basename(file) + if not file_dir: + file_dir = "." + sum_file = self.gen_sum_file(file, nsys_cmd) + # read kernel summary file + df = self.pd.read_csv(sum_file) + # annotate kernel to their categories + assert engine_model.get(engine), f"engine {engine} unknown" + assert engine_model[engine].get(model), f"model {model} unknown" + # remove nsys-rep from file_name for shorter x-label + file_name = file_name.replace(".nsys-rep", "") + df["Model_Engine"] = f"{model}_{engine}_{file_name}_{idx}" + self.anno_gpu_kernname(df, engine_model[engine][model]) + # patch in non-gpu time + gpu_sec = round(df["Elapsed Time (sec)"].sum(), 1) + total_sec = round(float(total_sec), 1) + if total_sec < gpu_sec: + logger.warning( + "Elapsed sec %.2f < GPU sec %.2f resetting Elapsed sec ", + total_sec, + gpu_sec, + ) + total_sec = gpu_sec + nongpu_row = self.make_nongpu_row(df, total_sec - gpu_sec) + df = self.pd.concat([df, nongpu_row], ignore_index=True) + combined_df = self.pd.concat([combined_df, df], ignore_index=True) + if out_dir is None: + out_dir = "." + else: + os.makedirs(out_dir, exist_ok=True) + # generate html file + self.make_html(combined_df, out_dir, title) + + +def parse_tuple(s): + return tuple(s.split(",")) + + +def main(): + logging.basicConfig( + format=("%(asctime)s - %(levelname)s - %(message)s"), level=logging.INFO + ) + parser = argparse.ArgumentParser( + description=( + "Process nsys rep and generate kernel non-overlapped cycles. \n" + "Example:\n" + "gputrc2graph.py --in_file d1.nsys-rep,sglang,llama,100 \n" + "d2.nsys-rep,sglang,gpt-oss,102 " + '--out_dir results/ --title "Model=gpt-oss SGLANG chart"' + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + # load supported engine_model + engine_model_supported = load_engine_model() + # Get a string representation of supported engine/model combinations + engine_model_supported_str = ", ".join( + f"{engine}:[{', '.join(models.keys())}]" + for engine, models in engine_model_supported.items() + ) + parser.add_argument( + "--in_file", + type=parse_tuple, + nargs="+", + help=( + "list of (nsys-rep, engine, model, elapsed_nonprofiled_sec) " + "separated by space. Elapsed_nonprofiled_sec is runtime without " + "profiling used to calculate non-gpu time. Specify 0 to use " + "elapsed time from nsys-rep but that might inflate non-gpu time. " + f"Available engine:[model] are: {engine_model_supported_str} " + f"Example: --infile d1.nsys-rep,sglan,llama,100 " + "d2.nsys-rep,sglang,gpt-oss,102" + ), + required=True, + ) + parser.add_argument("--out_dir", help=("output dir for result.csv/html")) + parser.add_argument("--title", help=("title for html chart")) + parser.add_argument( + "--nsys_cmd", + help=("nsys cmd, e.g. /usr/bin/nsys, Default: nsys"), + default="nsys", + ) + args = parser.parse_args() + gputrace = GPUTrace2Graph() + gputrace.gen_graph( + args.in_file, args.out_dir, args.title, args.nsys_cmd, engine_model_supported + ) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/profiler/nsys_profile_tools/sglang_engine_model.json b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/profiler/nsys_profile_tools/sglang_engine_model.json new file mode 100644 index 00000000..253cc762 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/profiler/nsys_profile_tools/sglang_engine_model.json @@ -0,0 +1,61 @@ +{ + "sglang": { + "llama": { + "gemm|nvjet": "gemm", + "fused_moe_kernel|GroupProblemShape|group_gemm_starts|bmm_|GemmUniversal": "moe_gemm", + "moe|sigmoid": "moe", + "CatArrayBatched|prepare_inputs": "prepare_next", + "ncclDevKernel|cross_device_reduce": "nccl_and_custom_ar", + "_norm_|Norm": "norm", + "topk": "topk", + "act_and_mul_": "activation", + "Rotary": "rope", + "SoftMax": "softmax", + "flash|fmha": "attn", + "elementwise": "elementwise", + "fp8_quant|cvt_|quantize": "quantize", + "reduce_kernel": "reduce", + "triton": "triton_kernel", + "CUDA mem": "non-gpu-H_D_memops", + ".*": "misc" + }, + "ds": { + "block_fp8_matmul": "block_fp8_gemm", + "gemm|matmul|nvjet": "gemm", + "fused_moe_kernel": "moe_gemm", + "moe|expert|sigmoid": "moe", + "CatArrayBatched|write_req_to": "prepare_next", + "ncclDevKernel|cross_device_reduce|all_gather": "nccl_and_custom_ar", + "Norm": "norm", + "topk": "topk", + "activation|act_and_mul": "activation", + "compute_position_kernel": "rope", + "elementwise": "elementwise", + "fp8_quant|quant_fp8|quantize": "quantize", + "SoftMax": "softmax", + "reduce": "reduce", + "_fwd_|create_flash|::mla::|KVCache": "attn", + "CUDA mem": "non-gpu-H_D_memops", + ".*": "misc" + }, + "gpt-oss": { + "gemm|nvjet": "gemm", + "fused_moe_kernel|_group_gemm|GroupProblemShape|GemmUniversal|bmm_|matmul_ogs_|_topk_forward|_combined_routing|_sum_bitmatrix_rows|_compute_writeback_idx": "moe_gemm", + "moe|sigmoid": "moe", + "CatArrayBatched|prepare_inputs": "prepare_next", + "_norm_|Norm": "norm", + "ncclDevKernel|cross_device_reduce|allreduce": "nccl_and_custom_ar", + "topk|TopK": "topk", + "act_and_mul_": "activation", + "Rotary": "rope", + "SoftMax": "softmax", + "flash|fmha": "attn", + "elementwise": "elementwise", + "fp8_quant|cvt_|quantize": "quantize", + "reduce_kernel": "reduce", + "triton": "triton_kernel", + "CUDA mem": "non-gpu-H_D_memops", + ".*": "misc" + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/README.md new file mode 100644 index 00000000..8b623fc3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/README.md @@ -0,0 +1,45 @@ +# Runtime examples + +The below examples will mostly need you to start a server in a separate terminal before you can execute them. Please see in the code for detailed instruction. + +## Native API + +* `lora.py`: An example how to use LoRA adapters. +* `multimodal_embedding.py`: An example how perform [multi modal embedding](Alibaba-NLP/gme-Qwen2-VL-2B-Instruct). +* `openai_batch_chat.py`: An example how to process batch requests for chat completions. +* `openai_batch_complete.py`: An example how to process batch requests for text completions. +* **`openai_chat_with_response_prefill.py`**: + An example that demonstrates how to [prefill a response](https://eugeneyan.com/writing/prompting/#prefill-claudes-responses) using the OpenAI API by enabling the `continue_final_message` parameter. + When enabled, the final (partial) assistant message is removed and its content is used as a prefill so that the model continues that message rather than starting a new turn. See [Anthropic's prefill example](https://docs.anthropic.com/en/docs/build-with-claude/prompt-engineering/prefill-claudes-response#example-structured-data-extraction-with-prefilling) for more context. +* `reward_model.py`: An example how to extract scores from a reward model. +* `vertex_predict.py`: An example how to deploy a model to [Vertex AI](https://cloud.google.com/vertex-ai?hl=en). + +## Engine + +The `engine` folder contains that examples that show how to use [Offline Engine API](https://docs.sglang.io/basic_usage/offline_engine_api.html#Offline-Engine-API) for common workflows. + +* `custom_server.py`: An example how to deploy a custom server. +* `embedding.py`: An example how to extract embeddings. +* `launch_engine.py`: An example how to launch the Engine. +* `offline_batch_inference_eagle.py`: An example how to perform speculative decoding using [EAGLE](https://docs.sglang.io/advanced_features/speculative_decoding.html). +* `offline_batch_inference_torchrun.py`: An example how to perform inference using [torchrun](https://pytorch.org/docs/stable/elastic/run.html). +* `offline_batch_inference_vlm.py`: An example how to use VLMs with the engine. +* `offline_batch_inference.py`: An example how to use the engine to perform inference on a batch of examples. + +## Hidden States + +The `hidden_states` folder contains examples on how to extract hidden states using SGLang. Please note that this might degrade throughput due to cuda graph rebuilding. + +* `hidden_states_engine.py`: An example how to extract hidden states using the Engine API. +* `hidden_states_server.py`: An example how to extract hidden states using the Server API. + +## Multimodal + +SGLang supports multimodal inputs for various model architectures. The `multimodal` folder contains examples showing how to use urls, files or encoded data to make requests to multimodal models. Examples include querying the [Llava-OneVision](multimodal/llava_onevision_server.py) model (image, multi-image, video), Llava-backed [Qwen-Llava](multimodal/qwen_llava_server.py) and [Llama3-Llava](multimodal/llama3_llava_server.py) models (image, multi-image), and Mistral AI's [Pixtral](multimodal/pixtral_server.py) (image, multi-image). + + +## Token In, Token Out + +The folder `token_in_token_out` shows how to perform inference, where we provide tokens and get tokens as response. + +* `token_in_token_out_{llm|vlm}_{engine|server}.py`: Shows how to perform token in, token out workflow for llm/vlm using either the engine or native API. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/custom_server.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/custom_server.py new file mode 100644 index 00000000..b190a463 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/custom_server.py @@ -0,0 +1,53 @@ +from sanic import Sanic, text +from sanic.response import json + +import sglang as sgl + +engine = None + +# Create an instance of the Sanic app +app = Sanic("sanic-server") + + +# Define an asynchronous route handler +@app.route("/generate", methods=["POST"]) +async def generate(request): + prompt = request.json.get("prompt") + if not prompt: + return json({"error": "Prompt is required"}, status=400) + + # async_generate returns a dict + result = await engine.async_generate(prompt) + + return text(result["text"]) + + +@app.route("/generate_stream", methods=["POST"]) +async def generate_stream(request): + prompt = request.json.get("prompt") + + if not prompt: + return json({"error": "Prompt is required"}, status=400) + + # async_generate returns a dict + result = await engine.async_generate(prompt, stream=True) + + # https://sanic.dev/en/guide/advanced/streaming.md#streaming + # init the response + response = await request.respond() + + # result is an async generator + async for chunk in result: + await response.send(chunk["text"]) + + await response.eof() + + +def run_server(): + global engine + engine = sgl.Engine(model_path="meta-llama/Meta-Llama-3.1-8B-Instruct") + app.run(host="0.0.0.0", port=8000, single_process=True) + + +if __name__ == "__main__": + run_server() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/embedding.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/embedding.py new file mode 100644 index 00000000..b927a188 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/embedding.py @@ -0,0 +1,27 @@ +import sglang as sgl + + +def main(): + # Sample prompts. + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + # Create an LLM. + llm = sgl.Engine( + model_path="Alibaba-NLP/gte-Qwen2-1.5B-instruct", is_embedding=True + ) + + outputs = llm.encode(prompts) + # Print the outputs (embedding vectors) + for prompt, output in zip(prompts, outputs): + print("===============================") + print(f"Prompt: {prompt}\nEmbedding vector: {output['embedding']}") + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/fastapi_engine_inference.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/fastapi_engine_inference.py new file mode 100644 index 00000000..f5da9d71 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/fastapi_engine_inference.py @@ -0,0 +1,195 @@ +""" +FastAPI server example for text generation using SGLang Engine and demonstrating client usage. + +Starts the server, sends requests to it, and prints responses. + +Usage: +python fastapi_engine_inference.py --model-path Qwen/Qwen2.5-0.5B-Instruct --tp_size 1 --host 127.0.0.1 --port 8000 [--startup-timeout 60] +""" + +import os +import subprocess +import time +from contextlib import asynccontextmanager + +import requests +from fastapi import FastAPI, Request + +import sglang as sgl +from sglang.utils import terminate_process + +engine = None + + +# Use FastAPI's lifespan manager to initialize/shutdown the engine +@asynccontextmanager +async def lifespan(app: FastAPI): + """Manages SGLang engine initialization during server startup.""" + global engine + # Initialize the SGLang engine when the server starts + # Adjust model_path and other engine arguments as needed + print("Loading SGLang engine...") + engine = sgl.Engine( + model_path=os.getenv("MODEL_PATH"), tp_size=int(os.getenv("TP_SIZE")) + ) + print("SGLang engine loaded.") + yield + # Clean up engine resources when the server stops (optional, depends on engine needs) + print("Shutting down SGLang engine...") + # engine.shutdown() # Or other cleanup if available/necessary + print("SGLang engine shutdown.") + + +app = FastAPI(lifespan=lifespan) + + +@app.post("/generate") +async def generate_text(request: Request): + """FastAPI endpoint to handle text generation requests.""" + global engine + if not engine: + return {"error": "Engine not initialized"}, 503 + + try: + data = await request.json() + prompt = data.get("prompt") + max_new_tokens = data.get("max_new_tokens", 128) + temperature = data.get("temperature", 0.7) + + if not prompt: + return {"error": "Prompt is required"}, 400 + + # Use async_generate for non-blocking generation + state = await engine.async_generate( + prompt, + sampling_params={ + "max_new_tokens": max_new_tokens, + "temperature": temperature, + }, + # Add other parameters like stop, top_p etc. as needed + ) + + return {"generated_text": state["text"]} + except Exception as e: + return {"error": str(e)}, 500 + + +# Helper function to start the server +def start_server(args, timeout=60): + """Starts the Uvicorn server as a subprocess and waits for it to be ready.""" + base_url = f"http://{args.host}:{args.port}" + command = [ + "python", + "-m", + "uvicorn", + "fastapi_engine_inference:app", + f"--host={args.host}", + f"--port={args.port}", + ] + + process = subprocess.Popen(command, stdout=None, stderr=None) + + start_time = time.perf_counter() + with requests.Session() as session: + while time.perf_counter() - start_time < timeout: + try: + # Check the /docs endpoint which FastAPI provides by default + response = session.get( + f"{base_url}/docs", timeout=5 + ) # Add a request timeout + if response.status_code == 200: + print(f"Server {base_url} is ready (responded on /docs)") + return process + except requests.ConnectionError: + # Specific exception for connection refused/DNS error etc. + pass + except requests.Timeout: + # Specific exception for request timeout + print(f"Health check to {base_url}/docs timed out, retrying...") + pass + except requests.RequestException as e: + # Catch other request exceptions + print(f"Health check request error: {e}, retrying...") + pass + # Use a shorter sleep interval for faster startup detection + time.sleep(1) + + # If loop finishes, raise the timeout error + # Attempt to terminate the failed process before raising + if process: + print( + "Server failed to start within timeout, attempting to terminate process..." + ) + terminate_process(process) # Use the imported terminate_process + raise TimeoutError( + f"Server failed to start at {base_url} within the timeout period." + ) + + +def send_requests(server_url, prompts, max_new_tokens, temperature): + """Sends generation requests to the running server for a list of prompts.""" + # Iterate through prompts and send requests + for i, prompt in enumerate(prompts): + print(f"\n[{i+1}/{len(prompts)}] Sending prompt: '{prompt}'") + payload = { + "prompt": prompt, + "max_new_tokens": max_new_tokens, + "temperature": temperature, + } + + try: + response = requests.post(f"{server_url}/generate", json=payload, timeout=60) + + result = response.json() + + print(f"Prompt: {prompt}\nResponse: {result['generated_text']}") + + except requests.exceptions.Timeout: + print(f" Error: Request timed out for prompt '{prompt}'") + except requests.exceptions.RequestException as e: + print(f" Error sending request for prompt '{prompt}': {e}") + + +if __name__ == "__main__": + """Main entry point for the script.""" + + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="127.0.0.1") + parser.add_argument("--port", type=int, default=8000) + parser.add_argument("--model-path", type=str, default="Qwen/Qwen2.5-0.5B-Instruct") + parser.add_argument("--tp_size", type=int, default=1) + parser.add_argument( + "--startup-timeout", + type=int, + default=60, + help="Time in seconds to wait for the server to be ready (default: %(default)s)", + ) + args = parser.parse_args() + + # Pass the model to the child uvicorn process via an env var + os.environ["MODEL_PATH"] = args.model_path + os.environ["TP_SIZE"] = str(args.tp_size) + + # Start the server + process = start_server(args, timeout=args.startup_timeout) + + # Define the prompts and sampling parameters + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + max_new_tokens = 64 + temperature = 0.1 + + # Define server url + server_url = f"http://{args.host}:{args.port}" + + # Send requests to the server + send_requests(server_url, prompts, max_new_tokens, temperature) + + # Terminate the server process + terminate_process(process) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/launch_engine.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/launch_engine.py new file mode 100644 index 00000000..b97e0bcd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/launch_engine.py @@ -0,0 +1,17 @@ +""" +This example demonstrates how to launch the offline engine. +""" + +import sglang as sgl + + +def main(): + llm = sgl.Engine(model_path="meta-llama/Meta-Llama-3.1-8B-Instruct") + llm.generate("What is the capital of France?") + llm.shutdown() + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/offline_batch_inference.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/offline_batch_inference.py new file mode 100644 index 00000000..92e68dcd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/offline_batch_inference.py @@ -0,0 +1,43 @@ +""" +Usage: +python3 offline_batch_inference.py --model meta-llama/Llama-3.1-8B-Instruct +""" + +import argparse +import dataclasses + +import sglang as sgl +from sglang.srt.server_args import ServerArgs + + +def main( + server_args: ServerArgs, +): + # Sample prompts. + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + # Create a sampling params object. + sampling_params = {"temperature": 0.8, "top_p": 0.95} + + # Create an LLM. + llm = sgl.Engine(**dataclasses.asdict(server_args)) + + outputs = llm.generate(prompts, sampling_params) + # Print the outputs. + for prompt, output in zip(prompts, outputs): + print("===============================") + print(f"Prompt: {prompt}\nGenerated text: {output['text']}") + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + args = parser.parse_args() + server_args = ServerArgs.from_cli_args(args) + main(server_args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/offline_batch_inference_async.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/offline_batch_inference_async.py new file mode 100644 index 00000000..578962d7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/offline_batch_inference_async.py @@ -0,0 +1,65 @@ +""" +Usage: +python offline_batch_inference_async.py --model-path Qwen/Qwen2-VL-7B-Instruct + +Note: +This demo shows the usage of async generation, +which is useful to implement an online-like generation with batched inference. +""" + +import argparse +import asyncio +import dataclasses +import time + +import sglang as sgl +from sglang.srt.server_args import ServerArgs + + +class InferenceEngine: + def __init__(self, **kwargs): + self.engine = sgl.Engine(**kwargs) + + async def generate(self, prompt, sampling_params): + result = await self.engine.async_generate(prompt, sampling_params) + return result + + +async def run_server(server_args): + inference = InferenceEngine(**dataclasses.asdict(server_args)) + + # Sample prompts. + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] * 100 + + # Create a sampling params object. + sampling_params = {"temperature": 0.8, "top_p": 0.95} + + # Run the generation tasks concurrently in async mode. + tasks = [] + for prompt in prompts: + task = asyncio.create_task(inference.generate(prompt, sampling_params)) + tasks.append(task) + + # Get and print the result + for task in tasks: + await task + while True: + if not task.done(): + time.sleep(1) + else: + result = task.result() + print(f"Generated text: {result['text']}") + break + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + args = parser.parse_args() + server_args = ServerArgs.from_cli_args(args) + asyncio.run(run_server(server_args)) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/offline_batch_inference_eagle.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/offline_batch_inference_eagle.py new file mode 100644 index 00000000..a7a89ef5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/offline_batch_inference_eagle.py @@ -0,0 +1,38 @@ +import sglang as sgl + + +def main(): + # Sample prompts. + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + + # Create a sampling params object. + sampling_params = {"temperature": 0, "max_new_tokens": 30} + + # Create an LLM. + llm = sgl.Engine( + model_path="meta-llama/Llama-2-7b-chat-hf", + speculative_algorithm="EAGLE", + speculative_draft_model_path="lmsys/sglang-EAGLE-llama2-chat-7B", + speculative_num_steps=3, + speculative_eagle_topk=4, + speculative_num_draft_tokens=16, + cuda_graph_max_bs=8, + ) + + outputs = llm.generate(prompts, sampling_params) + + # Print the outputs. + for prompt, output in zip(prompts, outputs): + print("===============================") + print(f"Prompt: {prompt}\nGenerated text: {output['text']}") + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/offline_batch_inference_qwen_1m.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/offline_batch_inference_qwen_1m.py new file mode 100644 index 00000000..664efa6d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/offline_batch_inference_qwen_1m.py @@ -0,0 +1,74 @@ +""" +Usage: +python3 offline_batch_inference.py +""" + +from urllib.request import urlopen + +import sglang as sgl + + +def load_prompt() -> str: + # Test cases with various lengths can be found at: + # + # https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-1M/test-data/64k.txt + # https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-1M/test-data/200k.txt + # https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-1M/test-data/600k.txt + # https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-1M/test-data/1m.txt + + with urlopen( + "https://qianwen-res.oss-cn-beijing.aliyuncs.com" + "/Qwen2.5-1M/test-data/64k.txt", + timeout=5, + ) as response: + prompt = response.read().decode("utf-8") + return prompt + + +# Processing the prompt. +def process_requests(llm: sgl.Engine, prompts: list[str]) -> None: + # Create a sampling params object. + sampling_params = { + "temperature": 0.7, + "top_p": 0.8, + "top_k": 20, + "repetition_penalty": 1.05, + "max_new_tokens": 256, + } + # Generate texts from the prompts. + outputs = llm.generate(prompts, sampling_params) + # Print the outputs. + for output in outputs: + prompt_token_ids = output["meta_info"]["prompt_tokens"] + generated_text = output["text"] + print( + f"Prompt length: {prompt_token_ids}, " f"Generated text: {generated_text!r}" + ) + + +# Create an LLM. +def initialize_engine() -> sgl.Engine: + llm = sgl.Engine( + model_path="Qwen/Qwen2.5-7B-Instruct-1M", + context_length=1048576, + page_size=256, + attention_backend="dual_chunk_flash_attn", + tp_size=4, + disable_radix_cache=True, + enable_mixed_chunk=False, + enable_torch_compile=False, + chunked_prefill_size=131072, + mem_fraction_static=0.6, + log_level="DEBUG", + ) + return llm + + +def main(): + llm = initialize_engine() + prompt = load_prompt() + process_requests(llm, [prompt]) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/offline_batch_inference_vlm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/offline_batch_inference_vlm.py new file mode 100644 index 00000000..939e6910 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/offline_batch_inference_vlm.py @@ -0,0 +1,52 @@ +""" +Usage: +python offline_batch_inference_vlm.py --model-path Qwen/Qwen2-VL-7B-Instruct +""" + +import argparse +import dataclasses + +import sglang as sgl +from sglang.srt.parser.conversation import chat_templates +from sglang.srt.server_args import ServerArgs + + +def main( + server_args: ServerArgs, +): + vlm = sgl.Engine(**dataclasses.asdict(server_args)) + + conv = chat_templates[server_args.chat_template].copy() + image_token = conv.image_token + + image_url = "https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true" + + prompt = f"What's in this image?\n{image_token}" + + sampling_params = { + "temperature": 0.001, + "max_new_tokens": 30, + } + + output = vlm.generate( + prompt=prompt, + image_data=image_url, + sampling_params=sampling_params, + ) + + print("===============================") + print(f"Prompt: {prompt}") + print(f"Generated text: {output['text']}") + + vlm.shutdown() + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + args = parser.parse_args() + + server_args = ServerArgs.from_cli_args(args) + main(server_args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/readme.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/readme.md new file mode 100644 index 00000000..fc6e89a6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/readme.md @@ -0,0 +1,54 @@ +# SGLang Engine + +SGLang provides a direct inference engine without the need for an HTTP server. There are generally these use cases: + +- [Offline Batch Inference](#offline-batch-inference) +- [Embedding Generation](#embedding-generation) +- [Custom Server](#custom-server) +- [Token-In-Token-Out for RLHF](#token-in-token-out-for-rlhf) +- [Inference Using FastAPI](#inference-using-fastapi) + +## Examples + +### [Offline Batch Inference](./offline_batch_inference.py) + +In this example, we launch an SGLang engine and feed a batch of inputs for inference. If you provide a very large batch, the engine will intelligently schedule the requests to process efficiently and prevent OOM (Out of Memory) errors. + +### [Embedding Generation](./embedding.py) + +In this example, we launch an SGLang engine and feed a batch of inputs for embedding generation. + +### [Custom Server](./custom_server.py) + +This example demonstrates how to create a custom server on top of the SGLang Engine. We use [Sanic](https://sanic.dev/en/) as an example. The server supports both non-streaming and streaming endpoints. + +#### Steps + +1. Install Sanic: + + ```bash + pip install sanic + ``` + +2. Run the server: + + ```bash + python custom_server + ``` + +3. Send requests: + + ```bash + curl -X POST http://localhost:8000/generate -H "Content-Type: application/json" -d '{"prompt": "The Transformer architecture is..."}' + curl -X POST http://localhost:8000/generate_stream -H "Content-Type: application/json" -d '{"prompt": "The Transformer architecture is..."}' --no-buffer + ``` + + This will send both non-streaming and streaming requests to the server. + +### [Token-In-Token-Out for RLHF](../token_in_token_out) + +In this example, we launch an SGLang engine, feed tokens as input and generate tokens as output. + +### [Inference Using FastAPI](fastapi_engine_inference.py) + +This example demonstrates how to create a FastAPI server that uses the SGLang engine for text generation. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/save_remote_state.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/save_remote_state.py new file mode 100644 index 00000000..a428195c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/save_remote_state.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +Saves each worker's model state dict directly to a checkpoint, which enables a +fast load path for large tensor-parallel models where each worker only needs to +read its own shard rather than the entire checkpoint. + +Example usage: + +python save_remote_state.py \ + --model-path /path/to/load \ + --tensor-parallel-size 8 \ + --remote-model-save-url [protocol]://[host]:[port]/[model_name] \ + +Then, the model can be loaded with + +llm = Engine( + model_path="[protocol]://[host]:[port]/[model_name]", + tensor_parallel_size=8, +) +""" +import dataclasses +from argparse import ArgumentParser +from pathlib import Path + +from sglang import Engine, ServerArgs + +parser = ArgumentParser() +ServerArgs.add_cli_args(parser) + +parser.add_argument( + "--remote-model-save-url", + required=True, + type=str, + help="remote address to store model weights", +) +parser.add_argument( + "--remote-draft-model-save-url", + default=None, + type=str, + help="remote address to store draft model weights", +) + + +def main(args): + engine_args = ServerArgs.from_cli_args(args) + model_path = engine_args.model_path + if not Path(model_path).is_dir(): + raise ValueError("model path must be a local directory") + # Create LLM instance from arguments + llm = Engine(**dataclasses.asdict(engine_args)) + llm.save_remote_model( + url=args.remote_model_save_url, draft_url=args.remote_draft_model_save_url + ) + print("save remote (draft) model successfully") + + +if __name__ == "__main__": + args = parser.parse_args() + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/save_sharded_state.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/save_sharded_state.py new file mode 100644 index 00000000..80ad5321 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/engine/save_sharded_state.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +Saves each worker's model state dict directly to a checkpoint, which enables a +fast load path for large tensor-parallel models where each worker only needs to +read its own shard rather than the entire checkpoint. + +Example usage: + +python save_sharded_state.py \ + --model-path /path/to/load \ + --quantization deepspeedfp \ + --tensor-parallel-size 8 \ + --output /path/to/save + +Then, the model can be loaded with + +llm = Engine( + model_path="/path/to/save", + load_format="sharded_state", + quantization="deepspeedfp", + tensor_parallel_size=8, +) +""" +import dataclasses +import os +import shutil +from argparse import ArgumentParser +from pathlib import Path + +from sglang import Engine, ServerArgs + +parser = ArgumentParser() +ServerArgs.add_cli_args(parser) + +parser.add_argument( + "--output", "-o", required=True, type=str, help="path to output checkpoint" +) +parser.add_argument( + "--file-pattern", type=str, help="string pattern of saved filenames" +) +parser.add_argument( + "--max-file-size", + type=str, + default=5 * 1024**3, + help="max size (in bytes) of each safetensors file", +) + + +def main(args): + engine_args = ServerArgs.from_cli_args(args) + model_path = engine_args.model_path + if not Path(model_path).is_dir(): + raise ValueError("model path must be a local directory") + # Create LLM instance from arguments + llm = Engine(**dataclasses.asdict(engine_args)) + Path(args.output).mkdir(exist_ok=True) + llm.save_sharded_model( + path=args.output, pattern=args.file_pattern, max_size=args.max_file_size + ) + + # Copy metadata files to output directory + for file in os.listdir(model_path): + if os.path.splitext(file)[1] not in (".bin", ".pt", ".safetensors"): + if os.path.isdir(os.path.join(model_path, file)): + shutil.copytree( + os.path.join(model_path, file), os.path.join(args.output, file) + ) + else: + shutil.copy(os.path.join(model_path, file), args.output) + + +if __name__ == "__main__": + args = parser.parse_args() + main(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/hidden_states/hidden_states_engine.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/hidden_states/hidden_states_engine.py new file mode 100644 index 00000000..60ab302c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/hidden_states/hidden_states_engine.py @@ -0,0 +1,66 @@ +""" +Usage: +python hidden_states.py + +Note that each time you change the `return_hidden_states` parameter, +the cuda graph will be recaptured, which might lead to a performance hit. +So avoid getting hidden states and completions alternately. +""" + +import torch + +import sglang as sgl + + +def main(): + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + # Create an LLM. + llm = sgl.Engine( + model_path="Alibaba-NLP/gte-Qwen2-1.5B-instruct", + enable_return_hidden_states=True, + ) + + sampling_params = { + "temperature": 0.8, + "top_p": 0.95, + "max_new_tokens": 10, + } + + outputs = llm.generate( + prompts, sampling_params=sampling_params, return_hidden_states=True + ) + + llm.shutdown() + + for prompt, output in zip(prompts, outputs): + for i in range(len(output["meta_info"]["hidden_states"])): + output["meta_info"]["hidden_states"][i] = torch.tensor( + output["meta_info"]["hidden_states"][i], dtype=torch.bfloat16 + ) + print("===============================") + print( + f"Prompt: {prompt}\n" + f"Generated text: {output['text']}\n" + f"Prompt_Tokens: {output['meta_info']['prompt_tokens']}\t" + f"Completion_tokens: {output['meta_info']['completion_tokens']}" + ) + print("Hidden states: ") + hidden_states = torch.cat( + [ + i.unsqueeze(0) if len(i.shape) == 1 else i + for i in output["meta_info"]["hidden_states"] + ] + ) + print(hidden_states) + print() + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/hidden_states/hidden_states_server.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/hidden_states/hidden_states_server.py new file mode 100644 index 00000000..b04f7437 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/hidden_states/hidden_states_server.py @@ -0,0 +1,81 @@ +""" +Usage: + +python hidden_states_server.py + +Note that each time you change the `return_hidden_states` parameter, +the cuda graph will be recaptured, which might lead to a performance hit. +So avoid getting hidden states and completions alternately. +""" + +import requests +import torch + +from sglang.test.test_utils import is_in_ci +from sglang.utils import terminate_process, wait_for_server + +if is_in_ci(): + from docs.backend.patch import launch_server_cmd +else: + from sglang.utils import launch_server_cmd + + +def main(): + # Launch the server + server_process, port = launch_server_cmd( + "python -m sglang.launch_server --model-path Alibaba-NLP/gte-Qwen2-1.5B-instruct --enable-return-hidden-states --host 0.0.0.0" + ) + wait_for_server(f"http://localhost:{port}") + + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + + sampling_params = { + "temperature": 0.8, + "top_p": 0.95, + "max_new_tokens": 10, + } + + json_data = { + "text": prompts, + "sampling_params": sampling_params, + "return_hidden_states": True, + } + + response = requests.post( + f"http://localhost:{port}/generate", + json=json_data, + ) + + terminate_process(server_process) + + outputs = response.json() + for prompt, output in zip(prompts, outputs): + for i in range(len(output["meta_info"]["hidden_states"])): + output["meta_info"]["hidden_states"][i] = torch.tensor( + output["meta_info"]["hidden_states"][i], dtype=torch.bfloat16 + ) + print("===============================") + print( + f"Prompt: {prompt}\n" + f"Generated text: {output['text']}\n" + f"Prompt_Tokens: {output['meta_info']['prompt_tokens']}\t" + f"Completion_tokens: {output['meta_info']['completion_tokens']}" + ) + print("Hidden states: ") + hidden_states = torch.cat( + [ + i.unsqueeze(0) if len(i.shape) == 1 else i + for i in output["meta_info"]["hidden_states"] + ] + ) + print(hidden_states) + print() + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/lora.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/lora.py new file mode 100644 index 00000000..181dc231 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/lora.py @@ -0,0 +1,67 @@ +""" +OpenAI-compatible LoRA adapter usage with SGLang. + +Server Setup: + python -m sglang.launch_server \\ + --model meta-llama/Llama-3.1-8B-Instruct \\ + --enable-lora \\ + --lora-paths sql=/path/to/sql python=/path/to/python +""" + +import openai + +client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="EMPTY") + + +def main(): + print("SGLang OpenAI-Compatible LoRA Examples\n") + + # Example 1: NEW - Adapter in model parameter (OpenAI-compatible) + print("1. Chat with LoRA adapter in model parameter:") + response = client.chat.completions.create( + model="meta-llama/Llama-3.1-8B-Instruct:sql", # ← adapter:name syntax + messages=[{"role": "user", "content": "Convert to SQL: show all users"}], + max_tokens=50, + ) + print(f" Response: {response.choices[0].message.content}\n") + + # Example 2: Completions API with adapter + print("2. Completion with LoRA adapter:") + response = client.completions.create( + model="meta-llama/Llama-3.1-8B-Instruct:python", + prompt="def fibonacci(n):", + max_tokens=50, + ) + print(f" Response: {response.choices[0].text}\n") + + # Example 3: OLD - Backward compatible with explicit lora_path + print("3. Backward compatible (explicit lora_path):") + response = client.chat.completions.create( + model="meta-llama/Llama-3.1-8B-Instruct", + messages=[{"role": "user", "content": "Convert to SQL: show all users"}], + extra_body={"lora_path": "sql"}, + max_tokens=50, + ) + print(f" Response: {response.choices[0].message.content}\n") + + # Example 4: Base model (no adapter) + print("4. Base model without adapter:") + response = client.chat.completions.create( + model="meta-llama/Llama-3.1-8B-Instruct", + messages=[{"role": "user", "content": "Hello!"}], + max_tokens=30, + ) + print(f" Response: {response.choices[0].message.content}\n") + + print("All examples completed!") + + +if __name__ == "__main__": + try: + main() + except Exception as e: + print(f"Error: {e}") + print( + "\nEnsure server is running:\n" + " python -m sglang.launch_server --model ... --enable-lora --lora-paths ..." + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/multimodal/llama3_llava_server.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/multimodal/llama3_llava_server.py new file mode 100644 index 00000000..a8409af7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/multimodal/llama3_llava_server.py @@ -0,0 +1,111 @@ +""" +Usage: +# Installing latest llava-next: pip install git+https://github.com/LLaVA-VL/LLaVA-NeXT.git +# Installing latest sglang. + +# Endpoint Service CLI: +python -m sglang.launch_server --model-path lmms-lab/llama3-llava-next-8b --port=30000 + +python3 llama3_llava_server.py + +Output: +"Friends posing for a fun photo with a life-sized teddy bear, creating a playful and memorable moment." +""" + +import argparse +import asyncio +import copy +import json + +import aiohttp +import requests +from llava.conversation import conv_llava_llama_3 + + +async def send_request(url, data, delay=0): + await asyncio.sleep(delay) + async with aiohttp.ClientSession() as session: + async with session.post(url, json=data) as resp: + output = await resp.json() + return output + + +async def test_concurrent(args): + url = f"{args.host}:{args.port}" + + prompt = "\nPlease generate caption towards this image." + conv_template = copy.deepcopy(conv_llava_llama_3) + conv_template.append_message(role=conv_template.roles[0], message=prompt) + conv_template.append_message(role=conv_template.roles[1], message=None) + prompt_with_template = conv_template.get_prompt() + response = [] + for i in range(1): + response.append( + send_request( + url + "/generate", + { + "text": prompt_with_template, + "image_data": "https://farm4.staticflickr.com/3175/2653711032_804ff86d81_z.jpg", + "sampling_params": { + "max_new_tokens": 1024, + "temperature": 0, + "top_p": 1.0, + "presence_penalty": 2, + "frequency_penalty": 2, + "stop": "<|eot_id|>", + }, + }, + ) + ) + + rets = await asyncio.gather(*response) + for ret in rets: + print(ret["text"]) + + +def test_streaming(args): + url = f"{args.host}:{args.port}" + prompt = "\nPlease generate caption towards this image." + conv_template = copy.deepcopy(conv_llava_llama_3) + conv_template.append_message(role=conv_template.roles[0], message=prompt) + conv_template.append_message(role=conv_template.roles[1], message=None) + prompt_with_template = conv_template.get_prompt() + pload = { + "text": prompt_with_template, + "sampling_params": { + "max_new_tokens": 1024, + "temperature": 0, + "top_p": 1.0, + "presence_penalty": 2, + "frequency_penalty": 2, + "stop": "<|eot_id|>", + }, + "image_data": "https://farm4.staticflickr.com/3175/2653711032_804ff86d81_z.jpg", + "stream": True, + } + response = requests.post( + url + "/generate", + json=pload, + stream=True, + ) + + prev = 0 + for chunk in response.iter_lines(decode_unicode=False): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]": + break + data = json.loads(chunk[5:].strip("\n")) + output = data["text"].strip() + print(output[prev:], end="", flush=True) + prev = len(output) + print("") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=30000) + args = parser.parse_args() + asyncio.run(test_concurrent(args)) + test_streaming(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/multimodal/llava_onevision_server.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/multimodal/llava_onevision_server.py new file mode 100644 index 00000000..2cf16e3b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/multimodal/llava_onevision_server.py @@ -0,0 +1,264 @@ +""" +Usage: + +python3 -m sglang.launch_server --model-path lmms-lab/llava-onevision-qwen2-72b-ov --port=30000 --tp-size=8 + +python3 llava_onevision_server.py +""" + +import io +import os +import sys +import time + +import numpy as np +import openai +import pybase64 +import requests +from decord import VideoReader, cpu +from PIL import Image + +# pip install httpx==0.23.3 +# pip install decord +# pip install protobuf==3.20.0 + + +def download_video(url, cache_dir): + file_path = os.path.join(cache_dir, "jobs.mp4") + os.makedirs(cache_dir, exist_ok=True) + + response = requests.get(url) + response.raise_for_status() + + with open(file_path, "wb") as f: + f.write(response.content) + + print(f"File downloaded and saved to: {file_path}") + return file_path + + +def create_openai_client(base_url): + return openai.Client(api_key="EMPTY", base_url=base_url) + + +def image_stream_request_test(client): + print("----------------------Image Stream Request Test----------------------") + stream_request = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://raw.githubusercontent.com/sgl-project/sglang/main/assets/logo.png" + }, + }, + { + "type": "text", + "text": "Please describe this image. Please list the benchmarks and the models.", + }, + ], + }, + ], + temperature=0.7, + max_tokens=1024, + stream=True, + ) + stream_response = "" + + for chunk in stream_request: + if chunk.choices[0].delta.content is not None: + content = chunk.choices[0].delta.content + stream_response += content + sys.stdout.write(content) + sys.stdout.flush() + + print("-" * 30) + + +def multi_image_stream_request_test(client): + print( + "----------------------Multi-Images Stream Request Test----------------------" + ) + stream_request = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://raw.githubusercontent.com/sgl-project/sglang/main/assets/logo.png" + }, + "modalities": "multi-images", + }, + { + "type": "image_url", + "image_url": { + "url": "https://raw.githubusercontent.com/sgl-project/sglang/main/examples/assets/example_image.png" + }, + "modalities": "multi-images", + }, + { + "type": "text", + "text": "I have shown you two images. Please describe the two images to me.", + }, + ], + }, + ], + temperature=0.7, + max_tokens=1024, + stream=True, + ) + stream_response = "" + + for chunk in stream_request: + if chunk.choices[0].delta.content is not None: + content = chunk.choices[0].delta.content + stream_response += content + sys.stdout.write(content) + sys.stdout.flush() + + print("-" * 30) + + +def video_stream_request_test(client, video_path): + print("------------------------Video Stream Request Test----------------------") + messages = prepare_video_messages(video_path) + + video_request = client.chat.completions.create( + model="default", + messages=messages, + temperature=0, + max_tokens=1024, + stream=True, + ) + print("-" * 30) + video_response = "" + + for chunk in video_request: + if chunk.choices[0].delta.content is not None: + content = chunk.choices[0].delta.content + video_response += content + sys.stdout.write(content) + sys.stdout.flush() + print("-" * 30) + + +def image_speed_test(client): + print("----------------------Image Speed Test----------------------") + start_time = time.perf_counter() + request = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://raw.githubusercontent.com/sgl-project/sglang/main/assets/logo.png" + }, + }, + { + "type": "text", + "text": "Please describe this image. Please list the benchmarks and the models.", + }, + ], + }, + ], + temperature=0, + max_tokens=1024, + ) + end_time = time.perf_counter() + response = request.choices[0].message.content + print(response) + print("-" * 30) + print_speed_test_results(request, start_time, end_time) + + +def video_speed_test(client, video_path): + print("------------------------Video Speed Test------------------------") + messages = prepare_video_messages(video_path) + + start_time = time.perf_counter() + video_request = client.chat.completions.create( + model="default", + messages=messages, + temperature=0, + max_tokens=1024, + ) + end_time = time.perf_counter() + video_response = video_request.choices[0].message.content + print(video_response) + print("-" * 30) + print_speed_test_results(video_request, start_time, end_time) + + +def prepare_video_messages(video_path): + max_frames_num = 32 + vr = VideoReader(video_path, ctx=cpu(0)) + total_frame_num = len(vr) + uniform_sampled_frames = np.linspace( + 0, total_frame_num - 1, max_frames_num, dtype=int + ) + frame_idx = uniform_sampled_frames.tolist() + frames = vr.get_batch(frame_idx).asnumpy() + + base64_frames = [] + for frame in frames: + pil_img = Image.fromarray(frame) + buff = io.BytesIO() + pil_img.save(buff, format="JPEG") + base64_str = pybase64.b64encode(buff.getvalue()).decode("utf-8") + base64_frames.append(base64_str) + + messages = [{"role": "user", "content": []}] + + for base64_frame in base64_frames: + frame_format = { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{base64_frame}"}, + "modalities": "video", + } + messages[0]["content"].append(frame_format) + + prompt = {"type": "text", "text": "Please describe the video in detail."} + messages[0]["content"].append(prompt) + + return messages + + +def print_speed_test_results(request, start_time, end_time): + total_tokens = request.usage.total_tokens + completion_tokens = request.usage.completion_tokens + prompt_tokens = request.usage.prompt_tokens + + print(f"Total tokens: {total_tokens}") + print(f"Completion tokens: {completion_tokens}") + print(f"Prompt tokens: {prompt_tokens}") + print(f"Time taken: {end_time - start_time} seconds") + print(f"Token per second: {total_tokens / (end_time - start_time)}") + print(f"Completion token per second: {completion_tokens / (end_time - start_time)}") + print(f"Prompt token per second: {prompt_tokens / (end_time - start_time)}") + + +def main(): + url = "https://raw.githubusercontent.com/EvolvingLMMs-Lab/sglang/dev/onevision_local/assets/jobs.mp4" + cache_dir = os.path.expanduser("~/.cache") + video_path = download_video(url, cache_dir) + + client = create_openai_client("http://127.0.0.1:30000/v1") + + image_stream_request_test(client) + multi_image_stream_request_test(client) + video_stream_request_test(client, video_path) + image_speed_test(client) + video_speed_test(client, video_path) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/multimodal/pixtral_server.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/multimodal/pixtral_server.py new file mode 100644 index 00000000..d907de14 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/multimodal/pixtral_server.py @@ -0,0 +1,127 @@ +""" +Usage: +# Run a Pixtral model with SGLang: +# HuggingFace: +python -m sglang.launch_server --model-path mistral-community/pixtral-12b --port=30000 +# ModelScope: +python -m sglang.launch_server --model-path AI-ModelScope/pixtral-12b --port=30000 + +# Then test it with: +python pixtral_server.py + +This script tests Pixtral model with both single and multiple images. +""" + +import argparse +import asyncio +import json + +import aiohttp +import requests + +IMAGE_TOKEN_SEP = "\n[IMG]" +ROUTE = "/generate" + + +async def send_request(url, data, delay=0): + await asyncio.sleep(delay) + async with aiohttp.ClientSession() as session: + async with session.post(url, json=data) as resp: + output = await resp.json() + return output + + +async def test_concurrent(args): + url = f"{args.host}:{args.port}{ROUTE}" + + # Single image test + if args.single_image: + prompt = f"[INST]Describe this image in detail.{IMAGE_TOKEN_SEP}[/INST]" + image_url = "https://picsum.photos/id/237/400/300" + modality = ["image"] + # Multiple images test + else: + image_urls = [ + "https://picsum.photos/id/237/400/300", + "https://picsum.photos/id/27/500/500", + ] + prompt = f"[INST]How many photos are there? Describe each in a very short sentence.{IMAGE_TOKEN_SEP * len(image_urls)}[/INST]" + image_url = image_urls + modality = ["multi-images"] + + response = await send_request( + url, + { + "text": prompt, + "image_data": image_url, + "sampling_params": { + "max_new_tokens": 100, + "temperature": 0.7, + "top_p": 0.9, + }, + "modalities": modality, + }, + ) + + print(f"Response: {response}") + if "text" in response: + print("\nOutput text:", response["text"]) + + +def test_streaming(args): + url = f"{args.host}:{args.port}/generate" + + # Single image test + if args.single_image: + prompt = f"[INST]Describe this image in detail.{IMAGE_TOKEN_SEP}[/INST]" + image_data = "https://picsum.photos/id/237/400/300" + modality = ["image"] + # Multiple images test + else: + image_urls = [ + "https://picsum.photos/id/237/400/300", + "https://picsum.photos/id/27/500/500", + ] + prompt = f"[INST]How many photos are there? Describe each in a very short sentence.{IMAGE_TOKEN_SEP * len(image_urls)}[/INST]" + image_data = image_urls + modality = ["multi-images"] + + pload = { + "text": prompt, + "image_data": image_data, + "sampling_params": {"max_new_tokens": 100, "temperature": 0.7, "top_p": 0.9}, + "modalities": modality, + "stream": True, + } + + response = requests.post(url, json=pload, stream=True) + + print("Streaming response:") + prev = 0 + for chunk in response.iter_lines(decode_unicode=False): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]": + break + data = json.loads(chunk[5:].strip("\n")) + output = data["text"].strip() + print(output[prev:], end="", flush=True) + prev = len(output) + print("\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=30000) + parser.add_argument( + "--single-image", + action="store_true", + help="Test with single image instead of multiple images", + ) + parser.add_argument("--no-stream", action="store_true", help="Don't test streaming") + args = parser.parse_args() + + asyncio.run(test_concurrent(args)) + if not args.no_stream: + test_streaming(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/multimodal/qwen_llava_server.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/multimodal/qwen_llava_server.py new file mode 100644 index 00000000..d8b3226e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/multimodal/qwen_llava_server.py @@ -0,0 +1,111 @@ +""" +Usage: +# Installing latest llava-next: pip install git+https://github.com/LLaVA-VL/LLaVA-NeXT.git +# Installing latest sglang. + +# Endpoint Service CLI: +python -m sglang.launch_server --model-path lmms-lab/llava-next-72b --port=30000 --tp-size=8 + +python3 qwen_llava_server.py + +Output: +"Two children pose with a large teddy bear, one holding a smaller stuffed bear, in a room with an American flag and potted plants." +""" + +import argparse +import asyncio +import copy +import json + +import aiohttp +import requests +from llava.conversation import conv_qwen + + +async def send_request(url, data, delay=0): + await asyncio.sleep(delay) + async with aiohttp.ClientSession() as session: + async with session.post(url, json=data) as resp: + output = await resp.json() + return output + + +async def test_concurrent(args): + url = f"{args.host}:{args.port}" + + prompt = "\nPlease generate caption towards this image." + conv_template = copy.deepcopy(conv_qwen) + conv_template.append_message(role=conv_template.roles[0], message=prompt) + conv_template.append_message(role=conv_template.roles[1], message=None) + prompt_with_template = conv_template.get_prompt() + response = [] + for i in range(1): + response.append( + send_request( + url + "/generate", + { + "text": prompt_with_template, + "image_data": "https://farm4.staticflickr.com/3175/2653711032_804ff86d81_z.jpg", + "sampling_params": { + "max_new_tokens": 1024, + "temperature": 0, + "top_p": 1.0, + "presence_penalty": 2, + "frequency_penalty": 2, + "stop": "<|im_end|>", + }, + }, + ) + ) + + rets = await asyncio.gather(*response) + for ret in rets: + print(ret["text"]) + + +def test_streaming(args): + url = f"{args.host}:{args.port}" + prompt = "\nPlease generate caption towards this image." + conv_template = copy.deepcopy(conv_qwen) + conv_template.append_message(role=conv_template.roles[0], message=prompt) + conv_template.append_message(role=conv_template.roles[1], message=None) + prompt_with_template = conv_template.get_prompt() + pload = { + "text": prompt_with_template, + "sampling_params": { + "max_new_tokens": 1024, + "temperature": 0, + "top_p": 1.0, + "presence_penalty": 2, + "frequency_penalty": 2, + "stop": "<|im_end|>", + }, + "image_data": "https://farm4.staticflickr.com/3175/2653711032_804ff86d81_z.jpg", + "stream": True, + } + response = requests.post( + url + "/generate", + json=pload, + stream=True, + ) + + prev = 0 + for chunk in response.iter_lines(decode_unicode=False): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]": + break + data = json.loads(chunk[5:].strip("\n")) + output = data["text"].strip() + print(output[prev:], end="", flush=True) + prev = len(output) + print("") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=30000) + args = parser.parse_args() + asyncio.run(test_concurrent(args)) + test_streaming(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/multimodal_embedding.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/multimodal_embedding.py new file mode 100644 index 00000000..4e8d748b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/multimodal_embedding.py @@ -0,0 +1,18 @@ +# launch server +# python -m sglang.launch_server --model-path Alibaba-NLP/gme-Qwen2-VL-2B-Instruct --is-embedding + +import requests + +url = "http://127.0.0.1:30000" + +text_input = "Represent this image in embedding space." +image_path = "https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild/resolve/main/images/023.jpg" + +payload = { + "model": "gme-qwen2-vl", + "input": [{"text": text_input}, {"image": image_path}], +} + +response = requests.post(url + "/v1/embeddings", json=payload).json() + +print("Embeddings:", [x.get("embedding") for x in response.get("data", [])]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/openai_chat_with_response_prefill.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/openai_chat_with_response_prefill.py new file mode 100644 index 00000000..6d803a1d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/openai_chat_with_response_prefill.py @@ -0,0 +1,53 @@ +""" +Usage: +1) Launch the server in one terminal: + python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --port 30000 + +2) Run this script in another terminal: + python openai_chat_with_response_prefill.py + +This example demonstrates two chat completion calls: +- One with continue_final_message enabled (the final assistant message is used as a prefill). +- One without continue_final_message (the final assistant message remains, starting a new turn). +""" + +import openai + +client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="EMPTY") + +messages = [ + {"role": "system", "content": "You are a helpful AI assistant."}, + { + "role": "user", + "content": """ +Extract the name, size, price, and color from this product description as a JSON object: + + +The SmartHome Mini is a compact smart home assistant available in black or white for only $49.99. +At just 5 inches wide, it lets you control lights, thermostats, and other connected devices via voice or app— +no matter where you place it in your home. +This affordable little hub brings convenient hands-free control to your smart devices. + +""", + }, + {"role": "assistant", "content": "{\n"}, +] + +# Calling the API with continue_final_message enabled. +print("=== Prefill with continue_final_messagem ===") +response_with = client.chat.completions.create( + model="meta-llama/Llama-3.1-8B-Instruct", + messages=messages, + temperature=0, + extra_body={"continue_final_message": True}, +) +print(response_with.choices[0].message.content) + +# Calling the API without continue_final_message (using default behavior). +print("\n=== Prefill without continue_final_message ===") +response_without = client.chat.completions.create( + model="meta-llama/Llama-3.1-8B-Instruct", + messages=messages, + temperature=0, +) +print(response_without.choices[0].message.content) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/reward_model.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/reward_model.py new file mode 100644 index 00000000..1a1177e6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/reward_model.py @@ -0,0 +1,32 @@ +# launch server +# python -m sglang.launch_server --model LxzGordon/URM-LLaMa-3.1-8B --is-embedding + +import requests + +url = "http://127.0.0.1:30000" + +PROMPT = ( + "What is the range of the numeric output of a sigmoid node in a neural network?" +) +RESPONSE1 = "The output of a sigmoid node is bounded between -1 and 1." +RESPONSE2 = "The output of a sigmoid node is bounded between 0 and 1." + +json_data = { + "conv": [ + [ + {"role": "user", "content": PROMPT}, + {"role": "assistant", "content": RESPONSE1}, + ], + [ + {"role": "user", "content": PROMPT}, + {"role": "assistant", "content": RESPONSE2}, + ], + ], +} +response = requests.post( + url + "/classify", + json=json_data, +).json() + +print(response) +print("scores:", [x["embedding"] for x in response]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/token_in_token_out/token_in_token_out_llm_engine.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/token_in_token_out/token_in_token_out_llm_engine.py new file mode 100644 index 00000000..11453f93 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/token_in_token_out/token_in_token_out_llm_engine.py @@ -0,0 +1,43 @@ +""" +This example demonstrates how to provide tokenized ids to LLM as input instead of text prompt, i.e. a token-in-token-out workflow. +""" + +import sglang as sgl +from sglang.srt.utils.hf_transformers_utils import get_tokenizer + +MODEL_PATH = "meta-llama/Llama-3.1-8B-Instruct" + + +def main(): + # Sample prompts. + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + # Create a sampling params object. + sampling_params = {"temperature": 0.8, "top_p": 0.95} + + # Tokenize inputs + tokenizer = get_tokenizer(MODEL_PATH) + token_ids_list = [tokenizer.encode(prompt) for prompt in prompts] + + # Create an LLM. + llm = sgl.Engine(model_path=MODEL_PATH, skip_tokenizer_init=True) + + outputs = llm.generate(input_ids=token_ids_list, sampling_params=sampling_params) + # Print the outputs. + for prompt, output in zip(prompts, outputs): + decode_output = tokenizer.decode(output["output_ids"]) + print("===============================") + print( + f"Prompt: {prompt}\nGenerated token ids: {output['output_ids']}\nGenerated text: {decode_output}" + ) + print() + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/token_in_token_out/token_in_token_out_llm_server.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/token_in_token_out/token_in_token_out_llm_server.py new file mode 100644 index 00000000..7e498f51 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/token_in_token_out/token_in_token_out_llm_server.py @@ -0,0 +1,68 @@ +""" +Usage: + +python token_in_token_out_llm_server.py + +""" + +import requests + +from sglang.srt.utils.hf_transformers_utils import get_tokenizer +from sglang.test.test_utils import is_in_ci +from sglang.utils import terminate_process, wait_for_server + +if is_in_ci(): + from docs.backend.patch import launch_server_cmd +else: + from sglang.utils import launch_server_cmd + + +MODEL_PATH = "meta-llama/Llama-3.1-8B-Instruct" + + +def main(): + # Launch the server + server_process, port = launch_server_cmd( + f"python -m sglang.launch_server --model-path {MODEL_PATH} --skip-tokenizer-init --host 0.0.0.0" + ) + wait_for_server(f"http://localhost:{port}") + + # Sample prompts. + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + + # Create a sampling params object. + sampling_params = {"temperature": 0.8, "top_p": 0.95} + + # Tokenize inputs + tokenizer = get_tokenizer(MODEL_PATH) + token_ids_list = [tokenizer.encode(prompt) for prompt in prompts] + + json_data = { + "input_ids": token_ids_list, + "sampling_params": sampling_params, + } + + response = requests.post( + f"http://localhost:{port}/generate", + json=json_data, + ) + + outputs = response.json() + for prompt, output in zip(prompts, outputs): + print("===============================") + decode_output = tokenizer.decode(output["output_ids"]) + print( + f"Prompt: {prompt}\nGenerated token ids: {output['output_ids']}\nGenerated text: {decode_output}" + ) + print() + + terminate_process(server_process) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/token_in_token_out/token_in_token_out_vlm_engine.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/token_in_token_out/token_in_token_out_vlm_engine.py new file mode 100644 index 00000000..bf6f1971 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/token_in_token_out/token_in_token_out_vlm_engine.py @@ -0,0 +1,74 @@ +import argparse +import dataclasses +from typing import Tuple + +from transformers import AutoProcessor + +from sglang import Engine +from sglang.lang.chat_template import get_chat_template_by_model_path +from sglang.srt.configs.model_config import ModelConfig +from sglang.srt.server_args import ServerArgs +from sglang.test.test_utils import DEFAULT_IMAGE_URL + + +def get_input_ids( + server_args: ServerArgs, model_config: ModelConfig +) -> Tuple[list[int], list]: + chat_template = get_chat_template_by_model_path(model_config.model_path) + text = f"{chat_template.image_token}What is in this picture?" + image_data = [DEFAULT_IMAGE_URL] + + processor = AutoProcessor.from_pretrained( + model_config.model_path, trust_remote_code=server_args.trust_remote_code + ) + + input_ids = ( + processor.tokenizer( + text=[text], + return_tensors="pt", + ) + .input_ids[0] + .tolist() + ) + + return input_ids, image_data + + +def token_in_out_example( + server_args: ServerArgs, +): + input_ids, image_data = get_input_ids( + server_args, + ModelConfig( + server_args.model_path, + trust_remote_code=server_args.trust_remote_code, + model_override_args=server_args.json_model_override_args, + ), + ) + backend = Engine(**dataclasses.asdict(server_args)) + + output = backend.generate( + input_ids=input_ids, + image_data=image_data, + sampling_params={ + "temperature": 0.8, + "max_new_tokens": 32, + }, + ) + + print("===============================") + print(f"Output token ids: ", output["output_ids"]) + + backend.shutdown() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + args = [ + "--model-path=Qwen/Qwen2-VL-2B", + ] + args = parser.parse_args(args=args) + server_args = ServerArgs.from_cli_args(args) + server_args.skip_tokenizer_init = True + token_in_out_example(server_args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/token_in_token_out/token_in_token_out_vlm_server.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/token_in_token_out/token_in_token_out_vlm_server.py new file mode 100644 index 00000000..392e1bf0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/token_in_token_out/token_in_token_out_vlm_server.py @@ -0,0 +1,78 @@ +""" +Usage: + +python token_in_token_out_vlm_server.py + +""" + +from typing import Tuple + +import requests +from transformers import AutoProcessor + +from sglang.lang.chat_template import get_chat_template_by_model_path +from sglang.test.test_utils import DEFAULT_IMAGE_URL, is_in_ci +from sglang.utils import terminate_process, wait_for_server + +if is_in_ci(): + from docs.backend.patch import launch_server_cmd +else: + from sglang.utils import launch_server_cmd + + +MODEL_PATH = "Qwen/Qwen2-VL-2B" + + +def get_input_ids() -> Tuple[list[int], list]: + chat_template = get_chat_template_by_model_path(MODEL_PATH) + text = f"{chat_template.image_token}What is in this picture?" + image_data = [DEFAULT_IMAGE_URL] + + processor = AutoProcessor.from_pretrained(MODEL_PATH) + + input_ids = ( + processor.tokenizer( + text=[text], + return_tensors="pt", + ) + .input_ids[0] + .tolist() + ) + + return input_ids, image_data + + +def main(): + # Launch the server + server_process, port = launch_server_cmd( + f"python -m sglang.launch_server --model-path {MODEL_PATH} --skip-tokenizer-init --host 0.0.0.0" + ) + wait_for_server(f"http://localhost:{port}") + + input_ids, image_data = get_input_ids() + + sampling_params = { + "temperature": 0.8, + "max_new_tokens": 32, + } + + json_data = { + "input_ids": input_ids, + "image_data": image_data, + "sampling_params": sampling_params, + } + + response = requests.post( + f"http://localhost:{port}/generate", + json=json_data, + ) + + output = response.json() + print("===============================") + print(f"Output token ids: ", output["output_ids"]) + + terminate_process(server_process) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/vertex_predict.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/vertex_predict.py new file mode 100644 index 00000000..58a41b1c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/runtime/vertex_predict.py @@ -0,0 +1,66 @@ +""" +Usage: +python -m sglang.launch_server --model meta-llama/Llama-2-7b-hf --port 30000 +python vertex_predict.py + +This example shows the request and response formats of the prediction route for +Google Cloud Vertex AI Online Predictions. + +Vertex AI SDK for Python is recommended for deploying models to Vertex AI +instead of a local server. After deploying the model to a Vertex AI Online +Prediction Endpoint, send requests via the Python SDK: + +response = endpoint.predict( + instances=[ + {"text": "The capital of France is"}, + {"text": "What is a car?"}, + ], + parameters={"sampling_params": {"max_new_tokens": 16}}, +) +print(response.predictions) + +More details about get online predictions from Vertex AI can be found at +https://cloud.google.com/vertex-ai/docs/predictions/get-online-predictions. +""" + +from dataclasses import dataclass +from typing import List, Optional + +import requests + + +@dataclass +class VertexPrediction: + predictions: List + + +class LocalVertexEndpoint: + def __init__(self) -> None: + self.base_url = "http://127.0.0.1:30000" + + def predict(self, instances: List[dict], parameters: Optional[dict] = None): + response = requests.post( + self.base_url + "/vertex_generate", + json={ + "instances": instances, + "parameters": parameters, + }, + ) + return VertexPrediction(predictions=response.json()["predictions"]) + + +endpoint = LocalVertexEndpoint() + +# Predict with a single prompt. +response = endpoint.predict(instances=[{"text": "The capital of France is"}]) +print(response.predictions) + +# Predict with multiple prompts and parameters. +response = endpoint.predict( + instances=[ + {"text": "The capital of France is"}, + {"text": "What is a car?"}, + ], + parameters={"sampling_params": {"max_new_tokens": 16}}, +) +print(response.predictions) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/sagemaker/deploy_and_serve_endpoint.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/sagemaker/deploy_and_serve_endpoint.py new file mode 100644 index 00000000..e518183c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/sagemaker/deploy_and_serve_endpoint.py @@ -0,0 +1,69 @@ +import json + +import boto3 +from sagemaker import serializers +from sagemaker.model import Model +from sagemaker.predictor import Predictor + +boto_session = boto3.session.Session() +sm_client = boto_session.client("sagemaker") +sm_role = boto_session.resource("iam").Role("SageMakerRole").arn + +endpoint_name = "" +image_uri = "" +model_id = ( + "" # eg: Qwen/Qwen3-0.6B from https://huggingface.co/Qwen/Qwen3-0.6B +) +hf_token = "" +prompt = "" + +model = Model( + name=endpoint_name, + image_uri=image_uri, + role=sm_role, + env={ + "SM_SGLANG_MODEL_PATH": model_id, + "HF_TOKEN": hf_token, + }, +) +print("Model created successfully") +print("Starting endpoint deployment (this may take 10-15 minutes)...") + +endpoint_config = model.deploy( + instance_type="ml.g5.12xlarge", + initial_instance_count=1, + endpoint_name=endpoint_name, + inference_ami_version="al2-ami-sagemaker-inference-gpu-3-1", + wait=True, +) +print("Endpoint deployment completed successfully") + + +print(f"Creating predictor for endpoint: {endpoint_name}") +predictor = Predictor( + endpoint_name=endpoint_name, + serializer=serializers.JSONSerializer(), +) + +payload = { + "model": model_id, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": 2400, + "temperature": 0.01, + "top_p": 0.9, + "top_k": 50, +} +print(f"Sending inference request with prompt: '{prompt[:50]}...'") +response = predictor.predict(payload) +print("Inference request completed successfully") + +if isinstance(response, bytes): + response = response.decode("utf-8") + +if isinstance(response, str): + try: + response = json.loads(response) + except json.JSONDecodeError: + print("Warning: Response is not valid JSON. Returning as string.") + +print(f"Received model response: '{response}'") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/usage/modelopt_quantize_and_export.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/usage/modelopt_quantize_and_export.py new file mode 100755 index 00000000..4394d917 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/examples/usage/modelopt_quantize_and_export.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +""" +Example: ModelOpt Quantization and Export with SGLang + +This example demonstrates the streamlined workflow for quantizing a model with +ModelOpt and automatically exporting it for deployment with SGLang. +""" + +import argparse +import os +from typing import Optional + +import torch + +import sglang as sgl +from sglang.srt.configs.device_config import DeviceConfig +from sglang.srt.configs.load_config import LoadConfig +from sglang.srt.configs.model_config import ModelConfig +from sglang.srt.distributed.parallel_state import ( + init_distributed_environment, + initialize_model_parallel, +) +from sglang.srt.model_loader.loader import get_model_loader + + +def _validate_export(export_dir: str) -> bool: + """Validate that an exported model directory contains the expected files.""" + import glob + + required_files = ["config.json", "tokenizer_config.json"] + + if not os.path.exists(export_dir): + return False + + # Check required files + for file in required_files: + if not os.path.exists(os.path.join(export_dir, file)): + return False + + # Check for model files using pattern matching to handle sharded models + model_patterns = [ + "model*.safetensors", + "pytorch_model*.bin", + ] + + has_model_file = False + for pattern in model_patterns: + matching_files = glob.glob(os.path.join(export_dir, pattern)) + if matching_files: + has_model_file = True + break + + return has_model_file + + +def _get_export_info(export_dir: str) -> Optional[dict]: + """Get information about an exported model.""" + import json + + if not _validate_export(export_dir): + return None + + try: + config_path = os.path.join(export_dir, "config.json") + with open(config_path, "r") as f: + config = json.load(f) + + return { + "model_type": config.get("model_type", "unknown"), + "architectures": config.get("architectures", []), + "quantization_config": config.get("quantization_config", {}), + "export_dir": export_dir, + } + except Exception: + return None + + +def quantize_and_export_model( + model_path: str, + export_dir: str, + quantization_method: str = "modelopt_fp8", + checkpoint_save_path: Optional[str] = None, + device: str = "cuda", +) -> None: + """ + Quantize a model with ModelOpt and export it for SGLang deployment. + + Args: + model_path: Path to the original model + export_dir: Directory to export the quantized model + quantization_method: Quantization method ("modelopt_fp8" or "modelopt_fp4") + checkpoint_save_path: Optional path to save ModelOpt checkpoint + device: Device to use for quantization + """ + print("🚀 Starting ModelOpt quantization and export workflow") + print(f"📥 Input model: {model_path}") + print(f"📤 Export directory: {export_dir}") + print(f"⚙️ Quantization method: {quantization_method}") + + # Initialize minimal distributed environment for single GPU quantization + if not torch.distributed.is_initialized(): + print("🔧 Initializing distributed environment...") + # Set up environment variables for single-process distributed + os.environ["RANK"] = "0" + os.environ["WORLD_SIZE"] = "1" + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = "12355" # Use a different port than tests + os.environ["LOCAL_RANK"] = "0" + + init_distributed_environment( + world_size=1, + rank=0, + local_rank=0, + backend="nccl" if device == "cuda" else "gloo", + ) + initialize_model_parallel( + tensor_model_parallel_size=1, + pipeline_model_parallel_size=1, + ) + + # Configure model loading with ModelOpt quantization and export + model_config = ModelConfig( + model_path=model_path, + quantization=quantization_method, # Use unified quantization flag + trust_remote_code=True, + ) + + load_config = LoadConfig( + modelopt_checkpoint_save_path=checkpoint_save_path, + modelopt_export_path=export_dir, + ) + device_config = DeviceConfig(device=device) + + # Load and quantize the model (export happens automatically) + print("🔄 Loading and quantizing model...") + model_loader = get_model_loader(load_config, model_config) + + try: + model_loader.load_model( + model_config=model_config, + device_config=device_config, + ) + print("✅ Model quantized successfully!") + + # Validate the export + if _validate_export(export_dir): + print("✅ Export validation passed!") + + info = _get_export_info(export_dir) + if info: + print("📋 Model info:") + print(f" - Type: {info['model_type']}") + print(f" - Architecture: {info['architectures']}") + print(f" - Quantization: {info['quantization_config']}") + else: + print("❌ Export validation failed!") + return + + except Exception as e: + print(f"❌ Quantization failed: {e}") + return + + print("\n🎉 Workflow completed successfully!") + print(f"📁 Quantized model exported to: {export_dir}") + print("\n🚀 To use the exported model:") + print( + f" python -m sglang.launch_server --model-path {export_dir} --quantization modelopt" + ) + print("\n # Or in Python:") + print(" import sglang as sgl") + print(f" llm = sgl.Engine(model_path='{export_dir}', quantization='modelopt')") + print(" # Note: 'modelopt' auto-detects FP4/FP8 from model config") + + +def deploy_exported_model( + export_dir: str, + host: str = "127.0.0.1", + port: int = 30000, +) -> None: + """ + Deploy an exported ModelOpt quantized model with SGLang. + + Args: + export_dir: Directory containing the exported model + host: Host to bind the server to + port: Port to bind the server to + """ + print(f"🚀 Deploying exported model from: {export_dir}") + + # Validate export first + if not _validate_export(export_dir): + print("❌ Invalid export directory!") + return + + try: + # Launch SGLang engine with the exported model + # Using generic "modelopt" for auto-detection of FP4/FP8 + llm = sgl.Engine( + model_path=export_dir, + quantization="modelopt", + host=host, + port=port, + ) + + print("✅ Model deployed successfully!") + print(f"🌐 Server running at http://{host}:{port}") + + # Example inference + prompts = ["Hello, how are you?", "What is the capital of France?"] + sampling_params = {"temperature": 0.8, "top_p": 0.95, "max_new_tokens": 100} + + print("\n🧪 Running example inference...") + outputs = llm.generate(prompts, sampling_params) + + for i, output in enumerate(outputs): + print(f"Prompt {i+1}: {prompts[i]}") + print(f"Output: {output['text']}") + print() + + except Exception as e: + print(f"❌ Deployment failed: {e}") + + +def main(): + parser = argparse.ArgumentParser( + description="ModelOpt Quantization and Export with SGLang", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Quantize and export a model (recommended workflow) + python modelopt_quantize_and_export.py quantize \\ + --model-path TinyLlama/TinyLlama-1.1B-Chat-v1.0 \\ + --export-dir ./quantized_model \\ + --quantization-method modelopt_fp8 + + # Deploy a pre-exported model + python modelopt_quantize_and_export.py deploy \\ + --export-dir ./quantized_model + """, + ) + + subparsers = parser.add_subparsers(dest="command", help="Available commands") + + # Quantize command + quantize_parser = subparsers.add_parser( + "quantize", help="Quantize and export a model" + ) + quantize_parser.add_argument( + "--model-path", required=True, help="Path to the model to quantize" + ) + quantize_parser.add_argument( + "--export-dir", required=True, help="Directory to export the quantized model" + ) + quantize_parser.add_argument( + "--quantization-method", + choices=["modelopt_fp8", "modelopt_fp4"], + default="modelopt_fp8", + help="Quantization method to use", + ) + quantize_parser.add_argument( + "--checkpoint-save-path", help="Optional path to save ModelOpt checkpoint" + ) + quantize_parser.add_argument( + "--device", default="cuda", help="Device to use for quantization" + ) + + # TODO: Quantize-and-serve command removed due to compatibility issues + # Use the separate quantize-then-deploy workflow instead + + # Deploy command + deploy_parser = subparsers.add_parser("deploy", help="Deploy an exported model") + deploy_parser.add_argument( + "--export-dir", required=True, help="Directory containing the exported model" + ) + deploy_parser.add_argument( + "--host", default="127.0.0.1", help="Host to bind the server to" + ) + deploy_parser.add_argument( + "--port", type=int, default=30000, help="Port to bind the server to" + ) + + args = parser.parse_args() + + if args.command == "quantize": + quantize_and_export_model( + model_path=args.model_path, + export_dir=args.export_dir, + quantization_method=args.quantization_method, + checkpoint_save_path=args.checkpoint_save_path, + device=args.device, + ) + elif args.command == "deploy": + deploy_exported_model( + export_dir=args.export_dir, + host=args.host, + port=args.port, + ) + else: + parser.print_help() + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/pyproject.toml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/pyproject.toml new file mode 100755 index 00000000..b5919cd0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/pyproject.toml @@ -0,0 +1,167 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "sglang" +version = "0.5.6" +description = "SGLang is a fast serving framework for large language models and vision language models." +readme = "README.md" +requires-python = ">=3.10" +license = { file = "LICENSE" } +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", +] + +dependencies = [ + "IPython", + "aiohttp", + "anthropic>=0.20.0", + "av ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' and platform_machine == 'armv7l')", + "blobfile==3.0.0", + "build", + "compressed-tensors", + "cuda-python", + "decord2", + "datasets", + "einops", + "fastapi", + "flashinfer_python==0.5.3", # keep it aligned with jit-cache version in Dockerfile + "flashinfer_cubin==0.5.3", + "gguf", + "hf_transfer", + "huggingface_hub", + "interegular", + "llguidance>=0.7.11,<0.8.0", + "modelscope", + "msgspec", + "ninja", + "numpy", + "nvidia-cutlass-dsl==4.2.1", + "nvidia-ml-py", + "openai-harmony==0.0.4", + "openai==2.6.1", + "orjson", + "outlines==0.1.11", + "packaging", + "partial_json_parser", + "pillow", + "prometheus-client>=0.20.0", + "psutil", + "py-spy", + "pybase64", + "pydantic", + "python-multipart", + "pyzmq>=25.1.2", + "requests", + "scipy", + "sentencepiece", + "setproctitle", + "sgl-kernel==0.3.18.post2", + "soundfile==0.13.1", + "tiktoken", + "timm==1.0.16", + "torch_memory_saver==0.0.9", + "torch==2.9.1", + "torchaudio==2.9.1", + "torchcodec==0.7.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec does not exist in those systems. If not provided, transformer will use torchvision instead by default. + "torchvision", + "torchao==0.9.0", + "tqdm", + "transformers==4.57.1", + "uvicorn", + "uvloop", + "xgrammar==0.1.27", + + "grpcio==1.75.1", # keep it align with compile_proto.py + "grpcio-tools==1.75.1", # keep it align with compile_proto.py + "grpcio-reflection==1.75.1", # required by srt/entrypoints/grpc_server.py + "grpcio-health-checking==1.75.1", # required for Kubernetes gRPC health probes +] + +[project.optional-dependencies] +checkpoint-engine = ["checkpoint-engine==0.1.2"] +diffusion = [ + "PyYAML==6.0.1", + "cloudpickle", + "diffusers==0.35.2", + "imageio==2.36.0", + "imageio-ffmpeg==0.5.1", + "moviepy>=2.0.0", + "opencv-python==4.10.0.84", + "remote-pdb", + "st_attn ==0.0.7", + "vsa==0.0.4", + "yunchang==0.6.3.post1", +] + +[tool.uv.extra-build-dependencies] +st-attn = ["torch", "setuptools"] +vsa = ["torch", "setuptools"] + +test = [ + "accelerate", + "expecttest", + "jsonlines", + "matplotlib", + "pandas", + "peft", + "pytest", + "sentence_transformers", + "tabulate", +] +dev = ["sglang[test]"] +tracing = [ + "opentelemetry-api", + "opentelemetry-exporter-otlp", + "opentelemetry-exporter-otlp-proto-grpc", + "opentelemetry-sdk", +] + +[project.urls] +"Homepage" = "https://github.com/sgl-project/sglang" +"Bug Tracker" = "https://github.com/sgl-project/sglang/issues" + +[project.scripts] +sglang = "sglang.cli.main:main" + +[tool.setuptools.package-data] +"sglang" = [ + "srt/layers/moe/fused_moe_triton/configs/*/*.json", + "srt/layers/quantization/configs/*.json", + "srt/mem_cache/storage/hf3fs/hf3fs_utils.cpp", + "srt/mem_cache/cpp_radix_tree/*.cpp", + "srt/mem_cache/cpp_radix_tree/*.h", + "srt/speculative/cpp_ngram/*.cpp", + "srt/speculative/cpp_ngram/*.h", + "jit_kernel/include/sgl_kernel/*.h", + "jit_kernel/include/sgl_kernel/*.cuh", + "jit_kernel/csrc/*.cuh" +] + +[tool.setuptools.packages.find] +exclude = [ + "assets*", + "benchmark*", + "docs*", + "dist*", + "playground*", + "scripts*", + "tests*", +] + +[tool.wheel] +exclude = [ + "assets*", + "benchmark*", + "docs*", + "dist*", + "playground*", + "scripts*", + "tests*", +] + +[tool.codespell] +ignore-words-list = "ans, als, hel, boostrap, childs, te, vas, hsa, ment" +skip = "*.json,*.jsonl,*.patch,*.txt" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/pyproject_cpu.toml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/pyproject_cpu.toml new file mode 100644 index 00000000..a537bc36 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/pyproject_cpu.toml @@ -0,0 +1,134 @@ +# https://docs.sglang.io/platforms/cpu_server.html +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "sglang" +version = "0.5.6" +description = "SGLang is a fast serving framework for large language models and vision language models." +readme = "README.md" +requires-python = ">=3.10" +license = { file = "LICENSE" } +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", +] + +dependencies = [ + "IPython", + "aiohttp", + "anthropic>=0.20.0", + "blobfile==3.0.0", + "build", + "compressed-tensors", + "datasets", + "decord", + "einops", + "fastapi", + "gguf", + "hf_transfer", + "huggingface_hub", + "intel-openmp", + "interegular", + "llguidance>=0.7.11,<0.8.0", + "modelscope", + "msgspec", + "ninja", + "numpy", + "openai-harmony==0.0.4", + "openai==1.99.1", + "orjson", + "outlines==0.1.11", + "packaging", + "partial_json_parser", + "pillow", + "prometheus-client>=0.20.0", + "psutil", + "py-spy", + "pybase64", + "pydantic", + "python-multipart", + "pyzmq>=25.1.2", + "requests", + "scipy", + "sentencepiece", + "setproctitle", + "soundfile==0.13.1", + "tiktoken", + "timm==1.0.16", + "torchao==0.9.0", + "tqdm", + "transformers==4.57.1", + "uvicorn", + "uvloop", + "xgrammar==0.1.27", + "grpcio==1.75.1", # keep it align with compile_proto.py + "grpcio-tools==1.75.1", # keep it align with compile_proto.py + "grpcio-reflection==1.75.1", # required by srt/entrypoints/grpc_server.py +] + +[project.optional-dependencies] +tracing = [ + "opentelemetry-sdk", + "opentelemetry-api", + "opentelemetry-exporter-otlp", + "opentelemetry-exporter-otlp-proto-grpc", +] +test = [ + "accelerate", + "expecttest", + "jsonlines", + "matplotlib", + "pandas", + "peft", + "pytest", + "sentence_transformers", + "tabulate", +] +all = [] +dev = ["sglang[test]"] + +[project.urls] +"Homepage" = "https://github.com/sgl-project/sglang" +"Bug Tracker" = "https://github.com/sgl-project/sglang/issues" + +[tool.setuptools.package-data] +"sglang" = [ + "srt/layers/moe/fused_moe_triton/configs/*/*.json", + "srt/layers/quantization/configs/*.json", + "srt/mem_cache/storage/hf3fs/hf3fs_utils.cpp", + "srt/mem_cache/cpp_radix_tree/*.cpp", + "srt/mem_cache/cpp_radix_tree/*.h", + "srt/speculative/cpp_ngram/*.cpp", + "srt/speculative/cpp_ngram/*.h", + "jit_kernel/include/sgl_kernel/*.h", + "jit_kernel/include/sgl_kernel/*.cuh", + "jit_kernel/csrc/*.cuh" +] + +[tool.setuptools.packages.find] +exclude = [ + "assets*", + "benchmark*", + "docs*", + "dist*", + "playground*", + "scripts*", + "tests*", +] + +[tool.wheel] +exclude = [ + "assets*", + "benchmark*", + "docs*", + "dist*", + "playground*", + "scripts*", + "tests*", +] + +[tool.codespell] +ignore-words-list = "ans, als, hel, boostrap, childs, te, vas, hsa, ment" +skip = "*.json,*.jsonl,*.patch,*.txt" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/pyproject_other.toml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/pyproject_other.toml new file mode 100755 index 00000000..3e854caa --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/pyproject_other.toml @@ -0,0 +1,156 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "sglang" +version = "0.5.6" +description = "SGLang is a fast serving framework for large language models and vision language models." +readme = "README.md" +requires-python = ">=3.10" +license = { file = "LICENSE" } +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", +] +dependencies = ["aiohttp", "requests", "tqdm", "numpy", "IPython", "setproctitle"] + +[project.optional-dependencies] +runtime_common = [ + "IPython", + "aiohttp", + "anthropic>=0.20.0", + "blobfile==3.0.0", + "build", + "compressed-tensors", + "decord2", + "datasets", + "einops", + "fastapi", + "gguf", + "hf_transfer", + "huggingface_hub", + "interegular", + "llguidance>=0.7.11,<0.8.0", + "modelscope", + "msgspec", + "ninja", + "numpy", + "openai-harmony==0.0.4", + "openai==1.99.1", + "orjson", + "outlines==0.1.11", + "packaging", + "partial_json_parser", + "pillow", + "prometheus-client>=0.20.0", + "psutil", + "py-spy", + "pybase64", + "pydantic", + "python-multipart", + "pyzmq>=25.1.2", + "requests", + "scipy", + "sentencepiece", + "setproctitle", + "soundfile==0.13.1", + "tiktoken", + "timm==1.0.16", + "torchao==0.9.0", + "tqdm", + "transformers==4.57.1", + "uvicorn", + "uvloop", + "xgrammar==0.1.27", + "grpcio==1.75.1", # keep it align with compile_proto.py + "grpcio-tools==1.75.1", # keep it align with compile_proto.py + "grpcio-reflection==1.75.1", # required by srt/entrypoints/grpc_server.py +] + +tracing = [ + "opentelemetry-sdk", + "opentelemetry-api", + "opentelemetry-exporter-otlp", + "opentelemetry-exporter-otlp-proto-grpc", +] + +# HIP (Heterogeneous-computing Interface for Portability) for AMD +# => base docker rocm/vllm-dev:20250114, not from public vllm whl +srt_hip = [ + "sglang[runtime_common]", + "torch", + "petit_kernel==0.0.2", + "wave-lang==3.8.2", +] + +# https://docs.sglang.io/platforms/ascend_npu.html +srt_npu = ["sglang[runtime_common]"] + +# For Intel Gaudi(device : hpu) follow the installation guide +# https://docs.vllm.ai/en/latest/getting_started/gaudi-installation.html +srt_hpu = ["sglang[runtime_common]"] + +test = [ + "accelerate", + "expecttest", + "gguf", + "jsonlines", + "matplotlib", + "pandas", + "peft", + "pytest", + "sentence_transformers", + "tabulate", +] +all_hip = ["sglang[srt_hip]"] +all_npu = ["sglang[srt_npu]"] +all_hpu = ["sglang[srt_hpu]"] + +dev_hip = ["sglang[all_hip]", "sglang[test]"] +dev_npu = ["sglang[all_npu]", "sglang[test]"] +dev_hpu = ["sglang[all_hpu]", "sglang[test]"] + +[project.urls] +"Homepage" = "https://github.com/sgl-project/sglang" +"Bug Tracker" = "https://github.com/sgl-project/sglang/issues" + +[tool.setuptools.package-data] +"sglang" = [ + "srt/layers/moe/fused_moe_triton/configs/*/*.json", + "srt/layers/quantization/configs/*.json", + "srt/mem_cache/storage/hf3fs/hf3fs_utils.cpp", + "srt/mem_cache/cpp_radix_tree/*.cpp", + "srt/mem_cache/cpp_radix_tree/*.h", + "srt/speculative/cpp_ngram/*.cpp", + "srt/speculative/cpp_ngram/*.h", + "jit_kernel/include/sgl_kernel/*.h", + "jit_kernel/include/sgl_kernel/*.cuh", + "jit_kernel/csrc/*.cuh" +] + +[tool.setuptools.packages.find] +exclude = [ + "assets*", + "benchmark*", + "docs*", + "dist*", + "playground*", + "scripts*", + "tests*", +] + +[tool.wheel] +exclude = [ + "assets*", + "benchmark*", + "docs*", + "dist*", + "playground*", + "scripts*", + "tests*", +] + +[tool.codespell] +ignore-words-list = "ans, als, hel, boostrap, childs, te, vas, hsa, ment" +skip = "*.json,*.jsonl,*.patch,*.txt" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/pyproject_xpu.toml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/pyproject_xpu.toml new file mode 100644 index 00000000..a058e6f4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/pyproject_xpu.toml @@ -0,0 +1,138 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "sglang" +version = "0.5.6" +description = "SGLang is a fast serving framework for large language models and vision language models." +readme = "README.md" +requires-python = ">=3.10" +license = { file = "LICENSE" } +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", +] + +dependencies = [ + "torch==2.9.0", + "torchcodec==0.8.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec does not exist in those systems. If not provided, transformer will use torchvision instead by default. + "av ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' and platform_machine == 'armv7l')", + "torchaudio==2.9.0", + "torchvision", + "sgl-kernel @ git+https://github.com/sgl-project/sgl-kernel-xpu.git", + "IPython", + "aiohttp", + "anthropic>=0.20.0", + "blobfile==3.0.0", + "build", + "compressed-tensors", + "datasets", + "decord", + "einops", + "fastapi", + "gguf", + "hf_transfer", + "huggingface_hub", + "interegular", + "llguidance>=0.7.11,<0.8.0", + "modelscope", + "msgspec", + "ninja", + "numpy", + "openai-harmony==0.0.4", + "openai==1.99.1", + "orjson", + "outlines==0.1.11", + "packaging", + "partial_json_parser", + "pillow", + "prometheus-client>=0.20.0", + "psutil", + "py-spy", + "pybase64", + "pydantic", + "python-multipart", + "pyzmq>=25.1.2", + "requests", + "scipy", + "sentencepiece", + "setproctitle", + "soundfile==0.13.1", + "tiktoken", + "timm==1.0.16", + "torchao==0.9.0", + "tqdm", + "transformers==4.57.1", + "uvicorn", + "uvloop", + # "xgrammar==0.1.24", , xgrammar depends on CUDA PyTorch and Triton only + "grpcio==1.75.1", # keep it align with compile_proto.py + "grpcio-tools==1.75.1", # keep it align with compile_proto.py + "grpcio-reflection==1.75.1", # required by srt/entrypoints/grpc_server.py +] + +[project.optional-dependencies] +tracing = [ + "opentelemetry-sdk", + "opentelemetry-api", + "opentelemetry-exporter-otlp", + "opentelemetry-exporter-otlp-proto-grpc", +] +test = [ + "accelerate", + "expecttest", + "jsonlines", + "matplotlib", + "pandas", + "peft", + "pytest", + "sentence_transformers", + "tabulate", +] +all = [] +dev = ["sglang[test]"] + +[project.urls] +"Homepage" = "https://github.com/sgl-project/sglang" +"Bug Tracker" = "https://github.com/sgl-project/sglang/issues" + +[tool.setuptools.package-data] +"sglang" = [ + "srt/layers/moe/fused_moe_triton/configs/*/*.json", + "srt/layers/quantization/configs/*.json", + "srt/mem_cache/storage/hf3fs/hf3fs_utils.cpp", + "srt/mem_cache/cpp_radix_tree/*.cpp", + "srt/mem_cache/cpp_radix_tree/*.h", + "srt/speculative/cpp_ngram/*.cpp", + "srt/speculative/cpp_ngram/*.h", + "jit_kernel/include/sgl_kernel/*.h", + "jit_kernel/include/sgl_kernel/*.cuh", + "jit_kernel/csrc/*.cuh" +] + +[tool.setuptools.packages.find] +exclude = [ + "assets*", + "benchmark*", + "docs*", + "dist*", + "playground*", + "scripts*", + "tests*", +] + +[tool.wheel] +exclude = [ + "assets*", + "benchmark*", + "docs*", + "dist*", + "playground*", + "scripts*", + "tests*", +] + +[tool.codespell] +ignore-words-list = "ans, als, hel, boostrap, childs, te, vas, hsa, ment" +skip = "*.json,*.jsonl,*.patch,*.txt" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/README.md new file mode 100644 index 00000000..4d9cf8c2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/README.md @@ -0,0 +1,17 @@ +# Code Structure + +- `eval`: The evaluation utilities. +- `lang`: The frontend language. +- `srt`: The backend engine for running local models. (SRT = SGLang Runtime). +- `test`: The test utilities. +- `api.py`: The public APIs. +- `bench_offline_throughput.py`: Benchmark the performance in the offline mode. +- `bench_one_batch.py`: Benchmark the latency of running a single static batch without a server. +- `bench_one_batch_server.py`: Benchmark the latency of running a single batch with a server. +- `bench_serving.py`: Benchmark online serving with dynamic requests. +- `check_env.py`: Check the environment variables and dependencies. +- `global_config.py`: The global configs and constants. +- `launch_server.py`: The entry point for launching a local server. +- `profiler.py`: The profiling entry point to send profile requests. +- `utils.py`: Common utilities. +- `version.py`: Version info. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/__init__.py new file mode 100644 index 00000000..509b145a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/__init__.py @@ -0,0 +1,83 @@ +# SGLang public APIs + +# Frontend Language APIs +from sglang.global_config import global_config +from sglang.lang.api import ( + Engine, + Runtime, + assistant, + assistant_begin, + assistant_end, + flush_cache, + function, + gen, + gen_int, + gen_string, + get_server_info, + image, + select, + separate_reasoning, + set_default_backend, + system, + system_begin, + system_end, + user, + user_begin, + user_end, + video, +) +from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint +from sglang.lang.choices import ( + greedy_token_selection, + token_length_normalized, + unconditional_likelihood_normalized, +) + +# Lazy import some libraries +from sglang.utils import LazyImport +from sglang.version import __version__ + +Anthropic = LazyImport("sglang.lang.backend.anthropic", "Anthropic") +LiteLLM = LazyImport("sglang.lang.backend.litellm", "LiteLLM") +OpenAI = LazyImport("sglang.lang.backend.openai", "OpenAI") +VertexAI = LazyImport("sglang.lang.backend.vertexai", "VertexAI") + +# Runtime Engine APIs +ServerArgs = LazyImport("sglang.srt.server_args", "ServerArgs") +Engine = LazyImport("sglang.srt.entrypoints.engine", "Engine") + +__all__ = [ + "Engine", + "Runtime", + "assistant", + "assistant_begin", + "assistant_end", + "flush_cache", + "function", + "gen", + "gen_int", + "gen_string", + "get_server_info", + "image", + "select", + "separate_reasoning", + "set_default_backend", + "system", + "system_begin", + "system_end", + "user", + "user_begin", + "user_end", + "video", + "RuntimeEndpoint", + "greedy_token_selection", + "token_length_normalized", + "unconditional_likelihood_normalized", + "ServerArgs", + "Anthropic", + "LiteLLM", + "OpenAI", + "VertexAI", + "global_config", + "__version__", +] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/bench_offline_throughput.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/bench_offline_throughput.py new file mode 100644 index 00000000..294d3f68 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/bench_offline_throughput.py @@ -0,0 +1,476 @@ +""" +Benchmark the throughput in the offline mode. +It accepts server arguments (the same as launch_server.py) and benchmark arguments (the same as bench_serving.py). + +# Usage +## Sharegpt dataset with default args +python -m sglang.bench_offline_throughput --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --num-prompts 10 + +## Random dataset with default args +python -m sglang.bench_offline_throughput --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --dataset-name random --random-input 1024 --random-output 1024 +""" + +import argparse +import asyncio +import dataclasses +import inspect +import json +import logging +import os +import random +import time +from typing import Dict, List, Optional + +import numpy as np + +from sglang.bench_serving import ( + DatasetRow, + get_dataset, + get_tokenizer, + sample_random_requests, + set_ulimit, +) +from sglang.lang.backend.runtime_endpoint import Runtime +from sglang.srt.entrypoints.engine import Engine +from sglang.srt.server_args import ServerArgs + + +@dataclasses.dataclass +class BenchArgs: + backend: str = "engine" + result_filename: str = "" + dataset_name: str = "sharegpt" + dataset_path: str = "" + num_prompts: int = 1000 + sharegpt_output_len: Optional[int] = None + sharegpt_context_len: Optional[int] = None + random_input_len: int = 1024 + random_output_len: int = 1024 + random_range_ratio: float = 0.0 + gsp_num_groups: int = 64 + gsp_prompts_per_group: int = 16 + gsp_system_prompt_len: int = 2048 + gsp_question_len: int = 128 + gsp_output_len: int = 256 + seed: int = 1 + disable_ignore_eos: bool = False + extra_request_body: Optional[str] = None + apply_chat_template: bool = False + profile: bool = False + skip_warmup: bool = False + do_not_exit: bool = False + prompt_suffix: str = "" + return_logprob: bool = False + logprob_start_len: int = -1 + + @staticmethod + def add_cli_args(parser: argparse.ArgumentParser): + parser.add_argument("--backend", type=str, default=BenchArgs.backend) + parser.add_argument( + "--result-filename", type=str, default=BenchArgs.result_filename + ) + parser.add_argument( + "--dataset-name", + type=str, + default="sharegpt", + choices=["sharegpt", "random", "generated-shared-prefix"], + help="Name of the dataset to benchmark on.", + ) + parser.add_argument( + "--dataset-path", type=str, default="", help="Path to the dataset." + ) + parser.add_argument( + "--num-prompts", + type=int, + default=BenchArgs.num_prompts, + help="Number of prompts to process. Default is 1000.", + ) + parser.add_argument( + "--sharegpt-output-len", + type=int, + default=BenchArgs.sharegpt_output_len, + help="Output length for each request. Overrides the output length from the ShareGPT dataset.", + ) + parser.add_argument( + "--sharegpt-context-len", + type=int, + default=BenchArgs.sharegpt_context_len, + help="The context length of the model for the ShareGPT dataset. Requests longer than the context length will be dropped.", + ) + parser.add_argument( + "--random-input-len", + type=int, + default=BenchArgs.random_input_len, + help="Number of input tokens per request, used only for random dataset.", + ) + parser.add_argument( + "--random-output-len", + type=int, + default=BenchArgs.random_output_len, + help="Number of output tokens per request, used only for random dataset.", + ) + parser.add_argument( + "--random-range-ratio", + type=float, + default=BenchArgs.random_range_ratio, + help="Range of sampled ratio of input/output length, " + "used only for random dataset.", + ) + parser.add_argument( + "--gsp-num-groups", + type=int, + default=BenchArgs.gsp_num_groups, + help="Number of groups with shared prefix, used" + "only for generate-shared-prefix", + ) + parser.add_argument( + "--gsp-prompts-per-group", + type=int, + default=BenchArgs.gsp_prompts_per_group, + help="Number of prompts per group of shared prefix, used" + "only for generate-shared-prefix", + ) + parser.add_argument( + "--gsp-system-prompt-len", + type=int, + default=BenchArgs.gsp_system_prompt_len, + help="System prompt length, used" "only for generate-shared-prefix", + ) + parser.add_argument( + "--gsp-question-len", + type=int, + default=BenchArgs.gsp_question_len, + help="Question length, used" "only for generate-shared-prefix", + ) + parser.add_argument( + "--gsp-output-len", + type=int, + default=BenchArgs.gsp_output_len, + help="Target length in tokens for outputs in generated-shared-prefix dataset", + ) + parser.add_argument("--seed", type=int, default=1, help="The random seed.") + parser.add_argument( + "--disable-ignore-eos", + action="store_true", + help="Disable ignore EOS token", + ) + parser.add_argument( + "--extra-request-body", + metavar='{"key1": "value1", "key2": "value2"}', + type=str, + default=BenchArgs.extra_request_body, + help="Append given JSON object to the request payload. You can use this to specify" + "additional generate params like sampling params.", + ) + parser.add_argument( + "--apply-chat-template", + action="store_true", + help="Apply chat template", + ) + parser.add_argument( + "--profile", + action="store_true", + help="Use Torch Profiler. The endpoint must be launched with " + "SGLANG_TORCH_PROFILER_DIR to enable profiler.", + ) + parser.add_argument( + "--skip-warmup", + action="store_true", + help="Skip the warmup batches.", + ) + parser.add_argument( + "--do-not-exit", + action="store_true", + help="Do not exit the program. This is useful for nsys profile with --duration and --delay.", + ) + parser.add_argument( + "--prompt-suffix", + type=str, + default="", + help="Suffix applied to the end of all user prompts, followed by assistant prompt suffix.", + ) + parser.add_argument( + "--return-logprob", + action="store_true", + help="Enable returning log probabilities.", + ) + parser.add_argument( + "--logprob-start-len", + type=int, + default=-1, + help="Start length for logprob. -1 means only return logprobs for output tokens (default). 0 means return logprobs for all tokens including input.", + ) + + @classmethod + def from_cli_args(cls, args: argparse.Namespace): + attrs = [attr.name for attr in dataclasses.fields(cls)] + return cls(**{attr: getattr(args, attr) for attr in attrs}) + + +def throughput_test_once( + backend_name: str, + backend, + reqs: List[DatasetRow], + ignore_eos: bool, + extra_request_body: Dict, + profile: bool, + return_logprob: bool = False, + logprob_start_len: int = -1, +): + measurement_results = { + "backend": backend_name, + "successful_requests": len(reqs), + "total_latency": -1, + "total_input_tokens": sum(r.prompt_len for r in reqs), + "total_output_tokens": -1, + "request_throughput": -1, + "input_throughput": -1, + "output_throughput": -1, + "total_throughput": -1, + } + + prompt = [r.prompt for r in reqs] + sampling_params = [ + { + "temperature": 0, + "max_new_tokens": r.output_len, + "ignore_eos": ignore_eos, + **extra_request_body, + } + for r in reqs + ] + + if profile: + assert ( + "SGLANG_TORCH_PROFILER_DIR" in os.environ + ), "Please set SGLANG_TORCH_PROFILER_DIR." + os.makedirs(os.environ["SGLANG_TORCH_PROFILER_DIR"], exist_ok=True) + backend.start_profile() + + st = time.perf_counter() + gen_out = backend.generate( + prompt=prompt, + sampling_params=sampling_params, + return_logprob=return_logprob, + logprob_start_len=logprob_start_len, + ) + latency = time.perf_counter() - st + + if profile: + dir = os.getenv("SGLANG_TORCH_PROFILER_DIR") + known_files = set(os.listdir(dir)) + backend.stop_profile() + monitor_trace_file(known_files, dir) + + if backend_name == "runtime": + gen_out = json.loads(gen_out) + + server_info = backend.get_server_info() + + measurement_results["total_latency"] = latency + measurement_results["total_output_tokens"] = sum( + o["meta_info"]["completion_tokens"] for o in gen_out + ) + measurement_results["request_throughput"] = ( + measurement_results["successful_requests"] / latency + ) + measurement_results["input_throughput"] = ( + measurement_results["total_input_tokens"] / latency + ) + measurement_results["output_throughput"] = ( + measurement_results["total_output_tokens"] / latency + ) + measurement_results["total_throughput"] = ( + measurement_results["total_input_tokens"] + + measurement_results["total_output_tokens"] + ) / latency + + if inspect.isawaitable(server_info): + server_info = asyncio.run(server_info) + + measurement_results["last_gen_throughput"] = server_info["internal_states"][0][ + "last_gen_throughput" + ] + + return measurement_results + + +def monitor_trace_file(known_files, directory, interval=1): + print(f"Monitoring {directory} for new trace files...") + + while True: + flag = False + time.sleep(interval) + current_files = set(os.listdir(directory)) + + new_files = current_files - known_files + for new_file in new_files: + new_file_path = os.path.join(directory, new_file) + print(f"New file detected: {new_file}") + + previous_size = 0 + while True: + try: + current_size = os.path.getsize(new_file_path) + except FileNotFoundError: + print(f"File {new_file} is no longer accessible.") + break + + if current_size > previous_size: + previous_size = current_size + else: + flag = True + break + + time.sleep(interval) + if flag: + break + + +def throughput_test( + server_args: ServerArgs, + bench_args: BenchArgs, +): + if bench_args.backend == "engine": + backend = Engine(**dataclasses.asdict(server_args)) + if not backend: + raise ValueError("Please provide valid engine arguments") + elif bench_args.backend == "runtime": + backend = Runtime(**dataclasses.asdict(server_args)) + else: + raise ValueError('Please set backend to either "engine" or "runtime"') + + tokenizer_id = server_args.tokenizer_path or server_args.model_path + tokenizer = get_tokenizer(tokenizer_id) + + # Set global environments + set_ulimit() + random.seed(bench_args.seed) + np.random.seed(bench_args.seed) + + # Parse args + extra_request_body = {} + if bench_args.extra_request_body: + extra_request_body = json.loads(args.extra_request_body) + + # Read dataset + input_requests = get_dataset(bench_args, tokenizer) + + warmup_requests = sample_random_requests( + input_len=256, + output_len=16, + num_prompts=min(bench_args.num_prompts, 16), + range_ratio=1.0, + tokenizer=tokenizer, + dataset_path=bench_args.dataset_path, + ) + + # Warm up + if not bench_args.skip_warmup: + logging.info("\nWarmup...") + throughput_test_once( + backend_name=bench_args.backend, + backend=backend, + reqs=warmup_requests, + ignore_eos=not bench_args.disable_ignore_eos, + extra_request_body=extra_request_body, + profile=False, + return_logprob=bench_args.return_logprob, + logprob_start_len=bench_args.logprob_start_len, + ) + time.sleep(0.5) + + logging.info("\nBenchmark...") + result = throughput_test_once( + backend_name=bench_args.backend, + backend=backend, + reqs=input_requests, + ignore_eos=not bench_args.disable_ignore_eos, + extra_request_body=extra_request_body, + profile=bench_args.profile, + return_logprob=bench_args.return_logprob, + logprob_start_len=bench_args.logprob_start_len, + ) + backend.shutdown() + + if bench_args.result_filename: + with open(bench_args.result_filename, "a") as fout: + fout.write(json.dumps(result) + "\n") + + print( + "\n{s:{c}^{n}}".format(s=" Offline Throughput Benchmark Result ", n=50, c="=") + ) + print("{:<40} {:<10}".format("Backend:", result["backend"])) + print("{:<40} {:<10}".format("Successful requests:", result["successful_requests"])) + print("{:<40} {:<10.2f}".format("Benchmark duration (s):", result["total_latency"])) + print("{:<40} {:<10}".format("Total input tokens:", result["total_input_tokens"])) + print( + "{:<40} {:<10}".format("Total generated tokens:", result["total_output_tokens"]) + ) + print( + "{:<40} {:<10.2f}".format( + "Last generation throughput (tok/s):", result["last_gen_throughput"] + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Request throughput (req/s):", result["request_throughput"] + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Input token throughput (tok/s):", result["input_throughput"] + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Output token throughput (tok/s):", result["output_throughput"] + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Total token throughput (tok/s):", result["total_throughput"] + ) + ) + print("=" * 50) + + return result + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + BenchArgs.add_cli_args(parser) + args = parser.parse_args() + + # handling ModelScope model downloads + if os.getenv("SGLANG_USE_MODELSCOPE", "false").lower() in ("true", "1"): + if os.path.exists(args.model_path): + print(f"Using local model path: {args.model_path}") + else: + try: + from modelscope import snapshot_download + + print(f"Using ModelScope to download model: {args.model_path}") + + # download the model and replace args.model_path + args.model_path = snapshot_download( + args.model_path, + ) + print(f"Model downloaded to: {args.model_path}") + except Exception as e: + print(f"ModelScope download failed: {str(e)}") + raise e + + server_args = ServerArgs.from_cli_args(args) + bench_args = BenchArgs.from_cli_args(args) + + logging.basicConfig( + level=getattr(logging, server_args.log_level.upper()), + format="%(message)s", + ) + + throughput_test(server_args, bench_args) + + while bench_args.do_not_exit: + pass diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/bench_one_batch.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/bench_one_batch.py new file mode 100644 index 00000000..25b16d31 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/bench_one_batch.py @@ -0,0 +1,795 @@ +""" +Benchmark the latency of running a single static batch without a server. + +This script does not launch a server and uses the low-level APIs. +It accepts server arguments (the same as launch_server.py) and benchmark arguments (e.g., batch size, input lengths). + +# Usage (latency test) +## with dummy weights: +python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3-8B-Instruct --load-format dummy +## sweep through multiple data points and store (append) the results in a jsonl file: +python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3-8B-Instruct --batch 1 12 14 --input-len 256 512 --output-len 32 256 --run-name test_run +## run with profiling: +python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3-8B-Instruct --batch 1 12 14 --input-len 256 512 --profile +## run with profiling to custom directory: +export SGLANG_TORCH_PROFILER_DIR=/root/sglang/profile_log +python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3-8B-Instruct --batch 1 --input-len 256 --profile +## run with CUDA profiler (nsys): +nsys profile --force-overwrite=true -o bench_one_batch python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3-8B-Instruct --batch 1 --input-len 256 --profile --profile-activities CUDA_PROFILER +# Usage (correctness test): +python -m sglang.bench_one_batch --model-path TinyLlama/TinyLlama-1.1B-Chat-v0.4 --correct + +## Reference output (of the correctness test above, can be gpu dependent): +input_ids=[[1, 450, 7483, 310, 3444, 338], [1, 450, 7483, 310, 278, 3303, 13187, 290, 338], [1, 20628, 338, 263, 6575, 1460, 2462, 322, 306, 763]] + +prefill logits (first half): tensor([[-10.0312, -9.5000, 0.8931, ..., -4.9414, -3.2422, -3.3633], + [-10.0312, -9.5000, 0.8931, ..., -4.9414, -3.2422, -3.3633], + [ -9.1875, -10.2500, 2.7129, ..., -4.3359, -4.0664, -4.1328]], + device='cuda:0') + +prefill logits (final): tensor([[-8.3125, -7.1172, 3.3457, ..., -4.9570, -4.1328, -3.4141], + [-8.9141, -9.0156, 4.1445, ..., -4.9922, -4.4961, -4.0781], + [-9.6328, -9.0547, 4.0195, ..., -5.3047, -4.7148, -4.4570]], + device='cuda:0') + +========== Prompt 0 ========== + The capital of France is Paris. +The capital of the United States is Washington, D.C. + + +========== Prompt 1 ========== + The capital of the United Kindom is London. +The capital of the United Kingdom is London. +The capital of the + +========== Prompt 2 ========== + Today is a sunny day and I like to go for a walk in the park. +I'm going to the park +""" + +import argparse +import copy +import dataclasses +import itertools +import json +import logging +import multiprocessing +import os +import time +from types import SimpleNamespace +from typing import Tuple + +import numpy as np +import torch +import torch.distributed as dist + +from sglang.srt.configs.model_config import ModelConfig +from sglang.srt.distributed.parallel_state import destroy_distributed_environment +from sglang.srt.entrypoints.engine import _set_envs_and_config +from sglang.srt.layers.moe import initialize_moe_config +from sglang.srt.managers.schedule_batch import Req, ScheduleBatch +from sglang.srt.managers.scheduler_dp_attn_mixin import prepare_mlp_sync_batch_raw +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_executor.model_runner import ModelRunner +from sglang.srt.sampling.sampling_params import SamplingParams +from sglang.srt.server_args import PortArgs, ServerArgs +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm +from sglang.srt.utils import ( + configure_logger, + get_bool_env_var, + is_cuda_alike, + is_xpu, + kill_process_tree, + maybe_reindex_device_id, + require_mlp_sync, + require_mlp_tp_gather, + set_gpu_proc_affinity, + suppress_other_loggers, +) +from sglang.srt.utils.hf_transformers_utils import get_tokenizer + +profile_activities = [torch.profiler.ProfilerActivity.CPU] + [ + profiler_activity + for available, profiler_activity in [ + (is_cuda_alike(), torch.profiler.ProfilerActivity.CUDA), + (is_xpu(), torch.profiler.ProfilerActivity.XPU), + ] + if available +] + + +def start_profile(profile_activities, profile_record_shapes=False, rank_print=print): + """ + Abstracted function to start profiling based on profile_activities. + Returns profiler object (or None). + """ + if "CUDA_PROFILER" in profile_activities: + try: + torch.cuda.cudart().cudaProfilerStart() + rank_print("CUDA Profiler started (nsys will begin capturing)") + except Exception as e: + rank_print(f"Failed to start CUDA profiler: {e}") + return None + else: + activities = [] + if "CPU" in profile_activities: + activities.append(torch.profiler.ProfilerActivity.CPU) + if "GPU" in profile_activities: + activities.append(torch.profiler.ProfilerActivity.CUDA) + if activities: + profiler = torch.profiler.profile( + activities=activities, + with_stack=True, + record_shapes=profile_record_shapes, + ) + profiler.start() + return profiler + return None + + +def stop_profile( + profiler, + profile_activities, + rank_print=print, + save_trace=False, + trace_filename=None, + stage=None, +): + """ + Abstracted function to stop profiling based on profile_activities. + Optionally saves trace results and prints completion messages. + """ + if "CUDA_PROFILER" in profile_activities: + try: + torch.cuda.cudart().cudaProfilerStop() + rank_print("CUDA Profiler stopped (nsys should dump traces)") + except Exception as e: + rank_print(f"Failed to stop CUDA profiler: {e}") + elif profiler is not None: + profiler.stop() + + if save_trace: + if profiler is not None: + if trace_filename: + _save_profile_trace_results(profiler, trace_filename) + stage_desc = f"for {stage}" if stage else "" + rank_print( + f"torch profiler chrome trace {stage_desc} saved to {trace_filename}" + ) + if "CUDA_PROFILER" in profile_activities: + rank_print(f"CUDA profiler trace for {stage} completed") + + +@dataclasses.dataclass +class BenchArgs: + run_name: str = "default" + batch_size: Tuple[int] = (1,) + input_len: Tuple[int] = (1024,) + output_len: Tuple[int] = (16,) + prompt_filename: str = "" + result_filename: str = "result.jsonl" + correctness_test: bool = False + # This is only used for correctness test + cut_len: int = 4 + log_decode_step: int = 0 + profile: bool = False + profile_record_shapes: bool = False + profile_activities: Tuple[str] = ("CPU", "GPU") + profile_stage: str = "all" + profile_filename_prefix: str = "profile" + + @staticmethod + def add_cli_args(parser: argparse.ArgumentParser): + parser.add_argument("--run-name", type=str, default=BenchArgs.run_name) + parser.add_argument( + "--batch-size", type=int, nargs="+", default=BenchArgs.batch_size + ) + parser.add_argument( + "--input-len", type=int, nargs="+", default=BenchArgs.input_len + ) + parser.add_argument( + "--output-len", type=int, nargs="+", default=BenchArgs.output_len + ) + parser.add_argument( + "--prompt-filename", type=str, default=BenchArgs.prompt_filename + ) + parser.add_argument( + "--result-filename", type=str, default=BenchArgs.result_filename + ) + parser.add_argument("--correctness-test", action="store_true") + parser.add_argument("--cut-len", type=int, default=BenchArgs.cut_len) + parser.add_argument( + "--log-decode-step", + type=int, + default=BenchArgs.log_decode_step, + help="Log decode latency by step, default is set to zero to disable.", + ) + parser.add_argument("--profile", action="store_true", help="Enable profiling.") + parser.add_argument( + "--profile-record-shapes", + action="store_true", + help="Record tensor shapes in profiling results.", + ) + parser.add_argument( + "--profile-activities", + type=str, + nargs="+", + default=["CPU", "GPU"], + choices=["CPU", "GPU", "CUDA_PROFILER"], + help="Profiler activities: CPU, GPU, CUDA_PROFILER. If CPU/GPU, use torch profiler. If CUDA_PROFILER, use CUDA profiler.", + ) + parser.add_argument( + "--profile-stage", + type=str, + default=BenchArgs.profile_stage, + choices=["all", "prefill", "decode"], + help="Which stage to profile: all, prefill, or decode only.", + ) + parser.add_argument( + "--profile-filename-prefix", + type=str, + default=BenchArgs.profile_filename_prefix, + help="Prefix of the profiling file names. The full profiling result file(s) be " + '"[profile_filename_prefix]_batch[batch_size]_input[input_len]_output[output_len].trace.json.gz"', + ) + + @classmethod + def from_cli_args(cls, args: argparse.Namespace): + # use the default value's type to cast the args into correct types. + attrs = [(attr.name, type(attr.default)) for attr in dataclasses.fields(cls)] + return cls( + **{attr: attr_type(getattr(args, attr)) for attr, attr_type in attrs} + ) + + +def load_model(server_args, port_args, gpu_id, tp_rank): + suppress_other_loggers() + rank_print = print if tp_rank == 0 else lambda *args, **kwargs: None + moe_ep_rank = tp_rank // (server_args.tp_size // server_args.ep_size) + + model_config = ModelConfig.from_server_args(server_args) + model_runner = ModelRunner( + model_config=model_config, + mem_fraction_static=server_args.mem_fraction_static, + gpu_id=gpu_id, + tp_rank=tp_rank, + tp_size=server_args.tp_size, + moe_ep_rank=moe_ep_rank, + moe_ep_size=server_args.ep_size, + pp_rank=0, + pp_size=1, + nccl_port=port_args.nccl_port, + server_args=server_args, + ) + rank_print(f"max_total_num_tokens={model_runner.max_total_num_tokens}") + tokenizer = get_tokenizer( + server_args.tokenizer_path, + tokenizer_mode=server_args.tokenizer_mode, + trust_remote_code=server_args.trust_remote_code, + ) + if server_args.tp_size > 1: + dist.barrier() + return model_runner, tokenizer + + +def prepare_inputs_for_correctness_test(bench_args, tokenizer, custom_prompts): + prompts = ( + custom_prompts + if custom_prompts + else [ + "The capital of France is", + "The capital of the United Kindom is", + "Today is a sunny day and I like", + ] + ) + input_ids = [tokenizer.encode(p) for p in prompts] + sampling_params = SamplingParams( + temperature=0, + max_new_tokens=BenchArgs.output_len, + ) + + reqs = [] + for i in range(len(prompts)): + assert len(input_ids[i]) > bench_args.cut_len + + tmp_input_ids = input_ids[i][: bench_args.cut_len] + req = Req( + rid=i, + origin_input_text=prompts[i], + origin_input_ids=tmp_input_ids, + sampling_params=sampling_params, + ) + req.fill_ids = req.origin_input_ids + req.extend_input_len = len(req.fill_ids) - len(req.prefix_indices) + req.logprob_start_len = len(req.origin_input_ids) - 1 + reqs.append(req) + + return input_ids, reqs + + +def prepare_extend_inputs_for_correctness_test( + bench_args, input_ids, reqs, model_runner +): + for i in range(len(reqs)): + req = reqs[i] + req.fill_ids += input_ids[i][bench_args.cut_len :] + req.prefix_indices = model_runner.req_to_token_pool.req_to_token[ + i, : bench_args.cut_len + ] + req.extend_input_len = len(req.fill_ids) - len(req.prefix_indices) + req.logprob_start_len = len(req.origin_input_ids) - 1 + return reqs + + +def prepare_synthetic_inputs_for_latency_test( + batch_size, input_len, custom_inputs=None +): + input_ids = ( + custom_inputs + if custom_inputs + else np.random.randint(0, 10000, (batch_size, input_len), dtype=np.int32) + ) + sampling_params = SamplingParams( + temperature=0, + max_new_tokens=BenchArgs.output_len, + ) + + reqs = [] + for i in range(len(input_ids)): + req = Req( + rid=i, + origin_input_text="", + origin_input_ids=list(input_ids[i]), + sampling_params=sampling_params, + ) + req.fill_ids = req.origin_input_ids + req.extend_input_len = len(req.fill_ids) - len(req.prefix_indices) + req.logprob_start_len = len(req.origin_input_ids) - 1 + reqs.append(req) + + return reqs + + +@torch.no_grad +def extend(reqs, model_runner): + # Create dummy tree_cache for benchmarks (no prefix caching, just allocation) + dummy_tree_cache = SimpleNamespace( + page_size=model_runner.server_args.page_size, + device=model_runner.device, + token_to_kv_pool_allocator=model_runner.token_to_kv_pool_allocator, + ) + + batch = ScheduleBatch.init_new( + reqs=reqs, + req_to_token_pool=model_runner.req_to_token_pool, + token_to_kv_pool_allocator=model_runner.token_to_kv_pool_allocator, + tree_cache=dummy_tree_cache, + model_config=model_runner.model_config, + enable_overlap=False, + spec_algorithm=SpeculativeAlgorithm.NONE, + ) + batch.prepare_for_extend() + _maybe_prepare_mlp_sync_batch(batch, model_runner) + model_worker_batch = batch.get_model_worker_batch() + forward_batch = ForwardBatch.init_new(model_worker_batch, model_runner) + logits_output, _ = model_runner.forward(forward_batch) + next_token_ids = model_runner.sample(logits_output, forward_batch) + return next_token_ids, logits_output.next_token_logits, batch + + +@torch.no_grad +def decode(input_token_ids, batch, model_runner): + batch.output_ids = input_token_ids + batch.prepare_for_decode() + _maybe_prepare_mlp_sync_batch(batch, model_runner) + model_worker_batch = batch.get_model_worker_batch() + forward_batch = ForwardBatch.init_new(model_worker_batch, model_runner) + logits_output, _ = model_runner.forward(forward_batch) + next_token_ids = model_runner.sample(logits_output, forward_batch) + return next_token_ids, logits_output.next_token_logits + + +def _maybe_prepare_mlp_sync_batch(batch: ScheduleBatch, model_runner): + if require_mlp_sync(model_runner.server_args): + prepare_mlp_sync_batch_raw( + batch, + dp_size=model_runner.server_args.dp_size, + attn_tp_size=1, + tp_group=model_runner.tp_group, + get_idle_batch=None, + disable_cuda_graph=model_runner.server_args.disable_cuda_graph, + require_mlp_tp_gather=require_mlp_tp_gather(model_runner.server_args), + disable_overlap_schedule=model_runner.server_args.disable_overlap_schedule, + offload_tags=set(), + ) + + +def _read_prompts_from_file(prompt_file, rank_print): + """Read custom prompts from the file specified by `--prompt-filename`.""" + if not prompt_file: + return [] + if not os.path.exists(prompt_file): + rank_print( + f"Custom prompt file {prompt_file} not found. Using default inputs..." + ) + return [] + with open(prompt_file, "r") as pf: + return pf.readlines() + + +def _get_torch_profiler_output_dir(): + return os.environ.get("SGLANG_TORCH_PROFILER_DIR", "/tmp") + + +def _create_torch_profiler_filename( + profile_filename_prefix, batch_size, input_len, output_len, stage +): + output_dir = _get_torch_profiler_output_dir() + filename = f"{profile_filename_prefix}_batch{batch_size}_input{input_len}_output{output_len}_{stage}.trace.json.gz" + return os.path.join(output_dir, filename) + + +def _save_profile_trace_results(profiler, filename): + parent_dir = os.path.dirname(os.path.abspath(filename)) + os.makedirs(parent_dir, exist_ok=True) + profiler.export_chrome_trace(filename) + print( + profiler.key_averages(group_by_input_shape=True).table( + sort_by="self_cpu_time_total" + ) + ) + + +def correctness_test( + server_args, + port_args, + bench_args, + gpu_id, + tp_rank, +): + # Configure the logger + configure_logger(server_args, prefix=f" TP{tp_rank}") + rank_print = print if tp_rank == 0 else lambda *args, **kwargs: None + + # Load the model + model_runner, tokenizer = load_model(server_args, port_args, gpu_id, tp_rank) + + # Prepare inputs + custom_prompts = _read_prompts_from_file(bench_args.prompt_filename, rank_print) + input_ids, reqs = prepare_inputs_for_correctness_test( + bench_args, tokenizer, custom_prompts + ) + rank_print(f"\n{input_ids=}\n") + + if bench_args.cut_len > 0: + # Prefill + next_token_ids, next_token_logits, batch = extend(reqs, model_runner) + rank_print(f"prefill logits (first half): {next_token_logits} \n") + + # Prepare extend inputs + reqs = prepare_extend_inputs_for_correctness_test( + bench_args, input_ids, reqs, model_runner + ) + + # Extend (prefill w/ KV cache) + next_token_ids, next_token_logits, batch = extend(reqs, model_runner) + rank_print(f"prefill logits (final): {next_token_logits} \n") + + # Decode + output_ids = [input_ids[i] + [next_token_ids[i]] for i in range(len(input_ids))] + for _ in range(bench_args.output_len[0] - 1): + next_token_ids, _ = decode(next_token_ids, batch, model_runner) + next_token_ids_list = next_token_ids.tolist() + for i in range(len(reqs)): + output_ids[i].append(next_token_ids_list[i]) + + # Print output texts + for i in range(len(reqs)): + rank_print(f"========== Prompt {i} ==========") + rank_print(tokenizer.decode(output_ids[i]), "\n") + + +def synchronize(device): + torch.get_device_module(device).synchronize() + + +def latency_test_run_once( + run_name, + model_runner, + rank_print, + reqs, + batch_size, + input_len, + output_len, + device, + log_decode_step, + profile, + profile_record_shapes, + profile_activities, + profile_filename_prefix, + profile_stage, + tp_rank, +): + max_batch_size = model_runner.max_total_num_tokens // (input_len + output_len) + if batch_size > max_batch_size: + rank_print( + f"skipping ({batch_size}, {input_len}, {output_len}) due to max batch size limit" + ) + return + + model_runner.req_to_token_pool.clear() + model_runner.token_to_kv_pool_allocator.clear() + + measurement_results = { + "run_name": run_name, + "batch_size": batch_size, + "input_len": input_len, + "output_len": output_len, + } + + tot_latency = 0 + + profiler = None + enable_profile_prefill = profile and profile_stage in ["all", "prefill"] + if enable_profile_prefill: + profiler = start_profile( + profile_activities, + profile_record_shapes=profile_record_shapes, + rank_print=rank_print, + ) + + synchronize(device) + tic = time.perf_counter() + next_token_ids, _, batch = extend(reqs, model_runner) + synchronize(device) + prefill_latency = time.perf_counter() - tic + + if enable_profile_prefill: + trace_filename = _create_torch_profiler_filename( + profile_filename_prefix, batch_size, input_len, output_len, "prefill" + ) + stop_profile( + profiler, + profile_activities, + rank_print=rank_print, + save_trace=True, + trace_filename=trace_filename, + stage="prefill", + ) + + tot_latency += prefill_latency + throughput = input_len * batch_size / prefill_latency + rank_print( + f"Prefill. latency: {prefill_latency:6.5f} s, throughput: {throughput:9.2f} token/s" + ) + measurement_results["prefill_latency"] = prefill_latency + measurement_results["prefill_throughput"] = throughput + + decode_latencies = [] + profile_step_of_interest = output_len // 2 + enable_profile_decode = profile and profile_stage in ["all", "decode"] + for i in range(output_len - 1): + synchronize(device) + profiler = None + if enable_profile_decode and i == profile_step_of_interest: + profiler = start_profile( + profile_activities, + profile_record_shapes=profile_record_shapes, + rank_print=rank_print, + ) + + tic = time.perf_counter() + next_token_ids, _ = decode(next_token_ids, batch, model_runner) + synchronize(device) + latency = time.perf_counter() - tic + + if enable_profile_decode and i == profile_step_of_interest: + trace_filename = _create_torch_profiler_filename( + profile_filename_prefix, batch_size, input_len, output_len, "decode" + ) + stop_profile( + profiler, + profile_activities, + rank_print=rank_print, + save_trace=True, + trace_filename=trace_filename, + stage="decode", + ) + + tot_latency += latency + throughput = batch_size / latency + decode_latencies.append(latency) + if i < 5 or (log_decode_step > 0 and i % log_decode_step == 0): + rank_print( + f"Decode {i}. Batch size: {batch_size}, latency: {latency:6.5f} s, throughput: {throughput:9.2f} token/s" + ) + + # Record decode timing from 2nd output + if output_len > 1: + med_decode_latency = np.median(decode_latencies) + med_decode_throughput = batch_size / med_decode_latency + rank_print( + f"Decode. median latency: {med_decode_latency:6.5f} s, median throughput: {med_decode_throughput:9.2f} token/s" + ) + measurement_results["median_decode_latency"] = med_decode_latency + measurement_results["median_decode_throughput"] = med_decode_throughput + + throughput = (input_len + output_len) * batch_size / tot_latency + rank_print( + f"Total. latency: {tot_latency:6.3f} s, throughput: {throughput:9.2f} token/s" + ) + measurement_results["total_latency"] = tot_latency + measurement_results["overall_throughput"] = throughput + return measurement_results + + +def latency_test( + server_args, + port_args, + bench_args, + gpu_id, + tp_rank, +): + initialize_moe_config(server_args) + + # Set CPU affinity + if get_bool_env_var("SGLANG_SET_CPU_AFFINITY"): + set_gpu_proc_affinity( + server_args.pp_size, server_args.tp_size, server_args.nnodes, tp_rank + ) + + # Configure the logger + configure_logger(server_args, prefix=f" TP{tp_rank}") + rank_print = print if tp_rank == 0 else lambda *args, **kwargs: None + + # Load the model + model_runner, tokenizer = load_model(server_args, port_args, gpu_id, tp_rank) + + # Prepare inputs for warm up + reqs = prepare_synthetic_inputs_for_latency_test( + bench_args.batch_size[0], bench_args.input_len[0] + ) + + # Warm up + rank_print("Warmup ...") + latency_test_run_once( + bench_args.run_name, + model_runner, + rank_print, + reqs, + bench_args.batch_size[0], + bench_args.input_len[0], + min(32, bench_args.output_len[0]), # shorter decoding to speed up the warmup + server_args.device, + log_decode_step=0, + profile=False, + profile_record_shapes=False, + profile_activities=("CPU", "GPU"), + profile_filename_prefix="", + profile_stage="all", + tp_rank=tp_rank, + ) + + rank_print("Benchmark ...") + + custom_inputs = _read_prompts_from_file(bench_args.prompt_filename, rank_print) + custom_inputs = [tokenizer.encode(p.strip()) for p in custom_inputs] + custom_input_len = len(custom_inputs) + + # Run the sweep + result_list = [] + for bs, il, ol in itertools.product( + bench_args.batch_size, bench_args.input_len, bench_args.output_len + ): + bs_aligned_inputs = [] + if custom_inputs: + if custom_input_len == bs: + bs_aligned_inputs = custom_inputs + elif custom_input_len > bs: + rank_print( + f"Custom input size ({custom_input_len}) is larger than batch_size ({bs}). " + f"Using the first {bs} prompts." + ) + bs_aligned_inputs = copy.deepcopy(custom_inputs[:bs]) + else: + rank_print( + f"Custom input size ({custom_input_len}) is smaller than batch_size ({bs}). " + f"Pad to the desired batch_size with the last prompt." + ) + bs_aligned_inputs = copy.deepcopy(custom_inputs) + bs_aligned_inputs.extend( + [bs_aligned_inputs[-1]] * (bs - custom_input_len) + ) + + reqs = prepare_synthetic_inputs_for_latency_test(bs, il, bs_aligned_inputs) + ret = latency_test_run_once( + bench_args.run_name, + model_runner, + rank_print, + reqs, + bs, + il, + ol, + server_args.device, + bench_args.log_decode_step, + bench_args.profile if tp_rank == 0 else None, + bench_args.profile_record_shapes if tp_rank == 0 else None, + bench_args.profile_activities, + bench_args.profile_filename_prefix, + bench_args.profile_stage, + tp_rank, + ) + if ret is not None: + result_list.append(ret) + + # Write results in jsonlines format on rank 0. + if tp_rank == 0 and bench_args.result_filename: + with open(bench_args.result_filename, "a") as fout: + for result in result_list: + fout.write(json.dumps(result) + "\n") + + if server_args.tp_size > 1: + destroy_distributed_environment() + + +def main(server_args, bench_args): + server_args.cuda_graph_max_bs = max(bench_args.batch_size) + + _set_envs_and_config(server_args) + + if server_args.model_path: + if bench_args.correctness_test: + work_func = correctness_test + else: + work_func = latency_test + else: + raise ValueError( + "Provide --model-path for running the tests or " + "provide --result-filename for plotting the results" + ) + + port_args = PortArgs.init_new(server_args) + + if server_args.tp_size == 1: + work_func(server_args, port_args, bench_args, 0, 0) + else: + workers = [] + for tp_rank in range(server_args.tp_size): + with maybe_reindex_device_id(tp_rank) as gpu_id: + proc = multiprocessing.Process( + target=work_func, + args=( + server_args, + port_args, + bench_args, + gpu_id, + tp_rank, + ), + ) + proc.start() + workers.append(proc) + + for proc in workers: + proc.join() + + proc.terminate() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + BenchArgs.add_cli_args(parser) + args = parser.parse_args() + server_args = ServerArgs.from_cli_args(args) + bench_args = BenchArgs.from_cli_args(args) + + logging.basicConfig( + level=getattr(logging, server_args.log_level.upper()), + format="%(message)s", + ) + + try: + main(server_args, bench_args) + finally: + if server_args.tp_size != 1: + kill_process_tree(os.getpid(), include_parent=False) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/bench_one_batch_server.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/bench_one_batch_server.py new file mode 100644 index 00000000..159a3a93 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/bench_one_batch_server.py @@ -0,0 +1,605 @@ +""" +Benchmark the latency of running a single batch with a server. + +This script launches a server and uses the HTTP interface. +It accepts server arguments (the same as launch_server.py) and benchmark arguments (e.g., batch size, input lengths). + +Usage: +python3 -m sglang.bench_one_batch_server --model meta-llama/Meta-Llama-3.1-8B --batch-size 1 16 64 --input-len 1024 --output-len 8 + +python3 -m sglang.bench_one_batch_server --model None --base-url http://localhost:30000 --batch-size 16 --input-len 1024 --output-len 8 +python3 -m sglang.bench_one_batch_server --model None --base-url http://localhost:30000 --batch-size 16 --input-len 1024 --output-len 8 --show-report --profile --profile-by-stage +python3 -m sglang.bench_one_batch_server --model None --base-url http://localhost:30000 --batch-size 16 --input-len 1024 --output-len 8 --output-path results.json --profile +""" + +import argparse +import dataclasses +import itertools +import json +import multiprocessing +import os +import random +import time +from typing import List, Optional, Tuple + +import numpy as np +import requests +from pydantic import BaseModel +from transformers import AutoProcessor, PreTrainedTokenizer + +from sglang.bench_serving import ( + get_processor, + get_tokenizer, + sample_mmmu_requests, + sample_random_requests, +) +from sglang.profiler import run_profile +from sglang.srt.entrypoints.http_server import launch_server +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import is_blackwell, kill_process_tree +from sglang.test.nightly_bench_utils import save_results_as_pydantic_models +from sglang.test.test_utils import is_in_ci, write_github_step_summary + + +@dataclasses.dataclass +class BenchArgs: + run_name: str = "default" + batch_size: Tuple[int] = (1,) + input_len: Tuple[int] = (1024,) + output_len: Tuple[int] = (16,) + temperature: float = 0.0 + return_logprob: bool = False + client_stream_interval: int = 1 + input_len_step_percentage: float = 0.0 + base_url: str = "" + skip_warmup: bool = False + show_report: bool = False + profile: bool = False + profile_steps: int = 5 + profile_by_stage: bool = False + profile_prefix: Optional[str] = None + profile_output_dir: Optional[str] = None + dataset_path: str = "" + dataset_name: str = "random" + parallel_batch: bool = False + result_filename: str = "result.jsonl" + pydantic_result_filename: Optional[str] = None + append_to_github_summary: bool = True + seed: int = 42 + + @staticmethod + def add_cli_args(parser: argparse.ArgumentParser): + parser.add_argument("--run-name", type=str, default=BenchArgs.run_name) + parser.add_argument( + "--batch-size", type=int, nargs="+", default=BenchArgs.batch_size + ) + parser.add_argument( + "--input-len", type=int, nargs="+", default=BenchArgs.input_len + ) + parser.add_argument( + "--output-len", type=int, nargs="+", default=BenchArgs.output_len + ) + parser.add_argument("--temperature", type=float, default=BenchArgs.temperature) + parser.add_argument("--return-logprob", action="store_true") + parser.add_argument( + "--client-stream-interval", + type=int, + default=BenchArgs.client_stream_interval, + ) + parser.add_argument( + "--input-len-step-percentage", + type=float, + default=BenchArgs.input_len_step_percentage, + ) + parser.add_argument("--base-url", type=str, default=BenchArgs.base_url) + parser.add_argument("--skip-warmup", action="store_true") + parser.add_argument("--show-report", action="store_true") + parser.add_argument("--profile", action="store_true") + parser.add_argument( + "--profile-steps", type=int, default=BenchArgs.profile_steps + ) + parser.add_argument("--profile-by-stage", action="store_true") + parser.add_argument( + "--profile-prefix", + type=str, + default=BenchArgs.profile_prefix, + ) + parser.add_argument( + "--profile-output-dir", + type=str, + default=BenchArgs.profile_output_dir, + ) + parser.add_argument( + "--dataset-path", + type=str, + default=BenchArgs.dataset_path, + help="Path to the dataset.", + ) + parser.add_argument( + "--dataset-name", + type=str, + default=BenchArgs.dataset_name, + choices=["mmmu", "random"], + help="Name of the dataset to benchmark on.", + ) + parser.add_argument("--parallel-batch", action="store_true") + parser.add_argument( + "--result-filename", + type=str, + default=BenchArgs.result_filename, + help="Store the results line by line in the JSON Line format to this file.", + ) + parser.add_argument( + "--pydantic-result-filename", + type=str, + default=BenchArgs.pydantic_result_filename, + help="Store the results as pydantic models in the JSON format to this file.", + ) + parser.add_argument( + "--no-append-to-github-summary", + action="store_false", + dest="append_to_github_summary", + help="Disable appending the output of this run to github ci summary", + ) + parser.add_argument("--seed", type=int, default=BenchArgs.seed) + + @classmethod + def from_cli_args(cls, args: argparse.Namespace): + attrs = [attr.name for attr in dataclasses.fields(cls)] + return cls(**{attr: getattr(args, attr) for attr in attrs}) + + +class BenchOneCaseResult(BaseModel): + run_name: str + batch_size: int + input_len: int + output_len: int + latency: float + input_throughput: float + output_throughput: float + overall_throughput: float + last_ttft: float + last_gen_throughput: float + acc_length: float + profile_link: Optional[str] = None + + def dump_to_jsonl(self, result_filename: str): + with open(result_filename, "a") as fout: + res = { + "run_name": self.run_name, + "batch_size": self.batch_size, + "input_len": self.input_len, + "output_len": self.output_len, + "latency": round(self.latency, 4), + "input_throughput": round(self.input_throughput, 2), + "output_throughput": round(self.output_throughput, 2), + "overall_throughput": round(self.overall_throughput, 2), + "last_ttft": round(self.last_ttft, 4), + "last_gen_throughput": round(self.last_gen_throughput, 2), + "acc_length": round(self.acc_length, 2), + } + fout.write(json.dumps(res) + "\n") + + +def launch_server_internal(server_args): + try: + launch_server(server_args) + except Exception as e: + raise e + finally: + kill_process_tree(os.getpid(), include_parent=False) + + +def launch_server_process(server_args: ServerArgs): + proc = multiprocessing.Process(target=launch_server_internal, args=(server_args,)) + proc.start() + base_url = f"http://{server_args.host}:{server_args.port}" + timeout = 600 + + start_time = time.time() + while time.time() - start_time < timeout: + try: + headers = { + "Content-Type": "application/json; charset=utf-8", + } + response = requests.get(f"{base_url}/v1/models", headers=headers) + if response.status_code == 200: + return proc, base_url + except requests.RequestException: + pass + time.sleep(10) + raise TimeoutError("Server failed to start within the timeout period.") + + +def run_one_case( + url: str, + batch_size: int, + input_len: int, + output_len: int, + temperature: float, + return_logprob: bool, + stream_interval: int, + input_len_step_percentage: float, + run_name: str, + result_filename: str, + tokenizer: PreTrainedTokenizer | AutoProcessor, + profile: bool = False, + profile_steps: int = BenchArgs.profile_steps, + profile_by_stage: bool = False, + profile_prefix: Optional[str] = BenchArgs.profile_prefix, + profile_output_dir: Optional[str] = BenchArgs.profile_output_dir, + dataset_name: str = BenchArgs.dataset_name, + dataset_path: str = BenchArgs.dataset_path, + parallel_batch: bool = False, +): + requests.post(url + "/flush_cache") + + # Load input token ids + # TODO: reuse bench_serving.get_dataset ? + if dataset_name == "mmmu": + input_requests = sample_mmmu_requests( + num_requests=batch_size, + processor=tokenizer, + fixed_output_len=output_len, + random_sample=False, + ) + elif dataset_name == "random": + input_requests = sample_random_requests( + input_len=input_len, + output_len=output_len, + num_prompts=batch_size, + range_ratio=1.0, + tokenizer=tokenizer, + dataset_path=dataset_path, + random_sample=True, + return_text=False, + ) + + # Load sampling parameters + use_structured_outputs = False + if use_structured_outputs: + texts = [] + for _ in range(batch_size): + texts.append( + "Human: What is the capital city of france? can you give as many trivial information as possible about that city? answer in json.\n" + * 50 + + "Assistant:" + ) + json_schema = "$$ANY$$" + else: + json_schema = None + + payload = { + "sampling_params": { + "temperature": temperature, + "max_new_tokens": output_len, + "ignore_eos": True, + "json_schema": json_schema, + "stream_interval": stream_interval, + }, + "return_logprob": return_logprob, + "stream": True, + **({"parallel_batch": parallel_batch} if parallel_batch else {}), + } + if dataset_name == "mmmu": + # vlm + input_ids = [] + # for vlms, tokenizer is an instance of AutoProcessor + tokenizer = tokenizer.tokenizer + for input_req in input_requests: + input_ids += [tokenizer.encode(input_req.prompt)] + payload["image_data"] = [req.image_data for req in input_requests] + + else: + input_ids = [req.prompt for req in input_requests] + + payload["input_ids"] = input_ids + + # Turn on profiler + profile_link = None + if profile: + profile_link: str = run_profile( + url=url, + num_steps=profile_steps, + activities=["CPU", "GPU"], + output_dir=profile_output_dir, + profile_by_stage=profile_by_stage, + profile_prefix=profile_prefix, + ) + + # Run the request + tic = time.perf_counter() + response = requests.post( + url + "/generate", + json=payload, + stream=True, + ) + + # Get the TTFT of the last request in the batch + last_ttft = 0.0 + for chunk in response.iter_lines(decode_unicode=False): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]": + break + data = json.loads(chunk[5:].strip("\n")) + if "error" in data: + raise RuntimeError(f"Request has failed. {data}.") + + assert ( + data["meta_info"]["finish_reason"] is None + or data["meta_info"]["finish_reason"]["type"] == "length" + ) + if data["meta_info"]["completion_tokens"] == 1: + last_ttft = time.perf_counter() - tic + + # Compute metrics + latency = time.perf_counter() - tic + input_throughput = batch_size * input_len / last_ttft + output_throughput = batch_size * output_len / (latency - last_ttft) + overall_throughput = batch_size * (input_len + output_len) / latency + + server_info = requests.get(url + "/get_server_info").json() + internal_state = server_info.get("internal_states", [{}]) + last_gen_throughput = internal_state[0].get("last_gen_throughput", None) or -1 + acc_length = internal_state[0].get("avg_spec_accept_length", None) or -1 + + # Print results + print(f"batch size: {batch_size}") + print(f"input_len: {input_len}") + print(f"output_len: {output_len}") + print(f"latency: {latency:.2f} s") + print(f"input throughput: {input_throughput:.2f} tok/s") + if output_len != 1: + print(f"output throughput: {output_throughput:.2f} tok/s") + print(f"last_ttft: {last_ttft:.2f} s") + print(f"last generation throughput: {last_gen_throughput:.2f} tok/s") + if acc_length > 0: + print(f"acc_length: {acc_length:.2f} ") + + # Dump results + result = BenchOneCaseResult( + run_name=run_name, + batch_size=batch_size, + input_len=input_len, + output_len=output_len, + latency=latency, + input_throughput=input_throughput, + output_throughput=output_throughput, + overall_throughput=overall_throughput, + last_ttft=last_ttft, + last_gen_throughput=last_gen_throughput, + acc_length=acc_length, + profile_link=profile_link, + ) + + # Save and return the results + if result_filename: + result.dump_to_jsonl(result_filename) + + return result + + +def should_skip_due_to_token_capacity( + batch_size, input_len, output_len, skip_token_capacity_threshold +): + if batch_size * (input_len + output_len) > skip_token_capacity_threshold: + print( + "=" * 8 + + f"Skip benchmark {batch_size=} * ({input_len=} + {output_len=}) = {batch_size * (input_len + output_len)} > {skip_token_capacity_threshold=} due to kv cache limit." + + "=" * 8 + ) + return True + return False + + +def get_report_summary( + results: List[BenchOneCaseResult], bench_args: BenchArgs, server_args: ServerArgs +): + summary = ( + f"\nInput lens: {bench_args.input_len}. Output lens: {bench_args.output_len}.\n" + ) + summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | acc length | ITL (ms) | input cost ($/1M) | output cost ($/1M) |" + + if bench_args.profile: + summary += " profile |" + + summary += "\n" + summary += "| ---------- | --------- | ----------- | ------------------------- | ------------------------- | ---------- | -------- | ----------------- | ------------------ |" + + if bench_args.profile: + summary += "-------------|" + summary += "\n" + + if is_blackwell(): + hourly_cost_per_gpu = 4 # $4/hour for one B200 + else: + hourly_cost_per_gpu = 2 # $2/hour for one H100 + input_util = 0.7 + + # sort result by input_len + results.sort(key=lambda x: x.input_len) + for res in results: + hourly_cost = hourly_cost_per_gpu * server_args.tp_size + accept_length = round(res.acc_length, 2) if res.acc_length > 0 else "n/a" + line = ( + f"| {res.batch_size} | " + f"{res.input_len} | " + f"{res.latency:.2f} | " + f"{res.input_throughput:.2f} | " + f"{res.output_throughput:.2f} | " + f"{accept_length} | " + f"{1 / (res.output_throughput/res.batch_size) * 1000:.2f} | " + f"{1e6 / (res.input_throughput * input_util) / 3600 * hourly_cost:.2f} | " + f"{1e6 / res.output_throughput / 3600 * hourly_cost:.2f} |" + ) + if bench_args.profile: + if res.profile_link: + line += f" [Profile]({res.profile_link}) |" + else: + line += f" n/a |" + line += "\n" + summary += line + + return summary + + +def run_benchmark(server_args: ServerArgs, bench_args: BenchArgs): + if bench_args.base_url: + proc, base_url = None, bench_args.base_url + else: + proc, base_url = launch_server_process(server_args) + + # Get tokenizer + server_info = requests.get(base_url + "/get_server_info").json() + if "tokenizer_path" in server_info: + tokenizer_path = server_info["tokenizer_path"] + elif "prefill" in server_info: + tokenizer_path = server_info["prefill"][0]["tokenizer_path"] + if bench_args.dataset_name == "mmmu": + # mmmu implies this is a MLLM + tokenizer = get_processor(tokenizer_path) + else: + tokenizer = get_tokenizer(tokenizer_path) + + # Get token capacity + internal_state = server_info.get("internal_states", [{}]) + skip_token_capacity_threshold = ( + internal_state[0].get("memory_usage", {}).get("token_capacity", 1000000000) + ) + + # Warmup + if not bench_args.skip_warmup: + print("=" * 8 + " Warmup Begin " + "=" * 8) + print(f"Warmup with batch_size={bench_args.batch_size}") + for bs in bench_args.batch_size: + run_one_case( + base_url, + batch_size=bs, + input_len=1024, + output_len=16, + temperature=bench_args.temperature, + return_logprob=bench_args.return_logprob, + stream_interval=bench_args.client_stream_interval, + input_len_step_percentage=bench_args.input_len_step_percentage, + run_name="", + result_filename="", + tokenizer=tokenizer, + dataset_name=bench_args.dataset_name, + dataset_path=bench_args.dataset_path, + parallel_batch=bench_args.parallel_batch, + ) + print("=" * 8 + " Warmup End " + "=" * 8 + "\n") + + results = [] + profile_results = [] + try: + # Benchmark all cases + for bs, il, ol in itertools.product( + bench_args.batch_size, bench_args.input_len, bench_args.output_len + ): + if should_skip_due_to_token_capacity( + bs, il, ol, skip_token_capacity_threshold + ): + continue + results.append( + run_one_case( + base_url, + bs, + il, + ol, + temperature=bench_args.temperature, + return_logprob=bench_args.return_logprob, + stream_interval=bench_args.client_stream_interval, + input_len_step_percentage=bench_args.input_len_step_percentage, + run_name=bench_args.run_name, + result_filename=bench_args.result_filename, + tokenizer=tokenizer, + dataset_name=bench_args.dataset_name, + dataset_path=bench_args.dataset_path, + parallel_batch=bench_args.parallel_batch, + ) + ) + + # Profile all cases + if bench_args.profile: + try: + for bs, il, ol in itertools.product( + bench_args.batch_size, bench_args.input_len, bench_args.output_len + ): + if should_skip_due_to_token_capacity( + bs, il, ol, skip_token_capacity_threshold + ): + continue + profile_prefix = ( + bench_args.profile_prefix or "" + ) + f"bs-{bs}-il-{il}" + profile_results.append( + run_one_case( + base_url, + bs, + il, + ol, + temperature=bench_args.temperature, + return_logprob=bench_args.return_logprob, + stream_interval=bench_args.client_stream_interval, + input_len_step_percentage=bench_args.input_len_step_percentage, + run_name=bench_args.run_name, + result_filename=bench_args.result_filename, + tokenizer=tokenizer, + dataset_name=bench_args.dataset_name, + dataset_path=bench_args.dataset_path, + parallel_batch=bench_args.parallel_batch, + profile=bench_args.profile, + profile_steps=bench_args.profile_steps, + profile_by_stage=bench_args.profile_by_stage, + profile_prefix=profile_prefix, + profile_output_dir=bench_args.profile_output_dir, + ) + ) + + # Replace the profile link + for res, profile_res in zip(results, profile_results): + res.profile_link = profile_res.profile_link + except Exception as e: + print(f"Error profiling, there will be no profile trace dump: {e}") + finally: + if proc: + kill_process_tree(proc.pid) + + print(f"\nResults are saved to {bench_args.result_filename}") + + if not bench_args.show_report: + return + + # Print summary + summary = get_report_summary(results, bench_args, server_args) + print(summary) + + if is_in_ci() and bench_args.append_to_github_summary: + write_github_step_summary(summary) + else: + print(summary) + + # Save results as pydantic models in the JSON format + if bench_args.pydantic_result_filename: + save_results_as_pydantic_models( + results, + pydantic_result_filename=bench_args.pydantic_result_filename, + model_path=server_args.model_path, + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + BenchArgs.add_cli_args(parser) + args = parser.parse_args() + + random.seed(args.seed) + np.random.seed(args.seed) + + server_args = ServerArgs.from_cli_args(args) + bench_args = BenchArgs.from_cli_args(args) + + run_benchmark(server_args, bench_args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/bench_serving.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/bench_serving.py new file mode 100644 index 00000000..00267489 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/bench_serving.py @@ -0,0 +1,2867 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/6366efc67b0aedd2c1721c14385370e50b297fb3/benchmarks/backend_request_func.py +# Adapted from https://github.com/vllm-project/vllm/blob/6366efc67b0aedd2c1721c14385370e50b297fb3/benchmarks/benchmark_serving.py + +""" +Benchmark online serving with dynamic requests. + +Usage: +python3 -m sglang.bench_serving --backend sglang --num-prompt 10 + +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 3000 --random-input 1024 --random-output 1024 --random-range-ratio 0.5 +""" + +import argparse +import asyncio +import importlib.util +import io +import json +import os +import pickle +import random +import resource +import shutil +import sys +import time +import traceback +import warnings +from argparse import ArgumentParser +from dataclasses import dataclass, field +from datetime import datetime +from functools import lru_cache +from json import JSONDecodeError +from pathlib import Path +from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple, Union + +import aiohttp +import numpy as np +import pybase64 +import requests +from datasets import load_dataset +from PIL import Image +from tqdm.asyncio import tqdm +from transformers import ( + AutoProcessor, + AutoTokenizer, + PreTrainedTokenizer, + PreTrainedTokenizerBase, + PreTrainedTokenizerFast, +) + +ASSISTANT_SUFFIX = "Assistant:" + +TERM_PLOTLIB_AVAILABLE = (importlib.util.find_spec("termplotlib") is not None) and ( + shutil.which("gnuplot") is not None +) + +global args + + +# don't want to import sglang package here +def _get_bool_env_var(name: str, default: str = "false") -> bool: + value = os.getenv(name, default) + return value.lower() in ("true", "1") + + +def _create_bench_client_session(): + # When the pressure is big, the read buffer could be full before aio thread read + # the content. We increase the read_bufsize from 64K to 10M. + # Define constants for timeout and buffer size for clarity and maintainability + BENCH_AIOHTTP_TIMEOUT_SECONDS = 6 * 60 * 60 # 6 hours + BENCH_AIOHTTP_READ_BUFSIZE_BYTES = 10 * 1024**2 # 10 MB + + aiohttp_timeout = aiohttp.ClientTimeout(total=BENCH_AIOHTTP_TIMEOUT_SECONDS) + return aiohttp.ClientSession( + timeout=aiohttp_timeout, read_bufsize=BENCH_AIOHTTP_READ_BUFSIZE_BYTES + ) + + +@dataclass +class RequestFuncInput: + prompt: str + api_url: str + prompt_len: int + output_len: int + model: str + lora_name: str + image_data: Optional[List[str]] + extra_request_body: Dict[str, Any] + timestamp: Optional[float] = None + + +@dataclass +class RequestFuncOutput: + generated_text: str = "" + success: bool = False + latency: float = 0.0 + ttft: float = 0.0 # Time to first token + itl: List[float] = field(default_factory=list) # List of inter-token latencies + text_chunks: List[str] = field(default_factory=list) + prompt_len: int = 0 + error: str = "" + output_len: int = 0 + start_time: float = 0.0 + + @staticmethod + def init_new(request_func_input: RequestFuncInput): + output = RequestFuncOutput() + output.prompt_len = request_func_input.prompt_len + return output + + +def remove_prefix(text: str, prefix: str) -> str: + return text[len(prefix) :] if text.startswith(prefix) else text + + +def remove_suffix(text: str, suffix: str) -> str: + return text[: -len(suffix)] if text.endswith(suffix) else text + + +def get_auth_headers() -> Dict[str, str]: + openai_api_key = os.environ.get("OPENAI_API_KEY") + if openai_api_key: + return {"Authorization": f"Bearer {openai_api_key}"} + else: + api_key = os.environ.get("API_KEY") + if api_key: + return {"Authorization": f"{api_key}"} + return {} + + +# trt llm does not support ignore_eos +# https://github.com/triton-inference-server/tensorrtllm_backend/issues/505 +async def async_request_trt_llm( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + assert api_url.endswith("generate_stream") + + async with _create_bench_client_session() as session: + payload = { + "accumulate_tokens": True, + "text_input": request_func_input.prompt, + "temperature": 0.000001, + "top_p": 1.0, + "max_tokens": request_func_input.output_len, + "stream": True, + "min_length": request_func_input.output_len, + "end_id": 1048576, + **request_func_input.extra_request_body, + } + if args.disable_ignore_eos: + del payload["min_length"] + del payload["end_id"] + output = RequestFuncOutput.init_new(request_func_input) + + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + try: + async with session.post(url=api_url, json=payload) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data:") + + data = json.loads(chunk) + output.generated_text += data["text_output"] + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = timestamp - st + output.ttft = ttft + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + most_recent_timestamp = timestamp + + output.latency = most_recent_timestamp - st + output.success = True + output.output_len = request_func_input.output_len + + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +# set ignore_eos True by default +async def async_request_openai_completions( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + assert api_url.endswith( + "completions" + ), "OpenAI Completions API URL must end with 'completions'." + + prompt = request_func_input.prompt + + async with _create_bench_client_session() as session: + payload = { + "model": request_func_input.model, + "prompt": prompt, + "temperature": 0.0, + "best_of": 1, + "max_tokens": request_func_input.output_len, + "stream": not args.disable_stream, + "ignore_eos": not args.disable_ignore_eos, + **request_func_input.extra_request_body, + } + + # hack to accommodate different LoRA conventions between SGLang and vLLM. + if request_func_input.lora_name: + payload["model"] = request_func_input.lora_name + payload["lora_path"] = request_func_input.lora_name + + if request_func_input.image_data: + payload.update({"image_data": request_func_input.image_data}) + + headers = get_auth_headers() + + output = RequestFuncOutput.init_new(request_func_input) + + generated_text = "" + output_len = request_func_input.output_len + ttft = 0.0 + st = time.perf_counter() + output.start_time = st + most_recent_timestamp = st + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + # NOTE: Some completion API might have a last + # usage summary response without a token so we + # want to check a token was generated + if data["choices"][0]["text"]: + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft = ttft + + # Decoding phase + else: + output.text_chunks.append( + data["choices"][0]["text"] + ) + output.itl.append(timestamp - most_recent_timestamp) + + most_recent_timestamp = timestamp + generated_text += data["choices"][0]["text"] + output_len = (data.get("usage") or {}).get( + "completion_tokens", output_len + ) + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.output_len = output_len + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +async def async_request_openai_chat_completions( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + """Makes a request to the OpenAI Chat Completions API. + + Handles both streaming and non-streaming responses, including support + for image data in messages. Calculates and returns various performance + metrics. + + Args: + request_func_input: Input parameters for the request. + pbar: Optional tqdm progress bar to update. + + Returns: + RequestFuncOutput: Output of the request, including generated text, + latency, TTFT, ITL, and success status. + """ + api_url = request_func_input.api_url + assert api_url.endswith( + "chat/completions" + ), "OpenAI Chat Completions API URL must end with 'chat/completions'." + + if request_func_input.image_data: + # Build multi-image content: a list of image_url entries followed by the text + content_items = [ + { + "type": "image_url", + "image_url": {"url": img_url}, + } + for img_url in request_func_input.image_data + ] + content_items.append({"type": "text", "text": request_func_input.prompt}) + messages = [ + { + "role": "user", + "content": content_items, + }, + ] + else: + messages = [{"role": "user", "content": request_func_input.prompt}] + + async with _create_bench_client_session() as session: + payload = { + "model": request_func_input.model, + "messages": messages, + "temperature": 0.0, + "max_completion_tokens": request_func_input.output_len, + "stream": not args.disable_stream, + "ignore_eos": not args.disable_ignore_eos, + **request_func_input.extra_request_body, + } + + # hack to accommodate different LoRA conventions between SGLang and vLLM. + if request_func_input.lora_name: + payload["model"] = request_func_input.lora_name + payload["lora_path"] = request_func_input.lora_name + + headers = get_auth_headers() + + output = RequestFuncOutput.init_new(request_func_input) + + generated_text = "" + output_len = request_func_input.output_len + ttft = 0.0 + st = time.perf_counter() + output.start_time = st + most_recent_timestamp = st + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + if args.disable_stream: + # Non-streaming response + response_json = await response.json() + output.generated_text = response_json["choices"][0]["message"][ + "content" + ] + output.success = True + output.latency = time.perf_counter() - st + output.ttft = ( + output.latency + ) # For non-streaming, TTFT = total latency + output.output_len = response_json.get("usage", {}).get( + "completion_tokens", output_len + ) + else: + # Streaming response + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + # Check if this chunk contains content + delta = data.get("choices", [{}])[0].get("delta", {}) + content = delta.get("content", "") + + if content: + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = timestamp - st + output.ttft = ttft + + # Decoding phase + else: + output.text_chunks.append(content) + output.itl.append( + timestamp - most_recent_timestamp + ) + + most_recent_timestamp = timestamp + generated_text += content + + # Check for usage info in final chunk + output_len = (data.get("usage") or {}).get( + "completion_tokens", output_len + ) + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.output_len = output_len + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +async def async_request_truss( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + + prompt = request_func_input.prompt + + async with _create_bench_client_session() as session: + payload = { + "model": request_func_input.model, + "prompt": prompt, + "temperature": 0.0, + "best_of": 1, + "max_tokens": request_func_input.output_len, + "stream": not args.disable_stream, + "ignore_eos": not args.disable_ignore_eos, + **request_func_input.extra_request_body, + } + headers = get_auth_headers() + + output = RequestFuncOutput.init_new(request_func_input) + + generated_text = "" + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + # NOTE: Some completion API might have a last + # usage summary response without a token so we + # want to check a token was generated + if data["choices"][0]["text"]: + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft = ttft + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + most_recent_timestamp = timestamp + generated_text += data["choices"][0]["text"] + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.output_len = request_func_input.output_len + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +async def async_request_sglang_generate( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + prompt = request_func_input.prompt + + async with _create_bench_client_session() as session: + payload = { + ("text" if isinstance(prompt, str) else "input_ids"): prompt, + "sampling_params": { + "temperature": 0.0, + "max_new_tokens": request_func_input.output_len, + "ignore_eos": not args.disable_ignore_eos, + }, + "stream": not args.disable_stream, + "lora_path": request_func_input.lora_name, + "return_logprob": args.return_logprob, + "logprob_start_len": -1, + **request_func_input.extra_request_body, + } + + # Add image data if available (list of image urls/base64) + if request_func_input.image_data: + payload["image_data"] = request_func_input.image_data + + headers = get_auth_headers() + + output = RequestFuncOutput.init_new(request_func_input) + + generated_text = "" + output_len = request_func_input.output_len + ttft = 0.0 + st = time.perf_counter() + output.start_time = st + most_recent_timestamp = st + last_output_len = 0 + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + # NOTE: Some completion API might have a last + # usage summary response without a token so we + # want to check a token was generated + if "text" in data and data["text"]: + timestamp = time.perf_counter() + generated_text = data["text"] + output_len = data["meta_info"]["completion_tokens"] + + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft = ttft + + # Decoding phase + else: + num_new_tokens = output_len - last_output_len + if num_new_tokens == 0: + continue + chunk_gap = timestamp - most_recent_timestamp + adjust_itl = chunk_gap / num_new_tokens + output.itl.extend([adjust_itl] * num_new_tokens) + + most_recent_timestamp = timestamp + last_output_len = output_len + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.output_len = output_len + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + print(f"{output.error=}") + + if pbar: + pbar.update(1) + return output + + +async def async_request_gserver( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + raise NotImplementedError() + + +async def async_request_profile(api_url: str) -> RequestFuncOutput: + async with _create_bench_client_session() as session: + output = RequestFuncOutput() + try: + body = { + "activities": getattr(args, "profile_activities", []), + "num_steps": getattr(args, "profile_num_steps", None), + "profile_by_stage": getattr(args, "profile_by_stage", None), + "profile_stages": getattr(args, "profile_stages", None), + } + print(f"async_request_profile {api_url=} {body=}") + async with session.post(url=api_url, json=body) as response: + if response.status == 200: + output.success = True + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + return output + + +def _build_profile_urls( + profile_prefill_url: Optional[List[str]], + profile_decode_url: Optional[List[str]], +) -> List[Tuple[str, str]]: + """Build profile URLs list from prefill/decode URL arguments. + + Returns: + List of (worker_type, url) tuples. e.g., [("Prefill-0", "http://..."), ("Decode-0", "http://...")] + """ + profile_urls = [] + if profile_prefill_url: + for idx, url in enumerate(profile_prefill_url): + profile_urls.append((f"Prefill-{idx}", url)) + if profile_decode_url: + for idx, url in enumerate(profile_decode_url): + profile_urls.append((f"Decode-{idx}", url)) + return profile_urls + + +async def _call_profile_pd(profile_urls: List[Tuple[str, str]], mode: str) -> None: + """Call profile endpoint (start/stop) on PD separated workers. + + Args: + profile_urls: List of (worker_type, url) tuples + mode: "start" or "stop" + """ + endpoint = "/start_profile" if mode == "start" else "/stop_profile" + action = "Starting" if mode == "start" else "Stopping" + action_past = "started" if mode == "start" else "stopped" + + print(f"{action} profiler...") + + for worker_type, url in profile_urls: + profile_output = await async_request_profile(api_url=url + endpoint) + if profile_output.success: + print(f"Profiler {action_past} for {worker_type} worker at {url}") + else: + print( + f"Failed to {mode} profiler for {worker_type} worker at {url}: {profile_output.error}" + ) + + +def get_model(pretrained_model_name_or_path: str) -> str: + if os.getenv("SGLANG_USE_MODELSCOPE", "false").lower() == "true": + import huggingface_hub.constants + from modelscope import snapshot_download + + model_path = snapshot_download( + model_id=pretrained_model_name_or_path, + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE, + ignore_file_pattern=[".*.pt", ".*.safetensors", ".*.bin"], + ) + + return model_path + return pretrained_model_name_or_path + + +def get_tokenizer( + pretrained_model_name_or_path: str, +) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]: + assert ( + pretrained_model_name_or_path is not None + and pretrained_model_name_or_path != "" + ) + if pretrained_model_name_or_path.endswith( + ".json" + ) or pretrained_model_name_or_path.endswith(".model"): + from sglang.srt.utils.hf_transformers_utils import get_tokenizer + + return get_tokenizer(pretrained_model_name_or_path) + + if pretrained_model_name_or_path is not None and not os.path.exists( + pretrained_model_name_or_path + ): + pretrained_model_name_or_path = get_model(pretrained_model_name_or_path) + return AutoTokenizer.from_pretrained( + pretrained_model_name_or_path, trust_remote_code=True + ) + + +def get_processor( + pretrained_model_name_or_path: str, +) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]: + assert ( + pretrained_model_name_or_path is not None + and pretrained_model_name_or_path != "" + ) + if pretrained_model_name_or_path.endswith( + ".json" + ) or pretrained_model_name_or_path.endswith(".model"): + from sglang.srt.utils.hf_transformers_utils import get_processor + + return get_processor(pretrained_model_name_or_path) + + if pretrained_model_name_or_path is not None and not os.path.exists( + pretrained_model_name_or_path + ): + pretrained_model_name_or_path = get_model(pretrained_model_name_or_path) + return AutoProcessor.from_pretrained( + pretrained_model_name_or_path, trust_remote_code=True + ) + + +def get_dataset(args, tokenizer, model_id=None): + tokenize_prompt = getattr(args, "tokenize_prompt", False) + if args.dataset_name == "sharegpt": + assert not tokenize_prompt + input_requests = sample_sharegpt_requests( + dataset_path=args.dataset_path, + num_requests=args.num_prompts, + tokenizer=tokenizer, + fixed_output_len=args.sharegpt_output_len, + context_len=args.sharegpt_context_len, + prompt_suffix=args.prompt_suffix, + apply_chat_template=args.apply_chat_template, + ) + elif args.dataset_name.startswith("random"): + input_requests = sample_random_requests( + input_len=args.random_input_len, + output_len=args.random_output_len, + num_prompts=args.num_prompts, + range_ratio=args.random_range_ratio, + tokenizer=tokenizer, + dataset_path=args.dataset_path, + random_sample=args.dataset_name == "random", + return_text=not tokenize_prompt, + ) + elif args.dataset_name == "image": + processor = get_processor(model_id) + input_requests = sample_image_requests( + num_requests=args.num_prompts, + image_count=args.image_count, + input_len=args.random_input_len, + output_len=args.random_output_len, + range_ratio=args.random_range_ratio, + processor=processor, + image_content=args.image_content, + image_format=args.image_format, + image_resolution=args.image_resolution, + backend=args.backend, + ) + elif args.dataset_name == "generated-shared-prefix": + assert not tokenize_prompt + input_requests = sample_generated_shared_prefix_requests( + num_groups=args.gsp_num_groups, + prompts_per_group=args.gsp_prompts_per_group, + system_prompt_len=args.gsp_system_prompt_len, + question_len=args.gsp_question_len, + output_len=args.gsp_output_len, + tokenizer=tokenizer, + args=args, + ) + elif args.dataset_name == "mmmu": + processor = get_processor(model_id) + input_requests = sample_mmmu_requests( + num_requests=args.num_prompts, + processor=processor, + backend=args.backend, + fixed_output_len=args.random_output_len, + random_sample=True, + ) + elif args.dataset_name == "mooncake": + # For mooncake, we don't generate the prompts here. + # We just load the raw trace data. The async generator will handle the rest. + if not args.dataset_path: + local_path = os.path.join("/tmp", args.mooncake_workload + "_trace.jsonl") + else: + local_path = args.dataset_path + + if not os.path.exists(local_path): + download_and_cache_file( + MOONCAKE_DATASET_URL[args.mooncake_workload], local_path + ) + + with open(local_path, "r") as f: + all_requests_data = [json.loads(line) for line in f if line.strip()] + + # Limit the number of requests based on --num-prompts + input_requests = all_requests_data[: args.num_prompts] + else: + raise ValueError(f"Unknown dataset: {args.dataset_name}") + return input_requests + + +ASYNC_REQUEST_FUNCS = { + "sglang": async_request_sglang_generate, + "sglang-native": async_request_sglang_generate, + "sglang-oai": async_request_openai_completions, + "sglang-oai-chat": async_request_openai_chat_completions, + "vllm": async_request_openai_completions, + "vllm-chat": async_request_openai_chat_completions, + "lmdeploy": async_request_openai_completions, + "lmdeploy-chat": async_request_openai_chat_completions, + "trt": async_request_trt_llm, + "gserver": async_request_gserver, + "truss": async_request_truss, +} + + +@dataclass +class BenchmarkMetrics: + completed: int + total_input: int + total_input_text: int + total_input_vision: int + total_output: int + total_output_retokenized: int + request_throughput: float + input_throughput: float + output_throughput: float + output_throughput_retokenized: float + total_throughput: float + total_throughput_retokenized: float + mean_ttft_ms: float + median_ttft_ms: float + std_ttft_ms: float + p99_ttft_ms: float + mean_tpot_ms: float + median_tpot_ms: float + std_tpot_ms: float + p99_tpot_ms: float + mean_itl_ms: float + median_itl_ms: float + std_itl_ms: float + p95_itl_ms: float + p99_itl_ms: float + max_itl_ms: float + mean_e2e_latency_ms: float + median_e2e_latency_ms: float + std_e2e_latency_ms: float + p99_e2e_latency_ms: float + concurrency: float + max_output_tokens_per_s: float = 0.0 + max_concurrent_requests: int = 0 + + +SHAREGPT_URL = "https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json" +MOONCAKE_DATASET_URL = { + "mooncake": "https://raw.githubusercontent.com/kvcache-ai/Mooncake/main/FAST25-release/arxiv-trace/mooncake_trace.jsonl", + "conversation": "https://raw.githubusercontent.com/kvcache-ai/Mooncake/main/FAST25-release/traces/conversation_trace.jsonl", + "synthetic": "https://raw.githubusercontent.com/kvcache-ai/Mooncake/main/FAST25-release/traces/synthetic_trace.jsonl", + "toolagent": "https://raw.githubusercontent.com/kvcache-ai/Mooncake/main/FAST25-release/traces/toolagent_trace.jsonl", +} + + +def download_and_cache_file(url: str, filename: Optional[str] = None): + """Read and cache a file from a url.""" + if filename is None: + filename = os.path.join("/tmp", url.split("/")[-1]) + + # Check if the cache file already exists + if is_file_valid_json(filename): + return filename + + print(f"Downloading from {url} to {filename}") + + # Stream the response to show the progress bar + response = requests.get(url, stream=True) + response.raise_for_status() # Check for request errors + + # Total size of the file in bytes + total_size = int(response.headers.get("content-length", 0)) + chunk_size = 1024 # Download in chunks of 1KB + + # Use tqdm to display the progress bar + with open(filename, "wb") as f, tqdm( + desc=filename, + total=total_size, + unit="B", + unit_scale=True, + unit_divisor=1024, + ) as bar: + for chunk in response.iter_content(chunk_size=chunk_size): + f.write(chunk) + bar.update(len(chunk)) + + return filename + + +def is_file_valid_json(path): + if not os.path.isfile(path): + return False + + # TODO can fuse into the real file open later + try: + with open(path) as f: + json.load(f) + return True + except JSONDecodeError as e: + print( + f"{path} exists but json loading fails ({e=}), thus treat as invalid file" + ) + return False + + +@dataclass +class DatasetRow: + prompt: str + prompt_len: int + output_len: int + text_prompt_len: Optional[int] = None + vision_prompt_len: Optional[int] = None + image_data: Optional[List[str]] = None + timestamp: Optional[float] = None + + def __post_init__(self): + if self.text_prompt_len is None: + self.text_prompt_len = self.prompt_len + if self.vision_prompt_len is None: + self.vision_prompt_len = 0 + + +async def get_mooncake_request_over_time( + input_requests: List[Dict], + tokenizer: PreTrainedTokenizerBase, + slowdown_factor: float, + num_rounds: int, +) -> AsyncGenerator[DatasetRow, None]: + """ + An async generator that yields requests based on the timestamps in the Mooncake trace file, + with support for multi-round sessions. + """ + if not input_requests: + return + + input_requests.sort(key=lambda r: r["timestamp"]) + + start_time = time.perf_counter() + trace_start_time_ms = input_requests[0]["timestamp"] + + for record in input_requests: + # Calculate when this entire session should start + relative_arrival_time_s = (record["timestamp"] - trace_start_time_ms) / 1000.0 + target_arrival_time_s = relative_arrival_time_s * slowdown_factor + + current_elapsed_time_s = time.perf_counter() - start_time + sleep_duration_s = target_arrival_time_s - current_elapsed_time_s + if sleep_duration_s > 0: + await asyncio.sleep(sleep_duration_s) + + # Once the session starts, generate all rounds for it as a burst + # This simulates a user engaging in a multi-turn conversation + + # Base user query constructed from hash_ids + user_query_base = "" + hash_ids = record.get("hash_ids", []) + for hash_id in hash_ids: + user_query_base += f"{hash_id}" + " ".join( + ["hi"] * 128 + ) # Shorter for multi-round + user_query_base += "Tell me a story based on this context." + + output_len_per_round = record.get("output_length", 256) + chat_history = [] + + for i in range(num_rounds): + # Add user query for the current round + chat_history.append( + {"role": "user", "content": f"Round {i + 1}: {user_query_base}"} + ) + + # Form the full prompt from history + try: + full_prompt_text = tokenizer.apply_chat_template( + chat_history, + tokenize=False, + add_generation_prompt=True, + return_dict=False, + ) + except Exception: + full_prompt_text = "\n".join( + [f"{msg['role']}: {msg['content']}" for msg in chat_history] + ) + + prompt_len = len(tokenizer.encode(full_prompt_text)) + + yield DatasetRow( + prompt=full_prompt_text, + prompt_len=prompt_len, + output_len=output_len_per_round, + ) + + # Add a placeholder assistant response for the next round's context + # We use a placeholder because we don't know the real response + placeholder_response = " ".join(["story"] * output_len_per_round) + chat_history.append({"role": "assistant", "content": placeholder_response}) + + +def sample_mmmu_requests( + num_requests: int, + processor: AutoProcessor | AutoTokenizer, + backend: str = "sglang", + fixed_output_len: Optional[int] = None, + random_sample: bool = True, +) -> List[DatasetRow]: + """ + Sample requests from the MMMU dataset using HuggingFace datasets. + + Args: + num_requests: Number of requests to sample. + fixed_output_len: If provided, use this fixed output length for all requests. + random_sample: Whether to randomly sample or take the first N. + + Returns: + List of tuples (prompt, prompt_token_len, output_token_len). + """ + print("Loading MMMU dataset from HuggingFace...") + + try: + print("Attempting to load MMMU Math dataset...") + mmmu_dataset = load_dataset("MMMU/MMMU", "Math", split="test") + print( + f"Successfully loaded MMMU Math dataset from HuggingFace with {len(mmmu_dataset)} examples" + ) + except Exception as e: + print(f"Failed to load MMMU Math dataset: {e}") + raise ValueError(f"Failed to load MMMU dataset: {e}") + + # Sample from the dataset + if len(mmmu_dataset) > num_requests: + if random_sample: + # Random sample + indices = random.sample(range(len(mmmu_dataset)), num_requests) + sample_dataset = mmmu_dataset.select(indices) + else: + # Take first N + sample_dataset = mmmu_dataset.select( + range(min(num_requests, len(mmmu_dataset))) + ) + else: + print(f"Dataset has less than {num_requests} examples, using all examples") + sample_dataset = mmmu_dataset + + print(f"Selected {len(sample_dataset)} examples for benchmarking") + + # Create prompts + filtered_dataset = [] + + for i, example in enumerate(sample_dataset): + try: + # Extract image_1 + image = example.get("image_1") + + if image is not None: + if hasattr(image, "save"): + # Convert RGBA images to RGB before encoding + if image.mode == "RGBA": + image = image.convert("RGB") + + # Encode image to base64 (save as PNG to support palette/alpha modes) + buffered = io.BytesIO() + image.save(buffered, format="PNG") + img_str = pybase64.b64encode(buffered.getvalue()).decode("utf-8") + image_data = f"data:image/png;base64,{img_str}" + else: + continue + + # Extract the question + question = example.get("question") + + # Construct the prompt + text_prompt = f"Question: {question}\n\nAnswer: " + output_len = fixed_output_len if fixed_output_len is not None else 256 + data_row = create_mm_data_row( + text_prompt, [image], [image_data], output_len, processor, backend + ) + filtered_dataset.append(data_row) + + except Exception as e: + print(f"Error processing example {i}: {e}") + + print(f"\nCreated {len(filtered_dataset)} MMMU prompts") + return filtered_dataset + + +def sample_sharegpt_requests( + dataset_path: str, + num_requests: int, + tokenizer: PreTrainedTokenizerBase, + fixed_output_len: Optional[int] = None, + context_len: Optional[int] = None, + prompt_suffix: Optional[str] = "", + apply_chat_template=False, +) -> List[DatasetRow]: + if fixed_output_len is not None and fixed_output_len < 4: + raise ValueError("output_len too small") + + # Download sharegpt if necessary + if not is_file_valid_json(dataset_path) and dataset_path == "": + dataset_path = download_and_cache_file(SHAREGPT_URL) + + # Load the dataset. + with open(dataset_path) as f: + dataset = json.load(f) + + # Filter out the conversations with less than 2 turns. + dataset = [ + data + for data in dataset + if len(data.get("conversations", data.get("conversation", []))) >= 2 + ] + # Only keep the first two turns of each conversation. + dataset = [ + ( + data.get("conversations", data.get("conversation", []))[0]["value"], + data.get("conversations", data.get("conversation", []))[1]["value"], + ) + for data in dataset + ] + + # Shuffle the dataset. + random.shuffle(dataset) + + # Filter out sequences that are too long or too short + filtered_dataset: List[DatasetRow] = [] + for i in range(len(dataset)): + if len(filtered_dataset) == num_requests: + break + + # Tokenize the prompts and completions. + prompt = dataset[i][0] + if prompt_suffix: + prompt = ( + remove_suffix(prompt, ASSISTANT_SUFFIX) + + prompt_suffix + + ASSISTANT_SUFFIX + ) + + if apply_chat_template: + prompt = tokenizer.apply_chat_template( + [{"role": "user", "content": prompt}], + add_generation_prompt=True, + tokenize=False, + return_dict=False, + ) + if tokenizer.bos_token: + prompt = prompt.replace(tokenizer.bos_token, "") + + prompt_token_ids = tokenizer.encode(prompt) + completion = dataset[i][1] + completion_token_ids = tokenizer.encode(completion) + prompt_len = len(prompt_token_ids) + output_len = ( + len(completion_token_ids) if fixed_output_len is None else fixed_output_len + ) + + if prompt_len < 2 or output_len < 2: + # Prune too short sequences. + continue + + if context_len and prompt_len + output_len > context_len: + # Prune too long sequences. + continue + + filtered_dataset.append( + DatasetRow( + prompt=prompt, + prompt_len=prompt_len, + output_len=output_len, + ) + ) + + print(f"#Input tokens: {np.sum([x.prompt_len for x in filtered_dataset])}") + print(f"#Output tokens: {np.sum([x.output_len for x in filtered_dataset])}") + return filtered_dataset + + +def sample_random_requests( + input_len: int, + output_len: int, + num_prompts: int, + range_ratio: float, + tokenizer: PreTrainedTokenizerBase, + dataset_path: str, + random_sample: bool = True, + return_text: bool = True, +) -> List[DatasetRow]: + input_lens = np.random.randint( + max(int(input_len * range_ratio), 1), + input_len + 1, + size=num_prompts, + ) + output_lens = np.random.randint( + int(output_len * range_ratio), + output_len + 1, + size=num_prompts, + ) + + if random_sample: + # Sample token ids from ShareGPT and repeat/truncate them to satisfy the input_lens + + # Download sharegpt if necessary + if not is_file_valid_json(dataset_path): + dataset_path = download_and_cache_file(SHAREGPT_URL) + + # Load the dataset. + with open(dataset_path) as f: + dataset = json.load(f) + # Filter out the conversations with less than 2 turns. + dataset = [ + data + for data in dataset + if len(data.get("conversations", data.get("conversation", []))) >= 2 + ] + # Only keep the first two turns of each conversation. + dataset = [ + ( + data.get("conversations", data.get("conversation", []))[0]["value"], + data.get("conversations", data.get("conversation", []))[1]["value"], + ) + for data in dataset + ] + # Shuffle the dataset. + random.shuffle(dataset) + + # Filter out sequences that are too long or too short + input_requests: List[DatasetRow] = [] + for data in dataset: + i = len(input_requests) + if i == num_prompts: + break + + # Tokenize the prompts and completions. + prompt = data[0] + prompt_token_ids = tokenizer.encode(prompt) + prompt_len = len(prompt_token_ids) + + # Skip empty prompt + if prompt_len == 0: + continue + + if prompt_len > input_lens[i]: + input_ids = prompt_token_ids[: input_lens[i]] + else: + ratio = (input_lens[i] + prompt_len - 1) // prompt_len + input_ids = (prompt_token_ids * ratio)[: input_lens[i]] + input_content = input_ids + if return_text: + input_content = tokenizer.decode(input_content) + input_requests.append( + DatasetRow( + prompt=input_content, + prompt_len=int(input_lens[i]), + output_len=int(output_lens[i]), + ) + ) + else: + # Sample token ids from random integers. This can cause some NaN issues. + offsets = np.random.randint(0, tokenizer.vocab_size, size=num_prompts) + input_requests = [] + for i in range(num_prompts): + input_content = [ + (offsets[i] + i + j) % tokenizer.vocab_size + for j in range(input_lens[i]) + ] + if return_text: + input_content = tokenizer.decode(input_content) + input_requests.append( + DatasetRow( + prompt=input_content, + prompt_len=int(input_lens[i]), + output_len=int(output_lens[i]), + ) + ) + + print(f"#Input tokens: {np.sum(input_lens)}") + print(f"#Output tokens: {np.sum(output_lens)}") + return input_requests + + +def parse_image_resolution(image_resolution: str) -> Tuple[int, int]: + """Parse image resolution into (width, height). + + Supports presets '1080p', '720p', '360p' and custom 'heightxwidth' format + (e.g., '1080x1920' means height=1080, width=1920). + """ + resolution_to_size = { + "4k": (3840, 2160), + "1080p": (1920, 1080), + "720p": (1280, 720), + "360p": (640, 360), + } + if image_resolution in resolution_to_size: + return resolution_to_size[image_resolution] + + res = image_resolution.strip().lower() + if "x" in res: + parts = res.split("x") + if len(parts) == 2 and parts[0].isdigit() and parts[1].isdigit(): + height = int(parts[0]) + width = int(parts[1]) + if height > 0 and width > 0: + return (width, height) + + raise ValueError( + f"Unsupported image resolution: {image_resolution}. " + "Choose from 4k, 1080p, 720p, 360p, or provide custom 'heightxwidth' (e.g., 1080x1920)." + ) + + +def create_mm_data_row( + text_prompt, images: list, images_base64, output_len, processor, backend +): + try: + if type(processor).__name__ == "Phi4MMProcessor": + # <|endoftext10|> is the image token used in the phi-4-multimodal model. + content_items = text_prompt.replace("image 1", "|endoftext10|") + else: + content_items = [ + {"type": "image", "image": {"url": image_base64}} + for image_base64 in images_base64 + ] + content_items.append({"type": "text", "text": text_prompt}) + prompt_str = processor.apply_chat_template( + [{"role": "user", "content": content_items}], + add_generation_prompt=True, + tokenize=False, + ) + except Exception as e: + # Note (Xinyuan): This is a workaround for an issue where some tokenizers do not support content as a list. (e.g. InternVL) + print(f"Error applying chat template: {e}, fallback to tag") + # Some tokenizers do not support list content; fall back to a placeholder in the text + prompt_str = f"{text_prompt}" + + # Calculate total tokens (text + vision) + prompt_len = processor( + text=[prompt_str], + images=images, + padding=False, + return_tensors="pt", + )["input_ids"].numel() + + # Calculate text-only tokens + try: + # Create text-only version of the prompt + text_only_prompt = processor.apply_chat_template( + [{"role": "user", "content": text_prompt}], + add_generation_prompt=True, + tokenize=False, + ) + text_prompt_len = processor( + text=[text_only_prompt], + padding=False, + return_tensors="pt", + )["input_ids"].numel() + except Exception: + # Fallback: just tokenize the text prompt directly + tokenizer_to_use = ( + processor.tokenizer if hasattr(processor, "tokenizer") else processor + ) + text_prompt_len = len(tokenizer_to_use.encode(text_prompt)) + + # Vision tokens = total tokens - text tokens + vision_prompt_len = prompt_len - text_prompt_len + + use_raw_prompt = backend in [ + "sglang", + "sglang-oai", + "sglang-oai-chat", + "vllm", + "vllm-chat", + "lmdeploy", + "lmdeploy-chat", + ] + return DatasetRow( + prompt=text_prompt if use_raw_prompt else prompt_str, + prompt_len=prompt_len, + output_len=output_len, + text_prompt_len=text_prompt_len, + vision_prompt_len=vision_prompt_len, + image_data=images_base64, + ) + + +def sample_image_requests( + num_requests: int, + image_count: int, + input_len: int, + output_len: int, + range_ratio: float, + processor: AutoProcessor, + image_content: str, + image_format: str, + image_resolution: str, + backend: str, +) -> List[DatasetRow]: + """Generate requests with images. + + - Each request includes ``image_count`` images. + - Supported resolutions: 4k (3840x2160), 1080p (1920x1080), 720p (1280x720), 360p (640x360), + or custom 'heightxwidth' (e.g., 1080x1920). + - Text lengths follow the 'random' dataset sampling rule. ``prompt_len`` + only counts text tokens and excludes image data. + """ + + # Parse resolution (supports presets and 'heightxwidth') + width, height = parse_image_resolution(image_resolution) + + # Check for potentially problematic combinations and warn user + if width * height >= 1920 * 1080 and image_count * num_requests >= 100: + warnings.warn( + f"High resolution ({width}x{height}) with {image_count * num_requests} total images " + f"may take a long time. Consider reducing resolution or image count.", + UserWarning, + stacklevel=2, + ) + + # Sample text lengths + input_lens = np.random.randint( + max(int(input_len * range_ratio), 1), input_len + 1, size=num_requests + ) + output_lens = np.random.randint( + int(output_len * range_ratio), output_len + 1, size=num_requests + ) + + def _gen_random_image_data_uri( + width: int = width, height: int = height + ) -> (Image, str, int): + if image_content == "blank": + # Generate blank white image + arr = np.full((height, width, 3), 255, dtype=np.uint8) + else: + # Generate random colored image + arr = (np.random.rand(height, width, 3) * 255).astype(np.uint8) + img = Image.fromarray(arr) + buf = io.BytesIO() + img.save(buf, format=image_format, quality=85) + encoded = pybase64.b64encode(buf.getvalue()).decode("utf-8") + image_data = f"data:image/{image_format};base64,{encoded}" + image_bytes = len(image_data.encode("utf-8")) + return img, image_data, image_bytes + + dataset: List[DatasetRow] = [] + total_image_bytes = 0 + for i in range(num_requests): + # Generate text prompt + text_prompt = gen_mm_prompt( + processor.tokenizer, + processor.image_token_id if hasattr(processor, "image_token_id") else None, + int(input_lens[i]), + ) + + # Generate image list + images, images_base64, images_bytes = zip( + *[_gen_random_image_data_uri() for _ in range(image_count)] + ) + total_image_bytes += sum(list(images_bytes)) + + data_row = create_mm_data_row( + text_prompt, + list(images), + list(images_base64), + int(output_lens[i]), + processor, + backend, + ) + + dataset.append(data_row) + + print(f"#Input tokens: {np.sum([x.prompt_len for x in dataset])}") + print(f"#Output tokens: {np.sum([x.output_len for x in dataset])}") + print( + f"\nCreated {len(dataset)} {image_content} {image_format} images with average {total_image_bytes // num_requests} bytes per request" + ) + return dataset + + +@lru_cache(maxsize=1) +def get_available_tokens(tokenizer): + """Get all available token ids from the tokenizer vocabulary.""" + return list(tokenizer.get_vocab().values()) + + +def gen_prompt(tokenizer, token_num): + """Generate a random prompt of specified token length using tokenizer vocabulary.""" + all_available_tokens = get_available_tokens(tokenizer) + selected_tokens = random.choices(all_available_tokens, k=token_num) + return tokenizer.decode(selected_tokens) + + +def gen_mm_prompt(tokenizer, image_pad_id, token_num): + """Generate a random prompt of specified token length using tokenizer vocabulary.""" + all_available_tokens = list(tokenizer.get_vocab().values()) + if image_pad_id: + all_available_tokens.remove(image_pad_id) + selected_tokens = random.choices(all_available_tokens, k=token_num) + return tokenizer.decode(selected_tokens) + + +def get_gen_prefix_cache_path(args, tokenizer): + """Create cache directory under ~/.cache/sglang/benchmark""" + cache_dir = Path.home() / ".cache" / "sglang" / "benchmark" + + # Create a unique cache filename based on the generation parameters + cache_key = ( + f"gen_shared_prefix_{args.seed}_{args.gsp_num_groups}_{args.gsp_prompts_per_group}_" + f"{args.gsp_system_prompt_len}_{args.gsp_question_len}_{args.gsp_output_len}_" + f"{tokenizer.__class__.__name__}.pkl" + ) + return cache_dir / cache_key + + +def sample_generated_shared_prefix_requests( + num_groups: int, + prompts_per_group: int, + system_prompt_len: int, + question_len: int, + output_len: int, + tokenizer: PreTrainedTokenizerBase, + args: argparse.Namespace, +) -> List[DatasetRow]: + """Generate benchmark requests with shared system prompts using random tokens and caching.""" + cache_path = get_gen_prefix_cache_path(args, tokenizer) + + # Try to load from cache first + if cache_path.exists(): + print(f"\nLoading cached generated input data from {cache_path}") + with open(cache_path, "rb") as f: + return pickle.load(f) + + print("\nGenerating new input data...") + + # Generate system prompts for each group + system_prompts = [] + for _ in range(num_groups): + system_prompt = gen_prompt(tokenizer, system_prompt_len) + system_prompts.append(system_prompt) + + # Generate questions + questions = [] + for _ in range(num_groups * prompts_per_group): + question = gen_prompt(tokenizer, question_len) + questions.append(question) + + # Combine system prompts with questions + input_requests = [] + total_input_tokens = 0 + total_output_tokens = 0 + + for group_idx in tqdm(range(num_groups), desc="Generating system prompt"): + system_prompt = system_prompts[group_idx] + for prompt_idx in tqdm( + range(prompts_per_group), desc="Generating questions", leave=False + ): + question = questions[group_idx * prompts_per_group + prompt_idx] + full_prompt = f"{system_prompt}\n\n{question}" + prompt_len = len(tokenizer.encode(full_prompt)) + + input_requests.append( + DatasetRow( + prompt=full_prompt, + prompt_len=prompt_len, + output_len=output_len, + ) + ) + total_input_tokens += prompt_len + total_output_tokens += output_len + + # Shuffle questions + random.shuffle(input_requests) + + # Print statistics + print(f"\nGenerated shared prefix dataset statistics:") + print(f"Number of groups: {num_groups}") + print(f"Prompts per group: {prompts_per_group}") + print(f"Total prompts: {len(input_requests)}") + print(f"Total input tokens: {total_input_tokens}") + print(f"Total output tokens: {total_output_tokens}") + print( + f"Average system prompt length: {sum(len(tokenizer.encode(sp)) for sp in system_prompts) / len(system_prompts):.1f} tokens" + ) + print( + f"Average question length: {sum(len(tokenizer.encode(q)) for q in questions) / len(questions):.1f} tokens\n" + ) + + # Save to cache + cache_path.parent.mkdir(parents=True, exist_ok=True) + print(f"Caching generated input data to {cache_path}") + with open(cache_path, "wb") as f: + pickle.dump(input_requests, f) + + return input_requests + + +async def get_request( + input_requests: List[DatasetRow], + request_rate: float, + use_trace_timestamps: bool = False, + slowdown_factor: float = 1.0, +) -> AsyncGenerator[DatasetRow, None]: + if use_trace_timestamps: + print( + f"Using trace timestamps for request generation with slowdown factor {slowdown_factor}." + ) + # Sort requests by timestamp for correct replay + input_requests.sort(key=lambda r: r.timestamp) + + start_time = time.perf_counter() + trace_start_time_ms = input_requests[0].timestamp if input_requests else 0 + + for request in input_requests: + trace_time_s = (request.timestamp - trace_start_time_ms) / 1000.0 + target_arrival_time = start_time + (trace_time_s * slowdown_factor) + + sleep_duration = target_arrival_time - time.perf_counter() + if sleep_duration > 0: + await asyncio.sleep(sleep_duration) + + yield request + else: + input_requests_iter = iter(input_requests) + for request in input_requests_iter: + yield request + + if request_rate == float("inf"): + # If the request rate is infinity, then we don't need to wait. + continue + + # Sample the request interval from the exponential distribution. + interval = np.random.exponential(1.0 / request_rate) + # The next request will be sent after the interval. + await asyncio.sleep(interval) + + +def calculate_metrics( + input_requests: List[DatasetRow], + outputs: List[RequestFuncOutput], + dur_s: float, + tokenizer: PreTrainedTokenizerBase, + backend: str, + accept_length: Optional[float] = None, + plot_throughput: bool = False, +) -> Tuple[BenchmarkMetrics, List[int]]: + output_lens: List[int] = [] + retokenized_output_lens: List[int] = [] + total_input = 0 + total_input_text = 0 + total_input_vision = 0 + completed = 0 + itls: List[float] = [] + tpots: List[float] = [] + ttfts: List[float] = [] + e2e_latencies: List[float] = [] + retokenized_itls: List[float] = [] + + use_retokenized_itl = ( + accept_length is not None + and accept_length > 0 + and backend in ("sglang-oai", "sglang-oai-chat") + ) + + for i in range(len(outputs)): + if outputs[i].success: + output_len = outputs[i].output_len + output_lens.append(output_len) + retokenized_output_len = len( + tokenizer.encode(outputs[i].generated_text, add_special_tokens=False) + ) + retokenized_output_lens.append(retokenized_output_len) + total_input += input_requests[i].prompt_len + total_input_text += input_requests[i].text_prompt_len + total_input_vision += input_requests[i].vision_prompt_len + if output_len > 1: + tpots.append((outputs[i].latency - outputs[i].ttft) / (output_len - 1)) + if use_retokenized_itl: + for k, itl in enumerate(outputs[i].itl): + num_tokens = len( + tokenizer.encode( + outputs[i].text_chunks[k], add_special_tokens=False + ) + ) + adjusted_itl = itl / num_tokens + retokenized_itls.extend([adjusted_itl] * num_tokens) + else: + itls += outputs[i].itl + ttfts.append(outputs[i].ttft) + + e2e_latencies.append(outputs[i].latency) + + completed += 1 + else: + output_lens.append(0) + retokenized_output_lens.append(0) + + if completed == 0: + warnings.warn( + "All requests failed. This is likely due to a misconfiguration " + "on the benchmark arguments.", + stacklevel=2, + ) + + max_output_tokens_per_s = 0.0 + max_concurrent_requests = 0 + + successful_outputs = [output for output in outputs if output.success] + if successful_outputs: + min_start_time = min(output.start_time for output in successful_outputs) + max_end_time = max( + output.start_time + output.latency for output in successful_outputs + ) + + duration_seconds = int(np.ceil(max_end_time - min_start_time)) + 1 + tokens_per_second = np.zeros(duration_seconds) + concurrent_requests_per_second = np.zeros(duration_seconds) + + for output in outputs: + if not output.success: + continue + + token_times = [output.start_time + output.ttft] + current_time = token_times[0] + for itl_value in output.itl: + current_time += itl_value + token_times.append(current_time) + + for token_time in token_times: + second_bucket = int(token_time - min_start_time) + if 0 <= second_bucket < duration_seconds: + tokens_per_second[second_bucket] += 1 + + request_start_second = int(output.start_time - min_start_time) + request_end_second = int( + (output.start_time + output.latency) - min_start_time + ) + + for second in range( + request_start_second, min(request_end_second + 1, duration_seconds) + ): + concurrent_requests_per_second[second] += 1 + + if len(tokens_per_second) > 0: + max_output_tokens_per_s = float(np.max(tokens_per_second)) + max_concurrent_requests = int(np.max(concurrent_requests_per_second)) + + if plot_throughput: + if TERM_PLOTLIB_AVAILABLE: + import termplotlib as tpl + + fig = tpl.figure() + fig.plot( + np.arange(len(tokens_per_second)), + tokens_per_second, + title="Output tokens per second", + xlabel="Time (s)", + ) + fig.plot( + np.arange(len(concurrent_requests_per_second)), + concurrent_requests_per_second, + title="Concurrent requests per second", + xlabel="Time (s)", + ) + fig.show() + else: + print("tip: install termplotlib and gnuplot to plot the metrics") + + itls = retokenized_itls if use_retokenized_itl else itls + metrics = BenchmarkMetrics( + completed=completed, + total_input=total_input, + total_input_text=total_input_text, + total_input_vision=total_input_vision, + total_output=sum(output_lens), + total_output_retokenized=sum(retokenized_output_lens), + request_throughput=completed / dur_s, + input_throughput=total_input / dur_s, + output_throughput=sum(output_lens) / dur_s, + output_throughput_retokenized=sum(retokenized_output_lens) / dur_s, + total_throughput=(total_input + sum(output_lens)) / dur_s, + total_throughput_retokenized=(total_input + sum(retokenized_output_lens)) + / dur_s, + mean_ttft_ms=np.mean(ttfts or 0) + * 1000, # ttfts is empty if streaming is not supported by backend + median_ttft_ms=np.median(ttfts or 0) * 1000, + std_ttft_ms=np.std(ttfts or 0) * 1000, + p99_ttft_ms=np.percentile(ttfts or 0, 99) * 1000, + mean_tpot_ms=np.mean(tpots or 0) * 1000, + median_tpot_ms=np.median(tpots or 0) * 1000, + std_tpot_ms=np.std(tpots or 0) * 1000, + p99_tpot_ms=np.percentile(tpots or 0, 99) * 1000, + mean_itl_ms=np.mean(itls or 0) * 1000, + median_itl_ms=np.median(itls or 0) * 1000, + std_itl_ms=np.std(itls or 0) * 1000, + p95_itl_ms=np.percentile(itls or 0, 95) * 1000, + p99_itl_ms=np.percentile(itls or 0, 99) * 1000, + max_itl_ms=np.max(itls or 0) * 1000, + mean_e2e_latency_ms=np.mean(e2e_latencies) * 1000, + median_e2e_latency_ms=np.median(e2e_latencies) * 1000, + std_e2e_latency_ms=np.std(e2e_latencies) * 1000, + p99_e2e_latency_ms=np.percentile(e2e_latencies, 99) * 1000, + concurrency=np.sum(e2e_latencies) / dur_s, + max_output_tokens_per_s=max_output_tokens_per_s, + max_concurrent_requests=max_concurrent_requests, + ) + + return metrics, output_lens + + +async def benchmark( + backend: str, + api_url: str, + base_url: str, + model_id: str, + tokenizer: PreTrainedTokenizerBase, + input_requests: List[DatasetRow], + request_rate: float, + max_concurrency: Optional[int], + disable_tqdm: bool, + lora_names: List[str], + lora_request_distribution: Optional[str], + lora_zipf_alpha: Optional[float], + extra_request_body: Dict[str, Any], + profile: bool, + pd_separated: bool = False, + flush_cache: bool = False, + warmup_requests: int = 1, + use_trace_timestamps: bool = False, + mooncake_slowdown_factor=1.0, + mooncake_num_rounds=1, + profile_prefill_url: Optional[List[str]] = None, + profile_decode_url: Optional[List[str]] = None, +): + if backend in ASYNC_REQUEST_FUNCS: + request_func = ASYNC_REQUEST_FUNCS[backend] + else: + raise ValueError(f"Unknown backend: {backend}") + + # Limit concurrency + # From https://github.com/vllm-project/vllm/pull/9390 + semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else None + + async def limited_request_func(request_func_input, pbar): + if semaphore is None: + return await request_func(request_func_input=request_func_input, pbar=pbar) + async with semaphore: + return await request_func(request_func_input=request_func_input, pbar=pbar) + + # Warmup + print(f"Starting warmup with {warmup_requests} sequences...") + + # Handle the data structure difference for the warmup request + if args.dataset_name == "mooncake": + # For mooncake, input_requests is a list of dicts. + # We need to build a temporary DatasetRow for the warmup phase. + warmup_record = input_requests[0] + + # Build prompt from hash_ids, just like in the async generator + hash_ids = warmup_record.get("hash_ids", []) + prompt_text = "" + for hash_id in hash_ids: + prompt_text += f"{hash_id}" + " ".join(["hi"] * 512) + prompt_text += "Can you tell me a detailed story in 1000 words?" + + output_len = warmup_record.get("output_length", 32) + prompt_len = len(tokenizer.encode(prompt_text)) + + # Create a temporary DatasetRow object for warmup + test_request = DatasetRow( + prompt=prompt_text, + prompt_len=prompt_len, + output_len=output_len, + image_data=None, # Mooncake doesn't have image data + ) + else: + # For all other datasets, input_requests is a list of DatasetRow objects + test_request = input_requests[0] + + if lora_names is not None and len(lora_names) != 0: + lora_name = lora_names[0] + else: + lora_name = None + + # Create the test input once + test_input = RequestFuncInput( + model=model_id, + prompt=test_request.prompt, + api_url=api_url, + prompt_len=test_request.prompt_len, + output_len=min(test_request.output_len, 32), + lora_name=lora_name, + image_data=test_request.image_data, + extra_request_body=extra_request_body, + ) + + # Run warmup requests + warmup_tasks = [] + for _ in range(warmup_requests): + warmup_tasks.append( + asyncio.create_task(request_func(request_func_input=test_input)) + ) + + warmup_outputs = await asyncio.gather(*warmup_tasks) + + # Check if at least one warmup request succeeded + if warmup_requests > 0 and not any(output.success for output in warmup_outputs): + raise ValueError( + "Warmup failed - Please make sure benchmark arguments " + f"are correctly specified. Error: {warmup_outputs[0].error}" + ) + else: + print( + f"Warmup completed with {args.warmup_requests} sequences. Starting main benchmark run..." + ) + + # Flush cache + if ("sglang" in backend and _get_bool_env_var("SGLANG_IS_IN_CI")) or flush_cache: + requests.post(base_url + "/flush_cache", headers=get_auth_headers()) + + time.sleep(1.0) + + # Build profile URLs for PD separated mode (do this once at the beginning) + pd_profile_urls = [] + if profile and pd_separated: + pd_profile_urls = _build_profile_urls(profile_prefill_url, profile_decode_url) + if not pd_profile_urls: + print( + "Warning: PD separated mode requires --profile-prefill-url or --profile-decode-url" + ) + print("Skipping profiler start. Please specify worker URLs for profiling.") + + # Start profiler + if profile: + if pd_separated: + if pd_profile_urls: + await _call_profile_pd(pd_profile_urls, "start") + else: + print("Starting profiler...") + profile_output = await async_request_profile( + api_url=base_url + "/start_profile" + ) + if profile_output.success: + print("Profiler started") + + # Run all requests + benchmark_start_time = time.perf_counter() + tasks: List[asyncio.Task] = [] + pbar_total = len(input_requests) + if ( + backend == "sglang" and args.dataset_name == "mooncake" + ): # Assuming mooncake is mainly for sglang or similar backends + print("Using time-based Mooncake request scheduler, ignoring --request-rate.") + request_generator = get_mooncake_request_over_time( + input_requests, tokenizer, mooncake_slowdown_factor, mooncake_num_rounds + ) + print( + f"Starting Mooncake trace replay. Sessions: {len(input_requests)}, Rounds per session: {mooncake_num_rounds}. Slowdown factor: {mooncake_slowdown_factor}" + ) + pbar_total *= args.mooncake_num_rounds + else: + request_generator = get_request(input_requests, request_rate) + + # Prepare LoRA request distribution parameters + if lora_request_distribution == "distinct": + lora_idx = 0 + elif lora_request_distribution == "skewed": + weights = np.array([lora_zipf_alpha**-i for i in range(len(lora_names))]) + lora_probs = weights / np.sum(weights) + else: + lora_idx = None + lora_probs = None + + pbar = None if disable_tqdm else tqdm(total=pbar_total) + async for request in request_generator: + if lora_names is not None and len(lora_names) != 0: + if lora_request_distribution == "uniform": + lora_name = random.choice(lora_names) + elif lora_request_distribution == "distinct": + lora_name = lora_names[lora_idx] + lora_idx = (lora_idx + 1) % len(lora_names) + else: + assert ( + lora_request_distribution == "skewed" + ), f"Unexpected lora_request_distribution: {lora_request_distribution}. Expected 'skewed'." + + lora_name = np.random.choice(lora_names, p=lora_probs) + else: + lora_name = None + + request_func_input = RequestFuncInput( + model=model_id, + prompt=request.prompt, + api_url=api_url, + prompt_len=request.prompt_len, + output_len=request.output_len, + lora_name=lora_name, + image_data=request.image_data, + extra_request_body=extra_request_body, + timestamp=request.timestamp, + ) + + tasks.append( + asyncio.create_task( + limited_request_func(request_func_input=request_func_input, pbar=pbar) + ) + ) + outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks) + + # Stop profiler + if profile: + if pd_separated: + if pd_profile_urls: + await _call_profile_pd(pd_profile_urls, "stop") + else: + if getattr(args, "profile_num_steps", None) is None: + print("Stopping profiler...") + profile_output = await async_request_profile( + api_url=base_url + "/stop_profile" + ) + if profile_output.success: + print("Profiler stopped") + + if pbar is not None: + pbar.close() + + if "sglang" in backend: + server_info = requests.get( + base_url + "/get_server_info", headers=get_auth_headers() + ) + if server_info.status_code == 200: + server_info_json = server_info.json() + if "decode" in server_info_json: + server_info_json = server_info_json["decode"][0] + if ( + "internal_states" in server_info_json + and server_info_json["internal_states"] + ): + accept_length = server_info_json["internal_states"][0].get( + "avg_spec_accept_length", None + ) + else: + accept_length = None + else: + accept_length = None + else: + accept_length = None + + # Compute metrics and print results + benchmark_duration = time.perf_counter() - benchmark_start_time + metrics, output_lens = calculate_metrics( + input_requests=input_requests, + outputs=outputs, + dur_s=benchmark_duration, + tokenizer=tokenizer, + backend=backend, + accept_length=accept_length, + plot_throughput=args.plot_throughput, + ) + + print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="=")) + print("{:<40} {:<10}".format("Backend:", backend)) + print( + "{:<40} {:<10}".format( + "Traffic request rate:", "trace" if use_trace_timestamps else request_rate + ) + ) + print( + "{:<40} {:<10}".format( + "Max request concurrency:", + max_concurrency if max_concurrency else "not set", + ) + ) + print("{:<40} {:<10}".format("Successful requests:", metrics.completed)) + print("{:<40} {:<10.2f}".format("Benchmark duration (s):", benchmark_duration)) + print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input)) + print("{:<40} {:<10}".format("Total input text tokens:", metrics.total_input_text)) + print( + "{:<40} {:<10}".format("Total input vision tokens:", metrics.total_input_vision) + ) + print("{:<40} {:<10}".format("Total generated tokens:", metrics.total_output)) + print( + "{:<40} {:<10}".format( + "Total generated tokens (retokenized):", metrics.total_output_retokenized + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Request throughput (req/s):", metrics.request_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Input token throughput (tok/s):", metrics.input_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Output token throughput (tok/s):", metrics.output_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Peak output token throughput (tok/s):", metrics.max_output_tokens_per_s + ) + ) + print( + "{:<40} {:<10}".format( + "Peak concurrent requests:", metrics.max_concurrent_requests + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Total token throughput (tok/s):", metrics.total_throughput + ) + ) + print("{:<40} {:<10.2f}".format("Concurrency:", metrics.concurrency)) + if accept_length: + print("{:<40} {:<10.2f}".format("Accept length:", accept_length)) + print("{s:{c}^{n}}".format(s="End-to-End Latency", n=50, c="-")) + print( + "{:<40} {:<10.2f}".format("Mean E2E Latency (ms):", metrics.mean_e2e_latency_ms) + ) + print( + "{:<40} {:<10.2f}".format( + "Median E2E Latency (ms):", metrics.median_e2e_latency_ms + ) + ) + print("{s:{c}^{n}}".format(s="Time to First Token", n=50, c="-")) + print("{:<40} {:<10.2f}".format("Mean TTFT (ms):", metrics.mean_ttft_ms)) + print("{:<40} {:<10.2f}".format("Median TTFT (ms):", metrics.median_ttft_ms)) + print("{:<40} {:<10.2f}".format("P99 TTFT (ms):", metrics.p99_ttft_ms)) + print( + "{s:{c}^{n}}".format(s="Time per Output Token (excl. 1st token)", n=50, c="-") + ) + print("{:<40} {:<10.2f}".format("Mean TPOT (ms):", metrics.mean_tpot_ms)) + print("{:<40} {:<10.2f}".format("Median TPOT (ms):", metrics.median_tpot_ms)) + print("{:<40} {:<10.2f}".format("P99 TPOT (ms):", metrics.p99_tpot_ms)) + print("{s:{c}^{n}}".format(s="Inter-Token Latency", n=50, c="-")) + print("{:<40} {:<10.2f}".format("Mean ITL (ms):", metrics.mean_itl_ms)) + print("{:<40} {:<10.2f}".format("Median ITL (ms):", metrics.median_itl_ms)) + print("{:<40} {:<10.2f}".format("P95 ITL (ms):", metrics.p95_itl_ms)) + print("{:<40} {:<10.2f}".format("P99 ITL (ms):", metrics.p99_itl_ms)) + print("{:<40} {:<10.2f}".format("Max ITL (ms):", metrics.max_itl_ms)) + print("=" * 50) + + resp = requests.get(base_url + "/get_server_info", headers=get_auth_headers()) + server_info = resp.json() if resp.status_code == 200 else None + + if ( + metrics.median_ttft_ms is not None + and metrics.mean_itl_ms is not None + and metrics.output_throughput is not None + ): + result = { + # Arguments + "tag": getattr(args, "tag", None), + "backend": args.backend, + "dataset_name": args.dataset_name, + "request_rate": "trace" if use_trace_timestamps else request_rate, + "max_concurrency": max_concurrency, + "sharegpt_output_len": args.sharegpt_output_len, + "random_input_len": args.random_input_len, + "random_output_len": args.random_output_len, + "random_range_ratio": args.random_range_ratio, + # Information + "server_info": server_info, + # Results + "duration": benchmark_duration, + "completed": metrics.completed, + "total_input_tokens": metrics.total_input, + "total_input_text_tokens": metrics.total_input_text, + "total_input_vision_tokens": metrics.total_input_vision, + "total_output_tokens": metrics.total_output, + "total_output_tokens_retokenized": metrics.total_output_retokenized, + "request_throughput": metrics.request_throughput, + "input_throughput": metrics.input_throughput, + "output_throughput": metrics.output_throughput, + "total_throughput": metrics.total_throughput, + "mean_e2e_latency_ms": metrics.mean_e2e_latency_ms, + "median_e2e_latency_ms": metrics.median_e2e_latency_ms, + "std_e2e_latency_ms": metrics.std_e2e_latency_ms, + "p99_e2e_latency_ms": metrics.p99_e2e_latency_ms, + "mean_ttft_ms": metrics.mean_ttft_ms, + "median_ttft_ms": metrics.median_ttft_ms, + "std_ttft_ms": metrics.std_ttft_ms, + "p99_ttft_ms": metrics.p99_ttft_ms, + "mean_tpot_ms": metrics.mean_tpot_ms, + "median_tpot_ms": metrics.median_tpot_ms, + "std_tpot_ms": metrics.std_tpot_ms, + "p99_tpot_ms": metrics.p99_tpot_ms, + "mean_itl_ms": metrics.mean_itl_ms, + "median_itl_ms": metrics.median_itl_ms, + "std_itl_ms": metrics.std_itl_ms, + "p95_itl_ms": metrics.p95_itl_ms, + "p99_itl_ms": metrics.p99_itl_ms, + "concurrency": metrics.concurrency, + "accept_length": accept_length, + "max_output_tokens_per_s": metrics.max_output_tokens_per_s, + "max_concurrent_requests": metrics.max_concurrent_requests, + } + else: + print(f"Error running benchmark for request rate: {request_rate}") + print("-" * 30) + + # Determine output file name + if args.output_file: + output_file_name = args.output_file + else: + now = datetime.now().strftime("%m%d") + if args.dataset_name == "image": + output_file_name = ( + f"{args.backend}_{now}_{args.num_prompts}_{args.random_input_len}_" + f"{args.random_output_len}_{args.image_count}imgs_" + f"{args.image_resolution}.jsonl" + ) + elif args.dataset_name.startswith("random"): + output_file_name = f"{args.backend}_{now}_{args.num_prompts}_{args.random_input_len}_{args.random_output_len}.jsonl" + else: + output_file_name = ( + f"{args.backend}_{now}_{args.num_prompts}_{args.dataset_name}.jsonl" + ) + + result_details = { + "input_lens": [output.prompt_len for output in outputs], + "output_lens": output_lens, + "ttfts": [output.ttft for output in outputs], + "itls": [output.itl for output in outputs], + "generated_texts": [output.generated_text for output in outputs], + "errors": [output.error for output in outputs], + } + + # Append results to a JSONL file + with open(output_file_name, "a") as file: + if args.output_details: + result_for_dump = result | result_details + else: + result_for_dump = result + file.write(json.dumps(result_for_dump) + "\n") + + return result | result_details + + +def check_chat_template(model_path): + try: + tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + return "chat_template" in tokenizer.init_kwargs + except Exception as e: + print(f"Fail to load tokenizer config with error={e}") + return False + + +def set_global_args(args_: argparse.Namespace): + """Set the global args.""" + global args + args = args_ + + +def run_benchmark(args_: argparse.Namespace): + global args + args = args_ + + # Set default value for max_concurrency if not present + if not hasattr(args, "max_concurrency"): + args.max_concurrency = None + + # Set default value for warmup_requests if not present + if not hasattr(args, "warmup_requests"): + args.warmup_requests = 1 + + if not hasattr(args, "output_details"): + args.output_details = False + + if not hasattr(args, "tokenize_prompt"): + args.tokenize_prompt = False + + if not hasattr(args, "plot_throughput"): + args.plot_throughput = False + + if not hasattr(args, "use_trace_timestamps"): + args.use_trace_timestamps = False + if not hasattr(args, "mooncake_slowdown_factor"): + args.mooncake_slowdown_factor = 1.0 + + if not hasattr(args, "mooncake_slowdown_factor"): + args.mooncake_slowdown_factor = 1.0 + + if not hasattr(args, "mooncake_num_rounds"): + args.mooncake_num_rounds = 1 + + if not hasattr(args, "served_model_name"): + args.served_model_name = None + + print(f"benchmark_args={args}") + + # Set global environments + set_ulimit() + random.seed(args.seed) + np.random.seed(args.seed) + + extra_request_body = {} + if args.extra_request_body: + extra_request_body = json.loads(args.extra_request_body) + + if args.tokenize_prompt: + assert ( + args.backend == "sglang" + ), "`--tokenize-prompt` only compatible with `--backend sglang` currently" + + # Set url + if args.port is None: + args.port = { + "sglang": 30000, + "sglang-native": 30000, + "sglang-oai": 30000, + "lmdeploy": 23333, + "vllm": 8000, + "trt": 8000, + "gserver": 9988, + "truss": 8080, + }.get(args.backend, 30000) + + model_url = ( + f"{args.base_url}/v1/models" + if args.base_url + else f"http://{args.host}:{args.port}/v1/models" + ) + + if args.backend in ["sglang", "sglang-native"]: + api_url = ( + f"{args.base_url}/generate" + if args.base_url + else f"http://{args.host}:{args.port}/generate" + ) + elif args.backend in ["sglang-oai", "vllm", "lmdeploy"]: + api_url = ( + f"{args.base_url}/v1/completions" + if args.base_url + else f"http://{args.host}:{args.port}/v1/completions" + ) + elif args.backend in ["sglang-oai-chat", "vllm-chat", "lmdeploy-chat"]: + api_url = ( + f"{args.base_url}/v1/chat/completions" + if args.base_url + else f"http://{args.host}:{args.port}/v1/chat/completions" + ) + elif args.backend == "trt": + api_url = ( + f"{args.base_url}/v2/models/ensemble/generate_stream" + if args.base_url + else f"http://{args.host}:{args.port}/v2/models/ensemble/generate_stream" + ) + if args.model is None: + print("Please provide a model using `--model` when using `trt` backend.") + sys.exit(1) + elif args.backend == "gserver": + api_url = args.base_url if args.base_url else f"{args.host}:{args.port}" + args.model = args.model or "default" + elif args.backend == "truss": + api_url = ( + f"{args.base_url}/v1/models/model:predict" + if args.base_url + else f"http://{args.host}:{args.port}/v1/models/model:predict" + ) + base_url = ( + f"http://{args.host}:{args.port}" if args.base_url is None else args.base_url + ) + + # Get model name + if args.model is None: + if args.backend == "truss": + print( + "Please provide a model with `--model` when using truss backend. e.g. --model meta-llama/Llama-3.1-8B-Instruct" + ) + sys.exit(1) + try: + response = requests.get(model_url, headers=get_auth_headers()) + model_list = response.json().get("data", []) + args.model = model_list[0]["id"] if model_list else None + except Exception as e: + print(f"Failed to fetch model from {model_url}. Error: {e}") + print( + "Please specify the correct host and port using `--host` and `--port`." + ) + sys.exit(1) + + if args.model is None: + print("No model specified or found. Please provide a model using `--model`.") + sys.exit(1) + + if not check_chat_template(args.model): + print( + "\nWARNING It is recommended to use the `Chat` or `Instruct` model for benchmarking.\n" + "Because when the tokenizer counts the output tokens, if there is gibberish, it might count incorrectly.\n" + ) + + if args.dataset_name in ["image", "mmmu"]: + args.apply_chat_template = True + assert ( + not args.tokenize_prompt + ), "`--tokenize-prompt` not compatible with image dataset" + + if args.lora_request_distribution in ["distinct", "skewed"]: + assert ( + args.lora_name is not None and len(args.lora_name) > 1 + ), "More than 1 LoRA adapter must be specified via --lora-name to use 'distinct' or 'skewed' request distribution." + + assert ( + args.lora_zipf_alpha > 1 + ), f"Got invalid value for --lora-zipf-alpha of {args.lora_zipf_alpha}. It must be greater than 1." + + print(f"{args}\n") + + # Read dataset + backend = args.backend + model_id = args.served_model_name or args.model + tokenizer_id = args.tokenizer if args.tokenizer is not None else args.model + tokenizer = get_tokenizer(tokenizer_id) + input_requests = get_dataset(args, tokenizer, model_id) + + # compatible with SimpleNamespace + if not hasattr(args, "flush_cache"): + args.flush_cache = False + + # Prepare LoRA arguments + lora_request_distribution = ( + args.lora_request_distribution if args.lora_name is not None else None + ) + + lora_zipf_alpha = ( + args.lora_zipf_alpha + if args.lora_name is not None and args.lora_request_distribution == "skewed" + else None + ) + + return asyncio.run( + benchmark( + backend=backend, + api_url=api_url, + base_url=base_url, + model_id=model_id, + tokenizer=tokenizer, + input_requests=input_requests, + request_rate=args.request_rate, + max_concurrency=args.max_concurrency, + disable_tqdm=args.disable_tqdm, + lora_names=args.lora_name, + lora_request_distribution=lora_request_distribution, + lora_zipf_alpha=lora_zipf_alpha, + extra_request_body=extra_request_body, + profile=args.profile, + pd_separated=args.pd_separated, + flush_cache=args.flush_cache, + warmup_requests=args.warmup_requests, + use_trace_timestamps=args.use_trace_timestamps, + mooncake_slowdown_factor=args.mooncake_slowdown_factor, + mooncake_num_rounds=args.mooncake_num_rounds, + profile_prefill_url=getattr(args, "profile_prefill_url", None), + profile_decode_url=getattr(args, "profile_decode_url", None), + ) + ) + + +def set_ulimit(target_soft_limit=65535): + resource_type = resource.RLIMIT_NOFILE + current_soft, current_hard = resource.getrlimit(resource_type) + + if current_soft < target_soft_limit: + try: + resource.setrlimit(resource_type, (target_soft_limit, current_hard)) + except ValueError as e: + print(f"Fail to set RLIMIT_NOFILE: {e}") + + +class LoRAPathAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, []) + for lora_name in values: + getattr(namespace, self.dest).append(lora_name) + + +if __name__ == "__main__": + parser = ArgumentParser(description="Benchmark the online serving throughput.") + parser.add_argument( + "--backend", + type=str, + choices=list(ASYNC_REQUEST_FUNCS.keys()), + default="sglang", + help="Must specify a backend, depending on the LLM Inference Engine.", + ) + parser.add_argument( + "--base-url", + type=str, + default=None, + help="Server or API base url if not using http host and port.", + ) + parser.add_argument( + "--host", type=str, default="0.0.0.0", help="Default host is 0.0.0.0." + ) + parser.add_argument( + "--port", + type=int, + help="If not set, the default port is configured according to its default value for different LLM Inference Engines.", + ) + parser.add_argument( + "--dataset-name", + type=str, + default="sharegpt", + choices=[ + "sharegpt", + "random", + "random-ids", + "generated-shared-prefix", + "mmmu", + "image", + "mooncake", + ], + help="Name of the dataset to benchmark on.", + ) + parser.add_argument( + "--dataset-path", type=str, default="", help="Path to the dataset." + ) + parser.add_argument( + "--model", + type=str, + help="Name or path of the model. If not set, the default model will request /v1/models for conf.", + ) + parser.add_argument( + "--served-model-name", + type=str, + help="The name of the model as served by the serving service. If not set, this defaults to the value of --model.", + ) + parser.add_argument( + "--tokenizer", + type=str, + help="Name or path of the tokenizer. If not set, using the model conf.", + ) + parser.add_argument( + "--num-prompts", + type=int, + default=1000, + help="Number of prompts to process. Default is 1000.", + ) + parser.add_argument( + "--sharegpt-output-len", + type=int, + default=None, + help="Output length for each request. Overrides the output length from the ShareGPT dataset.", + ) + parser.add_argument( + "--sharegpt-context-len", + type=int, + default=None, + help="The context length of the model for the ShareGPT dataset. Requests longer than the context length will be dropped.", + ) + parser.add_argument( + "--random-input-len", + type=int, + default=1024, + help="Number of input tokens per request, used only for random and image dataset.", + ) + parser.add_argument( + "--random-output-len", + default=1024, + type=int, + help="Number of output tokens per request, used only for random and image dataset.", + ) + parser.add_argument( + "--random-range-ratio", + type=float, + default=0.0, + help="Range of sampled ratio of input/output length, " + "used only for random and image dataset.", + ) + # image dataset args + parser.add_argument( + "--image-count", + type=int, + default=1, + help="Number of images per request (only available with the image dataset)", + ) + parser.add_argument( + "--image-resolution", + type=str, + default="1080p", + help=( + "Resolution of images for image dataset. " + "Supports presets 4k/1080p/720p/360p or custom 'heightxwidth' (e.g., 1080x1920)." + ), + ) + parser.add_argument( + "--image-format", + type=str, + default="jpeg", + help=("Format of images for image dataset. " "Supports jpeg and png."), + ) + parser.add_argument( + "--image-content", + type=str, + default="random", + help=("Content for images for image dataset. " "Supports random and blank."), + ) + parser.add_argument( + "--request-rate", + type=float, + default=float("inf"), + help="Number of requests per second. If this is inf, then all the requests are sent at time 0. " + "Otherwise, we use Poisson process to synthesize the request arrival times. Default is inf.", + ) + parser.add_argument( + "--use-trace-timestamps", + action="store_true", + help="Use timestamps from the trace file for request scheduling. Only valid for 'mooncake' dataset.", + ) + parser.add_argument( + "--max-concurrency", + type=int, + default=None, + help="Maximum number of concurrent requests. This can be used " + "to help simulate an environment where a higher level component " + "is enforcing a maximum number of concurrent requests. While the " + "--request-rate argument controls the rate at which requests are " + "initiated, this argument will control how many are actually allowed " + "to execute at a time. This means that when used in combination, the " + "actual request rate may be lower than specified with --request-rate, " + "if the server is not processing requests fast enough to keep up.", + ) + parser.add_argument("--output-file", type=str, help="Output JSONL file name.") + parser.add_argument( + "--output-details", action="store_true", help="Output details of benchmarking." + ) + parser.add_argument( + "--disable-tqdm", + action="store_true", + help="Specify to disable tqdm progress bar.", + ) + parser.add_argument( + "--disable-stream", + action="store_true", + help="Disable streaming mode.", + ) + parser.add_argument( + "--return-logprob", + action="store_true", + help="Return logprob.", + ) + parser.add_argument("--seed", type=int, default=1, help="The random seed.") + parser.add_argument( + "--disable-ignore-eos", + action="store_true", + help="Disable ignoring EOS.", + ) + parser.add_argument( + "--extra-request-body", + metavar='{"key1": "value1", "key2": "value2"}', + type=str, + help="Append given JSON object to the request payload. You can use this to specify" + "additional generate params like sampling params.", + ) + parser.add_argument( + "--apply-chat-template", + action="store_true", + help="Apply chat template", + ) + parser.add_argument( + "--profile", + action="store_true", + help="Use Torch Profiler. The endpoint must be launched with " + "SGLANG_TORCH_PROFILER_DIR to enable profiler.", + ) + parser.add_argument( + "--plot-throughput", + action="store_true", + help="Plot throughput and concurrent requests over time. Requires termplotlib and gnuplot.", + ) + # TODO unify all these + parser.add_argument( + "--profile-activities", + type=str, + nargs="+", + default=["CPU", "GPU"], + choices=["CPU", "GPU", "CUDA_PROFILER"], + ) + parser.add_argument("--profile-num-steps", type=int, default=None) + parser.add_argument("--profile-by-stage", action="store_true", default=False) + parser.add_argument("--profile-stages", nargs="+", default=None) + parser.add_argument( + "--lora-name", + type=str, + nargs="*", + default=None, + action=LoRAPathAction, + help="The names of LoRA adapters. You can provide a list of names in the format {name} {name} {name}...", + ) + parser.add_argument( + "--lora-request-distribution", + type=str, + default="uniform", + choices=[ + "uniform", + "distinct", + "skewed", + ], + help="What distribution to sample the LoRA adapters specified in --lora-name. Borrowed from the Punica paper. " + "'distinct' distribution means selecting a new LoRA adapter for every request. " + "'skewed' distribution follows the Zipf distribution, where the number of requests " + "to model i specified in --lora-name is α times the number of requests for model i+1, " + "where α > 1.", + ) + parser.add_argument( + "--lora-zipf-alpha", + type=float, + default=1.5, + help="The parameter to use for the Zipf distribution when --lora-request-distribution='skewed'.", + ) + parser.add_argument( + "--prompt-suffix", + type=str, + default="", + help="Suffix applied to the end of all user prompts, followed by assistant prompt suffix.", + ) + parser.add_argument( + "--pd-separated", + action="store_true", + help="Benchmark PD disaggregation server", + ) + + # Create a mutually exclusive group for profiling URLs + # In PD separated mode, prefill and decode workers must be profiled separately + profile_url_group = parser.add_mutually_exclusive_group() + profile_url_group.add_argument( + "--profile-prefill-url", + type=str, + nargs="*", + default=None, + help="URL(s) of the prefill worker(s) for profiling in PD separated mode. " + "Can specify multiple URLs: --profile-prefill-url http://localhost:30000 http://localhost:30001. " + "NOTE: Cannot be used together with --profile-decode-url. " + "In PD separated mode, prefill and decode workers must be profiled separately.", + ) + profile_url_group.add_argument( + "--profile-decode-url", + type=str, + nargs="*", + default=None, + help="URL(s) of the decode worker(s) for profiling in PD separated mode. " + "Can specify multiple URLs: --profile-decode-url http://localhost:30010 http://localhost:30011. " + "NOTE: Cannot be used together with --profile-prefill-url. " + "In PD separated mode, prefill and decode workers must be profiled separately.", + ) + parser.add_argument( + "--flush-cache", + action="store_true", + help="Flush the cache before running the benchmark", + ) + parser.add_argument( + "--warmup-requests", + type=int, + default=1, + help="Number of warmup requests to run before the benchmark", + ) + parser.add_argument( + "--tokenize-prompt", + action="store_true", + help="Use integer ids instead of string for inputs. Useful to control prompt lengths accurately", + ) + + group = parser.add_argument_group("generated-shared-prefix dataset arguments") + group.add_argument( + "--gsp-num-groups", + type=int, + default=64, + help="Number of system prompt groups for generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-prompts-per-group", + type=int, + default=16, + help="Number of prompts per system prompt group for generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-system-prompt-len", + type=int, + default=2048, + help="Target length in tokens for system prompts in generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-question-len", + type=int, + default=128, + help="Target length in tokens for questions in generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-output-len", + type=int, + default=256, + help="Target length in tokens for outputs in generated-shared-prefix dataset", + ) + mooncake_group = parser.add_argument_group("mooncake dataset arguments") + mooncake_group.add_argument( + "--mooncake-slowdown-factor", + type=float, + default=1.0, + help="Slowdown factor for replaying the mooncake trace. " + "A value of 2.0 means the replay is twice as slow. " + "NOTE: --request-rate is IGNORED in mooncake mode.", + ) + mooncake_group.add_argument( + "--mooncake-num-rounds", + type=int, + default=1, + help="Number of conversation rounds for each session in the mooncake dataset. " + "A value > 1 will enable true multi-turn session benchmarking.", + ) + mooncake_group.add_argument( + "--mooncake-workload", + type=str, + default="conversation", + choices=[ + "mooncake", + "conversation", + "synthetic", + "toolagent", + ], + help="Underlying workload for the mooncake dataset.", + ) + parser.add_argument( + "--tag", type=str, default=None, help="The tag to be dumped to output." + ) + args = parser.parse_args() + run_benchmark(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/check_env.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/check_env.py new file mode 100644 index 00000000..18fa94af --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/check_env.py @@ -0,0 +1,433 @@ +"""Check environment configurations and dependency versions.""" + +import importlib.metadata +import os +import resource +import subprocess +import sys +from abc import abstractmethod +from collections import OrderedDict, defaultdict + +import torch + +from sglang.srt.utils import is_hip, is_npu + + +def is_cuda_v2(): + return torch.version.cuda is not None + + +# List of packages to check versions +PACKAGE_LIST = [ + "sglang", + "sgl_kernel", + "flashinfer_python", + "flashinfer_cubin", + "flashinfer_jit_cache", + "triton", + "transformers", + "torchao", + "numpy", + "aiohttp", + "fastapi", + "hf_transfer", + "huggingface_hub", + "interegular", + "modelscope", + "orjson", + "outlines", + "packaging", + "psutil", + "pydantic", + "python-multipart", + "pyzmq", + "torchao", + "uvicorn", + "uvloop", + "vllm", + "xgrammar", + "openai", + "tiktoken", + "anthropic", + "litellm", + "decord2", +] + + +class BaseEnv: + """Base class for environment check""" + + def __init__(self): + self.package_list = PACKAGE_LIST + + @abstractmethod + def get_info(self) -> dict: + """ + Get CUDA-related information if available. + """ + raise NotImplementedError + + @abstractmethod + def get_topology(self) -> dict: + raise NotImplementedError + + def get_package_versions(self) -> dict: + """ + Get versions of specified packages. + """ + versions = {} + for package in self.package_list: + package_name = package.split("==")[0].split(">=")[0].split("<=")[0] + try: + version = importlib.metadata.version(package_name) + versions[package_name] = version + except ModuleNotFoundError: + versions[package_name] = "Module Not Found" + return versions + + def get_device_info(self): + """ + Get information about available GPU devices. + """ + devices = defaultdict(list) + capabilities = defaultdict(list) + for k in range(torch.cuda.device_count()): + devices[torch.cuda.get_device_name(k)].append(str(k)) + capability = torch.cuda.get_device_capability(k) + capabilities[f"{capability[0]}.{capability[1]}"].append(str(k)) + + gpu_info = {} + for name, device_ids in devices.items(): + gpu_info[f"GPU {','.join(device_ids)}"] = name + + if len(capabilities) == 1: + # All GPUs have the same compute capability + cap, gpu_ids = list(capabilities.items())[0] + gpu_info[f"GPU {','.join(gpu_ids)} Compute Capability"] = cap + else: + # GPUs have different compute capabilities + for cap, gpu_ids in capabilities.items(): + gpu_info[f"GPU {','.join(gpu_ids)} Compute Capability"] = cap + + return gpu_info + + def get_hypervisor_vendor(self) -> dict: + try: + output = subprocess.check_output(["lscpu"], text=True) + for line in output.split("\n"): + if "Hypervisor vendor:" in line: + return {"Hypervisor vendor:": line.split(":")[1].strip()} + return {} + except: + return {} + + def get_ulimit_soft(self) -> dict: + ulimit_soft, _ = resource.getrlimit(resource.RLIMIT_NOFILE) + return {"ulimit soft": ulimit_soft} + + def check_env(self): + """ + Check and print environment information. + """ + env_info = OrderedDict() + env_info["Python"] = sys.version.replace("\n", "") + env_info.update(self.get_info()) + env_info["PyTorch"] = torch.__version__ + env_info.update(self.get_package_versions()) + env_info.update(self.get_topology()) + env_info.update(self.get_hypervisor_vendor()) + env_info.update(self.get_ulimit_soft()) + + for k, v in env_info.items(): + print(f"{k}: {v}") + + +class GPUEnv(BaseEnv): + """Environment checker for Nvidia GPU""" + + def get_info(self): + cuda_info = {"CUDA available": torch.cuda.is_available()} + + if cuda_info["CUDA available"]: + cuda_info.update(self.get_device_info()) + cuda_info.update(self._get_cuda_version_info()) + + return cuda_info + + def _get_cuda_version_info(self): + """ + Get CUDA version information. + """ + from torch.utils.cpp_extension import CUDA_HOME + + cuda_info = {"CUDA_HOME": CUDA_HOME} + + if CUDA_HOME and os.path.isdir(CUDA_HOME): + cuda_info.update(self._get_nvcc_info()) + cuda_info.update(self._get_cuda_driver_version()) + + return cuda_info + + def _get_nvcc_info(self): + """ + Get NVCC version information. + """ + from torch.utils.cpp_extension import CUDA_HOME + + try: + nvcc = os.path.join(CUDA_HOME, "bin/nvcc") + nvcc_output = ( + subprocess.check_output(f'"{nvcc}" -V', shell=True) + .decode("utf-8") + .strip() + ) + return { + "NVCC": nvcc_output[ + nvcc_output.rfind("Cuda compilation tools") : nvcc_output.rfind( + "Build" + ) + ].strip() + } + except subprocess.SubprocessError: + return {"NVCC": "Not Available"} + + def _get_cuda_driver_version(self): + """ + Get CUDA driver version. + """ + versions = set() + try: + output = subprocess.check_output( + [ + "nvidia-smi", + "--query-gpu=driver_version", + "--format=csv,noheader,nounits", + ] + ) + versions = set(output.decode().strip().split("\n")) + if len(versions) == 1: + return {"CUDA Driver Version": versions.pop()} + else: + return {"CUDA Driver Versions": ", ".join(sorted(versions))} + except subprocess.SubprocessError: + return {"CUDA Driver Version": "Not Available"} + + def get_topology(self): + """ + Get GPU topology information. + """ + try: + result = subprocess.run( + ["nvidia-smi", "topo", "-m"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=True, + ) + return { + "NVIDIA Topology": ( + "\n" + result.stdout if result.returncode == 0 else None + ) + } + except subprocess.SubprocessError: + return {} + + +class HIPEnv(BaseEnv): + """Environment checker for ROCm/HIP""" + + def get_info(self): + cuda_info = {"ROCM available": torch.cuda.is_available()} + + if cuda_info["ROCM available"]: + cuda_info.update(self.get_device_info()) + cuda_info.update(self._get_cuda_version_info()) + + return cuda_info + + def _get_cuda_version_info(self): + from torch.utils.cpp_extension import ROCM_HOME as ROCM_HOME + + cuda_info = {"ROCM_HOME": ROCM_HOME} + + if ROCM_HOME and os.path.isdir(ROCM_HOME): + cuda_info.update(self._get_hipcc_info()) + cuda_info.update(self._get_rocm_driver_version()) + + return cuda_info + + def _get_hipcc_info(self): + from torch.utils.cpp_extension import ROCM_HOME + + try: + hipcc = os.path.join(ROCM_HOME, "bin/hipcc") + hipcc_output = ( + subprocess.check_output(f'"{hipcc}" --version', shell=True) + .decode("utf-8") + .strip() + ) + return { + "HIPCC": hipcc_output[ + hipcc_output.rfind("HIP version") : hipcc_output.rfind("AMD clang") + ].strip() + } + except subprocess.SubprocessError: + return {"HIPCC": "Not Available"} + + def _get_rocm_driver_version(self): + try: + output = subprocess.check_output( + [ + "rocm-smi", + "--showdriverversion", + "--csv", + ] + ) + versions = set(output.decode().strip().split("\n")) + versions.discard("name, value") + ver = versions.pop() + ver = ver.replace('"Driver version", ', "").replace('"', "") + + return {"ROCM Driver Version": ver} + except subprocess.SubprocessError: + return {"ROCM Driver Version": "Not Available"} + + def get_topology(self): + try: + result = subprocess.run( + ["rocm-smi", "--showtopotype"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=True, + ) + return { + "AMD Topology": "\n" + result.stdout if result.returncode == 0 else None + } + except subprocess.SubprocessError: + return {} + + +class NPUEnv(BaseEnv): + """Environment checker for Ascend NPU""" + + EXTRA_PACKAGE_LIST = [ + "torch_npu", + "sgl-kernel-npu", + "deep_ep", + ] + + def __init__(self): + super().__init__() + self.package_list.extend(NPUEnv.EXTRA_PACKAGE_LIST) + + def get_info(self): + cuda_info = {"NPU available": torch.npu.is_available()} + if cuda_info["NPU available"]: + cuda_info.update(self.get_device_info()) + cuda_info.update(self._get_cann_version_info()) + + return cuda_info + + def get_device_info(self): + """ + Get information about available NPUs. + Need to override due to torch_npu interface differences. + """ + devices = defaultdict(list) + for k in range(torch.npu.device_count()): + devices[torch.npu.get_device_name(k)].append(str(k)) + + npu_info = {} + for name, device_ids in devices.items(): + npu_info[f"NPU {','.join(device_ids)}"] = name + + return npu_info + + def _get_cann_version_info(self): + cann_envs = ["ASCEND_TOOLKIT_HOME", "ASCEND_INSTALL_PATH"] + for var in cann_envs: + path = os.environ.get(var) + if path and os.path.exists(path): + CANN_HOME = path + break + else: + default_path = "/usr/local/Ascend/ascend-toolkit/latest" + CANN_HOME = default_path if os.path.exists(default_path) else None + + if CANN_HOME: + npu_info = {"CANN_HOME": CANN_HOME} + npu_info.update(self._get_cann_info(CANN_HOME)) + npu_info.update(self._get_ascend_driver_version()) + return npu_info + else: + return {"CANN_HOME": "Not found"} + + def _get_cann_info(self, CANN_HOME: str): + cann_info = {} + cann_version_file = os.path.join(CANN_HOME, "version.cfg") + if os.path.exists(cann_version_file): + with open(cann_version_file, "r", encoding="utf-8") as f: + f.readline() # discard first line comment in version.cfg + cann_info["CANN"] = f.readline().split("[")[1].split("]")[0] + else: + cann_info["CANN"] = "Not Available" + try: + bisheng = os.path.join(CANN_HOME, "compiler/ccec_compiler/bin/bisheng") + bisheng_output = ( + subprocess.check_output([bisheng, "--version"]).decode("utf-8").strip() + ) + cann_info["BiSheng"] = bisheng_output.split("\n")[0].strip() + except subprocess.SubprocessError: + cann_info["BiSheng"] = "Not Available" + return cann_info + + def _get_ascend_driver_version(self): + try: + output = subprocess.check_output( + [ + "npu-smi", + "info", + "-t", + "board", + "-i", + "0", + ] + ) + for line in output.decode().strip().split("\n"): + if "Software Version" in line: + version = line.split(":")[-1].strip() + break + else: + version = "Not Available" + + return {"Ascend Driver Version": version} + except subprocess.SubprocessError: + return {"Ascend Driver Version": "Not Available"} + + def get_topology(self): + try: + result = subprocess.run( + ["npu-smi", "info", "-t", "topo"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=True, + ) + return { + "Ascend Topology": ( + "\n" + result.stdout if result.returncode == 0 else None + ) + } + except subprocess.SubprocessError: + return {} + + +if __name__ == "__main__": + if is_cuda_v2(): + env = GPUEnv() + elif is_hip(): + env = HIPEnv() + elif is_npu(): + env = NPUEnv() + env.check_env() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/cli/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/cli/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/cli/generate.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/cli/generate.py new file mode 100644 index 00000000..894a1175 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/cli/generate.py @@ -0,0 +1,33 @@ +import argparse + +from sglang.cli.utils import get_is_diffusion_model, get_model_path + + +def generate(args, extra_argv): + # If help is requested, show generate subcommand help without requiring --model-path + if any(h in extra_argv for h in ("-h", "--help")): + from sglang.multimodal_gen.runtime.entrypoints.cli.generate import ( + add_multimodal_gen_generate_args, + ) + + parser = argparse.ArgumentParser(description="SGLang Multimodal Generation") + add_multimodal_gen_generate_args(parser) + parser.parse_args(extra_argv) + return + + model_path = get_model_path(extra_argv) + is_diffusion_model = get_is_diffusion_model(model_path) + if is_diffusion_model: + from sglang.multimodal_gen.runtime.entrypoints.cli.generate import ( + add_multimodal_gen_generate_args, + generate_cmd, + ) + + parser = argparse.ArgumentParser(description="SGLang Multimodal Generation") + add_multimodal_gen_generate_args(parser) + parsed_args = parser.parse_args(extra_argv) + generate_cmd(parsed_args) + else: + raise Exception( + f"Generate subcommand is not yet supported for model: {model_path}" + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/cli/main.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/cli/main.py new file mode 100644 index 00000000..e8d3b755 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/cli/main.py @@ -0,0 +1,26 @@ +import argparse + +from sglang.cli.generate import generate +from sglang.cli.serve import serve + + +def main(): + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="subcommand", required=True) + + serve_parser = subparsers.add_parser( + "serve", + help="Launch the SGLang server.", + add_help=False, # Defer help to the specific parser + ) + serve_parser.set_defaults(func=serve) + + generate_parser = subparsers.add_parser( + "generate", + help="Run inference on a multimodal model.", + add_help=False, # Defer help to the specific parser + ) + generate_parser.set_defaults(func=generate) + + args, extra_argv = parser.parse_known_args() + args.func(args, extra_argv) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/cli/serve.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/cli/serve.py new file mode 100644 index 00000000..855d6335 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/cli/serve.py @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import logging +import os + +from sglang.cli.utils import get_is_diffusion_model, get_model_path +from sglang.srt.utils import kill_process_tree + +logger = logging.getLogger(__name__) + + +def serve(args, extra_argv): + if any(h in extra_argv for h in ("-h", "--help")): + # Since the server type is determined by the model, and we don't have a model path, + # we can't show the exact help. Instead, we show a general help message and then + # the help for both possible server types. + print( + "Usage: sglang serve --model-path [additional-arguments]\n" + ) + print( + "This command can launch either a standard language model server or a diffusion model server." + ) + print("The server type is determined by the model path.\n") + print("For specific arguments, please provide a model_path.") + print("\n--- Help for Standard Language Model Server ---") + from sglang.srt.server_args import prepare_server_args + + try: + prepare_server_args(["--help"]) + except SystemExit: + pass # argparse --help calls sys.exit + + print("\n--- Help for Diffusion Model Server ---") + from sglang.multimodal_gen.runtime.entrypoints.cli.serve import ( + add_multimodal_gen_serve_args, + ) + + parser = argparse.ArgumentParser(description="SGLang Diffusion Model Serving") + add_multimodal_gen_serve_args(parser) + parser.print_help() + return + + model_path = get_model_path(extra_argv) + try: + is_diffusion_model = get_is_diffusion_model(model_path) + if is_diffusion_model: + logger.info("Diffusion model detected") + + if is_diffusion_model: + # Logic for Diffusion Models + from sglang.multimodal_gen.runtime.entrypoints.cli.serve import ( + add_multimodal_gen_serve_args, + execute_serve_cmd, + ) + + parser = argparse.ArgumentParser( + description="SGLang Diffusion Model Serving" + ) + add_multimodal_gen_serve_args(parser) + parsed_args, remaining_argv = parser.parse_known_args(extra_argv) + + execute_serve_cmd(parsed_args, remaining_argv) + else: + # Logic for Standard Language Models + from sglang.launch_server import run_server + from sglang.srt.server_args import prepare_server_args + + # Add a dummy argument for the program name, expected by prepare_server_args + # as it typically processes sys.argv + server_args = prepare_server_args(extra_argv) + + run_server(server_args) + finally: + kill_process_tree(os.getpid(), include_parent=False) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/cli/utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/cli/utils.py new file mode 100644 index 00000000..57068fc4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/cli/utils.py @@ -0,0 +1,152 @@ +import hashlib +import json +import logging +import os +import tempfile +from typing import Optional + +import filelock +from huggingface_hub import hf_hub_download + +logger = logging.getLogger(__name__) + +temp_dir = tempfile.gettempdir() + + +def _get_lock(model_name_or_path: str, cache_dir: Optional[str] = None): + lock_dir = cache_dir or temp_dir + os.makedirs(os.path.dirname(lock_dir), exist_ok=True) + model_name = model_name_or_path.replace("/", "-") + hash_name = hashlib.sha256(model_name.encode()).hexdigest() + lock_file_name = hash_name + model_name + ".lock" + lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name), mode=0o666) + return lock + + +# Copied and adapted from hf_diffusers_utils.py +def _maybe_download_model( + model_name_or_path: str, local_dir: str | None = None, download: bool = True +) -> str: + """ + Resolve a model path. If it's a local directory, return it. + If it's a Hugging Face Hub ID, download only the config file + (`model_index.json` or `config.json`) and return its directory. + + Args: + model_name_or_path: Local path or Hugging Face Hub model ID + local_dir: Local directory to save the downloaded file (if any) + download: Whether to download from Hugging Face Hub when needed + + Returns: + Local directory path that contains the downloaded config file, or the original local directory. + """ + + if os.path.exists(model_name_or_path): + logger.info("Model already exists locally") + return model_name_or_path + + if not download: + return model_name_or_path + + with _get_lock(model_name_or_path): + # Try `model_index.json` first (diffusers models) + try: + logger.info( + "Downloading model_index.json from HF Hub for %s...", + model_name_or_path, + ) + file_path = hf_hub_download( + repo_id=model_name_or_path, + filename="model_index.json", + local_dir=local_dir, + ) + logger.info("Downloaded to %s", file_path) + return os.path.dirname(file_path) + except Exception as e_index: + logger.debug("model_index.json not found or failed: %s", e_index) + + # Fallback to `config.json` + try: + logger.info( + "Downloading config.json from HF Hub for %s...", model_name_or_path + ) + file_path = hf_hub_download( + repo_id=model_name_or_path, + filename="config.json", + local_dir=local_dir, + ) + logger.info("Downloaded to %s", file_path) + return os.path.dirname(file_path) + except Exception as e_config: + raise ValueError( + ( + "Could not find model locally at %s and failed to download " + "model_index.json/config.json from HF Hub: %s" + ) + % (model_name_or_path, e_config) + ) from e_config + + +# Copied and adapted from hf_diffusers_utils.py +def is_diffusers_model_path(model_path: str) -> True: + """ + Verify if the model directory contains a valid diffusers configuration. + + Args: + model_path: Path to the model directory + + Returns: + The loaded model configuration as a dictionary if the model is a diffusers model + None if the model is not a diffusers model + """ + + # Prefer model_index.json which indicates a diffusers pipeline + config_path = os.path.join(model_path, "model_index.json") + if not os.path.exists(config_path): + return False + + # Load the config + with open(config_path) as f: + config = json.load(f) + + # Verify diffusers version exists + if "_diffusers_version" not in config: + return False + return True + + +def get_is_diffusion_model(model_path: str): + model_path = _maybe_download_model(model_path) + is_diffusion_model = is_diffusers_model_path(model_path) + if is_diffusion_model: + logger.info("Diffusion model detected") + return is_diffusion_model + + +def get_model_path(extra_argv): + # Find the model_path argument + model_path = None + for i, arg in enumerate(extra_argv): + if arg == "--model-path": + if i + 1 < len(extra_argv): + model_path = extra_argv[i + 1] + break + elif arg.startswith("--model-path="): + model_path = arg.split("=", 1)[1] + break + + if model_path is None: + # Fallback for --help or other cases where model-path is not provided + if any(h in extra_argv for h in ["-h", "--help"]): + raise Exception( + "Usage: sglang serve --model-path [additional-arguments]\n\n" + "This command can launch either a standard language model server or a diffusion model server.\n" + "The server type is determined by the model path.\n" + "For specific arguments, please provide a model_path." + ) + else: + raise Exception( + "Error: --model-path is required. " + "Please provide the path to the model." + ) + return model_path diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/compile_deep_gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/compile_deep_gemm.py new file mode 100644 index 00000000..2386a46c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/compile_deep_gemm.py @@ -0,0 +1,191 @@ +""" +Compile DeepGEMM Kernels for a model with specify server arguments + +This script launches a server for capturing DeepGEMM calls and then compiles the kernels. +It accepts server arguments (the same as launch_server.py). + +Usage: +python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code + +""" + +import argparse +import dataclasses +import multiprocessing +import os +import time + +import requests + +from sglang.srt.disaggregation.utils import FAKE_BOOTSTRAP_HOST +from sglang.srt.entrypoints.http_server import launch_server +from sglang.srt.entrypoints.warmup import warmup +from sglang.srt.environ import envs +from sglang.srt.managers.io_struct import GenerateReqInput +from sglang.srt.managers.tokenizer_manager import TokenizerManager +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import kill_process_tree + +multiprocessing.set_start_method("spawn", force=True) + +# Reduce warning +envs.SGLANG_IN_DEEPGEMM_PRECOMPILE_STAGE.set(True) +# Force enable deep gemm +envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(True) +# Force enable mha chunked kv for DeepSeek V3 to avoid missing kv_b_proj DeepGEMM case +os.environ["SGL_CHUNKED_PREFIX_CACHE_THRESHOLD"] = "0" + + +@dataclasses.dataclass +class CompileArgs: + timeout: int = 3600 + + @staticmethod + def add_cli_args(parser: argparse.ArgumentParser): + parser.add_argument("--timeout", type=int, default=CompileArgs.timeout) + + @classmethod + def from_cli_args(cls, args: argparse.Namespace): + # use the default value's type to cast the args into correct types. + attrs = [(attr.name, type(attr.default)) for attr in dataclasses.fields(cls)] + return cls( + **{attr: attr_type(getattr(args, attr)) for attr, attr_type in attrs} + ) + + +@warmup("compile-deep-gemm") +async def warm_up_compile( + disaggregation_mode: str, tokenizer_manager: TokenizerManager +): + print("\nGenerate warm up request for compiling DeepGEMM...\n") + generate_req_input = GenerateReqInput( + input_ids=[0, 1, 2, 3], + sampling_params={ + "temperature": 0.0, + "max_new_tokens": 8, + "ignore_eos": True, + }, + ) + if disaggregation_mode != "null": + generate_req_input.bootstrap_room = 0 + generate_req_input.bootstrap_host = FAKE_BOOTSTRAP_HOST + + await tokenizer_manager.generate_request(generate_req_input, None).__anext__() + + +def launch_server_internal(server_args): + try: + launch_server(server_args) + except Exception as e: + raise e + finally: + kill_process_tree(os.getpid(), include_parent=False) + + +def launch_server_process_and_send_one_request( + server_args: ServerArgs, compile_args: CompileArgs +): + proc = multiprocessing.Process(target=launch_server_internal, args=(server_args,)) + proc.start() + base_url = f"http://{server_args.host}:{server_args.port}" + timeout = compile_args.timeout + + start_time = time.perf_counter() + while time.perf_counter() - start_time < timeout: + try: + headers = { + "Content-Type": "application/json; charset=utf-8", + } + if server_args.node_rank == 0: + response = requests.get(f"{base_url}/v1/models", headers=headers) + else: + # This http api is created by launch_dummy_health_check_server for none-rank0 node. + response = requests.get(f"{base_url}/health", headers=headers) + if response.status_code == 200: + # Rank-0 node send a request to sync with other node and then return. + if server_args.node_rank == 0: + payload = { + "input_ids": [0, 1, 2, 3], + "sampling_params": { + "max_new_tokens": 8, + "temperature": 0, + }, + } + # In PD mode, include fake bootstrap fields so workers don't assert + if server_args.disaggregation_mode != "null": + payload["bootstrap_host"] = FAKE_BOOTSTRAP_HOST + payload["bootstrap_room"] = 0 + + response = requests.post( + f"{base_url}/generate", + json=payload, + timeout=600, + ) + if response.status_code != 200: + error = response.json() + raise RuntimeError(f"Sync request failed: {error}") + # Other nodes should wait for the exit signal from Rank-0 node. + else: + start_time_waiting = time.perf_counter() + while proc.is_alive(): + if time.perf_counter() - start_time_waiting < timeout: + time.sleep(10) + else: + raise TimeoutError("Waiting for main node timeout!") + return proc + except requests.RequestException: + pass + time.sleep(10) + raise TimeoutError( + "DeepGEMM Kernels compilation timeout." + "\n\nFeel free and please restart the command." + ) + + +def refine_server_args(server_args: ServerArgs, compile_args: CompileArgs): + # Disable cuda graph and torch compile to save time + server_args.disable_cuda_graph = True + server_args.enable_torch_compile = False + print(f"Disable CUDA Graph and Torch Compile to save time...") + + # Set watchdog timeout to compile_args.timeout because compilation will take a long time + server_args.watchdog_timeout = compile_args.timeout + server_args.warmups = "compile-deep-gemm" + + +def run_compile(server_args: ServerArgs, compile_args: CompileArgs): + print( + "Begin DeepGEMM Kernels compilation...\n" + "It may take a long time and timeout maybe raised " + "while the compilation is still in progress.\n" + "Just feel free to restart the command " + "until the compilation is fully finished.\n" + ) + + proc = launch_server_process_and_send_one_request(server_args, compile_args) + + print("\nDeepGEMM Kernels compilation finished successfully.") + + # Sleep for safety + time.sleep(10) + if proc.is_alive(): + # This is the rank0 node. + kill_process_tree(proc.pid) + else: + try: + kill_process_tree(proc.pid) + except Exception: + pass + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + CompileArgs.add_cli_args(parser) + args = parser.parse_args() + server_args = ServerArgs.from_cli_args(args) + compile_args = CompileArgs.from_cli_args(args) + + refine_server_args(server_args, compile_args) + + run_compile(server_args, compile_args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/eval/llama3_eval.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/eval/llama3_eval.py new file mode 100644 index 00000000..253cdf27 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/eval/llama3_eval.py @@ -0,0 +1,315 @@ +# Adapt from https://github.com/fw-ai/llm_eval_meta + +import argparse +import asyncio +import os +import pickle +import re +import shutil +from collections import defaultdict +from dataclasses import dataclass + +import httpx +import numpy as np +import openai +from datasets import load_dataset +from openai import AsyncOpenAI +from tqdm import tqdm + +# Mapping providers to their clients and models +provider_to_models = { + "b10": { + "8b": "meta-llama/Llama-3.1-8B-Instruct", + "70b": "meta-llama/Llama-3.1-70B-Instruct", + "405b": "meta-llama/Llama-3.1-405B-Instruct", + }, + "oai": { + "8b": "meta-llama/Llama-3.1-8B-Instruct", + "70b": "meta-llama/Llama-3.1-70B-Instruct", + "405b": "meta-llama/Llama-3.1-405B-Instruct", + }, + "sgl": { + "8b": "meta-llama/Llama-3.1-8B-Instruct", + "70b": "meta-llama/Llama-3.1-70B-Instruct", + "405b": "meta-llama/Llama-3.1-405B-Instruct", + }, +} + + +async def fetch_responses( + client, prompt, semaphore, index, provider, model_size, output_dir, max_tokens +): + output_file = os.path.join(output_dir, f"response_{index}.pkl") + if os.path.exists(output_file): + print(f"File {output_file} already exists, skipping.") + return + + async with semaphore: + response = await client.completions.create( + model=provider_to_models[provider][model_size], + prompt=prompt, + temperature=0.0, + max_tokens=max_tokens, + ) + if isinstance(response, openai.BadRequestError): + with open(output_file, "wb") as f: + pickle.dump("bad_response", f) + assert isinstance(response, openai.types.completion.Completion) + # Save response to a file + with open(output_file, "wb") as f: + pickle.dump(response, f) + + +TASK_TO_MAX_TOKENS = { + "evals__mmlu__details": 1, + "evals__mmlu__0_shot__cot__details": 1024, + # Official meta uses 1024, but a small % (.05) of questions are answered correctly after relaxing + "evals__mmlu_pro__details": 2048, + "evals__gsm8k__details": 1024, +} + +TASK_TO_EVAL_SET = { + "mmlu": "evals__mmlu__details", + "mmlu_cot": "evals__mmlu__0_shot__cot__details", + "mmlu_pro": "evals__mmlu_pro__details", + "gsm8k": "evals__gsm8k__details", +} + + +class CustomAsyncHTTPXClient(httpx.AsyncClient): + async def send(self, request: httpx.Request, *args, **kwargs) -> httpx.Response: + request.url = httpx.URL( + f"https://model-{os.getenv('MODEL_ID')}.api.baseten.co/development/predict" + ) + return await super().send(request, *args, **kwargs) + + +def get_client(provider): + if provider not in "b10": + if os.getenv("OPENAI_API_KEY") == None: + os.environ["OPENAI_API_KEY"] = "EMPTY" + return { + "oai": AsyncOpenAI(base_url="http://127.0.0.1:8000/v1/"), + "b10": AsyncOpenAI( + api_key=f"Api-Key {os.getenv('OPENAI_API_KEY')}", + base_url=f"https://model-{os.getenv('MODEL_ID')}.api.baseten.co/development/predict", + http_client=CustomAsyncHTTPXClient(), + ), + "sgl": AsyncOpenAI(base_url="http://127.0.0.1:30000/v1/"), + }[provider] + + +# Define the benchmark function +async def benchmark(args): + ds = load_dataset( + "meta-llama/Llama-3.1-405B-Instruct-evals", + f"Llama-3.1-405B-Instruct-{TASK_TO_EVAL_SET[args.task]}", + ) + semaphore = asyncio.Semaphore(args.concurrency) # Limit to 16 concurrent tasks + + if args.num_examples is None: + args.num_examples = len(ds["latest"]["input_final_prompts"]) + prompts = ds["latest"]["input_final_prompts"][: args.num_examples] + + # Create the output directory if it does not exist + os.makedirs(args.output_dir, exist_ok=True) + + tasks = [] + # Create the tasks with tqdm progress bar + max_tokens = TASK_TO_MAX_TOKENS[TASK_TO_EVAL_SET[args.task]] + client = get_client(args.provider) + for idx, prompt in enumerate(tqdm(prompts, desc="Creating tasks")): + tasks.append( + asyncio.create_task( + fetch_responses( + client, + f"<|begin_of_text|>{prompt[0]}", + semaphore, + idx, + args.provider, + args.model_size, + args.output_dir, + max_tokens=max_tokens, + ) + ) + ) + + # Run the tasks with tqdm progress bar + for future in tqdm( + asyncio.as_completed(tasks), total=len(tasks), desc="Processing tasks" + ): + await future + + +def get_mmlu_answer(response): + if response is not None: + return response.choices[0].text.lstrip().rstrip().upper().replace(".", "") + return None + + +def get_mmlu_cot_answer(response): + pattern = r"The best answer is (.+)\.?" + match = re.search(pattern, response.choices[0].text) + if match: + return match.group(1).replace(".", "").replace("*", "") + + pattern = r"the best answer is (.+)\.?" + match = re.search(pattern, response.choices[0].text) + if match: + return match.group(1).replace(".", "") + + pattern = r"The correct answer is (.+)\.?" + match = re.search(pattern, response.choices[0].text) + if match: + return match.group(1).replace(".", "") + + pattern = r"the correct answer is (.+)\.?" + match = re.search(pattern, response.choices[0].text) + if match: + return match.group(1).replace(".", "") + + +def get_answer_gsm8k(response): + pattern = r"The final answer is (.+)\.?" + match = re.search(pattern, response.choices[0].text) + if match: + s = match.group(1) + for ok_symbol in ["%", "$"]: + s = s.replace(ok_symbol, "") + return s + + +TASK_TO_ANSWER_EXTRACTOR = { + "evals__mmlu__details": get_mmlu_answer, + "evals__mmlu__0_shot__cot__details": get_mmlu_cot_answer, + "evals__gsm8k__details": get_answer_gsm8k, + "evals__mmlu_pro__details": get_mmlu_cot_answer, +} + + +def get_dataset_from_task(task, response_path, model_size): + ds_405b = load_dataset( + f"meta-llama/Llama-3.1-405B-Instruct-evals", + f"Llama-3.1-405B-Instruct-{task}", + ) + ds_405b_hash_order = [x[0] for x in ds_405b["latest"]["input_final_prompts_hash"]] + + if "70b" in model_size or "8b" in model_size: + if "70" in model_size: + ref_model_ds = load_dataset( + f"meta-llama/Llama-3.1-70B-Instruct-evals", + f"Llama-3.1-70B-Instruct-{task}", + ) + else: + ref_model_ds = load_dataset( + f"meta-llama/Llama-3.1-8B-Instruct-evals", + f"Llama-3.1-8B-Instruct-{task}", + ) + + hash_to_row = {} + for row in ref_model_ds["latest"]: + hash_to_row[row["input_final_prompts_hash"][0]] = row + reordered_rows = [] + for prompt_hash in ds_405b_hash_order: + reordered_rows.append(hash_to_row[prompt_hash]) + ref_model_ds["latest"] = reordered_rows + return ref_model_ds + + return ds_405b + + +def analyze(task, response_path, model_size): + ds = get_dataset_from_task(task, response_path, model_size) + + responses = [] + total = len(ds["latest"]) + + for i in range(0, total): + response = pickle.load( + open(os.path.join(response_path, f"response_{i}.pkl"), "rb") + ) + responses.append(response) + + @dataclass + class Stats: + correct: int = 0 + total: int = 0 + meta_correct: int = 0 + + average: float = None + + subtask_name_to_stats = defaultdict(lambda: Stats()) + + for response, ds_row in zip(responses, ds["latest"]): + model_answer = TASK_TO_ANSWER_EXTRACTOR[task](response) + + subtask = ds_row["subtask_name"] + + is_eval_correct = model_answer in ds_row["input_correct_responses"] + if is_eval_correct: + subtask_name_to_stats[subtask].correct += 1 + + if ds_row["is_correct"]: + subtask_name_to_stats[subtask].meta_correct += 1 + + subtask_name_to_stats[subtask].total += 1 + + micro_stats = Stats() + for subtask, stats in subtask_name_to_stats.items(): + stats.average = stats.correct / stats.total + stats.meta_average = stats.meta_correct / stats.total + + micro_stats.correct += stats.correct + micro_stats.total += stats.total + micro_stats.meta_correct += stats.meta_correct + + micro_stats.average = micro_stats.correct / micro_stats.total + micro_stats.meta_average = micro_stats.meta_correct / micro_stats.total + + print("Macro average", np.mean([x.average for x in subtask_name_to_stats.values()])) + print( + "Meta Macro average", + np.mean([x.meta_average for x in subtask_name_to_stats.values()]), + ) + print("Micro average", micro_stats.average) + print("Meta Micro average", micro_stats.meta_average) + + +# Entry point for the script +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Script to run model with specified parameters." + ) + parser.add_argument( + "--model-size", + type=str, + default="8b", + help="Size of the model (e.g., 8b or 70b)", + ) + parser.add_argument( + "--provider", + type=str, + default="sgl", + help="Provider name (e.g., sgl, oai, b10)", + ) + parser.add_argument( + "--task", + type=str, + required=True, + help="Task (e.g., mmlu, mmlu_cot, mmlu_pro, gsm8k)", + ) + parser.add_argument( + "--num-examples", type=int, default=None, help="Number of examples to process" + ) + parser.add_argument("--concurrency", type=int, default=16) + parser.add_argument( + "--output-dir", + type=str, + default="tmp-output-dir", + help="Directory to save responses", + ) + + args = parser.parse_args() + asyncio.run(benchmark(args)) + analyze(TASK_TO_EVAL_SET[args.task], args.output_dir, args.model_size) + shutil.rmtree("tmp-output-dir", ignore_errors=True) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/eval/loogle_eval.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/eval/loogle_eval.py new file mode 100644 index 00000000..895362cd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/eval/loogle_eval.py @@ -0,0 +1,164 @@ +import argparse +import asyncio +import os +import pickle +from pathlib import Path +from typing import List + +import openai +import torch +from bert_score import BERTScorer +from datasets import load_dataset +from tqdm import tqdm + + +def get_client(api_url: str) -> openai.AsyncOpenAI: + if os.getenv("OPENAI_API_KEY") is None: + os.environ["OPENAI_API_KEY"] = "EMPTY" + return openai.AsyncOpenAI(base_url=api_url) + + +def get_dataset(): + return load_dataset("bigai-nlco/LooGLE", "longdep_qa", split="test") + + +async def fetch_response( + client: openai.AsyncOpenAI, + context: str, + question: str, + semaphore: asyncio.Semaphore, + index: int, + model: str, + output_dir: Path, +): + output_file = output_dir / f"response_{index}.pkl" + if output_file.exists(): + return + + prompt = ( + "Please answer the question based on the long texts below.\n" + f"{context}\n" + f"Question: {question}\n" + "Answer:" + ) + messages = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": prompt}, + ] + + async with semaphore: + try: + response = await client.chat.completions.create( + model=model, + messages=messages, + temperature=0.0, + max_tokens=512, + ) + except openai.BadRequestError as e: + with open(output_file, "wb") as f: + pickle.dump({"error": str(e)}, f) + return + + with open(output_file, "wb") as f: + pickle.dump(response, f) + + +async def benchmark(args): + dataset = get_dataset() + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + client = get_client(args.api_url) + semaphore = asyncio.Semaphore(args.max_concurrency) + + tasks: List[asyncio.Task] = [] + for idx, ex in enumerate(dataset): + if idx >= args.num_prompts: + break + tasks.append( + asyncio.create_task( + fetch_response( + client, + ex["context"], + ex["question"], + semaphore, + idx, + args.model, + output_dir, + ) + ) + ) + + for _ in tqdm( + asyncio.as_completed(tasks), total=len(tasks), desc="Running benchmark" + ): + await _ + + +def analyse(args): + dataset = get_dataset() + output_dir = Path(args.output_dir) + + device = "cuda" if torch.cuda.is_available() else "cpu" + scorer = BERTScorer(lang="en", device=device) + + hyps: List[str] = [] + refs: List[str] = [] + for idx, ex in enumerate(tqdm(dataset, desc="Loading responses")): + if idx >= args.num_prompts: + break + pkl_file = output_dir / f"response_{idx}.pkl" + if not pkl_file.exists(): + raise FileNotFoundError(pkl_file) + + response = pickle.load(open(pkl_file, "rb")) + if isinstance(response, dict) and "error" in response: + continue + + hyps.append(response.choices[0].message.content.strip()) + refs.append(ex["answer"]) + + if not hyps: + print("No valid responses to score!") + return + + batch_size = 64 + all_f1: List[float] = [] + for i in tqdm(range(0, len(hyps), batch_size), desc="Scoring batches"): + h_batch = hyps[i : i + batch_size] + r_batch = refs[i : i + batch_size] + _, _, f1_scores = scorer.score(h_batch, r_batch, verbose=False) + all_f1.extend([float(x) for x in f1_scores]) + + avg = sum(all_f1) / len(all_f1) + print(f"Average BERTScore (F1): {avg:.2%}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run benchmark and evaluation in one go." + ) + parser.add_argument( + "--api-url", + default="http://127.0.0.1:30000/v1", + help="OpenAI‑compatible API base URL", + ) + parser.add_argument( + "--model", + default="meta-llama/Llama-4-Maverick-17B-128E-Instruct", + help="Model name or ID, only used for model name", + ) + parser.add_argument( + "--max-concurrency", type=int, default=144, help="Maximum concurrent requests" + ) + parser.add_argument( + "--output-dir", default="tmp-output-dir", help="Directory for cached responses" + ) + parser.add_argument( + "--num-prompts", type=int, default=10000, help="Number of prompts to run" + ) + args = parser.parse_args() + + asyncio.run(benchmark(args)) + + analyse(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/global_config.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/global_config.py new file mode 100644 index 00000000..fcd65b5e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/global_config.py @@ -0,0 +1,29 @@ +"""Global configurations""" + +# FIXME: deprecate this file and move all usage to sglang.srt.environ or sglang.__init__.py + + +class GlobalConfig: + """ + Store some global constants. + """ + + def __init__(self): + # Verbosity level + # 0: do not output anything + # 2: output final text after every run + self.verbosity = 0 + + # Default backend of the language + self.default_backend = None + + # Output tokenization configs + self.skip_special_tokens_in_output = True + self.spaces_between_special_tokens_in_out = True + + # Language frontend interpreter optimization configs + self.enable_precache_with_tracing = True + self.enable_parallel_encoding = True + + +global_config = GlobalConfig() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/.clang-format b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/.clang-format new file mode 100644 index 00000000..75fe1387 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/.clang-format @@ -0,0 +1,19 @@ +BasedOnStyle: Google +IndentWidth: 2 +ColumnLimit: 120 +AllowShortFunctionsOnASingleLine: Empty +DerivePointerAlignment: false +PointerAlignment: Left +NamespaceIndentation: None +SortIncludes: true +AllowShortLoopsOnASingleLine: false +BinPackParameters: false # Prevents packing parameters in declarations +BinPackArguments: false # Prevents packing arguments in function calls +AlignAfterOpenBracket: AlwaysBreak # Forces a break after the opening parenthesis +AlignOperands: Align # Aligns arguments vertically +PenaltyBreakBeforeFirstCallParameter: 1 # Encourages breaking before the first argument +PenaltyReturnTypeOnItsOwnLine: 100 # Keeps return type with function name + +IncludeCategories: + - Regex: '^$' + Priority: 0 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/csrc/cuda_wait_value.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/csrc/cuda_wait_value.cuh new file mode 100644 index 00000000..5874ce1e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/csrc/cuda_wait_value.cuh @@ -0,0 +1,38 @@ +#include +#include + +#include + +#include + +namespace { + +__global__ void wait_flag_kernel(const int32_t* flag, int32_t target) { + const volatile int32_t* vflag = (volatile const int32_t*)flag; + + while (*vflag != target) { +#if __CUDA_ARCH__ >= 700 + __nanosleep(100); +#else + // Note: This falls back to an inefficient busy-wait on pre-Volta architectures. +#endif + } +} + +auto stream_wait_value(const tvm::ffi::TensorView flag, std::int32_t value) -> void { + using namespace host; + + auto length = SymbolicSize{"length"}; + TensorMatcher({length}).with_dtype().with_device().verify(flag); + RuntimeCheck(length.unwrap() >= 1, "wait_flag expects a non-empty tensor."); + + auto* ptr = static_cast(flag.data_ptr()); + const auto stream = LaunchKernel::resolve_device(flag.device()); + + constexpr int blocks = 1; + constexpr int threads = 1; + wait_flag_kernel<<>>(ptr, value); + RuntimeDeviceCheck(cudaGetLastError()); +} + +} // namespace diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/csrc/hicache.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/csrc/hicache.cuh new file mode 100644 index 00000000..e52ecbd3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/csrc/hicache.cuh @@ -0,0 +1,264 @@ +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace { + +struct HicacheKernelParams { + void* __restrict__ k_cache_dst; + void* __restrict__ v_cache_dst; + const void* __restrict__ indices_dst; + void* __restrict__ k_cache_src; + void* __restrict__ v_cache_src; + const void* __restrict__ indices_src; + std::size_t length; + std::size_t kv_cache_src_stride; + std::size_t kv_cache_dst_stride; + std::size_t num_layers = 0; // only used in all_layer transfer +}; + +template < + std::integral T, + std::size_t kElementSize, + std::size_t kUnroll, + std::size_t kBlockQuota, + std::size_t kNumThreads, + std::size_t kMaxOccupancy> +__global__ __launch_bounds__(kNumThreads, kMaxOccupancy) void hicache_transfer_per_layer( + const __grid_constant__ HicacheKernelParams params) { + // each warp acts as a worker + using namespace device; + static_assert(kNumThreads % kWarpThreads == 0); + static_assert(kWarpThreads % kUnroll == 0); + + constexpr auto kWarpThreads = device::kWarpThreads / kUnroll; + constexpr auto kWarpsPerBlock = kNumThreads / kWarpThreads; + constexpr auto kWorkers = kWarpsPerBlock * kBlockQuota; + + const auto& [ + k_cache_dst, v_cache_dst, indices_dst, // dst + k_cache_src, v_cache_src, indices_src, // src + length, kv_cache_src_stride, kv_cache_dst_stride, _ // metadata + ] = params; + const auto warp_id = blockIdx.x * kWarpsPerBlock + threadIdx.x / kWarpThreads; + + // force to transfer 128 bytes per iteration + // since the PCIe transaction size is 128 bytes aligned + constexpr auto kGranularity = 128 / kWarpThreads; + + for (auto i = warp_id; i < length; i += kWorkers) { + const auto pos_src = static_cast(indices_src)[i]; + const auto pos_dst = static_cast(indices_dst)[i]; + const auto src_k = pointer::offset(k_cache_src, pos_src * kv_cache_src_stride); + const auto dst_k = pointer::offset(k_cache_dst, pos_dst * kv_cache_dst_stride); + const auto src_v = pointer::offset(v_cache_src, pos_src * kv_cache_src_stride); + const auto dst_v = pointer::offset(v_cache_dst, pos_dst * kv_cache_dst_stride); + const auto vec_k = warp::load_vec(src_k); + const auto vec_v = warp::load_vec(src_v); + warp::store_vec(dst_k, vec_k); + warp::store_vec(dst_v, vec_v); + } +} + +template < + std::integral T, + std::size_t kElementSize, + std::size_t kUnroll, + std::size_t kBlockQuota, + std::size_t kNumThreads, + std::size_t kMaxOccupancy> +__global__ __launch_bounds__(kNumThreads, kMaxOccupancy) void hicache_transfer_all_layer( + const __grid_constant__ HicacheKernelParams params) { + // each warp acts as a worker + using namespace device; + using src_ptr_t = std::add_pointer_t; + using dst_ptr_t = std::add_pointer_t; + + static_assert(kNumThreads % kWarpThreads == 0); + constexpr auto kWarpThreads = device::kWarpThreads / kUnroll; + constexpr auto kWarpsPerBlock = static_cast(kNumThreads) / kWarpThreads; + constexpr auto kWorkers = kWarpsPerBlock * kBlockQuota; + + const auto& [ + k_ptr_dst, v_ptr_dst, indices_dst, // dst + k_ptr_src, v_ptr_src, indices_src, // src + length, kv_cache_src_stride, kv_cache_dst_stride, num_layers // metadata + ] = params; + const auto warp_id = blockIdx.x * kWarpsPerBlock + threadIdx.x / kWarpThreads; + + // force to transfer 128 bytes per iteration + // since the PCIe transaction size is 128 bytes aligned + constexpr auto kGranularity = 128 / kWarpThreads; + + for (auto i = warp_id; i < length; i += kWorkers) { + const auto pos_src = static_cast(indices_src)[i]; + const auto pos_dst = static_cast(indices_dst)[i]; + for (std::size_t layer = 0; layer < num_layers; ++layer) { + const auto k_cache_src = static_cast(k_ptr_src)[layer]; + const auto v_cache_src = static_cast(v_ptr_src)[layer]; + const auto k_cache_dst = static_cast(k_ptr_dst)[layer]; + const auto v_cache_dst = static_cast(v_ptr_dst)[layer]; + const auto src_k = pointer::offset(k_cache_src, pos_src * kv_cache_src_stride); + const auto dst_k = pointer::offset(k_cache_dst, pos_dst * kv_cache_dst_stride); + const auto src_v = pointer::offset(v_cache_src, pos_src * kv_cache_src_stride); + const auto dst_v = pointer::offset(v_cache_dst, pos_dst * kv_cache_dst_stride); + const auto vec_k = warp::load_vec(src_k); + const auto vec_v = warp::load_vec(src_v); + warp::store_vec(dst_k, vec_k); + warp::store_vec(dst_v, vec_v); + } + } +} + +template < + std::size_t kElementSize, + std::size_t kUnroll, + std::size_t kBlockQuota, + std::size_t kNumThreads, + std::size_t kMaxOccupancy> +struct HiCacheKernel { + template + static constexpr auto _kernel_one = + hicache_transfer_per_layer; + template + static constexpr auto _kernel_all = + hicache_transfer_all_layer; + + static void run_one( + const tvm::ffi::TensorView k_cache_dst, + const tvm::ffi::TensorView v_cache_dst, + const tvm::ffi::TensorView indices_dst, + const tvm::ffi::TensorView k_cache_src, + const tvm::ffi::TensorView v_cache_src, + const tvm::ffi::TensorView indices_src) { + using namespace host; + + auto D = SymbolicSize{"D"}; // cache dimension + auto N = SymbolicSize{"N"}; // src kv stride + auto M = SymbolicSize{"M"}; // dst kv stride + auto L = SymbolicSize{"L"}; // indices length + auto cache_dtype = SymbolicDType{}; + auto indices_dtype = SymbolicDType{}; + auto indices_device = SymbolicDevice{}; + + TensorMatcher({-1, D}) // + .with_strides({N, 1}) + .with_dtype(cache_dtype) + .with_device() + .verify(k_cache_src) + .verify(v_cache_src); + TensorMatcher({-1, D}) // + .with_strides({M, 1}) + .with_dtype(cache_dtype) + .with_device() + .verify(k_cache_dst) + .verify(v_cache_dst); + TensorMatcher({L}) // + .with_dtype(indices_dtype) + .with_device(indices_device) + .verify(indices_src) + .verify(indices_dst); + + // verify dimension match + const auto dtype_size = dtype_bytes(cache_dtype.unwrap()); + const auto element_bytes = D.unwrap() * dtype_size; + RuntimeCheck(kElementSize == element_bytes, "HicacheKernel: cache dimension mismatch."); + + const auto k_cache_dst_ptr = k_cache_dst.data_ptr(); + const auto v_cache_dst_ptr = v_cache_dst.data_ptr(); + const auto k_cache_src_ptr = k_cache_src.data_ptr(); + const auto v_cache_src_ptr = v_cache_src.data_ptr(); + const auto indices_dst_ptr = indices_dst.data_ptr(); + const auto indices_src_ptr = indices_src.data_ptr(); + const auto length = static_cast(L.unwrap()); + const auto kv_cache_src_stride = static_cast(N.unwrap()) * dtype_size; + const auto kv_cache_dst_stride = static_cast(M.unwrap()) * dtype_size; + const auto use_int32 = indices_dtype.unwrap().bits == 32; + const auto device = indices_device.unwrap(); + + constexpr auto kWorkersPerBlock = kNumThreads / (device::kWarpThreads / kUnroll); + const auto num_blocks = std::min(div_ceil(length, kWorkersPerBlock), kBlockQuota); + const auto params = HicacheKernelParams{ + .k_cache_dst = k_cache_dst_ptr, + .v_cache_dst = v_cache_dst_ptr, + .indices_dst = indices_dst_ptr, + .k_cache_src = k_cache_src_ptr, + .v_cache_src = v_cache_src_ptr, + .indices_src = indices_src_ptr, + .length = length, + .kv_cache_src_stride = kv_cache_src_stride, + .kv_cache_dst_stride = kv_cache_dst_stride, + }; + const auto kernel = use_int32 ? _kernel_one : _kernel_one; + LaunchKernel(num_blocks, kNumThreads, device)(kernel, params); + } + + static void run_all( + const tvm::ffi::TensorView k_ptr_dst, + const tvm::ffi::TensorView v_ptr_dst, + const tvm::ffi::TensorView indices_dst, + const tvm::ffi::TensorView k_ptr_src, + const tvm::ffi::TensorView v_ptr_src, + const tvm::ffi::TensorView indices_src, + const std::size_t kv_src_stride, + const std::size_t kv_dst_stride) { + using namespace host; + + auto N = SymbolicSize{"N"}; // num layers + auto L = SymbolicSize{"L"}; // indices length + auto dtype_ = SymbolicDType{}; + auto device_ = SymbolicDevice{}; + + TensorMatcher({N}) // + .with_dtype() + .with_device(device_) + .verify(k_ptr_src) + .verify(v_ptr_src) + .verify(k_ptr_dst) + .verify(v_ptr_dst); + TensorMatcher({L}) // + .with_dtype(dtype_) + .with_device(device_) + .verify(indices_src) + .verify(indices_dst); + + // verify dimension match + const auto k_cache_dst_ptr = k_ptr_dst.data_ptr(); + const auto v_cache_dst_ptr = v_ptr_dst.data_ptr(); + const auto k_cache_src_ptr = k_ptr_src.data_ptr(); + const auto v_cache_src_ptr = v_ptr_src.data_ptr(); + const auto indices_dst_ptr = indices_dst.data_ptr(); + const auto indices_src_ptr = indices_src.data_ptr(); + const auto length = static_cast(L.unwrap()); + const auto use_int32 = dtype_.unwrap().bits == 32; + const auto device = device_.unwrap(); + + constexpr auto kWorkersPerBlock = kNumThreads / (device::kWarpThreads / kUnroll); + const auto num_blocks = std::min(div_ceil(length, kWorkersPerBlock), kBlockQuota); + const auto params = HicacheKernelParams{ + .k_cache_dst = k_cache_dst_ptr, + .v_cache_dst = v_cache_dst_ptr, + .indices_dst = indices_dst_ptr, + .k_cache_src = k_cache_src_ptr, + .v_cache_src = v_cache_src_ptr, + .indices_src = indices_src_ptr, + .length = length, + .kv_cache_src_stride = kv_src_stride, + .kv_cache_dst_stride = kv_dst_stride, + .num_layers = static_cast(N.unwrap()), + }; + const auto kernel = use_int32 ? _kernel_all : _kernel_all; + LaunchKernel(num_blocks, kNumThreads, device)(kernel, params); + } +}; + +} // namespace diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/cuda_wait_value.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/cuda_wait_value.py new file mode 100644 index 00000000..27665dbb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/cuda_wait_value.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from functools import lru_cache +from typing import TYPE_CHECKING + +import torch + +from sglang.jit_kernel.utils import load_jit + +if TYPE_CHECKING: + import torch + from tvm_ffi.module import Module + + +@lru_cache(maxsize=1) +def _jit_stream_wait_value_module() -> Module: + return load_jit( + "cuda_wait_value", + cuda_files=["cuda_wait_value.cuh"], + cuda_wrappers=[("stream_wait_value", "stream_wait_value")], + ) + + +def stream_wait_value(flag: torch.Tensor, value: int) -> None: + module = _jit_stream_wait_value_module() + module.stream_wait_value(flag, value) + + +class Event: + def __init__(self) -> None: + self.flag = torch.zeros(1, dtype=torch.int32, device="cuda") + + def record(self, value: int = 1) -> None: + self.flag[0] = value + + def wait(self, value: int = 1) -> None: + stream_wait_value(self.flag, value) + + +def test_wait_before_record(event: Event | torch.cuda.Event): + stream_a = torch.cuda.Stream() + stream_b = torch.cuda.Stream() + + with torch.cuda.stream(stream_a): + event.wait() + + stream_a.synchronize() + + with torch.cuda.stream(stream_b): + event.record() + + +def main(): + import threading + import time + + block_thead = threading.Thread( + target=test_wait_before_record, args=(Event(),), daemon=True + ) + block_thead.start() + + non_block_thread = threading.Thread( + target=test_wait_before_record, args=(torch.cuda.Event(),) + ) + non_block_thread.start() + + print("Checking if custom Event blocks the stream...", flush=True) + for _ in range(5): + print(f"{block_thead.is_alive()=}, {non_block_thread.is_alive()=}", flush=True) + time.sleep(1) + + assert block_thead.is_alive(), "Custom Event did not block as expected" + assert not non_block_thread.is_alive(), "torch.cuda.Event should not block" + print("=" * 40) + print("Test completed successfully.") + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/hicache.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/hicache.py new file mode 100644 index 00000000..1d015fe0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/hicache.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import logging +from functools import lru_cache +from typing import TYPE_CHECKING + +from sglang.jit_kernel.utils import load_jit, make_cpp_args + +if TYPE_CHECKING: + import torch + from tvm_ffi.module import Module + +DEFAULT_BLOCK_QUOTA = 2 + + +@lru_cache(maxsize=None) +def _jit_hicache_module(*, element_size: int, unroll: int, block_quota: int) -> Module: + num_threads, occupancy = 1024, 1 + args = make_cpp_args( + element_size, + unroll, + block_quota, + num_threads, + occupancy, + ) + return load_jit( + "hicache", + *args, + cuda_files=["hicache.cuh"], + cuda_wrappers=[ + ("launch_one", f"HiCacheKernel<{args}>::run_one"), + ("launch_all", f"HiCacheKernel<{args}>::run_all"), + ], + ) + + +def can_use_hicache_jit_kernel( + *, + element_size: int, + unroll: int | None = None, # can be tuned for performance + block_quota: int | None = None, # can be tuned for less interference +) -> bool: + try: + unroll = unroll or _default_unroll(element_size) + block_quota = block_quota or DEFAULT_BLOCK_QUOTA + _jit_hicache_module( + element_size=element_size, + unroll=unroll, + block_quota=block_quota, + ) + return True + except Exception as e: + logger = logging.getLogger(__name__) + logger.warning(f"Failed to load JIT HiCache kernel: {e}") + return False + + +def _default_unroll(element_size: int) -> int: + if element_size <= 512: + return 4 + + if element_size <= 1024: + return 2 + + # fallback: no unroll + return 1 + + +def transfer_hicache_one_layer( + k_cache_dst: torch.Tensor, + v_cache_dst: torch.Tensor, + indices_dst: torch.Tensor, + k_cache_src: torch.Tensor, + v_cache_src: torch.Tensor, + indices_src: torch.Tensor, + *, + element_dim: int | None = None, + unroll: int | None = None, # can be tuned for performance + block_quota: int | None = None, # can be tuned for less interference +) -> None: + element_dim = element_dim or k_cache_dst.size(-1) + k_cache_src = k_cache_src.view(-1, element_dim) + v_cache_src = v_cache_src.view(-1, element_dim) + k_cache_dst = k_cache_dst.view(-1, element_dim) + v_cache_dst = v_cache_dst.view(-1, element_dim) + element_size = element_dim * k_cache_dst.element_size() + block_quota = block_quota or DEFAULT_BLOCK_QUOTA + unroll = unroll or _default_unroll(element_size) + module = _jit_hicache_module( + element_size=element_size, + unroll=unroll, + block_quota=block_quota, + ) + module.launch_one( + k_cache_dst, + v_cache_dst, + indices_dst, + k_cache_src, + v_cache_src, + indices_src, + ) + + +def transfer_hicache_all_layer( + k_ptr_dst: torch.Tensor, + v_ptr_dst: torch.Tensor, + indices_dst: torch.Tensor, + k_ptr_src: torch.Tensor, + v_ptr_src: torch.Tensor, + indices_src: torch.Tensor, + *, + kv_cache_src_stride_bytes: int, + kv_cache_dst_stride_bytes: int, + element_size: int | None = None, + unroll: int | None = None, # can be tuned for performance + block_quota: int | None = None, # can be tuned for less interference +) -> None: + if element_size is None: # assume both contiguous + assert kv_cache_dst_stride_bytes == kv_cache_src_stride_bytes + element_size = kv_cache_dst_stride_bytes + + block_quota = block_quota or DEFAULT_BLOCK_QUOTA + unroll = unroll or _default_unroll(element_size) + module = _jit_hicache_module( + element_size=element_size, + unroll=unroll, + block_quota=block_quota, + ) + module.launch_all( + k_ptr_dst, + v_ptr_dst, + indices_dst, + k_ptr_src, + v_ptr_src, + indices_src, + kv_cache_src_stride_bytes, + kv_cache_dst_stride_bytes, + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/include/sgl_kernel/tensor.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/include/sgl_kernel/tensor.h new file mode 100644 index 00000000..8208149e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/include/sgl_kernel/tensor.h @@ -0,0 +1,487 @@ +#pragma once +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace host { + +namespace stdr = std::ranges; +namespace stdv = std::views; + +namespace details { + +struct SizeRef; +struct DTypeRef; +struct DeviceRef; + +template +struct dtype_trait {}; + +template +struct dtype_trait { + inline static constexpr auto value = DLDataType{ + .code = std::is_signed_v ? DLDataTypeCode::kDLInt : DLDataTypeCode::kDLUInt, + .bits = static_cast(sizeof(T) * 8), + .lanes = 1}; +}; + +template +struct dtype_trait { + inline static constexpr auto value = + DLDataType{.code = DLDataTypeCode::kDLFloat, .bits = static_cast(sizeof(T) * 8), .lanes = 1}; +}; + +inline constexpr auto kAnyDeviceID = -1; +inline constexpr auto kAnySize = static_cast(-1); +inline constexpr auto kNullSize = static_cast(-1); +inline constexpr auto kNullDType = static_cast(18u); +inline constexpr auto kNullDevice = static_cast(-1); + +template +inline constexpr auto kDTypeList = std::array{dtype_trait::value...}; + +template +inline constexpr auto kDeviceList = std::array{ + DLDevice{.device_type = static_cast(Codes), .device_id = kAnyDeviceID}...}; + +template +struct PrintAbleSpan { + explicit PrintAbleSpan(std::span data) : data(data) {} + std::span data; +}; + +// define DLDataType comparison and printing in root namespace +inline constexpr auto kDeviceStringMap = [] { + constexpr auto map = std::array, 16>{ + std::pair{DLDeviceType::kDLCPU, "cpu"}, + std::pair{DLDeviceType::kDLCUDA, "cuda"}, + std::pair{DLDeviceType::kDLCUDAHost, "cuda_host"}, + std::pair{DLDeviceType::kDLOpenCL, "opencl"}, + std::pair{DLDeviceType::kDLVulkan, "vulkan"}, + std::pair{DLDeviceType::kDLMetal, "metal"}, + std::pair{DLDeviceType::kDLVPI, "vpi"}, + std::pair{DLDeviceType::kDLROCM, "rocm"}, + std::pair{DLDeviceType::kDLROCMHost, "rocm_host"}, + std::pair{DLDeviceType::kDLExtDev, "ext_dev"}, + std::pair{DLDeviceType::kDLCUDAManaged, "cuda_managed"}, + std::pair{DLDeviceType::kDLOneAPI, "oneapi"}, + std::pair{DLDeviceType::kDLWebGPU, "webgpu"}, + std::pair{DLDeviceType::kDLHexagon, "hexagon"}, + std::pair{DLDeviceType::kDLMAIA, "maia"}, + std::pair{DLDeviceType::kDLTrn, "trn"}, + }; + constexpr auto max_type = stdr::max(map | stdv::keys); + auto result = std::array{}; + for (const auto& [code, name] : map) { + result[static_cast(code)] = name; + } + return result; +}(); + +struct PrintableDevice { + DLDevice device; +}; + +inline auto& operator<<(std::ostream& os, DLDevice device) { + const auto& mapping = kDeviceStringMap; + const auto entry = static_cast(device.device_type); + host::RuntimeCheck(entry < mapping.size()); + const auto name = mapping[entry]; + host::RuntimeCheck(!name.empty(), "Unknown device: ", int(device.device_type)); + os << name; + if (device.device_id != kAnyDeviceID) os << "[" << device.device_id << "]"; + return os; +} + +inline auto& operator<<(std::ostream& os, PrintableDevice pd) { + return os << pd.device; +} + +template +inline auto& operator<<(std::ostream& os, PrintAbleSpan span) { + os << "["; + for (const auto i : stdv::iota(std::size_t{0}, span.data.size())) { + if (i > 0) { + os << ", "; + } + os << span.data[i]; + } + os << "]"; + return os; +} + +} // namespace details + +struct SymbolicSize { + public: + SymbolicSize(std::string_view annotation = {}) : m_value(details::kNullSize), m_annotation(annotation) {} + + auto get_name() const -> std::string_view { + return m_annotation; + } + auto set_value(int64_t value) -> void { + host::RuntimeCheck(!this->has_value(), "Size value already set"); + m_value = value; + } + auto has_value() const -> bool { + return m_value != details::kNullSize; + } + auto get_value() const -> std::optional { + return this->has_value() ? std::optional{m_value} : std::nullopt; + } + auto unwrap() const -> int64_t { + host::RuntimeCheck(this->has_value(), "Size value is not set"); + return m_value; + } + + SymbolicSize(const SymbolicSize&) = delete; + SymbolicSize& operator=(const SymbolicSize&) = delete; + + auto verify(int64_t dim) -> void { + if (this->has_value()) { + host::RuntimeCheck(m_value == dim, "Size mismatch: expected ", m_value, " but got ", dim); + } else { + this->set_value(dim); + } + } + + private: + std::int64_t m_value; + std::string_view m_annotation; +}; + +inline auto operator==(DLDevice lhs, DLDevice rhs) -> bool { + return lhs.device_type == rhs.device_type && lhs.device_id == rhs.device_id; +} + +struct SymbolicDType { + public: + SymbolicDType() : m_value({details::kNullDType, 0, 0}) {} + + auto set_value(DLDataType value) -> void { + host::RuntimeCheck(!this->has_value(), "Dtype value already set"); + host::RuntimeCheck( + m_check(value), "Dtype value [", value, "] not in the allowed options: ", details::PrintAbleSpan{m_options}); + m_value = value; + } + auto has_value() const -> bool { + return m_value.code != details::kNullDType; + } + auto get_value() const -> std::optional { + return this->has_value() ? std::optional{m_value} : std::nullopt; + } + auto unwrap() const -> DLDataType { + host::RuntimeCheck(this->has_value(), "Dtype value is not set"); + return m_value; + } + + auto set_options(std::span options) -> void { + m_options = options; + } + template + auto set_options() -> void { + m_options = details::kDTypeList; + } + + auto verify(DLDataType dtype) -> void { + if (this->has_value()) { + host::RuntimeCheck(m_value == dtype, "DType mismatch: expected ", m_value, " but got ", dtype); + } else { + this->set_value(dtype); + } + } + + private: + auto m_check(DLDataType value) const -> bool { + return stdr::empty(m_options) || (stdr::find(m_options, value) != stdr::end(m_options)); + } + + std::span m_options; + DLDataType m_value; +}; + +struct SymbolicDevice { + public: + SymbolicDevice() : m_value({details::kNullDevice, details::kAnyDeviceID}) {} + + auto set_value(DLDevice value) -> void { + host::RuntimeCheck(!this->has_value(), "Device value already set"); + host::RuntimeCheck( + m_check(value), + "Device value [", + details::PrintableDevice{value}, + "] not in the allowed options: ", + details::PrintAbleSpan{m_options}); + m_value = value; + } + auto has_value() const -> bool { + return m_value.device_type != details::kNullDevice; + } + auto get_value() const -> std::optional { + return this->has_value() ? std::optional{m_value} : std::nullopt; + } + auto unwrap() const -> DLDevice { + host::RuntimeCheck(this->has_value(), "Device value is not set"); + return m_value; + } + + auto set_options(std::span options) -> void { + m_options = options; + } + template + auto set_options() -> void { + m_options = details::kDeviceList; + } + + auto verify(DLDevice device) -> void { + if (this->has_value()) { + host::RuntimeCheck( + m_value == device, + "Device mismatch: expected ", + details::PrintableDevice{m_value}, + " but got ", + details::PrintableDevice{device}); + } else { + this->set_value(device); + } + } + + private: + auto m_check(DLDevice value) const -> bool { + return stdr::empty(m_options) || (stdr::any_of(m_options, [value](const DLDevice& opt) { + // device type must exactly match + if (opt.device_type != value.device_type) return false; + // device id can be wildcarded + return opt.device_id == details::kAnyDeviceID || opt.device_id == value.device_id; + })); + } + + std::span m_options; + DLDevice m_value; +}; + +namespace details { + +template +struct BaseRef { + public: + BaseRef(const BaseRef&) = delete; + BaseRef& operator=(const BaseRef&) = delete; + + auto operator->() const -> T* { + return m_ref; + } + auto operator*() const -> T& { + return *m_ref; + } + auto rebind(T& other) -> void { + m_ref = &other; + } + + explicit BaseRef() : m_ref(&m_cache), m_cache() {} + BaseRef(T& size) : m_ref(&size), m_cache() {} + + private: + T* m_ref; + T m_cache; +}; + +struct SizeRef : BaseRef { + using BaseRef::BaseRef; + SizeRef(int64_t value) { + if (value != kAnySize) { + (**this).set_value(value); + } else { + // otherwise, we can match any size + } + } + + auto value_or_name(std::size_t dim) const -> std::string { + if (const auto value = (**this).get_value()) { + return std::to_string(*value); + } else { + const auto annotation = (**this).get_name(); + if (annotation.empty()) { + return "dim#" + std::to_string(dim); + } else { + return static_cast(annotation); + } + } + } +}; + +struct DTypeRef : BaseRef { + using BaseRef::BaseRef; + DTypeRef(DLDataType options) { + (**this).set_value(options); + } + DTypeRef(std::initializer_list options) { + (**this).set_options(options); + } + DTypeRef(std::span options) { + (**this).set_options(options); + } +}; + +struct DeviceRef : BaseRef { + using BaseRef::BaseRef; + DeviceRef(DLDevice options) { + (**this).set_value(options); + } + DeviceRef(std::initializer_list options) { + (**this).set_options(options); + } + DeviceRef(std::span options) { + (**this).set_options(options); + } +}; + +} // namespace details + +struct TensorMatcher { + private: + using SizeRef = details::SizeRef; + using DTypeRef = details::DTypeRef; + using DeviceRef = details::DeviceRef; + using Loc_t = std::source_location; + + public: + TensorMatcher(const TensorMatcher&) = delete; + TensorMatcher& operator=(const TensorMatcher&) = delete; + + explicit TensorMatcher(std::initializer_list shape) : m_shape(shape), m_strides(), m_dtype() {} + + auto with_strides(std::initializer_list strides) && -> TensorMatcher&& { + // no partial update allowed + host::RuntimeCheck(m_strides.size() == 0, "Strides already specified"); + host::RuntimeCheck(m_shape.size() == strides.size(), "Strides size must match shape size"); + m_strides = strides; + return std::move(*this); + } + + template + auto with_dtype(DTypeRef&& dtype) && -> TensorMatcher&& { + m_init_dtype(); + m_dtype.rebind(*dtype); + return std::move(*this); + } + + template + auto with_dtype() && -> TensorMatcher&& { + static_assert(sizeof...(Ts) > 0, "At least one dtype option must be specified"); + m_init_dtype(); + m_dtype->set_options(); + return std::move(*this); + } + + template + auto with_device(DeviceRef&& device) && -> TensorMatcher&& { + m_init_device(); + m_device.rebind(*device); + return std::move(*this); + } + + template + auto with_device() && -> TensorMatcher&& { + static_assert(sizeof...(Codes) > 0, "At least one device option must be specified"); + m_init_device(); + m_device->set_options(); + return std::move(*this); + } + + // once we start verification, we cannot modify anymore + auto verify(tvm::ffi::TensorView view, Loc_t loc = Loc_t::current()) const&& -> const TensorMatcher&& { + try { + this->m_verify_impl(view); + } catch (PanicError& e) { + auto oss = std::ostringstream{}; + oss << "Tensor match failed for " << this->debug_str() << " at " << loc.file_name() << ":" << loc.line() + << "\n- Root cause: " << e.detail(); + throw PanicError(std::move(oss).str()); + } + return std::move(*this); + } + + auto debug_str() const -> std::string { + auto oss = std::ostringstream{}; + oss << "Tensor<"; + std::size_t dim = 0; + for (const auto& size_ref : m_shape) { + if (dim > 0) { + oss << ", "; + } + oss << size_ref.value_or_name(dim++); + } + oss << ">"; + if (m_strides.size() > 0) { + oss << " [strides=<"; + dim = 0; + for (const auto& stride_ref : m_strides) { + if (dim > 0) { + oss << ", "; + } + oss << stride_ref.value_or_name(dim++); + } + oss << ">]"; + } + return std::move(oss).str(); + } + + private: + auto m_verify_impl(tvm::ffi::TensorView view) const -> void { + const auto dim = static_cast(view.dim()); + host::RuntimeCheck(dim == m_shape.size(), "Tensor dimension mismatch: expected ", m_shape.size(), " but got ", dim); + for (const auto i : stdv::iota(std::size_t{0}, dim)) { + m_shape[i]->verify(view.size(i)); + } + if (this->m_has_strides()) { + for (const auto i : stdv::iota(std::size_t{0}, dim)) { + m_strides[i]->verify(view.stride(i)); + } + } else { + host::RuntimeCheck(view.is_contiguous(), "Tensor is not contiguous as expected"); + } + // since we may use the same matcher to verify again, we will force to check + m_dtype->verify(view.dtype()); + m_device->verify(view.device()); + } + + auto m_init_dtype() -> void { + host::RuntimeCheck(!m_has_dtype, "DType already specified"); + m_has_dtype = true; + } + auto m_init_device() -> void { + host::RuntimeCheck(!m_has_device, "Device already specified"); + m_has_device = true; + } + auto m_has_strides() const -> bool { + return !m_strides.empty(); + } + + std::span m_shape; + std::span m_strides; + DTypeRef m_dtype; + DeviceRef m_device; + bool m_has_dtype = false; + bool m_has_device = false; +}; + +} // namespace host diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/include/sgl_kernel/utils.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/include/sgl_kernel/utils.cuh new file mode 100644 index 00000000..cf03d8c0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/include/sgl_kernel/utils.cuh @@ -0,0 +1,101 @@ +#pragma once + +#include + +#include +#include + +#include +#include +#include +#include + +namespace device { + +inline constexpr auto kWarpThreads = 32u; + +namespace pointer { + +// we only allow void * pointer arithmetic for safety + +template +__always_inline __device__ auto offset(T* ptr, U... offset) -> void* { + static_assert(std::is_same_v, "Pointer arithmetic is only allowed for void* pointers"); + return static_cast(ptr) + (... + offset); +} + +template +__always_inline __device__ auto offset(const T* ptr, U... offset) -> const void* { + static_assert(std::is_same_v, "Pointer arithmetic is only allowed for void* pointers"); + return static_cast(ptr) + (... + offset); +} + +} // namespace pointer + +} // namespace device + +namespace host { + +inline auto +RuntimeDeviceCheck(::cudaError_t error, std::source_location location = std::source_location::current()) -> void { + if (error != ::cudaSuccess) { + [[unlikely]]; + ::host::panic(location, "CUDA error: ", ::cudaGetErrorString(error)); + } +} + +inline auto RuntimeCudaCheck(std::source_location location = std::source_location::current()) -> void { + return RuntimeDeviceCheck(::cudaGetLastError(), location); +} + +template +inline void set_smem_once(std::size_t smem_size) { + static const auto last_smem_size = [&] { + RuntimeDeviceCheck(::cudaFuncSetAttribute(F, ::cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + return smem_size; + }(); + RuntimeCheck( + smem_size <= last_smem_size, + "Dynamic shared memory size exceeds the previously set maximum size: ", + last_smem_size, + " bytes"); +} + +struct LaunchKernel { + public: + explicit LaunchKernel( + dim3 grid_dim, dim3 block_dim, DLDevice device, std::size_t dynamic_shared_mem_bytes = 0) noexcept + : m_config(s_make_config(grid_dim, block_dim, resolve_device(device), dynamic_shared_mem_bytes)) {} + + explicit LaunchKernel( + dim3 grid_dim, dim3 block_dim, cudaStream_t stream, std::size_t dynamic_shared_mem_bytes = 0) noexcept + : m_config(s_make_config(grid_dim, block_dim, stream, dynamic_shared_mem_bytes)) {} + + static auto resolve_device(DLDevice device) -> cudaStream_t { + return static_cast(::TVMFFIEnvGetStream(device.device_type, device.device_id)); + } + + LaunchKernel(const LaunchKernel&) = delete; + LaunchKernel& operator=(const LaunchKernel&) = delete; + + template + auto operator()(T&& kernel, Args&&... args) const -> void { + host::RuntimeDeviceCheck(::cudaLaunchKernelEx(&m_config, kernel, std::forward(args)...)); + } + + private: + static auto + s_make_config(dim3 grid_dim, dim3 block_dim, cudaStream_t stream, std::size_t smem) -> cudaLaunchConfig_t { + auto config = ::cudaLaunchConfig_t{}; + config.gridDim = grid_dim; + config.blockDim = block_dim; + config.dynamicSmemBytes = smem; + config.stream = stream; + config.numAttrs = 0; + return config; + } + cudaLaunchConfig_t m_config; + /// TODO: We can add a queue to store the attributes if needed in the future. +}; + +} // namespace host diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/include/sgl_kernel/utils.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/include/sgl_kernel/utils.h new file mode 100644 index 00000000..fd9723df --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/include/sgl_kernel/utils.h @@ -0,0 +1,88 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +namespace host { + +struct PanicError : public std::runtime_error { + public: + // copy and move constructors + explicit PanicError(std::string msg) : runtime_error(msg), m_message(std::move(msg)) {} + auto detail() const -> std::string_view { + const auto sv = std::string_view{m_message}; + const auto pos = sv.find(": "); + return pos == std::string_view::npos ? sv : sv.substr(pos + 2); + } + + private: + std::string m_message; +}; + +template +[[noreturn]] +inline auto panic(std::source_location location, Args&&... args) -> void { + std::ostringstream os; + os << "Runtime check failed at " << location.file_name() << ":" << location.line(); + if constexpr (sizeof...(args) > 0) { + os << ": "; + (os << ... << std::forward(args)); + } else { + os << " in " << location.function_name(); + } + throw PanicError(std::move(os).str()); +} + +template +struct RuntimeCheck { + using Loc_t = std::source_location; + template + explicit RuntimeCheck(Cond&& condition, Args&&... args, Loc_t location = Loc_t::current()) { + if (!condition) { + [[unlikely]]; + ::host::panic(location, std::forward(args)...); + } + } +}; + +template +explicit RuntimeCheck(Cond&&, Args&&...) -> RuntimeCheck; + +template +inline constexpr auto div_ceil(T a, U b) { + return (a + b - 1) / b; +} + +template +inline constexpr auto div_ceil(T a, U b) { + return (a + b - 1) / b; +} + +inline auto dtype_bytes(DLDataType dtype) -> std::size_t { + return static_cast(dtype.bits / 8); +} + +namespace pointer { + +// we only allow void * pointer arithmetic for safety + +template +inline auto offset(T* ptr, U... offset) -> void* { + static_assert(std::is_same_v, "Pointer arithmetic is only allowed for void* pointers"); + return static_cast(ptr) + (... + offset); +} + +template +inline auto offset(const T* ptr, U... offset) -> const void* { + static_assert(std::is_same_v, "Pointer arithmetic is only allowed for void* pointers"); + return static_cast(ptr) + (... + offset); +} + +} // namespace pointer + +} // namespace host diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/include/sgl_kernel/warp.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/include/sgl_kernel/warp.cuh new file mode 100644 index 00000000..904531f3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/include/sgl_kernel/warp.cuh @@ -0,0 +1,145 @@ +#pragma once +#include + +#include +#include +#include + +namespace device::warp { + +namespace details { + +template +inline constexpr auto get_mem_package() { + if constexpr (kUnit == 16) { + return uint4{}; + } else if constexpr (kUnit == 8) { + return uint2{}; + } else if constexpr (kUnit == 4) { + return uint1{}; + } else { + static_assert(kUnit == 16 || kUnit == 8 || kUnit == 4, "Unsupported memory package size"); + } +} + +inline constexpr auto default_unit_size(std::size_t x) -> std::size_t { + if (x % (16 * kWarpThreads) == 0) return 16; + if (x % (8 * kWarpThreads) == 0) return 8; + if (x % (4 * kWarpThreads) == 0) return 4; + return 0; // trigger static assert in _get_mem_package +} + +template +using mem_package_t = decltype(get_mem_package()); + +template +struct storage_vec { + T data[N]; +}; + +__always_inline __device__ auto load_nc(const uint1* __restrict__ src) -> uint1 { + uint32_t tmp; + asm volatile("ld.global.cs.b32 %0,[%1];" : "=r"(tmp) : "l"(src)); + return uint1{tmp}; +} + +__always_inline __device__ auto load_nc(const uint2* __restrict__ src) -> uint2 { + uint32_t tmp0, tmp1; + asm volatile("ld.global.cs.v2.b32 {%0,%1},[%2];" : "=r"(tmp0), "=r"(tmp1) : "l"(src)); + return uint2{tmp0, tmp1}; +} + +__always_inline __device__ auto load_nc(const uint4* __restrict__ src) -> uint4 { + uint32_t tmp0, tmp1, tmp2, tmp3; + asm volatile("ld.global.cs.v4.b32 {%0,%1,%2,%3},[%4];" : "=r"(tmp0), "=r"(tmp1), "=r"(tmp2), "=r"(tmp3) : "l"(src)); + return uint4{tmp0, tmp1, tmp2, tmp3}; +} + +__always_inline __device__ void store_nc(uint1* __restrict__ dst, const uint1& value) { + uint32_t tmp = value.x; + asm volatile("st.global.cs.b32 [%0],%1;" ::"l"(dst), "r"(tmp)); +} + +__always_inline __device__ void store_nc(uint2* __restrict__ dst, const uint2& value) { + uint32_t tmp0 = value.x; + uint32_t tmp1 = value.y; + asm volatile("st.global.cs.v2.b32 [%0],{%1,%2};" ::"l"(dst), "r"(tmp0), "r"(tmp1)); +} + +__always_inline __device__ void store_nc(uint4* __restrict__ dst, const uint4& value) { + uint32_t tmp0 = value.x; + uint32_t tmp1 = value.y; + uint32_t tmp2 = value.z; + uint32_t tmp3 = value.w; + asm volatile("st.global.cs.v4.b32 [%0],{%1,%2,%3,%4};" ::"l"(dst), "r"(tmp0), "r"(tmp1), "r"(tmp2), "r"(tmp3)); +} + +} // namespace details + +template < + std::size_t kBytes, + std::size_t kUnit = details::default_unit_size(kBytes), + std::size_t kThreads = ::device::kWarpThreads> +__always_inline __device__ void copy(void* __restrict__ dst, const void* __restrict__ src) { + using Package = details::mem_package_t; + constexpr auto kBytesPerLoop = sizeof(Package) * kThreads; + constexpr auto kLoopCount = kBytes / kBytesPerLoop; + static_assert(kBytes % kBytesPerLoop == 0, "kBytes must be multiple of 128 bytes"); + + const auto dst_packed = static_cast(dst); + const auto src_packed = static_cast(src); + const auto lane_id = threadIdx.x % kThreads; + +#pragma unroll kLoopCount + for (std::size_t i = 0; i < kLoopCount; ++i) { + const auto j = i * kThreads + lane_id; + dst_packed[j] = src_packed[j]; + } +} + +template < + std::size_t kBytes, + std::size_t kUnit = details::default_unit_size(kBytes), + std::size_t kThreads = ::device::kWarpThreads> +__always_inline __device__ auto load_vec(const void* __restrict__ src) { + using Package = details::mem_package_t; + constexpr auto kBytesPerLoop = sizeof(Package) * kThreads; + constexpr auto kLoopCount = kBytes / kBytesPerLoop; + static_assert(kBytes % kBytesPerLoop == 0, "kBytes must be multiple of 128 bytes"); + + const auto src_packed = static_cast(src); + const auto lane_id = threadIdx.x % kThreads; + details::storage_vec vec; + +#pragma unroll kLoopCount + for (std::size_t i = 0; i < kLoopCount; ++i) { + const auto j = i * kThreads + lane_id; + vec.data[i] = details::load_nc(src_packed + j); + } + + return vec; +} + +template < + std::size_t kBytes, + std::size_t kUnit = details::default_unit_size(kBytes), + std::size_t kThreads = ::device::kWarpThreads, + typename Tp> +__always_inline __device__ void store_vec(void* __restrict__ dst, const Tp& vec) { + using Package = details::mem_package_t; + constexpr auto kBytesPerLoop = sizeof(Package) * kThreads; + constexpr auto kLoopCount = kBytes / kBytesPerLoop; + static_assert(kBytes % kBytesPerLoop == 0, "kBytes must be multiple of 128 bytes"); + static_assert(std::is_same_v>); + + const auto dst_packed = static_cast(dst); + const auto lane_id = threadIdx.x % kThreads; + +#pragma unroll kLoopCount + for (std::size_t i = 0; i < kLoopCount; ++i) { + const auto j = i * kThreads + lane_id; + details::store_nc(dst_packed + j, vec.data[i]); + } +} + +} // namespace device::warp diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/utils.py new file mode 100644 index 00000000..6462cf41 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/jit_kernel/utils.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import pathlib +from functools import lru_cache +from typing import TYPE_CHECKING, List, Tuple, TypeAlias, Union + +if TYPE_CHECKING: + from tvm_ffi import Module + + +def _make_wrapper(tup: Tuple[str, str]) -> str: + export_name, kernel_name = tup + return f"TVM_FFI_DLL_EXPORT_TYPED_FUNC({export_name}, ({kernel_name}));" + + +@lru_cache() +def _resolve_kernel_path() -> pathlib.Path: + cur_dir = pathlib.Path(__file__).parent.resolve() + + # first, try this directory structure + def _environment_install(): + candidate = cur_dir.resolve() + if (candidate / "include").exists() and (candidate / "csrc").exists(): + return candidate + return None + + def _package_install(): + # TODO: support find path by package + return None + + path = _environment_install() or _package_install() + if path is None: + raise RuntimeError("Cannot find sgl-kernel/jit path") + return path + + +KERNEL_PATH = _resolve_kernel_path() +DEFAULT_INCLUDE = [str(KERNEL_PATH / "include")] +DEFAULT_CFLAGS = ["-std=c++20", "-O3"] +DEFAULT_CUDA_CFLAGS = ["-std=c++20", "-O3", "--expt-relaxed-constexpr"] +DEFAULT_LDFLAGS = [] +CPP_TEMPLATE_TYPE: TypeAlias = Union[int, float, bool] + + +class CPPArgList(list[str]): + def __str__(self) -> str: + return ", ".join(self) + + +def make_cpp_args(*args: CPP_TEMPLATE_TYPE) -> CPPArgList: + def _convert(arg: CPP_TEMPLATE_TYPE) -> str: + if isinstance(arg, bool): + return "true" if arg else "false" + if isinstance(arg, (int, float)): + return str(arg) + raise TypeError(f"Unsupported argument type for cpp template: {type(arg)}") + + return CPPArgList(_convert(arg) for arg in args) + + +def load_jit( + *args: str, + cpp_files: List[str] | None = None, + cuda_files: List[str] | None = None, + cpp_wrappers: List[Tuple[str, str]] | None = None, + cuda_wrappers: List[Tuple[str, str]] | None = None, + extra_cflags: List[str] | None = None, + extra_cuda_cflags: List[str] | None = None, + extra_ldflags: List[str] | None = None, + extra_include_paths: List[str] | None = None, + build_directory: str | None = None, +) -> Module: + from tvm_ffi.cpp import load_inline + + cpp_files = cpp_files or [] + cuda_files = cuda_files or [] + cpp_wrappers = cpp_wrappers or [] + cuda_wrappers = cuda_wrappers or [] + extra_cflags = extra_cflags or [] + extra_cuda_cflags = extra_cuda_cflags or [] + extra_ldflags = extra_ldflags or [] + extra_include_paths = extra_include_paths or [] + + # include cpp files + cpp_paths = [(KERNEL_PATH / "csrc" / f).resolve() for f in cpp_files] + cpp_sources = [f'#include "{path}"' for path in cpp_paths] + cpp_sources += [_make_wrapper(tup) for tup in cpp_wrappers] + + # include cuda files + cuda_paths = [(KERNEL_PATH / "csrc" / f).resolve() for f in cuda_files] + cuda_sources = [f'#include "{path}"' for path in cuda_paths] + cuda_sources += [_make_wrapper(tup) for tup in cuda_wrappers] + + return load_inline( + "sgl_kernel_jit_" + "_".join(str(arg) for arg in args), + cpp_sources=cpp_sources, + cuda_sources=cuda_sources, + extra_cflags=DEFAULT_CFLAGS + extra_cflags, + extra_cuda_cflags=DEFAULT_CUDA_CFLAGS + extra_cuda_cflags, + extra_ldflags=DEFAULT_LDFLAGS + extra_ldflags, + extra_include_paths=DEFAULT_INCLUDE + extra_include_paths, + build_directory=build_directory, + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/api.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/api.py new file mode 100644 index 00000000..745c656e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/api.py @@ -0,0 +1,292 @@ +"""Public APIs of the language.""" + +import re +from typing import Callable, List, Optional, Union + +from sglang.global_config import global_config +from sglang.lang.backend.base_backend import BaseBackend +from sglang.lang.choices import ChoicesSamplingMethod, token_length_normalized +from sglang.lang.ir import ( + SglExpr, + SglExprList, + SglFunction, + SglGen, + SglImage, + SglRoleBegin, + SglRoleEnd, + SglSelect, + SglSeparateReasoning, + SglVideo, +) + + +def function( + func: Optional[Callable] = None, num_api_spec_tokens: Optional[int] = None +): + if func: + return SglFunction(func, num_api_spec_tokens=num_api_spec_tokens) + + def decorator(func): + return SglFunction(func, num_api_spec_tokens=num_api_spec_tokens) + + return decorator + + +def Runtime(*args, **kwargs): + # Avoid importing unnecessary dependency + from sglang.lang.backend.runtime_endpoint import Runtime + + return Runtime(*args, **kwargs) + + +def Engine(*args, **kwargs): + # Avoid importing unnecessary dependency + from sglang.srt.entrypoints.engine import Engine + + return Engine(*args, **kwargs) + + +def set_default_backend(backend: BaseBackend): + global_config.default_backend = backend + + +def flush_cache(backend: Optional[BaseBackend] = None): + backend = backend or global_config.default_backend + if backend is None: + return False + + # If backend is Runtime + if hasattr(backend, "endpoint"): + backend = backend.endpoint + return backend.flush_cache() + + +def get_server_info(backend: Optional[BaseBackend] = None): + backend = backend or global_config.default_backend + if backend is None: + return None + + # If backend is Runtime + if hasattr(backend, "endpoint"): + backend = backend.endpoint + return backend.get_server_info() + + +def gen( + name: Optional[str] = None, + max_tokens: Optional[int] = None, + min_tokens: Optional[int] = None, + n: Optional[int] = None, + stop: Optional[Union[str, List[str]]] = None, + stop_token_ids: Optional[List[int]] = None, + stop_regex: Optional[Union[str, List[str]]] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + top_k: Optional[int] = None, + min_p: Optional[float] = None, + frequency_penalty: Optional[float] = None, + presence_penalty: Optional[float] = None, + ignore_eos: Optional[bool] = None, + return_logprob: Optional[bool] = None, + logprob_start_len: Optional[int] = None, + top_logprobs_num: Optional[int] = None, + return_text_in_logprobs: Optional[bool] = None, + dtype: Optional[Union[type, str]] = None, + choices: Optional[List[str]] = None, + choices_method: Optional[ChoicesSamplingMethod] = None, + regex: Optional[str] = None, + json_schema: Optional[str] = None, +): + """Call the model to generate. See the meaning of the arguments in docs/backend/sampling_params.md""" + + if choices: + return SglSelect( + name, + choices, + 0.0 if temperature is None else temperature, + token_length_normalized if choices_method is None else choices_method, + ) + + # check regex is valid + if regex is not None: + try: + re.compile(regex) + except re.error as e: + raise e + + return SglGen( + name, + max_tokens, + min_tokens, + n, + stop, + stop_token_ids, + stop_regex, + temperature, + top_p, + top_k, + min_p, + frequency_penalty, + presence_penalty, + ignore_eos, + return_logprob, + logprob_start_len, + top_logprobs_num, + return_text_in_logprobs, + dtype, + regex, + json_schema, + ) + + +def gen_int( + name: Optional[str] = None, + max_tokens: Optional[int] = None, + n: Optional[int] = None, + stop: Optional[Union[str, List[str]]] = None, + stop_token_ids: Optional[List[int]] = None, + stop_regex: Optional[Union[str, List[str]]] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + top_k: Optional[int] = None, + min_p: Optional[float] = None, + frequency_penalty: Optional[float] = None, + presence_penalty: Optional[float] = None, + ignore_eos: Optional[bool] = None, + return_logprob: Optional[bool] = None, + logprob_start_len: Optional[int] = None, + top_logprobs_num: Optional[int] = None, + return_text_in_logprobs: Optional[bool] = None, +): + return SglGen( + name, + max_tokens, + None, + n, + stop, + stop_token_ids, + stop_regex, + temperature, + top_p, + top_k, + min_p, + frequency_penalty, + presence_penalty, + ignore_eos, + return_logprob, + logprob_start_len, + top_logprobs_num, + return_text_in_logprobs, + int, + None, + ) + + +def gen_string( + name: Optional[str] = None, + max_tokens: Optional[int] = None, + n: Optional[int] = None, + stop: Optional[Union[str, List[str]]] = None, + stop_token_ids: Optional[List[int]] = None, + stop_regex: Optional[Union[str, List[str]]] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + top_k: Optional[int] = None, + min_p: Optional[float] = None, + frequency_penalty: Optional[float] = None, + presence_penalty: Optional[float] = None, + ignore_eos: Optional[bool] = None, + return_logprob: Optional[bool] = None, + logprob_start_len: Optional[int] = None, + top_logprobs_num: Optional[int] = None, + return_text_in_logprobs: Optional[bool] = None, +): + return SglGen( + name, + max_tokens, + None, + n, + stop, + stop_token_ids, + stop_regex, + temperature, + top_p, + top_k, + min_p, + frequency_penalty, + presence_penalty, + ignore_eos, + return_logprob, + logprob_start_len, + top_logprobs_num, + return_text_in_logprobs, + str, + None, + ) + + +def image(expr: SglExpr): + return SglImage(expr) + + +def video(path: str, num_frames: int): + return SglVideo(path, num_frames) + + +def select( + name: Optional[str] = None, + choices: Optional[List[str]] = None, + temperature: float = 0.0, + choices_method: ChoicesSamplingMethod = token_length_normalized, +): + assert choices is not None + return SglSelect(name, choices, temperature, choices_method) + + +def _role_common(name: str, expr: Optional[SglExpr] = None): + if expr is None: + return SglExprList([SglRoleBegin(name), SglRoleEnd(name)]) + else: + return SglExprList([SglRoleBegin(name), expr, SglRoleEnd(name)]) + + +def system(expr: Optional[SglExpr] = None): + return _role_common("system", expr) + + +def user(expr: Optional[SglExpr] = None): + return _role_common("user", expr) + + +def assistant(expr: Optional[SglExpr] = None): + return _role_common("assistant", expr) + + +def system_begin(): + return SglRoleBegin("system") + + +def system_end(): + return SglRoleEnd("system") + + +def user_begin(): + return SglRoleBegin("user") + + +def user_end(): + return SglRoleEnd("user") + + +def assistant_begin(): + return SglRoleBegin("assistant") + + +def assistant_end(): + return SglRoleEnd("assistant") + + +def separate_reasoning( + expr: Optional[SglExpr] = None, model_type: Optional[str] = None +): + return SglExprList([expr, SglSeparateReasoning(model_type, expr=expr)]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/anthropic.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/anthropic.py new file mode 100644 index 00000000..4918a170 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/anthropic.py @@ -0,0 +1,73 @@ +from sglang.lang.backend.base_backend import BaseBackend +from sglang.lang.chat_template import get_chat_template +from sglang.lang.interpreter import StreamExecutor +from sglang.lang.ir import SglSamplingParams + +try: + import anthropic +except ImportError as e: + anthropic = e + + +class Anthropic(BaseBackend): + def __init__(self, model_name, *args, **kwargs): + super().__init__() + + if isinstance(anthropic, Exception): + raise anthropic + + self.model_name = model_name + self.chat_template = get_chat_template("claude") + self.client = anthropic.Anthropic(*args, **kwargs) + + def get_chat_template(self): + return self.chat_template + + def generate( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + if s.messages_: + messages = s.messages_ + else: + messages = [{"role": "user", "content": s.text_}] + + if messages and messages[0]["role"] == "system": + system = messages.pop(0)["content"] + else: + system = "" + + ret = self.client.messages.create( + model=self.model_name, + system=system, + messages=messages, + **sampling_params.to_anthropic_kwargs(), + ) + comp = ret.content[0].text + + return comp, {} + + def generate_stream( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + if s.messages_: + messages = s.messages_ + else: + messages = [{"role": "user", "content": s.text_}] + + if messages and messages[0]["role"] == "system": + system = messages.pop(0)["content"] + else: + system = "" + + with self.client.messages.stream( + model=self.model_name, + system=system, + messages=messages, + **sampling_params.to_anthropic_kwargs(), + ) as stream: + for text in stream.text_stream: + yield text, {} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/base_backend.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/base_backend.py new file mode 100644 index 00000000..62dd5041 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/base_backend.py @@ -0,0 +1,82 @@ +from typing import List, Optional, Union + +from sglang.lang.chat_template import get_chat_template +from sglang.lang.choices import ChoicesDecision, ChoicesSamplingMethod +from sglang.lang.interpreter import StreamExecutor +from sglang.lang.ir import SglSamplingParams + + +class BaseBackend: + def __init__(self) -> None: + self.support_concate_and_append = False + self.chat_template = get_chat_template("default") + + def get_model_name(self): + raise NotImplementedError() + + def get_chat_template(self): + return self.chat_template + + def cache_prefix(self, prefix_str: str): + pass + + def uncache_prefix(self, rid: str): + pass + + def end_request(self, rid: Union[str, List[str]]): + pass + + def begin_program(self, s: StreamExecutor): + pass + + def end_program(self, s: Union[StreamExecutor, List[StreamExecutor]]): + pass + + def commit_lazy_operations(self, s: StreamExecutor): + pass + + def fork_program( + self, + src: StreamExecutor, + dst: List[StreamExecutor], + position_ids_offset: Optional[List[int]] = None, + ): + pass + + def fill_image(self, s: StreamExecutor): + pass + + def generate( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + raise NotImplementedError() + + def generate_stream( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + raise NotImplementedError() + + def select( + self, + s: StreamExecutor, + choices: List[str], + temperature: float, + choices_method: Optional[ChoicesSamplingMethod] = None, + ) -> ChoicesDecision: + raise NotImplementedError() + + def concatenate_and_append(self, src_rids: List[str], dst_rid: str): + raise NotImplementedError() + + def shutdown(self): + pass + + def flush_cache(self): + pass + + def get_server_info(self): + pass diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/litellm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/litellm.py new file mode 100644 index 00000000..5803b543 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/litellm.py @@ -0,0 +1,90 @@ +from typing import Mapping, Optional + +from sglang.lang.backend.base_backend import BaseBackend +from sglang.lang.chat_template import get_chat_template_by_model_path +from sglang.lang.interpreter import StreamExecutor +from sglang.lang.ir import SglSamplingParams + +try: + import litellm +except ImportError as e: + litellm = e + litellm.num_retries = 1 + + +class LiteLLM(BaseBackend): + def __init__( + self, + model_name, + chat_template=None, + api_key=None, + organization: Optional[str] = None, + base_url: Optional[str] = None, + timeout: Optional[float] = 600, + max_retries: Optional[int] = litellm.num_retries, + default_headers: Optional[Mapping[str, str]] = None, + ): + super().__init__() + + if isinstance(litellm, Exception): + raise litellm + + self.model_name = model_name + + self.chat_template = chat_template or get_chat_template_by_model_path( + model_name + ) + + self.client_params = { + "api_key": api_key, + "organization": organization, + "base_url": base_url, + "timeout": timeout, + "max_retries": max_retries, + "default_headers": default_headers, + } + + def get_chat_template(self): + return self.chat_template + + def generate( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + if s.messages_: + messages = s.messages_ + else: + messages = [{"role": "user", "content": s.text_}] + + ret = litellm.completion( + model=self.model_name, + messages=messages, + **self.client_params, + **sampling_params.to_litellm_kwargs(), + ) + comp = ret.choices[0].message.content + + return comp, {} + + def generate_stream( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + if s.messages_: + messages = s.messages_ + else: + messages = [{"role": "user", "content": s.text_}] + + ret = litellm.completion( + model=self.model_name, + messages=messages, + stream=True, + **self.client_params, + **sampling_params.to_litellm_kwargs(), + ) + for chunk in ret: + text = chunk.choices[0].delta.content + if text is not None: + yield text, {} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/openai.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/openai.py new file mode 100644 index 00000000..a2d006bb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/openai.py @@ -0,0 +1,475 @@ +import dataclasses +import logging +import time +import warnings +from typing import List, Optional, Union + +import numpy as np + +from sglang.lang.backend.base_backend import BaseBackend +from sglang.lang.chat_template import ChatTemplate, get_chat_template_by_model_path +from sglang.lang.choices import ChoicesDecision, ChoicesSamplingMethod +from sglang.lang.interpreter import StreamExecutor +from sglang.lang.ir import SglSamplingParams + +try: + import openai + import tiktoken +except ImportError as e: + openai = tiktoken = e + + +logger = logging.getLogger(__name__) + + +def create_logit_bias_int(tokenizer): + """Get logit bias for integer numbers.""" + int_token_ids = [] + + tokens = tokenizer._mergeable_ranks + for token, token_id in tokens.items(): + s = tokenizer.decode([token_id]) + if all([c.isdigit() for c in s]) or s in [" "]: + int_token_ids.append(token_id) + if len(int_token_ids) >= 300: # OpenAI API limit + break + special_tokens = tokenizer._special_tokens + mask = {t: 100 for t in int_token_ids[:299]} + mask[special_tokens["<|endoftext|>"]] = 100 + return mask + + +INSTRUCT_MODEL_NAMES = [ + "gpt-3.5-turbo-instruct", +] + + +@dataclasses.dataclass +class TokenUsage: + prompt_tokens: int + completion_tokens: int + + def reset(self): + self.prompt_tokens = self.completion_tokens = 0 + + +class OpenAI(BaseBackend): + def __init__( + self, + model_name: str, + is_chat_model: Optional[bool] = None, + chat_template: Optional[ChatTemplate] = None, + is_azure: bool = False, + *args, + **kwargs, + ): + super().__init__() + + if isinstance(openai, Exception): + raise openai + + if is_azure: + self.client = openai.AzureOpenAI(*args, **kwargs) + else: + self.client = openai.OpenAI(*args, **kwargs) + + self.model_name = model_name + try: + self.tokenizer = tiktoken.encoding_for_model(model_name) + except KeyError: + self.tokenizer = tiktoken.get_encoding("cl100k_base") + self.logit_bias_int = create_logit_bias_int(self.tokenizer) + + self.chat_template = chat_template or get_chat_template_by_model_path( + model_name + ) + + if is_chat_model is not None: + self.is_chat_model = is_chat_model + else: + if model_name in INSTRUCT_MODEL_NAMES: + self.is_chat_model = False + else: + self.is_chat_model = True + + self.chat_prefix = self.chat_template.role_prefix_and_suffix["assistant"][0] + + # Usage + self.token_usage = TokenUsage(0, 0) + + # API speculative execution + # TODO(ying): This does not support multi-threading (run_batch) + self.spec_kwargs = {} + self.spec_format = [] + self.spec_max_num_tries = 3 + + def get_chat_template(self): + return self.chat_template + + def _prepare_spec_execution( + self, + sampling_params: SglSamplingParams, + num_api_spec_tokens: int, + spec_var_name: str, + ): + if "max_tokens" not in self.spec_kwargs: + self.spec_kwargs["max_tokens"] = num_api_spec_tokens + else: + assert self.spec_kwargs["max_tokens"] == num_api_spec_tokens + + params = sampling_params.to_openai_kwargs() + for key, value in params.items(): + if key in ["stop"]: + continue + if key in ["max_tokens"]: + warnings.warn( + "The parameter max_tokens will be overwritten by speculated number of tokens." + ) + continue + if key not in self.spec_kwargs: + self.spec_kwargs[key] = value + else: + assert ( + value == self.spec_kwargs[key] + ), "sampling parameters should be consistent if turn on api speculative execution." + self.spec_format.append( + {"text": "", "stop": params["stop"], "name": spec_var_name} + ) + return "", {} + + def generate( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + spec_var_name: str = None, + ): + if sampling_params.dtype is None: + if self.is_chat_model: + if s.num_api_spec_tokens is None: + if not s.text_.endswith(self.chat_prefix): + raise RuntimeError( + "This use case is not supported if api speculative execution is off. " + "For OpenAI chat models, sgl.gen must be right after sgl.assistant. " + "Example of adding api speculative execution: @function(num_api_spec_tokens=128)." + ) + prompt = s.messages_ + else: + return self._prepare_spec_execution( + sampling_params, s.num_api_spec_tokens, spec_var_name + ) + else: + prompt = s.text_ + + kwargs = sampling_params.to_openai_kwargs() + if ( + self.model_name.startswith("o1") + or self.model_name.startswith("o3") + or "o1" in self.model_name + ): + kwargs.pop("max_tokens", None) + else: + kwargs.pop("max_completion_tokens", None) + + comp = openai_completion( + client=self.client, + token_usage=self.token_usage, + is_chat=self.is_chat_model, + model=self.model_name, + prompt=prompt, + **kwargs, + ) + # Keep the returned list (or string) as is. + elif sampling_params.dtype in [str, "str", "string"]: + assert ( + not self.is_chat_model + ), "constrained type not supported on chat model" + kwargs = sampling_params.to_openai_kwargs() + kwargs.pop("stop") + comp = openai_completion( + client=self.client, + token_usage=self.token_usage, + is_chat=self.is_chat_model, + model=self.model_name, + prompt=s.text_ + '"', + stop='"', + **kwargs, + ) + # Wrap each element in quotes if we have a list. + if isinstance(comp, list): + comp = ['"' + x + '"' for x in comp] + else: + comp = '"' + comp + '"' + elif sampling_params.dtype in [int, "int"]: + assert ( + not self.is_chat_model + ), "constrained type not supported on chat model" + kwargs = sampling_params.to_openai_kwargs() + kwargs.pop("stop") + comp = openai_completion( + client=self.client, + token_usage=self.token_usage, + is_chat=self.is_chat_model, + model=self.model_name, + prompt=s.text_, + logit_bias=self.logit_bias_int, + stop=[" "], + **kwargs, + ) + # Leave as a list if that's what is returned. + else: + raise ValueError(f"Unknown dtype: {sampling_params.dtype}") + + return comp, {} + + def spec_fill(self, value: str): + assert self.is_chat_model + self.spec_format.append({"text": value, "stop": None, "name": None}) + + def spec_pattern_match(self, comp): + for i, term in enumerate(self.spec_format): + text = term["text"] + if text != "": + if comp.startswith(text): + comp = comp[len(text) :] + else: + return False + else: + pos = comp.find(term["stop"]) + if pos != -1: + term["text"] = comp[:pos] + comp = comp[pos:] + else: + if i == len(self.spec_format) - 1: + term["text"] = comp + else: + return False + return True + + def role_end_generate( + self, + s: StreamExecutor, + ): + if s.num_api_spec_tokens is None or not s.text_.endswith(self.chat_prefix): + return + + comp = "" + if not all(x["name"] is None for x in self.spec_format): + # TODO(ying): throw errors or warnings + for i in range(self.spec_max_num_tries): + comp = openai_completion( + client=self.client, + token_usage=self.token_usage, + is_chat=self.is_chat_model, + model=self.model_name, + prompt=s.messages_, + **self.spec_kwargs, + ) + # Use a string for pattern matching. + comp_for_match = comp[0] if isinstance(comp, list) else comp + if self.spec_pattern_match(comp_for_match): + break + + for term in self.spec_format: + s.text_ += term["text"] + name = term["name"] + if name is not None: + s.variables[name] = term["text"] + s.meta_info[name] = {} + s.variable_event[name].set() + + self.spec_kwargs = {} + self.spec_format = [] + + def generate_stream( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + if sampling_params.dtype is None: + if self.is_chat_model: + if not s.text_.endswith(self.chat_prefix): + raise RuntimeError( + "This use case is not supported. " + "For OpenAI chat models, sgl.gen must be right after sgl.assistant" + ) + prompt = s.messages_ + else: + prompt = s.text_ + + kwargs = sampling_params.to_openai_kwargs() + generator = openai_completion_stream( + client=self.client, + token_usage=self.token_usage, + is_chat=self.is_chat_model, + model=self.model_name, + prompt=prompt, + **kwargs, + ) + return generator + else: + raise ValueError(f"Unknown dtype: {sampling_params.dtype}") + + def select( + self, + s: StreamExecutor, + choices: List[str], + temperature: float, + choices_method: ChoicesSamplingMethod, + ) -> ChoicesDecision: + """Note: `choices_method` is not used by the OpenAI backend.""" + if self.is_chat_model: + raise NotImplementedError( + "select/choices is not supported for chat models. " + "Please try to use a non-chat model such as gpt-3.5-turbo-instruct" + ) + + n_choices = len(choices) + token_ids = [self.tokenizer.encode(x) for x in choices] + scores = [0] * n_choices + valid = [len(x) > 0 for x in token_ids] + prompt_tokens = self.tokenizer.encode(s.text_) + + max_len = max([len(x) for x in token_ids]) + for step in range(max_len): + # Build logit bias + logit_bias = {} + for i in range(n_choices): + if valid[i]: + logit_bias[token_ids[i][step]] = 100 + + # Call API + ret = self.client.completions.create( + model=self.model_name, + prompt=prompt_tokens, + logit_bias=logit_bias, + max_tokens=1, + temperature=temperature, + ) + ret_str = ret.choices[0].text + ret_token = self.tokenizer.encode(ret_str)[0] + self.token_usage.prompt_tokens += ret.usage.prompt_tokens + self.token_usage.completion_tokens = ret.usage.completion_tokens + + # TODO: + # 1. return logits as the scores + # 2. compute logits of the full choice + # 3. consider chunk-based decoding + + # Update valid + hit = False + for i in range(n_choices): + if valid[i]: + if step == len(token_ids[i]) - 1: + valid[i] = False + + if ret_token == token_ids[i][step]: + scores[i] += 1 + hit = True + else: + valid[i] = False + assert hit + + if np.sum(valid) <= 1: + break + + prompt_tokens.append(ret_token) + + return ChoicesDecision( + decision=choices[np.argmax(scores)], + meta_info={"scores": scores}, + ) + + +def openai_completion( + client, token_usage, is_chat=None, retries=3, prompt=None, **kwargs +) -> Union[str, List[str]]: + # if "ebnf" is in kwargs, warn and remove + if "ebnf" in kwargs: + warnings.warn("EBNF is not officially supported by OpenAI endpoints. Ignoring.") + del kwargs["ebnf"] + + for attempt in range(retries): + try: + if is_chat: + if "stop" in kwargs and kwargs["stop"] is None: + kwargs.pop("stop") + ret = client.chat.completions.create(messages=prompt, **kwargs) + if len(ret.choices) == 1: + comp = ret.choices[0].message.content + else: + comp = [c.message.content for c in ret.choices] + else: + ret = client.completions.create(prompt=prompt, **kwargs) + if isinstance(prompt, (list, tuple)): + comp = [c.text for c in ret.choices] + else: + comp = ret.choices[0].text + if len(ret.choices) > 1: + comp = [c.text for c in ret.choices] + + token_usage.prompt_tokens += ret.usage.prompt_tokens + token_usage.completion_tokens += ret.usage.completion_tokens + break + except (openai.APIError, openai.APIConnectionError, openai.RateLimitError) as e: + logger.error(f"OpenAI Error: {e}. Waiting 5 seconds...") + time.sleep(5) + if attempt == retries - 1: + raise e + except Exception as e: + logger.error(f"RuntimeError {e}.") + raise e + + return comp + + +def openai_completion_stream( + client, token_usage, is_chat=None, retries=3, prompt=None, **kwargs +): + # if "ebnf" is in kwargs, warn and remove + if "ebnf" in kwargs: + warnings.warn("EBNF is not officially supported by OpenAI endpoints. Ignoring.") + del kwargs["ebnf"] + + for attempt in range(retries): + try: + if is_chat: + if "stop" in kwargs and kwargs["stop"] is None: + kwargs.pop("stop") + generator = client.chat.completions.create( + messages=prompt, + stream=True, + stream_options={"include_usage": True}, + **kwargs, + ) + for ret in generator: + if len(ret.choices) == 0: + continue + try: + content = ret.choices[0].delta.content + except IndexError: + content = None + yield content or "", {} + else: + generator = client.completions.create( + prompt=prompt, + stream=True, + stream_options={"include_usage": True}, + **kwargs, + ) + for ret in generator: + if len(ret.choices) == 0: + continue + content = ret.choices[0].text + yield content or "", {} + + token_usage.prompt_tokens += ret.usage.prompt_tokens + token_usage.completion_tokens += ret.usage.completion_tokens + break + except (openai.APIError, openai.APIConnectionError, openai.RateLimitError) as e: + logger.error(f"OpenAI Error: {e}. Waiting 5 seconds...") + time.sleep(5) + if attempt == retries - 1: + raise e + except Exception as e: + logger.error(f"RuntimeError {e}.") + raise e diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/runtime_endpoint.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/runtime_endpoint.py new file mode 100644 index 00000000..1573ca68 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/runtime_endpoint.py @@ -0,0 +1,527 @@ +import atexit +import json +import multiprocessing +import warnings +from typing import Dict, List, Optional, Union + +import aiohttp +import requests + +from sglang.global_config import global_config +from sglang.lang.backend.base_backend import BaseBackend +from sglang.lang.chat_template import get_chat_template, get_chat_template_by_model_path +from sglang.lang.choices import ChoicesDecision, ChoicesSamplingMethod +from sglang.lang.interpreter import StreamExecutor +from sglang.lang.ir import ( + REGEX_BOOL, + REGEX_FLOAT, + REGEX_INT, + REGEX_STR, + SglSamplingParams, +) +from sglang.utils import http_request + + +class RuntimeEndpoint(BaseBackend): + def __init__( + self, + base_url: str, + api_key: Optional[str] = None, + verify: Optional[str] = None, + chat_template_name: Optional[str] = None, + ): + super().__init__() + self.support_concate_and_append = True + + self.base_url = base_url + self.api_key = api_key + self.verify = verify + + res = http_request( + self.base_url + "/get_model_info", + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + self.model_info = res.json() + + if chat_template_name: + self.chat_template = get_chat_template(chat_template_name) + else: + self.chat_template = get_chat_template_by_model_path( + self.model_info["model_path"] + ) + + def get_model_name(self): + return self.model_info["model_path"] + + def flush_cache(self): + res = http_request( + self.base_url + "/flush_cache", + api_key=self.api_key, + verify=self.verify, + method="POST", + ) + self._assert_success(res) + + def get_server_info(self): + res = http_request( + self.base_url + "/get_server_info", + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + return res.json() + + def get_chat_template(self): + return self.chat_template + + def cache_prefix(self, prefix_str: str): + res = http_request( + self.base_url + "/generate", + json={"text": prefix_str, "sampling_params": {"max_new_tokens": 0}}, + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + + def start_profile(self): + res = http_request( + self.base_url + "/start_profile", + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + + def stop_profile(self): + res = http_request( + self.base_url + "/stop_profile", + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + + def commit_lazy_operations(self, s: StreamExecutor): + data = {"text": s.text_, "sampling_params": {"max_new_tokens": 0}} + self._add_images(s, data) + res = http_request( + self.base_url + "/generate", + json=data, + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + + def fill_image(self, s: StreamExecutor): + data = {"text": s.text_, "sampling_params": {"max_new_tokens": 0}} + self._add_images(s, data) + res = http_request( + self.base_url + "/generate", + json=data, + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + + def _handle_dtype_to_regex(self, sampling_params: SglSamplingParams): + if sampling_params.dtype is None: + return + + if sampling_params.stop == (): + sampling_params.stop = [] + + dtype_regex = None + if sampling_params.dtype in ["int", int]: + + dtype_regex = REGEX_INT + sampling_params.stop.extend([" ", "\n"]) + elif sampling_params.dtype in ["float", float]: + + dtype_regex = REGEX_FLOAT + sampling_params.stop.extend([" ", "\n"]) + elif sampling_params.dtype in ["str", str]: + + dtype_regex = REGEX_STR + elif sampling_params.dtype in ["bool", bool]: + + dtype_regex = REGEX_BOOL + else: + raise RuntimeError(f"Invalid dtype: {sampling_params.dtype}") + + if dtype_regex is not None and sampling_params.regex is not None: + warnings.warn( + f"Both dtype and regex are set. Only dtype will be used. dtype: {sampling_params.dtype}, regex: {sampling_params.regex}" + ) + + sampling_params.regex = dtype_regex + + def generate( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + self._handle_dtype_to_regex(sampling_params) + data = { + "text": s.text_, + "sampling_params": { + "skip_special_tokens": global_config.skip_special_tokens_in_output, + "spaces_between_special_tokens": global_config.spaces_between_special_tokens_in_out, + **sampling_params.to_srt_kwargs(), + }, + } + + for item in [ + "return_logprob", + "logprob_start_len", + "top_logprobs_num", + "return_text_in_logprobs", + ]: + value = getattr(sampling_params, item, None) + if value is not None: + data[item] = value + + self._add_images(s, data) + + res = http_request( + self.base_url + "/generate", + json=data, + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + + obj = res.json() + comp = obj["text"] + return comp, obj["meta_info"] + + def generate_stream( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + self._handle_dtype_to_regex(sampling_params) + + data = { + "text": s.text_, + "sampling_params": { + "skip_special_tokens": global_config.skip_special_tokens_in_output, + "spaces_between_special_tokens": global_config.spaces_between_special_tokens_in_out, + **sampling_params.to_srt_kwargs(), + }, + } + + for item in [ + "return_logprob", + "logprob_start_len", + "top_logprobs_num", + "return_text_in_logprobs", + ]: + value = getattr(sampling_params, item, None) + if value is not None: + data[item] = value + + data["stream"] = True + self._add_images(s, data) + + res = http_request( + self.base_url + "/generate", + json=data, + stream=True, + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + pos = 0 + + for chunk in res.iter_lines(decode_unicode=False): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]": + break + data = json.loads(chunk[5:].strip("\n")) + chunk_text = data["text"][pos:] + meta_info = data["meta_info"] + pos += len(chunk_text) + yield chunk_text, meta_info + + def select( + self, + s: StreamExecutor, + choices: List[str], + temperature: float, + choices_method: ChoicesSamplingMethod, + ) -> ChoicesDecision: + assert temperature <= 1e-5 + + # Cache common prefix + data = {"text": s.text_, "sampling_params": {"max_new_tokens": 0}} + obj = self._generate_http_request(s, data) + prompt_len = obj["meta_info"]["prompt_tokens"] + logprob_start_len = max(prompt_len - 2, 0) # For token healing + + # Compute logprob + data = { + "text": [s.text_ + c for c in choices], + "sampling_params": { + "max_new_tokens": 0, + "temperature": 0, + }, + "return_logprob": True, + "return_text_in_logprobs": True, + "logprob_start_len": logprob_start_len, + } + obj = self._generate_http_request(s, data) + + input_token_logprobs = [r["meta_info"]["input_token_logprobs"] for r in obj] + output_token_logprobs = [r["meta_info"]["output_token_logprobs"] for r in obj] + normalized_prompt_logprobs = [ + compute_normalized_prompt_logprobs(r["meta_info"]["input_token_logprobs"]) + for r in obj + ] + + # Remove extra token if no token healing occurred + for i in range(len(input_token_logprobs)): + healed_token_str = input_token_logprobs[i][0][-1] + if s.text_.endswith(healed_token_str): + healed_token_logprob = input_token_logprobs[i][0][0] + normalized_prompt_logprobs[i] = ( + normalized_prompt_logprobs[i] * len(input_token_logprobs[i]) + - healed_token_logprob + ) / (len(input_token_logprobs[i]) - 1) + input_token_logprobs[i] = input_token_logprobs[i][1:] + + # Compute unconditional logprobs if required + if choices_method.requires_unconditional_logprobs: + input_ids = [[el[1] for el in subl] for subl in input_token_logprobs] + data = { + "input_ids": input_ids, + "sampling_params": {"max_new_tokens": 0}, + "return_logprob": True, + } + obj = self._generate_http_request(s, data) + unconditional_token_logprobs = [ + r["meta_info"]["input_token_logprobs"] for r in obj + ] + else: + unconditional_token_logprobs = None + + return choices_method( + choices=choices, + normalized_prompt_logprobs=normalized_prompt_logprobs, + input_token_logprobs=input_token_logprobs, + output_token_logprobs=output_token_logprobs, + unconditional_token_logprobs=unconditional_token_logprobs, + ) + + def concatenate_and_append(self, src_rids: List[str], dst_rid: str): + res = http_request( + self.base_url + "/concate_and_append_request", + json={"src_rids": src_rids, "dst_rid": dst_rid}, + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + + def _generate_http_request(self, s: StreamExecutor, data): + self._add_images(s, data) + res = http_request( + self.base_url + "/generate", + json=data, + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + return res.json() + + def _add_images(self, s: StreamExecutor, data): + if s.images_: + assert len(s.images_) == 1, "Only support one image." + data["image_data"] = s.images_[0][1] + + def _assert_success(self, res): + if res.status_code != 200: + try: + content = res.json() + except json.JSONDecodeError: + content = res.text + raise RuntimeError(content) + + +def compute_normalized_prompt_logprobs(input_logprobs): + values = [x[0] for x in input_logprobs if x[0]] + return sum(values) / len(values) + + +class Runtime: + """ + A wrapper for the HTTP server. + This is used for launching the server in a python program without + using the command line interface. + + It is mainly used for the frontend language. + You should use the Engine class if you want to do normal offline processing without the frontend language. + """ + + def __init__( + self, + log_level: str = "error", + *args, + **kwargs, + ): + """See the arguments in server_args.py::ServerArgs""" + # We delay the import of any `sglang.srt` components in `sglang.lang`, so users can run + # client code without installing SRT server and its dependency if they want. + from sglang.srt.entrypoints.http_server import launch_server + from sglang.srt.server_args import ServerArgs + from sglang.srt.utils import is_port_available + + self.server_args = ServerArgs(*args, log_level=log_level, **kwargs) + + # Pre-allocate ports + for port in range(self.server_args.port, 40000): + if is_port_available(port): + break + self.server_args.port = port + + self.url = self.server_args.url() + self.generate_url = self.url + "/generate" + + # NOTE: We store pid instead of proc to fix some issues during __delete__ + self.pid = None + pipe_reader, pipe_writer = multiprocessing.Pipe(duplex=False) + + ctx = multiprocessing.get_context("spawn") + proc = ctx.Process( + target=launch_server, + args=(self.server_args, pipe_writer), + ) + proc.start() + pipe_writer.close() + self.pid = proc.pid + + # Before python program terminates, call shutdown implicitly. Therefore, users don't have to explicitly call .shutdown() + atexit.register(self.shutdown) + + # TODO: remove this pipe_writer mechanism and use `/health_generate` instead. + try: + init_state = pipe_reader.recv() + except EOFError: + init_state = "" + + if init_state != "ready": + self.shutdown() + raise RuntimeError( + "Initialization failed. Please see the error messages above." + ) + + self.endpoint = RuntimeEndpoint(self.url) + + def shutdown(self): + from sglang.srt.utils import kill_process_tree + + if self.pid is not None: + kill_process_tree(self.pid) + self.pid = None + + def start_profile(self): + self.endpoint.start_profile() + + def stop_profile(self): + self.endpoint.stop_profile() + + def cache_prefix(self, prefix: str): + self.endpoint.cache_prefix(prefix) + + def get_tokenizer(self): + from sglang.srt.utils.hf_transformers_utils import get_tokenizer + + return get_tokenizer( + self.server_args.tokenizer_path, + tokenizer_mode=self.server_args.tokenizer_mode, + trust_remote_code=self.server_args.trust_remote_code, + revision=self.server_args.revision, + ) + + async def async_generate( + self, + prompt: str, + sampling_params: Optional[Dict] = None, + ): + if self.server_args.skip_tokenizer_init: + json_data = { + "input_ids": prompt, + "sampling_params": sampling_params, + "stream": True, + } + else: + json_data = { + "text": prompt, + "sampling_params": sampling_params, + "stream": True, + } + pos = 0 + + timeout = aiohttp.ClientTimeout(total=3 * 3600) + async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session: + async with session.post(self.generate_url, json=json_data) as response: + async for chunk, _ in response.content.iter_chunks(): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]\n\n": + break + data = json.loads(chunk[5:].strip("\n")) + if "text" in data: + cur = data["text"][pos:] + if cur: + yield cur + pos += len(cur) + else: + yield data + + add_request = async_generate + + def generate( + self, + prompt: Union[str, List[str]], + sampling_params: Optional[Dict] = None, + return_logprob: Optional[Union[List[bool], bool]] = False, + logprob_start_len: Optional[Union[List[int], int]] = None, + top_logprobs_num: Optional[Union[List[int], int]] = None, + lora_path: Optional[List[Optional[str]]] = None, + ): + json_data = { + "text": prompt, + "sampling_params": sampling_params, + "return_logprob": return_logprob, + "logprob_start_len": logprob_start_len, + "top_logprobs_num": top_logprobs_num, + "lora_path": lora_path, + } + assert not isinstance(lora_path, list) or len(lora_path) == len(prompt) + response = requests.post( + self.url + "/generate", + json=json_data, + ) + return json.dumps(response.json()) + + def encode( + self, + prompt: Union[str, List[str], List[Dict], List[List[Dict]]], + ): + json_data = {"text": prompt} + response = requests.post(self.url + "/encode", json=json_data) + return json.dumps(response.json()) + + async def get_server_info(self): + async with aiohttp.ClientSession() as session: + async with session.get(f"{self.url}/get_server_info") as response: + if response.status == 200: + return await response.json() + else: + error_data = await response.json() + raise RuntimeError( + f"Failed to get server info. {error_data['error']['message']}" + ) + + def __del__(self): + self.shutdown() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/vertexai.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/vertexai.py new file mode 100644 index 00000000..3d51fb13 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/backend/vertexai.py @@ -0,0 +1,148 @@ +import os +import warnings + +from sglang.lang.backend.base_backend import BaseBackend +from sglang.lang.chat_template import get_chat_template +from sglang.lang.interpreter import StreamExecutor +from sglang.lang.ir import SglSamplingParams + +try: + import vertexai + from vertexai.preview.generative_models import ( + GenerationConfig, + GenerativeModel, + Image, + ) +except ImportError as e: + GenerativeModel = e + + +class VertexAI(BaseBackend): + def __init__(self, model_name, safety_settings=None): + super().__init__() + + if isinstance(GenerativeModel, Exception): + raise GenerativeModel + + project_id = os.environ["GCP_PROJECT_ID"] + location = os.environ.get("GCP_LOCATION") + vertexai.init(project=project_id, location=location) + + self.model_name = model_name + self.chat_template = get_chat_template("default") + self.safety_settings = safety_settings + + def get_chat_template(self): + return self.chat_template + + def generate( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + if s.messages_: + prompt = self.messages_to_vertexai_input(s.messages_) + else: + # single-turn + prompt = ( + self.text_to_vertexai_input(s.text_, s.cur_images) + if s.cur_images + else s.text_ + ) + ret = GenerativeModel(self.model_name).generate_content( + prompt, + generation_config=GenerationConfig(**sampling_params.to_vertexai_kwargs()), + safety_settings=self.safety_settings, + ) + + comp = ret.text + + return comp, {} + + def generate_stream( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + if s.messages_: + prompt = self.messages_to_vertexai_input(s.messages_) + else: + # single-turn + prompt = ( + self.text_to_vertexai_input(s.text_, s.cur_images) + if s.cur_images + else s.text_ + ) + generator = GenerativeModel(self.model_name).generate_content( + prompt, + stream=True, + generation_config=GenerationConfig(**sampling_params.to_vertexai_kwargs()), + safety_settings=self.safety_settings, + ) + for ret in generator: + yield ret.text, {} + + def text_to_vertexai_input(self, text, images): + input = [] + # split with image token + text_segs = text.split(self.chat_template.image_token) + for image_path, image_base64_data in images: + text_seg = text_segs.pop(0) + if text_seg != "": + input.append(text_seg) + input.append(Image.from_bytes(image_base64_data)) + text_seg = text_segs.pop(0) + if text_seg != "": + input.append(text_seg) + return input + + def messages_to_vertexai_input(self, messages): + vertexai_message = [] + # from openai message format to vertexai message format + for msg in messages: + if isinstance(msg["content"], str): + text = msg["content"] + else: + text = msg["content"][0]["text"] + + if msg["role"] == "system": + warnings.warn("Warning: system prompt is not supported in VertexAI.") + vertexai_message.append( + { + "role": "user", + "parts": [{"text": "System prompt: " + text}], + } + ) + vertexai_message.append( + { + "role": "model", + "parts": [{"text": "Understood."}], + } + ) + continue + if msg["role"] == "user": + vertexai_msg = { + "role": "user", + "parts": [{"text": text}], + } + elif msg["role"] == "assistant": + vertexai_msg = { + "role": "model", + "parts": [{"text": text}], + } + + # images + if isinstance(msg["content"], list) and len(msg["content"]) > 1: + for image in msg["content"][1:]: + assert image["type"] == "image_url" + vertexai_msg["parts"].append( + { + "inline_data": { + "data": image["image_url"]["url"].split(",")[1], + "mime_type": "image/jpeg", + } + } + ) + + vertexai_message.append(vertexai_msg) + return vertexai_message diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/chat_template.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/chat_template.py new file mode 100644 index 00000000..212d07e0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/chat_template.py @@ -0,0 +1,668 @@ +import re +from dataclasses import dataclass +from enum import Enum, auto +from typing import Callable, Dict, List, Tuple + + +class ChatTemplateStyle(Enum): + PLAIN = auto() + LLAMA2 = auto() + + +@dataclass +class ChatTemplate: + name: str + default_system_prompt: str + role_prefix_and_suffix: Dict[str, Tuple[str, str]] + stop_str: List[str] = () + image_token: str = "" + audio_token: str = ""), + }, + image_token=" \n", + ) +) + +register_chat_template( + ChatTemplate( + name="llama-2-chat", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ("<>\n", "\n<>\n\n"), + "user": ("[INST] ", " [/INST]"), + "assistant": ("", " "), + }, + style=ChatTemplateStyle.LLAMA2, + ) +) + +# Reference: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/blob/main/chat_template.json +register_chat_template( + ChatTemplate( + name="mistral", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ("[SYSTEM_PROMPT] ", " [/SYSTEM_PROMPT]"), + "user": ("[INST] ", " [/INST]"), + "assistant": ("", " "), + }, + stop_str=("",), + image_token="[IMG]", + ) +) + +register_chat_template( + ChatTemplate( + name="llama-3-instruct", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ( + "<|start_header_id|>system<|end_header_id|>\n\n", + "<|eot_id|>", + ), + "user": ( + "<|start_header_id|>user<|end_header_id|>\n\n", + "<|eot_id|>", + ), + "assistant": ( + "<|start_header_id|>assistant<|end_header_id|>\n\n", + "<|eot_id|>", + ), + }, + stop_str=("<|eot_id|>",), + image_token="<|image|>", + ) +) + +# https://huggingface.co/openbmb/MiniCPM-V-2_6 +register_chat_template( + ChatTemplate( + name="minicpmv", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ("", " "), + "user": ("user:", " "), + "assistant": ("assistant:", ""), + }, + stop_str=("<|im_end|>", "<|endoftext|>"), + image_token="(./)", + ) +) + +register_chat_template( + ChatTemplate( + name="janus-pro", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ( + "", + "", + ), + "User": ( + "<｜User｜>", + "", + ), + "assistant": ( + "<｜Assistant｜>", + "<｜end▁of▁sentence｜>", + ), + }, + stop_str=("<｜end▁of▁sentence｜>",), + image_token="\n", + ) +) + +# https://huggingface.co/openbmb/MiniCPM-o-2_6 +register_chat_template( + ChatTemplate( + name="minicpmo", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ("", " "), + "user": ("user:", " "), + "assistant": ("assistant:", ""), + }, + stop_str=("<|im_end|>", "<|endoftext|>"), + image_token="(./)", + audio_token="()", + ) +) + +register_chat_template( + ChatTemplate( + name="janus", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ( + "", + "", + ), + "user": ( + "<｜User｜>", + "", + ), + "assistant": ( + "<｜Assistant｜>", + "<｜end▁of▁sentence｜>", + ), + }, + stop_str=("<｜end▁of▁sentence｜>",), + image_token="\n", + ) +) + +# The difference between "llama-3-instruct-llava" and "llama-3-instruct" is that llava uses a different image_token. +register_chat_template( + ChatTemplate( + name="llama-3-instruct-llava", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ( + "<|start_header_id|>system<|end_header_id|>\n\n", + "<|eot_id|>", + ), + "user": ( + "<|start_header_id|>user<|end_header_id|>\n\n", + "<|eot_id|>", + ), + "assistant": ( + "<|start_header_id|>assistant<|end_header_id|>\n\n", + "<|eot_id|>", + ), + }, + stop_str=("<|eot_id|>",), + image_token="\n", + ) +) + +# Reference: https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct/blob/main/chat_template.json +register_chat_template( + ChatTemplate( + name="llama-4", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ( + "<|header_start|>system<|header_end|>\n\n", + "<|eot|>", + ), + "user": ( + "<|header_start|>user<|header_end|>\n\n", + "<|eot|>", + ), + "assistant": ( + "<|header_start|>assistant<|header_end|>\n\n", + "<|eot|>", + ), + }, + stop_str=("<|eot|>",), + image_token="<|image|>", + ) +) + +# Reference: https://modelscope.cn/models/01ai/Yi-1.5-34B-Chat/file/view/master?fileName=tokenizer_config.json&status=1 +register_chat_template( + ChatTemplate( + name="yi-1.5", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ("", ""), + "user": ("<|im_start|>user\n", "<|im_end|>\n<|im_start|>assistant\n"), + "assistant": ("", "<|im_end|>\n"), + }, + style=ChatTemplateStyle.PLAIN, + stop_str=("<|im_end|>",), + ) +) + +# Reference: https://github.com/01-ai/Yi/tree/main/VL#major-difference-with-llava +register_chat_template( + ChatTemplate( + name="yi-vl", + default_system_prompt=( + "This is a chat between an inquisitive human and an AI assistant. Assume the role of the AI assistant. Read all the images carefully, and respond to the human's questions with informative, helpful, detailed and polite answers." + "这是一个好奇的人类和一个人工智能助手之间的对话。假设你扮演这个AI助手的角色。仔细阅读所有的图像，并对人类的问题做出信息丰富、有帮助、详细的和礼貌的回答。" + ), + role_prefix_and_suffix={ + "system": ("", "\n\n"), + "user": ("### Human:", "\n"), + "assistant": ("### Assistant:", "\n"), + }, + image_token=" \n", + ) +) + +register_chat_template( + ChatTemplate( + name="gemma-it", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ("", ""), + "user": ("user\n", "\n"), + "assistant": ("model\n", "\n"), + }, + style=ChatTemplateStyle.PLAIN, + ) +) + +register_chat_template( + ChatTemplate( + name="dbrx-instruct", + default_system_prompt="You are DBRX, created by Databricks. You were last updated in December 2023. You answer questions based on information available up to that point.\nYOU PROVIDE SHORT RESPONSES TO SHORT QUESTIONS OR STATEMENTS, but provide thorough responses to more complex and open-ended questions.\nYou assist with various tasks, from writing to coding (using markdown for code blocks — remember to use ``` with code, JSON, and tables).\n(You do not have real-time data access or code execution capabilities. You avoid stereotyping and provide balanced perspectives on controversial topics. You do not provide song lyrics, poems, or news articles and do not divulge details of your training data.)\nThis is your system prompt, guiding your responses. Do not reference it, just respond to the user. If you find yourself talking about this message, stop. You should be responding appropriately and usually that means not mentioning this.\nYOU DO NOT MENTION ANY OF THIS INFORMATION ABOUT YOURSELF UNLESS THE INFORMATION IS DIRECTLY PERTINENT TO THE USER'S QUERY.", + role_prefix_and_suffix={ + "system": ("<|im_start|>system\n", "<|im_end|>"), + "user": ("\n<|im_start|>user\n", "<|im_end|>"), + "assistant": ("\n<|im_start|>assistant\n", "<|im_end|>"), + }, + stop_str=("<|im_end|>",), + ) +) + +register_chat_template( + ChatTemplate( + name="c4ai-command-r", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ( + "<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>", + "<|END_OF_TURN_TOKEN|>", + ), + "user": ("<|START_OF_TURN_TOKEN|><|USER_TOKEN|>", "<|END_OF_TURN_TOKEN|>"), + "assistant": ( + "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", + "<|END_OF_TURN_TOKEN|>", + ), + }, + style=ChatTemplateStyle.PLAIN, + ) +) + +# Adapted from https://huggingface.co/OpenGVLab/InternVL2-4B/blob/main/modeling_intern_vit.py +register_chat_template( + ChatTemplate( + name="internvl-2-5", + default_system_prompt="你是书生·万象，英文名是InternVL，是由上海人工智能实验室、清华大学及多家合作单位联合开发的多模态大语言模型。", + role_prefix_and_suffix={ + "system": ("<|im_start|>system\n", "<|im_end|>\n"), + "user": ("<|im_start|>user\n", "<|im_end|>\n"), + "assistant": ("<|im_start|>assistant\n", "<|im_end|>\n"), + }, + stop_str=["<|im_end|>", "<|action_end|>"], + ) +) + +register_chat_template( + ChatTemplate( + name="interns1", + default_system_prompt="You are an AI assistant whose name is Intern-S1 (书生大模型).\n- Intern-S1 (书生大模型) is a vision-language model that is developed by Shanghai AI Laboratory (上海人工智能实验室). It is designed to be helpful, honest, and harmless.\n- Intern-S1 (书生大模型) can understand and communicate fluently in the language chosen by the user such as English and 中文.\nYou are an expert reasoner with extensive experience in all areas. You approach problems through systematic thinking and rigorous reasoning. Your response should reflect deep understanding and precise logical thinking, making your solution path and reasoning clear to others. Please put your thinking process within ... tags.", + role_prefix_and_suffix={ + "system": ("<|im_start|>system\n", "<|im_end|>\n"), + "user": ("<|im_start|>user\n", "<|im_end|>\n"), + "assistant": ("<|im_start|>assistant\n", "<|im_end|>\n"), + }, + stop_str=["<|im_end|>", "<|action_end|>"], + ) +) + +register_chat_template( + ChatTemplate( + name="granite-3-instruct", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ( + "<|start_of_role|>system<|end_of_role|>", + "<|end_of_text|>", + ), + "user": ( + "<|start_of_role|>user<|end_of_role|>", + "<|end_of_text|>", + ), + "assistant": ( + "<|start_of_role|>assistant<|end_of_role|>", + "<|end_of_text|>", + ), + }, + stop_str=("<|end_of_text|>",), + ) +) + +register_chat_template( + ChatTemplate( + name="deepseek-v3", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ( + "", + "", + ), + "user": ( + "<｜User｜>", + "", + ), + "assistant": ( + "<｜Assistant｜>", + "<｜end▁of▁sentence｜>", + ), + }, + stop_str=("<｜end▁of▁sentence｜>",), + ) +) + +# Reference: https://huggingface.co/docs/transformers/main/model_doc/glm4_v#usage-example +register_chat_template( + ChatTemplate( + name="glm-4v", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ("<|system|>\n", "\n"), + "user": ("<|user|>\n", "\n"), + "assistant": ("<|assistant|>\n", "\n"), + }, + style=ChatTemplateStyle.PLAIN, + stop_str=["<|user|>", "<|endoftext|>", "<|observation|>"], + image_token="<|image|>", + ) +) + + +@register_chat_template_matching_function +def match_deepseek(model_path: str): + if re.search(r"deepseek-(v3|r1)", model_path, re.IGNORECASE) and not re.search( + r"base", model_path, re.IGNORECASE + ): + return "deepseek-v3" + + +@register_chat_template_matching_function +def match_orion(model_path: str): + if "orion" in model_path.lower(): + return "claude" + + +@register_chat_template_matching_function +def match_deepseek_janus_pro(model_path: str): + if re.search(r"janus", model_path, re.IGNORECASE): + return "janus-pro" + + +@register_chat_template_matching_function +def match_dbrx(model_path: str): + if re.search(r"dbrx", model_path, re.IGNORECASE) and re.search( + r"instruct", model_path, re.IGNORECASE + ): + return "dbrx-instruct" + + +@register_chat_template_matching_function +def match_vicuna(model_path: str): + if re.search(r"vicuna|llava-v1\.5|llava-next-video-7b", model_path, re.IGNORECASE): + return "vicuna_v1.1" + + +@register_chat_template_matching_function +def match_llama2_chat(model_path: str): + if re.search( + r"llama-2.*chat|codellama.*instruct", + model_path, + re.IGNORECASE, + ): + return "llama-2-chat" + + +@register_chat_template_matching_function +def match_mistral(model_path: str): + if re.search(r"pixtral|(mistral|mixtral).*instruct", model_path, re.IGNORECASE): + return "mistral" + + +@register_chat_template_matching_function +def match_llama3_instruct(model_path: str): + if re.search(r"llama-3.*instruct", model_path, re.IGNORECASE): + return "llama-3-instruct" + + +@register_chat_template_matching_function +def match_chat_ml(model_path: str): + if re.search(r"tinyllama", model_path, re.IGNORECASE): + return "chatml" + if re.search(r"qwen.*vl", model_path, re.IGNORECASE): + return "qwen2-vl" + if re.search(r"glm[-_]?4(\.\d+)?v", model_path, re.IGNORECASE): + return "glm-4v" + if re.search(r"qwen.*(chat|instruct)", model_path, re.IGNORECASE) and not re.search( + r"llava", model_path, re.IGNORECASE + ): + return "qwen" + if re.search( + r"llava-v1\.6-34b|llava-v1\.6-yi-34b|llava-next-video-34b|llava-onevision-qwen2", + model_path, + re.IGNORECASE, + ): + return "chatml-llava" + + +@register_chat_template_matching_function +def match_chat_yi(model_path: str): + if re.search(r"yi-vl", model_path, re.IGNORECASE) and not re.search( + r"llava", model_path, re.IGNORECASE + ): + return "yi-vl" + elif re.search(r"yi-1\.5.*chat", model_path, re.IGNORECASE): + return "yi-1.5" + + +@register_chat_template_matching_function +def match_gemma_it(model_path: str): + if re.search(r"gemma.*it", model_path, re.IGNORECASE): + return "gemma-it" + + +@register_chat_template_matching_function +def match_openbmb_minicpm(model_path: str): + if re.search(r"minicpm-v", model_path, re.IGNORECASE): + return "minicpmv" + elif re.search(r"minicpm-o", model_path, re.IGNORECASE): + return "minicpmo" + + +@register_chat_template_matching_function +def match_c4ai_command_r(model_path: str): + if re.search(r"c4ai-command-r", model_path, re.IGNORECASE): + return "c4ai-command-r" + + +@register_chat_template_matching_function +def match_granite_instruct(model_path: str): + if re.search(r"granite.*instruct", model_path, re.IGNORECASE): + return "granite-3-instruct" + + +@register_chat_template_matching_function +def match_gemma3_instruct(model_path: str): + if re.search(r"gemma-3", model_path, re.IGNORECASE): + return "gemma-it" + + +@register_chat_template_matching_function +def match_internvl_chat(model_path: str): + if re.search(r"internvl2_5", model_path, re.IGNORECASE): + return "internvl-2-5" + + +@register_chat_template_matching_function +def match_interns1_chat(model_path: str): + if re.search(r"intern-s1", model_path, re.IGNORECASE): + return "interns1" + if re.search(r"interns1", model_path, re.IGNORECASE): + return "interns1" + + +if __name__ == "__main__": + messages = [ + {"role": "system", "content": None}, # None means default + # {"role": "system", "content": "You are a helpful, respectful and honest assistant."}, + {"role": "user", "content": "Hello!"}, + {"role": "assistant", "content": "Hi!"}, + {"role": "user", "content": "What can you do?"}, + {"role": "assistant", "content": "I can chat with you."}, + ] + + template = get_chat_template("llama-2-chat") + print(template.get_prompt(messages)) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/choices.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/choices.py new file mode 100644 index 00000000..e52c6b36 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/choices.py @@ -0,0 +1,164 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +import numpy as np + + +@dataclass +class ChoicesDecision: + decision: str + meta_info: Optional[Dict[str, Any]] = None + + +class ChoicesSamplingMethod(ABC): + + @property + def requires_unconditional_logprobs(self) -> bool: + return False + + @abstractmethod + def __call__( + self, + *, + choices: List[str], + normalized_prompt_logprobs: List[float], + input_token_logprobs: List[List[Any]], + output_token_logprobs: List[List[Any]], + unconditional_token_logprobs: Optional[List[List[Any]]] = None, + ) -> ChoicesDecision: ... + + +class TokenLengthNormalized(ChoicesSamplingMethod): + + def __call__( + self, + *, + choices: List[str], + normalized_prompt_logprobs: List[float], + input_token_logprobs: List[List[Any]], + output_token_logprobs: List[List[Any]], + unconditional_token_logprobs: Optional[List[List[Any]]] = None, + ) -> ChoicesDecision: + """Select the option with the highest token length normalized prompt logprob.""" + best_choice = choices[np.argmax(normalized_prompt_logprobs)] + meta_info = { + "normalized_prompt_logprobs": normalized_prompt_logprobs, + "input_token_logprobs": input_token_logprobs, + "output_token_logprobs": output_token_logprobs, + } + return ChoicesDecision(decision=best_choice, meta_info=meta_info) + + +token_length_normalized = TokenLengthNormalized() + + +class GreedyTokenSelection(ChoicesSamplingMethod): + + def __call__( + self, + *, + choices: List[str], + normalized_prompt_logprobs: List[float], + input_token_logprobs: List[List[Any]], + output_token_logprobs: List[List[Any]], + unconditional_token_logprobs: Optional[List[List[Any]]] = None, + ) -> ChoicesDecision: + """Select the option based on greedy logprob selection. For overlapping options + where one option is a subset of a longer option, extend the shorter option using + its average logprob for comparison against the longer option.""" + + num_options = len(choices) + max_tokens = max(len(option) for option in input_token_logprobs) + logprob_matrix = self._build_logprob_matrix( + input_token_logprobs, max_tokens, num_options + ) + remaining = self._greedy_selection(logprob_matrix, num_options, max_tokens) + + best_choice = choices[remaining[0]] + meta_info = { + "normalized_prompt_logprobs": normalized_prompt_logprobs, + "input_token_logprobs": input_token_logprobs, + "output_token_logprobs": output_token_logprobs, + "greedy_logprob_matrix": logprob_matrix.tolist(), + } + return ChoicesDecision(decision=best_choice, meta_info=meta_info) + + def _build_logprob_matrix(self, input_token_logprobs, max_tokens, num_options): + logprob_matrix = np.zeros((num_options, max_tokens)) + for i, option in enumerate(input_token_logprobs): + actual_logprobs = [token[0] for token in option] + avg_logprob = np.mean(actual_logprobs) + logprob_matrix[i, : len(option)] = actual_logprobs + if len(option) < max_tokens: + logprob_matrix[i, len(option) :] = avg_logprob + return logprob_matrix + + def _greedy_selection(self, logprob_matrix, num_options, max_tokens): + remaining = np.arange(num_options) + for j in range(max_tokens): + max_logprob = np.max(logprob_matrix[remaining, j]) + remaining = remaining[logprob_matrix[remaining, j] == max_logprob] + if len(remaining) == 1: + break + return remaining + + +greedy_token_selection = GreedyTokenSelection() + + +class UnconditionalLikelihoodNormalized(ChoicesSamplingMethod): + + @property + def requires_unconditional_logprobs(self) -> bool: + return True + + def __call__( + self, + *, + choices: List[str], + normalized_prompt_logprobs: List[float], + input_token_logprobs: List[List[Any]], + output_token_logprobs: List[List[Any]], + unconditional_token_logprobs: Optional[List[List[Any]]] = None, + ) -> ChoicesDecision: + """Select the option with the highest average token logprob once normalized by + the unconditional token logprobs. + + The first unconditional token logprob is assumed to be None. If so, it is + replaced with 0 for the purposes of normalization.""" + + if unconditional_token_logprobs is None: + raise ValueError( + "Unconditional token logprobs are required for this method." + ) + + normalized_unconditional_prompt_logprobs = self._normalize_logprobs( + input_token_logprobs, unconditional_token_logprobs + ) + + best_choice = choices[np.argmax(normalized_unconditional_prompt_logprobs)] + meta_info = { + "normalized_prompt_logprobs": normalized_prompt_logprobs, + "input_token_logprobs": input_token_logprobs, + "output_token_logprobs": output_token_logprobs, + "unconditional_token_logprobs": unconditional_token_logprobs, + "normalized_unconditional_prompt_logprobs": normalized_unconditional_prompt_logprobs, + } + return ChoicesDecision(decision=best_choice, meta_info=meta_info) + + def _normalize_logprobs(self, input_token_logprobs, unconditional_token_logprobs): + normalized_unconditional_prompt_logprobs = [] + for inputs, unconditionals in zip( + input_token_logprobs, unconditional_token_logprobs + ): + inputs_logprobs = np.array([token[0] for token in inputs]) + unconditionals_logprobs = np.array([token[0] for token in unconditionals]) + unconditionals_logprobs[0] = unconditionals_logprobs[0] or 0 + normalized_unconditional_prompt_logprobs.append( + float(np.mean(inputs_logprobs - unconditionals_logprobs)) + ) + return normalized_unconditional_prompt_logprobs + + +unconditional_likelihood_normalized = UnconditionalLikelihoodNormalized() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/interpreter.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/interpreter.py new file mode 100644 index 00000000..0b59e91b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/interpreter.py @@ -0,0 +1,1061 @@ +"""The interpreter that executes SGL programs""" + +import asyncio +import contextvars +import copy +import multiprocessing +import queue +import threading +import uuid +import warnings +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from typing import Any, Callable, Dict, List, Optional + +import tqdm + +from sglang.global_config import global_config +from sglang.lang.ir import ( + SglCommitLazy, + SglConcateAndAppend, + SglConstantText, + SglExpr, + SglExprList, + SglGen, + SglImage, + SglRoleBegin, + SglRoleEnd, + SglSelect, + SglSeparateReasoning, + SglVariable, + SglVarScopeBegin, + SglVarScopeEnd, + SglVideo, +) +from sglang.utils import ( + encode_image_base64, + encode_video_base64, + get_exception_traceback, +) + + +def run_internal(state, program, func_args, func_kwargs, sync): + try: + state.ret_value = program.func(state, *func_args, **func_kwargs) + except Exception as e: + raise e + finally: + state.stream_executor.end() + + if sync: + state.stream_executor.sync() + + if global_config.verbosity >= 2: + print(state.text()) + + +def run_program( + program, + backend, + func_args, + func_kwargs, + default_sampling_para, + stream, + sync=False, + use_thread=True, +): + if hasattr(backend, "endpoint"): + backend = backend.endpoint + assert backend is not None, "Please specify a backend" + func_kwargs.update(program.bind_arguments) + stream_executor = StreamExecutor( + backend, + func_kwargs, + default_sampling_para, + chat_template=None, + stream=stream, + num_api_spec_tokens=program.num_api_spec_tokens, + use_thread=use_thread, + ) + state = ProgramState(stream_executor) + + if stream: + t = threading.Thread( + target=run_internal, args=(state, program, func_args, func_kwargs, sync) + ) + t.start() + return state + else: + run_internal(state, program, func_args, func_kwargs, sync) + return state + + +def run_program_batch( + program, + backend, + batch_arguments, + default_sampling_para, + num_threads, + progress_bar, + generator_style=False, +): + if hasattr(backend, "endpoint"): + backend = backend.endpoint + + # Pre-cache the common prefix for a batch. The prefix is extracted by tracing the program. + if global_config.enable_precache_with_tracing and len(batch_arguments) > 1: + cache_program(program, backend) + + # Run all programs + if num_threads == "auto": + num_threads = max(96, multiprocessing.cpu_count() * 16) + num_threads = min(num_threads, len(batch_arguments)) + + if generator_style: + return _run_program_batch_generator( + program, + backend, + batch_arguments, + default_sampling_para, + num_threads, + progress_bar, + ) + + # Original code path when generator_style=False + if num_threads == 1: + rets = [] + if progress_bar: + for arguments in tqdm.tqdm(batch_arguments): + rets.append( + run_program( + program, + backend, + (), + arguments, + default_sampling_para, + False, + True, + ) + ) + else: + for arguments in batch_arguments: + rets.append( + run_program( + program, + backend, + (), + arguments, + default_sampling_para, + False, + True, + ) + ) + else: + if progress_bar: + pbar = tqdm.tqdm(total=len(batch_arguments)) + + with ThreadPoolExecutor(num_threads) as executor: + futures = [] + for arguments in batch_arguments: + futures.append( + executor.submit( + run_program, + program, + backend, + (), + arguments, + default_sampling_para, + False, + True, + ) + ) + if progress_bar: + futures[-1].add_done_callback(lambda _: pbar.update()) + + rets = [f.result() for f in futures] + rets[-1].sync() + + if progress_bar: + pbar.close() + + return rets + + +def _run_program_batch_generator( + program, + backend, + batch_arguments, + default_sampling_para, + num_threads, + progress_bar, +): + """Helper function that yields results one by one using chunking to avoid overwhelming ThreadPoolExecutor.""" + if num_threads == 1: + iterator = tqdm.tqdm(batch_arguments) if progress_bar else batch_arguments + for arguments in iterator: + yield run_program( + program, + backend, + (), + arguments, + default_sampling_para, + False, + True, + ) + else: + pbar = tqdm.tqdm(total=len(batch_arguments)) if progress_bar else None + + # Process in chunks to avoid overwhelming ThreadPoolExecutor + # Otherwise, ThreadPoolExecutor.submit will block after adding certain number of tasks + # so we will never reach "yield" until all tasks are done + chunk_size = 200 + + with ThreadPoolExecutor(num_threads) as executor: + for chunk_start in range(0, len(batch_arguments), chunk_size): + chunk_end = min(chunk_start + chunk_size, len(batch_arguments)) + chunk_futures = [] + + # Submit chunk of tasks + for i in range(chunk_start, chunk_end): + future = executor.submit( + run_program, + program, + backend, + (), + batch_arguments[i], + default_sampling_para, + False, + True, + ) + if pbar: + future.add_done_callback(lambda _: pbar.update()) + chunk_futures.append(future) + + # Yield results from this chunk as they complete + for future in chunk_futures: + yield future.result() + + if pbar: + pbar.close() + + +def cache_program(program, backend): + from sglang.lang.tracer import extract_prefix_by_tracing + + prefix = extract_prefix_by_tracing(program, backend) + if prefix and len(prefix) > 64: + backend.cache_prefix(prefix) + + +class StreamExecutor: + """A stream executor that executes SGL expressions in a background thread.""" + + def __init__( + self, + backend, + arguments, + default_sampling_para, + chat_template, + stream, + num_api_spec_tokens=None, + use_thread=True, + ): + from sglang.lang.backend.base_backend import BaseBackend + + self.sid = uuid.uuid4().hex + self.backend: BaseBackend = backend + self.arguments: Dict[str, Any] = arguments + self.default_sampling_para = default_sampling_para + self.stream = stream + + self.variables = {} # Dict[name: str -> value: str] + self.variable_event = {} # Dict[name: str -> event: threading.Event] + self.meta_info = {} # Dict[name: str -> info: str] + self.is_finished = False + self.error_ = None + + # For completion + self.text_ = "" # The full text + + # For chat + self.messages_ = [] # The messages in the OpenAI API format + self.chat_template = chat_template or self.backend.get_chat_template() + self.cur_role = None + self.cur_role_begin_pos = None + + # For vision + self.images_ = [] + self.cur_images = [] + + # For fork/join + self.fork_start_text_pos = None + + # For speculative execution + self.num_api_spec_tokens = num_api_spec_tokens + self.speculated_text = "" + + # Worker thread + self.use_thread = use_thread + if self.use_thread: + self.queue = queue.Queue() + + def _run_worker_in_context(): + self._thread_worker_func() + + self.worker = threading.Thread( + target=contextvars.copy_context().run, args=(_run_worker_in_context,) + ) + self.worker.start() + + # For streaming + if stream: + self.stream_text_event = threading.Event() + self.stream_var_event = {} + else: + self.stream_text_event = None + self.stream_var_event = None + + def submit(self, expr: SglExpr): + self._init_var_event(expr) + + if self.use_thread: + self.queue.put(expr) + else: + self._execute(expr) + + def sync(self): + if self.use_thread: + self.queue.join() + + def get_var(self, name): + if name in self.variable_event: + self.variable_event[name].wait() + return self.variables[name] + + def set_var(self, name, value): + self.variables[name] = value + + def get_meta_info(self, name, timeout=None): + if name in self.variable_event: + got = self.variable_event[name].wait(timeout) + if not got: + raise TimeoutError(f"Timeout while waiting for event '{name}'") + ret = self.meta_info.get(name, None) + return ret + + def fork( + self, + size: int = 1, + position_ids_offset: Optional[List[int]] = None, + ): + if size > 1 and str(self.text_): + self.submit(SglCommitLazy()) + + self.sync() + size = int(size) + + exes = [ + StreamExecutor( + self.backend, + self.arguments, + self.default_sampling_para, + self.chat_template, + self.stream, + ) + for _ in range(size) + ] + for i in range(size): + exes[i].variables = dict(self.variables) + exes[i].text_ = str(self.text_) + exes[i].messages_ = list(self.messages_) + exes[i].cur_role = self.cur_role + exes[i].cur_role_begin_pos = self.cur_role_begin_pos + exes[i].fork_start_text_pos = len(self.text_) + exes[i].images_ = list(self.images_) + + # TODO(ying): handle API speculative execution + + return exes + + def text(self): + self.sync() + return self.text_ + + def messages(self): + self.sync() + return self.messages_ + + def error(self): + self.sync() + return self.error_ + + def end(self): + if self.use_thread: + if self.worker.is_alive(): + self.queue.put(None) + self.backend.end_program(self) + + def _thread_worker_func(self): + error = None + + while True: + expr = self.queue.get() + if expr is None: + self.queue.task_done() + break + + try: + self._execute(expr) + except Exception as e: + warnings.warn(f"Error in stream_executor: {get_exception_traceback()}") + error = e + break + self.queue.task_done() + if self.stream_text_event: + self.stream_text_event.set() + + # Clean the queue and events + if error is not None: + try: + while True: + self.queue.task_done() + self.queue.get_nowait() + except queue.Empty: + pass + for name in self.variable_event: + self.variable_event[name].set() + if self.stream_var_event: + for name in self.stream_var_event: + self.stream_var_event[name].set() + self.error_ = error + + if self.stream_text_event: + self.stream_text_event.set() + + self.is_finished = True + + def _execute(self, other): + if isinstance(other, str): + other = SglConstantText(other) + + assert isinstance(other, SglExpr), f"{other}" + + if isinstance(other, SglConstantText): + self._execute_fill(other.value) + elif isinstance(other, SglGen): + self._execute_gen(other) + elif isinstance(other, SglSelect): + self._execute_select(other) + elif isinstance(other, SglExprList): + for x in other.expr_list: + self._execute(x) + elif isinstance(other, SglRoleBegin): + self._execute_role_begin(other) + elif isinstance(other, SglRoleEnd): + self._execute_role_end(other) + elif isinstance(other, SglImage): + self._execute_image(other) + elif isinstance(other, SglVideo): + self._execute_video(other) + elif isinstance(other, SglVariable): + self._execute_variable(other) + elif isinstance(other, SglVarScopeBegin): + self._execute_var_scope_begin(other) + elif isinstance(other, SglVarScopeEnd): + self._execute_var_scope_end(other) + elif isinstance(other, SglCommitLazy): + self._execute_commit_lazy_operations(other) + elif isinstance(other, SglConcateAndAppend): + if ( + global_config.enable_parallel_encoding + and self.backend.support_concate_and_append + ): + self._execute_concatenate_and_append_kv_cache(other) + else: + self._execute_concatenate_and_append_text(other) + elif isinstance(other, SglSeparateReasoning): + self._execute_separate_reasoning(other) + else: + raise ValueError(f"Unknown type: {type(other)}") + + def _execute_fill(self, value: str, prefix=False): + value = str(value) + + if ( + self.cur_role == "assistant" + and self.num_api_spec_tokens is not None + and self.backend.is_chat_model + and not prefix + ): + self.backend.spec_fill(value) + return + + if self.speculated_text.startswith(value): + self.speculated_text = self.speculated_text[len(value) :] + else: + self.speculated_text = "" + + self.text_ += value + + def _execute_image(self, expr: SglImage): + path = expr.path + + base64_data = encode_image_base64(path) + + self.images_.append((path, base64_data)) + self.cur_images.append((path, base64_data)) + self.text_ += self.chat_template.image_token + + def _execute_video(self, expr: SglVideo): + path = expr.path + num_frames = expr.num_frames + + base64_data = encode_video_base64(path, num_frames) + + self.images_.append((path, base64_data)) + self.cur_images.append((path, base64_data)) + self.text_ += self.chat_template.image_token + + def _spec_gen(self, sampling_params): + stop = sampling_params.stop + max_new_tokens = sampling_params.max_new_tokens + meta_info = {} + + def regen(): + nonlocal meta_info + + sampling_params.max_new_tokens = max( + sampling_params.max_new_tokens, self.num_api_spec_tokens + ) + sampling_params.stop = None + self.speculated_text, meta_info = self.backend.generate( + self, sampling_params=sampling_params + ) + + def find_stop(): + if isinstance(stop, str): + return self.speculated_text.find(stop) + elif isinstance(stop, (tuple, list)): + pos = -1 + for stop_str in stop: + stop_pos = self.speculated_text.find(stop_str) + if stop_pos != -1 and (pos == -1 or stop_pos < pos): + pos = stop_pos + return pos + else: + raise Exception("Wrong type of stop in sampling parameters.") + + if stop is None: + if len(self.speculated_text) < max_new_tokens: + regen() + comp = self.speculated_text[:max_new_tokens] + self.speculated_text = self.speculated_text[max_new_tokens:] + elif isinstance(stop, (str, list, tuple)): + if self.speculated_text == "": + regen() + stop_pos = find_stop() + if stop_pos == -1: + stop_pos = min( + sampling_params.max_new_tokens, + len(self.speculated_text), + ) + comp = self.speculated_text[:stop_pos] + self.speculated_text = self.speculated_text[stop_pos:] + else: + raise ValueError("Wrong type of stop in sampling parameters.") + + return comp, meta_info + + def _execute_gen(self, expr: SglGen): + sampling_params = self._resolve_sampling_params(expr.sampling_params) + name = expr.name + if not self.stream: + if self.num_api_spec_tokens is None: + comp, meta_info = self.backend.generate( + self, + sampling_params=sampling_params, + ) + + else: + if self.backend.is_chat_model: + # Speculative execution on models with only chat interface. + # Store the calls into a temporary list. + # They will be lazily executed later. + comp, meta_info = self.backend.generate( + self, + sampling_params=sampling_params, + spec_var_name=name, + ) + return + + else: # Speculative execution on models with completion interface + comp, meta_info = self._spec_gen(sampling_params) + if isinstance(comp, list): + self.text_ += comp[0] + else: + assert isinstance(comp, str) + self.text_ += comp + + self.variables[name] = comp + self.meta_info[name] = meta_info + self.variable_event[name].set() + else: + assert ( + self.num_api_spec_tokens is None + ), "stream is not supported with api speculative execution" + generator = self.backend.generate_stream( + self, sampling_params=sampling_params + ) + + self.variables[name] = "" + self.stream_var_event[name].set() + + for comp, meta_info in generator: + self.text_ += comp + self.variables[name] += comp + self.meta_info[name] = meta_info + self.stream_var_event[name].set() + self.stream_text_event.set() + + self.variable_event[name].set() + self.stream_var_event[name].set() + + def _execute_select(self, expr: SglSelect): + choices_decision = self.backend.select( + self, expr.choices, expr.temperature, expr.choices_method + ) + if expr.name is not None: + name = expr.name + self.variables[name] = choices_decision.decision + self.meta_info[name] = choices_decision.meta_info + self.variable_event[name].set() + if self.stream_var_event: + self.stream_var_event[name].set() + self.text_ += choices_decision.decision + + def _execute_variable(self, expr: SglVariable): + src_executor = expr.source_stream_executor + value = src_executor.get_var(expr.name) + self._execute_fill(value) + + def _execute_role_begin(self, expr: SglRoleBegin): + assert self.cur_role is None, "Nested roles are not allowed." + + if len(self.messages_) == 0 and expr.role != "system": + # Insert the default system message + default_system = self.chat_template.default_system_prompt + if default_system: + self._execute_role_begin(SglRoleBegin("system")) + self._execute_fill(default_system) + self._execute_role_end(SglRoleEnd("system")) + + self.cur_role = expr.role + + prefix, _ = self.chat_template.get_prefix_and_suffix(expr.role, self.messages_) + + self._execute_fill(prefix, prefix=True) + self.cur_role_begin_pos = len(self.text_) + + def _execute_role_end(self, expr: SglRoleEnd): + if ( + self.cur_role == "assistant" + and self.num_api_spec_tokens is not None + and self.backend.is_chat_model + ): + # Execute the stored lazy generation calls + self.backend.role_end_generate(self) + self.cur_role = None + + new_text = self.text_[self.cur_role_begin_pos :].lstrip() + + _, suffix = self.chat_template.get_prefix_and_suffix(expr.role, self.messages_) + self._execute_fill(suffix) + + if self.cur_images: + # OpenAI vision API format + last_msg = { + "role": expr.role, + "content": [{"type": "text", "text": new_text}], + } + for image_path, image_base64_data in self.cur_images: + last_msg["content"].append( + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{image_base64_data}" + }, + } + ) + self.messages_.append(last_msg) + self.cur_images = [] + else: + # OpenAI chat API format + self.messages_.append({"role": expr.role, "content": new_text}) + + def _execute_var_scope_begin(self, expr: SglVarScopeBegin): + self.variables[expr.name] = int(len(self.text_)) + + def _execute_var_scope_end(self, expr: SglVarScopeEnd): + self.variables[expr.name] = self.text_[self.variables[expr.name] :] + self.variable_event[expr.name].set() + + def _execute_commit_lazy_operations(self, expr: SglCommitLazy): + self.backend.commit_lazy_operations(self) + + def _execute_concatenate_and_append_text(self, expr: SglConcateAndAppend): + new_text = "" + for s in expr.states: + exe = s.stream_executor + exe.sync() + new_text += exe.text_[exe.fork_start_text_pos :] + + self._execute_fill(new_text) + + def _execute_concatenate_and_append_kv_cache(self, expr: SglConcateAndAppend): + self_len = len(self.text_) + + for i, s in enumerate(expr.states): + exe = s.stream_executor + exe.submit(SglCommitLazy()) + + for i, s in enumerate(expr.states): + exe = s.stream_executor + exe.sync() + assert exe.fork_start_text_pos == self_len + self.text_ += exe.text_[exe.fork_start_text_pos :] + + src_rids = [state.stream_executor.sid for state in expr.states] + self.backend.concatenate_and_append(src_rids, self.sid) + + def _execute_separate_reasoning(self, expr: SglSeparateReasoning): + if self.stream: + # separate reasoning for stream is not supported + return + + if ( + self.cur_role == "assistant" + and self.num_api_spec_tokens is not None + and self.backend.is_chat_model + ): + # Execute the stored lazy generation calls + self.backend.role_end_generate(self) + + from sglang.srt.parser.reasoning_parser import ReasoningParser + + reasoning_parser = ReasoningParser(expr.model_type) + other = expr.expr + if not other: + return + elif isinstance(other, SglGen) or isinstance(other, SglSelect): + cur_text = self.get_var(other.name) + reasoning, normal_text = reasoning_parser.parse_non_stream(cur_text) + reasoning_name = expr.process_name_for_reasoning(other.name) + self.set_var(other.name, normal_text) + self.set_var(reasoning_name, reasoning) + # the variable is ready to be used + self.variable_event[reasoning_name].set() + self.text_ = self.text_[: self.cur_role_begin_pos] + normal_text + elif isinstance(other, SglExprList): + for x in other.expr_list: + self._execute_separate_reasoning( + SglSeparateReasoning(expr.model_type, x) + ) + + def _init_var_event(self, expr): + if isinstance( + expr, (SglGen, SglSelect, SglVarScopeBegin, SglSeparateReasoning) + ): + self.variable_event[expr.name] = threading.Event() + if self.stream: + self.stream_var_event[expr.name] = threading.Event() + elif isinstance(expr, SglExprList): + for e in expr.expr_list: + self._init_var_event(e) + + def _resolve_sampling_params(self, sampling_params): + """ + Construct sampling param based on default + override values + + The default values of sampling are populated in `default_sampling_para` via sgl.function.run(...sampling_args) + , and `sampling_params` contains the override values from sgl.gen(). + + Here we use default_sampling_para as the base and override the values if they exist in `sampling_params`. + It also extends the stop tokens based on the chat template. + """ + + # deepcopy is required because the dict has lists inside + clone = copy.deepcopy(self.default_sampling_para) + + for item in [ + "max_new_tokens", + "min_new_tokens", + "n", + "stop", + "stop_token_ids", + "stop_regex", + "temperature", + "top_p", + "top_k", + "min_p", + "frequency_penalty", + "presence_penalty", + "ignore_eos", + "return_logprob", + "logprob_start_len", + "top_logprobs_num", + "return_text_in_logprobs", + "dtype", + "regex", + "json_schema", + ]: + value = getattr(sampling_params, item, None) + if value is not None: + setattr(clone, item, value) + + if self.chat_template.stop_str: + if clone.stop == (): + clone.stop = [] + elif isinstance(clone.stop, str): + clone.stop = [clone.stop] + clone.stop += self.chat_template.stop_str + + return clone + + def __del__(self): + self.end() + + +class ProgramState: + """The state of an SGL program.""" + + def __init__(self, stream_executor: StreamExecutor): + self.stream_executor = stream_executor + + def _role_common(self, name: str, expr: Optional[SglExpr] = None): + if expr is not None: + role_expr = SglExprList([SglRoleBegin(name), expr, SglRoleEnd(name)]) + self.stream_executor.submit(role_expr) + return role_expr + else: + + @contextmanager + def role_scope(): + self.stream_executor.submit(SglRoleBegin(name)) + yield + self.stream_executor.submit(SglRoleEnd(name)) + + return role_scope() + + def system(self, expr: Optional[SglExpr] = None): + return self._role_common("system", expr) + + def user(self, expr: Optional[SglExpr] = None): + return self._role_common("user", expr) + + def assistant(self, expr: Optional[SglExpr] = None): + return self._role_common("assistant", expr) + + @contextmanager + def var_scope(self, name: str): + self.stream_executor.submit(SglVarScopeBegin(name)) + yield + self.stream_executor.submit(SglVarScopeEnd(name)) + + def fork( + self, + size: int = 1, + position_ids_offset: Optional[List[int]] = None, + ): + stream_executors = self.stream_executor.fork(size, position_ids_offset) + states = [ProgramState(x) for x in stream_executors] + state_group = ProgramStateGroup(states, self) + return state_group + + @contextmanager + def copy(self, position_ids_offset: Optional[List[int]] = None): + state_group = self.fork(1, position_ids_offset) + try: + yield state_group[0] + finally: + state_group.join() + + def text(self): + return self.stream_executor.text() + + def messages(self): + return self.stream_executor.messages() + + def sync(self): + return self.stream_executor.sync() + + def error(self): + return self.stream_executor.error() + + def text_iter(self, var_name: Optional[str] = None): + if self.stream_executor.stream: + prev = 0 + if var_name is None: + event = self.stream_executor.stream_text_event + while True: + event.wait() + event.clear() + out = str(self.stream_executor.text_[prev:]) + prev += len(out) + if out: + yield out + if self.stream_executor.is_finished: + break + else: + event = None + while not event: + if var_name in self.stream_executor.stream_var_event: + event = self.stream_executor.stream_var_event[var_name] + if self.stream_executor.is_finished: + yield "" + return + + while True: + event.wait() + event.clear() + out = str(self.stream_executor.variables[var_name][prev:]) + prev += len(out) + if out: + yield out + if self.stream_executor.variable_event[var_name].is_set(): + break + else: + if var_name is None: + yield self.text() + else: + yield self.get_var(var_name) + + async def text_async_iter( + self, var_name: Optional[str] = None, return_meta_data: bool = False + ): + loop = asyncio.get_running_loop() + + if self.stream_executor.stream: + prev = 0 + if var_name is None: + event = self.stream_executor.stream_text_event + while True: + await loop.run_in_executor(None, event.wait) + event.clear() + out = str(self.stream_executor.text_[prev:]) + prev += len(out) + if out: + yield out + if self.stream_executor.is_finished: + break + else: + event = None + while not event: + if var_name in self.stream_executor.stream_var_event: + event = self.stream_executor.stream_var_event[var_name] + if self.stream_executor.is_finished: + yield "" + return + + while True: + await loop.run_in_executor(None, event.wait) + event.clear() + out = str(self.stream_executor.variables[var_name][prev:]) + prev += len(out) + if out: + if return_meta_data: + yield out, self.stream_executor.meta_info[var_name] + else: + yield out + if self.stream_executor.variable_event[var_name].is_set(): + break + else: + if var_name is None: + yield self.text() + else: + yield self.get_var(var_name) + + def get_var(self, name): + return self.stream_executor.get_var(name) + + def set_var(self, name, value): + return self.stream_executor.set_var(name, value) + + def get_meta_info(self, name): + return self.stream_executor.get_meta_info(name) + + def __iadd__(self, other): + if other is None: + raise ValueError("Tried to append None to state.") + self.stream_executor.submit(other) + return self + + def __getitem__(self, name): + return self.get_var(name) + + def __setitem__(self, name, value): + self.set_var(name, value) + + def __contains__(self, name): + return name in self.stream_executor.variables + + def __del__(self): + self.stream_executor.end() + + def __repr__(self) -> str: + return f"ProgramState({self.text()})" + + +class ProgramStateGroup: + def __init__( + self, states: List[ProgramState], src_state: Optional[ProgramState] = None + ): + self.states = states + self.src_state = src_state + + def join(self, mode: str = "gather_variable"): + if mode == "gather_variable": + # Copy variables back + src_vars = self.src_state.stream_executor.variables + src_var_set = set(src_vars.keys()) + for child_state in self.states: + child_state.stream_executor.sync() + child_vars = child_state.stream_executor.variables + new_vars = set(child_vars.keys()) - src_var_set + + for k in new_vars: + if k in src_vars: + src_vars[k].append(child_vars[k]) + else: + src_vars[k] = [child_vars[k]] + elif mode == "concate_and_append": + # Concatenate and append KV cache + self.src_state += SglConcateAndAppend(self.states) + # Need a sync here. Otherwise, `states` can be deleted. + self.src_state.stream_executor.sync() + else: + raise ValueError(f"Invalid join mode: {mode}") + + for s in self.states: + s.stream_executor.end() + + def __getitem__(self, i: int): + return self.states[i] + + def __setitem__(self, i: int, value): + assert self.states[i] == value + + def __iadd__(self, other): + if isinstance(other, Callable): + # lambda function + for i in range(len(self.states)): + self.states[i] += other(i) + elif isinstance(other, SglExpr): + for i in range(len(self.states)): + self.states[i] += other + elif isinstance(other, (list, tuple)): + for i in range(len(self.states)): + self.states[i] += other[i] + else: + raise ValueError(f"Invalid value: {other}") + + return self diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/ir.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/ir.py new file mode 100644 index 00000000..43da723b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/ir.py @@ -0,0 +1,643 @@ +"""The intermediate representation.""" + +import dataclasses +import inspect +import warnings +from typing import List, Optional, Union + +from sglang.global_config import global_config +from sglang.lang.choices import ChoicesSamplingMethod + +REGEX_INT = r"[-+]?[0-9]+[ \n]*" +REGEX_FLOAT = r"[-+]?[0-9]*\.?[0-9]+[ \n]*" +REGEX_BOOL = r"(True|False)" +REGEX_STR = r"\"[\w\d\s]*\"" # bugs with regex r"\".*\"" in interegular pkg + + +@dataclasses.dataclass +class SglSamplingParams: + max_new_tokens: int = 128 + min_new_tokens: int = 0 + n: int = 1 + stop: Union[str, List[str]] = () + stop_token_ids: Optional[List[int]] = () + stop_regex: Optional[Union[str, List[str]]] = () + temperature: float = 1.0 + top_p: float = 1.0 + top_k: int = -1 # -1 means disable + min_p: float = 0.0 + frequency_penalty: float = 0.0 + presence_penalty: float = 0.0 + ignore_eos: bool = False + return_logprob: Optional[bool] = None + logprob_start_len: Optional[int] = (None,) + top_logprobs_num: Optional[int] = (None,) + return_text_in_logprobs: Optional[bool] = (None,) + json_schema: Optional[str] = None + + # for constrained generation, not included in to_xxx_kwargs + dtype: Optional[str] = None + regex: Optional[str] = None + + def clone(self): + return SglSamplingParams( + self.max_new_tokens, + self.min_new_tokens, + self.n, + self.stop, + self.stop_token_ids, + self.stop_regex, + self.temperature, + self.top_p, + self.top_k, + self.min_p, + self.frequency_penalty, + self.presence_penalty, + self.ignore_eos, + self.return_logprob, + self.logprob_start_len, + self.top_logprobs_num, + self.return_text_in_logprobs, + self.json_schema, + ) + + def to_openai_kwargs(self): + # OpenAI does not support top_k, so we drop it here + if self.regex is not None: + warnings.warn("Regular expression is not supported in the OpenAI backend.") + return { + "max_tokens": self.max_new_tokens, + "max_completion_tokens": self.max_new_tokens, + "n": self.n, + "stop": self.stop or None, + "temperature": self.temperature, + "top_p": self.top_p, + "frequency_penalty": self.frequency_penalty, + "presence_penalty": self.presence_penalty, + } + + def to_vertexai_kwargs(self): + if self.regex is not None: + warnings.warn( + "Regular expression is not supported in the VertexAI backend." + ) + return { + "candidate_count": 1, + "max_output_tokens": self.max_new_tokens, + "stop_sequences": self.stop, + "temperature": self.temperature, + "top_p": self.top_p, + "top_k": self.top_k if self.top_k > 0 else None, + } + + def to_anthropic_kwargs(self): + # Anthropic does not support frequency_penalty or presence_penalty, so we drop it here + if self.regex is not None: + warnings.warn( + "Regular expression is not supported in the Anthropic backend." + ) + return { + "max_tokens": self.max_new_tokens, + "stop_sequences": ( + self.stop if isinstance(self.stop, (list, tuple)) else [self.stop] + ), + "temperature": self.temperature, + "top_p": self.top_p, + "top_k": self.top_k, + } + + def to_litellm_kwargs(self): + if self.regex is not None: + warnings.warn("Regular expression is not supported in the LiteLLM backend.") + return { + "max_tokens": self.max_new_tokens, + "stop": self.stop or None, + "temperature": self.temperature, + "top_p": self.top_p, + "frequency_penalty": self.frequency_penalty, + "presence_penalty": self.presence_penalty, + } + + def to_srt_kwargs(self): + return { + "max_new_tokens": self.max_new_tokens, + "min_new_tokens": self.min_new_tokens, + "n": self.n, + "stop": self.stop, + "stop_token_ids": self.stop_token_ids, + "stop_regex": self.stop_regex, + "temperature": self.temperature, + "top_p": self.top_p, + "top_k": self.top_k, + "min_p": self.min_p, + "frequency_penalty": self.frequency_penalty, + "presence_penalty": self.presence_penalty, + "ignore_eos": self.ignore_eos, + "regex": self.regex, + "json_schema": self.json_schema, + } + + +class SglFunction: + def __init__(self, func, num_api_spec_tokens=None, bind_arguments=None): + self.func = func + self.num_api_spec_tokens = num_api_spec_tokens + self.bind_arguments = bind_arguments or {} + self.pin_prefix_rid = None + + # Parse arguments + argspec = inspect.getfullargspec(func) + assert argspec.args[0] == "s", 'The first argument must be "s"' + self.arg_names = argspec.args[1:] + self.arg_defaults = argspec.defaults if argspec.defaults is not None else [] + + def bind(self, **kwargs): + assert all(key in self.arg_names for key in kwargs) + + new_bind_dict = {**self.bind_arguments, **kwargs} + return SglFunction(self.func, bind_arguments=new_bind_dict) + + def run( + self, + *args, + max_new_tokens: int = 128, + n: int = 1, + stop: Optional[Union[str, List[str]]] = None, + stop_token_ids: Optional[List[int]] = None, + stop_regex: Optional[Union[str, List[str]]] = None, + temperature: float = 1.0, + top_p: float = 1.0, + top_k: int = -1, + min_p: float = 0.0, + frequency_penalty: float = 0.0, + presence_penalty: float = 0.0, + ignore_eos: bool = False, + return_logprob: Optional[bool] = None, + logprob_start_len: Optional[int] = None, + top_logprobs_num: Optional[int] = None, + return_text_in_logprobs: Optional[bool] = None, + stream: bool = False, + backend=None, + use_thread: bool = True, + **kwargs, + ): + from sglang.lang.interpreter import run_program + + # avoid using [] as the default arg: https://nikos7am.com/posts/mutable-default-arguments/ + if stop is None: + stop = [] + if stop_token_ids is None: + stop_token_ids = [] + if stop_regex is None: + stop_regex = [] + + default_sampling_para = SglSamplingParams( + max_new_tokens=max_new_tokens, + n=n, + stop=stop, + stop_token_ids=stop_token_ids, + stop_regex=stop_regex, + temperature=temperature, + top_p=top_p, + top_k=top_k, + min_p=min_p, + frequency_penalty=frequency_penalty, + presence_penalty=presence_penalty, + ignore_eos=ignore_eos, + return_logprob=return_logprob, + logprob_start_len=logprob_start_len, + top_logprobs_num=top_logprobs_num, + return_text_in_logprobs=return_text_in_logprobs, + ) + backend = backend or global_config.default_backend + return run_program( + self, + backend, + args, + kwargs, + default_sampling_para, + stream, + use_thread=use_thread, + ) + + def run_batch( + self, + batch_kwargs, + *, + max_new_tokens: int = 128, + n: int = 1, + stop: Optional[Union[str, List[str]]] = None, + stop_token_ids: Optional[List[int]] = None, + stop_regex: Optional[Union[str, List[str]]] = None, + temperature: float = 1.0, + top_p: float = 1.0, + top_k: int = -1, + min_p: float = 0.0, + frequency_penalty: float = 0.0, + presence_penalty: float = 0.0, + ignore_eos: bool = False, + return_logprob: Optional[bool] = None, + logprob_start_len: Optional[int] = None, + top_logprobs_num: Optional[int] = None, + return_text_in_logprobs: Optional[bool] = None, + backend=None, + num_threads: Union[str, int] = "auto", + progress_bar: bool = False, + generator_style: bool = False, + ): + from sglang.lang.interpreter import run_program_batch + + if stop is None: + stop = [] + if stop_token_ids is None: + stop_token_ids = [] + if stop_regex is None: + stop_regex = [] + + assert isinstance(batch_kwargs, (list, tuple)) + if len(batch_kwargs) == 0: + return [] + if not isinstance(batch_kwargs[0], dict): + num_programs = len(batch_kwargs) + # change the list of argument values to dict of arg_name -> arg_value + batch_kwargs = [ + {self.arg_names[i]: v for i, v in enumerate(arg_values)} + for arg_values in batch_kwargs + if isinstance(arg_values, (list, tuple)) + and len(self.arg_names) - len(self.arg_defaults) + <= len(arg_values) + <= len(self.arg_names) + ] + # Ensure to raise an exception if the number of arguments mismatch + if len(batch_kwargs) != num_programs: + raise Exception("Given arguments mismatch the SGL function signature") + + default_sampling_para = SglSamplingParams( + max_new_tokens=max_new_tokens, + n=n, + stop=stop, + stop_token_ids=stop_token_ids, + stop_regex=stop_regex, + temperature=temperature, + top_p=top_p, + top_k=top_k, + min_p=min_p, + frequency_penalty=frequency_penalty, + presence_penalty=presence_penalty, + ignore_eos=ignore_eos, + return_logprob=return_logprob, + logprob_start_len=logprob_start_len, + top_logprobs_num=top_logprobs_num, + return_text_in_logprobs=return_text_in_logprobs, + ) + backend = backend or global_config.default_backend + return run_program_batch( + self, + backend, + batch_kwargs, + default_sampling_para, + num_threads, + progress_bar, + generator_style=generator_style, + ) + + def trace(self, *, backend=None, **kwargs): + from sglang.lang.tracer import trace_program + + backend = backend or global_config.default_backend + return trace_program(self, kwargs, backend) + + def cache(self, backend=None): + from sglang.lang.interpreter import cache_program + + backend = backend or global_config.default_backend + return cache_program(self, backend) + + def __call__(self, *args, **kwargs): + from sglang.lang.tracer import TracingScope + + tracing_scope = TracingScope.get_current_scope() + if tracing_scope is None: + return self.run(*args, **kwargs) + else: + kwargs["backend"] = tracing_scope.tracer_state.backend + return self.trace(*args, **kwargs) + + +class SglExpr: + node_ct = 0 + + def __init__(self): + self.node_id = SglExpr.node_ct + self.prev_node = None + self.pid = None + SglExpr.node_ct += 1 + + def __add__(self, other): + if isinstance(other, str): + other = SglConstantText(other) + assert isinstance(other, SglExpr) + + return self.concatenate_ir(self, other) + + def __radd__(self, other): + if isinstance(other, str): + other = SglConstantText(other) + assert isinstance(other, SglExpr), f"{other}" + + return self.concatenate_ir(other, self) + + def concatenate_ir(self, a, b): + if isinstance(a, SglExprList): + if isinstance(b, SglExprList): + return SglExprList(a.expr_list + b.expr_list) + else: + return SglExprList(a.expr_list + [b]) + elif isinstance(b, SglExprList): + return SglExprList([a] + b.expr_list) + + return SglExprList([a, b]) + + def print_graph_dfs(self): + ret = [""] + visited = set() + + def dfs_print(x): + if x is None or x in visited: + return + visited.add(x) + + # Print dependency + if x.prev_node is not None: + dfs_print(x.prev_node) + + if isinstance(x, SglExprList): + for y in x.expr_list: + dfs_print(y) + # elif isinstance(x, SglRole): + # dfs_print(x.expr) + elif isinstance(x, SglVariable): + dfs_print(x.source) + + # Print the node itself + if isinstance(x, (SglFork, SglGetForkItem)): + ret[0] += f"%{x.node_id} = {x}\n" + else: + if x.prev_node is not None: + ret[0] += ( + f"%{x.node_id} = %{x.prev_node.node_id} + " + str(x) + "\n" + ) + else: + ret[0] += f"%{x.node_id} = " + str(x) + "\n" + + dfs_print(self) + return ret[0] + + +class SglExprList(SglExpr): + def __init__(self, expr_list: List[SglExpr]): + super().__init__() + self.expr_list = expr_list + + def __repr__(self): + return f"ExprList({self.expr_list})" + + +class SglArgument(SglExpr): + def __init__(self, name: str, value: str): + super().__init__() + self.name = name + self.value = value + + def __repr__(self): + return f"Argument(name={self.name}, value={repr(self.value)})" + + def __len__(self): + return len(self.value) + + def __getitem__(self, i): + return self.value[i] + + def __int__(self): + return self.value + + def __bool__(self): + return self.value + + def __format__(self, *args): + raise TypeError( + "Cannot put argument inside a f-string. " + "This is not compatible with the tracer. " + ) + + +class SglImage(SglExpr): + def __init__(self, path: str): + self.path = path + + def __repr__(self) -> str: + return f"SglImage({self.path})" + + +class SglVideo(SglExpr): + def __init__(self, path: str, num_frames: int): + self.path = path + self.num_frames = num_frames + + def __repr__(self) -> str: + return f"SglVideo({self.path}, {self.num_frames})" + + +class SglGen(SglExpr): + def __init__( + self, + name: Optional[str] = None, + max_new_tokens: Optional[int] = None, + min_new_tokens: Optional[int] = None, + n: Optional[int] = None, + stop: Optional[Union[str, List[str]]] = None, + stop_token_ids: Optional[List[int]] = None, + stop_regex: Optional[Union[str, List[str]]] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + top_k: Optional[int] = None, + min_p: Optional[float] = None, + frequency_penalty: Optional[float] = None, + presence_penalty: Optional[float] = None, + ignore_eos: Optional[bool] = None, + return_logprob: Optional[bool] = None, + logprob_start_len: Optional[int] = None, + top_logprobs_num: Optional[int] = None, + return_text_in_logprobs: Optional[bool] = None, + dtype: Optional[type] = None, + regex: Optional[str] = None, + json_schema: Optional[str] = None, + ): + """Call the model to generate. See the meaning of the arguments in docs/backend/sampling_params.md""" + super().__init__() + self.name = name + self.sampling_params = SglSamplingParams( + max_new_tokens=max_new_tokens, + min_new_tokens=min_new_tokens, + n=n, + stop=stop, + stop_regex=stop_regex, + stop_token_ids=stop_token_ids, + temperature=temperature, + top_p=top_p, + top_k=top_k, + min_p=min_p, + frequency_penalty=frequency_penalty, + presence_penalty=presence_penalty, + ignore_eos=ignore_eos, + return_logprob=return_logprob, + logprob_start_len=logprob_start_len, + top_logprobs_num=top_logprobs_num, + return_text_in_logprobs=return_text_in_logprobs, + dtype=dtype, + regex=regex, + json_schema=json_schema, + ) + + def __repr__(self): + return f"Gen('{self.name}')" + + +class SglConstantText(SglExpr): + def __init__(self, value: str): + super().__init__() + self.value = value + + def __repr__(self): + return f"Constant({repr(self.value)})" + + +class SglRoleBegin(SglExpr): + def __init__(self, role: str): + super().__init__() + self.role = role + + def __repr__(self): + return f"RoleBegin({self.role})" + + +class SglRoleEnd(SglExpr): + def __init__(self, role: str): + super().__init__() + self.role = role + + def __repr__(self): + return f"RoleEnd({self.role})" + + +class SglSelect(SglExpr): + + def __init__( + self, + name: str, + choices: List[str], + temperature: float, + choices_method: ChoicesSamplingMethod, + ): + super().__init__() + self.name = name + self.choices = choices + self.temperature = temperature + self.choices_method = choices_method + + def __repr__(self): + return f"Select({self.name}, choices={self.choices}, choices_method={self.choices_method})" + + +class SglFork(SglExpr): + def __init__(self, number: int, position_ids_offset=None): + super().__init__() + self.number = number + self.position_ids_offset = position_ids_offset + + def __repr__(self): + return ( + f"Fork(%{self.prev_node.node_id}, number={self.number}, " + f"position_ids_offset={self.position_ids_offset})" + ) + + +class SglGetForkItem(SglExpr): + def __init__(self, index: int): + super().__init__() + self.index = index + + def __repr__(self): + return f"GetForkItem(%{self.prev_node.node_id}, index={self.index})" + + +class SglVariable(SglExpr): + def __init__(self, name: str, source): + super().__init__() + self.name = name + self.source = source + + def __repr__(self): + return f"Variable('{self.name}', source=%{self.source.node_id})" + + +class SglVarScopeBegin(SglExpr): + def __init__(self, name: str): + super().__init__() + self.name = name + + def __repr__(self): + return f"VarScopeBegin('{self.name}')" + + +class SglVarScopeEnd(SglExpr): + def __init__(self, name: str): + super().__init__() + self.name = name + + def __repr__(self): + return f"VarScopeEnd('{self.name}')" + + +class SglConcateAndAppend(SglExpr): + def __init__(self, states): + super().__init__() + self.states = states + + def __repr__(self): + return f"ConcatenateAndAppend('{self.states}')" + + +class SglCommitLazy(SglExpr): + def __init__(self): + super().__init__() + + def __repr__(self): + return "CommitLazy()" + + +class SglSeparateReasoning(SglExpr): + def __init__(self, model_type: str, expr: SglExpr): + super().__init__() + self.model_type = model_type + + self.expr = expr + self.name = None + self._process_expr(expr) + + def process_name_for_reasoning(self, name): + if not name: + raise ValueError("name must be provided") + return f"{name}_reasoning_content" + + def _process_expr(self, expr): + if isinstance(expr, SglGen): + self.name = self.process_name_for_reasoning(expr.name) + elif isinstance(expr, SglSelect): + self.name = self.process_name_for_reasoning(expr.name) + elif isinstance(expr, SglExprList): + for x in expr.expr_list: + self._process_expr(x) + + def __repr__(self): + return f"SeparateReasoning(model_type={self.model_type}, name={self.name})" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/tracer.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/tracer.py new file mode 100644 index 00000000..0a2a744f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/lang/tracer.py @@ -0,0 +1,279 @@ +"""Tracing a program.""" + +import uuid +from typing import Any, Dict, List, Optional + +from sglang.lang.backend.base_backend import BaseBackend +from sglang.lang.interpreter import ProgramState, ProgramStateGroup +from sglang.lang.ir import ( + SglArgument, + SglConstantText, + SglExpr, + SglExprList, + SglFork, + SglGen, + SglGetForkItem, + SglRoleBegin, + SglRoleEnd, + SglSelect, + SglVariable, + SglVarScopeBegin, + SglVarScopeEnd, +) + + +class StopTracing(Exception): + pass + + +def extract_prefix_by_tracing(program, backend): + # Create dummy arguments + dummy_arguments = {name: SglArgument(name, None) for name in program.arg_names} + arguments = dummy_arguments + arguments.update(program.bind_arguments) + + # Trace + tracer = TracerProgramState(backend, arguments, only_trace_prefix=True) + try: + with TracingScope(tracer): + tracer.ret_value = program.func(tracer, **arguments) + except (StopTracing, TypeError, AttributeError): + # Some exceptions may not be caught + pass + + # Run and cache prefix + prefix = "" + for expr in tracer.flatten_nodes(): + if isinstance(expr, SglConstantText): + prefix += expr.value + else: + break + return prefix + + +def trace_program(program, arguments, backend): + # Create dummy backend + if backend is None: + backend = BaseBackend() + + # Create dummy arguments + dummy_arguments = { + name: SglArgument(name, None) + for name in program.arg_names + if name not in arguments + } + arguments.update(dummy_arguments) + arguments.update(program.bind_arguments) + + # Trace + tracer = TracerProgramState(backend, arguments, only_trace_prefix=False) + with TracingScope(tracer): + tracer.ret_value = program.func(tracer, **arguments) + return tracer + + +class TracerProgramState(ProgramState): + def __init__(self, backend, arguments, only_trace_prefix): + self.pid = uuid.uuid4().hex + self.backend = backend + self.arguments: Dict[str, Any] = arguments + self.only_trace_prefix = only_trace_prefix + + if hasattr(backend, "endpoint"): + self.backend = backend.endpoint + + self.nodes = [] + self.last_node = None + self.variables = {} + self.ret_value = None + + # For completion + + # For chat + self.messages_ = [] + self.cur_role = None + self.chat_template = self.backend.get_chat_template() + + # For multi states + self.child_states = [] + + cur_scope = TracingScope.get_current_scope() + if cur_scope is not None: + cur_scope.add_child_state(self) + + ################################## + ########### Public API ########### + ################################## + + def fork(self, size: int = 1, position_ids_offset: Optional[List[int]] = None): + assert size >= 1 + + if self.only_trace_prefix: + raise StopTracing() + + fork_node = SglFork(size) + fork_node.prev_node = self.last_node + + states = [ + TracerProgramState(self.backend, self.arguments, self.only_trace_prefix) + for _ in range(size) + ] + + for i in range(size): + node = SglGetForkItem(i) + node.prev_node = fork_node + states[i].last_node = node + states[i].variables = dict(self.variables) + states[i].messages_ = list(self.messages_) + states[i].cur_role = self.cur_role + states[i].chat_template = self.chat_template + + state_group = ProgramStateGroup(states, self) + + return state_group + + ################################## + ########## Internal API ########## + ################################## + + def _append_node(self, other: SglExpr): + self.nodes.append(other) + other.prev_node = self.last_node + self.last_node = other + + def _execute(self, other: SglExpr): + if isinstance(other, str): + other = SglConstantText(other) + + other.pid = self.pid + + if isinstance(other, SglConstantText): + self._execute_fill(other) + elif isinstance(other, SglGen): + self._execute_gen(other) + elif isinstance(other, SglSelect): + self._execute_select(other) + elif isinstance(other, SglExprList): + for x in other.expr_list: + self._execute(x) + elif isinstance(other, SglRoleBegin): + self._execute_role_begin(other) + elif isinstance(other, SglRoleEnd): + self._execute_role_end(other) + elif isinstance(other, SglVarScopeBegin): + self._execute_var_scope_begin(other) + elif isinstance(other, SglVarScopeEnd): + self._execute_var_scope_end(other) + else: + if self.only_trace_prefix: + raise StopTracing() + else: + self._append_node(other) + + return self + + def __iadd__(self, other): + self._execute(other) + return self + + def _execute_fill(self, expr: SglConstantText): + if isinstance(expr, str): + expr = SglConstantText(expr) + self._append_node(expr) + + def _execute_gen(self, expr: SglGen): + name = expr.name if expr.name is not None else "gen_" + str(len(self.variables)) + new_node = SglVariable(name, source=expr) + self.variables[name] = new_node + self._append_node(expr) + + def _execute_select(self, expr: SglSelect): + name = ( + expr.name if expr.name is not None else "select_" + str(len(self.variables)) + ) + new_node = SglVariable(name, source=expr) + self.variables[name] = new_node + self._append_node(expr) + + def _execute_role_begin(self, expr: SglRoleBegin): + assert self.cur_role is None, "Nested roles are not allowed." + + if len(self.messages_) == 0 and expr.role != "system": + # Insert default system message + default_system = self.chat_template.default_system_prompt + if default_system: + self._execute_role_begin(SglRoleBegin("system")) + self._execute_fill(default_system) + self._execute_role_end(SglRoleEnd("system")) + + self.cur_role = expr.role + + prefix, suffix = self.chat_template.get_prefix_and_suffix( + expr.role, self.messages_ + ) + + self._execute_fill(prefix) + + def _execute_role_end(self, expr: SglRoleEnd): + prefix, suffix = self.chat_template.get_prefix_and_suffix( + expr.role, self.messages_ + ) + + self._execute_fill(suffix) + + self.messages_.append({"role": expr.role, "content": ""}) + + self.cur_role = None + + def _execute_var_scope_end(self, expr: SglVarScopeEnd): + new_node = SglVariable(expr.name, source=self.last_node) + self.variables[expr.name] = new_node + + def get_var(self, name): + ret = self.arguments.get(name, None) + if ret is not None: + return ret + + v = self.variables[name] + return SglVariable(v.name, v.source) + + def flatten_nodes(self): + def traverse(cur): + if isinstance(cur, SglExprList): + for child in cur.expr_list: + traverse(child) + else: + ret.append(cur) + + ret = [] + for x in self.nodes: + traverse(x) + return ret + + def __del__(self): + pass + + +class TracingScope: + cur_scope = None + + def __init__(self, tracer_state: TracerProgramState): + self.tracer_state = tracer_state + self.last_scope = TracingScope.cur_scope + + def __enter__(self): + TracingScope.cur_scope = self + return self + + def __exit__(self, exc_type, exc_value, traceback): + TracingScope.cur_scope = self.last_scope + + @staticmethod + def get_current_scope(): + return TracingScope.cur_scope + + def add_child_state(self, state: TracerProgramState): + cur_scope = self + while cur_scope is not None: + cur_scope.tracer_state.child_states.append(state) + cur_scope = cur_scope.last_scope diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/launch_server.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/launch_server.py new file mode 100644 index 00000000..9e3e82a7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/launch_server.py @@ -0,0 +1,29 @@ +"""Launch the inference server.""" + +import asyncio +import os +import sys + +from sglang.srt.server_args import prepare_server_args +from sglang.srt.utils import kill_process_tree + + +def run_server(server_args): + """Run the server based on server_args.grpc_mode.""" + if server_args.grpc_mode: + from sglang.srt.entrypoints.grpc_server import serve_grpc + + asyncio.run(serve_grpc(server_args)) + else: + from sglang.srt.entrypoints.http_server import launch_server + + launch_server(server_args) + + +if __name__ == "__main__": + server_args = prepare_server_args(sys.argv[1:]) + + try: + run_server(server_args) + finally: + kill_process_tree(os.getpid(), include_parent=False) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/README.md new file mode 100644 index 00000000..68c9fb4b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/README.md @@ -0,0 +1,76 @@ +

+ +**SGLang diffusion is an inference framework for accelerated image/video generation.** + +SGLang diffusion features an end-to-end unified pipeline for accelerating diffusion models. It is designed to be modular and extensible, allowing users to easily add new models and optimizations. + +## Key Features + +SGLang Diffusion has the following features: + - Broad model support: Wan series, FastWan series, Hunyuan, Qwen-Image, Qwen-Image-Edit, Flux + - Fast inference speed: enpowered by highly optimized kernel from sgl-kernel and efficient scheduler loop + - Ease of use: OpenAI-compatible api, CLI, and python sdk support + - Diverse hardware support: H100, H200, A100, B200, 4090 + +## Getting Started + +```bash +uv pip install 'sglang[diffusion]' --prerelease=allow +``` + +For more installation methods (e.g. pypi, uv, docker), check [install.md](https://github.com/sgl-project/sglang/tree/main/python/sglang/multimodal_gen/docs/install.md). + + +## Inference + +Here's a minimal example to generate a video using the default settings: + +```python +from sglang.multimodal_gen import DiffGenerator + +def main(): + # Create a diff generator from a pre-trained model + generator = DiffGenerator.from_pretrained( + model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + num_gpus=1, # Adjust based on your hardware + ) + + # Provide a prompt for your video + prompt = "A curious raccoon peers through a vibrant field of yellow sunflowers, its eyes wide with interest." + + # Generate the video + video = generator.generate( + prompt, + return_frames=True, # Also return frames from this call (defaults to False) + output_path="my_videos/", # Controls where videos are saved + save_output=True + ) + +if __name__ == '__main__': + main() +``` + +Or, more simply, with the CLI: + +```bash +sglang generate --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \ + --text-encoder-cpu-offload --pin-cpu-memory \ + --prompt "A curious raccoon" \ + --save-output +``` + +For more usage examples (e.g. OpenAI compatible API, server mode), check [cli.md](https://github.com/sgl-project/sglang/tree/main/python/sglang/multimodal_gen/docs/cli.md). + +## Contributing + +All contributions are welcome. The contribution guide is available [here](https://github.com/sgl-project/sglang/tree/main/python/sglang/multimodal_gen/docs/contributing.md). + +## Acknowledgement + +We learnt and reused code from the following projects: + +- [FastVideo](https://github.com/hao-ai-lab/FastVideo.git). The major components of this repo are based on a fork of FastVide on Sept. 24, 2025. +- [xDiT](https://github.com/xdit-project/xDiT). We used the parallelism library from it. +- [diffusers](https://github.com/huggingface/diffusers) We used the pipeline design from it. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/__init__.py new file mode 100644 index 00000000..75182221 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/__init__.py @@ -0,0 +1,6 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo +from sglang.multimodal_gen.configs.pipeline_configs import PipelineConfig +from sglang.multimodal_gen.configs.sample import SamplingParams +from sglang.multimodal_gen.runtime.entrypoints.diffusion_generator import DiffGenerator + +__all__ = ["DiffGenerator", "PipelineConfig", "SamplingParams"] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/benchmarks/compare_perf.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/benchmarks/compare_perf.py new file mode 100644 index 00000000..2dfb087c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/benchmarks/compare_perf.py @@ -0,0 +1,216 @@ +import argparse +import json +import re +from datetime import datetime +from typing import Any, Dict, List, Tuple + + +def calculate_diff(base: float, new: float) -> Tuple[float, float]: + """Returns (diff, diff_percent).""" + diff = new - base + if base == 0: + percent = 0.0 + else: + percent = (diff / base) * 100 + return diff, percent + + +def calculate_upper_bound(baseline: float, rel_tol: float, min_abs_tol: float) -> float: + """Calculates the upper bound for performance regression check.""" + rel_limit = baseline * (1 + rel_tol) + abs_limit = baseline + min_abs_tol + return max(rel_limit, abs_limit) + + +def calculate_lower_bound(baseline: float, rel_tol: float, min_abs_tol: float) -> float: + """Calculates the lower bound for performance improvement check.""" + rel_lower = baseline * (1 - rel_tol) + abs_lower = baseline - min_abs_tol + return min(rel_lower, abs_lower) + + +def get_perf_status_emoji( + baseline: float, + new: float, + rel_tol: float = 0.1, + min_abs_tol: float = 120.0, +) -> str: + """ + Determines the status emoji based on performance difference. + + Logic: + Upper bound (Slower): max(baseline * (1 + rel_tol), baseline + min_abs_tol) + Lower bound (Faster): min(baseline * (1 - rel_tol), baseline - min_abs_tol) + """ + upper_bound = calculate_upper_bound(baseline, rel_tol, min_abs_tol) + lower_bound = calculate_lower_bound(baseline, rel_tol, min_abs_tol) + + if new > upper_bound: + return "🔴" + elif new < lower_bound: + return "🟢" + else: + return "⚪️" + + +def consolidate_steps( + steps_list: List[Dict[str, Any]], +) -> Tuple[Dict[str, float], List[str], Dict[str, int]]: + """ + Aggregates specific repeating steps (like denoising_step_*) into groups. + Returns: + - aggregated_durations: {name: duration_ms} + - ordered_names: list of names in execution order + - counts: {name: count_of_steps_aggregated} + """ + durations = {} + counts = {} + ordered_names = [] + seen_names = set() + + # Regex for steps to group + # Group "denoising_step_0", "denoising_step_1" -> "Denoising Loop" + denoise_pattern = re.compile(r"^denoising_step_(\d+)$") + denoising_group_name = "Denoising Loop" + + for step in steps_list: + name = step.get("name", "unknown") + dur = step.get("duration_ms", 0.0) + + match = denoise_pattern.match(name) + if match: + key = denoising_group_name + if key not in durations: + durations[key] = 0.0 + counts[key] = 0 + if key not in seen_names: + ordered_names.append(key) + seen_names.add(key) + durations[key] += dur + counts[key] += 1 + else: + # Standard stage (preserve order) + if name not in durations: + durations[name] = 0.0 + counts[name] = 0 + if name not in seen_names: + ordered_names.append(name) + seen_names.add(name) + durations[name] += dur + counts[name] += 1 + + return durations, ordered_names, counts + + +def _load_benchmark_file(file_path: str) -> Dict[str, Any]: + """Loads a benchmark JSON file.""" + with open(file_path, "r", encoding="utf-8") as f: + return json.load(f) + + +def compare_benchmarks( + baseline_path: str, new_path: str, output_format: str = "markdown" +): + """ + Compares two benchmark JSON files and prints a report. + """ + try: + base_data = _load_benchmark_file(baseline_path) + new_data = _load_benchmark_file(new_path) + except Exception as e: + print(f"Error loading benchmark files: {e}") + return + + base_e2e = base_data.get("total_duration_ms", 0) + new_e2e = new_data.get("total_duration_ms", 0) + + diff_ms, diff_pct = calculate_diff(base_e2e, new_e2e) + + if diff_pct < -2.0: + status = "✅" + elif diff_pct > 2.0: + status = "❌" + else: + status = "" + + # --- Stage Breakdown --- + base_durations, base_order, base_counts = consolidate_steps( + base_data.get("steps", []) + ) + new_durations, new_order, new_counts = consolidate_steps(new_data.get("steps", [])) + + # Merge orders: Start with New order (execution order), append any missing from Base + combined_order = list(new_order) + for name in base_order: + if name not in combined_order: + combined_order.append(name) + + stage_rows = [] + for stage in combined_order: + b_val = base_durations.get(stage, 0.0) + n_val = new_durations.get(stage, 0.0) + b_count = base_counts.get(stage, 1) + n_count = new_counts.get(stage, 1) + + s_diff, s_pct = calculate_diff(b_val, n_val) + + # Format count string if aggregated + count_str = "" + if stage == "Denoising Loop": + count_str = ( + f" ({n_count} steps)" + if n_count == b_count + else f" ({b_count}->{n_count} steps)" + ) + + # filter noise: show if diff is > 0.5ms OR if it's a major stage (like Denoising Loop) + # always show Denoising Loop or stages with significant duration/diff + stage_rows.append((stage + count_str, b_val, n_val, s_diff, s_pct)) + + if output_format == "markdown": + print("### Performance Comparison Report\n") + + # Summary Table + print("#### 1. High-level Summary") + print("| Metric | Baseline | New | Diff | Status |") + print("| :--- | :--- | :--- | :--- | :--- |") + print( + f"| **E2E Latency** | {base_e2e:.2f} ms | {new_e2e:.2f} ms | **{diff_ms:+.2f} ms ({diff_pct:+.1f}%)** | {status} |" + ) + print( + f"| **Throughput** | {1000 / base_e2e if base_e2e else 0:.2f} req/s | {1000 / new_e2e if new_e2e else 0:.2f} req/s | - | - |" + ) + print("\n") + + # Detailed Breakdown + print("#### 2. Stage Breakdown") + print( + "| Stage Name | Baseline (ms) | New (ms) | Diff (ms) | Diff (%) | Status |" + ) + print("| :--- | :--- | :--- | :--- | :--- | :--- |") + for name, b, n, d, p in stage_rows: + name_str = name + status_emoji = get_perf_status_emoji(b, n) + print( + f"| {name_str} | {b:.2f} | {n:.2f} | {d:+.2f} | {p:+.1f}% | {status_emoji} |" + ) + print("\n") + + # Metadata + print("
") + print("Metadata\n") + print(f"- Baseline Commit: `{base_data.get('commit_hash', 'N/A')}`") + print(f"- New Commit: `{new_data.get('commit_hash', 'N/A')}`") + print(f"- Timestamp: {datetime.now().isoformat()}") + print("
") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Compare two sglang-diffusion performance JSON files." + ) + parser.add_argument("baseline", help="Path to the baseline JSON file") + parser.add_argument("new", help="Path to the new JSON file") + args = parser.parse_args() + + compare_benchmarks(args.baseline, args.new) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/__init__.py new file mode 100644 index 00000000..dfff5f2c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/__init__.py @@ -0,0 +1,3 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# Configs for pipelines, and pipeline modules (in models folder) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/backend/vmoba/wan_1.3B_77_448_832.json b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/backend/vmoba/wan_1.3B_77_448_832.json new file mode 100644 index 00000000..1e55b5f2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/backend/vmoba/wan_1.3B_77_448_832.json @@ -0,0 +1,16 @@ +{ + "temporal_chunk_size": 2, + "temporal_topk": 2, + "spatial_chunk_size": [4, 13], + "spatial_topk": 6, + "st_chunk_size": [4, 4, 13], + "st_topk": 18, + "moba_select_mode": "topk", + "moba_threshold": 0.25, + "moba_threshold_type": "query_head", + "first_full_layer": 0, + "first_full_step": 12, + "temporal_layer": 1, + "spatial_layer": 1, + "st_layer": 1 +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/backend/vmoba/wan_1.3B_77_480_832.json b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/backend/vmoba/wan_1.3B_77_480_832.json new file mode 100644 index 00000000..ddf66f48 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/backend/vmoba/wan_1.3B_77_480_832.json @@ -0,0 +1,16 @@ +{ + "temporal_chunk_size": 2, + "temporal_topk": 3, + "spatial_chunk_size": [3, 4], + "spatial_topk": 20, + "st_chunk_size": [4, 6, 4], + "st_topk": 15, + "moba_select_mode": "threshold", + "moba_threshold": 0.25, + "moba_threshold_type": "query_head", + "first_full_layer": 0, + "first_full_step": 12, + "temporal_layer": 1, + "spatial_layer": 1, + "st_layer": 1 +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/__init__.py new file mode 100644 index 00000000..62c0aadf --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/__init__.py @@ -0,0 +1,8 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +from sglang.multimodal_gen.configs.models.base import ModelConfig +from sglang.multimodal_gen.configs.models.dits.base import DiTConfig +from sglang.multimodal_gen.configs.models.encoders.base import EncoderConfig +from sglang.multimodal_gen.configs.models.vaes.base import VAEConfig + +__all__ = ["ModelConfig", "VAEConfig", "DiTConfig", "EncoderConfig"] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/base.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/base.py new file mode 100644 index 00000000..6de428ad --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/base.py @@ -0,0 +1,105 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field, fields +from typing import Any, Dict + +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +# 1. ArchConfig contains all fields from diffuser's/transformer's config.json (i.e. all fields related to the architecture of the model) +# 2. ArchConfig should be inherited & overridden by each model arch_config +# 3. Any field in ArchConfig is fixed upon initialization, and should be hidden away from users +@dataclass +class ArchConfig: + stacked_params_mapping: list[tuple[str, str, str]] = field( + default_factory=list + ) # mapping from huggingface weight names to custom names + extra_attrs: Dict[str, Any] = field(default_factory=dict) + + def __getattr__(self, name: str): + d = object.__getattribute__(self, "__dict__") + extras = d.get("extra_attrs") + if extras is not None and name in extras: + return extras[name] + raise AttributeError( + f"'{self.__class__.__name__}' object has no attribute '{name}'" + ) + + def __setattr__(self, key, value): + if key in type(self).__dataclass_fields__: + object.__setattr__(self, key, value) + else: + d = object.__getattribute__(self, "__dict__") + extras = d.get("extra_attrs") + if extras is None: + extras = {} + d["extra_attrs"] = extras + extras[key] = value + + +@dataclass +class ModelConfig: + # Every model config parameter can be categorized into either ArchConfig or everything else + # Diffuser/Transformer parameters + arch_config: ArchConfig = field(default_factory=ArchConfig) + + # sglang-diffusion-specific parameters here + # i.e. STA, quantization, teacache + + def __getattr__(self, name): + # Only called if 'name' is not found in ModelConfig directly + if hasattr(self.arch_config, name): + return getattr(self.arch_config, name) + raise AttributeError( + f"'{type(self).__name__}' object has no attribute '{name}'" + ) + + def __getstate__(self): + # Return a dictionary of attributes to pickle + # Convert to dict and exclude any problematic attributes + state = self.__dict__.copy() + return state + + def __setstate__(self, state): + # Restore instance attributes from the unpickled state + self.__dict__.update(state) + + # This should be used only when loading from transformers/diffusers + def update_model_arch(self, source_model_dict: dict[str, Any]) -> None: + """ + Update arch_config with source_model_dict + """ + arch_config = self.arch_config + valid_fields = {f.name for f in fields(arch_config)} + + for key, value in source_model_dict.items(): + setattr(arch_config, key, value) + # else: + # raise AttributeError( + # f"{type(arch_config).__name__} has no field '{key}'" + # ) + + if hasattr(arch_config, "__post_init__"): + arch_config.__post_init__() + + def update_model_config(self, source_model_dict: dict[str, Any]) -> None: + assert ( + "arch_config" not in source_model_dict + ), "Source model config shouldn't contain arch_config." + + valid_fields = {f.name for f in fields(self)} + + for key, value in source_model_dict.items(): + if key in valid_fields: + setattr(self, key, value) + else: + logger.warning( + "%s does not contain field '%s'!", type(self).__name__, key + ) + raise AttributeError(f"Invalid field: {key}") + + if hasattr(self, "__post_init__"): + self.__post_init__() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/__init__.py new file mode 100644 index 00000000..67e6d97b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/__init__.py @@ -0,0 +1,7 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +from sglang.multimodal_gen.configs.models.dits.hunyuanvideo import HunyuanVideoConfig +from sglang.multimodal_gen.configs.models.dits.stepvideo import StepVideoConfig +from sglang.multimodal_gen.configs.models.dits.wanvideo import WanVideoConfig + +__all__ = ["HunyuanVideoConfig", "WanVideoConfig", "StepVideoConfig"] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/base.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/base.py new file mode 100644 index 00000000..77fecfb4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/base.py @@ -0,0 +1,76 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field +from typing import Any + +from sglang.multimodal_gen.configs.models.base import ArchConfig, ModelConfig +from sglang.multimodal_gen.runtime.layers.quantization import QuantizationConfig +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum + + +@dataclass +class DiTArchConfig(ArchConfig): + _fsdp_shard_conditions: list = field(default_factory=list) + _compile_conditions: list = field(default_factory=list) + + # convert weights name from HF-format to SGLang-dit-format + param_names_mapping: dict = field(default_factory=dict) + + # convert weights name from misc-format to HF-format + # usually applicable if the LoRA is trained with official repo implementation + lora_param_names_mapping: dict = field(default_factory=dict) + + # Reverse mapping for saving checkpoints: custom -> hf + reverse_param_names_mapping: dict = field(default_factory=dict) + _supported_attention_backends: set[AttentionBackendEnum] = field( + default_factory=lambda: { + AttentionBackendEnum.SLIDING_TILE_ATTN, + AttentionBackendEnum.SAGE_ATTN, + AttentionBackendEnum.FA, + AttentionBackendEnum.TORCH_SDPA, + AttentionBackendEnum.VIDEO_SPARSE_ATTN, + AttentionBackendEnum.VMOBA_ATTN, + AttentionBackendEnum.SAGE_ATTN_THREE, + } + ) + + hidden_size: int = 0 + num_attention_heads: int = 0 + num_channels_latents: int = 0 + exclude_lora_layers: list[str] = field(default_factory=list) + boundary_ratio: float | None = None + + def __post_init__(self) -> None: + if not self._compile_conditions: + self._compile_conditions = self._fsdp_shard_conditions.copy() + + +@dataclass +class DiTConfig(ModelConfig): + arch_config: DiTArchConfig = field(default_factory=DiTArchConfig) + + # sglang-diffusion DiT-specific parameters + prefix: str = "" + quant_config: QuantizationConfig | None = None + + @staticmethod + def add_cli_args(parser: Any, prefix: str = "dit-config") -> Any: + """Add CLI arguments for DiTConfig fields""" + parser.add_argument( + f"--{prefix}.prefix", + type=str, + dest=f"{prefix.replace('-', '_')}.prefix", + default=DiTConfig.prefix, + help="Prefix for the DiT model", + ) + + parser.add_argument( + f"--{prefix}.quant-config", + type=str, + dest=f"{prefix.replace('-', '_')}.quant_config", + default=None, + help="Quantization configuration for the DiT model", + ) + + return parser diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/flux.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/flux.py new file mode 100644 index 00000000..f50c9b1e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/flux.py @@ -0,0 +1,42 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field +from typing import Tuple + +from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig + + +@dataclass +class FluxArchConfig(DiTArchConfig): + patch_size: int = 1 + in_channels: int = 64 + out_channels: int | None = None + num_layers: int = 19 + num_single_layers: int = 38 + attention_head_dim: int = 128 + num_attention_heads: int = 24 + joint_attention_dim: int = 4096 + pooled_projection_dim: int = 768 + guidance_embeds: bool = False + axes_dims_rope: Tuple[int, int, int] = (16, 56, 56) + + param_names_mapping: dict = field( + default_factory=lambda: { + r"transformer\.(\w*)\.(.*)$": r"\1.\2", + } + ) + + def __post_init__(self): + super().__post_init__() + self.out_channels = self.out_channels or self.in_channels + self.hidden_size = self.num_attention_heads * self.attention_head_dim + self.num_channels_latents = self.out_channels + + +@dataclass +class FluxConfig(DiTConfig): + + arch_config: DiTArchConfig = field(default_factory=FluxArchConfig) + + prefix: str = "Flux" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/hunyuanvideo.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/hunyuanvideo.py new file mode 100644 index 00000000..1cae921f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/hunyuanvideo.py @@ -0,0 +1,184 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field + +import torch + +from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig + + +def is_double_block(n: str, m) -> bool: + return "double" in n and str.isdigit(n.split(".")[-1]) + + +def is_single_block(n: str, m) -> bool: + return "single" in n and str.isdigit(n.split(".")[-1]) + + +def is_refiner_block(n: str, m) -> bool: + return "refiner" in n and str.isdigit(n.split(".")[-1]) + + +def is_txt_in(n: str, m) -> bool: + return n.split(".")[-1] == "txt_in" + + +@dataclass +class HunyuanVideoArchConfig(DiTArchConfig): + _fsdp_shard_conditions: list = field( + default_factory=lambda: [is_double_block, is_single_block, is_refiner_block] + ) + + _compile_conditions: list = field( + default_factory=lambda: [is_double_block, is_single_block, is_txt_in] + ) + + param_names_mapping: dict = field( + default_factory=lambda: { + # 1. context_embedder.time_text_embed submodules (specific rules, applied first): + r"^context_embedder\.time_text_embed\.timestep_embedder\.linear_1\.(.*)$": r"txt_in.t_embedder.mlp.fc_in.\1", + r"^context_embedder\.time_text_embed\.timestep_embedder\.linear_2\.(.*)$": r"txt_in.t_embedder.mlp.fc_out.\1", + r"^context_embedder\.proj_in\.(.*)$": r"txt_in.input_embedder.\1", + r"^context_embedder\.time_text_embed\.text_embedder\.linear_1\.(.*)$": r"txt_in.c_embedder.fc_in.\1", + r"^context_embedder\.time_text_embed\.text_embedder\.linear_2\.(.*)$": r"txt_in.c_embedder.fc_out.\1", + r"^context_embedder\.token_refiner\.refiner_blocks\.(\d+)\.norm1\.(.*)$": r"txt_in.refiner_blocks.\1.norm1.\2", + r"^context_embedder\.token_refiner\.refiner_blocks\.(\d+)\.norm2\.(.*)$": r"txt_in.refiner_blocks.\1.norm2.\2", + r"^context_embedder\.token_refiner\.refiner_blocks\.(\d+)\.attn\.to_q\.(.*)$": ( + r"txt_in.refiner_blocks.\1.self_attn_qkv.\2", + 0, + 3, + ), + r"^context_embedder\.token_refiner\.refiner_blocks\.(\d+)\.attn\.to_k\.(.*)$": ( + r"txt_in.refiner_blocks.\1.self_attn_qkv.\2", + 1, + 3, + ), + r"^context_embedder\.token_refiner\.refiner_blocks\.(\d+)\.attn\.to_v\.(.*)$": ( + r"txt_in.refiner_blocks.\1.self_attn_qkv.\2", + 2, + 3, + ), + r"^context_embedder\.token_refiner\.refiner_blocks\.(\d+)\.attn\.to_out\.0\.(.*)$": r"txt_in.refiner_blocks.\1.self_attn_proj.\2", + r"^context_embedder\.token_refiner\.refiner_blocks\.(\d+)\.ff\.net\.0(?:\.proj)?\.(.*)$": r"txt_in.refiner_blocks.\1.mlp.fc_in.\2", + r"^context_embedder\.token_refiner\.refiner_blocks\.(\d+)\.ff\.net\.2(?:\.proj)?\.(.*)$": r"txt_in.refiner_blocks.\1.mlp.fc_out.\2", + r"^context_embedder\.token_refiner\.refiner_blocks\.(\d+)\.norm_out\.linear\.(.*)$": r"txt_in.refiner_blocks.\1.adaLN_modulation.linear.\2", + # 3. x_embedder mapping: + r"^x_embedder\.proj\.(.*)$": r"img_in.proj.\1", + # 4. Top-level time_text_embed mappings: + r"^time_text_embed\.timestep_embedder\.linear_1\.(.*)$": r"time_in.mlp.fc_in.\1", + r"^time_text_embed\.timestep_embedder\.linear_2\.(.*)$": r"time_in.mlp.fc_out.\1", + r"^time_text_embed\.guidance_embedder\.linear_1\.(.*)$": r"guidance_in.mlp.fc_in.\1", + r"^time_text_embed\.guidance_embedder\.linear_2\.(.*)$": r"guidance_in.mlp.fc_out.\1", + r"^time_text_embed\.text_embedder\.linear_1\.(.*)$": r"vector_in.fc_in.\1", + r"^time_text_embed\.text_embedder\.linear_2\.(.*)$": r"vector_in.fc_out.\1", + # 5. transformer_blocks mapping: + r"^transformer_blocks\.(\d+)\.norm1\.linear\.(.*)$": r"double_blocks.\1.img_mod.linear.\2", + r"^transformer_blocks\.(\d+)\.norm1_context\.linear\.(.*)$": r"double_blocks.\1.txt_mod.linear.\2", + r"^transformer_blocks\.(\d+)\.attn\.norm_q\.(.*)$": r"double_blocks.\1.img_attn_q_norm.\2", + r"^transformer_blocks\.(\d+)\.attn\.norm_k\.(.*)$": r"double_blocks.\1.img_attn_k_norm.\2", + r"^transformer_blocks\.(\d+)\.attn\.to_q\.(.*)$": ( + r"double_blocks.\1.img_attn_qkv.\2", + 0, + 3, + ), + r"^transformer_blocks\.(\d+)\.attn\.to_k\.(.*)$": ( + r"double_blocks.\1.img_attn_qkv.\2", + 1, + 3, + ), + r"^transformer_blocks\.(\d+)\.attn\.to_v\.(.*)$": ( + r"double_blocks.\1.img_attn_qkv.\2", + 2, + 3, + ), + r"^transformer_blocks\.(\d+)\.attn\.add_q_proj\.(.*)$": ( + r"double_blocks.\1.txt_attn_qkv.\2", + 0, + 3, + ), + r"^transformer_blocks\.(\d+)\.attn\.add_k_proj\.(.*)$": ( + r"double_blocks.\1.txt_attn_qkv.\2", + 1, + 3, + ), + r"^transformer_blocks\.(\d+)\.attn\.add_v_proj\.(.*)$": ( + r"double_blocks.\1.txt_attn_qkv.\2", + 2, + 3, + ), + r"^transformer_blocks\.(\d+)\.attn\.to_out\.0\.(.*)$": r"double_blocks.\1.img_attn_proj.\2", + # Corrected: merge attn.to_add_out into the main projection. + r"^transformer_blocks\.(\d+)\.attn\.to_add_out\.(.*)$": r"double_blocks.\1.txt_attn_proj.\2", + r"^transformer_blocks\.(\d+)\.attn\.norm_added_q\.(.*)$": r"double_blocks.\1.txt_attn_q_norm.\2", + r"^transformer_blocks\.(\d+)\.attn\.norm_added_k\.(.*)$": r"double_blocks.\1.txt_attn_k_norm.\2", + r"^transformer_blocks\.(\d+)\.ff\.net\.0(?:\.proj)?\.(.*)$": r"double_blocks.\1.img_mlp.fc_in.\2", + r"^transformer_blocks\.(\d+)\.ff\.net\.2(?:\.proj)?\.(.*)$": r"double_blocks.\1.img_mlp.fc_out.\2", + r"^transformer_blocks\.(\d+)\.ff_context\.net\.0(?:\.proj)?\.(.*)$": r"double_blocks.\1.txt_mlp.fc_in.\2", + r"^transformer_blocks\.(\d+)\.ff_context\.net\.2(?:\.proj)?\.(.*)$": r"double_blocks.\1.txt_mlp.fc_out.\2", + # 6. single_transformer_blocks mapping: + r"^single_transformer_blocks\.(\d+)\.attn\.norm_q\.(.*)$": r"single_blocks.\1.q_norm.\2", + r"^single_transformer_blocks\.(\d+)\.attn\.norm_k\.(.*)$": r"single_blocks.\1.k_norm.\2", + r"^single_transformer_blocks\.(\d+)\.attn\.to_q\.(.*)$": ( + r"single_blocks.\1.linear1.\2", + 0, + 4, + ), + r"^single_transformer_blocks\.(\d+)\.attn\.to_k\.(.*)$": ( + r"single_blocks.\1.linear1.\2", + 1, + 4, + ), + r"^single_transformer_blocks\.(\d+)\.attn\.to_v\.(.*)$": ( + r"single_blocks.\1.linear1.\2", + 2, + 4, + ), + r"^single_transformer_blocks\.(\d+)\.proj_mlp\.(.*)$": ( + r"single_blocks.\1.linear1.\2", + 3, + 4, + ), + # Corrected: map proj_out to modulation.linear rather than a separate proj_out branch. + r"^single_transformer_blocks\.(\d+)\.proj_out\.(.*)$": r"single_blocks.\1.linear2.\2", + r"^single_transformer_blocks\.(\d+)\.norm\.linear\.(.*)$": r"single_blocks.\1.modulation.linear.\2", + # 7. Final layers mapping: + r"^norm_out\.linear\.(.*)$": r"final_layer.adaLN_modulation.linear.\1", + r"^proj_out\.(.*)$": r"final_layer.linear.\1", + } + ) + + reverse_param_names_mapping: dict = field(default_factory=lambda: {}) + + patch_size: int = 2 + patch_size_t: int = 1 + in_channels: int = 16 + out_channels: int = 16 + num_attention_heads: int = 24 + attention_head_dim: int = 128 + mlp_ratio: float = 4.0 + num_layers: int = 20 + num_single_layers: int = 40 + num_refiner_layers: int = 2 + rope_axes_dim: tuple[int, int, int] = (16, 56, 56) + guidance_embeds: bool = False + dtype: torch.dtype | None = None + text_embed_dim: int = 4096 + pooled_projection_dim: int = 768 + rope_theta: int = 256 + qk_norm: str = "rms_norm" + exclude_lora_layers: list[str] = field( + default_factory=lambda: ["img_in", "txt_in", "time_in", "vector_in"] + ) + + def __post_init__(self): + super().__post_init__() + self.hidden_size: int = self.attention_head_dim * self.num_attention_heads + self.num_channels_latents: int = self.in_channels + + +@dataclass +class HunyuanVideoConfig(DiTConfig): + arch_config: DiTArchConfig = field(default_factory=HunyuanVideoArchConfig) + + prefix: str = "Hunyuan" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/qwenimage.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/qwenimage.py new file mode 100644 index 00000000..f9003264 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/qwenimage.py @@ -0,0 +1,41 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field +from typing import Tuple + +from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig + + +@dataclass +class QwenImageArchConfig(DiTArchConfig): + patch_size: int = 1 + in_channels: int = 64 + out_channels: int | None = None + num_layers: int = 19 + num_single_layers: int = 38 + attention_head_dim: int = 128 + num_attention_heads: int = 24 + joint_attention_dim: int = 4096 + pooled_projection_dim: int = 768 + guidance_embeds: bool = False + axes_dims_rope: Tuple[int, int, int] = (16, 56, 56) + + param_names_mapping: dict = field( + default_factory=lambda: { + r"^(transformer_blocks\.\d+\.attn\..*\.lora_[AB])\.default$": r"\1", + } + ) + + def __post_init__(self): + super().__post_init__() + self.out_channels = self.out_channels or self.in_channels + self.hidden_size = self.num_attention_heads * self.attention_head_dim + self.num_channels_latents = self.out_channels + + +@dataclass +class QwenImageDitConfig(DiTConfig): + arch_config: DiTArchConfig = field(default_factory=QwenImageArchConfig) + + prefix: str = "qwenimage" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/stepvideo.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/stepvideo.py new file mode 100644 index 00000000..1d7fe21a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/stepvideo.py @@ -0,0 +1,64 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field + +from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig + + +def is_transformer_blocks(n, m): + return "transformer_blocks" in n and n.split(".")[-1].isdigit() + + +@dataclass +class StepVideoArchConfig(DiTArchConfig): + _fsdp_shard_conditions: list = field( + default_factory=lambda: [is_transformer_blocks] + ) + + param_names_mapping: dict = field( + default_factory=lambda: { + # transformer block + r"^transformer_blocks\.(\d+)\.norm1\.(weight|bias)$": r"transformer_blocks.\1.norm1.norm.\2", + r"^transformer_blocks\.(\d+)\.norm2\.(weight|bias)$": r"transformer_blocks.\1.norm2.norm.\2", + r"^transformer_blocks\.(\d+)\.ff\.net\.0\.proj\.weight$": r"transformer_blocks.\1.ff.fc_in.weight", + r"^transformer_blocks\.(\d+)\.ff\.net\.2\.weight$": r"transformer_blocks.\1.ff.fc_out.weight", + # adanorm block + r"^adaln_single\.emb\.timestep_embedder\.linear_1\.(weight|bias)$": r"adaln_single.emb.mlp.fc_in.\1", + r"^adaln_single\.emb\.timestep_embedder\.linear_2\.(weight|bias)$": r"adaln_single.emb.mlp.fc_out.\1", + # caption projection + r"^caption_projection\.linear_1\.(weight|bias)$": r"caption_projection.fc_in.\1", + r"^caption_projection\.linear_2\.(weight|bias)$": r"caption_projection.fc_out.\1", + } + ) + + num_attention_heads: int = 48 + attention_head_dim: int = 128 + in_channels: int = 64 + out_channels: int | None = 64 + num_layers: int = 48 + dropout: float = 0.0 + patch_size: int = 1 + norm_type: str = "ada_norm_single" + norm_elementwise_affine: bool = False + norm_eps: float = 1e-6 + caption_channels: int | list[int] | tuple[int, ...] | None = field( + default_factory=lambda: [6144, 1024] + ) + attention_type: str | None = "torch" + use_additional_conditions: bool | None = False + exclude_lora_layers: list[str] = field(default_factory=lambda: []) + + def __post_init__(self): + self.hidden_size = self.num_attention_heads * self.attention_head_dim + self.out_channels = ( + self.in_channels if self.out_channels is None else self.out_channels + ) + self.num_channels_latents = self.out_channels + + +@dataclass +class StepVideoConfig(DiTConfig): + arch_config: DiTArchConfig = field(default_factory=StepVideoArchConfig) + + prefix: str = "StepVideo" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/wanvideo.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/wanvideo.py new file mode 100644 index 00000000..3a9939c4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/wanvideo.py @@ -0,0 +1,102 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field + +from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig + + +def is_blocks(n: str, m) -> bool: + return "blocks" in n and str.isdigit(n.split(".")[-1]) + + +@dataclass +class WanVideoArchConfig(DiTArchConfig): + _fsdp_shard_conditions: list = field(default_factory=lambda: [is_blocks]) + + param_names_mapping: dict = field( + default_factory=lambda: { + r"^patch_embedding\.(.*)$": r"patch_embedding.proj.\1", + r"^condition_embedder\.text_embedder\.linear_1\.(.*)$": r"condition_embedder.text_embedder.fc_in.\1", + r"^condition_embedder\.text_embedder\.linear_2\.(.*)$": r"condition_embedder.text_embedder.fc_out.\1", + r"^condition_embedder\.time_embedder\.linear_1\.(.*)$": r"condition_embedder.time_embedder.mlp.fc_in.\1", + r"^condition_embedder\.time_embedder\.linear_2\.(.*)$": r"condition_embedder.time_embedder.mlp.fc_out.\1", + r"^condition_embedder\.time_proj\.(.*)$": r"condition_embedder.time_modulation.linear.\1", + r"^condition_embedder\.image_embedder\.ff\.net\.0\.proj\.(.*)$": r"condition_embedder.image_embedder.ff.fc_in.\1", + r"^condition_embedder\.image_embedder\.ff\.net\.2\.(.*)$": r"condition_embedder.image_embedder.ff.fc_out.\1", + r"^blocks\.(\d+)\.attn1\.to_q\.(.*)$": r"blocks.\1.to_q.\2", + r"^blocks\.(\d+)\.attn1\.to_k\.(.*)$": r"blocks.\1.to_k.\2", + r"^blocks\.(\d+)\.attn1\.to_v\.(.*)$": r"blocks.\1.to_v.\2", + r"^blocks\.(\d+)\.attn1\.to_out\.0\.(.*)$": r"blocks.\1.to_out.\2", + r"^blocks\.(\d+)\.attn1\.norm_q\.(.*)$": r"blocks.\1.norm_q.\2", + r"^blocks\.(\d+)\.attn1\.norm_k\.(.*)$": r"blocks.\1.norm_k.\2", + r"^blocks\.(\d+)\.attn2\.to_out\.0\.(.*)$": r"blocks.\1.attn2.to_out.\2", + r"^blocks\.(\d+)\.ffn\.net\.0\.proj\.(.*)$": r"blocks.\1.ffn.fc_in.\2", + r"^blocks\.(\d+)\.ffn\.net\.2\.(.*)$": r"blocks.\1.ffn.fc_out.\2", + r"^blocks\.(\d+)\.norm2\.(.*)$": r"blocks.\1.self_attn_residual_norm.norm.\2", + } + ) + + reverse_param_names_mapping: dict = field(default_factory=lambda: {}) + + # Some LoRA adapters use the original official layer names instead of hf layer names, + # so apply this before the param_names_mapping + lora_param_names_mapping: dict = field( + default_factory=lambda: { + r"^blocks\.(\d+)\.self_attn\.q\.(.*)$": r"blocks.\1.attn1.to_q.\2", + r"^blocks\.(\d+)\.self_attn\.k\.(.*)$": r"blocks.\1.attn1.to_k.\2", + r"^blocks\.(\d+)\.self_attn\.v\.(.*)$": r"blocks.\1.attn1.to_v.\2", + r"^blocks\.(\d+)\.self_attn\.o\.(.*)$": r"blocks.\1.attn1.to_out.0.\2", + r"^blocks\.(\d+)\.cross_attn\.q\.(.*)$": r"blocks.\1.attn2.to_q.\2", + r"^blocks\.(\d+)\.cross_attn\.k\.(.*)$": r"blocks.\1.attn2.to_k.\2", + r"^blocks\.(\d+)\.cross_attn\.v\.(.*)$": r"blocks.\1.attn2.to_v.\2", + r"^blocks\.(\d+)\.cross_attn\.o\.(.*)$": r"blocks.\1.attn2.to_out.0.\2", + r"^blocks\.(\d+)\.ffn\.0\.(.*)$": r"blocks.\1.ffn.fc_in.\2", + r"^blocks\.(\d+)\.ffn\.2\.(.*)$": r"blocks.\1.ffn.fc_out.\2", + } + ) + + patch_size: tuple[int, int, int] = (1, 2, 2) + text_len = 512 + num_attention_heads: int = 40 + attention_head_dim: int = 128 + in_channels: int = 16 + out_channels: int = 16 + text_dim: int = 4096 + freq_dim: int = 256 + ffn_dim: int = 13824 + num_layers: int = 40 + cross_attn_norm: bool = True + qk_norm: str = "rms_norm_across_heads" + eps: float = 1e-6 + image_dim: int | None = None + added_kv_proj_dim: int | None = None + rope_max_seq_len: int = 1024 + pos_embed_seq_len: int | None = None + exclude_lora_layers: list[str] = field(default_factory=lambda: ["embedder"]) + + # Wan MoE + boundary_ratio: float | None = None + + # Causal Wan + local_attn_size: int = ( + -1 + ) # Window size for temporal local attention (-1 indicates global attention) + sink_size: int = ( + 0 # Size of the attention sink, we keep the first `sink_size` frames unchanged when rolling the KV cache + ) + num_frames_per_block: int = 3 + sliding_window_num_frames: int = 21 + + def __post_init__(self): + super().__post_init__() + self.out_channels = self.out_channels or self.in_channels + self.hidden_size = self.num_attention_heads * self.attention_head_dim + self.num_channels_latents = self.out_channels + + +@dataclass +class WanVideoConfig(DiTConfig): + arch_config: DiTArchConfig = field(default_factory=WanVideoArchConfig) + + prefix: str = "Wan" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/zimage.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/zimage.py new file mode 100644 index 00000000..7d3576fc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/dits/zimage.py @@ -0,0 +1,40 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field +from typing import Tuple + +from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig + + +@dataclass +class ZImageArchConfig(DiTArchConfig): + all_patch_size: Tuple[int, ...] = (2,) + all_f_patch_size: Tuple[int, ...] = (1,) + in_channels: int = 16 + out_channels: int | None = None + dim: int = 3840 + num_layers: int = 30 + n_refiner_layers: int = 2 + num_attention_heads: int = 30 + n_kv_heads: int = 30 + norm_eps: float = 1e-5 + qk_norm: bool = True + cap_feat_dim: int = 2560 + rope_theta: float = 256.0 + t_scale: float = 1000.0 + axes_dims: Tuple[int, int, int] = (32, 48, 48) + axes_lens: Tuple[int, int, int] = (1024, 512, 512) + + def __post_init__(self): + super().__post_init__() + self.out_channels = self.out_channels or self.in_channels + self.num_channels_latents = self.in_channels + self.hidden_size = self.dim + + +@dataclass +class ZImageDitConfig(DiTConfig): + arch_config: ZImageArchConfig = field(default_factory=ZImageArchConfig) + + prefix: str = "zimage" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/__init__.py new file mode 100644 index 00000000..70851bfa --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/__init__.py @@ -0,0 +1,25 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +from sglang.multimodal_gen.configs.models.encoders.base import ( + BaseEncoderOutput, + EncoderConfig, + ImageEncoderConfig, + TextEncoderConfig, +) +from sglang.multimodal_gen.configs.models.encoders.clip import ( + CLIPTextConfig, + CLIPVisionConfig, +) +from sglang.multimodal_gen.configs.models.encoders.llama import LlamaConfig +from sglang.multimodal_gen.configs.models.encoders.t5 import T5Config + +__all__ = [ + "EncoderConfig", + "TextEncoderConfig", + "ImageEncoderConfig", + "BaseEncoderOutput", + "CLIPTextConfig", + "CLIPVisionConfig", + "LlamaConfig", + "T5Config", +] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/base.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/base.py new file mode 100644 index 00000000..514b1561 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/base.py @@ -0,0 +1,86 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field +from typing import Any + +import torch + +from sglang.multimodal_gen.configs.models.base import ArchConfig, ModelConfig +from sglang.multimodal_gen.runtime.layers.quantization import QuantizationConfig +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum + + +@dataclass +class EncoderArchConfig(ArchConfig): + _fsdp_shard_conditions: list = field(default_factory=lambda: []) + architectures: list[str] = field(default_factory=lambda: []) + _supported_attention_backends: set[AttentionBackendEnum] = field( + default_factory=lambda: { + AttentionBackendEnum.FA, + AttentionBackendEnum.TORCH_SDPA, + } + ) + output_hidden_states: bool = False + use_return_dict: bool = True + + +@dataclass +class TextEncoderArchConfig(EncoderArchConfig): + vocab_size: int = 0 + hidden_size: int = 0 + num_hidden_layers: int = 0 + num_attention_heads: int = 0 + pad_token_id: int = 0 + eos_token_id: int = 0 + text_len: int = 0 + hidden_state_skip_layer: int = 0 + decoder_start_token_id: int = 0 + output_past: bool = True + scalable_attention: bool = True + tie_word_embeddings: bool = False + stacked_params_mapping: list[tuple[str, str, str]] = field( + default_factory=list + ) # mapping from huggingface weight names to custom names + tokenizer_kwargs: dict[str, Any] = field(default_factory=dict) + _fsdp_shard_conditions: list = field(default_factory=lambda: []) + + def __post_init__(self) -> None: + self.tokenizer_kwargs = { + "truncation": True, + "max_length": self.text_len, + "return_tensors": "pt", + } + + +@dataclass +class ImageEncoderArchConfig(EncoderArchConfig): + pass + + +@dataclass +class BaseEncoderOutput: + last_hidden_state: torch.FloatTensor | None = None + pooler_output: torch.FloatTensor | None = None + hidden_states: tuple[torch.FloatTensor, ...] | None = None + attentions: tuple[torch.FloatTensor, ...] | None = None + attention_mask: torch.Tensor | None = None + + +@dataclass +class EncoderConfig(ModelConfig): + arch_config: ArchConfig = field(default_factory=EncoderArchConfig) + + prefix: str = "" + quant_config: QuantizationConfig | None = None + lora_config: Any | None = None + + +@dataclass +class TextEncoderConfig(EncoderConfig): + arch_config: ArchConfig = field(default_factory=TextEncoderArchConfig) + + +@dataclass +class ImageEncoderConfig(EncoderConfig): + arch_config: ArchConfig = field(default_factory=ImageEncoderArchConfig) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/clip.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/clip.py new file mode 100644 index 00000000..6b36fc88 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/clip.py @@ -0,0 +1,95 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field + +from sglang.multimodal_gen.configs.models.encoders.base import ( + ImageEncoderArchConfig, + ImageEncoderConfig, + TextEncoderArchConfig, + TextEncoderConfig, +) + + +def _is_transformer_layer(n: str, m) -> bool: + return "layers" in n and str.isdigit(n.split(".")[-1]) + + +def _is_embeddings(n: str, m) -> bool: + return n.endswith("embeddings") + + +@dataclass +class CLIPTextArchConfig(TextEncoderArchConfig): + vocab_size: int = 49408 + hidden_size: int = 512 + intermediate_size: int = 2048 + projection_dim: int = 512 + num_hidden_layers: int = 12 + num_attention_heads: int = 8 + max_position_embeddings: int = 77 + hidden_act: str = "quick_gelu" + layer_norm_eps: float = 1e-5 + dropout: float = 0.0 + attention_dropout: float = 0.0 + initializer_range: float = 0.02 + initializer_factor: float = 1.0 + pad_token_id: int = 1 + bos_token_id: int = 49406 + eos_token_id: int = 49407 + text_len: int = 77 + stacked_params_mapping: list[tuple[str, str, str]] = field( + default_factory=lambda: [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ] + ) + _fsdp_shard_conditions: list = field( + default_factory=lambda: [_is_transformer_layer, _is_embeddings] + ) + + +@dataclass +class CLIPVisionArchConfig(ImageEncoderArchConfig): + hidden_size: int = 768 + intermediate_size: int = 3072 + projection_dim: int = 512 + num_hidden_layers: int = 12 + num_attention_heads: int = 12 + num_channels: int = 3 + image_size: int = 224 + patch_size: int = 32 + hidden_act: str = "quick_gelu" + layer_norm_eps: float = 1e-5 + dropout: float = 0.0 + attention_dropout: float = 0.0 + initializer_range: float = 0.02 + initializer_factor: float = 1.0 + stacked_params_mapping: list[tuple[str, str, str]] = field( + default_factory=lambda: [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ] + ) + + +@dataclass +class CLIPTextConfig(TextEncoderConfig): + arch_config: TextEncoderArchConfig = field(default_factory=CLIPTextArchConfig) + + num_hidden_layers_override: int | None = None + require_post_norm: bool | None = None + prefix: str = "clip" + + +@dataclass +class CLIPVisionConfig(ImageEncoderConfig): + arch_config: ImageEncoderArchConfig = field(default_factory=CLIPVisionArchConfig) + + num_hidden_layers_override: int | None = None + require_post_norm: bool | None = None + prefix: str = "clip" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/llama.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/llama.py new file mode 100644 index 00000000..41d98cab --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/llama.py @@ -0,0 +1,69 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field + +from sglang.multimodal_gen.configs.models.encoders.base import ( + TextEncoderArchConfig, + TextEncoderConfig, +) + + +def _is_transformer_layer(n: str, m) -> bool: + return "layers" in n and str.isdigit(n.split(".")[-1]) + + +def _is_embeddings(n: str, m) -> bool: + return n.endswith("embed_tokens") + + +def _is_final_norm(n: str, m) -> bool: + return n.endswith("norm") + + +@dataclass +class LlamaArchConfig(TextEncoderArchConfig): + vocab_size: int = 32000 + hidden_size: int = 4096 + intermediate_size: int = 11008 + num_hidden_layers: int = 32 + num_attention_heads: int = 32 + num_key_value_heads: int | None = None + hidden_act: str = "silu" + max_position_embeddings: int = 2048 + initializer_range: float = 0.02 + rms_norm_eps: float = 1e-6 + use_cache: bool = True + pad_token_id: int = 0 + bos_token_id: int = 1 + eos_token_id: int = 2 + pretraining_tp: int = 1 + tie_word_embeddings: bool = False + rope_theta: float = 10000.0 + rope_scaling: float | None = None + attention_bias: bool = False + attention_dropout: float = 0.0 + mlp_bias: bool = False + head_dim: int | None = None + hidden_state_skip_layer: int = 2 + text_len: int = 256 + stacked_params_mapping: list[tuple[str, str, str]] = field( + default_factory=lambda: [ + # (param_name, shard_name, shard_id) + (".qkv_proj", ".q_proj", "q"), + (".qkv_proj", ".k_proj", "k"), + (".qkv_proj", ".v_proj", "v"), + (".gate_up_proj", ".gate_proj", 0), # type: ignore + (".gate_up_proj", ".up_proj", 1), # type: ignore + ] + ) + _fsdp_shard_conditions: list = field( + default_factory=lambda: [_is_transformer_layer, _is_embeddings, _is_final_norm] + ) + + +@dataclass +class LlamaConfig(TextEncoderConfig): + arch_config: TextEncoderArchConfig = field(default_factory=LlamaArchConfig) + + prefix: str = "llama" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/qwen_image.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/qwen_image.py new file mode 100644 index 00000000..02f57ca6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/qwen_image.py @@ -0,0 +1,68 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field + +from sglang.multimodal_gen.configs.models.encoders.base import ( + TextEncoderArchConfig, + TextEncoderConfig, +) + + +def _is_transformer_layer(n: str, m) -> bool: + return "layers" in n and str.isdigit(n.split(".")[-1]) + + +def _is_embeddings(n: str, m) -> bool: + return n.endswith("embed_tokens") + + +def _is_final_norm(n: str, m) -> bool: + return n.endswith("norm") + + +@dataclass +class QwenImageArchConfig(TextEncoderArchConfig): + vocab_size: int = 32000 + hidden_size: int = 4096 + intermediate_size: int = 11008 + num_hidden_layers: int = 32 + num_attention_heads: int = 32 + num_key_value_heads: int | None = None + hidden_act: str = "silu" + max_position_embeddings: int = 2048 + initializer_range: float = 0.02 + rms_norm_eps: float = 1e-6 + use_cache: bool = True + pad_token_id: int = -1 + eos_token_id: int = 2 + pretraining_tp: int = 1 + tie_word_embeddings: bool = False + rope_theta: float = 10000.0 + rope_scaling: float | None = None + attention_bias: bool = False + attention_dropout: float = 0.0 + mlp_bias: bool = False + head_dim: int | None = None + hidden_state_skip_layer: int = 2 + text_len: int = 256 + + stacked_params_mapping: list[tuple[str, str, str]] = field( + default_factory=lambda: [ + # (param_name, shard_name, shard_id) + (".qkv_proj", ".q_proj", "q"), + (".qkv_proj", ".k_proj", "k"), + (".qkv_proj", ".v_proj", "v"), + (".gate_up_proj", ".gate_proj", 0), # type: ignore + (".gate_up_proj", ".up_proj", 1), # type: ignore + ] + ) + _fsdp_shard_conditions: list = field( + default_factory=lambda: [_is_transformer_layer, _is_embeddings, _is_final_norm] + ) + + +@dataclass +class Qwen2_5VLConfig(TextEncoderConfig): + arch_config: TextEncoderArchConfig = field(default_factory=QwenImageArchConfig) + # prefix: str = "qwen_image" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/t5.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/t5.py new file mode 100644 index 00000000..3fd9b2f1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/encoders/t5.py @@ -0,0 +1,86 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field + +from sglang.multimodal_gen.configs.models.encoders.base import ( + TextEncoderArchConfig, + TextEncoderConfig, +) + + +def _is_transformer_layer(n: str, m) -> bool: + return "block" in n and str.isdigit(n.split(".")[-1]) + + +def _is_embeddings(n: str, m) -> bool: + return n.endswith("shared") + + +def _is_final_layernorm(n: str, m) -> bool: + return n.endswith("final_layer_norm") + + +@dataclass +class T5ArchConfig(TextEncoderArchConfig): + vocab_size: int = 32128 + d_model: int = 512 + d_kv: int = 64 + d_ff: int = 2048 + num_layers: int = 6 + num_decoder_layers: int | None = None + num_heads: int = 8 + relative_attention_num_buckets: int = 32 + relative_attention_max_distance: int = 128 + dropout_rate: float = 0.1 + layer_norm_epsilon: float = 1e-6 + initializer_factor: float = 1.0 + feed_forward_proj: str = "relu" + dense_act_fn: str = "" + is_gated_act: bool = False + is_encoder_decoder: bool = True + use_cache: bool = True + pad_token_id: int = 0 + eos_token_id: int = 1 + classifier_dropout: float = 0.0 + text_len: int = 512 + stacked_params_mapping: list[tuple[str, str, str]] = field( + default_factory=lambda: [ + # (param_name, shard_name, shard_id) + (".qkv_proj", ".q", "q"), + (".qkv_proj", ".k", "k"), + (".qkv_proj", ".v", "v"), + ] + ) + _fsdp_shard_conditions: list = field( + default_factory=lambda: [ + _is_transformer_layer, + _is_embeddings, + _is_final_layernorm, + ] + ) + + # Referenced from https://github.com/huggingface/transformers/blob/main/src/transformers/models/t5/configuration_t5.py + def __post_init__(self): + super().__post_init__() + act_info = self.feed_forward_proj.split("-") + self.dense_act_fn: str = act_info[-1] + self.is_gated_act: bool = act_info[0] == "gated" + if self.feed_forward_proj == "gated-gelu": + self.dense_act_fn = "gelu_new" + + self.tokenizer_kwargs = { + "padding": "max_length", + "truncation": True, + "max_length": self.text_len, + "add_special_tokens": True, + "return_attention_mask": True, + "return_tensors": "pt", + } + + +@dataclass +class T5Config(TextEncoderConfig): + arch_config: TextEncoderArchConfig = field(default_factory=T5ArchConfig) + + prefix: str = "t5" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/__init__.py new file mode 100644 index 00000000..e9b47861 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/__init__.py @@ -0,0 +1,11 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +from sglang.multimodal_gen.configs.models.vaes.hunyuanvae import HunyuanVAEConfig +from sglang.multimodal_gen.configs.models.vaes.stepvideovae import StepVideoVAEConfig +from sglang.multimodal_gen.configs.models.vaes.wanvae import WanVAEConfig + +__all__ = [ + "HunyuanVAEConfig", + "WanVAEConfig", + "StepVideoVAEConfig", +] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/base.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/base.py new file mode 100644 index 00000000..344a37e6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/base.py @@ -0,0 +1,156 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +import argparse +import dataclasses +from dataclasses import dataclass, field +from typing import Any + +import torch + +from sglang.multimodal_gen.configs.models.base import ArchConfig, ModelConfig +from sglang.multimodal_gen.utils import StoreBoolean + + +@dataclass +class VAEArchConfig(ArchConfig): + scaling_factor: float | torch.Tensor = 0 + + temporal_compression_ratio: int = 4 + # or vae_scale_factor? + spatial_compression_ratio: int = 8 + + +@dataclass +class VAEConfig(ModelConfig): + arch_config: VAEArchConfig = field(default_factory=VAEArchConfig) + + # sglang-diffusion VAE-specific parameters + load_encoder: bool = True + load_decoder: bool = True + + tile_sample_min_height: int = 256 + tile_sample_min_width: int = 256 + tile_sample_min_num_frames: int = 16 + tile_sample_stride_height: int = 192 + tile_sample_stride_width: int = 192 + tile_sample_stride_num_frames: int = 12 + blend_num_frames: int = 0 + + use_tiling: bool = True + use_temporal_tiling: bool = True + use_parallel_tiling: bool = True + use_temporal_scaling_frames: bool = True + + def __post_init__(self): + self.blend_num_frames = ( + self.tile_sample_min_num_frames - self.tile_sample_stride_num_frames + ) + + def post_init(self): + pass + + @staticmethod + def add_cli_args(parser: Any, prefix: str = "vae-config") -> Any: + """Add CLI arguments for VAEConfig fields""" + parser.add_argument( + f"--{prefix}.load-encoder", + action=StoreBoolean, + dest=f"{prefix.replace('-', '_')}.load_encoder", + default=VAEConfig.load_encoder, + help="Whether to load the VAE encoder", + ) + parser.add_argument( + f"--{prefix}.load-decoder", + action=StoreBoolean, + dest=f"{prefix.replace('-', '_')}.load_decoder", + default=VAEConfig.load_decoder, + help="Whether to load the VAE decoder", + ) + parser.add_argument( + f"--{prefix}.tile-sample-min-height", + type=int, + dest=f"{prefix.replace('-', '_')}.tile_sample_min_height", + default=VAEConfig.tile_sample_min_height, + help="Minimum height for VAE tile sampling", + ) + parser.add_argument( + f"--{prefix}.tile-sample-min-width", + type=int, + dest=f"{prefix.replace('-', '_')}.tile_sample_min_width", + default=VAEConfig.tile_sample_min_width, + help="Minimum width for VAE tile sampling", + ) + parser.add_argument( + f"--{prefix}.tile-sample-min-num-frames", + type=int, + dest=f"{prefix.replace('-', '_')}.tile_sample_min_num_frames", + default=VAEConfig.tile_sample_min_num_frames, + help="Minimum number of frames for VAE tile sampling", + ) + parser.add_argument( + f"--{prefix}.tile-sample-stride-height", + type=int, + dest=f"{prefix.replace('-', '_')}.tile_sample_stride_height", + default=VAEConfig.tile_sample_stride_height, + help="Stride height for VAE tile sampling", + ) + parser.add_argument( + f"--{prefix}.tile-sample-stride-width", + type=int, + dest=f"{prefix.replace('-', '_')}.tile_sample_stride_width", + default=VAEConfig.tile_sample_stride_width, + help="Stride width for VAE tile sampling", + ) + parser.add_argument( + f"--{prefix}.tile-sample-stride-num-frames", + type=int, + dest=f"{prefix.replace('-', '_')}.tile_sample_stride_num_frames", + default=VAEConfig.tile_sample_stride_num_frames, + help="Stride number of frames for VAE tile sampling", + ) + parser.add_argument( + f"--{prefix}.blend-num-frames", + type=int, + dest=f"{prefix.replace('-', '_')}.blend_num_frames", + default=VAEConfig.blend_num_frames, + help="Number of frames to blend for VAE tile sampling", + ) + parser.add_argument( + f"--{prefix}.use-tiling", + action=StoreBoolean, + dest=f"{prefix.replace('-', '_')}.use_tiling", + default=VAEConfig.use_tiling, + help="Whether to use tiling for VAE", + ) + parser.add_argument( + f"--{prefix}.use-temporal-tiling", + action=StoreBoolean, + dest=f"{prefix.replace('-', '_')}.use_temporal_tiling", + default=VAEConfig.use_temporal_tiling, + help="Whether to use temporal tiling for VAE", + ) + parser.add_argument( + f"--{prefix}.use-parallel-tiling", + action=StoreBoolean, + dest=f"{prefix.replace('-', '_')}.use_parallel_tiling", + default=VAEConfig.use_parallel_tiling, + help="Whether to use parallel tiling for VAE", + ) + + return parser + + def get_vae_scale_factor(self): + return 2 ** (len(self.arch_config.block_out_channels) - 1) + + def encode_sample_mode(self): + return "argmax" + + @classmethod + def from_cli_args(cls, args: argparse.Namespace) -> "VAEConfig": + kwargs = {} + for attr in dataclasses.fields(cls): + value = getattr(args, attr.name, None) + if value is not None: + kwargs[attr.name] = value + return cls(**kwargs) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/flux.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/flux.py new file mode 100644 index 00000000..53bd4461 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/flux.py @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field + +from sglang.multimodal_gen.configs.models.vaes.base import VAEArchConfig, VAEConfig + + +@dataclass +class FluxVAEArchConfig(VAEArchConfig): + spatial_compression_ratio: int = 1 + + base_dim: int = 96 + decoder_base_dim: int | None = None + z_dim: int = 16 + dim_mult: tuple[int, ...] = (1, 2, 4, 4) + num_res_blocks: int = 2 + attn_scales: tuple[float, ...] = () + temperal_downsample: tuple[bool, ...] = (False, True, True) + dropout: float = 0.0 + + is_residual: bool = False + in_channels: int = 3 + out_channels: int = 3 + patch_size: int | None = None + scale_factor_temporal: int = 4 + scale_factor_spatial: int = 8 + clip_output: bool = True + + +@dataclass +class Flux2VAEArchConfig(FluxVAEArchConfig): + pass + + +@dataclass +class FluxVAEConfig(VAEConfig): + arch_config: FluxVAEArchConfig = field(default_factory=FluxVAEArchConfig) + + use_feature_cache: bool = True + + use_tiling: bool = False + use_temporal_tiling: bool = False + use_parallel_tiling: bool = False + + def __post_init__(self): + self.blend_num_frames = ( + self.tile_sample_min_num_frames - self.tile_sample_stride_num_frames + ) * 2 + + def post_init(self): + self.arch_config.vae_scale_factor = 2 ** ( + len(self.arch_config.block_out_channels) - 1 + ) + self.arch_config.spatial_compression_ratio = self.arch_config.vae_scale_factor + + +@dataclass +class Flux2VAEConfig(FluxVAEConfig): + arch_config: Flux2VAEArchConfig = field(default_factory=Flux2VAEArchConfig) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/hunyuanvae.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/hunyuanvae.py new file mode 100644 index 00000000..601b72d5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/hunyuanvae.py @@ -0,0 +1,41 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field + +from sglang.multimodal_gen.configs.models.vaes.base import VAEArchConfig, VAEConfig + + +@dataclass +class HunyuanVAEArchConfig(VAEArchConfig): + in_channels: int = 3 + out_channels: int = 3 + latent_channels: int = 16 + down_block_types: tuple[str, ...] = ( + "HunyuanVideoDownBlock3D", + "HunyuanVideoDownBlock3D", + "HunyuanVideoDownBlock3D", + "HunyuanVideoDownBlock3D", + ) + up_block_types: tuple[str, ...] = ( + "HunyuanVideoUpBlock3D", + "HunyuanVideoUpBlock3D", + "HunyuanVideoUpBlock3D", + "HunyuanVideoUpBlock3D", + ) + block_out_channels: tuple[int, ...] = (128, 256, 512, 512) + layers_per_block: int = 2 + act_fn: str = "silu" + norm_num_groups: int = 32 + scaling_factor: float = 0.476986 + spatial_compression_ratio: int = 8 + temporal_compression_ratio: int = 4 + mid_block_add_attention: bool = True + + def __post_init__(self): + self.spatial_compression_ratio: int = 2 ** (len(self.block_out_channels) - 1) + + +@dataclass +class HunyuanVAEConfig(VAEConfig): + arch_config: VAEArchConfig = field(default_factory=HunyuanVAEArchConfig) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/qwenimage.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/qwenimage.py new file mode 100644 index 00000000..d08ab5da --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/qwenimage.py @@ -0,0 +1,53 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field + +from sglang.multimodal_gen.configs.models.vaes.base import VAEArchConfig, VAEConfig + + +@dataclass +class QwenImageVAEArchConfig(VAEArchConfig): + spatial_compression_ratio: int = 1 + + base_dim: int = 96 + decoder_base_dim: int | None = None + z_dim: int = 16 + dim_mult: tuple[int, ...] = (1, 2, 4, 4) + num_res_blocks: int = 2 + attn_scales: tuple[float, ...] = () + temperal_downsample: tuple[bool, ...] = (False, True, True) + dropout: float = 0.0 + + is_residual: bool = False + in_channels: int = 3 + out_channels: int = 3 + patch_size: int | None = None + scale_factor_temporal: int = 4 + scale_factor_spatial: int = 8 + clip_output: bool = True + + +@dataclass +class QwenImageVAEConfig(VAEConfig): + arch_config: QwenImageVAEArchConfig = field(default_factory=QwenImageVAEArchConfig) + + use_feature_cache: bool = True + + use_tiling: bool = False + use_temporal_tiling: bool = False + use_parallel_tiling: bool = False + + def get_vae_scale_factor(self): + return 2 ** len(self.arch_config.temperal_downsample) + + def __post_init__(self): + self.blend_num_frames = ( + self.tile_sample_min_num_frames - self.tile_sample_stride_num_frames + ) * 2 + + def post_init(self): + self.arch_config.vae_scale_factor = 2 ** ( + len(self.arch_config.temperal_downsample) + ) + self.arch_config.spatial_compression_ratio = self.arch_config.vae_scale_factor diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/stepvideovae.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/stepvideovae.py new file mode 100644 index 00000000..6794e979 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/stepvideovae.py @@ -0,0 +1,31 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field + +from sglang.multimodal_gen.configs.models.vaes.base import VAEArchConfig, VAEConfig + + +@dataclass +class StepVideoVAEArchConfig(VAEArchConfig): + in_channels: int = 3 + out_channels: int = 3 + z_channels: int = 64 + num_res_blocks: int = 2 + version: int = 2 + frame_len: int = 17 + world_size: int = 1 + + spatial_compression_ratio: int = 16 + temporal_compression_ratio: int = 8 + + scaling_factor: float = 1.0 + + +@dataclass +class StepVideoVAEConfig(VAEConfig): + arch_config: VAEArchConfig = field(default_factory=StepVideoVAEArchConfig) + use_tiling: bool = False + use_temporal_tiling: bool = False + use_parallel_tiling: bool = False + use_temporal_scaling_frames: bool = False diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/wanvae.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/wanvae.py new file mode 100644 index 00000000..a1bd77eb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/models/vaes/wanvae.py @@ -0,0 +1,88 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field + +import torch + +from sglang.multimodal_gen.configs.models.vaes.base import VAEArchConfig, VAEConfig + + +@dataclass +class WanVAEArchConfig(VAEArchConfig): + base_dim: int = 96 + decoder_base_dim: int | None = None + z_dim: int = 16 + dim_mult: tuple[int, ...] = (1, 2, 4, 4) + num_res_blocks: int = 2 + attn_scales: tuple[float, ...] = () + temperal_downsample: tuple[bool, ...] = (False, True, True) + dropout: float = 0.0 + latents_mean: tuple[float, ...] = ( + -0.7571, + -0.7089, + -0.9113, + 0.1075, + -0.1745, + 0.9653, + -0.1517, + 1.5508, + 0.4134, + -0.0715, + 0.5517, + -0.3632, + -0.1922, + -0.9497, + 0.2503, + -0.2921, + ) + latents_std: tuple[float, ...] = ( + 2.8184, + 1.4541, + 2.3275, + 2.6558, + 1.2196, + 1.7708, + 2.6052, + 2.0743, + 3.2687, + 2.1526, + 2.8652, + 1.5579, + 1.6382, + 1.1253, + 2.8251, + 1.9160, + ) + is_residual: bool = False + in_channels: int = 3 + out_channels: int = 3 + patch_size: int | None = None + scale_factor_temporal: int = 4 + scale_factor_spatial: int = 8 + clip_output: bool = True + + def __post_init__(self): + self.scaling_factor: torch.tensor = 1.0 / torch.tensor(self.latents_std).view( + 1, self.z_dim, 1, 1, 1 + ) + self.shift_factor: torch.tensor = torch.tensor(self.latents_mean).view( + 1, self.z_dim, 1, 1, 1 + ) + self.temporal_compression_ratio = self.scale_factor_temporal + self.spatial_compression_ratio = self.scale_factor_spatial + + +@dataclass +class WanVAEConfig(VAEConfig): + arch_config: WanVAEArchConfig = field(default_factory=WanVAEArchConfig) + use_feature_cache: bool = True + + use_tiling: bool = False + use_temporal_tiling: bool = False + use_parallel_tiling: bool = False + + def __post_init__(self): + self.blend_num_frames = ( + self.tile_sample_min_num_frames - self.tile_sample_stride_num_frames + ) * 2 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/__init__.py new file mode 100644 index 00000000..250c0d7a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/__init__.py @@ -0,0 +1,35 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +from sglang.multimodal_gen.configs.pipeline_configs.base import ( + PipelineConfig, + SlidingTileAttnConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.flux import FluxPipelineConfig +from sglang.multimodal_gen.configs.pipeline_configs.hunyuan import ( + FastHunyuanConfig, + HunyuanConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.stepvideo import StepVideoT2VConfig +from sglang.multimodal_gen.configs.pipeline_configs.wan import ( + SelfForcingWanT2V480PConfig, + WanI2V480PConfig, + WanI2V720PConfig, + WanT2V480PConfig, + WanT2V720PConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.zimage import ZImagePipelineConfig + +__all__ = [ + "HunyuanConfig", + "FastHunyuanConfig", + "FluxPipelineConfig", + "PipelineConfig", + "SlidingTileAttnConfig", + "WanT2V480PConfig", + "WanI2V480PConfig", + "WanT2V720PConfig", + "WanI2V720PConfig", + "StepVideoT2VConfig", + "SelfForcingWanT2V480PConfig", + "ZImagePipelineConfig", +] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/base.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/base.py new file mode 100644 index 00000000..5ffc9278 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/base.py @@ -0,0 +1,669 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +import json +from collections.abc import Callable +from dataclasses import asdict, dataclass, field, fields +from enum import Enum, auto +from typing import Any + +import numpy as np +import PIL +import torch +from einops import rearrange + +from sglang.multimodal_gen.configs.models import ( + DiTConfig, + EncoderConfig, + ModelConfig, + VAEConfig, +) +from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput +from sglang.multimodal_gen.configs.utils import update_config_from_args +from sglang.multimodal_gen.runtime.distributed import ( + get_sp_parallel_rank, + get_sp_world_size, + sequence_model_parallel_all_gather, +) +from sglang.multimodal_gen.runtime.models.vision_utils import get_default_height_width +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.utils import ( + FlexibleArgumentParser, + StoreBoolean, + shallow_asdict, +) + +logger = init_logger(__name__) + + +# NOTE: possible duplication with DataType, WorkloadType +# this may focus on the model's original ability +class ModelTaskType(Enum): + I2V = auto() # Image to Video + T2V = auto() # Text to Video + TI2V = auto() # Text and Image to Video + T2I = auto() # Text to Image + I2I = auto() # Image to Image + + def is_image_gen(self): + return self == ModelTaskType.T2I or self == ModelTaskType.I2I + + +class STA_Mode(str, Enum): + """STA (Sliding Tile Attention) modes.""" + + STA_INFERENCE = "STA_inference" + STA_SEARCHING = "STA_searching" + STA_TUNING = "STA_tuning" + STA_TUNING_CFG = "STA_tuning_cfg" + NONE = None + + +def preprocess_text(prompt: str) -> str: + return prompt + + +def postprocess_text(output: BaseEncoderOutput, _text_inputs) -> torch.tensor: + raise NotImplementedError + + +def shard_rotary_emb_for_sp(emb): + """ + Shard rotary embeddings [S, D] along sequence for SP. + If S is not divisible by SP degree, pad by repeating the last row. + """ + # Sequence Parallelism: slice image RoPE to local shard if enabled + try: + from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + get_sp_parallel_rank, + get_sp_world_size, + ) + + sp_world_size = get_sp_world_size() + except Exception: + sp_world_size = 1 + seq_len = emb.shape[0] + if seq_len % sp_world_size != 0: + pad_len = sp_world_size - (seq_len % sp_world_size) + pad = emb[-1:].repeat(pad_len, 1) + emb = torch.cat([emb, pad], dim=0) + if sp_world_size > 1: + try: + rank = get_sp_parallel_rank() + except Exception: + rank = 0 + seq_len = emb.shape[0] + local_len = seq_len // sp_world_size + start = rank * local_len + end = start + local_len + emb = emb[start:end] + return emb + else: + return emb + + +# config for a single pipeline +@dataclass +class PipelineConfig: + """The base configuration class for a generation pipeline.""" + + task_type: ModelTaskType = ModelTaskType.I2I + + model_path: str = "" + pipeline_config_path: str | None = None + + # generation parameters + # controls the timestep embedding generation + should_use_guidance: bool = True + embedded_cfg_scale: float = 6.0 + flow_shift: float | None = None + disable_autocast: bool = False + + # Model configuration + dit_config: DiTConfig = field(default_factory=DiTConfig) + dit_precision: str = "bf16" + + # VAE configuration + vae_config: VAEConfig = field(default_factory=VAEConfig) + vae_precision: str = "fp32" + vae_tiling: bool = True + vae_sp: bool = True + + # Image encoder configuration + image_encoder_config: EncoderConfig = field(default_factory=EncoderConfig) + image_encoder_precision: str = "fp32" + + # Text encoder configuration + DEFAULT_TEXT_ENCODER_PRECISIONS = ("fp32",) + text_encoder_configs: tuple[EncoderConfig, ...] = field( + default_factory=lambda: (EncoderConfig(),) + ) + # See PRECISION_TO_TYPE for detailed mapping + text_encoder_precisions: tuple[str, ...] = field(default_factory=lambda: ("fp32",)) + text_encoder_extra_args: list[dict] = field(default_factory=lambda: [{}]) + + # image encoding + image_encoder_extra_args: dict = field(default_factory=lambda: {}) + + def postprocess_image(self, image): + return image.last_hidden_state + + preprocess_text_funcs: tuple[Callable[[str], str], ...] = field( + default_factory=lambda: (preprocess_text,) + ) + + # get prompt_embeds from encoder output + postprocess_text_funcs: tuple[Callable[[BaseEncoderOutput], torch.tensor], ...] = ( + field(default_factory=lambda: (postprocess_text,)) + ) + + # StepVideo specific parameters + pos_magic: str | None = None + neg_magic: str | None = None + timesteps_scale: bool | None = None + + # STA (Sliding Tile Attention) parameters + mask_strategy_file_path: str | None = None + STA_mode: STA_Mode = STA_Mode.STA_INFERENCE + skip_time_steps: int = 15 + + # DMD parameters + dmd_denoising_steps: list[int] | None = field(default=None) + + # Wan2.2 TI2V parameters + boundary_ratio: float | None = None + + # Compilation + # enable_torch_compile: bool = False + + # calculate the adjust size for condition image + # width: original condition image width + # height: original condition image height + def calculate_condition_image_size(self, image, width, height) -> tuple[int, int]: + vae_scale_factor = self.vae_config.arch_config.spatial_compression_ratio + height, width = get_default_height_width(image, vae_scale_factor, height, width) + return width, height + + ## For timestep preparation stage + + def prepare_sigmas(self, sigmas, num_inference_steps): + return sigmas + + ## For ImageVAEEncodingStage + def preprocess_condition_image( + self, image, target_width, target_height, _vae_image_processor + ): + """ + preprocess the condition image, returns (image, final_image_width, final_image_height) + """ + return image.resize( + (target_width, target_height), PIL.Image.Resampling.LANCZOS + ), (target_width, target_height) + + def prepare_image_processor_kwargs(self, batch): + return {} + + def postprocess_image_latent(self, latent_condition, batch): + vae_arch_config = self.vae_config.arch_config + spatial_compression_ratio = vae_arch_config.spatial_compression_ratio + temporal_compression_ratio = vae_arch_config.temporal_compression_ratio + num_frames = batch.num_frames + latent_height = batch.height // spatial_compression_ratio + latent_width = batch.width // spatial_compression_ratio + mask_lat_size = torch.ones(1, 1, num_frames, latent_height, latent_width) + mask_lat_size[:, :, 1:] = 0 + first_frame_mask = mask_lat_size[:, :, 0:1] + first_frame_mask = torch.repeat_interleave( + first_frame_mask, + repeats=temporal_compression_ratio, + dim=2, + ) + mask_lat_size = torch.concat( + [first_frame_mask, mask_lat_size[:, :, 1:, :]], dim=2 + ) + mask_lat_size = mask_lat_size.view( + 1, + -1, + temporal_compression_ratio, + latent_height, + latent_width, + ) + mask_lat_size = mask_lat_size.transpose(1, 2) + mask_lat_size = mask_lat_size.to(latent_condition.device) + image_latents = torch.concat([mask_lat_size, latent_condition], dim=1) + return image_latents + + def slice_noise_pred(self, noise, latents): + return noise + + def adjust_num_frames(self, num_frames): + return num_frames + + # tokenize the prompt + def tokenize_prompt(self, prompt: list[str], tokenizer, tok_kwargs) -> dict: + return tokenizer(prompt, **tok_kwargs) + + def prepare_latent_shape(self, batch, batch_size, num_frames): + height = batch.height // self.vae_config.arch_config.spatial_compression_ratio + width = batch.width // self.vae_config.arch_config.spatial_compression_ratio + + # Calculate latent shape + shape = ( + batch_size, + self.dit_config.num_channels_latents, + num_frames, + height, + width, + ) + + return shape + + def get_decode_scale_and_shift(self, device, dtype, vae): + vae_arch_config = self.vae_config.arch_config + scaling_factor = getattr(vae_arch_config, "scaling_factor", None) + if scaling_factor is None: + scaling_factor = getattr(vae, "scaling_factor", None) + + shift_factor = getattr(vae_arch_config, "shift_factor", None) + if shift_factor is None: + shift_factor = getattr(vae, "shift_factor", None) + return scaling_factor, shift_factor + + # called after latents are prepared + def maybe_pack_latents(self, latents, batch_size, batch): + return latents + + def maybe_prepare_latent_ids(self, latents): + return None + + # called after vae encode + def postprocess_vae_encode(self, image_latents, vae): + return image_latents + + # called after scale_and_shift, before vae decoding + def preprocess_decoding(self, latents): + return latents + + def gather_latents_for_sp(self, latents): + # For video latents [B, C, T_local, H, W], gather along time dim=2 + latents = sequence_model_parallel_all_gather(latents, dim=2) + return latents + + def shard_latents_for_sp(self, batch, latents): + # general logic for video models + sp_world_size, rank_in_sp_group = get_sp_world_size(), get_sp_parallel_rank() + if latents.dim() != 5: + return latents, False + time_dim = latents.shape[2] + if time_dim > 0 and time_dim % sp_world_size == 0: + sharded_tensor = rearrange( + latents, "b c (n t) h w -> b c n t h w", n=sp_world_size + ).contiguous() + sharded_tensor = sharded_tensor[:, :, rank_in_sp_group, :, :, :] + return sharded_tensor, True + return latents, False + + def get_pos_prompt_embeds(self, batch): + return batch.prompt_embeds + + def get_neg_prompt_embeds(self, batch): + return batch.negative_prompt_embeds + + def post_denoising_loop(self, latents, batch): + return latents + + def prepare_pos_cond_kwargs(self, batch, device, rotary_emb, dtype): + return {} + + def prepare_neg_cond_kwargs(self, batch, device, rotary_emb, dtype): + return {} + + @staticmethod + def add_cli_args( + parser: FlexibleArgumentParser, prefix: str = "" + ) -> FlexibleArgumentParser: + prefix_with_dot = f"{prefix}." if (prefix.strip() != "") else "" + + # model_path will be conflicting with the model_path in ServerArgs, + # so we add it separately if prefix is not empty + if prefix_with_dot != "": + parser.add_argument( + f"--{prefix_with_dot}model-path", + type=str, + dest=f"{prefix_with_dot.replace('-', '_')}model_path", + default=PipelineConfig.model_path, + help="Path to the pretrained model", + ) + + parser.add_argument( + f"--{prefix_with_dot}pipeline-config-path", + type=str, + dest=f"{prefix_with_dot.replace('-', '_')}pipeline_config_path", + default=PipelineConfig.pipeline_config_path, + help="Path to the pipeline config", + ) + parser.add_argument( + f"--{prefix_with_dot}embedded-cfg-scale", + type=float, + dest=f"{prefix_with_dot.replace('-', '_')}embedded_cfg_scale", + default=PipelineConfig.embedded_cfg_scale, + help="Embedded CFG scale", + ) + parser.add_argument( + f"--{prefix_with_dot}flow-shift", + type=float, + dest=f"{prefix_with_dot.replace('-', '_')}flow_shift", + default=PipelineConfig.flow_shift, + help="Flow shift parameter", + ) + + # DiT configuration + parser.add_argument( + f"--{prefix_with_dot}dit-precision", + type=str, + dest=f"{prefix_with_dot.replace('-', '_')}dit_precision", + default=PipelineConfig.dit_precision, + choices=["fp32", "fp16", "bf16"], + help="Precision for the DiT model", + ) + + # VAE configuration + parser.add_argument( + f"--{prefix_with_dot}vae-precision", + type=str, + dest=f"{prefix_with_dot.replace('-', '_')}vae_precision", + default=PipelineConfig.vae_precision, + choices=["fp32", "fp16", "bf16"], + help="Precision for VAE", + ) + parser.add_argument( + f"--{prefix_with_dot}vae-tiling", + action=StoreBoolean, + dest=f"{prefix_with_dot.replace('-', '_')}vae_tiling", + default=PipelineConfig.vae_tiling, + help="Enable VAE tiling", + ) + parser.add_argument( + f"--{prefix_with_dot}vae-sp", + action=StoreBoolean, + dest=f"{prefix_with_dot.replace('-', '_')}vae_sp", + help="Enable VAE spatial parallelism", + ) + + # Text encoder configuration + parser.add_argument( + f"--{prefix_with_dot}text-encoder-precisions", + nargs="+", + type=str, + dest=f"{prefix_with_dot.replace('-', '_')}text_encoder_precisions", + default=PipelineConfig.DEFAULT_TEXT_ENCODER_PRECISIONS, + choices=["fp32", "fp16", "bf16"], + help="Precision for each text encoder", + ) + + # Image encoder configuration + parser.add_argument( + f"--{prefix_with_dot}image-encoder-precision", + type=str, + dest=f"{prefix_with_dot.replace('-', '_')}image_encoder_precision", + default=PipelineConfig.image_encoder_precision, + choices=["fp32", "fp16", "bf16"], + help="Precision for image encoder", + ) + parser.add_argument( + f"--{prefix_with_dot}pos_magic", + type=str, + dest=f"{prefix_with_dot.replace('-', '_')}pos_magic", + default=PipelineConfig.pos_magic, + help="Positive magic prompt for sampling, used in stepvideo", + ) + parser.add_argument( + f"--{prefix_with_dot}neg_magic", + type=str, + dest=f"{prefix_with_dot.replace('-', '_')}neg_magic", + default=PipelineConfig.neg_magic, + help="Negative magic prompt for sampling, used in stepvideo", + ) + parser.add_argument( + f"--{prefix_with_dot}timesteps_scale", + type=bool, + dest=f"{prefix_with_dot.replace('-', '_')}timesteps_scale", + default=PipelineConfig.timesteps_scale, + help="Bool for applying scheduler scale in set_timesteps, used in stepvideo", + ) + + # DMD parameters + parser.add_argument( + f"--{prefix_with_dot}dmd-denoising-steps", + type=parse_int_list, + default=PipelineConfig.dmd_denoising_steps, + help="Comma-separated list of denoising steps (e.g., '1000,757,522')", + ) + + # Add VAE configuration arguments + from sglang.multimodal_gen.configs.models.vaes.base import VAEConfig + + VAEConfig.add_cli_args(parser, prefix=f"{prefix_with_dot}vae-config") + + # Add DiT configuration arguments + from sglang.multimodal_gen.configs.models.dits.base import DiTConfig + + DiTConfig.add_cli_args(parser, prefix=f"{prefix_with_dot}dit-config") + + return parser + + def update_config_from_dict(self, args: dict[str, Any], prefix: str = "") -> None: + prefix_with_dot = f"{prefix}." if (prefix.strip() != "") else "" + update_config_from_args(self, args, prefix, pop_args=True) + update_config_from_args( + self.vae_config, args, f"{prefix_with_dot}vae_config", pop_args=True + ) + update_config_from_args( + self.dit_config, args, f"{prefix_with_dot}dit_config", pop_args=True + ) + + @classmethod + def from_kwargs( + cls, kwargs: dict[str, Any], config_cli_prefix: str = "" + ) -> "PipelineConfig": + """ + Load PipelineConfig from kwargs Dictionary. + kwargs: dictionary of kwargs + config_cli_prefix: prefix of CLI arguments for this PipelineConfig instance + """ + from sglang.multimodal_gen.registry import get_model_info + + prefix_with_dot = ( + f"{config_cli_prefix}." if (config_cli_prefix.strip() != "") else "" + ) + model_path: str | None = kwargs.get( + prefix_with_dot + "model_path", None + ) or kwargs.get("model_path") + pipeline_config_or_path: str | PipelineConfig | dict[str, Any] | None = ( + kwargs.get(prefix_with_dot + "pipeline_config", None) + or kwargs.get("pipeline_config") + ) + if model_path is None: + raise ValueError("model_path is required in kwargs") + + # 1. Get the pipeline config class from the registry + model_info = get_model_info(model_path) + + pipeline_config = model_info.pipeline_config_cls() + + # 2. Load PipelineConfig from a json file or a PipelineConfig object if provided + if isinstance(pipeline_config_or_path, str): + pipeline_config.load_from_json(pipeline_config_or_path) + kwargs[prefix_with_dot + "pipeline_config_path"] = pipeline_config_or_path + elif isinstance(pipeline_config_or_path, PipelineConfig): + pipeline_config = pipeline_config_or_path + elif isinstance(pipeline_config_or_path, dict): + pipeline_config.update_pipeline_config(pipeline_config_or_path) + + # 3. Update PipelineConfig from CLI arguments if provided + kwargs[prefix_with_dot + "model_path"] = model_path + pipeline_config.update_config_from_dict(kwargs, config_cli_prefix) + return pipeline_config + + def check_pipeline_config(self) -> None: + if self.vae_sp and not self.vae_tiling: + raise ValueError( + "Currently enabling vae_sp requires enabling vae_tiling, please set --vae-tiling to True." + ) + + if len(self.text_encoder_configs) != len(self.text_encoder_precisions): + raise ValueError( + f"Length of text encoder configs ({len(self.text_encoder_configs)}) must be equal to length of text encoder precisions ({len(self.text_encoder_precisions)})" + ) + + if len(self.text_encoder_configs) != len(self.preprocess_text_funcs): + raise ValueError( + f"Length of text encoder configs ({len(self.text_encoder_configs)}) must be equal to length of text preprocessing functions ({len(self.preprocess_text_funcs)})" + ) + + if len(self.preprocess_text_funcs) != len(self.postprocess_text_funcs): + raise ValueError( + f"Length of text postprocess functions ({len(self.postprocess_text_funcs)}) must be equal to length of text preprocessing functions ({len(self.preprocess_text_funcs)})" + ) + + def dump_to_json(self, file_path: str): + output_dict = shallow_asdict(self) + del_keys = [] + for key, value in output_dict.items(): + if isinstance(value, ModelConfig): + model_dict = asdict(value) + # Model Arch Config should be hidden away from the users + model_dict.pop("arch_config") + output_dict[key] = model_dict + elif isinstance(value, tuple) and all( + isinstance(v, ModelConfig) for v in value + ): + model_dicts = [] + for v in value: + model_dict = asdict(v) + # Model Arch Config should be hidden away from the users + model_dict.pop("arch_config") + model_dicts.append(model_dict) + output_dict[key] = model_dicts + elif isinstance(value, tuple) and all(callable(f) for f in value): + # Skip dumping functions + del_keys.append(key) + + for key in del_keys: + output_dict.pop(key, None) + + with open(file_path, "w") as f: + json.dump(output_dict, f, indent=2) + + def load_from_json(self, file_path: str): + with open(file_path) as f: + input_pipeline_dict = json.load(f) + self.update_pipeline_config(input_pipeline_dict) + + def update_pipeline_config(self, source_pipeline_dict: dict[str, Any]) -> None: + for f in fields(self): + key = f.name + if key in source_pipeline_dict: + current_value = getattr(self, key) + new_value = source_pipeline_dict[key] + + # If it's a nested ModelConfig, update it recursively + if isinstance(current_value, ModelConfig): + current_value.update_model_config(new_value) + elif isinstance(current_value, tuple) and all( + isinstance(v, ModelConfig) for v in current_value + ): + assert len(current_value) == len( + new_value + ), "Users shouldn't delete or add text encoder config objects in your json" + for target_config, source_config in zip( + current_value, new_value, strict=True + ): + target_config.update_model_config(source_config) + else: + setattr(self, key, new_value) + + if hasattr(self, "__post_init__"): + self.__post_init__() + + +@dataclass +class ImagePipelineConfig(PipelineConfig): + """Base config for image generation pipelines with token-like latents [B, S, D].""" + + def _prepare_sigmas(self, sigmas, num_inference_steps): + sigmas = ( + np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) + if sigmas is None + else sigmas + ) + return sigmas + + def shard_latents_for_sp(self, batch, latents): + sp_world_size, rank_in_sp_group = get_sp_world_size(), get_sp_parallel_rank() + seq_len = latents.shape[1] + + # Pad to next multiple of SP degree if needed + if seq_len % sp_world_size != 0: + pad_len = sp_world_size - (seq_len % sp_world_size) + pad = torch.zeros( + (latents.shape[0], pad_len, latents.shape[2]), + dtype=latents.dtype, + device=latents.device, + ) + latents = torch.cat([latents, pad], dim=1) + # Record padding length for later unpad + batch.sp_seq_pad = int(getattr(batch, "sp_seq_pad", 0)) + pad_len + + sharded_tensor = rearrange( + latents, "b (n s) d -> b n s d", n=sp_world_size + ).contiguous() + sharded_tensor = sharded_tensor[:, rank_in_sp_group, :, :] + return sharded_tensor, True + + def gather_latents_for_sp(self, latents): + # For image latents [B, S_local, D], gather along sequence dim=1 + latents = sequence_model_parallel_all_gather(latents, dim=1) + return latents + + def _unpad_and_unpack_latents(self, latents, batch): + vae_scale_factor = self.vae_config.arch_config.vae_scale_factor + channels = self.dit_config.arch_config.in_channels + batch_size = latents.shape[0] + + height = 2 * (int(batch.height) // (vae_scale_factor * 2)) + width = 2 * (int(batch.width) // (vae_scale_factor * 2)) + + # If SP padding was applied, remove extra tokens before reshaping + target_tokens = (height // 2) * (width // 2) + if latents.shape[1] > target_tokens: + latents = latents[:, :target_tokens, :] + + latents = latents.view(batch_size, height // 2, width // 2, channels // 4, 2, 2) + latents = latents.permute(0, 3, 1, 4, 2, 5) + return latents, batch_size, channels, height, width + + +@dataclass +class SlidingTileAttnConfig(PipelineConfig): + """Configuration for sliding tile attention.""" + + # Override any BaseConfig defaults as needed + # Add sliding tile specific parameters + window_size: int = 16 + stride: int = 8 + + # You can provide custom defaults for inherited fields + height: int = 576 + width: int = 1024 + + # Additional configuration specific to sliding tile attention + pad_to_square: bool = False + use_overlap_optimization: bool = True + + +def parse_int_list(value: str) -> list[int]: + """Parse a comma-separated string of integers into a list.""" + if not value: + return [] + return [int(x.strip()) for x in value.split(",")] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/flux.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/flux.py new file mode 100644 index 00000000..d4bed105 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/flux.py @@ -0,0 +1,576 @@ +import math +from dataclasses import dataclass, field +from typing import Callable, List, Optional + +import PIL +import torch +from diffusers.image_processor import VaeImageProcessor + +from sglang.multimodal_gen.configs.models import DiTConfig, EncoderConfig, VAEConfig +from sglang.multimodal_gen.configs.models.dits.flux import FluxConfig +from sglang.multimodal_gen.configs.models.encoders import ( + BaseEncoderOutput, + CLIPTextConfig, + T5Config, + TextEncoderConfig, +) +from sglang.multimodal_gen.configs.models.encoders.base import TextEncoderArchConfig +from sglang.multimodal_gen.configs.models.encoders.qwen_image import ( + _is_transformer_layer, +) +from sglang.multimodal_gen.configs.models.vaes.flux import Flux2VAEConfig, FluxVAEConfig +from sglang.multimodal_gen.configs.pipeline_configs.base import ( + ImagePipelineConfig, + ModelTaskType, + preprocess_text, + shard_rotary_emb_for_sp, +) +from sglang.multimodal_gen.configs.pipeline_configs.hunyuan import ( + clip_postprocess_text, + clip_preprocess_text, +) +from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import _pack_latents +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device + + +def t5_postprocess_text(outputs: BaseEncoderOutput, _text_inputs) -> torch.Tensor: + return outputs.last_hidden_state + + +@dataclass +class FluxPipelineConfig(ImagePipelineConfig): + """Configuration for the FLUX pipeline.""" + + embedded_cfg_scale: float = 3.5 + + task_type: ModelTaskType = ModelTaskType.T2I + + vae_tiling: bool = False + + vae_sp: bool = False + + dit_config: DiTConfig = field(default_factory=FluxConfig) + # VAE + vae_config: VAEConfig = field(default_factory=FluxVAEConfig) + + # Text encoding stage + text_encoder_configs: tuple[EncoderConfig, ...] = field( + default_factory=lambda: (CLIPTextConfig(), T5Config()) + ) + + text_encoder_precisions: tuple[str, ...] = field( + default_factory=lambda: ("bf16", "bf16") + ) + + preprocess_text_funcs: tuple[Callable[[str], str], ...] = field( + default_factory=lambda: (clip_preprocess_text, preprocess_text), + ) + + postprocess_text_funcs: tuple[Callable[[str], str], ...] = field( + default_factory=lambda: (clip_postprocess_text, t5_postprocess_text) + ) + + text_encoder_extra_args: list[dict] = field( + default_factory=lambda: [ + dict( + max_length=77, + padding="max_length", + truncation=True, + return_overflowing_tokens=False, + return_length=False, + ), + None, + ] + ) + + def prepare_sigmas(self, sigmas, num_inference_steps): + return self._prepare_sigmas(sigmas, num_inference_steps) + + def prepare_latent_shape(self, batch, batch_size, num_frames): + height = 2 * ( + batch.height // (self.vae_config.arch_config.vae_scale_factor * 2) + ) + width = 2 * (batch.width // (self.vae_config.arch_config.vae_scale_factor * 2)) + num_channels_latents = self.dit_config.arch_config.in_channels // 4 + shape = (batch_size, num_channels_latents, height, width) + return shape + + def maybe_pack_latents(self, latents, batch_size, batch): + height = 2 * ( + batch.height // (self.vae_config.arch_config.vae_scale_factor * 2) + ) + width = 2 * (batch.width // (self.vae_config.arch_config.vae_scale_factor * 2)) + num_channels_latents = self.dit_config.arch_config.in_channels // 4 + # pack latents + return _pack_latents(latents, batch_size, num_channels_latents, height, width) + + def get_pos_prompt_embeds(self, batch): + return batch.prompt_embeds[1] + + def get_neg_prompt_embeds(self, batch): + return batch.negative_prompt_embeds[1] + + def _prepare_latent_image_ids(self, original_height, original_width, device): + vae_scale_factor = self.vae_config.arch_config.vae_scale_factor + height = int(original_height) // (vae_scale_factor * 2) + width = int(original_width) // (vae_scale_factor * 2) + latent_image_ids = torch.zeros(height, width, 3, device=device) + latent_image_ids[..., 1] = ( + latent_image_ids[..., 1] + torch.arange(height, device=device)[:, None] + ) + latent_image_ids[..., 2] = ( + latent_image_ids[..., 2] + torch.arange(width, device=device)[None, :] + ) + + latent_image_id_height, latent_image_id_width, latent_image_id_channels = ( + latent_image_ids.shape + ) + + latent_image_ids = latent_image_ids.reshape( + latent_image_id_height * latent_image_id_width, latent_image_id_channels + ) + + return latent_image_ids + + def get_freqs_cis(self, prompt_embeds, width, height, device, rotary_emb, batch): + txt_ids = torch.zeros(prompt_embeds.shape[1], 3, device=device) + img_ids = self._prepare_latent_image_ids( + original_height=height, + original_width=width, + device=device, + ) + + # NOTE(mick): prepare it here, to avoid unnecessary computations + img_cos, img_sin = rotary_emb.forward(img_ids) + img_cos = shard_rotary_emb_for_sp(img_cos) + img_sin = shard_rotary_emb_for_sp(img_sin) + + txt_cos, txt_sin = rotary_emb.forward(txt_ids) + + cos = torch.cat([txt_cos, img_cos], dim=0).to(device=device) + sin = torch.cat([txt_sin, img_sin], dim=0).to(device=device) + return cos, sin + + def post_denoising_loop(self, latents, batch): + # unpack latents for flux + ( + latents, + batch_size, + channels, + height, + width, + ) = self._unpad_and_unpack_latents(latents, batch) + latents = latents.reshape(batch_size, channels // (2 * 2), height, width) + return latents + + def prepare_pos_cond_kwargs(self, batch, device, rotary_emb, dtype): + return { + "freqs_cis": self.get_freqs_cis( + batch.prompt_embeds[1], + batch.width, + batch.height, + device, + rotary_emb, + batch, + ), + "pooled_projections": ( + batch.pooled_embeds[0] if batch.pooled_embeds else None + ), + } + + def prepare_neg_cond_kwargs(self, batch, device, rotary_emb, dtype): + return { + "freqs_cis": self.get_freqs_cis( + batch.negative_prompt_embeds[1], + batch.width, + batch.height, + device, + rotary_emb, + batch, + ), + "pooled_projections": ( + batch.neg_pooled_embeds[0] if batch.neg_pooled_embeds else None + ), + } + + +def _prepare_latent_ids( + latents: torch.Tensor, # (B, C, H, W) +): + r""" + Generates 4D position coordinates (T, H, W, L) for latent tensors. + + Args: + latents (torch.Tensor): + Latent tensor of shape (B, C, H, W) + + Returns: + torch.Tensor: + Position IDs tensor of shape (B, H*W, 4) All batches share the same coordinate structure: T=0, + H=[0..H-1], W=[0..W-1], L=0 + """ + + batch_size, _, height, width = latents.shape + + t = torch.arange(1) # [0] - time dimension + h = torch.arange(height) + w = torch.arange(width) + l = torch.arange(1) # [0] - layer dimension + + # Create position IDs: (H*W, 4) + latent_ids = torch.cartesian_prod(t, h, w, l) + + # Expand to batch: (B, H*W, 4) + latent_ids = latent_ids.unsqueeze(0).expand(batch_size, -1, -1) + return latent_ids + + +def _unpack_latents_with_ids( + x: torch.Tensor, x_ids: torch.Tensor +) -> list[torch.Tensor]: + """ + using position ids to scatter tokens into place + """ + x_list = [] + x_ids = x_ids.to(device=x.device) + for data, pos in zip(x, x_ids): + _, ch = data.shape # noqa: F841 + h_ids = pos[:, 1].to(torch.int64) + w_ids = pos[:, 2].to(torch.int64) + + h = torch.max(h_ids) + 1 + w = torch.max(w_ids) + 1 + + flat_ids = h_ids * w + w_ids + + out = torch.zeros((h * w, ch), device=data.device, dtype=data.dtype) + out.scatter_(0, flat_ids.unsqueeze(1).expand(-1, ch), data) + + # reshape from (H * W, C) to (H, W, C) and permute to (C, H, W) + + out = out.view(h, w, ch).permute(2, 0, 1) + x_list.append(out) + + return torch.stack(x_list, dim=0) + + +def _patchify_latents(latents): + batch_size, num_channels_latents, height, width = latents.shape + latents = latents.view( + batch_size, num_channels_latents, height // 2, 2, width // 2, 2 + ) + latents = latents.permute(0, 1, 3, 5, 2, 4) + latents = latents.reshape( + batch_size, num_channels_latents * 4, height // 2, width // 2 + ) + return latents + + +def _unpatchify_latents(latents): + batch_size, num_channels_latents, height, width = latents.shape + latents = latents.reshape( + batch_size, num_channels_latents // (2 * 2), 2, 2, height, width + ) + latents = latents.permute(0, 1, 4, 2, 5, 3) + latents = latents.reshape( + batch_size, num_channels_latents // (2 * 2), height * 2, width * 2 + ) + return latents + + +def _prepare_text_ids( + x: torch.Tensor, # (B, L, D) or (L, D) + t_coord: Optional[torch.Tensor] = None, +): + B, L, _ = x.shape + out_ids = [] + + for i in range(B): + t = torch.arange(1) if t_coord is None else t_coord[i] + h = torch.arange(1) + w = torch.arange(1) + l = torch.arange(L) + + coords = torch.cartesian_prod(t, h, w, l) + out_ids.append(coords) + + return torch.stack(out_ids) + + +def _prepare_image_ids( + image_latents: List[torch.Tensor], # [(1, C, H, W), (1, C, H, W), ...] + scale: int = 10, +): + if not isinstance(image_latents, list): + raise ValueError( + f"Expected `image_latents` to be a list, got {type(image_latents)}." + ) + + # create time offset for each reference image + t_coords = [scale + scale * t for t in torch.arange(0, len(image_latents))] + t_coords = [t.view(-1) for t in t_coords] + + image_latent_ids = [] + for x, t in zip(image_latents, t_coords): + x = x.squeeze(0) + _, height, width = x.shape + + x_ids = torch.cartesian_prod( + t, torch.arange(height), torch.arange(width), torch.arange(1) + ) + image_latent_ids.append(x_ids) + + image_latent_ids = torch.cat(image_latent_ids, dim=0) + image_latent_ids = image_latent_ids.unsqueeze(0) + + return image_latent_ids + + +def flux2_postprocess_text(outputs: BaseEncoderOutput, _text_inputs) -> torch.Tensor: + hidden_states_layers: list[int] = [10, 20, 30] + + out = torch.stack([outputs.hidden_states[k] for k in hidden_states_layers], dim=1) + batch_size, num_channels, seq_len, hidden_dim = out.shape + prompt_embeds = out.permute(0, 2, 1, 3).reshape( + batch_size, seq_len, num_channels * hidden_dim + ) + + return prompt_embeds + + +@dataclass +class Flux2MistralTextArchConfig(TextEncoderArchConfig): + stacked_params_mapping: list[tuple[str, str, str]] = field( + default_factory=lambda: [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ] + ) + _fsdp_shard_conditions: list = field( + default_factory=lambda: [_is_transformer_layer] + ) + + def __post_init__(self): + self.tokenizer_kwargs = { + "padding": "max_length", + "truncation": True, + "max_length": 512, + "add_special_tokens": True, + "return_attention_mask": True, + "return_tensors": "pt", + } + + +@dataclass +class Flux2MistralTextConfig(TextEncoderConfig): + arch_config: TextEncoderArchConfig = field( + default_factory=Flux2MistralTextArchConfig + ) + + +def format_text_input(prompts: List[str], system_message: str = None): + # Remove [IMG] tokens from prompts to avoid Pixtral validation issues + # when truncation is enabled. The processor counts [IMG] tokens and fails + # if the count changes after truncation. + cleaned_txt = [prompt.replace("[IMG]", "") for prompt in prompts] + + return [ + [ + { + "role": "system", + "content": [{"type": "text", "text": system_message}], + }, + {"role": "user", "content": [{"type": "text", "text": prompt}]}, + ] + for prompt in cleaned_txt + ] + + +def flux_2_preprocess_text(prompt: str): + system_message = "You are an AI that reasons about image descriptions. You give structured responses focusing on object relationships, object attribution and actions without speculation." + return format_text_input([prompt], system_message=system_message) + + +# Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage.QwenImagePipeline._pack_latents +def flux2_pack_latents(latents): + batch_size, num_channels, height, width = latents.shape + latents = latents.reshape(batch_size, num_channels, height * width).permute(0, 2, 1) + + return latents + + +@dataclass +class Flux2PipelineConfig(FluxPipelineConfig): + embedded_cfg_scale: float = 4.0 + + task_type: ModelTaskType = ModelTaskType.I2I + + text_encoder_precisions: tuple[str, ...] = field(default_factory=lambda: ("bf16",)) + + text_encoder_configs: tuple[EncoderConfig, ...] = field( + default_factory=lambda: (Flux2MistralTextConfig(),) + ) + preprocess_text_funcs: tuple[Callable[[str], str], ...] = field( + default_factory=lambda: (flux_2_preprocess_text,), + ) + + postprocess_text_funcs: tuple[Callable[[str], str], ...] = field( + default_factory=lambda: (flux2_postprocess_text,) + ) + vae_config: VAEConfig = field(default_factory=Flux2VAEConfig) + + def tokenize_prompt(self, prompts: list[str], tokenizer, tok_kwargs) -> dict: + # flatten to 1-d list + prompts = [p for prompt in prompts for p in prompt] + inputs = tokenizer.apply_chat_template( + prompts, + add_generation_prompt=False, + tokenize=True, + return_dict=True, + return_tensors="pt", + padding="max_length", + truncation=True, + # 2048 from official github repo, 512 from diffusers + max_length=512, + ) + + return inputs + + def prepare_latent_shape(self, batch, batch_size, num_frames): + height = 2 * ( + batch.height // (self.vae_config.arch_config.vae_scale_factor * 2) + ) + width = 2 * (batch.width // (self.vae_config.arch_config.vae_scale_factor * 2)) + num_channels_latents = self.dit_config.arch_config.in_channels + shape = (batch_size, num_channels_latents, height // 2, width // 2) + return shape + + def get_pos_prompt_embeds(self, batch): + return batch.prompt_embeds[0] + + def get_neg_prompt_embeds(self, batch): + return batch.negative_prompt_embeds[0] + + def calculate_condition_image_size( + self, image, width, height + ) -> Optional[tuple[int, int]]: + target_area: int = 1024 * 1024 + if width is not None and height is not None: + if width * height > target_area: + scale = math.sqrt(target_area / (width * height)) + width = int(width * scale) + height = int(height * scale) + return width, height + + return None + + def preprocess_condition_image( + self, image, target_width, target_height, vae_image_processor: VaeImageProcessor + ): + img = image.resize((target_width, target_height), PIL.Image.Resampling.LANCZOS) + image_width, image_height = img.size + vae_scale_factor = self.vae_config.arch_config.vae_scale_factor + multiple_of = vae_scale_factor * 2 + image_width = (image_width // multiple_of) * multiple_of + image_height = (image_height // multiple_of) * multiple_of + img = vae_image_processor.preprocess( + img, height=image_height, width=image_width, resize_mode="crop" + ) + return img, (image_width, image_height) + + def postprocess_image_latent(self, latent_condition, batch): + batch_size = batch.batch_size + # get image_latent_ids right after scale & shift + image_latent_ids = _prepare_image_ids([latent_condition]) + image_latent_ids = image_latent_ids.repeat(batch_size, 1, 1) + image_latent_ids = image_latent_ids.to(get_local_torch_device()) + batch.condition_image_latent_ids = image_latent_ids + + # latent: (1, 128, 32, 32) + packed = self.maybe_pack_latents( + latent_condition, None, batch + ) # (1, 1024, 128) + packed = packed.squeeze(0) # (1024, 128) - remove batch dim + + # Concatenate all reference tokens along sequence dimension + image_latents = packed.unsqueeze(0) # (1, N*1024, 128) + image_latents = image_latents.repeat(batch_size, 1, 1) + return image_latents + + def get_freqs_cis(self, prompt_embeds, width, height, device, rotary_emb, batch): + + txt_ids = _prepare_text_ids(prompt_embeds).to(device=device) + + img_ids = batch.latent_ids + if batch.image_latent is not None: + image_latent_ids = batch.condition_image_latent_ids + img_ids = torch.cat([img_ids, image_latent_ids], dim=1).to(device=device) + + if img_ids.ndim == 3: + img_ids = img_ids[0] + if txt_ids.ndim == 3: + txt_ids = txt_ids[0] + + # NOTE(mick): prepare it here, to avoid unnecessary computations + img_cos, img_sin = rotary_emb.forward(img_ids) + img_cos = shard_rotary_emb_for_sp(img_cos) + img_sin = shard_rotary_emb_for_sp(img_sin) + + txt_cos, txt_sin = rotary_emb.forward(txt_ids) + + cos = torch.cat([txt_cos, img_cos], dim=0).to(device=device) + sin = torch.cat([txt_sin, img_sin], dim=0).to(device=device) + return cos, sin + + def prepare_pos_cond_kwargs(self, batch, device, rotary_emb, dtype): + return { + "freqs_cis": self.get_freqs_cis( + batch.prompt_embeds[0], + batch.width, + batch.height, + device, + rotary_emb, + batch, + ) + } + + def prepare_neg_cond_kwargs(self, batch, device, rotary_emb, dtype): + return {} + + def maybe_pack_latents(self, latents, batch_size, batch): + return flux2_pack_latents(latents) + + def maybe_prepare_latent_ids(self, latents): + return _prepare_latent_ids(latents) + + def postprocess_vae_encode(self, image_latents, vae): + # patchify + image_latents = _patchify_latents(image_latents) + return image_latents + + def preprocess_decoding(self, latents): + latents = _unpatchify_latents(latents) + return latents + + def get_decode_scale_and_shift(self, device, dtype, vae): + vae_arch_config = self.vae_config.arch_config + latents_bn_mean = ( + vae.bn.running_mean.view(1, -1, 1, 1).to(device=device).to(device, dtype) + ) + latents_bn_std = torch.sqrt( + vae.bn.running_var.view(1, -1, 1, 1) + vae_arch_config.batch_norm_eps + ).to(device, dtype) + return 1 / latents_bn_std, latents_bn_mean + + def post_denoising_loop(self, latents, batch): + latent_ids = batch.latent_ids + latents = _unpack_latents_with_ids(latents, latent_ids) + + return latents + + def slice_noise_pred(self, noise, latents): + # remove noise over input image + noise = noise[:, : latents.size(1) :] + return noise diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/hunyuan.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/hunyuan.py new file mode 100644 index 00000000..d45dfadb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/hunyuan.py @@ -0,0 +1,114 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import TypedDict + +import torch + +from sglang.multimodal_gen.configs.models import DiTConfig, EncoderConfig, VAEConfig +from sglang.multimodal_gen.configs.models.dits import HunyuanVideoConfig +from sglang.multimodal_gen.configs.models.encoders import ( + BaseEncoderOutput, + CLIPTextConfig, + LlamaConfig, +) +from sglang.multimodal_gen.configs.models.vaes import HunyuanVAEConfig +from sglang.multimodal_gen.configs.pipeline_configs.base import ( + ModelTaskType, + PipelineConfig, +) + +PROMPT_TEMPLATE_ENCODE_VIDEO = ( + "<|start_header_id|>system<|end_header_id|>\n\nDescribe the video by detailing the following aspects: " + "1. The main content and theme of the video." + "2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects." + "3. Actions, events, behaviors temporal relationships, physical movement changes of the objects." + "4. background environment, light, style and atmosphere." + "5. camera angles, movements, and transitions used in the video:<|eot_id|>" + "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>" +) + + +class PromptTemplate(TypedDict): + template: str + crop_start: int + + +prompt_template_video: PromptTemplate = { + "template": PROMPT_TEMPLATE_ENCODE_VIDEO, + "crop_start": 95, +} + + +def llama_preprocess_text(prompt: str) -> str: + return prompt_template_video["template"].format(prompt) + + +def llama_postprocess_text(outputs: BaseEncoderOutput, _text_inputs) -> torch.tensor: + hidden_state_skip_layer = 2 + assert outputs.hidden_states is not None + hidden_states: tuple[torch.Tensor, ...] = outputs.hidden_states + last_hidden_state: torch.tensor = hidden_states[-(hidden_state_skip_layer + 1)] + crop_start = prompt_template_video.get("crop_start", -1) + last_hidden_state = last_hidden_state[:, crop_start:] + return last_hidden_state + + +def clip_preprocess_text(prompt: str) -> str: + return prompt + + +def clip_postprocess_text(outputs: BaseEncoderOutput, _text_inputs) -> torch.tensor: + pooler_output: torch.tensor = outputs.pooler_output + return pooler_output + + +@dataclass +class HunyuanConfig(PipelineConfig): + """Base configuration for HunYuan pipeline architecture.""" + + task_type: ModelTaskType = ModelTaskType.T2V + + # HunyuanConfig-specific parameters with defaults + # DiT + dit_config: DiTConfig = field(default_factory=HunyuanVideoConfig) + # VAE + vae_config: VAEConfig = field(default_factory=HunyuanVAEConfig) + # Denoising stage + embedded_cfg_scale: int = 6 + flow_shift: int = 7 + + # Text encoding stage + text_encoder_configs: tuple[EncoderConfig, ...] = field( + default_factory=lambda: (LlamaConfig(), CLIPTextConfig()) + ) + preprocess_text_funcs: tuple[Callable[[str], str], ...] = field( + default_factory=lambda: (llama_preprocess_text, clip_preprocess_text) + ) + postprocess_text_funcs: tuple[Callable[[BaseEncoderOutput], torch.tensor], ...] = ( + field(default_factory=lambda: (llama_postprocess_text, clip_postprocess_text)) + ) + + # Precision for each component + dit_precision: str = "bf16" + vae_precision: str = "fp16" + text_encoder_precisions: tuple[str, ...] = field( + default_factory=lambda: ("fp16", "fp16") + ) + + def __post_init__(self): + self.vae_config.load_encoder = False + self.vae_config.load_decoder = True + + +@dataclass +class FastHunyuanConfig(HunyuanConfig): + """Configuration specifically optimized for FastHunyuan weights.""" + + # Override HunyuanConfig defaults + flow_shift: int = 17 + + # No need to re-specify guidance_scale or embedded_cfg_scale as they + # already have the desired values from HunyuanConfig diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/qwen_image.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/qwen_image.py new file mode 100644 index 00000000..035c71e0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/qwen_image.py @@ -0,0 +1,330 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +from dataclasses import dataclass, field +from typing import Callable + +import torch + +from sglang.multimodal_gen.configs.models import DiTConfig, EncoderConfig, VAEConfig +from sglang.multimodal_gen.configs.models.dits.qwenimage import QwenImageDitConfig +from sglang.multimodal_gen.configs.models.encoders.qwen_image import Qwen2_5VLConfig +from sglang.multimodal_gen.configs.models.vaes.qwenimage import QwenImageVAEConfig +from sglang.multimodal_gen.configs.pipeline_configs.base import ( + ImagePipelineConfig, + ModelTaskType, + shard_rotary_emb_for_sp, +) +from sglang.multimodal_gen.runtime.models.vision_utils import resize +from sglang.multimodal_gen.utils import calculate_dimensions + + +def _extract_masked_hidden(hidden_states: torch.Tensor, mask: torch.Tensor): + bool_mask = mask.bool() + valid_lengths = bool_mask.sum(dim=1) + selected = hidden_states[bool_mask] + split_result = torch.split(selected, valid_lengths.tolist(), dim=0) + + return split_result + + +def qwen_image_preprocess_text(prompt): + prompt_template_encode = "<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n" + + template = prompt_template_encode + txt = template.format(prompt) + return txt + + +def qwen_image_postprocess_text(outputs, _text_inputs, drop_idx=34): + # squeeze the batch dim + hidden_states = outputs.hidden_states[-1] + split_hidden_states = _extract_masked_hidden( + hidden_states, _text_inputs.attention_mask + ) + split_hidden_states = [e[drop_idx:] for e in split_hidden_states] + max_seq_len = max([e.size(0) for e in split_hidden_states]) + prompt_embeds = torch.stack( + [ + torch.cat([u, u.new_zeros(max_seq_len - u.size(0), u.size(1))]) + for u in split_hidden_states + ] + ) + return prompt_embeds + + +# Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage.QwenImagePipeline._pack_latents +def _pack_latents(latents, batch_size, num_channels_latents, height, width): + latents = latents.view( + batch_size, num_channels_latents, height // 2, 2, width // 2, 2 + ) + latents = latents.permute(0, 2, 4, 1, 3, 5) + latents = latents.reshape( + batch_size, (height // 2) * (width // 2), num_channels_latents * 4 + ) + + return latents + + +@dataclass +class QwenImagePipelineConfig(ImagePipelineConfig): + """Configuration for the QwenImage pipeline.""" + + should_use_guidance: bool = False + task_type: ModelTaskType = ModelTaskType.T2I + + vae_tiling: bool = False + + vae_sp: bool = False + + dit_config: DiTConfig = field(default_factory=QwenImageDitConfig) + # VAE + vae_config: VAEConfig = field(default_factory=QwenImageVAEConfig) + + # Text encoding stage + text_encoder_configs: tuple[EncoderConfig, ...] = field( + default_factory=lambda: (Qwen2_5VLConfig(),) + ) + + text_encoder_precisions: tuple[str, ...] = field(default_factory=lambda: ("bf16",)) + + preprocess_text_funcs: tuple[Callable[[str], str], ...] = field( + default_factory=lambda: (qwen_image_preprocess_text,) + ) + + postprocess_text_funcs: tuple[Callable[[str], str], ...] = field( + default_factory=lambda: (qwen_image_postprocess_text,) + ) + text_encoder_extra_args: list[dict] = field( + default_factory=lambda: [ + dict( + padding=True, + truncation=True, + ), + None, + ] + ) + + def prepare_sigmas(self, sigmas, num_inference_steps): + return self._prepare_sigmas(sigmas, num_inference_steps) + + def prepare_image_processor_kwargs(self, batch): + if batch.prompt: + prompt_template_encode = "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>{}<|im_end|>\n<|im_start|>assistant\n" + txt = prompt_template_encode.format(batch.prompt) + return dict(text=[txt], padding=True) + else: + return {} + + def get_vae_scale_factor(self): + return self.vae_config.arch_config.vae_scale_factor + + def prepare_latent_shape(self, batch, batch_size, num_frames): + vae_scale_factor = self.vae_config.arch_config.vae_scale_factor + height = 2 * (batch.height // (vae_scale_factor * 2)) + width = 2 * (batch.width // (vae_scale_factor * 2)) + num_channels_latents = self.dit_config.arch_config.in_channels // 4 + shape = (batch_size, 1, num_channels_latents, height, width) + return shape + + def maybe_pack_latents(self, latents, batch_size, batch): + height = 2 * ( + batch.height // (self.vae_config.arch_config.vae_scale_factor * 2) + ) + width = 2 * (batch.width // (self.vae_config.arch_config.vae_scale_factor * 2)) + num_channels_latents = self.dit_config.arch_config.in_channels // 4 + # pack latents + return _pack_latents(latents, batch_size, num_channels_latents, height, width) + + def get_decode_scale_and_shift(self, device, dtype, vae): + vae_arch_config = self.vae_config.arch_config + scaling_factor = 1.0 / torch.tensor( + vae_arch_config.latents_std, device=device + ).view(1, vae_arch_config.z_dim, 1, 1, 1).to(device, dtype) + shift_factor = ( + torch.tensor(vae_arch_config.latents_mean) + .view(1, vae_arch_config.z_dim, 1, 1, 1) + .to(device, dtype) + ) + return scaling_factor, shift_factor + + @staticmethod + def get_freqs_cis(img_shapes, txt_seq_lens, rotary_emb, device, dtype): + # img_shapes: for global entire image + img_freqs, txt_freqs = rotary_emb(img_shapes, txt_seq_lens, device=device) + + img_cos, img_sin = ( + img_freqs.real.to(dtype=dtype), + img_freqs.imag.to(dtype=dtype), + ) + txt_cos, txt_sin = ( + txt_freqs.real.to(dtype=dtype), + txt_freqs.imag.to(dtype=dtype), + ) + + return (img_cos, img_sin), (txt_cos, txt_sin) + + def _prepare_cond_kwargs(self, batch, prompt_embeds, rotary_emb, device, dtype): + batch_size = prompt_embeds[0].shape[0] + height = batch.height + width = batch.width + vae_scale_factor = self.vae_config.arch_config.vae_scale_factor + + img_shapes = [ + [ + ( + 1, + height // vae_scale_factor // 2, + width // vae_scale_factor // 2, + ) + ] + ] * batch_size + txt_seq_lens = [prompt_embeds[0].shape[1]] + + (img_cos, img_sin), (txt_cos, txt_sin) = self.get_freqs_cis( + img_shapes, txt_seq_lens, rotary_emb, device, dtype + ) + + img_cos = shard_rotary_emb_for_sp(img_cos) + img_sin = shard_rotary_emb_for_sp(img_sin) + return { + "txt_seq_lens": txt_seq_lens, + "freqs_cis": ((img_cos, img_sin), (txt_cos, txt_sin)), + } + + def prepare_pos_cond_kwargs(self, batch, device, rotary_emb, dtype): + return self._prepare_cond_kwargs( + batch, batch.prompt_embeds, rotary_emb, device, dtype + ) + + def prepare_neg_cond_kwargs(self, batch, device, rotary_emb, dtype): + return self._prepare_cond_kwargs( + batch, batch.negative_prompt_embeds, rotary_emb, device, dtype + ) + + def post_denoising_loop(self, latents, batch): + # unpack latents for qwen-image + ( + latents, + batch_size, + channels, + height, + width, + ) = self._unpad_and_unpack_latents(latents, batch) + latents = latents.reshape(batch_size, channels // (2 * 2), 1, height, width) + return latents + + +@dataclass +class QwenImageEditPipelineConfig(QwenImagePipelineConfig): + """Configuration for the QwenImageEdit pipeline.""" + + task_type: ModelTaskType = ModelTaskType.I2I + + def _prepare_edit_cond_kwargs( + self, batch, prompt_embeds, rotary_emb, device, dtype + ): + batch_size = batch.latents.shape[0] + assert batch_size == 1 + height = batch.height + width = batch.width + image_size = batch.original_condition_image_size + edit_width, edit_height, _ = calculate_dimensions( + 1024 * 1024, image_size[0] / image_size[1] + ) + vae_scale_factor = self.get_vae_scale_factor() + + img_shapes = [ + [ + ( + 1, + height // vae_scale_factor // 2, + width // vae_scale_factor // 2, + ), + ( + 1, + edit_height // vae_scale_factor // 2, + edit_width // vae_scale_factor // 2, + ), + ], + ] * batch_size + txt_seq_lens = [prompt_embeds[0].shape[1]] + (img_cos, img_sin), (txt_cos, txt_sin) = QwenImagePipelineConfig.get_freqs_cis( + img_shapes, txt_seq_lens, rotary_emb, device, dtype + ) + + # perform sp shard on noisy image tokens + noisy_img_seq_len = ( + 1 * (height // vae_scale_factor // 2) * (width // vae_scale_factor // 2) + ) + + noisy_img_cos = shard_rotary_emb_for_sp(img_cos[:noisy_img_seq_len, :]) + noisy_img_sin = shard_rotary_emb_for_sp(img_sin[:noisy_img_seq_len, :]) + + # concat back the img_cos for input image (since it is not sp-shared later) + img_cos = torch.cat([noisy_img_cos, img_cos[noisy_img_seq_len:, :]], dim=0).to( + device=device + ) + img_sin = torch.cat([noisy_img_sin, img_sin[noisy_img_seq_len:, :]], dim=0).to( + device=device + ) + + return { + "txt_seq_lens": txt_seq_lens, + "freqs_cis": ((img_cos, img_sin), (txt_cos, txt_sin)), + } + + def preprocess_condition_image( + self, image, target_width, target_height, _vae_image_processor + ): + return resize(image, target_height, target_width, resize_mode="default"), ( + target_width, + target_height, + ) + + def postprocess_image_latent(self, latent_condition, batch): + batch_size = batch.batch_size + if batch_size > latent_condition.shape[0]: + if batch_size % latent_condition.shape[0] == 0: + # expand init_latents for batch_size + additional_image_per_prompt = batch_size // latent_condition.shape[0] + image_latents = latent_condition.repeat( + additional_image_per_prompt, 1, 1, 1 + ) + else: + raise ValueError( + f"Cannot duplicate `image` of batch size {latent_condition.shape[0]} to {batch_size} text prompts." + ) + else: + image_latents = latent_condition + image_latent_height, image_latent_width = image_latents.shape[3:] + num_channels_latents = self.dit_config.arch_config.in_channels // 4 + image_latents = _pack_latents( + image_latents, + batch_size, + num_channels_latents, + image_latent_height, + image_latent_width, + ) + + return image_latents + + def prepare_pos_cond_kwargs(self, batch, device, rotary_emb, dtype): + return self._prepare_edit_cond_kwargs( + batch, batch.prompt_embeds, rotary_emb, device, dtype + ) + + def prepare_neg_cond_kwargs(self, batch, device, rotary_emb, dtype): + return self._prepare_edit_cond_kwargs( + batch, batch.negative_prompt_embeds, rotary_emb, device, dtype + ) + + def calculate_condition_image_size(self, image, width, height) -> tuple[int, int]: + calculated_width, calculated_height, _ = calculate_dimensions( + 1024 * 1024, width / height + ) + return calculated_width, calculated_height + + def slice_noise_pred(self, noise, latents): + # remove noise over input image + noise = noise[:, : latents.size(1)] + return noise diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/stepvideo.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/stepvideo.py new file mode 100644 index 00000000..aff18e5c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/stepvideo.py @@ -0,0 +1,36 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field + +from sglang.multimodal_gen.configs.models import DiTConfig, VAEConfig +from sglang.multimodal_gen.configs.models.dits import StepVideoConfig +from sglang.multimodal_gen.configs.models.vaes import StepVideoVAEConfig +from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig + + +@dataclass +class StepVideoT2VConfig(PipelineConfig): + """Base configuration for StepVideo pipeline architecture.""" + + # WanConfig-specific parameters with defaults + # DiT + dit_config: DiTConfig = field(default_factory=StepVideoConfig) + # VAE + vae_config: VAEConfig = field(default_factory=StepVideoVAEConfig) + vae_tiling: bool = False + vae_sp: bool = False + + # Denoising stage + flow_shift: int = 13 + timesteps_scale: bool = False + pos_magic: str = ( + "超高清、HDR 视频、环境光、杜比全景声、画面稳定、流畅动作、逼真的细节、专业级构图、超现实主义、自然、生动、超细节、清晰。" + ) + neg_magic: str = ( + "画面暗、低分辨率、不良手、文本、缺少手指、多余的手指、裁剪、低质量、颗粒状、签名、水印、用户名、模糊。" + ) + + # Precision for each component + precision: str = "bf16" + vae_precision: str = "bf16" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py new file mode 100644 index 00000000..9e7f83ec --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py @@ -0,0 +1,212 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from collections.abc import Callable +from dataclasses import dataclass, field + +import torch + +from sglang.multimodal_gen.configs.models import DiTConfig, EncoderConfig, VAEConfig +from sglang.multimodal_gen.configs.models.dits import WanVideoConfig +from sglang.multimodal_gen.configs.models.encoders import ( + BaseEncoderOutput, + CLIPVisionConfig, + T5Config, +) +from sglang.multimodal_gen.configs.models.vaes import WanVAEConfig +from sglang.multimodal_gen.configs.pipeline_configs.base import ( + ModelTaskType, + PipelineConfig, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +def t5_postprocess_text(outputs: BaseEncoderOutput, _text_inputs) -> torch.Tensor: + mask: torch.Tensor = outputs.attention_mask + hidden_state: torch.Tensor = outputs.last_hidden_state + seq_lens = mask.gt(0).sum(dim=1).long() + assert torch.isnan(hidden_state).sum() == 0 + prompt_embeds = [u[:v] for u, v in zip(hidden_state, seq_lens, strict=True)] + prompt_embeds_tensor: torch.Tensor = torch.stack( + [ + torch.cat([u, u.new_zeros(512 - u.size(0), u.size(1))]) + for u in prompt_embeds + ], + dim=0, + ) + return prompt_embeds_tensor + + +@dataclass +class WanI2VCommonConfig(PipelineConfig): + # for all wan i2v pipelines + def adjust_num_frames(self, num_frames): + vae_scale_factor_temporal = self.vae_config.arch_config.scale_factor_temporal + if num_frames % vae_scale_factor_temporal != 1: + logger.warning( + f"`num_frames - 1` has to be divisible by {vae_scale_factor_temporal}. Rounding to the nearest number." + ) + num_frames = ( + num_frames // vae_scale_factor_temporal * vae_scale_factor_temporal + 1 + ) + return num_frames + return num_frames + + +@dataclass +class WanT2V480PConfig(PipelineConfig): + """Base configuration for Wan T2V 1.3B pipeline architecture.""" + + task_type: ModelTaskType = ModelTaskType.T2V + # WanConfig-specific parameters with defaults + # DiT + dit_config: DiTConfig = field(default_factory=WanVideoConfig) + + # VAE + vae_config: VAEConfig = field(default_factory=WanVAEConfig) + vae_tiling: bool = False + vae_sp: bool = False + + # Denoising stage + flow_shift: float | None = 3.0 + + # Text encoding stage + text_encoder_configs: tuple[EncoderConfig, ...] = field( + default_factory=lambda: (T5Config(),) + ) + postprocess_text_funcs: tuple[Callable[[BaseEncoderOutput], torch.Tensor], ...] = ( + field(default_factory=lambda: (t5_postprocess_text,)) + ) + + # Precision for each component + precision: str = "bf16" + vae_precision: str = "fp32" + text_encoder_precisions: tuple[str, ...] = field(default_factory=lambda: ("fp32",)) + + # WanConfig-specific added parameters + + def __post_init__(self): + self.vae_config.load_encoder = False + self.vae_config.load_decoder = True + + +@dataclass +class WanT2V720PConfig(WanT2V480PConfig): + """Base configuration for Wan T2V 14B 720P pipeline architecture.""" + + # WanConfig-specific parameters with defaults + + # Denoising stage + flow_shift: float | None = 5.0 + + +@dataclass +class WanI2V480PConfig(WanT2V480PConfig, WanI2VCommonConfig): + """Base configuration for Wan I2V 14B 480P pipeline architecture.""" + + # WanConfig-specific parameters with defaults + task_type: ModelTaskType = ModelTaskType.I2V + # Precision for each component + image_encoder_config: EncoderConfig = field(default_factory=CLIPVisionConfig) + image_encoder_precision: str = "fp32" + + image_encoder_extra_args: dict = field( + default_factory=lambda: dict( + output_hidden_states=True, + ) + ) + + def postprocess_image(self, image): + return image.hidden_states[-2] + + def __post_init__(self) -> None: + self.vae_config.load_encoder = True + self.vae_config.load_decoder = True + + +@dataclass +class WanI2V720PConfig(WanI2V480PConfig): + """Base configuration for Wan I2V 14B 720P pipeline architecture.""" + + # WanConfig-specific parameters with defaults + + # Denoising stage + flow_shift: float | None = 5.0 + + +@dataclass +class FastWan2_1_T2V_480P_Config(WanT2V480PConfig): + """Base configuration for FastWan T2V 1.3B 480P pipeline architecture with DMD""" + + # WanConfig-specific parameters with defaults + + # Denoising stage + flow_shift: float | None = 8.0 + dmd_denoising_steps: list[int] | None = field( + default_factory=lambda: [1000, 757, 522] + ) + + +@dataclass +class Wan2_2_TI2V_5B_Config(WanT2V480PConfig, WanI2VCommonConfig): + flow_shift: float | None = 5.0 + task_type: ModelTaskType = ModelTaskType.TI2V + expand_timesteps: bool = True + # ti2v, 5B + vae_stride = (4, 16, 16) + + def prepare_latent_shape(self, batch, batch_size, num_frames): + F = num_frames + z_dim = self.vae_config.arch_config.z_dim + vae_stride = self.vae_stride + oh = batch.height + ow = batch.width + shape = (batch_size, z_dim, F, oh // vae_stride[1], ow // vae_stride[2]) + return shape + + def __post_init__(self) -> None: + self.vae_config.load_encoder = True + self.vae_config.load_decoder = True + self.dit_config.expand_timesteps = self.expand_timesteps + + +@dataclass +class FastWan2_2_TI2V_5B_Config(Wan2_2_TI2V_5B_Config): + flow_shift: float | None = 5.0 + dmd_denoising_steps: list[int] | None = field( + default_factory=lambda: [1000, 757, 522] + ) + + +@dataclass +class Wan2_2_T2V_A14B_Config(WanT2V480PConfig): + flow_shift: float | None = 12.0 + boundary_ratio: float | None = 0.875 + + def __post_init__(self) -> None: + self.dit_config.boundary_ratio = self.boundary_ratio + + +@dataclass +class Wan2_2_I2V_A14B_Config(WanI2V480PConfig): + flow_shift: float | None = 5.0 + boundary_ratio: float | None = 0.900 + + def __post_init__(self) -> None: + super().__post_init__() + self.dit_config.boundary_ratio = self.boundary_ratio + + +# ============================================= +# ============= Causal Self-Forcing ============= +# ============================================= +@dataclass +class SelfForcingWanT2V480PConfig(WanT2V480PConfig): + is_causal: bool = True + flow_shift: float | None = 5.0 + dmd_denoising_steps: list[int] | None = field( + default_factory=lambda: [1000, 750, 500, 250] + ) + warp_denoising_step: bool = True diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/zimage.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/zimage.py new file mode 100644 index 00000000..b9e86b6a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/pipeline_configs/zimage.py @@ -0,0 +1,142 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo +from dataclasses import dataclass, field +from typing import Callable + +import torch + +from sglang.multimodal_gen.configs.models import DiTConfig, EncoderConfig, VAEConfig +from sglang.multimodal_gen.configs.models.dits.zimage import ZImageDitConfig +from sglang.multimodal_gen.configs.models.encoders import ( + BaseEncoderOutput, + TextEncoderConfig, +) +from sglang.multimodal_gen.configs.models.vaes.flux import FluxVAEConfig +from sglang.multimodal_gen.configs.pipeline_configs.base import ( + ImagePipelineConfig, + ModelTaskType, +) + + +def zimage_preprocess_text(prompt: str): + messages = [ + {"role": "user", "content": prompt}, + ] + return messages + + +def zimage_postprocess_text(outputs: BaseEncoderOutput, _text_inputs) -> torch.Tensor: + device = outputs.hidden_states[-2].device + prompt_mask = _text_inputs.attention_mask.to(device).bool() + return outputs.hidden_states[-2][0][prompt_mask[0]] + + +class TransformersModelConfig(EncoderConfig): + tokenizer_kwargs: dict = field(default_factory=lambda: {}) + + +@dataclass +class ZImagePipelineConfig(ImagePipelineConfig): + + should_use_guidance: bool = False + task_type: ModelTaskType = ModelTaskType.T2I + + dit_config: DiTConfig = field(default_factory=ZImageDitConfig) + vae_config: VAEConfig = field(default_factory=FluxVAEConfig) + text_encoder_configs: tuple[EncoderConfig, ...] = field( + default_factory=lambda: (TextEncoderConfig(),) + ) + + preprocess_text_funcs: tuple[Callable, ...] = field( + default_factory=lambda: (zimage_preprocess_text,) + ) + + postprocess_text_funcs: tuple[Callable, ...] = field( + default_factory=lambda: (zimage_postprocess_text,) + ) + + def tokenize_prompt(self, prompts: list[str], tokenizer, tok_kwargs) -> dict: + # flatten to 1-d list + inputs = tokenizer.apply_chat_template( + prompts, + tokenize=True, + add_generation_prompt=True, + enable_thinking=True, + padding="max_length", + max_length=512, # TODO (yhyang201): set max length according to config + truncation=True, + return_tensors="pt", + return_dict=True, + ) + return inputs + + def post_denoising_loop(self, latents, batch): + bs, channels, num_frames, height, width = latents.shape + return latents.view(bs, channels, height, width) + + def get_freqs_cis(self, prompt_embeds, width, height, device, rotary_emb, batch): + def create_coordinate_grid(size, start=None, device=None): + if start is None: + start = (0 for _ in size) + + axes = [ + torch.arange(x0, x0 + span, dtype=torch.int32, device=device) + for x0, span in zip(start, size) + ] + grids = torch.meshgrid(axes, indexing="ij") + return torch.stack(grids, dim=-1) + + PATCH_SIZE = 2 + F_PATCH_SIZE = 1 + SEQ_MULTI_OF = 32 + + cap_ori_len = prompt_embeds.size(0) + cap_padding_len = (-cap_ori_len) % SEQ_MULTI_OF + cap_padded_pos_ids = create_coordinate_grid( + size=(cap_ori_len + cap_padding_len, 1, 1), + start=(1, 0, 0), + device=device, + ).flatten(0, 2) + + C = self.dit_config.num_channels_latents + F = 1 + H = height // self.vae_config.arch_config.spatial_compression_ratio + W = width // self.vae_config.arch_config.spatial_compression_ratio + + pH, pW = PATCH_SIZE, PATCH_SIZE + pF = F_PATCH_SIZE + F_tokens, H_tokens, W_tokens = F // pF, H // pH, W // pW + image_ori_len = F_tokens * H_tokens * W_tokens + image_padding_len = (-image_ori_len) % SEQ_MULTI_OF + + image_ori_pos_ids = create_coordinate_grid( + size=(F_tokens, H_tokens, W_tokens), + start=(cap_ori_len + cap_padding_len + 1, 0, 0), + device=device, + ).flatten(0, 2) + image_padding_pos_ids = ( + create_coordinate_grid( + size=(1, 1, 1), + start=(0, 0, 0), + device=device, + ) + .flatten(0, 2) + .repeat(image_padding_len, 1) + ) + image_padded_pos_ids = torch.cat( + [image_ori_pos_ids, image_padding_pos_ids], dim=0 + ) + cap_freqs_cis = rotary_emb(cap_padded_pos_ids) + x_freqs_cis = rotary_emb(image_padded_pos_ids) + return (cap_freqs_cis, x_freqs_cis) + + def prepare_pos_cond_kwargs(self, batch, device, rotary_emb, dtype): + return { + "freqs_cis": self.get_freqs_cis( + batch.prompt_embeds[0], + batch.width, + batch.height, + device, + rotary_emb, + batch, + ), + } diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/__init__.py new file mode 100644 index 00000000..1d2574de --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/__init__.py @@ -0,0 +1,5 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams + +__all__ = ["SamplingParams"] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/flux.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/flux.py new file mode 100644 index 00000000..657142f2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/flux.py @@ -0,0 +1,25 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass + +from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams + + +@dataclass +class FluxSamplingParams(SamplingParams): + # Video parameters + # height: int = 1024 + # width: int = 1024 + num_frames: int = 1 + # Denoising stage + guidance_scale: float = 1.0 + negative_prompt: str = None + num_inference_steps: int = 50 + + def __post_init__(self): + default_sample_size = 128 + vae_scale_factor = 8 + # FIXME + # self.height = default_sample_size * vae_scale_factor + # self.width = default_sample_size * vae_scale_factor diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/hunyuan.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/hunyuan.py new file mode 100644 index 00000000..d6e33a81 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/hunyuan.py @@ -0,0 +1,37 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field + +from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams +from sglang.multimodal_gen.configs.sample.teacache import TeaCacheParams + + +@dataclass +class HunyuanSamplingParams(SamplingParams): + num_inference_steps: int = 50 + + num_frames: int = 125 + height: int = 720 + width: int = 1280 + fps: int = 24 + + guidance_scale: float = 1.0 + + teacache_params: TeaCacheParams = field( + default_factory=lambda: TeaCacheParams( + teacache_thresh=0.15, + coefficients=[ + 7.33226126e02, + -4.01131952e02, + 6.75869174e01, + -3.14987800e00, + 9.61237896e-02, + ], + ) + ) + + +@dataclass +class FastHunyuanSamplingParam(HunyuanSamplingParams): + num_inference_steps: int = 6 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/qwenimage.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/qwenimage.py new file mode 100644 index 00000000..c3270395 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/qwenimage.py @@ -0,0 +1,18 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass + +from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams + + +@dataclass +class QwenImageSamplingParams(SamplingParams): + # Video parameters + # height: int = 1024 + # width: int = 1024 + negative_prompt: str = " " + num_frames: int = 1 + # Denoising stage + guidance_scale: float = 4.0 + num_inference_steps: int = 50 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/sampling_params.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/sampling_params.py new file mode 100644 index 00000000..ef691b1a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/sampling_params.py @@ -0,0 +1,629 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +import argparse +import dataclasses +import hashlib +import json +import math +import os.path +import re +import time +import unicodedata +import uuid +from dataclasses import dataclass +from enum import Enum, auto +from typing import Any + +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.utils import align_to + +logger = init_logger(__name__) + + +def _json_safe(obj: Any): + """ + Recursively convert objects to JSON-serializable forms. + - Enums -> their name + - Sets/Tuples -> lists + - Dicts/Lists -> recursively processed + """ + if isinstance(obj, Enum): + return obj.name + if isinstance(obj, dict): + return {k: _json_safe(v) for k, v in obj.items()} + if isinstance(obj, (list, tuple, set)): + return [_json_safe(v) for v in obj] + return obj + + +def generate_request_id() -> str: + return str(uuid.uuid4()) + + +def _sanitize_filename(name: str, replacement: str = "_", max_length: int = 150) -> str: + """Create a filesystem- and ffmpeg-friendly filename. + + - Normalize to ASCII (drop accents and unsupported chars) + - Replace spaces with underscores + - Replace any char not in [A-Za-z0-9_.-] with replacement + - Collapse multiple underscores + - Trim leading/trailing dots/underscores and limit length + """ + normalized = unicodedata.normalize("NFKD", name) + ascii_name = normalized.encode("ascii", "ignore").decode("ascii") + ascii_name = ascii_name.replace(" ", "_") + ascii_name = re.sub(r"[^A-Za-z0-9._-]", replacement, ascii_name) + ascii_name = re.sub(r"_+", "_", ascii_name).strip("._") + if not ascii_name: + ascii_name = "output" + if max_length and len(ascii_name) > max_length: + ascii_name = ascii_name[:max_length] + return ascii_name + + +class DataType(Enum): + IMAGE = auto() + VIDEO = auto() + + def get_default_extension(self) -> str: + if self == DataType.IMAGE: + return "jpg" + else: + return "mp4" + + +@dataclass +class SamplingParams: + """ + Sampling parameters for generation. + """ + + data_type: DataType = DataType.VIDEO + + request_id: str | None = None + + # All fields below are copied from ForwardBatch + + # Image inputs + image_path: str | None = None + + # Text inputs + prompt: str | list[str] | None = None + negative_prompt: str = ( + "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards" + ) + prompt_path: str | None = None + output_path: str = "outputs/" + output_file_name: str | None = None + + # Batch info + num_outputs_per_prompt: int = 1 + seed: int = 1024 + + # Original dimensions (before VAE scaling) + num_frames: int = 125 + num_frames_round_down: bool = ( + False # Whether to round down num_frames if it's not divisible by num_gpus + ) + height: int | None = None + width: int | None = None + # NOTE: this is temporary, we need a way to know if width or height is not provided, or do the image resize earlier + height_not_provided: bool = False + width_not_provided: bool = False + fps: int = 24 + + # Denoising parameters + num_inference_steps: int = 50 + guidance_scale: float = 1.0 + guidance_rescale: float = 0.0 + boundary_ratio: float | None = None + + # TeaCache parameters + enable_teacache: bool = False + + # Profiling + profile: bool = False + num_profiled_timesteps: int = 2 + + # Debugging + debug: bool = False + perf_dump_path: str | None = None + + # Misc + save_output: bool = True + return_frames: bool = False + return_trajectory_latents: bool = False # returns all latents for each timestep + return_trajectory_decoded: bool = False # returns decoded latents for each timestep + + def _set_output_file_ext(self): + # add extension if needed + if not any( + self.output_file_name.endswith(ext) + for ext in [".mp4", ".jpg", ".png", ".webp"] + ): + self.output_file_name = ( + f"{self.output_file_name}.{self.data_type.get_default_extension()}" + ) + + def _set_output_file_name(self): + # settle output_file_name + if ( + self.output_file_name is None + and self.prompt + and isinstance(self.prompt, str) + ): + # generate a random filename + # get a hash of current params + params_dict = dataclasses.asdict(self) + # Avoid recursion + params_dict["output_file_name"] = "" + + # Convert to a stable JSON string + params_str = json.dumps(_json_safe(params_dict), sort_keys=True) + # Create a hash + hasher = hashlib.sha256() + hasher.update(params_str.encode("utf-8")) + param_hash = hasher.hexdigest()[:8] + + timestamp = time.strftime("%Y%m%d-%H%M%S") + base = f"{self.prompt[:100]}_{timestamp}_{param_hash}" + self.output_file_name = base + + if self.output_file_name is None: + timestamp = time.strftime("%Y%m%d-%H%M%S") + self.output_file_name = f"output_{timestamp}" + + self.output_file_name = _sanitize_filename(self.output_file_name) + + # Ensure a proper extension is present + self._set_output_file_ext() + + def __post_init__(self) -> None: + assert self.num_frames >= 1 + self.data_type = DataType.VIDEO if self.num_frames > 1 else DataType.IMAGE + + if self.width is None: + self.width_not_provided = True + self.width = 1280 + if self.height is None: + self.height_not_provided = True + self.height = 720 + + def check_sampling_param(self): + if self.prompt_path and not self.prompt_path.endswith(".txt"): + raise ValueError("prompt_path must be a txt file") + + def _adjust( + self, + server_args: ServerArgs, + ): + """ + final adjustment, called after merged with user params + """ + pipeline_config = server_args.pipeline_config + if not isinstance(self.prompt, str): + raise TypeError(f"`prompt` must be a string, but got {type(self.prompt)}") + + # Process negative prompt + if self.negative_prompt is not None and not self.negative_prompt.isspace(): + # avoid stripping default negative prompt: ' ' for qwen-image + self.negative_prompt = self.negative_prompt.strip() + + # Validate dimensions + if self.num_frames <= 0: + raise ValueError( + f"height, width, and num_frames must be positive integers, got " + f"height={self.height}, width={self.width}, " + f"num_frames={self.num_frames}" + ) + + if pipeline_config.task_type.is_image_gen(): + # settle num_frames + logger.debug(f"Setting num_frames to 1 because this is a image-gen model") + self.num_frames = 1 + self.data_type = DataType.IMAGE + else: + # NOTE: We must apply adjust_num_frames BEFORE the SP alignment logic below. + # If we apply it after, adjust_num_frames might modify the frame count + # and break the divisibility constraint (alignment) required by num_gpus. + self.num_frames = server_args.pipeline_config.adjust_num_frames( + self.num_frames + ) + + # Adjust number of frames based on number of GPUs for video task + use_temporal_scaling_frames = ( + pipeline_config.vae_config.use_temporal_scaling_frames + ) + num_frames = self.num_frames + num_gpus = server_args.num_gpus + temporal_scale_factor = ( + pipeline_config.vae_config.arch_config.temporal_compression_ratio + ) + + if use_temporal_scaling_frames: + orig_latent_num_frames = (num_frames - 1) // temporal_scale_factor + 1 + else: # stepvideo only + orig_latent_num_frames = self.num_frames // 17 * 3 + + if orig_latent_num_frames % server_args.num_gpus != 0: + # Adjust latent frames to be divisible by number of GPUs + if self.num_frames_round_down: + # Ensure we have at least 1 batch per GPU + new_latent_num_frames = ( + max(1, (orig_latent_num_frames // num_gpus)) * num_gpus + ) + else: + new_latent_num_frames = ( + math.ceil(orig_latent_num_frames / num_gpus) * num_gpus + ) + + if use_temporal_scaling_frames: + # Convert back to number of frames, ensuring num_frames-1 is a multiple of temporal_scale_factor + new_num_frames = ( + new_latent_num_frames - 1 + ) * temporal_scale_factor + 1 + else: # stepvideo only + # Find the least common multiple of 3 and num_gpus + divisor = math.lcm(3, num_gpus) + # Round up to the nearest multiple of this LCM + new_latent_num_frames = ( + (new_latent_num_frames + divisor - 1) // divisor + ) * divisor + # Convert back to actual frames using the StepVideo formula + new_num_frames = new_latent_num_frames // 3 * 17 + + logger.info( + "Adjusting number of frames from %s to %s based on number of GPUs (%s)", + self.num_frames, + new_num_frames, + server_args.num_gpus, + ) + self.num_frames = new_num_frames + + self._set_output_file_name() + self.log(server_args=server_args) + + def update(self, source_dict: dict[str, Any]) -> None: + for key, value in source_dict.items(): + if hasattr(self, key): + setattr(self, key, value) + else: + logger.exception("%s has no attribute %s", type(self).__name__, key) + + self.__post_init__() + + @classmethod + def from_pretrained(cls, model_path: str, **kwargs) -> "SamplingParams": + from sglang.multimodal_gen.registry import get_model_info + + model_info = get_model_info(model_path) + sampling_params: SamplingParams = model_info.sampling_param_cls(**kwargs) + return sampling_params + + @staticmethod + def from_user_sampling_params_args(model_path: str, server_args, *args, **kwargs): + sampling_params = SamplingParams.from_pretrained(model_path) + + user_sampling_params = SamplingParams(*args, **kwargs) + # TODO: refactor + sampling_params._merge_with_user_params(user_sampling_params) + sampling_params._adjust(server_args) + + return sampling_params + + def output_size_str(self) -> str: + return f"{self.width}x{self.height}" + + def seconds(self) -> float: + return self.num_frames / self.fps + + @staticmethod + def add_cli_args(parser: Any) -> Any: + """Add CLI arguments for SamplingParam fields""" + parser.add_argument("--data-type", type=str, nargs="+", default=DataType.VIDEO) + parser.add_argument( + "--num-frames-round-down", + action="store_true", + default=SamplingParams.num_frames_round_down, + ) + parser.add_argument( + "--enable-teacache", + action="store_true", + default=SamplingParams.enable_teacache, + ) + parser.add_argument( + "--profile", + action="store_true", + default=SamplingParams.profile, + help="Enable torch profiler for denoising stage", + ) + parser.add_argument( + "--debug", + action="store_true", + default=SamplingParams.debug, + help="", + ) + parser.add_argument( + "--num-profiled-timesteps", + type=int, + default=SamplingParams.num_profiled_timesteps, + help="Number of timesteps to profile after warmup", + ) + parser.add_argument( + "--prompt", + type=str, + default=SamplingParams.prompt, + help="Text prompt for generation", + ) + parser.add_argument( + "--negative-prompt", + type=str, + default=SamplingParams.negative_prompt, + help="Negative text prompt for generation", + ) + parser.add_argument( + "--prompt-path", + type=str, + default=SamplingParams.prompt_path, + help="Path to a text file containing the prompt", + ) + parser.add_argument( + "--output-path", + type=str, + default=SamplingParams.output_path, + help="Path to save the generated image/video", + ) + parser.add_argument( + "--output-file-name", + type=str, + default=SamplingParams.output_file_name, + help="Name of the output file", + ) + parser.add_argument( + "--num-outputs-per-prompt", + type=int, + default=SamplingParams.num_outputs_per_prompt, + help="Number of outputs to generate per prompt", + ) + parser.add_argument( + "--seed", + type=int, + default=SamplingParams.seed, + help="Random seed for generation", + ) + parser.add_argument( + "--num-frames", + type=int, + default=SamplingParams.num_frames, + help="Number of frames to generate", + ) + parser.add_argument( + "--height", + type=int, + default=SamplingParams.height, + help="Height of generated output", + ) + parser.add_argument( + "--width", + type=int, + default=SamplingParams.width, + help="Width of generated output", + ) + # resolution shortcuts + parser.add_argument( + "--4k", + action="store_true", + dest="resolution_4k", + help="Set resolution to 4K (3840x2160)", + ) + parser.add_argument( + "--2k", + action="store_true", + dest="resolution_2k", + help="Set resolution to 2K (2560x1440)", + ) + parser.add_argument( + "--1080p", + action="store_true", + dest="resolution_1080p", + help="Set resolution to 1080p (1920x1080)", + ) + parser.add_argument( + "--720p", + action="store_true", + dest="resolution_720p", + help="Set resolution to 720p (1280x720)", + ) + + parser.add_argument( + "--fps", + type=int, + default=SamplingParams.fps, + help="Frames per second for saved output", + ) + parser.add_argument( + "--num-inference-steps", + type=int, + default=SamplingParams.num_inference_steps, + help="Number of denoising steps", + ) + parser.add_argument( + "--guidance-scale", + type=float, + default=SamplingParams.guidance_scale, + help="Classifier-free guidance scale", + ) + parser.add_argument( + "--guidance-rescale", + type=float, + default=SamplingParams.guidance_rescale, + help="Guidance rescale factor", + ) + parser.add_argument( + "--boundary-ratio", + type=float, + default=SamplingParams.boundary_ratio, + help="Boundary timestep ratio", + ) + parser.add_argument( + "--save-output", + action="store_true", + default=SamplingParams.save_output, + help="Whether to save the output to disk", + ) + parser.add_argument( + "--no-save-output", + action="store_false", + dest="save_output", + help="Don't save the output to disk", + ) + parser.add_argument( + "--return-frames", + action="store_true", + default=SamplingParams.return_frames, + help="Whether to return the raw frames", + ) + parser.add_argument( + "--image-path", + type=str, + default=SamplingParams.image_path, + help="Path to input image for image-to-video generation", + ) + parser.add_argument( + "--moba-config-path", + type=str, + default=None, + help="Path to a JSON file containing V-MoBA specific configurations.", + ) + parser.add_argument( + "--return-trajectory-latents", + action="store_true", + default=SamplingParams.return_trajectory_latents, + help="Whether to return the trajectory", + ) + parser.add_argument( + "--return-trajectory-decoded", + action="store_true", + default=SamplingParams.return_trajectory_decoded, + help="Whether to return the decoded trajectory", + ) + return parser + + @classmethod + def get_cli_args(cls, args: argparse.Namespace): + # handle resolution shortcuts + if hasattr(args, "resolution_4k") and args.resolution_4k: + args.width = 3840 + args.height = 2160 + elif hasattr(args, "resolution_2k") and args.resolution_2k: + args.width = 2560 + args.height = 1440 + elif hasattr(args, "resolution_1080p") and args.resolution_1080p: + args.width = 1920 + args.height = 1080 + elif hasattr(args, "resolution_720p") and args.resolution_720p: + args.width = 1280 + args.height = 720 + + attrs = [attr.name for attr in dataclasses.fields(cls)] + args.height_not_provided = False + args.width_not_provided = False + return {attr: getattr(args, attr) for attr in attrs} + + def output_file_path(self): + return os.path.join(self.output_path, self.output_file_name) + + def _merge_with_user_params(self, user_params): + """ + Merges parameters from a user-provided SamplingParams object. + + This method updates the current object with values from `user_params`, + but skips any fields that are explicitly defined in the current object's + subclass. This is to preserve model-specific optimal parameters. + It also skips fields that the user has not changed from the default + in `user_params`. + """ + if user_params is None: + return + + # user is not allowed to modify any param defined in the SamplingParams subclass + subclass_defined_fields = set(type(self).__annotations__.keys()) + + # Compare against current instance to avoid constructing a default instance + default_params = SamplingParams() + + for field in dataclasses.fields(user_params): + field_name = field.name + user_value = getattr(user_params, field_name) + default_value = getattr(default_params, field_name) + + # A field is considered user-modified if its value is different from + # the default, with an exception for `output_file_name` which is + # auto-generated with a random component. + is_user_modified = ( + user_value != default_value + if field_name != "output_file_name" + else user_params.output_file_path is not None + ) + if is_user_modified and field_name not in subclass_defined_fields: + if hasattr(self, field_name): + setattr(self, field_name, user_value) + + self.height_not_provided = user_params.height_not_provided + self.width_not_provided = user_params.width_not_provided + self.__post_init__() + + @property + def n_tokens(self) -> int: + # Calculate latent sizes + if self.height and self.width: + latents_size = [ + (self.num_frames - 1) // 4 + 1, + self.height // 8, + self.width // 8, + ] + n_tokens = latents_size[0] * latents_size[1] * latents_size[2] + else: + n_tokens = -1 + return n_tokens + + def output_file_path(self): + return os.path.join(self.output_path, self.output_file_name) + + def log(self, server_args: ServerArgs): + # TODO: in some cases (e.g., TI2I), height and weight might be undecided at this moment + if self.height: + target_height = align_to(self.height, 16) + else: + target_height = -1 + if self.width: + target_width = align_to(self.width, 16) + else: + target_width = -1 + + # Log sampling parameters + debug_str = f"""Sampling params: + width: {target_width} + height: {target_height} + num_frames: {self.num_frames} + prompt: {self.prompt} + neg_prompt: {self.negative_prompt} + seed: {self.seed} + infer_steps: {self.num_inference_steps} + num_outputs_per_prompt: {self.num_outputs_per_prompt} + guidance_scale: {self.guidance_scale} + embedded_guidance_scale: {server_args.pipeline_config.embedded_cfg_scale} + n_tokens: {self.n_tokens} + flow_shift: {server_args.pipeline_config.flow_shift} + image_path: {self.image_path} + save_output: {self.save_output} + output_file_path: {self.output_file_path()} + """ # type: ignore[attr-defined] + logger.info(debug_str) + + +@dataclass +class CacheParams: + cache_type: str = "none" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/stepvideo.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/stepvideo.py new file mode 100644 index 00000000..4fff150b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/stepvideo.py @@ -0,0 +1,22 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass + +from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams + + +@dataclass +class StepVideoT2VSamplingParams(SamplingParams): + # Video parameters + height: int = 720 + width: int = 1280 + num_frames: int = 81 + + # Denoising stage + guidance_scale: float = 9.0 + num_inference_steps: int = 50 + + # neg magic and pos magic + # pos_magic: str = "超高清、HDR 视频、环境光、杜比全景声、画面稳定、流畅动作、逼真的细节、专业级构图、超现实主义、自然、生动、超细节、清晰。" + # neg_magic: str = "画面暗、低分辨率、不良手、文本、缺少手指、多余的手指、裁剪、低质量、颗粒状、签名、水印、用户名、模糊。" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/teacache.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/teacache.py new file mode 100644 index 00000000..ada71d0b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/teacache.py @@ -0,0 +1,43 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field + +from sglang.multimodal_gen.configs.sample.sampling_params import CacheParams + + +@dataclass +class TeaCacheParams(CacheParams): + cache_type: str = "teacache" + teacache_thresh: float = 0.0 + coefficients: list[float] = field(default_factory=list) + + +@dataclass +class WanTeaCacheParams(CacheParams): + # Unfortunately, TeaCache is very different for Wan than other models + cache_type: str = "teacache" + teacache_thresh: float = 0.0 + use_ret_steps: bool = True + ret_steps_coeffs: list[float] = field(default_factory=list) + non_ret_steps_coeffs: list[float] = field(default_factory=list) + + @property + def coefficients(self) -> list[float]: + if self.use_ret_steps: + return self.ret_steps_coeffs + else: + return self.non_ret_steps_coeffs + + @property + def ret_steps(self) -> int: + if self.use_ret_steps: + return 5 * 2 + else: + return 1 * 2 + + def get_cutoff_steps(self, num_inference_steps: int) -> int: + if self.use_ret_steps: + return num_inference_steps * 2 + else: + return num_inference_steps * 2 - 2 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/wan.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/wan.py new file mode 100644 index 00000000..9a23fb0b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/wan.py @@ -0,0 +1,217 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field + +from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams +from sglang.multimodal_gen.configs.sample.teacache import WanTeaCacheParams + + +@dataclass +class WanT2V_1_3B_SamplingParams(SamplingParams): + # Video parameters + height: int = 480 + width: int = 832 + num_frames: int = 81 + fps: int = 16 + + # Denoising stage + guidance_scale: float = 3.0 + negative_prompt: str = ( + "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards" + ) + num_inference_steps: int = 50 + + teacache_params: WanTeaCacheParams = field( + default_factory=lambda: WanTeaCacheParams( + teacache_thresh=0.08, + ret_steps_coeffs=[ + -5.21862437e04, + 9.23041404e03, + -5.28275948e02, + 1.36987616e01, + -4.99875664e-02, + ], + non_ret_steps_coeffs=[ + 2.39676752e03, + -1.31110545e03, + 2.01331979e02, + -8.29855975e00, + 1.37887774e-01, + ], + ) + ) + + +@dataclass +class WanT2V_14B_SamplingParams(SamplingParams): + # Video parameters + height: int = 720 + width: int = 1280 + num_frames: int = 81 + fps: int = 16 + + # Denoising stage + guidance_scale: float = 5.0 + negative_prompt: str = ( + "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards" + ) + num_inference_steps: int = 50 + + teacache_params: WanTeaCacheParams = field( + default_factory=lambda: WanTeaCacheParams( + teacache_thresh=0.20, + use_ret_steps=False, + ret_steps_coeffs=[ + -3.03318725e05, + 4.90537029e04, + -2.65530556e03, + 5.87365115e01, + -3.15583525e-01, + ], + non_ret_steps_coeffs=[ + -5784.54975374, + 5449.50911966, + -1811.16591783, + 256.27178429, + -13.02252404, + ], + ) + ) + + +@dataclass +class WanI2V_14B_480P_SamplingParam(WanT2V_1_3B_SamplingParams): + # Denoising stage + guidance_scale: float = 5.0 + num_inference_steps: int = 50 + # num_inference_steps: int = 40 + + teacache_params: WanTeaCacheParams = field( + default_factory=lambda: WanTeaCacheParams( + teacache_thresh=0.26, + ret_steps_coeffs=[ + -3.03318725e05, + 4.90537029e04, + -2.65530556e03, + 5.87365115e01, + -3.15583525e-01, + ], + non_ret_steps_coeffs=[ + -5784.54975374, + 5449.50911966, + -1811.16591783, + 256.27178429, + -13.02252404, + ], + ) + ) + + +@dataclass +class WanI2V_14B_720P_SamplingParam(WanT2V_14B_SamplingParams): + # Denoising stage + guidance_scale: float = 5.0 + num_inference_steps: int = 50 + # num_inference_steps: int = 40 + + teacache_params: WanTeaCacheParams = field( + default_factory=lambda: WanTeaCacheParams( + teacache_thresh=0.3, + ret_steps_coeffs=[ + -3.03318725e05, + 4.90537029e04, + -2.65530556e03, + 5.87365115e01, + -3.15583525e-01, + ], + non_ret_steps_coeffs=[ + -5784.54975374, + 5449.50911966, + -1811.16591783, + 256.27178429, + -13.02252404, + ], + ) + ) + + +@dataclass +class FastWanT2V480PConfig(WanT2V_1_3B_SamplingParams): + # DMD parameters + # dmd_denoising_steps: list[int] | None = field(default_factory=lambda: [1000, 757, 522]) + num_inference_steps: int = 3 + num_frames: int = 61 + height: int = 448 + width: int = 832 + fps: int = 16 + + +# ============================================= +# ============= Wan2.1 Fun Models ============= +# ============================================= +@dataclass +class Wan2_1_Fun_1_3B_InP_SamplingParams(SamplingParams): + """Sampling parameters for Wan2.1 Fun 1.3B InP model.""" + + height: int = 480 + width: int = 832 + num_frames: int = 81 + fps: int = 16 + negative_prompt: str | None = ( + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ) + guidance_scale: float = 6.0 + num_inference_steps: int = 50 + + +# ============================================= +# ============= Wan2.2 TI2V Models ============= +# ============================================= +@dataclass +class Wan2_2_Base_SamplingParams(SamplingParams): + """Sampling parameters for Wan2.2 TI2V 5B model.""" + + negative_prompt: str | None = ( + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ) + + +@dataclass +class Wan2_2_TI2V_5B_SamplingParam(Wan2_2_Base_SamplingParams): + """Sampling parameters for Wan2.2 TI2V 5B model.""" + + height: int = 704 + width: int = 1280 + num_frames: int = 121 + fps: int = 24 + guidance_scale: float = 5.0 + num_inference_steps: int = 50 + + +@dataclass +class Wan2_2_T2V_A14B_SamplingParam(Wan2_2_Base_SamplingParams): + guidance_scale: float = 4.0 # high_noise + guidance_scale_2: float = 3.0 # low_noise + num_inference_steps: int = 40 + fps: int = 16 + # NOTE(will): default boundary timestep is tracked by PipelineConfig, but + # can be overridden during sampling + + +@dataclass +class Wan2_2_I2V_A14B_SamplingParam(Wan2_2_Base_SamplingParams): + guidance_scale: float = 3.5 # high_noise + guidance_scale_2: float = 3.5 # low_noise + num_inference_steps: int = 40 + fps: int = 16 + # NOTE(will): default boundary timestep is tracked by PipelineConfig, but + # can be overridden during sampling + + +# ============================================= +# ============= Causal Self-Forcing ============= +# ============================================= +@dataclass +class SelfForcingWanT2V480PConfig(WanT2V_1_3B_SamplingParams): + pass diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/zimage.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/zimage.py new file mode 100644 index 00000000..f4530e97 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/sample/zimage.py @@ -0,0 +1,33 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field + +from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams +from sglang.multimodal_gen.configs.sample.teacache import TeaCacheParams + + +@dataclass +class ZImageSamplingParams(SamplingParams): + num_inference_steps: int = 9 + + num_frames: int = 1 + negative_prompt: str = None + # height: int = 720 + # width: int = 1280 + # fps: int = 24 + + guidance_scale: float = 0.0 + + teacache_params: TeaCacheParams = field( + default_factory=lambda: TeaCacheParams( + teacache_thresh=0.15, + coefficients=[ + 7.33226126e02, + -4.01131952e02, + 6.75869174e01, + -3.14987800e00, + 9.61237896e-02, + ], + ) + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/utils.py new file mode 100644 index 00000000..d2cc69ad --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/configs/utils.py @@ -0,0 +1,61 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +import argparse +from typing import Any + + +def update_config_from_args( + config: Any, args_dict: dict[str, Any], prefix: str = "", pop_args: bool = False +) -> bool: + """ + Update configuration object from arguments dictionary. + + Args: + config: The configuration object to update + args_dict: Dictionary containing arguments + prefix: Prefix for the configuration parameters in the args_dict. + If None, assumes direct attribute mapping without prefix. + """ + # Handle top-level attributes (no prefix) + args_not_to_remove = [ + "model_path", + ] + args_to_remove = [] + if prefix.strip() == "": + for key, value in args_dict.items(): + if hasattr(config, key) and value is not None: + if key == "text_encoder_precisions" and isinstance(value, list): + setattr(config, key, tuple(value)) + else: + setattr(config, key, value) + if pop_args: + args_to_remove.append(key) + else: + # Handle nested attributes with prefix + prefix_with_dot = f"{prefix}." + for key, value in args_dict.items(): + if key.startswith(prefix_with_dot) and value is not None: + attr_name = key[len(prefix_with_dot) :] + if hasattr(config, attr_name): + setattr(config, attr_name, value) + if pop_args: + args_to_remove.append(key) + + if pop_args: + for key in args_to_remove: + if key not in args_not_to_remove: + args_dict.pop(key) + + return len(args_to_remove) > 0 + + +def clean_cli_args(args: argparse.Namespace) -> dict[str, Any]: + """ + Clean the arguments by removing the ones that not explicitly provided by the user. + """ + provided_args = {} + for k, v in vars(args).items(): + if v is not None and hasattr(args, "_provided") and k in args._provided: + provided_args[k] = v + + return provided_args diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/csrc/attn/vmoba_attn/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/csrc/attn/vmoba_attn/README.md new file mode 100644 index 00000000..7b41bd51 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/csrc/attn/vmoba_attn/README.md @@ -0,0 +1,31 @@ +# Attention Kernel Used in SGLang diffusion + +## VMoBA: Mixture-of-Block Attention for Video Diffusion Models (VMoBA) + +### Installation +Please ensure that you have installed FlashAttention version **2.7.1 or higher**, as some interfaces have changed in recent releases. + +### Usage + +You can use `moba_attn_varlen` in the following ways: + +**Install from source:** +```bash +python setup.py install +``` + +**Import after installation:** +```python +from vmoba import moba_attn_varlen +``` + +**Or import directly from the project root:** +```python +from csrc.attn.vmoba_attn.vmoba import moba_attn_varlen +``` + +### Verify if you have successfully installed + +```bash +python csrc/attn/vmoba_attn/vmoba/vmoba.py +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/csrc/attn/vmoba_attn/setup.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/csrc/attn/vmoba_attn/setup.py new file mode 100644 index 00000000..3a1bdb67 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/csrc/attn/vmoba_attn/setup.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: Apache-2.0 + +from setuptools import find_packages, setup + +PACKAGE_NAME = "vmoba" +VERSION = "0.0.0" +AUTHOR = "JianzongWu" +DESCRIPTION = "VMoBA: Mixture-of-Block Attention for Video Diffusion Models" +URL = "https://github.com/KwaiVGI/VMoBA" + +setup( + name=PACKAGE_NAME, + version=VERSION, + author=AUTHOR, + description=DESCRIPTION, + url=URL, + packages=find_packages(), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + ], + python_requires=">=3.12", + install_requires=[ + "flash-attn >= 2.7.1", + ], +) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/csrc/attn/vmoba_attn/tests/test_vmoba_attn.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/csrc/attn/vmoba_attn/tests/test_vmoba_attn.py new file mode 100644 index 00000000..f4304bda --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/csrc/attn/vmoba_attn/tests/test_vmoba_attn.py @@ -0,0 +1,137 @@ +# SPDX-License-Identifier: Apache-2.0 + +import random + +import pytest +import torch +from csrc.attn.vmoba_attn.vmoba import moba_attn_varlen + + +def generate_test_data( + batch_size, total_seqlen, num_heads, head_dim, dtype, device="cuda" +): + """ + Generates random data for testing the variable-length attention function. + """ + torch.manual_seed(42) + random.seed(42) + torch.cuda.manual_seed_all(42) + + # Generate sequence lengths for each item in the batch + if batch_size > 1: + # Ensure sequence lengths are reasonably distributed + avg_seqlen = total_seqlen // batch_size + seqlens = [ + random.randint(avg_seqlen // 2, avg_seqlen + avg_seqlen // 2) + for _ in range(batch_size - 1) + ] + remaining_len = total_seqlen - sum(seqlens) + if remaining_len > 0: + seqlens.append(remaining_len) + else: # Adjust if sum exceeds total_seqlen + seqlens.append(avg_seqlen) + current_sum = sum(seqlens) + seqlens[-1] -= current_sum - total_seqlen + # Ensure all lengths are positive + seqlens = [max(1, s) for s in seqlens] + # Final adjustment to match total_seqlen + seqlens[-1] += total_seqlen - sum(seqlens) + + else: + seqlens = [total_seqlen] + + cu_seqlens = torch.tensor( + [0] + list(torch.cumsum(torch.tensor(seqlens), 0)), + device=device, + dtype=torch.int32, + ) + max_seqlen = max(seqlens) if seqlens else 0 + + q = torch.randn( + (total_seqlen, num_heads, head_dim), + dtype=dtype, + device=device, + requires_grad=False, + ) + k = torch.randn( + (total_seqlen, num_heads, head_dim), + dtype=dtype, + device=device, + requires_grad=False, + ) + v = torch.randn( + (total_seqlen, num_heads, head_dim), + dtype=dtype, + device=device, + requires_grad=False, + ) + + return q, k, v, cu_seqlens, max_seqlen + + +@pytest.mark.parametrize("batch_size", [1, 2]) +@pytest.mark.parametrize("total_seqlen", [512, 1024]) +@pytest.mark.parametrize("num_heads", [8]) +@pytest.mark.parametrize("head_dim", [64]) +@pytest.mark.parametrize("moba_chunk_size", [64]) +@pytest.mark.parametrize("moba_topk", [2, 4]) +@pytest.mark.parametrize("select_mode", ["topk", "threshold"]) +@pytest.mark.parametrize("threshold_type", ["query_head", "head_global", "overall"]) +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +def test_moba_attn_varlen_forward( + batch_size, + total_seqlen, + num_heads, + head_dim, + moba_chunk_size, + moba_topk, + select_mode, + threshold_type, + dtype, +): + """ + Tests the forward pass of moba_attn_varlen for basic correctness. + It checks output shape, dtype, and for the presence of NaNs/Infs. + """ + if dtype == torch.float32: + pytest.skip("float32 is not supported in flash attention") + + q, k, v, cu_seqlens, max_seqlen = generate_test_data( + batch_size, total_seqlen, num_heads, head_dim, dtype + ) + + # Ensure chunk size is not larger than the smallest sequence length + min_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).min().item() + if moba_chunk_size > min_seqlen: + pytest.skip( + "moba_chunk_size is larger than the minimum sequence length in the batch" + ) + + try: + output = moba_attn_varlen( + q=q, + k=k, + v=v, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + moba_chunk_size=moba_chunk_size, + moba_topk=moba_topk, + select_mode=select_mode, + threshold_type=threshold_type, + simsum_threshold=0.5, # A reasonable default for threshold mode + ) + except Exception as e: + pytest.fail(f"moba_attn_varlen forward pass failed with exception: {e}") + + # 1. Check output shape + assert ( + output.shape == q.shape + ), f"Expected output shape {q.shape}, but got {output.shape}" + + # 2. Check output dtype + assert ( + output.dtype == q.dtype + ), f"Expected output dtype {q.dtype}, but got {output.dtype}" + + # 3. Check for NaNs or Infs in the output + assert torch.all(torch.isfinite(output)), "Output contains NaN or Inf values" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/csrc/attn/vmoba_attn/vmoba/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/csrc/attn/vmoba_attn/vmoba/__init__.py new file mode 100644 index 00000000..8119387c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/csrc/attn/vmoba_attn/vmoba/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +from .vmoba import moba_attn_varlen, process_moba_input, process_moba_output diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/csrc/attn/vmoba_attn/vmoba/vmoba.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/csrc/attn/vmoba_attn/vmoba/vmoba.py new file mode 100644 index 00000000..8a29360a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/csrc/attn/vmoba_attn/vmoba/vmoba.py @@ -0,0 +1,1086 @@ +# SPDX-License-Identifier: Apache-2.0 +# Adapt from https://github.com/KwaiVGI/VMoBA/blob/main/src/vmoba.py + +import random +import time +from typing import Tuple + +import torch + +try: + from flash_attn import ( # Use the new flash attention function + flash_attn_varlen_func, + ) + from flash_attn.flash_attn_interface import ( + _flash_attn_varlen_backward, + _flash_attn_varlen_forward, + ) +except ImportError: + + def _unsupported(*args, **kwargs): + raise ImportError( + "flash-attn is not installed. Please install it, e.g., `pip install flash-attn`." + ) + + _flash_attn_varlen_forward = _unsupported + _flash_attn_varlen_backward = _unsupported + flash_attn_varlen_func = _unsupported + +from functools import lru_cache + +from einops import rearrange + + +@lru_cache(maxsize=16) +def calc_chunks(cu_seqlen, moba_chunk_size): + """ + Calculate chunk boundaries. + + For vision tasks we include all chunks (even the last one which might be shorter) + so that every chunk can be selected. + """ + batch_sizes = cu_seqlen[1:] - cu_seqlen[:-1] + batch_num_chunk = (batch_sizes + (moba_chunk_size - 1)) // moba_chunk_size + cu_num_chunk = torch.ones( + batch_num_chunk.numel() + 1, + device=cu_seqlen.device, + dtype=batch_num_chunk.dtype, + ) + cu_num_chunk[1:] = batch_num_chunk.cumsum(dim=0) + num_chunk = cu_num_chunk[-1] + chunk_sizes = torch.full( + (num_chunk + 1,), moba_chunk_size, dtype=torch.int32, device=cu_seqlen.device + ) + chunk_sizes[0] = 0 + batch_last_chunk_size = batch_sizes - (batch_num_chunk - 1) * moba_chunk_size + chunk_sizes[cu_num_chunk[1:]] = batch_last_chunk_size + cu_chunk = chunk_sizes.cumsum(dim=-1, dtype=torch.int32) + chunk_to_batch = torch.zeros( + (num_chunk,), dtype=torch.int32, device=cu_seqlen.device + ) + chunk_to_batch[cu_num_chunk[1:-1]] = 1 + chunk_to_batch = chunk_to_batch.cumsum(dim=0, dtype=torch.int32) + + # Do not filter out any chunk + filtered_chunk_indices = torch.arange( + num_chunk, device=cu_seqlen.device, dtype=torch.int32 + ) + num_filtered_chunk = num_chunk + + return cu_chunk, filtered_chunk_indices, num_filtered_chunk, chunk_to_batch + + +# --- Threshold Selection Helper Functions --- + + +def _select_threshold_query_head( + gate: torch.Tensor, + valid_gate_mask: torch.Tensor, + gate_self_chunk_mask: torch.Tensor, + simsum_threshold: float, +) -> torch.Tensor: + """ + Selects chunks for each pair based on threshold. + Normalization and sorting happen along the chunk dimension (dim=0). + """ + C, H, S = gate.shape + eps = 1e-6 + + # LSE‐style normalization per (across chunks) + gate_masked = torch.where(valid_gate_mask, gate, -torch.inf) # Use -inf for max + gate_min_val = torch.where(valid_gate_mask, gate, torch.inf) # Use +inf for min + + row_min = gate_min_val.amin(dim=0) # (H, S) + row_max = gate_masked.amax(dim=0) # (H, S) + denom = row_max - row_min + denom = torch.where( + denom <= eps, torch.ones_like(denom), denom + ) # avoid divide‑by‑zero + + gate_norm = (gate - row_min.unsqueeze(0)) / denom.unsqueeze(0) + gate_norm = torch.where(valid_gate_mask, gate_norm, 0.0) # (C, H, S) + + # 1) pull out the self‐chunk’s normalized weight for each + self_norm = (gate_norm * gate_self_chunk_mask).sum(dim=0) # (H, S) + + # 2) compute how much more normalized weight we need beyond self + total_norm_sum = gate_norm.sum(dim=0) # (H, S) + remain_ratio = simsum_threshold - self_norm / (total_norm_sum + eps) # (H, S) + remain_ratio = torch.clamp( + remain_ratio, min=0.0 + ) # if already ≥ thresh, no extra needed + + # 3) zero out the self‐chunk in a copy, so we only sort “others” + others_norm = gate_norm.clone() + others_norm[gate_self_chunk_mask] = 0.0 + + # 4) sort the other chunks by descending norm, per + sorted_norm, sorted_idx = torch.sort( + others_norm, descending=True, dim=0 + ) # (C, H, S) + + # 5) cumulative‑sum the sorted norms per + cumsum_others = sorted_norm.cumsum(dim=0) # (C, H, S) + + # 6) for each , find the smallest k where cumsum_ratio ≥ remain_ratio + ratio = cumsum_others / (total_norm_sum.unsqueeze(0) + eps) # (C, H, S) + cond = ratio >= remain_ratio.unsqueeze(0) # (C, H, S) boolean mask + any_cond = cond.any(dim=0) # (H, S) + # Find the index of the first True value along dim 0. If none, use C-1. + cutoff = torch.where( + any_cond, + cond.float().argmax(dim=0), + torch.full_like(any_cond, fill_value=C - 1), + ) # (H, S) + + # 7) build a mask in sorted order up to that cutoff + idx_range = torch.arange(C, device=gate.device).view(-1, 1, 1) # (C, 1, 1) + sorted_mask = idx_range <= cutoff.unsqueeze(0) # (C, H, S) + + # 8) scatter it back to original chunk order + others_mask = torch.zeros_like(gate, dtype=torch.bool) + others_mask.scatter_(0, sorted_idx, sorted_mask) + + # 9) finally, include every self‐chunk plus all selected others + final_gate_mask = valid_gate_mask & (others_mask | gate_self_chunk_mask) + + return final_gate_mask + + +def _select_threshold_block( + gate: torch.Tensor, + valid_gate_mask: torch.Tensor, + gate_self_chunk_mask: torch.Tensor, + simsum_threshold: float, +) -> torch.Tensor: + """ + Selects pairs for each block based on threshold. + Normalization and sorting happen across the head and sequence dimensions (dim=1, 2). + """ + C, H, S = gate.shape + HS = H * S + eps = 1e-6 + + # LSE‐style normalization per block (across heads and queries) + gate_masked = torch.where(valid_gate_mask, gate, -torch.inf) # Use -inf for max + gate_min_val = torch.where(valid_gate_mask, gate, torch.inf) # Use +inf for min + + block_max = gate_masked.amax(dim=(1, 2), keepdim=True) # (C, 1, 1) + block_min = gate_min_val.amin(dim=(1, 2), keepdim=True) # (C, 1, 1) + block_denom = block_max - block_min + block_denom = torch.where( + block_denom <= eps, torch.ones_like(block_denom), block_denom + ) # (C, 1, 1) + + gate_norm = (gate - block_min) / block_denom # (C, H, S) + gate_norm = torch.where(valid_gate_mask, gate_norm, 0.0) # (C, H, S) + + # 1) identify normalized weights of entries that *are* self-chunks (from query perspective) + self_norm_entries = gate_norm * gate_self_chunk_mask # (C, H, S) + # Sum these weights *per block* + self_norm_sum_per_block = self_norm_entries.sum(dim=(1, 2)) # (C,) + + # 2) compute how much more normalized weight each block needs beyond its self-chunk contributions + total_norm_sum_per_block = gate_norm.sum(dim=(1, 2)) # (C,) + remain_ratio = simsum_threshold - self_norm_sum_per_block / ( + total_norm_sum_per_block + eps + ) # (C,) + remain_ratio = torch.clamp(remain_ratio, min=0.0) # (C,) + + # 3) zero out the self‐chunk entries in a copy, so we only sort “others” + others_norm = gate_norm.clone() + others_norm[gate_self_chunk_mask] = 0.0 # Zero out self entries + + # 4) sort the other pairs by descending norm, per block + others_flat = others_norm.contiguous().view(C, HS) # (C, H*S) + sorted_others_flat, sorted_indices_flat = torch.sort( + others_flat, dim=1, descending=True + ) # (C, H*S) + + # 5) cumulative‑sum the sorted norms per block + cumsum_others_flat = sorted_others_flat.cumsum(dim=1) # (C, H*S) + + # 6) for each block, find the smallest k where cumsum_ratio ≥ remain_ratio + ratio_flat = cumsum_others_flat / ( + total_norm_sum_per_block.unsqueeze(1) + eps + ) # (C, H*S) + cond_flat = ratio_flat >= remain_ratio.unsqueeze(1) # (C, H*S) boolean mask + any_cond = cond_flat.any(dim=1) # (C,) + # Find the index of the first True value along dim 1. If none, use HS-1. + cutoff_flat = torch.where( + any_cond, + cond_flat.float().argmax(dim=1), + torch.full_like(any_cond, fill_value=HS - 1), + ) # (C,) + + # 7) build a mask in sorted order up to that cutoff per block + idx_range_flat = torch.arange(HS, device=gate.device).unsqueeze(0) # (1, H*S) + sorted_mask_flat = idx_range_flat <= cutoff_flat.unsqueeze(1) # (C, H*S) + + # 8) scatter it back to original order per block + others_mask_flat = torch.zeros_like(others_flat, dtype=torch.bool) # (C, H*S) + others_mask_flat.scatter_(1, sorted_indices_flat, sorted_mask_flat) + others_mask = others_mask_flat.view(C, H, S) # (C, H, S) + + # 9) finally, include every self‐chunk entry plus all selected others + final_gate_mask = valid_gate_mask & (others_mask | gate_self_chunk_mask) + + return final_gate_mask + + +def _select_threshold_overall( + gate: torch.Tensor, + valid_gate_mask: torch.Tensor, + gate_self_chunk_mask: torch.Tensor, + simsum_threshold: float, +) -> torch.Tensor: + """ + Selects triplets globally based on threshold. + Normalization and sorting happen across all valid entries. + """ + C, H, S = gate.shape + CHS = C * H * S + eps = 1e-6 + + # LSE‐style normalization globally across all valid entries + gate_masked = torch.where(valid_gate_mask, gate, -torch.inf) # Use -inf for max + gate_min_val = torch.where(valid_gate_mask, gate, torch.inf) # Use +inf for min + + overall_max = gate_masked.max() # scalar + overall_min = gate_min_val.min() # scalar + overall_denom = overall_max - overall_min + overall_denom = torch.where( + overall_denom <= eps, + torch.tensor(1.0, device=gate.device, dtype=gate.dtype), + overall_denom, + ) + + gate_norm = (gate - overall_min) / overall_denom # (C, H, S) + gate_norm = torch.where(valid_gate_mask, gate_norm, 0.0) # (C, H, S) + + # 1) identify normalized weights of entries that *are* self-chunks + self_norm_entries = gate_norm * gate_self_chunk_mask # (C, H, S) + # Sum these weights globally + self_norm_sum_overall = self_norm_entries.sum() # scalar + + # 2) compute how much more normalized weight is needed globally beyond self-chunk contributions + total_norm_sum_overall = gate_norm.sum() # scalar + remain_ratio = simsum_threshold - self_norm_sum_overall / ( + total_norm_sum_overall + eps + ) # scalar + remain_ratio = torch.clamp(remain_ratio, min=0.0) # scalar + + # 3) zero out the self‐chunk entries in a copy, so we only sort “others” + others_norm = gate_norm.clone() + others_norm[gate_self_chunk_mask] = 0.0 # Zero out self entries + + # 4) sort all other entries by descending norm, globally + others_flat = others_norm.flatten() # (C*H*S,) + valid_others_mask_flat = ( + valid_gate_mask.flatten() & ~gate_self_chunk_mask.flatten() + ) # Mask for valid, non-self entries + + # Only sort the valid 'other' entries + valid_others_indices = torch.where(valid_others_mask_flat)[0] + valid_others_values = others_flat[valid_others_indices] + + sorted_others_values, sort_perm = torch.sort( + valid_others_values, descending=True + ) # (N_valid_others,) + sorted_original_indices = valid_others_indices[ + sort_perm + ] # Original indices in C*H*S space, sorted by value + + # 5) cumulative‑sum the sorted valid 'other' norms globally + cumsum_others_values = sorted_others_values.cumsum(dim=0) # (N_valid_others,) + + # 6) find the smallest k where cumsum_ratio ≥ remain_ratio globally + ratio_values = cumsum_others_values / ( + total_norm_sum_overall + eps + ) # (N_valid_others,) + cond_values = ratio_values >= remain_ratio # (N_valid_others,) boolean mask + any_cond = cond_values.any() # scalar + + # Find the index of the first True value in the *sorted* list. If none, use all valid others. + cutoff_idx_in_sorted = torch.where( + any_cond, + cond_values.float().argmax(dim=0), + torch.tensor( + len(sorted_others_values) - 1, device=gate.device, dtype=torch.long + ), + ) + + # 7) build a mask selecting the top-k others based on the cutoff + # Select the original indices corresponding to the top entries in the sorted list + selected_other_indices = sorted_original_indices[: cutoff_idx_in_sorted + 1] + + # 8) create the mask in the original flat shape + others_mask_flat = torch.zeros_like(others_flat, dtype=torch.bool) # (C*H*S,) + if selected_other_indices.numel() > 0: # Check if any 'other' indices were selected + others_mask_flat[selected_other_indices] = True + others_mask = others_mask_flat.view(C, H, S) # (C, H, S) + + # 9) finally, include every self‐chunk entry plus all selected others + final_gate_mask = valid_gate_mask & (others_mask | gate_self_chunk_mask) + + return final_gate_mask + + +def _select_threshold_head_global( + gate: torch.Tensor, + valid_gate_mask: torch.Tensor, + gate_self_chunk_mask: torch.Tensor, + simsum_threshold: float, +) -> torch.Tensor: + """ + Selects globally for each head based on threshold. + """ + C, H, S = gate.shape + eps = 1e-6 + + # 1) LSE‐style normalization per head (across chunks and sequence dims) + gate_masked = torch.where(valid_gate_mask, gate, -torch.inf) + gate_min_val = torch.where(valid_gate_mask, gate, torch.inf) + + max_per_head = gate_masked.amax(dim=(0, 2), keepdim=True) # (1, H, 1) + min_per_head = gate_min_val.amin(dim=(0, 2), keepdim=True) # (1, H, 1) + denom = max_per_head - min_per_head + denom = torch.where(denom <= eps, torch.ones_like(denom), denom) + + gate_norm = (gate - min_per_head) / denom + gate_norm = torch.where(valid_gate_mask, gate_norm, 0.0) # (C, H, S) + + # 2) sum normalized self‐chunk contributions per head + self_norm_sum = (gate_norm * gate_self_chunk_mask).sum(dim=(0, 2)) # (H,) + + # 3) total normalized sum per head + total_norm_sum = gate_norm.sum(dim=(0, 2)) # (H,) + + # 4) how much more normalized weight needed per head + remain_ratio = simsum_threshold - self_norm_sum / (total_norm_sum + eps) # (H,) + remain_ratio = torch.clamp(remain_ratio, min=0.0) + + # 5) zero out self‐chunk entries to focus on "others" + others_norm = gate_norm.clone() + others_norm[gate_self_chunk_mask] = 0.0 # (C, H, S) + + # 6) flatten chunk and sequence dims, per head + CS = C * S + others_flat = others_norm.permute(1, 0, 2).reshape(H, CS) # (H, C*S) + valid_flat = ( + (valid_gate_mask & ~gate_self_chunk_mask).permute(1, 0, 2).reshape(H, CS) + ) # (H, C*S) + + # 7) vectorized selection of “others” per head + masked_flat = torch.where(valid_flat, others_flat, torch.zeros_like(others_flat)) + sorted_vals, sorted_idx = torch.sort( + masked_flat, dim=1, descending=True + ) # (H, C*S) + + cumsum_vals = sorted_vals.cumsum(dim=1) # (H, C*S) + ratio_vals = cumsum_vals / (total_norm_sum.unsqueeze(1) + eps) # (H, C*S) + cond = ratio_vals >= remain_ratio.unsqueeze(1) # (H, C*S) + + has_cutoff = cond.any(dim=1) # (H,) + default = torch.full((H,), CS - 1, device=gate.device, dtype=torch.long) + cutoff = torch.where(has_cutoff, cond.float().argmax(dim=1), default) # (H,) + + idx_range = torch.arange(CS, device=gate.device).unsqueeze(0) # (1, C*S) + sorted_mask = idx_range <= cutoff.unsqueeze(1) # (H, C*S) + + selected_flat = torch.zeros_like(valid_flat) # (H, C*S) + selected_flat.scatter_(1, sorted_idx, sorted_mask) # (H, C*S) + + # 8) reshape selection mask back to (C, H, S) + others_mask = selected_flat.reshape(H, C, S).permute(1, 0, 2) # (C, H, S) + + # 9) include self‐chunks plus selected others, and obey valid mask + final_gate_mask = valid_gate_mask & (gate_self_chunk_mask | others_mask) + + return final_gate_mask + + +class MixedAttention(torch.autograd.Function): + @staticmethod + def forward( + ctx, + q, + k, + v, + self_attn_cu_seqlen, + moba_q, + moba_kv, + moba_cu_seqlen_q, + moba_cu_seqlen_kv, + max_seqlen, + moba_chunk_size, + moba_q_sh_indices, + ): + ctx.max_seqlen = max_seqlen + ctx.moba_chunk_size = moba_chunk_size + ctx.softmax_scale = softmax_scale = q.shape[-1] ** (-0.5) + + # Non-causal self-attention branch + # return out, softmax_lse, S_dmask, rng_state + self_attn_out_sh, self_attn_lse_hs, _, _ = _flash_attn_varlen_forward( + q=q, + k=k, + v=v, + cu_seqlens_q=self_attn_cu_seqlen, + cu_seqlens_k=self_attn_cu_seqlen, + max_seqlen_q=max_seqlen, + max_seqlen_k=max_seqlen, + softmax_scale=softmax_scale, + causal=False, + dropout_p=0.0, + ) + # MOBA attention branch (non-causal) + moba_attn_out, moba_attn_lse_hs, _, _ = _flash_attn_varlen_forward( + q=moba_q, + k=moba_kv[:, 0], + v=moba_kv[:, 1], + cu_seqlens_q=moba_cu_seqlen_q, + cu_seqlens_k=moba_cu_seqlen_kv, + max_seqlen_q=max_seqlen, + max_seqlen_k=moba_chunk_size, + softmax_scale=softmax_scale, + causal=False, + dropout_p=0.0, + ) + + self_attn_lse_sh = self_attn_lse_hs.t().contiguous() + moba_attn_lse = moba_attn_lse_hs.t().contiguous() + + output = torch.zeros( + (q.shape[0], q.shape[1], q.shape[2]), device=q.device, dtype=torch.float32 + ) + output_2d = output.view(-1, q.shape[2]) + + max_lse_1d = self_attn_lse_sh.view(-1) + max_lse_1d = max_lse_1d.index_reduce( + 0, moba_q_sh_indices, moba_attn_lse.view(-1), "amax" + ) + self_attn_lse_sh = self_attn_lse_sh - max_lse_1d.view_as(self_attn_lse_sh) + moba_attn_lse = ( + moba_attn_lse.view(-1) + .sub(max_lse_1d.index_select(0, moba_q_sh_indices)) + .reshape_as(moba_attn_lse) + ) + + mixed_attn_se_sh = self_attn_lse_sh.exp() + moba_attn_se = moba_attn_lse.exp() + + mixed_attn_se_sh.view(-1).index_add_( + 0, moba_q_sh_indices, moba_attn_se.view(-1) + ) + mixed_attn_lse_sh = mixed_attn_se_sh.log() + + # Combine self-attention output + factor = (self_attn_lse_sh - mixed_attn_lse_sh).exp() # [S, H] + self_attn_out_sh = self_attn_out_sh * factor.unsqueeze(-1) + output_2d += self_attn_out_sh.reshape_as(output_2d) + + # Combine MOBA attention output + mixed_attn_lse = ( + mixed_attn_lse_sh.view(-1) + .index_select(0, moba_q_sh_indices) + .view_as(moba_attn_lse) + ) + factor = (moba_attn_lse - mixed_attn_lse).exp() # [S, H] + moba_attn_out = moba_attn_out * factor.unsqueeze(-1) + raw_attn_out = moba_attn_out.view(-1, moba_attn_out.shape[-1]) + output_2d.index_add_(0, moba_q_sh_indices, raw_attn_out) + output = output.to(q.dtype) + mixed_attn_lse_sh = mixed_attn_lse_sh + max_lse_1d.view_as(mixed_attn_se_sh) + ctx.save_for_backward( + output, + mixed_attn_lse_sh, + q, + k, + v, + self_attn_cu_seqlen, + moba_q, + moba_kv, + moba_cu_seqlen_q, + moba_cu_seqlen_kv, + moba_q_sh_indices, + ) + + return output + + @staticmethod + def backward(ctx, d_output): + + max_seqlen = ctx.max_seqlen + moba_chunk_size = ctx.moba_chunk_size + softmax_scale = ctx.softmax_scale + + ( + output, + mixed_attn_vlse_sh, + q, + k, + v, + self_attn_cu_seqlen, + moba_q, + moba_kv, + moba_cu_seqlen_q, + moba_cu_seqlen_kv, + moba_q_sh_indices, + ) = ctx.saved_tensors + + d_output = d_output.contiguous() + + dq = torch.empty_like(q) + dk = torch.empty_like(k) + dv = torch.empty_like(v) + _ = _flash_attn_varlen_backward( + dout=d_output, + q=q, + k=k, + v=v, + out=output, + softmax_lse=mixed_attn_vlse_sh.t().contiguous(), + dq=dq, + dk=dk, + dv=dv, + cu_seqlens_q=self_attn_cu_seqlen, + cu_seqlens_k=self_attn_cu_seqlen, + max_seqlen_q=max_seqlen, + max_seqlen_k=max_seqlen, + softmax_scale=softmax_scale, + causal=False, + dropout_p=0.0, + softcap=0.0, + alibi_slopes=None, + deterministic=True, + window_size_left=-1, + window_size_right=-1, + ) + + headdim = q.shape[-1] + d_moba_output = ( + d_output.view(-1, headdim).index_select(0, moba_q_sh_indices).unsqueeze(1) + ) + moba_output = ( + output.view(-1, headdim).index_select(0, moba_q_sh_indices).unsqueeze(1) + ) + + mixed_attn_vlse = ( + mixed_attn_vlse_sh.view(-1).index_select(0, moba_q_sh_indices).view(1, -1) + ) + + dmq = torch.empty_like(moba_q) + dmkv = torch.empty_like(moba_kv) + _ = _flash_attn_varlen_backward( + dout=d_moba_output, + q=moba_q, + k=moba_kv[:, 0], + v=moba_kv[:, 1], + out=moba_output, + softmax_lse=mixed_attn_vlse, + dq=dmq, + dk=dmkv[:, 0], + dv=dmkv[:, 1], + cu_seqlens_q=moba_cu_seqlen_q, + cu_seqlens_k=moba_cu_seqlen_kv, + max_seqlen_q=max_seqlen, + max_seqlen_k=moba_chunk_size, + softmax_scale=softmax_scale, + causal=False, + dropout_p=0.0, + softcap=0.0, + alibi_slopes=None, + deterministic=True, + window_size_left=-1, + window_size_right=-1, + ) + + return dq, dk, dv, None, dmq, dmkv, None, None, None, None, None + + +def moba_attn_varlen( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + cu_seqlens: torch.Tensor, + max_seqlen: int, + moba_chunk_size: int, + moba_topk: int, + select_mode: str = "threshold", # "topk" or "threshold" + simsum_threshold: float = 0.25, + threshold_type: str = "query_head", +) -> torch.Tensor: + """ + Accelerated MOBA attention for vision tasks with proper LSE normalization. + + This version: + - Splits KV into chunks. + - For each query head, selects the top-k relevant KV chunks (including the self chunk) + by amplifying the diagonal (self-chunk) logits. + - Aggregates the attention outputs from the selected chunks using a log-sum-exp + reduction so that attending to each query over the selected chunks is equivalent + to the original algorithm. + """ + # Stack keys and values. + kv = torch.stack((k, v), dim=1) + seqlen, num_head, head_dim = q.shape + + # Compute chunk boundaries. + cu_chunk, filtered_chunk_indices, num_filtered_chunk, chunk_to_batch = calc_chunks( + cu_seqlens, moba_chunk_size + ) + + self_attn_cu_seqlen = cu_chunk + + # Update top-k selection to include the self chunk. + moba_topk = min(moba_topk, num_filtered_chunk) + + # --- Build filtered KV from chunks --- + chunk_starts = cu_chunk[filtered_chunk_indices] # [num_filtered_chunk] + chunk_ends = cu_chunk[filtered_chunk_indices + 1] # [num_filtered_chunk] + chunk_lengths = chunk_ends - chunk_starts # [num_filtered_chunk] + max_chunk_len = int(chunk_lengths.max().item()) + + range_tensor = torch.arange( + max_chunk_len, device=kv.device, dtype=chunk_starts.dtype + ).unsqueeze(0) + indices = chunk_starts.unsqueeze(1) + range_tensor + indices = torch.clamp(indices, max=kv.shape[0] - 1) + valid_mask = range_tensor < chunk_lengths.unsqueeze(1) + gathered = kv[indices.view(-1)].view( + num_filtered_chunk, max_chunk_len, *kv.shape[1:] + ) + gathered = gathered * valid_mask.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1).type_as( + gathered + ) + + # Compute key_gate_weight over valid tokens. + key_values = gathered[ + :, :, 0 + ].float() # [num_filtered_chunk, max_chunk_len, num_head, head_dim] + valid_mask_exp = valid_mask.unsqueeze(-1).unsqueeze(-1) + key_sum = (key_values * valid_mask_exp).sum(dim=1) + divisor = valid_mask.sum(dim=1).unsqueeze(-1).unsqueeze(-1) + key_gate_weight = key_sum / divisor # [num_filtered_chunk, num_head, head_dim] + + # Compute gate logits between key_gate_weight and queries. + q_float = q.float() + # gate = torch.einsum("nhd,shd->nhs", key_gate_weight, q_float) # [num_filtered_chunk, num_head, seqlen] + gate = torch.bmm( + key_gate_weight.permute(1, 0, 2), q_float.permute(1, 0, 2).transpose(1, 2) + ).permute(1, 0, 2) + + # Amplify the diagonal (self chunk) contributions. + gate_seq_idx = ( + torch.arange(seqlen, device=q.device, dtype=torch.int32) + .unsqueeze(0) + .expand(num_filtered_chunk, seqlen) + ) + chunk_start = cu_chunk[filtered_chunk_indices] # [num_filtered_chunk] + chunk_end = cu_chunk[filtered_chunk_indices + 1] # [num_filtered_chunk] + gate_self_chunk_mask = ( + ( + (gate_seq_idx >= chunk_start.unsqueeze(1)) + & (gate_seq_idx < chunk_end.unsqueeze(1)) + ) + .unsqueeze(1) + .expand(-1, num_head, -1) + ) + amplification_factor = 1e9 # Example factor; adjust as needed. + origin_gate = gate.clone() + gate = gate.clone() + if select_mode == "topk": + gate[gate_self_chunk_mask] += amplification_factor + + # Exclude positions that are outside the valid batch boundaries. + batch_starts = cu_seqlens[chunk_to_batch[filtered_chunk_indices]] + batch_ends = cu_seqlens[chunk_to_batch[filtered_chunk_indices] + 1] + gate_batch_start_mask = gate_seq_idx < batch_starts.unsqueeze(1) + gate_batch_end_mask = gate_seq_idx >= batch_ends.unsqueeze(1) + gate_inf_mask = gate_batch_start_mask | gate_batch_end_mask + gate.masked_fill_(gate_inf_mask.unsqueeze(1), -float("inf")) + + if select_mode == "topk": + # We amplify self‐chunk in gate already, so self entries will rank highest. + valid_gate_mask = gate != -float("inf") + if threshold_type == "query_head": + # === per‐ top-k across chunks (original behavior) === + # gate: (C, H, S) + _, gate_topk_idx = torch.topk( + gate, k=moba_topk, dim=0, largest=True, sorted=False + ) + gate_idx_mask = torch.zeros_like(gate, dtype=torch.bool) + gate_idx_mask.scatter_(0, gate_topk_idx, True) + gate_mask = valid_gate_mask & gate_idx_mask + elif threshold_type == "overall": + # === global top-k across all (chunk, head, seq) entries === + C, H, S = gate.shape + flat_gate = gate.flatten() + flat_mask = valid_gate_mask.flatten() + flat_gate_masked = torch.where(flat_mask, flat_gate, -float("inf")) + # pick topk global entries + vals, idx = torch.topk( + flat_gate_masked, k=moba_topk * H * S, largest=True, sorted=False + ) + others_mask_flat = torch.zeros_like(flat_mask, dtype=torch.bool) + others_mask_flat[idx] = True + gate_mask = (valid_gate_mask.flatten() & others_mask_flat).view(gate.shape) + elif threshold_type == "head_global": + # per-head top-k across all chunks and sequence positions + C, H, S = gate.shape + CS = C * S + flat_gate = gate.permute(1, 0, 2).reshape(H, CS) + flat_valid = valid_gate_mask.permute(1, 0, 2).reshape(H, CS) + flat_gate_masked = torch.where( + flat_valid, flat_gate, torch.full_like(flat_gate, -float("inf")) + ) + # pick top-k indices per head + _, topk_idx = torch.topk( + flat_gate_masked, k=moba_topk * S, dim=1, largest=True, sorted=False + ) + gate_idx_flat = torch.zeros_like(flat_valid, dtype=torch.bool) + gate_idx_flat.scatter_(1, topk_idx, True) + gate_mask = gate_idx_flat.reshape(H, C, S).permute(1, 0, 2) + else: + raise ValueError( + f"Invalid threshold_type for topk: {threshold_type}. " + "Choose 'query_head', 'block', or 'overall'." + ) + elif select_mode == "threshold": + # Delegate to the specific thresholding function + valid_gate_mask = gate != -float("inf") # (num_chunk, num_head, seqlen) + if threshold_type == "query_head": + gate_mask = _select_threshold_query_head( + gate, valid_gate_mask, gate_self_chunk_mask, simsum_threshold + ) + elif threshold_type == "block": + gate_mask = _select_threshold_block( + gate, valid_gate_mask, gate_self_chunk_mask, simsum_threshold + ) + elif threshold_type == "overall": + gate_mask = _select_threshold_overall( + gate, valid_gate_mask, gate_self_chunk_mask, simsum_threshold + ) + elif threshold_type == "head_global": + gate_mask = _select_threshold_head_global( + gate, valid_gate_mask, gate_self_chunk_mask, simsum_threshold + ) + else: + raise ValueError( + f"Invalid threshold_type: {threshold_type}. Choose 'query_head', 'block', or 'overall'." + ) + else: + raise ValueError( + f"Invalid select_mode: {select_mode}. Choose 'topk' or 'threshold'." + ) + + # eliminate self_chunk in MoBA branch + gate_mask = gate_mask & ~gate_self_chunk_mask + # if gate_mask is all false, perform flash_attn instead + if gate_mask.sum() == 0: + return flash_attn_varlen_func( + q, k, v, cu_seqlens, cu_seqlens, max_seqlen, max_seqlen, causal=False + ) + + # Determine which query positions are selected. + # nonzero_indices has shape [N, 3] where each row is [chunk_index, head_index, seq_index]. + moba_q_indices = gate_mask.reshape(gate_mask.shape[0], -1).nonzero(as_tuple=True)[ + -1 + ] # [(h s k)] + moba_q_sh_indices = (moba_q_indices % seqlen) * num_head + ( + moba_q_indices // seqlen + ) + moba_q = ( + rearrange(q, "s h d -> (h s) d").index_select(0, moba_q_indices).unsqueeze(1) + ) + + # Build cumulative sequence lengths for the selected queries. + moba_seqlen_q = gate_mask.sum(dim=-1).flatten() + q_zero_mask = moba_seqlen_q == 0 + valid_expert_mask = ~q_zero_mask + if q_zero_mask.sum() > 0: + moba_seqlen_q = moba_seqlen_q[valid_expert_mask] + moba_cu_seqlen_q = torch.cat( + ( + torch.tensor([0], device=q.device, dtype=moba_seqlen_q.dtype), + moba_seqlen_q.cumsum(dim=0), + ), + dim=0, + ).to(torch.int32) + + # Rearrange gathered KV for the MOBA branch. + experts_tensor = rearrange(gathered, "nc cl two h d -> (nc h) cl two d") + valid_expert_lengths = ( + chunk_lengths.unsqueeze(1) + .expand(num_filtered_chunk, num_head) + .reshape(-1) + .to(torch.int32) + ) + if q_zero_mask.sum() > 0: + experts_tensor = experts_tensor[valid_expert_mask] + valid_expert_lengths = valid_expert_lengths[valid_expert_mask] + + seq_range = torch.arange( + experts_tensor.shape[1], device=experts_tensor.device + ).unsqueeze(0) + mask = seq_range < valid_expert_lengths.unsqueeze(1) + moba_kv = experts_tensor[mask] # Shape: ((nc h cl_valid) two d) + moba_kv = moba_kv.unsqueeze(2) # Shape: ((nc h cl_valid) two 1 d) + + moba_cu_seqlen_kv = torch.cat( + [ + torch.zeros(1, device=experts_tensor.device, dtype=torch.int32), + valid_expert_lengths.cumsum(dim=0), + ], + dim=0, + ).to(torch.int32) + + assert ( + moba_cu_seqlen_kv.shape == moba_cu_seqlen_q.shape + ), f"Mismatch between moba_cu_seqlen_kv.shape and moba_cu_seqlen_q.shape: {moba_cu_seqlen_kv.shape} vs {moba_cu_seqlen_q.shape}" + + return MixedAttention.apply( + q, + k, + v, + self_attn_cu_seqlen, + moba_q, + moba_kv, + moba_cu_seqlen_q, + moba_cu_seqlen_kv, + max_seqlen, + moba_chunk_size, + moba_q_sh_indices, + ) + + +def process_moba_input( + x, + patch_resolution, + chunk_size, +): + """ + Process inputs for the attention function. + + Args: + x (torch.Tensor): Input tensor with shape [batch_size, num_patches, num_heads, head_dim]. + patch_resolution (tuple): Tuple containing the patch resolution (t, h, w). + chunk_size (int): Size of the chunk. (maybe tuple or int, according to chunk type) + + Returns: + torch.Tensor: Processed input tensor. + """ + if isinstance(chunk_size, float) or isinstance(chunk_size, int): + moba_chunk_size = int(chunk_size * patch_resolution[1] * patch_resolution[2]) + else: + assert isinstance( + chunk_size, (Tuple, list) + ), f"chunk_size should be a tuple, list, or int, now it is: {type(chunk_size)}" + if len(chunk_size) == 2: + assert ( + patch_resolution[1] % chunk_size[0] == 0 + and patch_resolution[2] % chunk_size[1] == 0 + ), f"spatial patch_resolution {patch_resolution[1:]} should be divisible by 2d chunk_size {chunk_size}" + nch, ncw = ( + patch_resolution[1] // chunk_size[0], + patch_resolution[2] // chunk_size[1], + ) + x = rearrange( + x, + "b (t nch ch ncw cw) n d -> b (nch ncw t ch cw) n d", + t=patch_resolution[0], + nch=nch, + ncw=ncw, + ch=chunk_size[0], + cw=chunk_size[1], + ) + moba_chunk_size = patch_resolution[0] * chunk_size[0] * chunk_size[1] + elif len(chunk_size) == 3: + assert ( + patch_resolution[0] % chunk_size[0] == 0 + and patch_resolution[1] % chunk_size[1] == 0 + and patch_resolution[2] % chunk_size[2] == 0 + ), f"patch_resolution {patch_resolution} should be divisible by 3d chunk_size {chunk_size}" + nct, nch, ncw = ( + patch_resolution[0] // chunk_size[0], + patch_resolution[1] // chunk_size[1], + patch_resolution[2] // chunk_size[2], + ) + x = rearrange( + x, + "b (nct ct nch ch ncw cw) n d -> b (nct nch ncw ct ch cw) n d", + nct=nct, + nch=nch, + ncw=ncw, + ct=chunk_size[0], + ch=chunk_size[1], + cw=chunk_size[2], + ) + moba_chunk_size = chunk_size[0] * chunk_size[1] * chunk_size[2] + else: + raise ValueError( + f"chunk_size should be a int, or a tuple of length 2 or 3, now it is: {len(chunk_size)}" + ) + + return x, moba_chunk_size + + +def process_moba_output( + x, + patch_resolution, + chunk_size, +): + if isinstance(chunk_size, float) or isinstance(chunk_size, int): + pass + elif len(chunk_size) == 2: + x = rearrange( + x, + "b (nch ncw t ch cw) n d -> b (t nch ch ncw cw) n d", + nch=patch_resolution[1] // chunk_size[0], + ncw=patch_resolution[2] // chunk_size[1], + t=patch_resolution[0], + ch=chunk_size[0], + cw=chunk_size[1], + ) + elif len(chunk_size) == 3: + x = rearrange( + x, + "b (nct nch ncw ct ch cw) n d -> b (nct ct nch ch ncw cw) n d", + nct=patch_resolution[0] // chunk_size[0], + nch=patch_resolution[1] // chunk_size[1], + ncw=patch_resolution[2] // chunk_size[2], + ct=chunk_size[0], + ch=chunk_size[1], + cw=chunk_size[2], + ) + + return x + + +# TEST +def generate_data(batch_size, seqlen, num_head, head_dim, dtype): + random.seed(0) + torch.manual_seed(0) + torch.cuda.manual_seed(0) + device = torch.cuda.current_device() + + q = torch.randn((batch_size, seqlen, num_head, head_dim), requires_grad=True).to( + dtype=dtype, device="cuda" + ) + k = torch.randn((batch_size, seqlen, num_head, head_dim), requires_grad=True).to( + dtype=dtype, device="cuda" + ) + v = torch.randn((batch_size, seqlen, num_head, head_dim), requires_grad=True).to( + dtype=dtype, device="cuda" + ) + print(f"q.shape: {q.shape}, k.shape: {k.shape}, v.shape: {v.shape}") + cu_seqlens = torch.arange( + 0, q.shape[0] * q.shape[1] + 1, q.shape[1], dtype=torch.int32, device="cuda" + ) + max_seqlen = q.shape[1] + q = rearrange(q, "b s ... -> (b s) ...") + k = rearrange(k, "b s ... -> (b s) ...") + v = rearrange(v, "b s ... -> (b s) ...") + + return q, k, v, cu_seqlens, max_seqlen + + +def test_attn_varlen_moba_speed( + batch, + head, + seqlen, + head_dim, + moba_chunk_size, + moba_topk, + dtype=torch.bfloat16, + select_mode="threshold", + simsum_threshold=0.25, + threshold_type="query_head", +): + """Speed test comparing flash_attn vs moba_attention""" + # Get data + q, k, v, cu_seqlen, max_seqlen = generate_data(batch, seqlen, head, head_dim, dtype) + print( + f"batch:{batch} head:{head} seqlen:{seqlen} chunk:{moba_chunk_size} topk:{moba_topk} select_mode: {select_mode} simsum_threshold:{simsum_threshold}" + ) + vo_grad = torch.randn_like(q) + + # Warmup + warmup_iters = 3 + perf_test_iters = 10 + + # Warmup + for _ in range(warmup_iters): + o = flash_attn_varlen_func( + q, k, v, cu_seqlen, cu_seqlen, max_seqlen, max_seqlen, causal=False + ) + torch.autograd.backward(o, vo_grad) + + torch.cuda.synchronize() + start_flash = time.perf_counter() + for _ in range(perf_test_iters): + o = flash_attn_varlen_func( + q, k, v, cu_seqlen, cu_seqlen, max_seqlen, max_seqlen, causal=False + ) + torch.autograd.backward(o, vo_grad) + + torch.cuda.synchronize() + time_flash = (time.perf_counter() - start_flash) / perf_test_iters * 1000 + + # Warmup + for _ in range(warmup_iters): + om = moba_attn_varlen( + q, + k, + v, + cu_seqlen, + max_seqlen, + moba_chunk_size=moba_chunk_size, + moba_topk=moba_topk, + select_mode=select_mode, + simsum_threshold=simsum_threshold, + threshold_type=threshold_type, + ) + torch.autograd.backward(om, vo_grad) + + torch.cuda.synchronize() + start_moba = time.perf_counter() + for _ in range(perf_test_iters): + om = moba_attn_varlen( + q, + k, + v, + cu_seqlen, + max_seqlen, + moba_chunk_size=moba_chunk_size, + moba_topk=moba_topk, + select_mode=select_mode, + simsum_threshold=simsum_threshold, + threshold_type=threshold_type, + ) + torch.autograd.backward(om, vo_grad) + + torch.cuda.synchronize() + time_moba = (time.perf_counter() - start_moba) / perf_test_iters * 1000 + + print(f"Flash: {time_flash:.2f}ms, MoBA: {time_moba:.2f}ms") + print(f"Speedup: {time_flash / time_moba:.2f}x") + + +if __name__ == "__main__": + """ + CUDA_VISIBLE_DEVICES=1 \ + python -u csrc/attn/vmoba_attn/vmoba/vmoba.py + """ + test_attn_varlen_moba_speed( + batch=1, + head=12, + seqlen=32760, + head_dim=128, + moba_chunk_size=32760 // 3 // 6 // 4, + moba_topk=3, + select_mode="threshold", + simsum_threshold=0.3, + threshold_type="query_head", + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/cli.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/cli.md new file mode 100644 index 00000000..4b0f29c7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/cli.md @@ -0,0 +1,184 @@ +# SGLang diffusion CLI Inference + +The SGLang-diffusion CLI provides a quick way to access the inference pipeline for image and video generation. + +## Prerequisites + +- A working SGLang diffusion installation and the `sglang` CLI available in `$PATH`. +- Python 3.11+ if you plan to use the OpenAI Python SDK. + + +## Supported Arguments + +### Server Arguments + +- `--model-path {MODEL_PATH}`: Path to the model or model ID +- `--num-gpus {NUM_GPUS}`: Number of GPUs to use +- `--tp-size {TP_SIZE}`: Tensor parallelism size (only for the encoder; should not be larger than 1 if text encoder offload is enabled, as layer-wise offload plus prefetch is faster) +- `--sp-size {SP_SIZE}`: Sequence parallelism size (typically should match the number of GPUs) +- `--ulysses-degree {ULYSSES_DEGREE}`: The degree of DeepSpeed-Ulysses-style SP in USP +- `--ring-degree {RING_DEGREE}`: The degree of ring attention-style SP in USP + + +### Sampling Parameters + +- `--prompt {PROMPT}`: Text description for the video you want to generate +- `--num-inference-steps {STEPS}`: Number of denoising steps +- `--negative-prompt {PROMPT}`: Negative prompt to guide generation away from certain concepts +- `--seed {SEED}`: Random seed for reproducible generation + + +#### Image/Video Configuration + +- `--height {HEIGHT}`: Height of the generated output +- `--width {WIDTH}`: Width of the generated output +- `--num-frames {NUM_FRAMES}`: Number of frames to generate +- `--fps {FPS}`: Frames per second for the saved output, if this is a video-generation task + + +#### Output Options + +- `--output-path {PATH}`: Directory to save the generated video +- `--save-output`: Whether to save the image/video to disk +- `--return-frames`: Whether to return the raw frames + +### Using Configuration Files + +Instead of specifying all parameters on the command line, you can use a configuration file: + +```bash +sglang generate --config {CONFIG_FILE_PATH} +``` + +The configuration file should be in JSON or YAML format with the same parameter names as the CLI options. Command-line arguments take precedence over settings in the configuration file, allowing you to override specific values while keeping the rest from the configuration file. + +Example configuration file (config.json): + +```json +{ + "model_path": "FastVideo/FastHunyuan-diffusers", + "prompt": "A beautiful woman in a red dress walking down a street", + "output_path": "outputs/", + "num_gpus": 2, + "sp_size": 2, + "tp_size": 1, + "num_frames": 45, + "height": 720, + "width": 1280, + "num_inference_steps": 6, + "seed": 1024, + "fps": 24, + "precision": "bf16", + "vae_precision": "fp16", + "vae_tiling": true, + "vae_sp": true, + "vae_config": { + "load_encoder": false, + "load_decoder": true, + "tile_sample_min_height": 256, + "tile_sample_min_width": 256 + }, + "text_encoder_precisions": [ + "fp16", + "fp16" + ], + "mask_strategy_file_path": null, + "enable_torch_compile": false +} +``` + +Or using YAML format (config.yaml): + +```yaml +model_path: "FastVideo/FastHunyuan-diffusers" +prompt: "A beautiful woman in a red dress walking down a street" +output_path: "outputs/" +num_gpus: 2 +sp_size: 2 +tp_size: 1 +num_frames: 45 +height: 720 +width: 1280 +num_inference_steps: 6 +seed: 1024 +fps: 24 +precision: "bf16" +vae_precision: "fp16" +vae_tiling: true +vae_sp: true +vae_config: + load_encoder: false + load_decoder: true + tile_sample_min_height: 256 + tile_sample_min_width: 256 +text_encoder_precisions: + - "fp16" + - "fp16" +mask_strategy_file_path: null +enable_torch_compile: false +``` + + +To see all the options, you can use the `--help` flag: + +```bash +sglang generate --help +``` + +## Serve + +Launch the SGLang diffusion HTTP server and interact with it using the OpenAI SDK and curl. + +### Start the server + +Use the following command to launch the server: + +```bash +SERVER_ARGS=( + --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers + --text-encoder-cpu-offload + --pin-cpu-memory + --num-gpus 4 + --ulysses-degree=2 + --ring-degree=2 +) + +sglang serve "${SERVER_ARGS[@]}" +``` + +- **--model-path**: Which model to load. The example uses `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`. +- **--port**: HTTP port to listen on (the default here is `30010`). + +For detailed API usage, including Image, Video Generation and LoRA management, please refer to the [OpenAI API Documentation](openai_api.md). + + +## Generate + +Run a one-off generation task without launching a persistent server. + +To use it, pass both server arguments and sampling parameters in one command, after the `generate` subcommand, for example: + +```bash +SERVER_ARGS=( + --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers + --text-encoder-cpu-offload + --pin-cpu-memory + --num-gpus 4 + --ulysses-degree=2 + --ring-degree=2 +) + +SAMPLING_ARGS=( + --prompt "A curious raccoon" + --save-output + --output-path outputs + --output-file-name "A curious raccoon.mp4" +) + +sglang generate "${SERVER_ARGS[@]}" "${SAMPLING_ARGS[@]}" +``` + +Once the generation task has finished, the server will shut down automatically. + +> [!NOTE] +> The HTTP server-related arguments are ignored in this subcommand. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/contributing.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/contributing.md new file mode 100644 index 00000000..fb8b4456 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/contributing.md @@ -0,0 +1,56 @@ +# Contributing to SGLang Diffusion + +This guide outlines the requirements for contributing to the SGLang Diffusion module (`sglang.multimodal_gen`). + +## 1. Commit Message Convention + +We follow a structured commit message format to maintain a clean history. + +**Format:** +```text +[diffusion] : +``` + +**Examples:** +- `[diffusion] cli: add --perf-dump-path argument` +- `[diffusion] scheduler: fix deadlock in batch processing` +- `[diffusion] model: support Stable Diffusion 3.5` + +**Rules:** +- **Prefix**: Always start with `[diffusion]`. +- **Scope** (Optional): `cli`, `scheduler`, `model`, `pipeline`, `docs`, etc. +- **Subject**: Imperative mood, short and clear (e.g., "add feature" not "added feature"). + +## 2. Performance Reporting + +For PRs that impact **latency**, **throughput**, or **memory usage**, you **should** provide a performance comparison report. + +### How to Generate a Report + +1. **Baseline**: run the benchmark (for a single generation task) + ```bash + $ sglang generate --model-path --prompt "A benchmark prompt" --perf-dump-path baseline.json + ``` + +2. **New**: run the same benchmark, without modifying any server_args or sampling_params + ```bash + $ sglang generate --model-path --prompt "A benchmark prompt" --perf-dump-path new.json + ``` + +3. **Compare**: run the compare script, which will print a Markdown table to the console + ```bash + $ python python/sglang/multimodal_gen/benchmarks/compare_perf.py baseline.json new.json + ### Performance Comparison Report + ... + ``` +4. **Paste**: paste the table into the PR description + +## 3. CI-Based Change Protection + +Consider adding tests to the `pr-test` or `nightly-test` suites to safeguard your changes, especially for PRs that: + +1. support a new model +2. support or fix important features +3. significantly improve performance + +See [test](https://github.com/sgl-project/sglang/tree/main/python/sglang/multimodal_gen/test) for examples diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/install.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/install.md new file mode 100644 index 00000000..894a414b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/install.md @@ -0,0 +1,48 @@ +# Install SGLang-diffusion + +You can install sglang-diffusion using one of the methods below. + +This page primarily applies to common NVIDIA GPU platforms. + +## Method 1: With pip or uv + +It is recommended to use uv for a faster installation: + +```bash +pip install --upgrade pip +pip install uv +uv pip install "sglang[diffusion]" --prerelease=allow +``` + +## Method 2: From source + +```bash +# Use the latest release branch +git clone https://github.com/sgl-project/sglang.git +cd sglang + +# Install the Python packages +pip install --upgrade pip +pip install -e "python[diffusion]" + +# With uv +uv pip install -e "python[diffusion]" --prerelease=allow +``` + +## Method 3: Using Docker + +The Docker images are available on Docker Hub at [lmsysorg/sglang](), built from the [Dockerfile](https://github.com/sgl-project/sglang/tree/main/docker). +Replace `` below with your HuggingFace Hub [token](https://huggingface.co/docs/hub/en/security-tokens). + +```bash +docker run --gpus all \ + --shm-size 32g \ + -p 30000:30000 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --env "HF_TOKEN=" \ + --ipc=host \ + lmsysorg/sglang:dev \ + sglang generate --model-path black-forest-labs/FLUX.1-dev \ + --prompt "A logo With Bold Large text: SGL Diffusion" \ + --save-output +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/openai_api.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/openai_api.md new file mode 100644 index 00000000..568104f1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/openai_api.md @@ -0,0 +1,281 @@ +# SGLang Diffusion OpenAI API + +The SGLang diffusion HTTP server implements an OpenAI-compatible API for image and video generation, as well as LoRA adapter management. + +## Serve + +Launch the server using the `sglang serve` command. + +### Start the server + +```bash +SERVER_ARGS=( + --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers + --text-encoder-cpu-offload + --pin-cpu-memory + --num-gpus 4 + --ulysses-degree=2 + --ring-degree=2 + --port 30010 +) + +sglang serve "${SERVER_ARGS[@]}" +``` + +- **--model-path**: Path to the model or model ID. +- **--port**: HTTP port to listen on (default: `30000`). + +--- + +## Endpoints + +### Image Generation + +The server implements an OpenAI-compatible Images API under the `/v1/images` namespace. + +#### Create an image + +**Endpoint:** `POST /v1/images/generations` + +**Python Example (b64_json response):** + +```python +import base64 +from openai import OpenAI + +client = OpenAI(api_key="sk-proj-1234567890", base_url="http://localhost:30010/v1") + +img = client.images.generate( + prompt="A calico cat playing a piano on stage", + size="1024x1024", + n=1, + response_format="b64_json", +) + +image_bytes = base64.b64decode(img.data[0].b64_json) +with open("output.png", "wb") as f: + f.write(image_bytes) +``` + +**Curl Example:** + +```bash +curl -sS -X POST "http://localhost:30010/v1/images/generations" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-proj-1234567890" \ + -d '{ + "prompt": "A calico cat playing a piano on stage", + "size": "1024x1024", + "n": 1, + "response_format": "b64_json" + }' +``` + +> **Note** +> The `response_format=url` option is not supported for `POST /v1/images/generations` and will return a `400` error. + +#### Edit an image + +**Endpoint:** `POST /v1/images/edits` + +This endpoint accepts a multipart form upload with an input image and a text prompt. The server can return either a base64-encoded image or a URL to download the image. + +**Curl Example (b64_json response):** + +```bash +curl -sS -X POST "http://localhost:30010/v1/images/edits" \ + -H "Authorization: Bearer sk-proj-1234567890" \ + -F "image=@input.png" \ + -F "prompt=A calico cat playing a piano on stage" \ + -F "size=1024x1024" \ + -F "response_format=b64_json" +``` + +**Curl Example (URL response):** + +```bash +curl -sS -X POST "http://localhost:30010/v1/images/edits" \ + -H "Authorization: Bearer sk-proj-1234567890" \ + -F "image=@input.png" \ + -F "prompt=A calico cat playing a piano on stage" \ + -F "size=1024x1024" \ + -F "response_format=url" +``` + +#### Download image content + +When `response_format=url` is used with `POST /v1/images/edits`, the API returns a relative URL like `/v1/images//content`. + +**Endpoint:** `GET /v1/images/{image_id}/content` + +**Curl Example:** + +```bash +curl -sS -L "http://localhost:30010/v1/images//content" \ + -H "Authorization: Bearer sk-proj-1234567890" \ + -o output.png +``` + +### Video Generation + +The server implements a subset of the OpenAI Videos API under the `/v1/videos` namespace. + +#### Create a video + +**Endpoint:** `POST /v1/videos` + +**Python Example:** + +```python +from openai import OpenAI + +client = OpenAI(api_key="sk-proj-1234567890", base_url="http://localhost:30010/v1") + +video = client.videos.create( + prompt="A calico cat playing a piano on stage", + size="1280x720" +) +print(f"Video ID: {video.id}, Status: {video.status}") +``` + +**Curl Example:** + +```bash +curl -sS -X POST "http://localhost:30010/v1/videos" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-proj-1234567890" \ + -d '{ + "prompt": "A calico cat playing a piano on stage", + "size": "1280x720" + }' +``` + +#### List videos + +**Endpoint:** `GET /v1/videos` + +**Python Example:** + +```python +videos = client.videos.list() +for item in videos.data: + print(item.id, item.status) +``` + +**Curl Example:** + +```bash +curl -sS -X GET "http://localhost:30010/v1/videos" \ + -H "Authorization: Bearer sk-proj-1234567890" +``` + +#### Download video content + +**Endpoint:** `GET /v1/videos/{video_id}/content` + +**Python Example:** + +```python +import time + +# Poll for completion +while True: + page = client.videos.list() + item = next((v for v in page.data if v.id == video_id), None) + if item and item.status == "completed": + break + time.sleep(5) + +# Download content +resp = client.videos.download_content(video_id=video_id) +with open("output.mp4", "wb") as f: + f.write(resp.read()) +``` + +**Curl Example:** + +```bash +curl -sS -L "http://localhost:30010/v1/videos//content" \ + -H "Authorization: Bearer sk-proj-1234567890" \ + -o output.mp4 +``` + +--- + +### LoRA Management + +The server supports dynamic loading, merging, and unmerging of LoRA adapters. + +**Important Notes:** +- Mutual Exclusion: Only one LoRA can be *merged* (active) at a time +- Switching: To switch LoRAs, you must first `unmerge` the current one, then `set` the new one +- Caching: The server caches loaded LoRA weights in memory. Switching back to a previously loaded LoRA (same path) has little cost + +#### Set LoRA Adapter + +Loads a LoRA adapter and merges its weights into the model. + +**Endpoint:** `POST /v1/set_lora` + +**Parameters:** +- `lora_nickname` (string, required): A unique identifier for this LoRA +- `lora_path` (string, optional): Path to the `.safetensors` file or Hugging Face repo ID. Required for the first load; optional if re-activating a cached nickname + +**Curl Example:** + +```bash +curl -X POST http://localhost:30010/v1/set_lora \ + -H "Content-Type: application/json" \ + -d '{ + "lora_nickname": "lora_name", + "lora_path": "/path/to/lora.safetensors" + }' +``` + + +#### Merge LoRA Weights + +Manually merges the currently set LoRA weights into the base model. + +> [!NOTE] +> `set_lora` automatically performs a merge, so this is typically only needed if you have manually unmerged but want to re-apply the same LoRA without calling `set_lora` again.* + +**Endpoint:** `POST /v1/merge_lora_weights` + +**Curl Example:** + +```bash +curl -X POST http://localhost:30010/v1/merge_lora_weights \ + -H "Content-Type: application/json" +``` + + +#### Unmerge LoRA Weights + +Unmerges the currently active LoRA weights from the base model, restoring it to its original state. This **must** be called before setting a different LoRA. + +**Endpoint:** `POST /v1/unmerge_lora_weights` + +**Curl Example:** + +```bash +curl -X POST http://localhost:30010/v1/unmerge_lora_weights \ + -H "Content-Type: application/json" +``` + +### Example: Switching LoRAs + +1. Set LoRA A: + ```bash + curl -X POST http://localhost:30010/v1/set_lora -d '{"lora_nickname": "lora_a", "lora_path": "path/to/A"}' + ``` +2. Generate with LoRA A... +3. Unmerge LoRA A: + ```bash + curl -X POST http://localhost:30010/v1/unmerge_lora_weights + ``` +4. Set LoRA B: + ```bash + curl -X POST http://localhost:30010/v1/set_lora -d '{"lora_nickname": "lora_b", "lora_path": "path/to/B"}' + ``` +5. Generate with LoRA B... diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/support_matrix.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/support_matrix.md new file mode 100644 index 00000000..d8eeb037 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/support_matrix.md @@ -0,0 +1,48 @@ +# Compatibility Matrix + +The table below shows every supported model and the optimizations supported for them. + +The symbols used have the following meanings: + +- ✅ = Full compatibility +- ❌ = No compatibility +- ⭕ = Does not apply to this model + +## Models x Optimization + +The `HuggingFace Model ID` can be passed directly to `from_pretrained()` methods, and sglang-diffusion will use the +optimal +default parameters when initializing and generating videos. + +### Video Generation Models + +| Model Name | Hugging Face Model ID | Resolutions | TeaCache | Sliding Tile Attn | Sage Attn | Video Sparse Attention (VSA) | +|:-----------------------------|:--------------------------------------------------|:--------------------|:--------:|:-----------------:|:---------:|:----------------------------:| +| FastWan2.1 T2V 1.3B | `FastVideo/FastWan2.1-T2V-1.3B-Diffusers` | 480p | ⭕ | ⭕ | ⭕ | ✅ | +| FastWan2.2 TI2V 5B Full Attn | `FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers` | 720p | ⭕ | ⭕ | ⭕ | ✅ | +| Wan2.2 TI2V 5B | `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | 720p | ⭕ | ⭕ | ✅ | ⭕ | +| Wan2.2 T2V A14B | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 480p
720p | ❌ | ❌ | ✅ | ⭕ | +| Wan2.2 I2V A14B | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | 480p
720p | ❌ | ❌ | ✅ | ⭕ | +| HunyuanVideo | `hunyuanvideo-community/HunyuanVideo` | 720×1280
544×960 | ❌ | ✅ | ✅ | ⭕ | +| FastHunyuan | `FastVideo/FastHunyuan-diffusers` | 720×1280
544×960 | ❌ | ✅ | ✅ | ⭕ | +| Wan2.1 T2V 1.3B | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers` | 480p | ✅ | ✅ | ✅ | ⭕ | +| Wan2.1 T2V 14B | `Wan-AI/Wan2.1-T2V-14B-Diffusers` | 480p, 720p | ✅ | ✅ | ✅ | ⭕ | +| Wan2.1 I2V 480P | `Wan-AI/Wan2.1-I2V-14B-480P-Diffusers` | 480p | ✅ | ✅ | ✅ | ⭕ | +| Wan2.1 I2V 720P | `Wan-AI/Wan2.1-I2V-14B-720P-Diffusers` | 720p | ✅ | ✅ | ✅ | ⭕ | + +**Note**: Wan2.2 TI2V 5B has some quality issues when performing I2V generation. We are working on fixing this issue. + +### Image Generation Models + +| Model Name | HuggingFace Model ID | Resolutions | +|:----------------|:-------------------------------|:---------------| +| FLUX.1-dev | `black-forest-labs/FLUX.1-dev` | Any resolution | +| FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | Any resolution | +| Qwen Image | `Qwen/Qwen-Image` | Any resolution | +| Qwen Image Edit | `Qwen/Qwen-Image-Edit` | Any resolution | + +## Special requirements + +### Sliding Tile Attention + +- Currently, only Hopper GPUs (H100s) are supported. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/support_new_models.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/support_new_models.md new file mode 100644 index 00000000..e51bd68d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/docs/support_new_models.md @@ -0,0 +1,107 @@ +# How to Support New Diffusion Models + +This document explains how to add support for new diffusion models in SGLang diffusion. + +## Architecture Overview + +SGLang diffusion is engineered for both performance and flexibility, built upon a modular pipeline architecture. This +design allows developers to easily construct complex, customized pipelines for various diffusion models by combining and +reusing different components. + +At its core, the architecture revolves around two key concepts, as highlighted in our [blog post](https://lmsys.org/blog/2025-11-07-sglang-diffusion/#architecture): + +- **`ComposedPipeline`**: This class orchestrates a series of `PipelineStage`s to define the complete generation process for a specific model. It acts as the main entry point for a model and manages the data flow between the different stages of the diffusion process. +- **`PipelineStage`**: Each stage is a modular component that encapsulates a common function within the diffusion process. Examples include prompt encoding, the denoising loop, or VAE decoding. These stages are designed to be self-contained and reusable across different pipelines. + +## Key Components for Implementation + +To add support for a new diffusion model, you will primarily need to define or configure the following components: + +1. **`PipelineConfig`**: This is a dataclass that holds all the static configurations for your model pipeline. It includes paths to model components (like UNet, VAE, text encoders), precision settings (e.g., `fp16`, `bf16`), and other model-specific architectural parameters. Each model typically has its own subclass of `PipelineConfig`. + +2. **`SamplingParams`**: This dataclass defines the parameters that control the generation process at runtime. These are the user-provided inputs for a generation request, such as the `prompt`, `negative_prompt`, `guidance_scale`, `num_inference_steps`, `seed`, output dimensions (`height`, `width`), etc. + +3. **`ComposedPipeline` (not a config)**: This is the central class where you define the structure of your model's generation pipeline. You will create a new class that inherits from `ComposedPipelineBase` and, within it, instantiate and chain together the necessary `PipelineStage`s in the correct order. See `ComposedPipelineBase` and `PipelineStage` base definitions: + - [`ComposedPipelineBase`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/runtime/pipelines/composed_pipeline_base.py) + - [`PipelineStage`]( https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/runtime/pipelines/stages/base.py) + - [Central registry (models/config mapping)](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/registry.py) + +4. **Modules (components referenced by the pipeline)**: Each pipeline references a set of modules that are loaded from the model repository (e.g., Diffusers `model_index.json`) and assembled via the registry/loader. Common modules include: + - `text_encoder`: Encodes text prompts into embeddings + - `tokenizer`: Tokenizes raw text input for the text encoder(s). + - `processor`: Preprocesses images and extracts features; often used in image-to-image tasks. + - `image_encoder`: Specialized image feature extractor (may be distinct from or combined with `processor`). + - `dit/transformer`: The core denoising network (DiT/UNet architecture) operating in latent space. + - `scheduler`: Controls the timestep schedule and denoising dynamics throughout inference. + - `vae`: Variational Autoencoder for encoding/decoding between pixel space and latent space. + +## Available Pipeline Stages + +You can build your custom `ComposedPipeline` by combining the following available stages as your will. Each stage is responsible for a specific part of the generation process. + +| Stage Class | Description | +| -------------------------------- | ------------------------------------------------------------------------------------------------------- | +| `InputValidationStage` | Validates the user-provided `SamplingParams` to ensure they are correct before starting the pipeline. | +| `TextEncodingStage` | Encodes text prompts into embeddings using one or more text encoders. | +| `ImageEncodingStage` | Encodes input images into embeddings, often used in image-to-image tasks. | +| `ImageVAEEncodingStage` | Specifically encodes an input image into the latent space using a Variational Autoencoder (VAE). | +| `ConditioningStage` | Prepares the conditioning tensors (e.g., from text or image embeddings) for the denoising loop. | +| `TimestepPreparationStage` | Prepares the scheduler's timesteps for the diffusion process. | +| `LatentPreparationStage` | Creates the initial noisy latent tensor that will be denoised. | +| `DenoisingStage` | Executes the main denoising loop, iteratively applying the model (e.g., UNet) to refine the latents. | +| `DecodingStage` | Decodes the final latent tensor from the denoising loop back into pixel space (e.g., an image) using the VAE. | +| `DmdDenoisingStage` | A specialized denoising stage for certain model architectures. | +| `CausalDMDDenoisingStage` | A specialized causal denoising stage for specific video models. | + +## Example: Implementing `Qwen-Image-Edit` + +To illustrate the process, let's look at how `Qwen-Image-Edit` is implemented. The typical implementation order is: + +1. **Analyze Required Modules**: + - Study the target model's components by examining its `model_index.json` or Diffusers implementation to identify required modules: + - `processor`: Image preprocessing and feature extraction + - `scheduler`: Diffusion timestep scheduling + - `text_encoder`: Text-to-embedding conversion + - `tokenizer`: Text tokenization for the encoder + - `transformer`: Core DiT denoising network + - `vae`: Variational autoencoder for latent encoding/decoding + +2. **Create Configs**: + - **PipelineConfig**: [`QwenImageEditPipelineConfig`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/configs/pipelines/qwen_image.py) defines model-specific parameters, precision settings, preprocessing functions, and latent shape calculations. + - **SamplingParams**: [`QwenImageSamplingParams`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/configs/sample/qwenimage.py) sets runtime defaults like `num_frames=1`, `guidance_scale=4.0`, `num_inference_steps=50`. + +3. **Implement Model Components**: + - Adapt or implement specific model components in the appropriate directories: + - **DiT/Transformer**: Implement in [`runtime/models/dits/`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/runtime/models/dits/) - e.g., [`qwen_image.py`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py) for Qwen's DiT architecture + - **Encoders**: Implement in [`runtime/models/encoders/`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/runtime/models/encoders/) - e.g., text encoders like [`qwen2_5vl.py`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py) + - **VAEs**: Implement in [`runtime/models/vaes/`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/runtime/models/vaes/) - e.g., [`autoencoder_kl_qwenimage.py`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py) + - **Schedulers**: Implement in [`runtime/models/schedulers/`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/runtime/models/schedulers/) if needed + - These components handle the core model logic, attention mechanisms, and data transformations specific to the target diffusion model. + +4. **Define Pipeline Class**: + - The [`QwenImageEditPipeline`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/runtime/architectures/basic/qwen_image/qwen_image.py) class inherits from `ComposedPipelineBase` and orchestrates stages sequentially. + - Declare required modules via `_required_config_modules` and implement the pipeline stages: + + ```python + class QwenImageEditPipeline(ComposedPipelineBase): + pipeline_name = "QwenImageEditPipeline" # Matches Diffusers model_index.json + _required_config_modules = ["processor", "scheduler", "text_encoder", "tokenizer", "transformer", "vae"] + + def create_pipeline_stages(self, server_args: ServerArgs): + """Set up pipeline stages sequentially.""" + self.add_stage(stage_name="input_validation_stage", stage=InputValidationStage()) + self.add_stage(stage_name="prompt_encoding_stage_primary", stage=ImageEncodingStage(...)) + self.add_stage(stage_name="image_encoding_stage_primary", stage=ImageVAEEncodingStage(...)) + self.add_stage(stage_name="timestep_preparation_stage", stage=TimestepPreparationStage(...)) + self.add_stage(stage_name="latent_preparation_stage", stage=LatentPreparationStage(...)) + self.add_stage(stage_name="conditioning_stage", stage=ConditioningStage()) + self.add_stage(stage_name="denoising_stage", stage=DenoisingStage(...)) + self.add_stage(stage_name="decoding_stage", stage=DecodingStage(...)) + ``` + The pipeline is constructed by adding stages in order. `Qwen-Image-Edit` uses `ImageEncodingStage` (for prompt and image processing) and `ImageVAEEncodingStage` (for latent extraction) before standard denoising and decoding. + +5. **Register Configs**: + - Register the configs in the central registry ([`registry.py`](https://github.com/sgl-project/sglang/blob/main/python/sglang/multimodal_gen/registry.py)) via `_register_configs` to enable automatic loading and instantiation for the model. Modules are automatically loaded and injected based on the config and repository structure. + +By following this pattern of defining configurations and composing pipelines, you can integrate new diffusion models +into SGLang with ease. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/envs.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/envs.py new file mode 100644 index 00000000..56418e72 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/envs.py @@ -0,0 +1,328 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo +import importlib.util + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/envs.py +import logging +import os +from collections.abc import Callable +from typing import TYPE_CHECKING, Any + +import diffusers +import torch +from packaging import version + +from sglang.multimodal_gen.runtime.utils.common import get_bool_env_var + +logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + SGLANG_DIFFUSION_RINGBUFFER_WARNING_INTERVAL: int = 60 + SGLANG_DIFFUSION_NCCL_SO_PATH: str | None = None + LD_LIBRARY_PATH: str | None = None + LOCAL_RANK: int = 0 + CUDA_VISIBLE_DEVICES: str | None = None + SGLANG_DIFFUSION_CACHE_ROOT: str = os.path.expanduser("~/.cache/sgl_diffusion") + SGLANG_DIFFUSION_CONFIG_ROOT: str = os.path.expanduser("~/.config/sgl_diffusion") + SGLANG_DIFFUSION_CONFIGURE_LOGGING: int = 1 + SGLANG_DIFFUSION_LOGGING_LEVEL: str = "INFO" + SGLANG_DIFFUSION_LOGGING_PREFIX: str = "" + SGLANG_DIFFUSION_LOGGING_CONFIG_PATH: str | None = None + SGLANG_DIFFUSION_TRACE_FUNCTION: int = 0 + SGLANG_DIFFUSION_WORKER_MULTIPROC_METHOD: str = "fork" + SGLANG_DIFFUSION_TARGET_DEVICE: str = "cuda" + MAX_JOBS: str | None = None + NVCC_THREADS: str | None = None + CMAKE_BUILD_TYPE: str | None = None + VERBOSE: bool = False + SGLANG_DIFFUSION_SERVER_DEV_MODE: bool = False + SGLANG_DIFFUSION_STAGE_LOGGING: bool = False + + +def _is_hip(): + has_rocm = torch.version.hip is not None + return has_rocm + + +def _is_cuda(): + has_cuda = torch.version.cuda is not None + return has_cuda + + +def _is_musa(): + try: + if hasattr(torch, "musa") and torch.musa.is_available(): + return True + except ModuleNotFoundError: + return False + + +def _is_mps(): + return torch.backends.mps.is_available() + + +class PackagesEnvChecker: + _instance = None + + def __new__(cls): + if cls._instance is None: + cls._instance = super(PackagesEnvChecker, cls).__new__(cls) + cls._instance.initialize() + return cls._instance + + def initialize(self): + self.packages_info = { + "has_aiter": self.check_aiter(), + "diffusers_version": self.check_diffusers_version(), + } + + def check_aiter(self): + """ + Checks whether ROCm AITER library is installed + """ + try: + + logger.info("Using AITER as the attention library") + return True + except: + if _is_hip(): + logger.warning( + f'Using AMD GPUs, but library "aiter" is not installed, ' + "defaulting to other attention mechanisms" + ) + return False + + def check_flash_attn(self): + if not torch.cuda.is_available(): + return False + if _is_musa(): + logger.info( + "Flash Attention library is not supported on MUSA for the moment." + ) + return False + try: + return True + except ImportError: + logger.warning( + f'Flash Attention library "flash_attn" not found, ' + f"using pytorch attention implementation" + ) + return False + + def check_long_ctx_attn(self): + if not torch.cuda.is_available(): + return False + try: + return importlib.util.find_spec("yunchang") is not None + except ImportError: + logger.warning( + f'Ring Flash Attention library "yunchang" not found, ' + f"using pytorch attention implementation" + ) + return False + + def check_diffusers_version(self): + if version.parse( + version.parse(diffusers.__version__).base_version + ) < version.parse("0.30.0"): + raise RuntimeError( + f"Diffusers version: {version.parse(version.parse(diffusers.__version__).base_version)} is not supported," + f"please upgrade to version > 0.30.0" + ) + return version.parse(version.parse(diffusers.__version__).base_version) + + def get_packages_info(self): + return self.packages_info + + +PACKAGES_CHECKER = PackagesEnvChecker() + + +def get_default_cache_root() -> str: + return os.getenv( + "XDG_CACHE_HOME", + os.path.join(os.path.expanduser("~"), ".cache"), + ) + + +def get_default_config_root() -> str: + return os.getenv( + "XDG_CONFIG_HOME", + os.path.join(os.path.expanduser("~"), ".config"), + ) + + +def maybe_convert_int(value: str | None) -> int | None: + if value is None: + return None + return int(value) + + +# The begin-* and end* here are used by the documentation generator +# to extract the used env vars. + +# begin-env-vars-definition + +environment_variables: dict[str, Callable[[], Any]] = { + # ================== Installation Time Env Vars ================== + # Target device of sglang-diffusion, supporting [cuda (by default), + # rocm, neuron, cpu, openvino] + "SGLANG_DIFFUSION_TARGET_DEVICE": lambda: os.getenv( + "SGLANG_DIFFUSION_TARGET_DEVICE", "cuda" + ), + # Maximum number of compilation jobs to run in parallel. + # By default this is the number of CPUs + "MAX_JOBS": lambda: os.getenv("MAX_JOBS", None), + # Number of threads to use for nvcc + # By default this is 1. + # If set, `MAX_JOBS` will be reduced to avoid oversubscribing the CPU. + "NVCC_THREADS": lambda: os.getenv("NVCC_THREADS", None), + # If set, sgl_diffusion will use precompiled binaries (*.so) + "SGLANG_DIFFUSION_USE_PRECOMPILED": lambda: bool( + os.environ.get("SGLANG_DIFFUSION_USE_PRECOMPILED") + ) + or bool(os.environ.get("SGLANG_DIFFUSION_PRECOMPILED_WHEEL_LOCATION")), + # CMake build type + # If not set, defaults to "Debug" or "RelWithDebInfo" + # Available options: "Debug", "Release", "RelWithDebInfo" + "CMAKE_BUILD_TYPE": lambda: os.getenv("CMAKE_BUILD_TYPE"), + # If set, sgl_diffusion will print verbose logs during installation + "VERBOSE": lambda: bool(int(os.getenv("VERBOSE", "0"))), + # Root directory for FASTVIDEO configuration files + # Defaults to `~/.config/sgl_diffusion` unless `XDG_CONFIG_HOME` is set + # Note that this not only affects how sgl_diffusion finds its configuration files + # during runtime, but also affects how sgl_diffusion installs its configuration + # files during **installation**. + "SGLANG_DIFFUSION_CONFIG_ROOT": lambda: os.path.expanduser( + os.getenv( + "SGLANG_DIFFUSION_CONFIG_ROOT", + os.path.join(get_default_config_root(), "sgl_diffusion"), + ) + ), + # ================== Runtime Env Vars ================== + # Root directory for FASTVIDEO cache files + # Defaults to `~/.cache/sgl_diffusion` unless `XDG_CACHE_HOME` is set + "SGLANG_DIFFUSION_CACHE_ROOT": lambda: os.path.expanduser( + os.getenv( + "SGLANG_DIFFUSION_CACHE_ROOT", + os.path.join(get_default_cache_root(), "sgl_diffusion"), + ) + ), + # Interval in seconds to log a warning message when the ring buffer is full + "SGLANG_DIFFUSION_RINGBUFFER_WARNING_INTERVAL": lambda: int( + os.environ.get("SGLANG_DIFFUSION_RINGBUFFER_WARNING_INTERVAL", "60") + ), + # Path to the NCCL library file. It is needed because nccl>=2.19 brought + # by PyTorch contains a bug: https://github.com/NVIDIA/nccl/issues/1234 + "SGLANG_DIFFUSION_NCCL_SO_PATH": lambda: os.environ.get( + "SGLANG_DIFFUSION_NCCL_SO_PATH", None + ), + # when `SGLANG_DIFFUSION_NCCL_SO_PATH` is not set, sgl_diffusion will try to find the nccl + # library file in the locations specified by `LD_LIBRARY_PATH` + "LD_LIBRARY_PATH": lambda: os.environ.get("LD_LIBRARY_PATH", None), + # Internal flag to enable Dynamo fullgraph capture + "SGLANG_DIFFUSION_TEST_DYNAMO_FULLGRAPH_CAPTURE": lambda: bool( + os.environ.get("SGLANG_DIFFUSION_TEST_DYNAMO_FULLGRAPH_CAPTURE", "1") != "0" + ), + # local rank of the process in the distributed setting, used to determine + # the GPU device id + "LOCAL_RANK": lambda: int(os.environ.get("LOCAL_RANK", "0")), + # used to control the visible devices in the distributed setting + "CUDA_VISIBLE_DEVICES": lambda: os.environ.get("CUDA_VISIBLE_DEVICES", None), + # timeout for each iteration in the engine + "SGLANG_DIFFUSION_ENGINE_ITERATION_TIMEOUT_S": lambda: int( + os.environ.get("SGLANG_DIFFUSION_ENGINE_ITERATION_TIMEOUT_S", "60") + ), + # Logging configuration + # If set to 0, sgl_diffusion will not configure logging + # If set to 1, sgl_diffusion will configure logging using the default configuration + # or the configuration file specified by SGLANG_DIFFUSION_LOGGING_CONFIG_PATH + "SGLANG_DIFFUSION_CONFIGURE_LOGGING": lambda: int( + os.getenv("SGLANG_DIFFUSION_CONFIGURE_LOGGING", "1") + ), + "SGLANG_DIFFUSION_LOGGING_CONFIG_PATH": lambda: os.getenv( + "SGLANG_DIFFUSION_LOGGING_CONFIG_PATH" + ), + # this is used for configuring the default logging level + "SGLANG_DIFFUSION_LOGGING_LEVEL": lambda: os.getenv( + "SGLANG_DIFFUSION_LOGGING_LEVEL", "INFO" + ), + # if set, SGLANG_DIFFUSION_LOGGING_PREFIX will be prepended to all log messages + "SGLANG_DIFFUSION_LOGGING_PREFIX": lambda: os.getenv( + "SGLANG_DIFFUSION_LOGGING_PREFIX", "" + ), + # Trace function calls + # If set to 1, sgl_diffusion will trace function calls + # Useful for debugging + "SGLANG_DIFFUSION_TRACE_FUNCTION": lambda: int( + os.getenv("SGLANG_DIFFUSION_TRACE_FUNCTION", "0") + ), + # Path to the attention configuration file. Only used for sliding tile + # attention for now. + "SGLANG_DIFFUSION_ATTENTION_CONFIG": lambda: ( + None + if os.getenv("SGLANG_DIFFUSION_ATTENTION_CONFIG", None) is None + else os.path.expanduser(os.getenv("SGLANG_DIFFUSION_ATTENTION_CONFIG", ".")) + ), + # Use dedicated multiprocess context for workers. + # Both spawn and fork work + "SGLANG_DIFFUSION_WORKER_MULTIPROC_METHOD": lambda: os.getenv( + "SGLANG_DIFFUSION_WORKER_MULTIPROC_METHOD", "fork" + ), + # Enables torch profiler if set. Path to the directory where torch profiler + # traces are saved. Note that it must be an absolute path. + "SGLANG_DIFFUSION_TORCH_PROFILER_DIR": lambda: ( + None + if os.getenv("SGLANG_DIFFUSION_TORCH_PROFILER_DIR", None) is None + else os.path.expanduser(os.getenv("SGLANG_DIFFUSION_TORCH_PROFILER_DIR", ".")) + ), + # If set, sgl_diffusion will run in development mode, which will enable + # some additional endpoints for developing and debugging, + # e.g. `/reset_prefix_cache` + "SGLANG_DIFFUSION_SERVER_DEV_MODE": lambda: get_bool_env_var( + "SGLANG_DIFFUSION_SERVER_DEV_MODE" + ), + # If set, sgl_diffusion will enable stage logging, which will print the time + # taken for each stage + "SGLANG_DIFFUSION_STAGE_LOGGING": lambda: get_bool_env_var( + "SGLANG_DIFFUSION_STAGE_LOGGING" + ), +} + + +# end-env-vars-definition + + +def __getattr__(name: str): + # lazy evaluation of environment variables + if name in environment_variables: + return environment_variables[name]() + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def __dir__(): + return list(environment_variables.keys()) + + +def get_torch_distributed_backend() -> str: + if torch.cuda.is_available(): + return "nccl" + elif _is_musa(): + return "mccl" + elif _is_mps(): + return "gloo" + else: + raise NotImplementedError( + "No Accelerators(AMD/NV/MTT GPU, AMD MI instinct accelerators) available" + ) + + +def get_device(local_rank: int) -> torch.device: + if torch.cuda.is_available(): + return torch.device("cuda", local_rank) + elif _is_musa(): + return torch.device("musa", local_rank) + elif _is_mps(): + return torch.device("mps") + else: + return torch.device("cpu") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/registry.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/registry.py new file mode 100644 index 00000000..1e2fafeb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/registry.py @@ -0,0 +1,423 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +Central registry for multimodal models. + +This module provides a centralized registry for multimodal models, including pipelines +and sampling parameters. It allows for easy registration and retrieval of model +information based on model paths or other identifiers. +""" + +import dataclasses +import importlib +import os +import pkgutil +import re +from functools import lru_cache +from typing import Any, Callable, Dict, List, Optional, Tuple, Type + +from sglang.multimodal_gen.configs.pipeline_configs import ( + FastHunyuanConfig, + FluxPipelineConfig, + HunyuanConfig, + StepVideoT2VConfig, + WanI2V480PConfig, + WanI2V720PConfig, + WanT2V480PConfig, + WanT2V720PConfig, + ZImagePipelineConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig +from sglang.multimodal_gen.configs.pipeline_configs.flux import Flux2PipelineConfig +from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import ( + QwenImageEditPipelineConfig, + QwenImagePipelineConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.wan import ( + FastWan2_1_T2V_480P_Config, + FastWan2_2_TI2V_5B_Config, + Wan2_2_I2V_A14B_Config, + Wan2_2_T2V_A14B_Config, + Wan2_2_TI2V_5B_Config, +) +from sglang.multimodal_gen.configs.sample.flux import FluxSamplingParams +from sglang.multimodal_gen.configs.sample.hunyuan import ( + FastHunyuanSamplingParam, + HunyuanSamplingParams, +) +from sglang.multimodal_gen.configs.sample.qwenimage import QwenImageSamplingParams +from sglang.multimodal_gen.configs.sample.stepvideo import StepVideoT2VSamplingParams +from sglang.multimodal_gen.configs.sample.wan import ( + FastWanT2V480PConfig, + Wan2_1_Fun_1_3B_InP_SamplingParams, + Wan2_2_I2V_A14B_SamplingParam, + Wan2_2_T2V_A14B_SamplingParam, + Wan2_2_TI2V_5B_SamplingParam, + WanI2V_14B_480P_SamplingParam, + WanI2V_14B_720P_SamplingParam, + WanT2V_1_3B_SamplingParams, + WanT2V_14B_SamplingParams, +) +from sglang.multimodal_gen.configs.sample.zimage import ZImageSamplingParams +from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( + ComposedPipelineBase, +) +from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import ( + maybe_download_model_index, + verify_model_config_and_directory, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + +# --- Part 1: Pipeline Discovery --- + +_PIPELINE_REGISTRY: Dict[str, Type[ComposedPipelineBase]] = {} + + +def _discover_and_register_pipelines(): + """ + Automatically discover and register all ComposedPipelineBase subclasses. + This function scans the 'sglang.multimodal_gen.runtime.pipelines' package, + finds modules with an 'EntryClass' attribute, and maps the class's 'pipeline_name' + to the class itself in a global registry. + """ + if _PIPELINE_REGISTRY: # run only once + return + + package_name = "sglang.multimodal_gen.runtime.pipelines" + package = importlib.import_module(package_name) + + for _, module_name, ispkg in pkgutil.walk_packages( + package.__path__, package.__name__ + "." + ): + if not ispkg: + pipeline_module = importlib.import_module(module_name) + if hasattr(pipeline_module, "EntryClass"): + entry_cls = pipeline_module.EntryClass + entry_cls_list = ( + [entry_cls] if not isinstance(entry_cls, list) else entry_cls + ) + + for cls in entry_cls_list: + if hasattr(cls, "pipeline_name"): + if cls.pipeline_name in _PIPELINE_REGISTRY: + logger.warning( + f"Duplicate pipeline name '{cls.pipeline_name}' found. Overwriting." + ) + _PIPELINE_REGISTRY[cls.pipeline_name] = cls + logger.debug( + f"Registering pipelines complete, {len(_PIPELINE_REGISTRY)} pipelines registered" + ) + + +# --- Part 2: Config Registration --- +@dataclasses.dataclass +class ConfigInfo: + """Encapsulates all configuration information required to register a + diffusers model within this framework.""" + + sampling_param_cls: Any + pipeline_config_cls: Type[PipelineConfig] + + +# The central registry mapping a model name to its configuration information +_CONFIG_REGISTRY: Dict[str, ConfigInfo] = {} + +# Mappings from Hugging Face model paths to our internal model names +_MODEL_HF_PATH_TO_NAME: Dict[str, str] = {} + +# Detectors to identify model families from paths or class names +_MODEL_NAME_DETECTORS: List[Tuple[str, Callable[[str], bool]]] = [] + + +def register_configs( + sampling_param_cls: Any, + pipeline_config_cls: Type[PipelineConfig], + hf_model_paths: Optional[List[str]] = None, + model_detectors: Optional[List[Callable[[str], bool]]] = None, +): + """ + Registers configuration classes for a new model family. + """ + model_id = str(len(_CONFIG_REGISTRY)) + + _CONFIG_REGISTRY[model_id] = ConfigInfo( + sampling_param_cls=sampling_param_cls, + pipeline_config_cls=pipeline_config_cls, + ) + if hf_model_paths: + for path in hf_model_paths: + if path in _MODEL_HF_PATH_TO_NAME: + logger.warning( + f"Model path '{path}' is already mapped to '{_MODEL_HF_PATH_TO_NAME[path]}' and will be overwritten by '{model_id}'." + ) + _MODEL_HF_PATH_TO_NAME[path] = model_id + + if model_detectors: + for detector in model_detectors: + _MODEL_NAME_DETECTORS.append((model_id, detector)) + + +def _get_config_info(model_path: str) -> Optional[ConfigInfo]: + """ + Gets the ConfigInfo for a given model path using mappings and detectors. + """ + # 1. Exact match + if model_path in _MODEL_HF_PATH_TO_NAME: + model_id = _MODEL_HF_PATH_TO_NAME[model_path] + logger.debug(f"Resolved model path '{model_path}' from exact path match.") + return _CONFIG_REGISTRY.get(model_id) + + # 2. Partial match: find the best (longest) match against all registered model hf paths. + cleaned_model_path = re.sub(r"--", "/", model_path.lower()) + all_model_hf_paths = sorted(_MODEL_HF_PATH_TO_NAME.keys(), key=len, reverse=True) + for model_hf_path in all_model_hf_paths: + if model_hf_path.lower() in cleaned_model_path: + logger.debug( + f"Resolved model name '{model_hf_path}' from partial path match." + ) + model_id = _MODEL_HF_PATH_TO_NAME[model_hf_path] + return _CONFIG_REGISTRY.get(model_id) + + # 3. Use detectors + if os.path.exists(model_path): + config = verify_model_config_and_directory(model_path) + else: + config = maybe_download_model_index(model_path) + + pipeline_name = config.get("_class_name", "").lower() + + matched_model_names = [] + for model_id, detector in _MODEL_NAME_DETECTORS: + if detector(model_path.lower()) or detector(pipeline_name): + logger.debug( + f"Matched model name '{model_id}' using a registered detector." + ) + matched_model_names += [model_id] + + if len(matched_model_names) >= 1: + if len(matched_model_names) > 1: + logger.warning( + f"More than one model name is matched, using the first matched" + ) + model_id = matched_model_names[0] + return _CONFIG_REGISTRY.get(model_id) + else: + raise RuntimeError(f"No model info found for model path: {model_path}") + + +# --- Part 3: Main Resolver --- + + +@dataclasses.dataclass +class ModelInfo: + """ + Encapsulates all configuration information required to register a + diffusers model within this framework. + """ + + pipeline_cls: Type[ComposedPipelineBase] + sampling_param_cls: Any + pipeline_config_cls: Type[PipelineConfig] + + +@lru_cache(maxsize=1) +def get_model_info(model_path: str) -> Optional[ModelInfo]: + """ + Resolves all necessary classes (pipeline, sampling, config) for a given model path. + + This function serves as the main entry point for model resolution. It performs two main tasks: + 1. Dynamically resolves the pipeline class by reading 'model_index.json' and matching + '_class_name' against an auto-discovered registry of pipeline implementations. + 2. Resolves the associated configuration classes (for sampling and pipeline) using a + manually registered mapping based on the model path. + """ + # 1. Discover all available pipeline classes and cache them + _discover_and_register_pipelines() + + # 2. Get pipeline class from model's model_index.json + try: + if os.path.exists(model_path): + config = verify_model_config_and_directory(model_path) + else: + config = maybe_download_model_index(model_path) + except Exception as e: + logger.error(f"Could not read model config for '{model_path}': {e}") + return None + + pipeline_class_name = config.get("_class_name") + if not pipeline_class_name: + logger.error(f"'_class_name' not found in model_index.json for '{model_path}'") + return None + + pipeline_cls = _PIPELINE_REGISTRY.get(pipeline_class_name) + if not pipeline_cls: + logger.error( + f"Pipeline class '{pipeline_class_name}' specified in '{model_path}' is not a registered EntryClass in the framework. " + f"Available pipelines: {list(_PIPELINE_REGISTRY.keys())}" + ) + return None + + # 3. Get configuration classes (sampling, pipeline config) + config_info = _get_config_info(model_path) + if not config_info: + logger.error( + f"Could not resolve configuration for model '{model_path}'. " + "It is not a registered model path or detected by any registered model family detectors. " + f"Known model paths: {list(_MODEL_HF_PATH_TO_NAME.keys())}" + ) + return None + + # 4. Combine the complete model info + model_info = ModelInfo( + pipeline_cls=pipeline_cls, + sampling_param_cls=config_info.sampling_param_cls, + pipeline_config_cls=config_info.pipeline_config_cls, + ) + logger.info(f"Found model info: {model_info}") + + return model_info + + +# Registration of model configs +def _register_configs(): + # Hunyuan + register_configs( + sampling_param_cls=HunyuanSamplingParams, + pipeline_config_cls=HunyuanConfig, + hf_model_paths=[ + "hunyuanvideo-community/HunyuanVideo", + ], + model_detectors=[lambda hf_id: "hunyuan" in hf_id.lower()], + ) + register_configs( + sampling_param_cls=FastHunyuanSamplingParam, + pipeline_config_cls=FastHunyuanConfig, + hf_model_paths=[ + "FastVideo/FastHunyuan-diffusers", + ], + ) + + # StepVideo + register_configs( + sampling_param_cls=StepVideoT2VSamplingParams, + pipeline_config_cls=StepVideoT2VConfig, + hf_model_paths=[ + "FastVideo/stepvideo-t2v-diffusers", + ], + model_detectors=[lambda hf_id: "stepvideo" in hf_id.lower()], + ) + + # Wan + register_configs( + sampling_param_cls=WanT2V_1_3B_SamplingParams, + pipeline_config_cls=WanT2V480PConfig, + hf_model_paths=[ + "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + ], + model_detectors=[lambda hf_id: "wanpipeline" in hf_id.lower()], + ) + register_configs( + sampling_param_cls=WanT2V_14B_SamplingParams, + pipeline_config_cls=WanT2V720PConfig, + hf_model_paths=[ + "Wan-AI/Wan2.1-T2V-14B-Diffusers", + ], + ) + register_configs( + sampling_param_cls=WanI2V_14B_480P_SamplingParam, + pipeline_config_cls=WanI2V480PConfig, + hf_model_paths=[ + "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers", + ], + model_detectors=[lambda hf_id: "wanimagetovideo" in hf_id.lower()], + ) + register_configs( + sampling_param_cls=WanI2V_14B_720P_SamplingParam, + pipeline_config_cls=WanI2V720PConfig, + hf_model_paths=[ + "Wan-AI/Wan2.1-I2V-14B-720P-Diffusers", + ], + ) + register_configs( + sampling_param_cls=Wan2_1_Fun_1_3B_InP_SamplingParams, + pipeline_config_cls=WanI2V480PConfig, + hf_model_paths=[ + "weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers", + ], + ) + register_configs( + sampling_param_cls=Wan2_2_TI2V_5B_SamplingParam, + pipeline_config_cls=Wan2_2_TI2V_5B_Config, + hf_model_paths=[ + "Wan-AI/Wan2.2-TI2V-5B-Diffusers", + ], + ) + + register_configs( + sampling_param_cls=Wan2_2_TI2V_5B_SamplingParam, + pipeline_config_cls=FastWan2_2_TI2V_5B_Config, + hf_model_paths=[ + "FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers", + "FastVideo/FastWan2.2-TI2V-5B-Diffusers", + ], + ) + + register_configs( + sampling_param_cls=Wan2_2_T2V_A14B_SamplingParam, + pipeline_config_cls=Wan2_2_T2V_A14B_Config, + hf_model_paths=["Wan-AI/Wan2.2-T2V-A14B-Diffusers"], + ) + register_configs( + sampling_param_cls=Wan2_2_I2V_A14B_SamplingParam, + pipeline_config_cls=Wan2_2_I2V_A14B_Config, + hf_model_paths=["Wan-AI/Wan2.2-I2V-A14B-Diffusers"], + ) + register_configs( + sampling_param_cls=FastWanT2V480PConfig, + pipeline_config_cls=FastWan2_1_T2V_480P_Config, + hf_model_paths=[ + "FastVideo/FastWan2.1-T2V-1.3B-Diffusers", + ], + ) + + # FLUX + register_configs( + sampling_param_cls=FluxSamplingParams, + pipeline_config_cls=FluxPipelineConfig, + hf_model_paths=[ + "black-forest-labs/FLUX.1-dev", + ], + model_detectors=[lambda hf_id: "flux.1" in hf_id.lower()], + ) + register_configs( + sampling_param_cls=FluxSamplingParams, + pipeline_config_cls=Flux2PipelineConfig, + hf_model_paths=[ + "black-forest-labs/FLUX.2-dev", + ], + model_detectors=[lambda hf_id: "flux.2" in hf_id.lower()], + ) + register_configs( + sampling_param_cls=ZImageSamplingParams, + pipeline_config_cls=ZImagePipelineConfig, + hf_model_paths=[ + "Tongyi-MAI/Z-Image-Turbo", + ], + model_detectors=[lambda hf_id: "z-image" in hf_id.lower()], + ) + + # Qwen-Image + register_configs( + sampling_param_cls=QwenImageSamplingParams, + pipeline_config_cls=QwenImagePipelineConfig, + hf_model_paths=["Qwen/Qwen-Image"], + ) + register_configs( + sampling_param_cls=QwenImageSamplingParams, + pipeline_config_cls=QwenImageEditPipelineConfig, + hf_model_paths=["Qwen/Qwen-Image-Edit"], + ) + + +_register_configs() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/__init__.py new file mode 100644 index 00000000..9edfd5c6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/__init__.py @@ -0,0 +1,55 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +from sglang.multimodal_gen.runtime.distributed.communication_op import * +from sglang.multimodal_gen.runtime.distributed.group_coordinator import ( + get_local_torch_device, +) +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + cleanup_dist_env_and_memory, + get_dp_group, + get_dp_rank, + get_dp_world_size, + get_sp_group, + get_sp_parallel_rank, + get_sp_world_size, + get_tp_group, + get_tp_rank, + get_tp_world_size, + get_world_group, + get_world_rank, + get_world_size, + init_distributed_environment, + initialize_model_parallel, + maybe_init_distributed_environment_and_model_parallel, + model_parallel_is_initialized, +) +from sglang.multimodal_gen.runtime.distributed.utils import * + +__all__ = [ + # Initialization + "init_distributed_environment", + "initialize_model_parallel", + "cleanup_dist_env_and_memory", + "model_parallel_is_initialized", + "maybe_init_distributed_environment_and_model_parallel", + # World group + "get_world_group", + "get_world_rank", + "get_world_size", + # Data parallel group + "get_dp_group", + "get_dp_rank", + "get_dp_world_size", + # Sequence parallel group + "get_sp_group", + "get_sp_parallel_rank", + "get_sp_world_size", + # Tensor parallel group + "get_tp_group", + "get_tp_rank", + "get_tp_world_size", + # Get torch device + "get_local_torch_device", +] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/communication_op.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/communication_op.py new file mode 100644 index 00000000..61672ca4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/communication_op.py @@ -0,0 +1,55 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/distributed/communication_op.py + +import torch +import torch.distributed + +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + get_cfg_group, + get_sp_group, + get_tp_group, +) + + +def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor: + """All-reduce the input tensor across model parallel group.""" + return get_tp_group().all_reduce(input_) + + +def tensor_model_parallel_all_gather( + input_: torch.Tensor, dim: int = -1 +) -> torch.Tensor: + """All-gather the input tensor across model parallel group.""" + return get_tp_group().all_gather(input_, dim) + + +# TODO: remove model, make it sequence_parallel +def sequence_model_parallel_all_to_all_4D( + input_: torch.Tensor, scatter_dim: int = 2, gather_dim: int = 1 +) -> torch.Tensor: + """All-to-all communication of 4D tensors (e.g. QKV matrices) across sequence parallel group.""" + return get_sp_group().all_to_all_4D(input_, scatter_dim, gather_dim) + + +def sequence_model_parallel_all_gather( + input_: torch.Tensor, dim: int = -1 +) -> torch.Tensor: + """All-gather the input tensor across model parallel group.""" + return get_sp_group().all_gather(input_, dim) + + +def cfg_model_parallel_all_gather( + input_: torch.Tensor, dim: int = -1, separate_tensors: bool = False +) -> torch.Tensor: + """All-gather the input tensor across model parallel group.""" + return get_cfg_group().all_gather(input_, dim, separate_tensors) + + +def cfg_model_parallel_all_reduce( + input_: torch.Tensor, + op: torch._C._distributed_c10d.ReduceOp = torch._C._distributed_c10d.ReduceOp.SUM, +) -> torch.Tensor: + """All-reduce the input tensor across CFG parallel group.""" + return get_cfg_group().all_reduce(input_, op=op) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/__init__.py new file mode 100644 index 00000000..af2eb7d1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/__init__.py @@ -0,0 +1 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/base_device_communicator.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/base_device_communicator.py new file mode 100644 index 00000000..01bdf1c2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/base_device_communicator.py @@ -0,0 +1,297 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/distributed/device_communicators/base_device_communicator.py + +from typing import Any + +import torch +import torch.distributed as dist +from torch import Tensor +from torch.distributed import ProcessGroup, ReduceOp + + +class DistributedAutograd: + """Collection of autograd functions for distributed operations. + + This class provides custom autograd functions for distributed operations like all_reduce, + all_gather, and all_to_all. Each operation is implemented as a static inner class with + proper forward and backward implementations. + """ + + class AllReduce(torch.autograd.Function): + """Differentiable all_reduce operation. + + The gradient of all_reduce is another all_reduce operation since the operation + combines values from all ranks equally. + """ + + @staticmethod + def forward( + ctx: Any, + group: ProcessGroup, + input_: Tensor, + op: dist.ReduceOp | None = None, + ) -> Tensor: + ctx.group = group + ctx.op = op + output = input_.clone() + dist.all_reduce(output, group=group, op=op) + return output + + @staticmethod + def backward(ctx: Any, grad_output: Tensor) -> tuple[None, Tensor, None]: + grad_output = grad_output.clone() + dist.all_reduce(grad_output, group=ctx.group, op=ctx.op) + return None, grad_output, None + + class AllGather(torch.autograd.Function): + """Differentiable all_gather operation. + + The operation gathers tensors from all ranks and concatenates them along a specified dimension. + The backward pass uses reduce_scatter to efficiently distribute gradients back to source ranks. + """ + + @staticmethod + def forward( + ctx: Any, group: ProcessGroup, input_: Tensor, world_size: int, dim: int + ) -> Tensor: + ctx.group = group + ctx.world_size = world_size + ctx.dim = dim + ctx.input_shape = input_.shape + + input_size = input_.size() + output_size = (input_size[0] * world_size,) + input_size[1:] + output_tensor = torch.empty( + output_size, dtype=input_.dtype, device=input_.device + ) + + dist.all_gather_into_tensor(output_tensor, input_, group=group) + + output_tensor = output_tensor.reshape((world_size,) + input_size) + output_tensor = output_tensor.movedim(0, dim) + output_tensor = output_tensor.reshape( + input_size[:dim] + + (world_size * input_size[dim],) + + input_size[dim + 1 :] + ) + return output_tensor + + @staticmethod + def backward(ctx: Any, grad_output: Tensor) -> tuple[None, Tensor, None, None]: + # Split the gradient tensor along the gathered dimension + dim_size = grad_output.size(ctx.dim) // ctx.world_size + grad_chunks = grad_output.reshape( + grad_output.shape[: ctx.dim] + + (ctx.world_size, dim_size) + + grad_output.shape[ctx.dim + 1 :] + ) + grad_chunks = grad_chunks.movedim(ctx.dim, 0) + + # Each rank only needs its corresponding gradient + grad_input = torch.empty( + ctx.input_shape, dtype=grad_output.dtype, device=grad_output.device + ) + dist.reduce_scatter_tensor( + grad_input, grad_chunks.contiguous(), group=ctx.group + ) + + return None, grad_input, None, None + + class AllToAll4D(torch.autograd.Function): + """Differentiable all_to_all operation specialized for 4D tensors. + + This operation is particularly useful for attention operations where we need to + redistribute data across ranks for efficient parallel processing. + + The operation supports two modes: + 1. scatter_dim=2, gather_dim=1: Used for redistributing attention heads + 2. scatter_dim=1, gather_dim=2: Used for redistributing sequence dimensions + """ + + @staticmethod + def forward( + ctx: Any, + group: ProcessGroup, + input_: Tensor, + world_size: int, + scatter_dim: int, + gather_dim: int, + ) -> Tensor: + ctx.group = group + ctx.world_size = world_size + ctx.scatter_dim = scatter_dim + ctx.gather_dim = gather_dim + + if world_size == 1: + return input_ + + assert ( + input_.dim() == 4 + ), f"input must be 4D tensor, got {input_.dim()} and shape {input_.shape}" + + if scatter_dim == 2 and gather_dim == 1: + bs, shard_seqlen, hn, hd = input_.shape + seqlen = shard_seqlen * world_size + shard_hn = hn // world_size + + input_ = input_.transpose(0, 2).contiguous() # hn, shard_seqlen, bs, hd + output = torch.empty_like(input_) + + dist.all_to_all_single( + output, input_, group=group + ) # hn, shard_seqlen, bs, hd + + output = torch.cat( + output.split(shard_hn), dim=1 + ) # sharded hn, seqlen, bs, hd + + output = output.transpose( + 0, 2 + ).contiguous() # bs, seqlen, sharded_hn, hd + + return output + elif scatter_dim == 1 and gather_dim == 2: + bs, seqlen, shard_hn, hd = input_.shape + hn = shard_hn * world_size + shard_seqlen = seqlen // world_size + + input_ = input_.transpose(0, 2).contiguous() # shard_hn, seqlen, bs, hd + + input_ = ( + input_.reshape(shard_hn, world_size, shard_seqlen, bs, hd) + .transpose(0, 1) + .reshape(shard_hn * world_size, shard_seqlen, bs, hd) + .contiguous() + ) + + output = torch.empty_like(input_) + + dist.all_to_all_single(output, input_, group=group) + + output = output.transpose( + 0, 2 + ).contiguous() # bs, seqlen, sharded_hn, hd + + return output + else: + raise RuntimeError( + f"Invalid scatter_dim={scatter_dim}, gather_dim={gather_dim}. " + f"Only (scatter_dim=2, gather_dim=1) and (scatter_dim=1, gather_dim=2) are supported." + ) + + @staticmethod + def backward( + ctx: Any, grad_output: Tensor + ) -> tuple[None, Tensor, None, None, None]: + if ctx.world_size == 1: + return None, grad_output, None, None, None + + # For backward pass, we swap scatter_dim and gather_dim + output = DistributedAutograd.AllToAll4D.apply( + ctx.group, grad_output, ctx.world_size, ctx.gather_dim, ctx.scatter_dim + ) + return None, output, None, None, None + + +class DeviceCommunicatorBase: + """ + Base class for device-specific communicator with autograd support. + It can use the `cpu_group` to initialize the communicator. + If the device has PyTorch integration (PyTorch can recognize its + communication backend), the `device_group` will also be given. + """ + + def __init__( + self, + cpu_group: ProcessGroup, + device: torch.device | None = None, + device_group: ProcessGroup | None = None, + unique_name: str = "", + ): + self.device = device or torch.device("cpu") + self.cpu_group = cpu_group + self.device_group = device_group + self.unique_name = unique_name + self.rank = dist.get_rank(cpu_group) + self.world_size = dist.get_world_size(cpu_group) + self.ranks = dist.get_process_group_ranks(cpu_group) + self.global_rank = dist.get_rank() + self.global_world_size = dist.get_world_size() + self.rank_in_group = dist.get_group_rank(self.cpu_group, self.global_rank) + + def all_reduce( + self, input_: torch.Tensor, op: dist.ReduceOp | None = ReduceOp.SUM + ) -> torch.Tensor: + """Performs an all_reduce operation with gradient support.""" + return DistributedAutograd.AllReduce.apply(self.device_group, input_, op) + + def all_gather(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor: + """Performs an all_gather operation with gradient support.""" + if dim < 0: + dim += input_.dim() + return DistributedAutograd.AllGather.apply( + self.device_group, input_, self.world_size, dim + ) + + def all_to_all_4D( + self, input_: torch.Tensor, scatter_dim: int = 2, gather_dim: int = 1 + ) -> torch.Tensor: + """Performs a 4D all-to-all operation with gradient support.""" + return DistributedAutograd.AllToAll4D.apply( + self.device_group, input_, self.world_size, scatter_dim, gather_dim + ) + + def gather( + self, input_: torch.Tensor, dst: int = 0, dim: int = -1 + ) -> torch.Tensor | None: + """ + NOTE: We assume that the input tensor is on the same device across + all the ranks. + NOTE: `dst` is the local rank of the destination rank. + """ + world_size = self.world_size + assert ( + -input_.dim() <= dim < input_.dim() + ), f"Invalid dim ({dim}) for input tensor with shape {input_.size()}" + if dim < 0: + # Convert negative dim to positive. + dim += input_.dim() + + # Allocate output tensor. + if self.rank_in_group == dst: + gather_list = [torch.empty_like(input_) for _ in range(world_size)] + else: + gather_list = None + # Gather. + torch.distributed.gather( + input_, gather_list, dst=self.ranks[dst], group=self.device_group + ) + if self.rank_in_group == dst: + output_tensor = torch.cat(gather_list, dim=dim) + else: + output_tensor = None + return output_tensor + + def send(self, tensor: torch.Tensor, dst: int | None = None) -> None: + """Sends a tensor to the destination rank in a non-blocking way""" + """NOTE: `dst` is the local rank of the destination rank.""" + if dst is None: + dst = (self.rank_in_group + 1) % self.world_size + torch.distributed.send(tensor, self.ranks[dst], self.device_group) + + def recv( + self, size: torch.Size, dtype: torch.dtype, src: int | None = None + ) -> torch.Tensor: + """Receives a tensor from the source rank.""" + """NOTE: `src` is the local rank of the source rank.""" + if src is None: + src = (self.rank_in_group - 1) % self.world_size + + tensor = torch.empty(size, dtype=dtype, device=self.device) + torch.distributed.recv(tensor, self.ranks[src], self.device_group) + return tensor + + def destroy(self) -> None: + pass diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/cpu_communicator.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/cpu_communicator.py new file mode 100644 index 00000000..434cf384 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/cpu_communicator.py @@ -0,0 +1,161 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from: https://github.com/vllm-project/vllm/blob/main/vllm/distributed/device_communicators/cpu_communicator.py + +import os + +import torch +from torch.distributed import ProcessGroup + +from .base_device_communicator import DeviceCommunicatorBase + + +class CpuCommunicator(DeviceCommunicatorBase): + + def __init__( + self, + cpu_group: ProcessGroup, + device: torch.device | None = None, + device_group: ProcessGroup | None = None, + unique_name: str = "", + ): + from sglang.multimodal_gen.runtime.platforms import current_platform + from sglang.multimodal_gen.runtime.platforms.interface import CpuArchEnum + + super().__init__(cpu_group, device, device_group, unique_name) + self.dist_module = torch.distributed + + if ( + (current_platform.get_cpu_architecture() == CpuArchEnum.X86) + and hasattr(torch.ops._C, "init_shm_manager") + and unique_name.startswith("tp") + ): + self.dist_module = _CPUSHMDistributed(self) + + def all_reduce( + self, + input_: torch.Tensor, + op: torch.distributed.ReduceOp | None = torch.distributed.ReduceOp.SUM, + ) -> torch.Tensor: + self.dist_module.all_reduce(input_, group=self.device_group, op=op) + return input_ + + def gather( + self, input_: torch.Tensor, dst: int = 0, dim: int = -1 + ) -> torch.Tensor | None: + """ + NOTE: We assume that the input tensor is on the same device across + all the ranks. + NOTE: `dst` is the local rank of the destination rank. + """ + world_size = self.world_size + assert ( + -input_.dim() <= dim < input_.dim() + ), f"Invalid dim ({dim}) for input tensor with shape {input_.size()}" + if dim < 0: + # Convert negative dim to positive. + dim += input_.dim() + + # Allocate output tensor. + if self.rank_in_group == dst: + gather_list = [torch.empty_like(input_) for _ in range(world_size)] + else: + gather_list = None + + # Gather. + self.dist_module.gather( + input_, gather_list, dst=self.ranks[dst], group=self.device_group + ) + + if self.rank_in_group == dst: + output_tensor = torch.cat(gather_list, dim=dim) + else: + output_tensor = None + return output_tensor + + def all_gather(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor: + if dim < 0: + # Convert negative dim to positive. + dim += input_.dim() + input_size = input_.size() + # NOTE: we have to use concat-style all-gather here, + # stack-style all-gather has compatibility issues with + # torch.compile . see https://github.com/pytorch/pytorch/issues/138795 + output_size = (input_size[0] * self.world_size,) + input_size[1:] + # Allocate output tensor. + output_tensor = torch.empty( + output_size, dtype=input_.dtype, device=input_.device + ) + # All-gather. + self.dist_module.all_gather_into_tensor( + output_tensor, input_, group=self.device_group + ) + + # Reshape + output_tensor = output_tensor.reshape((self.world_size,) + input_size) + output_tensor = output_tensor.movedim(0, dim) + output_tensor = output_tensor.reshape( + input_size[:dim] + + (self.world_size * input_size[dim],) + + input_size[dim + 1 :] + ) + return output_tensor + + +class _CPUSHMDistributed: + + def __init__(self, communicator: CpuCommunicator): + instance_identifier = os.environ["VLLM_DIST_IDENT"] + unique_name = communicator.unique_name + instance_identifier = f"{instance_identifier}-{unique_name}" + self.communicator = communicator + + group_ranks = [str(rank) for rank in self.communicator.ranks] + shm_group_identifier = f"[{'-'.join(group_ranks)}]" + self.group_name = f"{instance_identifier}-{shm_group_identifier}-cpushm" + + self.handle = self._init_cpu_shm() + + def _init_cpu_shm(self) -> int: + handle = torch.ops._C.init_shm_manager( + self.group_name, + self.communicator.world_size, + self.communicator.rank, + ) + torch.distributed.barrier(self.communicator.device_group) + torch.ops._C.join_shm_manager( + handle, + self.group_name, + ) + torch.distributed.barrier(self.communicator.device_group) + + return int(handle) + + def all_reduce( + self, input: torch.Tensor, group: ProcessGroup | None = None + ) -> None: + torch.ops._C.shm_allreduce(self.handle, input) + + def gather( + self, + input: torch.Tensor, + gather_list: list[torch.Tensor] | None, + dst: int = -1, + group: ProcessGroup | None = None, + ) -> None: + # Note: different from the torch gather, here we use local dst rank. + torch.ops._C.shm_gather( + self.handle, + input, + gather_list, + torch.distributed.get_group_rank(group, dst), + ) + + def all_gather_into_tensor( + self, + output: torch.Tensor, + input: torch.Tensor, + group: ProcessGroup | None = None, + ) -> None: + torch.ops._C.shm_all_gather(self.handle, input, output) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/cuda_communicator.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/cuda_communicator.py new file mode 100644 index 00000000..c128c69f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/cuda_communicator.py @@ -0,0 +1,79 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/distributed/device_communicators/cuda_communicator.py + +import torch +from torch.distributed import ProcessGroup + +from sglang.multimodal_gen.runtime.distributed.device_communicators.base_device_communicator import ( + DeviceCommunicatorBase, +) + + +class CudaCommunicator(DeviceCommunicatorBase): + + def __init__( + self, + cpu_group: ProcessGroup, + device: torch.device | None = None, + device_group: ProcessGroup | None = None, + unique_name: str = "", + ): + super().__init__(cpu_group, device, device_group, unique_name) + + from sglang.multimodal_gen.runtime.distributed.device_communicators.pynccl import ( + PyNcclCommunicator, + ) + + self.pynccl_comm: PyNcclCommunicator | None = None + if self.world_size > 1: + self.pynccl_comm = PyNcclCommunicator( + group=self.cpu_group, + device=self.device, + ) + + def all_reduce(self, input_, op: torch.distributed.ReduceOp | None = None): + pynccl_comm = self.pynccl_comm + assert pynccl_comm is not None + out = pynccl_comm.all_reduce(input_, op=op) + if out is None: + # fall back to the default all-reduce using PyTorch. + # this usually happens during testing. + # when we run the model, allreduce only happens for the TP + # group, where we always have either custom allreduce or pynccl. + out = input_.clone() + torch.distributed.all_reduce(out, group=self.device_group, op=op) + return out + + def send(self, tensor: torch.Tensor, dst: int | None = None) -> None: + """Sends a tensor to the destination rank in a non-blocking way""" + """NOTE: `dst` is the local rank of the destination rank.""" + if dst is None: + dst = (self.rank_in_group + 1) % self.world_size + + pynccl_comm = self.pynccl_comm + if pynccl_comm is not None and not pynccl_comm.disabled: + pynccl_comm.send(tensor, dst) + else: + torch.distributed.send(tensor, self.ranks[dst], self.device_group) + + def recv( + self, size: torch.Size, dtype: torch.dtype, src: int | None = None + ) -> torch.Tensor: + """Receives a tensor from the source rank.""" + """NOTE: `src` is the local rank of the source rank.""" + if src is None: + src = (self.rank_in_group - 1) % self.world_size + + tensor = torch.empty(size, dtype=dtype, device=self.device) + pynccl_comm = self.pynccl_comm + if pynccl_comm is not None and not pynccl_comm.disabled: + pynccl_comm.recv(tensor, src) + else: + torch.distributed.recv(tensor, self.ranks[src], self.device_group) + return tensor + + def destroy(self) -> None: + if self.pynccl_comm is not None: + self.pynccl_comm = None diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/pynccl.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/pynccl.py new file mode 100644 index 00000000..2d1ef558 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/pynccl.py @@ -0,0 +1,258 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/distributed/device_communicators/pynccl.py + +# ===================== import region ===================== +import torch +import torch.distributed as dist +from torch.distributed import ProcessGroup, ReduceOp + +from sglang.multimodal_gen.runtime.distributed.device_communicators.pynccl_wrapper import ( + NCCLLibrary, + buffer_type, + cudaStream_t, + ncclComm_t, + ncclDataTypeEnum, + ncclRedOpTypeEnum, + ncclUniqueId, +) +from sglang.multimodal_gen.runtime.distributed.utils import StatelessProcessGroup +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.utils import current_stream + +logger = init_logger(__name__) + + +class PyNcclCommunicator: + + def __init__( + self, + group: ProcessGroup | StatelessProcessGroup, + device: int | str | torch.device, + library_path: str | None = None, + ): + """ + Args: + group: the process group to work on. If None, it will use the + default process group. + device: the device to bind the PyNcclCommunicator to. If None, + it will be bind to f"cuda:{local_rank}". + library_path: the path to the NCCL library. If None, it will + use the default library path. + It is the caller's responsibility to make sure each communicator + is bind to a unique device. + """ + if not isinstance(group, StatelessProcessGroup): + assert dist.is_initialized() + assert ( + dist.get_backend(group) != dist.Backend.NCCL + ), "PyNcclCommunicator should be attached to a non-NCCL group." + # note: this rank is the rank in the group + self.rank = dist.get_rank(group) + self.world_size = dist.get_world_size(group) + else: + self.rank = group.rank + self.world_size = group.world_size + + self.group = group + + # if world_size == 1, no need to create communicator + if self.world_size == 1: + self.available = False + self.disabled = True + return + try: + self.nccl = NCCLLibrary(library_path) + except Exception: + # disable because of missing NCCL library + # e.g. in a non-GPU environment + self.available = False + self.disabled = True + return + + self.available = True + self.disabled = False + + logger.info("sglang-diffusion is using nccl==%s", self.nccl.ncclGetVersion()) + + if self.rank == 0: + # get the unique id from NCCL + self.unique_id = self.nccl.ncclGetUniqueId() + else: + # construct an empty unique id + self.unique_id = ncclUniqueId() + + if not isinstance(group, StatelessProcessGroup): + tensor = torch.ByteTensor(list(self.unique_id.internal)) + ranks = dist.get_process_group_ranks(group) + # arg `src` in `broadcast` is the global rank + dist.broadcast(tensor, src=ranks[0], group=group) + byte_list = tensor.tolist() + for i, byte in enumerate(byte_list): + self.unique_id.internal[i] = byte + else: + self.unique_id = group.broadcast_obj(self.unique_id, src=0) + if isinstance(device, int): + device = torch.device(f"cuda:{device}") + elif isinstance(device, str): + device = torch.device(device) + # now `device` is a `torch.device` object + assert isinstance(device, torch.device) + self.device = device + # nccl communicator and stream will use this device + # `torch.cuda.device` is a context manager that changes the + # current cuda device to the specified one + with torch.cuda.device(device): + self.comm: ncclComm_t = self.nccl.ncclCommInitRank( + self.world_size, self.unique_id, self.rank + ) + + stream = current_stream() + # A small all_reduce for warmup. + data = torch.zeros(1, device=device) + self.all_reduce(data) + if stream is not None: + stream.synchronize() + del data + + def all_reduce( + self, in_tensor: torch.Tensor, op: ReduceOp = ReduceOp.SUM, stream=None + ) -> torch.Tensor: + if self.disabled: + return None + # nccl communicator created on a specific device + # will only work on tensors on the same device + # otherwise it will cause "illegal memory access" + assert in_tensor.device == self.device, ( + f"this nccl communicator is created to work on {self.device}, " + f"but the input tensor is on {in_tensor.device}" + ) + + out_tensor = torch.empty_like(in_tensor) + + if stream is None: + stream = current_stream() + self.nccl.ncclAllReduce( + buffer_type(in_tensor.data_ptr()), + buffer_type(out_tensor.data_ptr()), + in_tensor.numel(), + ncclDataTypeEnum.from_torch(in_tensor.dtype), + ncclRedOpTypeEnum.from_torch(op), + self.comm, + cudaStream_t(stream.cuda_stream), + ) + return out_tensor + + def all_gather( + self, output_tensor: torch.Tensor, input_tensor: torch.Tensor, stream=None + ): + if self.disabled: + return + # nccl communicator created on a specific device + # will only work on tensors on the same device + # otherwise it will cause "illegal memory access" + assert input_tensor.device == self.device, ( + f"this nccl communicator is created to work on {self.device}, " + f"but the input tensor is on {input_tensor.device}" + ) + if stream is None: + stream = current_stream() + self.nccl.ncclAllGather( + buffer_type(input_tensor.data_ptr()), + buffer_type(output_tensor.data_ptr()), + input_tensor.numel(), + ncclDataTypeEnum.from_torch(input_tensor.dtype), + self.comm, + cudaStream_t(stream.cuda_stream), + ) + + def reduce_scatter( + self, + output_tensor: torch.Tensor, + input_tensor: torch.Tensor, + op: ReduceOp = ReduceOp.SUM, + stream=None, + ): + if self.disabled: + return + # nccl communicator created on a specific device + # will only work on tensors on the same device + # otherwise it will cause "illegal memory access" + assert input_tensor.device == self.device, ( + f"this nccl communicator is created to work on {self.device}, " + f"but the input tensor is on {input_tensor.device}" + ) + if stream is None: + stream = current_stream() + self.nccl.ncclReduceScatter( + buffer_type(input_tensor.data_ptr()), + buffer_type(output_tensor.data_ptr()), + output_tensor.numel(), + ncclDataTypeEnum.from_torch(input_tensor.dtype), + ncclRedOpTypeEnum.from_torch(op), + self.comm, + cudaStream_t(stream.cuda_stream), + ) + + def send(self, tensor: torch.Tensor, dst: int, stream=None): + if self.disabled: + return + assert tensor.device == self.device, ( + f"this nccl communicator is created to work on {self.device}, " + f"but the input tensor is on {tensor.device}" + ) + if stream is None: + stream = current_stream() + self.nccl.ncclSend( + buffer_type(tensor.data_ptr()), + tensor.numel(), + ncclDataTypeEnum.from_torch(tensor.dtype), + dst, + self.comm, + cudaStream_t(stream.cuda_stream), + ) + + def recv(self, tensor: torch.Tensor, src: int, stream=None): + if self.disabled: + return + assert tensor.device == self.device, ( + f"this nccl communicator is created to work on {self.device}, " + f"but the input tensor is on {tensor.device}" + ) + if stream is None: + stream = current_stream() + self.nccl.ncclRecv( + buffer_type(tensor.data_ptr()), + tensor.numel(), + ncclDataTypeEnum.from_torch(tensor.dtype), + src, + self.comm, + cudaStream_t(stream.cuda_stream), + ) + + def broadcast(self, tensor: torch.Tensor, src: int, stream=None): + if self.disabled: + return + assert tensor.device == self.device, ( + f"this nccl communicator is created to work on {self.device}, " + f"but the input tensor is on {tensor.device}" + ) + if stream is None: + stream = current_stream() + if src == self.rank: + sendbuff = buffer_type(tensor.data_ptr()) + # NCCL requires the sender also to have a receive buffer + recvbuff = buffer_type(tensor.data_ptr()) + else: + sendbuff = buffer_type() + recvbuff = buffer_type(tensor.data_ptr()) + self.nccl.ncclBroadcast( + sendbuff, + recvbuff, + tensor.numel(), + ncclDataTypeEnum.from_torch(tensor.dtype), + src, + self.comm, + cudaStream_t(stream.cuda_stream), + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/pynccl_wrapper.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/pynccl_wrapper.py new file mode 100644 index 00000000..598e7be9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/device_communicators/pynccl_wrapper.py @@ -0,0 +1,450 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/distributed/device_communicators/pynccl_wrapper.py + +# This file is a pure Python wrapper for the NCCL library. +# The main purpose is to use NCCL combined with CUDA graph. +# Before writing this script, we tried the following approach: +# 1. We tried to use `cupy`, it calls NCCL correctly, but `cupy` itself +# often gets stuck when initializing the NCCL communicator. +# 2. We tried to use `torch.distributed`, but `torch.distributed.all_reduce` +# contains many other potential cuda APIs, that are not allowed during +# capturing the CUDA graph. For further details, please check +# https://discuss.pytorch.org/t/pytorch-cudagraph-with-nccl-operation-failed/ . +# +# Another rejected idea is to write a C/C++ binding for NCCL. It is usually +# doable, but we often encounter issues related with nccl versions, and need +# to switch between different versions of NCCL. See +# https://github.com/NVIDIA/nccl/issues/1234 for more details. +# A C/C++ binding is not flexible enough to handle this. It requires +# recompilation of the code every time we want to switch between different +# versions. This current implementation, with a **pure** Python wrapper, is +# more flexible. We can easily switch between different versions of NCCL by +# changing the environment variable `SGLANG_DIFFUSION_NCCL_SO_PATH`, or the `so_file` +# variable in the code. + +# TODO(will): support SGLANG_DIFFUSION_NCCL_SO_PATH + +import ctypes +import platform +from dataclasses import dataclass +from typing import Any + +import torch +from torch.distributed import ReduceOp + +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.utils import find_nccl_library + +logger = init_logger(__name__) + +# === export types and functions from nccl to Python === +# for the original nccl definition, please check +# https://github.com/NVIDIA/nccl/blob/master/src/nccl.h.in + +ncclResult_t = ctypes.c_int +ncclComm_t = ctypes.c_void_p + + +class ncclUniqueId(ctypes.Structure): + _fields_ = [("internal", ctypes.c_byte * 128)] + + +cudaStream_t = ctypes.c_void_p +buffer_type = ctypes.c_void_p + +ncclDataType_t = ctypes.c_int + + +class ncclDataTypeEnum: + ncclInt8 = 0 + ncclChar = 0 + ncclUint8 = 1 + ncclInt32 = 2 + ncclInt = 2 + ncclUint32 = 3 + ncclInt64 = 4 + ncclUint64 = 5 + ncclFloat16 = 6 + ncclHalf = 6 + ncclFloat32 = 7 + ncclFloat = 7 + ncclFloat64 = 8 + ncclDouble = 8 + ncclBfloat16 = 9 + ncclNumTypes = 10 + + @classmethod + def from_torch(cls, dtype: torch.dtype) -> int: + if dtype == torch.int8: + return cls.ncclInt8 + if dtype == torch.uint8: + return cls.ncclUint8 + if dtype == torch.int32: + return cls.ncclInt32 + if dtype == torch.int64: + return cls.ncclInt64 + if dtype == torch.float16: + return cls.ncclFloat16 + if dtype == torch.float32: + return cls.ncclFloat32 + if dtype == torch.float64: + return cls.ncclFloat64 + if dtype == torch.bfloat16: + return cls.ncclBfloat16 + raise ValueError(f"Unsupported dtype: {dtype}") + + +ncclRedOp_t = ctypes.c_int + + +class ncclRedOpTypeEnum: + ncclSum = 0 + ncclProd = 1 + ncclMax = 2 + ncclMin = 3 + ncclAvg = 4 + ncclNumOps = 5 + + @classmethod + def from_torch(cls, op: ReduceOp) -> int: + if op == ReduceOp.SUM: + return cls.ncclSum + if op == ReduceOp.PRODUCT: + return cls.ncclProd + if op == ReduceOp.MAX: + return cls.ncclMax + if op == ReduceOp.MIN: + return cls.ncclMin + if op == ReduceOp.AVG: + return cls.ncclAvg + raise ValueError(f"Unsupported op: {op}") + + +@dataclass +class Function: + name: str + restype: Any + argtypes: list[Any] + + +class NCCLLibrary: + exported_functions = [ + # const char* ncclGetErrorString(ncclResult_t result) + Function("ncclGetErrorString", ctypes.c_char_p, [ncclResult_t]), + # ncclResult_t ncclGetVersion(int *version); + Function("ncclGetVersion", ncclResult_t, [ctypes.POINTER(ctypes.c_int)]), + # ncclResult_t ncclGetUniqueId(ncclUniqueId* uniqueId); + Function("ncclGetUniqueId", ncclResult_t, [ctypes.POINTER(ncclUniqueId)]), + # ncclResult_t ncclCommInitRank( + # ncclComm_t* comm, int nranks, ncclUniqueId commId, int rank); + # note that ncclComm_t is a pointer type, so the first argument + # is a pointer to a pointer + Function( + "ncclCommInitRank", + ncclResult_t, + [ctypes.POINTER(ncclComm_t), ctypes.c_int, ncclUniqueId, ctypes.c_int], + ), + # ncclResult_t ncclAllReduce( + # const void* sendbuff, void* recvbuff, size_t count, + # ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, + # cudaStream_t stream); + # note that cudaStream_t is a pointer type, so the last argument + # is a pointer + Function( + "ncclAllReduce", + ncclResult_t, + [ + buffer_type, + buffer_type, + ctypes.c_size_t, + ncclDataType_t, + ncclRedOp_t, + ncclComm_t, + cudaStream_t, + ], + ), + # ncclResult_t ncclAllGather( + # const void* sendbuff, void* recvbuff, size_t count, + # ncclDataType_t datatype, ncclComm_t comm, + # cudaStream_t stream); + # note that cudaStream_t is a pointer type, so the last argument + # is a pointer + Function( + "ncclAllGather", + ncclResult_t, + [ + buffer_type, + buffer_type, + ctypes.c_size_t, + ncclDataType_t, + ncclComm_t, + cudaStream_t, + ], + ), + # ncclResult_t ncclReduceScatter( + # const void* sendbuff, void* recvbuff, size_t count, + # ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, + # cudaStream_t stream); + # note that cudaStream_t is a pointer type, so the last argument + # is a pointer + Function( + "ncclReduceScatter", + ncclResult_t, + [ + buffer_type, + buffer_type, + ctypes.c_size_t, + ncclDataType_t, + ncclRedOp_t, + ncclComm_t, + cudaStream_t, + ], + ), + # ncclResult_t ncclSend( + # const void* sendbuff, size_t count, ncclDataType_t datatype, + # int dest, ncclComm_t comm, cudaStream_t stream); + Function( + "ncclSend", + ncclResult_t, + [ + buffer_type, + ctypes.c_size_t, + ncclDataType_t, + ctypes.c_int, + ncclComm_t, + cudaStream_t, + ], + ), + # ncclResult_t ncclRecv( + # void* recvbuff, size_t count, ncclDataType_t datatype, + # int src, ncclComm_t comm, cudaStream_t stream); + Function( + "ncclRecv", + ncclResult_t, + [ + buffer_type, + ctypes.c_size_t, + ncclDataType_t, + ctypes.c_int, + ncclComm_t, + cudaStream_t, + ], + ), + # ncclResult_t ncclBroadcast( + # const void* sendbuff, void* recvbuff, size_t count, + # ncclDataType_t datatype, int root, ncclComm_t comm, + # cudaStream_t stream); + Function( + "ncclBroadcast", + ncclResult_t, + [ + buffer_type, + buffer_type, + ctypes.c_size_t, + ncclDataType_t, + ctypes.c_int, + ncclComm_t, + cudaStream_t, + ], + ), + # be cautious! this is a collective call, it will block until all + # processes in the communicator have called this function. + # because Python object destruction can happen in random order, + # it is better not to call it at all. + # ncclResult_t ncclCommDestroy(ncclComm_t comm); + Function("ncclCommDestroy", ncclResult_t, [ncclComm_t]), + ] + + # class attribute to store the mapping from the path to the library + # to avoid loading the same library multiple times + path_to_library_cache: dict[str, Any] = {} + + # class attribute to store the mapping from library path + # to the corresponding dictionary + path_to_dict_mapping: dict[str, dict[str, Any]] = {} + + def __init__(self, so_file: str | None = None): + + so_file = so_file or find_nccl_library() + + try: + if so_file not in NCCLLibrary.path_to_dict_mapping: + lib = ctypes.CDLL(so_file) + NCCLLibrary.path_to_library_cache[so_file] = lib + self.lib = NCCLLibrary.path_to_library_cache[so_file] + except Exception as e: + logger.error( + "Failed to load NCCL library from %s ." + "It is expected if you are not running on NVIDIA/AMD GPUs." + "Otherwise, the nccl library might not exist, be corrupted " + "or it does not support the current platform %s." + "If you already have the library, please set the " + "environment variable SGLANG_DIFFUSION_NCCL_SO_PATH" + " to point to the correct nccl library path.", + so_file, + platform.platform(), + ) + raise e + + if so_file not in NCCLLibrary.path_to_dict_mapping: + _funcs: dict[str, Any] = {} + for func in NCCLLibrary.exported_functions: + f = getattr(self.lib, func.name) + f.restype = func.restype + f.argtypes = func.argtypes + _funcs[func.name] = f + NCCLLibrary.path_to_dict_mapping[so_file] = _funcs + self._funcs = NCCLLibrary.path_to_dict_mapping[so_file] + + def ncclGetErrorString(self, result: ncclResult_t) -> str: + return str(self._funcs["ncclGetErrorString"](result).decode("utf-8")) + + def NCCL_CHECK(self, result: ncclResult_t) -> None: + if result != 0: + error_str = self.ncclGetErrorString(result) + raise RuntimeError(f"NCCL error: {error_str}") + + def ncclGetVersion(self) -> str: + version = ctypes.c_int() + self.NCCL_CHECK(self._funcs["ncclGetVersion"](ctypes.byref(version))) + version_str = str(version.value) + # something like 21903 --> "2.19.3" + major = version_str[0].lstrip("0") + minor = version_str[1:3].lstrip("0") + patch = version_str[3:].lstrip("0") + return f"{major}.{minor}.{patch}" + + def ncclGetUniqueId(self) -> ncclUniqueId: + unique_id = ncclUniqueId() + self.NCCL_CHECK(self._funcs["ncclGetUniqueId"](ctypes.byref(unique_id))) + return unique_id + + def ncclCommInitRank( + self, world_size: int, unique_id: ncclUniqueId, rank: int + ) -> ncclComm_t: + comm = ncclComm_t() + self.NCCL_CHECK( + self._funcs["ncclCommInitRank"]( + ctypes.byref(comm), world_size, unique_id, rank + ) + ) + return comm + + def ncclAllReduce( + self, + sendbuff: buffer_type, + recvbuff: buffer_type, + count: int, + datatype: int, + op: int, + comm: ncclComm_t, + stream: cudaStream_t, + ) -> None: + # `datatype` actually should be `ncclDataType_t` + # and `op` should be `ncclRedOp_t` + # both are aliases of `ctypes.c_int` + # when we pass int to a function, it will be converted to `ctypes.c_int` + # by ctypes automatically + self.NCCL_CHECK( + self._funcs["ncclAllReduce"]( + sendbuff, recvbuff, count, datatype, op, comm, stream + ) + ) + + def ncclReduceScatter( + self, + sendbuff: buffer_type, + recvbuff: buffer_type, + count: int, + datatype: int, + op: int, + comm: ncclComm_t, + stream: cudaStream_t, + ) -> None: + # `datatype` actually should be `ncclDataType_t` + # and `op` should be `ncclRedOp_t` + # both are aliases of `ctypes.c_int` + # when we pass int to a function, it will be converted to `ctypes.c_int` + # by ctypes automatically + self.NCCL_CHECK( + self._funcs["ncclReduceScatter"]( + sendbuff, recvbuff, count, datatype, op, comm, stream + ) + ) + + def ncclAllGather( + self, + sendbuff: buffer_type, + recvbuff: buffer_type, + count: int, + datatype: int, + comm: ncclComm_t, + stream: cudaStream_t, + ) -> None: + # `datatype` actually should be `ncclDataType_t` + # which is an aliases of `ctypes.c_int` + # when we pass int to a function, it will be converted to `ctypes.c_int` + # by ctypes automatically + self.NCCL_CHECK( + self._funcs["ncclAllGather"]( + sendbuff, recvbuff, count, datatype, comm, stream + ) + ) + + def ncclSend( + self, + sendbuff: buffer_type, + count: int, + datatype: int, + dest: int, + comm: ncclComm_t, + stream: cudaStream_t, + ) -> None: + self.NCCL_CHECK( + self._funcs["ncclSend"](sendbuff, count, datatype, dest, comm, stream) + ) + + def ncclRecv( + self, + recvbuff: buffer_type, + count: int, + datatype: int, + src: int, + comm: ncclComm_t, + stream: cudaStream_t, + ) -> None: + self.NCCL_CHECK( + self._funcs["ncclRecv"](recvbuff, count, datatype, src, comm, stream) + ) + + def ncclBroadcast( + self, + sendbuff: buffer_type, + recvbuff: buffer_type, + count: int, + datatype: int, + root: int, + comm: ncclComm_t, + stream: cudaStream_t, + ) -> None: + self.NCCL_CHECK( + self._funcs["ncclBroadcast"]( + sendbuff, recvbuff, count, datatype, root, comm, stream + ) + ) + + def ncclCommDestroy(self, comm: ncclComm_t) -> None: + self.NCCL_CHECK(self._funcs["ncclCommDestroy"](comm)) + + +__all__ = [ + "NCCLLibrary", + "ncclDataTypeEnum", + "ncclRedOpTypeEnum", + "ncclUniqueId", + "ncclComm_t", + "cudaStream_t", + "buffer_type", +] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py new file mode 100644 index 00000000..dd42b875 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py @@ -0,0 +1,1226 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# Copyright 2024 xDiT team. +# Adapted from +# https://github.com/vllm-project/vllm/blob/main/vllm/distributed/parallel_state.py +# Copyright 2023 The vLLM team. +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +import pickle +from collections import namedtuple +from contextlib import contextmanager +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +import torch.distributed +from torch.cuda import synchronize +from torch.distributed import Backend, ProcessGroup + +from sglang.multimodal_gen import envs +from sglang.multimodal_gen.runtime.distributed.device_communicators.base_device_communicator import ( + DeviceCommunicatorBase, +) +from sglang.multimodal_gen.runtime.distributed.device_communicators.cpu_communicator import ( + CpuCommunicator, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +try: + import torch_musa # noqa: F401 + from torch_musa.core.device import synchronize +except ModuleNotFoundError: + pass + +logger = init_logger(__name__) + +TensorMetadata = namedtuple("TensorMetadata", ["device", "dtype", "size"]) + + +_group_name_counter: dict[str, int] = {} + + +def get_local_torch_device() -> torch.device: + """Return the torch device for the current rank.""" + from sglang.multimodal_gen.runtime.platforms import current_platform + + return ( + torch.device(f"cuda:{envs.LOCAL_RANK}") + if current_platform.is_cuda_alike() + else torch.device("mps") + ) + + +def _get_unique_name(name: str) -> str: + """Get a unique name for the group. + Example: + _get_unique_name("tp") -> "tp:0" + _get_unique_name("tp") -> "tp:1" + """ + if name not in _group_name_counter: + _group_name_counter[name] = 0 + newname = f"{name}:{_group_name_counter[name]}" + _group_name_counter[name] += 1 + return newname + + +def _split_tensor_dict( + tensor_dict: Dict[str, Union[torch.Tensor, Any]], prefix: str = "" +) -> Tuple[List[Tuple[str, Any]], List[torch.Tensor]]: + """Split the tensor dictionary into two parts: + 1. A list of (key, value) pairs. If the value is a tensor, it is replaced + by its metadata. + 2. A list of tensors. + + If the Tensor is nested under `tensor_dict["key1"]["key2"]`, the key of its + metadata will be "key1%key2". + """ + metadata_list: List[Tuple[str, Any]] = [] + tensor_list = [] + for key, value in tensor_dict.items(): + assert "%" not in key, ( + "Avoid having '%' in key " + "as it is used as a separator for nested entries." + ) + if isinstance(value, torch.Tensor): + # Note: we cannot use `value.device` here, + # because it contains not only the device type but also the device + # index (e.g. "cuda:0"). We only need the device type. + # receiving side will set the device index. + device = value.device.type + metadata_list.append( + ( + prefix + key, + TensorMetadata(device, value.dtype, value.size()), + ) + ) + tensor_list.append(value) + elif isinstance(value, dict): + if len(value) == 0: + metadata_list.append((prefix + key, value)) + inner_metadata_list, inner_tensor_list = _split_tensor_dict( + value, prefix + key + "%" + ) + metadata_list.extend(inner_metadata_list) + tensor_list.extend(inner_tensor_list) + else: + metadata_list.append((prefix + key, value)) + return metadata_list, tensor_list + + +def _update_nested_dict(nested_dict, flattened_key, value): + key_splits = flattened_key.split("%") + cur_dict = nested_dict + for k in key_splits[:-1]: + if k not in cur_dict: + cur_dict[k] = {} + cur_dict = cur_dict[k] + cur_dict[key_splits[-1]] = value + + +@dataclass +class GraphCaptureContext: + stream: torch.cuda.Stream | None + + +class GroupCoordinator: + """ + PyTorch ProcessGroup wrapper for a group of processes. + PyTorch ProcessGroup is bound to one specific communication backend, + e.g. NCCL, Gloo, MPI, etc. + GroupCoordinator takes charge of all the communication operations among + the processes in the group. It can route the communication to + a specific implementation (e.g. switch allreduce implementation + based on the tensor size and cuda graph mode). + """ + + # available attributes: + rank: int # global rank + ranks: List[int] # global ranks in the group + world_size: int # size of the group + # difference between `local_rank` and `rank_in_group`: + # if we have a group of size 4 across two nodes: + # Process | Node | Rank | Local Rank | Rank in Group + # 0 | 0 | 0 | 0 | 0 + # 1 | 0 | 1 | 1 | 1 + # 2 | 1 | 2 | 0 | 2 + # 3 | 1 | 3 | 1 | 3 + local_rank: int # local rank in the current node, used to assign devices + rank_in_group: int # rank inside the group + cpu_group: ProcessGroup # group for CPU communication + device_group: ProcessGroup # group for device communication + use_device_communicator: bool # whether to use device communicator + device_communicator: DeviceCommunicatorBase # device communicator + + def __init__( + self, + group_ranks: List[List[int]], + local_rank: int, + torch_distributed_backend: Union[str, Backend], + use_device_communicator: bool = True, + use_message_queue_broadcaster: bool = False, + group_name: str | None = None, + ): + self.unique_name = _get_unique_name(group_name) + self.rank = torch.distributed.get_rank() + self.local_rank = local_rank + self.device_group = None + self.cpu_group = None + + for ranks in group_ranks: + device_group = torch.distributed.new_group( + ranks, backend=torch_distributed_backend + ) + # a group with `gloo` backend, to allow direct coordination between + # processes through the CPU. + cpu_group = torch.distributed.new_group(ranks, backend="gloo") + if self.rank in ranks: + self.ranks = ranks + self.world_size = len(ranks) + self.rank_in_group = ranks.index(self.rank) + self.device_group = device_group + self.cpu_group = cpu_group + + assert self.cpu_group is not None, f"{group_ranks=}, {local_rank=}" + assert self.device_group is not None + + # TODO: fix it for other platforms + self.device = get_local_torch_device() + + from sglang.multimodal_gen.runtime.platforms import current_platform + + self.use_device_communicator = use_device_communicator + + self.device_communicator: DeviceCommunicatorBase = None # type: ignore + if use_device_communicator and self.world_size > 1: + # Platform-aware device communicator selection + if current_platform.is_cuda_alike(): + from sglang.multimodal_gen.runtime.distributed.device_communicators.cuda_communicator import ( + CudaCommunicator, + ) + + self.device_communicator = CudaCommunicator( + cpu_group=self.cpu_group, + device=self.device, + device_group=self.device_group, + unique_name=self.unique_name, + ) + else: + # For MPS and CPU, use the CPU communicator + self.device_communicator = CpuCommunicator( + cpu_group=self.cpu_group, + device=self.device, + device_group=self.device_group, + unique_name=self.unique_name, + ) + + self.mq_broadcaster = None + + # TODO(will): check if this is needed + # self.use_custom_op_call = current_platform.is_cuda_alike() + self.use_custom_op_call = False + + @property + def first_rank(self): + """Return the global rank of the first process in the group""" + return self.ranks[0] + + @property + def last_rank(self): + """Return the global rank of the last process in the group""" + return self.ranks[-1] + + @property + def is_first_rank(self): + """Return whether the caller is the first process in the group""" + return self.rank == self.first_rank + + @property + def is_last_rank(self): + """Return whether the caller is the last process in the group""" + return self.rank == self.last_rank + + @property + def next_rank(self): + """Return the global rank of the process that follows the caller""" + rank_in_group = self.rank_in_group + world_size = self.world_size + return self.ranks[(rank_in_group + 1) % world_size] + + @property + def prev_rank(self): + """Return the global rank of the process that precedes the caller""" + rank_in_group = self.rank_in_group + world_size = self.world_size + return self.ranks[(rank_in_group - 1) % world_size] + + @property + def group_next_rank(self): + """Return the group rank of the process that follows the caller""" + rank_in_group = self.rank_in_group + world_size = self.world_size + return (rank_in_group + 1) % world_size + + @property + def group_prev_rank(self): + """Return the group rank of the process that precedes the caller""" + rank_in_group = self.rank_in_group + world_size = self.world_size + return (rank_in_group - 1) % world_size + + @property + def skip_rank(self): + """Return the global rank of the process that skip connects with the caller""" + rank_in_group = self.rank_in_group + world_size = self.world_size + return self.ranks[(world_size - rank_in_group - 1) % world_size] + + @property + def group_skip_rank(self): + """Return the group rank of the process that skip connects with the caller""" + rank_in_group = self.rank_in_group + world_size = self.world_size + return (world_size - rank_in_group - 1) % world_size + + @contextmanager + def graph_capture(self, graph_capture_context: GraphCaptureContext | None = None): + # Platform-aware graph capture + from sglang.multimodal_gen.runtime.platforms import current_platform + + if current_platform.is_cuda_alike(): + if graph_capture_context is None: + stream = torch.cuda.Stream() + graph_capture_context = GraphCaptureContext(stream) + else: + stream = graph_capture_context.stream + + # ensure all initialization operations complete before attempting to + # capture the graph on another stream + curr_stream = torch.cuda.current_stream() + if curr_stream != stream: + stream.wait_stream(curr_stream) + + with torch.cuda.stream(stream): + yield graph_capture_context + else: + # For non-CUDA platforms (MPS, CPU), just yield the context without stream management + if graph_capture_context is None: + # Create a dummy context for non-CUDA platforms + graph_capture_context = GraphCaptureContext(None) + yield graph_capture_context + + def all_to_all_4D( + self, input_: torch.Tensor, scatter_dim: int = 2, gather_dim: int = 1 + ) -> torch.Tensor: + if self.world_size == 1: + return input_ + return self.device_communicator.all_to_all_4D(input_, scatter_dim, gather_dim) + + def all_reduce( + self, + input_: torch.Tensor, + op=torch._C._distributed_c10d.ReduceOp.SUM, + async_op: bool = False, + ) -> torch.Tensor: + """ + NOTE: This operation will be applied in-place or out-of-place. + Always assume this function modifies its input, but use the return + value as the output. + """ + # Bypass the function if we are using only 1 GPU. + if self.world_size == 1: + return input_ + else: + torch.distributed.all_reduce( + input_, op=op, group=self.device_group, async_op=async_op + ) + return input_ + + def all_gather( + self, input_: torch.Tensor, dim: int = 0, separate_tensors: bool = False + ) -> Union[torch.Tensor, List[torch.Tensor]]: + world_size = self.world_size + # Bypass the function if we are using only 1 GPU. + if world_size == 1: + return input_ + assert ( + -input_.dim() <= dim < input_.dim() + ), f"Invalid dim ({dim}) for input tensor with shape {input_.size()}" + if dim < 0: + # Convert negative dim to positive. + dim += input_.dim() + # Allocate output tensor. + input_size = list(input_.size()) + input_size[0] *= world_size + output_tensor = torch.empty( + input_size, dtype=input_.dtype, device=input_.device + ) + # All-gather. + torch.distributed.all_gather_into_tensor( + output_tensor, input_, group=self.device_group + ) + if dim != 0: + input_size[0] //= world_size + output_tensor = output_tensor.reshape( + [ + world_size, + ] + + input_size + ) + output_tensor = output_tensor.movedim(0, dim) + + if separate_tensors: + tensor_list = [ + output_tensor.reshape(-1) + .narrow(0, input_.numel() * i, input_.numel()) + .view_as(input_) + for i in range(world_size) + ] + return tensor_list + else: + input_size = list(input_.size()) + input_size[dim] = input_size[dim] * world_size + # Reshape + output_tensor = output_tensor.reshape(input_size) + return output_tensor + + def gather(self, input_: torch.Tensor, dst: int = 0, dim: int = -1) -> torch.Tensor: + """ + NOTE: We assume that the input tensor is on the same device across + all the ranks. + NOTE: `dst` is the local rank of the destination rank. + """ + world_size = self.world_size + # Bypass the function if we are using only 1 GPU. + if world_size == 1: + return input_ + assert ( + -input_.dim() <= dim < input_.dim() + ), f"Invalid dim ({dim}) for input tensor with shape {input_.size()}" + if dim < 0: + # Convert negative dim to positive. + dim += input_.dim() + # Allocate output tensor. + if self.rank_in_group == dst: + gather_list = [torch.empty_like(input_) for _ in range(world_size)] + else: + gather_list = None + # Gather. + torch.distributed.gather( + input_, gather_list, dst=self.ranks[dst], group=self.device_group + ) + if self.rank_in_group == dst: + output_tensor = torch.cat(gather_list, dim=dim) + else: + output_tensor = None + return output_tensor + + def broadcast(self, input_: torch.Tensor, src: int = 0, async_op: bool = False): + """Broadcast the input tensor. + NOTE: `src` is the local rank of the source rank. + """ + assert src < self.world_size, f"Invalid src rank ({src})" + + # Bypass the function if we are using only 1 GPU. + if self.world_size == 1: + return input_ + # Broadcast. + torch.distributed.broadcast( + input_, + src=self.ranks[src], + group=self.device_group, + async_op=async_op, + ) + return input_ + + def broadcast_object(self, obj: Optional[Any] = None, src: int = 0): + """Broadcast the input object. + NOTE: `src` is the local rank of the source rank. + """ + assert src < self.world_size, f"Invalid src rank ({src})" + + # Bypass the function if we are using only 1 GPU. + if self.world_size == 1: + return obj + if self.shm_broadcaster is not None: + assert src == 0, "Shared memory broadcaster only supports src=0" + return self.shm_broadcaster.broadcast_object(obj) + if self.rank_in_group == src: + torch.distributed.broadcast_object_list( + [obj], src=self.ranks[src], group=self.cpu_group + ) + return obj + else: + recv = [None] + torch.distributed.broadcast_object_list( + recv, src=self.ranks[src], group=self.cpu_group + ) + return recv[0] + + def broadcast_object_list( + self, + obj_list: List[Any], + src: int = 0, + group: Optional[ProcessGroup] = None, + ): + """Broadcast the input object list. + NOTE: `src` is the local rank of the source rank. + """ + assert src < self.world_size, f"Invalid src rank ({src})" + + # Bypass the function if we are using only 1 GPU. + if self.world_size == 1: + return obj_list + # Broadcast. + torch.distributed.broadcast_object_list( + obj_list, src=self.ranks[src], group=self.device_group + ) + return obj_list + + def send_object(self, obj: Any, dst: int) -> None: + """Send the input object list to the destination rank.""" + """NOTE: `dst` is the local rank of the destination rank.""" + + assert dst < self.world_size, f"Invalid dst rank ({dst})" + + assert dst != self.rank, ( + "Invalid destination rank. Destination rank is the same " + "as the current rank." + ) + + # Serialize object to tensor and get the size as well + object_tensor = torch.frombuffer(pickle.dumps(obj), dtype=torch.uint8) + + size_tensor = torch.tensor( + [object_tensor.numel()], dtype=torch.long, device="cpu" + ) + + # Send object size + + torch.distributed.send(size_tensor, dst=self.ranks[dst], group=self.cpu_group) + + # Send object + torch.distributed.send(object_tensor, dst=self.ranks[dst], group=self.cpu_group) + + return None + + def recv_object(self, src: int) -> Any: + """Receive the input object list from the source rank.""" + """NOTE: `src` is the local rank of the source rank.""" + + assert src < self.world_size, f"Invalid src rank ({src})" + + assert ( + src != self.rank + ), "Invalid source rank. Source rank is the same as the current rank." + + size_tensor = torch.empty(1, dtype=torch.long, device="cpu") + + # Receive object size + rank_size = torch.distributed.recv( + size_tensor, src=self.ranks[src], group=self.cpu_group + ) + + # Tensor to receive serialized objects into. + object_tensor = torch.empty( # type: ignore[call-overload] + size_tensor.item(), # type: ignore[arg-type] + dtype=torch.uint8, + device="cpu", + ) + + rank_object = torch.distributed.recv( + object_tensor, src=self.ranks[src], group=self.cpu_group + ) + + assert ( + rank_object == rank_size + ), "Received object sender rank does not match the size sender rank." + + obj = pickle.loads(object_tensor.numpy().tobytes()) + + return obj + + def broadcast_tensor_dict( + self, + tensor_dict: Optional[Dict[str, Union[torch.Tensor, Any]]] = None, + src: int = 0, + group: Optional[ProcessGroup] = None, + metadata_group: Optional[ProcessGroup] = None, + ) -> Optional[Dict[str, Union[torch.Tensor, Any]]]: + """Broadcast the input tensor dictionary. + NOTE: `src` is the local rank of the source rank. + """ + # Bypass the function if we are using only 1 GPU. + if not torch.distributed.is_initialized() or self.world_size == 1: + return tensor_dict + + group = self.device_group + metadata_group = self.cpu_group + assert src < self.world_size, f"Invalid src rank ({src})" + src = self.ranks[src] + + rank = self.rank + if rank == src: + metadata_list: List[Tuple[Any, Any]] = [] + assert isinstance( + tensor_dict, dict + ), f"Expecting a dictionary, got {type(tensor_dict)}" + metadata_list, tensor_list = _split_tensor_dict(tensor_dict) + # `metadata_list` lives in CPU memory. + # `broadcast_object_list` has serialization & deserialization, + # all happening on CPU. Therefore, we can use the CPU group. + self.broadcast_object(metadata_list, src=src) + async_handles = [] + for tensor in tensor_list: + if tensor.numel() == 0: + # Skip broadcasting empty tensors. + continue + if tensor.is_cpu: + # use metadata_group for CPU tensors + handle = torch.distributed.broadcast( + tensor, src=src, group=metadata_group, async_op=True + ) + else: + # use group for GPU tensors + handle = torch.distributed.broadcast( + tensor, src=src, group=group, async_op=True + ) + async_handles.append(handle) + for async_handle in async_handles: + async_handle.wait() + + else: + metadata_list = self.broadcast_object(None, src=src) + tensor_dict = {} + async_handles = [] + for key, value in metadata_list: + if isinstance(value, TensorMetadata): + tensor = torch.empty( + value.size, dtype=value.dtype, device=value.device + ) + if tensor.numel() == 0: + # Skip broadcasting empty tensors. + _update_nested_dict(tensor_dict, key, tensor) + continue + if tensor.is_cpu: + # use metadata_group for CPU tensors + handle = torch.distributed.broadcast( + tensor, src=src, group=metadata_group, async_op=True + ) + else: + # use group for GPU tensors + handle = torch.distributed.broadcast( + tensor, src=src, group=group, async_op=True + ) + async_handles.append(handle) + _update_nested_dict(tensor_dict, key, tensor) + else: + _update_nested_dict(tensor_dict, key, value) + for async_handle in async_handles: + async_handle.wait() + return tensor_dict + + def send_tensor_dict( + self, + tensor_dict: Dict[str, Union[torch.Tensor, Any]], + dst: Optional[int] = None, + ) -> Optional[Dict[str, Union[torch.Tensor, Any]]]: + """Send the input tensor dictionary. + NOTE: `dst` is the local rank of the source rank. + """ + # Bypass the function if we are using only 1 GPU. + if not torch.distributed.is_initialized() or self.world_size == 1: + return tensor_dict + + group = self.device_group + metadata_group = self.cpu_group + + if dst is None: + dst = self.group_next_rank + assert dst < self.world_size, f"Invalid dst rank ({dst})" + + metadata_list: List[Tuple[Any, Any]] = [] + assert isinstance( + tensor_dict, dict + ), f"Expecting a dictionary, got {type(tensor_dict)}" + metadata_list, tensor_list = _split_tensor_dict(tensor_dict) + # `metadata_list` lives in CPU memory. + # `send_object_list` has serialization & deserialization, + # all happening on CPU. Therefore, we can use the CPU group. + self.send_object(metadata_list, dst=dst) + for tensor in tensor_list: + if tensor.numel() == 0: + # Skip sending empty tensors. + continue + if tensor.is_cpu: + # use metadata_group for CPU tensors + torch.distributed.send( + tensor, dst=self.ranks[dst], group=metadata_group + ) + else: + # use group for GPU tensors + torch.distributed.send(tensor, dst=self.ranks[dst], group=group) + return None + + def recv_tensor_dict( + self, src: Optional[int] = None + ) -> Optional[Dict[str, Union[torch.Tensor, Any]]]: + """Recv the input tensor dictionary. + NOTE: `src` is the local rank of the source rank. + """ + # Bypass the function if we are using only 1 GPU. + if not torch.distributed.is_initialized() or self.world_size == 1: + return None + + group = self.device_group + metadata_group = self.cpu_group + + if src is None: + src = self.group_prev_rank + assert src < self.world_size, f"Invalid src rank ({src})" + + recv_metadata_list = self.recv_object(src=src) + tensor_dict: Dict[str, Any] = {} + for key, value in recv_metadata_list: + if isinstance(value, TensorMetadata): + tensor = torch.empty(value.size, dtype=value.dtype, device=value.device) + if tensor.numel() == 0: + # Skip broadcasting empty tensors. + _update_nested_dict(tensor_dict, key, tensor) + continue + if tensor.is_cpu: + # use metadata_group for CPU tensors + torch.distributed.recv( + tensor, src=self.ranks[src], group=metadata_group + ) + else: + # use group for GPU tensors + torch.distributed.recv(tensor, src=self.ranks[src], group=group) + _update_nested_dict(tensor_dict, key, tensor) + else: + _update_nested_dict(tensor_dict, key, value) + return tensor_dict + + def barrier(self): + """Barrier synchronization among the group. + NOTE: don't use `device_group` here! `barrier` in NCCL is + terrible because it is internally a broadcast operation with + secretly created GPU tensors. It is easy to mess up the current + device. Use the CPU group instead. + """ + torch.distributed.barrier(group=self.cpu_group) + + def send(self, tensor: torch.Tensor, dst: Optional[int] = None) -> None: + """Sends a tensor to the destination rank in a non-blocking way""" + """NOTE: `dst` is the rank_in_group of the destination rank.""" + if dst is None: + dst = self.group_next_rank + + torch.distributed.send( + tensor, + self.ranks[dst], + group=( + self.device_groups[self.rank_in_group % 2] + if self.world_size == 2 + else self.device_group + ), + ) + + def recv( + self, size: torch.Size, dtype: torch.dtype, src: Optional[int] = None + ) -> torch.Tensor: + """Receives a tensor from the src rank.""" + """NOTE: `src` is the rank_in_group of the source rank.""" + if src is None: + src = self.group_prev_rank + + tensor = torch.empty(size, dtype=dtype, device=self.device) + torch.distributed.recv( + tensor, + self.ranks[src], + ( + self.device_groups[(self.rank_in_group + 1) % 2] + if self.world_size == 2 + else self.device_group + ), + ) + return tensor + + def destroy(self) -> None: + if self.device_group is not None: + torch.distributed.destroy_process_group(self.device_group) + self.device_group = None + if self.cpu_group is not None: + torch.distributed.destroy_process_group(self.cpu_group) + self.cpu_group = None + if self.device_communicator is not None: + self.device_communicator.destroy() + if self.mq_broadcaster is not None: + self.mq_broadcaster = None + + +class PipelineGroupCoordinator(GroupCoordinator): + """ + available attributes: + rank: int # global rank + ranks: List[int] # global ranks in the group + world_size: int # size of the group + difference between `local_rank` and `rank_in_group`: + if we have a group of size 4 across two nodes: + Process | Node | Rank | Local Rank | Rank in Group + 0 | 0 | 0 | 0 | 0 + 1 | 0 | 1 | 1 | 1 + 2 | 1 | 2 | 0 | 2 + 3 | 1 | 3 | 1 | 3 + local_rank: int # local rank used to assign devices + rank_in_group: int # rank inside the group + cpu_group: ProcessGroup # group for CPU communication + device_group: ProcessGroup # group for device communication + """ + + def __init__( + self, + group_ranks: List[List[int]], + local_rank: int, + torch_distributed_backend: Union[str, Backend], + group_name: str | None = None, + ): + super().__init__( + group_ranks=group_ranks, + local_rank=local_rank, + torch_distributed_backend=torch_distributed_backend, + group_name=group_name, + ) + self.rank = torch.distributed.get_rank() + self.local_rank = local_rank + self.device_group = None + self.cpu_group = None + self.cpu_groups = [] + self.device_groups = [] + if len(group_ranks[0]) > 2 or len(group_ranks[0]) == 1: + for ranks in group_ranks: + device_group = torch.distributed.new_group( + ranks, backend=torch_distributed_backend + ) + # a group with `gloo` backend, to allow direct coordination between + # processes through the CPU. + cpu_group = torch.distributed.new_group(ranks, backend="gloo") + if self.rank in ranks: + self.ranks = ranks + self.world_size = len(ranks) + self.rank_in_group = ranks.index(self.rank) + self.device_group = device_group + self.cpu_group = cpu_group + # when pipeline parallelism is 2, we need to create two groups to avoid + # communication stall. + # *_group_0_1 represents the group for communication from device 0 to + # device 1. + # *_group_1_0 represents the group for communication from device 1 to + # device 0. + elif len(group_ranks[0]) == 2: + for ranks in group_ranks: + device_group_0_1 = torch.distributed.new_group( + ranks, backend=torch_distributed_backend + ) + device_group_1_0 = torch.distributed.new_group( + ranks, backend=torch_distributed_backend + ) + # a group with `gloo` backend, to allow direct coordination between + # processes through the CPU. + cpu_group_0_1 = torch.distributed.new_group(ranks, backend="gloo") + cpu_group_1_0 = torch.distributed.new_group(ranks, backend="gloo") + if self.rank in ranks: + self.ranks = ranks + self.world_size = len(ranks) + self.rank_in_group = ranks.index(self.rank) + self.device_groups = [device_group_0_1, device_group_1_0] + self.cpu_groups = [cpu_group_0_1, cpu_group_1_0] + self.device_group = device_group_0_1 + self.cpu_group = cpu_group_0_1 + + assert self.cpu_group is not None + assert self.device_group is not None + + self.device = envs.get_device(local_rank) + + self.recv_buffer_set: bool = False + self.recv_tasks_queue: List[Tuple[str, int]] = [] + self.receiving_tasks: List[Tuple[torch.distributed.Work, str, int]] = [] + self.dtype: Optional[torch.dtype] = None + self.num_pipefusion_patches: Optional[int] = None + + self.recv_shape: Dict[str, Dict[int, torch.Size]] = {} + self.send_shape: Dict[str, Dict[int, torch.Size]] = {} + self.recv_buffer: Dict[str, Dict[int, torch.Size]] = {} + + self.skip_tensor_recv_buffer_set: bool = False + self.recv_skip_tasks_queue: List[Union[int, Tuple[str, int]]] = [] + self.receiving_skip_tasks: List[Tuple[torch.distributed.Work, str, int]] = [] + self.skip_tensor_recv_buffer: Optional[ + Union[List[torch.Tensor], torch.Tensor] + ] = None + self.skip_device_group = None + for ranks in group_ranks: + skip_device_group = torch.distributed.new_group( + ranks, backend=torch_distributed_backend + ) + if self.rank in ranks: + self.skip_device_group = skip_device_group + assert self.skip_device_group is not None + + def reset_buffer(self): + self.recv_tasks_queue = [] + self.receiving_tasks = [] + self.recv_shape = {} + self.send_shape = {} + self.recv_buffer = {} + + self.recv_skip_tasks_queue = [] + self.receiving_skip_tasks = [] + self.skip_tensor_recv_buffer = {} + + def set_config(self, dtype: torch.dtype): + self.dtype = dtype + + def set_recv_buffer( + self, + num_pipefusion_patches: int, + patches_shape_list: List[List[int]], + feature_map_shape: List[int], + dtype: torch.dtype, + ): + assert isinstance(dtype, torch.dtype), "dtype must be a torch.dtype object" + assert ( + isinstance(num_pipefusion_patches, int) and num_pipefusion_patches >= 1 + ), "num_pipefusion_patches must be greater than or equal to 1" + self.dtype = dtype + self.num_pipefusion_patches = num_pipefusion_patches + self.recv_buffer = [ + torch.zeros(*shape, dtype=self.dtype, device=self.device) + for shape in patches_shape_list + ] + self.recv_buffer.append( + torch.zeros(*feature_map_shape, dtype=self.dtype, device=self.device) + ) + self.recv_buffer_set = True + + def set_extra_tensors_recv_buffer( + self, + name: str, + shape: List[int], + num_buffers: int = 1, + dtype: torch.dtype = torch.float16, + ): + self.extra_tensors_recv_buffer[name] = [ + torch.zeros(*shape, dtype=dtype, device=self.device) + for _ in range(num_buffers) + ] + + def _check_shape_and_buffer( + self, + tensor_send_to_next=None, + recv_prev=False, + name: Optional[str] = None, + segment_idx: int = 0, + ): + send_flag = False + name = name or "latent" + if tensor_send_to_next is not None: + shape_list = self.send_shape.get(name, None) + if shape_list is None: + self.send_shape[name] = {segment_idx: tensor_send_to_next.shape} + send_flag = True + elif shape_list.get(segment_idx, None) is None: + self.send_shape[name][segment_idx] = tensor_send_to_next.shape + send_flag = True + + recv_flag = False + if recv_prev: + shape_list = self.recv_shape.get(name, None) + if shape_list is None: + recv_flag = True + elif shape_list.get(segment_idx, None) is None: + recv_flag = True + + recv_prev_shape = self._communicate_shapes( + tensor_send_to_next=tensor_send_to_next if send_flag else None, + recv_prev=recv_flag, + ) + + if recv_flag: + if self.recv_shape.get(name, None) is None: + self.recv_shape[name] = {segment_idx: recv_prev_shape} + else: + self.recv_shape[name][segment_idx] = recv_prev_shape + + if self.recv_buffer.get(name, None) is None: + self.recv_buffer[name] = { + segment_idx: torch.zeros( + recv_prev_shape, device=self.device, dtype=self.dtype + ) + } + else: + if self.recv_buffer[name].get(segment_idx, None) is not None: + logger.warning( + f"Recv buffer [name: {name}, segment_idx: {segment_idx}] already exist. updating..." + ) + self.recv_buffer[name][segment_idx] = torch.zeros( + recv_prev_shape, device=self.device, dtype=self.dtype + ) + + def _communicate_shapes(self, tensor_send_to_next=None, recv_prev=False): + """Communicate tensor shapes between stages. Used to communicate + tensor shapes before the actual tensor communication happens. + + Args: + tensor_send_next: tensor to send to next rank (no tensor sent if + set to None). + recv_prev: boolean for whether tensor should be received from + previous rank. + """ + + ops = [] + if recv_prev: + recv_prev_dim_tensor = torch.empty( + (1), device=self.device, dtype=torch.int64 + ) + recv_prev_dim_op = torch.distributed.P2POp( + torch.distributed.irecv, + recv_prev_dim_tensor, + self.prev_rank, + self.device_group, + ) + ops.append(recv_prev_dim_op) + + if tensor_send_to_next is not None: + send_next_dim_tensor = torch.tensor( + tensor_send_to_next.dim(), device=self.device, dtype=torch.int64 + ) + send_next_dim_op = torch.distributed.P2POp( + torch.distributed.isend, + send_next_dim_tensor, + self.next_rank, + self.device_group, + ) + ops.append(send_next_dim_op) + + if len(ops) > 0: + reqs = torch.distributed.batch_isend_irecv(ops) + for req in reqs: + req.wait() + + # To protect against race condition when using batch_isend_irecv(). + # should take this out once the bug with batch_isend_irecv is resolved. + synchronize() + + ops = [] + recv_prev_shape_tensor = None + if recv_prev: + recv_prev_shape_tensor = torch.empty( + torch.Size(recv_prev_dim_tensor), + device=self.device, + dtype=torch.int64, + ) + recv_prev_shape_op = torch.distributed.P2POp( + torch.distributed.irecv, + recv_prev_shape_tensor, + self.prev_rank, + self.device_group, + ) + ops.append(recv_prev_shape_op) + + if tensor_send_to_next is not None: + send_next_shape_tensor = torch.tensor( + tensor_send_to_next.size(), + device=self.device, + dtype=torch.int64, + ) + send_next_shape_op = torch.distributed.P2POp( + torch.distributed.isend, + send_next_shape_tensor, + self.next_rank, + self.device_group, + ) + ops.append(send_next_shape_op) + + if len(ops) > 0: + reqs = torch.distributed.batch_isend_irecv(ops) + for req in reqs: + req.wait() + + synchronize() + + recv_prev_shape = [0, 0, 0] + if recv_prev_shape_tensor is not None: + recv_prev_shape = recv_prev_shape_tensor + return torch.Size(recv_prev_shape) + + def pipeline_send( + self, tensor: torch.Tensor, name: str = "latent", segment_idx: int = -1 + ) -> None: + tensor = tensor.contiguous() + self._check_shape_and_buffer( + tensor_send_to_next=tensor, name=name, segment_idx=segment_idx + ) + self._pipeline_isend(tensor).wait() + + def pipeline_isend( + self, tensor: torch.Tensor, name: str = "latent", segment_idx: int = -1 + ) -> None: + tensor = tensor.contiguous() + self._check_shape_and_buffer( + tensor_send_to_next=tensor, name=name, segment_idx=segment_idx + ) + self._pipeline_isend(tensor) + + def pipeline_recv(self, idx: int = -1, name: str = "latent") -> torch.Tensor: + name = name or "latent" + self._check_shape_and_buffer(recv_prev=True, name=name, segment_idx=idx) + self._pipeline_irecv(self.recv_buffer[name][idx]).wait() + return self.recv_buffer[name][idx] + + def add_pipeline_recv_task(self, idx: int = -1, name: str = "latent"): + name = name or "latent" + self.recv_tasks_queue.append((name, idx)) + + def recv_next(self): + if len(self.recv_tasks_queue) == 0: + raise ValueError("No more tasks to receive") + elif len(self.recv_tasks_queue) > 0: + name, idx = self.recv_tasks_queue.pop(0) + self._check_shape_and_buffer(recv_prev=True, name=name, segment_idx=idx) + self.receiving_tasks.append( + (self._pipeline_irecv(self.recv_buffer[name][idx]), name, idx) + ) + + def get_pipeline_recv_data( + self, idx: int = -1, name: str = "latent" + ) -> torch.Tensor: + assert ( + len(self.receiving_tasks) > 0 + ), "No tasks to receive, call add_pipeline_recv_task first" + receiving_task = self.receiving_tasks.pop(0) + receiving_task[0].wait() + assert ( + receiving_task[1] == name and receiving_task[2] == idx + ), "Received tensor does not match the requested" + return self.recv_buffer[name][idx] + + def _pipeline_irecv(self, tensor: torch.tensor): + return torch.distributed.irecv( + tensor, + src=self.prev_rank, + group=( + self.device_groups[(self.rank_in_group + 1) % 2] + if self.world_size == 2 + else self.device_group + ), + ) + + def _pipeline_isend(self, tensor: torch.tensor): + return torch.distributed.isend( + tensor, + dst=self.next_rank, + group=( + self.device_groups[self.rank_in_group % 2] + if self.world_size == 2 + else self.device_group + ), + ) + + def set_skip_tensor_recv_buffer( + self, + patches_shape_list: List[List[int]], + feature_map_shape: List[int], + ): + self.skip_tensor_recv_buffer = [ + torch.zeros(*shape, dtype=self.dtype, device=self.device) + for shape in patches_shape_list + ] + self.skip_tensor_recv_buffer.append( + torch.zeros(*feature_map_shape, dtype=self.dtype, device=self.device) + ) + self.skip_tensor_recv_buffer_set = True + + def pipeline_send_skip(self, tensor: torch.Tensor) -> None: + tensor = tensor.contiguous() + self._pipeline_isend_skip(tensor).wait() + + def pipeline_isend_skip(self, tensor: torch.Tensor) -> None: + tensor = tensor.contiguous() + self._pipeline_isend_skip(tensor) + + def pipeline_recv_skip(self, idx: int = -1) -> torch.Tensor: + self._pipeline_irecv_skip(self.skip_tensor_recv_buffer[idx]).wait() + return self.skip_tensor_recv_buffer[idx] + + def add_pipeline_recv_skip_task(self, idx: int = -1): + self.recv_skip_tasks_queue.append(idx) + + def get_pipeline_recv_skip_data(self, idx: int = -1) -> torch.Tensor: + assert ( + len(self.receiving_skip_tasks) > 0 + ), "No tasks to receive, call add_pipeline_recv_skip_task first" + receiving_skip_task = self.receiving_skip_tasks.pop(0) + receiving_skip_task[0].wait() + assert ( + receiving_skip_task[2] == idx + ), "Received tensor does not match the requested" + return self.skip_tensor_recv_buffer[idx] + + def recv_skip_next(self): + if len(self.recv_skip_tasks_queue) == 0: + raise ValueError("No more tasks to receive") + elif len(self.recv_skip_tasks_queue) > 0: + task = self.recv_skip_tasks_queue.pop(0) + idx = task + self.receiving_skip_tasks.append( + ( + self._pipeline_irecv_skip(self.skip_tensor_recv_buffer[idx]), + None, + idx, + ) + ) + + def _pipeline_irecv_skip(self, tensor: torch.tensor): + return torch.distributed.irecv( + tensor, src=self.skip_rank, group=self.skip_device_group + ) + + def _pipeline_isend_skip(self, tensor: torch.tensor): + return torch.distributed.isend( + tensor, dst=self.skip_rank, group=self.skip_device_group + ) + + +class SequenceParallelGroupCoordinator(GroupCoordinator): + def __init__( + self, + group_ranks: List[List[int]], + local_rank: int, + torch_distributed_backend: Union[str, Backend], + group_name: str | None = None, + **kwargs, + ): + super().__init__( + group_ranks=group_ranks, + local_rank=local_rank, + torch_distributed_backend=torch_distributed_backend, + group_name=group_name, + ) + ulysses_group = kwargs.get("ulysses_group", None) + ring_group = kwargs.get("ring_group", None) + if ulysses_group is None: + raise RuntimeError( + f"Please pass argument 'ulysses_group' when calling init func of SequenceParallelGroupCoordinator" + ) + if ring_group is None: + raise RuntimeError( + f"Please pass argument 'ring_group' when calling init func of SequenceParallelGroupCoordinator" + ) + self.ulysses_group = ulysses_group + self.ring_group = ring_group + + self.ulysses_world_size = torch.distributed.get_world_size(self.ulysses_group) + self.ulysses_rank = torch.distributed.get_rank(self.ulysses_group) + self.ring_world_size = torch.distributed.get_world_size(self.ring_group) + self.ring_rank = torch.distributed.get_rank(self.ring_group) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py new file mode 100644 index 00000000..82dbb588 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py @@ -0,0 +1,1144 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/distributed/parallel_state.py +# Copyright 2023 The vLLM team. +# Adapted from +# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# Adapted from +# Copyright 2024 xDiT team. +# Adapted from +# https://github.com/vllm-project/vllm/blob/main/vllm/distributed/parallel_state.py +# Copyright 2023 The vLLM team. +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + +"""sglang-diffusion distributed state. + +It takes over the control of the distributed environment from PyTorch. +The typical workflow is: + +- call `init_distributed_environment` to initialize the distributed environment. +- call `initialize_model_parallel` or `ensure_model_parallel_initialized` to + initialize the model parallel groups. + +- any code dealing with the distributed stuff + +- call `destroy_model_parallel` to destroy the model parallel groups. +- call `destroy_distributed_environment` to destroy the distributed environment. + +If you only need to use the distributed environment without model parallelism, + you can skip the model parallel initialization and destruction steps. +""" +import contextlib +import os +import weakref +from collections import namedtuple +from collections.abc import Callable +from contextlib import contextmanager +from multiprocessing import shared_memory +from typing import Any, List, Optional +from unittest.mock import patch + +import torch +import torch.distributed +from torch.distributed import ProcessGroup + +import sglang.multimodal_gen.envs as envs +from sglang.multimodal_gen.runtime.distributed.utils import StatelessProcessGroup +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +from ..utils.distributed import RankGenerator +from .group_coordinator import ( + GroupCoordinator, + PipelineGroupCoordinator, + SequenceParallelGroupCoordinator, + get_local_torch_device, +) + +logger = init_logger(__name__) + +_WORLD: Optional[GroupCoordinator] = None +_TP: Optional[GroupCoordinator] = None +_SP: Optional[SequenceParallelGroupCoordinator] = None +_PP: Optional[PipelineGroupCoordinator] = None +_CFG: Optional[GroupCoordinator] = None +_DP: Optional[GroupCoordinator] = None +_DIT: Optional[GroupCoordinator] = None +_VAE: Optional[GroupCoordinator] = None + +logger = init_logger(__name__) + +TensorMetadata = namedtuple("TensorMetadata", ["device", "dtype", "size"]) + + +def _split_tensor_dict( + tensor_dict: dict[str, torch.Tensor | Any] +) -> tuple[list[tuple[str, Any]], list[torch.Tensor]]: + """Split the tensor dictionary into two parts: + 1. A list of (key, value) pairs. If the value is a tensor, it is replaced + by its metadata. + 2. A list of tensors. + """ + metadata_list: list[tuple[str, Any]] = [] + tensor_list: list[torch.Tensor] = [] + for key, value in tensor_dict.items(): + if isinstance(value, torch.Tensor): + # Note: we cannot use `value.device` here, + # because it contains not only the device type but also the device + # index (e.g. "cuda:0"). We only need the device type. + # receiving side will set the device index. + device = value.device.type + metadata_list.append( + (key, TensorMetadata(device, value.dtype, value.size())) + ) + tensor_list.append(value) + else: + metadata_list.append((key, value)) + return metadata_list, tensor_list + + +_groups: dict[str, Callable[[], Optional["GroupCoordinator"]]] = {} + + +def _register_group(group: "GroupCoordinator") -> None: + _groups[group.unique_name] = weakref.ref(group) + + +def all_reduce(tensor: torch.Tensor, group_name: str) -> torch.Tensor: + assert group_name in _groups, f"Group {group_name} is not found." + group = _groups[group_name]() + if group is None: + raise ValueError(f"Group {group_name} is destroyed.") + return group._all_reduce_out_place(tensor) + + +def all_reduce_fake(tensor: torch.Tensor, group_name: str) -> torch.Tensor: + return torch.empty_like(tensor) + + +_WORLD: GroupCoordinator | None = None +_NODE: GroupCoordinator | None = None + + +def get_world_group() -> GroupCoordinator: + assert _WORLD is not None, "world group is not initialized" + return _WORLD + + +def init_world_group( + ranks: list[int], local_rank: int, backend: str +) -> GroupCoordinator: + return GroupCoordinator( + group_ranks=[ranks], + local_rank=local_rank, + torch_distributed_backend=backend, + use_device_communicator=True, + group_name="world", + ) + + +# xDiT +def init_parallel_group_coordinator( + group_ranks: List[List[int]], + local_rank: int, + backend: str, + parallel_mode: str, + **kwargs, +) -> GroupCoordinator: + """ + Returns a Group Coordinator for the given parallel mode + """ + assert parallel_mode in [ + "data", + "pipeline", + "tensor", + "sequence", + "classifier_free_guidance", + ], f"parallel_mode {parallel_mode} is not supported" + if parallel_mode == "pipeline": + return PipelineGroupCoordinator( + group_ranks=group_ranks, + local_rank=local_rank, + torch_distributed_backend=backend, + group_name="pp_group", + ) + elif parallel_mode == "sequence": + return SequenceParallelGroupCoordinator( + group_ranks=group_ranks, + local_rank=local_rank, + torch_distributed_backend=backend, + group_name="sp_group", + **kwargs, + ) + else: + # fallback to GroupCoordinator + return GroupCoordinator( + group_ranks=group_ranks, + local_rank=local_rank, + torch_distributed_backend=backend, + group_name="cfg_group", + ) + + +# def init_parallel_group_coordinator( +# group_ranks: list[list[int]], +# local_rank: int, +# backend: str, +# use_message_queue_broadcaster: bool = False, +# group_name: str | None = None, +# ) -> GroupCoordinator: +# return GroupCoordinator( +# group_ranks=group_ranks, +# local_rank=local_rank, +# torch_distributed_backend=backend, +# use_device_communicator=True, +# use_message_queue_broadcaster=use_message_queue_broadcaster, +# group_name=group_name, +# ) + + +_TP: GroupCoordinator | None = None + + +def get_tp_group() -> GroupCoordinator: + assert _TP is not None, "tensor model parallel group is not initialized" + return _TP + + +_ENABLE_CUSTOM_ALL_REDUCE = True + + +def set_custom_all_reduce(enable: bool): + global _ENABLE_CUSTOM_ALL_REDUCE + _ENABLE_CUSTOM_ALL_REDUCE = enable + + +def init_distributed_environment( + world_size: int = 1, + rank: int = 0, + distributed_init_method: str = "env://", + local_rank: int = 0, + backend: str = "nccl", + device_id: torch.device | None = None, +): + # Determine the appropriate backend based on the platform + from sglang.multimodal_gen.runtime.platforms import current_platform + + if backend == "nccl" and not current_platform.is_cuda_alike(): + # Use gloo backend for non-CUDA platforms (MPS, CPU) + backend = "gloo" + logger.info("Using gloo backend for %s platform", current_platform.device_name) + + logger.debug( + "world_size=%d rank=%d local_rank=%d " "distributed_init_method=%s backend=%s", + world_size, + rank, + local_rank, + distributed_init_method, + backend, + ) + if not torch.distributed.is_initialized(): + assert distributed_init_method is not None, ( + "distributed_init_method must be provided when initializing " + "distributed environment" + ) + + # For MPS, don't pass device_id as it doesn't support device indices + extra_args = {} if current_platform.is_mps() else dict(device_id=device_id) + torch.distributed.init_process_group( + backend=backend, + init_method=distributed_init_method, + world_size=world_size, + rank=rank, + **extra_args, + ) + # set the local rank + # local_rank is not available in torch ProcessGroup, + # see https://github.com/pytorch/pytorch/issues/122816 + if local_rank == -1: + # local rank not set, this usually happens in single-node + # setting, where we can use rank as local rank + if distributed_init_method == "env://": + local_rank = envs.LOCAL_RANK + else: + local_rank = rank + global _WORLD + if _WORLD is None: + ranks = list(range(torch.distributed.get_world_size())) + _WORLD = init_world_group(ranks, local_rank, backend) + else: + assert ( + _WORLD.world_size == torch.distributed.get_world_size() + ), "world group already initialized with a different world size" + + +_SP: GroupCoordinator | None = None + + +def get_sp_group() -> SequenceParallelGroupCoordinator: + assert _SP is not None, "pipeline model parallel group is not initialized" + return _SP + + +_DP: GroupCoordinator | None = None + + +def get_dp_group() -> GroupCoordinator: + assert _DP is not None, "data parallel group is not initialized" + return _DP + + +# xDiT +def initialize_model_parallel( + data_parallel_size: int = 1, + classifier_free_guidance_degree: int = 1, + sequence_parallel_degree: Optional[int] = None, + ulysses_degree: int = 1, + ring_degree: int = 1, + tensor_parallel_degree: int = 1, + pipeline_parallel_degree: int = 1, + vae_parallel_size: int = 0, + backend: Optional[str] = None, +) -> None: + """ + Initialize model parallel groups. + + Arguments: + data_parallel_size: number of data parallelism groups. + classifier_free_guidance_degree: number of GPUs used for Classifier Free Guidance (CFG) + sequence_parallel_degree: number of GPUs used for sequence parallelism. sequence_parallel_degree = ulysses_degree * ring_degree + ulysses_degree: number of GPUs used for ulysses sequence parallelism. + ring_degree: number of GPUs used for ring sequence parallelism. + tensor_parallel_degree: number of GPUs used for tensor parallelism. + pipeline_parallel_degree: number of GPUs used for pipeline parallelism. + backend: distributed backend of pytorch collective comm. + + Let's say we have a total of 16 GPUs denoted by g0 ... g15 and we + use 2 groups to parallelize the batch dim(dp), 2 groups to parallelize + split batch caused by CFG, and 2 GPUs to parallelize sequence. + + dp_degree (2) * cfg_degree (2) * sp_degree (2) * pp_degree (2) = 16. + + The present function will create 8 data-parallel groups, + 8 CFG group, 8 pipeline-parallel group, and + 8 sequence-parallel groups: + 8 data-parallel groups: + [g0, g8], [g1, g9], [g2, g10], [g3, g11], + [g4, g12], [g5, g13], [g6, g14], [g7, g15] + 8 CFG-parallel groups: + [g0, g4], [g1, g5], [g2, g6], [g3, g7], + [g8, g12], [g9, g13], [g10, g14], [g11, g15] + 8 sequence-parallel groups: + [g0, g1], [g2, g3], [g4, g5], [g6, g7], + [g8, g9], [g10, g11], [g12, g13], [g14, g15] + 8 pipeline-parallel groups: + [g0, g2], [g4, g6], [g8, g10], [g12, g14], + [g1, g3], [g5, g7], [g9, g11], [g13, g15] + Note that for efficiency, the caller should make sure adjacent ranks + are on the same DGX box. For example if we are using 2 DGX-1 boxes + with a total of 16 GPUs, rank 0 to 7 belong to the first box and + ranks 8 to 15 belong to the second box. + """ + + if backend is None: + backend = envs.get_torch_distributed_backend() + # Get world size and rank. Ensure some consistencies. + assert torch.distributed.is_initialized() + world_size: int = torch.distributed.get_world_size() + backend = backend or torch.distributed.get_backend(get_world_group().device_group) + + dit_parallel_size = ( + data_parallel_size + * classifier_free_guidance_degree + * sequence_parallel_degree + * pipeline_parallel_degree + * tensor_parallel_degree + ) + + if world_size < dit_parallel_size: + raise RuntimeError( + f"world_size ({world_size}) is less than " + f"tensor_parallel_degree ({tensor_parallel_degree}) x " + f"pipeline_parallel_degree ({pipeline_parallel_degree}) x" + f"sequence_parallel_degree ({sequence_parallel_degree}) x" + f"classifier_free_guidance_degree " + f"({classifier_free_guidance_degree}) x" + f"data_parallel_degree ({data_parallel_size})" + ) + + rank_generator: RankGenerator = RankGenerator( + tensor_parallel_degree, + sequence_parallel_degree, + pipeline_parallel_degree, + classifier_free_guidance_degree, + data_parallel_size, + "tp-sp-pp-cfg-dp", + ) + global _DP + assert _DP is None, "data parallel group is already initialized" + _DP = init_parallel_group_coordinator( + group_ranks=rank_generator.get_ranks("dp"), + local_rank=get_world_group().local_rank, + backend=backend, + parallel_mode="data", + ) + + global _CFG + assert _CFG is None, "classifier_free_guidance group is already initialized" + _CFG = init_parallel_group_coordinator( + group_ranks=rank_generator.get_ranks("cfg"), + local_rank=get_world_group().local_rank, + backend=backend, + parallel_mode="classifier_free_guidance", + ) + global _PP + assert _PP is None, "pipeline model parallel group is already initialized" + _PP = init_parallel_group_coordinator( + group_ranks=rank_generator.get_ranks("pp"), + local_rank=get_world_group().local_rank, + backend=backend, + parallel_mode="pipeline", + ) + + global _SP + assert _SP is None, "sequence parallel group is already initialized" + + from yunchang import set_seq_parallel_pg + from yunchang.globals import PROCESS_GROUP + + set_seq_parallel_pg( + sp_ulysses_degree=ulysses_degree, + sp_ring_degree=ring_degree, + rank=get_world_group().rank_in_group, + world_size=dit_parallel_size, + ) + + _SP = init_parallel_group_coordinator( + group_ranks=rank_generator.get_ranks("sp"), + local_rank=get_world_group().local_rank, + backend=backend, + parallel_mode="sequence", + ulysses_group=PROCESS_GROUP.ULYSSES_PG, + ring_group=PROCESS_GROUP.RING_PG, + ) + + global _TP + assert _TP is None, "Tensor parallel group is already initialized" + _TP = init_parallel_group_coordinator( + group_ranks=rank_generator.get_ranks("tp"), + local_rank=get_world_group().local_rank, + backend=backend, + parallel_mode="tensor", + ) + + if vae_parallel_size > 0: + init_vae_group(dit_parallel_size, vae_parallel_size, backend) + init_dit_group(dit_parallel_size, backend) + + +# + + +# def initialize_model_parallel( +# tensor_model_parallel_size: int = 1, +# sequence_model_parallel_size: int = 1, +# data_parallel_size: int = 1, +# backend: str | None = None, +# ) -> None: +# """ +# Initialize model parallel groups. +# +# Arguments: +# tensor_model_parallel_size: number of GPUs used for tensor model +# parallelism (used for language encoder). +# sequence_model_parallel_size: number of GPUs used for sequence model +# parallelism (used for DiT). +# """ +# # Get world size and rank. Ensure some consistencies. +# assert ( +# _WORLD is not None +# ), "world group is not initialized, please call init_distributed_environment first" +# world_size: int = get_world_size() +# backend = backend or torch.distributed.get_backend(get_world_group().device_group) +# assert ( +# world_size >= tensor_model_parallel_size +# ), f"world_size({world_size}) must be greater than or equal to tensor_model_parallel_size({tensor_model_parallel_size})" +# num_tensor_model_parallel_groups: int = world_size // tensor_model_parallel_size +# global _TP +# assert _TP is None, "tensor model parallel group is already initialized" +# group_ranks = [] +# for i in range(num_tensor_model_parallel_groups): +# ranks = list( +# range(i * tensor_model_parallel_size, (i + 1) * tensor_model_parallel_size) +# ) +# group_ranks.append(ranks) +# +# # message queue broadcaster is only used in tensor model parallel group +# _TP = init_parallel_group_coordinator( +# group_ranks, +# get_world_group().local_rank, +# backend, +# use_message_queue_broadcaster=True, +# group_name="tp", +# ) +# +# # Build the sequence model-parallel groups. +# num_sequence_model_parallel_groups: int = world_size // sequence_model_parallel_size +# global _SP +# assert _SP is None, "sequence model parallel group is already initialized" +# group_ranks = [] +# +# # Since SP is incompatible with TP and PP, we can use a simpler group creation logic +# for i in range(num_sequence_model_parallel_groups): +# # Create groups of consecutive ranks +# ranks = list( +# range( +# i * sequence_model_parallel_size, (i + 1) * sequence_model_parallel_size +# ) +# ) +# group_ranks.append(ranks) +# +# _SP = init_parallel_group_coordinator( +# group_ranks, get_world_group().local_rank, backend, group_name="sp" +# ) +# +# # Build the data parallel groups. +# num_data_parallel_groups: int = sequence_model_parallel_size +# global _DP +# assert _DP is None, "data parallel group is already initialized" +# group_ranks = [] +# +# for i in range(num_data_parallel_groups): +# ranks = list(range(i, world_size, num_data_parallel_groups)) +# group_ranks.append(ranks) +# +# _DP = init_parallel_group_coordinator( +# group_ranks, get_world_group().local_rank, backend, group_name="dp" +# ) +# + + +def get_sp_world_size() -> int: + """Return world size for the sequence model parallel group.""" + return get_sp_group().world_size + + +def get_sp_parallel_rank() -> int: + """Return my rank for the sequence model parallel group.""" + return get_sp_group().rank_in_group + + +def get_world_size() -> int: + """Return world size for the world group.""" + return get_world_group().world_size + + +def get_world_rank() -> int: + """Return my rank for the world group.""" + return get_world_group().rank + + +def get_dp_world_size() -> int: + """Return world size for the data parallel group.""" + return get_dp_group().world_size + + +def get_dp_rank() -> int: + """Return my rank for the data parallel group.""" + return get_dp_group().rank_in_group + + +def maybe_init_distributed_environment_and_model_parallel( + tp_size: int, + sp_size: int, + enable_cfg_parallel: bool, + ulysses_degree: int = 1, + ring_degree: int = 1, + dp_size: int = 1, + distributed_init_method: str = "env://", +): + from sglang.multimodal_gen.runtime.platforms import current_platform + + if _WORLD is not None and model_parallel_is_initialized(): + # make sure the tp and sp sizes are correct + assert ( + get_tp_world_size() == tp_size + ), f"You are trying to initialize model parallel groups with size {tp_size}, but they are already initialized with size {get_tp_world_size()}" + assert ( + get_sp_world_size() == sp_size + ), f"You are trying to initialize model parallel groups with size {sp_size}, but they are already initialized with size {get_sp_world_size()}" + return + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + world_size = int(os.environ.get("WORLD_SIZE", 1)) + rank = int(os.environ.get("RANK", 0)) + device = get_local_torch_device() + logger.info( + "Initializing distributed environment with world_size=%d, device=%s", + world_size, + device, + main_process_only=False, + ) + + init_distributed_environment( + world_size=world_size, + rank=rank, + local_rank=local_rank, + distributed_init_method=distributed_init_method, + device_id=device, + ) + initialize_model_parallel( + data_parallel_size=dp_size, + classifier_free_guidance_degree=2 if enable_cfg_parallel else 1, + tensor_parallel_degree=tp_size, + ulysses_degree=ulysses_degree, + ring_degree=ring_degree, + sequence_parallel_degree=sp_size, + ) + + # Only set CUDA device if we're on a CUDA platform + if current_platform.is_cuda_alike(): + device = torch.device(f"cuda:{local_rank}") + torch.cuda.set_device(device) + + +def model_parallel_is_initialized() -> bool: + """Check if tensor, sequence parallel groups are initialized.""" + return _TP is not None and _SP is not None and _DP is not None and _CFG is not None + + +_TP_STATE_PATCHED = False + + +@contextmanager +def patch_tensor_parallel_group(tp_group: GroupCoordinator): + """Patch the tp group temporarily until this function ends. + + This method is for draft workers of speculative decoding to run draft model + with different tp degree from that of target model workers. + + Args: + tp_group (GroupCoordinator): the tp group coordinator + """ + global _TP_STATE_PATCHED + assert not _TP_STATE_PATCHED, "Should not call when it's already patched" + + _TP_STATE_PATCHED = True + old_tp_group = get_tp_group() + global _TP + _TP = tp_group + try: + yield + finally: + # restore the original state + _TP_STATE_PATCHED = False + _TP = old_tp_group + + +def get_tp_world_size() -> int: + """Return world size for the tensor model parallel group.""" + return get_tp_group().world_size + + +def get_tp_rank() -> int: + """Return my rank for the tensor model parallel group.""" + return get_tp_group().rank_in_group + + +def destroy_distributed_environment() -> None: + global _WORLD + if _WORLD: + _WORLD.destroy() + _WORLD = None + if torch.distributed.is_initialized(): + torch.distributed.destroy_process_group() + + +def cleanup_dist_env_and_memory(shutdown_ray: bool = False): + destroy_model_parallel() + destroy_distributed_environment() + with contextlib.suppress(AssertionError): + torch.distributed.destroy_process_group() + if shutdown_ray: + import ray # Lazy import Ray + + ray.shutdown() + + +def is_the_same_node_as( + pg: ProcessGroup | StatelessProcessGroup, source_rank: int = 0 +) -> list[int]: + """ + This is a collective operation that returns if each rank is in the same node + as the source rank. It tests if processes are attached to the same + memory system (shared access to shared memory). + """ + if isinstance(pg, ProcessGroup): + assert ( + torch.distributed.get_backend(pg) != torch.distributed.Backend.NCCL + ), "in_the_same_node_as should be tested with a non-NCCL group." + # local rank inside the group + rank = torch.distributed.get_rank(group=pg) + world_size = torch.distributed.get_world_size(group=pg) + + # global ranks of the processes in the group + ranks = torch.distributed.get_process_group_ranks(pg) + else: + rank = pg.rank + world_size = pg.world_size + ranks = list(range(world_size)) + + # local tensor in each process to store the result + is_in_the_same_node = torch.tensor([0] * world_size, dtype=torch.int32) + + magic_message = b"magic_message" + shm = None + + try: + with contextlib.suppress(OSError): + if rank == source_rank: + # create a shared memory segment + shm = shared_memory.SharedMemory(create=True, size=128) + shm.buf[: len(magic_message)] = magic_message + if isinstance(pg, ProcessGroup): + torch.distributed.broadcast_object_list( + [shm.name], src=ranks[source_rank], group=pg + ) + else: + pg.broadcast_obj(shm.name, src=source_rank) + is_in_the_same_node[rank] = 1 + else: + # try to open the shared memory segment + if isinstance(pg, ProcessGroup): + recv = [None] + torch.distributed.broadcast_object_list( + recv, src=ranks[source_rank], group=pg + ) + name = recv[0] + else: + name = pg.broadcast_obj(None, src=source_rank) + # fix to https://stackoverflow.com/q/62748654/9191338 + # Python incorrectly tracks shared memory even if it is not + # created by the process. The following patch is a workaround. + with patch( + "multiprocessing.resource_tracker.register", + lambda *args, **kwargs: None, + ): + shm = shared_memory.SharedMemory(name=name) + if shm.buf[: len(magic_message)] == magic_message: + is_in_the_same_node[rank] = 1 + except Exception as e: + logger.error("Error ignored in is_in_the_same_node: %s", e) + finally: + if shm: + shm.close() + + if isinstance(pg, ProcessGroup): + torch.distributed.barrier(group=pg) + else: + pg.barrier() + + # clean up the shared memory segment + with contextlib.suppress(OSError): + if rank == source_rank and shm: + shm.unlink() + + if isinstance(pg, ProcessGroup): + torch.distributed.all_reduce(is_in_the_same_node, group=pg) + aggregated_data = is_in_the_same_node + else: + aggregated_data = torch.zeros_like(is_in_the_same_node) + for i in range(world_size): + rank_data = pg.broadcast_obj(is_in_the_same_node, src=i) + aggregated_data += rank_data + + return [x == 1 for x in aggregated_data.tolist()] + + +def initialize_tensor_parallel_group( + tensor_model_parallel_size: int = 1, + backend: str | None = None, + group_name_suffix: str = "", +) -> GroupCoordinator: + """Initialize a tensor parallel group for a specific model. + + This function creates a tensor parallel group that can be used with the + patch_tensor_parallel_group context manager. It allows different models + to use different tensor parallelism configurations. + + Arguments: + tensor_model_parallel_size: number of GPUs used for tensor model parallelism. + backend: communication backend to use. + group_name_suffix: optional suffix to make the group name unique. + + Returns: + A GroupCoordinator for tensor parallelism that can be used with + the patch_tensor_parallel_group context manager. + + Example usage: + ```python + # Initialize tensor parallel group for model1 + tp_group_model1 = initialize_tensor_parallel_group( + tensor_model_parallel_size=4, + group_name_suffix="model1" + ) + + # Use tensor parallelism for model1 + with patch_tensor_parallel_group(tp_group_model1): + # Run model1 with tensor parallelism + output1 = model1(input1) + ``` + """ + # Get world size and rank. Ensure some consistencies. + assert torch.distributed.is_initialized() + world_size: int = torch.distributed.get_world_size() + backend = backend or torch.distributed.get_backend(get_world_group().device_group) + + # Ensure the world size is compatible with the parallelism configuration + assert ( + world_size % tensor_model_parallel_size == 0 + ), f"World size ({world_size}) must be divisible by tensor_model_parallel_size ({tensor_model_parallel_size})" + + # Build the tensor model-parallel groups. + num_tensor_model_parallel_groups: int = world_size // tensor_model_parallel_size + tp_group_ranks = [] + for i in range(num_tensor_model_parallel_groups): + ranks = list( + range(i * tensor_model_parallel_size, (i + 1) * tensor_model_parallel_size) + ) + tp_group_ranks.append(ranks) + + # Create TP group coordinator with a unique name + group_name = f"tp_{group_name_suffix}" if group_name_suffix else "tp" + tp_group = init_parallel_group_coordinator( + tp_group_ranks, + get_world_group().local_rank, + backend, + use_message_queue_broadcaster=True, + group_name=group_name, + ) + + return tp_group + + +def initialize_sequence_parallel_group( + sequence_model_parallel_size: int = 1, + backend: str | None = None, + group_name_suffix: str = "", +) -> GroupCoordinator: + """Initialize a sequence parallel group for a specific model. + + This function creates a sequence parallel group that can be used with the + patch_sequence_parallel_group context manager. It allows different models + to use different sequence parallelism configurations. + + Arguments: + sequence_model_parallel_size: number of GPUs used for sequence model parallelism. + backend: communication backend to use. + group_name_suffix: optional suffix to make the group name unique. + + Returns: + A GroupCoordinator for sequence parallelism that can be used with + the patch_sequence_parallel_group context manager. + + Example usage: + ```python + # Initialize sequence parallel group for model2 + sp_group_model2 = initialize_sequence_parallel_group( + sequence_model_parallel_size=2, + group_name_suffix="model2" + ) + + # Use sequence parallelism for model2 + with patch_sequence_parallel_group(sp_group_model2): + # Run model2 with sequence parallelism + output2 = model2(input2) + ``` + """ + # Get world size and rank. Ensure some consistencies. + assert torch.distributed.is_initialized() + world_size: int = torch.distributed.get_world_size() + backend = backend or torch.distributed.get_backend(get_world_group().device_group) + + # Ensure the world size is compatible with the parallelism configuration + assert ( + world_size % sequence_model_parallel_size == 0 + ), f"World size ({world_size}) must be divisible by sequence_model_parallel_size ({sequence_model_parallel_size})" + + # Build the sequence model-parallel groups. + num_sequence_model_parallel_groups: int = world_size // sequence_model_parallel_size + sp_group_ranks = [] + + for i in range(num_sequence_model_parallel_groups): + # Create groups of consecutive ranks + ranks = list( + range( + i * sequence_model_parallel_size, (i + 1) * sequence_model_parallel_size + ) + ) + sp_group_ranks.append(ranks) + + # Create SP group coordinator with a unique name + group_name = f"sp_{group_name_suffix}" if group_name_suffix else "sp" + sp_group = init_parallel_group_coordinator( + sp_group_ranks, get_world_group().local_rank, backend, group_name=group_name + ) + + return sp_group + + +# * QUERY +def get_world_group() -> GroupCoordinator: + assert _WORLD is not None, "world group is not initialized" + return _WORLD + + +# TP +def get_tp_group() -> GroupCoordinator: + assert _TP is not None, "tensor model parallel group is not initialized" + return _TP + + +def get_tensor_model_parallel_world_size(): + """Return world size for the tensor model parallel group.""" + return get_tp_group().world_size + + +def get_tensor_model_parallel_rank(): + """Return my rank for the tensor model parallel group.""" + return get_tp_group().rank_in_group + + +def get_sequence_parallel_world_size(): + """Return world size for the sequence parallel group.""" + return get_sp_group().world_size + + +def get_sequence_parallel_rank(): + """Return my rank for the sequence parallel group.""" + return get_sp_group().rank_in_group + + +def get_ulysses_parallel_world_size(): + return get_sp_group().ulysses_world_size + + +def get_ulysses_parallel_rank(): + return get_sp_group().ulysses_rank + + +def get_ring_parallel_world_size(): + return get_sp_group().ring_world_size + + +def get_ring_parallel_rank(): + return get_sp_group().ring_rank + + +# PP +def get_pp_group() -> PipelineGroupCoordinator: + assert _PP is not None, "pipeline model parallel group is not initialized" + return _PP + + +def get_pipeline_parallel_world_size(): + """Return world size for the pipeline model parallel group.""" + return get_pp_group().world_size + + +def get_pipeline_parallel_rank(): + """Return my rank for the pipeline model parallel group.""" + return get_pp_group().rank_in_group + + +def is_pipeline_first_stage(): + """Return True if in the first pipeline model parallel stage, False otherwise.""" + return get_pipeline_parallel_rank() == 0 + + +def is_pipeline_last_stage(): + """Return True if in the last pipeline model parallel stage, False otherwise.""" + return get_pipeline_parallel_rank() == (get_pipeline_parallel_world_size() - 1) + + +# CFG +def get_cfg_group() -> GroupCoordinator: + assert ( + _CFG is not None + ), "classifier_free_guidance parallel group is not initialized" + return _CFG + + +def get_classifier_free_guidance_world_size(): + """Return world size for the classifier_free_guidance parallel group.""" + return get_cfg_group().world_size + + +def get_classifier_free_guidance_rank(): + """Return my rank for the classifier_free_guidance parallel group.""" + return get_cfg_group().rank_in_group + + +# DP +def get_dp_group() -> GroupCoordinator: + assert _DP is not None, "pipeline model parallel group is not initialized" + return _DP + + +def get_data_parallel_world_size(): + """Return world size for the data parallel group.""" + return get_dp_group().world_size + + +def get_data_parallel_rank(): + """Return my rank for the data parallel group.""" + return get_dp_group().rank_in_group + + +def is_dp_last_group(): + """Return True if in the last data parallel group, False otherwise.""" + return ( + get_sequence_parallel_rank() == (get_sequence_parallel_world_size() - 1) + and get_classifier_free_guidance_rank() + == (get_classifier_free_guidance_world_size() - 1) + and get_pipeline_parallel_rank() == (get_pipeline_parallel_world_size() - 1) + ) + + +def get_dit_world_size(): + """Return world size for the DiT model (excluding VAE).""" + return ( + get_data_parallel_world_size() + * get_classifier_free_guidance_world_size() + * get_sequence_parallel_world_size() + * get_pipeline_parallel_world_size() + * get_tensor_model_parallel_world_size() + ) + + +# Add VAE getter functions +def get_vae_parallel_group() -> GroupCoordinator: + assert _VAE is not None, "VAE parallel group is not initialized" + return _VAE + + +def get_vae_parallel_world_size(): + """Return world size for the VAE parallel group.""" + return get_vae_parallel_group().world_size + + +def get_vae_parallel_rank(): + """Return my rank for the VAE parallel group.""" + return get_vae_parallel_group().rank_in_group + + +# * SET + + +def init_world_group( + ranks: List[int], local_rank: int, backend: str +) -> GroupCoordinator: + return GroupCoordinator( + group_ranks=[ranks], + local_rank=local_rank, + torch_distributed_backend=backend, + ) + + +def model_parallel_is_initialized(): + """Check if tensor and pipeline parallel groups are initialized.""" + return ( + _DP is not None + and _CFG is not None + and _SP is not None + and _PP is not None + and _TP is not None + ) + + +def init_dit_group( + dit_parallel_size: int, + backend: str, +): + global _DIT + _DIT = torch.distributed.new_group( + ranks=list(range(dit_parallel_size)), backend=backend + ) + + +def get_dit_group(): + assert _DIT is not None, "DIT group is not initialized" + return _DIT + + +def init_vae_group( + dit_parallel_size: int, + vae_parallel_size: int, + backend: str, +): + # Initialize VAE group first + global _VAE + assert _VAE is None, "VAE parallel group is already initialized" + vae_ranks = list(range(dit_parallel_size, dit_parallel_size + vae_parallel_size)) + _VAE = torch.distributed.new_group(ranks=vae_ranks, backend=backend) + + +def destroy_model_parallel() -> None: + """Set the groups to none and destroy them.""" + global _TP + if _TP: + _TP.destroy() + _TP = None + + global _SP + if _SP: + _SP.destroy() + _SP = None + + global _DP + if _DP: + _DP.destroy() + _DP = None + + +# xDit +# def destroy_model_parallel(): +# """Set the groups to none and destroy them.""" +# global _DP +# if _DP: +# _DP.destroy() +# _DP = None +# +# global _CFG +# if _CFG: +# _CFG.destroy() +# _CFG = None +# +# global _SP +# if _SP: +# _SP.destroy() +# _SP = None +# +# global _TP +# if _TP: +# _TP.destroy() +# _TP = None +# +# global _PP +# if _PP: +# _PP.destroy() +# _PP = None +# +# global _VAE +# if _VAE: +# _VAE.destroy() +# _VAE = None + + +def destroy_distributed_environment(): + global _WORLD + if _WORLD: + _WORLD.destroy() + _WORLD = None + if torch.distributed.is_initialized(): + torch.distributed.destroy_process_group() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/utils.py new file mode 100644 index 00000000..2d84f8b5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/distributed/utils.py @@ -0,0 +1,195 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/distributed/utils.py + +# Copyright 2023 The vLLM team. +# Adapted from +# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/utils.py +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +import dataclasses +import pickle +import time +from collections import deque +from collections.abc import Sequence +from typing import Any + +import torch +from torch.distributed import TCPStore + +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +def ensure_divisibility(numerator, denominator) -> None: + """Ensure that numerator is divisible by the denominator.""" + assert numerator % denominator == 0, "{} is not divisible by {}".format( + numerator, denominator + ) + + +def divide(numerator: int, denominator: int) -> int: + """Ensure that numerator is divisible by the denominator and return + the division value.""" + ensure_divisibility(numerator, denominator) + return numerator // denominator + + +def split_tensor_along_last_dim( + tensor: torch.Tensor, + num_partitions: int, + contiguous_split_chunks: bool = False, +) -> Sequence[torch.Tensor]: + """Split a tensor along its last dimension. + + Arguments: + tensor: input tensor. + num_partitions: number of partitions to split the tensor + contiguous_split_chunks: If True, make each chunk contiguous + in memory. + + Returns: + A list of Tensors + """ + # Get the size and dimension. + last_dim = tensor.dim() - 1 + last_dim_size = divide(tensor.size()[last_dim], num_partitions) + # Split. + tensor_list = torch.split(tensor, last_dim_size, dim=last_dim) + # NOTE: torch.split does not create contiguous tensors by default. + if contiguous_split_chunks: + return tuple(chunk.contiguous() for chunk in tensor_list) + + return tuple(tensor_list) + + +@dataclasses.dataclass +class StatelessProcessGroup: + """A dataclass to hold a metadata store, and the rank, world_size of the + group. Only use it to communicate metadata between processes. + For data-plane communication, create NCCL-related objects. + """ + + rank: int + world_size: int + store: torch._C._distributed_c10d.Store + data_expiration_seconds: int = 3600 # 1 hour + + # dst rank -> counter + send_dst_counter: dict[int, int] = dataclasses.field(default_factory=dict) + # src rank -> counter + recv_src_counter: dict[int, int] = dataclasses.field(default_factory=dict) + broadcast_send_counter: int = 0 + broadcast_recv_src_counter: dict[int, int] = dataclasses.field(default_factory=dict) + + # A deque to store the data entries, with key and timestamp. + entries: deque[tuple[str, float]] = dataclasses.field(default_factory=deque) + + def __post_init__(self): + assert self.rank < self.world_size + self.send_dst_counter = {i: 0 for i in range(self.world_size)} + self.recv_src_counter = {i: 0 for i in range(self.world_size)} + self.broadcast_recv_src_counter = {i: 0 for i in range(self.world_size)} + + def send_obj(self, obj: Any, dst: int): + """Send an object to a destination rank.""" + self.expire_data() + key = f"send_to/{dst}/{self.send_dst_counter[dst]}" + self.store.set(key, pickle.dumps(obj)) + self.send_dst_counter[dst] += 1 + self.entries.append((key, time.perf_counter())) + + def expire_data(self) -> None: + """Expire data that is older than `data_expiration_seconds` seconds.""" + while self.entries: + # check the oldest entry + key, timestamp = self.entries[0] + if time.perf_counter() - timestamp > self.data_expiration_seconds: + self.store.delete_key(key) + self.entries.popleft() + else: + break + + def recv_obj(self, src: int) -> Any: + """Receive an object from a source rank.""" + obj = pickle.loads( + self.store.get(f"send_to/{self.rank}/{self.recv_src_counter[src]}") + ) + self.recv_src_counter[src] += 1 + return obj + + def broadcast_obj(self, obj: Any | None, src: int) -> Any: + """Broadcast an object from a source rank to all other ranks. + It does not clean up after all ranks have received the object. + Use it for limited times, e.g., for initialization. + """ + if self.rank == src: + self.expire_data() + key = f"broadcast_from/{src}/" f"{self.broadcast_send_counter}" + self.store.set(key, pickle.dumps(obj)) + self.broadcast_send_counter += 1 + self.entries.append((key, time.perf_counter())) + return obj + else: + key = f"broadcast_from/{src}/" f"{self.broadcast_recv_src_counter[src]}" + recv_obj = pickle.loads(self.store.get(key)) + self.broadcast_recv_src_counter[src] += 1 + return recv_obj + + def all_gather_obj(self, obj: Any) -> list[Any]: + """All gather an object from all ranks.""" + gathered_objs = [] + for i in range(self.world_size): + if i == self.rank: + gathered_objs.append(obj) + self.broadcast_obj(obj, src=self.rank) + else: + recv_obj = self.broadcast_obj(None, src=i) + gathered_objs.append(recv_obj) + return gathered_objs + + def barrier(self): + """A barrier to synchronize all ranks.""" + for i in range(self.world_size): + if i == self.rank: + self.broadcast_obj(None, src=self.rank) + else: + self.broadcast_obj(None, src=i) + + @staticmethod + def create( + host: str, + port: int, + rank: int, + world_size: int, + data_expiration_seconds: int = 3600, + ) -> "StatelessProcessGroup": + """A replacement for `torch.distributed.init_process_group` that does not + pollute the global state. + + If we have process A and process B called `torch.distributed.init_process_group` + to form a group, and then we want to form another group with process A, B, C, + D, it is not possible in PyTorch, because process A and process B have already + formed a group, and process C and process D cannot join that group. This + function is a workaround for this issue. + + `torch.distributed.init_process_group` is a global call, while this function + is a stateless call. It will return a `StatelessProcessGroup` object that can be + used for exchanging metadata. With this function, process A and process B + can call `StatelessProcessGroup.create` to form a group, and then process A, B, + C, and D can call `StatelessProcessGroup.create` to form another group. + """ # noqa + store = TCPStore( + host_name=host, + port=port, + world_size=world_size, + is_master=(rank == 0), + ) + + return StatelessProcessGroup( + rank=rank, + world_size=world_size, + store=store, + data_expiration_seconds=data_expiration_seconds, + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/__init__.py new file mode 100644 index 00000000..af2eb7d1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/__init__.py @@ -0,0 +1 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/__init__.py new file mode 100644 index 00000000..af2eb7d1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/__init__.py @@ -0,0 +1 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/cli_types.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/cli_types.py new file mode 100644 index 00000000..2e5107ec --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/cli_types.py @@ -0,0 +1,28 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/entrypoints/cli/types.py + +import argparse + +from sglang.multimodal_gen.utils import FlexibleArgumentParser + + +class CLISubcommand: + """Base class for CLI subcommands""" + + name: str + + def cmd(self, args: argparse.Namespace) -> None: + """Execute the command with the given arguments""" + raise NotImplementedError + + def validate(self, args: argparse.Namespace) -> None: + """Validate the arguments for this command""" + pass + + def subparser_init( + self, subparsers: argparse._SubParsersAction + ) -> FlexibleArgumentParser: + """Initialize the subparser for this command""" + raise NotImplementedError diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/generate.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/generate.py new file mode 100644 index 00000000..9b4f8b59 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/generate.py @@ -0,0 +1,151 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/entrypoints/cli/serve.py + +import argparse +import dataclasses +import os +from typing import cast + +import sglang.multimodal_gen.envs as envs +from sglang.multimodal_gen import DiffGenerator +from sglang.multimodal_gen.configs.sample.sampling_params import ( + SamplingParams, + generate_request_id, +) +from sglang.multimodal_gen.runtime.entrypoints.cli.cli_types import CLISubcommand +from sglang.multimodal_gen.runtime.entrypoints.cli.utils import ( + RaiseNotImplementedAction, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.runtime.utils.perf_logger import ( + PerformanceLogger, + RequestTimings, +) +from sglang.multimodal_gen.utils import FlexibleArgumentParser + +logger = init_logger(__name__) + + +def add_multimodal_gen_generate_args(parser: argparse.ArgumentParser): + """Add the arguments for the generate command.""" + parser.add_argument( + "--config", + type=str, + default="", + required=False, + help="Read CLI options from a config JSON or YAML file. If provided, --model-path and --prompt are optional.", + ) + parser.add_argument( + "--perf-dump-path", + type=str, + default=None, + required=False, + help="Path to dump the performance metrics (JSON) for the run.", + ) + + parser = ServerArgs.add_cli_args(parser) + parser = SamplingParams.add_cli_args(parser) + + parser.add_argument( + "--text-encoder-configs", + action=RaiseNotImplementedAction, + help="JSON array of text encoder configurations (NOT YET IMPLEMENTED)", + ) + + return parser + + +def maybe_dump_performance(args: argparse.Namespace, server_args, prompt: str, results): + """dump performance if necessary""" + if not (args.perf_dump_path and results): + return + + if isinstance(results, list): + result = results[0] if results else {} + else: + result = results + + timings_dict = result.get("timings") + if not (args.perf_dump_path and timings_dict): + return + + timings = RequestTimings(request_id=timings_dict.get("request_id")) + timings.stages = timings_dict.get("stages", {}) + timings.total_duration_ms = timings_dict.get("total_duration_ms", 0) + + PerformanceLogger.dump_benchmark_report( + file_path=args.perf_dump_path, + timings=timings, + meta={ + "prompt": prompt, + "model": server_args.model_path, + }, + tag="cli_generate", + ) + + +def generate_cmd(args: argparse.Namespace): + """The entry point for the generate command.""" + # FIXME(mick): do not hard code + args.request_id = generate_request_id() + + # Auto-enable stage logging if dump path is provided + if args.perf_dump_path: + os.environ["SGLANG_DIFFUSION_STAGE_LOGGING"] = "True" + envs.SGLANG_DIFFUSION_STAGE_LOGGING = True + + server_args = ServerArgs.from_cli_args(args) + sampling_params_kwargs = SamplingParams.get_cli_args(args) + generator = DiffGenerator.from_pretrained( + model_path=server_args.model_path, server_args=server_args + ) + + results = generator.generate(sampling_params_kwargs=sampling_params_kwargs) + + prompt = sampling_params_kwargs.get("prompt", None) + maybe_dump_performance(args, server_args, prompt, results) + + +class GenerateSubcommand(CLISubcommand): + """The `generate` subcommand for the sglang-diffusion CLI""" + + def __init__(self) -> None: + self.name = "generate" + super().__init__() + self.init_arg_names = self._get_init_arg_names() + self.generation_arg_names = self._get_generation_arg_names() + + def _get_init_arg_names(self) -> list[str]: + """Get names of arguments for DiffGenerator initialization""" + return ["num_gpus", "tp_size", "sp_size", "model_path"] + + def _get_generation_arg_names(self) -> list[str]: + """Get names of arguments for generate_video method""" + return [field.name for field in dataclasses.fields(SamplingParams)] + + def cmd(self, args: argparse.Namespace) -> None: + generate_cmd(args) + + def validate(self, args: argparse.Namespace) -> None: + """Validate the arguments for this command""" + if args.num_gpus is not None and args.num_gpus <= 0: + raise ValueError("Number of gpus must be positive") + + if args.config and not os.path.exists(args.config): + raise ValueError(f"Config file not found: {args.config}") + + def subparser_init( + self, subparsers: argparse._SubParsersAction + ) -> FlexibleArgumentParser: + generate_parser = subparsers.add_parser( + "generate", + help="Run inference on a model", + usage="sgl_diffusion generate (--model-path MODEL_PATH_OR_ID --prompt PROMPT) | --config CONFIG_FILE [OPTIONS]", + ) + + generate_parser = add_multimodal_gen_generate_args(generate_parser) + + return cast(FlexibleArgumentParser, generate_parser) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/main.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/main.py new file mode 100644 index 00000000..c35dec33 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/main.py @@ -0,0 +1,44 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/entrypoints/cli/main.py + +from sglang.multimodal_gen.runtime.entrypoints.cli.cli_types import CLISubcommand +from sglang.multimodal_gen.runtime.entrypoints.cli.generate import GenerateSubcommand +from sglang.multimodal_gen.runtime.entrypoints.cli.serve import ServeSubcommand +from sglang.multimodal_gen.utils import FlexibleArgumentParser + + +def generate_cmd_init() -> list[CLISubcommand]: + return [GenerateSubcommand(), ServeSubcommand()] + + +def cmd_init() -> list[CLISubcommand]: + """Initialize all commands from separate modules""" + commands = [] + commands.extend(generate_cmd_init()) + return commands + + +def main() -> None: + parser = FlexibleArgumentParser(description="sglang-diffusion CLI") + parser.add_argument("-v", "--version", action="version", version="0.1.0") + + subparsers = parser.add_subparsers(required=False, dest="subparser") + + cmds = {} + for cmd in cmd_init(): + cmd.subparser_init(subparsers).set_defaults(dispatch_function=cmd.cmd) + cmds[cmd.name] = cmd + args = parser.parse_args() + if args.subparser in cmds: + cmds[args.subparser].validate(args) + + if hasattr(args, "dispatch_function"): + args.dispatch_function(args) + else: + parser.print_help() + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/serve.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/serve.py new file mode 100644 index 00000000..5f939a28 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/serve.py @@ -0,0 +1,69 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import os +from typing import cast + +from sglang.multimodal_gen.runtime.entrypoints.cli.cli_types import CLISubcommand +from sglang.multimodal_gen.runtime.launch_server import launch_server +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.utils import FlexibleArgumentParser + +logger = init_logger(__name__) + + +def add_multimodal_gen_serve_args(parser: argparse.ArgumentParser): + """Add the arguments for the serve command.""" + parser.add_argument( + "--config", + type=str, + default="", + required=False, + help="Read CLI options from a config JSON or YAML file.", + ) + return ServerArgs.add_cli_args(parser) + + +def execute_serve_cmd(args: argparse.Namespace, unknown_args: list[str] | None = None): + """The entry point for the serve command.""" + server_args = ServerArgs.from_cli_args(args, unknown_args) + server_args.post_init_serve() + launch_server(server_args) + + +class ServeSubcommand(CLISubcommand): + """The `serve` subcommand for the sglang-diffusion CLI""" + + def __init__(self) -> None: + self.name = "serve" + super().__init__() + + def cmd( + self, args: argparse.Namespace, unknown_args: list[str] | None = None + ) -> None: + execute_serve_cmd(args, unknown_args) + + def validate(self, args: argparse.Namespace) -> None: + """Validate the arguments for this command""" + if args.config and not os.path.exists(args.config): + raise ValueError(f"Config file not found: {args.config}") + + def subparser_init( + self, subparsers: argparse._SubParsersAction + ) -> FlexibleArgumentParser: + serve_parser = subparsers.add_parser( + "serve", + help="Launch the server and start FastAPI listener.", + usage="sgl_diffusion serve --model-path MODEL_PATH_OR_ID [OPTIONS]", + ) + + serve_parser = add_multimodal_gen_serve_args(serve_parser) + + return cast(FlexibleArgumentParser, serve_parser) + + +def cmd_init() -> list[CLISubcommand]: + return [ServeSubcommand()] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/utils.py new file mode 100644 index 00000000..a4fc7527 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/cli/utils.py @@ -0,0 +1,74 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import os +import subprocess +import sys + +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class RaiseNotImplementedAction(argparse.Action): + + def __call__(self, parser, namespace, values, option_string=None): + raise NotImplementedError(f"The {option_string} option is not yet implemented") + + +def launch_distributed( + num_gpus: int, args: list[str], master_port: int | None = None +) -> int: + """ + Launch a distributed job with the given arguments + + Args: + num_gpus: Number of GPUs to use + args: Arguments to pass to v1_sgl_diffusion_inference.py (defaults to sys.argv[1:]) + master_port: Port for the master process (default: random) + """ + + current_env = os.environ.copy() + python_executable = sys.executable + project_root = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../../..") + ) + main_script = os.path.join( + project_root, "sgl_diffusion/sample/v1_sgl_diffusion_inference.py" + ) + + cmd = [ + python_executable, + "-m", + "torch.distributed.run", + f"--nproc_per_node={num_gpus}", + ] + + if master_port is not None: + cmd.append(f"--master_port={master_port}") + + cmd.append(main_script) + cmd.extend(args) + + logger.info("Running inference with %d GPU(s)", num_gpus) + logger.info("Launching command: %s", " ".join(cmd)) + + current_env["PYTHONIOENCODING"] = "utf-8" + process = subprocess.Popen( + cmd, + env=current_env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + bufsize=1, + encoding="utf-8", + errors="replace", + ) + + if process.stdout: + for line in iter(process.stdout.readline, ""): + print(line.strip()) + + return process.wait() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/diffusion_generator.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/diffusion_generator.py new file mode 100644 index 00000000..d296450f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/diffusion_generator.py @@ -0,0 +1,459 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +DiffGenerator module for sglang-diffusion. + +This module provides a consolidated interface for generating videos using +diffusion models. +""" + +import multiprocessing as mp +import os +import time +from typing import Any + +import imageio +import numpy as np +import torch +import torchvision +from einops import rearrange + +from sglang.multimodal_gen.configs.sample.sampling_params import ( + DataType, + SamplingParams, +) +from sglang.multimodal_gen.runtime.entrypoints.openai.utils import ( + MergeLoraWeightsReq, + SetLoraReq, + UnmergeLoraWeightsReq, +) +from sglang.multimodal_gen.runtime.entrypoints.utils import prepare_request +from sglang.multimodal_gen.runtime.launch_server import launch_server +from sglang.multimodal_gen.runtime.pipelines_core import Req +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch +from sglang.multimodal_gen.runtime.server_args import PortArgs, ServerArgs +from sglang.multimodal_gen.runtime.sync_scheduler_client import sync_scheduler_client +from sglang.multimodal_gen.runtime.utils.logging_utils import ( + init_logger, + log_batch_completion, + log_generation_timer, + suppress_loggers, + suppress_other_loggers, +) + +suppress_loggers(["imageio", "imageio_ffmpeg", "PIL", "PIL_Image"]) + +logger = init_logger(__name__) + +# TODO: move to somewhere appropriate +try: + # Set the start method to 'spawn' to avoid CUDA errors in forked processes. + # This must be done at the top level of the module, before any CUDA context + # or other processes are initialized. + mp.set_start_method("spawn", force=True) +except RuntimeError: + # The start method can only be set once per program execution. + pass + + +# TODO: rename +class DiffGenerator: + """ + A unified class for generating images/videos using diffusion models. + + This class provides a simple interface for image/video generation with rich + customization options, similar to popular frameworks like HF Diffusers. + """ + + def __init__( + self, + server_args: ServerArgs, + ): + """ + Initialize the generator. + + Args: + server_args: The inference arguments + """ + self.server_args = server_args + self.port_args = PortArgs.from_server_args(server_args) + + # The executor is now a client to the Scheduler service + self.local_scheduler_process: list[mp.Process] | None = None + self.owns_scheduler_client: bool = False + self._current_lora_path: str | None = None + self._current_lora_nickname: str | None = None + self._is_lora_merged: bool = False + + @classmethod + def from_pretrained( + cls, + **kwargs, + ) -> "DiffGenerator": + """ + Create a DiffGenerator from a pretrained model. + + Args: + **kwargs: Additional arguments to customize model loading, set any ServerArgs or PipelineConfig attributes here. + + Returns: + The created DiffGenerator + + Priority level: Default pipeline config < User's pipeline config < User's kwargs + """ + # If users also provide some kwargs, it will override the ServerArgs and PipelineConfig. + + if (server_args := kwargs.get("server_args", None)) is not None: + if isinstance(server_args, ServerArgs): + pass + elif isinstance(server_args, dict): + server_args = ServerArgs.from_kwargs(**server_args) + else: + server_args = ServerArgs.from_kwargs(**kwargs) + + return cls.from_server_args(server_args) + + @classmethod + def from_server_args(cls, server_args: ServerArgs) -> "DiffGenerator": + """ + Create a DiffGenerator with the specified arguments. + + Args: + server_args: The inference arguments + + Returns: + The created DiffGenerator + """ + instance = cls( + server_args=server_args, + ) + is_local_mode = server_args.is_local_mode + logger.info(f"Local mode: {is_local_mode}") + if is_local_mode: + instance.local_scheduler_process = instance._start_local_server_if_needed() + else: + # In remote mode, we just need to connect and check. + sync_scheduler_client.initialize(server_args) + instance._check_remote_scheduler() + + # In both modes, this DiffGenerator instance is responsible for the client's lifecycle. + instance.owns_scheduler_client = True + return instance + + def _start_local_server_if_needed( + self, + ) -> list[mp.Process]: + """Check if a local server is running; if not, start it and return the process handles.""" + # First, we need a client to test the server. Initialize it temporarily. + sync_scheduler_client.initialize(self.server_args) + + processes = launch_server(self.server_args, launch_http_server=False) + + return processes + + def _check_remote_scheduler(self): + """Check if the remote scheduler is accessible.""" + if not sync_scheduler_client.ping(): + raise ConnectionError( + f"Could not connect to remote scheduler at " + f"{self.server_args.scheduler_endpoint()} with `local mode` as False. " + "Please ensure the server is running." + ) + logger.info( + f"Successfully connected to remote scheduler at " + f"{self.server_args.scheduler_endpoint()}." + ) + + def post_process_sample( + self, + sample: torch.Tensor, + data_type: DataType, + fps: int, + save_output: bool = True, + save_file_path: str = None, + ): + """ + Process a single sample output and save output if necessary + """ + # Process outputs + if sample.dim() == 3: + # for images, dim t is missing + sample = sample.unsqueeze(1) + sample = rearrange(sample, "c t h w -> t c h w") + frames = [] + # TODO: this can be batched + for x in sample: + x = torchvision.utils.make_grid(x, nrow=6) + x = x.transpose(0, 1).transpose(1, 2).squeeze(-1) + frames.append((x * 255).numpy().astype(np.uint8)) + + # Save outputs if requested + if save_output: + if save_file_path: + os.makedirs(os.path.dirname(save_file_path), exist_ok=True) + with suppress_other_loggers(): + if data_type == DataType.VIDEO: + imageio.mimsave( + save_file_path, + frames, + fps=fps, + format=data_type.get_default_extension(), + ) + else: + imageio.imwrite(save_file_path, frames[0]) + logger.info("Saved output to %s", save_file_path) + else: + logger.warning("No output path provided, output not saved") + + return frames + + def generate( + self, + sampling_params_kwargs: dict | None = None, + ) -> dict[str, Any] | list[np.ndarray] | list[dict[str, Any]] | None: + """ + Generate a image/video based on the given prompt. + + Args: + + Returns: + Either the output dictionary, list of frames, or list of results for batch processing + """ + # 1. prepare requests + prompt = sampling_params_kwargs.get("prompt", None) + prompts: list[str] = [] + # Handle batch processing from text file + if self.server_args.prompt_file_path is not None: + prompt_txt_path = self.server_args.prompt_file_path + if not os.path.exists(prompt_txt_path): + raise FileNotFoundError( + f"Prompt text file not found: {prompt_txt_path}" + ) + # Read prompts from file + with open(prompt_txt_path, encoding="utf-8") as f: + prompts.extend(line.strip() for line in f if line.strip()) + + if not prompts: + raise ValueError(f"No prompts found in file: {prompt_txt_path}") + + logger.info("Found %d prompts in %s", len(prompts), prompt_txt_path) + elif prompt is not None: + if isinstance(prompt, str): + prompts.append(prompt) + elif isinstance(prompt, list): + prompts.extend(prompt) + else: + raise ValueError("Either prompt or prompt_txt must be provided") + + sampling_params = SamplingParams.from_user_sampling_params_args( + self.server_args.model_path, + server_args=self.server_args, + **sampling_params_kwargs, + ) + + requests: list[Req] = [] + for output_idx, p in enumerate(prompts): + sampling_params.prompt = p + requests.append( + prepare_request( + server_args=self.server_args, + sampling_params=sampling_params, + ) + ) + + results = [] + total_start_time = time.perf_counter() + # 2. send requests to scheduler, one at a time + # TODO: send batch when supported + for request_idx, req in enumerate(requests): + try: + with log_generation_timer( + logger, req.prompt, request_idx + 1, len(requests) + ) as timer: + output_batch = self._send_to_scheduler_and_wait_for_response([req]) + if output_batch.error: + raise Exception(f"{output_batch.error}") + + if output_batch.output is None: + logger.error( + "Received empty output from scheduler for prompt %d", + request_idx + 1, + ) + continue + for output_idx, sample in enumerate(output_batch.output): + num_outputs = len(output_batch.output) + frames = self.post_process_sample( + sample, + fps=req.fps, + save_output=req.save_output, + save_file_path=req.output_file_path( + num_outputs, output_idx + ), + data_type=req.data_type, + ) + + result_item: dict[str, Any] = { + "samples": sample, + "frames": frames, + "prompts": req.prompt, + "size": (req.height, req.width, req.num_frames), + "generation_time": timer.duration, + "timings": ( + output_batch.timings.to_dict() + if output_batch.timings + else {} + ), + "trajectory": output_batch.trajectory_latents, + "trajectory_timesteps": output_batch.trajectory_timesteps, + "trajectory_decoded": output_batch.trajectory_decoded, + "prompt_index": output_idx, + } + results.append(result_item) + except Exception: + continue + + total_gen_time = time.perf_counter() - total_start_time + log_batch_completion(logger, len(results), total_gen_time) + + if len(results) == 0: + return None + else: + if requests[0].return_frames: + results = [r["frames"] for r in results] + if len(results) == 1: + return results[0] + return results + + def _send_to_scheduler_and_wait_for_response(self, batch: list[Req]) -> OutputBatch: + """ + Sends a request to the scheduler and waits for a response. + """ + return sync_scheduler_client.forward(batch) + + # LoRA + def _send_lora_request(self, req: Any, success_msg: str, failure_msg: str): + response = sync_scheduler_client.forward(req) + if isinstance(response, dict) and response.get("status") == "ok": + logger.info(success_msg) + else: + error_msg = ( + response.get("message", "Unknown error") + if isinstance(response, dict) + else "Unknown response format" + ) + raise RuntimeError(f"{failure_msg}: {error_msg}") + + def set_lora(self, lora_nickname: str, lora_path: str | None = None) -> None: + req = SetLoraReq(lora_nickname=lora_nickname, lora_path=lora_path) + self._send_lora_request( + req, + f"Successfully set LoRA adapter: {lora_nickname}", + "Failed to set LoRA adapter", + ) + + def unmerge_lora_weights(self) -> None: + req = UnmergeLoraWeightsReq() + self._send_lora_request( + req, + "Successfully unmerged LoRA weights", + "Failed to unmerge LoRA weights", + ) + self._is_lora_merged = False + + def merge_lora_weights(self) -> None: + req = MergeLoraWeightsReq() + self._send_lora_request( + req, "Successfully merged LoRA weights", "Failed to merge LoRA weights" + ) + self._is_lora_merged = True + + def _ensure_lora_state( + self, + lora_path: str | None, + lora_nickname: str | None = None, + merge_lora: bool = True, + ) -> None: + if lora_path is None: + if self._is_lora_merged: + self.unmerge_lora_weights() + self._current_lora_path = None + self._current_lora_nickname = None + self._is_lora_merged = False + return + + lora_nickname = lora_nickname or self.server_args.lora_nickname + + if self._current_lora_path != lora_path: + if self._is_lora_merged: + self.unmerge_lora_weights() + self._is_lora_merged = False + self.set_lora(lora_nickname, lora_path) + self._current_lora_path = lora_path + self._current_lora_nickname = lora_nickname + self._is_lora_merged = False + + if merge_lora and not self._is_lora_merged: + self.merge_lora_weights() + elif not merge_lora: + self._is_lora_merged = False + + def generate_with_lora( + self, + prompt: str | list[str] | None = None, + sampling_params: SamplingParams | None = None, + *, + lora_path: str | None = None, + lora_nickname: str | None = None, + merge_lora: bool = True, + **kwargs, + ): + self._ensure_lora_state( + lora_path=lora_path, lora_nickname=lora_nickname, merge_lora=merge_lora + ) + return self.generate( + prompt=prompt, + sampling_params=sampling_params, + **kwargs, + ) + + def shutdown(self): + """ + Shutdown the generator. + If in local mode, it also shuts down the scheduler server. + """ + # sends the shutdown command to the server + if self.local_scheduler_process: + logger.info("Waiting for local worker processes to terminate...") + for process in self.local_scheduler_process: + process.join(timeout=10) + if process.is_alive(): + logger.warning( + f"Local worker {process.name} did not terminate gracefully, forcing." + ) + process.terminate() + self.local_scheduler_process = None + + if self.owns_scheduler_client: + sync_scheduler_client.close() + self.owns_scheduler_client = False + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.shutdown() + + def __del__(self): + if self.owns_scheduler_client: + logger.warning( + "Generator was garbage collected without being shut down. " + "Attempting to shut down the local server and client." + ) + self.shutdown() + elif self.local_scheduler_process: + logger.warning( + "Generator was garbage collected without being shut down. " + "Attempting to shut down the local server." + ) + self.shutdown() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py new file mode 100644 index 00000000..489a08e2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py @@ -0,0 +1,80 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +import asyncio +from contextlib import asynccontextmanager + +from fastapi import APIRouter, FastAPI + +from sglang.multimodal_gen.runtime.entrypoints.openai import image_api, video_api +from sglang.multimodal_gen.runtime.server_args import ServerArgs, prepare_server_args +from sglang.multimodal_gen.runtime.utils.logging_utils import configure_logger + + +@asynccontextmanager +async def lifespan(app: FastAPI): + from sglang.multimodal_gen.runtime.scheduler_client import ( + run_zeromq_broker, + scheduler_client, + ) + + # 1. Initialize the singleton client that connects to the backend Scheduler + server_args = app.state.server_args + scheduler_client.initialize(server_args) + + # 2. Start the ZMQ Broker in the background to handle offline requests + broker_task = asyncio.create_task(run_zeromq_broker(server_args)) + + yield + + # On shutdown + print("FastAPI app is shutting down...") + broker_task.cancel() + scheduler_client.close() + + +# Health router +health_router = APIRouter() + + +@health_router.get("/health") +async def health(): + return {"status": "ok"} + + +@health_router.get("/health_generate") +async def health_generate(): + # TODO : health generate endpoint + return {"status": "ok"} + + +def create_app(server_args: ServerArgs): + """ + Create and configure the FastAPI application instance. + """ + app = FastAPI(lifespan=lifespan) + + app.include_router(health_router) + + from sglang.multimodal_gen.runtime.entrypoints.openai import common_api + + app.include_router(common_api.router) + app.include_router(image_api.router) + app.include_router(video_api.router) + + app.state.server_args = server_args + return app + + +if __name__ == "__main__": + import uvicorn + + server_args = prepare_server_args([]) + configure_logger(server_args) + app = create_app(server_args) + uvicorn.run( + app, + host=server_args.host, + port=server_args.port, + log_config=None, + reload=False, # Set to True during development for auto-reloading + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/common_api.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/common_api.py new file mode 100644 index 00000000..2dcbeb77 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/common_api.py @@ -0,0 +1,62 @@ +from typing import Any, Optional + +from fastapi import APIRouter, Body, HTTPException + +from sglang.multimodal_gen.runtime.entrypoints.openai.utils import ( + MergeLoraWeightsReq, + SetLoraReq, + UnmergeLoraWeightsReq, +) +from sglang.multimodal_gen.runtime.scheduler_client import scheduler_client +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +router = APIRouter(prefix="/v1") +logger = init_logger(__name__) + + +async def _handle_lora_request(req: Any, success_msg: str, failure_msg: str): + try: + response = await scheduler_client.forward(req) + if isinstance(response, dict) and response.get("status") == "ok": + return {"status": "ok", "message": success_msg} + else: + error_msg = ( + response.get("message", "Unknown error") + if isinstance(response, dict) + else "Unknown response format" + ) + raise HTTPException(status_code=500, detail=f"{failure_msg}: {error_msg}") + except Exception as e: + if isinstance(e, HTTPException): + raise + logger.error(f"Error during '{failure_msg}': {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post("/set_lora") +async def set_lora( + lora_nickname: str = Body(..., embed=True), + lora_path: Optional[str] = Body(None, embed=True), +): + req = SetLoraReq(lora_nickname=lora_nickname, lora_path=lora_path) + return await _handle_lora_request( + req, + f"Successfully set LoRA adapter: {lora_nickname}", + "Failed to set LoRA adapter", + ) + + +@router.post("/merge_lora_weights") +async def merge_lora_weights(): + req = MergeLoraWeightsReq() + return await _handle_lora_request( + req, "Successfully merged LoRA weights", "Failed to merge LoRA weights" + ) + + +@router.post("/unmerge_lora_weights") +async def unmerge_lora_weights(): + req = UnmergeLoraWeightsReq() + return await _handle_lora_request( + req, "Successfully unmerged LoRA weights", "Failed to unmerge LoRA weights" + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/image_api.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/image_api.py new file mode 100644 index 00000000..7ac86c2c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/image_api.py @@ -0,0 +1,228 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +import base64 +import os +import time +from typing import List, Optional + +from fastapi import APIRouter, File, Form, HTTPException, Path, Query, UploadFile +from fastapi.responses import FileResponse + +from sglang.multimodal_gen.configs.sample.sampling_params import ( + SamplingParams, + generate_request_id, +) +from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import ( + ImageGenerationsRequest, + ImageResponse, + ImageResponseData, +) +from sglang.multimodal_gen.runtime.entrypoints.openai.stores import IMAGE_STORE +from sglang.multimodal_gen.runtime.entrypoints.openai.utils import ( + _parse_size, + _save_upload_to_path, + process_generation_batch, +) +from sglang.multimodal_gen.runtime.entrypoints.utils import prepare_request +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.scheduler_client import scheduler_client +from sglang.multimodal_gen.runtime.server_args import get_global_server_args +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +router = APIRouter(prefix="/v1/images", tags=["images"]) +logger = init_logger(__name__) + + +def _choose_ext(output_format: Optional[str], background: Optional[str]) -> str: + # Normalize and choose extension + fmt = (output_format or "").lower() + if fmt in {"png", "webp", "jpeg", "jpg"}: + return "jpg" if fmt == "jpeg" else fmt + # If transparency requested, prefer png + if (background or "auto").lower() == "transparent": + return "png" + # Default + return "jpg" + + +def _build_sampling_params_from_request( + request_id: str, + prompt: str, + n: int, + size: Optional[str], + output_format: Optional[str], + background: Optional[str], + image_path: Optional[str] = None, +) -> SamplingParams: + if size is None: + width, height = None, None + else: + width, height = _parse_size(size) + ext = _choose_ext(output_format, background) + server_args = get_global_server_args() + # Build user params + sampling_params = SamplingParams.from_user_sampling_params_args( + model_path=server_args.model_path, + request_id=request_id, + prompt=prompt, + image_path=image_path, + num_frames=1, # image + width=width, + height=height, + num_outputs_per_prompt=max(1, min(int(n or 1), 10)), + save_output=True, + server_args=server_args, + output_file_name=f"{request_id}.{ext}", + ) + return sampling_params + + +def _build_req_from_sampling(s: SamplingParams) -> Req: + return Req( + request_id=s.request_id, + data_type=s.data_type, + prompt=s.prompt, + image_path=s.image_path, + height=s.height, + width=s.width, + fps=1, + num_frames=s.num_frames, + seed=s.seed, + output_path=s.output_path, + output_file_name=s.output_file_name, + num_outputs_per_prompt=s.num_outputs_per_prompt, + save_output=s.save_output, + ) + + +@router.post("/generations", response_model=ImageResponse) +async def generations( + request: ImageGenerationsRequest, +): + request_id = generate_request_id() + sampling = _build_sampling_params_from_request( + request_id=request_id, + prompt=request.prompt, + n=request.n or 1, + size=request.size, + output_format=request.output_format, + background=request.background, + ) + batch = prepare_request( + server_args=get_global_server_args(), + sampling_params=sampling, + ) + # Run synchronously for images and save to disk + save_file_path = await process_generation_batch(scheduler_client, batch) + + await IMAGE_STORE.upsert( + request_id, + { + "id": request_id, + "created_at": int(time.time()), + "file_path": save_file_path, + }, + ) + + resp_format = (request.response_format or "b64_json").lower() + if resp_format == "b64_json": + with open(save_file_path, "rb") as f: + b64 = base64.b64encode(f.read()).decode("utf-8") + return ImageResponse( + data=[ + ImageResponseData( + b64_json=b64, + revised_prompt=request.prompt, + ) + ] + ) + else: + # Return error, not supported + raise HTTPException( + status_code=400, detail="response_format=url is not supported" + ) + + +@router.post("/edits", response_model=ImageResponse) +async def edits( + image: Optional[List[UploadFile]] = File(None), + image_array: Optional[List[UploadFile]] = File(None, alias="image[]"), + prompt: str = Form(...), + mask: Optional[UploadFile] = File(None), + model: Optional[str] = Form(None), + n: Optional[int] = Form(1), + response_format: Optional[str] = Form(None), + size: Optional[str] = Form(None), + output_format: Optional[str] = Form(None), + background: Optional[str] = Form("auto"), + user: Optional[str] = Form(None), +): + request_id = generate_request_id() + # Resolve images from either `image` or `image[]` (OpenAI SDK sends `image[]` when list is provided) + images = image or image_array + if not images or len(images) == 0: + raise HTTPException(status_code=422, detail="Field 'image' is required") + + # Save first input image; additional images or mask are not yet used by the pipeline + uploads_dir = os.path.join("outputs", "uploads") + os.makedirs(uploads_dir, exist_ok=True) + first_image = images[0] + input_path = os.path.join(uploads_dir, f"{request_id}_{first_image.filename}") + await _save_upload_to_path(first_image, input_path) + + sampling = _build_sampling_params_from_request( + request_id=request_id, + prompt=prompt, + n=n or 1, + size=size, + output_format=output_format, + background=background, + image_path=input_path, + ) + batch = _build_req_from_sampling(sampling) + + save_file_path = await process_generation_batch(scheduler_client, batch) + + await IMAGE_STORE.upsert( + request_id, + { + "id": request_id, + "created_at": int(time.time()), + "file_path": save_file_path, + }, + ) + + # Default to b64_json to align with gpt-image-1 behavior in OpenAI examples + if (response_format or "b64_json").lower() == "b64_json": + with open(save_file_path, "rb") as f: + b64 = base64.b64encode(f.read()).decode("utf-8") + return ImageResponse( + data=[ImageResponseData(b64_json=b64, revised_prompt=prompt)] + ) + else: + url = f"/v1/images/{request_id}/content" + return ImageResponse(data=[ImageResponseData(url=url, revised_prompt=prompt)]) + + +@router.get("/{image_id}/content") +async def download_image_content( + image_id: str = Path(...), variant: Optional[str] = Query(None) +): + item = await IMAGE_STORE.get(image_id) + if not item: + raise HTTPException(status_code=404, detail="Image not found") + + file_path = item.get("file_path") + if not file_path or not os.path.exists(file_path): + raise HTTPException(status_code=404, detail="Image is still being generated") + + ext = os.path.splitext(file_path)[1].lower() + media_type = "image/jpeg" + if ext == ".png": + media_type = "image/png" + elif ext == ".webp": + media_type = "image/webp" + + return FileResponse( + path=file_path, media_type=media_type, filename=os.path.basename(file_path) + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py new file mode 100644 index 00000000..00800ab1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py @@ -0,0 +1,65 @@ +import time +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, Field + + +# Image API protocol models +class ImageResponseData(BaseModel): + b64_json: Optional[str] = None + url: Optional[str] = None + revised_prompt: Optional[str] = None + + +class ImageResponse(BaseModel): + created: int = Field(default_factory=lambda: int(time.time())) + data: List[ImageResponseData] + + +class ImageGenerationsRequest(BaseModel): + prompt: str + model: Optional[str] = None + n: Optional[int] = 1 + quality: Optional[str] = "auto" + response_format: Optional[str] = "url" # url | b64_json + size: Optional[str] = "1024x1024" # e.g., 1024x1024 + style: Optional[str] = "vivid" + background: Optional[str] = "auto" # transparent | opaque | auto + output_format: Optional[str] = None # png | jpeg | webp + user: Optional[str] = None + + +# Video API protocol models +class VideoResponse(BaseModel): + id: str + object: str = "video" + model: str = "sora-2" + status: str = "queued" + progress: int = 0 + created_at: int = Field(default_factory=lambda: int(time.time())) + size: str = "720x1280" + seconds: str = "4" + quality: str = "standard" + remixed_from_video_id: Optional[str] = None + completed_at: Optional[int] = None + expires_at: Optional[int] = None + error: Optional[Dict[str, Any]] = None + + +class VideoGenerationsRequest(BaseModel): + prompt: str + input_reference: Optional[str] = None + model: Optional[str] = None + seconds: Optional[int] = 4 + size: Optional[str] = "720x1280" + fps: Optional[int] = None + num_frames: Optional[int] = None + + +class VideoListResponse(BaseModel): + data: List[VideoResponse] + object: str = "list" + + +class VideoRemixRequest(BaseModel): + prompt: str diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/stores.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/stores.py new file mode 100644 index 00000000..f924de81 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/stores.py @@ -0,0 +1,46 @@ +import asyncio +from typing import Any, Dict, List, Optional + + +class AsyncDictStore: + """A small async-safe in-memory key-value store for dict items. + + This encapsulates the usual pattern of a module-level dict guarded by + an asyncio.Lock and provides simple CRUD methods that are safe to call + concurrently from FastAPI request handlers and background tasks. + """ + + def __init__(self) -> None: + self._items: Dict[str, Dict[str, Any]] = {} + self._lock = asyncio.Lock() + + async def upsert(self, key: str, value: Dict[str, Any]) -> None: + async with self._lock: + self._items[key] = value + + async def update_fields( + self, key: str, updates: Dict[str, Any] + ) -> Optional[Dict[str, Any]]: + async with self._lock: + item = self._items.get(key) + if item is None: + return None + item.update(updates) + return item + + async def get(self, key: str) -> Optional[Dict[str, Any]]: + async with self._lock: + return self._items.get(key) + + async def pop(self, key: str) -> Optional[Dict[str, Any]]: + async with self._lock: + return self._items.pop(key, None) + + async def list_values(self) -> List[Dict[str, Any]]: + async with self._lock: + return list(self._items.values()) + + +# Global stores shared by OpenAI entrypoints +VIDEO_STORE = AsyncDictStore() +IMAGE_STORE = AsyncDictStore() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/utils.py new file mode 100644 index 00000000..ec7a3cb1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/utils.py @@ -0,0 +1,125 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo +import dataclasses +import os +import time +from typing import Optional + +import imageio +import numpy as np +import torch +import torchvision +from einops import rearrange +from fastapi import UploadFile + +from sglang.multimodal_gen.configs.sample.sampling_params import DataType +from sglang.multimodal_gen.runtime.utils.logging_utils import ( + init_logger, + log_batch_completion, + log_generation_timer, +) + +logger = init_logger(__name__) + + +@dataclasses.dataclass +class SetLoraReq: + lora_nickname: str + lora_path: Optional[str] = None + + +@dataclasses.dataclass +class MergeLoraWeightsReq: + pass + + +@dataclasses.dataclass +class UnmergeLoraWeightsReq: + pass + + +def post_process_sample( + sample: torch.Tensor, + data_type: DataType, + fps: int, + save_output: bool = True, + save_file_path: str = None, +): + """ + Process sample output and save video if necessary + """ + # Process outputs + if sample.dim() == 3: + # for images, dim t is missing + sample = sample.unsqueeze(1) + videos = rearrange(sample, "c t h w -> t c h w") + frames = [] + for x in videos: + x = torchvision.utils.make_grid(x, nrow=6) + x = x.transpose(0, 1).transpose(1, 2).squeeze(-1) + frames.append((x * 255).numpy().astype(np.uint8)) + + # Save outputs if requested + if save_output: + if save_file_path: + os.makedirs(os.path.dirname(save_file_path), exist_ok=True) + if data_type == DataType.VIDEO: + imageio.mimsave( + save_file_path, + frames, + fps=fps, + format=data_type.get_default_extension(), + ) + else: + imageio.imwrite(save_file_path, frames[0]) + logger.info(f"Saved output to {save_file_path}") + else: + logger.info(f"No output path provided, output not saved") + + return frames + + +def _parse_size(size: str) -> tuple[int, int]: + try: + parts = size.lower().replace(" ", "").split("x") + if len(parts) != 2: + raise ValueError + w, h = int(parts[0]), int(parts[1]) + return w, h + except Exception: + # Fallback to default portrait 720x1280 + return 720, 1280 + + +# Helpers +async def _save_upload_to_path(upload: UploadFile, target_path: str) -> str: + os.makedirs(os.path.dirname(target_path), exist_ok=True) + content = await upload.read() + with open(target_path, "wb") as f: + f.write(content) + return target_path + + +async def process_generation_batch( + scheduler_client, + batch, +): + total_start_time = time.perf_counter() + with log_generation_timer(logger, batch.prompt): + result = await scheduler_client.forward([batch]) + + if result.output is None: + raise RuntimeError("Model generation returned no output.") + + save_file_path = str(os.path.join(batch.output_path, batch.output_file_name)) + post_process_sample( + result.output[0], + batch.data_type, + batch.fps, + batch.save_output, + save_file_path, + ) + + total_time = time.perf_counter() - total_start_time + log_batch_completion(logger, 1, total_time) + + return save_file_path diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py new file mode 100644 index 00000000..c346fc52 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py @@ -0,0 +1,265 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +import asyncio +import json +import os +import time +from typing import Any, Dict, Optional + +from fastapi import ( + APIRouter, + File, + Form, + HTTPException, + Path, + Query, + Request, + UploadFile, +) +from fastapi.responses import FileResponse + +from sglang.multimodal_gen.configs.sample.sampling_params import ( + SamplingParams, + generate_request_id, +) +from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import ( + VideoGenerationsRequest, + VideoListResponse, + VideoResponse, +) +from sglang.multimodal_gen.runtime.entrypoints.openai.stores import VIDEO_STORE +from sglang.multimodal_gen.runtime.entrypoints.openai.utils import ( + _parse_size, + _save_upload_to_path, + process_generation_batch, +) +from sglang.multimodal_gen.runtime.entrypoints.utils import prepare_request +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.server_args import get_global_server_args +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) +router = APIRouter(prefix="/v1/videos", tags=["videos"]) + + +# NOTE(mick): the sampling params needs to be further adjusted +# FIXME: duplicated with the one in `image_api.py` +def _build_sampling_params_from_request( + request_id: str, request: VideoGenerationsRequest +) -> SamplingParams: + if request.size is None: + width, height = None, None + else: + width, height = _parse_size(request.size) + seconds = request.seconds if request.seconds is not None else 4 + # Prefer user-provided fps/num_frames from request; fallback to defaults + fps_default = 24 + fps = request.fps if request.fps is not None else fps_default + # If user provides num_frames, use it directly; otherwise derive from seconds * fps + derived_num_frames = fps * seconds + num_frames = ( + request.num_frames if request.num_frames is not None else derived_num_frames + ) + server_args = get_global_server_args() + sampling_params = SamplingParams.from_user_sampling_params_args( + model_path=server_args.model_path, + request_id=request_id, + prompt=request.prompt, + num_frames=num_frames, + fps=fps, + width=width, + height=height, + image_path=request.input_reference, + save_output=True, + server_args=server_args, + output_file_name=request_id, + ) + + return sampling_params + + +# extract metadata which http_server needs to know +def _video_job_from_sampling( + request_id: str, req: VideoGenerationsRequest, sampling: SamplingParams +) -> Dict[str, Any]: + size_str = f"{sampling.width}x{sampling.height}" + seconds = int(round((sampling.num_frames or 0) / float(sampling.fps or 24))) + return { + "id": request_id, + "object": "video", + "model": req.model or "sora-2", + "status": "queued", + "progress": 0, + "created_at": int(time.time()), + "size": size_str, + "seconds": str(seconds), + "quality": "standard", + "file_path": sampling.output_file_path(), + } + + +async def _dispatch_job_async(job_id: str, batch: Req) -> None: + from sglang.multimodal_gen.runtime.scheduler_client import scheduler_client + + try: + await process_generation_batch(scheduler_client, batch) + await VIDEO_STORE.update_fields( + job_id, + {"status": "completed", "progress": 100, "completed_at": int(time.time())}, + ) + except Exception as e: + logger.error(f"{e}") + await VIDEO_STORE.update_fields( + job_id, {"status": "failed", "error": {"message": str(e)}} + ) + + +# TODO: support image to video generation +@router.post("", response_model=VideoResponse) +async def create_video( + request: Request, + # multipart/form-data fields (optional; used only when content-type is multipart) + prompt: Optional[str] = Form(None), + input_reference: Optional[UploadFile] = File(None), + model: Optional[str] = Form(None), + seconds: Optional[int] = Form(None), + size: Optional[str] = Form(None), + fps: Optional[int] = Form(None), + num_frames: Optional[int] = Form(None), + extra_body: Optional[str] = Form(None), +): + content_type = request.headers.get("content-type", "").lower() + request_id = generate_request_id() + + if "multipart/form-data" in content_type: + if not prompt: + raise HTTPException(status_code=400, detail="prompt is required") + if input_reference is None: + raise HTTPException( + status_code=400, detail="input_reference file is required" + ) + + uploads_dir = os.path.join("outputs", "uploads") + os.makedirs(uploads_dir, exist_ok=True) + input_path = os.path.join( + uploads_dir, f"{request_id}_{input_reference.filename}" + ) + await _save_upload_to_path(input_reference, input_path) + + # Parse extra_body JSON (if provided in multipart form) to get fps/num_frames overrides + extra_from_form: Dict[str, Any] = {} + if extra_body: + try: + extra_from_form = json.loads(extra_body) + except Exception: + extra_from_form = {} + + fps_val = fps if fps is not None else extra_from_form.get("fps") + num_frames_val = ( + num_frames if num_frames is not None else extra_from_form.get("num_frames") + ) + + req = VideoGenerationsRequest( + prompt=prompt, + input_reference=input_path, + model=model, + seconds=seconds if seconds is not None else 4, + size=size or "720x1280", + fps=fps_val, + num_frames=num_frames_val, + ) + else: + try: + body = await request.json() + except Exception: + body = {} + try: + # If client uses extra_body, merge it into the top-level payload + payload: Dict[str, Any] = dict(body or {}) + extra = payload.pop("extra_body", None) + if isinstance(extra, dict): + # Shallow-merge: only keys like fps/num_frames are expected + payload.update(extra) + req = VideoGenerationsRequest(**payload) + except Exception as e: + raise HTTPException(status_code=400, detail=f"Invalid request body: {e}") + + logger.debug(f"Server received from create_video endpoint: req={req}") + + sampling_params = _build_sampling_params_from_request(request_id, req) + job = _video_job_from_sampling(request_id, req, sampling_params) + await VIDEO_STORE.upsert(request_id, job) + + # Build Req for scheduler + batch = prepare_request( + server_args=get_global_server_args(), + sampling_params=sampling_params, + ) + # Enqueue the job asynchronously and return immediately + asyncio.create_task(_dispatch_job_async(request_id, batch)) + return VideoResponse(**job) + + +@router.get("", response_model=VideoListResponse) +async def list_videos( + after: Optional[str] = Query(None), + limit: Optional[int] = Query(None, ge=1, le=100), + order: Optional[str] = Query("desc"), +): + # Normalize order + order = (order or "desc").lower() + if order not in ("asc", "desc"): + order = "desc" + jobs = await VIDEO_STORE.list_values() + + reverse = order != "asc" + jobs.sort(key=lambda j: j.get("created_at", 0), reverse=reverse) + + if after is not None: + try: + idx = next(i for i, j in enumerate(jobs) if j["id"] == after) + jobs = jobs[idx + 1 :] + except StopIteration: + jobs = [] + + if limit is not None: + jobs = jobs[:limit] + items = [VideoResponse(**j) for j in jobs] + return VideoListResponse(data=items) + + +@router.get("/{video_id}", response_model=VideoResponse) +async def retrieve_video(video_id: str = Path(...)): + job = await VIDEO_STORE.get(video_id) + if not job: + raise HTTPException(status_code=404, detail="Video not found") + return VideoResponse(**job) + + +# TODO: support aborting a job. +@router.delete("/{video_id}", response_model=VideoResponse) +async def delete_video(video_id: str = Path(...)): + job = await VIDEO_STORE.pop(video_id) + if not job: + raise HTTPException(status_code=404, detail="Video not found") + # Mark as deleted in response semantics + job["status"] = "deleted" + return VideoResponse(**job) + + +@router.get("/{video_id}/content") +async def download_video_content( + video_id: str = Path(...), variant: Optional[str] = Query(None) +): + job = await VIDEO_STORE.get(video_id) + if not job: + raise HTTPException(status_code=404, detail="Video not found") + + file_path = job.get("file_path") + if not file_path or not os.path.exists(file_path): + raise HTTPException(status_code=404, detail="Generation is still in-progress") + + media_type = "video/mp4" # default variant + return FileResponse( + path=file_path, media_type=media_type, filename=os.path.basename(file_path) + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/utils.py new file mode 100644 index 00000000..9cd56f1e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/entrypoints/utils.py @@ -0,0 +1,42 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +DiffGenerator module for sglang-diffusion. + +This module provides a consolidated interface for generating videos using +diffusion models. +""" + + +from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.utils import shallow_asdict + +logger = init_logger(__name__) + + +def prepare_request( + server_args: ServerArgs, + sampling_params: SamplingParams, +) -> Req: + """ + Settle SamplingParams according to ServerArgs + + """ + # Create a copy of inference args to avoid modifying the original + req = Req( + **shallow_asdict(sampling_params), + VSA_sparsity=server_args.VSA_sparsity, + ) + req.adjust_size(server_args) + + if req.width <= 0 or req.height <= 0: + raise ValueError( + f"Height, width must be positive integers, got " + f"height={req.height}, width={req.width}" + ) + + return req diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/launch_server.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/launch_server.py new file mode 100644 index 00000000..0f34166a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/launch_server.py @@ -0,0 +1,142 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +import multiprocessing as mp + +import uvicorn + +from sglang.multimodal_gen.runtime.entrypoints.http_server import create_app +from sglang.multimodal_gen.runtime.managers.gpu_worker import run_scheduler_process +from sglang.multimodal_gen.runtime.server_args import ServerArgs, set_global_server_args +from sglang.multimodal_gen.runtime.utils.logging_utils import ( + configure_logger, + logger, + suppress_other_loggers, +) + + +def launch_server(server_args: ServerArgs, launch_http_server: bool = True): + """ + Args: + launch_http_server: False for offline local mode + """ + configure_logger(server_args) + suppress_other_loggers() + + # Start a new server with multiple worker processes + logger.info("Starting server...") + + num_gpus = server_args.num_gpus + processes = [] + + # Pipes for master to talk to slaves + task_pipes_to_slaves_w = [] + task_pipes_to_slaves_r = [] + for _ in range(num_gpus - 1): + r, w = mp.Pipe(duplex=False) + task_pipes_to_slaves_r.append(r) + task_pipes_to_slaves_w.append(w) + + # Pipes for slaves to talk to master + result_pipes_from_slaves_w = [] + result_pipes_from_slaves_r = [] + for _ in range(num_gpus - 1): + r, w = mp.Pipe(duplex=False) + result_pipes_from_slaves_r.append(r) + result_pipes_from_slaves_w.append(w) + + # Launch all worker processes + master_port = server_args.master_port or (server_args.master_port + 100) + scheduler_pipe_readers = [] + scheduler_pipe_writers = [] + + for i in range(num_gpus): + reader, writer = mp.Pipe(duplex=False) + scheduler_pipe_writers.append(writer) + if i == 0: # Master worker + process = mp.Process( + target=run_scheduler_process, + args=( + i, # local_rank + i, # rank + master_port, + server_args, + writer, + None, # No task pipe to read from master + None, # No result pipe to write to master + task_pipes_to_slaves_w, + result_pipes_from_slaves_r, + ), + name=f"sglang-diffusionWorker-{i}", + daemon=True, + ) + else: # Slave workers + process = mp.Process( + target=run_scheduler_process, + args=( + i, # local_rank + i, # rank + master_port, + server_args, + writer, + None, # No task pipe to read from master + None, # No result pipe to write to master + task_pipes_to_slaves_r[i - 1], + result_pipes_from_slaves_w[i - 1], + ), + name=f"sglang-diffusionWorker-{i}", + daemon=True, + ) + scheduler_pipe_readers.append(reader) + process.start() + processes.append(process) + + # Wait for all workers to be ready + scheduler_infos = [] + for writer in scheduler_pipe_writers: + writer.close() + + # Close unused pipe ends in parent process + for p in task_pipes_to_slaves_w: + p.close() + for p in task_pipes_to_slaves_r: + p.close() + for p in result_pipes_from_slaves_w: + p.close() + for p in result_pipes_from_slaves_r: + p.close() + + for i, reader in enumerate(scheduler_pipe_readers): + try: + data = reader.recv() + except EOFError: + logger.error( + f"Rank {i} scheduler is dead. Please check if there are relevant logs." + ) + processes[i].join() + logger.error(f"Exit code: {processes[i].exitcode}") + raise + + if data["status"] != "ready": + raise RuntimeError( + "Initialization failed. Please see the error messages above." + ) + scheduler_infos.append(data) + reader.close() + + logger.debug("All workers are ready") + + if launch_http_server: + logger.info("Starting FastAPI server.") + + # set for endpoints to access global_server_args + set_global_server_args(server_args) + + app = create_app(server_args) + uvicorn.run( + app, + log_config=None, + log_level=server_args.log_level, + host=server_args.host, + port=server_args.port, + reload=False, + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/__init__.py new file mode 100644 index 00000000..af2eb7d1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/__init__.py @@ -0,0 +1 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/activation.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/activation.py new file mode 100644 index 00000000..4eff9ba1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/activation.py @@ -0,0 +1,129 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/model_executor/layers/activation.py +"""Custom activation functions.""" +import math +from typing import Any + +import torch +import torch.nn as nn +import torch.nn.functional as F + +# TODO (will): remove this dependency +from sglang.multimodal_gen.runtime.layers.custom_op import CustomOp + + +@CustomOp.register("silu_and_mul") +class SiluAndMul(CustomOp): + """An activation function for SwiGLU. + + The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[-1] // 2. + + Shapes: + x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) + return: (num_tokens, d) or (batch_size, seq_len, d) + """ + + def __init__(self) -> None: + super().__init__() + + def forward_cuda(self, *args, **kwargs) -> Any: + return self.forward_native(*args, **kwargs) + + def forward_native(self, x: torch.Tensor) -> torch.Tensor: + """PyTorch-native implementation equivalent to forward().""" + d = x.shape[-1] // 2 + return F.silu(x[..., :d]) * x[..., d:] + + +@CustomOp.register("gelu_and_mul") +class GeluAndMul(CustomOp): + """An activation function for GeGLU. + + The function computes x -> GELU(x[:d]) * x[d:] where d = x.shape[-1] // 2. + + Shapes: + x: (batch_size, seq_len, 2 * d) or (num_tokens, 2 * d) + return: (batch_size, seq_len, d) or (num_tokens, d) + """ + + def __init__(self, approximate: str = "none"): + super().__init__() + self.approximate = approximate + if approximate not in ("none", "tanh"): + raise ValueError(f"Unknown approximate mode: {approximate}") + + def forward_cuda(self, *args, **kwargs) -> Any: + return self.forward_native(*args, **kwargs) + + def forward_native(self, x: torch.Tensor) -> torch.Tensor: + """PyTorch-native implementation equivalent to forward().""" + d = x.shape[-1] // 2 + return F.gelu(x[..., :d], approximate=self.approximate) * x[..., d:] + + def extra_repr(self) -> str: + return f"approximate={repr(self.approximate)}" + + +@CustomOp.register("gelu_new") +class NewGELU(CustomOp): + + def __init__(self): + super().__init__() + + def forward_cuda(self, *args, **kwargs) -> Any: + return self.forward_native(*args, **kwargs) + + def forward_native(self, x: torch.Tensor) -> torch.Tensor: + """PyTorch-native implementation equivalent to forward().""" + c = math.sqrt(2.0 / math.pi) + return 0.5 * x * (1.0 + torch.tanh(c * (x + 0.044715 * torch.pow(x, 3.0)))) + + +@CustomOp.register("quick_gelu") +class QuickGELU(CustomOp): + # https://github.com/huggingface/transformers/blob/main/src/transformers/activations.py#L90 + def __init__(self): + super().__init__() + + def forward_cuda(self, *args, **kwargs) -> Any: + return self.forward_native(*args, **kwargs) + + def forward_native(self, x: torch.Tensor) -> torch.Tensor: + """PyTorch-native implementation equivalent to forward().""" + return x * torch.sigmoid(1.702 * x) + + +_ACTIVATION_REGISTRY = { + "gelu": nn.GELU, + "gelu_new": NewGELU, + "gelu_pytorch_tanh": lambda: nn.GELU(approximate="tanh"), + "relu": nn.ReLU, + "silu": nn.SiLU, + "quick_gelu": QuickGELU, +} + + +def get_act_fn(act_fn_name: str) -> nn.Module: + """Get an activation function by name.""" + act_fn_name = act_fn_name.lower() + if act_fn_name not in _ACTIVATION_REGISTRY: + raise ValueError(f"Activation function {act_fn_name!r} is not supported.") + + return _ACTIVATION_REGISTRY[act_fn_name]() + + +_ACTIVATION_AND_MUL_REGISTRY = { + "gelu": GeluAndMul, + "silu": SiluAndMul, +} + + +def get_act_and_mul_fn(act_fn_name: str) -> nn.Module: + """Get an activation-and-mul (i.e. SiluAndMul) function by name.""" + act_fn_name = act_fn_name.lower() + if act_fn_name not in _ACTIVATION_AND_MUL_REGISTRY: + raise ValueError(f"Activation function {act_fn_name!r} is not supported.") + + return _ACTIVATION_AND_MUL_REGISTRY[act_fn_name]() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/STA_configuration.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/STA_configuration.py new file mode 100644 index 00000000..9635a674 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/STA_configuration.py @@ -0,0 +1,414 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +import json +import os +from collections import defaultdict +from typing import Any + +import numpy as np + +from sglang.multimodal_gen.utils import dict_to_3d_list + + +def configure_sta( + mode: str = "STA_searching", + layer_num: int = 40, + time_step_num: int = 50, + head_num: int = 40, + **kwargs, +) -> list[list[list[Any]]]: + """ + Configure Sliding Tile Attention (STA) parameters based on the specified mode. + + Parameters: + ---------- + mode : str + The STA mode to use. Options are: + - 'STA_searching': Generate a set of mask candidates for initial search + - 'STA_tuning': Select best mask strategy based on previously saved results + - 'STA_inference': Load and use a previously tuned mask strategy + layer_num: int, number of layers + time_step_num: int, number of timesteps + head_num: int, number of heads + + **kwargs : dict + Mode-specific parameters: + + For 'STA_searching': + - mask_candidates: list of str, optional, mask candidates to use + - mask_selected: list of int, optional, indices of selected masks + + For 'STA_tuning': + - mask_search_files_path: str, required, path to mask search results + - mask_candidates: list of str, optional, mask candidates to use + - mask_selected: list of int, optional, indices of selected masks + - skip_time_steps: int, optional, number of time steps to use full attention (default 12) + - save_dir: str, optional, directory to save mask strategy (default "mask_candidates") + + For 'STA_inference': + - load_path: str, optional, path to load mask strategy (default "mask_candidates/mask_strategy.json") + """ + valid_modes = ["STA_searching", "STA_tuning", "STA_inference", "STA_tuning_cfg"] + if mode not in valid_modes: + raise ValueError(f"Mode must be one of {valid_modes}, got {mode}") + + if mode == "STA_searching": + # Get parameters with defaults + mask_candidates: list[str] | None = kwargs.get("mask_candidates") + if mask_candidates is None: + raise ValueError("mask_candidates is required for STA_searching mode") + mask_selected: list[int] = kwargs.get( + "mask_selected", list(range(len(mask_candidates))) + ) + + # Parse selected masks + selected_masks: list[list[int]] = [] + for index in mask_selected: + mask = mask_candidates[index] + masks_list = [int(x) for x in mask.split(",")] + selected_masks.append(masks_list) + + # Create 3D mask structure with fixed dimensions (t=50, l=60) + masks_3d: list[list[list[list[int]]]] = [] + for i in range(time_step_num): # Fixed t dimension = 50 + row = [] + for j in range(layer_num): # Fixed l dimension = 60 + row.append(selected_masks) # Add all masks at each position + masks_3d.append(row) + + return masks_3d + + elif mode == "STA_tuning": + # Get required parameters + mask_search_files_path: str | None = kwargs.get("mask_search_files_path") + if not mask_search_files_path: + raise ValueError("mask_search_files_path is required for STA_tuning mode") + + # Get optional parameters with defaults + mask_candidates_tuning: list[str] | None = kwargs.get("mask_candidates") + if mask_candidates_tuning is None: + raise ValueError("mask_candidates is required for STA_tuning mode") + mask_selected_tuning: list[int] = kwargs.get( + "mask_selected", list(range(len(mask_candidates_tuning))) + ) + skip_time_steps_tuning: int | None = kwargs.get("skip_time_steps") + save_dir_tuning: str | None = kwargs.get("save_dir", "mask_candidates") + + # Parse selected masks + selected_masks_tuning: list[list[int]] = [] + for index in mask_selected_tuning: + mask = mask_candidates_tuning[index] + masks_list = [int(x) for x in mask.split(",")] + selected_masks_tuning.append(masks_list) + + # Read JSON results + results = read_specific_json_files(mask_search_files_path) + averaged_results = average_head_losses(results, selected_masks_tuning) + + # Add full attention mask for specific cases + full_attention_mask_tuning: list[int] | None = kwargs.get("full_attention_mask") + if full_attention_mask_tuning is not None: + selected_masks_tuning.append(full_attention_mask_tuning) + + # Select best mask strategy + timesteps_tuning: int = kwargs.get("timesteps", time_step_num) + if skip_time_steps_tuning is None: + skip_time_steps_tuning = 12 + mask_strategy, sparsity, strategy_counts = select_best_mask_strategy( + averaged_results, + selected_masks_tuning, + skip_time_steps_tuning, + timesteps_tuning, + head_num, + ) + + # Save mask strategy + if save_dir_tuning is not None: + os.makedirs(save_dir_tuning, exist_ok=True) + file_path = os.path.join( + save_dir_tuning, f"mask_strategy_s{skip_time_steps_tuning}.json" + ) + with open(file_path, "w") as f: + json.dump(mask_strategy, f, indent=4) + print(f"Successfully saved mask_strategy to {file_path}") + + # Print sparsity and strategy counts for information + print(f"Overall sparsity: {sparsity:.4f}") + print("\nStrategy usage counts:") + total_heads = time_step_num * layer_num * head_num # Fixed dimensions + for strategy, count in strategy_counts.items(): + print(f"Strategy {strategy}: {count} heads ({count/total_heads*100:.2f}%)") + + # Convert dictionary to 3D list with fixed dimensions + mask_strategy_3d = dict_to_3d_list( + mask_strategy, t_max=time_step_num, l_max=layer_num, h_max=head_num + ) + + return mask_strategy_3d + elif mode == "STA_tuning_cfg": + # Get required parameters for both positive and negative paths + mask_search_files_path_pos: str | None = kwargs.get( + "mask_search_files_path_pos" + ) + mask_search_files_path_neg: str | None = kwargs.get( + "mask_search_files_path_neg" + ) + save_dir_cfg: str | None = kwargs.get("save_dir") + + if ( + not mask_search_files_path_pos + or not mask_search_files_path_neg + or not save_dir_cfg + ): + raise ValueError( + "mask_search_files_path_pos, mask_search_files_path_neg, and save_dir are required for STA_tuning_cfg mode" + ) + + # Get optional parameters with defaults + mask_candidates_cfg: list[str] | None = kwargs.get("mask_candidates") + if mask_candidates_cfg is None: + raise ValueError("mask_candidates is required for STA_tuning_cfg mode") + mask_selected_cfg: list[int] = kwargs.get( + "mask_selected", list(range(len(mask_candidates_cfg))) + ) + skip_time_steps_cfg: int | None = kwargs.get("skip_time_steps") + + # Parse selected masks + selected_masks_cfg: list[list[int]] = [] + for index in mask_selected_cfg: + mask = mask_candidates_cfg[index] + masks_list = [int(x) for x in mask.split(",")] + selected_masks_cfg.append(masks_list) + + # Read JSON results for both positive and negative paths + pos_results = read_specific_json_files(mask_search_files_path_pos) + neg_results = read_specific_json_files(mask_search_files_path_neg) + # Combine positive and negative results into one list + combined_results = pos_results + neg_results + + # Average the combined results + averaged_results = average_head_losses(combined_results, selected_masks_cfg) + + # Add full attention mask for specific cases + full_attention_mask_cfg: list[int] | None = kwargs.get("full_attention_mask") + if full_attention_mask_cfg is not None: + selected_masks_cfg.append(full_attention_mask_cfg) + + timesteps_cfg: int = kwargs.get("timesteps", time_step_num) + if skip_time_steps_cfg is None: + skip_time_steps_cfg = 12 + # Select best mask strategy using combined results + mask_strategy, sparsity, strategy_counts = select_best_mask_strategy( + averaged_results, + selected_masks_cfg, + skip_time_steps_cfg, + timesteps_cfg, + head_num, + ) + + # Save mask strategy + os.makedirs(save_dir_cfg, exist_ok=True) + file_path = os.path.join( + save_dir_cfg, f"mask_strategy_s{skip_time_steps_cfg}.json" + ) + with open(file_path, "w") as f: + json.dump(mask_strategy, f, indent=4) + print(f"Successfully saved mask_strategy to {file_path}") + + # Print sparsity and strategy counts for information + print(f"Overall sparsity: {sparsity:.4f}") + print("\nStrategy usage counts:") + total_heads = time_step_num * layer_num * head_num # Fixed dimensions + for strategy, count in strategy_counts.items(): + print(f"Strategy {strategy}: {count} heads ({count/total_heads*100:.2f}%)") + + # Convert dictionary to 3D list with fixed dimensions + mask_strategy_3d = dict_to_3d_list( + mask_strategy, t_max=time_step_num, l_max=layer_num, h_max=head_num + ) + + return mask_strategy_3d + + else: # STA_inference + # Get parameters with defaults + load_path: str | None = kwargs.get( + "load_path", "mask_candidates/mask_strategy.json" + ) + if load_path is None: + raise ValueError("load_path is required for STA_inference mode") + + # Load previously saved mask strategy + with open(load_path) as f: + mask_strategy = json.load(f) + + # Convert dictionary to 3D list with fixed dimensions + mask_strategy_3d = dict_to_3d_list( + mask_strategy, t_max=time_step_num, l_max=layer_num, h_max=head_num + ) + + return mask_strategy_3d + + +# Helper functions + + +def read_specific_json_files(folder_path: str) -> list[dict[str, Any]]: + """Read and parse JSON files containing mask search results.""" + json_contents: list[dict[str, Any]] = [] + + # List files only in the current directory (no walk) + files = os.listdir(folder_path) + # Filter files + matching_files = [f for f in files if "mask" in f and f.endswith(".json")] + print(f"Found {len(matching_files)} matching files: {matching_files}") + + for file_name in matching_files: + file_path = os.path.join(folder_path, file_name) + with open(file_path) as file: + data = json.load(file) + json_contents.append(data) + + return json_contents + + +def average_head_losses( + results: list[dict[str, Any]], selected_masks: list[list[int]] +) -> dict[str, dict[str, np.ndarray]]: + """Average losses across all prompts for each mask strategy.""" + # Initialize a dictionary to store the averaged results + averaged_losses: dict[str, dict[str, np.ndarray]] = {} + loss_type = "L2_loss" + # Get all loss types (e.g., 'L2_loss') + averaged_losses[loss_type] = {} + + for mask in selected_masks: + mask_str = str(mask) + data_shape = np.array(results[0][loss_type][mask_str]).shape + accumulated_data = np.zeros(data_shape) + + # Sum across all prompts + for prompt_result in results: + accumulated_data += np.array(prompt_result[loss_type][mask_str]) + + # Average by dividing by number of prompts + averaged_data = accumulated_data / len(results) + averaged_losses[loss_type][mask_str] = averaged_data + + return averaged_losses + + +def select_best_mask_strategy( + averaged_results: dict[str, dict[str, np.ndarray]], + selected_masks: list[list[int]], + skip_time_steps: int = 12, + timesteps: int = 50, + head_num: int = 40, +) -> tuple[dict[str, list[int]], float, dict[str, int]]: + """Select the best mask strategy for each head based on loss minimization.""" + best_mask_strategy: dict[str, list[int]] = {} + loss_type = "L2_loss" + # Get the shape of time steps and layers + layers = len(averaged_results[loss_type][str(selected_masks[0])][0]) + + # Counter for sparsity calculation + total_tokens = 0 # total number of masked tokens + total_length = 0 # total sequence length + + strategy_counts: dict[str, int] = {str(strategy): 0 for strategy in selected_masks} + full_attn_strategy = selected_masks[-1] # Last strategy is full attention + print(f"Strategy {full_attn_strategy}, skip first {skip_time_steps} steps ") + + for t in range(timesteps): + for layer_idx in range(layers): + for h in range(head_num): + if t < skip_time_steps: # First steps use full attention + strategy = full_attn_strategy + else: + # Get losses for this head across all strategies + head_losses = [] + for strategy in selected_masks[:-1]: # Exclude full attention + head_losses.append( + averaged_results[loss_type][str(strategy)][t][layer_idx][h] + ) + + # Find which strategy gives minimum loss + best_strategy_idx = np.argmin(head_losses) + strategy = selected_masks[best_strategy_idx] + + best_mask_strategy[f"{t}_{layer_idx}_{h}"] = strategy + + # Calculate sparsity + nums = strategy # strategy is already a list of numbers + total_tokens += ( + nums[0] * nums[1] * nums[2] + ) # masked tokens for chosen strategy + total_length += ( + full_attn_strategy[0] + * full_attn_strategy[1] + * full_attn_strategy[2] + ) + + # Count strategy usage + strategy_counts[str(strategy)] += 1 + + overall_sparsity = 1 - total_tokens / total_length + + return best_mask_strategy, overall_sparsity, strategy_counts + + +def save_mask_search_results( + mask_search_final_result: list[dict[str, list[float]]], + prompt: str, + mask_strategies: list[str], + output_dir: str = "output/mask_search_result/", +) -> str | None: + if not mask_search_final_result: + print("No mask search results to save") + return None + + # Create result dictionary with defaultdict for nested lists + mask_search_dict: dict[str, dict[str, list[list[float]]]] = { + "L2_loss": defaultdict(list), + "L1_loss": defaultdict(list), + } + + mask_selected = list(range(len(mask_strategies))) + selected_masks: list[list[int]] = [] + for index in mask_selected: + mask = mask_strategies[index] + masks_list = [int(x) for x in mask.split(",")] + selected_masks.append(masks_list) + + # Process each mask strategy + for i, mask_strategy in enumerate(selected_masks): + mask_strategy_str = str(mask_strategy) + # Process L2 loss + step_results: list[list[float]] = [] + for step_data in mask_search_final_result: + if isinstance(step_data, dict) and "L2_loss" in step_data: + layer_losses = [float(loss) for loss in step_data["L2_loss"]] + step_results.append(layer_losses) + mask_search_dict["L2_loss"][mask_strategy_str] = step_results + + step_results = [] + for step_data in mask_search_final_result: + if isinstance(step_data, dict) and "L1_loss" in step_data: + layer_losses = [float(loss) for loss in step_data["L1_loss"]] + step_results.append(layer_losses) + mask_search_dict["L1_loss"][mask_strategy_str] = step_results + + # Create the output directory if it doesn't exist + os.makedirs(output_dir, exist_ok=True) + + # Create a filename based on the first 20 characters of the prompt + filename = prompt[:50].replace(" ", "_") + filepath = os.path.join(output_dir, f"mask_search_{filename}.json") + + # Save the results to a JSON file + with open(filepath, "w") as f: + json.dump(mask_search_dict, f, indent=4) + + print(f"Successfully saved mask research results to {filepath}") + + return filepath diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/__init__.py new file mode 100644 index 00000000..1b40782b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/__init__.py @@ -0,0 +1,28 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import ( + AttentionBackend, + AttentionMetadata, + AttentionMetadataBuilder, +) +from sglang.multimodal_gen.runtime.layers.attention.layer import ( + LocalAttention, + UlyssesAttention, + UlyssesAttention_VSA, + USPAttention, +) +from sglang.multimodal_gen.runtime.layers.attention.selector import get_attn_backend + +__all__ = [ + "USPAttention", + "LocalAttention", + "UlyssesAttention", + "UlyssesAttention_VSA", + "AttentionBackend", + "AttentionMetadata", + "AttentionMetadataBuilder", + # "AttentionState", + "get_attn_backend", +] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/__init__.py new file mode 100644 index 00000000..af2eb7d1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/__init__.py @@ -0,0 +1 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/aiter.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/aiter.py new file mode 100644 index 00000000..b96aad6a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/aiter.py @@ -0,0 +1,101 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +import aiter +import torch + +from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import ( + AttentionBackend, + AttentionImpl, + AttentionMetadata, + AttentionMetadataBuilder, +) + + +class AITerBackend(AttentionBackend): + """ + Backend for AITemplate attention implementation. + """ + + @staticmethod + def get_name() -> str: + return "AITER" + + @staticmethod + def get_impl_cls() -> type["AITerImpl"]: + return AITerImpl + + @staticmethod + def get_metadata_cls() -> type["AttentionMetadata"]: + # AITer backend does not require special metadata. + return AttentionMetadata + + @staticmethod + def get_builder_cls() -> type["AttentionMetadataBuilder"]: + raise NotImplementedError("AITer backend does not have a metadata builder.") + + +class AITerImpl(AttentionImpl): + """ + Implementation of attention using AITemplate. + """ + + def __init__( + self, + num_heads: int, + head_size: int, + softmax_scale: float, + causal: bool = False, + num_kv_heads: int | None = None, + prefix: str = "", + dropout_p: float = 0.0, + **extra_impl_args, + ) -> None: + super().__init__( + num_heads=num_heads, + head_size=head_size, + softmax_scale=softmax_scale, + causal=causal, + num_kv_heads=num_kv_heads, + prefix=prefix, + **extra_impl_args, + ) + if num_kv_heads is not None and num_kv_heads != num_heads: + raise NotImplementedError( + "AITer backend does not support Grouped Query Attention yet." + ) + self.causal = causal + self.dropout_p = dropout_p + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: AttentionMetadata | None = None, + ) -> torch.Tensor: + """ + Performs attention using aiter.flash_attn_func. + + Args: + query: Query tensor of shape [batch_size, num_heads, seq_len, head_dim] + key: Key tensor of shape [batch_size, num_heads, seq_len, head_dim] + value: Value tensor of shape [batch_size, num_heads, seq_len, head_dim] + attn_metadata: Metadata for the attention operation (unused). + + Returns: + Output tensor of shape [batch_size, num_heads, seq_len, head_dim] + """ + # aiter.flash_attn_func expects tensors in [B, H, S, D] layout, + # which is what ring_attn provides. + output, _ = aiter.flash_attn_func( + query, + key, + value, + dropout_p=self.dropout_p, + causal=self.causal, + return_attn_probs=False, + return_lse=True, + ) + return output diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/attention_backend.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/attention_backend.py new file mode 100644 index 00000000..3463ef05 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/attention_backend.py @@ -0,0 +1,180 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/attention/backends/abstract.py + +from abc import ABC, abstractmethod +from dataclasses import dataclass, fields +from typing import TYPE_CHECKING, Any, Generic, Protocol, TypeVar + +if TYPE_CHECKING: + pass + +import torch + + +class AttentionBackend(ABC): + """Abstract class for attention backends.""" + + # For some attention backends, we allocate an output tensor before + # calling the custom op. When piecewise cudagraph is enabled, this + # makes sure the output tensor is allocated inside the cudagraph. + accept_output_buffer: bool = False + + @staticmethod + @abstractmethod + def get_name() -> str: + raise NotImplementedError + + @staticmethod + @abstractmethod + def get_impl_cls() -> type["AttentionImpl"]: + raise NotImplementedError + + @staticmethod + @abstractmethod + def get_metadata_cls() -> type["AttentionMetadata"]: + raise NotImplementedError + + # @staticmethod + # @abstractmethod + # def get_state_cls() -> Type["AttentionState"]: + # raise NotImplementedError + + # @classmethod + # def make_metadata(cls, *args, **kwargs) -> "AttentionMetadata": + # return cls.get_metadata_cls()(*args, **kwargs) + + @staticmethod + @abstractmethod + def get_builder_cls() -> type["AttentionMetadataBuilder"]: + return None + + +@dataclass +class AttentionMetadata: + """Attention metadata for prefill and decode batched together.""" + + # Current step of diffusion process + current_timestep: int + + def asdict_zerocopy(self, skip_fields: set[str] | None = None) -> dict[str, Any]: + """Similar to dataclasses.asdict, but avoids deepcopying.""" + if skip_fields is None: + skip_fields = set() + # Note that if we add dataclasses as fields, they will need + # similar handling. + return { + field.name: getattr(self, field.name) + for field in fields(self) + if field.name not in skip_fields + } + + +T = TypeVar("T", bound=AttentionMetadata) + + +class AttentionMetadataBuilder(ABC, Generic[T]): + """Abstract class for attention metadata builders.""" + + @abstractmethod + def __init__(self) -> None: + """Create the builder, remember some configuration and parameters.""" + raise NotImplementedError + + @abstractmethod + def prepare(self) -> None: + """Prepare for one batch.""" + raise NotImplementedError + + @abstractmethod + def build( + self, + **kwargs: dict[str, Any], + ) -> AttentionMetadata: + """Build attention metadata with on-device tensors.""" + raise NotImplementedError + + +class AttentionLayer(Protocol): + + _k_scale: torch.Tensor + _v_scale: torch.Tensor + _k_scale_float: float + _v_scale_float: float + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: ... + + +class AttentionImpl(ABC, Generic[T]): + + @abstractmethod + def __init__( + self, + num_heads: int, + head_size: int, + softmax_scale: float, + causal: bool = False, + num_kv_heads: int | None = None, + prefix: str = "", + **extra_impl_args, + ) -> None: + raise NotImplementedError + + def preprocess_qkv(self, qkv: torch.Tensor, attn_metadata: T) -> torch.Tensor: + """Preprocess QKV tensor before performing attention operation. + + Default implementation returns the tensor unchanged. + Subclasses can override this to implement custom preprocessing + like reshaping, tiling, scaling, or other transformations. + + Called AFTER all_to_all for distributed attention + + Args: + qkv: The query-key-value tensor + attn_metadata: Metadata for the attention operation + + Returns: + Processed QKV tensor + """ + return qkv + + def postprocess_output( + self, + output: torch.Tensor, + attn_metadata: T, + ) -> torch.Tensor: + """Postprocess the output tensor after the attention operation. + + Default implementation returns the tensor unchanged. + Subclasses can override this to implement custom postprocessing + like untiling, scaling, or other transformations. + + Called BEFORE all_to_all for distributed attention + + Args: + output: The output tensor from the attention operation + attn_metadata: Metadata for the attention operation + + Returns: + Postprocessed output tensor + """ + + return output + + @abstractmethod + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: T, + ) -> torch.Tensor: + raise NotImplementedError diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/flash_attn.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/flash_attn.py new file mode 100644 index 00000000..2394de5f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/flash_attn.py @@ -0,0 +1,139 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass +from typing import Any + +import torch + +from sglang.multimodal_gen.runtime.managers.forward_context import get_forward_context + +try: + from sgl_kernel.flash_attn import flash_attn_varlen_func + + # flash_attn 3 no longer have a different API, see following commit: + # https://github.com/Dao-AILab/flash-attention/commit/ed209409acedbb2379f870bbd03abce31a7a51b7 + flash_attn_func = flash_attn_varlen_func +except ImportError as e: + raise e + +from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import ( + AttentionBackend, + AttentionImpl, + AttentionMetadata, + AttentionMetadataBuilder, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + +fa_ver = 3 + + +def set_fa_ver(ver: int): + global fa_ver + fa_ver = ver + + +@dataclass +class FlashAttentionMetadata: + # Sequence lengths for the forward batch + # Maximum sequence length for query + max_seqlen_q: int = 1 + # Maximum sequence length for key + max_seqlen_k: int = 0 + # Cumulative sequence lengths for query + cu_seqlens_q: torch.Tensor = None + # Cumulative sequence lengths for key + cu_seqlens_k: torch.Tensor = None + + +class FlashAttentionMetadataBuilder(AttentionMetadataBuilder): + + def __init__(self): + pass + + def prepare(self): + pass + + def build( # type: ignore + self, + raw_latent_shape=list, + **kwargs: dict[str, Any], + ) -> FlashAttentionMetadata: + # TODO: put empty values here to be set at first-run, since the q_len calculation can be complicated + return FlashAttentionMetadata(max_seqlen_q=None, max_seqlen_k=None) + + +class FlashAttentionBackend(AttentionBackend): + accept_output_buffer: bool = True + + @staticmethod + def get_supported_head_sizes() -> list[int]: + return [32, 64, 96, 128, 160, 192, 224, 256] + + @staticmethod + def get_name() -> str: + return "FLASH_ATTN" + + @staticmethod + def get_impl_cls() -> type["FlashAttentionImpl"]: + return FlashAttentionImpl + + @staticmethod + def get_metadata_cls() -> type["AttentionMetadata"]: + raise NotImplementedError + + @staticmethod + def get_builder_cls() -> type["AttentionMetadataBuilder"]: + return FlashAttentionMetadataBuilder + + +class FlashAttentionImpl(AttentionImpl): + + def __init__( + self, + num_heads: int, + head_size: int, + causal: bool, + softmax_scale: float, + num_kv_heads: int | None = None, + prefix: str = "", + **extra_impl_args, + ) -> None: + self.causal = causal + self.softmax_scale = softmax_scale + self.attention_metadata = FlashAttentionMetadata() + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: AttentionMetadata = None, + *, + return_softmax_lse: bool = False, + ): + attn_metadata: FlashAttentionMetadata = get_forward_context().attn_metadata + if attn_metadata is not None and attn_metadata.max_seqlen_q is None: + attn_metadata.max_seqlen_q = query.shape[1] + attn_metadata.max_seqlen_k = key.shape[1] + max_seqlen_q = attn_metadata.max_seqlen_q + max_seqlen_k = attn_metadata.max_seqlen_k + else: + max_seqlen_q = query.shape[1] + max_seqlen_k = key.shape[1] + output = flash_attn_func( + q=query, # type: ignore[no-untyped-call] + k=key, + v=value, + cu_seqlens_q=None, + cu_seqlens_k=None, + max_seqlen_q=max_seqlen_q, + max_seqlen_k=max_seqlen_k, + softmax_scale=self.softmax_scale, + causal=self.causal, + return_softmax_lse=return_softmax_lse, + ver=fa_ver, + ) + return output diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/flash_attn_2.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/flash_attn_2.py new file mode 100644 index 00000000..df795e06 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/flash_attn_2.py @@ -0,0 +1,78 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +import torch + +from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import ( + AttentionBackend, + AttentionImpl, + AttentionMetadata, + AttentionMetadataBuilder, +) +from sglang.multimodal_gen.runtime.layers.attention.backends.flash_attn import ( + flash_attn_func, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class FlashAttention2Backend(AttentionBackend): + accept_output_buffer: bool = True + + @staticmethod + def get_supported_head_sizes() -> list[int]: + return [32, 64, 96, 128, 160, 192, 224, 256] + + @staticmethod + def get_name() -> str: + return "FA" + + @staticmethod + def get_impl_cls() -> type["FlashAttention2Impl"]: + return FlashAttention2Impl + + @staticmethod + def get_metadata_cls() -> type["AttentionMetadata"]: + raise NotImplementedError + + @staticmethod + def get_builder_cls() -> type["AttentionMetadataBuilder"]: + raise NotImplementedError + + +class FlashAttention2Impl(AttentionImpl): + + def __init__( + self, + num_heads: int, + head_size: int, + causal: bool, + softmax_scale: float, + num_kv_heads: int | None = None, + prefix: str = "", + **extra_impl_args, + ) -> None: + self.causal = causal + self.softmax_scale = softmax_scale + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: AttentionMetadata, + ): + output = flash_attn_func( + q=query, # type: ignore[no-untyped-call] + k=key, + v=value, + cu_seqlens_q=None, + cu_seqlens_k=None, + max_seqlen_q=None, + max_seqlen_k=None, + softmax_scale=self.softmax_scale, + causal=self.causal, + ) + return output diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/sage_attn.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/sage_attn.py new file mode 100644 index 00000000..3563ddd1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/sage_attn.py @@ -0,0 +1,70 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +import torch +from sageattention import sageattn + +from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import ( # FlashAttentionMetadata, + AttentionBackend, + AttentionImpl, + AttentionMetadata, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class SageAttentionBackend(AttentionBackend): + + accept_output_buffer: bool = True + + @staticmethod + def get_supported_head_sizes() -> list[int]: + return [32, 64, 96, 128, 160, 192, 224, 256] + + @staticmethod + def get_name() -> str: + return "SAGE_ATTN" + + @staticmethod + def get_impl_cls() -> type["SageAttentionImpl"]: + return SageAttentionImpl + + # @staticmethod + # def get_metadata_cls() -> Type["AttentionMetadata"]: + # return FlashAttentionMetadata + + +class SageAttentionImpl(AttentionImpl): + + def __init__( + self, + num_heads: int, + head_size: int, + causal: bool, + softmax_scale: float, + num_kv_heads: int | None = None, + prefix: str = "", + **extra_impl_args, + ) -> None: + self.causal = causal + self.softmax_scale = softmax_scale + self.dropout = extra_impl_args.get("dropout_p", 0.0) + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + output = sageattn( + query, + key, + value, + # since input is (batch_size, seq_len, head_num, head_dim) + tensor_layout="NHD", + is_causal=self.causal, + ) + return output diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/sage_attn3.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/sage_attn3.py new file mode 100644 index 00000000..fd5b6f2b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/sage_attn3.py @@ -0,0 +1,78 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +import torch + +from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import ( + AttentionBackend, + AttentionImpl, + AttentionMetadata, + AttentionMetadataBuilder, +) +from sglang.multimodal_gen.runtime.layers.attention.backends.sageattn.api import ( + sageattn_blackwell, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class SageAttention3Backend(AttentionBackend): + + accept_output_buffer: bool = True + + @staticmethod + def get_supported_head_sizes() -> list[int]: + return [64, 128, 256] + + @staticmethod + def get_name() -> str: + return "SAGE_ATTN_THREE" + + @staticmethod + def get_impl_cls() -> type["SageAttention3Impl"]: + return SageAttention3Impl + + @staticmethod + def get_metadata_cls() -> type["AttentionMetadata"]: + raise NotImplementedError + + @staticmethod + def get_builder_cls() -> type["AttentionMetadataBuilder"]: + raise NotImplementedError + + # @staticmethod + # def get_metadata_cls() -> Type["AttentionMetadata"]: + # return FlashAttentionMetadata + + +class SageAttention3Impl(AttentionImpl): + + def __init__( + self, + num_heads: int, + head_size: int, + causal: bool, + softmax_scale: float, + num_kv_heads: int | None = None, + prefix: str = "", + **extra_impl_args, + ) -> None: + self.causal = causal + self.softmax_scale = softmax_scale + self.dropout = extra_impl_args.get("dropout_p", 0.0) + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + query = query.transpose(1, 2) + key = key.transpose(1, 2) + value = value.transpose(1, 2) + output = sageattn_blackwell(query, key, value, is_causal=self.causal) + output = output.transpose(1, 2) + return output diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/sdpa.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/sdpa.py new file mode 100644 index 00000000..bfa3b430 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/sdpa.py @@ -0,0 +1,77 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +import torch + +from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import ( # FlashAttentionMetadata, + AttentionBackend, + AttentionImpl, + AttentionMetadata, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class SDPABackend(AttentionBackend): + + accept_output_buffer: bool = True + + @staticmethod + def get_supported_head_sizes() -> list[int]: + return [32, 64, 96, 128, 160, 192, 224, 256] + + @staticmethod + def get_name() -> str: + return "SDPA" + + @staticmethod + def get_impl_cls() -> type["SDPAImpl"]: + return SDPAImpl + + # @staticmethod + # def get_metadata_cls() -> Type["AttentionMetadata"]: + # return FlashAttentionMetadata + + +class SDPAImpl(AttentionImpl): + + def __init__( + self, + num_heads: int, + head_size: int, + causal: bool, + softmax_scale: float, + num_kv_heads: int | None = None, + prefix: str = "", + **extra_impl_args, + ) -> None: + self.causal = causal + self.softmax_scale = softmax_scale + self.dropout = extra_impl_args.get("dropout_p", 0.0) + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + # transpose to bs, heads, seq_len, head_dim + query = query.transpose(1, 2) + key = key.transpose(1, 2) + value = value.transpose(1, 2) + attn_kwargs = { + "attn_mask": None, + "dropout_p": self.dropout, + "is_causal": self.causal, + "scale": self.softmax_scale, + } + if query.shape[1] != key.shape[1]: + attn_kwargs["enable_gqa"] = True + output = torch.nn.functional.scaled_dot_product_attention( + query, key, value, **attn_kwargs + ) + output = output.transpose(1, 2) + return output diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/sliding_tile_attn.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/sliding_tile_attn.py new file mode 100644 index 00000000..6db3785f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/sliding_tile_attn.py @@ -0,0 +1,313 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +import json +from dataclasses import dataclass +from typing import Any + +import torch +from einops import rearrange + +import sglang.multimodal_gen.envs as envs +from sglang.multimodal_gen.runtime.distributed import get_sp_group +from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import ( + AttentionBackend, + AttentionImpl, + AttentionMetadata, + AttentionMetadataBuilder, +) +from sglang.multimodal_gen.runtime.managers.forward_context import ( + ForwardContext, + get_forward_context, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.utils import dict_to_3d_list + +try: + from st_attn import sliding_tile_attention + + st_attn_backend_available = True +except Exception: + st_attn_backend_available = False + +logger = init_logger(__name__) + + +class RangeDict(dict): + + def __getitem__(self, item: int) -> str: + for key in self.keys(): + if isinstance(key, tuple): + low, high = key + if low <= item <= high: + return str(super().__getitem__(key)) + elif key == item: + return str(super().__getitem__(key)) + raise KeyError(f"seq_len {item} not supported for STA") + + +class SlidingTileAttentionBackend(AttentionBackend): + accept_output_buffer: bool = True + + @staticmethod + def get_supported_head_sizes() -> list[int]: + # TODO(will-refactor): check this + return [32, 64, 96, 128, 160, 192, 224, 256] + + @staticmethod + def get_name() -> str: + return "SLIDING_TILE_ATTN" + + @staticmethod + def get_impl_cls() -> type["SlidingTileAttentionImpl"]: + return SlidingTileAttentionImpl + + @staticmethod + def get_metadata_cls() -> type["SlidingTileAttentionMetadata"]: + return SlidingTileAttentionMetadata + + @staticmethod + def get_builder_cls() -> type["SlidingTileAttentionMetadataBuilder"]: + return SlidingTileAttentionMetadataBuilder + + +@dataclass +class SlidingTileAttentionMetadata(AttentionMetadata): + current_timestep: int + STA_param: list[ + list[Any] + ] # each timestep with one metadata, shape [num_layers, num_heads] + + +class SlidingTileAttentionMetadataBuilder(AttentionMetadataBuilder): + + def __init__(self): + pass + + def prepare(self): + pass + + def build( # type: ignore + self, + STA_param: list[list[Any]], + current_timestep: int, + **kwargs: dict[str, Any], + ) -> SlidingTileAttentionMetadata: + param = STA_param + if param is None: + return SlidingTileAttentionMetadata( + current_timestep=current_timestep, STA_param=[] + ) + return SlidingTileAttentionMetadata( + current_timestep=current_timestep, STA_param=param[current_timestep] + ) + + +class SlidingTileAttentionImpl(AttentionImpl): + + def __init__( + self, + num_heads: int, + head_size: int, + causal: bool, + softmax_scale: float, + num_kv_heads: int | None = None, + prefix: str = "", + **extra_impl_args, + ) -> None: + if not st_attn_backend_available: + raise ValueError("st attn not supported") + # TODO(will-refactor): for now this is the mask strategy, but maybe we should + # have a more general config for STA? + config_file = envs.SGLANG_DIFFUSION_ATTENTION_CONFIG + if config_file is None: + raise ValueError("SGLANG_DIFFUSION_ATTENTION_CONFIG is not set") + + # TODO(kevin): get mask strategy for different STA modes + with open(config_file) as f: + mask_strategy = json.load(f) + self.mask_strategy = dict_to_3d_list(mask_strategy) + + self.prefix = prefix + sp_group = get_sp_group() + self.sp_size = sp_group.world_size + # STA config + self.STA_base_tile_size = [6, 8, 8] + self.dit_seq_shape_mapping = RangeDict( + { + (115200, 115456): "30x48x80", + 82944: "36x48x48", + 69120: "18x48x80", + } + ) + self.full_window_mapping = { + "30x48x80": [5, 6, 10], + "36x48x48": [6, 6, 6], + "18x48x80": [3, 6, 10], + } + + def tile(self, x: torch.Tensor) -> torch.Tensor: + return rearrange( + x, + "b (n_t ts_t n_h ts_h n_w ts_w) h d -> b (n_t n_h n_w ts_t ts_h ts_w) h d", + n_t=self.full_window_size[0], + n_h=self.full_window_size[1], + n_w=self.full_window_size[2], + ts_t=self.STA_base_tile_size[0], + ts_h=self.STA_base_tile_size[1], + ts_w=self.STA_base_tile_size[2], + ) + + def untile(self, x: torch.Tensor) -> torch.Tensor: + x = rearrange( + x, + "b (n_t n_h n_w ts_t ts_h ts_w) h d -> b (n_t ts_t n_h ts_h n_w ts_w) h d", + n_t=self.full_window_size[0], + n_h=self.full_window_size[1], + n_w=self.full_window_size[2], + ts_t=self.STA_base_tile_size[0], + ts_h=self.STA_base_tile_size[1], + ts_w=self.STA_base_tile_size[2], + ) + return x + + def preprocess_qkv( + self, + qkv: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + img_sequence_length = qkv.shape[1] + self.dit_seq_shape_str = self.dit_seq_shape_mapping[img_sequence_length] + self.full_window_size = self.full_window_mapping[self.dit_seq_shape_str] + self.dit_seq_shape_int = list(map(int, self.dit_seq_shape_str.split("x"))) + self.img_seq_length = ( + self.dit_seq_shape_int[0] + * self.dit_seq_shape_int[1] + * self.dit_seq_shape_int[2] + ) + return self.tile(qkv) + + def postprocess_output( + self, + output: torch.Tensor, + attn_metadata: SlidingTileAttentionMetadata, + ) -> torch.Tensor: + return self.untile(output) + + def forward( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + attn_metadata: SlidingTileAttentionMetadata, + ) -> torch.Tensor: + if self.mask_strategy is None: + raise ValueError("mask_strategy cannot be None for SlidingTileAttention") + if self.mask_strategy[0] is None: + raise ValueError("mask_strategy[0] cannot be None for SlidingTileAttention") + + timestep = attn_metadata.current_timestep + forward_context: ForwardContext = get_forward_context() + forward_batch = forward_context.forward_batch + if forward_batch is None: + raise ValueError("forward_batch cannot be None") + # pattern:'.double_blocks.0.attn.impl' or '.single_blocks.0.attn.impl' + layer_idx = int(self.prefix.split(".")[-3]) + if attn_metadata.STA_param is None or len(attn_metadata.STA_param) <= layer_idx: + raise ValueError("Invalid STA_param") + STA_param = attn_metadata.STA_param[layer_idx] + + text_length = q.shape[1] - self.img_seq_length + has_text = text_length > 0 + + query = q.transpose(1, 2).contiguous() + key = k.transpose(1, 2).contiguous() + value = v.transpose(1, 2).contiguous() + + head_num = query.size(1) + sp_group = get_sp_group() + current_rank = sp_group.rank_in_group + start_head = current_rank * head_num + + # searching or tuning mode + if len(STA_param) < head_num * sp_group.world_size: + sparse_attn_hidden_states_all = [] + full_mask_window = STA_param[-1] + for window_size in STA_param[:-1]: + sparse_hidden_states = sliding_tile_attention( + query, + key, + value, + [window_size] * head_num, + text_length, + has_text, + self.dit_seq_shape_str, + ).transpose(1, 2) + sparse_attn_hidden_states_all.append(sparse_hidden_states) + + hidden_states = sliding_tile_attention( + query, + key, + value, + [full_mask_window] * head_num, + text_length, + has_text, + self.dit_seq_shape_str, + ).transpose(1, 2) + + attn_L2_loss = [] + attn_L1_loss = [] + # average loss across all heads + for sparse_attn_hidden_states in sparse_attn_hidden_states_all: + # L2 loss + attn_L2_loss_ = ( + torch.mean( + (sparse_attn_hidden_states.float() - hidden_states.float()) + ** 2, + dim=[0, 1, 3], + ) + .cpu() + .numpy() + ) + attn_L2_loss_ = [round(float(x), 6) for x in attn_L2_loss_] + attn_L2_loss.append(attn_L2_loss_) + # L1 loss + attn_L1_loss_ = ( + torch.mean( + torch.abs( + sparse_attn_hidden_states.float() - hidden_states.float() + ), + dim=[0, 1, 3], + ) + .cpu() + .numpy() + ) + attn_L1_loss_ = [round(float(x), 6) for x in attn_L1_loss_] + attn_L1_loss.append(attn_L1_loss_) + + layer_loss_save = {"L2_loss": attn_L2_loss, "L1_loss": attn_L1_loss} + + if forward_batch.is_cfg_negative: + if forward_batch.mask_search_final_result_neg is not None: + forward_batch.mask_search_final_result_neg[timestep].append( + layer_loss_save + ) + else: + if forward_batch.mask_search_final_result_pos is not None: + forward_batch.mask_search_final_result_pos[timestep].append( + layer_loss_save + ) + else: + windows = [STA_param[head_idx + start_head] for head_idx in range(head_num)] + + hidden_states = sliding_tile_attention( + query, + key, + value, + windows, + text_length, + has_text, + self.dit_seq_shape_str, + ).transpose(1, 2) + + return hidden_states diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn.py new file mode 100644 index 00000000..6fe34292 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn.py @@ -0,0 +1,331 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +import functools +import math +from dataclasses import dataclass + +import torch + +try: + from vsa import video_sparse_attn +except ImportError: + video_sparse_attn = None + +from typing import Any + +from sglang.multimodal_gen.runtime.distributed import get_sp_group +from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import ( + AttentionBackend, + AttentionImpl, + AttentionMetadata, + AttentionMetadataBuilder, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) +VSA_TILE_SIZE = (4, 4, 4) + + +@functools.lru_cache(maxsize=10) +def get_tile_partition_indices( + dit_seq_shape: tuple[int, int, int], + tile_size: tuple[int, int, int], + device: torch.device, +) -> torch.LongTensor: + T, H, W = dit_seq_shape + ts, hs, ws = tile_size + indices = torch.arange(T * H * W, device=device, dtype=torch.long).reshape(T, H, W) + ls = [] + for t in range(math.ceil(T / ts)): + for h in range(math.ceil(H / hs)): + for w in range(math.ceil(W / ws)): + ls.append( + indices[ + t * ts : min(t * ts + ts, T), + h * hs : min(h * hs + hs, H), + w * ws : min(w * ws + ws, W), + ].flatten() + ) + index = torch.cat(ls, dim=0) + return index + + +@functools.lru_cache(maxsize=10) +def get_reverse_tile_partition_indices( + dit_seq_shape: tuple[int, int, int], + tile_size: tuple[int, int, int], + device: torch.device, +) -> torch.LongTensor: + return torch.argsort(get_tile_partition_indices(dit_seq_shape, tile_size, device)) + + +@functools.lru_cache(maxsize=10) +def construct_variable_block_sizes( + dit_seq_shape: tuple[int, int, int], + num_tiles: tuple[int, int, int], + device: torch.device, +) -> torch.LongTensor: + """ + Compute the number of valid (non‑padded) tokens inside every + (ts_t × ts_h × ts_w) tile after padding ‑‑ flattened in the order + (t‑tile, h‑tile, w‑tile) that `rearrange` uses. + + Returns + ------- + torch.LongTensor # shape: [∏ full_window_size] + """ + # unpack + t, h, w = dit_seq_shape + ts_t, ts_h, ts_w = VSA_TILE_SIZE + n_t, n_h, n_w = num_tiles + + def _sizes(dim_len: int, tile: int, n_tiles: int) -> torch.LongTensor: + """Vector with the size of each tile along one dimension.""" + sizes = torch.full((n_tiles,), tile, dtype=torch.int, device=device) + # size of last (possibly partial) tile + remainder = dim_len - (n_tiles - 1) * tile + sizes[-1] = remainder if remainder > 0 else tile + return sizes + + t_sizes = _sizes(t, ts_t, n_t) # [n_t] + h_sizes = _sizes(h, ts_h, n_h) # [n_h] + w_sizes = _sizes(w, ts_w, n_w) # [n_w] + + # broadcast‑multiply to get voxels per tile, then flatten + block_sizes = ( + t_sizes[:, None, None] # [n_t, 1, 1] + * h_sizes[None, :, None] # [1, n_h, 1] + * w_sizes[None, None, :] # [1, 1, n_w] + ).reshape( + -1 + ) # [n_t * n_h * n_w] + + return block_sizes + + +@functools.lru_cache(maxsize=10) +def get_non_pad_index( + variable_block_sizes: torch.LongTensor, + max_block_size: int, +): + n_win = variable_block_sizes.shape[0] + device = variable_block_sizes.device + starts_pad = torch.arange(n_win, device=device) * max_block_size + index_pad = ( + starts_pad[:, None] + torch.arange(max_block_size, device=device)[None, :] + ) + index_mask = ( + torch.arange(max_block_size, device=device)[None, :] + < variable_block_sizes[:, None] + ) + return index_pad[index_mask] + + +class VideoSparseAttentionBackend(AttentionBackend): + + accept_output_buffer: bool = True + + @staticmethod + def get_supported_head_sizes() -> list[int]: + return [64, 128] + + @staticmethod + def get_name() -> str: + return "VIDEO_SPARSE_ATTN" + + @staticmethod + def get_impl_cls() -> type["VideoSparseAttentionImpl"]: + return VideoSparseAttentionImpl + + @staticmethod + def get_metadata_cls() -> type["VideoSparseAttentionMetadata"]: + return VideoSparseAttentionMetadata + + @staticmethod + def get_builder_cls() -> type["VideoSparseAttentionMetadataBuilder"]: + return VideoSparseAttentionMetadataBuilder + + +@dataclass +class VideoSparseAttentionMetadata(AttentionMetadata): + current_timestep: int + dit_seq_shape: list[int] + VSA_sparsity: float + num_tiles: list[int] + total_seq_length: int + tile_partition_indices: torch.LongTensor + reverse_tile_partition_indices: torch.LongTensor + variable_block_sizes: torch.LongTensor + non_pad_index: torch.LongTensor + + # adaption for FastWan2.1-T2V-1.3B-Diffusers + # Sequence lengths for the forward batch + # Maximum sequence length for query + max_seqlen_q: int = 1 + # Maximum sequence length for key + max_seqlen_k: int = 0 + + +class VideoSparseAttentionMetadataBuilder(AttentionMetadataBuilder): + + def __init__(self): + pass + + def prepare(self): + pass + + def build( # type: ignore + self, + current_timestep: int, + raw_latent_shape: tuple[int, int, int], + patch_size: tuple[int, int, int], + VSA_sparsity: float, + device: torch.device, + **kwargs: dict[str, Any], + ) -> VideoSparseAttentionMetadata: + patch_size = patch_size + dit_seq_shape = ( + raw_latent_shape[0] // patch_size[0], + raw_latent_shape[1] // patch_size[1], + raw_latent_shape[2] // patch_size[2], + ) + + num_tiles = ( + math.ceil(dit_seq_shape[0] / VSA_TILE_SIZE[0]), + math.ceil(dit_seq_shape[1] / VSA_TILE_SIZE[1]), + math.ceil(dit_seq_shape[2] / VSA_TILE_SIZE[2]), + ) + total_seq_length = math.prod(dit_seq_shape) + + tile_partition_indices = get_tile_partition_indices( + dit_seq_shape, VSA_TILE_SIZE, device + ) + reverse_tile_partition_indices = get_reverse_tile_partition_indices( + dit_seq_shape, VSA_TILE_SIZE, device + ) + variable_block_sizes = construct_variable_block_sizes( + dit_seq_shape, num_tiles, device + ) + non_pad_index = get_non_pad_index( + variable_block_sizes, math.prod(VSA_TILE_SIZE) + ) + + return VideoSparseAttentionMetadata( + current_timestep=current_timestep, + dit_seq_shape=dit_seq_shape, # type: ignore + VSA_sparsity=VSA_sparsity, # type: ignore + num_tiles=num_tiles, # type: ignore + total_seq_length=total_seq_length, # type: ignore + tile_partition_indices=tile_partition_indices, # type: ignore + reverse_tile_partition_indices=reverse_tile_partition_indices, + variable_block_sizes=variable_block_sizes, + non_pad_index=non_pad_index, + ) + + +class VideoSparseAttentionImpl(AttentionImpl): + + def __init__( + self, + num_heads: int, + head_size: int, + causal: bool, + softmax_scale: float, + num_kv_heads: int | None = None, + prefix: str = "", + **extra_impl_args, + ) -> None: + self.prefix = prefix + sp_group = get_sp_group() + self.sp_size = sp_group.world_size + + def tile( + self, + x: torch.Tensor, + num_tiles: list[int], + tile_partition_indices: torch.LongTensor, + non_pad_index: torch.LongTensor, + ) -> torch.Tensor: + t_padded_size = num_tiles[0] * VSA_TILE_SIZE[0] + h_padded_size = num_tiles[1] * VSA_TILE_SIZE[1] + w_padded_size = num_tiles[2] * VSA_TILE_SIZE[2] + + x_padded = torch.zeros( + ( + x.shape[0], + t_padded_size * h_padded_size * w_padded_size, + x.shape[-2], + x.shape[-1], + ), + device=x.device, + dtype=x.dtype, + ) + x_padded[:, non_pad_index] = x[:, tile_partition_indices] + return x_padded + + def untile( + self, + x: torch.Tensor, + reverse_tile_partition_indices: torch.LongTensor, + non_pad_index: torch.LongTensor, + ) -> torch.Tensor: + x = x[:, non_pad_index][:, reverse_tile_partition_indices] + return x + + def preprocess_qkv( + self, + qkv: torch.Tensor, + attn_metadata: VideoSparseAttentionMetadata, + ) -> torch.Tensor: + return self.tile( + qkv, + attn_metadata.num_tiles, + attn_metadata.tile_partition_indices, + attn_metadata.non_pad_index, + ) + + def postprocess_output( + self, + output: torch.Tensor, + attn_metadata: VideoSparseAttentionMetadata, + ) -> torch.Tensor: + return self.untile( + output, + attn_metadata.reverse_tile_partition_indices, + attn_metadata.non_pad_index, + ) + + def forward( # type: ignore[override] + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + gate_compress: torch.Tensor, + attn_metadata: VideoSparseAttentionMetadata, + ) -> torch.Tensor: + query = query.transpose(1, 2).contiguous() + key = key.transpose(1, 2).contiguous() + value = value.transpose(1, 2).contiguous() + gate_compress = gate_compress.transpose(1, 2).contiguous() + + VSA_sparsity = attn_metadata.VSA_sparsity + + cur_topk = math.ceil( + (1 - VSA_sparsity) + * (attn_metadata.total_seq_length / math.prod(VSA_TILE_SIZE)) + ) + + if video_sparse_attn is None: + raise NotImplementedError("video_sparse_attn is not installed") + hidden_states = video_sparse_attn( + query, + key, + value, + variable_block_sizes=attn_metadata.variable_block_sizes, + topk=cur_topk, + block_size=VSA_TILE_SIZE, + compress_attn_weight=gate_compress, + ).transpose(1, 2) + + return hidden_states diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/vmoba.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/vmoba.py new file mode 100644 index 00000000..5709601d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/backends/vmoba.py @@ -0,0 +1,258 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +import re +from dataclasses import dataclass + +import torch +from einops import rearrange +from kernel.attn.vmoba_attn.vmoba import ( + moba_attn_varlen, + process_moba_input, + process_moba_output, +) + +from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import ( + AttentionBackend, + AttentionImpl, + AttentionMetadata, + AttentionMetadataBuilder, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class VMOBAAttentionBackend(AttentionBackend): + + accept_output_buffer: bool = True + + @staticmethod + def get_name() -> str: + return "VMOBA_ATTN" + + @staticmethod + def get_impl_cls() -> type["VMOBAAttentionImpl"]: + return VMOBAAttentionImpl + + @staticmethod + def get_metadata_cls() -> type["VideoMobaAttentionMetadata"]: + return VideoMobaAttentionMetadata + + @staticmethod + def get_builder_cls() -> type["VideoMobaAttentionMetadataBuilder"]: + return VideoMobaAttentionMetadataBuilder + + +@dataclass +class VideoMobaAttentionMetadata(AttentionMetadata): + current_timestep: int + + temporal_chunk_size: int + temporal_topk: int + spatial_chunk_size: tuple[int, int] + spatial_topk: int + st_chunk_size: tuple[int, int, int] + st_topk: int + + moba_select_mode: str + moba_threshold: float + moba_threshold_type: str + patch_resolution: list[int] + + first_full_step: int = 12 + first_full_layer: int = 0 + # temporal_layer -> spatial_layer -> st_layer + temporal_layer: int = 1 + spatial_layer: int = 1 + st_layer: int = 1 + + +def pad_input(hidden_states, indices, batch, seqlen): + """ + Arguments: + hidden_states: (total_nnz, ...), where total_nnz = number of tokens in selected in attention_mask. + indices: (total_nnz), the indices that represent the non-masked tokens of the original padded input sequence. + batch: int, batch size for the padded sequence. + seqlen: int, maximum sequence length for the padded sequence. + Return: + hidden_states: (batch, seqlen, ...) + """ + dim = hidden_states.shape[1:] + output = torch.zeros( + (batch * seqlen), *dim, device=hidden_states.device, dtype=hidden_states.dtype + ) + output[indices] = hidden_states + return rearrange(output, "(b s) ... -> b s ...", b=batch) + + +class VideoMobaAttentionMetadataBuilder(AttentionMetadataBuilder): + + def __init__(self): + pass + + def prepare(self): + pass + + def build( # type: ignore + self, + current_timestep: int, + raw_latent_shape: tuple[int, int, int], + patch_size: tuple[int, int, int], + temporal_chunk_size: int, + temporal_topk: int, + spatial_chunk_size: tuple[int, int], + spatial_topk: int, + st_chunk_size: tuple[int, int, int], + st_topk: int, + moba_select_mode: str = "threshold", + moba_threshold: float = 0.25, + moba_threshold_type: str = "query_head", + device: torch.device = None, + first_full_layer: int = 0, + first_full_step: int = 12, + temporal_layer: int = 1, + spatial_layer: int = 1, + st_layer: int = 1, + **kwargs, + ) -> VideoMobaAttentionMetadata: + if device is None: + device = torch.device("cpu") + assert ( + raw_latent_shape[0] % patch_size[0] == 0 + and raw_latent_shape[1] % patch_size[1] == 0 + and raw_latent_shape[2] % patch_size[2] == 0 + ), f"spatial patch_resolution {raw_latent_shape} should be divisible by patch_size {patch_size}" + patch_resolution = [ + t // pt for t, pt in zip(raw_latent_shape, patch_size, strict=False) + ] + + return VideoMobaAttentionMetadata( + current_timestep=current_timestep, + temporal_chunk_size=temporal_chunk_size, + temporal_topk=temporal_topk, + spatial_chunk_size=spatial_chunk_size, + spatial_topk=spatial_topk, + st_chunk_size=st_chunk_size, + st_topk=st_topk, + moba_select_mode=moba_select_mode, + moba_threshold=moba_threshold, + moba_threshold_type=moba_threshold_type, + patch_resolution=patch_resolution, + first_full_layer=first_full_layer, + first_full_step=first_full_step, + temporal_layer=temporal_layer, + spatial_layer=spatial_layer, + st_layer=st_layer, + ) + + +class VMOBAAttentionImpl(AttentionImpl): + + def __init__( + self, + num_heads, + head_size, + softmax_scale, + causal=False, + num_kv_heads=None, + prefix="", + **extra_impl_args, + ) -> None: + self.prefix = prefix + self.layer_idx = self._get_layer_idx(prefix) + + self.pad_input = pad_input + + def _get_layer_idx(self, prefix: str) -> int | None: + match = re.search(r"blocks\.(\d+)", prefix) + if not match: + raise ValueError(f"Invalid prefix: {prefix}") + return int(match.group(1)) + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + """ + query: [B, L, H, D] + key: [B, L, H, D] + value: [B, L, H, D] + attn_metadata: AttentionMetadata + """ + batch_size, sequence_length, num_heads, head_dim = query.shape + + # select chunk type according to layer idx: + loop_layer_num = ( + attn_metadata.temporal_layer + + attn_metadata.spatial_layer + + attn_metadata.st_layer + ) + moba_layer = self.layer_idx - attn_metadata.first_full_layer + if moba_layer % loop_layer_num < attn_metadata.temporal_layer: + moba_chunk_size = attn_metadata.temporal_chunk_size + moba_topk = attn_metadata.temporal_topk + elif ( + moba_layer % loop_layer_num + < attn_metadata.temporal_layer + attn_metadata.spatial_layer + ): + moba_chunk_size = attn_metadata.spatial_chunk_size + moba_topk = attn_metadata.spatial_topk + elif ( + moba_layer % loop_layer_num + < attn_metadata.temporal_layer + + attn_metadata.spatial_layer + + attn_metadata.st_layer + ): + moba_chunk_size = attn_metadata.st_chunk_size + moba_topk = attn_metadata.st_topk + + query, chunk_size = process_moba_input( + query, attn_metadata.patch_resolution, moba_chunk_size + ) + key, chunk_size = process_moba_input( + key, attn_metadata.patch_resolution, moba_chunk_size + ) + value, chunk_size = process_moba_input( + value, attn_metadata.patch_resolution, moba_chunk_size + ) + max_seqlen = query.shape[1] + indices_q = torch.arange( + 0, query.shape[0] * query.shape[1], device=query.device + ) + cu_seqlens = torch.arange( + 0, + query.shape[0] * query.shape[1] + 1, + query.shape[1], + dtype=torch.int32, + device=query.device, + ) + query = rearrange(query, "b s ... -> (b s) ...") + key = rearrange(key, "b s ... -> (b s) ...") + value = rearrange(value, "b s ... -> (b s) ...") + + # current_timestep=attn_metadata.current_timestep + hidden_states = moba_attn_varlen( + query, + key, + value, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + moba_chunk_size=chunk_size, + moba_topk=moba_topk, + select_mode=attn_metadata.moba_select_mode, + simsum_threshold=attn_metadata.moba_threshold, + threshold_type=attn_metadata.moba_threshold_type, + ) + hidden_states = self.pad_input( + hidden_states, indices_q, batch_size, sequence_length + ) + hidden_states = process_moba_output( + hidden_states, attn_metadata.patch_resolution, moba_chunk_size + ) + + return hidden_states diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/layer.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/layer.py new file mode 100644 index 00000000..df4f377d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/layer.py @@ -0,0 +1,396 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from typing import Type + +import torch +import torch.nn as nn + +from sglang.multimodal_gen.runtime.distributed.communication_op import ( + sequence_model_parallel_all_gather, + sequence_model_parallel_all_to_all_4D, +) +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + get_ring_parallel_world_size, + get_sequence_parallel_world_size, + get_sp_parallel_rank, + get_sp_world_size, + get_ulysses_parallel_world_size, +) +from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import ( + AttentionImpl, +) +from sglang.multimodal_gen.runtime.layers.attention.selector import ( + backend_name_to_enum, + get_attn_backend, +) +from sglang.multimodal_gen.runtime.layers.usp import ( + _usp_input_all_to_all, + _usp_output_all_to_all, + ring_attn, +) +from sglang.multimodal_gen.runtime.managers.forward_context import ( + ForwardContext, + get_forward_context, +) +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum +from sglang.multimodal_gen.utils import get_compute_dtype + + +class UlyssesAttention(nn.Module): + """Ulysses-style SequenceParallelism attention layer.""" + + def __init__( + self, + num_heads: int, + head_size: int, + num_kv_heads: int | None = None, + softmax_scale: float | None = None, + causal: bool = False, + supported_attention_backends: set[AttentionBackendEnum] | None = None, + prefix: str = "", + **extra_impl_args, + ) -> None: + super().__init__() + if softmax_scale is None: + self.softmax_scale = head_size**-0.5 + else: + self.softmax_scale = softmax_scale + + if num_kv_heads is None: + num_kv_heads = num_heads + + dtype = get_compute_dtype() + attn_backend = get_attn_backend( + head_size, dtype, supported_attention_backends=supported_attention_backends + ) + impl_cls = attn_backend.get_impl_cls() + + self.attn_impl = impl_cls( + num_heads=num_heads, + head_size=head_size, + causal=causal, + softmax_scale=self.softmax_scale, + num_kv_heads=num_kv_heads, + prefix=f"{prefix}.impl", + **extra_impl_args, + ) + self.num_heads = num_heads + self.head_size = head_size + self.num_kv_heads = num_kv_heads + self.backend = backend_name_to_enum(attn_backend.get_name()) + self.dtype = dtype + + @torch.compiler.disable + def forward( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + replicated_q: torch.Tensor | None = None, + replicated_k: torch.Tensor | None = None, + replicated_v: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + """Forward pass for distributed attention. + + Args: + q (torch.Tensor): Query tensor [batch_size, seq_len, num_heads, head_dim] + k (torch.Tensor): Key tensor [batch_size, seq_len, num_heads, head_dim] + v (torch.Tensor): Value tensor [batch_size, seq_len, num_heads, head_dim] + replicated_q (Optional[torch.Tensor]): Replicated query tensor, typically for text tokens + replicated_k (Optional[torch.Tensor]): Replicated key tensor + replicated_v (Optional[torch.Tensor]): Replicated value tensor + + Returns: + Tuple[torch.Tensor, Optional[torch.Tensor]]: A tuple containing: + - o (torch.Tensor): Output tensor after attention for the main sequence + - replicated_o (Optional[torch.Tensor]): Output tensor for replicated tokens, if provided + """ + # Check input shapes + assert q.dim() == 4 and k.dim() == 4 and v.dim() == 4, "Expected 4D tensors" + batch_size, seq_len, num_heads, head_dim = q.shape + local_rank = get_sp_parallel_rank() + world_size = get_sp_world_size() + + forward_context: ForwardContext = get_forward_context() + ctx_attn_metadata = forward_context.attn_metadata + + # Stack QKV + qkv = torch.cat([q, k, v], dim=0) # [3, seq_len, num_heads, head_dim] + + # Redistribute heads across sequence dimension + qkv = sequence_model_parallel_all_to_all_4D(qkv, scatter_dim=2, gather_dim=1) + # Apply backend-specific preprocess_qkv + qkv = self.attn_impl.preprocess_qkv(qkv, ctx_attn_metadata) + + # Concatenate with replicated QKV if provided + if replicated_q is not None: + assert replicated_k is not None and replicated_v is not None + replicated_qkv = torch.cat( + [replicated_q, replicated_k, replicated_v], dim=0 + ) # [3, seq_len, num_heads, head_dim] + heads_per_rank = num_heads // world_size + replicated_qkv = replicated_qkv[ + :, :, local_rank * heads_per_rank : (local_rank + 1) * heads_per_rank + ] + qkv = torch.cat([qkv, replicated_qkv], dim=1) + + q, k, v = qkv.chunk(3, dim=0) + + output = self.attn_impl.forward(q, k, v, ctx_attn_metadata) + + # Redistribute back if using sequence parallelism + replicated_output = None + if replicated_q is not None: + replicated_output = output[:, seq_len * world_size :] + output = output[:, : seq_len * world_size] + # TODO: make this asynchronous + replicated_output = sequence_model_parallel_all_gather( + replicated_output.contiguous(), dim=2 + ) + # Apply backend-specific postprocess_output + output = self.attn_impl.postprocess_output(output, ctx_attn_metadata) + + output = sequence_model_parallel_all_to_all_4D( + output, scatter_dim=1, gather_dim=2 + ) + return output, replicated_output + + +class UlyssesAttention_VSA(UlyssesAttention): + """Distributed attention layer with VSA support.""" + + @torch.compiler.disable + def forward( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + replicated_q: torch.Tensor | None = None, + replicated_k: torch.Tensor | None = None, + replicated_v: torch.Tensor | None = None, + gate_compress: torch.Tensor | None = None, + ) -> torch.Tensor: + """Forward pass for distributed attention. + + Args: + q (torch.Tensor): Query tensor [batch_size, seq_len, num_heads, head_dim] + k (torch.Tensor): Key tensor [batch_size, seq_len, num_heads, head_dim] + v (torch.Tensor): Value tensor [batch_size, seq_len, num_heads, head_dim] + gate_compress (torch.Tensor): Gate compress tensor [batch_size, seq_len, num_heads, head_dim] + replicated_q (Optional[torch.Tensor]): Replicated query tensor, typically for text tokens + replicated_k (Optional[torch.Tensor]): Replicated key tensor + replicated_v (Optional[torch.Tensor]): Replicated value tensor + + Returns: + Tuple[torch.Tensor, Optional[torch.Tensor]]: A tuple containing: + - o (torch.Tensor): Output tensor after attention for the main sequence + - replicated_o (Optional[torch.Tensor]): Output tensor for replicated tokens, if provided + """ + # Check text tokens are not supported for VSA now + assert ( + replicated_q is None and replicated_k is None and replicated_v is None + ), "Replicated QKV is not supported for VSA now" + # Check input shapes + assert q.dim() == 4 and k.dim() == 4 and v.dim() == 4, "Expected 4D tensors" + + forward_context: ForwardContext = get_forward_context() + ctx_attn_metadata = forward_context.attn_metadata + + # Stack QKV + qkvg = torch.cat( + [q, k, v, gate_compress], dim=0 + ) # [3, seq_len, num_heads, head_dim] + + # Redistribute heads across sequence dimension + qkvg = sequence_model_parallel_all_to_all_4D(qkvg, scatter_dim=2, gather_dim=1) + + qkvg = self.attn_impl.preprocess_qkv(qkvg, ctx_attn_metadata) + + q, k, v, gate_compress = qkvg.chunk(4, dim=0) + output = self.attn_impl.forward( + q, k, v, gate_compress=gate_compress, attn_metadata=ctx_attn_metadata + ) # type: ignore[call-arg] + + # Apply backend-specific postprocess_output + output = self.attn_impl.postprocess_output(output, ctx_attn_metadata) + + output = sequence_model_parallel_all_to_all_4D( + output, scatter_dim=1, gather_dim=2 + ) + + return output + + +class LocalAttention(nn.Module): + """Attention layer.""" + + def __init__( + self, + num_heads: int, + head_size: int, + num_kv_heads: int | None = None, + softmax_scale: float | None = None, + causal: bool = False, + supported_attention_backends: set[AttentionBackendEnum] | None = None, + **extra_impl_args, + ) -> None: + super().__init__() + if softmax_scale is None: + self.softmax_scale = head_size**-0.5 + else: + self.softmax_scale = softmax_scale + if num_kv_heads is None: + num_kv_heads = num_heads + + dtype = get_compute_dtype() + attn_backend = get_attn_backend( + head_size, dtype, supported_attention_backends=supported_attention_backends + ) + impl_cls = attn_backend.get_impl_cls() + self.attn_impl = impl_cls( + num_heads=num_heads, + head_size=head_size, + softmax_scale=self.softmax_scale, + num_kv_heads=num_kv_heads, + causal=causal, + **extra_impl_args, + ) + self.num_heads = num_heads + self.head_size = head_size + self.num_kv_heads = num_kv_heads + self.backend = backend_name_to_enum(attn_backend.get_name()) + self.dtype = dtype + + def forward( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + ) -> torch.Tensor: + """ + Apply local attention between query, key and value tensors. + + Args: + q (torch.Tensor): Query tensor of shape [batch_size, seq_len, num_heads, head_dim] + k (torch.Tensor): Key tensor of shape [batch_size, seq_len, num_heads, head_dim] + v (torch.Tensor): Value tensor of shape [batch_size, seq_len, num_heads, head_dim] + + Returns: + torch.Tensor: Output tensor after local attention + """ + # Check input shapes + assert q.dim() == 4 and k.dim() == 4 and v.dim() == 4, "Expected 4D tensors" + + forward_context: ForwardContext = get_forward_context() + ctx_attn_metadata = forward_context.attn_metadata + + output = self.attn_impl.forward(q, k, v, attn_metadata=ctx_attn_metadata) + return output + + +class USPAttention(nn.Module): + """ + Ulysses Sequence Parallelism with Ring Attention. + + This class implements the USP algorithm, which is a combination of + Ulysses-style all-to-all communication for sequence-head dimension sharding + and Ring Attention for fine-grained sequence parallelism within subgroups. + """ + + def __init__( + self, + num_heads: int, + head_size: int, + num_kv_heads: int | None = None, + softmax_scale: float | None = None, + causal: bool = False, + supported_attention_backends: set[AttentionBackendEnum] | None = None, + prefix: str = "", + dropout_rate: float = 0.0, + **extra_impl_args, + ) -> None: + super().__init__() + if softmax_scale is None: + self.softmax_scale = head_size**-0.5 + else: + self.softmax_scale = softmax_scale + + if num_kv_heads is None: + num_kv_heads = num_heads + + dtype = get_compute_dtype() + attn_backend = get_attn_backend( + head_size, dtype, supported_attention_backends=supported_attention_backends + ) + impl_cls: Type["AttentionImpl"] = attn_backend.get_impl_cls() + self.attn_impl = impl_cls( + num_heads=num_heads, + head_size=head_size, + causal=causal, + softmax_scale=self.softmax_scale, + num_kv_heads=num_kv_heads, + prefix=f"{prefix}.impl", + **extra_impl_args, + ) + self.num_heads = num_heads + self.head_size = head_size + self.num_kv_heads = num_kv_heads + self.backend = backend_name_to_enum(attn_backend.get_name()) + self.dtype = dtype + self.causal = causal + self.dropout_p = dropout_rate + + def forward( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + replicated_q: torch.Tensor | None = None, + replicated_k: torch.Tensor | None = None, + replicated_v: torch.Tensor | None = None, + ) -> torch.Tensor: + """ + Forward pass for USPAttention. + + q, k, v: [B, S_local, H, D] + + Note: Replicated tensors are not supported in this implementation. + """ + assert ( + replicated_q is None and replicated_k is None and replicated_v is None + ), "USPAttention does not support replicated_qkv." + forward_context: ForwardContext = get_forward_context() + ctx_attn_metadata = forward_context.attn_metadata + if get_sequence_parallel_world_size() == 1: + # No sequence parallelism, just run local attention. + out = self.attn_impl.forward(q, k, v, ctx_attn_metadata) + return out + + # Ulysses-style All-to-All for sequence/head sharding + if get_ulysses_parallel_world_size() > 1: + # -> [B, S, H_local, D] + q = _usp_input_all_to_all(q, head_dim=2) + k = _usp_input_all_to_all(k, head_dim=2) + v = _usp_input_all_to_all(v, head_dim=2) + + # Ring Attention within subgroups or local attention + if get_ring_parallel_world_size() > 1: + out = ring_attn( + q, + k, + v, + attn_impl=self.attn_impl, + is_causal=self.causal, + dropout_p=self.dropout_p, + ) + else: + # -> [B, S, H_local, D] + out = self.attn_impl.forward(q, k, v, ctx_attn_metadata) + + # Ulysses-style All-to-All to restore original sharding + if get_ulysses_parallel_world_size() > 1: + # -> [B, S_local, H, D] + out = _usp_output_all_to_all(out, head_dim=2) + + return out diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/selector.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/selector.py new file mode 100644 index 00000000..b5d589f7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/attention/selector.py @@ -0,0 +1,197 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/attention/selector.py + +import os +from collections.abc import Generator +from contextlib import contextmanager +from functools import cache +from typing import cast + +import torch + +from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import ( + AttentionBackend, +) +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum +from sglang.multimodal_gen.runtime.server_args import get_global_server_args +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.utils import STR_BACKEND_ENV_VAR, resolve_obj_by_qualname + +logger = init_logger(__name__) + + +def backend_name_to_enum(backend_name: str) -> AttentionBackendEnum | None: + """ + Convert a string backend name to a _Backend enum value. + + Returns: + * _Backend: enum value if backend_name is a valid in-tree type + * None: otherwise it's an invalid in-tree type or an out-of-tree platform is + loaded. + """ + assert backend_name is not None + return ( + AttentionBackendEnum[backend_name] + if backend_name in AttentionBackendEnum.__members__ + else None + ) + + +def get_env_variable_attn_backend() -> AttentionBackendEnum | None: + """ + Get the backend override specified by the sglang-diffusion attention + backend environment variable, if one is specified. + + Returns: + + * _Backend enum value if an override is specified + * None otherwise + """ + backend_name = os.environ.get(STR_BACKEND_ENV_VAR) + return None if backend_name is None else backend_name_to_enum(backend_name) + + +# Global state allows a particular choice of backend +# to be forced, overriding the logic which auto-selects +# a backend based on system & workload configuration +# (default behavior if this variable is None) +# +# THIS SELECTION TAKES PRECEDENCE OVER THE +# FASTVIDEO ATTENTION BACKEND ENVIRONMENT VARIABLE +forced_attn_backend: AttentionBackendEnum | None = None + + +def global_force_attn_backend(attn_backend: AttentionBackendEnum | None) -> None: + """ + Force all attention operations to use a specified backend. + + Passing `None` for the argument re-enables automatic + backend selection., + + Arguments: + + * attn_backend: backend selection (None to revert to auto) + """ + global forced_attn_backend + forced_attn_backend = attn_backend + + +def get_global_forced_attn_backend() -> AttentionBackendEnum | None: + """ + Get the currently-forced choice of attention backend, + or None if auto-selection is currently enabled. + """ + return forced_attn_backend + + +def get_attn_backend( + head_size: int, + dtype: torch.dtype, + supported_attention_backends: set[AttentionBackendEnum] | None = None, +) -> type[AttentionBackend]: + if supported_attention_backends is not None: + # Sort the backend names to ensure consistent cache key + be_tuple = tuple( + sorted(list(supported_attention_backends), key=lambda b: b.name) + ) + else: + be_tuple = None + return _cached_get_attn_backend(head_size, dtype, be_tuple) + + +@cache +def _cached_get_attn_backend( + head_size: int, + dtype: torch.dtype, + supported_attention_backends: tuple[AttentionBackendEnum] | None = None, +) -> type[AttentionBackend]: + # Check whether a particular choice of backend was + # previously forced. + # + # THIS SELECTION OVERRIDES THE SGLANG_DIFFUSION_ATTENTION_BACKEND + # ENVIRONMENT VARIABLE. + from sglang.multimodal_gen.runtime.platforms import current_platform + + supported_attention_backends = set(supported_attention_backends) + if not supported_attention_backends: + raise ValueError("supported_attention_backends is empty") + selected_backend = None + backend_by_global_setting: AttentionBackendEnum | None = ( + get_global_forced_attn_backend() + ) + if backend_by_global_setting is not None: + selected_backend = backend_by_global_setting + else: + # Check the server arguments for a backend override + server_args = get_global_server_args() + if server_args.attention_backend is not None: + try: + selected_backend = AttentionBackendEnum[ + server_args.attention_backend.upper() + ] + + except KeyError: + raise ValueError( + f"Invalid attention backend '{server_args.attention_backend}' specified via command line. " + f"Available options are: {[e.name.lower() for e in AttentionBackendEnum]}" + ) + + # get device-specific attn_backend + if selected_backend is None: + logger.debug(f"Attention backend not specified") + elif ( + not supported_attention_backends + or selected_backend not in supported_attention_backends + ): + supported_attention_backends_str = [ + supported_attention_backend.__str__() + for supported_attention_backend in supported_attention_backends + ] + logger.debug( + f"Selected attention backend: '{selected_backend}' not in supported attention backends: {supported_attention_backends_str}" + ) + selected_backend = None + + attention_cls = current_platform.get_attn_backend_cls_str( + selected_backend, head_size, dtype + ) + if not attention_cls: + raise ValueError( + f"Invalid attention backend for {current_platform.device_name}" + ) + return cast(type[AttentionBackend], resolve_obj_by_qualname(attention_cls)) + + +@contextmanager +def global_force_attn_backend_context_manager( + attn_backend: AttentionBackendEnum, +) -> Generator[None, None, None]: + """ + Globally force a sglang-diffusion attention backend override within a + context manager, reverting the global attention backend + override to its prior state upon exiting the context + manager. + + Arguments: + + * attn_backend: attention backend to force + + Returns: + + * Generator + """ + + # Save the current state of the global backend override (if any) + original_value = get_global_forced_attn_backend() + + # Globally force the new backend override + global_force_attn_backend(attn_backend) + + # Yield control back to the enclosed code block + try: + yield + finally: + # Revert the original global backend override, if any + global_force_attn_backend(original_value) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/custom_op.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/custom_op.py new file mode 100644 index 00000000..abc2f123 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/custom_op.py @@ -0,0 +1,110 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/model_executor/custom_op.py + +from collections.abc import Callable +from typing import Any + +import torch.nn as nn + +from sglang.multimodal_gen.runtime.utils.common import ( + is_cpu, + is_cuda, + is_hip, + is_npu, + is_xpu, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + +_is_cuda = is_cuda() +_is_hip = is_hip() +_is_cpu = is_cpu() +_is_npu = is_npu() +_is_xpu = is_xpu() + + +class CustomOp(nn.Module): + """ + Base class for custom ops. + Dispatches the forward method to the appropriate backend. + """ + + def __init__(self) -> None: + super().__init__() + self._forward_method = self.dispatch_forward() + + def forward(self, *args, **kwargs) -> Any: + return self._forward_method(*args, **kwargs) + + def forward_native(self, *args, **kwargs) -> Any: + """PyTorch-native implementation of the forward method. + This method is optional. If implemented, it can be used with compilers + such as torch.compile or PyTorch XLA. Also, it can be used for testing + purposes. + """ + raise NotImplementedError + + def forward_cuda(self, *args, **kwargs) -> Any: + raise NotImplementedError + + def forward_cpu(self, *args, **kwargs) -> Any: + # By default, we assume that CPU ops are compatible with CUDA ops. + return self.forward_cuda(*args, **kwargs) + + def forward_tpu(self, *args, **kwargs) -> Any: + # By default, we assume that TPU ops are compatible with the + # PyTorch-native implementation. + # NOTE(woosuk): This is a placeholder for future extensions. + return self.forward_native(*args, **kwargs) + + def forward_oot(self, *args, **kwargs) -> Any: + # By default, we assume that OOT ops are compatible with the + # PyTorch-native implementation. + return self.forward_native(*args, **kwargs) + + def dispatch_forward(self) -> Callable: + if _is_cuda: + return self.forward_cuda + elif _is_hip: + return self.forward_hip + elif _is_npu: + return self.forward_npu + elif _is_xpu: + return self.forward_xpu + else: + return self.forward_native + + @classmethod + def enabled(cls) -> bool: + # since we are not using Inductor, we always return True + return True + + @staticmethod + def default_on() -> bool: + """ + On by default if level < CompilationLevel.PIECEWISE + Specifying 'all' or 'none' in custom_op takes precedence. + """ + raise NotImplementedError + + # Dictionary of all custom ops (classes, indexed by registered name). + # To check if an op with a name is enabled, call .enabled() on the class. + # Examples: + # - MyOp.enabled() + # - op_registry["my_op"].enabled() + op_registry: dict[str, type["CustomOp"]] = {} + + # Decorator to register custom ops. + @classmethod + def register(cls, name: str) -> Callable: + + def decorator(op_cls): + assert name not in cls.op_registry, f"Duplicate op name: {name}" + op_cls.name = name + cls.op_registry[name] = op_cls + return op_cls + + return decorator diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/layernorm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/layernorm.py new file mode 100644 index 00000000..ec8f680a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/layernorm.py @@ -0,0 +1,429 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/model_executor/layers/layernorm.py +"""Custom normalization layers.""" +from typing import Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from sglang.multimodal_gen.runtime.layers.custom_op import CustomOp +from sglang.multimodal_gen.runtime.layers.triton_ops import ( + fuse_scale_shift_kernel, + norm_infer, + rms_norm_fn, +) +from sglang.multimodal_gen.runtime.utils.common import ( + get_bool_env_var, + is_cpu, + is_cuda, + is_hip, + is_npu, + is_xpu, +) + +_is_cuda = is_cuda() +_is_hip = is_hip() +_is_npu = is_npu() +_is_cpu = is_cpu() +_is_xpu = is_xpu() + +from sgl_kernel import fused_add_rmsnorm, rmsnorm + + +# Copied and adapted from sglang +@CustomOp.register("rms_norm") +class RMSNorm(CustomOp): + """Root mean square normalization. + + Computes x -> w * x / sqrt(E[x^2] + eps) where w is the learned weight. + Refer to https://arxiv.org/abs/1910.07467 + """ + + def __init__( + self, + hidden_size: int, + eps: float = 1e-6, + dtype: torch.dtype = torch.float32, + var_hidden_size: Optional[int] = None, + ) -> None: + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + self.hidden_size = hidden_size + self.variance_size_override = ( + None if var_hidden_size == hidden_size else var_hidden_size + ) + if get_bool_env_var("SGLANG_ENABLE_DETERMINISTIC_INFERENCE"): + self._forward_method = self.forward_native + + def forward_triton(self, x: torch.Tensor, residual: Optional[torch.Tensor] = None): + return rms_norm_fn( + x, self.weight, bias=None, residual=residual, eps=self.variance_epsilon + ) + + def forward_cuda( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + shape = x.shape + x = x.reshape(-1, shape[-1]) + if residual is not None: + residual_shape = residual.shape + residual = residual.view(-1, shape[-1]) + + if x.dtype == torch.float: + # fp32 + out = self.forward_triton(x, residual) + elif self.variance_size_override is not None: + return self.forward_native(x, residual) + elif residual is not None: + fused_add_rmsnorm(x, residual, self.weight.data, self.variance_epsilon) + return x.view(shape), residual.view(residual_shape) + else: + out = rmsnorm(x, self.weight.data, self.variance_epsilon) + out = out.view(shape) + return out + + def forward_native( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + if not x.is_contiguous(): + x = x.contiguous() + orig_dtype = x.dtype + x = x.to(torch.float32) + if residual is not None: + x = x + residual.to(torch.float32) + residual = x.to(orig_dtype) + + hidden_size = x.shape[-1] + if hidden_size != self.hidden_size: + raise ValueError( + "Expected hidden_size to be " + f"{self.hidden_size}, but found: {hidden_size}" + ) + + if self.variance_size_override is None: + x_var = x + else: + if hidden_size < self.variance_size_override: + raise ValueError( + "Expected hidden_size to be at least " + f"{self.variance_size_override}, but found: {hidden_size}" + ) + + x_var = x[..., : self.variance_size_override] + + variance = x_var.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + self.variance_epsilon) + x = (x * self.weight).to(orig_dtype) + if residual is None: + return x + else: + return x, residual + + def forward_cpu( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + return self.forward_native(x, residual) + + def extra_repr(self) -> str: + s = f"hidden_size={self.weight.data.size(0)}" + s += f", eps={self.variance_epsilon}" + return s + + +# Copied and adapted from sglang +@CustomOp.register("layer_norm") +class LayerNorm(CustomOp): + def __init__( + self, + hidden_size: int, + eps=1e-5, + bias: bool = True, + elementwise_affine=True, + device=None, + dtype=None, + ) -> None: + super().__init__() + self.eps = eps + factory_kwargs = {"device": device, "dtype": dtype} + self.hidden_size = hidden_size + if elementwise_affine: + self.weight = torch.nn.Parameter(torch.empty(hidden_size, **factory_kwargs)) + self.bias = ( + torch.nn.Parameter(torch.empty(hidden_size, **factory_kwargs)) + if bias + else None + ) + else: + self.register_parameter("weight", None) + self.register_parameter("bias", None) + # Lazy cache for ones vector (not a registered buffer to avoid FSDP/meta issues) + self._weight_fallback_cache = None + + def _get_weight_fallback(self, x: torch.Tensor) -> torch.Tensor: + wf = getattr(self, "_weight_fallback_cache", None) + if ( + wf is None + or wf.device != x.device + or wf.dtype != x.dtype + or wf.numel() != self.hidden_size + ): + wf = torch.ones(self.hidden_size, device=x.device, dtype=x.dtype) + self._weight_fallback_cache = wf + return wf + + def forward_triton(self, x: torch.Tensor): + # Fast inference kernel without residual/dropout branches + return norm_infer( + x.view(-1, self.hidden_size), + self.weight, + self.bias, + eps=self.eps, + is_rms_norm=False, + ).view(x.shape) + + def forward_cuda( + self, + x: torch.Tensor, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + shape = x.shape + x = x.view(-1, self.hidden_size) + return self.forward_triton(x).view(shape) + + @torch.compile(backend="inductor") + def forward_native( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + input_dtype = x.dtype + mean = x.mean(-1, keepdim=True) + variance = (x - mean).pow(2).mean(-1, keepdim=True) + x = (x - mean) * torch.rsqrt(variance + self.eps) + if self.weight is not None: + x = self.weight * x + # if no affine, this is a no-op + if self.bias is not None: + x = x + self.bias + return x.to(input_dtype) + + def forward_cpu( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + return self.forward_native(x, residual) + + def extra_repr(self) -> str: + s = f"hidden_size={self.weight.data.size(0)}" + s += f", eps={self.variance_epsilon}" + return s + + +class ScaleResidual(nn.Module): + """ + Applies gated residual connection. + """ + + def __init__(self, prefix: str = ""): + super().__init__() + + def forward( + self, residual: torch.Tensor, x: torch.Tensor, gate: torch.Tensor + ) -> torch.Tensor: + """Apply gated residual connection.""" + # x.shape: [batch_size, seq_len, inner_dim] + if gate.dim() == 4: + # gate.shape: [batch_size, num_frames, 1, inner_dim] + num_frames = gate.shape[1] + frame_seqlen = x.shape[1] // num_frames + return residual + ( + x.unflatten(dim=1, sizes=(num_frames, frame_seqlen)) * gate + ).flatten(1, 2) + else: + # gate.shape: [batch_size, 1, inner_dim] + return residual + x * gate + + +# adapted from Diffusers: https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/normalization.py +# NOTE(will): Needed to match behavior of diffusers and wan2.1 even while using +# FSDP's MixedPrecisionPolicy +class FP32LayerNorm(nn.LayerNorm): + def forward(self, inputs: torch.Tensor) -> torch.Tensor: + origin_dtype = inputs.dtype + return F.layer_norm( + inputs.float(), + self.normalized_shape, + self.weight.float() if self.weight is not None else None, + self.bias.float() if self.bias is not None else None, + self.eps, + ).to(origin_dtype) + + +class ScaleResidualLayerNormScaleShift(nn.Module): + """ + Fused operation that combines: + 1. Gated residual connection + 2. LayerNorm + 3. Scale and shift operations + + This reduces memory bandwidth by combining memory-bound operations. + """ + + def __init__( + self, + hidden_size: int, + norm_type: str = "rms", + eps: float = 1e-6, + elementwise_affine: bool = False, + dtype: torch.dtype = torch.float32, + compute_dtype: torch.dtype | None = None, + prefix: str = "", + ): + super().__init__() + if norm_type == "rms": + self.norm = RMSNorm( + hidden_size, has_weight=elementwise_affine, eps=eps, dtype=dtype + ) + elif norm_type == "layer": + if compute_dtype == torch.float32: + self.norm = FP32LayerNorm( + hidden_size, elementwise_affine=elementwise_affine, eps=eps + ) + else: + self.norm = LayerNorm( + hidden_size, + elementwise_affine=elementwise_affine, + eps=eps, + dtype=dtype, + ) + else: + raise NotImplementedError(f"Norm type {norm_type} not implemented") + + def forward( + self, + residual: torch.Tensor, + x: torch.Tensor, + gate: torch.Tensor | int, + shift: torch.Tensor, + scale: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + """ + Apply gated residual connection, followed by layernorm and + scale/shift in a single fused operation. + + Returns: + Tuple containing: + - normalized and modulated output of shape: [batch_size, seq_len, inner_dim] + - residual value (value after residual connection + but before normalization) + """ + # x.shape: [batch_size, seq_len, inner_dim] + # Apply residual connection with gating + if isinstance(gate, int): + # used by cross-attention, should be 1 + assert gate == 1 + residual_output = residual + x + elif isinstance(gate, torch.Tensor): + if gate.dim() == 4: + # gate.shape: [batch_size, num_frames, 1, inner_dim] + num_frames = gate.shape[1] + frame_seqlen = x.shape[1] // num_frames + residual_output = residual + ( + x.unflatten(dim=1, sizes=(num_frames, frame_seqlen)) * gate + ).flatten(1, 2) + else: + # used by bidirectional self attention + # gate.shape: [batch_size, 1, inner_dim] + residual_output = residual + x * gate + else: + raise ValueError(f"Gate type {type(gate)} not supported") + # residual_output.shape: [batch_size, seq_len, inner_dim] + + # Apply normalization + normalized = self.norm(residual_output) + + # modulated = fused_scale_shift( + # normalized, + # scale, + # shift, + # ) + modulated = fuse_scale_shift_kernel( + normalized, + scale, + shift, + ) + return modulated, residual_output + + +class LayerNormScaleShift(nn.Module): + """ + Fused operation that combines LayerNorm with scale and shift operations. + This reduces memory bandwidth by combining memory-bound operations. + """ + + def __init__( + self, + hidden_size: int, + norm_type: str = "rms", + eps: float = 1e-6, + elementwise_affine: bool = False, + dtype: torch.dtype = torch.float32, + compute_dtype: torch.dtype | None = None, + prefix: str = "", + ): + super().__init__() + self.compute_dtype = compute_dtype + if norm_type == "rms": + self.norm = RMSNorm(hidden_size, has_weight=elementwise_affine, eps=eps) + elif norm_type == "layer": + if self.compute_dtype == torch.float32: + self.norm = FP32LayerNorm( + hidden_size, elementwise_affine=elementwise_affine, eps=eps + ) + else: + self.norm = nn.LayerNorm( + hidden_size, + elementwise_affine=elementwise_affine, + eps=eps, + dtype=dtype, + ) + else: + raise NotImplementedError(f"Norm type {norm_type} not implemented") + + def forward( + self, x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor + ) -> torch.Tensor: + """Apply ln followed by scale and shift in a single fused operation.""" + # x.shape: [batch_size, seq_len, inner_dim] + normalized = self.norm(x) + if self.compute_dtype == torch.float32: + normalized = normalized.float() + + if scale.dim() == 4: + # scale.shape: [batch_size, num_frames, 1, inner_dim] + num_frames = scale.shape[1] + frame_seqlen = normalized.shape[1] // num_frames + output = ( + normalized.unflatten(dim=1, sizes=(num_frames, frame_seqlen)) + * (1.0 + scale) + + shift + ).flatten(1, 2) + else: + # scale.shape: [batch_size, 1, inner_dim] + # shift.shape: [batch_size, 1, inner_dim] + output = normalized * (1.0 + scale) + shift + + if self.compute_dtype == torch.float32: + output = output.to(x.dtype) + + return output diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/linear.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/linear.py new file mode 100644 index 00000000..65c71372 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/linear.py @@ -0,0 +1,1057 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/model_executor/layers/linear.py + +from abc import abstractmethod + +import torch +import torch.nn.functional as F +from torch.nn.parameter import Parameter + +from sglang.multimodal_gen.runtime.distributed import ( + divide, + get_tp_rank, + get_tp_world_size, + split_tensor_along_last_dim, + tensor_model_parallel_all_gather, + tensor_model_parallel_all_reduce, +) +from sglang.multimodal_gen.runtime.layers.quantization.base_config import ( + QuantizationConfig, + QuantizeMethodBase, +) + +# yapf: disable +from sglang.multimodal_gen.runtime.models.parameter import ( + BasevLLMParameter, + BlockQuantScaleParameter, + PackedColumnParameter, + PackedvLLMParameter, + PerTensorScaleParameter, + RowvLLMParameter, +) + +# yapf: enable +from sglang.multimodal_gen.runtime.models.utils import set_weight_attrs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + +WEIGHT_LOADER_V2_SUPPORTED = [ + "CompressedTensorsLinearMethod", + "AWQMarlinLinearMethod", + "AWQLinearMethod", + "GPTQMarlinLinearMethod", + "Fp8LinearMethod", + "MarlinLinearMethod", + "QQQLinearMethod", + "GPTQMarlin24LinearMethod", + "TPUInt8LinearMethod", + "GPTQLinearMethod", + "FBGEMMFp8LinearMethod", + "ModelOptFp8LinearMethod", + "IPEXAWQLinearMethod", + "IPEXGPTQLinearMethod", + "HQQMarlinMethod", + "QuarkLinearMethod", +] + + +def adjust_scalar_to_fused_array( + param: torch.Tensor, loaded_weight: torch.Tensor, shard_id: str | int +) -> tuple[torch.Tensor, torch.Tensor]: + """For fused modules (QKV and MLP) we have an array of length + N that holds 1 scale for each "logical" matrix. So the param + is an array of length N. The loaded_weight corresponds to + one of the shards on disk. Here, we slice the param based on + the shard_id for loading. + """ + qkv_idxs = {"q": 0, "k": 1, "v": 2} + + if isinstance(shard_id, str): + shard_id = qkv_idxs[shard_id] + elif not isinstance(shard_id, int): + raise ValueError(f"Unknown Shard Id {shard_id}") + + # AutoFP8 scales do not have a shape + # compressed-tensors scales do have a shape + if len(loaded_weight.shape) != 0: + assert loaded_weight.shape[0] == 1 + loaded_weight = loaded_weight[0] + + return param[shard_id], loaded_weight + + +class LinearMethodBase(QuantizeMethodBase): + """Base class for different (maybe quantized) linear methods.""" + + @abstractmethod + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: list[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ) -> None: + """Create weights for a linear layer. + The weights will be set as attributes of the layer. + + Args: + layer: The layer that is using the LinearMethodBase factory. + input_size_per_partition: Size of the weight input dim on rank X. + output_partition_sizes: Sizes of the output dim of each logical + weight on rank X. E.g., output_partition_sizes for QKVLinear + is a list contains the width of Wq, Wk, Wv on rank X. + input_size: Size of the input dim of the weight across all ranks. + output_size: Size of the output dim of the weight across all ranks. + params_dtype: Datatype of the parameters. + """ + raise NotImplementedError + + @abstractmethod + def apply( + self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None = None + ) -> torch.Tensor: + """Apply the weights in layer to the input tensor. + Expects create_weights to have been called before on the layer.""" + raise NotImplementedError + + +class UnquantizedLinearMethod(LinearMethodBase): + """Linear method without quantization.""" + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: list[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ) -> None: + weight = Parameter( + torch.empty( + sum(output_partition_sizes), + input_size_per_partition, + dtype=params_dtype, + ), + requires_grad=False, + ) + set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0}) + layer.register_parameter("weight", weight) + set_weight_attrs(weight, extra_weight_attrs) + + def apply( + self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None = None + ) -> torch.Tensor: + output = ( + F.linear(x, layer.weight, bias) + if torch.cuda.is_available() or bias is None + else F.linear(x, layer.weight, bias.to(x.dtype)) + ) # NOTE: this line assumes that we are using amp when using cuda and is needed to account for the fact that amp isn't supported in mps + return output + + +class LinearBase(torch.nn.Module): + """Base linear layer. + + Args: + input_size: input dimension of the linear layer. + output_size: output dimension of the linear layer. + bias: If true, add bias. + skip_bias_add: If true, skip adding bias but instead return it. + params_dtype: Data type for the parameters. + quant_config: Quantization configure. + """ + + def __init__( + self, + input_size: int, + output_size: int, + skip_bias_add: bool = False, + params_dtype: torch.dtype | None = None, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ): + super().__init__() + + # Keep input parameters + self.input_size = input_size + self.output_size = output_size + self.skip_bias_add = skip_bias_add + if params_dtype is None: + params_dtype = torch.get_default_dtype() + self.params_dtype = params_dtype + self.quant_config = quant_config + self.prefix = prefix + if quant_config is None: + self.quant_method: QuantizeMethodBase | None = UnquantizedLinearMethod() + else: + self.quant_method = quant_config.get_quant_method(self, prefix=prefix) + + def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, Parameter | None]: + raise NotImplementedError + + +class ReplicatedLinear(LinearBase): + """Replicated linear layer. + + Args: + input_size: input dimension of the linear layer. + output_size: output dimension of the linear layer. + bias: If true, add bias. + skip_bias_add: If true, skip adding bias but instead return it. + params_dtype: Data type for the parameters. + quant_config: Quantization configure. + prefix: The name of the layer in the state dict, including all parents + (e.g. model.layers.0.qkv_proj) + """ + + def __init__( + self, + input_size: int, + output_size: int, + bias: bool = True, + skip_bias_add: bool = False, + params_dtype: torch.dtype | None = None, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ): + super().__init__( + input_size, + output_size, + skip_bias_add, + params_dtype, + quant_config, + prefix=prefix, + ) + + # All the linear layer supports quant method. + assert self.quant_method is not None + self.quant_method.create_weights( + self, + self.input_size, + [self.output_size], + self.input_size, + self.output_size, + self.params_dtype, + weight_loader=self.weight_loader, + ) + + if bias: + self.bias = Parameter( + torch.empty( + self.output_size, + dtype=self.params_dtype, + ) + ) + set_weight_attrs( + self.bias, + { + "output_dim": 0, + "weight_loader": self.weight_loader, + }, + ) + else: + self.register_parameter("bias", None) + + def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor) -> None: + # If the weight on disk does not have a shape, give it one + # (such scales for AutoFp8). + if len(loaded_weight.shape) == 0: + loaded_weight = loaded_weight.reshape(1) + + assert param.size() == loaded_weight.size(), ( + f"Tried to load weights of size {loaded_weight.size()}" + f"to a parameter of size {param.size()}" + ) + param.data.copy_(loaded_weight) + + def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, Parameter | None]: + bias = self.bias if not self.skip_bias_add else None + assert self.quant_method is not None + output = self.quant_method.apply(self, x, bias) + output_bias = self.bias if self.skip_bias_add else None + return output, output_bias + + def extra_repr(self) -> str: + s = f"in_features={self.input_size}" + s += f", output_features={self.output_size}" + s += f", bias={self.bias is not None}" + return s + + +class ColumnParallelLinear(LinearBase): + """Linear layer with column parallelism. + + The linear layer is defined as Y = XA + b. A is parallelized along + its second dimension as A = [A_1, ..., A_p]. + + Args: + input_size: first dimension of matrix A. + output_size: second dimension of matrix A. + bias: If true, add bias. + gather_output: If true, call all-gather on output and make Y available + to all GPUs, otherwise, every GPU will have its output + which is Y_i = XA_i + skip_bias_add: This was added to enable performance optimizations where + bias can be fused with other element-wise operations. we + skip adding bias but instead return it. + params_dtype: Data type for the parameters. + quant_config: Quantization configure. + output_sizes: list of output sizes packed into one output, like for QKV + the list would be size 3. + prefix: The name of the layer in the state dict, including all parents + (e.g. model.layers.0.qkv_proj) + """ + + def __init__( + self, + input_size: int, + output_size: int, + bias: bool = True, + gather_output: bool = False, + skip_bias_add: bool = False, + params_dtype: torch.dtype | None = None, + quant_config: QuantizationConfig | None = None, + output_sizes: list[int] | None = None, + prefix: str = "", + ): + # Divide the weight matrix along the last dimension. + self.tp_size = get_tp_world_size() + self.input_size_per_partition = input_size + self.output_size_per_partition = divide(output_size, self.tp_size) + self.output_partition_sizes = [self.output_size_per_partition] + # If QKV or MergedColumn, use output size of each partition. + if hasattr(self, "output_sizes"): + self.output_partition_sizes = [ + divide(output_size, self.tp_size) for output_size in self.output_sizes + ] + + super().__init__( + input_size, output_size, skip_bias_add, params_dtype, quant_config, prefix + ) + + self.gather_output = gather_output + + if output_sizes is None: + output_sizes = [output_size] + + assert self.quant_method is not None + self.quant_method.create_weights( + layer=self, + input_size_per_partition=self.input_size_per_partition, + output_partition_sizes=self.output_partition_sizes, + input_size=self.input_size, + output_size=self.output_size, + params_dtype=self.params_dtype, + weight_loader=( + self.weight_loader_v2 + if self.quant_method.__class__.__name__ in WEIGHT_LOADER_V2_SUPPORTED + else self.weight_loader + ), + ) + if bias: + self.bias = Parameter( + torch.empty( + self.output_size_per_partition, + dtype=params_dtype, + ) + ) + set_weight_attrs( + self.bias, + { + "output_dim": 0, + "weight_loader": self.weight_loader, + }, + ) + else: + self.register_parameter("bias", None) + + def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor) -> None: + tp_rank = get_tp_rank() + output_dim = getattr(param, "output_dim", None) + + is_sharded_weight = getattr(param, "is_sharded_weight", False) + is_sharded_weight = is_sharded_weight + + param_data = param.data + if output_dim is not None and not is_sharded_weight: + shard_size = param_data.shape[output_dim] + start_idx = tp_rank * shard_size + loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size) + + # Special case for loading scales off disk, which often do not + # have a shape (such as in the case of AutoFP8). + if len(loaded_weight.shape) == 0: + loaded_weight = loaded_weight.reshape(1) + + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + def weight_loader_v2(self, param: Parameter, loaded_weight: torch.Tensor) -> None: + # Special case for loading scales off disk, which often do not + # have a shape (such as in the case of AutoFP8). + if len(loaded_weight.shape) == 0: + assert loaded_weight.numel() == 1 + loaded_weight = loaded_weight.reshape(1) + param.load_column_parallel_weight(loaded_weight=loaded_weight) + + def forward(self, input_: torch.Tensor) -> tuple[torch.Tensor, Parameter | None]: + bias = self.bias if not self.skip_bias_add else None + + # Matrix multiply. + assert self.quant_method is not None + output_parallel = self.quant_method.apply(self, input_, bias) + if self.gather_output: + # All-gather across the partitions. + output = tensor_model_parallel_all_gather(output_parallel) + else: + output = output_parallel + output_bias = self.bias if self.skip_bias_add else None + return output, output_bias + + def extra_repr(self) -> str: + s = f"in_features={self.input_size}" + s += f", output_features={self.output_size_per_partition}" + s += f", bias={self.bias is not None}" + s += f", tp_size={get_tp_world_size()}" + s += f", gather_output={self.gather_output}" + return s + + +class MergedColumnParallelLinear(ColumnParallelLinear): + """Packed linear layers with column parallelism. + + Similar to ColumnParallelLinear, but the weight matrix is concatenated + along the output dimension. When the weight matrix is loaded, the + different partitions are sharded separately. + + Args: + input_size: input dimension of the linear layer. + output_sizes: list of output dimensions of the linear layer. + bias: If true, add bias. + gather_output: If true, call all-gather on output and make the output + available to all GPUs, otherwise, every GPU will have + its own output. + skip_bias_add: This was added to enable performance optimizations where + bias can be fused with other element-wise operations. we + skip adding bias but instead return it. + params_dtype: Data type for the parameters. + quant_config: Quantization configure. + prefix: The name of the layer in the state dict, including all parents + (e.g. model.layers.0.qkv_proj) + """ + + def __init__( + self, + input_size: int, + output_sizes: list[int], + bias: bool = True, + gather_output: bool = False, + skip_bias_add: bool = False, + params_dtype: torch.dtype | None = None, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ): + self.output_sizes = output_sizes + tp_size = get_tp_world_size() + assert all(output_size % tp_size == 0 for output_size in output_sizes) + super().__init__( + input_size=input_size, + output_size=sum(output_sizes), + bias=bias, + gather_output=gather_output, + skip_bias_add=skip_bias_add, + params_dtype=params_dtype, + quant_config=quant_config, + prefix=prefix, + ) + + def weight_loader( + self, + param: Parameter, + loaded_weight: torch.Tensor, + loaded_shard_id: int | None = None, + ) -> None: + + param_data = param.data + output_dim = getattr(param, "output_dim", None) + # Special case for AQLM codebooks. + is_metadata = getattr(param, "is_metadata", False) + # Special case for per-tensor scale to load scalar into fused array. + needs_scalar_to_array = getattr(param, "needs_scalar_to_array", False) + + if loaded_shard_id is None: + # Loaded weight is already fused on disk (mlp). + # (e.g., Phi-3's gate_up_proj). + if output_dim is None: + if needs_scalar_to_array: + param_data, loaded_weight = adjust_scalar_to_fused_array( + param_data, loaded_weight, 0 + ) + + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + return + current_shard_offset = 0 + shard_offsets: list[tuple[int, int, int]] = [] + for i, output_size in enumerate(self.output_sizes): + shard_offsets.append((i, current_shard_offset, output_size)) + current_shard_offset += output_size + for shard_id, shard_offset, shard_size in shard_offsets: + loaded_weight_shard = loaded_weight.narrow( + output_dim, shard_offset, shard_size + ) + self.weight_loader(param, loaded_weight_shard, shard_id) + return + + assert loaded_shard_id < len(self.output_sizes) + tp_rank = get_tp_rank() + tp_size = get_tp_world_size() + if output_dim is not None: + shard_offset = sum(self.output_sizes[:loaded_shard_id]) // tp_size + shard_size = self.output_sizes[loaded_shard_id] // tp_size + + is_sharded_weight = getattr(param, "is_sharded_weight", False) + # bitsandbytes loads the weights of the specific portion + # no need to narrow + is_sharded_weight = is_sharded_weight + + param_data = param_data.narrow(output_dim, shard_offset, shard_size) + start_idx = tp_rank * shard_size + if not is_sharded_weight: + loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size) + # Special case for AQLM codebooks. + elif is_metadata: + # metadata indicates fixed size concatenated along dim 0 + shard_size = loaded_weight.shape[0] + shard_offset = loaded_shard_id * shard_size + param_data = param_data.narrow(0, shard_offset, shard_size) + + # Special case for per-tensor scales in fused case. + elif needs_scalar_to_array: + param_data, loaded_weight = adjust_scalar_to_fused_array( + param_data, loaded_weight, loaded_shard_id + ) + + else: + ignore_warning = getattr(param, "ignore_warning", False) + if not ignore_warning: + logger.warning( + "Loading a weight without `output_dim` attribute in " + "MergedColumnParallelLinear, assume the weight is " + "the same for all partitions." + ) + + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + def _load_fused_module_from_checkpoint( + self, param: BasevLLMParameter, loaded_weight: torch.Tensor + ) -> None: + """ + Handle special case for models where MLP layers are already + fused on disk. In this case, we have no shard id. This function + determmines the shard id by splitting these layers and then calls + the weight loader using the shard id. + + An example of a model with these fused layers: + https://huggingface.co/microsoft/Phi-3-mini-4k-instruct + """ + + current_shard_offset = 0 + shard_offsets: list[tuple[int, int, int]] = [] + for i, output_size in enumerate(self.output_sizes): + shard_offsets.append((i, current_shard_offset, output_size)) + current_shard_offset += output_size + + for shard_id, shard_offset, shard_size in shard_offsets: + # Special case for Quantization. + # If quantized, we need to adjust the offset and size to account + # for the packing. + if ( + isinstance(param, PackedColumnParameter | PackedvLLMParameter) + and param.packed_dim == param.output_dim + ): + shard_size, shard_offset = param.adjust_shard_indexes_for_packing( + shard_size=shard_size, shard_offset=shard_offset + ) + + loaded_weight_shard = loaded_weight.narrow( + param.output_dim, shard_offset, shard_size + ) + self.weight_loader_v2(param, loaded_weight_shard, shard_id) + + def weight_loader_v2( + self, + param: BasevLLMParameter, + loaded_weight: torch.Tensor, + loaded_shard_id: int | None = None, + ) -> None: + if loaded_shard_id is None: + if isinstance(param, PerTensorScaleParameter): + param.load_merged_column_weight(loaded_weight=loaded_weight, shard_id=0) + return + elif type(param) in (RowvLLMParameter, BasevLLMParameter): + param.load_merged_column_weight(loaded_weight=loaded_weight) + return + # TODO: @dsikka - move to parameter.py + self._load_fused_module_from_checkpoint(param, loaded_weight) + return + + assert loaded_shard_id < len(self.output_sizes) + + tp_size = get_tp_world_size() + + if isinstance(param, BlockQuantScaleParameter): + raise NotImplementedError("FP8 is not implemented yet") + # FIXME(will): add fp8 support + # from vllm.model_executor.layers.quantization.fp8 import ( + # Fp8LinearMethod, Fp8MoEMethod) + # assert self.quant_method is not None + # assert isinstance(self.quant_method, + # (Fp8LinearMethod, Fp8MoEMethod)) + # weight_block_size = self.quant_method.quant_config.weight_block_size + # assert weight_block_size is not None + # block_n, _ = weight_block_size[0], weight_block_size[1] + # shard_offset = ( + # (sum(self.output_sizes[:loaded_shard_id]) + block_n - 1) // + # block_n) // tp_size + # shard_size = ((self.output_sizes[loaded_shard_id] + block_n - 1) // + # block_n // tp_size) + else: + shard_offset = sum(self.output_sizes[:loaded_shard_id]) // tp_size + shard_size = self.output_sizes[loaded_shard_id] // tp_size + + param.load_merged_column_weight( + loaded_weight=loaded_weight, + shard_id=loaded_shard_id, + shard_offset=shard_offset, + shard_size=shard_size, + ) + + +class QKVParallelLinear(ColumnParallelLinear): + """Linear layers for the attention's QKV transformation. + + Linear layers for the linear transformation of the query, key, and value + vectors in the attention layer. The weight matrix is concatenated along + the output dimension. The layer is parallelized along the head dimension. + When the number of key/value heads is smaller than the number of query + heads (e.g., multi-query/grouped-query attention), the key/value head may + be replicated while the query heads are partitioned. + + Args: + hidden_size: input hidden state size of the transformer. + head_size: size of each attention head. + total_num_heads: total number of attention query heads. + total_num_kv_heads: total number of attention key/value heads. If + None, assume total_num_kv_heads = total_num_heads. + bias: If true, add bias. + skip_bias_add: This was added to enable performance optimizations where + bias can be fused with other element-wise operations. we + skip adding bias but instead return it. + params_dtype: Data type for the parameters. + quant_config: Quantization configure. + prefix: The name of the layer in the state dict, including all parents + (e.g. model.layers.0.qkv_proj) + """ + + def __init__( + self, + hidden_size: int, + head_size: int, + total_num_heads: int, + total_num_kv_heads: int | None = None, + bias: bool = True, + skip_bias_add: bool = False, + params_dtype: torch.dtype | None = None, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ): + self.hidden_size = hidden_size + self.head_size = head_size + self.total_num_heads = total_num_heads + if total_num_kv_heads is None: + total_num_kv_heads = total_num_heads + self.total_num_kv_heads = total_num_kv_heads + # Divide the weight matrix along the last dimension. + tp_size = get_tp_world_size() + self.num_heads = divide(self.total_num_heads, tp_size) + if tp_size >= self.total_num_kv_heads: + self.num_kv_heads = 1 + self.num_kv_head_replicas = divide(tp_size, self.total_num_kv_heads) + else: + self.num_kv_heads = divide(self.total_num_kv_heads, tp_size) + self.num_kv_head_replicas = 1 + input_size = self.hidden_size + output_size = ( + (self.num_heads + 2 * self.num_kv_heads) * tp_size * self.head_size + ) + self.output_sizes = [ + self.num_heads * self.head_size * tp_size, # q_proj + self.num_kv_heads * self.head_size * tp_size, # k_proj + self.num_kv_heads * self.head_size * tp_size, # v_proj + ] + + super().__init__( + input_size=input_size, + output_size=output_size, + bias=bias, + gather_output=False, + skip_bias_add=skip_bias_add, + params_dtype=params_dtype, + quant_config=quant_config, + prefix=prefix, + ) + + def _get_shard_offset_mapping(self, loaded_shard_id: str) -> int | None: + shard_offset_mapping = { + "q": 0, + "k": self.num_heads * self.head_size, + "v": (self.num_heads + self.num_kv_heads) * self.head_size, + "total": (self.num_heads + 2 * self.num_kv_heads) * self.head_size, + } + return shard_offset_mapping.get(loaded_shard_id) + + def _get_shard_size_mapping(self, loaded_shard_id: str) -> int | None: + shard_size_mapping = { + "q": self.num_heads * self.head_size, + "k": self.num_kv_heads * self.head_size, + "v": self.num_kv_heads * self.head_size, + } + return shard_size_mapping.get(loaded_shard_id) + + def _load_fused_module_from_checkpoint( + self, param: BasevLLMParameter, loaded_weight: torch.Tensor + ): + """ + Handle special case for models where QKV layers are already + fused on disk. In this case, we have no shard id. This function + determmines the shard id by splitting these layers and then calls + the weight loader using the shard id. + + An example of a model with these fused layers: + https://huggingface.co/microsoft/Phi-3-mini-4k-instruct + """ + shard_offsets = [ + # (shard_id, shard_offset, shard_size) + ("q", 0, self.total_num_heads * self.head_size), + ( + "k", + self.total_num_heads * self.head_size, + self.total_num_kv_heads * self.head_size, + ), + ( + "v", + (self.total_num_heads + self.total_num_kv_heads) * self.head_size, + self.total_num_kv_heads * self.head_size, + ), + ] + + for shard_id, shard_offset, shard_size in shard_offsets: + # Special case for Quantization. + # If quantized, we need to adjust the offset and size to account + # for the packing. + if ( + isinstance(param, PackedColumnParameter | PackedvLLMParameter) + and param.packed_dim == param.output_dim + ): + shard_size, shard_offset = param.adjust_shard_indexes_for_packing( + shard_size=shard_size, shard_offset=shard_offset + ) + + loaded_weight_shard = loaded_weight.narrow( + param.output_dim, shard_offset, shard_size + ) + self.weight_loader_v2(param, loaded_weight_shard, shard_id) + + def weight_loader_v2( + self, + param: BasevLLMParameter, + loaded_weight: torch.Tensor, + loaded_shard_id: str | None = None, + ): + if loaded_shard_id is None: # special case for certain models + if isinstance(param, PerTensorScaleParameter): + param.load_qkv_weight(loaded_weight=loaded_weight, shard_id=0) + return + elif type(param) in (RowvLLMParameter, BasevLLMParameter): + param.load_qkv_weight(loaded_weight=loaded_weight) + return + # TODO: @dsikka - move to parameter.py + self._load_fused_module_from_checkpoint(param, loaded_weight) + return + + assert loaded_shard_id in ["q", "k", "v"] + + shard_offset = self._get_shard_offset_mapping(loaded_shard_id) + shard_size = self._get_shard_size_mapping(loaded_shard_id) + + param.load_qkv_weight( + loaded_weight=loaded_weight, + num_heads=self.num_kv_head_replicas, + shard_id=loaded_shard_id, + shard_offset=shard_offset, + shard_size=shard_size, + ) + + def weight_loader( + self, + param: Parameter, + loaded_weight: torch.Tensor, + loaded_shard_id: str | None = None, + ): + + param_data = param.data + output_dim = getattr(param, "output_dim", None) + # Special case for AQLM codebooks. + is_metadata = getattr(param, "is_metadata", False) + + # Special case for per-tensor scales in fused case. + needs_scalar_to_array = getattr(param, "needs_scalar_to_array", False) + + if loaded_shard_id is None: + # Loaded weight is already fused on disk (qkv). + # (e.g., Phi-3's qkv_proj). + if output_dim is None: + if needs_scalar_to_array: + param_data, loaded_weight = adjust_scalar_to_fused_array( + param_data, loaded_weight, 0 + ) + + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + return + shard_offsets = [ + # (shard_id, shard_offset, shard_size) + ("q", 0, self.total_num_heads * self.head_size), + ( + "k", + self.total_num_heads * self.head_size, + self.total_num_kv_heads * self.head_size, + ), + ( + "v", + (self.total_num_heads + self.total_num_kv_heads) * self.head_size, + self.total_num_kv_heads * self.head_size, + ), + ] + + for shard_id, shard_offset, shard_size in shard_offsets: + + loaded_weight_shard = loaded_weight.narrow( + output_dim, shard_offset, shard_size + ) + self.weight_loader(param, loaded_weight_shard, shard_id) + return + + tp_rank = get_tp_rank() + assert loaded_shard_id in ["q", "k", "v"] + + # If output dim is defined, use the default loading process. + if output_dim is not None: + if loaded_shard_id == "q": + shard_offset = 0 + shard_size = self.num_heads * self.head_size + elif loaded_shard_id == "k": + shard_offset = self.num_heads * self.head_size + shard_size = self.num_kv_heads * self.head_size + elif loaded_shard_id == "v": + shard_offset = (self.num_heads + self.num_kv_heads) * self.head_size + shard_size = self.num_kv_heads * self.head_size + + is_sharded_weight = getattr(param, "is_sharded_weight", False) + # bitsandbytes loads the weights of the specific portion + # no need to narrow + is_sharded_weight = is_sharded_weight + + shard_idx = 0 + param_data = param_data.narrow(output_dim, shard_offset, shard_size) + if loaded_shard_id == "q": + shard_idx = tp_rank + else: + shard_idx = tp_rank // self.num_kv_head_replicas + start_idx = shard_idx * shard_size + + if not is_sharded_weight: + loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size) + + # Special case for for AQLM codebooks. + elif is_metadata: + # metadata indicates fixed size concatenated along dim 0 + shard_size = loaded_weight.shape[0] + shard_index = ["q", "k", "v"].index(loaded_shard_id) + param_data = param_data.narrow(0, shard_index * shard_size, shard_size) + # Special case for per-tensor scales in fused case. + elif needs_scalar_to_array: + param_data, loaded_weight = adjust_scalar_to_fused_array( + param_data, loaded_weight, loaded_shard_id + ) + else: + ignore_warning = getattr(param, "ignore_warning", False) + if not ignore_warning: + logger.warning( + "Loading a weight without `output_dim` attribute in " + "QKVParallelLinear, assume the weight is the same " + "for all partitions." + ) + + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + +class RowParallelLinear(LinearBase): + """Linear layer with row parallelism. + + The linear layer is defined as Y = XA + b. A is parallelized along + its first dimension and X along its second dimension as: + - - + | A_1 | + | . | + A = | . | X = [X_1, ..., X_p] + | . | + | A_p | + - - + Arguments: + input_size: first dimension of matrix A. + output_size: second dimension of matrix A. + bias: If true, add bias. Note that bias is not parallelized. + input_is_parallel: If true, we assume that the input is already + split across the GPUs and we do not split + again. + skip_bias_add: This was added to enable performance optimization where + bias can be fused with other element-wise operations. + We skip adding bias but instead return it. + params_dtype: Data type for the parameters. + quant_config: Quantization configure. + """ + + def __init__( + self, + input_size: int, + output_size: int, + bias: bool = True, + input_is_parallel: bool = True, + skip_bias_add: bool = False, + params_dtype: torch.dtype | None = None, + reduce_results: bool = True, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ): + # Divide the weight matrix along the first dimension. + self.tp_rank = get_tp_rank() + self.tp_size = get_tp_world_size() + self.input_size_per_partition = divide(input_size, self.tp_size) + self.output_size_per_partition = output_size + self.output_partition_sizes = [output_size] + + super().__init__( + input_size, output_size, skip_bias_add, params_dtype, quant_config, prefix + ) + + self.input_is_parallel = input_is_parallel + self.reduce_results = reduce_results + + assert self.quant_method is not None + self.quant_method.create_weights( + layer=self, + input_size_per_partition=self.input_size_per_partition, + output_partition_sizes=self.output_partition_sizes, + input_size=self.input_size, + output_size=self.output_size, + params_dtype=self.params_dtype, + weight_loader=( + self.weight_loader_v2 + if self.quant_method.__class__.__name__ in WEIGHT_LOADER_V2_SUPPORTED + else self.weight_loader + ), + ) + if not reduce_results and (bias and not skip_bias_add): + raise ValueError( + "When not reduce the results, adding bias to the " + "results can lead to incorrect results" + ) + + if bias: + self.bias = Parameter(torch.empty(self.output_size, dtype=params_dtype)) + set_weight_attrs( + self.bias, + { + "output_dim": 0, + "weight_loader": self.weight_loader, + }, + ) + else: + self.register_parameter("bias", None) + + def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor): + tp_rank = get_tp_rank() + input_dim = getattr(param, "input_dim", None) + is_sharded_weight = getattr(param, "is_sharded_weight", False) + # bitsandbytes loads the weights of the specific portion + # no need to narrow + is_sharded_weight = is_sharded_weight + + param_data = param.data + if input_dim is not None and not is_sharded_weight: + shard_size = param_data.shape[input_dim] + start_idx = tp_rank * shard_size + loaded_weight = loaded_weight.narrow(input_dim, start_idx, shard_size) + + # Special case for loading scales off disk, which often do not + # have a shape (such as in the case of AutoFP8). + if len(loaded_weight.shape) == 0: + loaded_weight = loaded_weight.reshape(1) + + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + def weight_loader_v2(self, param: BasevLLMParameter, loaded_weight: torch.Tensor): + + # Special case for loading scales off disk, which often do not + # have a shape (such as in the case of AutoFP8). + if len(loaded_weight.shape) == 0: + assert loaded_weight.numel() == 1 + loaded_weight = loaded_weight.reshape(1) + + param.load_row_parallel_weight(loaded_weight=loaded_weight) + + def forward(self, input_) -> tuple[torch.Tensor, Parameter | None]: + if self.input_is_parallel: + input_parallel = input_ + else: + tp_rank = get_tp_rank() + splitted_input = split_tensor_along_last_dim( + input_, num_partitions=self.tp_size + ) + input_parallel = splitted_input[tp_rank].contiguous() + + # Matrix multiply. + assert self.quant_method is not None + # Only fuse bias add into GEMM for rank 0 (this ensures that + # bias will not get added more than once in TP>1 case) + bias_ = None if (self.tp_rank > 0 or self.skip_bias_add) else self.bias + output_parallel = self.quant_method.apply(self, input_parallel, bias=bias_) + if self.reduce_results and self.tp_size > 1: + output = tensor_model_parallel_all_reduce(output_parallel) + else: + output = output_parallel + + output_bias = self.bias if self.skip_bias_add else None + + return output, output_bias + + def extra_repr(self) -> str: + s = f"input_features={self.input_size_per_partition}" + s += f", output_features={self.output_size}" + s += f", bias={self.bias is not None}" + s += f", tp_size={self.tp_size}" + s += f", reduce_results={self.reduce_results}" + return s diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/lora/linear.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/lora/linear.py new file mode 100644 index 00000000..63092612 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/lora/linear.py @@ -0,0 +1,433 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Code adapted from SGLang https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/lora/layers.py + + +import torch +from torch import nn +from torch.distributed._composable.fsdp import ( + CPUOffloadPolicy, + OffloadPolicy, + fully_shard, +) +from torch.distributed.tensor import DTensor + +from sglang.multimodal_gen.runtime.distributed import ( + get_local_torch_device, + get_tp_rank, + split_tensor_along_last_dim, + tensor_model_parallel_all_gather, + tensor_model_parallel_all_reduce, +) +from sglang.multimodal_gen.runtime.layers.linear import ( + ColumnParallelLinear, + LinearBase, + MergedColumnParallelLinear, + QKVParallelLinear, + ReplicatedLinear, + RowParallelLinear, +) +from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import ( + VocabParallelEmbedding, +) +from sglang.multimodal_gen.utils import get_mixed_precision_state + +torch._dynamo.config.recompile_limit = 16 + + +class BaseLayerWithLoRA(nn.Module): + + def __init__( + self, + base_layer: nn.Module, + lora_rank: int | None = None, + lora_alpha: int | None = None, + ): + super().__init__() + self.base_layer: nn.Module = base_layer + + self.merged: bool = False + self.cpu_weight = base_layer.weight.to("cpu") + # indicates adapter weights don't contain this layer + # (which shouldn't normally happen, but we want to separate it from the case of erroneous merging) + self.disable_lora: bool = False + self.lora_rank = lora_rank + self.lora_alpha = lora_alpha + self.lora_path: str | None = None + + self.lora_A = None + self.lora_B = None + + @torch.compile() + def forward(self, x: torch.Tensor) -> torch.Tensor: + lora_A = self.lora_A + lora_B = self.lora_B + if isinstance(self.lora_B, DTensor): + lora_B = self.lora_B.to_local() + lora_A = self.lora_A.to_local() + + if not self.merged and not self.disable_lora: + lora_A_sliced = self.slice_lora_a_weights(lora_A.to(x, non_blocking=True)) + lora_B_sliced = self.slice_lora_b_weights(lora_B.to(x, non_blocking=True)) + delta = x @ lora_A_sliced.T @ lora_B_sliced.T + if self.lora_alpha != self.lora_rank: + delta = delta * ( + self.lora_alpha / self.lora_rank # type: ignore + ) # type: ignore + out, output_bias = self.base_layer(x) + return out + delta, output_bias + else: + out, output_bias = self.base_layer(x) + return out.to(x), output_bias + + def slice_lora_a_weights(self, A: torch.Tensor) -> torch.Tensor: + return A + + def slice_lora_b_weights(self, B: torch.Tensor) -> torch.Tensor: + return B + + def set_lora_weights( + self, + A: torch.Tensor, + B: torch.Tensor, + lora_path: str | None = None, + ) -> None: + self.lora_A = torch.nn.Parameter( + A + ) # share storage with weights in the pipeline + self.lora_B = torch.nn.Parameter(B) + self.disable_lora = False + self.merge_lora_weights() + self.lora_path = lora_path + + @torch.no_grad() + def merge_lora_weights(self) -> None: + if self.disable_lora: + return + + if self.merged: + self.unmerge_lora_weights() + assert ( + self.lora_A is not None and self.lora_B is not None + ), "LoRA weights not set. Please set them first." + if isinstance(self.base_layer.weight, DTensor): + mesh = self.base_layer.weight.data.device_mesh + unsharded_base_layer = ReplicatedLinear( + input_size=self.base_layer.input_size, + output_size=self.base_layer.output_size, + bias=getattr(self.base_layer, "bias", None) is not None, + skip_bias_add=self.base_layer.skip_bias_add, + params_dtype=self.base_layer.params_dtype, + quant_config=self.base_layer.quant_config, + prefix=self.base_layer.prefix, + ) + # Using offload param is on CPU, so current_device is for "CPU -> GPU -> merge -> CPU" + current_device = self.base_layer.weight.data.device + data = self.base_layer.weight.data.to( + get_local_torch_device() + ).full_tensor() + data += self.slice_lora_b_weights(self.lora_B).to( + data + ) @ self.slice_lora_a_weights(self.lora_A).to(data) + unsharded_base_layer.weight = nn.Parameter(data.to(current_device)) + if isinstance(getattr(self.base_layer, "bias", None), DTensor): + unsharded_base_layer.bias = nn.Parameter( + self.base_layer.bias.to(get_local_torch_device(), non_blocking=True) + .full_tensor() + .to(current_device) + ) + + offload_policy = ( + CPUOffloadPolicy() if "cpu" in str(current_device) else OffloadPolicy() + ) + mp_policy = get_mixed_precision_state().mp_policy + + self.base_layer = fully_shard( + unsharded_base_layer, + mesh=mesh, + mp_policy=mp_policy, + offload_policy=offload_policy, + ) + else: + current_device = self.base_layer.weight.data.device + data = self.base_layer.weight.data.to(get_local_torch_device()) + data += self.slice_lora_b_weights( + self.lora_B.to(data) + ) @ self.slice_lora_a_weights(self.lora_A.to(data)) + self.base_layer.weight.data = data.to(current_device, non_blocking=True) + + self.merged = True + + @torch.no_grad() + # @torch.compile(dynamic=True) + def unmerge_lora_weights(self) -> None: + if self.disable_lora: + return + + if not self.merged: + raise ValueError( + "LoRA weights not merged. Please merge them first before unmerging." + ) + + # avoid precision loss + if isinstance(self.base_layer.weight, DTensor): + device = self.base_layer.weight.data.device + self.base_layer.weight = nn.Parameter( + self.cpu_weight.to(device, non_blocking=True) + ) + else: + self.base_layer.weight.data = self.cpu_weight.data.to( + self.base_layer.weight, non_blocking=True + ) + + self.merged = False + + +class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA): + """ + Vocab parallel embedding layer with support for LoRA (Low-Rank Adaptation). + + Note: The current version does not yet implement the LoRA functionality. + This class behaves exactly the same as the base VocabParallelEmbedding. + Future versions will integrate LoRA functionality to support efficient parameter fine-tuning. + """ + + def __init__( + self, + base_layer: VocabParallelEmbedding, + ) -> None: + super().__init__(base_layer) + + def forward(self, input_: torch.Tensor) -> torch.Tensor: + raise NotImplementedError( + "We don't support VocabParallelEmbeddingWithLoRA yet." + ) + + +class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA): + + def __init__( + self, + base_layer: ColumnParallelLinear, + lora_rank: int | None = None, + lora_alpha: int | None = None, + ) -> None: + super().__init__(base_layer, lora_rank, lora_alpha) + + def forward(self, input_: torch.Tensor) -> torch.Tensor: + # duplicate the logic in ColumnParallelLinear + bias = self.base_layer.bias if not self.base_layer.skip_bias_add else None + output_parallel = self.base_layer.quant_method.apply( + self.base_layer, input_, bias + ) + if self.base_layer.gather_output: + output = tensor_model_parallel_all_gather(output_parallel) + else: + output = output_parallel + output_bias = self.base_layer.bias if self.base_layer.skip_bias_add else None + return output, output_bias + + def slice_lora_a_weights(self, A: torch.Tensor) -> torch.Tensor: + return A + + def slice_lora_b_weights(self, B: torch.Tensor) -> torch.Tensor: + tp_rank = get_tp_rank() + shard_size = self.base_layer.output_partition_sizes[0] + start_idx = tp_rank * shard_size + end_idx = (tp_rank + 1) * shard_size + B = B[start_idx:end_idx, :] + return B + + +class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA): + + def __init__( + self, + base_layer: MergedColumnParallelLinear, + lora_rank: int | None = None, + lora_alpha: int | None = None, + ) -> None: + super().__init__(base_layer, lora_rank, lora_alpha) + + def slice_lora_a_weights(self, A: torch.Tensor) -> torch.Tensor: + return A.to(self.base_layer.weight) + + def slice_lora_b_weights(self, B: torch.Tensor) -> torch.Tensor: + tp_rank = get_tp_rank() + # Since the outputs for both gate and up are identical, we use a random one. + shard_size = self.base_layer.output_partition_sizes[0] + start_idx = tp_rank * shard_size + end_idx = (tp_rank + 1) * shard_size + return B[:, start_idx:end_idx, :] + + +class QKVParallelLinearWithLoRA(ColumnParallelLinearWithLoRA): + + def __init__( + self, + base_layer: QKVParallelLinear, + lora_rank: int | None = None, + lora_alpha: int | None = None, + ) -> None: + super().__init__(base_layer, lora_rank, lora_alpha) + + def slice_lora_a_weights(self, A: torch.Tensor) -> torch.Tensor: + return A + + def slice_lora_b_weights( + self, B: list[torch.Tensor] + ) -> tuple[torch.Tensor, torch.Tensor]: + tp_rank = get_tp_rank() + B_q, B_kv = B + base_layer = self.base_layer + q_proj_shard_size = base_layer.q_proj_shard_size + kv_proj_shard_size = base_layer.kv_proj_shard_size + num_kv_head_replicas = base_layer.num_kv_head_replicas + + q_start_idx = q_proj_shard_size * tp_rank + q_end_idx = q_start_idx + q_proj_shard_size + + kv_shard_id = tp_rank // num_kv_head_replicas + kv_start_idx = kv_proj_shard_size * kv_shard_id + kv_end_idx = kv_start_idx + kv_proj_shard_size + + return B_q[q_start_idx:q_end_idx, :], B_kv[:, kv_start_idx:kv_end_idx, :] + + +class RowParallelLinearWithLoRA(BaseLayerWithLoRA): + + def __init__( + self, + base_layer: RowParallelLinear, + lora_rank: int | None = None, + lora_alpha: int | None = None, + ) -> None: + super().__init__(base_layer, lora_rank, lora_alpha) + + def forward(self, input_: torch.Tensor): + # duplicate the logic in RowParallelLinear + if self.base_layer.input_is_parallel: + input_parallel = input_ + else: + tp_rank = get_tp_rank() + splitted_input = split_tensor_along_last_dim( + input_, num_partitions=self.base_layer.tp_size + ) + input_parallel = splitted_input[tp_rank].contiguous() + output_parallel = self.base_layer.quant_method.apply( + self.base_layer, input_parallel + ) + + if self.set_lora: + output_parallel = self.apply_lora(output_parallel, input_parallel) + + if self.base_layer.reduce_results and self.base_layer.tp_size > 1: + output_ = tensor_model_parallel_all_reduce(output_parallel) + else: + output_ = output_parallel + + if not self.base_layer.skip_bias_add: + output = ( + output_ + self.base_layer.bias + if self.base_layer.bias is not None + else output_ + ) + output_bias = None + else: + output = output_ + output_bias = self.base_layer.bias + return output, output_bias + + def slice_lora_a_weights(self, A: torch.Tensor) -> torch.Tensor: + tp_rank = get_tp_rank() + shard_size = self.base_layer.input_size_per_partition + start_idx = tp_rank * shard_size + end_idx = (tp_rank + 1) * shard_size + A = A[:, start_idx:end_idx].contiguous() + return A + + def slice_lora_b_weights(self, B: torch.Tensor) -> torch.Tensor: + return B + + +class LinearWithLoRA(BaseLayerWithLoRA): + """ + Wrapper for standard torch.nn.Linear to support LoRA. + Unlike custom LinearBase classes, nn.Linear.forward() returns a single tensor, + not a tuple of (output, bias). + """ + + def __init__( + self, + base_layer: nn.Linear, + lora_rank: int | None = None, + lora_alpha: int | None = None, + ) -> None: + super().__init__(base_layer, lora_rank, lora_alpha) + + @torch.compile() + def forward(self, x: torch.Tensor) -> torch.Tensor: + lora_A = self.lora_A + lora_B = self.lora_B + if isinstance(self.lora_B, DTensor): + lora_B = self.lora_B.to_local() + lora_A = self.lora_A.to_local() + + if not self.merged and not self.disable_lora: + lora_A_sliced = self.slice_lora_a_weights(lora_A.to(x, non_blocking=True)) + lora_B_sliced = self.slice_lora_b_weights(lora_B.to(x, non_blocking=True)) + delta = x @ lora_A_sliced.T @ lora_B_sliced.T + if self.lora_alpha != self.lora_rank: + delta = delta * ( + self.lora_alpha / self.lora_rank # type: ignore + ) # type: ignore + # nn.Linear.forward() returns a single tensor, not a tuple + out = self.base_layer(x) + return out + delta + else: + # nn.Linear.forward() returns a single tensor + out = self.base_layer(x) + return out + + +def wrap_with_lora_layer( + layer: nn.Module, + lora_rank: int | None = None, + lora_alpha: int | None = None, +) -> BaseLayerWithLoRA | None: + """ + transform the given layer to its corresponding LoRA layer + """ + supported_layer_types: dict[ + type[LinearBase] | type[nn.Linear], type[BaseLayerWithLoRA] + ] = { + # the order matters + # VocabParallelEmbedding: VocabParallelEmbeddingWithLoRA, + QKVParallelLinear: QKVParallelLinearWithLoRA, + MergedColumnParallelLinear: MergedColumnParallelLinearWithLoRA, + ColumnParallelLinear: ColumnParallelLinearWithLoRA, + RowParallelLinear: RowParallelLinearWithLoRA, + ReplicatedLinear: BaseLayerWithLoRA, + nn.Linear: LinearWithLoRA, + } + for src_layer_type, lora_layer_type in supported_layer_types.items(): + if isinstance(layer, src_layer_type): # type: ignore[arg-type] + ret = lora_layer_type( + layer, + lora_rank=lora_rank, + lora_alpha=lora_alpha, + ) + return ret + return None + + +# source: https://github.com/vllm-project/vllm/blob/93b38bea5dd03e1b140ca997dfaadef86f8f1855/vllm/lora/utils.py#L9 +def replace_submodule( + model: nn.Module, module_name: str, new_module: nn.Module +) -> nn.Module: + """Replace a submodule in a model with a new module.""" + parent = model.get_submodule(".".join(module_name.split(".")[:-1])) + target_name = module_name.split(".")[-1] + setattr(parent, target_name, new_module) + return new_module diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/mlp.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/mlp.py new file mode 100644 index 00000000..17918e2a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/mlp.py @@ -0,0 +1,46 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +import torch +import torch.nn as nn + +from sglang.multimodal_gen.runtime.layers.activation import get_act_fn +from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear + + +class MLP(nn.Module): + """ + MLP for DiT blocks, NO gated linear units + """ + + def __init__( + self, + input_dim: int, + mlp_hidden_dim: int, + output_dim: int | None = None, + bias: bool = True, + act_type: str = "gelu_pytorch_tanh", + dtype: torch.dtype | None = None, + prefix: str = "", + ): + super().__init__() + self.fc_in = ReplicatedLinear( + input_dim, + mlp_hidden_dim, # For activation func like SiLU that need 2x width + bias=bias, + params_dtype=dtype, + ) + + self.act = get_act_fn(act_type) + if output_dim is None: + output_dim = input_dim + self.fc_out = ReplicatedLinear( + mlp_hidden_dim, output_dim, bias=bias, params_dtype=dtype + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x, _ = self.fc_in(x) + x = self.act(x) + x, _ = self.fc_out(x) + return x diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/quantization/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/quantization/__init__.py new file mode 100644 index 00000000..0d6c7979 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/quantization/__init__.py @@ -0,0 +1,71 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +from typing import Literal, get_args + +from sglang.multimodal_gen.runtime.layers.quantization.base_config import ( + QuantizationConfig, +) + +QuantizationMethods = Literal[None] + +QUANTIZATION_METHODS: list[str] = list(get_args(QuantizationMethods)) + +# The customized quantization methods which will be added to this dict. +_CUSTOMIZED_METHOD_TO_QUANT_CONFIG = {} + + +def register_quantization_config(quantization: str): + """Register a customized vllm quantization config. + + When a quantization method is not supported by vllm, you can register a customized + quantization config to support it. + + Args: + quantization (str): The quantization method name. + + Examples: + >>> from sglang.multimodal_gen.runtime.layers.quantization import register_quantization_config + >>> from sglang.multimodal_gen.runtime.layers.quantization import get_quantization_config + >>> from sglang.multimodal_gen.runtime.layers.quantization.base_config import QuantizationConfig + >>> + >>> @register_quantization_config("my_quant") + ... class MyQuantConfig(QuantizationConfig): + ... pass + >>> + >>> get_quantization_config("my_quant") + + """ # noqa: E501 + + def _wrapper(quant_config_cls): + if quantization in QUANTIZATION_METHODS: + raise ValueError( + f"The quantization method `{quantization}` is already exists." + ) + if not issubclass(quant_config_cls, QuantizationConfig): + raise ValueError( + "The quantization config must be a subclass of " "`QuantizationConfig`." + ) + _CUSTOMIZED_METHOD_TO_QUANT_CONFIG[quantization] = quant_config_cls + QUANTIZATION_METHODS.append(quantization) + return quant_config_cls + + return _wrapper + + +def get_quantization_config(quantization: str) -> type[QuantizationConfig]: + if quantization not in QUANTIZATION_METHODS: + raise ValueError(f"Invalid quantization method: {quantization}") + + method_to_config: dict[str, type[QuantizationConfig]] = {} + # Update the `method_to_config` with customized quantization methods. + method_to_config.update(_CUSTOMIZED_METHOD_TO_QUANT_CONFIG) + + return method_to_config[quantization] + + +all = [ + "QuantizationMethods", + "QuantizationConfig", + "get_quantization_config", + "QUANTIZATION_METHODS", +] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/quantization/base_config.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/quantization/base_config.py new file mode 100644 index 00000000..ffb275a8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/quantization/base_config.py @@ -0,0 +1,152 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/model_executor/layers/quantization/base_config.py + +import inspect +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any + +import torch +from torch import nn + +if TYPE_CHECKING: + from sglang.multimodal_gen.runtime.layers.quantization import QuantizationMethods +else: + QuantizationMethods = str + + +class QuantizeMethodBase(ABC): + """Base class for different quantized methods.""" + + @abstractmethod + def create_weights( + self, layer: torch.nn.Module, *weight_args, **extra_weight_attrs + ): + """Create weights for a layer. + + The weights will be set as attributes of the layer.""" + raise NotImplementedError + + @abstractmethod + def apply(self, layer: torch.nn.Module, *args, **kwargs) -> torch.Tensor: + """Apply the weights in layer to the input tensor. + + Expects create_weights to have been called before on the layer.""" + raise NotImplementedError + + # Not required functions + def embedding(self, layer: torch.nn.Module, *args, **kwargs) -> torch.Tensor: + """Gather embeddings in the layer based on indices in the input tensor. + + Expects create_weights to have been called before on the layer.""" + raise NotImplementedError + + def process_weights_after_loading(self, layer: nn.Module) -> None: + """Process the weight after loading. + + This can be used for example, to transpose weights for computation. + """ + return + + +def method_has_implemented_embedding(method_class: type[QuantizeMethodBase]) -> bool: + """ + Not all quant methods have embedding implemented, so we need to check that + it exists for our given method. We check this by making sure the function + has been changed from the base implementation. + """ + base_embedding = inspect.getattr_static(QuantizeMethodBase, "embedding", None) + class_embedding = inspect.getattr_static(method_class, "embedding", None) + + return class_embedding is not None and class_embedding is not base_embedding + + +class QuantizationConfig(ABC): + """Base class for quantization configs.""" + + def __init__(self): + super().__init__() + # mapping is updated by models as they initialize + self.packed_modules_mapping: dict[str, list[str]] = dict() + + @abstractmethod + def get_name(self) -> QuantizationMethods: + """Name of the quantization method.""" + raise NotImplementedError + + @abstractmethod + def get_supported_act_dtypes(self) -> list[torch.dtype]: + """List of supported activation dtypes.""" + raise NotImplementedError + + @classmethod + @abstractmethod + def get_min_capability(cls) -> int: + """Minimum GPU capability to support the quantization method. + + E.g., 70 for Volta, 75 for Turing, 80 for Ampere. + This requirement is due to the custom CUDA kernels used by the + quantization method. + """ + raise NotImplementedError + + @staticmethod + @abstractmethod + def get_config_filenames() -> list[str]: + """List of filenames to search for in the model directory.""" + raise NotImplementedError + + @classmethod + @abstractmethod + def from_config(cls, config: dict[str, Any]) -> "QuantizationConfig": + """Create a config class from the model's quantization config.""" + raise NotImplementedError + + @classmethod + def override_quantization_method( + cls, hf_quant_cfg, user_quant + ) -> QuantizationMethods | None: + """ + Detects if this quantization method can support a given checkpoint + format by overriding the user specified quantization method -- + this method should only be overwritten by subclasses in exceptional + circumstances + """ + return None + + @staticmethod + def get_from_keys(config: dict[str, Any], keys: list[str]) -> Any: + """Get a value from the model's quantization config.""" + for key in keys: + if key in config: + return config[key] + raise ValueError( + f"Cannot find any of {keys} in the model's " "quantization config." + ) + + @staticmethod + def get_from_keys_or(config: dict[str, Any], keys: list[str], default: Any) -> Any: + """Get a optional value from the model's quantization config.""" + try: + return QuantizationConfig.get_from_keys(config, keys) + except ValueError: + return default + + @abstractmethod + def get_quant_method( + self, layer: torch.nn.Module, prefix: str + ) -> QuantizeMethodBase | None: + """Get the quantize method to use for the quantized layer. + + Args: + layer: The layer for the quant method. + prefix: The full name of the layer in the state dict + Returns: + The quantize method. None if the given layer doesn't support quant + method. + """ + raise NotImplementedError + + def get_cache_scale(self, name: str) -> str | None: + return None diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/rotary_embedding.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/rotary_embedding.py new file mode 100644 index 00000000..c0a58903 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/rotary_embedding.py @@ -0,0 +1,889 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/model_executor/layers/rotary_embedding.py + +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.33.2/src/transformers/models/llama/modeling_llama.py +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Rotary Positional Embeddings.""" +import functools +from collections import OrderedDict +from typing import Any + +import torch + +from sglang.multimodal_gen.runtime.distributed.parallel_state import get_sp_group +from sglang.multimodal_gen.runtime.layers.custom_op import CustomOp +from sglang.multimodal_gen.runtime.layers.triton_ops import apply_rotary_embedding +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +def _rotate_neox(x: torch.Tensor) -> torch.Tensor: + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def _rotate_gptj(x: torch.Tensor) -> torch.Tensor: + x1 = x[..., ::2] + x2 = x[..., 1::2] + x = torch.stack((-x2, x1), dim=-1) + return x.flatten(-2) + + +def _apply_rotary_emb( + x: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + is_neox_style: bool, + interleaved: bool = False, +) -> torch.Tensor: + """ + Args: + x: [num_tokens, num_heads, head_size] or [num_tokens, head_size] + cos: [num_tokens, head_size // 2] + sin: [num_tokens, head_size // 2] + is_neox_style: Whether to use the Neox-style or GPT-J-style rotary + positional embeddings. + """ + # cos = cos.unsqueeze(-2).to(x.dtype) + # sin = sin.unsqueeze(-2).to(x.dtype) + if is_neox_style: + cos = cos.unsqueeze(-2) + sin = sin.unsqueeze(-2) + if is_neox_style: + x1, x2 = torch.chunk(x, 2, dim=-1) + else: + x1 = x[..., ::2] + x2 = x[..., 1::2] + o1 = (x1.float() * cos - x2.float() * sin).type_as(x) + o2 = (x2.float() * cos + x1.float() * sin).type_as(x) + return torch.cat((o1, o2), dim=-1) + else: + return apply_rotary_embedding(x, cos, sin, interleaved) + + +@CustomOp.register("rotary_embedding") +class RotaryEmbedding(CustomOp): + """Original rotary positional embedding.""" + + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int | float, + is_neox_style: bool, + dtype: torch.dtype, + ) -> None: + super().__init__() + self.head_size = head_size + self.rotary_dim = rotary_dim + self.max_position_embeddings = max_position_embeddings + self.base = base + self.is_neox_style = is_neox_style + self.dtype = dtype + + cache = self._compute_cos_sin_cache() + cache = cache.to(dtype) + self.cos_sin_cache: torch.Tensor + self.register_buffer("cos_sin_cache", cache, persistent=False) + + def _compute_inv_freq(self, base: int | float) -> torch.Tensor: + """Compute the inverse frequency.""" + # NOTE(woosuk): To exactly match the HF implementation, we need to + # use CPU to compute the cache and then move it to GPU. However, we + # create the cache on GPU for faster initialization. This may cause + # a slight numerical difference between the HF implementation and ours. + inv_freq = 1.0 / ( + base + ** ( + torch.arange(0, self.rotary_dim, 2, dtype=torch.float) / self.rotary_dim + ) + ) + return inv_freq + + def _compute_cos_sin_cache(self) -> torch.Tensor: + """Compute the cos and sin cache.""" + inv_freq = self._compute_inv_freq(self.base) + t = torch.arange(self.max_position_embeddings, dtype=torch.float) + + freqs = torch.einsum("i,j -> ij", t, inv_freq) + cos = freqs.cos() + sin = freqs.sin() + cache = torch.cat((cos, sin), dim=-1) + return cache + + def forward_cuda(self, *args, **kwargs) -> Any: + return self.forward_native(*args, **kwargs) + + def forward_native( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + offsets: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """A PyTorch-native implementation of forward().""" + if offsets is not None: + positions = positions + offsets + positions = positions.flatten() + num_tokens = positions.shape[0] + cos_sin = self.cos_sin_cache.index_select(0, positions) + cos, sin = cos_sin.chunk(2, dim=-1) + + query_shape = query.shape + query = query.view(num_tokens, -1, self.head_size) + query_rot = query[..., : self.rotary_dim] + query_pass = query[..., self.rotary_dim :] + query_rot = _apply_rotary_emb(query_rot, cos, sin, self.is_neox_style) + query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) + + key_shape = key.shape + key = key.view(num_tokens, -1, self.head_size) + key_rot = key[..., : self.rotary_dim] + key_pass = key[..., self.rotary_dim :] + key_rot = _apply_rotary_emb(key_rot, cos, sin, self.is_neox_style) + key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) + return query, key + + def extra_repr(self) -> str: + s = f"head_size={self.head_size}, rotary_dim={self.rotary_dim}" + s += f", max_position_embeddings={self.max_position_embeddings}" + s += f", base={self.base}, is_neox_style={self.is_neox_style}" + return s + + +class OneDRotaryEmbedding(torch.nn.Module): + """1D rotary positional embedding with caching.""" + + def __init__( + self, + dim: int, + theta: float = 10000.0, + theta_rescale_factor: float = 1.0, + interpolation_factor: float = 1.0, + dtype: torch.dtype = torch.float32, + use_real: bool = False, + repeat_interleave_real: bool = False, + ): + super().__init__() + assert dim % 2 == 0 + self.dim = dim + self.theta = theta + self.theta_rescale_factor = theta_rescale_factor + self.interpolation_factor = interpolation_factor + # dtype of freqs + self.dtype = dtype + self.use_real = use_real + self.repeat_interleave_real = repeat_interleave_real + + def build_freqs(self, device): + freqs = 1.0 / ( + self.theta + ** ( + torch.arange(0, self.dim, 2, dtype=self.dtype, device=device)[ + : (self.dim // 2) + ] + / self.dim + ).to(device=device) + ) + return freqs + + def build_freqs_outer(self, pos: torch.Tensor, device): + theta = self.theta + # proposed by reddit user bloc97, to rescale rotary embeddings to longer sequence length without fine-tuning + # has some connection to NTK literature + if self.theta_rescale_factor != 1.0: + theta *= self.theta_rescale_factor ** (self.dim / (self.dim - 2)) + + freqs = self.build_freqs(device) + + freqs = torch.outer(pos * self.interpolation_factor, freqs) + freqs_cos = freqs.cos() + freqs_sin = freqs.sin() + + if self.use_real and self.repeat_interleave_real: + freqs_cos = freqs_cos.repeat_interleave(2, dim=1) + freqs_sin = freqs_sin.repeat_interleave(2, dim=1) + + return freqs_cos.float(), freqs_sin.float() + + @functools.lru_cache(maxsize=16) + def forward_from_grid( + self, seq_len: int, start_pos: int, device_str: str + ) -> tuple[torch.Tensor, torch.Tensor]: + device = torch.device(device_str) + pos = torch.arange( + start_pos, start_pos + seq_len, dtype=self.dtype, device=device + ) + + freqs_cos, freqs_sin = self.build_freqs_outer(pos, device) + return freqs_cos, freqs_sin + + def forward(self, pos: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """ + Calculates 1D rotary embeddings for the given positions. + + This method converts the input tensor to a hashable representation + and calls a cached helper method to perform the computation. + """ + pos_tuple = tuple(pos.tolist()) + device_str = str(pos.device) + return self._forward_cached(pos_tuple, device_str) + + @functools.lru_cache(maxsize=16) + def _forward_cached( + self, pos_tuple: tuple, device_str: str + ) -> tuple[torch.Tensor, torch.Tensor]: + """ + The core implementation that computes 1D rotary embeddings. + This method is wrapped by an LRU cache. + """ + device = torch.device(device_str) + pos = torch.as_tensor(pos_tuple, dtype=self.dtype, device=device) + freqs_cos, freqs_sin = self.build_freqs_outer(pos, device) + return freqs_cos, freqs_sin + + +class NDRotaryEmbedding(torch.nn.Module): + """N-dimensional rotary positional embedding.""" + + def __init__( + self, + rope_dim_list: list[int], + rope_theta: float, + theta_rescale_factor: float | list[float] = 1.0, + interpolation_factor: float | list[float] = 1.0, + use_real: bool = False, + repeat_interleave_real: bool = False, + dtype: torch.dtype = torch.float32, + ): + super().__init__() + self.rope_dim_list = rope_dim_list + self.ndim = len(rope_dim_list) + self.rope_theta = rope_theta + # dtype of freqs + # does not control the output dtype + self.dtype = dtype + + if isinstance(theta_rescale_factor, (int, float)): + self.theta_rescale_factor = [theta_rescale_factor] * self.ndim + elif isinstance(theta_rescale_factor, list) and len(theta_rescale_factor) == 1: + self.theta_rescale_factor = [theta_rescale_factor[0]] * self.ndim + else: + self.theta_rescale_factor = theta_rescale_factor + assert ( + len(self.theta_rescale_factor) == self.ndim + ), "len(theta_rescale_factor) should equal to len(rope_dim_list)" + + if isinstance(interpolation_factor, (int, float)): + self.interpolation_factor = [interpolation_factor] * self.ndim + elif isinstance(interpolation_factor, list) and len(interpolation_factor) == 1: + self.interpolation_factor = [interpolation_factor[0]] * self.ndim + else: + self.interpolation_factor = interpolation_factor + assert ( + len(self.interpolation_factor) == self.ndim + ), "len(interpolation_factor) should equal to len(rope_dim_list)" + + self.rope_generators: list[OneDRotaryEmbedding] = torch.nn.ModuleList() + _config_to_gen_idx: dict[tuple, int] = {} + self.dim_idx_to_gen_idx: list[int] = [] + + for i in range(self.ndim): + dim = self.rope_dim_list[i] + rescale = self.theta_rescale_factor[i] + interp = self.interpolation_factor[i] + + config_key = (dim, rescale, interp, use_real, repeat_interleave_real) + if config_key not in _config_to_gen_idx: + generator = OneDRotaryEmbedding( + dim=dim, + theta=self.rope_theta, + theta_rescale_factor=rescale, + interpolation_factor=interp, + dtype=self.dtype, + use_real=use_real, + repeat_interleave_real=repeat_interleave_real, + ) + _config_to_gen_idx[config_key] = len(self.rope_generators) + self.rope_generators.append(generator) + + gen_idx = _config_to_gen_idx[config_key] + self.dim_idx_to_gen_idx.append(gen_idx) + + def forward(self, positions: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """ + Calculates n-d rotary embeddings for given absolute positions. + + Args: + positions (torch.Tensor): A tensor of shape `[num_tokens, ndim]` + containing the integer coordinates for each token. + + Returns: + A tuple of (cos, sin) tensors. + """ + # Caching wrapper: convert tensor to a hashable tuple of tuples. + pos_tuple = tuple(map(tuple, positions.tolist())) + device_str = str(positions.device) + return self._forward_cached(pos_tuple, device_str) + + @functools.lru_cache(maxsize=16) + def _forward_cached( + self, pos_tuple: tuple[tuple[int, ...], ...], device_str: str + ) -> tuple[torch.Tensor, torch.Tensor]: + """ + The core implementation that computes embeddings from a position tensor. + This method is wrapped by an LRU cache. + """ + device = torch.device(device_str) + positions = torch.tensor(pos_tuple, dtype=torch.long, device=device) + return self.forward_uncached(pos=positions) + + def forward_uncached(self, pos: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """ + The core implementation that computes embeddings from a position tensor. + This method is wrapped by an LRU cache. + """ + device = pos.device + + # Pre-allocate the final tensors for efficiency. + num_tokens = pos.shape[0] + first_generator = self.rope_generators[0] + if first_generator.use_real and first_generator.repeat_interleave_real: + head_dim = sum(self.rope_dim_list) + else: + head_dim = sum(self.rope_dim_list) // 2 + + cos = torch.empty((num_tokens, head_dim), device=device, dtype=self.dtype) + sin = torch.empty((num_tokens, head_dim), device=device, dtype=self.dtype) + + col_offset = 0 + for i in range(self.ndim): + # Extract position coordinates for the current dimension for all tokens. + pos_i = pos[:, i].to(self.dtype) + + # Get the appropriate 1D generator. + gen_idx = self.dim_idx_to_gen_idx[i] + generator = self.rope_generators[gen_idx] + + # Calculate 1D embeddings. + cos_1d, sin_1d = generator(pos_i) + + slice_width = cos_1d.shape[1] + cos[:, col_offset : col_offset + slice_width] = cos_1d + sin[:, col_offset : col_offset + slice_width] = sin_1d + col_offset += slice_width + + return cos.float(), sin.float() + + def forward_from_grid( + self, + grid_size: tuple[int, ...], + shard_dim: int = 0, + start_frame: int = 0, + device: torch.device | str | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """ + Handles sp internally + """ + # Caching wrapper: use grid parameters directly as the key. + # grid_tuple = _to_tuple(grid_size, dim=self.ndim) + device_str = str(device) if device is not None else "cpu" + return self._forward_cached_from_grid( + grid_size, shard_dim, start_frame, device_str + ) + + @functools.lru_cache(maxsize=16) + def _forward_cached_from_grid( + self, + grid_size: tuple[int, ...], + shard_dim: int, + start_frame: int, + device_str: str, + ) -> tuple[torch.Tensor, torch.Tensor]: + """ + Computes embeddings for a structured grid, using a highly efficient + implementation that avoids materializing the full position tensor. + This method is wrapped by an LRU cache. + """ + device = torch.device(device_str) + sp_group = get_sp_group() + sp_rank = sp_group.rank_in_group + sp_world_size = sp_group.world_size + + sizes = _to_tuple(grid_size, dim=self.ndim) + starts = (0,) * self.ndim + + # Apply sequence parallel sharding to the sizes and compute shard offset + shard_sizes = list(sizes) + shard_offsets = [0] * self.ndim + if sp_world_size > 1: + assert sizes[shard_dim] % sp_world_size == 0, ( + f"Dimension {shard_dim} with size {sizes[shard_dim]} is not divisible " + f"by sequence parallel world size {sp_world_size}" + ) + shard_size = sizes[shard_dim] // sp_world_size + shard_offsets[shard_dim] = sp_rank * shard_size + shard_sizes[shard_dim] = shard_size + + # Pre-allocate outputs on the requested device to avoid CPU ops and extra cats + num_tokens = 1 + for s in shard_sizes: + num_tokens *= int(s) + head_dim_half = sum(self.rope_dim_list) // 2 + cos = torch.empty((num_tokens, head_dim_half), device=device, dtype=self.dtype) + sin = torch.empty((num_tokens, head_dim_half), device=device, dtype=self.dtype) + + # Compute per-axis 1D embeddings once and expand via repeats to [N, d_i/2] + col_offset = 0 + for i in range(self.ndim): + dim_i = self.rope_dim_list[i] + dim_i_half = dim_i // 2 + size_i = int(shard_sizes[i]) + + # Starting position for this axis, with optional frame offset for time axis (i==0) + base_offset = starts[i] + if i == 0 and start_frame > 0: + base_offset += start_frame + if sp_world_size > 1 and i == shard_dim: + base_offset += shard_offsets[i] + + gen_idx = self.dim_idx_to_gen_idx[i] + generator = self.rope_generators[gen_idx] + cos_1d, sin_1d = generator.forward_from_grid( + size_i, base_offset, device_str + ) + + # Expand to [num_tokens, dim_i/2] matching flatten order (last dims vary fastest) + repeats_per_entry = 1 + for j in range(i + 1, self.ndim): + repeats_per_entry *= int(shard_sizes[j]) + tile_count = 1 + for j in range(0, i): + tile_count *= int(shard_sizes[j]) + + cos_expanded = cos_1d.repeat_interleave(repeats_per_entry, dim=0) + sin_expanded = sin_1d.repeat_interleave(repeats_per_entry, dim=0) + if tile_count > 1: + cos_expanded = cos_expanded.repeat(tile_count, 1) + sin_expanded = sin_expanded.repeat(tile_count, 1) + + cos[:, col_offset : col_offset + dim_i_half] = cos_expanded + sin[:, col_offset : col_offset + dim_i_half] = sin_expanded + col_offset += dim_i_half + + return cos.float(), sin.float() + + +def _to_tuple(x: int | tuple[int, ...], dim: int = 2) -> tuple[int, ...]: + if isinstance(x, int): + return (x,) * dim + elif len(x) == dim: + return x + else: + raise ValueError(f"Expected length {dim} or int, but got {x}") + + +def get_meshgrid_nd( + start: int | tuple[int, ...], + *args: int | tuple[int, ...], + dim: int = 2, + device: torch.device | str | None = None, + dtype: torch.dtype = torch.float32, +) -> torch.Tensor: + """ + Get n-D meshgrid with start, stop and num. + + Args: + start (int or tuple): If len(args) == 0, start is num; If len(args) == 1, start is start, args[0] is stop, + step is 1; If len(args) == 2, start is start, args[0] is stop, args[1] is num. For n-dim, start/stop/num + should be int or n-tuple. If n-tuple is provided, the meshgrid will be stacked following the dim order in + n-tuples. + *args: See above. + dim (int): Dimension of the meshgrid. Defaults to 2. + + Returns: + grid (np.ndarray): [dim, ...] + """ + if len(args) == 0: + # start is grid_size + num = _to_tuple(start, dim=dim) + start = (0,) * dim + stop = num + elif len(args) == 1: + # start is start, args[0] is stop, step is 1 + start = _to_tuple(start, dim=dim) + stop = _to_tuple(args[0], dim=dim) + num = tuple(stop[i] - start[i] for i in range(dim)) + elif len(args) == 2: + # start is start, args[0] is stop, args[1] is num + start = _to_tuple(start, dim=dim) # Left-Top eg: 12,0 + stop = _to_tuple(args[0], dim=dim) # Right-Bottom eg: 20,32 + num = _to_tuple(args[1], dim=dim) # Target Size eg: 32,124 + else: + raise ValueError(f"len(args) should be 0, 1 or 2, but got {len(args)}") + + # PyTorch implement of np.linspace(start[i], stop[i], num[i], endpoint=False) + axis_grid = [] + for i in range(dim): + a, b, n = start[i], stop[i], num[i] + g = torch.linspace(a, b, n + 1, dtype=dtype, device=device)[:n] + axis_grid.append(g) + grid = torch.meshgrid(*axis_grid, indexing="ij") # dim x [W, H, D] + grid = torch.stack(grid, dim=0) # [dim, W, H, D] + + return grid + + +def get_1d_rotary_pos_embed( + dim: int, + pos: torch.FloatTensor | int, + theta: float = 10000.0, + theta_rescale_factor: float = 1.0, + interpolation_factor: float = 1.0, + dtype: torch.dtype = torch.float32, + device: torch.device | str | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """ + Precompute the frequency tensor for complex exponential (cis) with given dimensions. + (Note: `cis` means `cos + i * sin`, where i is the imaginary unit.) + + This function calculates a frequency tensor with complex exponential using the given dimension 'dim' + and the end index 'end'. The 'theta' parameter scales the frequencies. + + Args: + dim (int): Dimension of the frequency tensor. + pos (int or torch.FloatTensor): Position indices for the frequency tensor. [S] or scalar + theta (float, optional): Scaling factor for frequency computation. Defaults to 10000.0. + theta_rescale_factor (float, optional): Rescale factor for theta. Defaults to 1.0. + interpolation_factor (float, optional): Factor to scale positions. Defaults to 1.0. + + Returns: + freqs_cos, freqs_sin: Precomputed frequency tensor with real and imaginary parts separately. [S, D] + """ + if isinstance(pos, int): + pos = torch.arange(pos, dtype=dtype, device=device) + elif ( + isinstance(pos, torch.Tensor) + and device is not None + and pos.device != torch.device(device) + ): + # Ensure positions are on the requested device to avoid implicit CPU ops. + pos = pos.to(device) + + # proposed by reddit user bloc97, to rescale rotary embeddings to longer sequence length without fine-tuning + # has some connection to NTK literature + if theta_rescale_factor != 1.0: + theta *= theta_rescale_factor ** (dim / (dim - 2)) + + freqs = 1.0 / ( + theta + ** (torch.arange(0, dim, 2, device=device)[: (dim // 2)].to(dtype) / dim).to( + device=device + ) + ) # [D/2] + freqs = torch.outer(pos * interpolation_factor, freqs) # [S, D/2] + freqs_cos = freqs.cos() # [S, D/2] + freqs_sin = freqs.sin() # [S, D/2] + return freqs_cos, freqs_sin + + +def get_nd_rotary_pos_embed( + rope_dim_list, + start, + *args, + theta=10000.0, + theta_rescale_factor: float | list[float] = 1.0, + interpolation_factor: float | list[float] = 1.0, + shard_dim: int = 0, + sp_rank: int = 0, + sp_world_size: int = 1, + dtype: torch.dtype = torch.float32, + start_frame: int = 0, + device: torch.device | str | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """ + This is a n-d version of precompute_freqs_cis, which is a RoPE for tokens with n-d structure. + Supports sequence parallelism by allowing sharding of a specific dimension. + + Args: + rope_dim_list (list of int): Dimension of each rope. len(rope_dim_list) should equal to n. + sum(rope_dim_list) should equal to head_dim of attention layer. + start (int | tuple of int | list of int): If len(args) == 0, start is num; If len(args) == 1, start is start, + args[0] is stop, step is 1; If len(args) == 2, start is start, args[0] is stop, args[1] is num. + *args: See above. + theta (float): Scaling factor for frequency computation. Defaults to 10000.0. + theta_rescale_factor (float): Rescale factor for theta. Defaults to 1.0. + interpolation_factor (float): Factor to scale positions. Defaults to 1.0. + shard_dim (int): Which dimension to shard for sequence parallelism. Defaults to 0. + sp_rank (int): Rank in the sequence parallel group. Defaults to 0. + sp_world_size (int): World size of the sequence parallel group. Defaults to 1. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: (cos, sin) tensors of shape [HW, D/2] + """ + # Determine per-axis sizes for the (possibly sharded) grid without materializing it + ndim = len(rope_dim_list) + if len(args) == 0: + # start is grid_size + sizes = _to_tuple(start, dim=ndim) + starts = (0,) * ndim + elif len(args) == 1: + # start is start, args[0] is stop, step is 1 + starts = _to_tuple(start, dim=ndim) + stops = _to_tuple(args[0], dim=ndim) + sizes = tuple(stops[i] - starts[i] for i in range(ndim)) + elif len(args) == 2: + # start is start, args[0] is stop, args[1] is num + starts = _to_tuple(start, dim=ndim) + _ = _to_tuple(args[0], dim=ndim) # stop, unused here + sizes = _to_tuple(args[1], dim=ndim) + else: + raise ValueError(f"len(args) should be 0, 1 or 2, but got {len(args)}") + + assert ( + shard_dim < ndim + ), f"shard_dim {shard_dim} must be less than number of dimensions {ndim}" + + # Apply sequence parallel sharding to the sizes and compute shard offset + shard_sizes = list(sizes) + shard_offsets = [0] * ndim + if sp_world_size > 1: + assert sizes[shard_dim] % sp_world_size == 0, ( + f"Dimension {shard_dim} with size {sizes[shard_dim]} is not divisible " + f"by sequence parallel world size {sp_world_size}" + ) + shard_size = sizes[shard_dim] // sp_world_size + shard_offsets[shard_dim] = sp_rank * shard_size + shard_sizes[shard_dim] = shard_size + + # Handle theta scaling/interpolation factor per-axis + if isinstance(theta_rescale_factor, int | float): + theta_rescale_factor = [theta_rescale_factor] * ndim + elif isinstance(theta_rescale_factor, list) and len(theta_rescale_factor) == 1: + theta_rescale_factor = [theta_rescale_factor[0]] * ndim + assert ( + len(theta_rescale_factor) == ndim + ), "len(theta_rescale_factor) should equal to len(rope_dim_list)" + + if isinstance(interpolation_factor, int | float): + interpolation_factor = [interpolation_factor] * ndim + elif isinstance(interpolation_factor, list) and len(interpolation_factor) == 1: + interpolation_factor = [interpolation_factor[0]] * ndim + assert ( + len(interpolation_factor) == ndim + ), "len(interpolation_factor) should equal to len(rope_dim_list)" + + # Pre-allocate outputs on the requested device to avoid CPU ops and extra cats + num_tokens = 1 + for s in shard_sizes: + num_tokens *= int(s) + head_dim_half = sum(rope_dim_list) // 2 + cos = torch.empty((num_tokens, head_dim_half), device=device, dtype=dtype) + sin = torch.empty((num_tokens, head_dim_half), device=device, dtype=dtype) + # Compute per-axis 1D embeddings once and expand via repeats to [N, d_i/2] + col_offset = 0 + for i in range(ndim): + dim_i = int(rope_dim_list[i]) + dim_i_half = dim_i // 2 + size_i = int(shard_sizes[i]) + + # Starting position for this axis, with optional frame offset for time axis (i==0) + base_offset = starts[i] + if i == 0 and start_frame > 0: + base_offset += start_frame + if sp_world_size > 1 and i == shard_dim: + base_offset += shard_offsets[i] + + pos_i = torch.arange(size_i, device=device, dtype=dtype) + base_offset + + cos_1d, sin_1d = get_1d_rotary_pos_embed( + dim_i, + pos_i, + theta=theta, + theta_rescale_factor=theta_rescale_factor[i], + interpolation_factor=interpolation_factor[i], + dtype=dtype, + device=device, + ) # [size_i, dim_i/2] + + # Expand to [num_tokens, dim_i/2] matching flatten order (last dims vary fastest) + repeats_per_entry = 1 + for j in range(i + 1, ndim): + repeats_per_entry *= int(shard_sizes[j]) + tile_count = 1 + for j in range(0, i): + tile_count *= int(shard_sizes[j]) + + cos_expanded = cos_1d.repeat_interleave(repeats_per_entry, dim=0) + sin_expanded = sin_1d.repeat_interleave(repeats_per_entry, dim=0) + if tile_count > 1: + cos_expanded = cos_expanded.repeat(tile_count, 1) + sin_expanded = sin_expanded.repeat(tile_count, 1) + + cos[:, col_offset : col_offset + dim_i_half] = cos_expanded + sin[:, col_offset : col_offset + dim_i_half] = sin_expanded + col_offset += dim_i_half + + return cos, sin + + +def get_rotary_pos_embed( + rope_sizes, + hidden_size, + heads_num, + rope_dim_list, + rope_theta, + theta_rescale_factor=1.0, + interpolation_factor=1.0, + shard_dim: int = 0, + dtype: torch.dtype = torch.float32, + start_frame: int = 0, + device: torch.device | str | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """ + Generate rotary positional embeddings for the given sizes. + + Args: + rope_sizes: Tuple of dimensions (t, h, w) + hidden_size: Hidden dimension size + heads_num: Number of attention heads + rope_dim_list: List of dimensions for each axis, or None + rope_theta: Base for frequency calculations + theta_rescale_factor: Rescale factor for theta. Defaults to 1.0 + interpolation_factor: Factor to scale positions. Defaults to 1.0 + shard_dim: Which dimension to shard for sequence parallelism. Defaults to 0. + + Returns: + Tuple of (cos, sin) tensors for rotary embeddings + """ + + target_ndim = 3 + head_dim = hidden_size // heads_num + + if rope_dim_list is None: + rope_dim_list = [head_dim // target_ndim for _ in range(target_ndim)] + + assert ( + sum(rope_dim_list) == head_dim + ), "sum(rope_dim_list) should equal to head_dim of attention layer" + + # Get SP info - now handled within NDRotaryEmbedding + # sp_group = get_sp_group() + # sp_rank = sp_group.rank_in_group + # sp_world_size = sp_group.world_size + + # Simple LRU cache keyed by parameters + global _ND_ROPE_CACHE + key = ( + tuple(rope_dim_list), + float(rope_theta), + ( + tuple(theta_rescale_factor) + if isinstance(theta_rescale_factor, list) + else float(theta_rescale_factor) + ), + ( + tuple(interpolation_factor) + if isinstance(interpolation_factor, list) + else float(interpolation_factor) + ), + dtype, + ) + + cache_hit = key in _ND_ROPE_CACHE + if cache_hit: + rope_emb = _ND_ROPE_CACHE.pop(key) + _ND_ROPE_CACHE[key] = rope_emb # move to end (most-recent) + else: + rope_emb = NDRotaryEmbedding( + rope_dim_list=rope_dim_list, + rope_theta=rope_theta, + theta_rescale_factor=theta_rescale_factor, + interpolation_factor=interpolation_factor, + dtype=dtype, + ) + _ND_ROPE_CACHE[key] = rope_emb + if len(_ND_ROPE_CACHE) > 16: + # pop least-recently-used + _ND_ROPE_CACHE.pop(next(iter(_ND_ROPE_CACHE))) + + freqs_cos, freqs_sin = rope_emb.forward_from_grid( + grid_size=_to_tuple(rope_sizes, dim=3), + shard_dim=shard_dim, + start_frame=start_frame, + device=device, + ) + return freqs_cos, freqs_sin + + +_ROPE_DICT: dict[tuple, RotaryEmbedding] = {} +_ND_ROPE_CACHE: "OrderedDict[tuple, NDRotaryEmbedding]" = OrderedDict() +_ROPE_3D_CACHE: "OrderedDict[tuple, tuple[torch.Tensor, torch.Tensor]]" = OrderedDict() + + +def get_rope( + head_size: int, + rotary_dim: int, + max_position: int, + base: int | float, + is_neox_style: bool = True, + rope_scaling: dict[str, Any] | None = None, + dtype: torch.dtype | None = None, + partial_rotary_factor: float = 1.0, +) -> RotaryEmbedding: + if dtype is None: + dtype = torch.get_default_dtype() + if rope_scaling is not None: + # Transforms every value that is a list into a tuple for caching calls + rope_scaling_tuple = { + k: tuple(v) if isinstance(v, list) else v for k, v in rope_scaling.items() + } + rope_scaling_args = tuple(rope_scaling_tuple.items()) + else: + rope_scaling_args = None + if partial_rotary_factor < 1.0: + rotary_dim = int(rotary_dim * partial_rotary_factor) + key = ( + head_size, + rotary_dim, + max_position, + base, + is_neox_style, + rope_scaling_args, + dtype, + ) + if key in _ROPE_DICT: + return _ROPE_DICT[key] + + if rope_scaling is None: + rotary_emb = RotaryEmbedding( + head_size, rotary_dim, max_position, base, is_neox_style, dtype + ) + else: + raise ValueError(f"Unknown RoPE scaling {rope_scaling}") + _ROPE_DICT[key] = rotary_emb + return rotary_emb diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/triton_ops.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/triton_ops.py new file mode 100644 index 00000000..2a8d96af --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/triton_ops.py @@ -0,0 +1,948 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# TODO: for temporary usage, expecting a refactor +from typing import Optional + +import torch +import triton # type: ignore +import triton.language as tl # type: ignore +from torch import Tensor + + +@triton.autotune( + configs=[ + triton.Config({"BLOCK_N": 64}, num_warps=2), + triton.Config({"BLOCK_N": 128}, num_warps=4), + triton.Config({"BLOCK_N": 256}, num_warps=4), + triton.Config({"BLOCK_N": 512}, num_warps=4), + triton.Config({"BLOCK_N": 1024}, num_warps=8), + ], + key=["inner_dim"], +) +@triton.jit +def _fused_scale_shift_4d_kernel( + output_ptr, + normalized_ptr, + scale_ptr, + shift_ptr, + rows, + inner_dim, + seq_len, + num_frames, + frame_seqlen, + BLOCK_N: tl.constexpr, +): + pid_row = tl.program_id(0) + pid_col = tl.program_id(1) + + col_offsets = pid_col * BLOCK_N + tl.arange(0, BLOCK_N) + mask = col_offsets < inner_dim + + # Pointers for normalized and output + row_base = pid_row * inner_dim + norm_ptrs = normalized_ptr + row_base + col_offsets + out_ptrs = output_ptr + row_base + col_offsets + + # Pointers for scale and shift for 4D + b_idx = pid_row // seq_len + t_idx = pid_row % seq_len + frame_idx_in_batch = t_idx // frame_seqlen + + scale_row_idx = b_idx * num_frames + frame_idx_in_batch + scale_ptrs = scale_ptr + scale_row_idx * inner_dim + col_offsets + shift_ptrs = shift_ptr + scale_row_idx * inner_dim + col_offsets + + normalized = tl.load(norm_ptrs, mask=mask, other=0.0) + scale = tl.load(scale_ptrs, mask=mask, other=0.0) + shift = tl.load(shift_ptrs, mask=mask, other=0.0) + + one = tl.full([BLOCK_N], 1.0, dtype=scale.dtype) + output = normalized * (one + scale) + shift + + tl.store(out_ptrs, output, mask=mask) + + +@triton.jit +def fuse_scale_shift_kernel_blc_opt( + x_ptr, + shift_ptr, + scale_ptr, + y_ptr, + B, + L, + C, + stride_x_b, + stride_x_l, + stride_x_c, + stride_s_b, + stride_s_l, + stride_s_c, + stride_sc_b, + stride_sc_l, + stride_sc_c, + SCALE_IS_SCALAR: tl.constexpr, + SHIFT_IS_SCALAR: tl.constexpr, + BLOCK_L: tl.constexpr, + BLOCK_C: tl.constexpr, +): + pid_l = tl.program_id(0) + pid_c = tl.program_id(1) + pid_b = tl.program_id(2) + + l_offsets = pid_l * BLOCK_L + tl.arange(0, BLOCK_L) + c_offsets = pid_c * BLOCK_C + tl.arange(0, BLOCK_C) + + mask_l = l_offsets < L + mask_c = c_offsets < C + mask = mask_l[:, None] & mask_c[None, :] + + x_off = ( + pid_b * stride_x_b + + l_offsets[:, None] * stride_x_l + + c_offsets[None, :] * stride_x_c + ) + x = tl.load(x_ptr + x_off, mask=mask, other=0) + + if SHIFT_IS_SCALAR: + shift_val = tl.load(shift_ptr) + shift = tl.full((BLOCK_L, BLOCK_C), shift_val, dtype=shift_val.dtype) + else: + s_off = ( + pid_b * stride_s_b + + l_offsets[:, None] * stride_s_l + + c_offsets[None, :] * stride_s_c + ) + shift = tl.load(shift_ptr + s_off, mask=mask, other=0) + + if SCALE_IS_SCALAR: + scale_val = tl.load(scale_ptr) + scale = tl.full((BLOCK_L, BLOCK_C), scale_val, dtype=scale_val.dtype) + else: + sc_off = ( + pid_b * stride_sc_b + + l_offsets[:, None] * stride_sc_l + + c_offsets[None, :] * stride_sc_c + ) + scale = tl.load(scale_ptr + sc_off, mask=mask, other=0) + + y = x * (1 + scale) + shift + tl.store(y_ptr + x_off, y, mask=mask) + + +def fuse_scale_shift_kernel( + x: torch.Tensor, + scale: torch.Tensor, + shift: torch.Tensor, + block_l: int = 128, + block_c: int = 128, +): + assert x.is_cuda and scale.is_cuda + assert x.is_contiguous() + + B, L, C = x.shape + output = torch.empty_like(x) + + if scale.dim() == 4: + # scale/shift: [B, F, 1, C] + rows = B * L + x_2d = x.view(rows, C) + output_2d = output.view(rows, C) + grid = lambda META: (rows, triton.cdiv(C, META["BLOCK_N"])) + num_frames = scale.shape[1] + assert ( + L % num_frames == 0 + ), "seq_len must be divisible by num_frames for 4D scale/shift" + frame_seqlen = L // num_frames + + # Compact [B, F, C] without the singleton dim into [B*F, C] + scale_reshaped = scale.squeeze(2).reshape(-1, C).contiguous() + shift_reshaped = shift.squeeze(2).reshape(-1, C).contiguous() + + _fused_scale_shift_4d_kernel[grid]( + output_2d, + x_2d, + scale_reshaped, + shift_reshaped, + rows, + C, + L, + num_frames, + frame_seqlen, + ) + else: + # 2D: [B, C] or [1, C] -> treat as [B, 1, C] and broadcast over L + # 3D: [B, L, C] (or broadcastable variants like [B, 1, C], [1, L, C], [1, 1, C]) + # Also support scalar (0D or 1-element) + if scale.dim() == 0 or (scale.dim() == 1 and scale.numel() == 1): + scale_blc = scale.reshape(1) + elif scale.dim() == 2: + scale_blc = scale[:, None, :] + elif scale.dim() == 3: + scale_blc = scale + else: + raise ValueError("scale must be 0D/1D(1)/2D/3D or 4D") + + if shift.dim() == 0 or (shift.dim() == 1 and shift.numel() == 1): + shift_blc = shift.reshape(1) + elif shift.dim() == 2: + shift_blc = shift[:, None, :] + elif shift.dim() == 3: + shift_blc = shift + else: + # broadcast later via expand if possible + shift_blc = shift + + need_scale_scalar = scale_blc.dim() == 1 and scale_blc.numel() == 1 + need_shift_scalar = shift_blc.dim() == 1 and shift_blc.numel() == 1 + + if not need_scale_scalar: + scale_exp = scale_blc.expand(B, L, C) + s_sb, s_sl, s_sc = scale_exp.stride() + else: + s_sb = s_sl = s_sc = 0 + + if not need_shift_scalar: + shift_exp = shift_blc.expand(B, L, C) + sh_sb, sh_sl, sh_sc = shift_exp.stride() + else: + sh_sb = sh_sl = sh_sc = 0 + + # If both scalars and both zero, copy fast-path + if need_scale_scalar and need_shift_scalar: + if (scale_blc.abs().max() == 0) and (shift_blc.abs().max() == 0): + output.copy_(x) + return output + + grid = (triton.cdiv(L, block_l), triton.cdiv(C, block_c), B) + fuse_scale_shift_kernel_blc_opt[grid]( + x, + shift_blc if need_shift_scalar else shift_exp, + scale_blc if need_scale_scalar else scale_exp, + output, + B, + L, + C, + x.stride(0), + x.stride(1), + x.stride(2), + sh_sb, + sh_sl, + sh_sc, + s_sb, + s_sl, + s_sc, + SCALE_IS_SCALAR=need_scale_scalar, + SHIFT_IS_SCALAR=need_shift_scalar, + BLOCK_L=block_l, + BLOCK_C=block_c, + num_warps=4, + num_stages=2, + ) + return output + + +@triton.autotune( + configs=[ + triton.Config({"BLOCK_HS_HALF": 32}, num_warps=2), + triton.Config({"BLOCK_HS_HALF": 64}, num_warps=4), + triton.Config({"BLOCK_HS_HALF": 128}, num_warps=4), + triton.Config({"BLOCK_HS_HALF": 256}, num_warps=8), + ], + key=["head_size", "interleaved"], +) +@triton.jit +def _rotary_embedding_kernel( + output_ptr, + x_ptr, + cos_ptr, + sin_ptr, + num_heads, + head_size, + num_tokens, + stride_x_row, + stride_cos_row, + stride_sin_row, + interleaved: tl.constexpr, + BLOCK_HS_HALF: tl.constexpr, +): + row_idx = tl.program_id(0) + token_idx = (row_idx // num_heads) % num_tokens + + x_row_ptr = x_ptr + row_idx * stride_x_row + cos_row_ptr = cos_ptr + token_idx * stride_cos_row + sin_row_ptr = sin_ptr + token_idx * stride_sin_row + output_row_ptr = output_ptr + row_idx * stride_x_row + + # half size for x1 and x2 + head_size_half = head_size // 2 + + for block_start in range(0, head_size_half, BLOCK_HS_HALF): + offsets_half = block_start + tl.arange(0, BLOCK_HS_HALF) + mask = offsets_half < head_size_half + + cos_vals = tl.load(cos_row_ptr + offsets_half, mask=mask, other=0.0) + sin_vals = tl.load(sin_row_ptr + offsets_half, mask=mask, other=0.0) + + offsets_x1 = 2 * offsets_half + offsets_x2 = 2 * offsets_half + 1 + + x1_vals = tl.load(x_row_ptr + offsets_x1, mask=mask, other=0.0) + x2_vals = tl.load(x_row_ptr + offsets_x2, mask=mask, other=0.0) + + x1_fp32 = x1_vals.to(tl.float32) + x2_fp32 = x2_vals.to(tl.float32) + cos_fp32 = cos_vals.to(tl.float32) + sin_fp32 = sin_vals.to(tl.float32) + o1_vals = tl.fma(-x2_fp32, sin_fp32, x1_fp32 * cos_fp32) + o2_vals = tl.fma(x1_fp32, sin_fp32, x2_fp32 * cos_fp32) + + tl.store(output_row_ptr + offsets_x1, o1_vals.to(x1_vals.dtype), mask=mask) + tl.store(output_row_ptr + offsets_x2, o2_vals.to(x2_vals.dtype), mask=mask) + + +def apply_rotary_embedding( + x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, interleaved: bool = False +) -> torch.Tensor: + output = torch.empty_like(x) + + if x.dim() > 3: + bsz, num_tokens, num_heads, head_size = x.shape + else: + num_tokens, num_heads, head_size = x.shape + bsz = 1 + + assert head_size % 2 == 0, "head_size must be divisible by 2" + + x_reshaped = x.view(-1, head_size) + output_reshaped = output.view(-1, head_size) + + # num_tokens per head, 1 token per block + grid = (bsz * num_tokens * num_heads,) + + if interleaved and cos.shape[-1] == head_size: + cos = cos[..., ::2].contiguous() + sin = sin[..., ::2].contiguous() + else: + cos = cos.contiguous() + sin = sin.contiguous() + + _rotary_embedding_kernel[grid]( + output_reshaped, + x_reshaped, + cos, + sin, + num_heads, + head_size, + num_tokens, + x_reshaped.stride(0), + cos.stride(0), + sin.stride(0), + interleaved, + ) + + return output + + +# RMSNorm-fp32 +def maybe_contiguous_lastdim(x): + return x.contiguous() if x is not None and x.stride(-1) != 1 else x + + +def maybe_contiguous(x): + return x.contiguous() if x is not None else None + + +def triton_autotune_configs(): + # Return configs with a valid warp count for the current device + configs = [] + # Maximum threads per block is architecture-dependent in theory, but in reality all are 1024 + max_threads_per_block = 1024 + # Default to warp size 32 if not defined by device + warp_size = getattr( + torch.cuda.get_device_properties(torch.cuda.current_device()), "warp_size", 32 + ) + # Autotune for warp counts which are powers of 2 and do not exceed thread per block limit + return [ + triton.Config({}, num_warps=warp_count) + for warp_count in [1, 2, 4, 8, 16, 32] + if warp_count * warp_size <= max_threads_per_block + ] + # return [triton.Config({}, num_warps=8)] + + +# Copied from flash-attn +@triton.autotune( + configs=triton_autotune_configs(), + key=[ + "N", + "HAS_RESIDUAL", + "STORE_RESIDUAL_OUT", + "IS_RMS_NORM", + "HAS_BIAS", + "HAS_WEIGHT", + "HAS_X1", + "HAS_W1", + "HAS_B1", + ], +) +# torch compile doesn't like triton.heuristics, so we set these manually when calling the kernel +# @triton.heuristics({"HAS_BIAS": lambda args: args["B"] is not None}) +# @triton.heuristics({"HAS_RESIDUAL": lambda args: args["RESIDUAL"] is not None}) +# @triton.heuristics({"HAS_X1": lambda args: args["X1"] is not None}) +# @triton.heuristics({"HAS_W1": lambda args: args["W1"] is not None}) +# @triton.heuristics({"HAS_B1": lambda args: args["B1"] is not None}) +@triton.jit +def _layer_norm_fwd_1pass_kernel( + X, # pointer to the input + Y, # pointer to the output + W, # pointer to the weights + B, # pointer to the biases + RESIDUAL, # pointer to the residual + X1, + W1, + B1, + Y1, + RESIDUAL_OUT, # pointer to the residual + ROWSCALE, + SEEDS, # Dropout seeds for each row + DROPOUT_MASK, + DROPOUT_MASK1, + Mean, # pointer to the mean + Rstd, # pointer to the 1/std + stride_x_row, # how much to increase the pointer when moving by 1 row + stride_y_row, + stride_res_row, + stride_res_out_row, + stride_x1_row, + stride_y1_row, + M, # number of rows in X + N, # number of columns in X + eps, # epsilon to avoid division by zero + dropout_p, # Dropout probability + zero_centered_weight, # If true, add 1.0 to the weight + IS_RMS_NORM: tl.constexpr, + BLOCK_N: tl.constexpr, + HAS_RESIDUAL: tl.constexpr, + STORE_RESIDUAL_OUT: tl.constexpr, + HAS_WEIGHT: tl.constexpr, + HAS_BIAS: tl.constexpr, + HAS_DROPOUT: tl.constexpr, + STORE_DROPOUT_MASK: tl.constexpr, + HAS_ROWSCALE: tl.constexpr, + HAS_X1: tl.constexpr, + HAS_W1: tl.constexpr, + HAS_B1: tl.constexpr, +): + # Map the program id to the row of X and Y it should compute. + row = tl.program_id(0) + X += row * stride_x_row + Y += row * stride_y_row + if HAS_RESIDUAL: + RESIDUAL += row * stride_res_row + if STORE_RESIDUAL_OUT: + RESIDUAL_OUT += row * stride_res_out_row + if HAS_X1: + X1 += row * stride_x1_row + if HAS_W1: + Y1 += row * stride_y1_row + # Compute mean and variance + cols = tl.arange(0, BLOCK_N) + x = tl.load(X + cols, mask=cols < N, other=0.0).to(tl.float32) + if HAS_ROWSCALE: + rowscale = tl.load(ROWSCALE + row).to(tl.float32) + x *= rowscale + if HAS_DROPOUT: + # Compute dropout mask + # 7 rounds is good enough, and reduces register pressure + keep_mask = ( + tl.rand(tl.load(SEEDS + row).to(tl.uint32), cols, n_rounds=7) > dropout_p + ) + x = tl.where(keep_mask, x / (1.0 - dropout_p), 0.0) + if STORE_DROPOUT_MASK: + tl.store(DROPOUT_MASK + row * N + cols, keep_mask, mask=cols < N) + if HAS_X1: + x1 = tl.load(X1 + cols, mask=cols < N, other=0.0).to(tl.float32) + if HAS_ROWSCALE: + rowscale = tl.load(ROWSCALE + M + row).to(tl.float32) + x1 *= rowscale + if HAS_DROPOUT: + # Compute dropout mask + # 7 rounds is good enough, and reduces register pressure + keep_mask = ( + tl.rand(tl.load(SEEDS + M + row).to(tl.uint32), cols, n_rounds=7) + > dropout_p + ) + x1 = tl.where(keep_mask, x1 / (1.0 - dropout_p), 0.0) + if STORE_DROPOUT_MASK: + tl.store(DROPOUT_MASK1 + row * N + cols, keep_mask, mask=cols < N) + x += x1 + if HAS_RESIDUAL: + residual = tl.load(RESIDUAL + cols, mask=cols < N, other=0.0).to(tl.float32) + x += residual + if STORE_RESIDUAL_OUT: + tl.store(RESIDUAL_OUT + cols, x, mask=cols < N) + if not IS_RMS_NORM: + mean = tl.sum(x, axis=0) / N + tl.store(Mean + row, mean) + xbar = tl.where(cols < N, x - mean, 0.0) + var = tl.sum(xbar * xbar, axis=0) / N + else: + xbar = tl.where(cols < N, x, 0.0) + var = tl.sum(xbar * xbar, axis=0) / N + rstd = 1 / tl.sqrt(var + eps) + tl.store(Rstd + row, rstd) + # Normalize and apply linear transformation + mask = cols < N + if HAS_WEIGHT: + w = tl.load(W + cols, mask=mask).to(tl.float32) + if zero_centered_weight: + w += 1.0 + if HAS_BIAS: + b = tl.load(B + cols, mask=mask).to(tl.float32) + x_hat = (x - mean) * rstd if not IS_RMS_NORM else x * rstd + if HAS_WEIGHT: + y = x_hat * w + b if HAS_BIAS else x_hat * w + else: + y = x_hat + b if HAS_BIAS else x_hat + # Write output + tl.store(Y + cols, y, mask=mask) + if HAS_W1: + w1 = tl.load(W1 + cols, mask=mask).to(tl.float32) + if zero_centered_weight: + w1 += 1.0 + if HAS_B1: + b1 = tl.load(B1 + cols, mask=mask).to(tl.float32) + y1 = x_hat * w1 + b1 if HAS_B1 else x_hat * w1 + tl.store(Y1 + cols, y1, mask=mask) + + +def _layer_norm_fwd( + x: Tensor, + weight: Tensor, + bias: Tensor, + eps: float, + residual: Optional[Tensor] = None, + x1: Optional[Tensor] = None, + weight1: Optional[Tensor] = None, + bias1: Optional[Tensor] = None, + dropout_p: float = 0.0, + rowscale: Optional[Tensor] = None, + out_dtype: Optional[torch.dtype] = None, + residual_dtype: Optional[torch.dtype] = None, + zero_centered_weight: bool = False, + is_rms_norm: bool = False, + return_dropout_mask: bool = False, + out: Optional[Tensor] = None, + residual_out: Optional[Tensor] = None, +) -> (Tensor, Tensor, Tensor, Tensor, Tensor, Tensor, Tensor, Tensor): + # Need to wrap to handle the case where residual_out is a alias of x, which makes torch.library + # and torch.compile unhappy. Also allocate memory for out and residual_out if they are None + # so that _layer_norm_fwd_impl doesn't have to return them. + if out is None: + out = torch.empty_like(x, dtype=x.dtype if out_dtype is None else out_dtype) + if residual is not None: + residual_dtype = residual.dtype + if residual_out is None and ( + residual is not None + or (residual_dtype is not None and residual_dtype != x.dtype) + or dropout_p > 0.0 + or rowscale is not None + or x1 is not None + ): + residual_out = torch.empty_like( + x, dtype=residual_dtype if residual_dtype is not None else x.dtype + ) + else: + residual_out = None + y1, mean, rstd, seeds, dropout_mask, dropout_mask1 = _layer_norm_fwd_impl( + x, + weight, + bias, + eps, + out, + residual=residual, + x1=x1, + weight1=weight1, + bias1=bias1, + dropout_p=dropout_p, + rowscale=rowscale, + zero_centered_weight=zero_centered_weight, + is_rms_norm=is_rms_norm, + return_dropout_mask=return_dropout_mask, + residual_out=residual_out, + ) + # residual_out is None if residual is None and residual_dtype == input_dtype and dropout_p == 0.0 + if residual_out is None: + residual_out = x + return out, y1, mean, rstd, residual_out, seeds, dropout_mask, dropout_mask1 + + +# [2025-04-28] torch.library.triton_op ignores the schema argument, but here we need the schema +# since we're returning a tuple of tensors +def _layer_norm_fwd_impl( + x: Tensor, + weight: Optional[Tensor], + bias: Tensor, + eps: float, + out: Tensor, + residual: Optional[Tensor] = None, + x1: Optional[Tensor] = None, + weight1: Optional[Tensor] = None, + bias1: Optional[Tensor] = None, + dropout_p: float = 0.0, + rowscale: Optional[Tensor] = None, + zero_centered_weight: bool = False, + is_rms_norm: bool = False, + return_dropout_mask: bool = False, + residual_out: Optional[Tensor] = None, +) -> (Tensor, Tensor, Tensor, Tensor, Tensor, Tensor): + M, N = x.shape + assert x.stride(-1) == 1 + if residual is not None: + assert residual.stride(-1) == 1 + assert residual.shape == (M, N) + if weight is not None: + assert weight.shape == (N,) + assert weight.stride(-1) == 1 + if bias is not None: + assert bias.stride(-1) == 1 + assert bias.shape == (N,) + if x1 is not None: + assert x1.shape == x.shape + assert rowscale is None + assert x1.stride(-1) == 1 + if weight1 is not None: + assert weight1.shape == (N,) + assert weight1.stride(-1) == 1 + if bias1 is not None: + assert bias1.shape == (N,) + assert bias1.stride(-1) == 1 + if rowscale is not None: + assert rowscale.is_contiguous() + assert rowscale.shape == (M,) + assert out.shape == x.shape + assert out.stride(-1) == 1 + if residual_out is not None: + assert residual_out.shape == x.shape + assert residual_out.stride(-1) == 1 + if weight1 is not None: + y1 = torch.empty_like(out) + assert y1.stride(-1) == 1 + else: + y1 = None + mean = ( + torch.empty((M,), dtype=torch.float32, device=x.device) + if not is_rms_norm + else None + ) + rstd = torch.empty((M,), dtype=torch.float32, device=x.device) + if dropout_p > 0.0: + seeds = torch.randint( + 2**32, (M if x1 is None else 2 * M,), device=x.device, dtype=torch.int64 + ) + else: + seeds = None + if return_dropout_mask and dropout_p > 0.0: + dropout_mask = torch.empty(M, N, device=x.device, dtype=torch.bool) + if x1 is not None: + dropout_mask1 = torch.empty(M, N, device=x.device, dtype=torch.bool) + else: + dropout_mask1 = None + else: + dropout_mask, dropout_mask1 = None, None + # Less than 64KB per feature: enqueue fused kernel + MAX_FUSED_SIZE = 65536 // x.element_size() + BLOCK_N = min(MAX_FUSED_SIZE, triton.next_power_of_2(N)) + if N > BLOCK_N: + raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.") + with torch.cuda.device(x.device.index): + torch.library.wrap_triton(_layer_norm_fwd_1pass_kernel)[(M,)]( + x, + out, + weight if weight is not None else x, # unused when HAS_WEIGHT == False + bias, + residual, + x1, + weight1, + bias1, + y1, + residual_out, + rowscale, + seeds, + dropout_mask, + dropout_mask1, + mean, + rstd, + x.stride(0), + out.stride(0), + residual.stride(0) if residual is not None else 0, + residual_out.stride(0) if residual_out is not None else 0, + x1.stride(0) if x1 is not None else 0, + y1.stride(0) if y1 is not None else 0, + M, + N, + eps, + dropout_p, + # Passing bool make torch inductor very unhappy since it then tries to compare to int_max + int(zero_centered_weight), + is_rms_norm, + BLOCK_N, + residual is not None, + residual_out is not None, + weight is not None, + bias is not None, + dropout_p > 0.0, + dropout_mask is not None, + rowscale is not None, + HAS_X1=x1 is not None, + HAS_W1=weight1 is not None, + HAS_B1=bias1 is not None, + ) + return y1, mean, rstd, seeds, dropout_mask, dropout_mask1 + + +class LayerNormFn: + + @staticmethod + def forward( + x, + weight, + bias, + residual=None, + x1=None, + weight1=None, + bias1=None, + eps=1e-6, + dropout_p=0.0, + rowscale=None, + prenorm=False, + residual_in_fp32=False, + zero_centered_weight=False, + is_rms_norm=False, + return_dropout_mask=False, + out_dtype=None, + out=None, + residual_out=None, + ): + x_shape_og = x.shape + # reshape input data into 2D tensor + x = maybe_contiguous_lastdim(x.reshape(-1, x.shape[-1])) + if residual is not None: + assert residual.shape == x_shape_og + residual = maybe_contiguous_lastdim( + residual.reshape(-1, residual.shape[-1]) + ) + if x1 is not None: + assert x1.shape == x_shape_og + assert rowscale is None, "rowscale is not supported with parallel LayerNorm" + x1 = maybe_contiguous_lastdim(x1.reshape(-1, x1.shape[-1])) + # weight can be None when elementwise_affine=False for LayerNorm + if weight is not None: + weight = weight.contiguous() + bias = maybe_contiguous(bias) + weight1 = maybe_contiguous(weight1) + bias1 = maybe_contiguous(bias1) + if rowscale is not None: + rowscale = rowscale.reshape(-1).contiguous() + residual_dtype = ( + residual.dtype + if residual is not None + else (torch.float32 if residual_in_fp32 else None) + ) + if out is not None: + out = out.reshape(-1, out.shape[-1]) + if residual_out is not None: + residual_out = residual_out.reshape(-1, residual_out.shape[-1]) + y, y1, mean, rstd, residual_out, seeds, dropout_mask, dropout_mask1 = ( + _layer_norm_fwd( + x, + weight, + bias, + eps, + residual, + x1, + weight1, + bias1, + dropout_p=dropout_p, + rowscale=rowscale, + out_dtype=out_dtype, + residual_dtype=residual_dtype, + zero_centered_weight=zero_centered_weight, + is_rms_norm=is_rms_norm, + return_dropout_mask=return_dropout_mask, + out=out, + residual_out=residual_out, + ) + ) + y = y.reshape(x_shape_og) + return y + + +def layer_norm_fn( + x, + weight, + bias, + residual=None, + x1=None, + weight1=None, + bias1=None, + eps=1e-6, + dropout_p=0.0, + rowscale=None, + prenorm=False, + residual_in_fp32=False, + zero_centered_weight=False, + is_rms_norm=False, + return_dropout_mask=False, + out_dtype=None, + out=None, + residual_out=None, +): + return LayerNormFn.forward( + x, + weight, + bias, + residual, + x1, + weight1, + bias1, + eps, + dropout_p, + rowscale, + prenorm, + residual_in_fp32, + zero_centered_weight, + is_rms_norm, + return_dropout_mask, + out_dtype, + out, + residual_out, + ) + + +@triton.jit +def _norm_infer_kernel( + X, + Y, + W, + B, + stride_x_row, + stride_y_row, + M, + N, + eps, + IS_RMS_NORM: tl.constexpr, + HAS_WEIGHT: tl.constexpr, + HAS_BIAS: tl.constexpr, + BLOCK_N: tl.constexpr, +): + row = tl.program_id(0) + X += row * stride_x_row + Y += row * stride_y_row + if HAS_WEIGHT: + W += 0 + if HAS_BIAS: + B += 0 + cols = tl.arange(0, BLOCK_N) + x = tl.load(X + cols, mask=cols < N, other=0.0).to(tl.float32) + if not IS_RMS_NORM: + mean = tl.sum(x, axis=0) / N + xbar = tl.where(cols < N, x - mean, 0.0) + var = tl.sum(xbar * xbar, axis=0) / N + else: + xbar = tl.where(cols < N, x, 0.0) + var = tl.sum(xbar * xbar, axis=0) / N + rstd = 1 / tl.sqrt(var + eps) + x_hat = (x - mean) * rstd if not IS_RMS_NORM else x * rstd + if HAS_WEIGHT: + w = tl.load(W + cols, mask=cols < N, other=1.0).to(tl.float32) + y = x_hat * w + else: + y = x_hat + if HAS_BIAS: + b = tl.load(B + cols, mask=cols < N, other=0.0).to(tl.float32) + y += b + tl.store(Y + cols, y, mask=cols < N) + + +def norm_infer( + x: Tensor, + weight: Optional[Tensor], + bias: Optional[Tensor], + eps: float, + is_rms_norm: bool = False, + out: Optional[Tensor] = None, +): + M, N = x.shape + x = x.contiguous() + if weight is not None: + assert weight.shape == (N,) + assert weight.stride(-1) == 1 + if bias is not None: + assert bias.shape == (N,) + assert bias.stride(-1) == 1 + if out is None: + out = torch.empty_like(x) + MAX_FUSED_SIZE = 65536 // x.element_size() + BLOCK_N = min(MAX_FUSED_SIZE, triton.next_power_of_2(N)) + if N > BLOCK_N: + raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.") + num_warps = min(max(BLOCK_N // 256, 1), 8) + _norm_infer_kernel[(M,)]( + x, + out, + weight if weight is not None else x, # dummy when HAS_WEIGHT=False + bias if bias is not None else x, # dummy when HAS_BIAS=False + x.stride(0), + out.stride(0), + M, + N, + eps, + IS_RMS_NORM=is_rms_norm, + HAS_WEIGHT=weight is not None, + HAS_BIAS=bias is not None, + BLOCK_N=BLOCK_N, + num_warps=num_warps, + ) + return out + + +def rms_norm_fn( + x, + weight, + bias, + residual=None, + x1=None, + weight1=None, + bias1=None, + eps=1e-6, + dropout_p=0.0, + rowscale=None, + prenorm=False, + residual_in_fp32=False, + zero_centered_weight=False, + return_dropout_mask=False, + out_dtype=None, + out=None, + residual_out=None, +): + return LayerNormFn.forward( + x, + weight, + bias, + residual, + x1, + weight1, + bias1, + eps, + dropout_p, + rowscale, + prenorm, + residual_in_fp32, + zero_centered_weight, + True, + return_dropout_mask, + out_dtype, + out, + residual_out, + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/usp.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/usp.py new file mode 100644 index 00000000..4f3804c9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/usp.py @@ -0,0 +1,255 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +import logging +from typing import TYPE_CHECKING + +import torch +import torch.distributed._functional_collectives as ft_c +from packaging.version import parse +from torch.distributed.tensor.experimental._attention import _cp_options + +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + get_sp_group, + get_ulysses_parallel_world_size, +) + +_cp_options.enable_load_balance = False + +if TYPE_CHECKING: + from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import ( + AttentionImpl, + ) + +logger = logging.getLogger(__name__) + + +def _maybe_wait(tensor: torch.Tensor) -> torch.Tensor: + """ + When tracing the code, the result tensor is not an AsyncCollectiveTensor, + so we cannot call ``wait()``. + """ + if isinstance(tensor, ft_c.AsyncCollectiveTensor): + return tensor.wait() + return tensor + + +def _usp_all_to_all_single(x: torch.Tensor) -> torch.Tensor: + ulysses_pg = get_sp_group().ulysses_group + assert ulysses_pg is not None, "Ulysses process group is not initialized." + x_shape = x.shape + x = x.flatten() + x = ft_c.all_to_all_single( + x, output_split_sizes=None, input_split_sizes=None, group=ulysses_pg + ) + x = _maybe_wait(x) + x = x.reshape(x_shape) + return x + + +def _usp_input_all_to_all(x: torch.Tensor, head_dim: int = 1) -> torch.Tensor: + """ + Perform Ulysses-style input all-to-all over the head dimension. + + Default layout expects heads at dim=1 and sequence at dim=2: + [b, h, s_local, d] -> [b, h // world_size, s_global, d] + + If heads are at dim=2 (input is [b, s_local, h, d]), set head_dim=2, and the + function returns [b, s_global, h // world_size, d], preserving the original + head/sequence dim ordering. + + Args: + x: A 4D tensor with layout [b, *, *, d] where '*' are sequence and heads + head_dim: Which dimension index corresponds to heads (1 or 2) + + Returns: + Tensor with the same dim order as input, with heads sharded and sequence gathered. + """ + world_size = get_ulysses_parallel_world_size() + if world_size <= 1: + return x + + assert x.ndim == 4, f"x must have 4 dimensions, got {x.ndim}" + assert head_dim in (1, 2), f"head_dim must be 1 or 2, got {head_dim}" + seq_dim = 1 if head_dim == 2 else 2 + + # Bring to canonical [b, h, s, d] + if head_dim == 1 and seq_dim == 2: + x_c = x + else: + x_c = x.permute(0, head_dim, seq_dim, 3).contiguous() + + b, h, s, d = x_c.shape + assert ( + h % world_size == 0 + ), f"h ({h}) must be divisible by world_size ({world_size})" + + # [b, h, s, d] -> [h, b, s, d] + x_c = x_c.permute(1, 0, 2, 3).contiguous() + # all-to-all along h + x_c = _usp_all_to_all_single(x_c) + # -> [b, h // world, s * world, d] + x_c = ( + x_c.reshape(world_size, h // world_size, b, -1, d) + .permute(2, 1, 0, 3, 4) + .reshape(b, h // world_size, -1, d) + ) + + if head_dim == 1 and seq_dim == 2: + return x_c + + # Map back to original ordering, preserving head/seq positions + new_order = [0, None, None, 3] + new_order[head_dim] = 1 + new_order[seq_dim] = 2 + return x_c.permute(tuple(new_order)).contiguous() + + +def _usp_output_all_to_all(x: torch.Tensor, head_dim: int = 1) -> torch.Tensor: + """ + Perform Ulysses-style output all-to-all over the head dimension (inverse of input). + + Default layout expects heads at dim=1 and sequence at dim=2: + [b, h // world_size, s_global, d] -> [b, h, s_local, d] + + If heads are at dim=2 (input is [b, s_global, h // world_size, d]), set head_dim=2, + and the function returns [b, s_local, h, d], preserving the original head/sequence + dim ordering. + + Args: + x: A 4D tensor with layout [b, *, *, d] where '*' are sequence and heads + head_dim: Which dimension index corresponds to heads (1 or 2) + + Returns: + Tensor with the same dim order as input, with heads gathered and sequence sharded. + """ + world_size = get_ulysses_parallel_world_size() + if world_size <= 1: + return x + + assert x.ndim == 4, f"x must have 4 dimensions, got {x.ndim}" + assert head_dim in (1, 2), f"head_dim must be 1 or 2, got {head_dim}" + seq_dim = 1 if head_dim == 2 else 2 + + # Bring to canonical [b, h, s, d] + if head_dim == 1 and seq_dim == 2: + x_c = x + else: + x_c = x.permute(0, head_dim, seq_dim, 3).contiguous() + + b, h, s, d = x_c.shape + assert ( + s % world_size == 0 + ), f"s ({s}) must be divisible by world_size ({world_size})" + + # [b, h, s, d] -> [s, b, h, d] + x_c = x_c.permute(2, 0, 1, 3).contiguous() + x_c = _usp_all_to_all_single(x_c) + # -> [b, h * world, s // world, d] + x_c = ( + x_c.reshape(world_size, s // world_size, b, -1, d) + .permute(2, 0, 3, 1, 4) + .reshape(b, -1, s // world_size, d) + ) + + if head_dim == 1 and seq_dim == 2: + return x_c + + # Map back to original ordering, preserving head/seq positions + new_order = [0, None, None, 3] + new_order[head_dim] = 1 + new_order[seq_dim] = 2 + return x_c.permute(tuple(new_order)).contiguous() + + +def ring_attn( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_impl: "AttentionImpl", + is_causal: bool = False, + dropout_p: float = 0.0, +): + """ + Ring Attention implementation. + + This function implements Ring Attention, a strategy for distributed attention + computation that reduces peak memory usage. It accepts a generic attention + implementation (`attn_impl`) which is called by the underlying PyTorch + distributed attention primitive. + + Args: + query, key, value: The input tensors for attention. + attn_impl: An instance of an attention implementation backend + (e.g., FlashAttentionImpl) whose `forward` method will be + used as the computational kernel. + is_causal: Whether to apply causal masking. + dropout_p: Dropout probability. + """ + # torch.distributed.tensor.experimental._attention is not a public API, + from torch.distributed.tensor.experimental._attention import ( + _templated_ring_attention, + ) + + ring_pg = get_sp_group().ring_group + assert ring_pg is not None, "Ring process group is not initialized." + + # Ring attention primitives expect tensors in [B, H, S, D] layout. + # We permute the inputs here. + query = torch.permute(query, [0, 2, 1, 3]).contiguous() + key = torch.permute(key, [0, 2, 1, 3]).contiguous() + value = torch.permute(value, [0, 2, 1, 3]).contiguous() + + # Create an adapter function that matches the signature expected by + # _templated_ring_attention. The `attn_impl` already has dropout and + # causal settings configured during its initialization. + + # Note: Please be aware that Attention Backend and Ring Attention may require different QKV tensor shapes. + # For example, FlashAttention expects the format to be BSHD. + def attn_callable_adapter(q, k, v, *args, **kwargs): + # We ignore the dropout_p and is_causal passed by _templated_ring_attention + # and rely on the pre-configured attn_impl. + # The `attn_metadata` is not available here, so we pass None. + # This is a limitation we must accept when using this experimental API. + q = torch.permute(q, [0, 2, 1, 3]) + k = torch.permute(k, [0, 2, 1, 3]) + v = torch.permute(v, [0, 2, 1, 3]) + # logger.warning(f"Warning: return_s·oftmax_lse is only supported for FlashAttentionImpl") + output, softmax_lse, *rest = attn_impl.forward( + q, + k, + v, + attn_metadata=None, + return_softmax_lse=True, + ) + output = torch.permute(output, [0, 2, 1, 3]) + return output, softmax_lse, *rest + + # Starting from torch 2.6.0, _templated_ring_attention expects an integer + # segment_id for the attention function. + use_segment_id = parse(torch.__version__).release >= parse("2.6.0").release + + attn_kwargs = dict( + mesh=ring_pg, + op=attn_callable_adapter, + dropout_p=dropout_p, + is_causal=is_causal, + query=query, + key=key, + value=value, + ) + + if use_segment_id: + # For torch >= 2.6, segment_id is required. The value '1' is a placeholder + # as we are not using complex segmentation features. + out, *_ = _templated_ring_attention( + seq_dim=1, # segment_id + **attn_kwargs, + ) + else: + out, *_ = _templated_ring_attention( + **attn_kwargs, + ) + + # Permute the output back to [B, S, H, D] layout. + output = torch.permute(out, [0, 2, 1, 3]) + return output diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/utils.py new file mode 100644 index 00000000..615ebc38 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/utils.py @@ -0,0 +1,24 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/model_executor/layers/utils.py +"""Utility methods for model layers.""" + +import torch + + +def get_token_bin_counts_and_mask( + tokens: torch.Tensor, + vocab_size: int, + num_seqs: int, +) -> tuple[torch.Tensor, torch.Tensor]: + # Compute the bin counts for the tokens. + # vocab_size + 1 for padding. + bin_counts = torch.zeros( + (num_seqs, vocab_size + 1), dtype=torch.long, device=tokens.device + ) + bin_counts.scatter_add_(1, tokens, torch.ones_like(tokens)) + bin_counts = bin_counts[:, :vocab_size] + mask = bin_counts > 0 + + return bin_counts, mask diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/visual_embedding.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/visual_embedding.py new file mode 100644 index 00000000..d556ab58 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/visual_embedding.py @@ -0,0 +1,190 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +import math + +import torch +import torch.nn as nn + +from sglang.multimodal_gen.runtime.layers.activation import get_act_fn +from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear +from sglang.multimodal_gen.runtime.layers.mlp import MLP + + +class PatchEmbed(nn.Module): + """2D Image to Patch Embedding + + Image to Patch Embedding using Conv2d + + A convolution based approach to patchifying a 2D image w/ embedding projection. + + Based on the impl in https://github.com/google-research/vision_transformer + + Hacked together by / Copyright 2020 Ross Wightman + + Remove the _assert function in forward function to be compatible with multi-resolution images. + """ + + def __init__( + self, + patch_size=16, + in_chans=3, + embed_dim=768, + norm_layer=None, + flatten=True, + bias=True, + dtype=None, + prefix: str = "", + ): + super().__init__() + # Convert patch_size to 2-tuple + if isinstance(patch_size, list | tuple): + if len(patch_size) == 1: + patch_size = (patch_size[0], patch_size[0]) + else: + patch_size = (patch_size, patch_size) + + self.patch_size = patch_size + self.flatten = flatten + + self.proj = nn.Conv3d( + in_chans, + embed_dim, + kernel_size=patch_size, + stride=patch_size, + bias=bias, + dtype=dtype, + ) + self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() + + def forward(self, x): + x = self.proj(x) + if self.flatten: + x = x.flatten(2).transpose(1, 2) # BCHW -> BNC + x = self.norm(x) + return x + + +class TimestepEmbedder(nn.Module): + """ + Embeds scalar timesteps into vector representations. + """ + + def __init__( + self, + hidden_size, + act_layer="silu", + frequency_embedding_size=256, + max_period=10000, + dtype=None, + freq_dtype=torch.float32, + prefix: str = "", + ): + super().__init__() + self.frequency_embedding_size = frequency_embedding_size + self.max_period = max_period + + self.mlp = MLP( + frequency_embedding_size, + hidden_size, + hidden_size, + act_type=act_layer, + dtype=dtype, + ) + self.freq_dtype = freq_dtype + + def forward( + self, t: torch.Tensor, timestep_seq_len: int | None = None + ) -> torch.Tensor: + t_freq = timestep_embedding( + t, self.frequency_embedding_size, self.max_period, dtype=self.freq_dtype + ).to(self.mlp.fc_in.weight.dtype) + if timestep_seq_len is not None: + assert ( + t_freq.shape[0] % timestep_seq_len == 0 + ), "timestep length is not divisible by timestep_seq_len" + batch_size = t_freq.shape[0] // timestep_seq_len + t_freq = t_freq.unflatten(0, (batch_size, timestep_seq_len)) + # t_freq = t_freq.to(self.mlp.fc_in.weight.dtype) + t_emb = self.mlp(t_freq) + return t_emb + + +def timestep_embedding( + t: torch.Tensor, + dim: int, + max_period: int = 10000, + dtype: torch.dtype = torch.float32, +) -> torch.Tensor: + """ + Create sinusoidal timestep embeddings. + + Args: + t: Tensor of shape [B] with timesteps + dim: Embedding dimension + max_period: Controls the minimum frequency of the embeddings + + Returns: + Tensor of shape [B, dim] with embeddings + """ + half = dim // 2 + freqs = torch.exp( + -math.log(max_period) + * torch.arange(start=0, end=half, dtype=dtype, device=t.device) + / half + ) + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + return embedding + + +class ModulateProjection(nn.Module): + """Modulation layer for DiT blocks.""" + + def __init__( + self, + hidden_size: int, + factor: int = 2, + act_layer: str = "silu", + dtype: torch.dtype | None = None, + prefix: str = "", + ): + super().__init__() + self.factor = factor + self.hidden_size = hidden_size + self.linear = ReplicatedLinear( + hidden_size, hidden_size * factor, bias=True, params_dtype=dtype + ) + self.act = get_act_fn(act_layer) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.act(x) + x, _ = self.linear(x) + return x + + +def unpatchify(x, t, h, w, patch_size, channels) -> torch.Tensor: + """ + Convert patched representation back to image space. + + Args: + x: Tensor of shape [B, T*H*W, C*P_t*P_h*P_w] + t, h, w: Temporal and spatial dimensions + + Returns: + Unpatchified tensor of shape [B, C, T*P_t, H*P_h, W*P_w] + """ + assert x.ndim == 3, f"x.ndim: {x.ndim}" + assert len(patch_size) == 3, f"patch_size: {patch_size}" + assert t * h * w == x.shape[1], f"t * h * w: {t * h * w}, x.shape[1]: {x.shape[1]}" + c = channels + pt, ph, pw = patch_size + + x = x.reshape(shape=(x.shape[0], t, h, w, c, pt, ph, pw)) + x = torch.einsum("nthwcopq->nctohpwq", x) + imgs = x.reshape(shape=(x.shape[0], c, t * pt, h * ph, w * pw)) + + return imgs diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/vocab_parallel_embedding.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/vocab_parallel_embedding.py new file mode 100644 index 00000000..fbddaab4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/layers/vocab_parallel_embedding.py @@ -0,0 +1,480 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +from collections.abc import Sequence +from dataclasses import dataclass + +import torch +import torch.nn.functional as F +from torch.nn.parameter import Parameter, UninitializedParameter + +from sglang.multimodal_gen.runtime.distributed import ( + divide, + get_tp_rank, + get_tp_world_size, + tensor_model_parallel_all_reduce, +) +from sglang.multimodal_gen.runtime.layers.quantization.base_config import ( + QuantizationConfig, + QuantizeMethodBase, + method_has_implemented_embedding, +) +from sglang.multimodal_gen.runtime.models.parameter import BasevLLMParameter +from sglang.multimodal_gen.runtime.models.utils import set_weight_attrs +from sglang.multimodal_gen.runtime.platforms import current_platform + +DEFAULT_VOCAB_PADDING_SIZE = 64 + + +class UnquantizedEmbeddingMethod(QuantizeMethodBase): + """Unquantized method for embeddings.""" + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: list[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + """Create weights for embedding layer.""" + + weight = Parameter( + torch.empty( + sum(output_partition_sizes), + input_size_per_partition, + dtype=params_dtype, + ), + requires_grad=False, + ) + set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0}) + layer.register_parameter("weight", weight) + set_weight_attrs(weight, extra_weight_attrs) + + def apply( + self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None = None + ) -> torch.Tensor: + return F.linear(x, layer.weight, bias) + + def embedding(self, layer: torch.nn.Module, input_: torch.Tensor) -> torch.Tensor: + return F.embedding(input_, layer.weight) + + +def pad_vocab_size(vocab_size: int, pad_to: int = DEFAULT_VOCAB_PADDING_SIZE) -> int: + """Pad the vocab size to the given value.""" + return ((vocab_size + pad_to - 1) // pad_to) * pad_to + + +def vocab_range_from_per_partition_vocab_size( + per_partition_vocab_size: int, rank: int, offset: int = 0 +) -> Sequence[int]: + index_f = rank * per_partition_vocab_size + index_l = index_f + per_partition_vocab_size + return index_f + offset, index_l + offset + + +def vocab_range_from_global_vocab_size( + global_vocab_size: int, rank: int, world_size: int, offset: int = 0 +) -> Sequence[int]: + per_partition_vocab_size = divide(global_vocab_size, world_size) + return vocab_range_from_per_partition_vocab_size( + per_partition_vocab_size, rank, offset=offset + ) + + +@dataclass +class VocabParallelEmbeddingShardIndices: + """Indices for a shard of a vocab parallel embedding.""" + + padded_org_vocab_start_index: int + padded_org_vocab_end_index: int + padded_added_vocab_start_index: int + padded_added_vocab_end_index: int + + org_vocab_start_index: int + org_vocab_end_index: int + added_vocab_start_index: int + added_vocab_end_index: int + + @property + def num_org_elements(self) -> int: + return self.org_vocab_end_index - self.org_vocab_start_index + + @property + def num_added_elements(self) -> int: + return self.added_vocab_end_index - self.added_vocab_start_index + + @property + def num_org_elements_padded(self) -> int: + return self.padded_org_vocab_end_index - self.padded_org_vocab_start_index + + @property + def num_added_elements_padded(self) -> int: + return self.padded_added_vocab_end_index - self.padded_added_vocab_start_index + + @property + def num_org_vocab_padding(self) -> int: + return self.num_org_elements_padded - self.num_org_elements + + @property + def num_added_vocab_padding(self) -> int: + return self.num_added_elements_padded - self.num_added_elements + + @property + def num_elements_padded(self) -> int: + return self.num_org_elements_padded + self.num_added_elements_padded + + def __post_init__(self): + # sanity checks + assert self.padded_org_vocab_start_index <= self.padded_org_vocab_end_index + assert self.padded_added_vocab_start_index <= self.padded_added_vocab_end_index + + assert self.org_vocab_start_index <= self.org_vocab_end_index + assert self.added_vocab_start_index <= self.added_vocab_end_index + + assert self.org_vocab_start_index <= self.padded_org_vocab_start_index + assert self.added_vocab_start_index <= self.padded_added_vocab_start_index + assert self.org_vocab_end_index <= self.padded_org_vocab_end_index + assert self.added_vocab_end_index <= self.padded_added_vocab_end_index + + assert self.num_org_elements <= self.num_org_elements_padded + assert self.num_added_elements <= self.num_added_elements_padded + + +@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend) +def get_masked_input_and_mask( + input_: torch.Tensor, + org_vocab_start_index: int, + org_vocab_end_index: int, + num_org_vocab_padding: int, + added_vocab_start_index: int, + added_vocab_end_index: int, +) -> tuple[torch.Tensor, torch.Tensor]: + # torch.compile will fuse all of the pointwise ops below + # into a single kernel, making it very fast + org_vocab_mask = (input_ >= org_vocab_start_index) & (input_ < org_vocab_end_index) + added_vocab_mask = (input_ >= added_vocab_start_index) & ( + input_ < added_vocab_end_index + ) + added_offset = ( + added_vocab_start_index + - (org_vocab_end_index - org_vocab_start_index) + - num_org_vocab_padding + ) + valid_offset = (org_vocab_start_index * org_vocab_mask) + ( + added_offset * added_vocab_mask + ) + vocab_mask = org_vocab_mask | added_vocab_mask + input_ = vocab_mask * (input_ - valid_offset) + return input_, ~vocab_mask + + +class VocabParallelEmbedding(torch.nn.Module): + """Embedding parallelized in the vocabulary dimension. + + Adapted from torch.nn.Embedding, note that we pad the vocabulary size to + make sure it is divisible by the number of model parallel GPUs. + + In order to support various loading methods, we ensure that LoRA-added + embeddings are always at the end of TP-sharded tensors. In other words, + we shard base embeddings and LoRA embeddings separately (both padded), + and place them in the same tensor. + In this example, we will have the original vocab size = 1010, + added vocab size = 16 and padding to 64. Therefore, the total + vocab size with padding will be 1088 (because we first pad 1010 to + 1024, add 16, and then pad to 1088). + Therefore, the tensor format looks like the following: + TP1, rank 0 (no sharding): + |< --------BASE-------- >|< -BASE PADDING-- >|< -----LORA------ >|< -LORA PADDING-- >| + corresponding token_id: | 0 | 1 | ... | 1009 | -1 | ... | -1 | 1010 | ... | 1015 | -1 | ... | -1 | + index: | 0 | 1 | ... | 1009 | 1010 | ... | 1023 | 1024 | ... | 1039 | 1040 | ... | 1087 | + + TP2, rank 0: + |< --------------------BASE--------------------- >|< -----LORA------ >|< -LORA PADDING- >| + corresponding token_id: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 1000 | ... | 1015 | -1 | ... | -1 | + index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 527 | 520 | ... | 543 | + TP2, rank 1: + |< -----------BASE----------- >|< -BASE PADDING- >|< -----------LORA PADDING----------- >| + corresponding token_id: | 512 | 513 | 514 | ... | 1009 | -1 | ... | -1 | -1 | ... | -1 | -1 | ... | -1 | + index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 519 | 520 | ... | 543 | + + Args: + num_embeddings: vocabulary size. + embedding_dim: size of hidden state. + params_dtype: type of the parameters. + org_num_embeddings: original vocabulary size (without LoRA). + padding_size: padding size for the vocabulary. + quant_config: quant config for the layer + prefix: full name of the layer in the state dict + """ # noqa: E501 + + def __init__( + self, + num_embeddings: int, + embedding_dim: int, + params_dtype: torch.dtype | None = None, + org_num_embeddings: int | None = None, + padding_size: int = DEFAULT_VOCAB_PADDING_SIZE, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ): + super().__init__() + + # Keep the input dimensions. + tp_rank = get_tp_rank() + self.tp_size = get_tp_world_size() + self.num_embeddings = num_embeddings + self.padding_size = padding_size + self.org_vocab_size = org_num_embeddings or num_embeddings + num_added_embeddings = num_embeddings - self.org_vocab_size + self.org_vocab_size_padded = pad_vocab_size( + self.org_vocab_size, self.padding_size + ) + self.num_embeddings_padded = pad_vocab_size( + self.org_vocab_size_padded + num_added_embeddings, self.padding_size + ) + assert self.org_vocab_size_padded <= self.num_embeddings_padded + + self.shard_indices = self._get_indices( + self.num_embeddings_padded, + self.org_vocab_size_padded, + self.num_embeddings, + self.org_vocab_size, + tp_rank, + self.tp_size, + ) + self.embedding_dim = embedding_dim + + quant_method = None + if quant_config is not None: + quant_method = quant_config.get_quant_method(self, prefix=prefix) + if quant_method is None: + quant_method = UnquantizedEmbeddingMethod() + + # If we are making an embedding layer, then our quantization linear + # method must implement the embedding operation. If we are another + # layer type like ParallelLMHead, this is not important. + is_embedding_layer = type(self.__class__) is VocabParallelEmbedding + quant_method_implements_embedding = method_has_implemented_embedding( + type(quant_method) + ) + if is_embedding_layer and not quant_method_implements_embedding: + raise NotImplementedError( + f"The class {type(quant_method).__name__} must implement " + "the 'embedding' method, see UnquantizedEmbeddingMethod." + ) + + self.quant_method: QuantizeMethodBase = quant_method + + if params_dtype is None: + params_dtype = torch.get_default_dtype() + # Divide the weight matrix along the vocaburaly dimension. + self.num_added_embeddings = self.num_embeddings - self.org_vocab_size + self.num_embeddings_per_partition = divide( + self.num_embeddings_padded, self.tp_size + ) + assert ( + self.shard_indices.num_elements_padded == self.num_embeddings_per_partition + ) + self.num_org_embeddings_per_partition = ( + self.shard_indices.org_vocab_end_index + - self.shard_indices.org_vocab_start_index + ) + self.num_added_embeddings_per_partition = ( + self.shard_indices.added_vocab_end_index + - self.shard_indices.added_vocab_start_index + ) + + self.quant_method.create_weights( + self, + self.embedding_dim, + [self.num_embeddings_per_partition], + self.embedding_dim, + self.num_embeddings_padded, + params_dtype=params_dtype, + weight_loader=self.weight_loader, + ) + + @classmethod + def _get_indices( + cls, + vocab_size_padded: int, + org_vocab_size_padded: int, + vocab_size: int, + org_vocab_size: int, + tp_rank: int, + tp_size: int, + ) -> VocabParallelEmbeddingShardIndices: + """Get start and end indices for vocab parallel embedding, following the + layout outlined in the class docstring, based on the given tp_rank and + tp_size.""" + num_added_embeddings_padded = vocab_size_padded - org_vocab_size_padded + padded_org_vocab_start_index, padded_org_vocab_end_index = ( + vocab_range_from_global_vocab_size(org_vocab_size_padded, tp_rank, tp_size) + ) + padded_added_vocab_start_index, padded_added_vocab_end_index = ( + vocab_range_from_global_vocab_size( + num_added_embeddings_padded, tp_rank, tp_size, offset=org_vocab_size + ) + ) + # remove padding + org_vocab_start_index = min(padded_org_vocab_start_index, org_vocab_size) + org_vocab_end_index = min(padded_org_vocab_end_index, org_vocab_size) + added_vocab_start_index = min(padded_added_vocab_start_index, vocab_size) + added_vocab_end_index = min(padded_added_vocab_end_index, vocab_size) + return VocabParallelEmbeddingShardIndices( + padded_org_vocab_start_index, + padded_org_vocab_end_index, + padded_added_vocab_start_index, + padded_added_vocab_end_index, + org_vocab_start_index, + org_vocab_end_index, + added_vocab_start_index, + added_vocab_end_index, + ) + + def get_sharded_to_full_mapping(self) -> list[int] | None: + """Get a mapping that can be used to reindex the gathered + logits for sampling. + + During sampling, we gather logits from all ranks. The relationship + of index->token_id will follow the same format as outlined in the class + docstring. However, after the gather, we want to reindex the final + logits tensor to map index->token_id one-to-one (the index is always + equal the token_id it corresponds to). The indices returned by this + method allow us to do that. + """ + if self.tp_size < 2: + return None + + base_embeddings: list[int] = [] + added_embeddings: list[int] = [] + padding: list[int] = [] + for tp_rank in range(self.tp_size): + shard_indices = self._get_indices( + self.num_embeddings_padded, + self.org_vocab_size_padded, + self.num_embeddings, + self.org_vocab_size, + tp_rank, + self.tp_size, + ) + range_start = self.num_embeddings_per_partition * tp_rank + range_end = self.num_embeddings_per_partition * (tp_rank + 1) + base_embeddings.extend( + range(range_start, range_start + shard_indices.num_org_elements) + ) + padding.extend( + range( + range_start + shard_indices.num_org_elements, + range_start + shard_indices.num_org_elements_padded, + ) + ) + added_embeddings.extend( + range( + range_start + shard_indices.num_org_elements_padded, + range_start + + shard_indices.num_org_elements_padded + + shard_indices.num_added_elements, + ) + ) + padding.extend( + range( + range_start + + shard_indices.num_org_elements_padded + + shard_indices.num_added_elements, + range_start + + shard_indices.num_org_elements_padded + + shard_indices.num_added_elements_padded, + ) + ) + assert ( + range_start + + shard_indices.num_org_elements_padded + + shard_indices.num_added_elements_padded + == range_end + ) + ret = base_embeddings + added_embeddings + padding + assert len(ret) == self.num_embeddings_padded + return ret + + def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor): + output_dim = getattr(param, "output_dim", None) + packed_dim = getattr(param, "packed_dim", None) + + # If the parameter is a gguf weight, then load it directly. + if getattr(param, "is_gguf_weight_type", None): + param.data.copy_(loaded_weight) + param.weight_type = loaded_weight.item() + return + elif isinstance(param, UninitializedParameter): + shape = list(loaded_weight.shape) + if output_dim is not None: + shape[output_dim] = self.num_embeddings_per_partition + param.materialize(tuple(shape), dtype=loaded_weight.dtype) + + # If parameter does not have output dim, then it should + # be copied onto all gpus (e.g. g_idx for act_order gptq). + if output_dim is None: + assert param.data.shape == loaded_weight.shape + param.data.copy_(loaded_weight) + return + + # Shard indexes for loading the weight + start_idx = self.shard_indices.org_vocab_start_index + shard_size = self.shard_indices.org_vocab_end_index - start_idx + + # If param packed on the same dim we are sharding on, then + # need to adjust offsets of loaded weight by pack_factor. + if packed_dim is not None and packed_dim == output_dim: + packed_factor = ( + param.packed_factor + if isinstance(param, BasevLLMParameter) + else param.pack_factor + ) + assert loaded_weight.shape[output_dim] == ( + self.org_vocab_size // param.packed_factor + ) + start_idx = start_idx // packed_factor + shard_size = shard_size // packed_factor + else: + assert loaded_weight.shape[output_dim] == self.org_vocab_size + + # Copy the data. Select chunk corresponding to current shard. + loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size) + + param[: loaded_weight.shape[0]].data.copy_(loaded_weight) + param[loaded_weight.shape[0] :].data.fill_(0) + + def forward(self, input_): + if self.tp_size > 1: + # Build the mask. + masked_input, input_mask = get_masked_input_and_mask( + input_, + self.shard_indices.org_vocab_start_index, + self.shard_indices.org_vocab_end_index, + self.shard_indices.num_org_vocab_padding, + self.shard_indices.added_vocab_start_index, + self.shard_indices.added_vocab_end_index, + ) + else: + masked_input = input_ + # Get the embeddings. + output_parallel = self.quant_method.embedding(self, masked_input.long()) + # Mask the output embedding. + if self.tp_size > 1: + output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0) + # Reduce across all the model parallel GPUs. + output = tensor_model_parallel_all_reduce(output_parallel) + return output + + def extra_repr(self) -> str: + s = f"num_embeddings={self.num_embeddings_per_partition}" + s += f", embedding_dim={self.embedding_dim}" + s += f", org_vocab_size={self.org_vocab_size}" + s += f", num_embeddings_padded={self.num_embeddings_padded}" + s += f", tp_size={self.tp_size}" + return s diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/loader/component_loader.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/loader/component_loader.py new file mode 100644 index 00000000..3e93e80a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/loader/component_loader.py @@ -0,0 +1,766 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +import dataclasses +import glob +import json +import os +import time +from abc import ABC, abstractmethod +from collections.abc import Generator, Iterable +from copy import deepcopy +from typing import Any, cast + +import torch +import torch.distributed as dist +import torch.nn as nn +from safetensors.torch import load_file as safetensors_load_file +from torch.distributed import init_device_mesh +from transformers import AutoImageProcessor, AutoProcessor, AutoTokenizer +from transformers.utils import SAFE_WEIGHTS_INDEX_NAME + +from sglang.multimodal_gen.configs.models import EncoderConfig, ModelConfig +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.loader.fsdp_load import ( + maybe_load_fsdp_model, + shard_model, +) +from sglang.multimodal_gen.runtime.loader.utils import set_default_torch_dtype +from sglang.multimodal_gen.runtime.loader.weight_utils import ( + filter_duplicate_safetensors_files, + filter_files_not_needed_for_inference, + pt_weights_iterator, + safetensors_weights_iterator, +) +from sglang.multimodal_gen.runtime.models.registry import ModelRegistry +from sglang.multimodal_gen.runtime.platforms import current_platform +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import ( + get_config, + get_diffusers_component_config, + get_hf_config, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.utils import PRECISION_TO_TYPE + +logger = init_logger(__name__) + + +class skip_init_modules: + def __enter__(self): + # Save originals + self._orig_reset = {} + for cls in (nn.Linear, nn.Conv1d, nn.Conv2d, nn.Conv3d): + self._orig_reset[cls] = cls.reset_parameters + cls.reset_parameters = lambda self: None # skip init + + def __exit__(self, exc_type, exc_value, traceback): + # restore originals + for cls, orig in self._orig_reset.items(): + cls.reset_parameters = orig + + +def _normalize_module_type(module_type: str) -> str: + """Normalize module types like 'text_encoder_2' -> 'text_encoder'.""" + if module_type.endswith("_2"): + return module_type[:-2] + return module_type + + +def _clean_hf_config_inplace(model_config: dict) -> None: + """Remove common extraneous HF fields if present.""" + for key in ( + "_name_or_path", + "transformers_version", + "model_type", + "tokenizer_class", + "torch_dtype", + ): + model_config.pop(key, None) + + +def _list_safetensors_files(model_path: str) -> list[str]: + """List all .safetensors files under a directory.""" + return sorted(glob.glob(os.path.join(str(model_path), "*.safetensors"))) + + +def load_native(library, component_module_path: str, server_args: ServerArgs): + if library == "transformers": + from transformers import AutoModel + + config = get_hf_config( + component_module_path, + trust_remote_code=server_args.trust_remote_code, + revision=server_args.revision, + ) + return AutoModel.from_pretrained( + component_module_path, + config=config, + trust_remote_code=server_args.trust_remote_code, + revision=server_args.revision, + ) + elif library == "diffusers": + import diffusers + + config = get_diffusers_component_config(model_path=component_module_path) + class_name = config.pop("_class_name", None) + if class_name: + cls = getattr(diffusers, class_name) + return cls.from_pretrained( + component_module_path, revision=server_args.revision, **config + ) + else: + raise ValueError("Cannot determine class name for generic diffusers loader") + else: + raise ValueError(f"Unsupported library: {library}") + + +class ComponentLoader(ABC): + """Base class for loading a specific type of model component.""" + + def __init__(self, device=None) -> None: + self.device = device + + def should_offload(self, server_args, model_config: ModelConfig | None = None): + raise NotImplementedError() + + def target_device(self, should_offload): + if should_offload: + return ( + torch.device("mps") + if current_platform.is_mps() + else torch.device("cpu") + ) + else: + return get_local_torch_device() + + def load( + self, + component_model_path: str, + server_args: ServerArgs, + module_name: str, + transformers_or_diffusers: str, + ): + """ + Template method that standardizes logging around the core load implementation. + The priority of loading method is: + 1. load customized module + 2. load native diffusers/transformers module + If all of the above methods failed, an error will be thrown + + """ + logger.info("Loading %s from %s", module_name, component_model_path) + try: + component = self.load_customized( + component_model_path, server_args, module_name + ) + source = "customized" + except Exception as _e: + # fallback to native version + component = self.load_native( + component_model_path, server_args, transformers_or_diffusers + ) + should_offload = self.should_offload(server_args) + target_device = self.target_device(should_offload) + component = component.to(device=target_device) + source = "native" + logger.warning( + "Native module %s: %s is loaded, performance may be sub-optimal", + module_name, + component.__class__.__name__, + ) + + if component is None: + logger.warning("Loaded %s returned None", module_name) + else: + logger.info( + f"Loaded %s: %s from: {source}", + module_name, + component.__class__.__name__, + ) + return component + + def load_native( + self, + component_model_path: str, + server_args: ServerArgs, + transformers_or_diffusers: str, + ): + """ + Load the component using the native library (transformers/diffusers). + """ + return load_native(transformers_or_diffusers, component_model_path, server_args) + + def load_customized( + self, component_model_path: str, server_args: ServerArgs, module_name: str + ): + """ + Load the customized version component, implemented and optimized in SGL-diffusion + """ + raise NotImplementedError( + f"load_customized not implemented for {self.__class__.__name__}" + ) + + @abstractmethod + def load_customized( + self, model_path: str, server_args: ServerArgs, module_name: str + ) -> Any: + """Implement the minimal core load logic in subclasses.""" + raise NotImplementedError + + @classmethod + def for_module_type( + cls, module_type: str, transformers_or_diffusers: str + ) -> "ComponentLoader": + """ + Factory method to create a component loader for a specific module type. + + Args: + module_type: Type of module (e.g., "vae", "text_encoder", "transformer", "scheduler") + transformers_or_diffusers: Whether the module is from transformers or diffusers + + Returns: + A component loader for the specified module type + """ + # Map of module types to their loader classes and expected library + module_type = _normalize_module_type(module_type) + module_loaders = { + "scheduler": (SchedulerLoader, "diffusers"), + "transformer": (TransformerLoader, "diffusers"), + "vae": (VAELoader, "diffusers"), + "text_encoder": (TextEncoderLoader, "transformers"), + "tokenizer": (TokenizerLoader, "transformers"), + "image_processor": (ImageProcessorLoader, "transformers"), + "image_encoder": (ImageEncoderLoader, "transformers"), + "processor": (AutoProcessorLoader, "transformers"), + } + + if module_type in module_loaders: + loader_cls, expected_library = module_loaders[module_type] + # Assert that the library matches what's expected for this module type + assert ( + transformers_or_diffusers == expected_library + ), f"{module_type} must be loaded from {expected_library}, got {transformers_or_diffusers}" + return loader_cls() + + # For unknown module types, use a generic loader + logger.warning( + "No specific loader found for module type: %s. Using generic loader.", + module_type, + ) + return GenericComponentLoader(transformers_or_diffusers) + + +class TextEncoderLoader(ComponentLoader): + """Loader for text encoders.""" + + @dataclasses.dataclass + class Source: + """A source for weights.""" + + model_or_path: str + """The model ID or path.""" + + prefix: str = "" + """A prefix to prepend to all weights.""" + + fall_back_to_pt: bool = True + """Whether .pt weights can be used.""" + + allow_patterns_overrides: list[str] | None = None + """If defined, weights will load exclusively using these patterns.""" + + counter_before_loading_weights: float = 0.0 + counter_after_loading_weights: float = 0.0 + + def should_offload(self, server_args, model_config: ModelConfig | None = None): + should_offload = server_args.text_encoder_cpu_offload + fsdp_shard_conditions = getattr(model_config, "_fsdp_shard_conditions", []) + use_cpu_offload = should_offload and len(fsdp_shard_conditions) > 0 + return use_cpu_offload + + def _prepare_weights( + self, + model_name_or_path: str, + fall_back_to_pt: bool, + allow_patterns_overrides: list[str] | None, + ) -> tuple[str, list[str], bool]: + """Prepare weights for the model. + + If the model is not local, it will be downloaded.""" + # model_name_or_path = (self._maybe_download_from_modelscope( + # model_name_or_path, revision) or model_name_or_path) + + is_local = os.path.isdir(model_name_or_path) + assert is_local, "Model path must be a local directory" + + use_safetensors = False + index_file = SAFE_WEIGHTS_INDEX_NAME + allow_patterns = ["*.safetensors", "*.bin"] + + if fall_back_to_pt: + allow_patterns += ["*.pt"] + + if allow_patterns_overrides is not None: + allow_patterns = allow_patterns_overrides + + hf_folder = model_name_or_path + + hf_weights_files: list[str] = [] + for pattern in allow_patterns: + hf_weights_files += glob.glob(os.path.join(hf_folder, pattern)) + if len(hf_weights_files) > 0: + if pattern == "*.safetensors": + use_safetensors = True + break + + if use_safetensors: + hf_weights_files = filter_duplicate_safetensors_files( + hf_weights_files, hf_folder, index_file + ) + else: + hf_weights_files = filter_files_not_needed_for_inference(hf_weights_files) + + if len(hf_weights_files) == 0: + raise RuntimeError( + f"Cannot find any model weights with `{model_name_or_path}`" + ) + + return hf_folder, hf_weights_files, use_safetensors + + def _get_weights_iterator( + self, source: "Source", to_cpu: bool + ) -> Generator[tuple[str, torch.Tensor], None, None]: + """get an iterator for the model weights based on the load format.""" + hf_folder, hf_weights_files, use_safetensors = self._prepare_weights( + source.model_or_path, + source.fall_back_to_pt, + source.allow_patterns_overrides, + ) + if use_safetensors: + weights_iterator = safetensors_weights_iterator( + hf_weights_files, to_cpu=to_cpu + ) + else: + weights_iterator = pt_weights_iterator(hf_weights_files, to_cpu=to_cpu) + + if self.counter_before_loading_weights == 0.0: + self.counter_before_loading_weights = time.perf_counter() + # apply the prefix. + return ((source.prefix + name, tensor) for (name, tensor) in weights_iterator) + + def _get_all_weights( + self, + model: nn.Module, + model_path: str, + to_cpu: bool, + ) -> Generator[tuple[str, torch.Tensor], None, None]: + primary_weights = TextEncoderLoader.Source( + model_path, + prefix="", + fall_back_to_pt=getattr(model, "fall_back_to_pt_during_load", True), + allow_patterns_overrides=getattr(model, "allow_patterns_overrides", None), + ) + yield from self._get_weights_iterator(primary_weights, to_cpu) + + secondary_weights = cast( + Iterable[TextEncoderLoader.Source], + getattr(model, "secondary_weights", ()), + ) + for source in secondary_weights: + yield from self._get_weights_iterator(source, to_cpu) + + def load_customized( + self, component_model_path: str, server_args: ServerArgs, module_name: str + ): + """Load the text encoders based on the model path, and inference args.""" + # model_config: PretrainedConfig = get_hf_config( + # model=model_path, + # trust_remote_code=server_args.trust_remote_code, + # revision=server_args.revision, + # model_override_args=None, + # ) + diffusers_pretrained_config = get_config( + component_model_path, trust_remote_code=True + ) + model_config = get_diffusers_component_config(model_path=component_model_path) + _clean_hf_config_inplace(model_config) + logger.info("HF model config: %s", model_config) + + def is_not_first_encoder(module_name): + return "2" in module_name + + # TODO(mick): had to throw an exception for different text-encoder arch + if not is_not_first_encoder(module_name): + encoder_config = server_args.pipeline_config.text_encoder_configs[0] + encoder_config.update_model_arch(model_config) + for key, value in diffusers_pretrained_config.__dict__.items(): + setattr(encoder_config.arch_config, key, value) + encoder_dtype = server_args.pipeline_config.text_encoder_precisions[0] + else: + assert len(server_args.pipeline_config.text_encoder_configs) == 2 + encoder_config = server_args.pipeline_config.text_encoder_configs[1] + encoder_config.update_model_arch(model_config) + encoder_dtype = server_args.pipeline_config.text_encoder_precisions[1] + # TODO(will): add support for other dtypes + return self.load_model( + component_model_path, + encoder_config, + server_args, + encoder_dtype, + ) + + def load_model( + self, + model_path: str, + model_config: EncoderConfig, + server_args: ServerArgs, + dtype: str = "fp16", + cpu_offload_flag: bool | None = None, + ): + # Determine CPU offload behavior and target device + + local_torch_device = get_local_torch_device() + should_offload = self.should_offload(server_args, model_config) + with set_default_torch_dtype(PRECISION_TO_TYPE[dtype]): + with local_torch_device, skip_init_modules(): + architectures = getattr(model_config, "architectures", []) + model_cls, _ = ModelRegistry.resolve_model_cls(architectures) + model = model_cls(model_config) + + weights_to_load = {name for name, _ in model.named_parameters()} + loaded_weights = model.load_weights( + self._get_all_weights(model, model_path, to_cpu=should_offload) + ) + self.counter_after_loading_weights = time.perf_counter() + logger.info( + "Loading weights took %.2f seconds", + self.counter_after_loading_weights + - self.counter_before_loading_weights, + ) + + # Explicitly move model to target device after loading weights + model = model.to(local_torch_device) + + if should_offload: + # Disable FSDP for MPS as it's not compatible + if current_platform.is_mps(): + logger.info( + "Disabling FSDP sharding for MPS platform as it's not compatible" + ) + else: + mesh = init_device_mesh( + "cuda", + mesh_shape=(1, dist.get_world_size()), + mesh_dim_names=("offload", "replicate"), + ) + shard_model( + model, + cpu_offload=True, + reshard_after_forward=True, + mesh=mesh["offload"], + fsdp_shard_conditions=model_config.arch_config._fsdp_shard_conditions + or getattr(model, "_fsdp_shard_conditions", None), + pin_cpu_memory=server_args.pin_cpu_memory, + ) + # We only enable strict check for non-quantized models + # that have loaded weights tracking currently. + # if loaded_weights is not None: + weights_not_loaded = weights_to_load - loaded_weights + if weights_not_loaded: + raise ValueError( + "Following model weights were not initialized from " + f"checkpoint: {weights_not_loaded}" + ) + + return model.eval() + + +class ImageEncoderLoader(TextEncoderLoader): + def should_offload(self, server_args, model_config: ModelConfig | None = None): + should_offload = server_args.image_encoder_cpu_offload + fsdp_shard_conditions = getattr(model_config, "_fsdp_shard_conditions", []) + use_cpu_offload = should_offload and len(fsdp_shard_conditions) > 0 + return use_cpu_offload + + def load_customized( + self, component_model_path: str, server_args: ServerArgs, *args + ): + """Load the text encoders based on the model path, and inference args.""" + # model_config: PretrainedConfig = get_hf_config( + # model=model_path, + # trust_remote_code=server_args.trust_remote_code, + # revision=server_args.revision, + # model_override_args=None, + # ) + with open(os.path.join(component_model_path, "config.json")) as f: + model_config = json.load(f) + _clean_hf_config_inplace(model_config) + logger.info("HF model config: %s", model_config) + + encoder_config = server_args.pipeline_config.image_encoder_config + encoder_config.update_model_arch(model_config) + + # Always start with local device; load_model will adjust for offload if needed + should_offload = self.should_offload(server_args) + # TODO(will): add support for other dtypes + return self.load_model( + component_model_path, + encoder_config, + server_args, + server_args.pipeline_config.image_encoder_precision, + cpu_offload_flag=server_args.image_encoder_cpu_offload, + ) + + +class ImageProcessorLoader(ComponentLoader): + """Loader for image processor.""" + + def load_customized( + self, component_model_path: str, server_args: ServerArgs, module_name: str + ) -> Any: + return AutoImageProcessor.from_pretrained(component_model_path, use_fast=True) + + +class AutoProcessorLoader(ComponentLoader): + """Loader for auto processor.""" + + def load_customized( + self, component_model_path: str, server_args: ServerArgs, module_name: str + ) -> Any: + return AutoProcessor.from_pretrained(component_model_path) + + +class TokenizerLoader(ComponentLoader): + """Loader for tokenizers.""" + + def load_customized( + self, component_model_path: str, server_args: ServerArgs, module_name: str + ) -> Any: + return AutoTokenizer.from_pretrained( + component_model_path, + padding_size="right", + ) + + +class VAELoader(ComponentLoader): + """Loader for VAE.""" + + def should_offload(self, server_args, cpu_offload_flag, model_config): + return True + + def load_customized( + self, component_model_path: str, server_args: ServerArgs, *args + ): + """Load the VAE based on the model path, and inference args.""" + config = get_diffusers_component_config(model_path=component_model_path) + class_name = config.pop("_class_name") + assert ( + class_name is not None + ), "Model config does not contain a _class_name attribute. Only diffusers format is supported." + + server_args.model_paths["vae"] = component_model_path + + logger.info("HF model config: %s", config) + vae_config = server_args.pipeline_config.vae_config + vae_config.update_model_arch(config) + + # NOTE: some post init logics are only available after updated with config + vae_config.post_init() + + target_device = self.target_device(server_args.vae_cpu_offload) + + with set_default_torch_dtype( + PRECISION_TO_TYPE[server_args.pipeline_config.vae_precision] + ), skip_init_modules(): + vae_cls, _ = ModelRegistry.resolve_model_cls(class_name) + vae = vae_cls(vae_config).to(target_device) + + # Find all safetensors files + safetensors_list = _list_safetensors_files(component_model_path) + # TODO(PY) + assert ( + len(safetensors_list) == 1 + ), f"Found {len(safetensors_list)} safetensors files in {component_model_path}" + loaded = safetensors_load_file(safetensors_list[0]) + vae.load_state_dict( + loaded, strict=False + ) # We might only load encoder or decoder + + return vae.eval() + + +class TransformerLoader(ComponentLoader): + """Loader for transformer.""" + + def load_customized( + self, component_model_path: str, server_args: ServerArgs, *args + ): + """Load the transformer based on the model path, and inference args.""" + config = get_diffusers_component_config(model_path=component_model_path) + hf_config = deepcopy(config) + cls_name = config.pop("_class_name") + if cls_name is None: + raise ValueError( + "Model config does not contain a _class_name attribute. " + "Only diffusers format is supported." + ) + + logger.info("transformer cls_name: %s", cls_name) + if server_args.override_transformer_cls_name is not None: + cls_name = server_args.override_transformer_cls_name + logger.info("Overriding transformer cls_name to %s", cls_name) + + server_args.model_paths["transformer"] = component_model_path + + # Config from Diffusers supersedes sgl_diffusion's model config + dit_config = server_args.pipeline_config.dit_config + dit_config.update_model_arch(config) + + model_cls, _ = ModelRegistry.resolve_model_cls(cls_name) + + # Find all safetensors files + safetensors_list = _list_safetensors_files(component_model_path) + if not safetensors_list: + raise ValueError(f"No safetensors files found in {component_model_path}") + + # Check if we should use custom initialization weights + custom_weights_path = getattr( + server_args, "init_weights_from_safetensors", None + ) + use_custom_weights = False + + if use_custom_weights: + logger.info( + "Using custom initialization weights from: %s", custom_weights_path + ) + assert ( + custom_weights_path is not None + ), "Custom initialization weights must be provided" + if os.path.isdir(custom_weights_path): + safetensors_list = _list_safetensors_files(custom_weights_path) + else: + assert custom_weights_path.endswith( + ".safetensors" + ), "Custom initialization weights must be a safetensors file" + safetensors_list = [custom_weights_path] + + logger.info( + "Loading model from %s safetensors files: %s", + len(safetensors_list), + safetensors_list, + ) + + default_dtype = PRECISION_TO_TYPE[server_args.pipeline_config.dit_precision] + + # Load the model using FSDP loader + logger.info("Loading %s, default_dtype: %s", cls_name, default_dtype) + assert server_args.hsdp_shard_dim is not None + model = maybe_load_fsdp_model( + model_cls=model_cls, + init_params={"config": dit_config, "hf_config": hf_config}, + weight_dir_list=safetensors_list, + device=get_local_torch_device(), + hsdp_replicate_dim=server_args.hsdp_replicate_dim, + hsdp_shard_dim=server_args.hsdp_shard_dim, + cpu_offload=server_args.dit_cpu_offload, + pin_cpu_memory=server_args.pin_cpu_memory, + fsdp_inference=server_args.use_fsdp_inference, + # TODO(will): make these configurable + default_dtype=default_dtype, + param_dtype=torch.bfloat16, + reduce_dtype=torch.float32, + output_dtype=None, + ) + + total_params = sum(p.numel() for p in model.parameters()) + logger.info("Loaded model with %.2fB parameters", total_params / 1e9) + + assert ( + next(model.parameters()).dtype == default_dtype + ), "Model dtype does not match default dtype" + + model = model.eval() + return model + + +class SchedulerLoader(ComponentLoader): + """Loader for scheduler.""" + + def load_customized( + self, component_model_path: str, server_args: ServerArgs, *args + ): + """Load the scheduler based on the model path, and inference args.""" + config = get_diffusers_component_config(model_path=component_model_path) + + class_name = config.pop("_class_name") + assert ( + class_name is not None + ), "Model config does not contain a _class_name attribute. Only diffusers format is supported." + + scheduler_cls, _ = ModelRegistry.resolve_model_cls(class_name) + + scheduler = scheduler_cls(**config) + if server_args.pipeline_config.flow_shift is not None: + scheduler.set_shift(server_args.pipeline_config.flow_shift) + if server_args.pipeline_config.timesteps_scale is not None: + scheduler.set_timesteps_scale(server_args.pipeline_config.timesteps_scale) + return scheduler + + +class GenericComponentLoader(ComponentLoader): + """Generic loader for components that don't have a specific loader.""" + + def __init__(self, library="transformers") -> None: + super().__init__() + self.library = library + + +class PipelineComponentLoader: + """ + Utility class for loading pipeline components. + This replaces the chain of if-else statements in load_pipeline_module. + """ + + @staticmethod + def load_module( + module_name: str, + component_model_path: str, + transformers_or_diffusers: str, + server_args: ServerArgs, + ): + """ + Load a pipeline module. + + Args: + module_name: Name of the module (e.g., "vae", "text_encoder", "transformer", "scheduler") + component_model_path: Path to the component model + transformers_or_diffusers: Whether the module is from transformers or diffusers + + Returns: + The loaded module + """ + logger.info( + "Loading %s using %s from %s", + module_name, + transformers_or_diffusers, + component_model_path, + ) + + # Get the appropriate loader for this module type + loader = ComponentLoader.for_module_type(module_name, transformers_or_diffusers) + + try: + # Load the module + return loader.load( + component_model_path, + server_args, + module_name, + transformers_or_diffusers, + ) + except Exception as e: + logger.error( + f"Error while loading component: {module_name}, {component_model_path=}" + ) + raise e diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/loader/fsdp_load.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/loader/fsdp_load.py new file mode 100644 index 00000000..38c73c90 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/loader/fsdp_load.py @@ -0,0 +1,314 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +# Adapted from torchtune +# Copyright 2024 The TorchTune Authors. +# Copyright 2025 The sglang-diffusion Authors. + +import contextlib +from collections.abc import Callable, Generator +from itertools import chain +from typing import Any + +import torch +from torch import nn +from torch.distributed import DeviceMesh, init_device_mesh +from torch.distributed._tensor import distribute_tensor +from torch.distributed.fsdp import ( + CPUOffloadPolicy, + FSDPModule, + MixedPrecisionPolicy, + fully_shard, +) +from torch.nn.modules.module import _IncompatibleKeys + +from sglang.multimodal_gen.runtime.loader.utils import ( + get_param_names_mapping, + hf_to_custom_state_dict, +) +from sglang.multimodal_gen.runtime.loader.weight_utils import ( + safetensors_weights_iterator, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.utils import set_mixed_precision_policy + +logger = init_logger(__name__) + + +# TODO(PY): move this to utils elsewhere +@contextlib.contextmanager +def set_default_dtype(dtype: torch.dtype) -> Generator[None, None, None]: + """ + Context manager to set torch's default dtype. + + Args: + dtype (torch.dtype): The desired default dtype inside the context manager. + + Returns: + ContextManager: context manager for setting default dtype. + + Example: + >>> with set_default_dtype(torch.bfloat16): + >>> x = torch.tensor([1, 2, 3]) + >>> x.dtype + torch.bfloat16 + + + """ + old_dtype = torch.get_default_dtype() + torch.set_default_dtype(dtype) + try: + yield + finally: + torch.set_default_dtype(old_dtype) + + +# TODO(PY): add compile option +def maybe_load_fsdp_model( + model_cls: type[nn.Module], + init_params: dict[str, Any], + weight_dir_list: list[str], + device: torch.device, + hsdp_replicate_dim: int, + hsdp_shard_dim: int, + default_dtype: torch.dtype, + param_dtype: torch.dtype, + reduce_dtype: torch.dtype, + cpu_offload: bool = False, + fsdp_inference: bool = False, + output_dtype: torch.dtype | None = None, + pin_cpu_memory: bool = True, +) -> torch.nn.Module: + """ + Load the model with FSDP if is training, else load the model without FSDP. + """ + # NOTE(will): cast_forward_inputs=True shouldn't be needed as we are + # manually casting the inputs to the model + mp_policy = MixedPrecisionPolicy( + param_dtype, reduce_dtype, output_dtype, cast_forward_inputs=False + ) + + set_mixed_precision_policy( + param_dtype=param_dtype, + reduce_dtype=reduce_dtype, + output_dtype=output_dtype, + mp_policy=mp_policy, + ) + + with set_default_dtype(default_dtype), torch.device("meta"): + model = model_cls(**init_params) + + # Check if we should use FSDP + use_fsdp = fsdp_inference + + # Disable FSDP for MPS as it's not compatible + from sglang.multimodal_gen.runtime.platforms import current_platform + + if current_platform.is_mps(): + use_fsdp = False + logger.info("Disabling FSDP for MPS platform as it's not compatible") + + if use_fsdp: + world_size = hsdp_replicate_dim * hsdp_shard_dim + if not fsdp_inference: + hsdp_replicate_dim = world_size + hsdp_shard_dim = 1 + + device_mesh = init_device_mesh( + "cuda", + # (Replicate(), Shard(dim=0)) + mesh_shape=(hsdp_replicate_dim, hsdp_shard_dim), + mesh_dim_names=("replicate", "shard"), + ) + shard_model( + model, + cpu_offload=cpu_offload, + reshard_after_forward=True, + mp_policy=mp_policy, + mesh=device_mesh, + fsdp_shard_conditions=model._fsdp_shard_conditions, + pin_cpu_memory=pin_cpu_memory, + ) + + weight_iterator = safetensors_weights_iterator(weight_dir_list) + param_names_mapping_fn = get_param_names_mapping(model.param_names_mapping) + load_model_from_full_model_state_dict( + model, + weight_iterator, + device, + default_dtype, + strict=True, + cpu_offload=cpu_offload, + param_names_mapping=param_names_mapping_fn, + ) + for n, p in chain(model.named_parameters(), model.named_buffers()): + if p.is_meta: + raise RuntimeError(f"Unexpected param or buffer {n} on meta device.") + # Avoid unintended computation graph accumulation during inference + if isinstance(p, torch.nn.Parameter): + p.requires_grad = False + return model + + +def shard_model( + model, + *, + cpu_offload: bool, + reshard_after_forward: bool = True, + mp_policy: MixedPrecisionPolicy | None = MixedPrecisionPolicy(), # noqa + mesh: DeviceMesh | None = None, + fsdp_shard_conditions: list[Callable[[str, nn.Module], bool]] = [], # noqa + pin_cpu_memory: bool = True, +) -> None: + """ + Utility to shard a model with FSDP using the PyTorch Distributed fully_shard API. + + This method will over the model's named modules from the bottom-up and apply shard modules + based on whether they meet any of the criteria from shard_conditions. + + Args: + model (TransformerDecoder): Model to shard with FSDP. + cpu_offload (bool): If set to True, FSDP will offload parameters, gradients, and optimizer + states to CPU. + reshard_after_forward (bool): Whether to reshard parameters and buffers after + the forward pass. Setting this to True corresponds to the FULL_SHARD sharding strategy + from FSDP1, while setting it to False corresponds to the SHARD_GRAD_OP sharding strategy. + mesh (Optional[DeviceMesh]): Device mesh to use for FSDP sharding under multiple parallelism. + Default to None. + fsdp_shard_conditions (List[Callable[[str, nn.Module], bool]]): A list of functions to determine + which modules to shard with FSDP. + pin_cpu_memory (bool): If set to True, FSDP will pin the CPU memory of the offloaded parameters. + + Raises: + ValueError: If no layer modules were sharded, indicating that no shard_condition was triggered. + """ + if fsdp_shard_conditions is None or len(fsdp_shard_conditions) == 0: + logger.warning( + "The FSDP shard condition list is empty or None. No modules will be sharded in %s", + type(model).__name__, + ) + return + + fsdp_kwargs = { + "reshard_after_forward": reshard_after_forward, + "mesh": mesh, + "mp_policy": mp_policy, + } + if cpu_offload: + fsdp_kwargs["offload_policy"] = CPUOffloadPolicy(pin_memory=pin_cpu_memory) + + # iterating in reverse to start with + # lowest-level modules first + num_layers_sharded = 0 + # TODO(will): don't reshard after forward for the last layer to save on the + # all-gather that will immediately happen Shard the model with FSDP, + for n, m in reversed(list(model.named_modules())): + if any([shard_condition(n, m) for shard_condition in fsdp_shard_conditions]): + fully_shard(m, **fsdp_kwargs) + num_layers_sharded += 1 + + if num_layers_sharded == 0: + raise ValueError( + "No layer modules were sharded. Please check if shard conditions are working as expected." + ) + + # Finally shard the entire model to account for any stragglers + fully_shard(model, **fsdp_kwargs) + + +# TODO(PY): device mesh for cfg parallel +def load_model_from_full_model_state_dict( + model: FSDPModule | torch.nn.Module, + full_sd_iterator: Generator[tuple[str, torch.Tensor], None, None], + device: torch.device, + param_dtype: torch.dtype, + strict: bool = False, + cpu_offload: bool = False, + param_names_mapping: Callable[[str], tuple[str, Any, Any]] | None = None, +) -> _IncompatibleKeys: + """ + Converting full state dict into a sharded state dict + and loading it into FSDP model (if training) or normal huggingface model + Args: + model (Union[FSDPModule, torch.nn.Module]): Model to generate fully qualified names for cpu_state_dict + full_sd_iterator (Generator): an iterator yielding (param_name, tensor) pairs + device (torch.device): device used to move full state dict tensors + param_dtype (torch.dtype): dtype used to move full state dict tensors + strict (bool): flag to check if to load the model in strict mode + cpu_offload (bool): flag to check if FSDP offload is enabled + param_names_mapping (Optional[Callable[[str], str]]): a function that maps full param name to sharded param name + Returns: + ``NamedTuple`` with ``missing_keys`` and ``unexpected_keys`` fields: + * **missing_keys** is a list of str containing the missing keys + * **unexpected_keys** is a list of str containing the unexpected keys + + Raises: + NotImplementedError: If got FSDP with more than 1D. + """ + meta_sd = model.state_dict() + sharded_sd = {} + custom_param_sd, reverse_param_names_mapping = hf_to_custom_state_dict( + full_sd_iterator, param_names_mapping + ) # type: ignore + for target_param_name, full_tensor in custom_param_sd.items(): + meta_sharded_param = meta_sd.get(target_param_name) + if meta_sharded_param is None: + raise ValueError( + f"Parameter {target_param_name} not found in custom model state dict. The hf to custom mapping may be incorrect." + ) + if not hasattr(meta_sharded_param, "device_mesh"): + full_tensor = full_tensor.to(device=device, dtype=param_dtype) + # In cases where parts of the model aren't sharded, some parameters will be plain tensors + sharded_tensor = full_tensor + else: + full_tensor = full_tensor.to(device=device, dtype=param_dtype) + sharded_tensor = distribute_tensor( + full_tensor, + meta_sharded_param.device_mesh, + meta_sharded_param.placements, + ) + if cpu_offload: + sharded_tensor = sharded_tensor.cpu() + sharded_sd[target_param_name] = nn.Parameter(sharded_tensor) + + model.reverse_param_names_mapping = reverse_param_names_mapping + unused_keys = set(meta_sd.keys()) - set(sharded_sd.keys()) + if unused_keys: + logger.warning("Found unloaded parameters in meta state dict: %s", unused_keys) + + # List of allowed parameter name patterns + ALLOWED_NEW_PARAM_PATTERNS = ["gate_compress"] # Can be extended as needed + for new_param_name in unused_keys: + if not any(pattern in new_param_name for pattern in ALLOWED_NEW_PARAM_PATTERNS): + logger.error( + "Unsupported new parameter: %s. Allowed patterns: %s", + new_param_name, + ALLOWED_NEW_PARAM_PATTERNS, + ) + raise ValueError( + f"New parameter '{new_param_name}' is not supported. " + f"Currently only parameters containing {ALLOWED_NEW_PARAM_PATTERNS} are allowed." + ) + meta_sharded_param = meta_sd.get(new_param_name) + if not hasattr(meta_sharded_param, "device_mesh"): + # Initialize with zeros + sharded_tensor = torch.zeros_like( + meta_sharded_param, device=device, dtype=param_dtype + ) + else: + # Initialize with zeros and distribute + full_tensor = torch.zeros_like( + meta_sharded_param, device=device, dtype=param_dtype + ) + sharded_tensor = distribute_tensor( + full_tensor, + meta_sharded_param.device_mesh, + meta_sharded_param.placements, + ) + if cpu_offload: + sharded_tensor = sharded_tensor.cpu() + sharded_sd[new_param_name] = nn.Parameter(sharded_tensor) + + # choose `assign=True` since we cannot call `copy_` on meta tensor + return model.load_state_dict(sharded_sd, strict=strict, assign=True) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/loader/utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/loader/utils.py new file mode 100644 index 00000000..9f375b9a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/loader/utils.py @@ -0,0 +1,102 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +"""Utilities for selecting and loading models.""" +import contextlib +import re +from collections import defaultdict +from collections.abc import Callable, Iterator +from typing import Any + +import torch + +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +@contextlib.contextmanager +def set_default_torch_dtype(dtype: torch.dtype): + """Sets the default torch dtype to the given dtype.""" + old_dtype = torch.get_default_dtype() + torch.set_default_dtype(dtype) + yield + torch.set_default_dtype(old_dtype) + + +def get_param_names_mapping( + mapping_dict: dict[str, str] +) -> Callable[[str], tuple[str, Any, Any]]: + """ + Creates a mapping function that transforms parameter names using regex patterns. + + Args: + mapping_dict (Dict[str, str]): Dictionary mapping regex patterns to replacement patterns + + Returns: + Callable[[str], str]: A function that maps parameter names from source to target format + """ + + def mapping_fn(name: str) -> tuple[str, Any, Any]: + # Try to match and transform the name using the regex patterns in mapping_dict + for pattern, replacement in mapping_dict.items(): + match = re.match(pattern, name) + if match: + merge_index = None + total_split_params = None + if isinstance(replacement, tuple): + merge_index = replacement[1] + total_split_params = replacement[2] + replacement = replacement[0] + name = re.sub(pattern, replacement, name) + return name, merge_index, total_split_params + + # If no pattern matches, return the original name + return name, None, None + + return mapping_fn + + +def hf_to_custom_state_dict( + hf_param_sd: dict[str, torch.Tensor] | Iterator[tuple[str, torch.Tensor]], + param_names_mapping: Callable[[str], tuple[str, Any, Any]], +) -> tuple[dict[str, torch.Tensor], dict[str, tuple[str, Any, Any]]]: + """ + Converts a Hugging Face parameter state dictionary to a custom parameter state dictionary. + + Args: + hf_param_sd (Dict[str, torch.Tensor]): The Hugging Face parameter state dictionary + param_names_mapping (Callable[[str], tuple[str, Any, Any]]): A function that maps parameter names from source to target format + + Returns: + custom_param_sd (Dict[str, torch.Tensor]): The custom formatted parameter state dict + reverse_param_names_mapping (Dict[str, Tuple[str, Any, Any]]): Maps back from custom to hf + """ + custom_param_sd = {} + to_merge_params = defaultdict(dict) # type: ignore + reverse_param_names_mapping = {} + if isinstance(hf_param_sd, dict): + hf_param_sd = hf_param_sd.items() # type: ignore + for source_param_name, full_tensor in hf_param_sd: # type: ignore + target_param_name, merge_index, num_params_to_merge = param_names_mapping( + source_param_name + ) + reverse_param_names_mapping[target_param_name] = ( + source_param_name, + merge_index, + num_params_to_merge, + ) + if merge_index is not None: + to_merge_params[target_param_name][merge_index] = full_tensor + if len(to_merge_params[target_param_name]) == num_params_to_merge: + # cat at output dim according to the merge_index order + sorted_tensors = [ + to_merge_params[target_param_name][i] + for i in range(num_params_to_merge) + ] + full_tensor = torch.cat(sorted_tensors, dim=0) + del to_merge_params[target_param_name] + else: + continue + custom_param_sd[target_param_name] = full_tensor + return custom_param_sd, reverse_param_names_mapping diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/loader/weight_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/loader/weight_utils.py new file mode 100644 index 00000000..2bda6ee6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/loader/weight_utils.py @@ -0,0 +1,300 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/model_executor/model_loader/weight_utils.py +"""Utilities for downloading and initializing model weights.""" +import hashlib +import json +import os +import tempfile +from collections.abc import Generator +from pathlib import Path + +import filelock +import huggingface_hub.constants +import torch +from safetensors.torch import safe_open +from tqdm.auto import tqdm + +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + +# use system-level temp directory for file locks, so that multiple users +# can share the same lock without error. +# lock files in the temp directory will be automatically deleted when the +# system reboots, so users will not complain about annoying lock files +temp_dir = tempfile.gettempdir() + + +def enable_hf_transfer() -> None: + """automatically activates hf_transfer""" + if "HF_HUB_ENABLE_HF_TRANSFER" not in os.environ: + try: + # enable hf hub transfer if available + import hf_transfer # type: ignore # noqa + + huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER = True + except ImportError: + pass + + +enable_hf_transfer() + + +class DisabledTqdm(tqdm): + + def __init__(self, *args, **kwargs): + kwargs["disable"] = True + super().__init__(*args, **kwargs) + + +def get_lock(model_name_or_path: str | Path, cache_dir: str | None = None): + lock_dir = cache_dir or temp_dir + model_name_or_path = str(model_name_or_path) + os.makedirs(os.path.dirname(lock_dir), exist_ok=True) + model_name = model_name_or_path.replace("/", "-") + hash_name = hashlib.sha256(model_name.encode()).hexdigest() + # add hash to avoid conflict with old users' lock files + lock_file_name = hash_name + model_name + ".lock" + # mode 0o666 is required for the filelock to be shared across users + lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name), mode=0o666) + return lock + + +# For models like Mistral-7B-v0.3, there are both sharded +# safetensors files and a consolidated safetensors file. +# Passing both of these to the weight loader functionality breaks. +# So, we use the index_file to +# look up which safetensors files should be used. +def filter_duplicate_safetensors_files( + hf_weights_files: list[str], hf_folder: str, index_file: str +) -> list[str]: + # model.safetensors.index.json is a mapping from keys in the + # torch state_dict to safetensors file holding that weight. + index_file_name = os.path.join(hf_folder, index_file) + if not os.path.isfile(index_file_name): + return hf_weights_files + + # Iterate through the weight_map (weight_name: safetensors files) + # to identify weights that we should use. + with open(index_file_name) as f: + weight_map = json.load(f)["weight_map"] + weight_files_in_index = set() + for weight_name in weight_map: + weight_files_in_index.add(os.path.join(hf_folder, weight_map[weight_name])) + # Filter out any fields that are not found in the index file. + hf_weights_files = [f for f in hf_weights_files if f in weight_files_in_index] + return hf_weights_files + + +def filter_files_not_needed_for_inference(hf_weights_files: list[str]) -> list[str]: + """ + Exclude files that are not needed for inference. + + See https://github.com/huggingface/transformers/blob/v4.34.0/src/transformers/trainer.py#L227-L233 + """ + blacklist = [ + "training_args.bin", + "optimizer.bin", + "optimizer.pt", + "scheduler.pt", + "scaler.pt", + ] + hf_weights_files = [ + f for f in hf_weights_files if not any(f.endswith(x) for x in blacklist) + ] + return hf_weights_files + + +# explicitly use pure text format, with a newline at the end +# this makes it impossible to see the animation in the progress bar +# but will avoid messing up with ray or multiprocessing, which wraps +# each line of output with some prefix. +_BAR_FORMAT = "{desc}: {percentage:3.0f}% Completed | {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]\n" # noqa: E501 + + +def _validate_safetensors_file(file_path: str) -> bool: + """ + Validate that a safetensors file is readable and not corrupted. + + Args: + file_path: Path to the safetensors file + + Returns: + True if file is valid, False if corrupted + """ + try: + with safe_open(file_path, framework="pt", device="cpu") as f: + _ = list(f.keys()) + return True + except Exception as e: + logger.error( + "Corrupted safetensors file detected: %s - %s: %s", + file_path, + type(e).__name__, + str(e), + ) + return False + + +def safetensors_weights_iterator( + hf_weights_files: list[str], + to_cpu: bool = True, +) -> Generator[tuple[str, torch.Tensor], None, None]: + """Iterate over the weights in the model safetensor files.""" + enable_tqdm = ( + not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0 + ) + device = "cpu" if to_cpu else str(get_local_torch_device()) + + # Validate files before loading + corrupted_files = [ + st_file + for st_file in hf_weights_files + if not _validate_safetensors_file(st_file) + ] + + if corrupted_files: + # Delete corrupted files (both symlink and blob if applicable) + for file_path in corrupted_files: + try: + if os.path.islink(file_path): + blob_path = os.path.realpath(file_path) + os.remove(file_path) + logger.info( + "Removed corrupted symlink: %s", os.path.basename(file_path) + ) + if os.path.exists(blob_path): + os.remove(blob_path) + logger.info( + "Removed corrupted blob: %s", os.path.basename(blob_path) + ) + elif os.path.isfile(file_path): + os.remove(file_path) + logger.info( + "Removed corrupted file: %s", os.path.basename(file_path) + ) + except Exception as e: + logger.warning("Failed to remove corrupted file %s: %s", file_path, e) + + raise RuntimeError( + f"Found {len(corrupted_files)} corrupted safetensors file(s). " + f"Files have been removed: {[os.path.basename(f) for f in corrupted_files]}. " + "Please retry - the files will be re-downloaded automatically." + ) + + for st_file in tqdm( + hf_weights_files, + desc="Loading safetensors checkpoint shards", + disable=not enable_tqdm, + bar_format=_BAR_FORMAT, + ): + with safe_open(st_file, framework="pt", device=device) as f: + for name in f.keys(): # noqa: SIM118 + param = f.get_tensor(name) + yield name, param + + +def pt_weights_iterator( + hf_weights_files: list[str], + to_cpu: bool = True, +) -> Generator[tuple[str, torch.Tensor], None, None]: + """Iterate over the weights in the model bin/pt files.""" + device = "cpu" if to_cpu else str(get_local_torch_device()) + enable_tqdm = ( + not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0 + ) + for bin_file in tqdm( + hf_weights_files, + desc="Loading pt checkpoint shards", + disable=not enable_tqdm, + bar_format=_BAR_FORMAT, + ): + state = torch.load(bin_file, map_location=device, weights_only=True) + yield from state.items() + del state + + +def default_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None: + """Default weight loader.""" + try: + if param.numel() == 1 and loaded_weight.numel() == 1: + # Sometimes scalar values aren't considered tensors with shapes + # so if both param and loaded_weight are a scalar, + # "broadcast" instead of copy + param.data.fill_(loaded_weight.item()) + else: + assert param.size() == loaded_weight.size(), ( + f"Attempted to load weight ({loaded_weight.size()}) " + f"into parameter ({param.size()})" + ) + + param.data.copy_(loaded_weight) + except Exception: + # NOTE: This exception is added for the purpose of setting breakpoint to + # debug weight loading issues. + raise + + +def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> str | None: + """Remap the name of FP8 k/v_scale parameters. + + This function handles the remapping of FP8 k/v_scale parameter names. + It detects if the given name ends with a suffix and attempts to remap + it to the expected name format in the model. If the remapped name is not + found in the params_dict, a warning is printed and None is returned. + + Args: + name (str): The original loaded checkpoint parameter name. + params_dict (dict): Dictionary containing the model's named parameters. + + Returns: + str: The remapped parameter name if successful, or the original name + if no remapping is needed. + None: If the remapped name is not found in params_dict. + """ + if name.endswith(".kv_scale"): + logger.warning_once( + "DEPRECATED. Found kv_scale in the checkpoint. " + "This format is deprecated in favor of separate k_scale and " + "v_scale tensors and will be removed in a future release. " + "Functionally, we will remap kv_scale to k_scale and duplicate " + "k_scale to v_scale" + ) + # NOTE: we remap the deprecated kv_scale to k_scale + remapped_name = name.replace(".kv_scale", ".attn.k_scale") + if remapped_name not in params_dict: + logger.warning_once( + f"Found kv_scale in the checkpoint (e.g. {name}), " + "but not found the expected name in the model " + f"(e.g. {remapped_name}). kv_scale is " + "not loaded." + ) + return None + return remapped_name + + possible_scale_names = [".k_scale", ".v_scale"] + modelopt_scale_names = [".self_attn.k_proj.k_scale", ".self_attn.v_proj.v_scale"] + for scale_name in possible_scale_names: + if name.endswith(scale_name): + if any(mo_scale_name in name for mo_scale_name in modelopt_scale_names): + remapped_name = name.replace( + f".self_attn.{scale_name[1]}_proj{scale_name}", + f".self_attn.attn{scale_name}", + ) + else: + remapped_name = name.replace(scale_name, f".attn{scale_name}") + if remapped_name not in params_dict: + logger.warning_once( + f"Found {scale_name} in the checkpoint (e.g. {name}), " + "but not found the expected name in the model " + f"(e.g. {remapped_name}). {scale_name} is " + "not loaded." + ) + return None + return remapped_name + + # If there were no matches, return the untouched param name + return name diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/managers/forward_context.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/managers/forward_context.py new file mode 100644 index 00000000..e506929c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/managers/forward_context.py @@ -0,0 +1,120 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/forward_context.py +import time +from collections import defaultdict +from contextlib import contextmanager +from dataclasses import dataclass +from typing import TYPE_CHECKING, Optional, Type + +import torch + +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +if TYPE_CHECKING: + from sglang.multimodal_gen.runtime.layers.attention import AttentionMetadata + from sglang.multimodal_gen.runtime.pipelines_core import Req + +logger = init_logger(__name__) + +# TODO(will): check if this is needed +# track_batchsize: bool = envs.SGLANG_DIFFUSION_LOG_BATCHSIZE_INTERVAL >= 0 +track_batchsize: bool = False +last_logging_time: float = 0 +forward_start_time: float = 0 +# batchsize_logging_interval: float = envs.SGLANG_DIFFUSION_LOG_BATCHSIZE_INTERVAL +batchsize_logging_interval: float = 1000 +batchsize_forward_time: defaultdict = defaultdict(list) + + +@dataclass +class ForwardContext: + current_timestep: int + # TODO(will): check this arg + # copy from vllm_config.compilation_config.static_forward_context + # attn_layers: Dict[str, Any] + # TODO: extend to support per-layer dynamic forward context + attn_metadata: "AttentionMetadata" # set dynamically for each forward pass + forward_batch: Optional["Req"] = None + attention_backend_cls: Optional[Type] = None + + def set_attn_backend_cls(self, attention_backend_cls: Type): + if self.attention_backend_cls: + if self.attention_backend_cls != attention_backend_cls: + raise RuntimeError( + f"Different types of attention backend in a same context detected, previous: {self.attention_backend_cls}, new: {attention_backend_cls}" + ) + else: + self.attention_backend_cls = attention_backend_cls + + +_forward_context: Optional["ForwardContext"] = None + + +def get_forward_context() -> "ForwardContext": + """Get the current forward context.""" + assert _forward_context is not None, ( + "Forward context is not set. " + "Please use `set_forward_context` to set the forward context." + ) + return _forward_context + + +# TODO(will): finalize the interface +@contextmanager +def set_forward_context( + current_timestep, attn_metadata, forward_batch: Optional["Req"] = None +): + """A context manager that stores the current forward context, + can be attention metadata, etc. + Here we can inject common logic for every model forward pass. + """ + global forward_start_time + need_to_track_batchsize = track_batchsize and attn_metadata is not None + if need_to_track_batchsize: + forward_start_time = time.perf_counter() + global _forward_context + prev_context = _forward_context + _forward_context = ForwardContext( + current_timestep=current_timestep, + attn_metadata=attn_metadata, + forward_batch=forward_batch, + ) + + try: + yield + finally: + global last_logging_time, batchsize_logging_interval + if need_to_track_batchsize: + if hasattr(attn_metadata, "num_prefill_tokens"): + # for v0 attention backends + batchsize = ( + attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens + ) + else: + # for v1 attention backends + batchsize = attn_metadata.num_input_tokens + now = time.perf_counter() + # time measurement is in milliseconds + batchsize_forward_time[batchsize].append((now - forward_start_time) * 1000) + if now - last_logging_time > batchsize_logging_interval: + last_logging_time = now + forward_stats = [] + for bs, times in batchsize_forward_time.items(): + if len(times) <= 1: + # can be cudagraph / profiling run + continue + medium = torch.quantile(torch.tensor(times), q=0.5).item() + medium = round(medium, 2) + forward_stats.append((bs, len(times), medium)) + forward_stats.sort(key=lambda x: x[1], reverse=True) + if forward_stats: + logger.info( + ( + "Batchsize forward time stats " + "(batchsize, count, median_time(ms)): %s" + ), + forward_stats, + ) + _forward_context = prev_context diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py new file mode 100644 index 00000000..7fa2014f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py @@ -0,0 +1,191 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +import multiprocessing as mp +import os +import time +from typing import List + +import torch +from setproctitle import setproctitle + +from sglang.multimodal_gen.runtime.distributed import ( + get_sp_group, + maybe_init_distributed_environment_and_model_parallel, +) +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + get_cfg_group, + get_tp_group, +) +from sglang.multimodal_gen.runtime.pipelines_core import Req, build_pipeline +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch +from sglang.multimodal_gen.runtime.server_args import PortArgs, ServerArgs +from sglang.multimodal_gen.runtime.utils.common import set_cuda_arch +from sglang.multimodal_gen.runtime.utils.logging_utils import ( + configure_logger, + init_logger, + suppress_other_loggers, +) +from sglang.multimodal_gen.runtime.utils.perf_logger import ( + PerformanceLogger, + RequestTimings, +) + +logger = init_logger(__name__) + +CYAN = "\033[1;36m" +RESET = "\033[0;0m" + + +class GPUWorker: + """ + A worker that executes the model on a single GPU. + """ + + def __init__( + self, + local_rank: int, + rank: int, + master_port: int, + server_args: ServerArgs, + ): + self.local_rank = local_rank + self.rank = rank + self.master_port = master_port + # FIXME: should we use tcp as distribute init method? + self.server_args = server_args + self.pipeline = None + + self.init_device_and_model() + self.sp_group = get_sp_group() + self.sp_cpu_group = self.sp_group.cpu_group + self.tp_group = get_tp_group() + self.tp_cpu_group = self.tp_group.cpu_group + + self.cfg_group = get_cfg_group() + self.cfg_cpu_group = self.cfg_group.cpu_group + + def init_device_and_model(self) -> None: + """Initialize the device and load the model.""" + setproctitle(f"sgl_diffusion::scheduler_TP{self.local_rank}") + torch.cuda.set_device(self.local_rank) + # Set environment variables for distributed initialization + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = str(self.master_port) + os.environ["LOCAL_RANK"] = str(self.local_rank) + os.environ["RANK"] = str(self.rank) + os.environ["WORLD_SIZE"] = str(self.server_args.num_gpus) + # Initialize the distributed environment + maybe_init_distributed_environment_and_model_parallel( + tp_size=self.server_args.tp_size, + enable_cfg_parallel=self.server_args.enable_cfg_parallel, + ulysses_degree=self.server_args.ulysses_degree, + ring_degree=self.server_args.ring_degree, + sp_size=self.server_args.sp_degree, + dp_size=self.server_args.dp_size, + ) + + self.pipeline = build_pipeline(self.server_args) + + logger.info( + f"Worker {self.rank}: Initialized device, model, and distributed environment." + ) + + def execute_forward(self, batch: List[Req]) -> OutputBatch: + """ + Execute a forward pass. + """ + assert self.pipeline is not None + # TODO: dealing with first req for now + req = batch[0] + output_batch = None + try: + start_time = time.monotonic() + timings = RequestTimings(request_id=req.request_id) + req.timings = timings + + output_batch = self.pipeline.forward(req, self.server_args) + duration_ms = (time.monotonic() - start_time) * 1000 + + if output_batch.timings: + output_batch.timings.total_duration_ms = duration_ms + PerformanceLogger.log_request_summary(timings=output_batch.timings) + except Exception as e: + if output_batch is None: + from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import ( + OutputBatch, + ) + + output_batch = OutputBatch() + output_batch.error = f"Error executing request {req.request_id}: {e}" + finally: + return output_batch + + def set_lora(self, lora_nickname: str, lora_path: str | None = None) -> None: + """ + Set the LoRA adapter for the pipeline. + """ + assert self.pipeline is not None + self.pipeline.set_lora(lora_nickname, lora_path) + + def merge_lora_weights(self) -> None: + """ + Merge LoRA weights. + """ + assert self.pipeline is not None + self.pipeline.merge_lora_weights() + + def unmerge_lora_weights(self) -> None: + """ + Unmerge LoRA weights. + """ + assert self.pipeline is not None + self.pipeline.unmerge_lora_weights() + + +def run_scheduler_process( + local_rank: int, + rank: int, + master_port: int, + server_args: ServerArgs, + pipe_writer: mp.connection.Connection, + # For all workers: pipe to receive tasks from rank 0 + task_pipe_r: mp.connection.Connection, + # For slave workers: pipe to send results back to rank 0 + result_pipe_w: mp.connection.Connection | None, + # For rank 0 worker only: pipes to send tasks to slaves + task_pipes_to_slaves: list[mp.connection.Connection] | None = None, + # For rank 0 worker only: pipes to receive results from slaves + result_pipes_from_slaves: list[mp.connection.Connection] | None = None, +) -> None: + """ + The entry point for the worker process. + Rank 0 acts as the master, handling ZMQ requests and coordinating slaves. + Ranks > 0 act as slaves, waiting for tasks from the master. + """ + configure_logger(server_args) + suppress_other_loggers() + set_cuda_arch() + + port_args = PortArgs.from_server_args(server_args) + + # start the scheduler event loop + assert task_pipes_to_slaves is not None + assert result_pipes_from_slaves is not None + from sglang.multimodal_gen.runtime.managers.scheduler import Scheduler + + scheduler = Scheduler( + server_args, + gpu_id=rank, + port_args=port_args, + task_pipes_to_slaves=task_pipes_to_slaves, + result_pipes_from_slaves=result_pipes_from_slaves, + ) + logger.info(f"Worker {rank}: Scheduler loop started.") + pipe_writer.send( + { + "status": "ready", + } + ) + scheduler.event_loop() + logger.info(f"Worker {rank}: Shutdown complete.") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/managers/scheduler.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/managers/scheduler.py new file mode 100644 index 00000000..35da11dc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/managers/scheduler.py @@ -0,0 +1,230 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from typing import Any, List + +import zmq + +from sglang.multimodal_gen.runtime.entrypoints.openai.utils import ( + MergeLoraWeightsReq, + SetLoraReq, + UnmergeLoraWeightsReq, +) +from sglang.multimodal_gen.runtime.managers.gpu_worker import GPUWorker +from sglang.multimodal_gen.runtime.pipelines_core import Req +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch +from sglang.multimodal_gen.runtime.server_args import ( + PortArgs, + ServerArgs, + set_global_server_args, +) +from sglang.multimodal_gen.runtime.utils.common import get_zmq_socket +from sglang.multimodal_gen.runtime.utils.distributed import broadcast_pyobj +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class Scheduler: + """ + Runs the main event loop for the rank 0 worker. + It listens for external requests via ZMQ and coordinates with other workers. + This class does NOT manage worker processes. + """ + + def __init__( + self, + server_args: ServerArgs, + gpu_id: int, + port_args: PortArgs, + task_pipes_to_slaves: list = None, + result_pipes_from_slaves: list = None, + ): + self.server_args = server_args + self.port_args = port_args + + set_global_server_args(server_args=server_args) + + # Inter-process Communication + self.context = zmq.Context(io_threads=2) + endpoint = server_args.scheduler_endpoint() + if gpu_id == 0: + self.receiver, actual_endpoint = get_zmq_socket( + self.context, zmq.REP, endpoint, True + ) + logger.info(f"Scheduler bind at endpoint: {actual_endpoint}") + else: + self.receiver = None + + worker = GPUWorker( + local_rank=gpu_id, + master_port=port_args.master_port, + rank=gpu_id, + server_args=server_args, + ) + self.worker = worker + self.task_pipes_to_slaves = task_pipes_to_slaves + self.result_pipes_from_slaves = result_pipes_from_slaves + self.gpu_id = gpu_id + self._running = True + + self.request_handlers = { + SetLoraReq: self._handle_set_lora, + MergeLoraWeightsReq: self._handle_merge_lora, + UnmergeLoraWeightsReq: self._handle_unmerge_lora, + Req: self._handle_generation, + } + + def _handle_set_lora(self, reqs: List[Any]): + # TODO: return set status + req = reqs[0] + self.worker.set_lora(req.lora_nickname, req.lora_path) + return {"status": "ok"} + + def _handle_merge_lora(self, _reqs: List[Any]): + self.worker.merge_lora_weights() + return {"status": "ok"} + + def _handle_unmerge_lora(self, _reqs: List[Any]): + self.worker.unmerge_lora_weights() + return {"status": "ok"} + + def _handle_generation(self, reqs: List[Req]): + return self.worker.execute_forward(reqs) + + def return_result(self, output_batch: OutputBatch): + """ + replies to client, only on rank 0 + """ + if self.receiver is not None: + self.receiver.send_pyobj(output_batch) + + def recv_reqs(self) -> List[Any]: + """ + For non-main schedulers, reqs are broadcasted from main using broadcast_pyobj + """ + if self.receiver is not None: + try: + recv_reqs = self.receiver.recv_pyobj() + except zmq.ZMQError: + # re-raise or handle appropriately to let the outer loop continue + raise + + # Ensure recv_reqs is a list + if not isinstance(recv_reqs, list): + recv_reqs = [recv_reqs] + else: + recv_reqs = None + + # TODO: fix this condition + if self.server_args.sp_degree != 1: + recv_reqs = broadcast_pyobj( + recv_reqs, + self.worker.sp_group.rank, + self.worker.sp_cpu_group, + src=self.worker.sp_group.ranks[0], + ) + + if self.server_args.enable_cfg_parallel: + recv_reqs = broadcast_pyobj( + recv_reqs, + self.worker.cfg_group.rank, + self.worker.cfg_cpu_group, + src=self.worker.cfg_group.ranks[0], + ) + + if self.server_args.tp_size > 1: + recv_reqs = broadcast_pyobj( + recv_reqs, + self.worker.tp_group.rank, + self.worker.tp_cpu_group, + src=self.worker.tp_group.ranks[0], + ) + + assert recv_reqs is not None + + return recv_reqs + + # TODO: queueing, cancellation + def event_loop(self) -> None: + """ + The main event loop that listens for ZMQ requests. + Handles abortion + """ + + logger.info( + f"Rank 0 scheduler listening on tcp://*:{self.server_args.scheduler_port}" + ) + + while self._running: + reqs = None + # 1: receive requests + try: + reqs = self.recv_reqs() + except Exception as e: + logger.error( + f"Error receiving requests in scheduler event loop: {e}", + exc_info=True, + ) + continue + + # 2: execute, make sure a reply is always sent + try: + first_req = reqs[0] if reqs else None + + handler = self.request_handlers.get(type(first_req)) + if handler: + output_batch = handler(reqs) + else: + output_batch = { + "status": "error", + "message": f"Unknown request type: {type(first_req)}", + } + except Exception as e: + logger.error( + f"Error executing request in scheduler event loop: {e}", + exc_info=True, + ) + # Determine appropriate error response format + output_batch = ( + OutputBatch(error=str(e)) + if reqs and isinstance(reqs[0], Req) + else {"status": "error", "message": str(e)} + ) + + try: + self.return_result(output_batch) + except zmq.ZMQError as e: + # Reply failed; log and keep loop alive to accept future requests + logger.error(f"ZMQ error sending reply: {e}") + continue + + logger.info("Scheduler event loop terminated.") + if self.receiver is not None: + self.receiver.close() + self.context.term() + + def _broadcast_task(self, payload: dict[str, Any]) -> None: + """Broadcast a task to all slave worker processes.""" + method = payload["method"] + kwargs = {k: v for k, v in payload.items() if k != "method"} + task = {"method": method, "kwargs": kwargs} + for pipe in self.task_pipes_to_slaves: + pipe.send(task) + + def _execute_on_rank0(self, payload: dict[str, Any]) -> dict[str, Any]: + """Execute task locally on the rank 0 worker.""" + method = payload["method"] + kwargs = {k: v for k, v in payload.items() if k != "method"} + handler = getattr(self.worker, method, None) + if handler: + result = handler(**kwargs) + return {"status": "ok", "result": result} + return {"status": "error", "error": f"Unknown method: {method}"} + + def _collect_slave_results(self) -> List[dict[str, Any]]: + """Collect results from all slave worker processes.""" + results = [] + for pipe in self.result_pipes_from_slaves: + results.append(pipe.recv()) + return results diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/__init__.py new file mode 100644 index 00000000..af2eb7d1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/__init__.py @@ -0,0 +1 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/base.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/base.py new file mode 100644 index 00000000..886a6a33 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/base.py @@ -0,0 +1,134 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from abc import ABC, abstractmethod +from typing import Any + +import torch +from torch import nn + +from sglang.multimodal_gen.configs.models import DiTConfig +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum + + +# TODO +class BaseDiT(nn.Module, ABC): + _fsdp_shard_conditions: list = [] + _compile_conditions: list = [] + param_names_mapping: dict + reverse_param_names_mapping: dict + hidden_size: int + num_attention_heads: int + num_channels_latents: int + # always supports torch_sdpa + _supported_attention_backends: set[AttentionBackendEnum] = ( + DiTConfig()._supported_attention_backends + ) + + def __init_subclass__(cls) -> None: + required_class_attrs = [ + "_fsdp_shard_conditions", + "param_names_mapping", + "_compile_conditions", + ] + super().__init_subclass__() + for attr in required_class_attrs: + if not hasattr(cls, attr): + raise AttributeError( + f"Subclasses of BaseDiT must define '{attr}' class variable" + ) + + def __init__(self, config: DiTConfig, hf_config: dict[str, Any], **kwargs) -> None: + super().__init__() + self.config = config + self.hf_config = hf_config + if not self.supported_attention_backends: + raise ValueError( + f"Subclass {self.__class__.__name__} must define _supported_attention_backends" + ) + + @abstractmethod + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor | list[torch.Tensor], + timestep: torch.LongTensor, + encoder_hidden_states_image: torch.Tensor | list[torch.Tensor] | None = None, + guidance=None, + **kwargs, + ) -> torch.Tensor: + pass + + def __post_init__(self) -> None: + required_attrs = ["hidden_size", "num_attention_heads", "num_channels_latents"] + for attr in required_attrs: + if not hasattr(self, attr): + raise AttributeError( + f"Subclasses of BaseDiT must define '{attr}' instance variable" + ) + + @property + def supported_attention_backends(self) -> set[AttentionBackendEnum]: + return self._supported_attention_backends + + @property + def device(self) -> torch.device: + """Get the device of the model.""" + return next(self.parameters()).device + + +class CachableDiT(BaseDiT): + """ + An intermediate base class that adds TeaCache optimization functionality to DiT models. + TeaCache accelerates inference by selectively skipping redundant computation when consecutive + diffusion steps are similar enough. + """ + + # These are required class attributes that should be overridden by concrete implementations + _fsdp_shard_conditions = [] + param_names_mapping = {} + reverse_param_names_mapping = {} + lora_param_names_mapping: dict = {} + # Ensure these instance attributes are properly defined in subclasses + hidden_size: int + num_attention_heads: int + num_channels_latents: int + # always supports torch_sdpa + _supported_attention_backends: set[AttentionBackendEnum] = ( + DiTConfig()._supported_attention_backends + ) + + def __init__(self, config: DiTConfig, **kwargs) -> None: + super().__init__(config, **kwargs) + + self.cnt = 0 + self.teacache_thresh = 0 + self.coefficients: list[float] = [] + + # NOTE(will): Only wan2.1 needs these, so we are hardcoding it here + if self.config.prefix == "wan": + self.use_ret_steps = self.config.cache_config.use_ret_steps + self.is_even = False + self.previous_residual_even: torch.Tensor | None = None + self.previous_residual_odd: torch.Tensor | None = None + self.accumulated_rel_l1_distance_even = 0 + self.accumulated_rel_l1_distance_odd = 0 + self.should_calc_even = True + self.should_calc_odd = True + else: + self.accumulated_rel_l1_distance = 0 + self.previous_modulated_input = None + self.previous_resiual = None + self.previous_e0_even: torch.Tensor | None = None + self.previous_e0_odd: torch.Tensor | None = None + + def maybe_cache_states( + self, hidden_states: torch.Tensor, original_hidden_states: torch.Tensor + ) -> None: + pass + + def should_skip_forward_for_cached_states(self, **kwargs: dict[str, Any]) -> bool: + return False + + def retrieve_cached_states(self, hidden_states: torch.Tensor) -> torch.Tensor: + raise NotImplementedError("maybe_retrieve_cached_states is not implemented") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/causal_wanvideo.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/causal_wanvideo.py new file mode 100644 index 00000000..2789ebdf --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/causal_wanvideo.py @@ -0,0 +1,851 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +import math +from typing import Any + +import torch +import torch.nn as nn +from torch.nn.attention.flex_attention import ( + BlockMask, + create_block_mask, + flex_attention, +) + +# wan 1.3B model has a weird channel / head configurations and require max-autotune to work with flexattention +# see https://github.com/pytorch/pytorch/issues/133254 +# change to default for other models +flex_attention = torch.compile( + flex_attention, dynamic=False, mode="max-autotune-no-cudagraphs" +) +import torch.distributed as dist + +from sglang.multimodal_gen.configs.models.dits import WanVideoConfig +from sglang.multimodal_gen.runtime.distributed.parallel_state import get_sp_world_size +from sglang.multimodal_gen.runtime.layers.attention import LocalAttention +from sglang.multimodal_gen.runtime.layers.layernorm import ( + FP32LayerNorm, + LayerNormScaleShift, + RMSNorm, + ScaleResidual, + ScaleResidualLayerNormScaleShift, +) +from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear +from sglang.multimodal_gen.runtime.layers.mlp import MLP +from sglang.multimodal_gen.runtime.layers.rotary_embedding import ( + _apply_rotary_emb, + get_rotary_pos_embed, +) +from sglang.multimodal_gen.runtime.layers.visual_embedding import PatchEmbed +from sglang.multimodal_gen.runtime.models.dits.base import BaseDiT +from sglang.multimodal_gen.runtime.models.dits.wanvideo import ( + WanT2VCrossAttention, + WanTimeTextImageEmbedding, +) +from sglang.multimodal_gen.runtime.platforms import ( + AttentionBackendEnum, + current_platform, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class CausalWanSelfAttention(nn.Module): + + def __init__( + self, + dim: int, + num_heads: int, + local_attn_size: int = -1, + sink_size: int = 0, + qk_norm=True, + eps=1e-6, + parallel_attention=False, + ) -> None: + assert dim % num_heads == 0 + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.local_attn_size = local_attn_size + self.sink_size = sink_size + self.qk_norm = qk_norm + self.eps = eps + self.parallel_attention = parallel_attention + self.max_attention_size = ( + 32760 if local_attn_size == -1 else local_attn_size * 1560 + ) + + # Scaled dot product attention + self.attn = LocalAttention( + num_heads=num_heads, + head_size=self.head_dim, + dropout_rate=0, + softmax_scale=None, + causal=False, + supported_attention_backends=( + AttentionBackendEnum.FA, + AttentionBackendEnum.TORCH_SDPA, + ), + ) + + def forward( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + freqs_cis: tuple[torch.Tensor, torch.Tensor], + block_mask: BlockMask, + kv_cache: dict | None = None, + current_start: int = 0, + cache_start: int | None = None, + ): + r""" + Args: + x(Tensor): Shape [B, L, num_heads, C / num_heads] + seq_lens(Tensor): Shape [B] + grid_sizes(Tensor): Shape [B, 3], the second dimension contains (F, H, W) + freqs(Tensor): Rope freqs, shape [1024, C / num_heads / 2] + """ + if cache_start is None: + cache_start = current_start + + cos, sin = freqs_cis + roped_query = _apply_rotary_emb(q, cos, sin, is_neox_style=False).type_as(v) + roped_key = _apply_rotary_emb(k, cos, sin, is_neox_style=False).type_as(v) + + if kv_cache is None: + # Padding for flex attention + padded_length = math.ceil(q.shape[1] / 128) * 128 - q.shape[1] + padded_roped_query = torch.cat( + [ + roped_query, + torch.zeros( + [q.shape[0], padded_length, q.shape[2], q.shape[3]], + device=q.device, + dtype=v.dtype, + ), + ], + dim=1, + ) + + padded_roped_key = torch.cat( + [ + roped_key, + torch.zeros( + [k.shape[0], padded_length, k.shape[2], k.shape[3]], + device=k.device, + dtype=v.dtype, + ), + ], + dim=1, + ) + + padded_v = torch.cat( + [ + v, + torch.zeros( + [v.shape[0], padded_length, v.shape[2], v.shape[3]], + device=v.device, + dtype=v.dtype, + ), + ], + dim=1, + ) + + x = flex_attention( + query=padded_roped_query.transpose(2, 1), + key=padded_roped_key.transpose(2, 1), + value=padded_v.transpose(2, 1), + block_mask=block_mask, + )[:, :, :-padded_length].transpose(2, 1) + else: + frame_seqlen = q.shape[1] + current_end = current_start + roped_query.shape[1] + sink_tokens = self.sink_size * frame_seqlen + # If we are using local attention and the current KV cache size is larger than the local attention size, we need to truncate the KV cache + kv_cache_size = kv_cache["k"].shape[1] + num_new_tokens = roped_query.shape[1] + if ( + self.local_attn_size != -1 + and (current_end > kv_cache["global_end_index"].item()) + and ( + num_new_tokens + kv_cache["local_end_index"].item() > kv_cache_size + ) + ): + # Calculate the number of new tokens added in this step + # Shift existing cache content left to discard oldest tokens + # Clone the source slice to avoid overlapping memory error + num_evicted_tokens = ( + num_new_tokens + kv_cache["local_end_index"].item() - kv_cache_size + ) + num_rolled_tokens = ( + kv_cache["local_end_index"].item() + - num_evicted_tokens + - sink_tokens + ) + kv_cache["k"][ + :, sink_tokens : sink_tokens + num_rolled_tokens + ] = kv_cache["k"][ + :, + sink_tokens + + num_evicted_tokens : sink_tokens + + num_evicted_tokens + + num_rolled_tokens, + ].clone() + kv_cache["v"][ + :, sink_tokens : sink_tokens + num_rolled_tokens + ] = kv_cache["v"][ + :, + sink_tokens + + num_evicted_tokens : sink_tokens + + num_evicted_tokens + + num_rolled_tokens, + ].clone() + # Insert the new keys/values at the end + local_end_index = ( + kv_cache["local_end_index"].item() + + current_end + - kv_cache["global_end_index"].item() + - num_evicted_tokens + ) + local_start_index = local_end_index - num_new_tokens + kv_cache["k"][:, local_start_index:local_end_index] = roped_key + kv_cache["v"][:, local_start_index:local_end_index] = v + else: + # Assign new keys/values directly up to current_end + local_end_index = ( + kv_cache["local_end_index"].item() + + current_end + - kv_cache["global_end_index"].item() + ) + local_start_index = local_end_index - num_new_tokens + kv_cache["k"] = kv_cache["k"].detach() + kv_cache["v"] = kv_cache["v"].detach() + # logger.info("kv_cache['k'] is in comp graph: %s", kv_cache["k"].requires_grad or kv_cache["k"].grad_fn is not None) + kv_cache["k"][:, local_start_index:local_end_index] = roped_key + kv_cache["v"][:, local_start_index:local_end_index] = v + x = self.attn( + roped_query, + kv_cache["k"][ + :, + max(0, local_end_index - self.max_attention_size) : local_end_index, + ], + kv_cache["v"][ + :, + max(0, local_end_index - self.max_attention_size) : local_end_index, + ], + ) + kv_cache["global_end_index"].fill_(current_end) + kv_cache["local_end_index"].fill_(local_end_index) + + return x + + +class CausalWanTransformerBlock(nn.Module): + + def __init__( + self, + dim: int, + ffn_dim: int, + num_heads: int, + local_attn_size: int = -1, + sink_size: int = 0, + qk_norm: str = "rms_norm_across_heads", + cross_attn_norm: bool = False, + eps: float = 1e-6, + added_kv_proj_dim: int | None = None, + supported_attention_backends: set[AttentionBackendEnum] | None = None, + prefix: str = "", + ): + super().__init__() + + # 1. Self-attention + self.norm1 = FP32LayerNorm(dim, eps, elementwise_affine=False) + self.to_q = ReplicatedLinear(dim, dim, bias=True) + self.to_k = ReplicatedLinear(dim, dim, bias=True) + self.to_v = ReplicatedLinear(dim, dim, bias=True) + + self.to_out = ReplicatedLinear(dim, dim, bias=True) + self.attn1 = CausalWanSelfAttention( + dim, + num_heads, + local_attn_size=local_attn_size, + sink_size=sink_size, + qk_norm=qk_norm, + eps=eps, + ) + self.hidden_dim = dim + self.num_attention_heads = num_heads + self.local_attn_size = local_attn_size + dim_head = dim // num_heads + if qk_norm == "rms_norm": + self.norm_q = RMSNorm(dim_head, eps=eps) + self.norm_k = RMSNorm(dim_head, eps=eps) + elif qk_norm == "rms_norm_across_heads": + # LTX applies qk norm across all heads + self.norm_q = RMSNorm(dim, eps=eps) + self.norm_k = RMSNorm(dim, eps=eps) + else: + print("QK Norm type not supported") + raise Exception + assert cross_attn_norm is True + self.self_attn_residual_norm = ScaleResidualLayerNormScaleShift( + dim, + norm_type="layer", + eps=eps, + elementwise_affine=True, + dtype=torch.float32, + compute_dtype=torch.float32, + ) + + # 2. Cross-attention + # Only T2V for now + self.attn2 = WanT2VCrossAttention(dim, num_heads, qk_norm=qk_norm, eps=eps) + self.cross_attn_residual_norm = ScaleResidualLayerNormScaleShift( + dim, + norm_type="layer", + eps=eps, + elementwise_affine=False, + dtype=torch.float32, + compute_dtype=torch.float32, + ) + + # 3. Feed-forward + self.ffn = MLP(dim, ffn_dim, act_type="gelu_pytorch_tanh") + self.mlp_residual = ScaleResidual() + + self.scale_shift_table = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + temb: torch.Tensor, + freqs_cis: tuple[torch.Tensor, torch.Tensor], + block_mask: BlockMask, + kv_cache: dict | None = None, + crossattn_cache: dict | None = None, + current_start: int = 0, + cache_start: int | None = None, + ) -> torch.Tensor: + # hidden_states.shape: [batch_size, seq_length, inner_dim] + # temb.shape: [batch_size, num_frames, 6, inner_dim] + if hidden_states.dim() == 4: + hidden_states = hidden_states.squeeze(1) + num_frames = temb.shape[1] + frame_seqlen = hidden_states.shape[1] // num_frames + bs, seq_length, _ = hidden_states.shape + orig_dtype = hidden_states.dtype + # assert orig_dtype != torch.float32 + e = self.scale_shift_table + temb.float() + # e.shape: [batch_size, num_frames, 6, inner_dim] + assert e.shape == (bs, num_frames, 6, self.hidden_dim) + shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa = e.chunk( + 6, dim=2 + ) + # *_msa.shape: [batch_size, num_frames, 1, inner_dim] + assert shift_msa.dtype == torch.float32 + + # 1. Self-attention + norm_hidden_states = ( + ( + self.norm1(hidden_states.float()).unflatten( + dim=1, sizes=(num_frames, frame_seqlen) + ) + * (1 + scale_msa) + + shift_msa + ) + .flatten(1, 2) + .to(orig_dtype) + ) + query, _ = self.to_q(norm_hidden_states) + key, _ = self.to_k(norm_hidden_states) + value, _ = self.to_v(norm_hidden_states) + + if self.norm_q is not None: + query = self.norm_q(query) + if self.norm_k is not None: + key = self.norm_k(key) + + query = query.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) + key = key.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) + value = value.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) + + attn_output = self.attn1( + query, + key, + value, + freqs_cis, + block_mask, + kv_cache, + current_start, + cache_start, + ) + attn_output = attn_output.flatten(2) + attn_output, _ = self.to_out(attn_output) + attn_output = attn_output.squeeze(1) + + null_shift = null_scale = torch.zeroes( + (1,), device=hidden_states.device, dtype=hidden_states.dtype + ) + norm_hidden_states, hidden_states = self.self_attn_residual_norm( + hidden_states, attn_output, gate_msa, null_shift, null_scale + ) + norm_hidden_states, hidden_states = norm_hidden_states.to( + orig_dtype + ), hidden_states.to(orig_dtype) + + # 2. Cross-attention + attn_output = self.attn2( + norm_hidden_states, + context=encoder_hidden_states, + context_lens=None, + crossattn_cache=crossattn_cache, + ) + norm_hidden_states, hidden_states = self.cross_attn_residual_norm( + hidden_states, attn_output, 1, c_shift_msa, c_scale_msa + ) + norm_hidden_states, hidden_states = norm_hidden_states.to( + orig_dtype + ), hidden_states.to(orig_dtype) + + # 3. Feed-forward + ff_output = self.ffn(norm_hidden_states) + hidden_states = self.mlp_residual(hidden_states, ff_output, c_gate_msa) + hidden_states = hidden_states.to(orig_dtype) + + return hidden_states + + +class CausalWanTransformer3DModel(BaseDiT): + _fsdp_shard_conditions = WanVideoConfig()._fsdp_shard_conditions + _compile_conditions = WanVideoConfig()._compile_conditions + _supported_attention_backends = WanVideoConfig()._supported_attention_backends + param_names_mapping = WanVideoConfig().param_names_mapping + reverse_param_names_mapping = WanVideoConfig().reverse_param_names_mapping + lora_param_names_mapping = WanVideoConfig().lora_param_names_mapping + + def __init__(self, config: WanVideoConfig, hf_config: dict[str, Any]) -> None: + super().__init__(config=config, hf_config=hf_config) + + inner_dim = config.num_attention_heads * config.attention_head_dim + self.hidden_size = config.hidden_size + self.num_attention_heads = config.num_attention_heads + self.attention_head_dim = config.attention_head_dim + self.in_channels = config.in_channels + self.out_channels = config.out_channels + self.num_channels_latents = config.num_channels_latents + self.patch_size = config.patch_size + self.text_len = config.text_len + self.local_attn_size = config.local_attn_size + + # 1. Patch & position embedding + self.patch_embedding = PatchEmbed( + in_chans=config.in_channels, + embed_dim=inner_dim, + patch_size=config.patch_size, + flatten=False, + ) + + # 2. Condition embeddings + self.condition_embedder = WanTimeTextImageEmbedding( + dim=inner_dim, + time_freq_dim=config.freq_dim, + text_embed_dim=config.text_dim, + image_embed_dim=config.image_dim, + ) + + # 3. Transformer blocks + self.blocks = nn.ModuleList( + [ + CausalWanTransformerBlock( + inner_dim, + config.ffn_dim, + config.num_attention_heads, + config.local_attn_size, + config.sink_size, + config.qk_norm, + config.cross_attn_norm, + config.eps, + config.added_kv_proj_dim, + self._supported_attention_backends, + prefix=f"{config.prefix}.blocks.{i}", + ) + for i in range(config.num_layers) + ] + ) + + # 4. Output norm & projection + self.norm_out = LayerNormScaleShift( + inner_dim, + norm_type="layer", + eps=config.eps, + elementwise_affine=False, + dtype=torch.float32, + compute_dtype=torch.float32, + ) + self.proj_out = nn.Linear( + inner_dim, config.out_channels * math.prod(config.patch_size) + ) + self.scale_shift_table = nn.Parameter( + torch.randn(1, 2, inner_dim) / inner_dim**0.5 + ) + + self.gradient_checkpointing = False + + # Causal-specific + self.block_mask = None + self.num_frame_per_block = config.arch_config.num_frames_per_block + assert self.num_frame_per_block <= 3 + self.independent_first_frame = False + + self.__post_init__() + + @staticmethod + def _prepare_blockwise_causal_attn_mask( + device: torch.device | str, + num_frames: int = 21, + frame_seqlen: int = 1560, + num_frame_per_block=1, + local_attn_size=-1, + ) -> BlockMask: + """ + we will divide the token sequence into the following format + [1 latent frame] [1 latent frame] ... [1 latent frame] + We use flexattention to construct the attention mask + """ + total_length = num_frames * frame_seqlen + + # we do right padding to get to a multiple of 128 + padded_length = math.ceil(total_length / 128) * 128 - total_length + + ends = torch.zeros( + total_length + padded_length, device=device, dtype=torch.long + ) + + # Block-wise causal mask will attend to all elements that are before the end of the current chunk + frame_indices = torch.arange( + start=0, + end=total_length, + step=frame_seqlen * num_frame_per_block, + device=device, + ) + + for tmp in frame_indices: + ends[tmp : tmp + frame_seqlen * num_frame_per_block] = ( + tmp + frame_seqlen * num_frame_per_block + ) + + def attention_mask(b, h, q_idx, kv_idx): + if local_attn_size == -1: + return (kv_idx < ends[q_idx]) | (q_idx == kv_idx) + else: + return ( + (kv_idx < ends[q_idx]) + & (kv_idx >= (ends[q_idx] - local_attn_size * frame_seqlen)) + ) | (q_idx == kv_idx) + # return ((kv_idx < total_length) & (q_idx < total_length)) | (q_idx == kv_idx) # bidirectional mask + + block_mask = create_block_mask( + attention_mask, + B=None, + H=None, + Q_LEN=total_length + padded_length, + KV_LEN=total_length + padded_length, + _compile=False, + device=device, + ) + + if not dist.is_initialized() or dist.get_rank() == 0: + print( + f" cache a block wise causal mask with block size of {num_frame_per_block} frames" + ) + print(block_mask) + + # import imageio + # import numpy as np + # from torch.nn.attention.flex_attention import create_mask + + # mask = create_mask(attention_mask, B=None, H=None, Q_LEN=total_length + + # padded_length, KV_LEN=total_length + padded_length, device=device) + # import cv2 + # mask = cv2.resize(mask[0, 0].cpu().float().numpy(), (1024, 1024)) + # imageio.imwrite("mask_%d.jpg" % (0), np.uint8(255. * mask)) + + return block_mask + + def _forward_inference( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor | list[torch.Tensor], + timestep: torch.LongTensor, + encoder_hidden_states_image: torch.Tensor | list[torch.Tensor] | None = None, + kv_cache: dict = None, + crossattn_cache: dict = None, + current_start: int = 0, + cache_start: int = 0, + start_frame: int = 0, + **kwargs, + ) -> torch.Tensor: + r""" + Run the diffusion model with kv caching. + See Algorithm 2 of CausVid paper https://arxiv.org/abs/2412.07772 for details. + This function will be run for num_frame times. + Process the latent frames one by one (1560 tokens each) + """ + + orig_dtype = hidden_states.dtype + if not isinstance(encoder_hidden_states, torch.Tensor): + encoder_hidden_states = encoder_hidden_states[0] + if ( + isinstance(encoder_hidden_states_image, list) + and len(encoder_hidden_states_image) > 0 + ): + encoder_hidden_states_image = encoder_hidden_states_image[0] + else: + encoder_hidden_states_image = None + + batch_size, num_channels, num_frames, height, width = hidden_states.shape + p_t, p_h, p_w = self.patch_size + post_patch_num_frames = num_frames // p_t + post_patch_height = height // p_h + post_patch_width = width // p_w + + # Get rotary embeddings + d = self.hidden_size // self.num_attention_heads + rope_dim_list = [d - 4 * (d // 6), 2 * (d // 6), 2 * (d // 6)] + freqs_cos, freqs_sin = get_rotary_pos_embed( + ( + post_patch_num_frames * get_sp_world_size(), + post_patch_height, + post_patch_width, + ), + self.hidden_size, + self.num_attention_heads, + rope_dim_list, + dtype=torch.float32 if current_platform.is_mps() else torch.float64, + rope_theta=10000, + start_frame=start_frame, # Assume that start_frame is 0 when kv_cache is None + ) + freqs_cos = freqs_cos.to(hidden_states.device) + freqs_sin = freqs_sin.to(hidden_states.device) + freqs_cis = ( + (freqs_cos.float(), freqs_sin.float()) if freqs_cos is not None else None + ) + + hidden_states = self.patch_embedding(hidden_states) + hidden_states = hidden_states.flatten(2).transpose(1, 2) + + temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image = ( + self.condition_embedder( + timestep.flatten(), encoder_hidden_states, encoder_hidden_states_image + ) + ) + timestep_proj = timestep_proj.unflatten(1, (6, self.hidden_size)).unflatten( + dim=0, sizes=timestep.shape + ) + + if encoder_hidden_states_image is not None: + encoder_hidden_states = torch.concat( + [encoder_hidden_states_image, encoder_hidden_states], dim=1 + ) + + encoder_hidden_states = ( + encoder_hidden_states.to(orig_dtype) + if current_platform.is_mps() + else encoder_hidden_states + ) # cast to orig_dtype for MPS + + assert encoder_hidden_states.dtype == orig_dtype + + # 4. Transformer blocks + for block_index, block in enumerate(self.blocks): + if torch.is_grad_enabled() and self.gradient_checkpointing: + causal_kwargs = { + "kv_cache": kv_cache[block_index], + "current_start": current_start, + "cache_start": cache_start, + "block_mask": self.block_mask, + } + hidden_states = self._gradient_checkpointing_func( + block, + hidden_states, + encoder_hidden_states, + timestep_proj, + freqs_cis, + **causal_kwargs, + ) + else: + causal_kwargs = { + "kv_cache": kv_cache[block_index], + "crossattn_cache": crossattn_cache[block_index], + "current_start": current_start, + "cache_start": cache_start, + "block_mask": self.block_mask, + } + hidden_states = block( + hidden_states, + encoder_hidden_states, + timestep_proj, + freqs_cis, + **causal_kwargs, + ) + + # 5. Output norm, projection & unpatchify + temb = temb.unflatten(dim=0, sizes=timestep.shape).unsqueeze(2) + shift, scale = (self.scale_shift_table.unsqueeze(1) + temb).chunk(2, dim=2) + hidden_states = self.norm_out(hidden_states, shift, scale) + hidden_states = self.proj_out(hidden_states) + + hidden_states = hidden_states.reshape( + batch_size, + post_patch_num_frames, + post_patch_height, + post_patch_width, + p_t, + p_h, + p_w, + -1, + ) + hidden_states = hidden_states.permute(0, 7, 1, 4, 2, 5, 3, 6) + output = hidden_states.flatten(6, 7).flatten(4, 5).flatten(2, 3) + + return output + + def _forward_train( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor | list[torch.Tensor], + timestep: torch.LongTensor, + encoder_hidden_states_image: torch.Tensor | list[torch.Tensor] | None = None, + start_frame: int = 0, + **kwargs, + ) -> torch.Tensor: + + orig_dtype = hidden_states.dtype + if not isinstance(encoder_hidden_states, torch.Tensor): + encoder_hidden_states = encoder_hidden_states[0] + if ( + isinstance(encoder_hidden_states_image, list) + and len(encoder_hidden_states_image) > 0 + ): + encoder_hidden_states_image = encoder_hidden_states_image[0] + else: + encoder_hidden_states_image = None + + batch_size, num_channels, num_frames, height, width = hidden_states.shape + p_t, p_h, p_w = self.patch_size + post_patch_num_frames = num_frames // p_t + post_patch_height = height // p_h + post_patch_width = width // p_w + + # Get rotary embeddings + d = self.hidden_size // self.num_attention_heads + rope_dim_list = [d - 4 * (d // 6), 2 * (d // 6), 2 * (d // 6)] + freqs_cos, freqs_sin = get_rotary_pos_embed( + ( + post_patch_num_frames * get_sp_world_size(), + post_patch_height, + post_patch_width, + ), + self.hidden_size, + self.num_attention_heads, + rope_dim_list, + dtype=torch.float32 if current_platform.is_mps() else torch.float64, + rope_theta=10000, + start_frame=start_frame, + ) + freqs_cos = freqs_cos.to(hidden_states.device) + freqs_sin = freqs_sin.to(hidden_states.device) + freqs_cis = ( + (freqs_cos.float(), freqs_sin.float()) if freqs_cos is not None else None + ) + + # Construct blockwise causal attn mask + if self.block_mask is None: + self.block_mask = self._prepare_blockwise_causal_attn_mask( + device=hidden_states.device, + num_frames=num_frames, + frame_seqlen=post_patch_height * post_patch_width, + num_frame_per_block=self.num_frame_per_block, + local_attn_size=self.local_attn_size, + ) + + hidden_states = self.patch_embedding(hidden_states) + hidden_states = hidden_states.flatten(2).transpose(1, 2) + + temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image = ( + self.condition_embedder( + timestep.flatten(), encoder_hidden_states, encoder_hidden_states_image + ) + ) + timestep_proj = timestep_proj.unflatten(1, (6, self.hidden_size)).unflatten( + dim=0, sizes=timestep.shape + ) + + if encoder_hidden_states_image is not None: + encoder_hidden_states = torch.concat( + [encoder_hidden_states_image, encoder_hidden_states], dim=1 + ) + + encoder_hidden_states = ( + encoder_hidden_states.to(orig_dtype) + if current_platform.is_mps() + else encoder_hidden_states + ) # cast to orig_dtype for MPS + + assert encoder_hidden_states.dtype == orig_dtype + + # 4. Transformer blocks + if torch.is_grad_enabled() and self.gradient_checkpointing: + for block in self.blocks: + hidden_states = self._gradient_checkpointing_func( + block, + hidden_states, + encoder_hidden_states, + timestep_proj, + freqs_cis, + block_mask=self.block_mask, + ) + else: + for block in self.blocks: + hidden_states = block( + hidden_states, + encoder_hidden_states, + timestep_proj, + freqs_cis, + block_mask=self.block_mask, + ) + + # 5. Output norm, projection & unpatchify + temb = temb.unflatten(dim=0, sizes=timestep.shape).unsqueeze(2) + shift, scale = (self.scale_shift_table.unsqueeze(1) + temb).chunk(2, dim=2) + hidden_states = self.norm_out(hidden_states, shift, scale) + hidden_states = self.proj_out(hidden_states) + + hidden_states = hidden_states.reshape( + batch_size, + post_patch_num_frames, + post_patch_height, + post_patch_width, + p_t, + p_h, + p_w, + -1, + ) + hidden_states = hidden_states.permute(0, 7, 1, 4, 2, 5, 3, 6) + output = hidden_states.flatten(6, 7).flatten(4, 5).flatten(2, 3) + + return output + + def forward(self, *args, **kwargs): + if kwargs.get("kv_cache") is not None: + return self._forward_inference(*args, **kwargs) + else: + return self._forward_train(*args, **kwargs) + + +EntryClass = CausalWanTransformer3DModel diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/flux.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/flux.py new file mode 100644 index 00000000..fbb75242 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/flux.py @@ -0,0 +1,558 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# Copyright 2025 Black Forest Labs, The HuggingFace Team and The InstantX Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn as nn +from diffusers.models.attention import AttentionModuleMixin, FeedForward +from diffusers.models.embeddings import ( + CombinedTimestepGuidanceTextProjEmbeddings, + CombinedTimestepTextProjEmbeddings, +) +from diffusers.models.modeling_outputs import Transformer2DModelOutput +from diffusers.models.normalization import ( + AdaLayerNormContinuous, + AdaLayerNormZero, + AdaLayerNormZeroSingle, +) +from torch.nn import LayerNorm as LayerNorm + +from sglang.multimodal_gen.configs.models.dits.flux import FluxConfig +from sglang.multimodal_gen.runtime.layers.attention import USPAttention + +# from sglang.multimodal_gen.runtime.layers.layernorm import LayerNorm as LayerNorm +from sglang.multimodal_gen.runtime.layers.layernorm import RMSNorm +from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear +from sglang.multimodal_gen.runtime.layers.mlp import MLP +from sglang.multimodal_gen.runtime.layers.rotary_embedding import ( + NDRotaryEmbedding, + _apply_rotary_emb, +) +from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT +from sglang.multimodal_gen.runtime.platforms import ( + AttentionBackendEnum, + current_platform, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) # pylint: disable=invalid-name + + +def _get_projections(attn: "FluxAttention", hidden_states, encoder_hidden_states=None): + query, _ = attn.to_q(hidden_states) + key, _ = attn.to_k(hidden_states) + value, _ = attn.to_v(hidden_states) + + encoder_query = encoder_key = encoder_value = None + if encoder_hidden_states is not None and attn.added_kv_proj_dim is not None: + encoder_query, _ = attn.add_q_proj(encoder_hidden_states) + encoder_key, _ = attn.add_k_proj(encoder_hidden_states) + encoder_value, _ = attn.add_v_proj(encoder_hidden_states) + + return query, key, value, encoder_query, encoder_key, encoder_value + + +def _get_fused_projections( + attn: "FluxAttention", hidden_states, encoder_hidden_states=None +): + query, key, value = attn.to_qkv(hidden_states).chunk(3, dim=-1) + + encoder_query = encoder_key = encoder_value = None + if encoder_hidden_states is not None and hasattr(attn, "to_added_qkv"): + encoder_query, encoder_key, encoder_value = attn.to_added_qkv( + encoder_hidden_states + ).chunk(3, dim=-1) + + return query, key, value, encoder_query, encoder_key, encoder_value + + +def _get_qkv_projections( + attn: "FluxAttention", hidden_states, encoder_hidden_states=None +): + if attn.fused_projections: + return _get_fused_projections(attn, hidden_states, encoder_hidden_states) + return _get_projections(attn, hidden_states, encoder_hidden_states) + + +class FluxAttention(torch.nn.Module, AttentionModuleMixin): + + def __init__( + self, + query_dim: int, + num_heads: int = 8, + dim_head: int = 64, + dropout: float = 0.0, + bias: bool = False, + added_kv_proj_dim: Optional[int] = None, + added_proj_bias: Optional[bool] = True, + out_bias: bool = True, + eps: float = 1e-5, + out_dim: int = None, + context_pre_only: Optional[bool] = None, + pre_only: bool = False, + ): + super().__init__() + + self.head_dim = dim_head + self.inner_dim = out_dim if out_dim is not None else dim_head * num_heads + self.query_dim = query_dim + self.use_bias = bias + self.dropout = dropout + self.out_dim = out_dim if out_dim is not None else query_dim + self.context_pre_only = context_pre_only + self.pre_only = pre_only + self.heads = out_dim // dim_head if out_dim is not None else num_heads + self.added_kv_proj_dim = added_kv_proj_dim + self.added_proj_bias = added_proj_bias + + self.norm_q = RMSNorm(dim_head, eps=eps) + + self.norm_k = RMSNorm(dim_head, eps=eps) + self.to_q = ReplicatedLinear(query_dim, self.inner_dim, bias=bias) + self.to_k = ReplicatedLinear(query_dim, self.inner_dim, bias=bias) + self.to_v = ReplicatedLinear(query_dim, self.inner_dim, bias=bias) + + if not self.pre_only: + self.to_out = torch.nn.ModuleList([]) + self.to_out.append( + ReplicatedLinear(self.inner_dim, self.out_dim, bias=out_bias) + ) + if dropout != 0.0: + self.to_out.append(torch.nn.Dropout(dropout)) + + if added_kv_proj_dim is not None: + self.norm_added_q = RMSNorm(dim_head, eps=eps) + self.norm_added_k = RMSNorm(dim_head, eps=eps) + self.add_q_proj = ReplicatedLinear( + added_kv_proj_dim, self.inner_dim, bias=added_proj_bias + ) + self.add_k_proj = ReplicatedLinear( + added_kv_proj_dim, self.inner_dim, bias=added_proj_bias + ) + self.add_v_proj = ReplicatedLinear( + added_kv_proj_dim, self.inner_dim, bias=added_proj_bias + ) + self.to_add_out = ReplicatedLinear(self.inner_dim, query_dim, bias=out_bias) + + self.attn = USPAttention( + num_heads=num_heads, + head_size=self.head_dim, + dropout_rate=0, + softmax_scale=None, + causal=False, + supported_attention_backends={ + AttentionBackendEnum.FA, + AttentionBackendEnum.TORCH_SDPA, + AttentionBackendEnum.SAGE_ATTN, + }, + ) + + def forward( + self, + x: torch.Tensor, + encoder_hidden_states: Optional[torch.Tensor] = None, + freqs_cis=None, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + query, key, value, encoder_query, encoder_key, encoder_value = ( + _get_qkv_projections(self, x, encoder_hidden_states) + ) + + query = query.unflatten(-1, (self.heads, -1)) + key = key.unflatten(-1, (self.heads, -1)) + value = value.unflatten(-1, (self.heads, -1)) + query = self.norm_q(query) + key = self.norm_k(key) + + if self.added_kv_proj_dim is not None: + encoder_query = encoder_query.unflatten(-1, (self.heads, -1)) + encoder_key = encoder_key.unflatten(-1, (self.heads, -1)) + encoder_value = encoder_value.unflatten(-1, (self.heads, -1)) + + encoder_query = self.norm_added_q(encoder_query) + encoder_key = self.norm_added_k(encoder_key) + + bsz, seq_len, _, _ = query.shape + query = torch.cat([encoder_query, query], dim=1) + key = torch.cat([encoder_key, key], dim=1) + value = torch.cat([encoder_value, value], dim=1) + + if freqs_cis is not None: + cos, sin = freqs_cis + query = _apply_rotary_emb( + query, cos, sin, is_neox_style=False, interleaved=False + ) + key = _apply_rotary_emb( + key, cos, sin, is_neox_style=False, interleaved=False + ) + + x = self.attn(query, key, value) + x = x.flatten(2, 3) + x = x.to(query.dtype) + + if encoder_hidden_states is not None: + encoder_hidden_states, x = x.split_with_sizes( + [ + encoder_hidden_states.shape[1], + x.shape[1] - encoder_hidden_states.shape[1], + ], + dim=1, + ) + x, _ = self.to_out[0](x) + if len(self.to_out) == 2: + x = self.to_out[1](x) + encoder_hidden_states, _ = self.to_add_out(encoder_hidden_states) + + return x, encoder_hidden_states + else: + return x + + +class FluxSingleTransformerBlock(nn.Module): + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + mlp_ratio: float = 4.0, + ): + super().__init__() + self.mlp_hidden_dim = int(dim * mlp_ratio) + + self.norm = AdaLayerNormZeroSingle(dim) + self.proj_mlp = ReplicatedLinear(dim, self.mlp_hidden_dim) + self.act_mlp = nn.GELU(approximate="tanh") + self.proj_out = ReplicatedLinear(dim + self.mlp_hidden_dim, dim) + + self.attn = FluxAttention( + query_dim=dim, + dim_head=attention_head_dim, + num_heads=num_attention_heads, + out_dim=dim, + bias=True, + eps=1e-6, + pre_only=True, + ) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + temb: torch.Tensor, + freqs_cis: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + joint_attention_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + text_seq_len = encoder_hidden_states.shape[1] + hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim=1) + + residual = hidden_states + norm_hidden_states, gate = self.norm(hidden_states, emb=temb) + proj_hidden_states, _ = self.proj_mlp(norm_hidden_states) + mlp_hidden_states = self.act_mlp(proj_hidden_states) + joint_attention_kwargs = joint_attention_kwargs or {} + attn_output = self.attn( + x=norm_hidden_states, + freqs_cis=freqs_cis, + **joint_attention_kwargs, + ) + + hidden_states = torch.cat([attn_output, mlp_hidden_states], dim=2) + gate = gate.unsqueeze(1) + proj_out, _ = self.proj_out(hidden_states) + hidden_states = gate * proj_out + hidden_states = residual + hidden_states + if hidden_states.dtype == torch.float16: + hidden_states = hidden_states.clip(-65504, 65504) + + encoder_hidden_states, hidden_states = ( + hidden_states[:, :text_seq_len], + hidden_states[:, text_seq_len:], + ) + return encoder_hidden_states, hidden_states + + +class FluxTransformerBlock(nn.Module): + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + qk_norm: str = "rms_norm", + eps: float = 1e-6, + ): + super().__init__() + + self.norm1 = AdaLayerNormZero(dim) + self.norm1_context = AdaLayerNormZero(dim) + + self.attn = FluxAttention( + query_dim=dim, + added_kv_proj_dim=dim, + dim_head=attention_head_dim, + num_heads=num_attention_heads, + out_dim=dim, + context_pre_only=False, + bias=True, + eps=eps, + ) + + self.norm2 = LayerNorm(dim, eps=1e-6, elementwise_affine=False) + self.ff = MLP( + input_dim=dim, mlp_hidden_dim=dim * 4, output_dim=dim, act_type="gelu" + ) + self.ff = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate") + + self.norm2_context = LayerNorm(dim, eps=1e-6, elementwise_affine=False) + self.ff_context = MLP( + input_dim=dim, mlp_hidden_dim=dim * 4, output_dim=dim, act_type="gelu" + ) + + self.ff_context = FeedForward( + dim=dim, dim_out=dim, activation_fn="gelu-approximate" + ) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + temb: torch.Tensor, + freqs_cis: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + joint_attention_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1( + hidden_states, emb=temb + ) + + norm_encoder_hidden_states, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = ( + self.norm1_context(encoder_hidden_states, emb=temb) + ) + + joint_attention_kwargs = joint_attention_kwargs or {} + # Attention. + attention_outputs = self.attn( + x=norm_hidden_states, + encoder_hidden_states=norm_encoder_hidden_states, + freqs_cis=freqs_cis, + **joint_attention_kwargs, + ) + + if len(attention_outputs) == 2: + attn_output, context_attn_output = attention_outputs + elif len(attention_outputs) == 3: + attn_output, context_attn_output, ip_attn_output = attention_outputs + + # Process attention outputs for the `hidden_states`. + attn_output = gate_msa.unsqueeze(1) * attn_output + hidden_states = hidden_states + attn_output + norm_hidden_states = self.norm2(hidden_states) + norm_hidden_states = ( + norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None] + ) + + ff_output = self.ff(norm_hidden_states) + ff_output = gate_mlp.unsqueeze(1) * ff_output + + hidden_states = hidden_states + ff_output + + if len(attention_outputs) == 3: + hidden_states = hidden_states + ip_attn_output + # Process attention outputs for the `encoder_hidden_states`. + context_attn_output = c_gate_msa.unsqueeze(1) * context_attn_output + encoder_hidden_states = encoder_hidden_states + context_attn_output + + norm_encoder_hidden_states = self.norm2_context(encoder_hidden_states) + norm_encoder_hidden_states = ( + norm_encoder_hidden_states * (1 + c_scale_mlp[:, None]) + + c_shift_mlp[:, None] + ) + + context_ff_output = self.ff_context(norm_encoder_hidden_states) + encoder_hidden_states = ( + encoder_hidden_states + c_gate_mlp.unsqueeze(1) * context_ff_output + ) + if encoder_hidden_states.dtype == torch.float16: + encoder_hidden_states = encoder_hidden_states.clip(-65504, 65504) + + return encoder_hidden_states, hidden_states + + +class FluxPosEmbed(nn.Module): + # modified from https://github.com/black-forest-labs/flux/blob/c00d7c60b085fce8058b9df845e036090873f2ce/src/flux/modules/layers.py#L11 + def __init__(self, theta: int, axes_dim: List[int]): + super().__init__() + self.rope = NDRotaryEmbedding( + rope_dim_list=axes_dim, + rope_theta=theta, + use_real=False, + repeat_interleave_real=False, + dtype=torch.float32 if current_platform.is_mps() else torch.float64, + ) + + def forward(self, ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + pos = ids.float() + # freqs_cos, freqs_sin = self.rope.forward(positions=pos) + freqs_cos, freqs_sin = self.rope.forward_uncached(pos=pos) + return freqs_cos.contiguous().float(), freqs_sin.contiguous().float() + + +class FluxTransformer2DModel(CachableDiT): + """ + The Transformer model introduced in Flux. + + Reference: https://blackforestlabs.ai/announcing-black-forest-labs/ + """ + + def __init__(self, config: FluxConfig, hf_config: dict[str, Any]) -> None: + super().__init__(config=config, hf_config=hf_config) + self.config = config.arch_config + + self.out_channels = ( + getattr(self.config, "out_channels", None) or self.config.in_channels + ) + self.inner_dim = ( + self.config.num_attention_heads * self.config.attention_head_dim + ) + + self.rotary_emb = FluxPosEmbed(theta=10000, axes_dim=self.config.axes_dims_rope) + + text_time_guidance_cls = ( + CombinedTimestepGuidanceTextProjEmbeddings + if self.config.guidance_embeds + else CombinedTimestepTextProjEmbeddings + ) + self.time_text_embed = text_time_guidance_cls( + embedding_dim=self.inner_dim, + pooled_projection_dim=self.config.pooled_projection_dim, + ) + + self.context_embedder = ReplicatedLinear( + self.config.joint_attention_dim, self.inner_dim + ) + self.x_embedder = ReplicatedLinear(self.config.in_channels, self.inner_dim) + self.transformer_blocks = nn.ModuleList( + [ + FluxTransformerBlock( + dim=self.inner_dim, + num_attention_heads=self.config.num_attention_heads, + attention_head_dim=self.config.attention_head_dim, + ) + for _ in range(self.config.num_layers) + ] + ) + + self.single_transformer_blocks = nn.ModuleList( + [ + FluxSingleTransformerBlock( + dim=self.inner_dim, + num_attention_heads=self.config.num_attention_heads, + attention_head_dim=self.config.attention_head_dim, + ) + for _ in range(self.config.num_single_layers) + ] + ) + + self.norm_out = AdaLayerNormContinuous( + self.inner_dim, self.inner_dim, elementwise_affine=False, eps=1e-6 + ) + self.proj_out = ReplicatedLinear( + self.inner_dim, + self.config.patch_size * self.config.patch_size * self.out_channels, + bias=True, + ) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor = None, + pooled_projections: torch.Tensor = None, + timestep: torch.LongTensor = None, + guidance: torch.Tensor = None, + freqs_cis: torch.Tensor = None, + joint_attention_kwargs: Optional[Dict[str, Any]] = None, + ) -> Union[torch.Tensor, Transformer2DModelOutput]: + """ + The [`FluxTransformer2DModel`] forward method. + + Args: + hidden_states (`torch.Tensor` of shape `(batch_size, image_sequence_length, in_channels)`): + Input `hidden_states`. + encoder_hidden_states (`torch.Tensor` of shape `(batch_size, text_sequence_length, joint_attention_dim)`): + Conditional embeddings (embeddings computed from the input conditions such as prompts) to use. + pooled_projections (`torch.Tensor` of shape `(batch_size, projection_dim)`): Embeddings projected + from the embeddings of input conditions. + timestep ( `torch.LongTensor`): + Used to indicate denoising step. + guidance (`torch.Tensor`): + Guidance embeddings. + joint_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + + """ + if ( + joint_attention_kwargs is not None + and joint_attention_kwargs.get("scale", None) is not None + ): + logger.warning( + "Passing `scale` via `joint_attention_kwargs` when not using the PEFT backend is ineffective." + ) + hidden_states, _ = self.x_embedder(hidden_states) + + temb = ( + self.time_text_embed(timestep, pooled_projections) + if guidance is None + else self.time_text_embed(timestep, guidance, pooled_projections) + ) + + encoder_hidden_states, _ = self.context_embedder(encoder_hidden_states) + + if ( + joint_attention_kwargs is not None + and "ip_adapter_image_embeds" in joint_attention_kwargs + ): + ip_adapter_image_embeds = joint_attention_kwargs.pop( + "ip_adapter_image_embeds" + ) + ip_hidden_states = self.encoder_hid_proj(ip_adapter_image_embeds) + joint_attention_kwargs.update({"ip_hidden_states": ip_hidden_states}) + + for index_block, block in enumerate(self.transformer_blocks): + encoder_hidden_states, hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + temb=temb, + freqs_cis=freqs_cis, + joint_attention_kwargs=joint_attention_kwargs, + ) + + for index_block, block in enumerate(self.single_transformer_blocks): + encoder_hidden_states, hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + temb=temb, + freqs_cis=freqs_cis, + joint_attention_kwargs=joint_attention_kwargs, + ) + + hidden_states = self.norm_out(hidden_states, temb) + + output, _ = self.proj_out(hidden_states) + + return output + + +EntryClass = FluxTransformer2DModel diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py new file mode 100644 index 00000000..3ff593f8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py @@ -0,0 +1,814 @@ +# Copyright 2025 Black Forest Labs, The HuggingFace Team and The InstantX Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Dict, Optional, Tuple + +import torch +import torch.nn as nn +from diffusers.models.attention import AttentionModuleMixin +from diffusers.models.embeddings import ( + TimestepEmbedding, + Timesteps, + get_1d_rotary_pos_embed, +) +from diffusers.models.normalization import AdaLayerNormContinuous + +from sglang.multimodal_gen.configs.models.dits.flux import FluxConfig +from sglang.multimodal_gen.runtime.layers.attention import USPAttention +from sglang.multimodal_gen.runtime.layers.layernorm import RMSNorm +from sglang.multimodal_gen.runtime.layers.rotary_embedding import _apply_rotary_emb +from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) # pylint: disable=invalid-name + + +def _get_projections(attn: "Flux2Attention", hidden_states, encoder_hidden_states=None): + query = attn.to_q(hidden_states) + key = attn.to_k(hidden_states) + value = attn.to_v(hidden_states) + + encoder_query = encoder_key = encoder_value = None + if encoder_hidden_states is not None and attn.added_kv_proj_dim is not None: + encoder_query = attn.add_q_proj(encoder_hidden_states) + encoder_key = attn.add_k_proj(encoder_hidden_states) + encoder_value = attn.add_v_proj(encoder_hidden_states) + + return query, key, value, encoder_query, encoder_key, encoder_value + + +def _get_fused_projections( + attn: "Flux2Attention", hidden_states, encoder_hidden_states=None +): + query, key, value = attn.to_qkv(hidden_states).chunk(3, dim=-1) + + encoder_query = encoder_key = encoder_value = (None,) + if encoder_hidden_states is not None and hasattr(attn, "to_added_qkv"): + encoder_query, encoder_key, encoder_value = attn.to_added_qkv( + encoder_hidden_states + ).chunk(3, dim=-1) + + return query, key, value, encoder_query, encoder_key, encoder_value + + +def _get_qkv_projections( + attn: "Flux2Attention", hidden_states, encoder_hidden_states=None +): + if attn.fused_projections: + return _get_fused_projections(attn, hidden_states, encoder_hidden_states) + return _get_projections(attn, hidden_states, encoder_hidden_states) + + +class Flux2SwiGLU(nn.Module): + """ + Flux 2 uses a SwiGLU-style activation in the transformer feedforward sub-blocks, but with the linear projection + layer fused into the first linear layer of the FF sub-block. Thus, this module has no trainable parameters. + """ + + def __init__(self): + super().__init__() + self.gate_fn = nn.SiLU() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x1, x2 = x.chunk(2, dim=-1) + x = self.gate_fn(x1) * x2 + return x + + +class Flux2FeedForward(nn.Module): + def __init__( + self, + dim: int, + dim_out: Optional[int] = None, + mult: float = 3.0, + inner_dim: Optional[int] = None, + bias: bool = False, + ): + super().__init__() + if inner_dim is None: + inner_dim = int(dim * mult) + dim_out = dim_out or dim + + # Flux2SwiGLU will reduce the dimension by half + self.linear_in = nn.Linear(dim, inner_dim * 2, bias=bias) + self.act_fn = Flux2SwiGLU() + self.linear_out = nn.Linear(inner_dim, dim_out, bias=bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.linear_in(x) + x = self.act_fn(x) + x = self.linear_out(x) + return x + + +class Flux2Attention(torch.nn.Module, AttentionModuleMixin): + + def __init__( + self, + query_dim: int, + num_heads: int = 8, + dim_head: int = 64, + dropout: float = 0.0, + bias: bool = False, + added_kv_proj_dim: Optional[int] = None, + added_proj_bias: Optional[bool] = True, + out_bias: bool = True, + eps: float = 1e-5, + out_dim: int = None, + elementwise_affine: bool = True, + ): + super().__init__() + + self.head_dim = dim_head + self.inner_dim = out_dim if out_dim is not None else dim_head * num_heads + self.query_dim = query_dim + self.out_dim = out_dim if out_dim is not None else query_dim + self.heads = out_dim // dim_head if out_dim is not None else num_heads + + self.use_bias = bias + self.dropout = dropout + + self.added_kv_proj_dim = added_kv_proj_dim + self.added_proj_bias = added_proj_bias + + self.to_q = torch.nn.Linear(query_dim, self.inner_dim, bias=bias) + self.to_k = torch.nn.Linear(query_dim, self.inner_dim, bias=bias) + self.to_v = torch.nn.Linear(query_dim, self.inner_dim, bias=bias) + + # QK Norm + self.norm_q = RMSNorm(dim_head, eps=eps) + self.norm_k = RMSNorm(dim_head, eps=eps) + + self.to_out = torch.nn.ModuleList([]) + self.to_out.append(torch.nn.Linear(self.inner_dim, self.out_dim, bias=out_bias)) + self.to_out.append(torch.nn.Dropout(dropout)) + + if added_kv_proj_dim is not None: + self.norm_added_q = RMSNorm(dim_head, eps=eps) + self.norm_added_k = RMSNorm(dim_head, eps=eps) + self.add_q_proj = torch.nn.Linear( + added_kv_proj_dim, self.inner_dim, bias=added_proj_bias + ) + self.add_k_proj = torch.nn.Linear( + added_kv_proj_dim, self.inner_dim, bias=added_proj_bias + ) + self.add_v_proj = torch.nn.Linear( + added_kv_proj_dim, self.inner_dim, bias=added_proj_bias + ) + self.to_add_out = torch.nn.Linear(self.inner_dim, query_dim, bias=out_bias) + + self.attn = USPAttention( + num_heads=num_heads, + head_size=self.head_dim, + dropout_rate=0, + softmax_scale=None, + causal=False, + supported_attention_backends={ + AttentionBackendEnum.FA, + AttentionBackendEnum.TORCH_SDPA, + AttentionBackendEnum.SAGE_ATTN, + }, + ) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: Optional[torch.Tensor] = None, + freqs_cis: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + ) -> torch.Tensor: + query, key, value, encoder_query, encoder_key, encoder_value = ( + _get_qkv_projections(self, hidden_states, encoder_hidden_states) + ) + + query = query.unflatten(-1, (self.heads, -1)) + key = key.unflatten(-1, (self.heads, -1)) + value = value.unflatten(-1, (self.heads, -1)) + + query = self.norm_q(query) + key = self.norm_k(key) + + if self.added_kv_proj_dim is not None: + encoder_query = encoder_query.unflatten(-1, (self.heads, -1)) + encoder_key = encoder_key.unflatten(-1, (self.heads, -1)) + encoder_value = encoder_value.unflatten(-1, (self.heads, -1)) + + encoder_query = self.norm_added_q(encoder_query) + encoder_key = self.norm_added_k(encoder_key) + + query = torch.cat([encoder_query, query], dim=1) + key = torch.cat([encoder_key, key], dim=1) + value = torch.cat([encoder_value, value], dim=1) + + if freqs_cis is not None: + cos, sin = freqs_cis + query = _apply_rotary_emb( + query, cos, sin, is_neox_style=False, interleaved=True + ) + key = _apply_rotary_emb( + key, cos, sin, is_neox_style=False, interleaved=True + ) + + hidden_states = self.attn(query, key, value) + + hidden_states = hidden_states.flatten(2, 3) + hidden_states = hidden_states.to(query.dtype) + + if encoder_hidden_states is not None: + encoder_hidden_states, hidden_states = hidden_states.split_with_sizes( + [ + encoder_hidden_states.shape[1], + hidden_states.shape[1] - encoder_hidden_states.shape[1], + ], + dim=1, + ) + encoder_hidden_states = self.to_add_out(encoder_hidden_states) + + hidden_states = self.to_out[0](hidden_states) + hidden_states = self.to_out[1](hidden_states) + + if encoder_hidden_states is not None: + return hidden_states, encoder_hidden_states + else: + return hidden_states + + +class Flux2ParallelSelfAttention(torch.nn.Module, AttentionModuleMixin): + """ + Flux 2 parallel self-attention for the Flux 2 single-stream transformer blocks. + + This implements a parallel transformer block, where the attention QKV projections are fused to the feedforward (FF) + input projections, and the attention output projections are fused to the FF output projections. See the [ViT-22B + paper](https://arxiv.org/abs/2302.05442) for a visual depiction of this type of transformer block. + """ + + # Does not support QKV fusion as the QKV projections are always fused + _supports_qkv_fusion = False + + def __init__( + self, + query_dim: int, + num_heads: int = 8, + dim_head: int = 64, + dropout: float = 0.0, + bias: bool = False, + out_bias: bool = True, + eps: float = 1e-5, + out_dim: int = None, + elementwise_affine: bool = True, + mlp_ratio: float = 4.0, + mlp_mult_factor: int = 2, + ): + super().__init__() + + self.head_dim = dim_head + self.inner_dim = out_dim if out_dim is not None else dim_head * num_heads + self.query_dim = query_dim + self.out_dim = out_dim if out_dim is not None else query_dim + self.heads = out_dim // dim_head if out_dim is not None else num_heads + + self.use_bias = bias + self.dropout = dropout + + self.mlp_ratio = mlp_ratio + self.mlp_hidden_dim = int(query_dim * self.mlp_ratio) + self.mlp_mult_factor = mlp_mult_factor + + # Fused QKV projections + MLP input projection + self.to_qkv_mlp_proj = torch.nn.Linear( + self.query_dim, + self.inner_dim * 3 + self.mlp_hidden_dim * self.mlp_mult_factor, + bias=bias, + ) + self.mlp_act_fn = Flux2SwiGLU() + + # QK Norm + self.norm_q = RMSNorm(dim_head, eps=eps) + self.norm_k = RMSNorm(dim_head, eps=eps) + + # Fused attention output projection + MLP output projection + self.to_out = torch.nn.Linear( + self.inner_dim + self.mlp_hidden_dim, self.out_dim, bias=out_bias + ) + + self.attn = USPAttention( + num_heads=num_heads, + head_size=self.head_dim, + dropout_rate=0, + softmax_scale=None, + causal=False, + supported_attention_backends={ + AttentionBackendEnum.FA, + AttentionBackendEnum.TORCH_SDPA, + AttentionBackendEnum.SAGE_ATTN, + }, + ) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + freqs_cis: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + **kwargs, + ) -> torch.Tensor: + # Parallel in (QKV + MLP in) projection + hidden_states = self.to_qkv_mlp_proj(hidden_states) + qkv, mlp_hidden_states = torch.split( + hidden_states, + [3 * self.inner_dim, self.mlp_hidden_dim * self.mlp_mult_factor], + dim=-1, + ) + + # Handle the attention logic + query, key, value = qkv.chunk(3, dim=-1) + + query = query.unflatten(-1, (self.heads, -1)) + key = key.unflatten(-1, (self.heads, -1)) + value = value.unflatten(-1, (self.heads, -1)) + + query = self.norm_q(query) + key = self.norm_k(key) + + if freqs_cis is not None: + cos, sin = freqs_cis + query = _apply_rotary_emb( + query, cos, sin, is_neox_style=False, interleaved=True + ) + key = _apply_rotary_emb( + key, cos, sin, is_neox_style=False, interleaved=True + ) + hidden_states = self.attn(query, key, value) + hidden_states = hidden_states.flatten(2, 3) + hidden_states = hidden_states.to(query.dtype) + + # Handle the feedforward (FF) logic + mlp_hidden_states = self.mlp_act_fn(mlp_hidden_states) + + # Concatenate and parallel output projection + hidden_states = torch.cat([hidden_states, mlp_hidden_states], dim=-1) + hidden_states = self.to_out(hidden_states) + + return hidden_states + + +class Flux2SingleTransformerBlock(nn.Module): + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + mlp_ratio: float = 3.0, + eps: float = 1e-6, + bias: bool = False, + ): + super().__init__() + + self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) + + # Note that the MLP in/out linear layers are fused with the attention QKV/out projections, respectively; this + # is often called a "parallel" transformer block. See the [ViT-22B paper](https://arxiv.org/abs/2302.05442) + # for a visual depiction of this type of transformer block. + self.attn = Flux2ParallelSelfAttention( + query_dim=dim, + dim_head=attention_head_dim, + num_heads=num_attention_heads, + out_dim=dim, + bias=bias, + out_bias=bias, + eps=eps, + mlp_ratio=mlp_ratio, + mlp_mult_factor=2, + ) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: Optional[torch.Tensor], + temb_mod_params: Tuple[torch.Tensor, torch.Tensor, torch.Tensor], + freqs_cis: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + joint_attention_kwargs: Optional[Dict[str, Any]] = None, + split_hidden_states: bool = False, + text_seq_len: Optional[int] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + # If encoder_hidden_states is None, hidden_states is assumed to have encoder_hidden_states already + # concatenated + if encoder_hidden_states is not None: + text_seq_len = encoder_hidden_states.shape[1] + hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim=1) + + mod_shift, mod_scale, mod_gate = temb_mod_params + + norm_hidden_states = self.norm(hidden_states) + norm_hidden_states = (1 + mod_scale) * norm_hidden_states + mod_shift + + joint_attention_kwargs = joint_attention_kwargs or {} + attn_output = self.attn( + hidden_states=norm_hidden_states, + freqs_cis=freqs_cis, + **joint_attention_kwargs, + ) + + hidden_states = hidden_states + mod_gate * attn_output + if hidden_states.dtype == torch.float16: + hidden_states = hidden_states.clip(-65504, 65504) + + if split_hidden_states: + encoder_hidden_states, hidden_states = ( + hidden_states[:, :text_seq_len], + hidden_states[:, text_seq_len:], + ) + return encoder_hidden_states, hidden_states + else: + return hidden_states + + +class Flux2TransformerBlock(nn.Module): + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + mlp_ratio: float = 3.0, + eps: float = 1e-6, + bias: bool = False, + ): + super().__init__() + self.mlp_hidden_dim = int(dim * mlp_ratio) + + self.norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) + self.norm1_context = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) + + self.attn = Flux2Attention( + query_dim=dim, + added_kv_proj_dim=dim, + dim_head=attention_head_dim, + num_heads=num_attention_heads, + out_dim=dim, + bias=bias, + added_proj_bias=bias, + out_bias=bias, + eps=eps, + ) + + self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) + self.ff = Flux2FeedForward(dim=dim, dim_out=dim, mult=mlp_ratio, bias=bias) + + self.norm2_context = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) + self.ff_context = Flux2FeedForward( + dim=dim, dim_out=dim, mult=mlp_ratio, bias=bias + ) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + temb_mod_params_img: Tuple[ + Tuple[torch.Tensor, torch.Tensor, torch.Tensor], ... + ], + temb_mod_params_txt: Tuple[ + Tuple[torch.Tensor, torch.Tensor, torch.Tensor], ... + ], + freqs_cis: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + joint_attention_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + joint_attention_kwargs = joint_attention_kwargs or {} + + # Modulation parameters shape: [1, 1, self.dim] + (shift_msa, scale_msa, gate_msa), (shift_mlp, scale_mlp, gate_mlp) = ( + temb_mod_params_img + ) + (c_shift_msa, c_scale_msa, c_gate_msa), ( + c_shift_mlp, + c_scale_mlp, + c_gate_mlp, + ) = temb_mod_params_txt + + # Img stream + norm_hidden_states = self.norm1(hidden_states) + norm_hidden_states = (1 + scale_msa) * norm_hidden_states + shift_msa + + # Conditioning txt stream + norm_encoder_hidden_states = self.norm1_context(encoder_hidden_states) + norm_encoder_hidden_states = ( + 1 + c_scale_msa + ) * norm_encoder_hidden_states + c_shift_msa + + # Attention on concatenated img + txt stream + attention_outputs = self.attn( + hidden_states=norm_hidden_states, + encoder_hidden_states=norm_encoder_hidden_states, + freqs_cis=freqs_cis, + **joint_attention_kwargs, + ) + + attn_output, context_attn_output = attention_outputs + + # Process attention outputs for the image stream (`hidden_states`). + attn_output = gate_msa * attn_output + hidden_states = hidden_states + attn_output + + norm_hidden_states = self.norm2(hidden_states) + norm_hidden_states = norm_hidden_states * (1 + scale_mlp) + shift_mlp + + ff_output = self.ff(norm_hidden_states) + hidden_states = hidden_states + gate_mlp * ff_output + + # Process attention outputs for the text stream (`encoder_hidden_states`). + context_attn_output = c_gate_msa * context_attn_output + encoder_hidden_states = encoder_hidden_states + context_attn_output + + norm_encoder_hidden_states = self.norm2_context(encoder_hidden_states) + norm_encoder_hidden_states = ( + norm_encoder_hidden_states * (1 + c_scale_mlp) + c_shift_mlp + ) + + context_ff_output = self.ff_context(norm_encoder_hidden_states) + encoder_hidden_states = encoder_hidden_states + c_gate_mlp * context_ff_output + if encoder_hidden_states.dtype == torch.float16: + encoder_hidden_states = encoder_hidden_states.clip(-65504, 65504) + + return encoder_hidden_states, hidden_states + + +class Flux2TimestepGuidanceEmbeddings(nn.Module): + def __init__( + self, in_channels: int = 256, embedding_dim: int = 6144, bias: bool = False + ): + super().__init__() + + self.time_proj = Timesteps( + num_channels=in_channels, flip_sin_to_cos=True, downscale_freq_shift=0 + ) + self.timestep_embedder = TimestepEmbedding( + in_channels=in_channels, time_embed_dim=embedding_dim, sample_proj_bias=bias + ) + + self.guidance_embedder = TimestepEmbedding( + in_channels=in_channels, time_embed_dim=embedding_dim, sample_proj_bias=bias + ) + + def forward(self, timestep: torch.Tensor, guidance: torch.Tensor) -> torch.Tensor: + timesteps_proj = self.time_proj(timestep) + timesteps_emb = self.timestep_embedder( + timesteps_proj.to(timestep.dtype) + ) # (N, D) + + guidance_proj = self.time_proj(guidance) + guidance_emb = self.guidance_embedder( + guidance_proj.to(guidance.dtype) + ) # (N, D) + + time_guidance_emb = timesteps_emb + guidance_emb + + return time_guidance_emb + + +class Flux2Modulation(nn.Module): + def __init__(self, dim: int, mod_param_sets: int = 2, bias: bool = False): + super().__init__() + self.mod_param_sets = mod_param_sets + + self.linear = nn.Linear(dim, dim * 3 * self.mod_param_sets, bias=bias) + self.act_fn = nn.SiLU() + + def forward( + self, temb: torch.Tensor + ) -> Tuple[Tuple[torch.Tensor, torch.Tensor, torch.Tensor], ...]: + mod = self.act_fn(temb) + mod = self.linear(mod) + + if mod.ndim == 2: + mod = mod.unsqueeze(1) + mod_params = torch.chunk(mod, 3 * self.mod_param_sets, dim=-1) + # Return tuple of 3-tuples of modulation params shift/scale/gate + return tuple( + mod_params[3 * i : 3 * (i + 1)] for i in range(self.mod_param_sets) + ) + + +class Flux2PosEmbed(nn.Module): + # modified from https://github.com/black-forest-labs/flux/blob/c00d7c60b085fce8058b9df845e036090873f2ce/src/flux/modules/layers.py#L11 + def __init__(self, theta: int, axes_dim: list[int]): + super().__init__() + self.theta = theta + self.axes_dim = axes_dim + + def forward(self, ids: torch.Tensor) -> torch.Tensor: + # Expected ids shape: [S, len(self.axes_dim)] + cos_out = [] + sin_out = [] + pos = ids.float() + is_mps = ids.device.type == "mps" + is_npu = ids.device.type == "npu" + freqs_dtype = torch.float32 if (is_mps or is_npu) else torch.float64 + # Unlike Flux 1, loop over len(self.axes_dim) rather than ids.shape[-1] + for i in range(len(self.axes_dim)): + cos, sin = get_1d_rotary_pos_embed( + self.axes_dim[i], + pos[..., i], + theta=self.theta, + repeat_interleave_real=True, + use_real=True, + freqs_dtype=freqs_dtype, + ) + cos_out.append(cos) + sin_out.append(sin) + freqs_cos = torch.cat(cos_out, dim=-1).to(ids.device) + freqs_sin = torch.cat(sin_out, dim=-1).to(ids.device) + return freqs_cos, freqs_sin + + +class Flux2Transformer2DModel(CachableDiT): + """ + The Transformer model introduced in Flux 2. + + Reference: https://blackforestlabs.ai/announcing-black-forest-labs/ + + """ + + def __init__(self, config: FluxConfig, hf_config: dict[str, Any]): + super().__init__(config=config, hf_config=hf_config) + patch_size: int = config.patch_size + in_channels: int = config.in_channels + out_channels: Optional[int] = config.out_channels + num_layers: int = config.num_layers + num_single_layers: int = config.num_single_layers + attention_head_dim: int = config.attention_head_dim + num_attention_heads: int = config.num_attention_heads + joint_attention_dim: int = config.joint_attention_dim + timestep_guidance_channels: int = config.timestep_guidance_channels + mlp_ratio: float = config.mlp_ratio + axes_dims_rope: Tuple[int, ...] = config.axes_dims_rope + rope_theta: int = config.rope_theta + eps: float = config.eps + self.out_channels = out_channels or in_channels + self.inner_dim = num_attention_heads * attention_head_dim + + # 1. Sinusoidal positional embedding for RoPE on image and text tokens + self.rotary_emb = Flux2PosEmbed(theta=rope_theta, axes_dim=axes_dims_rope) + + # 2. Combined timestep + guidance embedding + self.time_guidance_embed = Flux2TimestepGuidanceEmbeddings( + in_channels=timestep_guidance_channels, + embedding_dim=self.inner_dim, + bias=False, + ) + + # 3. Modulation (double stream and single stream blocks share modulation parameters, resp.) + # Two sets of shift/scale/gate modulation parameters for the double stream attn and FF sub-blocks + self.double_stream_modulation_img = Flux2Modulation( + self.inner_dim, mod_param_sets=2, bias=False + ) + self.double_stream_modulation_txt = Flux2Modulation( + self.inner_dim, mod_param_sets=2, bias=False + ) + # Only one set of modulation parameters as the attn and FF sub-blocks are run in parallel for single stream + self.single_stream_modulation = Flux2Modulation( + self.inner_dim, mod_param_sets=1, bias=False + ) + + # 4. Input projections + self.x_embedder = nn.Linear(in_channels, self.inner_dim, bias=False) + self.context_embedder = nn.Linear( + joint_attention_dim, self.inner_dim, bias=False + ) + + # 5. Double Stream Transformer Blocks + self.transformer_blocks = nn.ModuleList( + [ + Flux2TransformerBlock( + dim=self.inner_dim, + num_attention_heads=num_attention_heads, + attention_head_dim=attention_head_dim, + mlp_ratio=mlp_ratio, + eps=eps, + bias=False, + ) + for _ in range(num_layers) + ] + ) + + # 6. Single Stream Transformer Blocks + self.single_transformer_blocks = nn.ModuleList( + [ + Flux2SingleTransformerBlock( + dim=self.inner_dim, + num_attention_heads=num_attention_heads, + attention_head_dim=attention_head_dim, + mlp_ratio=mlp_ratio, + eps=eps, + bias=False, + ) + for _ in range(num_single_layers) + ] + ) + + # 7. Output layers + self.norm_out = AdaLayerNormContinuous( + self.inner_dim, + self.inner_dim, + elementwise_affine=False, + eps=eps, + bias=False, + ) + self.proj_out = nn.Linear( + self.inner_dim, patch_size * patch_size * self.out_channels, bias=False + ) + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor = None, + timestep: torch.LongTensor = None, + guidance: torch.Tensor = None, + freqs_cis: torch.Tensor = None, + joint_attention_kwargs: Optional[Dict[str, Any]] = None, + ) -> torch.Tensor: + """ + The [`FluxTransformer2DModel`] forward method. + + Args: + hidden_states (`torch.Tensor` of shape `(batch_size, image_sequence_length, in_channels)`): + Input `hidden_states`. + encoder_hidden_states (`torch.Tensor` of shape `(batch_size, text_sequence_length, joint_attention_dim)`): + Conditional embeddings (embeddings computed from the input conditions such as prompts) to use. + timestep ( `torch.LongTensor`): + Used to indicate denoising step. + joint_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + + """ + # 0. Handle input arguments + if joint_attention_kwargs is not None: + joint_attention_kwargs = joint_attention_kwargs.copy() + lora_scale = joint_attention_kwargs.pop("scale", 1.0) + else: + lora_scale = 1.0 + + num_txt_tokens = encoder_hidden_states.shape[1] + + # 1. Calculate timestep embedding and modulation parameters + timestep = timestep.to(hidden_states.dtype) + guidance = guidance.to(hidden_states.dtype) + + temb = self.time_guidance_embed(timestep, guidance) + + double_stream_mod_img = self.double_stream_modulation_img(temb) + double_stream_mod_txt = self.double_stream_modulation_txt(temb) + single_stream_mod = self.single_stream_modulation(temb)[0] + + # 2. Input projection for image (hidden_states) and conditioning text (encoder_hidden_states) + hidden_states = self.x_embedder(hidden_states) + encoder_hidden_states = self.context_embedder(encoder_hidden_states) + + # 3. Calculate RoPE embeddings from image and text tokens + # NOTE: the below logic means that we can't support batched inference with images of different resolutions or + # text prompts of different lengths. Is this a use case we want to support? + # 4. Double Stream Transformer Blocks + for index_block, block in enumerate(self.transformer_blocks): + encoder_hidden_states, hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + temb_mod_params_img=double_stream_mod_img, + temb_mod_params_txt=double_stream_mod_txt, + freqs_cis=freqs_cis, + joint_attention_kwargs=joint_attention_kwargs, + ) + # Concatenate text and image streams for single-block inference + hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim=1) + + # 5. Single Stream Transformer Blocks + for index_block, block in enumerate(self.single_transformer_blocks): + hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=None, + temb_mod_params=single_stream_mod, + freqs_cis=freqs_cis, + joint_attention_kwargs=joint_attention_kwargs, + ) + # Remove text tokens from concatenated stream + hidden_states = hidden_states[:, num_txt_tokens:, ...] + + # 6. Output layers + hidden_states = self.norm_out(hidden_states, temb) + output = self.proj_out(hidden_states) + + return output + + +EntryClass = Flux2Transformer2DModel diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/hunyuanvideo.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/hunyuanvideo.py new file mode 100644 index 00000000..ad9e10dd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/hunyuanvideo.py @@ -0,0 +1,961 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +from typing import Any + +import numpy as np +import torch +import torch.nn as nn + +from sglang.multimodal_gen.configs.models.dits import HunyuanVideoConfig +from sglang.multimodal_gen.configs.sample.teacache import TeaCacheParams +from sglang.multimodal_gen.runtime.distributed.parallel_state import get_sp_world_size +from sglang.multimodal_gen.runtime.layers.attention import ( + LocalAttention, + UlyssesAttention, +) +from sglang.multimodal_gen.runtime.layers.layernorm import ( + LayerNormScaleShift, + RMSNorm, + ScaleResidual, + ScaleResidualLayerNormScaleShift, +) +from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear +from sglang.multimodal_gen.runtime.layers.mlp import MLP +from sglang.multimodal_gen.runtime.layers.rotary_embedding import ( + _apply_rotary_emb, + get_rotary_pos_embed, +) +from sglang.multimodal_gen.runtime.layers.visual_embedding import ( + ModulateProjection, + PatchEmbed, + TimestepEmbedder, + unpatchify, +) +from sglang.multimodal_gen.runtime.managers.forward_context import get_forward_context +from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT +from sglang.multimodal_gen.runtime.models.utils import modulate +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum + + +class MMDoubleStreamBlock(nn.Module): + """ + A multimodal DiT block with separate modulation for text and image/video, + using distributed attention and linear layers. + """ + + def __init__( + self, + hidden_size: int, + num_attention_heads: int, + mlp_ratio: float, + dtype: torch.dtype | None = None, + supported_attention_backends: set[AttentionBackendEnum] | None = None, + prefix: str = "", + ): + super().__init__() + + self.deterministic = False + self.num_attention_heads = num_attention_heads + head_dim = hidden_size // num_attention_heads + mlp_hidden_dim = int(hidden_size * mlp_ratio) + + # Image modulation components + self.img_mod = ModulateProjection( + hidden_size, + factor=6, + act_layer="silu", + dtype=dtype, + prefix=f"{prefix}.img_mod", + ) + + # Fused operations for image stream + self.img_attn_norm = LayerNormScaleShift( + hidden_size, norm_type="layer", elementwise_affine=False, dtype=dtype + ) + self.img_attn_residual_mlp_norm = ScaleResidualLayerNormScaleShift( + hidden_size, norm_type="layer", elementwise_affine=False, dtype=dtype + ) + self.img_mlp_residual = ScaleResidual() + + # Image attention components + self.img_attn_qkv = ReplicatedLinear( + hidden_size, + hidden_size * 3, + bias=True, + params_dtype=dtype, + prefix=f"{prefix}.img_attn_qkv", + ) + + self.img_attn_q_norm = RMSNorm(head_dim, eps=1e-6, dtype=dtype) + self.img_attn_k_norm = RMSNorm(head_dim, eps=1e-6, dtype=dtype) + + self.img_attn_proj = ReplicatedLinear( + hidden_size, + hidden_size, + bias=True, + params_dtype=dtype, + prefix=f"{prefix}.img_attn_proj", + ) + + self.img_mlp = MLP( + hidden_size, + mlp_hidden_dim, + bias=True, + dtype=dtype, + prefix=f"{prefix}.img_mlp", + ) + + # Text modulation components + self.txt_mod = ModulateProjection( + hidden_size, + factor=6, + act_layer="silu", + dtype=dtype, + prefix=f"{prefix}.txt_mod", + ) + + # Fused operations for text stream + self.txt_attn_norm = LayerNormScaleShift( + hidden_size, norm_type="layer", elementwise_affine=False, dtype=dtype + ) + self.txt_attn_residual_mlp_norm = ScaleResidualLayerNormScaleShift( + hidden_size, norm_type="layer", elementwise_affine=False, dtype=dtype + ) + self.txt_mlp_residual = ScaleResidual() + + # Text attention components + self.txt_attn_qkv = ReplicatedLinear( + hidden_size, hidden_size * 3, bias=True, params_dtype=dtype + ) + + # QK norm layers for text + self.txt_attn_q_norm = RMSNorm(head_dim, eps=1e-6, dtype=dtype) + self.txt_attn_k_norm = RMSNorm(head_dim, eps=1e-6, dtype=dtype) + + self.txt_attn_proj = ReplicatedLinear( + hidden_size, hidden_size, bias=True, params_dtype=dtype + ) + + self.txt_mlp = MLP(hidden_size, mlp_hidden_dim, bias=True, dtype=dtype) + + # Use UlyssesAttention to replace Distributed attention + self.attn = UlyssesAttention( + num_heads=num_attention_heads, + head_size=head_dim, + causal=False, + supported_attention_backends=supported_attention_backends, + prefix=f"{prefix}.attn", + ) + + def forward( + self, + img: torch.Tensor, + txt: torch.Tensor, + vec: torch.Tensor, + freqs_cis: tuple, + ) -> tuple[torch.Tensor, torch.Tensor]: + # Process modulation vectors + img_mod_outputs = self.img_mod(vec) + ( + img_attn_shift, + img_attn_scale, + img_attn_gate, + img_mlp_shift, + img_mlp_scale, + img_mlp_gate, + ) = torch.chunk(img_mod_outputs, 6, dim=-1) + + txt_mod_outputs = self.txt_mod(vec) + ( + txt_attn_shift, + txt_attn_scale, + txt_attn_gate, + txt_mlp_shift, + txt_mlp_scale, + txt_mlp_gate, + ) = torch.chunk(txt_mod_outputs, 6, dim=-1) + + # Prepare image for attention using fused operation + img_attn_input = self.img_attn_norm(img, img_attn_shift, img_attn_scale) + # Get QKV for image + img_qkv, _ = self.img_attn_qkv(img_attn_input) + batch_size, image_seq_len = img_qkv.shape[0], img_qkv.shape[1] + + # Split QKV + img_qkv = img_qkv.view( + batch_size, image_seq_len, 3, self.num_attention_heads, -1 + ) + img_q, img_k, img_v = img_qkv[:, :, 0], img_qkv[:, :, 1], img_qkv[:, :, 2] + + # Apply QK-Norm if needed + + img_q = self.img_attn_q_norm(img_q.contiguous()).to(img_v) + img_k = self.img_attn_k_norm(img_k.contiguous()).to(img_v) + # Apply rotary embeddings + cos, sin = freqs_cis + img_q, img_k = _apply_rotary_emb( + img_q, cos, sin, is_neox_style=False + ), _apply_rotary_emb(img_k, cos, sin, is_neox_style=False) + # Prepare text for attention using fused operation + txt_attn_input = self.txt_attn_norm(txt, txt_attn_shift, txt_attn_scale) + + # Get QKV for text + txt_qkv, _ = self.txt_attn_qkv(txt_attn_input) + batch_size, text_seq_len = txt_qkv.shape[0], txt_qkv.shape[1] + + # Split QKV + txt_qkv = txt_qkv.view( + batch_size, text_seq_len, 3, self.num_attention_heads, -1 + ) + txt_q, txt_k, txt_v = txt_qkv[:, :, 0], txt_qkv[:, :, 1], txt_qkv[:, :, 2] + + # Apply QK-Norm if needed + txt_q = self.txt_attn_q_norm(txt_q.contiguous()).to(txt_q.dtype) + txt_k = self.txt_attn_k_norm(txt_k.contiguous()).to(txt_k.dtype) + + # Run distributed attention + img_attn, txt_attn = self.attn(img_q, img_k, img_v, txt_q, txt_k, txt_v) + img_attn_out, _ = self.img_attn_proj( + img_attn.view(batch_size, image_seq_len, -1) + ) + # Use fused operation for residual connection, normalization, and modulation + img_mlp_input, img_residual = self.img_attn_residual_mlp_norm( + img, img_attn_out, img_attn_gate, img_mlp_shift, img_mlp_scale + ) + + # Process image MLP + img_mlp_out = self.img_mlp(img_mlp_input) + img = self.img_mlp_residual(img_residual, img_mlp_out, img_mlp_gate) + + # Process text attention output + txt_attn_out, _ = self.txt_attn_proj( + txt_attn.reshape(batch_size, text_seq_len, -1) + ) + + # Use fused operation for residual connection, normalization, and modulation + txt_mlp_input, txt_residual = self.txt_attn_residual_mlp_norm( + txt, txt_attn_out, txt_attn_gate, txt_mlp_shift, txt_mlp_scale + ) + + # Process text MLP + txt_mlp_out = self.txt_mlp(txt_mlp_input) + txt = self.txt_mlp_residual(txt_residual, txt_mlp_out, txt_mlp_gate) + + return img, txt + + +class MMSingleStreamBlock(nn.Module): + """ + A DiT block with parallel linear layers using distributed attention + and tensor parallelism. + """ + + def __init__( + self, + hidden_size: int, + num_attention_heads: int, + mlp_ratio: float = 4.0, + dtype: torch.dtype | None = None, + supported_attention_backends: set[AttentionBackendEnum] | None = None, + prefix: str = "", + ): + super().__init__() + + self.deterministic = False + self.hidden_size = hidden_size + self.num_attention_heads = num_attention_heads + head_dim = hidden_size // num_attention_heads + mlp_hidden_dim = int(hidden_size * mlp_ratio) + self.mlp_hidden_dim = mlp_hidden_dim + + # Combined QKV and MLP input projection + self.linear1 = ReplicatedLinear( + hidden_size, + hidden_size * 3 + mlp_hidden_dim, + bias=True, + params_dtype=dtype, + prefix=f"{prefix}.linear1", + ) + + # Combined projection and MLP output + self.linear2 = ReplicatedLinear( + hidden_size + mlp_hidden_dim, + hidden_size, + bias=True, + params_dtype=dtype, + prefix=f"{prefix}.linear2", + ) + + # QK norm layers + self.q_norm = RMSNorm(head_dim, eps=1e-6, dtype=dtype) + self.k_norm = RMSNorm(head_dim, eps=1e-6, dtype=dtype) + + # Fused operations with better naming + self.input_norm_scale_shift = LayerNormScaleShift( + hidden_size, + norm_type="layer", + eps=1e-6, + elementwise_affine=False, + dtype=dtype, + ) + self.output_residual = ScaleResidual() + + # Activation function + self.mlp_act = nn.GELU(approximate="tanh") + + # Modulation + self.modulation = ModulateProjection( + hidden_size, + factor=3, + act_layer="silu", + dtype=dtype, + prefix=f"{prefix}.modulation", + ) + + # Use UlyssesAttention to replace Distributed attention + self.attn = UlyssesAttention( + num_heads=num_attention_heads, + head_size=head_dim, + causal=False, + supported_attention_backends=supported_attention_backends, + prefix=f"{prefix}.attn", + ) + + def forward( + self, + x: torch.Tensor, + vec: torch.Tensor, + txt_len: int, + freqs_cis: tuple[torch.Tensor, torch.Tensor], + ) -> torch.Tensor: + # Process modulation + mod_shift, mod_scale, mod_gate = self.modulation(vec).chunk(3, dim=-1) + + # Apply pre-norm and modulation using fused operation + x_mod = self.input_norm_scale_shift(x, mod_shift, mod_scale) + + # Get combined projections + linear1_out, _ = self.linear1(x_mod) + + # Split into QKV and MLP parts + qkv, mlp = torch.split( + linear1_out, [3 * self.hidden_size, self.mlp_hidden_dim], dim=-1 + ) + + # Process QKV + batch_size, seq_len = qkv.shape[0], qkv.shape[1] + qkv = qkv.view(batch_size, seq_len, 3, self.num_attention_heads, -1) + q, k, v = qkv[:, :, 0], qkv[:, :, 1], qkv[:, :, 2] + + # Apply QK-Norm + q = self.q_norm(q.contiguous()).to(v.dtype) + k = self.k_norm(k.contiguous()).to(v.dtype) + + # Split into image and text parts + img_q, txt_q = q[:, :-txt_len], q[:, -txt_len:] + img_k, txt_k = k[:, :-txt_len], k[:, -txt_len:] + img_v, txt_v = v[:, :-txt_len], v[:, -txt_len:] + # Apply rotary embeddings to image parts + cos, sin = freqs_cis + img_q, img_k = _apply_rotary_emb( + img_q, cos, sin, is_neox_style=False + ), _apply_rotary_emb(img_k, cos, sin, is_neox_style=False) + + # Run distributed attention + img_attn_output, txt_attn_output = self.attn( + img_q, img_k, img_v, txt_q, txt_k, txt_v + ) + attn_output = torch.cat((img_attn_output, txt_attn_output), dim=1).view( + batch_size, seq_len, -1 + ) + # Process MLP activation + mlp_output = self.mlp_act(mlp) + + # Combine attention and MLP outputs + combined = torch.cat((attn_output, mlp_output), dim=-1) + + # Final projection + output, _ = self.linear2(combined) + + # Apply residual connection with gating using fused operation + return self.output_residual(x, output, mod_gate) + + +class HunyuanVideoTransformer3DModel(CachableDiT): + """ + HunyuanVideo Transformer backbone adapted for distributed training. + + This implementation uses distributed attention and linear layers for efficient + parallel processing across multiple GPUs. + + Based on the architecture from: + - Flux.1: https://github.com/black-forest-labs/flux + - MMDiT: http://arxiv.org/abs/2403.03206 + """ + + # PY: we make the input args the same as HF config + + # shard single stream, double stream blocks, and refiner_blocks + _fsdp_shard_conditions = HunyuanVideoConfig()._fsdp_shard_conditions + _compile_conditions = HunyuanVideoConfig()._compile_conditions + _supported_attention_backends = HunyuanVideoConfig()._supported_attention_backends + param_names_mapping = HunyuanVideoConfig().param_names_mapping + reverse_param_names_mapping = HunyuanVideoConfig().reverse_param_names_mapping + lora_param_names_mapping = HunyuanVideoConfig().lora_param_names_mapping + + def __init__(self, config: HunyuanVideoConfig, hf_config: dict[str, Any]): + super().__init__(config=config, hf_config=hf_config) + + self.patch_size = [config.patch_size_t, config.patch_size, config.patch_size] + self.in_channels = config.in_channels + self.num_channels_latents = config.num_channels_latents + self.out_channels = ( + config.in_channels if config.out_channels is None else config.out_channels + ) + self.unpatchify_channels = self.out_channels + self.guidance_embeds = config.guidance_embeds + self.rope_dim_list = list(config.rope_axes_dim) + self.rope_theta = config.rope_theta + self.text_states_dim = config.text_embed_dim + self.text_states_dim_2 = config.pooled_projection_dim + # TODO(will): hack? + self.dtype = config.dtype + + pe_dim = config.hidden_size // config.num_attention_heads + if sum(config.rope_axes_dim) != pe_dim: + raise ValueError( + f"Got {config.rope_axes_dim} but expected positional dim {pe_dim}" + ) + + self.hidden_size = config.hidden_size + self.num_attention_heads = config.num_attention_heads + self.num_channels_latents = config.num_channels_latents + + # Image projection + self.img_in = PatchEmbed( + self.patch_size, + self.in_channels, + self.hidden_size, + dtype=config.dtype, + prefix=f"{config.prefix}.img_in", + ) + + self.txt_in = SingleTokenRefiner( + self.text_states_dim, + config.hidden_size, + config.num_attention_heads, + depth=config.num_refiner_layers, + dtype=config.dtype, + prefix=f"{config.prefix}.txt_in", + ) + + # Time modulation + self.time_in = TimestepEmbedder( + self.hidden_size, + act_layer="silu", + dtype=config.dtype, + prefix=f"{config.prefix}.time_in", + ) + + # Text modulation + self.vector_in = MLP( + self.text_states_dim_2, + self.hidden_size, + self.hidden_size, + act_type="silu", + dtype=config.dtype, + prefix=f"{config.prefix}.vector_in", + ) + + # Guidance modulation + self.guidance_in = ( + TimestepEmbedder( + self.hidden_size, + act_layer="silu", + dtype=config.dtype, + prefix=f"{config.prefix}.guidance_in", + ) + if self.guidance_embeds + else None + ) + + # Double blocks + self.double_blocks = nn.ModuleList( + [ + MMDoubleStreamBlock( + config.hidden_size, + config.num_attention_heads, + mlp_ratio=config.mlp_ratio, + dtype=config.dtype, + supported_attention_backends=self._supported_attention_backends, + prefix=f"{config.prefix}.double_blocks.{i}", + ) + for i in range(config.num_layers) + ] + ) + + # Single blocks + self.single_blocks = nn.ModuleList( + [ + MMSingleStreamBlock( + config.hidden_size, + config.num_attention_heads, + mlp_ratio=config.mlp_ratio, + dtype=config.dtype, + supported_attention_backends=self._supported_attention_backends, + prefix=f"{config.prefix}.single_blocks.{i+config.num_layers}", + ) + for i in range(config.num_single_layers) + ] + ) + + self.final_layer = FinalLayer( + config.hidden_size, + self.patch_size, + self.out_channels, + dtype=config.dtype, + prefix=f"{config.prefix}.final_layer", + ) + + self.__post_init__() + + # TODO: change the input the FORWARD_BATCH Dict + # TODO: change output to a dict + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor | list[torch.Tensor], + timestep: torch.LongTensor, + encoder_hidden_states_image: torch.Tensor | list[torch.Tensor] | None = None, + guidance=None, + **kwargs, + ): + """ + Forward pass of the HunyuanDiT model. + + Args: + hidden_states: Input image/video latents [B, C, T, H, W] + encoder_hidden_states: Text embeddings [B, L, D] + timestep: Diffusion timestep + guidance: Guidance scale for CFG + + Returns: + Tuple of (output) + """ + forward_context = get_forward_context() + forward_batch = forward_context.forward_batch + enable_teacache = forward_batch is not None and forward_batch.enable_teacache + + if guidance is None: + guidance = torch.tensor( + [6016.0], device=hidden_states.device, dtype=hidden_states.dtype + ) + + img = x = hidden_states + t = timestep + + # Split text embeddings - first token is global, rest are per-token + if isinstance(encoder_hidden_states, torch.Tensor): + txt = encoder_hidden_states[:, 1:] + text_states_2 = encoder_hidden_states[:, 0, : self.text_states_dim_2] + else: + txt = encoder_hidden_states[0] + text_states_2 = encoder_hidden_states[1] + + # Get spatial dimensions + _, _, ot, oh, ow = x.shape # codespell:ignore + tt, th, tw = ( + ot // self.patch_size[0], # codespell:ignore + oh // self.patch_size[1], + ow // self.patch_size[2], + ) + + # Get rotary embeddings + freqs_cos, freqs_sin = get_rotary_pos_embed( + (tt * get_sp_world_size(), th, tw), + self.hidden_size, + self.num_attention_heads, + self.rope_dim_list, + self.rope_theta, + ) + freqs_cos = freqs_cos.to(x.device) + freqs_sin = freqs_sin.to(x.device) + # Prepare modulation vectors + vec = self.time_in(t) + + # Add text modulation + vec = vec + self.vector_in(text_states_2) + + # Add guidance modulation if needed + if self.guidance_in and guidance is not None: + vec = vec + self.guidance_in(guidance) + # Embed image and text + img = self.img_in(img) + txt = self.txt_in(txt, t) + txt_seq_len = txt.shape[1] + img_seq_len = img.shape[1] + + freqs_cis = (freqs_cos, freqs_sin) if freqs_cos is not None else None + + should_skip_forward = self.should_skip_forward_for_cached_states( + img=img, vec=vec + ) + + if should_skip_forward: + img = self.retrieve_cached_states(img) + else: + if enable_teacache: + original_img = img.clone() + + # Process through double stream blocks + for index, block in enumerate(self.double_blocks): + double_block_args = [img, txt, vec, freqs_cis] + img, txt = block(*double_block_args) + # Merge txt and img to pass through single stream blocks + x = torch.cat((img, txt), 1) + + # Process through single stream blocks + if len(self.single_blocks) > 0: + for index, block in enumerate(self.single_blocks): + single_block_args = [ + x, + vec, + txt_seq_len, + freqs_cis, + ] + x = block(*single_block_args) + + # Extract image features + img = x[:, :img_seq_len, ...] + + if enable_teacache: + self.maybe_cache_states(img, original_img) + + # Final layer processing + img = self.final_layer(img, vec) + # Unpatchify to get original shape + img = unpatchify(img, tt, th, tw, self.patch_size, self.out_channels) + + return img + + def maybe_cache_states( + self, hidden_states: torch.Tensor, original_hidden_states: torch.Tensor + ) -> None: + self.previous_residual = hidden_states - original_hidden_states + + def should_skip_forward_for_cached_states(self, **kwargs) -> bool: + + forward_context = get_forward_context() + forward_batch = forward_context.forward_batch + if forward_batch is None: + return False + current_timestep = forward_context.current_timestep + enable_teacache = forward_batch.enable_teacache + + if not enable_teacache: + return False + raise NotImplementedError("teacache is not supported yet for HunyuanVideo") + + teacache_params = forward_batch.teacache_params + assert teacache_params is not None, "teacache_params is not initialized" + assert isinstance( + teacache_params, TeaCacheParams + ), "teacache_params is not a TeaCacheParams" + num_inference_steps = forward_batch.num_inference_steps + teache_thresh = teacache_params.teacache_thresh + + coefficients = teacache_params.coefficients + + if current_timestep == 0: + self.cnt = 0 + + inp = kwargs["img"].clone() + vec_ = kwargs["vec"].clone() + # convert to DTensor + vec_ = torch.distributed.tensor.DTensor.from_local( + vec_, + torch.distributed.DeviceMesh( + "cuda", list(range(get_sp_world_size())), mesh_dim_names=("dp",) + ), + [torch.distributed.tensor.Replicate()], + ) + + inp = torch.distributed.tensor.DTensor.from_local( + inp, + torch.distributed.DeviceMesh( + "cuda", list(range(get_sp_world_size())), mesh_dim_names=("dp",) + ), + [torch.distributed.tensor.Replicate()], + ) + + # txt_ = kwargs["txt"].clone() + + # inp = img.clone() + # vec_ = vec.clone() + # txt_ = txt.clone() + ( + img_mod1_shift, + img_mod1_scale, + img_mod1_gate, + img_mod2_shift, + img_mod2_scale, + img_mod2_gate, + ) = ( + self.double_blocks[0].img_mod(vec_).chunk(6, dim=-1) + ) + normed_inp = self.double_blocks[0].img_attn_norm.norm(inp) + modulated_inp = modulate(normed_inp, shift=img_mod1_shift, scale=img_mod1_scale) + if self.cnt == 0 or self.cnt == num_inference_steps - 1: + should_calc = True + self.accumulated_rel_l1_distance = 0 + else: + coefficients = [ + 7.33226126e02, + -4.01131952e02, + 6.75869174e01, + -3.14987800e00, + 9.61237896e-02, + ] + rescale_func = np.poly1d(coefficients) + assert ( + self.previous_modulated_input is not None + ), "previous_modulated_input is not initialized" + self.accumulated_rel_l1_distance += rescale_func( + ( + (modulated_inp - self.previous_modulated_input).abs().mean() + / self.previous_modulated_input.abs().mean() + ) + .cpu() + .item() + ) + if self.accumulated_rel_l1_distance < teache_thresh: + should_calc = False + else: + should_calc = True + self.accumulated_rel_l1_distance = 0 + self.previous_modulated_input = modulated_inp + self.cnt += 1 + + return not should_calc + + def retrieve_cached_states(self, hidden_states: torch.Tensor) -> torch.Tensor: + return hidden_states + self.previous_residual + + +class SingleTokenRefiner(nn.Module): + """ + A token refiner that processes text embeddings with attention to improve + their representation for cross-attention with image features. + """ + + def __init__( + self, + in_channels, + hidden_size, + num_attention_heads, + depth=2, + qkv_bias=True, + dtype=None, + prefix: str = "", + ) -> None: + super().__init__() + + # Input projection + self.input_embedder = ReplicatedLinear( + in_channels, + hidden_size, + bias=True, + params_dtype=dtype, + prefix=f"{prefix}.input_embedder", + ) + + # Timestep embedding + self.t_embedder = TimestepEmbedder( + hidden_size, act_layer="silu", dtype=dtype, prefix=f"{prefix}.t_embedder" + ) + + # Context embedding + self.c_embedder = MLP( + in_channels, + hidden_size, + hidden_size, + act_type="silu", + dtype=dtype, + prefix=f"{prefix}.c_embedder", + ) + + # Refiner blocks + self.refiner_blocks = nn.ModuleList( + [ + IndividualTokenRefinerBlock( + hidden_size, + num_attention_heads, + qkv_bias=qkv_bias, + dtype=dtype, + prefix=f"{prefix}.refiner_blocks.{i}", + ) + for i in range(depth) + ] + ) + + def forward(self, x, t): + # Get timestep embeddings + timestep_aware_representations = self.t_embedder(t) + + # Get context-aware representations + + context_aware_representations = torch.mean(x, dim=1) + + context_aware_representations = self.c_embedder(context_aware_representations) + c = timestep_aware_representations + context_aware_representations + # Project input + x, _ = self.input_embedder(x) + # Process through refiner blocks + for block in self.refiner_blocks: + x = block(x, c) + return x + + +class IndividualTokenRefinerBlock(nn.Module): + """ + A transformer block for refining individual tokens with self-attention. + """ + + def __init__( + self, + hidden_size, + num_attention_heads, + mlp_ratio=4.0, + qkv_bias=True, + dtype=None, + prefix: str = "", + ) -> None: + super().__init__() + self.num_attention_heads = num_attention_heads + mlp_hidden_dim = int(hidden_size * mlp_ratio) + + # Normalization and attention + self.norm1 = nn.LayerNorm( + hidden_size, eps=1e-6, elementwise_affine=True, dtype=dtype + ) + + self.self_attn_qkv = ReplicatedLinear( + hidden_size, + hidden_size * 3, + bias=qkv_bias, + params_dtype=dtype, + prefix=f"{prefix}.self_attn_qkv", + ) + + self.self_attn_proj = ReplicatedLinear( + hidden_size, + hidden_size, + bias=qkv_bias, + params_dtype=dtype, + prefix=f"{prefix}.self_attn_proj", + ) + + # MLP + self.norm2 = nn.LayerNorm( + hidden_size, eps=1e-6, elementwise_affine=True, dtype=dtype + ) + self.mlp = MLP( + hidden_size, + mlp_hidden_dim, + bias=True, + act_type="silu", + dtype=dtype, + prefix=f"{prefix}.mlp", + ) + + # Modulation + self.adaLN_modulation = ModulateProjection( + hidden_size, + factor=2, + act_layer="silu", + dtype=dtype, + prefix=f"{prefix}.adaLN_modulation", + ) + + # Scaled dot product attention + self.attn = LocalAttention( + num_heads=num_attention_heads, + head_size=hidden_size // num_attention_heads, + # TODO: remove hardcode; remove STA + supported_attention_backends=( + AttentionBackendEnum.FA, + AttentionBackendEnum.TORCH_SDPA, + ), + ) + + def forward(self, x, c): + # Get modulation parameters + gate_msa, gate_mlp = self.adaLN_modulation(c).chunk(2, dim=-1) + # Self-attention + norm_x = self.norm1(x) + qkv, _ = self.self_attn_qkv(norm_x) + + batch_size, seq_len = qkv.shape[0], qkv.shape[1] + qkv = qkv.view(batch_size, seq_len, 3, self.num_attention_heads, -1) + q, k, v = qkv[:, :, 0], qkv[:, :, 1], qkv[:, :, 2] + + # Run scaled dot product attention + attn_output = self.attn(q, k, v) # [B, L, H, D] + attn_output = attn_output.reshape(batch_size, seq_len, -1) # [B, L, H*D] + + # Project and apply residual connection with gating + attn_out, _ = self.self_attn_proj(attn_output) + x = x + attn_out * gate_msa.unsqueeze(1) + + # MLP + mlp_out = self.mlp(self.norm2(x)) + x = x + mlp_out * gate_mlp.unsqueeze(1) + + return x + + +class FinalLayer(nn.Module): + """ + The final layer of DiT that projects features to pixel space. + """ + + def __init__( + self, hidden_size, patch_size, out_channels, dtype=None, prefix: str = "" + ) -> None: + super().__init__() + + # Normalization + self.norm_final = nn.LayerNorm( + hidden_size, eps=1e-6, elementwise_affine=False, dtype=dtype + ) + + output_dim = patch_size[0] * patch_size[1] * patch_size[2] * out_channels + + self.linear = ReplicatedLinear( + hidden_size, + output_dim, + bias=True, + params_dtype=dtype, + prefix=f"{prefix}.linear", + ) + + # Modulation + self.adaLN_modulation = ModulateProjection( + hidden_size, + factor=2, + act_layer="silu", + dtype=dtype, + prefix=f"{prefix}.adaLN_modulation", + ) + + def forward(self, x, c): + # What the heck HF? Why you change the scale and shift order here??? + scale, shift = self.adaLN_modulation(c).chunk(2, dim=-1) + x = self.norm_final(x) * (1.0 + scale.unsqueeze(1)) + shift.unsqueeze(1) + x, _ = self.linear(x) + return x + + +EntryClass = HunyuanVideoTransformer3DModel diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py new file mode 100644 index 00000000..989d6d52 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py @@ -0,0 +1,650 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +import functools +from typing import Any, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn as nn +from diffusers.models.attention import FeedForward +from diffusers.models.embeddings import TimestepEmbedding, Timesteps +from diffusers.models.modeling_outputs import Transformer2DModelOutput +from diffusers.models.normalization import AdaLayerNormContinuous + +from sglang.multimodal_gen.configs.models.dits.qwenimage import QwenImageDitConfig +from sglang.multimodal_gen.runtime.layers.attention import USPAttention +from sglang.multimodal_gen.runtime.layers.layernorm import LayerNorm, RMSNorm +from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear +from sglang.multimodal_gen.runtime.layers.triton_ops import ( + apply_rotary_embedding, + fuse_scale_shift_kernel, +) +from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) # pylint: disable=invalid-name + + +class QwenTimestepProjEmbeddings(nn.Module): + def __init__(self, embedding_dim): + super().__init__() + + self.time_proj = Timesteps( + num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=0, scale=1000 + ) + self.timestep_embedder = TimestepEmbedding( + in_channels=256, time_embed_dim=embedding_dim + ) + + def forward(self, timestep, hidden_states): + timesteps_proj = self.time_proj(timestep) + timesteps_emb = self.timestep_embedder( + timesteps_proj.to(dtype=hidden_states.dtype) + ) # (N, D) + + conditioning = timesteps_emb + + return conditioning + + +class QwenEmbedRope(nn.Module): + def __init__(self, theta: int, axes_dim: List[int], scale_rope=False): + super().__init__() + self.theta = theta + self.axes_dim = axes_dim + pos_index = torch.arange(4096) + neg_index = torch.arange(4096).flip(0) * -1 - 1 + self.pos_freqs = torch.cat( + [ + self.rope_params(pos_index, self.axes_dim[0], self.theta), + self.rope_params(pos_index, self.axes_dim[1], self.theta), + self.rope_params(pos_index, self.axes_dim[2], self.theta), + ], + dim=1, + ) + self.neg_freqs = torch.cat( + [ + self.rope_params(neg_index, self.axes_dim[0], self.theta), + self.rope_params(neg_index, self.axes_dim[1], self.theta), + self.rope_params(neg_index, self.axes_dim[2], self.theta), + ], + dim=1, + ) + + # self.rope = NDRotaryEmbedding( + # rope_dim_list=axes_dim, + # rope_theta=theta, + # use_real=False, + # repeat_interleave_real=False, + # dtype=torch.float32 if current_platform.is_mps() else torch.float64, + # ) + + # DO NOT USING REGISTER BUFFER HERE, IT WILL CAUSE COMPLEX NUMBERS LOSE ITS IMAGINARY PART + self.scale_rope = scale_rope + + def rope_params(self, index, dim, theta=10000): + """ + Args: + index: [0, 1, 2, 3] 1D Tensor representing the position index of the token + """ + device = index.device + assert dim % 2 == 0 + freqs = torch.outer( + index, + ( + 1.0 + / torch.pow( + theta, + torch.arange(0, dim, 2, device=device).to(torch.float32).div(dim), + ) + ).to(device=device), + ) + freqs = torch.polar(torch.ones_like(freqs), freqs) + return freqs + + def forward( + self, + video_fhw: Union[Tuple[int, int, int], List[Tuple[int, int, int]]], + txt_seq_lens: List[int], + device: torch.device, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Args: + video_fhw (`Tuple[int, int, int]` or `List[Tuple[int, int, int]]`): + A list of 3 integers [frame, height, width] representing the shape of the video. + txt_seq_lens (`List[int]`): + A list of integers of length batch_size representing the length of each text prompt. + device: (`torch.device`): + The device on which to perform the RoPE computation. + """ + # When models are initialized under a "meta" device context (e.g. init_empty_weights), + # tensors created during __init__ become meta tensors. Calling .to(...) on a meta tensor + # raises "Cannot copy out of meta tensor". Rebuild the frequencies on the target device + # in that case; otherwise move them if just on a different device. + if getattr(self.pos_freqs, "device", torch.device("meta")).type == "meta": + pos_index = torch.arange(4096, device=device) + neg_index = torch.arange(4096, device=device).flip(0) * -1 - 1 + self.pos_freqs = torch.cat( + [ + self.rope_params(pos_index, self.axes_dim[0], self.theta), + self.rope_params(pos_index, self.axes_dim[1], self.theta), + self.rope_params(pos_index, self.axes_dim[2], self.theta), + ], + dim=1, + ).to(device=device) + self.neg_freqs = torch.cat( + [ + self.rope_params(neg_index, self.axes_dim[0], self.theta), + self.rope_params(neg_index, self.axes_dim[1], self.theta), + self.rope_params(neg_index, self.axes_dim[2], self.theta), + ], + dim=1, + ).to(device=device) + elif self.pos_freqs.device != device: + self.pos_freqs = self.pos_freqs.to(device) + self.neg_freqs = self.neg_freqs.to(device) + + if isinstance(video_fhw, list): + video_fhw = video_fhw[0] + if not isinstance(video_fhw, list): + video_fhw = [video_fhw] + + vid_freqs = [] + max_vid_index = 0 + for idx, fhw in enumerate(video_fhw): + frame, height, width = fhw + # RoPE frequencies are cached via a lru_cache decorator on _compute_video_freqs + video_freq = self._compute_video_freqs(frame, height, width, idx) + video_freq = video_freq.to(device) + vid_freqs.append(video_freq) + + if self.scale_rope: + max_vid_index = max(height // 2, width // 2, max_vid_index) + else: + max_vid_index = max(height, width, max_vid_index) + + max_len = max(txt_seq_lens) + txt_freqs = self.pos_freqs[max_vid_index : max_vid_index + max_len, ...] + vid_freqs = torch.cat(vid_freqs, dim=0).to(device=device) + return vid_freqs, txt_freqs + + @functools.lru_cache(maxsize=128) + def _compute_video_freqs( + self, frame: int, height: int, width: int, idx: int = 0 + ) -> torch.Tensor: + seq_lens = frame * height * width + freqs_pos = self.pos_freqs.split([x // 2 for x in self.axes_dim], dim=1) + freqs_neg = self.neg_freqs.split([x // 2 for x in self.axes_dim], dim=1) + + freqs_frame = ( + freqs_pos[0][idx : idx + frame] + .view(frame, 1, 1, -1) + .expand(frame, height, width, -1) + ) + if self.scale_rope: + freqs_height = torch.cat( + [freqs_neg[1][-(height - height // 2) :], freqs_pos[1][: height // 2]], + dim=0, + ) + freqs_height = freqs_height.view(1, height, 1, -1).expand( + frame, height, width, -1 + ) + freqs_width = torch.cat( + [freqs_neg[2][-(width - width // 2) :], freqs_pos[2][: width // 2]], + dim=0, + ) + freqs_width = freqs_width.view(1, 1, width, -1).expand( + frame, height, width, -1 + ) + else: + freqs_height = ( + freqs_pos[1][:height] + .view(1, height, 1, -1) + .expand(frame, height, width, -1) + ) + freqs_width = ( + freqs_pos[2][:width] + .view(1, 1, width, -1) + .expand(frame, height, width, -1) + ) + + freqs = torch.cat([freqs_frame, freqs_height, freqs_width], dim=-1).reshape( + seq_lens, -1 + ) + return freqs.clone().contiguous() + + +class QwenImageCrossAttention(nn.Module): + + def __init__( + self, + dim: int, # query_dim + num_heads: int, + head_dim: int, + window_size=(-1, -1), + added_kv_proj_dim: int = None, + out_bias: bool = True, + qk_norm=True, # rmsnorm + eps=1e-6, + pre_only=False, + context_pre_only: bool = False, + parallel_attention=False, + out_dim: int = None, + ) -> None: + assert dim % num_heads == 0 + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.window_size = window_size + self.qk_norm = qk_norm + self.eps = eps + self.parallel_attention = parallel_attention + + # layers + self.to_q = ReplicatedLinear(dim, dim) + self.to_k = ReplicatedLinear(dim, dim) + self.to_v = ReplicatedLinear(dim, dim) + if self.qk_norm: + self.norm_q = RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity() + self.norm_k = RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity() + self.inner_dim = out_dim if out_dim is not None else head_dim * num_heads + self.inner_kv_dim = self.inner_dim + if added_kv_proj_dim is not None: + self.add_k_proj = ReplicatedLinear( + added_kv_proj_dim, self.inner_kv_dim, bias=True + ) + self.add_v_proj = ReplicatedLinear( + added_kv_proj_dim, self.inner_kv_dim, bias=True + ) + if context_pre_only is not None: + self.add_q_proj = ReplicatedLinear( + added_kv_proj_dim, self.inner_dim, bias=True + ) + + if context_pre_only is not None and not context_pre_only: + self.to_add_out = ReplicatedLinear(self.inner_dim, self.dim, bias=out_bias) + else: + self.to_add_out = None + + if not pre_only: + self.to_out = nn.ModuleList([]) + self.to_out.append( + ReplicatedLinear(self.inner_dim, self.dim, bias=out_bias) + ) + else: + self.to_out = None + + self.norm_added_q = RMSNorm(head_dim, eps=eps) + self.norm_added_k = RMSNorm(head_dim, eps=eps) + + # Scaled dot product attention + self.attn = USPAttention( + num_heads=num_heads, + head_size=self.head_dim, + dropout_rate=0, + softmax_scale=None, + causal=False, + supported_attention_backends={ + AttentionBackendEnum.FA, + AttentionBackendEnum.TORCH_SDPA, + }, + ) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + image_rotary_emb: tuple[torch.Tensor, torch.Tensor], + **cross_attention_kwargs, + ): + seq_len_txt = encoder_hidden_states.shape[1] + + # Compute QKV for image stream (sample projections) + img_query, _ = self.to_q(hidden_states) + img_key, _ = self.to_k(hidden_states) + img_value, _ = self.to_v(hidden_states) + + # Compute QKV for text stream (context projections) + txt_query, _ = self.add_q_proj(encoder_hidden_states) + txt_key, _ = self.add_k_proj(encoder_hidden_states) + txt_value, _ = self.add_v_proj(encoder_hidden_states) + + # Reshape for multi-head attention + img_query = img_query.unflatten(-1, (self.num_heads, -1)) + img_key = img_key.unflatten(-1, (self.num_heads, -1)) + img_value = img_value.unflatten(-1, (self.num_heads, -1)) + + txt_query = txt_query.unflatten(-1, (self.num_heads, -1)) + txt_key = txt_key.unflatten(-1, (self.num_heads, -1)) + txt_value = txt_value.unflatten(-1, (self.num_heads, -1)) + + # Apply QK normalization + if self.norm_q is not None: + img_query = self.norm_q(img_query) + if self.norm_k is not None: + img_key = self.norm_k(img_key) + if self.norm_added_q is not None: + txt_query = self.norm_added_q(txt_query) + if self.norm_added_k is not None: + txt_key = self.norm_added_k(txt_key) + + # Apply RoPE + if image_rotary_emb is not None: + (img_cos, img_sin), (txt_cos, txt_sin) = image_rotary_emb + img_query = apply_rotary_embedding( + img_query, img_cos, img_sin, interleaved=True + ) + img_key = apply_rotary_embedding( + img_key, img_cos, img_sin, interleaved=True + ) + txt_query = apply_rotary_embedding( + txt_query, txt_cos, txt_sin, interleaved=True + ) + txt_key = apply_rotary_embedding( + txt_key, txt_cos, txt_sin, interleaved=True + ) + + # Concatenate for joint attention + # Order: [text, image] + joint_query = torch.cat([txt_query, img_query], dim=1) + joint_key = torch.cat([txt_key, img_key], dim=1) + joint_value = torch.cat([txt_value, img_value], dim=1) + + # Compute joint attention + joint_hidden_states = self.attn( + joint_query, + joint_key, + joint_value, + ) + + # Reshape back + joint_hidden_states = joint_hidden_states.flatten(2, 3) + joint_hidden_states = joint_hidden_states.to(joint_query.dtype) + + # Split attention outputs back + txt_attn_output = joint_hidden_states[:, :seq_len_txt, :] # Text part + img_attn_output = joint_hidden_states[:, seq_len_txt:, :] # Image part + + # Apply output projections + img_attn_output, _ = self.to_out[0](img_attn_output) + if len(self.to_out) > 1: + (img_attn_output,) = self.to_out[1](img_attn_output) # dropout + + txt_attn_output, _ = self.to_add_out(txt_attn_output) + + return img_attn_output, txt_attn_output + + +class QwenImageTransformerBlock(nn.Module): + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + qk_norm: str = "rms_norm", + eps: float = 1e-6, + ): + super().__init__() + + self.dim = dim + self.num_attention_heads = num_attention_heads + self.attention_head_dim = attention_head_dim + + # Image processing modules + self.img_mod = nn.Sequential( + nn.SiLU(), + nn.Linear( + dim, 6 * dim, bias=True + ), # For scale, shift, gate for norm1 and norm2 + ) + self.img_norm1 = LayerNorm(dim, elementwise_affine=False, eps=eps) + + self.attn = QwenImageCrossAttention( + dim=dim, + num_heads=num_attention_heads, + added_kv_proj_dim=dim, + context_pre_only=False, + head_dim=attention_head_dim, + ) + self.img_norm2 = LayerNorm(dim, eps=eps, elementwise_affine=False) + self.img_mlp = FeedForward( + dim=dim, dim_out=dim, activation_fn="gelu-approximate" + ) + + # Text processing modules + self.txt_mod = nn.Sequential( + nn.SiLU(), + nn.Linear( + dim, 6 * dim, bias=True + ), # For scale, shift, gate for norm1 and norm2 + ) + self.txt_norm1 = LayerNorm(dim, elementwise_affine=False, eps=eps) + # Text doesn't need separate attention - it's handled by img_attn joint computation + self.txt_norm2 = LayerNorm(dim, elementwise_affine=False, eps=eps) + self.txt_mlp = FeedForward( + dim=dim, dim_out=dim, activation_fn="gelu-approximate" + ) + + def _modulate(self, x, mod_params): + """Apply modulation to input tensor""" + shift, scale, gate = mod_params.chunk(3, dim=-1) + return fuse_scale_shift_kernel(x, scale, shift), gate.unsqueeze(1) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + encoder_hidden_states_mask: torch.Tensor, + temb: torch.Tensor, + image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + joint_attention_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Get modulation parameters for both streams + img_mod_params = self.img_mod(temb) # [B, 6*dim] + txt_mod_params = self.txt_mod(temb) # [B, 6*dim] + + # Split modulation parameters for norm1 and norm2 + img_mod1, img_mod2 = img_mod_params.chunk(2, dim=-1) # Each [B, 3*dim] + txt_mod1, txt_mod2 = txt_mod_params.chunk(2, dim=-1) # Each [B, 3*dim] + + # Process image stream - norm1 + modulation + + img_normed = self.img_norm1(hidden_states) + + img_modulated, img_gate1 = self._modulate(img_normed, img_mod1) + + # Process text stream - norm1 + modulation + txt_normed = self.txt_norm1(encoder_hidden_states) + txt_modulated, txt_gate1 = self._modulate(txt_normed, txt_mod1) + + # Use QwenAttnProcessor2_0 for joint attention computation + # This directly implements the DoubleStreamLayerMegatron logic: + # 1. Computes QKV for both streams + # 2. Applies QK normalization and RoPE + # 3. Concatenates and runs joint attention + # 4. Splits results back to separate streams + joint_attention_kwargs = joint_attention_kwargs or {} + attn_output = self.attn( + hidden_states=img_modulated, # Image stream (will be processed as "sample") + encoder_hidden_states=txt_modulated, # Text stream (will be processed as "context") + encoder_hidden_states_mask=encoder_hidden_states_mask, + image_rotary_emb=image_rotary_emb, + **joint_attention_kwargs, + ) + + # QwenAttnProcessor2_0 returns (img_output, txt_output) when encoder_hidden_states is provided + img_attn_output, txt_attn_output = attn_output + + # Apply attention gates and add residual (like in Megatron) + hidden_states = hidden_states + img_gate1 * img_attn_output + + encoder_hidden_states = encoder_hidden_states + txt_gate1 * txt_attn_output + + # Process image stream - norm2 + MLP + img_normed2 = self.img_norm2(hidden_states) + img_modulated2, img_gate2 = self._modulate(img_normed2, img_mod2) + img_mlp_output = self.img_mlp(img_modulated2) + hidden_states = hidden_states + img_gate2 * img_mlp_output + + # Process text stream - norm2 + MLP + txt_normed2 = self.txt_norm2(encoder_hidden_states) + txt_modulated2, txt_gate2 = self._modulate(txt_normed2, txt_mod2) + txt_mlp_output = self.txt_mlp(txt_modulated2) + encoder_hidden_states = encoder_hidden_states + txt_gate2 * txt_mlp_output + + # Clip to prevent overflow for fp16 + if encoder_hidden_states.dtype == torch.float16: + encoder_hidden_states = encoder_hidden_states.clip(-65504, 65504) + if hidden_states.dtype == torch.float16: + hidden_states = hidden_states.clip(-65504, 65504) + + return encoder_hidden_states, hidden_states + + +class QwenImageTransformer2DModel(CachableDiT): + """ + The Transformer model introduced in Qwen. + + """ + + _supports_gradient_checkpointing = True + _no_split_modules = ["QwenImageTransformerBlock"] + _skip_layerwise_casting_patterns = ["pos_embed", "norm"] + _repeated_blocks = ["QwenImageTransformerBlock"] + + def __init__( + self, + config: QwenImageDitConfig, + hf_config: dict[str, Any], + ): + super().__init__(config=config, hf_config=hf_config) + patch_size = config.arch_config.patch_size + in_channels = config.arch_config.in_channels + out_channels = config.arch_config.out_channels + num_layers = config.arch_config.num_layers + attention_head_dim = config.arch_config.attention_head_dim + num_attention_heads = config.arch_config.num_attention_heads + joint_attention_dim = config.arch_config.joint_attention_dim + axes_dims_rope = config.arch_config.axes_dims_rope + self.out_channels = out_channels or in_channels + self.inner_dim = num_attention_heads * attention_head_dim + + self.rotary_emb = QwenEmbedRope( + theta=10000, axes_dim=list(axes_dims_rope), scale_rope=True + ) + + self.time_text_embed = QwenTimestepProjEmbeddings(embedding_dim=self.inner_dim) + + self.txt_norm = RMSNorm(joint_attention_dim, eps=1e-6) + + self.img_in = nn.Linear(in_channels, self.inner_dim) + self.txt_in = nn.Linear(joint_attention_dim, self.inner_dim) + + self.transformer_blocks = nn.ModuleList( + [ + QwenImageTransformerBlock( + dim=self.inner_dim, + num_attention_heads=num_attention_heads, + attention_head_dim=attention_head_dim, + ) + for _ in range(num_layers) + ] + ) + + self.norm_out = AdaLayerNormContinuous( + self.inner_dim, self.inner_dim, elementwise_affine=False, eps=1e-6 + ) + self.proj_out = nn.Linear( + self.inner_dim, patch_size * patch_size * self.out_channels, bias=True + ) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor = None, + encoder_hidden_states_mask: torch.Tensor = None, + timestep: torch.LongTensor = None, + txt_seq_lens: Optional[List[int]] = None, + freqs_cis: tuple[torch.Tensor, torch.Tensor] = None, + guidance: torch.Tensor = None, # TODO: this should probably be removed + attention_kwargs: Optional[Dict[str, Any]] = None, + controlnet_block_samples=None, + return_dict: bool = True, + ) -> Union[torch.Tensor, Transformer2DModelOutput]: + """ + The [`QwenTransformer2DModel`] forward method. + + Args: + hidden_states (`torch.Tensor` of shape `(batch_size, image_sequence_length, in_channels)`): + Input `hidden_states`. + encoder_hidden_states (`torch.Tensor` of shape `(batch_size, text_sequence_length, joint_attention_dim)`): + Conditional embeddings (embeddings computed from the input conditions such as prompts) to use. + encoder_hidden_states_mask (`torch.Tensor` of shape `(batch_size, text_sequence_length)`): + Mask of the input conditions. + timestep ( `torch.LongTensor`): + Used to indicate denoising step. + attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.transformer_2d.Transformer2DModelOutput`] instead of a plain + tuple. + + Returns: + If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a + `tuple` where the first element is the sample tensor. + """ + if ( + attention_kwargs is not None + and attention_kwargs.get("scale", None) is not None + ): + logger.warning( + "Passing `scale` via `joint_attention_kwargs` when not using the PEFT backend is ineffective." + ) + + if isinstance(encoder_hidden_states, list): + encoder_hidden_states = encoder_hidden_states[0] + + hidden_states = self.img_in(hidden_states) + + timestep = (timestep / 1000).to(hidden_states.dtype) + encoder_hidden_states = self.txt_norm(encoder_hidden_states) + encoder_hidden_states = self.txt_in(encoder_hidden_states) + + temb = self.time_text_embed(timestep, hidden_states) + + image_rotary_emb = freqs_cis + for index_block, block in enumerate(self.transformer_blocks): + encoder_hidden_states, hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + encoder_hidden_states_mask=encoder_hidden_states_mask, + temb=temb, + image_rotary_emb=image_rotary_emb, + joint_attention_kwargs=attention_kwargs, + ) + + # controlnet residual + if controlnet_block_samples is not None: + interval_control = len(self.transformer_blocks) / len( + controlnet_block_samples + ) + interval_control = int(np.ceil(interval_control)) + hidden_states = ( + hidden_states + + controlnet_block_samples[index_block // interval_control] + ) + + # Use only the image part (hidden_states) from the dual-stream blocks + hidden_states = self.norm_out(hidden_states, temb) + + output = self.proj_out(hidden_states) + return output + + +EntryClass = QwenImageTransformer2DModel diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/stepvideo.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/stepvideo.py new file mode 100644 index 00000000..529c4995 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/stepvideo.py @@ -0,0 +1,729 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# Copyright 2025 StepFun Inc. All Rights Reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# ============================================================================== +from typing import Any + +import torch +from einops import rearrange, repeat +from torch import nn + +from sglang.multimodal_gen.configs.models.dits import StepVideoConfig +from sglang.multimodal_gen.runtime.distributed.parallel_state import get_sp_world_size +from sglang.multimodal_gen.runtime.layers.attention import LocalAttention, USPAttention +from sglang.multimodal_gen.runtime.layers.layernorm import LayerNormScaleShift +from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear +from sglang.multimodal_gen.runtime.layers.mlp import MLP +from sglang.multimodal_gen.runtime.layers.rotary_embedding import ( + _apply_rotary_emb, + get_rotary_pos_embed, +) +from sglang.multimodal_gen.runtime.layers.visual_embedding import TimestepEmbedder +from sglang.multimodal_gen.runtime.models.dits.base import BaseDiT +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum + + +class PatchEmbed2D(nn.Module): + """2D Image to Patch Embedding + + Image to Patch Embedding using Conv2d + + A convolution based approach to patchifying a 2D image w/ embedding projection. + + Based on the impl in https://github.com/google-research/vision_transformer + + Hacked together by / Copyright 2020 Ross Wightman + + Remove the _assert function in forward function to be compatible with multi-resolution images. + """ + + def __init__( + self, + patch_size=16, + in_chans=3, + embed_dim=768, + norm_layer=None, + flatten=True, + bias=True, + dtype=None, + prefix: str = "", + ): + super().__init__() + # Convert patch_size to 2-tuple + if isinstance(patch_size, list | tuple): + if len(patch_size) == 1: + patch_size = (patch_size[0], patch_size[0]) + else: + patch_size = (patch_size, patch_size) + + self.patch_size = patch_size + self.flatten = flatten + + self.proj = nn.Conv2d( + in_chans, + embed_dim, + kernel_size=patch_size, + stride=patch_size, + bias=bias, + dtype=dtype, + ) + self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() + + def forward(self, x): + x = self.proj(x) + if self.flatten: + x = x.flatten(2).transpose(1, 2) # BCHW -> BNC + x = self.norm(x) + return x + + +class StepVideoRMSNorm(nn.Module): + + def __init__( + self, + dim: int, + elementwise_affine=True, + eps: float = 1e-6, + device=None, + dtype=None, + ): + """ + Initialize the RMSNorm normalization layer. + + Args: + dim (int): The dimension of the input tensor. + eps (float, optional): A small value added to the denominator for numerical stability. Default is 1e-6. + + Attributes: + eps (float): A small value added to the denominator for numerical stability. + weight (nn.Parameter): Learnable scaling parameter. + + """ + factory_kwargs = {"device": device, "dtype": dtype} + super().__init__() + self.eps = eps + if elementwise_affine: + self.weight = nn.Parameter(torch.ones(dim, **factory_kwargs)) + + def _norm(self, x) -> torch.Tensor: + """ + Apply the RMSNorm normalization to the input tensor. + + Args: + x (torch.Tensor): The input tensor. + + Returns: + torch.Tensor: The normalized tensor. + + """ + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + + def forward(self, x): + """ + Forward pass through the RMSNorm layer. + + Args: + x (torch.Tensor): The input tensor. + + Returns: + torch.Tensor: The output tensor after applying RMSNorm. + + """ + output = self._norm(x.float()).type_as(x) + if hasattr(self, "weight"): + output = output * self.weight + return output + + +class SelfAttention(nn.Module): + + def __init__( + self, + hidden_dim, + head_dim, + rope_split: tuple[int, int, int] = (64, 32, 32), + bias: bool = False, + with_rope: bool = True, + with_qk_norm: bool = True, + attn_type: str = "torch", + supported_attention_backends=( + AttentionBackendEnum.FA, + AttentionBackendEnum.TORCH_SDPA, + ), + ): + super().__init__() + self.head_dim = head_dim + self.hidden_dim = hidden_dim + self.rope_split = list(rope_split) + self.n_heads = hidden_dim // head_dim + + self.wqkv = ReplicatedLinear(hidden_dim, hidden_dim * 3, bias=bias) + self.wo = ReplicatedLinear(hidden_dim, hidden_dim, bias=bias) + + self.with_rope = with_rope + self.with_qk_norm = with_qk_norm + if self.with_qk_norm: + self.q_norm = StepVideoRMSNorm(head_dim, elementwise_affine=True) + self.k_norm = StepVideoRMSNorm(head_dim, elementwise_affine=True) + + # self.core_attention = self.attn_processor(attn_type=attn_type) + self.parallel = attn_type == "parallel" + self.attn = USPAttention( + num_heads=self.n_heads, + head_size=head_dim, + causal=False, + supported_attention_backends=supported_attention_backends, + ) + + def _apply_rope(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor): + """ + x: [B, S, H, D] + cos: [S, D/2] where D = head_dim = sum(self.rope_split) + sin: [S, D/2] + returns x with rotary applied exactly as v0 did + """ + B, S, H, D = x.shape + # 1) split cos/sin per chunk + half_splits = [c // 2 for c in self.rope_split] # [32,16,16] for [64,32,32] + cos_splits = cos.split(half_splits, dim=1) + sin_splits = sin.split(half_splits, dim=1) + + outs = [] + idx = 0 + for chunk_size, cos_i, sin_i in zip( + self.rope_split, cos_splits, sin_splits, strict=True + ): + # slice the corresponding channels + x_chunk = x[..., idx : idx + chunk_size] # [B,S,H,chunk_size] + idx += chunk_size + + # flatten to [S, B*H, chunk_size] + x_flat = rearrange(x_chunk, "b s h d -> s (b h) d") + + # apply rotary on *that* chunk + out_flat = _apply_rotary_emb(x_flat, cos_i, sin_i, is_neox_style=True) + + # restore [B,S,H,chunk_size] + out = rearrange(out_flat, "s (b h) d -> b s h d", b=B, h=H) + outs.append(out) + + # concatenate back to [B,S,H,D] + return torch.cat(outs, dim=-1) + + def forward( + self, + x, + cu_seqlens=None, + max_seqlen=None, + rope_positions=None, + cos_sin=None, + attn_mask=None, + mask_strategy=None, + ): + + B, S, _ = x.shape + xqkv, _ = self.wqkv(x) + xqkv = xqkv.view(*x.shape[:-1], self.n_heads, 3 * self.head_dim) + q, k, v = torch.split(xqkv, [self.head_dim] * 3, dim=-1) # [B,S,H,D] + + if self.with_qk_norm: + q = self.q_norm(q) + k = self.k_norm(k) + + if self.with_rope: + if rope_positions is not None: + F, Ht, W = rope_positions + assert F * Ht * W == S, "rope_positions mismatches sequence length" + + cos, sin = cos_sin + cos = cos.to(x.device, dtype=x.dtype) + sin = sin.to(x.device, dtype=x.dtype) + + q = self._apply_rope(q, cos, sin) + k = self._apply_rope(k, cos, sin) + + output = self.attn(q, k, v) # [B,heads,S,D] + + output = rearrange(output, "b s h d -> b s (h d)") + output, _ = self.wo(output) + + return output + + +class CrossAttention(nn.Module): + + def __init__( + self, + hidden_dim, + head_dim, + bias=False, + with_qk_norm=True, + supported_attention_backends=( + AttentionBackendEnum.FA, + AttentionBackendEnum.TORCH_SDPA, + ), + ) -> None: + super().__init__() + self.head_dim = head_dim + self.n_heads = hidden_dim // head_dim + + self.wq = ReplicatedLinear(hidden_dim, hidden_dim, bias=bias) + self.wkv = ReplicatedLinear(hidden_dim, hidden_dim * 2, bias=bias) + self.wo = ReplicatedLinear(hidden_dim, hidden_dim, bias=bias) + + self.with_qk_norm = with_qk_norm + if self.with_qk_norm: + self.q_norm = StepVideoRMSNorm(head_dim, elementwise_affine=True) + self.k_norm = StepVideoRMSNorm(head_dim, elementwise_affine=True) + + self.attn = LocalAttention( + num_heads=self.n_heads, + head_size=head_dim, + causal=False, + supported_attention_backends=supported_attention_backends, + ) + + def forward( + self, x: torch.Tensor, encoder_hidden_states: torch.Tensor, attn_mask=None + ) -> torch.Tensor: + + xq, _ = self.wq(x) + xq = xq.view(*xq.shape[:-1], self.n_heads, self.head_dim) + + xkv, _ = self.wkv(encoder_hidden_states) + xkv = xkv.view(*xkv.shape[:-1], self.n_heads, 2 * self.head_dim) + + xk, xv = torch.split(xkv, [self.head_dim] * 2, dim=-1) ## seq_len, n, dim + + if self.with_qk_norm: + xq = self.q_norm(xq) + xk = self.k_norm(xk) + + output = self.attn(xq, xk, xv) + + output = rearrange(output, "b s h d -> b s (h d)") + output, _ = self.wo(output) + + return output + + +class AdaLayerNormSingle(nn.Module): + r""" + Norm layer adaptive layer norm single (adaLN-single). + + As proposed in PixArt-Alpha (see: https://arxiv.org/abs/2310.00426; Section 2.3). + + Parameters: + embedding_dim (`int`): The size of each embedding vector. + use_additional_conditions (`bool`): To use additional conditions for normalization or not. + """ + + def __init__(self, embedding_dim: int, time_step_rescale=1000): + super().__init__() + + self.emb = TimestepEmbedder(embedding_dim) + + self.silu = nn.SiLU() + self.linear = ReplicatedLinear(embedding_dim, 6 * embedding_dim, bias=True) + + self.time_step_rescale = time_step_rescale ## timestep usually in [0, 1], we rescale it to [0,1000] for stability + + def forward( + self, + timestep: torch.Tensor, + added_cond_kwargs: dict[str, torch.Tensor] | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + embedded_timestep = self.emb(timestep * self.time_step_rescale) + + out, _ = self.linear(self.silu(embedded_timestep)) + + return out, embedded_timestep + + +class StepVideoTransformerBlock(nn.Module): + r""" + A basic Transformer block. + + Parameters: + dim (`int`): The number of channels in the input and output. + num_attention_heads (`int`): The number of heads to use for multi-head attention. + attention_head_dim (`int`): The number of channels in each head. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + cross_attention_dim (`int`, *optional*): The size of the encoder_hidden_states vector for cross attention. + activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward. + num_embeds_ada_norm (: + obj: `int`, *optional*): The number of diffusion steps used during training. See `Transformer2DModel`. + attention_bias (: + obj: `bool`, *optional*, defaults to `False`): Configure if the attentions should contain a bias parameter. + only_cross_attention (`bool`, *optional*): + Whether to use only cross-attention layers. In this case two cross attention layers are used. + double_self_attention (`bool`, *optional*): + Whether to use two self-attention layers. In this case no cross attention layers are used. + upcast_attention (`bool`, *optional*): + Whether to upcast the attention computation to float32. This is useful for mixed precision training. + norm_elementwise_affine (`bool`, *optional*, defaults to `True`): + Whether to use learnable elementwise affine parameters for normalization. + norm_type (`str`, *optional*, defaults to `"layer_norm"`): + The normalization layer to use. Can be `"layer_norm"`, `"ada_norm"` or `"ada_norm_zero"`. + final_dropout (`bool` *optional*, defaults to False): + Whether to apply a final dropout after the last feed-forward layer. + positional_embeddings (`str`, *optional*, defaults to `None`): + The type of positional embeddings to apply to. + num_positional_embeddings (`int`, *optional*, defaults to `None`): + The maximum number of positional embeddings to apply. + """ + + def __init__( + self, + dim: int, + attention_head_dim: int, + norm_eps: float = 1e-5, + ff_inner_dim: int | None = None, + ff_bias: bool = False, + attention_type: str = "torch", + ): + super().__init__() + self.dim = dim + self.norm1 = LayerNormScaleShift( + dim, norm_type="layer", elementwise_affine=True, eps=norm_eps + ) + self.attn1 = SelfAttention( + dim, + attention_head_dim, + bias=False, + with_rope=True, + with_qk_norm=True, + ) + + self.norm2 = LayerNormScaleShift( + dim, norm_type="layer", elementwise_affine=True, eps=norm_eps + ) + self.attn2 = CrossAttention( + dim, attention_head_dim, bias=False, with_qk_norm=True + ) + + self.ff = MLP( + input_dim=dim, + mlp_hidden_dim=dim * 4 if ff_inner_dim is None else ff_inner_dim, + act_type="gelu_pytorch_tanh", + bias=ff_bias, + ) + + self.scale_shift_table = nn.Parameter(torch.randn(6, dim) / dim**0.5) + + @torch.no_grad() + def forward( + self, + q: torch.Tensor, + kv: torch.Tensor, + t_expand: torch.LongTensor, + attn_mask=None, + rope_positions: list | None = None, + cos_sin=None, + mask_strategy=None, + ) -> torch.Tensor: + + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( + torch.clone(chunk) + for chunk in ( + self.scale_shift_table[None] + t_expand.reshape(-1, 6, self.dim) + ).chunk(6, dim=1) + ) + + scale_shift_q = self.norm1( + q, scale=scale_msa.squeeze(1), shift=shift_msa.squeeze(1) + ) + + attn_q = self.attn1( + scale_shift_q, + rope_positions=rope_positions, + cos_sin=cos_sin, + mask_strategy=mask_strategy, + ) + + q = attn_q * gate_msa + q + + attn_q = self.attn2(q, kv, attn_mask) + + q = attn_q + q + + scale_shift_q = self.norm2( + q, scale=scale_mlp.squeeze(1), shift=shift_mlp.squeeze(1) + ) + + ff_output = self.ff(scale_shift_q) + + q = ff_output * gate_mlp + q + + return q + + +class StepVideoModel(BaseDiT): + # (Optional) Keep the same attribute for compatibility with splitting, etc. + _fsdp_shard_conditions = [ + lambda n, m: "transformer_blocks" in n and n.split(".")[-1].isdigit(), + # lambda n, m: "pos_embed" in n # If needed for the patch embedding. + ] + param_names_mapping = StepVideoConfig().param_names_mapping + reverse_param_names_mapping = StepVideoConfig().reverse_param_names_mapping + lora_param_names_mapping = StepVideoConfig().lora_param_names_mapping + _supported_attention_backends = StepVideoConfig()._supported_attention_backends + + def __init__(self, config: StepVideoConfig, hf_config: dict[str, Any]) -> None: + super().__init__(config=config, hf_config=hf_config) + self.num_attention_heads = config.num_attention_heads + self.attention_head_dim = config.attention_head_dim + self.in_channels = config.in_channels + self.out_channels = config.out_channels + self.num_layers = config.num_layers + self.dropout = config.dropout + self.patch_size = config.patch_size + self.norm_type = config.norm_type + self.norm_elementwise_affine = config.norm_elementwise_affine + self.norm_eps = config.norm_eps + self.use_additional_conditions = config.use_additional_conditions + self.caption_channels = config.caption_channels + self.attention_type = config.attention_type + self.num_channels_latents = config.num_channels_latents + # Compute inner dimension. + self.hidden_size = config.hidden_size + + # Image/video patch embedding. + self.pos_embed = PatchEmbed2D( + patch_size=self.patch_size, + in_chans=self.in_channels, + embed_dim=self.hidden_size, + ) + + self._rope_cache: dict[tuple, tuple[torch.Tensor, torch.Tensor]] = {} + # Transformer blocks. + self.transformer_blocks = nn.ModuleList( + [ + StepVideoTransformerBlock( + dim=self.hidden_size, + attention_head_dim=self.attention_head_dim, + attention_type=self.attention_type, + ) + for _ in range(self.num_layers) + ] + ) + + # Output blocks. + self.norm_out = LayerNormScaleShift( + self.hidden_size, + norm_type="layer", + eps=self.norm_eps, + elementwise_affine=self.norm_elementwise_affine, + ) + self.scale_shift_table = nn.Parameter( + torch.randn(2, self.hidden_size) / (self.hidden_size**0.5) + ) + self.proj_out = ReplicatedLinear( + self.hidden_size, self.patch_size * self.patch_size * self.out_channels + ) + # Time modulation via adaptive layer norm. + self.adaln_single = AdaLayerNormSingle(self.hidden_size) + + # Set up caption conditioning. + if isinstance(self.caption_channels, int): + caption_channel = self.caption_channels + else: + caption_channel, clip_channel = self.caption_channels + self.clip_projection = ReplicatedLinear(clip_channel, self.hidden_size) + self.caption_norm = nn.LayerNorm( + caption_channel, + eps=self.norm_eps, + elementwise_affine=self.norm_elementwise_affine, + ) + self.caption_projection = MLP( + input_dim=caption_channel, + mlp_hidden_dim=self.hidden_size, + act_type="gelu_pytorch_tanh", + ) + + # Flag to indicate if using parallel attention. + self.parallel = self.attention_type == "parallel" + + self.__post_init__() + + def patchfy(self, hidden_states) -> torch.Tensor: + hidden_states = rearrange(hidden_states, "b f c h w -> (b f) c h w") + hidden_states = self.pos_embed(hidden_states) + return hidden_states + + def prepare_attn_mask( + self, encoder_attention_mask, encoder_hidden_states, q_seqlen + ) -> tuple[torch.Tensor, torch.Tensor]: + kv_seqlens = encoder_attention_mask.sum(dim=1).int() + mask = torch.zeros( + [len(kv_seqlens), q_seqlen, max(kv_seqlens)], + dtype=torch.bool, + device=encoder_attention_mask.device, + ) + encoder_hidden_states = encoder_hidden_states[:, : max(kv_seqlens)] + for i, kv_len in enumerate(kv_seqlens): + mask[i, :, :kv_len] = 1 + return encoder_hidden_states, mask + + def block_forward( + self, + hidden_states, + encoder_hidden_states=None, + t_expand=None, + rope_positions=None, + cos_sin=None, + attn_mask=None, + parallel=True, + mask_strategy=None, + ) -> torch.Tensor: + + for i, block in enumerate(self.transformer_blocks): + hidden_states = block( + hidden_states, + encoder_hidden_states, + t_expand=t_expand, + attn_mask=attn_mask, + rope_positions=rope_positions, + cos_sin=cos_sin, + mask_strategy=mask_strategy[i], + ) + + return hidden_states + + def _get_rope( + self, + rope_positions: tuple[int, int, int], + dtype: torch.dtype, + device: torch.device, + ): + F, Ht, W = rope_positions + key = (F, Ht, W, dtype) + if key not in self._rope_cache: + cos, sin = get_rotary_pos_embed( + rope_sizes=(F * get_sp_world_size(), Ht, W), + hidden_size=self.hidden_size, + heads_num=self.hidden_size // self.attention_head_dim, + rope_dim_list=(64, 32, 32), # same split you used + rope_theta=1.0e4, + dtype=torch.float32, # build once in fp32 + ) + # move & cast once + self._rope_cache[key] = ( + cos.to(device, dtype=dtype), + sin.to(device, dtype=dtype), + ) + return self._rope_cache[key] + + @torch.inference_mode() + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor | None = None, + t_expand: torch.LongTensor | None = None, + encoder_hidden_states_2: torch.Tensor | None = None, + added_cond_kwargs: dict[str, torch.Tensor] | None = None, + encoder_attention_mask: torch.Tensor | None = None, + fps: torch.Tensor | None = None, + return_dict: bool = True, + mask_strategy=None, + guidance=None, + ): + assert hidden_states.ndim == 5 + "hidden_states's shape should be (bsz, f, ch, h ,w)" + frame = hidden_states.shape[2] + hidden_states = rearrange(hidden_states, "b c f h w -> b f c h w", f=frame) + if mask_strategy is None: + mask_strategy = [None, None] + bsz, frame, _, height, width = hidden_states.shape + height, width = height // self.patch_size, width // self.patch_size + + hidden_states = self.patchfy(hidden_states) + len_frame = hidden_states.shape[1] + + t_expand, embedded_timestep = self.adaln_single(t_expand) + encoder_hidden_states = self.caption_projection( + self.caption_norm(encoder_hidden_states) + ) + + if encoder_hidden_states_2 is not None and hasattr(self, "clip_projection"): + clip_embedding, _ = self.clip_projection(encoder_hidden_states_2) + encoder_hidden_states = torch.cat( + [clip_embedding, encoder_hidden_states], dim=1 + ) + + hidden_states = rearrange( + hidden_states, "(b f) l d-> b (f l) d", b=bsz, f=frame, l=len_frame + ).contiguous() + encoder_hidden_states, attn_mask = self.prepare_attn_mask( + encoder_attention_mask, encoder_hidden_states, q_seqlen=frame * len_frame + ) + + cos_sin = self._get_rope( + (frame, height, width), hidden_states.dtype, hidden_states.device + ) + + hidden_states = self.block_forward( + hidden_states, + encoder_hidden_states, + t_expand=t_expand, + rope_positions=[frame, height, width], + cos_sin=cos_sin, + attn_mask=attn_mask, + parallel=self.parallel, + mask_strategy=mask_strategy, + ) + + hidden_states = rearrange( + hidden_states, "b (f l) d -> (b f) l d", b=bsz, f=frame, l=len_frame + ) + + embedded_timestep = repeat( + embedded_timestep, "b d -> (b f) d", f=frame + ).contiguous() + + shift, scale = ( + self.scale_shift_table[None] + embedded_timestep[:, None] + ).chunk(2, dim=1) + hidden_states = self.norm_out( + hidden_states, shift=shift.squeeze(1), scale=scale.squeeze(1) + ) + # Modulation + hidden_states, _ = self.proj_out(hidden_states) + + # unpatchify + hidden_states = hidden_states.reshape( + shape=( + -1, + height, + width, + self.patch_size, + self.patch_size, + self.out_channels, + ) + ) + + hidden_states = rearrange(hidden_states, "n h w p q c -> n c h p w q") + output = hidden_states.reshape( + shape=( + -1, + self.out_channels, + height * self.patch_size, + width * self.patch_size, + ) + ) + + output = rearrange(output, "(b f) c h w -> b c f h w", f=frame) + return output + + +EntryClass = StepVideoModel diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py new file mode 100644 index 00000000..cb674e49 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py @@ -0,0 +1,945 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +import math +from typing import Any + +import numpy as np +import torch +import torch.nn as nn + +from sglang.multimodal_gen.configs.models.dits import WanVideoConfig +from sglang.multimodal_gen.configs.sample.wan import WanTeaCacheParams +from sglang.multimodal_gen.runtime.distributed.parallel_state import get_sp_world_size +from sglang.multimodal_gen.runtime.layers.attention import ( + UlyssesAttention_VSA, + USPAttention, +) +from sglang.multimodal_gen.runtime.layers.layernorm import ( + FP32LayerNorm, + LayerNormScaleShift, + RMSNorm, + ScaleResidual, + ScaleResidualLayerNormScaleShift, +) +from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear +from sglang.multimodal_gen.runtime.layers.mlp import MLP +from sglang.multimodal_gen.runtime.layers.rotary_embedding import ( + NDRotaryEmbedding, + _apply_rotary_emb, +) +from sglang.multimodal_gen.runtime.layers.visual_embedding import ( + ModulateProjection, + PatchEmbed, + TimestepEmbedder, +) +from sglang.multimodal_gen.runtime.managers.forward_context import get_forward_context +from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT +from sglang.multimodal_gen.runtime.platforms import ( + AttentionBackendEnum, + current_platform, +) +from sglang.multimodal_gen.runtime.server_args import get_global_server_args +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class WanImageEmbedding(torch.nn.Module): + + def __init__(self, in_features: int, out_features: int): + super().__init__() + + self.norm1 = FP32LayerNorm(in_features) + self.ff = MLP(in_features, in_features, out_features, act_type="gelu") + self.norm2 = FP32LayerNorm(out_features) + + def forward(self, encoder_hidden_states_image: torch.Tensor) -> torch.Tensor: + dtype = encoder_hidden_states_image.dtype + hidden_states = self.norm1(encoder_hidden_states_image) + hidden_states = self.ff(hidden_states) + hidden_states = self.norm2(hidden_states).to(dtype) + return hidden_states + + +class WanTimeTextImageEmbedding(nn.Module): + + def __init__( + self, + dim: int, + time_freq_dim: int, + text_embed_dim: int, + image_embed_dim: int | None = None, + ): + super().__init__() + + self.time_embedder = TimestepEmbedder( + dim, frequency_embedding_size=time_freq_dim, act_layer="silu" + ) + self.time_modulation = ModulateProjection(dim, factor=6, act_layer="silu") + self.text_embedder = MLP( + text_embed_dim, dim, dim, bias=True, act_type="gelu_pytorch_tanh" + ) + + self.image_embedder = None + if image_embed_dim is not None: + self.image_embedder = WanImageEmbedding(image_embed_dim, dim) + + def forward( + self, + timestep: torch.Tensor, + encoder_hidden_states: torch.Tensor, + encoder_hidden_states_image: torch.Tensor | None = None, + timestep_seq_len: int | None = None, + ): + temb = self.time_embedder(timestep, timestep_seq_len) + timestep_proj = self.time_modulation(temb) + + encoder_hidden_states = self.text_embedder(encoder_hidden_states) + if encoder_hidden_states_image is not None: + assert self.image_embedder is not None + encoder_hidden_states_image = self.image_embedder( + encoder_hidden_states_image + ) + + return temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image + + +class WanSelfAttention(nn.Module): + + def __init__( + self, + dim: int, + num_heads: int, + window_size=(-1, -1), + qk_norm=True, + eps=1e-6, + parallel_attention=False, + supported_attention_backends: set[AttentionBackendEnum] | None = None, + ) -> None: + assert dim % num_heads == 0 + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.window_size = window_size + self.qk_norm = qk_norm + self.eps = eps + self.parallel_attention = parallel_attention + + # layers + self.to_q = ReplicatedLinear(dim, dim) + self.to_k = ReplicatedLinear(dim, dim) + self.to_v = ReplicatedLinear(dim, dim) + self.to_out = ReplicatedLinear(dim, dim) + self.norm_q = RMSNorm(dim, eps=eps) if qk_norm else nn.Identity() + self.norm_k = RMSNorm(dim, eps=eps) if qk_norm else nn.Identity() + + # Scaled dot product attention + self.attn = USPAttention( + num_heads=num_heads, + head_size=self.head_dim, + dropout_rate=0, + softmax_scale=None, + causal=False, + supported_attention_backends=supported_attention_backends, + ) + + def forward(self, x: torch.Tensor, context: torch.Tensor, context_lens: int): + r""" + Args: + x(Tensor): Shape [B, L, num_heads, C / num_heads] + seq_lens(Tensor): Shape [B] + grid_sizes(Tensor): Shape [B, 3], the second dimension contains (F, H, W) + freqs(Tensor): Rope freqs, shape [1024, C / num_heads / 2] + """ + pass + + +class WanT2VCrossAttention(WanSelfAttention): + + def forward(self, x, context, context_lens, crossattn_cache=None): + r""" + Args: + x(Tensor): Shape [B, L1, C] + context(Tensor): Shape [B, L2, C] + context_lens(Tensor): Shape [B] + """ + b, n, d = x.size(0), self.num_heads, self.head_dim + + # compute query, key, value + q = self.norm_q(self.to_q(x)[0]).view(b, -1, n, d) + + if crossattn_cache is not None: + if not crossattn_cache["is_init"]: + crossattn_cache["is_init"] = True + k = self.norm_k(self.to_k(context)[0]).view(b, -1, n, d) + v = self.to_v(context)[0].view(b, -1, n, d) + crossattn_cache["k"] = k + crossattn_cache["v"] = v + else: + k = crossattn_cache["k"] + v = crossattn_cache["v"] + else: + k = self.norm_k(self.to_k(context)[0]).view(b, -1, n, d) + v = self.to_v(context)[0].view(b, -1, n, d) + + # compute attention + x = self.attn(q, k, v) + + # output + x = x.flatten(2) + x, _ = self.to_out(x) + return x + + +class WanI2VCrossAttention(WanSelfAttention): + + def __init__( + self, + dim: int, + num_heads: int, + window_size=(-1, -1), + qk_norm=True, + eps=1e-6, + supported_attention_backends: set[AttentionBackendEnum] | None = None, + ) -> None: + # VSA should not be in supported_attention_backends + super().__init__( + dim, + num_heads, + window_size, + qk_norm, + eps, + supported_attention_backends=supported_attention_backends, + ) + + self.add_k_proj = ReplicatedLinear(dim, dim) + self.add_v_proj = ReplicatedLinear(dim, dim) + self.norm_added_k = RMSNorm(dim, eps=eps) if qk_norm else nn.Identity() + self.norm_added_q = RMSNorm(dim, eps=eps) if qk_norm else nn.Identity() + + def forward(self, x, context, context_lens): + r""" + Args: + x(Tensor): Shape [B, L1, C] + context(Tensor): Shape [B, L2, C] + context_lens(Tensor): Shape [B] + """ + context_img = context[:, :257] + context = context[:, 257:] + b, n, d = x.size(0), self.num_heads, self.head_dim + + # compute query, key, value + q = self.norm_q(self.to_q(x)[0]).view(b, -1, n, d) + k = self.norm_k(self.to_k(context)[0]).view(b, -1, n, d) + v = self.to_v(context)[0].view(b, -1, n, d) + k_img = self.norm_added_k(self.add_k_proj(context_img)[0]).view(b, -1, n, d) + v_img = self.add_v_proj(context_img)[0].view(b, -1, n, d) + img_x = self.attn(q, k_img, v_img) + # compute attention + x = self.attn(q, k, v) + + # output + x = x.flatten(2) + img_x = img_x.flatten(2) + x = x + img_x + x, _ = self.to_out(x) + return x + + +class WanTransformerBlock(nn.Module): + + def __init__( + self, + dim: int, + ffn_dim: int, + num_heads: int, + qk_norm: str = "rms_norm_across_heads", + cross_attn_norm: bool = False, + eps: float = 1e-6, + added_kv_proj_dim: int | None = None, + supported_attention_backends: set[AttentionBackendEnum] | None = None, + prefix: str = "", + ): + super().__init__() + + # 1. Self-attention + self.norm1 = FP32LayerNorm(dim, eps, elementwise_affine=False) + self.to_q = ReplicatedLinear(dim, dim, bias=True) + self.to_k = ReplicatedLinear(dim, dim, bias=True) + self.to_v = ReplicatedLinear(dim, dim, bias=True) + + self.to_out = ReplicatedLinear(dim, dim, bias=True) + self.attn1 = USPAttention( + num_heads=num_heads, + head_size=dim // num_heads, + causal=False, + supported_attention_backends=supported_attention_backends, + prefix=f"{prefix}.attn1", + ) + + self.hidden_dim = dim + self.num_attention_heads = num_heads + dim_head = dim // num_heads + if qk_norm == "rms_norm": + self.norm_q = RMSNorm(dim_head, eps=eps) + self.norm_k = RMSNorm(dim_head, eps=eps) + elif qk_norm == "rms_norm_across_heads": + # LTX applies qk norm across all heads + self.norm_q = RMSNorm(dim, eps=eps) + self.norm_k = RMSNorm(dim, eps=eps) + else: + logger.error("QK Norm type not supported") + raise Exception + assert cross_attn_norm is True + self.self_attn_residual_norm = ScaleResidualLayerNormScaleShift( + dim, + norm_type="layer", + eps=eps, + elementwise_affine=True, + dtype=torch.float32, + compute_dtype=torch.float32, + ) + + # 2. Cross-attention + if added_kv_proj_dim is not None: + # I2V + self.attn2 = WanI2VCrossAttention( + dim, + num_heads, + qk_norm=qk_norm, + eps=eps, + supported_attention_backends=supported_attention_backends, + ) + else: + # T2V + self.attn2 = WanT2VCrossAttention( + dim, + num_heads, + qk_norm=qk_norm, + eps=eps, + supported_attention_backends=supported_attention_backends, + ) + self.cross_attn_residual_norm = ScaleResidualLayerNormScaleShift( + dim, + norm_type="layer", + eps=eps, + elementwise_affine=False, + dtype=torch.float32, + compute_dtype=torch.float32, + ) + + # 3. Feed-forward + self.ffn = MLP(dim, ffn_dim, act_type="gelu_pytorch_tanh") + self.mlp_residual = ScaleResidual() + + self.scale_shift_table = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + temb: torch.Tensor, + freqs_cis: tuple[torch.Tensor, torch.Tensor], + ) -> torch.Tensor: + if hidden_states.dim() == 4: + hidden_states = hidden_states.squeeze(1) + bs, seq_length, _ = hidden_states.shape + orig_dtype = hidden_states.dtype + if temb.dim() == 4: + # temb: batch_size, seq_len, 6, inner_dim (wan2.2 ti2v) + shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa = ( + self.scale_shift_table.unsqueeze(0) + temb.float() + ).chunk(6, dim=2) + # batch_size, seq_len, 1, inner_dim + shift_msa = shift_msa.squeeze(2) + scale_msa = scale_msa.squeeze(2) + gate_msa = gate_msa.squeeze(2) + c_shift_msa = c_shift_msa.squeeze(2) + c_scale_msa = c_scale_msa.squeeze(2) + c_gate_msa = c_gate_msa.squeeze(2) + else: + # temb: batch_size, 6, inner_dim (wan2.1/wan2.2 14B) + e = self.scale_shift_table + temb.float() + shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa = ( + e.chunk(6, dim=1) + ) + + assert shift_msa.dtype == torch.float32 + + # 1. Self-attention + norm1 = self.norm1(hidden_states.float()) + norm_hidden_states = (norm1 * (1 + scale_msa) + shift_msa).to(orig_dtype) + query, _ = self.to_q(norm_hidden_states) + key, _ = self.to_k(norm_hidden_states) + value, _ = self.to_v(norm_hidden_states) + + if self.norm_q is not None: + query = self.norm_q(query) + if self.norm_k is not None: + key = self.norm_k(key) + + query = query.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) + key = key.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) + value = value.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) + + # Apply rotary embeddings + cos, sin = freqs_cis + query, key = _apply_rotary_emb( + query, cos, sin, is_neox_style=False + ), _apply_rotary_emb(key, cos, sin, is_neox_style=False) + attn_output = self.attn1(query, key, value) + attn_output = attn_output.flatten(2) + attn_output, _ = self.to_out(attn_output) + attn_output = attn_output.squeeze(1) + + null_shift = null_scale = torch.zeros( + (1,), device=hidden_states.device, dtype=hidden_states.dtype + ) + norm_hidden_states, hidden_states = self.self_attn_residual_norm( + hidden_states, attn_output, gate_msa, null_shift, null_scale + ) + norm_hidden_states, hidden_states = norm_hidden_states.to( + orig_dtype + ), hidden_states.to(orig_dtype) + + # 2. Cross-attention + attn_output = self.attn2( + norm_hidden_states, context=encoder_hidden_states, context_lens=None + ) + norm_hidden_states, hidden_states = self.cross_attn_residual_norm( + hidden_states, attn_output, 1, c_shift_msa, c_scale_msa + ) + norm_hidden_states, hidden_states = norm_hidden_states.to( + orig_dtype + ), hidden_states.to(orig_dtype) + + # 3. Feed-forward + ff_output = self.ffn(norm_hidden_states) + hidden_states = self.mlp_residual(hidden_states, ff_output, c_gate_msa) + hidden_states = hidden_states.to(orig_dtype) + + return hidden_states + + +class WanTransformerBlock_VSA(nn.Module): + + def __init__( + self, + dim: int, + ffn_dim: int, + num_heads: int, + qk_norm: str = "rms_norm_across_heads", + cross_attn_norm: bool = False, + eps: float = 1e-6, + added_kv_proj_dim: int | None = None, + supported_attention_backends: set[AttentionBackendEnum] | None = None, + prefix: str = "", + ): + super().__init__() + + # 1. Self-attention + self.norm1 = FP32LayerNorm(dim, eps, elementwise_affine=False) + self.to_q = ReplicatedLinear(dim, dim, bias=True) + self.to_k = ReplicatedLinear(dim, dim, bias=True) + self.to_v = ReplicatedLinear(dim, dim, bias=True) + self.to_gate_compress = ReplicatedLinear(dim, dim, bias=True) + + self.to_out = ReplicatedLinear(dim, dim, bias=True) + self.attn1 = UlyssesAttention_VSA( + num_heads=num_heads, + head_size=dim // num_heads, + causal=False, + supported_attention_backends=supported_attention_backends, + prefix=f"{prefix}.attn1", + ) + self.hidden_dim = dim + self.num_attention_heads = num_heads + dim_head = dim // num_heads + if qk_norm == "rms_norm": + self.norm_q = RMSNorm(dim_head, eps=eps) + self.norm_k = RMSNorm(dim_head, eps=eps) + elif qk_norm == "rms_norm_across_heads": + # LTX applies qk norm across all heads + self.norm_q = RMSNorm(dim, eps=eps) + self.norm_k = RMSNorm(dim, eps=eps) + else: + logger.error("QK Norm type not supported") + raise Exception + assert cross_attn_norm is True + self.self_attn_residual_norm = ScaleResidualLayerNormScaleShift( + dim, + norm_type="layer", + eps=eps, + elementwise_affine=True, + dtype=torch.float32, + compute_dtype=torch.float32, + ) + + if AttentionBackendEnum.VIDEO_SPARSE_ATTN in supported_attention_backends: + supported_attention_backends.remove(AttentionBackendEnum.VIDEO_SPARSE_ATTN) + # 2. Cross-attention + if added_kv_proj_dim is not None: + # I2V + self.attn2 = WanI2VCrossAttention( + dim, + num_heads, + qk_norm=qk_norm, + eps=eps, + supported_attention_backends=supported_attention_backends, + ) + else: + # T2V + self.attn2 = WanT2VCrossAttention( + dim, + num_heads, + qk_norm=qk_norm, + eps=eps, + supported_attention_backends=supported_attention_backends, + ) + self.cross_attn_residual_norm = ScaleResidualLayerNormScaleShift( + dim, + norm_type="layer", + eps=eps, + elementwise_affine=False, + dtype=torch.float32, + compute_dtype=torch.float32, + ) + + # 3. Feed-forward + self.ffn = MLP(dim, ffn_dim, act_type="gelu_pytorch_tanh") + self.mlp_residual = ScaleResidual() + + self.scale_shift_table = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + temb: torch.Tensor, + freqs_cis: tuple[torch.Tensor, torch.Tensor], + ) -> torch.Tensor: + if hidden_states.dim() == 4: + hidden_states = hidden_states.squeeze(1) + bs, seq_length, _ = hidden_states.shape + orig_dtype = hidden_states.dtype + # assert orig_dtype != torch.float32 + e = self.scale_shift_table + temb.float() + shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa = e.chunk( + 6, dim=1 + ) + assert shift_msa.dtype == torch.float32 + + # 1. Self-attention + norm_hidden_states = ( + self.norm1(hidden_states.float()) * (1 + scale_msa) + shift_msa + ).to(orig_dtype) + query, _ = self.to_q(norm_hidden_states) + key, _ = self.to_k(norm_hidden_states) + value, _ = self.to_v(norm_hidden_states) + gate_compress, _ = self.to_gate_compress(norm_hidden_states) + + if self.norm_q is not None: + query = self.norm_q(query) + if self.norm_k is not None: + key = self.norm_k(key) + + query = query.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) + key = key.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) + value = value.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) + gate_compress = gate_compress.squeeze(1).unflatten( + 2, (self.num_attention_heads, -1) + ) + + # Apply rotary embeddings + cos, sin = freqs_cis + query, key = _apply_rotary_emb( + query, cos, sin, is_neox_style=False + ), _apply_rotary_emb(key, cos, sin, is_neox_style=False) + + attn_output = self.attn1(query, key, value, gate_compress=gate_compress) + attn_output = attn_output.flatten(2) + attn_output, _ = self.to_out(attn_output) + attn_output = attn_output.squeeze(1) + + null_shift = null_scale = torch.zeros((1,), device=hidden_states.device) + norm_hidden_states, hidden_states = self.self_attn_residual_norm( + hidden_states, attn_output, gate_msa, null_shift, null_scale + ) + norm_hidden_states, hidden_states = norm_hidden_states.to( + orig_dtype + ), hidden_states.to(orig_dtype) + + # 2. Cross-attention + attn_output = self.attn2( + norm_hidden_states, context=encoder_hidden_states, context_lens=None + ) + norm_hidden_states, hidden_states = self.cross_attn_residual_norm( + hidden_states, attn_output, 1, c_shift_msa, c_scale_msa + ) + norm_hidden_states, hidden_states = norm_hidden_states.to( + orig_dtype + ), hidden_states.to(orig_dtype) + + # 3. Feed-forward + ff_output = self.ffn(norm_hidden_states) + hidden_states = self.mlp_residual(hidden_states, ff_output, c_gate_msa) + hidden_states = hidden_states.to(orig_dtype) + + return hidden_states + + +class WanTransformer3DModel(CachableDiT): + _fsdp_shard_conditions = WanVideoConfig()._fsdp_shard_conditions + _compile_conditions = WanVideoConfig()._compile_conditions + _supported_attention_backends = WanVideoConfig()._supported_attention_backends + param_names_mapping = WanVideoConfig().param_names_mapping + reverse_param_names_mapping = WanVideoConfig().reverse_param_names_mapping + lora_param_names_mapping = WanVideoConfig().lora_param_names_mapping + + def __init__(self, config: WanVideoConfig, hf_config: dict[str, Any]) -> None: + super().__init__(config=config, hf_config=hf_config) + + inner_dim = config.num_attention_heads * config.attention_head_dim + self.hidden_size = config.hidden_size + self.num_attention_heads = config.num_attention_heads + self.in_channels = config.in_channels + self.out_channels = config.out_channels + self.num_channels_latents = config.num_channels_latents + self.patch_size = config.patch_size + self.text_len = config.text_len + + # 1. Patch & position embedding + self.patch_embedding = PatchEmbed( + in_chans=config.in_channels, + embed_dim=inner_dim, + patch_size=config.patch_size, + flatten=False, + ) + + # 2. Condition embeddings + self.condition_embedder = WanTimeTextImageEmbedding( + dim=inner_dim, + time_freq_dim=config.freq_dim, + text_embed_dim=config.text_dim, + image_embed_dim=config.image_dim, + ) + + # 3. Transformer blocks + attn_backend = get_global_server_args().attention_backend + transformer_block = ( + WanTransformerBlock_VSA + if (attn_backend and attn_backend.lower() == "video_sparse_attn") + else WanTransformerBlock + ) + self.blocks = nn.ModuleList( + [ + transformer_block( + inner_dim, + config.ffn_dim, + config.num_attention_heads, + config.qk_norm, + config.cross_attn_norm, + config.eps, + config.added_kv_proj_dim, + self._supported_attention_backends + | {AttentionBackendEnum.VIDEO_SPARSE_ATTN}, + prefix=f"{config.prefix}.blocks.{i}", + ) + for i in range(config.num_layers) + ] + ) + + # 4. Output norm & projection + self.norm_out = LayerNormScaleShift( + inner_dim, + norm_type="layer", + eps=config.eps, + elementwise_affine=False, + dtype=torch.float32, + compute_dtype=torch.float32, + ) + self.proj_out = nn.Linear( + inner_dim, config.out_channels * math.prod(config.patch_size) + ) + self.scale_shift_table = nn.Parameter( + torch.randn(1, 2, inner_dim) / inner_dim**0.5 + ) + + # For type checking + self.previous_e0_even = None + self.previous_e0_odd = None + self.previous_residual_even = None + self.previous_residual_odd = None + self.is_even = True + self.should_calc_even = True + self.should_calc_odd = True + self.accumulated_rel_l1_distance_even = 0 + self.accumulated_rel_l1_distance_odd = 0 + self.cnt = 0 + self.__post_init__() + + # misc + self.sp_size = get_sp_world_size() + + # Get rotary embeddings + d = self.hidden_size // self.num_attention_heads + self.rope_dim_list = [d - 4 * (d // 6), 2 * (d // 6), 2 * (d // 6)] + + self.rotary_emb = NDRotaryEmbedding( + rope_dim_list=self.rope_dim_list, + rope_theta=10000, + dtype=torch.float32 if current_platform.is_mps() else torch.float64, + ) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor | list[torch.Tensor], + timestep: torch.LongTensor, + encoder_hidden_states_image: torch.Tensor | list[torch.Tensor] | None = None, + guidance=None, + **kwargs, + ) -> torch.Tensor: + forward_batch = get_forward_context().forward_batch + enable_teacache = forward_batch is not None and forward_batch.enable_teacache + + orig_dtype = hidden_states.dtype + if not isinstance(encoder_hidden_states, torch.Tensor): + encoder_hidden_states = encoder_hidden_states[0] + if ( + isinstance(encoder_hidden_states_image, list) + and len(encoder_hidden_states_image) > 0 + ): + encoder_hidden_states_image = encoder_hidden_states_image[0] + else: + encoder_hidden_states_image = None + + batch_size, num_channels, num_frames, height, width = hidden_states.shape + + p_t, p_h, p_w = self.patch_size + post_patch_num_frames = num_frames // p_t + post_patch_height = height // p_h + post_patch_width = width // p_w + + # The rotary embedding layer correctly handles SP offsets internally. + freqs_cos, freqs_sin = self.rotary_emb.forward_from_grid( + ( + post_patch_num_frames * self.sp_size, + post_patch_height, + post_patch_width, + ), + shard_dim=0, + start_frame=0, + device=hidden_states.device, + ) + assert freqs_cos.dtype == torch.float32 + assert freqs_cos.device == hidden_states.device + freqs_cis = ( + (freqs_cos.float(), freqs_sin.float()) if freqs_cos is not None else None + ) + + hidden_states = self.patch_embedding(hidden_states) + hidden_states = hidden_states.flatten(2).transpose(1, 2) + # timestep shape: batch_size, or batch_size, seq_len (wan 2.2 ti2v) + if timestep.dim() == 2: + # ti2v + ts_seq_len = timestep.shape[1] + timestep = timestep.flatten() # batch_size * seq_len + else: + ts_seq_len = None + + temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image = ( + self.condition_embedder( + timestep, + encoder_hidden_states, + encoder_hidden_states_image, + timestep_seq_len=ts_seq_len, + ) + ) + if ts_seq_len is not None: + # batch_size, seq_len, 6, inner_dim + timestep_proj = timestep_proj.unflatten(2, (6, -1)) + else: + # batch_size, 6, inner_dim + timestep_proj = timestep_proj.unflatten(1, (6, -1)) + + if encoder_hidden_states_image is not None: + encoder_hidden_states = torch.concat( + [encoder_hidden_states_image, encoder_hidden_states], dim=1 + ) + + encoder_hidden_states = ( + encoder_hidden_states.to(orig_dtype) + if current_platform.is_mps() + else encoder_hidden_states + ) # cast to orig_dtype for MPS + + assert encoder_hidden_states.dtype == orig_dtype + + # 4. Transformer blocks + # if caching is enabled, we might be able to skip the forward pass + should_skip_forward = self.should_skip_forward_for_cached_states( + timestep_proj=timestep_proj, temb=temb + ) + + if should_skip_forward: + hidden_states = self.retrieve_cached_states(hidden_states) + else: + # if teacache is enabled, we need to cache the original hidden states + if enable_teacache: + original_hidden_states = hidden_states.clone() + + for block in self.blocks: + hidden_states = block( + hidden_states, encoder_hidden_states, timestep_proj, freqs_cis + ) + # if teacache is enabled, we need to cache the original hidden states + if enable_teacache: + self.maybe_cache_states(hidden_states, original_hidden_states) + # 5. Output norm, projection & unpatchify + if temb.dim() == 3: + # batch_size, seq_len, inner_dim (wan 2.2 ti2v) + shift, scale = ( + self.scale_shift_table.unsqueeze(0) + temb.unsqueeze(2) + ).chunk(2, dim=2) + shift = shift.squeeze(2) + scale = scale.squeeze(2) + else: + # batch_size, inner_dim + shift, scale = (self.scale_shift_table + temb.unsqueeze(1)).chunk(2, dim=1) + + hidden_states = self.norm_out(hidden_states, shift, scale) + hidden_states = self.proj_out(hidden_states) + + hidden_states = hidden_states.reshape( + batch_size, + post_patch_num_frames, + post_patch_height, + post_patch_width, + p_t, + p_h, + p_w, + -1, + ) + hidden_states = hidden_states.permute(0, 7, 1, 4, 2, 5, 3, 6) + output = hidden_states.flatten(6, 7).flatten(4, 5).flatten(2, 3) + + return output + + def maybe_cache_states( + self, hidden_states: torch.Tensor, original_hidden_states: torch.Tensor + ) -> None: + if self.is_even: + self.previous_residual_even = ( + hidden_states.squeeze(0) - original_hidden_states + ) + else: + self.previous_residual_odd = ( + hidden_states.squeeze(0) - original_hidden_states + ) + + def should_skip_forward_for_cached_states(self, **kwargs) -> bool: + + forward_context = get_forward_context() + forward_batch = forward_context.forward_batch + if forward_batch is None or not forward_batch.enable_teacache: + return False + teacache_params = forward_batch.teacache_params + assert teacache_params is not None, "teacache_params is not initialized" + assert isinstance( + teacache_params, WanTeaCacheParams + ), "teacache_params is not a WanTeaCacheParams" + current_timestep = forward_context.current_timestep + num_inference_steps = forward_batch.num_inference_steps + + # initialize the coefficients, cutoff_steps, and ret_steps + coefficients = teacache_params.coefficients + use_ret_steps = teacache_params.use_ret_steps + cutoff_steps = teacache_params.get_cutoff_steps(num_inference_steps) + ret_steps = teacache_params.ret_steps + teacache_thresh = teacache_params.teacache_thresh + + if current_timestep == 0: + self.cnt = 0 + + timestep_proj = kwargs["timestep_proj"] + temb = kwargs["temb"] + modulated_inp = timestep_proj if use_ret_steps else temb + + if self.cnt % 2 == 0: # even -> condition + self.is_even = True + if self.cnt < ret_steps or self.cnt >= cutoff_steps: + self.should_calc_even = True + self.accumulated_rel_l1_distance_even = 0 + else: + assert ( + self.previous_e0_even is not None + ), "previous_e0_even is not initialized" + assert ( + self.accumulated_rel_l1_distance_even is not None + ), "accumulated_rel_l1_distance_even is not initialized" + rescale_func = np.poly1d(coefficients) + self.accumulated_rel_l1_distance_even += rescale_func( + ( + (modulated_inp - self.previous_e0_even).abs().mean() + / self.previous_e0_even.abs().mean() + ) + .cpu() + .item() + ) + if self.accumulated_rel_l1_distance_even < teacache_thresh: + self.should_calc_even = False + else: + self.should_calc_even = True + self.accumulated_rel_l1_distance_even = 0 + self.previous_e0_even = modulated_inp.clone() + + else: # odd -> unconditon + self.is_even = False + if self.cnt < ret_steps or self.cnt >= cutoff_steps: + self.should_calc_odd = True + self.accumulated_rel_l1_distance_odd = 0 + else: + assert ( + self.previous_e0_odd is not None + ), "previous_e0_odd is not initialized" + assert ( + self.accumulated_rel_l1_distance_odd is not None + ), "accumulated_rel_l1_distance_odd is not initialized" + rescale_func = np.poly1d(coefficients) + self.accumulated_rel_l1_distance_odd += rescale_func( + ( + (modulated_inp - self.previous_e0_odd).abs().mean() + / self.previous_e0_odd.abs().mean() + ) + .cpu() + .item() + ) + if self.accumulated_rel_l1_distance_odd < teacache_thresh: + self.should_calc_odd = False + else: + self.should_calc_odd = True + self.accumulated_rel_l1_distance_odd = 0 + self.previous_e0_odd = modulated_inp.clone() + self.cnt += 1 + should_skip_forward = False + if self.is_even: + if not self.should_calc_even: + should_skip_forward = True + else: + if not self.should_calc_odd: + should_skip_forward = True + + return should_skip_forward + + def retrieve_cached_states(self, hidden_states: torch.Tensor) -> torch.Tensor: + if self.is_even: + return hidden_states + self.previous_residual_even + else: + return hidden_states + self.previous_residual_odd + + +EntryClass = WanTransformer3DModel diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/zimage.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/zimage.py new file mode 100644 index 00000000..bdc38d52 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/dits/zimage.py @@ -0,0 +1,605 @@ +import math +from typing import Any, List, Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from sglang.multimodal_gen.configs.models.dits.zimage import ZImageDitConfig +from sglang.multimodal_gen.runtime.layers.attention import USPAttention +from sglang.multimodal_gen.runtime.layers.layernorm import RMSNorm +from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear +from sglang.multimodal_gen.runtime.layers.rotary_embedding import _apply_rotary_emb +from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + +ADALN_EMBED_DIM = 256 +SEQ_MULTI_OF = 32 + + +class SelectFirstElement(nn.Module): + def __init__(self): + super().__init__() + + def forward(self, x): + return x[0] + + +class TimestepEmbedder(nn.Module): + def __init__(self, out_size, mid_size=None, frequency_embedding_size=256): + super().__init__() + if mid_size is None: + mid_size = out_size + + self.mlp = nn.ModuleList( + [ + ReplicatedLinear(frequency_embedding_size, mid_size, bias=True), + nn.SiLU(), + ReplicatedLinear(mid_size, out_size, bias=True), + ] + ) + + self.frequency_embedding_size = frequency_embedding_size + + @staticmethod + def timestep_embedding(t, dim, max_period=10000): + with torch.amp.autocast("cuda", enabled=False): + half = dim // 2 + freqs = torch.exp( + -math.log(max_period) + * torch.arange(start=0, end=half, dtype=torch.float32, device=t.device) + / half + ) + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat( + [embedding, torch.zeros_like(embedding[:, :1])], dim=-1 + ) + return embedding + + def forward(self, t): + t_freq = self.timestep_embedding(t, self.frequency_embedding_size).to( + self.mlp[0].weight.dtype + ) + t_emb, _ = self.mlp[0](t_freq) + t_emb = self.mlp[1](t_emb) + t_emb, _ = self.mlp[2](t_emb) + return t_emb + + +class FeedForward(nn.Module): + def __init__(self, dim: int, hidden_dim: int): + super().__init__() + self.w1 = ReplicatedLinear(dim, hidden_dim, bias=False) + self.w2 = ReplicatedLinear(hidden_dim, dim, bias=False) + self.w3 = ReplicatedLinear(dim, hidden_dim, bias=False) + + def _forward_silu_gating(self, x1, x3): + return F.silu(x1) * x3 + + def forward(self, x): + x1, _ = self.w1(x) + x3, _ = self.w3(x) + out, _ = self.w2(self._forward_silu_gating(x1, x3)) + return out + + +class ZImageAttention(nn.Module): + def __init__( + self, + dim: int, + num_heads: int, + num_kv_heads: int, + qk_norm: bool = True, + eps: float = 1e-6, + ) -> None: + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.num_kv_heads = num_kv_heads + self.head_dim = dim // num_heads + self.qk_norm = qk_norm + + self.to_q = ReplicatedLinear(dim, dim, bias=False) + self.to_k = ReplicatedLinear(dim, self.head_dim * num_kv_heads, bias=False) + self.to_v = ReplicatedLinear(dim, self.head_dim * num_kv_heads, bias=False) + + if self.qk_norm: + self.norm_q = RMSNorm(self.head_dim, eps=eps) + self.norm_k = RMSNorm(self.head_dim, eps=eps) + else: + self.norm_q = None + self.norm_k = None + + self.to_out = nn.ModuleList([ReplicatedLinear(dim, dim, bias=False)]) + + self.attn = USPAttention( + num_heads=num_heads, + head_size=self.head_dim, + num_kv_heads=num_kv_heads, + dropout_rate=0, + softmax_scale=None, + causal=False, + supported_attention_backends={ + AttentionBackendEnum.FA, + AttentionBackendEnum.TORCH_SDPA, + }, + ) + + def forward( + self, + hidden_states: torch.Tensor, + freqs_cis: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + ): + q, _ = self.to_q(hidden_states) + k, _ = self.to_k(hidden_states) + v, _ = self.to_v(hidden_states) + + q = q.view(*q.shape[:-1], self.num_heads, self.head_dim) + k = k.view(*k.shape[:-1], self.num_kv_heads, self.head_dim) + v = v.view(*v.shape[:-1], self.num_kv_heads, self.head_dim) + + if self.norm_q is not None: + q = self.norm_q(q) + if self.norm_k is not None: + k = self.norm_k(k) + + # Apply RoPE + def apply_rotary_emb( + x_in: torch.Tensor, + freqs_cis: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + cos, sin = freqs_cis + x_out = _apply_rotary_emb(x_in, cos, sin, is_neox_style=False) + return x_out + + if freqs_cis is not None: + q = apply_rotary_emb(q, freqs_cis) + k = apply_rotary_emb(k, freqs_cis) + + hidden_states = self.attn(q, k, v) + hidden_states = hidden_states.flatten(2) + + hidden_states, _ = self.to_out[0](hidden_states) + + return hidden_states + + +class ZImageTransformerBlock(nn.Module): + def __init__( + self, + layer_id: int, + dim: int, + n_heads: int, + n_kv_heads: int, + norm_eps: float, + qk_norm: bool, + modulation=True, + ): + super().__init__() + self.dim = dim + self.head_dim = dim // n_heads + self.layer_id = layer_id + self.modulation = modulation + + self.attention = ZImageAttention( + dim=dim, + num_heads=n_heads, + num_kv_heads=n_kv_heads, + qk_norm=qk_norm, + eps=1e-5, + ) + + self.feed_forward = FeedForward(dim=dim, hidden_dim=int(dim / 3 * 8)) + + self.attention_norm1 = RMSNorm(dim, eps=norm_eps) + self.ffn_norm1 = RMSNorm(dim, eps=norm_eps) + + self.attention_norm2 = RMSNorm(dim, eps=norm_eps) + self.ffn_norm2 = RMSNorm(dim, eps=norm_eps) + + if modulation: + self.adaLN_modulation = nn.Sequential( + ReplicatedLinear(min(dim, ADALN_EMBED_DIM), 4 * dim, bias=True) + ) + + def forward( + self, + x: torch.Tensor, + freqs_cis: Tuple[torch.Tensor, torch.Tensor], + adaln_input: Optional[torch.Tensor] = None, + ): + if self.modulation: + assert adaln_input is not None + scale_msa_gate, _ = self.adaLN_modulation(adaln_input) + scale_msa, gate_msa, scale_mlp, gate_mlp = scale_msa_gate.unsqueeze( + 1 + ).chunk(4, dim=2) + gate_msa, gate_mlp = gate_msa.tanh(), gate_mlp.tanh() + scale_msa, scale_mlp = 1.0 + scale_msa, 1.0 + scale_mlp + + # Attention block + attn_out = self.attention( + self.attention_norm1(x) * scale_msa, + freqs_cis=freqs_cis, + ) + x = x + gate_msa * self.attention_norm2(attn_out) + + # FFN block + x = x + gate_mlp * self.ffn_norm2( + self.feed_forward( + self.ffn_norm1(x) * scale_mlp, + ) + ) + else: + # Attention block + attn_out = self.attention( + self.attention_norm1(x), + freqs_cis=freqs_cis, + ) + x = x + self.attention_norm2(attn_out) + + # FFN block + x = x + self.ffn_norm2( + self.feed_forward( + self.ffn_norm1(x), + ) + ) + + return x + + +class FinalLayer(nn.Module): + def __init__(self, hidden_size, out_channels): + super().__init__() + self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.linear = ReplicatedLinear(hidden_size, out_channels, bias=True) + + self.act = nn.SiLU() + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), + ReplicatedLinear(min(hidden_size, ADALN_EMBED_DIM), hidden_size, bias=True), + ) + + def forward(self, x, c): + scale, _ = self.adaLN_modulation(c) + scale = 1.0 + scale + x = self.norm_final(x) * scale.unsqueeze(1) + x, _ = self.linear(x) + return x + + +class RopeEmbedder: + def __init__( + self, + theta: float = 256.0, + axes_dims: List[int] = (16, 56, 56), + axes_lens: List[int] = (64, 128, 128), + ): + self.theta = theta + self.axes_dims = axes_dims + self.axes_lens = axes_lens + assert len(axes_dims) == len( + axes_lens + ), "axes_dims and axes_lens must have the same length" + + self.cos_cached = None + self.sin_cached = None + + @staticmethod + def precompute_freqs(dim: List[int], end: List[int], theta: float = 256.0): + with torch.device("cpu"): + cos_list = [] + sin_list = [] + for i, (d, e) in enumerate(zip(dim, end)): + freqs = 1.0 / ( + theta + ** (torch.arange(0, d, 2, dtype=torch.float64, device="cpu") / d) + ) + timestep = torch.arange(e, device=freqs.device, dtype=torch.float64) + freqs = torch.outer(timestep, freqs).float() + + cos_list.append(torch.cos(freqs)) + sin_list.append(torch.sin(freqs)) + + return cos_list, sin_list + + def __call__(self, ids: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Args: + ids: [batch, len(axes_dims)] or [seq_len, len(axes_dims)] + Returns: + cos: [batch/seq, head_dim // 2] + sin: [batch/seq, head_dim // 2] + """ + assert ids.ndim == 2 + assert ids.shape[-1] == len(self.axes_dims) + device = ids.device + + if self.cos_cached is None: + self.cos_cached, self.sin_cached = self.precompute_freqs( + self.axes_dims, self.axes_lens, theta=self.theta + ) + self.cos_cached = [c.to(device) for c in self.cos_cached] + self.sin_cached = [s.to(device) for s in self.sin_cached] + else: + if self.cos_cached[0].device != device: + self.cos_cached = [c.to(device) for c in self.cos_cached] + self.sin_cached = [s.to(device) for s in self.sin_cached] + + cos_out = [] + sin_out = [] + for i in range(len(self.axes_dims)): + index = ids[:, i] + cos_out.append(self.cos_cached[i][index]) + sin_out.append(self.sin_cached[i][index]) + + return torch.cat(cos_out, dim=-1), torch.cat(sin_out, dim=-1) + + +class ZImageTransformer2DModel(CachableDiT): + _supports_gradient_checkpointing = True + _no_split_modules = ["ZImageTransformerBlock"] + + def __init__( + self, + config: ZImageDitConfig, + hf_config: dict[str, Any], + ) -> None: + super().__init__(config=config, hf_config=hf_config) + + self.config_data = config # Store config + arch_config = config.arch_config + + self.in_channels = arch_config.in_channels + self.out_channels = arch_config.out_channels + self.all_patch_size = arch_config.all_patch_size + self.all_f_patch_size = arch_config.all_f_patch_size + self.dim = arch_config.dim + self.n_heads = arch_config.num_attention_heads + + self.rope_theta = arch_config.rope_theta + self.t_scale = arch_config.t_scale + self.gradient_checkpointing = False + + assert len(self.all_patch_size) == len(self.all_f_patch_size) + + all_x_embedder = {} + all_final_layer = {} + for patch_idx, (patch_size, f_patch_size) in enumerate( + zip(self.all_patch_size, self.all_f_patch_size) + ): + x_embedder = ReplicatedLinear( + f_patch_size * patch_size * patch_size * self.in_channels, + self.dim, + bias=True, + ) + all_x_embedder[f"{patch_size}-{f_patch_size}"] = x_embedder + + final_layer = FinalLayer( + self.dim, patch_size * patch_size * f_patch_size * self.out_channels + ) + all_final_layer[f"{patch_size}-{f_patch_size}"] = final_layer + + self.all_x_embedder = nn.ModuleDict(all_x_embedder) + self.all_final_layer = nn.ModuleDict(all_final_layer) + + self.noise_refiner = nn.ModuleList( + [ + ZImageTransformerBlock( + 1000 + layer_id, + self.dim, + self.n_heads, + arch_config.n_kv_heads, + arch_config.norm_eps, + arch_config.qk_norm, + modulation=True, + ) + for layer_id in range(arch_config.n_refiner_layers) + ] + ) + self.context_refiner = nn.ModuleList( + [ + ZImageTransformerBlock( + layer_id, + self.dim, + self.n_heads, + arch_config.n_kv_heads, + arch_config.norm_eps, + arch_config.qk_norm, + modulation=False, + ) + for layer_id in range(arch_config.n_refiner_layers) + ] + ) + self.t_embedder = TimestepEmbedder( + min(self.dim, ADALN_EMBED_DIM), mid_size=1024 + ) + + self.cap_embedder = nn.Sequential( + RMSNorm(arch_config.cap_feat_dim, eps=arch_config.norm_eps), + ReplicatedLinear(arch_config.cap_feat_dim, self.dim, bias=True), + ) + + self.x_pad_token = nn.Parameter(torch.empty((1, self.dim))) + self.cap_pad_token = nn.Parameter(torch.empty((1, self.dim))) + + self.layers = nn.ModuleList( + [ + ZImageTransformerBlock( + layer_id, + self.dim, + self.n_heads, + arch_config.n_kv_heads, + arch_config.norm_eps, + arch_config.qk_norm, + ) + for layer_id in range(arch_config.num_layers) + ] + ) + head_dim = self.dim // self.n_heads + assert head_dim == sum(arch_config.axes_dims) + self.axes_dims = arch_config.axes_dims + self.axes_lens = arch_config.axes_lens + + self.rotary_emb = RopeEmbedder( + theta=self.rope_theta, axes_dims=self.axes_dims, axes_lens=self.axes_lens + ) + + def unpatchify( + self, x: List[torch.Tensor], size: List[Tuple], patch_size, f_patch_size + ) -> List[torch.Tensor]: + pH = pW = patch_size + pF = f_patch_size + bsz = len(x) + assert len(size) == bsz + for i in range(bsz): + F, H, W = size[i] + ori_len = (F // pF) * (H // pH) * (W // pW) + # "f h w pf ph pw c -> c (f pf) (h ph) (w pw)" + x[i] = ( + x[i][:ori_len] + .view(F // pF, H // pH, W // pW, pF, pH, pW, self.out_channels) + .permute(6, 0, 3, 1, 4, 2, 5) + .reshape(self.out_channels, F, H, W) + ) + return x + + @staticmethod + def create_coordinate_grid(size, start=None, device=None): + if start is None: + start = (0 for _ in size) + + axes = [ + torch.arange(x0, x0 + span, dtype=torch.int32, device=device) + for x0, span in zip(start, size) + ] + grids = torch.meshgrid(axes, indexing="ij") + return torch.stack(grids, dim=-1) + + def patchify_and_embed( + self, + all_image: List[torch.Tensor], + all_cap_feats: List[torch.Tensor], + patch_size: int, + f_patch_size: int, + ): + assert len(all_image) == len(all_cap_feats) == 1 + + image = all_image[0] # C, F, H, W + cap_feat = all_cap_feats[0] # L, D + pH = pW = patch_size + pF = f_patch_size + device = image.device + + all_image_out = [] + all_image_size = [] + all_cap_feats_out = [] + + # ------------ Process Caption ------------ + cap_ori_len = cap_feat.size(0) + cap_padding_len = (-cap_ori_len) % SEQ_MULTI_OF + + # padded feature + cap_padded_feat = torch.cat( + [cap_feat, cap_feat[-1:].repeat(cap_padding_len, 1)], + dim=0, + ) + all_cap_feats_out.append(cap_padded_feat) + + # ------------ Process Image ------------ + C, F, H, W = image.size() + all_image_size.append((F, H, W)) + + F_tokens, H_tokens, W_tokens = F // pF, H // pH, W // pW + image = image.view(C, F_tokens, pF, H_tokens, pH, W_tokens, pW) + # "c f pf h ph w pw -> (f h w) (pf ph pw c)" + image = image.permute(1, 3, 5, 2, 4, 6, 0).reshape( + F_tokens * H_tokens * W_tokens, pF * pH * pW * C + ) + image_ori_len = image.size(0) + image_padding_len = (-image_ori_len) % SEQ_MULTI_OF + + # padded feature + image_padded_feat = torch.cat( + [image, image[-1:].repeat(image_padding_len, 1)], + dim=0, + ) + all_image_out.append(image_padded_feat) + + return ( + all_image_out, + all_cap_feats_out, + all_image_size, + ) + + def forward( + self, + hidden_states: List[torch.Tensor], + encoder_hidden_states: List[torch.Tensor], + timestep, + guidance=0, + patch_size=2, + f_patch_size=1, + freqs_cis=None, + **kwargs, + ): + assert patch_size in self.all_patch_size + assert f_patch_size in self.all_f_patch_size + + x = hidden_states + cap_feats = encoder_hidden_states + timestep = 1000.0 - timestep + t = timestep + bsz = 1 + device = x[0].device + t = self.t_embedder(t) + adaln_input = t.type_as(x) + ( + x, + cap_feats, + x_size, + ) = self.patchify_and_embed(x, cap_feats, patch_size, f_patch_size) + + x = torch.cat(x, dim=0) + x, _ = self.all_x_embedder[f"{patch_size}-{f_patch_size}"](x) + x_freqs_cis = freqs_cis[1] + + x = x.unsqueeze(0) + x_freqs_cis = x_freqs_cis + for layer in self.noise_refiner: + x = layer(x, x_freqs_cis, adaln_input) + + cap_feats = torch.cat(cap_feats, dim=0) + + cap_feats, _ = self.cap_embedder(cap_feats) + + cap_freqs_cis = freqs_cis[0] + + cap_feats = cap_feats.unsqueeze(0) + for layer in self.context_refiner: + cap_feats = layer(cap_feats, cap_freqs_cis) + + unified = torch.cat([x, cap_feats], dim=1) + unified_freqs_cis = ( + torch.cat([x_freqs_cis[0], cap_freqs_cis[0]], dim=0), + torch.cat([x_freqs_cis[1], cap_freqs_cis[1]], dim=0), + ) + + for layer in self.layers: + unified = layer(unified, unified_freqs_cis, adaln_input) + + unified = self.all_final_layer[f"{patch_size}-{f_patch_size}"]( + unified, adaln_input + ) + unified = list(unified.unbind(dim=0)) + x = self.unpatchify(unified, x_size, patch_size, f_patch_size) + + return -x[0] + + +EntryClass = ZImageTransformer2DModel diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/base.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/base.py new file mode 100644 index 00000000..7b3eece9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/base.py @@ -0,0 +1,71 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +from abc import ABC, abstractmethod +from dataclasses import field + +import torch +from torch import nn + +from sglang.multimodal_gen.configs.models.encoders import ( + BaseEncoderOutput, + ImageEncoderConfig, + TextEncoderConfig, +) +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum + + +class TextEncoder(nn.Module, ABC): + _fsdp_shard_conditions: list = field(default_factory=lambda: []) + _stacked_params_mapping: list[tuple[str, str, str]] = field(default_factory=list) + _supported_attention_backends: set[AttentionBackendEnum] = ( + TextEncoderConfig()._supported_attention_backends + ) + + def __init__(self, config: TextEncoderConfig) -> None: + super().__init__() + self.config = config + self._fsdp_shard_conditions = config.arch_config._fsdp_shard_conditions + self._stacked_params_mapping = config.arch_config.stacked_params_mapping + if not self.supported_attention_backends: + raise ValueError( + f"Subclass {self.__class__.__name__} must define _supported_attention_backends" + ) + + @abstractmethod + def forward( + self, + input_ids: torch.Tensor | None, + position_ids: torch.Tensor | None = None, + attention_mask: torch.Tensor | None = None, + inputs_embeds: torch.Tensor | None = None, + output_hidden_states: bool | None = None, + **kwargs, + ) -> BaseEncoderOutput: + pass + + @property + def supported_attention_backends(self) -> set[AttentionBackendEnum]: + return self._supported_attention_backends + + +class ImageEncoder(nn.Module, ABC): + _supported_attention_backends: set[AttentionBackendEnum] = ( + ImageEncoderConfig()._supported_attention_backends + ) + + def __init__(self, config: ImageEncoderConfig) -> None: + super().__init__() + self.config = config + if not self.supported_attention_backends: + raise ValueError( + f"Subclass {self.__class__.__name__} must define _supported_attention_backends" + ) + + @abstractmethod + def forward(self, pixel_values: torch.Tensor, **kwargs) -> BaseEncoderOutput: + pass + + @property + def supported_attention_backends(self) -> set[AttentionBackendEnum]: + return self._supported_attention_backends diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/bert.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/bert.py new file mode 100644 index 00000000..5a423e51 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/bert.py @@ -0,0 +1,46 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# type: ignore +import os + +import torch +import torch.nn as nn +from transformers import BertModel, BertTokenizer + + +class HunyuanClip(nn.Module): + """ + Hunyuan clip code copied from https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/hunyuandit/pipeline_hunyuandit.py + hunyuan's clip used BertModel and BertTokenizer, so we copy it. + """ + + def __init__(self, model_dir, max_length=77): + super().__init__() + + self.max_length = max_length + self.tokenizer = BertTokenizer.from_pretrained( + os.path.join(model_dir, "tokenizer") + ) + self.text_encoder = BertModel.from_pretrained( + os.path.join(model_dir, "clip_text_encoder") + ) + + @torch.no_grad + def forward(self, prompts, with_mask=True): + self.device = next(self.text_encoder.parameters()).device + text_inputs = self.tokenizer( + prompts, + padding="max_length", + max_length=self.max_length, + truncation=True, + return_attention_mask=True, + return_tensors="pt", + ) + prompt_embeds = self.text_encoder( + text_inputs.input_ids.to(self.device), + attention_mask=( + text_inputs.attention_mask.to(self.device) if with_mask else None + ), + ) + return prompt_embeds.last_hidden_state, prompt_embeds.pooler_output diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/clip.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/clip.py new file mode 100644 index 00000000..ec80e387 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/clip.py @@ -0,0 +1,700 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/model_executor/models/clip.py +# Adapted from transformers: https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/clip/modeling_clip.py +"""Minimal implementation of CLIPVisionModel intended to be only used +within a vision language model.""" +from collections.abc import Iterable +from typing import Optional + +import torch +import torch.nn as nn + +from sglang.multimodal_gen.configs.models.encoders import ( + BaseEncoderOutput, + CLIPTextConfig, + CLIPVisionConfig, +) +from sglang.multimodal_gen.runtime.distributed import divide, get_tp_world_size +from sglang.multimodal_gen.runtime.layers.activation import get_act_fn +from sglang.multimodal_gen.runtime.layers.attention import LocalAttention +from sglang.multimodal_gen.runtime.layers.linear import ( + ColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear, +) +from sglang.multimodal_gen.runtime.layers.quantization import QuantizationConfig + +# TODO: support quantization +# from vllm.model_executor.layers.quantization import QuantizationConfig +from sglang.multimodal_gen.runtime.loader.weight_utils import default_weight_loader +from sglang.multimodal_gen.runtime.models.encoders.base import ImageEncoder, TextEncoder +from sglang.multimodal_gen.runtime.models.encoders.vision import ( + resolve_visual_encoder_outputs, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +# Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/clip/modeling_clip.py#L164 # noqa +class CLIPVisionEmbeddings(nn.Module): + + def __init__(self, config: CLIPVisionConfig): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.image_size = config.image_size + self.patch_size = config.patch_size + assert self.image_size % self.patch_size == 0 + + self.class_embedding = nn.Parameter(torch.randn(self.embed_dim)) + + self.patch_embedding = nn.Conv2d( + in_channels=config.num_channels, + out_channels=self.embed_dim, + kernel_size=self.patch_size, + stride=self.patch_size, + bias=False, + ) + + self.num_patches = (self.image_size // self.patch_size) ** 2 + self.num_positions = self.num_patches + 1 + self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim) + self.register_buffer( + "position_ids", + torch.arange(self.num_positions).expand((1, -1)), + persistent=False, + ) + + def forward(self, pixel_values: torch.Tensor) -> torch.Tensor: + batch_size = pixel_values.shape[0] + target_dtype = self.patch_embedding.weight.dtype + patch_embeds = self.patch_embedding( + pixel_values.to(dtype=target_dtype) + ) # shape = [*, width, grid, grid] + patch_embeds = patch_embeds.flatten(2).transpose(1, 2) + + class_embeds = self.class_embedding.expand(batch_size, 1, -1) + embeddings = torch.cat([class_embeds, patch_embeds], dim=1) + embeddings = embeddings + self.position_embedding(self.position_ids) + + return embeddings + + +class CLIPTextEmbeddings(nn.Module): + + def __init__(self, config: CLIPTextConfig): + super().__init__() + self.config = config + embed_dim = config.hidden_size + + self.token_embedding = nn.Embedding(config.vocab_size, embed_dim) + self.position_embedding = nn.Embedding( + config.max_position_embeddings, embed_dim + ) + + # position_ids (1, len position emb) is contiguous in memory and exported when serialized + self.register_buffer( + "position_ids", + torch.arange(config.max_position_embeddings).expand((1, -1)), + persistent=False, + ) + + def forward( + self, + input_ids: torch.LongTensor | None = None, + position_ids: torch.LongTensor | None = None, + inputs_embeds: torch.FloatTensor | None = None, + ) -> torch.Tensor: + if input_ids is not None: + seq_length = input_ids.shape[-1] + elif inputs_embeds is not None: + seq_length = inputs_embeds.shape[-2] + else: + raise ValueError("Either input_ids or inputs_embeds must be provided.") + + max_position_embedding = self.position_embedding.weight.shape[0] + + if seq_length > max_position_embedding: + raise ValueError( + f"Sequence length must be less than max_position_embeddings (got `sequence length`: " + f"{seq_length} and max_position_embeddings: {max_position_embedding}" + ) + + if position_ids is None: + position_ids = self.position_ids[:, :seq_length] + + if inputs_embeds is None: + inputs_embeds = self.token_embedding(input_ids) + + position_embeddings = self.position_embedding(position_ids) + embeddings = inputs_embeds + position_embeddings + + return embeddings + + +class CLIPAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + config: CLIPVisionConfig | CLIPTextConfig, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.embed_dim // self.num_heads + if self.head_dim * self.num_heads != self.embed_dim: + raise ValueError( + "embed_dim must be divisible by num_heads " + f"(got `embed_dim`: {self.embed_dim} and `num_heads`:" + f" {self.num_heads})." + ) + self.scale = self.head_dim**-0.5 + self.dropout = config.attention_dropout + + self.qkv_proj = QKVParallelLinear( + hidden_size=self.embed_dim, + head_size=self.head_dim, + total_num_heads=self.num_heads, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) + + self.out_proj = RowParallelLinear( + input_size=self.embed_dim, + output_size=self.embed_dim, + quant_config=quant_config, + prefix=f"{prefix}.out_proj", + ) + + self.tp_size = get_tp_world_size() + self.num_heads_per_partition = divide(self.num_heads, self.tp_size) + + self.attn = LocalAttention( + self.num_heads_per_partition, + self.head_dim, + self.num_heads_per_partition, + softmax_scale=self.scale, + causal=False, + supported_attention_backends=config._supported_attention_backends, + ) + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return ( + tensor.view(bsz, seq_len, self.num_heads, self.head_dim) + .transpose(1, 2) + .contiguous() + ) + + def forward( + self, + hidden_states: torch.Tensor, + ): + """Input shape: Batch x Time x Channel""" + + qkv_states, _ = self.qkv_proj(hidden_states) + query_states, key_states, value_states = qkv_states.chunk(3, dim=-1) + # use flash_attn_func + query_states = query_states.reshape( + query_states.shape[0], + query_states.shape[1], + self.num_heads_per_partition, + self.head_dim, + ) + key_states = key_states.reshape( + key_states.shape[0], + key_states.shape[1], + self.num_heads_per_partition, + self.head_dim, + ) + value_states = value_states.reshape( + value_states.shape[0], + value_states.shape[1], + self.num_heads_per_partition, + self.head_dim, + ) + attn_output = self.attn(query_states, key_states, value_states) + + attn_output = attn_output.reshape( + attn_output.shape[0], + attn_output.shape[1], + self.num_heads_per_partition * self.head_dim, + ) + attn_output, _ = self.out_proj(attn_output) + + return attn_output, None + + +class CLIPMLP(nn.Module): + + def __init__( + self, + config: CLIPVisionConfig | CLIPTextConfig, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.config = config + self.activation_fn = get_act_fn(config.hidden_act) + self.fc1 = ColumnParallelLinear( + config.hidden_size, + config.intermediate_size, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.fc1", + ) + self.fc2 = RowParallelLinear( + config.intermediate_size, + config.hidden_size, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.fc2", + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states, _ = self.fc1(hidden_states) + hidden_states = self.activation_fn(hidden_states) + hidden_states, _ = self.fc2(hidden_states) + + return hidden_states + + +class CLIPEncoderLayer(nn.Module): + + def __init__( + self, + config: CLIPTextConfig | CLIPVisionConfig, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.self_attn = CLIPAttention( + config, + quant_config=quant_config, + prefix=f"{prefix}.self_attn", + ) + self.layer_norm1 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.mlp = CLIPMLP(config, quant_config=quant_config, prefix=f"{prefix}.mlp") + self.layer_norm2 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + residual = hidden_states + + hidden_states = self.layer_norm1(hidden_states) + hidden_states, _ = self.self_attn(hidden_states=hidden_states) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.layer_norm2(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + return hidden_states + + +class CLIPEncoder(nn.Module): + """ + Transformer encoder consisting of `config.num_hidden_layers` self + attention layers. Each layer is a [`CLIPEncoderLayer`]. + + Args: + config: CLIPConfig + """ + + def __init__( + self, + config: CLIPVisionConfig | CLIPTextConfig, + quant_config: QuantizationConfig | None = None, + num_hidden_layers_override: int | None = None, + prefix: str = "", + ) -> None: + super().__init__() + + self.config = config + + if num_hidden_layers_override is None: + num_hidden_layers = config.num_hidden_layers + else: + num_hidden_layers = num_hidden_layers_override + self.layers = nn.ModuleList( + [ + CLIPEncoderLayer( + config=config, + quant_config=quant_config, + prefix=f"{prefix}.layers.{layer_idx}", + ) + for layer_idx in range(num_hidden_layers) + ] + ) + + def forward( + self, inputs_embeds: torch.Tensor, return_all_hidden_states: bool + ) -> torch.Tensor | list[torch.Tensor]: + hidden_states_pool = [inputs_embeds] + hidden_states = inputs_embeds + + for idx, encoder_layer in enumerate(self.layers): + hidden_states = encoder_layer(hidden_states) + if return_all_hidden_states: + hidden_states_pool.append(hidden_states) + # If we have multiple feature sample layers, we return all hidden + # states in order and grab the ones we need by index. + if return_all_hidden_states: + return hidden_states_pool + return [hidden_states] + + +class CLIPTextTransformer(nn.Module): + + def __init__( + self, + config: CLIPTextConfig, + quant_config: QuantizationConfig | None = None, + num_hidden_layers_override: int | None = None, + prefix: str = "", + ): + super().__init__() + self.config = config + embed_dim = config.hidden_size + + self.embeddings = CLIPTextEmbeddings(config) + + self.encoder = CLIPEncoder( + config, + quant_config=quant_config, + num_hidden_layers_override=num_hidden_layers_override, + prefix=prefix, + ) + + self.final_layer_norm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps) + + # For `pooled_output` computation + self.eos_token_id = config.eos_token_id + + def forward( + self, + input_ids: torch.Tensor | None, + position_ids: torch.Tensor | None = None, + attention_mask: torch.Tensor | None = None, + inputs_embeds: torch.Tensor | None = None, + output_hidden_states: bool | None = None, + ) -> BaseEncoderOutput: + r""" + Returns: + + """ + output_hidden_states = ( + output_hidden_states + if output_hidden_states is not None + else self.config.output_hidden_states + ) + + if input_ids is None: + raise ValueError("You have to specify input_ids") + + input_shape = input_ids.size() + input_ids = input_ids.view(-1, input_shape[-1]) + + hidden_states = self.embeddings(input_ids=input_ids, position_ids=position_ids) + + # CLIP's text model uses causal mask, prepare it here. + # https://github.com/openai/CLIP/blob/cfcffb90e69f37bf2ff1e988237a0fbe41f33c04/clip/model.py#L324 + # causal_attention_mask = _create_4d_causal_attention_mask( + # input_shape, hidden_states.dtype, device=hidden_states.device + # ) + + # # expand attention_mask + # if attention_mask is not None and not self._use_flash_attention_2: + # raise NotImplementedError("attention_mask is not supported for CLIPTextTransformer") + # # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + # attention_mask = _prepare_4d_attention_mask(attention_mask, hidden_states.dtype) + + encoder_outputs = self.encoder( + inputs_embeds=hidden_states, + # attention_mask=attention_mask, + # causal_attention_mask=causal_attention_mask, + # output_attentions=output_attentions, + return_all_hidden_states=output_hidden_states, + # return_dict=return_dict, + ) + + last_hidden_state = encoder_outputs[-1] + last_hidden_state = self.final_layer_norm(last_hidden_state) + + if self.eos_token_id == 2: + # The `eos_token_id` was incorrect before PR #24773: Let's keep what have been done here. + # A CLIP model with such `eos_token_id` in the config can't work correctly with extra new tokens added + # ------------------------------------------------------------ + # text_embeds.shape = [batch_size, sequence_length, transformer.width] + # take features from the eot embedding (eot_token is the highest number in each sequence) + # casting to torch.int for onnx compatibility: argmax doesn't support int64 inputs with opset 14 + pooled_output = last_hidden_state[ + torch.arange( + last_hidden_state.shape[0], device=last_hidden_state.device + ), + input_ids.to(dtype=torch.int, device=last_hidden_state.device).argmax( + dim=-1 + ), + ] + else: + # The config gets updated `eos_token_id` from PR #24773 (so the use of exta new tokens is possible) + pooled_output = last_hidden_state[ + torch.arange( + last_hidden_state.shape[0], device=last_hidden_state.device + ), + # We need to get the first position of `eos_token_id` value (`pad_token_ids` might equal to `eos_token_id`) + # Note: we assume each sequence (along batch dim.) contains an `eos_token_id` (e.g. prepared by the tokenizer) + ( + input_ids.to(dtype=torch.int, device=last_hidden_state.device) + == self.eos_token_id + ) + .int() + .argmax(dim=-1), + ] + + return BaseEncoderOutput( + last_hidden_state=last_hidden_state, + pooler_output=pooled_output, + hidden_states=encoder_outputs, + # attentions=encoder_outputs.attentions, + ) + + +class CLIPTextModel(TextEncoder): + + def __init__( + self, + config: CLIPTextConfig, + ) -> None: + super().__init__(config) + self.text_model = CLIPTextTransformer( + config=config, quant_config=config.quant_config, prefix=config.prefix + ) + + def forward( + self, + input_ids: torch.Tensor | None, + position_ids: torch.Tensor | None = None, + attention_mask: torch.Tensor | None = None, + inputs_embeds: torch.Tensor | None = None, + output_hidden_states: bool | None = None, + **kwargs, + ) -> BaseEncoderOutput: + + outputs: BaseEncoderOutput = self.text_model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + output_hidden_states=output_hidden_states, + ) + return outputs + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + + # Define mapping for stacked parameters + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ] + params_dict = dict(self.named_parameters()) + loaded_params: set[str] = set() + for name, loaded_weight in weights: + # Handle q_proj, k_proj, v_proj -> qkv_proj mapping + for param_name, weight_name, shard_id in stacked_params_mapping: + if weight_name in name: + # Replace the weight name with the parameter name + model_param_name = name.replace(weight_name, param_name) + + if model_param_name in params_dict: + param = params_dict[model_param_name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + loaded_params.add(model_param_name) + break + else: + # Use default weight loader for all other parameters + if name in params_dict: + param = params_dict[name] + weight_loader = getattr( + param, "weight_loader", default_weight_loader + ) + weight_loader(param, loaded_weight) + loaded_params.add(name) + + return loaded_params + + +class CLIPVisionTransformer(nn.Module): + + def __init__( + self, + config: CLIPVisionConfig, + quant_config: QuantizationConfig | None = None, + num_hidden_layers_override: int | None = None, + require_post_norm: bool | None = None, + prefix: str = "", + ) -> None: + super().__init__() + + self.config = config + embed_dim = config.hidden_size + + self.embeddings = CLIPVisionEmbeddings(config) + + # NOTE: This typo of "layrnorm" is not fixed on purpose to match + # the original transformers code and name of the model weights. + self.pre_layrnorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps) + + self.encoder = CLIPEncoder( + config=config, + quant_config=quant_config, + num_hidden_layers_override=num_hidden_layers_override, + prefix=f"{prefix}.encoder", + ) + + num_hidden_layers = config.num_hidden_layers + if len(self.encoder.layers) > config.num_hidden_layers: + raise ValueError( + f"The original encoder only has {num_hidden_layers} " + f"layers, but you requested {len(self.encoder.layers)} layers." + ) + + # If possible, skip post_layernorm to conserve memory + if require_post_norm is None: + require_post_norm = len(self.encoder.layers) == num_hidden_layers + + if require_post_norm: + self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps) + else: + self.post_layernorm = None + + def forward( + self, + pixel_values: torch.Tensor, + output_hidden_states: Optional[bool] = None, + feature_sample_layers: list[int] | None = None, + ) -> BaseEncoderOutput: + + hidden_states = self.embeddings(pixel_values) + hidden_states = self.pre_layrnorm(hidden_states) + + return_all_hidden_states = output_hidden_states or ( + feature_sample_layers is not None + ) + + # Produces either the last layer output or all of the hidden states, + # depending on if we have feature_sample_layers or not + encoder_outputs = self.encoder( + inputs_embeds=hidden_states, + return_all_hidden_states=return_all_hidden_states, + ) + + if not return_all_hidden_states: + encoder_outputs = encoder_outputs[0] + + # Handle post-norm (if applicable) and stacks feature layers if needed + encoder_outputs = resolve_visual_encoder_outputs( + encoder_outputs, + feature_sample_layers, + self.post_layernorm, + self.config.num_hidden_layers, + ) + + if return_all_hidden_states: + return BaseEncoderOutput(hidden_states=encoder_outputs) + + return BaseEncoderOutput(last_hidden_state=encoder_outputs) + + +class CLIPVisionModel(ImageEncoder): + config_class = CLIPVisionConfig + main_input_name = "pixel_values" + packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]} + + def __init__(self, config: CLIPVisionConfig) -> None: + super().__init__(config) + self.vision_model = CLIPVisionTransformer( + config=config, + quant_config=config.quant_config, + num_hidden_layers_override=config.num_hidden_layers_override, + require_post_norm=config.require_post_norm, + prefix=f"{config.prefix}.vision_model", + ) + + def forward( + self, + pixel_values: torch.Tensor, + feature_sample_layers: list[int] | None = None, + output_hidden_states: Optional[bool] = None, + **kwargs, + ) -> BaseEncoderOutput: + base_encoder_output = self.vision_model( + pixel_values, + output_hidden_states=output_hidden_states, + feature_sample_layers=feature_sample_layers, + ) + + return base_encoder_output + + @property + def device(self): + return next(self.parameters()).device + + # (TODO) Add prefix argument for filtering out weights to be loaded + # ref: https://github.com/vllm-project/vllm/pull/7186#discussion_r1734163986 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + + params_dict = dict(self.named_parameters()) + loaded_params: set[str] = set() + layer_count = len(self.vision_model.encoder.layers) + + for name, loaded_weight in weights: + if name.startswith("visual_projection"): + continue + # post_layernorm is not needed in CLIPVisionModel + if ( + name.startswith("vision_model.post_layernorm") + and self.vision_model.post_layernorm is None + ): + continue + + # omit layers when num_hidden_layers_override is set + if name.startswith("vision_model.encoder.layers"): + layer_idx = int(name.split(".")[3]) + if layer_idx >= layer_count: + continue + + for ( + param_name, + weight_name, + shard_id, + ) in self.config.arch_config.stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(name) + return loaded_params + + +class BertModel(CLIPTextModel): + pass + + +EntryClass = [CLIPTextModel, CLIPVisionModel] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/llama.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/llama.py new file mode 100644 index 00000000..ea208f12 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/llama.py @@ -0,0 +1,459 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/model_executor/models/llama.py + +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only LLaMA model compatible with HuggingFace weights.""" +from collections.abc import Iterable +from typing import Any + +import torch +from torch import nn + +# from ..utils import (extract_layer_index) +from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput, LlamaConfig +from sglang.multimodal_gen.runtime.distributed import get_tp_world_size +from sglang.multimodal_gen.runtime.layers.activation import SiluAndMul + +# from vllm.model_executor.layers.quantization import QuantizationConfig +from sglang.multimodal_gen.runtime.layers.attention import LocalAttention +from sglang.multimodal_gen.runtime.layers.layernorm import RMSNorm +from sglang.multimodal_gen.runtime.layers.linear import ( + MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear, +) +from sglang.multimodal_gen.runtime.layers.quantization import QuantizationConfig +from sglang.multimodal_gen.runtime.layers.rotary_embedding import get_rope +from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import ( + VocabParallelEmbedding, +) +from sglang.multimodal_gen.runtime.loader.weight_utils import ( + default_weight_loader, + maybe_remap_kv_scale_name, +) +from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder + + +class LlamaMLP(nn.Module): + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: QuantizationConfig | None = None, + bias: bool = False, + prefix: str = "", + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + input_size=hidden_size, + output_sizes=[intermediate_size] * 2, + # output_size=intermediate_size, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.gate_up_proj", + ) + self.down_proj = RowParallelLinear( + input_size=intermediate_size, + output_size=hidden_size, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.down_proj", + ) + if hidden_act != "silu": + raise ValueError( + f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now." + ) + self.act_fn = SiluAndMul() + + def forward(self, x): + x, _ = self.gate_up_proj(x) + x = self.act_fn(x) + x, _ = self.down_proj(x) + return x + + +class LlamaAttention(nn.Module): + + def __init__( + self, + config: LlamaConfig, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + rope_theta: float = 10000, + rope_scaling: dict[str, Any] | None = None, + max_position_embeddings: int = 8192, + quant_config: QuantizationConfig | None = None, + bias: bool = False, + bias_o_proj: bool = False, + prefix: str = "", + ) -> None: + super().__init__() + # layer_idx = extract_layer_index(prefix) + self.hidden_size = hidden_size + tp_size = get_tp_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + # MistralConfig has an optional head_dim introduced by Mistral-Nemo + self.head_dim = getattr( + config, "head_dim", self.hidden_size // self.total_num_heads + ) + # Phi models introduced a partial_rotary_factor parameter in the config + partial_rotary_factor = getattr(config, "partial_rotary_factor", 1) + self.rotary_dim = int(partial_rotary_factor * self.head_dim) + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + self.max_position_embeddings = max_position_embeddings + + self.qkv_proj = QKVParallelLinear( + hidden_size=hidden_size, + head_size=self.head_dim, + total_num_heads=self.total_num_heads, + total_num_kv_heads=self.total_num_kv_heads, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) + + self.o_proj = RowParallelLinear( + input_size=self.total_num_heads * self.head_dim, + output_size=hidden_size, + bias=bias_o_proj, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + + is_neox_style = True + is_gguf = ( + quant_config + and hasattr(quant_config, "get_name") + and quant_config.get_name() == "gguf" + ) + if is_gguf and config.model_type == "llama": + is_neox_style = False + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.rotary_dim, + max_position=max_position_embeddings, + base=int(rope_theta), + rope_scaling=rope_scaling, + is_neox_style=is_neox_style, + ) + + self.attn = LocalAttention( + self.num_heads, + self.head_dim, + self.num_kv_heads, + softmax_scale=self.scaling, + causal=True, + supported_attention_backends=config._supported_attention_backends, + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + # attn_output = self.attn(q, k, v) + # use flash_attn_func + # TODO (Attn abstraction and backend) + # reshape q, k, v to (batch_size, seq_len, num_heads, head_dim) + batch_size = q.shape[0] + seq_len = q.shape[1] + q = q.reshape(batch_size, seq_len, self.num_heads, self.head_dim) + k = k.reshape(batch_size, seq_len, self.num_kv_heads, self.head_dim) + v = v.reshape(batch_size, seq_len, self.num_kv_heads, self.head_dim) + # import pdb; pdb.set_trace() + # attn_output = flash_attn_varlen_func(q, k, v, softmax_scale=self.scaling, causal=True) + attn_output = self.attn(q, k, v) + attn_output = attn_output.reshape( + batch_size, seq_len, self.num_heads * self.head_dim + ) + + output, _ = self.o_proj(attn_output) + return output + + +class LlamaDecoderLayer(nn.Module): + + def __init__( + self, + config: LlamaConfig, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + rope_theta = getattr(config, "rope_theta", 10000) + rope_scaling = getattr(config, "rope_scaling", None) + if rope_scaling is not None and getattr( + config, "original_max_position_embeddings", None + ): + rope_scaling["original_max_position_embeddings"] = ( + config.original_max_position_embeddings + ) + max_position_embeddings = getattr(config, "max_position_embeddings", 8192) + # Support abacusai/Smaug-72B-v0.1 with attention_bias + # Support internlm/internlm-7b with bias + attention_bias = getattr(config, "attention_bias", False) or getattr( + config, "bias", False + ) + bias_o_proj = attention_bias + # support internlm/internlm3-8b with qkv_bias + if hasattr(config, "qkv_bias"): + attention_bias = config.qkv_bias + + self.self_attn = LlamaAttention( + config=config, + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=getattr( + config, "num_key_value_heads", config.num_attention_heads + ), + rope_theta=rope_theta, + rope_scaling=rope_scaling, + max_position_embeddings=max_position_embeddings, + quant_config=quant_config, + bias=attention_bias, + bias_o_proj=bias_o_proj, + prefix=f"{prefix}.self_attn", + ) + self.mlp = LlamaMLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + bias=getattr(config, "mlp_bias", False), + prefix=f"{prefix}.mlp", + ) + self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + residual: torch.Tensor | None, + ) -> tuple[torch.Tensor, torch.Tensor]: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm(hidden_states, residual) + + hidden_states = self.self_attn(positions=positions, hidden_states=hidden_states) + + # Fully Connected + hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) + hidden_states = self.mlp(hidden_states) + return hidden_states, residual + + +class LlamaModel(TextEncoder): + + def __init__( + self, + config: LlamaConfig, + ): + super().__init__(config) + + self.config = config + self.quant_config = self.config.quant_config + if config.lora_config is not None: + max_loras = 1 + lora_vocab_size = 1 + if hasattr(config.lora_config, "max_loras"): + max_loras = config.lora_config.max_loras + if hasattr(config.lora_config, "lora_extra_vocab_size"): + lora_vocab_size = config.lora_config.lora_extra_vocab_size + lora_vocab = lora_vocab_size * max_loras + else: + lora_vocab = 0 + self.vocab_size = config.vocab_size + lora_vocab + self.org_vocab_size = config.vocab_size + + self.embed_tokens = VocabParallelEmbedding( + self.vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + quant_config=config.quant_config, + ) + + self.layers = nn.ModuleList( + [ + LlamaDecoderLayer( + config=config, + quant_config=config.quant_config, + prefix=f"{config.prefix}.layers.{i}", + ) + for i in range(config.num_hidden_layers) + ] + ) + + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) + + def forward( + self, + input_ids: torch.Tensor | None, + position_ids: torch.Tensor | None = None, + attention_mask: torch.Tensor | None = None, + inputs_embeds: torch.Tensor | None = None, + output_hidden_states: bool | None = None, + **kwargs, + ) -> BaseEncoderOutput: + output_hidden_states = ( + output_hidden_states + if output_hidden_states is not None + else self.config.output_hidden_states + ) + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.get_input_embeddings(input_ids) + residual = None + + if position_ids is None: + position_ids = torch.arange( + 0, hidden_states.shape[1], device=hidden_states.device + ).unsqueeze(0) + + all_hidden_states: tuple[Any, ...] | None = () if output_hidden_states else None + for layer in self.layers: + if all_hidden_states is not None: + # TODO + all_hidden_states += ( + (hidden_states,) + if residual is None + else (hidden_states + residual,) + ) + hidden_states, residual = layer(position_ids, hidden_states, residual) + + hidden_states, _ = self.norm(hidden_states, residual) + + # add hidden states from the last decoder layer + if all_hidden_states is not None: + all_hidden_states += (hidden_states,) + + # TODO(will): maybe unify the output format with other models and use + # our own class + output = BaseEncoderOutput( + last_hidden_state=hidden_states, + # past_key_values=past_key_values if use_cache else None, + hidden_states=all_hidden_states, + # attentions=all_self_attns, + ) + + return output + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + + params_dict = dict(self.named_parameters()) + loaded_params: set[str] = set() + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name: + # Models trained using ColossalAI may include these tensors in + # the checkpoint. Skip them. + continue + # if (self.quant_config is not None and + # (scale_name := self.quant_config.get_cache_scale(name))): + # # Loading kv cache quantization scales + # param = params_dict[scale_name] + # weight_loader = getattr(param, "weight_loader", + # default_weight_loader) + # loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else + # loaded_weight[0]) + # weight_loader(param, loaded_weight) + # loaded_params.add(scale_name) + # continue + if "scale" in name: + # Remapping the name of FP8 kv-scale. + kv_scale_name: str | None = maybe_remap_kv_scale_name(name, params_dict) + if kv_scale_name is None: + continue + else: + name = kv_scale_name + for ( + param_name, + weight_name, + shard_id, + ) in self.config.arch_config.stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + + if name not in params_dict: + continue + + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + + if name not in params_dict: + continue + + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(name) + return loaded_params + + +EntryClass = LlamaModel diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/mistral_3.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/mistral_3.py new file mode 100644 index 00000000..fef6ece6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/mistral_3.py @@ -0,0 +1,459 @@ +# coding=utf-8 +# Copyright 2025 HuggingFace Inc. team. All rights reserved. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Iterable, Optional, Union + +import torch +from torch import nn +from transformers import Cache, DynamicCache, LlavaConfig, Mistral3Config, MistralConfig +from transformers.integrations.sdpa_attention import sdpa_attention_forward +from transformers.masking_utils import create_causal_mask +from transformers.modeling_outputs import BaseModelOutputWithPast +from transformers.models.mistral3.modeling_mistral3 import ( + Mistral3CausalLMOutputWithPast, + Mistral3ModelOutputWithPast, +) +from transformers.models.mistral.modeling_mistral import ( + MistralMLP, + MistralRMSNorm, + MistralRotaryEmbedding, + apply_rotary_pos_emb, +) + +from sglang.multimodal_gen.runtime.layers.attention import USPAttention +from sglang.multimodal_gen.runtime.loader.weight_utils import default_weight_loader +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). + The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to + (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand( + batch, num_key_value_heads, n_rep, slen, head_dim + ) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +class MistralAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: MistralConfig, layer_idx: int): + super().__init__() + self.config = config + self.layer_idx = layer_idx + self.num_key_value_groups = ( + config.num_attention_heads // config.num_key_value_heads + ) + + self.head_dim = ( + getattr(config, "head_dim", None) + or config.hidden_size // config.num_attention_heads + ) + self.num_key_value_groups = ( + config.num_attention_heads // config.num_key_value_heads + ) + self.scaling = self.head_dim**-0.5 + self.attention_dropout = config.attention_dropout + self.is_causal = True + self.q_proj = nn.Linear( + config.hidden_size, config.num_attention_heads * self.head_dim, bias=False + ) + self.k_proj = nn.Linear( + config.hidden_size, config.num_key_value_heads * self.head_dim, bias=False + ) + self.v_proj = nn.Linear( + config.hidden_size, config.num_key_value_heads * self.head_dim, bias=False + ) + self.o_proj = nn.Linear( + config.num_attention_heads * self.head_dim, config.hidden_size, bias=False + ) + self.is_causal = True + self.num_heads = config.num_attention_heads + self.num_key_value_heads = config.num_key_value_heads + self.attn = USPAttention( + num_heads=self.num_heads, + head_size=self.head_dim, + dropout_rate=0, + softmax_scale=None, + causal=False, + supported_attention_backends={ + AttentionBackendEnum.FA, + AttentionBackendEnum.TORCH_SDPA, + }, + ) + + def forward( + self, + hidden_states: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor], + attention_mask: Optional[torch.Tensor], + past_key_values: Optional[Cache] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs, + ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, self.head_dim) + + query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2) + key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2) + value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) + + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb( + query_states, key_states, cos, sin + ) + + if past_key_values is not None: + # sin and cos are specific to RoPE models; cache_position needed for the static cache + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_values.update( + key_states, value_states, self.layer_idx, cache_kwargs + ) + + attention_interface = sdpa_attention_forward + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0, + scaling=self.scaling, + sliding_window=getattr( + self.config, "sliding_window", None + ), # main diff with Llama + **kwargs, + ) + + attn_output = attn_output.reshape(*input_shape, -1).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output + + +class MistralDecoderLayer(nn.Module): + def __init__(self, config: MistralConfig, layer_idx: int): + super().__init__() + self.hidden_size = config.hidden_size + self.self_attn = MistralAttention(config=config, layer_idx=layer_idx) + self.mlp = MistralMLP(config) + self.input_layernorm = MistralRMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) + self.post_attention_layernorm = MistralRMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + use_cache: Optional[bool] = False, + cache_position: Optional[torch.LongTensor] = None, + position_embeddings: Optional[ + tuple[torch.Tensor, torch.Tensor] + ] = None, # necessary, but kept here for BC + **kwargs, + ) -> torch.Tensor: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + # Self Attention + hidden_states = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + cache_position=cache_position, + position_embeddings=position_embeddings, + **kwargs, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + return hidden_states + + +class MistralModel(nn.Module): + def __init__(self, config: MistralConfig): + super().__init__() + self.config = config + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding( + config.vocab_size, config.hidden_size, self.padding_idx + ) + self.layers = nn.ModuleList( + [ + MistralDecoderLayer(config, layer_idx) + for layer_idx in range(config.num_hidden_layers) + ] + ) + self.norm = MistralRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.rotary_emb = MistralRotaryEmbedding(config=config) + self.gradient_checkpointing = False + self.config._attn_implementation = "sdpa" + + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + output_hidden_states: Optional[bool] = None, + **kwargs, + ) -> BaseModelOutputWithPast: + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError( + "You must specify exactly one of input_ids or inputs_embeds" + ) + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + + if use_cache and past_key_values is None: + past_key_values = DynamicCache(config=self.config) + + if cache_position is None: + past_seen_tokens = ( + past_key_values.get_seq_length() if past_key_values is not None else 0 + ) + cache_position = torch.arange( + past_seen_tokens, + past_seen_tokens + inputs_embeds.shape[1], + device=inputs_embeds.device, + ) + + if position_ids is None: + position_ids = cache_position.unsqueeze(0) + mask_function = create_causal_mask + causal_mask = mask_function( + config=self.config, + input_embeds=inputs_embeds, + attention_mask=attention_mask, + cache_position=cache_position, + past_key_values=past_key_values, + position_ids=position_ids, + ) + + hidden_states = inputs_embeds + position_embeddings = self.rotary_emb(hidden_states, position_ids) + + hidden_states_pool = [] + for decoder_layer in self.layers[: self.config.num_hidden_layers]: + hidden_states = decoder_layer( + hidden_states, + attention_mask=causal_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + cache_position=cache_position, + position_embeddings=position_embeddings, + **kwargs, + ) + if output_hidden_states: + hidden_states_pool.append(hidden_states) + + hidden_states = self.norm(hidden_states) + if output_hidden_states: + hidden_states_pool.append(hidden_states) + + return BaseModelOutputWithPast( + hidden_states=hidden_states_pool, + last_hidden_state=hidden_states, + past_key_values=past_key_values if use_cache else None, + ) + + +class Mistral3Model(nn.Module): + _checkpoint_conversion_mapping = {"language_model.model": "language_model"} + + def __init__(self, config: Mistral3Config): + super().__init__() + self.language_model = MistralModel(config.text_config) + self.config = config + + def get_input_embeddings(self): + return self.language_model.embed_tokens + + def set_decoder(self, decoder): + self.language_model = decoder + + def get_decoder(self): + return self.language_model + + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + output_hidden_states: Optional[bool] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidoutput_hidden_statesden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + image_sizes: Optional[torch.Tensor] = None, + **kwargs, + ) -> Union[tuple, Mistral3ModelOutputWithPast]: + output_attentions = False + output_hidden_states = True + + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError( + "You must specify exactly one of input_ids or inputs_embeds" + ) + + if inputs_embeds is None: + inputs_embeds = self.get_input_embeddings()(input_ids) + + outputs: BaseModelOutputWithPast = self.language_model( + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=True, + cache_position=cache_position, + **kwargs, + ) + + return Mistral3ModelOutputWithPast( + last_hidden_state=outputs.last_hidden_state, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +class Mistral3ForConditionalGeneration(nn.Module): + _checkpoint_conversion_mapping = { + "^language_model.model": "model.language_model", + "^multi_modal_projector": "model.multi_modal_projector", + "^language_model.lm_head": "lm_head", + } + _tied_weights_keys = ["lm_head.weight"] + + def __init__(self, config: LlavaConfig): + super().__init__() + self.model = Mistral3Model(config.arch_config) + + def get_input_embeddings(self): + return self.model.get_input_embeddings() + + def set_decoder(self, decoder): + self.model.set_decoder(decoder) + + def get_decoder(self): + return self.model.get_decoder() + + # Make modules available through conditional class for BC + @property + def language_model(self): + return self.model.language_model + + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + output_hidden_states: Optional[bool] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + return_dict: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + logits_to_keep: Union[int, torch.Tensor] = 0, + image_sizes: Optional[torch.Tensor] = None, + **kwargs, + ) -> Union[tuple, Mistral3CausalLMOutputWithPast]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Example: + + """ + output_hidden_states = True + + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_hidden_states=output_hidden_states, + return_dict=True, + cache_position=cache_position, + image_sizes=image_sizes, + **kwargs, + ) + + return Mistral3CausalLMOutputWithPast( + hidden_states=outputs.hidden_states, + ) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + # Define mapping for stacked parameters + params_dict = dict(self.named_parameters()) + loaded_params: set[str] = set() + for name, loaded_weight in weights: + name_lower = name.lower() + if ( + "vision" in name_lower + or "multi" in name_lower + or "lm_head" in name_lower + ): + continue + final_name = name.replace("language_model.model.", "model.language_model.") + + if final_name in params_dict: + param = params_dict[final_name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(final_name) + else: + logger.warning(f"Param {name=} {final_name=} from weight is not loaded") + + return loaded_params + + +EntryClass = Mistral3ForConditionalGeneration diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py new file mode 100644 index 00000000..c354f923 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py @@ -0,0 +1,1180 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +from types import SimpleNamespace + +from transformers import ( + Cache, + DynamicCache, + PretrainedConfig, + Qwen2_5_VLTextConfig, + Qwen2RMSNorm, +) +from transformers.masking_utils import ( + create_causal_mask, + create_sliding_window_causal_mask, +) +from transformers.modeling_flash_attention_utils import FlashAttentionKwargs +from transformers.modeling_outputs import BaseModelOutputWithPast +from transformers.utils import TransformersKwargs, is_torchdynamo_compiling + +from sglang.multimodal_gen.configs.models.encoders.qwen_image import Qwen2_5VLConfig +from sglang.multimodal_gen.runtime.layers.attention import LocalAttention +from sglang.multimodal_gen.runtime.layers.linear import ( + MergedColumnParallelLinear, + RowParallelLinear, +) +from sglang.multimodal_gen.runtime.layers.quantization import QuantizationConfig +from sglang.multimodal_gen.runtime.loader.weight_utils import default_weight_loader +from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum +from sglang.multimodal_gen.runtime.utils.common import add_prefix + +# coding=utf-8 +# Adapted from +# https://github.com/huggingface/transformers/blob/19e6e80e10118f855137b90740936c0b11ac397f/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +# Copyright 2024 The Qwen team. +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only Qwen2-VL model compatible with HuggingFace weights.""" +import logging +from typing import Callable, Iterable, Optional, Tuple, Union + +try: + from typing import Unpack # type: ignore[attr-defined] +except ImportError: + # Python 3.10 and below + from typing_extensions import Unpack + +import torch +import torch.nn as nn +from transformers.activations import ACT2FN +from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import ( + Qwen2_5_VisionTransformerPretrainedModel, + Qwen2_5_VLAttention, + Qwen2_5_VLCausalLMOutputWithPast, + Qwen2_5_VLModelOutputWithPast, + Qwen2_5_VLRotaryEmbedding, + Qwen2MLP, + apply_multimodal_rotary_pos_emb, + eager_attention_forward, +) + +logger = logging.getLogger(__name__) + + +class Qwen2_5_VLAttention(nn.Module): + """ + Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer + and "Generating Long Sequences with Sparse Transformers". + """ + + def __init__(self, config: Qwen2_5_VLTextConfig, layer_idx: Optional[int] = None): + super().__init__() + self.config = config + self.layer_idx = layer_idx + if layer_idx is None: + logger.warn( + f"Instantiating {self.__class__.__name__} without passing `layer_idx` is not recommended and will " + "to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` " + "when creating this class." + ) + + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.is_causal = True + self.attention_dropout = config.attention_dropout + self.rope_scaling = config.rope_scaling + self.scaling = self.head_dim**-0.5 + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {self.num_heads})." + ) + self.q_proj = nn.Linear( + self.hidden_size, self.num_heads * self.head_dim, bias=True + ) + self.k_proj = nn.Linear( + self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True + ) + self.v_proj = nn.Linear( + self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True + ) + self.o_proj = nn.Linear( + self.num_heads * self.head_dim, self.hidden_size, bias=False + ) + self.sliding_window = ( + config.sliding_window + if config.layer_types[layer_idx] == "sliding_attention" + else None + ) + + self.rotary_emb = Qwen2_5_VLRotaryEmbedding(config=config) + self.attn = LocalAttention( + num_heads=self.num_heads, + head_size=self.head_dim, + num_kv_heads=self.num_key_value_heads, + softmax_scale=self.scaling, + causal=True, + supported_attention_backends=( + AttentionBackendEnum.FA, + AttentionBackendEnum.TORCH_SDPA, + ), + ) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + cache_position: Optional[torch.LongTensor] = None, + position_embeddings: Optional[ + tuple[torch.Tensor, torch.Tensor] + ] = None, # necessary, but kept here for BC + **kwargs: Unpack[FlashAttentionKwargs], + ) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]: + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2) + + cos, sin = position_embeddings + query_states, key_states = apply_multimodal_rotary_pos_emb( + query_states, key_states, cos, sin, self.rope_scaling["mrope_section"] + ) + + if past_key_values is not None: + cache_kwargs = { + "sin": sin, + "cos": cos, + "cache_position": cache_position, + } # Specific to RoPE models + key_states, value_states = past_key_values.update( + key_states, value_states, self.layer_idx, cache_kwargs + ) + + attention_interface: Callable = eager_attention_forward + # if self.config._attn_implementation != "eager": + # attention_interface = ALL_ATTENTION_FUNCTIONS["sdpa"] + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + value_states = value_states.transpose(1, 2) + attn_output = self.attn(query_states, key_states, value_states) + # + # attn_output, attn_weights = attention_interface( + # self, + # query_states, + # key_states, + # value_states, + # attention_mask, + # dropout=0.0 if not self.training else self.attention_dropout, + # scaling=self.scaling, + # sliding_window=self.sliding_window, + # position_ids=position_ids, # pass positions for FA2 + # **kwargs, + # ) + + attn_output = attn_output.reshape(bsz, q_len, -1).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output + + +class Qwen2_5_VLDecoderLayer(nn.Module): + def __init__(self, config: Qwen2_5_VLTextConfig, layer_idx: int): + super().__init__() + self.hidden_size = config.hidden_size + + if ( + config.use_sliding_window + and config._attn_implementation != "flash_attention_2" + ): + logger.warning( + f"Sliding Window Attention is enabled but not implemented for `{config._attn_implementation}`; " + "unexpected results may be encountered." + ) + self.self_attn = Qwen2_5_VLAttention(config, layer_idx) + + self.mlp = Qwen2MLP(config) + self.input_layernorm = Qwen2RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = Qwen2RMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) + self.attention_type = config.layer_types[layer_idx] + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[tuple[torch.Tensor]] = None, + output_attentions: Optional[bool] = False, + use_cache: Optional[bool] = False, + cache_position: Optional[torch.LongTensor] = None, + position_embeddings: Optional[ + tuple[torch.Tensor, torch.Tensor] + ] = None, # necessary, but kept here for BC + **kwargs: Unpack[FlashAttentionKwargs], + ) -> tuple[ + torch.FloatTensor, Optional[tuple[torch.FloatTensor, torch.FloatTensor]] + ]: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`, *optional*): attention mask of size + `(batch, sequence_length)` where padding elements are indicated by 0. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + past_key_values (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states + cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): + Indices depicting the position of the input sequence tokens in the sequence. + position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*): + Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`, + with `head_dim` being the embedding dimension of each attention head. + kwargs (`dict`, *optional*): + Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code + into the model + """ + + residual = hidden_states + + hidden_states = self.input_layernorm(hidden_states) + + # Self Attention + hidden_states = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + output_attentions=output_attentions, + use_cache=use_cache, + cache_position=cache_position, + position_embeddings=position_embeddings, + **kwargs, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + return hidden_states + + +class Qwen2_5_VLMLP(nn.Module): + def __init__( + self, + in_features: int, + hidden_features: int = None, + bias: bool = True, + hidden_act="silu", + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + input_size=in_features, + output_sizes=[hidden_features] * 2, # [gate_proj, up_proj] + bias=bias, + quant_config=quant_config, + prefix=add_prefix("gate_up_proj", prefix), + ) + self.down_proj = RowParallelLinear( + hidden_features, + in_features, + bias=bias, + quant_config=quant_config, + prefix=add_prefix("down_proj", prefix), + ) + self.act = ACT2FN[hidden_act] + + def forward(self, x: torch.Tensor) -> torch.Tensor: + gate_up, _ = self.gate_up_proj(x) + gate, up = gate_up.chunk(2, dim=-1) + x = self.act(gate) * up + x_down, _ = self.down_proj(x) + return x_down + + +class Qwen2_5_VLTextModel(nn.Module): + def __init__(self, config: PretrainedConfig): + super().__init__() + self.config = config + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding( + config.vocab_size, config.hidden_size, self.padding_idx + ) + self.layers = nn.ModuleList( + [ + Qwen2_5_VLDecoderLayer(config, layer_idx) + for layer_idx in range(config.num_hidden_layers) + ] + ) + self._attn_implementation = config._attn_implementation + self.norm = Qwen2RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.rotary_emb = Qwen2_5_VLRotaryEmbedding(config=config) + self.has_sliding_layers = "sliding_attention" in self.config.layer_types + + self.gradient_checkpointing = False + # Initialize weights and apply final processing + # self.post_init() + + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs: Unpack[FlashAttentionKwargs], + ) -> Union[tuple, BaseModelOutputWithPast]: + output_attentions = ( + output_attentions + if output_attentions is not None + else self.config.output_attentions + ) + output_hidden_states = ( + output_hidden_states + if output_hidden_states is not None + else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + + return_dict = ( + return_dict if return_dict is not None else self.config.use_return_dict + ) + + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError( + "You must specify exactly one of input_ids or inputs_embeds" + ) + + # torch.jit.trace() doesn't support cache objects in the output + if use_cache and past_key_values is None and not torch.jit.is_tracing(): + past_key_values = DynamicCache(config=self.config) + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + + if cache_position is None: + past_seen_tokens = ( + past_key_values.get_seq_length() if past_key_values is not None else 0 + ) + cache_position = torch.arange( + past_seen_tokens, + past_seen_tokens + inputs_embeds.shape[1], + device=inputs_embeds.device, + ) + + # the hard coded `3` is for temporal, height and width. + if position_ids is None: + position_ids = cache_position.view(1, 1, -1).expand( + 3, inputs_embeds.shape[0], -1 + ) + elif position_ids.ndim == 2: + position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) + + # NOTE: we need to pass text position ids for packing. Qwen2-VL uses 3D positions + # where each dim indicates visual spatial positions for temporal/height/width grids. + # There are two scenarios when FA2-like packed masking might be activated. + # 1. User specifically passed packed `position_ids` and no attention mask. + # In this case we expect the user to create correct position ids for all 3 grids + # and prepend text-only position ids to it. The final tensor will be [4, bs, seq-len] + # 2. User runs forward with no attention mask and no position ids. In this case, position ids + # are prepared by the model (`get_rope_index`) as `[4, bs, seq-len]` tensor. Text-only positions are + # prepended by us when creating positions so that the mask is constructed correctly. NOTE: failing to pass + # text-only positions will cause incorrect mask construction, do not change `prepare_input_for_generation` + if position_ids.ndim == 3 and position_ids.shape[0] == 4: + text_position_ids = position_ids[0] + position_ids = position_ids[1:] + else: + text_position_ids = position_ids[0] + + # It may already have been prepared by e.g. `generate` + if not isinstance(causal_mask_mapping := attention_mask, dict): + # Prepare mask arguments + mask_kwargs = { + "config": self.config, + "input_embeds": inputs_embeds, + "attention_mask": attention_mask, + "cache_position": cache_position, + "past_key_values": past_key_values, + "position_ids": text_position_ids, + } + # Create the masks + causal_mask_mapping = { + "full_attention": create_causal_mask(**mask_kwargs), + } + # The sliding window alternating layers are not always activated depending on the config + if self.has_sliding_layers: + causal_mask_mapping["sliding_attention"] = ( + create_sliding_window_causal_mask(**mask_kwargs) + ) + + hidden_states = inputs_embeds + + # create position embeddings to be shared across the decoder layers + position_embeddings = self.rotary_emb(hidden_states, position_ids) + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + + for decoder_layer in self.layers: + if output_hidden_states: + all_hidden_states += (hidden_states,) + + hidden_states = decoder_layer( + hidden_states, + attention_mask=causal_mask_mapping[decoder_layer.attention_type], + position_ids=text_position_ids, + past_key_values=past_key_values, + output_attentions=output_attentions, + use_cache=use_cache, + cache_position=cache_position, + position_embeddings=position_embeddings, + **kwargs, + ) + + hidden_states = self.norm(hidden_states) + + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + if not return_dict: + return tuple( + v + for v in [ + hidden_states, + past_key_values, + all_hidden_states, + all_self_attns, + ] + if v is not None + ) + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=past_key_values, + hidden_states=all_hidden_states, + attentions=all_self_attns, + ) + + +class Qwen2_5_VLModel(nn.Module): + base_model_prefix = "" + _checkpoint_conversion_mapping = {"^model": "language_model"} + # Reference: fix gemma3 grad acc #37208 + accepts_loss_kwargs = False + _no_split_modules = ["Qwen2_5_VLDecoderLayer", "Qwen2_5_VLVisionBlock"] + + def __init__(self, config): + super().__init__() + self.visual = Qwen2_5_VisionTransformerPretrainedModel._from_config( + config.vision_config + ) + self.language_model = Qwen2_5_VLTextModel(config.text_config) + self.visual.to(torch.get_default_dtype()) + self.rope_deltas = None # cache rope_deltas here + self.config = config + # Initialize weights and apply final processing + # self.post_init() + + def get_input_embeddings(self): + return self.language_model.embed_tokens + + def set_input_embeddings(self, value): + self.language_model.embed_tokens = value + + def set_decoder(self, decoder): + self.language_model = decoder + + def get_decoder(self): + return self.language_model + + def get_rope_index( + self, + input_ids: Optional[torch.LongTensor] = None, + image_grid_thw: Optional[torch.LongTensor] = None, + video_grid_thw: Optional[torch.LongTensor] = None, + second_per_grid_ts: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """ + Calculate the 3D rope index based on image and video's temporal, height and width in LLM. + + Explanation: + Each embedding sequence contains vision embedding and text embedding or just contains text embedding. + + For pure text embedding sequence, the rotary position embedding has no difference with modern LLMs. + Examples: + input_ids: [T T T T T], here T is for text. + temporal position_ids: [0, 1, 2, 3, 4] + height position_ids: [0, 1, 2, 3, 4] + width position_ids: [0, 1, 2, 3, 4] + + For vision and text embedding sequence, we calculate 3D rotary position embedding for vision part + and 1D rotary position embedding for text part. + Examples: + Temporal (Time): 3 patches, representing different segments of the video in time. + Height: 2 patches, dividing each frame vertically. + Width: 2 patches, dividing each frame horizontally. + We also have some important parameters: + fps (Frames Per Second): The video's frame rate, set to 1. This means one frame is processed each second. + tokens_per_second: This is a crucial parameter. It dictates how many "time-steps" or "temporal tokens" are conceptually packed into a one-second interval of the video. In this case, we have 25 tokens per second. So each second of the video will be represented with 25 separate time points. It essentially defines the temporal granularity. + temporal_patch_size: The number of frames that compose one temporal patch. Here, it's 2 frames. + interval: The step size for the temporal position IDs, calculated as tokens_per_second * temporal_patch_size / fps. In this case, 25 * 2 / 1 = 50. This means that each temporal patch will be have a difference of 50 in the temporal position IDs. + input_ids: [V V V V V V V V V V V V T T T T T], here V is for vision. + vision temporal position_ids: [0, 0, 0, 0, 50, 50, 50, 50, 100, 100, 100, 100] + vision height position_ids: [0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1] + vision width position_ids: [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1] + text temporal position_ids: [101, 102, 103, 104, 105] + text height position_ids: [101, 102, 103, 104, 105] + text width position_ids: [101, 102, 103, 104, 105] + Here we calculate the text start position_ids as the max vision position_ids plus 1. + + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + second_per_grid_ts (`torch.Tensor` of shape `(num_videos)`, *optional*): + The time interval (in seconds) for each grid along the temporal dimension in the 3D position IDs. + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + Returns: + position_ids (`torch.LongTensor` of shape `(3, batch_size, sequence_length)`) + mrope_position_deltas (`torch.Tensor` of shape `(batch_size)`) + """ + spatial_merge_size = self.config.vision_config.spatial_merge_size + image_token_id = self.config.image_token_id + video_token_id = self.config.video_token_id + vision_start_token_id = self.config.vision_start_token_id + mrope_position_deltas = [] + if input_ids is not None and ( + image_grid_thw is not None or video_grid_thw is not None + ): + total_input_ids = input_ids + if attention_mask is None: + attention_mask = torch.ones_like(total_input_ids) + position_ids = torch.ones( + 3, + input_ids.shape[0], + input_ids.shape[1], + dtype=input_ids.dtype, + device=input_ids.device, + ) + image_index, video_index = 0, 0 + attention_mask = attention_mask.to(total_input_ids.device) + for i, input_ids in enumerate(total_input_ids): + input_ids = input_ids[attention_mask[i] == 1] + image_nums, video_nums = 0, 0 + vision_start_indices = torch.argwhere( + input_ids == vision_start_token_id + ).squeeze(1) + vision_tokens = input_ids[vision_start_indices + 1] + image_nums = (vision_tokens == image_token_id).sum() + video_nums = (vision_tokens == video_token_id).sum() + input_tokens = input_ids.tolist() + llm_pos_ids_list: list = [] + st = 0 + remain_images, remain_videos = image_nums, video_nums + for _ in range(image_nums + video_nums): + if image_token_id in input_tokens and remain_images > 0: + ed_image = input_tokens.index(image_token_id, st) + else: + ed_image = len(input_tokens) + 1 + if video_token_id in input_tokens and remain_videos > 0: + ed_video = input_tokens.index(video_token_id, st) + else: + ed_video = len(input_tokens) + 1 + if ed_image < ed_video: + t, h, w = ( + image_grid_thw[image_index][0], + image_grid_thw[image_index][1], + image_grid_thw[image_index][2], + ) + second_per_grid_t = 0 + image_index += 1 + remain_images -= 1 + ed = ed_image + + else: + t, h, w = ( + video_grid_thw[video_index][0], + video_grid_thw[video_index][1], + video_grid_thw[video_index][2], + ) + if second_per_grid_ts is not None: + second_per_grid_t = second_per_grid_ts[video_index] + else: + second_per_grid_t = 1.0 + video_index += 1 + remain_videos -= 1 + ed = ed_video + llm_grid_t, llm_grid_h, llm_grid_w = ( + t.item(), + h.item() // spatial_merge_size, + w.item() // spatial_merge_size, + ) + text_len = ed - st + + st_idx = ( + llm_pos_ids_list[-1].max() + 1 + if len(llm_pos_ids_list) > 0 + else 0 + ) + llm_pos_ids_list.append( + torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx + ) + + range_tensor = torch.arange(llm_grid_t).view(-1, 1) + expanded_range = range_tensor.expand(-1, llm_grid_h * llm_grid_w) + + ## normalize type, send to device. + second_per_grid_t = torch.as_tensor( + second_per_grid_t, + dtype=range_tensor.dtype, + device=range_tensor.device, + ) + + time_tensor = ( + expanded_range + * second_per_grid_t + * self.config.vision_config.tokens_per_second + ) + + time_tensor_long = time_tensor.long() + t_index = time_tensor_long.flatten() + + h_index = ( + torch.arange(llm_grid_h) + .view(1, -1, 1) + .expand(llm_grid_t, -1, llm_grid_w) + .flatten() + ) + w_index = ( + torch.arange(llm_grid_w) + .view(1, 1, -1) + .expand(llm_grid_t, llm_grid_h, -1) + .flatten() + ) + llm_pos_ids_list.append( + torch.stack([t_index, h_index, w_index]) + text_len + st_idx + ) + st = ed + llm_grid_t * llm_grid_h * llm_grid_w + + if st < len(input_tokens): + st_idx = ( + llm_pos_ids_list[-1].max() + 1 + if len(llm_pos_ids_list) > 0 + else 0 + ) + text_len = len(input_tokens) - st + llm_pos_ids_list.append( + torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx + ) + + llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1) + position_ids[..., i, attention_mask[i] == 1] = llm_positions.to( + position_ids.device + ) + mrope_position_deltas.append( + llm_positions.max() + 1 - len(total_input_ids[i]) + ) + mrope_position_deltas = torch.tensor( + mrope_position_deltas, device=input_ids.device + ).unsqueeze(1) + return position_ids, mrope_position_deltas + else: + if attention_mask is not None: + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + position_ids = ( + position_ids.unsqueeze(0) + .expand(3, -1, -1) + .to(attention_mask.device) + ) + max_position_ids = position_ids.max(0, keepdim=False)[0].max( + -1, keepdim=True + )[0] + mrope_position_deltas = max_position_ids + 1 - attention_mask.shape[-1] + else: + position_ids = ( + torch.arange(input_ids.shape[1], device=input_ids.device) + .view(1, 1, -1) + .expand(3, input_ids.shape[0], -1) + ) + mrope_position_deltas = torch.zeros( + [input_ids.shape[0], 1], + device=input_ids.device, + dtype=input_ids.dtype, + ) + + return position_ids, mrope_position_deltas + + def get_video_features( + self, + pixel_values_videos: torch.FloatTensor, + video_grid_thw: Optional[torch.LongTensor] = None, + ): + """ + Encodes videos into continuous embeddings that can be forwarded to the language model. + + Args: + pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input videos. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + """ + pixel_values_videos = pixel_values_videos.type(self.visual.dtype) + video_embeds = self.visual(pixel_values_videos, grid_thw=video_grid_thw) + split_sizes = ( + video_grid_thw.prod(-1) // self.visual.spatial_merge_size**2 + ).tolist() + video_embeds = torch.split(video_embeds, split_sizes) + return video_embeds + + def get_image_features( + self, + pixel_values: torch.FloatTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + ): + """ + Encodes images into continuous embeddings that can be forwarded to the language model. + + Args: + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): + The tensors corresponding to the input images. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + """ + pixel_values = pixel_values.type(self.visual.dtype) + image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) + split_sizes = ( + image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2 + ).tolist() + image_embeds = torch.split(image_embeds, split_sizes) + return image_embeds + + def get_placeholder_mask( + self, + input_ids: torch.LongTensor, + inputs_embeds: torch.FloatTensor, + image_features: torch.FloatTensor = None, + video_features: torch.FloatTensor = None, + ): + """ + Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is + equal to the length of multimodal features. If the lengths are different, an error is raised. + """ + if input_ids is None: + special_image_mask = inputs_embeds == self.get_input_embeddings()( + torch.tensor( + self.config.image_token_id, + dtype=torch.long, + device=inputs_embeds.device, + ) + ) + special_image_mask = special_image_mask.all(-1) + special_video_mask = inputs_embeds == self.get_input_embeddings()( + torch.tensor( + self.config.video_token_id, + dtype=torch.long, + device=inputs_embeds.device, + ) + ) + special_video_mask = special_video_mask.all(-1) + else: + special_image_mask = input_ids == self.config.image_token_id + special_video_mask = input_ids == self.config.video_token_id + + n_image_tokens = special_image_mask.sum() + special_image_mask = ( + special_image_mask.unsqueeze(-1) + .expand_as(inputs_embeds) + .to(inputs_embeds.device) + ) + if ( + image_features is not None + and inputs_embeds[special_image_mask].numel() != image_features.numel() + ): + raise ValueError( + f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {image_features.shape[0]}" + ) + + n_video_tokens = special_video_mask.sum() + special_video_mask = ( + special_video_mask.unsqueeze(-1) + .expand_as(inputs_embeds) + .to(inputs_embeds.device) + ) + if ( + video_features is not None + and inputs_embeds[special_video_mask].numel() != video_features.numel() + ): + raise ValueError( + f"Videos features and video tokens do not match: tokens: {n_video_tokens}, features {video_features.shape[0]}" + ) + + return special_image_mask, special_video_mask + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + pixel_values: Optional[torch.Tensor] = None, + pixel_values_videos: Optional[torch.FloatTensor] = None, + image_grid_thw: Optional[torch.LongTensor] = None, + video_grid_thw: Optional[torch.LongTensor] = None, + rope_deltas: Optional[torch.LongTensor] = None, + cache_position: Optional[torch.LongTensor] = None, + second_per_grid_ts: Optional[torch.Tensor] = None, + **kwargs: Unpack[TransformersKwargs], + ) -> Union[tuple, Qwen2_5_VLModelOutputWithPast]: + r""" + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. + second_per_grid_ts (`torch.Tensor` of shape `(num_videos)`, *optional*): + The time interval (in seconds) for each grid along the temporal dimension in the 3D position IDs. + """ + + output_attentions = ( + output_attentions + if output_attentions is not None + else self.config.output_attentions + ) + output_hidden_states = ( + output_hidden_states + if output_hidden_states is not None + else self.config.output_hidden_states + ) + return_dict = ( + return_dict if return_dict is not None else self.config.use_return_dict + ) + + if inputs_embeds is None: + inputs_embeds = self.get_input_embeddings()(input_ids) + + if pixel_values is not None: + image_embeds = self.get_image_features(pixel_values, image_grid_thw) + image_embeds = torch.cat(image_embeds, dim=0).to( + inputs_embeds.device, inputs_embeds.dtype + ) + image_mask, _ = self.get_placeholder_mask( + input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds + ) + inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) + + if pixel_values_videos is not None: + video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw) + video_embeds = torch.cat(video_embeds, dim=0).to( + inputs_embeds.device, inputs_embeds.dtype + ) + _, video_mask = self.get_placeholder_mask( + input_ids, inputs_embeds=inputs_embeds, video_features=video_embeds + ) + inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds) + + if position_ids is None: + # Calculate RoPE index once per generation in the pre-fill stage only. + # When compiling, we can't check tensor values thus we check only input length + # It is safe to assume that `length!=1` means we're in pre-fill because compiled + # models currently cannot do asssisted decoding + prefill_compiled_stage = is_torchdynamo_compiling() and ( + (input_ids is not None and input_ids.shape[1] != 1) + or (inputs_embeds is not None and inputs_embeds.shape[1] != 1) + ) + prefill_noncompiled_stage = not is_torchdynamo_compiling() and ( + (cache_position is not None and cache_position[0] == 0) + or (past_key_values is None or past_key_values.get_seq_length() == 0) + ) + if ( + prefill_compiled_stage or prefill_noncompiled_stage + ) or self.rope_deltas is None: + position_ids, rope_deltas = self.get_rope_index( + input_ids, + image_grid_thw, + video_grid_thw, + second_per_grid_ts=second_per_grid_ts, + attention_mask=attention_mask, + ) + self.rope_deltas = rope_deltas + else: + batch_size, seq_length, _ = inputs_embeds.shape + position_ids = torch.arange(seq_length, device=inputs_embeds.device) + position_ids = position_ids.view(1, 1, -1).expand(3, batch_size, -1) + if cache_position is not None: + delta = (cache_position[0] + self.rope_deltas).to( + inputs_embeds.device + ) + else: + delta = torch.zeros( + (batch_size, seq_length), device=inputs_embeds.device + ) + delta = delta.repeat_interleave(batch_size // delta.shape[0], dim=1) + position_ids += delta.to(position_ids.device) + + outputs = self.language_model( + input_ids=None, + position_ids=position_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=True, + cache_position=cache_position, + **kwargs, + ) + + output = Qwen2_5_VLModelOutputWithPast( + last_hidden_state=outputs.last_hidden_state, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + rope_deltas=self.rope_deltas, + ) + return output if return_dict else output.to_tuple() + + +class DotDict(dict): + def __init__(self, mapping): + super().__init__() + for key, value in mapping.items(): + if isinstance(value, dict): + value = DotDict(value) # 递归转换 + elif isinstance(value, list): + # 如果是 list，且元素是 dict 也递归转换 + value = [ + DotDict(item) if isinstance(item, dict) else item for item in value + ] + self[key] = value + + def __getattr__(self, item): + try: + return self[item] + except KeyError: + raise AttributeError(f"No attribute '{item}'") + + def __setattr__(self, key, value): + self[key] = value + + def __delattr__(self, key): + del self[key] + + +def dict_to_namespace(d): + for k, v in d.items(): + if isinstance(v, dict): + d[k] = dict_to_namespace(v) + elif isinstance(v, list): + d[k] = [dict_to_namespace(i) if isinstance(i, dict) else i for i in v] + return SimpleNamespace(**d) + + +class Qwen2_5_VLForConditionalGeneration(TextEncoder): + # BitandBytes specific attributes + default_bitsandbytes_target_modules = [ + ".gate_up_proj.", + ".down_proj.", + ".q_proj.", + ".k_proj.", + ".v_proj.", + ".o_proj.", + ] + bitsandbytes_stacked_params_mapping = { + # shard_name, weight_name, index + "q_proj": ("qkv_proj", 0), + "k_proj": ("qkv_proj", 1), + "v_proj": ("qkv_proj", 2), + "gate_proj": ("gate_up_proj", 0), + "up_proj": ("gate_up_proj", 1), + } + + def __init__( + self, + config: Qwen2_5VLConfig, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__(config) + config = config.arch_config + self.model = Qwen2_5_VLModel(config) + self.lm_head = nn.Linear( + config.text_config.hidden_size, config.text_config.vocab_size, bias=False + ) + + self.config = config + + def get_input_embeddings(self): + return self.model.embed_tokens + + @torch.no_grad() + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + pixel_values: Optional[torch.Tensor] = None, + pixel_values_videos: Optional[torch.FloatTensor] = None, + image_grid_thw: Optional[torch.LongTensor] = None, + video_grid_thw: Optional[torch.LongTensor] = None, + rope_deltas: Optional[torch.LongTensor] = None, + cache_position: Optional[torch.LongTensor] = None, + second_per_grid_ts: Optional[torch.Tensor] = None, + logits_to_keep: Union[int, torch.Tensor] = 0, + **kwargs: Unpack[TransformersKwargs], + ): + """Run forward pass for Qwen2_5-VL. + + Args: + input_ids: Flattened (concatenated) input_ids corresponding to a + batch. + positions: Flattened (concatenated) position ids corresponding to a + batch. + **NOTE**: If mrope is enabled (default setting for Qwen2-VL + opensource models), the shape will be `(3, seq_len)`, + otherwise it will be `(seq_len,). + (Use input_metadata.mrope_positions to replace it) + """ + output_attentions = False + output_hidden_states = ( + output_hidden_states + if output_hidden_states is not None + else self.config.output_hidden_states + ) + + outputs = self.model( + input_ids=input_ids, + pixel_values=pixel_values, + pixel_values_videos=pixel_values_videos, + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + second_per_grid_ts=second_per_grid_ts, + position_ids=position_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=True, + cache_position=cache_position, + **kwargs, + ) + + hidden_states = outputs[0] + + # Only compute necessary logits, and do not upcast them to float if we are not computing the loss + slice_indices = ( + slice(-logits_to_keep, None) + if isinstance(logits_to_keep, int) + else logits_to_keep + ) + logits = self.lm_head(hidden_states[:, slice_indices, :]) + return Qwen2_5_VLCausalLMOutputWithPast( + loss=None, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + rope_deltas=outputs.rope_deltas, + ) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + loaded_params: set[str] = set() + + params_dict = dict(self.named_parameters(remove_duplicate=False)) + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + + name = name.replace("model.", "model.language_model.") + if "visual." in name: + name = name.replace("visual.", "model.visual.") + try: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + param = params_dict[name] + except KeyError: + print(params_dict.keys()) + raise + + weight_loader = getattr(param, "weight_loader", default_weight_loader) + loaded_weight = loaded_weight.to(param.dtype) + weight_loader(param, loaded_weight) + loaded_params.add(name) + return loaded_params + + def get_embed_and_head(self): + return self.model.embed_tokens.weight, self.lm_head.weight + + +EntryClass = Qwen2_5_VLForConditionalGeneration diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/stepllm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/stepllm.py new file mode 100644 index 00000000..dae653ea --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/stepllm.py @@ -0,0 +1,614 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# type: ignore +# Copyright 2025 StepFun Inc. All Rights Reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# ============================================================================== +import os +from functools import wraps + +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange +from transformers import PretrainedConfig, PreTrainedModel + +from sglang.multimodal_gen.runtime.models.dits.stepvideo import StepVideoRMSNorm + + +class EmptyInitOnDevice(torch.overrides.TorchFunctionMode): + + def __init__(self, device=None): + self.device = device + + def __torch_function__(self, func, types, args=(), kwargs=None): + kwargs = kwargs or {} + if getattr(func, "__module__", None) == "torch.nn.init": + if "tensor" in kwargs: + return kwargs["tensor"] + else: + return args[0] + if ( + self.device is not None + and func in torch.utils._device._device_constructors() + and kwargs.get("device") is None + ): + kwargs["device"] = self.device + return func(*args, **kwargs) + + +def with_empty_init(func): + + @wraps(func) + def wrapper(*args, **kwargs): + with EmptyInitOnDevice("cpu"): + return func(*args, **kwargs) + + return wrapper + + +class LLaMaEmbedding(nn.Module): + """Language model embeddings. + + Arguments: + hidden_size: hidden size + vocab_size: vocabulary size + max_sequence_length: maximum size of sequence. This + is used for positional embedding + embedding_dropout_prob: dropout probability for embeddings + init_method: weight initialization method + num_tokentypes: size of the token-type embeddings. 0 value + will ignore this embedding + """ + + def __init__( + self, + cfg, + ): + super().__init__() + self.hidden_size = cfg.hidden_size + self.params_dtype = cfg.params_dtype + self.fp32_residual_connection = cfg.fp32_residual_connection + self.embedding_weights_in_fp32 = cfg.embedding_weights_in_fp32 + self.word_embeddings = torch.nn.Embedding( + cfg.padded_vocab_size, + self.hidden_size, + ) + self.embedding_dropout = torch.nn.Dropout(cfg.hidden_dropout) + + def forward(self, input_ids): + # Embeddings. + if self.embedding_weights_in_fp32: + self.word_embeddings = self.word_embeddings.to(torch.float32) + embeddings = self.word_embeddings(input_ids) + if self.embedding_weights_in_fp32: + embeddings = embeddings.to(self.params_dtype) + self.word_embeddings = self.word_embeddings.to(self.params_dtype) + + # Data format change to avoid explicit transposes : [b s h] --> [s b h]. + embeddings = embeddings.transpose(0, 1).contiguous() + + # If the input flag for fp32 residual connection is set, convert for float. + if self.fp32_residual_connection: + embeddings = embeddings.float() + + # Dropout. + embeddings = self.embedding_dropout(embeddings) + + return embeddings + + +class StepChatTokenizer: + """Step Chat Tokenizer""" + + def __init__( + self, + model_file, + name="StepChatTokenizer", + bot_token="<|BOT|>", # Begin of Turn + eot_token="<|EOT|>", # End of Turn + call_start_token="<|CALL_START|>", # Call Start + call_end_token="<|CALL_END|>", # Call End + think_start_token="<|THINK_START|>", # Think Start + think_end_token="<|THINK_END|>", # Think End + mask_start_token="<|MASK_1e69f|>", # Mask start + mask_end_token="<|UNMASK_1e69f|>", # Mask end + ): + import sentencepiece + + self._tokenizer = sentencepiece.SentencePieceProcessor(model_file=model_file) + + self._vocab = {} + self._inv_vocab = {} + + self._special_tokens = {} + self._inv_special_tokens = {} + + self._t5_tokens = [] + + for idx in range(self._tokenizer.get_piece_size()): + text = self._tokenizer.id_to_piece(idx) + self._inv_vocab[idx] = text + self._vocab[text] = idx + + if self._tokenizer.is_control(idx) or self._tokenizer.is_unknown(idx): + self._special_tokens[text] = idx + self._inv_special_tokens[idx] = text + + self._unk_id = self._tokenizer.unk_id() + self._bos_id = self._tokenizer.bos_id() + self._eos_id = self._tokenizer.eos_id() + + for token in [ + bot_token, + eot_token, + call_start_token, + call_end_token, + think_start_token, + think_end_token, + ]: + assert token in self._vocab, f"Token '{token}' not found in tokenizer" + assert ( + token in self._special_tokens + ), f"Token '{token}' is not a special token" + + for token in [mask_start_token, mask_end_token]: + assert token in self._vocab, f"Token '{token}' not found in tokenizer" + + self._bot_id = self._tokenizer.piece_to_id(bot_token) + self._eot_id = self._tokenizer.piece_to_id(eot_token) + self._call_start_id = self._tokenizer.piece_to_id(call_start_token) + self._call_end_id = self._tokenizer.piece_to_id(call_end_token) + self._think_start_id = self._tokenizer.piece_to_id(think_start_token) + self._think_end_id = self._tokenizer.piece_to_id(think_end_token) + self._mask_start_id = self._tokenizer.piece_to_id(mask_start_token) + self._mask_end_id = self._tokenizer.piece_to_id(mask_end_token) + + self._underline_id = self._tokenizer.piece_to_id("\u2581") + + @property + def vocab(self): + return self._vocab + + @property + def inv_vocab(self): + return self._inv_vocab + + @property + def vocab_size(self): + return self._tokenizer.vocab_size() + + def tokenize(self, text: str) -> list[int]: + return self._tokenizer.encode_as_ids(text) + + def detokenize(self, token_ids: list[int]) -> str: + return self._tokenizer.decode_ids(token_ids) + + +class Tokens: + + def __init__( + self, input_ids, cu_input_ids, attention_mask, cu_seqlens, max_seq_len + ) -> None: + self.input_ids = input_ids + self.attention_mask = attention_mask + self.cu_input_ids = cu_input_ids + self.cu_seqlens = cu_seqlens + self.max_seq_len = max_seq_len + + def to(self, device): + self.input_ids = self.input_ids.to(device) + self.attention_mask = self.attention_mask.to(device) + self.cu_input_ids = self.cu_input_ids.to(device) + self.cu_seqlens = self.cu_seqlens.to(device) + return self + + +class Wrapped_StepChatTokenizer(StepChatTokenizer): + + def __call__( + self, + text, + max_length=320, + padding="max_length", + truncation=True, + return_tensors="pt", + ): + # [bos, ..., eos, pad, pad, ..., pad] + self.BOS = 1 + self.EOS = 2 + self.PAD = 2 + out_tokens = [] + attn_mask = [] + if len(text) == 0: + part_tokens = [self.BOS] + [self.EOS] + valid_size = len(part_tokens) + if len(part_tokens) < max_length: + part_tokens += [self.PAD] * (max_length - valid_size) + out_tokens.append(part_tokens) + attn_mask.append([1] * valid_size + [0] * (max_length - valid_size)) + else: + for part in text: + part_tokens = self.tokenize(part) + part_tokens = part_tokens[ + : (max_length - 2) + ] # leave 2 space for bos and eos + part_tokens = [self.BOS] + part_tokens + [self.EOS] + valid_size = len(part_tokens) + if len(part_tokens) < max_length: + part_tokens += [self.PAD] * (max_length - valid_size) + out_tokens.append(part_tokens) + attn_mask.append([1] * valid_size + [0] * (max_length - valid_size)) + + out_tokens = torch.tensor(out_tokens, dtype=torch.long) + attn_mask = torch.tensor(attn_mask, dtype=torch.long) + + # padding y based on tp size + padded_len = 0 + padded_flag = False + if padded_len > 0: + padded_flag = True + if padded_flag: + pad_tokens = torch.tensor( + [[self.PAD] * max_length], device=out_tokens.device + ) + pad_attn_mask = torch.tensor( + [[1] * padded_len + [0] * (max_length - padded_len)], + device=attn_mask.device, + ) + out_tokens = torch.cat([out_tokens, pad_tokens], dim=0) + attn_mask = torch.cat([attn_mask, pad_attn_mask], dim=0) + + # cu_seqlens + cu_out_tokens = out_tokens.masked_select(attn_mask != 0).unsqueeze(0) + seqlen = attn_mask.sum(dim=1).tolist() + cu_seqlens = torch.cumsum(torch.tensor([0] + seqlen), 0).to( + device=out_tokens.device, dtype=torch.int32 + ) + max_seq_len = max(seqlen) + return Tokens(out_tokens, cu_out_tokens, attn_mask, cu_seqlens, max_seq_len) + + +def flash_attn_func( + q, + k, + v, + dropout_p=0.0, + softmax_scale=None, + causal=True, + return_attn_probs=False, + tp_group_rank=0, + tp_group_size=1, +): + softmax_scale = q.size(-1) ** (-0.5) if softmax_scale is None else softmax_scale + return torch.ops.Optimus.fwd( + q, + k, + v, + None, + dropout_p, + softmax_scale, + causal, + return_attn_probs, + None, + tp_group_rank, + tp_group_size, + )[0] + + +class FlashSelfAttention(torch.nn.Module): + + def __init__( + self, + attention_dropout=0.0, + ): + super().__init__() + self.dropout_p = attention_dropout + + def forward(self, q, k, v, cu_seqlens=None, max_seq_len=None): + if cu_seqlens is None: + output = flash_attn_func(q, k, v, dropout_p=self.dropout_p) + else: + raise ValueError("cu_seqlens is not supported!") + + return output + + +def safediv(n, d): + q, r = divmod(n, d) + assert r == 0 + return q + + +class MultiQueryAttention(nn.Module): + + def __init__(self, cfg, layer_id=None): + super().__init__() + + self.head_dim = cfg.hidden_size // cfg.num_attention_heads + self.max_seq_len = cfg.seq_length + self.use_flash_attention = cfg.use_flash_attn + assert self.use_flash_attention, "FlashAttention is required!" + + self.n_groups = cfg.num_attention_groups + self.tp_size = 1 + self.n_local_heads = cfg.num_attention_heads + self.n_local_groups = self.n_groups + + self.wqkv = nn.Linear( + cfg.hidden_size, + cfg.hidden_size + self.head_dim * 2 * self.n_groups, + bias=False, + ) + self.wo = nn.Linear( + cfg.hidden_size, + cfg.hidden_size, + bias=False, + ) + + # assert self.use_flash_attention, 'non-Flash attention not supported yet.' + self.core_attention = FlashSelfAttention( + attention_dropout=cfg.attention_dropout + ) + # self.core_attention = LocalAttention( + # num_heads = self.n_local_heads, + # head_size = self.head_dim, + # # num_kv_heads = self.n_local_groups, + # casual = True, + # supported_attention_backends = [_Backend.FLASH_ATTN, _Backend.TORCH_SDPA], # RIVER TODO + # ) + self.layer_id = layer_id + + def forward( + self, + x: torch.Tensor, + mask: torch.Tensor | None, + cu_seqlens: torch.Tensor | None, + max_seq_len: torch.Tensor | None, + ): + seqlen, bsz, dim = x.shape + xqkv = self.wqkv(x) + + xq, xkv = torch.split( + xqkv, + (dim // self.tp_size, self.head_dim * 2 * self.n_groups // self.tp_size), + dim=-1, + ) + + # gather on 1st dimension + xq = xq.view(seqlen, bsz, self.n_local_heads, self.head_dim) + xkv = xkv.view(seqlen, bsz, self.n_local_groups, 2 * self.head_dim) + xk, xv = xkv.chunk(2, -1) + + # rotary embedding + flash attn + xq = rearrange(xq, "s b h d -> b s h d") + xk = rearrange(xk, "s b h d -> b s h d") + xv = rearrange(xv, "s b h d -> b s h d") + + # q_per_kv = self.n_local_heads // self.n_local_groups + # if q_per_kv > 1: + # b, s, h, d = xk.size() + # if h == 1: + # xk = xk.expand(b, s, q_per_kv, d) + # xv = xv.expand(b, s, q_per_kv, d) + # else: + # ''' To cover the cases where h > 1, we have + # the following implementation, which is equivalent to: + # xk = xk.repeat_interleave(q_per_kv, dim=-2) + # xv = xv.repeat_interleave(q_per_kv, dim=-2) + # but can avoid calling aten::item() that involves cpu. + # ''' + # idx = torch.arange(q_per_kv * h, device=xk.device).reshape(q_per_kv, -1).permute(1, 0).flatten() + # xk = torch.index_select(xk.repeat(1, 1, q_per_kv, 1), 2, idx).contiguous() + # xv = torch.index_select(xv.repeat(1, 1, q_per_kv, 1), 2, idx).contiguous() + if self.use_flash_attention: + output = self.core_attention(xq, xk, xv) + # reduce-scatter only support first dimension now + output = rearrange(output, "b s h d -> s b (h d)").contiguous() + else: + xq, xk, xv = [ + rearrange(x, "b s ... -> s b ...").contiguous() for x in (xq, xk, xv) + ] + output = self.core_attention(xq, xk, xv) # , mask) + output = self.wo(output) + return output + + +class FeedForward(nn.Module): + + def __init__( + self, + cfg, + dim: int, + hidden_dim: int, + layer_id: int, + multiple_of: int = 256, + ): + super().__init__() + + hidden_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of) + + def swiglu(x): + x = torch.chunk(x, 2, dim=-1) + return F.silu(x[0]) * x[1] + + self.swiglu = swiglu + + self.w1 = nn.Linear( + dim, + 2 * hidden_dim, + bias=False, + ) + self.w2 = nn.Linear( + hidden_dim, + dim, + bias=False, + ) + + def forward(self, x): + x = self.swiglu(self.w1(x)) + output = self.w2(x) + return output + + +class TransformerBlock(nn.Module): + + def __init__(self, cfg, layer_id: int): + super().__init__() + + self.n_heads = cfg.num_attention_heads + self.dim = cfg.hidden_size + self.head_dim = cfg.hidden_size // cfg.num_attention_heads + self.attention = MultiQueryAttention( + cfg, + layer_id=layer_id, + ) + + self.feed_forward = FeedForward( + cfg, + dim=cfg.hidden_size, + hidden_dim=cfg.ffn_hidden_size, + layer_id=layer_id, + ) + self.layer_id = layer_id + self.attention_norm = StepVideoRMSNorm( + cfg.hidden_size, + eps=cfg.layernorm_epsilon, + ) + self.ffn_norm = StepVideoRMSNorm( + cfg.hidden_size, + eps=cfg.layernorm_epsilon, + ) + + def forward( + self, + x: torch.Tensor, + mask: torch.Tensor | None, + cu_seqlens: torch.Tensor | None, + max_seq_len: torch.Tensor | None, + ): + residual = self.attention.forward( + self.attention_norm(x), mask, cu_seqlens, max_seq_len + ) + h = x + residual + ffn_res = self.feed_forward.forward(self.ffn_norm(h)) + out = h + ffn_res + return out + + +class Transformer(nn.Module): + + def __init__( + self, + config, + max_seq_size=8192, + ): + super().__init__() + self.num_layers = config.num_layers + self.layers = self._build_layers(config) + + def _build_layers(self, config): + layers = torch.nn.ModuleList() + for layer_id in range(self.num_layers): + layers.append( + TransformerBlock( + config, + layer_id=layer_id + 1, + ) + ) + return layers + + def forward( + self, + hidden_states, + attention_mask, + cu_seqlens=None, + max_seq_len=None, + ): + + if max_seq_len is not None and not isinstance(max_seq_len, torch.Tensor): + max_seq_len = torch.tensor(max_seq_len, dtype=torch.int32, device="cpu") + + for lid, layer in enumerate(self.layers): + hidden_states = layer( + hidden_states, + attention_mask, + cu_seqlens, + max_seq_len, + ) + return hidden_states + + +class Step1Model(PreTrainedModel): + config_class = PretrainedConfig + + @with_empty_init + def __init__( + self, + config, + ): + super().__init__(config) + self.tok_embeddings = LLaMaEmbedding(config) + self.transformer = Transformer(config) + + def forward( + self, + input_ids=None, + attention_mask=None, + ): + + hidden_states = self.tok_embeddings(input_ids) + + hidden_states = self.transformer( + hidden_states, + attention_mask, + ) + return hidden_states + + +class STEP1TextEncoder(torch.nn.Module): + + def __init__(self, model_dir, max_length=320): + super().__init__() + self.max_length = max_length + self.text_tokenizer = Wrapped_StepChatTokenizer( + os.path.join(model_dir, "step1_chat_tokenizer.model") + ) + text_encoder = Step1Model.from_pretrained(model_dir) + self.text_encoder = text_encoder.eval().to(torch.bfloat16) + + @torch.no_grad + def forward(self, prompts, with_mask=True, max_length=None): + self.device = next(self.text_encoder.parameters()).device + + with torch.no_grad(), torch.amp.autocast("cuda", dtype=torch.bfloat16): + if type(prompts) is str: + prompts = [prompts] + txt_tokens = self.text_tokenizer( + prompts, + max_length=max_length or self.max_length, + padding="max_length", + truncation=True, + return_tensors="pt", + ) + y = self.text_encoder( + txt_tokens.input_ids.to(self.device), + attention_mask=( + txt_tokens.attention_mask.to(self.device) if with_mask else None + ), + ) + y_mask = txt_tokens.attention_mask + return y.transpose(0, 1), y_mask + + +EntryClass = STEP1TextEncoder diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/t5.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/t5.py new file mode 100644 index 00000000..048308ad --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/t5.py @@ -0,0 +1,716 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from transformers: https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/t5/modeling_t5.py + +# Derived from T5 implementation posted on HuggingFace; license below: +# +# coding=utf-8 +# Copyright 2018 Mesh TensorFlow authors, T5 Authors and HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch T5 & UMT5 model.""" + +import math +from collections.abc import Iterable +from dataclasses import dataclass + +import torch +import torch.nn.functional as F +from torch import nn + +from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput, T5Config +from sglang.multimodal_gen.runtime.distributed import get_tp_rank, get_tp_world_size +from sglang.multimodal_gen.runtime.layers.activation import get_act_fn +from sglang.multimodal_gen.runtime.layers.layernorm import RMSNorm +from sglang.multimodal_gen.runtime.layers.linear import ( + MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear, +) +from sglang.multimodal_gen.runtime.layers.quantization import QuantizationConfig +from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import ( + VocabParallelEmbedding, +) +from sglang.multimodal_gen.runtime.loader.weight_utils import default_weight_loader +from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder +from sglang.multimodal_gen.runtime.platforms import current_platform + + +class AttentionType: + """ + Attention type. + Use string to be compatible with `torch.compile`. + """ + + # Decoder attention between previous layer Q/K/V + DECODER = "decoder" + # Encoder attention between previous layer Q/K/V for encoder-decoder + ENCODER = "encoder" + # Encoder attention between previous layer Q/K/V + ENCODER_ONLY = "encoder_only" + # Attention between dec. Q and enc. K/V for encoder-decoder + ENCODER_DECODER = "encoder_decoder" + + +_seen_keys = set() # 用集合记录已经出现过的 key + + +@dataclass +class AttentionMetadata: + attn_bias: torch.Tensor + + +class T5DenseActDense(nn.Module): + + def __init__( + self, config: T5Config, quant_config: QuantizationConfig | None = None + ): + super().__init__() + self.wi = MergedColumnParallelLinear(config.d_model, [config.d_ff], bias=False) + self.wo = RowParallelLinear( + config.d_ff, config.d_model, bias=False, quant_config=quant_config + ) + self.act = get_act_fn(config.dense_act_fn) + + def forward(self, hidden_states) -> torch.Tensor: + hidden_states, _ = self.wi(hidden_states) + hidden_states = self.act(hidden_states) + hidden_states, _ = self.wo(hidden_states) + return hidden_states + + +class T5DenseGatedActDense(nn.Module): + + def __init__( + self, config: T5Config, quant_config: QuantizationConfig | None = None + ): + super().__init__() + self.wi_0 = MergedColumnParallelLinear( + config.d_model, [config.d_ff], bias=False, quant_config=quant_config + ) + self.wi_1 = MergedColumnParallelLinear( + config.d_model, [config.d_ff], bias=False, quant_config=quant_config + ) + # Should not run in fp16 unless mixed-precision is used, + # see https://github.com/huggingface/transformers/issues/20287. + self.wo = RowParallelLinear( + config.d_ff, config.d_model, bias=False, quant_config=quant_config + ) + self.act = get_act_fn(config.dense_act_fn) + + def forward(self, hidden_states) -> torch.Tensor: + hidden_gelu = self.act(self.wi_0(hidden_states)[0]) + hidden_linear, _ = self.wi_1(hidden_states) + hidden_states = hidden_gelu * hidden_linear + hidden_states, _ = self.wo(hidden_states) + return hidden_states + + +class T5LayerFF(nn.Module): + + def __init__( + self, config: T5Config, quant_config: QuantizationConfig | None = None + ): + super().__init__() + if config.is_gated_act: + self.DenseReluDense = T5DenseGatedActDense( + config, quant_config=quant_config + ) + else: + self.DenseReluDense = T5DenseActDense(config, quant_config=quant_config) + + self.layer_norm = RMSNorm(config.d_model, eps=config.layer_norm_epsilon) + + def forward(self, hidden_states) -> torch.Tensor: + forwarded_states = self.layer_norm(hidden_states) + forwarded_states = self.DenseReluDense(forwarded_states) + hidden_states = hidden_states + forwarded_states + return hidden_states + + +# T5 has attn_bias and does not use softmax scaling +class T5MultiHeadAttention(nn.Module): + + def __init__(self) -> None: + super().__init__() + + def forward(self, q, k, v, attn_bias=None): + b, _, n, c = q.shape + attn = torch.einsum("binc,bjnc->bnij", q, k) + if attn_bias is not None: + attn += attn_bias + + attn = F.softmax(attn.float(), dim=-1).type_as(attn) + x = torch.einsum("bnij,bjnc->binc", attn, v) + x = x.reshape(b, -1, n * c) + return x + + +class T5Attention(nn.Module): + + def __init__( + self, + config: T5Config, + attn_type: str, + has_relative_attention_bias=False, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ): + super().__init__() + self.attn_type = attn_type + # Cross-attention has no relative pos encoding anyway + self.is_decoder = attn_type == AttentionType.DECODER + self.has_relative_attention_bias = has_relative_attention_bias + self.relative_attention_num_buckets = config.relative_attention_num_buckets + self.relative_attention_max_distance = config.relative_attention_max_distance + self.d_model = config.d_model + self.key_value_proj_dim = config.d_kv + self.total_num_heads = self.total_num_kv_heads = config.num_heads + + # Partition heads across multiple tensor parallel GPUs. + tp_world_size = get_tp_world_size() + assert config.num_heads % tp_world_size == 0 + self.n_heads = config.num_heads // tp_world_size + + self.inner_dim = self.n_heads * self.key_value_proj_dim + # No GQA in t5. + # self.n_kv_heads = self.n_heads + + self.qkv_proj = QKVParallelLinear( + self.d_model, + self.d_model // self.total_num_heads, + self.total_num_heads, + self.total_num_kv_heads, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) + + self.attn = T5MultiHeadAttention() + + if self.has_relative_attention_bias: + self.relative_attention_bias = VocabParallelEmbedding( + self.relative_attention_num_buckets, + self.total_num_heads, + org_num_embeddings=self.relative_attention_num_buckets, + padding_size=self.relative_attention_num_buckets, + quant_config=quant_config, + ) + self.o = RowParallelLinear( + self.d_model, + self.d_model, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + + @staticmethod + def _relative_position_bucket( + relative_position, bidirectional=True, num_buckets=32, max_distance=128 + ) -> torch.Tensor: + """ + Adapted from Mesh Tensorflow: + https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593 + Translate relative position to a bucket number for relative attention. + The relative position is defined as memory_position - query_position, + i.e. the distance in tokens from the attending position to the + attended-to position. If bidirectional=False, then positive relative + positions are invalid. We use smaller buckets for small absolute + relative_position and larger buckets for larger absolute + relative_positions. All relative positions >=max_distance map to the + same bucket. All relative positions <=-max_distance map to the same + bucket. This should allow for more graceful generalization to longer + sequences than the model has been trained on + Args: + relative_position: an int32 Tensor + bidirectional: a boolean - whether the attention is bidirectional + num_buckets: an integer + max_distance: an integer + Returns: + a Tensor with the same shape as relative_position, containing int32 + values in the range [0, num_buckets) + """ # noqa: E501 + relative_buckets = 0 + if bidirectional: + num_buckets //= 2 + relative_buckets += (relative_position > 0).to(torch.long) * num_buckets + relative_position = torch.abs(relative_position) + else: + relative_position = -torch.min( + relative_position, torch.zeros_like(relative_position) + ) + # now relative_position is in the range [0, inf) + + # half of the buckets are for exact increments in positions + max_exact = num_buckets // 2 + is_small = relative_position < max_exact + + # The other half of the buckets are for logarithmically bigger bins + # in positions up to max_distance + relative_position_if_large = max_exact + ( + torch.log(relative_position.float() / max_exact) + / math.log(max_distance / max_exact) + * (num_buckets - max_exact) + ).to(torch.long) + relative_position_if_large = torch.min( + relative_position_if_large, + torch.full_like(relative_position_if_large, num_buckets - 1), + ) + + relative_buckets += torch.where( + is_small, relative_position, relative_position_if_large + ) + return relative_buckets + + def compute_bias(self, query_length, key_length, device=None) -> torch.Tensor: + """Compute binned relative position bias""" + if device is None: + device = self.relative_attention_bias.weight.device + context_position = torch.arange(query_length, dtype=torch.long, device=device)[ + :, None + ] + memory_position = torch.arange(key_length, dtype=torch.long, device=device)[ + None, : + ] + # max_seq_len, nh + relative_position = memory_position - context_position + relative_position_bucket = self._relative_position_bucket( + relative_position, # shape (query_length, key_length) + bidirectional=(not self.is_decoder), + num_buckets=self.relative_attention_num_buckets, + max_distance=self.relative_attention_max_distance, + ) + values = self.relative_attention_bias( + relative_position_bucket + ) # shape (query_length, key_length, num_heads) + x = values.permute([2, 0, 1]).unsqueeze( + 0 + ) # shape (1, num_heads, query_length, key_length) + return x + + def forward( + self, + hidden_states: torch.Tensor, # (num_tokens, d_model) + attention_mask: torch.Tensor, + attn_metadata: AttentionMetadata | None = None, + ) -> torch.Tensor: + bs, seq_len, _ = hidden_states.shape + num_seqs = bs + n, c = self.n_heads, self.d_model // self.total_num_heads + qkv, _ = self.qkv_proj(hidden_states) + # Projection of 'own' hidden state (self-attention). No GQA here. + q, k, v = qkv.split(self.inner_dim, dim=-1) + q = q.reshape(bs, seq_len, n, c) + k = k.reshape(bs, seq_len, n, c) + v = v.reshape(bs, seq_len, n, c) + + assert attn_metadata is not None + attn_bias = attn_metadata.attn_bias + # Not compatible with CP here (as all encoder-decoder models), + # as it assumes homogeneous batch (prefills or decodes). + if self.has_relative_attention_bias: + # Self-attention. Compute T5 relative positional encoding. + # The bias term is computed on longest sequence in batch. Biases + # for shorter sequences are slices of the longest. + assert self.attn_type == AttentionType.ENCODER + attn_bias = self.compute_bias(seq_len, seq_len).repeat(num_seqs, 1, 1, 1) + attn_metadata.attn_bias = attn_bias + else: + # Encoder/Decoder Self-Attention Layer, attn bias already cached. + assert attn_bias is not None + + if attention_mask is not None: + attention_mask = ( + attention_mask.view(bs, 1, 1, -1) + if attention_mask.ndim == 2 + else attention_mask.unsqueeze(1) + ) + mask_val = -1e4 if current_platform.is_mps() else torch.finfo(q.dtype).min + attn_bias.masked_fill_(attention_mask == 0, mask_val) + + if get_tp_world_size() > 1: + rank = get_tp_rank() + attn_bias = attn_bias[ + :, rank * self.n_heads : (rank + 1) * self.n_heads, :, : + ] + + attn_output = self.attn(q, k, v, attn_bias) + output, _ = self.o(attn_output) + return output + + +class T5LayerSelfAttention(nn.Module): + + def __init__( + self, + config, + has_relative_attention_bias=False, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ): + super().__init__() + self.SelfAttention = T5Attention( + config, + AttentionType.DECODER if "decoder" in prefix else AttentionType.ENCODER, + has_relative_attention_bias=has_relative_attention_bias, + quant_config=quant_config, + prefix=f"{prefix}.SelfAttention", + ) + self.layer_norm = RMSNorm(config.d_model, eps=config.layer_norm_epsilon) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor, + attn_metadata: AttentionMetadata | None = None, + ) -> torch.Tensor: + normed_hidden_states = self.layer_norm(hidden_states) + + attention_output = self.SelfAttention( + hidden_states=normed_hidden_states, + attention_mask=attention_mask, + attn_metadata=attn_metadata, + ) + + hidden_states = hidden_states + attention_output + + return hidden_states + + +class T5LayerCrossAttention(nn.Module): + + def __init__( + self, config, quant_config: QuantizationConfig | None = None, prefix: str = "" + ): + super().__init__() + self.EncDecAttention = T5Attention( + config, + AttentionType.ENCODER_DECODER, + has_relative_attention_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.EncDecAttention", + ) + self.layer_norm = RMSNorm(config.d_model, eps=config.layer_norm_epsilon) + + def forward( + self, + hidden_states: torch.Tensor, + attn_metadata: AttentionMetadata | None = None, + ) -> torch.Tensor: + normed_hidden_states = self.layer_norm(hidden_states) + attention_output = self.EncDecAttention( + hidden_states=normed_hidden_states, + attn_metadata=attn_metadata, + ) + hidden_states = hidden_states + attention_output + return hidden_states + + +class T5Block(nn.Module): + + def __init__( + self, + config: T5Config, + is_decoder: bool, + has_relative_attention_bias=False, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ): + super().__init__() + self.is_decoder = is_decoder + self.layer = nn.ModuleList() + self.layer.append( + T5LayerSelfAttention( + config, + has_relative_attention_bias=has_relative_attention_bias, + quant_config=quant_config, + prefix=f"{prefix}.self_attn", + ) + ) + + if self.is_decoder: + self.layer.append( + T5LayerCrossAttention( + config, quant_config=quant_config, prefix=f"{prefix}.cross_attn" + ) + ) + + self.layer.append(T5LayerFF(config, quant_config=quant_config)) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor, + attn_metadata: AttentionMetadata | None = None, + ) -> torch.Tensor: + + if attention_mask is None: + attention_mask = torch.ones( + hidden_states.shape[:2], device=hidden_states.device + ) + + hidden_states = self.layer[0]( + hidden_states=hidden_states, + attention_mask=attention_mask, + attn_metadata=attn_metadata, + ) + + if self.is_decoder: + hidden_states = self.layer[1]( + hidden_states=hidden_states, attn_metadata=attn_metadata + ) + + # Apply Feed Forward layer + hidden_states = self.layer[-1](hidden_states) + + return hidden_states + + +class T5Stack(nn.Module): + + def __init__( + self, + config: T5Config, + is_decoder: bool, + n_layers: int, + embed_tokens=None, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + is_umt5: bool = False, + ): + super().__init__() + self.embed_tokens = embed_tokens + self.is_umt5 = is_umt5 + if is_umt5: + self.block = nn.ModuleList( + [ + T5Block( + config, + is_decoder=is_decoder, + has_relative_attention_bias=True, + quant_config=quant_config, + prefix=f"{prefix}.blocks.{i}", + ) + for i in range(n_layers) + ] + ) + else: + # Only the first block has relative positional encoding. + self.block = nn.ModuleList( + [ + T5Block( + config, + is_decoder=is_decoder, + has_relative_attention_bias=i == 0, + quant_config=quant_config, + prefix=f"{prefix}.blocks.{i}", + ) + for i in range(n_layers) + ] + ) + self.final_layer_norm = RMSNorm(config.d_model, eps=config.layer_norm_epsilon) + + def forward( + self, + input_ids: torch.Tensor, + attention_mask: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + hidden_states = self.embed_tokens(input_ids) + + for idx, block in enumerate(self.block): + hidden_states = block( + hidden_states=hidden_states, + attention_mask=attention_mask, + attn_metadata=attn_metadata, + ) + + hidden_states = self.final_layer_norm(hidden_states) + return hidden_states + + +class T5EncoderModel(TextEncoder): + + def __init__(self, config: T5Config, prefix: str = ""): + super().__init__(config) + + quant_config = None + + self.shared = VocabParallelEmbedding( + config.vocab_size, config.d_model, org_num_embeddings=config.vocab_size + ) + + self.encoder = T5Stack( + config, + False, + config.num_layers, + self.shared, + quant_config=quant_config, + prefix=f"{prefix}.encoder", + is_umt5=False, + ) + + def get_input_embeddings(self): + return self.shared + + def forward( + self, + input_ids: torch.Tensor | None, + position_ids: torch.Tensor | None = None, + attention_mask: torch.Tensor | None = None, + inputs_embeds: torch.Tensor | None = None, + output_hidden_states: bool | None = None, + **kwargs, + ) -> BaseEncoderOutput: + attn_metadata = AttentionMetadata(None) + hidden_states = self.encoder( + input_ids=input_ids, + attention_mask=attention_mask, + attn_metadata=attn_metadata, + ) + + return BaseEncoderOutput(last_hidden_state=hidden_states) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + (".qkv_proj", ".q", "q"), + (".qkv_proj", ".k", "k"), + (".qkv_proj", ".v", "v"), + ] + params_dict = dict(self.named_parameters()) + loaded_params: set[str] = set() + for name, loaded_weight in weights: + loaded = False + if "decoder" in name or "lm_head" in name: + continue + for param_name, weight_name, shard_id in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + + if name not in params_dict: + continue + + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + loaded = True + break + if not loaded: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + + if name not in params_dict: + continue + + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(name) + return loaded_params + + +class UMT5EncoderModel(TextEncoder): + + def __init__(self, config: T5Config, prefix: str = ""): + super().__init__(config) + + quant_config = None + + self.shared = VocabParallelEmbedding( + config.vocab_size, config.d_model, org_num_embeddings=config.vocab_size + ) + + self.encoder = T5Stack( + config, + False, + config.num_layers, + self.shared, + quant_config=quant_config, + prefix=f"{prefix}.encoder", + is_umt5=True, + ) + + def get_input_embeddings(self): + return self.shared + + def forward( + self, + input_ids: torch.Tensor | None, + position_ids: torch.Tensor | None = None, + attention_mask: torch.Tensor | None = None, + inputs_embeds: torch.Tensor | None = None, + output_hidden_states: bool | None = None, + **kwargs, + ) -> BaseEncoderOutput: + attn_metadata = AttentionMetadata(None) + hidden_states = self.encoder( + input_ids=input_ids, + attention_mask=attention_mask, + attn_metadata=attn_metadata, + ) + + return BaseEncoderOutput( + last_hidden_state=hidden_states, + attention_mask=attention_mask, + ) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + params_dict = dict(self.named_parameters()) + loaded_params: set[str] = set() + for name, loaded_weight in weights: + loaded = False + if "decoder" in name or "lm_head" in name: + continue + for ( + param_name, + weight_name, + shard_id, + ) in self.config.arch_config.stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + + if name not in params_dict: + continue + + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + loaded = True + break + if not loaded: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + + if name not in params_dict: + continue + + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(name) + return loaded_params + + +EntryClass = [T5EncoderModel, UMT5EncoderModel] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/vision.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/vision.py new file mode 100644 index 00000000..3150abf1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/encoders/vision.py @@ -0,0 +1,96 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/model_executor/models/vision.py + +from abc import ABC, abstractmethod +from typing import Generic, TypeVar + +import torch +from transformers import PretrainedConfig + +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + +_C = TypeVar("_C", bound=PretrainedConfig) + + +class VisionEncoderInfo(ABC, Generic[_C]): + + def __init__(self, vision_config: _C) -> None: + super().__init__() + + self.vision_config = vision_config + + @abstractmethod + def get_num_image_tokens( + self, + *, + image_width: int, + image_height: int, + ) -> int: + raise NotImplementedError + + @abstractmethod + def get_max_image_tokens(self) -> int: + raise NotImplementedError + + @abstractmethod + def get_image_size(self) -> int: + raise NotImplementedError + + @abstractmethod + def get_patch_size(self) -> int: + raise NotImplementedError + + @abstractmethod + def get_patch_grid_length(self) -> int: + raise NotImplementedError + + +def resolve_visual_encoder_outputs( + encoder_outputs: torch.Tensor | list[torch.Tensor], + feature_sample_layers: list[int] | None, + post_layer_norm: torch.nn.LayerNorm | None, + max_possible_layers: int, +) -> torch.Tensor: + """Given the outputs a visual encoder module that may correspond to the + output of the last layer, or a list of hidden states to be stacked, + handle post normalization and resolve it into a single output tensor. + + Args: + encoder_outputs: Output of encoder's last layer or all hidden states. + feature_sample_layers: Optional layer indices to grab from the encoder + outputs; if provided, encoder outputs must be a list. + post_layer_norm: Post norm to apply to the output of the encoder. + max_possible_layers: Total layers in the fully loaded visual encoder. + + """ + if feature_sample_layers is None: + if post_layer_norm is not None: + return post_layer_norm(encoder_outputs) + return encoder_outputs + + # Get the hidden states corresponding to the layer indices. + # Negative values are relative to the full visual encoder, + # so offset them depending on how many layers were loaded. + # NOTE: this assumes that encoder_outputs is a list containing + # the inputs to the visual encoder, followed by the hidden states + # of each layer. + num_loaded_layers = len(encoder_outputs) - 1 + offset = max_possible_layers - num_loaded_layers + hs_pool = [ + ( + encoder_outputs[layer_idx] + if layer_idx >= 0 + else encoder_outputs[layer_idx + offset] + ) + for layer_idx in feature_sample_layers + ] + + # Apply post-norm on the final hidden state if we are using it + uses_last_layer = feature_sample_layers[-1] in (len(hs_pool) - 1, -1) + if post_layer_norm is not None and uses_last_layer: + hs_pool[-1] = post_layer_norm(encoder_outputs) + return torch.cat(hs_pool, dim=-1) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/parameter.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/parameter.py new file mode 100644 index 00000000..ba9b42c6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/parameter.py @@ -0,0 +1,423 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/model_executor/parameter.py + +from collections.abc import Callable +from fractions import Fraction +from typing import Any + +import torch +from torch.nn import Parameter + +from sglang.multimodal_gen.runtime.distributed import get_tp_rank +from sglang.multimodal_gen.runtime.models.utils import _make_synced_weight_loader +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class BasevLLMParameter(Parameter): + """ + Base parameter for vLLM linear layers. Extends the torch.nn.parameter + by taking in a linear weight loader. Will copy the loaded weight + into the parameter when the provided weight loader is called. + """ + + def __new__(cls, data: torch.Tensor, **kwargs): + + return super().__new__(cls, data=data, requires_grad=False) + + def __init__(self, data: torch.Tensor, weight_loader: Callable): + """ + Initialize the BasevLLMParameter + + :param data: torch tensor with the parameter data + :param weight_loader: weight loader callable + + :returns: a torch.nn.parameter + """ + + # During weight loading, we often do something like: + # narrowed_tensor = param.data.narrow(0, offset, len) + # narrowed_tensor.copy_(real_weight) + # expecting narrowed_tensor and param.data to share the same storage. + # However, on TPUs, narrowed_tensor will lazily propagate to the base + # tensor, which is param.data, leading to the redundant memory usage. + # This sometimes causes OOM errors during model loading. To avoid this, + # we sync the param tensor after its weight loader is called. + from sglang.multimodal_gen.runtime.platforms import current_platform + + if current_platform.is_tpu(): + weight_loader = _make_synced_weight_loader(weight_loader) + + self._weight_loader = weight_loader + + @property + def weight_loader(self): + return self._weight_loader + + def _is_1d_and_scalar(self, loaded_weight: torch.Tensor): + cond1 = self.data.ndim == 1 and self.data.numel() == 1 + cond2 = loaded_weight.ndim == 0 and loaded_weight.numel() == 1 + return cond1 and cond2 + + def _assert_and_load(self, loaded_weight: torch.Tensor) -> None: + assert self.data.shape == loaded_weight.shape or self._is_1d_and_scalar( + loaded_weight + ) + self.data.copy_(loaded_weight) + + def load_column_parallel_weight(self, loaded_weight: torch.Tensor) -> None: + self._assert_and_load(loaded_weight) + + def load_row_parallel_weight(self, loaded_weight: torch.Tensor) -> None: + self._assert_and_load(loaded_weight) + + def load_merged_column_weight(self, loaded_weight: torch.Tensor, **kwargs) -> None: + self._assert_and_load(loaded_weight) + + def load_qkv_weight(self, loaded_weight: torch.Tensor, **kwargs) -> None: + self._assert_and_load(loaded_weight) + + +class _ColumnvLLMParameter(BasevLLMParameter): + """ + Private class defining weight loading functionality + (load_merged_column_weight, load_qkv_weight) + for parameters being loaded into linear layers with column + parallelism. This includes QKV and MLP layers which are + not already fused on disk. Requires an output dimension + to be defined. Called within the weight loader of + each of the column parallel linear layers. + """ + + def __init__(self, output_dim: int, **kwargs): + self._output_dim = output_dim + super().__init__(**kwargs) + + @property + def output_dim(self): + return self._output_dim + + def load_column_parallel_weight(self, loaded_weight: torch.Tensor) -> None: + tp_rank = get_tp_rank() + shard_size = self.data.shape[self.output_dim] + loaded_weight = loaded_weight.narrow( + self.output_dim, tp_rank * shard_size, shard_size + ) + assert self.data.shape == loaded_weight.shape + self.data.copy_(loaded_weight) + + def load_merged_column_weight(self, loaded_weight: torch.Tensor, **kwargs) -> None: + + shard_offset = kwargs.get("shard_offset") + shard_size = kwargs.get("shard_size") + if shard_offset is None or shard_size is None: + raise ValueError("shard_offset and shard_size must be provided") + if ( + isinstance(self, PackedColumnParameter | PackedvLLMParameter) + and self.packed_dim == self.output_dim + ): + shard_size, shard_offset = self.adjust_shard_indexes_for_packing( + shard_offset=shard_offset, shard_size=shard_size + ) + + param_data = self.data + + tp_rank = get_tp_rank() + param_data = param_data.narrow(self.output_dim, shard_offset, shard_size) + loaded_weight = loaded_weight.narrow( + self.output_dim, tp_rank * shard_size, shard_size + ) + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + def load_qkv_weight(self, loaded_weight: torch.Tensor, **kwargs) -> None: + + shard_offset = kwargs.get("shard_offset") + shard_size = kwargs.get("shard_size") + shard_id = kwargs.get("shard_id") + num_heads = kwargs.get("num_heads") + + assert shard_offset is not None + assert shard_size is not None + assert shard_id is not None + assert num_heads is not None + + if ( + isinstance(self, PackedColumnParameter | PackedvLLMParameter) + and self.output_dim == self.packed_dim + ): + shard_size, shard_offset = self.adjust_shard_indexes_for_packing( + shard_offset=shard_offset, shard_size=shard_size + ) + + param_data = self.data + tp_rank = get_tp_rank() + shard_id = tp_rank if shard_id == "q" else tp_rank // num_heads + param_data = param_data.narrow(self.output_dim, shard_offset, shard_size) + loaded_weight = loaded_weight.narrow( + self.output_dim, shard_id * shard_size, shard_size + ) + + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + +class RowvLLMParameter(BasevLLMParameter): + """ + Parameter class defining weight_loading functionality + (load_row_parallel_weight) for parameters being loaded + into linear layers with row parallel functionality. + Requires an input_dim to be defined. + """ + + def __init__(self, input_dim: int, **kwargs): + self._input_dim = input_dim + super().__init__(**kwargs) + + @property + def input_dim(self): + return self._input_dim + + def load_row_parallel_weight(self, loaded_weight: torch.Tensor) -> None: + tp_rank = get_tp_rank() + shard_size = self.data.shape[self.input_dim] + loaded_weight = loaded_weight.narrow( + self.input_dim, tp_rank * shard_size, shard_size + ) + + if len(loaded_weight.shape) == 0: + loaded_weight = loaded_weight.reshape(1) + + assert self.data.shape == loaded_weight.shape + self.data.copy_(loaded_weight) + + +class ModelWeightParameter(_ColumnvLLMParameter, RowvLLMParameter): + """ + Parameter class for linear layer weights. Uses both column and + row parallelism. + """ + + pass + + +class GroupQuantScaleParameter(_ColumnvLLMParameter, RowvLLMParameter): + """ + Parameter class for weight scales loaded for weights with + grouped quantization. Uses both column and row parallelism. + """ + + pass + + +class ChannelQuantScaleParameter(_ColumnvLLMParameter): + """ + Parameter class for weight scales loaded for weights with + channel-wise quantization. Equivalent to _ColumnvLLMParameter. + """ + + pass + + +class PerTensorScaleParameter(BasevLLMParameter): + """ + Parameter class for scales where the number of scales is + equivalent to the number of logical matrices in fused linear + layers (e.g. for QKV, there are 3 scales loaded from disk). + This is relevant to weights with per-tensor quantization. + Adds functionality to map the scalers to a shard during + weight loading. + + Note: additional parameter manipulation may be handled + for each quantization config specifically, within + process_weights_after_loading + """ + + def __init__(self, **kwargs): + self.qkv_idxs = {"q": 0, "k": 1, "v": 2} + super().__init__(**kwargs) + + def _shard_id_as_int(self, shard_id: str | int) -> int: + if isinstance(shard_id, int): + return shard_id + + # if not int, assume shard_id for qkv + # map to int and return + assert isinstance(shard_id, str) + assert shard_id in self.qkv_idxs + return self.qkv_idxs[shard_id] + + # For row parallel layers, no sharding needed + # load weight into parameter as is + def load_row_parallel_weight(self, *args, **kwargs) -> None: + super().load_row_parallel_weight(*args, **kwargs) + + def load_merged_column_weight(self, *args, **kwargs) -> None: + self._load_into_shard_id(*args, **kwargs) + + def load_qkv_weight(self, *args, **kwargs) -> None: + self._load_into_shard_id(*args, **kwargs) + + def load_column_parallel_weight(self, *args, **kwargs) -> None: + super().load_row_parallel_weight(*args, **kwargs) + + def _load_into_shard_id( + self, loaded_weight: torch.Tensor, shard_id: str | int, **kwargs + ): + """ + Slice the parameter data based on the shard id for + loading. + """ + + param_data = self.data + shard_id = self._shard_id_as_int(shard_id) + + # AutoFP8 scales do not have a shape + # compressed-tensors scales do have a shape + if len(loaded_weight.shape) != 0: + assert loaded_weight.shape[0] == 1 + loaded_weight = loaded_weight[0] + + param_data = param_data[shard_id] + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + +class PackedColumnParameter(_ColumnvLLMParameter): + """ + Parameter for model parameters which are packed on disk + and support column parallelism only. See PackedvLLMParameter + for more details on the packed properties. + """ + + def __init__(self, packed_factor: int | Fraction, packed_dim: int, **kwargs): + self._packed_factor = packed_factor + self._packed_dim = packed_dim + super().__init__(**kwargs) + + @property + def packed_dim(self): + return self._packed_dim + + @property + def packed_factor(self): + return self._packed_factor + + def adjust_shard_indexes_for_packing( + self, shard_size, shard_offset + ) -> tuple[Any, Any]: + return _adjust_shard_indexes_for_packing( + shard_size=shard_size, + shard_offset=shard_offset, + packed_factor=self.packed_factor, + ) + + +class PackedvLLMParameter(ModelWeightParameter): + """ + Parameter for model weights which are packed on disk. + Example: GPTQ Marlin weights are int4 or int8, packed into int32. + Extends the ModelWeightParameter to take in the + packed factor, the packed dimension, and optionally, marlin + tile size for marlin kernels. Adjusts the shard_size and + shard_offset for fused linear layers model weight loading + by accounting for packing and optionally, marlin tile size. + """ + + def __init__(self, packed_factor: int | Fraction, packed_dim: int, **kwargs): + self._packed_factor = packed_factor + self._packed_dim = packed_dim + super().__init__(**kwargs) + + @property + def packed_dim(self): + return self._packed_dim + + @property + def packed_factor(self): + return self._packed_factor + + def adjust_shard_indexes_for_packing(self, shard_size, shard_offset): + return _adjust_shard_indexes_for_packing( + shard_size=shard_size, + shard_offset=shard_offset, + packed_factor=self.packed_factor, + ) + + +class BlockQuantScaleParameter(_ColumnvLLMParameter, RowvLLMParameter): + """ + Parameter class for weight scales loaded for weights with + block-wise quantization. Uses both column and row parallelism. + """ + + pass + + +def permute_param_layout_( + param: BasevLLMParameter, input_dim: int, output_dim: int, **kwargs +) -> BasevLLMParameter: + """ + Permute a parameter's layout to the specified input and output dimensions, + useful for forcing the parameter into a known layout, for example, if I need + a packed (quantized) weight matrix to be in the layout + {input_dim = 0, output_dim = 1, packed_dim = 0} + then I can call: + permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0) + to ensure x is in the correct layout (permuting it to the correct layout if + required, asserting if it cannot get it to the correct layout) + """ + + curr_input_dim = getattr(param, "input_dim", None) + curr_output_dim = getattr(param, "output_dim", None) + + if curr_input_dim is None or curr_output_dim is None: + assert param.data.dim() == 2, ( + "permute_param_layout_ only supports 2D parameters when either " + "input_dim or output_dim is not set" + ) + + # if one of the dimensions is not set, set it to the opposite of the other + # we can only do this since we asserted the parameter is 2D above + if curr_input_dim is None: + assert curr_output_dim is not None, "either input or output dim must be set" + curr_input_dim = (curr_output_dim + 1) % 2 + if curr_output_dim is None: + assert curr_input_dim is not None, "either input or output dim must be set" + curr_output_dim = (curr_input_dim + 1) % 2 + + # create permutation from the current layout to the layout with + # self.input_dim at input_dim and self.output_dim at output_dim preserving + # other dimensions + perm = [ + i for i in range(param.data.dim()) if i not in [curr_input_dim, curr_output_dim] + ] + perm.insert(input_dim, curr_input_dim) + perm.insert(output_dim, curr_output_dim) + + if "packed_dim" in kwargs: + assert ( + hasattr(param, "packed_dim") + and param.packed_dim == perm[kwargs["packed_dim"]] + ), "permute_param_layout_ currently doesn't support repacking" + + param.data = param.data.permute(*perm) + if hasattr(param, "_input_dim"): + param._input_dim = input_dim + if hasattr(param, "_output_dim"): + param._output_dim = output_dim + if "packed_dim" in kwargs and hasattr(param, "_packed_dim"): + param._packed_dim = kwargs["packed_dim"] + + return param + + +def _adjust_shard_indexes_for_packing( + shard_size, shard_offset, packed_factor +) -> tuple[Any, Any]: + shard_size = shard_size // packed_factor + shard_offset = shard_offset // packed_factor + return shard_size, shard_offset diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/registry.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/registry.py new file mode 100644 index 00000000..2ee26257 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/registry.py @@ -0,0 +1,365 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/model_executor/models/registry.py + +import ast +import importlib +import os +import pickle +import subprocess +import sys +import tempfile +from abc import ABC, abstractmethod +from collections.abc import Callable, Set +from dataclasses import dataclass, field +from functools import lru_cache +from typing import NoReturn, TypeVar, cast + +import cloudpickle +from torch import nn + +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + +MODELS_PATH = os.path.dirname(__file__) +COMPONENT_DIRS = [ + d + for d in os.listdir(MODELS_PATH) + if os.path.isdir(os.path.join(MODELS_PATH, d)) + and not d.startswith("__") + and not d.startswith(".") +] + +_IMAGE_ENCODER_MODELS: dict[str, tuple] = { + # "HunyuanVideoTransformer3DModel": ("image_encoder", "hunyuanvideo", "HunyuanVideoImageEncoder"), + "CLIPVisionModelWithProjection": ("encoders", "clip", "CLIPVisionModel"), +} + + +@lru_cache(maxsize=None) +def _discover_and_register_models() -> dict[str, tuple[str, str, str]]: + discovered_models = _IMAGE_ENCODER_MODELS + for component in COMPONENT_DIRS: + component_path = os.path.join(MODELS_PATH, component) + for filename in os.listdir(component_path): + if not filename.endswith(".py"): + continue + + mod_relname = filename[:-3] + filepath = os.path.join(component_path, filename) + try: + with open(filepath, "r", encoding="utf-8") as f: + source = f.read() + tree = ast.parse(source, filename=filename) + + entry_class_node = None + first_class_def = None + + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + for target in node.targets: + if ( + isinstance(target, ast.Name) + and target.id == "EntryClass" + ): + entry_class_node = node + break + if first_class_def is None and isinstance(node, ast.ClassDef): + first_class_def = node + if entry_class_node and first_class_def: + model_cls_name_list = [] + value_node = entry_class_node.value + + # EntryClass = ClassName + if isinstance(value_node, ast.Name): + model_cls_name_list.append(value_node.id) + # EntryClass = ["...", ClassName, ...] + elif isinstance(value_node, (ast.List, ast.Tuple)): + for elt in value_node.elts: + if isinstance(elt, ast.Constant): + model_cls_name_list.append(elt.value) + elif isinstance(elt, ast.Name): + model_cls_name_list.append(elt.id) + + if model_cls_name_list: + for model_cls_str in model_cls_name_list: + if model_cls_str in discovered_models: + logger.warning( + f"Duplicate architecture found: {model_cls_str}. It will be overwritten." + ) + model_arch = model_cls_str + discovered_models[model_arch] = ( + component, + mod_relname, + model_cls_str, + ) + + except Exception as e: + logger.warning(f"Could not parse {filepath} to find models: {e}") + + return discovered_models + + +_SGLANG_DIFFUSION_MODELS = _discover_and_register_models() + +_SUBPROCESS_COMMAND = [ + sys.executable, + "-m", + "sglang.multimodal_gen.runtime.models.dits.registry", +] + +_T = TypeVar("_T") + + +@dataclass(frozen=True) +class _ModelInfo: + architecture: str + + @staticmethod + def from_model_cls(model: type[nn.Module]) -> "_ModelInfo": + return _ModelInfo( + architecture=model.__name__, + ) + + +class _BaseRegisteredModel(ABC): + + @abstractmethod + def inspect_model_cls(self) -> _ModelInfo: + raise NotImplementedError + + @abstractmethod + def load_model_cls(self) -> type[nn.Module]: + raise NotImplementedError + + +@dataclass(frozen=True) +class _RegisteredModel(_BaseRegisteredModel): + """ + Represents a model that has already been imported in the main process. + """ + + interfaces: _ModelInfo + model_cls: type[nn.Module] + + @staticmethod + def from_model_cls(model_cls: type[nn.Module]): + return _RegisteredModel( + interfaces=_ModelInfo.from_model_cls(model_cls), + model_cls=model_cls, + ) + + def inspect_model_cls(self) -> _ModelInfo: + return self.interfaces + + def load_model_cls(self) -> type[nn.Module]: + return self.model_cls + + +def _run_in_subprocess(fn: Callable[[], _T]) -> _T: + # NOTE: We use a temporary directory instead of a temporary file to avoid + # issues like https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file + with tempfile.TemporaryDirectory() as tempdir: + output_filepath = os.path.join(tempdir, "registry_output.tmp") + + # `cloudpickle` allows pickling lambda functions directly + input_bytes = cloudpickle.dumps((fn, output_filepath)) + + # cannot use `sys.executable __file__` here because the script + # contains relative imports + returned = subprocess.run( + _SUBPROCESS_COMMAND, input=input_bytes, capture_output=True + ) + + # check if the subprocess is successful + try: + returned.check_returncode() + except Exception as e: + # wrap raised exception to provide more information + raise RuntimeError( + f"Error raised in subprocess:\n" f"{returned.stderr.decode()}" + ) from e + + with open(output_filepath, "rb") as f: + return cast(_T, pickle.load(f)) + + +@dataclass(frozen=True) +class _LazyRegisteredModel(_BaseRegisteredModel): + """ + Represents a model that has not been imported in the main process. + """ + + module_name: str + component_name: str + class_name: str + + # Performed in another process to avoid initializing CUDA + def inspect_model_cls(self) -> _ModelInfo: + return _run_in_subprocess( + lambda: _ModelInfo.from_model_cls(self.load_model_cls()) + ) + + def load_model_cls(self) -> type[nn.Module]: + mod = importlib.import_module(self.module_name) + return cast(type[nn.Module], getattr(mod, self.class_name)) + + +@lru_cache(maxsize=128) +def _try_load_model_cls( + model_arch: str, + model: _BaseRegisteredModel, +) -> type[nn.Module] | None: + from sglang.multimodal_gen.runtime.platforms import current_platform + + current_platform.verify_model_arch(model_arch) + try: + return model.load_model_cls() + except Exception: + logger.exception("Ignore import error when loading '%s'", model_arch) + return None + + +@lru_cache(maxsize=128) +def _try_inspect_model_cls( + model_arch: str, + model: _BaseRegisteredModel, +) -> _ModelInfo | None: + try: + return model.inspect_model_cls() + except Exception: + logger.exception("Error in inspecting model architecture '%s'", model_arch) + return None + + +@dataclass +class _ModelRegistry: + # Keyed by model_arch + registered_models: dict[str, _BaseRegisteredModel] = field(default_factory=dict) + + def get_supported_archs(self) -> Set[str]: + return self.registered_models.keys() + + def register_model( + self, + model_arch: str, + model_cls: type[nn.Module] | str, + ) -> None: + """ + Register an external model to be used in vLLM. + + :code:`model_cls` can be either: + + - A :class:`torch.nn.Module` class directly referencing the model. + - A string in the format :code:`:` which can be used to + lazily import the model. This is useful to avoid initializing CUDA + when importing the model and thus the related error + :code:`RuntimeError: Cannot re-initialize CUDA in forked subprocess`. + """ + if model_arch in self.registered_models: + logger.warning( + "Model architecture %s is already registered, and will be " + "overwritten by the new model class %s.", + model_arch, + model_cls, + ) + + if isinstance(model_cls, str): + split_str = model_cls.split(":") + if len(split_str) != 2: + msg = "Expected a string in the format `:`" + raise ValueError(msg) + + model = _LazyRegisteredModel(*split_str) + else: + model = _RegisteredModel.from_model_cls(model_cls) + + self.registered_models[model_arch] = model + + def _raise_for_unsupported(self, architectures: list[str]) -> NoReturn: + all_supported_archs = self.get_supported_archs() + + if any(arch in all_supported_archs for arch in architectures): + raise ValueError( + f"Model architectures {architectures} failed " + "to be inspected. Please check the logs for more details." + ) + + raise ValueError( + f"Model architectures {architectures} are not supported for now. " + f"Supported architectures: {all_supported_archs}" + ) + + def _try_load_model_cls(self, model_arch: str) -> type[nn.Module] | None: + if model_arch not in self.registered_models: + return None + + return _try_load_model_cls(model_arch, self.registered_models[model_arch]) + + def _try_inspect_model_cls(self, model_arch: str) -> _ModelInfo | None: + if model_arch not in self.registered_models: + return None + + return _try_inspect_model_cls(model_arch, self.registered_models[model_arch]) + + def _normalize_archs( + self, + architectures: str | list[str], + ) -> list[str]: + if isinstance(architectures, str): + architectures = [architectures] + if not architectures: + logger.warning("No model architectures are specified") + + normalized_arch = [] + for arch in architectures: + if arch not in self.registered_models: + raise Exception( + f"Unsupported model architecture: {arch}. Registered architectures: {self.registered_models=}" + ) + normalized_arch.append(arch) + return normalized_arch + + def inspect_model_cls( + self, + architectures: str | list[str], + ) -> tuple[_ModelInfo, str]: + architectures = self._normalize_archs(architectures) + + for arch in architectures: + model_info = self._try_inspect_model_cls(arch) + if model_info is not None: + return (model_info, arch) + + return self._raise_for_unsupported(architectures) + + def resolve_model_cls( + self, + architectures: str | list[str], + ) -> tuple[type[nn.Module], str]: + architectures = self._normalize_archs(architectures) + + for arch in architectures: + model_cls = self._try_load_model_cls(arch) + if model_cls is not None: + return (model_cls, arch) + + return self._raise_for_unsupported(architectures) + + +ModelRegistry = _ModelRegistry( + { + model_arch: _LazyRegisteredModel( + module_name=f"sglang.multimodal_gen.runtime.models.{component_name}.{mod_relname}", + component_name=component_name, + class_name=cls_name, + ) + for model_arch, ( + component_name, + mod_relname, + cls_name, + ) in _SGLANG_DIFFUSION_MODELS.items() + } +) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/schedulers/base.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/schedulers/base.py new file mode 100644 index 00000000..eb4e3bdd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/schedulers/base.py @@ -0,0 +1,37 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +from abc import ABC, abstractmethod + +import torch + + +class BaseScheduler(ABC): + timesteps: torch.Tensor + order: int + num_train_timesteps: int + + def __init__(self, *args, **kwargs) -> None: + # Check if subclass has defined all required properties + required_attributes = ["timesteps", "order", "num_train_timesteps"] + + for attr in required_attributes: + if not hasattr(self, attr): + raise AttributeError( + f"Subclasses of BaseScheduler must define '{attr}' property" + ) + + @abstractmethod + def set_shift(self, shift: float) -> None: + pass + + @abstractmethod + def set_timesteps(self, *args, **kwargs) -> None: + pass + + @abstractmethod + def scale_model_input( + self, sample: torch.Tensor, timestep: int | None = None + ) -> torch.Tensor: + pass diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_flow_match_euler_discrete.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_flow_match_euler_discrete.py new file mode 100644 index 00000000..d184802b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_flow_match_euler_discrete.py @@ -0,0 +1,698 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +# Copyright 2024 Stability AI, Katherine Crowson and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# +# Modified from diffusers==0.29.2 +# +# ============================================================================== +import math +from dataclasses import dataclass +from typing import Any + +import numpy as np +import scipy.stats +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.schedulers.scheduling_utils import SchedulerMixin +from diffusers.utils import BaseOutput + +from sglang.multimodal_gen.runtime.models.schedulers.base import BaseScheduler +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +@dataclass +class FlowMatchEulerDiscreteSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + """ + + prev_sample: torch.FloatTensor + + +class FlowMatchEulerDiscreteScheduler(SchedulerMixin, ConfigMixin, BaseScheduler): + """ + Euler scheduler. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + shift (`float`, defaults to 1.0): + The shift value for the timestep schedule. + use_dynamic_shifting (`bool`, defaults to False): + Whether to apply timestep shifting on-the-fly based on the image resolution. + base_shift (`float`, defaults to 0.5): + Value to stabilize image generation. Increasing `base_shift` reduces variation and image is more consistent + with desired output. + max_shift (`float`, defaults to 1.15): + Value change allowed to latent vectors. Increasing `max_shift` encourages more variation and image may be + more exaggerated or stylized. + base_image_seq_len (`int`, defaults to 256): + The base image sequence length. + max_image_seq_len (`int`, defaults to 4096): + The maximum image sequence length. + invert_sigmas (`bool`, defaults to False): + Whether to invert the sigmas. + shift_terminal (`float`, defaults to None): + The end value of the shifted timestep schedule. + use_karras_sigmas (`bool`, defaults to False): + Whether to use Karras sigmas for step sizes in the noise schedule during sampling. + use_exponential_sigmas (`bool`, defaults to False): + Whether to use exponential sigmas for step sizes in the noise schedule during sampling. + use_beta_sigmas (`bool`, defaults to False): + Whether to use beta sigmas for step sizes in the noise schedule during sampling. + time_shift_type (`str`, defaults to "exponential"): + The type of dynamic resolution-dependent timestep shifting to apply. Either "exponential" or "linear". + stochastic_sampling (`bool`, defaults to False): + Whether to use stochastic sampling. + """ + + _compatibles: list[Any] = [] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + shift: float = 1.0, + use_dynamic_shifting: bool = False, + base_shift: float | None = 0.5, + max_shift: float | None = 1.15, + base_image_seq_len: int | None = 256, + max_image_seq_len: int | None = 4096, + invert_sigmas: bool = False, + shift_terminal: float | None = None, + use_karras_sigmas: bool | None = False, + use_exponential_sigmas: bool | None = False, + use_beta_sigmas: bool | None = False, + time_shift_type: str = "exponential", + stochastic_sampling: bool = False, + ): + if ( + sum( + [ + self.config.use_beta_sigmas, + self.config.use_exponential_sigmas, + self.config.use_karras_sigmas, + ] + ) + > 1 + ): + raise ValueError( + "Only one of `config.use_beta_sigmas`, `config.use_exponential_sigmas`, `config.use_karras_sigmas` can be used." + ) + if time_shift_type not in {"exponential", "linear"}: + raise ValueError( + "`time_shift_type` must either be 'exponential' or 'linear'." + ) + + timesteps = np.linspace( + 1, num_train_timesteps, num_train_timesteps, dtype=np.float32 + )[::-1].copy() + timesteps = torch.from_numpy(timesteps).to(dtype=torch.float32) + + sigmas = timesteps / num_train_timesteps + if not use_dynamic_shifting: + # when use_dynamic_shifting is True, we apply the timestep shifting on the fly based on the image resolution + sigmas = shift * sigmas / (1 + (shift - 1) * sigmas) + + self.timesteps = sigmas * num_train_timesteps + self.num_train_timesteps = num_train_timesteps + + self._step_index: int | None = None + self._begin_index: int | None = None + + self._shift = shift + + self.sigmas = sigmas.to("cpu") # to avoid too much CPU/GPU communication + self.sigma_min = self.sigmas[-1].item() + self.sigma_max = self.sigmas[0].item() + BaseScheduler.__init__(self) + + @property + def shift(self) -> float: + """ + The value used for shifting. + """ + return self._shift + + @property + def step_index(self) -> int | None: + """ + The index counter for current timestep. It will increase 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self) -> int | None: + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0) -> None: + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def set_shift(self, shift: float) -> None: + self._shift = shift + + def scale_noise( + self, + sample: torch.FloatTensor, + timestep: float | torch.FloatTensor, + noise: torch.FloatTensor | None = None, + ) -> torch.FloatTensor: + """ + Forward process in flow-matching + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`int`, *optional*): + The current timestep in the diffusion chain. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + # Make sure sigmas and timesteps have the same device and dtype as original_samples + sigmas = self.sigmas.to(device=sample.device, dtype=sample.dtype) + + if sample.device.type == "mps" and torch.is_floating_point(timestep): + # mps does not support float64 + schedule_timesteps = self.timesteps.to(sample.device, dtype=torch.float32) + assert isinstance(timestep, torch.Tensor) + timestep = timestep.to(sample.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(sample.device) + assert isinstance(timestep, torch.Tensor) + timestep = timestep.to(sample.device) + + # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index + if self.begin_index is None: + step_indices = [ + self.index_for_timestep(t, schedule_timesteps) for t in timestep + ] + elif self.step_index is not None: + # add_noise is called after first denoising step (for inpainting) + step_indices = [self.step_index] * timestep.shape[0] + else: + # add noise is called before first denoising step to create initial latent(img2img) + step_indices = [self.begin_index] * timestep.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(sample.shape): + sigma = sigma.unsqueeze(-1) + + sample = sigma * noise + (1.0 - sigma) * sample + + return sample + + def _sigma_to_t(self, sigma: float) -> float: + return sigma * self.config.num_train_timesteps + + def time_shift( + self, mu: float, sigma: float, t: torch.Tensor | np.ndarray + ) -> torch.Tensor | np.ndarray: + if self.config.time_shift_type == "exponential": + return self._time_shift_exponential(mu, sigma, t) + elif self.config.time_shift_type == "linear": + return self._time_shift_linear(mu, sigma, t) + else: + raise ValueError(f"Unknown time_shift_type: {self.config.time_shift_type}") + + def stretch_shift_to_terminal(self, t: torch.Tensor) -> torch.Tensor: + r""" + Stretches and shifts the timestep schedule to ensure it terminates at the configured `shift_terminal` config + value. + + Reference: + https://github.com/Lightricks/LTX-Video/blob/a01a171f8fe3d99dce2728d60a73fecf4d4238ae/ltx_video/schedulers/rf.py#L51 + + Args: + t (`torch.Tensor`): + A tensor of timesteps to be stretched and shifted. + + Returns: + `torch.Tensor`: + A tensor of adjusted timesteps such that the final value equals `self.config.shift_terminal`. + """ + one_minus_z = 1 - t + scale_factor = one_minus_z[-1] / (1 - self.config.shift_terminal) + stretched_t = 1 - (one_minus_z / scale_factor) + return stretched_t + + def set_timesteps( + self, + num_inference_steps: int | None = None, + device: str | torch.device = None, + sigmas: list[float] | None = None, + mu: float | None = None, + timesteps: list[float] | None = None, + ) -> None: + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`, *optional*): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + sigmas (`List[float]`, *optional*): + Custom values for sigmas to be used for each diffusion step. If `None`, the sigmas are computed + automatically. + mu (`float`, *optional*): + Determines the amount of shifting applied to sigmas when performing resolution-dependent timestep + shifting. + timesteps (`List[float]`, *optional*): + Custom values for timesteps to be used for each diffusion step. If `None`, the timesteps are computed + automatically. + """ + + if self.config.use_dynamic_shifting and mu is None: + raise ValueError( + "`mu` must be passed when `use_dynamic_shifting` is set to be `True`" + ) + + if ( + sigmas is not None + and timesteps is not None + and len(sigmas) != len(timesteps) + ): + raise ValueError("`sigmas` and `timesteps` should have the same length") + + if num_inference_steps is not None: + if (sigmas is not None and len(sigmas) != num_inference_steps) or ( + timesteps is not None and len(timesteps) != num_inference_steps + ): + raise ValueError( + "`sigmas` and `timesteps` should have the same length as num_inference_steps, if `num_inference_steps` is provided" + ) + else: + if sigmas is not None: + num_inference_steps = len(sigmas) + elif timesteps is not None: + num_inference_steps = len(timesteps) + else: + raise ValueError( + "Either num_inference_steps, sigmas, or timesteps must be provided" + ) + + self.num_inference_steps = num_inference_steps + + # 1. Prepare default sigmas + is_timesteps_provided = timesteps is not None + + timesteps_array: np.ndarray | None = None + if is_timesteps_provided: + assert timesteps is not None + timesteps_array = np.array(timesteps).astype(np.float32) + + sigmas_array: np.ndarray + if sigmas is None: + if timesteps_array is None: + timesteps_array = np.linspace( + self._sigma_to_t(self.sigma_max), + self._sigma_to_t(self.sigma_min), + num_inference_steps, + ) + sigmas_array = timesteps_array / self.config.num_train_timesteps + else: + sigmas_array = np.array(sigmas).astype(np.float32) + num_inference_steps = len(sigmas_array) + + # 2. Perform timestep shifting. Either no shifting is applied, or resolution-dependent shifting of + # "exponential" or "linear" type is applied + if self.config.use_dynamic_shifting: + assert mu is not None, "mu cannot be None when use_dynamic_shifting is True" + sigmas_array = self.time_shift(mu, 1.0, sigmas_array) + else: + sigmas_array = ( + self.shift * sigmas_array / (1 + (self.shift - 1) * sigmas_array) + ) + + # 3. If required, stretch the sigmas schedule to terminate at the configured `shift_terminal` value + if self.config.shift_terminal: + sigmas_tensor = torch.from_numpy(sigmas_array).to(dtype=torch.float32) + sigmas_tensor = self.stretch_shift_to_terminal(sigmas_tensor) + sigmas_array = sigmas_tensor.numpy() + + # 4. If required, convert sigmas to one of karras, exponential, or beta sigma schedules + if self.config.use_karras_sigmas: + sigmas_tensor = torch.from_numpy(sigmas_array).to(dtype=torch.float32) + sigmas_tensor = self._convert_to_karras( + in_sigmas=sigmas_tensor, num_inference_steps=num_inference_steps + ) + sigmas_array = sigmas_tensor.numpy() + elif self.config.use_exponential_sigmas: + sigmas_tensor = torch.from_numpy(sigmas_array).to(dtype=torch.float32) + sigmas_tensor = self._convert_to_exponential( + in_sigmas=sigmas_tensor, num_inference_steps=num_inference_steps + ) + sigmas_array = sigmas_tensor.numpy() + elif self.config.use_beta_sigmas: + sigmas_tensor = torch.from_numpy(sigmas_array).to(dtype=torch.float32) + sigmas_tensor = self._convert_to_beta( + in_sigmas=sigmas_tensor, num_inference_steps=num_inference_steps + ) + sigmas_array = sigmas_tensor.numpy() + + # 5. Convert sigmas and timesteps to tensors and move to specified device + sigmas_tensor = torch.from_numpy(sigmas_array).to( + dtype=torch.float32, device=device + ) + if not is_timesteps_provided: + timesteps_tensor = sigmas_tensor * self.config.num_train_timesteps + else: + assert timesteps_array is not None + timesteps_tensor = torch.from_numpy(timesteps_array).to( + dtype=torch.float32, device=device + ) + + # 6. Append the terminal sigma value. + # If a model requires inverted sigma schedule for denoising but timesteps without inversion, the + # `invert_sigmas` flag can be set to `True`. This case is only required in Mochi + if self.config.invert_sigmas: + sigmas_tensor = 1.0 - sigmas_tensor + timesteps_tensor = sigmas_tensor * self.config.num_train_timesteps + sigmas_tensor = torch.cat( + [sigmas_tensor, torch.ones(1, device=sigmas_tensor.device)] + ) + else: + sigmas_tensor = torch.cat( + [sigmas_tensor, torch.zeros(1, device=sigmas_tensor.device)] + ) + + self.timesteps = timesteps_tensor + self.sigmas = sigmas_tensor + self._step_index = None + self._begin_index = None + + def index_for_timestep( + self, + timestep: float | torch.FloatTensor, + schedule_timesteps: torch.Tensor | None = None, + ) -> int: + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + indices = (schedule_timesteps == timestep).nonzero() + + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + pos = 1 if len(indices) > 1 else 0 + + return indices[pos].item() + + def _init_step_index(self, timestep: float | torch.FloatTensor) -> None: + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def step( + self, + model_output: torch.FloatTensor, + timestep: int | torch.Tensor, + sample: torch.FloatTensor, + s_churn: float = 0.0, + s_tmin: float = 0.0, + s_tmax: float = float("inf"), + s_noise: float = 1.0, + generator: torch.Generator | None = None, + per_token_timesteps: torch.Tensor | None = None, + return_dict: bool = True, + ) -> FlowMatchEulerDiscreteSchedulerOutput | tuple[torch.FloatTensor, ...]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`int` or `torch.Tensor`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + s_churn (`float`): + s_tmin (`float`): + s_tmax (`float`): + s_noise (`float`, defaults to 1.0): + Scaling factor for noise added to the sample. + generator (`torch.Generator`, *optional*): + A random number generator. + per_token_timesteps (`torch.Tensor`, *optional*): + The timesteps for each token in the sample. + return_dict (`bool`): + Whether or not to return a + [`~schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteSchedulerOutput`] or tuple. + + Returns: + [`~schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteSchedulerOutput`] or `tuple`: + If return_dict is `True`, + [`~schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteSchedulerOutput`] is returned, + otherwise a tuple is returned where the first element is the sample tensor. + """ + + if isinstance(timestep, int | torch.IntTensor | torch.LongTensor): + raise ValueError( + ( + "Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to" + " `FlowMatchEulerDiscreteScheduler.step()` is not supported. Make sure to pass" + " one of the `scheduler.timesteps` as a timestep." + ), + ) + + if self.step_index is None: + self._init_step_index(timestep) + + # Upcast to avoid precision issues when computing prev_sample + sample = sample.to(torch.float32) + + if per_token_timesteps is not None: + per_token_sigmas = per_token_timesteps / self.config.num_train_timesteps + + sigmas = self.sigmas[:, None, None] + lower_mask = sigmas < per_token_sigmas[None] - 1e-6 + lower_sigmas = lower_mask * sigmas + lower_sigmas, _ = lower_sigmas.max(dim=0) + + current_sigma = per_token_sigmas[..., None] + next_sigma = lower_sigmas[..., None] + dt = current_sigma - next_sigma + else: + assert self.step_index is not None, "step_index should not be None" + sigma_idx = self.step_index + sigma = self.sigmas[sigma_idx] + sigma_next = self.sigmas[sigma_idx + 1] + + current_sigma = sigma + next_sigma = sigma_next + dt = sigma_next - sigma + + if self.config.stochastic_sampling: + x0 = sample - current_sigma * model_output + noise = torch.randn_like(sample) + prev_sample = (1.0 - next_sigma) * x0 + next_sigma * noise + else: + prev_sample = sample + dt * model_output + + # upon completion increase step index by one + assert self._step_index is not None, "_step_index should not be None" + self._step_index += 1 + if per_token_timesteps is None: + # Cast sample back to model compatible dtype + prev_sample = prev_sample.to(model_output.dtype) + + if isinstance(prev_sample, torch.Tensor | float) and not return_dict: + return (prev_sample,) + + return FlowMatchEulerDiscreteSchedulerOutput(prev_sample=prev_sample) + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras + def _convert_to_karras( + self, in_sigmas: torch.Tensor, num_inference_steps: int + ) -> torch.Tensor: + """Constructs the noise schedule of Karras et al. (2022).""" + + # Hack to make sure that other schedulers which copy this function don't break + # TODO: Add this logic to the other schedulers + if hasattr(self.config, "sigma_min"): + sigma_min = self.config.sigma_min + else: + sigma_min = None + + if hasattr(self.config, "sigma_max"): + sigma_max = self.config.sigma_max + else: + sigma_max = None + + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + + rho = 7.0 # 7.0 is the value used in the paper + ramp = np.linspace(0, 1, num_inference_steps) + min_inv_rho = sigma_min ** (1 / rho) + max_inv_rho = sigma_max ** (1 / rho) + sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + return sigmas + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_exponential + def _convert_to_exponential( + self, in_sigmas: torch.Tensor, num_inference_steps: int + ) -> torch.Tensor: + """Constructs an exponential noise schedule.""" + + # Hack to make sure that other schedulers which copy this function don't break + # TODO: Add this logic to the other schedulers + if hasattr(self.config, "sigma_min"): + sigma_min = self.config.sigma_min + else: + sigma_min = None + + if hasattr(self.config, "sigma_max"): + sigma_max = self.config.sigma_max + else: + sigma_max = None + + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + + sigmas = np.exp( + np.linspace(math.log(sigma_max), math.log(sigma_min), num_inference_steps) + ) + return sigmas + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_beta + def _convert_to_beta( + self, + in_sigmas: torch.Tensor, + num_inference_steps: int, + alpha: float = 0.6, + beta: float = 0.6, + ) -> torch.Tensor: + """From "Beta Sampling is All You Need" [arXiv:2407.12173] (Lee et. al, 2024)""" + + # Hack to make sure that other schedulers which copy this function don't break + # TODO: Add this logic to the other schedulers + if hasattr(self.config, "sigma_min"): + sigma_min = self.config.sigma_min + else: + sigma_min = None + + if hasattr(self.config, "sigma_max"): + sigma_max = self.config.sigma_max + else: + sigma_max = None + + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + + sigmas = np.array( + [ + sigma_min + (ppf * (sigma_max - sigma_min)) + for ppf in [ + scipy.stats.beta.ppf(timestep, alpha, beta) + for timestep in 1 - np.linspace(0, 1, num_inference_steps) + ] + ] + ) + return sigmas + + def _time_shift_exponential( + self, mu: float, sigma: float, t: torch.Tensor | np.ndarray + ) -> torch.Tensor | np.ndarray: + if isinstance(t, np.ndarray): + return np.exp(mu) / (np.exp(mu) + (1 / t - 1) ** sigma) + else: + return math.exp(mu) / (math.exp(mu) + (1 / t - 1) ** sigma) + + def _time_shift_linear( + self, mu: float, sigma: float, t: torch.Tensor | np.ndarray + ) -> torch.Tensor | np.ndarray: + return mu / (mu + (1 / t - 1) ** sigma) + + def add_noise( + self, + clean_latent: torch.Tensor, + noise: torch.Tensor, + timestep: torch.IntTensor, + ) -> torch.Tensor: + """ + Args: + clean_latent: the clean latent with shape [B, C, H, W], + where B is batch_size or batch_size * num_frames + noise: the noise with shape [B, C, H, W] + timestep: the timestep with shape [1] or [bs * num_frames] or [bs, num_frames] + + Returns: + the corrupted latent with shape [B, C, H, W] + """ + # If timestep is [bs, num_frames] + if timestep.ndim == 2: + timestep = timestep.flatten(0, 1) + assert timestep.numel() == clean_latent.shape[0] + elif timestep.ndim == 1: + # If timestep is [1] + if timestep.shape[0] == 1: + timestep = timestep.expand(clean_latent.shape[0]) + else: + assert timestep.numel() == clean_latent.shape[0] + else: + raise ValueError(f"[add_noise] Invalid timestep shape: {timestep.shape}") + # timestep shape should be [B] + self.sigmas = self.sigmas.to(noise.device) + self.timesteps = self.timesteps.to(noise.device) + timestep_id = torch.argmin( + (self.timesteps.unsqueeze(0) - timestep.unsqueeze(1)).abs(), dim=1 + ) + sigma = self.sigmas[timestep_id].reshape(-1, 1, 1, 1) + sample = (1 - sigma) * clean_latent + sigma * noise + return sample.type_as(noise) + + def scale_model_input( + self, sample: torch.Tensor, timestep: int | None = None + ) -> torch.Tensor: + return sample + + def __len__(self) -> int: + return 0 + + +EntryClass = FlowMatchEulerDiscreteScheduler diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_flow_unipc_multistep.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_flow_unipc_multistep.py new file mode 100644 index 00000000..1e6b84e0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_flow_unipc_multistep.py @@ -0,0 +1,853 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Copied from https://github.com/huggingface/diffusers/blob/v0.31.0/src/diffusers/schedulers/scheduling_unipc_multistep.py +# Convert unipc for flow matching +# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved. + +import math +from typing import Any + +import numpy as np +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.schedulers.scheduling_utils import ( + KarrasDiffusionSchedulers, + SchedulerMixin, + SchedulerOutput, +) +from diffusers.utils import deprecate + +from sglang.multimodal_gen.runtime.models.schedulers.base import BaseScheduler + + +class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler): + """ + `UniPCMultistepScheduler` is a training-free framework designed for the fast sampling of diffusion models. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + solver_order (`int`, default `2`): + The UniPC order which can be any positive integer. The effective order of accuracy is `solver_order + 1` + due to the UniC. It is recommended to use `solver_order=2` for guided sampling, and `solver_order=3` for + unconditional sampling. + prediction_type (`str`, defaults to "flow_prediction"): + Prediction type of the scheduler function; must be `flow_prediction` for this scheduler, which predicts + the flow of the diffusion process. + thresholding (`bool`, defaults to `False`): + Whether to use the "dynamic thresholding" method. This is unsuitable for latent-space diffusion models such + as Stable Diffusion. + dynamic_thresholding_ratio (`float`, defaults to 0.995): + The ratio for the dynamic thresholding method. Valid only when `thresholding=True`. + sample_max_value (`float`, defaults to 1.0): + The threshold value for dynamic thresholding. Valid only when `thresholding=True` and `predict_x0=True`. + predict_x0 (`bool`, defaults to `True`): + Whether to use the updating algorithm on the predicted x0. + solver_type (`str`, default `bh2`): + Solver type for UniPC. It is recommended to use `bh1` for unconditional sampling when steps < 10, and `bh2` + otherwise. + lower_order_final (`bool`, default `True`): + Whether to use lower-order solvers in the final steps. Only valid for < 15 inference steps. This can + stabilize the sampling of DPMSolver for steps < 15, especially for steps <= 10. + disable_corrector (`list`, default `[]`): + Decides which step to disable the corrector to mitigate the misalignment between `epsilon_theta(x_t, c)` + and `epsilon_theta(x_t^c, c)` which can influence convergence for a large guidance scale. Corrector is + usually disabled during the first few steps. + solver_p (`SchedulerMixin`, default `None`): + Any other scheduler that if specified, the algorithm becomes `solver_p + UniC`. + use_karras_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use Karras sigmas for step sizes in the noise schedule during the sampling process. If `True`, + the sigmas are determined according to a sequence of noise levels {σi}. + use_exponential_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use exponential sigmas for step sizes in the noise schedule during the sampling process. + timestep_spacing (`str`, defaults to `"linspace"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + final_sigmas_type (`str`, defaults to `"zero"`): + The final `sigma` value for the noise schedule during the sampling process. If `"sigma_min"`, the final + sigma is the same as the last sigma in the training schedule. If `zero`, the final sigma is set to 0. + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + solver_order: int = 2, + prediction_type: str = "flow_prediction", + shift: float | None = 1.0, + use_dynamic_shifting=False, + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + sample_max_value: float = 1.0, + predict_x0: bool = True, + solver_type: str = "bh2", + lower_order_final: bool = True, + disable_corrector: tuple = (), + solver_p: SchedulerMixin = None, + timestep_spacing: str = "linspace", + steps_offset: int = 0, + final_sigmas_type: str | None = "zero", # "zero", "sigma_min" + **kwargs, + ): + + if solver_type not in ["bh1", "bh2"]: + if solver_type in ["midpoint", "heun", "logrho"]: + self.register_to_config(solver_type="bh2") + else: + raise NotImplementedError( + f"{solver_type} is not implemented for {self.__class__}" + ) + + self.predict_x0 = predict_x0 + # setable values + self.num_inference_steps: int | None = None + alphas = np.linspace(1, 1 / num_train_timesteps, num_train_timesteps)[ + ::-1 + ].copy() + sigmas = 1.0 - alphas + sigmas = torch.from_numpy(sigmas).to(dtype=torch.float32) + + if not use_dynamic_shifting: + # when use_dynamic_shifting is True, we apply the timestep shifting on the fly based on the image resolution + assert shift is not None + sigmas = shift * sigmas / (1 + (shift - 1) * sigmas) # pyright: ignore + + self.sigmas = sigmas + self.sigma_min = self.sigmas[-1].item() + self.sigma_max = self.sigmas[0].item() + + self.timesteps = sigmas * num_train_timesteps + self.num_train_timesteps = num_train_timesteps + + self.model_outputs = [None] * solver_order + self.timestep_list: list[Any | None] = [None] * solver_order + self.lower_order_nums = 0 + self.disable_corrector = list(disable_corrector) + self.solver_p = solver_p + self.last_sample = None + self._step_index: int | None = None + self._begin_index: int | None = None + + BaseScheduler.__init__(self) + + @property + def step_index(self): + """ + The index counter for current timestep. It will increase 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + def set_shift(self, shift: float) -> None: + self.config.shift = shift + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + # Modified from diffusers.schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteScheduler.set_timesteps + def set_timesteps( + self, + num_inference_steps: int | None = None, + device: str | torch.device = None, + sigmas: list[float] | None = None, + mu: float | None | None = None, + shift: float | None | None = None, + ): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + Args: + num_inference_steps (`int`): + Total number of the spacing of the time steps. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + + if self.config.use_dynamic_shifting and mu is None: + raise ValueError( + " you have to pass a value for `mu` when `use_dynamic_shifting` is set to be `True`" + ) + + if sigmas is None: + assert num_inference_steps is not None + sigmas = np.linspace( + self.sigma_max, self.sigma_min, num_inference_steps + 1 + ).copy()[ + :-1 + ] # pyright: ignore + + if self.config.use_dynamic_shifting: + assert mu is not None + sigmas = self.time_shift(mu, 1.0, sigmas) # pyright: ignore + else: + if shift is None: + shift = self.config.shift + assert isinstance(sigmas, np.ndarray) + sigmas = shift * sigmas / (1 + (shift - 1) * sigmas) # pyright: ignore + + if self.config.final_sigmas_type == "sigma_min": + sigma_last = ((1 - self.alphas_cumprod[0]) / self.alphas_cumprod[0]) ** 0.5 + elif self.config.final_sigmas_type == "zero": + sigma_last = 0 + else: + raise ValueError( + f"`final_sigmas_type` must be one of 'zero', or 'sigma_min', but got {self.config.final_sigmas_type}" + ) + + timesteps = sigmas * self.config.num_train_timesteps + sigmas = np.concatenate([sigmas, [sigma_last]]).astype( + np.float32 + ) # pyright: ignore + + self.sigmas = torch.from_numpy(sigmas).to(device=device) + self.timesteps = torch.from_numpy(timesteps).to( + device=device, dtype=torch.int64 + ) + + self.num_inference_steps = len(timesteps) + + self.model_outputs = [ + None, + ] * self.config.solver_order + self.lower_order_nums = 0 + self.last_sample = None + if self.solver_p: + self.solver_p.set_timesteps(self.num_inference_steps, device=device) + + # add an index counter for schedulers that allow duplicated timesteps + self._step_index = None + self._begin_index = None + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample + def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor: + """ + "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the + prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by + s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing + pixels from saturation at each step. We find that dynamic thresholding results in significantly better + photorealism as well as better image-text alignment, especially when using very large guidance weights." + + https://arxiv.org/abs/2205.11487 + """ + dtype = sample.dtype + batch_size, channels, *remaining_dims = sample.shape + + if dtype not in (torch.float32, torch.float64): + sample = ( + sample.float() + ) # upcast for quantile calculation, and clamp not implemented for cpu half + + # Flatten sample for doing quantile calculation along each image + sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) + + abs_sample = sample.abs() # "a certain percentile absolute pixel value" + + s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1) + s = torch.clamp( + s, min=1, max=self.config.sample_max_value + ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] + s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0 + sample = ( + torch.clamp(sample, -s, s) / s + ) # "we threshold xt0 to the range [-s, s] and then divide by s" + + sample = sample.reshape(batch_size, channels, *remaining_dims) + sample = sample.to(dtype) + + return sample + + # Copied from diffusers.schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteScheduler._sigma_to_t + def _sigma_to_t(self, sigma): + return sigma * self.config.num_train_timesteps + + def _sigma_to_alpha_sigma_t(self, sigma) -> tuple[Any, Any]: + return 1 - sigma, sigma + + # Copied from diffusers.schedulers.scheduling_flow_match_euler_discrete.set_timesteps + def time_shift(self, mu: float, sigma: float, t: torch.Tensor): + return math.exp(mu) / (math.exp(mu) + (1 / t - 1) ** sigma) + + def convert_model_output( + self, + model_output: torch.Tensor, + *args, + sample: torch.Tensor = None, + **kwargs, + ) -> torch.Tensor: + r""" + Convert the model output to the corresponding type the UniPC algorithm needs. + + Args: + model_output (`torch.Tensor`): + The direct output from the learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.Tensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.Tensor`: + The converted model output. + """ + timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None) + if sample is None: + if len(args) > 1: + sample = args[1] + else: + raise ValueError("missing `sample` as a required keyword argument") + if timestep is not None: + deprecate( + "timesteps", + "1.0.0", + "Passing `timesteps` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + + if self.predict_x0: + if self.config.prediction_type == "flow_prediction": + sigma_t = self.sigmas[self.step_index] + x0_pred = sample - sigma_t * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`," + " `v_prediction` or `flow_prediction` for the UniPCMultistepScheduler." + ) + + if self.config.thresholding: + x0_pred = self._threshold_sample(x0_pred) + + return x0_pred + else: + if self.config.prediction_type == "flow_prediction": + sigma_t = self.sigmas[self.step_index] + epsilon = sample - (1 - sigma_t) * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`," + " `v_prediction` or `flow_prediction` for the UniPCMultistepScheduler." + ) + + if self.config.thresholding: + sigma_t = self.sigmas[self.step_index] + x0_pred = sample - sigma_t * model_output + x0_pred = self._threshold_sample(x0_pred) + epsilon = model_output + x0_pred + + return epsilon + + def multistep_uni_p_bh_update( + self, + model_output: torch.Tensor, + *args, + sample: torch.Tensor = None, + order: int | None = None, # pyright: ignore + **kwargs, + ) -> torch.Tensor: + """ + One step for the UniP (B(h) version). Alternatively, `self.solver_p` is used if is specified. + + Args: + model_output (`torch.Tensor`): + The direct output from the learned diffusion model at the current timestep. + prev_timestep (`int`): + The previous discrete timestep in the diffusion chain. + sample (`torch.Tensor`): + A current instance of a sample created by the diffusion process. + order (`int`): + The order of UniP at this timestep (corresponds to the *p* in UniPC-p). + + Returns: + `torch.Tensor`: + The sample tensor at the previous timestep. + """ + prev_timestep = args[0] if len(args) > 0 else kwargs.pop("prev_timestep", None) + if sample is None: + if len(args) > 1: + sample = args[1] + else: + raise ValueError(" missing `sample` as a required keyword argument") + if order is None: + if len(args) > 2: + order = args[2] + else: + raise ValueError(" missing `order` as a required keyword argument") + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + model_output_list = self.model_outputs + + s0 = self.timestep_list[-1] + m0 = model_output_list[-1] + x = sample + + if self.solver_p: + x_t = self.solver_p.step(model_output, s0, x).prev_sample + return x_t + + sigma_t, sigma_s0 = ( + self.sigmas[self.step_index + 1], + self.sigmas[self.step_index], + ) # pyright: ignore + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + + h = lambda_t - lambda_s0 + device = sample.device + + rks = [] + D1s: list[Any] | None = [] + sigmas = self.sigmas.to(device=device) + for i in range(1, order): + si = self.step_index - i # pyright: ignore + mi = model_output_list[-(i + 1)] + alpha_si, sigma_si = self._sigma_to_alpha_sigma_t(sigmas[si]) + lambda_si = torch.log(alpha_si) - torch.log(sigma_si) + rk = (lambda_si - lambda_s0) / h + rks.append(rk) + assert mi is not None + D1s.append((mi - m0) / rk) # pyright: ignore + + if len(rks) > 0: + rks = torch.stack(rks) + one = torch.ones(1, device=device, dtype=rks.dtype) + rks = torch.cat([rks, one]) + else: + rks = torch.ones(1, device=device, dtype=h.dtype) + + R = [] + b = [] + + hh = -h if self.predict_x0 else h + h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1 + h_phi_k = h_phi_1 / hh - 1 + + factorial_i = 1 + + if self.config.solver_type == "bh1": + B_h = hh + elif self.config.solver_type == "bh2": + B_h = torch.expm1(hh) + else: + raise NotImplementedError() + + for i in range(1, order + 1): + R.append(torch.pow(rks, i - 1)) + b.append(h_phi_k * factorial_i / B_h) + factorial_i *= i + 1 + h_phi_k = h_phi_k / hh - 1 / factorial_i + + R = torch.stack(R) + b = torch.stack(b) + + if D1s is not None and len(D1s) > 0: + D1s = torch.stack(D1s, dim=1) # (B, K) + # for order 2, we use a simplified version + if order == 2: + rhos_p = 0.5 * torch.ones(1, dtype=x.dtype, device=device) + else: + assert isinstance(R, torch.Tensor) + rhos_p = torch.linalg.solve(R[:-1, :-1], b[:-1]).to(device).to(x.dtype) + else: + D1s = None + + if self.predict_x0: + x_t_ = sigma_t / sigma_s0 * x - alpha_t * h_phi_1 * m0 + if D1s is not None: + pred_res = torch.einsum( + "k,bkc...->bc...", rhos_p, D1s + ) # pyright: ignore + else: + pred_res = 0 + x_t = x_t_ - alpha_t * B_h * pred_res + else: + x_t_ = alpha_t / alpha_s0 * x - sigma_t * h_phi_1 * m0 + if D1s is not None: + pred_res = torch.einsum( + "k,bkc...->bc...", rhos_p, D1s + ) # pyright: ignore + else: + pred_res = 0 + x_t = x_t_ - sigma_t * B_h * pred_res + + x_t = x_t.to(x.dtype) + return x_t + + def multistep_uni_c_bh_update( + self, + this_model_output: torch.Tensor, + *args, + last_sample: torch.Tensor = None, + this_sample: torch.Tensor = None, + order: int | None = None, # pyright: ignore + **kwargs, + ) -> torch.Tensor: + """ + One step for the UniC (B(h) version). + + Args: + this_model_output (`torch.Tensor`): + The model outputs at `x_t`. + this_timestep (`int`): + The current timestep `t`. + last_sample (`torch.Tensor`): + The generated sample before the last predictor `x_{t-1}`. + this_sample (`torch.Tensor`): + The generated sample after the last predictor `x_{t}`. + order (`int`): + The `p` of UniC-p at this step. The effective order of accuracy should be `order + 1`. + + Returns: + `torch.Tensor`: + The corrected sample tensor at the current timestep. + """ + this_timestep = args[0] if len(args) > 0 else kwargs.pop("this_timestep", None) + if last_sample is None: + if len(args) > 1: + last_sample = args[1] + else: + raise ValueError(" missing`last_sample` as a required keyword argument") + if this_sample is None: + if len(args) > 2: + this_sample = args[2] + else: + raise ValueError(" missing`this_sample` as a required keyword argument") + if order is None: + if len(args) > 3: + order = args[3] + else: + raise ValueError(" missing`order` as a required keyword argument") + if this_timestep is not None: + deprecate( + "this_timestep", + "1.0.0", + "Passing `this_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + model_output_list = self.model_outputs + + m0 = model_output_list[-1] + x = last_sample + x_t = this_sample + model_t = this_model_output + + sigma_t, sigma_s0 = ( + self.sigmas[self.step_index], + self.sigmas[self.step_index - 1], + ) # pyright: ignore + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + + h = lambda_t - lambda_s0 + device = this_sample.device + + # Build rks and D1s fully on device to avoid any host-device sync + # Fast paths for small orders (common cases: 1 or 2) + if order == 1: + rks = torch.ones(1, device=device, dtype=h.dtype) + D1s = None + elif order == 2: + # order == 2 -> only one historical point is used + si = self.step_index - 2 # i = 1 + mi = model_output_list[-2] + alpha_si, sigma_si = self._sigma_to_alpha_sigma_t(self.sigmas[si]) + lambda_si = torch.log(alpha_si) - torch.log(sigma_si) + rk = (lambda_si - lambda_s0) / h # 0-dim tensor on device + # rks = [rk, 1.0] but keep it on device without list->tensor sync + rks = torch.stack((rk, torch.ones_like(rk))) + assert mi is not None + # D1s shape: (B, K=1, C, ...) to match later einsum over K + D1s = ((mi - m0) / rk).unsqueeze(1) # pyright: ignore + else: + rks_list = [] + D1s_list = [] + for i in range(1, order): + si = self.step_index - (i + 1) + mi = model_output_list[-(i + 1)] + alpha_si, sigma_si = self._sigma_to_alpha_sigma_t(self.sigmas[si]) + lambda_si = torch.log(alpha_si) - torch.log(sigma_si) + rk = (lambda_si - lambda_s0) / h + rks_list.append(rk) + assert mi is not None + D1s_list.append((mi - m0) / rk) # pyright: ignore + + # Append 1.0 as a device tensor to rks + rks = torch.stack(rks_list + [torch.ones_like(rks_list[0])]) + D1s = torch.stack(D1s_list, dim=1) if len(D1s_list) > 0 else None + + R = [] + b = [] + + hh = -h if self.predict_x0 else h + h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1 + h_phi_k = h_phi_1 / hh - 1 + + factorial_i = 1 + + if self.config.solver_type == "bh1": + B_h = hh + elif self.config.solver_type == "bh2": + B_h = torch.expm1(hh) + else: + raise NotImplementedError() + + for i in range(1, order + 1): + R.append(torch.pow(rks, i - 1)) + b.append(h_phi_k * factorial_i / B_h) + factorial_i *= i + 1 + h_phi_k = h_phi_k / hh - 1 / factorial_i + + R = torch.stack(R) + # Avoid torch.tensor(list_of_gpu_scalars) which syncs to host + b = torch.stack(b) + + # D1s is already prepared above for order==2; remains None for order==1 + + # for order 1, we use a simplified version + if order == 1: + rhos_c = 0.5 * torch.ones(1, dtype=x.dtype, device=device) + elif order == 2: + # Manually solve the 2x2 linear system to avoid device synchronization from torch.linalg.solve + # R = [[1, 1], [rk, 1]], where rk = rks[0] + rk = rks[0] + det = 1 - rk + # Using Cramer's rule to solve for rhos_c = [x0, x1] + # x0 = (b0 - b1) / det + # x1 = (b1 - rk * b0) / det + rhos_c_0 = (b[0] - b[1]) / det + rhos_c_1 = (b[1] - rk * b[0]) / det + rhos_c = torch.stack([rhos_c_0, rhos_c_1]) + else: + rhos_c = torch.linalg.solve(R, b).to(device).to(x.dtype) + + if self.predict_x0: + x_t_ = sigma_t / sigma_s0 * x - alpha_t * h_phi_1 * m0 + if D1s is not None: + corr_res = torch.einsum("k,bkc...->bc...", rhos_c[:-1], D1s) + else: + corr_res = 0 + D1_t = model_t - m0 + x_t = x_t_ - alpha_t * B_h * (corr_res + rhos_c[-1] * D1_t) + else: + x_t_ = alpha_t / alpha_s0 * x - sigma_t * h_phi_1 * m0 + if D1s is not None: + corr_res = torch.einsum("k,bkc...->bc...", rhos_c[:-1], D1s) + else: + corr_res = 0 + D1_t = model_t - m0 + x_t = x_t_ - sigma_t * B_h * (corr_res + rhos_c[-1] * D1_t) + x_t = x_t.to(x.dtype) + return x_t + + def index_for_timestep(self, timestep, schedule_timesteps=None) -> int: + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + indices = (schedule_timesteps == timestep).nonzero() + + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + pos = 1 if len(indices) > 1 else 0 + step_index: int = indices[pos].item() + + return step_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._init_step_index + def _init_step_index(self, timestep) -> None: + """ + Initialize the step_index counter for the scheduler. + """ + + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def step( + self, + model_output: torch.Tensor, + timestep: int | torch.Tensor, + sample: torch.Tensor, + return_dict: bool = True, + generator=None, + ) -> SchedulerOutput | tuple: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with + the multistep UniPC. + + Args: + model_output (`torch.Tensor`): + The direct output from learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.Tensor`): + A current instance of a sample created by the diffusion process. + return_dict (`bool`): + Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`. + + Returns: + [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_utils.SchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to call 'set_timesteps' after creating the scheduler" + ) + + if self.step_index is None: + self._init_step_index(timestep) + + use_corrector = ( + self.step_index > 0 + and self.step_index - 1 not in self.disable_corrector + and self.last_sample is not None # pyright: ignore + ) + + sample = sample.to(model_output.device) + model_output_convert = self.convert_model_output(model_output, sample=sample) + + if use_corrector: + sample = self.multistep_uni_c_bh_update( + this_model_output=model_output_convert, + last_sample=self.last_sample, + this_sample=sample, + order=self.this_order, + ) + + for i in range(self.config.solver_order - 1): + self.model_outputs[i] = self.model_outputs[i + 1] + self.timestep_list[i] = self.timestep_list[i + 1] + + self.model_outputs[-1] = model_output_convert + self.timestep_list[-1] = timestep # pyright: ignore + + if self.config.lower_order_final: + this_order = min( + self.config.solver_order, len(self.timesteps) - self.step_index + ) # pyright: ignore + else: + this_order = self.config.solver_order + + self.this_order: int = min( + this_order, self.lower_order_nums + 1 + ) # warmup for multistep + assert self.this_order > 0 + + self.last_sample = sample + prev_sample = self.multistep_uni_p_bh_update( + model_output=model_output, # pass the original non-converted model output, in case solver-p is used + sample=sample, + order=self.this_order, + ) + + if self.lower_order_nums < self.config.solver_order: + self.lower_order_nums += 1 + + # upon completion increase step index by one + assert self._step_index is not None + self._step_index += 1 # pyright: ignore + + if not return_dict: + return (prev_sample,) + + return SchedulerOutput(prev_sample=prev_sample) + + def scale_model_input(self, sample: torch.Tensor, *args, **kwargs) -> torch.Tensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.Tensor`): + The input sample. + + Returns: + `torch.Tensor`: + A scaled input sample. + """ + return sample + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.add_noise + def add_noise( + self, + original_samples: torch.Tensor, + noise: torch.Tensor, + timesteps: torch.IntTensor, + ) -> torch.Tensor: + # Make sure sigmas and timesteps have the same device and dtype as original_samples + sigmas = self.sigmas.to( + device=original_samples.device, dtype=original_samples.dtype + ) + if original_samples.device.type == "mps" and torch.is_floating_point(timesteps): + # mps does not support float64 + schedule_timesteps = self.timesteps.to( + original_samples.device, dtype=torch.float32 + ) + timesteps = timesteps.to(original_samples.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(original_samples.device) + timesteps = timesteps.to(original_samples.device) + + # begin_index is None when the scheduler is used for training or pipeline does not implement set_begin_index + if self.begin_index is None: + step_indices = [ + self.index_for_timestep(t, schedule_timesteps) for t in timesteps + ] + elif self.step_index is not None: + # add_noise is called after first denoising step (for inpainting) + step_indices = [self.step_index] * timesteps.shape[0] + else: + # add noise is called before first denoising step to create initial latent(img2img) + step_indices = [self.begin_index] * timesteps.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(original_samples.shape): + sigma = sigma.unsqueeze(-1) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + noisy_samples = alpha_t * original_samples + sigma_t * noise + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps + + +EntryClass = FlowUniPCMultistepScheduler diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_self_forcing_flow_match.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_self_forcing_flow_match.py new file mode 100644 index 00000000..9a4749a6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_self_forcing_flow_match.py @@ -0,0 +1,142 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.schedulers.scheduling_utils import SchedulerMixin +from diffusers.utils import BaseOutput + +from sglang.multimodal_gen.runtime.models.schedulers.base import BaseScheduler +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class SelfForcingFlowMatchSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + """ + + prev_sample: torch.FloatTensor + + +class SelfForcingFlowMatchScheduler(BaseScheduler, ConfigMixin, SchedulerMixin): + config_name = "scheduler_config.json" + order = 1 + + @register_to_config + def __init__( + self, + num_inference_steps=100, + num_train_timesteps=1000, + shift=3.0, + sigma_max=1.0, + sigma_min=0.003 / 1.002, + inverse_timesteps=False, + extra_one_step=False, + reverse_sigmas=False, + *args, + **kwargs, + ): + self.num_train_timesteps = num_train_timesteps + self.shift = shift + self.sigma_max = sigma_max + self.sigma_min = sigma_min + self.inverse_timesteps = inverse_timesteps + self.extra_one_step = extra_one_step + self.reverse_sigmas = reverse_sigmas + self.set_timesteps(num_inference_steps) + + def set_timesteps( + self, + num_inference_steps=100, + denoising_strength=1.0, + return_dict=False, + **kwargs, + ): + sigma_start = ( + self.sigma_min + (self.sigma_max - self.sigma_min) * denoising_strength + ) + if self.extra_one_step: + self.sigmas = torch.linspace( + sigma_start, self.sigma_min, num_inference_steps + 1 + )[:-1] + else: + self.sigmas = torch.linspace( + sigma_start, self.sigma_min, num_inference_steps + ) + if self.inverse_timesteps: + self.sigmas = torch.flip(self.sigmas, dims=[0]) + self.sigmas = self.shift * self.sigmas / (1 + (self.shift - 1) * self.sigmas) + if self.reverse_sigmas: + self.sigmas = 1 - self.sigmas + self.timesteps = self.sigmas * self.num_train_timesteps + + def step( + self, + model_output: torch.FloatTensor, + timestep: torch.FloatTensor, + sample: torch.FloatTensor, + to_final=False, + return_dict=False, + **kwargs, + ): + if timestep.ndim == 2: + timestep = timestep.flatten(0, 1) + elif timestep.ndim == 0: + # handles the case where timestep is a scalar, this occurs when we + # use this scheduler for ODE trajectory + timestep = timestep.unsqueeze(0) + + self.sigmas = self.sigmas.to(model_output.device) + self.timesteps = self.timesteps.to(model_output.device) + timestep = timestep.to(model_output.device) + + timestep_id = torch.argmin( + (self.timesteps.unsqueeze(0) - timestep.unsqueeze(1)).abs(), dim=1 + ) + sigma = self.sigmas[timestep_id].reshape(-1, 1, 1, 1) + if to_final or (timestep_id + 1 >= len(self.timesteps)).any(): + sigma_ = 1 if (self.inverse_timesteps or self.reverse_sigmas) else 0 + else: + sigma_ = self.sigmas[timestep_id + 1].reshape(-1, 1, 1, 1) + prev_sample = sample + model_output * (sigma_ - sigma) + if isinstance(prev_sample, torch.Tensor | float) and not return_dict: + return (prev_sample,) + return SelfForcingFlowMatchSchedulerOutput(prev_sample=prev_sample) + + def add_noise(self, original_samples, noise, timestep): + """ + Diffusion forward corruption process. + Input: + - clean_latent: the clean latent with shape [B*T, C, H, W] + - noise: the noise with shape [B*T, C, H, W] + - timestep: the timestep with shape [B*T] + Output: the corrupted latent with shape [B*T, C, H, W] + """ + if timestep.ndim == 2: + timestep = timestep.flatten(0, 1) + self.sigmas = self.sigmas.to(noise.device) + self.timesteps = self.timesteps.to(noise.device) + timestep_id = torch.argmin( + (self.timesteps.unsqueeze(0) - timestep.unsqueeze(1)).abs(), dim=1 + ) + sigma = self.sigmas[timestep_id].reshape(-1, 1, 1, 1) + sample = (1 - sigma) * original_samples + sigma * noise + return sample.type_as(noise) + + def scale_model_input( + self, sample: torch.Tensor, timestep: int | None = None + ) -> torch.Tensor: + return sample + + def set_shift(self, shift: float) -> None: + self.shift = shift + + +EntryClass = SelfForcingFlowMatchScheduler diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_unipc_multistep.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_unipc_multistep.py new file mode 100644 index 00000000..df5e9b83 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_unipc_multistep.py @@ -0,0 +1,1207 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# Copyright 2025 TSAIL Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: check https://huggingface.co/papers/2302.04867 and https://github.com/wl-zhao/UniPC for more info +# The codebase is modified based on https://github.com/huggingface/diffusers/blob/main/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py +# ============================================================================== +# +# Modified from diffusers==0.35.0.dev0 +# +# ============================================================================== + +import math + +import numpy as np +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.schedulers.scheduling_utils import ( + KarrasDiffusionSchedulers, + SchedulerMixin, + SchedulerOutput, +) +from diffusers.utils import deprecate, is_scipy_available + +from sglang.multimodal_gen.runtime.models.schedulers.base import BaseScheduler + +if is_scipy_available(): + import scipy.stats + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_transform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +# Copied from diffusers.schedulers.scheduling_ddim.rescale_zero_terminal_snr +def rescale_zero_terminal_snr(betas): + """ + Rescales betas to have zero terminal SNR Based on https://huggingface.co/papers/2305.08891 (Algorithm 1) + + + Args: + betas (`torch.Tensor`): + the betas that the scheduler is being initialized with. + + Returns: + `torch.Tensor`: rescaled betas with zero terminal SNR + """ + # Convert betas to alphas_bar_sqrt + alphas = 1.0 - betas + alphas_cumprod = torch.cumprod(alphas, dim=0) + alphas_bar_sqrt = alphas_cumprod.sqrt() + + # Store old values. + alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone() + alphas_bar_sqrt_T = alphas_bar_sqrt[-1].clone() + + # Shift so the last timestep is zero. + alphas_bar_sqrt -= alphas_bar_sqrt_T + + # Scale so the first timestep is back to the old value. + alphas_bar_sqrt *= alphas_bar_sqrt_0 / (alphas_bar_sqrt_0 - alphas_bar_sqrt_T) + + # Convert alphas_bar_sqrt to betas + alphas_bar = alphas_bar_sqrt**2 # Revert sqrt + alphas = alphas_bar[1:] / alphas_bar[:-1] # Revert cumprod + alphas = torch.cat([alphas_bar[0:1], alphas]) + betas = 1 - alphas + + return betas + + +class UniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler): + """ + `UniPCMultistepScheduler` is a training-free framework designed for the fast sampling of diffusion models. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.0001): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.02): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + solver_order (`int`, default `2`): + The UniPC order which can be any positive integer. The effective order of accuracy is `solver_order + 1` + due to the UniC. It is recommended to use `solver_order=2` for guided sampling, and `solver_order=3` for + unconditional sampling. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + thresholding (`bool`, defaults to `False`): + Whether to use the "dynamic thresholding" method. This is unsuitable for latent-space diffusion models such + as Stable Diffusion. + dynamic_thresholding_ratio (`float`, defaults to 0.995): + The ratio for the dynamic thresholding method. Valid only when `thresholding=True`. + sample_max_value (`float`, defaults to 1.0): + The threshold value for dynamic thresholding. Valid only when `thresholding=True` and `predict_x0=True`. + predict_x0 (`bool`, defaults to `True`): + Whether to use the updating algorithm on the predicted x0. + solver_type (`str`, default `bh2`): + Solver type for UniPC. It is recommended to use `bh1` for unconditional sampling when steps < 10, and `bh2` + otherwise. + lower_order_final (`bool`, default `True`): + Whether to use lower-order solvers in the final steps. Only valid for < 15 inference steps. This can + stabilize the sampling of DPMSolver for steps < 15, especially for steps <= 10. + disable_corrector (`list`, default `[]`): + Decides which step to disable the corrector to mitigate the misalignment between `epsilon_theta(x_t, c)` + and `epsilon_theta(x_t^c, c)` which can influence convergence for a large guidance scale. Corrector is + usually disabled during the first few steps. + solver_p (`SchedulerMixin`, default `None`): + Any other scheduler that if specified, the algorithm becomes `solver_p + UniC`. + use_karras_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use Karras sigmas for step sizes in the noise schedule during the sampling process. If `True`, + the sigmas are determined according to a sequence of noise levels {σi}. + use_exponential_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use exponential sigmas for step sizes in the noise schedule during the sampling process. + use_beta_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use beta sigmas for step sizes in the noise schedule during the sampling process. Refer to [Beta + Sampling is All You Need](https://huggingface.co/papers/2407.12173) for more information. + use_flow_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use flow sigmas for step sizes in the noise schedule during the sampling process. + timestep_spacing (`str`, defaults to `"linspace"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + final_sigmas_type (`str`, defaults to `"zero"`): + The final `sigma` value for the noise schedule during the sampling process. If `"sigma_min"`, the final + sigma is the same as the last sigma in the training schedule. If `zero`, the final sigma is set to 0. + rescale_betas_zero_snr (`bool`, defaults to `False`): + Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and + dark samples instead of limiting it to samples with medium brightness. Loosely related to + [`--offset_noise`](https://github.com/huggingface/diffusers/blob/74fd735eb073eb1d774b1ab4154a0876eb82f055/examples/dreambooth/train_dreambooth.py#L506). + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: np.ndarray | list[float] | None = None, + solver_order: int = 2, + prediction_type: str = "epsilon", + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + sample_max_value: float = 1.0, + predict_x0: bool = True, + solver_type: str = "bh2", + lower_order_final: bool = True, + disable_corrector: list[int] = [], + solver_p: SchedulerMixin = None, + use_karras_sigmas: bool | None = False, + use_exponential_sigmas: bool | None = False, + use_beta_sigmas: bool | None = False, + use_flow_sigmas: bool | None = False, + flow_shift: float | None = 1.0, + timestep_spacing: str = "linspace", + steps_offset: int = 0, + final_sigmas_type: str | None = "zero", # "zero", "sigma_min" + rescale_betas_zero_snr: bool = False, + use_dynamic_shifting: bool = False, + time_shift_type: str = "exponential", + ): + if self.config.use_beta_sigmas and not is_scipy_available(): + raise ImportError( + "Make sure to install scipy if you want to use beta sigmas." + ) + if ( + sum( + [ + self.config.use_beta_sigmas, + self.config.use_exponential_sigmas, + self.config.use_karras_sigmas, + ] + ) + > 1 + ): + raise ValueError( + "Only one of `config.use_beta_sigmas`, `config.use_exponential_sigmas`, `config.use_karras_sigmas` can be used." + ) + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace( + beta_start, beta_end, num_train_timesteps, dtype=torch.float32 + ) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = ( + torch.linspace( + beta_start**0.5, + beta_end**0.5, + num_train_timesteps, + dtype=torch.float32, + ) + ** 2 + ) + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError( + f"{beta_schedule} is not implemented for {self.__class__}" + ) + + if rescale_betas_zero_snr: + self.betas = rescale_zero_terminal_snr(self.betas) + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + + if rescale_betas_zero_snr: + # Close to 0 without being 0 so first sigma is not inf + # FP16 smallest positive subnormal works well here + self.alphas_cumprod[-1] = 2**-24 + + # Currently we only support VP-type noise schedule + self.alpha_t = torch.sqrt(self.alphas_cumprod) + self.sigma_t = torch.sqrt(1 - self.alphas_cumprod) + self.lambda_t = torch.log(self.alpha_t) - torch.log(self.sigma_t) + self.sigmas = ((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5 + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + if solver_type not in ["bh1", "bh2"]: + if solver_type in ["midpoint", "heun", "logrho"]: + self.register_to_config(solver_type="bh2") + else: + raise NotImplementedError( + f"{solver_type} is not implemented for {self.__class__}" + ) + + self.predict_x0 = predict_x0 + # setable values + self.num_inference_steps = None + timesteps = np.linspace( + 0, num_train_timesteps - 1, num_train_timesteps, dtype=np.float32 + )[::-1].copy() + self.timesteps = torch.from_numpy(timesteps) + self.num_train_timesteps = num_train_timesteps + self.model_outputs = [None] * solver_order + self.timestep_list = [None] * solver_order + self.lower_order_nums = 0 + self.disable_corrector = disable_corrector + self.solver_p = solver_p + self.last_sample = None + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + BaseScheduler.__init__(self) + + @property + def step_index(self): + """ + The index counter for current timestep. It will increase 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + def set_shift(self, shift: float) -> None: + self.config.flow_shift = shift + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def set_timesteps( + self, + num_inference_steps: int, + device: str | torch.device = None, + mu: float | None = None, + ): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://huggingface.co/papers/2305.08891 + if mu is not None: + assert ( + self.config.use_dynamic_shifting + and self.config.time_shift_type == "exponential" + ) + self.config.flow_shift = np.exp(mu) + if self.config.timestep_spacing == "linspace": + timesteps = ( + np.linspace( + 0, self.config.num_train_timesteps - 1, num_inference_steps + 1 + ) + .round()[::-1][:-1] + .copy() + .astype(np.int64) + ) + elif self.config.timestep_spacing == "leading": + step_ratio = self.config.num_train_timesteps // (num_inference_steps + 1) + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = ( + (np.arange(0, num_inference_steps + 1) * step_ratio) + .round()[::-1][:-1] + .copy() + .astype(np.int64) + ) + timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = self.config.num_train_timesteps / num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = ( + np.arange(self.config.num_train_timesteps, 0, -step_ratio) + .round() + .copy() + .astype(np.int64) + ) + timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'." + ) + + sigmas = np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5) + if self.config.use_karras_sigmas: + log_sigmas = np.log(sigmas) + sigmas = np.flip(sigmas).copy() + sigmas = self._convert_to_karras( + in_sigmas=sigmas, num_inference_steps=num_inference_steps + ) + timesteps = np.array( + [self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas] + ).round() + if self.config.final_sigmas_type == "sigma_min": + sigma_last = sigmas[-1] + elif self.config.final_sigmas_type == "zero": + sigma_last = 0 + else: + raise ValueError( + f"`final_sigmas_type` must be one of 'zero', or 'sigma_min', but got {self.config.final_sigmas_type}" + ) + sigmas = np.concatenate([sigmas, [sigma_last]]).astype(np.float32) + elif self.config.use_exponential_sigmas: + log_sigmas = np.log(sigmas) + sigmas = np.flip(sigmas).copy() + sigmas = self._convert_to_exponential( + in_sigmas=sigmas, num_inference_steps=num_inference_steps + ) + timesteps = np.array( + [self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas] + ) + if self.config.final_sigmas_type == "sigma_min": + sigma_last = sigmas[-1] + elif self.config.final_sigmas_type == "zero": + sigma_last = 0 + else: + raise ValueError( + f"`final_sigmas_type` must be one of 'zero', or 'sigma_min', but got {self.config.final_sigmas_type}" + ) + sigmas = np.concatenate([sigmas, [sigma_last]]).astype(np.float32) + elif self.config.use_beta_sigmas: + log_sigmas = np.log(sigmas) + sigmas = np.flip(sigmas).copy() + sigmas = self._convert_to_beta( + in_sigmas=sigmas, num_inference_steps=num_inference_steps + ) + timesteps = np.array( + [self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas] + ) + if self.config.final_sigmas_type == "sigma_min": + sigma_last = sigmas[-1] + elif self.config.final_sigmas_type == "zero": + sigma_last = 0 + else: + raise ValueError( + f"`final_sigmas_type` must be one of 'zero', or 'sigma_min', but got {self.config.final_sigmas_type}" + ) + sigmas = np.concatenate([sigmas, [sigma_last]]).astype(np.float32) + elif self.config.use_flow_sigmas: + alphas = np.linspace( + 1, 1 / self.config.num_train_timesteps, num_inference_steps + 1 + ) + sigmas = 1.0 - alphas + sigmas = np.flip( + self.config.flow_shift + * sigmas + / (1 + (self.config.flow_shift - 1) * sigmas) + )[:-1].copy() + timesteps = (sigmas * self.config.num_train_timesteps).copy() + if self.config.final_sigmas_type == "sigma_min": + sigma_last = sigmas[-1] + elif self.config.final_sigmas_type == "zero": + sigma_last = 0 + else: + raise ValueError( + f"`final_sigmas_type` must be one of 'zero', or 'sigma_min', but got {self.config.final_sigmas_type}" + ) + sigmas = np.concatenate([sigmas, [sigma_last]]).astype(np.float32) + else: + sigmas = np.interp(timesteps, np.arange(0, len(sigmas)), sigmas) + if self.config.final_sigmas_type == "sigma_min": + sigma_last = ( + (1 - self.alphas_cumprod[0]) / self.alphas_cumprod[0] + ) ** 0.5 + elif self.config.final_sigmas_type == "zero": + sigma_last = 0 + else: + raise ValueError( + f"`final_sigmas_type` must be one of 'zero', or 'sigma_min', but got {self.config.final_sigmas_type}" + ) + sigmas = np.concatenate([sigmas, [sigma_last]]).astype(np.float32) + + self.sigmas = torch.from_numpy(sigmas) + self.timesteps = torch.from_numpy(timesteps).to( + device=device, dtype=torch.int64 + ) + + self.num_inference_steps = len(timesteps) + + self.model_outputs = [ + None, + ] * self.config.solver_order + self.lower_order_nums = 0 + self.last_sample = None + if self.solver_p: + self.solver_p.set_timesteps(self.num_inference_steps, device=device) + + # add an index counter for schedulers that allow duplicated timesteps + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample + def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor: + """ + "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the + prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by + s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing + pixels from saturation at each step. We find that dynamic thresholding results in significantly better + photorealism as well as better image-text alignment, especially when using very large guidance weights." + + https://huggingface.co/papers/2205.11487 + """ + dtype = sample.dtype + batch_size, channels, *remaining_dims = sample.shape + + if dtype not in (torch.float32, torch.float64): + sample = ( + sample.float() + ) # upcast for quantile calculation, and clamp not implemented for cpu half + + # Flatten sample for doing quantile calculation along each image + sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) + + abs_sample = sample.abs() # "a certain percentile absolute pixel value" + + s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1) + s = torch.clamp( + s, min=1, max=self.config.sample_max_value + ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] + s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0 + sample = ( + torch.clamp(sample, -s, s) / s + ) # "we threshold xt0 to the range [-s, s] and then divide by s" + + sample = sample.reshape(batch_size, channels, *remaining_dims) + sample = sample.to(dtype) + + return sample + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t + def _sigma_to_t(self, sigma, log_sigmas): + # get log sigma + log_sigma = np.log(np.maximum(sigma, 1e-10)) + + # get distribution + dists = log_sigma - log_sigmas[:, np.newaxis] + + # get sigmas range + low_idx = ( + np.cumsum((dists >= 0), axis=0) + .argmax(axis=0) + .clip(max=log_sigmas.shape[0] - 2) + ) + high_idx = low_idx + 1 + + low = log_sigmas[low_idx] + high = log_sigmas[high_idx] + + # interpolate sigmas + w = (low - log_sigma) / (low - high) + w = np.clip(w, 0, 1) + + # transform interpolation to time range + t = (1 - w) * low_idx + w * high_idx + t = t.reshape(sigma.shape) + return t + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._sigma_to_alpha_sigma_t + def _sigma_to_alpha_sigma_t(self, sigma): + if self.config.use_flow_sigmas: + alpha_t = 1 - sigma + sigma_t = sigma + else: + alpha_t = 1 / ((sigma**2 + 1) ** 0.5) + sigma_t = sigma * alpha_t + + return alpha_t, sigma_t + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras + def _convert_to_karras( + self, in_sigmas: torch.Tensor, num_inference_steps + ) -> torch.Tensor: + """Constructs the noise schedule of Karras et al. (2022).""" + + # Hack to make sure that other schedulers which copy this function don't break + # TODO: Add this logic to the other schedulers + if hasattr(self.config, "sigma_min"): + sigma_min = self.config.sigma_min + else: + sigma_min = None + + if hasattr(self.config, "sigma_max"): + sigma_max = self.config.sigma_max + else: + sigma_max = None + + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + + rho = 7.0 # 7.0 is the value used in the paper + ramp = np.linspace(0, 1, num_inference_steps) + min_inv_rho = sigma_min ** (1 / rho) + max_inv_rho = sigma_max ** (1 / rho) + sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + return sigmas + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_exponential + def _convert_to_exponential( + self, in_sigmas: torch.Tensor, num_inference_steps: int + ) -> torch.Tensor: + """Constructs an exponential noise schedule.""" + + # Hack to make sure that other schedulers which copy this function don't break + # TODO: Add this logic to the other schedulers + if hasattr(self.config, "sigma_min"): + sigma_min = self.config.sigma_min + else: + sigma_min = None + + if hasattr(self.config, "sigma_max"): + sigma_max = self.config.sigma_max + else: + sigma_max = None + + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + + sigmas = np.exp( + np.linspace(math.log(sigma_max), math.log(sigma_min), num_inference_steps) + ) + return sigmas + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_beta + def _convert_to_beta( + self, + in_sigmas: torch.Tensor, + num_inference_steps: int, + alpha: float = 0.6, + beta: float = 0.6, + ) -> torch.Tensor: + """From "Beta Sampling is All You Need" [arXiv:2407.12173] (Lee et. al, 2024)""" + + # Hack to make sure that other schedulers which copy this function don't break + # TODO: Add this logic to the other schedulers + if hasattr(self.config, "sigma_min"): + sigma_min = self.config.sigma_min + else: + sigma_min = None + + if hasattr(self.config, "sigma_max"): + sigma_max = self.config.sigma_max + else: + sigma_max = None + + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + + sigmas = np.array( + [ + sigma_min + (ppf * (sigma_max - sigma_min)) + for ppf in [ + scipy.stats.beta.ppf(timestep, alpha, beta) + for timestep in 1 - np.linspace(0, 1, num_inference_steps) + ] + ] + ) + return sigmas + + def convert_model_output( + self, + model_output: torch.Tensor, + *args, + sample: torch.Tensor = None, + **kwargs, + ) -> torch.Tensor: + r""" + Convert the model output to the corresponding type the UniPC algorithm needs. + + Args: + model_output (`torch.Tensor`): + The direct output from the learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.Tensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.Tensor`: + The converted model output. + """ + timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None) + if sample is None: + if len(args) > 1: + sample = args[1] + else: + raise ValueError("missing `sample` as a required keyword argument") + if timestep is not None: + deprecate( + "timesteps", + "1.0.0", + "Passing `timesteps` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + + if self.predict_x0: + if self.config.prediction_type == "epsilon": + x0_pred = (sample - sigma_t * model_output) / alpha_t + elif self.config.prediction_type == "sample": + x0_pred = model_output + elif self.config.prediction_type == "v_prediction": + x0_pred = alpha_t * sample - sigma_t * model_output + elif self.config.prediction_type == "flow_prediction": + sigma_t = self.sigmas[self.step_index] + x0_pred = sample - sigma_t * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, " + "`v_prediction`, or `flow_prediction` for the UniPCMultistepScheduler." + ) + + if self.config.thresholding: + x0_pred = self._threshold_sample(x0_pred) + + return x0_pred + else: + if self.config.prediction_type == "epsilon": + return model_output + elif self.config.prediction_type == "sample": + epsilon = (sample - alpha_t * model_output) / sigma_t + return epsilon + elif self.config.prediction_type == "v_prediction": + epsilon = alpha_t * model_output + sigma_t * sample + return epsilon + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or" + " `v_prediction` for the UniPCMultistepScheduler." + ) + + def multistep_uni_p_bh_update( + self, + model_output: torch.Tensor, + *args, + sample: torch.Tensor = None, + order: int = None, + **kwargs, + ) -> torch.Tensor: + """ + One step for the UniP (B(h) version). Alternatively, `self.solver_p` is used if is specified. + + Args: + model_output (`torch.Tensor`): + The direct output from the learned diffusion model at the current timestep. + prev_timestep (`int`): + The previous discrete timestep in the diffusion chain. + sample (`torch.Tensor`): + A current instance of a sample created by the diffusion process. + order (`int`): + The order of UniP at this timestep (corresponds to the *p* in UniPC-p). + + Returns: + `torch.Tensor`: + The sample tensor at the previous timestep. + """ + prev_timestep = args[0] if len(args) > 0 else kwargs.pop("prev_timestep", None) + if sample is None: + if len(args) > 1: + sample = args[1] + else: + raise ValueError("missing `sample` as a required keyword argument") + if order is None: + if len(args) > 2: + order = args[2] + else: + raise ValueError("missing `order` as a required keyword argument") + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + model_output_list = self.model_outputs + + s0 = self.timestep_list[-1] + m0 = model_output_list[-1] + x = sample + + if self.solver_p: + x_t = self.solver_p.step(model_output, s0, x).prev_sample + return x_t + + sigma_t, sigma_s0 = ( + self.sigmas[self.step_index + 1], + self.sigmas[self.step_index], + ) + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + + h = lambda_t - lambda_s0 + device = sample.device + + rks = [] + D1s = [] + for i in range(1, order): + si = self.step_index - i + mi = model_output_list[-(i + 1)] + alpha_si, sigma_si = self._sigma_to_alpha_sigma_t(self.sigmas[si]) + lambda_si = torch.log(alpha_si) - torch.log(sigma_si) + rk = (lambda_si - lambda_s0) / h + rks.append(rk) + D1s.append((mi - m0) / rk) + + rks.append(1.0) + rks = torch.tensor(rks, device=device) + + R = [] + b = [] + + hh = -h if self.predict_x0 else h + h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1 + h_phi_k = h_phi_1 / hh - 1 + + factorial_i = 1 + + if self.config.solver_type == "bh1": + B_h = hh + elif self.config.solver_type == "bh2": + B_h = torch.expm1(hh) + else: + raise NotImplementedError() + + for i in range(1, order + 1): + R.append(torch.pow(rks, i - 1)) + b.append(h_phi_k * factorial_i / B_h) + factorial_i *= i + 1 + h_phi_k = h_phi_k / hh - 1 / factorial_i + + R = torch.stack(R) + b = torch.tensor(b, device=device) + + if len(D1s) > 0: + D1s = torch.stack(D1s, dim=1) # (B, K) + # for order 2, we use a simplified version + if order == 2: + rhos_p = torch.tensor([0.5], dtype=x.dtype, device=device) + else: + rhos_p = torch.linalg.solve(R[:-1, :-1], b[:-1]).to(device).to(x.dtype) + else: + D1s = None + + if self.predict_x0: + x_t_ = sigma_t / sigma_s0 * x - alpha_t * h_phi_1 * m0 + if D1s is not None: + pred_res = torch.einsum("k,bkc...->bc...", rhos_p, D1s) + else: + pred_res = 0 + x_t = x_t_ - alpha_t * B_h * pred_res + else: + x_t_ = alpha_t / alpha_s0 * x - sigma_t * h_phi_1 * m0 + if D1s is not None: + pred_res = torch.einsum("k,bkc...->bc...", rhos_p, D1s) + else: + pred_res = 0 + x_t = x_t_ - sigma_t * B_h * pred_res + + x_t = x_t.to(x.dtype) + return x_t + + def multistep_uni_c_bh_update( + self, + this_model_output: torch.Tensor, + *args, + last_sample: torch.Tensor = None, + this_sample: torch.Tensor = None, + order: int = None, + **kwargs, + ) -> torch.Tensor: + """ + One step for the UniC (B(h) version). + + Args: + this_model_output (`torch.Tensor`): + The model outputs at `x_t`. + this_timestep (`int`): + The current timestep `t`. + last_sample (`torch.Tensor`): + The generated sample before the last predictor `x_{t-1}`. + this_sample (`torch.Tensor`): + The generated sample after the last predictor `x_{t}`. + order (`int`): + The `p` of UniC-p at this step. The effective order of accuracy should be `order + 1`. + + Returns: + `torch.Tensor`: + The corrected sample tensor at the current timestep. + """ + this_timestep = args[0] if len(args) > 0 else kwargs.pop("this_timestep", None) + if last_sample is None: + if len(args) > 1: + last_sample = args[1] + else: + raise ValueError("missing `last_sample` as a required keyword argument") + if this_sample is None: + if len(args) > 2: + this_sample = args[2] + else: + raise ValueError("missing `this_sample` as a required keyword argument") + if order is None: + if len(args) > 3: + order = args[3] + else: + raise ValueError("missing `order` as a required keyword argument") + if this_timestep is not None: + deprecate( + "this_timestep", + "1.0.0", + "Passing `this_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + model_output_list = self.model_outputs + + m0 = model_output_list[-1] + x = last_sample + x_t = this_sample + model_t = this_model_output + + sigma_t, sigma_s0 = ( + self.sigmas[self.step_index], + self.sigmas[self.step_index - 1], + ) + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + + h = lambda_t - lambda_s0 + device = this_sample.device + + rks = [] + D1s = [] + for i in range(1, order): + si = self.step_index - (i + 1) + mi = model_output_list[-(i + 1)] + alpha_si, sigma_si = self._sigma_to_alpha_sigma_t(self.sigmas[si]) + lambda_si = torch.log(alpha_si) - torch.log(sigma_si) + rk = (lambda_si - lambda_s0) / h + rks.append(rk) + D1s.append((mi - m0) / rk) + + rks.append(1.0) + rks = torch.tensor(rks, device=device) + + R = [] + b = [] + + hh = -h if self.predict_x0 else h + h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1 + h_phi_k = h_phi_1 / hh - 1 + + factorial_i = 1 + + if self.config.solver_type == "bh1": + B_h = hh + elif self.config.solver_type == "bh2": + B_h = torch.expm1(hh) + else: + raise NotImplementedError() + + for i in range(1, order + 1): + R.append(torch.pow(rks, i - 1)) + b.append(h_phi_k * factorial_i / B_h) + factorial_i *= i + 1 + h_phi_k = h_phi_k / hh - 1 / factorial_i + + R = torch.stack(R) + b = torch.tensor(b, device=device) + + if len(D1s) > 0: + D1s = torch.stack(D1s, dim=1) + else: + D1s = None + + # for order 1, we use a simplified version + if order == 1: + rhos_c = torch.tensor([0.5], dtype=x.dtype, device=device) + else: + rhos_c = torch.linalg.solve(R, b).to(device).to(x.dtype) + + if self.predict_x0: + x_t_ = sigma_t / sigma_s0 * x - alpha_t * h_phi_1 * m0 + if D1s is not None: + corr_res = torch.einsum("k,bkc...->bc...", rhos_c[:-1], D1s) + else: + corr_res = 0 + D1_t = model_t - m0 + x_t = x_t_ - alpha_t * B_h * (corr_res + rhos_c[-1] * D1_t) + else: + x_t_ = alpha_t / alpha_s0 * x - sigma_t * h_phi_1 * m0 + if D1s is not None: + corr_res = torch.einsum("k,bkc...->bc...", rhos_c[:-1], D1s) + else: + corr_res = 0 + D1_t = model_t - m0 + x_t = x_t_ - sigma_t * B_h * (corr_res + rhos_c[-1] * D1_t) + x_t = x_t.to(x.dtype) + return x_t + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.index_for_timestep + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + index_candidates = (schedule_timesteps == timestep).nonzero() + + if len(index_candidates) == 0: + step_index = len(self.timesteps) - 1 + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + elif len(index_candidates) > 1: + step_index = index_candidates[1].item() + else: + step_index = index_candidates[0].item() + + return step_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._init_step_index + def _init_step_index(self, timestep): + """ + Initialize the step_index counter for the scheduler. + """ + + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def step( + self, + model_output: torch.Tensor, + timestep: int | torch.Tensor, + sample: torch.Tensor, + return_dict: bool = True, + ) -> SchedulerOutput | tuple: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with + the multistep UniPC. + + Args: + model_output (`torch.Tensor`): + The direct output from learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.Tensor`): + A current instance of a sample created by the diffusion process. + return_dict (`bool`): + Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`. + + Returns: + [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_utils.SchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to call 'set_timesteps' after creating the scheduler" + ) + + if self.step_index is None: + self._init_step_index(timestep) + + use_corrector = ( + self.step_index > 0 + and self.step_index - 1 not in self.disable_corrector + and self.last_sample is not None + ) + + model_output_convert = self.convert_model_output(model_output, sample=sample) + if use_corrector: + sample = self.multistep_uni_c_bh_update( + this_model_output=model_output_convert, + last_sample=self.last_sample, + this_sample=sample, + order=self.this_order, + ) + + for i in range(self.config.solver_order - 1): + self.model_outputs[i] = self.model_outputs[i + 1] + self.timestep_list[i] = self.timestep_list[i + 1] + + self.model_outputs[-1] = model_output_convert + self.timestep_list[-1] = timestep + + if self.config.lower_order_final: + this_order = min( + self.config.solver_order, len(self.timesteps) - self.step_index + ) + else: + this_order = self.config.solver_order + + self.this_order = min( + this_order, self.lower_order_nums + 1 + ) # warmup for multistep + assert self.this_order > 0 + + self.last_sample = sample + prev_sample = self.multistep_uni_p_bh_update( + model_output=model_output, # pass the original non-converted model output, in case solver-p is used + sample=sample, + order=self.this_order, + ) + + if self.lower_order_nums < self.config.solver_order: + self.lower_order_nums += 1 + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample,) + + return SchedulerOutput(prev_sample=prev_sample) + + def scale_model_input(self, sample: torch.Tensor, *args, **kwargs) -> torch.Tensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.Tensor`): + The input sample. + + Returns: + `torch.Tensor`: + A scaled input sample. + """ + return sample + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.add_noise + def add_noise( + self, + original_samples: torch.Tensor, + noise: torch.Tensor, + timesteps: torch.IntTensor, + ) -> torch.Tensor: + # Make sure sigmas and timesteps have the same device and dtype as original_samples + sigmas = self.sigmas.to( + device=original_samples.device, dtype=original_samples.dtype + ) + if original_samples.device.type == "mps" and torch.is_floating_point(timesteps): + # mps does not support float64 + schedule_timesteps = self.timesteps.to( + original_samples.device, dtype=torch.float32 + ) + timesteps = timesteps.to(original_samples.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(original_samples.device) + timesteps = timesteps.to(original_samples.device) + + # begin_index is None when the scheduler is used for training or pipeline does not implement set_begin_index + if self.begin_index is None: + step_indices = [ + self.index_for_timestep(t, schedule_timesteps) for t in timesteps + ] + elif self.step_index is not None: + # add_noise is called after first denoising step (for inpainting) + step_indices = [self.step_index] * timesteps.shape[0] + else: + # add noise is called before first denoising step to create initial latent(img2img) + step_indices = [self.begin_index] * timesteps.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(original_samples.shape): + sigma = sigma.unsqueeze(-1) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + noisy_samples = alpha_t * original_samples + sigma_t * noise + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps + + +EntryClass = UniPCMultistepScheduler diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/utils.py new file mode 100644 index 00000000..73a2f062 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/utils.py @@ -0,0 +1,148 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/model_executor/utils.py +"""Utils for model executor.""" +from typing import Any + +import torch + + +def set_weight_attrs( + weight: torch.Tensor, + weight_attrs: dict[str, Any] | None, +): + """Set attributes on a weight tensor. + + This method is used to set attributes on a weight tensor. This method + will not overwrite existing attributes. + + Args: + weight: The weight tensor. + weight_attrs: A dictionary of attributes to set on the weight tensor. + """ + if weight_attrs is None: + return + for key, value in weight_attrs.items(): + assert not hasattr(weight, key), f"Overwriting existing tensor attribute: {key}" + + # NOTE(woosuk): During weight loading, we often do something like: + # narrowed_tensor = param.data.narrow(0, offset, len) + # narrowed_tensor.copy_(real_weight) + # expecting narrowed_tensor and param.data to share the same storage. + # However, on TPUs, narrowed_tensor will lazily propagate to the base + # tensor, which is param.data, leading to the redundant memory usage. + # This sometimes causes OOM errors during model loading. To avoid this, + # we sync the param tensor after its weight loader is called. + # TODO(woosuk): Remove this hack once we have a better solution. + from sglang.multimodal_gen.runtime.platforms import current_platform + + if current_platform.is_tpu() and key == "weight_loader": + value = _make_synced_weight_loader(value) + setattr(weight, key, value) + + +def _make_synced_weight_loader(original_weight_loader) -> Any: + + def _synced_weight_loader(param, *args, **kwargs): + original_weight_loader(param, *args, **kwargs) + torch._sync(param) + + return _synced_weight_loader + + +def extract_layer_index(layer_name: str) -> int: + """ + Extract the layer index from the module name. + Examples: + - "encoder.layers.0" -> 0 + - "encoder.layers.1.self_attn" -> 1 + - "2.self_attn" -> 2 + - "model.encoder.layers.0.sub.1" -> ValueError + """ + subnames = layer_name.split(".") + int_vals: list[int] = [] + for subname in subnames: + try: + int_vals.append(int(subname)) + except ValueError: + continue + assert len(int_vals) == 1, ( + f"layer name {layer_name} should" " only contain one integer" + ) + return int_vals[0] + + +def modulate( + x: torch.Tensor, + shift: torch.Tensor | None = None, + scale: torch.Tensor | None = None, +) -> torch.Tensor: + """modulate by shift and scale + + Args: + x (torch.Tensor): input tensor. + shift (torch.Tensor, optional): shift tensor. Defaults to None. + scale (torch.Tensor, optional): scale tensor. Defaults to None. + + Returns: + torch.Tensor: the output tensor after modulate. + """ + if scale is None and shift is None: + return x + elif shift is None: + return x * (1 + scale.unsqueeze(1)) # type: ignore[union-attr] + elif scale is None: + return x + shift.unsqueeze(1) # type: ignore[union-attr] + else: + return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze( + 1 + ) # type: ignore[union-attr] + + +def pred_noise_to_pred_video( + pred_noise: torch.Tensor, + noise_input_latent: torch.Tensor, + timestep: torch.Tensor, + scheduler: Any, +) -> torch.Tensor: + """ + Convert predicted noise to clean latent. + + Args: + pred_noise: the predicted noise with shape [B, C, H, W] + where B is batch_size or batch_size * num_frames + noise_input_latent: the noisy latent with shape [B, C, H, W], + timestep: the timestep with shape [1] or [bs * num_frames] or [bs, num_frames] + scheduler: the scheduler + + Returns: + the predicted video with shape [B, C, H, W] + """ + # If timestep is [bs, num_frames] + if timestep.ndim == 2: + timestep = timestep.flatten(0, 1) + assert timestep.numel() == noise_input_latent.shape[0] + elif timestep.ndim == 1: + # If timestep is [1] + if timestep.shape[0] == 1: + timestep = timestep.expand(noise_input_latent.shape[0]) + else: + assert timestep.numel() == noise_input_latent.shape[0] + else: + raise ValueError( + f"[pred_noise_to_pred_video] Invalid timestep shape: {timestep.shape}" + ) + # timestep shape should be [B] + dtype = pred_noise.dtype + device = pred_noise.device + pred_noise = pred_noise.double().to(device) + noise_input_latent = noise_input_latent.double().to(device) + sigmas = scheduler.sigmas.double().to(device) + timesteps = scheduler.timesteps.double().to(device) + timestep_id = torch.argmin( + (timesteps.unsqueeze(0) - timestep.unsqueeze(1)).abs(), dim=1 + ) + sigma_t = sigmas[timestep_id].reshape(-1, 1, 1, 1) + pred_video = noise_input_latent - sigma_t * pred_noise + return pred_video.to(dtype) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder.py new file mode 100644 index 00000000..91fa447e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder.py @@ -0,0 +1,585 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +from typing import Dict, Optional, Tuple, Union + +import torch +from diffusers.models.attention_processor import ( + ADDED_KV_ATTENTION_PROCESSORS, + CROSS_ATTENTION_PROCESSORS, + Attention, + AttentionProcessor, + AttnAddedKVProcessor, + AttnProcessor, + FusedAttnProcessor2_0, +) +from diffusers.models.autoencoders.vae import ( + Decoder, + DecoderOutput, + DiagonalGaussianDistribution, + Encoder, +) +from diffusers.models.modeling_outputs import AutoencoderKLOutput +from torch import nn + +from sglang.multimodal_gen.configs.models.vaes.flux import FluxVAEConfig + + +class AutoencoderKL(nn.Module): + r""" + A VAE model with KL loss for encoding images into latents and decoding latent representations into images. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + + Parameters: + in_channels (int, *optional*, defaults to 3): Number of channels in the input image. + out_channels (int, *optional*, defaults to 3): Number of channels in the output. + down_block_types (`Tuple[str]`, *optional*, defaults to `("DownEncoderBlock2D",)`): + Tuple of downsample block types. + up_block_types (`Tuple[str]`, *optional*, defaults to `("UpDecoderBlock2D",)`): + Tuple of upsample block types. + block_out_channels (`Tuple[int]`, *optional*, defaults to `(64,)`): + Tuple of block output channels. + act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use. + latent_channels (`int`, *optional*, defaults to 4): Number of channels in the latent space. + sample_size (`int`, *optional*, defaults to `32`): Sample input size. + scaling_factor (`float`, *optional*, defaults to 0.18215): + The component-wise standard deviation of the trained latent space computed using the first batch of the + training set. This is used to scale the latent space to have unit variance when training the diffusion + model. The latents are scaled with the formula `z = z * scaling_factor` before being passed to the + diffusion model. When decoding, the latents are scaled back to the original scale with the formula: `z = 1 + / scaling_factor * z`. For more details, refer to sections 4.3.2 and D.1 of the [High-Resolution Image + Synthesis with Latent Diffusion Models](https://huggingface.co/papers/2112.10752) paper. + force_upcast (`bool`, *optional*, default to `True`): + If enabled it will force the VAE to run in float32 for high image resolution pipelines, such as SD-XL. VAE + can be fine-tuned / trained to a lower range without losing too much precision in which case `force_upcast` + can be set to `False` - see: https://huggingface.co/madebyollin/sdxl-vae-fp16-fix + mid_block_add_attention (`bool`, *optional*, default to `True`): + If enabled, the mid_block of the Encoder and Decoder will have attention blocks. If set to false, the + mid_block will only have resnet blocks + """ + + _supports_gradient_checkpointing = True + _no_split_modules = ["BasicTransformerBlock", "ResnetBlock2D"] + + def __init__( + self, + config: FluxVAEConfig, + ): + super().__init__() + self.config = config + arch_config = config.arch_config + + in_channels = arch_config.in_channels + out_channels = arch_config.out_channels + down_block_types = arch_config.down_block_types + up_block_types = arch_config.up_block_types + block_out_channels = arch_config.block_out_channels + layers_per_block = arch_config.layers_per_block + act_fn = arch_config.act_fn + latent_channels = arch_config.latent_channels + norm_num_groups = arch_config.norm_num_groups + sample_size = arch_config.sample_size + use_quant_conv = arch_config.use_quant_conv + use_post_quant_conv = arch_config.use_post_quant_conv + mid_block_add_attention = arch_config.mid_block_add_attention + + # pass init params to Encoder + self.encoder = Encoder( + in_channels=in_channels, + out_channels=latent_channels, + down_block_types=down_block_types, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + act_fn=act_fn, + norm_num_groups=norm_num_groups, + double_z=True, + mid_block_add_attention=mid_block_add_attention, + ) + + # pass init params to Decoder + self.decoder = Decoder( + in_channels=latent_channels, + out_channels=out_channels, + up_block_types=up_block_types, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + norm_num_groups=norm_num_groups, + act_fn=act_fn, + mid_block_add_attention=mid_block_add_attention, + ) + + self.quant_conv = ( + nn.Conv2d(2 * latent_channels, 2 * latent_channels, 1) + if use_quant_conv + else None + ) + self.post_quant_conv = ( + nn.Conv2d(latent_channels, latent_channels, 1) + if use_post_quant_conv + else None + ) + + self.use_slicing = False + self.use_tiling = False + + # only relevant if vae tiling is enabled + self.tile_sample_min_size = sample_size + sample_size = ( + self.config.sample_size[0] + if isinstance(self.config.sample_size, (list, tuple)) + else self.config.sample_size + ) + self.tile_latent_min_size = int( + sample_size / (2 ** (len(self.config.block_out_channels) - 1)) + ) + self.tile_overlap_factor = 0.25 + + def enable_tiling(self, use_tiling: bool = True): + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + """ + self.use_tiling = use_tiling + + def disable_tiling(self): + r""" + Disable tiled VAE decoding. If `enable_tiling` was previously enabled, this method will go back to computing + decoding in one step. + """ + self.enable_tiling(False) + + def enable_slicing(self): + r""" + Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to + compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. + """ + self.use_slicing = True + + def disable_slicing(self): + r""" + Disable sliced VAE decoding. If `enable_slicing` was previously enabled, this method will go back to computing + decoding in one step. + """ + self.use_slicing = False + + @property + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors( + name: str, + module: torch.nn.Module, + processors: Dict[str, AttentionProcessor], + ): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor() + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor( + self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]] + ): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_default_attn_processor + def set_default_attn_processor(self): + """ + Disables custom attention processors and sets the default attention implementation. + """ + if all( + proc.__class__ in ADDED_KV_ATTENTION_PROCESSORS + for proc in self.attn_processors.values() + ): + processor = AttnAddedKVProcessor() + elif all( + proc.__class__ in CROSS_ATTENTION_PROCESSORS + for proc in self.attn_processors.values() + ): + processor = AttnProcessor() + else: + raise ValueError( + f"Cannot call `set_default_attn_processor` when attention processors are of type {next(iter(self.attn_processors.values()))}" + ) + + self.set_attn_processor(processor) + + def _encode(self, x: torch.Tensor) -> torch.Tensor: + batch_size, num_channels, height, width = x.shape + + if self.use_tiling and ( + width > self.tile_sample_min_size or height > self.tile_sample_min_size + ): + return self._tiled_encode(x) + + enc = self.encoder(x) + if self.quant_conv is not None: + enc = self.quant_conv(enc) + + return enc + + def encode( + self, x: torch.Tensor, return_dict: bool = True + ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: + """ + Encode a batch of images into latents. + + Args: + x (`torch.Tensor`): Input batch of images. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.autoencoder_kl.AutoencoderKLOutput`] instead of a plain tuple. + + Returns: + The latent representations of the encoded images. If `return_dict` is True, a + [`~models.autoencoder_kl.AutoencoderKLOutput`] is returned, otherwise a plain `tuple` is returned. + """ + if self.use_slicing and x.shape[0] > 1: + encoded_slices = [self._encode(x_slice) for x_slice in x.split(1)] + h = torch.cat(encoded_slices) + else: + h = self._encode(x) + + posterior = DiagonalGaussianDistribution(h) + + if not return_dict: + return (posterior,) + + return AutoencoderKLOutput(latent_dist=posterior) + + def _decode( + self, z: torch.Tensor, return_dict: bool = True + ) -> Union[DecoderOutput, torch.Tensor]: + if self.use_tiling and ( + z.shape[-1] > self.tile_latent_min_size + or z.shape[-2] > self.tile_latent_min_size + ): + return self.tiled_decode(z, return_dict=return_dict) + + if self.post_quant_conv is not None: + z = self.post_quant_conv(z) + + dec = self.decoder(z) + + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) + + def decode(self, z: torch.FloatTensor) -> Union[DecoderOutput, torch.FloatTensor]: + """ + Decode a batch of images. + + Args: + z (`torch.Tensor`): Input batch of latent vectors. + + Returns: + [`~models.vae.DecoderOutput`] or `tuple`: + If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is + returned. + + """ + + if self.use_slicing and z.shape[0] > 1: + decoded_slices = [self._decode(z_slice).sample for z_slice in z.split(1)] + decoded = torch.cat(decoded_slices) + else: + decoded = self._decode(z).sample + + return decoded + + def blend_v( + self, a: torch.Tensor, b: torch.Tensor, blend_extent: int + ) -> torch.Tensor: + blend_extent = min(a.shape[2], b.shape[2], blend_extent) + for y in range(blend_extent): + b[:, :, y, :] = a[:, :, -blend_extent + y, :] * (1 - y / blend_extent) + b[ + :, :, y, : + ] * (y / blend_extent) + return b + + def blend_h( + self, a: torch.Tensor, b: torch.Tensor, blend_extent: int + ) -> torch.Tensor: + blend_extent = min(a.shape[3], b.shape[3], blend_extent) + for x in range(blend_extent): + b[:, :, :, x] = a[:, :, :, -blend_extent + x] * (1 - x / blend_extent) + b[ + :, :, :, x + ] * (x / blend_extent) + return b + + def _tiled_encode(self, x: torch.Tensor) -> torch.Tensor: + r"""Encode a batch of images using a tiled encoder. + + When this option is enabled, the VAE will split the input tensor into tiles to compute encoding in several + steps. This is useful to keep memory use constant regardless of image size. The end result of tiled encoding is + different from non-tiled encoding because each tile uses a different encoder. To avoid tiling artifacts, the + tiles overlap and are blended together to form a smooth output. You may still see tile-sized changes in the + output, but they should be much less noticeable. + + Args: + x (`torch.Tensor`): Input batch of images. + + Returns: + `torch.Tensor`: + The latent representation of the encoded videos. + """ + + overlap_size = int(self.tile_sample_min_size * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_latent_min_size * self.tile_overlap_factor) + row_limit = self.tile_latent_min_size - blend_extent + + # Split the image into 512x512 tiles and encode them separately. + rows = [] + for i in range(0, x.shape[2], overlap_size): + row = [] + for j in range(0, x.shape[3], overlap_size): + tile = x[ + :, + :, + i : i + self.tile_sample_min_size, + j : j + self.tile_sample_min_size, + ] + tile = self.encoder(tile) + if self.config.use_quant_conv: + tile = self.quant_conv(tile) + row.append(tile) + rows.append(row) + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_extent) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent) + result_row.append(tile[:, :, :row_limit, :row_limit]) + result_rows.append(torch.cat(result_row, dim=3)) + + enc = torch.cat(result_rows, dim=2) + return enc + + def tiled_encode( + self, x: torch.Tensor, return_dict: bool = True + ) -> AutoencoderKLOutput: + r"""Encode a batch of images using a tiled encoder. + + When this option is enabled, the VAE will split the input tensor into tiles to compute encoding in several + steps. This is useful to keep memory use constant regardless of image size. The end result of tiled encoding is + different from non-tiled encoding because each tile uses a different encoder. To avoid tiling artifacts, the + tiles overlap and are blended together to form a smooth output. You may still see tile-sized changes in the + output, but they should be much less noticeable. + + Args: + x (`torch.Tensor`): Input batch of images. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.autoencoder_kl.AutoencoderKLOutput`] instead of a plain tuple. + + Returns: + [`~models.autoencoder_kl.AutoencoderKLOutput`] or `tuple`: + If return_dict is True, a [`~models.autoencoder_kl.AutoencoderKLOutput`] is returned, otherwise a plain + `tuple` is returned. + """ + deprecation_message = ( + "The tiled_encode implementation supporting the `return_dict` parameter is deprecated. In the future, the " + "implementation of this method will be replaced with that of `_tiled_encode` and you will no longer be able " + "to pass `return_dict`. You will also have to create a `DiagonalGaussianDistribution()` from the returned value." + ) + # deprecate("tiled_encode", "1.0.0", deprecation_message, standard_warn=False) + + overlap_size = int(self.tile_sample_min_size * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_latent_min_size * self.tile_overlap_factor) + row_limit = self.tile_latent_min_size - blend_extent + + # Split the image into 512x512 tiles and encode them separately. + rows = [] + for i in range(0, x.shape[2], overlap_size): + row = [] + for j in range(0, x.shape[3], overlap_size): + tile = x[ + :, + :, + i : i + self.tile_sample_min_size, + j : j + self.tile_sample_min_size, + ] + tile = self.encoder(tile) + if self.config.use_quant_conv: + tile = self.quant_conv(tile) + row.append(tile) + rows.append(row) + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_extent) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent) + result_row.append(tile[:, :, :row_limit, :row_limit]) + result_rows.append(torch.cat(result_row, dim=3)) + + moments = torch.cat(result_rows, dim=2) + posterior = DiagonalGaussianDistribution(moments) + + if not return_dict: + return (posterior,) + + return AutoencoderKLOutput(latent_dist=posterior) + + def tiled_decode( + self, z: torch.Tensor, return_dict: bool = True + ) -> Union[DecoderOutput, torch.Tensor]: + r""" + Decode a batch of images using a tiled decoder. + + Args: + z (`torch.Tensor`): Input batch of latent vectors. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.vae.DecoderOutput`] instead of a plain tuple. + + Returns: + [`~models.vae.DecoderOutput`] or `tuple`: + If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is + returned. + """ + overlap_size = int(self.tile_latent_min_size * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_sample_min_size * self.tile_overlap_factor) + row_limit = self.tile_sample_min_size - blend_extent + + # Split z into overlapping 64x64 tiles and decode them separately. + # The tiles have an overlap to avoid seams between tiles. + rows = [] + for i in range(0, z.shape[2], overlap_size): + row = [] + for j in range(0, z.shape[3], overlap_size): + tile = z[ + :, + :, + i : i + self.tile_latent_min_size, + j : j + self.tile_latent_min_size, + ] + if self.config.use_post_quant_conv: + tile = self.post_quant_conv(tile) + decoded = self.decoder(tile) + row.append(decoded) + rows.append(row) + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_extent) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent) + result_row.append(tile[:, :, :row_limit, :row_limit]) + result_rows.append(torch.cat(result_row, dim=3)) + + dec = torch.cat(result_rows, dim=2) + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) + + def forward( + self, + sample: torch.Tensor, + sample_posterior: bool = False, + generator: Optional[torch.Generator] = None, + ) -> Union[DecoderOutput, torch.Tensor]: + r""" + Args: + sample (`torch.Tensor`): Input sample. + sample_posterior (`bool`, *optional*, defaults to `False`): + Whether to sample from the posterior. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`DecoderOutput`] instead of a plain tuple. + """ + x = sample + posterior = self.encode(x).latent_dist + if sample_posterior: + z = posterior.sample(generator=generator) + else: + z = posterior.mode() + dec = self.decode(z).sample + + return dec + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.fuse_qkv_projections + def fuse_qkv_projections(self): + """ + Enables fused QKV projections. For self-attention modules, all projection matrices (i.e., query, key, value) + are fused. For cross-attention modules, key and value projection matrices are fused. + + > [!WARNING] > This API is 🧪 experimental. + """ + self.original_attn_processors = None + + for _, attn_processor in self.attn_processors.items(): + if "Added" in str(attn_processor.__class__.__name__): + raise ValueError( + "`fuse_qkv_projections()` is not supported for models having added KV projections." + ) + + self.original_attn_processors = self.attn_processors + + for module in self.modules(): + if isinstance(module, Attention): + module.fuse_projections(fuse=True) + + self.set_attn_processor(FusedAttnProcessor2_0()) + + +EntryClass = AutoencoderKL diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_flux2.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_flux2.py new file mode 100644 index 00000000..9df72c09 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_flux2.py @@ -0,0 +1,524 @@ +import math +from typing import Dict, Optional, Tuple, Union + +import torch +import torch.nn as nn +from diffusers.models.attention_processor import ( + ADDED_KV_ATTENTION_PROCESSORS, + CROSS_ATTENTION_PROCESSORS, + AttentionProcessor, + AttnAddedKVProcessor, + AttnProcessor, +) +from diffusers.models.autoencoders.vae import ( + Decoder, + DecoderOutput, + DiagonalGaussianDistribution, + Encoder, +) +from diffusers.models.modeling_outputs import AutoencoderKLOutput + +from sglang.multimodal_gen.configs.models.vaes.flux import Flux2VAEConfig +from sglang.multimodal_gen.runtime.models.vaes.common import ParallelTiledVAE + + +class AutoencoderKLFlux2(nn.Module, ParallelTiledVAE): + r""" + A VAE model with KL loss for encoding images into latents and decoding latent representations into images. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + + Parameters: + """ + + _supports_gradient_checkpointing = True + _no_split_modules = ["BasicTransformerBlock", "ResnetBlock2D"] + + def __init__( + self, + config: Flux2VAEConfig, + ): + super().__init__() + + self.config = config + arch_config = config.arch_config + + in_channels: int = arch_config.in_channels + out_channels: int = arch_config.out_channels + down_block_types: Tuple[str, ...] = arch_config.down_block_types + up_block_types: Tuple[str, ...] = arch_config.up_block_types + block_out_channels: Tuple[int, ...] = arch_config.block_out_channels + layers_per_block: int = arch_config.layers_per_block + act_fn: str = arch_config.act_fn + latent_channels: int = arch_config.latent_channels + norm_num_groups: int = arch_config.norm_num_groups + sample_size: int = arch_config.sample_size + force_upcast: bool = arch_config.force_upcast + use_quant_conv: bool = arch_config.use_quant_conv + use_post_quant_conv: bool = arch_config.use_post_quant_conv + mid_block_add_attention: bool = arch_config.mid_block_add_attention + batch_norm_eps: float = arch_config.batch_norm_eps + batch_norm_momentum: float = arch_config.batch_norm_momentum + patch_size: Tuple[int, int] = arch_config.patch_size + # pass init params to Encoder + self.encoder = Encoder( + in_channels=in_channels, + out_channels=latent_channels, + down_block_types=down_block_types, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + act_fn=act_fn, + norm_num_groups=norm_num_groups, + double_z=True, + mid_block_add_attention=mid_block_add_attention, + ) + + # pass init params to Decoder + self.decoder = Decoder( + in_channels=latent_channels, + out_channels=out_channels, + up_block_types=up_block_types, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + norm_num_groups=norm_num_groups, + act_fn=act_fn, + mid_block_add_attention=mid_block_add_attention, + ) + + self.quant_conv = ( + nn.Conv2d(2 * latent_channels, 2 * latent_channels, 1) + if use_quant_conv + else None + ) + self.post_quant_conv = ( + nn.Conv2d(latent_channels, latent_channels, 1) + if use_post_quant_conv + else None + ) + + self.bn = nn.BatchNorm2d( + math.prod(patch_size) * latent_channels, + eps=batch_norm_eps, + momentum=batch_norm_momentum, + affine=False, + track_running_stats=True, + ) + + self.use_slicing = False + self.use_tiling = False + + # only relevant if vae tiling is enabled + self.tile_sample_min_size = self.config.sample_size + sample_size = ( + self.config.sample_size[0] + if isinstance(self.config.sample_size, (list, tuple)) + else self.config.sample_size + ) + self.tile_latent_min_size = int( + sample_size / (2 ** (len(self.config.block_out_channels) - 1)) + ) + self.tile_overlap_factor = 0.25 + + @property + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors( + name: str, + module: torch.nn.Module, + processors: Dict[str, AttentionProcessor], + ): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor() + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor( + self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]] + ): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_default_attn_processor + def set_default_attn_processor(self): + """ + Disables custom attention processors and sets the default attention implementation. + """ + if all( + proc.__class__ in ADDED_KV_ATTENTION_PROCESSORS + for proc in self.attn_processors.values() + ): + processor = AttnAddedKVProcessor() + elif all( + proc.__class__ in CROSS_ATTENTION_PROCESSORS + for proc in self.attn_processors.values() + ): + processor = AttnProcessor() + else: + raise ValueError( + f"Cannot call `set_default_attn_processor` when attention processors are of type {next(iter(self.attn_processors.values()))}" + ) + + self.set_attn_processor(processor) + + def _encode(self, x: torch.Tensor) -> torch.Tensor: + batch_size, num_channels, height, width = x.shape + + if self.use_tiling and ( + width > self.tile_sample_min_size or height > self.tile_sample_min_size + ): + return self._tiled_encode(x) + + enc = self.encoder(x) + if self.quant_conv is not None: + enc = self.quant_conv(enc) + + return enc + + def encode( + self, x: torch.Tensor, return_dict: bool = True + ) -> Union[DiagonalGaussianDistribution]: + """ + Encode a batch of images into latents. + + Args: + x (`torch.Tensor`): Input batch of images. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.autoencoder_kl.AutoencoderKLOutput`] instead of a plain tuple. + + Returns: + The latent representations of the encoded images. If `return_dict` is True, a + [`~models.autoencoder_kl.AutoencoderKLOutput`] is returned, otherwise a plain `tuple` is returned. + """ + + if x.ndim == 5: + assert x.shape[2] == 1 + x = x.squeeze(2) + + if self.use_slicing and x.shape[0] > 1: + encoded_slices = [self._encode(x_slice) for x_slice in x.split(1)] + h = torch.cat(encoded_slices) + else: + h = self._encode(x) + + posterior = DiagonalGaussianDistribution(h) + return posterior + + def _decode( + self, z: torch.Tensor, return_dict: bool = True + ) -> Union[DecoderOutput, torch.Tensor]: + if self.use_tiling and ( + z.shape[-1] > self.tile_latent_min_size + or z.shape[-2] > self.tile_latent_min_size + ): + return self.tiled_decode(z, return_dict=return_dict) + + if self.post_quant_conv is not None: + z = self.post_quant_conv(z) + + dec = self.decoder(z) + + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) + + def decode( + self, z: torch.FloatTensor, return_dict: bool = True, generator=None + ) -> Union[DecoderOutput, torch.FloatTensor]: + """ + Decode a batch of images. + + Args: + z (`torch.Tensor`): Input batch of latent vectors. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.vae.DecoderOutput`] instead of a plain tuple. + + Returns: + [`~models.vae.DecoderOutput`] or `tuple`: + If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is + returned. + + """ + if self.use_slicing and z.shape[0] > 1: + decoded_slices = [self._decode(z_slice).sample for z_slice in z.split(1)] + decoded = torch.cat(decoded_slices) + else: + decoded = self._decode(z).sample + + return decoded + + def blend_v( + self, a: torch.Tensor, b: torch.Tensor, blend_extent: int + ) -> torch.Tensor: + blend_extent = min(a.shape[2], b.shape[2], blend_extent) + for y in range(blend_extent): + b[:, :, y, :] = a[:, :, -blend_extent + y, :] * (1 - y / blend_extent) + b[ + :, :, y, : + ] * (y / blend_extent) + return b + + def blend_h( + self, a: torch.Tensor, b: torch.Tensor, blend_extent: int + ) -> torch.Tensor: + blend_extent = min(a.shape[3], b.shape[3], blend_extent) + for x in range(blend_extent): + b[:, :, :, x] = a[:, :, :, -blend_extent + x] * (1 - x / blend_extent) + b[ + :, :, :, x + ] * (x / blend_extent) + return b + + def _tiled_encode(self, x: torch.Tensor) -> torch.Tensor: + r"""Encode a batch of images using a tiled encoder. + + When this option is enabled, the VAE will split the input tensor into tiles to compute encoding in several + steps. This is useful to keep memory use constant regardless of image size. The end result of tiled encoding is + different from non-tiled encoding because each tile uses a different encoder. To avoid tiling artifacts, the + tiles overlap and are blended together to form a smooth output. You may still see tile-sized changes in the + output, but they should be much less noticeable. + + Args: + x (`torch.Tensor`): Input batch of images. + + Returns: + `torch.Tensor`: + The latent representation of the encoded videos. + """ + + overlap_size = int(self.tile_sample_min_size * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_latent_min_size * self.tile_overlap_factor) + row_limit = self.tile_latent_min_size - blend_extent + + # Split the image into 512x512 tiles and encode them separately. + rows = [] + for i in range(0, x.shape[2], overlap_size): + row = [] + for j in range(0, x.shape[3], overlap_size): + tile = x[ + :, + :, + i : i + self.tile_sample_min_size, + j : j + self.tile_sample_min_size, + ] + tile = self.encoder(tile) + if self.config.use_quant_conv: + tile = self.quant_conv(tile) + row.append(tile) + rows.append(row) + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_extent) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent) + result_row.append(tile[:, :, :row_limit, :row_limit]) + result_rows.append(torch.cat(result_row, dim=3)) + + enc = torch.cat(result_rows, dim=2) + return enc + + def tiled_encode( + self, x: torch.Tensor, return_dict: bool = True + ) -> AutoencoderKLOutput: + r"""Encode a batch of images using a tiled encoder. + + When this option is enabled, the VAE will split the input tensor into tiles to compute encoding in several + steps. This is useful to keep memory use constant regardless of image size. The end result of tiled encoding is + different from non-tiled encoding because each tile uses a different encoder. To avoid tiling artifacts, the + tiles overlap and are blended together to form a smooth output. You may still see tile-sized changes in the + output, but they should be much less noticeable. + + Args: + x (`torch.Tensor`): Input batch of images. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.autoencoder_kl.AutoencoderKLOutput`] instead of a plain tuple. + + Returns: + [`~models.autoencoder_kl.AutoencoderKLOutput`] or `tuple`: + If return_dict is True, a [`~models.autoencoder_kl.AutoencoderKLOutput`] is returned, otherwise a plain + `tuple` is returned. + """ + deprecation_message = ( + "The tiled_encode implementation supporting the `return_dict` parameter is deprecated. In the future, the " + "implementation of this method will be replaced with that of `_tiled_encode` and you will no longer be able " + "to pass `return_dict`. You will also have to create a `DiagonalGaussianDistribution()` from the returned value." + ) + + overlap_size = int(self.tile_sample_min_size * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_latent_min_size * self.tile_overlap_factor) + row_limit = self.tile_latent_min_size - blend_extent + + # Split the image into 512x512 tiles and encode them separately. + rows = [] + for i in range(0, x.shape[2], overlap_size): + row = [] + for j in range(0, x.shape[3], overlap_size): + tile = x[ + :, + :, + i : i + self.tile_sample_min_size, + j : j + self.tile_sample_min_size, + ] + tile = self.encoder(tile) + if self.config.use_quant_conv: + tile = self.quant_conv(tile) + row.append(tile) + rows.append(row) + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_extent) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent) + result_row.append(tile[:, :, :row_limit, :row_limit]) + result_rows.append(torch.cat(result_row, dim=3)) + + moments = torch.cat(result_rows, dim=2) + posterior = DiagonalGaussianDistribution(moments) + + if not return_dict: + return (posterior,) + + return AutoencoderKLOutput(latent_dist=posterior) + + def tiled_decode( + self, z: torch.Tensor, return_dict: bool = True + ) -> Union[DecoderOutput, torch.Tensor]: + r""" + Decode a batch of images using a tiled decoder. + + Args: + z (`torch.Tensor`): Input batch of latent vectors. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.vae.DecoderOutput`] instead of a plain tuple. + + Returns: + [`~models.vae.DecoderOutput`] or `tuple`: + If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is + returned. + """ + overlap_size = int(self.tile_latent_min_size * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_sample_min_size * self.tile_overlap_factor) + row_limit = self.tile_sample_min_size - blend_extent + + # Split z into overlapping 64x64 tiles and decode them separately. + # The tiles have an overlap to avoid seams between tiles. + rows = [] + for i in range(0, z.shape[2], overlap_size): + row = [] + for j in range(0, z.shape[3], overlap_size): + tile = z[ + :, + :, + i : i + self.tile_latent_min_size, + j : j + self.tile_latent_min_size, + ] + if self.config.use_post_quant_conv: + tile = self.post_quant_conv(tile) + decoded = self.decoder(tile) + row.append(decoded) + rows.append(row) + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_extent) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent) + result_row.append(tile[:, :, :row_limit, :row_limit]) + result_rows.append(torch.cat(result_row, dim=3)) + + dec = torch.cat(result_rows, dim=2) + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) + + def forward( + self, + sample: torch.Tensor, + sample_posterior: bool = False, + return_dict: bool = True, + generator: Optional[torch.Generator] = None, + ) -> Union[DecoderOutput, torch.Tensor]: + r""" + Args: + sample (`torch.Tensor`): Input sample. + sample_posterior (`bool`, *optional*, defaults to `False`): + Whether to sample from the posterior. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`DecoderOutput`] instead of a plain tuple. + """ + x = sample + posterior = self.encode(x).latent_dist + if sample_posterior: + z = posterior.sample(generator=generator) + else: + z = posterior.mode() + dec = self.decode(z).sample + + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) + + +EntryClass = AutoencoderKLFlux2 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py new file mode 100644 index 00000000..e860b313 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py @@ -0,0 +1,1170 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +from typing import Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.models.activations import get_activation +from diffusers.models.autoencoders.vae import ( + DecoderOutput, + DiagonalGaussianDistribution, +) +from diffusers.models.modeling_outputs import AutoencoderKLOutput + +from sglang.multimodal_gen.configs.models.vaes.qwenimage import QwenImageVAEConfig +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) # pylint: disable=invalid-name + +CACHE_T = 2 + + +class QwenImageCausalConv3d(nn.Conv3d): + r""" + A custom 3D causal convolution layer with feature caching support. + + This layer extends the standard Conv3D layer by ensuring causality in the time dimension and handling feature + caching for efficient inference. + + Args: + in_channels (int): Number of channels in the input image + out_channels (int): Number of channels produced by the convolution + kernel_size (int or tuple): Size of the convolving kernel + stride (int or tuple, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to all three sides of the input. Default: 0 + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: Union[int, Tuple[int, int, int]], + stride: Union[int, Tuple[int, int, int]] = 1, + padding: Union[int, Tuple[int, int, int]] = 0, + ) -> None: + super().__init__( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + ) + + # Set up causal padding + self._padding = ( + self.padding[2], + self.padding[2], + self.padding[1], + self.padding[1], + 2 * self.padding[0], + 0, + ) + self.padding = (0, 0, 0) + + def forward(self, x, cache_x=None): + padding = list(self._padding) + if cache_x is not None and self._padding[4] > 0: + cache_x = cache_x.to(x.device) + x = torch.cat([cache_x, x], dim=2) + padding[4] -= cache_x.shape[2] + x = F.pad(x, padding) + return super().forward(x) + + +class QwenImageRMS_norm(nn.Module): + r""" + A custom RMS normalization layer. + + Args: + dim (int): The number of dimensions to normalize over. + channel_first (bool, optional): Whether the input tensor has channels as the first dimension. + Default is True. + images (bool, optional): Whether the input represents image data. Default is True. + bias (bool, optional): Whether to include a learnable bias term. Default is False. + """ + + def __init__( + self, + dim: int, + channel_first: bool = True, + images: bool = True, + bias: bool = False, + ) -> None: + super().__init__() + broadcastable_dims = (1, 1, 1) if not images else (1, 1) + shape = (dim, *broadcastable_dims) if channel_first else (dim,) + + self.channel_first = channel_first + self.scale = dim**0.5 + self.gamma = nn.Parameter(torch.ones(shape)) + self.bias = nn.Parameter(torch.zeros(shape)) if bias else 0.0 + + def forward(self, x): + return ( + F.normalize(x, dim=(1 if self.channel_first else -1)) + * self.scale + * self.gamma + + self.bias + ) + + +class QwenImageUpsample(nn.Upsample): + r""" + Perform upsampling while ensuring the output tensor has the same data type as the input. + + Returns: + torch.Tensor: Upsampled tensor with the same data type as the input. + """ + + def forward(self, x): + return super().forward(x.float()).type_as(x) + + +class QwenImageResample(nn.Module): + r""" + A custom resampling module for 2D and 3D data. + + Args: + dim (int): The number of input/output channels. + mode (str): The resampling mode. Must be one of: + - 'none': No resampling (identity operation). + - 'upsample2d': 2D upsampling with nearest-exact interpolation and convolution. + - 'upsample3d': 3D upsampling with nearest-exact interpolation, convolution, and causal 3D convolution. + - 'downsample2d': 2D downsampling with zero-padding and convolution. + - 'downsample3d': 3D downsampling with zero-padding, convolution, and causal 3D convolution. + """ + + def __init__(self, dim: int, mode: str) -> None: + super().__init__() + self.dim = dim + self.mode = mode + + # layers + if mode == "upsample2d": + self.resample = nn.Sequential( + QwenImageUpsample(scale_factor=(2.0, 2.0), mode="nearest-exact"), + nn.Conv2d(dim, dim // 2, 3, padding=1), + ) + elif mode == "upsample3d": + self.resample = nn.Sequential( + QwenImageUpsample(scale_factor=(2.0, 2.0), mode="nearest-exact"), + nn.Conv2d(dim, dim // 2, 3, padding=1), + ) + self.time_conv = QwenImageCausalConv3d( + dim, dim * 2, (3, 1, 1), padding=(1, 0, 0) + ) + + elif mode == "downsample2d": + self.resample = nn.Sequential( + nn.ZeroPad2d((0, 1, 0, 1)), nn.Conv2d(dim, dim, 3, stride=(2, 2)) + ) + elif mode == "downsample3d": + self.resample = nn.Sequential( + nn.ZeroPad2d((0, 1, 0, 1)), nn.Conv2d(dim, dim, 3, stride=(2, 2)) + ) + self.time_conv = QwenImageCausalConv3d( + dim, dim, (3, 1, 1), stride=(2, 1, 1), padding=(0, 0, 0) + ) + + else: + self.resample = nn.Identity() + + def forward(self, x, feat_cache=None, feat_idx=[0]): + b, c, t, h, w = x.size() + if self.mode == "upsample3d": + if feat_cache is not None: + idx = feat_idx[0] + if feat_cache[idx] is None: + feat_cache[idx] = "Rep" + feat_idx[0] += 1 + else: + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if ( + cache_x.shape[2] < 2 + and feat_cache[idx] is not None + and feat_cache[idx] != "Rep" + ): + # cache last frame of last two chunk + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :] + .unsqueeze(2) + .to(cache_x.device), + cache_x, + ], + dim=2, + ) + if ( + cache_x.shape[2] < 2 + and feat_cache[idx] is not None + and feat_cache[idx] == "Rep" + ): + cache_x = torch.cat( + [torch.zeros_like(cache_x).to(cache_x.device), cache_x], + dim=2, + ) + if feat_cache[idx] == "Rep": + x = self.time_conv(x) + else: + x = self.time_conv(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + + x = x.reshape(b, 2, c, t, h, w) + x = torch.stack((x[:, 0, :, :, :, :], x[:, 1, :, :, :, :]), 3) + x = x.reshape(b, c, t * 2, h, w) + t = x.shape[2] + x = x.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w) + x = self.resample(x) + x = x.view(b, t, x.size(1), x.size(2), x.size(3)).permute(0, 2, 1, 3, 4) + + if self.mode == "downsample3d": + if feat_cache is not None: + idx = feat_idx[0] + if feat_cache[idx] is None: + feat_cache[idx] = x.clone() + feat_idx[0] += 1 + else: + cache_x = x[:, :, -1:, :, :].clone() + x = self.time_conv( + torch.cat([feat_cache[idx][:, :, -1:, :, :], x], 2) + ) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + return x + + +class QwenImageResidualBlock(nn.Module): + r""" + A custom residual block module. + + Args: + in_dim (int): Number of input channels. + out_dim (int): Number of output channels. + dropout (float, optional): Dropout rate for the dropout layer. Default is 0.0. + non_linearity (str, optional): Type of non-linearity to use. Default is "silu". + """ + + def __init__( + self, + in_dim: int, + out_dim: int, + dropout: float = 0.0, + non_linearity: str = "silu", + ) -> None: + super().__init__() + self.in_dim = in_dim + self.out_dim = out_dim + self.nonlinearity = get_activation(non_linearity) + + # layers + self.norm1 = QwenImageRMS_norm(in_dim, images=False) + self.conv1 = QwenImageCausalConv3d(in_dim, out_dim, 3, padding=1) + self.norm2 = QwenImageRMS_norm(out_dim, images=False) + self.dropout = nn.Dropout(dropout) + self.conv2 = QwenImageCausalConv3d(out_dim, out_dim, 3, padding=1) + self.conv_shortcut = ( + QwenImageCausalConv3d(in_dim, out_dim, 1) + if in_dim != out_dim + else nn.Identity() + ) + + def forward(self, x, feat_cache=None, feat_idx=[0]): + # Apply shortcut connection + h = self.conv_shortcut(x) + + # First normalization and activation + x = self.norm1(x) + x = self.nonlinearity(x) + + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), + cache_x, + ], + dim=2, + ) + + x = self.conv1(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv1(x) + + # Second normalization and activation + x = self.norm2(x) + x = self.nonlinearity(x) + + # Dropout + x = self.dropout(x) + + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), + cache_x, + ], + dim=2, + ) + + x = self.conv2(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv2(x) + + # Add residual connection + return x + h + + +class QwenImageAttentionBlock(nn.Module): + r""" + Causal self-attention with a single head. + + Args: + dim (int): The number of channels in the input tensor. + """ + + def __init__(self, dim): + super().__init__() + self.dim = dim + + # layers + self.norm = QwenImageRMS_norm(dim) + self.to_qkv = nn.Conv2d(dim, dim * 3, 1) + self.proj = nn.Conv2d(dim, dim, 1) + + def forward(self, x): + identity = x + batch_size, channels, time, height, width = x.size() + + x = x.permute(0, 2, 1, 3, 4).reshape(batch_size * time, channels, height, width) + x = self.norm(x) + + # compute query, key, value + qkv = self.to_qkv(x) + qkv = qkv.reshape(batch_size * time, 1, channels * 3, -1) + qkv = qkv.permute(0, 1, 3, 2).contiguous() + q, k, v = qkv.chunk(3, dim=-1) + + # apply attention + x = F.scaled_dot_product_attention(q, k, v) + + x = ( + x.squeeze(1) + .permute(0, 2, 1) + .reshape(batch_size * time, channels, height, width) + ) + + # output projection + x = self.proj(x) + + # Reshape back: [(b*t), c, h, w] -> [b, c, t, h, w] + x = x.view(batch_size, time, channels, height, width) + x = x.permute(0, 2, 1, 3, 4) + + return x + identity + + +class QwenImageMidBlock(nn.Module): + """ + Middle block for QwenImageVAE encoder and decoder. + + Args: + dim (int): Number of input/output channels. + dropout (float): Dropout rate. + non_linearity (str): Type of non-linearity to use. + """ + + def __init__( + self, + dim: int, + dropout: float = 0.0, + non_linearity: str = "silu", + num_layers: int = 1, + ): + super().__init__() + self.dim = dim + + # Create the components + resnets = [QwenImageResidualBlock(dim, dim, dropout, non_linearity)] + attentions = [] + for _ in range(num_layers): + attentions.append(QwenImageAttentionBlock(dim)) + resnets.append(QwenImageResidualBlock(dim, dim, dropout, non_linearity)) + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + self.gradient_checkpointing = False + + def forward(self, x, feat_cache=None, feat_idx=[0]): + # First residual block + x = self.resnets[0](x, feat_cache, feat_idx) + + # Process through attention and residual blocks + for attn, resnet in zip(self.attentions, self.resnets[1:]): + if attn is not None: + x = attn(x) + + x = resnet(x, feat_cache, feat_idx) + + return x + + +class QwenImageEncoder3d(nn.Module): + r""" + A 3D encoder module. + + Args: + dim (int): The base number of channels in the first layer. + z_dim (int): The dimensionality of the latent space. + dim_mult (list of int): Multipliers for the number of channels in each block. + num_res_blocks (int): Number of residual blocks in each block. + attn_scales (list of float): Scales at which to apply attention mechanisms. + temperal_downsample (list of bool): Whether to downsample temporally in each block. + dropout (float): Dropout rate for the dropout layers. + non_linearity (str): Type of non-linearity to use. + """ + + def __init__( + self, + dim=128, + z_dim=4, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_downsample=[True, True, False], + dropout=0.0, + non_linearity: str = "silu", + ): + super().__init__() + # dim = config.arch_config.dim + # z_dim = config.arch_config.z_dim + # dim_mult = config.arch_config.dim_mult + # num_res_blocks = config.arch_config.num_res_blocks + # attn_scales = config.arch_config.attn_scales + # temperal_downsample = config.arch_config.temperal_downsample + # dropout = config.arch_config.dropout + # non_linearity = config.arch_config.non_linearity + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.temperal_downsample = temperal_downsample + self.nonlinearity = get_activation(non_linearity) + + # dimensions + dims = [dim * u for u in [1] + dim_mult] + scale = 1.0 + + # init block + self.conv_in = QwenImageCausalConv3d(3, dims[0], 3, padding=1) + + # downsample blocks + self.down_blocks = nn.ModuleList([]) + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + # residual (+attention) blocks + for _ in range(num_res_blocks): + self.down_blocks.append( + QwenImageResidualBlock(in_dim, out_dim, dropout) + ) + if scale in attn_scales: + self.down_blocks.append(QwenImageAttentionBlock(out_dim)) + in_dim = out_dim + + # downsample block + if i != len(dim_mult) - 1: + mode = "downsample3d" if temperal_downsample[i] else "downsample2d" + self.down_blocks.append(QwenImageResample(out_dim, mode=mode)) + scale /= 2.0 + + # middle blocks + self.mid_block = QwenImageMidBlock( + out_dim, dropout, non_linearity, num_layers=1 + ) + + # output blocks + self.norm_out = QwenImageRMS_norm(out_dim, images=False) + self.conv_out = QwenImageCausalConv3d(out_dim, z_dim, 3, padding=1) + + self.gradient_checkpointing = False + + def forward(self, x, feat_cache=None, feat_idx=[0]): + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), + cache_x, + ], + dim=2, + ) + x = self.conv_in(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv_in(x) + + ## downsamples + for layer in self.down_blocks: + if feat_cache is not None: + x = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + ## middle + x = self.mid_block(x, feat_cache, feat_idx) + + ## head + x = self.norm_out(x) + x = self.nonlinearity(x) + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), + cache_x, + ], + dim=2, + ) + x = self.conv_out(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv_out(x) + return x + + +class QwenImageUpBlock(nn.Module): + """ + A block that handles upsampling for the QwenImageVAE decoder. + + Args: + in_dim (int): Input dimension + out_dim (int): Output dimension + num_res_blocks (int): Number of residual blocks + dropout (float): Dropout rate + upsample_mode (str, optional): Mode for upsampling ('upsample2d' or 'upsample3d') + non_linearity (str): Type of non-linearity to use + """ + + def __init__( + self, + in_dim: int, + out_dim: int, + num_res_blocks: int, + dropout: float = 0.0, + upsample_mode: Optional[str] = None, + non_linearity: str = "silu", + ): + super().__init__() + self.in_dim = in_dim + self.out_dim = out_dim + + # Create layers list + resnets = [] + # Add residual blocks and attention if needed + current_dim = in_dim + for _ in range(num_res_blocks + 1): + resnets.append( + QwenImageResidualBlock(current_dim, out_dim, dropout, non_linearity) + ) + current_dim = out_dim + + self.resnets = nn.ModuleList(resnets) + + # Add upsampling layer if needed + self.upsamplers = None + if upsample_mode is not None: + self.upsamplers = nn.ModuleList( + [QwenImageResample(out_dim, mode=upsample_mode)] + ) + + self.gradient_checkpointing = False + + def forward(self, x, feat_cache=None, feat_idx=[0]): + """ + Forward pass through the upsampling block. + + Args: + x (torch.Tensor): Input tensor + feat_cache (list, optional): Feature cache for causal convolutions + feat_idx (list, optional): Feature index for cache management + + Returns: + torch.Tensor: Output tensor + """ + for resnet in self.resnets: + if feat_cache is not None: + x = resnet(x, feat_cache, feat_idx) + else: + x = resnet(x) + + if self.upsamplers is not None: + if feat_cache is not None: + x = self.upsamplers[0](x, feat_cache, feat_idx) + else: + x = self.upsamplers[0](x) + return x + + +class QwenImageDecoder3d(nn.Module): + r""" + A 3D decoder module. + + Args: + dim (int): The base number of channels in the first layer. + z_dim (int): The dimensionality of the latent space. + dim_mult (list of int): Multipliers for the number of channels in each block. + num_res_blocks (int): Number of residual blocks in each block. + attn_scales (list of float): Scales at which to apply attention mechanisms. + temperal_upsample (list of bool): Whether to upsample temporally in each block. + dropout (float): Dropout rate for the dropout layers. + non_linearity (str): Type of non-linearity to use. + """ + + def __init__( + self, + dim=128, + z_dim=4, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_upsample=[False, True, True], + dropout=0.0, + non_linearity: str = "silu", + ): + super().__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.temperal_upsample = temperal_upsample + + self.nonlinearity = get_activation(non_linearity) + + # dimensions + dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + scale = 1.0 / 2 ** (len(dim_mult) - 2) + + # init block + self.conv_in = QwenImageCausalConv3d(z_dim, dims[0], 3, padding=1) + + # middle blocks + self.mid_block = QwenImageMidBlock( + dims[0], dropout, non_linearity, num_layers=1 + ) + + # upsample blocks + self.up_blocks = nn.ModuleList([]) + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + # residual (+attention) blocks + if i > 0: + in_dim = in_dim // 2 + + # Determine if we need upsampling + upsample_mode = None + if i != len(dim_mult) - 1: + upsample_mode = "upsample3d" if temperal_upsample[i] else "upsample2d" + + # Create and add the upsampling block + up_block = QwenImageUpBlock( + in_dim=in_dim, + out_dim=out_dim, + num_res_blocks=num_res_blocks, + dropout=dropout, + upsample_mode=upsample_mode, + non_linearity=non_linearity, + ) + self.up_blocks.append(up_block) + + # Update scale for next iteration + if upsample_mode is not None: + scale *= 2.0 + + # output blocks + self.norm_out = QwenImageRMS_norm(out_dim, images=False) + self.conv_out = QwenImageCausalConv3d(out_dim, 3, 3, padding=1) + + self.gradient_checkpointing = False + + def forward(self, x, feat_cache=None, feat_idx=[0]): + ## conv1 + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), + cache_x, + ], + dim=2, + ) + x = self.conv_in(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv_in(x) + + ## middle + x = self.mid_block(x, feat_cache, feat_idx) + + ## upsamples + for up_block in self.up_blocks: + x = up_block(x, feat_cache, feat_idx) + + ## head + x = self.norm_out(x) + x = self.nonlinearity(x) + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), + cache_x, + ], + dim=2, + ) + x = self.conv_out(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv_out(x) + return x + + +class AutoencoderKLQwenImage(nn.Module): + r""" + A VAE model with KL loss for encoding videos into latents and decoding latent representations into videos. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + """ + + _supports_gradient_checkpointing = False + + # fmt: off + def __init__( + self, + config: QwenImageVAEConfig, + ) -> None: + # fmt: on + super().__init__() + base_dim = config.arch_config.base_dim + z_dim = config.arch_config.z_dim + dim_mult = config.arch_config.dim_mult + num_res_blocks = config.arch_config.num_res_blocks + attn_scales = config.arch_config.attn_scales + temperal_downsample = config.arch_config.temperal_downsample + dropout = config.arch_config.dropout + # non_linearity = config.arch_config.non_linearity + self.z_dim = z_dim + self.temperal_downsample = temperal_downsample + self.temperal_upsample = temperal_downsample[::-1] + + self.encoder = QwenImageEncoder3d( + base_dim, z_dim * 2, dim_mult, num_res_blocks, attn_scales, self.temperal_downsample, dropout + ) + self.quant_conv = QwenImageCausalConv3d(z_dim * 2, z_dim * 2, 1) + self.post_quant_conv = QwenImageCausalConv3d(z_dim, z_dim, 1) + + self.decoder = QwenImageDecoder3d( + base_dim, z_dim, dim_mult, num_res_blocks, attn_scales, self.temperal_upsample, dropout + ) + + self.spatial_compression_ratio = 2 ** len(self.temperal_downsample) + + # When decoding a batch of video latents at a time, one can save memory by slicing across the batch dimension + # to perform decoding of a single video latent at a time. + self.use_slicing = False + + # When decoding spatially large video latents, the memory requirement is very high. By breaking the video latent + # frames spatially into smaller tiles and performing multiple forward passes for decoding, and then blending the + # intermediate tiles together, the memory requirement can be lowered. + self.use_tiling = False + + # The minimal tile height and width for spatial tiling to be used + self.tile_sample_min_height = 256 + self.tile_sample_min_width = 256 + + # The minimal distance between two spatial tiles + self.tile_sample_stride_height = 192 + self.tile_sample_stride_width = 192 + + # Precompute and cache conv counts for encoder and decoder for clear_cache speedup + self._cached_conv_counts = { + "decoder": sum(isinstance(m, QwenImageCausalConv3d) for m in self.decoder.modules()) + if self.decoder is not None + else 0, + "encoder": sum(isinstance(m, QwenImageCausalConv3d) for m in self.encoder.modules()) + if self.encoder is not None + else 0, + } + cuda_device = get_local_torch_device() + # FIXME: hardcode + dtype = torch.bfloat16 + latent_channels = config.arch_config.z_dim + + self.shift_factor = ( + torch.tensor( + config.arch_config.latents_mean + ) + .view(1, latent_channels, 1, 1, 1) + .to(cuda_device, dtype) + ) + latents_std_tensor = torch.tensor(config.arch_config.latents_std, dtype=dtype, device=cuda_device) + self.scaling_factor = (1.0 / latents_std_tensor).view(1, latent_channels, 1, 1, 1) + + def enable_tiling( + self, + tile_sample_min_height: Optional[int] = None, + tile_sample_min_width: Optional[int] = None, + tile_sample_stride_height: Optional[float] = None, + tile_sample_stride_width: Optional[float] = None, + ) -> None: + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + + Args: + tile_sample_min_height (`int`, *optional*): + The minimum height required for a sample to be separated into tiles across the height dimension. + tile_sample_min_width (`int`, *optional*): + The minimum width required for a sample to be separated into tiles across the width dimension. + tile_sample_stride_height (`int`, *optional*): + The minimum amount of overlap between two consecutive vertical tiles. This is to ensure that there are + no tiling artifacts produced across the height dimension. + tile_sample_stride_width (`int`, *optional*): + The stride between two consecutive horizontal tiles. This is to ensure that there are no tiling + artifacts produced across the width dimension. + """ + self.use_tiling = True + self.tile_sample_min_height = tile_sample_min_height or self.tile_sample_min_height + self.tile_sample_min_width = tile_sample_min_width or self.tile_sample_min_width + self.tile_sample_stride_height = tile_sample_stride_height or self.tile_sample_stride_height + self.tile_sample_stride_width = tile_sample_stride_width or self.tile_sample_stride_width + + def disable_tiling(self) -> None: + r""" + Disable tiled VAE decoding. If `enable_tiling` was previously enabled, this method will go back to computing + decoding in one step. + """ + self.use_tiling = False + + def enable_slicing(self) -> None: + r""" + Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to + compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. + """ + self.use_slicing = True + + def disable_slicing(self) -> None: + r""" + Disable sliced VAE decoding. If `enable_slicing` was previously enabled, this method will go back to computing + decoding in one step. + """ + self.use_slicing = False + + def clear_cache(self): + def _count_conv3d(model): + count = 0 + for m in model.modules(): + if isinstance(m, QwenImageCausalConv3d): + count += 1 + return count + + self._conv_num = _count_conv3d(self.decoder) + self._conv_idx = [0] + self._feat_map = [None] * self._conv_num + # cache encode + self._enc_conv_num = _count_conv3d(self.encoder) + self._enc_conv_idx = [0] + self._enc_feat_map = [None] * self._enc_conv_num + + def _encode(self, x: torch.Tensor): + _, _, num_frame, height, width = x.shape + + if self.use_tiling and (width > self.tile_sample_min_width or height > self.tile_sample_min_height): + return self.tiled_encode(x) + + self.clear_cache() + iter_ = 1 + (num_frame - 1) // 4 + for i in range(iter_): + self._enc_conv_idx = [0] + if i == 0: + out = self.encoder(x[:, :, :1, :, :], feat_cache=self._enc_feat_map, feat_idx=self._enc_conv_idx) + else: + out_ = self.encoder( + x[:, :, 1 + 4 * (i - 1): 1 + 4 * i, :, :], + feat_cache=self._enc_feat_map, + feat_idx=self._enc_conv_idx, + ) + out = torch.cat([out, out_], 2) + + enc = self.quant_conv(out) + self.clear_cache() + return enc + + def encode( + self, x: torch.Tensor, return_dict: bool = True + ) -> DiagonalGaussianDistribution: + r""" + Encode a batch of images into latents. + + Args: + x (`torch.Tensor`): Input batch of images. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.autoencoder_kl.AutoencoderKLOutput`] instead of a plain tuple. + + Returns: + The latent representations of the encoded videos. If `return_dict` is True, a + [`~models.autoencoder_kl.AutoencoderKLOutput`] is returned, otherwise a plain `tuple` is returned. + """ + if self.use_slicing and x.shape[0] > 1: + encoded_slices = [self._encode(x_slice) for x_slice in x.split(1)] + h = torch.cat(encoded_slices) + else: + h = self._encode(x) + posterior = DiagonalGaussianDistribution(h) + + return posterior + + def _decode(self, z: torch.Tensor, return_dict: bool = True): + _, _, num_frame, height, width = z.shape + tile_latent_min_height = self.tile_sample_min_height // self.spatial_compression_ratio + tile_latent_min_width = self.tile_sample_min_width // self.spatial_compression_ratio + + if self.use_tiling and (width > tile_latent_min_width or height > tile_latent_min_height): + return self.tiled_decode(z, return_dict=return_dict) + + self.clear_cache() + x = self.post_quant_conv(z) + for i in range(num_frame): + self._conv_idx = [0] + if i == 0: + out = self.decoder(x[:, :, i: i + 1, :, :], feat_cache=self._feat_map, feat_idx=self._conv_idx) + else: + out_ = self.decoder(x[:, :, i: i + 1, :, :], feat_cache=self._feat_map, feat_idx=self._conv_idx) + out = torch.cat([out, out_], 2) + + out = torch.clamp(out, min=-1.0, max=1.0) + self.clear_cache() + if not return_dict: + return (out,) + + return DecoderOutput(sample=out) + + def decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + r""" + Decode a batch of images. + + Args: + z (`torch.Tensor`): Input batch of latent vectors. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.vae.DecoderOutput`] instead of a plain tuple. + + Returns: + [`~models.vae.DecoderOutput`] or `tuple`: + If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is + returned. + """ + if self.use_slicing and z.shape[0] > 1: + decoded_slices = [self._decode(z_slice).sample for z_slice in z.split(1)] + decoded = torch.cat(decoded_slices) + else: + decoded = self._decode(z).sample + + return decoded + + def blend_v(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor: + blend_extent = min(a.shape[-2], b.shape[-2], blend_extent) + for y in range(blend_extent): + b[:, :, :, y, :] = a[:, :, :, -blend_extent + y, :] * (1 - y / blend_extent) + b[:, :, :, y, :] * ( + y / blend_extent + ) + return b + + def blend_h(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor: + blend_extent = min(a.shape[-1], b.shape[-1], blend_extent) + for x in range(blend_extent): + b[:, :, :, :, x] = a[:, :, :, :, -blend_extent + x] * (1 - x / blend_extent) + b[:, :, :, :, x] * ( + x / blend_extent + ) + return b + + def tiled_encode(self, x: torch.Tensor) -> AutoencoderKLOutput: + r"""Encode a batch of images using a tiled encoder. + + Args: + x (`torch.Tensor`): Input batch of videos. + + Returns: + `torch.Tensor`: + The latent representation of the encoded videos. + """ + _, _, num_frames, height, width = x.shape + latent_height = height // self.spatial_compression_ratio + latent_width = width // self.spatial_compression_ratio + + tile_latent_min_height = self.tile_sample_min_height // self.spatial_compression_ratio + tile_latent_min_width = self.tile_sample_min_width // self.spatial_compression_ratio + tile_latent_stride_height = self.tile_sample_stride_height // self.spatial_compression_ratio + tile_latent_stride_width = self.tile_sample_stride_width // self.spatial_compression_ratio + + blend_height = tile_latent_min_height - tile_latent_stride_height + blend_width = tile_latent_min_width - tile_latent_stride_width + + # Split x into overlapping tiles and encode them separately. + # The tiles have an overlap to avoid seams between tiles. + rows = [] + for i in range(0, height, self.tile_sample_stride_height): + row = [] + for j in range(0, width, self.tile_sample_stride_width): + self.clear_cache() + time = [] + frame_range = 1 + (num_frames - 1) // 4 + for k in range(frame_range): + self._enc_conv_idx = [0] + if k == 0: + tile = x[:, :, :1, i: i + self.tile_sample_min_height, j: j + self.tile_sample_min_width] + else: + tile = x[ + :, + :, + 1 + 4 * (k - 1): 1 + 4 * k, + i: i + self.tile_sample_min_height, + j: j + self.tile_sample_min_width, + ] + tile = self.encoder(tile, feat_cache=self._enc_feat_map, feat_idx=self._enc_conv_idx) + tile = self.quant_conv(tile) + time.append(tile) + row.append(torch.cat(time, dim=2)) + rows.append(row) + self.clear_cache() + + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_height) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_width) + result_row.append(tile[:, :, :, :tile_latent_stride_height, :tile_latent_stride_width]) + result_rows.append(torch.cat(result_row, dim=-1)) + + enc = torch.cat(result_rows, dim=3)[:, :, :, :latent_height, :latent_width] + return enc + + def tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + r""" + Decode a batch of images using a tiled decoder. + + Args: + z (`torch.Tensor`): Input batch of latent vectors. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.vae.DecoderOutput`] instead of a plain tuple. + + Returns: + [`~models.vae.DecoderOutput`] or `tuple`: + If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is + returned. + """ + _, _, num_frames, height, width = z.shape + sample_height = height * self.spatial_compression_ratio + sample_width = width * self.spatial_compression_ratio + + tile_latent_min_height = self.tile_sample_min_height // self.spatial_compression_ratio + tile_latent_min_width = self.tile_sample_min_width // self.spatial_compression_ratio + tile_latent_stride_height = self.tile_sample_stride_height // self.spatial_compression_ratio + tile_latent_stride_width = self.tile_sample_stride_width // self.spatial_compression_ratio + + blend_height = self.tile_sample_min_height - self.tile_sample_stride_height + blend_width = self.tile_sample_min_width - self.tile_sample_stride_width + + # Split z into overlapping tiles and decode them separately. + # The tiles have an overlap to avoid seams between tiles. + rows = [] + for i in range(0, height, tile_latent_stride_height): + row = [] + for j in range(0, width, tile_latent_stride_width): + self.clear_cache() + time = [] + for k in range(num_frames): + self._conv_idx = [0] + tile = z[:, :, k: k + 1, i: i + tile_latent_min_height, j: j + tile_latent_min_width] + tile = self.post_quant_conv(tile) + decoded = self.decoder(tile, feat_cache=self._feat_map, feat_idx=self._conv_idx) + time.append(decoded) + row.append(torch.cat(time, dim=2)) + rows.append(row) + self.clear_cache() + + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_height) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_width) + result_row.append(tile[:, :, :, : self.tile_sample_stride_height, : self.tile_sample_stride_width]) + result_rows.append(torch.cat(result_row, dim=-1)) + + dec = torch.cat(result_rows, dim=3)[:, :, :, :sample_height, :sample_width] + + if not return_dict: + return (dec,) + return DecoderOutput(sample=dec) + + def forward( + self, + sample: torch.Tensor, + sample_posterior: bool = False, + return_dict: bool = True, + generator: Optional[torch.Generator] = None, + ) -> Union[DecoderOutput, torch.Tensor]: + """ + Args: + sample (`torch.Tensor`): Input sample. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`DecoderOutput`] instead of a plain tuple. + """ + x = sample + posterior = self.encode(x).latent_dist + if sample_posterior: + z = posterior.sample(generator=generator) + else: + z = posterior.mode() + dec = self.decode(z, return_dict=return_dict) + return dec + + +EntryClass = AutoencoderKLQwenImage diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/common.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/common.py new file mode 100644 index 00000000..58dad250 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/common.py @@ -0,0 +1,648 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +from abc import ABC, abstractmethod +from collections.abc import Iterator +from math import prod +from typing import Optional, cast + +import numpy as np +import torch +import torch.distributed as dist +from diffusers.models.autoencoders.vae import DiagonalGaussianDistribution +from diffusers.utils.torch_utils import randn_tensor + +from sglang.multimodal_gen.configs.models import VAEConfig +from sglang.multimodal_gen.runtime.distributed import ( + get_sp_parallel_rank, + get_sp_world_size, +) + + +class ParallelTiledVAE(ABC): + tile_sample_min_height: int + tile_sample_min_width: int + tile_sample_min_num_frames: int + tile_sample_stride_height: int + tile_sample_stride_width: int + tile_sample_stride_num_frames: int + blend_num_frames: int + use_tiling: bool + use_temporal_tiling: bool + use_parallel_tiling: bool + + def __init__(self, config: VAEConfig, **kwargs) -> None: + self.config = config + self.tile_sample_min_height = config.tile_sample_min_height + self.tile_sample_min_width = config.tile_sample_min_width + self.tile_sample_min_num_frames = config.tile_sample_min_num_frames + self.tile_sample_stride_height = config.tile_sample_stride_height + self.tile_sample_stride_width = config.tile_sample_stride_width + self.tile_sample_stride_num_frames = config.tile_sample_stride_num_frames + self.blend_num_frames = config.blend_num_frames + self.use_tiling = config.use_tiling + self.use_temporal_tiling = config.use_temporal_tiling + self.use_parallel_tiling = config.use_parallel_tiling + + def to(self, device) -> "ParallelTiledVAE": + # TODO: implement this + return self + + @property + def device(self): + return next(self.parameters()).device + + @property + def temporal_compression_ratio(self) -> int: + return cast(int, self.config.temporal_compression_ratio) + + @property + def spatial_compression_ratio(self) -> int: + return cast(int, self.config.spatial_compression_ratio) + + @property + def scaling_factor(self) -> float | torch.Tensor: + return cast(float | torch.Tensor, self.config.scaling_factor) + + @abstractmethod + def _encode(self, *args, **kwargs) -> torch.Tensor: + pass + + @abstractmethod + def _decode(self, *args, **kwargs) -> torch.Tensor: + pass + + def encode(self, x: torch.Tensor) -> DiagonalGaussianDistribution: + batch_size, num_channels, num_frames, height, width = x.shape + latent_num_frames = (num_frames - 1) // self.temporal_compression_ratio + 1 + + if ( + self.use_tiling + and self.use_temporal_tiling + and num_frames > self.tile_sample_min_num_frames + ): + latents = self.tiled_encode(x)[:, :, :latent_num_frames] + elif self.use_tiling and ( + width > self.tile_sample_min_width or height > self.tile_sample_min_height + ): + latents = self.spatial_tiled_encode(x)[:, :, :latent_num_frames] + else: + latents = self._encode(x)[:, :, :latent_num_frames] + return DiagonalGaussianDistribution(latents) + + def decode(self, z: torch.Tensor) -> torch.Tensor: + batch_size, num_channels, num_frames, height, width = z.shape + tile_latent_min_height = ( + self.tile_sample_min_height // self.spatial_compression_ratio + ) + tile_latent_min_width = ( + self.tile_sample_stride_width // self.spatial_compression_ratio + ) + tile_latent_min_num_frames = ( + self.tile_sample_min_num_frames // self.temporal_compression_ratio + ) + num_sample_frames = (num_frames - 1) * self.temporal_compression_ratio + 1 + + if self.use_tiling and self.use_parallel_tiling and get_sp_world_size() > 1: + return self.parallel_tiled_decode(z)[:, :, :num_sample_frames] + if ( + self.use_tiling + and self.use_temporal_tiling + and num_frames > tile_latent_min_num_frames + ): + return self.tiled_decode(z)[:, :, :num_sample_frames] + + if self.use_tiling and ( + width > tile_latent_min_width or height > tile_latent_min_height + ): + return self.spatial_tiled_decode(z)[:, :, :num_sample_frames] + + return self._decode(z)[:, :, :num_sample_frames] + + def blend_v( + self, a: torch.Tensor, b: torch.Tensor, blend_extent: int + ) -> torch.Tensor: + blend_extent = min(a.shape[-2], b.shape[-2], blend_extent) + for y in range(blend_extent): + b[:, :, :, y, :] = a[:, :, :, -blend_extent + y, :] * ( + 1 - y / blend_extent + ) + b[:, :, :, y, :] * (y / blend_extent) + return b + + def blend_h( + self, a: torch.Tensor, b: torch.Tensor, blend_extent: int + ) -> torch.Tensor: + blend_extent = min(a.shape[-1], b.shape[-1], blend_extent) + for x in range(blend_extent): + b[:, :, :, :, x] = a[:, :, :, :, -blend_extent + x] * ( + 1 - x / blend_extent + ) + b[:, :, :, :, x] * (x / blend_extent) + return b + + def blend_t( + self, a: torch.Tensor, b: torch.Tensor, blend_extent: int + ) -> torch.Tensor: + blend_extent = min(a.shape[-3], b.shape[-3], blend_extent) + for x in range(blend_extent): + b[:, :, x, :, :] = a[:, :, -blend_extent + x, :, :] * ( + 1 - x / blend_extent + ) + b[:, :, x, :, :] * (x / blend_extent) + return b + + def spatial_tiled_encode(self, x: torch.Tensor) -> torch.Tensor: + r"""Encode a batch of images using a tiled encoder. + + Args: + x (`torch.Tensor`): Input batch of videos. + + Returns: + `torch.Tensor`: + The latent representation of the encoded videos. + """ + _, _, _, height, width = x.shape + # latent_height = height // self.spatial_compression_ratio + # latent_width = width // self.spatial_compression_ratio + + tile_latent_min_height = ( + self.tile_sample_min_height // self.spatial_compression_ratio + ) + tile_latent_min_width = ( + self.tile_sample_min_width // self.spatial_compression_ratio + ) + tile_latent_stride_height = ( + self.tile_sample_stride_height // self.spatial_compression_ratio + ) + tile_latent_stride_width = ( + self.tile_sample_stride_width // self.spatial_compression_ratio + ) + + blend_height = tile_latent_min_height - tile_latent_stride_height + blend_width = tile_latent_min_width - tile_latent_stride_width + + # Split x into overlapping tiles and encode them separately. + # The tiles have an overlap to avoid seams between tiles. + rows = [] + for i in range(0, height, self.tile_sample_stride_height): + row = [] + for j in range(0, width, self.tile_sample_stride_width): + tile = x[ + :, + :, + :, + i : i + self.tile_sample_min_height, + j : j + self.tile_sample_min_width, + ] + tile = self._encode(tile) + row.append(tile) + rows.append(row) + + return self._merge_spatial_tiles( + rows, + blend_height, + blend_width, + tile_latent_stride_height, + tile_latent_stride_width, + ) + + def _parallel_data_generator( + self, gathered_results, gathered_dim_metadata + ) -> Iterator[tuple[torch.Tensor, int]]: + global_idx = 0 + for i, per_rank_metadata in enumerate(gathered_dim_metadata): + _start_shape = 0 + for shape in per_rank_metadata: + mul_shape = prod(shape) + yield ( + gathered_results[ + i, _start_shape : _start_shape + mul_shape + ].reshape(shape), + global_idx, + ) + _start_shape += mul_shape + global_idx += 1 + + def parallel_tiled_decode(self, z: torch.FloatTensor) -> torch.FloatTensor: + """ + Parallel version of tiled_decode that distributes both temporal and spatial computation across GPUs + """ + world_size, rank = get_sp_world_size(), get_sp_parallel_rank() + B, C, T, H, W = z.shape + + # Calculate parameters + tile_latent_min_height = ( + self.tile_sample_min_height // self.spatial_compression_ratio + ) + tile_latent_min_width = ( + self.tile_sample_min_width // self.spatial_compression_ratio + ) + tile_latent_min_num_frames = ( + self.tile_sample_min_num_frames // self.temporal_compression_ratio + ) + tile_latent_stride_height = ( + self.tile_sample_stride_height // self.spatial_compression_ratio + ) + tile_latent_stride_width = ( + self.tile_sample_stride_width // self.spatial_compression_ratio + ) + tile_latent_stride_num_frames = ( + self.tile_sample_stride_num_frames // self.temporal_compression_ratio + ) + + blend_height = self.tile_sample_min_height - self.tile_sample_stride_height + blend_width = self.tile_sample_min_width - self.tile_sample_stride_width + + # Calculate tile dimensions + num_t_tiles = ( + T + tile_latent_stride_num_frames - 1 + ) // tile_latent_stride_num_frames + num_h_tiles = (H + tile_latent_stride_height - 1) // tile_latent_stride_height + num_w_tiles = (W + tile_latent_stride_width - 1) // tile_latent_stride_width + total_spatial_tiles = num_h_tiles * num_w_tiles + total_tiles = num_t_tiles * total_spatial_tiles + + # Calculate tiles per rank and padding + tiles_per_rank = (total_tiles + world_size - 1) // world_size + start_tile_idx = rank * tiles_per_rank + end_tile_idx = min((rank + 1) * tiles_per_rank, total_tiles) + + local_results = [] + local_dim_metadata = [] + # Process assigned tiles + for local_idx, global_idx in enumerate(range(start_tile_idx, end_tile_idx)): + t_idx = global_idx // total_spatial_tiles + spatial_idx = global_idx % total_spatial_tiles + h_idx = spatial_idx // num_w_tiles + w_idx = spatial_idx % num_w_tiles + + # Calculate positions + t_start = t_idx * tile_latent_stride_num_frames + h_start = h_idx * tile_latent_stride_height + w_start = w_idx * tile_latent_stride_width + + # Extract and process tile + tile = z[ + :, + :, + t_start : t_start + tile_latent_min_num_frames + 1, + h_start : h_start + tile_latent_min_height, + w_start : w_start + tile_latent_min_width, + ] + + # Process tile + tile = self._decode(tile) + + if t_start > 0: + tile = tile[:, :, 1:, :, :] + + # Store metadata + shape = tile.shape + # Store decoded data (flattened) + decoded_flat = tile.reshape(-1) + local_results.append(decoded_flat) + local_dim_metadata.append(shape) + + results = torch.cat(local_results, dim=0).contiguous() + del local_results + # first gather size to pad the results + local_size = torch.tensor( + [results.size(0)], device=results.device, dtype=torch.int64 + ) + all_sizes = [ + torch.zeros(1, device=results.device, dtype=torch.int64) + for _ in range(world_size) + ] + dist.all_gather(all_sizes, local_size) + max_size = max(size.item() for size in all_sizes) + padded_results = torch.zeros(max_size, device=results.device) + padded_results[: results.size(0)] = results + del results + + # Gather all results + gathered_dim_metadata = [None] * world_size + gathered_results = ( + torch.zeros_like(padded_results) + .repeat(world_size, *[1] * len(padded_results.shape)) + .contiguous() + ) # use contiguous to make sure it won't copy data in the following operations + # TODO (PY): use sgl_diffusion distributed methods + dist.all_gather_into_tensor(gathered_results, padded_results) + dist.all_gather_object(gathered_dim_metadata, local_dim_metadata) + # Process gathered results + data: list = [ + [[[] for _ in range(num_w_tiles)] for _ in range(num_h_tiles)] + for _ in range(num_t_tiles) + ] + for current_data, global_idx in self._parallel_data_generator( + gathered_results, gathered_dim_metadata + ): + t_idx = global_idx // total_spatial_tiles + spatial_idx = global_idx % total_spatial_tiles + h_idx = spatial_idx // num_w_tiles + w_idx = spatial_idx % num_w_tiles + data[t_idx][h_idx][w_idx] = current_data + # Merge results + result_slices = [] + last_slice_data = None + for i, tem_data in enumerate(data): + slice_data = self._merge_spatial_tiles( + tem_data, + blend_height, + blend_width, + self.tile_sample_stride_height, + self.tile_sample_stride_width, + ) + if i > 0: + slice_data = self.blend_t( + last_slice_data, slice_data, self.blend_num_frames + ) + result_slices.append( + slice_data[:, :, : self.tile_sample_stride_num_frames, :, :] + ) + else: + result_slices.append( + slice_data[:, :, : self.tile_sample_stride_num_frames + 1, :, :] + ) + last_slice_data = slice_data + dec = torch.cat(result_slices, dim=2) + + return dec + + def _merge_spatial_tiles( + self, tiles, blend_height, blend_width, stride_height, stride_width + ) -> torch.Tensor: + """Helper function to merge spatial tiles with blending""" + result_rows = [] + for i, row in enumerate(tiles): + result_row = [] + for j, tile in enumerate(row): + if i > 0: + tile = self.blend_v(tiles[i - 1][j], tile, blend_height) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_width) + result_row.append(tile[:, :, :, :stride_height, :stride_width]) + result_rows.append(torch.cat(result_row, dim=-1)) + return torch.cat(result_rows, dim=-2) + + def spatial_tiled_decode(self, z: torch.Tensor) -> torch.Tensor: + r""" + Decode a batch of images using a tiled decoder. + + Args: + z (`torch.Tensor`): Input batch of latent vectors. + + Returns: + `torch.Tensor`: + The decoded images. + """ + + _, _, _, height, width = z.shape + # sample_height = height * self.spatial_compression_ratio + # sample_width = width * self.spatial_compression_ratio + + tile_latent_min_height = ( + self.tile_sample_min_height // self.spatial_compression_ratio + ) + tile_latent_min_width = ( + self.tile_sample_min_width // self.spatial_compression_ratio + ) + tile_latent_stride_height = ( + self.tile_sample_stride_height // self.spatial_compression_ratio + ) + tile_latent_stride_width = ( + self.tile_sample_stride_width // self.spatial_compression_ratio + ) + + blend_height = self.tile_sample_min_height - self.tile_sample_stride_height + blend_width = self.tile_sample_min_width - self.tile_sample_stride_width + + # Split z into overlapping tiles and decode them separately. + # The tiles have an overlap to avoid seams between tiles. + rows = [] + for i in range(0, height, tile_latent_stride_height): + row = [] + for j in range(0, width, tile_latent_stride_width): + tile = z[ + :, + :, + :, + i : i + tile_latent_min_height, + j : j + tile_latent_min_width, + ] + decoded = self._decode(tile) + row.append(decoded) + rows.append(row) + return self._merge_spatial_tiles( + rows, + blend_height, + blend_width, + self.tile_sample_stride_height, + self.tile_sample_stride_width, + ) + + def tiled_encode(self, x: torch.Tensor) -> torch.Tensor: + _, _, num_frames, height, width = x.shape + + # tile_latent_min_num_frames = self.tile_sample_min_num_frames // self.temporal_compression_ratio + tile_latent_stride_num_frames = ( + self.tile_sample_stride_num_frames // self.temporal_compression_ratio + ) + + row = [] + for i in range(0, num_frames, self.tile_sample_stride_num_frames): + tile = x[:, :, i : i + self.tile_sample_min_num_frames + 1, :, :] + if self.use_tiling and ( + height > self.tile_sample_min_height + or width > self.tile_sample_min_width + ): + tile = self.spatial_tiled_encode(tile) + else: + tile = self._encode(tile) + if i > 0: + tile = tile[:, :, 1:, :, :] + row.append(tile) + result_row = [] + for i, tile in enumerate(row): + if i > 0: + tile = self.blend_t(row[i - 1], tile, self.blend_num_frames) + result_row.append(tile[:, :, :tile_latent_stride_num_frames, :, :]) + else: + result_row.append(tile[:, :, : tile_latent_stride_num_frames + 1, :, :]) + enc = torch.cat(result_row, dim=2) + return enc + + def tiled_decode(self, z: torch.Tensor) -> torch.Tensor: + batch_size, num_channels, num_frames, height, width = z.shape + + tile_latent_min_height = ( + self.tile_sample_min_height // self.spatial_compression_ratio + ) + tile_latent_min_width = ( + self.tile_sample_min_width // self.spatial_compression_ratio + ) + tile_latent_min_num_frames = ( + self.tile_sample_min_num_frames // self.temporal_compression_ratio + ) + tile_latent_stride_num_frames = ( + self.tile_sample_stride_num_frames // self.temporal_compression_ratio + ) + + row = [] + for i in range(0, num_frames, tile_latent_stride_num_frames): + tile = z[:, :, i : i + tile_latent_min_num_frames + 1, :, :] + if self.use_tiling and ( + tile.shape[-1] > tile_latent_min_width + or tile.shape[-2] > tile_latent_min_height + ): + decoded = self.spatial_tiled_decode(tile) + else: + decoded = self._decode(tile) + if i > 0: + decoded = decoded[:, :, 1:, :, :] + row.append(decoded) + result_row = [] + for i, tile in enumerate(row): + if i > 0: + tile = self.blend_t(row[i - 1], tile, self.blend_num_frames) + result_row.append( + tile[:, :, : self.tile_sample_stride_num_frames, :, :] + ) + else: + result_row.append( + tile[:, :, : self.tile_sample_stride_num_frames + 1, :, :] + ) + + dec = torch.cat(result_row, dim=2) + return dec + + def enable_tiling( + self, + tile_sample_min_height: int | None = None, + tile_sample_min_width: int | None = None, + tile_sample_min_num_frames: int | None = None, + tile_sample_stride_height: int | None = None, + tile_sample_stride_width: int | None = None, + tile_sample_stride_num_frames: int | None = None, + blend_num_frames: int | None = None, + use_tiling: bool | None = None, + use_temporal_tiling: bool | None = None, + use_parallel_tiling: bool | None = None, + ) -> None: + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + + Args: + tile_sample_min_height (`int`, *optional*): + The minimum height required for a sample to be separated into tiles across the height dimension. + tile_sample_min_width (`int`, *optional*): + The minimum width required for a sample to be separated into tiles across the width dimension. + tile_sample_min_num_frames (`int`, *optional*): + The minimum number of frames required for a sample to be separated into tiles across the frame + dimension. + tile_sample_stride_height (`int`, *optional*): + The minimum amount of overlap between two consecutive vertical tiles. This is to ensure that there are + no tiling artifacts produced across the height dimension. + tile_sample_stride_width (`int`, *optional*): + The stride between two consecutive horizontal tiles. This is to ensure that there are no tiling + artifacts produced across the width dimension. + tile_sample_stride_num_frames (`int`, *optional*): + The stride between two consecutive frame tiles. This is to ensure that there are no tiling artifacts + produced across the frame dimension. + """ + self.use_tiling = True + self.tile_sample_min_height = ( + tile_sample_min_height or self.tile_sample_min_height + ) + self.tile_sample_min_width = tile_sample_min_width or self.tile_sample_min_width + self.tile_sample_min_num_frames = ( + tile_sample_min_num_frames or self.tile_sample_min_num_frames + ) + self.tile_sample_stride_height = ( + tile_sample_stride_height or self.tile_sample_stride_height + ) + self.tile_sample_stride_width = ( + tile_sample_stride_width or self.tile_sample_stride_width + ) + self.tile_sample_stride_num_frames = ( + tile_sample_stride_num_frames or self.tile_sample_stride_num_frames + ) + if blend_num_frames is not None: + self.blend_num_frames = blend_num_frames + else: + self.blend_num_frames = ( + self.tile_sample_min_num_frames - self.tile_sample_stride_num_frames + ) + self.use_tiling = use_tiling or self.use_tiling + self.use_temporal_tiling = use_temporal_tiling or self.use_temporal_tiling + self.use_parallel_tiling = use_parallel_tiling or self.use_parallel_tiling + + def disable_tiling(self) -> None: + r""" + Disable tiled VAE decoding. If `enable_tiling` was previously enabled, this method will go back to computing + decoding in one step. + """ + self.use_tiling = False + + +# adapted from https://github.com/huggingface/diffusers/blob/e7ffeae0a191f710881d1fbde00cd6ff025e81f2/src/diffusers/models/autoencoders/vae.py#L691 +class DiagonalGaussianDistribution: + + def __init__(self, parameters: torch.Tensor, deterministic: bool = False): + self.parameters = parameters + self.mean, self.logvar = torch.chunk(parameters, 2, dim=1) + self.logvar = torch.clamp(self.logvar, -30.0, 20.0) + self.deterministic = deterministic + self.std = torch.exp(0.5 * self.logvar) + self.var = torch.exp(self.logvar) + if self.deterministic: + self.var = self.std = torch.zeros_like( + self.mean, device=self.parameters.device, dtype=self.parameters.dtype + ) + + def sample(self, generator: torch.Generator | None = None) -> torch.Tensor: + # make sure sample is on the same device as the parameters and has same dtype + sample = randn_tensor( + self.mean.shape, + generator=generator, + device=self.parameters.device, + dtype=self.parameters.dtype, + ) + x = self.mean + self.std * sample + return x + + def kl( + self, other: Optional["DiagonalGaussianDistribution"] = None + ) -> torch.Tensor: + if self.deterministic: + return torch.Tensor([0.0]) + else: + if other is None: + return 0.5 * torch.sum( + torch.pow(self.mean, 2) + self.var - 1.0 - self.logvar, + dim=[1, 2, 3], + ) + else: + return 0.5 * torch.sum( + torch.pow(self.mean - other.mean, 2) / other.var + + self.var / other.var + - 1.0 + - self.logvar + + other.logvar, + dim=[1, 2, 3], + ) + + def nll( + self, sample: torch.Tensor, dims: tuple[int, ...] = (1, 2, 3) + ) -> torch.Tensor: + if self.deterministic: + return torch.Tensor([0.0]) + logtwopi = np.log(2.0 * np.pi) + return 0.5 * torch.sum( + logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var, + dim=dims, + ) + + def mode(self) -> torch.Tensor: + return self.mean diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/hunyuanvae.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/hunyuanvae.py new file mode 100644 index 00000000..a9de61a5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/hunyuanvae.py @@ -0,0 +1,852 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from diffusers + +# Copyright 2024 The Hunyuan Team, The HuggingFace Team and The sglang-diffusion Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from sglang.multimodal_gen.configs.models.vaes import HunyuanVAEConfig +from sglang.multimodal_gen.runtime.layers.activation import get_act_fn +from sglang.multimodal_gen.runtime.models.vaes.common import ParallelTiledVAE + + +def prepare_causal_attention_mask( + num_frames: int, + height_width: int, + dtype: torch.dtype, + device: torch.device, + batch_size: int | None = None, +) -> torch.Tensor: + indices = torch.arange(1, num_frames + 1, dtype=torch.int32, device=device) + indices_blocks = indices.repeat_interleave(height_width) + x, y = torch.meshgrid(indices_blocks, indices_blocks, indexing="xy") + mask = torch.where(x <= y, 0, -float("inf")).to(dtype=dtype) + + if batch_size is not None: + mask = mask.unsqueeze(0).expand(batch_size, -1, -1) + return mask + + +class HunyuanVAEAttention(nn.Module): + + def __init__( + self, in_channels, heads, dim_head, eps, norm_num_groups, bias + ) -> None: + super().__init__() + self.in_channels = in_channels + self.heads = heads + self.dim_head = dim_head + self.eps = eps + self.norm_num_groups = norm_num_groups + self.bias = bias + + inner_dim = heads * dim_head + + # Define the projection layers + self.to_q = nn.Linear(in_channels, inner_dim, bias=bias) + self.to_k = nn.Linear(in_channels, inner_dim, bias=bias) + self.to_v = nn.Linear(in_channels, inner_dim, bias=bias) + self.to_out = nn.Sequential(nn.Linear(inner_dim, in_channels, bias=bias)) + + # Optional normalization layers + self.group_norm = nn.GroupNorm( + norm_num_groups, in_channels, eps=eps, affine=True + ) + + def forward( + self, hidden_states: torch.Tensor, attention_mask: torch.Tensor | None = None + ) -> torch.Tensor: + residual = hidden_states + + batch_size, sequence_length, _ = hidden_states.shape + + hidden_states = self.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2) + + # Project to query, key, value + query = self.to_q(hidden_states) + key = self.to_k(hidden_states) + value = self.to_v(hidden_states) + + # Reshape for multi-head attention + head_dim = self.dim_head + + query = query.view(batch_size, -1, self.heads, head_dim).transpose(1, 2) + key = key.view(batch_size, -1, self.heads, head_dim).transpose(1, 2) + value = value.view(batch_size, -1, self.heads, head_dim).transpose(1, 2) + + # Perform scaled dot-product attention + hidden_states = F.scaled_dot_product_attention( + query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False + ) + + # Reshape back + hidden_states = hidden_states.transpose(1, 2).reshape( + batch_size, -1, self.heads * head_dim + ) + hidden_states = hidden_states.to(query.dtype) + + # Linear projection + hidden_states = self.to_out(hidden_states) + + # Residual connection and rescale + hidden_states = hidden_states + residual + + return hidden_states + + +class HunyuanVideoCausalConv3d(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int | tuple[int, int, int] = 3, + stride: int | tuple[int, int, int] = 1, + padding: int | tuple[int, int, int] = 0, + dilation: int | tuple[int, int, int] = 1, + bias: bool = True, + pad_mode: str = "replicate", + ) -> None: + super().__init__() + + kernel_size = ( + (kernel_size, kernel_size, kernel_size) + if isinstance(kernel_size, int) + else kernel_size + ) + + self.pad_mode = pad_mode + self.time_causal_padding = ( + kernel_size[0] // 2, + kernel_size[0] // 2, + kernel_size[1] // 2, + kernel_size[1] // 2, + kernel_size[2] - 1, + 0, + ) + + self.conv = nn.Conv3d( + in_channels, out_channels, kernel_size, stride, padding, dilation, bias=bias + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = F.pad( + hidden_states, self.time_causal_padding, mode=self.pad_mode + ) + return self.conv(hidden_states) + + +class HunyuanVideoUpsampleCausal3D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int | None = None, + kernel_size: int = 3, + stride: int = 1, + bias: bool = True, + upsample_factor: tuple[int, ...] = (2, 2, 2), + ) -> None: + super().__init__() + + out_channels = out_channels or in_channels + self.upsample_factor = upsample_factor + + self.conv = HunyuanVideoCausalConv3d( + in_channels, out_channels, kernel_size, stride, bias=bias + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + num_frames = hidden_states.size(2) + + first_frame, other_frames = hidden_states.split((1, num_frames - 1), dim=2) + first_frame = F.interpolate( + first_frame.squeeze(2), + scale_factor=self.upsample_factor[1:], + mode="nearest", + ).unsqueeze(2) + + if num_frames > 1: + # See: https://github.com/pytorch/pytorch/issues/81665 + # Unless you have a version of pytorch where non-contiguous implementation of F.interpolate + # is fixed, this will raise either a runtime error, or fail silently with bad outputs. + # If you are encountering an error here, make sure to try running encoding/decoding with + # `vae.enable_tiling()` first. If that doesn't work, open an issue at: + # https://github.com/huggingface/diffusers/issues + other_frames = other_frames.contiguous() + other_frames = F.interpolate( + other_frames, scale_factor=self.upsample_factor, mode="nearest" + ) + hidden_states = torch.cat((first_frame, other_frames), dim=2) + else: + hidden_states = first_frame + + hidden_states = self.conv(hidden_states) + return hidden_states + + +class HunyuanVideoDownsampleCausal3D(nn.Module): + + def __init__( + self, + channels: int, + out_channels: int | None = None, + padding: int = 1, + kernel_size: int = 3, + bias: bool = True, + stride=2, + ) -> None: + super().__init__() + out_channels = out_channels or channels + + self.conv = HunyuanVideoCausalConv3d( + channels, out_channels, kernel_size, stride, padding, bias=bias + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.conv(hidden_states) + return hidden_states + + +class HunyuanVideoResnetBlockCausal3D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int | None = None, + dropout: float = 0.0, + groups: int = 32, + eps: float = 1e-6, + non_linearity: str = "silu", + ) -> None: + super().__init__() + out_channels = out_channels or in_channels + + self.nonlinearity = get_act_fn(non_linearity) + + self.norm1 = nn.GroupNorm(groups, in_channels, eps=eps, affine=True) + self.conv1 = HunyuanVideoCausalConv3d(in_channels, out_channels, 3, 1, 0) + + self.norm2 = nn.GroupNorm(groups, out_channels, eps=eps, affine=True) + self.dropout = nn.Dropout(dropout) + self.conv2 = HunyuanVideoCausalConv3d(out_channels, out_channels, 3, 1, 0) + + self.conv_shortcut = None + if in_channels != out_channels: + self.conv_shortcut = HunyuanVideoCausalConv3d( + in_channels, out_channels, 1, 1, 0 + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = hidden_states.contiguous() + residual = hidden_states + + hidden_states = self.norm1(hidden_states) + hidden_states = self.nonlinearity(hidden_states) + hidden_states = self.conv1(hidden_states) + + hidden_states = self.norm2(hidden_states) + hidden_states = self.nonlinearity(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.conv2(hidden_states) + + if self.conv_shortcut is not None: + residual = self.conv_shortcut(residual) + + hidden_states = hidden_states + residual + return hidden_states + + +class HunyuanVideoMidBlock3D(nn.Module): + + def __init__( + self, + in_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_act_fn: str = "silu", + resnet_groups: int = 32, + add_attention: bool = True, + attention_head_dim: int = 1, + ) -> None: + super().__init__() + resnet_groups = ( + resnet_groups if resnet_groups is not None else min(in_channels // 4, 32) + ) + self.add_attention = add_attention + + # There is always at least one resnet + resnets = [ + HunyuanVideoResnetBlockCausal3D( + in_channels=in_channels, + out_channels=in_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + non_linearity=resnet_act_fn, + ) + ] + attentions: list[HunyuanVAEAttention | None] = [] + + for _ in range(num_layers): + if self.add_attention: + attentions.append( + HunyuanVAEAttention( + in_channels, + heads=in_channels // attention_head_dim, + dim_head=attention_head_dim, + eps=resnet_eps, + norm_num_groups=resnet_groups, + bias=True, + ) + ) + else: + attentions.append(None) + + resnets.append( + HunyuanVideoResnetBlockCausal3D( + in_channels=in_channels, + out_channels=in_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + non_linearity=resnet_act_fn, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + self.gradient_checkpointing = False + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + if torch.is_grad_enabled() and self.gradient_checkpointing: + hidden_states = self._gradient_checkpointing_func( + self.resnets[0], hidden_states + ) + + for attn, resnet in zip(self.attentions, self.resnets[1:], strict=True): + if attn is not None: + batch_size, num_channels, num_frames, height, width = ( + hidden_states.shape + ) + hidden_states = hidden_states.permute(0, 2, 3, 4, 1).flatten(1, 3) + attention_mask = prepare_causal_attention_mask( + num_frames, + height * width, + hidden_states.dtype, + hidden_states.device, + batch_size=batch_size, + ) + hidden_states = attn(hidden_states, attention_mask=attention_mask) + hidden_states = hidden_states.unflatten( + 1, (num_frames, height, width) + ).permute(0, 4, 1, 2, 3) + + hidden_states = self._gradient_checkpointing_func(resnet, hidden_states) + + else: + hidden_states = self.resnets[0](hidden_states) + + for attn, resnet in zip(self.attentions, self.resnets[1:], strict=True): + if attn is not None: + batch_size, num_channels, num_frames, height, width = ( + hidden_states.shape + ) + hidden_states = hidden_states.permute(0, 2, 3, 4, 1).flatten(1, 3) + attention_mask = prepare_causal_attention_mask( + num_frames, + height * width, + hidden_states.dtype, + hidden_states.device, + batch_size=batch_size, + ) + hidden_states = attn(hidden_states, attention_mask=attention_mask) + hidden_states = hidden_states.unflatten( + 1, (num_frames, height, width) + ).permute(0, 4, 1, 2, 3) + + hidden_states = resnet(hidden_states) + + return hidden_states + + +class HunyuanVideoDownBlock3D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_act_fn: str = "silu", + resnet_groups: int = 32, + add_downsample: bool = True, + downsample_stride: tuple[int, ...] | int = 2, + downsample_padding: int = 1, + ) -> None: + super().__init__() + resnets = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + HunyuanVideoResnetBlockCausal3D( + in_channels=in_channels, + out_channels=out_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + non_linearity=resnet_act_fn, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + HunyuanVideoDownsampleCausal3D( + out_channels, + out_channels=out_channels, + padding=downsample_padding, + stride=downsample_stride, + ) + ] + ) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + if torch.is_grad_enabled() and self.gradient_checkpointing: + for resnet in self.resnets: + hidden_states = self._gradient_checkpointing_func(resnet, hidden_states) + else: + for resnet in self.resnets: + hidden_states = resnet(hidden_states) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + return hidden_states + + +class HunyuanVideoUpBlock3D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_act_fn: str = "silu", + resnet_groups: int = 32, + add_upsample: bool = True, + upsample_scale_factor: tuple[int, ...] = (2, 2, 2), + ) -> None: + super().__init__() + resnets = [] + + for i in range(num_layers): + input_channels = in_channels if i == 0 else out_channels + + resnets.append( + HunyuanVideoResnetBlockCausal3D( + in_channels=input_channels, + out_channels=out_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + non_linearity=resnet_act_fn, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList( + [ + HunyuanVideoUpsampleCausal3D( + out_channels, + out_channels=out_channels, + upsample_factor=upsample_scale_factor, + ) + ] + ) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + if torch.is_grad_enabled() and self.gradient_checkpointing: + for resnet in self.resnets: + hidden_states = self._gradient_checkpointing_func(resnet, hidden_states) + + else: + for resnet in self.resnets: + hidden_states = resnet(hidden_states) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states) + + return hidden_states + + +class HunyuanVideoEncoder3D(nn.Module): + r""" + Causal encoder for 3D video-like data introduced in [Hunyuan Video](https://huggingface.co/papers/2412.03603). + """ + + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + down_block_types: tuple[str, ...] = ( + "HunyuanVideoDownBlock3D", + "HunyuanVideoDownBlock3D", + "HunyuanVideoDownBlock3D", + "HunyuanVideoDownBlock3D", + ), + block_out_channels: tuple[int, ...] = (128, 256, 512, 512), + layers_per_block: int = 2, + norm_num_groups: int = 32, + act_fn: str = "silu", + double_z: bool = True, + mid_block_add_attention=True, + temporal_compression_ratio: int = 4, + spatial_compression_ratio: int = 8, + ) -> None: + super().__init__() + + self.conv_in = HunyuanVideoCausalConv3d( + in_channels, block_out_channels[0], kernel_size=3, stride=1 + ) + self.mid_block: HunyuanVideoMidBlock3D | None = None + self.down_blocks = nn.ModuleList([]) + + output_channel = block_out_channels[0] + for i, down_block_type in enumerate(down_block_types): + if down_block_type != "HunyuanVideoDownBlock3D": + raise ValueError(f"Unsupported down_block_type: {down_block_type}") + + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + num_spatial_downsample_layers = int(np.log2(spatial_compression_ratio)) + num_time_downsample_layers = int(np.log2(temporal_compression_ratio)) + + if temporal_compression_ratio == 4: + add_spatial_downsample = bool(i < num_spatial_downsample_layers) + add_time_downsample = bool( + i >= (len(block_out_channels) - 1 - num_time_downsample_layers) + and not is_final_block + ) + elif temporal_compression_ratio == 8: + add_spatial_downsample = bool(i < num_spatial_downsample_layers) + add_time_downsample = bool(i < num_time_downsample_layers) + else: + raise ValueError( + f"Unsupported time_compression_ratio: {temporal_compression_ratio}" + ) + + downsample_stride_HW = (2, 2) if add_spatial_downsample else (1, 1) + downsample_stride_T = (2,) if add_time_downsample else (1,) + downsample_stride = tuple(downsample_stride_T + downsample_stride_HW) + + down_block = HunyuanVideoDownBlock3D( + num_layers=layers_per_block, + in_channels=input_channel, + out_channels=output_channel, + add_downsample=bool(add_spatial_downsample or add_time_downsample), + resnet_eps=1e-6, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + downsample_stride=downsample_stride, + downsample_padding=0, + ) + + self.down_blocks.append(down_block) + + self.mid_block = HunyuanVideoMidBlock3D( + in_channels=block_out_channels[-1], + resnet_eps=1e-6, + resnet_act_fn=act_fn, + attention_head_dim=block_out_channels[-1], + resnet_groups=norm_num_groups, + add_attention=mid_block_add_attention, + ) + + self.conv_norm_out = nn.GroupNorm( + num_channels=block_out_channels[-1], num_groups=norm_num_groups, eps=1e-6 + ) + self.conv_act = nn.SiLU() + + conv_out_channels = 2 * out_channels if double_z else out_channels + self.conv_out = HunyuanVideoCausalConv3d( + block_out_channels[-1], conv_out_channels, kernel_size=3 + ) + + self.gradient_checkpointing = False + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.conv_in(hidden_states) + + if torch.is_grad_enabled() and self.gradient_checkpointing: + for down_block in self.down_blocks: + hidden_states = self._gradient_checkpointing_func( + down_block, hidden_states + ) + + hidden_states = self._gradient_checkpointing_func( + self.mid_block, hidden_states + ) + else: + for down_block in self.down_blocks: + hidden_states = down_block(hidden_states) + assert self.mid_block is not None + hidden_states = self.mid_block(hidden_states) + + hidden_states = self.conv_norm_out(hidden_states) + hidden_states = self.conv_act(hidden_states) + hidden_states = self.conv_out(hidden_states) + + return hidden_states + + +class HunyuanVideoDecoder3D(nn.Module): + r""" + Causal decoder for 3D video-like data introduced in [Hunyuan Video](https://huggingface.co/papers/2412.03603). + """ + + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + up_block_types: tuple[str, ...] = ( + "HunyuanVideoUpBlock3D", + "HunyuanVideoUpBlock3D", + "HunyuanVideoUpBlock3D", + "HunyuanVideoUpBlock3D", + ), + block_out_channels: tuple[int, ...] = (128, 256, 512, 512), + layers_per_block: int = 2, + norm_num_groups: int = 32, + act_fn: str = "silu", + mid_block_add_attention=True, + time_compression_ratio: int = 4, + spatial_compression_ratio: int = 8, + ): + super().__init__() + self.layers_per_block = layers_per_block + + self.conv_in = HunyuanVideoCausalConv3d( + in_channels, block_out_channels[-1], kernel_size=3, stride=1 + ) + self.up_blocks = nn.ModuleList([]) + + # mid + self.mid_block = HunyuanVideoMidBlock3D( + in_channels=block_out_channels[-1], + resnet_eps=1e-6, + resnet_act_fn=act_fn, + attention_head_dim=block_out_channels[-1], + resnet_groups=norm_num_groups, + add_attention=mid_block_add_attention, + ) + + # up + reversed_block_out_channels = list(reversed(block_out_channels)) + output_channel = reversed_block_out_channels[0] + for i, up_block_type in enumerate(up_block_types): + if up_block_type != "HunyuanVideoUpBlock3D": + raise ValueError(f"Unsupported up_block_type: {up_block_type}") + + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + num_spatial_upsample_layers = int(np.log2(spatial_compression_ratio)) + num_time_upsample_layers = int(np.log2(time_compression_ratio)) + + if time_compression_ratio == 4: + add_spatial_upsample = bool(i < num_spatial_upsample_layers) + add_time_upsample = bool( + i >= len(block_out_channels) - 1 - num_time_upsample_layers + and not is_final_block + ) + else: + raise ValueError( + f"Unsupported time_compression_ratio: {time_compression_ratio}" + ) + + upsample_scale_factor_HW = (2, 2) if add_spatial_upsample else (1, 1) + upsample_scale_factor_T = (2,) if add_time_upsample else (1,) + upsample_scale_factor = tuple( + upsample_scale_factor_T + upsample_scale_factor_HW + ) + + up_block = HunyuanVideoUpBlock3D( + num_layers=self.layers_per_block + 1, + in_channels=prev_output_channel, + out_channels=output_channel, + add_upsample=bool(add_spatial_upsample or add_time_upsample), + upsample_scale_factor=upsample_scale_factor, + resnet_eps=1e-6, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + ) + + self.up_blocks.append(up_block) + prev_output_channel = output_channel + + # out + self.conv_norm_out = nn.GroupNorm( + num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=1e-6 + ) + self.conv_act = nn.SiLU() + self.conv_out = HunyuanVideoCausalConv3d( + block_out_channels[0], out_channels, kernel_size=3 + ) + + self.gradient_checkpointing = False + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.conv_in(hidden_states) + + if torch.is_grad_enabled() and self.gradient_checkpointing: + hidden_states = self._gradient_checkpointing_func( + self.mid_block, hidden_states + ) + + for up_block in self.up_blocks: + hidden_states = self._gradient_checkpointing_func( + up_block, hidden_states + ) + else: + hidden_states = self.mid_block(hidden_states) + + for up_block in self.up_blocks: + hidden_states = up_block(hidden_states) + + # post-process + hidden_states = self.conv_norm_out(hidden_states) + hidden_states = self.conv_act(hidden_states) + hidden_states = self.conv_out(hidden_states) + + return hidden_states + + +class AutoencoderKLHunyuanVideo(nn.Module, ParallelTiledVAE): + r""" + A VAE model with KL loss for encoding videos into latents and decoding latent representations into videos. + Introduced in [HunyuanVideo](https://huggingface.co/papers/2412.03603). + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + """ + + _supports_gradient_checkpointing = True + + def __init__( + self, + config: HunyuanVAEConfig, + ) -> None: + nn.Module.__init__(self) + ParallelTiledVAE.__init__(self, config) + + # TODO(will): only pass in config. We do this by manually defining a + # config for hunyuan vae + self.block_out_channels = config.block_out_channels + + if config.load_encoder: + self.encoder = HunyuanVideoEncoder3D( + in_channels=config.in_channels, + out_channels=config.latent_channels, + down_block_types=config.down_block_types, + block_out_channels=config.block_out_channels, + layers_per_block=config.layers_per_block, + norm_num_groups=config.norm_num_groups, + act_fn=config.act_fn, + double_z=True, + mid_block_add_attention=config.mid_block_add_attention, + temporal_compression_ratio=config.temporal_compression_ratio, + spatial_compression_ratio=config.spatial_compression_ratio, + ) + self.quant_conv = nn.Conv3d( + 2 * config.latent_channels, 2 * config.latent_channels, kernel_size=1 + ) + + if config.load_decoder: + self.decoder = HunyuanVideoDecoder3D( + in_channels=config.latent_channels, + out_channels=config.out_channels, + up_block_types=config.up_block_types, + block_out_channels=config.block_out_channels, + layers_per_block=config.layers_per_block, + norm_num_groups=config.norm_num_groups, + act_fn=config.act_fn, + time_compression_ratio=config.temporal_compression_ratio, + spatial_compression_ratio=config.spatial_compression_ratio, + mid_block_add_attention=config.mid_block_add_attention, + ) + self.post_quant_conv = nn.Conv3d( + config.latent_channels, config.latent_channels, kernel_size=1 + ) + + def _encode(self, x: torch.Tensor) -> torch.Tensor: + x = self.encoder(x) + enc = self.quant_conv(x) + return enc + + def _decode(self, z: torch.Tensor) -> torch.Tensor: + z = self.post_quant_conv(z) + dec = self.decoder(z) + return dec + + def forward( + self, + sample: torch.Tensor, + sample_posterior: bool = False, + generator: torch.Generator | None = None, + ) -> torch.Tensor: + r""" + Args: + sample (`torch.Tensor`): Input sample. + sample_posterior (`bool`, *optional*, defaults to `False`): + Whether to sample from the posterior. + """ + x = sample + posterior = self.encode(x).latent_dist + if sample_posterior: + z = posterior.sample(generator=generator) + else: + z = posterior.mode() + dec = self.decode(z) + return dec + + +EntryClass = AutoencoderKLHunyuanVideo diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/stepvideovae.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/stepvideovae.py new file mode 100644 index 00000000..d202b7a6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/stepvideovae.py @@ -0,0 +1,1184 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2025 StepFun Inc. All Rights Reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# ============================================================================== +from typing import Any + +import torch +from einops import rearrange +from torch import nn +from torch.nn import functional as F + +from sglang.multimodal_gen.configs.models.vaes import StepVideoVAEConfig +from sglang.multimodal_gen.runtime.models.vaes.common import ParallelTiledVAE + + +def base_group_norm(x, norm_layer, act_silu=False, channel_last=False) -> torch.Tensor: + if hasattr(base_group_norm, "spatial") and base_group_norm.spatial: + assert channel_last + x_shape = x.shape + x = x.flatten(0, 1) + if channel_last: + # Permute to NCHW format + x = x.permute(0, 3, 1, 2) + + out = F.group_norm( + x.contiguous(), + norm_layer.num_groups, + norm_layer.weight, + norm_layer.bias, + norm_layer.eps, + ) + if act_silu: + out = F.silu(out) + + if channel_last: + # Permute back to NHWC format + out = out.permute(0, 2, 3, 1) + + out = out.view(x_shape) + else: + if channel_last: + # Permute to NCHW format + x = x.permute(0, 3, 1, 2) + out = F.group_norm( + x.contiguous(), + norm_layer.num_groups, + norm_layer.weight, + norm_layer.bias, + norm_layer.eps, + ) + if act_silu: + out = F.silu(out) + if channel_last: + # Permute back to NHWC format + out = out.permute(0, 2, 3, 1) + return out + + +def base_conv2d(x, conv_layer, channel_last=False, residual=None) -> torch.Tensor: + if channel_last: + x = x.permute(0, 3, 1, 2) # NHWC to NCHW + out = F.conv2d( + x, + conv_layer.weight, + conv_layer.bias, + stride=conv_layer.stride, + padding=conv_layer.padding, + ) + if residual is not None: + if channel_last: + residual = residual.permute(0, 3, 1, 2) # NHWC to NCHW + out += residual + if channel_last: + out = out.permute(0, 2, 3, 1) # NCHW to NHWC + return out + + +def base_conv3d( + x, conv_layer, channel_last=False, residual=None, only_return_output=False +) -> torch.Tensor: + if only_return_output: + size = cal_outsize( + x.shape, conv_layer.weight.shape, conv_layer.stride, conv_layer.padding + ) + return torch.empty(size, device=x.device, dtype=x.dtype) + if channel_last: + x = x.permute(0, 4, 1, 2, 3) # NDHWC to NCDHW + out = F.conv3d( + x, + conv_layer.weight, + conv_layer.bias, + stride=conv_layer.stride, + padding=conv_layer.padding, + ) + if residual is not None: + if channel_last: + residual = residual.permute(0, 4, 1, 2, 3) # NDHWC to NCDHW + out += residual + if channel_last: + out = out.permute(0, 2, 3, 4, 1) # NCDHW to NDHWC + return out + + +def cal_outsize(input_sizes, kernel_sizes, stride, padding) -> list: + stride_d, stride_h, stride_w = stride + padding_d, padding_h, padding_w = padding + dilation_d, dilation_h, dilation_w = 1, 1, 1 + + in_d = input_sizes[1] + in_h = input_sizes[2] + in_w = input_sizes[3] + + kernel_d = kernel_sizes[2] + kernel_h = kernel_sizes[3] + kernel_w = kernel_sizes[4] + out_channels = kernel_sizes[0] + + out_d = calc_out_(in_d, padding_d, dilation_d, kernel_d, stride_d) + out_h = calc_out_(in_h, padding_h, dilation_h, kernel_h, stride_h) + out_w = calc_out_(in_w, padding_w, dilation_w, kernel_w, stride_w) + size = [input_sizes[0], out_d, out_h, out_w, out_channels] + return size + + +def calc_out_( + in_size: int, padding: int, dilation: int, kernel: int, stride: int +) -> int: + return (in_size + 2 * padding - dilation * (kernel - 1) - 1) // stride + 1 + + +def base_conv3d_channel_last(x, conv_layer, residual=None) -> torch.Tensor: + in_numel = x.numel() + out_numel = int(x.numel() * conv_layer.out_channels / conv_layer.in_channels) + if (in_numel >= 2**30) or (out_numel >= 2**30): + assert conv_layer.stride[0] == 1, "time split asks time stride = 1" + + B, T, H, W, C = x.shape + K = conv_layer.kernel_size[0] + + chunks = 4 + chunk_size = T // chunks + + if residual is None: + out_nhwc = base_conv3d( + x, + conv_layer, + channel_last=True, + residual=residual, + only_return_output=True, + ) + else: + out_nhwc = residual + + assert B == 1 + for i in range(chunks): + if i == chunks - 1: + xi = x[:1, chunk_size * i :] + out_nhwci = out_nhwc[:1, chunk_size * i :] + else: + xi = x[:1, chunk_size * i : chunk_size * (i + 1) + K - 1] + out_nhwci = out_nhwc[:1, chunk_size * i : chunk_size * (i + 1)] + if residual is not None: + if i == chunks - 1: + ri = residual[:1, chunk_size * i :] + else: + ri = residual[:1, chunk_size * i : chunk_size * (i + 1)] + else: + ri = None + out_nhwci.copy_(base_conv3d(xi, conv_layer, channel_last=True, residual=ri)) + else: + out_nhwc = base_conv3d(x, conv_layer, channel_last=True, residual=residual) + return out_nhwc + + +class Upsample2D(nn.Module): + + def __init__( + self, channels, use_conv=False, use_conv_transpose=False, out_channels=None + ) -> None: + super().__init__() + self.channels = channels + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.use_conv_transpose = use_conv_transpose + + if use_conv: + self.conv = nn.Conv2d(self.channels, self.out_channels, 3, padding=1) + else: + assert "Not Supported" + self.conv = nn.ConvTranspose2d(channels, self.out_channels, 4, 2, 1) + + def forward(self, x, output_size=None) -> torch.Tensor: + assert x.shape[-1] == self.channels + + if self.use_conv_transpose: + return self.conv(x) + + if output_size is None: + x = ( + F.interpolate( + x.permute(0, 3, 1, 2).to(memory_format=torch.channels_last), + scale_factor=2.0, + mode="nearest", + ) + .permute(0, 2, 3, 1) + .contiguous() + ) + else: + x = ( + F.interpolate( + x.permute(0, 3, 1, 2).to(memory_format=torch.channels_last), + size=output_size, + mode="nearest", + ) + .permute(0, 2, 3, 1) + .contiguous() + ) + + # x = self.conv(x) + x = base_conv2d(x, self.conv, channel_last=True) + return x + + +class Downsample2D(nn.Module): + + def __init__(self, channels, use_conv=False, out_channels=None, padding=1) -> None: + super().__init__() + self.channels = channels + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.padding = padding + stride = 2 + + if use_conv: + self.conv = nn.Conv2d( + self.channels, self.out_channels, 3, stride=stride, padding=padding + ) + else: + assert self.channels == self.out_channels + self.conv = nn.AvgPool2d(kernel_size=stride, stride=stride) + + def forward(self, x) -> torch.Tensor: + assert x.shape[-1] == self.channels + if self.use_conv and self.padding == 0: + pad = (0, 0, 0, 1, 0, 1) + x = F.pad(x, pad, mode="constant", value=0) + + assert x.shape[-1] == self.channels + # x = self.conv(x) + x = base_conv2d(x, self.conv, channel_last=True) + return x + + +class CausalConv(nn.Module): + + def __init__(self, chan_in, chan_out, kernel_size, **kwargs) -> None: + super().__init__() + + if isinstance(kernel_size, int): + kernel_size = ( + kernel_size if isinstance(kernel_size, tuple) else ((kernel_size,) * 3) + ) + time_kernel_size, height_kernel_size, width_kernel_size = kernel_size + + self.dilation = kwargs.pop("dilation", 1) + self.stride = kwargs.pop("stride", 1) + if isinstance(self.stride, int): + self.stride = (self.stride, 1, 1) + time_pad = self.dilation * (time_kernel_size - 1) + max((1 - self.stride[0]), 0) + height_pad = height_kernel_size // 2 + width_pad = width_kernel_size // 2 + self.time_causal_padding = ( + width_pad, + width_pad, + height_pad, + height_pad, + time_pad, + 0, + ) + self.time_uncausal_padding = ( + width_pad, + width_pad, + height_pad, + height_pad, + 0, + 0, + ) + + self.conv = nn.Conv3d( + chan_in, + chan_out, + kernel_size, + stride=self.stride, + dilation=self.dilation, + **kwargs, + ) + self.chan_in = chan_in + self.chan_out = chan_out + self.is_first_run = True + + def forward(self, x, is_init=True, residual=None) -> torch.Tensor: + x = nn.functional.pad( + x, self.time_causal_padding if is_init else self.time_uncausal_padding + ) + x = self.conv(x) + if residual is not None: + x.add_(residual) + return x + + +class ChannelDuplicatingPixelUnshuffleUpSampleLayer3D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + factor: int, + ) -> None: + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.factor = factor + assert out_channels * factor**3 % in_channels == 0 + self.repeats = out_channels * factor**3 // in_channels + + def forward(self, x: torch.Tensor, is_init=True) -> torch.Tensor: + x = x.repeat_interleave(self.repeats, dim=1) + x = x.view( + x.size(0), + self.out_channels, + self.factor, + self.factor, + self.factor, + x.size(2), + x.size(3), + x.size(4), + ) + x = x.permute(0, 1, 5, 2, 6, 3, 7, 4).contiguous() + x = x.view( + x.size(0), + self.out_channels, + x.size(2) * self.factor, + x.size(4) * self.factor, + x.size(6) * self.factor, + ) + x = x[:, :, self.factor - 1 :, :, :] + return x + + +class ConvPixelShuffleUpSampleLayer3D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int, + factor: int, + ) -> None: + super().__init__() + self.factor = factor + out_ratio = factor**3 + self.conv = CausalConv( + in_channels, out_channels * out_ratio, kernel_size=kernel_size + ) + + def forward(self, x: torch.Tensor, is_init=True) -> torch.Tensor: + x = self.conv(x, is_init) + x = self.pixel_shuffle_3d(x, self.factor) + return x + + @staticmethod + def pixel_shuffle_3d(x: torch.Tensor, factor: int) -> torch.Tensor: + batch_size, channels, depth, height, width = x.size() + new_channels = channels // (factor**3) + new_depth = depth * factor + new_height = height * factor + new_width = width * factor + + x = x.view( + batch_size, new_channels, factor, factor, factor, depth, height, width + ) + x = x.permute(0, 1, 5, 2, 6, 3, 7, 4).contiguous() + x = x.view(batch_size, new_channels, new_depth, new_height, new_width) + x = x[:, :, factor - 1 :, :, :] + return x + + +class ConvPixelUnshuffleDownSampleLayer3D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int, + factor: int, + ) -> None: + super().__init__() + self.factor = factor + out_ratio = factor**3 + assert out_channels % out_ratio == 0 + self.conv = CausalConv( + in_channels, out_channels // out_ratio, kernel_size=kernel_size + ) + + def forward(self, x: torch.Tensor, is_init=True) -> torch.Tensor: + x = self.conv(x, is_init) + x = self.pixel_unshuffle_3d(x, self.factor) + return x + + @staticmethod + def pixel_unshuffle_3d(x: torch.Tensor, factor: int) -> torch.Tensor: + pad = (0, 0, 0, 0, factor - 1, 0) # (left, right, top, bottom, front, back) + x = F.pad(x, pad) + B, C, D, H, W = x.shape + x = x.view(B, C, D // factor, factor, H // factor, factor, W // factor, factor) + x = x.permute(0, 1, 3, 5, 7, 2, 4, 6).contiguous() + x = x.view(B, C * factor**3, D // factor, H // factor, W // factor) + return x + + +class PixelUnshuffleChannelAveragingDownSampleLayer3D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + factor: int, + ) -> None: + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.factor = factor + assert in_channels * factor**3 % out_channels == 0 + self.group_size = in_channels * factor**3 // out_channels + + def forward(self, x: torch.Tensor, is_init=True) -> torch.Tensor: + pad = ( + 0, + 0, + 0, + 0, + self.factor - 1, + 0, + ) # (left, right, top, bottom, front, back) + x = F.pad(x, pad) + B, C, D, H, W = x.shape + x = x.view( + B, + C, + D // self.factor, + self.factor, + H // self.factor, + self.factor, + W // self.factor, + self.factor, + ) + x = x.permute(0, 1, 3, 5, 7, 2, 4, 6).contiguous() + x = x.view( + B, C * self.factor**3, D // self.factor, H // self.factor, W // self.factor + ) + x = x.view( + B, + self.out_channels, + self.group_size, + D // self.factor, + H // self.factor, + W // self.factor, + ) + x = x.mean(dim=2) + return x + + +def base_group_norm_with_zero_pad( + x, norm_layer, act_silu=True, pad_size=2 +) -> torch.Tensor: + out_shape = list(x.shape) + out_shape[1] += pad_size + out = torch.empty(out_shape, dtype=x.dtype, device=x.device) + out[:, pad_size:] = base_group_norm( + x, norm_layer, act_silu=act_silu, channel_last=True + ) + out[:, :pad_size] = 0 + return out + + +class CausalConvChannelLast(CausalConv): + time_causal_padding: tuple[Any, ...] + time_uncausal_padding: tuple[Any, ...] + + def __init__(self, chan_in, chan_out, kernel_size, **kwargs) -> None: + super().__init__(chan_in, chan_out, kernel_size, **kwargs) + + self.time_causal_padding = (0, 0) + self.time_causal_padding + self.time_uncausal_padding = (0, 0) + self.time_uncausal_padding + + def forward(self, x, is_init=True, residual=None) -> torch.Tensor: + if self.is_first_run: + self.is_first_run = False + # self.conv.weight = nn.Parameter(self.conv.weight.permute(0,2,3,4,1).contiguous()) + + x = nn.functional.pad( + x, self.time_causal_padding if is_init else self.time_uncausal_padding + ) + + x = base_conv3d_channel_last(x, self.conv, residual=residual) + return x + + +class CausalConvAfterNorm(CausalConv): + + def __init__(self, chan_in, chan_out, kernel_size, **kwargs) -> None: + super().__init__(chan_in, chan_out, kernel_size, **kwargs) + + if self.time_causal_padding == (1, 1, 1, 1, 2, 0): + self.conv = nn.Conv3d( + chan_in, + chan_out, + kernel_size, + stride=self.stride, + dilation=self.dilation, + padding=(0, 1, 1), + **kwargs, + ) + else: + self.conv = nn.Conv3d( + chan_in, + chan_out, + kernel_size, + stride=self.stride, + dilation=self.dilation, + **kwargs, + ) + self.is_first_run = True + + def forward(self, x, is_init=True, residual=None) -> torch.Tensor: + if self.is_first_run: + self.is_first_run = False + + if self.time_causal_padding == (1, 1, 1, 1, 2, 0): + pass + else: + x = nn.functional.pad(x, self.time_causal_padding).contiguous() + + x = base_conv3d_channel_last(x, self.conv, residual=residual) + return x + + +class AttnBlock(nn.Module): + + def __init__(self, in_channels) -> None: + super().__init__() + + self.norm = nn.GroupNorm(num_groups=32, num_channels=in_channels) + self.q = CausalConvChannelLast(in_channels, in_channels, kernel_size=1) + self.k = CausalConvChannelLast(in_channels, in_channels, kernel_size=1) + self.v = CausalConvChannelLast(in_channels, in_channels, kernel_size=1) + self.proj_out = CausalConvChannelLast(in_channels, in_channels, kernel_size=1) + + def attention(self, x, is_init=True) -> torch.Tensor: + x = base_group_norm(x, self.norm, act_silu=False, channel_last=True) + q = self.q(x, is_init) + k = self.k(x, is_init) + v = self.v(x, is_init) + + b, t, h, w, c = q.shape + q, k, v = map(lambda x: rearrange(x, "b t h w c -> b 1 (t h w) c"), (q, k, v)) + x = nn.functional.scaled_dot_product_attention(q, k, v, is_causal=True) + x = rearrange(x, "b 1 (t h w) c -> b t h w c", t=t, h=h, w=w) + + return x + + def forward(self, x): + x = x.permute(0, 2, 3, 4, 1).contiguous() + h = self.attention(x) + x = self.proj_out(h, residual=x) + x = x.permute(0, 4, 1, 2, 3) + return x + + +class Resnet3DBlock(nn.Module): + + def __init__( + self, + in_channels, + out_channels=None, + temb_channels=512, + conv_shortcut=False, + ) -> None: + super().__init__() + + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + self.out_channels = out_channels + + self.norm1 = nn.GroupNorm(num_groups=32, num_channels=in_channels) + self.conv1 = CausalConvAfterNorm(in_channels, out_channels, kernel_size=3) + if temb_channels > 0: + self.temb_proj = nn.Linear(temb_channels, out_channels) + + self.norm2 = nn.GroupNorm(num_groups=32, num_channels=out_channels) + self.conv2 = CausalConvAfterNorm(out_channels, out_channels, kernel_size=3) + + assert conv_shortcut is False + self.use_conv_shortcut = conv_shortcut + if self.in_channels != self.out_channels: + if self.use_conv_shortcut: + self.conv_shortcut = CausalConvAfterNorm( + in_channels, out_channels, kernel_size=3 + ) + else: + self.nin_shortcut = CausalConvAfterNorm( + in_channels, out_channels, kernel_size=1 + ) + + def forward(self, x, temb=None, is_init=True) -> torch.Tensor: + x = x.permute(0, 2, 3, 4, 1).contiguous() + + h = base_group_norm_with_zero_pad(x, self.norm1, act_silu=True, pad_size=2) + h = self.conv1(h) + if temb is not None: + h = h + self.temb_proj(nn.functional.silu(temb))[:, :, None, None] + + x = self.nin_shortcut(x) if self.in_channels != self.out_channels else x + + h = base_group_norm_with_zero_pad(h, self.norm2, act_silu=True, pad_size=2) + x = self.conv2(h, residual=x) + + x = x.permute(0, 4, 1, 2, 3) + return x + + +class Downsample3D(nn.Module): + + def __init__(self, in_channels, with_conv, stride) -> None: + super().__init__() + + self.with_conv = with_conv + if with_conv: + self.conv = CausalConv( + in_channels, in_channels, kernel_size=3, stride=stride + ) + + def forward(self, x, is_init=True) -> torch.Tensor: + if self.with_conv: + x = self.conv(x, is_init) + else: + x = nn.functional.avg_pool3d(x, kernel_size=2, stride=2) + return x + + +class VideoEncoder(nn.Module): + + def __init__( + self, + ch=32, + ch_mult=(4, 8, 16, 16), + num_res_blocks=2, + in_channels=3, + z_channels=16, + double_z=True, + down_sampling_layer=(1, 2), + resamp_with_conv=True, + version=1, + ) -> None: + super().__init__() + + temb_ch = 0 + + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + + # downsampling + self.conv_in = CausalConv(in_channels, ch, kernel_size=3) + self.down_sampling_layer = down_sampling_layer + + in_ch_mult = (1,) + tuple(ch_mult) + self.down = nn.ModuleList() + for i_level in range(self.num_resolutions): + block = nn.ModuleList() + attn = nn.ModuleList() + block_in = ch * in_ch_mult[i_level] + block_out = ch * ch_mult[i_level] + for i_block in range(self.num_res_blocks): + block.append( + Resnet3DBlock( + in_channels=block_in, + out_channels=block_out, + temb_channels=temb_ch, + ) + ) + block_in = block_out + down = nn.Module() + down.block = block + down.attn = attn + if i_level != self.num_resolutions - 1: + if i_level in self.down_sampling_layer: + down.downsample = Downsample3D( + block_in, resamp_with_conv, stride=(2, 2, 2) + ) + else: + down.downsample = Downsample2D( + block_in, resamp_with_conv, padding=0 + ) # DIFF + self.down.append(down) + + # middle + self.mid = nn.Module() + self.mid.block_1 = Resnet3DBlock( + in_channels=block_in, out_channels=block_in, temb_channels=temb_ch + ) + self.mid.attn_1 = AttnBlock(block_in) + self.mid.block_2 = Resnet3DBlock( + in_channels=block_in, out_channels=block_in, temb_channels=temb_ch + ) + + # end + self.norm_out = nn.GroupNorm(num_groups=32, num_channels=block_in) + self.version = version + if version == 2: + channels = 4 * z_channels * 2**3 + self.conv_patchify = ConvPixelUnshuffleDownSampleLayer3D( + block_in, channels, kernel_size=3, factor=2 + ) + self.shortcut_pathify = PixelUnshuffleChannelAveragingDownSampleLayer3D( + block_in, channels, 2 + ) + self.shortcut_out = PixelUnshuffleChannelAveragingDownSampleLayer3D( + channels, 2 * z_channels if double_z else z_channels, 1 + ) + self.conv_out = CausalConvChannelLast( + channels, 2 * z_channels if double_z else z_channels, kernel_size=3 + ) + else: + self.conv_out = CausalConvAfterNorm( + block_in, 2 * z_channels if double_z else z_channels, kernel_size=3 + ) + + @torch.inference_mode() + def forward(self, x, video_frame_num, is_init=True) -> torch.Tensor: + # timestep embedding + temb = None + + t = video_frame_num + + # downsampling + h = self.conv_in(x, is_init) + + # make it real channel last, but behave like normal layout + h = h.permute(0, 2, 3, 4, 1).contiguous().permute(0, 4, 1, 2, 3) + + for i_level in range(self.num_resolutions): + for i_block in range(self.num_res_blocks): + h = self.down[i_level].block[i_block](h, temb, is_init) + if len(self.down[i_level].attn) > 0: + h = self.down[i_level].attn[i_block](h) + + if i_level != self.num_resolutions - 1: + if isinstance(self.down[i_level].downsample, Downsample2D): + _, _, t, _, _ = h.shape + h = rearrange(h, "b c t h w -> (b t) h w c", t=t) + h = self.down[i_level].downsample(h) + h = rearrange(h, "(b t) h w c -> b c t h w", t=t) + else: + h = self.down[i_level].downsample(h, is_init) + + h = self.mid.block_1(h, temb, is_init) + h = self.mid.attn_1(h) + h = self.mid.block_2(h, temb, is_init) + + h = h.permute(0, 2, 3, 4, 1).contiguous() # b c l h w -> b l h w c + if self.version == 2: + h = base_group_norm(h, self.norm_out, act_silu=True, channel_last=True) + h = h.permute(0, 4, 1, 2, 3).contiguous() + shortcut = self.shortcut_pathify(h, is_init) + h = self.conv_patchify(h, is_init) + h = h.add_(shortcut) + shortcut = self.shortcut_out(h, is_init).permute(0, 2, 3, 4, 1) + h = self.conv_out(h.permute(0, 2, 3, 4, 1).contiguous(), is_init) + h = h.add_(shortcut) + else: + h = base_group_norm_with_zero_pad( + h, self.norm_out, act_silu=True, pad_size=2 + ) + h = self.conv_out(h, is_init) + h = h.permute(0, 4, 1, 2, 3) # b l h w c -> b c l h w + + h = rearrange(h, "b c t h w -> b t c h w") + return h + + +class Res3DBlockUpsample(nn.Module): + + def __init__( + self, input_filters, num_filters, down_sampling_stride, down_sampling=False + ) -> None: + super().__init__() + + self.input_filters = input_filters + self.num_filters = num_filters + + self.act_ = nn.SiLU(inplace=True) + + self.conv1 = CausalConvChannelLast( + num_filters, num_filters, kernel_size=[3, 3, 3] + ) + self.norm1 = nn.GroupNorm(32, num_filters) + + self.conv2 = CausalConvChannelLast( + num_filters, num_filters, kernel_size=[3, 3, 3] + ) + self.norm2 = nn.GroupNorm(32, num_filters) + + self.down_sampling = down_sampling + if down_sampling: + self.down_sampling_stride = down_sampling_stride + else: + self.down_sampling_stride = [1, 1, 1] + + if num_filters != input_filters or down_sampling: + self.conv3 = CausalConvChannelLast( + input_filters, + num_filters, + kernel_size=[1, 1, 1], + stride=self.down_sampling_stride, + ) + self.norm3 = nn.GroupNorm(32, num_filters) + + def forward(self, x, is_init=False) -> torch.Tensor: + x = x.permute(0, 2, 3, 4, 1).contiguous() + + residual = x + + h = self.conv1(x, is_init) + h = base_group_norm(h, self.norm1, act_silu=True, channel_last=True) + + h = self.conv2(h, is_init) + h = base_group_norm(h, self.norm2, act_silu=False, channel_last=True) + + if self.down_sampling or self.num_filters != self.input_filters: + x = self.conv3(x, is_init) + x = base_group_norm(x, self.norm3, act_silu=False, channel_last=True) + + h.add_(x) + h = self.act_(h) + if residual is not None: + h.add_(residual) + + h = h.permute(0, 4, 1, 2, 3) + return h + + +class Upsample3D(nn.Module): + + def __init__(self, in_channels, scale_factor=2) -> None: + super().__init__() + + self.scale_factor = scale_factor + self.conv3d = Res3DBlockUpsample( + input_filters=in_channels, + num_filters=in_channels, + down_sampling_stride=(1, 1, 1), + down_sampling=False, + ) + + def forward(self, x, is_init=True, is_split=True) -> torch.Tensor: + b, c, t, h, w = x.shape + + # x = x.permute(0,2,3,4,1).contiguous().permute(0,4,1,2,3).to(memory_format=torch.channels_last_3d) + if is_split: + split_size = c // 8 + x_slices = torch.split(x, split_size, dim=1) + x = [ + nn.functional.interpolate(x, scale_factor=self.scale_factor) + for x in x_slices + ] + x = torch.cat(x, dim=1) + else: + x = nn.functional.interpolate(x, scale_factor=self.scale_factor) + + x = self.conv3d(x, is_init) + return x + + +class VideoDecoder(nn.Module): + + def __init__( + self, + ch=128, + z_channels=16, + out_channels=3, + ch_mult=(1, 2, 4, 4), + num_res_blocks=2, + temporal_up_layers=(2, 3), + temporal_downsample=4, + resamp_with_conv=True, + version=1, + ) -> None: + super().__init__() + + temb_ch = 0 + + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.temporal_downsample = temporal_downsample + + block_in = ch * ch_mult[self.num_resolutions - 1] + self.version = version + if version == 2: + channels = 4 * z_channels * 2**3 + self.conv_in = CausalConv(z_channels, channels, kernel_size=3) + self.shortcut_in = ChannelDuplicatingPixelUnshuffleUpSampleLayer3D( + z_channels, channels, 1 + ) + self.conv_unpatchify = ConvPixelShuffleUpSampleLayer3D( + channels, block_in, kernel_size=3, factor=2 + ) + self.shortcut_unpathify = ChannelDuplicatingPixelUnshuffleUpSampleLayer3D( + channels, block_in, 2 + ) + else: + self.conv_in = CausalConv(z_channels, block_in, kernel_size=3) + + # middle + self.mid = nn.Module() + self.mid.block_1 = Resnet3DBlock( + in_channels=block_in, out_channels=block_in, temb_channels=temb_ch + ) + self.mid.attn_1 = AttnBlock(block_in) + self.mid.block_2 = Resnet3DBlock( + in_channels=block_in, out_channels=block_in, temb_channels=temb_ch + ) + + # upsampling + self.up_id = len(temporal_up_layers) + self.video_frame_num = 1 + self.cur_video_frame_num = self.video_frame_num // 2**self.up_id + 1 + self.up = nn.ModuleList() + for i_level in reversed(range(self.num_resolutions)): + block = nn.ModuleList() + attn = nn.ModuleList() + block_out = ch * ch_mult[i_level] + for i_block in range(self.num_res_blocks + 1): + block.append( + Resnet3DBlock( + in_channels=block_in, + out_channels=block_out, + temb_channels=temb_ch, + ) + ) + block_in = block_out + up = nn.Module() + up.block = block + up.attn = attn + if i_level != 0: + if i_level in temporal_up_layers: + up.upsample = Upsample3D(block_in) + self.cur_video_frame_num = self.cur_video_frame_num * 2 + else: + up.upsample = Upsample2D(block_in, resamp_with_conv) + self.up.insert(0, up) # prepend to get consistent order + + # end + self.norm_out = nn.GroupNorm(num_groups=32, num_channels=block_in) + self.conv_out = CausalConvAfterNorm(block_in, out_channels, kernel_size=3) + + @torch.inference_mode() + def forward(self, z, is_init=True) -> torch.Tensor: + z = rearrange(z, "b t c h w -> b c t h w") + h = self.conv_in(z, is_init=is_init) + if self.version == 2: + shortcut = self.shortcut_in(z, is_init=is_init) + h = h.add_(shortcut) + shortcut = self.shortcut_unpathify(h, is_init=is_init) + h = self.conv_unpatchify(h, is_init=is_init) + h = h.add_(shortcut) + + temb = None + + h = h.permute(0, 2, 3, 4, 1).contiguous().permute(0, 4, 1, 2, 3) + h = self.mid.block_1(h, temb, is_init=is_init) + h = self.mid.attn_1(h) + h = h.permute(0, 2, 3, 4, 1).contiguous().permute(0, 4, 1, 2, 3) + h = self.mid.block_2(h, temb, is_init=is_init) + + # upsampling + for i_level in reversed(range(self.num_resolutions)): + for i_block in range(self.num_res_blocks + 1): + h = h.permute(0, 2, 3, 4, 1).contiguous().permute(0, 4, 1, 2, 3) + h = self.up[i_level].block[i_block](h, temb, is_init=is_init) + if len(self.up[i_level].attn) > 0: + h = self.up[i_level].attn[i_block](h) + if i_level != 0: + if isinstance(self.up[i_level].upsample, Upsample2D): + B = h.size(0) + h = h.permute(0, 2, 3, 4, 1).flatten(0, 1) + h = self.up[i_level].upsample(h) + h = h.unflatten(0, (B, -1)).permute(0, 4, 1, 2, 3) + else: + h = self.up[i_level].upsample(h, is_init=is_init) + + # end + h = h.permute(0, 2, 3, 4, 1) # b c l h w -> b l h w c + h = base_group_norm_with_zero_pad(h, self.norm_out, act_silu=True, pad_size=2) + h = self.conv_out(h) + h = h.permute(0, 4, 1, 2, 3) + + if is_init: + h = h[:, :, (self.temporal_downsample - 1) :] + return h + + +def rms_norm(input, normalized_shape, eps=1e-6) -> torch.Tensor: + dtype = input.dtype + input = input.to(torch.float32) + variance = ( + input.pow(2) + .flatten(-len(normalized_shape)) + .mean(-1)[(...,) + (None,) * len(normalized_shape)] + ) + input = input * torch.rsqrt(variance + eps) + return input.to(dtype) + + +class DiagonalGaussianDistribution: + + def __init__( + self, + parameters, + deterministic=False, + rms_norm_mean=False, + only_return_mean=False, + ) -> None: + self.parameters = parameters + self.mean, self.logvar = torch.chunk(parameters, 2, dim=-3) # N,[X],C,H,W + self.logvar = torch.clamp(self.logvar, -30.0, 20.0) + self.std = torch.exp(0.5 * self.logvar) + self.var = torch.exp(self.logvar) + self.deterministic = deterministic + if self.deterministic: + self.var = self.std = torch.zeros_like( + self.mean, device=self.parameters.device, dtype=self.parameters.dtype + ) + if rms_norm_mean: + self.mean = rms_norm(self.mean, self.mean.size()[1:]) + self.only_return_mean = only_return_mean + + def sample(self, generator=None) -> torch.Tensor: + # make sure sample is on the same device + # as the parameters and has same dtype + sample = torch.randn( + self.mean.shape, generator=generator, device=self.parameters.device + ) + sample = sample.to(dtype=self.parameters.dtype) + x = self.mean + self.std * sample + if self.only_return_mean: + return self.mean + else: + return x + + +class AutoencoderKLStepvideo(nn.Module, ParallelTiledVAE): + + def __init__( + self, + config: StepVideoVAEConfig, + ) -> None: + nn.Module.__init__(self) + ParallelTiledVAE.__init__(self, config) + + self.frame_len = config.frame_len + + if config.version == 2: + self.latent_len = 3 + base_group_norm.spatial = True # type: ignore[attr-defined] + else: + self.latent_len = 5 + base_group_norm.spatial = False # type: ignore[attr-defined] + + self.encoder = VideoEncoder( + in_channels=config.in_channels, + z_channels=config.z_channels, + num_res_blocks=config.num_res_blocks, + version=config.version, + ) + + self.decoder = VideoDecoder( + z_channels=config.z_channels, + out_channels=config.out_channels, + num_res_blocks=config.num_res_blocks, + version=config.version, + ) + + self.world_size = config.world_size + # self.is_init = True + + def load_state_dict(self, state_dict, strict=True): + remapped = {} + for key, value in state_dict.items(): + if key.startswith("decoder.conv_out."): + # move “decoder.conv_out.weight” → “decoder.conv_out.conv.weight” + suffix = key[len("decoder.conv_out.") :] + remapped[f"decoder.conv_out.conv.{suffix}"] = value + else: + remapped[key] = value + super().load_state_dict(remapped, strict=strict) + + def _encode(self, x, is_init_image=True) -> torch.Tensor: + # b, len, c, h, w = x.size() + b, c, len, h, w = x.size() + # x = rearrange(x, 'b l c h w -> b c l h w').contiguous() + z = self.encoder(x, len, True) # 下采样[1, 4, 8, 16, 16] + return z + + @torch.inference_mode() + def encode(self, x): + # b (nc cf) c h w -> (b nc) cf c h w -> encode -> (b nc) cf c h w -> b (nc cf) c h w + chunks = list(x.split(self.frame_len, dim=1)) + for i in range(len(chunks)): + chunks[i] = self._encode(chunks[i], True) + z = torch.cat(chunks, dim=1) + + posterior = DiagonalGaussianDistribution(z) + return posterior.sample() + + def _decode(self, z) -> torch.Tensor: + + chunks = list(z.split(self.latent_len, dim=2)) + for i in range(len(chunks)): + chunks[i] = chunks[i].permute(0, 2, 1, 3, 4) + chunks[i] = chunks[i].to(next(self.decoder.parameters()).dtype) + chunks[i] = self.decoder(chunks[i], is_init=True) + x = torch.cat(chunks, dim=2) + return x + + def decode(self, z) -> torch.Tensor: + num_frames = z.size(2) + dec = ParallelTiledVAE.decode(self, z).permute(0, 2, 1, 3, 4) + dec = self.mix(dec).permute(0, 2, 1, 3, 4) + num_sample_frames = num_frames // 3 * 17 + return dec[:, :, :num_sample_frames] + + def mix(self, x) -> torch.Tensor: + remain_scale = 0.6 + mix_scale = 1.0 - remain_scale + front = slice(self.frame_len - 1, x.size(1) - 1, self.frame_len) + back = slice(self.frame_len, x.size(1), self.frame_len) + x[:, back] = x[:, back] * remain_scale + x[:, front] * mix_scale + x[:, front] = x[:, front] * remain_scale + x[:, back] * mix_scale + return x + + def forward( + self, + sample: torch.Tensor, + sample_posterior: bool = False, + generator: torch.Generator | None = None, + ) -> torch.Tensor: + """ + Args: + sample (`torch.Tensor`): Input sample. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`DecoderOutput`] instead of a plain tuple. + """ + x = sample + posterior = self.encode(x).latent_dist + if sample_posterior: + z = posterior.sample(generator=generator) + else: + z = posterior.mode() + dec = self.decode(z) + return dec + + +EntryClass = AutoencoderKLStepvideo diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py new file mode 100644 index 00000000..1018d43b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py @@ -0,0 +1,1343 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +# Copyright 2025 The Wan Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import contextvars +from contextlib import contextmanager + +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange + +from sglang.multimodal_gen.configs.models.vaes import WanVAEConfig +from sglang.multimodal_gen.runtime.layers.activation import get_act_fn +from sglang.multimodal_gen.runtime.models.vaes.common import ( + DiagonalGaussianDistribution, + ParallelTiledVAE, +) +from sglang.multimodal_gen.runtime.platforms import current_platform + +CACHE_T = 2 + +is_first_frame = contextvars.ContextVar("is_first_frame", default=False) +feat_cache = contextvars.ContextVar("feat_cache", default=None) +feat_idx = contextvars.ContextVar("feat_idx", default=0) +first_chunk = contextvars.ContextVar("first_chunk", default=None) + + +@contextmanager +def forward_context( + first_frame_arg=False, feat_cache_arg=None, feat_idx_arg=None, first_chunk_arg=None +): + is_first_frame_token = is_first_frame.set(first_frame_arg) + feat_cache_token = feat_cache.set(feat_cache_arg) + feat_idx_token = feat_idx.set(feat_idx_arg) + first_chunk_token = first_chunk.set(first_chunk_arg) + try: + yield + finally: + is_first_frame.reset(is_first_frame_token) + feat_cache.reset(feat_cache_token) + feat_idx.reset(feat_idx_token) + first_chunk.reset(first_chunk_token) + + +class AvgDown3D(nn.Module): + + def __init__( + self, + in_channels, + out_channels, + factor_t, + factor_s=1, + ): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.factor_t = factor_t + self.factor_s = factor_s + self.factor = self.factor_t * self.factor_s * self.factor_s + + assert in_channels * self.factor % out_channels == 0 + self.group_size = in_channels * self.factor // out_channels + + def forward(self, x: torch.Tensor) -> torch.Tensor: + pad_t = (self.factor_t - x.shape[2] % self.factor_t) % self.factor_t + pad = (0, 0, 0, 0, pad_t, 0) + x = F.pad(x, pad) + B, C, T, H, W = x.shape + x = x.view( + B, + C, + T // self.factor_t, + self.factor_t, + H // self.factor_s, + self.factor_s, + W // self.factor_s, + self.factor_s, + ) + x = x.permute(0, 1, 3, 5, 7, 2, 4, 6).contiguous() + x = x.view( + B, + C * self.factor, + T // self.factor_t, + H // self.factor_s, + W // self.factor_s, + ) + x = x.view( + B, + self.out_channels, + self.group_size, + T // self.factor_t, + H // self.factor_s, + W // self.factor_s, + ) + x = x.mean(dim=2) + return x + + +class DupUp3D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + factor_t, + factor_s=1, + ): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + + self.factor_t = factor_t + self.factor_s = factor_s + self.factor = self.factor_t * self.factor_s * self.factor_s + + assert out_channels * self.factor % in_channels == 0 + self.repeats = out_channels * self.factor // in_channels + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = x.repeat_interleave(self.repeats, dim=1) + x = x.view( + x.size(0), + self.out_channels, + self.factor_t, + self.factor_s, + self.factor_s, + x.size(2), + x.size(3), + x.size(4), + ) + x = x.permute(0, 1, 5, 2, 6, 3, 7, 4).contiguous() + x = x.view( + x.size(0), + self.out_channels, + x.size(2) * self.factor_t, + x.size(4) * self.factor_s, + x.size(6) * self.factor_s, + ) + + _first_chunk = first_chunk.get() + if _first_chunk: + x = x[:, :, self.factor_t - 1 :, :, :] + return x + + +class WanCausalConv3d(nn.Conv3d): + r""" + A custom 3D causal convolution layer with feature caching support. + + This layer extends the standard Conv3D layer by ensuring causality in the time dimension and handling feature + caching for efficient inference. + + Args: + in_channels (int): Number of channels in the input image + out_channels (int): Number of channels produced by the convolution + kernel_size (int or tuple): Size of the convolving kernel + stride (int or tuple, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to all three sides of the input. Default: 0 + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int | tuple[int, int, int], + stride: int | tuple[int, int, int] = 1, + padding: int | tuple[int, int, int] = 0, + ) -> None: + super().__init__( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + ) + self.padding: tuple[int, int, int] + # Set up causal padding + self._padding: tuple[int, ...] = ( + self.padding[2], + self.padding[2], + self.padding[1], + self.padding[1], + 2 * self.padding[0], + 0, + ) + self.padding = (0, 0, 0) + + def forward(self, x, cache_x=None): + padding = list(self._padding) + if cache_x is not None and self._padding[4] > 0: + cache_x = cache_x.to(x.device) + x = torch.cat([cache_x, x], dim=2) + padding[4] -= cache_x.shape[2] + x = F.pad(x, padding) + x = ( + x.to(self.weight.dtype) if current_platform.is_mps() else x + ) # casting needed for mps since amp isn't supported + return super().forward(x) + + +class WanRMS_norm(nn.Module): + r""" + A custom RMS normalization layer. + + Args: + dim (int): The number of dimensions to normalize over. + channel_first (bool, optional): Whether the input tensor has channels as the first dimension. + Default is True. + images (bool, optional): Whether the input represents image data. Default is True. + bias (bool, optional): Whether to include a learnable bias term. Default is False. + """ + + def __init__( + self, + dim: int, + channel_first: bool = True, + images: bool = True, + bias: bool = False, + ) -> None: + super().__init__() + broadcastable_dims = (1, 1, 1) if not images else (1, 1) + shape = (dim, *broadcastable_dims) if channel_first else (dim,) + + self.channel_first = channel_first + self.scale = dim**0.5 + self.gamma = nn.Parameter(torch.ones(shape)) + self.bias = nn.Parameter(torch.zeros(shape)) if bias else 0.0 + + def forward(self, x): + return ( + F.normalize(x, dim=(1 if self.channel_first else -1)) + * self.scale + * self.gamma + + self.bias + ) + + +class WanUpsample(nn.Upsample): + r""" + Perform upsampling while ensuring the output tensor has the same data type as the input. + + Args: + x (torch.Tensor): Input tensor to be upsampled. + + Returns: + torch.Tensor: Upsampled tensor with the same data type as the input. + """ + + def forward(self, x): + return super().forward(x.float()).type_as(x) + + +class WanResample(nn.Module): + r""" + A custom resampling module for 2D and 3D data. + + Args: + dim (int): The number of input/output channels. + mode (str): The resampling mode. Must be one of: + - 'none': No resampling (identity operation). + - 'upsample2d': 2D upsampling with nearest-exact interpolation and convolution. + - 'upsample3d': 3D upsampling with nearest-exact interpolation, convolution, and causal 3D convolution. + - 'downsample2d': 2D downsampling with zero-padding and convolution. + - 'downsample3d': 3D downsampling with zero-padding, convolution, and causal 3D convolution. + """ + + def __init__(self, dim: int, mode: str, upsample_out_dim: int = None) -> None: + super().__init__() + self.dim = dim + self.mode = mode + + # default to dim //2 + if upsample_out_dim is None: + upsample_out_dim = dim // 2 + + # layers + if mode == "upsample2d": + self.resample = nn.Sequential( + WanUpsample(scale_factor=(2.0, 2.0), mode="nearest-exact"), + nn.Conv2d(dim, upsample_out_dim, 3, padding=1), + ) + elif mode == "upsample3d": + self.resample = nn.Sequential( + WanUpsample(scale_factor=(2.0, 2.0), mode="nearest-exact"), + nn.Conv2d(dim, upsample_out_dim, 3, padding=1), + ) + self.time_conv = WanCausalConv3d(dim, dim * 2, (3, 1, 1), padding=(1, 0, 0)) + + elif mode == "downsample2d": + self.resample = nn.Sequential( + nn.ZeroPad2d((0, 1, 0, 1)), nn.Conv2d(dim, dim, 3, stride=(2, 2)) + ) + elif mode == "downsample3d": + self.resample = nn.Sequential( + nn.ZeroPad2d((0, 1, 0, 1)), nn.Conv2d(dim, dim, 3, stride=(2, 2)) + ) + self.time_conv = WanCausalConv3d( + dim, dim, (3, 1, 1), stride=(2, 1, 1), padding=(0, 0, 0) + ) + + else: + self.resample = nn.Identity() + + def forward(self, x): + b, c, t, h, w = x.size() + first_frame = is_first_frame.get() + if first_frame: + assert t == 1 + _feat_cache = feat_cache.get() + _feat_idx = feat_idx.get() + if self.mode == "upsample3d": + if _feat_cache is not None: + idx = _feat_idx + if _feat_cache[idx] is None: + _feat_cache[idx] = "Rep" + _feat_idx += 1 + else: + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if ( + cache_x.shape[2] < 2 + and _feat_cache[idx] is not None + and _feat_cache[idx] != "Rep" + ): + # cache last frame of last two chunk + cache_x = torch.cat( + [ + _feat_cache[idx][:, :, -1, :, :] + .unsqueeze(2) + .to(cache_x.device), + cache_x, + ], + dim=2, + ) + if ( + cache_x.shape[2] < 2 + and _feat_cache[idx] is not None + and _feat_cache[idx] == "Rep" + ): + cache_x = torch.cat( + [torch.zeros_like(cache_x).to(cache_x.device), cache_x], + dim=2, + ) + if _feat_cache[idx] == "Rep": + x = self.time_conv(x) + else: + x = self.time_conv(x, _feat_cache[idx]) + _feat_cache[idx] = cache_x + _feat_idx += 1 + + x = x.reshape(b, 2, c, t, h, w) + x = torch.stack((x[:, 0, :, :, :, :], x[:, 1, :, :, :, :]), 3) + x = x.reshape(b, c, t * 2, h, w) + feat_cache.set(_feat_cache) + feat_idx.set(_feat_idx) + elif not first_frame and hasattr(self, "time_conv"): + x = self.time_conv(x) + x = x.reshape(b, 2, c, t, h, w) + x = torch.stack((x[:, 0, :, :, :, :], x[:, 1, :, :, :, :]), 3) + x = x.reshape(b, c, t * 2, h, w) + t = x.shape[2] + x = x.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w) + x = self.resample(x) + x = x.view(b, t, x.size(1), x.size(2), x.size(3)).permute(0, 2, 1, 3, 4) + + _feat_cache = feat_cache.get() + _feat_idx = feat_idx.get() + if self.mode == "downsample3d": + if _feat_cache is not None: + idx = _feat_idx + if _feat_cache[idx] is None: + _feat_cache[idx] = x.clone() + _feat_idx += 1 + else: + cache_x = x[:, :, -1:, :, :].clone() + x = self.time_conv( + torch.cat([_feat_cache[idx][:, :, -1:, :, :], x], 2) + ) + _feat_cache[idx] = cache_x + _feat_idx += 1 + feat_cache.set(_feat_cache) + feat_idx.set(_feat_idx) + elif not first_frame and hasattr(self, "time_conv"): + x = self.time_conv(x) + return x + + +class WanResidualBlock(nn.Module): + r""" + A custom residual block module. + + Args: + in_dim (int): Number of input channels. + out_dim (int): Number of output channels. + dropout (float, optional): Dropout rate for the dropout layer. Default is 0.0. + non_linearity (str, optional): Type of non-linearity to use. Default is "silu". + """ + + def __init__( + self, + in_dim: int, + out_dim: int, + dropout: float = 0.0, + non_linearity: str = "silu", + ) -> None: + super().__init__() + self.in_dim = in_dim + self.out_dim = out_dim + self.nonlinearity = get_act_fn(non_linearity) + + # layers + self.norm1 = WanRMS_norm(in_dim, images=False) + self.conv1 = WanCausalConv3d(in_dim, out_dim, 3, padding=1) + self.norm2 = WanRMS_norm(out_dim, images=False) + self.dropout = nn.Dropout(dropout) + self.conv2 = WanCausalConv3d(out_dim, out_dim, 3, padding=1) + self.conv_shortcut = ( + WanCausalConv3d(in_dim, out_dim, 1) if in_dim != out_dim else nn.Identity() + ) + + def forward(self, x): + # Apply shortcut connection + h = self.conv_shortcut(x) + + # First normalization and activation + x = self.norm1(x) + x = self.nonlinearity(x) + + _feat_cache = feat_cache.get() + _feat_idx = feat_idx.get() + if _feat_cache is not None: + idx = _feat_idx + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and _feat_cache[idx] is not None: + cache_x = torch.cat( + [ + _feat_cache[idx][:, :, -1, :, :] + .unsqueeze(2) + .to(cache_x.device), + cache_x, + ], + dim=2, + ) + + x = self.conv1(x, _feat_cache[idx]) + _feat_cache[idx] = cache_x + _feat_idx += 1 + feat_cache.set(_feat_cache) + feat_idx.set(_feat_idx) + else: + x = self.conv1(x) + + # Second normalization and activation + x = self.norm2(x) + x = self.nonlinearity(x) + + # Dropout + x = self.dropout(x) + + _feat_cache = feat_cache.get() + _feat_idx = feat_idx.get() + if _feat_cache is not None: + idx = _feat_idx + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and _feat_cache[idx] is not None: + cache_x = torch.cat( + [ + _feat_cache[idx][:, :, -1, :, :] + .unsqueeze(2) + .to(cache_x.device), + cache_x, + ], + dim=2, + ) + + x = self.conv2(x, _feat_cache[idx]) + _feat_cache[idx] = cache_x + _feat_idx += 1 + feat_cache.set(_feat_cache) + feat_idx.set(_feat_idx) + else: + x = self.conv2(x) + + # Add residual connection + return x + h + + +class WanAttentionBlock(nn.Module): + r""" + Causal self-attention with a single head. + + Args: + dim (int): The number of channels in the input tensor. + """ + + def __init__(self, dim) -> None: + super().__init__() + self.dim = dim + + # layers + self.norm = WanRMS_norm(dim) + self.to_qkv = nn.Conv2d(dim, dim * 3, 1) + self.proj = nn.Conv2d(dim, dim, 1) + + def forward(self, x): + identity = x + batch_size, channels, time, height, width = x.size() + + x = x.permute(0, 2, 1, 3, 4).reshape(batch_size * time, channels, height, width) + x = self.norm(x) + + # compute query, key, value + qkv = self.to_qkv(x) + qkv = qkv.reshape(batch_size * time, 1, channels * 3, -1) + qkv = qkv.permute(0, 1, 3, 2).contiguous() + q, k, v = qkv.chunk(3, dim=-1) + + # apply attention + x = F.scaled_dot_product_attention(q, k, v) + + x = ( + x.squeeze(1) + .permute(0, 2, 1) + .reshape(batch_size * time, channels, height, width) + ) + + # output projection + x = self.proj(x) + + # Reshape back: [(b*t), c, h, w] -> [b, c, t, h, w] + x = x.view(batch_size, time, channels, height, width) + x = x.permute(0, 2, 1, 3, 4) + + return x + identity + + +class WanMidBlock(nn.Module): + """ + Middle block for WanVAE encoder and decoder. + + Args: + dim (int): Number of input/output channels. + dropout (float): Dropout rate. + non_linearity (str): Type of non-linearity to use. + """ + + def __init__( + self, + dim: int, + dropout: float = 0.0, + non_linearity: str = "silu", + num_layers: int = 1, + ): + super().__init__() + self.dim = dim + + # Create the components + resnets = [WanResidualBlock(dim, dim, dropout, non_linearity)] + attentions = [] + for _ in range(num_layers): + attentions.append(WanAttentionBlock(dim)) + resnets.append(WanResidualBlock(dim, dim, dropout, non_linearity)) + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + self.gradient_checkpointing = False + + def forward(self, x): + # First residual block + x = self.resnets[0](x) + + # Process through attention and residual blocks + for attn, resnet in zip(self.attentions, self.resnets[1:], strict=True): + if attn is not None: + x = attn(x) + + x = resnet(x) + + return x + + +class WanResidualDownBlock(nn.Module): + + def __init__( + self, + in_dim, + out_dim, + dropout, + num_res_blocks, + temperal_downsample=False, + down_flag=False, + ): + super().__init__() + + # Shortcut path with downsample + self.avg_shortcut = AvgDown3D( + in_dim, + out_dim, + factor_t=2 if temperal_downsample else 1, + factor_s=2 if down_flag else 1, + ) + + # Main path with residual blocks and downsample + resnets = [] + for _ in range(num_res_blocks): + resnets.append(WanResidualBlock(in_dim, out_dim, dropout)) + in_dim = out_dim + self.resnets = nn.ModuleList(resnets) + + # Add the final downsample block + if down_flag: + mode = "downsample3d" if temperal_downsample else "downsample2d" + self.downsampler = WanResample(out_dim, mode=mode) + else: + self.downsampler = None + + def forward(self, x): + x_copy = x.clone() + for resnet in self.resnets: + x = resnet(x) + if self.downsampler is not None: + x = self.downsampler(x) + + return x + self.avg_shortcut(x_copy) + + +class WanEncoder3d(nn.Module): + r""" + A 3D encoder module. + + Args: + dim (int): The base number of channels in the first layer. + z_dim (int): The dimensionality of the latent space. + dim_mult (list of int): Multipliers for the number of channels in each block. + num_res_blocks (int): Number of residual blocks in each block. + attn_scales (list of float): Scales at which to apply attention mechanisms. + temperal_downsample (list of bool): Whether to downsample temporally in each block. + dropout (float): Dropout rate for the dropout layers. + non_linearity (str): Type of non-linearity to use. + """ + + def __init__( + self, + in_channels: int = 3, + dim=128, + z_dim=4, + dim_mult=(1, 2, 4, 4), + num_res_blocks=2, + attn_scales=(), + temperal_downsample=(True, True, False), + dropout=0.0, + non_linearity: str = "silu", + is_residual: bool = False, # wan 2.2 vae use a residual downblock + ): + super().__init__() + self.dim = dim + self.z_dim = z_dim + dim_mult = list(dim_mult) + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = list(attn_scales) + self.temperal_downsample = list(temperal_downsample) + self.nonlinearity = get_act_fn(non_linearity) + + # dimensions + dims = [dim * u for u in [1] + dim_mult] + scale = 1.0 + + # init block + self.conv_in = WanCausalConv3d(in_channels, dims[0], 3, padding=1) + + # downsample blocks + self.down_blocks = nn.ModuleList([]) + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:], strict=True)): + # residual (+attention) blocks + if is_residual: + self.down_blocks.append( + WanResidualDownBlock( + in_dim, + out_dim, + dropout, + num_res_blocks, + temperal_downsample=( + temperal_downsample[i] if i != len(dim_mult) - 1 else False + ), + down_flag=i != len(dim_mult) - 1, + ) + ) + else: + for _ in range(num_res_blocks): + self.down_blocks.append(WanResidualBlock(in_dim, out_dim, dropout)) + if scale in attn_scales: + self.down_blocks.append(WanAttentionBlock(out_dim)) + in_dim = out_dim + + # downsample block + if i != len(dim_mult) - 1: + mode = "downsample3d" if temperal_downsample[i] else "downsample2d" + self.down_blocks.append(WanResample(out_dim, mode=mode)) + scale /= 2.0 + + # middle blocks + self.mid_block = WanMidBlock(out_dim, dropout, non_linearity, num_layers=1) + + # output blocks + self.norm_out = WanRMS_norm(out_dim, images=False) + self.conv_out = WanCausalConv3d(out_dim, z_dim, 3, padding=1) + + self.gradient_checkpointing = False + + def forward(self, x): + _feat_cache = feat_cache.get() + _feat_idx = feat_idx.get() + if _feat_cache is not None: + idx = _feat_idx + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and _feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat( + [ + _feat_cache[idx][:, :, -1, :, :] + .unsqueeze(2) + .to(cache_x.device), + cache_x, + ], + dim=2, + ) + x = self.conv_in(x, _feat_cache[idx]) + _feat_cache[idx] = cache_x + _feat_idx += 1 + feat_cache.set(_feat_cache) + feat_idx.set(_feat_idx) + else: + x = self.conv_in(x) + + ## downsamples + for layer in self.down_blocks: + x = layer(x) + + ## middle + x = self.mid_block(x) + + ## head + x = self.norm_out(x) + x = self.nonlinearity(x) + + _feat_cache = feat_cache.get() + _feat_idx = feat_idx.get() + if _feat_cache is not None: + idx = _feat_idx + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and _feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat( + [ + _feat_cache[idx][:, :, -1, :, :] + .unsqueeze(2) + .to(cache_x.device), + cache_x, + ], + dim=2, + ) + x = self.conv_out(x, _feat_cache[idx]) + _feat_cache[idx] = cache_x + _feat_idx += 1 + feat_cache.set(_feat_cache) + feat_idx.set(_feat_idx) + else: + x = self.conv_out(x) + return x + + +# adapted from: https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/autoencoders/autoencoder_kl_wan.py +class WanResidualUpBlock(nn.Module): + """ + A block that handles upsampling for the WanVAE decoder. + Args: + in_dim (int): Input dimension + out_dim (int): Output dimension + num_res_blocks (int): Number of residual blocks + dropout (float): Dropout rate + temperal_upsample (bool): Whether to upsample on temporal dimension + up_flag (bool): Whether to upsample or not + non_linearity (str): Type of non-linearity to use + """ + + def __init__( + self, + in_dim: int, + out_dim: int, + num_res_blocks: int, + dropout: float = 0.0, + temperal_upsample: bool = False, + up_flag: bool = False, + non_linearity: str = "silu", + ): + super().__init__() + self.in_dim = in_dim + self.out_dim = out_dim + + if up_flag: + self.avg_shortcut = DupUp3D( + in_dim, + out_dim, + factor_t=2 if temperal_upsample else 1, + factor_s=2, + ) + else: + self.avg_shortcut = None + + # create residual blocks + resnets = [] + current_dim = in_dim + for _ in range(num_res_blocks + 1): + resnets.append( + WanResidualBlock(current_dim, out_dim, dropout, non_linearity) + ) + current_dim = out_dim + + self.resnets = nn.ModuleList(resnets) + + # Add upsampling layer if needed + if up_flag: + upsample_mode = "upsample3d" if temperal_upsample else "upsample2d" + self.upsampler = WanResample( + out_dim, mode=upsample_mode, upsample_out_dim=out_dim + ) + else: + self.upsampler = None + + self.gradient_checkpointing = False + + def forward(self, x): + """ + Forward pass through the upsampling block. + Args: + x (torch.Tensor): Input tensor + feat_cache (list, optional): Feature cache for causal convolutions + feat_idx (list, optional): Feature index for cache management + Returns: + torch.Tensor: Output tensor + """ + if self.avg_shortcut is not None: + x_copy = x.clone() + + for resnet in self.resnets: + x = resnet(x) + + if self.upsampler is not None: + x = self.upsampler(x) + + if self.avg_shortcut is not None: + x = x + self.avg_shortcut(x_copy) + + return x + + +class WanUpBlock(nn.Module): + """ + A block that handles upsampling for the WanVAE decoder. + + Args: + in_dim (int): Input dimension + out_dim (int): Output dimension + num_res_blocks (int): Number of residual blocks + dropout (float): Dropout rate + upsample_mode (str, optional): Mode for upsampling ('upsample2d' or 'upsample3d') + non_linearity (str): Type of non-linearity to use + """ + + def __init__( + self, + in_dim: int, + out_dim: int, + num_res_blocks: int, + dropout: float = 0.0, + upsample_mode: str | None = None, + non_linearity: str = "silu", + ): + super().__init__() + self.in_dim = in_dim + self.out_dim = out_dim + + # Create layers list + resnets = [] + # Add residual blocks and attention if needed + current_dim = in_dim + for _ in range(num_res_blocks + 1): + resnets.append( + WanResidualBlock(current_dim, out_dim, dropout, non_linearity) + ) + current_dim = out_dim + + self.resnets = nn.ModuleList(resnets) + + # Add upsampling layer if needed + self.upsamplers = None + if upsample_mode is not None: + self.upsamplers = nn.ModuleList([WanResample(out_dim, mode=upsample_mode)]) + + self.gradient_checkpointing = False + + def forward(self, x): + """ + Forward pass through the upsampling block. + + Args: + x (torch.Tensor): Input tensor + feat_cache (list, optional): Feature cache for causal convolutions + feat_idx (list, optional): Feature index for cache management + + Returns: + torch.Tensor: Output tensor + """ + for resnet in self.resnets: + x = resnet(x) + + if self.upsamplers is not None: + x = self.upsamplers[0](x) + return x + + +class WanDecoder3d(nn.Module): + r""" + A 3D decoder module. + + Args: + dim (int): The base number of channels in the first layer. + z_dim (int): The dimensionality of the latent space. + dim_mult (list of int): Multipliers for the number of channels in each block. + num_res_blocks (int): Number of residual blocks in each block. + attn_scales (list of float): Scales at which to apply attention mechanisms. + temperal_upsample (list of bool): Whether to upsample temporally in each block. + dropout (float): Dropout rate for the dropout layers. + non_linearity (str): Type of non-linearity to use. + """ + + def __init__( + self, + dim=128, + z_dim=4, + dim_mult=(1, 2, 4, 4), + num_res_blocks=2, + attn_scales=(), + temperal_upsample=(False, True, True), + dropout=0.0, + non_linearity: str = "silu", + out_channels: int = 3, + is_residual: bool = False, + ): + super().__init__() + self.dim = dim + self.z_dim = z_dim + dim_mult = list(dim_mult) + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = list(attn_scales) + self.temperal_upsample = list(temperal_upsample) + + self.nonlinearity = get_act_fn(non_linearity) + + # dimensions + dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + + # init block + self.conv_in = WanCausalConv3d(z_dim, dims[0], 3, padding=1) + + # middle blocks + self.mid_block = WanMidBlock(dims[0], dropout, non_linearity, num_layers=1) + + # upsample blocks + self.up_blocks = nn.ModuleList([]) + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:], strict=True)): + # residual (+attention) blocks + if i > 0 and not is_residual: + # wan vae 2.1 + in_dim = in_dim // 2 + + # determine if we need upsampling + up_flag = i != len(dim_mult) - 1 + # determine upsampling mode, if not upsampling, set to None + upsample_mode = None + if up_flag and temperal_upsample[i]: + upsample_mode = "upsample3d" + elif up_flag: + upsample_mode = "upsample2d" + + # Create and add the upsampling block + if is_residual: + up_block = WanResidualUpBlock( + in_dim=in_dim, + out_dim=out_dim, + num_res_blocks=num_res_blocks, + dropout=dropout, + temperal_upsample=temperal_upsample[i] if up_flag else False, + up_flag=up_flag, + non_linearity=non_linearity, + ) + else: + up_block = WanUpBlock( + in_dim=in_dim, + out_dim=out_dim, + num_res_blocks=num_res_blocks, + dropout=dropout, + upsample_mode=upsample_mode, + non_linearity=non_linearity, + ) + self.up_blocks.append(up_block) + + # output blocks + self.norm_out = WanRMS_norm(out_dim, images=False) + self.conv_out = WanCausalConv3d(out_dim, out_channels, 3, padding=1) + + self.gradient_checkpointing = False + + def forward(self, x): + ## conv1 + _feat_cache = feat_cache.get() + _feat_idx = feat_idx.get() + if _feat_cache is not None: + idx = _feat_idx + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and _feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat( + [ + _feat_cache[idx][:, :, -1, :, :] + .unsqueeze(2) + .to(cache_x.device), + cache_x, + ], + dim=2, + ) + x = self.conv_in(x, _feat_cache[idx]) + _feat_cache[idx] = cache_x + _feat_idx += 1 + feat_cache.set(_feat_cache) + feat_idx.set(_feat_idx) + else: + x = self.conv_in(x) + + ## middle + x = self.mid_block(x) + + ## upsamples + for up_block in self.up_blocks: + x = up_block(x) + + ## head + x = self.norm_out(x) + x = self.nonlinearity(x) + _feat_cache = feat_cache.get() + _feat_idx = feat_idx.get() + if _feat_cache is not None: + idx = _feat_idx + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and _feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat( + [ + _feat_cache[idx][:, :, -1, :, :] + .unsqueeze(2) + .to(cache_x.device), + cache_x, + ], + dim=2, + ) + x = self.conv_out(x, _feat_cache[idx]) + _feat_cache[idx] = cache_x + _feat_idx += 1 + feat_cache.set(_feat_cache) + feat_idx.set(_feat_idx) + else: + x = self.conv_out(x) + return x + + +def patchify(x, patch_size): + if patch_size == 1: + return x + + if x.dim() == 4: + x = rearrange(x, "b c (h q) (w r) -> b (c r q) h w", q=patch_size, r=patch_size) + elif x.dim() == 5: + x = rearrange( + x, + "b c f (h q) (w r) -> b (c r q) f h w", + q=patch_size, + r=patch_size, + ) + else: + raise ValueError(f"Invalid input shape: {x.shape}") + + return x + + +def unpatchify(x, patch_size): + if patch_size == 1: + return x + + if x.dim() == 4: + x = rearrange(x, "b (c r q) h w -> b c (h q) (w r)", q=patch_size, r=patch_size) + elif x.dim() == 5: + x = rearrange( + x, + "b (c r q) f h w -> b c f (h q) (w r)", + q=patch_size, + r=patch_size, + ) + + return x + + +class AutoencoderKLWan(nn.Module, ParallelTiledVAE): + r""" + A VAE model with KL loss for encoding videos into latents and decoding latent representations into videos. + Introduced in [Wan 2.1]. + """ + + _supports_gradient_checkpointing = False + + def __init__( + self, + config: WanVAEConfig, + ) -> None: + nn.Module.__init__(self) + ParallelTiledVAE.__init__(self, config) + + self.z_dim = config.z_dim + self.temperal_downsample = list(config.temperal_downsample) + self.temperal_upsample = list(config.temperal_downsample)[::-1] + + if config.decoder_base_dim is None: + decoder_base_dim = config.base_dim + else: + decoder_base_dim = config.decoder_base_dim + + self.latents_mean = list(config.latents_mean) + self.latents_std = list(config.latents_std) + self.shift_factor = config.shift_factor + + if config.load_encoder: + self.encoder = WanEncoder3d( + in_channels=config.in_channels, + dim=config.base_dim, + z_dim=self.z_dim * 2, + dim_mult=config.dim_mult, + num_res_blocks=config.num_res_blocks, + attn_scales=config.attn_scales, + temperal_downsample=self.temperal_downsample, + dropout=config.dropout, + is_residual=config.is_residual, + ) + self.quant_conv = WanCausalConv3d(self.z_dim * 2, self.z_dim * 2, 1) + self.post_quant_conv = WanCausalConv3d(self.z_dim, self.z_dim, 1) + + if config.load_decoder: + self.decoder = WanDecoder3d( + dim=decoder_base_dim, + z_dim=self.z_dim, + dim_mult=config.dim_mult, + num_res_blocks=config.num_res_blocks, + attn_scales=config.attn_scales, + temperal_upsample=self.temperal_upsample, + dropout=config.dropout, + out_channels=config.out_channels, + is_residual=config.is_residual, + ) + + self.use_feature_cache = config.use_feature_cache + + def clear_cache(self) -> None: + + def _count_conv3d(model) -> int: + count = 0 + for m in model.modules(): + if isinstance(m, WanCausalConv3d): + count += 1 + return count + + if self.config.load_decoder: + self._conv_num = _count_conv3d(self.decoder) + self._conv_idx = 0 + self._feat_map = [None] * self._conv_num + # cache encode + if self.config.load_encoder: + self._enc_conv_num = _count_conv3d(self.encoder) + self._enc_conv_idx = 0 + self._enc_feat_map = [None] * self._enc_conv_num + + def encode(self, x: torch.Tensor) -> torch.Tensor: + if self.use_feature_cache: + self.clear_cache() + if self.config.patch_size is not None: + x = patchify(x, patch_size=self.config.patch_size) + with forward_context( + feat_cache_arg=self._enc_feat_map, feat_idx_arg=self._enc_conv_idx + ): + t = x.shape[2] + iter_ = 1 + (t - 1) // 4 + for i in range(iter_): + feat_idx.set(0) + if i == 0: + out = self.encoder(x[:, :, :1, :, :]) + else: + out_ = self.encoder(x[:, :, 1 + 4 * (i - 1) : 1 + 4 * i, :, :]) + out = torch.cat([out, out_], 2) + enc = self.quant_conv(out) + mu, logvar = enc[:, : self.z_dim, :, :, :], enc[:, self.z_dim :, :, :, :] + enc = torch.cat([mu, logvar], dim=1) + enc = DiagonalGaussianDistribution(enc) + self.clear_cache() + else: + for block in self.encoder.down_blocks: + if isinstance(block, WanResample) and block.mode == "downsample3d": + _padding = list(block.time_conv._padding) + _padding[4] = 2 + block.time_conv._padding = tuple(_padding) + enc = ParallelTiledVAE.encode(self, x) + + return enc + + def _encode(self, x: torch.Tensor, first_frame=False) -> torch.Tensor: + with forward_context(first_frame_arg=first_frame): + out = self.encoder(x) + enc = self.quant_conv(out) + mu, logvar = enc[:, : self.z_dim, :, :, :], enc[:, self.z_dim :, :, :, :] + enc = torch.cat([mu, logvar], dim=1) + return enc + + def tiled_encode(self, x: torch.Tensor) -> torch.Tensor: + first_frame = x[:, :, 0, :, :].unsqueeze(2) + first_frame = self._encode(first_frame, first_frame=True) + + enc = ParallelTiledVAE.tiled_encode(self, x) + enc = enc[:, :, 1:] + enc = torch.cat([first_frame, enc], dim=2) + return enc + + def spatial_tiled_encode(self, x: torch.Tensor) -> torch.Tensor: + first_frame = x[:, :, 0, :, :].unsqueeze(2) + first_frame = self._encode(first_frame, first_frame=True) + + enc = ParallelTiledVAE.spatial_tiled_encode(self, x) + enc = enc[:, :, 1:] + enc = torch.cat([first_frame, enc], dim=2) + return enc + + def decode(self, z: torch.Tensor) -> torch.Tensor: + if self.use_feature_cache: + self.clear_cache() + iter_ = z.shape[2] + x = self.post_quant_conv(z) + with forward_context( + feat_cache_arg=self._feat_map, feat_idx_arg=self._conv_idx + ): + for i in range(iter_): + feat_idx.set(0) + if i == 0: + first_chunk.set(True) + out = self.decoder(x[:, :, i : i + 1, :, :]) + else: + first_chunk.set(False) + out_ = self.decoder(x[:, :, i : i + 1, :, :]) + out = torch.cat([out, out_], 2) + + if self.config.patch_size is not None: + out = unpatchify(out, patch_size=self.config.patch_size) + + out = out.float() + out = torch.clamp(out, min=-1.0, max=1.0) + self.clear_cache() + else: + out = ParallelTiledVAE.decode(self, z) + + return out + + def _decode(self, z: torch.Tensor, first_frame=False) -> torch.Tensor: + x = self.post_quant_conv(z) + with forward_context(first_frame_arg=first_frame): + out = self.decoder(x) + + out = torch.clamp(out, min=-1.0, max=1.0) + + return out + + def tiled_decode(self, z: torch.Tensor) -> torch.Tensor: + self.blend_num_frames *= 2 + dec = ParallelTiledVAE.tiled_decode(self, z) + start_frame_idx = self.temporal_compression_ratio - 1 + dec = dec[:, :, start_frame_idx:] + return dec + + def spatial_tiled_decode(self, z: torch.Tensor) -> torch.Tensor: + dec = ParallelTiledVAE.spatial_tiled_decode(self, z) + start_frame_idx = self.temporal_compression_ratio - 1 + dec = dec[:, :, start_frame_idx:] + return dec + + def parallel_tiled_decode(self, z: torch.FloatTensor) -> torch.FloatTensor: + self.blend_num_frames *= 2 + dec = ParallelTiledVAE.parallel_tiled_decode(self, z) + start_frame_idx = self.temporal_compression_ratio - 1 + dec = dec[:, :, start_frame_idx:] + return dec + + def forward( + self, + sample: torch.Tensor, + sample_posterior: bool = False, + generator: torch.Generator | None = None, + ) -> torch.Tensor: + """ + Args: + sample (`torch.Tensor`): Input sample. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`DecoderOutput`] instead of a plain tuple. + """ + x = sample + posterior = self.encode(x).latent_dist + if sample_posterior: + z = posterior.sample(generator=generator) + else: + z = posterior.mode() + dec = self.decode(z) + return dec + + +EntryClass = AutoencoderKLWan diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vision_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vision_utils.py new file mode 100644 index 00000000..c50eefc2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/models/vision_utils.py @@ -0,0 +1,304 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +import os +import tempfile +from collections.abc import Callable +from urllib.parse import unquote, urlparse + +import imageio +import numpy as np +import PIL.Image +import PIL.ImageOps +import requests +import torch +from packaging import version + +from sglang.multimodal_gen.runtime.utils.logging_utils import suppress_other_loggers + +if version.parse(version.parse(PIL.__version__).base_version) >= version.parse("9.1.0"): + PIL_INTERPOLATION = { + "linear": PIL.Image.Resampling.BILINEAR, + "bilinear": PIL.Image.Resampling.BILINEAR, + "bicubic": PIL.Image.Resampling.BICUBIC, + "lanczos": PIL.Image.Resampling.LANCZOS, + "nearest": PIL.Image.Resampling.NEAREST, + } +else: + PIL_INTERPOLATION = { + "linear": PIL.Image.LINEAR, + "bilinear": PIL.Image.BILINEAR, + "bicubic": PIL.Image.BICUBIC, + "lanczos": PIL.Image.LANCZOS, + "nearest": PIL.Image.NEAREST, + } + + +def pil_to_numpy(images: list[PIL.Image.Image] | PIL.Image.Image) -> np.ndarray: + r""" + Convert a PIL image or a list of PIL images to NumPy arrays. + + Args: + images (`PIL.Image.Image` or `List[PIL.Image.Image]`): + The PIL image or list of images to convert to NumPy format. + + Returns: + `np.ndarray`: + A NumPy array representation of the images. + """ + if not isinstance(images, list): + images = [images] + images = [np.array(image).astype(np.float32) / 255.0 for image in images] + images_arr: np.ndarray = np.stack(images, axis=0) + + return images_arr + + +def numpy_to_pt(images: np.ndarray) -> torch.Tensor: + r""" + Convert a NumPy image to a PyTorch tensor. + + Args: + images (`np.ndarray`): + The NumPy image array to convert to PyTorch format. + + Returns: + `torch.Tensor`: + A PyTorch tensor representation of the images. + """ + if images.ndim == 3: + images = images[..., None] + + images = torch.from_numpy(images.transpose(0, 3, 1, 2)) + return images + + +def normalize(images: np.ndarray | torch.Tensor) -> np.ndarray | torch.Tensor: + r""" + Normalize an image array to [-1,1]. + + Args: + images (`np.ndarray` or `torch.Tensor`): + The image array to normalize. + + Returns: + `np.ndarray` or `torch.Tensor`: + The normalized image array. + """ + return 2.0 * images - 1.0 + + +# adapted from diffusers.utils import load_image +def load_image( + image: str | PIL.Image.Image, + convert_method: Callable[[PIL.Image.Image], PIL.Image.Image] | None = None, +) -> PIL.Image.Image: + """ + Loads `image` to a PIL Image. + + Args: + image (`str` or `PIL.Image.Image`): + The image to convert to the PIL Image format. + convert_method (Callable[[PIL.Image.Image], PIL.Image.Image], *optional*): + A conversion method to apply to the image after loading it. When set to `None` the image will be converted + "RGB". + + Returns: + `PIL.Image.Image`: + A PIL Image. + """ + if isinstance(image, str): + if image.startswith("http://") or image.startswith("https://"): + with suppress_other_loggers(not_suppress_on_main_rank=True): + image = PIL.Image.open(requests.get(image, stream=True).raw) + elif os.path.isfile(image): + image = PIL.Image.open(image) + else: + raise ValueError( + f"Incorrect path or URL. URLs must start with `http://` or `https://`, and {image} is not a valid path." + ) + elif isinstance(image, PIL.Image.Image): + image = image + else: + raise ValueError( + "Incorrect format used for the image. Should be a URL linking to an image, a local path, or a PIL image." + ) + + image = PIL.ImageOps.exif_transpose(image) + + if convert_method is not None: + image = convert_method(image) + else: + image = image.convert("RGB") + + return image + + +# adapted from diffusers.utils import load_video +def load_video( + video: str, + convert_method: ( + Callable[[list[PIL.Image.Image]], list[PIL.Image.Image]] | None + ) = None, +) -> list[PIL.Image.Image]: + """ + Loads `video` to a list of PIL Image. + Args: + video (`str`): + A URL or Path to a video to convert to a list of PIL Image format. + convert_method (Callable[[List[PIL.Image.Image]], List[PIL.Image.Image]], *optional*): + A conversion method to apply to the video after loading it. When set to `None` the images will be converted + to "RGB". + Returns: + `List[PIL.Image.Image]`: + The video as a list of PIL images. + """ + is_url = video.startswith("http://") or video.startswith("https://") + is_file = os.path.isfile(video) + was_tempfile_created = False + + if not (is_url or is_file): + raise ValueError( + f"Incorrect path or URL. URLs must start with `http://` or `https://`, and {video} is not a valid path." + ) + + if is_url: + response = requests.get(video, stream=True) + if response.status_code != 200: + raise ValueError( + f"Failed to download video. Status code: {response.status_code}" + ) + + parsed_url = urlparse(video) + file_name = os.path.basename(unquote(parsed_url.path)) + + suffix = os.path.splitext(file_name)[1] or ".mp4" + with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as temp_file: + video_path = temp_file.name + video_data = response.iter_content(chunk_size=8192) + for chunk in video_data: + temp_file.write(chunk) + + video = video_path + + pil_images = [] + if video.endswith(".gif"): + gif = PIL.Image.open(video) + try: + while True: + pil_images.append(gif.copy()) + gif.seek(gif.tell() + 1) + except EOFError: + pass + + else: + try: + imageio.plugins.ffmpeg.get_exe() + except AttributeError: + raise AttributeError( + "`Unable to find an ffmpeg installation on your machine. Please install via `pip install imageio-ffmpeg" + ) from None + + with imageio.get_reader(video) as reader: + # Read all frames + for frame in reader: + pil_images.append(PIL.Image.fromarray(frame)) + + if was_tempfile_created: + os.remove(video_path) + + if convert_method is not None: + pil_images = convert_method(pil_images) + + return pil_images + + +def get_default_height_width( + image: PIL.Image.Image | np.ndarray | torch.Tensor, + vae_scale_factor: int, + height: int | None = None, + width: int | None = None, +) -> tuple[int, int]: + r""" + Returns the height and width of the image, downscaled to the next integer multiple of `vae_scale_factor`. + + Args: + image (`Union[PIL.Image.Image, np.ndarray, torch.Tensor]`): + The image input, which can be a PIL image, NumPy array, or PyTorch tensor. If it is a NumPy array, it + should have shape `[batch, height, width]` or `[batch, height, width, channels]`. If it is a PyTorch + tensor, it should have shape `[batch, channels, height, width]`. + height (`Optional[int]`, *optional*, defaults to `None`): + The height of the preprocessed image. If `None`, the height of the `image` input will be used. + width (`Optional[int]`, *optional*, defaults to `None`): + The width of the preprocessed image. If `None`, the width of the `image` input will be used. + + Returns: + `Tuple[int, int]`: + A tuple containing the height and width, both resized to the nearest integer multiple of + `vae_scale_factor`. + """ + + if height is None: + if isinstance(image, PIL.Image.Image): + height = image.height + elif isinstance(image, torch.Tensor): + height = image.shape[2] + else: + height = image.shape[1] + + if width is None: + if isinstance(image, PIL.Image.Image): + width = image.width + elif isinstance(image, torch.Tensor): + width = image.shape[3] + else: + width = image.shape[2] + + width, height = ( + x - x % vae_scale_factor for x in (width, height) + ) # resize to integer multiple of vae_scale_factor + + return height, width + + +def resize( + image: PIL.Image.Image | np.ndarray | torch.Tensor, + height: int, + width: int, + resize_mode: str = "default", # "default", "fill", "crop" + resample: str = "lanczos", +) -> PIL.Image.Image | np.ndarray | torch.Tensor: + """ + Resize image. + + Args: + image (`PIL.Image.Image`, `np.ndarray` or `torch.Tensor`): + The image input, can be a PIL image, numpy array or pytorch tensor. + height (`int`): + The height to resize to. + width (`int`): + The width to resize to. + resize_mode (`str`, *optional*, defaults to `default`): + The resize mode to use, can be one of `default` or `fill`. If `default`, will resize the image to fit + within the specified width and height, and it may not maintaining the original aspect ratio. If `fill`, + will resize the image to fit within the specified width and height, maintaining the aspect ratio, and + then center the image within the dimensions, filling empty with data from image. If `crop`, will resize + the image to fit within the specified width and height, maintaining the aspect ratio, and then center + the image within the dimensions, cropping the excess. Note that resize_mode `fill` and `crop` are only + supported for PIL image input. + + Returns: + `PIL.Image.Image`, `np.ndarray` or `torch.Tensor`: + The resized image. + """ + if resize_mode != "default" and not isinstance(image, PIL.Image.Image): + raise ValueError( + f"Only PIL image input is supported for resize_mode {resize_mode}" + ) + assert isinstance(image, PIL.Image.Image) + if resize_mode == "default": + image = image.resize((width, height), resample=PIL_INTERPOLATION[resample]) + else: + raise ValueError(f"resize_mode {resize_mode} is not supported") + return image diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/__init__.py new file mode 100644 index 00000000..af2eb7d1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/__init__.py @@ -0,0 +1 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/flux.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/flux.py new file mode 100644 index 00000000..d738c8ba --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/flux.py @@ -0,0 +1,124 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +from sglang.multimodal_gen.runtime.pipelines_core import LoRAPipeline +from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( + ComposedPipelineBase, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages import ( + ConditioningStage, + DecodingStage, + DenoisingStage, + InputValidationStage, + LatentPreparationStage, + TextEncodingStage, + TimestepPreparationStage, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +# TODO(will): move PRECISION_TO_TYPE to better place + +logger = init_logger(__name__) + + +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + + +def prepare_mu(batch: Req, server_args: ServerArgs): + height = batch.height + width = batch.width + vae_scale_factor = ( + server_args.pipeline_config.vae_config.arch_config.vae_scale_factor + ) + image_seq_len = (int(height) // vae_scale_factor) * (int(width) // vae_scale_factor) + + mu = calculate_shift( + image_seq_len, + # hard code, since scheduler_config is not in PipelineConfig now + 256, + 4096, + 0.5, + 1.15, + ) + return "mu", mu + + +class FluxPipeline(LoRAPipeline, ComposedPipelineBase): + pipeline_name = "FluxPipeline" + + _required_config_modules = [ + "text_encoder", + "text_encoder_2", + "tokenizer", + "tokenizer_2", + "vae", + "transformer", + "scheduler", + ] + + def create_pipeline_stages(self, server_args: ServerArgs): + """Set up pipeline stages with proper dependency injection.""" + + self.add_stage( + stage_name="input_validation_stage", stage=InputValidationStage() + ) + + self.add_stage( + stage_name="prompt_encoding_stage_primary", + stage=TextEncodingStage( + text_encoders=[ + self.get_module("text_encoder"), + self.get_module("text_encoder_2"), + ], + tokenizers=[ + self.get_module("tokenizer"), + self.get_module("tokenizer_2"), + ], + ), + ) + + self.add_stage(stage_name="conditioning_stage", stage=ConditioningStage()) + + self.add_stage( + stage_name="timestep_preparation_stage", + stage=TimestepPreparationStage( + scheduler=self.get_module("scheduler"), + prepare_extra_set_timesteps_kwargs=[prepare_mu], + ), + ) + + self.add_stage( + stage_name="latent_preparation_stage", + stage=LatentPreparationStage( + scheduler=self.get_module("scheduler"), + transformer=self.get_module("transformer"), + ), + ) + + self.add_stage( + stage_name="denoising_stage", + stage=DenoisingStage( + transformer=self.get_module("transformer"), + scheduler=self.get_module("scheduler"), + ), + ) + + self.add_stage( + stage_name="decoding_stage", stage=DecodingStage(vae=self.get_module("vae")) + ) + + +EntryClass = FluxPipeline diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/flux_2.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/flux_2.py new file mode 100644 index 00000000..58ce257a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/flux_2.py @@ -0,0 +1,124 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo +# SPDX-License-Identifier: Apache-2.0 + +from diffusers.image_processor import VaeImageProcessor + +from sglang.multimodal_gen.runtime.pipelines_core import LoRAPipeline, Req +from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( + ComposedPipelineBase, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages import ( + ConditioningStage, + DecodingStage, + DenoisingStage, + ImageVAEEncodingStage, + InputValidationStage, + LatentPreparationStage, + TextEncodingStage, + TimestepPreparationStage, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +def compute_empirical_mu(batch: Req, server_args: ServerArgs): + num_steps = batch.num_inference_steps + image_seq_len = batch.raw_latent_shape[1] + a1, b1 = 8.73809524e-05, 1.89833333 + a2, b2 = 0.00016927, 0.45666666 + + if image_seq_len > 4300: + mu = a2 * image_seq_len + b2 + return "mu", float(mu) + + m_200 = a2 * image_seq_len + b2 + m_10 = a1 * image_seq_len + b1 + + a = (m_200 - m_10) / 190.0 + b = m_200 - 200.0 * a + mu = a * num_steps + b + + return "mu", float(mu) + + +class Flux2Pipeline(LoRAPipeline, ComposedPipelineBase): + pipeline_name = "Flux2Pipeline" + + _required_config_modules = [ + "text_encoder", + "tokenizer", + "vae", + "transformer", + "scheduler", + ] + + def create_pipeline_stages(self, server_args: ServerArgs): + """Set up pipeline stages with proper dependency injection.""" + + self.add_stage( + stage_name="input_validation_stage", + stage=InputValidationStage( + vae_image_processor=VaeImageProcessor( + vae_scale_factor=server_args.pipeline_config.vae_config.arch_config.vae_scale_factor + * 2 + ), + ), + ) + + self.add_stage( + stage_name="prompt_encoding_stage_primary", + stage=TextEncodingStage( + text_encoders=[ + self.get_module("text_encoder"), + ], + tokenizers=[ + self.get_module("tokenizer"), + ], + ), + ) + + self.add_stage( + stage_name="image_encoding_stage_primary", + stage=ImageVAEEncodingStage( + vae_image_processor=VaeImageProcessor( + vae_scale_factor=server_args.pipeline_config.vae_config.arch_config.vae_scale_factor + * 2 + ), + vae=self.get_module("vae"), + ), + ) + + self.add_stage(stage_name="conditioning_stage", stage=ConditioningStage()) + + self.add_stage( + stage_name="latent_preparation_stage", + stage=LatentPreparationStage( + scheduler=self.get_module("scheduler"), + transformer=self.get_module("transformer"), + ), + ) + + self.add_stage( + stage_name="timestep_preparation_stage", + stage=TimestepPreparationStage( + scheduler=self.get_module("scheduler"), + prepare_extra_set_timesteps_kwargs=[compute_empirical_mu], + ), + ) + + self.add_stage( + stage_name="denoising_stage", + stage=DenoisingStage( + transformer=self.get_module("transformer"), + scheduler=self.get_module("scheduler"), + ), + ) + + self.add_stage( + stage_name="decoding_stage", stage=DecodingStage(vae=self.get_module("vae")) + ) + + +EntryClass = Flux2Pipeline diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/hunyuan_pipeline.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/hunyuan_pipeline.py new file mode 100644 index 00000000..0be68fc4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/hunyuan_pipeline.py @@ -0,0 +1,95 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +Hunyuan video diffusion pipeline implementation. + +This module contains an implementation of the Hunyuan video diffusion pipeline +using the modular pipeline architecture. +""" + + +from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( + ComposedPipelineBase, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages import ( + ConditioningStage, + DecodingStage, + DenoisingStage, + InputValidationStage, + LatentPreparationStage, + TextEncodingStage, + TimestepPreparationStage, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +# TODO(will): move PRECISION_TO_TYPE to better place + +logger = init_logger(__name__) + + +class HunyuanVideoPipeline(ComposedPipelineBase): + + pipeline_name = "HunyuanVideoPipeline" + + _required_config_modules = [ + "text_encoder", + "text_encoder_2", + "tokenizer", + "tokenizer_2", + "vae", + "transformer", + "scheduler", + ] + + def create_pipeline_stages(self, server_args: ServerArgs): + """Set up pipeline stages with proper dependency injection.""" + + self.add_stage( + stage_name="input_validation_stage", stage=InputValidationStage() + ) + + self.add_stage( + stage_name="prompt_encoding_stage_primary", + stage=TextEncodingStage( + text_encoders=[ + self.get_module("text_encoder"), + self.get_module("text_encoder_2"), + ], + tokenizers=[ + self.get_module("tokenizer"), + self.get_module("tokenizer_2"), + ], + ), + ) + + self.add_stage(stage_name="conditioning_stage", stage=ConditioningStage()) + + self.add_stage( + stage_name="timestep_preparation_stage", + stage=TimestepPreparationStage(scheduler=self.get_module("scheduler")), + ) + + self.add_stage( + stage_name="latent_preparation_stage", + stage=LatentPreparationStage( + scheduler=self.get_module("scheduler"), + transformer=self.get_module("transformer"), + ), + ) + + self.add_stage( + stage_name="denoising_stage", + stage=DenoisingStage( + transformer=self.get_module("transformer"), + scheduler=self.get_module("scheduler"), + ), + ) + + self.add_stage( + stage_name="decoding_stage", stage=DecodingStage(vae=self.get_module("vae")) + ) + + +EntryClass = HunyuanVideoPipeline diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/qwen_image.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/qwen_image.py new file mode 100644 index 00000000..fabaae54 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/qwen_image.py @@ -0,0 +1,187 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +from sglang.multimodal_gen.runtime.pipelines_core import LoRAPipeline +from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( + ComposedPipelineBase, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages import ( + DecodingStage, + DenoisingStage, + ImageEncodingStage, + ImageVAEEncodingStage, + InputValidationStage, + LatentPreparationStage, + TextEncodingStage, + TimestepPreparationStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.conditioning import ( + ConditioningStage, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +# TODO(will): move PRECISION_TO_TYPE to better place + +logger = init_logger(__name__) + + +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + + +def prepare_mu(batch: Req, server_args: ServerArgs): + height = batch.height + width = batch.width + vae_scale_factor = server_args.pipeline_config.vae_config.vae_scale_factor + image_seq_len = (int(height) // vae_scale_factor) * (int(width) // vae_scale_factor) + + mu = calculate_shift( + image_seq_len, + # hard code, since scheduler_config is not in PipelineConfig now + 256, + 4096, + 0.5, + 1.15, + ) + return "mu", mu + + +class QwenImagePipeline(LoRAPipeline, ComposedPipelineBase): + pipeline_name = "QwenImagePipeline" + + _required_config_modules = [ + "text_encoder", + "tokenizer", + "vae", + "transformer", + "scheduler", + ] + + def create_pipeline_stages(self, server_args: ServerArgs): + """Set up pipeline stages with proper dependency injection.""" + + self.add_stage( + stage_name="input_validation_stage", stage=InputValidationStage() + ) + + self.add_stage( + stage_name="prompt_encoding_stage_primary", + stage=TextEncodingStage( + text_encoders=[ + self.get_module("text_encoder"), + ], + tokenizers=[ + self.get_module("tokenizer"), + ], + ), + ) + + self.add_stage(stage_name="conditioning_stage", stage=ConditioningStage()) + + self.add_stage( + stage_name="timestep_preparation_stage", + stage=TimestepPreparationStage( + scheduler=self.get_module("scheduler"), + prepare_extra_set_timesteps_kwargs=[prepare_mu], + ), + ) + + self.add_stage( + stage_name="latent_preparation_stage", + stage=LatentPreparationStage( + scheduler=self.get_module("scheduler"), + transformer=self.get_module("transformer"), + ), + ) + + self.add_stage( + stage_name="denoising_stage", + stage=DenoisingStage( + transformer=self.get_module("transformer"), + scheduler=self.get_module("scheduler"), + ), + ) + + self.add_stage( + stage_name="decoding_stage", stage=DecodingStage(vae=self.get_module("vae")) + ) + + +class QwenImageEditPipeline(LoRAPipeline, ComposedPipelineBase): + pipeline_name = "QwenImageEditPipeline" + + _required_config_modules = [ + "processor", + "scheduler", + "text_encoder", + "tokenizer", + "transformer", + "vae", + ] + + def create_pipeline_stages(self, server_args: ServerArgs): + """Set up pipeline stages with proper dependency injection.""" + + self.add_stage( + stage_name="input_validation_stage", stage=InputValidationStage() + ) + + self.add_stage( + stage_name="prompt_encoding_stage_primary", + stage=ImageEncodingStage( + image_processor=self.get_module("processor"), + text_encoder=self.get_module("text_encoder"), + ), + ) + + self.add_stage( + stage_name="image_encoding_stage_primary", + stage=ImageVAEEncodingStage( + vae=self.get_module("vae"), + ), + ) + + self.add_stage( + stage_name="timestep_preparation_stage", + stage=TimestepPreparationStage( + scheduler=self.get_module("scheduler"), + prepare_extra_set_timesteps_kwargs=[prepare_mu], + ), + ) + + self.add_stage( + stage_name="latent_preparation_stage", + stage=LatentPreparationStage( + scheduler=self.get_module("scheduler"), + transformer=self.get_module("transformer"), + ), + ) + + self.add_stage(stage_name="conditioning_stage", stage=ConditioningStage()) + + self.add_stage( + stage_name="denoising_stage", + stage=DenoisingStage( + transformer=self.get_module("transformer"), + scheduler=self.get_module("scheduler"), + ), + ) + + self.add_stage( + stage_name="decoding_stage", stage=DecodingStage(vae=self.get_module("vae")) + ) + + +EntryClass = [QwenImagePipeline, QwenImageEditPipeline] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/stepvideo_pipeline.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/stepvideo_pipeline.py new file mode 100644 index 00000000..9d2e80c2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/stepvideo_pipeline.py @@ -0,0 +1,182 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# type: ignore +# SPDX-License-Identifier: Apache-2.0 +""" +Hunyuan video diffusion pipeline implementation. + +This module contains an implementation of the Hunyuan video diffusion pipeline +using the modular pipeline architecture. +""" + +import os +from typing import Any + +import torch +from huggingface_hub import hf_hub_download + +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.loader.component_loader import ( + PipelineComponentLoader, +) +from sglang.multimodal_gen.runtime.models.encoders.bert import ( + HunyuanClip, # type: ignore +) +from sglang.multimodal_gen.runtime.models.encoders.stepllm import STEP1TextEncoder +from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( + ComposedPipelineBase, +) +from sglang.multimodal_gen.runtime.pipelines_core.lora_pipeline import LoRAPipeline +from sglang.multimodal_gen.runtime.pipelines_core.stages import ( + DecodingStage, + DenoisingStage, + InputValidationStage, + LatentPreparationStage, + StepvideoPromptEncodingStage, + TimestepPreparationStage, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class StepVideoPipeline(LoRAPipeline, ComposedPipelineBase): + pipeline_name = "StepVideoPipeline" + + _required_config_modules = ["transformer", "scheduler", "vae"] + + def create_pipeline_stages(self, server_args: ServerArgs): + """Set up pipeline stages with proper dependency injection.""" + + self.add_stage( + stage_name="input_validation_stage", stage=InputValidationStage() + ) + + self.add_stage( + stage_name="prompt_encoding_stage", + stage=StepvideoPromptEncodingStage( + stepllm=self.get_module("text_encoder"), + clip=self.get_module("text_encoder_2"), + ), + ) + + self.add_stage( + stage_name="timestep_preparation_stage", + stage=TimestepPreparationStage(scheduler=self.get_module("scheduler")), + ) + + self.add_stage( + stage_name="latent_preparation_stage", + stage=LatentPreparationStage( + scheduler=self.get_module("scheduler"), + transformer=self.get_module("transformer"), + ), + ) + + self.add_stage( + stage_name="denoising_stage", + stage=DenoisingStage( + transformer=self.get_module("transformer"), + scheduler=self.get_module("scheduler"), + ), + ) + + self.add_stage( + stage_name="decoding_stage", stage=DecodingStage(vae=self.get_module("vae")) + ) + + def build_llm(self, model_dir, device) -> torch.nn.Module: + text_encoder = ( + STEP1TextEncoder(model_dir, max_length=320).to(torch.bfloat16).eval() + ) + return text_encoder + + def build_clip(self, model_dir, device) -> HunyuanClip: + clip = HunyuanClip(model_dir, max_length=77).eval() + return clip + + def initialize_pipeline(self, server_args: ServerArgs): + """ + Initialize the pipeline. + """ + target_device = get_local_torch_device() + llm_dir = os.path.join(self.model_path, "step_llm") + clip_dir = os.path.join(self.model_path, "hunyuan_clip") + text_enc = self.build_llm(llm_dir, target_device) + clip_enc = self.build_clip(clip_dir, target_device) + self.add_module("text_encoder", text_enc) + self.add_module("text_encoder_2", clip_enc) + lib_path = ( + os.path.join( + server_args.model_path, + "lib/liboptimus_ths-torch2.5-cu124.cpython-310-x86_64-linux-gnu.so", + ) + if os.path.isdir(server_args.model_path) # local checkout + else hf_hub_download( + repo_id=server_args.model_path, + filename="lib/liboptimus_ths-torch2.5-cu124.cpython-310-x86_64-linux-gnu.so", + ) + ) + torch.ops.load_library(lib_path) + + def load_modules( + self, + server_args: ServerArgs, + loaded_modules: dict[str, torch.nn.Module] | None = None, + ) -> dict[str, Any]: + """ + Load the modules from the config. + """ + model_index = self._load_config() + logger.info("Loading pipeline modules from config: %s", model_index) + + # remove keys that are not pipeline modules + model_index.pop("_class_name") + model_index.pop("_diffusers_version") + + # some sanity checks + assert ( + len(model_index) > 1 + ), "model_index.json must contain at least one pipeline module" + + required_modules = ["transformer", "scheduler", "vae"] + for module_name in required_modules: + if module_name not in model_index: + raise ValueError( + f"model_index.json must contain a {module_name} module" + ) + logger.info("Diffusers config passed sanity checks") + + # all the component models used by the pipeline + modules = {} + for module_name, ( + transformers_or_diffusers, + architecture, + ) in model_index.items(): + component_model_path = os.path.join(self.model_path, module_name) + module = PipelineComponentLoader.load_module( + module_name=module_name, + component_model_path=component_model_path, + transformers_or_diffusers=transformers_or_diffusers, + server_args=server_args, + ) + logger.info("Loaded module %s from %s", module_name, component_model_path) + + if module_name in modules: + logger.warning("Overwriting module %s", module_name) + modules[module_name] = module + + required_modules = self.required_config_modules + # Check if all required modules were loaded + for module_name in required_modules: + if module_name not in modules or modules[module_name] is None: + raise ValueError( + f"Required module {module_name} was not loaded properly" + ) + + return modules + + +EntryClass = StepVideoPipeline diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/wan_causal_dmd_pipeline.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/wan_causal_dmd_pipeline.py new file mode 100644 index 00000000..b103ee0a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/wan_causal_dmd_pipeline.py @@ -0,0 +1,81 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +Wan causal DMD pipeline implementation. + +This module wires the causal DMD denoising stage into the modular pipeline. +""" + +from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( + ComposedPipelineBase, +) +from sglang.multimodal_gen.runtime.pipelines_core.lora_pipeline import LoRAPipeline + +# isort: off +from sglang.multimodal_gen.runtime.pipelines_core.stages import ( + ConditioningStage, + DecodingStage, + CausalDMDDenoisingStage, + InputValidationStage, + LatentPreparationStage, + TextEncodingStage, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +# isort: on + +logger = init_logger(__name__) + + +class WanCausalDMDPipeline(LoRAPipeline, ComposedPipelineBase): + pipeline_name = "WanCausalDMDPipeline" + + _required_config_modules = [ + "text_encoder", + "tokenizer", + "vae", + "transformer", + "scheduler", + ] + + def create_pipeline_stages(self, server_args: ServerArgs) -> None: + """Set up pipeline stages with proper dependency injection.""" + + self.add_stage( + stage_name="input_validation_stage", stage=InputValidationStage() + ) + + self.add_stage( + stage_name="prompt_encoding_stage", + stage=TextEncodingStage( + text_encoders=[self.get_module("text_encoder")], + tokenizers=[self.get_module("tokenizer")], + ), + ) + + self.add_stage(stage_name="conditioning_stage", stage=ConditioningStage()) + + self.add_stage( + stage_name="latent_preparation_stage", + stage=LatentPreparationStage( + scheduler=self.get_module("scheduler"), + transformer=self.get_module("transformer", None), + ), + ) + + self.add_stage( + stage_name="denoising_stage", + stage=CausalDMDDenoisingStage( + transformer=self.get_module("transformer"), + scheduler=self.get_module("scheduler"), + ), + ) + + self.add_stage( + stage_name="decoding_stage", stage=DecodingStage(vae=self.get_module("vae")) + ) + + +EntryClass = WanCausalDMDPipeline diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/wan_dmd_pipeline.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/wan_dmd_pipeline.py new file mode 100644 index 00000000..3c973834 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/wan_dmd_pipeline.py @@ -0,0 +1,101 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +Wan video diffusion pipeline implementation. + +This module contains an implementation of the Wan video diffusion pipeline +using the modular pipeline architecture. +""" + +from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, +) +from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( + ComposedPipelineBase, +) +from sglang.multimodal_gen.runtime.pipelines_core.lora_pipeline import LoRAPipeline +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +# isort: off +from sglang.multimodal_gen.runtime.pipelines_core.stages import ( + ConditioningStage, + DecodingStage, + DmdDenoisingStage, + InputValidationStage, + LatentPreparationStage, + TextEncodingStage, + TimestepPreparationStage, +) + +# isort: on + +logger = init_logger(__name__) + + +class WanDMDPipeline(LoRAPipeline, ComposedPipelineBase): + """ + Wan video diffusion pipeline with LoRA support. + """ + + pipeline_name = "WanDMDPipeline" + + _required_config_modules = [ + "text_encoder", + "tokenizer", + "vae", + "transformer", + "scheduler", + ] + + def initialize_pipeline(self, server_args: ServerArgs): + + self.modules["scheduler"] = FlowMatchEulerDiscreteScheduler( + shift=server_args.pipeline_config.flow_shift + ) + + def create_pipeline_stages(self, server_args: ServerArgs) -> None: + """Set up pipeline stages with proper dependency injection.""" + + self.add_stage( + stage_name="input_validation_stage", stage=InputValidationStage() + ) + + self.add_stage( + stage_name="prompt_encoding_stage", + stage=TextEncodingStage( + text_encoders=[self.get_module("text_encoder")], + tokenizers=[self.get_module("tokenizer")], + ), + ) + + self.add_stage(stage_name="conditioning_stage", stage=ConditioningStage()) + + self.add_stage( + stage_name="timestep_preparation_stage", + stage=TimestepPreparationStage(scheduler=self.get_module("scheduler")), + ) + + self.add_stage( + stage_name="latent_preparation_stage", + stage=LatentPreparationStage( + scheduler=self.get_module("scheduler"), + transformer=self.get_module("transformer", None), + ), + ) + + self.add_stage( + stage_name="denoising_stage", + stage=DmdDenoisingStage( + transformer=self.get_module("transformer"), + scheduler=self.get_module("scheduler"), + ), + ) + + self.add_stage( + stage_name="decoding_stage", stage=DecodingStage(vae=self.get_module("vae")) + ) + + +EntryClass = WanDMDPipeline diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/wan_i2v_dmd_pipeline.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/wan_i2v_dmd_pipeline.py new file mode 100644 index 00000000..7a49eef9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/wan_i2v_dmd_pipeline.py @@ -0,0 +1,113 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +Wan video diffusion pipeline implementation. + +This module contains an implementation of the Wan video diffusion pipeline +using the modular pipeline architecture. +""" + +from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( + ComposedPipelineBase, +) +from sglang.multimodal_gen.runtime.pipelines_core.lora_pipeline import LoRAPipeline +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +# isort: off +from sglang.multimodal_gen.runtime.pipelines_core.stages import ( + ImageEncodingStage, + ConditioningStage, + DecodingStage, + DmdDenoisingStage, + ImageVAEEncodingStage, + InputValidationStage, + LatentPreparationStage, + TextEncodingStage, + TimestepPreparationStage, +) + +# isort: on +from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, +) + +logger = init_logger(__name__) + + +class WanImageToVideoDmdPipeline(LoRAPipeline, ComposedPipelineBase): + pipeline_name = "WanImageToVideoDmdPipeline" + + _required_config_modules = [ + "text_encoder", + "tokenizer", + "vae", + "transformer", + "scheduler", + "image_encoder", + "image_processor", + ] + + def initialize_pipeline(self, server_args: ServerArgs): + self.modules["scheduler"] = FlowMatchEulerDiscreteScheduler( + shift=server_args.pipeline_config.flow_shift + ) + + def create_pipeline_stages(self, server_args: ServerArgs): + """Set up pipeline stages with proper dependency injection.""" + + self.add_stage( + stage_name="input_validation_stage", stage=InputValidationStage() + ) + + self.add_stage( + stage_name="prompt_encoding_stage", + stage=TextEncodingStage( + text_encoders=[self.get_module("text_encoder")], + tokenizers=[self.get_module("tokenizer")], + ), + ) + + self.add_stage( + stage_name="image_encoding_stage", + stage=ImageEncodingStage( + image_encoder=self.get_module("image_encoder"), + image_processor=self.get_module("image_processor"), + ), + ) + + self.add_stage(stage_name="conditioning_stage", stage=ConditioningStage()) + + self.add_stage( + stage_name="timestep_preparation_stage", + stage=TimestepPreparationStage(scheduler=self.get_module("scheduler")), + ) + + self.add_stage( + stage_name="latent_preparation_stage", + stage=LatentPreparationStage( + scheduler=self.get_module("scheduler"), + transformer=self.get_module("transformer"), + ), + ) + + self.add_stage( + stage_name="image_latent_preparation_stage", + stage=ImageVAEEncodingStage(vae=self.get_module("vae")), + ) + + self.add_stage( + stage_name="denoising_stage", + stage=DmdDenoisingStage( + transformer=self.get_module("transformer"), + scheduler=self.get_module("scheduler"), + ), + ) + + self.add_stage( + stage_name="decoding_stage", stage=DecodingStage(vae=self.get_module("vae")) + ) + + +EntryClass = WanImageToVideoDmdPipeline diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/wan_i2v_pipeline.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/wan_i2v_pipeline.py new file mode 100644 index 00000000..93a19687 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/wan_i2v_pipeline.py @@ -0,0 +1,118 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +Wan video diffusion pipeline implementation. + +This module contains an implementation of the Wan video diffusion pipeline +using the modular pipeline architecture. +""" + +from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( + ComposedPipelineBase, +) +from sglang.multimodal_gen.runtime.pipelines_core.lora_pipeline import LoRAPipeline +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +# isort: off +from sglang.multimodal_gen.runtime.pipelines_core.stages import ( + ImageEncodingStage, + ConditioningStage, + DecodingStage, + DenoisingStage, + ImageVAEEncodingStage, + InputValidationStage, + LatentPreparationStage, + TextEncodingStage, + TimestepPreparationStage, +) + +# isort: on +from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_unipc_multistep import ( + FlowUniPCMultistepScheduler, +) + +logger = init_logger(__name__) + + +class WanImageToVideoPipeline(LoRAPipeline, ComposedPipelineBase): + pipeline_name = "WanImageToVideoPipeline" + + _required_config_modules = [ + "text_encoder", + "tokenizer", + "vae", + "transformer", + "scheduler", + "image_encoder", + "image_processor", + ] + + def initialize_pipeline(self, server_args: ServerArgs): + self.modules["scheduler"] = FlowUniPCMultistepScheduler( + shift=server_args.pipeline_config.flow_shift + ) + + def create_pipeline_stages(self, server_args: ServerArgs): + """Set up pipeline stages with proper dependency injection.""" + + self.add_stage( + stage_name="input_validation_stage", stage=InputValidationStage() + ) + + self.add_stage( + stage_name="prompt_encoding_stage", + stage=TextEncodingStage( + text_encoders=[self.get_module("text_encoder")], + tokenizers=[self.get_module("tokenizer")], + ), + ) + + if ( + self.get_module("image_encoder") is not None + and self.get_module("image_processor") is not None + ): + self.add_stage( + stage_name="image_encoding_stage", + stage=ImageEncodingStage( + image_encoder=self.get_module("image_encoder"), + image_processor=self.get_module("image_processor"), + ), + ) + + self.add_stage(stage_name="conditioning_stage", stage=ConditioningStage()) + + self.add_stage( + stage_name="timestep_preparation_stage", + stage=TimestepPreparationStage(scheduler=self.get_module("scheduler")), + ) + + self.add_stage( + stage_name="latent_preparation_stage", + stage=LatentPreparationStage( + scheduler=self.get_module("scheduler"), + transformer=self.get_module("transformer"), + ), + ) + + self.add_stage( + stage_name="image_latent_preparation_stage", + stage=ImageVAEEncodingStage(vae=self.get_module("vae")), + ) + + self.add_stage( + stage_name="denoising_stage", + stage=DenoisingStage( + transformer=self.get_module("transformer"), + transformer_2=self.get_module("transformer_2"), + scheduler=self.get_module("scheduler"), + ), + ) + + self.add_stage( + stage_name="decoding_stage", stage=DecodingStage(vae=self.get_module("vae")) + ) + + +EntryClass = WanImageToVideoPipeline diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/wan_pipeline.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/wan_pipeline.py new file mode 100644 index 00000000..8f1cbfc2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/wan_pipeline.py @@ -0,0 +1,101 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +Wan video diffusion pipeline implementation. + +This module contains an implementation of the Wan video diffusion pipeline +using the modular pipeline architecture. +""" + +from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_unipc_multistep import ( + FlowUniPCMultistepScheduler, +) +from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( + ComposedPipelineBase, +) +from sglang.multimodal_gen.runtime.pipelines_core.lora_pipeline import LoRAPipeline +from sglang.multimodal_gen.runtime.pipelines_core.stages import ( + ConditioningStage, + DecodingStage, + DenoisingStage, + InputValidationStage, + LatentPreparationStage, + TextEncodingStage, + TimestepPreparationStage, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class WanPipeline(LoRAPipeline, ComposedPipelineBase): + """ + Wan video diffusion pipeline with LoRA support. + """ + + pipeline_name = "WanPipeline" + + _required_config_modules = [ + "text_encoder", + "tokenizer", + "vae", + "transformer", + "scheduler", + ] + + def initialize_pipeline(self, server_args: ServerArgs): + # We use UniPCMScheduler from Wan2.1 official repo, not the one in diffusers. + self.modules["scheduler"] = FlowUniPCMultistepScheduler( + shift=server_args.pipeline_config.flow_shift + ) + + def create_pipeline_stages(self, server_args: ServerArgs) -> None: + """Set up pipeline stages with proper dependency injection.""" + + self.add_stage( + stage_name="input_validation_stage", stage=InputValidationStage() + ) + + self.add_stage( + stage_name="prompt_encoding_stage", + stage=TextEncodingStage( + text_encoders=[self.get_module("text_encoder")], + tokenizers=[self.get_module("tokenizer")], + ), + ) + + self.add_stage(stage_name="conditioning_stage", stage=ConditioningStage()) + + self.add_stage( + stage_name="timestep_preparation_stage", + stage=TimestepPreparationStage(scheduler=self.get_module("scheduler")), + ) + + self.add_stage( + stage_name="latent_preparation_stage", + stage=LatentPreparationStage( + scheduler=self.get_module("scheduler"), + transformer=self.get_module("transformer", None), + ), + ) + + self.add_stage( + stage_name="denoising_stage", + stage=DenoisingStage( + transformer=self.get_module("transformer"), + transformer_2=self.get_module("transformer_2", None), + scheduler=self.get_module("scheduler"), + vae=self.get_module("vae"), + pipeline=self, + ), + ) + + self.add_stage( + stage_name="decoding_stage", + stage=DecodingStage(vae=self.get_module("vae"), pipeline=self), + ) + + +EntryClass = WanPipeline diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/zimage_pipeline.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/zimage_pipeline.py new file mode 100644 index 00000000..f8fd441d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines/zimage_pipeline.py @@ -0,0 +1,116 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo +# SPDX-License-Identifier: Apache-2.0 + + +from sglang.multimodal_gen.runtime.pipelines_core import LoRAPipeline, Req +from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( + ComposedPipelineBase, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages import ( + ConditioningStage, + DecodingStage, + DenoisingStage, + InputValidationStage, + LatentPreparationStage, + TextEncodingStage, + TimestepPreparationStage, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + + +def prepare_mu(batch: Req, server_args: ServerArgs): + height = batch.height + width = batch.width + vae_scale_factor = server_args.pipeline_config.vae_config.vae_scale_factor + image_seq_len = ((int(height) // vae_scale_factor) // 2) * ( + (int(width) // vae_scale_factor) // 2 + ) + mu = calculate_shift( + image_seq_len, + # hard code, since scheduler_config is not in PipelineConfig now + 256, + 4096, + 0.5, + 1.15, + ) + return "mu", mu + + +class ZImagePipeline(LoRAPipeline, ComposedPipelineBase): + pipeline_name = "ZImagePipeline" + + _required_config_modules = [ + "text_encoder", + "tokenizer", + "vae", + "transformer", + "scheduler", + ] + + def create_pipeline_stages(self, server_args: ServerArgs): + """Set up pipeline stages with proper dependency injection.""" + + self.add_stage( + stage_name="input_validation_stage", stage=InputValidationStage() + ) + + self.add_stage( + stage_name="prompt_encoding_stage_primary", + stage=TextEncodingStage( + text_encoders=[ + self.get_module("text_encoder"), + ], + tokenizers=[ + self.get_module("tokenizer"), + ], + ), + ) + + self.add_stage(stage_name="conditioning_stage", stage=ConditioningStage()) + + self.add_stage( + stage_name="timestep_preparation_stage", + stage=TimestepPreparationStage( + scheduler=self.get_module("scheduler"), + prepare_extra_set_timesteps_kwargs=[prepare_mu], + ), + ) + + self.add_stage( + stage_name="latent_preparation_stage", + stage=LatentPreparationStage( + scheduler=self.get_module("scheduler"), + transformer=self.get_module("transformer"), + ), + ) + + self.add_stage( + stage_name="denoising_stage", + stage=DenoisingStage( + transformer=self.get_module("transformer"), + scheduler=self.get_module("scheduler"), + ), + ) + + self.add_stage( + stage_name="decoding_stage", stage=DecodingStage(vae=self.get_module("vae")) + ) + + +EntryClass = ZImagePipeline diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/__init__.py new file mode 100644 index 00000000..32243af3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/__init__.py @@ -0,0 +1,62 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +Diffusion pipelines for sglang.multimodal_gen. + +This package contains diffusion pipelines for generating videos and images. +""" + +from typing import cast + +from sglang.multimodal_gen.registry import get_model_info +from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( + ComposedPipelineBase, +) +from sglang.multimodal_gen.runtime.pipelines_core.lora_pipeline import LoRAPipeline +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import ( + maybe_download_model, + verify_model_config_and_directory, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class PipelineWithLoRA(LoRAPipeline, ComposedPipelineBase): + """Type for a pipeline that has both ComposedPipelineBase and LoRAPipeline functionality.""" + + pass + + +def build_pipeline( + server_args: ServerArgs, +) -> PipelineWithLoRA: + """ + Only works with valid hf diffusers configs. (model_index.json) + We want to build a pipeline based on the inference args mode_path: + 1. download the model from the hub if it's not already downloaded + 2. verify the model config and directory + 3. based on the config, determine the pipeline class + """ + model_path = server_args.model_path + model_info = get_model_info(model_path) + + pipeline_cls = model_info.pipeline_cls + + # instantiate the pipelines + pipeline = pipeline_cls(model_path, server_args) + + logger.info("Pipelines instantiated") + + return cast(PipelineWithLoRA, pipeline) + + +__all__ = [ + "build_pipeline", + "ComposedPipelineBase", + "Req", + "LoRAPipeline", +] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/composed_pipeline_base.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/composed_pipeline_base.py new file mode 100644 index 00000000..175c14fe --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/composed_pipeline_base.py @@ -0,0 +1,365 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +Base class for composed pipelines. + +This module defines the base class for pipelines that are composed of multiple stages. +""" + +import argparse +import os +from abc import ABC, abstractmethod +from typing import Any, cast + +import torch +from tqdm import tqdm + +from sglang.multimodal_gen.configs.pipeline_configs import PipelineConfig +from sglang.multimodal_gen.runtime.loader.component_loader import ( + PipelineComponentLoader, +) +from sglang.multimodal_gen.runtime.pipelines_core.executors.pipeline_executor import ( + PipelineExecutor, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages import PipelineStage +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import ( + maybe_download_model, + verify_model_config_and_directory, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class ComposedPipelineBase(ABC): + """ + Base class for pipelines composed of multiple stages. + + This class provides the framework for creating pipelines by composing multiple + stages together. Each stage is responsible for a specific part of the diffusion + process, and the pipeline orchestrates the execution of these stages. + """ + + is_video_pipeline: bool = False # To be overridden by video pipelines + # should contains only the modules to be loaded + _required_config_modules: list[str] = [] + _extra_config_module_map: dict[str, str] = {} + server_args: ServerArgs | None = None + modules: dict[str, Any] = {} + post_init_called: bool = False + executor: PipelineExecutor | None = None + + # the name of the pipeline it associated with, in diffusers + pipeline_name: str + + def is_lora_effective(self): + return False + + def is_lora_set(self): + return False + + def __init__( + self, + model_path: str, + server_args: ServerArgs, + required_config_modules: list[str] | None = None, + loaded_modules: dict[str, torch.nn.Module] | None = None, + executor: PipelineExecutor | None = None, + ): + """ + Initialize the pipeline. After __init__, the pipeline should be ready to + use. The pipeline should be stateless and not hold any batch state. + """ + self.server_args = server_args + + self.model_path: str = model_path + self._stages: list[PipelineStage] = [] + self._stage_name_mapping: dict[str, PipelineStage] = {} + self.executor = executor or self.build_executor(server_args=server_args) + + if required_config_modules is not None: + self._required_config_modules = required_config_modules + + if self._required_config_modules is None: + raise NotImplementedError("Subclass must set _required_config_modules") + # temp disable for duplicate initialing tp + # maybe_init_distributed_environment_and_model_parallel( + # server_args.tp_size, server_args.sp_size + # ) + + # Load modules directly in initialization + logger.info("Loading pipeline modules...") + self.modules = self.load_modules(server_args, loaded_modules) + + def build_executor(self, server_args: ServerArgs): + # TODO + from sglang.multimodal_gen.runtime.pipelines_core.executors.parallel_executor import ( + ParallelExecutor, + ) + + # return SyncExecutor(server_args=server_args) + return ParallelExecutor(server_args=server_args) + + def post_init(self) -> None: + assert self.server_args is not None, "server_args must be set" + if self.post_init_called: + return + self.post_init_called = True + + self.initialize_pipeline(self.server_args) + if self.server_args.enable_torch_compile: + self.modules["transformer"] = torch.compile(self.modules["transformer"]) + logger.info("Torch Compile enabled for DiT") + + logger.info("Creating pipeline stages...") + self.create_pipeline_stages(self.server_args) + + @classmethod + def from_pretrained( + cls, + model_path: str, + device: str | None = None, + torch_dtype: torch.dtype | None = None, + pipeline_config: str | PipelineConfig | None = None, + args: argparse.Namespace | None = None, + required_config_modules: list[str] | None = None, + loaded_modules: dict[str, torch.nn.Module] | None = None, + **kwargs, + ) -> "ComposedPipelineBase": + """ + Load a pipeline from a pretrained model. + loaded_modules: Optional[Dict[str, torch.nn.Module]] = None, + If provided, loaded_modules will be used instead of loading from config/pretrained weights. + """ + kwargs["model_path"] = model_path + server_args = ServerArgs.from_kwargs(**kwargs) + + logger.info("server_args in from_pretrained: %s", server_args) + + pipe = cls( + model_path, + server_args, + required_config_modules=required_config_modules, + loaded_modules=loaded_modules, + ) + pipe.post_init() + return pipe + + def get_module(self, module_name: str, default_value: Any = None) -> Any: + if module_name not in self.modules: + return default_value + return self.modules[module_name] + + def add_module(self, module_name: str, module: Any): + self.modules[module_name] = module + + def _load_config(self) -> dict[str, Any]: + model_path = maybe_download_model(self.model_path) + self.model_path = model_path + # server_args.downloaded_model_path = model_path + logger.info("Model path: %s", model_path) + config = verify_model_config_and_directory(model_path) + return cast(dict[str, Any], config) + + @property + def required_config_modules(self) -> list[str]: + """ + List of modules that are required by the pipeline. The names should match + the diffusers directory and model_index.json file. These modules will be + loaded using the PipelineComponentLoader and made available in the + modules dictionary. Access these modules using the get_module method. + + class ConcretePipeline(ComposedPipelineBase): + _required_config_modules = ["vae", "text_encoder", "transformer", "scheduler", "tokenizer"] + + + @property + def required_config_modules(self): + return self._required_config_modules + """ + return self._required_config_modules + + @property + def stages(self) -> list[PipelineStage]: + """ + List of stages in the pipeline. + """ + return self._stages + + @abstractmethod + def create_pipeline_stages(self, server_args: ServerArgs): + """ + Create the inference pipeline stages. + """ + raise NotImplementedError + + def initialize_pipeline(self, server_args: ServerArgs): + """ + Initialize the pipeline. + """ + return + + def load_modules( + self, + server_args: ServerArgs, + loaded_modules: dict[str, torch.nn.Module] | None = None, + ) -> dict[str, Any]: + """ + Load the modules from the config. + loaded_modules: Optional[Dict[str, torch.nn.Module]] = None, + If provided, loaded_modules will be used instead of loading from config/pretrained weights. + """ + + model_index = self._load_config() + logger.info("Loading pipeline modules from config: %s", model_index) + + # remove keys that are not pipeline modules + model_index.pop("_class_name") + model_index.pop("_diffusers_version") + if ( + "boundary_ratio" in model_index + and model_index["boundary_ratio"] is not None + ): + logger.info( + "MoE pipeline detected. Adding transformer_2 to self.required_config_modules..." + ) + self.required_config_modules.append("transformer_2") + logger.info( + "MoE pipeline detected. Setting boundary ratio to %s", + model_index["boundary_ratio"], + ) + server_args.pipeline_config.dit_config.boundary_ratio = model_index[ + "boundary_ratio" + ] + + model_index.pop("boundary_ratio", None) + # used by Wan2.2 ti2v + model_index.pop("expand_timesteps", None) + + # some sanity checks + assert ( + len(model_index) > 1 + ), "model_index.json must contain at least one pipeline module" + + model_index = { + required_module: model_index[required_module] + for required_module in self.required_config_modules + } + + for module_name in self.required_config_modules: + if ( + module_name not in model_index + and module_name in self._extra_config_module_map + ): + extra_module_value = self._extra_config_module_map[module_name] + logger.warning( + "model_index.json does not contain a %s module, but found {%s: %s} in _extra_config_module_map, adding to model_index.", + module_name, + module_name, + extra_module_value, + ) + if extra_module_value in model_index: + logger.info( + "Using module %s for %s", extra_module_value, module_name + ) + model_index[module_name] = model_index[extra_module_value] + continue + else: + raise ValueError( + f"Required module key: {module_name} value: {model_index.get(module_name)} was not found in loaded modules {model_index.keys()}" + ) + + # all the component models used by the pipeline + required_modules = self.required_config_modules + logger.info("Loading required components: %s", required_modules) + + components = {} + for module_name, ( + transformers_or_diffusers, + architecture, + ) in tqdm(iterable=model_index.items(), desc="Loading required modules"): + + if transformers_or_diffusers is None: + logger.warning( + "Module %s in model_index.json has null value, removing from required_config_modules", + module_name, + ) + if module_name in self.required_config_modules: + self.required_config_modules.remove(module_name) + continue + if module_name not in required_modules: + logger.info("Skipping module %s", module_name) + continue + if loaded_modules is not None and module_name in loaded_modules: + logger.info("Using module %s already provided", module_name) + components[module_name] = loaded_modules[module_name] + continue + + # we load the module from the extra config module map if it exists + if module_name in self._extra_config_module_map: + load_module_name = self._extra_config_module_map[module_name] + else: + load_module_name = module_name + + component_model_path = os.path.join(self.model_path, load_module_name) + module = PipelineComponentLoader.load_module( + module_name=load_module_name, + component_model_path=component_model_path, + transformers_or_diffusers=transformers_or_diffusers, + server_args=server_args, + ) + logger.info("Loaded module %s from %s", module_name, component_model_path) + + if module_name in components: + logger.warning("Overwriting module %s", module_name) + components[module_name] = module + + # Check if all required modules were loaded + for module_name in required_modules: + if module_name not in components or components[module_name] is None: + raise ValueError( + f"Required module key: {module_name} value: {components.get(module_name)} was not found in loaded modules {components.keys()}" + ) + + return components + + def add_stage(self, stage_name: str, stage: PipelineStage): + assert self.modules is not None, "No modules are registered" + self._stages.append(stage) + self._stage_name_mapping[stage_name] = stage + setattr(self, stage_name, stage) + + # TODO(will): don't hardcode no_grad + @torch.no_grad() + def forward( + self, + batch: Req, + server_args: ServerArgs, + ) -> Req: + """ + Generate a video or image using the pipeline. + + Args: + batch: The batch to generate from. + server_args: The inference arguments. + Returns: + Req: The batch with the generated video or image. + """ + if not self.post_init_called: + self.post_init() + + if self.is_lora_set() and not self.is_lora_effective(): + logger.warning( + "LoRA adapter is set, but not effective. Please make sure the LoRA weights are merged" + ) + + # Execute each stage + logger.info( + "Running pipeline stages: %s", + list(self._stage_name_mapping.keys()), + main_process_only=True, + ) + return self.executor.execute(self.stages, batch, server_args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/executors/parallel_executor.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/executors/parallel_executor.py new file mode 100644 index 00000000..dabe531c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/executors/parallel_executor.py @@ -0,0 +1,92 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +from typing import List + +import torch + +from sglang.multimodal_gen.runtime.distributed import get_sp_group +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + get_cfg_group, + get_classifier_free_guidance_rank, +) +from sglang.multimodal_gen.runtime.pipelines_core import Req +from sglang.multimodal_gen.runtime.pipelines_core.executors.pipeline_executor import ( + PipelineExecutor, + Timer, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import ( + PipelineStage, + StageParallelismType, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.distributed import broadcast_pyobj + + +class ParallelExecutor(PipelineExecutor): + """ + The correctness of the execution relies on the parallelism_type declared by stages + + """ + + def collect_from_main(self, batches: list[Req]): + + # TODO: fix this condition + if self.server_args.sp_degree != 1: + sp_group = get_sp_group() + batches = broadcast_pyobj( + batches, + sp_group.rank, + sp_group.cpu_group, + src=sp_group.ranks[0], + ) + + if self.server_args.enable_cfg_parallel: + batches = broadcast_pyobj( + batches, + self.worker.cfg_group.rank, + self.worker.cfg_cpu_group, + src=self.worker.cfg_group.ranks[0], + ) + + def execute( + self, + stages: List[PipelineStage], + batch: Req, + server_args: ServerArgs, + ) -> Req: + rank = get_classifier_free_guidance_rank() + cfg_rank = get_classifier_free_guidance_rank() + cfg_group = get_cfg_group() + + # TODO: decide when to gather on main when CFG_PARALLEL -> MAIN_RANK_ONLY + for stage in stages: + with Timer(stage.__class__.__name__): + paradigm = stage.parallelism_type + + if paradigm == StageParallelismType.MAIN_RANK_ONLY: + if rank == 0: + batch = stage(batch, server_args) + # obj_list = [batch] if rank == 0 else [] + # + # broadcasted_list = broadcast_pyobj( + # obj_list, rank=rank, dist_group=cfg_group.cpu_group, src=0 + # ) + # if rank != 0: + # batch = broadcasted_list[0] + torch.distributed.barrier() + + elif paradigm == StageParallelismType.CFG_PARALLEL: + obj_list = [batch] if rank == 0 else [] + broadcasted_list = broadcast_pyobj( + obj_list, rank=rank, dist_group=cfg_group.cpu_group, src=0 + ) + if rank != 0: + batch = broadcasted_list[0] + batch = stage(batch, server_args) + + torch.distributed.barrier() + + elif paradigm == StageParallelismType.REPLICATED: + batch = stage(batch, server_args) + + return batch diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/executors/pipeline_executor.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/executors/pipeline_executor.py new file mode 100644 index 00000000..917af320 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/executors/pipeline_executor.py @@ -0,0 +1,59 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +Base class for all pipeline executors. +""" + +from abc import ABC, abstractmethod +from typing import List + +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages import PipelineStage +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.runtime.utils.perf_logger import StageProfiler + +logger = init_logger(__name__) + + +class Timer(StageProfiler): + """ + A wrapper around StageProfiler to maintain backward compatibility. + It forces simple logging behavior (log start/end) regardless of env vars. + """ + + def __init__(self, name="Stage"): + super().__init__(stage_name=name, timings=None, simple_log=True, logger=logger) + + +class PipelineExecutor(ABC): + """ + Abstract base class for all pipeline executors. + + Executors orchestrate the execution of pipeline, with managing the parallel and communications required by stages + + """ + + def __init__(self, server_args): + self.server_args = server_args + + @abstractmethod + def execute( + self, + stages: List[PipelineStage], + batch: Req, + server_args: ServerArgs, + ) -> Req: + """ + Execute the pipeline stages. + + Args: + stages: A list of pipeline stages to execute. + batch: The batch to process. + server_args: The server arguments. + + Returns: + The processed batch. + """ + raise NotImplementedError diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/executors/sync_executor.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/executors/sync_executor.py new file mode 100644 index 00000000..5e429b3e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/executors/sync_executor.py @@ -0,0 +1,39 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +Synchronous pipeline executor implementation. +""" +from typing import List + +from sglang.multimodal_gen.runtime.pipelines_core.executors.pipeline_executor import ( + PipelineExecutor, + Timer, + logger, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages import PipelineStage +from sglang.multimodal_gen.runtime.server_args import ServerArgs + + +class SyncExecutor(PipelineExecutor): + """ + A simple synchronous executor that runs stages sequentially. + """ + + def execute( + self, + stages: List[PipelineStage], + batch: Req, + server_args: ServerArgs, + ) -> Req: + """ + Execute the pipeline stages sequentially. + """ + logger.info("Running pipeline stages sequentially with SyncExecutor.") + + for stage in stages: + with Timer(stage.__class__.__name__): + batch = stage(batch, server_args) + + return batch diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/lora_pipeline.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/lora_pipeline.py new file mode 100644 index 00000000..33f8ad6c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/lora_pipeline.py @@ -0,0 +1,284 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +import os +from collections import defaultdict +from collections.abc import Hashable +from typing import Any + +import torch +import torch.distributed as dist +from safetensors.torch import load_file + +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.layers.lora.linear import ( + BaseLayerWithLoRA, + replace_submodule, + wrap_with_lora_layer, +) +from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping +from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( + ComposedPipelineBase, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import maybe_download_lora +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +# to avoid deadlocks when forking +os.environ["TOKENIZERS_PARALLELISM"] = "false" + +logger = init_logger(__name__) + + +class LoRAPipeline(ComposedPipelineBase): + """ + Pipeline that supports injecting LoRA adapters into the diffusion transformer. + """ + + # [lora_nickname][target_LoRA_weight_name_in_SGLang_dit] = weight + # e.g., [jinx][transformer_blocks.0.attn.to_v.lora_A] + lora_adapters: dict[str, dict[str, torch.Tensor]] = defaultdict( + dict + ) # state dicts of loaded lora adapters + loaded_adapter_paths: dict[str, str] = {} # nickname -> lora_path + cur_adapter_name: str = "" + cur_adapter_path: str = "" + # [dit_layer_name] = wrapped_lora_layer + lora_layers: dict[str, BaseLayerWithLoRA] = {} + lora_layers_critic: dict[str, BaseLayerWithLoRA] = {} + server_args: ServerArgs + exclude_lora_layers: list[str] = [] + device: torch.device = get_local_torch_device() + lora_target_modules: list[str] | None = None + lora_path: str | None = None + lora_nickname: str = "default" + lora_rank: int | None = None + lora_alpha: int | None = None + lora_initialized: bool = False + is_lora_merged: bool = False + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.device = get_local_torch_device() + self.exclude_lora_layers = ( + self.server_args.pipeline_config.dit_config.arch_config.exclude_lora_layers + ) + self.lora_target_modules = self.server_args.lora_target_modules + self.lora_path = self.server_args.lora_path + self.lora_nickname = self.server_args.lora_nickname + if self.lora_path is not None: + self.convert_to_lora_layers() + self.set_lora( + self.lora_nickname, self.lora_path # type: ignore + ) # type: ignore + + def is_target_layer(self, module_name: str) -> bool: + if self.lora_target_modules is None: + return True + return any( + target_name in module_name for target_name in self.lora_target_modules + ) + + def convert_to_lora_layers(self) -> None: + """ + Unified method to convert the transformer to a LoRA transformer. + """ + if self.lora_initialized: + return + self.lora_initialized = True + converted_count = 0 + for name, layer in self.modules["transformer"].named_modules(): + if not self.is_target_layer(name): + continue + + excluded = any( + exclude_layer in name for exclude_layer in self.exclude_lora_layers + ) + if excluded: + continue + + lora_layer = wrap_with_lora_layer( + layer, + lora_rank=self.lora_rank, + lora_alpha=self.lora_alpha, + ) + if lora_layer is not None: + self.lora_layers[name] = lora_layer + replace_submodule(self.modules["transformer"], name, lora_layer) + converted_count += 1 + logger.info("Converted %d layers to LoRA layers", converted_count) + + if "fake_score_transformer" in self.modules: + for name, layer in self.modules["fake_score_transformer"].named_modules(): + if not self.is_target_layer(name): + continue + layer = wrap_with_lora_layer( + layer, + lora_rank=self.lora_rank, + lora_alpha=self.lora_alpha, + ) + if layer is not None: + self.lora_layers_critic[name] = layer + replace_submodule( + self.modules["fake_score_transformer"], name, layer + ) + converted_count += 1 + logger.info( + "Converted %d layers to LoRA layers in the critic model", + converted_count, + ) + + def is_lora_effective(self): + return self.is_lora_merged + + def is_lora_set(self): + return self.lora_initialized and self.cur_adapter_name is not None + + def load_lora_adapter(self, lora_path: str, lora_nickname: str, rank: int): + """ + Load the LoRA, and setup the lora_adapters for later weight replacement + """ + assert lora_path is not None + lora_local_path = maybe_download_lora(lora_path) + lora_state_dict = load_file(lora_local_path) + + if lora_nickname in self.lora_adapters: + self.lora_adapters[lora_nickname].clear() + + config = self.server_args.pipeline_config.dit_config.arch_config + + param_names_mapping_fn = get_param_names_mapping( + config.param_names_mapping + or self.modules["transformer"].param_names_mapping + ) + lora_param_names_mapping_fn = get_param_names_mapping( + config.lora_param_names_mapping + or self.modules["transformer"].lora_param_names_mapping + ) + + to_merge_params: defaultdict[Hashable, dict[Any, Any]] = defaultdict(dict) + for name, weight in lora_state_dict.items(): + name = name.replace("diffusion_model.", "") + name = name.replace(".weight", "") + # misc-format -> HF-format + name, _, _ = lora_param_names_mapping_fn(name) + # HF-format (LoRA) -> SGLang-dit-format + target_name, merge_index, num_params_to_merge = param_names_mapping_fn(name) + # for (in_dim, r) @ (r, out_dim), we only merge (r, out_dim * n) where n is the number of linear layers to fuse + # see param mapping in HunyuanVideoArchConfig + if merge_index is not None and "lora_B" in name: + to_merge_params[target_name][merge_index] = weight + if len(to_merge_params[target_name]) == num_params_to_merge: + # cat at output dim according to the merge_index order + sorted_tensors = [ + to_merge_params[target_name][i] + for i in range(num_params_to_merge) + ] + weight = torch.cat(sorted_tensors, dim=1) + del to_merge_params[target_name] + else: + continue + + print(f"{name} -> {target_name}") + if target_name in self.lora_adapters[lora_nickname]: + raise ValueError( + f"Dit target weight name {target_name} already exists in lora_adapters[{lora_nickname}]" + ) + self.lora_adapters[lora_nickname][target_name] = weight.to(self.device) + self.cur_adapter_path = lora_path + self.loaded_adapter_paths[lora_nickname] = lora_path + logger.info("Rank %d: loaded LoRA adapter %s", rank, lora_path) + + def set_lora( + self, lora_nickname: str, lora_path: str | None = None + ): # type: ignore + """ + Load a LoRA adapter into the pipeline and merge it into the transformer. + Args: + lora_nickname: The "nick name" of the adapter when referenced in the pipeline. + lora_path: The path to the adapter, either a local path or a Hugging Face repo id. + """ + if self.is_lora_merged and self.cur_adapter_name != lora_nickname: + raise ValueError( + f"LoRA '{self.cur_adapter_name}' is currently merged. " + "Please call 'unmerge_lora_weights' before setting a new LoRA." + ) + + if lora_nickname not in self.lora_adapters and lora_path is None: + raise ValueError( + f"Adapter {lora_nickname} not found in the pipeline. Please provide lora_path to load it." + ) + if not self.lora_initialized: + self.convert_to_lora_layers() + + adapter_updated = False + rank = dist.get_rank() + + should_load = False + if lora_path is not None: + if lora_nickname not in self.loaded_adapter_paths: + should_load = True + elif self.loaded_adapter_paths[lora_nickname] != lora_path: + should_load = True + + if should_load: + adapter_updated = True + self.load_lora_adapter(lora_path, lora_nickname, rank) + + if ( + not adapter_updated + and self.cur_adapter_name == lora_nickname + and self.is_lora_merged + ): + return + self.cur_adapter_name = lora_nickname + + # Merge the new adapter + adapted_count = 0 + for name, layer in self.lora_layers.items(): + lora_A_name = name + ".lora_A" + lora_B_name = name + ".lora_B" + if ( + lora_A_name in self.lora_adapters[lora_nickname] + and lora_B_name in self.lora_adapters[lora_nickname] + ): + layer.set_lora_weights( + self.lora_adapters[lora_nickname][lora_A_name], + self.lora_adapters[lora_nickname][lora_B_name], + lora_path=lora_path, + ) + adapted_count += 1 + else: + if rank == 0: + logger.warning( + "LoRA adapter %s does not contain the weights for layer '%s'. LoRA will not be applied to it.", + lora_path, + name, + ) + layer.disable_lora = True + self.is_lora_merged = True + logger.info( + "Rank %d: LoRA adapter %s applied to %d layers", + rank, + lora_path, + adapted_count, + ) + + def merge_lora_weights(self) -> None: + if self.is_lora_merged: + logger.warning("LoRA weights are already merged") + return + + for name, layer in self.lora_layers.items(): + layer.merge_lora_weights() + logger.info("LoRA weights merged") + self.is_lora_merged = True + + def unmerge_lora_weights(self) -> None: + if not self.is_lora_merged: + logger.warning("LoRA weights are not merged.") + return + + for name, layer in self.lora_layers.items(): + layer.unmerge_lora_weights() + self.is_lora_merged = False diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py new file mode 100644 index 00000000..781d31e5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py @@ -0,0 +1,241 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Inspired by SGLang: https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/model_executor/forward_batch_info.py +""" +Data structures for functional pipeline processing. + +This module defines the dataclasses used to pass state between pipeline components +in a functional manner, reducing the need for explicit parameter passing. +""" + +from __future__ import annotations + +import os +import pprint +from dataclasses import asdict, dataclass, field +from typing import TYPE_CHECKING, Any, Optional + +import PIL.Image +import torch + +from sglang.multimodal_gen.configs.sample.sampling_params import DataType +from sglang.multimodal_gen.configs.sample.teacache import ( + TeaCacheParams, + WanTeaCacheParams, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs + +if TYPE_CHECKING: + + from sglang.multimodal_gen.runtime.utils.perf_logger import RequestTimings + + +@dataclass +class Req: + """ + Complete state passed through the pipeline execution. + + This dataclass contains all information needed during the diffusion pipeline + execution, allowing methods to update specific components without needing + to manage numerous individual parameters. + """ + + # TODO(will): double check that args are separate from server_args + # properly. Also maybe think about providing an abstraction for pipeline + # specific arguments. + data_type: DataType + + request_id: str | None = None + + generator: torch.Generator | list[torch.Generator] | None = None + + # Image inputs + image_path: str | None = None + # Image encoder hidden states + image_embeds: list[torch.Tensor] = field(default_factory=list) + + original_condition_image_size: tuple[int, int] = None + condition_image: torch.Tensor | PIL.Image.Image | None = None + pixel_values: torch.Tensor | PIL.Image.Image | None = None + preprocessed_image: torch.Tensor | None = None + + # Text inputs + prompt: str | list[str] | None = None + negative_prompt: str | list[str] | None = None + prompt_path: str | None = None + output_path: str = "outputs/" + # without extension + output_file_name: str | None = None + output_file_ext: str | None = None + # Primary encoder embeddings + prompt_embeds: list[torch.Tensor] | torch.Tensor = field(default_factory=list) + negative_prompt_embeds: list[torch.Tensor] | None = None + prompt_attention_mask: list[torch.Tensor] | None = None + negative_attention_mask: list[torch.Tensor] | None = None + clip_embedding_pos: list[torch.Tensor] | None = None + clip_embedding_neg: list[torch.Tensor] | None = None + + pooled_embeds: list[torch.Tensor] = field(default_factory=list) + neg_pooled_embeds: list[torch.Tensor] = field(default_factory=list) + + # Additional text-related parameters + max_sequence_length: int | None = None + prompt_template: dict[str, Any] | None = None + do_classifier_free_guidance: bool = False + + # Batch info + num_outputs_per_prompt: int = 1 + seed: int | None = None + seeds: list[int] | None = None + + # Tracking if embeddings are already processed + is_prompt_processed: bool = False + + # Latent tensors + latents: torch.Tensor | None = None + # Flux-2 + latent_ids: torch.Tensor | None = None + + raw_latent_shape: torch.Tensor | None = None + noise_pred: torch.Tensor | None = None + # vae-encoded condition image + image_latent: torch.Tensor | list[torch.Tensor] | None = None + condition_image_latent_ids: torch.Tensor | list[torch.Tensor] | None = None + + # Latent dimensions + height_latents: list[int] | int | None = None + width_latents: list[int] | int | None = None + num_frames: list[int] | int = 1 # Default for image models + num_frames_round_down: bool = ( + False # Whether to round down num_frames if it's not divisible by num_gpus + ) + + # Original dimensions (before VAE scaling) + height: list[int] | int | None = None + width: list[int] | int | None = None + fps: list[int] | int | None = None + height_not_provided: bool = False + width_not_provided: bool = False + + # Timesteps + timesteps: torch.Tensor | None = None + timestep: torch.Tensor | float | int | None = None + step_index: int | None = None + boundary_ratio: float | None = None + + # Scheduler parameters + num_inference_steps: int = 50 + guidance_scale: float = 1.0 + guidance_scale_2: float | None = None + guidance_rescale: float = 0.0 + eta: float = 0.0 + sigmas: list[float] | None = None + + n_tokens: int | None = None + + # Other parameters that may be needed by specific schedulers + extra_step_kwargs: dict[str, Any] = field(default_factory=dict) + + # Component modules (populated by the pipeline) + modules: dict[str, Any] = field(default_factory=dict) + + return_trajectory_latents: bool = False + return_trajectory_decoded: bool = False + trajectory_timesteps: list[torch.Tensor] | None = None + trajectory_latents: torch.Tensor | None = None + + # Extra parameters that might be needed by specific pipeline implementations + extra: dict[str, Any] = field(default_factory=dict) + + # Misc + save_output: bool = True + return_frames: bool = False + + # TeaCache parameters + enable_teacache: bool = False + teacache_params: TeaCacheParams | WanTeaCacheParams | None = None + + # STA parameters + STA_param: list | None = None + is_cfg_negative: bool = False + mask_search_final_result_pos: list[list] | None = None + mask_search_final_result_neg: list[list] | None = None + + # VSA parameters + VSA_sparsity: float = 0.0 + + # stage logging + timings: Optional["RequestTimings"] = None + + # profile + profile: bool = False + num_profiled_timesteps: int = 8 + + # debugging + debug: bool = False + # dummy for now + perf_dump_path: str | None = None + + # results + output: torch.Tensor | None = None + + @property + def batch_size(self): + # Determine batch size + if isinstance(self.prompt, list): + batch_size = len(self.prompt) + elif self.prompt is not None: + batch_size = 1 + else: + batch_size = self.prompt_embeds[0].shape[0] + + # Adjust batch size for number of videos per prompt + batch_size *= self.num_outputs_per_prompt + return batch_size + + def output_file_path(self, num_outputs, output_idx): + output_file_name = self.output_file_name + if num_outputs > 1 and output_file_name: + base, ext = os.path.splitext(output_file_name) + output_file_name = f"{base}_{output_idx}{ext}" + + return ( + os.path.join(self.output_path, output_file_name) + if output_file_name + else None + ) + + def __post_init__(self): + """Initialize dependent fields after dataclass initialization.""" + # Set do_classifier_free_guidance based on guidance scale and negative prompt + if self.guidance_scale > 1.0 and self.negative_prompt is not None: + self.do_classifier_free_guidance = True + if self.negative_prompt_embeds is None: + self.negative_prompt_embeds = [] + if self.guidance_scale_2 is None: + self.guidance_scale_2 = self.guidance_scale + + def adjust_size(self, server_args: ServerArgs): + if self.height is None or self.width is None: + self.width = 1280 + self.height = 720 + + def __str__(self): + return pprint.pformat(asdict(self), indent=2, width=120) + + +@dataclass +class OutputBatch: + """ + Final output (after pipeline completion) + """ + + output: torch.Tensor | None = None + trajectory_timesteps: list[torch.Tensor] | None = None + trajectory_latents: torch.Tensor | None = None + trajectory_decoded: list[torch.Tensor] | None = None + error: str | None = None + + # logged timings info, directly from Req.timings + timings: Optional["RequestTimings"] = None diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/__init__.py new file mode 100644 index 00000000..3c962b0b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/__init__.py @@ -0,0 +1,59 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +Pipeline stages for diffusion models. + +This package contains the various stages that can be composed to create +complete diffusion pipelines. +""" + +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.causal_denoising import ( + CausalDMDDenoisingStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.conditioning import ( + ConditioningStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising_dmd import ( + DmdDenoisingStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.encoding import EncodingStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.image_encoding import ( + ImageEncodingStage, + ImageVAEEncodingStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.input_validation import ( + InputValidationStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.latent_preparation import ( + LatentPreparationStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.stepvideo_encoding import ( + StepvideoPromptEncodingStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import ( + TextEncodingStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.timestep_preparation import ( + TimestepPreparationStage, +) + +__all__ = [ + "PipelineStage", + "InputValidationStage", + "TimestepPreparationStage", + "LatentPreparationStage", + "ConditioningStage", + "DenoisingStage", + "DmdDenoisingStage", + "CausalDMDDenoisingStage", + "EncodingStage", + "DecodingStage", + "ImageEncodingStage", + "ImageVAEEncodingStage", + "TextEncodingStage", + "StepvideoPromptEncodingStage", +] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py new file mode 100644 index 00000000..e79ea187 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py @@ -0,0 +1,233 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +Base classes for pipeline stages. + +This module defines the abstract base classes for pipeline stages that can be +composed to create complete diffusion pipelines. +""" + +from abc import ABC, abstractmethod +from enum import Enum, auto + +import torch + +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + VerificationResult, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs, get_global_server_args +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.runtime.utils.perf_logger import StageProfiler + +logger = init_logger(__name__) + + +class StageParallelismType(Enum): + # execute on all gpus + REPLICATED = auto() + # executed on main rank only + MAIN_RANK_ONLY = auto() + # this stage requires a cfg-parallel + CFG_PARALLEL = auto() + + +class StageVerificationError(Exception): + """Exception raised when stage verification fails.""" + + pass + + +class PipelineStage(ABC): + """ + Abstract base class for all pipeline stages. + + A pipeline stage represents a discrete step in the diffusion process that can be + composed with other stages to create a complete pipeline. Each stage is responsible + for a specific part of the process, such as prompt encoding, latent preparation, etc. + """ + + def __init__(self): + self.server_args = get_global_server_args() + + def log_info(self, msg, *args): + """Logs an informational message with the stage name as a prefix.""" + logger.info(f"[{self.__class__.__name__}] {msg}", *args) + + def log_warning(self, msg, *args): + """Logs a warning message with the stage name as a prefix.""" + logger.warning(f"[{self.__class__.__name__}] {msg}", *args) + + def log_error(self, msg, *args): + """Logs an error message with the stage name as a prefix.""" + logger.error(f"[{self.__class__.__name__}] {msg}", *args) + + def log_debug(self, msg, *args): + """Logs a debug message with the stage name as a prefix.""" + logger.debug(f"[{self.__class__.__name__}] {msg}", *args) + + def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """ + Verify the input for the stage. + + Example: + from sglang.multimodal_gen.runtime.pipelines.stages.validators import V, VerificationResult + + def verify_input(self, batch, server_args): + result = VerificationResult() + result.add_check("height", batch.height, V.positive_int_divisible(8)) + result.add_check("width", batch.width, V.positive_int_divisible(8)) + result.add_check("image_latent", batch.image_latent, V.is_tensor) + return result + + Args: + batch: The current batch information. + server_args: The inference arguments. + + Returns: + A VerificationResult containing the verification status. + + """ + # Default implementation - no verification + return VerificationResult() + + def maybe_free_model_hooks(self): + pass + + # execute on all ranks by default + @property + def parallelism_type(self) -> StageParallelismType: + # if get_global_server_args().enable_cfg_parallel: + # return StageParallelismType.MAIN_RANK_ONLY + return StageParallelismType.REPLICATED + + def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """ + Verify the output for the stage. + + Args: + batch: The current batch information. + server_args: The inference arguments. + + Returns: + A VerificationResult containing the verification status. + """ + # Default implementation - no verification + return VerificationResult() + + def _run_verification( + self, + verification_result: VerificationResult, + stage_name: str, + verification_type: str, + ) -> None: + """ + Run verification and raise errors if any checks fail. + + Args: + verification_result: Results from verify_input or verify_output + stage_name: Name of the current stage + verification_type: "input" or "output" + """ + if not verification_result.is_valid(): + failed_fields = verification_result.get_failed_fields() + if failed_fields: + # Get detailed failure information + detailed_summary = verification_result.get_failure_summary() + + failed_fields_str = ", ".join(failed_fields) + error_msg = ( + f"{verification_type.capitalize()} verification failed for {stage_name}: " + f"Failed fields: {failed_fields_str}\n" + f"Details: {detailed_summary}" + ) + raise StageVerificationError(error_msg) + + @property + def device(self) -> torch.device: + """Get the device for this stage.""" + return torch.device("cuda" if torch.cuda.is_available() else "cpu") + + def set_logging(self, enable: bool): + """ + Enable or disable logging for this stage. + + Args: + enable: Whether to enable logging. + """ + self._enable_logging = enable + + def __call__( + self, + batch: Req, + server_args: ServerArgs, + ) -> Req: + """ + Execute the stage's processing on the batch with optional verification and logging. + Should not be overridden by subclasses. + + Args: + batch: The current batch information. + server_args: The inference arguments. + + Returns: + The updated batch information after this stage's processing. + """ + stage_name = self.__class__.__name__ + # Check if verification is enabled (simple approach for prototype) + enable_verification = getattr(server_args, "enable_stage_verification", False) + + if enable_verification: + # Pre-execution input verification + try: + input_result = self.verify_input(batch, server_args) + self._run_verification(input_result, stage_name, "input") + except Exception as e: + logger.error("Input verification failed for %s: %s", stage_name, str(e)) + raise + + # Execute the actual stage logic with unified profiling + with StageProfiler(stage_name, logger=logger, timings=batch.timings): + result = self.forward(batch, server_args) + + if enable_verification: + # Post-execution output verification + try: + output_result = self.verify_output(result, server_args) + self._run_verification(output_result, stage_name, "output") + except Exception as e: + logger.error( + "Output verification failed for %s: %s", stage_name, str(e) + ) + raise + + return result + + @abstractmethod + def forward( + self, + batch: Req, + server_args: ServerArgs, + ) -> Req: + """ + Forward pass of the stage's processing. + + This method should be implemented by subclasses to provide the forward + processing logic for the stage. + + Args: + batch: The current batch information. + server_args: The inference arguments. + + Returns: + The updated batch information after this stage's processing. + """ + raise NotImplementedError + + def backward( + self, + batch: Req, + server_args: ServerArgs, + ) -> Req: + raise NotImplementedError diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/causal_denoising.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/causal_denoising.py new file mode 100644 index 00000000..d06dc142 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/causal_denoising.py @@ -0,0 +1,508 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +import torch # type: ignore + +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context +from sglang.multimodal_gen.runtime.models.utils import pred_noise_to_pred_video +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + StageValidators as V, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + VerificationResult, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +try: + from sglang.multimodal_gen.runtime.layers.attention.backends.sliding_tile_attn import ( + SlidingTileAttentionBackend, + ) + + st_attn_available = True +except ImportError: + st_attn_available = False + SlidingTileAttentionBackend = None # type: ignore + +try: + from sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn import ( + VideoSparseAttentionBackend, + ) + + vsa_available = True +except ImportError: + vsa_available = False + VideoSparseAttentionBackend = None # type: ignore + +logger = init_logger(__name__) + + +class CausalDMDDenoisingStage(DenoisingStage): + """ + Denoising stage for causal diffusion. + """ + + def __init__(self, transformer, scheduler) -> None: + super().__init__(transformer, scheduler) + # KV and cross-attention cache state (initialized on first forward) + self.kv_cache1: list | None = None + self.crossattn_cache: list | None = None + # Model-dependent constants (aligned with causal_inference.py assumptions) + self.num_transformer_blocks = self.transformer.config.arch_config.num_layers + self.num_frames_per_block = ( + self.transformer.config.arch_config.num_frames_per_block + ) + self.sliding_window_num_frames = ( + self.transformer.config.arch_config.sliding_window_num_frames + ) + + try: + self.local_attn_size = getattr( + self.transformer.model, "local_attn_size", -1 + ) # type: ignore + except Exception: + self.local_attn_size = -1 + + def forward( + self, + batch: Req, + server_args: ServerArgs, + ) -> Req: + target_dtype = torch.bfloat16 + autocast_enabled = ( + target_dtype != torch.float32 + ) and not server_args.disable_autocast + + latent_seq_length = batch.latents.shape[-1] * batch.latents.shape[-2] + patch_ratio = ( + self.transformer.config.arch_config.patch_size[-1] + * self.transformer.config.arch_config.patch_size[-2] + ) + self.frame_seq_length = latent_seq_length // patch_ratio + # TODO(will): make this a parameter once we add i2v support + independent_first_frame = self.transformer.independent_first_frame + + # Timesteps for DMD + timesteps = torch.tensor( + server_args.pipeline_config.dmd_denoising_steps, dtype=torch.long + ).cpu() + + if server_args.pipeline_config.warp_denoising_step: + logger.info("Warping timesteps...") + scheduler_timesteps = torch.cat( + (self.scheduler.timesteps.cpu(), torch.tensor([0], dtype=torch.float32)) + ) + timesteps = scheduler_timesteps[1000 - timesteps] + timesteps = timesteps.to(get_local_torch_device()) + logger.info("Using timesteps: %s", timesteps) + + # Image kwargs (kept empty unless caller provides compatible args) + image_kwargs: dict = {} + + pos_cond_kwargs = self.prepare_extra_func_kwargs( + self.transformer.forward, + { + # "encoder_hidden_states_2": batch.clip_embedding_pos, + "encoder_attention_mask": batch.prompt_attention_mask, + }, + ) + + # STA + if st_attn_available and self.attn_backend == SlidingTileAttentionBackend: + self.prepare_sta_param(batch, server_args) + + # Latents and prompts + assert batch.latents is not None, "latents must be provided" + latents = batch.latents # [B, C, T, H, W] + b, c, t, h, w = latents.shape + prompt_embeds = batch.prompt_embeds + assert torch.isnan(prompt_embeds[0]).sum() == 0 + + # Initialize or reset caches + if self.kv_cache1 is None: + self._initialize_kv_cache( + batch_size=latents.shape[0], dtype=target_dtype, device=latents.device + ) + self._initialize_crossattn_cache( + batch_size=latents.shape[0], + max_text_len=server_args.pipeline_config.text_encoder_configs[ + 0 + ].arch_config.text_len, + dtype=target_dtype, + device=latents.device, + ) + else: + assert self.crossattn_cache is not None + # reset cross-attention cache + for block_index in range(self.num_transformer_blocks): + self.crossattn_cache[block_index]["is_init"] = False # type: ignore + # reset kv cache pointers + for block_index in range(len(self.kv_cache1)): + self.kv_cache1[block_index]["global_end_index"] = ( + torch.tensor( # type: ignore + [0], dtype=torch.long, device=latents.device + ) + ) + self.kv_cache1[block_index]["local_end_index"] = ( + torch.tensor( # type: ignore + [0], dtype=torch.long, device=latents.device + ) + ) + + # Optional: cache context features from provided image latents prior to generation + current_start_frame = 0 + if getattr(batch, "image_latent", None) is not None: + image_latent = batch.image_latent + assert image_latent is not None + input_frames = image_latent.shape[2] + # timestep zero (or configured context noise) for cache warm-up + t_zero = torch.zeros( + [latents.shape[0]], device=latents.device, dtype=torch.long + ) + if independent_first_frame and input_frames >= 1: + # warm-up with the very first frame independently + image_first_btchw = ( + image_latent[:, :, :1, :, :].to(target_dtype).permute(0, 2, 1, 3, 4) + ) + with torch.autocast( + device_type="cuda", dtype=target_dtype, enabled=autocast_enabled + ): + _ = self.transformer( + image_first_btchw, + prompt_embeds, + t_zero, + kv_cache=self.kv_cache1, + crossattn_cache=self.crossattn_cache, + current_start=current_start_frame * self.frame_seq_length, + **image_kwargs, + **pos_cond_kwargs, + ) + current_start_frame += 1 + remaining_frames = input_frames - 1 + else: + remaining_frames = input_frames + + # process remaining input frames in blocks of num_frame_per_block + while remaining_frames > 0: + block = min(self.num_frames_per_block, remaining_frames) + ref_btchw = ( + image_latent[ + :, :, current_start_frame : current_start_frame + block, :, : + ] + .to(target_dtype) + .permute(0, 2, 1, 3, 4) + ) + with torch.autocast( + device_type="cuda", dtype=target_dtype, enabled=autocast_enabled + ): + _ = self.transformer( + ref_btchw, + prompt_embeds, + t_zero, + kv_cache=self.kv_cache1, + crossattn_cache=self.crossattn_cache, + current_start=current_start_frame * self.frame_seq_length, + **image_kwargs, + **pos_cond_kwargs, + ) + current_start_frame += block + remaining_frames -= block + + # Base position offset from any cache warm-up + pos_start_base = current_start_frame + + # Determine block sizes + if not independent_first_frame or ( + independent_first_frame and batch.image_latent is not None + ): + if t % self.num_frames_per_block != 0: + raise ValueError( + "num_frames must be divisible by num_frames_per_block for causal DMD denoising" + ) + num_blocks = t // self.num_frames_per_block + block_sizes = [self.num_frames_per_block] * num_blocks + start_index = 0 + else: + if (t - 1) % self.num_frames_per_block != 0: + raise ValueError( + "(num_frames - 1) must be divisible by num_frame_per_block when independent_first_frame=True" + ) + num_blocks = (t - 1) // self.num_frames_per_block + block_sizes = [1] + [self.num_frames_per_block] * num_blocks + start_index = 0 + + # DMD loop in causal blocks + with self.progress_bar(total=len(block_sizes) * len(timesteps)) as progress_bar: + for current_num_frames in block_sizes: + current_latents = latents[ + :, :, start_index : start_index + current_num_frames, :, : + ] + # use BTCHW for DMD conversion routines + noise_latents_btchw = current_latents.permute(0, 2, 1, 3, 4) + video_raw_latent_shape = noise_latents_btchw.shape + + for i, t_cur in enumerate(timesteps): + # Copy for pred conversion + noise_latents = noise_latents_btchw.clone() + latent_model_input = current_latents.to(target_dtype) + + if ( + batch.image_latent is not None + and independent_first_frame + and start_index == 0 + ): + latent_model_input = torch.cat( + [latent_model_input, batch.image_latent.to(target_dtype)], + dim=2, + ) + + # Prepare inputs + t_expand = t_cur.repeat(latent_model_input.shape[0]) + + # Attention metadata if needed + if ( + vsa_available + and self.attn_backend == VideoSparseAttentionBackend + ): + self.attn_metadata_builder_cls = ( + self.attn_backend.get_builder_cls() + ) + if self.attn_metadata_builder_cls is not None: + self.attn_metadata_builder = ( + self.attn_metadata_builder_cls() + ) + attn_metadata = self.attn_metadata_builder.build( # type: ignore + current_timestep=i, # type: ignore + raw_latent_shape=( + current_num_frames, + h, + w, + ), # type: ignore + patch_size=server_args.pipeline_config.dit_config.patch_size, # type: ignore + STA_param=batch.STA_param, # type: ignore + VSA_sparsity=server_args.VSA_sparsity, # type: ignore + device=get_local_torch_device(), # type: ignore + ) # type: ignore + assert ( + attn_metadata is not None + ), "attn_metadata cannot be None" + else: + attn_metadata = None + else: + attn_metadata = None + + with ( + torch.autocast( + device_type="cuda", + dtype=target_dtype, + enabled=autocast_enabled, + ), + set_forward_context( + current_timestep=i, + attn_metadata=attn_metadata, + forward_batch=batch, + ), + ): + # Run transformer; follow DMD stage pattern + t_expanded_noise = t_cur * torch.ones( + (latent_model_input.shape[0], 1), + device=latent_model_input.device, + dtype=torch.long, + ) + pred_noise_btchw = self.transformer( + latent_model_input, + prompt_embeds, + t_expanded_noise, + kv_cache=self.kv_cache1, + crossattn_cache=self.crossattn_cache, + current_start=(pos_start_base + start_index) + * self.frame_seq_length, + start_frame=start_index, + **image_kwargs, + **pos_cond_kwargs, + ).permute(0, 2, 1, 3, 4) + + # Convert pred noise to pred video with FM Euler scheduler utilities + pred_video_btchw = pred_noise_to_pred_video( + pred_noise=pred_noise_btchw.flatten(0, 1), + noise_input_latent=noise_latents.flatten(0, 1), + timestep=t_expand, + scheduler=self.scheduler, + ).unflatten(0, pred_noise_btchw.shape[:2]) + + if i < len(timesteps) - 1: + next_timestep = timesteps[i + 1] * torch.ones( + [1], dtype=torch.long, device=pred_video_btchw.device + ) + noise = torch.randn( + video_raw_latent_shape, + dtype=pred_video_btchw.dtype, + generator=( + batch.generator[0] + if isinstance(batch.generator, list) + else batch.generator + ), + ).to(self.device) + noise_btchw = noise + noise_latents_btchw = self.scheduler.add_noise( + pred_video_btchw.flatten(0, 1), + noise_btchw.flatten(0, 1), + next_timestep, + ).unflatten(0, pred_video_btchw.shape[:2]) + current_latents = noise_latents_btchw.permute(0, 2, 1, 3, 4) + else: + current_latents = pred_video_btchw.permute(0, 2, 1, 3, 4) + + if progress_bar is not None: + progress_bar.update() + + # Write back and advance + latents[:, :, start_index : start_index + current_num_frames, :, :] = ( + current_latents + ) + + # Re-run with context timestep to update KV cache using clean context + context_noise = getattr(server_args.pipeline_config, "context_noise", 0) + t_context = torch.ones( + [latents.shape[0]], device=latents.device, dtype=torch.long + ) * int(context_noise) + context_bcthw = current_latents.to(target_dtype) + with ( + torch.autocast( + device_type="cuda", dtype=target_dtype, enabled=autocast_enabled + ), + set_forward_context( + current_timestep=0, + attn_metadata=attn_metadata, + forward_batch=batch, + ), + ): + t_expanded_context = t_context.unsqueeze(1) + _ = self.transformer( + context_bcthw, + prompt_embeds, + t_expanded_context, + kv_cache=self.kv_cache1, + crossattn_cache=self.crossattn_cache, + current_start=(pos_start_base + start_index) + * self.frame_seq_length, + start_frame=start_index, + **image_kwargs, + **pos_cond_kwargs, + ) + start_index += current_num_frames + + batch.latents = latents + return batch + + def _initialize_kv_cache(self, batch_size, dtype, device) -> None: + """ + Initialize a Per-GPU KV cache aligned with the Wan model assumptions. + """ + kv_cache1 = [] + num_attention_heads = self.transformer.num_attention_heads + attention_head_dim = self.transformer.attention_head_dim + if self.local_attn_size != -1: + kv_cache_size = self.local_attn_size * self.frame_seq_length + else: + kv_cache_size = self.frame_seq_length * self.sliding_window_num_frames + + for _ in range(self.num_transformer_blocks): + kv_cache1.append( + { + "k": torch.zeros( + [ + batch_size, + kv_cache_size, + num_attention_heads, + attention_head_dim, + ], + dtype=dtype, + device=device, + ), + "v": torch.zeros( + [ + batch_size, + kv_cache_size, + num_attention_heads, + attention_head_dim, + ], + dtype=dtype, + device=device, + ), + "global_end_index": torch.tensor( + [0], dtype=torch.long, device=device + ), + "local_end_index": torch.tensor( + [0], dtype=torch.long, device=device + ), + } + ) + + self.kv_cache1 = kv_cache1 + + def _initialize_crossattn_cache( + self, batch_size, max_text_len, dtype, device + ) -> None: + """ + Initialize a Per-GPU cross-attention cache aligned with the Wan model assumptions. + """ + crossattn_cache = [] + num_attention_heads = self.transformer.num_attention_heads + attention_head_dim = self.transformer.attention_head_dim + for _ in range(self.num_transformer_blocks): + crossattn_cache.append( + { + "k": torch.zeros( + [ + batch_size, + max_text_len, + num_attention_heads, + attention_head_dim, + ], + dtype=dtype, + device=device, + ), + "v": torch.zeros( + [ + batch_size, + max_text_len, + num_attention_heads, + attention_head_dim, + ], + dtype=dtype, + device=device, + ), + "is_init": False, + } + ) + self.crossattn_cache = crossattn_cache + + def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Verify denoising stage inputs.""" + result = VerificationResult() + result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)]) + result.add_check("prompt_embeds", batch.prompt_embeds, V.list_not_empty) + result.add_check("image_embeds", batch.image_embeds, V.is_list) + result.add_check( + "image_latent", batch.image_latent, V.none_or_tensor_with_dims(5) + ) + result.add_check( + "num_inference_steps", batch.num_inference_steps, V.positive_int + ) + result.add_check("guidance_scale", batch.guidance_scale, V.non_negative_float) + result.add_check("eta", batch.eta, V.non_negative_float) + result.add_check("generator", batch.generator, V.generator_or_list_generators) + result.add_check( + "do_classifier_free_guidance", + batch.do_classifier_free_guidance, + V.bool_value, + ) + result.add_check( + "negative_prompt_embeds", + batch.negative_prompt_embeds, + lambda x: not batch.do_classifier_free_guidance or V.list_not_empty(x), + ) + return result diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/conditioning.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/conditioning.py new file mode 100644 index 00000000..f0cb4569 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/conditioning.py @@ -0,0 +1,107 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +Conditioning stage for diffusion pipelines. +""" + +import torch + +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + StageValidators as V, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + VerificationResult, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class ConditioningStage(PipelineStage): + """ + Stage for applying conditioning to the diffusion process. + + This stage handles the application of conditioning, such as classifier-free guidance, + to the diffusion process. + """ + + @torch.no_grad() + def forward( + self, + batch: Req, + server_args: ServerArgs, + ) -> Req: + """ + Apply conditioning to the diffusion process. + + Args: + batch: The current batch information. + server_args: The inference arguments. + + Returns: + The batch with applied conditioning. + """ + # TODO!! + if not batch.do_classifier_free_guidance: + return batch + else: + return batch + + logger.info("batch.negative_prompt_embeds: %s", batch.negative_prompt_embeds) + logger.info( + "do_classifier_free_guidance: %s", batch.do_classifier_free_guidance + ) + logger.info("cfg_scale: %s", batch.guidance_scale) + + # Ensure negative prompt embeddings are available + assert ( + batch.negative_prompt_embeds is not None + ), "Negative prompt embeddings are required for classifier-free guidance" + + # Concatenate primary embeddings and masks + batch.prompt_embeds = torch.cat( + [batch.negative_prompt_embeds, batch.prompt_embeds] + ) + if batch.attention_mask is not None: + batch.attention_mask = torch.cat( + [batch.negative_attention_mask, batch.attention_mask] + ) + + # Concatenate secondary embeddings and masks if present + if batch.prompt_embeds_2 is not None: + batch.prompt_embeds_2 = torch.cat( + [batch.negative_prompt_embeds_2, batch.prompt_embeds_2] + ) + if batch.attention_mask_2 is not None: + batch.attention_mask_2 = torch.cat( + [batch.negative_attention_mask_2, batch.attention_mask_2] + ) + + return batch + + def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Verify conditioning stage inputs.""" + result = VerificationResult() + result.add_check( + "do_classifier_free_guidance", + batch.do_classifier_free_guidance, + V.bool_value, + ) + result.add_check("guidance_scale", batch.guidance_scale, V.non_negative_float) + result.add_check("prompt_embeds", batch.prompt_embeds, V.list_not_empty) + result.add_check( + "negative_prompt_embeds", + batch.negative_prompt_embeds, + lambda x: not batch.do_classifier_free_guidance or V.list_not_empty(x), + ) + return result + + def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Verify conditioning stage outputs.""" + result = VerificationResult() + result.add_check("prompt_embeds", batch.prompt_embeds, V.list_not_empty) + return result diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py new file mode 100644 index 00000000..2d9cf7f0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py @@ -0,0 +1,213 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +Decoding stage for diffusion pipelines. +""" + +import weakref + +import torch + +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.loader.component_loader import VAELoader +from sglang.multimodal_gen.runtime.models.vaes.common import ParallelTiledVAE +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import ( + PipelineStage, + StageParallelismType, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + VerificationResult, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs, get_global_server_args +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.utils import PRECISION_TO_TYPE + +logger = init_logger(__name__) + + +class DecodingStage(PipelineStage): + """ + Stage for decoding latent representations into pixel space. + + This stage handles the decoding of latent representations into the final + output format (e.g., pixel values). + """ + + def __init__(self, vae, pipeline=None) -> None: + self.vae: ParallelTiledVAE = vae + self.pipeline = weakref.ref(pipeline) if pipeline else None + + @property + def parallelism_type(self) -> StageParallelismType: + if get_global_server_args().enable_cfg_parallel: + return StageParallelismType.MAIN_RANK_ONLY + return StageParallelismType.REPLICATED + + def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Verify decoding stage inputs.""" + result = VerificationResult() + # Denoised latents for VAE decoding: [batch_size, channels, frames, height_latents, width_latents] + # result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)]) + return result + + def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Verify decoding stage outputs.""" + result = VerificationResult() + # Decoded video/images: [batch_size, channels, frames, height, width] + # result.add_check("output", batch.output, [V.is_tensor, V.with_dims(5)]) + return result + + def scale_and_shift(self, latents: torch.Tensor, server_args): + scaling_factor, shift_factor = ( + server_args.pipeline_config.get_decode_scale_and_shift( + latents.device, latents.dtype, self.vae + ) + ) + + # 1. scale + if isinstance(scaling_factor, torch.Tensor): + latents = latents / scaling_factor.to(latents.device, latents.dtype) + else: + latents = latents / scaling_factor + + # 2. apply shifting if needed + if shift_factor is not None: + if isinstance(shift_factor, torch.Tensor): + latents += shift_factor.to(latents.device, latents.dtype) + else: + latents += shift_factor + return latents + + @torch.no_grad() + def decode(self, latents: torch.Tensor, server_args: ServerArgs) -> torch.Tensor: + """ + Decode latent representations into pixel space using VAE. + + Args: + latents: Input latent tensor with shape (batch, channels, frames, height_latents, width_latents) + server_args: Configuration containing: + - disable_autocast: Whether to disable automatic mixed precision (default: False) + - pipeline_config.vae_precision: VAE computation precision ("fp32", "fp16", "bf16") + - pipeline_config.vae_tiling: Whether to enable VAE tiling for memory efficiency + + Returns: + Decoded video tensor with shape (batch, channels, frames, height, width), + normalized to [0, 1] range and moved to CPU as float32 + """ + self.vae = self.vae.to(get_local_torch_device()) + latents = latents.to(get_local_torch_device()) + # Setup VAE precision + vae_dtype = PRECISION_TO_TYPE[server_args.pipeline_config.vae_precision] + vae_autocast_enabled = ( + vae_dtype != torch.float32 + ) and not server_args.disable_autocast + + # scale and shift + latents = self.scale_and_shift(latents, server_args) + latents = server_args.pipeline_config.preprocess_decoding(latents) + + # Decode latents + with torch.autocast( + device_type="cuda", dtype=vae_dtype, enabled=vae_autocast_enabled + ): + try: + # TODO: make it more specific + if server_args.pipeline_config.vae_tiling: + self.vae.enable_tiling() + except Exception: + pass + if not vae_autocast_enabled: + latents = latents.to(vae_dtype) + image = self.vae.decode(latents) + + # De-normalize image to [0, 1] range + image = (image / 2 + 0.5).clamp(0, 1) + return image + + @torch.no_grad() + def forward( + self, + batch: Req, + server_args: ServerArgs, + ) -> OutputBatch: + """ + Decode latent representations into pixel space. + + This method processes the batch through the VAE decoder, converting latent + representations to pixel-space video/images. It also optionally decodes + trajectory latents for visualization purposes. + + Args: + batch: The current batch containing: + - latents: Tensor to decode (batch, channels, frames, height_latents, width_latents) + - return_trajectory_decoded (optional): Flag to decode trajectory latents + - trajectory_latents (optional): Latents at different timesteps + - trajectory_timesteps (optional): Corresponding timesteps + server_args: Configuration containing: + - output_type: "latent" to skip decoding, otherwise decode to pixels + - vae_cpu_offload: Whether to offload VAE to CPU after decoding + - model_loaded: Track VAE loading state + - model_paths: Path to VAE model if loading needed + + Returns: + Modified batch with: + - output: Decoded frames (batch, channels, frames, height, width) as CPU float32 + - trajectory_decoded (if requested): List of decoded frames per timestep + """ + # load vae if not already loaded (used for memory constrained devices) + pipeline = self.pipeline() if self.pipeline else None + if not server_args.model_loaded["vae"]: + loader = VAELoader() + self.vae = loader.load(server_args.model_paths["vae"], server_args) + if pipeline: + pipeline.add_module("vae", self.vae) + server_args.model_loaded["vae"] = True + + if server_args.output_type == "latent": + frames = batch.latents + else: + frames = self.decode(batch.latents, server_args) + + # decode trajectory latents if needed + if batch.return_trajectory_decoded: + trajectory_decoded = [] + assert ( + batch.trajectory_latents is not None + ), "batch should have trajectory latents" + for idx in range(batch.trajectory_latents.shape[1]): + # batch.trajectory_latents is [batch_size, timesteps, channels, frames, height, width] + cur_latent = batch.trajectory_latents[:, idx, :, :, :, :] + cur_timestep = batch.trajectory_timesteps[idx] + logger.info("decoding trajectory latent for timestep: %s", cur_timestep) + decoded_frames = self.decode(cur_latent, server_args) + trajectory_decoded.append(decoded_frames.cpu().float()) + else: + trajectory_decoded = None + + # Convert to CPU float32 for compatibility + frames = frames.cpu().float() + + # Update batch with decoded image + output_batch = OutputBatch( + output=frames, + trajectory_timesteps=batch.trajectory_timesteps, + trajectory_latents=batch.trajectory_latents, + trajectory_decoded=trajectory_decoded, + timings=batch.timings, + ) + + # Offload models if needed + self.maybe_free_model_hooks() + + if server_args.vae_cpu_offload: + self.vae.to("cpu") + + if torch.backends.mps.is_available(): + del self.vae + if pipeline is not None and "vae" in pipeline.modules: + del pipeline.modules["vae"] + server_args.model_loaded["vae"] = False + + return output_batch diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py new file mode 100644 index 00000000..aae4af50 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py @@ -0,0 +1,1387 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +Denoising stage for diffusion pipelines. +""" + +import inspect +import math +import os +import time +import weakref +from collections.abc import Iterable +from functools import lru_cache +from typing import Any + +import torch +import torch.profiler +from einops import rearrange +from tqdm.auto import tqdm + +from sglang.multimodal_gen.configs.pipeline_configs.base import ModelTaskType, STA_Mode +from sglang.multimodal_gen.configs.pipeline_configs.wan import Wan2_2_TI2V_5B_Config +from sglang.multimodal_gen.runtime.distributed import ( + cfg_model_parallel_all_reduce, + get_local_torch_device, + get_sp_parallel_rank, + get_sp_world_size, + get_world_group, +) +from sglang.multimodal_gen.runtime.distributed.communication_op import ( + sequence_model_parallel_all_gather, +) +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + get_cfg_group, + get_classifier_free_guidance_rank, + get_world_rank, +) +from sglang.multimodal_gen.runtime.layers.attention.backends.flash_attn import ( + FlashAttentionBackend, +) +from sglang.multimodal_gen.runtime.layers.attention.selector import get_attn_backend +from sglang.multimodal_gen.runtime.layers.attention.STA_configuration import ( + configure_sta, + save_mask_search_results, +) +from sglang.multimodal_gen.runtime.loader.component_loader import TransformerLoader +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import ( + PipelineStage, + StageParallelismType, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + StageValidators as V, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + VerificationResult, +) +from sglang.multimodal_gen.runtime.platforms.interface import AttentionBackendEnum +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.runtime.utils.perf_logger import StageProfiler +from sglang.multimodal_gen.utils import dict_to_3d_list, masks_like + +try: + from sglang.multimodal_gen.runtime.layers.attention.backends.sliding_tile_attn import ( + SlidingTileAttentionBackend, + ) + + st_attn_available = True +except ImportError: + st_attn_available = False + +try: + from sglang.multimodal_gen.runtime.layers.attention.backends.vmoba import ( + VMOBAAttentionBackend, + ) + from sglang.multimodal_gen.utils import is_vmoba_available + + vmoba_attn_available = is_vmoba_available() +except ImportError: + vmoba_attn_available = False + +try: + from sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn import ( + VideoSparseAttentionBackend, + ) + + vsa_available = True +except ImportError: + vsa_available = False + +logger = init_logger(__name__) + + +class DenoisingStage(PipelineStage): + """ + Stage for running the denoising loop in diffusion pipelines. + + This stage handles the iterative denoising process that transforms + the initial noise into the final output. + """ + + def __init__( + self, transformer, scheduler, pipeline=None, transformer_2=None, vae=None + ) -> None: + super().__init__() + self.transformer = transformer + self.transformer_2 = transformer_2 + + hidden_size = self.server_args.pipeline_config.dit_config.hidden_size + num_attention_heads = ( + self.server_args.pipeline_config.dit_config.num_attention_heads + ) + attn_head_size = hidden_size // num_attention_heads + + # torch compile + if self.server_args.enable_torch_compile: + full_graph = False + self.transformer = torch.compile( + self.transformer, mode="max-autotune", fullgraph=full_graph + ) + self.transformer_2 = ( + torch.compile( + self.transformer_2, mode="max-autotune", fullgraph=full_graph + ) + if transformer_2 is not None + else None + ) + + self.scheduler = scheduler + self.vae = vae + self.pipeline = weakref.ref(pipeline) if pipeline else None + + self.attn_backend = get_attn_backend( + head_size=attn_head_size, + dtype=torch.float16, # TODO(will): hack + supported_attention_backends={ + AttentionBackendEnum.SLIDING_TILE_ATTN, + AttentionBackendEnum.VIDEO_SPARSE_ATTN, + AttentionBackendEnum.VMOBA_ATTN, + AttentionBackendEnum.FA, + AttentionBackendEnum.TORCH_SDPA, + AttentionBackendEnum.SAGE_ATTN_THREE, + }, # hack + ) + + # cfg + self.guidance = None + + # misc + self.profiler = None + + @lru_cache(maxsize=8) + def _build_guidance(self, batch_size, target_dtype, device, guidance_val): + """Builds a guidance tensor. This method is cached.""" + return ( + torch.full( + (batch_size,), + guidance_val, + dtype=torch.float32, + device=device, + ).to(target_dtype) + * 1000.0 + ) + + def get_or_build_guidance(self, bsz: int, dtype, device): + """ + Get the guidance tensor, using a cached version if available. + + This method retrieves a cached guidance tensor using `_build_guidance`. + The caching is based on batch size, dtype, device, and the guidance value, + preventing repeated tensor creation within the denoising loop. + """ + if self.server_args.pipeline_config.should_use_guidance: + # TODO: should the guidance_scale be picked-up from sampling_params? + guidance_val = self.server_args.pipeline_config.embedded_cfg_scale + return self._build_guidance(bsz, dtype, device, guidance_val) + else: + return None + + @property + def parallelism_type(self) -> StageParallelismType: + # return StageParallelismType.CFG_PARALLEL if get_global_server_args().enable_cfg_parallel else StageParallelismType.REPLICATED + return StageParallelismType.REPLICATED + + def _preprocess_latents_for_ti2v( + self, latents, target_dtype, batch, server_args: ServerArgs + ): + # FIXME: should probably move to latent preparation stage, to handle with offload + # Wan2.2 TI2V directly replaces the first frame of the latent with + # the image latent instead of appending along the channel dim + assert batch.image_latent is None, "TI2V task should not have image latents" + assert self.vae is not None, "VAE is not provided for TI2V task" + self.vae = self.vae.to(batch.condition_image.device) + z = self.vae.encode(batch.condition_image).mean.float() + if self.vae.device != "cpu" and server_args.vae_cpu_offload: + self.vae = self.vae.to("cpu") + if hasattr(self.vae, "shift_factor") and self.vae.shift_factor is not None: + if isinstance(self.vae.shift_factor, torch.Tensor): + z -= self.vae.shift_factor.to(z.device, z.dtype) + else: + z -= self.vae.shift_factor + + if isinstance(self.vae.scaling_factor, torch.Tensor): + z = z * self.vae.scaling_factor.to(z.device, z.dtype) + else: + z = z * self.vae.scaling_factor + # z: [B, C, 1, H, W] + latent_model_input = latents.to(target_dtype) + # Keep as [B, C, T, H, W] for proper broadcasting + assert latent_model_input.ndim == 5 + + # Create mask with proper shape [B, C, T, H, W] + latent_for_mask = latent_model_input.squeeze(0) # [C, T, H, W] + _, reserved_frames_masks = masks_like([latent_for_mask], zero=True) + reserved_frames_mask = reserved_frames_masks[0].unsqueeze(0) # [1, C, T, H, W] + + # replace GLOBAL first frame with image - proper broadcasting + # z: [B, C, 1, H, W], reserved_frames_mask: [1, C, T, H, W] + # Both will broadcast correctly + latents = ( + 1.0 - reserved_frames_mask + ) * z + reserved_frames_mask * latent_model_input + assert latents.ndim == 5 + latents = latents.to(get_local_torch_device()) + batch.latents = latents + + F = batch.num_frames + temporal_scale = ( + server_args.pipeline_config.vae_config.arch_config.scale_factor_temporal + ) + spatial_scale = ( + server_args.pipeline_config.vae_config.arch_config.scale_factor_spatial + ) + patch_size = server_args.pipeline_config.dit_config.arch_config.patch_size + seq_len = ( + ((F - 1) // temporal_scale + 1) + * (batch.height // spatial_scale) + * (batch.width // spatial_scale) + // (patch_size[1] * patch_size[2]) + ) + seq_len = int(math.ceil(seq_len / get_sp_world_size())) * get_sp_world_size() + return seq_len, z, reserved_frames_masks + + def _postprocess_latents_for_ti2v(self, z, reserved_frames_masks, batch): + rank_in_sp_group = get_sp_parallel_rank() + sp_world_size = get_sp_world_size() + + if getattr(batch, "did_sp_shard_latents", False): + # Shard z (image latent) along time dimension + # z shape: [1, C, 1, H, W] - only first frame + # Only rank 0 has the first frame after sharding + if z.shape[2] == 1: + # z is single frame, only rank 0 needs it + if rank_in_sp_group == 0: + z_sp = z + else: + # Other ranks don't have the first frame + z_sp = None + else: + # Should not happen for TI2V + z_sp = z + + # Shard reserved_frames_mask along time dimension to match sharded latents + # reserved_frames_mask is a list from masks_like, extract reserved_frames_mask[0] first + # reserved_frames_mask[0] shape: [C, T, H, W] + # All ranks need their portion of reserved_frames_mask for timestep calculation + if reserved_frames_masks is not None: + reserved_frames_mask = reserved_frames_masks[ + 0 + ] # Extract tensor from list + time_dim = reserved_frames_mask.shape[1] # [C, T, H, W] + if time_dim > 0 and time_dim % sp_world_size == 0: + reserved_frames_mask_sp_tensor = rearrange( + reserved_frames_mask, + "c (n t) h w -> c n t h w", + n=sp_world_size, + ).contiguous() + reserved_frames_mask_sp_tensor = reserved_frames_mask_sp_tensor[ + :, rank_in_sp_group, :, :, : + ] + reserved_frames_mask_sp = ( + reserved_frames_mask_sp_tensor # Store as tensor, not list + ) + else: + reserved_frames_mask_sp = reserved_frames_mask + else: + reserved_frames_mask_sp = None + else: + # SP not enabled or latents not sharded + z_sp = z + reserved_frames_mask_sp = ( + reserved_frames_masks[0] if reserved_frames_masks is not None else None + ) # Extract tensor + + return reserved_frames_mask_sp, z_sp + + def _prepare_denoising_loop(self, batch: Req, server_args: ServerArgs): + """ + Prepare all necessary invariant variables for the denoising loop. + + Args: + batch: The current batch information. + server_args: The inference arguments. + + Returns: + A dictionary containing all the prepared variables for the denoising loop. + """ + pipeline = self.pipeline() if self.pipeline else None + if not server_args.model_loaded["transformer"]: + loader = TransformerLoader() + self.transformer = loader.load( + server_args.model_paths["transformer"], server_args + ) + if self.server_args.enable_torch_compile: + self.transformer = torch.compile( + self.transformer, mode="max-autotune", fullgraph=True + ) + if pipeline: + pipeline.add_module("transformer", self.transformer) + server_args.model_loaded["transformer"] = True + + # Prepare extra step kwargs for scheduler + extra_step_kwargs = self.prepare_extra_func_kwargs( + self.scheduler.step, + {"generator": batch.generator, "eta": batch.eta}, + ) + + # Setup precision and autocast settings + target_dtype = torch.bfloat16 + autocast_enabled = ( + target_dtype != torch.float32 + ) and not server_args.disable_autocast + + # Get timesteps and calculate warmup steps + timesteps = batch.timesteps + if timesteps is None: + raise ValueError("Timesteps must be provided") + num_inference_steps = batch.num_inference_steps + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + + # Prepare image latents and embeddings for I2V generation + image_embeds = batch.image_embeds + if len(image_embeds) > 0: + image_embeds = [ + image_embed.to(target_dtype) for image_embed in image_embeds + ] + + # Prepare STA parameters + if st_attn_available and self.attn_backend == SlidingTileAttentionBackend: + self.prepare_sta_param(batch, server_args) + + # Get latents and embeddings + latents = batch.latents + prompt_embeds = batch.prompt_embeds + # Removed Tensor truthiness assert to avoid GPU sync + neg_prompt_embeds = None + if batch.do_classifier_free_guidance: + neg_prompt_embeds = batch.negative_prompt_embeds + assert neg_prompt_embeds is not None + # Removed Tensor truthiness assert to avoid GPU sync + + # (Wan2.2) Calculate timestep to switch from high noise expert to low noise expert + boundary_ratio = server_args.pipeline_config.dit_config.boundary_ratio + if batch.boundary_ratio is not None: + logger.info( + "Overriding boundary ratio from %s to %s", + boundary_ratio, + batch.boundary_ratio, + ) + boundary_ratio = batch.boundary_ratio + + if boundary_ratio is not None: + boundary_timestep = boundary_ratio * self.scheduler.num_train_timesteps + else: + boundary_timestep = None + + # specifically for Wan2_2_TI2V_5B_Config, not applicable for FastWan2_2_TI2V_5B_Config + should_preprocess_for_wan_ti2v = ( + server_args.pipeline_config.task_type == ModelTaskType.TI2V + and batch.condition_image is not None + and type(server_args.pipeline_config) is Wan2_2_TI2V_5B_Config + ) + + # TI2V specific preparations - before SP sharding + if should_preprocess_for_wan_ti2v: + seq_len, z, reserved_frames_masks = self._preprocess_latents_for_ti2v( + latents, target_dtype, batch, server_args + ) + else: + seq_len, z, reserved_frames_masks = ( + None, + None, + None, + ) + + # Handle sequence parallelism after TI2V processing + self._preprocess_sp_latents(batch, server_args) + latents = batch.latents + + # Shard z and reserved_frames_mask for TI2V if SP is enabled + if should_preprocess_for_wan_ti2v: + reserved_frames_mask_sp, z_sp = self._postprocess_latents_for_ti2v( + z, reserved_frames_masks, batch + ) + else: + reserved_frames_mask_sp, z_sp = ( + reserved_frames_masks[0] if reserved_frames_masks is not None else None + ), z + + guidance = self.get_or_build_guidance( + # TODO: replace with raw_latent_shape? + latents.shape[0], + latents.dtype, + latents.device, + ) + + image_kwargs = self.prepare_extra_func_kwargs( + self.transformer.forward, + { + # TODO: make sure on-device + "encoder_hidden_states_image": image_embeds, + "mask_strategy": dict_to_3d_list(None, t_max=50, l_max=60, h_max=24), + }, + ) + + pos_cond_kwargs = self.prepare_extra_func_kwargs( + self.transformer.forward, + { + "encoder_hidden_states_2": batch.clip_embedding_pos, + "encoder_attention_mask": batch.prompt_attention_mask, + } + | server_args.pipeline_config.prepare_pos_cond_kwargs( + batch, + self.device, + getattr(self.transformer, "rotary_emb", None), + dtype=target_dtype, + ), + ) + + if batch.do_classifier_free_guidance: + neg_cond_kwargs = self.prepare_extra_func_kwargs( + self.transformer.forward, + { + "encoder_hidden_states_2": batch.clip_embedding_neg, + "encoder_attention_mask": batch.negative_attention_mask, + } + | server_args.pipeline_config.prepare_neg_cond_kwargs( + batch, + self.device, + getattr(self.transformer, "rotary_emb", None), + dtype=target_dtype, + ), + ) + else: + neg_cond_kwargs = {} + + return { + "extra_step_kwargs": extra_step_kwargs, + "target_dtype": target_dtype, + "autocast_enabled": autocast_enabled, + "timesteps": timesteps, + "num_inference_steps": num_inference_steps, + "num_warmup_steps": num_warmup_steps, + "image_kwargs": image_kwargs, + "pos_cond_kwargs": pos_cond_kwargs, + "neg_cond_kwargs": neg_cond_kwargs, + "latents": latents, + "prompt_embeds": prompt_embeds, + "neg_prompt_embeds": neg_prompt_embeds, + "boundary_timestep": boundary_timestep, + "z": z_sp, # Use SP-sharded version + # ndim == 5 + "reserved_frames_mask": reserved_frames_mask_sp, # Use SP-sharded version + "seq_len": seq_len, + "guidance": guidance, + } + + def _post_denoising_loop( + self, + batch: Req, + latents: torch.Tensor, + trajectory_latents: list, + trajectory_timesteps: list, + server_args: ServerArgs, + ): + # Gather results if using sequence parallelism + if trajectory_latents: + trajectory_tensor = torch.stack(trajectory_latents, dim=1) + trajectory_timesteps_tensor = torch.stack(trajectory_timesteps, dim=0) + else: + trajectory_tensor = None + trajectory_timesteps_tensor = None + + # Gather results if using sequence parallelism + latents, trajectory_tensor = self._postprocess_sp_latents( + batch, latents, trajectory_tensor + ) + + if trajectory_tensor is not None and trajectory_timesteps_tensor is not None: + batch.trajectory_timesteps = trajectory_timesteps_tensor.cpu() + batch.trajectory_latents = trajectory_tensor.cpu() + + # Update batch with final latents + batch.latents = self.server_args.pipeline_config.post_denoising_loop( + latents, batch + ) + + # Save STA mask search results if needed + if ( + st_attn_available + and self.attn_backend == SlidingTileAttentionBackend + and server_args.STA_mode == STA_Mode.STA_SEARCHING + ): + self.save_sta_search_results(batch) + + # deallocate transformer if on mps + pipeline = self.pipeline() if self.pipeline else None + if torch.backends.mps.is_available(): + logger.info( + "Memory before deallocating transformer: %s", + torch.mps.current_allocated_memory(), + ) + del self.transformer + if pipeline is not None and "transformer" in pipeline.modules: + del pipeline.modules["transformer"] + server_args.model_loaded["transformer"] = False + logger.info( + "Memory after deallocating transformer: %s", + torch.mps.current_allocated_memory(), + ) + + def _preprocess_sp_latents(self, batch: Req, server_args: ServerArgs): + """Shard latents for Sequence Parallelism if applicable.""" + if get_sp_world_size() <= 1: + return + + if batch.latents is not None: + ( + batch.latents, + did_shard, + ) = server_args.pipeline_config.shard_latents_for_sp(batch, batch.latents) + batch.did_sp_shard_latents = did_shard + else: + batch.did_sp_shard_latents = False + + # For I2I tasks like QwenImageEdit, the image_latent (input image) should be + # replicated on all SP ranks, not sharded, as it provides global context. + if ( + server_args.pipeline_config.task_type != ModelTaskType.I2I + and batch.image_latent is not None + ): + batch.image_latent, _ = server_args.pipeline_config.shard_latents_for_sp( + batch, batch.image_latent + ) + + def _postprocess_sp_latents( + self, + batch: Req, + latents: torch.Tensor, + trajectory_tensor: torch.Tensor | None, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + """Gather latents after Sequence Parallelism if they were sharded.""" + if get_sp_world_size() > 1 and getattr(batch, "did_sp_shard_latents", False): + latents = self.server_args.pipeline_config.gather_latents_for_sp(latents) + if trajectory_tensor is not None: + # trajectory_tensor shapes: + # - video: [b, num_steps, c, t_local, h, w] -> gather on dim=3 + # - image: [b, num_steps, s_local, d] -> gather on dim=2 + trajectory_tensor = trajectory_tensor.to(get_local_torch_device()) + gather_dim = 3 if trajectory_tensor.dim() >= 5 else 2 + trajectory_tensor = sequence_model_parallel_all_gather( + trajectory_tensor, dim=gather_dim + ) + if gather_dim == 2 and hasattr(batch, "raw_latent_shape"): + orig_s = batch.raw_latent_shape[1] + if trajectory_tensor.shape[2] > orig_s: + trajectory_tensor = trajectory_tensor[:, :, :orig_s, :] + return latents, trajectory_tensor + + def start_profile(self, batch: Req): + if not batch.profile: + return + + logger.info("Starting Profiler...") + # Build activities dynamically to avoid CUDA hangs when CUDA is unavailable + activities = [torch.profiler.ProfilerActivity.CPU] + if torch.cuda.is_available(): + activities.append(torch.profiler.ProfilerActivity.CUDA) + + self.profiler = torch.profiler.profile( + activities=activities, + schedule=torch.profiler.schedule( + skip_first=0, + wait=0, + warmup=1, + active=batch.num_profiled_timesteps, + repeat=5, + ), + on_trace_ready=lambda _: torch.profiler.tensorboard_trace_handler( + f"./logs" + ), + record_shapes=True, + with_stack=True, + ) + self.profiler.start() + + def step_profile(self): + if self.profiler: + if torch.cuda.is_available(): + torch.cuda.synchronize() + self.profiler.step() + + def stop_profile(self, batch: Req): + try: + if self.profiler: + logger.info("Stopping Profiler...") + if torch.cuda.is_available(): + torch.cuda.synchronize() + self.profiler.stop() + request_id = batch.request_id if batch.request_id else "profile_trace" + log_dir = f"./logs" + os.makedirs(log_dir, exist_ok=True) + + rank = get_world_rank() + trace_path = os.path.abspath( + os.path.join(log_dir, f"{request_id}-rank{rank}.trace.json.gz") + ) + logger.info(f"Saving profiler traces to: {trace_path}") + self.profiler.export_chrome_trace(trace_path) + torch.distributed.barrier() + except Exception as e: + logger.error(f"{e}") + + def _manage_device_placement( + self, + model_to_use: torch.nn.Module, + model_to_offload: torch.nn.Module | None, + server_args: ServerArgs, + ): + """ + Manages the offload / load behavior of dit + """ + if not server_args.dit_cpu_offload: + return + + # Offload the unused model if it's on CUDA + if ( + model_to_offload is not None + and next(model_to_offload.parameters()).device.type == "cuda" + ): + model_to_offload.to("cpu") + + # Load the model to use if it's on CPU + if ( + model_to_use is not None + and next(model_to_use.parameters()).device.type == "cpu" + ): + model_to_use.to(get_local_torch_device()) + + def _select_and_manage_model( + self, + t_int: int, + boundary_timestep: float | None, + server_args: ServerArgs, + batch: Req, + ): + if boundary_timestep is None or t_int >= boundary_timestep: + # High-noise stage + current_model = self.transformer + model_to_offload = self.transformer_2 + current_guidance_scale = batch.guidance_scale + else: + # Low-noise stage + current_model = self.transformer_2 + model_to_offload = self.transformer + current_guidance_scale = batch.guidance_scale_2 + + self._manage_device_placement(current_model, model_to_offload, server_args) + + assert current_model is not None, "The model for the current step is not set." + return current_model, current_guidance_scale + + def expand_timestep_before_forward( + self, + batch: Req, + server_args: ServerArgs, + t_device, + target_dtype, + seq_len: int | None, + reserved_frames_mask, + ): + bsz = batch.raw_latent_shape[0] + should_preprocess_for_wan_ti2v = ( + server_args.pipeline_config.task_type == ModelTaskType.TI2V + and batch.condition_image is not None + and type(server_args.pipeline_config) is Wan2_2_TI2V_5B_Config + ) + + # expand timestep + if should_preprocess_for_wan_ti2v: + # Explicitly cast t_device to the target float type at the beginning. + # This ensures any precision-based rounding (e.g., float32(999.0) -> bfloat16(1000.0)) + # is applied consistently *before* it's used by any rank. + t_device_rounded = t_device.to(target_dtype) + + local_seq_len = seq_len + if get_sp_world_size() > 1 and getattr( + batch, "did_sp_shard_latents", False + ): + local_seq_len = seq_len // get_sp_world_size() + + if get_sp_parallel_rank() == 0 and reserved_frames_mask is not None: + # Rank 0 has the first frame, create a special timestep tensor + # NOTE: The spatial downsampling in the next line is suspicious but kept + # to match original model's potential training configuration. + temp_ts = ( + reserved_frames_mask[0][:, ::2, ::2] * t_device_rounded + ).flatten() + + # Pad to full local sequence length + temp_ts = torch.cat( + [ + temp_ts, + temp_ts.new_ones(local_seq_len - temp_ts.size(0)) + * t_device_rounded, + ] + ) + timestep = temp_ts.unsqueeze(0).repeat(bsz, 1) + else: + # Other ranks get a uniform timestep tensor of the correct shape [B, local_seq_len] + timestep = t_device.repeat(bsz, local_seq_len) + else: + timestep = t_device.repeat(bsz) + return timestep + + def post_forward_for_ti2v_task( + self, batch: Req, server_args: ServerArgs, reserved_frames_mask, latents, z + ): + """ + For Wan2.2 ti2v task, global first frame should be replaced with encoded image after each timestep + """ + should_preprocess_for_wan_ti2v = ( + server_args.pipeline_config.task_type == ModelTaskType.TI2V + and batch.condition_image is not None + and type(server_args.pipeline_config) is Wan2_2_TI2V_5B_Config + ) + if should_preprocess_for_wan_ti2v: + # Apply TI2V mask blending with SP-aware z and reserved_frames_mask. + # This ensures the first frame is always the condition image after each step. + # This is only applied on rank 0, where z is not None. + if z is not None and reserved_frames_mask is not None: + # z: [1, C, 1, H, W] + # latents: [1, C, T_local, H, W] + # reserved_frames_mask: [C, T_local, H, W] + # Unsqueeze mask to [1, C, T_local, H, W] for broadcasting. + # z will broadcast along the time dimension. + latents = ( + 1.0 - reserved_frames_mask.unsqueeze(0) + ) * z + reserved_frames_mask.unsqueeze(0) * latents + + return latents + + @torch.no_grad() + def forward( + self, + batch: Req, + server_args: ServerArgs, + ) -> Req: + """ + Run the denoising loop. + + Args: + batch: The current batch information. + server_args: The inference arguments. + + Returns: + The batch with denoised latents. + """ + # Prepare variables for the denoising loop + + prepared_vars = self._prepare_denoising_loop(batch, server_args) + extra_step_kwargs = prepared_vars["extra_step_kwargs"] + target_dtype = prepared_vars["target_dtype"] + autocast_enabled = prepared_vars["autocast_enabled"] + timesteps = prepared_vars["timesteps"] + num_inference_steps = prepared_vars["num_inference_steps"] + num_warmup_steps = prepared_vars["num_warmup_steps"] + image_kwargs = prepared_vars["image_kwargs"] + pos_cond_kwargs = prepared_vars["pos_cond_kwargs"] + neg_cond_kwargs = prepared_vars["neg_cond_kwargs"] + latents = prepared_vars["latents"] + boundary_timestep = prepared_vars["boundary_timestep"] + z = prepared_vars["z"] + reserved_frames_mask = prepared_vars["reserved_frames_mask"] + seq_len = prepared_vars["seq_len"] + guidance = prepared_vars["guidance"] + + # Initialize lists for ODE trajectory + trajectory_timesteps: list[torch.Tensor] = [] + trajectory_latents: list[torch.Tensor] = [] + + # Run denoising loop + denoising_start_time = time.time() + + self.start_profile(batch=batch) + + # to avoid device-sync caused by timestep comparison + timesteps_cpu = timesteps.cpu() + num_timesteps = timesteps_cpu.shape[0] + with torch.autocast( + device_type=("cuda" if torch.cuda.is_available() else "cpu"), + dtype=target_dtype, + enabled=autocast_enabled, + ): + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t_host in enumerate(timesteps_cpu): + # Skip if interrupted + if hasattr(self, "interrupt") and self.interrupt: + continue + + with StageProfiler( + f"denoising_step_{i}", logger=logger, timings=batch.timings + ): + t_int = int(t_host.item()) + t_device = timesteps[i] + current_model, current_guidance_scale = ( + self._select_and_manage_model( + t_int=t_int, + boundary_timestep=boundary_timestep, + server_args=server_args, + batch=batch, + ) + ) + + # Expand latents for I2V + latent_model_input = latents.to(target_dtype) + if batch.image_latent is not None: + assert ( + not server_args.pipeline_config.task_type + == ModelTaskType.TI2V + ), "image latents should not be provided for TI2V task" + latent_model_input = torch.cat( + [latent_model_input, batch.image_latent], dim=1 + ).to(target_dtype) + + timestep = self.expand_timestep_before_forward( + batch, + server_args, + t_device, + target_dtype, + seq_len, + reserved_frames_mask, + ) + + latent_model_input = self.scheduler.scale_model_input( + latent_model_input, t_device + ) + + # Predict noise residual + attn_metadata = self._build_attn_metadata(i, batch, server_args) + noise_pred = self._predict_noise_with_cfg( + current_model=current_model, + latent_model_input=latent_model_input, + timestep=timestep, + batch=batch, + timestep_index=i, + attn_metadata=attn_metadata, + target_dtype=target_dtype, + current_guidance_scale=current_guidance_scale, + image_kwargs=image_kwargs, + pos_cond_kwargs=pos_cond_kwargs, + neg_cond_kwargs=neg_cond_kwargs, + server_args=server_args, + guidance=guidance, + latents=latents, + ) + + # Compute the previous noisy sample + latents = self.scheduler.step( + model_output=noise_pred, + timestep=t_device, + sample=latents, + **extra_step_kwargs, + return_dict=False, + )[0] + + latents = self.post_forward_for_ti2v_task( + batch, server_args, reserved_frames_mask, latents, z + ) + + # save trajectory latents if needed + if batch.return_trajectory_latents: + trajectory_timesteps.append(t_host) + trajectory_latents.append(latents) + + # Update progress bar + if i == num_timesteps - 1 or ( + (i + 1) > num_warmup_steps + and (i + 1) % self.scheduler.order == 0 + and progress_bar is not None + ): + progress_bar.update() + + self.step_profile() + + self.stop_profile(batch) + + denoising_end_time = time.time() + + if num_timesteps > 0: + self.log_info( + "average time per step: %.4f seconds", + (denoising_end_time - denoising_start_time) / len(timesteps), + ) + + self._post_denoising_loop( + batch=batch, + latents=latents, + trajectory_latents=trajectory_latents, + trajectory_timesteps=trajectory_timesteps, + server_args=server_args, + ) + return batch + + # TODO: this will extends the preparation stage, should let subclass/passed-in variables decide which to prepare + def prepare_extra_func_kwargs(self, func, kwargs) -> dict[str, Any]: + """ + Prepare extra kwargs for the scheduler step / denoise step. + + Args: + func: The function to prepare kwargs for. + kwargs: The kwargs to prepare. + + Returns: + The prepared kwargs. + """ + extra_step_kwargs = {} + for k, v in kwargs.items(): + accepts = k in set(inspect.signature(func).parameters.keys()) + if accepts: + extra_step_kwargs[k] = v + return extra_step_kwargs + + def progress_bar( + self, iterable: Iterable | None = None, total: int | None = None + ) -> tqdm: + """ + Create a progress bar for the denoising process. + + Args: + iterable: The iterable to iterate over. + total: The total number of items. + + Returns: + A tqdm progress bar. + """ + local_rank = get_world_group().local_rank + if local_rank == 0: + return tqdm(iterable=iterable, total=total) + else: + return tqdm(iterable=iterable, total=total, disable=True) + + def rescale_noise_cfg( + self, noise_cfg, noise_pred_text, guidance_rescale=0.0 + ) -> torch.Tensor: + """ + Rescale noise prediction according to guidance_rescale. + + Based on findings of "Common Diffusion Noise Schedules and Sample Steps are Flawed" + (https://arxiv.org/pdf/2305.08891.pdf), Section 3.4. + + Args: + noise_cfg: The noise prediction with guidance. + noise_pred_text: The text-conditioned noise prediction. + guidance_rescale: The guidance rescale factor. + + Returns: + The rescaled noise prediction. + """ + std_text = noise_pred_text.std( + dim=list(range(1, noise_pred_text.ndim)), keepdim=True + ) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # Rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # Mix with the original results from guidance by factor guidance_rescale + noise_cfg = ( + guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg + ) + return noise_cfg + + def _build_attn_metadata( + self, i: int, batch: Req, server_args: ServerArgs + ) -> Any | None: + """ + Build attention metadata for custom attention backends. + + Args: + i: The current timestep index. + batch: The current batch information. + server_args: The inference arguments. + + Returns: + The attention metadata, or None if not applicable. + """ + attn_metadata = None + self.attn_metadata_builder_cls = self.attn_backend.get_builder_cls() + if self.attn_metadata_builder_cls: + self.attn_metadata_builder = self.attn_metadata_builder_cls() + if (st_attn_available and self.attn_backend == SlidingTileAttentionBackend) or ( + vsa_available and self.attn_backend == VideoSparseAttentionBackend + ): + attn_metadata = self.attn_metadata_builder.build( + current_timestep=i, + raw_latent_shape=batch.raw_latent_shape[2:5], + patch_size=server_args.pipeline_config.dit_config.patch_size, + STA_param=batch.STA_param, + VSA_sparsity=server_args.VSA_sparsity, + device=get_local_torch_device(), + ) + elif vmoba_attn_available and self.attn_backend == VMOBAAttentionBackend: + moba_params = server_args.moba_config.copy() + moba_params.update( + { + "current_timestep": i, + "raw_latent_shape": batch.raw_latent_shape[2:5], + "patch_size": server_args.pipeline_config.dit_config.patch_size, + "device": get_local_torch_device(), + } + ) + elif self.attn_backend == FlashAttentionBackend: + attn_metadata = self.attn_metadata_builder.build( + raw_latent_shape=batch.raw_latent_shape + ) + else: + return None + + assert attn_metadata is not None, "attn_metadata cannot be None" + + return attn_metadata + + def _predict_noise( + self, + current_model, + latent_model_input, + timestep, + prompt_embeds, + target_dtype, + guidance: torch.Tensor, + **kwargs, + ): + return current_model( + hidden_states=latent_model_input, + encoder_hidden_states=prompt_embeds, + timestep=timestep, + guidance=guidance, + **kwargs, + ) + + def _predict_noise_with_cfg( + self, + current_model: torch.nn.Module, + latent_model_input: torch.Tensor, + timestep, + batch: Req, + timestep_index: int, + attn_metadata, + target_dtype, + current_guidance_scale, + image_kwargs: dict[str, Any], + pos_cond_kwargs: dict[str, Any], + neg_cond_kwargs: dict[str, Any], + server_args, + guidance, + latents, + ): + """ + Predict the noise residual with classifier-free guidance. + + Args: + current_model: The transformer model to use for the current step. + latent_model_input: The input latents for the model. + timestep: The expanded timestep tensor. + batch: The current batch information. + timestep_index: The current timestep index. + attn_metadata: Attention metadata for custom backends. + target_dtype: The target data type for autocasting. + current_guidance_scale: The guidance scale for the current step. + image_kwargs: Keyword arguments for image conditioning. + pos_cond_kwargs: Keyword arguments for positive prompt conditioning. + neg_cond_kwargs: Keyword arguments for negative prompt conditioning. + + Returns: + The predicted noise. + """ + noise_pred_cond: torch.Tensor | None = None + noise_pred_uncond: torch.Tensor | None = None + cfg_rank = get_classifier_free_guidance_rank() + # positive pass + if not (server_args.enable_cfg_parallel and cfg_rank != 0): + batch.is_cfg_negative = False + with set_forward_context( + current_timestep=timestep_index, + attn_metadata=attn_metadata, + forward_batch=batch, + ): + noise_pred_cond = self._predict_noise( + current_model=current_model, + latent_model_input=latent_model_input, + timestep=timestep, + prompt_embeds=server_args.pipeline_config.get_pos_prompt_embeds( + batch + ), + target_dtype=target_dtype, + guidance=guidance, + **image_kwargs, + **pos_cond_kwargs, + ) + # TODO: can it be moved to after _predict_noise_with_cfg? + noise_pred_cond = server_args.pipeline_config.slice_noise_pred( + noise_pred_cond, latents + ) + if not batch.do_classifier_free_guidance: + # If CFG is disabled, we are done. Return the conditional prediction. + return noise_pred_cond + + # negative pass + if not server_args.enable_cfg_parallel or cfg_rank != 0: + batch.is_cfg_negative = True + with set_forward_context( + current_timestep=timestep_index, + attn_metadata=attn_metadata, + forward_batch=batch, + ): + noise_pred_uncond = self._predict_noise( + current_model=current_model, + latent_model_input=latent_model_input, + timestep=timestep, + prompt_embeds=server_args.pipeline_config.get_neg_prompt_embeds( + batch + ), + target_dtype=target_dtype, + guidance=guidance, + **image_kwargs, + **neg_cond_kwargs, + ) + noise_pred_uncond = server_args.pipeline_config.slice_noise_pred( + noise_pred_uncond, latents + ) + + # Combine predictions + if server_args.enable_cfg_parallel: + # Each rank computes its partial contribution and we sum via all-reduce: + # final = s*cond + (1-s)*uncond + if cfg_rank == 0: + assert noise_pred_cond is not None + partial = current_guidance_scale * noise_pred_cond + else: + assert noise_pred_uncond is not None + partial = (1 - current_guidance_scale) * noise_pred_uncond + + noise_pred = cfg_model_parallel_all_reduce(partial) + + # Guidance rescale: broadcast std(cond) from rank 0, compute std(cfg) locally + if batch.guidance_rescale > 0.0: + std_cfg = noise_pred.std( + dim=list(range(1, noise_pred.ndim)), keepdim=True + ) + if cfg_rank == 0: + assert noise_pred_cond is not None + std_text = noise_pred_cond.std( + dim=list(range(1, noise_pred_cond.ndim)), keepdim=True + ) + else: + std_text = torch.empty_like(std_cfg) + # Broadcast std_text from local src=0 to all ranks in CFG group + std_text = get_cfg_group().broadcast(std_text, src=0) + noise_pred_rescaled = noise_pred * (std_text / std_cfg) + noise_pred = ( + batch.guidance_rescale * noise_pred_rescaled + + (1 - batch.guidance_rescale) * noise_pred + ) + return noise_pred + else: + # Serial CFG: both cond and uncond are available locally + assert noise_pred_cond is not None and noise_pred_uncond is not None + noise_pred = noise_pred_uncond + current_guidance_scale * ( + noise_pred_cond - noise_pred_uncond + ) + + if batch.guidance_rescale > 0.0: + noise_pred = self.rescale_noise_cfg( + noise_pred, + noise_pred_cond, + guidance_rescale=batch.guidance_rescale, + ) + return noise_pred + + def prepare_sta_param(self, batch: Req, server_args: ServerArgs): + """ + Prepare Sliding Tile Attention (STA) parameters and settings. + + Args: + batch: The current batch information. + server_args: The inference arguments. + """ + # TODO(kevin): STA mask search, currently only support Wan2.1 with 69x768x1280 + STA_mode = server_args.STA_mode + skip_time_steps = server_args.skip_time_steps + if batch.timesteps is None: + raise ValueError("Timesteps must be provided") + timesteps_num = batch.timesteps.shape[0] + + logger.info("STA_mode: %s", STA_mode) + if (batch.num_frames, batch.height, batch.width) != ( + 69, + 768, + 1280, + ) and STA_mode != "STA_inference": + raise NotImplementedError( + "STA mask search/tuning is not supported for this resolution" + ) + + if ( + STA_mode == STA_Mode.STA_SEARCHING + or STA_mode == STA_Mode.STA_TUNING + or STA_mode == STA_Mode.STA_TUNING_CFG + ): + size = (batch.width, batch.height) + if size == (1280, 768): + # TODO: make it configurable + sparse_mask_candidates_searching = [ + "3, 1, 10", + "1, 5, 7", + "3, 3, 3", + "1, 6, 5", + "1, 3, 10", + "3, 6, 1", + ] + sparse_mask_candidates_tuning = [ + "3, 1, 10", + "1, 5, 7", + "3, 3, 3", + "1, 6, 5", + "1, 3, 10", + "3, 6, 1", + ] + full_mask = ["3,6,10"] + else: + raise NotImplementedError( + "STA mask search is not supported for this resolution" + ) + layer_num = self.transformer.config.num_layers + # specific for HunyuanVideo + if hasattr(self.transformer.config, "num_single_layers"): + layer_num += self.transformer.config.num_single_layers + head_num = self.transformer.config.num_attention_heads + + if STA_mode == STA_Mode.STA_SEARCHING: + STA_param = configure_sta( + mode=STA_Mode.STA_SEARCHING, + layer_num=layer_num, + head_num=head_num, + time_step_num=timesteps_num, + mask_candidates=sparse_mask_candidates_searching + full_mask, + # last is full mask; Can add more sparse masks while keep last one as full mask + ) + elif STA_mode == STA_Mode.STA_TUNING: + STA_param = configure_sta( + mode=STA_Mode.STA_TUNING, + layer_num=layer_num, + head_num=head_num, + time_step_num=timesteps_num, + mask_search_files_path=f"output/mask_search_result_pos_{size[0]}x{size[1]}/", + mask_candidates=sparse_mask_candidates_tuning, + full_attention_mask=[int(x) for x in full_mask[0].split(",")], + skip_time_steps=skip_time_steps, # Use full attention for first 12 steps + save_dir=f"output/mask_search_strategy_{size[0]}x{size[1]}/", # Custom save directory + timesteps=timesteps_num, + ) + elif STA_mode == STA_Mode.STA_TUNING_CFG: + STA_param = configure_sta( + mode=STA_Mode.STA_TUNING_CFG, + layer_num=layer_num, + head_num=head_num, + time_step_num=timesteps_num, + mask_search_files_path_pos=f"output/mask_search_result_pos_{size[0]}x{size[1]}/", + mask_search_files_path_neg=f"output/mask_search_result_neg_{size[0]}x{size[1]}/", + mask_candidates=sparse_mask_candidates_tuning, + full_attention_mask=[int(x) for x in full_mask[0].split(",")], + skip_time_steps=skip_time_steps, + save_dir=f"output/mask_search_strategy_{size[0]}x{size[1]}/", + timesteps=timesteps_num, + ) + elif STA_mode == STA_Mode.STA_INFERENCE: + import sglang.multimodal_gen.envs as envs + + config_file = envs.SGLANG_DIFFUSION_ATTENTION_CONFIG + if config_file is None: + raise ValueError("SGLANG_DIFFUSION_ATTENTION_CONFIG is not set") + STA_param = configure_sta( + mode=STA_Mode.STA_INFERENCE, + layer_num=layer_num, + head_num=head_num, + time_step_num=timesteps_num, + load_path=config_file, + ) + + batch.STA_param = STA_param + batch.mask_search_final_result_pos = [[] for _ in range(timesteps_num)] + batch.mask_search_final_result_neg = [[] for _ in range(timesteps_num)] + + def save_sta_search_results(self, batch: Req): + """ + Save the STA mask search results. + + Args: + batch: The current batch information. + """ + size = (batch.width, batch.height) + if size == (1280, 768): + # TODO: make it configurable + sparse_mask_candidates_searching = [ + "3, 1, 10", + "1, 5, 7", + "3, 3, 3", + "1, 6, 5", + "1, 3, 10", + "3, 6, 1", + ] + else: + raise NotImplementedError( + "STA mask search is not supported for this resolution" + ) + + if batch.mask_search_final_result_pos is not None and batch.prompt is not None: + save_mask_search_results( + [dict(layer_data) for layer_data in batch.mask_search_final_result_pos], + prompt=str(batch.prompt), + mask_strategies=sparse_mask_candidates_searching, + output_dir=f"output/mask_search_result_pos_{size[0]}x{size[1]}/", + ) + if batch.mask_search_final_result_neg is not None and batch.prompt is not None: + save_mask_search_results( + [dict(layer_data) for layer_data in batch.mask_search_final_result_neg], + prompt=str(batch.prompt), + mask_strategies=sparse_mask_candidates_searching, + output_dir=f"output/mask_search_result_neg_{size[0]}x{size[1]}/", + ) + + def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Verify denoising stage inputs.""" + result = VerificationResult() + result.add_check("timesteps", batch.timesteps, [V.is_tensor, V.min_dims(1)]) + # disable temporarily for image-generation models + # result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)]) + result.add_check("prompt_embeds", batch.prompt_embeds, V.list_not_empty) + result.add_check("image_embeds", batch.image_embeds, V.is_list) + # result.add_check( + # "image_latent", batch.image_latent, V.none_or_tensor_with_dims(5) + # ) + result.add_check( + "num_inference_steps", batch.num_inference_steps, V.positive_int + ) + result.add_check("guidance_scale", batch.guidance_scale, V.non_negative_float) + result.add_check("eta", batch.eta, V.non_negative_float) + result.add_check("generator", batch.generator, V.generator_or_list_generators) + result.add_check( + "do_classifier_free_guidance", + batch.do_classifier_free_guidance, + V.bool_value, + ) + result.add_check( + "negative_prompt_embeds", + batch.negative_prompt_embeds, + lambda x: not batch.do_classifier_free_guidance or V.list_not_empty(x), + ) + return result + + def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Verify denoising stage outputs.""" + result = VerificationResult() + # result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)]) + return result diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising_dmd.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising_dmd.py new file mode 100644 index 00000000..1ee183b0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising_dmd.py @@ -0,0 +1,204 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +import time + +import torch + +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context +from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, +) +from sglang.multimodal_gen.runtime.models.utils import pred_noise_to_pred_video +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages import DenoisingStage +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.runtime.utils.perf_logger import StageProfiler +from sglang.multimodal_gen.utils import dict_to_3d_list + +logger = init_logger(__name__) + + +class DmdDenoisingStage(DenoisingStage): + """ + Denoising stage for DMD. + """ + + def __init__(self, transformer, scheduler) -> None: + super().__init__(transformer, scheduler) + self.scheduler = FlowMatchEulerDiscreteScheduler(shift=8.0) + + def _preprocess_sp_latents(self, batch: Req, server_args: ServerArgs): + # 1. to shard latents (B, C, T, H, W) along dim 2 + super()._preprocess_sp_latents(batch, server_args) + + # 2. DMD expects (B, T, C, H, W) for the main latents in the loop + if batch.latents is not None: + batch.latents = batch.latents.permute(0, 2, 1, 3, 4) + + # Note: batch.image_latent is kept as (B, C, T, H, W) here + + def _postprocess_sp_latents( + self, + batch: Req, + latents: torch.Tensor, + trajectory_tensor: torch.Tensor | None, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + # 1. convert back from DMD's (B, T, C, H, W) to standard (B, C, T, H, W) + # this is because base gather_latents_for_sp expects dim=2 for T + latents = latents.permute(0, 2, 1, 3, 4) + + # 2. use base method to gather + return super()._postprocess_sp_latents(batch, latents, trajectory_tensor) + + def forward( + self, + batch: Req, + server_args: ServerArgs, + ) -> Req: + """ + Run the denoising loop. + """ + prepared_vars = self._prepare_denoising_loop(batch, server_args) + + target_dtype = prepared_vars["target_dtype"] + autocast_enabled = prepared_vars["autocast_enabled"] + num_warmup_steps = prepared_vars["num_warmup_steps"] + latents = prepared_vars["latents"] + video_raw_latent_shape = latents.shape + + timesteps = torch.tensor( + server_args.pipeline_config.dmd_denoising_steps, + dtype=torch.long, + device=get_local_torch_device(), + ) + + # prepare image_kwargs + image_embeds = batch.image_embeds + if len(image_embeds) > 0: + image_embeds = [img.to(target_dtype) for img in image_embeds] + + image_kwargs = self.prepare_extra_func_kwargs( + self.transformer.forward, + { + "encoder_hidden_states_image": image_embeds, + "mask_strategy": dict_to_3d_list(None, t_max=50, l_max=60, h_max=24), + }, + ) + + pos_cond_kwargs = prepared_vars["pos_cond_kwargs"] + prompt_embeds = prepared_vars["prompt_embeds"] + + denoising_loop_start_time = time.time() + self.start_profile(batch=batch) + + with self.progress_bar(total=len(timesteps)) as progress_bar: + for i, t in enumerate(timesteps): + # Skip if interrupted + if hasattr(self, "interrupt") and self.interrupt: + continue + + with StageProfiler( + f"denoising_step_{i}", logger=logger, timings=batch.timings + ): + # Expand latents for I2V + noise_latents = latents.clone() + latent_model_input = latents.to(target_dtype) + + if batch.image_latent is not None: + latent_model_input = torch.cat( + [ + latent_model_input, + batch.image_latent.permute(0, 2, 1, 3, 4), + ], + dim=2, + ).to(target_dtype) + assert not torch.isnan( + latent_model_input + ).any(), "latent_model_input contains nan" + + # Prepare inputs for transformer + t_expand = t.repeat(latent_model_input.shape[0]) + + guidance_expand = self.get_or_build_guidance( + latent_model_input.shape[0], + target_dtype, + get_local_torch_device(), + ) + + # Predict noise residual + with torch.autocast( + device_type="cuda", + dtype=target_dtype, + enabled=autocast_enabled, + ): + attn_metadata = self._build_attn_metadata(i, batch, server_args) + + batch.is_cfg_negative = False + with set_forward_context( + current_timestep=i, + attn_metadata=attn_metadata, + forward_batch=batch, + ): + # Run transformer + pred_noise = self.transformer( + latent_model_input.permute(0, 2, 1, 3, 4), + prompt_embeds, + t_expand, + guidance=guidance_expand, + **image_kwargs, + **pos_cond_kwargs, + ).permute(0, 2, 1, 3, 4) + + pred_video = pred_noise_to_pred_video( + pred_noise=pred_noise.flatten(0, 1), + noise_input_latent=noise_latents.flatten(0, 1), + timestep=t_expand, + scheduler=self.scheduler, + ).unflatten(0, pred_noise.shape[:2]) + + if i < len(timesteps) - 1: + next_timestep = timesteps[i + 1] * torch.ones( + [1], dtype=torch.long, device=pred_video.device + ) + noise = torch.randn( + video_raw_latent_shape, + dtype=pred_video.dtype, + generator=batch.generator[0], + ).to(self.device) + latents = self.scheduler.add_noise( + pred_video.flatten(0, 1), + noise.flatten(0, 1), + next_timestep, + ).unflatten(0, pred_video.shape[:2]) + else: + latents = pred_video + + # Update progress bar + if i == len(timesteps) - 1 or ( + (i + 1) > num_warmup_steps + and (i + 1) % self.scheduler.order == 0 + and progress_bar is not None + ): + progress_bar.update() + + self.step_profile() + + self.stop_profile(batch) + denoising_loop_end_time = time.time() + if len(timesteps) > 0: + self.log_info( + "average time per step: %.4f seconds", + (denoising_loop_end_time - denoising_loop_start_time) / len(timesteps), + ) + + self._post_denoising_loop( + batch=batch, + latents=latents, + trajectory_latents=[], + trajectory_timesteps=[], + server_args=server_args, + ) + + return batch diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/encoding.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/encoding.py new file mode 100644 index 00000000..6fe00884 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/encoding.py @@ -0,0 +1,105 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +Encoding stage for diffusion pipelines. +""" + +import torch + +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.models.vaes.common import ParallelTiledVAE +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + V, # Import validators +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + VerificationResult, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.utils import PRECISION_TO_TYPE + +logger = init_logger(__name__) + + +class EncodingStage(PipelineStage): + """ + Stage for encoding pixel space representations into latent space. + + This stage handles the encoding of pixel-space video/images into latent + representations for further processing in the diffusion pipeline. + """ + + def __init__(self, vae: ParallelTiledVAE) -> None: + self.vae: ParallelTiledVAE = vae + + @torch.no_grad() + def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Verify encoding stage inputs.""" + result = VerificationResult() + # Input video/images for VAE encoding: [batch_size, channels, frames, height, width] + result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)]) + return result + + def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Verify encoding stage outputs.""" + result = VerificationResult() + # Encoded latents: [batch_size, channels, frames, height_latents, width_latents] + result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)]) + return result + + def forward( + self, + batch: Req, + server_args: ServerArgs, + ) -> Req: + """ + Encode pixel space representations into latent space. + + Args: + batch: The current batch information. + server_args: The inference arguments. + + Returns: + The batch with encoded latents. + """ + assert batch.latents is not None and isinstance(batch.latents, torch.Tensor) + + self.vae = self.vae.to(get_local_torch_device()) + + # Setup VAE precision + vae_dtype = PRECISION_TO_TYPE[server_args.pipeline_config.vae_precision] + vae_autocast_enabled = ( + vae_dtype != torch.float32 + ) and not server_args.disable_autocast + + # Normalize input to [-1, 1] range (reverse of decoding normalization) + latents = (batch.latents * 2.0 - 1.0).clamp(-1, 1) + + # Move to appropriate device and dtype + latents = latents.to(get_local_torch_device()) + + # Encode image to latents + with torch.autocast( + device_type="cuda", dtype=vae_dtype, enabled=vae_autocast_enabled + ): + if server_args.pipeline_config.vae_tiling: + self.vae.enable_tiling() + # if server_args.vae_sp: + # self.vae.enable_parallel() + if not vae_autocast_enabled: + latents = latents.to(vae_dtype) + latents = self.vae.encode(latents).mean + + # Update batch with encoded latents + batch.latents = latents + + # Offload models if needed + self.maybe_free_model_hooks() + + if server_args.vae_cpu_offload: + self.vae.to("cpu") + + return batch diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/image_encoding.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/image_encoding.py new file mode 100644 index 00000000..5264c1b0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/image_encoding.py @@ -0,0 +1,357 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +Image encoding stages for I2V diffusion pipelines. + +This module contains implementations of image encoding stages for diffusion pipelines. +""" + +import PIL +import torch +from diffusers.models.autoencoders.vae import DiagonalGaussianDistribution + +from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import ( + qwen_image_postprocess_text, +) +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context +from sglang.multimodal_gen.runtime.models.vaes.common import ParallelTiledVAE +from sglang.multimodal_gen.runtime.models.vision_utils import ( + normalize, + numpy_to_pt, + pil_to_numpy, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + StageValidators as V, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + VerificationResult, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.utils import PRECISION_TO_TYPE + +logger = init_logger(__name__) + + +class ImageEncodingStage(PipelineStage): + """ + Stage for encoding image prompts into embeddings for diffusion models. + + This stage handles the encoding of image prompts into the embedding space + expected by the diffusion model. + """ + + def __init__( + self, + image_processor, + image_encoder=None, + text_encoder=None, + ) -> None: + """ + Initialize the prompt encoding stage. + + Args: + text_encoder: An encoder to encode input_ids and pixel values + """ + super().__init__() + self.image_processor = image_processor + self.image_encoder = image_encoder + self.text_encoder = text_encoder + + def move_to_device(self, device): + fields = [ + "image_processor", + "image_encoder", + ] + for field in fields: + processor = getattr(self, field, None) + if processor and hasattr(processor, "to"): + setattr(self, field, processor.to(device)) + + def encoding_qwen_image_edit(self, outputs, image_inputs): + # encoder hidden state + prompt_embeds = qwen_image_postprocess_text(outputs, image_inputs, 64) + return prompt_embeds + + @torch.no_grad() + def forward( + self, + batch: Req, + server_args: ServerArgs, + ) -> Req: + """ + Encode the prompt into image encoder hidden states. + + Args: + batch: The current batch information. + server_args: The inference arguments. + + Returns: + The batch with encoded prompt embeddings. + """ + + if batch.condition_image is None: + return batch + cuda_device = get_local_torch_device() + self.move_to_device(cuda_device) + + image = batch.condition_image + + image_processor_kwargs = ( + server_args.pipeline_config.prepare_image_processor_kwargs(batch) + ) + + image_inputs = self.image_processor( + images=image, return_tensors="pt", **image_processor_kwargs + ).to(cuda_device) + if self.image_encoder: + # if an image encoder is provided + with set_forward_context(current_timestep=0, attn_metadata=None): + outputs = self.image_encoder( + **image_inputs, + **server_args.pipeline_config.image_encoder_extra_args, + ) + image_embeds = server_args.pipeline_config.postprocess_image(outputs) + + batch.image_embeds.append(image_embeds) + elif self.text_encoder: + # if a text encoder is provided, e.g. Qwen-Image-Edit + # 1. neg prompt embeds + if batch.prompt: + prompt_template_encode = "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>{}<|im_end|>\n<|im_start|>assistant\n" + txt = prompt_template_encode.format(batch.negative_prompt) + neg_image_processor_kwargs = dict(text=[txt], padding=True) + else: + neg_image_processor_kwargs = {} + + neg_image_inputs = self.image_processor( + images=image, return_tensors="pt", **neg_image_processor_kwargs + ).to(get_local_torch_device()) + + with set_forward_context(current_timestep=0, attn_metadata=None): + outputs = self.text_encoder( + input_ids=image_inputs.input_ids, + attention_mask=image_inputs.attention_mask, + pixel_values=image_inputs.pixel_values, + image_grid_thw=image_inputs.image_grid_thw, + output_hidden_states=True, + ) + neg_outputs = self.text_encoder( + input_ids=neg_image_inputs.input_ids, + attention_mask=neg_image_inputs.attention_mask, + pixel_values=neg_image_inputs.pixel_values, + image_grid_thw=neg_image_inputs.image_grid_thw, + output_hidden_states=True, + ) + batch.prompt_embeds.append( + self.encoding_qwen_image_edit(outputs, image_inputs) + ) + + batch.negative_prompt_embeds.append( + self.encoding_qwen_image_edit(neg_outputs, neg_image_inputs) + ) + + self.move_to_device("cpu") + + return batch + + def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Verify image encoding stage inputs.""" + result = VerificationResult() + if batch.debug: + logger.debug(f"{batch.condition_image=}") + logger.debug(f"{batch.image_embeds=}") + result.add_check("pil_image", batch.condition_image, V.not_none) + result.add_check("image_embeds", batch.image_embeds, V.is_list) + return result + + def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Verify image encoding stage outputs.""" + result = VerificationResult() + # result.add_check("image_embeds", batch.image_embeds, V.list_of_tensors_dims(3)) + return result + + +class ImageVAEEncodingStage(PipelineStage): + """ + Stage for encoding pixel representations into latent space. + + This stage handles the encoding of pixel representations into the final + input format (e.g., image_latents). + """ + + def __init__(self, vae: ParallelTiledVAE, **kwargs) -> None: + super().__init__() + self.vae: ParallelTiledVAE = vae + + def forward( + self, + batch: Req, + server_args: ServerArgs, + ) -> Req: + """ + Encode pixel representations into latent space. + + Args: + batch: The current batch information. + server_args: The inference arguments. + + Returns: + The batch with encoded outputs. + """ + + if batch.condition_image is None: + return batch + + num_frames = batch.num_frames + + self.vae = self.vae.to(get_local_torch_device()) + + image = batch.condition_image + image = self.preprocess( + image, + ).to(get_local_torch_device(), dtype=torch.float32) + + # (B, C, H, W) -> (B, C, 1, H, W) + image = image.unsqueeze(2) + + if num_frames == 1: + video_condition = image + else: + video_condition = torch.cat( + [ + image, + image.new_zeros( + image.shape[0], + image.shape[1], + num_frames - 1, + image.shape[3], + image.shape[4], + ), + ], + dim=2, + ) + video_condition = video_condition.to( + device=get_local_torch_device(), dtype=torch.float32 + ) + + # Setup VAE precision + vae_dtype = PRECISION_TO_TYPE[server_args.pipeline_config.vae_precision] + vae_autocast_enabled = ( + vae_dtype != torch.float32 + ) and not server_args.disable_autocast + + # Encode Image + with torch.autocast( + device_type="cuda", dtype=vae_dtype, enabled=vae_autocast_enabled + ): + if server_args.pipeline_config.vae_tiling: + self.vae.enable_tiling() + # if server_args.vae_sp: + # self.vae.enable_parallel() + if not vae_autocast_enabled: + video_condition = video_condition.to(vae_dtype) + encoder_output: DiagonalGaussianDistribution = self.vae.encode( + video_condition + ) + + generator = batch.generator + if generator is None: + raise ValueError("Generator must be provided") + + sample_mode = server_args.pipeline_config.vae_config.encode_sample_mode() + + latent_condition = self.retrieve_latents( + encoder_output, generator, sample_mode=sample_mode + ) + latent_condition = server_args.pipeline_config.postprocess_vae_encode( + latent_condition, self.vae + ) + + scaling_factor, shift_factor = ( + server_args.pipeline_config.get_decode_scale_and_shift( + device=latent_condition.device, + dtype=latent_condition.dtype, + vae=self.vae, + ) + ) + + # apply shift & scale if needed + if isinstance(shift_factor, torch.Tensor): + shift_factor = shift_factor.to(latent_condition.device) + + if isinstance(scaling_factor, torch.Tensor): + scaling_factor = scaling_factor.to(latent_condition.device) + + latent_condition -= shift_factor + latent_condition = latent_condition * scaling_factor + + batch.image_latent = server_args.pipeline_config.postprocess_image_latent( + latent_condition, batch + ) + + self.maybe_free_model_hooks() + + self.vae.to("cpu") + + return batch + + def retrieve_latents( + self, + encoder_output: DiagonalGaussianDistribution, + generator: torch.Generator | None = None, + sample_mode: str = "sample", + ): + if sample_mode == "sample": + return encoder_output.sample(generator) + elif sample_mode == "argmax": + return encoder_output.mode() + else: + raise AttributeError("Could not access latents of provided encoder_output") + + def preprocess( + self, + image: torch.Tensor | PIL.Image.Image, + ) -> torch.Tensor: + + if isinstance(image, PIL.Image.Image): + image = pil_to_numpy(image) # to np + image = numpy_to_pt(image) # to pt + + do_normalize = True + if image.min() < 0: + do_normalize = False + if do_normalize: + image = normalize(image) + + return image + + def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Verify encoding stage inputs.""" + result = VerificationResult() + + assert batch.condition_image is None or ( + isinstance(batch.condition_image, PIL.Image.Image) + or isinstance(batch.condition_image, torch.Tensor) + ) + assert batch.height is not None and isinstance(batch.height, int) + assert batch.width is not None and isinstance(batch.width, int) + assert batch.num_frames is not None and isinstance(batch.num_frames, int) + + result.add_check("generator", batch.generator, V.generator_or_list_generators) + result.add_check("height", batch.height, V.positive_int) + result.add_check("width", batch.width, V.positive_int) + result.add_check("num_frames", batch.num_frames, V.positive_int) + return result + + def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Verify encoding stage outputs.""" + result = VerificationResult() + # result.add_check( + # "image_latent", batch.image_latent, [V.is_tensor, V.with_dims(5)] + # ) + return result diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/input_validation.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/input_validation.py new file mode 100644 index 00000000..8ad061d9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/input_validation.py @@ -0,0 +1,250 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +Input validation stage for diffusion pipelines. +""" +import numpy as np +import torch +import torchvision.transforms.functional as TF +from PIL import Image + +from sglang.multimodal_gen.configs.pipeline_configs import WanI2V480PConfig +from sglang.multimodal_gen.configs.pipeline_configs.base import ModelTaskType +from sglang.multimodal_gen.runtime.models.vision_utils import load_image, load_video +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + StageValidators, + VerificationResult, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.utils import best_output_size + +logger = init_logger(__name__) + +# Alias for convenience +V = StageValidators + + +# TODO: since this might change sampling params after logging, should be do this beforehand? + + +class InputValidationStage(PipelineStage): + """ + Stage for validating and preparing inputs for diffusion pipelines. + + This stage validates that all required inputs are present and properly formatted + before proceeding with the diffusion process. + + In this stage, input image and output image may be resized + """ + + def __init__(self, vae_image_processor=None): + super().__init__() + self.vae_image_processor = vae_image_processor + + def _generate_seeds(self, batch: Req, server_args: ServerArgs): + """Generate seeds for the inference""" + seed = batch.seed + num_videos_per_prompt = batch.num_outputs_per_prompt + + assert seed is not None + seeds = [seed + i for i in range(num_videos_per_prompt)] + batch.seeds = seeds + # Peiyuan: using GPU seed will cause A100 and H100 to generate different results... + # FIXME: the generator's in latent preparation stage seems to be different from seeds + batch.generator = [torch.Generator("cpu").manual_seed(seed) for seed in seeds] + + def preprocess_condition_image( + self, + batch: Req, + server_args: ServerArgs, + condition_image_width, + condition_image_height, + ): + """ + preprocess condition image + NOTE: condition image resizing is only allowed in InputValidationStage + """ + if server_args.pipeline_config.task_type == ModelTaskType.I2I: + # calculate new condition image size + calculated_size = ( + server_args.pipeline_config.calculate_condition_image_size( + batch.condition_image, + condition_image_width, + condition_image_height, + ) + ) + + # preprocess condition image if necessary + if calculated_size is not None: + calculated_width, calculated_height = calculated_size + condition_image, calculated_size = ( + server_args.pipeline_config.preprocess_condition_image( + batch.condition_image, + calculated_width, + calculated_height, + self.vae_image_processor, + ) + ) + batch.condition_image = condition_image + + # adjust output image size + calculated_width, calculated_height = calculated_size + width = batch.width or calculated_width + height = batch.height or calculated_height + multiple_of = ( + server_args.pipeline_config.vae_config.get_vae_scale_factor() * 2 + ) + width = width // multiple_of * multiple_of + height = height // multiple_of * multiple_of + batch.width = width + batch.height = height + elif server_args.pipeline_config.task_type == ModelTaskType.TI2V: + # duplicate with vae_image_processor + # further processing for ti2v task + img = batch.condition_image + ih, iw = img.height, img.width + patch_size = server_args.pipeline_config.dit_config.arch_config.patch_size + vae_stride = ( + server_args.pipeline_config.vae_config.arch_config.scale_factor_spatial + ) + dh, dw = patch_size[1] * vae_stride, patch_size[2] * vae_stride + max_area = 704 * 1280 + ow, oh = best_output_size(iw, ih, dw, dh, max_area) + + scale = max(ow / iw, oh / ih) + img = img.resize((round(iw * scale), round(ih * scale)), Image.LANCZOS) + logger.info("resized img height: %s, img width: %s", img.height, img.width) + + # center-crop + x1 = (img.width - ow) // 2 + y1 = (img.height - oh) // 2 + img = img.crop((x1, y1, x1 + ow, y1 + oh)) + assert img.width == ow and img.height == oh + + # to tensor + img = TF.to_tensor(img).sub_(0.5).div_(0.5).to(self.device).unsqueeze(1) + img = img.unsqueeze(0) + batch.height = oh + batch.width = ow + # TODO: should we store in a new field: pixel values? + batch.condition_image = img + + elif isinstance(server_args.pipeline_config, WanI2V480PConfig): + # TODO: could we merge with above? + # resize image only, Wan2.1 I2V + max_area = 720 * 1280 + aspect_ratio = condition_image_height / condition_image_width + mod_value = ( + server_args.pipeline_config.vae_config.arch_config.scale_factor_spatial + * server_args.pipeline_config.dit_config.arch_config.patch_size[1] + ) + height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value + width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value + + batch.condition_image = batch.condition_image.resize((width, height)) + batch.height = height + batch.width = width + + def forward( + self, + batch: Req, + server_args: ServerArgs, + ) -> Req: + """ + Validate and prepare inputs. + + Args: + batch: The current batch information. + server_args: The inference arguments. + + Returns: + The validated batch information. + """ + + self._generate_seeds(batch, server_args) + + # Ensure prompt is properly formatted + if batch.prompt is None and batch.prompt_embeds is None: + raise ValueError("Either `prompt` or `prompt_embeds` must be provided") + + # Ensure negative prompt is properly formatted if using classifier-free guidance + if ( + batch.do_classifier_free_guidance + and batch.negative_prompt is None + and batch.negative_prompt_embeds is None + ): + raise ValueError( + "For classifier-free guidance, either `negative_prompt` or " + "`negative_prompt_embeds` must be provided" + ) + + # Validate number of inference steps + if batch.num_inference_steps <= 0: + raise ValueError( + f"Number of inference steps must be positive, but got {batch.num_inference_steps}" + ) + + # Validate guidance scale if using classifier-free guidance + if batch.do_classifier_free_guidance and batch.guidance_scale < 0: + raise ValueError( + f"Guidance scale must be positive, but got {batch.guidance_scale}" + ) + + # for i2v, get image from image_path + # @TODO(Wei) hard-coded for wan2.2 5b ti2v for now. Should put this in image_encoding stage + if batch.image_path is not None: + if batch.image_path.endswith(".mp4"): + image = load_video(batch.image_path)[0] + else: + image = load_image(batch.image_path) + batch.condition_image = image + condition_image_width, condition_image_height = image.width, image.height + batch.original_condition_image_size = image.size + + self.preprocess_condition_image( + batch, server_args, condition_image_width, condition_image_height + ) + + return batch + + def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Verify input validation stage inputs.""" + result = VerificationResult() + result.add_check("seed", batch.seed, [V.not_none, V.non_negative_int]) + result.add_check( + "num_videos_per_prompt", batch.num_outputs_per_prompt, V.positive_int + ) + result.add_check( + "prompt_or_embeds", + None, + lambda _: V.string_or_list_strings(batch.prompt) + or V.list_not_empty(batch.prompt_embeds), + ) + + result.add_check( + "num_inference_steps", batch.num_inference_steps, V.positive_int + ) + result.add_check( + "guidance_scale", + batch.guidance_scale, + lambda x: not batch.do_classifier_free_guidance or V.non_negative_float(x), + ) + return result + + def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Verify input validation stage outputs.""" + result = VerificationResult() + result.add_check("height", batch.height, V.positive_int) + result.add_check("width", batch.width, V.positive_int) + # Validate height and width + if batch.height % 8 != 0 or batch.width % 8 != 0: + raise ValueError( + f"Height and width must be divisible by 8 but are {batch.height} and {batch.width}." + ) + result.add_check("seeds", batch.seeds, V.list_not_empty) + result.add_check("generator", batch.generator, V.generator_or_list_generators) + return result diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/latent_preparation.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/latent_preparation.py new file mode 100644 index 00000000..acd901b3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/latent_preparation.py @@ -0,0 +1,163 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +Latent preparation stage for diffusion pipelines. +""" +from diffusers.utils.torch_utils import randn_tensor + +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + StageValidators as V, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + VerificationResult, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class LatentPreparationStage(PipelineStage): + """ + Stage for preparing initial latent variables for the diffusion process. + + This stage handles the preparation of the initial latent variables that will be + denoised during the diffusion process. + """ + + def __init__(self, scheduler, transformer) -> None: + super().__init__() + self.scheduler = scheduler + self.transformer = transformer + + def forward( + self, + batch: Req, + server_args: ServerArgs, + ) -> Req: + """ + Prepare initial latent variables for the diffusion process. + + Args: + batch: The current batch information. + server_args: The inference arguments. + + Returns: + The batch with prepared latent variables. + """ + + latent_num_frames = None + # Adjust video length based on VAE version if needed + if hasattr(self, "adjust_video_length"): + latent_num_frames = self.adjust_video_length(batch, server_args) + + batch_size = batch.batch_size + + # Get required parameters + dtype = batch.prompt_embeds[0].dtype + device = get_local_torch_device() + generator = batch.generator + latents = batch.latents + num_frames = ( + latent_num_frames if latent_num_frames is not None else batch.num_frames + ) + height = batch.height + width = batch.width + + # TODO(will): remove this once we add input/output validation for stages + if height is None or width is None: + raise ValueError("Height and width must be provided") + + # Validate generator if it's a list + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + # Generate or use provided latents + if latents is None: + shape = server_args.pipeline_config.prepare_latent_shape( + batch, batch_size, num_frames + ) + latents = randn_tensor( + shape, generator=generator, device=device, dtype=dtype + ) + + latent_ids = server_args.pipeline_config.maybe_prepare_latent_ids(latents) + + if latent_ids is not None: + batch.latent_ids = latent_ids.to(device=device) + + latents = server_args.pipeline_config.maybe_pack_latents( + latents, batch_size, batch + ) + else: + latents = latents.to(device) + + # Scale the initial noise if needed + if hasattr(self.scheduler, "init_noise_sigma"): + latents = latents * self.scheduler.init_noise_sigma + # Update batch with prepared latents + batch.latents = latents + batch.raw_latent_shape = latents.shape + return batch + + def adjust_video_length(self, batch: Req, server_args: ServerArgs) -> int: + """ + Adjust video length based on VAE version. + + Args: + batch: The current batch information. + server_args: The inference arguments. + + Returns: + The batch with adjusted video length. + """ + + video_length = batch.num_frames + use_temporal_scaling_frames = ( + server_args.pipeline_config.vae_config.use_temporal_scaling_frames + ) + if use_temporal_scaling_frames: + temporal_scale_factor = ( + server_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + ) + latent_num_frames = (video_length - 1) // temporal_scale_factor + 1 + else: # stepvideo only + latent_num_frames = video_length // 17 * 3 + return int(latent_num_frames) + + def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Verify latent preparation stage inputs.""" + result = VerificationResult() + result.add_check( + "prompt_or_embeds", + None, + lambda _: V.string_or_list_strings(batch.prompt) + or V.list_not_empty(batch.prompt_embeds), + ) + result.add_check("prompt_embeds", batch.prompt_embeds, V.list_of_tensors) + result.add_check( + "num_videos_per_prompt", batch.num_outputs_per_prompt, V.positive_int + ) + result.add_check("generator", batch.generator, V.generator_or_list_generators) + result.add_check("num_frames", batch.num_frames, V.positive_int) + result.add_check("height", batch.height, V.positive_int) + result.add_check("width", batch.width, V.positive_int) + result.add_check("latents", batch.latents, V.none_or_tensor) + return result + + def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Verify latent preparation stage outputs.""" + result = VerificationResult() + if batch.debug: + logger.debug(f"{batch.raw_latent_shape=}") + # disable temporarily for image-generation models + # result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)]) + result.add_check("raw_latent_shape", batch.raw_latent_shape, V.is_tuple) + return result diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/stepvideo_encoding.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/stepvideo_encoding.py new file mode 100644 index 00000000..eed0edd9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/stepvideo_encoding.py @@ -0,0 +1,99 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +import torch + +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + StageValidators as V, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + VerificationResult, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +# The dedicated stepvideo prompt encoding stage. +class StepvideoPromptEncodingStage(PipelineStage): + """ + Stage for encoding prompts using the remote caption API. + + This stage applies the magic string transformations and calls + the remote caption service asynchronously to get: + - primary prompt embeddings, + - an attention mask, + - and a clip embedding. + """ + + def __init__(self, stepllm, clip) -> None: + super().__init__() + # self.caption_client = caption_client # This should have a call_caption(prompts: List[str]) method. + self.stepllm = stepllm + self.clip = clip + + @torch.no_grad() + def forward(self, batch: Req, server_args) -> Req: + + prompts = [batch.prompt + server_args.pipeline_config.pos_magic] + bs = len(prompts) + prompts += [server_args.pipeline_config.neg_magic] * bs + with set_forward_context(current_timestep=0, attn_metadata=None): + y, y_mask = self.stepllm(prompts) + clip_emb, _ = self.clip(prompts) + len_clip = clip_emb.shape[1] + y_mask = torch.nn.functional.pad(y_mask, (len_clip, 0), value=1) + pos_clip, neg_clip = clip_emb[:bs], clip_emb[bs:] + + # split positive vs negative text + batch.prompt_embeds = y[:bs] # [bs, seq_len, dim] + batch.negative_prompt_embeds = y[bs : 2 * bs] # [bs, seq_len, dim] + batch.prompt_attention_mask = y_mask[:bs] # [bs, seq_len] + batch.negative_attention_mask = y_mask[bs : 2 * bs] # [bs, seq_len] + batch.clip_embedding_pos = pos_clip + batch.clip_embedding_neg = neg_clip + return batch + + def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Verify stepvideo encoding stage inputs.""" + result = VerificationResult() + result.add_check("prompt", batch.prompt, V.string_not_empty) + return result + + def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Verify stepvideo encoding stage outputs.""" + result = VerificationResult() + result.add_check( + "prompt_embeds", batch.prompt_embeds, [V.is_tensor, V.with_dims(3)] + ) + result.add_check( + "negative_prompt_embeds", + batch.negative_prompt_embeds, + [V.is_tensor, V.with_dims(3)], + ) + result.add_check( + "prompt_attention_mask", + batch.prompt_attention_mask, + [V.is_tensor, V.with_dims(2)], + ) + result.add_check( + "negative_attention_mask", + batch.negative_attention_mask, + [V.is_tensor, V.with_dims(2)], + ) + result.add_check( + "clip_embedding_pos", + batch.clip_embedding_pos, + [V.is_tensor, V.with_dims(2)], + ) + result.add_check( + "clip_embedding_neg", + batch.clip_embedding_neg, + [V.is_tensor, V.with_dims(2)], + ) + return result diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/text_encoding.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/text_encoding.py new file mode 100644 index 00000000..6423e45d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/text_encoding.py @@ -0,0 +1,336 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +Prompt encoding stages for diffusion pipelines. + +This module contains implementations of prompt encoding stages for diffusion pipelines. +""" + +import torch + +from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput +from sglang.multimodal_gen.configs.pipeline_configs import FluxPipelineConfig +from sglang.multimodal_gen.configs.pipeline_configs.flux import Flux2PipelineConfig +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + StageValidators as V, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + VerificationResult, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class TextEncodingStage(PipelineStage): + """ + Stage for encoding text prompts into embeddings for diffusion models. + + This stage handles the encoding of text prompts into the embedding space + expected by the diffusion model. + """ + + def __init__(self, text_encoders, tokenizers) -> None: + """ + Initialize the prompt encoding stage. + + """ + super().__init__() + self.tokenizers = tokenizers + self.text_encoders = text_encoders + + @torch.no_grad() + def forward( + self, + batch: Req, + server_args: ServerArgs, + ) -> Req: + """ + Encode the prompt into text encoder hidden states. + + Args: + batch: The current batch information. + server_args: The inference arguments. + + Returns: + The batch with encoded prompt embeddings. + """ + assert len(self.tokenizers) == len(self.text_encoders) + assert len(self.text_encoders) == len( + server_args.pipeline_config.text_encoder_configs + ) + + # Encode positive prompt with all available encoders + assert batch.prompt is not None + prompt_text: str | list[str] = batch.prompt + + all_indices: list[int] = list(range(len(self.text_encoders))) + + prompt_embeds_list, prompt_masks_list, pooler_embeds_list = self.encode_text( + prompt_text, + server_args, + encoder_index=all_indices, + return_attention_mask=True, + ) + + for pe in prompt_embeds_list: + batch.prompt_embeds.append(pe) + + for pe in pooler_embeds_list: + batch.pooled_embeds.append(pe) + if batch.prompt_attention_mask is not None: + for am in prompt_masks_list: + batch.prompt_attention_mask.append(am) + + # Encode negative prompt if CFG is enabled + if batch.do_classifier_free_guidance: + assert isinstance(batch.negative_prompt, str) + neg_embeds_list, neg_masks_list, neg_pooler_embeds_list = self.encode_text( + batch.negative_prompt, + server_args, + encoder_index=all_indices, + return_attention_mask=True, + ) + + assert batch.negative_prompt_embeds is not None + + for ne in neg_embeds_list: + batch.negative_prompt_embeds.append(ne) + + for pe in neg_pooler_embeds_list: + batch.neg_pooled_embeds.append(pe) + if batch.negative_attention_mask is not None: + for nm in neg_masks_list: + batch.negative_attention_mask.append(nm) + + return batch + + def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Verify text encoding stage inputs.""" + result = VerificationResult() + result.add_check("prompt", batch.prompt, V.string_or_list_strings) + result.add_check( + "negative_prompt", + batch.negative_prompt, + lambda x: not batch.do_classifier_free_guidance or V.string_not_none(x), + ) + result.add_check( + "do_classifier_free_guidance", + batch.do_classifier_free_guidance, + V.bool_value, + ) + result.add_check("prompt_embeds", batch.prompt_embeds, V.is_list) + result.add_check( + "negative_prompt_embeds", batch.negative_prompt_embeds, V.none_or_list + ) + return result + + def prepare_tokenizer_kwargs(self, tokenizer_kwargs, **kwargs): + tok_kwargs = tokenizer_kwargs | kwargs + + return tok_kwargs + + @torch.no_grad() + def encode_text( + self, + text: str | list[str], + server_args: ServerArgs, + encoder_index: int | list[int] | None = None, + return_attention_mask: bool = False, + return_type: str = "list", # one of: "list", "dict", "stack" + device: torch.device | str | None = None, + dtype: torch.dtype | None = None, + max_length: int | None = None, + truncation: bool | None = None, + padding: bool | str | None = None, + return_overflowing_tokens=None, + return_length=None, + ): + """ + Encode plain text using selected text encoder(s) and return embeddings. + + Args: + text: A single string or a list of strings to encode. + server_args: The inference arguments providing pipeline config, + including tokenizer and encoder settings, preprocess and postprocess + functions. + encoder_index: Encoder selector by index. Accepts an int or list of ints. + return_attention_mask: If True, also return attention masks for each + selected encoder. + return_type: "list" (default) returns a list aligned with selection; + "dict" returns a dict keyed by encoder index as a string; "stack" stacks along a + new first dimension (requires matching shapes). + device: Optional device override for inputs; defaults to local torch device. + dtype: Optional dtype to cast returned embeddings to. + max_length: Optional per-call tokenizer override. + truncation: Optional per-call tokenizer override. + padding: Optional per-call tokenizer override. + + Returns: + Depending on return_type and return_attention_mask: + - list: List[Tensor] or (List[Tensor], List[Tensor]) + - dict: Dict[str, Tensor] or (Dict[str, Tensor], Dict[str, Tensor]) + - stack: Tensor of shape [num_encoders, ...] or a tuple with stacked + attention masks + """ + + assert len(self.tokenizers) == len(self.text_encoders) + assert len(self.text_encoders) == len( + server_args.pipeline_config.text_encoder_configs + ) + + # Resolve selection into indices + encoder_cfgs = server_args.pipeline_config.text_encoder_configs + if encoder_index is None: + indices: list[int] = [0] + elif isinstance(encoder_index, int): + indices = [encoder_index] + else: + indices = list(encoder_index) + # validate range + num_encoders = len(self.text_encoders) + for idx in indices: + if idx < 0 or idx >= num_encoders: + raise IndexError( + f"encoder index {idx} out of range [0, {num_encoders - 1}]" + ) + + # Validate indices are within range + num_encoders = len(self.text_encoders) + + # Normalize input to list[str] + assert isinstance(text, str | list) + if isinstance(text, str): + texts: list[str] = [text] + else: + texts = text + + embeds_list: list[torch.Tensor] = [] + pooled_embeds_list: list[torch.Tensor] = [] + + attn_masks_list: list[torch.Tensor] = [] + + preprocess_funcs = server_args.pipeline_config.preprocess_text_funcs + postprocess_funcs = server_args.pipeline_config.postprocess_text_funcs + text_encoder_extra_args = server_args.pipeline_config.text_encoder_extra_args + encoder_cfgs = server_args.pipeline_config.text_encoder_configs + + if return_type not in ("list", "dict", "stack"): + raise ValueError( + f"Invalid return_type '{return_type}'. Expected one of: 'list', 'dict', 'stack'" + ) + + target_device = device if device is not None else get_local_torch_device() + + for i in indices: + tokenizer = self.tokenizers[i] + text_encoder = self.text_encoders[i] + encoder_config = encoder_cfgs[i] + preprocess_func = preprocess_funcs[i] + postprocess_func = postprocess_funcs[i] + text_encoder_extra_arg = ( + text_encoder_extra_args[i] + if i < len(text_encoder_extra_args) and text_encoder_extra_args[i] + else {} + ) + + processed_text_list: list[str] = [] + for prompt_str in texts: + preprocessed = preprocess_func(prompt_str) + processed_text_list.append(preprocessed) + + # Prepare tokenizer args + tok_kwargs = self.prepare_tokenizer_kwargs( + encoder_config.tokenizer_kwargs, + **text_encoder_extra_arg, + ) + + text_inputs: dict = server_args.pipeline_config.tokenize_prompt( + processed_text_list, tokenizer, tok_kwargs + ).to(target_device) + + input_ids = text_inputs["input_ids"] + is_flux_v1 = isinstance( + server_args.pipeline_config, FluxPipelineConfig + ) and not isinstance(server_args.pipeline_config, Flux2PipelineConfig) + is_flux_t5 = is_flux_v1 and i == 1 + + if is_flux_t5: + attention_mask = torch.ones(input_ids.shape[:2], device=target_device) + else: + attention_mask = text_inputs["attention_mask"] + with set_forward_context(current_timestep=0, attn_metadata=None): + outputs: BaseEncoderOutput = text_encoder( + input_ids=input_ids, + attention_mask=attention_mask, + output_hidden_states=True, + use_cache=False, + ) + prompt_embeds = postprocess_func(outputs, text_inputs) + if dtype is not None: + prompt_embeds = prompt_embeds.to(dtype=dtype) + + embeds_list.append(prompt_embeds) + if is_flux_v1: + pooled_embeds_list.append(outputs.pooler_output) + if return_attention_mask: + attn_masks_list.append(attention_mask) + + # Shape results according to return_type + if return_type == "list": + if return_attention_mask: + return embeds_list, attn_masks_list, pooled_embeds_list + return embeds_list, pooled_embeds_list + + if return_type == "dict": + key_strs = [str(i) for i in indices] + embeds_dict = {k: v for k, v in zip(key_strs, embeds_list, strict=False)} + if return_attention_mask: + attn_dict = { + k: v for k, v in zip(key_strs, attn_masks_list, strict=False) + } + return embeds_dict, attn_dict + return embeds_dict + + # return_type == "stack" + # Validate shapes are compatible + base_shape = list(embeds_list[0].shape) + for t in embeds_list[1:]: + if list(t.shape) != base_shape: + raise ValueError( + f"Cannot stack embeddings with differing shapes: {[list(t.shape) for t in embeds_list]}" + ) + stacked_embeds = torch.stack(embeds_list, dim=0) + if return_attention_mask: + base_mask_shape = list(attn_masks_list[0].shape) + for m in attn_masks_list[1:]: + if list(m.shape) != base_mask_shape: + raise ValueError( + f"Cannot stack attention masks with differing shapes: {[list(m.shape) for m in attn_masks_list]}" + ) + stacked_masks = torch.stack(attn_masks_list, dim=0) + return stacked_embeds, stacked_masks + return stacked_embeds + + def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Verify text encoding stage outputs.""" + result = VerificationResult() + result.add_check( + "prompt_embeds", batch.prompt_embeds, V.list_of_tensors_min_dims(2) + ) + result.add_check( + "negative_prompt_embeds", + batch.negative_prompt_embeds, + lambda x: not batch.do_classifier_free_guidance + or V.list_of_tensors_with_min_dims(x, 2), + ) + if batch.debug: + logger.debug(f"{batch.prompt_embeds=}") + logger.debug(f"{batch.negative_prompt_embeds=}") + return result diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/timestep_preparation.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/timestep_preparation.py new file mode 100644 index 00000000..ca0e65f6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/timestep_preparation.py @@ -0,0 +1,148 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +Timestep preparation stages for diffusion pipelines. + +This module contains implementations of timestep preparation stages for diffusion pipelines. +""" + +import inspect +from typing import Any, Callable, Tuple + +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import ( + PipelineStage, + StageParallelismType, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + StageValidators as V, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + VerificationResult, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class TimestepPreparationStage(PipelineStage): + """ + Stage for preparing timesteps for the diffusion process. + + This stage handles the preparation of the timestep sequence that will be used + during the diffusion process. + """ + + def __init__( + self, + scheduler, + prepare_extra_set_timesteps_kwargs: list[ + Callable[[Req, ServerArgs], Tuple[str, Any]] + ] = [], + ) -> None: + self.scheduler = scheduler + self.prepare_extra_set_timesteps_kwargs = prepare_extra_set_timesteps_kwargs + + @property + def parallelism_type(self) -> StageParallelismType: + return StageParallelismType.REPLICATED + + def forward( + self, + batch: Req, + server_args: ServerArgs, + ) -> Req: + """ + Prepare timesteps for the diffusion process. + + Args: + batch: The current batch information. + server_args: The inference arguments. + + Returns: + The batch with prepared timesteps. + """ + scheduler = self.scheduler + device = get_local_torch_device() + num_inference_steps = batch.num_inference_steps + timesteps = batch.timesteps + sigmas = batch.sigmas + n_tokens = batch.n_tokens + + sigmas = server_args.pipeline_config.prepare_sigmas(sigmas, num_inference_steps) + + # Prepare extra kwargs for set_timesteps + extra_set_timesteps_kwargs = {} + if ( + n_tokens is not None + and "n_tokens" in inspect.signature(scheduler.set_timesteps).parameters + ): + extra_set_timesteps_kwargs["n_tokens"] = n_tokens + + for callee in self.prepare_extra_set_timesteps_kwargs: + key, value = callee(batch, server_args) + assert isinstance(key, str) + extra_set_timesteps_kwargs[key] = value + + # Handle custom timesteps or sigmas + if timesteps is not None and sigmas is not None: + raise ValueError( + "Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values" + ) + + if timesteps is not None: + accepts_timesteps = ( + "timesteps" in inspect.signature(scheduler.set_timesteps).parameters + ) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps( + timesteps=timesteps, device=device, **extra_set_timesteps_kwargs + ) + timesteps = scheduler.timesteps + elif sigmas is not None: + accept_sigmas = ( + "sigmas" in inspect.signature(scheduler.set_timesteps).parameters + ) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps( + sigmas=sigmas, device=device, **extra_set_timesteps_kwargs + ) + timesteps = scheduler.timesteps + else: + scheduler.set_timesteps( + num_inference_steps, device=device, **extra_set_timesteps_kwargs + ) + timesteps = scheduler.timesteps + + # Update batch with prepared timesteps + batch.timesteps = timesteps + self.log_debug(f"timesteps: {timesteps}") + return batch + + def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Verify timestep preparation stage inputs.""" + result = VerificationResult() + result.add_check( + "num_inference_steps", batch.num_inference_steps, V.positive_int + ) + result.add_check("timesteps", batch.timesteps, V.none_or_tensor) + result.add_check("sigmas", batch.sigmas, V.none_or_list) + result.add_check("n_tokens", batch.n_tokens, V.none_or_positive_int) + return result + + def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Verify timestep preparation stage outputs.""" + result = VerificationResult() + result.add_check("timesteps", batch.timesteps, [V.is_tensor, V.with_dims(1)]) + return result diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/validators.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/validators.py new file mode 100644 index 00000000..1ca9e992 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/pipelines_core/stages/validators.py @@ -0,0 +1,522 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +""" +Common validators for pipeline stage verification. + +This module provides reusable validation functions that can be used across +all pipeline stages for input/output verification. +""" + +from collections.abc import Callable +from typing import Any + +import torch + + +class StageValidators: + """Common validators for pipeline stages.""" + + @staticmethod + def not_none(value: Any) -> bool: + """Check if value is not None.""" + return value is not None + + @staticmethod + def positive_int(value: Any) -> bool: + """Check if value is a positive integer.""" + return isinstance(value, int) and value > 0 + + @staticmethod + def non_negative_int(value: Any) -> bool: + """Check if value is a non-negative float.""" + return isinstance(value, int | float) and value >= 0 + + @staticmethod + def positive_float(value: Any) -> bool: + """Check if value is a positive float.""" + return isinstance(value, int | float) and value > 0 + + @staticmethod + def non_negative_float(value: Any) -> bool: + """Check if value is a non-negative float.""" + return isinstance(value, int | float) and value >= 0 + + @staticmethod + def divisible_by(value: Any, divisor: int) -> bool: + """Check if value is divisible by divisor.""" + return value is not None and isinstance(value, int) and value % divisor == 0 + + @staticmethod + def is_tensor(value: Any) -> bool: + """Check if value is a torch tensor and doesn't contain NaN values.""" + if not isinstance(value, torch.Tensor): + return False + return not torch.isnan(value).any().item() + + @staticmethod + def tensor_with_dims(value: Any, dims: int) -> bool: + """Check if value is a tensor with specific dimensions and no NaN values.""" + if not isinstance(value, torch.Tensor): + return False + if value.dim() != dims: + return False + return not torch.isnan(value).any().item() + + @staticmethod + def tensor_min_dims(value: Any, min_dims: int) -> bool: + """Check if value is a tensor with at least min_dims dimensions and no NaN values.""" + if not isinstance(value, torch.Tensor): + return False + if value.dim() < min_dims: + return False + return not torch.isnan(value).any().item() + + @staticmethod + def tensor_shape_matches(value: Any, expected_shape: tuple) -> bool: + """Check if tensor shape matches expected shape (None for any size) and no NaN values.""" + if not isinstance(value, torch.Tensor): + return False + if len(value.shape) != len(expected_shape): + return False + for actual, expected in zip(value.shape, expected_shape, strict=True): + if expected is not None and actual != expected: + return False + return not torch.isnan(value).any().item() + + @staticmethod + def list_not_empty(value: Any) -> bool: + """Check if value is a non-empty list.""" + return isinstance(value, list) and len(value) > 0 + + @staticmethod + def list_length(value: Any, length: int) -> bool: + """Check if list has specific length.""" + return isinstance(value, list) and len(value) == length + + @staticmethod + def list_min_length(value: Any, min_length: int) -> bool: + """Check if list has at least min_length items.""" + return isinstance(value, list) and len(value) >= min_length + + @staticmethod + def string_not_empty(value: Any) -> bool: + """Check if value is a non-empty string.""" + return isinstance(value, str) and len(value.strip()) > 0 + + @staticmethod + def string_not_none(value: Any) -> bool: + """Check if value is a non-empty string.""" + return isinstance(value, str) and len(value) > 0 + + @staticmethod + def string_or_list_strings(value: Any) -> bool: + """Check if value is a string or list of strings.""" + if isinstance(value, str): + return True + if isinstance(value, list): + return all(isinstance(item, str) for item in value) + return False + + @staticmethod + def bool_value(value: Any) -> bool: + """Check if value is a boolean.""" + return isinstance(value, bool) + + @staticmethod + def generator_or_list_generators(value: Any) -> bool: + """Check if value is a Generator or list of Generators.""" + if isinstance(value, torch.Generator): + return True + if isinstance(value, list): + return all(isinstance(item, torch.Generator) for item in value) + return False + + @staticmethod + def is_list(value: Any) -> bool: + """Check if value is a list (can be empty).""" + return isinstance(value, list) + + @staticmethod + def is_tuple(value: Any) -> bool: + """Check if value is a tuple.""" + return isinstance(value, tuple) + + @staticmethod + def none_or_tensor(value: Any) -> bool: + """Check if value is None or a tensor without NaN values.""" + if value is None: + return True + if not isinstance(value, torch.Tensor): + return False + return not torch.isnan(value).any().item() + + @staticmethod + def list_of_tensors_with_dims(value: Any, dims: int) -> bool: + """Check if value is a non-empty list where all items are tensors with specific dimensions and no NaN values.""" + if not isinstance(value, list) or len(value) == 0: + return False + for item in value: + if not isinstance(item, torch.Tensor): + return False + if item.dim() != dims: + return False + if torch.isnan(item).any().item(): + return False + return True + + @staticmethod + def list_of_tensors(value: Any) -> bool: + """Check if value is a non-empty list where all items are tensors without NaN values.""" + if not isinstance(value, list) or len(value) == 0: + return False + for item in value: + if not isinstance(item, torch.Tensor): + return False + if torch.isnan(item).any().item(): + return False + return True + + @staticmethod + def list_of_tensors_with_min_dims(value: Any, min_dims: int) -> bool: + """Check if value is a non-empty list where all items are tensors with at least min_dims dimensions and no NaN values.""" + if not isinstance(value, list) or len(value) == 0: + return False + for item in value: + if not isinstance(item, torch.Tensor): + return False + if item.dim() < min_dims: + return False + if torch.isnan(item).any().item(): + return False + return True + + @staticmethod + def none_or_tensor_with_dims(dims: int) -> Callable[[Any], bool]: + """Return a validator that checks if value is None or a tensor with specific dimensions and no NaN values.""" + + def validator(value: Any) -> bool: + if value is None: + return True + if not isinstance(value, torch.Tensor): + return False + if value.dim() != dims: + return False + return not torch.isnan(value).any().item() + + return validator + + @staticmethod + def none_or_list(value: Any) -> bool: + """Check if value is None or a list.""" + return value is None or isinstance(value, list) + + @staticmethod + def none_or_positive_int(value: Any) -> bool: + """Check if value is None or a positive integer.""" + return value is None or (isinstance(value, int) and value > 0) + + # Helper methods that return functions for common patterns + @staticmethod + def with_dims(dims: int) -> Callable[[Any], bool]: + """Return a validator that checks if tensor has specific dimensions and no NaN values.""" + + def validator(value: Any) -> bool: + return StageValidators.tensor_with_dims(value, dims) + + return validator + + @staticmethod + def min_dims(min_dims: int) -> Callable[[Any], bool]: + """Return a validator that checks if tensor has at least min_dims dimensions and no NaN values.""" + + def validator(value: Any) -> bool: + return StageValidators.tensor_min_dims(value, min_dims) + + return validator + + @staticmethod + def divisible(divisor: int) -> Callable[[Any], bool]: + """Return a validator that checks if value is divisible by divisor.""" + + def validator(value: Any) -> bool: + return StageValidators.divisible_by(value, divisor) + + return validator + + @staticmethod + def positive_int_divisible(divisor: int) -> Callable[[Any], bool]: + """Return a validator that checks if value is a positive integer divisible by divisor.""" + + def validator(value: Any) -> bool: + return ( + isinstance(value, int) + and value > 0 + and StageValidators.divisible_by(value, divisor) + ) + + return validator + + @staticmethod + def list_of_tensors_dims(dims: int) -> Callable[[Any], bool]: + """Return a validator that checks if value is a list of tensors with specific dimensions and no NaN values.""" + + def validator(value: Any) -> bool: + return StageValidators.list_of_tensors_with_dims(value, dims) + + return validator + + @staticmethod + def list_of_tensors_min_dims(min_dims: int) -> Callable[[Any], bool]: + """Return a validator that checks if value is a list of tensors with at least min_dims dimensions and no NaN values.""" + + def validator(value: Any) -> bool: + return StageValidators.list_of_tensors_with_min_dims(value, min_dims) + + return validator + + +class ValidationFailure: + """Details about a specific validation failure.""" + + def __init__( + self, + validator_name: str, + actual_value: Any, + expected: str | None = None, + error_msg: str | None = None, + ): + self.validator_name = validator_name + self.actual_value = actual_value + self.expected = expected + self.error_msg = error_msg + + def __str__(self) -> str: + parts = [f"Validator '{self.validator_name}' failed"] + + if self.error_msg: + parts.append(f"Error: {self.error_msg}") + + # Add actual value info (but limit very long representations) + actual_str = self._format_value(self.actual_value) + parts.append(f"Actual: {actual_str}") + + if self.expected: + parts.append(f"Expected: {self.expected}") + + return ". ".join(parts) + + def _format_value(self, value: Any) -> str: + """Format a value for display in error messages.""" + if value is None: + return "None" + elif isinstance(value, torch.Tensor): + return f"tensor(shape={list(value.shape)}, dtype={value.dtype})" + elif isinstance(value, list): + if len(value) == 0: + return "[]" + elif len(value) <= 3: + item_strs = [self._format_value(item) for item in value] + return f"[{', '.join(item_strs)}]" + else: + return f"list(length={len(value)}, first_item={self._format_value(value[0])})" + elif isinstance(value, str): + if len(value) > 50: + return f"'{value[:47]}...'" + else: + return f"'{value}'" + else: + return f"{type(value).__name__}({value})" + + +class VerificationResult: + """Wrapper class for stage verification results.""" + + def __init__(self) -> None: + self._checks: dict[str, bool] = {} + self._failures: dict[str, list[ValidationFailure]] = {} + + def add_check( + self, + field_name: str, + value: Any, + validators: Callable[[Any], bool] | list[Callable[[Any], bool]], + ) -> "VerificationResult": + """ + Add a validation check for a field. + + Args: + field_name: Name of the field being checked + value: The actual value to validate + validators: Single validation function or list of validation functions. + Each function will be called with the value as its first argument. + + Returns: + Self for method chaining + + Examples: + # Single validator + result.add_check("tensor", my_tensor, V.is_tensor) + + # Multiple validators (all must pass) + result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)]) + + # Using partial functions for parameters + result.add_check("height", batch.height, [V.not_none, V.divisible(8)]) + """ + if not isinstance(validators, list): + validators = [validators] + + failures = [] + all_passed = True + + # Apply all validators and collect detailed failure info + for validator in validators: + try: + passed = validator(value) + if not passed: + all_passed = False + failure = self._create_validation_failure(validator, value) + failures.append(failure) + except Exception as e: + # If any validator raises an exception, consider the check failed + all_passed = False + validator_name = getattr(validator, "__name__", str(validator)) + failure = ValidationFailure( + validator_name=validator_name, + actual_value=value, + error_msg=f"Exception during validation: {str(e)}", + ) + failures.append(failure) + + self._checks[field_name] = all_passed + if not all_passed: + self._failures[field_name] = failures + + return self + + def _create_validation_failure( + self, validator: Callable, value: Any + ) -> ValidationFailure: + """Create a ValidationFailure with detailed information.""" + validator_name = getattr(validator, "__name__", str(validator)) + + # Try to extract meaningful expected value info based on validator type + expected = None + error_msg = None + + # Handle common validator patterns + if hasattr(validator, "__closure__") and validator.__closure__: + # This is likely a closure (like our helper functions) + if "dims" in validator_name or "with_dims" in str(validator): + if isinstance(value, torch.Tensor): + expected = f"tensor with {validator.__closure__[0].cell_contents} dimensions" + else: + expected = "tensor with specific dimensions" + elif "divisible" in str(validator): + expected = ( + f"integer divisible by {validator.__closure__[0].cell_contents}" + ) + + # Handle specific validator types and check for NaN values + if validator_name == "is_tensor": + expected = "torch.Tensor without NaN values" + if isinstance(value, torch.Tensor) and torch.isnan(value).any().item(): + error_msg = ( + f"tensor contains {torch.isnan(value).sum().item()} NaN values" + ) + elif validator_name == "positive_int": + expected = "positive integer" + elif validator_name == "not_none": + expected = "non-None value" + elif validator_name == "list_not_empty": + expected = "non-empty list" + elif validator_name == "bool_value": + expected = "boolean value" + elif ( + "tensor_with_dims" in validator_name or "tensor_min_dims" in validator_name + ): + if isinstance(value, torch.Tensor): + if torch.isnan(value).any().item(): + error_msg = f"tensor has {value.dim()} dimensions but contains {torch.isnan(value).sum().item()} NaN values" + else: + error_msg = f"tensor has {value.dim()} dimensions" + elif validator_name == "is_list": + expected = "list" + elif validator_name == "none_or_tensor": + expected = "None or tensor without NaN values" + if isinstance(value, torch.Tensor) and torch.isnan(value).any().item(): + error_msg = ( + f"tensor contains {torch.isnan(value).sum().item()} NaN values" + ) + elif validator_name == "list_of_tensors": + expected = "non-empty list of tensors without NaN values" + if isinstance(value, list) and len(value) > 0: + nan_count = 0 + for item in value: + if ( + isinstance(item, torch.Tensor) + and torch.isnan(item).any().item() + ): + nan_count += torch.isnan(item).sum().item() + if nan_count > 0: + error_msg = ( + f"list contains tensors with total {nan_count} NaN values" + ) + elif "list_of_tensors_with_dims" in validator_name: + expected = ( + "non-empty list of tensors with specific dimensions and no NaN values" + ) + if isinstance(value, list) and len(value) > 0: + nan_count = 0 + for item in value: + if ( + isinstance(item, torch.Tensor) + and torch.isnan(item).any().item() + ): + nan_count += torch.isnan(item).sum().item() + if nan_count > 0: + error_msg = ( + f"list contains tensors with total {nan_count} NaN values" + ) + + return ValidationFailure( + validator_name=validator_name, + actual_value=value, + expected=expected, + error_msg=error_msg, + ) + + def is_valid(self) -> bool: + """Check if all validations passed.""" + return all(self._checks.values()) + + def get_failed_fields(self) -> list[str]: + """Get list of fields that failed validation.""" + return [field for field, passed in self._checks.items() if not passed] + + def get_detailed_failures(self) -> dict[str, list[ValidationFailure]]: + """Get detailed failure information for each failed field.""" + return self._failures.copy() + + def get_failure_summary(self) -> str: + """Get a comprehensive summary of all validation failures.""" + if self.is_valid(): + return "All validations passed" + + summary_parts = [] + for field_name, failures in self._failures.items(): + field_summary = f"\n Field '{field_name}':" + for i, failure in enumerate(failures, 1): + field_summary += f"\n {i}. {failure}" + summary_parts.append(field_summary) + + return "Validation failures:" + "".join(summary_parts) + + def to_dict(self) -> dict: + """Convert to dictionary for backward compatibility.""" + return self._checks.copy() + + +# Alias for convenience +V = StageValidators diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/__init__.py new file mode 100644 index 00000000..c87fb3aa --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/__init__.py @@ -0,0 +1,172 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/platforms/__init__.py + +import traceback +from typing import TYPE_CHECKING + +# imported by other files, do not remove +from sglang.multimodal_gen.runtime.platforms.interface import ( # noqa: F401 + AttentionBackendEnum, + Platform, + PlatformEnum, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.utils import resolve_obj_by_qualname + +logger = init_logger(__name__) + + +def cuda_platform_plugin() -> str | None: + is_cuda = False + + try: + from sglang.multimodal_gen.utils import import_pynvml + + pynvml = import_pynvml() # type: ignore[no-untyped-call] + pynvml.nvmlInit() + try: + # NOTE: Edge case: sgl_diffusion cpu build on a GPU machine. + # Third-party pynvml can be imported in cpu build, + # we need to check if sgl_diffusion is built with cpu too. + # Otherwise, sgl_diffusion will always activate cuda plugin + # on a GPU machine, even if in a cpu build. + is_cuda = pynvml.nvmlDeviceGetCount() > 0 + finally: + pynvml.nvmlShutdown() + except Exception as e: + if "nvml" not in e.__class__.__name__.lower(): + # If the error is not related to NVML, re-raise it. + raise e + + # CUDA is supported on Jetson, but NVML may not be. + import os + + def cuda_is_jetson() -> bool: + return os.path.isfile("/etc/nv_tegra_release") or os.path.exists( + "/sys/class/tegra-firmware" + ) + + if cuda_is_jetson(): + is_cuda = True + if is_cuda: + logger.info("CUDA is available") + + return ( + "sglang.multimodal_gen.runtime.platforms.cuda.CudaPlatform" if is_cuda else None + ) + + +def mps_platform_plugin() -> str | None: + """Detect if MPS (Metal Performance Shaders) is available on macOS.""" + is_mps = False + + try: + import torch + + if torch.backends.mps.is_available(): + is_mps = True + logger.info("MPS (Metal Performance Shaders) is available") + except Exception as e: + logger.info("MPS detection failed: %s", e) + + return "sglang.multimodal_gen.runtime.platforms.mps.MpsPlatform" if is_mps else None + + +def cpu_platform_plugin() -> str | None: + """Detect if CPU platform should be used.""" + # CPU is always available as a fallback + return "sglang.multimodal_gen.runtime.platforms.cpu.CpuPlatform" + + +def rocm_platform_plugin() -> str | None: + is_rocm = False + + try: + import amdsmi + + amdsmi.amdsmi_init() + try: + if len(amdsmi.amdsmi_get_processor_handles()) > 0: + is_rocm = True + logger.info("ROCm platform is available") + finally: + amdsmi.amdsmi_shut_down() + except Exception as e: + logger.info("ROCm platform is unavailable: %s", e) + + return ( + "sglang.multimodal_gen.runtime.platforms.rocm.RocmPlatform" if is_rocm else None + ) + + +builtin_platform_plugins = { + "cuda": cuda_platform_plugin, + "rocm": rocm_platform_plugin, + "mps": mps_platform_plugin, + "cpu": cpu_platform_plugin, +} + + +def resolve_current_platform_cls_qualname() -> str: + # TODO(will): if we need to support other platforms, we should consider if + # vLLM's plugin architecture is suitable for our needs. + + # Try MPS first on macOS + platform_cls_qualname = mps_platform_plugin() + if platform_cls_qualname is not None: + return platform_cls_qualname + + # Fall back to ROCm + platform_cls_qualname = rocm_platform_plugin() + if platform_cls_qualname is not None: + return platform_cls_qualname + + # Fall back to CUDA + platform_cls_qualname = cuda_platform_plugin() + if platform_cls_qualname is not None: + return platform_cls_qualname + + # Fall back to CPU as last resort + platform_cls_qualname = cpu_platform_plugin() + if platform_cls_qualname is not None: + return platform_cls_qualname + + raise RuntimeError("No platform plugin found. Please check your " "installation.") + + +_current_platform: Platform | None = None +_init_trace: str = "" + +if TYPE_CHECKING: + current_platform: Platform + + +def __getattr__(name: str): + if name == "current_platform": + # lazy init current_platform. + # 1. out-of-tree platform plugins need `from sglang.multimodal_gen.runtime.platforms import + # Platform` so that they can inherit `Platform` class. Therefore, + # we cannot resolve `current_platform` during the import of + # `sglang.multimodal_gen.runtime.platforms`. + # 2. when users use out-of-tree platform plugins, they might run + # `import sgl_diffusion`, some sgl_diffusion internal code might access + # `current_platform` during the import, and we need to make sure + # `current_platform` is only resolved after the plugins are loaded + # (we have tests for this, if any developer violate this, they will + # see the test failures). + global _current_platform + if _current_platform is None: + platform_cls_qualname = resolve_current_platform_cls_qualname() + _current_platform = resolve_obj_by_qualname(platform_cls_qualname)() + global _init_trace + _init_trace = "".join(traceback.format_stack()) + return _current_platform + elif name in globals(): + return globals()[name] + else: + raise AttributeError(f"No attribute named '{name}' exists in {__name__}.") + + +__all__ = ["Platform", "PlatformEnum", "current_platform", "_init_trace"] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/cpu.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/cpu.py new file mode 100644 index 00000000..5186d248 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/cpu.py @@ -0,0 +1,61 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/platforms/cpu.py + +import platform + +import torch + +from sglang.multimodal_gen.runtime.platforms.interface import ( + CpuArchEnum, + Platform, + PlatformEnum, +) + + +class CpuPlatform(Platform): + _enum = PlatformEnum.CPU + device_name = "CPU" + device_type = "cpu" + dispatch_key = "CPU" + + @classmethod + def get_cpu_architecture(cls) -> CpuArchEnum: + """Get the CPU architecture.""" + machine = platform.machine().lower() + if machine in ("x86_64", "amd64", "i386", "i686"): + return CpuArchEnum.X86 + elif machine in ("arm64", "aarch64"): + return CpuArchEnum.ARM + else: + return CpuArchEnum.UNSPECIFIED + + @classmethod + def get_device_name(cls, device_id: int = 0) -> str: + return platform.processor() + + @classmethod + def get_device_uuid(cls, device_id: int = 0) -> str: + return platform.machine() + + @classmethod + def get_device_total_memory(cls, device_id: int = 0) -> int: + # This is a rough estimate for CPU memory + # In practice, you might want to use psutil or similar + return 0 + + @classmethod + def is_async_output_supported(cls, enforce_eager: bool | None) -> bool: + return True + + @classmethod + def get_current_memory_usage( + cls, device: torch.types.Device | None = None + ) -> float: + # For CPU, we can't easily get memory usage without additional libraries + return 0.0 + + @classmethod + def get_device_communicator_cls(cls) -> str: + return "sglang.multimodal_gen.runtime.distributed.device_communicators.cpu_communicator.CpuCommunicator" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/cuda.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/cuda.py new file mode 100644 index 00000000..90320dae --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/cuda.py @@ -0,0 +1,438 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/platforms/cuda.py +"""Code inside this file can safely assume cuda platform, e.g. importing +pynvml. However, it should not initialize cuda context. +""" + +import os +from collections.abc import Callable +from functools import lru_cache, wraps +from typing import TypeVar + +import torch +from typing_extensions import ParamSpec + +from sglang.multimodal_gen.runtime.platforms.interface import ( + AttentionBackendEnum, + DeviceCapability, + Platform, + PlatformEnum, +) +from sglang.multimodal_gen.runtime.utils.common import is_blackwell +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.utils import import_pynvml + +logger = init_logger(__name__) + +_P = ParamSpec("_P") +_R = TypeVar("_R") + +pynvml = import_pynvml() # type: ignore[no-untyped-call] + +# pytorch 2.5 uses cudnn sdpa by default, which will cause crash on some models +# see https://github.com/huggingface/diffusers/issues/9704 for details +torch.backends.cuda.enable_cudnn_sdp(False) + + +def device_id_to_physical_device_id(device_id: int) -> int: + if "CUDA_VISIBLE_DEVICES" in os.environ: + device_ids = os.environ["CUDA_VISIBLE_DEVICES"].split(",") + if device_ids == [""]: + msg = ( + "CUDA_VISIBLE_DEVICES is set to empty string, which means" + " GPU support is disabled. If you are using ray, please unset" + " the environment variable `CUDA_VISIBLE_DEVICES` inside the" + " worker/actor. " + "Check https://github.com/vllm-project/vllm/issues/8402 for" + " more information." + ) + raise RuntimeError(msg) + physical_device_id = device_ids[device_id] + return int(physical_device_id) + else: + return device_id + + +def with_nvml_context(fn: Callable[_P, _R]) -> Callable[_P, _R]: + @wraps(fn) + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: + pynvml.nvmlInit() + try: + return fn(*args, **kwargs) + finally: + pynvml.nvmlShutdown() + + return wrapper + + +class CudaPlatformBase(Platform): + _enum = PlatformEnum.CUDA + device_name: str = "cuda" + device_type: str = "cuda" + dispatch_key: str = "CUDA" + device_control_env_var: str = "CUDA_VISIBLE_DEVICES" + + @classmethod + def get_device_capability(cls, device_id: int = 0) -> DeviceCapability | None: + raise NotImplementedError + + @classmethod + def get_device_name(cls, device_id: int = 0) -> str: + raise NotImplementedError + + @classmethod + def get_device_total_memory(cls, device_id: int = 0) -> int: + raise NotImplementedError + + @classmethod + def is_async_output_supported(cls, enforce_eager: bool | None) -> bool: + if enforce_eager: + logger.warning( + "To see benefits of async output processing, enable CUDA " + "graph. Since, enforce-eager is enabled, async output " + "processor cannot be used" + ) + return False + return True + + @classmethod + def is_full_nvlink(cls, device_ids: list[int]) -> bool: + raise NotImplementedError + + @classmethod + def log_warnings(cls) -> None: + pass + + @classmethod + def get_current_memory_usage( + cls, device: torch.types.Device | None = None + ) -> float: + torch.cuda.reset_peak_memory_stats(device) + return float(torch.cuda.max_memory_allocated(device)) + + @classmethod + def get_attn_backend_cls_str( + cls, + selected_backend: AttentionBackendEnum | None, + head_size: int, + dtype: torch.dtype, + ) -> str: + # TODO(will): maybe come up with a more general interface for local attention + # if distributed is False, we always try to use Flash attn + if selected_backend == AttentionBackendEnum.SLIDING_TILE_ATTN: + try: + from st_attn import sliding_tile_attention # noqa: F401 + + from sglang.multimodal_gen.runtime.layers.attention.backends.sliding_tile_attn import ( # noqa: F401 + SlidingTileAttentionBackend, + ) + + logger.info("Using Sliding Tile Attention backend.") + + return "sglang.multimodal_gen.runtime.layers.attention.backends.sliding_tile_attn.SlidingTileAttentionBackend" + except ImportError as e: + logger.error( + "Failed to import Sliding Tile Attention backend: %s", str(e) + ) + raise ImportError( + "Sliding Tile Attention backend is not installed. " + ) from e + elif selected_backend == AttentionBackendEnum.SAGE_ATTN: + try: + from sageattention import sageattn # noqa: F401 + + from sglang.multimodal_gen.runtime.layers.attention.backends.sage_attn import ( # noqa: F401 + SageAttentionBackend, + ) + + logger.info("Using Sage Attention backend.") + + return "sglang.multimodal_gen.runtime.layers.attention.backends.sage_attn.SageAttentionBackend" + except ImportError as e: + logger.info(e) + logger.info( + "Sage Attention backend is not installed. Fall back to Flash Attention." + ) + elif selected_backend == AttentionBackendEnum.SAGE_ATTN_THREE: + try: + from sglang.multimodal_gen.runtime.layers.attention.backends.sage_attn3 import ( # noqa: F401 + SageAttention3Backend, + ) + from sglang.multimodal_gen.runtime.layers.attention.backends.sageattn.api import ( # noqa: F401 + sageattn_blackwell, + ) + + logger.info("Using Sage Attention 3 backend.") + + return "sglang.multimodal_gen.runtime.layers.attention.backends.sage_attn3.SageAttention3Backend" + except ImportError as e: + logger.info(e) + logger.info( + "Sage Attention 3 backend is not installed. Fall back to Flash Attention." + ) + elif selected_backend == AttentionBackendEnum.VIDEO_SPARSE_ATTN: + try: + from vsa import block_sparse_attn # noqa: F401 + + from sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn import ( # noqa: F401 + VideoSparseAttentionBackend, + ) + + logger.info("Using Video Sparse Attention backend.") + + return "sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn.VideoSparseAttentionBackend" + except ImportError as e: + logger.error( + "Failed to import Video Sparse Attention backend: %s", str(e) + ) + raise ImportError( + "Video Sparse Attention backend is not installed. " + ) from e + elif selected_backend == AttentionBackendEnum.VMOBA_ATTN: + try: + from kernel.attn.vmoba_attn.vmoba import moba_attn_varlen # noqa: F401 + + from sglang.multimodal_gen.runtime.layers.attention.backends.vmoba import ( # noqa: F401 + VMOBAAttentionBackend, + ) + + logger.info("Using Video MOBA Attention backend.") + + return "sglang.multimodal_gen.runtime.layers.attention.backends.vmoba.VMOBAAttentionBackend" + except ImportError as e: + logger.error( + "Failed to import Video MoBA Attention backend: %s", str(e) + ) + raise ImportError( + "Video MoBA Attention backend is not installed. " + ) from e + elif selected_backend == AttentionBackendEnum.AITER: + logger.info("Using AITer backend.") + return "sglang.multimodal_gen.runtime.layers.attention.backends.aiter.AITerBackend" + elif selected_backend == AttentionBackendEnum.TORCH_SDPA: + logger.info("Using Torch SDPA backend.") + return "sglang.multimodal_gen.runtime.layers.attention.backends.sdpa.SDPABackend" + elif selected_backend in [ + AttentionBackendEnum.FA, + ]: + if is_blackwell(): + from sglang.multimodal_gen.runtime.layers.attention.backends.flash_attn import ( + set_fa_ver, + ) + + set_fa_ver(4) + target_backend = AttentionBackendEnum.FA + elif selected_backend: + raise ValueError(f"Invalid attention backend for {cls.device_name}") + else: + if is_blackwell(): + from sglang.multimodal_gen.runtime.layers.attention.backends.flash_attn import ( + set_fa_ver, + ) + + set_fa_ver(4) + target_backend = AttentionBackendEnum.FA + + if not cls.has_device_capability(80): + logger.info( + "Cannot use FlashAttention backend for Volta and Turing " "GPUs." + ) + target_backend = AttentionBackendEnum.TORCH_SDPA + elif dtype not in (torch.float16, torch.bfloat16): + logger.info( + "Cannot use FlashAttention backend for dtype other than " + "torch.float16 or torch.bfloat16." + ) + target_backend = AttentionBackendEnum.TORCH_SDPA + + # FlashAttn is valid for the model, checking if the package is + # installed. + if target_backend == AttentionBackendEnum.FA: + try: + from sglang.multimodal_gen.runtime.layers.attention.backends.flash_attn import ( # noqa: F401 + FlashAttentionBackend, + ) + + supported_sizes = FlashAttentionBackend.get_supported_head_sizes() + if head_size not in supported_sizes: + logger.info( + "Cannot use FlashAttention backend for head size %d.", + head_size, + ) + target_backend = AttentionBackendEnum.TORCH_SDPA + except ImportError: + logger.info( + "Cannot use FlashAttention backend because the " + "flash_attn package is not found. " + "Make sure that flash_attn was built and installed " + "(on by default)." + ) + target_backend = AttentionBackendEnum.TORCH_SDPA + + if target_backend == AttentionBackendEnum.TORCH_SDPA: + logger.info("Using Torch SDPA backend.") + + return "sglang.multimodal_gen.runtime.layers.attention.backends.sdpa.SDPABackend" + + logger.info("Using FlashAttention (FA3 for hopper, FA4 for blackwell) backend") + + return "sglang.multimodal_gen.runtime.layers.attention.backends.flash_attn.FlashAttentionBackend" + + @classmethod + def get_device_communicator_cls(cls) -> str: + return "sglang.multimodal_gen.runtime.distributed.device_communicators.cuda_communicator.CudaCommunicator" # noqa + + +# NVML utils +# Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`, +# all the related functions work on real physical device ids. +# the major benefit of using NVML is that it will not initialize CUDA +class NvmlCudaPlatform(CudaPlatformBase): + + @classmethod + @lru_cache(maxsize=8) + @with_nvml_context + def get_device_capability(cls, device_id: int = 0) -> DeviceCapability | None: + try: + physical_device_id = device_id_to_physical_device_id(device_id) + handle = pynvml.nvmlDeviceGetHandleByIndex(physical_device_id) + major, minor = pynvml.nvmlDeviceGetCudaComputeCapability(handle) + return DeviceCapability(major=major, minor=minor) + except RuntimeError: + return None + + @classmethod + @lru_cache(maxsize=8) + @with_nvml_context + def has_device_capability( + cls, + capability: tuple[int, int] | int, + device_id: int = 0, + ) -> bool: + try: + return bool(super().has_device_capability(capability, device_id)) + except RuntimeError: + return False + + @classmethod + @lru_cache(maxsize=8) + @with_nvml_context + def get_device_name(cls, device_id: int = 0) -> str: + physical_device_id = device_id_to_physical_device_id(device_id) + return cls._get_physical_device_name(physical_device_id) + + @classmethod + @lru_cache(maxsize=8) + @with_nvml_context + def get_device_uuid(cls, device_id: int = 0) -> str: + physical_device_id = device_id_to_physical_device_id(device_id) + handle = pynvml.nvmlDeviceGetHandleByIndex(physical_device_id) + return str(pynvml.nvmlDeviceGetUUID(handle)) + + @classmethod + @lru_cache(maxsize=8) + @with_nvml_context + def get_device_total_memory(cls, device_id: int = 0) -> int: + physical_device_id = device_id_to_physical_device_id(device_id) + handle = pynvml.nvmlDeviceGetHandleByIndex(physical_device_id) + return int(pynvml.nvmlDeviceGetMemoryInfo(handle).total) + + @classmethod + @with_nvml_context + def is_full_nvlink(cls, physical_device_ids: list[int]) -> bool: + """ + query if the set of gpus are fully connected by nvlink (1 hop) + """ + handles = [pynvml.nvmlDeviceGetHandleByIndex(i) for i in physical_device_ids] + for i, handle in enumerate(handles): + for j, peer_handle in enumerate(handles): + if i < j: + try: + p2p_status = pynvml.nvmlDeviceGetP2PStatus( + handle, + peer_handle, + pynvml.NVML_P2P_CAPS_INDEX_NVLINK, + ) + if p2p_status != pynvml.NVML_P2P_STATUS_OK: + return False + except pynvml.NVMLError: + logger.exception( + "NVLink detection failed. This is normal if" + " your machine has no NVLink equipped." + ) + return False + return True + + @classmethod + def _get_physical_device_name(cls, device_id: int = 0) -> str: + handle = pynvml.nvmlDeviceGetHandleByIndex(device_id) + return str(pynvml.nvmlDeviceGetName(handle)) + + @classmethod + @with_nvml_context + def log_warnings(cls) -> None: + device_ids: int = pynvml.nvmlDeviceGetCount() + if device_ids > 1: + device_names = [cls._get_physical_device_name(i) for i in range(device_ids)] + if ( + len(set(device_names)) > 1 + and os.environ.get("CUDA_DEVICE_ORDER") != "PCI_BUS_ID" + ): + logger.warning( + "Detected different devices in the system: %s. Please" + " make sure to set `CUDA_DEVICE_ORDER=PCI_BUS_ID` to " + "avoid unexpected behavior.", + ", ".join(device_names), + ) + + +class NonNvmlCudaPlatform(CudaPlatformBase): + + @classmethod + def get_device_capability(cls, device_id: int = 0) -> DeviceCapability: + major, minor = torch.cuda.get_device_capability(device_id) + return DeviceCapability(major=major, minor=minor) + + @classmethod + def get_device_name(cls, device_id: int = 0) -> str: + return str(torch.cuda.get_device_name(device_id)) + + @classmethod + def get_device_total_memory(cls, device_id: int = 0) -> int: + device_props = torch.cuda.get_device_properties(device_id) + return int(device_props.total_memory) + + @classmethod + def is_full_nvlink(cls, physical_device_ids: list[int]) -> bool: + logger.exception( + "NVLink detection not possible, as context support was" + " not found. Assuming no NVLink available." + ) + return False + + +# Autodetect either NVML-enabled or non-NVML platform +# based on whether NVML is available. +nvml_available = False +try: + try: + pynvml.nvmlInit() + nvml_available = True + except Exception: + # On Jetson, NVML is not supported. + nvml_available = False +finally: + if nvml_available: + pynvml.nvmlShutdown() + +CudaPlatform = NvmlCudaPlatform if nvml_available else NonNvmlCudaPlatform + +try: + from sphinx.ext.autodoc.mock import _MockModule + + if not isinstance(pynvml, _MockModule): + CudaPlatform.log_warnings() +except ModuleNotFoundError: + CudaPlatform.log_warnings() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/interface.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/interface.py new file mode 100644 index 00000000..660f8e1c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/interface.py @@ -0,0 +1,252 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/platforms/interface.py +from __future__ import annotations + +import enum +import random +from typing import TYPE_CHECKING, NamedTuple + +import numpy as np +import torch + +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.utils import resolve_obj_by_qualname + +if TYPE_CHECKING: + from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import ( + AttentionImpl, + ) + +logger = init_logger(__name__) + + +class AttentionBackendEnum(enum.Enum): + FA = enum.auto() + SLIDING_TILE_ATTN = enum.auto() + TORCH_SDPA = enum.auto() + SAGE_ATTN = enum.auto() + SAGE_ATTN_THREE = enum.auto() + VIDEO_SPARSE_ATTN = enum.auto() + VMOBA_ATTN = enum.auto() + AITER = enum.auto() + NO_ATTENTION = enum.auto() + + def __str__(self): + return self.name.lower() + + +class PlatformEnum(enum.Enum): + CUDA = enum.auto() + ROCM = enum.auto() + TPU = enum.auto() + CPU = enum.auto() + MPS = enum.auto() + OOT = enum.auto() + UNSPECIFIED = enum.auto() + + +class CpuArchEnum(enum.Enum): + X86 = enum.auto() + ARM = enum.auto() + UNSPECIFIED = enum.auto() + + +class DeviceCapability(NamedTuple): + major: int + minor: int + + def as_version_str(self) -> str: + return f"{self.major}.{self.minor}" + + def to_int(self) -> int: + """ + Express device capability as an integer ````. + + It is assumed that the minor version is always a single digit. + """ + assert 0 <= self.minor < 10 + return self.major * 10 + self.minor + + +class Platform: + _enum: PlatformEnum + device_name: str + device_type: str + + # available dispatch keys: + # check https://github.com/pytorch/pytorch/blob/313dac6c1ca0fa0cde32477509cce32089f8532a/torchgen/model.py#L134 # noqa + # use "CPU" as a fallback for platforms not registered in PyTorch + dispatch_key: str = "CPU" + + # The torch.compile backend for compiling simple and + # standalone functions. The default value is "inductor" to keep + # the same behavior as PyTorch. + # NOTE: for the forward part of the model, vLLM has another separate + # compilation strategy. + simple_compile_backend: str = "inductor" + + supported_quantization: list[str] = [] + + def is_cuda(self) -> bool: + return self._enum == PlatformEnum.CUDA + + def is_rocm(self) -> bool: + return self._enum == PlatformEnum.ROCM + + def is_tpu(self) -> bool: + return self._enum == PlatformEnum.TPU + + def is_cpu(self) -> bool: + return self._enum == PlatformEnum.CPU + + def is_out_of_tree(self) -> bool: + return self._enum == PlatformEnum.OOT + + def is_cuda_alike(self) -> bool: + """Stateless version of :func:`torch.cuda.is_available`.""" + return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM) + + def is_mps(self) -> bool: + return self._enum == PlatformEnum.MPS + + @classmethod + def get_attn_backend_cls_str( + cls, + selected_backend: AttentionBackendEnum | None, + head_size: int, + dtype: torch.dtype, + ) -> str: + """Get the attention backend class of a device.""" + return "" + + @classmethod + def get_device_capability( + cls, + device_id: int = 0, + ) -> DeviceCapability | None: + """Stateless version of :func:`torch.cuda.get_device_capability`.""" + return None + + @classmethod + def has_device_capability( + cls, + capability: tuple[int, int] | int, + device_id: int = 0, + ) -> bool: + """ + Test whether this platform is compatible with a device capability. + + The ``capability`` argument can either be: + + - A tuple ``(major, minor)``. + - An integer ````. (See :meth:`DeviceCapability.to_int`) + """ + current_capability = cls.get_device_capability(device_id=device_id) + if current_capability is None: + return False + + if isinstance(capability, tuple): + return current_capability >= capability + + return current_capability.to_int() >= capability + + @classmethod + def get_device_name(cls, device_id: int = 0) -> str: + """Get the name of a device.""" + raise NotImplementedError + + @classmethod + def get_device_uuid(cls, device_id: int = 0) -> str: + """Get the uuid of a device, e.g. the PCI bus ID.""" + raise NotImplementedError + + @classmethod + def get_device_total_memory(cls, device_id: int = 0) -> int: + """Get the total memory of a device in bytes.""" + raise NotImplementedError + + @classmethod + def is_async_output_supported(cls, enforce_eager: bool | None) -> bool: + """ + Check if the current platform supports async output. + """ + raise NotImplementedError + + @classmethod + def inference_mode(cls): + """A device-specific wrapper of `torch.inference_mode`. + + This wrapper is recommended because some hardware backends such as TPU + do not support `torch.inference_mode`. In such a case, they will fall + back to `torch.no_grad` by overriding this method. + """ + return torch.inference_mode(mode=True) + + @classmethod + def seed_everything(cls, seed: int | None = None) -> None: + """ + Set the seed of each random module. + `torch.manual_seed` will set seed on all devices. + + Loosely based on: https://github.com/Lightning-AI/pytorch-lightning/blob/2.4.0/src/lightning/fabric/utilities/seed.py#L20 + """ + if seed is not None: + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + @classmethod + def verify_model_arch(cls, model_arch: str) -> None: + """ + Verify whether the current platform supports the specified model + architecture. + + - This will raise an Error or Warning based on the model support on + the current platform. + - By default all models are considered supported. + """ + pass + + @classmethod + def verify_quantization(cls, quant: str) -> None: + """ + Verify whether the quantization is supported by the current platform. + """ + if cls.supported_quantization and quant not in cls.supported_quantization: + raise ValueError( + f"{quant} quantization is currently not supported in " + f"{cls.device_name}." + ) + + @classmethod + def get_current_memory_usage( + cls, device: torch.types.Device | None = None + ) -> float: + """ + Return the memory usage in bytes. + """ + raise NotImplementedError + + @classmethod + def get_device_communicator_cls(cls) -> str: + """ + Get device specific communicator class for distributed communication. + """ + return "sglang.multimodal_gen.runtime.distributed.device_communicators.base_device_communicator.DeviceCommunicatorBase" # noqa + + @classmethod + def get_cpu_architecture(cls) -> CpuArchEnum: + """Get the CPU architecture of the current platform.""" + return CpuArchEnum.UNSPECIFIED + + def get_attn_backend(self, *args, **kwargs) -> AttentionImpl: + attention_cls_str = self.get_attn_backend_cls_str(*args, **kwargs) + return resolve_obj_by_qualname(attention_cls_str) + + +class UnspecifiedPlatform(Platform): + _enum = PlatformEnum.UNSPECIFIED + device_type = "" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/mps.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/mps.py new file mode 100644 index 00000000..2312ec05 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/mps.py @@ -0,0 +1,88 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 + +import torch + +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum +from sglang.multimodal_gen.runtime.platforms.interface import ( + DeviceCapability, + Platform, + PlatformEnum, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class MpsPlatform(Platform): + _enum = PlatformEnum.MPS + device_name: str = "mps" + device_type: str = "mps" + dispatch_key: str = "MPS" + device_control_env_var: str = "MPS_VISIBLE_DEVICES" + + @classmethod + def get_device_capability(cls, device_id: int = 0) -> DeviceCapability | None: + raise NotImplementedError + + @classmethod + def get_device_name(cls, device_id: int = 0) -> str: + raise NotImplementedError + + @classmethod + def get_device_uuid(cls, device_id: int = 0) -> str: + raise NotImplementedError + + @classmethod + def get_device_total_memory(cls, device_id: int = 0) -> int: + raise NotImplementedError + + @classmethod + def is_async_output_supported(cls, enforce_eager: bool | None) -> bool: + if enforce_eager: + logger.warning( + "To see benefits of async output processing, enable MPS " + "graph. Since, enforce-eager is enabled, async output " + "processor cannot be used" + ) + return False + return True + + @classmethod + def get_current_memory_usage( + cls, device: torch.types.Device | None = None + ) -> float: + return 0.0 + + @classmethod + def get_attn_backend_cls_str( + cls, + selected_backend: AttentionBackendEnum | None, + head_size: int, + dtype: torch.dtype, + ) -> str: + # MPS supports SDPA (Scaled Dot-Product Attention) which is the most compatible + logger.info("Using Torch SDPA backend for MPS.") + return ( + "sglang.multimodal_gen.runtime.layers.attention.backends.sdpa.SDPABackend" + ) + + @classmethod + def get_device_communicator_cls(cls) -> str: + # Use base communicator for MPS + return "sglang.multimodal_gen.runtime.distributed.device_communicators.base_device_communicator.DeviceCommunicatorBase" + + @classmethod + def seed_everything(cls, seed: int | None = None) -> None: + """Set the seed for MPS device.""" + if seed is not None: + import random + + import numpy as np + + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + # MPS doesn't have manual_seed_all like CUDA + # The manual_seed above should be sufficient diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/rocm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/rocm.py new file mode 100644 index 00000000..1e5c370d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/platforms/rocm.py @@ -0,0 +1,138 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from rocm/vllm: https://github.com/ROCm/vllm/blob/v0.7.3%2Brocm/vllm/platforms/rocm.py +""" +This file is a platform abstraction for ROCm GPUs, +adjusted to match the structure and interface of `cuda.py`. +""" + +import torch + +import sglang.multimodal_gen.envs as envs +from sglang.multimodal_gen.runtime.platforms.interface import ( + AttentionBackendEnum, + DeviceCapability, + Platform, + PlatformEnum, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +# ROCm uses the same torch.cuda interface +class RocmPlatform(Platform): + _enum = PlatformEnum.ROCM + device_name: str = "rocm" + device_type: str = "cuda" # torch uses 'cuda' backend string + dispatch_key: str = "CUDA" + device_control_env_var: str = "CUDA_VISIBLE_DEVICES" + + @classmethod + def get_device_capability(cls, device_id: int = 0) -> DeviceCapability: + major, minor = torch.cuda.get_device_capability(device_id) + return DeviceCapability(major=major, minor=minor) + + @classmethod + def get_device_name(cls, device_id: int = 0) -> str: + return str(torch.cuda.get_device_name(device_id)) + + @classmethod + def get_device_total_memory(cls, device_id: int = 0) -> int: + return torch.cuda.get_device_properties(device_id).total_memory + + @classmethod + def is_async_output_supported(cls, enforce_eager: bool | None) -> bool: + if enforce_eager: + logger.warning( + "To see benefits of async output processing, enable CUDA graph. " + "Since enforce-eager is enabled, async output processor cannot be used" + ) + return False + return True + + @classmethod + def log_warnings(cls) -> None: + pass # ROCm-specific warnings can be added here + + @classmethod + def get_current_memory_usage(cls, device: torch.device | None = None) -> float: + torch.cuda.reset_peak_memory_stats(device) + return float(torch.cuda.max_memory_allocated(device)) + + @classmethod + def get_attn_backend_cls_str( + cls, + selected_backend: AttentionBackendEnum | None, + head_size: int, + dtype: torch.dtype, + ) -> str: + logger.info( + "Trying SGLANG_DIFFUSION_ATTENTION_BACKEND=%s", + envs.SGLANG_DIFFUSION_ATTENTION_BACKEND, + ) + + if selected_backend == AttentionBackendEnum.TORCH_SDPA: + logger.info("Using Torch SDPA backend.") + return "sglang.multimodal_gen.runtime.layers.attention.backends.sdpa.SDPABackend" + + elif selected_backend in (AttentionBackendEnum.FA, None): + pass + + elif selected_backend in ( + AttentionBackendEnum.SLIDING_TILE_ATTN, + AttentionBackendEnum.SAGE_ATTN, + ): + raise ValueError( + f"{selected_backend.name} is not supported on {cls.device_name}." + ) + elif selected_backend: + raise ValueError( + f"Invalid attention backend for {cls.device_name}: {selected_backend}" + ) + + target_backend = AttentionBackendEnum.FA + if dtype not in (torch.float16, torch.bfloat16): + logger.info( + "Cannot use FlashAttention backend for dtype other than " + "torch.float16 or torch.bfloat16." + ) + target_backend = AttentionBackendEnum.TORCH_SDPA + + if target_backend == AttentionBackendEnum.FA: + try: + import flash_attn # noqa: F401 + + from sglang.multimodal_gen.runtime.layers.attention.backends.flash_attn import ( # noqa: F401 + FlashAttentionBackend, + ) + + supported_sizes = FlashAttentionBackend.get_supported_head_sizes() + if head_size not in supported_sizes: + logger.info( + "Cannot use FlashAttention-2 backend for head size %d.", + head_size, + ) + target_backend = AttentionBackendEnum.TORCH_SDPA + except ImportError: + logger.info( + "Cannot use FlashAttention backend because the " + "flash_attn package is not found. " + "Make sure that flash_attn was built and installed " + "(on by default)." + ) + target_backend = AttentionBackendEnum.TORCH_SDPA + + if target_backend == AttentionBackendEnum.TORCH_SDPA: + logger.info("Using Torch SDPA backend.") + + return "sglang.multimodal_gen.runtime.layers.attention.backends.sdpa.SDPABackend" + + logger.info("Using Flash Attention backend.") + + return "sglang.multimodal_gen.runtime.layers.attention.backends.flash_attn.FlashAttentionBackend" + + @classmethod + def get_device_communicator_cls(cls) -> str: + return "sglang.multimodal_gen.runtime.distributed.device_communicators.cuda_communicator.CudaCommunicator" # works for ROCm too diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/scheduler_client.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/scheduler_client.py new file mode 100644 index 00000000..c0c0a00a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/scheduler_client.py @@ -0,0 +1,149 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +import asyncio +from typing import Any + +import zmq +import zmq.asyncio + +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +# Using a singleton pattern to hold the ZMQ context and the socket connected to the scheduler +class SchedulerClient: + """ + A gateway for Scheduler, forwarding the ForwardBatch from http endpoints (or somewhere else) to background scheduler, with TCP socket + """ + + _instance = None + + def __new__(cls, *args, **kwargs): + if not cls._instance: + cls._instance = super(SchedulerClient, cls).__new__(cls) + return cls._instance + + def __init__(self, *args, **kwargs): + # Ensure the initialization runs only once for the singleton instance + if getattr(self, "_init_done", False): + return + # Queue + worker to strictly serialize ZeroMQ REQ/REP interactions + self._request_queue = asyncio.Queue() + self._worker_task = None + self._closing = False + self._init_done = True + + def initialize(self, server_args: ServerArgs): + self.server_args = server_args + self.context = zmq.asyncio.Context() + # This is the REQ socket used to connect to the backend Scheduler + self.scheduler_socket = self.context.socket(zmq.REQ) + scheduler_endpoint = server_args.scheduler_endpoint() + self.scheduler_socket.connect(scheduler_endpoint) + logger.info( + f"Scheduler client connected to backend scheduler at {scheduler_endpoint}" + ) + # Worker will be lazily started on the first forward call to ensure a running loop exists + + async def forward(self, batch: Any) -> Any: + """Enqueue a request to the backend Scheduler and await the reply.""" + if self._closing: + raise RuntimeError( + "SchedulerClient is closing; cannot forward new requests" + ) + + await self._ensure_worker_started() + + loop = asyncio.get_running_loop() + future = loop.create_future() + await self._request_queue.put((batch, future)) + return await future + + async def _ensure_worker_started(self): + # Start the worker only once and only when an event loop is running + if self._worker_task is None or self._worker_task.done(): + self._worker_task = asyncio.create_task(self._worker_loop()) + + async def _worker_loop(self): + while True: + try: + item = await self._request_queue.get() + try: + batch, future = item + except Exception: + # Malformed queue item; skip + self._request_queue.task_done() + continue + + try: + await self.scheduler_socket.send_pyobj(batch) + response = await self.scheduler_socket.recv_pyobj() + if not future.done(): + future.set_result(response) + except Exception as e: + if not future.done(): + future.set_exception(e) + finally: + self._request_queue.task_done() + except asyncio.CancelledError: + # Drain remaining items with cancellation error to avoid hanging waiters + while True: + try: + batch, future = self._request_queue.get_nowait() + except asyncio.QueueEmpty: + break + try: + if not future.done(): + future.set_exception(asyncio.CancelledError()) + finally: + self._request_queue.task_done() + raise + + def close(self): + self._closing = True + # Cancel worker if running + if self._worker_task is not None: + self._worker_task.cancel() + try: + self.scheduler_socket.close() + finally: + try: + self.context.term() + except Exception: + pass + + +# Singleton instance +scheduler_client = SchedulerClient() + + +async def run_zeromq_broker(server_args: ServerArgs): + """ + This function runs as a background task in the FastAPI process. + It listens for TCP requests from offline clients (e.g., DiffGenerator). + """ + ctx = zmq.asyncio.Context() + # This is the REP socket that listens for requests from DiffGenerator + socket = ctx.socket(zmq.REP) + broker_endpoint = f"tcp://*:{server_args.broker_port}" + socket.bind(broker_endpoint) + logger.info(f"ZMQ Broker is listening for offline jobs on {broker_endpoint}") + + while True: + try: + # 1. Receive a request from an offline client + request_batch = await socket.recv_pyobj() + logger.info("Broker received an offline job from a client.") + + # 2. Forward the request to the main Scheduler via the shared client + response_batch = await scheduler_client.forward(request_batch) + + # 3. Send the Scheduler's reply back to the offline client + await socket.send_pyobj(response_batch) + + except Exception as e: + logger.error(f"Error in ZMQ Broker: {e}", exc_info=True) + # A reply must be sent to prevent the client from hanging + await socket.send_pyobj({"status": "error", "message": str(e)}) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/server_args.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/server_args.py new file mode 100644 index 00000000..4abc5cbf --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/server_args.py @@ -0,0 +1,1053 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Inspired by SGLang: https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/server_args.py +"""The arguments of sglang-diffusion Inference.""" +import argparse +import dataclasses +import inspect +import json +import random +import sys +import tempfile +from contextlib import contextmanager +from dataclasses import field +from enum import Enum +from typing import Any, Optional + +from sglang.multimodal_gen.configs.pipeline_configs import FluxPipelineConfig +from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig, STA_Mode +from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import ( + QwenImageEditPipelineConfig, + QwenImagePipelineConfig, +) +from sglang.multimodal_gen.runtime.platforms import ( + AttentionBackendEnum, + current_platform, +) +from sglang.multimodal_gen.runtime.utils.common import ( + is_port_available, + is_valid_ipv6_address, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import ( + configure_logger, + init_logger, +) +from sglang.multimodal_gen.utils import FlexibleArgumentParser, StoreBoolean + +logger = init_logger(__name__) + +ZMQ_TCP_PORT_DELTA = 233 + + +def _is_torch_tensor(obj: Any) -> tuple[bool, Any]: + """Return (is_tensor, torch_module_or_None) without importing torch at module import time.""" + try: + import torch # type: ignore + + return isinstance(obj, torch.Tensor), torch + except Exception: + return False, None + + +def _sanitize_for_logging(obj: Any, key_hint: str | None = None) -> Any: + """Recursively convert objects to JSON-serializable forms for concise logging. + + Rules: + - Drop any field/dict key named 'param_names_mapping'. + - Render Enums using their value. + - Render torch.Tensor as a compact summary; if key name is 'scaling_factor', include stats. + - Dataclasses are expanded to dicts and sanitized recursively. + - Callables/functions are rendered as their qualified name. + - Fallback to str(...) for unknown types. + """ + # Handle simple types quickly + if obj is None or isinstance(obj, (str, int, float, bool)): + return obj + + # Enum -> value for readability + if isinstance(obj, Enum): + return obj.value + + # torch.Tensor handling (lazy import) + is_tensor, torch_mod = _is_torch_tensor(obj) + if is_tensor: + try: + ten = obj.detach().cpu() + if key_hint == "scaling_factor": + # Provide a compact, single-line summary for scaling_factor + stats = { + "shape": list(ten.shape), + "dtype": str(ten.dtype), + } + # Stats might fail for some dtypes; guard individually + try: + stats["min"] = float(ten.min().item()) + except Exception: + pass + try: + stats["max"] = float(ten.max().item()) + except Exception: + pass + try: + stats["mean"] = float(ten.float().mean().item()) + except Exception: + pass + return {"tensor": "scaling_factor", **stats} + # Generic tensor summary + return {"tensor": True, "shape": list(ten.shape), "dtype": str(ten.dtype)} + except Exception: + return "" + + # Dataclasses -> dict + if dataclasses.is_dataclass(obj): + result: dict[str, Any] = {} + for f in dataclasses.fields(obj): + if not f.repr: + continue + name = f.name + if "names_mapping" in name: # drop noisy mappings + continue + try: + value = getattr(obj, name) + except Exception: + continue + result[name] = _sanitize_for_logging(value, key_hint=name) + return result + + # Dicts -> sanitize keys/values; drop 'param_names_mapping' + if isinstance(obj, dict): + result_dict: dict[str, Any] = {} + for k, v in obj.items(): + try: + key_str = str(k) + except Exception: + key_str = "" + if key_str == "param_names_mapping": + continue + result_dict[key_str] = _sanitize_for_logging(v, key_hint=key_str) + return result_dict + + # Sequences/Sets -> list + if isinstance(obj, (list, tuple, set)): + return [_sanitize_for_logging(x) for x in obj] + + # Functions / Callables -> qualified name + try: + if inspect.isroutine(obj) or inspect.isclass(obj): + module = getattr(obj, "__module__", "") + qn = getattr(obj, "__qualname__", getattr(obj, "__name__", "")) + return f"{module}.{qn}" if module else qn + except Exception: + pass + + # Fallback: string representation + try: + return str(obj) + except Exception: + return "" + + +class ExecutionMode(str, Enum): + """ + Enumeration for different pipeline modes. + + Inherits from str to allow string comparison for backward compatibility. + """ + + INFERENCE = "inference" + + @classmethod + def from_string(cls, value: str) -> "ExecutionMode": + """Convert string to ExecutionMode enum.""" + try: + return cls(value.lower()) + except ValueError: + raise ValueError( + f"Invalid mode: {value}. Must be one of: {', '.join([m.value for m in cls])}" + ) from None + + @classmethod + def choices(cls) -> list[str]: + """Get all available choices as strings for argparse.""" + return [mode.value for mode in cls] + + +class WorkloadType(str, Enum): + """ + Enumeration for different workload types. + + Inherits from str to allow string comparison for backward compatibility. + """ + + I2V = "i2v" # Image to Video + T2V = "t2v" # Text to Video + T2I = "t2i" # Text to Image + I2I = "i2i" # Image to Image + + @classmethod + def from_string(cls, value: str) -> "WorkloadType": + """Convert string to WorkloadType enum.""" + try: + return cls(value.lower()) + except ValueError: + raise ValueError( + f"Invalid workload type: {value}. Must be one of: {', '.join([m.value for m in cls])}" + ) from None + + @classmethod + def choices(cls) -> list[str]: + """Get all available choices as strings for argparse.""" + return [workload.value for workload in cls] + + +# args for sgl_diffusion framework +@dataclasses.dataclass +class ServerArgs: + # Model and path configuration (for convenience) + model_path: str + + # Attention + attention_backend: str = None + + # Running mode + mode: ExecutionMode = ExecutionMode.INFERENCE + + # Workload type + workload_type: WorkloadType = WorkloadType.T2V + + # Cache strategy + cache_strategy: str = "none" + + # Distributed executor backend + distributed_executor_backend: str = "mp" + nccl_port: Optional[int] = None + + # HuggingFace specific parameters + trust_remote_code: bool = False + revision: str | None = None + + # Parallelism + num_gpus: int = 1 + tp_size: int = -1 + sp_degree: int = -1 + # sequence parallelism + ulysses_degree: Optional[int] = None + ring_degree: Optional[int] = None + # data parallelism + # number of data parallelism groups + dp_size: int = 1 + # number of gpu in a dp group + dp_degree: int = 1 + # cfg parallel + enable_cfg_parallel: bool = False + + hsdp_replicate_dim: int = 1 + hsdp_shard_dim: int = -1 + dist_timeout: int | None = None # timeout for torch.distributed + + pipeline_config: PipelineConfig = field(default_factory=PipelineConfig, repr=False) + + # LoRA parameters + # (Wenxuan) prefer to keep it here instead of in pipeline config to not make it complicated. + lora_path: str | None = None + lora_nickname: str = "default" # for swapping adapters in the pipeline + # can restrict layers to adapt, e.g. ["q_proj"] + # Will adapt only q, k, v, o by default. + lora_target_modules: list[str] | None = None + + output_type: str = "pil" + + # CPU offload parameters + dit_cpu_offload: bool = True + use_fsdp_inference: bool = False + text_encoder_cpu_offload: bool = True + image_encoder_cpu_offload: bool = True + vae_cpu_offload: bool = True + pin_cpu_memory: bool = True + + # STA (Sliding Tile Attention) parameters + mask_strategy_file_path: str | None = None + STA_mode: STA_Mode = STA_Mode.STA_INFERENCE + skip_time_steps: int = 15 + + # Compilation + enable_torch_compile: bool = False + + disable_autocast: bool = False + + # VSA parameters + VSA_sparsity: float = 0.0 # inference/validation sparsity + + # V-MoBA parameters + moba_config_path: str | None = None + moba_config: dict[str, Any] = field(default_factory=dict) + + # Master port for distributed inference + # TODO: do not hard code + master_port: int | None = None + + # http server endpoint config, would be ignored in local mode + host: str | None = None + port: int | None = None + + scheduler_port: int = 5555 + + # Stage verification + enable_stage_verification: bool = True + + # Prompt text file for batch processing + prompt_file_path: str | None = None + + # model paths for correct deallocation + model_paths: dict[str, str] = field(default_factory=dict) + model_loaded: dict[str, bool] = field( + default_factory=lambda: { + "transformer": True, + "vae": True, + } + ) + override_transformer_cls_name: str | None = None + + # # DMD parameters + # dmd_denoising_steps: List[int] | None = field(default=None) + + # MoE parameters used by Wan2.2 + boundary_ratio: float | None = None + + # Logging + log_level: str = "info" + + @property + def broker_port(self) -> int: + return self.port + 1 + + @property + def is_local_mode(self) -> bool: + """ + If no server is running when a generation task begins, 'local_mode' will be enabled: a dedicated server will be launched + """ + return self.host is None or self.port is None + + def __post_init__(self): + # Add randomization to avoid race condition when multiple servers start simultaneously + if self.attention_backend in ["fa3", "fa4"]: + self.attention_backend = "fa" + + initial_scheduler_port = self.scheduler_port + random.randint(0, 100) + self.scheduler_port = self.settle_port(initial_scheduler_port) + # TODO: remove hard code + initial_master_port = (self.master_port or 30005) + random.randint(0, 100) + self.master_port = self.settle_port(initial_master_port, 37) + if self.moba_config_path: + try: + with open(self.moba_config_path) as f: + self.moba_config = json.load(f) + logger.info("Loaded V-MoBA config from %s", self.moba_config_path) + except (FileNotFoundError, json.JSONDecodeError) as e: + logger.error( + "Failed to load V-MoBA config from %s: %s", self.moba_config_path, e + ) + raise + self.check_server_args() + + configure_logger(server_args=self) + + # log clean server_args + try: + safe_args = _sanitize_for_logging(self, key_hint="server_args") + logger.info("server_args: %s", json.dumps(safe_args, ensure_ascii=False)) + except Exception: + # Fallback to default repr if sanitization fails + logger.info(f"server_args: {self}") + + @staticmethod + def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: + # Model and path configuration + parser.add_argument( + "--model-path", + type=str, + help="The path of the model weights. This can be a local folder or a Hugging Face repo ID.", + ) + parser.add_argument( + "--model-dir", + type=str, + help="Directory containing StepVideo model", + ) + + # attention + parser.add_argument( + "--attention-backend", + type=str, + default=None, + choices=[e.name.lower() for e in AttentionBackendEnum] + ["fa3", "fa4"], + help="The attention backend to use. If not specified, the backend is automatically selected based on hardware and installed packages.", + ) + + # Running mode + parser.add_argument( + "--mode", + type=str, + choices=ExecutionMode.choices(), + default=ServerArgs.mode.value, + help="The mode to run SGLang-diffusion", + ) + + # Workload type + parser.add_argument( + "--workload-type", + type=str, + choices=WorkloadType.choices(), + default=ServerArgs.workload_type.value, + help="The workload type", + ) + + # distributed_executor_backend + parser.add_argument( + "--distributed-executor-backend", + type=str, + choices=["mp"], + default=ServerArgs.distributed_executor_backend, + help="The distributed executor backend to use", + ) + + # HuggingFace specific parameters + parser.add_argument( + "--trust-remote-code", + action=StoreBoolean, + default=ServerArgs.trust_remote_code, + help="Trust remote code when loading HuggingFace models", + ) + parser.add_argument( + "--revision", + type=str, + default=ServerArgs.revision, + help="The specific model version to use (can be a branch name, tag name, or commit id)", + ) + + # Parallelism + parser.add_argument( + "--num-gpus", + type=int, + default=ServerArgs.num_gpus, + help="The number of GPUs to use.", + ) + parser.add_argument( + "--tp-size", + type=int, + default=ServerArgs.tp_size, + help="The tensor parallelism size.", + ) + parser.add_argument( + "--sp-degree", + type=int, + default=ServerArgs.sp_degree, + help="The sequence parallelism size.", + ) + parser.add_argument( + "--ulysses-degree", + type=int, + default=ServerArgs.ulysses_degree, + help="Ulysses sequence parallel degree. Used in attention layer.", + ) + parser.add_argument( + "--ring-degree", + type=int, + default=ServerArgs.ring_degree, + help="Ring sequence parallel degree. Used in attention layer.", + ) + parser.add_argument( + "--enable-cfg-parallel", + action="store_true", + default=ServerArgs.enable_cfg_parallel, + help="Enable cfg parallel.", + ) + parser.add_argument( + "--data-parallel-size", + "--dp-size", + "--dp", + type=int, + default=ServerArgs.dp_size, + help="The data parallelism size.", + ) + + parser.add_argument( + "--hsdp-replicate-dim", + type=int, + default=ServerArgs.hsdp_replicate_dim, + help="The data parallelism size.", + ) + parser.add_argument( + "--hsdp-shard-dim", + type=int, + default=ServerArgs.hsdp_shard_dim, + help="The data parallelism shards.", + ) + parser.add_argument( + "--dist-timeout", + type=int, + default=ServerArgs.dist_timeout, + help="Set timeout for torch.distributed initialization.", + ) + + # Output type + parser.add_argument( + "--output-type", + type=str, + default=ServerArgs.output_type, + choices=["pil"], + help="Output type for the generated video", + ) + + # Prompt text file for batch processing + parser.add_argument( + "--prompt-file-path", + type=str, + default=ServerArgs.prompt_file_path, + help="Path to a text file containing prompts (one per line) for batch processing", + ) + + # STA (Sliding Tile Attention) parameters + parser.add_argument( + "--STA-mode", + type=str, + default=ServerArgs.STA_mode.value, + choices=[mode.value for mode in STA_Mode], + help="STA mode contains STA_inference, STA_searching, STA_tuning, STA_tuning_cfg, None", + ) + parser.add_argument( + "--skip-time-steps", + type=int, + default=ServerArgs.skip_time_steps, + help="Number of time steps to warmup (full attention) for STA", + ) + parser.add_argument( + "--mask-strategy-file-path", + type=str, + help="Path to mask strategy JSON file for STA", + ) + parser.add_argument( + "--enable-torch-compile", + action=StoreBoolean, + default=ServerArgs.enable_torch_compile, + help="Use torch.compile to speed up DiT inference." + + "However, will likely cause precision drifts. See (https://github.com/pytorch/pytorch/issues/145213)", + ) + + parser.add_argument( + "--dit-cpu-offload", + action=StoreBoolean, + help="Use CPU offload for DiT inference. Enable if run out of memory with FSDP.", + ) + parser.add_argument( + "--use-fsdp-inference", + action=StoreBoolean, + help="Use FSDP for inference by sharding the model weights. Latency is very low due to prefetch--enable if run out of memory.", + ) + parser.add_argument( + "--text-encoder-cpu-offload", + action=StoreBoolean, + help="Use CPU offload for text encoder. Enable if run out of memory.", + ) + parser.add_argument( + "--image-encoder-cpu-offload", + action=StoreBoolean, + help="Use CPU offload for image encoder. Enable if run out of memory.", + ) + parser.add_argument( + "--vae-cpu-offload", + action=StoreBoolean, + help="Use CPU offload for VAE. Enable if run out of memory.", + ) + parser.add_argument( + "--pin-cpu-memory", + action=StoreBoolean, + help='Pin memory for CPU offload. Only added as a temp workaround if it throws "CUDA error: invalid argument". ' + "Should be enabled in almost all cases", + ) + parser.add_argument( + "--disable-autocast", + action=StoreBoolean, + help="Disable autocast for denoising loop and vae decoding in pipeline sampling", + ) + + # VSA parameters + parser.add_argument( + "--VSA-sparsity", + type=float, + default=ServerArgs.VSA_sparsity, + help="Validation sparsity for VSA", + ) + + # Master port for distributed inference + parser.add_argument( + "--master-port", + type=int, + default=ServerArgs.master_port, + help="Master port for distributed inference. If not set, a random free port will be used.", + ) + parser.add_argument( + "--scheduler-port", + type=int, + default=ServerArgs.scheduler_port, + help="Port for the scheduler server.", + ) + parser.add_argument( + "--host", + type=str, + default=ServerArgs.host, + help="Host for the HTTP API server.", + ) + parser.add_argument( + "--port", + type=int, + default=ServerArgs.port, + help="Port for the HTTP API server.", + ) + + # Stage verification + parser.add_argument( + "--enable-stage-verification", + action=StoreBoolean, + default=ServerArgs.enable_stage_verification, + help="Enable input/output verification for pipeline stages", + ) + parser.add_argument( + "--override-transformer-cls-name", + type=str, + default=ServerArgs.override_transformer_cls_name, + help="Override transformer cls name", + ) + # LoRA + parser.add_argument( + "--lora-path", + type=str, + default=ServerArgs.lora_path, + help="The path to the LoRA adapter weights (can be local file path or HF hub id) to launch with", + ) + parser.add_argument( + "--lora-nickname", + type=str, + default=ServerArgs.lora_nickname, + help="The nickname for the LoRA adapter to launch with", + ) + # Add pipeline configuration arguments + PipelineConfig.add_cli_args(parser) + + # Logging + parser.add_argument( + "--log-level", + type=str, + default=ServerArgs.log_level, + help="The logging level of all loggers.", + ) + return parser + + def url(self): + if is_valid_ipv6_address(self.host): + return f"http://[{self.host}]:{self.port}" + else: + return f"http://{self.host}:{self.port}" + + def scheduler_endpoint(self): + """ + Internal endpoint for scheduler + + """ + scheduler_host = self.host or "localhost" + return f"tcp://{scheduler_host}:{self.scheduler_port}" + + def settle_port( + self, port: int, port_inc: int = 42, max_attempts: int = 100 + ) -> int: + """ + Find an available port with retry logic. + + Args: + port: Initial port to check + port_inc: Port increment for each attempt + max_attempts: Maximum number of attempts to find an available port + + Returns: + An available port number + + Raises: + RuntimeError: If no available port is found after max_attempts + """ + attempts = 0 + original_port = port + + while attempts < max_attempts: + if is_port_available(port): + if attempts > 0: + logger.info( + f"Port {original_port} was unavailable, using port {port} instead" + ) + return port + + attempts += 1 + if port < 60000: + port += port_inc + else: + # Wrap around with randomization to avoid collision + port = 5000 + random.randint(0, 1000) + + raise RuntimeError( + f"Failed to find available port after {max_attempts} attempts " + f"(started from port {original_port})" + ) + + def post_init_serve(self): + """ + Post init when in serve mode + """ + if self.host is None: + self.host = "localhost" + if self.port is None: + self.port = 3000 + self.port = self.settle_port(self.port) + + @classmethod + def from_cli_args( + cls, args: argparse.Namespace, unknown_args: list[str] | None = None + ) -> "ServerArgs": + if unknown_args is None: + unknown_args = [] + provided_args = cls.get_provided_args(args, unknown_args) + + # Handle config file + config_file = provided_args.get("config") + if config_file: + config_args = cls.load_config_file(config_file) + # Provided args override config file args + provided_args = {**config_args, **provided_args} + + # Handle special cases + # if "tp_size" in provided_args: + # provided_args["tp"] = provided_args.pop("tp_size") + + return cls.from_dict(provided_args) + + @classmethod + def from_dict(cls, kwargs: dict[str, Any]) -> "ServerArgs": + """Create a ServerArgs object from a dictionary.""" + attrs = [attr.name for attr in dataclasses.fields(cls)] + server_args_kwargs: dict[str, Any] = {} + + for attr in attrs: + if attr == "pipeline_config": + pipeline_config = PipelineConfig.from_kwargs(kwargs) + logger.debug(f"Using PipelineConfig: {type(pipeline_config)}") + server_args_kwargs["pipeline_config"] = pipeline_config + elif attr in kwargs: + server_args_kwargs[attr] = kwargs[attr] + + return cls(**server_args_kwargs) + + @staticmethod + def load_config_file(config_file: str) -> dict[str, Any]: + """Load a config file.""" + if config_file.endswith(".json"): + with open(config_file, "r") as f: + return json.load(f) + elif config_file.endswith((".yaml", ".yml")): + try: + import yaml + except ImportError: + raise ImportError( + "Please install PyYAML to use YAML config files. " + "`pip install pyyaml`" + ) + with open(config_file, "r") as f: + return yaml.safe_load(f) + else: + raise ValueError(f"Unsupported config file format: {config_file}") + + @classmethod + def from_kwargs(cls, **kwargs: Any) -> "ServerArgs": + # Convert mode string to enum if necessary + if "mode" in kwargs and isinstance(kwargs["mode"], str): + kwargs["mode"] = ExecutionMode.from_string(kwargs["mode"]) + + # Convert workload_type string to enum if necessary + if "workload_type" in kwargs and isinstance(kwargs["workload_type"], str): + kwargs["workload_type"] = WorkloadType.from_string(kwargs["workload_type"]) + + kwargs["pipeline_config"] = PipelineConfig.from_kwargs(kwargs) + return cls(**kwargs) + + @staticmethod + def get_provided_args( + args: argparse.Namespace, unknown_args: list[str] + ) -> dict[str, Any]: + """Get the arguments provided by the user.""" + provided_args = {} + # We need to check against the raw command-line arguments to see what was + # explicitly provided by the user, vs. what's a default value from argparse. + raw_argv = sys.argv + unknown_args + + # Create a set of argument names that were present on the command line. + # This handles both styles: '--arg=value' and '--arg value'. + provided_arg_names = set() + for arg in raw_argv: + if arg.startswith("--"): + # For '--arg=value', this gets 'arg'; for '--arg', this also gets 'arg'. + arg_name = arg.split("=", 1)[0].replace("-", "_").lstrip("_") + provided_arg_names.add(arg_name) + + # Populate provided_args if the argument from the namespace was on the command line. + for k, v in vars(args).items(): + if k in provided_arg_names: + provided_args[k] = v + + return provided_args + + def check_server_sp_args(self): + if self.sp_degree == -1: + # assume we leave all remaining gpus to sp + num_gpus_per_group = self.dp_size * self.tp_size + if self.enable_cfg_parallel: + num_gpus_per_group *= 2 + if self.num_gpus % num_gpus_per_group != 0: + raise ValueError(f"{self.num_gpus=} % {num_gpus_per_group} != 0") + self.sp_degree = self.num_gpus // num_gpus_per_group + + if ( + self.ulysses_degree is None + and self.ring_degree is None + and self.sp_degree != 1 + ): + self.ulysses_degree = self.sp_degree + logger.info( + f"Automatically set ulysses_degree=sp_degree={self.ulysses_degree} for best performance" + ) + + if self.ulysses_degree is None: + self.ulysses_degree = 1 + logger.info( + f"Ulysses degree not set, " f"using default value {self.ulysses_degree}" + ) + + if self.ring_degree is None: + self.ring_degree = 1 + logger.info( + f"Ring degree not set, " f"using default value {self.ring_degree}" + ) + + if self.ring_degree > 1: + if self.attention_backend != None and self.attention_backend != "fa": + raise ValueError( + "Ring Attention is only supported for flash attention backend for now" + ) + else: + self.attention_backend = "fa" + logger.info( + "Ring Attention is currently only supported for flash attention, attention_backend has been automatically set to flash attention" + ) + + if self.sp_degree == -1: + self.sp_degree = self.ring_degree * self.ulysses_degree + logger.info( + f"sequence_parallel_degree is not provided, using ring_degree * ulysses_degree = {self.sp_degree}" + ) + + if self.sp_degree != self.ring_degree * self.ulysses_degree: + raise ValueError( + f"sequence_parallel_degree is not equal to ring_degree * ulysses_degree, {self.sp_degree} != {self.ring_degree} * {self.ulysses_degree}" + ) + + def check_server_dp_args(self): + assert self.num_gpus % self.dp_size == 0, f"{self.num_gpus=}, {self.dp_size=}" + assert self.dp_size >= 1, "--dp-size must be natural number" + # NOTE: disable temporarily + # self.dp_degree = self.num_gpus // self.dp_size + logger.info(f"Setting dp_degree to: {self.dp_degree}") + if self.dp_size > 1: + raise ValueError("DP is not yet supported") + + def check_server_args(self) -> None: + """Validate inference arguments for consistency""" + if current_platform.is_mps(): + self.use_fsdp_inference = False + + # autocast + is_flux = ( + isinstance(self.pipeline_config, FluxPipelineConfig) + or isinstance(self.pipeline_config, QwenImagePipelineConfig) + or isinstance(self.pipeline_config, QwenImageEditPipelineConfig) + ) + if is_flux: + self.disable_autocast = True + + # Validate mode consistency + assert isinstance( + self.mode, ExecutionMode + ), f"Mode must be an ExecutionMode enum, got {type(self.mode)}" + assert ( + self.mode in ExecutionMode.choices() + ), f"Invalid execution mode: {self.mode}" + + # Validate workload type + assert isinstance( + self.workload_type, WorkloadType + ), f"Workload type must be a WorkloadType enum, got {type(self.workload_type)}" + assert ( + self.workload_type in WorkloadType.choices() + ), f"Invalid workload type: {self.workload_type}" + + if self.tp_size == -1: + self.tp_size = 1 + + if self.hsdp_shard_dim == -1: + self.hsdp_shard_dim = self.num_gpus + + assert ( + self.sp_degree <= self.num_gpus and self.num_gpus % self.sp_degree == 0 + ), "num_gpus must >= and be divisible by sp_size" + assert ( + self.hsdp_replicate_dim <= self.num_gpus + and self.num_gpus % self.hsdp_replicate_dim == 0 + ), "num_gpus must >= and be divisible by hsdp_replicate_dim" + assert ( + self.hsdp_shard_dim <= self.num_gpus + and self.num_gpus % self.hsdp_shard_dim == 0 + ), "num_gpus must >= and be divisible by hsdp_shard_dim" + + if self.num_gpus < max(self.tp_size, self.sp_degree): + self.num_gpus = max(self.tp_size, self.sp_degree) + + if self.pipeline_config is None: + raise ValueError("pipeline_config is not set in ServerArgs") + + self.pipeline_config.check_pipeline_config() + + # parallelism + self.check_server_dp_args() + # allocate all remaining gpus for sp-size + self.check_server_sp_args() + + if self.enable_cfg_parallel: + if self.num_gpus == 1: + raise ValueError( + "CFG Parallelism is enabled via `--enable-cfg-parallel`, while -num-gpus==1" + ) + + +@dataclasses.dataclass +class PortArgs: + # The ipc filename for scheduler (rank 0) to receive inputs from tokenizer (zmq) + scheduler_input_ipc_name: str + + # The port for nccl initialization (torch.dist) + nccl_port: int + + # The ipc filename for rpc call between Engine and Scheduler + rpc_ipc_name: str + + # The ipc filename for Scheduler to send metrics + metrics_ipc_name: str + + # Master port for distributed inference + master_port: int | None = None + + @staticmethod + def from_server_args( + server_args: ServerArgs, dp_rank: Optional[int] = None + ) -> "PortArgs": + if server_args.nccl_port is None: + nccl_port = server_args.scheduler_port + random.randint(100, 1000) + while True: + if is_port_available(nccl_port): + break + if nccl_port < 60000: + nccl_port += 42 + else: + nccl_port -= 43 + else: + nccl_port = server_args.nccl_port + + # Normal case, use IPC within a single node + return PortArgs( + scheduler_input_ipc_name=f"ipc://{tempfile.NamedTemporaryFile(delete=False).name}", + nccl_port=nccl_port, + rpc_ipc_name=f"ipc://{tempfile.NamedTemporaryFile(delete=False).name}", + metrics_ipc_name=f"ipc://{tempfile.NamedTemporaryFile(delete=False).name}", + master_port=server_args.master_port, + ) + + +# TODO: not sure what _current_server_args is for, using a _global_server_args instead +_current_server_args = None +_global_server_args = None + + +def prepare_server_args(argv: list[str]) -> ServerArgs: + """ + Prepare the inference arguments from the command line arguments. + + Args: + argv: The command line arguments. Typically, it should be `sys.argv[1:]` + to ensure compatibility with `parse_args` when no arguments are passed. + + Returns: + The inference arguments. + """ + parser = FlexibleArgumentParser() + ServerArgs.add_cli_args(parser) + raw_args = parser.parse_args(argv) + server_args = ServerArgs.from_cli_args(raw_args) + global _current_server_args + _current_server_args = server_args + return server_args + + +@contextmanager +def set_current_server_args(server_args: ServerArgs): + """ + Temporarily set the current sgl_diffusion config. + Used during model initialization. + We save the current sgl_diffusion config in a global variable, + so that all modules can access it, e.g. custom ops + can access the sgl_diffusion config to determine how to dispatch. + """ + global _current_server_args + old_server_args = _current_server_args + try: + _current_server_args = server_args + yield + finally: + _current_server_args = old_server_args + + +def set_global_server_args(server_args: ServerArgs): + """ + Set the global sgl_diffusion config for each process + """ + global _global_server_args + _global_server_args = server_args + + +def get_current_server_args() -> ServerArgs: + if _current_server_args is None: + # in ci, usually when we test custom ops/modules directly, + # we don't set the sgl_diffusion config. In that case, we set a default + # config. + # TODO(will): may need to handle this for CI. + raise ValueError("Current sgl_diffusion args is not set.") + return _current_server_args + + +def get_global_server_args() -> ServerArgs: + if _global_server_args is None: + # in ci, usually when we test custom ops/modules directly, + # we don't set the sgl_diffusion config. In that case, we set a default + # config. + # TODO(will): may need to handle this for CI. + raise ValueError("Global sgl_diffusion args is not set.") + return _global_server_args + + +def parse_int_list(value: str) -> list[int]: + if not value: + return [] + return [int(x.strip()) for x in value.split(",")] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/sync_scheduler_client.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/sync_scheduler_client.py new file mode 100644 index 00000000..3d32576f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/sync_scheduler_client.py @@ -0,0 +1,92 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo +from typing import Any + +import zmq + +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class SyncSchedulerClient: + """ + A synchronous, singleton client for communicating with the Scheduler service. + Designed for use in synchronous environments like the DiffGenerator or standalone scripts. + """ + + _instance = None + + def __new__(cls, *args, **kwargs): + if not cls._instance: + cls._instance = super(SyncSchedulerClient, cls).__new__(cls) + return cls._instance + + def initialize(self, server_args: ServerArgs): + if hasattr(self, "context") and not self.context.closed: + logger.warning( + "SyncSchedulerClient is already initialized. Re-initializing." + ) + self.close() + + self.server_args = server_args + self.context = zmq.Context() # Standard synchronous context + self.scheduler_socket = self.context.socket(zmq.REQ) + + # Set socket options for the main communication socket + self.scheduler_socket.setsockopt(zmq.LINGER, 0) + self.scheduler_socket.setsockopt( + zmq.RCVTIMEO, 6000000 + ) # 10 minute timeout for generation + + scheduler_endpoint = self.server_args.scheduler_endpoint() + self.scheduler_socket.connect(scheduler_endpoint) + logger.debug( + f"SyncSchedulerClient connected to backend scheduler at {scheduler_endpoint}" + ) + + def forward(self, batch: Any) -> Any: + """Sends a batch or request to the scheduler and waits for the response.""" + try: + self.scheduler_socket.send_pyobj(batch) + output_batch = self.scheduler_socket.recv_pyobj() + return output_batch + except zmq.error.Again: + logger.error("Timeout waiting for response from scheduler.") + raise TimeoutError("Scheduler did not respond in time.") + + def ping(self) -> bool: + """ + Checks if the scheduler server is alive using a temporary socket. + This avoids interfering with the state of the main REQ/REP socket. + """ + if not hasattr(self, "context") or self.context.closed: + logger.error("Cannot ping: client is not initialized.") + return False + + ping_socket = self.context.socket(zmq.REQ) + ping_socket.setsockopt(zmq.LINGER, 0) + ping_socket.setsockopt(zmq.RCVTIMEO, 2000) # 2-second timeout for pings + + endpoint = self.server_args.scheduler_endpoint() + + try: + ping_socket.connect(endpoint) + ping_socket.send_pyobj({"method": "ping"}) + ping_socket.recv_pyobj() + return True + except zmq.error.Again: + return False + finally: + ping_socket.close() + + def close(self): + """Closes the socket and terminates the context.""" + if hasattr(self, "scheduler_socket"): + self.scheduler_socket.close() + if hasattr(self, "context"): + self.context.term() + + +# Singleton instance for easy access +sync_scheduler_client = SyncSchedulerClient() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/utils/common.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/utils/common.py new file mode 100644 index 00000000..6907756e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/utils/common.py @@ -0,0 +1,340 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +import importlib +import ipaddress +import logging +import os +import platform +import signal +import socket +import sys +import threading +from functools import lru_cache + +import psutil +import torch +import zmq + +# use the native logger to avoid circular import +logger = logging.getLogger(__name__) + + +def kill_process_tree(parent_pid, include_parent: bool = True, skip_pid: int = None): + """Kill the process and all its child processes.""" + # Remove sigchld handler to avoid spammy logs. + if threading.current_thread() is threading.main_thread(): + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + + if parent_pid is None: + parent_pid = os.getpid() + include_parent = False + + try: + itself = psutil.Process(parent_pid) + except psutil.NoSuchProcess: + return + + children = itself.children(recursive=True) + for child in children: + if child.pid == skip_pid: + continue + try: + child.kill() + except psutil.NoSuchProcess: + pass + + if include_parent: + try: + if parent_pid == os.getpid(): + itself.kill() + sys.exit(0) + + itself.kill() + + # Sometime processes cannot be killed with SIGKILL (e.g, PID=1 launched by kubernetes), + # so we send an additional signal to kill them. + itself.send_signal(signal.SIGQUIT) + except psutil.NoSuchProcess: + pass + + +def add_prefix(name: str, prefix: str) -> str: + """Add a weight path prefix to a module name. + + Args: + name: base module name. + prefix: weight prefix str to added to the front of `name` concatenated with `.`. + + Returns: + The string `prefix.name` if prefix is non-empty, otherwise just `name`. + """ + return name if not prefix else f"{prefix}.{name}" + + +def is_valid_ipv6_address(address: str) -> bool: + try: + ipaddress.IPv6Address(address) + return True + except ValueError: + return False + + +def configure_ipv6(dist_init_addr): + addr = dist_init_addr + end = addr.find("]") + if end == -1: + raise ValueError("invalid IPv6 address format: missing ']'") + + host = addr[: end + 1] + + # this only validates the address without brackets: we still need the below checks. + # if it's invalid, immediately raise an error so we know it's not formatting issues. + if not is_valid_ipv6_address(host[1:end]): + raise ValueError(f"invalid IPv6 address: {host}") + + port_str = None + if len(addr) > end + 1: + if addr[end + 1] == ":": + port_str = addr[end + 2 :] + else: + raise ValueError("received IPv6 address format: expected ':' after ']'") + + if not port_str: + raise ValueError( + "a port must be specified in IPv6 address (format: [ipv6]:port)" + ) + + try: + port = int(port_str) + except ValueError: + raise ValueError(f"invalid port in IPv6 address: '{port_str}'") + return port, host + + +def is_port_available(port): + """Return whether a port is available.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(("", port)) + s.listen(1) + return True + except socket.error: + return False + except OverflowError: + return False + + +def get_zmq_socket( + context: zmq.Context, + socket_type: zmq.SocketType, + endpoint: str, + bind: bool, + max_bind_retries: int = 10, +) -> tuple[zmq.Socket, str]: + """ + Create and configure a ZMQ socket. + + Args: + context: ZMQ context + socket_type: Type of ZMQ socket + endpoint: Endpoint string (e.g., "tcp://localhost:5555") + bind: Whether to bind (True) or connect (False) + max_bind_retries: Maximum number of retries if bind fails due to address already in use + + Returns: + A tuple of (socket, actual_endpoint). The actual_endpoint may differ from the + requested endpoint if bind retry was needed. + """ + mem = psutil.virtual_memory() + total_mem = mem.total / 1024**3 + available_mem = mem.available / 1024**3 + if total_mem > 32 and available_mem > 16: + buf_size = int(0.5 * 1024**3) + else: + buf_size = -1 + + socket = context.socket(socket_type) + if endpoint.find("[") != -1: + socket.setsockopt(zmq.IPV6, 1) + + def set_send_opt(): + socket.setsockopt(zmq.SNDHWM, 0) + socket.setsockopt(zmq.SNDBUF, buf_size) + + def set_recv_opt(): + socket.setsockopt(zmq.RCVHWM, 0) + socket.setsockopt(zmq.RCVBUF, buf_size) + + if socket_type == zmq.PUSH: + set_send_opt() + elif socket_type == zmq.PULL: + set_recv_opt() + elif socket_type == zmq.DEALER: + set_send_opt() + set_recv_opt() + elif socket_type == zmq.REQ: + set_send_opt() + set_recv_opt() + elif socket_type == zmq.REP: + set_send_opt() + set_recv_opt() + else: + raise ValueError(f"Unsupported socket type: {socket_type}") + + if bind: + # Parse port from endpoint for retry logic + import re + + port_match = re.search(r":(\d+)$", endpoint) + + if port_match and max_bind_retries > 1: + original_port = int(port_match.group(1)) + last_exception = None + + for attempt in range(max_bind_retries): + try: + current_endpoint = endpoint + if attempt > 0: + # Try next port (increment by 42 to match settle_port logic) + current_port = original_port + attempt * 42 + current_endpoint = re.sub( + r":(\d+)$", f":{current_port}", endpoint + ) + logger.info( + f"ZMQ bind failed for port {original_port + (attempt - 1) * 42}, " + f"retrying with port {current_port} (attempt {attempt + 1}/{max_bind_retries})" + ) + + socket.bind(current_endpoint) + + if attempt > 0: + logger.warning( + f"Successfully bound ZMQ socket to {current_endpoint} after {attempt + 1} attempts. " + f"Original port {original_port} was unavailable." + ) + + return socket, current_endpoint + + except zmq.ZMQError as e: + last_exception = e + if e.errno == zmq.EADDRINUSE and attempt < max_bind_retries - 1: + # Address already in use, try next port + continue + elif attempt == max_bind_retries - 1: + # Last attempt failed + logger.error( + f"Failed to bind ZMQ socket after {max_bind_retries} attempts. " + f"Original endpoint: {endpoint}, Last tried port: {original_port + attempt * 42}" + ) + raise + else: + # Different error, raise immediately + raise + + # Should not reach here, but just in case + if last_exception: + raise last_exception + else: + # No retry logic needed (either no port in endpoint or max_bind_retries == 1) + socket.bind(endpoint) + return socket, endpoint + else: + socket.connect(endpoint) + return socket, endpoint + + return socket, endpoint + + +# https://pytorch.org/docs/stable/notes/hip.html#checking-for-hip +@lru_cache(maxsize=1) +def is_hip() -> bool: + return torch.version.hip is not None + + +@lru_cache(maxsize=1) +def is_cuda(): + return torch.cuda.is_available() and torch.version.cuda + + +@lru_cache(maxsize=1) +def is_cuda_alike(): + return is_cuda() or is_hip() + + +@lru_cache(maxsize=1) +def is_blackwell(): + if not is_cuda(): + return False + return torch.cuda.get_device_capability()[0] == 10 + + +@lru_cache(maxsize=1) +def is_hpu() -> bool: + return hasattr(torch, "hpu") and torch.hpu.is_available() + + +@lru_cache(maxsize=1) +def is_xpu() -> bool: + return hasattr(torch, "xpu") and torch.xpu.is_available() + + +@lru_cache(maxsize=1) +def is_npu() -> bool: + return hasattr(torch, "npu") and torch.npu.is_available() + + +@lru_cache(maxsize=1) +def is_host_cpu_x86() -> bool: + machine = platform.machine().lower() + return ( + machine in ("x86_64", "amd64", "i386", "i686") + and hasattr(torch, "cpu") + and torch.cpu.is_available() + ) + + +@lru_cache(maxsize=1) +def is_cpu() -> bool: + return os.getenv("SGLANG_USE_CPU_ENGINE", "0") == "1" and is_host_cpu_x86() + + +# cuda + + +def set_cuda_arch(): + capability = torch.cuda.get_device_capability() + arch = f"{capability[0]}.{capability[1]}" + os.environ["TORCH_CUDA_ARCH_LIST"] = f"{arch}{'+PTX' if arch == '9.0' else ''}" + + +def is_flashinfer_available(): + """ + Check whether flashinfer is available. + As of Oct. 6, 2024, it is only available on NVIDIA GPUs. + """ + # if not get_bool_env_var("SGLANG_IS_FLASHINFER_AVAILABLE", default="true"): + # return False + return importlib.util.find_spec("flashinfer") is not None and is_cuda() + + +# env var managements + +_warned_bool_env_var_keys = set() + + +def get_bool_env_var(name: str, default: str = "false") -> bool: + value = os.getenv(name, default) + value = str(value).strip().lower() + + truthy_values = {"1", "true", "yes", "y", "t", "on"} + falsy_values = {"0", "false", "no", "n", "f", "off", ""} + + if (value not in truthy_values) and (value not in falsy_values): + if value not in _warned_bool_env_var_keys: + logger.warning( + f"get_bool_env_var({name}) see non-understandable value={value} and treat as false" + ) + _warned_bool_env_var_keys.add(value) + + return value in truthy_values diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/utils/distributed.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/utils/distributed.py new file mode 100644 index 00000000..c89a31dc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/utils/distributed.py @@ -0,0 +1,231 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +import pickle +from typing import Any, List, Optional + +import numpy as np +import torch +import torch.distributed as dist + + +def broadcast_pyobj( + data: List[Any], + rank: int, + dist_group: Optional[torch.distributed.ProcessGroup] = None, + src: int = 0, + force_cpu_device: bool = True, +): + """Broadcast inputs from src rank to all other ranks with torch.dist backend. + The `rank` here refer to the source rank on global process group (regardless + of dist_group argument). + """ + device = torch.device( + "cuda" if torch.cuda.is_available() and not force_cpu_device else "cpu" + ) + + if rank == src: + if data is None or len(data) == 0: + tensor_size = torch.tensor([0], dtype=torch.long, device=device) + dist.broadcast(tensor_size, src=src, group=dist_group) + else: + serialized_data = pickle.dumps(data) + size = len(serialized_data) + + tensor_data = torch.ByteTensor( + np.frombuffer(serialized_data, dtype=np.uint8) + ).to(device) + tensor_size = torch.tensor([size], dtype=torch.long, device=device) + + dist.broadcast(tensor_size, src=src, group=dist_group) + dist.broadcast(tensor_data, src=src, group=dist_group) + return data + else: + tensor_size = torch.tensor([0], dtype=torch.long, device=device) + dist.broadcast(tensor_size, src=src, group=dist_group) + size = tensor_size.item() + + if size == 0: + return [] + + tensor_data = torch.empty(size, dtype=torch.uint8, device=device) + dist.broadcast(tensor_data, src=src, group=dist_group) + + serialized_data = bytes(tensor_data.cpu().numpy()) + data = pickle.loads(serialized_data) + return data + + +def generate_masked_orthogonal_rank_groups( + world_size: int, parallel_size: list[int], mask: list[bool] +) -> list[list[int]]: + """Generate orthogonal parallel groups based on the parallel size and mask. + + Arguments: + world_size (int): world size + + parallel_size (List[int]): + The parallel size of each orthogonal parallel type. For example, if + tensor_parallel_size = 2, pipeline_model_parallel_group = 3, data_parallel_size = 4, + and the parallel mapping order is tp-pp-dp, then the parallel_size = [2, 3, 4]. + + mask (List[bool]): + The mask controls which parallel methods the generated groups represent. If mask[i] is + True, it means the generated group contains the i-th parallelism method. For example, + if parallel_size = [tp_size, pp_size, dp_size], and mask = [True, False , True], then + the generated group is the `tp-dp` group, if the mask = [False, True, False], then the + generated group is the `pp` group. + + Algorithm: + For orthogonal parallelism, such as tp/dp/pp/cp, the global_rank and + + If we want to get the `dp_group` (tp_size * pp_size groups of dp_size ranks each. + For example, if the gpu size is 8 and order is 'tp-pp-dp', size is '2-2-2', and the + dp_group here is [[0, 4], [1, 5], [2, 6], [3, 7]].) + The tp_rank and pp_rank will be combined to form the `dp_group_index`. + dp_group_index = tp_rank + pp_rank * tp_size (2) + + So, Given that tp_rank and pp_rank satisfy equation (2), and dp_rank in + range(0, dp_size), the ranks in dp_group[dp_group_index] satisfies the + equation (1). + + This function solve this math problem. + + For example, if the parallel_size = [tp_size, dp_size, pp_size] = [2, 3, 4], + and the mask = [False, True, False]. Then, + dp_group_index(0) = tp_rank(0) + pp_rank(0) * 2 + dp_group_index(1) = tp_rank(1) + pp_rank(0) * 2 + ... + dp_group_index(7) = tp_rank(1) + pp_rank(3) * 2 + + dp_group[0] = 0 + range(0, 3) * 2 + 0 = [0, 2, 4] + dp_group[1] = 1 + range(0, 3) * 2 + 0 = [1, 3, 5] + ... + dp_group[7] = 1 + range(0, 3) * 2 + 3 * 2 * 3 = [19, 21, 23] + """ + + def prefix_product(a: List[int], init=1) -> List[int]: + r = [init] + for v in a: + init = init * v + r.append(init) + return r + + def inner_product(a: List[int], b: List[int]) -> int: + return sum([x * y for x, y in zip(a, b)]) + + def decompose(index, shape, stride=None): + """ + This function solve the math problem below: + There is an equation: + index = sum(idx[i] * stride[i]) + And given the value of index, stride. + Return the idx. + This function will used to get the pp/dp/pp_rank + from group_index and rank_in_group. + """ + if stride is None: + stride = prefix_product(shape) + idx = [(index // d) % s for s, d in zip(shape, stride)] + # stride is a prefix_product result. And the value of stride[-1] + # is not used. + assert ( + sum([x * y for x, y in zip(idx, stride[:-1])]) == index + ), "idx {} with shape {} mismatch the return idx {}".format(index, shape, idx) + return idx + + masked_shape = [s for s, m in zip(parallel_size, mask) if m] + unmasked_shape = [s for s, m in zip(parallel_size, mask) if not m] + + global_stride = prefix_product(parallel_size) + masked_stride = [d for d, m in zip(global_stride, mask) if m] + unmasked_stride = [d for d, m in zip(global_stride, mask) if not m] + + group_size = prefix_product(masked_shape)[-1] + num_of_group = world_size // group_size + + ranks = [] + for group_index in range(num_of_group): + # get indices from unmaksed for group_index. + decomposed_group_idx = decompose(group_index, unmasked_shape) + rank = [] + for rank_in_group in range(group_size): + # get indices from masked for rank_in_group. + decomposed_rank_idx = decompose(rank_in_group, masked_shape) + rank.append( + inner_product(decomposed_rank_idx, masked_stride) + + inner_product(decomposed_group_idx, unmasked_stride) + ) + ranks.append(rank) + return ranks + + +class RankGenerator(object): + def __init__( + self, + tp: int, + sp: int, + pp: int, + cfg: int, + dp: int, + order: str, + rank_offset: int = 0, + ) -> None: + self.tp = tp + self.sp = sp + self.pp = pp + self.cfg = cfg + self.dp = dp + self.rank_offset = rank_offset + self.world_size = tp * sp * pp * cfg * dp + + self.name_to_size = { + "tp": self.tp, + "sp": self.sp, + "pp": self.pp, + "cfg": self.cfg, + "dp": self.dp, + } + order = order.lower() + + for name in self.name_to_size.keys(): + if name not in order and self.name_to_size[name] != 1: + raise RuntimeError( + f"The size of ({name}) is ({self.name_to_size[name]}), but you haven't specified the order ({self.order})." + ) + elif name not in order: + order = order + "-" + name + + self.order = order + self.ordered_size = [] + + for token in order.split("-"): + self.ordered_size.append(self.name_to_size[token]) + + def get_mask(self, order: str, token: str): + ordered_token = order.split("-") + token = token.split("-") + mask = [False] * len(ordered_token) + for t in token: + mask[ordered_token.index(t)] = True + return mask + + def get_ranks(self, token): + """Get rank group by input token. + + Arguments: + token (str): + Specify the ranks type that want to get. If we want + to obtain multiple parallel types, we can use a hyphen + '-' to separate them. For example, if we want to obtain + the TP_DP group, the token should be 'tp-dp'. + + """ + mask = self.get_mask(self.order, token) + ranks = generate_masked_orthogonal_rank_groups( + self.world_size, self.ordered_size, mask + ) + if self.rank_offset > 0: + for rank_group in ranks: + for i in range(len(rank_group)): + rank_group[i] += self.rank_offset + return ranks diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py new file mode 100644 index 00000000..a3c83b6f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py @@ -0,0 +1,406 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from SGLang: https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/hf_transformers_utils.py + +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities for Huggingface Transformers.""" + +import contextlib +import hashlib +import json +import os +import tempfile +from functools import reduce +from pathlib import Path +from typing import Any, Optional, cast + +import filelock +from diffusers.loaders.lora_base import ( + _best_guess_weight_name, # watch out for potetential removal from diffusers +) +from huggingface_hub import snapshot_download +from transformers import AutoConfig, PretrainedConfig +from transformers.models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES + +from sglang.multimodal_gen.runtime.utils.logging_utils import ( + init_logger, + suppress_other_loggers, +) + +logger = init_logger(__name__) +_CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = { + # ChatGLMConfig.model_type: ChatGLMConfig, + # DbrxConfig.model_type: DbrxConfig, + # ExaoneConfig.model_type: ExaoneConfig, + # Qwen2_5_VLConfig.model_type: Qwen2_5_VLConfig, +} + +for name, cls in _CONFIG_REGISTRY.items(): + with contextlib.suppress(ValueError): + AutoConfig.register(name, cls) + + +def download_from_hf(model_path: str): + if os.path.exists(model_path): + return model_path + + return snapshot_download(model_path, allow_patterns=["*.json", "*.bin", "*.model"]) + + +def get_hf_config( + component_model_path: str, + trust_remote_code: bool, + revision: str | None = None, + model_override_args: dict | None = None, + **kwargs, +) -> PretrainedConfig: + is_gguf = check_gguf_file(component_model_path) + if is_gguf: + raise NotImplementedError("GGUF models are not supported.") + + config = AutoConfig.from_pretrained( + component_model_path, + trust_remote_code=trust_remote_code, + revision=revision, + **kwargs, + ) + if config.model_type in _CONFIG_REGISTRY: + config_class = _CONFIG_REGISTRY[config.model_type] + config = config_class.from_pretrained(component_model_path, revision=revision) + # NOTE(HandH1998): Qwen2VL requires `_name_or_path` attribute in `config`. + config._name_or_path = component_model_path + if model_override_args: + config.update(model_override_args) + + # Special architecture mapping check for GGUF models + if is_gguf: + if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES: + raise RuntimeError(f"Can't get gguf config for {config.model_type}.") + model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type] + config.update({"architectures": [model_type]}) + + return config + + +def get_config( + model: str, + trust_remote_code: bool, + revision: Optional[str] = None, + model_override_args: Optional[dict] = None, + **kwargs, +): + try: + config = AutoConfig.from_pretrained( + model, trust_remote_code=trust_remote_code, revision=revision, **kwargs + ) + except ValueError as e: + raise e + + return config + + +def load_dict(file_path): + if not os.path.exists(file_path): + return {} + try: + # Load the config directly from the file + with open(file_path) as f: + config_dict: dict[str, Any] = json.load(f) + if "_diffusers_version" in config_dict: + config_dict.pop("_diffusers_version") + # TODO(will): apply any overrides from inference args + return config_dict + except Exception as e: + raise RuntimeError( + f"Failed to load diffusers config from {file_path}: {e}" + ) from e + + +def get_diffusers_component_config( + model_path: str, +) -> dict[str, Any]: + """Gets a configuration of a submodule for the given diffusers model. + + Args: + model_path: the path of the submodule + + Returns: + The loaded configuration. + """ + + # Check if the model path exists + if os.path.exists(model_path): + # tokenizer + config_names = ["generation_config.json"] + # By default, we load config.json, but scheduler_config.json for scheduler + if "scheduler" in model_path: + config_names.append("scheduler_config.json") + else: + config_names.append("config.json") + + config_file_paths = [ + os.path.join(model_path, config_name) for config_name in config_names + ] + + combined_config = reduce( + lambda acc, path: acc | load_dict(path), config_file_paths, {} + ) + + return combined_config + else: + raise RuntimeError(f"Diffusers config file not found at {model_path}") + + +# Models don't use the same configuration key for determining the maximum +# context length. Store them here so we can sanely check them. +# NOTE: The ordering here is important. Some models have two of these and we +# have a preference for which value gets used. +CONTEXT_LENGTH_KEYS = [ + "max_sequence_length", + "seq_length", + "max_seq_len", + "model_max_length", + "max_position_embeddings", +] + + +def attach_additional_stop_token_ids(tokenizer): + # Special handling for stop token <|eom_id|> generated by llama 3 tool use. + if "<|eom_id|>" in tokenizer.get_added_vocab(): + tokenizer.additional_stop_token_ids = set( + [tokenizer.get_added_vocab()["<|eom_id|>"]] + ) + else: + tokenizer.additional_stop_token_ids = None + + +def check_gguf_file(model: str | os.PathLike) -> bool: + """Check if the file is a GGUF model.""" + model = Path(model) + if not model.is_file(): + return False + elif model.suffix == ".gguf": + return True + + with open(model, "rb") as f: + header = f.read(4) + return header == b"GGUF" + + +def get_lock(model_name_or_path: str): + lock_dir = tempfile.gettempdir() + os.makedirs(os.path.dirname(lock_dir), exist_ok=True) + model_name = model_name_or_path.replace("/", "-") + hash_name = hashlib.sha256(model_name.encode()).hexdigest() + # add hash to avoid conflict with old users' lock files + lock_file_name = hash_name + model_name + ".lock" + # mode 0o666 is required for the filelock to be shared across users + lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name), mode=0o666) + return lock + + +def maybe_download_lora( + model_name_or_path: str, local_dir: str | None = None, download: bool = True +) -> str: + """ + Check if the model path is a Hugging Face Hub model ID and download it if needed. + Args: + model_name_or_path: Local path or Hugging Face Hub model ID + local_dir: Local directory to save the model + download: Whether to download the model from Hugging Face Hub + + Returns: + Local path to the model + """ + + local_path = maybe_download_model(model_name_or_path, local_dir, download) + # return directly if local_path is a file + if os.path.isfile(local_path): + return local_path + + weight_name = _best_guess_weight_name( + model_name_or_path, file_extension=".safetensors" + ) + return os.path.join(local_path, weight_name) + + +def verify_model_config_and_directory(model_path: str) -> dict[str, Any]: + """ + Verify that the model directory contains a valid diffusers configuration. + + Args: + model_path: Path to the model directory + + Returns: + The loaded model configuration as a dictionary + """ + + # Check for model_index.json which is required for diffusers models + config_path = os.path.join(model_path, "model_index.json") + if not os.path.exists(config_path): + raise ValueError( + f"Model directory {model_path} does not contain model_index.json. " + "Only HuggingFace diffusers format is supported." + ) + + # Check for transformer and vae directories + transformer_dir = os.path.join(model_path, "transformer") + vae_dir = os.path.join(model_path, "vae") + + if not os.path.exists(transformer_dir): + raise ValueError( + f"Model directory {model_path} does not contain a transformer/ directory." + ) + + if not os.path.exists(vae_dir): + raise ValueError( + f"Model directory {model_path} does not contain a vae/ directory." + ) + + # Load the config + with open(config_path) as f: + config = json.load(f) + + # Verify diffusers version exists + if "_diffusers_version" not in config: + raise ValueError("model_index.json does not contain _diffusers_version") + + logger.info("Diffusers version: %s", config["_diffusers_version"]) + return cast(dict[str, Any], config) + + +def maybe_download_model_index(model_name_or_path: str) -> dict[str, Any]: + """ + Download and extract just the model_index.json for a Hugging Face model. + + Args: + model_name_or_path: Path or HF Hub model ID + + Returns: + The parsed model_index.json as a dictionary + """ + import tempfile + + from huggingface_hub import hf_hub_download + from huggingface_hub.errors import EntryNotFoundError + + # If it's a local path, verify it directly + if os.path.exists(model_name_or_path): + try: + return verify_model_config_and_directory(model_name_or_path) + except ValueError: + # Not a pipeline, maybe a single model. + config_path = os.path.join(model_name_or_path, "config.json") + if os.path.exists(config_path): + with open(config_path) as f: + config = json.load(f) + return config + raise + + # For remote models, download just the model_index.json + try: + with tempfile.TemporaryDirectory() as tmp_dir: + # Download just the model_index.json file + model_index_path = hf_hub_download( + repo_id=model_name_or_path, + filename="model_index.json", + local_dir=tmp_dir, + ) + + # Load the model_index.json + with open(model_index_path) as f: + config: dict[str, Any] = json.load(f) + + # Verify it has the required fields + if "_class_name" not in config: + raise ValueError( + f"model_index.json for {model_name_or_path} does not contain _class_name field" + ) + + if "_diffusers_version" not in config: + raise ValueError( + f"model_index.json for {model_name_or_path} does not contain _diffusers_version field" + ) + + # Add the pipeline name for downstream use + config["pipeline_name"] = config["_class_name"] + + logger.info( + "Downloaded model_index.json for %s, pipeline: %s", + model_name_or_path, + config["_class_name"], + ) + return config + except EntryNotFoundError: + logger.warning( + "model_index.json not found for %s. Assuming it is a single model and downloading it.", + model_name_or_path, + ) + local_path = maybe_download_model(model_name_or_path) + config_path = os.path.join(local_path, "config.json") + if not os.path.exists(config_path): + raise ValueError( + f"Failed to find config.json for {model_name_or_path} after failing to find model_index.json" + f"You might be looking for models ending with '-Diffusers'" + ) + with open(config_path) as f: + config = json.load(f) + return config + except Exception as e: + raise ValueError( + f"Failed to download or parse model_index.json for {model_name_or_path}: {e}" + ) from e + + +def maybe_download_model( + model_name_or_path: str, local_dir: str | None = None, download: bool = True +) -> str: + """ + Check if the model path is a Hugging Face Hub model ID and download it if needed. + + Args: + model_name_or_path: Local path or Hugging Face Hub model ID + local_dir: Local directory to save the model + download: Whether to download the model from Hugging Face Hub + + Returns: + Local path to the model + """ + + # If the path exists locally, return it + if os.path.exists(model_name_or_path): + logger.info("Model already exists locally") + return model_name_or_path + + # Otherwise, assume it's a HF Hub model ID and try to download it + try: + logger.info( + "Downloading model snapshot from HF Hub for %s...", model_name_or_path + ) + with get_lock(model_name_or_path).acquire( + poll_interval=2 + ), suppress_other_loggers(not_suppress_on_main_rank=True): + local_path = snapshot_download( + repo_id=model_name_or_path, + ignore_patterns=["*.onnx", "*.msgpack"], + local_dir=local_dir, + ) + logger.info("Downloaded model to %s", local_path) + return str(local_path) + except Exception as e: + raise ValueError( + f"Could not find model at {model_name_or_path} and failed to download from HF Hub: {e}" + ) from e diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/utils/logging_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/utils/logging_utils.py new file mode 100644 index 00000000..86d2e7b8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/utils/logging_utils.py @@ -0,0 +1,482 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/logger.py +"""Logging configuration for sglang.multimodal_gen.""" +import argparse +import datetime +import logging +import os +import sys +import time +import warnings +from contextlib import contextmanager +from functools import lru_cache, partial +from logging import Logger +from types import MethodType +from typing import Any, cast + +import sglang.multimodal_gen.envs as envs + +SGLANG_DIFFUSION_CONFIGURE_LOGGING = envs.SGLANG_DIFFUSION_CONFIGURE_LOGGING +SGLANG_DIFFUSION_LOGGING_CONFIG_PATH = envs.SGLANG_DIFFUSION_LOGGING_CONFIG_PATH +SGLANG_DIFFUSION_LOGGING_LEVEL = envs.SGLANG_DIFFUSION_LOGGING_LEVEL +SGLANG_DIFFUSION_LOGGING_PREFIX = envs.SGLANG_DIFFUSION_LOGGING_PREFIX + +RED = "\033[91m" +GREEN = "\033[92m" +YELLOW = "\033[93m" +RESET = "\033[0;0m" + +_FORMAT = ( + f"{SGLANG_DIFFUSION_LOGGING_PREFIX}%(levelname)s %(asctime)s " + "[%(filename)s: %(lineno)d] %(message)s" +) + +# _FORMAT = "[%(asctime)s] %(message)s" +_DATE_FORMAT = "%m-%d %H:%M:%S" + +DEFAULT_LOGGING_CONFIG = { + "formatters": { + "sgl_diffusion": { + "class": "sglang.multimodal_gen.runtime.utils.logging_utils.ColoredFormatter", + "datefmt": _DATE_FORMAT, + "format": _FORMAT, + }, + }, + "handlers": { + "sgl_diffusion": { + "class": "logging.StreamHandler", + "formatter": "sgl_diffusion", + "level": SGLANG_DIFFUSION_LOGGING_LEVEL, + "stream": "ext://sys.stdout", + }, + }, + "loggers": { + "sgl_diffusion": { + "handlers": ["sgl_diffusion"], + "level": "WARNING", + "propagate": False, + }, + }, + "root": { + "handlers": ["sgl_diffusion"], + "level": "DEBUG", + }, + "version": 1, + "disable_existing_loggers": False, +} + + +class NewLineFormatter(logging.Formatter): + """Adds logging prefix to newlines to align multi-line messages.""" + + def __init__(self, fmt, datefmt=None, style="%"): + logging.Formatter.__init__(self, fmt, datefmt, style) + + def format(self, record): + msg = logging.Formatter.format(self, record) + if record.message != "": + parts = msg.split(record.message) + msg = msg.replace("\n", "\r\n" + parts[0]) + return msg + + +class ColoredFormatter(NewLineFormatter): + """A logging formatter that adds color to log levels.""" + + LEVEL_COLORS = { + logging.ERROR: RED, + logging.WARNING: YELLOW, + } + + def format(self, record: logging.LogRecord) -> str: + """Adds color to the log level name.""" + original_levelname = record.levelname + color = self.LEVEL_COLORS.get(record.levelno) + if color: + record.levelname = f"{color}{original_levelname}{RESET}" + + formatted_message = super().format(record) + + if color: + record.levelname = original_levelname + + return formatted_message + + +class SortedHelpFormatter(argparse.HelpFormatter): + """SortedHelpFormatter that sorts arguments by their option strings.""" + + def add_arguments(self, actions): + actions = sorted(actions, key=lambda x: x.option_strings) + super().add_arguments(actions) + + +@lru_cache +def _print_info_once(logger: Logger, msg: str) -> None: + # Set the stacklevel to 2 to print the original caller's line info + logger.info(msg, stacklevel=2) + + +@lru_cache +def _print_warning_once(logger: Logger, msg: str) -> None: + # Set the stacklevel to 2 to print the original caller's line info + logger.warning(msg, stacklevel=2) + + +def get_is_main_process(): + try: + rank = int(os.environ["RANK"]) + except (KeyError, ValueError): + rank = 0 + return rank == 0 + + +def get_is_local_main_process(): + try: + rank = int(os.environ["LOCAL_RANK"]) + except (KeyError, ValueError): + rank = 0 + return rank == 0 + + +def _log_process_aware( + level: int, + logger_self: Logger, + msg: object, + *args: Any, + main_process_only: bool, + local_main_process_only: bool, + **kwargs: Any, +) -> None: + """Helper function to log a message if the process rank matches the criteria.""" + is_main_process = get_is_main_process() + is_local_main_process = get_is_local_main_process() + + should_log = ( + not main_process_only + and not local_main_process_only + or (main_process_only and is_main_process) + or (local_main_process_only and is_local_main_process) + ) + + if should_log: + # stacklevel=3 to show the original caller's location, + # as this function is called by the patched methods. + logger_self.log(level, msg, *args, stacklevel=3, **kwargs) + + +class _SGLDiffusionLogger(Logger): + """ + Note: + This class is just to provide type information. + We actually patch the methods directly on the :class:`logging.Logger` + instance to avoid conflicting with other libraries such as + `intel_extension_for_pytorch.utils._logger`. + """ + + def info_once(self, msg: str) -> None: + """ + As :meth:`info`, but subsequent calls with the same message + are silently dropped. + """ + _print_info_once(self, msg) + + def warning_once(self, msg: str) -> None: + """ + As :meth:`warning`, but subsequent calls with the same message + are silently dropped. + """ + _print_warning_once(self, msg) + + def info( # type: ignore[override] + self, + msg: object, + *args: Any, + main_process_only: bool = True, + local_main_process_only: bool = True, + **kwargs: Any, + ) -> None: ... + + def debug( # type: ignore[override] + self, + msg: object, + *args: Any, + main_process_only: bool = True, + local_main_process_only: bool = True, + **kwargs: Any, + ) -> None: ... + + def warning( # type: ignore[override] + self, + msg: object, + *args: Any, + main_process_only: bool = False, + local_main_process_only: bool = True, + **kwargs: Any, + ) -> None: ... + + def error( # type: ignore[override] + self, + msg: object, + *args: Any, + main_process_only: bool = False, + local_main_process_only: bool = True, + **kwargs: Any, + ) -> None: ... + + +def init_logger(name: str) -> _SGLDiffusionLogger: + """The main purpose of this function is to ensure that loggers are + retrieved in such a way that we can be sure the root sgl_diffusion logger has + already been configured.""" + + logger = logging.getLogger(name) + + # Patch instance methods + setattr(logger, "info_once", MethodType(_print_info_once, logger)) + setattr(logger, "warning_once", MethodType(_print_warning_once, logger)) + + def _create_patched_method( + level: int, + main_process_only_default: bool, + local_main_process_only_default: bool, + ): + def _method( + self: Logger, + msg: object, + *args: Any, + main_process_only: bool = main_process_only_default, + local_main_process_only: bool = local_main_process_only_default, + **kwargs: Any, + ) -> None: + _log_process_aware( + level, + self, + msg, + *args, + main_process_only=main_process_only, + local_main_process_only=local_main_process_only, + **kwargs, + ) + + return _method + + setattr( + logger, + "info", + MethodType(_create_patched_method(logging.INFO, True, True), logger), + ) + setattr( + logger, + "debug", + MethodType(_create_patched_method(logging.DEBUG, True, True), logger), + ) + setattr( + logger, + "warning", + MethodType(_create_patched_method(logging.WARNING, False, True), logger), + ) + setattr( + logger, + "error", + MethodType(_create_patched_method(logging.ERROR, False, True), logger), + ) + + return cast(_SGLDiffusionLogger, logger) + + +logger = init_logger(__name__) + + +def _trace_calls(log_path, root_dir, frame, event, arg=None): + if event in ["call", "return"]: + # Extract the filename, line number, function name, and the code object + filename = frame.f_code.co_filename + lineno = frame.f_lineno + func_name = frame.f_code.co_name + if not filename.startswith(root_dir): + # only log the functions in the sgl_diffusion root_dir + return + # Log every function call or return + try: + last_frame = frame.f_back + if last_frame is not None: + last_filename = last_frame.f_code.co_filename + last_lineno = last_frame.f_lineno + last_func_name = last_frame.f_code.co_name + else: + # initial frame + last_filename = "" + last_lineno = 0 + last_func_name = "" + with open(log_path, "a") as f: + ts = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f") + if event == "call": + f.write( + f"{ts} Call to" + f" {func_name} in {filename}:{lineno}" + f" from {last_func_name} in {last_filename}:" + f"{last_lineno}\n" + ) + else: + f.write( + f"{ts} Return from" + f" {func_name} in {filename}:{lineno}" + f" to {last_func_name} in {last_filename}:" + f"{last_lineno}\n" + ) + except NameError: + # modules are deleted during shutdown + pass + return partial(_trace_calls, log_path, root_dir) + + +def enable_trace_function_call(log_file_path: str, root_dir: str | None = None): + """ + Enable tracing of every function call in code under `root_dir`. + This is useful for debugging hangs or crashes. + `log_file_path` is the path to the log file. + `root_dir` is the root directory of the code to trace. If None, it is the + sgl_diffusion root directory. + + Note that this call is thread-level, any threads calling this function + will have the trace enabled. Other threads will not be affected. + """ + logger.warning( + "SGLANG_DIFFUSION_TRACE_FUNCTION is enabled. It will record every" + " function executed by Python. This will slow down the code. It " + "is suggested to be used for debugging hang or crashes only." + ) + logger.info("Trace frame log is saved to %s", log_file_path) + if root_dir is None: + # by default, this is the sgl_diffusion root directory + root_dir = os.path.dirname(os.path.dirname(__file__)) + sys.settrace(partial(_trace_calls, log_file_path, root_dir)) + + +def set_uvicorn_logging_configs(): + from uvicorn.config import LOGGING_CONFIG + + LOGGING_CONFIG["formatters"]["default"][ + "fmt" + ] = "[%(asctime)s] %(levelprefix)s %(message)s" + LOGGING_CONFIG["formatters"]["default"]["datefmt"] = "%Y-%m-%d %H:%M:%S" + LOGGING_CONFIG["formatters"]["access"][ + "fmt" + ] = '[%(asctime)s] %(levelprefix)s %(client_addr)s - "%(request_line)s" %(status_code)s' + LOGGING_CONFIG["formatters"]["access"]["datefmt"] = "%Y-%m-%d %H:%M:%S" + + +def configure_logger(server_args, prefix: str = ""): + log_format = f"[%(asctime)s{prefix}] %(message)s" + datefmt = "%m-%d %H:%M:%S" + logging.basicConfig( + level=getattr(logging, server_args.log_level.upper()), + format=log_format, + datefmt=datefmt, + force=True, + ) + + set_uvicorn_logging_configs() + + +def suppress_loggers(loggers_to_suppress: list[str]): + original_levels = {} + + for logger_name in loggers_to_suppress: + logger = logging.getLogger(logger_name) + original_levels[logger_name] = logger.level + logger.setLevel(logging.WARNING) + + return original_levels + + +@contextmanager +def suppress_other_loggers(not_suppress_on_main_rank: bool = False): + """ + A context manager to temporarily suppress specified loggers. + + Args: + not_suppress_on_main_rank (bool): If True, loggers will not be + suppressed on the main process (rank 0). + """ + # This is a global setting that we want to apply to all ranks + warnings.filterwarnings( + "ignore", category=UserWarning, message="The given NumPy array is not writable" + ) + + should_suppress = True + if not_suppress_on_main_rank: + if get_is_main_process() == 0: + should_suppress = False + + loggers_to_suppress = ["urllib3", "imageio", "imageio_ffmpeg", "PIL", "PIL_Image"] + original_levels = suppress_loggers(loggers_to_suppress) + + try: + yield + finally: + if should_suppress: + for logger_name, level in original_levels.items(): + logging.getLogger(logger_name).setLevel(level) + + +class GenerationTimer: + def __init__(self): + self.start_time = 0.0 + self.end_time = 0.0 + self.duration = 0.0 + + +@contextmanager +def log_generation_timer( + logger: logging.Logger, + prompt: str, + request_idx: int | None = None, + total_requests: int | None = None, +): + if request_idx is not None and total_requests is not None: + logger.info( + "Processing prompt %d/%d: %s", + request_idx, + total_requests, + prompt[:100], + ) + else: + max_len = 100 + suffix = "..." if len(prompt) > max_len else "" + logger.info(f"Processing prompt: {prompt[:100]}{suffix}") + + timer = GenerationTimer() + timer.start_time = time.perf_counter() + try: + yield timer + timer.end_time = time.perf_counter() + timer.duration = timer.end_time - timer.start_time + logger.info("Pixel data generated successfully in %.2f seconds", timer.duration) + except Exception as e: + if request_idx is not None: + logger.error( + "Failed to generate output for prompt %d: %s", + request_idx, + e, + exc_info=True, + ) + else: + logger.error( + f"Failed to generate output for prompt: {e}", + exc_info=True, + ) + raise + + +def log_batch_completion( + logger: logging.Logger, num_outputs: int, total_time: float +) -> None: + logger.info( + "Completed batch processing. Generated %d outputs in %.2f seconds.", + num_outputs, + total_time, + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/utils/perf_logger.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/utils/perf_logger.py new file mode 100644 index 00000000..16f94255 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/runtime/utils/perf_logger.py @@ -0,0 +1,263 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo +import dataclasses +import json +import logging +import os +import subprocess +import sys +import time +from datetime import datetime +from functools import lru_cache +from pathlib import Path +from typing import Any, Dict, Optional + +from dateutil.tz import UTC + +import sglang +import sglang.multimodal_gen.envs as envs +from sglang.multimodal_gen.runtime.utils.logging_utils import ( + _SGLDiffusionLogger, + get_is_main_process, +) + + +@dataclasses.dataclass +class RequestTimings: + """A lightweight data class to store performance timings for a single request.""" + + def __init__(self, request_id: str): + self.request_id = request_id + self.stages: Dict[str, float] = {} + self.steps: list[float] = [] + self.total_duration_ms: float = 0.0 + + def record_stage(self, stage_name: str, duration_s: float): + """Records the duration of a pipeline stage""" + self.stages[stage_name] = duration_s * 1000 # Store as milliseconds + + def record_steps(self, index: int, duration_s: float): + """Records the duration of a denoising step""" + assert index == len(self.steps) + self.steps.append(duration_s * 1000) + + def to_dict(self) -> Dict[str, Any]: + """Serializes the timing data to a dictionary.""" + return { + "request_id": self.request_id, + "stages": self.stages, + "steps": self.steps, + "total_duration_ms": self.total_duration_ms, + } + + +def get_diffusion_perf_log_dir() -> str: + """ + Determines the directory for performance logs. + """ + log_dir = os.environ.get("SGLANG_PERF_LOG_DIR") + if log_dir: + return os.path.abspath(log_dir) + if log_dir is None: + sglang_path = Path(sglang.__file__).resolve() + target_path = (sglang_path.parent / "../../.cache/logs").resolve() + return str(target_path) + return "" + + +@lru_cache(maxsize=1) +def get_git_commit_hash() -> str: + try: + commit_hash = os.environ.get("SGLANG_GIT_COMMIT") + if not commit_hash: + commit_hash = ( + subprocess.check_output( + ["git", "rev-parse", "HEAD"], stderr=subprocess.DEVNULL + ) + .strip() + .decode("utf-8") + ) + _CACHED_COMMIT_HASH = commit_hash + return commit_hash + except (subprocess.CalledProcessError, FileNotFoundError): + _CACHED_COMMIT_HASH = "N/A" + return "N/A" + + +@dataclasses.dataclass +class RequestPerfRecord: + request_id: str + + timestamp: str + commit_hash: str + tag: str + + stages: list[dict] + steps: list[float] + total_duration_ms: float + + def __init__( + self, + request_id, + commit_hash, + tag, + stages, + steps, + total_duration_ms, + timestamp=None, + ): + self.request_id = request_id + if timestamp is not None: + self.timestamp = timestamp + else: + self.timestamp = datetime.now(UTC).isoformat() + + self.commit_hash = commit_hash + self.tag = tag + self.stages = stages + self.steps = steps + self.total_duration_ms = total_duration_ms + + +class StageProfiler: + """ + A unified context manager, records timing information (usually of a single Stage or a step) into a provided RequestTimings object (usually from a Req). + """ + + def __init__( + self, + stage_name: str, + logger: _SGLDiffusionLogger, + timings: Optional["RequestTimings"], + simple_log: bool = False, + ): + self.stage_name = stage_name + self.timings = timings + self.logger = logger + self.simple_log = simple_log + self.start_time = 0.0 + + # Check env var at runtime to ensure we pick up changes (e.g. from CLI args) + self.metrics_enabled = envs.SGLANG_DIFFUSION_STAGE_LOGGING + + def __enter__(self): + if self.simple_log: + self.logger.info(f"[{self.stage_name}] started...") + + if (self.metrics_enabled and self.timings) or self.simple_log: + self.start_time = time.perf_counter() + + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if not ((self.metrics_enabled and self.timings) or self.simple_log): + return False + + execution_time_s = time.perf_counter() - self.start_time + + if exc_type: + self.logger.error( + "[%s] Error during execution after %.4f ms: %s", + self.stage_name, + execution_time_s * 1000, + exc_val, + exc_info=True, + ) + return False + + if self.simple_log: + self.logger.info( + f"[{self.stage_name}] finished in {execution_time_s:.4f} seconds" + ) + + if self.metrics_enabled and self.timings: + if "denoising_step_" in self.stage_name: + index = int(self.stage_name[len("denoising_step_") :]) + self.timings.record_steps(index, execution_time_s) + else: + self.timings.record_stage(self.stage_name, execution_time_s) + + return False + + +class PerformanceLogger: + """ + A global utility class for logging performance metrics for all request, categorized by request-id. + + Serves both as a runtime logger (stream to file) and a dump utility. + + Notice that ""RequestTimings"" stores the performance metrics of a single request + """ + + @classmethod + def dump_benchmark_report( + cls, + file_path: str, + timings: "RequestTimings", + meta: Optional[Dict[str, Any]] = None, + tag: str = "benchmark_dump", + ): + """ + Static method to dump a standardized benchmark report to a file. + Eliminates duplicate logic in CLI/Client code. + """ + formatted_steps = [ + {"name": name, "duration_ms": duration_ms} + for name, duration_ms in timings.stages.items() + ] + + report = { + "timestamp": datetime.now(UTC).isoformat(), + "request_id": timings.request_id, + "commit_hash": get_git_commit_hash(), + "tag": tag, + "total_duration_ms": timings.total_duration_ms, + "steps": formatted_steps, + "meta": meta or {}, + } + + try: + abs_path = os.path.abspath(file_path) + os.makedirs(os.path.dirname(abs_path), exist_ok=True) + with open(abs_path, "w", encoding="utf-8") as f: + json.dump(report, f, indent=2) + print(f"[Performance] Metrics dumped to: {abs_path}") + except IOError as e: + print(f"[Performance] Failed to dump metrics to {abs_path}: {e}") + logging.getLogger(__name__).error(f"Dump failed: {e}") + + @classmethod + def log_request_summary( + cls, + timings: "RequestTimings", + tag: str = "total_inference_time", + ): + """logs the stage metrics and total duration for a completed request + to the performance_log file. + """ + formatted_stages = [ + {"name": name, "execution_time_ms": duration_ms} + for name, duration_ms in timings.stages.items() + ] + + record = RequestPerfRecord( + timings.request_id, + commit_hash=get_git_commit_hash(), + tag="pipeline_stage_metrics", + stages=formatted_stages, + steps=timings.steps, + total_duration_ms=timings.total_duration_ms, + ) + + try: + if get_is_main_process(): + log_dir = get_diffusion_perf_log_dir() + if not os.path.exists(log_dir): + os.makedirs(log_dir, exist_ok=True) + + log_file = os.path.join(log_dir, "performance.log") + + with open(log_file, "a", encoding="utf-8") as f: + f.write(json.dumps(dataclasses.asdict(record)) + "\n") + + except (OSError, PermissionError) as e: + print(f"WARNING: Failed to log performance record: {e}", file=sys.stderr) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/__init__.py new file mode 100644 index 00000000..af2eb7d1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/__init__.py @@ -0,0 +1 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/cli/test_generate_common.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/cli/test_generate_common.py new file mode 100644 index 00000000..49dfa653 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/cli/test_generate_common.py @@ -0,0 +1,107 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +""" + Common generate cli test, one test for image and video each +""" + +import os +import unittest +from pathlib import Path + +from PIL import Image + +from sglang.multimodal_gen.test.test_utils import ( + TestCLIBase, + check_image_size, + is_mp4, + run_command, +) + + +class TestGenerate(TestCLIBase): + model_path = "black-forest-labs/FLUX.1-dev" + launch_file_name = "launch_flux.json" + output_name = "FLUX.1-dev, single gpu" + ext = "jpg" + + def test_generate_with_config(self): + test_dir = Path(__file__).parent + config_path = ( + (test_dir / ".." / "test_files" / self.launch_file_name) + .resolve() + .as_posix() + ) + command = [ + "sgl_diffusion", + "generate", + f"--config={config_path}", + ] + duration = run_command(command) + + self.assertIsNotNone(duration, f"Run command failed: {command}") + + # verify + self.verify_image(self.output_name) + + def test_generate_multiple_outputs(self): + command = [ + "sglang", + "generate", + "--prompt", + "A curious raccoon", + "--output-path=outputs", + f"--model-path={self.model_path}", + "--save-output", + f"--output-file-name={self.output_name}", + "--num-outputs-per-prompt=2", + "--width=720", + "--height=720", + ] + duration = run_command(command) + self.assertIsNotNone(duration, f"Run command failed: {command}") + + self.verify_image(f"{self.output_name}_0.{self.ext}") + self.verify_image(f"{self.output_name}_1.{self.ext}") + + def verify_image(self, output_name): + path = os.path.join("outputs", output_name) + with Image.open(path) as image: + check_image_size(self, image, 720, 720) + + def verify_video(self, output_name): + path = os.path.join("outputs", output_name) + with open(path, "rb") as f: + header = f.read(12) + assert is_mp4(header) + + +class TestWanGenerate(TestGenerate): + model_path = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" + launch_file_name = "launch_wan.json" + output_name = "Wan2.1-T2V-1.3B-Diffusers, single gpu" + ext = "mp4" + + def test_generate_multiple_outputs(self): + command = [ + "sglang", + "generate", + "--prompt", + "A curious raccoon", + "--output-path=outputs", + f"--model-path={self.model_path}", + "--save-output", + f"--output-file-name={self.output_name}", + "--num-outputs-per-prompt=2", + "--width=720", + "--height=720", + ] + duration = run_command(command) + self.assertIsNotNone(duration, f"Run command failed: {command}") + + self.verify_video(f"{self.output_name}_0.{self.ext}") + # FIXME: second video is a meaningless output + self.verify_video(f"{self.output_name}_1.{self.ext}") + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/cli/test_generate_t2i_perf.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/cli/test_generate_t2i_perf.py new file mode 100644 index 00000000..400e8c71 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/cli/test_generate_t2i_perf.py @@ -0,0 +1,84 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +import unittest +from pathlib import Path + +from sglang.multimodal_gen.configs.sample.sampling_params import DataType +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.test.test_utils import TestGenerateBase + +logger = init_logger(__name__) + + +class TestFlux_T2V(TestGenerateBase): + model_path = "black-forest-labs/FLUX.1-dev" + extra_args = [] + data_type: DataType = DataType.IMAGE + thresholds = { + "test_single_gpu": 6.5 * 1.05, + "test_usp": 8.3 * 1.05, + } + + def test_cfg_parallel(self): + pass + + def test_mixed(self): + pass + + +class TestQwenImage(TestGenerateBase): + model_path = "Qwen/Qwen-Image" + extra_args = [] + data_type: DataType = DataType.IMAGE + thresholds = { + "test_single_gpu": 10.4 * 1.05, + "test_usp": 20.2 * 1.05, + } + + def test_cfg_parallel(self): + pass + + def test_mixed(self): + pass + + +class TestQwenImageEdit(TestGenerateBase): + model_path = "Qwen/Qwen-Image-Edit" + extra_args = [] + data_type: DataType = DataType.IMAGE + thresholds = { + "test_single_gpu": 33.4 * 1.05, + "test_usp": 26.9 * 1.05, + } + + prompt: str | None = ( + "Change the rabbit's color to purple, with a flash light background." + ) + + def setUp(self): + test_dir = Path(__file__).parent + img_path = (test_dir / ".." / "test_files" / "rabbit.jpg").resolve().as_posix() + self.base_command = [ + "sglang", + "generate", + "--text-encoder-cpu-offload", + "--pin-cpu-memory", + f"--prompt", + f"{self.prompt}", + "--save-output", + "--log-level=debug", + f"--width={self.width}", + f"--height={self.height}", + f"--output-path={self.output_path}", + ] + [f"--image-path={img_path}"] + + def test_cfg_parallel(self): + pass + + def test_mixed(self): + pass + + +if __name__ == "__main__": + del TestGenerateBase + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/cli/test_generate_t2v_perf.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/cli/test_generate_t2v_perf.py new file mode 100644 index 00000000..1739bc8c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/cli/test_generate_t2v_perf.py @@ -0,0 +1,81 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +import unittest + +from sglang.multimodal_gen.configs.sample.sampling_params import DataType +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.test.test_utils import TestGenerateBase + +logger = init_logger(__name__) + + +class TestFastWan2_1_T2V(TestGenerateBase): + model_path = "FastVideo/FastWan2.1-T2V-1.3B-Diffusers" + extra_args = ["--attention-backend=video_sparse_attn"] + data_type: DataType = DataType.VIDEO + thresholds = { + "test_single_gpu": 13.0, + "test_cfg_parallel": 15.0, + "test_usp": 15.0, + "test_mixed": 15.0 * 1.05, + } + + # disabled for vsa + def test_usp(self): + pass + + +class TestFastWan2_2_T2V(TestGenerateBase): + model_path = "FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers" + extra_args = [] + data_type: DataType = DataType.VIDEO + thresholds = { + "test_single_gpu": 25.0, + "test_cfg_parallel": 30.0, + "test_usp": 30.0, + "test_mixed": 30.0, + } + + +class TestWan2_1_T2V(TestGenerateBase): + model_path = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" + extra_args = [] + data_type: DataType = DataType.VIDEO + thresholds = { + "test_single_gpu": 76.0 * 1.05, + "test_cfg_parallel": 46.5 * 1.05, + "test_usp": 39.8 * 1.05, + "test_mixed": 37.3 * 1.05, + } + + def test_mixed(self): + pass + + def test_cfg_parallel(self): + pass + + +class TestWan2_2_T2V(TestGenerateBase): + model_path = "Wan-AI/Wan2.2-T2V-A14B-Diffusers" + extra_args = [] + data_type: DataType = DataType.VIDEO + thresholds = { + "test_single_gpu": 904.3 * 1.05, + "test_cfg_parallel": 446, + "test_usp": 316 * 1.05, + "test_mixed": 159, + } + + def test_single_gpu(self): + pass + + def test_mixed(self): + pass + + def test_cfg_parallel(self): + pass + + +if __name__ == "__main__": + del TestGenerateBase + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/cli/test_generate_ti2v_perf.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/cli/test_generate_ti2v_perf.py new file mode 100644 index 00000000..3dc3c7c8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/cli/test_generate_ti2v_perf.py @@ -0,0 +1,71 @@ +import unittest + +from sglang.multimodal_gen.configs.sample.sampling_params import DataType +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.test.test_utils import TestGenerateBase + +logger = init_logger(__name__) + + +class TestGenerateTI2VBase(TestGenerateBase): + data_type: DataType = DataType.VIDEO + + @classmethod + def setUpClass(cls): + cls.base_command = [ + "sglang", + "generate", + "--prompt", + "Summer beach vacation style, a white cat wearing sunglasses sits on a surfboard. The fluffy-furred feline gazes directly at the camera with a relaxed expression. Blurred beach scenery forms the background featuring crystal-clear waters, distant green hills, and a blue sky dotted with white clouds. The cat assumes a naturally relaxed posture, as if savoring the sea breeze and warm sunlight. A close-up shot highlights the feline's intricate details and the refreshing atmosphere of the seaside.", + "--image-path", + "https://github.com/Wan-Video/Wan2.2/blob/990af50de458c19590c245151197326e208d7191/examples/i2v_input.JPG?raw=true", + "--save-output", + "--log-level=debug", + f"--output-path={cls.output_path}", + ] + cls.extra_args + + def test_single_gpu(self): + pass + + def test_cfg_parallel(self): + pass + + def test_mixed(self): + pass + + +class TestWan2_1_I2V_14B_480P(TestGenerateTI2VBase): + model_path = "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers" + thresholds = { + "test_usp": 557.9 * 1.05, + } + + +class TestWan2_1_I2V_14B_720P(TestGenerateTI2VBase): + model_path = "Wan-AI/Wan2.1-I2V-14B-720P-Diffusers" + thresholds = { + "test_usp": 558.4 * 1.05, + } + + +class TestWan2_2_TI2V_5B(TestGenerateTI2VBase): + model_path = "Wan-AI/Wan2.2-TI2V-5B-Diffusers" + # FIXME: doesn't work with vsa at the moment + # extra_args = ["--attention-backend=video_sparse_attn"] + thresholds = { + "test_usp": 82.3 * 1.05, + } + + +# OOM +# class TestWan2_2_I2V_A14B(TestGenerateTI2VBase): +# model_path = "Wan-AI/Wan2.2-I2V-A14B-Diffusers" +# # FIXME: doesn't work with vsa at the moment +# thresholds = { +# "test_usp": 66.3 * 1.05, +# } + + +if __name__ == "__main__": + del TestGenerateTI2VBase, TestGenerateBase + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/cli/test_serve.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/cli/test_serve.py new file mode 100644 index 00000000..156cf215 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/cli/test_serve.py @@ -0,0 +1,301 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +import asyncio +import base64 +import subprocess +import tempfile +import time +import unittest +import uuid +from contextlib import contextmanager +from pathlib import Path +from urllib.request import urlopen + +from openai import OpenAI + +from sglang.multimodal_gen.runtime.utils.common import kill_process_tree +from sglang.multimodal_gen.test.test_utils import is_mp4, is_png, wait_for_port + + +@contextmanager +def downloaded_temp_file(url: str, prefix: str = "i2v_input_", suffix: str = ".jpg"): + tmp_path = Path(tempfile.gettempdir()) / f"{prefix}{uuid.uuid4().hex}{suffix}" + with urlopen(url) as resp: + tmp_path.write_bytes(resp.read()) + try: + yield tmp_path + finally: + try: + tmp_path.unlink(missing_ok=True) + except Exception: + pass + + +def wait_for_video_completion(client, video_id, timeout=300, check_interval=3): + start = time.time() + video = client.videos.retrieve(video_id) + + while video.status not in ("completed", "failed"): + time.sleep(check_interval) + video = client.videos.retrieve(video_id) + assert time.time() - start < timeout, "video generate timeout" + + return video + + +class TestVideoHttpServer(unittest.TestCase): + model_name = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" + timeout = 500 + extra_args = [] + + def _create_wait_and_download( + self, client: OpenAI, prompt: str, size: str + ) -> bytes: + + video = client.videos.create(prompt=prompt, size=size) + video_id = video.id + self.assertEqual(video.status, "queued") + + video = wait_for_video_completion(client, video_id, timeout=self.timeout) + self.assertEqual(video.status, "completed", "video generate failed") + + response = client.videos.download_content( + video_id=video_id, + ) + content = response.read() + return content + + @classmethod + def setUpClass(cls): + cls.base_command = [ + "sglang", + "serve", + "--model-path", + f"{cls.model_name}", + "--port", + "30010", + ] + + process = subprocess.Popen( + cls.base_command + cls.extra_args, + # stdout=subprocess.PIPE, + # stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + cls.pid = process.pid + wait_for_port(host="127.0.0.1", port=30010) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.pid) + + def test_http_server_basic(self): + client = OpenAI( + api_key="sk-proj-1234567890", base_url="http://localhost:30010/v1" + ) + content = self._create_wait_and_download( + client, "A plane is taking off.", "832x480" + ) + self.assertTrue(is_mp4(content)) + + def test_concurrent_requests(self): + client = OpenAI( + api_key="sk-proj-1234567890", base_url="http://localhost:30010/v1" + ) + + num_requests = 2 + + async def generate_and_check_video(prompt, size): + content = await asyncio.to_thread( + self._create_wait_and_download, client, prompt, size + ) + self.assertTrue(is_mp4(content)) + + async def send_concurrent_requests(): + tasks = [ + generate_and_check_video( + "A ship is beside the port.", + "832x480", + ) + for _ in range(num_requests) + ] + await asyncio.gather(*tasks) + + asyncio.run(send_concurrent_requests()) + + +class TestImage2VideoHttpServer(unittest.TestCase): + model_name = "Wan-AI/Wan2.2-I2V-A14B-Diffusers" + timeout = 1200 + extra_args = [] + + def _create_wait_and_download( + self, client: OpenAI, prompt: str, size: str + ) -> bytes: + + image_url = "https://github.com/Wan-Video/Wan2.2/blob/990af50de458c19590c245151197326e208d7191/examples/i2v_input.JPG?raw=true" + with downloaded_temp_file( + image_url, prefix="i2v_input_", suffix=".jpg" + ) as tmp_path: + video = client.videos.create( + prompt=prompt, + input_reference=tmp_path, + size=size, + seconds=10, + extra_body={"fps": 16, "num_frames": 125}, + ) + # TODO: Some combinations of num_frames and fps may cause errors and need further investigation. + video_id = video.id + self.assertEqual(video.status, "queued") + + video = wait_for_video_completion(client, video_id, timeout=self.timeout) + self.assertEqual(video.status, "completed", "video generate failed") + + response = client.videos.download_content( + video_id=video_id, + ) + content = response.read() + return content + + @classmethod + def setUpClass(cls): + cls.base_command = [ + "sglang", + "serve", + "--model-path", + f"{cls.model_name}", + "--num-gpus", + "4", + "--ulysses-degree", + "4", + "--port", + "30010", + ] + + process = subprocess.Popen( + cls.base_command + cls.extra_args, + text=True, + bufsize=1, + ) + cls.pid = process.pid + wait_for_port(host="127.0.0.1", port=30010) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.pid) + + def test_http_server_basic(self): + client = OpenAI( + api_key="sk-proj-1234567890", base_url="http://localhost:30010/v1" + ) + content = self._create_wait_and_download( + client, "A cat surfing on the sea.", "832x480" + ) + self.assertTrue(is_mp4(content)) + + def test_concurrent_requests(self): + client = OpenAI( + api_key="sk-proj-1234567890", base_url="http://localhost:30010/v1" + ) + + num_requests = 2 + + async def generate_and_check_video(prompt, size): + content = await asyncio.to_thread( + self._create_wait_and_download, client, prompt, size + ) + self.assertTrue(is_mp4(content)) + + async def send_concurrent_requests(): + tasks = [ + generate_and_check_video( + "A cat surfing on the sea.", + "832x480", + ) + for _ in range(num_requests) + ] + await asyncio.gather(*tasks) + + asyncio.run(send_concurrent_requests()) + + +class TestImageHttpServer(unittest.TestCase): + + def _create_wait_and_download( + self, client: OpenAI, prompt: str, size: str + ) -> bytes: + img = client.images.generate( + model="gpt-image-1", + prompt=prompt, + n=1, + size=size, + response_format="b64_json", + output_format="png", + ) + image_bytes = base64.b64decode(img.data[0].b64_json) + return image_bytes + + @classmethod + def setUpClass(cls): + cls.base_command = [ + "sglang", + "serve", + "--model-path", + "Qwen/Qwen-Image", + "--port", + "30020", + ] + + process = subprocess.Popen( + cls.base_command, + # stdout=subprocess.PIPE, + # stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + cls.pid = process.pid + wait_for_port(host="127.0.0.1", port=30020) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.pid) + + def test_http_server_basic(self): + client = OpenAI( + api_key="sk-proj-1234567890", base_url="http://localhost:30020/v1" + ) + content = self._create_wait_and_download( + client, "A calico cat playing a piano on stage", "832x480" + ) + self.assertTrue(is_png(content)) + + def test_concurrent_requests(self): + client = OpenAI( + api_key="sk-proj-1234567890", base_url="http://localhost:30020/v1" + ) + + num_requests = 2 + + async def generate_and_check_image(prompt, size): + content = await asyncio.to_thread( + self._create_wait_and_download, client, prompt, size + ) + self.assertTrue(is_png(content)) + + async def send_concurrent_requests(): + tasks = [ + generate_and_check_image( + "A dog playing a piano on stage", + "832x480", + ) + for _ in range(num_requests) + ] + await asyncio.gather(*tasks) + + asyncio.run(send_concurrent_requests()) + + +if __name__ == "__main__": + # del TestPerform·anceBase + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/run_suite.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/run_suite.py new file mode 100644 index 00000000..206d06cc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/run_suite.py @@ -0,0 +1,173 @@ +""" +Test runner for multimodal_gen that manages test suites and parallel execution. + +Usage: + python3 run_suite.py --suite --partition-id --total-partitions + +Example: + python3 run_suite.py --suite 1-gpu --partition-id 0 --total-partitions 2 +""" + +import argparse +import os +import subprocess +import sys +from pathlib import Path + +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + +SUITES = { + "1-gpu": [ + "test_server_a.py", + "test_server_b.py", + # add new 1-gpu test files here + ], + "2-gpu": [ + "test_server_2_gpu_a.py", + "test_server_2_gpu_b.py", + # add new 2-gpu test files here + ], +} + + +def parse_args(): + parser = argparse.ArgumentParser(description="Run multimodal_gen test suite") + parser.add_argument( + "--suite", + type=str, + required=True, + choices=list(SUITES.keys()), + help="The test suite to run (e.g., 1-gpu, 2-gpu)", + ) + parser.add_argument( + "--partition-id", + type=int, + default=0, + help="Index of the current partition (for parallel execution)", + ) + parser.add_argument( + "--total-partitions", + type=int, + default=1, + help="Total number of partitions", + ) + parser.add_argument( + "--base-dir", + type=str, + default="server", + help="Base directory for tests relative to this script's parent", + ) + return parser.parse_args() + + +def run_pytest(files): + if not files: + print("No files to run.") + return 0 + + base_cmd = [sys.executable, "-m", "pytest", "-s", "-v", "--log-cli-level=INFO"] + + max_retries = 2 + # retry if the perf assertion failed, for {max_retries} times + for i in range(max_retries + 1): + cmd = list(base_cmd) + if i > 0: + cmd.append("--last-failed") + cmd.extend(files) + + if i > 0: + logger.info( + f"Performance assertion failed. Retrying ({i}/{max_retries}) with --last-failed..." + ) + + logger.info(f"Running command: {' '.join(cmd)}") + + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + + output_lines = [] + while True: + line = process.stdout.readline() + if not line and process.poll() is not None: + break + if line: + sys.stdout.write(line) + output_lines.append(line) + + returncode = process.poll() + + if returncode == 0: + return 0 + + # check if the failure is due to an assertion in test_server_utils.py + full_output = "".join(output_lines) + is_perf_assertion = ( + "multimodal_gen/test/server/test_server_utils.py" in full_output + and "AssertionError" in full_output + ) + + if not is_perf_assertion: + return returncode + + return returncode + + +def main(): + args = parse_args() + + # 1. resolve base path + current_file_path = Path(__file__).resolve() + test_root_dir = current_file_path.parent + target_dir = test_root_dir / args.base_dir + + if not target_dir.exists(): + print(f"Error: Target directory {target_dir} does not exist.") + sys.exit(1) + + # 2. get files from suite + suite_files_rel = SUITES[args.suite] + + suite_files_abs = [] + for f_rel in suite_files_rel: + f_abs = target_dir / f_rel + if not f_abs.exists(): + print(f"Warning: Test file {f_rel} not found in {target_dir}. Skipping.") + continue + suite_files_abs.append(str(f_abs)) + + if not suite_files_abs: + print(f"No valid test files found for suite '{args.suite}'.") + sys.exit(0) + + # 3. partitioning + my_files = [ + f + for i, f in enumerate(suite_files_abs) + if i % args.total_partitions == args.partition_id + ] + + print( + f"Suite: {args.suite} | Partition: {args.partition_id}/{args.total_partitions}" + ) + print(f"Selected {len(my_files)} files:") + for f in my_files: + print(f" - {os.path.basename(f)}") + + if not my_files: + print("No files assigned to this partition. Exiting success.") + sys.exit(0) + + # 4. execute + exit_code = run_pytest(my_files) + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/conftest.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/conftest.py new file mode 100644 index 00000000..96b49591 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/conftest.py @@ -0,0 +1,53 @@ +_GLOBAL_PERF_RESULTS = [] + + +def pytest_sessionfinish(session): + """ + This hook is called by pytest at the end of the entire test session. + It prints a consolidated summary of all performance results. + """ + if not _GLOBAL_PERF_RESULTS: + return + + print("\n\n" + "=" * 35 + " Performance Summary " + "=" * 35) + print( + f"{'Test Suite':<30} | {'Test Name':<20} | {'E2E (ms)':>12} | {'Avg Denoise (ms)':>18} | {'Median Denoise (ms)':>20}" + ) + print( + "-" * 30 + + "-+-" + + "-" * 20 + + "-+-" + + "-" * 12 + + "-+-" + + "-" * 18 + + "-+-" + + "-" * 20 + ) + + for entry in sorted(_GLOBAL_PERF_RESULTS, key=lambda x: x["class_name"]): + print( + f"{entry['class_name']:<30} | {entry['test_name']:<20} | {entry['e2e_ms']:>12.2f} | " + f"{entry['avg_denoise_ms']:>18.2f} | {entry['median_denoise_ms']:>20.2f}" + ) + + print("=" * 91) + + print("\n\n" + "=" * 36 + " Detailed Reports " + "=" * 37) + for entry in sorted(_GLOBAL_PERF_RESULTS, key=lambda x: x["class_name"]): + print(f"\n--- Details for {entry['class_name']} / {entry['test_name']} ---") + stage_report = ", ".join( + f"{name}:{duration:.2f}ms" + for name, duration in entry.get("stage_metrics", {}).items() + ) + if stage_report: + print(f" Stages: {stage_report}") + + sampled_steps = entry.get("sampled_steps") or {} + if sampled_steps: + step_report = ", ".join( + f"{idx}:{duration:.2f}ms" + for idx, duration in sorted(sampled_steps.items()) + ) + print(f" Sampled Steps: {step_report}") + print("=" * 91) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/perf_baselines.json b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/perf_baselines.json new file mode 100644 index 00000000..0757aa51 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/perf_baselines.json @@ -0,0 +1,1018 @@ +{ + "metadata": { + "model": "Diffusion Server", + "hardware": "CI H100 80GB pool", + "description": "Reference numbers captured from the CI diffusion server baseline run" + }, + "tolerances": { + "e2e": 0.1, + "denoise_stage": 0.05, + "non_denoise_stage": 0.4, + "denoise_step": 0.2, + "denoise_agg": 0.1 + }, + "improvement_reporting": { + "threshold": 0.2 + }, + "sampling": { + "step_fractions": [ + 0.0, + 0.2, + 0.4, + 0.6, + 0.8, + 1.0 + ], + "warmup_requests": { + "text": 1, + "image_edit": 0 + } + }, + "scenarios": { + "qwen_image_t2i": { + "notes": "Single-image generation using the default prompt", + "expected_e2e_ms": 74500.0, + "expected_avg_denoise_ms": 422.42, + "expected_median_denoise_ms": 410.62, + "stages_ms": { + "InputValidationStage": 0.1, + "TextEncodingStage": 834.2, + "ConditioningStage": 0.1, + "TimestepPreparationStage": 10.6, + "LatentPreparationStage": 11.8, + "DenoisingStage": 21202.6, + "DecodingStage": 751.1 + }, + "denoise_step_ms": { + "0": 1077.77, + "1": 345.13, + "2": 413.8, + "3": 405.49, + "4": 408.14, + "5": 409.06, + "6": 408.85, + "7": 410.53, + "8": 407.51, + "9": 409.44, + "10": 408.65, + "11": 410.14, + "12": 411.74, + "13": 409.59, + "14": 409.17, + "15": 410.78, + "16": 410.66, + "17": 410.58, + "18": 411.27, + "19": 410.51, + "20": 409.03, + "21": 410.16, + "22": 409.42, + "23": 411.03, + "24": 410.18, + "25": 409.72, + "26": 410.26, + "27": 410.21, + "28": 410.71, + "29": 470.76, + "30": 411.06, + "31": 410.1, + "32": 410.55, + "33": 410.77, + "34": 410.74, + "35": 411.75, + "36": 410.78, + "37": 411.56, + "38": 410.85, + "39": 411.08, + "40": 411.12, + "41": 411.1, + "42": 411.09, + "43": 410.87, + "44": 411.37, + "45": 411.68, + "46": 411.0, + "47": 410.09, + "48": 412.72, + "49": 410.42 + } + }, + "qwen_image_t2i_2_gpus": { + "stages_ms": { + "InputValidationStage": 0.04, + "TextEncodingStage": 693.2, + "ConditioningStage": 0.02, + "TimestepPreparationStage": 2.84, + "LatentPreparationStage": 9.13, + "DenoisingStage": 24529.77, + "DecodingStage": 612.79 + }, + "denoise_step_ms": { + "0": 405.94, + "1": 420.06, + "2": 414.79, + "3": 392.4, + "4": 408.14, + "5": 605.0, + "6": 469.39, + "7": 574.04, + "8": 539.61, + "9": 452.93, + "10": 279.36, + "11": 271.8, + "12": 438.26, + "13": 552.65, + "14": 576.1, + "15": 679.84, + "16": 543.0, + "17": 512.81, + "18": 522.27, + "19": 545.06, + "20": 545.85, + "21": 523.83, + "22": 519.36, + "23": 513.78, + "24": 532.54, + "25": 524.94, + "26": 542.59, + "27": 570.91, + "28": 568.73, + "29": 564.52, + "30": 564.57, + "31": 544.94, + "32": 496.81, + "33": 488.98, + "34": 457.18, + "35": 441.42, + "36": 437.44, + "37": 477.6, + "38": 429.17, + "39": 465.55, + "40": 448.25, + "41": 511.83, + "42": 450.6, + "43": 375.78, + "44": 504.4, + "45": 524.44, + "46": 535.22, + "47": 514.52, + "48": 431.58, + "49": 410.68 + }, + "expected_e2e_ms": 25850.45, + "expected_avg_denoise_ms": 490.43, + "expected_median_denoise_ms": 512.32 + }, + "flux_image_t2i": { + "stages_ms": { + "InputValidationStage": 0.03, + "TextEncodingStage": 81.49, + "ConditioningStage": 0.01, + "TimestepPreparationStage": 2.43, + "LatentPreparationStage": 6.29, + "DenoisingStage": 8381.3, + "DecodingStage": 653.03 + }, + "denoise_step_ms": { + "0": 165.27, + "1": 58.88, + "2": 166.85, + "3": 166.51, + "4": 166.77, + "5": 167.55, + "6": 172.4, + "7": 167.77, + "8": 167.51, + "9": 167.22, + "10": 168.19, + "11": 167.74, + "12": 168.48, + "13": 168.08, + "14": 168.16, + "15": 167.15, + "16": 167.05, + "17": 169.27, + "18": 167.96, + "19": 167.74, + "20": 168.21, + "21": 167.07, + "22": 167.35, + "23": 167.06, + "24": 169.28, + "25": 169.41, + "26": 168.92, + "27": 167.59, + "28": 167.57, + "29": 170.42, + "30": 166.24, + "31": 168.33, + "32": 168.56, + "33": 168.62, + "34": 167.28, + "35": 167.12, + "36": 168.21, + "37": 168.78, + "38": 168.89, + "39": 167.74, + "40": 168.57, + "41": 167.89, + "42": 168.03, + "43": 167.61, + "44": 167.75, + "45": 168.03, + "46": 168.81, + "47": 168.29, + "48": 168.64, + "49": 168.78 + }, + "expected_e2e_ms": 9275.51, + "expected_avg_denoise_ms": 165.83, + "expected_median_denoise_ms": 169.33 + }, + "flux_2_image_t2i": { + "stages_ms": { + "InputValidationStage": 0.05, + "TextEncodingStage": 530.93, + "ImageVAEEncodingStage": 0.0, + "ConditioningStage": 0.02, + "LatentPreparationStage": 12.71, + "TimestepPreparationStage": 2.91, + "DenoisingStage": 26403.1, + "DecodingStage": 286.85 + }, + "denoise_step_ms": { + "0": 511.3, + "1": 132.57, + "2": 541.19, + "3": 518.93, + "4": 541.2, + "5": 520.28, + "6": 532.47, + "7": 525.68, + "8": 538.25, + "9": 525.84, + "10": 526.13, + "11": 525.67, + "12": 524.63, + "13": 530.57, + "14": 530.46, + "15": 529.94, + "16": 532.47, + "17": 527.88, + "18": 527.7, + "19": 525.08, + "20": 525.72, + "21": 529.3, + "22": 522.59, + "23": 529.75, + "24": 523.46, + "25": 528.72, + "26": 526.92, + "27": 528.62, + "28": 522.77, + "29": 528.35, + "30": 528.05, + "31": 528.89, + "32": 525.34, + "33": 530.36, + "34": 529.19, + "35": 526.92, + "36": 528.16, + "37": 525.03, + "38": 527.33, + "39": 527.96, + "40": 527.81, + "41": 524.79, + "42": 528.46, + "43": 532.49, + "44": 526.95, + "45": 533.14, + "46": 529.32, + "47": 528.51, + "48": 532.14, + "49": 529.29 + }, + "expected_e2e_ms": 27648.69, + "expected_avg_denoise_ms": 520.09, + "expected_median_denoise_ms": 528.0 + }, + "flux_2_ti2i": { + "stages_ms": { + "InputValidationStage": 99.82, + "TextEncodingStage": 519.88, + "ImageVAEEncodingStage": 254.56, + "ConditioningStage": 0.01, + "LatentPreparationStage": 12.4, + "TimestepPreparationStage": 2.71, + "DenoisingStage": 54705.41, + "DecodingStage": 311.13 + }, + "denoise_step_ms": { + "0": 1067.03, + "1": 271.58, + "2": 1073.07, + "3": 1071.93, + "4": 1100.0, + "5": 1102.28, + "6": 1088.3, + "7": 1089.09, + "8": 1086.95, + "9": 1089.33, + "10": 1089.28, + "11": 1096.51, + "12": 1098.88, + "13": 1080.84, + "14": 1098.44, + "15": 1100.88, + "16": 1086.83, + "17": 1090.58, + "18": 1096.35, + "19": 1086.25, + "20": 1082.71, + "21": 1097.6, + "22": 1098.72, + "23": 1100.9, + "24": 1099.02, + "25": 1101.52, + "26": 1098.75, + "27": 1101.41, + "28": 1091.75, + "29": 1087.2, + "30": 1101.33, + "31": 1098.14, + "32": 1100.14, + "33": 1098.91, + "34": 1100.05, + "35": 1099.12, + "36": 1100.22, + "37": 1103.29, + "38": 1092.79, + "39": 1086.59, + "40": 1094.81, + "41": 1105.6, + "42": 1100.54, + "43": 1099.95, + "44": 1096.5, + "45": 1086.69, + "46": 1095.85, + "47": 1092.85, + "48": 1086.17, + "49": 1099.67 + }, + "expected_e2e_ms": 56308.23, + "expected_avg_denoise_ms": 1077.26, + "expected_median_denoise_ms": 1096.5 + }, + "flux_image_t2i_2_gpus": { + "stages_ms": { + "InputValidationStage": 0.03, + "TextEncodingStage": 74.47, + "ConditioningStage": 0.01, + "TimestepPreparationStage": 2.23, + "LatentPreparationStage": 6.17, + "DenoisingStage": 8400.49, + "DecodingStage": 381.56 + }, + "denoise_step_ms": { + "0": 166.27, + "1": 59.6, + "2": 167.31, + "3": 168.7, + "4": 168.83, + "5": 171.05, + "6": 174.64, + "7": 170.92, + "8": 169.69, + "9": 169.21, + "10": 167.71, + "11": 177.62, + "12": 166.44, + "13": 174.61, + "14": 170.43, + "15": 169.47, + "16": 167.24, + "17": 169.15, + "18": 169.51, + "19": 172.3, + "20": 172.19, + "21": 172.36, + "22": 168.39, + "23": 168.47, + "24": 170.55, + "25": 170.96, + "26": 168.43, + "27": 169.01, + "28": 169.62, + "29": 170.95, + "30": 171.83, + "31": 171.92, + "32": 170.1, + "33": 170.46, + "34": 169.91, + "35": 168.91, + "36": 170.27, + "37": 170.23, + "38": 169.62, + "39": 169.66, + "40": 169.57, + "41": 169.42, + "42": 168.59, + "43": 171.12, + "44": 169.6, + "45": 169.93, + "46": 171.23, + "47": 171.03, + "48": 170.14, + "49": 169.4 + }, + "expected_e2e_ms": 9006.3, + "expected_avg_denoise_ms": 167.89, + "expected_median_denoise_ms": 169.67 + }, + "zimage_image_t2i": { + "stages_ms": { + "InputValidationStage": 0.03, + "TextEncodingStage": 104.21, + "ConditioningStage": 0.01, + "TimestepPreparationStage": 1.33, + "LatentPreparationStage": 1.13, + "DenoisingStage": 850.85, + "DecodingStage": 289.32 + }, + "denoise_step_ms": { + "0": 101.56, + "1": 28.26, + "2": 101.74, + "3": 101.68, + "4": 102.19, + "5": 102.05, + "6": 102.03, + "7": 102.28, + "8": 105.54 + }, + "expected_e2e_ms": 1383.47, + "expected_avg_denoise_ms": 94.15, + "expected_median_denoise_ms": 102.03 + }, + "qwen_image_edit_ti2i": { + "notes": "single uploaded reference image, Qwen/Qwen-Image-Edit", + "expected_e2e_ms": 138500.0, + "expected_avg_denoise_ms": 720.0, + "expected_median_denoise_ms": 718.0, + "stages_ms": { + "InputValidationStage": 23, + "ImageEncodingStage": 1485.0, + "ImageVAEEncodingStage": 350.0, + "ConditioningStage": 0.13, + "TimestepPreparationStage": 13.78, + "LatentPreparationStage": 15.0, + "DenoisingStage": 36000.0, + "DecodingStage": 850.0 + }, + "denoise_step_ms": { + "0": 720.0, + "1": 720.0, + "2": 720.0, + "3": 720.0, + "4": 720.0, + "5": 720.0, + "6": 720.0, + "7": 720.0, + "8": 720.0, + "9": 720.0, + "10": 720.0, + "11": 720.0, + "12": 720.0, + "13": 720.0, + "14": 720.0, + "15": 720.0, + "16": 720.0, + "17": 720.0, + "18": 720.0, + "19": 720.0, + "20": 720.0, + "21": 720.0, + "22": 720.0, + "23": 720.0, + "24": 720.0, + "25": 720.0, + "26": 720.0, + "27": 720.0, + "28": 720.0, + "29": 720.0, + "30": 720.0, + "31": 720.0, + "32": 720.0, + "33": 720.0, + "34": 720.0, + "35": 720.0, + "36": 720.0, + "37": 720.0, + "38": 720.0, + "39": 720.0, + "40": 720.0, + "41": 720.0, + "42": 720.0, + "43": 720.0, + "44": 720.0, + "45": 720.0, + "46": 720.0, + "47": 720.0, + "48": 720.0, + "49": 720.0 + } + }, + "wan2_1_t2v_1.3b": { + "stages_ms": { + "InputValidationStage": 0.06, + "TextEncodingStage": 3595.12, + "ConditioningStage": 0.02, + "TimestepPreparationStage": 2.39, + "LatentPreparationStage": 15.27, + "DenoisingStage": 91099.4, + "DecodingStage": 4330.65, + "per_frame_generation": null + }, + "denoise_step_ms": { + "0": 2918.67, + "1": 1784.23, + "2": 1797.72, + "3": 1798.8, + "4": 1798.19, + "5": 1799.27, + "6": 1798.54, + "7": 1798.67, + "8": 1798.76, + "9": 1798.34, + "10": 1799.22, + "11": 1798.61, + "12": 1799.4, + "13": 1799.04, + "14": 1797.41, + "15": 1799.05, + "16": 1798.32, + "17": 1799.12, + "18": 1799.56, + "19": 1797.01, + "20": 1798.28, + "21": 1799.06, + "22": 1800.05, + "23": 1797.76, + "24": 1798.16, + "25": 1798.62, + "26": 1798.64, + "27": 1799.44, + "28": 1798.79, + "29": 1798.13, + "30": 1797.47, + "31": 1799.4, + "32": 1798.77, + "33": 1799.47, + "34": 1798.49, + "35": 1796.51, + "36": 1799.68, + "37": 1799.24, + "38": 1798.49, + "39": 1799.66, + "40": 1797.04, + "41": 1799.58, + "42": 1797.35, + "43": 1798.07, + "44": 1798.6, + "45": 1798.95, + "46": 1799.51, + "47": 1798.25, + "48": 1799.04, + "49": 1798.34 + }, + "expected_e2e_ms": 99083.75, + "expected_avg_denoise_ms": 1820.02, + "expected_median_denoise_ms": 1798.65 + }, + "wan2_2_ti2v_5b": { + "stages_ms": { + "InputValidationStage": 96.27, + "TextEncodingStage": 2238.81, + "ConditioningStage": 0.02, + "TimestepPreparationStage": 2.39, + "LatentPreparationStage": 27.62, + "DenoisingStage": 134069.79, + "DecodingStage": 13559.79, + "per_frame_generation": null + }, + "denoise_step_ms": { + "0": 3181.0, + "1": 2561.67, + "2": 2578.49, + "3": 2582.1, + "4": 2572.24, + "5": 2577.72, + "6": 2581.35, + "7": 2578.79, + "8": 2584.98, + "9": 2588.49, + "10": 2594.37, + "11": 2591.19, + "12": 2591.32, + "13": 2595.35, + "14": 2594.35, + "15": 2595.62, + "16": 2596.35, + "17": 2596.11, + "18": 2597.24, + "19": 2603.13, + "20": 2599.9, + "21": 2601.48, + "22": 2603.58, + "23": 2601.13, + "24": 2600.47, + "25": 2604.13, + "26": 2606.04, + "27": 2605.3, + "28": 2602.02, + "29": 2601.83, + "30": 2603.57, + "31": 2606.63, + "32": 2606.1, + "33": 2602.24, + "34": 2603.29, + "35": 2602.34, + "36": 2602.16, + "37": 2608.14, + "38": 2603.48, + "39": 2601.7, + "40": 2603.96, + "41": 2604.58, + "42": 2606.67, + "43": 2603.52, + "44": 2599.88, + "45": 2598.66, + "46": 2600.74, + "47": 2602.31, + "48": 2608.4, + "49": 2606.02 + }, + "expected_e2e_ms": 150004.2, + "expected_avg_denoise_ms": 2608.84, + "expected_median_denoise_ms": 2601.59 + }, + "fastwan2_2_ti2v_5b": { + "stages_ms": { + "InputValidationStage": 88.86, + "TextEncodingStage": 2327.87, + "ConditioningStage": 0.01, + "TimestepPreparationStage": 58.66, + "LatentPreparationStage": 28.55, + "DmdDenoisingStage": 4438.3, + "DecodingStage": 14177.77, + "per_frame_generation": null + }, + "denoise_step_ms": { + "0": 2022.21, + "1": 1263.17, + "2": 1149.59 + }, + "expected_e2e_ms": 21133.36, + "expected_avg_denoise_ms": 1478.32, + "expected_median_denoise_ms": 1263.17 + }, + "fast_hunyuan_video": { + "stages_ms": { + "InputValidationStage": 0.09, + "TextEncodingStage": 845.64, + "ConditioningStage": 0.04, + "TimestepPreparationStage": 125.22, + "LatentPreparationStage": 29.34, + "DenoisingStage": 3860.64, + "DecodingStage": 2580.55 + }, + "denoise_step_ms": { + "0": 2063.08, + "1": 164.02, + "2": 406.99, + "3": 407.95, + "4": 407.51, + "5": 404.2 + }, + "expected_e2e_ms": 7487.87, + "expected_avg_denoise_ms": 642.29, + "expected_median_denoise_ms": 407.25 + }, + "wan2_2_i2v_a14b_2gpu": { + "stages_ms": { + "InputValidationStage": 59.33, + "TextEncodingStage": 6062.41, + "ConditioningStage": 0.02, + "TimestepPreparationStage": 2.2, + "LatentPreparationStage": 8.93, + "ImageVAEEncodingStage": 2075.47, + "DenoisingStage": 382628.41, + "DecodingStage": 2820.89 + }, + "denoise_step_ms": { + "0": 31228.27, + "1": 7723.86, + "2": 7769.69, + "3": 7795.93, + "4": 7815.58, + "5": 7829.48, + "6": 7827.34, + "7": 7825.35, + "8": 7828.05, + "9": 7809.53, + "10": 7801.29, + "11": 7790.96, + "12": 7785.88, + "13": 7785.5, + "14": 7780.32, + "15": 55411.1, + "16": 7722.27, + "17": 7761.31, + "18": 7789.46, + "19": 7800.6, + "20": 7814.91, + "21": 7799.62, + "22": 7801.25, + "23": 7798.27, + "24": 7797.67, + "25": 7795.97, + "26": 7781.74, + "27": 7784.16, + "28": 7796.64, + "29": 7789.75, + "30": 7792.13, + "31": 7790.99, + "32": 7778.1, + "33": 7777.78, + "34": 7780.56, + "35": 7778.22, + "36": 7770.88, + "37": 7771.56, + "38": 7767.82, + "39": 7769.23 + }, + "expected_e2e_ms": 393606.77, + "expected_avg_denoise_ms": 9565.48, + "expected_median_denoise_ms": 7790.98 + }, + "wan2_1_i2v_14b_480P_2gpu": { + "stages_ms": { + "InputValidationStage": 38.23, + "TextEncodingStage": 3550.36, + "ImageEncodingStage": 3462.55, + "ConditioningStage": 0.01, + "TimestepPreparationStage": 2.6, + "LatentPreparationStage": 9.73, + "ImageVAEEncodingStage": 2290.98, + "DenoisingStage": 415021.17, + "DecodingStage": 3016.1, + "per_frame_generation": null + }, + "denoise_step_ms": { + "0": 10200.25, + "1": 8222.39, + "2": 8279.38, + "3": 8301.48, + "4": 8338.87, + "5": 8352.39, + "6": 8354.64, + "7": 8353.64, + "8": 8315.58, + "9": 8308.48, + "10": 8299.65, + "11": 8292.7, + "12": 8292.73, + "13": 8285.21, + "14": 8276.06, + "15": 8270.41, + "16": 8273.04, + "17": 8266.04, + "18": 8267.7, + "19": 8264.06, + "20": 8259.32, + "21": 8257.26, + "22": 8253.02, + "23": 8251.77, + "24": 8260.97, + "25": 8251.39, + "26": 8237.43, + "27": 8241.33, + "28": 8235.96, + "29": 8240.6, + "30": 8232.48, + "31": 8237.85, + "32": 8244.3, + "33": 8236.79, + "34": 8239.83, + "35": 8239.89, + "36": 8239.12, + "37": 8246.74, + "38": 8235.67, + "39": 8242.77, + "40": 8241.17, + "41": 8240.24, + "42": 8237.01, + "43": 8231.26, + "44": 8232.85, + "45": 8226.56, + "46": 8236.98, + "47": 8226.73, + "48": 8220.49, + "49": 8217.04 + }, + "expected_e2e_ms": 426697.37, + "expected_avg_denoise_ms": 8300.19, + "expected_median_denoise_ms": 8267.01 + }, + "wan2_1_i2v_14b_720P_2gpu": { + "stages_ms": { + "InputValidationStage": 53.67, + "TextEncodingStage": 2838, + "ImageEncodingStage": 3123.99, + "ConditioningStage": 0.01, + "TimestepPreparationStage": 3.39, + "LatentPreparationStage": 8.41, + "ImageVAEEncodingStage": 2261.05, + "DenoisingStage": 417418.12, + "DecodingStage": 2968.35 + }, + "denoise_step_ms": { + "0": 11848.08, + "1": 8220.3, + "2": 8274.3, + "3": 8298.9, + "4": 8303.34, + "5": 8322.44, + "6": 8314.37, + "7": 8318.54, + "8": 8304.94, + "9": 8303.04, + "10": 8305.22, + "11": 8296.22, + "12": 8289.2, + "13": 8294.19, + "14": 8294.87, + "15": 8285.96, + "16": 8284.98, + "17": 8281.61, + "18": 8277.35, + "19": 8287.46, + "20": 8280.3, + "21": 8279.18, + "22": 8279.37, + "23": 8280.16, + "24": 8282.67, + "25": 8272.14, + "26": 8279.37, + "27": 8271.66, + "28": 8274.6, + "29": 8272.88, + "30": 8273.76, + "31": 8266.17, + "32": 8267.77, + "33": 8266.88, + "34": 8263.14, + "35": 8265.97, + "36": 8267.76, + "37": 8268.03, + "38": 8262.24, + "39": 8261.4, + "40": 8263.65, + "41": 8272.46, + "42": 8254.9, + "43": 8261.03, + "44": 8252.92, + "45": 8262.49, + "46": 8253.67, + "47": 8254.92, + "48": 8257.08, + "49": 8236.56 + }, + "expected_e2e_ms": 427536.9, + "expected_avg_denoise_ms": 8348.21, + "expected_median_denoise_ms": 8274.45 + }, + "wan2_2_t2v_a14b_2gpu": { + "stages_ms": { + "InputValidationStage": 0.07, + "TextEncodingStage": 2507.83, + "ConditioningStage": 0.02, + "TimestepPreparationStage": 3.22, + "LatentPreparationStage": 2.99, + "DenoisingStage": 103136.69, + "DecodingStage": 1431.71 + }, + "denoise_step_ms": { + "0": 24471.86, + "1": 757.31, + "2": 760.07, + "3": 758.74, + "4": 762.4, + "5": 755.83, + "6": 760.06, + "7": 756.38, + "8": 755.38, + "9": 754.25, + "10": 754.51, + "11": 753.46, + "12": 753.67, + "13": 753.08, + "14": 754.83, + "15": 753.04, + "16": 754.28, + "17": 754.45, + "18": 758.19, + "19": 756.23, + "20": 755.14, + "21": 755.92, + "22": 759.52, + "23": 762.09, + "24": 756.8, + "25": 758.86, + "26": 48787.27, + "27": 758.5, + "28": 757.57, + "29": 757.16, + "30": 758.43, + "31": 763.31, + "32": 753.69, + "33": 754.91, + "34": 752.03, + "35": 763.65, + "36": 760.96, + "37": 754.31, + "38": 753.64, + "39": 756.95 + }, + "expected_e2e_ms": 106895.63, + "expected_avg_denoise_ms": 2550.47, + "expected_median_denoise_ms": 756.59 + }, + "wan2_1_t2v_14b_2gpu": { + "stages_ms": { + "InputValidationStage": 0.05, + "TextEncodingStage": 2310.34, + "ConditioningStage": 0.02, + "TimestepPreparationStage": 2.42, + "LatentPreparationStage": 27.7, + "DenoisingStage": 803631.52, + "DecodingStage": 8898.74, + "per_frame_generation": null + }, + "denoise_step_ms": { + "0": 17347.88, + "1": 15956.93, + "2": 16027.54, + "3": 16054.15, + "4": 16081.46, + "5": 16062.7, + "6": 16058.56, + "7": 16057.58, + "8": 16061.04, + "9": 16120.97, + "10": 16036.84, + "11": 16019.6, + "12": 16042.29, + "13": 16039.87, + "14": 16063.0, + "15": 16036.16, + "16": 16079.82, + "17": 16019.7, + "18": 16061.5, + "19": 16039.95, + "20": 16009.42, + "21": 16051.01, + "22": 16039.31, + "23": 16048.22, + "24": 16071.41, + "25": 16078.75, + "26": 16061.78, + "27": 16018.39, + "28": 16041.44, + "29": 16039.64, + "30": 16041.89, + "31": 16039.6, + "32": 16038.97, + "33": 15999.48, + "34": 16019.93, + "35": 16040.27, + "36": 16020.3, + "37": 16039.38, + "38": 15999.4, + "39": 16022.15, + "40": 16042.32, + "41": 16016.62, + "42": 15998.92, + "43": 16041.48, + "44": 15999.63, + "45": 16003.21, + "46": 15995.91, + "47": 16023.52, + "48": 16016.64, + "49": 16019.6 + }, + "expected_e2e_ms": 814884.71, + "expected_avg_denoise_ms": 16062.92, + "expected_median_denoise_ms": 16039.62 + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_2_gpu_a.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_2_gpu_a.py new file mode 100644 index 00000000..3668f63e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_2_gpu_a.py @@ -0,0 +1,25 @@ +""" +2 GPU tests +""" + +from __future__ import annotations + +import pytest + +from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401 + DiffusionServerBase, + diffusion_server, +) +from sglang.multimodal_gen.test.server.testcase_configs import ( + TWO_GPU_CASES_A, + DiffusionTestCase, +) + + +class TestDiffusionServerTwoGpu(DiffusionServerBase): + """Performance tests for 2-GPU diffusion cases.""" + + @pytest.fixture(params=TWO_GPU_CASES_A, ids=lambda c: c.id) + def case(self, request) -> DiffusionTestCase: + """Provide a DiffusionTestCase for each 2-GPU test.""" + return request.param diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_2_gpu_b.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_2_gpu_b.py new file mode 100644 index 00000000..2c9b5cdc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_2_gpu_b.py @@ -0,0 +1,25 @@ +""" +2 GPU tests +""" + +from __future__ import annotations + +import pytest + +from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401 + DiffusionServerBase, + diffusion_server, +) +from sglang.multimodal_gen.test.server.testcase_configs import ( + TWO_GPU_CASES_B, + DiffusionTestCase, +) + + +class TestDiffusionServerTwoGpu(DiffusionServerBase): + """Performance tests for 2-GPU diffusion cases.""" + + @pytest.fixture(params=TWO_GPU_CASES_B, ids=lambda c: c.id) + def case(self, request) -> DiffusionTestCase: + """Provide a DiffusionTestCase for each 2-GPU test.""" + return request.param diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_a.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_a.py new file mode 100644 index 00000000..fdf072ec --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_a.py @@ -0,0 +1,31 @@ +""" +Config-driven diffusion performance test with pytest parametrization. + + +If the actual run is significantly better than the baseline, the improved cases with their updated baseline will be printed +""" + +from __future__ import annotations + +import pytest + +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401 + DiffusionServerBase, + diffusion_server, +) +from sglang.multimodal_gen.test.server.testcase_configs import ( + ONE_GPU_CASES_A, + DiffusionTestCase, +) + +logger = init_logger(__name__) + + +class TestDiffusionServerOneGpu(DiffusionServerBase): + """Performance tests for 1-GPU diffusion cases.""" + + @pytest.fixture(params=ONE_GPU_CASES_A, ids=lambda c: c.id) + def case(self, request) -> DiffusionTestCase: + """Provide a DiffusionTestCase for each 1-GPU test.""" + return request.param diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_b.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_b.py new file mode 100644 index 00000000..1a0432db --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_b.py @@ -0,0 +1,31 @@ +""" +Config-driven diffusion performance test with pytest parametrization. + + +If the actual run is significantly better than the baseline, the improved cases with their updated baseline will be printed +""" + +from __future__ import annotations + +import pytest + +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401 + DiffusionServerBase, + diffusion_server, +) +from sglang.multimodal_gen.test.server.testcase_configs import ( + ONE_GPU_CASES_B, + DiffusionTestCase, +) + +logger = init_logger(__name__) + + +class TestDiffusionServerOneGpu(DiffusionServerBase): + """Performance tests for 1-GPU diffusion cases.""" + + @pytest.fixture(params=ONE_GPU_CASES_B, ids=lambda c: c.id) + def case(self, request) -> DiffusionTestCase: + """Provide a DiffusionTestCase for each 1-GPU test.""" + return request.param diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_common.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_common.py new file mode 100644 index 00000000..18b56f5d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_common.py @@ -0,0 +1,384 @@ +""" +Config-driven diffusion performance test with pytest parametrization. + + +If the actual run is significantly better than the baseline, the improved cases with their updated baseline will be printed +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any, Callable + +import openai +import pytest +from openai import OpenAI + +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.runtime.utils.perf_logger import RequestPerfRecord +from sglang.multimodal_gen.test.server.conftest import _GLOBAL_PERF_RESULTS +from sglang.multimodal_gen.test.server.test_server_utils import ( + VALIDATOR_REGISTRY, + PerformanceValidator, + ServerContext, + ServerManager, + WarmupRunner, + download_image_from_url, + get_generate_fn, +) +from sglang.multimodal_gen.test.server.testcase_configs import ( + BASELINE_CONFIG, + DiffusionTestCase, + PerformanceSummary, + ScenarioConfig, +) +from sglang.multimodal_gen.test.test_utils import ( + get_dynamic_server_port, + is_image_url, + read_perf_logs, + wait_for_req_perf_record, +) + +logger = init_logger(__name__) + + +@pytest.fixture +def diffusion_server(case: DiffusionTestCase) -> ServerContext: + """Start a diffusion server for a single case and tear it down afterwards.""" + default_port = get_dynamic_server_port() + port = int(os.environ.get("SGLANG_TEST_SERVER_PORT", default_port)) + server_args = case.server_args + sampling_params = case.sampling_params + extra_args = os.environ.get("SGLANG_TEST_SERVE_ARGS", "") + extra_args += ( + f" --num-gpus {server_args.num_gpus} --ulysses-degree {server_args.num_gpus}" + ) + + # start server + manager = ServerManager( + model=server_args.model_path, + port=port, + wait_deadline=float(os.environ.get("SGLANG_TEST_WAIT_SECS", "1200")), + extra_args=extra_args, + ) + ctx = manager.start() + + try: + # Reconstruct output size for OpenAI API + output_size = sampling_params.output_size + warmup = WarmupRunner( + port=ctx.port, + model=server_args.model_path, + prompt=sampling_params.prompt or "A colorful raccoon icon", + output_size=output_size, + ) + warmup.run_text_warmups(server_args.warmup_text) + + if ( + case.server_args.warmup_edit > 0 + and case.sampling_params.prompt + and sampling_params.image_path + ): + # Handle URL or local path + if is_image_url(sampling_params.image_path): + image_path = download_image_from_url(str(sampling_params.image_path)) + else: + image_path = Path(sampling_params.image_path) + + warmup.run_edit_warmups( + count=server_args.warmup_edit, + edit_prompt=sampling_params.prompt, + image_path=image_path, + ) + except Exception as exc: + logger.error("Warm-up failed for %s: %s", case.id, exc) + ctx.cleanup() + raise + + try: + yield ctx + finally: + ctx.cleanup() + + +class DiffusionServerBase: + """Performance tests for all diffusion models/scenarios. + + This single test class runs against all cases defined in ONE_GPU_CASES. + Each case gets its own server instance via the parametrized fixture. + """ + + _perf_results: list[dict[str, Any]] = [] + _improved_baselines: list[dict[str, Any]] = [] + + @classmethod + def setup_class(cls): + cls._perf_results = [] + cls._improved_baselines = [] + + @classmethod + def teardown_class(cls): + for result in cls._perf_results: + result["class_name"] = cls.__name__ + _GLOBAL_PERF_RESULTS.append(result) + + if cls._improved_baselines: + import json + + output = """ +--- POTENTIAL BASELINE IMPROVEMENTS DETECTED --- +The following test cases performed significantly better than their baselines. +Consider updating perf_baselines.json with the snippets below: +""" + for item in cls._improved_baselines: + output += ( + f'\n"{item["id"]}": {json.dumps(item["baseline"], indent=4)},\n' + ) + print(output) + + def _client(self, ctx: ServerContext) -> OpenAI: + """Get OpenAI client for the server.""" + return OpenAI( + api_key="sglang-anything", + base_url=f"http://localhost:{ctx.port}/v1", + ) + + def run_and_collect( + self, + ctx: ServerContext, + case_id: str, + generate_fn: Callable[[str, openai.Client], str], + ) -> RequestPerfRecord: + """Run generation and collect performance records.""" + log_path = ctx.perf_log_path + prev_len = len(read_perf_logs(log_path)) + log_wait_timeout = 30 + + client = self._client(ctx) + rid = generate_fn(case_id, client) + + req_perf_record, _ = wait_for_req_perf_record( + rid, + prev_len, + log_path, + timeout=log_wait_timeout, + ) + + return req_perf_record + + def _validate_and_record( + self, + case: DiffusionTestCase, + perf_record: RequestPerfRecord, + ) -> None: + """Validate metrics and record results.""" + is_baseline_generation_mode = os.environ.get("SGLANG_GEN_BASELINE", "0") == "1" + + scenario = BASELINE_CONFIG.scenarios.get(case.id) + missing_scenario = False + if scenario is None: + # Create dummy scenario to allow metric collection + scenario = type( + "DummyScenario", + (), + { + "expected_e2e_ms": 0, + "expected_avg_denoise_ms": 0, + "expected_median_denoise_ms": 0, + "stages_ms": {}, + "denoise_step_ms": {}, + }, + )() + if not is_baseline_generation_mode: + missing_scenario = True + + validator_name = case.server_args.custom_validator or "default" + validator_class = VALIDATOR_REGISTRY.get(validator_name, PerformanceValidator) + + validator = validator_class( + scenario=scenario, + tolerances=BASELINE_CONFIG.tolerances, + step_fractions=BASELINE_CONFIG.step_fractions, + ) + + summary = validator.collect_metrics(perf_record) + + if is_baseline_generation_mode or missing_scenario: + self._dump_baseline_for_testcase(case, summary, missing_scenario) + if missing_scenario: + pytest.fail(f"Testcase '{case.id}' not found in perf_baselines.json") + return + + self._check_for_improvement(case, summary, scenario) + + try: + validator.validate(perf_record, case.sampling_params.num_frames) + except AssertionError as e: + logger.error(f"Performance validation failed for {case.id}:\n{e}") + self._dump_baseline_for_testcase(case, summary, missing_scenario) + raise + + result = { + "test_name": case.id, + "modality": case.server_args.modality, + "e2e_ms": summary.e2e_ms, + "avg_denoise_ms": summary.avg_denoise_ms, + "median_denoise_ms": summary.median_denoise_ms, + "stage_metrics": summary.stage_metrics, + "sampled_steps": summary.sampled_steps, + } + + # video-specific metrics + if summary.frames_per_second: + result.update( + { + "frames_per_second": summary.frames_per_second, + "total_frames": summary.total_frames, + "avg_frame_time_ms": summary.avg_frame_time_ms, + } + ) + + self.__class__._perf_results.append(result) + + def _check_for_improvement( + self, + case: DiffusionTestCase, + summary: PerformanceSummary, + scenario: "ScenarioConfig", + ) -> None: + """Check for potential significant performance improvements and record them.""" + is_improved = False + threshold = BASELINE_CONFIG.improvement_threshold + + def is_sig_faster(actual, expected): + if expected == 0 or expected is None: + return False + return actual < expected * (1 - threshold) + + def safe_get_metric(metric_dict, key): + val = metric_dict.get(key) + return val if val is not None else float("inf") + + # Check for any significant improvement + if ( + is_sig_faster(summary.e2e_ms, scenario.expected_e2e_ms) + or is_sig_faster(summary.avg_denoise_ms, scenario.expected_avg_denoise_ms) + or is_sig_faster( + summary.median_denoise_ms, scenario.expected_median_denoise_ms + ) + ): + is_improved = True + # Combine metrics, always taking the better (lower) value + new_stages = { + stage: min( + safe_get_metric(summary.stage_metrics, stage), + safe_get_metric(scenario.stages_ms, stage), + ) + for stage in set(summary.stage_metrics) | set(scenario.stages_ms) + } + new_denoise_steps = { + step: min( + safe_get_metric(summary.all_denoise_steps, step), + safe_get_metric(scenario.denoise_step_ms, step), + ) + for step in set(summary.all_denoise_steps.keys()) + | set(scenario.denoise_step_ms) + } + + # Check for stage-level improvements + if not is_improved: + for stage, new_val in new_stages.items(): + if is_sig_faster(new_val, scenario.stages_ms.get(stage, float("inf"))): + is_improved = True + break + if not is_improved: + for step, new_val in new_denoise_steps.items(): + if is_sig_faster( + new_val, scenario.denoise_step_ms.get(step, float("inf")) + ): + is_improved = True + break + + if is_improved: + new_baseline = { + "stages_ms": {k: round(v, 2) for k, v in new_stages.items()}, + "denoise_step_ms": { + str(k): round(v, 2) for k, v in new_denoise_steps.items() + }, + "expected_e2e_ms": round( + min(summary.e2e_ms, scenario.expected_e2e_ms), 2 + ), + "expected_avg_denoise_ms": round( + min(summary.avg_denoise_ms, scenario.expected_avg_denoise_ms), 2 + ), + "expected_median_denoise_ms": round( + min(summary.median_denoise_ms, scenario.expected_median_denoise_ms), + 2, + ), + } + self._improved_baselines.append({"id": case.id, "baseline": new_baseline}) + + def _dump_baseline_for_testcase( + self, + case: DiffusionTestCase, + summary: "PerformanceSummary", + missing_scenario: bool = False, + ) -> None: + """Dump performance metrics as a JSON scenario for baselines.""" + import json + + denoise_steps_formatted = { + str(k): round(v, 2) for k, v in summary.all_denoise_steps.items() + } + stages_formatted = {k: round(v, 2) for k, v in summary.stage_metrics.items()} + + baseline = { + "stages_ms": stages_formatted, + "denoise_step_ms": denoise_steps_formatted, + "expected_e2e_ms": round(summary.e2e_ms, 2), + "expected_avg_denoise_ms": round(summary.avg_denoise_ms, 2), + "expected_median_denoise_ms": round(summary.median_denoise_ms, 2), + } + + # Video-specific metrics + if case.server_args.modality == "video": + if "per_frame_generation" not in baseline["stages_ms"]: + baseline["stages_ms"]["per_frame_generation"] = ( + round(summary.avg_frame_time_ms, 2) + if summary.avg_frame_time_ms + else None + ) + action = "add" if missing_scenario else "update" + output = f""" +{action} this baseline in the "scenarios" section of perf_baselines.json: + +"{case.id}": {json.dumps(baseline, indent=4)} + +""" + logger.error(output) + + def test_diffusion_perf( + self, + case: DiffusionTestCase, + diffusion_server: ServerContext, + ): + """Single parametrized test that runs for all cases. + + Pytest will execute this test once per case in ONE_GPU_CASES, + with test IDs like: + - test_diffusion_perf[qwen_image_text] + - test_diffusion_perf[qwen_image_edit] + - etc. + """ + generate_fn = get_generate_fn( + model_path=case.server_args.model_path, + modality=case.server_args.modality, + sampling_params=case.sampling_params, + ) + perf_record = self.run_and_collect( + diffusion_server, + case.id, + generate_fn, + ) + self._validate_and_record(case, perf_record) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_utils.py new file mode 100644 index 00000000..4e3e788a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/test_server_utils.py @@ -0,0 +1,696 @@ +""" +Server management and performance validation for diffusion tests. +""" + +from __future__ import annotations + +import base64 +import os +import shlex +import subprocess +import sys +import tempfile +import threading +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Sequence +from urllib.request import urlopen + +import pytest +from openai import Client, OpenAI + +from sglang.multimodal_gen.benchmarks.compare_perf import calculate_upper_bound +from sglang.multimodal_gen.runtime.utils.common import kill_process_tree +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.runtime.utils.perf_logger import RequestPerfRecord +from sglang.multimodal_gen.test.server.testcase_configs import ( + DiffusionSamplingParams, + PerformanceSummary, + ScenarioConfig, + ToleranceConfig, +) +from sglang.multimodal_gen.test.slack_utils import upload_file_to_slack +from sglang.multimodal_gen.test.test_utils import ( + is_image_url, + prepare_perf_log, + validate_image, + validate_openai_video, +) + +logger = init_logger(__name__) + + +def download_image_from_url(url: str) -> Path: + """Download an image from a URL to a temporary file. + + Args: + url: The URL of the image to download + + Returns: + Path to the downloaded temporary file + """ + logger.info(f"Downloading image from URL: {url}") + + # Determine file extension from URL + ext = ".jpg" # default + if url.lower().endswith((".png", ".jpeg", ".jpg", ".webp", ".gif")): + ext = url[url.rfind(".") :] + + # Create temporary file + temp_file = ( + Path(tempfile.gettempdir()) / f"diffusion_test_image_{int(time.time())}{ext}" + ) + + try: + with urlopen(url, timeout=30) as response: + temp_file.write_bytes(response.read()) + logger.info(f"Downloaded image to: {temp_file}") + return temp_file + except Exception as e: + logger.error(f"Failed to download image from {url}: {e}") + raise + + +@dataclass +class ServerContext: + """Context for a running diffusion server.""" + + port: int + process: subprocess.Popen + model: str + stdout_file: Path + perf_log_path: Path + log_dir: Path + _stdout_fh: Any = field(repr=False) + _log_thread: threading.Thread | None = field(default=None, repr=False) + + def cleanup(self) -> None: + """Clean up server resources.""" + try: + kill_process_tree(self.process.pid) + except Exception: + pass + try: + self._stdout_fh.flush() + self._stdout_fh.close() + except Exception: + pass + + +class ServerManager: + """Manages diffusion server lifecycle.""" + + def __init__( + self, + model: str, + port: int, + wait_deadline: float = 1200.0, + extra_args: str = "", + ): + self.model = model + self.port = port + self.wait_deadline = wait_deadline + self.extra_args = extra_args + + def start(self) -> ServerContext: + """Start the diffusion server and wait for readiness.""" + log_dir, perf_log_path = prepare_perf_log() + + safe_model_name = self.model.replace("/", "_") + stdout_path = ( + Path(tempfile.gettempdir()) + / f"sgl_server_{self.port}_{safe_model_name}.log" + ) + stdout_path.unlink(missing_ok=True) + + command = [ + "sglang", + "serve", + "--model-path", + self.model, + "--port", + str(self.port), + "--log-level=debug", + ] + if self.extra_args.strip(): + command.extend(self.extra_args.strip().split()) + + env = os.environ.copy() + env["SGLANG_DIFFUSION_STAGE_LOGGING"] = "1" + env["SGLANG_PERF_LOG_DIR"] = log_dir.as_posix() + + # TODO: unify with run_command + logger.info(f"Running command: {shlex.join(command)}") + + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + env=env, + ) + + log_thread = None + stdout_fh = stdout_path.open("w", encoding="utf-8", buffering=1) + if process.stdout: + + def _log_pipe(pipe: Any, file: Any) -> None: + """Read from pipe and write to file and stdout.""" + try: + with pipe: + for line in iter(pipe.readline, ""): + sys.stdout.write(line) + file.write(line) + file.flush() + except Exception as e: + logger.error("Log pipe thread error: %s", e) + finally: + file.close() + logger.debug("Log pipe thread finished.") + + log_thread = threading.Thread( + target=_log_pipe, args=(process.stdout, stdout_fh) + ) + log_thread.daemon = True + log_thread.start() + + logger.info( + "[server-test] Starting server pid=%s, model=%s, log=%s", + process.pid, + self.model, + stdout_path, + ) + + self._wait_for_ready(process, stdout_path) + + return ServerContext( + port=self.port, + process=process, + model=self.model, + stdout_file=stdout_path, + perf_log_path=perf_log_path, + log_dir=log_dir, + _stdout_fh=stdout_fh, + _log_thread=log_thread, + ) + + def _wait_for_ready(self, process: subprocess.Popen, stdout_path: Path) -> None: + """Wait for server to become ready.""" + start = time.time() + ready_message = "Application startup complete." + + while time.time() - start < self.wait_deadline: + if process.poll() is not None: + tail = self._get_log_tail(stdout_path) + raise RuntimeError( + f"Server exited early (code {process.returncode}).\n{tail}" + ) + + if stdout_path.exists(): + try: + content = stdout_path.read_text(encoding="utf-8", errors="ignore") + if ready_message in content: + logger.info("[server-test] Server ready") + return + except Exception as e: + logger.debug("Could not read log yet: %s", e) + + elapsed = int(time.time() - start) + logger.info("[server-test] Waiting for server... elapsed=%ss", elapsed) + time.sleep(5) + + tail = self._get_log_tail(stdout_path) + raise TimeoutError(f"Server not ready within {self.wait_deadline}s.\n{tail}") + + @staticmethod + def _get_log_tail(path: Path, lines: int = 200) -> str: + """Get the last N lines from a log file.""" + try: + content = path.read_text(encoding="utf-8", errors="ignore") + return "\n".join(content.splitlines()[-lines:]) + except Exception: + return "" + + +class WarmupRunner: + """Handles warmup requests for a server.""" + + def __init__( + self, + port: int, + model: str, + prompt: str, + output_size: str, + ): + self.client = OpenAI( + api_key="sglang-anything", + base_url=f"http://localhost:{port}/v1", + ) + self.model = model + self.prompt = prompt + self.output_size = output_size + + def run_text_warmups(self, count: int) -> None: + """Run text-to-image warmup requests.""" + if count <= 0: + return + + logger.info("[server-test] Running %s text warm-up(s)", count) + for _ in range(count): + result = self.client.images.generate( + model=self.model, + prompt=self.prompt, + n=1, + size=self.output_size, + response_format="b64_json", + ) + validate_image(result.data[0].b64_json) + + def run_edit_warmups( + self, + count: int, + edit_prompt: str, + image_path: Path, + ) -> None: + """Run image-edit warmup requests.""" + if count <= 0: + return + + if not image_path.exists(): + logger.warning( + "[server-test] Skipping edit warmup: image missing at %s", image_path + ) + return + + logger.info("[server-test] Running %s edit warm-up(s)", count) + for _ in range(count): + with image_path.open("rb") as fh: + result = self.client.images.edit( + model=self.model, + image=fh, + prompt=edit_prompt, + n=1, + size=self.output_size, + response_format="b64_json", + ) + validate_image(result.data[0].b64_json) + + +class PerformanceValidator: + """Validates performance metrics against expectations.""" + + is_video_gen: bool = False + + def __init__( + self, + scenario: ScenarioConfig, + tolerances: ToleranceConfig, + step_fractions: Sequence[float], + ): + self.scenario = scenario + self.tolerances = tolerances + self.step_fractions = step_fractions + self.is_baseline_generation_mode = ( + os.environ.get("SGLANG_GEN_BASELINE", "0") == "1" + ) + + def _assert_le( + self, + name: str, + actual: float, + expected: float, + tolerance: float, + min_abs_tolerance_ms: float = 20.0, + ): + """Assert that actual is less than or equal to expected within a tolerance. + + Uses the larger of relative tolerance or absolute tolerance to prevent + flaky failures on very fast operations. + """ + upper_bound = calculate_upper_bound(expected, tolerance, min_abs_tolerance_ms) + assert actual <= upper_bound, ( + f"Validation failed for '{name}'.\n" + f" Actual: {actual:.4f}ms\n" + f" Expected: {expected:.4f}ms\n" + f" Limit: {upper_bound:.4f}ms " + f"(rel_tol: {tolerance:.1%}, abs_pad: {min_abs_tolerance_ms}ms)" + ) + + def validate( + self, perf_record: RequestPerfRecord, *args, **kwargs + ) -> PerformanceSummary: + """Validate all performance metrics and return summary.""" + summary = self.collect_metrics(perf_record) + if self.is_baseline_generation_mode: + return summary + + self._validate_e2e(summary) + self._validate_denoise_agg(summary) + self._validate_denoise_steps(summary) + self._validate_stages(summary) + + return summary + + def collect_metrics( + self, + perf_record: RequestPerfRecord, + ) -> PerformanceSummary: + return PerformanceSummary.from_req_perf_record(perf_record, self.step_fractions) + + def _validate_e2e(self, summary: PerformanceSummary) -> None: + """Validate end-to-end performance.""" + assert summary.e2e_ms > 0, "E2E duration missing" + self._assert_le( + "E2E Latency", + summary.e2e_ms, + self.scenario.expected_e2e_ms, + self.tolerances.e2e, + ) + + def _validate_denoise_agg(self, summary: PerformanceSummary) -> None: + """Validate aggregate denoising metrics.""" + assert summary.avg_denoise_ms > 0, "Denoising step timings missing" + + self._assert_le( + "Average Denoise Step", + summary.avg_denoise_ms, + self.scenario.expected_avg_denoise_ms, + self.tolerances.denoise_agg, + ) + self._assert_le( + "Median Denoise Step", + summary.median_denoise_ms, + self.scenario.expected_median_denoise_ms, + self.tolerances.denoise_agg, + ) + + def _validate_denoise_steps(self, summary: PerformanceSummary) -> None: + """Validate individual denoising steps.""" + for idx, actual in summary.sampled_steps.items(): + expected = self.scenario.denoise_step_ms.get(idx) + if expected is None: + continue + # FIXME: hardcode, looser for first step + tolerance = 0.4 if idx == 0 else self.tolerances.denoise_step + + self._assert_le( + f"Denoise Step {idx}", + actual, + expected, + tolerance, + ) + + def _validate_stages(self, summary: PerformanceSummary) -> None: + """Validate stage-level metrics.""" + assert summary.stage_metrics, "Stage metrics missing" + + for stage, expected in self.scenario.stages_ms.items(): + if stage == "per_frame_generation" and self.is_video_gen: + continue + actual = summary.stage_metrics.get(stage) + assert actual is not None, f"Stage {stage} timing missing" + tolerance = ( + self.tolerances.denoise_stage + if stage == "DenoisingStage" + else self.tolerances.non_denoise_stage + ) + self._assert_le( + f"Stage '{stage}'", + actual, + expected, + tolerance, + min_abs_tolerance_ms=120.0, # relax absolute tolerance for non-denoising stages + ) + + +class VideoPerformanceValidator(PerformanceValidator): + """Extended validator for video diffusion with frame-level metrics.""" + + is_video_gen = True + + def validate( + self, + perf_record: RequestPerfRecord, + num_frames: int | None = None, + ) -> PerformanceSummary: + """Validate video metrics including frame generation rates.""" + summary = super().validate(perf_record) + + if num_frames and summary.e2e_ms > 0: + summary.total_frames = num_frames + summary.avg_frame_time_ms = summary.e2e_ms / num_frames + summary.frames_per_second = 1000.0 / summary.avg_frame_time_ms + + if not self.is_baseline_generation_mode: + self._validate_frame_rate(summary) + + return summary + + def _validate_frame_rate(self, summary: PerformanceSummary) -> None: + """Validate frame generation performance.""" + expected_frame_time = self.scenario.stages_ms.get("per_frame_generation") + if expected_frame_time and summary.avg_frame_time_ms: + self._assert_le( + "Average Frame Time", + summary.avg_frame_time_ms, + expected_frame_time, + self.tolerances.denoise_stage, + ) + + +# Registry of validators by name +VALIDATOR_REGISTRY = { + "default": PerformanceValidator, + "video": VideoPerformanceValidator, +} + + +def get_generate_fn( + model_path: str, + modality: str, + sampling_params: DiffusionSamplingParams, +) -> Callable[[str, Client], str]: + """Return appropriate generation function for the case.""" + + def _create_and_download_video( + client, + case_id, + *, + model: str, + size: str, + prompt: str | None = None, + seconds: int | None = None, + input_reference: Any | None = None, + ) -> str: + """ + Create a video job via /v1/videos, poll until completion, + then download the binary content and validate it. + """ + + create_kwargs: dict[str, Any] = { + "model": model, + "size": size, + } + if prompt is not None: + create_kwargs["prompt"] = prompt + if seconds is not None: + create_kwargs["seconds"] = seconds + if input_reference is not None: + create_kwargs["input_reference"] = input_reference # triggers multipart + + job = client.videos.create(**create_kwargs) # type: ignore[attr-defined] + video_id = job.id + + job_completed = False + is_baseline_generation_mode = os.environ.get("SGLANG_GEN_BASELINE", "0") == "1" + timeout = 3600.0 if is_baseline_generation_mode else 1200.0 + deadline = time.time() + timeout + while True: + page = client.videos.list() # type: ignore[attr-defined] + item = next((v for v in page.data if v.id == video_id), None) + + if item and getattr(item, "status", None) == "completed": + job_completed = True + break + + if time.time() > deadline: + break + + time.sleep(1) + + if not job_completed: + if is_baseline_generation_mode: + logger.warning( + f"{id}: video job {video_id} timed out during baseline generation. " + "Attempting to collect performance data anyway." + ) + return video_id + + pytest.fail(f"{id}: video job {video_id} did not complete in time") + + # download video + resp = client.videos.download_content(video_id=video_id) # type: ignore[attr-defined] + content = resp.read() + validate_openai_video(content) + + tmp_path = f"{video_id}.mp4" + with open(tmp_path, "wb") as f: + f.write(content) + upload_file_to_slack( + case_id=case_id, + model=model_path, + prompt=sampling_params.prompt, + file_path=tmp_path, + origin_file_path=sampling_params.image_path, + ) + os.remove(tmp_path) + + return video_id + + video_seconds = sampling_params.seconds or 4 + + def generate_image(case_id, client) -> str: + """T2I: Text to Image generation.""" + if not sampling_params.prompt: + pytest.skip(f"{id}: no text prompt configured") + + response = client.images.with_raw_response.generate( + model=model_path, + prompt=sampling_params.prompt, + n=1, + size=sampling_params.output_size, + response_format="b64_json", + ) + result = response.parse() + validate_image(result.data[0].b64_json) + + img_data = base64.b64decode(result.data[0].b64_json) + tmp_path = f"{result.created}.png" + with open(tmp_path, "wb") as f: + f.write(img_data) + upload_file_to_slack( + case_id=case_id, + model=model_path, + prompt=sampling_params.prompt, + file_path=tmp_path, + ) + os.remove(tmp_path) + + return str(result.created) + + def generate_image_edit(case_id, client) -> str: + """TI2I: Text + Image ? Image edit.""" + if not sampling_params.prompt or not sampling_params.image_path: + pytest.skip(f"{id}: no edit config") + + if is_image_url(sampling_params.image_path): + image_path = download_image_from_url(str(sampling_params.image_path)) + else: + image_path = Path(sampling_params.image_path) + if not image_path.exists(): + pytest.skip(f"{id}: file missing: {image_path}") + + with image_path.open("rb") as fh: + response = client.images.with_raw_response.edit( + model=model_path, + image=fh, + prompt=sampling_params.prompt, + n=1, + size=sampling_params.output_size, + response_format="b64_json", + ) + rid = response.headers.get("x-request-id", "") + + result = response.parse() + validate_image(result.data[0].b64_json) + + img_data = base64.b64decode(result.data[0].b64_json) + tmp_path = f"{rid}.png" + with open(tmp_path, "wb") as f: + f.write(img_data) + upload_file_to_slack( + case_id=case_id, + model=model_path, + prompt=sampling_params.prompt, + file_path=tmp_path, + origin_file_path=sampling_params.image_path, + ) + os.remove(tmp_path) + + return rid + + def generate_video(case_id, client) -> str: + """T2V: Text ? Video.""" + if not sampling_params.prompt: + pytest.skip(f"{id}: no text prompt configured") + + return _create_and_download_video( + client, + case_id, + model=model_path, + prompt=sampling_params.prompt, + size=sampling_params.output_size, + seconds=video_seconds, + ) + + def generate_image_to_video(case_id, client) -> str: + """I2V: Image ? Video (optional prompt).""" + if not sampling_params.image_path: + pytest.skip(f"{id}: no input image configured") + + if is_image_url(sampling_params.image_path): + image_path = download_image_from_url(str(sampling_params.image_path)) + else: + image_path = Path(sampling_params.image_path) + if not image_path.exists(): + pytest.skip(f"{id}: file missing: {image_path}") + + with image_path.open("rb") as fh: + return _create_and_download_video( + client, + case_id, + model=model_path, + prompt=sampling_params.prompt, + size=sampling_params.output_size, + seconds=video_seconds, + input_reference=fh, + ) + + def generate_text_image_to_video(case_id, client) -> str: + """TI2V: Text + Image ? Video.""" + if not sampling_params.prompt or not sampling_params.image_path: + pytest.skip(f"{id}: no edit config") + + if is_image_url(sampling_params.image_path): + image_path = download_image_from_url(str(sampling_params.image_path)) + else: + image_path = Path(sampling_params.image_path) + if not image_path.exists(): + pytest.skip(f"{id}: file missing: {image_path}") + + with image_path.open("rb") as fh: + return _create_and_download_video( + client, + case_id, + model=model_path, + prompt=sampling_params.prompt, + size=sampling_params.output_size, + seconds=video_seconds, + input_reference=fh, + ) + + if modality == "video": + if sampling_params.image_path and sampling_params.prompt: + fn = generate_text_image_to_video + elif sampling_params.image_path: + fn = generate_image_to_video + else: + fn = generate_video + elif sampling_params.prompt and sampling_params.image_path: + fn = generate_image_edit + else: + fn = generate_image + + return fn diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/testcase_configs.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/testcase_configs.py new file mode 100644 index 00000000..75022e0b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/server/testcase_configs.py @@ -0,0 +1,484 @@ +""" +Configuration and data structures for diffusion performance tests. + +Usage: + +pytest python/sglang/multimodal_gen/test/server/test_server_a.py +# for a single testcase, look for the name of the testcases in DIFFUSION_CASES +pytest python/sglang/multimodal_gen/test/server/test_server_a.py -k qwen_image_t2i + + +To add a new testcase: +1. add your testcase with case-id: `my_new_test_case_id` to DIFFUSION_CASES +2. run `SGLANG_GEN_BASELINE=1 pytest -s python/sglang/multimodal_gen/test/server/test_server_a.py -k my_new_test_case_id` +3. insert or override the corresponding scenario in `scenarios` section of perf_baselines.json with the output baseline of step-2 + + +""" + +from __future__ import annotations + +import json +import os +import statistics +from dataclasses import dataclass +from pathlib import Path +from typing import Sequence + +from sglang.multimodal_gen.runtime.utils.perf_logger import RequestPerfRecord + + +@dataclass +class ToleranceConfig: + """Tolerance ratios for performance validation.""" + + e2e: float + denoise_stage: float + non_denoise_stage: float + denoise_step: float + denoise_agg: float + + +@dataclass +class ScenarioConfig: + """Expected performance metrics for a test scenario.""" + + stages_ms: dict[str, float] + denoise_step_ms: dict[int, float] + expected_e2e_ms: float + expected_avg_denoise_ms: float + expected_median_denoise_ms: float + + +@dataclass +class BaselineConfig: + """Full baseline configuration.""" + + scenarios: dict[str, ScenarioConfig] + step_fractions: Sequence[float] + warmup_defaults: dict[str, int] + tolerances: ToleranceConfig + improvement_threshold: float + + @classmethod + def load(cls, path: Path) -> BaselineConfig: + """Load baseline configuration from JSON file.""" + with path.open("r", encoding="utf-8") as fh: + data = json.load(fh) + + tol_data = data["tolerances"] + tolerances = ToleranceConfig( + e2e=float(os.getenv("SGLANG_E2E_TOLERANCE", tol_data["e2e"])), + denoise_stage=float( + os.getenv("SGLANG_STAGE_TIME_TOLERANCE", tol_data["denoise_stage"]) + ), + non_denoise_stage=float( + os.getenv( + "SGLANG_NON_DENOISE_STAGE_TIME_TOLERANCE", + tol_data["non_denoise_stage"], + ) + ), + denoise_step=float( + os.getenv("SGLANG_DENOISE_STEP_TOLERANCE", tol_data["denoise_step"]) + ), + denoise_agg=float( + os.getenv("SGLANG_DENOISE_AGG_TOLERANCE", tol_data["denoise_agg"]) + ), + ) + + scenarios = {} + for name, cfg in data["scenarios"].items(): + scenarios[name] = ScenarioConfig( + stages_ms=cfg["stages_ms"], + denoise_step_ms={int(k): v for k, v in cfg["denoise_step_ms"].items()}, + expected_e2e_ms=float(cfg["expected_e2e_ms"]), + expected_avg_denoise_ms=float(cfg["expected_avg_denoise_ms"]), + expected_median_denoise_ms=float(cfg["expected_median_denoise_ms"]), + ) + + return cls( + scenarios=scenarios, + step_fractions=tuple(data["sampling"]["step_fractions"]), + warmup_defaults=data["sampling"].get("warmup_requests", {}), + tolerances=tolerances, + improvement_threshold=data.get("improvement_reporting", {}).get( + "threshold", 0.2 + ), + ) + + +@dataclass(frozen=True) +class DiffusionServerArgs: + """Configuration for a single model/scenario test case.""" + + model_path: str # HF repo or local path + modality: str = "image" # "image" or "video" or "3d" + + warmup_text: int = 1 # number of text-to-image/video warmups + warmup_edit: int = 0 # number of image/video-edit warmups + custom_validator: str | None = None # optional custom validator name + # resources + num_gpus: int = 1 + + +@dataclass(frozen=True) +class DiffusionSamplingParams: + """Configuration for a single model/scenario test case.""" + + output_size: str = "" + + # inputs and conditioning + prompt: str | None = None # text prompt for generation + image_path: Path | str | None = None # input image/video for editing (Path or URL) + + # duration + seconds: int = 1 # for video: duration in seconds + num_frames: int | None = None # for video: number of frames + fps: int | None = None # for video: frames per second + + +@dataclass(frozen=True) +class DiffusionTestCase: + """Configuration for a single model/scenario test case.""" + + id: str # pytest test id and scenario name + server_args: DiffusionServerArgs + sampling_params: DiffusionSamplingParams + + +def sample_step_indices( + step_map: dict[int, float], fractions: Sequence[float] +) -> list[int]: + if not step_map: + return [] + max_idx = max(step_map.keys()) + indices = set() + for fraction in fractions: + idx = min(max_idx, max(0, int(round(fraction * max_idx)))) + if idx in step_map: + indices.add(idx) + return sorted(indices) + + +@dataclass +class PerformanceSummary: + """Summary of performance of a request, built from RequestPerfRecord""" + + e2e_ms: float + avg_denoise_ms: float + median_denoise_ms: float + # { "stage_1": time_1, "stage_2": time_2 } + stage_metrics: dict[str, float] + step_metrics: list[float] + sampled_steps: dict[int, float] + all_denoise_steps: dict[int, float] + frames_per_second: float | None = None + total_frames: int | None = None + avg_frame_time_ms: float | None = None + + @staticmethod + def from_req_perf_record( + record: RequestPerfRecord, step_fractions: Sequence[float] + ): + """Collect all performance metrics into a summary without validation.""" + e2e_ms = record.total_duration_ms + + step_durations = record.steps + avg_denoise = 0.0 + median_denoise = 0.0 + if step_durations: + avg_denoise = sum(step_durations) / len(step_durations) + median_denoise = statistics.median(step_durations) + + per_step = {index: s for index, s in enumerate(step_durations)} + sample_indices = sample_step_indices(per_step, step_fractions) + sampled_steps = {idx: per_step[idx] for idx in sample_indices} + + # convert from list to dict + stage_metrics = {} + for item in record.stages: + if isinstance(item, dict) and "name" in item: + val = item.get("execution_time_ms", 0.0) + stage_metrics[item["name"]] = val + + return PerformanceSummary( + e2e_ms=e2e_ms, + avg_denoise_ms=avg_denoise, + median_denoise_ms=median_denoise, + stage_metrics=stage_metrics, + step_metrics=step_durations, + sampled_steps=sampled_steps, + all_denoise_steps=per_step, + ) + + +# All test cases with clean default values +# To test different models, simply add more DiffusionCase entries +ONE_GPU_CASES_A: list[DiffusionTestCase] = [ + # === Text to Image (T2I) === + DiffusionTestCase( + "qwen_image_t2i", + DiffusionServerArgs( + model_path="Qwen/Qwen-Image", + modality="image", + warmup_text=1, + warmup_edit=0, + ), + DiffusionSamplingParams( + prompt="A futuristic cityscape at sunset with flying cars", + output_size="1024x1024", + ), + ), + DiffusionTestCase( + "flux_image_t2i", + DiffusionServerArgs( + model_path="black-forest-labs/FLUX.1-dev", + modality="image", + warmup_text=1, + warmup_edit=0, + ), + DiffusionSamplingParams( + prompt="A futuristic cityscape at sunset with flying cars", + output_size="1024x1024", + ), + ), + DiffusionTestCase( + "flux_2_image_t2i", + DiffusionServerArgs( + model_path="black-forest-labs/FLUX.2-dev", + modality="image", + warmup_text=1, + warmup_edit=0, + ), + DiffusionSamplingParams( + prompt="A futuristic cityscape at sunset with flying cars", + output_size="1024x1024", + ), + ), + DiffusionTestCase( + "zimage_image_t2i", + DiffusionServerArgs( + model_path="Tongyi-MAI/Z-Image-Turbo", + modality="image", + warmup_text=1, + warmup_edit=0, + ), + DiffusionSamplingParams( + prompt="Doraemon is eating dorayaki.", + output_size="1024x1024", + ), + ), + # === Text and Image to Image (TI2I) === + # TODO: Timeout with Torch2.9. Add back when it can pass CI + # DiffusionTestCase( + # id="qwen_image_edit_ti2i", + # model_path="Qwen/Qwen-Image-Edit", + # modality="image", + # prompt=None, # not used for editing + # output_size="1024x1536", + # warmup_text=0, + # warmup_edit=1, + # edit_prompt="Convert 2D style to 3D style", + # image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg", + # ), +] + +ONE_GPU_CASES_B: list[DiffusionTestCase] = [ + # === Text to Video (T2V) === + DiffusionTestCase( + "wan2_1_t2v_1.3b", + DiffusionServerArgs( + model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + modality="video", + warmup_text=0, + warmup_edit=0, + custom_validator="video", + ), + DiffusionSamplingParams( + prompt="A curious raccoon", + output_size="848x480", + ), + ), + # NOTE(mick): flaky + # DiffusionTestCase( + # id="hunyuan_video", + # model_path="hunyuanvideo-community/HunyuanVideo", + # modality="video", + # prompt="A curious raccoon", + # output_size="720x480", + # warmup_text=0, + # warmup_edit=0, + # custom_validator="video", + # ), + DiffusionTestCase( + "flux_2_ti2i", + DiffusionServerArgs( + model_path="black-forest-labs/FLUX.2-dev", + modality="image", + warmup_text=0, + warmup_edit=1, + ), + DiffusionSamplingParams( + prompt="Convert 2D style to 3D style", + image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg", + ), + ), + DiffusionTestCase( + "fast_hunyuan_video", + DiffusionServerArgs( + model_path="FastVideo/FastHunyuan-diffusers", + modality="video", + warmup_text=0, + warmup_edit=0, + custom_validator="video", + ), + DiffusionSamplingParams( + prompt="A curious raccoon", + output_size="720x480", + ), + ), + # === Text and Image to Video (TI2V) === + DiffusionTestCase( + "wan2_2_ti2v_5b", + DiffusionServerArgs( + model_path="Wan-AI/Wan2.2-TI2V-5B-Diffusers", + modality="video", + warmup_text=0, + warmup_edit=0, + custom_validator="video", + ), + DiffusionSamplingParams( + output_size="832x1104", + prompt="Add dynamic motion to the scene", + image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg", + ), + ), + DiffusionTestCase( + "fastwan2_2_ti2v_5b", + DiffusionServerArgs( + model_path="FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers", + modality="video", + warmup_text=0, + warmup_edit=0, + custom_validator="video", + ), + DiffusionSamplingParams( + output_size="832x1104", + prompt="Add dynamic motion to the scene", + image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg", + ), + ), +] + +TWO_GPU_CASES_A = [ + # TODO: Timeout with Torch2.9. Add back when it can pass CI + # DiffusionTestCase( + # id="wan2_2_i2v_a14b_2gpu", + # model_path="Wan-AI/Wan2.2-I2V-A14B-Diffusers", + # modality="video", + # prompt="generate", + # warmup_text=0, + # warmup_edit=0, + # output_size="832x1104", + # edit_prompt="generate", + # image_path="https://github.com/Wan-Video/Wan2.2/blob/990af50de458c19590c245151197326e208d7191/examples/i2v_input.JPG?raw=true", + # custom_validator="video", + # num_gpus=2, + # num_frames=1, + # ), + DiffusionTestCase( + "wan2_2_t2v_a14b_2gpu", + DiffusionServerArgs( + model_path="Wan-AI/Wan2.2-T2V-A14B-Diffusers", + modality="video", + warmup_text=0, + warmup_edit=0, + custom_validator="video", + num_gpus=2, + ), + DiffusionSamplingParams( + prompt="A curious raccoon", + output_size="720x480", + ), + ), + DiffusionTestCase( + "wan2_1_t2v_14b_2gpu", + DiffusionServerArgs( + model_path="Wan-AI/Wan2.1-T2V-14B-Diffusers", + warmup_text=0, + warmup_edit=0, + modality="video", + num_gpus=2, + custom_validator="video", + ), + DiffusionSamplingParams( + prompt="A curious raccoon", + output_size="720x480", + ), + ), +] + +TWO_GPU_CASES_B = [ + DiffusionTestCase( + "wan2_1_i2v_14b_480P_2gpu", + DiffusionServerArgs( + model_path="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers", + warmup_text=0, + warmup_edit=0, + modality="video", + custom_validator="video", + num_gpus=2, + ), + DiffusionSamplingParams( + output_size="832x1104", + prompt="Add dynamic motion to the scene", + image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg", + ), + ), + DiffusionTestCase( + "wan2_1_i2v_14b_720P_2gpu", + DiffusionServerArgs( + model_path="Wan-AI/Wan2.1-I2V-14B-720P-Diffusers", + modality="video", + warmup_text=0, + warmup_edit=0, + custom_validator="video", + num_gpus=2, + ), + DiffusionSamplingParams( + prompt="Add dynamic motion to the scene", + image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg", + output_size="832x1104", + ), + ), + DiffusionTestCase( + "qwen_image_t2i_2_gpus", + DiffusionServerArgs( + model_path="Qwen/Qwen-Image", + modality="image", + warmup_text=1, + warmup_edit=0, + num_gpus=2, + ), + DiffusionSamplingParams( + prompt="A futuristic cityscape at sunset with flying cars", + output_size="1024x1024", + ), + ), + DiffusionTestCase( + "flux_image_t2i_2_gpus", + DiffusionServerArgs( + model_path="black-forest-labs/FLUX.1-dev", + modality="image", + warmup_text=1, + warmup_edit=0, + ), + DiffusionSamplingParams( + prompt="A futuristic cityscape at sunset with flying cars", + output_size="1024x1024", + ), + ), +] + +# Load global configuration +BASELINE_CONFIG = BaselineConfig.load(Path(__file__).with_name("perf_baselines.json")) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/slack_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/slack_utils.py new file mode 100644 index 00000000..7237c891 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/slack_utils.py @@ -0,0 +1,193 @@ +""" + This file upload the media generated in diffusion-nightly-test to a slack channel of SGLang +""" + +import logging +import os +import tempfile +from datetime import datetime +from urllib.parse import urlparse +from urllib.request import urlopen + +from sglang.multimodal_gen.runtime.utils.perf_logger import get_git_commit_hash + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +import inspect + +try: + import sglang.multimodal_gen.test.server.testcase_configs as configs + from sglang.multimodal_gen.test.server.testcase_configs import DiffusionTestCase + + ALL_CASES = [] + for name, value in inspect.getmembers(configs): + if name.endswith("_CASES") or "_CASES_" in name: + if ( + isinstance(value, list) + and len(value) > 0 + and isinstance(value[0], DiffusionTestCase) + ): + ALL_CASES.extend(value) + elif isinstance(value, list) and len(value) == 0: + # Assume empty list with matching name is a valid case list container + pass + + # Deduplicate cases by ID + seen_ids = set() + unique_cases = [] + for c in ALL_CASES: + if c.id not in seen_ids: + seen_ids.add(c.id) + unique_cases.append(c) + ALL_CASES = unique_cases + +except Exception as e: + logger.warning(f"Failed to import test cases: {e}") + ALL_CASES = [] + + +def _get_status_message(run_id, current_case_id, thread_messages=None): + date_str = datetime.now().strftime("%d/%m") + base_header = f""""*🧵 for nightly test of {date_str}* +*Git Revision:* {get_git_commit_hash()} +*GitHub Run ID:* {run_id} +*Total Tasks:* {len(ALL_CASES)} + +""" + + if not ALL_CASES: + return base_header + + default_emoji_for_case_in_progress = "⏳" + status_map = {c.id: default_emoji_for_case_in_progress for c in ALL_CASES} + + if thread_messages: + for msg in thread_messages: + text = msg.get("text", "") + # Look for case_id in the message (format: *Case ID:* `case_id`) + for c in ALL_CASES: + if f"*Case ID:* `{c.id}`" in text: + status_map[c.id] = "✅" + + if current_case_id: + status_map[current_case_id] = "✅" + + lines = [base_header, "", "*Tasks Status:*"] + + # Calculate padding + max_len = max(len(c.id) for c in ALL_CASES) if ALL_CASES else 10 + max_len = max(max_len, len("Case ID")) + + # Build markdown table inside a code block + table_lines = ["```"] + table_lines.append(f"| {'Case ID'.ljust(max_len)} | Status |") + table_lines.append(f"| {'-' * max_len} | :----: |") + + for c in ALL_CASES: + mark = status_map.get(c.id, default_emoji_for_case_in_progress) + table_lines.append(f"| {c.id.ljust(max_len)} | {mark} |") + + table_lines.append("```") + + lines.extend(table_lines) + + return "\n".join(lines) + + +def upload_file_to_slack( + case_id: str = None, + model: str = None, + prompt: str = None, + file_path: str = None, + origin_file_path: str = None, +) -> bool: + temp_path = None + try: + from slack_sdk import WebClient + + run_id = os.getenv("GITHUB_RUN_ID", "local") + + token = os.environ.get("SGLANG_DIFFUSION_SLACK_TOKEN") + if not token: + logger.info(f"Slack upload failed: no token") + return False + + if not file_path or not os.path.exists(file_path): + logger.info(f"Slack upload failed: no file path") + return False + + if origin_file_path and origin_file_path.startswith(("http", "https")): + suffix = os.path.splitext(urlparse(origin_file_path).path)[1] or ".tmp" + with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tf: + with urlopen(origin_file_path) as response: + tf.write(response.read()) + temp_path = tf.name + origin_file_path = temp_path + + uploads = [{"file": file_path, "title": "Generated Image"}] + if origin_file_path and os.path.exists(origin_file_path): + uploads.insert(0, {"file": origin_file_path, "title": "Original Image"}) + + message = ( + f"*Case ID:* `{case_id}`\n" f"*Model:* `{model}`\n" f"*Prompt:* {prompt}" + ) + + client = WebClient(token=token) + channel_id = "C0A02NDF7UY" + thread_ts = None + + parent_msg_text = None + try: + history = client.conversations_history(channel=channel_id, limit=100) + for msg in history.get("messages", []): + if f"*GitHub Run ID:* {run_id}" in msg.get("text", ""): + # Use thread_ts if it exists (msg is a reply), otherwise use ts (msg is a parent) + thread_ts = msg.get("thread_ts") or msg.get("ts") + parent_msg_text = msg.get("text", "") + logger.info(f"Found thread_ts: {thread_ts}") + break + except Exception as e: + logger.warning(f"Failed to search slack history: {e}") + + if not thread_ts: + try: + text = _get_status_message(run_id, case_id) + response = client.chat_postMessage(channel=channel_id, text=text) + thread_ts = response["ts"] + except Exception as e: + logger.warning(f"Failed to create parent thread: {e}") + + # Upload first to ensure it's in history + client.files_upload_v2( + channel=channel_id, + file_uploads=uploads, + initial_comment=message, + thread_ts=thread_ts, + ) + + # Then update status based on thread replies + if thread_ts: + try: + replies = client.conversations_replies( + channel=channel_id, ts=thread_ts, limit=200 + ) + messages = replies.get("messages", []) + new_text = _get_status_message(run_id, case_id, messages) + + # Only update if changed significantly (ignoring timestamp diffs if any) + # But here we just check text content + if new_text != parent_msg_text: + client.chat_update(channel=channel_id, ts=thread_ts, text=new_text) + except Exception as e: + logger.warning(f"Failed to update parent message: {e}") + + logger.info(f"File uploaded successfully: {os.path.basename(file_path)}") + return True + + except Exception as e: + logger.info(f"Slack upload failed: {e}") + return False + finally: + if temp_path and os.path.exists(temp_path): + os.remove(temp_path) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/test_files/launch_flux.json b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/test_files/launch_flux.json new file mode 100644 index 00000000..6a9d8382 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/test_files/launch_flux.json @@ -0,0 +1,11 @@ +{ + "model_path": "black-forest-labs/FLUX.1-dev", + "prompt": "A beautiful woman in a red dress walking down a street", + "text_encoder_cpu_offload": true, + "pin_cpu_memory": true, + "save_output": true, + "width": 720, + "height": 720, + "output_path": "outputs", + "output_file_name": "FLUX.1-dev, single gpu" +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/test_files/launch_wan.json b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/test_files/launch_wan.json new file mode 100644 index 00000000..eeb9ddf9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/test_files/launch_wan.json @@ -0,0 +1,11 @@ +{ + "model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + "prompt": "A beautiful woman in a red dress walking down a street", + "text_encoder_cpu_offload": true, + "pin_cpu_memory": true, + "save_output": true, + "width": 720, + "height": 720, + "output_path": "outputs", + "output_file_name": "Wan2.1-T2V-1.3B-Diffusers, single gpu" +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/test_offline_api.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/test_offline_api.py new file mode 100644 index 00000000..9c45e271 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/test_offline_api.py @@ -0,0 +1,75 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +""" + Testing the performance of generate command of sgl_diffusion' CLI +""" + +import unittest + +import torch + +from sglang.multimodal_gen.runtime.entrypoints.diffusion_generator import DiffGenerator +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class TestGeneratorAPIBase(unittest.TestCase): + # server args + server_kwargs = {} + + # sampling + output_path: str = "test_outputs" + + results = [] + + @classmethod + def setUpClass(cls): + cls.results = [] + + def verify_single_generation_result(self, result): + self.assertIsNotNone(result, "Generation failed") + self.assertTrue( + "samples" in result and isinstance(result["samples"], torch.Tensor), + f"Incorrect Generation result", + ) + + def _run_test(self, name, server_kwargs, test_key: str): + generator = DiffGenerator.from_pretrained(**server_kwargs) + result = generator.generate(prompt="A curious raccoon") + self.verify_single_generation_result(result) + + def test_single_gpu(self): + self._run_test( + name=self.server_kwargs["model_path"], + server_kwargs=self.server_kwargs | dict(num_gpus=1), + test_key="test_single_gpu", + ) + + def test_cfg_parallel(self): + self._run_test( + name=self.server_kwargs["model_path"], + server_kwargs=self.server_kwargs + | dict(num_gpus=2, enable_cfg_parallel=True), + test_key="test_cfg_parallel", + ) + + def test_multiple_prompts(self): + generator = DiffGenerator.from_pretrained( + **self.server_kwargs | dict(num_gpus=2, enable_cfg_parallel=True) + ) + prompts = ["A curious raccoon", "A curious cat"] + results = generator.generate(prompt=prompts) + + self.assertEqual(len(results), len(prompts), "Some generation tasks fail") + for result in results: + self.verify_single_generation_result(result) + + +class TestWan2_1_T2V(TestGeneratorAPIBase): + server_kwargs = {"model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers"} + + +if __name__ == "__main__": + del TestGeneratorAPIBase + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/test_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/test_utils.py new file mode 100644 index 00000000..7af786c4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/test/test_utils.py @@ -0,0 +1,420 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo +import base64 +import dataclasses +import json +import os +import shlex +import socket +import subprocess +import sys +import time +import unittest +from pathlib import Path +from typing import Optional + +from PIL import Image + +from sglang.multimodal_gen.configs.sample.sampling_params import DataType +from sglang.multimodal_gen.runtime.utils.common import get_bool_env_var +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.runtime.utils.perf_logger import ( + RequestPerfRecord, + get_diffusion_perf_log_dir, +) + +logger = init_logger(__name__) + + +def is_image_url(image_path: str | Path | None) -> bool: + """Check if image_path is a URL.""" + if image_path is None: + return False + return isinstance(image_path, str) and ( + image_path.startswith("http://") or image_path.startswith("https://") + ) + + +def run_command(command) -> Optional[float]: + """Runs a command and returns the execution time and status.""" + print(f"Running command: {shlex.join(command)}") + + duration = None + with subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + ) as process: + for line in process.stdout: + sys.stdout.write(line) + if "Pixel data generated" in line: + words = line.split(" ") + duration = float(words[-2]) + + if process.returncode == 0: + return duration + else: + print(f"Command failed with exit code {process.returncode}") + return None + + +def probe_port(host="127.0.0.1", port=30010, timeout=2.0) -> bool: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.settimeout(timeout) + try: + s.connect((host, port)) + return True + except OSError: + return False + + +def is_in_ci() -> bool: + return get_bool_env_var("SGLANG_IS_IN_CI") + + +def get_dynamic_server_port() -> int: + cuda_devices = os.environ.get("CUDA_VISIBLE_DEVICES", "0") + if not cuda_devices: + cuda_devices = "0" + try: + first_device_id = int(cuda_devices.split(",")[0].strip()[0]) + except (ValueError, IndexError): + first_device_id = 0 + + if is_in_ci(): + base_port = 10000 + first_device_id * 2000 + else: + base_port = 20000 + first_device_id * 1000 + + return base_port + 1000 + + +def is_mp4(data): + idx = data.find(b"ftyp") + return 0 <= idx <= 32 + + +def is_jpeg(data: bytes) -> bool: + # JPEG files start with: FF D8 FF + return data.startswith(b"\xff\xd8\xff") + + +def is_png(data): + # PNG files start with: 89 50 4E 47 0D 0A 1A 0A + return data.startswith(b"\x89PNG\r\n\x1a\n") + + +def wait_for_port(host="127.0.0.1", port=30010, deadline=300.0, interval=0.5): + end = time.time() + deadline + last_err = None + while time.time() < end: + if probe_port(host, port, timeout=interval): + return True + time.sleep(interval) + raise TimeoutError(f"Port {host}:{port} not ready. Last error: {last_err}") + + +def check_image_size(ut, image, width, height): + # check image size + ut.assertEqual(image.size, (width, height)) + + +def get_perf_log_dir() -> Path: + """Gets the performance log directory from the centralized sglang utility.""" + log_dir_str = get_diffusion_perf_log_dir() + if not log_dir_str: + raise RuntimeError( + "Performance logging is disabled (SGLANG_PERF_LOG_DIR is empty), " + "but a test tried to access the log directory." + ) + return Path(log_dir_str) + + +def _ensure_log_path(log_dir: Path) -> Path: + log_dir.mkdir(parents=True, exist_ok=True) + return log_dir / "performance.log" + + +def clear_perf_log(log_dir: Path) -> Path: + """Delete the perf log file so tests can watch for fresh entries.""" + log_path = _ensure_log_path(log_dir) + if log_path.exists(): + log_path.unlink() + logger.info("[server-test] Monitoring perf log at %s", log_path.as_posix()) + return log_path + + +def prepare_perf_log() -> tuple[Path, Path]: + """Convenience helper to resolve and clear the perf log in one call.""" + log_dir = get_perf_log_dir() + log_path = clear_perf_log(log_dir) + return log_dir, log_path + + +def read_perf_logs(log_path: Path) -> list[RequestPerfRecord]: + if not log_path.exists(): + return [] + records: list[RequestPerfRecord] = [] + with log_path.open("r", encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + record_dict = json.loads(line) + records.append(RequestPerfRecord(**record_dict)) + except json.JSONDecodeError: + continue + return records + + +def wait_for_req_perf_record( + request_id: str, + prev_len: int, + log_path: Path, + timeout: float = 30.0, +) -> tuple[RequestPerfRecord | None, int]: + """ + the stage metrics of this request should be in the performance_log file with {request-id} + """ + logger.info(f"Waiting for req perf record with request id: {request_id}") + deadline = time.time() + timeout + while time.time() < deadline: + records = read_perf_logs(log_path) + if len(records) >= prev_len + 1: + # FIXME: unable to get rid from openai apis, this is a hack. we should compare rid + # potential error when there are multiple servers + return records[-1], len(records) + + time.sleep(0.5) + + if os.environ.get("SGLANG_GEN_BASELINE", "0") == "1": + records = read_perf_logs(log_path) + return None, len(records) + + logger.error(f"record: {records}") + raise AssertionError(f"Timeout waiting for stage metrics for request {request_id} ") + + +def validate_image(b64_json: str) -> None: + """Decode and validate that image is PNG or JPEG.""" + image_bytes = base64.b64decode(b64_json) + assert is_png(image_bytes) or is_jpeg(image_bytes), "Image must be PNG or JPEG" + + +def validate_video(b64_json: str) -> None: + """Decode and validate that video is a valid format.""" + video_bytes = base64.b64decode(b64_json) + is_mp4 = ( + video_bytes[:4] == b"\x00\x00\x00\x18" or video_bytes[:4] == b"\x00\x00\x00\x1c" + ) + is_webm = video_bytes[:4] == b"\x1a\x45\xdf\xa3" + assert is_mp4 or is_webm, "Video must be MP4 or WebM" + + +def validate_openai_video(video_bytes: bytes) -> None: + """Validate that video is MP4 or WebM by magic bytes.""" + is_mp4 = ( + video_bytes.startswith(b"\x00\x00\x00\x18") + or video_bytes.startswith(b"\x00\x00\x00\x1c") + or video_bytes[4:8] == b"ftyp" + ) + is_webm = video_bytes.startswith(b"\x1a\x45\xdf\xa3") + assert is_mp4 or is_webm, "Video must be MP4 or WebM" + + +@dataclasses.dataclass +class TestResult: + name: str + key: str + duration: Optional[float] + succeed: bool + + @property + def duration_str(self): + return f"{self.duration:.4f}" if self.duration else "NA" + + +class TestCLIBase(unittest.TestCase): + model_path: str = None + extra_args = [] + data_type: DataType = None + # tested on h100 + thresholds = {} + + width: int = 720 + height: int = 720 + output_path: str = "test_outputs" + + base_command = [ + "sglang", + "generate", + "--text-encoder-cpu-offload", + "--pin-cpu-memory", + "--prompt", + "A curious raccoon", + "--save-output", + "--log-level=debug", + f"--width={width}", + f"--height={height}", + f"--output-path={output_path}", + ] + + results = [] + + @classmethod + def setUpClass(cls): + cls.results = [] + + def _run_command(self, name: str, model_path: str, test_key: str = "", args=[]): + command = ( + self.base_command + + [f"--model-path={model_path}"] + + shlex.split(args or "") + + ["--output-file-name", f"{name}"] + + self.extra_args + ) + duration = run_command(command) + status = "Success" if duration else "Failed" + succeed = duration is not None + + duration = float(duration) if succeed else None + self.results.append(TestResult(name, test_key, duration, succeed)) + + return name, duration, status + + +class TestGenerateBase(TestCLIBase): + model_path: str = None + extra_args = [] + data_type: DataType = None + # tested on h100 + thresholds = {} + + width: int = 720 + height: int = 720 + output_path: str = "test_outputs" + image_path: str | None = None + prompt: str | None = "A curious raccoon" + + base_command = [ + "sglang", + "generate", + # "--text-encoder-cpu-offload", + # "--pin-cpu-memory", + f"--prompt", + f"{prompt}", + "--save-output", + "--log-level=debug", + f"--width={width}", + f"--height={height}", + f"--output-path={output_path}", + ] + + results: list[TestResult] = [] + + @classmethod + def setUpClass(cls): + cls.results = [] + + @classmethod + def tearDownClass(cls): + # Print markdown table + print("\n## Test Results\n") + print("| Test Case | Duration | Status |") + print("|--------------------------------|----------|---------|") + test_keys = ["test_single_gpu", "test_cfg_parallel", "test_usp", "test_mixed"] + test_key_to_order = { + test_key: order for order, test_key in enumerate(test_keys) + } + + ordered_results: list[TestResult] = [None] * len(test_keys) + for result in cls.results: + order = test_key_to_order[result.key] + ordered_results[order] = result + + for result in ordered_results: + if not result: + continue + status = ( + "Succeed" + if ( + result.succeed + and float(result.duration) <= float(cls.thresholds[result.key]) + ) + else "Failed" + ) + print(f"| {result.name:<30} | {result.duration_str:<8} | {status:<7} |") + print() + durations = [result.duration_str for result in cls.results] + print(" | ".join([""] + durations + [""])) + + def _run_test(self, name: str, args, model_path: str, test_key: str): + time_threshold = self.thresholds[test_key] + name, duration, status = self._run_command( + name, args=args, model_path=model_path, test_key=test_key + ) + self.verify(status, name, duration, time_threshold) + + def verify(self, status, name, duration, time_threshold): + print("-" * 80) + print("\n" * 3) + + # test task status + self.assertEqual(status, "Success", f"{name} command failed") + self.assertIsNotNone(duration, f"Could not parse duration for {name}") + self.assertLessEqual( + duration, + time_threshold, + f"{name} failed with {duration:.4f}s > {time_threshold}s", + ) + + # test output file + path = os.path.join( + self.output_path, f"{name}.{self.data_type.get_default_extension()}" + ) + self.assertTrue(os.path.exists(path), f"Output file not exist for {path}") + if self.data_type == DataType.IMAGE: + with Image.open(path) as image: + check_image_size(self, image, self.width, self.height) + logger.info(f"{name} passed in {duration:.4f}s (threshold: {time_threshold}s)") + + def model_name(self): + return self.model_path.split("/")[-1] + + def test_single_gpu(self): + """single gpu""" + self._run_test( + name=f"{self.model_name()}_single_gpu", + args=None, + model_path=self.model_path, + test_key="test_single_gpu", + ) + + def test_cfg_parallel(self): + """cfg parallel""" + self._run_test( + name=f"{self.model_name()}_cfg_parallel", + args="--num-gpus 2 --enable-cfg-parallel", + model_path=self.model_path, + test_key="test_cfg_parallel", + ) + + def test_usp(self): + """usp""" + self._run_test( + name=f"{self.model_name()}_usp", + args="--num-gpus 4 --ulysses-degree=2 --ring-degree=2", + model_path=self.model_path, + test_key="test_usp", + ) + + def test_mixed(self): + """mixed""" + self._run_test( + name=f"{self.model_name()}_mixed", + args="--num-gpus 4 --ulysses-degree=2 --ring-degree=1 --enable-cfg-parallel", + model_path=self.model_path, + test_key="test_mixed", + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/third_party/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/third_party/__init__.py new file mode 100644 index 00000000..af2eb7d1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/third_party/__init__.py @@ -0,0 +1 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/third_party/pynvml.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/third_party/pynvml.py new file mode 100644 index 00000000..546dc8b8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/third_party/pynvml.py @@ -0,0 +1,7227 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# copied from https://pypi.org/project/nvidia-ml-py +# version 12.570.86 + +##### +# Copyright (c) 2011-2023, NVIDIA Corporation. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the NVIDIA Corporation nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +##### + +import os +import string +import sys +import threading + +## +# Python bindings for the NVML library +## +from ctypes import * +from functools import wraps + +## C Type mappings ## +## Enums +_nvmlEnableState_t = c_uint +NVML_FEATURE_DISABLED = 0 +NVML_FEATURE_ENABLED = 1 + +_nvmlBrandType_t = c_uint +NVML_BRAND_UNKNOWN = 0 +NVML_BRAND_QUADRO = 1 +NVML_BRAND_TESLA = 2 +NVML_BRAND_NVS = 3 +NVML_BRAND_GRID = ( + 4 # Deprecated from API reporting. Keeping definition for backward compatibility. +) +NVML_BRAND_GEFORCE = 5 +NVML_BRAND_TITAN = 6 +NVML_BRAND_NVIDIA_VAPPS = 7 # NVIDIA Virtual Applications +NVML_BRAND_NVIDIA_VPC = 8 # NVIDIA Virtual PC +NVML_BRAND_NVIDIA_VCS = 9 # NVIDIA Virtual Compute Server +NVML_BRAND_NVIDIA_VWS = 10 # NVIDIA RTX Virtual Workstation +NVML_BRAND_NVIDIA_CLOUD_GAMING = 11 # NVIDIA Cloud Gaming +NVML_BRAND_NVIDIA_VGAMING = NVML_BRAND_NVIDIA_CLOUD_GAMING # Deprecated from API reporting. Keeping definition for backward compatibility. +NVML_BRAND_QUADRO_RTX = 12 +NVML_BRAND_NVIDIA_RTX = 13 +NVML_BRAND_NVIDIA = 14 +NVML_BRAND_GEFORCE_RTX = 15 # Unused +NVML_BRAND_TITAN_RTX = 16 # Unused +NVML_BRAND_COUNT = 17 + +_nvmlTemperatureThresholds_t = c_uint +NVML_TEMPERATURE_THRESHOLD_SHUTDOWN = 0 +NVML_TEMPERATURE_THRESHOLD_SLOWDOWN = 1 +NVML_TEMPERATURE_THRESHOLD_MEM_MAX = 2 +NVML_TEMPERATURE_THRESHOLD_GPU_MAX = 3 +NVML_TEMPERATURE_THRESHOLD_ACOUSTIC_MIN = 4 +NVML_TEMPERATURE_THRESHOLD_ACOUSTIC_CURR = 5 +NVML_TEMPERATURE_THRESHOLD_ACOUSTIC_MAX = 6 +NVML_TEMPERATURE_THRESHOLD_GPS_CURR = 7 +NVML_TEMPERATURE_THRESHOLD_COUNT = 8 + +_nvmlTemperatureSensors_t = c_uint +NVML_TEMPERATURE_GPU = 0 +NVML_TEMPERATURE_COUNT = 1 + + +_nvmlComputeMode_t = c_uint +NVML_COMPUTEMODE_DEFAULT = 0 +NVML_COMPUTEMODE_EXCLUSIVE_THREAD = 1 ## Support Removed +NVML_COMPUTEMODE_PROHIBITED = 2 +NVML_COMPUTEMODE_EXCLUSIVE_PROCESS = 3 +NVML_COMPUTEMODE_COUNT = 4 + +_nvmlMemoryLocation_t = c_uint +NVML_MEMORY_LOCATION_L1_CACHE = 0 +NVML_MEMORY_LOCATION_L2_CACHE = 1 +NVML_MEMORY_LOCATION_DEVICE_MEMORY = 2 +NVML_MEMORY_LOCATION_DRAM = 2 +NVML_MEMORY_LOCATION_REGISTER_FILE = 3 +NVML_MEMORY_LOCATION_TEXTURE_MEMORY = 4 +NVML_MEMORY_LOCATION_TEXTURE_SHM = 5 +NVML_MEMORY_LOCATION_CBU = 6 +NVML_MEMORY_LOCATION_SRAM = 7 +NVML_MEMORY_LOCATION_COUNT = 8 + +NVML_NVLINK_MAX_LINKS = 18 + +# For backwards compatibility, maintain the incorrectly-named "LANES" define +NVML_NVLINK_MAX_LANES = NVML_NVLINK_MAX_LINKS + +_nvmlNvLinkErrorCounter_t = c_uint +NVML_NVLINK_ERROR_DL_REPLAY = 0 +NVML_NVLINK_ERROR_DL_RECOVERY = 1 +NVML_NVLINK_ERROR_DL_CRC_FLIT = 2 +NVML_NVLINK_ERROR_DL_CRC_DATA = 3 +NVML_NVLINK_ERROR_DL_ECC_DATA = 4 +NVML_NVLINK_ERROR_COUNT = 5 + +_nvmlNvLinkEccLaneErrorCounter_t = c_uint +NVML_NVLINK_ERROR_DL_ECC_LANE0 = 0 +NVML_NVLINK_ERROR_DL_ECC_LANE1 = 1 +NVML_NVLINK_ERROR_DL_ECC_LANE2 = 2 +NVML_NVLINK_ERROR_DL_ECC_LANE3 = 3 +NVML_NVLINK_ERROR_DL_ECC_COUNT = 5 + +_nvmlNvLinkCapability_t = c_uint +NVML_NVLINK_CAP_P2P_SUPPORTED = 0 +NVML_NVLINK_CAP_SYSMEM_ACCESS = 1 +NVML_NVLINK_CAP_P2P_ATOMICS = 2 +NVML_NVLINK_CAP_SYSMEM_ATOMICS = 3 +NVML_NVLINK_CAP_SLI_BRIDGE = 4 +NVML_NVLINK_CAP_VALID = 5 +NVML_NVLINK_CAP_COUNT = 6 + +_nvmlNvLinkUtilizationCountPktTypes_t = c_uint +NVML_NVLINK_COUNTER_PKTFILTER_NOP = 0x1 +NVML_NVLINK_COUNTER_PKTFILTER_READ = 0x2 +NVML_NVLINK_COUNTER_PKTFILTER_WRITE = 0x4 +NVML_NVLINK_COUNTER_PKTFILTER_RATOM = 0x8 +NVML_NVLINK_COUNTER_PKTFILTER_NRATOM = 0x10 +NVML_NVLINK_COUNTER_PKTFILTER_FLUSH = 0x20 +NVML_NVLINK_COUNTER_PKTFILTER_RESPDATA = 0x40 +NVML_NVLINK_COUNTER_PKTFILTER_RESPNODATA = 0x80 +NVML_NVLINK_COUNTER_PKTFILTER_ALL = 0xFF + +_nvmlNvLinkUtilizationCountUnits_t = c_uint +NVML_NVLINK_COUNTER_UNIT_CYCLES = 0 +NVML_NVLINK_COUNTER_UNIT_PACKETS = 1 +NVML_NVLINK_COUNTER_UNIT_BYTES = 2 +NVML_NVLINK_COUNTER_UNIT_RESERVED = 3 +NVML_NVLINK_COUNTER_UNIT_COUNT = 4 + +_nvmlNvLinkDeviceType_t = c_uint +NVML_NVLINK_DEVICE_TYPE_GPU = 0x00 +NVML_NVLINK_DEVICE_TYPE_IBMNPU = 0x01 +NVML_NVLINK_DEVICE_TYPE_SWITCH = 0x02 +NVML_NVLINK_DEVICE_TYPE_UNKNOWN = 0xFF + +# These are deprecated, instead use _nvmlMemoryErrorType_t +_nvmlEccBitType_t = c_uint +NVML_SINGLE_BIT_ECC = 0 +NVML_DOUBLE_BIT_ECC = 1 +NVML_ECC_ERROR_TYPE_COUNT = 2 + +_nvmlEccCounterType_t = c_uint +NVML_VOLATILE_ECC = 0 +NVML_AGGREGATE_ECC = 1 +NVML_ECC_COUNTER_TYPE_COUNT = 2 + +_nvmlMemoryErrorType_t = c_uint +NVML_MEMORY_ERROR_TYPE_CORRECTED = 0 +NVML_MEMORY_ERROR_TYPE_UNCORRECTED = 1 +NVML_MEMORY_ERROR_TYPE_COUNT = 2 + +_nvmlClockType_t = c_uint +NVML_CLOCK_GRAPHICS = 0 +NVML_CLOCK_SM = 1 +NVML_CLOCK_MEM = 2 +NVML_CLOCK_VIDEO = 3 +NVML_CLOCK_COUNT = 4 + +_nvmlClockId_t = c_uint +NVML_CLOCK_ID_CURRENT = 0 +NVML_CLOCK_ID_APP_CLOCK_TARGET = 1 +NVML_CLOCK_ID_APP_CLOCK_DEFAULT = 2 +NVML_CLOCK_ID_CUSTOMER_BOOST_MAX = 3 +NVML_CLOCK_ID_COUNT = 4 + +_nvmlDriverModel_t = c_uint +NVML_DRIVER_WDDM = 0 +NVML_DRIVER_WDM = 1 +NVML_DRIVER_MCDM = 2 + +NVML_MAX_GPU_PERF_PSTATES = 16 + +_nvmlPstates_t = c_uint +NVML_PSTATE_0 = 0 +NVML_PSTATE_1 = 1 +NVML_PSTATE_2 = 2 +NVML_PSTATE_3 = 3 +NVML_PSTATE_4 = 4 +NVML_PSTATE_5 = 5 +NVML_PSTATE_6 = 6 +NVML_PSTATE_7 = 7 +NVML_PSTATE_8 = 8 +NVML_PSTATE_9 = 9 +NVML_PSTATE_10 = 10 +NVML_PSTATE_11 = 11 +NVML_PSTATE_12 = 12 +NVML_PSTATE_13 = 13 +NVML_PSTATE_14 = 14 +NVML_PSTATE_15 = 15 +NVML_PSTATE_UNKNOWN = 32 + +_nvmlInforomObject_t = c_uint +NVML_INFOROM_OEM = 0 +NVML_INFOROM_ECC = 1 +NVML_INFOROM_POWER = 2 +NVML_INFOROM_DEN = 3 +NVML_INFOROM_COUNT = 4 + +_nvmlReturn_t = c_uint +NVML_SUCCESS = 0 +NVML_ERROR_UNINITIALIZED = 1 +NVML_ERROR_INVALID_ARGUMENT = 2 +NVML_ERROR_NOT_SUPPORTED = 3 +NVML_ERROR_NO_PERMISSION = 4 +NVML_ERROR_ALREADY_INITIALIZED = 5 +NVML_ERROR_NOT_FOUND = 6 +NVML_ERROR_INSUFFICIENT_SIZE = 7 +NVML_ERROR_INSUFFICIENT_POWER = 8 +NVML_ERROR_DRIVER_NOT_LOADED = 9 +NVML_ERROR_TIMEOUT = 10 +NVML_ERROR_IRQ_ISSUE = 11 +NVML_ERROR_LIBRARY_NOT_FOUND = 12 +NVML_ERROR_FUNCTION_NOT_FOUND = 13 +NVML_ERROR_CORRUPTED_INFOROM = 14 +NVML_ERROR_GPU_IS_LOST = 15 +NVML_ERROR_RESET_REQUIRED = 16 +NVML_ERROR_OPERATING_SYSTEM = 17 +NVML_ERROR_LIB_RM_VERSION_MISMATCH = 18 +NVML_ERROR_IN_USE = 19 +NVML_ERROR_MEMORY = 20 +NVML_ERROR_NO_DATA = 21 +NVML_ERROR_VGPU_ECC_NOT_SUPPORTED = 22 +NVML_ERROR_INSUFFICIENT_RESOURCES = 23 +NVML_ERROR_FREQ_NOT_SUPPORTED = 24 +NVML_ERROR_ARGUMENT_VERSION_MISMATCH = 25 +NVML_ERROR_DEPRECATED = 26 +NVML_ERROR_NOT_READY = 27 +NVML_ERROR_GPU_NOT_FOUND = 28 +NVML_ERROR_INVALID_STATE = 29 +NVML_ERROR_UNKNOWN = 999 + +_nvmlFanState_t = c_uint +NVML_FAN_NORMAL = 0 +NVML_FAN_FAILED = 1 + +_nvmlFanControlPolicy_t = c_uint +NVML_FAN_POLICY_TEMPERATURE_CONTINOUS_SW = 0 +NVML_FAN_POLICY_MANUAL = 1 + +_nvmlLedColor_t = c_uint +NVML_LED_COLOR_GREEN = 0 +NVML_LED_COLOR_AMBER = 1 + +_nvmlGpuOperationMode_t = c_uint +NVML_GOM_ALL_ON = 0 +NVML_GOM_COMPUTE = 1 +NVML_GOM_LOW_DP = 2 + +_nvmlPageRetirementCause_t = c_uint +NVML_PAGE_RETIREMENT_CAUSE_MULTIPLE_SINGLE_BIT_ECC_ERRORS = 0 +NVML_PAGE_RETIREMENT_CAUSE_DOUBLE_BIT_ECC_ERROR = 1 +NVML_PAGE_RETIREMENT_CAUSE_COUNT = 2 + +_nvmlRestrictedAPI_t = c_uint +NVML_RESTRICTED_API_SET_APPLICATION_CLOCKS = 0 +NVML_RESTRICTED_API_SET_AUTO_BOOSTED_CLOCKS = 1 +NVML_RESTRICTED_API_COUNT = 2 + +_nvmlBridgeChipType_t = c_uint +NVML_BRIDGE_CHIP_PLX = 0 +NVML_BRIDGE_CHIP_BRO4 = 1 +NVML_MAX_PHYSICAL_BRIDGE = 128 + +_nvmlValueType_t = c_uint +NVML_VALUE_TYPE_DOUBLE = 0 +NVML_VALUE_TYPE_UNSIGNED_INT = 1 +NVML_VALUE_TYPE_UNSIGNED_LONG = 2 +NVML_VALUE_TYPE_UNSIGNED_LONG_LONG = 3 +NVML_VALUE_TYPE_SIGNED_LONG_LONG = 4 +NVML_VALUE_TYPE_SIGNED_INT = 5 +NVML_VALUE_TYPE_UNSIGNED_SHORT = 6 +NVML_VALUE_TYPE_COUNT = 7 + +_nvmlNvlinkVersion_t = c_uint +NVML_NVLINK_VERSION_INVALID = 0 +NVML_NVLINK_VERSION_1_0 = 1 +NVML_NVLINK_VERSION_2_0 = 2 +NVML_NVLINK_VERSION_2_2 = 3 +NVML_NVLINK_VERSION_3_0 = 4 +NVML_NVLINK_VERSION_3_1 = 5 +NVML_NVLINK_VERSION_4_0 = 6 +NVML_NVLINK_VERSION_5_0 = 7 + +_nvmlPerfPolicyType_t = c_uint +NVML_PERF_POLICY_POWER = 0 +NVML_PERF_POLICY_THERMAL = 1 +NVML_PERF_POLICY_SYNC_BOOST = 2 +NVML_PERF_POLICY_BOARD_LIMIT = 3 +NVML_PERF_POLICY_LOW_UTILIZATION = 4 +NVML_PERF_POLICY_RELIABILITY = 5 +NVML_PERF_POLICY_TOTAL_APP_CLOCKS = 10 +NVML_PERF_POLICY_TOTAL_BASE_CLOCKS = 11 +NVML_PERF_POLICY_COUNT = 12 + +_nvmlEncoderQueryType_t = c_uint +NVML_ENCODER_QUERY_H264 = 0 +NVML_ENCODER_QUERY_HEVC = 1 +NVML_ENCODER_QUERY_AV1 = 2 +NVML_ENCODER_QUERY_UNKNOWN = 255 + +_nvmlFBCSessionType_t = c_uint +NVML_FBC_SESSION_TYPE_UNKNOWN = 0 +NVML_FBC_SESSION_TYPE_TOSYS = 1 +NVML_FBC_SESSION_TYPE_CUDA = 2 +NVML_FBC_SESSION_TYPE_VID = 3 +NVML_FBC_SESSION_TYPE_HWENC = 4 + +_nvmlDetachGpuState_t = c_uint +NVML_DETACH_GPU_KEEP = 0 +NVML_DETACH_GPU_REMOVE = 1 + +_nvmlPcieLinkState_t = c_uint +NVML_PCIE_LINK_KEEP = 0 +NVML_PCIE_LINK_SHUT_DOWN = 1 + +_nvmlSamplingType_t = c_uint +NVML_TOTAL_POWER_SAMPLES = 0 +NVML_GPU_UTILIZATION_SAMPLES = 1 +NVML_MEMORY_UTILIZATION_SAMPLES = 2 +NVML_ENC_UTILIZATION_SAMPLES = 3 +NVML_DEC_UTILIZATION_SAMPLES = 4 +NVML_PROCESSOR_CLK_SAMPLES = 5 +NVML_MEMORY_CLK_SAMPLES = 6 +NVML_MODULE_POWER_SAMPLES = 7 +NVML_JPG_UTILIZATION_SAMPLES = 8 +NVML_OFA_UTILIZATION_SAMPLES = 9 +NVML_SAMPLINGTYPE_COUNT = 10 + +_nvmlPcieUtilCounter_t = c_uint +NVML_PCIE_UTIL_TX_BYTES = 0 +NVML_PCIE_UTIL_RX_BYTES = 1 +NVML_PCIE_UTIL_COUNT = 2 + +_nvmlGpuTopologyLevel_t = c_uint +NVML_TOPOLOGY_INTERNAL = 0 +NVML_TOPOLOGY_SINGLE = 10 +NVML_TOPOLOGY_MULTIPLE = 20 +NVML_TOPOLOGY_HOSTBRIDGE = 30 +NVML_TOPOLOGY_NODE = 40 +NVML_TOPOLOGY_CPU = NVML_TOPOLOGY_NODE +NVML_TOPOLOGY_SYSTEM = 50 + +_nvmlGpuP2PCapsIndex_t = c_uint +NVML_P2P_CAPS_INDEX_READ = (0,) +NVML_P2P_CAPS_INDEX_WRITE = 1 +NVML_P2P_CAPS_INDEX_NVLINK = 2 +NVML_P2P_CAPS_INDEX_ATOMICS = 3 +# +# NVML_P2P_CAPS_INDEX_PROP is deprecated. +# Use NVML_P2P_CAPS_INDEX_PCI instead. +# +NVML_P2P_CAPS_INDEX_PROP = 4 +NVML_P2P_CAPS_INDEX_PCI = 4 +NVML_P2P_CAPS_INDEX_UNKNOWN = 5 + +_nvmlGpuP2PStatus_t = c_uint +NVML_P2P_STATUS_OK = 0 +NVML_P2P_STATUS_CHIPSET_NOT_SUPPORED = 1 +NVML_P2P_STATUS_CHIPSET_NOT_SUPPORTED = NVML_P2P_STATUS_CHIPSET_NOT_SUPPORED +NVML_P2P_STATUS_GPU_NOT_SUPPORTED = 2 +NVML_P2P_STATUS_IOH_TOPOLOGY_NOT_SUPPORTED = 3 +NVML_P2P_STATUS_DISABLED_BY_REGKEY = 4 +NVML_P2P_STATUS_NOT_SUPPORTED = 5 +NVML_P2P_STATUS_UNKNOWN = 6 + +_nvmlDeviceArchitecture_t = c_uint +NVML_DEVICE_ARCH_KEPLER = 2 +NVML_DEVICE_ARCH_MAXWELL = 3 +NVML_DEVICE_ARCH_PASCAL = 4 +NVML_DEVICE_ARCH_VOLTA = 5 +NVML_DEVICE_ARCH_TURING = 6 +NVML_DEVICE_ARCH_AMPERE = 7 +NVML_DEVICE_ARCH_ADA = 8 +NVML_DEVICE_ARCH_HOPPER = 9 +NVML_DEVICE_ARCH_BLACKWELL = 10 +NVML_DEVICE_ARCH_T23X = 11 +NVML_DEVICE_ARCH_UNKNOWN = 0xFFFFFFFF + +# PCI bus Types +_nvmlBusType_t = c_uint +NVML_BUS_TYPE_UNKNOWN = 0 +NVML_BUS_TYPE_PCI = 1 +NVML_BUS_TYPE_PCIE = 2 +NVML_BUS_TYPE_FPCI = 3 +NVML_BUS_TYPE_AGP = 4 + +_nvmlPowerSource_t = c_uint +NVML_POWER_SOURCE_AC = 0x00000000 +NVML_POWER_SOURCE_BATTERY = 0x00000001 +NVML_POWER_SOURCE_UNDERSIZED = 0x00000002 + +_nvmlAdaptiveClockInfoStatus_t = c_uint +NVML_ADAPTIVE_CLOCKING_INFO_STATUS_DISABLED = 0x00000000 +NVML_ADAPTIVE_CLOCKING_INFO_STATUS_ENABLED = 0x00000001 + +_nvmlClockLimitId_t = c_uint +NVML_CLOCK_LIMIT_ID_RANGE_START = 0xFFFFFF00 +NVML_CLOCK_LIMIT_ID_TDP = 0xFFFFFF01 +NVML_CLOCK_LIMIT_ID_UNLIMITED = 0xFFFFFF02 + +_nvmlPcieLinkMaxSpeed_t = c_uint +NVML_PCIE_LINK_MAX_SPEED_INVALID = 0x00000000 +NVML_PCIE_LINK_MAX_SPEED_2500MBPS = 0x00000001 +NVML_PCIE_LINK_MAX_SPEED_5000MBPS = 0x00000002 +NVML_PCIE_LINK_MAX_SPEED_8000MBPS = 0x00000003 +NVML_PCIE_LINK_MAX_SPEED_16000MBPS = 0x00000004 +NVML_PCIE_LINK_MAX_SPEED_32000MBPS = 0x00000005 +NVML_PCIE_LINK_MAX_SPEED_64000MBPS = 0x00000006 + +_nvmlPcieAtomicsCapability_t = c_uint +NVML_PCIE_ATOMICS_CAP_FETCHADD32 = 0x01 +NVML_PCIE_ATOMICS_CAP_FETCHADD64 = 0x02 +NVML_PCIE_ATOMICS_CAP_SWAP32 = 0x04 +NVML_PCIE_ATOMICS_CAP_SWAP64 = 0x08 +NVML_PCIE_ATOMICS_CAP_CAS32 = 0x10 +NVML_PCIE_ATOMICS_CAP_CAS64 = 0x20 +NVML_PCIE_ATOMICS_CAP_CAS128 = 0x40 +NVML_PCIE_ATOMICS_OPS_MAX = 7 + +_nvmlAffinityScope_t = c_uint +NVML_AFFINITY_SCOPE_NODE = 0 +NVML_AFFINITY_SCOPE_SOCKET = 1 + +_nvmlDeviceGpuRecoveryAction_t = c_uint +NVML_GPU_RECOVERY_ACTION_NONE = 0 +NVML_GPU_RECOVERY_ACTION_GPU_RESET = 1 +NVML_GPU_RECOVERY_ACTION_NODE_REBOOT = 2 +NVML_GPU_RECOVERY_ACTION_DRAIN_P2P = 3 +NVML_GPU_RECOVERY_ACTION_DRAIN_AND_RESET = 4 + +# C preprocessor defined values +nvmlFlagDefault = 0 +nvmlFlagForce = 1 +NVML_INIT_FLAG_NO_GPUS = 1 +NVML_INIT_FLAG_NO_ATTACH = 2 + +NVML_MAX_GPC_COUNT = 32 + +# buffer size +NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE = 16 +NVML_DEVICE_UUID_BUFFER_SIZE = 80 +NVML_DEVICE_UUID_V2_BUFFER_SIZE = 96 +NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE = 80 +NVML_SYSTEM_NVML_VERSION_BUFFER_SIZE = 80 +NVML_DEVICE_NAME_BUFFER_SIZE = 64 +NVML_DEVICE_NAME_V2_BUFFER_SIZE = 96 +NVML_DEVICE_SERIAL_BUFFER_SIZE = 30 +NVML_DEVICE_PART_NUMBER_BUFFER_SIZE = 80 +NVML_DEVICE_GPU_PART_NUMBER_BUFFER_SIZE = 80 +NVML_DEVICE_VBIOS_VERSION_BUFFER_SIZE = 32 +NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE = 32 +NVML_DEVICE_PCI_BUS_ID_BUFFER_V2_SIZE = 16 +NVML_GRID_LICENSE_BUFFER_SIZE = 128 +NVML_VGPU_NAME_BUFFER_SIZE = 64 +NVML_GRID_LICENSE_FEATURE_MAX_COUNT = 3 +NVML_VGPU_METADATA_OPAQUE_DATA_SIZE = sizeof(c_uint) + 256 +NVML_VGPU_PGPU_METADATA_OPAQUE_DATA_SIZE = 256 +NVML_DEVICE_GPU_FRU_PART_NUMBER_BUFFER_SIZE = ( + 0x14 # NV2080_GPU_MAX_PRODUCT_PART_NUMBER_LENGTH +) +NVML_PERF_MODES_BUFFER_SIZE = 2048 + +# Format strings +NVML_DEVICE_PCI_BUS_ID_LEGACY_FMT = "%04X:%02X:%02X.0" +NVML_DEVICE_PCI_BUS_ID_FMT = "%08X:%02X:%02X.0" + +NVML_VALUE_NOT_AVAILABLE_ulonglong = c_ulonglong(-1) +NVML_VALUE_NOT_AVAILABLE_uint = c_uint(-1) + +""" + Field Identifiers. + + All Identifiers pertain to a device. Each ID is only used once and is guaranteed never to change. +""" +NVML_FI_DEV_ECC_CURRENT = 1 # Current ECC mode. 1=Active. 0=Inactive +NVML_FI_DEV_ECC_PENDING = 2 # Pending ECC mode. 1=Active. 0=Inactive + +# ECC Count Totals +NVML_FI_DEV_ECC_SBE_VOL_TOTAL = 3 # Total single bit volatile ECC errors +NVML_FI_DEV_ECC_DBE_VOL_TOTAL = 4 # Total double bit volatile ECC errors +NVML_FI_DEV_ECC_SBE_AGG_TOTAL = 5 # Total single bit aggregate (persistent) ECC errors +NVML_FI_DEV_ECC_DBE_AGG_TOTAL = 6 # Total double bit aggregate (persistent) ECC errors +# Individual ECC locations +NVML_FI_DEV_ECC_SBE_VOL_L1 = 7 # L1 cache single bit volatile ECC errors +NVML_FI_DEV_ECC_DBE_VOL_L1 = 8 # L1 cache double bit volatile ECC errors +NVML_FI_DEV_ECC_SBE_VOL_L2 = 9 # L2 cache single bit volatile ECC errors +NVML_FI_DEV_ECC_DBE_VOL_L2 = 10 # L2 cache double bit volatile ECC errors +NVML_FI_DEV_ECC_SBE_VOL_DEV = 11 # Device memory single bit volatile ECC errors +NVML_FI_DEV_ECC_DBE_VOL_DEV = 12 # Device memory double bit volatile ECC errors +NVML_FI_DEV_ECC_SBE_VOL_REG = 13 # Register file single bit volatile ECC errors +NVML_FI_DEV_ECC_DBE_VOL_REG = 14 # Register file double bit volatile ECC errors +NVML_FI_DEV_ECC_SBE_VOL_TEX = 15 # Texture memory single bit volatile ECC errors +NVML_FI_DEV_ECC_DBE_VOL_TEX = 16 # Texture memory double bit volatile ECC errors +NVML_FI_DEV_ECC_DBE_VOL_CBU = 17 # CBU double bit volatile ECC errors +NVML_FI_DEV_ECC_SBE_AGG_L1 = 18 # L1 cache single bit aggregate (persistent) ECC errors +NVML_FI_DEV_ECC_DBE_AGG_L1 = 19 # L1 cache double bit aggregate (persistent) ECC errors +NVML_FI_DEV_ECC_SBE_AGG_L2 = 20 # L2 cache single bit aggregate (persistent) ECC errors +NVML_FI_DEV_ECC_DBE_AGG_L2 = 21 # L2 cache double bit aggregate (persistent) ECC errors +NVML_FI_DEV_ECC_SBE_AGG_DEV = ( + 22 # Device memory single bit aggregate (persistent) ECC errors +) +NVML_FI_DEV_ECC_DBE_AGG_DEV = ( + 23 # Device memory double bit aggregate (persistent) ECC errors +) +NVML_FI_DEV_ECC_SBE_AGG_REG = ( + 24 # Register File single bit aggregate (persistent) ECC errors +) +NVML_FI_DEV_ECC_DBE_AGG_REG = ( + 25 # Register File double bit aggregate (persistent) ECC errors +) +NVML_FI_DEV_ECC_SBE_AGG_TEX = ( + 26 # Texture memory single bit aggregate (persistent) ECC errors +) +NVML_FI_DEV_ECC_DBE_AGG_TEX = ( + 27 # Texture memory double bit aggregate (persistent) ECC errors +) +NVML_FI_DEV_ECC_DBE_AGG_CBU = 28 # CBU double bit aggregate ECC errors + +# Page Retirement +NVML_FI_DEV_RETIRED_SBE = 29 # Number of retired pages because of single bit errors +NVML_FI_DEV_RETIRED_DBE = 30 # Number of retired pages because of double bit errors +NVML_FI_DEV_RETIRED_PENDING = 31 # If any pages are pending retirement. 1=yes. 0=no. + +# NvLink Flit Error Counters +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L0 = ( + 32 # NVLink flow control CRC Error Counter for Lane 0 +) +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L1 = ( + 33 # NVLink flow control CRC Error Counter for Lane 1 +) +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L2 = ( + 34 # NVLink flow control CRC Error Counter for Lane 2 +) +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L3 = ( + 35 # NVLink flow control CRC Error Counter for Lane 3 +) +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L4 = ( + 36 # NVLink flow control CRC Error Counter for Lane 4 +) +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L5 = ( + 37 # NVLink flow control CRC Error Counter for Lane 5 +) +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL = ( + 38 # NVLink flow control CRC Error Counter total for all Lanes +) + +# NvLink CRC Data Error Counters +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L0 = ( + 39 # NVLink data CRC Error Counter for Lane 0 +) +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L1 = ( + 40 # NVLink data CRC Error Counter for Lane 1 +) +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L2 = ( + 41 # NVLink data CRC Error Counter for Lane 2 +) +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L3 = ( + 42 # NVLink data CRC Error Counter for Lane 3 +) +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L4 = ( + 43 # NVLink data CRC Error Counter for Lane 4 +) +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L5 = ( + 44 # NVLink data CRC Error Counter for Lane 5 +) +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL = ( + 45 # NvLink data CRC Error Counter total for all Lanes +) + +# NvLink Replay Error Counters +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L0 = 46 # NVLink Replay Error Counter for Lane 0 +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L1 = 47 # NVLink Replay Error Counter for Lane 1 +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L2 = 48 # NVLink Replay Error Counter for Lane 2 +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L3 = 49 # NVLink Replay Error Counter for Lane 3 +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L4 = 50 # NVLink Replay Error Counter for Lane 4 +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L5 = 51 # NVLink Replay Error Counter for Lane 5 +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL = ( + 52 # NVLink Replay Error Counter total for all Lanes +) + +# NvLink Recovery Error Counters +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L0 = ( + 53 # NVLink Recovery Error Counter for Lane 0 +) +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L1 = ( + 54 # NVLink Recovery Error Counter for Lane 1 +) +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L2 = ( + 55 # NVLink Recovery Error Counter for Lane 2 +) +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L3 = ( + 56 # NVLink Recovery Error Counter for Lane 3 +) +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L4 = ( + 57 # NVLink Recovery Error Counter for Lane 4 +) +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L5 = ( + 58 # NVLink Recovery Error Counter for Lane 5 +) +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL = ( + 59 # NVLink Recovery Error Counter total for all Lanes +) + +# NvLink Bandwidth Counters +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L0 = ( + 60 # NVLink Bandwidth Counter for Counter Set 0, Lane 0 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L1 = ( + 61 # NVLink Bandwidth Counter for Counter Set 0, Lane 1 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L2 = ( + 62 # NVLink Bandwidth Counter for Counter Set 0, Lane 2 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L3 = ( + 63 # NVLink Bandwidth Counter for Counter Set 0, Lane 3 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L4 = ( + 64 # NVLink Bandwidth Counter for Counter Set 0, Lane 4 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L5 = ( + 65 # NVLink Bandwidth Counter for Counter Set 0, Lane 5 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_TOTAL = ( + 66 # NVLink Bandwidth Counter Total for Counter Set 0, All Lanes +) + +# NvLink Bandwidth Counters +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L0 = ( + 67 # NVLink Bandwidth Counter for Counter Set 1, Lane 0 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L1 = ( + 68 # NVLink Bandwidth Counter for Counter Set 1, Lane 1 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L2 = ( + 69 # NVLink Bandwidth Counter for Counter Set 1, Lane 2 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L3 = ( + 70 # NVLink Bandwidth Counter for Counter Set 1, Lane 3 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L4 = ( + 71 # NVLink Bandwidth Counter for Counter Set 1, Lane 4 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L5 = ( + 72 # NVLink Bandwidth Counter for Counter Set 1, Lane 5 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_TOTAL = ( + 73 # NVLink Bandwidth Counter Total for Counter Set 1, All Lanes +) + +# Perf Policy Counters +NVML_FI_DEV_PERF_POLICY_POWER = 74 # Perf Policy Counter for Power Policy +NVML_FI_DEV_PERF_POLICY_THERMAL = 75 # Perf Policy Counter for Thermal Policy +NVML_FI_DEV_PERF_POLICY_SYNC_BOOST = 76 # Perf Policy Counter for Sync boost Policy +NVML_FI_DEV_PERF_POLICY_BOARD_LIMIT = 77 # Perf Policy Counter for Board Limit +NVML_FI_DEV_PERF_POLICY_LOW_UTILIZATION = ( + 78 # Perf Policy Counter for Low GPU Utilization Policy +) +NVML_FI_DEV_PERF_POLICY_RELIABILITY = 79 # Perf Policy Counter for Reliability Policy +NVML_FI_DEV_PERF_POLICY_TOTAL_APP_CLOCKS = ( + 80 # Perf Policy Counter for Total App Clock Policy +) +NVML_FI_DEV_PERF_POLICY_TOTAL_BASE_CLOCKS = ( + 81 # Perf Policy Counter for Total Base Clocks Policy +) + +# Memory temperatures +NVML_FI_DEV_MEMORY_TEMP = 82 # Memory temperature for the device + +# Energy Counter +NVML_FI_DEV_TOTAL_ENERGY_CONSUMPTION = ( + 83 # Total energy consumption for the GPU in mJ since the driver was last reloaded +) + +# NVLink Speed +NVML_FI_DEV_NVLINK_SPEED_MBPS_L0 = 84 +NVML_FI_DEV_NVLINK_SPEED_MBPS_L1 = 85 +NVML_FI_DEV_NVLINK_SPEED_MBPS_L2 = 86 +NVML_FI_DEV_NVLINK_SPEED_MBPS_L3 = 87 +NVML_FI_DEV_NVLINK_SPEED_MBPS_L4 = 88 +NVML_FI_DEV_NVLINK_SPEED_MBPS_L5 = 89 +NVML_FI_DEV_NVLINK_SPEED_MBPS_COMMON = 90 + +# NVLink Link Count +NVML_FI_DEV_NVLINK_LINK_COUNT = 91 + +# Page Retirement pending fields +NVML_FI_DEV_RETIRED_PENDING_SBE = 92 +NVML_FI_DEV_RETIRED_PENDING_DBE = 93 + +# PCIe replay and replay rollover counters +NVML_FI_DEV_PCIE_REPLAY_COUNTER = 94 +NVML_FI_DEV_PCIE_REPLAY_ROLLOVER_COUNTER = 95 + +# NvLink Flit Error Counters +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L6 = ( + 96 # NVLink flow control CRC Error Counter for Lane 6 +) +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L7 = ( + 97 # NVLink flow control CRC Error Counter for Lane 7 +) +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L8 = ( + 98 # NVLink flow control CRC Error Counter for Lane 8 +) +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L9 = ( + 99 # NVLink flow control CRC Error Counter for Lane 9 +) +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L10 = ( + 100 # NVLink flow control CRC Error Counter for Lane 10 +) +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L11 = ( + 101 # NVLink flow control CRC Error Counter for Lane 11 +) + +# NvLink CRC Data Error Counters +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L6 = ( + 102 # NVLink data CRC Error Counter for Lane 6 +) +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L7 = ( + 103 # NVLink data CRC Error Counter for Lane 7 +) +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L8 = ( + 104 # NVLink data CRC Error Counter for Lane 8 +) +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L9 = ( + 105 # NVLink data CRC Error Counter for Lane 9 +) +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L10 = ( + 106 # NVLink data CRC Error Counter for Lane 10 +) +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L11 = ( + 107 # NVLink data CRC Error Counter for Lane 11 +) + +# NvLink Replay Error Counters +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L6 = 108 # NVLink Replay Error Counter for Lane 6 +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L7 = 109 # NVLink Replay Error Counter for Lane 7 +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L8 = 110 # NVLink Replay Error Counter for Lane 8 +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L9 = 111 # NVLink Replay Error Counter for Lane 9 +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L10 = ( + 112 # NVLink Replay Error Counter for Lane 10 +) +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L11 = ( + 113 # NVLink Replay Error Counter for Lane 11 +) + +# NvLink Recovery Error Counters +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L6 = ( + 114 # NVLink Recovery Error Counter for Lane 6 +) +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L7 = ( + 115 # NVLink Recovery Error Counter for Lane 7 +) +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L8 = ( + 116 # NVLink Recovery Error Counter for Lane 8 +) +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L9 = ( + 117 # NVLink Recovery Error Counter for Lane 9 +) +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L10 = ( + 118 # NVLink Recovery Error Counter for Lane 10 +) +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L11 = ( + 119 # NVLink Recovery Error Counter for Lane 11 +) + +# NvLink Bandwidth Counters +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L6 = ( + 120 # NVLink Bandwidth Counter for Counter Set 0, Lane 6 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L7 = ( + 121 # NVLink Bandwidth Counter for Counter Set 0, Lane 7 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L8 = ( + 122 # NVLink Bandwidth Counter for Counter Set 0, Lane 8 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L9 = ( + 123 # NVLink Bandwidth Counter for Counter Set 0, Lane 9 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L10 = ( + 124 # NVLink Bandwidth Counter for Counter Set 0, Lane 10 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L11 = ( + 125 # NVLink Bandwidth Counter for Counter Set 0, Lane 11 +) + +# NvLink Bandwidth Counters +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L6 = ( + 126 # NVLink Bandwidth Counter for Counter Set 1, Lane 6 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L7 = ( + 127 # NVLink Bandwidth Counter for Counter Set 1, Lane 7 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L8 = ( + 128 # NVLink Bandwidth Counter for Counter Set 1, Lane 8 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L9 = ( + 129 # NVLink Bandwidth Counter for Counter Set 1, Lane 9 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L10 = ( + 130 # NVLink Bandwidth Counter for Counter Set 1, Lane 10 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L11 = ( + 131 # NVLink Bandwidth Counter for Counter Set 1, Lane 11 +) + +# NVLink Speed +NVML_FI_DEV_NVLINK_SPEED_MBPS_L6 = 132 +NVML_FI_DEV_NVLINK_SPEED_MBPS_L7 = 133 +NVML_FI_DEV_NVLINK_SPEED_MBPS_L8 = 134 +NVML_FI_DEV_NVLINK_SPEED_MBPS_L9 = 135 +NVML_FI_DEV_NVLINK_SPEED_MBPS_L10 = 136 +NVML_FI_DEV_NVLINK_SPEED_MBPS_L11 = 137 + +# NVLink Throughput Counters +NVML_FI_DEV_NVLINK_THROUGHPUT_DATA_TX = 138 # NVLink TX Data throughput in KiB +NVML_FI_DEV_NVLINK_THROUGHPUT_DATA_RX = 139 # NVLink RX Data throughput in KiB +NVML_FI_DEV_NVLINK_THROUGHPUT_RAW_TX = 140 # NVLink TX Data + protocol overhead in KiB +NVML_FI_DEV_NVLINK_THROUGHPUT_RAW_RX = 141 # NVLink RX Data + protocol overhead in KiB + +# Row Remapper +NVML_FI_DEV_REMAPPED_COR = 142 +NVML_FI_DEV_REMAPPED_UNC = 143 +NVML_FI_DEV_REMAPPED_PENDING = 144 +NVML_FI_DEV_REMAPPED_FAILURE = 145 + +# Remote device NVLink ID +NVML_FI_DEV_NVLINK_REMOTE_NVLINK_ID = 146 + +# Number of NVLinks connected to NVSwitch +NVML_FI_DEV_NVSWITCH_CONNECTED_LINK_COUNT = 147 + +# NvLink ECC Data Error Counters +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L0 = ( + 148 # < NVLink data ECC Error Counter for Link 0 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L1 = ( + 149 # < NVLink data ECC Error Counter for Link 1 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L2 = ( + 150 # < NVLink data ECC Error Counter for Link 2 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L3 = ( + 151 # < NVLink data ECC Error Counter for Link 3 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L4 = ( + 152 # < NVLink data ECC Error Counter for Link 4 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L5 = ( + 153 # < NVLink data ECC Error Counter for Link 5 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L6 = ( + 154 # < NVLink data ECC Error Counter for Link 6 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L7 = ( + 155 # < NVLink data ECC Error Counter for Link 7 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L8 = ( + 156 # < NVLink data ECC Error Counter for Link 8 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L9 = ( + 157 # < NVLink data ECC Error Counter for Link 9 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L10 = ( + 158 # < NVLink data ECC Error Counter for Link 10 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L11 = ( + 159 # < NVLink data ECC Error Counter for Link 11 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_TOTAL = ( + 160 # < NvLink data ECC Error Counter total for all Links +) + +NVML_FI_DEV_NVLINK_ERROR_DL_REPLAY = 161 +NVML_FI_DEV_NVLINK_ERROR_DL_RECOVERY = 162 +NVML_FI_DEV_NVLINK_ERROR_DL_CRC = 163 +NVML_FI_DEV_NVLINK_GET_SPEED = 164 +NVML_FI_DEV_NVLINK_GET_STATE = 165 +NVML_FI_DEV_NVLINK_GET_VERSION = 166 + +NVML_FI_DEV_NVLINK_GET_POWER_STATE = 167 +NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD = 168 + +NVML_FI_DEV_PCIE_L0_TO_RECOVERY_COUNTER = 169 + +NVML_FI_DEV_C2C_LINK_COUNT = 170 +NVML_FI_DEV_C2C_LINK_GET_STATUS = 171 +NVML_FI_DEV_C2C_LINK_GET_MAX_BW = 172 + +NVML_FI_DEV_PCIE_COUNT_CORRECTABLE_ERRORS = 173 +NVML_FI_DEV_PCIE_COUNT_NAKS_RECEIVED = 174 +NVML_FI_DEV_PCIE_COUNT_RECEIVER_ERROR = 175 +NVML_FI_DEV_PCIE_COUNT_BAD_TLP = 176 +NVML_FI_DEV_PCIE_COUNT_NAKS_SENT = 177 +NVML_FI_DEV_PCIE_COUNT_BAD_DLLP = 178 +NVML_FI_DEV_PCIE_COUNT_NON_FATAL_ERROR = 179 +NVML_FI_DEV_PCIE_COUNT_FATAL_ERROR = 180 +NVML_FI_DEV_PCIE_COUNT_UNSUPPORTED_REQ = 181 +NVML_FI_DEV_PCIE_COUNT_LCRC_ERROR = 182 +NVML_FI_DEV_PCIE_COUNT_LANE_ERROR = 183 + +NVML_FI_DEV_IS_RESETLESS_MIG_SUPPORTED = 184 + +NVML_FI_DEV_POWER_AVERAGE = 185 +NVML_FI_DEV_POWER_INSTANT = 186 +NVML_FI_DEV_POWER_MIN_LIMIT = 187 +NVML_FI_DEV_POWER_MAX_LIMIT = 188 +NVML_FI_DEV_POWER_DEFAULT_LIMIT = 189 +NVML_FI_DEV_POWER_CURRENT_LIMIT = 190 +NVML_FI_DEV_ENERGY = 191 +NVML_FI_DEV_POWER_REQUESTED_LIMIT = 192 + +NVML_FI_DEV_TEMPERATURE_SHUTDOWN_TLIMIT = 193 +NVML_FI_DEV_TEMPERATURE_SLOWDOWN_TLIMIT = 194 +NVML_FI_DEV_TEMPERATURE_MEM_MAX_TLIMIT = 195 +NVML_FI_DEV_TEMPERATURE_GPU_MAX_TLIMIT = 196 + +NVML_FI_DEV_PCIE_COUNT_TX_BYTES = 197 +NVML_FI_DEV_PCIE_COUNT_RX_BYTES = 198 + +NVML_FI_DEV_IS_MIG_MODE_INDEPENDENT_MIG_QUERY_CAPABLE = 199 + +NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_MAX = 200 + +NVML_FI_DEV_NVLINK_COUNT_XMIT_PACKETS = 201 +NVML_FI_DEV_NVLINK_COUNT_XMIT_BYTES = 202 +NVML_FI_DEV_NVLINK_COUNT_RCV_PACKETS = 203 +NVML_FI_DEV_NVLINK_COUNT_RCV_BYTES = 204 +NVML_FI_DEV_NVLINK_COUNT_VL15_DROPPED = 205 # Deprecated, do not use +NVML_FI_DEV_NVLINK_COUNT_MALFORMED_PACKET_ERRORS = 206 +NVML_FI_DEV_NVLINK_COUNT_BUFFER_OVERRUN_ERRORS = 207 +NVML_FI_DEV_NVLINK_COUNT_RCV_ERRORS = 208 +NVML_FI_DEV_NVLINK_COUNT_RCV_REMOTE_ERRORS = 209 +NVML_FI_DEV_NVLINK_COUNT_RCV_GENERAL_ERRORS = 210 +NVML_FI_DEV_NVLINK_COUNT_LOCAL_LINK_INTEGRITY_ERRORS = 211 +NVML_FI_DEV_NVLINK_COUNT_XMIT_DISCARDS = 212 + +NVML_FI_DEV_NVLINK_COUNT_LINK_RECOVERY_SUCCESSFUL_EVENTS = 213 +NVML_FI_DEV_NVLINK_COUNT_LINK_RECOVERY_FAILED_EVENTS = 214 +NVML_FI_DEV_NVLINK_COUNT_LINK_RECOVERY_EVENTS = 215 + +NVML_FI_DEV_NVLINK_COUNT_RAW_BER_LANE0 = 216 # Deprecated, do not use +NVML_FI_DEV_NVLINK_COUNT_RAW_BER_LANE1 = 217 # Deprecated, do not use +NVML_FI_DEV_NVLINK_COUNT_RAW_BER = 218 # Deprecated, do not use +NVML_FI_DEV_NVLINK_COUNT_EFFECTIVE_ERRORS = 219 +NVML_FI_DEV_NVLINK_COUNT_EFFECTIVE_BER = 220 +NVML_FI_DEV_NVLINK_COUNT_SYMBOL_ERRORS = 221 +NVML_FI_DEV_NVLINK_COUNT_SYMBOL_BER = 222 + +NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_MIN = 223 +NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_UNITS = ( + 224 # Values are in the form NVML_NVLINK_LOW_POWER_THRESHOLD_UNIT_* +) +NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_SUPPORTED = 225 + +NVML_FI_DEV_RESET_STATUS = ( + 226 # Deprecated use NVML_FI_DEV_GET_GPU_RECOVERY_ACTION instead +) +NVML_FI_DEV_DRAIN_AND_RESET_STATUS = ( + 227 # Deprecated use NVML_FI_DEV_GET_GPU_RECOVERY_ACTION instead +) +NVML_FI_DEV_PCIE_OUTBOUND_ATOMICS_MASK = 228 +NVML_FI_DEV_PCIE_INBOUND_ATOMICS_MASK = 229 +NVML_FI_DEV_GET_GPU_RECOVERY_ACTION = 230 + +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_0 = 235 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_1 = 236 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_2 = 237 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_3 = 238 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_4 = 239 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_5 = 240 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_6 = 241 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_7 = 242 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_8 = 243 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_9 = 244 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_10 = 245 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_11 = 246 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_12 = 247 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_13 = 248 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_14 = 249 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_15 = 250 +NVML_FI_PWR_SMOOTHING_ENABLED = 251 # Enablement (0/DISABLED or 1/ENABLED) +NVML_FI_PWR_SMOOTHING_PRIV_LVL = 252 # Current privilege level +NVML_FI_PWR_SMOOTHING_IMM_RAMP_DOWN_ENABLED = ( + 253 # Immediate ramp down enablement (0/DISABLED or 1/ENABLED) +) +NVML_FI_PWR_SMOOTHING_APPLIED_TMP_CEIL = 254 # Applied TMP ceiling value +NVML_FI_PWR_SMOOTHING_APPLIED_TMP_FLOOR = 255 # Applied TMP floor value +NVML_FI_PWR_SMOOTHING_MAX_PERCENT_TMP_FLOOR_SETTING = 256 # Max % TMP Floor value +NVML_FI_PWR_SMOOTHING_MIN_PERCENT_TMP_FLOOR_SETTING = 257 # Min % TMP Floor value +NVML_FI_PWR_SMOOTHING_HW_CIRCUITRY_PERCENT_LIFETIME_REMAINING = ( + 258 # HW Circuitry % lifetime remaining +) +NVML_FI_PWR_SMOOTHING_MAX_NUM_PRESET_PROFILES = 259 # Max number of preset profiles +NVML_FI_PWR_SMOOTHING_PROFILE_PERCENT_TMP_FLOOR = 260 # % TMP floor for a given profile +NVML_FI_PWR_SMOOTHING_PROFILE_RAMP_UP_RATE = ( + 261 # Ramp up rate in mW/s for a given profile +) +NVML_FI_PWR_SMOOTHING_PROFILE_RAMP_DOWN_RATE = ( + 262 # Ramp down rate in mW/s for a given profile +) +NVML_FI_PWR_SMOOTHING_PROFILE_RAMP_DOWN_HYST_VAL = ( + 263 # Ramp down hysteresis value in ms for a given profile +) +NVML_FI_PWR_SMOOTHING_ACTIVE_PRESET_PROFILE = 264 # Active preset profile number +NVML_FI_PWR_SMOOTHING_ADMIN_OVERRIDE_PERCENT_TMP_FLOOR = ( + 265 # % TMP floor for a given profile +) +NVML_FI_PWR_SMOOTHING_ADMIN_OVERRIDE_RAMP_UP_RATE = ( + 266 # Ramp up rate in mW/s for a given profile +) +NVML_FI_PWR_SMOOTHING_ADMIN_OVERRIDE_RAMP_DOWN_RATE = ( + 267 # Ramp down rate in mW/s for a given profile +) +NVML_FI_PWR_SMOOTHING_ADMIN_OVERRIDE_RAMP_DOWN_HYST_VAL = ( + 268 # Ramp down hysteresis value in ms for a given profile +) + +NVML_FI_MAX = 269 # One greater than the largest field ID defined above + +# NVML_FI_DEV_NVLINK_GET_STATE state enums +NVML_NVLINK_STATE_INACTIVE = 0x0 +NVML_NVLINK_STATE_ACTIVE = 0x1 +NVML_NVLINK_STATE_SLEEP = 0x2 + +NVML_NVLINK_LOW_POWER_THRESHOLD_UNIT_100US = ( + 0 # NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_UNITS +) +NVML_NVLINK_LOW_POWER_THRESHOLD_UNIT_50US = ( + 1 # NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_UNITS +) + +## Enums needed for the method nvmlDeviceGetVirtualizationMode and nvmlDeviceSetVirtualizationMode +NVML_GPU_VIRTUALIZATION_MODE_NONE = 0 # Represents Bare Metal GPU +NVML_GPU_VIRTUALIZATION_MODE_PASSTHROUGH = ( + 1 # Device is associated with GPU-Passthorugh +) +NVML_GPU_VIRTUALIZATION_MODE_VGPU = ( + 2 # Device is associated with vGPU inside virtual machine. +) +NVML_GPU_VIRTUALIZATION_MODE_HOST_VGPU = ( + 3 # Device is associated with VGX hypervisor in vGPU mode +) +NVML_GPU_VIRTUALIZATION_MODE_HOST_VSGA = ( + 4 # Device is associated with VGX hypervisor in vSGA mode +) + +## Lib loading ## +nvmlLib = None +libLoadLock = threading.Lock() +_nvmlLib_refcount = 0 # Incremented on each nvmlInit and decremented on nvmlShutdown + +## vGPU Management +_nvmlVgpuTypeId_t = c_uint +_nvmlVgpuInstance_t = c_uint + +_nvmlVgpuVmIdType_t = c_uint +NVML_VGPU_VM_ID_DOMAIN_ID = 0 +NVML_VGPU_VM_ID_UUID = 1 + +_nvmlGridLicenseFeatureCode_t = c_uint +NVML_GRID_LICENSE_FEATURE_CODE_UNKNOWN = 0 +NVML_GRID_LICENSE_FEATURE_CODE_VGPU = 1 +NVML_GRID_LICENSE_FEATURE_CODE_NVIDIA_RTX = 2 +NVML_GRID_LICENSE_FEATURE_CODE_VWORKSTATION = ( + 2 # deprecated, use NVML_GRID_LICENSE_FEATURE_CODE_NVIDIA_RTX. +) +NVML_GRID_LICENSE_FEATURE_CODE_GAMING = 3 +NVML_GRID_LICENSE_FEATURE_CODE_COMPUTE = 4 + +_nvmlGridLicenseExpiryStatus_t = c_uint8 +NVML_GRID_LICENSE_EXPIRY_NOT_AVAILABLE = (0,) # Expiry information not available +NVML_GRID_LICENSE_EXPIRY_INVALID = (1,) # Invalid expiry or error fetching expiry +NVML_GRID_LICENSE_EXPIRY_VALID = (2,) # Valid expiry +NVML_GRID_LICENSE_EXPIRY_NOT_APPLICABLE = (3,) # Expiry not applicable +NVML_GRID_LICENSE_EXPIRY_PERMANENT = (4,) # Permanent expiry + +_nvmlVgpuCapability_t = c_uint +NVML_VGPU_CAP_NVLINK_P2P = 0 # vGPU P2P over NVLink is supported +NVML_VGPU_CAP_GPUDIRECT = 1 # GPUDirect capability is supported +NVML_VGPU_CAP_MULTI_VGPU_EXCLUSIVE = ( + 2 # vGPU profile cannot be mixed with other vGPU profiles in same VM +) +NVML_VGPU_CAP_EXCLUSIVE_TYPE = ( + 3 # vGPU profile cannot run on a GPU alongside other profiles of different type +) +NVML_VGPU_CAP_EXCLUSIVE_SIZE = ( + 4 # vGPU profile cannot run on a GPU alongside other profiles of different size +) +NVML_VGPU_CAP_COUNT = 5 + +_nvmlVgpuDriverCapability_t = c_uint +NVML_VGPU_DRIVER_CAP_HETEROGENEOUS_MULTI_VGPU = ( + 0 # Supports mixing of different vGPU profiles within one guest VM +) +NVML_VGPU_DRIVER_CAP_WARM_UPDATE = 1 # Supports FSR and warm update of vGPU host driver without terminating the running guest VM +NVML_VGPU_DRIVER_CAP_COUNT = 2 + +_nvmlDeviceVgpuCapability_t = c_uint +NVML_DEVICE_VGPU_CAP_FRACTIONAL_MULTI_VGPU = 0 # Query whether the fractional vGPU profiles on this GPU can be used in multi-vGPU configurations +NVML_DEVICE_VGPU_CAP_HETEROGENEOUS_TIMESLICE_PROFILES = 1 # Query whether the GPU supports concurrent execution of timesliced vGPU profiles of differing types +NVML_DEVICE_VGPU_CAP_HETEROGENEOUS_TIMESLICE_SIZES = 2 # Query whether the GPU supports concurrent execution of timesliced vGPU profiles of differing framebuffer sizes +NVML_DEVICE_VGPU_CAP_READ_DEVICE_BUFFER_BW = 3 # Query the GPU's read_device_buffer expected bandwidth capacity in megabytes per second +NVML_DEVICE_VGPU_CAP_WRITE_DEVICE_BUFFER_BW = 4 # Query the GPU's write_device_buffer expected bandwidth capacity in megabytes per second +NVML_DEVICE_VGPU_CAP_DEVICE_STREAMING = ( + 5 # Query whether the vGPU profiles on the GPU supports migration data streaming +) +NVML_DEVICE_VGPU_CAP_MINI_QUARTER_GPU = ( + 6 # Set/Get support of mini-quarter vGPU profiles +) +NVML_DEVICE_VGPU_CAP_COMPUTE_MEDIA_ENGINE_GPU = ( + 7 # Set/Get support for compute media engine vGPU profiles +) +NVML_DEVICE_VGPU_CAP_WARM_UPDATE = ( + 8 # Query whether the GPU supports FSR and warm update +) +NVML_DEVICE_VGPU_CAP_HOMOGENEOUS_PLACEMENTS = 9 # Query whether the GPU supports reporting of placements of timesliced vGPU profiles with identical framebuffer sizes +NVML_DEVICE_VGPU_CAP_COUNT = 10 + +_nvmlVgpuGuestInfoState_t = c_uint +NVML_VGPU_INSTANCE_GUEST_INFO_STATE_UNINITIALIZED = 0 +NVML_VGPU_INSTANCE_GUEST_INFO_STATE_INITIALIZED = 1 + +_nvmlVgpuVmCompatibility_t = c_uint +NVML_VGPU_VM_COMPATIBILITY_NONE = 0x0 +NVML_VGPU_VM_COMPATIBILITY_COLD = 0x1 +NVML_VGPU_VM_COMPATIBILITY_HIBERNATE = 0x2 +NVML_VGPU_VM_COMPATIBILITY_SLEEP = 0x4 +NVML_VGPU_VM_COMPATIBILITY_LIVE = 0x8 + +_nvmlVgpuPgpuCompatibilityLimitCode_t = c_uint +NVML_VGPU_COMPATIBILITY_LIMIT_NONE = 0x0 +NVML_VGPU_COMPATIBILITY_LIMIT_HOST_DRIVER = 0x1 +NVML_VGPU_COMPATIBILITY_LIMIT_GUEST_DRIVER = 0x2 +NVML_VGPU_COMPATIBILITY_LIMIT_GPU = 0x4 +NVML_VGPU_COMPATIBILITY_LIMIT_OTHER = 0x80000000 + +_nvmlHostVgpuMode_t = c_uint +NVML_HOST_VGPU_MODE_NON_SRIOV = 0 +NVML_HOST_VGPU_MODE_SRIOV = 1 + +_nvmlConfComputeGpusReadyState_t = c_uint +NVML_CC_ACCEPTING_CLIENT_REQUESTS_FALSE = 0 +NVML_CC_ACCEPTING_CLIENT_REQUESTS_TRUE = 1 + +_nvmlConfComputeGpuCaps_t = c_uint +NVML_CC_SYSTEM_GPUS_CC_NOT_CAPABLE = 0 +NVML_CC_SYSTEM_GPUS_CC_CAPABLE = 1 + +_nvmlConfComputeCpuCaps_t = c_uint +NVML_CC_SYSTEM_CPU_CAPS_NONE = 0 +NVML_CC_SYSTEM_CPU_CAPS_AMD_SEV = 1 +NVML_CC_SYSTEM_CPU_CAPS_INTEL_TDX = 2 +NVML_CC_SYSTEM_CPU_CAPS_AMD_SEV_SNP = 3 +NVML_CC_SYSTEM_CPU_CAPS_AMD_SNP_VTOM = 4 + +_nvmlConfComputeDevToolsMode_t = c_uint +NVML_CC_SYSTEM_DEVTOOLS_MODE_OFF = 0 +NVML_CC_SYSTEM_DEVTOOLS_MODE_ON = 1 + +NVML_CC_SYSTEM_MULTIGPU_NONE = 0 +NVML_CC_SYSTEM_MULTIGPU_PROTECTED_PCIE = 1 + +NVML_CC_SYSTEM_ENVIRONMENT_UNAVAILABLE = 0 +NVML_CC_SYSTEM_ENVIRONMENT_SIM = 1 +NVML_CC_SYSTEM_ENVIRONMENT_PROD = 2 + +_nvmlConfComputeCcFeature_t = c_uint +NVML_CC_SYSTEM_FEATURE_DISABLED = 0 +NVML_CC_SYSTEM_FEATURE_ENABLED = 1 + +_nvmlConfComputeCcKeyRotationThreshAttackerAdv_t = c_uint +NVML_CC_KEY_ROTATION_THRESH_ATTACKER_ADVANTAGE_MIN = 50 +NVML_CC_KEY_ROTATION_THRESH_ATTACKER_ADVANTAGE_MAX = 65 + +# GSP firmware +NVML_GSP_FIRMWARE_VERSION_BUF_SIZE = 0x40 + + +class NVMLLibraryMismatchError(Exception): + pass + + +## Error Checking ## +class NVMLError(Exception): + _valClassMapping = dict() + # List of currently known error codes + _errcode_to_string = { + NVML_ERROR_UNINITIALIZED: "Uninitialized", + NVML_ERROR_INVALID_ARGUMENT: "Invalid Argument", + NVML_ERROR_NOT_SUPPORTED: "Not Supported", + NVML_ERROR_NO_PERMISSION: "Insufficient Permissions", + NVML_ERROR_ALREADY_INITIALIZED: "Already Initialized", + NVML_ERROR_NOT_FOUND: "Not Found", + NVML_ERROR_INSUFFICIENT_SIZE: "Insufficient Size", + NVML_ERROR_INSUFFICIENT_POWER: "Insufficient External Power", + NVML_ERROR_DRIVER_NOT_LOADED: "Driver Not Loaded", + NVML_ERROR_TIMEOUT: "Timeout", + NVML_ERROR_IRQ_ISSUE: "Interrupt Request Issue", + NVML_ERROR_LIBRARY_NOT_FOUND: "NVML Shared Library Not Found", + NVML_ERROR_FUNCTION_NOT_FOUND: "Function Not Found", + NVML_ERROR_CORRUPTED_INFOROM: "Corrupted infoROM", + NVML_ERROR_GPU_IS_LOST: "GPU is lost", + NVML_ERROR_RESET_REQUIRED: "GPU requires restart", + NVML_ERROR_OPERATING_SYSTEM: "The operating system has blocked the request.", + NVML_ERROR_LIB_RM_VERSION_MISMATCH: "RM has detected an NVML/RM version mismatch.", + NVML_ERROR_MEMORY: "Insufficient Memory", + NVML_ERROR_UNKNOWN: "Unknown Error", + } + + def __new__(typ, value): + """ + Maps value to a proper subclass of NVMLError. + See _extractNVMLErrorsAsClasses function for more details + """ + if typ == NVMLError: + typ = NVMLError._valClassMapping.get(value, typ) + obj = Exception.__new__(typ) + obj.value = value + return obj + + def __str__(self): + try: + if self.value not in NVMLError._errcode_to_string: + NVMLError._errcode_to_string[self.value] = str( + nvmlErrorString(self.value) + ) + return NVMLError._errcode_to_string[self.value] + except NVMLError: + return "NVML Error with code %d" % self.value + + def __eq__(self, other): + return self.value == other.value + + +def nvmlExceptionClass(nvmlErrorCode): + if nvmlErrorCode not in NVMLError._valClassMapping: + raise ValueError("nvmlErrorCode %s is not valid" % nvmlErrorCode) + return NVMLError._valClassMapping[nvmlErrorCode] + + +def _extractNVMLErrorsAsClasses(): + """ + Generates a hierarchy of classes on top of NVMLError class. + + Each NVML Error gets a new NVMLError subclass. This way try,except blocks can filter appropriate + exceptions more easily. + + NVMLError is a parent class. Each NVML_ERROR_* gets it's own subclass. + e.g. NVML_ERROR_ALREADY_INITIALIZED will be turned into NVMLError_AlreadyInitialized + """ + this_module = sys.modules[__name__] + nvmlErrorsNames = [x for x in dir(this_module) if x.startswith("NVML_ERROR_")] + for err_name in nvmlErrorsNames: + # e.g. Turn NVML_ERROR_ALREADY_INITIALIZED into NVMLError_AlreadyInitialized + class_name = "NVMLError_" + string.capwords( + err_name.replace("NVML_ERROR_", ""), "_" + ).replace("_", "") + err_val = getattr(this_module, err_name) + + def gen_new(val): + def new(typ): + obj = NVMLError.__new__(typ, val) + return obj + + return new + + new_error_class = type(class_name, (NVMLError,), {"__new__": gen_new(err_val)}) + new_error_class.__module__ = __name__ + setattr(this_module, class_name, new_error_class) + NVMLError._valClassMapping[err_val] = new_error_class + + +_extractNVMLErrorsAsClasses() + + +def _nvmlCheckReturn(ret): + if ret != NVML_SUCCESS: + raise NVMLError(ret) + return ret + + +## Function access ## +_nvmlGetFunctionPointer_cache = ( + dict() +) # function pointers are cached to prevent unnecessary libLoadLock locking + + +def _nvmlGetFunctionPointer(name): + global nvmlLib + + if name in _nvmlGetFunctionPointer_cache: + return _nvmlGetFunctionPointer_cache[name] + + libLoadLock.acquire() + try: + # ensure library was loaded + if nvmlLib == None: + raise NVMLError(NVML_ERROR_UNINITIALIZED) + try: + _nvmlGetFunctionPointer_cache[name] = getattr(nvmlLib, name) + return _nvmlGetFunctionPointer_cache[name] + except AttributeError: + raise NVMLError(NVML_ERROR_FUNCTION_NOT_FOUND) + finally: + # lock is always freed + libLoadLock.release() + + +## Alternative object +# Allows the object to be printed +# Allows mismatched types to be assigned +# - like None when the Structure variant requires c_uint +class nvmlFriendlyObject(object): + def __init__(self, dictionary): + for x in dictionary: + setattr(self, x, dictionary[x]) + + def __str__(self): + return self.__dict__.__str__() + + +def nvmlStructToFriendlyObject(struct): + d = {} + for x in struct._fields_: + key = x[0] + value = getattr(struct, key) + # only need to convert from bytes if bytes, no need to check python version. + d[key] = value.decode() if isinstance(value, bytes) else value + obj = nvmlFriendlyObject(d) + return obj + + +# pack the object so it can be passed to the NVML library +def nvmlFriendlyObjectToStruct(obj, model): + for x in model._fields_: + key = x[0] + value = obj.__dict__[key] + # any c_char_p in python3 needs to be bytes, default encoding works fine. + if sys.version_info >= (3,): + setattr(model, key, value.encode()) + else: + setattr(model, key, value) + return model + + +## Unit structures +class struct_c_nvmlUnit_t(Structure): + pass # opaque handle + + +c_nvmlUnit_t = POINTER(struct_c_nvmlUnit_t) + + +class _PrintableStructure(Structure): + """ + Abstract class that produces nicer __str__ output than ctypes.Structure. + e.g. instead of: + >>> print str(obj) + + this class will print + class_name(field_name: formatted_value, field_name: formatted_value) + + _fmt_ dictionary of -> + e.g. class that has _field_ 'hex_value', c_uint could be formatted with + _fmt_ = {"hex_value" : "%08X"} + to produce nicer output. + Default formatting string for all fields can be set with key "" like: + _fmt_ = {"" : "%d MHz"} # e.g all values are numbers in MHz. + If not set it's assumed to be just "%s" + + Exact format of returned str from this class is subject to change in the future. + """ + + _fmt_ = {} + + def __str__(self): + result = [] + for x in self._fields_: + key = x[0] + value = getattr(self, key) + fmt = "%s" + if key in self._fmt_: + fmt = self._fmt_[key] + elif "" in self._fmt_: + fmt = self._fmt_[""] + result.append(("%s: " + fmt) % (key, value)) + return self.__class__.__name__ + "(" + ", ".join(result) + ")" + + def __getattribute__(self, name): + res = super(_PrintableStructure, self).__getattribute__(name) + # need to convert bytes to unicode for python3 don't need to for python2 + # Python 2 strings are of both str and bytes + # Python 3 strings are not of type bytes + # ctypes should convert everything to the correct values otherwise + if isinstance(res, bytes): + if isinstance(res, str): + return res + return res.decode() + return res + + def __setattr__(self, name, value): + if isinstance(value, str): + # encoding a python2 string returns the same value, since python2 strings are bytes already + # bytes passed in python3 will be ignored. + value = value.encode() + super(_PrintableStructure, self).__setattr__(name, value) + + +class c_nvmlUnitInfo_t(_PrintableStructure): + _fields_ = [ + ("name", c_char * 96), + ("id", c_char * 96), + ("serial", c_char * 96), + ("firmwareVersion", c_char * 96), + ] + + +class c_nvmlC2cModeInfo_v1_t(_PrintableStructure): + _fields_ = [("isC2cEnabled", c_uint)] + + +nvmlC2cModeInfo_v1 = 0x1000008 + + +class c_nvmlLedState_t(_PrintableStructure): + _fields_ = [ + ("cause", c_char * 256), + ("color", _nvmlLedColor_t), + ] + + +class c_nvmlPSUInfo_t(_PrintableStructure): + _fields_ = [ + ("state", c_char * 256), + ("current", c_uint), + ("voltage", c_uint), + ("power", c_uint), + ] + + +class c_nvmlUnitFanInfo_t(_PrintableStructure): + _fields_ = [ + ("speed", c_uint), + ("state", _nvmlFanState_t), + ] + + +class c_nvmlUnitFanSpeeds_t(_PrintableStructure): + _fields_ = [("fans", c_nvmlUnitFanInfo_t * 24), ("count", c_uint)] + + +## Device structures +class struct_c_nvmlDevice_t(Structure): + pass # opaque handle + + +c_nvmlDevice_t = POINTER(struct_c_nvmlDevice_t) + + +class nvmlPciInfoExt_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("domain", c_uint), + ("bus", c_uint), + ("device", c_uint), + ("pciDeviceId", c_uint), + ("pciSubSystemId", c_uint), + ("baseClass", c_uint), + ("subClass", c_uint), + ("busId", c_char * NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE), + ] + _fmt_ = { + "version": "0x%04X", + "domain": "0x%04X", + "bus": "0x%02X", + "device": "0x%02X", + "pciDeviceId": "0x%08X", + "pciSubSystemId": "0x%08X", + "baseClass": "0x%01X", + "subClass": "0x%01X", + } + + +nvmlPciInfoExt_v1 = 0x1000040 + + +# Legacy pciInfo used for _v1 and _v2 +class nvmlPciInfo_v2_t(_PrintableStructure): + _fields_ = [ + ("busId", c_char * NVML_DEVICE_PCI_BUS_ID_BUFFER_V2_SIZE), + ("domain", c_uint), + ("bus", c_uint), + ("device", c_uint), + ("pciDeviceId", c_uint), + # Added in 2.285 + ("pciSubSystemId", c_uint), + ("reserved0", c_uint), + ("reserved1", c_uint), + ("reserved2", c_uint), + ("reserved3", c_uint), + ] + _fmt_ = { + "domain": "0x%04X", + "bus": "0x%02X", + "device": "0x%02X", + "pciDeviceId": "0x%08X", + "pciSubSystemId": "0x%08X", + } + + +class nvmlPciInfo_t(_PrintableStructure): + _fields_ = [ + # Moved to the new busId location below + ("busIdLegacy", c_char * NVML_DEVICE_PCI_BUS_ID_BUFFER_V2_SIZE), + ("domain", c_uint), + ("bus", c_uint), + ("device", c_uint), + ("pciDeviceId", c_uint), + # Added in 2.285 + ("pciSubSystemId", c_uint), + # New busId replaced the long deprecated and reserved fields with a + # field of the same size in 9.0 + ("busId", c_char * NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE), + ] + _fmt_ = { + "domain": "0x%08X", + "bus": "0x%02X", + "device": "0x%02X", + "pciDeviceId": "0x%08X", + "pciSubSystemId": "0x%08X", + } + + +class c_nvmlSystemDriverBranchInfo_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("branch", c_char * NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE), + ] + + +SystemDriverBranchInfo_v1 = 0x1000054 + + +class c_nvmlExcludedDeviceInfo_t(_PrintableStructure): + _fields_ = [("pci", nvmlPciInfo_t), ("uuid", c_char * NVML_DEVICE_UUID_BUFFER_SIZE)] + + +class nvmlNvLinkUtilizationControl_t(_PrintableStructure): + _fields_ = [ + ("units", _nvmlNvLinkUtilizationCountUnits_t), + ("pktfilter", _nvmlNvLinkUtilizationCountPktTypes_t), + ] + + +class c_nvmlMemory_t(_PrintableStructure): + _fields_ = [ + ("total", c_ulonglong), + ("free", c_ulonglong), + ("used", c_ulonglong), + ] + _fmt_ = {"": "%d B"} + + +class c_nvmlMemory_v2_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("total", c_ulonglong), + ("reserved", c_ulonglong), + ("free", c_ulonglong), + ("used", c_ulonglong), + ] + _fmt_ = {"": "%d B"} + + +nvmlMemory_v2 = 0x02000028 + + +class c_nvmlBAR1Memory_t(_PrintableStructure): + _fields_ = [ + ("bar1Total", c_ulonglong), + ("bar1Free", c_ulonglong), + ("bar1Used", c_ulonglong), + ] + _fmt_ = {"": "%d B"} + + +class nvmlClkMonFaultInfo_t(Structure): + _fields_ = [("clkApiDomain", c_uint), ("clkDomainFaultMask", c_uint)] + + +MAX_CLK_DOMAINS = 32 + + +class nvmlClkMonStatus_t(Structure): + _fields_ = [ + ("bGlobalStatus", c_uint), + ("clkMonListSize", c_uint), + ("clkMonList", nvmlClkMonFaultInfo_t * MAX_CLK_DOMAINS), + ] + + +# On Windows with the WDDM driver, usedGpuMemory is reported as None +# Code that processes this structure should check for None, I.E. +# +# if (info.usedGpuMemory == None): +# # TODO handle the error +# pass +# else: +# print("Using %d MiB of memory" % (info.usedGpuMemory / 1024 / 1024)) +# endif +# +# See NVML documentation for more information +class c_nvmlProcessInfo_v2_t(_PrintableStructure): + _fields_ = [ + ("pid", c_uint), + ("usedGpuMemory", c_ulonglong), + ("gpuInstanceId", c_uint), + ("computeInstanceId", c_uint), + ] + _fmt_ = {"usedGpuMemory": "%d B"} + + +c_nvmlProcessInfo_v3_t = c_nvmlProcessInfo_v2_t + +c_nvmlProcessInfo_t = c_nvmlProcessInfo_v3_t + +_nvmlProcessMode_t = c_uint +NVML_PROCESS_MODE_COMPUTE = 0 +NVML_PROCESS_MODE_GRAPHICS = 1 +NVML_PROCESS_MODE_MPS = 2 + + +class c_nvmlProcessDetail_v1_t(Structure): + _fields_ = [ + ("pid", c_uint), + ("usedGpuMemory", c_ulonglong), + ("gpuInstanceId", c_uint), + ("computeInstanceId", c_uint), + ("usedGpuCcProtectedMemory", c_ulonglong), + ] + + +class c_nvmlProcessDetailList_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("mode", _nvmlProcessMode_t), + ("numProcArrayEntries", c_uint), + ("procArray", POINTER(c_nvmlProcessDetail_v1_t)), + ] + _fmt_ = {"numProcArrayEntries": "%d B"} + + +c_nvmlProcessDetailList_t = c_nvmlProcessDetailList_v1_t + +nvmlProcessDetailList_v1 = 0x1000018 + + +class c_nvmlBridgeChipInfo_t(_PrintableStructure): + _fields_ = [ + ("type", _nvmlBridgeChipType_t), + ("fwVersion", c_uint), + ] + + +class c_nvmlBridgeChipHierarchy_t(_PrintableStructure): + _fields_ = [ + ("bridgeCount", c_uint), + ("bridgeChipInfo", c_nvmlBridgeChipInfo_t * 128), + ] + + +class c_nvmlEccErrorCounts_t(_PrintableStructure): + _fields_ = [ + ("l1Cache", c_ulonglong), + ("l2Cache", c_ulonglong), + ("deviceMemory", c_ulonglong), + ("registerFile", c_ulonglong), + ] + + +class c_nvmlUtilization_t(_PrintableStructure): + _fields_ = [ + ("gpu", c_uint), + ("memory", c_uint), + ] + _fmt_ = {"": "%d %%"} + + +# Added in 2.285 +class c_nvmlHwbcEntry_t(_PrintableStructure): + _fields_ = [ + ("hwbcId", c_uint), + ("firmwareVersion", c_char * 32), + ] + + +class c_nvmlValue_t(Union): + _fields_ = [ + ("dVal", c_double), + ("uiVal", c_uint), + ("ulVal", c_ulong), + ("ullVal", c_ulonglong), + ("sllVal", c_longlong), + ("siVal", c_int), + ("usVal", c_ushort), + ] + + +class c_nvmlSample_t(_PrintableStructure): + _fields_ = [ + ("timeStamp", c_ulonglong), + ("sampleValue", c_nvmlValue_t), + ] + + +class c_nvmlViolationTime_t(_PrintableStructure): + _fields_ = [ + ("referenceTime", c_ulonglong), + ("violationTime", c_ulonglong), + ] + + +class c_nvmlFieldValue_t(_PrintableStructure): + _fields_ = [ + ("fieldId", c_uint32), + ("scopeId", c_uint32), + ("timestamp", c_int64), + ("latencyUsec", c_int64), + ("valueType", _nvmlValueType_t), + ("nvmlReturn", _nvmlReturn_t), + ("value", c_nvmlValue_t), + ] + + +NVML_NVLINK_TOTAL_SUPPORTED_BW_MODES = 23 + +nvmlNvlinkSupportedBwModes_v1 = 0x100001C + + +class c_nvmlNvlinkSupportedBwModes_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("bwModes", c_uint8 * NVML_NVLINK_TOTAL_SUPPORTED_BW_MODES), + ("totalBwModes", c_uint8), + ] + + def __init__(self): + super(c_nvmlNvlinkSupportedBwModes_v1_t, self).__init__( + version=nvmlNvlinkSupportedBwModes_v1 + ) + + +nvmlNvlinkGetBwMode_v1 = 0x100000C + + +class c_nvmlNvlinkGetBwMode_v1_t(_PrintableStructure): + _fields_ = [("version", c_uint), ("bIsBest", c_uint), ("bwMode", c_uint8)] + + def __init__(self): + super(c_nvmlNvlinkGetBwMode_v1_t, self).__init__(version=nvmlNvlinkGetBwMode_v1) + + +nvmlNvlinkSetBwMode_v1 = 0x100000C + + +class c_nvmlNvlinkSetBwMode_v1_t(_PrintableStructure): + _fields_ = [("version", c_uint), ("bSetBest", c_uint), ("bwMode", c_uint8)] + + def __init__(self): + super(c_nvmlNvlinkSetBwMode_v1_t, self).__init__(version=nvmlNvlinkSetBwMode_v1) + + +class c_nvmlVgpuHeterogeneousMode_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("mode", c_uint), + ] + + +VgpuHeterogeneousMode_v1 = 0x1000008 + + +class c_nvmlVgpuPlacementId_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("placementId", c_uint), + ] + + +VgpuPlacementId_v1 = 0x1000008 + + +class c_nvmlVgpuPlacementList_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("count", c_uint), + ("placementSize", c_uint), + ("placementIds", POINTER(c_uint)), + ] + + +VgpuPlacementList_v1 = 0x1000018 + +NVML_VGPU_PGPU_HETEROGENEOUS_MODE = 0 +NVML_VGPU_PGPU_HOMOGENEOUS_MODE = 1 + + +class c_nvmlVgpuPlacementList_v2_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("placementSize", c_uint), + ("count", c_uint), + ("placementIds", POINTER(c_uint)), + ("mode", c_uint), + ] + + +VgpuPlacementList_v2 = 0x2000020 + + +class c_nvmlVgpuTypeBar1Info_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("bar1Size", c_ulonglong), + ] + + +VgpuTypeBar1Info_v1 = 0x1000010 + + +class c_nvmlVgpuInstanceUtilizationSample_t(_PrintableStructure): + _fields_ = [ + ("vgpuInstance", _nvmlVgpuInstance_t), + ("timeStamp", c_ulonglong), + ("smUtil", c_nvmlValue_t), + ("memUtil", c_nvmlValue_t), + ("encUtil", c_nvmlValue_t), + ("decUtil", c_nvmlValue_t), + ] + + +class c_nvmlVgpuInstanceUtilizationInfo_v1_t(_PrintableStructure): + _fields_ = [ + ("timeStamp", c_ulonglong), + ("vgpuInstance", _nvmlVgpuInstance_t), + ("smUtil", c_nvmlValue_t), + ("memUtil", c_nvmlValue_t), + ("encUtil", c_nvmlValue_t), + ("decUtil", c_nvmlValue_t), + ("jpgUtil", c_nvmlValue_t), + ("ofaUtil", c_nvmlValue_t), + ] + + +class c_nvmlVgpuInstancesUtilizationInfo_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("sampleValType", _nvmlValueType_t), + ("vgpuInstanceCount", c_uint), + ("lastSeenTimeStamp", c_ulonglong), + ("vgpuUtilArray", POINTER(c_nvmlVgpuInstanceUtilizationInfo_v1_t)), + ] + + +VgpuInstancesUtilizationInfo_v1 = 0x01000020 + + +class c_nvmlVgpuProcessUtilizationSample_t(_PrintableStructure): + _fields_ = [ + ("vgpuInstance", _nvmlVgpuInstance_t), + ("pid", c_uint), + ("processName", c_char * NVML_VGPU_NAME_BUFFER_SIZE), + ("timeStamp", c_ulonglong), + ("smUtil", c_uint), + ("memUtil", c_uint), + ("encUtil", c_uint), + ("decUtil", c_uint), + ] + + +class c_nvmlVgpuProcessUtilizationInfo_v1_t(_PrintableStructure): + _fields_ = [ + ("processName", c_char * NVML_VGPU_NAME_BUFFER_SIZE), + ("timeStamp", c_ulonglong), + ("vgpuInstance", _nvmlVgpuInstance_t), + ("pid", c_uint), + ("smUtil", c_uint), + ("memUtil", c_uint), + ("encUtil", c_uint), + ("decUtil", c_uint), + ("jpgUtil", c_uint), + ("ofaUtil", c_uint), + ] + + +class c_nvmlVgpuProcessesUtilizationInfo_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("vgpuProcessCount", c_uint), + ("lastSeenTimeStamp", c_ulonglong), + ("vgpuProcUtilArray", POINTER(c_nvmlVgpuProcessUtilizationInfo_v1_t)), + ] + + +VgpuProcessesUtilizationInfo_v1 = 0x01000018 + + +class nvmlVgpuRuntimeState_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("size", c_ulonglong), + ] + + +VgpuRuntimeState_v1 = 0x1000010 + + +class c_nvmlVgpuLicenseExpiry_t(_PrintableStructure): + _fields_ = [ + ("year", c_uint32), + ("month", c_uint16), + ("day", c_uint16), + ("hour", c_uint16), + ("min", c_uint16), + ("sec", c_uint16), + ("status", c_uint8), + ] + + +NVML_GRID_LICENSE_STATE_UNKNOWN = 0 +NVML_GRID_LICENSE_STATE_UNINITIALIZED = 1 +NVML_GRID_LICENSE_STATE_UNLICENSED_UNRESTRICTED = 2 +NVML_GRID_LICENSE_STATE_UNLICENSED_RESTRICTED = 3 +NVML_GRID_LICENSE_STATE_UNLICENSED = 4 +NVML_GRID_LICENSE_STATE_LICENSED = 5 + + +class c_nvmlVgpuLicenseInfo_t(_PrintableStructure): + _fields_ = [ + ("isLicensed", c_uint8), + ("licenseExpiry", c_nvmlVgpuLicenseExpiry_t), + ("currentState", c_uint), + ] + + +class c_nvmlEncoderSession_t(_PrintableStructure): + _fields_ = [ + ("sessionId", c_uint), + ("pid", c_uint), + ("vgpuInstance", _nvmlVgpuInstance_t), + ("codecType", c_uint), + ("hResolution", c_uint), + ("vResolution", c_uint), + ("averageFps", c_uint), + ("encodeLatency", c_uint), + ] + + +class c_nvmlProcessUtilizationSample_t(_PrintableStructure): + _fields_ = [ + ("pid", c_uint), + ("timeStamp", c_ulonglong), + ("smUtil", c_uint), + ("memUtil", c_uint), + ("encUtil", c_uint), + ("decUtil", c_uint), + ] + + +class c_nvmlProcessUtilizationInfo_v1_t(_PrintableStructure): + _fields_ = [ + ("timeStamp", c_ulonglong), + ("pid", c_uint), + ("smUtil", c_uint), + ("memUtil", c_uint), + ("encUtil", c_uint), + ("decUtil", c_uint), + ("jpgUtil", c_uint), + ("ofaUtil", c_uint), + ] + + +class c_nvmlProcessesUtilizationInfo_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("processSamplesCount", c_uint), + ("lastSeenTimeStamp", c_ulonglong), + ("procUtilArray", POINTER(c_nvmlProcessUtilizationInfo_v1_t)), + ] + + +ProcessesUtilizationInfo_v1 = 0x01000018 + + +class c_nvmlGridLicenseExpiry_t(_PrintableStructure): + _fields_ = [ + ("year", c_uint32), + ("month", c_uint16), + ("day", c_uint16), + ("hour", c_uint16), + ("min", c_uint16), + ("sec", c_uint16), + ("status", c_uint8), + ] + + +class c_nvmlGridLicensableFeature_v4_t(_PrintableStructure): + _fields_ = [ + ("featureCode", _nvmlGridLicenseFeatureCode_t), + ("featureState", c_uint), + ("licenseInfo", c_char * NVML_GRID_LICENSE_BUFFER_SIZE), + ("productName", c_char * NVML_GRID_LICENSE_BUFFER_SIZE), + ("featureEnabled", c_uint), + ("licenseExpiry", c_nvmlGridLicenseExpiry_t), + ] + + +class c_nvmlGridLicensableFeatures_v4_t(_PrintableStructure): + _fields_ = [ + ("isGridLicenseSupported", c_int), + ("licensableFeaturesCount", c_uint), + ( + "gridLicensableFeatures", + c_nvmlGridLicensableFeature_v4_t * NVML_GRID_LICENSE_FEATURE_MAX_COUNT, + ), + ] + + +class c_nvmlGridLicensableFeature_v3_t(_PrintableStructure): + _fields_ = [ + ("featureCode", _nvmlGridLicenseFeatureCode_t), + ("featureState", c_uint), + ("licenseInfo", c_char * NVML_GRID_LICENSE_BUFFER_SIZE), + ("productName", c_char * NVML_GRID_LICENSE_BUFFER_SIZE), + ("featureEnabled", c_uint), + ] + + +class c_nvmlGridLicensableFeatures_v3_t(_PrintableStructure): + _fields_ = [ + ("isGridLicenseSupported", c_int), + ("licensableFeaturesCount", c_uint), + ( + "gridLicensableFeatures", + c_nvmlGridLicensableFeature_v3_t * NVML_GRID_LICENSE_FEATURE_MAX_COUNT, + ), + ] + + +class c_nvmlGridLicensableFeature_v2_t(_PrintableStructure): + _fields_ = [ + ("featureCode", _nvmlGridLicenseFeatureCode_t), + ("featureState", c_uint), + ("licenseInfo", c_char * NVML_GRID_LICENSE_BUFFER_SIZE), + ("productName", c_char * NVML_GRID_LICENSE_BUFFER_SIZE), + ] + + +class c_nvmlGridLicensableFeatures_v2_t(_PrintableStructure): + _fields_ = [ + ("isGridLicenseSupported", c_int), + ("licensableFeaturesCount", c_uint), + ( + "gridLicensableFeatures", + c_nvmlGridLicensableFeature_v2_t * NVML_GRID_LICENSE_FEATURE_MAX_COUNT, + ), + ] + + +class c_nvmlGridLicensableFeature_t(_PrintableStructure): + _fields_ = [ + ("featureCode", _nvmlGridLicenseFeatureCode_t), + ("featureState", c_uint), + ("licenseInfo", c_char * NVML_GRID_LICENSE_BUFFER_SIZE), + ] + + +class c_nvmlGridLicensableFeatures_t(_PrintableStructure): + _fields_ = [ + ("isGridLicenseSupported", c_int), + ("licensableFeaturesCount", c_uint), + ( + "gridLicensableFeatures", + c_nvmlGridLicensableFeature_t * NVML_GRID_LICENSE_FEATURE_MAX_COUNT, + ), + ] + + +class c_nvmlMarginTemperature_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("marginTemperature", c_int), + ] + + +nvmlMarginTemperature_v1 = 0x1000008 + + +## Event structures +class struct_c_nvmlEventSet_t(Structure): + pass # opaque handle + + +c_nvmlEventSet_t = POINTER(struct_c_nvmlEventSet_t) + +nvmlEventTypeSingleBitEccError = 0x0000000000000001 +nvmlEventTypeDoubleBitEccError = 0x0000000000000002 +nvmlEventTypePState = 0x0000000000000004 +nvmlEventTypeXidCriticalError = 0x0000000000000008 +nvmlEventTypeClock = 0x0000000000000010 +nvmlEventTypePowerSourceChange = 0x0000000000000080 +nvmlEventMigConfigChange = 0x0000000000000100 +nvmlEventTypeSingleBitEccErrorStorm = 0x0000000000000200 +nvmlEventTypeDramRetirementEvent = 0x0000000000000400 +nvmlEventTypeDramRetirementFailure = 0x0000000000000800 +nvmlEventTypeNonFatalPoisonError = 0x0000000000001000 +nvmlEventTypeFatalPoisonError = 0x0000000000002000 +nvmlEventTypeGpuUnavailableError = 0x0000000000004000 +nvmlEventTypeGpuRecoveryAction = 0x0000000000008000 +nvmlEventTypeNone = 0x0000000000000000 +nvmlEventTypeAll = ( + nvmlEventTypeNone + | nvmlEventTypeSingleBitEccError + | nvmlEventTypeDoubleBitEccError + | nvmlEventTypePState + | nvmlEventTypeClock + | nvmlEventTypePowerSourceChange + | nvmlEventTypeXidCriticalError + | nvmlEventMigConfigChange + | nvmlEventTypeSingleBitEccErrorStorm + | nvmlEventTypeDramRetirementEvent + | nvmlEventTypeDramRetirementFailure + | nvmlEventTypeNonFatalPoisonError + | nvmlEventTypeFatalPoisonError + | nvmlEventTypeGpuUnavailableError + | nvmlEventTypeGpuRecoveryAction +) + +## Clock Event Reasons defines +nvmlClocksEventReasonGpuIdle = 0x0000000000000001 +nvmlClocksEventReasonApplicationsClocksSetting = 0x0000000000000002 +nvmlClocksEventReasonUserDefinedClocks = nvmlClocksEventReasonApplicationsClocksSetting # deprecated, use nvmlClocksEventReasonApplicationsClocksSetting +nvmlClocksEventReasonSwPowerCap = 0x0000000000000004 +nvmlClocksEventReasonHwSlowdown = 0x0000000000000008 +nvmlClocksEventReasonSyncBoost = 0x0000000000000010 +nvmlClocksEventReasonSwThermalSlowdown = 0x0000000000000020 +nvmlClocksEventReasonHwThermalSlowdown = 0x0000000000000040 +nvmlClocksEventReasonHwPowerBrakeSlowdown = 0x0000000000000080 +nvmlClocksEventReasonDisplayClockSetting = 0x0000000000000100 +nvmlClocksEventReasonNone = 0x0000000000000000 +nvmlClocksEventReasonAll = ( + nvmlClocksEventReasonNone + | nvmlClocksEventReasonGpuIdle + | nvmlClocksEventReasonApplicationsClocksSetting + | nvmlClocksEventReasonSwPowerCap + | nvmlClocksEventReasonHwSlowdown + | nvmlClocksEventReasonSyncBoost + | nvmlClocksEventReasonSwThermalSlowdown + | nvmlClocksEventReasonHwThermalSlowdown + | nvmlClocksEventReasonHwPowerBrakeSlowdown + | nvmlClocksEventReasonDisplayClockSetting +) + +## Following have been deprecated +nvmlClocksThrottleReasonGpuIdle = 0x0000000000000001 +nvmlClocksThrottleReasonApplicationsClocksSetting = 0x0000000000000002 +nvmlClocksThrottleReasonUserDefinedClocks = nvmlClocksThrottleReasonApplicationsClocksSetting # deprecated, use nvmlClocksThrottleReasonApplicationsClocksSetting +nvmlClocksThrottleReasonSwPowerCap = 0x0000000000000004 +nvmlClocksThrottleReasonHwSlowdown = 0x0000000000000008 +nvmlClocksThrottleReasonSyncBoost = 0x0000000000000010 +nvmlClocksThrottleReasonSwThermalSlowdown = 0x0000000000000020 +nvmlClocksThrottleReasonHwThermalSlowdown = 0x0000000000000040 +nvmlClocksThrottleReasonHwPowerBrakeSlowdown = 0x0000000000000080 +nvmlClocksThrottleReasonDisplayClockSetting = 0x0000000000000100 +nvmlClocksThrottleReasonNone = 0x0000000000000000 +nvmlClocksThrottleReasonAll = ( + nvmlClocksThrottleReasonNone + | nvmlClocksThrottleReasonGpuIdle + | nvmlClocksThrottleReasonApplicationsClocksSetting + | nvmlClocksThrottleReasonSwPowerCap + | nvmlClocksThrottleReasonHwSlowdown + | nvmlClocksThrottleReasonSyncBoost + | nvmlClocksThrottleReasonSwThermalSlowdown + | nvmlClocksThrottleReasonHwThermalSlowdown + | nvmlClocksThrottleReasonHwPowerBrakeSlowdown + | nvmlClocksThrottleReasonDisplayClockSetting +) + + +class c_nvmlEventData_t(_PrintableStructure): + _fields_ = [ + ("device", c_nvmlDevice_t), + ("eventType", c_ulonglong), + ("eventData", c_ulonglong), + ("gpuInstanceId", c_uint), + ("computeInstanceId", c_uint), + ] + _fmt_ = {"eventType": "0x%08X"} + + +class c_nvmlAccountingStats_t(_PrintableStructure): + _fields_ = [ + ("gpuUtilization", c_uint), + ("memoryUtilization", c_uint), + ("maxMemoryUsage", c_ulonglong), + ("time", c_ulonglong), + ("startTime", c_ulonglong), + ("isRunning", c_uint), + ("reserved", c_uint * 5), + ] + + +class c_nvmlVgpuVersion_t(Structure): + _fields_ = [("minVersion", c_uint), ("maxVersion", c_uint)] + + +class c_nvmlVgpuMetadata_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("revision", c_uint), + ("guestInfoState", _nvmlVgpuGuestInfoState_t), + ("guestDriverVersion", c_char * NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE), + ("hostDriverVersion", c_char * NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE), + ("reserved", c_uint * 6), + ("vgpuVirtualizationCaps", c_uint), + ("guestVgpuVersion", c_uint), + ("opaqueDataSize", c_uint), + ("opaqueData", c_char * NVML_VGPU_METADATA_OPAQUE_DATA_SIZE), + ] + + +class c_nvmlVgpuPgpuMetadata_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("revision", c_uint), + ("hostDriverVersion", c_char * NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE), + ("pgpuVirtualizationCaps", c_uint), + ("reserved", c_uint * 5), + ("hostSupportedVgpuRange", c_nvmlVgpuVersion_t), + ("opaqueDataSize", c_uint), + ("opaqueData", c_char * NVML_VGPU_PGPU_METADATA_OPAQUE_DATA_SIZE), + ] + + +class c_nvmlVgpuPgpuCompatibility_t(Structure): + _fields_ = [ + ("vgpuVmCompatibility", _nvmlVgpuVmCompatibility_t), + ("compatibilityLimitCode", _nvmlVgpuPgpuCompatibilityLimitCode_t), + ] + + +## vGPU scheduler policy defines +NVML_VGPU_SCHEDULER_POLICY_UNKNOWN = 0 +NVML_VGPU_SCHEDULER_POLICY_BEST_EFFORT = 1 +NVML_VGPU_SCHEDULER_POLICY_EQUAL_SHARE = 2 +NVML_VGPU_SCHEDULER_POLICY_FIXED_SHARE = 3 + +## Supported vGPU scheduler policy count +NVML_SUPPORTED_VGPU_SCHEDULER_POLICY_COUNT = 3 + +NVML_SCHEDULER_SW_MAX_LOG_ENTRIES = 200 + +NVML_VGPU_SCHEDULER_ARR_DEFAULT = 0 +NVML_VGPU_SCHEDULER_ARR_DISABLE = 1 +NVML_VGPU_SCHEDULER_ARR_ENABLE = 2 + + +class c_nvmlVgpuSchedDataWithARR_t(_PrintableStructure): + _fields_ = [ + ("avgFactor", c_uint), + ("timeslice", c_uint), + ] + + +class c_nvmlVgpuSchedData_t(_PrintableStructure): + _fields_ = [ + ("timeslice", c_uint), + ] + + +class c_nvmlVgpuSchedulerParams_t(Union): + _fields_ = [ + ("vgpuSchedDataWithARR", c_nvmlVgpuSchedDataWithARR_t), + ("vgpuSchedData", c_nvmlVgpuSchedData_t), + ] + + +class c_nvmlVgpuSchedulerLogEntry_t(_PrintableStructure): + _fields_ = [ + ("timestamp", c_ulonglong), + ("timeRunTotal", c_ulonglong), + ("timeRun", c_ulonglong), + ("swRunlistId", c_uint), + ("targetTimeSlice", c_ulonglong), + ("cumulativePreemptionTime", c_ulonglong), + ] + + +class c_nvmlVgpuSchedulerLog_t(_PrintableStructure): + _fields_ = [ + ("engineId", c_uint), + ("schedulerPolicy", c_uint), + ("arrMode", c_uint), + ("schedulerParams", c_nvmlVgpuSchedulerParams_t), + ("entriesCount", c_uint), + ( + "logEntries", + c_nvmlVgpuSchedulerLogEntry_t * NVML_SCHEDULER_SW_MAX_LOG_ENTRIES, + ), + ] + + +class c_nvmlVgpuSchedulerGetState_t(_PrintableStructure): + _fields_ = [ + ("schedulerPolicy", c_uint), + ("arrMode", c_uint), + ("schedulerParams", c_nvmlVgpuSchedulerParams_t), + ] + + +class c_nvmlVgpuSchedSetDataWithARR_t(_PrintableStructure): + _fields_ = [ + ("avgFactor", c_uint), + ("frequency", c_uint), + ] + + +class c_nvmlVgpuSchedSetData_t(_PrintableStructure): + _fields_ = [ + ("timeslice", c_uint), + ] + + +class c_nvmlVgpuSchedulerSetParams_t(Union): + _fields_ = [ + ("vgpuSchedDataWithARR", c_nvmlVgpuSchedSetDataWithARR_t), + ("vgpuSchedData", c_nvmlVgpuSchedSetData_t), + ] + + +class c_nvmlVgpuSchedulerSetState_t(_PrintableStructure): + _fields_ = [ + ("schedulerPolicy", c_uint), + ("enableARRMode", c_uint), + ("schedulerParams", c_nvmlVgpuSchedulerSetParams_t), + ] + + +class c_nvmlVgpuSchedulerCapabilities_t(_PrintableStructure): + _fields_ = [ + ("supportedSchedulers", c_uint * NVML_SUPPORTED_VGPU_SCHEDULER_POLICY_COUNT), + ("maxTimeslice", c_uint), + ("minTimeslice", c_uint), + ("isArrModeSupported", c_uint), + ("maxFrequencyForARR", c_uint), + ("minFrequencyForARR", c_uint), + ("maxAvgFactorForARR", c_uint), + ("minAvgFactorForARR", c_uint), + ] + + +class c_nvmlFBCStats_t(Structure): + _fields_ = [ + ("sessionsCount", c_uint), + ("averageFPS", c_uint), + ("averageLatency", c_uint), + ] + + +class c_nvmlFBCSession_t(_PrintableStructure): + _fields_ = [ + ("sessionId", c_uint), + ("pid", c_uint), + ("vgpuInstance", _nvmlVgpuInstance_t), + ("displayOrdinal", c_uint), + ("sessionType", c_uint), + ("sessionFlags", c_uint), + ("hMaxResolution", c_uint), + ("vMaxResolution", c_uint), + ("hResolution", c_uint), + ("vResolution", c_uint), + ("averageFPS", c_uint), + ("averageLatency", c_uint), + ] + + +NVML_DEVICE_MIG_DISABLE = 0x0 +NVML_DEVICE_MIG_ENABLE = 0x1 + +NVML_GPU_INSTANCE_PROFILE_1_SLICE = 0x0 +NVML_GPU_INSTANCE_PROFILE_2_SLICE = 0x1 +NVML_GPU_INSTANCE_PROFILE_3_SLICE = 0x2 +NVML_GPU_INSTANCE_PROFILE_4_SLICE = 0x3 +NVML_GPU_INSTANCE_PROFILE_7_SLICE = 0x4 +NVML_GPU_INSTANCE_PROFILE_8_SLICE = 0x5 +NVML_GPU_INSTANCE_PROFILE_6_SLICE = 0x6 +NVML_GPU_INSTANCE_PROFILE_1_SLICE_REV1 = 0x7 +NVML_GPU_INSTANCE_PROFILE_2_SLICE_REV1 = 0x8 +NVML_GPU_INSTANCE_PROFILE_1_SLICE_REV2 = 0x9 +NVML_GPU_INSTANCE_PROFILE_1_SLICE_GFX = 0xA +NVML_GPU_INSTANCE_PROFILE_2_SLICE_GFX = 0xB +NVML_GPU_INSTANCE_PROFILE_4_SLICE_GFX = 0xC +NVML_GPU_INSTANCE_PROFILE_COUNT = 0xD + + +class c_nvmlGpuInstancePlacement_t(Structure): + _fields_ = [("start", c_uint), ("size", c_uint)] + + +class c_nvmlGpuInstanceProfileInfo_t(Structure): + _fields_ = [ + ("id", c_uint), + ("isP2pSupported", c_uint), + ("sliceCount", c_uint), + ("instanceCount", c_uint), + ("multiprocessorCount", c_uint), + ("copyEngineCount", c_uint), + ("decoderCount", c_uint), + ("encoderCount", c_uint), + ("jpegCount", c_uint), + ("ofaCount", c_uint), + ("memorySizeMB", c_ulonglong), + ] + + +nvmlGpuInstanceProfileInfo_v2 = 0x02000098 + + +class c_nvmlGpuInstanceProfileInfo_v2_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("id", c_uint), + ("isP2pSupported", c_uint), + ("sliceCount", c_uint), + ("instanceCount", c_uint), + ("multiprocessorCount", c_uint), + ("copyEngineCount", c_uint), + ("decoderCount", c_uint), + ("encoderCount", c_uint), + ("jpegCount", c_uint), + ("ofaCount", c_uint), + ("memorySizeMB", c_ulonglong), + ("name", c_char * NVML_DEVICE_NAME_V2_BUFFER_SIZE), + ] + + def __init__(self): + super(c_nvmlGpuInstanceProfileInfo_v2_t, self).__init__( + version=nvmlGpuInstanceProfileInfo_v2 + ) + + +class c_nvmlGpuInstanceInfo_t(Structure): + _fields_ = [ + ("device", c_nvmlDevice_t), + ("id", c_uint), + ("profileId", c_uint), + ("placement", c_nvmlGpuInstancePlacement_t), + ] + + +class struct_c_nvmlGpuInstance_t(Structure): + pass # opaque handle + + +c_nvmlGpuInstance_t = POINTER(struct_c_nvmlGpuInstance_t) + +NVML_COMPUTE_INSTANCE_PROFILE_1_SLICE = 0x0 +NVML_COMPUTE_INSTANCE_PROFILE_2_SLICE = 0x1 +NVML_COMPUTE_INSTANCE_PROFILE_3_SLICE = 0x2 +NVML_COMPUTE_INSTANCE_PROFILE_4_SLICE = 0x3 +NVML_COMPUTE_INSTANCE_PROFILE_7_SLICE = 0x4 +NVML_COMPUTE_INSTANCE_PROFILE_8_SLICE = 0x5 +NVML_COMPUTE_INSTANCE_PROFILE_6_SLICE = 0x6 +NVML_COMPUTE_INSTANCE_PROFILE_1_SLICE_REV1 = 0x7 +NVML_COMPUTE_INSTANCE_PROFILE_COUNT = 0x8 + +NVML_COMPUTE_INSTANCE_ENGINE_PROFILE_SHARED = 0x0 +NVML_COMPUTE_INSTANCE_ENGINE_PROFILE_COUNT = 0x1 + + +class c_nvmlComputeInstancePlacement_t(Structure): + _fields_ = [("start", c_uint), ("size", c_uint)] + + +class c_nvmlComputeInstanceProfileInfo_t(Structure): + _fields_ = [ + ("id", c_uint), + ("sliceCount", c_uint), + ("instanceCount", c_uint), + ("multiprocessorCount", c_uint), + ("sharedCopyEngineCount", c_uint), + ("sharedDecoderCount", c_uint), + ("sharedEncoderCount", c_uint), + ("sharedJpegCount", c_uint), + ("sharedOfaCount", c_uint), + ] + + +nvmlComputeInstanceProfileInfo_v2 = 0x02000088 + + +class c_nvmlComputeInstanceProfileInfo_v2_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("id", c_uint), + ("sliceCount", c_uint), + ("instanceCount", c_uint), + ("multiprocessorCount", c_uint), + ("sharedCopyEngineCount", c_uint), + ("sharedDecoderCount", c_uint), + ("sharedEncoderCount", c_uint), + ("sharedJpegCount", c_uint), + ("sharedOfaCount", c_uint), + ("name", c_char * NVML_DEVICE_NAME_V2_BUFFER_SIZE), + ] + + def __init__(self): + super(c_nvmlComputeInstanceProfileInfo_v2_t, self).__init__( + version=nvmlComputeInstanceProfileInfo_v2 + ) + + +class c_nvmlComputeInstanceInfo_t(Structure): + _fields_ = [ + ("device", c_nvmlDevice_t), + ("gpuInstance", c_nvmlGpuInstance_t), + ("id", c_uint), + ("profileId", c_uint), + ("placement", c_nvmlComputeInstancePlacement_t), + ] + + +NVML_MAX_GPU_UTILIZATIONS = 8 +NVML_GPU_UTILIZATION_DOMAIN_GPU = 0 +NVML_GPU_UTILIZATION_DOMAIN_FB = 1 +NVML_GPU_UTILIZATION_DOMAIN_VID = 2 +NVML_GPU_UTILIZATION_DOMAIN_BUS = 3 + + +class c_nvmlGpuDynamicPstatesUtilization_t(Structure): + _fields_ = [ + ("bIsPresent", c_uint, 1), + ("percentage", c_uint), + ("incThreshold", c_uint), + ("decThreshold", c_uint), + ] + + +class c_nvmlGpuDynamicPstatesInfo_t(Structure): + _fields_ = [ + ("flags", c_uint), + ( + "utilization", + c_nvmlGpuDynamicPstatesUtilization_t * NVML_MAX_GPU_UTILIZATIONS, + ), + ] + + +NVML_MAX_THERMAL_SENSORS_PER_GPU = 3 + +NVML_THERMAL_TARGET_NONE = 0 +NVML_THERMAL_TARGET_GPU = 1 +NVML_THERMAL_TARGET_MEMORY = 2 +NVML_THERMAL_TARGET_POWER_SUPPLY = 4 +NVML_THERMAL_TARGET_BOARD = 8 +NVML_THERMAL_TARGET_VCD_BOARD = 9 +NVML_THERMAL_TARGET_VCD_INLET = 10 +NVML_THERMAL_TARGET_VCD_OUTLET = 11 +NVML_THERMAL_TARGET_ALL = 15 +NVML_THERMAL_TARGET_UNKNOWN = -1 + +NVML_THERMAL_CONTROLLER_NONE = 0 +NVML_THERMAL_CONTROLLER_GPU_INTERNAL = 1 +NVML_THERMAL_CONTROLLER_ADM1032 = 2 +NVML_THERMAL_CONTROLLER_ADT7461 = 3 +NVML_THERMAL_CONTROLLER_MAX6649 = 4 +NVML_THERMAL_CONTROLLER_MAX1617 = 5 +NVML_THERMAL_CONTROLLER_LM99 = 6 +NVML_THERMAL_CONTROLLER_LM89 = 7 +NVML_THERMAL_CONTROLLER_LM64 = 8 +NVML_THERMAL_CONTROLLER_G781 = 9 +NVML_THERMAL_CONTROLLER_ADT7473 = 10 +NVML_THERMAL_CONTROLLER_SBMAX6649 = 11 +NVML_THERMAL_CONTROLLER_VBIOSEVT = 12 +NVML_THERMAL_CONTROLLER_OS = 13 +NVML_THERMAL_CONTROLLER_NVSYSCON_CANOAS = 14 +NVML_THERMAL_CONTROLLER_NVSYSCON_E551 = 15 +NVML_THERMAL_CONTROLLER_MAX6649R = 16 +NVML_THERMAL_CONTROLLER_ADT7473S = 17 +NVML_THERMAL_CONTROLLER_UNKNOWN = -1 + + +class c_nvmlGpuThermalSensor_t(Structure): + _fields_ = [ + ("controller", c_int), + ("defaultMinTemp", c_int), + ("defaultMaxTemp", c_int), + ("currentTemp", c_int), + ("target", c_int), + ] + + +class c_nvmlGpuThermalSettings_t(Structure): + _fields_ = [ + ("count", c_uint), + ("sensor", c_nvmlGpuThermalSensor_t * NVML_MAX_THERMAL_SENSORS_PER_GPU), + ] + + +_nvmlCoolerControl_t = c_uint +NVML_THERMAL_COOLER_SIGNAL_NONE = 0 +NVML_THERMAL_COOLER_SIGNAL_TOGGLE = 1 +NVML_THERMAL_COOLER_SIGNAL_VARIABLE = 2 +NVML_THERMAL_COOLER_SIGNAL_COUNT = 3 + +_nvmlCoolerTarget_t = c_uint +NVML_THERMAL_COOLER_TARGET_NONE = 1 << 0 +NVML_THERMAL_COOLER_TARGET_GPU = 1 << 1 +NVML_THERMAL_COOLER_TARGET_MEMORY = 1 << 2 +NVML_THERMAL_COOLER_TARGET_POWER_SUPPLY = 1 << 3 +NVML_THERMAL_COOLER_TARGET_GPU_RELATED = ( + NVML_THERMAL_COOLER_TARGET_GPU + | NVML_THERMAL_COOLER_TARGET_MEMORY + | NVML_THERMAL_COOLER_TARGET_POWER_SUPPLY +) + + +class c_nvmlCoolerInfo_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("index", c_uint), + ("coolerControlType", _nvmlCoolerControl_t), + ("coolerTarget", _nvmlCoolerTarget_t), + ] + + +nvmlCoolerInfo_v1 = 0x1000010 + + +def nvmlDeviceGetCoolerInfo(handle): + c_coolerInfo = c_nvmlCoolerInfo_t() + c_coolerInfo.version = nvmlCoolerInfo_v1 + c_coolerInfo.index = 0 + fn = _nvmlGetFunctionPointer("nvmlDeviceGetCoolerInfo") + ret = fn(handle, byref(c_coolerInfo)) + _nvmlCheckReturn(ret) + return [c_coolerInfo.coolerControlType, c_coolerInfo.coolerTarget] + + +class struct_c_nvmlComputeInstance_t(Structure): + pass # opaque handle + + +c_nvmlComputeInstance_t = POINTER(struct_c_nvmlComputeInstance_t) + + +class c_nvmlDeviceAttributes(Structure): + _fields_ = [ + ("multiprocessorCount", c_uint), + ("sharedCopyEngineCount", c_uint), + ("sharedDecoderCount", c_uint), + ("sharedEncoderCount", c_uint), + ("sharedJpegCount", c_uint), + ("sharedOfaCount", c_uint), + ("gpuInstanceSliceCount", c_uint), + ("computeInstanceSliceCount", c_uint), + ("memorySizeMB", c_ulonglong), + ] + + +class c_nvmlRowRemapperHistogramValues(Structure): + _fields_ = [ + ("max", c_uint), + ("high", c_uint), + ("partial", c_uint), + ("low", c_uint), + ("none", c_uint), + ] + + +NVML_GPU_CERT_CHAIN_SIZE = 0x1000 +NVML_GPU_ATTESTATION_CERT_CHAIN_SIZE = 0x1400 +NVML_CC_GPU_CEC_NONCE_SIZE = 0x20 +NVML_CC_GPU_ATTESTATION_REPORT_SIZE = 0x2000 +NVML_CC_GPU_CEC_ATTESTATION_REPORT_SIZE = 0x1000 +NVML_CC_CEC_ATTESTATION_REPORT_NOT_PRESENT = 0 +NVML_CC_CEC_ATTESTATION_REPORT_PRESENT = 1 + + +class c_nvmlConfComputeSystemState_t(Structure): + _fields_ = [ + ("environment", c_uint), + ("ccFeature", c_uint), + ("devToolsMode", c_uint), + ] + + +nvmlSystemConfComputeSettings_v1 = 0x1000014 + + +class c_nvmlSystemConfComputeSettings_v1_t(Structure): + _fields_ = [ + ("version", c_uint), + ("environment", c_uint), + ("ccFeature", c_uint), + ("devToolsMode", c_uint), + ("multiGpuMode", c_uint), + ] + + def __init__(self): + super(c_nvmlSystemConfComputeSettings_v1_t, self).__init__( + version=nvmlSystemConfComputeSettings_v1 + ) + + +class c_nvmlConfComputeSystemCaps_t(Structure): + _fields_ = [ + ("cpuCaps", c_uint), + ("gpusCaps", c_uint), + ] + + +class c_nvmlConfComputeMemSizeInfo_t(Structure): + _fields_ = [ + ("protectedMemSizeKib", c_ulonglong), + ("unprotectedMemSizeKib", c_ulonglong), + ] + + +class c_nvmlConfComputeGpuCertificate_t(Structure): + _fields_ = [ + ("certChainSize", c_uint), + ("attestationCertChainSize", c_uint), + ("certChain", c_uint8 * NVML_GPU_CERT_CHAIN_SIZE), + ("attestationCertChain", c_uint8 * NVML_GPU_ATTESTATION_CERT_CHAIN_SIZE), + ] + + +class c_nvmlConfComputeGpuAttestationReport_t(Structure): + _fields_ = [ + ("isCecAttestationReportPresent", c_uint), + ("attestationReportSize", c_uint), + ("cecAttestationReportSize", c_uint), + ("nonce", c_uint8 * NVML_CC_GPU_CEC_NONCE_SIZE), + ("attestationReport", c_uint8 * NVML_CC_GPU_ATTESTATION_REPORT_SIZE), + ("cecAttestationReport", c_uint8 * NVML_CC_GPU_CEC_ATTESTATION_REPORT_SIZE), + ] + + +class c_nvmlConfComputeSetKeyRotationThresholdInfo_t(Structure): + _fields_ = [ + ("version", c_uint), + ("maxAttackerAdvantage", c_ulong), + ] + + +ConfComputeSetKeyRotationThresholdInfo_v1 = 0x1000010 + + +class c_nvmlConfComputeGetKeyRotationThresholdInfo_t(Structure): + _fields_ = [ + ("version", c_uint), + ("attackerAdvantage", c_ulong), + ] + + +ConfComputeGetKeyRotationThresholdInfo_v1 = 0x1000010 + + +## string/bytes conversion for ease of use +def convertStrBytes(func): + """ + In python 3, strings are unicode instead of bytes, and need to be converted for ctypes + Args from caller: (1, 'string', <__main__.c_nvmlDevice_t at 0xFFFFFFFF>) + Args passed to function: (1, b'string', <__main__.c_nvmlDevice_t at 0xFFFFFFFF)> + ---- + Returned from function: b'returned string' + Returned to caller: 'returned string' + """ + + @wraps(func) + def wrapper(*args, **kwargs): + # encoding a str returns bytes in python 2 and 3 + args = [arg.encode() if isinstance(arg, str) else arg for arg in args] + res = func(*args, **kwargs) + # In python 2, str and bytes are the same + # In python 3, str is unicode and should be decoded. + # Ctypes handles most conversions, this only effects c_char and char arrays. + if isinstance(res, bytes): + if isinstance(res, str): + return res + return res.decode() + return res + + if sys.version_info >= (3,): + return wrapper + return func + + +def throwOnVersionMismatch(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except NVMLError_FunctionNotFound: + raise NVMLLibraryMismatchError( + "Unversioned function called and the " + "pyNVML version does not match the NVML lib version. " + "Either use matching pyNVML and NVML lib versions or " + "use a versioned function such as " + func.__name__ + "_v2" + ) + + return wrapper + + +## C function wrappers ## +def nvmlInitWithFlags(flags): + _LoadNvmlLibrary() + + # + # Initialize the library + # + fn = _nvmlGetFunctionPointer("nvmlInitWithFlags") + ret = fn(flags) + _nvmlCheckReturn(ret) + + # Atomically update refcount + global _nvmlLib_refcount + libLoadLock.acquire() + _nvmlLib_refcount += 1 + libLoadLock.release() + return None + + +def nvmlInit(): + nvmlInitWithFlags(0) + return None + + +def _LoadNvmlLibrary(): + """ + Load the library if it isn't loaded already + """ + global nvmlLib + + if nvmlLib == None: + # lock to ensure only one caller loads the library + libLoadLock.acquire() + + try: + # ensure the library still isn't loaded + if nvmlLib == None: + try: + if sys.platform[:3] == "win": + # cdecl calling convention + try: + # Check for nvml.dll in System32 first for DCH drivers + nvmlLib = CDLL( + os.path.join( + os.getenv("WINDIR", "C:/Windows"), + "System32/nvml.dll", + ) + ) + except OSError as ose: + # If nvml.dll is not found in System32, it should be in ProgramFiles + # load nvml.dll from %ProgramFiles%/NVIDIA Corporation/NVSMI/nvml.dll + nvmlLib = CDLL( + os.path.join( + os.getenv("ProgramFiles", "C:/Program Files"), + "NVIDIA Corporation/NVSMI/nvml.dll", + ) + ) + else: + # assume linux + nvmlLib = CDLL("libnvidia-ml.so.1") + except OSError as ose: + _nvmlCheckReturn(NVML_ERROR_LIBRARY_NOT_FOUND) + if nvmlLib == None: + _nvmlCheckReturn(NVML_ERROR_LIBRARY_NOT_FOUND) + finally: + # lock is always freed + libLoadLock.release() + + +def nvmlShutdown(): + # + # Leave the library loaded, but shutdown the interface + # + fn = _nvmlGetFunctionPointer("nvmlShutdown") + ret = fn() + _nvmlCheckReturn(ret) + + # Atomically update refcount + global _nvmlLib_refcount + libLoadLock.acquire() + if 0 < _nvmlLib_refcount: + _nvmlLib_refcount -= 1 + libLoadLock.release() + return None + + +# Added in 2.285 +@convertStrBytes +def nvmlErrorString(result): + fn = _nvmlGetFunctionPointer("nvmlErrorString") + fn.restype = c_char_p # otherwise return is an int + ret = fn(result) + return ret + + +# Added in 2.285 +@convertStrBytes +def nvmlSystemGetNVMLVersion(): + c_version = create_string_buffer(NVML_SYSTEM_NVML_VERSION_BUFFER_SIZE) + fn = _nvmlGetFunctionPointer("nvmlSystemGetNVMLVersion") + ret = fn(c_version, c_uint(NVML_SYSTEM_NVML_VERSION_BUFFER_SIZE)) + _nvmlCheckReturn(ret) + return c_version.value + + +def nvmlSystemGetCudaDriverVersion(): + c_cuda_version = c_int() + fn = _nvmlGetFunctionPointer("nvmlSystemGetCudaDriverVersion") + ret = fn(byref(c_cuda_version)) + _nvmlCheckReturn(ret) + return c_cuda_version.value + + +def nvmlSystemGetCudaDriverVersion_v2(): + c_cuda_version = c_int() + fn = _nvmlGetFunctionPointer("nvmlSystemGetCudaDriverVersion_v2") + ret = fn(byref(c_cuda_version)) + _nvmlCheckReturn(ret) + return c_cuda_version.value + + +# Added in 2.285 +@convertStrBytes +def nvmlSystemGetProcessName(pid): + c_name = create_string_buffer(1024) + fn = _nvmlGetFunctionPointer("nvmlSystemGetProcessName") + ret = fn(c_uint(pid), c_name, c_uint(1024)) + _nvmlCheckReturn(ret) + return c_name.value + + +@convertStrBytes +def nvmlSystemGetDriverVersion(): + c_version = create_string_buffer(NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE) + fn = _nvmlGetFunctionPointer("nvmlSystemGetDriverVersion") + ret = fn(c_version, c_uint(NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE)) + _nvmlCheckReturn(ret) + return c_version.value + + +# Added in 2.285 +def nvmlSystemGetHicVersion(): + c_count = c_uint(0) + hics = None + fn = _nvmlGetFunctionPointer("nvmlSystemGetHicVersion") + + # get the count + ret = fn(byref(c_count), None) + + # this should only fail with insufficient size + if (ret != NVML_SUCCESS) and (ret != NVML_ERROR_INSUFFICIENT_SIZE): + raise NVMLError(ret) + + # If there are no hics + if c_count.value == 0: + return [] + + hic_array = c_nvmlHwbcEntry_t * c_count.value + hics = hic_array() + ret = fn(byref(c_count), hics) + _nvmlCheckReturn(ret) + return hics + + +def nvmlSystemGetDriverBranch(): + c_branchInfo = c_nvmlSystemDriverBranchInfo_v1_t(0) + c_branchInfo.version = SystemDriverBranchInfo_v1 + fn = _nvmlGetFunctionPointer("nvmlSystemGetDriverBranch") + ret = fn(byref(c_branchInfo), c_uint(NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE)) + _nvmlCheckReturn(ret) + return c_branchInfo + + +## Unit get functions +def nvmlUnitGetCount(): + c_count = c_uint() + fn = _nvmlGetFunctionPointer("nvmlUnitGetCount") + ret = fn(byref(c_count)) + _nvmlCheckReturn(ret) + return c_count.value + + +def nvmlUnitGetHandleByIndex(index): + c_index = c_uint(index) + unit = c_nvmlUnit_t() + fn = _nvmlGetFunctionPointer("nvmlUnitGetHandleByIndex") + ret = fn(c_index, byref(unit)) + _nvmlCheckReturn(ret) + return unit + + +def nvmlUnitGetUnitInfo(unit): + c_info = c_nvmlUnitInfo_t() + fn = _nvmlGetFunctionPointer("nvmlUnitGetUnitInfo") + ret = fn(unit, byref(c_info)) + _nvmlCheckReturn(ret) + return c_info + + +def nvmlUnitGetLedState(unit): + c_state = c_nvmlLedState_t() + fn = _nvmlGetFunctionPointer("nvmlUnitGetLedState") + ret = fn(unit, byref(c_state)) + _nvmlCheckReturn(ret) + return c_state + + +def nvmlUnitGetPsuInfo(unit): + c_info = c_nvmlPSUInfo_t() + fn = _nvmlGetFunctionPointer("nvmlUnitGetPsuInfo") + ret = fn(unit, byref(c_info)) + _nvmlCheckReturn(ret) + return c_info + + +def nvmlUnitGetTemperature(unit, type): + c_temp = c_uint() + fn = _nvmlGetFunctionPointer("nvmlUnitGetTemperature") + ret = fn(unit, c_uint(type), byref(c_temp)) + _nvmlCheckReturn(ret) + return c_temp.value + + +def nvmlUnitGetFanSpeedInfo(unit): + c_speeds = c_nvmlUnitFanSpeeds_t() + fn = _nvmlGetFunctionPointer("nvmlUnitGetFanSpeedInfo") + ret = fn(unit, byref(c_speeds)) + _nvmlCheckReturn(ret) + return c_speeds + + +# added to API +def nvmlUnitGetDeviceCount(unit): + c_count = c_uint(0) + # query the unit to determine device count + fn = _nvmlGetFunctionPointer("nvmlUnitGetDevices") + ret = fn(unit, byref(c_count), None) + if ret == NVML_ERROR_INSUFFICIENT_SIZE: + ret = NVML_SUCCESS + _nvmlCheckReturn(ret) + return c_count.value + + +def nvmlUnitGetDevices(unit): + c_count = c_uint(nvmlUnitGetDeviceCount(unit)) + device_array = c_nvmlDevice_t * c_count.value + c_devices = device_array() + fn = _nvmlGetFunctionPointer("nvmlUnitGetDevices") + ret = fn(unit, byref(c_count), c_devices) + _nvmlCheckReturn(ret) + return c_devices + + +## Device get functions +def nvmlDeviceGetCount(): + c_count = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetCount_v2") + ret = fn(byref(c_count)) + _nvmlCheckReturn(ret) + return c_count.value + + +def nvmlDeviceGetHandleByIndex(index): + c_index = c_uint(index) + device = c_nvmlDevice_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetHandleByIndex_v2") + ret = fn(c_index, byref(device)) + _nvmlCheckReturn(ret) + return device + + +@convertStrBytes +def nvmlDeviceGetHandleBySerial(serial): + c_serial = c_char_p(serial) + device = c_nvmlDevice_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetHandleBySerial") + ret = fn(c_serial, byref(device)) + _nvmlCheckReturn(ret) + return device + + +@convertStrBytes +def nvmlDeviceGetHandleByUUID(uuid): + c_uuid = c_char_p(uuid) + device = c_nvmlDevice_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetHandleByUUID") + ret = fn(c_uuid, byref(device)) + _nvmlCheckReturn(ret) + return device + + +@convertStrBytes +def nvmlDeviceGetHandleByPciBusId(pciBusId): + c_busId = c_char_p(pciBusId) + device = c_nvmlDevice_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetHandleByPciBusId_v2") + ret = fn(c_busId, byref(device)) + _nvmlCheckReturn(ret) + return device + + +@convertStrBytes +def nvmlDeviceGetName(handle): + c_name = create_string_buffer(NVML_DEVICE_NAME_V2_BUFFER_SIZE) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetName") + ret = fn(handle, c_name, c_uint(NVML_DEVICE_NAME_V2_BUFFER_SIZE)) + _nvmlCheckReturn(ret) + return c_name.value + + +class c_nvmlDevicePerfModes_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("str", c_char * NVML_PERF_MODES_BUFFER_SIZE), + ] + + +nvmlDevicePerfModes_v1 = 0x1000804 + + +@convertStrBytes +def nvmlDeviceGetPerformanceModes(handle): + perfModes = c_nvmlDevicePerfModes_v1_t() + perfModes.version = nvmlDevicePerfModes_v1 + fn = _nvmlGetFunctionPointer("nvmlDeviceGetPerformanceModes") + ret = fn(handle, byref(perfModes)) + _nvmlCheckReturn(ret) + return perfModes.str + + +class c_nvmlDeviceCurrentClockFreqs_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("str", c_char * NVML_PERF_MODES_BUFFER_SIZE), + ] + + +nvmlDeviceCurrentClockFreqs_v1 = 0x1000804 + + +@convertStrBytes +def nvmlDeviceGetCurrentClockFreqs(handle): + currentClockFreqs = c_nvmlDeviceCurrentClockFreqs_v1_t() + currentClockFreqs.version = nvmlDeviceCurrentClockFreqs_v1 + fn = _nvmlGetFunctionPointer("nvmlDeviceGetCurrentClockFreqs") + ret = fn(handle, byref(currentClockFreqs)) + _nvmlCheckReturn(ret) + return currentClockFreqs.str + + +def nvmlDeviceGetBoardId(handle): + c_id = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetBoardId") + ret = fn(handle, byref(c_id)) + _nvmlCheckReturn(ret) + return c_id.value + + +def nvmlDeviceGetMultiGpuBoard(handle): + c_multiGpu = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetMultiGpuBoard") + ret = fn(handle, byref(c_multiGpu)) + _nvmlCheckReturn(ret) + return c_multiGpu.value + + +def nvmlDeviceGetBrand(handle): + c_type = _nvmlBrandType_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetBrand") + ret = fn(handle, byref(c_type)) + _nvmlCheckReturn(ret) + return c_type.value + + +def nvmlDeviceGetC2cModeInfoV1(handle): + c_info = c_nvmlC2cModeInfo_v1_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetC2cModeInfoV") + ret = fn(handle, byref(c_info)) + _nvmlCheckReturn(ret) + return c_info + + +def nvmlDeviceGetC2cModeInfoV(handle): + return nvmlDeviceGetC2cModeInfoV1(handle) + + +@convertStrBytes +def nvmlDeviceGetBoardPartNumber(handle): + c_part_number = create_string_buffer(NVML_DEVICE_PART_NUMBER_BUFFER_SIZE) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetBoardPartNumber") + ret = fn(handle, c_part_number, c_uint(NVML_DEVICE_PART_NUMBER_BUFFER_SIZE)) + _nvmlCheckReturn(ret) + return c_part_number.value + + +@convertStrBytes +def nvmlDeviceGetSerial(handle): + c_serial = create_string_buffer(NVML_DEVICE_SERIAL_BUFFER_SIZE) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetSerial") + ret = fn(handle, c_serial, c_uint(NVML_DEVICE_SERIAL_BUFFER_SIZE)) + _nvmlCheckReturn(ret) + return c_serial.value + + +def nvmlDeviceGetModuleId(handle, moduleId=c_uint()): + isReference = type(moduleId) is not c_uint + moduleIdRef = moduleId if isReference else byref(moduleId) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetModuleId") + ret = fn(handle, moduleIdRef) + if isReference: + return ret + else: + _nvmlCheckReturn(ret) + return moduleId.value + + +def nvmlDeviceGetMemoryAffinity(handle, nodeSetSize, scope): + affinity_array = c_ulonglong * nodeSetSize + c_affinity = affinity_array() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemoryAffinity") + ret = fn(handle, nodeSetSize, byref(c_affinity), _nvmlAffinityScope_t(scope)) + _nvmlCheckReturn(ret) + return c_affinity + + +def nvmlDeviceGetCpuAffinityWithinScope(handle, cpuSetSize, scope): + affinity_array = c_ulonglong * cpuSetSize + c_affinity = affinity_array() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetCpuAffinityWithinScope") + ret = fn(handle, cpuSetSize, byref(c_affinity), _nvmlAffinityScope_t(scope)) + _nvmlCheckReturn(ret) + return c_affinity + + +def nvmlDeviceGetCpuAffinity(handle, cpuSetSize): + affinity_array = c_ulonglong * cpuSetSize + c_affinity = affinity_array() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetCpuAffinity") + ret = fn(handle, cpuSetSize, byref(c_affinity)) + _nvmlCheckReturn(ret) + return c_affinity + + +def nvmlDeviceSetCpuAffinity(handle): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetCpuAffinity") + ret = fn(handle) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceClearCpuAffinity(handle): + fn = _nvmlGetFunctionPointer("nvmlDeviceClearCpuAffinity") + ret = fn(handle) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceGetNumaNodeId(handle): + fn = _nvmlGetFunctionPointer("nvmlDeviceGetNumaNodeId") + node = c_int() + ret = fn(handle, byref(node)) + _nvmlCheckReturn(ret) + return node.value + + +def nvmlDeviceGetMinorNumber(handle): + c_minor_number = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetMinorNumber") + ret = fn(handle, byref(c_minor_number)) + _nvmlCheckReturn(ret) + return c_minor_number.value + + +@convertStrBytes +def nvmlDeviceGetUUID(handle): + c_uuid = create_string_buffer(NVML_DEVICE_UUID_V2_BUFFER_SIZE) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetUUID") + ret = fn(handle, c_uuid, c_uint(NVML_DEVICE_UUID_V2_BUFFER_SIZE)) + _nvmlCheckReturn(ret) + return c_uuid.value + + +@convertStrBytes +def nvmlDeviceGetInforomVersion(handle, infoRomObject): + c_version = create_string_buffer(NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetInforomVersion") + ret = fn( + handle, + _nvmlInforomObject_t(infoRomObject), + c_version, + c_uint(NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE), + ) + _nvmlCheckReturn(ret) + return c_version.value + + +# Added in 4.304 +@convertStrBytes +def nvmlDeviceGetInforomImageVersion(handle): + c_version = create_string_buffer(NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetInforomImageVersion") + ret = fn(handle, c_version, c_uint(NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE)) + _nvmlCheckReturn(ret) + return c_version.value + + +# Added in 4.304 +def nvmlDeviceGetInforomConfigurationChecksum(handle): + c_checksum = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetInforomConfigurationChecksum") + ret = fn(handle, byref(c_checksum)) + _nvmlCheckReturn(ret) + return c_checksum.value + + +# Added in 4.304 +def nvmlDeviceValidateInforom(handle): + fn = _nvmlGetFunctionPointer("nvmlDeviceValidateInforom") + ret = fn(handle) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceGetLastBBXFlushTime(handle): + c_timestamp = c_ulonglong() + c_durationUs = c_ulong() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetLastBBXFlushTime") + ret = fn(handle, byref(c_timestamp), byref(c_durationUs)) + _nvmlCheckReturn(ret) + return [c_timestamp.value, c_durationUs.value] + + +def nvmlDeviceGetDisplayMode(handle): + c_mode = _nvmlEnableState_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetDisplayMode") + ret = fn(handle, byref(c_mode)) + _nvmlCheckReturn(ret) + return c_mode.value + + +def nvmlDeviceGetDisplayActive(handle): + c_mode = _nvmlEnableState_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetDisplayActive") + ret = fn(handle, byref(c_mode)) + _nvmlCheckReturn(ret) + return c_mode.value + + +def nvmlDeviceGetPersistenceMode(handle): + c_state = _nvmlEnableState_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetPersistenceMode") + ret = fn(handle, byref(c_state)) + _nvmlCheckReturn(ret) + return c_state.value + + +def nvmlDeviceGetPciInfoExt(handle, c_info): + fn = _nvmlGetFunctionPointer("nvmlDeviceGetPciInfoExt") + ret = fn(handle, c_info) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceGetPciInfo_v3(handle): + c_info = nvmlPciInfo_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetPciInfo_v3") + ret = fn(handle, byref(c_info)) + _nvmlCheckReturn(ret) + return c_info + + +def nvmlDeviceGetPciInfo(handle): + return nvmlDeviceGetPciInfo_v3(handle) + + +def nvmlDeviceGetClockInfo(handle, type): + c_clock = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetClockInfo") + ret = fn(handle, _nvmlClockType_t(type), byref(c_clock)) + _nvmlCheckReturn(ret) + return c_clock.value + + +# Added in 2.285 +def nvmlDeviceGetMaxClockInfo(handle, type): + c_clock = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxClockInfo") + ret = fn(handle, _nvmlClockType_t(type), byref(c_clock)) + _nvmlCheckReturn(ret) + return c_clock.value + + +# Added in 4.304 +def nvmlDeviceGetApplicationsClock(handle, type): + c_clock = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetApplicationsClock") + ret = fn(handle, _nvmlClockType_t(type), byref(c_clock)) + _nvmlCheckReturn(ret) + return c_clock.value + + +def nvmlDeviceGetMaxCustomerBoostClock(handle, type): + c_clock = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxCustomerBoostClock") + ret = fn(handle, _nvmlClockType_t(type), byref(c_clock)) + _nvmlCheckReturn(ret) + return c_clock.value + + +def nvmlDeviceGetClock(handle, type, id): + c_clock = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetClock") + ret = fn(handle, _nvmlClockType_t(type), _nvmlClockId_t(id), byref(c_clock)) + _nvmlCheckReturn(ret) + return c_clock.value + + +# Added in 5.319 +def nvmlDeviceGetDefaultApplicationsClock(handle, type): + c_clock = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetDefaultApplicationsClock") + ret = fn(handle, _nvmlClockType_t(type), byref(c_clock)) + _nvmlCheckReturn(ret) + return c_clock.value + + +# Added in 4.304 +def nvmlDeviceGetSupportedMemoryClocks(handle): + # first call to get the size + c_count = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedMemoryClocks") + ret = fn(handle, byref(c_count), None) + + if ret == NVML_SUCCESS: + # special case, no clocks + return [] + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: + # typical case + clocks_array = c_uint * c_count.value + c_clocks = clocks_array() + + # make the call again + ret = fn(handle, byref(c_count), c_clocks) + _nvmlCheckReturn(ret) + + procs = [] + for i in range(c_count.value): + procs.append(c_clocks[i]) + + return procs + else: + # error case + raise NVMLError(ret) + + +# Added in 4.304 +def nvmlDeviceGetSupportedGraphicsClocks(handle, memoryClockMHz): + # first call to get the size + c_count = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedGraphicsClocks") + ret = fn(handle, c_uint(memoryClockMHz), byref(c_count), None) + + if ret == NVML_SUCCESS: + # special case, no clocks + return [] + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: + # typical case + clocks_array = c_uint * c_count.value + c_clocks = clocks_array() + + # make the call again + ret = fn(handle, c_uint(memoryClockMHz), byref(c_count), c_clocks) + _nvmlCheckReturn(ret) + + procs = [] + for i in range(c_count.value): + procs.append(c_clocks[i]) + + return procs + else: + # error case + raise NVMLError(ret) + + +def nvmlDeviceGetFanSpeed(handle): + c_speed = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetFanSpeed") + ret = fn(handle, byref(c_speed)) + _nvmlCheckReturn(ret) + return c_speed.value + + +def nvmlDeviceGetFanSpeed_v2(handle, fan): + c_speed = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetFanSpeed_v2") + ret = fn(handle, fan, byref(c_speed)) + _nvmlCheckReturn(ret) + return c_speed.value + + +class c_nvmlFanSpeedInfo_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("fan", c_uint), + ("speed", c_uint), + ] + + +nvmlFanSpeedInfo_v1 = 0x100000C + + +def nvmlDeviceGetFanSpeedRPM(handle): + c_fanSpeed = c_nvmlFanSpeedInfo_t() + c_fanSpeed.fan = 0 + c_fanSpeed.version = nvmlFanSpeedInfo_v1 + fn = _nvmlGetFunctionPointer("nvmlDeviceGetFanSpeedRPM") + ret = fn(handle, byref(c_fanSpeed)) + _nvmlCheckReturn(ret) + return c_fanSpeed.speed + + +def nvmlDeviceGetTargetFanSpeed(handle, fan): + c_speed = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetTargetFanSpeed") + ret = fn(handle, fan, byref(c_speed)) + _nvmlCheckReturn(ret) + return c_speed.value + + +def nvmlDeviceGetNumFans(device): + c_numFans = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetNumFans") + ret = fn(device, byref(c_numFans)) + _nvmlCheckReturn(ret) + return c_numFans.value + + +def nvmlDeviceSetDefaultFanSpeed_v2(handle, index): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetDefaultFanSpeed_v2") + ret = fn(handle, index) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlDeviceGetMinMaxFanSpeed(handle, minSpeed=c_uint(), maxSpeed=c_uint()): + isReference = (type(minSpeed) is not c_uint) or (type(maxSpeed) is not c_uint) + minSpeedRef = minSpeed if isReference else byref(minSpeed) + maxSpeedRef = maxSpeed if isReference else byref(maxSpeed) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetMinMaxFanSpeed") + ret = fn(handle, minSpeedRef, maxSpeedRef) + _nvmlCheckReturn(ret) + return NVML_SUCCESS if isReference else [minSpeed.value, maxSpeed.value] + + +def nvmlDeviceGetFanControlPolicy_v2(handle, fan, fanControlPolicy=c_uint()): + isReference = type(fanControlPolicy) is not c_uint + fanControlPolicyRef = fanControlPolicy if isReference else byref(fanControlPolicy) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetFanControlPolicy_v2") + ret = fn(handle, fan, fanControlPolicyRef) + _nvmlCheckReturn(ret) + return NVML_SUCCESS if isReference else fanControlPolicy.value + + +def nvmlDeviceSetFanControlPolicy(handle, fan, fanControlPolicy): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetFanControlPolicy") + ret = fn(handle, fan, _nvmlFanControlPolicy_t(fanControlPolicy)) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +class c_nvmlTemperature_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("sensorType", _nvmlTemperatureSensors_t), + ("temperature", c_int), + ] + + +nvmlTemperature_v1 = 0x100000C + + +def nvmlDeviceGetTemperatureV1(handle, sensor): + c_temp = c_nvmlTemperature_v1_t() + c_temp.version = nvmlTemperature_v1 + c_temp.sensorType = _nvmlTemperatureSensors_t(sensor) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetTemperatureV") + ret = fn(handle, byref(c_temp)) + _nvmlCheckReturn(ret) + return c_temp.temperature + + +def nvmlDeviceGetTemperatureV(handle, sensor, version=nvmlTemperature_v1): + if version == nvmlTemperature_v1: + return nvmlDeviceGetTemperatureV1(handle, sensor) + else: + raise NVMLError(NVML_ERROR_ARGUMENT_VERSION_MISMATCH) + + +# DEPRECATED use nvmlDeviceGetTemperatureV instead +def nvmlDeviceGetTemperature(handle, sensor): + c_temp = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetTemperature") + ret = fn(handle, _nvmlTemperatureSensors_t(sensor), byref(c_temp)) + _nvmlCheckReturn(ret) + return c_temp.value + + +def nvmlDeviceGetTemperatureThreshold(handle, threshold): + c_temp = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetTemperatureThreshold") + ret = fn(handle, _nvmlTemperatureThresholds_t(threshold), byref(c_temp)) + _nvmlCheckReturn(ret) + return c_temp.value + + +def nvmlDeviceSetTemperatureThreshold(handle, threshold, temp): + c_temp = c_uint() + c_temp.value = temp + fn = _nvmlGetFunctionPointer("nvmlDeviceSetTemperatureThreshold") + ret = fn(handle, _nvmlTemperatureThresholds_t(threshold), byref(c_temp)) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceGetMarginTemperature(handle): + c_marginTempInfo = c_nvmlMarginTemperature_v1_t() + c_marginTempInfo.version = nvmlMarginTemperature_v1 + fn = _nvmlGetFunctionPointer("nvmlDeviceGetMarginTemperature") + ret = fn(handle, byref(c_marginTempInfo)) + _nvmlCheckReturn(ret) + return c_marginTempInfo.marginTemperature + + +# DEPRECATED use nvmlDeviceGetPerformanceState +def nvmlDeviceGetPowerState(handle): + c_pstate = _nvmlPstates_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerState") + ret = fn(handle, byref(c_pstate)) + _nvmlCheckReturn(ret) + return c_pstate.value + + +def nvmlDeviceGetPerformanceState(handle): + c_pstate = _nvmlPstates_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetPerformanceState") + ret = fn(handle, byref(c_pstate)) + _nvmlCheckReturn(ret) + return c_pstate.value + + +def nvmlDeviceGetPowerManagementMode(handle): + c_pcapMode = _nvmlEnableState_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerManagementMode") + ret = fn(handle, byref(c_pcapMode)) + _nvmlCheckReturn(ret) + return c_pcapMode.value + + +def nvmlDeviceGetPowerManagementLimit(handle): + c_limit = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerManagementLimit") + ret = fn(handle, byref(c_limit)) + _nvmlCheckReturn(ret) + return c_limit.value + + +# Added in 4.304 +def nvmlDeviceGetPowerManagementLimitConstraints(handle): + c_minLimit = c_uint() + c_maxLimit = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerManagementLimitConstraints") + ret = fn(handle, byref(c_minLimit), byref(c_maxLimit)) + _nvmlCheckReturn(ret) + return [c_minLimit.value, c_maxLimit.value] + + +# Added in 4.304 +def nvmlDeviceGetPowerManagementDefaultLimit(handle): + c_limit = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerManagementDefaultLimit") + ret = fn(handle, byref(c_limit)) + _nvmlCheckReturn(ret) + return c_limit.value + + +# Added in 331 +def nvmlDeviceGetEnforcedPowerLimit(handle): + c_limit = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetEnforcedPowerLimit") + ret = fn(handle, byref(c_limit)) + _nvmlCheckReturn(ret) + return c_limit.value + + +def nvmlDeviceGetPowerUsage(handle): + c_watts = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerUsage") + ret = fn(handle, byref(c_watts)) + _nvmlCheckReturn(ret) + return c_watts.value + + +def nvmlDeviceGetTotalEnergyConsumption(handle): + c_millijoules = c_uint64() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetTotalEnergyConsumption") + ret = fn(handle, byref(c_millijoules)) + _nvmlCheckReturn(ret) + return c_millijoules.value + + +# Added in 4.304 +def nvmlDeviceGetGpuOperationMode(handle): + c_currState = _nvmlGpuOperationMode_t() + c_pendingState = _nvmlGpuOperationMode_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuOperationMode") + ret = fn(handle, byref(c_currState), byref(c_pendingState)) + _nvmlCheckReturn(ret) + return [c_currState.value, c_pendingState.value] + + +# Added in 4.304 +def nvmlDeviceGetCurrentGpuOperationMode(handle): + return nvmlDeviceGetGpuOperationMode(handle)[0] + + +# Added in 4.304 +def nvmlDeviceGetPendingGpuOperationMode(handle): + return nvmlDeviceGetGpuOperationMode(handle)[1] + + +def nvmlDeviceGetMemoryInfo(handle, version=None): + if not version: + c_memory = c_nvmlMemory_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemoryInfo") + else: + c_memory = c_nvmlMemory_v2_t() + c_memory.version = version + fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemoryInfo_v2") + ret = fn(handle, byref(c_memory)) + _nvmlCheckReturn(ret) + return c_memory + + +def nvmlDeviceGetBAR1MemoryInfo(handle): + c_bar1_memory = c_nvmlBAR1Memory_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetBAR1MemoryInfo") + ret = fn(handle, byref(c_bar1_memory)) + _nvmlCheckReturn(ret) + return c_bar1_memory + + +def nvmlDeviceGetComputeMode(handle): + c_mode = _nvmlComputeMode_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetComputeMode") + ret = fn(handle, byref(c_mode)) + _nvmlCheckReturn(ret) + return c_mode.value + + +def nvmlDeviceGetCudaComputeCapability(handle): + c_major = c_int() + c_minor = c_int() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetCudaComputeCapability") + ret = fn(handle, byref(c_major), byref(c_minor)) + _nvmlCheckReturn(ret) + return (c_major.value, c_minor.value) + + +def nvmlDeviceGetEccMode(handle): + c_currState = _nvmlEnableState_t() + c_pendingState = _nvmlEnableState_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetEccMode") + ret = fn(handle, byref(c_currState), byref(c_pendingState)) + _nvmlCheckReturn(ret) + return [c_currState.value, c_pendingState.value] + + +# added to API +def nvmlDeviceGetCurrentEccMode(handle): + return nvmlDeviceGetEccMode(handle)[0] + + +# added to API +def nvmlDeviceGetPendingEccMode(handle): + return nvmlDeviceGetEccMode(handle)[1] + + +def nvmlDeviceGetDefaultEccMode(handle): + c_defaultState = _nvmlEnableState_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetDefaultEccMode") + ret = fn(handle, byref(c_defaultState)) + _nvmlCheckReturn(ret) + return [c_defaultState.value] + + +def nvmlDeviceGetTotalEccErrors(handle, errorType, counterType): + c_count = c_ulonglong() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetTotalEccErrors") + ret = fn( + handle, + _nvmlMemoryErrorType_t(errorType), + _nvmlEccCounterType_t(counterType), + byref(c_count), + ) + _nvmlCheckReturn(ret) + return c_count.value + + +# This is deprecated, instead use nvmlDeviceGetMemoryErrorCounter +def nvmlDeviceGetDetailedEccErrors(handle, errorType, counterType): + c_counts = c_nvmlEccErrorCounts_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetDetailedEccErrors") + ret = fn( + handle, + _nvmlMemoryErrorType_t(errorType), + _nvmlEccCounterType_t(counterType), + byref(c_counts), + ) + _nvmlCheckReturn(ret) + return c_counts + + +# Added in 4.304 +def nvmlDeviceGetMemoryErrorCounter(handle, errorType, counterType, locationType): + c_count = c_ulonglong() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemoryErrorCounter") + ret = fn( + handle, + _nvmlMemoryErrorType_t(errorType), + _nvmlEccCounterType_t(counterType), + _nvmlMemoryLocation_t(locationType), + byref(c_count), + ) + _nvmlCheckReturn(ret) + return c_count.value + + +def nvmlDeviceGetUtilizationRates(handle): + c_util = c_nvmlUtilization_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetUtilizationRates") + ret = fn(handle, byref(c_util)) + _nvmlCheckReturn(ret) + return c_util + + +def nvmlDeviceGetEncoderUtilization(handle): + c_util = c_uint() + c_samplingPeriod = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetEncoderUtilization") + ret = fn(handle, byref(c_util), byref(c_samplingPeriod)) + _nvmlCheckReturn(ret) + return [c_util.value, c_samplingPeriod.value] + + +def nvmlDeviceGetDecoderUtilization(handle): + c_util = c_uint() + c_samplingPeriod = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetDecoderUtilization") + ret = fn(handle, byref(c_util), byref(c_samplingPeriod)) + _nvmlCheckReturn(ret) + return [c_util.value, c_samplingPeriod.value] + + +def nvmlDeviceGetJpgUtilization(handle): + c_util = c_uint() + c_samplingPeriod = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetJpgUtilization") + ret = fn(handle, byref(c_util), byref(c_samplingPeriod)) + _nvmlCheckReturn(ret) + return [c_util.value, c_samplingPeriod.value] + + +def nvmlDeviceGetOfaUtilization(handle): + c_util = c_uint() + c_samplingPeriod = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetOfaUtilization") + ret = fn(handle, byref(c_util), byref(c_samplingPeriod)) + _nvmlCheckReturn(ret) + return [c_util.value, c_samplingPeriod.value] + + +def nvmlDeviceGetPcieReplayCounter(handle): + c_replay = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetPcieReplayCounter") + ret = fn(handle, byref(c_replay)) + _nvmlCheckReturn(ret) + return c_replay.value + + +def nvmlDeviceGetDriverModel(handle): + c_currModel = _nvmlDriverModel_t() + c_pendingModel = _nvmlDriverModel_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetDriverModel") + ret = fn(handle, byref(c_currModel), byref(c_pendingModel)) + _nvmlCheckReturn(ret) + return [c_currModel.value, c_pendingModel.value] + + +# added to API +def nvmlDeviceGetCurrentDriverModel(handle): + return nvmlDeviceGetDriverModel(handle)[0] + + +# added to API +def nvmlDeviceGetPendingDriverModel(handle): + return nvmlDeviceGetDriverModel(handle)[1] + + +# Added in 2.285 +@convertStrBytes +def nvmlDeviceGetVbiosVersion(handle): + c_version = create_string_buffer(NVML_DEVICE_VBIOS_VERSION_BUFFER_SIZE) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetVbiosVersion") + ret = fn(handle, c_version, c_uint(NVML_DEVICE_VBIOS_VERSION_BUFFER_SIZE)) + _nvmlCheckReturn(ret) + return c_version.value + + +# Added in 2.285 +def nvmlDeviceGetComputeRunningProcesses_v2(handle): + # first call to get the size + c_count = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetComputeRunningProcesses_v2") + ret = fn(handle, byref(c_count), None) + if ret == NVML_SUCCESS: + # special case, no running processes + return [] + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: + # typical case + # oversize the array in case more processes are created + c_count.value = c_count.value * 2 + 5 + proc_array = c_nvmlProcessInfo_v2_t * c_count.value + c_procs = proc_array() + # make the call again + ret = fn(handle, byref(c_count), c_procs) + _nvmlCheckReturn(ret) + procs = [] + for i in range(c_count.value): + # use an alternative struct for this object + obj = nvmlStructToFriendlyObject(c_procs[i]) + if obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value: + # special case for WDDM on Windows, see comment above + obj.usedGpuMemory = None + procs.append(obj) + return procs + else: + # error case + raise NVMLError(ret) + + +# Added in 2.285 +def nvmlDeviceGetComputeRunningProcesses_v3(handle): + # first call to get the size + c_count = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetComputeRunningProcesses_v3") + ret = fn(handle, byref(c_count), None) + + if ret == NVML_SUCCESS: + # special case, no running processes + return [] + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: + # typical case + # oversize the array in case more processes are created + c_count.value = c_count.value * 2 + 5 + proc_array = c_nvmlProcessInfo_v3_t * c_count.value + c_procs = proc_array() + + # make the call again + ret = fn(handle, byref(c_count), c_procs) + _nvmlCheckReturn(ret) + + procs = [] + for i in range(c_count.value): + # use an alternative struct for this object + obj = nvmlStructToFriendlyObject(c_procs[i]) + if obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value: + # special case for WDDM on Windows, see comment above + obj.usedGpuMemory = None + procs.append(obj) + + return procs + else: + # error case + raise NVMLError(ret) + + +@throwOnVersionMismatch +def nvmlDeviceGetComputeRunningProcesses(handle): + return nvmlDeviceGetComputeRunningProcesses_v3(handle) + + +def nvmlDeviceGetGraphicsRunningProcesses_v2(handle): + # first call to get the size + c_count = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetGraphicsRunningProcesses_v2") + ret = fn(handle, byref(c_count), None) + if ret == NVML_SUCCESS: + # special case, no running processes + return [] + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: + # typical case + # oversize the array in case more processes are created + c_count.value = c_count.value * 2 + 5 + proc_array = c_nvmlProcessInfo_v2_t * c_count.value + c_procs = proc_array() + # make the call again + ret = fn(handle, byref(c_count), c_procs) + _nvmlCheckReturn(ret) + procs = [] + for i in range(c_count.value): + # use an alternative struct for this object + obj = nvmlStructToFriendlyObject(c_procs[i]) + if obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value: + # special case for WDDM on Windows, see comment above + obj.usedGpuMemory = None + procs.append(obj) + return procs + else: + # error case + raise NVMLError(ret) + + +def nvmlDeviceGetGraphicsRunningProcesses_v3(handle): + # first call to get the size + c_count = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetGraphicsRunningProcesses_v3") + ret = fn(handle, byref(c_count), None) + + if ret == NVML_SUCCESS: + # special case, no running processes + return [] + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: + # typical case + # oversize the array in case more processes are created + c_count.value = c_count.value * 2 + 5 + proc_array = c_nvmlProcessInfo_v3_t * c_count.value + c_procs = proc_array() + + # make the call again + ret = fn(handle, byref(c_count), c_procs) + _nvmlCheckReturn(ret) + + procs = [] + for i in range(c_count.value): + # use an alternative struct for this object + obj = nvmlStructToFriendlyObject(c_procs[i]) + if obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value: + # special case for WDDM on Windows, see comment above + obj.usedGpuMemory = None + procs.append(obj) + + return procs + else: + # error case + raise NVMLError(ret) + + +@throwOnVersionMismatch +def nvmlDeviceGetGraphicsRunningProcesses(handle): + return nvmlDeviceGetGraphicsRunningProcesses_v3(handle) + + +@throwOnVersionMismatch +def nvmlDeviceGetMPSComputeRunningProcesses(handle): + return nvmlDeviceGetMPSComputeRunningProcesses_v3(handle) + + +def nvmlDeviceGetMPSComputeRunningProcesses_v2(handle): + # first call to get the size + c_count = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetMPSComputeRunningProcesses_v2") + ret = fn(handle, byref(c_count), None) + + if ret == NVML_SUCCESS: + # special case, no running processes + return [] + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: + # typical case + # oversize the array in case more processes are created + c_count.value = c_count.value * 2 + 5 + proc_array = c_nvmlProcessInfo_v2_t * c_count.value + c_procs = proc_array() + + # make the call again + ret = fn(handle, byref(c_count), c_procs) + _nvmlCheckReturn(ret) + + procs = [] + for i in range(c_count.value): + # use an alternative struct for this object + obj = nvmlStructToFriendlyObject(c_procs[i]) + if obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value: + # special case for WDDM on Windows, see comment above + obj.usedGpuMemory = None + procs.append(obj) + + return procs + else: + # error case + raise NVMLError(ret) + + +def nvmlDeviceGetMPSComputeRunningProcesses_v3(handle): + # first call to get the size + c_count = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetMPSComputeRunningProcesses_v3") + ret = fn(handle, byref(c_count), None) + + if ret == NVML_SUCCESS: + # special case, no running processes + return [] + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: + # typical case + # oversize the array in case more processes are created + c_count.value = c_count.value * 2 + 5 + proc_array = c_nvmlProcessInfo_v3_t * c_count.value + c_procs = proc_array() + + # make the call again + ret = fn(handle, byref(c_count), c_procs) + _nvmlCheckReturn(ret) + + procs = [] + for i in range(c_count.value): + # use an alternative struct for this object + obj = nvmlStructToFriendlyObject(c_procs[i]) + if obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value: + # special case for WDDM on Windows, see comment above + obj.usedGpuMemory = None + procs.append(obj) + + return procs + else: + # error case + raise NVMLError(ret) + + +def nvmlDeviceGetRunningProcessDetailList(handle, version, mode): + c_processDetailList = c_nvmlProcessDetailList_t() + c_processDetailList.version = version + c_processDetailList.mode = mode + + fn = _nvmlGetFunctionPointer("nvmlDeviceGetRunningProcessDetailList") + + # first call to get the size + ret = fn(handle, byref(c_processDetailList)) + if ret == NVML_SUCCESS: + # special case, no running processes + return [] + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: + c_procs = c_nvmlProcessDetail_v1_t * c_processDetailList.numProcArrayEntries + c_processDetailList.procArray = cast( + (c_procs)(), POINTER(c_nvmlProcessDetail_v1_t) + ) + + # make the call again + ret = fn(handle, byref(c_processDetailList)) + _nvmlCheckReturn(ret) + + procs = [] + for i in range(c_processDetailList.numProcArrayEntries): + # use an alternative struct for this object + obj = c_processDetailList.procArray[i] + if obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value: + obj.usedGpuMemory = None + if obj.usedGpuCcProtectedMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value: + obj.usedGpuCcProtectedMemory = None + procs.append(obj) + + return procs + else: + # error case + raise NVMLError(ret) + + +def nvmlDeviceGetAutoBoostedClocksEnabled(handle): + c_isEnabled = _nvmlEnableState_t() + c_defaultIsEnabled = _nvmlEnableState_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetAutoBoostedClocksEnabled") + ret = fn(handle, byref(c_isEnabled), byref(c_defaultIsEnabled)) + _nvmlCheckReturn(ret) + return [c_isEnabled.value, c_defaultIsEnabled.value] + # Throws NVML_ERROR_NOT_SUPPORTED if hardware doesn't support setting auto boosted clocks + + +## Set functions +def nvmlUnitSetLedState(unit, color): + fn = _nvmlGetFunctionPointer("nvmlUnitSetLedState") + ret = fn(unit, _nvmlLedColor_t(color)) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceSetPersistenceMode(handle, mode): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetPersistenceMode") + ret = fn(handle, _nvmlEnableState_t(mode)) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceSetComputeMode(handle, mode): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetComputeMode") + ret = fn(handle, _nvmlComputeMode_t(mode)) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceSetEccMode(handle, mode): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetEccMode") + ret = fn(handle, _nvmlEnableState_t(mode)) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceClearEccErrorCounts(handle, counterType): + fn = _nvmlGetFunctionPointer("nvmlDeviceClearEccErrorCounts") + ret = fn(handle, _nvmlEccCounterType_t(counterType)) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceSetDriverModel(handle, model): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetDriverModel") + ret = fn(handle, _nvmlDriverModel_t(model)) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceSetAutoBoostedClocksEnabled(handle, enabled): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetAutoBoostedClocksEnabled") + ret = fn(handle, _nvmlEnableState_t(enabled)) + _nvmlCheckReturn(ret) + return None + # Throws NVML_ERROR_NOT_SUPPORTED if hardware doesn't support setting auto boosted clocks + + +def nvmlDeviceSetDefaultAutoBoostedClocksEnabled(handle, enabled, flags): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetDefaultAutoBoostedClocksEnabled") + ret = fn(handle, _nvmlEnableState_t(enabled), c_uint(flags)) + _nvmlCheckReturn(ret) + return None + # Throws NVML_ERROR_NOT_SUPPORTED if hardware doesn't support setting auto boosted clocks + + +def nvmlDeviceSetGpuLockedClocks(handle, minGpuClockMHz, maxGpuClockMHz): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetGpuLockedClocks") + ret = fn(handle, c_uint(minGpuClockMHz), c_uint(maxGpuClockMHz)) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceResetGpuLockedClocks(handle): + fn = _nvmlGetFunctionPointer("nvmlDeviceResetGpuLockedClocks") + ret = fn(handle) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceSetMemoryLockedClocks(handle, minMemClockMHz, maxMemClockMHz): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetMemoryLockedClocks") + ret = fn(handle, c_uint(minMemClockMHz), c_uint(maxMemClockMHz)) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceResetMemoryLockedClocks(handle): + fn = _nvmlGetFunctionPointer("nvmlDeviceResetMemoryLockedClocks") + ret = fn(handle) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceGetClkMonStatus(handle, c_clkMonInfo=nvmlClkMonStatus_t()): + isReference = type(c_clkMonInfo) is not nvmlClkMonStatus_t + c_clkMonInfoRef = c_clkMonInfo if isReference else byref(c_clkMonInfo) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetClkMonStatus") + ret = fn(handle, c_clkMonInfoRef) + _nvmlCheckReturn(ret) + return NVML_SUCCESS if isReference else c_clkMonInfo + + +# Added in 4.304 +def nvmlDeviceSetApplicationsClocks(handle, maxMemClockMHz, maxGraphicsClockMHz): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetApplicationsClocks") + ret = fn(handle, c_uint(maxMemClockMHz), c_uint(maxGraphicsClockMHz)) + _nvmlCheckReturn(ret) + return None + + +# Added in 4.304 +def nvmlDeviceResetApplicationsClocks(handle): + fn = _nvmlGetFunctionPointer("nvmlDeviceResetApplicationsClocks") + ret = fn(handle) + _nvmlCheckReturn(ret) + return None + + +# Added in 4.304 +def nvmlDeviceSetPowerManagementLimit(handle, limit): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetPowerManagementLimit") + ret = fn(handle, c_uint(limit)) + _nvmlCheckReturn(ret) + return None + + +# Added in 4.304 +def nvmlDeviceSetGpuOperationMode(handle, mode): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetGpuOperationMode") + ret = fn(handle, _nvmlGpuOperationMode_t(mode)) + _nvmlCheckReturn(ret) + return None + + +# Added in 2.285 +def nvmlEventSetCreate(): + fn = _nvmlGetFunctionPointer("nvmlEventSetCreate") + eventSet = c_nvmlEventSet_t() + ret = fn(byref(eventSet)) + _nvmlCheckReturn(ret) + return eventSet + + +# Added in 2.285 +def nvmlDeviceRegisterEvents(handle, eventTypes, eventSet): + fn = _nvmlGetFunctionPointer("nvmlDeviceRegisterEvents") + ret = fn(handle, c_ulonglong(eventTypes), eventSet) + _nvmlCheckReturn(ret) + return None + + +# Added in 2.285 +def nvmlDeviceGetSupportedEventTypes(handle): + c_eventTypes = c_ulonglong() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedEventTypes") + ret = fn(handle, byref(c_eventTypes)) + _nvmlCheckReturn(ret) + return c_eventTypes.value + + +# raises NVML_ERROR_TIMEOUT exception on timeout +def nvmlEventSetWait_v2(eventSet, timeoutms): + fn = _nvmlGetFunctionPointer("nvmlEventSetWait_v2") + data = c_nvmlEventData_t() + ret = fn(eventSet, byref(data), c_uint(timeoutms)) + _nvmlCheckReturn(ret) + return data + + +def nvmlEventSetWait(eventSet, timeoutms): + return nvmlEventSetWait_v2(eventSet, timeoutms) + + +# Added in 2.285 +def nvmlEventSetFree(eventSet): + fn = _nvmlGetFunctionPointer("nvmlEventSetFree") + ret = fn(eventSet) + _nvmlCheckReturn(ret) + return None + + +# Added in 3.295 +def nvmlDeviceOnSameBoard(handle1, handle2): + fn = _nvmlGetFunctionPointer("nvmlDeviceOnSameBoard") + onSameBoard = c_int() + ret = fn(handle1, handle2, byref(onSameBoard)) + _nvmlCheckReturn(ret) + return onSameBoard.value != 0 + + +# Added in 3.295 +def nvmlDeviceGetCurrPcieLinkGeneration(handle): + fn = _nvmlGetFunctionPointer("nvmlDeviceGetCurrPcieLinkGeneration") + gen = c_uint() + ret = fn(handle, byref(gen)) + _nvmlCheckReturn(ret) + return gen.value + + +# Added in 3.295 +def nvmlDeviceGetMaxPcieLinkGeneration(handle): + fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxPcieLinkGeneration") + gen = c_uint() + ret = fn(handle, byref(gen)) + _nvmlCheckReturn(ret) + return gen.value + + +# Added in 3.295 +def nvmlDeviceGetCurrPcieLinkWidth(handle): + fn = _nvmlGetFunctionPointer("nvmlDeviceGetCurrPcieLinkWidth") + width = c_uint() + ret = fn(handle, byref(width)) + _nvmlCheckReturn(ret) + return width.value + + +# Added in 3.295 +def nvmlDeviceGetMaxPcieLinkWidth(handle): + fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxPcieLinkWidth") + width = c_uint() + ret = fn(handle, byref(width)) + _nvmlCheckReturn(ret) + return width.value + + +def nvmlDeviceGetGpuMaxPcieLinkGeneration(handle): + fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuMaxPcieLinkGeneration") + gen = c_uint() + ret = fn(handle, byref(gen)) + _nvmlCheckReturn(ret) + return gen.value + + +# Added in 4.304 +def nvmlDeviceGetSupportedClocksThrottleReasons(handle): + c_reasons = c_ulonglong() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedClocksThrottleReasons") + ret = fn(handle, byref(c_reasons)) + _nvmlCheckReturn(ret) + return c_reasons.value + + +def nvmlDeviceGetSupportedClocksEventReasons(handle): + c_reasons = c_ulonglong() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedClocksEventReasons") + ret = fn(handle, byref(c_reasons)) + _nvmlCheckReturn(ret) + return c_reasons.value + + +# Added in 4.304 +def nvmlDeviceGetCurrentClocksThrottleReasons(handle): + c_reasons = c_ulonglong() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetCurrentClocksThrottleReasons") + ret = fn(handle, byref(c_reasons)) + _nvmlCheckReturn(ret) + return c_reasons.value + + +def nvmlDeviceGetCurrentClocksEventReasons(handle): + c_reasons = c_ulonglong() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetCurrentClocksEventReasons") + ret = fn(handle, byref(c_reasons)) + _nvmlCheckReturn(ret) + return c_reasons.value + + +# Added in 5.319 +def nvmlDeviceGetIndex(handle): + fn = _nvmlGetFunctionPointer("nvmlDeviceGetIndex") + c_index = c_uint() + ret = fn(handle, byref(c_index)) + _nvmlCheckReturn(ret) + return c_index.value + + +# Added in 5.319 +def nvmlDeviceGetAccountingMode(handle): + c_mode = _nvmlEnableState_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetAccountingMode") + ret = fn(handle, byref(c_mode)) + _nvmlCheckReturn(ret) + return c_mode.value + + +def nvmlDeviceSetAccountingMode(handle, mode): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetAccountingMode") + ret = fn(handle, _nvmlEnableState_t(mode)) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceClearAccountingPids(handle): + fn = _nvmlGetFunctionPointer("nvmlDeviceClearAccountingPids") + ret = fn(handle) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceGetAccountingStats(handle, pid): + stats = c_nvmlAccountingStats_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetAccountingStats") + ret = fn(handle, c_uint(pid), byref(stats)) + _nvmlCheckReturn(ret) + if stats.maxMemoryUsage == NVML_VALUE_NOT_AVAILABLE_ulonglong.value: + # special case for WDDM on Windows, see comment above + stats.maxMemoryUsage = None + return stats + + +def nvmlDeviceGetAccountingPids(handle): + count = c_uint(nvmlDeviceGetAccountingBufferSize(handle)) + pids = (c_uint * count.value)() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetAccountingPids") + ret = fn(handle, byref(count), pids) + _nvmlCheckReturn(ret) + return list(map(int, pids[0 : count.value])) + + +def nvmlDeviceGetAccountingBufferSize(handle): + bufferSize = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetAccountingBufferSize") + ret = fn(handle, byref(bufferSize)) + _nvmlCheckReturn(ret) + return int(bufferSize.value) + + +def nvmlDeviceGetRetiredPages(device, sourceFilter): + c_source = _nvmlPageRetirementCause_t(sourceFilter) + c_count = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetRetiredPages") + + # First call will get the size + ret = fn(device, c_source, byref(c_count), None) + + # this should only fail with insufficient size + if (ret != NVML_SUCCESS) and (ret != NVML_ERROR_INSUFFICIENT_SIZE): + raise NVMLError(ret) + + # call again with a buffer + # oversize the array for the rare cases where additional pages + # are retired between NVML calls + c_count.value = c_count.value * 2 + 5 + page_array = c_ulonglong * c_count.value + c_pages = page_array() + ret = fn(device, c_source, byref(c_count), c_pages) + _nvmlCheckReturn(ret) + return list(map(int, c_pages[0 : c_count.value])) + + +def nvmlDeviceGetRetiredPages_v2(device, sourceFilter): + c_source = _nvmlPageRetirementCause_t(sourceFilter) + c_count = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetRetiredPages_v2") + + # First call will get the size + ret = fn(device, c_source, byref(c_count), None) + + # this should only fail with insufficient size + if (ret != NVML_SUCCESS) and (ret != NVML_ERROR_INSUFFICIENT_SIZE): + raise NVMLError(ret) + + # call again with a buffer + # oversize the array for the rare cases where additional pages + # are retired between NVML calls + c_count.value = c_count.value * 2 + 5 + page_array = c_ulonglong * c_count.value + c_pages = page_array() + times_array = c_ulonglong * c_count.value + c_times = times_array() + ret = fn(device, c_source, byref(c_count), c_pages, c_times) + _nvmlCheckReturn(ret) + return [ + {"address": int(c_pages[i]), "timestamp": int(c_times[i])} + for i in range(c_count.value) + ] + + +def nvmlDeviceGetRetiredPagesPendingStatus(device): + c_pending = _nvmlEnableState_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetRetiredPagesPendingStatus") + ret = fn(device, byref(c_pending)) + _nvmlCheckReturn(ret) + return int(c_pending.value) + + +def nvmlDeviceGetAPIRestriction(device, apiType): + c_permission = _nvmlEnableState_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetAPIRestriction") + ret = fn(device, _nvmlRestrictedAPI_t(apiType), byref(c_permission)) + _nvmlCheckReturn(ret) + return int(c_permission.value) + + +def nvmlDeviceSetAPIRestriction(handle, apiType, isRestricted): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetAPIRestriction") + ret = fn(handle, _nvmlRestrictedAPI_t(apiType), _nvmlEnableState_t(isRestricted)) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceGetBridgeChipInfo(handle): + bridgeHierarchy = c_nvmlBridgeChipHierarchy_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetBridgeChipInfo") + ret = fn(handle, byref(bridgeHierarchy)) + _nvmlCheckReturn(ret) + return bridgeHierarchy + + +def nvmlDeviceGetSamples(device, sampling_type, timeStamp): + c_sampling_type = _nvmlSamplingType_t(sampling_type) + c_time_stamp = c_ulonglong(timeStamp) + c_sample_count = c_uint(0) + c_sample_value_type = _nvmlValueType_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetSamples") + + ## First Call gets the size + ret = fn( + device, + c_sampling_type, + c_time_stamp, + byref(c_sample_value_type), + byref(c_sample_count), + None, + ) + + # Stop if this fails + if ret != NVML_SUCCESS: + raise NVMLError(ret) + + sampleArray = c_sample_count.value * c_nvmlSample_t + c_samples = sampleArray() + ret = fn( + device, + c_sampling_type, + c_time_stamp, + byref(c_sample_value_type), + byref(c_sample_count), + c_samples, + ) + _nvmlCheckReturn(ret) + return (c_sample_value_type.value, c_samples[0 : c_sample_count.value]) + + +def nvmlDeviceGetViolationStatus(device, perfPolicyType): + c_perfPolicy_type = _nvmlPerfPolicyType_t(perfPolicyType) + c_violTime = c_nvmlViolationTime_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetViolationStatus") + + ## Invoke the method to get violation time + ret = fn(device, c_perfPolicy_type, byref(c_violTime)) + _nvmlCheckReturn(ret) + return c_violTime + + +def nvmlDeviceGetPcieThroughput(device, counter): + c_util = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetPcieThroughput") + ret = fn(device, _nvmlPcieUtilCounter_t(counter), byref(c_util)) + _nvmlCheckReturn(ret) + return c_util.value + + +def nvmlSystemGetTopologyGpuSet(cpuNumber): + c_count = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlSystemGetTopologyGpuSet") + + # First call will get the size + ret = fn(cpuNumber, byref(c_count), None) + + if ret != NVML_SUCCESS: + raise NVMLError(ret) + # call again with a buffer + device_array = c_nvmlDevice_t * c_count.value + c_devices = device_array() + ret = fn(cpuNumber, byref(c_count), c_devices) + _nvmlCheckReturn(ret) + return list(c_devices[0 : c_count.value]) + + +def nvmlDeviceGetTopologyNearestGpus(device, level): + c_count = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetTopologyNearestGpus") + + # First call will get the size + ret = fn(device, level, byref(c_count), None) + + if ret != NVML_SUCCESS: + raise NVMLError(ret) + + # call again with a buffer + device_array = c_nvmlDevice_t * c_count.value + c_devices = device_array() + ret = fn(device, level, byref(c_count), c_devices) + _nvmlCheckReturn(ret) + return list(c_devices[0 : c_count.value]) + + +def nvmlDeviceGetTopologyCommonAncestor(device1, device2): + c_level = _nvmlGpuTopologyLevel_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetTopologyCommonAncestor") + ret = fn(device1, device2, byref(c_level)) + _nvmlCheckReturn(ret) + return c_level.value + + +def nvmlDeviceGetNvLinkUtilizationCounter(device, link, counter): + c_rxcounter = c_ulonglong() + c_txcounter = c_ulonglong() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkUtilizationCounter") + ret = fn(device, link, counter, byref(c_rxcounter), byref(c_txcounter)) + _nvmlCheckReturn(ret) + return (c_rxcounter.value, c_txcounter.value) + + +def nvmlDeviceFreezeNvLinkUtilizationCounter(device, link, counter, freeze): + fn = _nvmlGetFunctionPointer("nvmlDeviceFreezeNvLinkUtilizationCounter") + ret = fn(device, link, counter, freeze) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceResetNvLinkUtilizationCounter(device, link, counter): + fn = _nvmlGetFunctionPointer("nvmlDeviceResetNvLinkUtilizationCounter") + ret = fn(device, link, counter) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceSetNvLinkUtilizationControl(device, link, counter, control, reset): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetNvLinkUtilizationControl") + ret = fn(device, link, counter, byref(control), reset) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceGetNvLinkUtilizationControl(device, link, counter): + c_control = nvmlNvLinkUtilizationControl_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkUtilizationControl") + ret = fn(device, link, counter, byref(c_control)) + _nvmlCheckReturn(ret) + return c_control + + +def nvmlDeviceGetNvLinkCapability(device, link, capability): + c_capResult = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkCapability") + ret = fn(device, link, capability, byref(c_capResult)) + _nvmlCheckReturn(ret) + return c_capResult.value + + +def nvmlDeviceGetNvLinkErrorCounter(device, link, counter): + c_result = c_ulonglong() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkErrorCounter") + ret = fn(device, link, counter, byref(c_result)) + _nvmlCheckReturn(ret) + return c_result.value + + +def nvmlDeviceResetNvLinkErrorCounters(device, link): + fn = _nvmlGetFunctionPointer("nvmlDeviceResetNvLinkErrorCounters") + ret = fn(device, link) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceGetNvLinkRemotePciInfo(device, link): + c_pci = nvmlPciInfo_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkRemotePciInfo_v2") + ret = fn(device, link, byref(c_pci)) + _nvmlCheckReturn(ret) + return c_pci + + +def nvmlDeviceGetNvLinkRemoteDeviceType(handle, link): + c_type = _nvmlNvLinkDeviceType_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkRemoteDeviceType") + ret = fn(handle, link, byref(c_type)) + _nvmlCheckReturn(ret) + return c_type.value + + +def nvmlDeviceGetNvLinkState(device, link): + c_isActive = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkState") + ret = fn(device, link, byref(c_isActive)) + _nvmlCheckReturn(ret) + return c_isActive.value + + +def nvmlDeviceGetNvLinkVersion(device, link): + c_version = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkVersion") + ret = fn(device, link, byref(c_version)) + _nvmlCheckReturn(ret) + return c_version.value + + +def nvmlDeviceModifyDrainState(pciInfo, newState): + fn = _nvmlGetFunctionPointer("nvmlDeviceModifyDrainState") + ret = fn(pointer(pciInfo), newState) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceQueryDrainState(pciInfo): + c_newState = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceQueryDrainState") + ret = fn(pointer(pciInfo), byref(c_newState)) + _nvmlCheckReturn(ret) + return c_newState.value + + +def nvmlDeviceRemoveGpu(pciInfo): + fn = _nvmlGetFunctionPointer("nvmlDeviceRemoveGpu") + ret = fn(pointer(pciInfo)) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceDiscoverGpus(pciInfo): + fn = _nvmlGetFunctionPointer("nvmlDeviceDiscoverGpus") + ret = fn(pointer(pciInfo)) + _nvmlCheckReturn(ret) + return None + + +def nvmlDeviceGetFieldValues(handle, fieldIds): + values_arr = c_nvmlFieldValue_t * len(fieldIds) + values = values_arr() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetFieldValues") + + for i, fieldId in enumerate(fieldIds): + try: + (values[i].fieldId, values[i].scopeId) = fieldId + except TypeError: + values[i].fieldId = fieldId + + ret = fn(handle, c_int32(len(fieldIds)), byref(values)) + _nvmlCheckReturn(ret) + return values + + +def nvmlDeviceClearFieldValues(handle, fieldIds): + values_arr = c_nvmlFieldValue_t * len(fieldIds) + values = values_arr() + fn = _nvmlGetFunctionPointer("nvmlDeviceClearFieldValues") + + for i, fieldId in enumerate(fieldIds): + try: + (values[i].fieldId, values[i].scopeId) = fieldId + except TypeError: + values[i].fieldId = fieldId + + ret = fn(handle, c_int32(len(fieldIds)), byref(values)) + _nvmlCheckReturn(ret) + return values + + +def nvmlDeviceGetVirtualizationMode(handle): + c_virtualization_mode = c_ulonglong() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetVirtualizationMode") + ret = fn(handle, byref(c_virtualization_mode)) + _nvmlCheckReturn(ret) + return c_virtualization_mode.value + + +def nvmlDeviceSetVirtualizationMode(handle, virtualization_mode): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetVirtualizationMode") + return fn(handle, virtualization_mode) + + +def nvmlDeviceGetVgpuHeterogeneousMode(handle): + c_vgpuHeterogeneousMode = c_nvmlVgpuHeterogeneousMode_v1_t(0) + c_vgpuHeterogeneousMode.version = VgpuHeterogeneousMode_v1 + fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuHeterogeneousMode") + ret = fn(handle, byref(c_vgpuHeterogeneousMode)) + _nvmlCheckReturn(ret) + return c_vgpuHeterogeneousMode.mode + + +def nvmlDeviceSetVgpuHeterogeneousMode(handle, heterogeneous_mode): + c_vgpuHeterogeneousMode = c_nvmlVgpuHeterogeneousMode_v1_t(0) + c_vgpuHeterogeneousMode.version = VgpuHeterogeneousMode_v1 + c_vgpuHeterogeneousMode.mode = heterogeneous_mode + fn = _nvmlGetFunctionPointer("nvmlDeviceSetVgpuHeterogeneousMode") + ret = fn(handle, byref(c_vgpuHeterogeneousMode)) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlVgpuInstanceGetPlacementId(vgpuInstance): + c_placement = c_nvmlVgpuPlacementId_v1_t(0) + c_placement.version = VgpuPlacementId_v1 + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetPlacementId") + ret = fn(vgpuInstance, byref(c_placement)) + _nvmlCheckReturn(ret) + return c_placement.placementId + + +def nvmlDeviceGetVgpuTypeSupportedPlacements(handle, vgpuTypeId, mode=0, version=1): + c_max_instances = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetMaxInstances") + ret = fn(handle, vgpuTypeId, byref(c_max_instances)) + _nvmlCheckReturn(ret) + + if version == 2: + c_vgpu_placements = c_nvmlVgpuPlacementList_v2_t() + c_vgpu_placements.version = VgpuPlacementList_v2 + c_vgpu_placements.count = c_max_instances.value + c_vgpu_placements.mode = mode + elif version == 1: + c_vgpu_placements = c_nvmlVgpuPlacementList_v1_t() + c_vgpu_placements.version = VgpuPlacementList_v1 + else: + raise NVMLError(NVML_ERROR_ARGUMENT_VERSION_MISMATCH) + + c_placements = c_uint * c_max_instances.value + c_vgpu_placements.placementIds = c_placements() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuTypeSupportedPlacements") + ret = fn(handle, vgpuTypeId, byref(c_vgpu_placements)) + _nvmlCheckReturn(ret) + return c_vgpu_placements + + +def nvmlDeviceGetVgpuTypeCreatablePlacements(handle, vgpuTypeId, version=1): + c_max_instances = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetMaxInstances") + ret = fn(handle, vgpuTypeId, byref(c_max_instances)) + _nvmlCheckReturn(ret) + + if version == 2: + c_vgpu_placements = c_nvmlVgpuPlacementList_v2_t() + c_vgpu_placements.version = VgpuPlacementList_v2 + c_vgpu_placements.count = c_max_instances.value + elif version == 1: + c_vgpu_placements = c_nvmlVgpuPlacementList_v1_t() + c_vgpu_placements.version = VgpuPlacementList_v1 + + c_placements = c_uint * c_max_instances.value + c_vgpu_placements.placementIds = c_placements() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuTypeCreatablePlacements") + ret = fn(handle, vgpuTypeId, byref(c_vgpu_placements)) + _nvmlCheckReturn(ret) + return c_vgpu_placements + + +def nvmlGetVgpuDriverCapabilities(capability): + c_capResult = c_uint() + fn = _nvmlGetFunctionPointer("nvmlGetVgpuDriverCapabilities") + ret = fn(_nvmlVgpuDriverCapability_t(capability), byref(c_capResult)) + _nvmlCheckReturn(ret) + return c_capResult.value + + +def nvmlDeviceGetVgpuCapabilities(handle, capability): + c_capResult = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuCapabilities") + ret = fn(handle, _nvmlDeviceVgpuCapability_t(capability), byref(c_capResult)) + _nvmlCheckReturn(ret) + return c_capResult.value + + +def nvmlDeviceSetVgpuCapabilities(handle, capability, state): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetVgpuCapabilities") + ret = fn(handle, _nvmlDeviceVgpuCapability_t(capability), state) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlDeviceGetSupportedVgpus(handle): + # first call to get the size + c_vgpu_count = c_uint(0) + + fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedVgpus") + ret = fn(handle, byref(c_vgpu_count), None) + + if ret == NVML_SUCCESS: + # special case, no supported vGPUs + return [] + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: + # typical case + vgpu_type_ids_array = _nvmlVgpuTypeId_t * c_vgpu_count.value + c_vgpu_type_ids = vgpu_type_ids_array() + + # make the call again + ret = fn(handle, byref(c_vgpu_count), c_vgpu_type_ids) + _nvmlCheckReturn(ret) + vgpus = [] + for i in range(c_vgpu_count.value): + vgpus.append(c_vgpu_type_ids[i]) + return vgpus + else: + # error case + raise NVMLError(ret) + + +def nvmlDeviceGetCreatableVgpus(handle): + # first call to get the size + c_vgpu_count = c_uint(0) + + fn = _nvmlGetFunctionPointer("nvmlDeviceGetCreatableVgpus") + ret = fn(handle, byref(c_vgpu_count), None) + + if ret == NVML_SUCCESS: + # special case, no supported vGPUs + return [] + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: + # typical case + vgpu_type_ids_array = _nvmlVgpuTypeId_t * c_vgpu_count.value + c_vgpu_type_ids = vgpu_type_ids_array() + + # make the call again + ret = fn(handle, byref(c_vgpu_count), c_vgpu_type_ids) + _nvmlCheckReturn(ret) + vgpus = [] + for i in range(c_vgpu_count.value): + vgpus.append(c_vgpu_type_ids[i]) + return vgpus + else: + # error case + raise NVMLError(ret) + + +def nvmlVgpuTypeGetGpuInstanceProfileId(vgpuTypeId): + c_profile_id = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetGpuInstanceProfileId") + ret = fn(vgpuTypeId, byref(c_profile_id)) + _nvmlCheckReturn(ret) + return c_profile_id.value + + +@convertStrBytes +def nvmlVgpuTypeGetClass(vgpuTypeId): + c_class = create_string_buffer(NVML_DEVICE_NAME_BUFFER_SIZE) + c_buffer_size = c_uint(NVML_DEVICE_NAME_BUFFER_SIZE) + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetClass") + ret = fn(vgpuTypeId, c_class, byref(c_buffer_size)) + _nvmlCheckReturn(ret) + return c_class.value + + +@convertStrBytes +def nvmlVgpuTypeGetName(vgpuTypeId): + c_name = create_string_buffer(NVML_DEVICE_NAME_BUFFER_SIZE) + c_buffer_size = c_uint(NVML_DEVICE_NAME_BUFFER_SIZE) + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetName") + ret = fn(vgpuTypeId, c_name, byref(c_buffer_size)) + _nvmlCheckReturn(ret) + return c_name.value + + +def nvmlVgpuTypeGetDeviceID(vgpuTypeId): + c_device_id = c_ulonglong(0) + c_subsystem_id = c_ulonglong(0) + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetDeviceID") + ret = fn(vgpuTypeId, byref(c_device_id), byref(c_subsystem_id)) + _nvmlCheckReturn(ret) + return (c_device_id.value, c_subsystem_id.value) + + +def nvmlVgpuTypeGetFramebufferSize(vgpuTypeId): + c_fb_size = c_ulonglong(0) + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetFramebufferSize") + ret = fn(vgpuTypeId, byref(c_fb_size)) + _nvmlCheckReturn(ret) + return c_fb_size.value + + +def nvmlVgpuTypeGetNumDisplayHeads(vgpuTypeId): + c_num_heads = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetNumDisplayHeads") + ret = fn(vgpuTypeId, byref(c_num_heads)) + _nvmlCheckReturn(ret) + return c_num_heads.value + + +def nvmlVgpuTypeGetResolution(vgpuTypeId): + c_xdim = c_uint(0) + c_ydim = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetResolution") + ret = fn(vgpuTypeId, 0, byref(c_xdim), byref(c_ydim)) + _nvmlCheckReturn(ret) + return (c_xdim.value, c_ydim.value) + + +@convertStrBytes +def nvmlVgpuTypeGetLicense(vgpuTypeId): + c_license = create_string_buffer(NVML_GRID_LICENSE_BUFFER_SIZE) + c_buffer_size = c_uint(NVML_GRID_LICENSE_BUFFER_SIZE) + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetLicense") + ret = fn(vgpuTypeId, c_license, c_buffer_size) + _nvmlCheckReturn(ret) + return c_license.value + + +def nvmlVgpuTypeGetFrameRateLimit(vgpuTypeId): + c_frl_config = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetFrameRateLimit") + ret = fn(vgpuTypeId, byref(c_frl_config)) + _nvmlCheckReturn(ret) + return c_frl_config.value + + +def nvmlVgpuTypeGetGspHeapSize(vgpuTypeId): + c_gsp_heap = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetGspHeapSize") + ret = fn(vgpuTypeId, byref(c_gsp_heap)) + _nvmlCheckReturn(ret) + return c_gsp_heap.value + + +def nvmlVgpuTypeGetFbReservation(vgpuTypeId): + c_fb_reservation = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetFbReservation") + ret = fn(vgpuTypeId, byref(c_fb_reservation)) + _nvmlCheckReturn(ret) + return c_fb_reservation.value + + +def nvmlVgpuInstanceGetRuntimeStateSize(vgpuInstance): + c_runtime_state = nvmlVgpuRuntimeState_v1_t() + c_runtime_state.version = VgpuRuntimeState_v1 + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetRuntimeStateSize") + ret = fn(vgpuInstance, byref(c_runtime_state)) + _nvmlCheckReturn(ret) + return c_runtime_state + + +def nvmlVgpuTypeGetMaxInstances(handle, vgpuTypeId): + c_max_instances = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetMaxInstances") + ret = fn(handle, vgpuTypeId, byref(c_max_instances)) + _nvmlCheckReturn(ret) + return c_max_instances.value + + +def nvmlVgpuTypeGetMaxInstancesPerVm(vgpuTypeId): + c_max_instances_per_vm = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetMaxInstancesPerVm") + ret = fn(vgpuTypeId, byref(c_max_instances_per_vm)) + _nvmlCheckReturn(ret) + return c_max_instances_per_vm.value + + +def nvmlVgpuTypeGetBAR1Info(vgpuTypeId): + c_bar1Info = c_nvmlVgpuTypeBar1Info_v1_t(0) + c_bar1Info.version = VgpuTypeBar1Info_v1 + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetBAR1Info") + ret = fn(vgpuTypeId, byref(c_bar1Info)) + _nvmlCheckReturn(ret) + return c_bar1Info + + +def nvmlDeviceGetActiveVgpus(handle): + # first call to get the size + c_vgpu_count = c_uint(0) + + fn = _nvmlGetFunctionPointer("nvmlDeviceGetActiveVgpus") + ret = fn(handle, byref(c_vgpu_count), None) + + if ret == NVML_SUCCESS: + # special case, no active vGPUs + return [] + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: + # typical case + vgpu_instance_array = _nvmlVgpuInstance_t * c_vgpu_count.value + c_vgpu_instances = vgpu_instance_array() + + # make the call again + ret = fn(handle, byref(c_vgpu_count), c_vgpu_instances) + _nvmlCheckReturn(ret) + vgpus = [] + for i in range(c_vgpu_count.value): + vgpus.append(c_vgpu_instances[i]) + return vgpus + else: + # error case + raise NVMLError(ret) + + +@convertStrBytes +def nvmlVgpuInstanceGetVmID(vgpuInstance): + c_vm_id = create_string_buffer(NVML_DEVICE_UUID_BUFFER_SIZE) + c_buffer_size = c_uint(NVML_GRID_LICENSE_BUFFER_SIZE) + c_vm_id_type = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetVmID") + ret = fn(vgpuInstance, byref(c_vm_id), c_buffer_size, byref(c_vm_id_type)) + _nvmlCheckReturn(ret) + return (c_vm_id.value, c_vm_id_type.value) + + +@convertStrBytes +def nvmlVgpuInstanceGetUUID(vgpuInstance): + c_uuid = create_string_buffer(NVML_DEVICE_UUID_BUFFER_SIZE) + c_buffer_size = c_uint(NVML_DEVICE_UUID_BUFFER_SIZE) + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetUUID") + ret = fn(vgpuInstance, byref(c_uuid), c_buffer_size) + _nvmlCheckReturn(ret) + return c_uuid.value + + +@convertStrBytes +def nvmlVgpuInstanceGetMdevUUID(vgpuInstance): + c_uuid = create_string_buffer(NVML_DEVICE_UUID_BUFFER_SIZE) + c_buffer_size = c_uint(NVML_DEVICE_UUID_BUFFER_SIZE) + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetMdevUUID") + ret = fn(vgpuInstance, byref(c_uuid), c_buffer_size) + _nvmlCheckReturn(ret) + return c_uuid.value + + +@convertStrBytes +def nvmlVgpuInstanceGetVmDriverVersion(vgpuInstance): + c_driver_version = create_string_buffer(NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE) + c_buffer_size = c_uint(NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE) + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetVmDriverVersion") + ret = fn(vgpuInstance, byref(c_driver_version), c_buffer_size) + _nvmlCheckReturn(ret) + return c_driver_version.value + + +def nvmlVgpuInstanceGetLicenseStatus(vgpuInstance): + c_license_status = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetLicenseStatus") + ret = fn(vgpuInstance, byref(c_license_status)) + _nvmlCheckReturn(ret) + return c_license_status.value + + +def nvmlVgpuInstanceGetLicenseInfo_v2(vgpuInstance): + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetLicenseInfo_v2") + c_license_info = c_nvmlVgpuLicenseInfo_t() + ret = fn(vgpuInstance, byref(c_license_info)) + _nvmlCheckReturn(ret) + return c_license_info + + +def nvmlVgpuInstanceGetLicenseInfo(vgpuInstance): + return nvmlVgpuInstanceGetLicenseInfo_v2(vgpuInstance) + + +def nvmlVgpuInstanceGetFrameRateLimit(vgpuInstance): + c_frl = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetFrameRateLimit") + ret = fn(vgpuInstance, byref(c_frl)) + _nvmlCheckReturn(ret) + return c_frl.value + + +def nvmlVgpuInstanceGetEccMode(vgpuInstance): + c_mode = _nvmlEnableState_t() + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetEccMode") + ret = fn(vgpuInstance, byref(c_mode)) + _nvmlCheckReturn(ret) + return c_mode.value + + +def nvmlVgpuInstanceGetType(vgpuInstance): + c_vgpu_type = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetType") + ret = fn(vgpuInstance, byref(c_vgpu_type)) + _nvmlCheckReturn(ret) + return c_vgpu_type.value + + +def nvmlVgpuInstanceGetEncoderCapacity(vgpuInstance): + c_encoder_capacity = c_ulonglong(0) + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetEncoderCapacity") + ret = fn(vgpuInstance, byref(c_encoder_capacity)) + _nvmlCheckReturn(ret) + return c_encoder_capacity.value + + +def nvmlVgpuInstanceSetEncoderCapacity(vgpuInstance, encoder_capacity): + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceSetEncoderCapacity") + return fn(vgpuInstance, encoder_capacity) + + +def nvmlVgpuInstanceGetFbUsage(vgpuInstance): + c_fb_usage = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetFbUsage") + ret = fn(vgpuInstance, byref(c_fb_usage)) + _nvmlCheckReturn(ret) + return c_fb_usage.value + + +def nvmlVgpuTypeGetCapabilities(vgpuTypeId, capability): + c_cap_result = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetCapabilities") + ret = fn(vgpuTypeId, _nvmlVgpuCapability_t(capability), byref(c_cap_result)) + _nvmlCheckReturn(ret) + return c_cap_result.value + + +def nvmlVgpuInstanceGetGpuInstanceId(vgpuInstance): + c_id = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetGpuInstanceId") + ret = fn(vgpuInstance, byref(c_id)) + _nvmlCheckReturn(ret) + return c_id.value + + +@convertStrBytes +def nvmlVgpuInstanceGetGpuPciId(vgpuInstance): + c_vgpuPciId = create_string_buffer(NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE) + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetGpuPciId") + ret = fn( + vgpuInstance, c_vgpuPciId, byref(c_uint(NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE)) + ) + _nvmlCheckReturn(ret) + return c_vgpuPciId.value + + +def nvmlDeviceGetVgpuUtilization(handle, timeStamp): + # first call to get the size + c_vgpu_count = c_uint(0) + c_time_stamp = c_ulonglong(timeStamp) + c_sample_value_type = _nvmlValueType_t() + + fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuUtilization") + ret = fn( + handle, c_time_stamp, byref(c_sample_value_type), byref(c_vgpu_count), None + ) + + if ret == NVML_SUCCESS: + # special case, no active vGPUs + return [] + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: + # typical case + sampleArray = c_vgpu_count.value * c_nvmlVgpuInstanceUtilizationSample_t + c_samples = sampleArray() + + # make the call again + ret = fn( + handle, + c_time_stamp, + byref(c_sample_value_type), + byref(c_vgpu_count), + c_samples, + ) + _nvmlCheckReturn(ret) + + return c_samples[0 : c_vgpu_count.value] + else: + # error case + raise NVMLError(ret) + + +def nvmlDeviceGetVgpuInstancesUtilizationInfo(handle, timeStamp): + # first call to get the size + c_time_stamp = c_ulonglong(timeStamp) + c_vgpuUtilInfo = c_nvmlVgpuInstancesUtilizationInfo_v1_t(0) + c_vgpuUtilInfo.version = VgpuInstancesUtilizationInfo_v1 + c_vgpuUtilInfo.sampleValType = _nvmlValueType_t() + c_vgpuUtilInfo.vgpuInstanceCount = c_uint(0) + c_vgpuUtilInfo.lastSeenTimeStamp = c_time_stamp + + fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuInstancesUtilizationInfo") + ret = fn(handle, byref(c_vgpuUtilInfo)) + + if ret == NVML_SUCCESS: + # special case, no active vGPUs + return [] + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: + # typical case + sampleArray = ( + c_vgpuUtilInfo.vgpuInstanceCount * c_nvmlVgpuInstanceUtilizationInfo_v1_t + ) + c_samples = sampleArray() + c_vgpuUtilInfo.vgpuUtilArray = c_samples + + # make the call again + ret = fn(handle, byref(c_vgpuUtilInfo)) + _nvmlCheckReturn(ret) + + return c_samples[0 : c_vgpuUtilInfo.vgpuInstanceCount] + else: + # error case + raise NVMLError(ret) + + +def nvmlDeviceGetP2PStatus(device1, device2, p2pIndex): + c_p2pstatus = _nvmlGpuP2PStatus_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetP2PStatus") + ret = fn(device1, device2, p2pIndex, byref(c_p2pstatus)) + _nvmlCheckReturn(ret) + return c_p2pstatus.value + + +def nvmlDeviceGetGridLicensableFeatures_v4(handle): + c_get_grid_licensable_features = c_nvmlGridLicensableFeatures_v4_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetGridLicensableFeatures_v4") + ret = fn(handle, byref(c_get_grid_licensable_features)) + _nvmlCheckReturn(ret) + + return c_get_grid_licensable_features + + +def nvmlDeviceGetGridLicensableFeatures(handle): + return nvmlDeviceGetGridLicensableFeatures_v4(handle) + + +def nvmlDeviceGetGspFirmwareVersion(handle, version=None): + isUserDefined = version is not None + if not isUserDefined: + version = (c_char * NVML_GSP_FIRMWARE_VERSION_BUF_SIZE)() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetGspFirmwareVersion") + ret = fn(handle, version) + _nvmlCheckReturn(ret) + return NVML_SUCCESS if isUserDefined else version.value + + +def nvmlDeviceGetGspFirmwareMode(handle, isEnabled=c_uint(), defaultMode=c_uint()): + isReference = type(isEnabled) is not c_uint + isEnabledRef = isEnabled if isReference else byref(isEnabled) + defaultModeRef = defaultMode if isReference else byref(defaultMode) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetGspFirmwareMode") + ret = fn(handle, isEnabledRef, defaultModeRef) + _nvmlCheckReturn(ret) + return NVML_SUCCESS if isReference else [isEnabled.value, defaultMode.value] + + +def nvmlDeviceGetEncoderCapacity(handle, encoderQueryType): + c_encoder_capacity = c_ulonglong(0) + c_encoderQuery_type = _nvmlEncoderQueryType_t(encoderQueryType) + + fn = _nvmlGetFunctionPointer("nvmlDeviceGetEncoderCapacity") + ret = fn(handle, c_encoderQuery_type, byref(c_encoder_capacity)) + _nvmlCheckReturn(ret) + return c_encoder_capacity.value + + +def nvmlDeviceGetVgpuProcessUtilization(handle, timeStamp): + # first call to get the size + c_vgpu_count = c_uint(0) + c_time_stamp = c_ulonglong(timeStamp) + + fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuProcessUtilization") + ret = fn(handle, c_time_stamp, byref(c_vgpu_count), None) + + if ret == NVML_SUCCESS: + # special case, no active vGPUs + return [] + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: + # typical case + sampleArray = c_vgpu_count.value * c_nvmlVgpuProcessUtilizationSample_t + c_samples = sampleArray() + + # make the call again + ret = fn(handle, c_time_stamp, byref(c_vgpu_count), c_samples) + _nvmlCheckReturn(ret) + + return c_samples[0 : c_vgpu_count.value] + else: + # error case + raise NVMLError(ret) + + +def nvmlDeviceGetVgpuProcessesUtilizationInfo(handle, timeStamp): + # first call to get the size + c_time_stamp = c_ulonglong(timeStamp) + c_vgpuProcUtilInfo = c_nvmlVgpuProcessesUtilizationInfo_v1_t(0) + c_vgpuProcUtilInfo.version = VgpuProcessesUtilizationInfo_v1 + c_vgpuProcUtilInfo.vgpuProcessCount = c_uint(0) + c_vgpuProcUtilInfo.lastSeenTimeStamp = c_time_stamp + + fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuProcessesUtilizationInfo") + ret = fn(handle, byref(c_vgpuProcUtilInfo)) + + if ret == NVML_SUCCESS: + # special case, no active vGPUs + return [] + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: + # typical case + sampleArray = ( + c_vgpuProcUtilInfo.vgpuProcessCount * c_nvmlVgpuProcessUtilizationInfo_v1_t + ) + c_samples = sampleArray() + c_vgpuProcUtilInfo.vgpuProcUtilArray = c_samples + + # make the call again + ret = fn(handle, byref(c_vgpuProcUtilInfo)) + _nvmlCheckReturn(ret) + + return c_samples[0 : c_vgpuProcUtilInfo.vgpuProcessCount] + else: + # error case + raise NVMLError(ret) + + +def nvmlDeviceGetEncoderStats(handle): + c_encoderCount = c_ulonglong(0) + c_encodeFps = c_ulonglong(0) + c_encoderLatency = c_ulonglong(0) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetEncoderStats") + ret = fn(handle, byref(c_encoderCount), byref(c_encodeFps), byref(c_encoderLatency)) + _nvmlCheckReturn(ret) + return (c_encoderCount.value, c_encodeFps.value, c_encoderLatency.value) + + +def nvmlDeviceGetEncoderSessions(handle): + # first call to get the size + c_session_count = c_uint(0) + + fn = _nvmlGetFunctionPointer("nvmlDeviceGetEncoderSessions") + ret = fn(handle, byref(c_session_count), None) + + if ret == NVML_SUCCESS: + if c_session_count.value != 0: + # typical case + session_array = c_nvmlEncoderSession_t * c_session_count.value + c_sessions = session_array() + + # make the call again + ret = fn(handle, byref(c_session_count), c_sessions) + _nvmlCheckReturn(ret) + sessions = [] + for i in range(c_session_count.value): + sessions.append(c_sessions[i]) + return sessions + else: + return [] # no active sessions + else: + # error case + raise NVMLError(ret) + + +def nvmlDeviceGetFBCStats(handle): + c_fbcStats = c_nvmlFBCStats_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetFBCStats") + ret = fn(handle, byref(c_fbcStats)) + _nvmlCheckReturn(ret) + return c_fbcStats + + +def nvmlDeviceGetFBCSessions(handle): + # first call to get the size + c_session_count = c_uint(0) + + fn = _nvmlGetFunctionPointer("nvmlDeviceGetFBCSessions") + ret = fn(handle, byref(c_session_count), None) + + if ret == NVML_SUCCESS: + if c_session_count.value != 0: + # typical case + session_array = c_nvmlFBCSession_t * c_session_count.value + c_sessions = session_array() + + # make the call again + ret = fn(handle, byref(c_session_count), c_sessions) + _nvmlCheckReturn(ret) + sessions = [] + for i in range(c_session_count.value): + sessions.append(c_sessions[i]) + return sessions + else: + return [] # no active sessions + else: + # error case + raise NVMLError(ret) + + +def nvmlVgpuInstanceGetEncoderStats(vgpuInstance): + c_encoderCount = c_ulonglong(0) + c_encodeFps = c_ulonglong(0) + c_encoderLatency = c_ulonglong(0) + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetEncoderStats") + ret = fn( + vgpuInstance, byref(c_encoderCount), byref(c_encodeFps), byref(c_encoderLatency) + ) + _nvmlCheckReturn(ret) + return (c_encoderCount.value, c_encodeFps.value, c_encoderLatency.value) + + +def nvmlVgpuInstanceGetEncoderSessions(vgpuInstance): + # first call to get the size + c_session_count = c_uint(0) + + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetEncoderSessions") + ret = fn(vgpuInstance, byref(c_session_count), None) + + if ret == NVML_SUCCESS: + if c_session_count.value != 0: + # typical case + session_array = c_nvmlEncoderSession_t * c_session_count.value + c_sessions = session_array() + + # make the call again + ret = fn(vgpuInstance, byref(c_session_count), c_sessions) + _nvmlCheckReturn(ret) + sessions = [] + for i in range(c_session_count.value): + sessions.append(c_sessions[i]) + return sessions + else: + return [] # no active sessions + else: + # error case + raise NVMLError(ret) + + +def nvmlVgpuInstanceGetFBCStats(vgpuInstance): + c_fbcStats = c_nvmlFBCStats_t() + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetFBCStats") + ret = fn(vgpuInstance, byref(c_fbcStats)) + _nvmlCheckReturn(ret) + return c_fbcStats + + +def nvmlVgpuInstanceGetFBCSessions(vgpuInstance): + # first call to get the size + c_session_count = c_uint(0) + + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetFBCSessions") + ret = fn(vgpuInstance, byref(c_session_count), None) + + if ret == NVML_SUCCESS: + if c_session_count.value != 0: + # typical case + session_array = c_nvmlFBCSession_t * c_session_count.value + c_sessions = session_array() + + # make the call again + ret = fn(vgpuInstance, byref(c_session_count), c_sessions) + _nvmlCheckReturn(ret) + sessions = [] + for i in range(c_session_count.value): + sessions.append(c_sessions[i]) + return sessions + else: + return [] # no active sessions + else: + # error case + raise NVMLError(ret) + + +def nvmlDeviceGetProcessUtilization(handle, timeStamp): + # first call to get the size + c_count = c_uint(0) + c_time_stamp = c_ulonglong(timeStamp) + + fn = _nvmlGetFunctionPointer("nvmlDeviceGetProcessUtilization") + ret = fn(handle, None, byref(c_count), c_time_stamp) + + if ret == NVML_ERROR_INSUFFICIENT_SIZE: + # typical case + sampleArray = c_count.value * c_nvmlProcessUtilizationSample_t + c_samples = sampleArray() + + # make the call again + ret = fn(handle, c_samples, byref(c_count), c_time_stamp) + _nvmlCheckReturn(ret) + + return c_samples[0 : c_count.value] + else: + # error case + raise NVMLError(ret) + + +def nvmlDeviceGetProcessesUtilizationInfo(handle, timeStamp): + # first call to get the size + c_time_stamp = c_ulonglong(timeStamp) + c_processesUtilInfo = c_nvmlProcessesUtilizationInfo_v1_t(0) + c_processesUtilInfo.version = ProcessesUtilizationInfo_v1 + c_processesUtilInfo.processSamplesCount = c_uint(0) + c_processesUtilInfo.lastSeenTimeStamp = c_time_stamp + + fn = _nvmlGetFunctionPointer("nvmlDeviceGetProcessesUtilizationInfo") + ret = fn(handle, byref(c_processesUtilInfo)) + + if ret == NVML_ERROR_INSUFFICIENT_SIZE: + # typical case + sampleArray = ( + c_processesUtilInfo.processSamplesCount * c_nvmlProcessUtilizationInfo_v1_t + ) + c_samples = sampleArray() + c_processesUtilInfo.procUtilArray = c_samples + + # make the call again + ret = fn(handle, byref(c_processesUtilInfo)) + _nvmlCheckReturn(ret) + + return c_samples[0 : c_processesUtilInfo.processSamplesCount] + else: + # error case + raise NVMLError(ret) + + +def nvmlVgpuInstanceGetMetadata(vgpuInstance): + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetMetadata") + c_vgpuMetadata = c_nvmlVgpuMetadata_t() + c_bufferSize = c_uint(0) + # Make the first NVML API call to get the c_bufferSize value. + # We have already allocated required buffer above. + ret = fn(vgpuInstance, byref(c_vgpuMetadata), byref(c_bufferSize)) + if ret == NVML_ERROR_INSUFFICIENT_SIZE: + ret = fn(vgpuInstance, byref(c_vgpuMetadata), byref(c_bufferSize)) + _nvmlCheckReturn(ret) + else: + raise NVMLError(ret) + return c_vgpuMetadata + + +def nvmlDeviceGetVgpuMetadata(handle): + fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuMetadata") + c_vgpuPgpuMetadata = c_nvmlVgpuPgpuMetadata_t() + c_bufferSize = c_uint(0) + # Make the first NVML API call to get the c_bufferSize value. + # We have already allocated required buffer above. + ret = fn(handle, byref(c_vgpuPgpuMetadata), byref(c_bufferSize)) + if ret == NVML_ERROR_INSUFFICIENT_SIZE: + ret = fn(handle, byref(c_vgpuPgpuMetadata), byref(c_bufferSize)) + _nvmlCheckReturn(ret) + else: + raise NVMLError(ret) + return c_vgpuPgpuMetadata + + +def nvmlGetVgpuCompatibility(vgpuMetadata, pgpuMetadata): + fn = _nvmlGetFunctionPointer("nvmlGetVgpuCompatibility") + c_vgpuPgpuCompatibility = c_nvmlVgpuPgpuCompatibility_t() + ret = fn(byref(vgpuMetadata), byref(pgpuMetadata), byref(c_vgpuPgpuCompatibility)) + _nvmlCheckReturn(ret) + return c_vgpuPgpuCompatibility + + +@convertStrBytes +def nvmlDeviceGetPgpuMetadataString(handle): + fn = _nvmlGetFunctionPointer("nvmlDeviceGetPgpuMetadataString") + c_pgpuMetadata = create_string_buffer(NVML_VGPU_PGPU_METADATA_OPAQUE_DATA_SIZE) + c_bufferSize = c_uint(0) + # Make the first NVML API call to get the c_bufferSize value. + # We have already allocated required buffer above. + ret = fn(handle, byref(c_pgpuMetadata), byref(c_bufferSize)) + if ret == NVML_ERROR_INSUFFICIENT_SIZE: + ret = fn(handle, byref(c_pgpuMetadata), byref(c_bufferSize)) + _nvmlCheckReturn(ret) + else: + raise NVMLError(ret) + return (c_pgpuMetadata.value, c_bufferSize.value) + + +def nvmlDeviceGetVgpuSchedulerLog(handle): + c_vgpu_sched_log = c_nvmlVgpuSchedulerLog_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuSchedulerLog") + ret = fn(handle, byref(c_vgpu_sched_log)) + _nvmlCheckReturn(ret) + return c_vgpu_sched_log + + +def nvmlDeviceGetVgpuSchedulerState(handle): + c_vgpu_sched_state = c_nvmlVgpuSchedulerGetState_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuSchedulerState") + ret = fn(handle, byref(c_vgpu_sched_state)) + _nvmlCheckReturn(ret) + return c_vgpu_sched_state + + +def nvmlDeviceGetVgpuSchedulerCapabilities(handle): + c_vgpu_sched_caps = c_nvmlVgpuSchedulerCapabilities_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuSchedulerCapabilities") + ret = fn(handle, byref(c_vgpu_sched_caps)) + _nvmlCheckReturn(ret) + return c_vgpu_sched_caps + + +def nvmlDeviceSetVgpuSchedulerState(handle, sched_state): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetVgpuSchedulerState") + ret = fn(handle, byref(sched_state)) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlSetVgpuVersion(vgpuVersion): + fn = _nvmlGetFunctionPointer("nvmlSetVgpuVersion") + ret = fn(byref(vgpuVersion)) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlGetVgpuVersion(supported=None, current=None): + isUserDefined = (supported is not None) or (current is not None) + if not isUserDefined: + supported = c_nvmlVgpuVersion_t() + current = c_nvmlVgpuVersion_t() + fn = _nvmlGetFunctionPointer("nvmlGetVgpuVersion") + ret = fn(byref(supported), byref(current)) + _nvmlCheckReturn(ret) + return ( + NVML_SUCCESS + if isUserDefined + else [ + (supported.minVersion, supported.maxVersion), + (current.minVersion, current.maxVersion), + ] + ) + + +def nvmlVgpuInstanceGetAccountingMode(vgpuInstance): + c_mode = _nvmlEnableState_t() + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetAccountingMode") + ret = fn(vgpuInstance, byref(c_mode)) + _nvmlCheckReturn(ret) + return c_mode.value + + +def nvmlVgpuInstanceGetAccountingPids(vgpuInstance): + c_pidCount = c_uint() + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetAccountingPids") + ret = fn(vgpuInstance, byref(c_pidCount), None) + if ret == NVML_ERROR_INSUFFICIENT_SIZE: + sampleArray = c_pidCount.value * c_uint + c_pidArray = sampleArray() + ret = fn(vgpuInstance, byref(c_pidCount), byref(c_pidArray)) + _nvmlCheckReturn(ret) + else: + raise NVMLError(ret) + return (c_pidCount, c_pidArray) + + +def nvmlVgpuInstanceGetAccountingStats(vgpuInstance, pid): + c_accountingStats = c_nvmlAccountingStats_t() + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetAccountingStats") + ret = fn(vgpuInstance, pid, byref(c_accountingStats)) + _nvmlCheckReturn(ret) + return c_accountingStats + + +def nvmlVgpuInstanceClearAccountingPids(vgpuInstance): + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceClearAccountingPids") + ret = fn(vgpuInstance) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlGetExcludedDeviceCount(): + c_count = c_uint() + fn = _nvmlGetFunctionPointer("nvmlGetExcludedDeviceCount") + ret = fn(byref(c_count)) + _nvmlCheckReturn(ret) + return c_count.value + + +def nvmlGetExcludedDeviceInfoByIndex(index): + c_index = c_uint(index) + info = c_nvmlExcludedDeviceInfo_t() + fn = _nvmlGetFunctionPointer("nvmlGetExcludedDeviceInfoByIndex") + ret = fn(c_index, byref(info)) + _nvmlCheckReturn(ret) + return info + + +def nvmlDeviceGetHostVgpuMode(handle): + c_host_vgpu_mode = _nvmlHostVgpuMode_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetHostVgpuMode") + ret = fn(handle, byref(c_host_vgpu_mode)) + _nvmlCheckReturn(ret) + return c_host_vgpu_mode.value + + +def nvmlDeviceSetMigMode(device, mode): + c_activationStatus = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceSetMigMode") + ret = fn(device, mode, byref(c_activationStatus)) + _nvmlCheckReturn(ret) + return c_activationStatus.value + + +def nvmlDeviceGetMigMode(device): + c_currentMode = c_uint() + c_pendingMode = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetMigMode") + ret = fn(device, byref(c_currentMode), byref(c_pendingMode)) + _nvmlCheckReturn(ret) + return [c_currentMode.value, c_pendingMode.value] + + +def nvmlDeviceGetGpuInstanceProfileInfo(device, profile, version=2): + if version == 2: + c_info = c_nvmlGpuInstanceProfileInfo_v2_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstanceProfileInfoV") + elif version == 1: + c_info = c_nvmlGpuInstanceProfileInfo_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstanceProfileInfo") + else: + raise NVMLError(NVML_ERROR_FUNCTION_NOT_FOUND) + ret = fn(device, profile, byref(c_info)) + _nvmlCheckReturn(ret) + return c_info + + +# Define function alias for the API exposed by NVML +nvmlDeviceGetGpuInstanceProfileInfoV = nvmlDeviceGetGpuInstanceProfileInfo + + +def nvmlDeviceGetGpuInstanceRemainingCapacity(device, profileId): + c_count = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstanceRemainingCapacity") + ret = fn(device, profileId, byref(c_count)) + _nvmlCheckReturn(ret) + return c_count.value + + +def nvmlDeviceGetGpuInstancePossiblePlacements( + device, profileId, placementsRef, countRef +): + fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstancePossiblePlacements_v2") + ret = fn(device, profileId, placementsRef, countRef) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlDeviceCreateGpuInstance(device, profileId): + c_instance = c_nvmlGpuInstance_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceCreateGpuInstance") + ret = fn(device, profileId, byref(c_instance)) + _nvmlCheckReturn(ret) + return c_instance + + +def nvmlDeviceCreateGpuInstanceWithPlacement(device, profileId, placement): + c_instance = c_nvmlGpuInstance_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceCreateGpuInstanceWithPlacement") + ret = fn(device, profileId, placement, byref(c_instance)) + _nvmlCheckReturn(ret) + return c_instance + + +def nvmlGpuInstanceDestroy(gpuInstance): + fn = _nvmlGetFunctionPointer("nvmlGpuInstanceDestroy") + ret = fn(gpuInstance) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlDeviceGetGpuInstances(device, profileId, gpuInstancesRef, countRef): + fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstances") + ret = fn(device, profileId, gpuInstancesRef, countRef) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlDeviceGetGpuInstanceById(device, gpuInstanceId): + c_instance = c_nvmlGpuInstance_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstanceById") + ret = fn(device, gpuInstanceId, byref(c_instance)) + _nvmlCheckReturn(ret) + return c_instance + + +def nvmlGpuInstanceGetInfo(gpuInstance): + c_info = c_nvmlGpuInstanceInfo_t() + fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetInfo") + ret = fn(gpuInstance, byref(c_info)) + _nvmlCheckReturn(ret) + return c_info + + +def nvmlGpuInstanceGetComputeInstanceProfileInfo( + device, profile, engProfile, version=2 +): + if version == 2: + c_info = c_nvmlComputeInstanceProfileInfo_v2_t() + fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstanceProfileInfoV") + elif version == 1: + c_info = c_nvmlComputeInstanceProfileInfo_t() + fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstanceProfileInfo") + else: + raise NVMLError(NVML_ERROR_FUNCTION_NOT_FOUND) + ret = fn(device, profile, engProfile, byref(c_info)) + _nvmlCheckReturn(ret) + return c_info + + +# Define function alias for the API exposed by NVML +nvmlGpuInstanceGetComputeInstanceProfileInfoV = ( + nvmlGpuInstanceGetComputeInstanceProfileInfo +) + + +def nvmlGpuInstanceGetComputeInstanceRemainingCapacity(gpuInstance, profileId): + c_count = c_uint() + fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstanceRemainingCapacity") + ret = fn(gpuInstance, profileId, byref(c_count)) + _nvmlCheckReturn(ret) + return c_count.value + + +def nvmlGpuInstanceGetComputeInstancePossiblePlacements( + gpuInstance, profileId, placementsRef, countRef +): + fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstancePossiblePlacements") + ret = fn(gpuInstance, profileId, placementsRef, countRef) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlGpuInstanceCreateComputeInstance(gpuInstance, profileId): + c_instance = c_nvmlComputeInstance_t() + fn = _nvmlGetFunctionPointer("nvmlGpuInstanceCreateComputeInstance") + ret = fn(gpuInstance, profileId, byref(c_instance)) + _nvmlCheckReturn(ret) + return c_instance + + +def nvmlGpuInstanceCreateComputeInstanceWithPlacement( + gpuInstance, profileId, placement +): + c_instance = c_nvmlComputeInstance_t() + fn = _nvmlGetFunctionPointer("nvmlGpuInstanceCreateComputeInstanceWithPlacement") + ret = fn(gpuInstance, profileId, placement, byref(c_instance)) + _nvmlCheckReturn(ret) + return c_instance + + +def nvmlComputeInstanceDestroy(computeInstance): + fn = _nvmlGetFunctionPointer("nvmlComputeInstanceDestroy") + ret = fn(computeInstance) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlGpuInstanceGetComputeInstances( + gpuInstance, profileId, computeInstancesRef, countRef +): + fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstances") + ret = fn(gpuInstance, profileId, computeInstancesRef, countRef) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlGpuInstanceGetComputeInstanceById(gpuInstance, computeInstanceId): + c_instance = c_nvmlComputeInstance_t() + fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstanceById") + ret = fn(gpuInstance, computeInstanceId, byref(c_instance)) + _nvmlCheckReturn(ret) + return c_instance + + +def nvmlComputeInstanceGetInfo_v2(computeInstance): + c_info = c_nvmlComputeInstanceInfo_t() + fn = _nvmlGetFunctionPointer("nvmlComputeInstanceGetInfo_v2") + ret = fn(computeInstance, byref(c_info)) + _nvmlCheckReturn(ret) + return c_info + + +def nvmlComputeInstanceGetInfo(computeInstance): + return nvmlComputeInstanceGetInfo_v2(computeInstance) + + +def nvmlDeviceIsMigDeviceHandle(device): + c_isMigDevice = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceIsMigDeviceHandle") + ret = fn(device, byref(c_isMigDevice)) + _nvmlCheckReturn(ret) + return c_isMigDevice + + +def nvmlDeviceGetGpuInstanceId(device): + c_gpuInstanceId = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstanceId") + ret = fn(device, byref(c_gpuInstanceId)) + _nvmlCheckReturn(ret) + return c_gpuInstanceId.value + + +def nvmlDeviceGetComputeInstanceId(device): + c_computeInstanceId = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetComputeInstanceId") + ret = fn(device, byref(c_computeInstanceId)) + _nvmlCheckReturn(ret) + return c_computeInstanceId.value + + +def nvmlDeviceGetMaxMigDeviceCount(device): + c_count = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxMigDeviceCount") + ret = fn(device, byref(c_count)) + _nvmlCheckReturn(ret) + return c_count.value + + +def nvmlDeviceGetMigDeviceHandleByIndex(device, index): + c_index = c_uint(index) + migDevice = c_nvmlDevice_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetMigDeviceHandleByIndex") + ret = fn(device, c_index, byref(migDevice)) + _nvmlCheckReturn(ret) + return migDevice + + +def nvmlDeviceGetDeviceHandleFromMigDeviceHandle(migDevice): + device = c_nvmlDevice_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetDeviceHandleFromMigDeviceHandle") + ret = fn(migDevice, byref(device)) + _nvmlCheckReturn(ret) + return device + + +def nvmlDeviceGetAttributes_v2(device): + c_attrs = c_nvmlDeviceAttributes() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetAttributes_v2") + ret = fn(device, byref(c_attrs)) + _nvmlCheckReturn(ret) + return c_attrs + + +def nvmlDeviceGetAttributes(device): + return nvmlDeviceGetAttributes_v2(device) + + +def nvmlDeviceGetRemappedRows(device): + fn = _nvmlGetFunctionPointer("nvmlDeviceGetRemappedRows") + c_corr = c_uint() + c_unc = c_uint() + c_bpending = c_uint() + c_bfailure = c_uint() + ret = fn(device, byref(c_corr), byref(c_unc), byref(c_bpending), byref(c_bfailure)) + _nvmlCheckReturn(ret) + return (c_corr.value, c_unc.value, c_bpending.value, c_bfailure.value) + + +def nvmlDeviceGetRowRemapperHistogram(device): + c_vals = c_nvmlRowRemapperHistogramValues() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetRowRemapperHistogram") + ret = fn(device, byref(c_vals)) + _nvmlCheckReturn(ret) + return c_vals + + +def nvmlDeviceGetArchitecture(device): + arch = _nvmlDeviceArchitecture_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetArchitecture") + ret = fn(device, byref(arch)) + _nvmlCheckReturn(ret) + return arch.value + + +def nvmlDeviceGetBusType(device): + c_busType = _nvmlBusType_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetBusType") + ret = fn(device, byref(c_busType)) + _nvmlCheckReturn(ret) + return c_busType.value + + +def nvmlDeviceGetIrqNum(device): + c_irqNum = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetIrqNum") + ret = fn(device, byref(c_irqNum)) + _nvmlCheckReturn(ret) + return c_irqNum.value + + +def nvmlDeviceGetNumGpuCores(device): + c_numCores = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetNumGpuCores") + ret = fn(device, byref(c_numCores)) + _nvmlCheckReturn(ret) + return c_numCores.value + + +def nvmlDeviceGetPowerSource(device): + c_powerSource = _nvmlPowerSource_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerSource") + ret = fn(device, byref(c_powerSource)) + _nvmlCheckReturn(ret) + return c_powerSource.value + + +def nvmlDeviceGetMemoryBusWidth(device): + c_memBusWidth = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemoryBusWidth") + ret = fn(device, byref(c_memBusWidth)) + _nvmlCheckReturn(ret) + return c_memBusWidth.value + + +def nvmlDeviceGetPcieLinkMaxSpeed(device): + c_speed = _nvmlPcieLinkMaxSpeed_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetPcieLinkMaxSpeed") + ret = fn(device, byref(c_speed)) + _nvmlCheckReturn(ret) + return c_speed.value + + +def nvmlDeviceGetAdaptiveClockInfoStatus(device): + c_adaptiveClockInfoStatus = _nvmlAdaptiveClockInfoStatus_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetAdaptiveClockInfoStatus") + ret = fn(device, byref(c_adaptiveClockInfoStatus)) + _nvmlCheckReturn(ret) + return c_adaptiveClockInfoStatus.value + + +def nvmlDeviceGetPcieSpeed(device): + c_speed = c_uint() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetPcieSpeed") + ret = fn(device, byref(c_speed)) + _nvmlCheckReturn(ret) + return c_speed.value + + +def nvmlDeviceGetDynamicPstatesInfo( + device, c_dynamicpstatesinfo=c_nvmlGpuDynamicPstatesInfo_t() +): + isReference = type(c_dynamicpstatesinfo) is not c_nvmlGpuDynamicPstatesInfo_t + dynamicpstatesinfoRef = ( + c_dynamicpstatesinfo if isReference else byref(c_dynamicpstatesinfo) + ) + + fn = _nvmlGetFunctionPointer("nvmlDeviceGetDynamicPstatesInfo") + ret = fn(device, dynamicpstatesinfoRef) + _nvmlCheckReturn(ret) + return NVML_SUCCESS if isReference else c_dynamicpstatesinfo + + +def nvmlDeviceSetFanSpeed_v2(handle, index, speed): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetFanSpeed_v2") + ret = fn(handle, index, speed) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlDeviceGetThermalSettings( + device, sensorindex, c_thermalsettings=c_nvmlGpuThermalSettings_t() +): + isReference = type(c_thermalsettings) is not c_nvmlGpuThermalSettings_t + thermalsettingsRef = c_thermalsettings if isReference else byref(c_thermalsettings) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetThermalSettings") + ret = fn(device, sensorindex, thermalsettingsRef) + _nvmlCheckReturn(ret) + return NVML_SUCCESS if isReference else c_thermalsettings.sensor[:] + + +def nvmlDeviceGetMinMaxClockOfPState( + device, clockType, pstate, minClockMHz=c_uint(), maxClockMHz=c_uint() +): + isReference = (type(minClockMHz) is not c_uint) or (type(maxClockMHz) is not c_uint) + minClockMHzRef = minClockMHz if isReference else byref(minClockMHz) + maxClockMHzRef = maxClockMHz if isReference else byref(maxClockMHz) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetMinMaxClockOfPState") + ret = fn( + device, + _nvmlClockType_t(clockType), + _nvmlClockType_t(pstate), + minClockMHzRef, + maxClockMHzRef, + ) + _nvmlCheckReturn(ret) + return NVML_SUCCESS if isReference else (minClockMHz.value, maxClockMHz.value) + + +class c_nvmlClockOffset_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("type", _nvmlClockType_t), + ("pstate", _nvmlPstates_t), + ("clockOffsetMHz", c_int), + ("minClockOffsetMHz", c_int), + ("maxClockOffsetMHz", c_int), + ] + + +nvmlClockOffset_v1 = 0x1000018 + + +def nvmlDeviceGetClockOffsets(device, info): + fn = _nvmlGetFunctionPointer("nvmlDeviceGetClockOffsets") + ret = fn(device, info) + return NVML_SUCCESS + + +def nvmlDeviceSetClockOffsets(device, info): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetClockOffsets") + ret = fn(device, info) + return NVML_SUCCESS + + +def nvmlDeviceGetSupportedPerformanceStates(device): + pstates = [] + c_count = c_uint(NVML_MAX_GPU_PERF_PSTATES) + c_size = sizeof(c_uint) * c_count.value + + # NOTE: use 'c_uint' to represent the size of the nvmlPstate_t enumeration. + pstates_array = _nvmlPstates_t * c_count.value + c_pstates = pstates_array() + + fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedPerformanceStates") + ret = fn(device, c_pstates, c_size) + _nvmlCheckReturn(ret) + + for value in c_pstates: + if value != NVML_PSTATE_UNKNOWN: + pstates.append(value) + + return pstates + + +def nvmlDeviceGetGpcClkVfOffset(device): + offset = c_int32() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpcClkVfOffset") + ret = fn(device, byref(offset)) + _nvmlCheckReturn(ret) + return offset.value + + +def nvmlDeviceSetGpcClkVfOffset(device, offset): + c_offset = c_int32(offset) + fn = _nvmlGetFunctionPointer("nvmlDeviceSetGpcClkVfOffset") + ret = fn(device, c_offset) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlDeviceGetGpcClkMinMaxVfOffset(device, minOffset=c_int(), maxOffset=c_int()): + isReference = (type(minOffset) is not c_int) or (type(maxOffset) is not c_int) + minOffsetRef = minOffset if isReference else byref(minOffset) + maxOffsetRef = maxOffset if isReference else byref(maxOffset) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpcClkMinMaxVfOffset") + ret = fn(device, minOffsetRef, maxOffsetRef) + _nvmlCheckReturn(ret) + return NVML_SUCCESS if isReference else (minOffset.value, maxOffset.value) + + +def nvmlDeviceGetMemClkVfOffset(device): + offset = c_int32() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemClkVfOffset") + ret = fn(device, byref(offset)) + _nvmlCheckReturn(ret) + return offset.value + + +def nvmlDeviceSetMemClkVfOffset(device, offset): + c_offset = c_int32(offset) + fn = _nvmlGetFunctionPointer("nvmlDeviceSetMemClkVfOffset") + ret = fn(device, c_offset) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlDeviceGetMemClkMinMaxVfOffset(device, minOffset=c_int(), maxOffset=c_int()): + isReference = (type(minOffset) is not c_int) or (type(maxOffset) is not c_int) + minOffsetRef = minOffset if isReference else byref(minOffset) + maxOffsetRef = maxOffset if isReference else byref(maxOffset) + + fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemClkMinMaxVfOffset") + ret = fn(device, minOffsetRef, maxOffsetRef) + _nvmlCheckReturn(ret) + return NVML_SUCCESS if isReference else (minOffset.value, maxOffset.value) + + +def nvmlSystemSetConfComputeGpusReadyState(state): + c_state = c_uint(state) + fn = _nvmlGetFunctionPointer("nvmlSystemSetConfComputeGpusReadyState") + ret = fn(c_state) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlSystemGetConfComputeGpusReadyState(): + c_state = c_uint() + fn = _nvmlGetFunctionPointer("nvmlSystemGetConfComputeGpusReadyState") + ret = fn(byref(c_state)) + _nvmlCheckReturn(ret) + return c_state.value + + +def nvmlSystemGetConfComputeCapabilities(): + c_ccSysCaps = c_nvmlConfComputeSystemCaps_t() + fn = _nvmlGetFunctionPointer("nvmlSystemGetConfComputeCapabilities") + ret = fn(byref(c_ccSysCaps)) + _nvmlCheckReturn(ret) + return c_ccSysCaps + + +def nvmlSystemGetConfComputeState(): + c_state = c_nvmlConfComputeSystemState_t() + fn = _nvmlGetFunctionPointer("nvmlSystemGetConfComputeState") + ret = fn(byref(c_state)) + _nvmlCheckReturn(ret) + return c_state + + +def nvmlSystemGetConfComputeSettings(settings): + fn = _nvmlGetFunctionPointer("nvmlSystemGetConfComputeSettings") + return fn(settings) + + +def nvmlDeviceSetConfComputeUnprotectedMemSize(device, c_ccMemSize): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetConfComputeUnprotectedMemSize") + ret = fn(device, c_ccMemSize) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlDeviceGetConfComputeMemSizeInfo(device): + c_ccMemSize = c_nvmlConfComputeMemSizeInfo_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetConfComputeMemSizeInfo") + ret = fn(device, byref(c_ccMemSize)) + _nvmlCheckReturn(ret) + return c_ccMemSize + + +def nvmlDeviceGetConfComputeProtectedMemoryUsage(device): + c_memory = c_nvmlMemory_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetConfComputeProtectedMemoryUsage") + ret = fn(device, byref(c_memory)) + _nvmlCheckReturn(ret) + return c_memory + + +def nvmlDeviceGetConfComputeGpuCertificate(device): + c_cert = c_nvmlConfComputeGpuCertificate_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetConfComputeGpuCertificate") + ret = fn(device, byref(c_cert)) + _nvmlCheckReturn(ret) + return c_cert + + +def nvmlDeviceGetConfComputeGpuAttestationReport(device, c_nonce): + c_attestReport = c_nvmlConfComputeGpuAttestationReport_t() + c_nonce_arr = (c_uint8 * len(c_nonce))(*(c_nonce)) + setattr(c_attestReport, "nonce", c_nonce_arr) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetConfComputeGpuAttestationReport") + ret = fn(device, byref(c_attestReport)) + _nvmlCheckReturn(ret) + return c_attestReport + + +def nvmlSystemSetConfComputeKeyRotationThresholdInfo(max_atk_adv): + c_keyRotationThrInfo = c_nvmlConfComputeSetKeyRotationThresholdInfo_t(0) + c_keyRotationThrInfo.version = ConfComputeSetKeyRotationThresholdInfo_v1 + c_keyRotationThrInfo.maxAttackerAdvantage = max_atk_adv + fn = _nvmlGetFunctionPointer("nvmlSystemSetConfComputeKeyRotationThresholdInfo") + ret = fn(byref(c_keyRotationThrInfo)) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlSystemGetConfComputeKeyRotationThresholdInfo(): + c_keyRotationThrInfo = c_nvmlConfComputeGetKeyRotationThresholdInfo_t(0) + c_keyRotationThrInfo.version = ConfComputeGetKeyRotationThresholdInfo_v1 + fn = _nvmlGetFunctionPointer("nvmlSystemGetConfComputeKeyRotationThresholdInfo") + ret = fn(byref(c_keyRotationThrInfo)) + _nvmlCheckReturn(ret) + return c_keyRotationThrInfo + + +## GPM ## +######### + +## Enums/defines + +#### GPM Metric Identifiers +NVML_GPM_METRIC_GRAPHICS_UTIL = ( + 1 # Percentage of time any compute/graphics app was active on the GPU. 0.0 - 100.0 +) +NVML_GPM_METRIC_SM_UTIL = 2 # Percentage of SMs that were busy. 0.0 - 100.0 +NVML_GPM_METRIC_SM_OCCUPANCY = ( + 3 # Percentage of warps that were active vs theoretical maximum. 0.0 - 100.0 +) +NVML_GPM_METRIC_INTEGER_UTIL = ( + 4 # Percentage of time the GPU's SMs were doing integer operations. 0.0 - 100.0 +) +NVML_GPM_METRIC_ANY_TENSOR_UTIL = ( + 5 # Percentage of time the GPU's SMs were doing ANY tensor operations. 0.0 - 100.0 +) +NVML_GPM_METRIC_DFMA_TENSOR_UTIL = ( + 6 # Percentage of time the GPU's SMs were doing DFMA tensor operations. 0.0 - 100.0 +) +NVML_GPM_METRIC_HMMA_TENSOR_UTIL = ( + 7 # Percentage of time the GPU's SMs were doing HMMA tensor operations. 0.0 - 100.0 +) +NVML_GPM_METRIC_IMMA_TENSOR_UTIL = ( + 9 # Percentage of time the GPU's SMs were doing IMMA tensor operations. 0.0 - 100.0 +) +NVML_GPM_METRIC_DRAM_BW_UTIL = ( + 10 # Percentage of DRAM bw used vs theoretical maximum. 0.0 - 100.0 +) +NVML_GPM_METRIC_FP64_UTIL = ( + 11 # Percentage of time the GPU's SMs were doing non-tensor FP64 math. 0.0 - 100.0 +) +NVML_GPM_METRIC_FP32_UTIL = ( + 12 # Percentage of time the GPU's SMs were doing non-tensor FP32 math. 0.0 - 100.0 +) +NVML_GPM_METRIC_FP16_UTIL = ( + 13 # Percentage of time the GPU's SMs were doing non-tensor FP16 math. 0.0 - 100.0 +) +NVML_GPM_METRIC_PCIE_TX_PER_SEC = 20 # PCIe traffic from this GPU in MiB/sec +NVML_GPM_METRIC_PCIE_RX_PER_SEC = 21 # PCIe traffic to this GPU in MiB/sec +NVML_GPM_METRIC_NVDEC_0_UTIL = 30 # Percent utilization of NVDEC 0. 0.0 - 100.0 +NVML_GPM_METRIC_NVDEC_1_UTIL = 31 # Percent utilization of NVDEC 1. 0.0 - 100.0 +NVML_GPM_METRIC_NVDEC_2_UTIL = 32 # Percent utilization of NVDEC 2. 0.0 - 100.0 +NVML_GPM_METRIC_NVDEC_3_UTIL = 33 # Percent utilization of NVDEC 3. 0.0 - 100.0 +NVML_GPM_METRIC_NVDEC_4_UTIL = 34 # Percent utilization of NVDEC 4. 0.0 - 100.0 +NVML_GPM_METRIC_NVDEC_5_UTIL = 35 # Percent utilization of NVDEC 5. 0.0 - 100.0 +NVML_GPM_METRIC_NVDEC_6_UTIL = 36 # Percent utilization of NVDEC 6. 0.0 - 100.0 +NVML_GPM_METRIC_NVDEC_7_UTIL = 37 # Percent utilization of NVDEC 7. 0.0 - 100.0 +NVML_GPM_METRIC_NVJPG_0_UTIL = 40 # Percent utilization of NVJPG 0. 0.0 - 100.0 +NVML_GPM_METRIC_NVJPG_1_UTIL = 41 # Percent utilization of NVJPG 1. 0.0 - 100.0 +NVML_GPM_METRIC_NVJPG_2_UTIL = 42 # Percent utilization of NVJPG 2. 0.0 - 100.0 +NVML_GPM_METRIC_NVJPG_3_UTIL = 43 # Percent utilization of NVJPG 3. 0.0 - 100.0 +NVML_GPM_METRIC_NVJPG_4_UTIL = 44 # Percent utilization of NVJPG 4. 0.0 - 100.0 +NVML_GPM_METRIC_NVJPG_5_UTIL = 45 # Percent utilization of NVJPG 5. 0.0 - 100.0 +NVML_GPM_METRIC_NVJPG_6_UTIL = 46 # Percent utilization of NVJPG 6. 0.0 - 100.0 +NVML_GPM_METRIC_NVJPG_7_UTIL = 47 # Percent utilization of NVJPG 7. 0.0 - 100.0 +NVML_GPM_METRIC_NVOFA_0_UTIL = 50 # Percent utilization of NVOFA 0. 0.0 - 100.0 +NVML_GPM_METRIC_NVOFA_1_UTIL = 51 # Percent utilization of NVOFA 1. 0.0 - 100.0 +NVML_GPM_METRIC_NVLINK_TOTAL_RX_PER_SEC = ( + 60 # NvLink read bandwidth for all links in MiB/sec +) +NVML_GPM_METRIC_NVLINK_TOTAL_TX_PER_SEC = ( + 61 # NvLink write bandwidth for all links in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L0_RX_PER_SEC = 62 # NvLink read bandwidth for link 0 in MiB/sec +NVML_GPM_METRIC_NVLINK_L0_TX_PER_SEC = ( + 63 # NvLink write bandwidth for link 0 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L1_RX_PER_SEC = 64 # NvLink read bandwidth for link 1 in MiB/sec +NVML_GPM_METRIC_NVLINK_L1_TX_PER_SEC = ( + 65 # NvLink write bandwidth for link 1 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L2_RX_PER_SEC = 66 # NvLink read bandwidth for link 2 in MiB/sec +NVML_GPM_METRIC_NVLINK_L2_TX_PER_SEC = ( + 67 # NvLink write bandwidth for link 2 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L3_RX_PER_SEC = 68 # NvLink read bandwidth for link 3 in MiB/sec +NVML_GPM_METRIC_NVLINK_L3_TX_PER_SEC = ( + 69 # NvLink write bandwidth for link 3 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L4_RX_PER_SEC = 70 # NvLink read bandwidth for link 4 in MiB/sec +NVML_GPM_METRIC_NVLINK_L4_TX_PER_SEC = ( + 71 # NvLink write bandwidth for link 4 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L5_RX_PER_SEC = 72 # NvLink read bandwidth for link 5 in MiB/sec +NVML_GPM_METRIC_NVLINK_L5_TX_PER_SEC = ( + 73 # NvLink write bandwidth for link 5 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L6_RX_PER_SEC = 74 # NvLink read bandwidth for link 6 in MiB/sec +NVML_GPM_METRIC_NVLINK_L6_TX_PER_SEC = ( + 75 # NvLink write bandwidth for link 6 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L7_RX_PER_SEC = 76 # NvLink read bandwidth for link 7 in MiB/sec +NVML_GPM_METRIC_NVLINK_L7_TX_PER_SEC = ( + 77 # NvLink write bandwidth for link 7 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L8_RX_PER_SEC = 78 # NvLink read bandwidth for link 8 in MiB/sec +NVML_GPM_METRIC_NVLINK_L8_TX_PER_SEC = ( + 79 # NvLink write bandwidth for link 8 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L9_RX_PER_SEC = 80 # NvLink read bandwidth for link 9 in MiB/sec +NVML_GPM_METRIC_NVLINK_L9_TX_PER_SEC = ( + 81 # NvLink write bandwidth for link 9 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L10_RX_PER_SEC = ( + 82 # NvLink read bandwidth for link 10 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L10_TX_PER_SEC = ( + 83 # NvLink write bandwidth for link 10 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L11_RX_PER_SEC = ( + 84 # NvLink read bandwidth for link 11 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L11_TX_PER_SEC = ( + 85 # NvLink write bandwidth for link 11 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L12_RX_PER_SEC = ( + 86 # NvLink read bandwidth for link 12 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L12_TX_PER_SEC = ( + 87 # NvLink write bandwidth for link 12 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L13_RX_PER_SEC = ( + 88 # NvLink read bandwidth for link 13 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L13_TX_PER_SEC = ( + 89 # NvLink write bandwidth for link 13 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L14_RX_PER_SEC = ( + 90 # NvLink read bandwidth for link 14 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L14_TX_PER_SEC = ( + 91 # NvLink write bandwidth for link 14 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L15_RX_PER_SEC = ( + 92 # NvLink read bandwidth for link 15 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L15_TX_PER_SEC = ( + 93 # NvLink write bandwidth for link 15 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L16_RX_PER_SEC = ( + 94 # NvLink read bandwidth for link 16 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L16_TX_PER_SEC = ( + 95 # NvLink write bandwidth for link 16 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L17_RX_PER_SEC = ( + 96 # NvLink read bandwidth for link 17 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L17_TX_PER_SEC = ( + 97 # NvLink write bandwidth for link 17 in MiB/sec +) +NVML_GPM_METRIC_MAX = 98 + +## Structs + + +class c_nvmlUnitInfo_t(_PrintableStructure): + _fields_ = [ + ("name", c_char * 96), + ("id", c_char * 96), + ("serial", c_char * 96), + ("firmwareVersion", c_char * 96), + ] + + +class struct_c_nvmlGpmSample_t(Structure): + pass # opaque handle + + +c_nvmlGpmSample_t = POINTER(struct_c_nvmlGpmSample_t) + + +class c_metricInfo_t(Structure): + _fields_ = [ + ("shortName", c_char_p), + ("longName", c_char_p), + ("unit", c_char_p), + ] + + +class c_nvmlGpmMetric_t(_PrintableStructure): + _fields_ = [ + ("metricId", c_uint), + ("nvmlReturn", _nvmlReturn_t), + ("value", c_double), + ("metricInfo", c_metricInfo_t), + ] + + +class c_nvmlGpmMetricsGet_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("numMetrics", c_uint), + ("sample1", c_nvmlGpmSample_t), + ("sample2", c_nvmlGpmSample_t), + ("metrics", c_nvmlGpmMetric_t * NVML_GPM_METRIC_MAX), + ] + + +NVML_GPM_METRICS_GET_VERSION = 1 + + +class c_nvmlGpmSupport_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("isSupportedDevice", c_uint), + ] + + +NVML_GPM_SUPPORT_VERSION = 1 + +## Functions + + +def nvmlGpmMetricsGet(metricsGet): + fn = _nvmlGetFunctionPointer("nvmlGpmMetricsGet") + ret = fn(byref(metricsGet)) + _nvmlCheckReturn(ret) + return metricsGet + + +def nvmlGpmSampleFree(gpmSample): + fn = _nvmlGetFunctionPointer("nvmlGpmSampleFree") + ret = fn(gpmSample) + _nvmlCheckReturn(ret) + return + + +def nvmlGpmSampleAlloc(): + gpmSample = c_nvmlGpmSample_t() + fn = _nvmlGetFunctionPointer("nvmlGpmSampleAlloc") + ret = fn(byref(gpmSample)) + _nvmlCheckReturn(ret) + return gpmSample + + +def nvmlGpmSampleGet(device, gpmSample): + fn = _nvmlGetFunctionPointer("nvmlGpmSampleGet") + ret = fn(device, gpmSample) + _nvmlCheckReturn(ret) + return gpmSample + + +def nvmlGpmMigSampleGet(device, gpuInstanceId, gpmSample): + fn = _nvmlGetFunctionPointer("nvmlGpmMigSampleGet") + ret = fn(device, gpuInstanceId, gpmSample) + _nvmlCheckReturn(ret) + return gpmSample + + +def nvmlGpmQueryDeviceSupport(device): + gpmSupport = c_nvmlGpmSupport_t() + gpmSupport.version = NVML_GPM_SUPPORT_VERSION + fn = _nvmlGetFunctionPointer("nvmlGpmQueryDeviceSupport") + ret = fn(device, byref(gpmSupport)) + _nvmlCheckReturn(ret) + return gpmSupport + + +def nvmlGpmSetStreamingEnabled(device, state): + c_state = c_uint(state) + fn = _nvmlGetFunctionPointer("nvmlGpmSetStreamingEnabled") + ret = fn(device, c_state) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlGpmQueryIfStreamingEnabled(device): + c_state = c_uint() + fn = _nvmlGetFunctionPointer("nvmlGpmQueryIfStreamingEnabled") + ret = fn(device, byref(c_state)) + _nvmlCheckReturn(ret) + return c_state.value + + +# Low Power Structure and Function + +NVML_NVLINK_POWER_STATE_HIGH_SPEED = 0x0 +NVML_NVLINK_POWER_STATE_LOW = 0x1 + +NVML_NVLINK_LOW_POWER_THRESHOLD_MIN = 0x1 +NVML_NVLINK_LOW_POWER_THRESHOLD_MAX = 0x1FFF +NVML_NVLINK_LOW_POWER_THRESHOLD_RESET = 0xFFFFFFFF +NVML_NVLINK_LOW_POWER_THRESHOLD_DEFAULT = NVML_NVLINK_LOW_POWER_THRESHOLD_RESET + + +class c_nvmlNvLinkPowerThres_t(Structure): + _fields_ = [ + ("lowPwrThreshold", c_uint), + ] + + +def nvmlDeviceSetNvLinkDeviceLowPowerThreshold(device, l1threshold): + c_info = c_nvmlNvLinkPowerThres_t() + c_info.lowPwrThreshold = l1threshold + fn = _nvmlGetFunctionPointer("nvmlDeviceSetNvLinkDeviceLowPowerThreshold") + ret = fn(device, byref(c_info)) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +NVML_GPU_FABRIC_UUID_LEN = 16 + +_nvmlGpuFabricState_t = c_uint +NVML_GPU_FABRIC_STATE_NOT_SUPPORTED = 0 +NVML_GPU_FABRIC_STATE_NOT_STARTED = 1 +NVML_GPU_FABRIC_STATE_IN_PROGRESS = 2 +NVML_GPU_FABRIC_STATE_COMPLETED = 3 + + +class c_nvmlGpuFabricInfo_t(_PrintableStructure): + _fields_ = [ + ("clusterUuid", c_char * NVML_DEVICE_UUID_BUFFER_SIZE), + ("status", _nvmlReturn_t), + ("cliqueId", c_uint32), + ("state", _nvmlGpuFabricState_t), + ] + + +NVML_GPU_FABRIC_HEALTH_MASK_DEGRADED_BW_NOT_SUPPORTED = 0 +NVML_GPU_FABRIC_HEALTH_MASK_DEGRADED_BW_TRUE = 1 +NVML_GPU_FABRIC_HEALTH_MASK_DEGRADED_BW_FALSE = 2 +NVML_GPU_FABRIC_HEALTH_MASK_SHIFT_DEGRADED_BW = 0 +NVML_GPU_FABRIC_HEALTH_MASK_WIDTH_DEGRADED_BW = 0x11 + +NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_RECOVERY_NOT_SUPPORTED = 0 +NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_RECOVERY_TRUE = 1 +NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_RECOVERY_FALSE = 2 +NVML_GPU_FABRIC_HEALTH_MASK_SHIFT_ROUTE_RECOVERY = 2 +NVML_GPU_FABRIC_HEALTH_MASK_WIDTH_ROUTE_RECOVERY = 0x11 + +NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_UNHEALTHY_NOT_SUPPORTED = 0 +NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_UNHEALTHY_TRUE = 1 +NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_UNHEALTHY_FALSE = 2 +NVML_GPU_FABRIC_HEALTH_MASK_SHIFT_ROUTE_UNHEALTHY = 4 +NVML_GPU_FABRIC_HEALTH_MASK_WIDTH_ROUTE_UNHEALTHY = 0x11 + +NVML_GPU_FABRIC_HEALTH_MASK_ACCESS_TIMEOUT_RECOVERY_NOT_SUPPORTED = 0 +NVML_GPU_FABRIC_HEALTH_MASK_ACCESS_TIMEOUT_RECOVERY_TRUE = 1 +NVML_GPU_FABRIC_HEALTH_MASK_ACCESS_TIMEOUT_RECOVERY_FALSE = 2 +NVML_GPU_FABRIC_HEALTH_MASK_SHIFT_ACCESS_TIMEOUT_RECOVERY = 6 +NVML_GPU_FABRIC_HEALTH_MASK_WIDTH_ACCESS_TIMEOUT_RECOVERY = 0x11 + +nvmlGpuFabricInfo_v2 = 0x02000024 + + +class c_nvmlGpuFabricInfoV_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("clusterUuid", c_char * NVML_GPU_FABRIC_UUID_LEN), + ("status", _nvmlReturn_t), + ("cliqueId", c_uint32), + ("state", _nvmlGpuFabricState_t), + ("healthMask", c_uint32), + ] + + def __init__(self): + super(c_nvmlGpuFabricInfoV_t, self).__init__(version=nvmlGpuFabricInfo_v2) + + +def nvmlDeviceGetGpuFabricInfo(device, gpuFabricInfo): + fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuFabricInfo") + ret = fn(device, gpuFabricInfo) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlDeviceGetGpuFabricInfoV(device, gpuFabricInfo): + fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuFabricInfoV") + ret = fn(device, gpuFabricInfo) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +###################### +## Enums/defines +#### NVML GPU NVLINK BW MODE +NVML_GPU_NVLINK_BW_MODE_FULL = 0x0 +NVML_GPU_NVLINK_BW_MODE_OFF = 0x1 +NVML_GPU_NVLINK_BW_MODE_MIN = 0x2 +NVML_GPU_NVLINK_BW_MODE_HALF = 0x3 +NVML_GPU_NVLINK_BW_MODE_3QUARTER = 0x4 +NVML_GPU_NVLINK_BW_MODE_COUNT = 0x5 + + +def nvmlSystemSetNvlinkBwMode(mode): + fn = _nvmlGetFunctionPointer("nvmlSystemSetNvlinkBwMode") + ret = fn(mode) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlSystemGetNvlinkBwMode(): + mode = c_uint() + fn = _nvmlGetFunctionPointer("nvmlSystemGetNvlinkBwMode") + ret = fn(byref(mode)) + _nvmlCheckReturn(ret) + return mode.value + + +_nvmlPowerScopeType_t = c_uint +NVML_POWER_SCOPE_GPU = 0 +NVML_POWER_SCOPE_MODULE = 1 +NVML_POWER_SCOPE_MEMORY = 2 + + +class c_nvmlPowerValue_v2_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("powerScope", _nvmlPowerScopeType_t), + ("powerValueMw", c_uint), + ] + _fmt_ = {"": "%d B"} + + +nvmlPowerValue_v2 = 0x0200000C + + +def nvmlDeviceSetPowerManagementLimit_v2( + device, powerScope, powerLimit, version=nvmlPowerValue_v2 +): + c_powerScope = _nvmlPowerScopeType_t(powerScope) + c_powerValue = c_nvmlPowerValue_v2_t() + c_powerValue.version = c_uint(version) + c_powerValue.powerScope = c_powerScope + c_powerValue.powerValueMw = c_uint(powerLimit) + fn = _nvmlGetFunctionPointer("nvmlDeviceSetPowerManagementLimit_v2") + ret = fn(device, byref(c_powerValue)) + return NVML_SUCCESS + + +class c_nvmlEccSramErrorStatus_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("aggregateUncParity", c_ulonglong), + ("aggregateUncSecDed", c_ulonglong), + ("aggregateCor", c_ulonglong), + ("volatileUncParity", c_ulonglong), + ("volatileUncSecDed", c_ulonglong), + ("volatileCor", c_ulonglong), + ("aggregateUncBucketL2", c_ulonglong), + ("aggregateUncBucketSm", c_ulonglong), + ("aggregateUncBucketPcie", c_ulonglong), + ("aggregateUncBucketMcu", c_ulonglong), + ("aggregateUncBucketOther", c_ulonglong), + ("bThresholdExceeded", c_uint), + ] + + def __init__(self): + super(c_nvmlEccSramErrorStatus_v1_t, self).__init__( + version=nvmlEccSramErrorStatus_v1 + ) + + +nvmlEccSramErrorStatus_v1 = 0x1000068 + + +def nvmlDeviceGetSramEccErrorStatus(device, status): + fn = _nvmlGetFunctionPointer("nvmlDeviceGetSramEccErrorStatus") + ret = fn(device, status) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +NVML_DEV_CAP_EGM = 1 << 0 +nvmlDeviceCapabilities_v1 = 0x1000008 + + +class c_nvmlDeviceCapabilities_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("capMask", c_uint), + ] + + def __init__(self): + super(c_nvmlDeviceCapabilities_v1_t, self).__init__( + version=nvmlDeviceCapabilities_v1 + ) + + +def nvmlDeviceGetCapabilities(device, caps): + fn = _nvmlGetFunctionPointer("nvmlDeviceGetCapabilities") + return fn(device, caps) + + +class c_nvmlPlatformInfo_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("ibGuid", c_char * 16), + ("rackGuid", c_char * 16), + ("chassisPhysicalSlotNumber", c_char), + ("computeSlotIndex", c_char), + ("nodeIndex", c_char), + ("peerType", c_char), + ("moduleId", c_char), + ] + + def __init__(self): + super(c_nvmlPlatformInfo_v1_t, self).__init__(version=nvmlPlatformInfo_v1) + + +nvmlPlatformInfo_v1 = 0x100002C + + +def nvmlDeviceGetPlatformInfo(device, platformInfo): + fn = _nvmlGetFunctionPointer("nvmlDeviceGetPlatformInfo") + ret = fn(device, platformInfo) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +class c_nvmlMask255_t(_PrintableStructure): + _fields_ = [ + ("mask", c_uint * 8), + ] + + +NVML_WORKLOAD_POWER_MAX_PROFILES = 255 +NVML_POWER_PROFILE_MAX_P = 0 +NVML_POWER_PROFILE_MAX_Q = 1 +NVML_POWER_PROFILE_COMPUTE = 2 +NVML_POWER_PROFILE_MEMORY_BOUND = 3 +NVML_POWER_PROFILE_NETWORK = 4 +NVML_POWER_PROFILE_BALANCED = 5 +NVML_POWER_PROFILE_LLM_INFERENCE = 6 +NVML_POWER_PROFILE_LLM_TRAINING = 7 +NVML_POWER_PROFILE_RBM = 8 +NVML_POWER_PROFILE_DCPCIE = 9 +NVML_POWER_PROFILE_HMMA_SPARSE = 10 +NVML_POWER_PROFILE_HMMA_DENSE = 11 +NVML_POWER_PROFILE_SYNC_BALANCED = 12 +NVML_POWER_PROFILE_HPC = 13 +NVML_POWER_PROFILE_MIG = 14 +NVML_POWER_PROFILE_MAX = 15 + +nvmlWorkloadPowerProfileInfo_v1 = 0x100002C + + +class c_nvmlWorkloadPowerProfileInfo_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("profileId", c_uint), + ("priority", c_uint), + ("conflictingmask", c_nvmlMask255_t), + ] + + def __init__(self): + super(c_nvmlWorkloadPowerProfileInfo_v1_t, self).__init__( + version=nvmlWorkloadPowerProfileInfo_v1 + ) + + +nvmlWorkloadPowerProfileProfilesInfo_v1 = 0x1002BF8 + + +class c_nvmlWorkloadPowerProfileProfilesInfo_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("perfProfilesMask", c_nvmlMask255_t), + ( + "perfProfile", + c_nvmlWorkloadPowerProfileInfo_v1_t * NVML_WORKLOAD_POWER_MAX_PROFILES, + ), + ] + + def __init__(self): + super(c_nvmlWorkloadPowerProfileProfilesInfo_v1_t, self).__init__( + version=nvmlWorkloadPowerProfileProfilesInfo_v1 + ) + + +nvmlWorkloadPowerProfileCurrentProfiles_v1 = 0x1000064 + + +class c_nvmlWorkloadPowerProfileCurrentProfiles_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("perfProfilesMask", c_nvmlMask255_t), + ("requestedProfilesMask", c_nvmlMask255_t), + ("enforcedProfilesMask", c_nvmlMask255_t), + ] + + def __init__(self): + super(c_nvmlWorkloadPowerProfileCurrentProfiles_v1_t, self).__init__( + version=nvmlWorkloadPowerProfileCurrentProfiles_v1 + ) + + +nvmlWorkloadPowerProfileRequestedProfiles_v1 = 0x1000024 + + +class c_nvmlWorkloadPowerProfileRequestedProfiles_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("requestedProfilesMask", c_nvmlMask255_t), + ] + + def __init__(self): + super(c_nvmlWorkloadPowerProfileRequestedProfiles_v1_t, self).__init__( + version=nvmlWorkloadPowerProfileRequestedProfiles_v1 + ) + + +def nvmlDeviceWorkloadPowerProfileGetProfilesInfo(device, profilesInfo): + fn = _nvmlGetFunctionPointer("nvmlDeviceWorkloadPowerProfileGetProfilesInfo") + ret = fn(device, profilesInfo) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlDeviceWorkloadPowerProfileGetCurrentProfiles(device, currentProfiles): + fn = _nvmlGetFunctionPointer("nvmlDeviceWorkloadPowerProfileGetCurrentProfiles") + ret = fn(device, currentProfiles) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlDeviceWorkloadPowerProfileSetRequestedProfiles(device, requestedProfiles): + fn = _nvmlGetFunctionPointer("nvmlDeviceWorkloadPowerProfileSetRequestedProfiles") + ret = fn(device, requestedProfiles) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlDeviceWorkloadPowerProfileClearRequestedProfiles(device, requestedProfiles): + fn = _nvmlGetFunctionPointer("nvmlDeviceWorkloadPowerProfileClearRequestedProfiles") + ret = fn(device, requestedProfiles) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlDeviceGetNvlinkSupportedBwModes(device, supportedBwModes): + fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvlinkSupportedBwModes") + ret = fn(device, supportedBwModes) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlDeviceGetNvlinkBwMode(device, getBwMode): + fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvlinkBwMode") + ret = fn(device, getBwMode) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +def nvmlDeviceSetNvlinkBwMode(device, setBwMode): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetNvlinkBwMode") + ret = fn(device, setBwMode) + _nvmlCheckReturn(ret) + return NVML_SUCCESS + + +nvmlDramEncryptionInfo_v1 = 0x01000008 + + +class c_nvmlDramEncryptionInfo_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("encryptionState", _nvmlEnableState_t), + ] + + def __init__(self): + super(c_nvmlDramEncryptionInfo_t, self).__init__( + version=nvmlDramEncryptionInfo_v1 + ) + + +def nvmlDeviceGetDramEncryptionMode(handle): + c_currState = c_nvmlDramEncryptionInfo_t() + c_pendingState = c_nvmlDramEncryptionInfo_t() + fn = _nvmlGetFunctionPointer("nvmlDeviceGetDramEncryptionMode") + ret = fn(handle, byref(c_currState), byref(c_pendingState)) + _nvmlCheckReturn(ret) + return [c_currState.encryptionState, c_pendingState.encryptionState] + + +# added to API +def nvmlDeviceGetCurrentDramEncryptionMode(handle): + return nvmlDeviceGetDramEncryptionMode(handle)[0] + + +# added to API +def nvmlDeviceGetPendingDramEncryptionMode(handle): + return nvmlDeviceGetDramEncryptionMode(handle)[1] + + +def nvmlDeviceSetDramEncryptionMode(handle, mode): + fn = _nvmlGetFunctionPointer("nvmlDeviceSetDramEncryptionMode") + c_dramEncryptionMode = c_nvmlDramEncryptionInfo_t() + c_dramEncryptionMode.encryptionState = mode + ret = fn(handle, byref(c_dramEncryptionMode)) + _nvmlCheckReturn(ret) + return None + + +# Power Smoothing defines +NVML_POWER_SMOOTHING_MAX_NUM_PROFILES = 5 +NVML_POWER_SMOOTHING_ADMIN_OVERRIDE_NOT_SET = 0xFFFFFFFF +NVML_POWER_SMOOTHING_PROFILE_PARAM_PERCENT_TMP_FLOOR = 0 +NVML_POWER_SMOOTHING_PROFILE_PARAM_RAMP_UP_RATE = 1 +NVML_POWER_SMOOTHING_PROFILE_PARAM_RAMP_DOWN_RATE = 2 +NVML_POWER_SMOOTHING_PROFILE_PARAM_RAMP_DOWN_HYSTERESIS = 3 + +nvmlPowerSmoothingState_v1 = 0x1000008 + + +class c_nvmlPowerSmoothingState_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("state", c_uint), + ] + + def __init__(self): + super(c_nvmlPowerSmoothingState_v1_t, self).__init__( + version=nvmlPowerSmoothingState_v1 + ) + + +nvmlPowerSmoothingProfile_v1 = 0x1000018 + + +class c_nvmlPowerSmoothingProfile_v1_t(_PrintableStructure): + _fields_ = [ + ("version", c_uint), + ("profileId", c_uint), + ("paramId", c_uint), + ("value", c_double), + ] + + def __init__(self): + super(c_nvmlPowerSmoothingProfile_v1_t, self).__init__( + version=nvmlPowerSmoothingProfile_v1 + ) + + +def nvmlDevicePowerSmoothingActivatePresetProfile(device, profile): + fn = _nvmlGetFunctionPointer("nvmlDevicePowerSmoothingActivatePresetProfile") + ret = fn(device, profile) + _nvmlCheckReturn(ret) + + +def nvmlDevicePowerSmoothingUpdatePresetProfileParam(device, profile): + fn = _nvmlGetFunctionPointer("nvmlDevicePowerSmoothingUpdatePresetProfileParam") + ret = fn(device, profile) + _nvmlCheckReturn(ret) + + +def nvmlDevicePowerSmoothingSetState(device, state): + fn = _nvmlGetFunctionPointer("nvmlDevicePowerSmoothingSetState") + ret = fn(device, state) + _nvmlCheckReturn(ret) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/utils.py new file mode 100644 index 00000000..6a8f5958 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/multimodal_gen/utils.py @@ -0,0 +1,813 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +# Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/utils.py + +import argparse +import ctypes +import importlib +import importlib.util +import inspect +import math +import os +import signal +import socket +import sys +import threading +import traceback +from collections.abc import Callable +from dataclasses import dataclass, fields, is_dataclass +from functools import lru_cache, partial, wraps +from typing import Any, TypeVar, cast + +import cloudpickle +import imageio +import numpy as np +import torch +import torchvision +import yaml +from einops import rearrange +from remote_pdb import RemotePdb +from torch.distributed.fsdp import MixedPrecisionPolicy + +import sglang.multimodal_gen.envs as envs +from sglang.multimodal_gen.runtime.utils.logging_utils import ( + SortedHelpFormatter, + init_logger, +) + +logger = init_logger(__name__) + +T = TypeVar("T") + +# TODO(will): used to convert server_args.precision to torch.dtype. Find a +# cleaner way to do this. +PRECISION_TO_TYPE = { + "fp32": torch.float32, + "fp16": torch.float16, + "bf16": torch.bfloat16, +} + +STR_BACKEND_ENV_VAR: str = "SGLANG_DIFFUSION_ATTENTION_BACKEND" +STR_ATTN_CONFIG_ENV_VAR: str = "SGLANG_DIFFUSION_ATTENTION_CONFIG" + + +def find_nccl_library() -> str: + """ + We either use the library file specified by the `VLLM_NCCL_SO_PATH` + environment variable, or we find the library file brought by PyTorch. + After importing `torch`, `libnccl.so.2` or `librccl.so.1` can be + found by `ctypes` automatically. + """ + so_file = envs.SGLANG_DIFFUSION_NCCL_SO_PATH + + # manually load the nccl library + if so_file: + logger.info( + "Found nccl from environment variable SGLANG_DIFFUSION_NCCL_SO_PATH=%s", + so_file, + ) + else: + if torch.version.cuda is not None: + so_file = "libnccl.so.2" + elif torch.version.hip is not None: + so_file = "librccl.so.1" + else: + raise ValueError("NCCL only supports CUDA and ROCm backends.") + logger.info("Found nccl from library %s", so_file) + return str(so_file) + + +prev_set_stream = torch.cuda.set_stream + +_current_stream = None + + +def _patched_set_stream(stream: torch.cuda.Stream | None) -> None: + global _current_stream + _current_stream = stream + if stream is not None: + prev_set_stream(stream) + + +torch.cuda.set_stream = _patched_set_stream + + +def current_stream() -> torch.cuda.Stream | None: + """ + replace `torch.cuda.current_stream()` with `sglang.multimodal_gen.utils.current_stream()`. + it turns out that `torch.cuda.current_stream()` is quite expensive, + as it will construct a new stream object at each call. + here we patch `torch.cuda.set_stream` to keep track of the current stream + directly, so that we can avoid calling `torch.cuda.current_stream()`. + + the underlying hypothesis is that we do not call `torch._C._cuda_setStream` + from C/C++ code. + """ + from sglang.multimodal_gen.runtime.platforms import current_platform + + # For non-CUDA platforms, return None + if not current_platform.is_cuda_alike(): + return None + + global _current_stream + if _current_stream is None: + # when this function is called before any stream is set, + # we return the default stream. + # On ROCm using the default 0 stream in combination with RCCL + # is hurting performance. Therefore creating a dedicated stream + # per process + _current_stream = ( + torch.cuda.Stream() + if current_platform.is_rocm() + else torch.cuda.current_stream() + ) + return _current_stream + + +class StoreBoolean(argparse.Action): + + def __init__(self, option_strings, dest, default=False, required=False, help=None): + super().__init__( + option_strings=option_strings, + dest=dest, + nargs="?", + const=True, + default=default, + required=required, + help=help, + ) + + def __call__(self, parser, namespace, values, option_string=None): + if values is None: + setattr(namespace, self.dest, True) + elif isinstance(values, str): + if values.lower() == "true": + setattr(namespace, self.dest, True) + elif values.lower() == "false": + setattr(namespace, self.dest, False) + else: + raise ValueError( + f"Invalid boolean value: {values}. " "Expected 'true' or 'false'." + ) + else: + setattr(namespace, self.dest, bool(values)) + + +class FlexibleArgumentParser(argparse.ArgumentParser): + """ArgumentParser that allows both underscore and dash in names.""" + + def __init__(self, *args, **kwargs) -> None: + # Set the default 'formatter_class' to SortedHelpFormatter + if "formatter_class" not in kwargs: + kwargs["formatter_class"] = SortedHelpFormatter + super().__init__(*args, **kwargs) + + def parse_args( # type: ignore[override] + self, args=None, namespace=None + ) -> argparse.Namespace: + if args is None: + args = sys.argv[1:] + + if any(arg.startswith("--config") for arg in args): + args = self._pull_args_from_config(args) + + # Convert underscores to dashes and vice versa in argument names + processed_args = [] + for arg in args: + if arg.startswith("--"): + if "=" in arg: + key, value = arg.split("=", 1) + key = "--" + key[len("--") :].replace("_", "-") + processed_args.append(f"{key}={value}") + else: + processed_args.append("--" + arg[len("--") :].replace("_", "-")) + elif arg.startswith("-O") and arg != "-O" and len(arg) == 2: + # allow -O flag to be used without space, e.g. -O3 + processed_args.append("-O") + processed_args.append(arg[2:]) + else: + processed_args.append(arg) + + namespace = super().parse_args(processed_args, namespace) + + # Track which arguments were explicitly provided + namespace._provided = set() + + i = 0 + while i < len(args): + arg = args[i] + if arg.startswith("--"): + # Handle --key=value format + if "=" in arg: + key = arg.split("=")[0][2:].replace("-", "_") + namespace._provided.add(key) + i += 1 + # Handle --key value format + else: + key = arg[2:].replace("-", "_") + namespace._provided.add(key) + # Skip the value if there is one + if i + 1 < len(args) and not args[i + 1].startswith("-"): + i += 2 + else: + i += 1 + else: + i += 1 + + return namespace # type: ignore[no-any-return] + + def _pull_args_from_config(self, args: list[str]) -> list[str]: + """Method to pull arguments specified in the config file + into the command-line args variable. + + The arguments in config file will be inserted between + the argument list. + + example: + ```yaml + port: 12323 + tensor-parallel-size: 4 + ``` + ```python + $: vllm {serve,chat,complete} "facebook/opt-12B" \ + --config config.yaml -tp 2 + $: args = [ + "serve,chat,complete", + "facebook/opt-12B", + '--config', 'config.yaml', + '-tp', '2' + ] + $: args = [ + "serve,chat,complete", + "facebook/opt-12B", + '--port', '12323', + '--tp-size', '4', + '-tp', '2' + ] + ``` + + Please note how the config args are inserted after the sub command. + this way the order of priorities is maintained when these are args + parsed by super(). + """ + index = -1 + config_arg = None + for i, arg in enumerate(args): + if arg.startswith("--config"): + if index != -1: + raise ValueError("More than one config file specified!") + index = i + config_arg = arg + + if config_arg is None: + return args + args_before_config = args[:index] + if "=" in config_arg: + file_path = config_arg.split("=", 1)[1] + args_after_config = args[index + 1 :] + else: + if index == len(args) - 1: + raise ValueError( + "No config file specified! " + "Please check your command-line arguments." + ) + file_path = args[index + 1] + args_after_config = args[index + 2 :] + + config_args = self._load_config_file(file_path) + + # 0th index is for {serve,chat,complete} + # followed by model_tag (only for serve) + # followed by config args + # followed by rest of cli args. + # maintaining this order will enforce the precedence + # of cli > config > defaults + if args[0] == "serve": + if index == 1: + raise ValueError( + "No model_tag specified! Please check your command-line" + " arguments." + ) + command = args_before_config[0] + model_tag = args_before_config[1] + other_args_before = args_before_config[2:] + args = ( + [command, model_tag] + + config_args + + other_args_before + + args_after_config + ) + else: + command = args_before_config[0] + other_args_before = args_before_config[1:] + args = [command] + config_args + other_args_before + args_after_config + + return args + + def _load_config_file(self, file_path: str) -> list[str]: + """Loads a yaml file and returns the key value pairs as a + flattened list with argparse like pattern + ```yaml + port: 12323 + tensor-parallel-size: 4 + vae_config: + load_encoder: false + load_decoder: true + ``` + returns: + processed_args: list[str] = [ + '--port': '12323', + '--tp-size': '4', + '--vae-config.load-encoder': 'false', + '--vae-config.load-decoder': 'true' + ] + """ + + extension: str = file_path.split(".")[-1] + if extension not in ("yaml", "yml", "json"): + raise ValueError( + "Config file must be of a yaml/yml/json type.\ + %s supplied", + extension, + ) + + processed_args: list[str] = [] + + config: dict[str, Any] = {} + try: + with open(file_path) as config_file: + config = yaml.safe_load(config_file) + except Exception as ex: + logger.error( + "Unable to read the config file at %s. \ + Make sure path is correct", + file_path, + ) + raise ex + + store_boolean_arguments = [ + action.dest for action in self._actions if isinstance(action, StoreBoolean) + ] + + def process_dict(prefix: str, d: dict[str, Any]): + for key, value in d.items(): + full_key = f"{prefix}.{key}" if prefix else key + + if isinstance(value, bool) and full_key not in store_boolean_arguments: + if value: + processed_args.append("--" + full_key) + else: + processed_args.append("--" + full_key) + processed_args.append("false") + elif isinstance(value, list): + processed_args.append("--" + full_key) + for item in value: + processed_args.append(str(item)) + elif isinstance(value, dict): + process_dict(full_key, value) + else: + processed_args.append("--" + full_key) + processed_args.append(str(value)) + + process_dict("", config) + + return processed_args + + +def warn_for_unimplemented_methods(cls: type[T]) -> type[T]: + """ + A replacement for `abc.ABC`. + When we use `abc.ABC`, subclasses will fail to instantiate + if they do not implement all abstract methods. + Here, we only require `raise NotImplementedError` in the + base class, and log a warning if the method is not implemented + in the subclass. + """ + + original_init = cls.__init__ + + def find_unimplemented_methods(self: object): + unimplemented_methods = [] + for attr_name in dir(self): + # bypass inner method + if attr_name.startswith("_"): + continue + + try: + attr = getattr(self, attr_name) + # get the func of callable method + if callable(attr): + attr_func = attr.__func__ + except AttributeError: + continue + src = inspect.getsource(attr_func) + if "NotImplementedError" in src: + unimplemented_methods.append(attr_name) + if unimplemented_methods: + method_names = ",".join(unimplemented_methods) + msg = f"Methods {method_names} not implemented in {self}" + logger.warning(msg) + + @wraps(original_init) + def wrapped_init(self, *args, **kwargs) -> None: + original_init(self, *args, **kwargs) + find_unimplemented_methods(self) + + type.__setattr__(cls, "__init__", wrapped_init) + return cls + + +def align_to(value: int, alignment: int) -> int: + """align height, width according to alignment + + Args: + value (int): height or width + alignment (int): target alignment factor + + Returns: + int: the aligned value + """ + return int(math.ceil(value / alignment) * alignment) + + +def resolve_obj_by_qualname(qualname: str) -> Any: + """ + Resolve an object by its fully qualified name. + """ + module_name, obj_name = qualname.rsplit(".", 1) + module = importlib.import_module(module_name) + return getattr(module, obj_name) + + +# From vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/utils.py +def import_pynvml(): + """ + Historical comments: + + libnvml.so is the library behind nvidia-smi, and + pynvml is a Python wrapper around it. We use it to get GPU + status without initializing CUDA context in the current process. + Historically, there are two packages that provide pynvml: + - `nvidia-ml-py` (https://pypi.org/project/nvidia-ml-py/): The official + wrapper. It is a dependency of sglang-diffusion, and is installed when users + install sglang-diffusion. It provides a Python module named `pynvml`. + - `pynvml` (https://pypi.org/project/pynvml/): An unofficial wrapper. + Prior to version 12.0, it also provides a Python module `pynvml`, + and therefore conflicts with the official one which is a standalone Python file. + This causes errors when both of them are installed. + Starting from version 12.0, it migrates to a new module + named `pynvml_utils` to avoid the conflict. + It is so confusing that many packages in the community use the + unofficial one by mistake, and we have to handle this case. + For example, `nvcr.io/nvidia/pytorch:24.12-py3` uses the unofficial + one, and it will cause errors, see the issue + https://github.com/vllm-project/vllm/issues/12847 for example. + After all the troubles, we decide to copy the official `pynvml` + module to our codebase, and use it directly. + """ + import sglang.multimodal_gen.third_party.pynvml as pynvml + + return pynvml + + +def update_environment_variables(envs: dict[str, str]): + for k, v in envs.items(): + if k in os.environ and os.environ[k] != v: + logger.warning( + "Overwriting environment variable %s " "from '%s' to '%s'", + k, + os.environ[k], + v, + ) + os.environ[k] = v + + +def run_method( + obj: Any, method: str | bytes | Callable, args: tuple[Any], kwargs: dict[str, Any] +) -> Any: + """ + Run a method of an object with the given arguments and keyword arguments. + If the method is string, it will be converted to a method using getattr. + If the method is serialized bytes and will be deserialized using + cloudpickle. + If the method is a callable, it will be called directly. + """ + if isinstance(method, bytes): + func = partial(cloudpickle.loads(method), obj) + elif isinstance(method, str): + try: + func = getattr(obj, method) + except AttributeError: + raise NotImplementedError( + f"Method {method!r} is not" " implemented." + ) from None + else: + func = partial(method, obj) # type: ignore + return func(*args, **kwargs) + + +def shallow_asdict(obj) -> dict[str, Any]: + if not is_dataclass(obj): + raise TypeError("Expected dataclass instance") + return {f.name: getattr(obj, f.name) for f in fields(obj)} + + +# TODO: validate that this is fine +def kill_itself_when_parent_died() -> None: + # if sys.platform == "linux": + # sigkill this process when parent worker manager dies + PR_SET_PDEATHSIG = 1 + import platform + + if platform.system() == "Linux": + libc = ctypes.CDLL("libc.so.6") + libc.prctl(PR_SET_PDEATHSIG, signal.SIGKILL) + # elif platform.system() == "Darwin": + # libc = ctypes.CDLL("libc.dylib") + # logger.warning("kill_itself_when_parent_died is only supported in linux.") + else: + logger.warning("kill_itself_when_parent_died is only supported in linux.") + + +def get_exception_traceback() -> str: + etype, value, tb = sys.exc_info() + err_str = "".join(traceback.format_exception(etype, value, tb)) + return err_str + + +class TypeBasedDispatcher: + + def __init__(self, mapping: list[tuple[type, Callable]]): + self._mapping = mapping + + def __call__(self, obj: Any): + for ty, fn in self._mapping: + if isinstance(obj, ty): + return fn(obj) + raise ValueError(f"Invalid object: {obj}") + + +# For non-torch.distributed debugging +def remote_breakpoint() -> None: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(("localhost", 0)) # Let the OS pick an ephemeral port. + port = s.getsockname()[1] + RemotePdb(host="localhost", port=port).set_trace() + + +@dataclass +class MixedPrecisionState: + param_dtype: torch.dtype | None = None + reduce_dtype: torch.dtype | None = None + output_dtype: torch.dtype | None = None + compute_dtype: torch.dtype | None = None + mp_policy: MixedPrecisionPolicy | None = None + + +# Thread-local storage for mixed precision state +_mixed_precision_state = threading.local() + + +def get_mixed_precision_state() -> MixedPrecisionState: + """Get the current mixed precision state.""" + if not hasattr(_mixed_precision_state, "state"): + raise ValueError("Mixed precision state not set") + return cast(MixedPrecisionState, _mixed_precision_state.state) + + +def set_mixed_precision_policy( + param_dtype: torch.dtype, + reduce_dtype: torch.dtype, + output_dtype: torch.dtype | None = None, + mp_policy: MixedPrecisionPolicy | None = None, +): + """Set mixed precision policy globally. + + Args: + param_dtype: Parameter dtype used for training + reduce_dtype: Reduction dtype used for gradients + output_dtype: Optional output dtype + """ + state = MixedPrecisionState( + param_dtype=param_dtype, + reduce_dtype=reduce_dtype, + output_dtype=output_dtype, + mp_policy=mp_policy, + ) + _mixed_precision_state.state = state + + +def get_compute_dtype() -> torch.dtype: + """Get the current compute dtype from mixed precision policy. + + Returns: + torch.dtype: The compute dtype to use, defaults to get_default_dtype() if no policy set + """ + if not hasattr(_mixed_precision_state, "state"): + return torch.get_default_dtype() + else: + state = get_mixed_precision_state() + return state.param_dtype + + +def dict_to_3d_list( + mask_strategy: dict[str, Any] | None = None, + t_max: int | None = None, + l_max: int | None = None, + h_max: int | None = None, +) -> list[list[list[torch.Tensor | None]]]: + """ + Convert a dictionary of mask indices to a 3D list of tensors. + Args: + mask_strategy: keys are "t_l_h", values are torch.Tensor masks. + t_max, l_max, h_max: if provided (all three), force the output shape to (t_max, l_max, h_max). + If all three are None, infer shape from the data. + """ + # Case 1: no data, but fixed shape requested + if mask_strategy is None: + assert ( + t_max is not None and l_max is not None and h_max is not None + ), "If mask_strategy is None, you must provide t_max, l_max, and h_max" + return [ + [[None for _ in range(h_max)] for _ in range(l_max)] for _ in range(t_max) + ] + + # Parse all keys into integer tuples + indices = [tuple(map(int, key.split("_"))) for key in mask_strategy] + + # Decide on dimensions + if t_max is None and l_max is None and h_max is None: + # fully dynamic: infer from data + max_timesteps_idx = max(t for t, _, _ in indices) + 1 + max_layer_idx = max(l for _, l, _ in indices) + 1 # noqa: E741 + max_head_idx = max(h for _, _, h in indices) + 1 + else: + # require all three to be provided + assert t_max is not None and l_max is not None and h_max is not None, ( + "Either supply none of (t_max, l_max, h_max) to infer dimensions, " + "or supply all three to fix the shape." + ) + max_timesteps_idx = t_max + max_layer_idx = l_max + max_head_idx = h_max + + # Preallocate + result = [ + [[None for _ in range(max_head_idx)] for _ in range(max_layer_idx)] + for _ in range(max_timesteps_idx) + ] + + # Fill in, skipping any out-of-bounds entries + for key, value in mask_strategy.items(): + t, l, h = map(int, key.split("_")) # noqa: E741 + if ( + 0 <= t < max_timesteps_idx + and 0 <= l < max_layer_idx + and 0 <= h < max_head_idx + ): + result[t][l][h] = value + # else: silently ignore any key that doesn't fit + + return result + + +def set_random_seed(seed: int) -> None: + from sglang.multimodal_gen.runtime.platforms import current_platform + + current_platform.seed_everything(seed) + + +@lru_cache(maxsize=1) +def is_vsa_available() -> bool: + return importlib.util.find_spec("vsa") is not None + + +@lru_cache(maxsize=1) +def is_vmoba_available() -> bool: + if importlib.util.find_spec("kernel.csrc.attn.vmoba_attn.vmoba") is None: + return False + try: + import flash_attn + + return flash_attn.__version__ >= "2.7.4" + except Exception: + return False + + +# adapted from: https://github.com/Wan-Video/Wan2.2/blob/main/wan/utils/utils.py +def masks_like( + tensors, zero=False, generator=None, p=0.2 +) -> tuple[list[torch.Tensor], list[torch.Tensor]]: + """ + Generate binary masks for Text-to-Image-to-Video (TI2V) tasks. + + Creates masks to control which frames should be preserved vs replaced. + Primarily used to fix the first frame to the input image while generating other frames. + + Args: + tensors: List of tensors with shape [C, T, H, W] + zero: If True, set first frame (dim 1, index 0) to zero. Default: False + generator: Optional random generator for stochastic masking + p: Probability of applying special noise when generator is provided. Default: 0.2 + + Returns: + Tuple of two lists of tensors: + - When zero=False: Both lists contain all-ones tensors + - When zero=True (no generator): First frame set to 0, others to 1 + - When zero=True (with generator): First frame set to small random values with probability p + + Example: + >>> latent = torch.randn(48, 69, 96, 160) # [C, T, H, W] + >>> _, mask = masks_like([latent], zero=True) + >>> # mask[0][:, 0] == 0 (first frame) + >>> # mask[0][:, 1:] == 1 (other frames) + >>> blended = (1.0 - mask[0]) * image + mask[0] * latent + >>> # Result: first frame = image, other frames = latent + """ + assert isinstance(tensors, list) + out1 = [torch.ones(u.shape, dtype=u.dtype, device=u.device) for u in tensors] + + out2 = [torch.ones(u.shape, dtype=u.dtype, device=u.device) for u in tensors] + + if zero: + if generator is not None: + for u, v in zip(out1, out2, strict=False): + random_num = torch.rand( + 1, generator=generator, device=generator.device + ).item() + if random_num < p: + u[:, 0] = ( + torch.normal( + mean=-3.5, + std=0.5, + size=(1,), + device=u.device, + generator=generator, + ) + .expand_as(u[:, 0]) + .exp() + ) + v[:, 0] = torch.zeros_like(v[:, 0]) + else: + u[:, 0] = u[:, 0] + v[:, 0] = v[:, 0] + + else: + for u, v in zip(out1, out2, strict=False): + u[:, 0] = torch.zeros_like(u[:, 0]) + v[:, 0] = torch.zeros_like(v[:, 0]) + + return out1, out2 + + +# adapted from: https://github.com/Wan-Video/Wan2.2/blob/main/wan/utils/utils.py +def best_output_size(w, h, dw, dh, expected_area): + # float output size + ratio = w / h + ow = (expected_area * ratio) ** 0.5 + oh = expected_area / ow + + # process width first + ow1 = int(ow // dw * dw) + oh1 = int(expected_area / ow1 // dh * dh) + assert ow1 % dw == 0 and oh1 % dh == 0 and ow1 * oh1 <= expected_area + ratio1 = ow1 / oh1 + + # process height first + oh2 = int(oh // dh * dh) + ow2 = int(expected_area / oh2 // dw * dw) + assert oh2 % dh == 0 and ow2 % dw == 0 and ow2 * oh2 <= expected_area + ratio2 = ow2 / oh2 + + # compare ratios + if max(ratio / ratio1, ratio1 / ratio) < max(ratio / ratio2, ratio2 / ratio): + return ow1, oh1 + else: + return ow2, oh2 + + +def save_decoded_latents_as_video( + decoded_latents: list[torch.Tensor], output_path: str, fps: int +): + # Process outputs + videos = rearrange(decoded_latents, "b c t h w -> t b c h w") + frames = [] + for x in videos: + x = torchvision.utils.make_grid(x, nrow=6) + x = x.transpose(0, 1).transpose(1, 2).squeeze(-1) + frames.append((x * 255).numpy().astype(np.uint8)) + + os.makedirs(os.path.dirname(output_path), exist_ok=True) + imageio.mimsave(output_path, frames, fps=fps, format="mp4") + + +def calculate_dimensions(target_area, ratio): + width = math.sqrt(target_area * ratio) + height = width / ratio + + width = round(width / 32) * 32 + height = round(height / 32) * 32 + + return width, height, None diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/profiler.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/profiler.py new file mode 100644 index 00000000..0ffa07f2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/profiler.py @@ -0,0 +1,155 @@ +""" +Run live profiling. + +Usage: +python3 -m sglang.profiler +""" + +import argparse +import json +import os +import time +from argparse import ArgumentParser +from pathlib import Path +from typing import List, Optional + +import requests + +PROFILER_DIR = os.getenv("SGLANG_TORCH_PROFILER_DIR", "/tmp") + + +def run_profile( + url: Optional[str], + num_steps: int, + activities: List[str], + output_dir: Optional[str] = None, + profile_by_stage: bool = False, + merge_profiles: bool = False, + profile_prefix: Optional[str] = None, +) -> str: + if output_dir is None: + output_dir = PROFILER_DIR + + output_dir = Path(os.path.abspath(os.path.normpath(output_dir))) / str(time.time()) + output_dir.mkdir(exist_ok=True, parents=True) + + print(f"Dump profiling traces to {output_dir}") + print( + f"Waiting for {num_steps} steps and the trace to be flushed.... ({profile_by_stage=})" + ) + + # Dump server args. + file_path = Path(output_dir) / "server_args.json" + if not file_path.exists(): + response = requests.get(url + "/get_server_info") + response.raise_for_status() + server_args_data = response.json() + with open(file_path, "w") as file: + file.write(json.dumps(server_args_data)) + + # Start profiler. The API replies when all steps are processed + # and files are generated. + json_data = { + "output_dir": str(output_dir), + "num_steps": str(num_steps), + "activities": activities, + "profile_by_stage": profile_by_stage, + "merge_profiles": merge_profiles, + "profile_prefix": profile_prefix, + } + + response = requests.post(url=url + "/start_profile", json=json_data) + response.raise_for_status() + + trace_link = str(output_dir) + return trace_link + + +if __name__ == "__main__": + parser = ArgumentParser(description="Benchmark the online serving throughput.") + parser.add_argument( + "--url", + type=str, + default="http://localhost:30000", + help="Server or API base url if not using http host and port.", + ) + parser.add_argument( + "--output-dir", + type=str, + default=None, + help="Profile directory to dump profile traces.", + ) + parser.add_argument( + "--num-steps", + type=int, + default=5, + help="The number of forward steps to profile.", + ) + parser.add_argument( + "--profile-by-stage", + action=argparse.BooleanOptionalAction, + type=bool, + default=False, + help="Whether to profile prefill and decode separately", + ) + parser.add_argument( + "--profile-prefix", + type=str, + help="The prefix of this profiler file.", + ) + parser.add_argument( + "--cpu", + action=argparse.BooleanOptionalAction, + type=bool, + default=True, + help="Whether to profile CPU activity", + ) + parser.add_argument( + "--gpu", + action=argparse.BooleanOptionalAction, + type=bool, + default=True, + help="Whether to profile GPU activity", + ) + parser.add_argument( + "--mem", + action=argparse.BooleanOptionalAction, + type=bool, + default=False, + help="Whether to memory usage (https://pytorch.org/memory_viz)", + ) + parser.add_argument( + "--rpd", + action=argparse.BooleanOptionalAction, + type=bool, + default=False, + help="Whether to use ROCM rpd profiler (https://github.com/ROCm/rocmProfileData)", + ) + parser.add_argument( + "--merge-profiles", + action=argparse.BooleanOptionalAction, + type=bool, + default=False, + help="Whether to merge profiles from all ranks into a single trace file", + ) + + args = parser.parse_args() + activities = [] + if args.cpu: + activities.append("CPU") + if args.gpu: + activities.append("GPU") + if args.mem: + activities.append("MEM") + if args.rpd: + activities.append("RPD") + + run_profile( + url=args.url, + num_steps=args.num_steps, + activities=activities, + output_dir=args.output_dir, + profile_by_stage=args.profile_by_stage, + profile_prefix=args.profile_prefix, + merge_profiles=args.merge_profiles, + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/_custom_ops.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/_custom_ops.py new file mode 100644 index 00000000..3353aa2e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/_custom_ops.py @@ -0,0 +1,165 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/_custom_ops.py +import logging +from typing import List, Optional, Tuple + +import torch + +from sglang.srt.utils import is_hip, is_hpu, is_npu + +logger = logging.getLogger(__name__) + + +if not is_hpu(): + try: + import sgl_kernel + except ImportError as e: + logger.warning("Failed to import from custom_ar with %r", e) + + +if not is_hip() and not is_npu(): + custom_op = sgl_kernel.allreduce + + # custom allreduce + def init_custom_ar( + ipc_tensors: List[torch.Tensor], + rank_data: torch.Tensor, + rank: int, + full_nvlink: bool, + ) -> int: + return custom_op.init_custom_ar(ipc_tensors, rank_data, rank, full_nvlink) + + def all_reduce( + fa: int, + inp: torch.Tensor, + out: torch.Tensor, + reg_buffer: int, + reg_buffer_sz_bytes: int, + ) -> None: + custom_op.all_reduce(fa, inp, out, reg_buffer, reg_buffer_sz_bytes) + + def dispose(fa: int) -> None: + custom_op.dispose(fa) + + def meta_size() -> int: + return custom_op.meta_size() + + def register_buffer(fa: int, ipc_tensors: List[int]) -> None: + return custom_op.register_buffer(fa, ipc_tensors) + + def get_graph_buffer_ipc_meta(fa: int) -> Tuple[List[int], List[int]]: + return custom_op.get_graph_buffer_ipc_meta(fa) + + def register_graph_buffers( + fa: int, handles: List[List[int]], offsets: List[List[int]] + ) -> None: + custom_op.register_graph_buffers(fa, handles, offsets) + +else: + # ROCM custom allreduce + + def init_custom_ar( + meta: torch.Tensor, + rank_data: torch.Tensor, + handles: List[str], + offsets: List[int], + rank: int, + full_nvlink: bool, + ) -> int: + return sgl_kernel.allreduce.init_custom_ar( + meta, rank_data, handles, offsets, rank, full_nvlink + ) + + def all_reduce_reg(fa: int, inp: torch.Tensor, out: torch.Tensor) -> None: + sgl_kernel.allreduce.all_reduce_reg(fa, inp, out) + + def all_reduce_unreg( + fa: int, inp: torch.Tensor, reg_buffer: torch.Tensor, out: torch.Tensor + ) -> None: + sgl_kernel.allreduce.all_reduce_unreg(fa, inp, reg_buffer, out) + + def dispose(fa: int) -> None: + sgl_kernel.allreduce.dispose(fa) + + def meta_size() -> int: + return sgl_kernel.allreduce.meta_size() + + def register_buffer( + fa: int, t: torch.Tensor, handles: List[str], offsets: List[int] + ) -> None: + return sgl_kernel.allreduce.register_buffer(fa, t, handles, offsets) + + def get_graph_buffer_ipc_meta(fa: int) -> Tuple[torch.Tensor, List[int]]: + return sgl_kernel.allreduce.get_graph_buffer_ipc_meta(fa) + + def register_graph_buffers( + fa: int, handles: List[str], offsets: List[List[int]] + ) -> None: + sgl_kernel.allreduce.register_graph_buffers(fa, handles, offsets) + + def allocate_meta_buffer(size: int) -> torch.Tensor: + return sgl_kernel.allreduce.allocate_meta_buffer(size) + + def get_meta_buffer_ipc_handle(inp: torch.Tensor) -> torch.Tensor: + return sgl_kernel.allreduce.get_meta_buffer_ipc_handle(inp) + + # ROCM custom quick allreduce + + def init_custom_qr( + rank: int, world_size: int, qr_max_size: Optional[int] = None + ) -> int: + return sgl_kernel.allreduce.init_custom_qr(world_size, rank, qr_max_size) + + def qr_get_handle(fa: int) -> torch.Tensor: + return sgl_kernel.allreduce.qr_get_handle(fa) + + def qr_open_handles(fa: int, handles: list[torch.Tensor]) -> None: + sgl_kernel.allreduce.qr_open_handles(fa, handles) + + def qr_all_reduce( + fa: int, + inp: torch.Tensor, + out: torch.Tensor, + quant_level: int, + cast_bf2half: bool, + ) -> None: + sgl_kernel.allreduce.qr_all_reduce(fa, inp, out, quant_level, cast_bf2half) + + def qr_destroy(fa: int) -> None: + sgl_kernel.allreduce.qr_destroy(fa) + + def qr_max_size() -> int: + return sgl_kernel.allreduce.qr_max_size() + + +def mscclpp_generate_unique_id() -> bytes: + return sgl_kernel.allreduce.mscclpp_generate_unique_id() + + +def mscclpp_init_context( + unique_id: bytes, + rank: int, + world_size: int, + scratch: torch.Tensor, + put_buffer: torch.Tensor, + nranks_per_node: int, + rank_to_node: List[int], + rank_to_ib: List[int], + context_selection: int, +) -> int: + return sgl_kernel.allreduce.mscclpp_init_context( + unique_id, + rank, + world_size, + scratch, + put_buffer, + nranks_per_node, + rank_to_node, + rank_to_ib, + context_selection, + ) + + +def mscclpp_allreduce( + context: int, inp: torch.Tensor, out: torch.Tensor, nthreads: int, nblocks: int +) -> None: + return sgl_kernel.allreduce.mscclpp_allreduce(context, inp, out, nthreads, nblocks) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_invariant_ops/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_invariant_ops/__init__.py new file mode 100644 index 00000000..3caa60f0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_invariant_ops/__init__.py @@ -0,0 +1,29 @@ +# Adapted from https://github.com/thinking-machines-lab/batch_invariant_ops/blob/main/batch_invariant_ops/__init__.py + +from .batch_invariant_ops import ( + AttentionBlockSize, + disable_batch_invariant_mode, + enable_batch_invariant_mode, + get_batch_invariant_attention_block_size, + is_batch_invariant_mode_enabled, + log_softmax, + matmul_persistent, + mean_dim, + rms_norm_batch_invariant, + set_batch_invariant_mode, +) + +__version__ = "0.1.0" + +__all__ = [ + "set_batch_invariant_mode", + "is_batch_invariant_mode_enabled", + "disable_batch_invariant_mode", + "enable_batch_invariant_mode", + "matmul_persistent", + "log_softmax", + "mean_dim", + "get_batch_invariant_attention_block_size", + "AttentionBlockSize", + "rms_norm_batch_invariant", +] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_invariant_ops/batch_invariant_ops.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_invariant_ops/batch_invariant_ops.py new file mode 100644 index 00000000..67bb1c5c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_invariant_ops/batch_invariant_ops.py @@ -0,0 +1,991 @@ +# Adapted from https://github.com/thinking-machines-lab/batch_invariant_ops/blob/main/batch_invariant_ops/batch_invariant_ops.py + +import contextlib +from collections import namedtuple +from collections.abc import Callable +from typing import Any, Dict + +import torch +import triton +import triton.language as tl + +from sglang.srt.layers.deep_gemm_wrapper.configurer import ENABLE_JIT_DEEPGEMM +from sglang.srt.utils.common import calc_diff, get_bool_env_var + +if ENABLE_JIT_DEEPGEMM: + import deep_gemm + +_ENABLE_MM_DEEPGEMM = get_bool_env_var( + "SGLANG_BATCH_INVARIANT_OPS_ENABLE_MM_DEEPGEMM", "1" +) +# If true, allows to fallback to batch variant gemm when the shape cannot be run in DeepGEMM +_ENABLE_MM_FALLBACK_VARIANT = get_bool_env_var( + "SGLANG_BATCH_INVARIANT_OPS_ENABLE_MM_FALLBACK_VARIANT", "0" +) +_ENABLE_MM_COMPARISON_TEST = get_bool_env_var( + "SGLANG_BATCH_INVARIANT_OPS_ENABLE_MM_COMPARISON_TEST" +) + +if not _ENABLE_MM_DEEPGEMM: + print("Disable DeepGEMM in batch invariant ops. Performance may be suboptimal.") + +__all__ = [ + "set_batch_invariant_mode", + "is_batch_invariant_mode_enabled", + "disable_batch_invariant_mode", + "enable_batch_invariant_mode", +] + + +def _matmul_launch_metadata( + grid: Callable[..., Any], kernel: Any, args: Dict[str, Any] +) -> Dict[str, Any]: + ret = {} + m, n, k = args["M"], args["N"], args["K"] + ret["name"] = f"{kernel.name} [M={m}, N={n}, K={k}]" + if "tiles_per_update" in args: + ret["name"] = ( + f"{kernel.name} [M={m}, N={n}, K={k}, tiles_per_update={args['tiles_per_update']:02}]" + ) + if "c_ptr" in args: + bytes_per_elem = args["c_ptr"].element_size() + else: + bytes_per_elem = 1 if args["FP8_OUTPUT"] else 2 + ret[f"flops{bytes_per_elem * 8}"] = 2.0 * m * n * k + ret["bytes"] = bytes_per_elem * (m * k + n * k + m * n) + return ret + + +@triton.jit +def _compute_pid(tile_id, num_pid_in_group, num_pid_m, GROUP_SIZE_M, NUM_SMS): + group_id = tile_id // num_pid_in_group + first_pid_m = group_id * GROUP_SIZE_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) + pid_m = first_pid_m + (tile_id % group_size_m) + pid_n = (tile_id % num_pid_in_group) // group_size_m + return pid_m, pid_n + + +@triton.jit(launch_metadata=_matmul_launch_metadata) +def matmul_kernel_persistent( + a_ptr, + b_ptr, + c_ptr, # + bias_ptr, + M, + N, + K, # + stride_am, + stride_ak, + stride_bk, + stride_bn, + stride_cm, + stride_cn, + BLOCK_SIZE_M: tl.constexpr, # + BLOCK_SIZE_N: tl.constexpr, # + BLOCK_SIZE_K: tl.constexpr, # + GROUP_SIZE_M: tl.constexpr, # + NUM_SMS: tl.constexpr, # + A_LARGE: tl.constexpr, + B_LARGE: tl.constexpr, + C_LARGE: tl.constexpr, + HAS_BIAS: tl.constexpr, +): + start_pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_SIZE_M) + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + k_tiles = tl.cdiv(K, BLOCK_SIZE_K) + num_tiles = num_pid_m * num_pid_n + + offs_k_for_mask = tl.arange(0, BLOCK_SIZE_K) + num_pid_in_group = GROUP_SIZE_M * num_pid_n + + for tile_id in tl.range(start_pid, num_tiles, NUM_SMS, flatten=True): + pid_m, pid_n = _compute_pid( + tile_id, num_pid_in_group, num_pid_m, GROUP_SIZE_M, NUM_SMS + ) + start_m = pid_m * BLOCK_SIZE_M + start_n = pid_n * BLOCK_SIZE_N + offs_am = start_m + tl.arange(0, BLOCK_SIZE_M) + offs_bn = start_n + tl.arange(0, BLOCK_SIZE_N) + if A_LARGE: + offs_am = offs_am.to(tl.int64) + if B_LARGE: + offs_bn = offs_bn.to(tl.int64) + offs_am = tl.where(offs_am < M, offs_am, 0) + offs_bn = tl.where(offs_bn < N, offs_bn, 0) + offs_am = tl.max_contiguous(tl.multiple_of(offs_am, BLOCK_SIZE_M), BLOCK_SIZE_M) + offs_bn = tl.max_contiguous(tl.multiple_of(offs_bn, BLOCK_SIZE_N), BLOCK_SIZE_N) + + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + for ki in range(k_tiles): + if A_LARGE or B_LARGE: + offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K).to(tl.int64) + else: + offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K) + a_ptrs = a_ptr + ( + offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak + ) + b_ptrs = b_ptr + ( + offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn + ) + + a = tl.load( + a_ptrs, mask=offs_k_for_mask[None, :] < K - ki * BLOCK_SIZE_K, other=0.0 + ) + b = tl.load( + b_ptrs, mask=offs_k_for_mask[:, None] < K - ki * BLOCK_SIZE_K, other=0.0 + ) + accumulator = tl.dot(a, b, accumulator) + + offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + if C_LARGE: + offs_cm = offs_cm.to(tl.int64) + offs_cn = offs_cn.to(tl.int64) + c_ptrs = c_ptr + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] + c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) + if HAS_BIAS: + bias_ptrs = bias_ptr + offs_cn + bias = tl.load(bias_ptrs, mask=offs_cn < N, other=0.0).to(tl.float32) + accumulator += bias + if c_ptr.dtype.element_ty == tl.float8e4nv: + c = accumulator.to(tl.float8e4nv) + elif c_ptr.dtype.element_ty == tl.bfloat16: + c = accumulator.to(tl.bfloat16) + elif c_ptr.dtype.element_ty == tl.float32: + c = accumulator.to(tl.float32) + else: + c = accumulator.to(tl.float16) + tl.store(c_ptrs, c, mask=c_mask) + + +def _matmul_persistent_triton( + a: torch.Tensor, b: torch.Tensor, bias: torch.Tensor | None = None +): + # Check constraints. + assert a.shape[1] == b.shape[0], "Incompatible dimensions" + assert a.dtype == b.dtype, "Incompatible dtypes" + assert ( + bias is None or bias.dim() == 1 + ), "Currently assuming bias is 1D, let Horace know if you run into this" + NUM_SMS = torch.cuda.get_device_properties("cuda").multi_processor_count + M, K = a.shape + K, N = b.shape + dtype = a.dtype + # Allocates output. + c = torch.empty((M, N), device=a.device, dtype=dtype) + + # 1D launch kernel where each block gets its own program. + def grid(META): + return ( + min( + NUM_SMS, + triton.cdiv(M, META["BLOCK_SIZE_M"]) + * triton.cdiv(N, META["BLOCK_SIZE_N"]), + ), + ) + + configs = { + torch.bfloat16: { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_stages": 3, + "num_warps": 8, + }, + torch.float16: { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_stages": 3, + "num_warps": 8, + }, + torch.float32: { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8, + "num_stages": 3, + "num_warps": 8, + }, + } + # print(a.device, b.device, c.device) + matmul_kernel_persistent[grid]( + a, + b, + c, # + bias, + M, + N, + K, # + a.stride(0), + a.stride(1), # + b.stride(0), + b.stride(1), # + c.stride(0), + c.stride(1), # + NUM_SMS=NUM_SMS, # + A_LARGE=a.numel() > 2**31, + B_LARGE=b.numel() > 2**31, + C_LARGE=c.numel() > 2**31, + HAS_BIAS=bias is not None, + **configs[dtype], + ) + return c + + +def _matmul_persistent_deepgemm( + a: torch.Tensor, b: torch.Tensor, bias: torch.Tensor | None = None +): + M, K = a.shape + K, N = b.shape + dtype = a.dtype + out = torch.empty((M, N), device=a.device, dtype=dtype) + + try: + deep_gemm.bf16_gemm_nn(a, b, out) + except RuntimeError as e: + raise RuntimeError( + f"DeepGEMM failed for matrix shapes M={M}, N={N}, K={K}. " + f"This typically occurs when dimensions are too small for DeepGEMM's TMA descriptors. " + f"Consider increasing MIN_DEEPGEMM_DIM in matmul_persistent() or disabling DeepGEMM " + f"for small matrices. Original error: {e}" + ) from e + + # TODO can this be put in DeepGEMM's `c`? + if bias is not None: + out += bias + + return out + + +def matmul_persistent( + a: torch.Tensor, b: torch.Tensor, bias: torch.Tensor | None = None +): + K, N = b.shape + + # DeepGEMM has minimum dimension requirements for TMA descriptors + MIN_DEEPGEMM_DIM = 16 + + if ( + _ENABLE_MM_DEEPGEMM + and ENABLE_JIT_DEEPGEMM + and (a.dtype == torch.bfloat16) + and (b.dtype == torch.bfloat16) + and a.is_contiguous() + and b.transpose(0, 1).is_contiguous() + and N >= MIN_DEEPGEMM_DIM + ): + if _ENABLE_MM_COMPARISON_TEST: + out_triton = _matmul_persistent_triton(a=a, b=b, bias=bias) + out_deepgemm = _matmul_persistent_deepgemm(a=a, b=b, bias=bias) + diff = calc_diff(out_triton, out_deepgemm) + assert diff < 0.0001, f"{diff=} {out_triton=} {out_deepgemm=}" + # can be enabled for debugging + # print( + # f"{diff=} " + # f"{(out_triton - out_deepgemm).abs().mean()=} " + # f"{(out_triton - out_deepgemm).abs().sum()=} " + # f"{torch.sum(out_triton != out_deepgemm)=} " + # ) + # print(f"{a=} {b=} {bias=} {out_triton=} {out_deepgemm=}") + return out_deepgemm + + return _matmul_persistent_deepgemm(a=a, b=b, bias=bias) + + if _ENABLE_MM_FALLBACK_VARIANT: + return torch.einsum("ik,kj->ij", a, b) + + return _matmul_persistent_triton(a=a, b=b, bias=bias) + + +@triton.jit +def _log_softmax_kernel( + input_ptr, + output_ptr, + input_row_stride, + output_row_stride, + n_cols, + BLOCK_SIZE: tl.constexpr, +): + """ + Compute log_softmax along the last dimension of a 2D tensor. + Each block handles one row of the input tensor. + """ + # Get the row index for this block + row_idx = tl.program_id(0).to(tl.int64) + + # Compute base pointers for input and output rows + row_start_ptr = input_ptr + row_idx * input_row_stride + output_row_start_ptr = output_ptr + row_idx * output_row_stride + + # Step 1: Find maximum value in the row for numerical stability + # Load first block to infer dtype and initialize max_val with correct type + col_idx_init = tl.arange(0, BLOCK_SIZE) + mask_init = col_idx_init < n_cols + vals_init = tl.load( + row_start_ptr + col_idx_init, mask=mask_init, other=-float("inf") + ) + max_val = tl.max(vals_init) + + # Continue with remaining blocks + for col_offset in range(BLOCK_SIZE, n_cols, BLOCK_SIZE): + col_idx = col_offset + tl.arange(0, BLOCK_SIZE) + mask = col_idx < n_cols + + # Load values + vals = tl.load(row_start_ptr + col_idx, mask=mask, other=-float("inf")) + + # Update maximum + max_val = tl.max(tl.maximum(vals, max_val)) + + # Step 2: Compute sum of exp(x - max_val) + # Initialize sum_exp with correct dtype by using tl.sum on a zero vector + sum_exp = tl.sum(tl.zeros([1], dtype=max_val.dtype)) + + for col_offset in range(0, n_cols, BLOCK_SIZE): + col_idx = col_offset + tl.arange(0, BLOCK_SIZE) + mask = col_idx < n_cols + + # Load values + vals = tl.load(row_start_ptr + col_idx, mask=mask, other=0.0) + + # Compute exp(x - max_val) and accumulate + exp_vals = tl.exp(vals - max_val) + sum_exp += tl.sum(tl.where(mask, exp_vals, 0.0)) + + # Compute log(sum_exp) + log_sum_exp = tl.log(sum_exp) + + # Step 3: Compute final log_softmax values: x - max_val - log_sum_exp + for col_offset in range(0, n_cols, BLOCK_SIZE): + col_idx = col_offset + tl.arange(0, BLOCK_SIZE) + mask = col_idx < n_cols + + # Load values + vals = tl.load(row_start_ptr + col_idx, mask=mask) + + # Compute log_softmax + output = vals - max_val - log_sum_exp + + # Store results + tl.store(output_row_start_ptr + col_idx, output, mask=mask) + + +def log_softmax(input: torch.Tensor, dim: int = -1) -> torch.Tensor: + """ + Compute log_softmax using Triton kernel. + + Args: + input: Input tensor + dim: Dimension along which to compute log_softmax (only -1 or last dim supported) + >> Stashed changes + Returns: + Tensor with log_softmax applied along the specified dimension + """ + if dim != -1 and dim != input.ndim - 1: + raise ValueError( + "This implementation only supports log_softmax along the last dimension" + ) + + # Flatten all dimensions except the last one + original_shape = input.shape + input_2d = input.reshape(-1, input.shape[-1]) + input_2d = input_2d.contiguous() + + n_rows, n_cols = input_2d.shape + + # Allocate output tensor + output = torch.empty_like(input_2d) + + # Choose block size based on the number of columns + BLOCK_SIZE = 1024 + + # Launch kernel with one block per row + grid = (n_rows,) + _log_softmax_kernel[grid]( + input_2d, + output, + input_2d.stride(0), + output.stride(0), + n_cols, + BLOCK_SIZE=BLOCK_SIZE, + ) + # Reshape output back to original shape + return output.reshape(original_shape) + + +@triton.jit +def mean_kernel( + input_ptr, + output_ptr, + input_stride0, + input_stride1, + input_stride2, + output_stride0, + output_stride1, + M, # size before reduction dim + N, # size of reduction dim + K, # size after reduction dim + BLOCK_SIZE: tl.constexpr, +): + """ + Kernel for computing mean along a single dimension. + Input is viewed as (M, N, K) where N is the dimension being reduced. + """ + # Program ID gives us which output element we're computing + pid = tl.program_id(0) + + # Compute output indices + m_idx = pid // K + k_idx = pid % K + + # Bounds check + if m_idx >= M or k_idx >= K: + return + + # Accumulate sum across reduction dimension + acc = 0.0 + for n_start in range(0, N, BLOCK_SIZE): + n_offsets = n_start + tl.arange(0, BLOCK_SIZE) + mask = n_offsets < N + + # Calculate input indices + input_idx = ( + m_idx * input_stride0 + n_offsets * input_stride1 + k_idx * input_stride2 + ) + + # Load and accumulate + vals = tl.load(input_ptr + input_idx, mask=mask, other=0.0) + acc += tl.sum(vals) + + # Compute mean and store + mean_val = acc / N + output_idx = m_idx * output_stride0 + k_idx * output_stride1 + tl.store(output_ptr + output_idx, mean_val) + + +def mean_dim( + input: torch.Tensor, + dim: int, + keepdim: bool = False, + dtype: torch.dtype | None = None, +) -> torch.Tensor: + """ + Triton implementation of torch.mean with single dimension reduction. + + Args: + input: Input tensor + dim: Single dimension along which to compute mean + keepdim: Whether to keep the reduced dimension + dtype: Output dtype. If None, uses input dtype (or float32 for integer inputs) + + Returns: + Tensor with mean values along specified dimension + """ + # Validate inputs + assert input.is_cuda, "Input must be a CUDA tensor" + assert ( + -input.ndim <= dim < input.ndim + ), f"Invalid dimension {dim} for tensor with {input.ndim} dimensions" + + # Handle negative dim + if dim < 0: + dim = dim + input.ndim + + # Handle dtype + if dtype is None: + if input.dtype in [torch.int8, torch.int16, torch.int32, torch.int64]: + dtype = torch.float32 + else: + dtype = input.dtype + + # Convert input to appropriate dtype if needed + if input.dtype != dtype: + input = input.to(dtype) + + # Get input shape and strides + shape = list(input.shape) + + # Calculate dimensions for kernel + M = 1 + for i in range(dim): + M *= shape[i] + + N = shape[dim] + + K = 1 + for i in range(dim + 1, len(shape)): + K *= shape[i] + + # Reshape input to 3D view (M, N, K) + input_3d = input.reshape(M, N, K) + + # Create output shape + if keepdim: + output_shape = shape.copy() + output_shape[dim] = 1 + else: + output_shape = shape[:dim] + shape[dim + 1 :] + + # Create output tensor + output = torch.empty(output_shape, dtype=dtype, device=input.device) + + # Reshape output for kernel + if keepdim: + output_2d = output.reshape(M, 1, K).squeeze(1) + else: + output_2d = output.reshape(M, K) + + # Launch kernel + grid = (M * K,) + BLOCK_SIZE = 1024 + + mean_kernel[grid]( + input_3d, + output_2d, + input_3d.stride(0), + input_3d.stride(1), + input_3d.stride(2), + output_2d.stride(0), + output_2d.stride(1) if output_2d.ndim > 1 else 0, + M, + N, + K, + BLOCK_SIZE, + ) + + return output + + +def mm_batch_invariant(a, b): + return matmul_persistent(a, b) + + +def addmm_batch_invariant(bias, a, b): + return matmul_persistent(a, b, bias=bias) + + +def _log_softmax_batch_invariant(input, dim, _half_to_float): + assert not _half_to_float, "not implemented" + return log_softmax(input, dim=dim) + + +def mean_batch_invariant(input, dim, keepdim=False, dtype: torch.dtype | None = None): + assert dtype is None or dtype == torch.float32, f"unsupported dtype: {dtype}" + if len(dim) == 1: + return mean_dim(input, dim[0], keepdim=keepdim) + else: + assert input.dtype in { + torch.float16, + torch.bfloat16, + torch.float32, + }, "only float types supported for now" + n_elems = 1 + for d in dim: + n_elems *= input.shape[d] + return torch.sum(input, dim=dim, keepdim=keepdim, dtype=torch.float32) / n_elems + + +@triton.jit +def bmm_kernel_persistent( + a_ptr, + b_ptr, + c_ptr, # + B, + M, + N, + K, # + stride_ab, + stride_am, + stride_ak, + stride_bb, + stride_bk, + stride_bn, + stride_cb, + stride_cm, + stride_cn, + BLOCK_SIZE_M: tl.constexpr, # + BLOCK_SIZE_N: tl.constexpr, # + BLOCK_SIZE_K: tl.constexpr, # + GROUP_SIZE_M: tl.constexpr, # + NUM_SMS: tl.constexpr, # + A_LARGE: tl.constexpr, + B_LARGE: tl.constexpr, + C_LARGE: tl.constexpr, +): + """ + Batched matrix multiplication kernel that processes batches in parallel. + Each tile processes a (BLOCK_SIZE_M, BLOCK_SIZE_N) output block for a specific batch. + """ + start_pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_SIZE_M) + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + k_tiles = tl.cdiv(K, BLOCK_SIZE_K) + num_tiles_per_batch = num_pid_m * num_pid_n + num_tiles_total = B * num_tiles_per_batch + + offs_k_for_mask = tl.arange(0, BLOCK_SIZE_K) + num_pid_in_group = GROUP_SIZE_M * num_pid_n + + # Process tiles in a deterministic order: batch-major ordering + for tile_id in tl.range(start_pid, num_tiles_total, NUM_SMS, flatten=True): + # Decompose tile_id into batch and within-batch tile + batch_idx = tile_id // num_tiles_per_batch + tile_in_batch = tile_id % num_tiles_per_batch + + pid_m, pid_n = _compute_pid( + tile_in_batch, num_pid_in_group, num_pid_m, GROUP_SIZE_M, NUM_SMS + ) + start_m = pid_m * BLOCK_SIZE_M + start_n = pid_n * BLOCK_SIZE_N + offs_am = start_m + tl.arange(0, BLOCK_SIZE_M) + offs_bn = start_n + tl.arange(0, BLOCK_SIZE_N) + if A_LARGE: + offs_am = offs_am.to(tl.int64) + if B_LARGE: + offs_bn = offs_bn.to(tl.int64) + offs_am = tl.where(offs_am < M, offs_am, 0) + offs_bn = tl.where(offs_bn < N, offs_bn, 0) + offs_am = tl.max_contiguous(tl.multiple_of(offs_am, BLOCK_SIZE_M), BLOCK_SIZE_M) + offs_bn = tl.max_contiguous(tl.multiple_of(offs_bn, BLOCK_SIZE_N), BLOCK_SIZE_N) + + # Add batch offset + if A_LARGE or B_LARGE: + batch_idx_typed = batch_idx.to(tl.int64) + else: + batch_idx_typed = batch_idx + + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + for ki in range(k_tiles): + if A_LARGE or B_LARGE: + offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K).to(tl.int64) + else: + offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K) + + a_ptrs = a_ptr + ( + batch_idx_typed * stride_ab + + offs_am[:, None] * stride_am + + offs_k[None, :] * stride_ak + ) + b_ptrs = b_ptr + ( + batch_idx_typed * stride_bb + + offs_k[:, None] * stride_bk + + offs_bn[None, :] * stride_bn + ) + + a = tl.load( + a_ptrs, mask=offs_k_for_mask[None, :] < K - ki * BLOCK_SIZE_K, other=0.0 + ) + b = tl.load( + b_ptrs, mask=offs_k_for_mask[:, None] < K - ki * BLOCK_SIZE_K, other=0.0 + ) + accumulator = tl.dot(a, b, accumulator) + + offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + if C_LARGE: + offs_cm = offs_cm.to(tl.int64) + offs_cn = offs_cn.to(tl.int64) + c_ptrs = ( + c_ptr + + batch_idx_typed * stride_cb + + stride_cm * offs_cm[:, None] + + stride_cn * offs_cn[None, :] + ) + c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) + + if c_ptr.dtype.element_ty == tl.float8e4nv: + c = accumulator.to(tl.float8e4nv) + elif c_ptr.dtype.element_ty == tl.bfloat16: + c = accumulator.to(tl.bfloat16) + elif c_ptr.dtype.element_ty == tl.float32: + c = accumulator.to(tl.float32) + else: + c = accumulator.to(tl.float16) + tl.store(c_ptrs, c, mask=c_mask) + + +def bmm_batch_invariant(a, b, *, out=None): + # Batched matrix multiply: (B, M, K) x (B, K, N) -> (B, M, N) + # Process batches in parallel with our persistent kernel + if a.ndim == 3 and b.ndim == 3: + # Check constraints + assert a.shape[0] == b.shape[0], "Batch sizes must match" + assert a.shape[2] == b.shape[1], "Incompatible dimensions" + assert a.dtype == b.dtype, "Incompatible dtypes" + + B = a.shape[0] + M = a.shape[1] + K = a.shape[2] + N = b.shape[2] + dtype = a.dtype + + # Allocate output + if out is None: + c = torch.empty((B, M, N), device=a.device, dtype=dtype) + else: + c = out + + NUM_SMS = torch.cuda.get_device_properties("cuda").multi_processor_count + + # Use fixed kernel configuration for determinism + configs = { + torch.bfloat16: { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_stages": 3, + "num_warps": 8, + }, + torch.float16: { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_stages": 3, + "num_warps": 8, + }, + torch.float32: { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8, + "num_stages": 3, + "num_warps": 8, + }, + } + + config = configs.get(dtype) + if config is None: + raise ValueError( + f"Unsupported dtype {dtype} for bmm_batch_invariant. " + f"Supported dtypes are: {list(configs.keys())}" + ) + + # Grid: limit by NUM_SMS for persistent kernel approach + num_tiles_per_batch = triton.cdiv(M, config["BLOCK_SIZE_M"]) * triton.cdiv( + N, config["BLOCK_SIZE_N"] + ) + num_tiles_total = B * num_tiles_per_batch + grid = (min(NUM_SMS, num_tiles_total),) + + bmm_kernel_persistent[grid]( + a, + b, + c, # + B, + M, + N, + K, # + a.stride(0), + a.stride(1), + a.stride(2), # + b.stride(0), + b.stride(1), + b.stride(2), # + c.stride(0), + c.stride(1), + c.stride(2), # + NUM_SMS=NUM_SMS, # + A_LARGE=a.numel() > 2**31, + B_LARGE=b.numel() > 2**31, + C_LARGE=c.numel() > 2**31, + **config, + ) + + return c + else: + raise ValueError( + f"bmm_batch_invariant expects 3D tensors, " + f"got shapes {a.shape} and {b.shape}" + ) + + +@triton.jit +def _rms_norm_kernel( + input_ptr, + weight_ptr, + output_ptr, + input_row_stride, + output_row_stride, + n_cols, + eps, + BLOCK_SIZE: tl.constexpr, +): + """ + Compute RMS normalization along the last dimension of a 2D tensor. + RMS Norm: y = x / sqrt(mean(x^2) + eps) * weight + Each block handles one row of the input tensor. + """ + row_idx = tl.program_id(0).to(tl.int64) + row_start_ptr = input_ptr + row_idx * input_row_stride + output_row_start_ptr = output_ptr + row_idx * output_row_stride + + # Step 1: Compute sum of squares in float32 to avoid overflow + sum_sq = tl.zeros([1], dtype=tl.float32) + for col_offset in range(0, n_cols, BLOCK_SIZE): + col_idx = col_offset + tl.arange(0, BLOCK_SIZE) + mask = col_idx < n_cols + + vals = tl.load(row_start_ptr + col_idx, mask=mask, other=0.0) + # Convert to float32 for accumulation to prevent overflow + vals_f32 = vals.to(tl.float32) + sq_vals = vals_f32 * vals_f32 + sum_sq += tl.sum(tl.where(mask, sq_vals, 0.0)) + + # Step 2: Compute RMS (root mean square) in float32 + mean_sq = sum_sq / n_cols + rms = tl.sqrt(mean_sq + eps) + inv_rms = 1.0 / rms + + # Step 3: Normalize and apply weight + for col_offset in range(0, n_cols, BLOCK_SIZE): + col_idx = col_offset + tl.arange(0, BLOCK_SIZE) + mask = col_idx < n_cols + vals = tl.load(row_start_ptr + col_idx, mask=mask, other=0.0) + weight = tl.load(weight_ptr + col_idx, mask=mask, other=1.0) + # Compute in float32 then convert back to input dtype + vals_f32 = vals.to(tl.float32) + weight_f32 = weight.to(tl.float32) + output_f32 = vals_f32 * inv_rms * weight_f32 + output = output_f32.to(vals.dtype) + tl.store(output_row_start_ptr + col_idx, output, mask=mask) + + +def rms_norm( + input: torch.Tensor, weight: torch.Tensor, eps: float = 1e-6 +) -> torch.Tensor: + """ + Compute RMS normalization using Triton kernel. + + RMS Norm normalizes the input by the root mean square and scales by weight: + output = input / sqrt(mean(input^2) + eps) * weight + + Args: + input: Input tensor of shape (..., hidden_size) + weight: Weight tensor of shape (hidden_size,) + eps: Small constant for numerical stability + + Returns: + Tensor with RMS normalization applied along the last dimension + """ + assert weight.dim() == 1, "Weight must be 1-dimensional" + assert input.shape[-1] == weight.shape[0], ( + f"Input last dimension ({input.shape[-1]}) must match " + f"weight dimension ({weight.shape[0]})" + ) + + # Flatten all dimensions except the last one + original_shape = input.shape + input_2d = input.reshape(-1, input.shape[-1]) + input_2d = input_2d.contiguous() + weight = weight.contiguous() + + n_rows, n_cols = input_2d.shape + + output = torch.empty_like(input_2d) + BLOCK_SIZE = 1024 + grid = (n_rows,) + _rms_norm_kernel[grid]( + input_2d, + weight, + output, + input_2d.stride(0), + output.stride(0), + n_cols, + eps, + BLOCK_SIZE=BLOCK_SIZE, + ) + return output.reshape(original_shape) + + +def rms_norm_batch_invariant( + input: torch.Tensor, weight: torch.Tensor, eps: float = 1e-6 +) -> torch.Tensor: + """ + Batch-invariant wrapper for RMS normalization. + + This function provides a deterministic, batch-invariant implementation + of RMS normalization for use with the batch_invariant mode. + + Adapted from @https://github.com/vllm-project/vllm/blob/66a168a197ba214a5b70a74fa2e713c9eeb3251a/vllm/model_executor/layers/batch_invariant.py#L649 + + Args: + input: Input tensor of shape (..., hidden_size) + weight: Weight tensor of shape (hidden_size,) + eps: Small constant for numerical stability + + Returns: + RMS normalized tensor + """ + return rms_norm(input, weight, eps=eps) + + +_batch_invariant_MODE = False +_batch_invariant_LIB = None +_original_torch_bmm = None + + +def is_batch_invariant_mode_enabled(): + return _batch_invariant_MODE + + +def enable_batch_invariant_mode( + enable_bmm: bool = True, +): + global _batch_invariant_MODE, _batch_invariant_LIB, _original_torch_bmm + if _batch_invariant_MODE: + return + + _batch_invariant_MODE = True + _batch_invariant_LIB = torch.library.Library("aten", "IMPL") + _batch_invariant_LIB.impl("aten::mm", mm_batch_invariant, "CUDA") + _batch_invariant_LIB.impl("aten::addmm", addmm_batch_invariant, "CUDA") + _batch_invariant_LIB.impl( + "aten::_log_softmax", _log_softmax_batch_invariant, "CUDA" + ) + _batch_invariant_LIB.impl("aten::mean.dim", mean_batch_invariant, "CUDA") + + if enable_bmm: + _batch_invariant_LIB.impl("aten::bmm", bmm_batch_invariant, "CUDA") + + # Also monkeypatch torch.bmm directly as a fallback + _original_torch_bmm = torch.bmm + torch.bmm = bmm_batch_invariant + + +def disable_batch_invariant_mode(): + global _batch_invariant_MODE, _batch_invariant_LIB, _original_torch_bmm + if _batch_invariant_LIB is not None: + _batch_invariant_LIB._destroy() + if _original_torch_bmm is not None: + torch.bmm = _original_torch_bmm + _original_torch_bmm = None + _batch_invariant_MODE = False + _batch_invariant_LIB = None + + +@contextlib.contextmanager +def set_batch_invariant_mode(enabled: bool = True): + global _batch_invariant_MODE, _batch_invariant_LIB + old_data = (_batch_invariant_MODE, _batch_invariant_LIB) + if enabled: + enable_batch_invariant_mode() + else: + disable_batch_invariant_mode() + yield + if _batch_invariant_LIB is not None: + _batch_invariant_LIB._destroy() + _batch_invariant_MODE, _batch_invariant_LIB = old_data + + +AttentionBlockSize = namedtuple("AttentionBlockSize", ["block_m", "block_n"]) + + +def get_batch_invariant_attention_block_size() -> AttentionBlockSize: + return AttentionBlockSize(block_m=16, block_n=16) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_overlap/operations.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_overlap/operations.py new file mode 100644 index 00000000..9d824587 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_overlap/operations.py @@ -0,0 +1,211 @@ +from __future__ import annotations + +import os +from contextlib import contextmanager +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, List, Sequence, Union + +import torch + +from sglang.srt.layers.dp_attention import set_dp_buffer_len + +if TYPE_CHECKING: + from sglang.srt.model_executor.forward_batch_info import ForwardBatch + +_ENABLE_PROFILE = bool(int(os.environ.get("SGLANG_OPERATIONS_ENABLE_PROFILE", "0"))) + +if _ENABLE_PROFILE: + import nvtx + + +def execute_operations(inputs, operations): + stages = _convert_operations_to_stages(operations) + executor = _StageExecutor("primary", stages, inputs=inputs) + for _ in range(executor.num_stages): + executor.next() + assert executor.done + return executor.output + + +def execute_overlapped_operations( + inputs_arr: Sequence, + operations_arr: Sequence, + delta_stages: Sequence[int], +) -> Sequence: + # Make it explicit for clarity; if we need multi-batch overlap, this can be generalized + inputs_a, inputs_b = inputs_arr + operations_a, operations_b = operations_arr + delta_stage_a, delta_stage_b = delta_stages + assert delta_stage_a == 0 + delta_stage = delta_stage_b + + stages_a = _convert_operations_to_stages(operations_a) + stages_b = _convert_operations_to_stages(operations_b) + executor_a = _StageExecutor("a", stages_a, inputs=inputs_a) + executor_b = _StageExecutor("b", stages_b, inputs=inputs_b) + + for _ in range(delta_stage): + executor_a.next() + + for _ in range(executor_a.num_stages - delta_stage): + executor_a.next() + executor_b.next() + + for _ in range(delta_stage): + executor_b.next() + + assert executor_a.done and executor_b.done + return [executor_a.output, executor_b.output] + + +class YieldOperation: + pass + + +@dataclass +class ExecutionOperation: + debug_name: str + fn: Callable + + +Operation = Union[YieldOperation, ExecutionOperation, Callable] +Stage = List[ExecutionOperation] + + +class _StageExecutor: + def __init__(self, debug_name: str, stages: List[Stage], inputs: dict): + self._debug_name = debug_name + self._stages = stages + self._index = 0 + self._stage_state = _StateDict() + self._stage_output = inputs + + # handling DP attention + forward_batch: ForwardBatch = inputs["forward_batch"] + self._global_dp_buffer_len = forward_batch.global_dp_buffer_len + self._local_dp_buffer_len = forward_batch.input_ids.shape[0] + self._global_num_tokens = forward_batch.global_num_tokens_cpu + self._is_dp_max_padding = forward_batch.dp_padding_mode.is_max_len() + + def next(self): + assert not self.done + + stage = self._stages[self._index] + + if self._global_dp_buffer_len is not None: + set_dp_buffer_len( + self._global_dp_buffer_len, + self._local_dp_buffer_len, + self._is_dp_max_padding, + self._global_num_tokens, + ) + + with _annotate_region(debug_name=f"{self._debug_name}{self._index}"): + for op in stage: + with _annotate_region(debug_name=op.debug_name): + self._stage_output = op.fn( + state=self._stage_state, + **( + self._stage_output if self._stage_output is not None else {} + ), + ) + + self._index += 1 + + @property + def output(self): + assert self.done + return self._stage_output + + @property + def done(self): + return self._index >= self.num_stages + + @property + def num_stages(self): + return len(self._stages) + + +@contextmanager +def _annotate_region(debug_name): + if _ENABLE_PROFILE: + with torch.autograd.profiler.record_function(debug_name): + with nvtx.annotate(debug_name): + yield + else: + yield + + +class _StateDict: + def __init__(self): + self._data = {} + + def __setattr__(self, key, value): + if key == "_data": + super().__setattr__(key, value) + return + assert ( + key not in self._data + ), f"`{key}` already exist, are you sure you want to override it?" + self._data[key] = value + + def __getattr__(self, item): + return self._data[item] + + def __delattr__(self, item): + del self._data[item] + + def pop(self, item): + return self._data.pop(item) + + def update(self, values: Dict[str, Any]): + for k, v in values.items(): + setattr(self, k, v) + + def get(self, item): + return self._data.get(item) + + def clear(self, expect_keys: Sequence[str]): + if set(self._data.keys()) != set(expect_keys): + raise Exception( + f"Unexpected keys when clearning. This may indicate you do not release memory early enough but leave it to here. {list(self._data.keys())=} {expect_keys=}" + ) + + self._data.clear() + + +def _convert_operations_to_stages(operations: List[Operation]) -> List[Stage]: + operations = _decorate_operations(operations) + operation_chunks = list( + _chunk_by_separator(operations, lambda op: isinstance(op, YieldOperation)) + ) + assert all(len(chunk) > 0 for chunk in operation_chunks) + return operation_chunks + + +def _chunk_by_separator( + items: List[Any], is_separator: Callable[[Any], bool] +) -> Generator[List[Any], None, None]: + pending_items = [] + for item in items: + if is_separator(item): + yield pending_items + pending_items = [] + else: + pending_items.append(item) + if len(pending_items) > 0: + yield pending_items + + +def _decorate_operations(operations: List[Operation], debug_name_prefix: str = ""): + return [_decorate_operation(op, debug_name_prefix) for op in operations] + + +def _decorate_operation(operation: Operation, debug_name_prefix: str): + if isinstance(operation, YieldOperation): + return operation + return ExecutionOperation( + debug_name=debug_name_prefix + + getattr(operation, "__name__", "unknown").replace("op_", ""), + fn=operation, + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_overlap/operations_strategy.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_overlap/operations_strategy.py new file mode 100644 index 00000000..152e4874 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_overlap/operations_strategy.py @@ -0,0 +1,211 @@ +from dataclasses import dataclass +from typing import List, Optional + +import torch + +from sglang.srt.batch_overlap import operations +from sglang.srt.batch_overlap.operations import Operation +from sglang.srt.layers.moe.token_dispatcher import DeepEPConfig +from sglang.srt.model_executor.forward_batch_info import ForwardMode + + +@dataclass +class OperationsStrategy: + operations: List[Operation] + deep_gemm_num_sms: Optional[int] = None + tbo_delta_stages: Optional[int] = None + + @classmethod + def concat(cls, items: List["OperationsStrategy"]) -> "OperationsStrategy": + return OperationsStrategy( + operations=[x for item in items for x in item.operations], + deep_gemm_num_sms=_assert_all_same( + [item.deep_gemm_num_sms for item in items] + ), + tbo_delta_stages=_assert_all_same( + [item.tbo_delta_stages for item in items] + ), + ) + + @staticmethod + def init_new_tbo( + layers: torch.nn.ModuleList, + forward_mode: ForwardMode, + ) -> "OperationsStrategy": + layer_name = layers[0].__class__.__name__ + if layer_name == "DeepseekV2DecoderLayer": + return OperationsStrategy.concat( + [ + _compute_moe_deepseek_layer_operations_strategy_tbo( + layer, forward_mode + ) + for layer in layers + ] + ) + elif layer_name == "Qwen3MoeDecoderLayer": + return OperationsStrategy.concat( + [ + _compute_moe_qwen3_layer_operations_strategy_tbo( + layer, forward_mode + ) + for layer in layers + ] + ) + else: + raise NotImplementedError + + +def _assert_all_same(items: List): + assert all(item == items[0] for item in items) + return items[0] + + +# -------------------------------- Strategy for DeepSeek --------------------------------------- + + +# TODO can refactor to make it more fancy if we have more complex strategies +def _compute_moe_deepseek_layer_operations_strategy_tbo( + layer: torch.nn.Module, + forward_mode: ForwardMode, +) -> OperationsStrategy: + assert layer.is_layer_sparse, "dense layer TBO not yet implemented" + if forward_mode == ForwardMode.EXTEND: + return _compute_moe_deepseek_blog_prefill(layer) + elif ( + forward_mode == ForwardMode.DECODE or forward_mode == ForwardMode.TARGET_VERIFY + ): + return _compute_moe_deepseek_blog_decode(layer) + else: + raise NotImplementedError(f"Unsupported {forward_mode=}") + + +def _compute_moe_deepseek_blog_prefill(layer): + device_properties = torch.cuda.get_device_properties(device="cuda") + total_num_sms = device_properties.multi_processor_count + deep_gemm_num_sms = total_num_sms - DeepEPConfig.get_instance().num_sms + + return OperationsStrategy( + deep_gemm_num_sms=deep_gemm_num_sms, + tbo_delta_stages=0, + operations=[ + layer.op_comm_prepare_attn, + layer.self_attn.op_prepare, + layer.self_attn.op_core, + layer.op_comm_prepare_mlp, + layer.mlp.op_gate, + layer.mlp.op_select_experts, + layer.mlp.op_dispatch_a, + operations.YieldOperation(), + layer.mlp.op_dispatch_b, + layer.mlp.op_experts, + layer.mlp.op_combine_a, + operations.YieldOperation(), + layer.mlp.op_shared_experts, + layer.mlp.op_combine_b, + layer.mlp.op_output, + layer.op_comm_postprocess_layer, + ], + ) + + +def _compute_moe_deepseek_blog_decode(layer): + return OperationsStrategy( + deep_gemm_num_sms=None, + tbo_delta_stages=2, + operations=[ + layer.op_comm_prepare_attn, + layer.self_attn.op_prepare, + operations.YieldOperation(), + layer.self_attn.op_core, + layer.op_comm_prepare_mlp, + layer.mlp.op_gate, + layer.mlp.op_select_experts, + operations.YieldOperation(), + layer.mlp.op_dispatch_a, + layer.mlp.op_shared_experts, + operations.YieldOperation(), + layer.mlp.op_dispatch_b, + layer.mlp.op_experts, + layer.mlp.op_combine_a, + operations.YieldOperation(), + layer.mlp.op_combine_b, + operations.YieldOperation(), + layer.mlp.op_output, + layer.op_comm_postprocess_layer, + ], + ) + + +# -------------------------------- Strategy for Qwen3 --------------------------------------- + + +# TODO: unstable, current strategy is almost the same as DeepSeek, keep redundant code here for +# convenience to adjust strategy +def _compute_moe_qwen3_layer_operations_strategy_tbo( + layer: torch.nn.Module, + forward_mode: ForwardMode, +) -> OperationsStrategy: + assert layer.is_layer_sparse, "qwen3 moe only support sparse layers" + if forward_mode == ForwardMode.EXTEND: + return _compute_moe_qwen3_prefill(layer) + elif ( + forward_mode == ForwardMode.DECODE or forward_mode == ForwardMode.TARGET_VERIFY + ): + return _compute_moe_qwen3_decode(layer) + else: + raise NotImplementedError(f"Unsupported {forward_mode=}") + + +def _compute_moe_qwen3_prefill(layer): + device_properties = torch.cuda.get_device_properties(device="cuda") + total_num_sms = device_properties.multi_processor_count + deep_gemm_num_sms = total_num_sms - DeepEPConfig.get_instance().num_sms + + return OperationsStrategy( + deep_gemm_num_sms=deep_gemm_num_sms, + tbo_delta_stages=0, + operations=[ + layer.op_comm_prepare_attn, + layer.self_attn.op_prepare, + layer.self_attn.op_core, + layer.op_comm_prepare_mlp, + layer.mlp.op_gate, + layer.mlp.op_select_experts, + layer.mlp.op_dispatch_a, + operations.YieldOperation(), + layer.mlp.op_dispatch_b, + layer.mlp.op_experts, + layer.mlp.op_combine_a, + operations.YieldOperation(), + layer.mlp.op_combine_b, + layer.mlp.op_output, + layer.op_comm_postprocess_layer, + ], + ) + + +def _compute_moe_qwen3_decode(layer): + return OperationsStrategy( + deep_gemm_num_sms=None, + tbo_delta_stages=2, + operations=[ + layer.op_comm_prepare_attn, + layer.self_attn.op_prepare, + operations.YieldOperation(), + layer.self_attn.op_core, + layer.op_comm_prepare_mlp, + layer.mlp.op_gate, + layer.mlp.op_select_experts, + operations.YieldOperation(), + layer.mlp.op_dispatch_a, + operations.YieldOperation(), + layer.mlp.op_dispatch_b, + layer.mlp.op_experts, + layer.mlp.op_combine_a, + operations.YieldOperation(), + layer.mlp.op_combine_b, + layer.mlp.op_output, + layer.op_comm_postprocess_layer, + operations.YieldOperation(), + ], + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_overlap/single_batch_overlap.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_overlap/single_batch_overlap.py new file mode 100644 index 00000000..7b324c37 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_overlap/single_batch_overlap.py @@ -0,0 +1,116 @@ +# Copyright 2025 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +import torch + +from sglang.srt.layers.moe import get_moe_runner_backend +from sglang.srt.layers.moe.utils import is_sbo_enabled +from sglang.srt.utils import get_int_env_var + + +class SboFlags: + # TODO may have: "enable_dispatch_shared_one_stream_overlap", "enable_dispatch_gateup_gemm_two_stream_overlap", ... + + @classmethod + def enable_combine_down_gemm_two_stream_overlap(cls): + return ( + is_sbo_enabled() + # currently only cutedsl backend supports it + and get_moe_runner_backend().is_flashinfer_cutedsl() + ) + + @classmethod + def enable_combine_shared_two_stream_overlap(cls): + return is_sbo_enabled() + + @classmethod + def fuse_shared_experts_inside_sbo(cls): + # TODO after antgroup's PR, should be `... or cls.enable_dispatch_shared_one_stream_overlap()` + return cls.enable_combine_shared_two_stream_overlap() + + +@dataclass +class CombineOverlapArgs: + # this "overlap" flag means overlapping with down gemm, not the general two-stream overlap + overlap: bool + stream: torch.cuda.Stream + wait_event: torch.cuda.Event + num_sms: int + signal: Optional[torch.Tensor] = None + threshold: int = 0 + + +@dataclass +class DownGemmOverlapArgs: + num_sms: int + signal: torch.Tensor + start_event: torch.cuda.Event + + +def compute_overlap_args(dispatch_output, alt_stream): + if not ( + SboFlags.enable_combine_down_gemm_two_stream_overlap() + or SboFlags.enable_combine_shared_two_stream_overlap() + ): + return None, None, {} + + hidden_states = dispatch_output.hidden_states + + num_local_experts, num_tokens_static, hidden_dim = hidden_states.shape + + total_num_sms = torch.cuda.get_device_properties( + device="cuda" + ).multi_processor_count + communicate_num_sms = get_int_env_var("SGLANG_DEEPEP_LL_COMBINE_SEND_NUM_SMS", 32) + compute_num_sms = total_num_sms - communicate_num_sms + + assert alt_stream is not None + combine_wait_event = torch.cuda.Event() + combine_overlap_args = CombineOverlapArgs( + overlap=False, + num_sms=communicate_num_sms, + stream=alt_stream, + wait_event=combine_wait_event, + ) + meta_overlap_args = dict( + compute_num_sms=compute_num_sms, + ) + down_gemm_overlap_args = None + + if SboFlags.enable_combine_down_gemm_two_stream_overlap(): + # TODO use zero_allocator to remove this `torch.zeros` call + # NOTE ours v2 use uint32 not int32 currently + combine_signal = torch.zeros( + num_local_experts, dtype=torch.uint32, device=hidden_states.device + ) + + down_gemm_overlap_args = DownGemmOverlapArgs( + signal=combine_signal, + start_event=combine_wait_event, + num_sms=compute_num_sms, + ) + combine_overlap_args.overlap = True + combine_overlap_args.signal = combine_signal + combine_overlap_args.threshold = compute_num_sms + else: + meta_overlap_args |= dict( + record_event_after_down=combine_wait_event, + ) + + return combine_overlap_args, down_gemm_overlap_args, meta_overlap_args diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_overlap/two_batch_overlap.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_overlap/two_batch_overlap.py new file mode 100644 index 00000000..becb54e4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/batch_overlap/two_batch_overlap.py @@ -0,0 +1,1027 @@ +from __future__ import annotations + +import copy +import dataclasses +import logging +from dataclasses import replace +from typing import TYPE_CHECKING, Dict, List, Optional, Sequence + +import torch + +from sglang.srt.batch_overlap.operations import ( + execute_operations, + execute_overlapped_operations, +) +from sglang.srt.batch_overlap.operations_strategy import OperationsStrategy +from sglang.srt.layers import deep_gemm_wrapper +from sglang.srt.layers.attention.base_attn_backend import AttentionBackend +from sglang.srt.layers.communicator import ( + CommunicateContext, + CommunicateSummableTensorPairFn, + ScatterMode, +) +from sglang.srt.layers.moe import ( + get_deepep_mode, + get_moe_a2a_backend, + get_tbo_token_distribution_threshold, + is_tbo_enabled, +) +from sglang.srt.layers.moe.token_dispatcher import ( + DeepEPDispatcher, + MooncakeEPDispatcher, +) +from sglang.srt.layers.moe.token_dispatcher.base import BaseDispatcher +from sglang.srt.managers.schedule_batch import ScheduleBatch +from sglang.srt.model_executor.forward_batch_info import ( + ForwardBatch, + ForwardMode, + compute_position, +) +from sglang.srt.server_args import get_global_server_args +from sglang.srt.speculative.spec_info import SpecInput +from sglang.srt.utils import BumpAllocator, empty_context, get_bool_env_var, is_hip + +if TYPE_CHECKING: + from sglang.srt.batch_overlap.single_batch_overlap import CombineOverlapArgs + from sglang.srt.layers.moe.token_dispatcher import DispatchOutput + from sglang.srt.speculative.eagle_info import EagleVerifyInput + +_is_hip = is_hip() + +_tbo_debug = get_bool_env_var("SGLANG_TBO_DEBUG") + +logger = logging.getLogger(__name__) + + +# -------------------------------- Compute Basic Info --------------------------------------- + + +def get_token_num_per_seq( + forward_mode: ForwardMode, + spec_info: Optional[SpecInput] = None, +): + if forward_mode.is_target_verify(): + return spec_info.draft_token_num + elif forward_mode.is_decode(): + return 1 + elif forward_mode.is_idle(): + return 0 + else: + # For extend, we should not use `token_num_per_seq`. + return None + + +# TODO: may smartly disable TBO when batch size is too small b/c it will slow down +def compute_split_seq_index( + forward_mode: ForwardMode, + num_tokens: int, + extend_lens: Optional[Sequence[int]], + token_num_per_seq: Optional[int], +) -> Optional[int]: + if forward_mode == ForwardMode.EXTEND: + assert extend_lens is not None + return _split_extend_seqs(extend_lens) + elif forward_mode.is_target_verify() or forward_mode.is_decode(): + assert token_num_per_seq is not None + return (num_tokens // token_num_per_seq) // 2 + elif forward_mode.is_idle() or forward_mode.is_prebuilt(): + assert num_tokens == 0 + return 0 + else: + raise NotImplementedError() + + +def _is_two_chunk_split_enabled(extend_lens: Sequence[int]) -> bool: + if extend_lens is None: + return False + + vanilla_split_seq_index = _split_array_by_balanced_sum(extend_lens) + left_sum = sum(extend_lens[:vanilla_split_seq_index]) + overall_sum = sum(extend_lens) + threshold = get_tbo_token_distribution_threshold() + assert threshold <= 0.5, f"{threshold=}" + return left_sum < overall_sum * threshold or left_sum > overall_sum * ( + 1 - threshold + ) + + +def _split_extend_seqs(arr: Sequence[int]) -> int: + if _is_two_chunk_split_enabled(arr): + return _split_array_by_cum_less_than_half(arr) + + return _split_array_by_balanced_sum(arr) + + +def _split_array_by_cum_less_than_half(arr: Sequence[int]) -> int: + left_sum = 0 + overall_sum = sum(arr) + half_sum = overall_sum // 2 + chosen_index = 0 + + for i in range(len(arr)): + left_sum += arr[i] + if left_sum > half_sum: + chosen_index = i + break + + return chosen_index + + +def _split_array_by_balanced_sum(arr: Sequence[int]) -> int: + overall_sum = sum(arr) + left_sum = 0 + min_diff = float("inf") + best_index = 0 + + for i in range(1, len(arr)): + left_sum += arr[i - 1] + right_sum = overall_sum - left_sum + diff = abs(left_sum - right_sum) + if diff <= min_diff: + min_diff = diff + best_index = i + else: + break + + return best_index + + +def _update_device_and_sum_field_from_cpu_field( + batch: ForwardBatch, cpu_field: str, device_field: str, sum_field: str = None +): + cpu_value = getattr(batch, cpu_field, None) + old_device_value = getattr(batch, device_field, None) + if ( + cpu_value is None + or old_device_value is None + or not (isinstance(cpu_value, torch.Tensor) or isinstance(cpu_value, list)) + ): + return + + new_device_value = ( + cpu_value + if isinstance(cpu_value, torch.Tensor) + else torch.tensor(cpu_value, dtype=old_device_value.dtype) + ).to(device=get_global_server_args().device, non_blocking=True) + setattr(batch, device_field, new_device_value) + + if sum_field is not None: + sum_value = ( + cpu_value.sum().item() + if isinstance(cpu_value, torch.Tensor) + else sum(cpu_value) + ) + setattr(batch, sum_field, sum_value) + + +def _compute_mask_offset(seq_index: int, spec_info: Optional[EagleVerifyInput]) -> int: + if seq_index == 0: + return 0 + + offset = 0 + max_seq_len = min(seq_index, spec_info.seq_lens_cpu.shape[0]) + for i in range(max_seq_len): + offset += ( + spec_info.seq_lens_cpu[i] + spec_info.draft_token_num + ) * spec_info.draft_token_num + return offset + + +def split_spec_info( + spec_info: Optional[EagleVerifyInput], + start_seq_index: int, + end_seq_index: int, + start_token_index: int, + end_token_index: int, +): + if spec_info is None: + return None + if spec_info.draft_token is not None: + draft_token = spec_info.draft_token[start_token_index:end_token_index] + else: + draft_token = None + if spec_info.custom_mask is not None and spec_info.draft_token is not None: + custom_mask_start = _compute_mask_offset(start_seq_index, spec_info) + if end_seq_index == spec_info.seq_lens_cpu.shape[0]: + custom_mask_end = spec_info.custom_mask.shape[0] + else: + custom_mask_end = _compute_mask_offset(end_seq_index, spec_info) + + if custom_mask_end > custom_mask_start: + custom_mask = spec_info.custom_mask[custom_mask_start:custom_mask_end] + else: + custom_mask = spec_info.custom_mask + else: + custom_mask = spec_info.custom_mask + if spec_info.positions is not None: + positions = spec_info.positions[start_token_index:end_token_index] + else: + positions = None + if spec_info.retrive_index is not None: + retrive_index = spec_info.retrive_index[start_seq_index:end_seq_index] + else: + retrive_index = None + if spec_info.retrive_next_token is not None: + retrive_next_token = spec_info.retrive_next_token[start_seq_index:end_seq_index] + else: + retrive_next_token = None + if spec_info.retrive_next_sibling is not None: + retrive_next_sibling = spec_info.retrive_next_sibling[ + start_seq_index:end_seq_index + ] + else: + retrive_next_sibling = None + if spec_info.retrive_cum_len is not None: + retrive_cum_len = spec_info.retrive_cum_len[start_seq_index:end_seq_index] + else: + retrive_cum_len = None + + if spec_info.seq_lens_cpu is not None: + seq_lens_cpu = spec_info.seq_lens_cpu[start_seq_index:end_seq_index] + else: + seq_lens_cpu = None + if seq_lens_cpu is not None: + seq_lens_sum = seq_lens_cpu.sum() + else: + seq_lens_sum = None + output_spec_info = replace( + spec_info, + custom_mask=custom_mask, + draft_token=draft_token, + positions=positions, + retrive_index=retrive_index, + retrive_next_token=retrive_next_token, + retrive_next_sibling=retrive_next_sibling, + retrive_cum_len=retrive_cum_len, + seq_lens_cpu=seq_lens_cpu, + seq_lens_sum=seq_lens_sum, + ) + return output_spec_info + + +def compute_split_token_index( + split_seq_index: int, + forward_mode: "ForwardMode", + extend_seq_lens: Optional[Sequence[int]], + token_num_per_seq: Optional[int], +) -> int: + if forward_mode == ForwardMode.EXTEND: + assert extend_seq_lens is not None + if _is_two_chunk_split_enabled(extend_seq_lens): + return sum(extend_seq_lens) // 2 + return sum(extend_seq_lens[:split_seq_index]) + elif forward_mode.is_target_verify() or forward_mode.is_decode(): + assert token_num_per_seq is not None + return split_seq_index * token_num_per_seq + elif forward_mode.is_idle(): + assert split_seq_index == 0 + return 0 + else: + raise NotImplementedError + + +def compute_split_indices_for_cuda_graph_replay( + forward_mode: ForwardMode, + cuda_graph_num_tokens: int, + spec_info: Optional[SpecInput], +): + forward_mode_for_tbo_split = ( + forward_mode if forward_mode != ForwardMode.IDLE else ForwardMode.DECODE + ) + token_num_per_seq = get_token_num_per_seq( + forward_mode=forward_mode, spec_info=spec_info + ) + tbo_split_seq_index = compute_split_seq_index( + forward_mode=forward_mode_for_tbo_split, + num_tokens=cuda_graph_num_tokens, + extend_lens=None, + token_num_per_seq=token_num_per_seq, + ) + tbo_split_token_index = compute_split_token_index( + split_seq_index=tbo_split_seq_index, + forward_mode=forward_mode_for_tbo_split, + extend_seq_lens=None, + token_num_per_seq=token_num_per_seq, + ) + return tbo_split_seq_index, tbo_split_token_index + + +# -------------------------------- Preparation --------------------------------------- + + +class TboCudaGraphRunnerPlugin: + def __init__(self): + self._tbo_children_num_token_non_padded = torch.zeros((2,), dtype=torch.int32) + + def capture_one_batch_size(self, batch: ForwardBatch, num_tokens: int): + if not is_tbo_enabled(): + return + token_num_per_seq = get_token_num_per_seq( + forward_mode=batch.forward_mode, spec_info=batch.spec_info + ) + + batch.tbo_split_seq_index = compute_split_seq_index( + forward_mode=batch.forward_mode, + num_tokens=num_tokens, + extend_lens=None, + token_num_per_seq=token_num_per_seq, + ) + # For simplicity, when two_batch_overlap is enabled, we only capture CUDA Graph for tbo=true + assert batch.tbo_split_seq_index is not None, f"{num_tokens=}" + + self._tbo_children_num_token_non_padded[...] = ( + TboForwardBatchPreparer.compute_tbo_children_num_token_non_padded(batch) + ) + + TboForwardBatchPreparer.prepare_raw( + batch, + tbo_children_num_token_non_padded=self._tbo_children_num_token_non_padded, + ) + + def replay_prepare( + self, + forward_mode: ForwardMode, + bs: int, + num_token_non_padded: int, + spec_info: Optional[SpecInput], + ): + token_num_per_seq = get_token_num_per_seq( + forward_mode=forward_mode, spec_info=spec_info + ) + tbo_split_seq_index, tbo_split_token_index = ( + compute_split_indices_for_cuda_graph_replay( + forward_mode=forward_mode, + cuda_graph_num_tokens=bs * token_num_per_seq, + spec_info=spec_info, + ) + ) + + self._tbo_children_num_token_non_padded[...] = ( + TboForwardBatchPreparer.compute_tbo_children_num_token_non_padded_raw( + tbo_split_token_index=tbo_split_token_index, + num_token_non_padded=num_token_non_padded, + ) + ) + + +class TboDPAttentionPreparer: + def prepare_all_gather( + self, + local_batch: ScheduleBatch, + ): + + deepep_mode = get_deepep_mode() + enable_a2a_moe = not get_moe_a2a_backend().is_none() + enable_two_batch_overlap = is_tbo_enabled() + + self.enable_two_batch_overlap = enable_two_batch_overlap + + if local_batch is not None: + token_num_per_seq = get_token_num_per_seq( + forward_mode=local_batch.forward_mode, spec_info=local_batch.spec_info + ) + + if ( + local_batch.forward_mode.is_target_verify() + or local_batch.forward_mode.is_decode() + ): + num_tokens = local_batch.batch_size() * token_num_per_seq + elif local_batch.forward_mode.is_prebuilt(): + num_tokens = 0 + else: + num_tokens = local_batch.extend_num_tokens + self.local_tbo_split_seq_index = compute_split_seq_index( + forward_mode=local_batch.forward_mode, + num_tokens=num_tokens, + extend_lens=local_batch.extend_lens, + token_num_per_seq=token_num_per_seq, + ) + resolved_deepep_mode = deepep_mode.resolve(local_batch.is_extend_in_batch) + local_can_run_tbo = (self.local_tbo_split_seq_index is not None) and not ( + ( + local_batch.forward_mode.is_extend() + and not local_batch.forward_mode.is_target_verify() + ) + and enable_a2a_moe + and (resolved_deepep_mode.is_low_latency()) + ) + else: + self.local_tbo_split_seq_index = 0 + local_can_run_tbo = True + + local_forward_mode = self._compute_local_forward_mode(local_batch) + + return local_can_run_tbo, local_forward_mode + + def compute_output(self, partial_global_info): + local_can_run_tbo_aggregated = min(partial_global_info[:, 0].tolist()) + forward_modes = partial_global_info[:, 1].tolist() + + global_forward_mode, forward_mode_agree = self._compute_global_forward_mode( + forward_modes + ) + + can_run_tbo = ( + self.enable_two_batch_overlap + and local_can_run_tbo_aggregated + and forward_mode_agree + ) + + tbo_split_seq_index = self.local_tbo_split_seq_index if can_run_tbo else None + global_forward_mode = global_forward_mode if can_run_tbo else None + return tbo_split_seq_index, global_forward_mode + + @staticmethod + def _compute_local_forward_mode(local_batch): + return ( + local_batch.forward_mode if local_batch is not None else ForwardMode.IDLE + ).value + + @staticmethod + def _compute_global_forward_mode(forward_modes): + forward_modes_excluding_idle = [ + x for x in forward_modes if x != ForwardMode.IDLE.value + ] + + if not forward_modes_excluding_idle: + return ForwardMode.IDLE, False + + forward_mode_agree = TboDPAttentionPreparer._is_all_same( + forward_modes_excluding_idle + ) + global_forward_mode = ( + ForwardMode(forward_modes_excluding_idle[0]) if forward_mode_agree else None + ) + return global_forward_mode, forward_mode_agree + + @staticmethod + def _is_all_same(x): + return all(value == x[0] for value in x) + + +class TboForwardBatchPreparer: + @classmethod + def prepare(cls, batch: ForwardBatch, is_draft_worker: bool = False): + if batch.tbo_split_seq_index is None or is_draft_worker: + return + + tbo_children_num_token_non_padded = ( + cls.compute_tbo_children_num_token_non_padded(batch) + ) + cls.prepare_raw( + batch, tbo_children_num_token_non_padded=tbo_children_num_token_non_padded + ) + + @classmethod + def prepare_raw( + cls, batch: ForwardBatch, tbo_children_num_token_non_padded: torch.Tensor + ): + from sglang.srt.layers.attention.tbo_backend import TboAttnBackend + + tbo_split_token_index = cls._compute_split_token_index(batch) + + is_enable_two_chunk = ( + batch.forward_mode == ForwardMode.EXTEND + and _is_two_chunk_split_enabled(batch.extend_seq_lens_cpu) + ) + + if _tbo_debug: + logger.info( + f"TboForwardBatchPreparer.prepare " + f"is_enable_two_chunk={is_enable_two_chunk} " + f"tbo_split_seq_index={batch.tbo_split_seq_index} " + f"tbo_split_token_index={tbo_split_token_index} " + f"extend_seq_lens={batch.extend_seq_lens_cpu} " + f"bs={batch.batch_size} " + f"forward_mode={batch.forward_mode}" + ) + + assert isinstance(batch.attn_backend, TboAttnBackend) + attn_backend_child_a, attn_backend_child_b = batch.attn_backend.children + + [out_num_token_non_padded_a, out_num_token_non_padded_b] = ( + tbo_children_num_token_non_padded + ) + + child_a = cls.filter_batch( + batch, + start_token_index=0, + end_token_index=tbo_split_token_index, + start_seq_index=0, + end_seq_index=( + batch.tbo_split_seq_index + 1 + if is_enable_two_chunk + else batch.tbo_split_seq_index + ), + output_attn_backend=attn_backend_child_a, + out_num_token_non_padded=out_num_token_non_padded_a, + ) + child_b = cls.filter_batch( + batch, + start_token_index=tbo_split_token_index, + end_token_index=batch.input_ids.shape[0], + start_seq_index=batch.tbo_split_seq_index, + end_seq_index=batch.batch_size, + output_attn_backend=attn_backend_child_b, + out_num_token_non_padded=out_num_token_non_padded_b, + ) + + if is_enable_two_chunk: + cls.derive_fields_related_to_seq_len_for_two_chunk( + batch, + child_a=child_a, + child_b=child_b, + tbo_split_seq_index=batch.tbo_split_seq_index, + ) + + assert batch.tbo_children is None + batch.tbo_children = [child_a, child_b] + + @classmethod + def derive_fields_related_to_seq_len_for_two_chunk( + cls, + batch: ForwardBatch, + *, + child_a: ForwardBatch, + child_b: ForwardBatch, + tbo_split_seq_index: int, + ): + extend_seq_lens_cpu = batch.extend_seq_lens_cpu + overall_seq_lens_sum = sum(extend_seq_lens_cpu) + half_seq_lens_sum = overall_seq_lens_sum // 2 + left_last_seq_token_num = half_seq_lens_sum - sum( + extend_seq_lens_cpu[:tbo_split_seq_index] + ) + right_first_seq_token_num = ( + extend_seq_lens_cpu[tbo_split_seq_index] - left_last_seq_token_num + ) + + # making deepcopy to be extra safe + child_a.extend_seq_lens_cpu = copy.deepcopy(child_a.extend_seq_lens_cpu) + child_a.extend_seq_lens_cpu[-1] = left_last_seq_token_num + child_b.extend_seq_lens_cpu = copy.deepcopy(child_b.extend_seq_lens_cpu) + child_b.extend_seq_lens_cpu[0] = right_first_seq_token_num + for child in [child_a, child_b]: + _update_device_and_sum_field_from_cpu_field( + batch=child, + cpu_field="extend_seq_lens_cpu", + device_field="extend_seq_lens", + sum_field="extend_num_tokens", + ) + + assert ( + child_a.extend_num_tokens == half_seq_lens_sum + ), f"{child_a.extend_num_tokens=}, {half_seq_lens_sum=}" + + child_a.seq_lens_cpu = copy.deepcopy(child_a.seq_lens_cpu) + child_a.seq_lens_cpu[-1] = ( + child_a.extend_seq_lens_cpu[-1] + child_a.extend_prefix_lens_cpu[-1] + ) + _update_device_and_sum_field_from_cpu_field( + batch=child_a, + cpu_field="seq_lens_cpu", + device_field="seq_lens", + sum_field="seq_lens_sum", + ) + + child_b.extend_prefix_lens_cpu = copy.deepcopy(child_b.extend_prefix_lens_cpu) + child_b.extend_prefix_lens_cpu[0] += left_last_seq_token_num + _update_device_and_sum_field_from_cpu_field( + batch=child_b, + cpu_field="extend_prefix_lens_cpu", + device_field="extend_prefix_lens", + sum_field=None, + ) + _, child_b.extend_start_loc = compute_position( + get_global_server_args().attention_backend, + child_b.extend_prefix_lens, + child_b.extend_seq_lens, + child_b.extend_num_tokens, + ) + + @classmethod + def filter_batch( + cls, + batch: ForwardBatch, + *, + start_token_index: int, + end_token_index: int, + start_seq_index: int, + end_seq_index: int, + output_attn_backend: AttentionBackend, + out_num_token_non_padded: torch.Tensor, + ): + assert ( + end_token_index >= start_token_index + ), f"{end_token_index=}, {start_token_index=}, batch={batch}" + num_tokens = batch.input_ids.shape[0] + num_seqs = batch.batch_size + + output_dict = dict() + + for key in [ + "input_ids", + "positions", + "out_cache_loc", + ]: + old_value = getattr(batch, key) + assert ( + old_value.shape[0] == num_tokens + ), f"{key=} {old_value=} {num_tokens=} {batch=}" + output_dict[key] = old_value[start_token_index:end_token_index] + + for key in [ + "req_pool_indices", + "seq_lens", + "seq_lens_cpu", + "extend_seq_lens", + "extend_prefix_lens", + "extend_start_loc", + "extend_prefix_lens_cpu", + "extend_seq_lens_cpu", + "extend_logprob_start_lens_cpu", + "lora_ids", + ]: + old_value = getattr(batch, key) + if old_value is None: + continue + elif batch.forward_mode.is_target_verify() and ( + key == "extend_seq_lens" + or key == "extend_prefix_lens" + or key == "extend_start_loc" + or key == "extend_prefix_lens_cpu" + or key == "extend_seq_lens_cpu" + or key == "extend_logprob_start_lens_cpu" + ): + output_dict[key] = None + continue + assert ( + len(old_value) == num_seqs + ), f"{key=} {old_value=} {num_seqs=} {batch=}" + output_dict[key] = old_value[start_seq_index:end_seq_index] + + spec_info = getattr(batch, "spec_info") + output_spec_info = split_spec_info( + spec_info=spec_info, + start_token_index=start_token_index, + end_token_index=end_token_index, + start_seq_index=start_seq_index, + end_seq_index=end_seq_index, + ) + output_dict["spec_info"] = output_spec_info + for key in [ + "forward_mode", + "is_extend_in_batch", + "return_logprob", + "req_to_token_pool", + "token_to_kv_pool", + "can_run_dp_cuda_graph", + "dp_padding_mode", + "global_forward_mode", + "is_prefill_only", + "spec_algorithm", + "capture_hidden_mode", + "padded_static_len", + "mrope_positions", # only used by qwen2-vl, thus not care + "split_index", # for split prefill + "orig_seq_lens", # only used by qwen-1m, thus not care + ]: + output_dict[key] = getattr(batch, key) + if not batch.forward_mode.is_target_verify(): + assert ( + _compute_extend_num_tokens(batch.input_ids, batch.forward_mode) + == batch.extend_num_tokens + ), f"{batch=}" + extend_num_tokens = _compute_extend_num_tokens( + output_dict["input_ids"], output_dict["forward_mode"] + ) + + # TODO improve, e.g. unify w/ `init_raw` + if ( + get_global_server_args().moe_dense_tp_size == 1 + and batch.global_dp_buffer_len is not None + ): + sum_len = end_token_index - start_token_index + global_dp_buffer_len = sum_len + else: + global_dp_buffer_len = None + + output_dict.update( + dict( + batch_size=end_seq_index - start_seq_index, + seq_lens_sum=( + output_dict["seq_lens_cpu"].sum() + if "seq_lens_cpu" in output_dict + else None + ), + extend_num_tokens=extend_num_tokens, + attn_backend=output_attn_backend, + num_token_non_padded=out_num_token_non_padded, + # TODO: handle it when we need TBO + DeepSeek V3.2 + num_token_non_padded_cpu=None, + tbo_split_seq_index=None, + tbo_parent_token_range=(start_token_index, end_token_index), + tbo_children=None, + global_num_tokens_gpu=None, + global_num_tokens_cpu=None, + global_dp_buffer_len=global_dp_buffer_len, + global_num_tokens_for_logprob_gpu=None, + global_num_tokens_for_logprob_cpu=None, + sampling_info=None, + # For logits and logprobs post processing, thus we do not care + temp_scaled_logprobs=False, + temperature=None, + top_p_normalized_logprobs=False, + top_p=None, + mm_inputs=None, + top_logprobs_nums=None, + token_ids_logprobs=None, + next_token_logits_buffer=None, + ) + ) + + errors = [] + for field in dataclasses.fields(ForwardBatch): + if getattr(batch, field.name) is not None and field.name not in output_dict: + errors.append( + f"Field {field.name} has value, but is not yet supported (value={getattr(batch, field.name)} batch={batch})" + ) + if len(errors) > 0: + raise Exception(f"{len(errors)} errors happen:\n" + "\n\n".join(errors)) + + return ForwardBatch(**output_dict) + + @classmethod + def compute_tbo_children_num_token_non_padded(cls, batch: ForwardBatch): + return cls.compute_tbo_children_num_token_non_padded_raw( + tbo_split_token_index=cls._compute_split_token_index(batch), + num_token_non_padded=len(batch.input_ids), + ) + + @classmethod + def compute_tbo_children_num_token_non_padded_raw( + cls, tbo_split_token_index: int, num_token_non_padded: int + ): + # TODO we may make padding on both sub-batches to make it slightly more balanced + value_a = min(tbo_split_token_index, num_token_non_padded) + value_b = max(0, num_token_non_padded - tbo_split_token_index) + return torch.tensor([value_a, value_b], dtype=torch.int32).to( + device=get_global_server_args().device, non_blocking=True + ) + + @classmethod + def _compute_split_token_index(cls, batch: ForwardBatch): + token_num_per_seq = get_token_num_per_seq( + forward_mode=batch.forward_mode, spec_info=batch.spec_info + ) + return compute_split_token_index( + split_seq_index=batch.tbo_split_seq_index, + forward_mode=batch.forward_mode, + extend_seq_lens=batch.extend_seq_lens_cpu, + token_num_per_seq=token_num_per_seq, + ) + + +def _compute_extend_num_tokens(input_ids, forward_mode: ForwardMode): + if ( + forward_mode.is_decode() + or forward_mode.is_idle() + or forward_mode.is_target_verify() + ): + return None + elif forward_mode.is_extend(): + return input_ids.shape[0] + raise NotImplementedError + + +# -------------------------------- Execution --------------------------------------- + + +def model_forward_maybe_tbo( + layers, + enable_tbo: bool, + positions: torch.Tensor, + forward_batch: ForwardBatch, + hidden_states: torch.Tensor, + input_data_scatter_mode: ScatterMode, + residual: Optional[torch.Tensor], + zero_allocator: Optional[BumpAllocator] = None, +): + inputs = dict( + positions=positions, + hidden_states=hidden_states, + forward_batch=forward_batch, + residual=residual, + zero_allocator=zero_allocator, + ) + layer_input_scatter_mode = layers[0].layer_scatter_modes.layer_input_mode + operations_strategy = OperationsStrategy.init_new_tbo( + layers, forward_batch.global_forward_mode + ) + if enable_tbo: + return _model_forward_tbo( + inputs=inputs, + operations_strategy=operations_strategy, + input_data_scatter_mode=input_data_scatter_mode, + layer_input_scatter_mode=layer_input_scatter_mode, + ) + else: + return _model_forward_non_tbo(inputs, operations_strategy) + + +def _model_forward_tbo( + inputs, + operations_strategy: OperationsStrategy, + input_data_scatter_mode: ScatterMode, + layer_input_scatter_mode: ScatterMode, +): + inputs_arr = _model_forward_tbo_split_inputs( + **inputs, + input_data_scatter_mode=input_data_scatter_mode, + layer_input_scatter_mode=layer_input_scatter_mode, + ) + del inputs + + context = ( + empty_context() + if _is_hip + else deep_gemm_wrapper.configure_deep_gemm_num_sms( + operations_strategy.deep_gemm_num_sms + ) + ) + + with context: + outputs_arr = execute_overlapped_operations( + inputs_arr=inputs_arr, + operations_arr=[operations_strategy.operations] * 2, + delta_stages=[0, operations_strategy.tbo_delta_stages], + ) + + return _model_forward_tbo_merge_outputs(*outputs_arr) + + +def _model_forward_non_tbo(inputs, operations_strategy: OperationsStrategy): + outputs = execute_operations(inputs, operations_strategy.operations) + return outputs["hidden_states"], outputs["residual"] + + +def _model_forward_tbo_split_inputs( + hidden_states: torch.Tensor, + residual: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + zero_allocator: Optional[BumpAllocator], + input_data_scatter_mode: ScatterMode, + layer_input_scatter_mode: ScatterMode, +) -> List[Dict]: + tbo_splitter_scatter_mode = ScatterMode.TP_ATTN_FULL + context = CommunicateContext.init_new() + + hidden_states, residual = CommunicateSummableTensorPairFn.execute( + hidden_states_input_mode=input_data_scatter_mode, + residual_input_mode=input_data_scatter_mode, + output_mode=tbo_splitter_scatter_mode, + hidden_states=hidden_states, + residual=residual, + forward_batch=forward_batch, + context=context, + ) + + inputs_arr = _model_forward_tbo_split_inputs_raw( + hidden_states=hidden_states, + residual=residual, + positions=positions, + forward_batch=forward_batch, + zero_allocator=zero_allocator, + ) + + def _post_transform(hidden_states, residual, forward_batch, **kwargs): + hidden_states, residual = CommunicateSummableTensorPairFn.execute( + hidden_states_input_mode=tbo_splitter_scatter_mode, + residual_input_mode=tbo_splitter_scatter_mode, + output_mode=layer_input_scatter_mode, + hidden_states=hidden_states, + residual=residual, + forward_batch=forward_batch, + context=context, + ) + return dict( + hidden_states=hidden_states, + residual=residual, + forward_batch=forward_batch, + **kwargs, + ) + + return [_post_transform(**inputs) for inputs in inputs_arr] + + +def _model_forward_tbo_split_inputs_raw( + hidden_states: torch.Tensor, + residual: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + zero_allocator: Optional[BumpAllocator], +) -> List[Dict]: + return [ + dict( + **_model_forward_filter_inputs( + hidden_states=hidden_states, + residual=residual, + positions=positions, + output_forward_batch=output_forward_batch, + tbo_subbatch_index=tbo_subbatch_index, + ), + **( + dict(zero_allocator=zero_allocator) + if zero_allocator is not None + else {} + ), + ) + for tbo_subbatch_index, output_forward_batch in enumerate( + forward_batch.tbo_children + ) + ] + + +def _model_forward_filter_inputs( + hidden_states: torch.Tensor, + residual: torch.Tensor, + positions: torch.Tensor, + output_forward_batch: ForwardBatch, + tbo_subbatch_index: int, +) -> Dict: + token_slice = slice(*output_forward_batch.tbo_parent_token_range) + return dict( + hidden_states=hidden_states[token_slice], + residual=None if residual is None else residual[token_slice], + positions=positions[token_slice], + forward_batch=output_forward_batch, + tbo_subbatch_index=tbo_subbatch_index, + ) + + +def _model_forward_tbo_merge_outputs(output_a, output_b): + def _handle_key(name): + value_a = output_a[name] + value_b = output_b[name] + assert (value_a is None) == (value_b is None) + if value_a is None: + return None + return torch.concat([value_a, value_b], dim=0) + + return _handle_key("hidden_states"), _handle_key("residual") + + +# -------------------------------- Utilities and wrappers --------------------------------------- + + +class MaybeTboDeepEPDispatcher(BaseDispatcher): + def __init__(self, **kwargs): + super().__init__() + num_inner_dispatchers = 2 if is_tbo_enabled() else 1 + if get_moe_a2a_backend().is_deepep(): + self._inners = [ + DeepEPDispatcher(**kwargs) for _ in range(num_inner_dispatchers) + ] + elif get_moe_a2a_backend().is_mooncake(): + self._inners = [ + MooncakeEPDispatcher(**kwargs) for _ in range(num_inner_dispatchers) + ] + + def _execute(self, name, tbo_subbatch_index: Optional[int] = None, **kwargs): + return getattr(self._inners[tbo_subbatch_index or 0], name)(**kwargs) + + def dispatch(self, **kwargs) -> DispatchOutput: + return self._execute("dispatch", **kwargs) + + def dispatch_a(self, **kwargs): + return self._execute("dispatch_a", **kwargs) + + def dispatch_b(self, **kwargs): + return self._execute("dispatch_b", **kwargs) + + def combine(self, **kwargs) -> torch.Tensor: + return self._execute("combine", **kwargs) + + def combine_a(self, **kwargs): + return self._execute("combine_a", **kwargs) + + def combine_b(self, **kwargs): + return self._execute("combine_b", **kwargs) + + def set_quant_config(self, quant_config: dict): + super().set_quant_config(quant_config) + for inner in self._inners: + inner.set_quant_config(quant_config) + + def set_overlap_args( + self, combine_overlap_args: CombineOverlapArgs, meta_overlap_args: dict + ): + super().set_overlap_args(combine_overlap_args, meta_overlap_args) + for inner in self._inners: + inner.set_overlap_args(combine_overlap_args, meta_overlap_args) + + def clear_overlap_args(self): + super().clear_overlap_args() + for inner in self._inners: + inner.clear_overlap_args() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/checkpoint_engine/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/checkpoint_engine/__init__.py new file mode 100644 index 00000000..d8a77f90 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/checkpoint_engine/__init__.py @@ -0,0 +1,9 @@ +""" +Checkpoint engine module for SGLang. + +This module provides functionality for updating model weights via checkpoint engine. +""" + +from sglang.srt.checkpoint_engine.update import main + +__all__ = ["main"] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/checkpoint_engine/checkpoint_engine_worker.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/checkpoint_engine/checkpoint_engine_worker.py new file mode 100644 index 00000000..dd8805e6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/checkpoint_engine/checkpoint_engine_worker.py @@ -0,0 +1,142 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Checkpoint-engine integration for SGLang. +This module provides weight update functionality via IPC for checkpoint-engine compatibility. +""" +import logging +from typing import Callable, Dict, Optional + +import torch +import zmq + +try: + from checkpoint_engine.worker import update_weights_from_ipc +except ImportError: + raise ImportError( + "checkpoint-engine is not installed. " + "Please install it with: pip install sglang[checkpoint-engine]" + ) + +logger = logging.getLogger(__name__) + + +class SGLangCheckpointEngineWorkerExtension: + """ + Worker extension for SGLang to support checkpoint-engine IPC weight updates. + This class provides the interface needed for checkpoint-engine integration. + """ + + def __init__(self): + self._zmq_ctx: Optional[zmq.Context] = None + + def get_device_uuid(self) -> str: + """Get the UUID of current device.""" + # We need to implement this to get the device UUID + # This will be overridden when integrated into SGLang's worker + raise NotImplementedError( + "This method should be overridden by SGLang integration" + ) + + def get_device_id(self) -> int: + """Get the device ID.""" + raise NotImplementedError( + "This method should be overridden by SGLang integration" + ) + + def get_model_loader(self) -> Callable: + """Get the model weight loader function.""" + raise NotImplementedError( + "This method should be overridden by SGLang integration" + ) + + def get_post_hook(self) -> Optional[Callable]: + """Get the post-processing hook after weight loading.""" + return None + + def update_weights_from_ipc(self, zmq_handles: Dict[str, str]): + """ + Update weights from IPC communication. + Args: + zmq_handles: Dict mapping device UUID to ZMQ socket path + """ + if self._zmq_ctx is None: + self._zmq_ctx = zmq.Context() + device_uuid = self.get_device_uuid() + device_id = self.get_device_id() + if device_uuid not in zmq_handles: + raise ValueError( + f"Device UUID {device_uuid} not found in zmq_handles: {list(zmq_handles.keys())}" + ) + update_weights_from_ipc( + self._zmq_ctx, + zmq_handles[device_uuid], + device_id=device_id, + run=self.get_model_loader(), + post_hook=self.get_post_hook(), + ) + + +class SGLangCheckpointEngineWorkerExtensionImpl(SGLangCheckpointEngineWorkerExtension): + """ + Implementation of SGLangCheckpointEngineWorkerExtension that integrates with SGLang's model runner. + This class provides the concrete implementation for checkpoint-engine IPC weight updates. + """ + + def __init__(self, model_runner): + super().__init__() + self.model_runner = model_runner + + def get_device_uuid(self) -> str: + """Get the UUID of current device.""" + # Get device UUID for current device + device_id = torch.cuda.current_device() + try: + return f"GPU-{torch.cuda.get_device_properties(device_id).uuid!s}" + except AssertionError as e: + raise ValueError(f"Failed to get GPU UUID for device {device_id}") from e + + def get_device_id(self) -> int: + """Get the device ID.""" + return torch.cuda.current_device() + + def get_model_loader(self) -> Callable: + """Get the model weight loader function.""" + return self.model_runner.model.load_weights + + def get_post_hook(self) -> Optional[Callable]: + """Get the post-processing hook after weight loading.""" + + def post_hook(): + # Perform post-processing after weight loading similar to DefaultModelLoader + try: + from sglang.srt.model_loader.loader import device_loading_context + + # Process quantization methods after loading weights + for _, module in self.model_runner.model.named_modules(): + quant_method = getattr(module, "quant_method", None) + if quant_method is not None: + # Move parameters to device if needed for quantization processing + target_device = torch.device( + "cuda", torch.cuda.current_device() + ) + with device_loading_context(module, target_device): + quant_method.process_weights_after_loading(module) + # Call model-specific post-loading hook if available + if hasattr(self.model_runner.model, "post_load_weights"): + self.model_runner.model.post_load_weights() + except Exception as e: + logger.warning(f"Post-hook processing failed: {e}") + + return post_hook diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/checkpoint_engine/update.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/checkpoint_engine/update.py new file mode 100644 index 00000000..93c8b4b6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/checkpoint_engine/update.py @@ -0,0 +1,317 @@ +""" +Usage: +1) Launch the server with wait-for-initial-weights option in one terminal: + python -m sglang.launch_server --model-path /workspace/Qwen/Qwen3-4B/ --tensor-parallel-size 2 --port 19730 --load-format dummy --checkpoint-engine-wait-weights-before-ready --mem-fraction-static 0.7 + +2) Torchrun this script in another terminal: + torchrun --nproc-per-node 2 update.py --update-method broadcast --checkpoint-path /workspace/Qwen/Qwen3-4B/ --inference-parallel-size 2 + +Or use the integrated entry point: + python -m sglang.srt.checkpoint_engine.update --update-method broadcast --checkpoint-path /workspace/Qwen/Qwen3-4B/ --inference-parallel-size 2 +""" + +import argparse +import json +import os +import pickle +import subprocess +import sys +import time +from collections import defaultdict +from collections.abc import Callable +from contextlib import contextmanager +from typing import Literal + +import httpx +import torch +import torch.distributed as dist +from safetensors import safe_open + +try: + from checkpoint_engine.ps import ParameterServer + from loguru import logger +except ImportError: + # Fallback for when checkpoint_engine is not available + ParameterServer = None + import logging + + logger = logging.getLogger(__name__) + + +@contextmanager +def timer(msg: str): + start = time.perf_counter() + yield + end = time.perf_counter() + logger.info(f"{msg} duration: {end - start:.2f} seconds") + + +def check_sglang_ready( + endpoint: str, inference_parallel_size: int, uds: str | None = None +): + rank = int(os.getenv("RANK", 0)) + if rank != rank // inference_parallel_size * inference_parallel_size: + return + retry_num = 0 + transport = None + if uds is not None: + transport = httpx.HTTPTransport(uds=uds) + with httpx.Client(transport=transport) as client: + while True: + try: + response = client.get(f"{endpoint}/ping", timeout=10) + response.raise_for_status() + break + except (httpx.ConnectError, httpx.HTTPStatusError) as e: + if retry_num % 10 == 0: + logger.warning( + f"fail to check sglang ready, retry {retry_num} times, error: {e}" + ) + retry_num += 1 + time.sleep(0.1) + + +def split_checkpoint_files( + checkpoint_path: str, rank: int, world_size: int +) -> list[str]: + checkpoint_files = [ + os.path.join(checkpoint_path, f) + for f in filter( + lambda x: x.endswith(".safetensors"), os.listdir(checkpoint_path) + ) + ] + files_per_rank = (len(checkpoint_files) + world_size - 1) // world_size + return checkpoint_files[rank * files_per_rank : (rank + 1) * files_per_rank] + + +def split_tensors( + checkpoint_path: str, rank: int, world_size: int +) -> dict[str, torch.Tensor]: + index_fn = os.path.join(checkpoint_path, "model.safetensors.index.json") + with open(index_fn) as f: + weight_map: dict[str, str] = json.load(f)["weight_map"] + weights_per_rank = (len(weight_map) + world_size - 1) // world_size + fn_tensors: dict[str, list[str]] = defaultdict(list) + weight_keys = list(weight_map.items()) + for name, file in weight_keys[ + rank * weights_per_rank : (rank + 1) * weights_per_rank + ]: + fn_tensors[file].append(name) + named_tensors = {} + for file, names in fn_tensors.items(): + with safe_open(os.path.join(checkpoint_path, file), framework="pt") as f: + for name in names: + named_tensors[name] = f.get_tensor(name) + return named_tensors + + +def req_inference( + endpoint: str, + inference_parallel_size: int, + timeout: float = 300.0, + uds: str | None = None, + weight_version: str | None = None, +) -> Callable[[list[tuple[str, str]]], None]: + rank = int(os.getenv("RANK", 0)) + src = rank // inference_parallel_size * inference_parallel_size + + def req_func(socket_paths: list[tuple[str, str]]): + if rank == src: + with httpx.Client(transport=httpx.HTTPTransport(uds=uds)) as client: + resp = client.post( + f"{endpoint}/update_weights_from_ipc", + json={ + "zmq_handles": dict( + socket_paths[src : src + inference_parallel_size] + ), + "flush_cache": True, + "weight_version": weight_version, + }, + timeout=timeout, + ) + resp.raise_for_status() + + return req_func + + +def update_weights( + ps, + checkpoint_name: str, + checkpoint_files: list[str], + named_tensors: dict[str, torch.Tensor], + req_func: Callable[[list[tuple[str, str]]], None], + inference_parallel_size: int, + endpoint: str, + save_metas_file: str | None = None, + update_method: Literal["broadcast", "p2p", "all"] = "broadcast", + uds: str | None = None, +): + ps.register_checkpoint( + checkpoint_name, files=checkpoint_files, named_tensors=named_tensors + ) + ps.init_process_group() + check_sglang_ready(endpoint, inference_parallel_size, uds) + dist.barrier() + with timer("Gather metas"): + ps.gather_metas(checkpoint_name) + if save_metas_file and int(os.getenv("RANK")) == 0: + with open(save_metas_file, "wb") as f: + pickle.dump(ps.get_metas(), f) + + if update_method == "broadcast" or update_method == "all": + with timer("Update weights without setting ranks"): + ps.update(checkpoint_name, req_func) + + if update_method == "p2p" or update_method == "all": + if update_method: + # sleep 2s to wait destroy process group + time.sleep(2) + with timer("Update weights with setting ranks"): + ps.update( + checkpoint_name, req_func, ranks=list(range(inference_parallel_size)) + ) + + +def join( + ps: ParameterServer, + checkpoint_name: str, + load_metas_file: str, + req_func: Callable[[list[tuple[str, str]]], None], + inference_parallel_size: int, + endpoint: str, + uds: str | None = None, +): + assert load_metas_file, "load_metas_file is required" + with open(load_metas_file, "rb") as f: + metas = pickle.load(f) + ps.init_process_group() + check_sglang_ready(endpoint, inference_parallel_size, uds) + dist.barrier() + with timer("Gather metas before join"): + ps.gather_metas(checkpoint_name) + ps.load_metas(metas) + with timer( + f"Update weights with setting ranks as range(0, {inference_parallel_size}) by using p2p" + ): + ps.update(checkpoint_name, req_func, ranks=list(range(inference_parallel_size))) + + +def run_with_torchrun(): + """Run the update script with torchrun automatically.""" + # Parse inference_parallel_size from command line arguments to determine nproc-per-node + inference_parallel_size = 8 # default + args = sys.argv[1:] # Skip the script name + + # Look for --inference-parallel-size in arguments + for i, arg in enumerate(args): + if arg == "--inference-parallel-size" and i + 1 < len(args): + try: + inference_parallel_size = int(args[i + 1]) + except ValueError: + pass + break + elif arg.startswith("--inference-parallel-size="): + try: + inference_parallel_size = int(arg.split("=", 1)[1]) + except ValueError: + pass + break + + # Build torchrun command + cmd = ["torchrun", f"--nproc-per-node={inference_parallel_size}", __file__] + args + + print(f"Running: {' '.join(cmd)}", file=sys.stderr) + + # Execute torchrun with the original script + try: + result = subprocess.run(cmd, check=False) + sys.exit(result.returncode) + except FileNotFoundError: + print( + "Error: torchrun command not found. Please ensure PyTorch is installed.", + file=sys.stderr, + ) + sys.exit(1) + except KeyboardInterrupt: + print("\nInterrupted by user", file=sys.stderr) + sys.exit(130) + + +def main(): + # Check if we're running under torchrun or need to invoke it + if os.getenv("RANK") is None: + # Not running under torchrun, so invoke it + run_with_torchrun() + return + + # Running under torchrun, proceed with normal execution + parser = argparse.ArgumentParser(description="Update weights example") + parser.add_argument("--checkpoint-path", type=str, default=None) + parser.add_argument("--save-metas-file", type=str, default=None) + parser.add_argument("--load-metas-file", type=str, default=None) + parser.add_argument("--sleep-time", type=int, default=0) + parser.add_argument("--endpoint", type=str, default="http://localhost:19730") + parser.add_argument("--inference-parallel-size", type=int, default=8) + parser.add_argument("--checkpoint-name", type=str, default="my-checkpoint-iter-0") + parser.add_argument("--update-method", type=str, default="broadcast") + parser.add_argument("--uds", type=str, default=None) + parser.add_argument("--weight-version", type=str, default=None) + args = parser.parse_args() + + # Get rank and world_size from environment (set by torchrun) + rank = int(os.getenv("RANK", 0)) + world_size = int(os.getenv("WORLD_SIZE", 1)) + + req_func = req_inference( + args.endpoint, + args.inference_parallel_size, + uds=args.uds, + weight_version=args.weight_version, + ) + + if ParameterServer is None: + print("Error: checkpoint_engine package not available", file=sys.stderr) + sys.exit(1) + + ps = ParameterServer(auto_pg=True) + ps._p2p_store = None + if args.load_metas_file: + join( + ps, + args.checkpoint_name, + args.load_metas_file, + req_func, + args.inference_parallel_size, + args.endpoint, + args.uds, + ) + else: + if args.checkpoint_path and os.path.exists( + os.path.join(args.checkpoint_path, "model.safetensors.index.json") + ): + named_tensors = split_tensors(args.checkpoint_path, rank, world_size) + checkpoint_files = [] + else: + checkpoint_files = ( + split_checkpoint_files(args.checkpoint_path, rank, world_size) + if args.checkpoint_path + else [] + ) + named_tensors = {} + update_weights( + ps, + args.checkpoint_name, + checkpoint_files, + named_tensors, + req_func, + args.inference_parallel_size, + args.endpoint, + args.save_metas_file, + args.update_method, + args.uds, + ) + time.sleep(args.sleep_time) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/backend.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/backend.py new file mode 100644 index 00000000..9892e472 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/backend.py @@ -0,0 +1,455 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.10.0/vllm/compilation/backend.py + + +import ast +import dataclasses +import logging +import os +import pprint +import time +from collections.abc import Sequence +from contextlib import contextmanager +from typing import Any, Callable, Optional + +import torch +import torch.fx as fx +from torch._dispatch.python import enable_python_dispatcher + +from sglang.srt.compilation.compilation_config import CompilationConfig +from sglang.srt.compilation.compilation_counter import compilation_counter +from sglang.srt.compilation.compiler_interface import EagerAdapter, InductorAdaptor +from sglang.srt.compilation.cuda_piecewise_backend import CUDAPiecewiseBackend +from sglang.srt.compilation.pass_manager import PostGradPassManager +from sglang.srt.utils.common import rank0_log + +logger = logging.getLogger(__name__) + + +SPLIT_OPS = [ + "sglang.unified_attention_with_output", + "sglang.gdn_with_output", +] + + +def add_split_ops(ops): + SPLIT_OPS.extend(ops) + + +def make_compiler(config: CompilationConfig): + if config.compiler == "eager": + return EagerAdapter() + elif config.compiler == "inductor": + return InductorAdaptor() + else: + raise ValueError(f"Unknown compiler: {config.compiler}") + + +class CompilerManager: + def __init__( + self, + config: CompilationConfig, + ): + self.cache = dict() + self.is_cache_updated = False + self.compiler = make_compiler(config) + + def compute_hash(self): + return self.compiler.compute_hash() + + def initialize_cache( + self, cache_dir: str, disable_cache: bool = False, prefix: str = "" + ): + self.disable_cache = disable_cache + self.cache_dir = cache_dir + self.cache_file_path = os.path.join(cache_dir, "sglang_compile_cache.py") + + if not disable_cache and os.path.exists(self.cache_file_path): + with open(self.cache_file_path) as f: + self.cache = ast.literal_eval(f.read()) + + self.compiler.initialize_cache( + cache_dir=cache_dir, disable_cache=disable_cache, prefix=prefix + ) + + def save_to_file(self): + if self.disable_cache or not self.is_cache_updated: + return + printer = pprint.PrettyPrinter(indent=4) + data = printer.pformat(self.cache) + with open(self.cache_file_path, "w") as f: + f.write(data) + + def load( + self, + graph: fx.GraphModule, + example_inputs: list[Any], + graph_index: int, + runtime_shape: Optional[int] = None, + ) -> Optional[Callable]: + handle = self.cache[(runtime_shape, graph_index, self.compiler.name)] + compiled_graph = self.compiler.load( + handle, graph, example_inputs, graph_index, runtime_shape + ) + if runtime_shape is None: + logger.debug( + "Directly load the %s-th graph for dynamic shape from %s via " + "handle %s", + graph_index, + self.compiler.name, + handle, + ) + else: + logger.debug( + "Directly load the %s-th graph for shape %s from %s via " "handle %s", + graph_index, + str(runtime_shape), + self.compiler.name, + handle, + ) + return compiled_graph + + def compile( + self, + graph: fx.GraphModule, + example_inputs, + inductor_config: dict[str, Any], + graph_index: int = 0, + num_graphs: int = 1, + runtime_shape: Optional[int] = None, + ) -> Any: + if graph_index == 0: + # before compiling the first graph, record the start time + global compilation_start_time + compilation_start_time = time.time() + + compilation_counter.num_backend_compilations += 1 + + compiled_graph = None + + # TODO(Yuwei): support cache loading + + # no compiler cached the graph, or the cache is disabled, + # we need to compile it + if isinstance(self.compiler, InductorAdaptor): + maybe_key = None + else: + maybe_key = f"artifact_shape_{runtime_shape}_subgraph_{graph_index}" + compiled_graph, handle = self.compiler.compile( + graph, example_inputs, inductor_config, runtime_shape, maybe_key + ) + + assert compiled_graph is not None, "Failed to compile the graph" + + # store the artifact in the cache + if handle is not None: + self.cache[(runtime_shape, graph_index, self.compiler.name)] = handle + compilation_counter.num_cache_entries_updated += 1 + self.is_cache_updated = True + if graph_index == 0: + # adds some info logging for the first graph + if runtime_shape is None: + logger.info("Cache the graph for dynamic shape for later use") + else: + logger.info( + "Cache the graph of shape %s for later use", str(runtime_shape) + ) + if runtime_shape is None: + logger.debug( + "Store the %s-th graph for dynamic shape from %s via " "handle %s", + graph_index, + self.compiler.name, + handle, + ) + else: + logger.debug( + "Store the %s-th graph for shape %s from %s via handle %s", + graph_index, + str(runtime_shape), + self.compiler.name, + handle, + ) + + # after compiling the last graph, record the end time + if graph_index == num_graphs - 1: + now = time.time() + elapsed = now - compilation_start_time + if runtime_shape is None: + logger.info("Compiling a graph for dynamic shape takes %.2f s", elapsed) + else: + logger.info( + "Compiling a graph for shape %s takes %.2f s", + runtime_shape, + elapsed, + ) + + return compiled_graph + + +@dataclasses.dataclass +class SplitItem: + submod_name: str + graph_id: int + is_splitting_graph: bool + graph: fx.GraphModule + + +def split_graph( + graph: fx.GraphModule, ops: list[str] +) -> tuple[fx.GraphModule, list[SplitItem]]: + # split graph by ops + subgraph_id = 0 + node_to_subgraph_id = {} + split_op_graphs = [] + for node in graph.graph.nodes: + if node.op in ("output", "placeholder"): + continue + if node.op == "call_function" and str(node.target) in ops: + subgraph_id += 1 + node_to_subgraph_id[node] = subgraph_id + split_op_graphs.append(subgraph_id) + subgraph_id += 1 + else: + node_to_subgraph_id[node] = subgraph_id + + # `keep_original_order` is important! + # otherwise pytorch might reorder the nodes and + # the semantics of the graph will change when we + # have mutations in the graph + split_gm = torch.fx.passes.split_module.split_module( + graph, None, lambda node: node_to_subgraph_id[node], keep_original_order=True + ) + + outputs = [] + + names = [name for (name, module) in split_gm.named_modules()] + + for name in names: + if "." in name or name == "": + # recursive child module or the root module + continue + + module = getattr(split_gm, name) + + graph_id = int(name.replace("submod_", "")) + outputs.append(SplitItem(name, graph_id, (graph_id in split_op_graphs), module)) + + # sort by intetger graph_id, rather than string name + outputs.sort(key=lambda x: x.graph_id) + + return split_gm, outputs + + +# we share the global graph pool among all the backends +global_graph_pool = None + +compilation_start_time = 0.0 + + +class PiecewiseCompileInterpreter(torch.fx.Interpreter): + def __init__( + self, + module: torch.fx.GraphModule, + compile_submod_names: list[str], + inductor_config: dict[str, Any], + graph_pool, + compile_config: CompilationConfig, + sglang_backend: "SGLangBackend", + ): + super().__init__(module) + from torch._guards import detect_fake_mode + + self.fake_mode = detect_fake_mode() + self.compile_submod_names = compile_submod_names + self.graph_pool = graph_pool + self.sglang_backend = sglang_backend + # When True, it annoyingly dumps the torch.fx.Graph on errors. + self.extra_traceback = False + self.inductor_config = inductor_config + self.compile_config = compile_config + + def run(self, *args): + fake_args = [ + self.fake_mode.from_tensor(t) if isinstance(t, torch.Tensor) else t + for t in args + ] + with self.fake_mode, enable_python_dispatcher(): + return super().run(*fake_args) + + def call_module( + self, + target: torch.fx.node.Target, + args: tuple[torch.fx.node.Argument, ...], + kwargs: dict[str, Any], + ) -> Any: + assert isinstance(target, str) + output = super().call_module(target, args, kwargs) + + if target in self.compile_submod_names: + index = self.compile_submod_names.index(target) + submod = self.fetch_attr(target) + sym_shape_indices = [ + i for i, x in enumerate(args) if isinstance(x, torch.SymInt) + ] + global compilation_start_time + compiled_graph_for_dynamic_shape = ( + self.sglang_backend.compiler_manager.compile( + submod, + args, + self.inductor_config, + graph_index=index, + num_graphs=len(self.compile_submod_names), + runtime_shape=None, + ) + ) + + self.module.__dict__[target] = CUDAPiecewiseBackend( + submod, + self.compile_config, + self.inductor_config, + self.graph_pool, + index, + len(self.compile_submod_names), + sym_shape_indices, + compiled_graph_for_dynamic_shape, + self.sglang_backend, + ) + + compilation_counter.num_piecewise_capturable_graphs_seen += 1 + + return output + + +model_tag: str = "backbone" + + +@contextmanager +def set_model_tag(tag: str): + """Context manager to set the model tag.""" + global model_tag + assert ( + tag != model_tag + ), f"Model tag {tag} is the same as the current tag {model_tag}." + old_tag = model_tag + model_tag = tag + try: + yield + finally: + model_tag = old_tag + + +class SGLangBackend: + + graph_pool: Any + _called: bool = False + # the graph we compiled + graph: fx.GraphModule + # the stiching graph module for all the piecewise graphs + split_gm: fx.GraphModule + piecewise_graphs: list[SplitItem] + returned_callable: Callable + # Inductor passes to run on the graph pre-defunctionalization + post_grad_passes: Sequence[Callable] + sym_tensor_indices: list[int] + input_buffers: list[torch.Tensor] + compiler_manager: CompilerManager + + def __init__( + self, + config: CompilationConfig, + graph_pool: Any, + ): + rank0_log(f"Initializing SGLangBackend") + assert graph_pool is not None + self.graph_pool = graph_pool + + self.post_grad_pass_manager = PostGradPassManager() + self.sym_tensor_indices = [] + self.input_buffers = [] + + self.compiler_manager = CompilerManager(config) + self.inductor_config = { + "enable_auto_functionalized_v2": False, + } + self.compile_config = config + + def configure_post_pass(self): + self.post_grad_pass_manager.configure() + self.inductor_config["post_grad_custom_post_pass"] = self.post_grad_pass_manager + + def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable: + rank0_log(f"SGLangBackend __call__") + base_cache_dir = os.path.expanduser( + os.getenv("SGLANG_CACHE_DIR", "~/.cache/sglang/") + ) + + cache_hash = self.compiler_manager.compute_hash() + cache_dir = os.path.join( + base_cache_dir, + "torch_compile_cache", + cache_hash, + ) + + os.makedirs(cache_dir, exist_ok=True) + rank = 0 + dp_rank = 0 + local_cache_dir = os.path.join(cache_dir, f"rank_{rank}_{dp_rank}", model_tag) + os.makedirs(local_cache_dir, exist_ok=True) + self.compiler_manager.initialize_cache( + local_cache_dir, disable_cache=False, prefix="" + ) + compilation_counter.num_graphs_seen += 1 + + assert not self._called, "SGLangBackend can only be called once" + + self.graph = graph + self.configure_post_pass() + + self.split_gm, self.piecewise_graphs = split_graph( + graph, + SPLIT_OPS, + ) + from torch._dynamo.utils import lazy_format_graph_code + + # depyf will hook lazy_format_graph_code and dump the graph + # for debugging, no need to print the graph here + lazy_format_graph_code("before split", self.graph) + lazy_format_graph_code("after split", self.split_gm) + + compilation_counter.num_piecewise_graphs_seen += len(self.piecewise_graphs) + + submod_names_to_compile = [ + item.submod_name + for item in self.piecewise_graphs + if not item.is_splitting_graph + ] + + PiecewiseCompileInterpreter( + self.split_gm, + submod_names_to_compile, + self.inductor_config, + self.graph_pool, + self.compile_config, + self, + ).run(*example_inputs) + + rank = torch.distributed.get_rank() + + if rank == 0: + graph_path = os.path.join( + local_cache_dir, f"computation_graph_{time.time()}.py" + ) + if not os.path.exists(graph_path): + # code adapted from https://github.com/thuml/depyf/blob/dab831108a752d1facc00acdd6d4243891845c37/depyf/explain/patched_lazy_format_graph_code.py#L30 # noqa + # use `print_readable` because it can include submodules + src = ( + "from __future__ import annotations\nimport torch\n" + + self.split_gm.print_readable(print_output=False) + ) + src = src.replace("", "GraphModule") + with open(graph_path, "w") as f: + f.write(src) + + rank0_log(f"Computation graph saved to {graph_path}") + + self._called = True + return self.split_gm diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/compilation_config.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/compilation_config.py new file mode 100644 index 00000000..5ddafe8a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/compilation_config.py @@ -0,0 +1,29 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.10.0/vllm/compilation/compilation_config.py + +from typing import List + + +# TODO(Yuwei): support better compile config support +class CompilationConfig: + def __init__( + self, + capture_sizes: List[int], + compiler: str = "eager", + enable_debug_mode: bool = False, + ): + self.traced_files = set() + self.capture_sizes = capture_sizes + self.compiler = compiler + self.enable_debug_mode = enable_debug_mode + + def add_traced_file(self, file_path: str): + self.traced_files.add(file_path) + + def get_traced_files(self): + return self.traced_files + + def get_capture_sizes(self): + return self.capture_sizes + + def get_enable_debug_mode(self): + return self.enable_debug_mode diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/compilation_counter.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/compilation_counter.py new file mode 100644 index 00000000..e973f8f2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/compilation_counter.py @@ -0,0 +1,47 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.10.0/vllm/compilation/compilation_counter.py + +import copy +import dataclasses +from contextlib import contextmanager + + +@dataclasses.dataclass +class CompilationCounter: + num_models_seen: int = 0 + num_graphs_seen: int = 0 + # including the splitting ops + num_piecewise_graphs_seen: int = 0 + # not including the splitting ops + num_piecewise_capturable_graphs_seen: int = 0 + num_backend_compilations: int = 0 + # Number of gpu_model_runner attempts to trigger CUDAGraphs capture + num_gpu_runner_capture_triggers: int = 0 + # Number of CUDAGraphs captured + num_cudagraph_captured: int = 0 + # InductorAdapter.compile calls + num_inductor_compiles: int = 0 + # EagerAdapter.compile calls + num_eager_compiles: int = 0 + # The number of time vLLM's compiler cache entry was updated + num_cache_entries_updated: int = 0 + # The number of standalone_compile compiled artifacts saved + num_compiled_artifacts_saved: int = 0 + # Number of times a model was loaded with CompilationLevel.DYNAMO_AS_IS + dynamo_as_is_count: int = 0 + + def clone(self) -> "CompilationCounter": + return copy.deepcopy(self) + + @contextmanager + def expect(self, **kwargs): + old = self.clone() + yield + for k, v in kwargs.items(): + assert getattr(self, k) - getattr(old, k) == v, ( + f"{k} not as expected, before it is {getattr(old, k)}" + f", after it is {getattr(self, k)}, " + f"expected diff is {v}" + ) + + +compilation_counter = CompilationCounter() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/compile.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/compile.py new file mode 100644 index 00000000..b9ff7f6b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/compile.py @@ -0,0 +1,215 @@ +import contextvars +import inspect +import logging +import os +import sys +import types +from contextlib import contextmanager +from dataclasses import dataclass +from typing import Any, Callable, Optional, Union + +import torch + +from sglang.srt.compilation.compilation_config import CompilationConfig +from sglang.srt.utils.common import rank0_log + +logger = logging.getLogger(__name__) + +_COMPILE_ENABLED = contextvars.ContextVar("_COMPILE_ENABLED", default=False) + + +@contextmanager +def set_compiled(enabled: bool = True): + token = _COMPILE_ENABLED.set(enabled) + try: + yield + finally: + _COMPILE_ENABLED.reset(token) + + +@dataclass +class IntermediateTensors: + """For all pipeline stages except the last, we need to return the hidden + states and residuals to be sent to the next stage. This data structure + contains the hidden states and residuals for a request. + + Each stage also needs to handle its own finished_sending and + finished_recving in case of kv transfer. + """ + + tensors: dict[str, torch.Tensor] + # [req_ids] + finished_sending: Optional[set[str]] = None + finished_recving: Optional[set[str]] = None + + def __init__(self, tensors): + # manually define this function, so that + # Dynamo knows `IntermediateTensors()` comes from this file. + # Otherwise, dataclass will generate this function by evaluating + # a string, and we will lose the information about the source file. + self.tensors = tensors + + def __getitem__(self, key: Union[str, slice]): + if isinstance(key, str): + return self.tensors[key] + elif isinstance(key, slice): + return self.__class__({k: v[key] for k, v in self.tensors.items()}) + + def __setitem__(self, key: str, value: torch.Tensor): + self.tensors[key] = value + + def items(self): + return self.tensors.items() + + def __len__(self): + return len(self.tensors) + + def __eq__(self, other: object): + return isinstance(other, self.__class__) and self + + def __repr__(self) -> str: + return f"IntermediateTensors(tensors={self.tensors})" + + +def _normalize_dims(dims, ndim: int): + dims = [dims] if isinstance(dims, int) else list(dims) + return [d if d >= 0 else ndim + d for d in dims] + + +class _MaybeIntermediateTensors: + """Duck-typed check to support your IntermediateTensors without importing.""" + + def __init__(self, obj): + self.is_intermediate = hasattr(obj, "tensors") and isinstance( + getattr(obj, "tensors"), dict + ) + self.obj = obj + + +def _mark_dynamic_on_value(val, dims): + if isinstance(val, torch.Tensor): + torch._dynamo.mark_dynamic(val, _normalize_dims(dims, val.ndim)) + else: + mit = _MaybeIntermediateTensors(val) + if mit.is_intermediate: + for t in mit.obj.tensors.values(): + torch._dynamo.mark_dynamic(t, _normalize_dims(dims, t.ndim)) + # else: ignore (None or non-tensor) + + +def _infer_dynamic_arg_dims_from_annotations(forward_fn): + sig = inspect.signature(forward_fn) + dyn = {} + for name, p in sig.parameters.items(): + ann = p.annotation + # Accept torch.Tensor / Optional[torch.Tensor] / your IntermediateTensors types by name + if ( + ann is torch.Tensor + or getattr(getattr(ann, "__args__", [None])[0], "__name__", "") == "Tensor" + ): + dyn[name] = 0 + elif getattr(ann, "__name__", "") in ("IntermediateTensors",) or any( + getattr(a, "__name__", "") == "IntermediateTensors" + for a in getattr(ann, "__args__", []) + ): + dyn[name] = 0 + elif ann == "torch.Tensor" or ann == "Optional[torch.Tensor]": + # For future import annotations (e.g. from __future__ import annotations), the annotation is a string + dyn[name] = 0 + if not dyn: + raise ValueError("No dynamic dims inferred; pass dynamic_arg_dims explicitly.") + return dyn + + +def install_torch_compiled( + module: torch.nn.Module, + *, + dynamic_arg_dims: dict[str, Union[int, list[int]]] | None = None, + backend_factory: Optional[Callable[[torch.fx.GraphModule, list], Callable]] = None, + compile_config: CompilationConfig = None, + fullgraph: bool = True, + graph_pool: Any = None, +): + rank0_log(f"install_torch_compiled") + unbound_fwd = module.__class__.forward + if not callable(unbound_fwd): + raise TypeError("module.__class__.forward must be callable") + original_code = unbound_fwd.__code__ + + dyn_map = dynamic_arg_dims or _infer_dynamic_arg_dims_from_annotations(unbound_fwd) + + if backend_factory is None: + from sglang.srt.compilation.backend import SGLangBackend + + backend_factory = lambda gm, ex: SGLangBackend(compile_config, graph_pool)( + gm, ex + ) + + compiled_codes: list[type(original_code)] = [] + state = {"compiled": False, "compiled_callable": None} + + def bytecode_hook(old_code, new_code): + if old_code is not original_code: + return + frame = sys._getframe() + while frame and frame.f_back: + frame = frame.f_back + if ( + frame.f_code.co_name == "_compile" + and os.path.basename(frame.f_code.co_filename) == "convert_frame.py" + ): + break + try: + dynamo_frame = frame.f_locals["frame"] + except Exception: + return + if dynamo_frame.f_code is not old_code: + return + if dynamo_frame.f_locals.get("self") is not module: + return + compiled_codes.append(new_code) + + torch._dynamo.convert_frame.register_bytecode_hook(bytecode_hook) + + def _ensure_compiled(self, *args, **kwargs): + """Compile on first use (with flag ON).""" + if state["compiled"]: + return + # Mark dynamic dims only when we are about to compile + sig = inspect.signature(unbound_fwd) + ba = sig.bind(self, *args, **kwargs) + ba.apply_defaults() + for name, dims in (dyn_map or {}).items(): + if name in ba.arguments: + val = ba.arguments[name] + if val is not None: + _mark_dynamic_on_value(val, dims) + + # Avoid cross-instance cache reuse + torch._dynamo.eval_frame.remove_from_cache(unbound_fwd.__code__) + + bound = types.MethodType(unbound_fwd, self) + compiled_callable = torch.compile( + bound, fullgraph=fullgraph, backend=backend_factory + ) + + # Trigger Dynamo so bytecode hook can capture + compiled_callable(*args, **kwargs) + + state["compiled"] = True + state["compiled_callable"] = compiled_callable + + def trampoline(self, *args, **kwargs): + use_compiled = _COMPILE_ENABLED.get() + if use_compiled: + if not state["compiled"]: + _ensure_compiled(self, *args, **kwargs) + + compiled_callable = state["compiled_callable"] + return compiled_callable(*args, **kwargs) + else: + # Explicitly run the original uncompiled forward + return unbound_fwd(self, *args, **kwargs) + + module.forward = types.MethodType(trampoline, module) + return module diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/compiler_interface.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/compiler_interface.py new file mode 100644 index 00000000..8310f75c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/compiler_interface.py @@ -0,0 +1,503 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.10.0/vllm/compilation/compiler_interface.py + +import contextlib +import copy +import hashlib +import os +from contextlib import ExitStack +from typing import Any, Callable, Optional +from unittest.mock import patch + +import torch +import torch._inductor.compile_fx +import torch.fx as fx + +from sglang.srt.compilation.compilation_counter import compilation_counter +from sglang.srt.compilation.inductor_pass import pass_context + + +class CompilerInterface: + """ + The interface for a compiler that can be used by vLLM. + """ + + # The name of the compiler, e.g. inductor. + # This is a class-level attribute. + name: str + + def initialize_cache( + self, cache_dir: str, disable_cache: bool = False, prefix: str = "" + ): + """ + when the vLLM process uses `cache_dir` as the cache directory, + the compiler should initialize itself with the cache directory, + e.g. by re-directing its own cache directory to a sub-directory. + + prefix can be used in combination with cache_dir to figure out the base + cache directory, e.g. there're multiple parts of model being compiled, + but we want to share the same cache directory for all of them. + + e.g. + cache_dir = "/path/to/dir/backbone", prefix = "backbone" + cache_dir = "/path/to/dir/eagle_head", prefix = "eagle_head" + """ + pass + + def compute_hash(self) -> str: + """ + Gather all the relevant information from the vLLM config, + to compute a hash so that we can cache the compiled model. + + See [`VllmConfig.compute_hash`][vllm.config.VllmConfig.compute_hash] + to check what information + is already considered by default. This function should only + consider the information that is specific to the compiler. + """ + return "" + + def compile( + self, + graph: fx.GraphModule, + example_inputs: list[Any], + compiler_config: dict[str, Any], + runtime_shape: Optional[int] = None, + key: Optional[str] = None, + ) -> tuple[Optional[Callable], Optional[Any]]: + """ + Compile the graph with the given example inputs and compiler config, + with a runtime shape. If the `runtime_shape` is None, it means + the `example_inputs` have a dynamic shape. Otherwise, the + `runtime_shape` specifies the shape of the inputs. Right now we only + support one variable shape for all inputs, which is the batchsize + (number of tokens) during inference. + + Dynamo will make sure `graph(*example_inputs)` is valid. + + The function should return a compiled callable function, as well as + a handle that can be used to directly load the compiled function. + + The handle should be a plain Python object, preferably a string or a + file path for readability. + + If the compiler doesn't support caching, it should return None for the + handle. If the compiler fails to compile the graph, it should return + None for the compiled function as well. + + `key` is required for StandaloneInductorAdapter, it specifies where to + save the compiled artifact. The compiled artifact gets saved to + `cache_dir/key`. + """ + return None, None + + def load( + self, + handle: Any, + graph: fx.GraphModule, + example_inputs: list[Any], + graph_index: int, + runtime_shape: Optional[int] = None, + ) -> Callable: + """ + Load the compiled function from the handle. + Raises an error if the handle is invalid. + + The handle is the second return value of the `compile` function. + """ + raise NotImplementedError("caching is not supported") + + +def get_inductor_factors() -> list[Any]: + factors: list[Any] = [] + # summarize system state + from torch._inductor.codecache import CacheBase + + system_factors = CacheBase.get_system() + factors.append(system_factors) + + # summarize pytorch state + from torch._inductor.codecache import torch_key + + torch_factors = torch_key() + factors.append(torch_factors) + return factors + + +class AlwaysHitShapeEnv: + """ + Why do we need this class: + + For normal `torch.compile` usage, every compilation will have + one Dynamo bytecode compilation and one Inductor compilation. + The Inductor compilation happens under the context of the + Dynamo bytecode compilation, and that context is used to + determine the dynamic shape information, etc. + + For our use case, we only run Dynamo bytecode compilation once, + and run Inductor compilation multiple times with different shapes + plus a general shape. The compilation for specific shapes happens + outside of the context of the Dynamo bytecode compilation. At that + time, we don't have shape environment to provide to Inductor, and + it will fail the Inductor code cache lookup. + + By providing a dummy shape environment that always hits, we can + make the Inductor code cache lookup always hit, and we can + compile the graph for different shapes as needed. + + The following dummy methods are obtained by trial-and-error + until it works. + """ + + def __init__(self) -> None: + self.guards: list[Any] = [] + + def evaluate_guards_expression(self, *args, **kwargs): + return True + + def get_pruned_guards(self, *args, **kwargs): + return [] + + def produce_guards_expression(self, *args, **kwargs): + return "" + + +class InductorAdaptor(CompilerInterface): + """ + The adaptor for the Inductor compiler, version 2.5, 2.6, 2.7. + """ + + name = "inductor" + + def compute_hash(self) -> str: + factors = get_inductor_factors() + hash_str = hashlib.md5( + str(factors).encode(), usedforsecurity=False + ).hexdigest()[:10] + return hash_str + + def initialize_cache( + self, cache_dir: str, disable_cache: bool = False, prefix: str = "" + ): + self.cache_dir = cache_dir + self.prefix = prefix + self.base_cache_dir = cache_dir[: -len(prefix)] if prefix else cache_dir + if disable_cache: + return + # redirect the cache directory to a sub-directory + # set flags so that Inductor and Triton store their cache + # in the cache_dir, then users only need to copy the cache_dir + # to another machine to reuse the cache. + inductor_cache = os.path.join(self.base_cache_dir, "inductor_cache") + os.makedirs(inductor_cache, exist_ok=True) + os.environ["TORCHINDUCTOR_CACHE_DIR"] = inductor_cache + triton_cache = os.path.join(self.base_cache_dir, "triton_cache") + os.makedirs(triton_cache, exist_ok=True) + os.environ["TRITON_CACHE_DIR"] = triton_cache + + def compile( + self, + graph: fx.GraphModule, + example_inputs: list[Any], + compiler_config: dict[str, Any], + runtime_shape: Optional[int] = None, + key: Optional[str] = None, + ) -> tuple[Optional[Callable], Optional[Any]]: + compilation_counter.num_inductor_compiles += 1 + from torch._inductor.compile_fx import compile_fx + + current_config = {} + if compiler_config is not None: + current_config.update(compiler_config) + + # disable remote cache + current_config["fx_graph_cache"] = True + current_config["fx_graph_remote_cache"] = False + + set_inductor_config(current_config, runtime_shape) + + # inductor can inplace modify the graph, so we need to copy it + # see https://github.com/pytorch/pytorch/issues/138980 + graph = copy.deepcopy(graph) + + # it's the first time we compile this graph + # the assumption is that we don't have nested Inductor compilation. + # compiled_fx_graph_hash will only be called once, and we can hook + # it to get the hash of the compiled graph directly. + + hash_str, file_path = None, None + from torch._inductor.codecache import FxGraphCache, compiled_fx_graph_hash + + if torch.__version__.startswith("2.5"): + original_load = FxGraphCache.load + original_load_name = "torch._inductor.codecache.FxGraphCache.load" + + def hijack_load(*args, **kwargs): + inductor_compiled_graph = original_load(*args, **kwargs) + nonlocal file_path + compiled_fn = inductor_compiled_graph.current_callable + file_path = compiled_fn.__code__.co_filename # noqa + if not file_path.startswith(self.base_cache_dir): + # hooked in the align_inputs_from_check_idxs function + # in torch/_inductor/utils.py + for cell in compiled_fn.__closure__: + if not callable(cell.cell_contents): + continue + if cell.cell_contents.__code__.co_filename.startswith( + self.base_cache_dir + ): + # this is the real file path compiled from Inductor + file_path = cell.cell_contents.__code__.co_filename + break + return inductor_compiled_graph + + hijacked_compile_fx_inner = ( + torch._inductor.compile_fx.compile_fx_inner + ) # noqa + elif torch.__version__ >= "2.6": + # function renamed in 2.6 + original_load_name = None + + def hijacked_compile_fx_inner(*args, **kwargs): + output = torch._inductor.compile_fx.compile_fx_inner(*args, **kwargs) + nonlocal hash_str + inductor_compiled_graph = output + if inductor_compiled_graph is not None: + nonlocal file_path + compiled_fn = inductor_compiled_graph.current_callable + file_path = compiled_fn.__code__.co_filename # noqa + if not file_path.startswith(self.base_cache_dir): + # hooked in the align_inputs_from_check_idxs function + # in torch/_inductor/utils.py + for cell in compiled_fn.__closure__: + if not callable(cell.cell_contents): + continue + code = cell.cell_contents.__code__ + if code.co_filename.startswith(self.base_cache_dir): + # this is the real file path + # compiled from Inductor + file_path = code.co_filename + break + hash_str = inductor_compiled_graph._fx_graph_cache_key + return output + + def hijack_compiled_fx_graph_hash(*args, **kwargs): + out = compiled_fx_graph_hash(*args, **kwargs) + nonlocal hash_str + hash_str = out[0] + return out + + def _check_can_cache(*args, **kwargs): + # no error means it can be cached. + # Inductor refuses to cache the graph outside of Dynamo + # tracing context, and also disables caching for graphs + # with high-order ops. + # For vLLM, in either case, we want to cache the graph. + # see https://github.com/pytorch/pytorch/blob/9f5ebf3fc609105a74eab4ccc24932d6353ff566/torch/_inductor/codecache.py#L1221 # noqa + return + + def _get_shape_env() -> AlwaysHitShapeEnv: + return AlwaysHitShapeEnv() + + with ExitStack() as stack: + # hijack to get the compiled graph itself + if original_load_name is not None: + stack.enter_context(patch(original_load_name, hijack_load)) + + # for hijacking the hash of the compiled graph + stack.enter_context( + patch( + "torch._inductor.codecache.compiled_fx_graph_hash", + hijack_compiled_fx_graph_hash, + ) + ) + + # for providing a dummy shape environment + stack.enter_context( + patch( + "torch._inductor.codecache.FxGraphCache._get_shape_env", + _get_shape_env, + ) + ) + + from torch._functorch._aot_autograd.autograd_cache import AOTAutogradCache + + # torch 2.8+ on main uses _get_shape_env in AOTAutogradCache + if hasattr(AOTAutogradCache, "_get_shape_env"): + stack.enter_context( + patch( + "torch._functorch._aot_autograd.autograd_cache.AOTAutogradCache._get_shape_env", + _get_shape_env, + ) + ) + + # for forcing the graph to be cached + stack.enter_context( + patch( + "torch._inductor.codecache.FxGraphCache._check_can_cache", + _check_can_cache, + ) + ) + + # Dynamo metrics context, see method for more details. + stack.enter_context(self.metrics_context()) + + # Disable remote caching. When these are on, on remote cache-hit, + # the monkey-patched functions never actually get called. + # vLLM today assumes and requires the monkey-patched functions to + # get hit. + # TODO(zou3519): we're going to replace this all with + # standalone_compile sometime. + + stack.enter_context( + torch._inductor.config.patch(fx_graph_remote_cache=False) + ) + # InductorAdaptor (unfortunately) requires AOTAutogradCache + # to be turned off to run. It will fail to acquire the hash_str + # and error if not. + # StandaloneInductorAdaptor (PyTorch 2.8+) fixes this problem. + stack.enter_context( + torch._functorch.config.patch(enable_autograd_cache=False) + ) + stack.enter_context( + torch._functorch.config.patch(enable_remote_autograd_cache=False) + ) + + with pass_context(runtime_shape): + compiled_graph = compile_fx( + graph, + example_inputs, + inner_compile=hijacked_compile_fx_inner, + config_patches=current_config, + ) + return compiled_graph, (hash_str, file_path) + + def load( + self, + handle: Any, + graph: fx.GraphModule, + example_inputs: list[Any], + graph_index: int, + runtime_shape: Optional[int] = None, + ) -> Callable: + assert isinstance(handle, tuple) + assert isinstance(handle[0], str) + assert isinstance(handle[1], str) + hash_str = handle[0] + + from torch._functorch._aot_autograd.autograd_cache import AOTAutogradCache + from torch._inductor.codecache import FxGraphCache + + with ExitStack() as exit_stack: + exit_stack.enter_context( + patch( + "torch._inductor.codecache.FxGraphCache._get_shape_env", + lambda *args, **kwargs: AlwaysHitShapeEnv(), + ) + ) + # torch 2.8+ on main uses _get_shape_env in AOTAutogradCache + if hasattr(AOTAutogradCache, "_get_shape_env"): + exit_stack.enter_context( + patch( + "torch._functorch._aot_autograd.autograd_cache.AOTAutogradCache._get_shape_env", + lambda *args, **kwargs: AlwaysHitShapeEnv(), + ) + ) + + # Dynamo metrics context, see method for more details. + exit_stack.enter_context(self.metrics_context()) + + if torch.__version__.startswith("2.5"): + inductor_compiled_graph = FxGraphCache._lookup_graph( + hash_str, example_inputs, True, False + ) + assert inductor_compiled_graph is not None, ( + "Inductor cache lookup failed. Please remove" + f"the cache directory and try again." # noqa + ) + elif torch.__version__ >= "2.6": + from torch._inductor.output_code import CompiledFxGraphConstantsWithGm + + constants = CompiledFxGraphConstantsWithGm(graph) + inductor_compiled_graph, _ = FxGraphCache._lookup_graph( + hash_str, example_inputs, True, None, constants + ) + assert inductor_compiled_graph is not None, ( + "Inductor cache lookup failed. Please remove" + f"the cache directory and try again." # noqa + ) + + # Inductor calling convention (function signature): + # f(list) -> tuple + # Dynamo calling convention (function signature): + # f(*args) -> Any + + # need to know if the graph returns a tuple + from torch._inductor.compile_fx import graph_returns_tuple + + returns_tuple = graph_returns_tuple(graph) + + # this is the callable we return to Dynamo to run + def compiled_graph(*args): + # convert args to list + list_args = list(args) + graph_output = inductor_compiled_graph(list_args) + # unpack the tuple if needed + if returns_tuple: + return graph_output + else: + return graph_output[0] + + return compiled_graph + + def metrics_context(self) -> contextlib.AbstractContextManager: + """ + This method returns the Dynamo metrics context (if it exists, + otherwise a null context). It is used by various compile components. + Present in torch>=2.6, it's used inside FxGraphCache in + torch==2.6 (but not after). It might also be used in various other + torch.compile internal functions. + + Because it is re-entrant, we always set it (even if entering via Dynamo + and the context was already entered). We might want to revisit if it + should be set at a different level of compilation. + + This is likely a bug in PyTorch: public APIs should not rely on + manually setting up internal contexts. But we also rely on non-public + APIs which might not provide these guarantees. + """ + import torch._dynamo.utils + + return torch._dynamo.utils.get_metrics_context() + + +def set_inductor_config(config, runtime_shape): + if isinstance(runtime_shape, int): + # for a specific batchsize, tuning triton kernel parameters + # can be beneficial + config["max_autotune"] = True + config["coordinate_descent_tuning"] = True + + +class EagerAdapter(CompilerInterface): + name = "eager" + + def compile( + self, + graph: fx.GraphModule, + example_inputs: list[Any], + compiler_config: dict[str, Any], + runtime_shape: Optional[int] = None, + key: Optional[str] = None, + num_graphs: int = 1, + ) -> tuple[Optional[Callable], Optional[Any]]: + return graph, None + + def load( + self, + handle: Any, + graph: fx.GraphModule, + example_inputs: list[Any], + graph_index: int, + runtime_shape: Optional[int] = None, + num_graphs: int = 1, + ) -> Callable: + raise NotImplementedError("eager compilation is not supported") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/cuda_piecewise_backend.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/cuda_piecewise_backend.py new file mode 100644 index 00000000..2e45d34d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/cuda_piecewise_backend.py @@ -0,0 +1,215 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.10.0/vllm/compilation/cuda_piecewise_backend.py + +import dataclasses +import logging +from contextlib import ExitStack +from typing import Any, Callable, Optional, Union +from unittest.mock import patch + +import torch +import torch.fx as fx +from sgl_kernel import weak_ref_tensor + +from sglang.srt.compilation.compilation_config import CompilationConfig +from sglang.srt.compilation.compilation_counter import compilation_counter + +logger = logging.getLogger(__name__) + + +def weak_ref_tensors( + tensors: Union[torch.Tensor, list[torch.Tensor], tuple[torch.Tensor]] +) -> Union[torch.Tensor, list[Any], tuple[Any], Any]: + """ + Convenience function to create weak references to tensors, + for single tensor, list of tensors or tuple of tensors. + """ + if isinstance(tensors, torch.Tensor): + return weak_ref_tensor(tensors) + if isinstance(tensors, list): + return [weak_ref_tensor(t) for t in tensors] + if isinstance(tensors, tuple): + return tuple(weak_ref_tensor(t) for t in tensors) + raise ValueError("Invalid type for tensors") + + +@dataclasses.dataclass +class ConcreteSizeEntry: + runtime_shape: int + need_to_compile: bool # the size is in compile_sizes + use_cudagraph: bool # the size is in cudagraph_capture_sizes + + compiled: bool = False + runnable: Callable = None # type: ignore + num_finished_warmup: int = 0 + cudagraph: Optional[torch.cuda.CUDAGraph] = None + output: Optional[Any] = None + + # for cudagraph debugging, track the input addresses + # during capture, and check if they are the same during replay + input_addresses: Optional[list[int]] = None + + +class CUDAPiecewiseBackend: + + def __init__( + self, + graph: fx.GraphModule, + compile_config: CompilationConfig, + inductor_config: dict[str, Any], + graph_pool: Any, + piecewise_compile_index: int, + total_piecewise_compiles: int, + sym_shape_indices: list[int], + compiled_graph_for_general_shape: Callable, + sglang_backend, + ): + """ + The backend for piecewise compilation. + It mainly handles the compilation and cudagraph capturing. + + We will compile `self.graph` once for the general shape, + and then compile for different shapes specified in + `compilation_config.compile_sizes`. + + Independently, we will capture cudagraph for different shapes. + + If a shape needs both compilation and cudagraph, we will + compile it first, and then capture cudagraph. + """ + self.graph = graph + self.inductor_config = inductor_config + self.graph_pool = graph_pool + self.piecewise_compile_index = piecewise_compile_index + self.total_piecewise_compiles = total_piecewise_compiles + self.sglang_backend = sglang_backend + + self.is_first_graph = piecewise_compile_index == 0 + self.is_last_graph = piecewise_compile_index == total_piecewise_compiles - 1 + + self.compile_sizes: set[int] = set([]) + self.compile_config = compile_config + self.cudagraph_capture_sizes: set[int] = set(compile_config.get_capture_sizes()) + + self.first_run_finished = False + + self.compiled_graph_for_general_shape = compiled_graph_for_general_shape # noqa + + self.sym_shape_indices = sym_shape_indices + + # the entries for different shapes that we need to either + # compile or capture cudagraph + self.concrete_size_entries: dict[int, ConcreteSizeEntry] = {} + + # to_be_compiled_sizes tracks the remaining sizes to compile, + # and updates during the compilation process, so we need to copy it + self.to_be_compiled_sizes: set[int] = self.compile_sizes.copy() + for shape in self.compile_sizes.union(self.cudagraph_capture_sizes): + self.concrete_size_entries[shape] = ConcreteSizeEntry( + runtime_shape=shape, + need_to_compile=shape in self.compile_sizes, + use_cudagraph=shape in self.cudagraph_capture_sizes, + ) + + def check_for_ending_compilation(self): + if self.is_last_graph and not self.to_be_compiled_sizes: + # no specific sizes to compile + # save the hash of the inductor graph for the next run + self.sglang_backend.compiler_manager.save_to_file() + + def __call__(self, *args) -> Any: + if not self.first_run_finished: + self.first_run_finished = True + self.check_for_ending_compilation() + return self.compiled_graph_for_general_shape(*args) + runtime_shape = args[self.sym_shape_indices[0]] + if runtime_shape not in self.concrete_size_entries: + # we don't need to do anything for this shape + return self.compiled_graph_for_general_shape(*args) + + entry = self.concrete_size_entries[runtime_shape] + + if entry.runnable is None: + entry.runnable = self.compiled_graph_for_general_shape + + if entry.need_to_compile and not entry.compiled: + entry.compiled = True + self.to_be_compiled_sizes.remove(runtime_shape) + # args are real arguments + entry.runnable = self.sglang_backend.compiler_manager.compile( + self.graph, + args, + self.inductor_config, + graph_index=self.piecewise_compile_index, + num_graphs=self.total_piecewise_compiles, + runtime_shape=runtime_shape, + ) + + # finished compilations for all required shapes + if self.is_last_graph and not self.to_be_compiled_sizes: + self.check_for_ending_compilation() + + # Skip CUDA graphs if this entry doesn't use them OR + # if we're supposed to skip them globally + # skip_cuda_graphs = get_forward_context().skip_cuda_graphs + # if not entry.use_cudagraph or skip_cuda_graphs: + # return entry.runnable(*args) + + if entry.cudagraph is None: + if entry.num_finished_warmup < 1: # noqa + entry.num_finished_warmup += 1 + return entry.runnable(*args) + + if self.compile_config.get_enable_debug_mode(): + input_addresses = [ + x.data_ptr() for x in args if isinstance(x, torch.Tensor) + ] + entry.input_addresses = input_addresses + cudagraph = torch.cuda.CUDAGraph() + + with ExitStack() as stack: + if not self.is_first_graph: + # during every model forward, we will capture + # many pieces of cudagraphs (roughly one per layer). + # running gc again and again across layers will + # make the cudagraph capture very slow. + # therefore, we only run gc for the first graph, + # and disable gc for the rest of the graphs. + stack.enter_context(patch("gc.collect", lambda: None)) + stack.enter_context(patch("torch.cuda.empty_cache", lambda: None)) + + # mind-exploding: carefully manage the reference and memory. + with torch.cuda.graph(cudagraph, pool=self.graph_pool): + # `output` is managed by pytorch's cudagraph pool + output = entry.runnable(*args) + if self.is_last_graph: + # by converting it to weak ref, + # the original `output` will immediately be released + # to save memory. It is only safe to do this for + # the last graph, because the output of the last graph + # will not be used by any other cuda graph. + output = weak_ref_tensors(output) + + # here we always use weak ref for the output + # to save memory + entry.output = weak_ref_tensors(output) + entry.cudagraph = cudagraph + + compilation_counter.num_cudagraph_captured += 1 + + # important: we need to return the output, rather than + # the weak ref of the output, so that pytorch can correctly + # manage the memory during cuda graph capture + return output + + if self.compile_config.get_enable_debug_mode(): + # check if the input addresses are the same + new_input_addresses = [ + x.data_ptr() for x in args if isinstance(x, torch.Tensor) + ] + assert new_input_addresses == entry.input_addresses, ( + "Input addresses for cudagraphs are different during replay." + f" Expected {entry.input_addresses}, got {new_input_addresses}" + ) + + entry.cudagraph.replay() + return entry.output diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/fix_functionalization.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/fix_functionalization.py new file mode 100644 index 00000000..8673e357 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/fix_functionalization.py @@ -0,0 +1,134 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.10.0/vllm/compilation/fix_functionalization.py + +import logging +import operator +from collections.abc import Iterable +from typing import Optional, Union + +import torch +from torch._higher_order_ops.auto_functionalize import auto_functionalized + +from sglang.srt.compilation.fx_utils import is_func +from sglang.srt.compilation.inductor_pass import SGLangInductorPass + +logger = logging.getLogger(__name__) + + +class FixFunctionalizationPass(SGLangInductorPass): + """ + This pass defunctionalizes certain nodes to avoid redundant tensor copies. + After this pass, DCE (dead-code elimination) should never be run, + as de-functionalized nodes may appear as dead code. + + To add new nodes to defunctionalize, add to the if-elif chain in __call__. + """ + + def __call__(self, graph: torch.fx.Graph): + self.begin() + self.dump_graph(graph, "before_fix_functionalization") + + self.nodes_to_remove: list[torch.fx.Node] = [] + count = 0 + for node in graph.nodes: + if not is_func(node, auto_functionalized): + continue # Avoid deep if-elif nesting + count += 1 + + self.dump_graph(graph, "before_fix_functionalization_cleanup") + + # Remove the nodes all at once + count_removed = len(self.nodes_to_remove) + for node in self.nodes_to_remove: + graph.erase_node(node) + + logger.debug( + "De-functionalized %s nodes, removed %s nodes", count, count_removed + ) + self.dump_graph(graph, "after_fix_functionalization") + self.end_and_log() + + def _remove(self, node_or_nodes: Union[torch.fx.Node, Iterable[torch.fx.Node]]): + """ + Stage a node (or nodes) for removal at the end of the pass. + """ + if isinstance(node_or_nodes, torch.fx.Node): + self.nodes_to_remove.append(node_or_nodes) + else: + self.nodes_to_remove.extend(node_or_nodes) + + def defunctionalize( + self, + graph: torch.fx.Graph, + node: torch.fx.Node, + mutated_args: dict[int, Union[torch.fx.Node, str]], + args: Optional[tuple[Union[torch.fx.Node, str], ...]] = None, + ): + """ + De-functionalize a node by replacing it with a call to the original. + It also replaces the getitem users with the mutated arguments. + See replace_users_with_mutated_args and insert_defunctionalized. + """ + self.replace_users_with_mutated_args(node, mutated_args) + self.insert_defunctionalized(graph, node, args=args) + self._remove(node) + + def replace_users_with_mutated_args( + self, node: torch.fx.Node, mutated_args: dict[int, Union[torch.fx.Node, str]] + ): + """ + Replace all getitem users of the auto-functionalized node with the + mutated arguments. + :param node: The auto-functionalized node + :param mutated_args: The mutated arguments, indexed by getitem index. + If the value of an arg is a string, `node.kwargs[arg]` is used. + """ + for idx, user in self.getitem_users(node).items(): + arg = mutated_args[idx] + arg = node.kwargs[arg] if isinstance(arg, str) else arg + user.replace_all_uses_with(arg) + self._remove(user) + + def getitem_users(self, node: torch.fx.Node) -> dict[int, torch.fx.Node]: + """ + Returns the operator.getitem users of the auto-functionalized node, + indexed by the index they are getting. + """ + users = {} + for user in node.users: + if is_func(user, operator.getitem): + idx = user.args[1] + users[idx] = user + return users + + def insert_defunctionalized( + self, + graph: torch.fx.Graph, + node: torch.fx.Node, + args: Optional[tuple[Union[torch.fx.Node, str], ...]] = None, + ): + """ + Insert a new defunctionalized node into the graph before node. + If one of the kwargs is 'out', provide args directly, + as node.kwargs cannot be used. + See https://github.com/pytorch/pytorch/blob/a00faf440888ffb724bad413f329a49e2b6388e7/torch/_inductor/lowering.py#L351 + + :param graph: Graph to insert the defunctionalized node into + :param node: The auto-functionalized node to defunctionalize + :param args: If we cannot use kwargs, specify args directly. + If an arg is a string, `node.kwargs[arg]` is used. + """ # noqa: E501 + assert is_func( + node, auto_functionalized + ), f"node must be auto-functionalized, is {node} instead" + + # Create a new call to the original function + with graph.inserting_before(node): + function = node.args[0] + if args is None: + graph.call_function(function, kwargs=node.kwargs) + else: + # Args passed as strings refer to items in node.kwargs + args = tuple( + node.kwargs[arg] if isinstance(arg, str) else arg for arg in args + ) + graph.call_function(function, args=args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/fx_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/fx_utils.py new file mode 100644 index 00000000..b2e863e6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/fx_utils.py @@ -0,0 +1,83 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.10.0/vllm/compilation/fx_utils.py + +import operator +from collections.abc import Iterable, Iterator +from typing import Optional + +from torch import fx +from torch._higher_order_ops.auto_functionalize import auto_functionalized +from torch._ops import OpOverload + + +def is_func(node: fx.Node, target) -> bool: + return node.op == "call_function" and node.target == target + + +def is_auto_func(node: fx.Node, op: OpOverload) -> bool: + return is_func(node, auto_functionalized) and node.args[0] == op + + +# Returns the first specified node with the given op (if it exists) +def find_specified_fn_maybe( + nodes: Iterable[fx.Node], op: OpOverload +) -> Optional[fx.Node]: + for node in nodes: + if node.target == op: + return node + return None + + +# Returns the first specified node with the given op +def find_specified_fn(nodes: Iterable[fx.Node], op: OpOverload) -> fx.Node: + node = find_specified_fn_maybe(nodes, op) + assert node is not None, f"Could not find {op} in nodes {nodes}" + return node + + +# Returns the first auto_functionalized node with the given op (if it exists) +def find_auto_fn_maybe(nodes: Iterable[fx.Node], op: OpOverload) -> Optional[fx.Node]: + for node in nodes: + if is_func(node, auto_functionalized) and node.args[0] == op: # noqa + return node + return None + + +# Returns the first auto_functionalized node with the given op +def find_auto_fn(nodes: Iterable[fx.Node], op: OpOverload) -> fx.Node: + node = find_auto_fn_maybe(nodes, op) + assert node is not None, f"Could not find {op} in nodes {nodes}" + return node + + +# Returns the getitem node that extracts the idx-th element from node +# (if it exists) +def find_getitem_maybe(node: fx.Node, idx: int) -> Optional[fx.Node]: + for user in node.users: + if is_func(user, operator.getitem) and user.args[1] == idx: + return user + return None + + +# Returns the getitem node that extracts the idx-th element from node +def find_getitem(node: fx.Node, idx: int) -> fx.Node: + ret = find_getitem_maybe(node, idx) + assert ret is not None, f"Could not find getitem {idx} in node {node}" + return ret + + +# An auto-functionalization-aware utility for finding nodes with a specific op +def find_op_nodes(op: OpOverload, graph: fx.Graph) -> Iterator[fx.Node]: + if not op._schema.is_mutable: + yield from graph.find_nodes(op="call_function", target=op) + + for n in graph.find_nodes(op="call_function", target=auto_functionalized): + if n.args[0] == op: + yield n + + +# Asserts that the node only has one user and returns it +# Even if a node has only 1 user, it might share storage with another node, +# which might need to be taken into account. +def get_only_user(node: fx.Node) -> fx.Node: + assert len(node.users) == 1 + return next(iter(node.users)) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/inductor_pass.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/inductor_pass.py new file mode 100644 index 00000000..acbde65b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/inductor_pass.py @@ -0,0 +1,140 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.10.0/vllm/compilation/inductor_pass.py + +import hashlib +import inspect +import json +import logging +import time +import types +from contextlib import contextmanager +from typing import Any, Callable, Optional, Union + +import torch +from torch import fx +from torch._dynamo.utils import lazy_format_graph_code +from torch._inductor.custom_graph_pass import CustomGraphPass + +logger = logging.getLogger(__name__) + +_pass_context = None + + +class PassContext: + + def __init__(self, runtime_shape: Optional[int]): + self.runtime_shape = runtime_shape + + +def get_pass_context() -> PassContext: + """Get the current pass context.""" + assert _pass_context is not None + return _pass_context + + +@contextmanager +def pass_context(runtime_shape: Optional[int]): + """A context manager that stores the current pass context, + usually it is a list of sizes to specialize. + """ + global _pass_context + prev_context = _pass_context + _pass_context = PassContext(runtime_shape) + try: + yield + finally: + _pass_context = prev_context + + +class InductorPass(CustomGraphPass): + """ + A custom graph pass that uses a hash of its source as the UUID. + This is defined as a convenience and should work in most cases. + """ + + def uuid(self) -> Any: + """ + Provide a unique identifier for the pass, used in Inductor code cache. + This should depend on the pass implementation, so that changes to the + pass result in recompilation. + By default, the object source is hashed. + """ + return InductorPass.hash_source(self) + + @staticmethod + def hash_source(*srcs: Union[str, Any]): + """ + Utility method to hash the sources of functions or objects. + :param srcs: strings or objects to add to the hash. + Objects and functions have their source inspected. + :return: + """ + hasher = hashlib.sha256() + for src in srcs: + if isinstance(src, str): + src_str = src + elif isinstance(src, types.FunctionType): + src_str = inspect.getsource(src) + else: + src_str = inspect.getsource(src.__class__) + hasher.update(src_str.encode("utf-8")) + return hasher.hexdigest() + + @staticmethod + def hash_dict(dict_: dict[Any, Any]): + """ + Utility method to hash a dictionary, can alternatively be used for uuid. + :return: A sha256 hash of the json rep of the dictionary. + """ + encoded = json.dumps(dict_, sort_keys=True).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + def is_applicable_for_shape(self, shape: Optional[int]): + return True + + +class CallableInductorPass(InductorPass): + """ + This class is a wrapper for a callable that automatically provides an + implementation of the UUID. + """ + + def __init__( + self, callable: Callable[[fx.Graph], None], uuid: Optional[Any] = None + ): + self.callable = callable + self._uuid = self.hash_source(callable) if uuid is None else uuid + + def __call__(self, graph: torch.fx.Graph): + self.callable(graph) + + def uuid(self) -> Any: + return self._uuid + + +class SGLangInductorPass(InductorPass): + + def __init__( + self, + ): + self.pass_name = self.__class__.__name__ + + def dump_graph(self, graph: torch.fx.Graph, stage: str): + lazy_format_graph_code(stage, graph.owning_module) + + def begin(self): + self._start_time = time.perf_counter_ns() + + def end_and_log(self): + self._end_time = time.perf_counter_ns() + duration_ms = float(self._end_time - self._start_time) / 1.0e6 + logger.debug("%s completed in %.1f ms", self.pass_name, duration_ms) + + +class PrinterInductorPass(SGLangInductorPass): + + def __init__(self, name: str): + super().__init__() + self.name = name + + def __call__(self, graph: torch.fx.Graph): + self.dump_graph(graph, self.name) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/pass_manager.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/pass_manager.py new file mode 100644 index 00000000..9173976f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/pass_manager.py @@ -0,0 +1,66 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.10.0/vllm/compilation/pass_manager.py + +import logging + +from torch import fx as fx + +from sglang.srt.compilation.fix_functionalization import FixFunctionalizationPass +from sglang.srt.compilation.inductor_pass import ( + CustomGraphPass, + InductorPass, + SGLangInductorPass, + get_pass_context, +) + +logger = logging.getLogger(__name__) + + +class PostGradPassManager(CustomGraphPass): + """ + The pass manager for post-grad passes. + It handles configuration, adding custom passes, and running passes. + It supports uuid for the Inductor code cache. That includes torch<2.6 + support using pickling (in .inductor_pass.CustomGraphPass). + + The order of the post-grad post-passes is: + 1. passes (constructor parameter) + 2. default passes (NoopEliminationPass, FusionPass) + 3. config["post_grad_custom_post_pass"] (if it exists) + 4. fix_functionalization + This way, all passes operate on a functionalized graph. + """ + + def __init__(self): + self.passes: list[SGLangInductorPass] = [] + + def __call__(self, graph: fx.Graph): + shape = get_pass_context().runtime_shape + for pass_ in self.passes: + if pass_.is_applicable_for_shape(shape): + pass_(graph) + + # always run fix_functionalization last + self.fix_functionalization(graph) + + def configure( + self, + ): + self.pass_config = dict() + self.fix_functionalization = FixFunctionalizationPass() + + def add(self, pass_: InductorPass): + assert isinstance(pass_, InductorPass) + self.passes.append(pass_) + + def uuid(self): + """ + The PostGradPassManager is set as a custom pass in the Inductor and + affects compilation caching. Its uuid depends on the UUIDs of all + dependent passes and the pass config. See InductorPass for more info. + """ + pass_manager_uuid = "fshdakhsa" + state = {"pass_config": pass_manager_uuid, "passes": []} + for pass_ in self.passes: + state["passes"].append(pass_.uuid()) + state["passes"].append(self.fix_functionalization.uuid()) + return InductorPass.hash_dict(state) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/piecewise_context_manager.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/piecewise_context_manager.py new file mode 100644 index 00000000..a0d3c7dd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/compilation/piecewise_context_manager.py @@ -0,0 +1,61 @@ +from contextlib import contextmanager +from dataclasses import dataclass +from typing import Any, List, Optional + +from sglang.srt.model_executor.forward_batch_info import ForwardBatch + +_in_piecewise_cuda_graph = False + + +def is_in_piecewise_cuda_graph(): + return _in_piecewise_cuda_graph + + +@contextmanager +def enable_piecewise_cuda_graph(): + global _in_piecewise_cuda_graph + _in_piecewise_cuda_graph = True + + yield + + _in_piecewise_cuda_graph = False + + +@dataclass +class ForwardContext: + def __init__(self): + self.forward_batch = None + self.attention_layer = None + + def set_forward_batch(self, forward_batch: ForwardBatch): + self.forward_batch = forward_batch + + def set_attention_layers(self, layers: List[Any]): + self.attention_layers = layers + + def set_quant_config(self, quant_config: Any): + self.quant_config = quant_config + + +_forward_context: Optional[ForwardContext] = None + + +def get_forward_context() -> Optional[ForwardContext]: + if _forward_context is None: + return None + return _forward_context + + +@contextmanager +def set_forward_context( + forward_batch: ForwardBatch, attention_layers: List[Any], quant_config: Any +): + global _forward_context + _forward_context = ForwardContext() + _forward_context.set_forward_batch(forward_batch) + _forward_context.set_attention_layers(attention_layers) + _forward_context.set_quant_config(quant_config) + try: + yield + finally: + _forward_context = None diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/__init__.py new file mode 100644 index 00000000..b35cc1dc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/__init__.py @@ -0,0 +1,47 @@ +from sglang.srt.configs.chatglm import ChatGLMConfig +from sglang.srt.configs.dbrx import DbrxConfig +from sglang.srt.configs.deepseekvl2 import DeepseekVL2Config +from sglang.srt.configs.dots_ocr import DotsOCRConfig +from sglang.srt.configs.dots_vlm import DotsVLMConfig +from sglang.srt.configs.exaone import ExaoneConfig +from sglang.srt.configs.falcon_h1 import FalconH1Config +from sglang.srt.configs.janus_pro import MultiModalityConfig +from sglang.srt.configs.jet_nemotron import JetNemotronConfig +from sglang.srt.configs.jet_vlm import JetVLMConfig +from sglang.srt.configs.kimi_linear import KimiLinearConfig +from sglang.srt.configs.kimi_vl import KimiVLConfig +from sglang.srt.configs.kimi_vl_moonvit import MoonViTConfig +from sglang.srt.configs.longcat_flash import LongcatFlashConfig +from sglang.srt.configs.nano_nemotron_vl import NemotronH_Nano_VL_V2_Config +from sglang.srt.configs.nemotron_h import NemotronHConfig +from sglang.srt.configs.olmo3 import Olmo3Config +from sglang.srt.configs.qwen3_next import Qwen3NextConfig +from sglang.srt.configs.step3_vl import ( + Step3TextConfig, + Step3VisionEncoderConfig, + Step3VLConfig, +) + +__all__ = [ + "ExaoneConfig", + "ChatGLMConfig", + "DbrxConfig", + "DeepseekVL2Config", + "LongcatFlashConfig", + "MultiModalityConfig", + "KimiVLConfig", + "MoonViTConfig", + "Step3VLConfig", + "Step3TextConfig", + "Step3VisionEncoderConfig", + "Olmo3Config", + "KimiLinearConfig", + "Qwen3NextConfig", + "DotsVLMConfig", + "DotsOCRConfig", + "FalconH1Config", + "NemotronHConfig", + "NemotronH_Nano_VL_V2_Config", + "JetNemotronConfig", + "JetVLMConfig", +] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/chatglm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/chatglm.py new file mode 100644 index 00000000..9370c218 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/chatglm.py @@ -0,0 +1,78 @@ +# Adapted from +# https://github.com/THUDM/ChatGLM2-6B +# https://github.com/vllm-project/vllm/blob/main/vllm/transformers_utils/configs/chatglm.py + +# ChatGLM2 and ChatGLM3 share the same config. +# ChatGLM4 is officially supported by Huggingface +# transformers >= 4.46.0 is required +# https://huggingface.co/docs/transformers/en/model_doc/glm +from transformers import PretrainedConfig + + +class ChatGLMConfig(PretrainedConfig): + model_type = "chatglm" + attribute_map = { + "num_hidden_layers": "num_layers", + "n_head_kv": "multi_query_group_num", + } + + def __init__( + self, + num_layers=28, + padded_vocab_size=65024, + hidden_size=4096, + ffn_hidden_size=13696, + kv_channels=128, + num_attention_heads=32, + seq_length=2048, + hidden_dropout=0.0, + attention_dropout=0.0, + layernorm_epsilon=1e-5, + rmsnorm=True, + apply_residual_connection_post_layernorm=False, + post_layer_norm=True, + add_bias_linear=False, + add_qkv_bias=False, + interleaved_qkv=False, + bias_dropout_fusion=True, + multi_query_attention=False, + multi_query_group_num=1, + apply_query_key_layer_scaling=True, + attention_softmax_in_fp32=True, + fp32_residual_connection=False, + quantization_bit=0, + pre_seq_len=None, + prefix_projection=False, + **kwargs + ): + self.num_layers = num_layers + self.vocab_size = padded_vocab_size + self.padded_vocab_size = padded_vocab_size + self.hidden_size = hidden_size + self.ffn_hidden_size = ffn_hidden_size + self.kv_channels = kv_channels + self.num_attention_heads = num_attention_heads + self.seq_length = seq_length + # It is to be compatible with long lora. + self.max_position_embeddings = seq_length + self.hidden_dropout = hidden_dropout + self.attention_dropout = attention_dropout + self.layernorm_epsilon = layernorm_epsilon + self.rmsnorm = rmsnorm + self.apply_residual_connection_post_layernorm = ( + apply_residual_connection_post_layernorm + ) + self.post_layer_norm = post_layer_norm + self.add_bias_linear = add_bias_linear + self.add_qkv_bias = add_qkv_bias + self.bias_dropout_fusion = bias_dropout_fusion + self.multi_query_attention = multi_query_attention + self.multi_query_group_num = multi_query_group_num + self.apply_query_key_layer_scaling = apply_query_key_layer_scaling + self.attention_softmax_in_fp32 = attention_softmax_in_fp32 + self.fp32_residual_connection = fp32_residual_connection + self.quantization_bit = quantization_bit + self.pre_seq_len = pre_seq_len + self.prefix_projection = prefix_projection + self.interleaved_qkv = interleaved_qkv + super().__init__(**kwargs) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/dbrx.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/dbrx.py new file mode 100644 index 00000000..75ccbde9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/dbrx.py @@ -0,0 +1,279 @@ +# Adapted from +# https://huggingface.co/databricks/dbrx-base/blob/main/configuration_dbrx.py +# https://github.com/vllm-project/vllm/blob/main/vllm/transformers_utils/configs/dbrx.py +"""Dbrx configuration.""" + +from typing import Any, Optional + +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + +logger = logging.get_logger(__name__) + +DBRX_PRETRAINED_CONFIG_ARCHIVE_MAP = {} # type: ignore + + +class DbrxAttentionConfig(PretrainedConfig): + """Configuration class for Dbrx Attention. + + [`DbrxAttention`] class. It is used to instantiate attention layers + according to the specified arguments, defining the layers architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + attn_pdrop (`float`, *optional*, defaults to 0.0): + The dropout probability for the attention layers. + clip_qkv (`float`, *optional*, defaults to None): + If not `None`, clip the queries, keys, and values in the attention layer to this value. + kv_n_heads (Optional[int]): For grouped_query_attention only, allow user to specify number of kv heads. + rope_theta (float): The base frequency for rope. + """ + + def __init__( + self, + attn_pdrop: float = 0, + clip_qkv: Optional[float] = None, + kv_n_heads: int = 1, + rope_theta: float = 10000.0, + **kwargs: Any, + ): + super().__init__(**kwargs) + self.attn_pdrop = attn_pdrop + self.clip_qkv = clip_qkv + self.kv_n_heads = kv_n_heads + self.rope_theta = rope_theta + + for k in ["model_type"]: + if k in kwargs: + kwargs.pop(k) + if len(kwargs) != 0: + raise ValueError(f"Found unknown {kwargs=}") + + @classmethod + def from_pretrained( + cls, pretrained_model_name_or_path: str, **kwargs: Any + ) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + + config_dict, kwargs = cls.get_config_dict( + pretrained_model_name_or_path, **kwargs + ) + + if config_dict.get("model_type") == "dbrx": + config_dict = config_dict["attn_config"] + + if ( + "model_type" in config_dict + and hasattr(cls, "model_type") + and config_dict["model_type"] != cls.model_type + ): + logger.warning( + "You are using a model of type %s to instantiate a model of " + "type %s. This is not supported for all configurations of " + "models and can yield errors.", + config_dict["model_type"], + cls.model_type, + ) + + return cls.from_dict(config_dict, **kwargs) + + +class DbrxFFNConfig(PretrainedConfig): + """Configuration class for Dbrx FFN. + + [`DbrxFFN`] class. It is used to instantiate feedforward layers according to + the specified arguments, defining the layers architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + ffn_act_fn (dict, optional): A dict specifying activation function for the FFN. + The dict should have a key 'name' with the value being the name of + the activation function along with any additional keyword arguments. + ffn_hidden_size (int, optional): The hidden size of the feedforward network. + moe_num_experts (int, optional): The number of experts in the mixture of experts layer. + moe_top_k (int, optional): The number of experts to use in the mixture of experts layer. + moe_jitter_eps (float, optional): The jitter epsilon for the mixture of experts layer. + moe_loss_weight (float, optional): The loss weight for the mixture of experts layer. + moe_normalize_expert_weights (float, optional): The normalization factor for the expert weights. + uniform_expert_assignment (bool, optional): Whether to use uniform expert assignment. + This should only be used for benchmarking purposes. + """ + + def __init__( + self, + ffn_act_fn: Optional[dict] = None, + ffn_hidden_size: int = 3584, + moe_num_experts: int = 4, + moe_top_k: int = 1, + moe_jitter_eps: Optional[float] = None, + moe_loss_weight: float = 0.01, + moe_normalize_expert_weights: Optional[float] = 1, + uniform_expert_assignment: bool = False, + **kwargs: Any, + ): + super().__init__() + if ffn_act_fn is None: + ffn_act_fn = {"name": "silu"} + self.ffn_act_fn = ffn_act_fn + self.ffn_hidden_size = ffn_hidden_size + self.moe_num_experts = moe_num_experts + self.moe_top_k = moe_top_k + self.moe_jitter_eps = moe_jitter_eps + self.moe_loss_weight = moe_loss_weight + self.moe_normalize_expert_weights = moe_normalize_expert_weights + self.uniform_expert_assignment = uniform_expert_assignment + + for k in ["model_type"]: + if k in kwargs: + kwargs.pop(k) + if len(kwargs) != 0: + raise ValueError(f"Found unknown {kwargs=}") + + @classmethod + def from_pretrained( + cls, pretrained_model_name_or_path: str, **kwargs: Any + ) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + + config_dict, kwargs = cls.get_config_dict( + pretrained_model_name_or_path, **kwargs + ) + + if config_dict.get("model_type") == "dbrx": + config_dict = config_dict["ffn_config"] + + if ( + "model_type" in config_dict + and hasattr(cls, "model_type") + and config_dict["model_type"] != cls.model_type + ): + logger.warning( + "You are using a model of type %s to instantiate a model of " + "type %s. This is not supported for all " + "configurations of models and can yield errors.", + config_dict["model_type"], + cls.model_type, + ) + + return cls.from_dict(config_dict, **kwargs) + + +class DbrxConfig(PretrainedConfig): + """Configuration class for Dbrx. + + [`DbrxModel`]. It is used to instantiate a Dbrx model according to the + specified arguments, defining the model architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + d_model (`int`, *optional*, defaults to 6144): + Dimensionality of the embeddings and hidden states. + n_heads (`int`, *optional*, defaults to 48): + Number of attention heads for each attention layer in the Transformer encoder. + n_layers (`int`, *optional*, defaults to 40): + Number of hidden layers in the Transformer encoder. + max_seq_len (`int`, *optional*, defaults to 32768): + The maximum sequence length of the model. + vocab_size (`int`, *optional*, defaults to 100352): + Vocabulary size of the Dbrx model. Defines the maximum number of different tokens that can be represented by + the `inputs_ids` passed when calling [`DbrxModel`]. + resid_pdrop (`float`, *optional*, defaults to 0.0): + The dropout probability applied to the attention output before combining with residual. + emb_pdrop (`float`, *optional*, defaults to 0.0): + The dropout probability for the embedding layer. + attn_config (`dict`, *optional*): + A dictionary used to configure the model's attention module. + ffn_config (`dict`, *optional*): + A dictionary used to configure the model's FFN module. + use_cache (`bool`, *optional*, defaults to `False`): + Whether or not the model should return the last key/values attentions (not used by all models). + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + output_router_logits (`bool`, *optional*, defaults to `False`): + Whether or not the router logits should be returned by the model. Enabling this will also + allow the model to output the auxiliary loss. See [here]() for more details + router_aux_loss_coef (`float`, *optional*, defaults to 0.001): + The aux loss factor for the total loss. + + + Example: + ```python + >>> from transformers import DbrxConfig, DbrxModel + + >>> # Initializing a Dbrx configuration + >>> configuration = DbrxConfig() + + >>> # Initializing a model (with random weights) from the configuration + >>> model = DbrxModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ``` + """ + + model_type = "dbrx" + attribute_map = { + "num_attention_heads": "n_heads", + "hidden_size": "d_model", + "num_hidden_layers": "n_layers", + "max_position_embeddings": "max_seq_len", + } + + def __init__( + self, + d_model: int = 2048, + n_heads: int = 16, + n_layers: int = 24, + max_seq_len: int = 2048, + vocab_size: int = 32000, + resid_pdrop: float = 0.0, + emb_pdrop: float = 0.0, + attn_config: Optional[DbrxAttentionConfig] = None, + ffn_config: Optional[DbrxFFNConfig] = None, + use_cache: bool = True, + initializer_range: float = 0.02, + output_router_logits: bool = False, + router_aux_loss_coef: float = 0.05, + **kwargs: Any, + ): + if attn_config is None: + self.attn_config = DbrxAttentionConfig() + elif isinstance(attn_config, dict): + self.attn_config = DbrxAttentionConfig(**attn_config) + else: + self.attn_config = attn_config + + if ffn_config is None: + self.ffn_config = DbrxFFNConfig() + elif isinstance(ffn_config, dict): + self.ffn_config = DbrxFFNConfig(**ffn_config) + else: + self.ffn_config = ffn_config + + self.d_model = d_model + self.n_heads = n_heads + self.n_layers = n_layers + self.max_seq_len = max_seq_len + self.vocab_size = vocab_size + self.resid_pdrop = resid_pdrop + self.emb_pdrop = emb_pdrop + self.use_cache = use_cache + self.initializer_range = initializer_range + self.output_router_logits = output_router_logits + self.router_aux_loss_coef = router_aux_loss_coef + + tie_word_embeddings = kwargs.pop("tie_word_embeddings", False) + if tie_word_embeddings: + raise ValueError("tie_word_embeddings is not supported for Dbrx models.") + + super().__init__( + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/deepseek_ocr.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/deepseek_ocr.py new file mode 100644 index 00000000..b1f2488d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/deepseek_ocr.py @@ -0,0 +1,794 @@ +import math +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple + +import torch +from PIL import Image, ImageOps +from transformers import ( + AutoProcessor, + LlamaTokenizerFast, + PretrainedConfig, + ProcessorMixin, +) + +from sglang.srt.multimodal.customized_mm_processor_utils import ( + register_customized_processor, +) +from sglang.srt.sampling.custom_logit_processor import ( + DeepseekOCRNoRepeatNGramLogitProcessor, +) + +BASE_SIZE = 1024 +IMAGE_SIZE = 640 +CROP_MODE = True +MIN_CROPS = 2 +MAX_CROPS = 6 # max:9; If your GPU memory is small, it is recommended to set it to 6. +MAX_CONCURRENCY = 100 # If you have limited GPU memory, lower the concurrency count. +NUM_WORKERS = 64 # image pre-process (resize/padding) workers +PRINT_NUM_VIS_TOKENS = False +SKIP_REPEAT = True +MODEL_PATH = "deepseek-ai/DeepSeek-OCR" # change to your model path + +NGRAM_NO_REPEAT_SIZE = 30 +NGRAM_NO_REPEAT_WINDOW = 90 +# Whitelist `` and `` token ids to allow table structures. +NGRAM_NO_REPEAT_WHITELIST = (128821, 128822) + +DEFAULT_CUSTOM_LOGIT_PROCESSOR = DeepseekOCRNoRepeatNGramLogitProcessor.to_str() + + +def get_default_ngram_custom_params() -> Dict[str, Any]: + """Return default custom params for the DeepSeek-OCR n-gram no repeat processor.""" + + return { + "ngram_size": NGRAM_NO_REPEAT_SIZE, + "window_size": NGRAM_NO_REPEAT_WINDOW, + "whitelist_token_ids": list(NGRAM_NO_REPEAT_WHITELIST), + } + + +PROMPT = "\n<|grounding|>Convert the document to markdown." + + +class DictOutput(object): + def items(self): + return self.__dict__.items() + + def keys(self): + return self.__dict__.keys() + + def __getitem__(self, item): + return self.__dict__[item] + + def __contains__(self, key): + return key in self.__dict__ + + def __setitem__(self, key, value): + self.__dict__[key] = value + + +@dataclass +class VLChatProcessorOutput(DictOutput): + input_ids: torch.LongTensor + target_ids: torch.LongTensor + images_crop: torch.LongTensor + pixel_values: ( + torch.Tensor + ) # rename from "images" to "pixel_values" for compatibility + images_seq_mask: torch.BoolTensor + images_spatial_crop: torch.LongTensor + + def __len__(self): + return len(self.input_ids) + + +class ImageTransform(object): + def __init__( + self, + mean: Optional[Tuple[float, float, float]] = (0.5, 0.5, 0.5), + std: Optional[Tuple[float, float, float]] = (0.5, 0.5, 0.5), + normalize: bool = True, + ): + self.mean = mean + self.std = std + self.normalize = normalize + + # only load torchvision.transforms when needed + try: + import torchvision.transforms as T + + # FIXME: add version check for gguf + except ImportError as err: + raise ImportError( + "Please install torchvision via `pip install torchvision` to use Deepseek-VL2." + ) from err + + transform_pipelines = [T.ToTensor()] + + if normalize: + transform_pipelines.append(T.Normalize(mean, std)) + + self.transform = T.Compose(transform_pipelines) + + def __call__(self, pil_img: Image.Image): + x = self.transform(pil_img) + return x + + +def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size): + best_ratio_diff = float("inf") + best_ratio = (1, 1) + area = width * height + for ratio in target_ratios: + target_aspect_ratio = ratio[0] / ratio[1] + ratio_diff = abs(aspect_ratio - target_aspect_ratio) + if ratio_diff < best_ratio_diff: + best_ratio_diff = ratio_diff + best_ratio = ratio + elif ratio_diff == best_ratio_diff: + if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]: + best_ratio = ratio + return best_ratio + + +def dynamic_preprocess( + image, min_num=MIN_CROPS, max_num=MAX_CROPS, image_size=640, use_thumbnail=False +): + orig_width, orig_height = image.size + aspect_ratio = orig_width / orig_height + + # calculate the existing image aspect ratio + target_ratios = set( + (i, j) + for n in range(min_num, max_num + 1) + for i in range(1, n + 1) + for j in range(1, n + 1) + if i * j <= max_num and i * j >= min_num + ) + target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1]) + + # find the closest aspect ratio to the target + target_aspect_ratio = find_closest_aspect_ratio( + aspect_ratio, target_ratios, orig_width, orig_height, image_size + ) + + # calculate the target width and height + target_width = image_size * target_aspect_ratio[0] + target_height = image_size * target_aspect_ratio[1] + blocks = target_aspect_ratio[0] * target_aspect_ratio[1] + + # resize the image + resized_img = image.resize((target_width, target_height)) + processed_images = [] + for i in range(blocks): + box = ( + (i % (target_width // image_size)) * image_size, + (i // (target_width // image_size)) * image_size, + ((i % (target_width // image_size)) + 1) * image_size, + ((i // (target_width // image_size)) + 1) * image_size, + ) + # split the image + split_img = resized_img.crop(box) + processed_images.append(split_img) + assert len(processed_images) == blocks + if use_thumbnail and len(processed_images) != 1: + thumbnail_img = image.resize((image_size, image_size)) + processed_images.append(thumbnail_img) + return processed_images, target_aspect_ratio + + +class DeepseekOCRProcessor(ProcessorMixin): + tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast") + attributes = ["tokenizer"] + + def __init__( + self, + tokenizer: LlamaTokenizerFast, + candidate_resolutions: Tuple[Tuple[int, int]], + patch_size: int, + downsample_ratio: int, + image_mean: Tuple[float, float, float] = (0.5, 0.5, 0.5), + image_std: Tuple[float, float, float] = (0.5, 0.5, 0.5), + normalize: bool = True, + image_token: str = "", + pad_token: str = "<｜▁pad▁｜>", + add_special_token: bool = False, + sft_format: str = "deepseek", + mask_prompt: bool = True, + ignore_id: int = -100, + **kwargs, + ): + + self.candidate_resolutions = candidate_resolutions + self.image_size = candidate_resolutions[0][0] + self.patch_size = patch_size + self.image_mean = image_mean + self.image_std = image_std + self.normalize = normalize + self.downsample_ratio = downsample_ratio + self.base_size = BASE_SIZE + self.image_transform = ImageTransform( + mean=image_mean, std=image_std, normalize=normalize + ) + self.tokenizer = tokenizer + # must set this，padding side with make a difference in batch inference + self.tokenizer.padding_side = "left" + + # add the pad_token as special token to use 'tokenizer.pad_token' and 'tokenizer.pad_token_id' + if tokenizer.pad_token is None: + self.tokenizer.add_special_tokens({"pad_token": pad_token}) + + # add image token + image_token_id = self.tokenizer.vocab.get(image_token) + if image_token_id is None: + special_tokens = [image_token] + special_tokens_dict = {"additional_special_tokens": special_tokens} + self.tokenizer.add_special_tokens(special_tokens_dict) + self.image_token_id = self.tokenizer.vocab.get(image_token) + + # add five special tokens for grounding-related tasks + # <|ref|>, <|/ref|>, <|det|>, <|/det|>, <|grounding|> + special_tokens = ["<|ref|>", "<|/ref|>", "<|det|>", "<|/det|>", "<|grounding|>"] + special_tokens_dict = {"additional_special_tokens": special_tokens} + self.tokenizer.add_special_tokens(special_tokens_dict) + + # add special tokens for SFT data + special_tokens = ["<|User|>", "<|Assistant|>"] + special_tokens_dict = {"additional_special_tokens": special_tokens} + self.tokenizer.add_special_tokens(special_tokens_dict) + + self.image_token = image_token + self.pad_token = pad_token + self.add_special_token = add_special_token + self.sft_format = sft_format + self.mask_prompt = mask_prompt + self.ignore_id = ignore_id + + super().__init__( + tokenizer, + **kwargs, + ) + + def format_messages_v2(self, messages: str, pil_images, max_req_input_len=-1): + """play the role of format_messages_v2 and get_images_info in the last version""" + tokenized_data = [] + masked_tokenized_data = [] # labels + images_list = [] + images_seq_mask = [] + images_spatial_crop = [] + + image_index = 0 + image_token_cnt = messages.count(self.image_token) + ( + input_ids, + images, + images_crop, + seq_mask, + spatial_crop, + num_image_tokens, + image_shapes, + ) = self.tokenize_with_images( + messages, + pil_images[image_index : image_index + image_token_cnt], + bos=True, + eos=True, + cropping=len(pil_images) <= 2, + ) + + image_index = image_token_cnt + images_list += images + images_seq_mask += seq_mask + images_spatial_crop = spatial_crop + + return ( + input_ids, + masked_tokenized_data, + images_list, + images_seq_mask, + images_spatial_crop, + images_crop, + ) + + @property + def bos_id(self): + return self.tokenizer.bos_token_id + + @property + def eos_id(self): + return self.tokenizer.eos_token_id + + @property + def pad_id(self): + return self.tokenizer.pad_token_id + + def encode(self, text: str, bos: bool = True, eos: bool = False): + t = self.tokenizer.encode(text, add_special_tokens=False) + + if bos: + t = [self.bos_id] + t + if eos: + t = t + [self.eos_id] + + return t + + def decode(self, t: List[int], **kwargs) -> str: + return self.tokenizer.decode(t, **kwargs) + + def process_one( + self, + prompt: str = None, + conversations: List[Dict[str, str]] = None, + images: List[Image.Image] = None, + apply_sft_format: bool = False, + inference_mode: bool = True, + system_prompt: str = "", + max_req_input_len: int = -1, + cropping: bool = True, + **kwargs, + ): + """ + + Args: + prompt (str): the formatted prompt; + conversations (List[Dict]): conversations with a list of messages; + images (List[ImageType]): the list of images; + apply_sft_format (bool): if prompt is not None, then apply the SFT format to prompt; + if conversations is not None, then it will always apply the SFT format to conversations; + inference_mode (bool): if True, then remove the last eos token; + system_prompt (str): the system prompt; + **kwargs: + + Returns: + outputs (BaseProcessorOutput): the output of the processor, + - input_ids (torch.LongTensor): [N + image tokens] + - target_ids (torch.LongTensor): [N + image tokens] + - images (torch.FloatTensor): [n_images, 3, H, W] + - image_id (int): the id of the image token + - num_image_tokens (List[int]): the number of image tokens + """ + + prompt = conversations or prompt + ( + input_ids, + masked_tokenized_str, + images_list, + images_seq_mask, + images_spatial_crop, + images_crop, + ) = self.format_messages_v2(prompt, images, max_req_input_len) + + target_ids = torch.LongTensor(masked_tokenized_str) + + if len(images_list) == 0: + images = torch.zeros((1, 3, self.image_size, self.image_size)) + else: + images = torch.stack(images_list, dim=0) + + images_spatial_crop = torch.stack( + [images_spatial_crop], dim=0 + ) # stack the tensor to make it a batch of 1 + + prepare = VLChatProcessorOutput( + input_ids=input_ids, + target_ids=target_ids, + images_crop=images_crop, + pixel_values=images, + images_seq_mask=images_seq_mask, + images_spatial_crop=images_spatial_crop, + ) + + return prepare + + def __call__( + self, + *, + prompt: str = None, + conversations: List[Dict[str, str]] = None, + images: List[Image.Image] = None, + apply_sft_format: bool = False, + inference_mode: bool = True, + system_prompt: str = "", + max_req_input_len: int = -1, + text: list[str] = None, + **kwargs, + ): + assert text is None or isinstance(text, list) + if text is not None: + text = text[0] + prepare = self.process_one( + prompt=prompt or text, + conversations=conversations, + images=images, + apply_sft_format=apply_sft_format, + inference_mode=inference_mode, + system_prompt=system_prompt, + max_req_input_len=max_req_input_len, + ) + + return prepare + + def find_all_indices(self, messages, target_value): + indices = [] + for index, item in enumerate(messages): + if item == target_value: + indices.append(index) + return indices + + def tokenize_with_images( + self, + conversation: str, + images: List[Image.Image], + bos: bool = True, + eos: bool = True, + cropping: bool = True, + ): + """Tokenize text with tags.""" + + conversation = conversation + assert conversation.count(self.image_token) == len(images) + text_splits = conversation.split(self.image_token) + images_list, images_crop_list, images_seq_mask, images_spatial_crop = ( + [], + [], + [], + [], + ) + image_shapes = [] + num_image_tokens = [] + tokenized_str = [] + for text_sep, image in zip(text_splits, images): + """encode text_sep""" + tokenized_sep = self.encode(text_sep, bos=False, eos=False) + + tokenized_str += tokenized_sep + images_seq_mask += [False] * len(tokenized_sep) + + image_shapes.append(image.size) + + if image.size[0] <= 640 and image.size[1] <= 640: + crop_ratio = [1, 1] + else: + if cropping: + images_crop_raw, crop_ratio = dynamic_preprocess( + image, image_size=IMAGE_SIZE + ) + else: + crop_ratio = [1, 1] + + """process the global view""" + if self.image_size <= 640 and not cropping: + image = image.resize((self.image_size, self.image_size)) + + global_view = ImageOps.pad( + image, + (self.base_size, self.base_size), + color=tuple(int(x * 255) for x in self.image_transform.mean), + ) + images_list.append(self.image_transform(global_view)) + + num_width_tiles, num_height_tiles = crop_ratio + images_spatial_crop.append([num_width_tiles, num_height_tiles]) + + if num_width_tiles > 1 or num_height_tiles > 1: + for i in range(len(images_crop_raw)): + images_crop_list.append(self.image_transform(images_crop_raw[i])) + + """add image tokens""" + num_queries = math.ceil( + (self.image_size // self.patch_size) / self.downsample_ratio + ) + num_queries_base = math.ceil( + (self.base_size // self.patch_size) / self.downsample_ratio + ) + + tokenized_image = ( + [self.image_token_id] * num_queries_base + [self.image_token_id] + ) * num_queries_base + tokenized_image += [self.image_token_id] + if num_width_tiles > 1 or num_height_tiles > 1: + tokenized_image += ( + [self.image_token_id] * (num_queries * num_width_tiles) + + [self.image_token_id] + ) * (num_queries * num_height_tiles) + tokenized_str += tokenized_image + + images_seq_mask += [True] * len(tokenized_image) + num_image_tokens.append(len(tokenized_image)) + + """process the last text split""" + tokenized_sep = self.encode(text_splits[-1], bos=False, eos=False) + + tokenized_str += tokenized_sep + images_seq_mask += [False] * len(tokenized_sep) + + """add the bos and eos tokens""" + if bos: + tokenized_str = [self.bos_id] + tokenized_str + images_seq_mask = [False] + images_seq_mask + if eos: + tokenized_str = tokenized_str + [self.eos_id] + images_seq_mask = images_seq_mask + [False] + + assert len(tokenized_str) == len( + images_seq_mask + ), f"tokenize_with_images func: tokenized_str's length {len(tokenized_str)} is not equal to imags_seq_mask's length {len(images_seq_mask)}" + + masked_tokenized_str = [] + for token_index in tokenized_str: + if token_index != self.image_token_id: + masked_tokenized_str.append(token_index) + else: + masked_tokenized_str.append(self.ignore_id) + + assert ( + len(tokenized_str) == len(images_seq_mask) == len(masked_tokenized_str) + ), ( + f"tokenized_str's length {len(tokenized_str)}, input_ids' length {len(masked_tokenized_str)}, " + f"imags_seq_mask's length {len(images_seq_mask)}, are not equal" + ) + input_ids = torch.LongTensor(tokenized_str) + target_ids = torch.LongTensor(masked_tokenized_str) + images_seq_mask = torch.tensor(images_seq_mask, dtype=torch.bool) + + # set input_ids < 0 | input_ids == self.image_token_id as ignore_id + target_ids[(input_ids < 0) | (input_ids == self.image_token_id)] = ( + self.ignore_id + ) + input_ids[input_ids < 0] = self.pad_id + + inference_mode = True + + if inference_mode: + # Remove the ending eos token + assert input_ids[-1] == self.eos_id + input_ids = input_ids[:-1] + target_ids = target_ids[:-1] + images_seq_mask = images_seq_mask[:-1] + + if len(images_list) == 0: + pixel_values = torch.zeros((1, 3, self.base_size, self.base_size)) + images_spatial_crop = torch.zeros((1, 1), dtype=torch.long) + images_crop = torch.zeros( + (1, 3, self.image_size, self.image_size) + ).unsqueeze(0) + else: + pixel_values = torch.stack(images_list, dim=0) + images_spatial_crop = torch.tensor(images_spatial_crop, dtype=torch.long) + if images_crop_list: + images_crop = torch.stack(images_crop_list, dim=0).unsqueeze(0) + else: + images_crop = torch.zeros( + (1, 3, self.image_size, self.image_size) + ).unsqueeze(0) + + input_ids = input_ids.unsqueeze(0) + return ( + input_ids, + pixel_values, + images_crop, + images_seq_mask, + images_spatial_crop, + num_image_tokens, + image_shapes, + ) + + +class VisionEncoderConfig(PretrainedConfig): + model_type: str = "vision" + + model_name: str = "vit_so400m_patch14_siglip_384.webli" + image_size: int = 384 + patch_size: int = 16 + width: int = 1024 + layers: int = 24 + heads: int = 16 + mlp_ratio: int = 4 + global_pool: str = "map" + ignore_head: bool = True + class_token: bool = False + num_classes: int = 0 + use_checkpoint: bool = False + weight_init: str = "skip" + deterministic: bool = False + num_recomputing_layers: int = 0 + + def __init__( + self, + model_name: str = "vit_so400m_patch14_siglip_384.webli", + image_size: int = 384, + patch_size: int = 16, + width: int = 1024, + layers: int = 24, + heads: int = 16, + mlp_ratio: int = 4, + global_pool: str = "map", + ignore_head: bool = True, + class_token: bool = False, + num_classes: int = 0, + use_checkpoint: bool = False, + **kwargs, + ): + self.model_name = model_name + self.image_size = image_size + self.patch_size = patch_size + self.width = width + self.layers = layers + self.heads = heads + self.mlp_ratio = mlp_ratio + self.global_pool = global_pool + self.ignore_head = ignore_head + self.class_token = class_token + self.num_classes = num_classes + self.use_checkpoint = use_checkpoint + + super().__init__(**kwargs) + + +class MlpProjectorConfig(PretrainedConfig): + model_type = "mlp_projector" + projector_type: str = "downsample_mlp_gelu" + input_dim: int = 1152 + n_embed: int = 2048 + depth: int = 2 + mlp_ratio: int = 1 + downsample_ratio: int = 2 + token_pooling: bool = False + + def __init__( + self, + projector_type: str = "downsample_mlp_gelu", + input_dim: int = 1152, + n_embed: int = 2048, + depth: int = 2, + mlp_ratio: int = 1, + downsample_ratio: int = 2, + **kwargs, + ): + self.projector_type = projector_type + self.input_dim = input_dim + self.n_embed = n_embed + self.depth = depth + self.mlp_ratio = mlp_ratio + self.downsample_ratio = downsample_ratio + + super().__init__(**kwargs) + + +class DeepseekV2Config(PretrainedConfig): + model_type = "deepseek_v2" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + vocab_size=102400, + hidden_size=4096, + intermediate_size=11008, + moe_intermediate_size=1407, + num_hidden_layers=30, + num_attention_heads=32, + num_key_value_heads=32, + n_shared_experts=None, + n_routed_experts=None, + ep_size=1, + routed_scaling_factor=1.0, + kv_lora_rank=512, + q_lora_rank=1536, + qk_rope_head_dim=64, + v_head_dim=128, + qk_nope_head_dim=128, + topk_method="gready", + n_group=None, + topk_group=None, + num_experts_per_tok=None, + moe_layer_freq=1, + first_k_dense_replace=0, + norm_topk_prob=False, + scoring_func="softmax", + aux_loss_alpha=0.001, + seq_aux=True, + hidden_act="silu", + max_position_embeddings=2048, + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + pad_token_id=None, + bos_token_id=100000, + eos_token_id=100001, + pretraining_tp=1, + tie_word_embeddings=False, + rope_theta=10000.0, + rope_scaling=None, + attention_bias=False, + attention_dropout=0.0, + use_mla=True, + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.moe_intermediate_size = moe_intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.n_shared_experts = n_shared_experts + self.n_routed_experts = n_routed_experts + self.ep_size = ep_size + self.routed_scaling_factor = routed_scaling_factor + self.kv_lora_rank = kv_lora_rank + self.q_lora_rank = q_lora_rank + self.qk_rope_head_dim = qk_rope_head_dim + self.v_head_dim = v_head_dim + self.qk_nope_head_dim = qk_nope_head_dim + self.topk_method = topk_method + self.n_group = n_group + self.topk_group = topk_group + self.num_experts_per_tok = num_experts_per_tok + self.moe_layer_freq = moe_layer_freq + self.first_k_dense_replace = first_k_dense_replace + self.norm_topk_prob = norm_topk_prob + self.scoring_func = scoring_func + self.aux_loss_alpha = aux_loss_alpha + self.seq_aux = seq_aux + # for backward compatibility + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = float(rms_norm_eps) + self.pretraining_tp = pretraining_tp + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + self.attention_bias = attention_bias + self.attention_dropout = attention_dropout + self.use_mla = use_mla + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + +@register_customized_processor(processor_class=DeepseekOCRProcessor) +class DeepseekVLV2Config(PretrainedConfig): + # model_type = "deepseek_vl_v2" + model_type = "deepseek-ocr" + vision_config: VisionEncoderConfig + projector_config: MlpProjectorConfig + + tile_tag: str = "2D" + global_view_pos: str = "head" + candidate_resolutions: tuple[tuple[int, int]] = ((384, 384),) + customized_processor_type: type[Any] = DeepseekOCRProcessor + + def __init__( + self, + tile_tag: str = "tile_tag", + global_view_pos: str = "head", + candidate_resolutions: tuple[tuple[int, int]] = ((384, 384),), + **kwargs, + ): + super().__init__(**kwargs) + + vision_config = kwargs.get("vision_config", {}) + self.vision_config = VisionEncoderConfig(**vision_config) + + projector_config = kwargs.get("projector_config", {}) + self.projector_config = MlpProjectorConfig(**projector_config) + + language_config = kwargs.get("language_config", {}) + self.text_config = DeepseekV2Config(**language_config) + + self.tile_tag = tile_tag + self.global_view_pos = global_view_pos + self.candidate_resolutions = candidate_resolutions + self.vocab_size = self.text_config.vocab_size + self.hidden_size = self.text_config.hidden_size + + +AutoProcessor.register(DeepseekVLV2Config, DeepseekOCRProcessor) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/deepseekvl2.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/deepseekvl2.py new file mode 100644 index 00000000..9621f058 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/deepseekvl2.py @@ -0,0 +1,687 @@ +import math +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple + +import torch +from PIL import Image, ImageOps +from transformers import ( + AutoProcessor, + LlamaTokenizerFast, + PretrainedConfig, + ProcessorMixin, +) + + +def select_best_resolution(image_size, candidate_resolutions): + # used for cropping + original_width, original_height = image_size + best_fit = None + max_effective_resolution = 0 + min_wasted_resolution = float("inf") + + for width, height in candidate_resolutions: + scale = min(width / original_width, height / original_height) + downscaled_width, downscaled_height = int(original_width * scale), int( + original_height * scale + ) + effective_resolution = min( + downscaled_width * downscaled_height, original_width * original_height + ) + wasted_resolution = (width * height) - effective_resolution + + if effective_resolution > max_effective_resolution or ( + effective_resolution == max_effective_resolution + and wasted_resolution < min_wasted_resolution + ): + max_effective_resolution = effective_resolution + min_wasted_resolution = wasted_resolution + best_fit = (width, height) + + return best_fit + + +class DictOutput(object): + def items(self): + return self.__dict__.items() + + def keys(self): + return self.__dict__.keys() + + def __getitem__(self, item): + return self.__dict__[item] + + def __contains__(self, key): + return key in self.__dict__ + + def __setitem__(self, key, value): + self.__dict__[key] = value + + +@dataclass +class VLChatProcessorOutput(DictOutput): + input_ids: torch.LongTensor + target_ids: torch.LongTensor + pixel_values: ( + torch.Tensor + ) # rename from "images" to "pixel_values" for compatibility + images_seq_mask: torch.BoolTensor + images_spatial_crop: torch.LongTensor + + def __len__(self): + return len(self.input_ids) + + +class ImageTransform(object): + def __init__( + self, + mean: Optional[Tuple[float, float, float]] = (0.5, 0.5, 0.5), + std: Optional[Tuple[float, float, float]] = (0.5, 0.5, 0.5), + normalize: bool = True, + ): + self.mean = mean + self.std = std + self.normalize = normalize + + # only load torchvision.transforms when needed + try: + import torchvision.transforms as T + + # FIXME: add version check for gguf + except ImportError as err: + raise ImportError( + "Please install torchvision via `pip install torchvision` to use Deepseek-VL2." + ) from err + + transform_pipelines = [T.ToTensor()] + + if normalize: + transform_pipelines.append(T.Normalize(mean, std)) + + self.transform = T.Compose(transform_pipelines) + + def __call__(self, pil_img: Image.Image): + x = self.transform(pil_img) + return x + + +class DeepseekVLV2Processor(ProcessorMixin): + tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast") + attributes = ["tokenizer"] + + def __init__( + self, + tokenizer: LlamaTokenizerFast, + candidate_resolutions: Tuple[Tuple[int, int]], + patch_size: int, + downsample_ratio: int, + image_mean: Tuple[float, float, float] = (0.5, 0.5, 0.5), + image_std: Tuple[float, float, float] = (0.5, 0.5, 0.5), + normalize: bool = True, + image_token: str = "", + pad_token: str = "<｜▁pad▁｜>", + add_special_token: bool = False, + sft_format: str = "deepseek", + mask_prompt: bool = True, + ignore_id: int = -100, + **kwargs, + ): + + self.candidate_resolutions = candidate_resolutions + self.image_size = candidate_resolutions[0][0] + self.patch_size = patch_size + self.image_mean = image_mean + self.image_std = image_std + self.normalize = normalize + self.downsample_ratio = downsample_ratio + + self.image_transform = ImageTransform( + mean=image_mean, std=image_std, normalize=normalize + ) + self.tokenizer = tokenizer + # must set this，padding side with make a difference in batch inference + self.tokenizer.padding_side = "left" + + # add the pad_token as special token to use 'tokenizer.pad_token' and 'tokenizer.pad_token_id' + if tokenizer.pad_token is None: + self.tokenizer.add_special_tokens({"pad_token": pad_token}) + + # add image token + image_token_id = self.tokenizer.vocab.get(image_token) + if image_token_id is None: + special_tokens = [image_token] + special_tokens_dict = {"additional_special_tokens": special_tokens} + self.tokenizer.add_special_tokens(special_tokens_dict) + self.image_token_id = self.tokenizer.vocab.get(image_token) + + # add five special tokens for grounding-related tasks + # <|ref|>, <|/ref|>, <|det|>, <|/det|>, <|grounding|> + special_tokens = ["<|ref|>", "<|/ref|>", "<|det|>", "<|/det|>", "<|grounding|>"] + special_tokens_dict = {"additional_special_tokens": special_tokens} + self.tokenizer.add_special_tokens(special_tokens_dict) + + # add special tokens for SFT data + special_tokens = ["<|User|>", "<|Assistant|>"] + special_tokens_dict = {"additional_special_tokens": special_tokens} + self.tokenizer.add_special_tokens(special_tokens_dict) + + self.image_token = image_token + self.pad_token = pad_token + self.add_special_token = add_special_token + self.sft_format = sft_format + self.mask_prompt = mask_prompt + self.ignore_id = ignore_id + + super().__init__( + tokenizer, + **kwargs, + ) + + def format_messages_v2(self, messages, pil_images, max_req_input_len=-1): + """play the role of format_messages_v2 and get_images_info in the last version""" + tokenized_data = [] + masked_tokenized_data = [] # labels + images_list = [] + images_seq_mask = [] + images_spatial_crop = [] + + image_index = 0 + image_token_cnt = messages.count(self.image_token) + tokenized_str, images, seq_mask, spatial_crop = self.tokenize_with_images( + messages, + pil_images[image_index : image_index + image_token_cnt], + bos=True, + eos=True, + cropping=len(pil_images) <= 2, + max_req_input_len=max_req_input_len, + ) + + image_index = image_token_cnt + tokenized_data += tokenized_str + if self.mask_prompt: + masked_tokenized_data += [self.ignore_id] * len(tokenized_str) + else: + masked_tokenized_data += tokenized_str + images_list += images + images_seq_mask += seq_mask + images_spatial_crop += spatial_crop + + assert len(tokenized_data) == len( + images_seq_mask + ), f"format_messages_v2: tokenized_str's length {len(tokenized_str)} is not equal to imags_seq_mask's length {len(images_seq_mask)}" + + return ( + tokenized_data, + masked_tokenized_data, + images_list, + images_seq_mask, + images_spatial_crop, + ) + + @property + def bos_id(self): + return self.tokenizer.bos_token_id + + @property + def eos_id(self): + return self.tokenizer.eos_token_id + + @property + def pad_id(self): + return self.tokenizer.pad_token_id + + def encode(self, text: str, bos: bool = True, eos: bool = False): + t = self.tokenizer.encode(text, add_special_tokens=False) + + if bos: + t = [self.bos_id] + t + if eos: + t = t + [self.eos_id] + + return t + + def decode(self, t: List[int], **kwargs) -> str: + return self.tokenizer.decode(t, **kwargs) + + def process_one( + self, + prompt: str = None, + conversations: List[Dict[str, str]] = None, + images: List[Image.Image] = None, + apply_sft_format: bool = False, + inference_mode: bool = True, + system_prompt: str = "", + max_req_input_len: int = -1, + **kwargs, + ): + """ + + Args: + prompt (str): the formatted prompt; + conversations (List[Dict]): conversations with a list of messages; + images (List[ImageType]): the list of images; + apply_sft_format (bool): if prompt is not None, then apply the SFT format to prompt; + if conversations is not None, then it will always apply the SFT format to conversations; + inference_mode (bool): if True, then remove the last eos token; + system_prompt (str): the system prompt; + **kwargs: + + Returns: + outputs (BaseProcessorOutput): the output of the processor, + - input_ids (torch.LongTensor): [N + image tokens] + - target_ids (torch.LongTensor): [N + image tokens] + - images (torch.FloatTensor): [n_images, 3, H, W] + - image_id (int): the id of the image token + - num_image_tokens (List[int]): the number of image tokens + """ + + assert ( + prompt is None or conversations is None + ), "prompt and conversations cannot be used at the same time." + + ( + tokenized_str, + masked_tokenized_str, + images_list, + images_seq_mask, + images_spatial_crop, + ) = self.format_messages_v2(conversations, images, max_req_input_len) + + assert ( + len(tokenized_str) == len(images_seq_mask) == len(masked_tokenized_str) + ), ( + f"tokenized_str's length {len(tokenized_str)}, input_ids' length {len(masked_tokenized_str)}, " + f"imags_seq_mask's length {len(images_seq_mask)}, are not equal" + ) + + input_ids = torch.LongTensor(tokenized_str) + target_ids = torch.LongTensor(masked_tokenized_str) + images_seq_mask = torch.tensor(images_seq_mask, dtype=torch.bool) + + # set input_ids < 0 | input_ids == self.image_token_id as ignore_id + target_ids[(input_ids < 0) | (input_ids == self.image_token_id)] = ( + self.ignore_id + ) + input_ids[input_ids < 0] = self.pad_id + + if inference_mode: + assert input_ids[-1] == self.eos_id + input_ids = input_ids[:-1] + target_ids = target_ids[:-1] + images_seq_mask = images_seq_mask[:-1] + + if len(images_list) == 0: + images = torch.zeros((1, 3, self.image_size, self.image_size)) + images_spatial_crop = torch.zeros((1, 2), dtype=torch.long) + else: + images = torch.stack(images_list, dim=0) + images_spatial_crop = torch.tensor(images_spatial_crop, dtype=torch.long) + + images_spatial_crop = torch.stack( + [images_spatial_crop], dim=0 + ) # stack the tensor to make it a batch of 1 + + prepare = VLChatProcessorOutput( + input_ids=input_ids, + target_ids=target_ids, + pixel_values=images, + images_seq_mask=images_seq_mask, + images_spatial_crop=images_spatial_crop, + ) + + return prepare + + def __call__( + self, + *, + prompt: str = None, + conversations: List[Dict[str, str]] = None, + images: List[Image.Image] = None, + apply_sft_format: bool = False, + inference_mode: bool = True, + system_prompt: str = "", + max_req_input_len: int = -1, + **kwargs, + ): + prepare = self.process_one( + prompt=prompt, + conversations=conversations, + images=images, + apply_sft_format=apply_sft_format, + inference_mode=inference_mode, + system_prompt=system_prompt, + max_req_input_len=max_req_input_len, + ) + + return prepare + + def find_all_indices(self, messages, target_value): + indices = [] + for index, item in enumerate(messages): + if item == target_value: + indices.append(index) + return indices + + def tokenize_with_images( + self, + conversation: str, + images: List[Image.Image], + bos: bool = True, + eos: bool = True, + cropping: bool = True, + max_req_input_len: int = -1, + ): + """Tokenize text with tags.""" + images_list, images_seq_mask, images_spatial_crop = [], [], [] + text_splits = conversation.split(self.image_token) + tokenized_str = [] + for text_sep, image in zip(text_splits, images): + """encode text_sep""" + tokenized_sep = self.encode(text_sep, bos=False, eos=False) + tokenized_str += tokenized_sep + images_seq_mask += [False] * len(tokenized_sep) + + """select best resolution for anyres""" + if cropping: + best_width, best_height = select_best_resolution( + image.size, self.candidate_resolutions + ) + else: + best_width, best_height = self.image_size, self.image_size + # print(image.size, (best_width, best_height)) # check the select_best_resolutions func + + """process the global view""" + global_view = ImageOps.pad( + image, + (self.image_size, self.image_size), + color=tuple(int(x * 255) for x in self.image_transform.mean), + ) + images_list.append(self.image_transform(global_view)) + + """process the local views""" + local_view = ImageOps.pad( + image, + (best_width, best_height), + color=tuple(int(x * 255) for x in self.image_transform.mean), + ) + for i in range(0, best_height, self.image_size): + for j in range(0, best_width, self.image_size): + images_list.append( + self.image_transform( + local_view.crop( + (j, i, j + self.image_size, i + self.image_size) + ) + ) + ) + + """record height / width crop num""" + num_width_tiles, num_height_tiles = ( + best_width // self.image_size, + best_height // self.image_size, + ) + images_spatial_crop.append([num_width_tiles, num_height_tiles]) + + """add image tokens""" + h = w = math.ceil( + (self.image_size // self.patch_size) / self.downsample_ratio + ) + # global views tokens h * (w + 1), 1 is for line separator + tokenized_image = [self.image_token_id] * h * (w + 1) + # add a separator between global and local views + tokenized_image += [self.image_token_id] + # local views tokens, (num_height_tiles * h) * (num_width_tiles * w + 1) + tokenized_image += ( + [self.image_token_id] + * (num_height_tiles * h) + * (num_width_tiles * w + 1) + ) + + tokenized_str += tokenized_image + images_seq_mask += [True] * len(tokenized_image) + # print(width_crop_num, height_crop_num, len(tokenized_image)) # test the correctness of the number of image-related tokens + + """process the last text split""" + tokenized_sep = self.encode(text_splits[-1], bos=False, eos=False) + # deal with video, limit with request len + if max_req_input_len > -1: + if max_req_input_len < len(tokenized_sep) + len(tokenized_str) - 1: + rest = max_req_input_len - len(tokenized_sep) - 1 - 1024 + tokenized_str = tokenized_str[:rest] + images_seq_mask = images_seq_mask[:rest] + tokenized_str += tokenized_sep + images_seq_mask += [False] * len(tokenized_sep) + + """add the bos and eos tokens""" + if bos: + tokenized_str = [self.bos_id] + tokenized_str + images_seq_mask = [False] + images_seq_mask + if eos: + tokenized_str = tokenized_str + [self.eos_id] + images_seq_mask = images_seq_mask + [False] + + assert len(tokenized_str) == len( + images_seq_mask + ), f"tokenize_with_images func: tokenized_str's length {len(tokenized_str)} is not equal to imags_seq_mask's length {len(images_seq_mask)}" + + return tokenized_str, images_list, images_seq_mask, images_spatial_crop + + +class DeepseekVL2VisionEncoderConfig(PretrainedConfig): + model_type: str = "vision" + + model_name: str = "siglip_large_patch16_384" + image_size: int = 384 + patch_size: int = 16 + width: int = 1024 + layers: int = 24 + heads: int = 16 + mlp_ratio: int = 4 + global_pool: str = "map" + ignore_head: bool = True + class_token: bool = False + num_classes: int = 0 + use_checkpoint: bool = False + weight_init: str = "skip" + deterministic: bool = False + num_recomputing_layers: int = 0 + + def __init__( + self, + model_name: str = "siglip_large_patch16_384", + image_size: int = 384, + patch_size: int = 16, + width: int = 1024, + layers: int = 24, + heads: int = 16, + mlp_ratio: int = 4, + global_pool: str = "map", + ignore_head: bool = True, + class_token: bool = False, + num_classes: int = 0, + use_checkpoint: bool = False, + **kwargs, + ): + self.model_name = model_name + self.image_size = image_size + self.patch_size = patch_size + self.width = width + self.layers = layers + self.heads = heads + self.mlp_ratio = mlp_ratio + self.global_pool = global_pool + self.ignore_head = ignore_head + self.class_token = class_token + self.num_classes = num_classes + self.use_checkpoint = use_checkpoint + + super().__init__(**kwargs) + + +class DeepseekVL2MlpProjectorConfig(PretrainedConfig): + model_type = "mlp_projector" + projector_type: str = "downsample_mlp_gelu" + input_dim: int = 1152 + n_embed: int = 2048 + depth: int = 2 + mlp_ratio: int = 1 + downsample_ratio: int = 2 + token_pooling: bool = False + + def __init__( + self, + projector_type: str = "downsample_mlp_gelu", + input_dim: int = 1152, + n_embed: int = 2048, + depth: int = 2, + mlp_ratio: int = 1, + downsample_ratio: int = 2, + **kwargs, + ): + self.projector_type = projector_type + self.input_dim = input_dim + self.n_embed = n_embed + self.depth = depth + self.mlp_ratio = mlp_ratio + self.downsample_ratio = downsample_ratio + + super().__init__(**kwargs) + + +class DeepseekV2Config(PretrainedConfig): + + model_type = "deepseek_v2" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + vocab_size=102400, + hidden_size=4096, + intermediate_size=11008, + moe_intermediate_size=1407, + num_hidden_layers=30, + num_attention_heads=32, + num_key_value_heads=32, + n_shared_experts=None, + n_routed_experts=None, + ep_size=1, + routed_scaling_factor=1.0, + kv_lora_rank=512, + q_lora_rank=1536, + qk_rope_head_dim=64, + v_head_dim=128, + qk_nope_head_dim=128, + topk_method="gready", + n_group=None, + topk_group=None, + num_experts_per_tok=None, + moe_layer_freq=1, + first_k_dense_replace=0, + norm_topk_prob=False, + scoring_func="softmax", + aux_loss_alpha=0.001, + seq_aux=True, + hidden_act="silu", + max_position_embeddings=2048, + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + pad_token_id=None, + bos_token_id=100000, + eos_token_id=100001, + pretraining_tp=1, + tie_word_embeddings=False, + rope_theta=10000.0, + rope_scaling=None, + attention_bias=False, + attention_dropout=0.0, + use_mla=True, + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.moe_intermediate_size = moe_intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.n_shared_experts = n_shared_experts + self.n_routed_experts = n_routed_experts + self.ep_size = ep_size + self.routed_scaling_factor = routed_scaling_factor + self.kv_lora_rank = kv_lora_rank + self.q_lora_rank = q_lora_rank + self.qk_rope_head_dim = qk_rope_head_dim + self.v_head_dim = v_head_dim + self.qk_nope_head_dim = qk_nope_head_dim + self.topk_method = topk_method + self.n_group = n_group + self.topk_group = topk_group + self.num_experts_per_tok = num_experts_per_tok + self.moe_layer_freq = moe_layer_freq + self.first_k_dense_replace = first_k_dense_replace + self.norm_topk_prob = norm_topk_prob + self.scoring_func = scoring_func + self.aux_loss_alpha = aux_loss_alpha + self.seq_aux = seq_aux + # for backward compatibility + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = float(rms_norm_eps) + self.pretraining_tp = pretraining_tp + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + self.attention_bias = attention_bias + self.attention_dropout = attention_dropout + self.use_mla = use_mla + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + +class DeepseekVL2Config(PretrainedConfig): + model_type = "deepseek_vl_v2" + vision_config: DeepseekVL2VisionEncoderConfig + projector_config: DeepseekVL2MlpProjectorConfig + language_config: DeepseekV2Config + + tile_tag: str = "2D" + global_view_pos: str = "head" + candidate_resolutions: Tuple[Tuple[int, int]] = ((384, 384),) + + def __init__( + self, + tile_tag: str = "tile_tag", + global_view_pos: str = "head", + candidate_resolutions: Tuple[Tuple[int, int]] = ((384, 384),), + **kwargs, + ): + super().__init__(**kwargs) + + vision_config = kwargs.get("vision_config", {}) + self.vision_config = DeepseekVL2VisionEncoderConfig(**vision_config) + + projector_config = kwargs.get("projector_config", {}) + self.projector_config = DeepseekVL2MlpProjectorConfig(**projector_config) + + language_config = kwargs.get("language_config", {}) + if isinstance(language_config, DeepseekV2Config): + self.language_config = language_config + else: + self.language_config = DeepseekV2Config(**language_config) + + self.tile_tag = tile_tag + self.global_view_pos = global_view_pos + self.candidate_resolutions = candidate_resolutions + self.architectures = ["DeepseekVL2ForCausalLM"] + + +AutoProcessor.register(DeepseekVL2Config, DeepseekVLV2Processor) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/device_config.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/device_config.py new file mode 100644 index 00000000..20b9af9b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/device_config.py @@ -0,0 +1,19 @@ +import logging +from typing import Optional + +import torch + +logger = logging.getLogger(__name__) + + +class DeviceConfig: + device: Optional[torch.device] + gpu_id: Optional[int] + + def __init__(self, device: str = "cuda", gpu_id: int = -1) -> None: + if device in ["cuda", "xpu", "hpu", "cpu", "npu"]: + self.device_type = device + else: + raise RuntimeError(f"Not supported device type: {device}") + self.device = torch.device(self.device_type) + self.gpu_id = gpu_id diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/dots_ocr.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/dots_ocr.py new file mode 100644 index 00000000..8b0693b8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/dots_ocr.py @@ -0,0 +1,64 @@ +from typing import Optional + +from transformers import AutoProcessor, Qwen2_5_VLProcessor +from transformers.image_processing_utils import BaseImageProcessor +from transformers.models.qwen2 import Qwen2Config + +from sglang.srt.configs.dots_vlm import DotsVisionConfig + + +class DotsOCRConfig(Qwen2Config): + model_type = "dots_ocr" + + def __init__( + self, + image_token_id=151665, + video_token_id=151656, + vision_config: Optional[dict] = None, + *args, + **kwargs + ): + super().__init__(*args, **kwargs) + self.image_token_id = image_token_id + self.video_token_id = video_token_id + self.vision_config = DotsVisionConfig(**(vision_config or {})) + + def save_pretrained(self, save_directory, **kwargs): + self._auto_class = None + super().save_pretrained(save_directory, **kwargs) + + +class DummyVideoProcessor(BaseImageProcessor): + model_input_names = ["pixel_values"] + + def __call__(self, *args, **kwargs): + return None + + +class DotsVLProcessor(Qwen2_5_VLProcessor): + def __init__( + self, + image_processor=None, + tokenizer=None, + video_processor=None, + chat_template=None, + **kwargs + ): + if video_processor is None: + video_processor = DummyVideoProcessor() + super().__init__( + image_processor, tokenizer, video_processor, chat_template=chat_template + ) + self.image_token = ( + "<|imgpad|>" + if not hasattr(tokenizer, "image_token") + else tokenizer.image_token + ) + self.image_token_id = ( + tokenizer.image_token_id + if getattr(tokenizer, "image_token_id", None) is not None + else tokenizer.convert_tokens_to_ids(self.image_token) + ) + + +AutoProcessor.register(DotsOCRConfig, DotsVLProcessor) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/dots_vlm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/dots_vlm.py new file mode 100644 index 00000000..dc921582 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/dots_vlm.py @@ -0,0 +1,134 @@ +from transformers import AutoProcessor, PretrainedConfig +from transformers.processing_utils import ProcessingKwargs + +try: + from transformers import Qwen2_5_VLProcessor +except ImportError: + raise ImportError( + "Qwen2_5_VLProcessor can not be found. Please upgrade your transformers version." + ) + +from sglang.srt.configs.deepseekvl2 import DeepseekV2Config + + +class DotsVisionConfig(PretrainedConfig): + model_type: str = "dots_vit" + + def __init__( + self, + embed_dim: int = 1536, # vision encoder embed size + hidden_size: int = 1536, # after merger hidden size + intermediate_size: int = 4224, + num_hidden_layers: int = 42, + num_attention_heads: int = 12, + num_channels: int = 3, + patch_size: int = 14, + spatial_merge_size: int = 2, + temporal_patch_size: int = 1, + rms_norm_eps: float = 1e-5, + use_bias: bool = False, + attn_implementation="flash_attention_2", # "eager","sdpa","flash_attention_2" + initializer_range=0.02, + init_merger_std=0.02, + is_causal=False, # ve causal forward + post_norm=True, + gradient_checkpointing=False, + **kwargs, + ): + super().__init__(**kwargs) + self.embed_dim = embed_dim + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.num_channels = num_channels + self.patch_size = patch_size + self.spatial_merge_size = spatial_merge_size + self.temporal_patch_size = temporal_patch_size + self.rms_norm_eps = rms_norm_eps + self.use_bias = use_bias + self.attn_implementation = attn_implementation + self.initializer_range = initializer_range + self.init_merger_std = init_merger_std + self.is_causal = is_causal + self.post_norm = post_norm + self.gradient_checkpointing = gradient_checkpointing + + +class DotsVLMConfig(PretrainedConfig): + model_type = "dots_vlm" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + vision_config = kwargs.get("vision_config", {}) + self.im_span_id = kwargs.get("image_token_id", 128815) + self.video_span_id = kwargs.get("video_token_id", 128836) + self.vision_config = DotsVisionConfig(**vision_config) + self.language_config = DeepseekV2Config(**kwargs) + self.architectures = ["DotsVLMForCausalLM"] + + +class DotsVLMProcessorKwargs(ProcessingKwargs, total=False): + _defaults = { + "text_kwargs": { + "padding": False, + }, + } + + +class DotsVLMProcessor(Qwen2_5_VLProcessor): + r""" + Constructs a DotsVLM processor which derives from Qwen2_5_VLProcessor, but overrides the image and video token ids. + Besides, its tokenizer is a LlamaTokenizerFast instead of Qwen2TokenizerFast. + [`DotsVLMProcessor`] offers all the functionalities of [`DotsVisionConfig`] and [`LlamaTokenizerFast`]. See the + [`~DotsVLMProcessor.__call__`] and [`~DotsVLMProcessor.decode`] for more information. + Args: + image_processor ([`Qwen2VLImageProcessor`], *optional*): + The image processor is a required input. + tokenizer ([`LlamaTokenizerFast`], *optional*): + The tokenizer is a required input. + chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages + in a chat into a tokenizable string. + """ + + attributes = ["image_processor", "tokenizer"] + + valid_kwargs = ["chat_template"] + + tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast") + + def __init__( + self, image_processor=None, tokenizer=None, chat_template=None, **kwargs + ): + super().__init__(image_processor, tokenizer, chat_template=chat_template) + self.image_token = ( + "<|imgpad|>" + if not hasattr(tokenizer, "image_token") + else tokenizer.image_token + ) + self.video_token = ( + "<|video_pad|>" + if not hasattr(tokenizer, "video_token") + else tokenizer.video_token + ) + self.img_token = ( + "<|img|>" if not hasattr(tokenizer, "img_token") else tokenizer.img_token + ) + self.endofimg_token = ( + "<|endofimg|>" + if not hasattr(tokenizer, "endofimg_token") + else tokenizer.endofimg_token + ) + self.image_token_id = ( + tokenizer.image_token_id + if getattr(tokenizer, "image_token_id", None) + else tokenizer.encode(self.image_token)[0] + ) + self.video_token_id = ( + tokenizer.video_token_id + if getattr(tokenizer, "video_token_id", None) + else tokenizer.encode(self.video_token)[0] + ) + + +AutoProcessor.register(DotsVLMConfig, DotsVLMProcessor) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/exaone.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/exaone.py new file mode 100644 index 00000000..7b0a2d29 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/exaone.py @@ -0,0 +1,195 @@ +# coding=utf-8 +# Copyright 2024 The LG AI Research EXAONE Lab. All rights reserved. +# Copyright 2024 The LG CNS AI Engineering Team. +# Copyright 2023-2024 SGLang Team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" EXAONE model configuration """ +from typing import Any, Dict + +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + +logger = logging.get_logger(__name__) + +EXAONE_PRETRAINED_CONFIG_ARCHIVE_MAP: Dict[str, Any] = {} + + +# ruff: noqa: E501 +class ExaoneConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a :class:`~transformers.ExaoneModel`. It is used to + instantiate a EXAONE model according to the specified arguments, defining the model architecture. Instantiating a + configuration with the defaults will yield a similar configuration to that of the Exaone + + Configuration objects inherit from :class:`~transformers.PretrainedConfig` and can be used to control the model + outputs. Read the documentation from :class:`~transformers.PretrainedConfig` for more information. + + + Args: + vocab_size (:obj:`int`, `optional`, defaults to 102400): + Vocabulary size of the EXAONE model. Defines the number of different tokens that can be represented by the + :obj:`inputs_ids` passed when calling :class:`~transformers.ExaoneModel`. Vocabulary size of the model. + Defines the different tokens that can be represented by the `inputs_ids` passed to the forward method of + :class:`~transformers.EXAONEModel`. + max_position_embeddings (:obj:`int`, `optional`, defaults to 2048): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + hidden_size (:obj:`int`, `optional`, defaults to 2048): + Dimensionality of the encoder layers and the pooler layer. + num_layers (:obj:`int`, `optional`, defaults to 32): + Number of hidden layers in the Transformer encoder. + num_attention_heads (:obj:`int`, `optional`, defaults to 32): + Number of attention heads for each attention layer in the Transformer decoder. + num_key_value_heads (:obj:`int`, `optional`): + This is the number of key_value heads that should be used to implement Grouped Query Attention. If + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if + `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed + by meanpooling all the original heads within that group. For more details checkout [this + paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to + `num_attention_heads`. + intermediate_size (:obj:`int`, `optional`, defaults to `hidden_size * 4`): + Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. + activation_function (:obj:`str` or :obj:`function`, `optional`, defaults to :obj:`"silu"`): + The non-linear activation function (function or string) in the decoder. + rope_theta (:obj:`float`, `optional`, defaults to 10000.0): + The base period of the RoPE embeddings. + rope_scaling (:obj:`Dict`, `optional`): + Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type + and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value + accordingly. + Expected contents: + `rope_type` (:obj:`str`): + The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope', + 'llama3'], with 'default' being the original RoPE implementation. + `factor` (:obj:`float`, `optional`): + Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In + most scaling types, a `factor` of x will enable the model to handle sequences of length x * + original maximum pre-trained length. + `original_max_position_embeddings` (:obj:`int`, `optional`): + Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during + pretraining. + `attention_factor` (:obj:`float`, `optional`): + Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention + computation. If unspecified, it defaults to value recommended by the implementation, using the + `factor` field to infer the suggested value. + `beta_fast` (:obj:`float`, `optional`): + Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear + ramp function. If unspecified, it defaults to 32. + `beta_slow` (:obj:`float`, `optional`): + Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear + ramp function. If unspecified, it defaults to 1. + `short_factor` (:obj:`List[float]`, `optional`): + Only used with 'longrope'. The scaling factor to be applied to short contexts (< + `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden + size divided by the number of attention heads divided by 2 + `long_factor` (:obj:`List[float]`, `optional`): + Only used with 'longrope'. The scaling factor to be applied to long contexts (< + `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden + size divided by the number of attention heads divided by 2 + `low_freq_factor` (:obj:`float`, `optional`): + Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE + `high_freq_factor` (:obj:`float`, `optional`): + Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE + embed_dropout (:obj:`float`, `optional`, defaults to 0.0): + The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler. + attention_dropout (:obj:`float`, `optional`, defaults to 0.0): + The dropout ratio for the attention probabilities. + layer_norm_epsilon (:obj:`float`, `optional`, defaults to 1e-5): + The epsilon used by the layer normalization layers. + initializer_range (:obj:`float`, `optional`, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + use_cache (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if ``configs.is_decoder=True``. + bos_token_id (:obj:`int`, `optional`, defaults to 0): + Beginning of stream token id. + eos_token_id (:obj:`int`, `optional`, defaults to 2): + End of stream token id. + tie_word_embeddings (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether to tie weight embeddings + gradient_checkpointing (:obj:`bool`, `optional`, defaults to :obj:`False`): + If True, use gradient checkpointing to save memory at the expense of slower backward pass. + + Example:: + + >>> from transformers import EXAONEModel, ExaoneConfig + + >>> # Initializing a EXAONE configuration + >>> configuration = ExaoneConfig() + + >>> # Initializing a model from configuration + >>> model = EXAONEModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.configs + """ + + model_type = "exaone" + keys_to_ignore_at_inference = ["past_key_values"] + attribute_map = {"num_hidden_layers": "num_layers"} + + def __init__( + self, + vocab_size=102400, + max_position_embeddings=2048, + hidden_size=2048, + num_layers=32, + num_attention_heads=32, + num_key_value_heads=None, + intermediate_size=None, + activation_function="silu", + rope_theta=10000.0, + rope_scaling=None, + embed_dropout=0.0, + attention_dropout=0.0, + layer_norm_epsilon=1e-5, + initializer_range=0.02, + use_cache=True, + bos_token_id=0, + eos_token_id=2, + tie_word_embeddings=True, + **kwargs + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.num_layers = num_layers + self.num_attention_heads = num_attention_heads + self.num_hidden_layers = num_layers + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + self.num_key_value_heads = num_key_value_heads + if intermediate_size: + self.intermediate_size = intermediate_size + else: + self.intermediate_size = hidden_size * 4 + self.activation_function = activation_function + self.embed_dropout = embed_dropout + self.attention_dropout = attention_dropout + self.layer_norm_epsilon = layer_norm_epsilon + self.initializer_range = initializer_range + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + + self.bos_token_id = bos_token_id + self.eos_token_id = eos_token_id + + super().__init__( + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/falcon_h1.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/falcon_h1.py new file mode 100644 index 00000000..1f524b89 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/falcon_h1.py @@ -0,0 +1,310 @@ +# coding=utf-8 +# Copyright 2024 TII and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Falcon-H1 model configuration""" + + +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + +from sglang.srt.configs.mamba_utils import Mamba2CacheParams, Mamba2StateShape + +logger = logging.get_logger(__name__) + + +class FalconH1Config(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`FalconH1Model`]. It is used to instantiate a + FalconH1Model model according to the specified arguments, defining the model architecture. Instantiating a configuration + with defaults taken from [ibm-fms/FalconH1-9.8b-2.2T-hf](https://huggingface.co/ibm-fms/FalconH1-9.8b-2.2T-hf). + The FalconH1Model is a hybrid [mamba2](https://github.com/state-spaces/mamba) architecture with SwiGLU. + The checkpoints are jointly trained by IBM, Princeton, and UIUC. + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + Args: + vocab_size (`int`, *optional*, defaults to 128000): + Vocabulary size of the FalconH1 model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`FalconH1Model`] + tie_word_embeddings (`bool`, *optional*, defaults to `False`): + Whether the model's input and output word embeddings should be tied. Note that this is only relevant if the + model has a output word embedding layer. + hidden_size (`int`, *optional*, defaults to 4096): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 14336): + Dimension of the MLP representations. + num_hidden_layers (`int`, *optional*, defaults to 32): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 32): + Number of attention heads for each attention layer in the Transformer encoder. + num_key_value_heads (`int`, *optional*, defaults to 8): + This is the number of key_value heads that should be used to implement Grouped Query Attention. If + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if + `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed + by meanpooling all the original heads within that group. For more details, check out [this + paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to `8`. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the decoder. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`float`, *optional*, defaults to 1e-05): + The epsilon used by the rms normalization layers. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. + num_logits_to_keep (`int` or `None`, *optional*, defaults to 1): + Number of prompt logits to calculate during generation. If `None`, all logits will be calculated. If an + integer value, only last `num_logits_to_keep` logits will be calculated. Default is 1 because only the + logits of the last prompt token are needed for generation. For long sequences, the logits for the entire + sequence may use a lot of memory so, setting `num_logits_to_keep=1` will reduce memory footprint + significantly. + pad_token_id (`int`, *optional*, defaults to 0): + The id of the padding token. + bos_token_id (`int`, *optional*, defaults to 1): + The id of the "beginning-of-sequence" token. + eos_token_id (`int`, *optional*, defaults to 2): + The id of the "end-of-sequence" token. + max_position_embeddings (`int`, *optional*, defaults to 8192): + Max cached sequence length for the model + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + mamba_d_ssm (`int`, *optional*, defaults to 1024): + The dimension of the SSM state space latents. + mamba_n_heads (`int`, *optional*, defaults to 128): + The number of mamba heads used in the v2 implementation. + mamba_d_head (`int`, *optional*, defaults to `"auto"`): + Head embedding dimension size + mamba_n_groups (`int`, *optional*, defaults to 1): + The number of the mamba groups used in the v2 implementation. + mamba_d_state (`int`, *optional*, defaults to 256): + The dimension the mamba state space latents + mamba_d_conv (`int`, *optional*, defaults to 4): + The size of the mamba convolution kernel + mamba_expand (`int`, *optional*, defaults to 2): + Expanding factor (relative to hidden_size) used to determine the mamba intermediate size + mamba_chunk_size (`int`, *optional*, defaults to 256): + The chunks in which to break the sequence when doing prefill/training + mamba_conv_bias (`bool`, *optional*, defaults to `True`): + Flag indicating whether or not to use bias in the convolution layer of the mamba mixer block. + mamba_proj_bias (`bool`, *optional*, defaults to `False`): + Flag indicating whether or not to use bias in the input and output projections (["in_proj", "out_proj"]) of the mamba mixer block + mamba_norm_before_gate (`bool`, *optional*, defaults to `True`): + Whether to use RMSNorm before the gate in the Mamba block + mamba_rms_norm (`bool`, *optional*, defaults to `False`): + Whether to use RMSNorm instead of LayerNorm in the Mamba block + projectors_bias (`bool`, *optional*, defaults to `False`): + Flag indicating whether or not to use bias in the input and output projections (["in_proj", "out_proj"]) of the attention block + rope_theta (`float`, *optional*, defaults to 100000.0): + The theta value used for the RoPE embeddings. + rope_scaling (`float`, *optional*): + The scaling value used for the RoPE embeddings. If `None`, no scaling is applied. + lm_head_multiplier (`float`, *optional*, defaults to 1.0): + The multiplier for the LM head. This is used to scale the output of the LM head. + embedding_multiplier (`float`, *optional*, defaults to 1.0): + The multiplier for the embedding layer. This is used to scale the output of the embedding layer. + mlp_multipliers (`list[float]`, *optional*): + The multipliers for the MLP layers. This is used to scale the output of the MLP layers. The first value is + the multiplier of gate layer, the second value is the multiplier of the down_proj layer. + key_multiplier (`float`, *optional*): + The multiplier for the key layer. This is used to scale the output of the key layer. + attention_out_multiplier (`float`, *optional*): + The multiplier for the attention output layer. This is used to scale the output of the attention output + attention_in_multiplier (`float`, *optional*): + The multiplier for the attention input layer. This is used to scale the output of the attention input layer. + ssm_multipliers (`list[float]`, *optional*): + The multipliers for the SSM layers. This is used to scale the output of the SSM layers. + ssm_in_multiplier (`float`, *optional*): + The multiplier for the SSM input layer. This is used to scale the output of the SSM input layer. + ssm_out_multiplier (`float`, *optional*): + The multiplier for the SSM output layer. This is used to scale the output of the SSM output layer. + """ + + model_type = "falcon_h1" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + vocab_size=128000, + tie_word_embeddings=False, + hidden_size=4096, + intermediate_size=14336, + num_hidden_layers=32, + num_attention_heads=32, + num_key_value_heads=8, + hidden_act="silu", + initializer_range=0.02, + rms_norm_eps=1e-5, + use_cache=True, + num_logits_to_keep=1, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + max_position_embeddings=8192, + attention_dropout=0.0, + mamba_d_ssm=1024, + mamba_n_heads=128, + mamba_d_head="auto", + mamba_n_groups=1, + mamba_d_state=256, + mamba_d_conv=4, + mamba_expand=2, + mamba_chunk_size=256, + mamba_conv_bias=True, + mamba_proj_bias=False, + mamba_norm_before_gate=True, + mamba_rms_norm=False, + projectors_bias=False, + rope_theta=100000.0, + rope_scaling=None, + lm_head_multiplier=1.0, + embedding_multiplier=1.0, + mlp_multipliers=None, + key_multiplier=None, + attention_out_multiplier=None, + attention_in_multiplier=None, + ssm_multipliers=None, + ssm_in_multiplier=None, + ssm_out_multiplier=None, + **kwargs, + ): + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.max_position_embeddings = max_position_embeddings + self.attention_dropout = attention_dropout + self.attention_bias = False + self.mlp_bias = False + + # for backward compatibility + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + + self.use_cache = use_cache + self.num_logits_to_keep = num_logits_to_keep + + self.rope_theta = rope_theta + self.rope_scaling = None + self.rope_scaling = rope_scaling + self.projectors_bias = projectors_bias + self.mamba_intermediate = mamba_intermediate = ( + mamba_expand * hidden_size if mamba_d_ssm is None else mamba_d_ssm + ) + + if mamba_intermediate % mamba_n_heads != 0: + raise ValueError("mamba_n_heads must divide mamba_expand * hidden_size") + + # for the mamba_v2, must satisfy the following + if mamba_d_head == "auto": + mamba_d_head = mamba_intermediate // mamba_n_heads + + if mamba_d_head * mamba_n_heads != mamba_intermediate: + raise ValueError( + "The dimensions for the Mamba head state do not match the model intermediate_size" + ) + + self.mamba_d_ssm = mamba_d_ssm + self.mamba_n_heads = mamba_n_heads + self.mamba_d_head = mamba_d_head + self.mamba_n_groups = mamba_n_groups + self.mamba_d_state = mamba_d_state + self.mamba_d_conv = mamba_d_conv + self.mamba_expand = mamba_expand + self.mamba_chunk_size = mamba_chunk_size + self.mamba_conv_bias = mamba_conv_bias + self.mamba_proj_bias = mamba_proj_bias + + self.mamba_norm_before_gate = mamba_norm_before_gate + self.mamba_rms_norm = mamba_rms_norm + + self.lm_head_multiplier = lm_head_multiplier + self.embedding_multiplier = embedding_multiplier + + if mlp_multipliers is not None: + self.mlp_multipliers = mlp_multipliers + else: + self.mlp_multipliers = [1.0, 1.0] + + if attention_out_multiplier is not None: + self.attention_out_multiplier = attention_out_multiplier + else: + self.attention_out_multiplier = 1.0 + + if attention_in_multiplier is not None: + self.attention_in_multiplier = attention_in_multiplier + else: + self.attention_in_multiplier = 1.0 + + if key_multiplier is not None: + self.key_multiplier = key_multiplier + else: + self.key_multiplier = 1.0 + + if ssm_multipliers is not None: + self.ssm_multipliers = ssm_multipliers + else: + self.ssm_multipliers = [1.0, 1.0, 1.0, 1.0, 1.0] + + if ssm_in_multiplier is not None: + self.ssm_in_multiplier = ssm_in_multiplier + else: + self.ssm_in_multiplier = 1.0 + + if ssm_out_multiplier is not None: + self.ssm_out_multiplier = ssm_out_multiplier + else: + self.ssm_out_multiplier = 1.0 + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + @property + def layers_block_type(self): + return ["falcon_h1" for i in range(self.num_hidden_layers)] + + @property + def full_attention_layer_ids(self): + # For Falcon-H1, we do have attention on all layers + return range(self.num_hidden_layers) + + @property + def linear_layer_ids(self): + # For Falcon-H1, we do have mamba on all layers + return range(self.num_hidden_layers) + + @property + def mamba2_cache_params(self): + from sglang.srt.layers.dp_attention import get_attention_tp_size + + shape = Mamba2StateShape.create( + tp_world_size=get_attention_tp_size(), + intermediate_size=self.mamba_intermediate, + n_groups=self.mamba_n_groups, + num_heads=self.mamba_n_heads, + head_dim=self.mamba_d_head, + state_size=self.mamba_d_state, + conv_kernel=self.mamba_d_conv, + ) + return Mamba2CacheParams(shape=shape, layers=self.linear_layer_ids) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/internvl.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/internvl.py new file mode 100644 index 00000000..3ba9c61c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/internvl.py @@ -0,0 +1,706 @@ +import copy +import os +from shutil import copyfile +from typing import Any, Dict, List, Optional, Tuple, Union + +import sentencepiece as spm +from transformers import ( + TOKENIZER_MAPPING, + GptOssConfig, + LlamaConfig, + PretrainedConfig, + PreTrainedTokenizer, + Qwen2Config, + Qwen3Config, + Qwen3MoeConfig, +) + +from sglang.utils import logger + +# Copied from: https://github.com/OpenGVLab/InternVL/blob/34a81000402bf8f716bab8c9b57aff1f6b436bd0/internvl_chat/internvl/model/internvl_chat/configuration_internvl_chat.py#L21 + + +VOCAB_FILES_NAMES = {"vocab_file": "./tokenizer.model"} + +PRETRAINED_VOCAB_FILES_MAP = {} + + +# Modified from transformers.model.llama.configuration_llama.LlamaConfig +class InternLM2Config(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`InternLM2Model`]. It is used to instantiate + an InternLM2 model according to the specified arguments, defining the model architecture. Instantiating a + configuration with the defaults will yield a similar configuration to that of the InternLM2-7B. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 32000): + Vocabulary size of the InternLM2 model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`InternLM2Model`] + hidden_size (`int`, *optional*, defaults to 4096): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 11008): + Dimension of the MLP representations. + num_hidden_layers (`int`, *optional*, defaults to 32): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 32): + Number of attention heads for each attention layer in the Transformer encoder. + num_key_value_heads (`int`, *optional*): + This is the number of key_value heads that should be used to implement Grouped Query Attention. If + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if + `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed + by meanpooling all the original heads within that group. For more details checkout [this + paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to + `num_attention_heads`. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the decoder. + max_position_embeddings (`int`, *optional*, defaults to 2048): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`float`, *optional*, defaults to 1e-12): + The epsilon used by the rms normalization layers. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. + tie_word_embeddings(`bool`, *optional*, defaults to `False`): + Whether to tie weight embeddings + Example: + + """ + + model_type = "internlm2" + _auto_class = "AutoConfig" + + def __init__( # pylint: disable=W0102 + self, + vocab_size=103168, + hidden_size=4096, + intermediate_size=11008, + num_hidden_layers=32, + num_attention_heads=32, + num_key_value_heads=None, + hidden_act="silu", + max_position_embeddings=2048, + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + tie_word_embeddings=False, + bias=True, + rope_theta=10000, + rope_scaling=None, + attn_implementation="eager", + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.bias = bias + + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + self.num_key_value_heads = num_key_value_heads + + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + self._rope_scaling_validation() + + self.attn_implementation = attn_implementation + if self.attn_implementation is None: + self.attn_implementation = "eager" + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + def _rope_scaling_validation(self): + """ + Validate the `rope_scaling` configuration. + """ + if self.rope_scaling is None: + return + + if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2: + raise ValueError( + "`rope_scaling` must be a dictionary with with two fields, `type` and `factor`, " + f"got {self.rope_scaling}" + ) + rope_scaling_type = self.rope_scaling.get("type", None) + rope_scaling_factor = self.rope_scaling.get("factor", None) + if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]: + raise ValueError( + f"`rope_scaling`'s type field must be one of ['linear', 'dynamic'], got {rope_scaling_type}" + ) + if ( + rope_scaling_factor is None + or not isinstance(rope_scaling_factor, (float, int)) + or rope_scaling_factor < 1.0 + ): + raise ValueError( + f"`rope_scaling`'s factor field must be a float|int >= 1, got {rope_scaling_factor=}, {type(rope_scaling_factor)=}" + ) + if isinstance(rope_scaling_factor, int): + rope_scaling_factor = float(rope_scaling_factor) + + +class InternVisionConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`InternVisionModel`]. It is used to + instantiate a vision encoder according to the specified arguments, defining the model architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + num_channels (`int`, *optional*, defaults to 3): + Number of color channels in the input images (e.g., 3 for RGB). + patch_size (`int`, *optional*, defaults to 14): + The size (resolution) of each patch. + image_size (`int`, *optional*, defaults to 224): + The size (resolution) of each image. + qkv_bias (`bool`, *optional*, defaults to `False`): + Whether to add a bias to the queries and values in the self-attention layers. + hidden_size (`int`, *optional*, defaults to 3200): + Dimensionality of the encoder layers and the pooler layer. + num_attention_heads (`int`, *optional*, defaults to 25): + Number of attention heads for each attention layer in the Transformer encoder. + intermediate_size (`int`, *optional*, defaults to 12800): + Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. + qk_normalization (`bool`, *optional*, defaults to `True`): + Whether to normalize the queries and keys in the self-attention layers. + num_hidden_layers (`int`, *optional*, defaults to 48): + Number of hidden layers in the Transformer encoder. + use_flash_attn (`bool`, *optional*, defaults to `True`): + Whether to use flash attention mechanism. + hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"selu"` and `"gelu_new"` ``"gelu"` are supported. + layer_norm_eps (`float`, *optional*, defaults to 1e-6): + The epsilon used by the layer normalization layers. + dropout (`float`, *optional*, defaults to 0.0): + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + drop_path_rate (`float`, *optional*, defaults to 0.0): + Dropout rate for stochastic depth. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + initializer_factor (`float`, *optional*, defaults to 0.1): + A factor for layer scale. + """ + + model_type = "intern_vit_6b" + + def __init__( + self, + num_channels=3, + patch_size=14, + image_size=224, + qkv_bias=False, + hidden_size=3200, + num_attention_heads=25, + intermediate_size=12800, + qk_normalization=True, + num_hidden_layers=48, + use_flash_attn=True, + hidden_act="gelu", + layer_norm_eps=1e-6, + dropout=0.0, + drop_path_rate=0.0, + attention_dropout=0.0, + initializer_range=0.02, + initializer_factor=0.1, + **kwargs, + ): + super().__init__(**kwargs) + + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.dropout = dropout + self.drop_path_rate = drop_path_rate + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.num_channels = num_channels + self.patch_size = patch_size + self.image_size = image_size + self.initializer_range = initializer_range + self.initializer_factor = initializer_factor + self.attention_dropout = attention_dropout + self.layer_norm_eps = layer_norm_eps + self.hidden_act = hidden_act + self.qkv_bias = qkv_bias + self.qk_normalization = qk_normalization + self.use_flash_attn = use_flash_attn + + @classmethod + def from_pretrained( + cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs + ) -> "PretrainedConfig": + config_dict, kwargs = cls.get_config_dict( + pretrained_model_name_or_path, **kwargs + ) + + if "vision_config" in config_dict: + config_dict = config_dict["vision_config"] + + if ( + "model_type" in config_dict + and hasattr(cls, "model_type") + and config_dict["model_type"] != cls.model_type + ): + logger.warning( + f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors." + ) + + return cls.from_dict(config_dict, **kwargs) + + +class InternVLChatConfig(PretrainedConfig): + model_type = "internvl_chat" + is_composition = True + + def __init__( + self, + vision_config=None, + llm_config=None, + use_backbone_lora=0, + use_llm_lora=0, + pad2square=False, + select_layer=-1, + force_image_size=None, + downsample_ratio=0.5, + template=None, + dynamic_image_size=False, + use_thumbnail=False, + ps_version="v1", + min_dynamic_patch=1, + max_dynamic_patch=6, + **kwargs, + ): + super().__init__(**kwargs) + + if vision_config is None: + vision_config = {"architectures": ["InternVisionModel"]} + logger.info( + "vision_config is None. Initializing the InternVisionConfig with default values." + ) + + if llm_config is None: + llm_config = {"architectures": ["InternLM2ForCausalLM"]} + logger.info( + "llm_config is None. Initializing the LlamaConfig config with default values (`LlamaConfig`)." + ) + + self.vision_config = InternVisionConfig(**vision_config) + if llm_config.get("architectures")[0] == "LlamaForCausalLM": + self.llm_config = LlamaConfig(**llm_config) + elif llm_config.get("architectures")[0] == "InternLM2ForCausalLM": + self.llm_config = InternLM2Config(**llm_config) + elif llm_config.get("architectures")[0] == "Qwen2ForCausalLM": + self.llm_config = Qwen2Config(**llm_config) + elif llm_config.get("architectures")[0] == "Qwen3MoeForCausalLM": + self.llm_config = Qwen3MoeConfig(**llm_config) + elif llm_config.get("architectures")[0] == "Qwen3ForCausalLM": + self.llm_config = Qwen3Config(**llm_config) + elif llm_config.get("architectures")[0] == "GptOssForCausalLM": + self.llm_config = GptOssConfig(**llm_config) + else: + raise ValueError( + "Unsupported architecture: {}".format( + llm_config.get("architectures")[0] + ) + ) + + self.use_backbone_lora = use_backbone_lora + self.use_llm_lora = use_llm_lora + self.pad2square = pad2square + self.select_layer = select_layer + self.force_image_size = force_image_size + self.downsample_ratio = downsample_ratio + self.template = template + self.dynamic_image_size = dynamic_image_size + self.use_thumbnail = use_thumbnail + self.ps_version = ps_version # pixel shuffle version + self.min_dynamic_patch = min_dynamic_patch + self.max_dynamic_patch = max_dynamic_patch + + self.hidden_size = self.llm_config.hidden_size + # By default, we use tie_word_embeddings=False for models of all sizes. + self.tie_word_embeddings = False + self.llm_config.tie_word_embeddings = self.tie_word_embeddings + + def to_dict(self): + """ + Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`]. + + Returns: + `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance, + """ + output = copy.deepcopy(self.__dict__) + output["vision_config"] = self.vision_config.to_dict() + output["llm_config"] = self.llm_config.to_dict() + output["model_type"] = self.__class__.model_type + output["use_backbone_lora"] = self.use_backbone_lora + output["use_llm_lora"] = self.use_llm_lora + output["select_layer"] = self.select_layer + output["force_image_size"] = self.force_image_size + output["downsample_ratio"] = self.downsample_ratio + output["template"] = self.template + output["dynamic_image_size"] = self.dynamic_image_size + output["use_thumbnail"] = self.use_thumbnail + output["ps_version"] = self.ps_version + output["min_dynamic_patch"] = self.min_dynamic_patch + output["max_dynamic_patch"] = self.max_dynamic_patch + + return output + + +# # Modified from transformers.model.llama.tokenization_llama_fast.LlamaTokenizerFast -> InternLM2TokenizerFast +# class InternLM2TokenizerFast(PreTrainedTokenizerFast): +# vocab_files_names = VOCAB_FILES_NAMES +# slow_tokenizer_class = InternLM2Tokenizer +# padding_side = 'left' +# model_input_names = ['input_ids', 'attention_mask'] +# _auto_class = 'AutoTokenizer' +# +# def __init__( +# self, +# vocab_file, +# unk_token='', +# bos_token='', +# eos_token='', +# pad_token='

YJ(;}>v?L6g5;1`FYy3Q%3#qG zlL;K|teG}b%F4Q*#A9>IHYBE#q{DvS_noD&q3h0BBT0R`qZV2Vvd-|nEK?D+Hpho9 zJm;xA}bbU$mwr1CsP$T7vRVQ~w{;087nI9UoZ(A+xMs z{Ut!e|E;I~dzBd_6ssW%%_YF2Id>skn-Wqt;d>%DxmJVytJeQoF zBoJUjPF?}hq>`sU$^Q;AQIVtszHeL&W? ze>e!CuZm_DUXlOxPN0zx{0$&Hb`BDN z%a2}QQ!NhsGCifI&Qy44UfU8^`gvs0y0cJdYVhMZJ$0bjvin8G0{Bj;H(SYKJphe!2yyyy9*g$ zbbhgFJ4qf23H4n#xqb*vU`)|{Ux-goSW7e6AH34^8<6`eJ@roufP{G2zov9?&A=irOAWAg8)kr^mH?r zhY_r9@-!V_FWAxa;Ju4Jn`3%)6Kar)43NUa)H)1qx6Etbsd+-5;LiN-rT0q6L3`&7 zU`w+G-6Sv`?k2J6d~&-G>KS4*K6f%p5WHd`kBwd5lG(; zN&O;cFeVG348r!8mEwvBZvc0ZQ79^gXUsb-D)@6%YljU@VtdJa9_2j?6}l#!fa+q| zAXc9nX%Q9H*LLp)Lod#}lkNI}RAY+aW((q!8M4*-7QU$Q8WAEl&WzxhlKdx-JVIur zA5+aGL#)hGiwFmu@((~ro2P(is?;#8@%DPy^UOpjs&f_EaDtUA zNAKniOnrV`$d-$?mP5o@Dvaa2hr1K;1Sd_;F^rnQ_B<>gukYWzp4UI(Q|k|1E^1)P zP}64M?-MK*9l>9kGD?WrLV;bj6q^$h|3GYRI@ZVBSN&STT2E4=KHSfHGn!)?#TVKP z^g|Q2F=UAqCwzUGkDYYq4oNeyeEAA~%S)*kePK67PcjTWw@aj;%nq7ic*eNBGQNIr zY#7v+TYT2luTsA1R%d|YxH(N`^0nz2eQi6$zgPZ#^ef%EssC)@>~8?#^PNcjq-oo_ zl$k~JNS&Y&|MH~4JFG&Dv7`!V4KenwJ_{KALk&DmS%}R%hNNuePBhRRG8}-4`s*1C&$gdW~NF-+ZLywjyUs{_6#7G9e zhe$mf+SbSdfXQl{OYA^XAR#IJk5&x~z+}}*OBtV73KiI8pL%0Ty0*HiS;Hj@ zn2yb8tJw1_M zKTlQ1$n4XfY`t$6PW(b=g&;({i>sL4G#D87rdhBGYI@3XZsunyeVt0p!2wP@_h^i` zA!A&_jlAkA`B(%4RbyACcKH9GQ!AP}j*U(e>wPl7mn-?Ob#Q>;uz051#>dhSH@P473XQds| zi5h2nxEOprn05D>^JhYk|Go=*<1xhf2jb;ZoTm{iLR2Q@o3FS6&Fd|TMkCsen`opK_P_V;{BUeQ#In9cErgI2@E{ek}fyv0pai#62AEOd0cS$Opvr<1KO0cOg0g zLB=?cbbq4k$>RBDqFrx8pubb*L;Kxr{FZ;(%u}a*F9s}oT@GvFlWz^1Z>^=jD9Nbc z1gayK3*k5JWY#&yYNxIx;$ngIoPBo!CI%W*kFwOPz16?e-sxqG33kk%Dund3T~@!m zv|n|-n$8h%loy-Z|GW)|+HSL-Ia23RD=vhCAfuVbZ%!g3f1L z(}cd6ehBlRR9T9^+Ak?9C>tz2*=Vb%5<~)TjB`{S9|9Js%5*J}GFp0YI*VFMiGfLm zmKnJ7Dao)p2XE|$ANdMok^1>c;4h87T3=SQkpT*CQhx)eUk-$U0_;Wx`(`}8nrJ-M zH+^qAK=)H-QXW9@Z%O;i%jRQO&JT z0<}^8z*#TQNJkL?-!*BY2J0U0YcaD6Ld)`BS(Y!F1iX@O@;E0KkQ5ix^`6vF8=-!{ z7$r9iqq{t){O#J>=89W_pLSIs_IsIj z1y9~nB`y8--t5I_Wt9ZQwlCkND?q6uW_Sh*pGcojnPobvHywy!2(CR9PTI5ln(gjE zIO;GJRiI}X!TJsG8}Lq~A-8uOG(?4Z{R+_knqzZid;5a4(Gdt&PRn)%B1Ic{jMf4`CxH>?ZIex0P20cSxUF_+Qbf(4%x3BwAL(`2% zy-cIXo;(LgPJC#Eelok5NtsM-8jCte;R?laxUQBowyeS_6_FQ+p)*%L>((hY!4&n} zdPd29>KluzBuPI^*0WqTE`y=bg;R?}l{i!iQ>}T8gt$WWc19*k5DV;0T(xUk$->Vp z0n-pwvN4_%-v32Vl130r87jeA0OMe3igocU-$Q5KFDZ$YbK%lLP$?;ANyOL8x(ZqO zHdpXRh;w2H7PIo@yC_b@b0nQ5ReJ-HoWjXH57>SKj!l{xxC|=CLHt97mp!7D&pxZ_ z8g-7j@jHi%%Ejg zG+Xt_A}IZ8iN|zJ)n>Xv+=j|<)~vx$VLYJyIYlsuhtQY8x@Q796R?`^Lln0qepNJ7}<=Y*`A$Ea#~LC`3rKXg)MCpUh7 z$*k?GxTVK&bUVYPkMNXbGpq5}t(HbQlvW<-=tNwpQiISYKR^9b8b@1Yck7P}D$;`m zrQRMgFGW@)4ir{*k360&Bj$q~_8;zztEPc%S(O6bmfRc~ja)K(qc*ytQrDg=8p@?Q z*yBwYRakT`>{#GVVwUD*?q{!236&Efzh)<0q8<0}iGw#^O=$gEqiAphLq1xh-|Lbh zO(wC=Z{Vpo(^-}SZVqJIpBX)P-mkao-%x!Yk$ z$UkALVG1;5O|hH&h)APR9DaB#SGc#&nq+GhUrd8E!&OXI%;%&HfT>7Q9}3?*NOMK4 z>FAcPO%%bax@n|R&^&$vLg}-JYJ`V)igJtt8gpS5)fPT--}n+vCZi)1nq}Kot$1n- zeaeaVhGp_~Ay}dG`9DyQB!NNYnwRlY@Y%MA>(4I7M+FVeiDJU$)SJ1w5--aO9v*hf ziB={Kx6}NKO-g-TggA#S=#4jLio%OZkol-t>}3_;JAjL894XWvl1_iMwuIYX6FIY+ zqFznM8#usLJ?b=IUvN2Kl(fJN6I6>;z6k|{$Ln_?2?n)V%ja=|F}@a+-;A4NeXt*O z_rqy_Kz3~~F5-JxKJWLiHQ>YOfqLkKDMW6VH!+$3NB}`=&Dt9?O3t>x{TSyOeo~;W zr(cizY3TRz*;H1zoG|3c7AFlzv#Y=HXifj+Y3Iey5X1&A&TN*N%KG?pz1AY9p^{34 znAui%mg!8pYzq=6HM&*Y>Btej!e7FmF)Sh2*{b64^+lHvv<_6VC*Eg z5&L>TMn=y_Rj3yRlHp#qR%}()!mPDBOy%m@H%8c&99W8N-iTw(|6!C77W^_V}5?wr>u6E?oxx^(~_mEu%I5Y7ctxcN~k=K*w_sy({2;0eEsQ z={kQQOrn80y=hLWk%qQYUgBc4C%}ru| z6vHSVe+GA^+iE2uho4XMgQD_;uUcOPigmw14+Wf^v`BoSy@{|^3>RY<4n_GhXhn4n z89fPoQgcRk9`m<`MbFEibKS>dM17m2_Uktw&Hz;{;`Ucm z=O*pkp5A=V#DY7Dl42fG0!0@HkueCrP&`TJ(HwbeszYxJB?pCRK8nTbHfT{kUI}p-5pzt3o$*V{42(;AV85; zALz|;9;7vZ=;~X9zKy^aqH_P)d~f+j!3^mXBPhN1fc!JD~4LqlLOwO>=N#=>3EZ4ZqgE}Y1NxZ@#y{Qmy{IEhHv8I$e|n1 zu2q{$Rsq-mIR6q@$WvX-pK9Xp{hfkajYP$`p8*C_{tm44&lA$6YI_eC&cdtHVhrnHeeGTeMyBcBm2!H~u}<^#@`xW7 zGUTlY+|)ouU7H1;cgYoW-lu`nzY-Wk_NsyWlx3B@abk`eHl*XXu>Y5CgkxQ^si?16D$~OGE#KTECt&K8KxnZD!^~Lfw2Mclq{7WXPoJT(@msp1! zd%W{}Kl4{*31E3dSH3vh&wibzpJvsWPOuk^7i||JGN*UW8o9_(-5Y+1ky5Xpv{9nH ztHTT32IrRb$JLX0=h$#9sg(aBGwcvYmJ9=QS@-NiUEoBGC#>AK9ynbBH)Y*h`v-`* z+^)6`wePlpiBp07nJWf{wbEgf*t&u1Cj5vm)8bda zcLLGcmLubM0t9rQOvYJ0^iDrgqg&0*`96&ds`ZjwH#Nnk;y2%^q0CY&6&0x0dC7|` zUuyT~o2z$nb31isI_>8{`&}tQ)hTq%8i>uqh?EDYKbc%XPjTIZ^B9ccgRqXNy>i)0 zy|R4nC)_RjYfu7Etiii)9ULIzw7bt+V!NLl%q%OOhpnVq#NqKog{Oqy5=#B5+l z3JvhtGfnVB4Wc0H88X+hdt}_|Vm02!r(gvkS{sgy-Ke2A@iij5d|x!RpwRu-hO7@A z?)Z^=Yt1T>5D%Z4z=$G}@M8!pxzskpKwY9eo$X4GdILqbLyTVK4;^GKYD_ z$45EWrl^ql@XtHUQ@zMVQ$mWFql$Hq;=m<=O%X-CpJ7LeX=fLHlI?5nN1gI}tjEAa zrAqXbNSuAcG9k5}HB+1X01tf6e`KtMck-ucnDJQTr(2~LUY4`_5PvUs%MX4E*)gzrhM6zM9=qFMaUJrK%?-U#r;Fxde1_Ll^Qcs_z}co$e=x$9j<$I1 zbw1Y*D31H5{Hs~%ujNihvTYCLOXn=JrXi=sUbq|+rVjhG)DJe7g!4>f83)qL^{WNfsCpyYu0+ zd|orW7?ZR8)G>0i2x#7C5GNQ5b_iZ-qcbgj6;rF`a2l3+BW2`e0^>y+HI(_X)Zet? zC)8jMcf~Vc3KxSSFYb^Awzrk0FUuC$T+>R0}Iu z)3$bj+bb-r? zX#_rAQGR%bK9sW*)1T3)lsMEf_k+66%_0Z;)TAMTd`ZFrwZ)3vPirK6FbIJagukM# zq(!Ea*-&hND>Dx^O){o*p>)@1D}eIAgw%BP*tx9|$;s>*Y#RCB9?#xZbY{`ZToihq zFHa_H+JHim8yVp0N}#tUesfF4oKQmnD2Z zAp{%Nsza8jc!d%j4a zVrZvnB%un4uBeh!=jHo!^906WoB1kIF(dLeFHH!$TmT(Vn+cT5RFu zvS{EyCTP5$rkVdKTqd=x5K|K+GgD_z)@QlgSj9nY#Pc`cvUjqGP%<)qob~v4Pec94 zn5j7@s#8;9rfw>3oB?ULbErYC9qBKd;lyT@Y2%v>O>Dpb#gXD6YuIRxDug(x^kZjv zxaCJDBuYO22lu1Z{e3CMW1>zjNb1;6CnE$E&u;yzLmjea4u*r}>EoyBTwVMJk4sIp z?sY_Fgv?8GXSKhiLT%;C^BX%Wn&Md|ZCG4sSZQ>$JaxO_v&n$6*KRv-VNBNH^`=uaU<3*ky~n zZ;DUOq3_?&^Y1`7_*QkS z__Z$bkxF58)IcJRya^I@dOys z$BRGkaODp!7|5#D)8=l*V9aNEsdM$18i+)a7}Pqf(r$n92Kl%~@7$8e@fX|A>y2mF zUJy;0t>zb8)SVbV* z0ZUz_lGVMVw6Pqz&fy}cQd=g+QesPSv?~MGcg83Lx?!Y}8tTGG8QkN8z<+560F(fq zT>>LlE+n)}ljvWNz~J^30jKPL^&T@ytl$3?0Z0BV%KS7>xh15?FETb;;&Sl;-H7#1 ziO8S-NsVI|gmOpZ_=M3PCEay0#kn@#ftUrVrmR)Y~CGN zC`qBHaD8Gp(GsnC(?-e|ocDw$KIA>87tdf5@r zcjG%tX}TAGEXjY}!~jC+M}q|4D@+8F2$ILb|D_M$DDuA-CQEvR+{^dKDixW3KoU;7 zvz7qN`rFT7%xLIt_d6w8|M`pD7TN8<*{NdWr{lRn=5{*qAv*|vwBFK`>Z=`tySuK& z7TL3dU;aFe6&I}V8=!Iix6uF7>!AM_mla?0=Qj#uCjkBLCMagC`aTVo)U_0LYv=1X zc6M}`7yhgK15727wpyZr^d|8&pb7=eP(4*&#U3$aps3OxTl9aYjp?tFoe<Y?F`%XLHBDs$Ig7y+WNma>a1?yr^Gw zNV&L9(=P|}5k!d2PKMxpN(JxP{4%28DD^1k^uP!$o%ixNo3sTUcvB5BvY3qhtQ@os zHgM-k&4lm34wXWGrW*R~z#6V>`i%*;M$<1A)!1w4`1}WX4>Bitr>KJUf7D3!PbJqI zS6^3;#ODmc!2aF%>vR{~Ecy za9LtcXoeH24EeTUj0VffXatjhv;df0U^d#*(Of!ONp*r40lBhyQ_>s>RJIu$^Ba!! z7#LP0{=wkvmR?RFNj9DgeNH@a3oV?a$d3g104!JNS$q;!iQZIU?5lFuNzWgGu^0fI z$iC%{AK?TH1rZ*N%@B})6;wHlWrpSY?DLM<6yp8DjABTLXUJ9n9*iC~9!4VHRi3WWoJaJx&vm-Wn&7@&=%p)SlR^R#LZ0JKg>=DdU0cvaEjp}cD zM2^LDtXNYujlDOBxcQw&E2ZDD2mEL{b`N*C7L%>Qa6{F8RQoyxXaZVSM7|htI_V(n zGMp_+Ra3Qga};*L3RQ)_D}Ao5UuD{>J_M4He?*J^=qlYPANdGC!~-NJ&yu^&NjiP_ zyG_}xkBX5=&Tp75uY1Qoc0Va%)MzR*E&RpU)AxN(zoj2Ge}N1Lya!@Vqq5{`gX5{I zigHDw{-6Lans$DF`(c|9sx@hT#bbSOczVcKRH$Ui(^*=y(+}3QLFj$La!beV(r!Ec zlvcvE4gfJrI~TmhBUCNvaj);OQEEeGh*iJwjrwj>d@{p2!LT3T@f?!VXQr*?l@-bm z9gG(=Wy&cOwH1TW#el}E6y+R{4HZ-^P#%3+Y()Yq8iI@6rJJ5sGrFvruHA6!zp-oW zG+p#W4}$bJN~DBBShP(T(Ge2NFZ5)+-&CY{e$7!mO@=h$h-zWk@06)B9%-966cMB_ z0(Wy>Fta9>UA-=tP-w6f_K-8cYXX$x@33{~6#9{NF-psZ$4e>$nIK(yY#`qRQ$)WX zV3(_|r(_VMfsX*`XfR=g&J8UKr<>A_w+dBlT)uoNQV?GnWdNIC`MMrrYx2>V(5xAk zg6o4^5kQ7TF|1Y$Is>%yl)_L}R{E}PQw=rWXIUDXq#myJ93fBX8QyqVIBeI?O$+W& z&3oT%76#@P#8h2`^!|GDKPPUt&33ifH z%Qvx5%91Fm?tPp|q8S>+R9llev2dFEi83eN9}fPkvQ8y4S%e>Sgmq;5PqB@_LOq~T zl;^0F!$3%6*UCwNl)qT#A=a^CRGbkk+9=FJ(%*70V+nl%i!3d8rPiljU0!|`lGj_A z3(Q#_X*As)-lerssLH zZy7ONE4sSBM9#yC9}J}^ixqIF!w16?*XPivOp_4q9E2s5A-rn?pmu2>FdF|^d`?(A zm{tmv33I{Rjq$@z1@`r&pdYn2OgS+Ou*TXCC*;h^xl-Eefhg$nIa?gkVE2R%7t*wql?4%Y0Q4Gl~%RKW(4~=$^I06Lwz^)i@;+hw;r7kG%j=g z+eGf)T(QJ*3FE%eAnknR4UWKjbkrt%*tSNow>G?I%yJ5(U2OMlJeZXAwE5O_{CS#T zUK>GoukRQelQN#yo-9kTs3^KZkz^E zBw*0S0M*O7*Rk;L)d9}{@@scILt)PQTeAnr8C7N9MR>d+%H*kGUM>DKn~y0n`I1nR zkU%C9_0tYe1r>&!j?6ozZh7`vL~0%3{I+1h6UC!V{CR{Ms79`@V&!O~UsKEaxncJd zGt$L;dbTC-g<6<;*PMnf9BsOUy^|?g)cA+1tOG4S?AcweO`)Y3vp56ssTv$B0p4X& zT788G&+K>58M~4?g`D3|+v~rqvmt!3(6`c6zW|8q3?t2F&8l4^L zP@D|84@xwUD`zlcHxLnup<>tfk+cY%(An}G={*(oFUzMXKv4K&3!;{2oL%re5%ET> z&+G@gw8mh_FJ^ac`X%fd&?i3v-m9?Ol?Rudx;?R=WfGYN4C_GBgR)^D;H%a7!ygo$ z_R@r*8bYYCWWWqZ@B7{}gRq&JnVN5D$y}#lFs&L!`E2d34QU6#n-bC`CQ*OCdUW1T zz>_!ky&ibc75>MY)C?|;E@R45H*1mVNn7XVYx8(eXXsc)YZA2G{AKQR>8_ZHf~w)q z^k=|Nwx6`Ar|?FJP7qqmtr!^SqFv_ruO|0~zOk1$nx9l!T5})#Zhdh5)))J@z!4TT z#GXI*P`stQZ@>i7d6?RdIsu2A%Vs}LOMWyB27=OwP`mJl7-rtLH%qrp_)lFSc7ve(W&YR_`^I;GX?i}=4)plGK>$5 z1#tQ1I&hu33nwldp4F|5nWW@P3?fsS!tU_=4qx6Ei_4s&sKI5N{P(an84JtGY8LS-y;YHD{`2r_wYu~UH- z$MWvnR6vMCpS()7Czko#)oYEqZ}cdg^%-tSd-QUx8%}|jI#A4)@#nsMO~u=~gEI*| zK&FWc*=KQ66H`6~&L%qZr{wNqO-Ra4r(oy2H`^s<;vj#T^fLK3_QxvCQ<(@URH=#n zU*R~>Z7Z3a;fCJ-wOw^lKwU$vj%tnZ{df|C?f23&LiCRMOgFyeIw+IGy5}IhxedRb;n<5d1(!l z?@?mi>Svw-O95}c#;fB+NND3EJv==d5+s9V7f1ewiJh|TWy_;Wc!+Z8$H~F8XTGT03 ze?j}9trfYPr>FmT_?Y%oR7x@;+yD8X(d1>Sm!jF9&|Ypgwp9MxiO5&Kl$H?+x%Dpl z*N6VvWPf~$zb5+WFRu&Pbk0iGShNW#QZr=ek^q%pNWKG0S>2a^`QK}YmX+~ddqj&E zlwW`QTPu?xSMyah94kCi@z5G0Um&7!{WV=u!cQ+L3LvJRFM4gmY5hXfe#qosQRA6< z!-1`h-SNV}IXl&c4J;&=!u8b9uwgMOQ6$&!;`TF*ZY}O{BE}g)6^tUV81BupeNunX zE}r0AV4Bft?wj9>^u3}SpSEt-d!p?ZP&`M1?ps119M8KnerFv0U4Pq8Row0R{cXDu z{QbYwGURvKJ1Q-e%tLoc=H2R#`M0l+J}qdb(Vn~Tg#V4VaC)0T&_4(~W`A#keDKJY z%>4=OhN4*q^$~8iu)a@4fi>taNnULJ@AE-RWggl~k4Yh?^dA;%e=)wOd-N%{r!6iu zee1_CV7kHAfw66Mb9P%h4E*;csdC4UvCE*5KV~!l)N&aV;HF#hJ@)ZXg1Hj%lf80ebz6?rx=?5()0Ff*E0`Y4)?f z#;xaRAZj2x6;|i_YfS%i=%ud3xv@FR$jDp!V|lqvd12;uP3`Xyfmg15uo3v_X_bI` zWYkY^lJou^7AW;QJ|F*q-B8hAWZpd3UA04{)ai?VBmEoJFTQf+U)5iB{UgNzv^10O z{V(f!>!<}w-RA8p%kWuvc+co}!}^3HzSO&B6$XlD`w^bVHz56F5z-1gd$%F&Pq%%s zzr137hD$&RAEgK&D#O*p|R$?DOHM9IN^!M=kjb_FhAttr$ zx3l8Nz-yh`8Z`G{#^k~r$t#>GxtGOFYxd!T=;0-4`mfFS2V<{+s=@fq{r2K~Uw>UE zj89hU*t_!Qm0l;jtIH2jDW9~rv|iBolw8X&xcc_Xv`gCXf}EPCv)NQAKwP{#6gRF( zI_pdI&*T_f82N^BfUR`BP}ZrO`D%Ab4z)pi(rR7x>6~cxC!P1Rsj4M>q3B|EvWe^5 zteAA@zkf|EMykTZU9($nvI@Ma>j-Ssja+19n&U2l|^lYKd&QSl7>F_f}ZIue`dkw#I)!iCq)e>TV?p9+7$W*gb zenW7kgU<2KDLS1ppC`Lfbt~Wb&qGg(M)6Jws7pTdu@^N8a2n!S>&R&NK888FiUuv-8n1%(uNC9>;Fq}-yUmv(=fthC9-NZHQzVw3 zx4bJ9^sNg45Jm^Ub4F-tJfqDs@t`^*PI;T#8s?-yv_?HIVJu@&()qO6t);oLf|dFs znx?0I9!NL$9~9V#g0*3CPE$##;VZh{iW8|EWXs4$=`dRxFp`K6Z2vrzGJrO9^>s947s?yrLPE=Od{b zO!rd1y7pl-BsGS=2_llPi`$Q83Kxu(j*@kbV0ItRIlQqfH-k(S9ZNfR^KS&J*Ag&F zZh~l0a~td0pEf*=u{j`ao;FYzy0%(&(eb^*Sc_f}FeaANac?(X+?(aP$}6XRtbqx{^C0l_bJ+}(Yp)F zZQKHctE>dMDAJPea*Yk|68^#R9Zt(rhGD;ff35Juv^xzS4}oh%)qxj}k{XMl*#Ock z$&Xq%JYJy8st1z*xQS?*T}4zv8mWk}19HSTbV1gz@12Ib_6_Rsy!OY@(-`mI?TEZR z>>`xxe)pzge?jY8`wrYWEuh5|w+2SeraGnhPgP7iq8?v}O^#SvjPIr;<&lbwb3NO`dD`#TB28 zc8db&3a#^k=(~dMHIyqERU&s?+qL_I-$&hbkW~=dKp^r3_*z9NM!g_sySwsuik19q zxA9Oo37gCvE6g!ZRU56Xv)Sxr_8N-(kmN57-76p&B8HtB#|(N37K zK#vN`h6`HOvfA)6$rIB!2EPlGE(mZy2j>K+NS%uGDE7OlUEMSqtOr&^DF7UET@C}g z)8_pSdJ>}Xstz%XTVA(%9HbRIV|o!}y>dFjw1%uqp`c=R?dbf3fTE5(Ne2&WRoPCW zsBcnaJkmG8Zb3ok-P~v5a*COxIBK27xBLhB#o@t12wql>Bo~9`=y6KyhJnfcCBE;l zfS#h(35TRv5@!dB(3kHN6M~okaljVrLmf{{zMm4x04aWl2#-lE0{L(^;2;sk|*G&0z9;h6mWo|4CsCE^dp zy^%T9;5cSCF?(pF!!7}#zWB(+L6MT7@>;){jYB-aNFd2PPEYB$wnH~!4=PG&Gt~J7 zUJiN!l;-UNr*Q6O2s`%vR8c0;HoG#!k?(X^>XWvj-wHWTAmH~R5aU*|4lmm|zmYe? z{n5gg7qGoM+2r80N4D*-$B13$8c%*P zkS7Evc!F+^mpV`z3ej1uaQNy!RO5uicRu3*fr5+l)SVBT?G0K_)>O32o0~gZPnMi` zbB?#PH@?pSIg4Y1_POg&8_o30padZMgz@y~`1Xn}rBCBoXAo;f7KXJ~Dr&k*-u zEc5E|xo_o6%4-QZywy^Sxh7afk{(2XAS52##Xx5=KnA3I9~w_55bC24i60Og6-G$# z%#)jqp$o|^RtmGSooO04s9xQ^pkctUO*x;YurCxfX^+3Rf=#@-maB#_RwtUO&X5o_ zR;rK1lgMjX15QuR@l%$tzHhl*VM6ruV>c^?=>r>HpATd&56$iKPUI>%1gVy99gLs9 zusVgfTSg*V@R@XcvSD&=>~ro@l4iMJxnKjexv)<575ASzzMZ@EW^*-WwdCoJht2d$ z6vpiG&62Erf@gXU^VjQ6&t--wD__jVRp8AHF4^rxfm#x7^%UaPw}?0;^oToo*~LO; zb+xR=cT@~F3ePw3HX3a-po!9jlCV>4sDKfv80pg&VouH@;EJPRIYWx-3v&E0Gc)8V zxCs_UZ!d=WHVCQ!1~V6aPtS!6X6I$#!gq*#P!bEi+E1;b4>DZ27EU zqOutk`VuOZri2cFt3x<_gYYp?rywiqsL~DO8Bd4?998rOku?x+low#Z7SSwL= zlm$YD;WRY3X2i_QpDT5wmeL}bG8T4Mm5)2J(Za6C+3!r}@LB7<&9JkonWWfMdD0!n z-TP50nLj9FcC*KsoG+aS@#`uIkn^sMtmIt4v8R) z!XW}kwsP$@ZhT5xePB3~Xj;YCRn46gz*J!mfx2FLh-H{F_{fk&*xl4y`}cW%FWC0c zg7$eP<}Bx$tHGVLdO=i~3_|StWZZCfkxLT~!t>3F0IDJ{slLfjn}RQbW>AlVIc?uL z(s=Z<4XnjyrLm=YIsW}-5qorDeTj@996L6^Bf;2gbC_3yc|_R*8F{OSMaFDV{>&B! zs(&sr+2Z>{7B7^+ijyYF=8B2lLnR+CIsPE{K!BRx7t0sgo2nlB0j#>P+;15*-~QCL zXWd#E5~pQU(dhK^@-{LM5cQa?n4T24dAxJ;W^mkl{^bdfh!)fQt?UCG=lZt;##E?| zMiF07Q}Eq8A=qtJXDGFr0kxTrx!?~~8>VMWI!{Z<`F6Q}UpIyP-cz1&;1VQ6t+Gwa z80a+yWXPB}UMzv*_&E{F7WIXC)>M$6f$M^GmlT(AVwng#3k0d%od*rMla3Gm#m7wv zhQHmTob3;~g1;_(FPR=9ZbAv39=`yKkZGa;U{04Bn@lJG0TaZe@%S3Y3T+eTdI>%fyHZGBC&Bt>$J`xl z`>6Y<%j{@(^;sl>qP|#dAF!+VFq5D#i4Ak{EE$;e_`bFG2RVa;EeYDo*Gr#7`YjKQ zWB5-*O*zcRq0W;eftciK(_LB*%eb0< z$yw^9GrrYbNske7+*F|%doA!jp5G|A|7E7d5V6e1ahS3<5y&~bWHNV3KR2hva7sU; z!1w<0PTb?|o%kK~jE}R2+<6Hv(yGiW?V{tPt)`z^!76QrIX*GLacT3lFM0BNAlNbu+CvQkHt>Zko=+v8~kjxk6vV zk=Z3s&d{c4bfC=!Ue*~C>TorJ1(RnI|5=soc=3d?kG-I6uWl6^ZbHrqBanP|>M;^J zYVg*ZNpCTIeRRdl$Kq<)zVKB43W#`y2n*m?yXW|t1se<_EHh)etR984^bomBtACxS zFrG!&NPc%Bxl8yRUw^xobVoXOq%W{(90RX1V+^KDaqhh;Fj&%W>-bI4yOFm2hD*hG zlDa91NEB-*FEZgHz=EV4oNN3$#>l3JXUit`ch_`eYQJanKK3zCk+aUz1?glE*@W(; z>OfS&=4764%y+b>t{6J7owp^&#;L!Fkkn$~3}tD)S^Bb|5Ua6!!fC6Ys%#U=uFPJy zEcdk|$*#A?M))Lp;2`8chPj2N3nH!RvIue#SZxS*4PvxT&@iJ4+P`7x6u%6NJLHI> zGgy3kzm8oiF?Xi064=l*c46&|^QD^LIXMZf=K=@+S_^SrohJBs@_0YqVH~+lY-PK| zvY2HoYycw1QFm1U9AcE!o$_K^iJLk(Tb&E;!nYy20SOb39Q5Pf6t;)XP21L4vvi-P zFuty_I-^MBzA|A%X89b;IB36yg#taV%bD@<0vTn2&K)xLwM|A%CB47Lq1-Z793!}5 zs_R|#$(DXNg$zV4tiz?F2U-ct{B>clLB1{A5%~8K~5n-AeG9;>X6^CoVEsO zX|k81^r4qYzbDNhp${&P4Zq#qqWm8~@8kdYdPcyV9$wL*&&bjq<)h%QC!hHU+n?eU zmhvSgJqP~hK*9aTkPE3;6I$x}515Dkj{IlqU#SimUe}}C+OwRLLZtCMi%ysF`+LwwiQrT)cq4;+|vZnr6rm(?xDyzP@IXsO4wJRnpO6C zeVX{LVgk)S06}S~N8raNJ=E05O+%6x@cnEWkpuUFGQg6`uGr#cwsSI~BaKC} z-7!Oi%H|g}DeDv8;-G2Yh5I|WlKw%_E35l;dHNP5n$^zO;}42q<>{<8@>Bl{Iz*vI zl9NVZ@BV+-z$Zv)tG(e?_fhb(Q00b;J|hFE_WGm1V=!O*UjRjL?O$|19?b9T><6d} zLlgy05``Qn3<*Il30+vdfw%46^?!Tv|Hy>@pReVgz2zC;-F*A2tHZkMzoRFi07-of zDcl~gXB4GK1u2eLTL$o-q9hNDtfV=9jX@e2m3eu3vnFaL?h;1Ou(mzYaOec4e77}vL z&b%e1FV@mzhGVO|o`gx`&YQ9ha+-35nh~qktt-P8 z3HNvgk^d3{HwOLvgfVu(Ta_WB7Pp4@HiVD$2`%#eB@@$Od&c=Szq!qiC zWddwYQh9+r;~_A?@OVQN$RLZ1HA}HIA!M?!L}~m@g}9lX+rc`RQodxqp> z!dy;HSdN)XTAM<(ZLnis6LEitJqX-4`g0mW4IBx8Wko765XZhw3)<6f^I;qtu{qa^zJ;Y3cpM^l4sVSP!!95tJ_BjT~f#C)q2{~E7g2)iH9Kq%g%|5=ZjI!#n`z`HL=@T#8cGgpUhJ4Cc`2L_s$lj|=l%Ek- zOVzYh*-7cjc~>}xn1e1@!MUJ;N&rJb^y*}a(|v86_Mp~$Qn)80CnuU_dl?h@=O=mH zA2g)Nq1?UiyT2m}D(+_<2|XkSFFxI2{G4`AEAB_Cz;C??ruJnzer!;{Q)8xna)*CyukJ21R~IqL2|c8 z=9|PO=c(c$(3L&&z8rSpb6Va&cK9}7c}eKoDEZ5zOGtqQa?}@2Zc(q_Td})c)};J# z_#`IhaV+gYI-*NB#mT&Ym^fdW!Av(S(o#S>Fet&|m5CGDf)#*qR>vl!w0BN6Tp@Lc z5yU$+rDtaU$kh9!bUYIt_0-%yBl0BbMRUhs`!sIbB4%D|HQ78|;v% zMGhn2gclMmlJAjuL*^Fj&W&>|U=R(+EaqX_-NCaB*4uS)!+@kF_# z5dQXKs|QWTNs*A;<>yP{VOBxt;Q`fDrpawgP@VE*MyzSSvt#}_!!95tV4BuLc(N&R zVz5LSS$%}fs5j}sQp8!9lDgCv#?$qqb5|Jdj~9s9Kzy?XmP#Vy->L!wFAcOGFC_LW zoj}BA^rxYj_o0`GF^F}$+ZW7j8{~pM9e+^Z&-XvJ2#G(c`wr08mtc!_<~$A<9a0VF z+yO2s3LZ!GILmpJ^>|2YEoI+q)UP#kLG?KKM(L)iZ7ovLKNY7HjdPIkgoCwTx2#^F z^fXtibdp_=O7oAGYT_=O8LUFf;~S?10!g&&R7tnz8kIkeCo{<`@15^moMxstjWw=vWD{p#}k|TO#8ih=qsrv)^wtOtd<&noLlCWH1-+z+J zZ8z7Pbeo#Skem=Vxg2%Ax%YB5$}M@Fr_UAurm+U%=SM%Pq%o>72*`xAs2dC|lQ*RfAxm-m2+_jWeKecs=@} zu{+e`PMDwIQ^cv`O1*(p(KswA48M?_fm`I+>s@XOjE^P5d)#u4-9ao|ljvC4In8!FT|r$6;b1Z{cX21+ zKw+0Z1@vBYw=Gc%q$7js5k`S`H$P1A%)<)t3x5+S4LasEt+Ndo@xNIUTzA|cd>vl~ zMAf5^c-A6#f(wV0NPPD6Ji<-#x(9Tff z&Ar{}UP*sYOpbhvC~y7wFV<8o-9(M<5_I;eULn@4e$P$3$^CX+%E(QL`?8P))BYFO zIlB!z2fSxKI$pO^0j}xA2&Ma$A1QLCgR@oyVwvWA1QPYSb=%#MJofJq&}B|tLX)Rb z=%r2reiHk$o9UjY=Z`vUSvbT_7Q)Dy<+IEbQox*(w_(6nfSwdaaccZ-NzVcMs|R&5 zvOQ$dTt;+x|8k^EWgFBYH?+IEN8@Y}*&fj4RyAiDGNxmbKvReM)-+F(s_=kDmYK`1 zn#2YJ^yc$3->1O2sqA-`tIb{Zl2JIXNbt_`mq~IfGWNSKNV16;6?+1FQBY7)b-b<{ zZ(iM0)|(R=J0%d%Hblg%MKM{nIWDhfQuC~MSj#QxFeQ!H zJT&DKEFe7%fi+5l^0U5GD-HF6*gY$p9!Km6uDJ_()t=}#otow(bbV{e!HHg%Sk$>m z;H>O5x~+#hcU=4Zhm!hZQU<~YwSoU)#o&>c&@6_~WtQvi_-=?N5(HLB=|l654eREQeZGt3q9Q+~^*2u)$E zHN#&CNr)auA(!IsXHL)F4NCWpavphquTr4~b$+y&DYLAmH<9xGdH=n?G|_c%^n73f zO8%Xz_<0-+oPnrB*fJct*EYm|(Y9O~Rnpq&Q@5GyUhH>pe28@H58#muzM)n=t=Fek zp%L$&+ARSqqq4eX_gy7$Y}%|S6NyVp>nXbj+f@j0tqlG#x)B z`RF}GY3&()Cjf=S-1aJ7O7^Ea?&>=h=dvjbYgyW=!`bM!Frqh9Evoh>lZh#c4RXyi zar3;l2EXhD0WIzI^QMed^5h2g%Vt}E(WL_i$JK+S{$nkQ9z`JfVFC+!-30;gc3^{W3=^Pjj>ZDH>13(UdLy(X23Muam9 z;rsE3Wo|19^vAK6RTBp>hoO@c`R6Va;9Xb6l2en?N#B1P)imbFj8EgWivZMg3{C7~88V#nE+0aXejHLCAa(Le+4gt)En&r*mU| za}fggj8$Nym_&X&{P2DFVTURA?GhGucu-NPneLp~$r0rzXfxXfr|EdqT*f}D9K;f) zhpa)^y@nCuho8nbGJrnU^`~rR{)^&?T-AFCMCHU-S-e`9JY|)g`H4XaOM#vBi{}NW zEJFhf+T-{h%$@6w?r?Jbh>deasp*~i61gQr&Ui>j)`8j1>4DNs>XC8yWWI(eW367n zXbyNy{-ESFYAgc342wZFv?*3vM#Zhl&7!SIZ~LuW^9fB4J;mZg94!!sf!fGMwp1V) zzud3mbdG~*Fu3R09~7Lfg}1A5*WA6%=?&jh^PXH^J|-USdCbr7VH%a9Nb?-p-ySdQ z`XYnv5HK|Ua{CfU9qyaWwUa3@abWM%P=aqWR8Eq z@A?I&zhIkGcX2^6O8RF@dJ+oYu6zBQ`d_LbQWy%QSz~$Vc@ozCS0xH#`kNPt@|RnI z)K{La1wUK(a*s}&-HlVBv>QBG(tE{)WNyR<8 zIbsI>!+z3*%hftAGjs(igB25tM5rz(GY4YdzgCvKN3Ywj58$2^i`HI%Mho##db?8J zWk##;b*>x^Q;)8TAR8K^+Q;|BrUGUnq971``n|61_>s zi!SYGE9WkMa~hvvYQ(I^Sw*q!&r*&MyiOf)%}g9cgKNyDW{QZk8v*6@f>1%RuR_=N zuOln}setH{nIbdix2wvU_H6G7{_fiFDE7?}A7&Iu#LeT2yZ;2mCra;5{_;cYyRS~$ zQP)RMUdrZb-07SgRH8(Gc;|WY`t_^BJ+^Z}gF3*!H+u8M?cLhHh>rK7c0$YE1|M-| zR{mvQFix%17d>@&LYaxo(7Mbv@)Y!yzuLZ!JNtFu(wqEuEKQ|&nsjS=e}X{2v7cIq(%~qm56Po=(W9LBecTBn(?#35&CF2qxGD?T)wF@uI&FzL5*CZvCKEzcC ziqbN^+r}mHce+ z#lnP!gW1Q+!Uxx~CvSC~kvPxSc@0sHfsFadT9~4nlisKWj7NFuHpc`iUd_!%mDz#m zcjrz&@B1lFhjwA4f<6+XJd)Z)4%!<_Gl}1qB5lwi87?b%iF40SH(mC8Wbi(a;W3+DqJc{ur-jh~Q#G zt<~o@wThNH6o#nMXx6*>^Yo z{p=bdiQT<$XMv+}L`fS{ySbqM9k)=HBYpx=vT|HC65>45)}O5VaK^NBFAl%b^iMES zD>@#Mm5>#~p4yFB`GO|I%BK+;p<-R|MQ)Y@Y8Sj}Xl*L&64?}fNeUYx;n1T+sz?o0 z_sYmKm!bjkgsuU8ykPOS1-@=cXz|My_u{gj(r<1dXv9Y-UMVh*saxw5Q(@X>EBwR{ zHs!Q%M(Zu!a_tIzDITv8)cwi+-P*dmsc1=^Rn}NlMpkP+qJ7A35{OPpj$F)fE~AP;UpCXm92Q-D_X8^q2e2dL=wf#HRWDqNVK#Lw2ksw z`HVvQPIn*zp`r>_9rl;flNj?-56|3c!@r2qfkKbo+7ggtfTL$$mYD2li@KrCze2=R zE+{?yjQ5HC$pys@oEz7X9~6GR zMoa3p0Ht!zM|-pI%|=Bk8`mpm%#qe|a1nXqfwOCrSa-nMb#s0Rq^o9quSA%a35WW^ zt3~WwI)x1jy{j2&Qe|!O(DJaf`w16cm3h@w9(y=O)R@t?^&G0Z40n=kM9MUgO6ylG zf7=mFNa!P^zHFfDN)2a-wy|0~+_W~*5h&5RTyVnb^ehty;F6>2yX^Fx`F?jW5M6SU z;-kv%4z6(GEIXBU*VEsn`CX{ibdVoAZ(Kh)SV2d&=s#@<(t*5=z%DY47~)A)bW!S$ z_pDHJ5YlH2RoqY`l|zinhc=8GoWqHx^s}+_v(FTeP;UrpHzayLndmSM$f33rk3CmX zuDs!CbQRO63=6=KpsnnbTT)UoB^0|k<59R zC2Q{fK~d+pxAZcM*7H^kQTQ3fANXXKb3zwB32)<4`>ZIuPFz%#keKF`Y)@tr8VL%T z)T&aUbeMzH>zFt@xMBst){0!1Gs!bDQnzo$^lQ&72IoH9ni9qnY_bzX_vpfTSr{QX z#b8Kr9G)X@V;qBAFH)a1c2>G=8O zS-Dt1nG7qgyxhLi3`=iK5}7@&tihBN--B_fftI#OXof=a$9lQGWQih}kfccwpr3ll({x{^A>PgTdU z$u;bgE4k;#tbJ|@V05UN30pBEd}ChB@Vt>ncMf8fsS-pE3B@kzXHIJMEXVP6|{Meec(3x|dljTB(vnM|%X5z|J?bdi=ZY;HU(=*~IVXeK)`#5?o zP-!^CP~$_2xFH8x9F&9+cl~2HPKk*EXzT{qlk3kybvjc}kh8C-1EANxbGbh}w3?Ur zaLB${0+K12h{Qcb=%MbNYcrLUvl*m7q4z2?S9-WJ#Zf-|Hh1dZpW^X2_2Ev>zJl*X z;(f_+~8-A@<~7MYtwejCx(AWOc2+uL^SlKN)6!$NfpVK|GI!E%C=1} zx#39hYaOnPwq|Ds8E##lL&N=e{j`PGeuK+?!xb1AG@nV{s%E*jUJr3v?44(RnOZPh z@0@8?g62Tz32?tRwMf1hgHFL&4NW9h#%nHka>?-+rxnB)oa|&XkYm%wP0yrTHTrXkgebN zOV_hIF}bNb{`6b}KJEC5Cz&`4-gMqqS~&rcZUj%S7lI^b0P3U$9-5Qpl10dD>Onoz z9)~FbclJGfywN?RTZ66tjI=BXRm~EfS5_ zq~V&asRoWc40X_)CGVSu8U66-<~mr8V~6qk(El=gX=%l01-DbL3`TprX10%nz*Mz`H>Y+RSAt)1>x2w&{!Hu% zy&6#{#i{9R1=pEz_i}pAG~Erg9Jy1FVmy&x^3FQ_NyG!Jd6izz`~32SUn@feqvm5t z`3)`-%}lvN`a#2ICFe@?+LD6F*?1lrlj&${HuzjXDQ|{JjXy7lhTyIS417ZodrJ&{ zWJBlQ4{VNgN?nZPo6g9@`#o1;lajlz&5qz7Ya*?oW1in03gAAYIU^}ab9}}5zJ{t1 z!%SD^leD+M4;(5ZWNMbLQz3AkFs29-r1}*fPm*z!(IO>$W3zB`ke1^mcr3J9Td32F z=&V95W?n>svg&GH2XJTTRKx!`KxwV^2W5{=W~h-rmnd%vdf#1MtK45XIhcF{=| z<4kdQS+{ETJy_o)ERC-+C6S;@rvLHMJ@>51BDhCdj5Yt^wC2XVqJQqXqN5XcmQ;kR zzn{E}r|kilb#74agRB9Ho0=wOAqo{j>ahE1*xuPqcFGRr{kx2jGP6cCU`4HzmL;*c zs&1lOUi$zsYj9G|Cp^LfhVdGj(k)Eu`2ZIz=@{j3*sisVJ63cz2da9=p_)YkNX7#h z!*k}@KNf5ZlUsuInf=nb#1%=sSG(Q3SF4Qd?2THex&u_tNO4X?H`z@N7n^&ICEV&? zx`*&bUy2nLH{wz6pOqk_gQ?H2-sLd&YUK^2OM8oJb*}u#??X4Tq@TEfLK1c3`UzEO zh%_Ye1ujEO&*;wW6b-`s4g=nq2v^$om1bSu9Q42j3VPRV;W@8P00J09hVOB zatdV8#sr5LaOeEkBhgru+SZvXDaP9wUmxA*(*{rYC0tY_^iiRZAAos?NPYP#dlnyi zITi%J1_;vlkWb?WwpFE7Ly}D;1Z%GbX|NPEN`qVUUhYv_e#xC!@nQl;XQCGw8=+d` zblIZw9I>Jh(HeY^{BE8!$!M^|xpjk6nyy2Yqg6&BxKd*HYHFa?^!>UIQNqWZlQ+ei1-rr!qmOU%t$!~|jiCQ8DiZHMDCU2(4FaT-g^03Z#=Eo4`ae_) zQGNN$!o=O>f9W7-v$_l03wO5*C&&LKh&(x4nCM?TzMzyp`4`kdw?1O)o#DCqkKKVr zceb>>d5+Tk@jpTd{u%Td>ie&{Mv*2vsb8}R@eRK@-1M+R#zwB?3S7DQI`%xYwrYaf z#&NejKYqM9yA;6YJ-`3`*Ls7>ZD4qHN?ylvh6D_y(=7Gl{L&K@sVQR?VI zH68!kC>eo;Q&Rs0kA>rfm_~$p>VQvX=HxT9%oyG~Jc6M=C?EfA-ZKduo1u2JD+*7$ zmLK&axc|z0jK+oOtw%8y4A43seJ z`riKjKZQWh(N4$U_wi=v-%9++r@DJx0|)FsC@hEf3$6>BEuXj~dH)}--ZHAKE?V0T zUZ7}kiWT<k|bX4P|$r~+hu?Hzx71j7Y}&|?*8ZH z;*R3sNX_L6n>oM#Yd{Jx3X=Q{06*Ph`~8por{EdY&}WRm8&O|hc)(v9QRgA?T@YdvGHvsj?*t$wz*VL5MV*?>veARB|6g2B_Wo z*LDZ+JN%2}K|)_i8iM@WTgWwP^I|J!LGS+=%_u?R+W+l%p&B1FLFSnt;m!RhKskMO zXyEO+Ky%;2(|bVKJ1)T8^~I&=bvJ-^WmQ1T7akw!0}~qI^)oNbyg% zul}Ax*KoT@AK%%6%_i5`*%&jkxlSM=EGh==%6azVXb;RV_bf??QFLMyqhjgff7VA~ z7eUqz^}J$!YVxCOfwy>VTT$*G&ithD7wL&m@$Dxu}fMi~JW&2FCD z)g=0`YN3}3JDI%24RCgKzHj4KUqi1NIjx5icA{*&wQ9EG7s zOlqUD>B*Mbt5N~hO>Sv&O+0xQS^nWTuI_oKk{Pd_^?}^`GZXuMPL*%d_If@Lt)>^1 zHYTcf_)nLlwI?0iP73@!qpieNlT95Kalu~Z>Q?KQCfrp|#0v)vQ^CZL>2RRPo?qgO zp;?Pw3s>M6-&m{69uCOb?7!I!P}!^@y6*6)S%zj=_otH|{l-pRQPD>6)tm(mpS3a3 z7CwNq!S&jz&-b$Sy{!= zrQI9NDExieO50Xu{+A&mSRBajoHeFdCG6#$Fg?5|if&*%Gt?Lm^UfRLd~;O;k`%G4 zsZ}?zyC@?;s8VRQOoFlYA2^=u-gAp|g2wqP;?2iW9LPUF4by+#H5)qpC_mpuaD&`- zR^2lE0TD1Kx)MWWC*EZv=cb|t9-HxO0Tr@_YYrjBgFI1;FHPaJkBZ{_!2y1g-GYA8 zxd>3MBT&q3;Rlz9gM9%)>kr!^4xsl*Ji5o1Pv!NebmDegzIt^FHoMY2-?T359|v-Q zTV%2y(cgqp=bQnM=5<T#HXo1JfIT;1EE4ZHLjruOVr1|B9x8%;Ec0t7sbOSfJ)Xt1nYa7 zoO0p}JvHE1g^Re1t3ngE5B1}`*2fc7z(1aEH=oqD-Q7Bk_wV zL*&yK5-a2SsHz7SOG^1xSzKO|_#%*FXw|bEtto!xDUO-+;#2wCDz%@@{lTCNp^k`E zAeD=@lIE7Ix7eMRT84y}VKqsuN(OGVY~`0w{bJQ{?T@%dWPYGfPl)lFF^!35r2JLK zm(?ySGu4kwXN6hABKBWd+NB#nOLCU4# ztBA>>0By+o2-2-(_w-uYcXH|>*hnn)!1a-}xD`&Fb@2Tln@s+=nVy}7yZuc&qI|bz z*q+WXv6Rx@X-FEB7`P+-axwgce3F`2+P!()M&zOJQ`zy4IV<&%C7Vq0V=T=e%9!LS z=>>HeZeWctU&~cfm^gAfPr8vIDf#e$oVkOM3t^Qi!&3&SZmnbscUtTB=*N#iSHa1y z)+D4R%r=ke=5BD2GJ<)YRA&n9xYdR&(_H*wAjEEdM`ey?h9rFUY1t> z2818^`H3l$YDQjdWsIFA=UAs)(3{<|C+AIh8da8iA+qsWQ>M9xT$MM5+}EZ}9)5mg zqpxgj5$ti1S#2>B8S^cca&{mH!V*>%AUNw%{YFV7_?F0RYIT`07UB>i8j>rC#@6bcr7_-a-s)i;kqb+=d#O^y3CL_ zF+O>nbKVF~$eBEaFK8ygj$(_GujHJALiqW6wF$^fmO}yk`ZT2N0y5$ZIl@Gs_4W1o z)AARyx2*Y@R5~cwpNmxY$fOUaeI**Aw-a!#tWKF3*n)_A8CbG2a`QlzhHA-M`9|D> zY{qr`#==>Frw*6)9@u_SY{2J3E0{`}0zhf>M5h-X0rX!tzfMcHwb?Fruy2n!P?C zM|U#hyAgl*Zj(XQRkb*_i`$dqLyA+blKU+~MJIOMG>7FqLFkvbD?uh0p*Q?5qJz~&S83&T*v ze4eB2d}=>XaAH<$j;P^mS?fURbBcCkH6wfQ0eJMXk+kcKa zqCH~#N#_{BTZY=t?I7fZu9PGy(`YhU@py-swktpPfg(=`0m{c}mVHmpQoYg|0-E`m zrQ?bV>NZcuQYxKM_vLK4NniQM&+8Ijl3QlyKwt`!WRR0IHU#~C4~-!ir}>WaT&>8& zRB=*lb)lMci@M6s`$=bCTcr&&;zy-TB?#1PpK<9M-ovd`5a;~9;Zmz9IWF>RQe%Og zR#Gspr8|nmwmDcC7MLm&y`z#{RNcH7sWOu5Q!228V(s8A$NTTLwcC=5T`Dfo+zFy;k50G(bP&T z&d1NNAoUVRL$RQ;!k-zYU9w|j$PZ_)Zf~&#YF4F>JzjhoVR$E`awPOJLHG1*TSvon zZTPWRT!;@$34w}hy8b65u9>BF#$0N#Hz~)i_Kr8QN@4=6t~EZ*5QVz>ykL|vwG1`p zg9HP(rG5j3a*+4Rd%QUC2)Qgf@hls3+=v}y?c=J5 zTb7kuG?mMmmo(PJf$Xt6JbWS#`PcN}G`1X8x+;s5M%!GjUwU_F1IhDgj zINIXPynhZQM(TSc%SReXU+a8&iLY8#S^Q}=n2?B)S6QA@qo+M><@A91LaSt`LhHxe ztAsm;VzQJ)@zEESt&Ft@^!wB$^vE?Xu+X8pQk1)u%(5(!U#PUOb}@$|OeMbjnrgw| z?Y)ya3ekXUS^Sbrt`DwujEiXeeNfji^pS^RcPRAs85@DHx?Vy&8TbQV#Wl2c94Z9s zW)QFXERFtdkaEX+efSfi^{ugAi@?L*Xgi>%=b~TZe`3aszt~rVOUJ_xWTCoxQGM{Y z$3XP8&wwP4Z>^>KCl3<<(u)T~dE-L-8leh6&h}_^b6Iii9)jLJwf3UkKA~JVLt8fB z0RTYj`i*mNSqlLD5uq3|4Q78lK>}o(4!I1axRwT|1K4^|jgRXQ%87a*K!{TFK-ay| zk?5W2BA^n0(szu2#*i*wNpk%L#MA#xP!Qz?r$2SNor^#6&=Mm8S)6+#$E)^sNpNJ) zI{@Ver#JBXLw}Ep_MKP&=iq{<>lyvdp4znpG=#$E;vAk0ucz^V9!>zzMBQCmHgf^+ z8OIy|sK632G8#9r+PY}u!r2WV*>MSKJrHcJQm41P_<@|y`6z#NN%9jud)`L*1DVr& zr~4jh{NdpUiD`SG7ft57?+Iu3!uCS+B_$Kf_Q=@Y|;_#wsUa~1K`UI zFRyjFI#-)tV`QHrWF^Ho6JO2iHeF?00)5-y*&)wZnS*)XV65deOWOhfvy}2TrcdL+J~>s01#84-%%{JzL!X`oFN+O~n5i_0Ku6_t0Auc)KHqj0@5kA7%W)z)Y z;(&R!wx4%y9aUa{U(P*qL?6GaDqUUbUOzC^xB^}I4TB|j<6PN1^O(5e<|S?Otd9!x zmd6rJCs`_YuqLHC+_d(H<&-BHJg_!PT=f`imq+aDbSN^FGWfp{Ex!BlCJLPkH#Nv< z=`{Iuk4GfqkSR*`J$s?CDX-V#?qT0j=MHSPvT{CkA7XFFj(9njyPsOEvVQq-Ml3ST z!MQcnT~}iy_tK5Q+PG11v@9?kDN&QILrhV%3x0w;VvOpbxPs3U9$;5Bs#A0SUX5&U zO~s3e&%csnEO~oxuNOfUL;OXIlkNu$?cgo{22`nC)-I!nY=1gazI90ASpg?!;W@bXu>!OKdOfGNLogwe#hEDoN#~@qqU0hGaTS1Zf4LDeQfJLceq(L3Qq89$u0Y*ALIJP&sX?$03#vP ziiZJ!(NFlpNeF%L8WVmyei-$}Y#tGu(sBIhQe2PIl)6c-W)xsI#0NFGxxTkeEzS-m zwr6Q>YC{XH%8hjd3Ygh^+xSC+M%Nty>*b{zIlQjV%D^O@Sr3{1o`GpN(PYAoCI>`~ zVL2kUw%E;{&MYkrI$An6UzE=J3@Poynfh{M-f|c=3UwIOk%GA>|3|I)q{P5TkwD+C zL?6trf9{`S`ue?1@#C>LgoekZiBkYb0a+zH2>-}G^}7GlR%TO0)^sr|yMwIY0q6|{ z(8ONqgMgrj;?Bf1M;2>Vl%4Q-zfm=2Fr`gpLII7h)5toB1&FALZt5pIuUELzbMINf|D(9mw%-Fbs9UnH8_9Ip9 zv<|h>)}OZ2HP%ZaXFIS3HAWtl3+19`s=BkQFg@t_1sJqhC0_m9x=W3cyyWU`{Ia@> z*Ob}pw3Kdlv6XM+kQq7Aw;go|j-`kYNkMrOZPeckTjhqrF zvt5Rz(NQ370?~9;Ks+xDf+u00SwBeEBKD@-W#R081Q>tB{lsD`oyeMoaj;}rU06Er_!i^U(zbUdGX8N>$4SPzC; zh~}zKV$E|m~%c)nUT856L(Brs%WiRRRF6$nI^kZ_*Tq&-fBU`P5*e+(XzZ$)yGirJN- zqjRBTU~EG6q*omYLEW9N*92DAh5uC%9uz~^3}2pn0cgHYP7O0oF2=7cmqn?5o9F|4 zrr7=J`SI$8#Il}NQmIIP>NU-R71(xeyI!2P7n8T5MLSy}HG>ktM_6(|#=@ z5@(6dqA1+?;HK<>aMsUmOPmFAT8xS;@!A4hbje2gE?0uaVfL$Rss}m7thY)^_cd@) zf3X2%AE2|Nco5+!nKGZh#)Q6G8^03aJR(Wa*(LPOkoG7=(@?uP>_mrVEtJ7H+!*oY zVV|X=N$GqTcb4SFqm!MbBYGqxwc}Y&h74VJIb`5wdY(xYKONQi-ySxG z8D6;Qi?}A$xgRkbs}PQoD3L>N%bFm9p&+Hz)q#YA#)rEJvKY*<1=ZAME|pu{wgzC1 zQKeBd(>0rc0n^WGhI43roYp4PwL9%$0`@b=z{d6Y=>la<+SklX&s|h5MwOOduLhJ^ zHRwY~%h(<}&$rs?iilt0&y%^k<$lB7X1AERDuvsBnz}boD&ymX1+TRm7nb2w8*^?8pP{mD`|-X3 zE3pK-e-(!TW>mxOMN|MNCfw*I#QY}Q;&M{k(fCW(qv3&yc6(r9^I9tf26vAQ{fuS} z-sx#XB{%3u^sv#pl!%nd74Hbo&_qz{^@dACit(`;bAA^@ANeUUl{0&*s_F9Wn*7>? zWQ@9V%d@sWaQu#XTB-06eE-JRY^4oqua^^CIJ(P)asoZf{;oK)Ol7qDv&m-dvrRh) zvZVq;=e6wUZxR?AUQ!kWQ6`D66y+ic4*#)dT39XB{cI}nz}2X=<%rT5&75dY%r{#r zQ;1NT3@71??U;f;d-=TQ`T4TmK4n64Kw4p7tyQO?ONjNOxSU8D%cUHmS!SLl+RKmo z5<+4nZ)h65kCzUiVrgw))~pGdAfJ{K2tHDBVQsbOS-)}AoFxPxFIRcf79TrDY~Q;i zq-jj0jMB8b#tJTpcMLthGEo!bv6+tyxW-G>nvZsAjD7a0Wyno!Ke9V_!ngT#ArHZh z^mG&IF56r~O-k#ziHzg1e;VEvgDuO|(oYIf+mUfwSf)B{iOe-HMCv1-|dk$FS2S(w!d751tpU{RtPJ6Wh z)udB4H9H%{HR)+FF}g!D?xazp?A2y=;J+vvbAXISbbyh;?|dlnygv-V4aNA${6>=7 zBw=|6@Yg;hoZ;h6u-cP8CgykV+as~aJkcNYe;$QSE#+qrd;y8JekXL>)d|&)X6Xls z0JP;Xs5T@KpiWbpi!4q7Uxr6oe*h|8-ugZDyhQGB`w=7!=-Ii0Z-xA0fNHr)W-y$# zwYug}`~UbGkl`>pC)8~G2E_^oj&t!?W@WUN=9^FpJU-~MAH)Tw)Kgv~)$3^cyAgpsqe9Hjkzd?Dw6aXhD31cfuLWmySer+@`WYNz-7zy(-aTrj1apQyHV70QX2 zy;Bu({lJ@6^@oZ5Cj1AVB1tR$seg(mh7*fm68>8v{cb{ahXo#x=EDYbs%}9*B z>kz{jZ#Pllyh|yI=-vwd-Wk}s`tFhMg66+U5&swWc-v`0%2OS$!G2y6Bd7?{h^lyj z$XAuB{@yEeqSQ=QhyvZbm*6u^k`SvJO1D zaMi@0uT`&L{<2iKbH*k7vbKyTSKEHc5^`H`C!4zXUO2j%ssi1F_AHI7R`*lb;z$3m zL%n03MAUi2wjLI*Tb=x)o(%D_K{Xr3}{GTX>akn{QtAiCJ+>{Qrd z!?vc2m`$x_1xlbDe`{Y^Tygkmyu8q|LsbAq)O{KxG~#k)&XZbIbx&Z(2fU=IEaf1O zfnk=(5rx^{a`;GybShzW)TRMSbFGt(mH?*&a@XZLrWuk$)^HtkD1MD6mUDHHOS%x* z%=sYF24-Or)pzhpy5Kvc2_r7 z(k_y|At(2q1W(PfLzY;xCco;ObPwa2&vrca;%OPFBXKd#h4|}pPUlHr1aqDegVcd{ zNj`p0eWo|i9pvPmFX{B(K>Ve8I{VtBa61@^Q6%r~fUQT27Dr4qr-Tzwoc znk(QT&%55eBg>D)s$rR&`e|)b^rl6pX+>JGRmnm85FD`rs|t0My$`*8+JwHOIAVAiYOYgfZ zm3csW+$-<*cChigCqll;^X_Rl{GCn13p^m8n>)Z#dI za2bN8y(}4y75UVA7w{;3Bi0f3)%$Pr_3GP6a@S?2h>u=3YZNe`G(-VVV5Cpn1VNrM zj7zMxF02IGEs^!8vW#^L$eN_=czR)b371W{AMdLr?ur$w+yHMpA~>;}uyU(Sn4H0l(L zDFR*PS(bijXD=9g?l9Z=@g4GPb15*f73&LuO@-2~rl=Q*=;pG;DHkH`i0*?Kd$@nNP5-wNp#gStDd%61fYNi3P>sH=^q&PB;Yl0aMB|^wP5T5NZbo$ybG6 z=3T-=N9{idij_DtsUMDT&i(ogI0m9d&A&i&mahd2&G>Su*dJRuE4j^x=WNNxK%{BT z%s9Y7dS(~rh|CSN2;;*UTin;qxTcFR6)g7Y5UV%k0|rhwvck5DM$vRGOv6BaPV-iF zy~n`$<5=H8dTAY?$Ct|v0j%?)Tm4AMo7zX$| z$i7_-GWKr5-w#nQ^UkOuWa-kpV*WS0);h2G(nIN9$#Es2u^4{R)Z|tiKTih^GxlOd z=@!x_9b7~^WCkM;MsrS&dpE&2y96`PMt)V&gPo-yb)Nh5qaIgrqjSU8wGTQhmvlvC zGIq{&gyHt>b`17fllWdj+pv~J4V;R7O*F>Nj2D#^uSLom))enCMpfD1*v#uaaaoZQ zja5R+I0GZaELDtHdiPqzgoGh+jI!*&56P)}`kGoBzZ8t=ZF*>`c0DBwfrsJs`S)E z&?f$_QX2Aaz>~b3A$ye0m&9;JDQ)E#xd3P1K?wx{$Ey^ghv(7Etm+6C$08lSm}H%# z^A?K{RUI62KJv966`wwtW8%_97{9;IPkG(PDYhS*dOzBqeo4%%8)j)L4zsDUrr``n zO7YTli{dst|C-xp;~&%SfJJ-g_~Pc{Im~r8CSZ?e!_?xEL%T2`lIMqJf8aYhWp`u! zL5rd&yr#)`h;Tt3L4T3q7q!B$G!t3M5S$2h(Oln3CZ>h|RBt@iyL4^FAvTJ!r;)r1 zOTNz!)7?#-V?O`*?x1GM@Fky-*=ScoifKHYx&QPfD-y+YD%DX@$&i&Q$;}?@KiHAC z$x_|KX^!2|t$dMK<0MaAdm~B&WyBe%Vb+hION^-LQbO@1C$ee|of*O`Psx{Og5)_ostXXUf)1;fx_?PsAaHz@##! z-8woJG{I@U*7Z^kFMPB;dV!SthK6;Q;%d#oQKP1y67SCAV>rYFC`5 za#lKq;1v&-pH%fOOU}e6yb+|9yTh`3Jhob!NngHZNMtJpP4wKTzjj(&vRvL|RQ8dj zP;=Pzwu%1^l94vGAH5yP7JrLIRG{q2s3jeKY;y#?qJYwlbJ#N}BRt>bego3{#C7}j z@Lw(Rj#QneAhv;YuHW3bT1{sU_TIbD)w~BK;$^E}ZXXFPJJ&r2w!3t7NyKzJA~S%= z+Q`|UEXRK7%vH-ZBShJ|IL&RP#`i6Zt&D-LCb$CXk?=jR{7$%|OZcAts-ksBq2#%x z9Mr4SKt}Gia;N52D(J4fUQ42V~S=E7DmapA67O=4YT)2~|(UadwGzE4ja^A_s% zJ@+mp^0inRm?CfH6O*gIjD{+`S`b(gR1)y@v6t_f{7<3CA@ds~mb(t2G#N_fvqnQS zQ3Su~i;@)Y6q$~%DV;NIYK*C70`;%I=44lz6*sxEn}Bf7q*7Cmkc)-uC+EUYlewF_ z4g*iyr>cq4n+5LnTK4huR~mS3xp5ELbMAv$Rs6Pqi408H8umVf6wYt-uo$;J(LTCH zY^D7ND&$ZyL74CTq$VtGnvi*R_^QL`W>JK)lJ~l-Ey3%q`G@%Um2`E`&99%jx^=Cr zHBz&6ksG;3hxoa&R2j#Fag6 zYY|dGct9x=gA^iNBfe&8_J-CNhNPjhp2ris5PbSCQ2Hnh{qL_w!!t~V6RBeu;BTZ; zD|IdzdDZ(j)?t};b@bV3{R7_r-@%QY;0)Iiq`%?Fpj_9Z5d1*>^550WO1FV{7Z}Hn z1ATvo=}3=6sd`%AwY`5M2%Ta)E$7fzLo>%cD0#Wa{C|jaFY*l+j^3Za%_rbNPw(*@ zbJ2#zu;h-rzsw)Ha!YW-k+%MHK!+s--bmI2Jao(cctt{Oa^$2a8G!ni?N8D@6uhv1 z_x2S)-|#nJa|Vg+kD~%6E#*^B-@S(=;w@FZCrvDp5?gzm%tiXc^P{8SaKQ;VMi_L^ zxh{%#4FGk5-{en0~J-!)++4{I>k$@xpl!;rJ~w*g`U zhQ82<)wV@=oTfSanPql|2Yo}9nPhX+aG^?tlK4mlUDOvsOu6!qPG%4qNXV-$(` zJx~In-zBcNFefv!Zd~}3#z^1(1GaPZcN7bhte#O_BiVxV$CiR9o2n+*thW1qKbhNv zq5HT0d?Vf{y>!7jbbJ!^2uTbPq#C?W7^<(^ObWqjLL9*J*pFS=vvHtEh_3b~oCCF; zmU+?@pPrlliRT)B14z$%HSynXJmuvFQ{ow;PM0ZOMGC%JIdJtqy>a5g$oOZTLCdtT z=+Vmj@q8y&K#~L11M&C(#g;2cWm8yO8)7nq>xHyuXlXAd9MqfBvacw{EL8y17kX+w5Ubp&*g$KkB(`dE28=aN*Y~@1?K?N_Yscq#sCl@Ms00i>%;Xu&IuBo*pceh z3LNml<%+Zy6(3IspdZw_8SjsfEBxJo01}4V$KL>jJn#wDpB;$-`+k0Y_HaT?5vS3p zjcOEJD(aiE@(wSf1lgnfGFEsMAi%7I5qNA4TR7P~@0Ft=a6RI=x;MRidx5e@*z{JB zr;%HiL1k1sp9ue{>f7dNsfx|KZu(iZP?bocG_{eylJva}bR_8EQQg$em$k?%~c#~k^DS;AVGZc@hCSo2K*FB4<0whpug zc%);EtaK)kjzQ>WuLG7A9_{Vqi0cm4YE&xU6mHrX@4|?A^qb7xr*KHTKFxJg8I+>?YGAjN_?F}U-k2|-yX1x3)Oh;jG3-*$n6JwFY63;( z=g9O-d|PK3qZ+H#k+u>j)u&xk=h*+6K1UPi_)>VAe!F#Wsnu`aG1m8WY;(el!oi_; zf&OEbFTrx9asvA+f9=bx45W+^Jpa)luO3k-P%#AKs_5R$R?vd5PhcEyp? z(q1ftauX0E|LcePJ;N7DNt4+eZJPF|Wg7uYKLlo$8kd#}7Shitt881{KaWn^*RNP< zQJ1$fXlG|_x*}&a^Zl1ThdFqGzw23+4Pm9zt0H4;@N+i5IE%|I8W5;-61t5`n8@ur zKT9{Nu)*+~Quc~gA%6i8{>)o6yNJU>4(ZrWm@CrT1L@l_yzuNz+=V7D$1j#m4a6DV z{!$njKBBhAVORJKh>YZ=4)`*0{%cwck9=XHX>S@fl^e(+U;xaGI_9ir4C5NsjWlvr zD;Za^wWhV4$+6L}`F?dKz0R3h3=Z_pW$eF{eofxWwiDryMHIy;cKp~-iOTs#%E7`a zu#2C?UFvz8!l(I=;&5v!GoqRN<+B_wQu~|Svgq>iyTrP6C*e;c8cb~?iRV~a&K!V} zO9NE~mN-DDSIhJ;h)`?eKbO>URaWnAg)9Cux6`>YRZdl18Xu%!x$?FtDNx#a8X(E> zJszQtf@EJn$TPjiR-q;7Ez`y&nr9wO#&}x6HZ8X9X<(5UN`vRKzleUhEA)fR=PkCK zDzy+GjuF$+Z6(z3>iYK zl&6$)r0WSv)S2EtnBJEB1wD;@8f8U@NLx-Jc6h9eLqYub zVdzutAF4;_1?1+|=|S^%2iW0kdiPZ#7OBxK&SEa9PboBfbi|#}C_A6`Ue(jN3dv|@ z8BLpNB8gHaR^sxT3?LU+Bqj>d6Wy_@2A!{BcScsTWyqlX(1cSyDC7ic>@hsRjZ!~N zimbk3-*H`Nd@U zttDSqLzWR}M!kl?g;=(%oM49oLi;{~YPqZljurHR-U2y%lq75%;%WMBb24BcdsJ+b z*LH%$F^_Vn(YLjhN^DeHOQJM$mnZokm_#(_YWgRT5h^8bk9|tR22I4*#w{*eVn|)v znNCg0rIEQK8>GwiNSGHrp;qeHQEwD*tg#tGWRfjs!d)6cRqb-4vwPt#)inj;yV6*t zOr%G4K`)eXnpp|i;&5m=>m=xEn=DSI2P# z>p2ChW?6jGM&kZ?BXH}aSCVwG)DYw)w@xEqJSsUuTRh(Vd~U+FuBoU<7ZdSwnqLR~)pW_44}tH>(8fAbju4?tSzOVCQOG z6Dtg_i9UFwih{aUh0N{UWo+1w-iv^o6sY^7F#~=4QY} z>zt3s0}j~Lq8Z2w%Pz_?S;K@VzIut^4)zL%M~&?fjqMu6)9_H z%QG)K>fQjM!;z^m0Hw}(p6}dsoE_OKD-6cjHqpwmvtMn=H|w-*q9InImqU^H*S1%} zUM;N$j|U6)(jMZN3*~7KU!`}03`Vt7K18`yOuP3OeP<(}vUtM@46%`Si%dM+9Q8)9 zALjD6v^G9>X}U}d9Oo78qli+OUV!{W-5PAEB19AFt3G>`q@{We@QsSg{PZF_PeVVX zpDy3yoQ;q5^kWIAgw5vmgr|jBN_X@n-VGRs^UC>^s%NO=9uub(~nr-L;*W0Gs1Jj6##Kswc;*!D{ z>!!K+`zVid3vymja3_Tn@Y`<@d9Mr^QSV0-qK*>D+%P$l>tJy_BH9^6%^LxjIEwZe zM3~Bs3=GzULxMHGM!|Y!pSC^IY?kJ#8qY5e7h2Y@vei2zT*Ui3i z-3?~eMa@s`Fr`QLNff^i-Wx7OrrDA;7;?khyx}zYcjogo&WQ6N3~p<#-sG=!z-ho< zY^&1NT&eBDlJ=?EMXSJ~fY2kV;nS0wn$jy6$uKGO!03&Ner@uFD6U9ETn1NrOFTF4 z^|Ds$tyX%&zC!2Q3qY~qmxc4z{T$1|PiZ=_MFbQ$R{|t}<&CEx%gbxbKL+>22|fEg zHOTxaTDglS;jNC(QDy#5HSkX%@Q(-m|L+CcWB)gzl$ia{qx%GRe~0v^s%U=q82E=J z^*`4lf4TmnKKwKCRtSKIm;MBMB!@#Iuz>LH-+)Tg)Z$FbFO-wB9NfWEw@;lpsE&Ue z`IwOiM&w%BR#Mt}m==B*u0mCz?~n%HFl4*nO)OkqZaSYJk+fbv`kGC40{~{`h!7{Q z_tyKeXMfZQp!nvv3cExl7(Y4qM zz&=3q9H#jO>HjToh9D~)1bw|r`fHaMBBjKfkXO1t`ZtmXl;tCep@FSul;{6O5nh9L zc*jwI_dz`o^fOPB_yqaKgF~0sB=_en49I+yh(~)6+yvk-?>k=I8Lz>PuQmaM=tH}~ z5m)da%ncYk>6{FpZ*1&mc-1PV?| za(>4J9l%-qIiyVbpN@D7QU^wt%6v)TGivQ`KH7BxQHhq(hb z?tQP%&=&#ucrpI*aL((e4A}7?K-gl)+xXk5WrJxFlZ@uF9ZLC-DG0Qz<^MsqnU2=* zXRh0!4y($!E#a$Jw+{S>cAzZh+8Vh?M^{&iJ-UiX>X}D7tk{;YTQ-O^FT1B1UjC`- zQlS)zS**;Z=_j5>TTwFOsQ1C~vh3aJ6W12Dw>YjitKla$XAdi8kT#^0(zk%sVR(tA zt`$4rVX*Z;H_A?%N<1AeSTN1AXlm$M&$hH?qT%AwNxfo~;P&%X)`TfxZpdiyD-jYf zYxmW{c+&^fW&%s+Pp@wJr&TKj928qqTF`Y*)7KUw3`N0SG;uq11yq|~gQ~Y99ep!z z)@Ocb%@&9HhfA!_HXHs-PLIaNd$r&F6HYPCoKgt(isf6z&DMk2 zq~O)2DmkGq`k1?!F98*)4+!#~278uPlL950A&1-phSqBmIU{4`g=S~nF(VYLeYIGqwN%Eb)fg)-3dJx{D|)v>7@ z4#oKWxKc3<_lwz1LL-8o{Yh%dm*_kF8RVHfROa)=d^1%dVb0{#Qz~bn*(5`^l|cNx zt{aCPiKXOSWq)o6%?icN%0&g|-gIsZ(5^A^R5Rp+E+X6aOL0=ABR1R}t&hS5D+-gO zZh8Ak;9v}t#>TrE*lqmz24`LMMWt0cd!hf1^{Q&v22p4MlmU%V$>XbGmDCb$J}tQS z$tu*tf6)}k&nOpB^vmaCmwZ)1d3wM55J8$K>ovjFGHe>Ic$<71_^ z7)ie|OinWs6#(jJdRumA&UK@`FMO@nB-nJV?pG;o%RJL)ZNX&vf(&d(H1d3hNq0<# zST$wTgV<7{f1MG5)&I6E8C_6l+&m zejowYvYX*Q<6-Pj1HXE?*eqqoL{ppGmB`{os<<~ndnVo!-QSUy)A6fyjr@pGcMH>p z455jySIIn9kqlIbQjZgGsh%h^rblv|HkG1Jr!x}z7}4}NyChKCQi8)G0nQs7q&9GR z*BA>gJ5|DQyUf&lf7{hwU2nrI4=3fX6PCh?qpnO@AR{#AU`%UPR(7c+HWG^3!ajzi zhjQ9vT&`3TGIN{bH-4WXa&9*sE8>`sV7{R={XI_ei;C~G*6*+qS$`2D&W4tlVLicJLb{wdSH`soyDq(CDYq3x zzGSC|^dk7RUi;A4(wnhT@^#|YWY~YUueGZ=mq1oR=`(EFLWr)ZE`v1vSlW)5;ERIY z)k-G5)#sOTws$qDf%daKS?h5hIwh^S_=+m^aw10AusJ+QO;~qlFDlql;c`&Nu#RdN zMrsf|)8wjb39ELUNHAmn%>5jWGoq8p;-vhxD%1)>vr9~8WUpM)6gS%hfBTIzVVC*Z z_3Eg@v^G2GHy|p>H#!Jj*39>9C9XIswXC$ZVXen&8f@a6OJscV={!Ci9#7I3jDx1y z+=ijYt|8vMLMMm{!4$=jf3zn~^ZrI~bwI=Ty7F|1Yw@IWh#=IOo({1j7pO{e)fc;Kj(?Y?y>zeHUBkZlC+G_i)-9QU%p+NCcoB|~{6xS9f6btU| z1efBp#l5&wB!u9_ofdcZAjOM&aC*|`dC&g7z4tid{F%W`)?y`Nv3~cQ^P2Uh^So^h z6zm8mvBqhuZK}h;+!`G$)%W#}ApI+L2aZZl&(`fN!5Xm|&9yiN?(xNOOlM+dzgSK* z%Vwa=KWS{4>4PF4$Cv#e*LDI_BJ4lDKD;hvhDa6RG*TFi9yq4GOuKuA*bJ-o=H{Z3WWnYy3_0#m(GO93cYKG3e)Cl+}}J z@k>PZL&;+XAg1+{Dv)P@$U6`aMfxN?BPrE^S6hBiy|uOtKApC&rwn;Z4SzTaod3`; zmy^B2+N~8yMKl*xRhdxGk_g;%A z_<+82m|&)T6H<+Py+6YjV{D6PfR;(FLMqj3ski31<+UFN@-o@*V_$t@{56q~L$LKI z=N}|2o6d_gd-!Z>?5jIbUODGI{h~PB#608`m=bhZrQ+G;YCnHNp+F`X7T=^&BB1Bq zY!X;+E;jJ+bf_;eo^73y$$WVQIi};?IjW$zh!<93fyG7yx=-|Qea;aj6(5sWQLQ!E zfNHIC#nKZSp5LsSNp;v3E2Mi30;iiK_2#R&oUw_(%&UKKbofg<2-|pt608@i31Jt! zp(tS@fR)zd&E+5JTeq^@O)=chwyjfZ-tx-Mi*=taC+TtcDNV%G=#_IQp}@r^4=htg z?*8es#Zx>6nAmIiv>JS*$?)%H>J(AqQb4?F$_4Bf6RS+9-FXvU#w=eri-cxvL(8&d zg@RyjC^>s{X9l0hD#Cx!e%ts=t%8yvuK_Ve9i6V37aR)Y3Acf%#-BL9R|rQhq0%RK z6wVCR#=>bKb^95|hFtya%9?wlN+pVWCRJ^7TyO)DW>mQ<@~@Jv~{q{dt7-PqGdpIk(Od$*je z^CO^@jSGTd`S@rwh7X^w0EvN0+na${=ZwgeNaVvDRR5S=%$ONft8EhNZB)Ifw;db^ z9OGDCDLFg8bEcokg3NpbSW`A$>-+8ptWOf4z2c|r_3zE8fCY>K05P}MpDJ%4JqV9S z0K&qJQ^A!Hmx!@~T*cp~=+-ZnvjYh^L5{@+Rr&+h5e{d(NT9#c~JKk7k0J zQUo1Elkp`0Z{9U5KD7Fn3Qn)1WTE%6NxdRRr|X|kSmR)QpO%)kKr|}P+ZfMtv22fv z?sr5x*EiQUS>=H1?A}2Du?=b69{^X~{VnACV{EU;gFwtLh{?wDMiOdHk;c;>fLO`; zZ?!SEMv;W3A0HhAKAXIMhCcfkt+u=I25qzL-*NB-_(>Y$9ondP?n)i)KNlZ3;W>A3 zdUtefLzLqnZ5y2pgkI$e&XMAV&XIw{sF(i;K4D{Q($t{u5CK z;BWb_OyS3}{FPP#`F$u=O8g|)^+&^XU^M1({hhepLw(j?%V(hKKf2hn31t#-RaEA3 zNc#FN^x`4;2YP3p*U8H0=)Md9IN3|??cn$HoCNjxETWga9y~^0p+lj!DgJ3?577UY zS@rnQ2r*$O;Tf7z(6#@Yrz|%sck}=-o+IsE-Rr+#s>eGdwJi@Rr)>;cP(3YjVcM?8 zj{%Nt%N{7d?6b;6ZO0J+4a4htR2Wnv5c5AV&KiOJnQPbH6#yVkeB+`1P(T3i0;d8o7T|#TqaM$B z8KXaKzq*Ts6#^Jn;;8NZ0DJ+`Ah;zqwaeIJbaM||g-h~5dc!li>2XKi{VI26genjbw*Au_3ld_p&MZk2ER30Tj z+dqK11>o15%z>GYPcQ4t3U32MFV{)W^?2$7EbYR#OfT!DGv#8W>AJS*D)S3A&sLkv zTbU9iMQ)$)>zg9YcTctNzPB2gKiQI8&=VWj#ZwpRoCb0L&(h93jEmlZr%O5un?;du zu5lMR2R5#TuVWc=w23g+hT$4WhJxnGKqhjjiBHGrAdTZ)VPyXTW{@PXVcelf>jcH3 z@98aLiFy6Q(S~AZ-Sj(GFHXjX6e7^iN;FTv+;JSlWD($8+RCFQ6ZO^VsRDM4$1XAV z_th+FY1GAuY%Z+OQmrytEyE|dr0)t+$phvYOm|Ne+pyc97J}(F7W5q z%ne#QS4l>Pvol)7_+q@8$6G0UxMs0K4H+NTR3-+C+CRYMd=DYGAZT; zRJH-@tjHV|{jMl8pOicH8z;v{dTc)s&mh>tiAs0#4+<`fcs5Pc3oS~g)2!WgJK(!z zvU!?TYI)OI^2u)|pZA#bnGh;++Y<6+H=^0Za?xI`vY>q|K?;-XnARt=S74c(E-2Kv zxxS6EkB_1xG5@~N%IfpWB{9KorsL_I@XldoUUu*w{&D5yHL}Mcr=2uMi z$Nm8H{1R_Y#sxWARf=W4ZE3lzS=9YjPjNM1EV$s8NRI61xjVvdJk7RICh9M)TaRR> zyJD8knXg$Qm8ZCwEEf+3?(a7D8uh)B*HPWSWUbPGxE`R627@qDpXI{`cA6$)npF3^ zX*bQJXtz?D-fvh_(b>nd5;T+V^3h7Y;CrGm8B;k@tV?xUasCNOme||t_zdJ)RM)xm zT$r`$Rrjq-a$pU8pYe-=d2ZfXQ!#uiqF@>$)Aty3v!r~Cn|V4j<^K9h!LvVf={ga= zP_@$$1`7Vn%H4Kwn4VuKyPEmZ_x1dD%!`#}q@ z=NtFVcL>=~Di(tL3K-cyS;Vpmwvsl~7cY`o598J6B^`+8g+6zN)3UtmYV!%*QY005 zC{=Qqp5uE%So>6T@iq4^AE%dV%}FY@(|HT*iPObc)X%P7`du8T;BaD<)sr_+#T-JAKIUoQq<^K4xb& z@*u3rK4GQZ%KmPhHliTY)uuNa5y%@u%|M=uewrQ8gUYtG+vrQWb%)M2E=NeUMRkJk5&MCeX}jrEjL94k&gJp+)GR23YCj$B?goibQvfo#3OW$saMRN3H#&%*RK2c0`GfS ze-asrIkT~*M3RNVZ*UqR-N&V$EOzQPL2}GZtQ?K>*oXWhJc?5D=E%O=U+gF-4CNod zUH*8xsc%tc{Y1o;MclZFS6w5!9t%(tsT!bGk85C5(k@lS5G*)?<;qP8Z+8A}@P)>bbkBLi!%0IQX%>0dX_~6Pu zOL|#yU1t_zVW}}ertyInYV@$ZX<(qRX@lv>(nViPo;ONVV@wH6swl8wGA4#ph4Qe>a8*jY)>Qew_DJrz&`m*_e?Prr{E*gUcex3j8=_Bg2S1D)^pZ zNzKc0&HghpSC=89$6^bW@0Ymo%q{SiaBa@3P&;p>fmKVdqP3Suh2i|0{EajY@eosiQNdO> zTh_3Fkr7Ya0FWx=B~N*Xp5vV16P9mmGed0qDtZ4)Xw11!wju_VYliD!=Di8(b|$3r zH5`#zw_+%ImPhX1Z&QA|7lBr4RZ z=r@`tb>-n^ZHSEf52oT&odd$yX;T>!IJwqg1o~nfiC9IWXW76 z{`SMIkm@F#pDf$d>)AIwR6H|lO^ZTZ1ZzaQff4%n_h$->7Sg6!IU#niVwMKT88@KCC1mjao;-dZ55=DE5tyt zF(?=|7gw$|+zK0|<)e6!l=X5kW*_JvvZCDjZ*fwk_Xv_gq{%wozOC_(2(WZ;PSD8u zYHRH-En!a26ig}qQ6OYddcO=|Sa?296R$@fjA2F;Vr#kpjjz(b)D>55jRxmHHX^_5 zviiG=*4fV5b5d8f+q}f*=GVL-&dK9h)J`gGPheKloB+=hPTj^8)0#X}{9N=iIjJe; zGn-GQ1u(?M@~+>L-<_^bG;^s%`2Ncly%ORC@l6C{Zx%*|SIf%6k_p00IIG zk;{6Y!HMN%lroS&DOt;!2jd;`)5-4M=AVo0Y!9HzB!0vXQY>Z2&z;X z?NS9?=uh0{DgTjqo*S$*KxZKN;yh9ye!<9>Z$JYwF1!sa<5X* zMWn@HBLm5iIih+5fNslvYnY~D*`C2tOJ?t!(sKhnFj0P$n0xp1EC}^=w#d828#$`2 zk0glHOnEl^oEZ`2j*6T@sqfA?_DPwimg(?o1Cq&D-QD_8 zoUCQN5v9beeK77i>2o zLX&*QdEe$l4tnBmLc3l)j^x8qC?;WkiX~Cp!lSFEL2}ix}Z6in#SEiLat_KDfxjl(V*V! z%@1vaMdCneJiAJK4QByVb77Y5B$5ZtJ2|}>J$Y2`F>Vk+ppfacjk`vzrCt@(R7=31 zORqsTrP9b+*ZFCqFR0?V;s*-`EnRe72T5xMML2Ga-cyN{D#*?o{j>IaSDfQ|hX&ai z%Ozc4hKz!dmqMD}FD_nm7**blUOJc}e#&~l*#y{c9I{ znd4C1fO1@;^cmDE!Y3DrqV`KgDIxX-$IB^w4G^tATU)=_ zjH>->c@HUNetJ3a!8V15HaO`-cgPP9>+rqS($!& zsQ11gUG1^Cd|E|Zf=YUFgowoTuLAx5Sw!GV)S4=TgLNq8_&ZmOO<0-^fi`u#igBpC%)3#R?EGcq6D44WwBVZMR^#l&KP|iIkT?56;vv&pWIk^?83>LVl zlC9@Q_hEhHo5WAF{r5J-(aa78=_!9VInOc|c^Hies_!B5%OkH)U{SC!mxviCCqK0JAWI0KK^nGinY3qLaAp9(oV?V0|dp+;&XF!;=aY>9xY zTVIDoVCq6*@4DT2jo=uXjll|bdSto+M9!i>AO+$6j6*tsMADyVF17&5f%0aZ zYy>R$r4W=I3nup&c!;)KdM@x;MlI>RfyGj)FnKzJ-C$Y`5p6K@%Ca?efk#7X->G^C zXu|MOChFn$NOQFeJwq%5$WE=~S|~9NSN=z?mz^ytw$|e%=I=v2`FcM#naS~(j^xeV zmioP6()TGNakpu>;4Q6I%8o3PW>SA<2Yz3FCJ zTRdv-Gfr1MWFs}=ah(_AoVPx*xh7v6V=eZzyg*;WKf)6ecvV^lo-AIQO)+at1T5r^ zv!(kEjt@RNnP1~DnE!5^AW0(og63o?jWUf)XHC&CE>r6H&LNw!ih!vshzw z{hR-4Oa*0Y-dz@JWe$GFlqTZQRcymk8-Gp66yL%Xt>TbVQRGMCInxvH)qO9O=NhZ( zaq`0%iT&;;cLlBtI4heGVX0azeOBo4hMX-HR!lzJu~p;+8j!z@X=yB!Va1}kOZ`7y zFTK%8Ql_Lx_Q$4#B8PsbL?JE6Q zxW8u~VD8Fn7GCO4qsCGLE)T8a!^KW(EEW{XM1i41X-aVM)&h27d@P{6#I6ZS1Abnb zuQsrt!y}4&K2I&Mj2lyzqU*hMWt5%-YjDf4@M4@~hECSNK zck3Ql(2gD-#+NEYm2K`Z!05NRWxRwP=4_ zGZ|guTr|esgdNGFAhImFuv&BLaWQH$b?Wkk=~Zqdlc1_<-jc($QTo=$U%W>G44Ey9EVf;>c+IE$R=a+4=W^t|G#A&GN=8nBpZs}&{ThcWnULXiGeH-}Uqf|Pw;eh{6 zVzG{qfc%-Q$V5RX6G1drwZ62a(vHf~E<`$oEQs5+LFrG7iQ z$Z#f_D~$&8G2%coiTGCS zFA3J}qyy(Lq(&ot%pOCSnaz<)0t8Y$Di53^)+u=? z{(b2XieEfYst#dhEB9C16H>>eZVgzgr26UKH^3Tx;&L%oPcULv^idw9nO8$6yq)zDUTvP4UGX%5oBzis?4l+9$9nXqXMuolYGx!on@YD+<9k*K6175~1 zq#Qhfrb~w+-}sm@<|O*Tb{8kXU#02CGkHNLG2@rdFdy|LGc3Fh zJmjDg#{&P2bMF#&Y9h=%}tM-G$@M|gu$x%2@jkWCwU z&R>2tuB@Zc*~QrpdGTkCM`HHWvZ}m2NOfd(_Vk@v9L47(`W@OvF6L;FI7bM=kLHul zoHq8Cs3nrb-$!TFp9c1jyHjQo1$d$d{PSO+p~%B{k2z&^a}M!As>YF@8%8y_62`aB z&m)>Tt=WYQO5q&@CBf=+%`Woo-NmU3>`FhJ_!#f=OeC9noGLF6Yho$=p!fNXw##GI z(9s*rNN?_xRSQUaMA%)@t(~fRH$iD-*5?c@3e3%#+yJ%0J95IEX#EF^ln3}_>dbwk zAYA7yC;oc zG4WD2THCLC$k~cN3aYAzs@y(43y)Dy;6ahnAa|KzwOt-w=JGDOWTC&RZbR3z=#v;( zY^&sLNQCrhP<|-q`_PcWF$w`b5Qq5R6i(BAagT}W->I-ap3 z=R^7)p_x!u@*b|OeRq5k9f-olq+S7!K1igKYtGxQv(t+W$OJx#{%D=-Co;=5P2o$+ z)vD^iTxw@I<8U@nE@HL5$sv1R!R_j&xx**1xA`n_xNI}EZ1e7%t}Z7gmf1ATHV<7P zP&RKX#ZZB*pv)4}hI=ECSJ}%tKVI3$YC>84b-z#O+C-Lx+4LQxelrw|td)YXREdQQ zrilArOS{`@T^Ze#Os^z)Q@N@opW!kXvRb}S{?UlP0w(^x-r>Kgr}Lgkx~>W=-!28$ zP&E`mFs+8Ho;D@ePz&{0+%>D}l{|0g0?v{!J`O6>{H1+GAVR?R`j>;VwlI|RWj~j7 zQD=QXBA;i(&;pw%#wexi=*zXY5Zuh$@hLh%+T=wNeXJ;pzK~xOPq6#(WHq-RU|k8{ z=51W%svL-tw8vmIck2Q8zCFeMcuqbI`#GE>xVFC~-9594HalO1tNIGme}hjC<_$SZ zk&;aA^X=|z+A=eP(&9Q2RSW4gQYZzq1}GyPGOG1E?YP?dU8{bky9ekh6!sxTGcL~1 z!fbWKi+SjQ;f=O%F%~wUQdz?;ymC@bNM2DX6E0`2ZGh>LxF&r_c=tua_nV&9PE!ES zo*=TLBM5h2rt3&S!%8)BN*tApyDm5N)aJr4oruUtQ1A;)@H4?kO3Zp722NL2$y_!O zxBgs&gc?xIM?73+n3&At;h0uiU3wPWZmDIup}74m?uZzHB&5}iW+F^|cc&lwjqd02 zPc9CixL{zk)S-ieysA)mw^X%v3HWRUF(31Y6bX4X4)r6Y;8if>{}}l>L(zc?E!`(D z5+U(QTmG71$rz94wdQhNmxfF$A6uz@>uTQe-tiN#ZVmEu42ozQQkjVP@ z${uPL_nYqueIi;;X=}9(;j$4x>caJLJz+;8jmYm>5?#vc)vQ`(&{=n z(FN+kL21~pBqLrCH4+wQA-EwHnd{YolR(*^3G4_m+trbhZLU4XYR311x}Q|`#LNTF z&KP732x(LiCiBEtk=2G$W{eb;X?}kIRMlLH6Tiv|TcqCKuXEOI3b`u5GB@3sCejPY z%+Ep>E4}w*M>p*sSSs6MkO~P7*W?L_u$l<3l(umC{wBIzHwvFchj* zAgVI$J=B&eJv~V35oBY!x&VobinsY(&TgYRam&=gWfSJ{4@dNK&0E^#F0C0eL&?I+ zp4a8zfXr#ihYI-OtM7ON6KxqFaG#tr^;yhoETswvk%kWSWQVKp_u$NG z#?2kY+v+v)QWVrZ5y~L15H=CFOYH34fc*|=raM4xqr?QoBPp1!6K3D)UoTSFpIt#d zTip`rLXILScY2LZYc7lIC4!AS_}kQqxgNWRKI#L{d9l|>f-DrTB9uctR?LAA@&chu zJI&|Q$2y*rDXz5MYcQBcaJIz?>RI;4nDe6Ny zES%Bfi~>yVvnJvqITRu1QLA*bVh#S}>;NsL*5S-az5o-`6&)=e;>XU{$5ME>;akx4Hv zu-VY6r5hyyq`m*U`!Hu zO&TWxTbVTyiPNY}g4%teXV&p3WD(>*6_YaDdYQa#XKZG?ybLVvLpG?4 z#Ck7)KIRZR>4hU?k0FCEplV3ZF_y{<+r7>@FJb|yT%XB{yQ#?h3;lEalM`NIdK+1w zR}$aAyVi0z2mzs=BV$JSg$`tXD)U!_7VRyge7)ak>t^1A)20Rph~BSUv9;tE#Sm$t z(SLR)S1Zc9deiH*XtCX*PBJwB0ZB~lDg89jgLS)&_)RQ#eQOD|8A;rKDjQLF=s=FX zO*D~Q+q^Gued-#RSK{pSxowf<@#Pdkb(3=ubj0M%kg<7!zt$ue}J4g zyU_WqMnbBd#vyrX+8y$}ccGPMVAt#b=ohm-ZC$TCtna9OQNe^)nPp&bAGr~5BN7{s zk|whURU0)r*Mr~X!vnDAp*G4!NmTKJJDY?Y6U&xqnZaUtogi8wW=b);L)OBdhRln_ zgYkQ|SMdmkK`NU0SH#hgu>11Yad4=GVrf}D;a5qIvZ|NNxhkWaP_a)_R7IM`ziEGa zeW8&s&HN=y)TyDs$|MwwUJyzF?|{%01fhH6PRL*yEcLE(F#vP8531;vl#B;2EVEz( z;R&rbZ2%E^C*h|#C;Cs$U$j~z>vP}`L3|Cv!JGAwBVdoyYqAj5cdHtjvHjAS4$IZWmw_(Zwo?0~ zUmHvkk$5xAu8s(TvQE6kj%DMci|gXPYE3eMzxhE$Y)z@=~qu3+R+VDQ5}+0Z;IcOOokQl*C;7$gWqr4)V-( z^SMQM>OFeDrY+LvMySfm>k=<^3VU#}{zM&_c)nAyX=vblObv})0PSWE>zL$Wrt(a4 z2C^9&sCvo_&{%^H(F4K3F+XY2y`;uL)_cVwPCY^o)%Rouwj15tr$%AkY0FNZWcHsz zTXY}#uc5s5U*56_9X5Jt7Vg+!NR?b_m?(WPE+EgK`<5Q(8aL-2Epv$dYnso#_A2TR zpoX84D&71cwh?LoX$HBAC||0%({Vf^2gisNUgh!d_%gJ&bZbBRU*Zn7)BhVv`@hB- zF{AK@W|T#(fdJ)~Gnz~YEU*r~6EQ=_4=nzhS^IBF)c<+K%+nkYhzSh*|6K6DWvqXG z84JR`@{1bf5j?xbezbI^tqdnL{pHRXL9IhY|Wfb}ylLX2Q#nQqw! z#dfTxwtpP-&~D@l;KT>pL1;mxPmkJ`S{g+HIZ$TKf2fQ4*CQ`e_Ja?Y19=5p%Jj1$ zV#21}<&?A6uOdiS93U>4_Z0DChEZX;=+)*IdR?wU8g;P;j`MljXNjJ#G$)P@GJ7+; zLYUvi?yuzmzpn`~Tu4G`G4wH;;0 z%v|xj86=F05c^_{?3abtDNH1ZGhGA?kUReJd~qz+ke58`Z_5bx&{|?5UWUjMD&NcK^ig|@ zpytvgu8bSUBr{om*2eLrB-MCw7Yw#U+UuBH*;UUUt`FI7X@2WMm9#=(V1zZ4M5B1Y ziWt3$VZqW>j63;za=tcn1*wDVc5`Ze+H;Q(CEig_x8RT@u2&N(tf6)vRA2IO37(1xZjIB6%-mlLOufK(tt@u zbNR&5pRed`{ammmy}!A9ie}yIw>YGJDGPG%q%cEN_L3mK?q8hI*;tmkN-c-_@2Ts} z9)7(xS~Gym&xCO0Yvd9088}N@DbApFkn$W&?07m7bA12~xpw4P2Q>`Uyz~(V2B%fd z6w|CzzP)vauLUiJI=Xlm?6TBQlsOd~-cK)eZ$H}i#DMuQbgAUg5k)xVFOCy^aeOQg z0UpARg!k^rMZJ|VZ-ydOM${>K7PsPZygo6=zQ+|o9P5)@feAh_=S`^NvBMJ+R+Q!% z%9F_0C|gd}I9`;*dZwg|83G-?#pht87~1dRwwJT^OrsvkgU(7!O=COQR%Lwoz-4T3 z%`Wv~*i)~>dkj+rrRHMWC`~KPjmk*ukO+uauu1eZpqtSON}P-fYIJonm??Sj{)say zLL8~DE~;SJe-B4GZG?JHbBALRR_<^ObJ{fNe8FUi<^+Fpr(Wag*c^|GHef~UR-!Ej+UPZ@S2dCci_fKo9>U!vt zrQwTz?1`BuiXDVBB-*M*105I!N|SAx*j^p>|Ky#SUE#lOx#@?S4$1w-z_6&`DP2305RYD^jhwD%YuRQAlzoUTfd&d>RrTM(N}`EjzA5| z+}m26CJW`7bj{%7>L@-VNWO*Hertq=z0Z|X8{ZOMlVjauc`h?FA+_Sz^>4M9*k^B2 zl24QH2lU1_3nhpKapiqhzE#r#)tGF!)`a<9Msw7}7v=LfWs1eIcg{xbkyaDIjokL; z^Y()pQ-l+L?RD z=$8L9#k64g97Y^!CWzUnQ2bGmdn#;52~2z>#tEsLw-)?RoU4Il`&cUt)uPpNUp^^r z@WOcRh~=<;bjrFSdvt}{0H$MsDnnP6y)Jd%?>Si;(HpJ4Q5NPre^PN?!E!*`G#_?5C|RSE9-gFWf# zdCNphx@9H;w6o~{^_@k27Hj6ttfWcAo^BMUX+lSRxnx7mgNs`=qIQIgQ zfc`AkTChBKZ|_%zO<5W+eTh^(rEjQsAl0cthTS6xBe#a{u&Ow>Xg)M%#Y@l*7&6a< z^fYL2;p&@ebf5$q*u=dDC$5oyTxpa$X4R-9ohmu_Ubow)QD=7kFXoSOr#9>_;>>MuhM0%~nvNb1g7OH>!Y7Uo;yjiKNa-S^AXK6mwN<$+##+_>-HNXsCgL&m}7b zoY-*T$d#-VUK^rz$TQV^IFybYL)uJn+ys3ZsWMTpXv zj_^M3in9N>OYB0v=U}d;T*plq zp7V6hrza(tNm;bM7sCM-Z%VX-b+0`nYl4JnEo_g~OFm_Ky7RHl)5bOoIZpSOC4;9D zHkq!@A0DPMP$jW@%6h);+HC2|AER}nHkd7q3Z~NJz};M-wgk2WDmg*5RJm!Vn{+Rn z3O+w_w~lxPSp#-{Tiil0yB>-jzw830o5c#8^$0vDD|-vm&Oo>Hzb4TDH&G$9!Xm74H5qkMdEtsyu;hSzZPGU_1QkLRl5~FHRe-8`drph|0VKbX@|P@;lri8A zmLHBYZ#}%(5SbJ-#Ev1Z`oyNdM^KuWRE5}HH?9Zk)M#Wh92@|Taw0|FAsu#M4=|M~ zd#e1cM$Qsq0>s{~G&&T$nK>@y#iqTAh348+HL(UhoVEY!5wWd@4peZJA^35_uC~Fe z9jggF){jT^S!R}fnW6rJ&x>=5NEs``ftdDjT{a`-7DL-K;@7D$Z_4E&2t(L3BX+8+1<>PxGS%43?F*F?sP)PT+Wcc>n{y z#KLgIym)y`IN(6dQuR2Ij_XzBl?)qua zPaY=aJ`qM1$o`2@;T@6sR~%b|=(Ni~kP6i3X*c~i8-BGfNR2$(NRV)v zWICYUb5FHRxYBJ^08dTzCzb5QHHBUbU2IVOWoiW)&EIhig9k2sGU$Aqj22k*&p>tx zV$+wqyRPJ<`Ufrl@@bhC$sYiRc#$zMc{|~A2 zUzX&5R5!p>d%C8JDuz=gH_$@vT$j~4k->O^evpI3yj zkN@c_c!}ly7vK3D1N4`QiHXkgZ@tOOr?Z+O*nOP;Fb^@&=^R_H#U;-+s8E*7zh~(2 z(wIJpJ`$H}+3?dp7a9GT2(B`DEF5r#Adv0SyN!~je&*^G75wYr6zPJD$qx#KFUG35 z(Jho43|@@MDQzsNPGo#?{$Brb&bljo$tcds%JRjk7p`n_8Np79;YiNF1XM2e+Eyuo zJDKvefEoT)eKqzQQ6I{SjMwCi?BR@l^2xG2H9I!-i@K_b6)4p8V3%g-`o^Y3{#Pf{_(#~wuEl_D>SC& z--DrjD%dd-p9kW`p5o-cci>Tnb1!}e7Z2l$DeVnf&r{4n^SSj#RZ&MJx@g+B4<#gb zkGG4@?XhUf*;KzRfrEoVZZ2!K7-;h9K|OSmwIs5kE!#S*eBd>FA+>NOo&}(WH^fJ| z>H0?IB-0%-4wq@VO=!Ey9(~TiO^Rb1I>S1lo59_7!ARz_I8>w*7H7()d7K~76W2wv zMwEEPp$_Dn9mp>L3c=s1`WN8t9f`m99nYAw(6GN4Zo0_cx8y?=0c*S}C=FKoU2k~W zRi1xdm$xX>x|4dg>)u2Lt3=)DqkhVktL2Tr7)w(0Gk^;?rWxuPLPYYlAKJwz1DLeV zFm*D5z1=^0df$SgFY$81-R@YXJ|1V|Q-T5qKw@#Ze*j}ct?oYl{iv;27Er)Q;`4&b z+sTr0?(qQcYm?%8S0ux*0h@!S|IS|juCiPT_2u`W#2HFyE2j6CA!|)N0fWbyw^MRX z+j5~;VootBE`EBKwa-GCc|@kwQ{t_gvz{mkO@tSPar&B^DOK-6-IU1rUv6Cc?H6YE zB@gAi=-biHKn=DAB71)$CyoOC#J}5Fnlz*El2^OEt;9Uv-FoVC56Ljafk<*w%mhBU!<%^U8Ie;wePJst zkwsaD&t4~(*<}t`q&3%54SH2o^TF?3*R9Mk`U#81;qPQ&<<&IW7{W?lG%|XDSa3mh zRO>GZThF91PaIl}Ym((y!&2Rwa*5xGS?Q(cq|moz&gK^K!W@)=u$pt5iEyod;vTlpaF!{`&L%@W>)Hxlp z3H+$QMsCv8#g@Tn+OAx5GS;{#IkoEGBsM$uvap`h=3Rn<`}e!>SSk<+PU%;-Mk4LV z0!nEhfN?#FMdNn|XWO7v#S6+Hk6ps$XuJ=N+^8sR=O?P?3uQ9q!3+*>sfaN0)WaRi z*vhOe-$dQqz5PWWM6TN^F4Xwl_Vjdm`U}a+8|91;P-NZ?v&l(9@T*_Qhntn-nDQpI z8LF2ZCsp$t^VdaV^F#M~-fVQS)SY^Qx>)h)Ss;l=@R*3jK3uU=LfXPiZ!x3XX66bm zTv?7ms31YER6zPfn(?&)<CR!W6?yea^sx8V+#JqaDhvHFr-MU*5jMUsHdn`CMCm z=F=U=ndW`b#UFr0(~eT*T|beG*GTK5-4&G^-)#6y)zE&#@e-n+qep=qDi6^ypTA>cRW2qPr`kIu(!`1x;-alN`y3lYJMrIf_saTBdVoo%Z z1%pCJH!zx7RTo|bXG!Zx5hQRIBU5iK2%$jXMWhvc{rd0Y0`o1X4zi_>OD z3Ni&`(5=uWRRIjTe6LiXG0FCVT7p)ni)hL~#a`N_)U;maZ7myHYMxzHP^BqJQ~}#p z-{cBn*FJYqJWt_od==+=^aeI7=O(+^?zd)_Wh~MYqsLA17hvxPyGx_=xQkKdZDtFZ zpl*NR?~5UIj}u$6Sg7#jG+QMG>Jo}CvFe30Oi$9|3ub$`L_CRFJflxor^a(%^B!GZ zI@XccQ6{~GpEnhO*Tn|Es=&aox^c_4Ct@sfXr@osI=aXK^xbvtda+t~Fnpc0Xf+c1 zlv``pl88I_An#;x4ZOCrYas983a0vN2;L}b>LceO67fA~GOCgiGE;!% znx+%+QTm$D6~L< z0>!mZ+`SZcNbm%AcP~zn7T4hJ5Q00wiWDgh#fujx?!}>SC%yMSGi%nGZ?Tn}WSyLI z-u>HqKi6SNz7#6Fh5IcxrI&pHBp{EXRgY@Iyl#P_wAz%r$|gghhJOHhyx{>6iU-b+ zYPrxpDc^fEZUCk zhSau9!l-h?(pthaZh7Ncka)5}@V!OF#_vhQ3%F>aokFX1<8k)a!=3Jz zyKw^^Lwa|Uiy})lKh|Ja;ueM49~d=?M^$9lcB-lci&2|Xt?Ev{43=3Lj;bn_X;{`B zpbee=ETGlp2RrM@y@q~j;g${H+`tgKZ%Xs@?&GsFzFN%WAa z-&;PR%#0uT?m>&x#)(a)_CtQ2o34Cugcky2nIbgWp0Q@nK6<)?z)4D3Af7qw^9OOi zFtxOYPL9bohj)-Ef%%yiB1$n+qKhC{U3jm9-n6xI)zDtJ{>=g=x9#ji=T7_dB1a~- zuBTaI-TMc*qHN|Lovq+_4h!b?ea`QRiTRsFKKPlLDctdKz$Yk~`&YE35TAku*fhpe zd=!i- zMx=kiYpCYA)VX0quKOcvOA4#oaci>Ti z#E@R_j>o+8cByJ7MXpFvriz?IB7Sx*lN8vwg1&4lN9o}ainFJD>G5wAnpPe++#EBn_<23OP{?T)7?XNfwTRJ{YsAp#bS$r*cyc>7Zj6a zTU8&5+A(K}P>hbQ#6MT}`jq(%pDt>GJJf8-Hpuj8o~75z{xEU2vN%SD4%xf=MWcy< zz&Siwk43Xt@IroyMJh=Zhxj^0zQRFUAM9_J)NOMDDjL#xD!!5q2$-jCi%X!avC1hI zes-?&b41?tK#mjFqf{kCz_EifhOgklLM=YRqB+yFy=fH5L`?CBnbIs=`M}=JsHT}| z3w!o9ilYAOQEWVB$h1-9j-?f9{8`_RH95<~*Dkh;e*3^L553yT{vvvi$>xObN#j$S z+0M$mzKhwKNl4Y#;!3cEfSy5>5muGylCHMpFMb)HQ|1x6?PqTcF}pXS8e};u=}M>4 zpVF$)v6*eGv9*g>4b*&QSjiIY4W*+O+6%F@o1c^1hsxe5i;012^E-*74-Rcc+B5aK z&g&4@1r^kB4#`jBzugvS_q=xJ^KzArmCIlfry$?`xdr68N9(=t!Fb$*vW9MaiTZ^E zu$;Xig?BU+6lV?@6O5B$PG<`-Y*uJ>Cub`?&adE#F{{Bjwjh z>v3`c0~MAIuWPTHi{L@BkY)KWDJxL!><_&pUsqVPGkAoX{@XkXvqqVg?Rab2^d{lx zz99+=Vs5LhTuIVwom`YpIZ?=Ha$3KiV7ypuk%Y>dWWlvB^_8PBVt2b!d6*xx!u(1| zJ!(ovsZ{8YXHY_w!8*;%+r3^QOs5NJ1{didnm9FY( z)xdUAyUp>d++U4++;bn;5_6?L`nVlnl=E(|0x`jS{C64!$yLEDv!QjqRt*Vp{F>w1 zH4GE2nTtI}49XkBBoUD-l=;bx*3!{r)-l3R0dLu;YE5MSX}hK~ENnZrP$13Hu^DjY zl&hd?tN$a-HTnDa2d^?UkK&4~q_pIT{^sQ_!)xr9^TxH`g*~au4p$*ja$}H;064Q2 zsE;Um>7#7$QEgx)a5Qcp(S6gbPm2MYwx^}^9SiW5nR9w^)BJpOt%ZTKFN0`VKJ`J+ zvVB+kLrh!wrD9%ApSlqnC#P(*W5d;Z0V1q0vt5=ZeFM&sG8L=VF4om@w0h;FU!cz$ zbI^Rqn}gS$2wFW_2_^U(;EX6k~hOS(3#h}VQW_=rsqsCSb$BH}oj_!wuRzkbpawWLtOoKiD zE#I_g=ukDVPE^+iaq;|T9*_NEuAG2uG3N}8Xu{FC>|c(+W$eh;jZC5J2s!Suf>n*UJ=P*b%DVr5V$|9yl1;D0EDf8~m8+@Q2%F|09tP&2iU%)J!8||6k?gKkx@B#{bn| z{`cy6HpR0aYwN22w!@+pB-NGzSbETc)kp!KIsWo2kxS_1|NanY|Fh}nl1Q6yzA})d zx-QfKPJQP<27%)|T@-kla5DQuRbL*oS^8| zg)R?4!pk;V`N6-pmbZ!_4cb1FzSLu9RB7TYnAfK>0U?}Lo%FKKN` zI~1!15_4WwP)Y{hj=$h?V0~|9&l=l@-iH>Pv;ATjw$ffqCi((B*V_p&C`1E{q8pywo6~^V42$sWSx|4w;Q~YlJ+bkPWsT&QCg=I7Imd)EuzZi zQ>QNC<1daZe%1qdR*Q+;PmWbBH!XSAyyOmZAf?DNYs)NUWR|{}Q-tVlu4e>2(^jdx zH1u+T04u~VJ15DbKuA%&pRhF64};6tpBrDvE3_s=$0)ulVinu&VLn#-d3HvhD)YtE~n zdT|Q=jeYCRl2i%eF2Pi0Ez~RxLfIq~^?@wrK8LF&;laUCJffJ`B%-$4@L+u&mlS{E zW4=-LJ|7Z&&x2a^*{d4Y%e<>3@$c?2^x(Piua(7DxNGEE&p^lu1s6!$+~I{u`(78_ z;n>~V#m35`t`89^y8Am0ZY{a;Hv4j_nW;Yl-7~FPovkT536e7#LMs^FBQ(!Tw_~GH z8^g`>S{QezOOz66ol8|O^RzVT>BWSntDL8Eqvm$V*K)lMHc+xkayp~8ld`^aPkG!l zW|?(wmXRQ%GM}cqPe`iq_orR&^wn35qQx}BNpg>jFKxDm_jPpk+S>EU--4{4JCs^u z=Vq}sfFxoESVNMnJ?YYBErm3ftae0~(@Hww{qFhrmWZ4R_Xjz(%j#z!?#KOw>h?^U z84+#MNc>vCy*8#l_yQPFXk9(pqQ)`jDBh=Gc6oicD&#E&VUUFM11D) zeaf-y9=tAGywfRL{zL!HfXaN~A3zVZ$+9e=Drbza8pho9$mVgh!YgllZE`T14s>CZ zAE`z=l&!e(XM9KDpzHsntZg>NQPgilKwA~e4nvxmn4q?c!h!NbgWWyeUFL{pH#bn#a7#5eql*_x`5EVboO13OmnokM;n*W-k<>!F2hCbo2 zs%=fEd;sVEo5A!{(u|W%{2dY{uU^EczF*P=-KWB+5Sno@;#8~rR8NU>FhxvlgNcnV zAMxUcw7cu|OCyrGpmWXah2KkDmd{N`U4Q4Pr)0eX@g|h(Fw)Cw$qilP2gceVH6lqb9W*;5v%%VQ`~Mw$O`h%O&DN8K!UUZ!ZkHS<#| zTiFldHgDjAp|1X2_QKi%@!R5?btdgRXj{3KrT5SvB%8@kY@K)K;Hb~;A3#>`0~5-H zxQ^8oPfG2v!L;P47l4f1z0GCeqaT+n?a+I?k(ot%9yqPKTh<}tMBRWFk)AAo09 zzexd#MUb$lHYFtU=_lRW@DcOvC4Kh|ty5pp&`@rSJrF#3e|CI99+S&DH-4AAMI5e@ zp{t|j(RvLX+-+JoLb*olYq{`_?{QjT>{_r57$XYjz20y zuFd=ds5e;JWwQJ_OZhnC2upd>Eb%ot$8G}T#NMfqB%{2;Xs1!q#sP+dG@SisBkW(7@@lY1G&w-y|% z1aNY&fSZb91O<%9YXzt+CJXNELy*k2_V1jpp!fb;F_o2-GgDW{3fH;W`-Ka2wr$hH z&WyZDE%z{IB6CpR8LNP;ns74C(dfm$!fBeipDZucX{i`3R!Y$BQ4{4)^13A`P^Q- zYMHVlmc-oox! z6VrHqX=cmBjyB|`Y-TyZ88n$@jWQt>Y5@pR<*Zz=N zl<52J1|2A^7_8e;O_lG!dcj?pnD@~(!dS61v}yFPVHQP7(#~G=6(0ZQKY;YjH_0l+ za$27k)kckpZ#M-MD2p9Oixx1399>}7tk2II&Y^*~alfNR6iNf$y|{rBuXl*q!k>u66W9hm zsg9)VPa6f+%0)h^N+(!q9$ZEXT2-u~&O2+nlnGq9IQBtBSEn@#r0Ck2-|;UU9Ip9K#&yV2cPqZKhXTio+3gM(LV|iVT+~!y zI9b}MsNv)4Ed^BKFmOt=zvrKEq90{?<|(s=Q(8EoC6#^V1ccM16zm0amm zx*4LV+mP}H{yV3ysNeah4MUecf?Yb!-1=aGnEJtOXSI)DYGVUlAHI9@Mn11fKF;76-tN7c$?W{!UnBWYPEWN88BL_@;~8t^-}o8b)Z*! zk_?KA)bU=d@C2G)zuPYf`_gi+yO-#14`mWi1WTWq&$_Ih>%DYeW|#ktCCT~^U?eMP zY1*p&gcz%Tik0gNXYbpe=Hu`O_Fu8J0xf3SBUkJUe&3;2x!@!UFh*xVZ*&^~o-G{a z*4fgXU<;EHYv{F!%ysX%^^F}=31Zuha$DB?a%5x%Y&V^(*5ZFdfNEp}4y8%JRF}jN z$~FCgL$@_@Oi*cDAn!gmL3fM^D$OOi8oEw$DB_YGmxBoGk(FLe;)%;}kN-ci=+S>svxt}&tAe>QyEfAqksO{;Jl zG(hZI%HUl&Aww2*U+IA$?Qkm?Q!XtTEgKHOXMfHxd^XP7-Z;r+ zZ&YuzX@8@H07xuQYieFiL+Bp>>!C!Jkg?duYvTQnLa9r#bA6aGs?O|+9LyX&q1@q% zmM$(`?~m3VPC50>ZK<{&xB0F>`JjKH(NhiHDw? zmf|mpWZI^im4mwG8_^hpI+2^fcxH_ztNleN)HPgNl!3F_U(}J~=KcYe&>IdKk@BA9 z9YCYF2XC2A?c@76lpo-KBR-2L`&qT1TMZ3tJ-Z@tX8#IX{)4tJ;xtKf37Ls?{fQEc3CNSsSq2b*CEMO04XFs^oQy6-f5LI|Cr}pv2N0l$ znxb-}yfJb3bkT!usS9v%R*CFH;u7JHEevtV3ZC1Y8_0&s=>2ym|Tf{_z76 zQ-T%@tm{GBfBT`uMSLv-Q1X{rLa)DGdm=#zkO!~=j->AL*@B=K!C0>Uc_0+g)>gxw zG5~~$lRoxkDuAKMY{ zb!*ozKVM%%fE2KMe)%{k z6ilQzh>UFUkaO&*h2mnxAw9K!0Z(d*Cn*ac>&z~)Ki#R!pJaVJZ zlANyBcP-!JJOn8NK;b=PaOi*+VESdjgBvj?yIa-S<0M` zefO$z2fS$lzgH&e!L>xM{a1D+1;rOgP56Pfj9ko2-x%j7{Pr!9Y(@zoPrtFJSbr5A z#sSXyw;VPVA<<6noC05Kk$m(@;N&EZvjrlP`Umxb^~fH@4r!C)3^B*t_WCV1Xf-d# z-CEx8!gDV1$BERU;km;*xbf&U&irG+Di{1JAJ%42tEUydjRXPkI2@RxD)KqZ z=uavu@|l9C`w0z6-0Hx}mxF?8d!g5|Z40K~9>`&HNljN`N9`GpAv1GKQbuz@goj>m zxRIYG2TjQ2lG0QrnS{990X8W}jz*}-?uwzX<(b+)XA*zGtX&)eL6E;|#B2A< z<+Gb@ep}sxD)Gs^K@jaDO9O{0e@KmXTLW^^IJ`Du4oZbl87ay6J=rb%sh*dG^&hm%a0G1KaLkgYg;ko;5txU&nsSNg5b^=(j1M={uuwZ?=NY18JMhjMu zd`7Cty~5D*W+&9&6BgU8Q_$K%eZ&{QG}zlnEiezxh2RDG-@I!VKCnHf_|*C!$qJG)q7Nj=(4-xP)`shibpuMy;x@^Q>EpWroH zzyD1R1bvQAFQ2*$1Ek=*!DOD!HjuteFN167x^eN?BMWx*GJZ<$YF31`fHL8I%GMiU zAyzE-b}4ckDarLbyYi|(!{D15NkC<2;l$ox_efT%&KBZvS*;P@W9nYuIf@d0g`O4L ztWh-Gh!B) zr}B~6<-9_&Wti&mYH?}FFUPsrtV?|(tb9R-87gOb`B4Yjrm}sIzLOH0v`g8o12#i_ zS6G=IN$ijEhA-EX?RSRU$K-tGpTXDH7d1Ywrl3c+j~Qe^tXR$g2La*&;A&WASqP`F zap-6^M~Tq%w@j{rN{w^1js^mPWfpBXkob-SqpXNM z@?4cLV0b^X#$p^bpB>Vq?!-~ZA_>rA0lcI~F-5O7YYwL$&jKf<_`B_9drpo0`L?8* zfSBC?%`Q8*F~3k`0*|LOA#gsg;8eY#_;DT?ylp7L6zs_Ioos`@H5j6+nkI6HB{ z1ivd5vu~22W0ewG@UeXPatT@jgDA%AG@Ch0`&{JhB4D(k_U6~@ciNT%(3Luo2N6N# z$oOX$4@)Yd7is*(bKFmJa~)e3GxOwZ#g(-q?d^$I)kIVCul=W#sF?jep~A7)INlS& z!lm`yXvrbi{VYm$uKX8evTwjk;AwI{QN}ds70s|hM-c(GVnQ-+eTB)$O`IR)pt4Sd zS1&ge#;3a8jB&-`zT2pRu>z&P$Qy!cVJfWzsc5yJ&!W~pI0|ppHJyS6W7|7cOJ{3tv{10=37z{6bt_XdhbP z(lOc!Z^SGI5-J`h=Lk=!Db@YHvy8&i-}8 zee{I4G(SSQS{6}HT(1<{B(#Cqo}V!KoLS+GEsSL}LGxUK>@rF|E9MBPUE`A?yLDv3 z-peqE9!8g0X0R(thj4VFz9a~WDmgG@_3#)hz;jl9nx33fm&#a&Ex0Aq@yFjFbKdq;kyX zZC$h{aI!9rs<+LwW$yo$^Dd@=Q@&aNXWq1!V@#?cnOG$CWBdS=Em@--n+mxDFf4TT*=qUc;b}vk`FH~v#SC1 z1zB%uiaOLhzqSKsf64qQWS%P_x1VHkIhXq;T`U-T7;xO(o)Y6Nd?KbdwwPCl&xLNi zvLO|m)WngqJF_J3Y_`B*^@251iGMaHAtli=120z!3n)me%ADt^-Rx)fIREh=^qe{7 z7mVo)c|ICme7#k(Nj{S$oSbE#pPB~qco1MyKl??Qo?AAKXd*(5z}G7vFh=AS-^PZ$ z-l$dt8ncwEynZlz5v4_`^7!X1IKIr|T((_Z^2IWlSh;T)&cYC)0_C(N`yxr7 z#o6@xi45xLQDXGVD$kTJkC0&vrb9zSD$_%;czuF*2CdwBqg=7bQ$5{R=ND>ATvGSzC=aHJEKIt}klbvM-`D^|aA zkiR-vGUCla@9fRTB9lNuKgkgO`s+g&lrAlK+#E!D#ad=!h=cj}{(KeLSXP(Iw97C$ z?oU_IKY&?_fpU>o zlx7uAXM}q%rXM3*M9kly06uhYJh}q;@3YMZb!~)Q!0(2`-y)H6we);bvld;{v3n*>kr}*32#BR zh&u#hVj^?4RJD@}*^lCeN3LM|UN_{Xi%xDl*5eTomSiuV03YuD0T=?na>1RCPR$Xn zjzWJP3;;Oqh{ZdD>&1hwTdGg50dSSmAnf-K*yj$KtlFKi<=qZ;U21%K=~83 zn(o1PFfjnp9FB}+bOBfgdQhIa({2Hu-!J>)U)*2ZZmyvHL}eH4R>=9wuEAJP6@4|t zjaJcQh4CC!Di}Q&sgwK-_8WAkU;RyT{PzT_MiRm;?#lr-x@3>++%vlNbM!YCyFup% zJp0Tu6axLiL5rJA)EGLVFQFNxtU$TSD4Kl1=g$UMYi97xZr771P+7pS=Nq`*VG#)r z*yvJ180A`C8K(cD$xJb|Y)A00(Lw;&Xaq3w*w{|dO;XJNV{>@%#I*|rGL|jYykYqn z11&@Q*ve$C_M+>y=9!;ZvYjGXkl_4CP!wzJ^DJU8CwXb#YYgD}UAp#4h?T9}378|N z*gV}UCStRCC18ezB`|^i6K76=ZE+e@PE70{!0Ee2FEQ;YFP~SVCdYUsIjP5Xvxw=X z?_(?D;`&(ADfTz%f6S%TU!NW5F$-dWF$BA+Xfo%ud?w_&3`J^s0+f!lv!)vaKh-|G z&{0s4tIS&*br6dZUXU5rGcNz`__W1argaPLheau=L|gHfuEwUPHIRDuPq(VQv4_(0 zU&wLtPf<^)p(Q!~Ts|oY^ee_7vuBo`>U;nWFAVNz1{(Z2Z5t8IVX`S^0%y-umZ#v# zy{;SHi^liV3KxxfEM4nyr-JXYhq0tnmTGTxXpfM^;m?zE@ivR8lr?d{csD^a)YUk+ zNTwtsnSQZ~-R}8hd^M$UWpOfe`my+f<5I}=;?eQr*Bxvcnp_WkjoByr7?=^=<-zrV zN$RxUa`UNbr}j-UO*HmQYU2x)##9@&9Y4j84EIO5AADx2rlQO4q`Gbd1T*_*46+Hq z5kDZuVpx{%)DM>2(s@!D7%GK_L`}VvhHAvLWdk+0Rn?~3nA9Y`d-T;NmYc*C0s6ye`Iz{7tnh_O;8t$^5X}?CopIQj(Ic3JHKXI?ZI*F7=4Xl*x z4bE)puHLt5DGLpL6HVxW8L@Avb!}Bp{@@!_b$A~wbHoYDWaYQ5RAPk^vrT+U*RX~} z%P>;Phcs9!B+@@mu$b8uH1Y^9zO@r(F3* zf>R*)@a3JhrcYD!uwy2turh`RB%DVMS1qUmK~jly!T7PIs7b@(=& zl$}P1AWJmv$8X1h#64C?bJ;|i<_kQ~o!YYwB#&9TU2BfIcEPk4AMcSkiecUeZjC+0 zcDbZ^;{jWI%^PQr&6#Pn^Zmh#k;=JM%F~##WwtwZX1qzi@^R1-CT2Jy%Zo9|!fn7l zc&;rkXh!x*y5S{nBeu0Y?iFr`VS<-(yOR#fJVV4lNYz+~VzKpbwdnNRMj5hBmY6V= zu>B{CkaaRm=lcGJb-jRgg+_SyD_xT^E6;k>=wX?e{EES8>0h<|fq66ciW>IcbDY>f zHR-y6Vm$r9ns8KAWQ0a<)N|MIjBuy;%Vb{fPvPBd(F+W`3g$R6)9^5|a1)gNcjqnT z>8wsZhuTG0#mVk?^6M|Ew(g$<54%WBTGcf!i^H-zDjMfbJtVP3K;M3JTHKP)8 zUbyyJHmkbQqm?7$G(T7MXR&ISf}}-E9dj9Vif{5T^@ zwVAe36y+f96otMHtTt3|wBf6aXbaxUi((?tMRlvzrkj#hWW(4m)tho&R4-wQ8(8&h z@kXl?E<^bEC2uf65=ftQal=vQ7=br03bA&)-u&@% zPy>mhRkD)6WMEdulG;*90!d3Uyv^A{<6u8a5AsaACY{e;;~S@KLdYAA_WLWKI{3O{ zV>$AUx%Sm3^9?w0w3n)UUJj>k!Ws&caz@px8foL^!{Zj47O5^YP9Pwr zyIS;m{{|Q=_6x`Ys?OcEar$#iqP%kX;V3NIb+!E&(N256WSFAwfn{o;YH!B2bisvs zhP}Ge3^^s!rtsMCR+IC&1^4_Snh%|*&vhGh#iFA$Hc?> zs&;}e6UzA5Etc(m+C(i|CYu)icCKLH^?oW5?=rW0n-enrNYPlt(e zvpzu&?|F$0hgKEG!krvvcZSQ9N*8i@Ho?Tq=PPD((2aT9%(xj^Ehd($cSVrW?ee2Z zWulR?G}9SI<-zuP^HiOQqb;Gd2|g=bU8veM!K;8adjp)vaF`UTHLH8jfxb!kR#)EBV9iw8a;Pnj93c#-k;Hi(ULWH)z^3B9MLJX~)ISCt z>5I%*Jg~PEAuysYh*(r@YEy|lnr(;T!12O*$V+Lb({+`G&r*q64B6=ra9uaj#Kf%K zEexlQcDk$;(w)P4hsV7Mo)qC(rn*T?N>!4#da2v=!WOCAz>X;ccRfrX0Jg^hW|B|f ztXAfq-TVQ>En{XNg=v($GYCiwk0VW&M|xl#oyRURziimhie=cTPpPhI<0m{UfTOpt zQp)MMJ4(`!$#7b_k@J+4Z-!PO@P4HUTBnL3K@Aph8JMx~NvPWo+AO=XK0~JMF4U3n z6ra>GEN9U+x4!B!+P(V1Ib^L8K}rX;p?++{q!W8W^*f;PzG9ja$LsIinqw`h{+{2Y zH$JDF)MV4e2EE1x$&J~`z4D`Z6`3q%XB8loL&WZWn>{2@ufN~w*uxKqhgaPwpr zYok>J2eM5;I!4SO>nEN^h=+Gh6k#-q8pmX_28M$dko8Xr87WPE#c6QJTGI4>;%E(# zJ-G86ojhtajXY^qyH`>eoU#4(bFxM0y3k_&8ZT5-w17`}_GF4%M%PCnkDHRmZX7kA znqUb)kDicPva~Ts_)GUbNo62gPVDLjpk?Mv+Z#oXebeWRZa0Z)bB`ZR zh80LYM`b7JIVWiH`Eyg4ZJ@z(eY}&Tq|!Bg;?IPsg;-dMrB$oQwxnYptWcQKoJ-k* z={EB_GmW$%v(FxN+8b8&LpN4Z+4zwobE7pawBr*c<=T!j<1iDC1q7v8w`r2rboAIm zHScb}p;C6F*c{~JIMMr7Hh2fYuHFe$z>e}zq5aqlvAeYiZhZGy!cLKdUZS+ zB51##wq}8`##$&a*CqDljCkcBl*IvqEOBRn1NM%XzEvqsV4P*PMf)f`Gi%6^&A6yE z+dxY7HcBP3v~+CYLH8*CV~6+4*cAyYoGpX)T=mg5<4nd5K|i+oKD9C`4((q>J?me! zym>K(z>>EHkzv)N1q`CkoGYSl$?p~`_%=||-OOPu$pRA}KDisWPZ-;Yk9${j%7m$n zs#)ulPnTARVt0!ttCd7Srtb|@RQA-jN=q{|03LEsa1NrRt62R;8`*8~Fz-a#*_3JZaqNPGkK*G^8W^9QqI7EG(s}ku1Vw56L(UI#tov_*3=jMePW5N}<7}kyv{# z4H%^bOcDM zOM4pk0qN}kKR$Xr!_v58pLSHAANRS|dns^0aoRv*9Y;=q#4=mDHl4;^AuHLGePFZf z4a>kC&Io1>X#I`EkB=DOByjP?87l-(>v1)6!XwblXqEk9{co|6*hR4qDGqS*!0mj?S@B1mFgq zc_>Kl`G13#DP5{IvT^$F-O`HXW*5W*M=SR$Bp|kj0HiX-PA3)YKVct^DuGoWY|8c= z6~oB+cW?|ctWL9pM4~1~{Y{%+BQ5l%a$iyVTt=a#Vt+wkQ5XQ+o)3ey-vi%?x1QUp z_6?HScYi_Gjf{>R=}n^zPSEu=*ZP}m$N!fZYmWSs=YTgyI1!UAV|TAF9^%db%h22^ zQRuS6<4q6RQ!Vmo*6CWV)9YQ5|EQsaKZ$0b&wgV={x)2ryg3e9SlE5UWeGdmP3pG8 ztiQ*7jDT+i|7Q4d^5yCbKllO}_*^|l@_d4Uk3rd{Z>OZFMb$jM08mxJZ~qPFhXvAP zK6)d4>V5zE?}JMW;@`DNoCo!xZ19VW_Km576R&%0=juTeR%HJF zlN9QM+s7BFqt7;WSbF@wKTV9L+Qx-ZH% z^6%uEkRhHJm|7+!!582`VD=F9@^ALpl8b?7~lLR*(=l^R@sO%HBzy~E|Te;~idW)ca zKrSg`OcySn)2vf}CCwKJt%Os*`1J`HYVQD?ci@O zo;iI}?w=F3vp(RVEAnkMv&O1{7rE(s$rbB=O>I zG!*^;)+JVzMYHZc1yp8FhpWu`RlK1iQ+Gceo5?UWIa`YP8+x;y+i=-aIartsx^~If zm^P+0VYCATsqWKVRL9Zdnb`_{k4b%lzN*}DAvF)^r70CJWeoI0R{u<_Swg5)FW1FY zOEk@t!Z?PK-*hckeVNIkwELFEpTbNL26xA-r;lgH<+i_3Vf$tkkme{B(G!rFteHiv zOCcC4FR#w~J~3rzgp|^TjZSQCzeSywsTr;eRTU1m;LFGurtgeO7byI4#pSK9V~YI6tX9TKS~o+Z^t^>YmCS|k{kB9U+LuWL1Kj#c zyA$dsLI#JQV#1xTh)%P1k9zmgsqv|sExWP>+tI}czw{-!r=n)!GzvOa{XFrs%NSO& z@)XXgs!hL-US)a31WH%e(H!TmI!J^7(XS5}Yaacp+LyVA2xAuOXFpsm`uz0jKDv-k zXc{{Ra}b-SE8Mk|-bct)Xu}A;szqNqUT9U>7#1DAF}y9W>XA{b_YHSeXVvz$m@%gK zBXW{bL^mEY&CIV>oK<|Tbprm(}8N>mtVMC$+ls)pr)P1JtV#K z#pjnM-IcyeWt&)<3ANsV)OJO33Dv6f`{r2ll6p6mHGM=M!k&gYl*W(@DlY-VkBeSB zvV1mm9n=rZCG1S54l(o`h?Cl8$24GD-&v*PvFP$WlR<>pJKvL3C!l?(wDaCtA zd23TjywK~giH-etl6IM?H(Osu-S*{VeN6U8}DYkpkCytNktBJ>260f!E8+dHDQspb}E>uBauW!V{>92lXJ>USLSa`oo2Z zm_00f#kGlPHH@E@THVYMh1ID0rBY|><~GxDox&@xL;l@5tDRNG&g--LZ;$@~boxpL z&FRTCQzkV)+GFrUIwciCn7sf51($y|d>O;dBg?&1#gveSSX8@8{=v|j+`vxm zy=_t`2W~mYp8XWxZP|&4rQv(J)b-yZdzd_z8YVLj@5$U)giHpX1-7ILYvht{ypa=r zB30Ut^B*4!%8?HJKi2&o42V^kJzsXA8DNr8LK1RP)d3RWcfA(cG~9X}S#@&yJacFL zqbtXXDUXW3z|4kJAZ;vpoC^E}j>bX?974N@+xW9R(!ta1M^e(N*4YeO3`041r$x=} zrKfFd^2wBtp2zzLCj1O<%f+Hao$j-`ozYiueIvDY&h0xG9$yY+i+6r#VRur;K3q_K zxG~qjRfza0Sv*uZlri>R3zLB@8ATfH8@()~`I$Osb1e2a#@Yo_L(|^B&}n78&J%<_ z@U5|8extZ$N1xV#FY?cx|2xl(oZ>nCN1oUFh5a^mg!kEwMek$(fG5Mq1>`BEBS$n( zsrGhwMd{^3lv8nIFI_yG{YlfD2*x*ywtkvGxbVBJq``{|yaQDR+&j7^FTD&UQP zD$naV=@%;Jc&Ur}BGlngd>7~(4yA$+>XQDQwfCQ1mR94aS#ml7dCa3~tqzn~x7}d| zF@z zhigpjonD}=%>k!hL%VnOSP!NKxkbva>k z)$=i4)N#Z1E5Z}nW$}vC?LOv7ATSpqA zme156D(O;m4temL16|SfA05F}is~MVB_7+!^_|DblP>qL#y{+Gd>0mO!x7?2mr;Ng zJ5C9z*_N{KEP^_Pegz5{ z=H3S{5&#_UoU^H~HC|kibNDc`eZRwY;s-%Yrx(7#d=4=(w;;aM5qj3lc~@LSW}oYMQ}L)y>2!YtN3$L9>O-BZ9D@YsYzTo zn`zhec(r0TGuQmG$0SlN#kyi{My&S=v>wy&`*XmI{tRHWrbd)T*$@sY;JJ<&FTux4 zt6np=o=humtGDrVR@_zT&Rt_C-?JxXB;t-8nmC$og_yNd8B4CR6wLGNRzr>FR|Z;r zP4(EmbS!G#j}qwG)R5T)3sl1yK84>^OE@~M%9&=xOPerkyX~k}I@!L#W!f(FJu5Bs zj?gRAbD!l1&wBms+-r5D>^Xt5MN$DW zy}dG#`gx^3&k*dCRX|3p%ENhTx=d);dO9C?K3PogCA|ylsHB=5r5((gnTkD9b#mrQ z@4My5bbVHCt3=hLdtdsnn>}8EM6&@UmUU1r$f7{eTAk(dJoXEKI&U~u5jZfIrD>js z5kJX)(aDAbN4Ew5ED5==_S8Q&T}`%-@M0TT%BWirk!^%zr2+~#q_1n(S~1xeB#Q)- ziMN0ecgLKt-c;Ey^3aHuj$|0{UC9=2UA;qca%_NZjcG}(P^qs!>q^c2Gxeo>e=iP- z4$}DU-}dSh90>mal;0*lKUvkvEpwv-kRdg4O*XI(Tumi zk3LBPe2gjJTEbg(2MJ}0^G(tB>8GZWs<0JwxqxCFKd=43WC9GQgh*8Tr*{h!tWol! zD8%c2ZP%*~b7{9XnS6vU1&o@a20`>D#?snS6avIg5#Z6@6Y|9c@8y}dG{3X2j%|H6 zl$^GaLuOR(h&n`b@WmD|%Tkls|0FFksy%y%6<);im+0FzMjI4ro<6G^@m?jB%Gspk z%v3b;uDJ=i=oosoa;~1d>A}cV_rygH0eU6aVFzNMODR!qNWeS@rKY(QE-V-RD$5Oy z0QR6D-KWQf8C5Z7K@PuL@4a1--mF%q>3MWRv{2xf)m6K^5L_WAw1c_Pe(C zByrD}Va!~;LHutKSXHOuc8YlqIscowQC9Z$)fGI{?1xtttOpU7!emsgZ+#FS!8u-9 z69q)8N6}#^hy{+`8J9F)IM(gkw|K$ubhwY^S%2}>J9wiw=_EzW)0(0k+0XssX_m9w z+VwS+b*5Gv!%|$d-hKMzE9M)z|M+>QRG%;I->_*ap;u5bs%K_#9bQk>>{s4Kt1jRq z&PiH$ah00N7o$w`JxVmXEl z?EERHt}1barbbo0xMU6kGfuJ4z&ia;6W(Kg5S5Y#8WY2Mup@0nG4hgJgUTtR{O>R5bb1%tZSrj1En+dbgU`eJ zt9vxH`g->*kNKQyV_FSsbpODPW&{5o5vw-*UjF>f2#-ouoprU>T*}tbU-iQbN$UQ8 zXsv4mQF)bA_C!IsoWAAf@2Tv2r~gB<^YX`Vy9v{3qHtPd{r=!{e6xlqLU9o@QQ2*j zRpR^k?8=6JyYHh1TI5STZ?Ei|c7b{{8q~LA_Q!QJG&KHq6Bj7E2b#dmZ*-Flu~{^< z=TFW*qYI#+q^xIXPmRn1uwCJUw~y|N{om75ZO_+rXiuM1^%e`kONQqI zoCxFG8ZYYa*Bbso4;)@s8(K&MDj);ZlmHn351{2uKH7&4ok&6EXPW?M@tdiluB0A~ z%coJpge3zuJ$_#0>@Fe>0XNO(L3xCKRwXQ+Fvz@RglBN>JvCxG)m1E>1@4VLRl6S! z9DTQ7kdRTnIn=0QkHw`MRwVH%IoU%+2mrQY!>6RP0uIz+()^BEXiT(pLl`W4uFrzF zM_bOeGQqV%$~@MGR(Hgi^Y!O>)qf;4x(YzG-`$tjbu4D;W4Jexo8itaSJxgl?K&OK zT~a<*;ep!c$T2d%l^ zq%o;)nXOHfd6sMY&HU@Ln$0}u{{PzFgrQGQC@IiVdVo7}44{Te9rWO3pGMI@d}>bvikNIXszc{GsD4fT z?kkq#JL+>Spn7ra1@+a->@O^M!YGI0{2EJJU{DF%x%&+^u?zVsmSu|kOWmjugrn`u>L|dx76N5rJgqim=tWVw{LEiB+|3a-G^zYDB*;DRWu) zI#;;x@eToS=FVbJWyDzg%o?1dwlN}iXQVjRf+CB_`R09h9g~ldfh8w0O7Ul~@b82I zgpyFb3-=|^P=1x2X_wJPSuF_`u-C9(!CUSz7jb>B$V{QUNgKNH+sN8^P3!(A1xxSu z>>1$f={Wr^BIkO%vVMBpyHnss4jx35+4yjzv;^^pK{M<{hqM^~Kvd!YTQn@=*ZEn6 z3G$U(eyh-<_1ON2N>-~?rKYomzm>R^*ZpnhIc~m|kR>_)m`-5&j_AugwWjAuXXR8F z*mNH_H#maFn_N=3#lprGHDpVip>Cg;oeh-O4Y{uqW$hk?FBZKnFF#cV{1ApUTY#kj zBbFk9)NMJ#U;#m@Z>jIc5h~k^rh?4<)r4j1*tH6X;t!unhP)Drp;1-u1?gnG`8M`i z0JSvvow!q0VPB5>luMGv*jZ@R z+MEZJk7cWHuJ^oCuV4DOGc2a3&JxzItY*w2ECJugPvoeXG|88MLP^Coxe^Y{Mc2cl zAlMW1LEhc;B?Z{fq$O_C%4jEJwV7FWy;#@aX1T)B#ckI}5qF}jA9hDZMVlen&goJn zSwC60Bb@UH*sPPWh1+Nyyq8SBt5e0|8+$A6Rxt2s7q_f|YdF=N&Y5x6z$H<-+y9{j z?^PgU*Xn+Mj!uh&a@&HZ+nt~}1l#Jb0%5L?<@DI#9&Px?OE{%rsBTnUQcYzS7bA5^ zvNh+*3#e)pZgtQ*!h9}##iDBMVg-?_+*kUa)2i&ocwxA+p6mUeAZ)d9)?+%+>KbAi zrd+wNO7aUM7(fpBW7m4AY34*Sda8AE^e^KRVYIr5YZ$ntAxdZF8l;M$Le1SJNwo%D zjU0E+hBk+ODu%!5Dc;icYQ`Qr@ zVfx9LF;9Q=7x*;WBP~PXoU;`j_dGaoW zfX!WRnf+XUul4HD#d~$3m8YJrS2=1Jm-1DTR?d9SZ-gC(hF1M+WHrI> zo|r)ez}{yn$uZF#n1XplZoA#$(KeW)X;n{Zgn`K-&BbKhZs1MIbsjUzWD&@iT@#Uvz0;$H$tR3yr<{E#x|Y^dB(@Vr zQ67-2$J;;5TF$2TsPqeqSeE?|>qG24W2zsT!O0AO#HXrC5aLFOnOx60qC6)Xd@QF< za43!NkOGfY&^fVpZ1mbqcXvTYxa}3iey1Uyp`XJ=?!2D{H80bYP>`+x4UYB&>A7}n zT9TJ}aZHH}C$Du?QmmIJuJ@GsmI(t6GogMf$hY3SKNHt3llT}5(00|?Hl$d++NNHk z$IESzZt96bnMd5KzjMPseV6Ok;w@__R;Veb>>O2$Zv$i*E%HiNFzby6$yQ-Vt~Od5 zHIeC6W%(&M)s8Adz?!oTE>#O%HC`Sdp{S49RbnI1Wuc4b($gUNg5w&U$NjtM<$amZ zEPu;UkdSRc0o0Zrys3pGKO~)7dot(_4oRx*OE}kfy(*LXcA4^pR0_D2RZ*hSLsCno zAQEEdvf@wP?90}WZ^m;W(b&#hLYb3)b2q67XdfSQ^%lJ|nc@mUobL^ulKUlcbG{G?mj7M=9$b%^%ng04bffgJFwIr8{L_OU%UGQL!(mB%tecL>RM?ZWYMa2G(& zo6%IBVp3O0vvFZ?-2UkSwv)b0%eheI-Bn`mV6J`fECm65!66p9p|*#W`X8E7X#S)% zPH!$JevTJOIMtAegBJ1&&crjb2}&W*f-C#F5m_!I-)@8%i2k-T8y+>>zo%ca2}ys zoIa;<1!IxDj!YkE*W{0zn)u)PWnnTyc5Ik}T+8LAI|d^NszcVW;&e;0Ass34`%1{@ zKr2+MvSU^ScHUJEW-5{TTtver^R#V~%4G~@0bf|M2f*8Mos{2tixNxo7bo@D8H7@~ zCN*d&D_9K+i#J{Bm@IK%)XVt4Yq#q?F=7P>p}JXbDo>=xgeY*d_kT7>Ce3XTsQ&}V3uG?7}) z-Yn(!JibdNhUsyNbbWcwqZUrjmPild$OKz|g$X2AF{e<3(>o6YqC9olFwGn%SSXoK zFumzif$WgiYqplYAK}2lq~rfQYh!n3KXf7zmChJ z@1#t_<~7VYW0p{5-b_=rtZVgEQU@S-lB~hdOuWd^4q1|9$3nhKx5y0=t5dAG#~vu- zrVJe~$>zeCRJ#Se7fDm_)kN=qQ8JMmGOXh@38vuQV=p$5Z>q`C07)0~F4*qt#$J^k z{=m=Zzk20vv@%4Ed^d&Q=~YVwF%$rxc1+(RE(Du`bYk!e+$`%NKz~ehEcN0GCUeHo zsi*8pY$4sUimFt#qkIuzR&#V#TDYa7e7Gel`j7$@w@7Wmy)5ZGt5BA#v#ExY|Ik7V z+@$4I?kX$`r6Pb-Zcs@IY)RsA{U8E%b|uwfMZTdk)~f@5s8`H_&=4y1i8n@ z^xvOg7bvKeN}0j{Ad?v&LhY+Qv%gj3!*_g(n|<&f+WXz88hEx9 zT-C{Tis}6P5GK?t)LAa-)+>W*orvl^a2YR|h9($PYMj{})C_bc4$YrW2WxcAcj&%P z<6MSnPVPtdM$OT8Kl?>g2w~FVEH7` z{nCIX6#W%B8Y_D=x;XUbg54!(K%KMKS&NozlXW^*K0RK3orIK>l!6+(q|8F|HUE%= zG#<}j!yRo}W_8_N}KhF50|f%b_Y`aVFWM!tHC;G8pp znhlipjwgA)4Q0r@Kor=sCAUZGMx&QE|x4Qwn32Sb>R>iVCBE zE|bgI1_r3+d4N^5;Ew_A4DWGWKWWPaXgXf5A4Ld-WTux0qM|~ih|2yGJAAz*x#mm} zy+>a%(}_tJ9IwN12rP-TND~%yce5p_FY#O|z&XpD_KiJi=*=#4k6Ga8*QAb*k>7O+5!jxlN2;p5DX0u!W}7c}xy>P=5|}sV{|F<@YLp5J zEL!i;k^lZu5`)*s#AO|UZ|C;p0gmk}SnvK!Sop!?m@E`2)~skje)}I96nZe;a`5Aj zde;aE^MqEUsweE-%UPY45JH(&)D=}3>|7U|U=7GL3)D=AzTit)ZBtsO8|j7&$!i4; zDMaq;`QBI?G_70X5dwyIE%nt&R)HYzTVIj8*+=ca78Dc%L64oYKTjM*i7B{VW4(@b zcluv*0fC7ldl>Pp(S8pN4dcbte`o^n^m1a1L!UnW%OgCaL9dV7$a&6~_nl0T>c{WY5+syOmJN!T;` zzscT@uPe77z9Y&6Qqvrc&TypvH@0^rWqi*vMGp&rOZv1EJe;un-E_v%`rDzv6S=&y zaB*5E^7kqmVwuXHW279Aerl;OuBpP3Bz{Xivtl3zou$kw)9QikfPDe#A z!(}rE!J8ZRr>gb7b(i5N(Sh@`2vO2PjQjiL=_f;ro{ynuc<|BIn66jR_m^#p#%O52 z_4XaaLT#z+WJ2x&176f!3j4Lnce8|sJDsA5WjT1;d%HNoXY4#D9%t}YZR>pm_G5Nz zFxWS*o}d00&(Fo*q8&rs9W;vH#Mm{ggD6`DI@!N6iC{-3+m~k#-~YB%gfY{`|F674 z7UQoKhYrKz?f*J0BqbI2)<-9t#fz3bQc*r5xgzH%oGV=xltTo^TkV8HOlliM zC=XxsGg4>yGy4y`EpI>Z%9)IRPMf|;jBSsEFS9xB?Wqe{|6AQMYIR|MMxDjQ4`W$a zap|{2BK~l=hqK3YJ-9#4{`2b4w7kq-!xpJSF>(Y_9F1P9(?oAm{z8l8zU=&47kdr_ z%3XK=Yg&r%_yPF!1Ji_GxmV0cA8hG&j7PS_Qg1MaC;#+7q)i(^$AOb=Aiu6AxQ=E?}QNFpR+ zl&LZ$0wK)$_=IVqASVx z2^MlmSVb@c!qqC&r_5HLkFQQdpG0Q(8!d7=Mn%vY->}lSme8M+Lf9hMXji_h+*gd< zB<$HgVYQHhQ&c7DRLFDMyvZut+I7kO1EH~uRURUwX#A8c7p#!gY5i(AZ>tFo+~I<< z*Xu&7IhCu^npp}m1IyY8FP_6o{hJO0%i}3eLyK~9re;>o82p3tZ_Cg2jd+X(M1^J{ zycMOB2Ul9Sz|twWy6nTAt@Xfp8qYyx!rHWyA^u*{Vtk6pxN|Vew{`KS;jT?|{2N5< zj?5~|Hkw4c$W9CIt$L!gE8O11%8`n;lc~fqlQXq_M=dD4U|$t@KZ*H}h@3c~vgm=8 z*XvEd6r3a^B=oc~Da+U`w09eX?IYz2o@{&Uaqtwm@E$iTArrDTElVFqsOFsVWNc|K zsMBi8YDL?foOlWEhM;m#>e}jkXB2}=ypt+PCs*h#G=RzW~0f|Y~y;9?7B8G{EY8O0V& zK|x$qpn+sf;Q0Wh21l>Chst%*e1V;?N*v4i92DCw!;#R}g1}OnGDTlg#fd$86TO`M zp(D##B;hhurkS6w_qncG-+4KUfnCms$|!>z6~gkx&%?>=h$p9SQ2|}rL-d1wqcY_| zaN`{EeLaU+bxH$GZW`AGG|T`G{Lee_i`7<&zt@(M3TW9B$Mtb*9ojEYuYFyl7&7uEbowIn4ze zjsq_Jj^MTXGBLfxweJFvwE~qA-l!9OfO(cw-{P&&(6x+y8rBdTNr^iMR}OjWFs3w} zKk9wja`Q#5$(h2QZ8B-np#BVkzbLwVY19wo%El0tOikUX;bT4?$k(v_z@c7TcmI}d z$T#Bw^8lH}Y9KW9QJTL5nv=aPU(2M@s>17&oV0tyEzH%Cpi3{xb@klm<3_>DZW+3g zNh$WMmDRh+-h3V$q8u7m0r-IUrV zWvgH|*m1~>wsOPe9g-!%2cu#+DZk)5=V3R(96unS&$+Q2kC}IYcZh5m)tIB-FM~pC3fe4T<6`}GV8%b96{?EIyYa#_C8{pZ!%iiov-Y#v}6mMkE zXu>+D)g(8i`^N#VJXBAM?xI1;{VuLRe#$hd9|Zpw0|khZ&M-5gfl>Ju9c(Lfgw@bpnIvh&Qe^gq`XX$pB}x5g+Rel8L67 zM~Ywb6YV6@3Sw01L6t=@OzG5T3@y?LS3UZd*rp6lDtfh@)q0NJBgo7y1Z+_7V;oljny0DMw<9bJ? zM$PlsB_akt_ULUcgkVhRq_CLBw%aec4e^=^ZE6s2q`dD&G_B#^RmlFyQg@m`cM2B9 zbfktCwCGt&EOvkFE@ejV+5J_0A=dAZuVKK8eeVr1KrYfH(F;yhwj2J3R>AcaM}Jgh z(DhtCswUwy-8%J6QcGhPy5SK5?^0@vH>rg47n#3Si-gH51sQc`L;JC8=)o$-ZgTcl z1;7IDtik2_t2-wmE$$6BgmU;u?b88IAl%l2{$=&FZ&8ek;Q#$u5K9*~V&AdAUIjkAkY~PHmv5f^bEhIVn;#TSL%2y7AtrC&O)D-SG z)lph=>fr1CnV`W2H z4-o0T!|UbSJ6T}bl?79qj-RBielY=g6s!YxVolPw*pJ?>Td-yPPZq6tW0&2^?(@d6 z?AA{FD1WbJfR}d=w5hA%)f!By{5l$#l%xpf7W};RH6yo)jw5Sr9ji!2baGe1O9($T zu3Mq;NIt2Xd8cZMG^5{Ihxfp=SF_nOx#1jkHOu>B_Go%r~RH#vjVjBj;R|fLVgtk%>Z1x)oC%ml798pnO3%$ zJvpXXpcS?!Ip2Izx=u-!X*NRw)sn01wvF9tZ1BAKG)(DmX;VoL?bDF$rkt%ZZ;2%= zan<$N=#S_yYXMT?kvp5$yk~3KHRXz%NZNTeMcN9P=b~R2#aduhS~c!$Q$JWHtsSa` zI`j-P&tUCkC(_J>T_E$tmL?;RpED|zqr!L@SF3ct9u)c=TN8Y{>m&jwBJ6FY{uHqR zx9b{T*!6`o?;QD^Nfz>FL1qgm^#%5-g+s;t@}^iqL%-NhtKo9$7nUlv&IvG zk<_?_!}3fFLYuHuG~Z{!5=>554Ek;xGEUYwoMaYfd||2*9uCN~ea?zHm$E4D?@`B5 zY(FdoC=CqEuE*fi{c3Lp`-`Q)W-PQNS^*)YI6Nt1c^gfWAa#q7Z~aM?&R(Kib|q38 z4Rt&j%2rpG1KO+ZAMxCwRWB{m9;Ws;#glVw%Pdix+^c~H-6%YoR|>BMC_$rlNLD3{ z=Rm1yMm6&?kuw2BZ1>%4`nrV2>bkSp7%guh#)g=ZO}UGNRK4wDD8P{JShtvF z=*`v2lP^@sb(k;Zaa~)A-3r_Gxk;{%V0&a6O5;Pyq(s^YuWxV&xw#^|Gc>TS8^3-H zC9IW5N*?^$v1Bl14DQ;)X^b@AbEBPXoXB+Sw>H6>go7pZN@PQZ zH0#`|?*fQq*kgRgK+qz#N_i2byMkPYj`!H{B{_PuX{i6CN_R1E<4c7@M_Lg&MIh2By^m@4hQ4*!z{oqL*o@+ zypNA6_4mR*fkCCxr7!Pq1WqXN_AEGnolqxfg!N?6Qc#^UuSLF{VxMD&CClVy-7i*G zmC#qlNiepyv#2~7KGQJ+LAe{P@+PpGQ5R@zo?BI-Snh73Ja^gh<%66?noE#SL1au< zvBstV@xGAlpjI-MBmd%yy4GhTmGHQgfBj2(AOA^0|}I+>Zsa5O6)6p?|h1Fc}W6 zw^AkEV`DL^kR3W_88(fX(%2Nnqu=_yr7fFHaJ6Lr@$7EsM_7wZ`m~|{IwxlnCMrsNj6yyqA5%?@FAW+z?^7CIV=t`W(R=-ZuAx5A4ERGHkW@PRhVb@R zW-=xn20zX^|6AS6`#QbShLC`IAvt#Da`qZmVIN#4j-}3_9OQ*qX;w+~MmxJQicK7r z5*Yb{BLFZ+-SM&-Bc%wa^NAWJGBKJjjIGyc;OPw-?X6(&Fd>Qgv9{F{X;vW26==P` z-SY&*l1!Iej#F%nDZ8nO%jRU7^p)(|H5Z533lIaRyw(WgHnpio*4Nk6M>vkoB~9vq z=IY#3yedODiK_<;=-3un&Aj6zL$01)HGOHMSlEnLL=nomi3ewug7LTBj38 zU^y2ZVJ3NDon2B-59fg}?HtxspLHx)F%y!$B?OAx-m3el%5ZJ?E(WSr2D2g+3n3* zj*7xmHL%EVKW)J$%*J(-T#-3hnR)6a_E;vr)!@dkhsBjEJi zoele`nCZNChly|2b#zCLxX)g*S4H!~>v1V35d2|f=GLzrM>pDk*#+!k%p8uz7(2sw zaYlkx1~=R7UD$6#*-A_^COjR_&TeCIIhpLl#*dgTBP5{&_Mx^9etz22{5i+sAHJOz zU@drLW1ruxeZ#JK%^fzWLZLc@N&oEbWz3=}3joDlc-6ay_Qw10_Gb03EP(d@itqlP zg9JTK@3HcK^oozE->hi+7V2~1`+vGhrgmjNHVs--;;m+3GtlvG+5er#QIdm*q{myw zZ(jplOjhi>%Woe2mnI%Z-(>$mM&C8pN1O+aZmDiRh+W5>J0+Dy_qDB5%%UM^M;%qq z{AkE7PePsp0%JTlP*-zj!|MQxk@%wK_67BGP$3aZZ zP+7d0PB;Egk+WSP{>8+bM@zhtT0Mt%l6Vo{Y37}dKD_w{(fIz?4nrw^aPmgY8?S$R zu-?JvWK*iE;pRDN${4Ef_=&F&Jq;qH z>gbG-XgJ&cvp~gEO5E(ek4`3Ed)fHp__UiN_p%|D7I6?Ax{ek0*MsHxgQOo5+-v+If1A3vjuIC_)oFWBbBuYcyR zd(``lUd6Xy=49rleC)4CS^t(6jY!+?xnu6VpGUOo`j7j+-679681eCOBpZ!=M7mg0 zo_7vOymz+u`D@0)NXN@ERu9m}eK@=jF`NB6_vcFghXy|2UesJ>{9lvH-(HoFzdo}O zoAn)L7V^lW=j)rOd{iFOCl82C5ccRhT$R;l{VVdz3%mT5RlluK%F@AD{>RTyI?Qk^&eW4w0t~%l>Jntm2D&17e~1>t1lGm z&6CdhRIg6x{${npU>asjU7WzbMKzMJzqcaruV#1izfxJMt}2f4e+L{wsG>pU#n%{2 z$}W<>&r{-G70v&9#v}-qzoZqvb|W7Y)mQS0c>ecFj!gR~<()A@q=(zXbJDu$)0OQ2 zgnE`!%Fn)RA$OXq-wU}rY=SGk&1>@4k^W9$KlXceK8L0yed+ss^5E^|nOnq)6+uzk zo4bFqy&F!u#{STQRo;Hb`Q5D2hJiq9H<$)q)?hY)N0N|K22+d3Yk+Ik90BI&Fo54s z5ns8kHX#kF1!38yI$6<-4{oLQ0XO+MP8tL$gaxl>>%AD8U!NG*So9FDiM$@UNTWJo zz@JP_Ew+A-4x^U+jmfH}_EaCi-j!X0nBV$|G_TQ6))$Pr95I&?>YmE?J@L@jGZPa= zSr%@kJ_xLzjN|7!{?W7bKSRU3+L&F*Tp@myHY1kn_%YNeInX1|njz}WWJpAoKUytO z*&7i4j;PZ4*~W@(!Uy|uJJWe?Iz(&^LZP>Mh@AxN?e> z!n|l{p36SKtyKkcxDs#tKX!lxkS+~N^jLusAaF>&xZzhy-*{J7 zVBXtsMaJcXCEv-~q*X!U{l2R=W5n}WeLX>kpC;%5jUp7-0bcy@RHQp=Hjml3(r?*P zLj(E9a=f9zpfWZGDe=G$3^OsN-XrgS>>zIw+qy{LRGH$r(+=BVtxp%%9F zZmO%e8{@^o)9anWv7~~j6F~UzLRnWiqgL?`kJ?OBsTXrH5~^{(_<*B!3gWaHNA3Gg z-;Xc01gi#x7EM}>Z1_|XrPw$O*`y3x%q|h7tmtw}&%2TZwNy&$kb3N|u=RM*lx!!11uKKeM(R{EgKQtKcg9?yZEF(4c$C<}6LO%$J zsyM_Goi~33lMl3V50LVvCzbH&7imElic{d=(ppls{`|2sQQXS$`RS~z;U-Gb8ud!v z33b2GV$qj zLx~u#Ff7!qKv9S=i8G}ia>|EP&YGos_R{flM`klmNtpYY-NKlWH!86&V8d2IB z`xN=%IY<@3a`o3@q6l^&;O;p0K@Thou_Py@lA*!Gk+IPs@&LQPgBt$O)vT2ym+h;>h2t~N)l4u5%E_3Q^2wwp znm%6C4AwMYTbpGmLv!tLMULKitDCCPG-h}O7{$`BkZx69wTm8)GZ6A~_Sh1=m2!}j zTvLf;DGBK$Do}cnDwdgQ3SNcCmU04gN}6ri>OfoCCKz}iULKYx)puCeqiq(^mKQ>m zxi*KN;M9O3PP(2i*l8WJMO|*d1u34EcBL@U(|R35Q&CRZx^m+&*P?Vr=1unQjZP})8}A+>Hz7iYb6Dy2ettg9EJdHqU# zg+H)xY8K^kf;Byp!AYC4=mkJiO-1SEqI}vMffaKRvywlGO&P8;L03kV5CRIFP+mc) zqOM_$V!&0&psM(%U{0CphJNU!eo4bZ%u^)hoU&ppWxos{j8&xi z@a%+@cb?9Agq14{qL?mccO3HsPj%17N{v9bw9i?q8P@-Kt?@neGadR--#B^d`ZsW? zq58aziI|UrPLYGaDk63TOCy~{gEYkf)3tI74JGc6$c_ zht62lutAlt=S*_l-?ow9Uq_)w>w<9&3mh{9H`&h;&VW|IjReAQ!(-v1IBmhhF#awXm93w{0%dwUyJ zPv zICOAtC-nFQ7B#Arj=kSjYT1p(?2qm&ACJ2m`M|PeLBGp67qvek%$ z)V?AsCj;&gaGmaJL<|#-*rz>=R2<&(>Acn7w5P0J@gw{Xjb|U1=62A)C^5yzX?%g3 zTKU(KbJA^6hOc^|M=(4>m-_P_r-)@LZr+>cX3X0Qkds2?aIOMg_{HPafiTpe3+Pn< z{1Xut*$jEdN;^X}{AZSjM8LxkyD_dubHRHesCa9vBIyFIm`7n{tG=PVcw&nV`vruSqhhri)u8o>;hyF3d$2Z#mkv_v#sGy1XP!y%@;VrW)vl(%l4Z=Wrc~Q1;hk zEot(qUUfN?Ve49bd)f3dbA*z=R5X>FFd3xtfsTZC)?uo;X14mL7gs@rqP1I0dU4Y_ z*m^|L`UCS4lrLVzx#*HNKZ8E6%BW#wl%AC0nbUH#;vmZr>LaNhaPR?6K$=;kGs=76 zCwIItXvVcEYuz_^(_092FBIerJ1N=4D{Ztv(mN<4#uq(*aw51bs}HH{tb&Wj&s=JB z@1PXv#k&;=-4*fIS|#lJr^v+8rTSgaJAH5QOTX>J8pX;5hj$he!{?}?vq2X2UDVIi zHj3h)F&>e8osarc_j$x6rvthN1;nD6>q_n z4>L0)=T_-+pL1wTdAP%*SWj2|Vz7^TbBKP-6_c^QNOMp~W_PIWRNG+fire*7&1bWE zDT&`z571R)>$FHqr1I_P7RtD@Ygu=lZARJ^$@%r2AseWQEd=1)8MEj6?@rXBS0cXm zi2D8jw9zO`Vl~jk`F)7Eq9Z?aIi)atEQaT~S}alOJ_hP}c#RVniMkV{Ms0|R%3hM^ zY`k=APwuBY{#j44@H5PsNP5+iLf~~1)8se)Cxc%_j@Ur0;WxL&Ozhd))pXt#;?zl8 z6Ma#IjXl}Nj_f}%?gRe>U^YiCygks>5Zf0NR!tzW?18|*GO!{3xvdXLkc=ej3Q?7G z8nO*peNqPQ2ub=+zINreu9pdw*!FxTCh+Hq`TF2x{pQft?iE!luakJYRbN8LNIF}knHIkj#M&@J$1^ORLU912F2Hwu;a&m zf@+n2Az6L9bM9rp_k`2bA{?K>&5(1nA39WRGlw##CfgQwzU%0!3>sRpAn7u(mTqrq zsdk54!G#FDN+)UdW5O=gy@W?zOP@l_W!_3sHxF1V7dB)H^`US}LqyEYczy;!A+NC zgNUfYD`w$}0x+0=&Qc@lw&q)i?jCOQewTYCfVROm|LE_sfM?X<-P|W*guDz7*l%NpUp`N>!8^YK5LW<;=FmMldM%eSf zr>E1_zDgiMjAmugtZUJwrPk0lWu5EEaGCv&S#tFL-?_B&sQ8m65vDZ$iv@4J!CdiF z_~dUiJIr$19OZI9@0C*#H}c;2< z{bpX(;$!S{T=1{tu1sxDBhc-Eg0yZmJs3=Zn8Zd8NM@sx^^a?l=83q)VEPT15lc%6<+o{fnoSMtWU)MYu*L4i8`bv(B96 zt?B!GQD9iK^NyWa2n;J`$n-pUL8ihe8XAlL25iVuz@#-Xajbf$r@s>E?;R{-($&%%Y&agB5F6x|-C=L?{nkHA0juT8^TzjiVa9}x z?8nrPOAF}YZ2w}82ConP;9w~Teik!+TA0=F?jw#6^Au$fCE;Go1dhBsCs}xB_|5;6 z#gG411pcdha;h{{{Wg@${#W`-6l~n3^(W`(^nWjMPHA-t#o763a!~mUjwWzT;`-3b{p1B@6rnG5W+V3HXXB(1tlev%Qm6{F-qPNR)6sn=O zW)}dhX}_7s?Xs3&9^dgq*A39tEt40eSDO0O>DONA^fCjt)Y45}qYb$*Wa;qMxoNOo z>gz+u)k>h5t%KXO{36vtgtdpVZe%CYQUt=?Uoe@BKX+Ez>u+BGj_JyWRinMf7f+_O zzp5ubQbo(mak3dNl{x1<+kCOmayG(FMS1B${#%Z_ZfWg&kn%%WBjx;*a{)Q?uT^16 zZW|)!MQD$kJ+(%LtGiCxInx+q(rVxG<2X|yymRPgl${ftI{_f}LQK3X;K;@xwT)8hNFL6@NuBWN&Ce zJYA+G+rXrIXZ1Odl|9PDem+E<$39tkUd@w5WqFNbvPeTzC~=B#$C_uCBgsfUCK+as z?}pbT)RGW>c#so?C}dK!{2*sBscBa@4zSQ&?-5xz5BnC%C&;ok^>gF=mX8r83hDyq zwMyU4454mq$C!b_)U<)d)a06o+urCwC+u9d95kXM78cd^fE6Adn5c18F z@*-`$CKV}?IWTlcANR_ICv}*J_yR`xcDhYtcLo6U;@8d{cqBSL(?tc|?4UG(+#evZ zwq&(J(Mda_eP%GR;Tzeq;n#GMCjF&(0~Jax9{};sQd9SAU^1qSY3sw8lZJ>-h0SSQ zyDlYMqWMDBu|OA=iqpXrKXSmy`i@!fcsgSeA-Px0S>V+F;p;7c+UommZ7R?LMT->+@VOJcyZ_c(%#SWz2}@cbC@CQ*)t(K zdm?-Ne`{Tfq!%>iQH*YYsa!HcP2^9lL?2}T-pFwj%P}nBvbs+gRxS3mr~EPH-ooz3 zx2!fW+0@6a76tv$G-NBP1kg@*wJSt!=5LBEzA0ILC*TWv!e?WE9;KGROsPk9&wFO& z-{^G5fqq%9k8F2X`Xx`&=G+K_*X54dCtLM0e?(Okq(i2SeE~*@&f=#vZ!`v;^?;Kf zDhP=m)z+w%w72M7>+0rxHysgH9WMlI-<{Lkw|I_glL~D&-o*L`TH9IDhj2)m72BY( zZ!P=~YXS%G6lDTw-_ctEGZ>3|l=dqvVhI~vF{^%11Qw@lUn;lp?{)1d<{5HBI&9}d zG%B@nl7pFtqkFF{SAP2pD+in`82XO76Z`b%fL=&=O@(g8xUv5$iIHJm!yyRjgdz~B z#&rp0wX*S;P)Uzrv#~vks(>)#x}>n;`d5+d*c8S|?-&GUe7PxGy_ptctK)OBE7~*} zSf+6ylJRWg-r0>XR17a4v3rWE`}s_+x{WitN= z`{#}Tp~R%5l6s=ydoRN6+Ei`pGn{(iS88~@k&WOfzN|50fq7lxM|g6D-(owX!O2)m zY-0@H;@1{EIunSBp!I;WW-XAQ@n|GJ2%J%Tum@W=JHfq55SUISN(URoA8XU{I&>Mt z5P-hwhn8aN0+J+j8o~!+9K-s2^$3lfhJm(Yit+vlU%qxv2$jthw%-0uu$hf3JrDSm zqTBziq~O}&sbl$bM!DDQUaCD~7GhQ~5^kLWAtxKhMIoKy$w?)-IFX-&0-N^_rPUS> z$RODnf>T>k{%ex_gBr}UNBMPKevVk<3nPzK8v$L#topHqo(zn*O20pm2{mgp5HG|N zu0ienyXkJojPG6sFjg6!S_b)1`>yeY(OYe6kONrF2mtG>mnz4NDEpR0gQ^H&q9c+?7;`?;JB!lBX(Doxrs#4lV`vyMox2*( zaYPn0^-x9!yWhx&o?e~&EDi_#o%hmI;cMg+%TP=NK4?!_$EIaMr6u~{=1xh-+#v0k zBGn7Q&X|;uLubHS)Ds918;|co04&rS!dONXpVj^HXr$Q~dv{2x7o6- z?rh2RWWnbP`>Q49^eFjv1tNuat86X;QR;OMlZq#(#MVbaGR7c!Sy>!C;tt2OMjTMd z0s+1!glrq^Fe$m1YV(M4(;knZFm*Y>qa{R(k740izx@x`W%cPLht_lvA%d^ixhw)4 zz9e$NbJ-cTrNL{a&cCy5P62d&|_#< zcI~+N-d=hDc0E(kI#(tm9b?;zXM{d;QyCmLl{r9xh>Y>dL<*-97D_xs0 zro6q*EQv0iZnfv;=R;h3;kBXS`{D|pGf@Sx)q0sAKTB}vTUbw8N5U7KH^BJG0Z(-b zANp@Nr4Qu|f8TmA$Rsc)-PpSQA|aVmoys(qQdppH8szh2jquW7JBo0IM-*E=4VtLl zc4z5KDxyQEgWgU7`ZFy22Z7*0K8xXJ{Uq}(PfU2gJ9aUzUR|e57VCmkQE=KKl2-oK z%J<|sd{0m`T*?1nn8puowf9-XmwCRQ1wb@(boB}+Kl|T3di9uHqS<3gWEWdI7noR_ z`~#T4kf~i*VX{fDV*yIQK}ZoZOcrFYH1<~3+(RhutogOgXN9~EQWI#_=C{4~`H`os zP8$b^D7fslu9{P$X7pqs#gKr zEMrrV3!5G#yoHJ{#NBRi%NHW`7kr|5-E z(SEdPGF9BRR?jEm$OmuszZZ*1d)EWK%xzRJ9;(#y^MtdH*-c@4EM^)PZ${QPAl)V= zZckDA%!AN=?@ws!|cd>6mp=M39vFKt=p2oScG)f9o( z7{cK0n0(xp+WX+yQHmsbyBQWZmr|amUe-=QdB2z@)jI)+sIx~eUcgx0@z~QTNin)> zjQmq`Toz8y>lUQaxDB}B0uG9S!V)Sa9fLvoIt|osx$Ig=hA9?aspaa`+<}rxIJ?B` z1jE;CZd=xJOn{Tu_Zp=`*4y8UPu2m^VN!v7l}5QXeQtKm(J|m0S+BHaS|RTMW5o)J zvL4G#G%`P{Z;wvVpX1zV4NlGpC~UqdMsXZhs#hx9*RdL$73mkB=~qH;^^GFu?~_lh zmvcs|bp^d85lQH^KQ|rsahLd7J)5Q~Zt7iZB z`#08(O5X`|J_?6(nSAe+>TbKGuR1uPcvLE>bMfMS>vdA9l~WPomAU+j8VNGbLgM9T zDv8D^G^yA#2JvSC<#^ntkRPb`$rHUdR9x_-w1e>CSpPW6shpu4RDy2D3rV8evUZLq z24Am{cx072WruBgG%gf!%M{jC@A)h(>RQI)KP2EV0hoK{U=7vF*yNrH|5)qh z9w;YkO+Mzfv~96H9V?OgIM z?wbDFNr?^tv}v-cs-UZko}*YMe)S`pj@_qi)iGz{0*>5vIu*vaOsuxc%_7}*S#4WK z>aFb@OAOOu6odsG&^=Lz57@Gpxyv5X1-F%pYndpA)tnGh7||xshm}Xnx;Cb{LPWw+ zfy`R|dvVpUWxp^o$GL`gri4ddthYQ-(Tv-%A7FUh@1ML;xWR?Tm8nHM_Uh6pRlS;& zWjWMo*)YMJzz0HJK5|&tuiot3?%xw;eRg=>p^){o_X{v2uuVGb9x6ytDa&GMMmIVfxaW0NR7<(@4 zF!WrsO|Q?KT^vRxFh7-bZPcT_jQZ;Hq8M6S1eiAm%Iko1w4wVv1$i&nD|`DpJ)C~v zlpYA!j z(qWDJJ?wu^pyc1TiaHZRn2CQ&OWTJeo>>!kg0!}4L&u6ErVOLgG`V#60)}vF`xeQ( zY5P}g(}i2XzSoca4ECN-<8K6+y=wzRQuK^RtuKhm^?cK(X_n4&p?%u!bl1cFXCdcr zH!Jox5OBb)>b^Dv02Ec%qnmaiNhDxEM@WMU0g)t@9Ns>C%F5)8?!k>O9u$vE^y2Pg zaGfpQ9_^?jcWS*pZ_|M!I)=4FvV0gA77Q-7&MZcy`O(N`w4KB+xx}?j9p_JXG3{Rm z%qvy&e^bhoc~!o&N(*@=FV9K9NDPV-a$jQt@m28>5q`zEyARHf4xOOsF*%AXR^#O! z36LWKmpamuvNv>~@9^s1w&8#O0NG0^6d8bjVs!Yg%E!wq!{AkG5`wRX^?VISF{)bk zc#dEEp0p3k$i>Rn;he(gt^bp;qn-Ja$@|}|F0}fb1QBV+e=R?R1dOqN%|0{)`!2-6 zpYM!`i15&ac!Ks73?hfyhC4d(dxJ7G&sw(dkNri6R?bP1m!2trxxJ;Af$BNHFe$O% zv)U9}EtO*3uB>?i;(S`^1P*z!PB^>*NRoTmqz(D-(W${%S9&4KhnhPyzqV86JbEo6 z`U&W+_TxnD*jwEMCBFEY(8vncX& zOFp-F2t6ND0gs}6qLFJSQOQP9hJYn_gqHAn?h$2m%5k{G@Dw9X0;xK(VTxTR6@58? zYnu0Eb>VIJv}v6R{r9cYP}*G{E|J7bjzs^Z`vBR25HdDsGPY?f=bX`G~{ zHsMZT^te67H+5~W_5^L-EmLG063uQ*-{56wmKb`q@QF4ZuaIHFcG3V`!f$`5&A0zB ztxJs7D4!Am3Y00~!pc!cD5VN%SFhGtalTzfLGupIj`zeba3rUu6Kot!(`|U+XrPvG zJiI0s)L&x^NRpG9Aexg4tAa&rB}xjc8(c_~f3-H>cys(nYDHyg+z9 zh$2oK@+DkswCV1KdnkPQdH$=SO4Je|85}0{t7#lPlN9*Al+mhXD12E|2}1zZa&$d; zt}0^c>d@&kr&`Z3bpqK%@8Wu+VXzwbvnEW)V-C!snkr;`{Br;XIBPpicPrp>V+IAz zkZeQ|G_4n>+o(!!e~#ga|#+KTu|0#73*>ME=lR5P2SU1$+v;r0HrhyI z$1R#=;)1P(yU|rAab^pPwx#R!ZsOlR!t10|>c5+tXU81^wzZ?P;imCEy+A+Rcvv*% zGzgP5JE!!%UsSyOG^!e@D<&8u>BV3P8tlo$8>+`EB6;k!{(Z%m6QuLXSQ|HOGTowQ zq7Y;P2{#fLmg3<1Qq`evyvu>f(*QRyG0Os=sIDP{dUD&(cMQ`mA;a2m$j~T$89@gX!V#G3SRIC!_4&$d7pDoG}f^ zSj;vneokdkv=mr3FG&B=+@j;&gLNiF>-L*^^dF3GD%l2h9^IIE8|yi@Gq1-p^o<(J zC4&Gnpj6$D(=nIjq`&I)&2imF^-x=h92I?|Iy-O?6=-;zD}JpIOiK^euOY}@9;5M=Pit_8i?q0N-{*-bOEKW+8KOXo1;eX( zu!)nD!!jJXE(E(;(1<^4i<6B^#rG9Yq?5Aj^3x+SxJTl21q{9-u5)lHHh{>5IO@Eh|JB5U$I=eQ+YR2x3^x z$$a7UaPwQzGaV$B|APnJin!RuZixfqxhG;tBz2EVWpF?svv)?(y5Ne;q7h-z%uvHG zHv=)80)|IN*`MP|DVmW-D}lX9i^TxkBSSEBhmkXsMMt=v*|~e=vN{_)K&UJP#yP_D zNH(S3ks#=NQ{rhlYp${P!+rN*HS0~OobxB z7)OvGt1x~~$ks$&yVmL`8ZMzzQex|i)&UH1+U||oxr(0Wq=3SW2AvIkXo2A*Nm4`V zZ4Ra4Adp-f$&^_zhin8PKJDl5Z;D#+>Vmg=zqB!`qBC^{>vTlt1b}fVHTK-d&4hpi@@KMN9m}Yj4901ffORW z{t)B`1XAiBjN~2ulx6^slG1gr-aIYc;<2=tH4mRgQln9!wS4$?9XOM>^2UY=f8l1} zu8KUq^4a3@3CB&!4QeKv%Ja1YzSXX(QkHh?WD>P78vw2L@q_`*)K2B!)I!?M3sVF? z&2jrfI21Z3PEab@i;|A9@9e+ro(LLw%F)~4WD?nV0&&viOVktFJqNDJ9Tg+Hjls%p zT(tGHO!oLrIu@Q2&p|^3+NJf>38{b&t7u9Y=fM4@cFsH23Puft8AXF$Qs-fOK$gER zJ;L`yQqMxYxHn@hZyA+D#`=+jz za^wCJy0_1D=e_Kr7`a9)ID0FEZUq6gPO0JXffCzQn*rx>nasA-xdZmP$>C~ElG0K^ zyT=ucb{ZPy?7QXLzb+FdSd{|PZ3DgXwcHWj>>cl?ngm$Pimh~K90YIQ>msoWRmP56 zz4;J^6R*Uv7zHM^j+L}Zd{^f~e@*=wb04uTo#qQy7qyqnjAP1!PD!`9Jgo z`TE<8zDuw1^u5`nV?X0Al|Dw$H-go z@ac)zCv(2oovq92p@O4EHE9CH-ZO<-H?b=$_M@Nw|G1p_JR7cFpc-Gv~o{$*smq-=SI(&;U(go%l70 zENhwjZ3YIJnF-d~mwY6X?=Ys9!!>;m{93Ob1>+U(cTS|CSV_sdfpS>FHRI zvf-^u?l5@eAYxy0f8a_xdP6;?i(8}%7*vSe$uj8I`Uk_iS^gq3CUZ9E^ilO=gZVC8 zN^7&DDLPcw)h7Q}w|U71==uH0&2+5Rw;9ugo>jaMT7z$~U*9kwh#^~FdDTFCdLz7z za4kZ?Eag?}rQ4ln+Jw*iKmt9(uC$-h4J>+LUm#Ii`U%fZh%0;u-wL91>wniaP%FXy zvmr-B^z1L_Ag7i38cz zo&LelJP@Lr3}}}l<@l2WVb}j#B#Y+g(k1>sHqpcPXBfPHV-)}DUf#b2Hwg~en7u>1 z2Y*!JpPSccVx!(NVJ+PCaq+ORkL-x&A4^E&-zoYw1m#6?(lS zkZX!&8hx}PzoqH_l=EC+{TGdkRm&GjA89%N^Zjr1X)vH8c-1Xhe0D*eDio=vRN2(h zTnElm{NX6tep=ld0xkGS6KbxEjdg3j`)_hv$%Q3;sFcj-RT}P8uFiP^fEiG73@;>JioH zMo0bEGm`utpGv3m$ohQM2Ue`zA7nj3cf4eINUN4^=XZShb2GK9(yY=27%46P|vx}zT`n5mtR@mqh zUfQtY5Xt<3-oRDQGxmB0mV?vbIQ#wb2q92FibT*y$J1do(_(Z{aBx_S76W(v-R@OB zt=*Q?XWd;bEaT82)p{sHGv>5gJ$AcCn2@QEMC6f!Xo>-QW(G&G+h@a6T8QDy+wZz* z93DeCk2rrKZF76O8te-rG}CQfDc;RP z-E(Y)bK_n%Q(w_4w}vdQ4>|i^tdN!JcDE0X?(qLzF-)k2rs@zs(5J#*ckdSFm=#@U0KYc82~+b19N!~Ntpb^myr!4W z$NC1Hzz?r`PW8oDKUqxq1vhY2d2$a2U~%W0SbiKU+k_qu3ng&?ab zxk^WrBR=MEf4l!NP&~QyW)HhNQ<e^6Zu4v^b8R&PIa~sO~cU zGp5z-OMsuKOI4i9+hYLQxSQ;D%u6a3a=Biizo9W z&`%SUt8L^&LzYf4#llRPHIP!sSo73T^13hN2-SKVq)OiU%1XZZQm9X9we3E_R!Fk?N93+Ky))8F0zo?7yU ziS6hnW%?A0r|QYYRj{~t;&9p6uGXmx3>k9{am_3Z)kWheg=bKDSOuQA@0z1}X!I0{ zuhJ%sTlfc`6l$dBStL8r?X=$n?%qTF8m~D=@AP&ri|>&CU?}`*PG&)eb}J?_;goP<>dt7zeCy9&D>o$Xa#2HjF7Oz z3KVh);@@vthY;#Tq=6TXRJf4zviW4(`(hMJ<0ng}45U*UU>ze5d%u_)y$I*qNoIUs z-Zfz~rKfR+MQu>Pv3$Jvq^8F#f3##%)WgnoYw~#6cKEE)HTJZTpGqa!FbwCta#1zf zK4T&=MJW-E?pQz%=yp&gEmX-ZrytsSQc-cylU1v&OGT2deHQ`k6ky1nphx6%Ka^n^x))L-=UeIm$l>ztc5-Z0j_aSHeI zcK-(hL8Ai9JIsde5@#_^$4%|UJdH^$Suz5So&{75)dTaax|RA9KdR4VbPF>nBtUJC zvp~A5MHlNNRyOR#_TR2Uuj~yCp@@=ILD(VrX+n{ZQdWawN+9-1Z|(CVng-+D(#h|U zM>MZtR_yZK1zc%dABa=(Yr-BQ&+JqnOoSmp%*=kW^!s7#+SOzwoe>ab}*n7l<^}2a{^g*?D@*@!b=HN(wKpu_S^Tf z?h+bsZ;}V9yDW(G8`g(gW-nB&?P##T;&3XJ4d+Wlk((% zm@@Pa_>6X#t~w_&nNs}*?Ay<2`B$xvs{%;wi;Mls>=NoW`ZQtK{AAwH6hr*SC~VY% zZ#^4f!NFq0^Ea~2Bz5KA%|$F?3-w8OuwsMGY^z$Dzec*|msG7+uUFGQ^+hNveF5mp zA?dj#HGG_??d7bRzG*UcJ#}qm=V;L>DdxJnmz=?g^47JjJaAg~@OC)tOm8opyaqvc z-CNjr7?*#Dk-F1FOd2KQzs5S?OC?1BHQAllJh3M^A?{Hi6j6uv*yx$-QvjPa=`7_(g^w$J~btk1p-2Tcc&BlQ6$1m9%2!;4(Kh zn&z{0XZVlCimccac@C|vaZvqYfAh;mfXyE^Q3^Kuv}hR@DutJoviHP3q$gs5SCpy5y6ymy1r|qdI=62H|EZ z5s1P=DC00Ig0uw>5|F0D1)OHf#8Wx1%cg!hQ47X-l-fkBPtrkxCEqDCtqbZ0Lm%>sd{QHLgC6KNp;>=#P3j+?SOajpf(cRlJP z+NBdG>%lPeqVRaOSjo zz!LpIj*|2&_tT#AP%>QJMFex$-eLL+1dZJ0kC!|KYJ5jh2qnUVna>76Fm~{F*mLgB z^cQr+x`j-rhV?H}+5;9a77i8x#aX*{!?E&g(rm~9k0krA&c+e1gwMqnv9Nw}G94CG zt)Q*rI@3`-v=G*llvLTwj@sVsb=-l=Z}+M*R%9l>$!pZIrls6*r?Bpe6M;f^Y+g$J z3Jn0iix+&%%x6r;w6SOsn`RrbXS&ZpQ-$=_`+F-&SrF@^_?hP9zivPR_%A^rQ<>mWIHWPrL4G^g1Z8HsEd^ZK)@b`#tJ=9yr;qeRYv%SNw-%$jj54^b2MmAbbossWeO<0;FfO(v z7>Zkt(qBV@xSi^XhS84=+BUbhXhg-@Pq0hA+*x;X8;XX2Jo%ChOTZJ9}2C;J1^ZIkVV{Q=a0EslslPlB}Lg zbp9cWk8$Y3K;f&dXiq0ha8~no_Bx!dC#yQ2^=y;a1xoJC)3F*A{YohYbiyo`lg45c zD~8i;s9s6lc1|BtM0kyObkpvtYANM>U7bdip|w4_^YL`&3?cTbJ8Pm0I6iAoH?yR?+$m`ZXkl>Mr?~-ZzFrCD zI$vH7{I=fC>KZ-ur~mZ@%a9JtVK4Uc+{j2j0n_ZG>h-wzp1aD?+qXd7pCkD%Hlx7K z@9j6mVk2f3-m|Rxe+o7iU-8fMK5m;WAJc1s(98a3IPDcHqFkfvrWV$`@4gAKtUqFL z%qtC`A5E)sTo6jYy|dtNw@&Pz6Uu5ndm2kGtUdCXp=Wx!^trnW#39+T=e}8y`>LBt zd_O0hoHwP-9jem+iE&2csEyGdIxD!XT$>j*8)dBtw4D+xCpp*DPflAa;Zhm)-9{`PCtEBQHKQ7im;{9bu1 zww0h{=siN`qv$gT`lp@ z<*&j2>A%GD;UI>yReg0t;X89(kKSB1g&+o^x_~_#d!T1pz z+%AGXYEB-1+^dVfoiH*O$4|~~emoRl9Xt}ju+vEW(^zv%=ZFEsIGJ79{r9>40mc4T zhyIU8!r~`-H^89j%H&S`E}&fxWBsL**GkgUEoWq-tDEp_EQWRzvtR=u&cwIshP4ZE zKd&DLPyfYtU_6d%SiCxFjch<8>$d9E(FqldQpNhmMKHq5Ogp7}1wC8|y;VNf_wxb4RWZoC8k zoT@h7;ZkM;FAEN@7fYtPhB>bX=~M-ARfRlfX*I;(Z&Y4j;(ue=yqDb~Te}13zD*S# zx!k$7o~J7Fm`}wO8UX3SOCW-*%wQA8e=yFZu@{hSxq6AaK`#OwWVI51S*Un9766CO zCe(8$_=cs6XGZoWyIJOf1M5aJpVlwF?N{GHPzk!fyDzsmVV0`DDZkSTbhxS93}^RJ zNwSi-sSsJOY$}3#{zUeg`gW6|cH`N1TxawUB5VEZU}!mrWdn+4fjDBn4R@1Y(|m7F zjRe1L`|%*?7C$&;0UmjJ^>ztVRg4UjRT zR!qzfHRyYzrZY5l)03I%=l|rwRd&9#CwUTduYGr+G+wVd2$@j|&mHVn=#0S>u$`)r zwrF$?JZdrlqxlPByue9|V2?hTR)i!HTJ{Uo6UC%taBG|-J*JI_y;RT`8{l%DL#=Bl4 zR-|k@b>!Ig55`YZ$QX?^RXeG0QTstvOc|DZp8oR*F{XgMTzn^F^XlmlGd;rxBjc#l zoNVJLdR;N?@|!l$$7dA097sKvQnkF^TUrkyya z2y0BH>GziRMXo&CTcsXFl^tl6J%<2?xmUy}+yizMdOQZ|;d1Z0OI6-mHPzIY8%5WX zV}qk%ZhGnnfW9WC{5sj6Gu<%Q1&zsRE5~DyaM3_>xgn|#I7!E-sS*nZ%z!#s!j&iaFD=W_orqiU{%*{t8 zMirx=Z_L^0JS=vm5*uqW^-eMcHKvjMQ-h}!#*Zme_{rWu^QSg8w~~6eZA~8tZf&bF z)EmF{wyE4JRQDgI?3T*-km>NS#z9><WbqB3_a;yNq4|k|gf|xdz#Nv{f0zA}p9Y{#5Hz|SvuxX|q>p+8 zOdZy;&Lgq1GNMO8)vJ@2?g`F1XFKPW=29SX|6qVo!MgSz)dDW(=0m$sJ(?hu*s8&L zoR3=R=_z-cDtVk_{19oL;jA6KV%`E_#MGG8QZy`+PuOehO248+=^^8%d-it^S>G~> z7(O`h()A}B3`kXZcRIGE_Fett#_#<-)MM<8qRpvOqMs)Ne}u6IGxf9nO|sQ!A^RN6 z&G875LfOQfar6=q5>_Q+dSZ`Z(F=dwIENPTQc9qVd6wC&+cWvmSu7C5E%nH;rkRoa2BY?(SM>qi|Tz z$gKS}ZdYCT(O_(Tg}Aa4mhyR3^mV531USrpx27O(hu*1&a;E$Zq2{|G>(H3Y&{-#Z z4nVD-O-uP!hcIrTr;aIhD)SJN5I1X`HJ=U{mCiZbvmcxxtVz$|$17|6nq`}v103G* z^&|kuMS+wOqI=HW54(s7W4h=25-W)t&R7QD=g^2uS|9 zZfX^Rc{=;Sk=Lnk8`LP5fPaV+_3mM3RE|VqcXR%H9R2vq5wU`rWQy@TLb_O$Pb4+u zZw2hF7c!A`^PT=$K0X>fu`I@rYhP^U9K|b_h5k2!`l2_QGaQ67G#sAwReK>6THx^} zel(WfkxPH7grmFhg=RAVVwIThYF`*_I~rS+;Oz9x^|wR(uDB~@QHEH8X;}}^jy8>F z-!8X85`wfG3XqrjE{ubRX(aa=36vcQU6AXUJW1Lfp&E>DeO)+E`dwHYo-b);EZ z5w%ORHNwvn2dBo2{53n(E&fMqDrn}%!2WOfV&h6Sucn53ZEJ;7V9^Wb=1xzVEK%j^ zGII`Kux>efv(zJ0`8&+sO}#1RbY9MzGOWn`8D5v|xZpSBAb!2FC(k z98Mb;b3Y$KY(eTrCmyp?%PFD>DrZ-KoXKL71G-dtIq}7M%1SY84|gA3txgSpo7%oQ zY9BzM(CR=#`$~;sp1>q=x#Gk9Fn;A(xTlbplZ}(&;CRZ!5Lnu~Y#GWvoAYGL}wv%{xA-I;Tw$^bf{;^jOZR zaYgin26~Chh!frKC;4V)NA8qo@8=D-pZA!N3V(?X%1k+a!(Z0{mTH9DvR@#JZqyRX z+abK+-S~nRv6%nWK3&${h~(DK#5xsR@(+ehoxa-73X9lLSHsY&F7FajQS!u!auYuM z@X-cmv{fnE@17m%zpJ4{+;nQZgtgq7BgQy3tC%wp+cNCe5I#P?GoE3aE0(Q8UhtVw z6V!%LA7g(;K4e()wykXNs;G2i88Xjlo&la5FCDKm3}?O6H=G+{d>3Br`KYY9NVLy$ zrxgh40Z$hGP%z4f9ajr^{KzxYfKV~n;)#5ehxcU)MQv) zg!-vV%<{Eb8D@9=Lk(#upB}+l2bJE2z`Mj2ClZH}^kIMRD{tKKkLiq_uxKaX5ZK^d z@#Lk;FEnfXq(G>HUscReKtBJpU354QQh%tX5Zld&5FL@>`F<-x!#A}1DKV!XmYrag z*2v)otba=?ig~l3o5d0G^;;1o>{BO zi}8#_Oe1%=rlsxwNLj7m@%T!0bPOid2tJwd$W)+0K5+)j=ite9C+?=rZXXzwOsOV| zFxln;BBiJf^eu}p8fv&W3tsc~HMlv^TqRj&m^ zcbbbe??%F;)pt^rwAF))?78jm9o$m7yxkV(m|fRuA7tg=rO>48ZEI+kX1(qA%^!W0 zIq5j9G$O7D%p?_+uJ^j7x*F4VgR9T{qcS(`?Rw7d27hR__dSptaqOn zGNZt(@o-~{w|D|-ybwdVdjp@}=P_2wml|kgfx%Ch0K}G}H-!}PB{4rap2`aVn7f`} zka;Of?e!#=c6;K75DqLV0$2A19Aflgscp{3E%)1}b*{pj4hXta%bPUMQkP9;u8(um zi7DTq1&`5}KVWDG6!Fp!08;oMs}KTut&FbTd_;Zg_$Z~fZDApJ2E$1s1%p-NG5pKj zJ^B^a4*&8D<6*#|g~P0~`+~po#aB!PG@#-9hkJYaeh5wyoGIH?L$ov@7bo&bBEcx( zpWclJ326QI-=@vK6`Oy@z%QRih8Q^Sj?SPTaUEU5$@%1ex;PcW4lRNuFw`*sQ zHw{y88fAXsey<%j;6k&2su*Ss&OH-c5p#mzsC zx;;JIeDwenC2s7-<>cEKG`+mk(C&yXF_j1mYbI-Y5?GM&s4YjWU{L!Y|Fq`5BE?^TH&>mV(58wX zL8@t^bptaaZ$_Bs7K!mjHyzb_Q^~8(q@7+FRdC#6OiMBuQ@Xnv1ZTp-80Vwg8R z!Z_&}iQ#K^B^M9Va<*}-1`b1`phhjh zv`lp*rRs^h+=SrN{BodyTpx}mWnvu{Q^Oj`JT%$k;B>CP&^N~P+>e%3JdX=qxU-UG zdl(y~s$n9U#W~}f6<~H=axPe6ZKW|R(8N*l^OUsxrYE~!znS4DkW?9I=!F^isF&F# zMQ2`e8^5WLYIbfsKB0Hw0x#cAIo{;t0Yp&?@=u7InT z8f+{}fP31GC zA+_J7a&}W&`n~#O+Q(gR(^q?pfB6SGo0f|Z zOWXfucgg$Ez{dC$rCkcL3?v88m5LtFCg2>SLReGszdAlQ&DUO?MgCNy6f&B>+5FwH za$C9Tk~X46f-A-nXSJ8mcB^}5x4Vg2{0D=)$A=&ZQ?*E=%ij?=OSB;>?!x8%*3r95 zbL>?F>jM5m4g7ZXIgPIBN$wO5Tsd~Ve5=RssebnF$yK1>bbZ#q_w^TD=uYbEXF>0Z zD&sOOtZy-YM6Vp&qN#Qyxo2-bGt_egD*NGyHWr`0#R<{oQOqctpZ^|a=%}24d;-E} ze@8r=32?$CThhVo4IM9-Do|P+TFTnl*S-U+E-FMx`V% z3}Q!zD8Enn%C%_SNUri04=gRUePpt}FSHS(DwmcWn!@pq18@sis(IEm)ER_5#v}=2 zii2hXq#A{ghWtjtrjCpw>!fLjqZ}EZ(Yormw4=1y0oJ;Dr^aVszbN^^CHW#ksu@Fz zzWQmRT%jVsRZ`Z5eVHhS3|KWx?2?>b4ZsZ4L087U3?Hi919kdw($jniThA;ps2>$= zmDJM6$?)`x^=uW((e-cNY{UcZ^zN;lQ=CURDHyZW^5>=>*njMEjmR5g8)qRrzNHIB zXq)vrgf=c66%Chf7#JoMtjW?27aPGCX_@Cfpzte=Bt}bGzHLw2*jR_Eu0LY6*@L|C5#NL~E$=q0iI%v7y`4^7gJ7X1pPdGV9GCgZ}dz|&TNRMT&v~9wi;lA45 zo=N#cvGPujyn{}$VFV-Zp{Fnd0FOSxY>>!$$`p#knSAk%V~XFnIC>$OqR_Ef3s6RbkP?NcAeF#d;ZhGlm$c~kI+8T``ISGZPXDHGEYXdn+ue7RJi*(K2Uh()?edfi# zb*7-`qJvLP(HU}CAhc1H4hiE-Gk_SZ@QYHm*Y~VT%Agb`rMYQJdF(v)zpG@7HVT*q?M$HJPUpN4l-2 z)`HXtoeCcV_qO&P7tpRIsg!^e!6ra(yp31Uuc=!h1`2n#vfyS$9jRqzDisc;ec?k3 zoJ)Q1_-XmVsO}72(|Pb+#**7^7>Ko9i{+;gZi0|ekf-2QEy?A3^_LA%aup>8uMt6JUGs+Ps!(M(z@Nnbl6U@mv`!=#XL40DAxSd zi#k7%>I$bh39<-;kB4u_3<~@@?Pq>!5*^$ElX770ezsaD>0a{UJpL}ZyhfjC$%DWq zo#!ThS_d58YPV^PmTrReGbN~{yhR~C4KtMYL>jjiN+y}U7c84eI#*6W@jX#{4RC?; zlgEXGSdS2YTUNUYWmD4n#Ktg2!U^`}+g0Y-%d)dq_oufD$L(SJ$JY*6UOR1eBgC|y z%qu`pSY~9uFQi+czeY|OESJJw#WB^edD#RqEL!F{ET7tPj>~&CH>IYiTKmlLFjV!X zC=`pbSpR#Rc^CHV?|_EG;s1}bw~T7Lf4e=QEwn&^LU9TdcPQ>sAV_d`D6WAbf#RhF zife*9gy2w|pe@C%xCALqu;Na8a{uq=oSA24&8#`Yi>yVG-&%R`%gT3O*WRBU@#oW~ zduS8TUMK#~%_RvJsc~CrLAm69vsT|s>A%#%DN(-6Ly3pItq` znsIG+HO+c}*!D6T8qFl8n) zK=Q({71}7!Jh(+EKvDEyG&_QYkh~}S;h1i)BCBKFJ~r=W*k4O%ig1B4FIH23^^f6 zEzLjSeh!~>FGe8GB;UJ=+??zJFpAm-Qc&)F=6;?0tJx;#lG@9va|i0`+Z@QT$o%R- zQo>ePaYTiq3w_V+hnj+4O=w0erk`t-hqHT69kKR~3M`G~)9&74c8=(Z>?L z-MO?#)@2FVgqLAQ$5F#>R8kU$)C_Dd%Ya+zWa=Rtd*^7w`LcNfylR8WHpKa$W~LoQ z$6y;>MfQBb&c(Vb?<@j{a??~*Hh^9)q?A!6aP+vnWyVZrQ-xAiSbllL1XqkzQFCHu zX?1LO7`a?{-`kh^d9|_7cIkSNsc)Z=?q8ON z2vErvBO_IbY!wFEh|MrHGP5N{(C|Bnxy4ZuGnSHQgd0>Vv3n2-e?hW!Y{Vz7BUSvI zjeifl;@kQD<|6{$Q6xGAaWNFZ<=$WY!+v-;ySo9i@)0HUIVF%AmJ-hg47Xa7BYLdx zuTN>UJYBD{lx0(d&S5yFnJQZH%AklrukTKn_NV^-uJ==tXGV$7y1;0k!3_ph+16L;KQP^ znXx~7hvEDCtfSL%juyl8>sOkl2f|tk^mgw5(2)PuKK@Td`Ty`A!5fU))15rC|9PE9 z>65sKo&B5D^;5oZAJjw3DxYiQtqJjair>{Dlw>zw}@A|F+p?18RdKJaN_ zv8>43T$~B7_s!~m+rfB|1o!IOiC^jXeU9{4C7G>=2}#SF#`;k5zb)(jxqva~KJr{9 zytCvl1{a1$(ury4Da%gBm&N;FihrH0xS);%G(&%@X?AtoZFwDC$BBWFl|{&db$2*k z)s`D<7~P3>=*n_h>`gvB#h1YQr1YZ^6te9BJa6pqU@l)@-P*Z$VXvj;At3YUm%?j? z^{abn#rr><9}IT)>u2W|f*EYuhHva!GvI|;vL&kJV@6M!DXEaQ>?VOKyfj<@TaKQ- z#HPh3VNk5Z6zweG1l-agYL8h1bN+gZ$Cw&hLii~fB{%9aZK8Fd%G3U;gV3?DUh*#~ zvH8hjYU@0-4c>!NuTeRs5<5y8>N!-L+d>2)?P8eFYk=KQdqk|?OPe>4O5kU#TbKCc zPcv2}J@L<{I>>fMGlc!P?i>TZ?u!zAeMVdDYqpfJLL`~Lx zjlTa);a+uhSEe@_Nz8Vq;C)t#FB~mpM^ui`vnbFH6czF@c)8Bbzz6QzY6=T0^L#DE z!;cAk<;S~k`)*O#aU6a}32j%~3i)vRl=1Y8@w7s)cknL;pGw~S?8S^l5qi_u1)Z#| z?Ndrfr_}#sp_GAX8s;gx#WF{ZIC^>(!mrX~dv94f-MU7|Ee~gjd$;J9a`@dsJf_{^ z(B9ULja%6?uB0oyY^(&22eMyehdLDfk(m+;O&&GqqO|8`Q(KXGq!&G)Ez?>E&@#bU(wkf32H0G~pYvm`Z+bX)-`KCUE*kc9_WF z$7eET++dH9(=Br7V_pgK%AuED%qo4xNz6oMgGqHS*<(nfz_?W^Z%WVENW~w3Ihr}a z*+^k9mC(bU8FEQT1jZa=*gP^DRPS3w76ZPYjYNY{+kki4`&WkM?0ZyPiqd)0%ATm! z=w?g;*ih@ z;bV5Arg3P_V*8)|Et-(bh5PBo=dhRZyEPis zjPJ4BX#gTO!(9v}hPt}Cdv$yfewq?+q|xpFT!r^V+|uu3V%2C9(sB8U)k-;wU|V0wmt^6L;{9cZlqWY%#)Y&7>N^81N;U&08Yu1#jPs{>qV zwdMBpRLtl&V$2P*^X`bZbGUXwkykNW+_%MFx$AdktWDf!L)s>arq9EN%yspuwnvjL8o-l?x~*6Jk$JQV8JZ$g zuM~;lOe%?c$y81eS}^v0v-btu1v11@ILb`xC5e+l6c2b^bMXY3{zZ1LnVp+3$=;qBF(PeSKldigx$=LCFhx@No-O1a!wbmoeat&<%n0^tWJihOs(9t zr~IMx{2V1a5i*3P=+^w!sQ%jIkydmQ05>Na_Vau*`T#5&E2TIn6N+{>=twI3r=l8n zI=dylZriz@9+!%o41|VXR;-B&RwxRo^9Kb93l>fJ$Xw}55vK>Koe8=npV zQtETxZn}5f^Dd%`4gO-Z>BM)fnS&kc$ON-@RE8#3L#Hf?aJQH@TkLcM8tHg+Jh!Yg zfc6|#isg1Z1z=tl03j(f_-2fBC5I(nE0Tm|Z^qbz-F*QSyHIf}^nmI*Lf^Xxu8%IiAazPD)6VC z`0a!>eMU+gLkI8BieXR#RwNa&e~Hsb-OKONtdjGtEsKsk(8lkxV=jkrYmFFfV&B-YAJM{&cTfyoSVUy1_?sOQmoA#c1H*P8r@XH29s#dv2kJ){9l%mfum9=S5Wy zj_5KRKXCe~Puax5cp?XsiZmG4ID--}Yw1|Fn_MKqy_1=S{}%(Fe)g5C8CTOH(lUA^ zA|rWtlwrnM-SKtRYMtj!Z71bW<(HB=9*7G-_jxqmUX4bHKj&1;pGiLp!#L*y;%XgzR zieYu@ul>j2$JoMZ*PpJ~rVQ;QxnvUjfW`KY`qHP=3gKNWFbN zY@8;@eSeXY0pc)<`6a>G3p4{JBaox?nGzf)xmMG5*Of#5{13AwTUuD6ZDDOKFolVe z>O8vd2y*=$J*8P=fh3;RSN(tiXP%hB@liXd%I7;tSv!` zUlWiEj9S8P!(5pwEZ0`&O`E;rZ8v8LwT&WA^XqTMa44@H7ctHMZ#gXX$XkVfZ)*I-@Dx3C#owI?^F5A|HBhI8 z94zWCN`qy7JKWTG{aL~4CtsYyp!T`0T!Nv{4b?^;q90mpIv(_u6vt0UVrZQ7Dlqkc0(OTa;= zcnl_s)(&2jmtO{s*9WZOED8K}>=B4BiENNen_9-q-1nYe48CAqmHJV$fTMZLho@{O z>O!)4V>hS@`^V-Lg#+bnUuVm18iWNG)fvh%w7ci467r;;-OQsITJ z%JAJ>`aNFFGO=M%efQdw1<>xoqE)x zqxF89ZwJDi++H)P22Tlfm*j&APRSBoC(;o02CSngw7>t1k0I_gz_m;F)w zIVovBGZ+(x?ECqzhnEhyJL_k>`}d{_I_IaQ^C!ws<0L6iPL0~tcD>ghPEYT@OcSS( zPU9Vuc>FFE)^RC@c&$~ zK$JlOeBvKTW@a}kIUWAcB#35O*H2%qZK@MCs#n&At6|B9BBzF5 z=))WCy7_+Zg>?MFbnm=8@8rbr^SAzw=*F<-=2`uPdE1rC{{Lh5i23oEWL(q^s$Vo& z*F+!;L$3XbHLm92*_n)+;=ib@$8n4uhi;l<_pTv3#F%Wt^h(wujfYv0f4xKL~;PJKNe zBdp4$vE4BHIwxog^ImDiG`^(*okGvHo9xYE+$A}Q?Few67UjHPcmPTX1r}T6=H(*g zczHFkPv0}NjJ;s&@m9)`X-w5kd`LN4ig+c@PpaV~p;5+FNrGm&`4t*8us4xR()i8n zLZ+%3S|XmnfV{F7(Rvf=&NdN8b}JHewO0axL)qg;9Zg>Lt#Tp5yfxk&F$>9+`>?sx z%>^D&*maD_Y_lGW+u7)Pc6|r}Gb(eLErvd~wy2LX)XeBbx9w z(Ct)Zkj5|3Yx4v6(=D7Sv6!7@O~oXbsnpaCLs`;f{wuKjfNyk_4ITj^oHZLZKB63l zNm=cYX#sTk4fm5>{8YK*l=W#_*VToV`nA-Gx3Zf!RLuO_T*v*Hq8bGYt8n^GVdQUG zl@gbo%5B`W9ryN*%VX?*x7;kWnfHM6gQ@})2g$?*aH^!XV~ zN_>3Fe`O&iS)@tYu6kPaVyXRnu>pmp!I8zq&fAH4tnh(7`e*!4j%Nu((6SQUf|A74 zNp4eqwCyN>txv=T=|dHLqs zH`6Fw7()(mX5WXAmv-&vQDWuKwMmysh6*9{BOOz-Xbs#kWb?JcF9OSppQXlp_-Iqu zbHp?V$-?QZz9sgmX~=}PwAQdG+Vqh1R(~v_@U~wc?Cn398y^EsnVLZCB6v;w{H3_j zATNZ5cD;bRxfkLgyH4FepkWcDD+*RECvh(NS=f#zcpnE9mlytYASlCq`MFa@%S)A~ zgT9P|%^Mq!fZ0V4as(CQuesu0$*p~uGpO6P9)veNZ}N9MUm!fHq!s~BXad0A{1g5a zbNcs{wtNms;>)HbhG;;*4(@bWB8LVuUpetyseH=&kHg`Z;x@|H-j#4wp0LmM**Zya zAHb?|)qP?I?|c+3$l`hynj1Of;(}e*APK)U$J}S7Isp3s|2&t9S-!oAB`JpCn|R5V zgZ8NbKR7Xz8C1#5MEk2a#adMO=~NTfA5uL_sig(>gzDX)w=*}(DZ^P)6=!tIVhfys zo`M{Uf-zj~E=yT>mtAD@6z{O&*ME(B}4vVXrH%bm_GK^(zo{264^*D zBmwju`HA#M>0SKNPKa`QqqDF$(u%Mt%E(M|d782-#Dt-8y3(IzMEik!B{NguD9*o9)7&GE&RLS#&Tzb|J;v7L=sG6*Q8<@XdIj%e?Y5=UH= z`$N#fhl--PEpu;6v{WjTqr|1F72-}&Bfx=rNs6Wtx_`fyvdxB zS4yW(tEA%(%8f>g$73h=7oABit$tKYSO z!?sCd>O6(~%HYUbx2!{ax4Opuq7dcQC3H6%19Q^yU*;wWt~zr+sJeW!=b*&lXZ$DJ z`!-5zqDYm734Q>!dv5z3(^XHu#wpeuM`kde7KxBj784NMIBrQxP0LK}MoSTpX6 z)0*!XAuw3`rG#JxwG%`I`xWNVSkmO?diOnrksM0UdyfQ@-B$hNX%3*+Svmv^Eu7N& zSKFGc84lOg9i)}?mx(UZmINlL}F!Hy${)n;6+&GYAY*bA9t(?3n9qg;& zU$}ryF{MI$O#rrH^ET(c%*LIaw)^2CiWfWL`ZF`=TT~2z(q@HQvisoi5Cl~8kNj5S zuFm2Ch(|n_q|=&~!Dr@Z?xv!n2MJ^gsGhJ&Gm}4mfB%Ix{T{y5;`zaKBC4v*)Q%nL zBk40QVXxlK{w>SFR!QK-a;QTJTw?V7LHZEvl{DX6cYG1(c||Co%F3YNA<>KkUC<@1<; z9$iWCY5Jm8?{O~^(XVCalXl7>O{Z3G`%x;nXT|y&P z3#V=d!6_M58qz}VFTu&0Ca)Ttbs#tJ(qOjTDl#gr1N8JU#}_~3vK(c3a5_w~7fvS` zU;)OC(mt7-_MJ)h518gj*_n?5(Z8(w{6Ic9v7yV6v|wZw4fcSS!#EvPoe0YYAH^tU z2&?T0u8I`#vJVY2>AZp&n;#ajtrV%~Kuob;-dKaV#}zKm&(D2;1JI<8rx78t4{3Y2 zCgS>s3JBK*`u+{2F>MZR-SRM)8-IVOMaic9FBUmncPh*Z^(T=2aOUr+%X531Z5wLC zyxi6< zyy6%}Z+d4FZLWC7s%y+$wh>d+P-I15iIUr>XFg9c_cT%MYX~ys&xEH)WB8iJO2(>Rg))`V7yLHvC67b*^M(E0H&azi^2H(@|)$^j&Aq4shc~iA)cwLr09CADXKPN{2RkHo)H43gas*(a%OveM&^cP!9zZuJtdD~6E_jhR=> zluntNQT}ddivUqG*R%W_as1;Md;cy_X;g4a@B}FAys(8r-6pwt`rlVQ=sGC@x*BW= zD$kFird?Cg&|M?59E6*TPFcjE#BgrnePf<4+U51yBgjnrD6@LF#FX)g2W#jO!x9b%-$T;9-xdZMfNj zeprgq$24+|+RkWIy-D~6^d1oUwQ=0lS`Wqvr-5~~)gDUp%N0K42|xH4DCGpWHpx5k zYswPC8elyvw{EIIQLmtnc%*%$l}Pr=C2cln)xgn&3wt`L7W)=mqLy1eo13(uZ^l3A zJ~2xQXl|hf)qZ{b2zy5}%~iD!vSe$ABv|L6Rx&E^WR8)X@i9kKLnQk1b7*%Z2X)r# ziL+r`Wwm-#;yPIvZWX2}e}27e#r|P5<&f=I{Ux_`W9)ITZ^5#8%9LGt*iGU^&1-Ll zAlf*Jsu<256=R?>ImSzCU9T?Bf}q2l*h0tIh4%IB_j8VY?RG-6M@Yr7WnKx^EyoeT zC)-OAe=*{evN#x>O$_cq%kTlMTH|2q%nT0~5M~zzGe4CZ%4v4&I!4JL_L#r>cQe8F zuw(wH=U!)AxkL8GX_p#R5CvQHmrpub-UxhNAz~0vNrLGsLlgmz=d){OZb7ml}ZOd0Q&A$fVz zPu6ox>cV)NB3O=PEY^jTDmIM z0Gq0XedHr?9)i1Mav6$C$7x?I1?N-zvw>!UWLEC{cuXsMEZu|A|B(Xv&n%*wkE3gD z;jvVmOer=)@yz@?3UqMz-xRN6m6MzZk^qGEXA?7(?)lYn^}GaCAj%tvSQqkE$K{%s4BeKWehwukv| z!uT=n|M`WAi8*&zQr*G7Z+Vmw3~1SmyF1`M|BJyI^`F8mERI&1-nc&jhc14Pug+E` zfo?qDbi4%4&4Y>Tt#o7SIF9l?y|DH(WmPQ((qc%!^c7@UxmUR@u3+xxS((c7f?g+; z>d}QFFD@jTP1LxVhUn4AhT=Sc1354 zET~%f(japDvxJm>(1Snv$H+@NwdE)1>jXn_L!lWYi=>H%&OZ2V z)fIufeATif?LXx1cHVPp(q_8>pRWuNyi~XXTKWzl*0|ZCcaqZ=qsdGQw$fEXWnpiN zf2bdgu#J9Q=a`*6oqU|oQf;t5WEvw;=0J&UoXTDrp>>epOK|pz?>0H8c*UesN7G$% z9JB?Kl0+%d?9}akKY0%awBc8xlhhH+zxCTf@;)|cco_IyD}CGu+Rk*X>V-Yi*fWX5 zQN~EtD|&}P-pYizbTep97i|_$$VrcDocuytBJxXOULz;pl)zKDGjlSoZ6?8Gep4s8 zS4t-(qCN329L)$6*cqRzpnB90{TsAS@ounzaSaVBths6^en3VIEf7|8eB9hX zLYd*664bc8#LOhmqm>xKXQKI&vY$uxpEd0Zt0-h1Y8=d$t2Vc2iQsChU@V2Dc*^N@ zT$1;-pD(CWdE?dRd>F^P5aNDYA7GZ@HL@%KbKx*pHe1SqsN&wwiYY$%09> zOZb(p>UG5BiKjck_3ci)3&exQv%|K=S>qY*&F|{{o0H88{-97ue98I2z0^w^*L-}x zqf$mjzPepD4^V5GF?`B1pt4Xc+J9$j@FLT?M+k+5au9HKHuAAH(g=9U{b5jFT3|}R zVyoR74R{1|-n1BHsPc9sIbFEH*qe=W6nw8NCENR_^c@ za7*mLwvLSyG4Q!p6UN*GWmOJrNaU`NiDIZ}o1%uRalmhMYWSXO&RY>rU0C)#yQ2Y& z&6!zZDXHsm(;$Xxk~?DzxCvNPqkSd+!)6B1;Bdz#_a;m%{?45mD>u1GDhI(Z6De2! z7sIre=P$;QpWaxa(tD$j8@8|Em-9@At$o^SD(cjLf~Pm|niCT)u7_D86NA`*Ov4hL zDD<)Fe7~AXx1t7Ggvqa?9#`qJLYyWuH)-S97n&w-*$rbg5~Z7zcRz(L)=mE%*Y zlfIKqR&)+qg05|qi_OO{jd-EE9$CDnNGJDv%ql|H27@sH*JSx&POA~0y*i1@XxcnD zf6#{7;GQpuZ8~TFLu7&XnmYg`_PvW`60 zooGbEn;L{|?Vv)2@WSj5Jt$w>-u-0pucm{fm-SfeC236kp08#tbYgVhAxa>dhI+|q zME3@a?Z~PcQ+FlDQqi9taGngfNKMKai3NWW>O_c|zfW5R??9?&BKYH-tL?3D;DB*N z3x}AshB+GPQG1+13(z1Q6|Tp%DYLiZjv^)*O6*qgbfv)2(#DivC65P@^rB{}4rp@% z?}y*@OpfL_0m)Jdv-th3;M(pQErC8kyMwKp)eT?`&9cAeBJcVCETc&Nq~CdhiSra;&Dii9 z&rY?mPgN{$i74HQloU;q4@^w73tgcjLsP4SgFz*f>G>NctnfZNlNqjFd|G3kd7Eb& zBU_b054X~~qtpT7-ac>N*2l4{AhIUL=BhZTxHitTl6(jW+4PkU#r4XVcg#H8#n0kN z5qD#iEzM@ek9C$jR868IucL4#!;0~QotX;Zc2oTXsDym8m2?3)kl9h6dd! z1K?6xh26t1>IFPr504xU1#j_o5ywFddgV;Y4pI}>%XQE25G%=t7BDSmoAU+latSpl zK+yz=Cr)u(v=`4vJ(=Yr#J)%T$kV}A{kN0u(`R?v28|-yi@LKh_R(%#;S(Vo`y;ke z^A8E;ctaLQ@Ja#r=^|FtO_QR^2#>ZWRM-qo-(94jfUBl;K@(3#pl4{ z7S^iUPhRqsT+iO3ojI&Gy904%N%(%pqVIXAnIxQM1oA`|W~Rgx`SSw(CMx40=L(7; z(Fr2>Q4j4Q^@auDRqHE(F~Hyp{%t3ehV!ug2i;Pw`2=AwN4>!p=w3sVy9mt!|7+=G z5F&cx^LndR$<(rHnGQ*jowELjps6sr)AsWBcfPnY&OW^P%o#Pi`ZydFDXC^pnEF0& zUn@R(@AQ11I1J5t;Jtq@u9I=rN>#HRQ+*>qZ(x2grHQYyItao}I4C1^tH8Cj^Kf!lfwS0q#vCSBtvV1nh@J`-kSXhmDJ3R8xe@K`Yz0w#RoA$G z-}!y}o1vN)^uc$5hAHBUia&8UW%Juxjkt56!*FJ|KT!6MXUmys?P&S zm+$SvTn`Vz8VRU)4H1p$*(OPkl9Wq$|4b!YkNLlsr5n*(2Bd%cp7qiPi1o?4MJ5tq zLR}vG_^96ED^Xurpc@{=s?EH_h_)a_8xylUYX<==m(d4_MqVa z5gB{P5aS*6%X_@oaGBJBzc^?p#xSS}S3Kl+ca$tBsXN{y)8FMd)+Rh{ z!>->aH;r}Zt+~JzbAM0RbF&88Ol636S-8>VPrurZ3;eM=5UB6Y%rwi6_2^Dy;pHLq z%Xk%-aSfPLVuxTwB<|qSdr0eadMNHGpAuwSaco3HbM&jV(p84{&2#zk?);r-F8wn6 zwhT{emqstr+H-cHk$RIcN|G5TlBFHp%qY<&7G8WUh)l&_R0R=(pePyva zu((R^J`nt{o1>wDHXn6_gx%D9?{*F2by%RZXyuO)i>#xAcSigu1?pvbUspFRrIU)= z=1<9iX{$8+xeXGu?a%^(!&N-Yy5-cwq`XXA0z~P#^GtTFw5Ho9nkD=_ddY6NoDq<^ zBQq-AVS>g}c}R(OfDO^`X~kjN&cxTLT_|FFG=gI3k;X~@fv<$6tEm(_d#0$@GgUC0 zuT6Y{%ZO=@znS3uAvJu;EkY&p+*^pC8$wQ2oWritR7mn78NcmRA}Q!#VJC|%F~;Le z+Le5i_LI(m;&{ZE0Kmb{U860hqRoP;m!%`<`$0-tI$!z%1otm+ z1Ph?5B@(h=GTPrC^6wPwcY7;}>UAPISI7q%)6YGh0#Umkrw9}@k{30QPmrJVo{G#H z9dM)K26@!Hz+?k4DI0N;^iFKX@k4qk0R`%KruqZd zDJS^d*nQn@Iepwnt%JqD=P?a!+)WOh)Oe1CK?TA>w0@O#w4fw^if*yA;vw6XmR(Z~ zVvXdFCfck?Uhlay2SVTRcvMSkxu~3TMg0~#)%i{1{P{9Gdz8Lj?c%Jhb95F&ar=B^ zCdqPwv|pP<6vmOq%;}$t;+TfPI^McMi47W+bUCyazBfGjT-`1%Xoq`XZKyORUc{!@ zL!C8@`nNMTLgaWCvmfvDn_!Tc}cHYLTYlk@KfWS$oyb z%QWwE=!v!=1z67d$*C~M)>%0i#;Svdy9e>V8te24NsT5d3G zhW;a7{|kHdUy4{yQZ*qFTk5}EUm%kETZT6uEv^5F<33lcxe%63u#G{BU>F+r^nVe~ zI2>Xp($<*&DX{K-^B+bRwp1y)@8I=w+5hHf|B-Y;_GbeV`x2QqmoBPj@Je+Ai;kR( zWh1EVil$PO`a$wUx8NZcgOmKXdkNz|IIt({D(XgKe3&Bz&F%tSuTAfR_*8Q0N&zOSo-w&>g=80vC=A~mZ{b!JIiWTbf z0$b{TCAk0V(><9#Y?k!@#rPjaHU$n&ZOc~~_XG7~T7_R842Ne7ACo1JE|1ZzTSuUe z(fZUH0WHey08yNXe~4KaSeF!-*(}zs?(U|Rjp%7?Q@3KXqIk=~0+BGa`v5sc%-V2}YoRz! zADCV5exw>KMLKZoE5SEFkK(S9oWCDv@JFcyPNi8aE>@{KE+t-ZSr@lYl}AaIYw;JE zs}&w7N!Oj4s4+fi%d{5ZPg5+>`qfvW^IATXfYYBc94n%*-_fje*DkD}0a{w-_9Cmw zS!cRXPi2@H$Klvm^p%9)Ih~ZGRJ(MO^Xd7*)>^$yeB!D9+uZA1WVC4Fp?X9GwNcEc zr*40Wx`wkgYA$}SzD%7^n*@d<^2MWboHLZIG^ZUDBZd?U%LuW5ztjAlEL5}kQAL90 zFd9w6Z@i|Y%qZM$=jh z;^T?{z4&uhKv--V-^NvgOe2;<=?RD44e8UANAenVg^f(?+XfrX$1^Y&Q&&Vurs-D- zk95_bR9lK?9u2QtYP@57OY@ORu76SoDjDn67D|^gdtn4ejO zm5eHR!|D~Qy5<$nkR!_PPBTWi&6gc3Om%8Zm{?D1Cv(v(ZT&y&sA=ad-4ZuFmdKG_ z(=+A)Rj!Y?D@pWM#FIo-0WBNV+-w?+pBi&Us}-U$O}N$&lhC`WHTsH4JyUW6p)-rE z2!Uk+J9?{~dnhe%P1^~j+Mw1(GoWQKDJfj8YvDamvHkSLBXEa7I_$1~{I!O4I9a$7 zr$^~6>MZ{Wjh?#ac39kTXaYlk?SF@q=uO{fZ4 z=Z43;*<%+s9(Pb-iCCjyJ*r{Y`#LUYpJlqZQPFvSr`FtWpFKnBIVzB}`CeH{e?MBz z2kXt|mUO5DMUs0skBYvMvM^$@4eyahqQ8?#!g~`LLYR2Q-ef}s+Ak{6gpak#GCh`A zVxBs@DgrKC@d=!&x#vn4aB+=IU@k0-myCfQSh3z&f@8Mz~^09N54Z4pP zrag}i*XS4iVxW91E6N1;M~&$>Ie&oI{6Iri?}(GbnYo9oR2Z#wK6|_`u(2+^@|iHr zvX*%F!ByhUCuU?}c0ax~Ub7#6@lW_quLl}>cnVcBB?di1$v3}ah()hJVYy&437}De zZ_t_PWLSpN?@*!|nd9YnoR{j~zul;mPFv9PI~Hc*yx{y5d#P33pIaSQ zs}mNng{uO3Y@KNSA?^rv7Hxlg)D@R9h3~UG?&L}-?4B?ZEMoQ-;~4HL{9%>5ql3ZC zL@zaY*=~S`ZK27zVen)Y{|L&xJF)d$8JBVW3dZE4{i=Um&1gw0HV7opU`H{@Q#L9H zg1u=z?iB969;QS)JwvRr2fV)2rOA2m=c%Qbpdx@3?3*+y@Adh6Ir3kD4Cz%YJ4l%W z*{J6&gIZC_y1@IR{z-nitQ*;HJPpflb`yGZ65LKW#p?%?AUg~T-i&4{YZdx zO_nOox^hOoZ@gJv=}YVAe39{V7&+LL%T`Q$1$0sWs27r$PmDqFeGLKQC5$zs`VT}Fye$76V26VnzDs5v25n_M*bp+JM?I6 zJPNFd?mgyoPh4KgSe$bDVADqxA&aU~bagAzUY9>1L9E+rcRor8lT6`uIbJf1XoS7F zR}jw6uZuaw=rzu4H))H?^wKwnEqW;GHJU2)}Z| zkF16DzyTG)$))3R8P5PAy+Ln-1syCS%9Ck-wl(ajz77*^w|&}!vIH6$*X%F7acF#a zODal3@6TpJ)^rM>u_z3f* zKF-`(=f0I+MIeTl(J&{8rX?Zl^2QQ~R-vp8=$)Ism3Q*9$LlLsWRKm=jOhAua%~P+ zn1iQ|^N3=n_#*KXQolKyf!SSwPR`D%b}Bk5c9)PeDjl?4sALxU0%J8)VIY3Od~11m zE!PP(Gt>6W*sb$}wTw~7y^!%O*t2X7_eoti53@#`{%oqLod!!4P>-qVHx53mU!@=s zu9vu9jP|$rn#?r)EU!}RYBNpuUo;!?N%)LLC?RP2VqvFNTcdMI)(-$F37gXzKEymq zcdcS+d~pXtF-BM4wV0d7T->=t05KF0bIj)wZO{8 zg&^hKl?Bi&bs(PRAR@^@=?*Z+8jIKp}MFRnSbRnKP&KWB6p zGCm#mKn7MPWMg%Si0PMv3#8R$p?FwEyNtolbX{C z(1mH_CFS=Q^*?Q>#M;Znc1mv#kRx9dJy~@GtfTi*ieXv(;|@|q+XhMlH|$i-g!R(-^D=iq zVr+@mKAUAcB#?&58|{;2CuedUU}z_0ny+PCnaDO#_SzUS2)}prXxSm*ku>cWFx05R zkP}nO3002!S1gg++0MjM|n8n8O2_RS`oa?j2Fnb+Znn=5vnc1%0B&z=gr5 z%RC3SlN|njWnhedQL>u|PBTYicQZV|h$wQ#aDP0(Y2loDlrFh3#MUD9#<;~bUI)eb zu1RY8u}1*>v&}^GHOS7vF1xyBR-!NQz6|f=CU-fsJHb z|BK<+ku{IU#B}N^jJ-+iU8b^Y3BLM1_L9r@+@I%*?Y;{yBeJV=0e@<6S=!t=PW=mm zs1OJ7D`!C{%U+JWX$-{a2qinikwlNudEI_%aEeMUo?3-0nQ)Wgr9DSF;n;>hoZ&dL zTBz>78(^NvAwHkkqGP#tE4g}nV^{;7?8^LHR55p219&u<8-esj7C8mEmI8#ySO(^a zI-*uVgBVA?JEX}+&GDfpU!>Z8amwh#nk~#(w64ZQ1g=PtpbV8v-o__*itCuN|DpKW zlru0X*~DlAwwOnjX3}JOshSe^=Oo8{{=8LU!JHyoYJGu6U+xP5pM@0pJ!R@G`eA>N z&&Dh+t4t=Db09&?JHOZ;)U4#guIVaiCGF58L%(I9EnG}%GVQE&3Y{M}SWBmHN)%*g zf6dG+VS;>=)b-jdAq;A5%X9jZs^Ste~X$?`W?i#;I_m=CguKG3GxNQj)$Y~ z7n7M8b>0>^*=s*!Qqu`X`E`cxiS#h`gyNQs1DOA+-3DiYS5qo*jD`i|=%>hiKqAWx zY85$sG)qQJ`)ifrs+>FK2l?2`<+i3E7go=kLXy5B^E3!{`F3a8$fstr$nTSo6rc0b z;*aV>pn2*Syr+yQrCnMe%Jcbk9|+uCc(28`J!kmJp`p(9bh;=OSD1GB!|%kVQnAG@ zNVXb|lV^dt>vixJvjI4aycoZjcWaAJcpkz8%)V4b`-^ANBYXXI75mkgI%FcU3Z?VD}&uH z`88BMF^w2Q_3PX*FuLQL{HQq)SWN<{SOdJ#S>te z)Nyz!>5jxawCR7v+eG0;;)MJhRC6j)d$D>ViZhFHM#r#CgCFH*{q|^VSxv~h>GrcQ zunCrkRFs!wB7A#$b}xBMgb@}$ck`#}Bh=1jiRq&# zZI+R%-)XmkV6hLK`CkmlCviZD@wOHnE#4*n;I1*uf2y}$|4i9w{vUL`Wl&p-7q=U% z*a3=Lp*RF>Deevl#oeVi1b2txP9RW8{UY7>mbi)bHNxpNagp^L`4#&{$ zFFwE|u*fz(fsxhJ2hFrG3mqMPQS#^!Mcy(PeqH^f;2(7SVM*E{-v=T+$&uC{>?NSE z;hqPpQcDw60_DocNo!mtg{IafiBW9|*P5WO5r9a));rzXWwp##J zXn1M9XkqHYOyShGCW$7*B)Ub|ZraV>q+yy=Zfx4ga6Xd8;m(!rv6`7exg8 zf)Sd2uQ1E6Psd$8!r81@jCD4VuBc`F&%C13dEhd;VNK&DTm(dGe48lI?rl+M(JO_G}lJM*~?$N zU#or^p%m5JT2Vk$Zj0Gpdk?barLJASUZxzdavK`Z;q$5YT-;0y|EyFe=2AN0(&yJb zoLq>sut$hIAqu5Jr-j(f81#-2K~}dwrJGqDGQ&*e?uo{H>%+#^H{*a4fOG4 zyLST)+FAgnYYt$XaJkeeoNVxBMQ>RT66we1k#?!)PW`w6uDmMa4Kc zsBlqB<5#W!u-O_tYlOm9=(U;vl_qAJeD0;6Z2Icv(Ta4oY z*w@l*07vcNh=fa1Nb_ryh4bv*75)2!A_Um?8`y^n|oweqfK$wYpykTRU%G)xS$j3xC=FdbsmSs$&qv8RAu+^ zT^gw{p?!{1dqLWG1(Di1;^U60xS_n{6AyRPe{^iiz}p{qWY1OX_-Q<{iH!^Ih`suq zTERb6P?J(T36(vOh5jPh&|eL2WQqpsweAl4(!PgWENoUzX_IyZX(_P7+%Fn-xw;^2-xC@a-J37 zn~#QHRd3TtXh?6;7LNo!kl?WA<$63XGa+)z|5Y*)H7Z9^kk&&1ar0`E6oKmQD=d!1)>=iq8 zwsC!t5lek##qwR*@CxTDQ13(c$fOQcq<$pr&_C}CSfw_b#Q(K@nl>UvA`=k{wWKMrylK`etXA&wuNJeK#GBpQc)X zVsr!*5gfo6>*%gqQasulS(LO?2+wxWF;mZ zmlb4Xo0ZYCP!xWB_4vpE+9e&`EFVLTCGQis5!R#E~}s)vp7y7@w}l)D|j@@Lm3EyFf?W;75=fT+Eb;8rxn(E zHBmYhYyDW|q~A@pqVbmU1v)V?lm7}kwEaKS8e*DPExag6Ievq=;`5Yf1W8g?FG$`h;RlhZHuJZ<%3~Hd8zLR3eILe~n9Yq@=)_*@ z;`tY#JZWevZy-Ey$>Q}rTcKR6UZFzUyt+ucfbO+blG@8MqV&`W%fedDBg37{wfqz9@xhHs4YwmVZm6VUIIIhg zMjSX#>@w6AI)}R4pi-wfeVWsjn0eDBv(~CbmF;$1(R;irmS5vm)XwpZX=22)^b*t* zj#G{A(hag~U_f@n&=_iX;gkdJ+IR)UR|Z*;tz)WGhc63>v%D2UH|*==k>w)CO@Nrg zR6zEemJ#GMEa<;cVV~Pmzl95IO$0?>QC?Q}@lGip`QkceM8nRyT}3QePJ9ffejjimcH0Nn!Dv~`$fUC zt5)PqLB76R$$zTDMsQVd>A%HKlbX!XP>$)(-m3}n zjW;v~yl>NO-=|_CRay{sGLOxSms<8Z;eV4Pv2>kL^U_gr*YhKB)QZa01DMNyyk{Y2 z^3A=3g+=VRrFxUUanDtDAcO9fPjzKQ^Ut@_4_iik+wW@ycg~q39z}$QUC5U?-N#tY zj=EMDV74f(N)dxCOkE*xew5f{qM5Rf_cPKCE_vz`PF0ZGe;(6XT~kx_!p_~3Vh_Xt zE1?bvTpcNH>kbAbtfj7dJ(%%%t9_Y*-NM$?)}7?>@xuWE8ZoxW?}6gOuUAiE@$g5m zp<&snHBYE0CG*LUAp{qrA;1z_rM_#f5Wcu3M#4Qy>t;eD(pJX7jP!OPa3(!NP)+6#)6ui@L?Zu#bEN}bD>V-Mw#wndm>8{5D_<>p2w*zeKOv1l2@kTX;!9L0B-M6K%spQYWSH(=w%WTc=15 zj<)C1x{AhFF3~K8eJkVYqQ^Fn?|?oQnE)+3{3r6|0$F*YN_r9DUtYMSq&gze~d=NTnfcXBWQwo0i|l5`+_yVN^_u@s`YKa*0d3rg1H~T#H6(M#Wy0N{84+nE?4;GPm7CMzvkX->ED0p)k`TnQj#vUEN~=I z=Xqk4Izla!N8b7dz81v(}*uyVwm)@th?pKS=>mot^ zmFdXUQVsar8N}(gv5&=(X#rE|I9gu}6Tr)>6<;EhVq%2L8J(pI#H*HQ)V%_g(s_v9 z>UxAHMNXQ-8kJ^dA_jjQDo$I{+Ngstb-L>t3--AjFGJ4#3#iEXw*-M6~HACaciV`Riad zA6Ih}3KZ$)WG-avKn&3ea`FlQY$pW4A_lCatNfngD@>apPG*aKidz`XRzWxmmE>Ql znwW(kPa^Hr@;nmOK=kcx)Kv2Fx%fKLu_e_L_8^wz@iiF|%kXnQ5)HQ7uKyXYy|PlGRNwxe%Sg+ELAcM?9IQO|3OU=J!MH}+gK z*Pue2PJcU~!(h?3M}qwmcYWCr~xG*~K<31Q4CI=I985E&w!`HTcaHOy_gUPjul7mjM0 z7?esW>J)hOuQ?lu;Y@sDKCz_>j8J5o%2xRqdY(RazCkZ=38P2pI2A*FM;%T7>-jVY z7W{a79DJ6rm~KJa$P|!n%=Qc2+z%aq-y?V*jv#)Uks1-l;R_G#DYH{y#|h|35yE zXjR7LQ^4u(J43wpV>PV*|QLkuVh(7 z@Uu)UF=Vb#ip3^9N}*XywFEGh)B^S&;YiLN1ci}NNc_mX2Ow;weZ(UKYoW_%L;cYQ zwS%|>>+)enO8I#`;1M(?U-j#i4}LO8Mkb{ zSAuEELTCX4Hf2Mw)V)-F`d?Rj@D;4klaZDh{Vu2*z;$F7{pe};foa9)V|;7|KJ1L1 zJ3gG!vWl?Fp6}D>)E=;>MEOvhVtCaU&Q8o|L5c!_t@VMi)wCK)dyL=3c-vIwcx8GJ zPYQqR@9vw$RLOhG9Wkh2_VC?eaCVEO#ICh-MFKSI$IcQTe(eO8Wd_l*HBkJkoX5%t zXJi}#uQvBKC1c|{$P#lf@&QhrWxHO~othC};*kQvvz*;2k^?130Y?kZhHQ-g&~gg` z=p$P$_~w>G09%V&36jj>woQgx@}QstymNq5)6zUu4Xivnuv%T8T@%fXUoH}e*5)Sd zvtFg}zNb{@sY9lqm9?RgOzWu{@y<7u2y`fM)8w*f2mV5q9Eedbdvo8aGhi~sc@nn@ zoteBHdzcf=UaltLS*Z+9DXI3^aZtY5+QDNx{=ExiZ8|uuL*TAfnbc~9@2J9m=xfv( zI2xwO2WqSs0vwfhZ`ZbW`1fbr za}Df|%C;7#+38jB4n24$gump_tdMBfJ5|4E^lwn=8S987@hyS1AYmNJpFcj&;Ne;c z4fwjpuCGpa%ulaS#`?s%`soj3Zt=DXb#QoM*?pVJAOu1LU4rjm)9#<0yPEhgc08FL z7tT!;D3OK)u*A*&0%i-G1*Z!_{sQK7W!;57-RhWdqHl;X*RrbPowXbn z9FHZU1Vh|@SBsb#^KO4Q!#T3uLGkn%jBImn;5L}tX zcc;-u*hfpBM+3iDDij3_v}!Cye;RE52cUHSY6Cz1=Uev8C$^^h4{wv#X1@G+nY=iT z@4P6?ZVZa^Ij819GysS-38$|c-V>RS$?5&DMF(%h;tyf;twRWgq; zmTRkxUJQ}*2+(N6?vym72jKlESzZ3k!&b`*8HxSWy>U8 zW1A6?!u?r2gE*MLuvp{UOo4J?lgnQ~^FYjZOeuwdwh|zMgr>eyul{16)7>0U15)L9 zYtvtD)Zr2lb9G?SE+m35;UO^Ra3RS9*;#Y`vPCvV%coy!UVj>_SLtV0JOiY;nv+(( zoXnWZYuX<5d;d{9|2+^({fU4==&cnt*bw}F_)_fyj^HI|;yL;2ZRK>Pj!dz{V-+;V6pJkF!Z8Y!HDb(klv`l%GUAuFBOTodlb2Qd@8wD~9hEi4 zApzqt*KpAwSktAxrb;WSDAJ*mbgSMpdpaT8JY03#`sv&-uuj5{YZu&MR@9Js5sziD z(q-sz)Fx~964F34?6z>&8X6Gq$aDe*qwiF+QfC~8R_y@Ke)PDJcs_8 zU#Y$?K8PNzZoNP2uMYTf@fV=`Aej2WNr8>$&3cv%Q7f^Vh^jrT$#o1I47ao`O#W%B(|!TQ~-DAB>Ed(j9QS|Or0 zuS>`Bz%Cnvrz-jqTwn;@hXj@gaJ@jPmCbKW^V=uRwjW$8VxVWb?5XjPp^*>kcA|vy zw>rjD(E7lP0Qh#Hb=|FQUI{HIsIX=g<(O+09B=Br)>ls<}B*E?r3@lJ2U}GjWF%Vln zSSq|+sq80^H4K}~Q)T#3{(!KpHFXL;)rMCUvcYpH*bmc4Jp^HEU^=Hd<`*BB^vb1P zIs(wM^nxPhr(@aBU)p8a)e#BU+%jjLFT^P9xLmt>v*_GvdUZw+E5EgVT+PQ{)9MuP z7%ba{9rppo*ztPXlPq?ZhazavcigB`;bZu-;{JuY2BHrGIBv|jnL$zzs%SoeEW1RN z)O(t^c*o^aP-Tks%zxO+lvBw^1TlE3GsBVE%)6FWLXknun4~jK=8jNw*uOj&?tfPF zYQub*Fml>xO-EAMlaf?XKzCp#_tM@~rmlTjTB=}*G)YV>x`hI6)IMPHu<&Eil5@Sa ze_2v&0(Y(_7ON^I@}Yix>N;53@h%1vH7pNSf%(GznaG7FPw_fectgb7Tbju3bgzCQ zVbLNXi~W^+@G=Anv(pOJB|(~MSK$ED0G1zi0N)K{<>*3e1^K<`)!P{TZ~F}gZHf9- z8P|n1%K7T3+>R2Rv?$Z3PqJ!rk<7)aqFHf&#IvBDy!j{g$pA`H`}l)x?G37dYyO`( zTP6Rw=c%UplQ{tFTfT({gt_vXz7^j@?2NWN8kkU~PNL$RUTvq;WnN#`DchZvb9=G|xU4Z`m~{ z4y{;~|6F#c3gW@%mR(u z+k^S|j~$F?w#I7NR=0;7g%twulUlqdIml@tp0Fo+jI76qFzx&qS0$g4tJ`!E8UEBp zV*AhTC9IHM95H$>T6^36iSEXjpDP6k?l*nP6@t~nuXOquoEOQK#2WJT1ueA*W@oR{ zl76;)EEh;lCCbQ}vLxV+!$l_$jQ=teAd_#^>JGBiEgEy_A^ziv+yg2OBd#Q}h0}=? zQ8}<`dhdI!>rg5lCJ;n|Z5g3oYkBL0rao{#n}>}K@sZ=>r>a{|vSctZ2pFqdw9`sV zadEQI`&s;421~6L&N{{jl{kH&D~G-NAYC`z!y+u)phw%9v5%Y2;Lk>?6m<;g@QeMt zJiczI2V?*zR8Gw=;wsqxCkQ8A_w#Bv-}ql4C~o5LFAN?rLSCCYMihqtY?BQelV5?+adp^nEcA>L$x;V|x1Blp9(BX(${X7^M6TxLf6zS^-G1(yMO31_0 z84`uuJm6sgZ~nf^kFfXgBS(-@W9O%3$GIoJ^I)0t##NVIt>81AqEBWElxvSikj4Q= z@xd;QD(>t{I<_K|0lniH^Efn#|64u$ub{gNb>aSPBg$YJIeWc=tBGxqGw`m zR6xQUtc)_v&Gs?YCY$wQ@45q-7hz{7%rpKHqY19d1ytrJMsm;5(B!J*U}@=s-%}?i z_mg@@v(V23I9(tj22~X>#PJvSl{y218s^@pFIBfq4hmof?F`a)NAX+knS?tI@`wAu z@vaHwK3|HE-d|5|D3+>3DuoUgjODV52sqIAt0+Pb^vu2~56v}%9jGDEK;cNPthBtT zLzoK|x@Jqp(v%8Qxb+6H(~rwirhm4MF0QVuQ9jqz$cz&Rvxi|ZsfEZ<=eh6EY4`@E znNMyrHZg`_@`AbmCD)`}SsMT~bdOqx#E7H-UZfAL9 z@WWkLpLI=$G?=J4H6M1PKO{PBK({U?t}j|jMgBN};_c3_{$_D;87&d>J|j8nW6bL?8ht#I1H(nm=8r`LDaA%A zurf6@J?(G7P^YKwyQ`^4{PRoVJUY_CYt}B$%;Xt~@D&>!d3VWwG=Ng5vM%;)G_okk z;BKPK61LHF9JrMC>oZ51q_h7$rLttxXD%L3MV-kv!2I)DCuri`X+Lz!`$ z8%0;rNIph$Ta*WpOO!ATOF=t#8EA+Z6?9oKX7gCEgp1LrV2*I@uskI8B@m;AXS?Ah zlC-AvZ=iCkA>F1dSRa~S5=5?`o=G7daiBPSrruCc|Hjtt9G`*da&_t7=v*;Ezy z{V=qkbYlSx-x;T#if#wx8~{gX=7C7eV1~q~rdzTmqwuD#zOPRbM6wlw7tJj-e9OB& zrwuh~vXc=cj#nho8-{-I;Vr@t$XNUk7HOxwiyx(F>gz@{pBRtB}z8Sun zrnHSB{IaQ8CwNlh5o(IIN0Y&Z(EL8gXgrq=RI~o+(;bqz)6b=aLQedD7lf!j)oT{N ze$BP?7x1B@_vRAR5SfF#&r59P^5LAy_2Pgu=yT@M(9#Xv9L?#ASC_UT0&mJ-fQ1^6 zzv0DN@FZCiIH%&~=jzQgkUfzDtZxenZXPZAa-YyLS^(wR^vO(3@T4{x-$)P;(e3}2 z24a?>^8WHIWridwqH3hf4yGCs^A9q5YK%%qi1U7ZpPn-P7qAlC%=94fHtliY)b9b` zoajokpi;z){^Bj`g`L;SYrvVr#d>G4cwV!ta?SkMT&@6a%To@29pBQ<{^JpV*|FF6 zJh-sPyZgnU3PyB>86F+2c-+DdLFk>2b2N2mm&pe>iA400bwM%6IPtaeeJ|UO=3;nfO zE1a>F)Afm#y%q+IfufHNp3Fh^V!y~XRocmoE|n{*goE^55TZEVzks_in{KzYQ9>-V z6t+^Am^P!qdrVt&Q~om$z@-Hd!W8J4O7XI&Yr{Qa597lt`U{W>t$aL2n!g1j?LBER z>B2gg!{lSA*!R(-`DtVeKyklSy3ff9n>VB{el1C5;Uh;8P)oEvFQO?|)R)-U+S*RV z`DMOJ1Q$nC&uCQFEH+J?mP($-zd9H?|DF6ND8~1xWZ>MSbxxWB@+jRZ=y0i4%8gl9HuIc(WkAD~zZB=5w+VR@Vk`lzHnob=E zMsuQ;mkwJVn8dq+%So_0KKR!qwiU2)os%pEF&6vxJbYAh(0gR|9j##>iJc|=et0Nz z$f9_zV>cuE>jr*fJ^pNjSQt|I@FF#uIwL&oE9#o{g_|`cZ?A>}<%0SO0@7R6j*ROdF?IS+9?=Iqd*!|eu z>5GjR_b0T`HCTvPjTArZF1A$urgIayjEAsbbgKU6LNn{NqC~6` zL8VzsMcXn@pt|O8$8(UPSm=4#Hp#GMM*4@4&Gp_ut>nzy=e>EupTuC+2~mv$)LUcy zv?jw?rcOQ$>t06vp*ybarQwY)jF!s-DTrzwJ z2Md?^*d|~RL<7R%4ZAEWONRQ5K94!x6aC8`xi)xCSi{Wv)bW%(JzkJ!)BxcFE$2G*V?$aSp)gSE)h4IDw1K-;Mdjn^|7 zj3bo{RIh~W)v88KsNXKsSooyjR4_=dCuVnoMA8em{peiH8d$G${R4|~CS;$= zsc=2Bf8QCGe!9Y+uZJ5E9(*N*uWwYY{}8)ev*>ecS0709@NFb{cPOrlqp4yB0Eb&X>qkZ@|CR=|l4S0; z@+23~3h1kqX4IFk33Kil{d)iK7m!k~NLqM3r?-E|_GwfYF`T^E%5?#&JA&E=g;$`M z!pM$(3vu>#Y7~u&n^e`uR9cxOjO@vHik^;}-n;g!p=ismHe53eMmlNSF80P#(<)?) z;Mbz+q(`fS+S|($LH_aqge(lpchWW0qzx%96KPv(o(m?#vLpPOH>7;J-re$|^E@B3 zQi|@R{eqf6HGDgsDAmjXKhFOX++rQAPQi#`@HriKNk=Oe$VEw2&b+6b3RhwcAcG9p zPyrMNjX!0Ed`YFLjC`a{qFa|mzh8qTEEyJ(E!K510HHhu2Xfz~KqRvTEK!dg_E`7r z{-Sbn`M-$;)HWN=@rz5uU;jey-|axb`383s3$0$#DtOWc76TFMxwUdl6jEPU8JHD} zLJ1aH<|t@E8epR*-sZ4gR(kCus+3OMY}0ihzE5lSs#y^a`}P%24_7RD{>DO-HzhJ9f3f z*lT4Kd(*GV5}^RmN^zyiHcj<64Tl4%X;BxbbZS-MG)bF47e|KB5?QH(C8||bpVFhK zBLwoyW^3j~Q%hQtw(_zQXEM&2QZKD`n&-wqJqd|KRi)R1fS;}XR(l8L)IOwjUMpM; zRv*#fmnPU|`LcP{_il0@E+Y}EH}Yl45AE!zx6dI|g0EFd!$TMQb2E)0>H<=YhzTi{ zmT@RY=Y+?X{t@QNyqV?tI68GUXYPdMqi6>bOZC>Y?wB;b_ThQBFi-@!>NkY~MX)4m z4OZW)c<5LbHdW-NWt(-Ah3oe*YgBv~6eO#$puKFF6s_>-z6HjrC*h$l6^I=2%|WtN zUD7x<6WH|)Ekt3;W;Y0?#`i>X@vlaSEc5WdP4!{nV&#cxZcY+=TXfNnJYGyifH^dc7r3>apUU1 z@$&Cnt-Ll49(fW4$dXtT&3sBP^wnslsJf0Y)PjFLlZ5N&P>=mu^_~T?@;f+wXhd~o z$+ac=RGvW3HE9=fM;xGoNm*V@3FkxKM9mJa!Q2VuDk9gmE6>heIck;8p5kP%vx}05 z;e$nD$ZPFdTb`4XpsA>V+Cz}%gpmTd%0M)y4Jp6v*I)1d1=vT;n7lDfnRPIGi?h%@ zP3ubMt0_pS8^o0kJK_|}6YQ-#jAM18CMnNlQLzHen@5ba1Nu$^D>y*g{n(ni_)a#?<>>EtWtF9HL>yipt~; zPIaST+px1NjjM)AAAq`lsR%ZWk=Hz?UFHVvcOS})9G`!_%#q|BE~Pq@EUFdUpUe5a zrh)^UorAgi{+!*PMF{ZRpR{8R$>I}|h4Q8yzbo$>JEU5c;FbJnFbt44+?AXsc{9rn zV^xeR!HTCHz)obxnBZ{L{291-2ThMCsov0FNp%Lwp+^t%W%LfbbMbN+`IzJ};)|4E zRv7;*Dl7a3G`6#Qb#+V*1<9jBfehcyv{a4WB-eJSig-~zmwR39kr4apzfft%yMj0j zKR(@8>+J16EF=B-e=FMm^MV$&V*rSzihTWlD>7fJ;|Tzg01N-$DJa`&Q-=!cW_qaau62bgmI2%Rp z3mnw4aC3V1aNH#3za>3WN+8UMn7W_d2XH+HetG4*?fI^Q+8~?)voo*p4)Z@rR!bROr8Q@|~{ePR(Anp;+*-6g?mbm3=#C&(=s9*&i#V!#jj1#{@<_U0qumyTvU zP~nF;!^_}e8j~QEPQSl^q5l>OMAWzG*2|TX{caHp9xq>#;y;Dav!N#8R4uGu??)lX zd3C)Qn^czAWLAO5wivohl{?-I`@ zB47OPSCQy${B$O=q_g)F(LK@ki6A*`GIK%1y+h^_q3Oe@ehy?lTqlt4Hh3JBl_wPQ z>8$TM!MpB_45#Np>XmJ~<$s_=a9}`6kT5p<{=*o`VP@g}(MKMO)3uB=M?2_vbBj+h zH9jAUk^c8cN;VKpo*c3Gz`1G}5Lc7-2NX-)1?pn0r#0`gjxEr89*v(?)79Sen;_d} z{!Omw%;m(cijxj?%z@?i?Mp5-0ht%gVV&PWHjeeog;#@D#oVaA6|xnIu+rfWD!a2^ zh25c03n79(5?Iva7`mjoja@%eK3G90oCV%|!X6y<^NeR(*sGJZta}7Yo~H#24bQv9 zQ5!47joS*Ov>2d3V;E5~(z8}Q4)z{Q%82~8agKR9>aOJAX^g3qbuR@&DXZlSt?*@V zib`2KE>Jx-MgtBcrInvg8#OXs)egPoX&||0YJQCh`v)Z)v~xXbHzPJ}ocpwMAE3 zdZJ=59#-y+#qZ~3E`+7cRMm?395n}&$hW~)V;PkBA!5p6<`Xha1*Hcr)-5}+h^H8; z{_Z*DThgtCfw(M1f@9j7+L6knq|5^vq%)+a{o2X~B3N-Y4a=gueQXf`3f|)MQ;BM0 zxsrGcZu8*Vf)#LZ@O%CQ8+zR>sW(RHdpuG65@WdAhkys7h59PT9wOs&Sg-J#tlyp1 zz9W~4P7C<`54_Pa364gR;U+OvJvGB9@J1K16Y*<%EbS|Xi_Ps0p8=R_p{8QIO?gx- z-^d?M{(gIrOZ`c0(DVAAuT!5m{;7>LREBf^N=W}|5XGjLfW~Cd!B*9XQvCjRi}^-U zgGLgJXpQna=p8VZQ1tO!G#o$X)5lD?UnVR3$+=r#wKy{j#fv4SRV5Q+!q39h<13T#@3{AjEeUVHfx^Fp|~CemALbDjWN$Iie=7>wJl}aW|M2#RCpNo=GxDYxqiR!hbw`Y}_QCt?gl_{`@`=k?h%O=OMYGX##`8MB#L0VKSz{nM(TV1n91|NWRER|&S(G7WM;N*ADDp|nprfT0M6 z-yU8OwtXGxA(qlwn!DT9Fow9TISi|r@~YY zk4NIGg`8bRV@LAY+MEp$-Y&IdB=%RSa5tTDrU~oogXbBwd@4ET=brATjLwBK!#Jw> zg(N@#s4_1vXB|leoS^oPqGy}KoVeW|5}^61w9v+(?B>0W!Ta5xZe`qa2K&1v`L+q-@P|4fTwBD$qel1A{-n=UOFPDEoMFE z{Cezik^{^(w_tNvk(=IwIm~y`I0vDh@Zk19{5+LaX-%zl@?4ku+CP%hmUP<0E#0Gr zr0Uu@D4%Fb&3G}b-=I@MZpJMR0=6F}5IQumHvYsEaV5?K(9YL)aG>%S@yzkE==PQU zF$dSmb8hPZ>u=Uig)Ku}PWD8BL-K*=_qWNT>esLg zp@HXi#PgriquP8}L+bC~CuS`;xLk}1x}8W>t_?m)DPB5ghaznlG%pIuIHI7G=BA#Gy=F?o0OiNqpFj}xo5N}&CQM;Q$8(SA+H*anuY7z*sVT0Rmv9ln zz$ZvFYt_MNQ?GVqmiODF9~-|dSwhpA0IvZ;(M^v8so6S|e*yozMw@{%{3C;ZmiBT< z8Xyj2Q#%7`0KzlV49)ztYft$8s7+WlFJ3~|4|KHWN(!C+*m^7)gL{CuG zmip8c7F^FLRKDe2L6e6nW+)Yu(UcwjO<+kM$&r?8Mw((M@_p7Zhj;9&iBY|xqQ%jg zCyuiYJ!>utH+MEEMLQQvJO*0O3{Fxu7}B5m9ov$?PRQaef$z$q!7BB-hg5$lX3d(~ zf$e(!z>hmX*trgxJN#7AW$j-BO$rT-Ai~N5X*X&oEr;c+5BZa3`7lIesmGos@zUN9 zG<>rlYbmeHA(J?3K1*H?!NE-%CMchoTds8#?R+^DPy6AJpgII+OiklKC8_{IC^%fz z^Un$#^Hxza^(cmwAL+9x9PY2;+s{+w-weMg1lKiC3k zOH+}t@W{#@;+gtFVt%P{zq>eYvXbwuYAgpCm+Q74b%_toELqX|mlxi~g`%BW8nTor zh~onRYyoV|OS5(T4uvvaZYb-O-SrT6BZ9)l4-;lAiWuZl+&|!9ugEK{-oZ$XlC*hI z;fg>7tmO@93w!mIl|Qv_Fx4?^L7lPT4%Zsn^z;c+(F%hFa;k#Mbj|L*+|gS4af^iH z3oO`mSlr9y(_;8XG?56Ft9L?X7}A;p+srqwm2OeJOA#Sk0`YRyz!e_7m`gN@xzn{V5uS zV(w*AP-JGGn%{WI;{3z)tL-ez0k`pzi=8hnnZTzNqvq5Vgd}0Ou(Ce2^>l*S$mJGY zVSJ}o);cushiC4jop8}2U(hH#ZQY8W-b1oL&o2RD%Hg))xomd5vppHE5pU3(;Vm5% zX^)eKuC{HjyK3P7=o1>)yOnLhH1hu8OQsO7oo1dx5#4#*NE|K8_R(YpCBCtgAtni} zOp?MvidhY%Bd0v%wEFvOgXTC@_H()~^`fL|sYVa)L+Y;XqFD}MEy{Spk6q99C4JRD z42&ixDm^tjD*!)RlJ>D7x^3yA9DW$U7+C@{D%mFKJVv{5DGUf}YJoTz?z3!(`m33Ph$}(Ep1~Q>|L-3eJtT; z<*Ypi8X5ft5|V#7byY#%YzK-MWLB0*QuhfPW)CdXgjYY-Yh+|$#LIJ&GZV?RvMRK! zAFn)hRwA=%Vrl0hjhO9-^&o+nqi?>11_$sR2j3uez=^_e_l}>BI~Pnv$Ei{2B|D4O z)`PqJveG)J?yy490}z*4RQob&v>j%tAd}ofr;qctEX22&WTA&SA19gyCy9@v zOskdRhlSa!JsLn-?q8a^{0uj-&*MZ;OZch3L7x8|*oYqnpTSTB74A95a$bL<+wDD{ zm@y1&^W9*wXDf;n@elh$y6_jEala)hesoT)^9Mm>rXT8mx5Yr?g{X0(@o=o&5$#Cy zj3@C%B*QaKF>?{o`{na1#G4gi9$U-Xe0U*4qj z>ws~p9iuh#SV%xi)8RCdz&K&8$RU34l~Uqf|C3uk%3!)jZR3cnB>ywyIsQ@iJ_}XJ znLF%!cWR!1*oL|@==|CFb!E9XX;Kx^|GFYdi4!^e4L|+@w7USv-eiC{00`AB7y~hS zKR)aS&?Ca?90OGdJ|40Ja~wbpLCP!6lQ3Qvy?gdGMP`TiLG}>Gq5z0dQ<@tA4Bze% zFgzbq81yCv#PK5wh$jWU3 zk$_kTu$;r|)DrL}O^&_xpSjua^9Vlqh?+yWxP{sYUfT?d-l)-SIyQQN5gp2r>9rJ# zODMM}UD~5r>F4;bFhgCF+pG=Pk405h#hk5QR$rz|rrwxS+M{gscbyFIoeWRM(OX7% zEt4mPaWCa+YgjS@LDDM?z6g5%oxDv-9=<|8#*ZIaU_7}?+AXi9A}vzk5-pqezl(V> zujL__MwMS|Z!NgY`6!6;y40glPh7yMHigP3rdMv0lt) z!iHil_m!YB67X8m(T-R%ilL2FMfcfvEArFXz9zt&?ZW+WT909Dr0k!}CQ zldUJSb92lhF#&Sds81M(>Xw!p)42^xrp3~Z9CX2AI#ky#yRAVbwiSZ8n61Z2wW^j0 z7&ru?SMneTbwezw1zzAsmr4!d&GyE%J!38XnM-5Wy8%I3L zz7-a>-tIlFwiMzsQBUP9k>CD*2z%?Vr~0i=8Ap+g#! zZiWFQ6_}yBQ;cXa*ucAFC+le-QV`JEM?^pwDI|}|TIVt#@|ByM0$%P42w8O+S zfnm)}FVgoRDppHP+2o3#HMDn~+$71{G;r;&jN*CrdKIVETonz zE6RQ9+9O=o$gzrx>Z;^6)(WL)n56g0+B(}if?s-+@TIa^nZJ~7?tQ3ZsjB1DA59TC zq{uPwI-iN3xdcwe9I{*Zl2*XC;PNV8LO(yZ*jb1`ap*5QDu0tpL54$?h2${Bs2FEZ zQ&w}in2$MkGbzLe0vneOO5DAt29Kd&JqOoqd3{d~$$M+#ZwC;We*F8dM&ZjR_XXvy}xHW>0 z^7@EKxBXik^EjUbM7{t45_v4tK{4o}C^6yQQBQeeEj##h+=Kenu63v`iVVY2>}1@K zu8gNM{TmtHAWd27Nh&Ggi&!t81l2eOHvvVzoJ^Q+;KIG-;A((% zl_;kTX~wYT!guxmkqgO$qO>_d?qVXmT6wS61~UPIO@iUj6wz>Kz4#siEtF@72Rjz1 zJ|ViqAVMF9IGMUk zmf_@RmYx(%pRg&l@gCc7VzHt@+FElw?;5$!Fa(C624k&+hn$5EG`L5@4bGq#etHRo zJ#!Nd@agovp=-eow}IWsca>y4>*+kc2(di2;uf715pvd8g%0h(BPC+shzg#UY??Lv zE3Lc^nBv962~NnHZdnU(Vr!3UFrwEEh}?qDK7C81%i2Hk#mym5);y5{ZYGjL=T)}aRQ0OlsyQjr; z!%E}+`FHjV+Hy80-ff0{C}JmVPZCwQB*+WZ|4k6$D|MfY&7y}fO}{7E?LZu{sP8&M zY~K6ENtKe0>%tIh$*B&zfK0^c3i zz%RuIe4{5Y?j;u~UU%K1zZ+s)#bAhBN2b)&r}S{x%~EWr^engh++G)XQS zmI57L=VUaMAL?A&XwUfD*Zip@Dy0|K(7a8!tni01SYH~}NYSMbg=5ikBOduXB_8p4 zqm{l;ElxR!jDP0Y*E>#P?;;Tm)In*4$C5C*W_Y*ls3p*qA&0w#^I}n(c~j4RRYLGS z-qRG3JWh=8$(cv^DaohH6~JxW$RyCa_wt3-Y?oI%CV3;ncDZW%YTUT?#T1R7as{ET zX+TA571y}@3TYW+3+Hdg>KT#!xn=GmU6#pDF4p1bl^E|ry?+4Tue1l)it0Teq1v73 z)S4?xSoi5hsD?U}a94s*Rc@ozoj|R3py_Gj;m>dWV!y4R;Gls&NLd0$5Crz86mvJopmp-3 zKJO2MwNo?JD-)r@Urn%_qmMKX9E%FNbA{esih4hLT*tw<48 zq%0eh!jfFkx%pkbK``c!Gcm=`OygySCPvS=?Is#wLNfdn`D3kU7!`|dl+(3E6|ppI zfCDk+X`-RuAHT8Ji6rzz0dgz|yCaILnZ_TN!)1w{r1snl{pwWj}Jnb_#;} z^gb9oq>alH4V4_9@DM$3Z93<)Uku$OqBv8y|Vp_K` z&x-wWB;5JfEh>pzZEy3TfTmk3He!!Je}vmZ37kt2An8eR@=PaWfm#TwCP-0dWw!Fo zX&P%;^AU$SNT~=@g<<}Br|uh$_%fN;FAdSgQU2>O|Cuk5Yb4>7>$e8}XL>27%BL-w zKbX@er$1!aykCjmwWRu1$Qzi_Fyt44X8n0V%NqC}8kXs$rg!=o_vYEB=@zw_(pdAa zkBs)K9IkeRw5JG^t6O?I5|qI}Q8Ijx*wajZmLiNadFJC0G2yW>M3~t!TLu@q3b10A z3iXs${XB%em~QKVwPSbH<~Mz4bUlbs-VTDKuraPdhRtVlhDG3@RtqWVATNUb94nq5 zUJ66*RMDdC_zI+$&#hq-VP|3lO9P6o0_pM?{+H_Y8MA9oCozW++Nbf8_`V@+J^4R`d>lcG$RRP++_h6q7@y}xLSsuSUtH_~u?^PKSCtr#y5%*DX z6!}uY9X6n2<29(2<#f@z;>wm&ByT9ulrQ8sM5X9Wg@WMXL{Ui(=liwwe%p?UPpM1O z)y9y0A<#{YY?(AvhM>7-Qh)~32=%4|x$eXZwZxP|r{}d=-jw%Y{Ine)Js=+9xVowT zjG3R0DfC%+5-oUP{)nq(aAH;4E2{?iITTQBmB|X84%KHGqMR?6jsZ!3EVc9A%^#Z#5$9KNw|Qx3DRr)KxdiOLYkpL_ijN>8@#)7IXULi{fv)-yZPKxhCHZx%wtS~j9 zjOG-`P;v8J@v(!qn!`7%;Y)jrxH3Iq`s|v`x`{cCgi-Z_jT&8WO9fxYf)Vl7sgz9|XQb=uYaq&ohsLi(hn*(F{d%{ttJPC zArGm&kDPO`T62m{BEwlU!+RW__LX<^sklIKJ-P~oGx7}N#8n;K&P`#%ZjJPkxMm+; zfKqPa*r#lub6$4t9V}|a=YX!T#>?+Ue1wM`{2s44o>yDG=ntp_rg<6evAhQ{3AYg{ z7i-zKVBMUmPbRxp-gK3my5wdyxK7sq0Gxr>l8 zNUKj!7K~OkMHh(oZPc|6qGjoR;5@fB_Fyx`fGiaEJ>Gvv^_dL4Yev{foE=~5lHTU( zb{)s#`S&LWf8RX7WpQ5md42ac902-;(>H`2-8n@X{=Ub~?vCo~40Hc8ssE3!o`hJK zSb4aNgzQP>u$ct3)EEzclFI>s_15?De#&|h^51Ac9P?29*UuGy)*oOb<^HS9NKOb7 z|8IUB*FeXGYZM@YmPu|#|%Jf>6hsf6x7!MU{$Vf!kSx{e~4E-P>MLXHG7y0 zp!Eg+#Ucc+PnNEJ2IJ!aR)0E4nQj39A?<&ze+4%GdGkPza^4Il;h7if#ou`VQ0S!j zyz%NKz?SAXh!rROG6F73wig zPh_KEEt6HUgHc~gGqb~~)1jy{|5GXM`}f3Hi?xjt=WbhS-tzb$Rm3}3PMhbNupzIl zN4R;;ji?-`Cq#PJr$(IR}5}KD!Zttir2qkgVENX$~3<=or)f=;%x>UU2%m;S)B z+@J>juM!EQ!KTq)uqiZKxj_O*x(}D|WGwtKZ5+GYF%uu1Aw-T1D3~uSZJxUn?UzlpqC;O039|6;>$9&?=RW-lo`Mr&qeozJATGRf9Iip%sinK|@XJl0|);96dP3qCW1q!l&I+l8V?LFY{e zzV?(XM6(LM@by_!Oi`YHQk*&CeRU5z*`R4=;EEdqED`+UK-P5#6p;S=pEBN!+ zr&?6hV^2yw(X>)uT!cSDp5y(QN(N5P@F*yt)Ga#ng2Q4j? zRWT@8;Q|Y^e#CB&o4+)$hDHq!l7QvV)^KKD_WaTPCMLN#(E-m+OlF*teoMb5`*f}g zkymRCOD0>{qqCdqUaUiwnudR;KJQ0+EZ;xw z{ReOoqR)5cc}4N$6~J&?`D6{zu3WV=wogG)n&6$5HXcCuL5xW8FSa>Y$-L+G(3Rp3 zYinCr7k{e{gIL&#Bz5m~$&vdL(b?W&Rrz{iqZj zQ!IAx13C(B*@9z#xWF88G90te4vGR)-Vj1Gm`>h!>$Fb@HaYbtG&f%4 zpYZT~z@e9{`bkUz8S{F6D=ggd_OB8n@>6HD@QbI}9j8Y;h^-*T%<1->=TTkSss`Li z5M;7E4LWwyl`k|^>7{sP?%rz+`J?>yw&bZY4TUl!b9q~nu4Ua6XQ{MI{N(+p8G(fy zaaG#7_~2OR0lyh{KBK_S*8(O%J_R8Rqu`_7o3}bb3(iivE1sFSuM?W<>huNqyUL}T zo>%KB1_>yg<6aWw^t=SK`=0=$RC#Giy%?2p2Xy8=&uBQW~HUAH9CW+qjh`?g-6u*oe! zXnAT6)#9}gdr>>RjtrSSRn+r+LCW>1VGa?RjRw9yJhO6?3%|fdl(UG9+jjDQ0Ks1! z$4OlZzHd#+Bu4M!DluO(e)hj`3AJG_3)xKn9c}e3w}}#0{TbDOoqSPU+62fBT%FCm zG3*#01+`UF7V&J%{0Hztsk8;U{6!T*+e>ZA$F|!k{iHA;wV!t1!tU?P_t=X41jbjH zwF*7uI3=^A>ux9PtJRvtB96@P#pd03?Qf%}{bs878wfKm#eK1P70&d^vuQ%M1Dx=} zip~k8*n5V(n+`!baBLx+PBa6 zYQO0&aEaY9Kb&#gl!CY+GZbhKJu|x3U4XP?HV;E|eStd6VrqNrUOr6mN#?}0l_L7l zWNs=^xZG0Ne1E*jHgcA~2pcS=%lEBFiQX7lZDC_oDaPzG-Sp&Mi_Wt z=Y_&yg>Yrq6L2;kq-wV>kGU)bXskP(LGB99&fuS5hk;+#q~GWIuwJ`)TAONK%qn^d zc8}=m%lLvhMQ4I?uWlt`B4T?d9S_dy7vyPL$FBcHrP~c>x@#~ zu+6~iBSi(_H8Os_6r|V|UV4o82-g*l$saiUl2WMiY*6E_B6ZxX7 zJn1xIY94>pq0=zsQLv4Iz#uB!VT{~Ux*yJs`MXqU_O-wxqBxzO(`L5#SN@xA)iLG# zNz0DZTKS0mD+8AKIxJ^Uq~gbIM{mXf?{Z?A?vUv8K|By8yi0S@T#&ceNw*W$NHVrv zS>4xFP~@HFSkn)YrHdU?yz3lD|LPL`Y{n-V4|B(2XkW8i+?u-4lm zm)~2bmkq%YZcF^%5(k&lb&}~-T%R*u8%1a(Q_QW`Y-*0EGxL66o;8~)@mL-r6P>-o zPgZ$fwG*RgC*+tw_!U@kQlP??DR?Kd_W`f{yMaqp0)#DPwmaJ-a!@JXvzCcID{$Bz zM$)oj3d6Sp{R61=Jm8c#A@!%|06LaD+`;oCzvAfOlu3QZopYK@F_ZdHMvUuOMnzrW zdH%*fz+~~6+OB_=?9pVg`h3em3*&(68|G#5M47hTIjwJQ+c9woUW!ftxl|APa1_UYL6c*r+Jy}Tgn zXVdiT?FaH!(K}b;XHB=#V0;c1m}W~Ay3CrZI{i1x87bnYB`^( zeQ!oZxl>A24wU@sHwZ1?>@052htSg-WWIVmR~LSm>%7CpYXnIwk4_Av2!^7I$lk~) zaGZvf;En}dit--{XIQCoQJT}P6Q+Z{jSb$18o|n(hEW$EWb;ms0DhZ|NGNhkem|~Ut z#+{3;MO#2ANm5)NLIh+S;sqgE7SlZQCYZVnLyLLRh|hG=JP=CU6eig^2A$yKntbyV zs34VmuARCNm8!e1eee*w_-tVYeX<<+@FK6fA@&m;ts&aO^K9l}dNRDIKN`rC>Xkpa zSxopibzZtzmbwX~c7xAGY0L2ExPhM5L zL{j(Y^VlD7P;4$3^U4-#sFm29(fF*@YTOoXwS(EY=VRe#qZ;7wX51Sg%>bY7K zGy{djsBw?=q2&sd>DeYG{Psl^%cGEkK&bl7eB1zi;R}MIOqHm#$f;FBP+kmOa+Kld zN`7v+c4C+PQ&A{Rc$G}m3$bG)fbL4qq-*0VS4N$jWX9;B*O>m&@MGN=cbIiH9+sV` zTXByet<*2&L*JGQ<(qk82)_V1>Z6H!4>ZKJ(TyH8F(0TO0UOCK=HK?Aixq2|21%}_ zbvtc9-JXk&!=d1$igS@{3|>k%oSDCF@VuA$XU0J}1=t{st|6?hM6Mb&aXVS~P-_Sp zgML-BW!^h(Je!UqU)Mx=Tz7gpf_;}(@I3Dw-DFOwqO%WovCFV<+{B)HP)Rs*Al;OV zk130B#vG@sK4URv$g@6AR9g)#nM>EFnIaGdtb22iyz-vmS^w;o>f>?EC0vjp&z7N} znw?w&t-7RcC+>sRmKGfwzWCC}=-0;9L+YeVv3Pkl_y)u}g2n99$UGFqlW5D}es=@# z1MlcMQ2(F@6M-2Y#KhZhk66S${`%2ab6@uo^os)jwzF)54L zT=5QmH!-N_;>X1AA-LNa*XVvP^cS>IGG}E&u3^7AK zz{lzGSk_xnynd^5WQOC_E6BdzGVk-d+rL;9O4U~|yG8H9^~3mMTk;F#g;&Hswh%oD z8S2X>OGIy9CiYdoKV7O!KWXW$5IY_bR$n3$bChK4r+HX9?K~e6<2A@$lkOHst2iNF zC_Xtk=PZt(9+`f&97!nlM@>|=g3M9utiv*3q@$tb&sFBp%+uDuwD_M$Dp6+$9JHy| zd$)tQIk~tf2~d9*!ARH-`dase(eCkYo#yr%%D)aQ=wqTTYWJzhOb$E)|Fx)dQ^+{^ z(--756@#pC8O=#5(m07v> zUq+r71R4Gjj$3#=!r7wc&h1DiP*|9sFeJKFbnQZa*bWVwNU&L?)FXN=A>&akRvCMG|Qsw+56<_XT3W%ywe3~I9-!QulDD@=355s2yK%j ziV+60Y6#52j1b$m#|-9ma;!?$swA4jqF5xidH5!6GqHdwwwYQmQvH!CW-LZC8Vq_c zpOCwan=K6@Ff`}JF!j{4c`r>AU@5(y_1aq+?!wPJLWT?1;eiXoe(rK*@|BYy&!)$x z_LsNyY$fxbaSXZILQ9q$6l)g0HABYHnQWxdi03+*4R2~~{D-Kz{ikn_Cq{N!Yn|J@ zePGonMu9BmqAySph06=VylE2;p;vUYO{6}JoF$}o7aB!0R2q4YxzyBr` z4tVwDnmO*|=k*qVMAOeXm>}3Vc*%LD>E34@C!3u$pmJadOYbOuICm`7fW3g%FXz3z zgJ3*OK(Kh@w?_btmjLplv7ni%2mkTbloOo)!76%qPcoMrynbDD>;&UOo%@(W8UVso z|66kTU>Mf0QXzfljm2!0X2~C}q&O(esGjVE3=8 zqSAoJJaHW<>{BwVEIZ~yzF@;upH?%&W|+@V>yT# z*AHlYdBb>U3&0iPI>U&@l7an|Xl2RrvgM5oj#6NbgKMtzK2axB#PDfXXu7L(D_53s z&cOEIW1A;#Y|8m;GyE;Z>4TPM2h)q^CBHgbIiM)o5;aYBEiFL5$0J+{>_W^zHc$|k z6%0Y*Ju&nzD=2?AeDAMx==m63LxNhwc|F->dxWA*EXJV z6vD@tSGZtvHv8(Ado-X+#^`842GSl}K&XQOZ-9!}U9SfJ&CTeGls~*PfvtyYY*?I) z1W3Bh;U0*rH05CURlZyG_j2F&QvVs%6m<#t1y?h=j4k)Ap1C=MS{bpz4aDM(iyP@- z>fz>~o%eVycb%?|lNEF=SH6Q2wNm`zBOa!WbU#PB-q+L^B)=4$)UVMEW19G2874gR znw=KG?qko7LoJ6VyQ7`hIlb1#7zRTf+eQ15>ZhEGsY90n&VGtSVQy9*Xp-7e7!zRQ-&s6TPSur;1P8k&EN=eF2rW z{u*ai$P_N|+O=(~!^*jC(}qgjTA;D9H}W-Fa~DMje8|KclGIu3|5`~qN3F%|u%f_) z@srz%=sx%v@PtU(vl|osgUk6b`u!*`UZHJT6Re3WDlyF%-rv@LrO-(J)Zj6y(L#QCr zI`tW6D6-l0&DoS9f6Ij|NaDV>SkZ!C?{J~M9H0*9<9t+}z0&-KK_o?GS$a>H^zV_^ z@x!yVdNqY1kF9N60eSA9rnZ{2yed5K6D@hwCDu<)tfBAu+Q-^ z#lJLN72A?uHdGnv2<_7b-S@Kpht9~n>yRk4mK-34aBXU$Y`!y@ zZD8W2D!G^s447DB41_n3Z1uA@UgV0sD#B2I2xbGZ98A^=Ow{P!d^pIVdDYZHP7Egv?&fbB(N<^?TG-+h!$W0cB`)&Ly~wHaVU$3jVqKmbGWTn} zz>22OiiA_u@)Nxf()}1@+6yY$6`$kS_ond=4H3#9kbUf7ylHDtSSayMpuMlQNlu*} z(RVDbYy;!EWLy3vf<{&8!*n1yIKDvEv#pCuP{&*g+tZ4!$KO-e%rDon77=JIDsW#^ z)XZ+#Vc>XXD2OR#XR~zyL+n1}DQbec5y<|S$C0GPW>>((Qg*8zl@nIVN*oXEflgY^Ey)%DF3wh$x9oZlI!`hAH?dH3-F=`zh z7`WuQw%TiJh?DiXFA(Pa-f)AoEVE4nrR;wU-)Zy^pIjpR6f(}#&V|65CUVhjOW&#v zMysyCMS(zet7hJOg*YO>$JZ(~{=*=AHWU%?Ip0@5Ck{pr0@jz^dyj;)ib=K7C9 zkX)Qw0X^1BW+!80@bJx4j$Yy? z*^a_p&|;_15gLorDCLzPomHWIo8~b#L}l{Yxmc~-=0fSMMO>29h@66mgs#_w=zzkU zg0LY!-?nD$hCl-hq+ZM189hl7uo7tjrL&;jnKH2#P#8(jv5 zc*45+Wkj`Cg|q7eTFFTcjt3Jcsx`W?g2bULbnKipKTQ#&DU~T8XC4)llvvl(ZEKzd zcZxS?{l&sc1{Z!+rc7A=<9&zh_Se6OP(Sz^$&^Lwcs>S z{&Vuk{xegfLaQbj9`Z6!_%nZ=-&*_{zwM;ITBGlg{{TC7@ACRg zeo<`M1ld!7D5T(novCqwTbbvmaYsDGj0CKl`MUGu(XzAWgJ$8pg=`z4qHx|C%@Fr( zP+$xX*!#LYx#Q}%uC)F<*A!;MH#>5w?~vpRDXPU9 z7|Cv?JbEW>b~%Ue4WJHtG6a&$s>B3+ZG*!kj2iWVC=q%q7g$T~4`t|=Fz~anWm`fl z3iC8%G_@%s6p1R?g}?6*`1wFv6rL!vQ!3R5F!rYh6&~ox6-&qsnmh=T#BhJ!5P`yR zzf+>|gD;}-@dd74KU^>UwXCdFgI>A7_P@CCu@*Wo$p_fxS!;#RMhhZBS#1aXfI%wn zTKtorTa|F_#*WNz53*pU_tB``Q}scxf?~!Gd-gu_^d!5_M|{_+1=gb<6vs3ed#0C) zzGE110S!zN)UtDCP#mypcv`@K7+dmJF3@8;SnKPB${yQ^qqMiNit4afw~P$m{;Mu* z7Gi4d!r0)~#iA77>L4}_?N_`KxS!(8yfX`WlX$7mpRennReH{>U^rvzvG2fHpO)e4 zUQbnP(igMo<2EDRGadrPC!q3tdy>50#<`;JbJuw@T56 z`)(@4Q*&?LF~Lf-A?30uOeuzGN@;XFuqoXrYDZB*WB!_UvKSj3kv7{Q`~9^NJ0s_- zL*6<+j(**AO>GksVZLE4l#Uo31y+?lfJ}Ip4(_QJ=I#5=ar^_kqaSTX@CC#d=CZZq z_a<(I1}jVc)X?uZ5Ao2;dHX5{w^kLFm1nTFAo1Qv?5m~}&F8w&J;Sh7&Kiv}zG{O&B+5IjFd1lR5Aphe}>~FwsogJ_?}<-PXfo zTXVMh%-YcOnsU;UWti`GEPtOnZPC7bXL$1Y0``Zam{_C3#sjEyplIcB!07Q3&1Gws zS7VgDS>F4}V8Kn+2rQrR@ex1vZ7d{rwkqpoz=eb_#7L6aHwl||7jzzkLssH zv7Zz+*;+0@xOSb179{Qspvq)<>Jk8GSc#u)ri*I+wWZ3?#9iEHe)9`uAy$Hhi>T&= z68{LlZxBySyAmVf>gl;#=fk%#EHEgq88F_$R^WyxT1!grx{-~ywS8hI!bv)77UO@X zthMB(xu`I5`*p6@ZbLBJwjt=+uozXZ{zqLjyd$|1V*4vqEqd%ScW;(2HTFQ^?0Dh9 z99Iw6u(=@WB)O?;tH_yOU5To&x~pz4;JXr(-GU;NpDXvu}RkUeciV!`M1 zzo5fkZAVKqTp`LvN0$jKYBD#m?W!&$PC68&C0ifsBVVlf#g#V;V;QsWdPi>;m4f(c z3>hm01~asxzeE(~&1yhvyj=QR_6z((FB|f-t>E?++FtDT9Qb-o)zu(YSHH|-C$8$( z1o>G8{^}Lo1}pwY@f{DmPL7BTmKiJzv04HXtTSKHbG=!>#Rv$PCiv!{m`E|UUetawYSh}=SOrOt%#RnOGWoD{$ zSTLEPYEjthO_OId$jM{N(19m__eNVZp2sSj$?m0EV=!lnbwRX?`dP) z>?3Ru!?CNjy$IWPAK9|9HAmEFktwEVEe&>#xYs=AJBs28uZyXQYBIo(8LA6z=FQ$Q zB_B1u>*N9W?UCqY#oF0;;zdfFl0Bwq)UI=rS;A|yGlc5Eas2RfIclc#dv#bPuFghC z*0bSB>q0lS`3rpooT%1T-IA;q*$%-k0Ftj{Sn5)WK~MlIbWqyBF)>dU^Hk3Xl~OEh z(=aF}thjJ9=Gj-F;OIN5I#D%3nrb7do9-Fe|3%TY<0*?)ES~+b&2SkpDRB0K$?R<1 zDR6-(Xc0ZJ{E&-peRz#LXOo3RgEQO*v z+=3vAorjgWTeDVykBuIc$NE8#ESoNSgqQRAr_wpkpDLq_J?wIl>{(9}J%T8jeluYv zp^(m1yV=jJgPQ&|!%=uI;exwM)}@Twh8FKvCTNY2;W{kx??_20#>Fi(QH=P?!+Cl+ zMmO2-IkaXs4cbw(NN6}pyWFZcGN~)(46%U*)ZOwFUtblBXhGvIPJgyp@6Twcug;JD zwIrP;2!>{=a09&`@2QmN%@&fAp%YC+Ap;)z&HSkaOU7Gi%#4fHU(@8`RrA@7nj zKC627uf6%qN12&!Padkzt(igWQ6fJc?yweOgd7(F=G_8mT~J@X~7M%92t7{a8VP4$E?mMEb?dB3P<<1FTWTe7Vm z{HaecLNxTpHXgxeSXoAWqo@_q5d<<2a!!Ad&Vk3Nebh36mo z9dqNJ9YRxB+8md-%^$RN2x(iE&Si?Rh}jF|rA_AhnFjQf2QeW9aoL>rI?7Te!t@oF z4wq;ZS&$8>aym;sya(Pj70&l}Q#WipEkX=_8#I*^I1(63CULXt$K> z%bS*_{qNRs{(nU%(*NZC0RP)^i-T>{x&HQu{K(8vygbOn*&pDFT0%-O9IXM!p4Xqu z=d83l_uT!}17y?wLO9JeGTj+Yjhz7~_ULVbMH;$xocjReD8llplTiQ=G=;UDRptQL z|8K;f|JvXFpPJZ%NAu6O-#qxHb|YLAy6V^V=WB>=T`tX0gi!4F@k2Dfx^H2HoWWil zQ^xdZuGehSkWy2xPVWcPB6hJD*}pF=dEYSXFu!mrbFWk|afCoYT~SHP3OFIsZj}z{ z@*eb%MlXw5;<}Cd)FZ3x*u3itGB!D-{Yj~)y~P)C9eyPO0}}FuR`z~J<-OYVqXzFR zw;GUj^{30(EPzer7ad=|&W?N(6sl$wv3JNQ7W7(8}6q5X&FM zx!v1Q?yP_rKUyF&Xny3nX*d6rJjPzLnvwNa<@b8)Puv}a!uvVQ)@r)(aV{Yn@xR_D zUL%_m4G$P<<);;u+TOVxO0A(%7#}#jAAbp}s6&ze0sFEgCZg;ihvQIfUA#rYHT8UZ z)yzLrWmkr-E3G>D+kK8@c83Q`%rRS*ugMYYLGa0NmtXH{!&Cm`uSQ>03@FERGfgd6 zzF`C6l431QLgNjosS(;KeJ@%(6$1z$kmyMPenuI^ev?8hm+$yCq0a8{vwr}~8?N6| z<@d`;$27T9{yfuVFaXO>pF>z?CpK*|Y%`U>M1&?_pufDyujaddgZWc}l1S!$M#DORX?gTj`{cvhNMN+I&@8E^zi zfi-q*IS)7gmQUWv%GSLJzc(+FoS9~cZQ7HCPBAu;qr;`=(G8B`Z*eF!ADRc@s3A$v zEKkirvsQ%$*8xKB<3|o9QKM3VmUoETQPv0Q{xMW*Y^iYc z%#3lrjw~6!_8uD!2*&8zj7mOVajE84 zjPeJ`$Z*qPX)TtvHtaGDMD%;w6ygYq$WQT)jo=VX-+&{mz*bYKAHCk-1TQJ9E5DC8q zf_|hzH(z?DS~?(rJOc?#*crzmd|3E)ByVcLYxg>i@mF4y)Rh#cwbdtN>;RaFjaqfm zi;60ViZa@%??p}9r$V5rh-bvSCq0S&P0E{sbQwAL_)ztxXeUtkkX)1m1xWyU6z!*r zSXefKWbMC2D{6_H8x+!*y?K3}XRG)zogMg+G(!Vz$A*l>JgH@ClONa6(19AT-OXs= zX@@;Vk{Q8Do)S|?ysAmu$|0M@@B=+ghg?+#lGhopm^ej{ECq7U7Nh6Avb+}79IcoJ zPE!jsZQYVOJNdNGQ)I$4)e=411ERd*)M`7{&^Q&0FjlX9b<4wu?t_?cGR{ik70Mc$ zM>4KQ;UY?OL=9yw-V7MQS?Os6b(tHr;dHKIKfDguDC$PeNJO$=dIvhp+Rq}#gfx8x zb)tzOC3V?Q=805AVKwINS6s|pT%ZpJ6^UU=hDY3bDj87bdS33jbijMw!uKr%wNLw2 zje)$=dRs3ctV|Jhicd&2*Px$f`C*3U`2BN}EPH}+>OFW=v+uo%lATU}5|aP^MUDm` zKsyaf8;D}3rhNls+9zwdG@Di9BtC(})Y+m@;pJM57}#lwQX!>z6EH)@xT-(;fSZ}? z&fKgdZNGmv3AI1N$IlETl~33K>!Q@K_n~+~LvS^^nXZ*Rj>?uJX+v^R&UJ+vT@xWO zQ?RD}!oXMUHG77k_5zV~-axTjIz&ysCDva1$@C&G1>YVjpU_MCq}GUlmydGa-(Wzs zKW;?_^2ntg%9JF)M%tbN*D;I8rj%f#8L%|iA%BzdrVF0Jx7|p2$v1=-aI>iqZ=LJu z>)`t6N?DMq`amZ`fu7#c!mm!R)?Sc+|M^n5^Qs|NZLKw}qQpiqJBN^I(8KEeC(p%x zDCJ3S$~&u9Br(6uixSPb3>=pIyks=5?!eC_%e!xljf* zqG9|~4yPgxz_Z0er|D;m%YKS&<(;u)A}1n(^sHk~;KQ8?v!M+<9~zJG-{7cbl6ugx zR!5EZ`LHjCrUs&)mbOh3jM&g!N6JEwNQGZo_-mf5SyXwoLPb|tS)j}0o@SBpbNw)~ z@>gM@;l{(f)h{-Uqq+%FzQhVLxszGSuAm#QHXz^Da6jBh-W(z?AW9D-qWlMlJO@=^ znxokH>;|c=vNB89t9GSx%lb#Vj%xa;QhFu7U1$CJgf3A05W2vjy7ehljm=(QAOjINW%Y zc5CT@sjn>TR-;-n{sGLc_jNS_-#Un_3Ne^#{F?jiX?j#d4TWEratMp3rza^M5DfGh zM>O1%Hnt~QHf-5K-)ZM@$;V_iT_95QkW~;P%_cE>u@gs2dZe8zi@Q~V)i2%(KSp1n z)miGin}~On=N*Qd;*C@GgQf z_M__!&5Ka;$$VM0Caw!l<1-L4X+e8?;|Ng=U0IrI#OZ<<*3C4KkyX?}BQ&?X=aD%Z zgULicRb z59 zXx$Z!w{;O@;n~`X$r=2v<3AFY#uRV7-vJj@ekXH#X9D8{E+;UBd4U`^?+k-D_^_g|FPmt)ZT>W|nee)&JHKdXXsDJ&!xxViB<-GW z^gJ-X*j1`EQ#U31Z{AO=m^*fU#0@Q=5qnsoCZP6e>ZZ^X&n1oa$@ew1gH@e#GG51T z&HBHTyY#I1VvaPKcXD*6?Qy@)OXqo*2hHY9{$4O~It2L3avki9a@Ca7IB*!7^od=+ z9N}uO@ROAJ2e{W#V7NsxAM?l6tV%J|DS8$?aQ<6oBxW>kn!~d?*KPkF&LhvnDgD^rO{%{v2p_R8yP${P`vvB$ zf_^EdH}c7QzV7~`>{>%O(o=4}WcFT3C%5X3YnBFEQK3|4OT{-Ol+{@I4{((t;eA|Z zisww9V+ApciXf~DPW{Mb4{(xJZeDUjoUhK(`|-y}#&Bh}*?2Y(r=4dc+T{4!-OU0^ zwVvH*#OupKi82l50H2%RKKlJ}o9ipTqYJ8l}BBoh|9 zp*!(QLY#bEKG|va@CSCZ^%Bz@E2WJX`=(8+a&s9*X+j|y zlp{HK#!cqca-!0!`3LGk2X&@(`AHcp{lobZP4m$3`~3W2syK)_Sf|{<5g=yzv8g_) zy2Fj4pX7b9tA1SDWOJm+~PmPWNLjr+NDQz-RIxHN*TndVs= z_B^}TvAw^E-46j?-(^BR2C4qxcggSOhSGbsAU(UWBNP1$yLhqk232Sw4J5UR2U=DK zq4V5Dl5PX=c*9d?Z1q?k+VuH16ji@N%b#~|lt19qH2e11E()p&&5Q-I7^!dCNb#$e z6&_A8K%GUxs-aa!02eU`A4gc2kX$Y-BDG|RlNry(3THBWm5U9|(+5COjKigvL9<&e z;fvN#UT)W~JV);)@Y#;+e}?}69N8b#P_OoU)zwMfa)Z?-h{jG&mk`b>02P_I)5aSc zACh%oK4Ew3o&zRsFdsa5qHfxEY~qXYD;Z=ueJ~q(2w00|nJ0p!(XnwVIl`s|JW8x6 zkkfn|&$>VyGO3r&BYlKP1(K_&CSY4NBx^)zE@3sk7J~l*LB&TP(>{m9po-C?Phlxi z^#eE@Ljm%q7I1J@mJq{bk{l_}(rMeAW}GxCcU34j)QP9qNtXSC2%60QQWNJEH;X>) z=u`WK7?Cp^8o+%TL29vggkchfZ&lA8q6I$Baw%mKlQLdM9RTe&*(&z=d80jEP zo^wr0zcQ+kQg&ux8+1_N%_HFm!y>QWcH~|#VR~|f9qT~RdnMv@Wa}oCrtbf&DWS15 zb0;%3Rf8w}AU5?zC; zyN_w*-%$=8Xw*L+PR+oV{ivEy0(jH7>S&DCH<~&{qe3hKJR z)HR2bLDPJEeQXDjdeeENti<=H_Js#X!<|L6U7B*=nW2=daYveUguBS+w&~PqL2H+m zAkS!aN_&NDSd5T1H26(oK0RuEL9S3z#(ntXi5ja8_F3#4`Sl2eDpF%gNp+Svvh zrv5+#8r|2O&Pac=IUD;r9Fz63)f_LU-J52DXd7GlZ*$!2r?;A*&Yq^`R;DM*)OKn* zNiMYQQYqV1Lv&=pQVF>ho`@fV3<4X&AxzG7#QDXncTl4{o^6Nc4 zhARfBdHM15&!y`VCuEKiuCh3@$;6EeO`ZYtV-Y-t8XC4&MU`g@Lu^odPr%0YXqeJmAKqo`c>X@>jgFgrD4%{ zmugy+qu{-^n37;9Pflte?tIXq0O(uTt>Mp&lYS|aIa0PHS+-L$4hp;?XHvls6|1}@ z|J)cF3m(ce9=k941hmn4d^|F|{TxU-+D2Fvyx2j#7!lMV^vbyv*~UL-i#!$`I2KDg z7*p8GF#`(sR!NZtybd8%LA;^Ke%*F?-x&nf`x(gw=gwN&*K9?bz7`fm*SiDbadcG1Um;eKYGQmNK4&QD6aT#oGAn9T9!u6z9CZ3L#q?E~S@5Jbv>Az4g^`1)--9axa~fM!`|PnbBzX5NPfic9507wMU}IdWCOp8tog4G~wC3XsjMh zJEaV7gtA@M%S|R8&Uz+1DCb7C{0l#=RrT{YgZ#q&&Pm>_j4OJ=!*h%K57`~J4jVsF zC2Q&VzpH_d+US_R)aaIS8)JuirHs(N*?0<*r*`YdJv$%Q{ZOUTbrNNMPP_$bvu*|+Amb|Z#-Tg(&O5Ocq|4)VJ|m!RJO@YeCLR#h(|(>r-EeMwRk4?8QO#wA zvgD?H&!qQNkD^d~wf*Agx0*#RlD)I zSkvf0L{{dw_Jb4Re$Hv9iuQdlO)+_@TDM!a{US|lL*ykNb{iH1#$g}3B>DCkGqibA zD3pgs5CAHDZThU>c3URM2z+sPNw5lg%>GUm${TB5G2|GQD5n06mWQcGHiJH16PQFF z4-yQa&4<(DjV@3u>I!{(%^D>$v~NzMt6TRuwyko?G;(NoY)!r+fl`*r4tDI) zYp+KOhD~0~zc;<9vjayR$mVjHm zQnD$4b{4I)Ppe7ZyYUzOeyylF`c{~0yQ0b)Ic!kQZ z2`xGH_E-SFaqjE|cVvbUUf)Jh>^rnWQY`y>rCgxX1Zd!9M1=^bCRiZ|&LpLItt-*1 znLwEqDsHK&jiq|Z65C4gZIpV8@)e!%3s#VmbU&0~ zEHG8AhZ;W_SlJ>jMsU`MgXpWjl=pURAvcR2p7{LuQ)%U}$%2sZtDLJWO-N1hAV@d( z4nsW6(&T44=EU=~PN8}&qvAL5mO*r~E{{w?mFvzM3rl9Cy>tbskgteZungs1kw5z6 zkPup7a7tijjU&0GNT%~{BSjR#Dk#=2Ap_GCtomv78f^v27WxOH=%aitt@ftp6Lx?& z82%52`|EE|1$afjh+d)&Staa{nRcp5@y&aY@4rg*7hh(bHF(w6KRki{;a6i}{|BS+ z`tQLHFMUPc&blb-#Ly`RKc}FwoUt8m`)qqdz_& zrvE^0Z?v?{MuQGD(t~u5vs~+ZiB4|t54g1&xH+W^i=>}gln+F_%8k!|Ju7aECu9)Y z^5;Dwk!x`;^k>m5r=xTEyW#sitV#HE$!^{Cf^T$$twMuCDpapOHE|HU{QRO;8Oa*( zhm&mjbTr#b{=FVazG-gT^bhgJ>FMd}T92D2hyZjw*3a+9C*XbA?P3yVYwFJIo;aJZ z=D4bmeKf+BfWEDt1#UQct>*yeRPu_RwIDLNr^ry#)wI?L9ggL#))FGPQL0T5=ohyr zEJvGCx(RE>fX5^9AO27p3i%8@7P*KT#JOS zMab048tAar30!1KiwOxUdKK@Q#&;}vlmx+Lb7e60;ah#-myTl;gG-1fe82fT?3F7o z6Flo^XHTvhS%JEmzc}r)zh|kIQL*;YRPyT=5NH!b!Z_cUXCCcRLiW`nbwa;gcF0$X zk1pEW?_JH8xhw{BtO8Al0*^m!x`>*cYx8iRVPf;7M`?Bb-bbmnZtB$UAL4V9!3Z%A z5l%G}t-#z)cp(OU9O)f3Czr#WFGnP*4=Dx0#;p_{yM1gVf~X*>^<^-HmQRFY8N<;!d}$hvJM{dFO$48~1RRuMgYA(;Lu7 zpf{2Gb%0Gt?xIv4%iF7~+ET0)`JV=qRp|A?XU-67P+rZr!xDHCAja@xI5*Pg!=|uD>OH-tT+m# zqYEuAnWxJMi`H43i)mgVpRWJJdH3nAu-#|!de6gs&J?ZHa=A5;`Uj(vqnhgBx`Ml- z{w9UZ8qP`K50C+rp!*QPoey;G1M5!C@ijLF^_%hIxK7^BV?P#VR$R=RTygCl3N{l8 z$isw{oaoFxd(H4QJ%)AEu0xyqSLOR!?un#?l8rbYP-}q?5;oGVpMPLKN?iJ%1vH=b z|50XVED+5-f6JHO3ZCjVxaz_(hHsYiZn6hfr9gkJo7bN&oW~UZ8@mirY5>2=cuzJDbE0_s&$aQD&e#(JvIR&YY~dufIBI_#Br{N;1=bqbv< zmHRn)$^K|V^Db{Cw6%?kw#m#|t(s7uv#Awp*=@z|Ea+jG@uC6|m@9Y+^CYT>DV%`| zm^zTdFWdf3g(@L!|G{`#-20W9xLBfd2CGJ7wisbXAglHb)HDQ^5uLSv?&AZdKTiuj zF%U_uVrenj=Cu-Pe?6A+R6Dj6IT=Crxf*DV9;-<_bTdysDcvBPNgUCn+DYwI;dwW= z#3fRsFv2L|#9$sH0?n9jj0yN){LJ7ZCzf_GtCmstvCVI11R;WxfEVx&2BmYsvWj-i z+j8Qz9GGmjY+SZ?>eywQ`*nZb#ghN9flf{AT`>|@zXZJ$Pe%3HyFSfw>@yTD&KHOC z4S30awBpH-9#gKG$mwH2=!O=NA_G|*P1K)Aa~fQl5C_*ChkZ+&u4+jmtLD5g0K$@$ zV^6|FftuTV>!nS|$-}$Y#GLu_b3s~D9n@HBCy#Xi@u`=bduXvMx`c;&+O2#j2f$Cn z{Z^7(SBNvL)e1?)ieAw|hegS9H7VtT0m{@rUNn@Ys0Ru7Q6lBYU^^t2cRv3~5z= zav$D4Fn=u%?1TmC*%)e9R$Z6Y{(^)I>I`f@c->$UHGX_#7vjgkY(z~tP1`~7Ww=!8 zFvL((S$_67#%ojcsE%E4|qW|o1$2^Rn+Xx0N2 zy9EKLtH=T(TW61>P~=A86MNMlUcMLbCTUHeEjuW}Fho1@sOT+~f&vhS`VUXiD9M`ifENu}uP{GP`?_3R{SLNBe~N)^W*Lmr+>NNiz*_$p zwD$}BlZlCX41{1vHIbC!53m<2 zOV)5tjtN@*;p_V&a+@rqXA_Gm?Ue2K35`WW+VA)Bp3g*P^l~$3B0R$NRe=e_ET2IH z(DX8>Y=a{;i8QAF`dblI5d)YJ2Nggp7YB`RV}+*8{~`U!w9Fuw<`Ki^cl)!{P-rch z#7tq0(OfV-V064Z$=EH5)OY{HGh=Q-CnK>GM(7Z$ZA}IzWvW=xX$(uQlLm0aIQ*h4 zV;YY*6mn=;l=_$_{=MPgIrPxxGKI$^GiV*$^I&OznK<&c(O-I;zS#NOZD839XX0=CMJ2RO(Y(Uqyl_azfaJ)K(;y9$vB;n626Kt)2f+@8qyk_P5k8qp zXBcdh(Bgsm?!3Yao}*OvR1&%oXkaOs-KG6uudM^$Hy`;$#juMNJ8}j+6%SJbDv1aq z>*B4HZ1gi#3LOM9bXf!i>*7hIVsCHrWyTrajQo)eU!r{feH>4)H_A`&DEzlx1dWa; zVdmQ-T4Pn}v*PbsXsE^a_+9%{Kl!XzLl1M*@Jx35?Am$X5y`=8i{j;-Gb2cb(9PeP zk`Z1MT@BovKC;>?b_3E_M8@~bBv7ZVF%``%N;TEAm+Spytx#9F@Cdd-CKQ;cFYx=#*t#l^vyO|&z|CH0g4q7*#&9X*I<+Y0328(iP65yGpB~^n# zcrqR^pq;5K&oek(UGIb$o3QAfp*&St*YMdz1nHTm%<-u%)4*I~3!wS|GHX!XxaC&G?1+B*?yx2)NE`T`29&aG#4XHf~GUe+n+!Up5X zAmh>*9ToAKt?m`|2fClnGn(qh-d{4FZN1DNJqBbLU_}6W)8nI_-7MXSi;4GnftoJGuxq0mptzoIUhw0eA&NNsAWc7re|bYctfwQXc;(Rvkqkn zkKnrwR;X)89JHIBB?-o@Y zWMDVP@_&VlQ?MT3EWtt-tAdwvxdk-9ZXqOzu3|RpTsW!k%_V# z9&au}sVOf0!6+kIDUqXiEY*yXc~!vcK8~EcWa{{aFkvn{VNFU}^kUPBL$ZdlzIYc~ z6zx>}ND!KDO`If?P>4_&QNM|MWpkijoMH?*|nzw}xkmUJ%ThoD2p{F%l>W2n?GQxwY~q@&{a zI+l3wlo0jr)=x4NPB^kzpA1UtS!JUzw52yIu6)m+{w8 z$*BgvmQ|-VL4mgpfD&dJH&JU#aHG)EUVAW|oRJ%MwS&3z6Q|cd6IqLTvnWA`VxLP> z>yZYJmPD+ICb6GuDfqwbKyyesr^UMZoXq|o9l8od}bS_58xX-~EQYUk((jpzi@eA*0}-s&gbWOCk4&Xo&(5LY$Z z5)ilf!twa!SYUKH=(ZrK`|hfS8aj)2-e^A|%TiEPsXjR@d#7R;r2=(OZWW;uH;f+f zWa3gnjbwpkpw2uhxHNqw#l$2mv2Mueu!eQdQ2E1J2cjZ%sZq4q(>3awV3J4dnl)DI zM!qJ8LiqFB-{xx9cbpvvqGz^`uzZ3Vq3kgXIJTKJyIQiGOsnYLHm16Vfq+5UR=!cf z#3WSuD=oS#N1*j$Qt0qFhw29Jg3AepyL{R<^d9NaP>i+Q>M6HONK^my-0|G}6HmS3 zLE+ZCjz&;tpX4oV#oEHbKNwqw>cq8yKY}l_ib+`B4SU#=mAmZ^7kd)@y8d#`ppu|v z@?OOHfP`hru88FUyc>Q?`vI-c>ZLAq*td|LdIoUg_{gMKL`m%IbWI**5M z{z|>&B6QI34>!ZqIV^*Aare(`|JVayEyuL6w3-_H!+zT@KIWEK$6jlyo8Wjh$ZoBQ z{L~Qy>xjU8Jw_8YFUVjY-K)(tVFOIs?=|8t(B9uIU#wjsU$Uuw%N0m$Ttt+wcIsC;I_n0V@WuzJl}WDCV9~KRIUMSR zOB5HQp8Y}yj_Ss)#!emti_qs2P?_|`6{nY>nGJ7~PO9!Xd)}`uMluH0B{Ep*I3cvvJw?s~#j)I_WuQrbWQV(fl$ij_WV-Lv)MuDmA5X-yh zHSGptxg1X-ae64tE~TMx<=eH=S{;$)$j6)8pGCSnGBuyLzy^=nh#j*{+rohIxR5^x zRgPQXW0KA$ZH}SN_F91-DNFnQClbr?{I#7x1Jn@T$o*jMm{(qrrQ|*;J5<5K%t7)W z44AK!{$SVTz4PI!Fhma#%60%zoNrOl1kK2$PJ^_-I6tQ7`u6~=CP1KL4sO=hL)TGnhVPzG=vSAdwb#I<>DdjK@g8;?O+#+>9>pIUZWTj5y7PG|5 ztt)&eBwfVTiHf$Ws+^|ZIR^f09(ZVu`5P$Qgp^U)$Xzym@|7`bIfgLzDO-2soj1wM zDkE$Rk^0l;nLQ-B1eVVSJ>S7yj;nTFNq!A>bYgCQBMGpJkSYzFC**Ycyi^6ETJUq$ zpf^GXtk1aqRs4v&;&zme2p3B_U$QJSt}AqQ8LRok=ylIgjRx3*9+s~N#Bn2_eP$Pu9{WcNKGEwE2`{k0yR(Foz& z(drtP;U_IOI}DE@>n46am4n%=y8R06bJl)40=`Xn0pUVcjkG;K1RE_&guh@irh{Hup zmemG3Qn8nH=9IK%!^gL7LQbE(evDP^6>3~$^tbJC8s~uxa3D$cO>6aIF#&fL;@eh< z)yjG~qK46Nld2Aw7V~YjKx-?wkLB5fM@$6ZP*F{K@R_aPgwV1RM<$OLY$xlm$e+Fm& zsFl-cO0FzV2<9=&A{L_!-f>c;b_!Q@-k|yx}?jSY?0N~ z2iqYW{S33+a8A{L8^|LVMDI7u(8qz-(G|C_Kmc+VL&SZ0jYW|fTOp64vv?1yUnU*> zc+;k(B)Mt-9ghN}M^Q-%gFg}E&=ur!KyvFU7083fO9|DE^={eKU_}-(TBe0EY{r+C z0WMW)U}cRd`reNEmpS*~%NztF%&4eK^XI7`!`R37ZGFDxdeXE*BCT6H3xuRY^~-Tp zZUBxr2_LyN&pykt`9R=drih7?y4G*=z%EIDb5A0z-XyYLyl=LcOPb<1H1(g#rF?+< z$dF81C7Kx0Lv*IOpnf)*($_?I4I$f+5Bm#y8i1vE(W)7B9P-d{Z12Gr44L`1w#mYo z4agmd%QCoFA%eX6y;dtHz6h>IN|0{Q&Ne7{wk)QxtF%JCOZVWod1`3Ui?t4+7-(l5 zd0IX^`(Cq`2bFAmLA9lsI+fBK#j^^;lt^7h#;2VpaFMm)ouEYTHUhsm=hH8?Ass=I zH~wP1;O)T=f5`p0tT+Pk5BQUSJWc6bqzn~0IUUWom!RdPF0sO3bLgKU46y*XBSR>cvOxS5G4VXEWbZ@ zs9NQ0&#)CgK1)A@T}*nEQA0En=W^7sU|8_t4#;t|f#$CszndH#Z*f&I}mFTb16^hLW0sG(>mR0^=B-o?gDMEAivevhd4n--t@8!u!r zpa>S88(XHAHWS&depq;!3Do9w*)UGuC}}kId^NxSIjM)LX<#Gkv&IJRv$=|L!4;N0 zUDVKaOQDEasYe*k!VwX*UVHc?`9~~SPWmQegXM_w1j{*lCYl7)tnoHQ5PXH&JG|x{2%{PvE8DgO zRL=eQuy>EJ^}PIT8lZIiz_Itm@bEiKbd@j#?vS9Q(p3nA3si(!sJuG7>(#ZVs*`_* z%F=)Jcc9^wenxNZkE?$$nmGC7N;9OHvtLZr%*LJ`^Co%f zME-9L3I9>8+e|eZo%AICf8;-kVPDVQPTci8vD{HXJH{}u9vSD>C(w{gU?;j#klTXjp>d2Mo+7&WhuM zSq%NHBB*sqjE@$`6eD{ApsxN1|1%}zG_>W>-ggBrML1NGU zyN9!5YH?Q6$YBNpb)N&b%0LY2*Ans2eUo;V-$Ai85{=N70Q0ED9!Wna;g5>h92d z!>s(UzodJRtF}ZgCV5bkB*^h~_hhY!D%#e(&xiTceimc`KI*cm5x>+{SNAPDBorrQ zXae$}p7z+QfVop1%cq^y`S~nOfTn#A=_s$ zXkBr-IuD-jED=<7kS_Vtemi0Q9dq5^!ItVcBZ^zVi08F2)Lu1GAKd>}WaU}iR(~q3 zh>YU0igJ7~oaN@S#Lf4+zlaIFQeG0TotN8Xgx89B=#J!re16p#(QHeLNM{5Fp^=-T zw{$wv1kunpsNiV^_R4N@l!3ShKjJA+auSMx%>k;azxU!J55kf%D0up^Dacf!%9-oPra* zj;qFIU(~Y<5~#AMGEklPu3^7xvu_lXVBR*@N2^2xY%B5a>Ei+HX+BARDH_vkjZ92) zO&UufZxz}1lm$gKI1TKf31kTH>sUcDs9#Ma0eDkrSAtvSqbfyuTs@$48sg7C3ycXL z6aK1$=S#{A1sPGNy&AG7n;>7zX5`8k5UeO-L$E1nX;swIB$WVO1YAdxBJknkE@z%E zn%o1}LpzW*X7g(d-<$EUT_JNOz}FIY0Z&@c{Y8=Ck}SddV{1tPqsQHQaL6`BZ+(15s2tt;cl}m)J2-?C&s>g}8fN z`l;M8wt10nbY+mRwq^GY0;^2s2C<#Epf%t(jHrFbUkToB&pW3ozvn24Y5SWcT|J#& z+9+e_p)<1p(V3BAjxRxj@KGBlB6MqmDgLLD`U zB6L|~SR2N5Oj!t3DyM27D;TDIQEp}OvX5O3k|T?MDfbc|c%eh?%u%kQwxW@8R%iB_ zB{+3PQzx;Q5ZD7_<1rgFNh@aRShklhXc`FYPr}Fh+yp3NE7{sNh~^^{{5tc|{kWkXR-)RscOvy2LOD)^$Q!i}^NZ-mMtA&;P_O!?Z9?7owBN|vU? zF7MH%ys)Bo#mV-4LmNt(tfBGFwTcq&ilE6X8L^AXFpBWTY6&2r{ij|$=*H2P>gQg` zBQIjZY80iu(%do_AcT>b(Y)723ec zqAT{?z$pwIk&!uR#hs)bIS)88*anzDEkgtFu(dOvu~&?joS|kS);eE3<=r#emnrA5 z56jL&iVxx=>nvjOSOPHKH{bZoQjdnl)vvf-+D?Q@D(fQVwY|CZCPZhoh`qtR1BGq0Uxqx~1B)3cBkgE6ZV^fEKxpcc7f47q`(1MyKWoUq6-PD`w_yz~ zK3V)zH;$p$4MhS3mY6Q}qJdVXj++=Z z!ahIHcCjSmOzL1_ZREtWY^6-+Shl@LLsvP?PXcwN5LOQH)1^7jKRtVj@DSkCf|vKo zEvqJFU;+r`{ywEqtZF7Klk>=@f^P-OzK=|<1}+#1f;e{9$9}9yC*G#dDyw&zxxB~* z0M%)QF9dU12ycUte~tDqtGAU;p#LeIZJkKP zKNuk3^M~+UiL=7B+CekSn+GQW`deLCjuGDjXw%0pOX6K@cP<-cJg|35mCjMqfvZN ziWKAkbPZ{04n5zjE!eJN4=SC?xjvWxQRUCk?aNcV+ z$Tmra3N24?O4JmD(<)>}8wI^n`H>S7UGdEOr&tmEJYWkRSjtLySMo2aOK1jpC<@_d>? z>{PDwx_yTV*KyV^R=+1S%~8KXJ#-B2Cp1%jQq>&dP*y-Ni?-M;Y1i%*aony}iUdsy zfBHr5X#!jk8#i)#Ov~c1m&Ed6%L)?fkzAPnWqC&ShXy?_cG-Xia5NY4FTdxCktKC37;h5QYi^giHSuizHy8oJ+8-sapet-xGmK&piEg1)Bho-~`moPk!0_QxdW@3Pz(2HV4( zrt*TkLtR!?8yQt+sdQvtY6A{)I}Q-&_S>COhcFmXz7c_8_{i@lhZ>#MDa!o22ltHY zpEL2BuC=$<`yjX&v|nuXRB6}oJOM4sZmg`x54CA+S-YlbuM2Q87k^AQ$C$^{{V4x&hq&Z8d!3u*?d>4) z`H15hC&v5J;FTX9g+Vpxt zHapiH%arKk-4-Q5hw=1ITx0Sq;1UX}s80}By`B+?uqw^@Yh>dS>!O{T1q?+UO(m{= zBcs=XVlcfV1$jy8>ElChUr8`LV^(qV*jpn_ji_$Oo6jSE%b%#q2=DN1e{uf8OyDc@ zhEtx2!$e+1E;$%;s#t^qQ9YR)LDyAwjcm-h_~ zPMqz8);#^?o%m2VKs+5|RRdSmku6--r^S*i#3svAE8879d>aLc6O2Vjr8$OBPD?$; zEE!TnGwgn3>DbZ^aT43cr;k2Jyfxw#OFeW`+M&`^Jg>u=)94fSW{Zm(#Lc-$=rMX1 z6rlYK)+T!-cs^+CbkSp&BU{;o6(=BH0VZ2k)hk+7eg#75ox-VZHr7fe-nDP zUuXMMI>Bg`^Jq9j=_!QEPf3v{DJu8HXprZh(uS<2>_siWY&pK0ol0{!WJ%8>QH0qM z*JKFXn21MRwa)TJq&Jw_49X zo*?DznN{)MQirEpqW=X9gC`fT!&(~I=yK{S(fdXbwvtL~N4;PltM9qvqfT^bCk@Hu zIcP|K*$fR!f?NvCzD{x3;N7=kQ7kS+6*pe-CB;Q^o|c~@FYxVrBDF>)dHu9!Jx`+g zq8TFMdPz?cfZ8b;jEiHlHRhMUdB?+Ova0JiG7LX9qQ1S7iajdvaY#$)!yB*QYnSDE zhTx&Dg6H9US5qYt^2A0b0>8cu? zunZtnuxj?+#49ejltMI5QoCWq`1GY)O9Q*Dh>FtTaz(%y?RNlJJ8i)kgc~S9BvL0H zSL>K}9wjJVj$BiD!1>}ZV&7}Se?zTg7PZwCViCkvGRJhLlvqiEC4;?iRnlXet!)T5 zTyW~nVh^u}yIT`BIfi%6nK5^-vJM~VzG*7|TxtTAR0Fjdi0h161;=n3<5EnKyM&4- zmKt58w|IKn)%vs?|K8*?8mNU+G*EjASU`QnZfZ^Z@yt3wBZoH0Xs(8P4`ikiYXO_` z>6E9zHxQ>5xY3wT`-<9o2EpJSh}~=Y=zMnZ!YR3g;@8Ejsv{SF2(<_`etJVE++D6r z>fI+k71Q*^+-K>`05iODd3pE?buL$es=k&u3{1!o72N_CZwQKPj4n=Eono#DiAKC9EM)I_mTqqXB#D6tS_@7=yv+86MVuM-G^J zR20!=%Z@q>AZ}J(#@qFOC2Rk8z~=+7`aQb%&iA$qeYXAuA5#wqti?!gIlj?6ajks& zsE#H4ub-I&&D9PZ$NcvUX4a5(%v1NCmiuU@n*A@kLx1z$VNX^3fH6JYj=N`{a1U&` z=~2-xd`NZ<7`*Pj*+yY-VZ^KbgR#9Ij46WZyU)3O>4YJQn(Ll$X`%az#%XV0x(q#U z`Uj&61K4!#@4nkzy6pDzXAkyrx8%-sS%TSb-i zi*R6cNugSEI2QbWHz`Tb%a68>{qA=0^BK-RPxi^aoj{`k9V`Nhv!X-nR#j`zdTLNiY`hhBlTq%9rs zv5!9R=76;@9L?co?v*su;nCi%Hd#h0#a{HpRHh)-HSG@fJPRStKLrn##Xk!g1Y8uX zPIOMm1iXccho|BMmU=;c0!iemJSQ$SXXqKrT_-+C}c!Pde5L_O*ZsMZ7;JCkbQ96_3;9QsQY_IU3-QrDNCSksxdP3f4XcTb~&yrC3*))>dh+M=~Ht9T+6{{}>Kf3^bC_MF# za#rtm;Vf3Cg{dPG>#?y@^}aM+j>xFp$puc{nqzOa);}4F?RLp*U3%;j;%%3t2f}_S z@ya&jkwFx@icuOLbBNzY&bLZ_C!KjX#8+{asHrBd2J|z?63B(WRtvgEo&Pmo0Ab%J zKkGb8y48Diymm}&q^`XoPinn%0G^c8V0Da_Wi%qvJHL-R%kfhtQZQmZc=P>Ak-Ze% z*sVC1xSLp5J11su1diMU2F+24f-*<(?^^lUtF+*Tag^?fv|)m!gBnN z{%ShmV}w+UNW?5zKd}aUO_uaDw<;ohU=xoxn3bsAV#ho1ZI20d`#frBTR1R@?V$`(JwM>6eyo87@h;oOr@kr00_SjVzSepGf?4CD(PCTeS^tu|H*_E5pJQhY;Y;s4OyDGota2FcAXXmKISDZF{eOFfiHX zR%<=e7OlUXL=v17A7VdUlC7KLhLzWC;=RO<{A%|PM&o%05V=GGlh~4Yr%mldLV@5@ zk&Et7EaSlklcn!8Z;9=t(SeunJcC4A8)!(^@XVq4Pb&(0mp9~Ao0WB-ZB&D8{sZ75 z0fdht(0j{kgxcbeOlrYSMJ0ad<$yg@9{Mr4Ns)YKx)<76I#W^qjnVe4md*U8)Sm&t zR|bZfLqA;C`!>7|X*;*&mp((_YQCs&OEkFp*!7I=hVkYLB&y6axe_6B{Up&cuVail`O> z)17CMQOrs)IQX22&F=$Zo|c2W*7522vRMg#v!Qj8UC1hBs;Hb$D+d-WN(%v&kc{L) zKcHj>$J&}D2Wf1Gx^K_2A}2l<6b(|Jnl8X3l!qcC48SZx)7&_{m$??KvG0t+U&NQ% z?9fzmDlJ>GEYdhgu+l7(R!a$R-aISR(klJkXKOyrE>;zJ{n5`Cl>Fg_JZx2i=QuVr z2f$@_Bxi)Fr^6+_jLAVj+o$-sg;yLB{tciJJC$W~C6?TB))<48&7%DlD868)x94O2ZP20=&8~RQ4y|gmUTw<CdEJPP7%c-Xl~b|x-~mZB_paAsedAMHEJ{i_Y(uedmP31GTi0G| zLUilk(ooeI1D5cjJQxEU@M^9RST=l-k+fZodQ?Uz^Kg_LX1?1|{$FIhWmwZ;AND^| zT50Kyk^=@{K9AWAo5fTYyuj!}Y2OEbD9q*D-)^4Wdg&-3Dc{Exlau~&=V z@7i^p-}C&OBErH;gsq!F%bw5YM|?N)I6awJ^#2V(p?l^RizOxJlyFcRFs9lGR5j1qN=iCc5`7*)W&=1B4hM zP?0vOJcFvxG~Zw)&)_&xdc4VE|7a_Psor@wMJ>)h05NLQRk$9fFp49L*-Y?7?2Fuw zzYr}S3s{*Io|V2x{bY(z^6Ax!qbRiKN#%Z4)xYJ%t-$3=BFiSy$?o8t$wjHcdUY+P zsClO_nMnSlmIgW(yV-(_hS`5(eC~?vMEs5n)3yPXDu+u8glO@%+T~9*J^qT}RD2im z7Gj{Zb4;?VyLVr2b;Y&Z`nu=+Z@vU``vv($ff4=K1F&m-B&?Xfg5tt6aHu6XMawT~ zCM6t;erWN}KR{wl@;|_rd>P!@E1|~6)bdWzBX2%$wV9eyReLFTAtqH*X#yPB=hZaz zo;)d{xERHdrSR)w*LL3nN>B@bs%GxBL@t}l2UJiIJ6gHhJ_yDA!Sq<$R0xi`X6?e4 zkMgi?==de4v~4ybm|@qoC&TiSr85LemyM^O{8a<=>=FJ;chl-l76~IgE@zQe1(r8lAmg9W9b1-cm=;PZn zs?`=>Oei(}sO0U(mPGXjl=gPvCxwwxVu^J@(bcO1o1pjC-shU@GbLU2aRW1a-XUxS zTUj+(!jI`A)DjC46+8r7Mn;+fO`y{kaSV)WoBdaZ(w}+`Orm+kmdC~OA;r4_6%IVk zKVFZgMLLZMRxgdp?p6swpN62k?|H5gD`_b{8oaIOXd#%T{q5lU-pBw(oLT&#E>TJ8 z%9GVEII&wO|JC$M_gJaWbk<-^Tfes<5>;9Wr3JG$zPeCnDabotpW^Yw;n<`Fr8|T9 zr&{n*#(XQY+&f7!qvM5-Q^mwnzf=}B{)7~7H&Ow`rQHJ?2Y!)(HK`ffl>r8Zt@#|d`5Rh+qVD~Q9xnq1*0Dt+g5x$xg*~;sP2*it*H+dWHEU( zNzHHe1=)R5?2+P$=&)Al2r)uSJV^`Ym{V$DctUg;P|^7hKn_)(;514pY!bW3omYV* z&u&wS7wxyHb_-r`UMlC-!LHHvSpWSdw5q>rkHT{w;{IR$E9qrOR-BP~m+2jc;fd}5G{RLR} ze$Dh7^zFsq-Qwv#VIl-?0ZohQZT%|#;3>FYA)Q-hv@_Glp`w-G8?25!Qn(-t{evH74By7KGT_hP1{xl9(F~>OP8}ojcNHx^fSF-T+ zpr2sfxCqoKD099_9FAA)K1q6{q~aOZA3K5T6L%i(v-H5Z<@UyDpe(0kktW@-O)WPd zFbxJy^AtsFP8XH-JSm6h_Ok1I{NgxEr^9-{+B+{zvNh!LOZx1m3%LcTB_-cZY(c^> zqdqO@Iuuv9Mv{ZFTyr2(Kzg-$c74U;0>i~q^Po~Z+vD=2?a(%;BrO}{gFk7ICLlpd z#F7J?s^%*$itNGZ5 zC!jHm1%P%gZ}G)#f~zPNnMr)DM|TF=B&Tt(J!C)s#`=4oPQPCpJV|xp57S>`(vA1w z+qv78uEn)4DRYm{>5g(-L^?07cnNjQxlx3iHl2U_g1^RdZjpdYK!dr z5UD#p>}zv12cs~Bt=t;FHdVc~i8jGgcuA6c_E1Ps9hgRnFU;`fYm2F^X+L}5NF(6GF`wg+o>~$!xDv# zCu;5Lov~ow_Toa=mUgdE-8T+30)g*PCjn|ybT5(s!|x4Cc*JSXi!IyC5KUh_FwQ}M z!t*pVQ0`YZF<7PPMRD&UaSv`xW3qa z?!57IGFW-87>Zjl@0l?p%6!e)RopUhwWQoMzCH8cE$Qz{H(p+uO^oPjGD`M6DdA0&6YlN1n{fEoO*|&o~ zZ5pK%^7Tf%73+X=xs?O8!EqfOjpD7d{*5~r$*xO&ITtdnFVEsq+Ok>j6ggc-s7XnU zmF@5lf|{AKq+2`1Bko3awO>>9@fPl}pW!@8K-&CQpN}W~UYx%9tr*(5#X3@};s#>s&%4j6H)9EldC%jGoMqAbyt!|2i#mkOVU};nu>$6v2>Hs6D)ND^i`${iHzZk9qQuaszBHozE+x@p&Vi(?4cdNA= zg9BS7At@i;XA&nR@H1_e7$w5jKo=^Yqe+QgetNdz#8I@@L9#_yvq1twsRahqEMBUml$eLt z^RPP=1CsP5gPl>cvKgGyQt?*1O;-yifQ&jRN5t3PdC60z-`?pO3lStO)>#Du>8v5Q6R{tNNu)e;{on>d7KZfe~#A`>utN}ZcaxtV!E@`}FDvQWnS?sM5@WWWfz zuRcU9wC4Ofq05?IW3wgs*Pu4WMlN2*ct3ZRsR!X&s>1Q)cW+WS+H^jI=`=+~ajD_( zFz0?KS64_%SAfz<|Ma_73UIcbep>fbtCrc7kwH|Et?3CHAJ^DWv+u!l6?|Vz61;$( zCW|SseR@F2?9#zT9{iA7`fuYJ2T&-eBz?f;clo#eOR)SEq@Nq6^XYlHJ;fuJPd4eg zvVqQV6IR1O=Of$Nssaz%iz!5;q=uHHbpPLq^c*w8%o-Py<$obOj{iw=O8-bct4glRuVht?aGU4&j$&R{|1($74=~@~UwHBl;D~P*sBKsU{#_ng zAiYVCeEU^5p4H$#>K+lc(&F;9_~psF4EH39xBMruiRf3*d?w|d{BsO;(!KD5$qm#Z zAQ4?V$HEn^6;2OO0dTIY{G3=#_<>nkc6WWbalRVHcDslPCcc&qzWi&2sQ~!>5Qao& z52eVz)NlU!iY5R1@iC^g0l>Vzwf?3wV10ox^Y0H#*oW>K5BK@D(OYy)-&47NfS9bm zUjb$J{@Xskcj!W2VBkH{k4b4WT zT~Ix4Qz`ZkmaynDxHRx})Q$%$lWfyt1I2_mRkIo(BH|^g;21<5pF_D1U(t>F3q| z27WH5Qfa4gi@?DL*F{OKBjk2>7t9eo`L9{PZl&gFWwVtkrPHm=iP~?vw78xpZ{G6l z6m!#mum|y{n2k`^w-ng;aH*w9Wn{|3Jd+yZ)DnYUwD1p9`~oentXPLU)ofe6=kjSB znRQ;kad7GRh*nw$ns$7?AdQG1s^UKe4Zw$j0DMI1~DR>m#cC`+i zYHA$U%zw=JebjCeR9sQ%tiv;_kWuv$OpWHe`Q!H9#rY6w@Vi zP7{kJzma=IRmr);7pnNJI(4Dn69s{c3$kH>43miQi5sIwo6pG8rxCT$h2m0I1mFnT8 zN#&6w_*GSquaY@vIoN5dTB9tTyU3TMWDmeW@!FcfM+7e`wc}@qRW6Fvtew>P6-+7W z88nJ^4QR39kwDkl4~As}6_SG=?gpEVY!K*62uIiT+>a&)U`~TwVTH}`BrEY@l^VF)xp9^SxSgG_s;O|RRY-wDy14Zd8}If8RXMx zX^6E0o_NhUP9>6h@As+AJ5}%A^lkFeD3L>8H|pmm&Ey zy6`>*o_WHDn0;sU>vrx^)Qwq1Nf}5wwm1K_%;#nHAW~x|p8;00zBGNDQcv%2T<*gk z?d30PDzw2(N=o&~Jpatg=Tf!wS0>VdS&w_1+P-2FqXh zaJ5Teu-4+r_QZvzCQo&)Hs0Z`!uAJUs`GEmxtw1YyGOEY3Kd97S*-ipwu%O7uO!GW#itXov| zxRX6BpXzE_a1Y*?R5%dOdr-Cvdr8Y=bDPfG`&Kqss-KFMGuR9FPGz5sS$7Di!xg>k zpC2P$LlmXfUaD|a-MxG_gmi*}im0{dk5^p9c7v5v=>s!ZHkW4iSuh4T@cb3XBHwLo zkG}0`9~5zI#pGd1jzr4thW;|iuA;V0$?VZ^o*b_-v-6{1vkv!LaLdt;)q3dtaFF}f(=u%(z^=V z3(NA10Z;t)T@ieo>oi{bec&k?kACKM)^3K6$9?wNEtge1of=QeARBfPG$W)*scfGI zS$-057jZf#7W#;=N^)}Hjt|cXSC@9tRC>soPr;})xZWo)(hlEHz`5&ooUEWyd3u{X zZcs>X=q{U@`{PUErWvkzhmUvd7FgL@&ybGz`O!-LhkPQVfWOS_yBz7GkAZ4` z8Ai~9xv&zZ`{Vrw`~{O}X4|%+0*Mi9ieQu3MQCBSM@L~TFHuA4?Q$ZWnTyxZ9-e|i z368=RtK3A|;4j`?NBsUV3%Q{Swd?d}Zk1_hLG`c<6A@-f+XeSA--%0-vZIH3GJnE|^eLUs zZ56HMf8@9FiRy>BdFC@pqEo|e^V;9OBBjelrWBNukY2Kug z8}Rh%0cZwvW^Z=}<0uaRoiXYO`~2*x^h`Oq+0{_u4sqa$U^Ev=+WN0zbgha-dfVm9 zN^jLq2C4zCzor^nhF=S|JeQ3})@a#PL^e)174oiC55X$^9U1!K9S!5N(h0_A)}eO+ z*tV8|)rN03-EUmpecGR@>RE1LyQ$KdG`b=NVli@E5#*7QcHjUi*!kIEM&3=uTjmPO z;5-u!rOL&`iYFyy&d-`q$E!;jVKENR)OPE{nY9{Y02^l*i&3w_^ym~Yh*dOrRwkt{x^*mrgacd;i>KeX2nqJdGQv)E;CLf~ zT9HcruaBUqJ@)E$YcHAF_GcUxcYQOcU?otzoNC%6l2Uy)9tR7itm0Uhk`jbLQ>3mX ztBh8~E`m+a5a(NRxmivO1RsPBm#7=SRcGHTHdnq_0hqn{QN;vBBxsCY%JhS|$Uaok zhc`I_m+fFL)y~FvmyY@Z3?QQ92N1XycN~xF*R?km!!s`P&@M*_1A1mPu0e-%Njk!j zYwv1M?U3N>TrwYaiY%L+nUlkYSyAR>kCzGFV zPhgSj8xxq`EU{x)+X%W{Gh;Qf4)%p64GPtgGw7ZPsK+jTY&kILYMVdCm8cU}(;*3%HPYxm-UKFms0Od8@>ktiy|Muw>`y6w z!R^@Z!5LqhNMs={ZY(D8=@Z3i)`Epg;f}}GIaIC)2|gH#N!^-2FP4^}Qy-D(S}j@& zUIR;6_AB8)b#bx8 z^0wLGcyXq{8;&!s9I>+HPL{M>3r>5&<2X1QU9zF7E;USUG5?Lh>pf^1do8~df>{`b zZxO)*`GZ}Gi3!kIO?aoZoX_{S0LIzp{%a|y&wcv$pJvw_dwE^g98qow>9=hfo02Jf zE_$%HsuYC)LN8^dbRTaCi8kE3lS|2YRaJ%A)&S@8!Be*G10vrAE>+n1?C2#!=2oUo zrXGQ?>++k>0o>w-eMO+x?<1hFx|@)&dfH}>6}-hrOQ`2Dz}jD1ls_2zs7 z+%xj-Np**JMXcy~?~Y_t4a%f4Yb^V9o4$!2u_zi#GqbH!zV}jexPi=wF)589 z@gtea{D8x^mn~wEV1Mog4TvMAKFCs#0qvEFjmMSb0IH^su6u=Ya!WLWJDVu=Q5kY% zv@)NaPLGqDpLYxKXu(93XkSeanSAV()t)4?EXU&?&B>f2?Y3VX&st60XF+}k$M$f`7daPuvIQqj zIE2@P0{fwwHjoY2R-6g138a;~)-<+*9SWd}BCp^+Czw~`q92z9nz_$3_q!3-$|6?f zczkW{qfun82B9&S*IHZibSrABX&v2-tThO+CfaWIu60m z1g;+ULy(kL2IVbxL$l*4)lF|sdMvUI8Efxe?&3JerTpZYZ)iKH87`L5oJiAvwP(`Z56UfBrKJh@Q`{2eX-rzg7S;4;RuSSAbC=;l{$g%wu>o zI^+2vwDwEEh6nPo9Y9g)uCiec@LK-U!1a555ny~_`{g;HqA+6ehxX2^G647gbF`s- zZa69drx$3qTRG#Co&R{+_Wu4a&}sXL5}q@a$ISe^9Hmup{DQRf&Gm-ub-w{B(IYD7 zow#hzkCaIOnew$HpcJbj*X&RhomA!7j<`kwfSemA64WO{lUG^*W-_(K1j}R9?A@Ud zv(ZrinSATJFNl-yg*r4&%y@ZdFMRUW3_yvt)S&@nVgJ!9$tr3jvp(Xo{(pF4R+tn5 zcz6GoDz*uv4cIwW4G9asj1#HO<8K;Va!0NO#IgVIq8TlcVxDi7vk8)LkAhculc_zL z5C*d-Cn0(5v!T@36dC487qF6(T-+*-Pg|jTHX?~X$(DVB z6A`{>1#fbwWgQwI>Zd0RHtP{9mx}BR&qnFHZ++DTR0bza$;&L76WLf&nsga#lc%_^ zKG_d@t(#a;lT&0qrKEJae}u{H$}Jy1+R%jkqE!kbh((#jG`2z7Y%;XI`zwGv^!mho zT!L}t*J<+KB$TtP!`o99s6)Ge5|0#E`OJ@f5r$9oBs5l{`#3zOKVhxQ)Qoanh+?$L zRQl^ffW~jD7p*3v?G0M>vG4J=_L60g5>!md!ca6F;bpZ-5HfYQZ|838t58fe z-SaG(#F8urCp~Ky`PV=kbzNd$OD)d~bJ1Yts`fLM?@|R$5>?~kt;1gPZu-ga(>BY6 zc000S6Lr{LlQ+$;EJH9`YxxEwa*#-TGm5}8eY3q|tYOSo+1tmucm;(R_`sPLf&bLN zax@OY5TpNF2&5n(e`iBU2A=mYf*MXDS1sPu`vp(qS-Cez(QBCCt2qc`c(pXyvXVXu zzB`Cr_09pCVnpZuSr%r7I1>#zbzpQ5zO|Wa8w|m7r;%IAW&A)-aMvLq9aEsU`l8y$ zl<8@d*iNLG)%!#EdhaSyb<*}5_P03$I;y|NDeNz}F-d-?;W(v`8W$;Nr8vS87}SgdGR*p~17o#qyf-#r07@d;posH!0<~6}6MM zDzzpG)GLgxG$>sj@uOJr25o@(O2_)wGj|N>^$jh4%6qP^&yniehLbF7ic?e>+$$q1 zmkm@Ko}Uy#R?Thr1vY*NEM4(0JP$O9APExS07_{sur6TD2Cx4hPH)&9Vq_7U>yiC@ zjvHTnadiA8S5>9MPKkXoYNpA?-L>FIL@&x@T z@D*7sHXVl&Mb%eIuD=-->=x|K>>tjnH5-*%T{btLtDEyDU+CsKuTH}HVn>!7ObMpF zQ1Y8C)#dIteF$bdza$Ah2SJ ze5j!0Ar{zPx%Uu7uVCWTd0=+675AM;A#|35pyr!_T20Sd`L2*h^&iN}uK97|pl|UE zS*D27LY4R4joY1{_otSGofEuQC+ZnJNrGsy<#n zLSTfgvD{9q`HeyS;a5j0LfTXcg`7W*6P&UQlX_ z>j!qG?uNupFt^HLf9$N1gM;dPY=QHYo}pvlw?MOP&vb`JE`oeD2)fd){aH&4x|GWb z>lDJs!()p2t~<~tR=O>1?ZFj#5pnqOTaWM@p*5a-v4?5w2nUw%kun=FkIieq3p=VF zOkqe-PN7(2`dZ=M1W(G2%1r$|csZX_$#Jx{x4914vdW<&l9YCvXB(mQM5Mvx1cl(_ zJ5qV}K8$^AoL$M2wI!hx#>UEX!5VS>9mfjsnsClea!CrZl{&Y>p>aezWu{o6rxxS1 zAM~9-(WA0O$afsPxm5k;ueIlpXjKNqSZyS;cQz?awhL(G`EcS$OK@k2lehG!E{U^h zuR*2J%D@HXrX)T7+m@{w#E^cm85NPzi=wgMZ>pwGM6wT~23UES4Jqm+$4+0_?f~Vc zTCXUo6-{&kp*&z1JiA(41*Yfmeot${a;QOoG|{COT!Uc3zexQNwx01sCQCWr86G`O ziBj}`s58gkr`F|jtIDEmY}_1yi!urO^GJLA9*~l&`#6ev)+r-FOt`qLUk?qU)7iVu zvwu2rt7JtIU&O`%!CMSTCukj}Yf1YDn5LskP2k+y3x%f{pwCWbBjaX)$=fG52-o89 z_Nz+wh$GsSlha7vQa47zpmIKbwh*$ME(|lXq_Z~WcV2b&lm?Qmd*MN6ANLEsy}Cj# z=;QkT0RI5s5GK_&QtoRlm>3yZUPG>1Pj;FB;EN}Jh~=+Asr0{88+}qw!&QH>cQ>^& zo#`WH(yM0)12}Dmzt|5A_!w7O4!mSF@$Y?RJyR znd8|~UxYls_KJ1!zkiik+E0h~c1t<#wQ zX*8X?o(icc1{D|O`dl8l8&HXw7vW^iw-LY@|(7hs&uMrd86LhnOi?nkWk$DC|FCq^+o*m z!PDb;oW~CioGqU=>$q1fJ=1=&9&X#hhoom_Fa3kD)m7-OcQhou^WrYoHvnuOWT`vfH zuIg2A@b12%<4w_&a1p|pB%<$_R?u?O!W!KIr)}BWC968iWc{Rm?(J%1RAh$|;xk3L zV&`bwM?Z3J^*Vq#tQ0ntT25n^nWG|}na@Mjr9Ccnbk8^35rs^d)ON4bwdbIwlN&3& ze!FQ|pQ|$RI4!P}V$XlGkzbl?362TGu>f38cs`{+1;qx5V&TDE7l^=xoR&CUNas> zHjmPwAd+b>+v$U)_IQ!AAGTMv%lz#5z;p_6IG^-c?}|O$gdy>#1O>nIgL639&)H3U zT7CWKfqeA{tM+2w!&EN*ugkR1SzOezNR3G)sE6!_S5%CBdSYFJS6hxI$&urkSCepB z8VwDtdo`sw{Ax-hy8tKz`+6<13kBXEbs3c0jrJG%CZVqh(|iR~0GcEsNq z$?(wkV$chx`{bw3>XK|-aR6)o2e+tbVbFiJVk^FtJYW1)v#iZT=qC|`${99zY!P2q zllyN88OASIH1nNNd-(xMRz1a6_1up0>XcO)CM29c%QI1a!o@JMEeZ@hcjkMLA1tcK z3@xR!xelyt=Btb~pZC{o)HekrZE$(N&n(Op4lJ=;&oi?u&PULV413T*bETMO-)8gc z1-s4C9;+;OI1_`;Hl|?k4Rzq3InLnc9Sw?NxGR9u_vE%8q-0B)j@o5qB~O#OY;?c+ z$4d*ES4P_n0rzQz;gd|J!Ca1x5*2xFAF5|6;S(Iio!fY+Ln(eP^?!AMd1t9P;}rAX z)LZjfw$fO*(J63O|9ob$sRjtVY12)1jfL6f+g5s6{;XP%&Hnu$Ra)L2G`qh9-vocj zbH(ZK$7xrSnH2*I zy*y>u7X3LZP}))-ONOC_y=qQqZn|=;FCgAAfDJPROa1-iIVc{eUfk&xxB|6#yUaiK zEd#;Q?EhU2qQBg7XspA$B%u&5ka-0Z`pnf*8Ce)Z=hORJ16&{a8STXq%!6{ewFWVy z3Szu>nSCi_?&S>beeLyz6dauZgBf_|6X@W8`Kc+3RP`p0kC1MVVp-JJZlXWuG}oGB z)9fwu9Y2I>3S0PXw_1k{hS+%q6mH{|m@5uuOG34vosF(hb7$NHRgPps5m_>++g63J zl{^Y|_CqQhD)Ej+Cg!4$ry2%~?1u%_63%3+S(U%Cvq!#0VMLX zX=OhchWkzC9jf*tPI|=6xJN%5X&U%a_ht7N@u_2U&7MIo*ve~EQvXDW6NUSS$u&f5 z6%pVB%VQTgF&E-MS-|*eEXrnVv?iJin(>Q1$XQ;W-4NQ#PSE5G&+38 zYxo&1+h>01XQ5UGI?rSAfZAjGRL*fUE*h^XZ1A5T`DRPlkDGg?*%o;_%4M#;({mB- zeK{0=RYWX-@q$7i4$mH(tixay=SQ>zlDFP(B2y&v1up(78gXmdPSex#V(R3Y_cM4S zTh1gPk#Oi$7bHfoJ&1$1jg`lSd88{z+C?#jd=z^fB|=IZ!Ghe?D=A;IB`7sM<>&iM z6JL>SXiCoD4D@UF{kSB|H;6*51h+eUzi0-!_iTObN!m$ zXCTw*#VhLC$yU5rwsS>mLa)6IsX0ZGT~QO0waBBB=a?32K}r-AxgV@u&{6YPU}JY} z8(YVTa!ys*@X**mpVGWRLrfYG0=4>ySIj{w^kvz6J(c1c7B%(*0&IkgyWcax_BF-7 z?gGyn^;9xQqi#nFzG+@uvz%~<4uyl(eZ4n zR%&4(UlbWUvZex{Mu;vi1A?T+>%EKRkzzS+(6KAQ&AP|Sb0+l=U_in*e%IF9Rijn! zCZoWO0#3wo-?I24@26*M{{XCEWH9CN?~A8${NHE0F)TQ{K0BvWyrIaauOG6=^!HqN z1$s!Zm;3|fLN@7VUM6Yxbzox)q$@T&$;Dhoa{giegA^tvH^>#v$ns#~x4%0IUF|8+$8vR+#zh#9yVayC-GtKgaxO$6TOpt>F_R|N6zc8!bQ0 zVv4%0|9Qn z*T&x7Z%VvvVIsZ%mi?)h7~Pp&95-N*Vu;_@-=g1*kR%)Vo@bKp_SgFlc>j57q$Tro zSm>uxo!+_`pKv7RfB0OlQ>+rT^S@>oF29b&#{~c^t>X*Ma=!8a*^ZEl|FNI}P_@Xt zS)2c6NHJ)&)9!5iMzJ<`F<;afGd&fe0 zte6VK$APEeZRoH0wL(n&D1g*Spg^C<^}S|1HrTP_$5MFu4JP}jlwyaQ>gLpY_Bn(qto%)CMp#H zO#YhuZ;5xz#M`jwvwi=#7!O{hbFh5jcRKBps&V5%{~{B zVZP%qeIM^G;NgiA``j>J;M0p;#j6-D$fqWAo;Ul4j@4D?F(Rp>F9msM5=Gx){`EqW zJpo>-kg43<(wa4U_UO6FV8amw!;-QtD!)#uCApc{nwy*PHL>{xp?c)Q?T*w$?bX3e zD!NOa{(RKhjo+;j=+(ug3y#Gz%rH+?(0^8_gJuQhMs@C=ygJm{kp3InR(xP6T0&Jf zPCJ04fAKlRaI;wKnxgNNJxNNxH1#z*MYU#)Kj8Mw^_RCsFv^P2h8{$nqIMaB;ptKp zTd&NY!4*|Y9Qm{m>#@2x3u zyLL-v?bI`ZCWg*dV(iD8KoX<)O-rl$gZ2d8aA;jX_SDC^Kf8FAMV*bxRtE*+o5OwSf z`7o;XO?rKAy>oqeV*PC;Pp=|R^3%tf%q*%!8rQtf1Kv!XPiK8r6Y`O*9g8QEnF7z@ z2J~{>=M<*me?r&EWp?&X3+s#&WvMGkD?c0j zeGk6%=6{pWT6lCZou>kLcp!K8sFv!FFMd;#)K(qKX?qYx8;=V6wl9Ngs}ATQ6YAw5 zaD3pSCD|$6jkTc@hfAjukR6;{hgw)7QL~4YIDe!Vg&~bk5><9~C~q#ZJ%NT^+)!(v z05nMJ#TEq>-akN?U4zVYg_L+&kKh zog|~^do$lU4U~5wXIPtc^LiY9u?kC=EbB&VC28=g;k%X=GGlX?GT4Lk-AwV-%+y`g z9Qz=Vue?+6iI`jH{7QEqBX!jT5+kUFB78++Puc{>eKb(22&*L#<(I50O`(daQ8jJs zo}XT4Rou)b>?5hfQ_ydU5>$6-yU?dgrn2yJ!gI499v5SdMx!nlINH_6umnF>btbqIVlwDD~amMQjmWd6dlu`rf&O_*Ku`+T38IO zR?vx7h<})SxBM(qf4Nk8Q`s#$c|?)GT`egr%BbOtHEF7$Hk!(eI}f+%x_|0ZpkAak z;e?~T%fx>AtXjNK{#b7`ys4T?{YSE6ODM6ha zgm7g1^FG**7vqK{!r3mLx9n5`vMCYPd4!;$5$_{8fkaq)RyQm){o%#@gbs zN!`f52hYhiTMrTZ$R!CC(n_gD2=5vVC0Ym0g~l{Py32ODAE5i&DOYB*;{HL2f++40 zM~>+_*oRZnM@+Z!8*XM}I;;+=ITz!19>_6EI*YX(DWrKzEnGk5LPrLS>NKRxZzndn zzIHTHODJUeApEoC^8Cy=8TV^0nRo}&Gi-0cBwL`Ia^1#1fSRASmGeJ9rbeni0}VL* zYN+CJNAc)-(}{NpV*Ki1fj@H55?zp&iW0|ie_P*HCp23=+^Spv`@#K~GGw`Y8K-iq zQxmiK#ukZfzAR&ZZC9pM4XXNDre_1L29MLc)URp`=T)fs8TX4v>{f%XPpM+}FJT#^ zO0@iv&d0b~xo|glO>CDoTj@iVvxe#+BdC9>Qcqb{@CMSHzi`OFcD3U)jd*fgInID{{I zFFn&A9-m@by#Hl9jQW;G+dT6h;BjoGx_BwA5T9eeZGD5lDQ&rp{{3=TfQ;!Li^{AX zRnxZn_c4<&*nkl)cY8*1NadkBV=b7@LbSpZM~o7|5KUYbFN!%zu|B5XZah{%eROnLY4nZwd!+9lHF@b{xj zex&%_N-s^u@xW|!)GSdj4G;uu*Rn0)P6FM)@UUI5DZ<6CkuRV%B`#jqYTY5xudafe z;CD8|=rL^$LP8}1@nT`eFh#nPE@SA{A5Z;tOHnC+Y7v8_v6QstSxMp z$knsP%Yk2}21-FmG5nqxAJm<46QDr>=O5)RFpMNxuVsuck0$0fHSTEt-vl;W!EU=XMAXH||$7E5JQKyb(61l8bTB!`wX@?}C^jj&)!S9pc zrtIPk5$A9&+Vi0jIT^kMI`js#TBoHcmm?$0llN`xS^K1(h%C;qpcI(jKDFc#^}1a_ z9<1EC+v{_d@y)Rd+Kl*qaflZkQsr6dZ?Tk$&!Sc&6U#( zQg@6f4#A&UU&X#d*DMJ@)FcR|WX`OJPkfxNU{ybl?EbPPe=`8rJifqeEaL6(Hax7tl) zbR1O91w^oWzgp)w%GV;IuUNRvZWEzV9&_u7JB92#GTcE9Sl)zl%39|8el)fAkP^|8 zwjm_LhbM+;N?;DZo`0g|(M@@EIAJp1I0*+cJCwa;J=GM{WhH5XE=y+R$ka!r9&tl& zBO$0|g*PlMm4-=Fs^_X*{udtuYU<;4n|JGn=(HHAZIRFCZ)lZ!>y&G~-Er@7(bg03 z+qia_<8~wpU$DnD6pFQ)?$^p~|gs$TIk*UI`ab?Ttm2~Xf==vcLfYOrA z9&@X{$CB{pM1@Dp#N(>)QU;ysL-OuCfA0OpTmOJHyC?N9dzI3e6>C~Ew-)SoPq{#} zNw5HRa^JE$&p>G#2%L+aFtmNenmSKT$S=V4@t4a>q%^pfwc#`W=M(M(;~L5Sm~Xsu zm0!%vFm!24SILL87Oj&P>kzNVg-7fYM*G*&lSZxk@HrBle6sr=44t_c$UBqLSBXoa| zH(ns#*!+079A%Lu2@5LV*x&|~+$O%i7vAFbVV;bVHKOI)2yuQNa;JQk_WaP$5M#Xm733iy4!=9{i(hqJ~_kIhW`d^W0x1R?=6M~qSASs+e=zk8s|KF?&Y75mA9H|tw z!LVEZ4CMU!%QX>V@s4dduWCa;L=~vAUm!UTUcQV*iczP+nWi48qVh|LMGZcn@$IvE zW)HJDVXvThy&i>$v9;lpg}!)h0lBgF`yBnKQPuEYNl+8p&zVt2|G>%3 zm%WBIQ5MqG=EXS z7yZolvTZE$nxuW^k1p6q!n*|>WSqzHw1c)Nu7gF5T!JiwTQq&`5Q1%fa-5LdPK3>3 zF$e+M(31@`+1DMfA%v=j^&dp8Nh4=aa&8!66OC62awWLPGN^EP`u}Rna%JVb+vG7C zt@8_8>zBHAv9w`eW=Iwars+=pfCg|W`J+w&QHjO(5wIE`ib>^>zH{}6vA%aFvSD_j z3IYyMofI_l*UU*AqJ0fUGf!f=8=)+n=~Zc<+I`m3?&wtM)jx^x2z`awC&E7 z3IIV7jR*nIN@R3UJX8)Ghi%JMau%TgOvim+A!w7l*ABKc2dSKw(h9_CdhApy_e_)h z-l7MG)lzN;aBU^H5v(5Qz1gy>ka3OSh$1qw_(wD6wnJ3hEQiQ$J0<96GUanIsbOUj zFi+>hyB6BTr*Iht8U_IMdsUS?W=4Qob!WL3ZXqda2PmjoF--P1r6X8RgjT zZdCACv0S({a=^u=X(TnXH9_3ow_CBOm>!>W+3(5=EKeUAu?xQ{>jcd1S?SL=3ao&p zyMnw%R3upVaDvYC#jiN8zNW5bL0>KmR&L)6x#~~!MFc&BP=u#qLw}N?TZfWhAqR8A z#(D}|cnY&wD85s8etbD-UU#99KOb|}r7FO1cjBZ-F96Fz$W3I-c`>o31TrePw&~<5 zT9#EByMi7H1HSHu+F4x>5%xC-l+|s@zerog0iL0~pMEpA$Dq@gZ{VEDF6U_?McNtv zu=TjaT55XT&(>K`atSWEh}kKaRcsw>9B%xR${C%45lEpi3vi)S<7VD;(Yj2uIhUbb z?4mr%B2%o(wDPy!AEbE(xJ@{0tNw17YB1+pK&n72cQQUY+I;A;Wpcl?^HAY)5M@`$ zT;j+L^;i`wqA>{E7koZ<<-O^6?IGQjpq^;7H6JnTcZ&iFy4JktUQJvLE^hr_z)2tr z`ER!+?T>-eljQn2jE9gN`?xkblwHmVQuY2|z&0<*yqV$EF8Coy-)pnM*k-4fXNyj?l#Rhz374(aK<$!goB=1ThzKnl$BNHup|z$wdbZnG_ws ze)G3vY5=(|3oRS=d?=TELTZy6`k*>N!(|V4)Q5c-X|ubVpAT=82Y(XFqR3WKbJIN<%9{cU;;{hFk)fN6^b?TEwhmxTAK! zU6}ngFE^*jgim?`+eI?MiSdpJ)MSrw*MWm%t+f4p}TLWx>}|#$b#}8iHU%6{SNID2A&(yD2cy3hB7$_gAHn z70;XX7?g6BiMk@CY5;{+PPLBCcD77EKs(&!(fYl*V2`3LEce^AgKOD1@7xMa?P-NrR}rdP=mmqCo-QHuZIL*XY1$&P{rZa#e*zoJQgu2P!Uv7cvf#(h~B= zrV(!8sH&7)DW?uPT0TWq4NoyIbeK{bhPY4%QlN3#h(@pJ47cQPzDZ(u_xL1=nHeY| z7F3b%K}FD2d-he|X=<>$G^S%8AN4_70kMTvZ#>(Kfj7@|{n0x?(*t?ptyyA)G<-Awa((j@8T0riRY$u!gh1 zQ%OGd$IVgA_l_W3AE@$lZ(_YV{{0?Bw1CO96`hK5TznDIcQIso;BV#l?_A3lR%;sF zA5oT<)iTOip_{yCQHKI3np~5n$n@p!Hyigv{`UDTe72Lq|HDnS^A8-Xgv-D~vaV4O zsne}@254HW{j#1IZNMO+Z7?A4NO+dhJ)e8mleG6npyUg4HPb)$hb~++cGVR&#&;Z} zE7AUek<_P1;AD~XvbBc5zLZ|-+swj#@#SmCgt42YqBqrEio@~x6#I1b$QUfc`D#|` ze|SN~#I}y3EYg+tPHaPCckfe1A?X?161?2#u$pi7O#PnoA|17Q9rdWRgMv+}QNJfC zVM1ZEO3%y(+5t@2{Ph(McTayvwe6)z;@<|@Kp)sk_kGkHRh#kg)vQqrz$kr$fKo$C zh`Yc7;Bg;J5I-+W0c3tYI5~}=(*)?m6+}-s<9z~=5fqTo&m3TqeYwwX7f!5bx3i(2 zjLGDoMz{8G0JyZ_KFOPJrrl$;wPoY9#}z16n*+KOZZ`ObiQ3wNq_{1~-6ynPc)|VB zX|C<$Np04W2b<+wlog7FD;wJd`5lS-&f4i6={TPAN|n^Rg-JN(ltH4>=J<+tMcT{} zsqbXwMX(thlO~nhR|m)6@xc860V5XOeY4G}2~vMpzXO@4PZ! zIIBEmGp1wz&fk5vF}?;Q1i-D6U?3<2H{TB|;HTAFVJL3yUJ+}ah$PjFMj*kK*Bo^Q z+#4&K#)x+-7_iFwJyFW4gB^cP{V1^n1n44Fb^&DUWywWb3sDnZ=$(l#@B4sq&)}?= zo}kP9*V2>(Jjzl9!4M9!kg$AU`Vd%v=(JH`obGcVw!Qy5FdHhG*U9I*0pw} zDPadVfRm?8F0*r}^qkhdU1vFtd?DkPDO2%p+Ed5MR%!gJmS66BvpLv%zs3pDX<_zl zc7ckHw%w9g*GYI4fFT9nQa`j5*YO&>p5><}nLd?sud)c$h#nUdX8K|ZePniIVqL## z#oynWBAowMCp4&&{&Kk19%WGSSLJ4Ru~`sn6BdwOM2x5(OcoI5y_(jKdxP=Cm3Uk4 z5#yVya~APy_u#ehbEa_c6-;oFNwnakMh+jhhF^!$JJG>_m4@p!)j#sqYA(>`e&h5K zUg}&$l>XJfZd5iG9j&Ch8z2tFF(Ilt;Xpe#hz42a_ix{LfV!7#YVA7Ro{s#yFOPp) z0%mkub~()Ec$anAn0*2pidDu=-}QANA=AFD-&*-b*P0eAY6@Knp0h{%fZ~idJ`8ef zgdmx)rp;Hw=hp+`L#2LN0LWp+?TOfrczc( zEthmf5c%bS2)c5req@Zr+-`XM`VZNhS>Z#K^i=iybq!gXuaeT>aPf~+^z@aK{0!8} zZuG=zcG%Q8mFD3Nu`zyeWMXo1EF=PcK{O~An83(N2z4Ga*Ntu5jgCeN3ek?T2Hl>w zt3-n{FBp<}GTv@;$;7_MJzDUE;vh+jI&r|S27og7xE1G0{rRh<^_;_nD9iCxTFGje z`na)SY&s9dWS}Ja z3*-JLE0P3j%uoUm6A$TXP!;SM(-)Tf?=E;qm0CGvu})mkyc{++$L0${;&lmGtRoBF z)*SA9*=dQ}&Kl);&JL}$0s~F8!^%dPuKP-i0?w_EaONp?x=Sys4?t%PLa>!h1EZ#7 zx8|n}6=T}RwA(e9ktM3Cew%ImTwv>SU>u_;`nQ6WV?$i<2rDu@Hr;^)X@?Lh* ze0_*lFzAp@x|3nqsAP_~BFy;8EuSi6eJLzq`g4Ll>Zb_^B0$v!gweG-qP}oCEdR3zz`Lp`CeP! zUDmxau(XjGR`N($)(p!(gNO3d*pznIoZ&#bXmygpp+~6{N9@M1CHLi1R6W;_npb4v zHb<<4fTm~(e_J+D$-uG=$kY*iO((-?4-~f+u=iw&xJTh8VGI;V$ugmT)Z4SX?wU$l zmw1<~Eq3))g;gm!kA((G<;W{N|@Bly?E24NuuUD>05g{?KiK%XEDIz+W6dt0{0 zk79%9Au3?pHEIsiM>a8y`F$Ut)-n!<#9(|VjUo+5uDq3+ zJ~I(mhwy-_pf@L1r&O8t2BKW6%Vg{$>?=zx!V4evb zs#t;bt}$?%??9)Nm{Nq-6ll|EDM-z{MxU%@0fGnU%=3tA19E%-YTRtg7QQ&Xn^3Qh7WR@R?JX(cckI|6gZ z)#~Ek7Sy}{`(tcywB5NB-bnwIPDwqJ-hEc}Eq;CN%u_1NR$2az3)Qn3sy^$4=O*vm zy*ZrGB7yU+^eap7m+%PhcJB&$bc_2=CVCF*gayLm0V-$+LlD0Hk1F;5rc?;H)~pZx z|6!BJi6npW-Ro)DINiW`zj-}hokWCDm6IKnU15D^DD2WZPfw!5uKzpvtfERfPLv`c zSu<|19%qXR+85m!$=FpBp)M1okC;`%jO}PkuuX>dJo@Yj6G`buU34|rf3|)WOOeYf zp=JKQCi|jtilj-LotWIajwG^t`U znnh`9BxJr3sm6NE=%7jC6;_lCtUwV00)B{(c4c(n1RbBpPlO7(jPgu6ve%l<>Tp<| z--O{-3(C(Y(51)!Dv3VZ*%N^~ns8>zx8^&b8jft!asn@);}RDM znMGLfV1()8Rf7ouDg*MnE1xH@Z?VjbYhTDl_vd1496k1~vNyh;xy-o;PMK|Na?S4v zqFW~XRLL^|YzJV5QT<@!yidO(UKZ&vq**xVk&(2LD-p7} zk=-KEIq9rbb;g9mvq!J(utexcHNmr*a1GxY)hdFB*>dzNFT5cmnl_W!{uELMr3fW& z1*>2G5og;~W}NERbgeQ~UhdH#)i2=afr zmVEtcc)0E|G`p+tw*Tw&v*zHmxUAUE*+y-r$@Xm?UTJ+=xhCxFzxCI3Z&ati=_YF0 zZAukTOP%tWB02mQOo%nc@_OX6G4a~Y5e|_||G-AV4l{HzFG`iK2^D>FX!33N#pY?3 zX?0<3Wd&mJeQA{wl;9Xd$6|A2Vw%yrqp=~Kk5~o@>s()J3@0F2HagiTO;nKCXr8ue zs3+YM_;a+ivQ{NwT8gK9`ceGKKF&=<%*N}}NmR`O7+P=OFouD3#v}rI4rls^QqTpL z%Xx|%inQ*r&GBAWJ!o%lgWo3RhJcHpu>{MO)#_r3hs4@#cR$BWK9+Qap`vOBs8m z`lVqpSvZzu8XYJE)`xZG-PWuUn4})}?8IEZ*hQSxG(8cv8M)c85?zxR&kBS|4)4d+ z9d&=i4N86rW8xc&Wd$Wrb107{t)<%Zvv9f20CYd_noOwcTJp&eX>T%$sHrwuWWTi) z82;|}44=#Xz>PQU4+YvOtQrg+#=ig3oY&=UBkRLiF!<_Ivy6o#z847xt$*>%N7y_B~C09tanx;M~z-@a?FIs zu_s_l7f{Dueu}yqA7+A_dThM%ZETV>iPmA>u+5Eo?*RGh>lzR5K(Re9&kTlI;18 zC>~b{fgZ$@YC>m?GCl(n>skRSgnI@{al@6AEVH&juG|7llV=z;~_t2J^jR%=xiETWxdmhwKr9iM)Y8hJGW-}zsxm3UTy;)rDD80wR)k!`wWo*NvY~W8AdA23MYU*7RE3Ho{QMvfv|~mz%o(%7GL^!)Tne} zJhn~&y-p?G@)W>}tm%W6XH*R+(8D#{2%i*)C0~is%LEU?P=MNye8x`B>uqGVMEtlS zj9(Lvmsxpr@mcuAQ`9E`SpR0Bi&6G;)~*E`n6TD#z5`bU_AQ)OtiZuxmb0(VDz4#i z!S61!|5s(dQ%brrRitX1B2UTsyW@0gfAmVg6LW7mp3?036hH zJ!^&6nBHx>iUS-g{@iQZcAKRp#OfE6;{)f57eEfb?}nX>UD(mX5>T@)sz&huSIVhq z8)T9B&;%pVW3LgesxZ9JRMg~bF$RmyJpKbWM1%dRy7VlQcd%7Q#*@d(ytr9-koA^s z`gP)?DFujYoPr|?(09JjI^MAnp}qwq`e z8yzAY_dW(sgpUeIR}YA;EIlFnXK`vE-iynuipytzM7_Q)o;=Lst$ey*J;K~pNp8(wEt9P5qKzjdDXXh$Ys%2%Xze%y zEVrTrKr=(bbI%nhfU9YQS0Y^%JqY4ApNxN{XpeYRn zkfuI!CMWDT6~V>96~J(^blxmwG2*dmuoC(DE&0gA?SmvWc2^$|9VxJL^_gUKZMBb} z!Ltl{d6F|`H_t6P+s1U0Nr&+W-1g<}Bef-zqi7g)id9Xw=Mp>ptc+Od@L||W!XuWw zX)6{JL)N~YjFZM|%vMa^j7p$k>@gs1a%7PIu;so=&V7Y}dU3=$Ra@x~N@VyyaDX$! z?EU-v*J_>E3E{T36A90baD90@3!QO&Ndk^+`j6=-5}wP6S}~W|mf$%z5^zdT$S%cf z(}}D)1@1Ekk96*-X6|mbkW63ay7qlD@&>wE7xPmU&nf5N^@&@iwi))4@Pd-1-3~n`bZObp_xmb;vai&&NBpLs1I@j!)IAs~WXCv;3 z%R{M~;DgJvnAqGwtDR1)0nMwocBCLA*rLMA0>?|~w5u`Q++&WCV%1AS?|` zwJ~@9?x=pnPrw$88TR>eKw{?-5aH>*S@tGU(ior&3ylw(3)ZNjJSb+SXP7vQa%?Ea z$>oLm%+#i(qI_|@6kkPtRW-CK1?@_Y#mqi`z7+d%eP(;u`!+Jjozbzz9N#Og-oY1n zY8}n01Wcl|zu@@un_w1m&K9+!YkTeWVL$_hh9DlT6fWKrp__5|o#}CDH+U3hGoa@1|)fVPMAuU~F@ zOdi)_04l?lFaEw6ZtGfDHs7(M+CRfV!tLsdidWA~9~ zjfj?MPGjyUaf>9BeH5fpK<@93_qyfRYNgiD6a1ROARZk7tZiD_>WGeRYN7L~CFrMN z`a|wE-uy6bc;j=ekPBe0{JrRXYcZFTrxpk%1-8QLz=6UnPIYjT7=RcaBFyn*#5kXr zhsaHVs*o)ecYTdC9o3pWjvbF-AO3`@qJ>17KJD!I-r1m^=8$yY4&ldn>$x;CjHF!? zhMJbDQy-ebVRtwLjD$563;ih+quseVd}_MFP3xMzO@tadOo5L<)NcJdkR1l1ly)P5 zm%wbvI%!KYhjb)&eOg%qgZ z@OgdeyrwL0(l&qwq#oP<%kUq#ac>`>xvDzTz2)HShA$gomDj1uz~=rbH4bXCO=Zms zX86%}mh}22x;}rWpx{Jx4Ncy?s<*u&7D!8%vDcPzXTObA0#C|J-2C3XQ7*x&T zLfD(FGCryLv$6;eIwWR;a=0V6jC+pOOdSK4i_T?tswwSAs5_H{VBZsj;qZulA5XcPBb=qqt~r2q=CBqoe8IyA)czykc@|Y|^~O zmQXa^yq}RrR;_<)hPr6cyCY2NdXJQ+%F+tR3TXzCxthcgrGi^Rb?XRYra4T1ew;2w%xfqK1H9XK7Jp#4yhH%b6W@?Id|M&q6Yb2WDNN#zz_= z{QSGijSJEGRM}EtuCbD``E*lG44}>yCPfXZVtx{$QQ_GlLX&DxNfHc=;>kUcNv*j> z>TTpJCo_~Y=h$%zC}pk7qO>*v)35DtWKoOXPP|E}>v$A(yU-AzHO}IEbe*r%R0yv@ zrIJ-9pi^XB<)LX9FCfY)V5A~E6J&0a6i-pw_p~^o^!7G=Zycov4t^YDtD9rbY|~CA zWnU{(6Yplg0KuvlELzLVh9E|5cpmW$)iRJc5UdMQW=WXhQAa;)aXT*v%H- zByMR<8kKaHMrMp)OUFeYR^0G@=sA-Tr8#(Z9tKaRj~NasBXlz-{|x4{{W?2~rqbA> z9>-d#)i*aEpvB2+A|Z55nIlk{YHso;x8^+92FUwj`WAj{{XWUeM469>*eyLdj*1c{ zQxO$RKN0{`3aBmR&JR0mFoMtVWvRYW*eBcN56ce{`WjbyG{ z7KpAQg+c{#q5t=dY@l_xweZsCux;FE@@J{eX8RvTMVK24>au)E6hU%39DMP;C^BT( zo6G}0M5{E>^?Ri+e(VWG%g>`)3PY!8K*-6W({) zjJe9TDY$MR8}`3fo&?;9#@|U;GzQCvV5YqPf~J2zBCytW!@=Z_KUi$8B}-N>E;zhYCLn-Gd%cD}Db%ULPF2>!zg1;JfZaKZNA^j^33`eFj zd}u<@9y#6H3WJ(A+uQj?z`*NpTKM=v@V-lS3pMEf;-1hOF`JL>jl=)DNy_n;$Nt1a zTK-WP=HJlW?s1I?xqbcc&<*;x2`7e|SR?BHcNz5v)_TLkd=qg;_xJl>dk&rgU0Lal zF!|pAbl~3(USLpy!X$u(+nBFkx~+Emn?y~)Y|#?H9)f!)H&@1y8A(*G$1tQA*GP89 zu?+BdL}h@$_~3C#CA%)T`P7uIq^LVlpuY6%CtXH%wGR3ViJ2?sf-`Zsg2d7eEtSPf zNi+4FZwiv<=TaWKW}rJGTmc!-A8(w~?!d3be6P)S-m0`Lmz~#W=-*nM^eND4e`kP6 zkeTxYoI*yJ(%B_#!qZ3TtX+m1jqRUyY#4-^M`Qf4XSwEPXlQVl31cLsw1(@ks!sf| zvrMEQX|JJPRtpqMZByxtT>XV?Z&Hsa3HaO$KNtk2Ljq=T^W&ftG)AKWy@6FaLJZtx zZpKW8!Rzl%rNyQvO7wE`*Owm0$@C3vGOn^}TE69scy<*ViulQxg>Z2eNx6sE%rA^5{H ziBg@fE9ZA>^6C)wEFQ(>bx~pRIbo#oyCaUN4^ldYRjN6NFdlO1M^J>GMI>A(3B7}0 zsVo*`He0}O;^am#ZDxQIwK(^EN|U7mpJ}AJb4oc&6z(W6!MGqqwO* z6}PaVrVwSnm7Gu2!a)JI&jCylOM?Wr-cnaOH>#F4Qc<%Hov>UeD3|uFyU3}FH9wG^ zJ_8E;qV06sauCFLF4em`y#se!E&6z%gBJZlRyFcB6Bd$P=o6CIf77YZfUrb`*GiaI z5=d2C+#g0%x#rvZ#-hwkP&TAWnep_qgoUO|+%fr2NeoxL2e5SOM^?8!De{~~hkk(q z5MB|FoT8k702-38f_~F)iP{^L36$|G?maYnwsu~6TAsJQa(@YK2audUF#&(cUv$!P zC;tnl_eRq;b5cUEv&V`b2p0}}7sCR#mBX$L_|-aZv~ZH`Jm0j1#rY@dI;AYAVVTYf zbEG2-VO?j5_fEJnYOWku3P zYkjl(-}m?<;6x9Z^bN#i4rXkr-pn}ny@Q+0p?M5{pO71~3F&HvC(ki>o+&)l1D03* zg$J|n@K7cD1bED{7Yut~$BRdf5?{qCgY2%fo)Oc=e9=VumtPS#M`>0<>8FX@X4|jk zx!KRRu#q}ZAuCG@OUQVh2zjcKyPJ`$@#LhjEh6;+Mlw`p_``t4`s&fR&Te;(gZqV> z=pw!y@9mpP_DPklR2ZVI&02QzYQHPruaCxt)*{!n!mPRZPN?W$&D}30us1MoDfS-F z2WRR!vY1ek$KlFK(LR?#CY^=H^br|fvq@fz!_umOGX9v3zQvpvkbDc3w(&p9iZ~XCCz9sDRRWO` zFt3!-E|`Dfn0+6WP_8HywQD;C;^3#b!8$;g2{}?;MJ`eUO8YH$1)Y9e3S&wB2aeDe zt!FP^!a~*Ij%p|h-gRcr+S*ekC~Dh_xi2`)`uQ~8qGAVcu8F7nI=F^cS3ZWl&U|g2 z;vH2fJP%1hFB&Yb%cI`#Oh^^t3c+rV!C#Fz5267R%T+E~W~{xhii#>;Y2pZ9RUv#d zyC_x;MNO(uauS;^w3+**W`7I$h!V`a-?FO5JOV*8EG*64{fsY& zo^%ul0Pjo2>c*_*huf;&uYQdPi?L^UPL-TgshKq8WTtPMA3N3y_MGAr|DH_5QkxMK zHtV96sOpwR`hG+*)hI$=7DJuTlAEMXvuvzfr2l3Wjd+s+^QG=SG}9&;prco7Iq4PX zBzc@PYK3Fmm1@kfAQCk{YF0NrkttipnQ9uOA0*{6-4HfpYH}J}TZ&XTy!U&sJ}*m6 z29=0tz@9HX5z)oMB_}88cST~#*?VLPg%qxq6lVro0AoU+m}6(ac)btLf12?nLF{B?{&bMOI37PA~$Ll$saF? z49C=7at(sCoxs5GH5}>U$MT0}CU++MK7eDBqB2gAc%lU=bt8Idql*NX zUd)#gr{t^sS%eB4Hh*$vARK5)EL9UsVDd!8! z#)Tu%?jE*T$vS@zCoZ<0OmIY)lbep>tzg?Xbw9dZHmQt@lXX%(yDEQ88Rq8lp8NyX z%#(cQFH04*k|=fY`tz{N66>L*MMc5?WsP`K**REI%vOuwX@O@f=ve~;PVND z0b%aw>TXDL)$|;Xz8lYX1XA~5EF&|Bb%$M`ZRSfo%cr`GGg5Eu%0slif3Y{vAe`Pe~7YY$oIEn&>ZCozi{(x3X>Ug_5WCEPODsv4Ds-Dl)8UMhk|7AEH-18z#E?70w(yH+VNbA5y z%uAxoiR6d$YR4d%@FYWL@KVrc&_A@P<|t>PgHfh^Pm8m+Ui)^pY>q1i7D~OYGFUTa zy8!pRncMbFx_0-W)hcsT1wK+Hh4hR}N-Cs!TuB{IxO%VA%}v~YM4>9^e`I4;<%+ghp*ex&VN1OnDR)ydJYUUs&8HD*#XY5i>511)ii_5elpM zYVe?pL`v7)W_ME6&oM`<)xUYT6eliOU$*-q;%^#%1$sXG1@o*vkUj`rG?O?OMvSxk z17~loCWLFAGpdogs>Jb&ILmz#HMs@65hgfYa3^(j@WfWwTYWZ*3F!# z2QmWCXNzMyxqsji(sjaC#M(L}bgNXSh3oF4-&kZ=h%sqEyvqLAw66(7VzhnHkGHM7 z1SC7!Z)2i6`Y(gFS?ZQw|BeU~T~mpi*bq@<&!Unfd?nqo6K>Dkx{6HF@e%@BPEeO* z75@XbY6NX8A;B>Cu|ySsITVldCbs!lM!zJ45u0{wUI#|6aHl5U|IKRe6@SCi?$g$py*uT;p{sM`Z+v7gMokh2w>6x38`=Nw2u8<-5_^xUXpA}u8 zmj92^xCvP&%jPa&k@XJorwDXJ2}_=No@IE4W9#x9q1Ox5*&x>Fw&0yo8W!a7cOT@c zULumJkCI*zFrUd8gN+{dkt`G*X?tIG>(sS&K5*uiRGemMQp!Mw1O>ed z4C2A5N&ZqQMFmpc)mY6k=ntlmm!B(5UBHIeSsi&MLrq++nqDX>w7cjYLc3dJnj|b7 zFOWwAORp>b=rN$nbb)N3l+rZWZ})8^u`KBofG==#9DP8$fuc8M!ysyIJo)MC7r|Xw@V-b< zfVj5@&S??%9c?*vTj-};duOu&>c&k;NiVw%i~(yjSle^7^)bmu0j5xHq@@FErDkJr)WIC#z3(_jvOk^7 zXUvD3HKr^<2cW;nJN=E&SrXtnn&(PrNHzrg35QiKuj6w&mz;TWE_<`yi9BSdFlDFv zBuaC~;|1x*F+Sk#34Z4c)oD9P_T*18DLrwBXa2ZI)ljSpK|>W!(04A$Md$F{JoQHg zNm!tJ9A@hG?VY{jF|$mx(4eGB@86eH4kvAsLBl31GLcAs5q_`&nKm*)g;LC7j0)Vn zy|}V)Bvm*KP0a(t)j)~>&Zq=Auqe;B>CS09mO_nFb?w`1DYmxV#7ae;(#kxYMAuYP z_K#&_5}>+uVxJu#nxy;d^+mfzbVd{8abkE|%K#xO!ZE4+-MS)6gDHlzz)XV|j-hlb zk>mj$eOLq?0wp)d90O`U?fKUsTQ?=en8wPwjU*Ja?yFBP5S02_iWwty9+#Ai14oCO zRv(auY8hp}Rvxx^A~QHOS`yf+$99p!^ZsIXyyOnIbV<#iWJ=d|GEUX2Wj;kq z_V*h8Q%o}u-9gNNAs?)Ap7e{q+oXi}11DY;(n=Q%qtKYPq@9LF0Y45EHSH)Gx}e?i za)52-a;CMEA3@Vo(*z^^&Q$5yns@Uc%X7ng!{ewFXDrr9bneLqRp`EDS}e2T@Xsk1 z1SkeLsYp#stW?5!b5>g%oT{84Uvh$33ece{&7ilk;z^Owqr!0Q`|-jp-!$CpT~sh3 zkHHDzBT(~51s0HD9`EX_UGm*?lNzhd%N(nr%>dY`CW0wSRcPu$cSj&!ZG^^3(Iuiw ztPAN$3eSN}-Pj&x3uRIy=X_%4-NQqpu~-uuBn__aX4bkj+%!7@VdXSZxY>m=2ps;| zp;5HHuJqMA!W;VxF(#R@DpPl_>|g$+cB8<-hS2_ZFp@17SgZW~QO~gKA2=)@2|NHD z7TqDme=5QOZrTbu!f+Z8=TH#;CSh0!A?m`3#=_v_u){|nX}sTmkSHMU`gh8lBr8RH zecR#hglPYLiH4Z)ZVK+d=r20i!Iwp3&l_1Jpi#!dX;}>S!#jBDQfe)Sn43EUvHfjo zsCrNVw$L`-TrA=Zg3X`T*U%~ialz+EpRn6|-m3#Y?Mxz2^8^f^K9!7_Z9>w$y~HguN>ir%v7be)U5&xFuB+w-es{ zVIW4@rdr!Pc7JrcMZU!JAv6tLEi+DLX>N)9zh^d5Dev=NUm@vA+ayDh$~v7Ic(Jdv z{i|E4X}7P;VZGIZGG_S~S+($5K!uU5#dV`E)FcT)l~3Ek+{AfiCGc<;b_Tai!jzmf z3IJJrQsZA8NY@di^ted4E-^e<*sTP#o5e4?5Wpb!Xc6@r?2(GMkoGviMS& z;QYQ;*|xY9PJK9f8c+z30>seb5l>EM7{7tk-;RZ%XG3OpXY^q85ZO1W4VDi}qbgS^ zG*RK?Jo}*f2Ik6aQ+yKA*6>Y_DO|`N#|u~0$lq}oLUS-X-F!fn0MbswZCR;>c@#bU+i&m(x>ZaZnRYJbw_!3T!@6YkF4rmxS@V} z7yb(y7oA$#k^ywOCV!oy5JG1%x)pG$c?l*12dF#xxgl$4w}5S3HK2uCI?)lyX+{+* zYTCmWYq5qqXy?Q|l}iBnddq+_tOJ{)G8$*=pyHLG#hl24&Bg<|*^+2ceg{2zRZ}z_ z*tSKIUk3oqfrYQJAwMn}q)dt9g8`)889)^o!y**WJFlr=bcd7 zQ?=o=@!?mPh>CVtKDdng@KlUKjycGJ^t3I(x^UU(7!(k6#uf-rr_rFBg=ejhenW?` zuiac@frNu{!Q6b-HM#GijW*-w&B{RJ0rA2@;L)zWJMIzLQxpLLRc#i7u8CGsKw)9A z;?LZ$#KK5qoLF%*0-hG5uC@v0`IV6vRm=t;04}6?)PcMdWz%|jQ<3k)x@ka)$z8=Bjo58H93;#8br#EQHR{?K z9W~ICIu#R=@!EF5;@+A#N%2*koM3fKbD~u9CHMsnpSq+WH{LTAkY`_pHV#qJOh=YY z|7T9;a-ZkM?sqkU4pE?gZLJb!U@K0`#EpidO>+$C#)O~Z>UI7M%4?=yUNzygv?4K^ zw+-;`6&rQ;#%mYC7Ly?tBGV$;jS8!)$@}C!J3{g~inNkh&?wG_4Yt8HOzFKV{ay%f7qJ~XX4uq+&y^^@#u-wz%k0Q_K_s91y=V$wd_;F#@KFX? zWw8}wQT8dE3PxgVk2UzVX-s^^k`V2^tpC^86N1#g<3P&y{v;V2O3$+%Z*}{fziaaa z3-da~g(a%VA+GNi{Tb+A`@s7FC40EYXKL4NlD`tQO=}MW)7R0#H7|oYBlU>tsAt;9 z4EoIquw4-65^urCQY$otSEWzAO!%o#_hy0$SlbID(_5w@8>N~-B|o{tF^KJ(j}p(W zo;S4U2VEw_TA-1SP$CAmEvc)?rP*|Jw#c1re0)X23`p-AFS+7~Rs+ zASI=MbjN6r+USnaA>BxKBc0OeZy%rUAJ3mVj@7a4c<;Vn_j#RHdI8JYQVEE2krMG` z^c^?zV#DxfyOumtWzpGHS?9{gGLS#%n0%rseL4~lTYe$>dT2Cf?hML?SNn}M>6QG< zzQJ==H4H@sNnLD1)Y{PDrWqKc2(tA)F~om<5gCZ#nA4|D1}0gK>{ZMEMbY`q%}nt+ zGP+p)Kta|?W8?#24t~K2Nvj{e z4RH(%uC@i~rYdcc9wH8+OUvV;Ob+bdzOoy5rU-UUrJ`JRi!w)G6}`uV{CIVWOa!%c z=JkitPh`$?2M!c$WX<<5HjYef%RNg?rH&gU>KvaFO`8b*5UFY|_p&y%hbt@&F2jm^ zYYBN25HH0=m{Ej`_xUrBhUf|$053DKW8bo+Mf~crcF=X~vsc9K(a_RQJFsryYh`qO(AasO<6YV?t}HXPt;5H2K-ayA5f?Wo<4@WJ_Tp(0iED~h zQ)(j9*A@%E2#iYu=M7FJ@QTEx%1EE*#wS6A%2dij^$$u2T3bb%`Z(l%P{#nC_Y^7I zP*_j#rA9713(BVstt0LZo&E6;Zt6S2WjaA8mrM$uazXFjNnxor!2(bwBR>|5Qf9_3 z?$8}hJA$Ac($q9f{Gj??Syg6Hq4cU|(jHmI7|stY0R|J^l!x-Q?Neg#CS)cH0#s)2 zX1_d8cY(*})`!;#)FE3)^#Pb?cma|VwTwMA^6-i?9T(Rxw$#k`9@-E2xk$XPGW*@* z-oD56oSC4v6?1fw`Xc$w?IwFY3~MP!(fDO%tWVJBa+SyXpC7z)dgV4)y;0>J5MBnq zB_`6>R0`Tgy0!=RCf^tMy>HQ~U!vZs_Lxh@B6ml@@tm7IqdMLoEjD|!yrv!@nG|=3 zRi9v@P*^4{U@GrwiyTp6SPY^LVpz2wxah5izKSTJ`?)t;Q=BX;XMIJgsjKlxAnSrq z^x0pO^XE+adJ#HDqV=kaKL*`!D{Bb+om+pV^b=K)>&&Ge;w&zbfK9~5%Tx{4yMror zZwcej6ki(-#PQ~Fy>z3)nv|&7MKF^A+(o_jeQ%dV% zH$_l>YC7nVpRcU1$e|<0{4SA|e~RVYqM0ZtX-DXAJEKDtD5Y&#nhs|*fn_Mj=F-Ov zFaMxiq5P3!{eCIe0(0)0^4BEj;*3)LfDkweM!jU5RVZXXm-egNp6KcHKt@bW)}aWt zl$BBR>iPImk(9&T)sehfFcc`8@I@5Yy0Nu|WeRD*4rhPHW6{!G8kce&4N0c*r>c_u z#;IPI8tkt;)M)$8d|}immx;lWtrbsXsa-4EA~@Up&m(}4%t3po&ib;RCH1PPB=vg! zMm`!@`mn5&(Ucu>Tc_9ZE7HSpM zge@~SsqG!@O@h7HIo{GC!9Ue?ZAI>nEUgQ6I+TaOloNcye^EsJ_?8Tu*!3boeD)D) zW7ZTjycH8isza|xshh5)QeJz+p;Aij{{m~k>5C~Puz42MrnX_*3yaD?cA-SM*vX0Q zu8;Kqg(yCCyGbdaLG7qR8Rhku@yNG7k&i5Se^I1Cg1W1&uFpxdI=S4Cg?{{a}tt)KEY;@Q@B|}v=9b}XO-`&#{h}yTz#V@K9CFAWA4TnrH zPAxiV?T5-h3Iwootb<6dEn*Ch)9OfWP8I|706&ww1do8bJEhBJlw=%;;hb}0(809A zl=sUSBE?6xv9h_hrA7J+v-C=sFJrpXDIe{ld`C2mDZ$j?%%{xmf6M~C#jR6^eBzA= zE{WVbXA7*I`gCU4@{s^+fpu4X6Q4vQO?;qpPtQ-Zy;YaGs!AU5FVo&FF~ja&YBu*W zCpQJTVt-E71tWy{7-%^%Lqwb%Wcu6gsg|h+s*EBr9vJ8N4;atuJgK`B`16G=Q!paYc1& zVBlf*rdN4je)Aj$YBt_?U7)RG!Jti^#oI3h$R+6gOe(`XIy(J+3 zjcP3HIa7C2Z9GQhb1)_ocv2a@`0zq2#u5%pbXl;ndH(>wl?afSajtEyxFyG)}pH8+xnDl@(oxjWifc&IH zIGRkvjNbD`soxYeHP%=qnU^|9MnT;~NKaJ)V5=^#AEsBH*d%W~%@;7`R?a3{#kP=W zO_{W~AuUHTymLM!F;gTQ=u378u>oNRj{+Nu`>?%c~5w+TgCXv^J?vg@k z-IhrAHiIK3N+LkGFyXUCV#IR>HbQ~-r$?JHBQQk(CS9D;ix%nJls`1hU3%$N9n$D5*L zQ3F^n1u+|~6RE^!r=!QlCz$?XmV5<)7fsYLkG$35KMW~FPovbZVgAw)RJ(pt(O*}N zxf&~TEv8k;^ewh$!;7nodEPgjKD4)DLv*>DO=NA|^vA*491Rf6kJ#8+I#oWD8;{zfHW8Bk5X!0;&R}3~PHqr&^z}W3M=!53cY0$- zux>iY%1Q)DpW|7kcB`*gSG%=|E;CU6MTvyfxgKqoM4ZF2lyJ|8o;C?q#L!lkKvv3m zUzGbWmE$F4JL1tkbbaJws$5y|*ughDOX?yG(YR0f(ZxaK5%sVa3ArPsv0>cwRm6~o zh>2s@v8(JFX8Ix)P}%B=ZgJ<-6x!V+OV%CTsu;Kpw9)1`j)cTxAv9ruO> zX(nm&JR)VD1?jM=C9O0tx*9kAgsV3gm@ur&1qx405R!sUiYf9w^lqeDgE?r#o!*iw zdf^?{CcAyZ#iTVNBiG*pLqdk)=Gt67*7f8*4(EI2U?;~JlEfMDMm85LB=A8ki2Bn> zQ}Mu)+!`vVbkig__dAQxZi16>Uz?ldm)ss!J;TDYDnioFIbS)WB_mo|f zb*881_MeICp4d;I8|%yU60Rx+P=A{U_kkR&8XtJB6mS%z?zf%pXQ;>R9*JnbB1MN| zdCDoIv!g&mlA!gJVCROYcLTq8`x0xp%je5|s~klU3s%CR<;)ThO^qNIR~Ium)p8+h zl8#YJRZp)-^FP`|I0g8EX;64tk{7;yiRQWapGgI0hFEn$;&)f)f>+uo$#L7nbMg?LW$5|hapxsRNwI|(3X zy2(JWwye{PZc{RW&~;v>p7iP<2EW1480=-spw2Wwot)d7VEH9S<`YY|grYm@+8nX7VQuTG>e(yX;^>=~xaj z5u&syV8CaVy{!2-2`nuzNc2*g4b5-aXd|<`#i|d6v#h_h5@}_TsBzvFNcIC$$FCLk zz`uY5r3YU+o67mC2q7pGBfXXSJ7Fn&`MXz)t_f%n8{pY`2s2)2_-vm>|KKz(evxcR zu4eNdKk@y;=_gff?yYPww|_J_jx;&tjX-#Uwu#^G&fLlkBU9f0W`eKfQ#X7wZZLq( zaq<-eg>fD27hV5}_Uf6N0xa{I-5ulDO%9!r^r9ui{?|2Bb zEunh81)Od~!IL|9;#+;a(ngi5J7VCA!m>Rh)|OR~&j6iXFny6H?EPHsa_8dwfs1x+ z^OaVPZ#_Bgop=Qs%IA-0oWIR>+Nl~~u;Qwr^a5AjNApC@$UI#1!f(g~3;nk}9|_j~ zUPa(gLpkKDm{rVdTz4@m4q%&(rBR};4YBZK)Z*g&=lm)Mxr+^`)9oeJMUu{?2|ay8 zeTn~#CQ)^XU#uA|I=WofU-GZ+|0`2`Md-x-k#hif(L=qvTmL7_Lx&xH?RfH`E?C_; zlIm`_Md6J5?KpeU`QOv}fmLS3+=-9sZp9r7%IT1*?|&So%r}32if{J!Z$Ch`Db+S2$x1NZ8g2APk zeO49^N2s)X3)d@8`cGw7-4gF54wWCjZKB|=-#wAU7eAHX`$%XzR&LBvM_qSI$V6`r z>rjrgoVcvkwY8WQAPw@bq9wx$Qttz;ZblMIlN24+Zs;jPgGh#`4V2P_~{~w(`RvCtv9M zy^4pLAfz8>G^YE52fzoLFbNzfYvE<1}Hc4wF` z_<%+y%vntd`@9+>dKH#fG~k6>9{npk;p={_`70rwH>ItqL%*#sGJYt}MOH43)?Dgy zUZ~Ryl=K_OT88!#qGg)%O~#XdqJ`f8C-cz>K54`+*e#l{!=3{%1vA#6%5o$G4o#f` z^AzBB>}LMn@2-3m?qvp(2wWOP$-e{O8&EI0TWuZKi2v7%Q?>DU3zE3hQ~(S2LzTCr zn`O?_>s5`C(ZY5y=^_x(!I0jNUWT(G@FUQs**yJwk>u(tNZ|Z4qUqv=5&;kSI@~20 z0bE&haea`DU*4b3;ZAwX*ODK1w+`}zo2MDlrqNjTi)9E&8N&HH!uM$cwqUmjkFxBu|J5UtUKJ@M-I{)d={LMdV!?7?@c-Gi#e7~x6Tt3-dEQ#TivFKF zv0E3k8-LqmaJ9bst=bquouip6>3j*VgwX&Lt2N?%g!2z*&^Usy&>%xL3bZ)ZgX0xo z51WCs(buM0tC4P0%hhB?2C7O$O{7N&vW$+;CxFuZgMWZ0!Hinc-l~r*MBYP@U^9gz z(pTUB30(j`F<+IQhE_=tN_LP=(ocF%75f^YyDN(3X#6=bJpSv$$bz(*PljsEQ-|x; zq?B-BoEKcjM&+Y}gn)^C8&zx83EnwE2HYlxuXJef@eq}3VP+Uh311k;cd1lU3(zXi z#0TfB4Z?xrsk)xuh->xH*ONFOZ z^EhgJ!8@s1jA#h6v;U+vWKNLt2zQK?l9b!Wd|!E;sX&)}>0O4Q31^R_^L{m%F9ZgL zi5zNg#{bcP7Gd+PSE-A?RkxLiEd$7#D4O5^v9z0IAnBT=kcpQecw(MhH9UC=?5x=} z_vWbX%rIRQ=grwfg?81{7yuM}LmRT*$ML?j2RPBIzhOY8fC=OqjH z-1roYf7y!q*0@C&_(()c5Z589PCA}7wjz45zu_Jq(>Bm=us`WyA-Jr%ivzp zz_9+HzO;3~XgB-JS$F)>BqE=s-A8~rB(~;@nJ(YSMMk6W3%eoPGotxJG&lE^4*j}v z?+1T+(vYuEvtkinHnJ#&G2Y?d?P_b>iF~$pT&8vs=W{5WPL4gJe^HE+K!;z^7R*%a zDYn+lkD^XjLuaCkx4FdHmU)wT0d+nf-%2pV?X)6Z?CkaO4(e{yC=K!+w;LUr^Dm{k zs1q9kKKJ!yBH^=Sof`Xi3}Mv8x>7qk$8SHDt1O2rmSjfPlk7a~I7yvIC)xSom`s6Z z=Zjgf?V@9=Y(2c{IG$&roroLcLRmG~4)6U%``qg2%*o;;VUVo&8-bM8U69sr(oACT zMJ#wqQl1~3`t4x=l!migGJFCSgdQ89?l5~<{{?HC_~eu-@?Ojmz+;BR2FJ9=U<9>j zG9V;%R3dmHKyV;e92uBQ&+yfIWj)rErq2+UgeJ1FluqQZM@oILkDx_}cl*}N%HuR| zTI4(AY!d{{d`~n5NsO%wAsTX+_cD<8=u&Lsj{b673m4zm<%-Yj`7T||8eL>;O}dN~ zx%n(JscUPg*hQZ)ia^?eBujoM?{17)fD*1VA)NyIf#>WICNeYX)ZrW$FLPn6I>lbZ zTpf^b=HpJ?D+9&J3T&{uQpikyz;Iy-0OAu{uw=_1Xz1#ky9g64*bYBfjTim&~W zMQ0!=DiI}8tQhv}Me>o=Z`}~Xh-H7$L{YYUaQu7>aWLb&(D0x;!Er~9s|B_yxsfbH z-UX(s9c_-RhF&KdXGI?Jm&&i_F^>`n#9JEYT(8-_{4G3oQ6wY{ZAckVpGCU^-O}dVEW29Z+Ro`*&~*?g zib^FAia4Cmw?wfC0jT}%&%!H+{Qe?*R8GTU+u6Tk)JecFbZ2UrBq_{BEhywqA70>_ z-(XxObviFMm&lc94&xh=&gEp-w~VoSimOUyo={H|Q$AGqP&F$1Glw zAF!XF$e2!nB5X&;gKRC`?0;n_9WqOztEHyuV<~N^3(V7R#gr?4I>$GK#ue=@XwmbM zrvH4;a3!a)yi9kx>lg@U2xA;!aMdrFtmgZRGH19-EaLWlmY~ksUq(Vd{Nu8sBz2{d zZJ+JB8%4RG+UT)uw%=<(}=w(97PomDF0Rr0BxOm^qw=C+masL(;xQ zg>mG}htwXksYNB*;$Uxe9a|%;p`lu)Q5btLlq?!VKs{3OxbAW%R`8?)DbZC9R-6cV z0!~Gco^@#on%Pji_5VmufBGS(@&lE2-PKW5-OYj>5|a#8N&btXE2*n86!^Nvwtr1Xq#QC?Nm!`moKfEK9@bs`+cYnGD9yLThvPo#y88%0v9 zxvfX3wtuL&y;-f$f2X+_Q&fgF(VCCCij_TW#ZvUEiG|ES{yh|3#S$vWx1Q zcV;+0%7gCYwltFpe}cK9h3%avfZRat{PN6^7YRrit_|(dz3!Cx)E(W@HMhpn@^jKL z5$PD6^dIIsJ5g$yc0v5_YE;XZM6Hy{ZTHLMGu6r}3RFMyN&L9f1h3z$3^9s$a263f~=f@xA;dK?UJntR>|Uc+8)wz1-;7FdVcG z^U3b4(je+pCVuZ<{6*=G#Odw+*0dHYbL5@B)*G=0D6)&e7kk0puITy~WiXu7V;4?E zdA%u2!`AX+Vbo`T0&h-7Tf=2B)P6@Rw4A+VHsyP}=GK-gahfRpIdzpIp z#y<0>zLvb)AZd`nkx~r!*la-tcx#D4U?rNQp0l-lp8I*Cv>U*^US>`REa(4Jai!fJmSs6+M zN4%Dc5-Z;7+r)eg#M`nPP+*DljQmCbsXwi3xICq%xpz@w&Ii|&Nz_^?lavZ8?9V^c zYq^eR3KdDoP>s612aIM|L2;250&hpo$8t>sWe56Iy>G*;A;v|pqWB1fU{}A-QCu;j zu6{y<_?>&i(Zq7D&hloB=4T}4uspmP?bH+xb)*nf^%W4@ngnCv4k zMxFf=ViP&xqWuAS;13%d)7byS!uNbIQ>^HNJ%j#d_#h^_QTKO;x%@xH;d!U`YLot8 zG5UP+@YOJ4XNO5l zE$O^xp|Oxl9pS~@erB)+AuiJT9F^VvG&}HhtnyPE=y-uc;5abq_>FBq`XK?v4TO!K zWzLUHd33HYQO9C5Z$}sFMdZo?E0qFhF})8|kc_W}<&PAo#0TaWvD8s2u%hD7#-loLXE&VYw}tDK z5j!14&2|D@*SM#62&U5rq-~nq2N|p^-=@Y<2E(ATNtfIpp3Y?nmkqtBu8<8k85+c9 z+7IwW;By$FU}_;WcC#k-GTl)fMuo4(9nH7Dj@QeVy7W!PJdvPfk+H;|agUS~S0Xg~ zvtX|iT@Zwwj4=2`p0?;!1tpfAwk@)hZ%RWhFp#xN&AEpUFE^Q9frD?84x86$qC1%@ z`iLMuY1zuzsR@_z`C~X~6CWMa)Ckeb9r>fyOfV^tBu24G8Vrmy*pvDsGqIrvU@KTR z=_4qJ@CP7u+i-ze^ktZ+vYSHWm~`GmCe;v)TJ?dY2dSO~D6aDzXRL2FZ0c$P#LY{A zED-I;mR1YuKEMlf)$ADlbEzj6Mv`N}fko~ub z%n3LeZXA-acLeuUczTKz@1OuYDx8Y$$O#hmE}yVsWg#0QDGZLnlo4BvVtlP7clqPM z@<{nLt-+rcjy2P@mh)-x&i--MvKDw{AUF3^@~Nn`9}3I7nVKZKXQf|_cVhL^ZiKYv z&f<~GtO=BFF)CNe8EulHp-Uy$gw(OAONoF_&mTJ}U6#ixym4r~n0&K*h*2!g?Ai{V z(mndcOL9GAp9NC4-xQn+XI)3NhP-FUq?Mq)_XEgqBBON5Gb(gLmmW9aQG~~vo{w)@ z^oO*r8u?y2pEn||mK6=Dt5;l1e&9R)H?0}oMAuuiJ(@m(DL zqz&I@IN-TMYC@Q8`m|bSuHJ|l7b4sI=%B8(#HJ;0!W4-_V(s3#({+$9%+2i3 z!pjS+SzGQY+97e{g2d`0yO!NDz7uLDnv}G|7eL}AvsiXGk#OfDvo1)=n72l_fZu68TgwLmenr-ACT~pR^d3z90s^3KZ zR{fn7rbQoVEMlf4E6A8wxOhf{9KZFZZE>FGlW~Q(;aP)Sbgf>LxLdtue0}-`_p#ir zh4K!W4KMF>gB>wEu}GV-yAZ&l!}w(88{I*nkC(=CL;BfWcNYY9adB4mNG{MK_g)`n zH1iPhJX|{mq-wf<2a;i$DKBevl&HR3oAZ$0<5&JIibQtmzp*^0 z@uD{6zQB!&7;9|z&{oIDr9f>%)_PD^7&7<88?Ra4W#W`)%FjB71zlMyp;}fr3OxKV zu)G=g!7+!l_;s=O-jdI%xd&ACS-FaBk5SNbH2EgaHe?0+*X~7A6cKxuNPQXC+<v1Ct9>-N!mNP` z0A}VxD&Rk`Q?T&v?KNCZNQ*j#9Z^BRQoPARpf>UPyIT*?Qi#DSK#CoF&E(f7z< z#p98Jm63GdkB;3o4GOBa$I`z$qC{NRZ@;-ECV|?(>Isk?V(7?+)>Bs?zr6CBXk@}$ z0%?N$UxBFjB-Ov*!z(+x7v*pGmY&2X@xdkbG|$l_s2cuJrq5B)V>q*B>qXYbwHoYc zC^B?hb_Vzt2mhkPOuh>1LQX05#~hP)djFyrw(|7|g?EK{hW#55sL;`&pkDF|pzN&q zggpBy(sJhXEHFTUhTI^15d}A#MOjCM3#D7)e=&RIur>UPe}>2ZakDS~N6#_@2%;nF zME<)K_HS;18s5R+bvAJ8@gP1D)Q`cgAZc6t9BnJq@kC`;V>HS2a-q!gQxL5)q2Vwd zB~USv8I;KV%(<|bUWhISnh2N=hax^af*~&D#|Ng}2TR%+D(Bp7?^waOoff!f$LbRr z!NwN;TzT{4pB=6=3<(aUT|CM(+56A*n!36_s+^VJkN1!L6t_RI?rRE4+vmy~65GMj zVDaH;AP9AiSHM8U0&)uE6-!<>cjvhxxXf>+!5~YuHnm9mS&U9x!QNfmw~tI$?1$4s zG}YG2!>PFKB0u-zy-S{ph>+r@sW8ZCGCC=c5?})GN^Ph#r;1DBH` zOACvcXAZ)U_M*yY(Z2s&E1Br&%Es2BjS)<4O+`3zV-9=o!MmtsPwL< zY#cJt-9Ggq5#I-@7O8T(7S|-W`ITKr;2J4<6!!Iaut35>IAfnKD{umuiHY26l+`x8 z{PT>Z`|Q}N=9@0CPKOKl2LJ>An$uT&!s}N=cJeOU+dzJ; z>%}Ka1u1Y+vJ0F8vibHopyg>L_cypijgsmops@a2E3{kch1r_x0KST0opj5W_+=Pz z6e^lvRXXM#t^iZGrhnm8o{$BLid}xW>j!Mm#GF=b=3szb;`8TU{Px?1kU^3FqWJG; zJ5#l3_N#5nI+AjQP6iqItAA`_&3nwfNM&VpuAxWFah-Cgg>_0h#(OQZro-w*(Ubnf zD1m$l56UJsM3P#IDnhx5IVy-CM%JjNOhznEo1dxd=2{POZV6Bob;0gIX;rQZL-P0I zq;`iR%@{k21mS=Q0Sf!LUOt&kxm32{(wmY@jX}=|v^86ANHB2!=-)coG|3#QP!Q5;LS5;tw)W*jf8~cnma2dW0zxA zkhCc@H7!$_G50;0;R<0M%L_}m2vtmal7+o~QFG&`mpY$$Us=a-1cyorGn)A;0cde( z8s^g`%s^Wy8~AUG4KDJ{UOA8BuOzEVFTi}AF9Q@bQ#J)i72{J!Wz|z9nh}cP6F{px zNMwwtU}0pmYWh^z-rnwp>=~XXFR{Yd8SOo4HY&l!>>+i=Sd z;`bpzXnm3M4pI%;iG+H|1QnSpzG;2hdAi(?fln5rofp;5G|pdt3+wv$e(F8BLxN*O zVddEU9bxNIT$RhdeC3 zNt$_ZW@O!y9Tn-1l*zSavJ%tJ<(4&srX1cx)cLiRi!?9k-IJs6wdZq3lHun22(nwv z@sg=6EwJP_8j8?H{SHcs823!XwXWV~|7oeKrabaQ7axmCwDhC%8s&DAp&$?*{duyx zpm>OPxxuFCRhk&cOpriub%8g4p@21U-erkJ$6c~h!?8n^=;e?1G!MP^kv3qP{E;XR zBcJ_kbF(~d(_{}VC5VpA)@|ZhfuD;bX<)g2@%E27-`9gV123e@4^V%mN zXlH*&bNw*v$*G>#Us73lKYmRa0^MMFIKKw{mtO+0N!q1TBKVln=kDwmR#`W)%lp=M z@QGyvk2nAdl{A!$>kMysu~~%qYUkdTh4KO3)9S#Q6|KS|zY|=om)@6>JCECSni5@n)LJn$>onM!$2>FcbTm6QugpwN?ZGJ4^>5u;Be9TF;oto12LNGPw{(d zv5Zpa_O;&TMK3J^)dZmcAEf~qW<6O$)o_M=scp8#+`BF_OD&+5M6{Dy9IlAO3S|jt z;fys;*&b(jX;A$~zKIxGj#}&v{zmx;?Qdb$lJRT-)kOXCefma~u%-

', +# sp_model_kwargs: Optional[Dict[str, Any]] = None, +# add_bos_token=True, +# add_eos_token=False, +# decode_with_prefix_space=False, +# clean_up_tokenization_spaces=False, +# **kwargs, +# ): +# super().__init__( +# vocab_file=vocab_file, +# unk_token=unk_token, +# bos_token=bos_token, +# eos_token=eos_token, +# pad_token=pad_token, +# sp_model_kwargs=sp_model_kwargs, +# add_bos_token=add_bos_token, +# add_eos_token=add_eos_token, +# decode_with_prefix_space=decode_with_prefix_space, +# clean_up_tokenization_spaces=clean_up_tokenization_spaces, +# **kwargs, +# ) +# self._add_bos_token = add_bos_token +# self._add_eos_token = add_eos_token +# self.update_post_processor() +# self.vocab_file = vocab_file +# +# @property +# def can_save_slow_tokenizer(self) -> bool: +# return os.path.isfile(self.vocab_file) if self.vocab_file else False +# +# def update_post_processor(self): +# """ +# Updates the underlying post processor with the current `bos_token` and `eos_token`. +# """ +# bos = self.bos_token +# bos_token_id = self.bos_token_id +# if bos is None and self.add_bos_token: +# raise ValueError('add_bos_token = True but bos_token = None') +# +# eos = self.eos_token +# eos_token_id = self.eos_token_id +# if eos is None and self.add_eos_token: +# raise ValueError('add_eos_token = True but eos_token = None') +# +# single = f"{(bos + ':0 ') if self.add_bos_token else ''}$A:0{(' ' + eos + ':0') if self.add_eos_token else ''}" +# pair = f"{single}{(' ' + bos + ':1') if self.add_bos_token else ''} $B:1{(' ' + eos + ':1') if self.add_eos_token else ''}" +# +# special_tokens = [] +# if self.add_bos_token: +# special_tokens.append((bos, bos_token_id)) +# if self.add_eos_token: +# special_tokens.append((eos, eos_token_id)) +# self._tokenizer.post_processor = processors.TemplateProcessing( +# single=single, pair=pair, special_tokens=special_tokens +# ) +# +# @property +# def add_eos_token(self): +# return self._add_eos_token +# +# @property +# def add_bos_token(self): +# return self._add_bos_token +# +# @add_eos_token.setter +# def add_eos_token(self, value): +# self._add_eos_token = value +# self.update_post_processor() +# +# @add_bos_token.setter +# def add_bos_token(self, value): +# self._add_bos_token = value +# self.update_post_processor() +# +# def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]: +# if not self.can_save_slow_tokenizer: +# raise ValueError( +# 'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow ' +# 'tokenizer.' +# ) +# +# if not os.path.isdir(save_directory): +# logger.error(f'Vocabulary path ({save_directory}) should be a directory') +# return +# out_vocab_file = os.path.join( +# save_directory, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'] +# ) +# +# if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file): +# copyfile(self.vocab_file, out_vocab_file) +# +# return (out_vocab_file,) + + +# Modified from transformers.model.llama.tokenization_llama.LlamaTokenizer +class InternLM2Tokenizer(PreTrainedTokenizer): + """ + Construct a InternLM2 tokenizer. Based on byte-level Byte-Pair-Encoding. + + Args: + vocab_file (`str`): + Path to the vocabulary file. + """ + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + model_input_names = ["input_ids", "attention_mask"] + _auto_class = "AutoTokenizer" + + def __init__( + self, + vocab_file, + unk_token="", + bos_token="", + eos_token="", + pad_token="", + sp_model_kwargs: Optional[Dict[str, Any]] = None, + add_bos_token=True, + add_eos_token=False, + decode_with_prefix_space=False, + clean_up_tokenization_spaces=False, + **kwargs, + ): + print("register succeed") + self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs + self.vocab_file = vocab_file + self.add_bos_token = add_bos_token + self.add_eos_token = add_eos_token + self.decode_with_prefix_space = decode_with_prefix_space + self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs) + self.sp_model.Load(vocab_file) + self._no_prefix_space_tokens = None + super().__init__( + bos_token=bos_token, + eos_token=eos_token, + unk_token=unk_token, + pad_token=pad_token, + clean_up_tokenization_spaces=clean_up_tokenization_spaces, + **kwargs, + ) + + @property + def no_prefix_space_tokens(self): + if self._no_prefix_space_tokens is None: + vocab = self.convert_ids_to_tokens(list(range(self.vocab_size))) + self._no_prefix_space_tokens = { + i for i, tok in enumerate(vocab) if not tok.startswith("▁") + } + return self._no_prefix_space_tokens + + @property + def vocab_size(self): + """Returns vocab size""" + return self.sp_model.get_piece_size() + + @property + def bos_token_id(self) -> Optional[int]: + return self.sp_model.bos_id() + + @property + def eos_token_id(self) -> Optional[int]: + return self.sp_model.eos_id() + + def get_vocab(self): + """Returns vocab as a dict""" + vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)} + vocab.update(self.added_tokens_encoder) + return vocab + + def _tokenize(self, text): + """Returns a tokenized string.""" + return self.sp_model.encode(text, out_type=str) + + def _convert_token_to_id(self, token): + """Converts a token (str) in an id using the vocab.""" + return self.sp_model.piece_to_id(token) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + token = self.sp_model.IdToPiece(index) + return token + + def _maybe_add_prefix_space(self, tokens, decoded): + if tokens and tokens[0] not in self.no_prefix_space_tokens: + return " " + decoded + else: + return decoded + + def convert_tokens_to_string(self, tokens): + """Converts a sequence of tokens (string) in a single string.""" + current_sub_tokens = [] + out_string = "" + prev_is_special = False + for token in tokens: + # make sure that special tokens are not decoded using sentencepiece model + if token in self.all_special_tokens: + if not prev_is_special: + out_string += " " + out_string += self.sp_model.decode(current_sub_tokens) + token + prev_is_special = True + current_sub_tokens = [] + else: + current_sub_tokens.append(token) + prev_is_special = False + out_string += self.sp_model.decode(current_sub_tokens) + out_string = self.clean_up_tokenization(out_string) + out_string = self._maybe_add_prefix_space(tokens=tokens, decoded=out_string) + return out_string[1:] + + def save_vocabulary( + self, save_directory, filename_prefix: Optional[str] = None + ) -> Tuple[str]: + """ + Save the vocabulary and special tokens file to a directory. + + Args: + save_directory (`str`): + The directory in which to save the vocabulary. + + Returns: + `Tuple(str)`: Paths to the files saved. + """ + if not os.path.isdir(save_directory): + logger.error(f"Vocabulary path ({save_directory}) should be a directory") + return + out_vocab_file = os.path.join( + save_directory, + (filename_prefix + "-" if filename_prefix else "") + + VOCAB_FILES_NAMES["vocab_file"], + ) + + if os.path.abspath(self.vocab_file) != os.path.abspath( + out_vocab_file + ) and os.path.isfile(self.vocab_file): + copyfile(self.vocab_file, out_vocab_file) + elif not os.path.isfile(self.vocab_file): + with open(out_vocab_file, "wb") as fi: + content_spiece_model = self.sp_model.serialized_model_proto() + fi.write(content_spiece_model) + + return (out_vocab_file,) + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + if self.add_bos_token: + bos_token_ids = [self.bos_token_id] + else: + bos_token_ids = [] + + output = bos_token_ids + token_ids_0 + + if token_ids_1 is not None: + output = output + token_ids_1 + + if self.add_eos_token: + output = output + [self.eos_token_id] + + return output + + def get_special_tokens_mask( + self, + token_ids_0: List[int], + token_ids_1: Optional[List[int]] = None, + already_has_special_tokens: bool = False, + ) -> List[int]: + """ + Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer `prepare_for_model` method. + + Args: + token_ids_0 (`List[int]`): + List of IDs. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + already_has_special_tokens (`bool`, *optional*, defaults to `False`): + Whether or not the token list is already formatted with special tokens for the model. + + Returns: + `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. + """ + if already_has_special_tokens: + return super().get_special_tokens_mask( + token_ids_0=token_ids_0, + token_ids_1=token_ids_1, + already_has_special_tokens=True, + ) + + if token_ids_1 is None: + return [1] + ([0] * len(token_ids_0)) + [1] + return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1] + + def create_token_type_ids_from_sequences( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """ + Create a mask from the two sequences passed to be used in a sequence-pair classification task. T5 does not make + use of token type ids, therefore a list of zeros is returned. + + Args: + token_ids_0 (`List[int]`): + List of IDs. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of zeros. + """ + eos = [self.eos_token_id] + + if token_ids_1 is None: + return len(token_ids_0 + eos) * [0] + return len(token_ids_0 + eos + token_ids_1 + eos) * [0] + + +TOKENIZER_MAPPING.register( + InternVLChatConfig, (InternLM2Tokenizer, None), exist_ok=True +) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/janus_pro.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/janus_pro.py new file mode 100644 index 00000000..d574953e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/janus_pro.py @@ -0,0 +1,634 @@ +# Adapted from: +# https://github.com/deepseek-ai/Janus/tree/main/janus/models + +from dataclasses import dataclass +from typing import Dict, List, Tuple, Union + +import numpy as np +import PIL +import torch +from PIL.Image import Image +from transformers import ( + BaseImageProcessor, + BatchFeature, + LlamaConfig, + LlamaTokenizerFast, + PretrainedConfig, + ProcessorMixin, +) +from transformers.image_utils import to_numpy_array + +from sglang.srt.configs.utils import register_image_processor, register_processor +from sglang.srt.multimodal.mm_utils import expand2square + + +class DictToObject(dict): + def __init__(self, dictionary): + super(self).__init__(dictionary) + + for key, value in dictionary.items(): + if isinstance(value, dict): + value = DictToObject(value) + setattr(self, key, value) + + +class VisionConfig(PretrainedConfig): + model_type = "vision" + cls: str = "" + params = {} + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + self.cls = kwargs.get("cls", "") + if not isinstance(self.cls, str): + self.cls = self.cls.__name__ + + self.params = kwargs.get("params", {}) + + +class GenAlignerConfig(PretrainedConfig): + model_type = "gen_aligner" + cls: str = "" + params = {} + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + self.cls = kwargs.get("cls", "") + if not isinstance(self.cls, str): + self.cls = self.cls.__name__ + + self.params = kwargs.get("params", {}) + + +class GenHeadConfig(PretrainedConfig): + model_type = "gen_head" + cls: str = "" + params = {} + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + self.cls = kwargs.get("cls", "") + if not isinstance(self.cls, str): + self.cls = self.cls.__name__ + + self.params = kwargs.get("params", {}) + + +class AlignerConfig(PretrainedConfig): + model_type = "aligner" + cls: str = "" + params = {} + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + self.cls = kwargs.get("cls", "") + if not isinstance(self.cls, str): + self.cls = self.cls.__name__ + + self.params = kwargs.get("params", {}) + + +class GenVisionConfig(PretrainedConfig): + model_type = "gen_vision" + cls: str = "" + params = {} + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + self.cls = kwargs.get("cls", "") + if not isinstance(self.cls, str): + self.cls = self.cls.__name__ + + self.params = kwargs.get("params", {}) + + +@dataclass +class SigLIPVisionCfg: + width: int = 1152 + layers: Union[Tuple[int, int, int, int], int] = 27 + heads: int = 16 + patch_size: int = 14 + image_size: Union[Tuple[int, int], int] = 336 + global_pool: str = "map" + mlp_ratio: float = 3.7362 + class_token: bool = False + num_classes: int = 0 + use_checkpoint: bool = False + + +class MultiModalityConfig(PretrainedConfig): + model_type = "multi_modality" + vision_config: VisionConfig + aligner_config: AlignerConfig + + gen_vision_config: GenVisionConfig + gen_aligner_config: GenAlignerConfig + gen_head_config: GenHeadConfig + + language_config: LlamaConfig + + def __init__(self, **kwargs): + super().__init__(**kwargs) + vision_config = kwargs.get("vision_config", {}) + self.vision_config = VisionConfig(**vision_config) + + aligner_config = kwargs.get("aligner_config", {}) + self.aligner_config = AlignerConfig(**aligner_config) + + gen_vision_config = kwargs.get("gen_vision_config", {}) + self.gen_vision_config = GenVisionConfig(**gen_vision_config) + + gen_aligner_config = kwargs.get("gen_aligner_config", {}) + self.gen_aligner_config = GenAlignerConfig(**gen_aligner_config) + + gen_head_config = kwargs.get("gen_head_config", {}) + self.gen_head_config = GenHeadConfig(**gen_head_config) + + language_config = kwargs.get("language_config", {}) + if isinstance(language_config, LlamaConfig): + self.language_config = language_config + else: + self.language_config = LlamaConfig(**language_config) + + +class VLMImageProcessor(BaseImageProcessor): + model_input_names = ["pixel_values"] + + def __init__( + self, + image_size: int, + min_size: int = 14, + image_mean: Union[Tuple[float, float, float], List[float]] = ( + 0.48145466, + 0.4578275, + 0.40821073, + ), + image_std: Union[Tuple[float, float, float], List[float]] = ( + 0.26862954, + 0.26130258, + 0.27577711, + ), + rescale_factor: float = 1.0 / 255.0, + do_normalize: bool = True, + **kwargs, + ): + super().__init__(**kwargs) + + self.image_size = image_size + self.rescale_factor = rescale_factor + self.image_mean = image_mean + self.image_std = image_std + self.min_size = min_size + self.do_normalize = do_normalize + + if image_mean is None: + self.background_color = (127, 127, 127) + else: + self.background_color = tuple([int(x * 255) for x in image_mean]) + + def resize(self, pil_img: Image) -> np.ndarray: + """ + + Args: + pil_img (PIL.Image): [H, W, 3] in PIL.Image in RGB + + Returns: + x (np.ndarray): [3, self.image_size, self.image_size] + """ + + width, height = pil_img.size + max_size = max(width, height) + + size = [ + max(int(height / max_size * self.image_size), self.min_size), + max(int(width / max_size * self.image_size), self.min_size), + ] + + if width <= 0 or height <= 0 or size[0] <= 0 or size[1] <= 0: + # print(f"orig size = {pil_img.size}, new size = {size}") + raise ValueError("Invalid size!") + + def resize( + pil_img, size, interpolation=PIL.Image.Resampling.BICUBIC, antialias=True + ): + if isinstance(size, int): + w, h = pil_img.size + if (w <= h and w == size) or (h <= w and h == size): + return pil_img + if w < h: + ow = size + oh = int(size * h / w) + else: + oh = size + ow = int(size * w / h) + size = (ow, oh) + else: + size = (size[1], size[0]) + + return pil_img.resize( + size, resample=interpolation, reducing_gap=None if antialias else 3.0 + ) + + pil_img = resize( + pil_img, size, interpolation=PIL.Image.Resampling.BICUBIC, antialias=True + ) + + pil_img = expand2square(pil_img, self.background_color) + x = to_numpy_array(pil_img) + + # [H, W, 3] -> [3, H, W] + x = np.transpose(x, (2, 0, 1)) + + return x + + def preprocess(self, images, return_tensors: str = "pt", **kwargs) -> BatchFeature: + # resize and pad to [self.image_size, self.image_size] + # then convert from [H, W, 3] to [3, H, W] + if not isinstance(images, list): + images = [images] + images: List[np.ndarray] = [self.resize(image) for image in images] + images = [image[:3, ...] for image in images] + + # rescale from [0, 255] -> [0, 1] + images = [ + self.rescale( + image=image, + scale=self.rescale_factor, + input_data_format="channels_first", + ) + for image in images + ] + + # normalize + if self.do_normalize: + images = [ + self.normalize( + image=image, + mean=self.image_mean, + std=self.image_std, + input_data_format="channels_first", + ) + for image in images + ] + data = {"pixel_values": images} + return BatchFeature(data=data, tensor_type=return_tensors) + + @property + def default_shape(self): + return [3, self.image_size, self.image_size] + + +class DictOutput(object): + def items(self): + return self.__dict__.items() + + def keys(self): + return self.__dict__.keys() + + def __getitem__(self, item): + return self.__dict__[item] + + def __contains__(self, key): + return key in self.__dict__ + + def __setitem__(self, key, value): + self.__dict__[key] = value + + +@dataclass +class VLChatProcessorOutput(DictOutput): + sft_format: str + input_ids: torch.Tensor + pixel_values: torch.Tensor + num_image_tokens: torch.IntTensor + + def __len__(self): + return len(self.input_ids) + + +@dataclass +class BatchedVLChatProcessorOutput(DictOutput): + sft_format: List[str] + input_ids: torch.Tensor + pixel_values: torch.Tensor + attention_mask: torch.Tensor + images_seq_mask: torch.BoolTensor + images_emb_mask: torch.BoolTensor + + +# FIXME: had to place Official Processor here, since image_processor module would not be imported in all threads, +# hence AutoProcessor registration would not be affective in some cases +class VLChatProcessor(ProcessorMixin): + image_processor_class = "AutoImageProcessor" + tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast") + + attributes = ["image_processor", "tokenizer"] + + def __init__( + self, + image_processor: VLMImageProcessor, + tokenizer: LlamaTokenizerFast, + image_tag: str = "", + image_start_tag: str = "", + image_end_tag: str = "", + pad_tag: str = "<｜▁pad▁｜>", + num_image_tokens: int = 576, + add_special_token: bool = False, + sft_format: str = "deepseek", + mask_prompt: bool = True, + ignore_id: int = -100, + **kwargs, + ): + self.image_processor = image_processor + self.tokenizer = tokenizer + + image_id = self.tokenizer.vocab.get(image_tag) + if image_id is None: + special_tokens = [image_tag] + special_tokens_dict = {"additional_special_tokens": special_tokens} + self.tokenizer.add_special_tokens(special_tokens_dict) + # print(f"Add image tag = {image_tag} to the tokenizer") + + self.image_tag = image_tag + self.image_start_tag = image_start_tag + self.image_end_tag = image_end_tag + self.pad_tag = pad_tag + + self.num_image_tokens = num_image_tokens + self.add_special_token = add_special_token + self.sft_format = sft_format + self.ignore_id = ignore_id + + super().__init__( + image_processor, + tokenizer, + **kwargs, + ) + + @property + def image_token(self): + return self.image_tag + + @property + def image_id(self) -> int: + image_id = self.tokenizer.vocab.get(self.image_tag) + return image_id + + @property + def image_start_id(self): + image_start_id = self.tokenizer.vocab.get(self.image_start_tag) + return image_start_id + + @property + def image_end_id(self): + image_end_id = self.tokenizer.vocab.get(self.image_end_tag) + return image_end_id + + @property + def image_start_token(self): + return self.image_start_tag + + @property + def image_end_token(self): + return self.image_end_tag + + @property + def pad_id(self): + pad_id = self.tokenizer.vocab.get(self.pad_tag) + return pad_id + + def add_image_token( + self, + image_indices: List[int], + input_ids: torch.LongTensor, + ): + """ + + Args: + image_indices (List[int]): [index_0, index_1, ..., index_j] + input_ids (torch.LongTensor): [N] + + Returns: + input_ids (torch.LongTensor): [N + image tokens] + num_image_tokens (torch.IntTensor): [n_images] + """ + + input_slices = [] + + start = 0 + for index in image_indices: + if self.add_special_token: + end = index + 1 + else: + end = index + + # original text tokens + input_slices.append(input_ids[start:end]) + + # add boi, image tokens, eoi and set the mask as False + input_slices.append(self.image_start_id * torch.ones((1), dtype=torch.long)) + input_slices.append( + self.image_id * torch.ones((self.num_image_tokens,), dtype=torch.long) + ) + input_slices.append(self.image_end_id * torch.ones((1), dtype=torch.long)) + start = index + 1 + + # the left part + input_slices.append(input_ids[start:]) + + # concat all slices + input_ids = torch.cat(input_slices, dim=0) + num_image_tokens = torch.IntTensor([self.num_image_tokens] * len(image_indices)) + + return input_ids, num_image_tokens + + def process_one( + self, + prompt: str = None, + images: List[Image] = None, + **kwargs, + ): + """ + + Args: + prompt (str): the formatted prompt; + images (List[ImageType]): the list of images; + **kwargs: + + Returns: + outputs (BaseProcessorOutput): the output of the processor, + - input_ids (torch.LongTensor): [N + image tokens] + - target_ids (torch.LongTensor): [N + image tokens] + - images (torch.FloatTensor): [n_images, 3, H, W] + - image_id (int): the id of the image token + - num_image_tokens (List[int]): the number of image tokens + """ + + sft_format = prompt + # tokenize + input_ids = self.tokenizer.encode(sft_format) + input_ids = torch.LongTensor(input_ids) + + # add image tokens to the input_ids + image_token_mask: torch.Tensor = (input_ids == self.image_id).to(torch.bool) + image_indices = image_token_mask.nonzero() + input_ids, num_image_tokens = self.add_image_token( + image_indices=image_indices, + input_ids=input_ids, + ) + + # load images + images_outputs = self.image_processor(images, return_tensors="pt") + + prepare = VLChatProcessorOutput( + sft_format=sft_format, + input_ids=input_ids, + pixel_values=images_outputs.pixel_values, + num_image_tokens=num_image_tokens, + ) + + return prepare + + def __call__( + self, + *, + prompt: str = None, + conversations: List[Dict[str, str]] = None, + images: List[Image] = None, + force_batchify: bool = True, + **kwargs, + ): + """ + + Args: + prompt (str): the formatted prompt; + conversations (List[Dict]): conversations with a list of messages; + images (List[ImageType]): the list of images; + force_batchify (bool): force batchify the inputs; + **kwargs: + + Returns: + outputs (BaseProcessorOutput): the output of the processor, + - input_ids (torch.LongTensor): [N + image tokens] + - images (torch.FloatTensor): [n_images, 3, H, W] + - image_id (int): the id of the image token + - num_image_tokens (List[int]): the number of image tokens + """ + + prepare = self.process_one( + prompt=prompt, conversations=conversations, images=images + ) + + if force_batchify: + prepare = self.batchify([prepare]) + + return prepare + + def batchify( + self, prepare_list: List[VLChatProcessorOutput] + ) -> BatchedVLChatProcessorOutput: + """ + Preprocesses the inputs for multimodal inference. + + Args: + prepare_list (List[VLChatProcessorOutput]): A list of VLChatProcessorOutput. + + Returns: + BatchedVLChatProcessorOutput: A dictionary of the inputs to use for multimodal inference. + """ + + batch_size = len(prepare_list) + sft_format = [] + n_images = [] + seq_lens = [] + for prepare in prepare_list: + n_images.append(len(prepare.num_image_tokens)) + seq_lens.append(len(prepare)) + + input_token_max_len = max(seq_lens) + max_n_images = max(1, max(n_images)) + + batched_input_ids = torch.full( + (batch_size, input_token_max_len), self.pad_id + ).long() # FIXME + batched_attention_mask = torch.zeros((batch_size, input_token_max_len)).long() + batched_pixel_values = torch.zeros( + (batch_size, max_n_images, *self.image_processor.default_shape) + ).float() + batched_images_seq_mask = torch.zeros((batch_size, input_token_max_len)).bool() + batched_images_emb_mask = torch.zeros( + (batch_size, max_n_images, self.num_image_tokens) + ).bool() + + for i, prepare in enumerate(prepare_list): + input_ids = prepare.input_ids + seq_len = len(prepare) + n_image = len(prepare.num_image_tokens) + # left-padding + batched_attention_mask[i, -seq_len:] = 1 + batched_input_ids[i, -seq_len:] = torch.LongTensor(input_ids) + batched_images_seq_mask[i, -seq_len:] = input_ids == self.image_id + + if n_image > 0: + batched_pixel_values[i, :n_image] = prepare.pixel_values + for j, n_image_tokens in enumerate(prepare.num_image_tokens): + batched_images_emb_mask[i, j, :n_image_tokens] = True + + sft_format.append(prepare.sft_format) + + batched_prepares = BatchedVLChatProcessorOutput( + input_ids=batched_input_ids, + attention_mask=batched_attention_mask, + pixel_values=batched_pixel_values, + images_seq_mask=batched_images_seq_mask, + images_emb_mask=batched_images_emb_mask, + sft_format=sft_format, + ) + + return batched_prepares + + +class VLMImageProcessorConfig(PretrainedConfig): + model_type = "deepseek_vlm" + image_size: int + min_size: int + image_mean: Union[Tuple[float, float, float], List[float]] + image_std: Union[Tuple[float, float, float], List[float]] + rescale_factor: float + do_normalize: bool + + def __init__( + self, + image_size: int, + min_size: int = 14, + image_mean: Union[Tuple[float, float, float], List[float]] = ( + 0.48145466, + 0.4578275, + 0.40821073, + ), + image_std: Union[Tuple[float, float, float], List[float]] = ( + 0.26862954, + 0.26130258, + 0.27577711, + ), + rescale_factor: float = 1.0 / 255.0, + do_normalize: bool = True, + **kwargs, + ): + self.image_size = image_size + self.min_size = min_size + self.image_mean = image_mean + self.image_std = image_std + self.rescale_factor = rescale_factor + self.do_normalize = do_normalize + + super().__init__(**kwargs) + + +register_processor(MultiModalityConfig, VLChatProcessor) +register_image_processor(MultiModalityConfig, VLMImageProcessor) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/jet_nemotron.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/jet_nemotron.py new file mode 100644 index 00000000..c05a2ec1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/jet_nemotron.py @@ -0,0 +1,74 @@ +from dataclasses import dataclass +from typing import Any + +from transformers.configuration_utils import PretrainedConfig + +from sglang.srt.configs.mamba_utils import Mamba2CacheParams, Mamba2StateShape + + +@dataclass +class JetBlockConfig: + mode: str + expand_v: float + num_heads: int + head_dim: int + norm_eps: str + conv_size: int + dconv_generator_reduction: int + dconv_implementation: str + + +class JetNemotronConfig(PretrainedConfig): + model_type: str = "jet_nemotron" + + efficient_attention_config: dict[str, dict[str, Any]] + hidden_act: str + hidden_size: int + initializer_range: float + intermediate_size: int + layer_types: list[str] + max_position_embeddings: int + num_attention_heads: int + num_key_value_heads: int + rms_norm_eps: float + rope_scaling: None + rope_theta: float + + @property + def full_attention_layer_ids(self) -> list[int]: + return [ + idx + for idx, layer_type in enumerate(self.layer_types) + if layer_type in ("attn", "swa") + ] + + @property + def linear_layer_ids(self) -> list[int]: + return [ + idx + for idx, layer_type in enumerate(self.layer_types) + if layer_type == "jet" + ] + + @property + def mamba2_cache_params(self) -> Mamba2CacheParams: + from sglang.srt.layers.dp_attention import get_attention_tp_size + + jet_block_config = JetBlockConfig(**self.efficient_attention_config["jet"]) + + num_heads = jet_block_config.num_heads + head_k_dim = jet_block_config.head_dim + head_v_dim = int(head_k_dim * jet_block_config.expand_v) + total_v_dim = num_heads * head_v_dim + + shape = Mamba2StateShape.create( + tp_world_size=get_attention_tp_size(), + intermediate_size=total_v_dim, + n_groups=num_heads, + num_heads=num_heads, + head_dim=head_v_dim, + state_size=head_k_dim, + conv_kernel=jet_block_config.conv_size, + ) + + return Mamba2CacheParams(shape=shape, layers=self.linear_layer_ids) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/jet_vlm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/jet_vlm.py new file mode 100644 index 00000000..9b8cba6e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/jet_vlm.py @@ -0,0 +1,53 @@ +from typing import Any + +from transformers.configuration_utils import PretrainedConfig +from transformers.models.siglip import SiglipVisionConfig + +from sglang.srt.configs.jet_nemotron import JetNemotronConfig +from sglang.srt.configs.mamba_utils import Mamba2CacheParams + + +class JetVLMConfig(PretrainedConfig): + model_type = "jet_vlm" + sub_configs = { + "text_config": JetNemotronConfig, + "vision_config": SiglipVisionConfig, + } + _auto_class = "AutoConfig" + + def __init__( + self, + *, + text_config: dict[str, Any] | None = None, + vision_config: dict[str, Any] | None = None, + image_token_id: int | None = None, + video_token_id: int | None = None, + **kwargs, + ): + self.text_config = ( + JetNemotronConfig(**text_config) + if text_config is not None + else JetNemotronConfig() + ) + self.vision_config = ( + SiglipVisionConfig(**vision_config) + if vision_config is not None + else SiglipVisionConfig() + ) + + self.image_token_id = image_token_id if image_token_id is not None else -1 + self.video_token_id = video_token_id if video_token_id is not None else -1 + + super().__init__(**kwargs) + + @property + def full_attention_layer_ids(self) -> list[int]: + return self.text_config.full_attention_layer_ids + + @property + def linear_layer_ids(self) -> list[int]: + return self.text_config.linear_layer_ids + + @property + def mamba2_cache_params(self) -> Mamba2CacheParams: + return self.text_config.mamba2_cache_params diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/kimi_linear.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/kimi_linear.py new file mode 100644 index 00000000..e7360904 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/kimi_linear.py @@ -0,0 +1,161 @@ +# Adapted from: https://github.com/vllm-project/vllm/blob/0384aa7150c4c9778efca041ffd1beb3ad2bd694/vllm/transformers_utils/configs/kimi_linear.py +from transformers.configuration_utils import PretrainedConfig + +from sglang.srt.configs.mamba_utils import KimiLinearCacheParams, KimiLinearStateShape + + +class KimiLinearConfig(PretrainedConfig): + model_type = "kimi_linear" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + model_type="kimi_linear", + vocab_size=163840, + hidden_size=4096, + head_dim=None, + intermediate_size=11008, + num_hidden_layers=32, + num_attention_heads=32, + num_key_value_heads=None, + hidden_act="silu", + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + rope_theta=10000.0, + rope_scaling=None, + tie_word_embeddings=False, + moe_intermediate_size: int | None = None, + moe_renormalize: bool = True, + moe_router_activation_func: str = "sigmoid", + num_experts: int | None = None, + num_experts_per_token: int | None = None, + num_shared_experts: int = 0, + routed_scaling_factor: float = 1.0, + first_k_dense_replace: int = 0, + moe_layer_freq: int = 1, + use_grouped_topk: bool = True, + num_expert_group: int = 1, + topk_group: int = 1, + q_lora_rank: int | None = None, + kv_lora_rank: int | None = None, + qk_nope_head_dim: int | None = None, + qk_rope_head_dim: int | None = None, + v_head_dim: int | None = None, + mla_use_nope: bool | None = False, + num_nextn_predict_layers: int = 0, + linear_attn_config: dict | None = None, + **kwargs, + ): + self.model_type = model_type + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.head_dim = ( + head_dim if head_dim is not None else hidden_size // num_attention_heads + ) + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + + # for backward compatibility + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + + self.q_lora_rank = q_lora_rank + self.kv_lora_rank = kv_lora_rank + self.qk_nope_head_dim = qk_nope_head_dim + self.qk_rope_head_dim = qk_rope_head_dim + self.v_head_dim = v_head_dim + self.mla_use_nope = mla_use_nope + # moe config + self.n_routed_experts = self.num_experts = num_experts + self.num_experts_per_token = num_experts_per_token + self.moe_renormalize = moe_renormalize + self.num_shared_experts = num_shared_experts + self.routed_scaling_factor = routed_scaling_factor + self.moe_router_activation_func = moe_router_activation_func + assert self.moe_router_activation_func in ("softmax", "sigmoid") + self.moe_intermediate_size = moe_intermediate_size + self.first_k_dense_replace = first_k_dense_replace + self.moe_layer_freq = moe_layer_freq + self.use_grouped_topk = use_grouped_topk + self.num_expert_group = num_expert_group + self.topk_group = topk_group + self.num_nextn_predict_layers = num_nextn_predict_layers + + if linear_attn_config is not None: + assert linear_attn_config["kda_layers"] is not None + assert linear_attn_config["full_attn_layers"] is not None + self.linear_attn_config = linear_attn_config + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + @property + def is_mla(self): + return ( + self.q_lora_rank is not None + or self.kv_lora_rank is not None + or self.qk_nope_head_dim is not None + or self.qk_rope_head_dim is not None + or self.v_head_dim is not None + or self.mla_use_nope is True + ) + + @property + def is_moe(self): + return self.num_experts is not None + + @property + def is_linear_attn(self) -> bool: + return not ( + self.linear_attn_config is None + or ( + isinstance(self.linear_attn_config, dict) + and self.linear_attn_config["kda_layers"] is not None + and len(self.linear_attn_config["kda_layers"]) == 0 + ) + ) + + def is_kda_layer(self, layer_idx: int): + return ( + self.linear_attn_config is not None + and (layer_idx + 1) in self.linear_attn_config["kda_layers"] + ) + + @property + def linear_layer_ids(self): + return [i for i in range(self.num_hidden_layers) if self.is_kda_layer(i)] + + @property + def full_attention_layer_ids(self): + return [i for i in range(self.num_hidden_layers) if not self.is_kda_layer(i)] + + @property + def mamba2_cache_params(self) -> KimiLinearCacheParams: + from sglang.srt.layers.dp_attention import get_attention_tp_size + + shape = KimiLinearStateShape.create( + tp_world_size=get_attention_tp_size(), + num_heads=self.linear_attn_config["num_heads"], + head_dim=self.linear_attn_config["head_dim"], + conv_kernel_size=self.linear_attn_config["short_conv_kernel_size"], + ) + + return KimiLinearCacheParams(shape=shape, layers=self.linear_layer_ids) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/kimi_vl.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/kimi_vl.py new file mode 100644 index 00000000..3c7d20f5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/kimi_vl.py @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: Apache-2.0 +# Adapted from https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct/blob/main/configuration_kimi_vl.py +from typing import Optional, Union + +from transformers.configuration_utils import PretrainedConfig + +from sglang.srt.configs.deepseekvl2 import DeepseekV2Config +from sglang.srt.configs.kimi_vl_moonvit import MoonViTConfig + + +class KimiVLConfig(PretrainedConfig): + model_type = "kimi_vl" + + def __init__( + self, + vision_config: Optional[Union[dict, MoonViTConfig]] = None, + text_config: Optional[Union[dict, DeepseekV2Config]] = None, + ignore_index: int = -100, + media_placeholder_token_id: int = 163605, + pad_token_id: int = 0, + **kwargs + ): + if vision_config is None: + vision_config = MoonViTConfig() + elif isinstance(vision_config, dict): + vision_config = MoonViTConfig(**vision_config) + self.vision_config = vision_config + + if text_config is None: + text_config = DeepseekV2Config() + elif isinstance(text_config, dict): + text_config = DeepseekV2Config(**text_config) + self.text_config = text_config + + self.ignore_index = ignore_index + self.media_placeholder_token_id = media_placeholder_token_id + + super().__init__(pad_token_id=pad_token_id, **kwargs) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/kimi_vl_moonvit.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/kimi_vl_moonvit.py new file mode 100644 index 00000000..166809eb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/kimi_vl_moonvit.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: Apache-2.0 +# Adapted from https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct/blob/main/configuration_kimi_vl.py +from transformers.configuration_utils import PretrainedConfig + + +class MoonViTConfig(PretrainedConfig): + model_type = "moonvit" + + def __init__( + self, + patch_size: int = 14, + init_pos_emb_height: int = 64, + init_pos_emb_width: int = 64, + num_attention_heads: int = 16, + num_hidden_layers: int = 27, + hidden_size: int = 1152, + intermediate_size: int = 4304, + merge_kernel_size: tuple[int, int] = (2, 2), + **kwargs, + ): + super().__init__(**kwargs) + self.patch_size = patch_size + # Positional embedding config + self.init_pos_emb_height = init_pos_emb_height + self.init_pos_emb_width = init_pos_emb_width + # Transformer config + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + # Patch merger config + self.merge_kernel_size = merge_kernel_size diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/load_config.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/load_config.py new file mode 100644 index 00000000..042eb322 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/load_config.py @@ -0,0 +1,121 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/config.py +import enum +import logging +from dataclasses import dataclass, field +from typing import List, Optional, Union + +import orjson + +from sglang.srt.configs.modelopt_config import ModelOptConfig +from sglang.srt.utils import is_hip + +logger = logging.getLogger(__name__) + + +class LoadFormat(str, enum.Enum): + AUTO = "auto" + PT = "pt" + SAFETENSORS = "safetensors" + NPCACHE = "npcache" + DUMMY = "dummy" + SHARDED_STATE = "sharded_state" + GGUF = "gguf" + BITSANDBYTES = "bitsandbytes" + MISTRAL = "mistral" + LAYERED = "layered" + JAX = "jax" + REMOTE = "remote" + REMOTE_INSTANCE = "remote_instance" + RDMA = "rdma" + LOCAL_CACHED = "local_cached" + + +@dataclass +class LoadConfig: + """ + download_dir: Directory to download and load the weights, default to the + default cache directory of huggingface. + load_format: The format of the model weights to load: + "auto" will try to load the weights in the safetensors format and + fall back to the pytorch bin format if safetensors format is + not available. + "pt" will load the weights in the pytorch bin format. + "safetensors" will load the weights in the safetensors format. + "npcache" will load the weights in pytorch format and store + a numpy cache to speed up the loading. + "dummy" will initialize the weights with random values, which is + mainly for profiling. + "bitsandbytes" will load nf4 type weights. + ignore_patterns: The list of patterns to ignore when loading the model. + Default to "original/**/*" to avoid repeated loading of llama's + checkpoints. + decryption_key_file: If set, decrypts the output files with a password read + from this file (after PBKDF2). + decrypt_max_concurrency: The maximum number of concurrent processes to decrypt the safetensor files. -1 means no limit. + + # ModelOpt-specific loading options + modelopt_checkpoint_restore_path: Optional[str] = None + modelopt_checkpoint_save_path: Optional[str] = None + modelopt_export_path: Optional[str] = None + """ + + load_format: Union[str, LoadFormat] = LoadFormat.AUTO + download_dir: Optional[str] = None + model_loader_extra_config: Optional[Union[str, dict]] = field(default_factory=dict) + ignore_patterns: Optional[Union[List[str], str]] = None + decryption_key_file: Optional[str] = None + decrypt_max_concurrency: int = -1 + tp_rank: Optional[int] = None + remote_instance_weight_loader_seed_instance_ip: Optional[str] = None + remote_instance_weight_loader_seed_instance_service_port: Optional[int] = None + remote_instance_weight_loader_send_weights_group_ports: Optional[List[int]] = None + + # ModelOpt-specific loading options + modelopt_checkpoint_restore_path: Optional[str] = None + modelopt_checkpoint_save_path: Optional[str] = None + modelopt_export_path: Optional[str] = None + + # ModelOpt configuration object + modelopt_config: Optional[ModelOptConfig] = None + + def __post_init__(self): + model_loader_extra_config = self.model_loader_extra_config or {} + if isinstance(model_loader_extra_config, str): + self.model_loader_extra_config = orjson.loads(model_loader_extra_config) + self._verify_load_format() + + if self.ignore_patterns is not None and len(self.ignore_patterns) > 0: + logger.info( + "Ignoring the following patterns when downloading weights: %s", + self.ignore_patterns, + ) + else: + self.ignore_patterns = ["original/**/*"] + + # Create ModelOptConfig if not provided + if self.modelopt_config is None: + self.modelopt_config = ModelOptConfig( + checkpoint_restore_path=self.modelopt_checkpoint_restore_path, + checkpoint_save_path=self.modelopt_checkpoint_save_path, + export_path=self.modelopt_export_path, + ) + + def _verify_load_format(self) -> None: + if not isinstance(self.load_format, str): + return + + load_format = self.load_format.lower() + self.load_format = LoadFormat(load_format) + + rocm_not_supported_load_format: List[str] = [] + if is_hip() and load_format in rocm_not_supported_load_format: + rocm_supported_load_format = [ + f + for f in LoadFormat.__members__ + if (f not in rocm_not_supported_load_format) + ] + raise ValueError( + f"load format '{load_format}' is not supported in ROCm. " + f"Supported load formats are " + f"{rocm_supported_load_format}" + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/longcat_flash.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/longcat_flash.py new file mode 100644 index 00000000..e6a2dfb0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/longcat_flash.py @@ -0,0 +1,104 @@ +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + +logger = logging.get_logger(__name__) + +FLASH_PRETRAINED_CONFIG_ARCHIVE_MAP = {} + + +class LongcatFlashConfig(PretrainedConfig): + model_type = "longcat_flash" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + vocab_size=131072, + hidden_size=6144, + intermediate_size=None, + ffn_hidden_size=12288, + expert_ffn_hidden_size=2048, + num_layers=28, + num_hidden_layers=None, + num_attention_heads=64, + ep_size=1, + kv_lora_rank=512, + q_lora_rank=1536, + qk_rope_head_dim=128, + qk_nope_head_dim=128, + v_head_dim=128, + n_routed_experts=512, + moe_topk=12, + norm_topk_prob=False, + max_position_embeddings=131072, + rms_norm_eps=1e-05, + use_cache=True, + pad_token_id=None, + bos_token_id=1, + eos_token_id=2, + pretraining_tp=1, + tie_word_embeddings=False, + rope_theta=10000000.0, + rope_scaling=None, + attention_bias=False, + attention_dropout=0.0, + mla_scale_q_lora=True, + mla_scale_kv_lora=True, + torch_dtype="bfloat16", + params_dtype="bfloat16", + rounter_params_dtype="float32", + router_bias=False, + topk_method=None, + routed_scaling_factor=6.0, + zero_expert_num=256, + zero_expert_type="identity", + nextn_use_scmoe=False, + num_nextn_predict_layers=1, + **kwargs, + ): + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + torch_dtype=torch_dtype, + params_dtype=params_dtype, + rounter_params_dtype=rounter_params_dtype, + topk_method=topk_method, + router_bias=router_bias, + nextn_use_scmoe=nextn_use_scmoe, + num_nextn_predict_layers=num_nextn_predict_layers, + **kwargs, + ) + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.num_hidden_layers = ( + num_hidden_layers if num_hidden_layers is not None else num_layers + ) + self.intermediate_size = ( + intermediate_size if intermediate_size is not None else ffn_hidden_size + ) + self.moe_intermediate_size = expert_ffn_hidden_size + self.num_attention_heads = num_attention_heads + self.ep_size = ep_size + self.kv_lora_rank = kv_lora_rank + self.q_lora_rank = q_lora_rank + self.qk_rope_head_dim = qk_rope_head_dim + self.v_head_dim = v_head_dim + self.qk_nope_head_dim = qk_nope_head_dim + self.n_routed_experts = n_routed_experts + self.moe_topk = moe_topk + self.norm_topk_prob = norm_topk_prob + self.rms_norm_eps = rms_norm_eps + self.pretraining_tp = pretraining_tp + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + self.attention_bias = attention_bias + self.attention_dropout = attention_dropout + self.mla_scale_q_lora = mla_scale_q_lora + self.mla_scale_kv_lora = mla_scale_kv_lora + self.zero_expert_num = zero_expert_num + self.zero_expert_type = zero_expert_type + self.routed_scaling_factor = routed_scaling_factor + self.hidden_act = "silu" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/mamba_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/mamba_utils.py new file mode 100644 index 00000000..d2ff3762 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/mamba_utils.py @@ -0,0 +1,183 @@ +# Copyright 2025 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Common config utils for mamba2 - NemotronH, FalconH1, Qwen3Next, etc.""" + +import os +from abc import ABC +from dataclasses import dataclass, field +from typing import List, Optional + +import numpy as np +import torch + +from sglang.srt.distributed.utils import divide + + +def extra_groups_for_head_shards(ngroups: int, tp_size: int): + """Compute the increase in group numbers to account for + replication in order to accompany the head shards.""" + + # in the case ngoups % tp_size == 0, this will be zero + if ngroups % tp_size == 0: + return 0 + + # for n_groups == 1, this is exactly tp_size - n_groups + return tp_size - ngroups + + +@dataclass(kw_only=True, frozen=True) +class Mamba2StateDType: + conv: torch.dtype + temporal: torch.dtype + + +CONV_DTYPE = torch.bfloat16 + + +def mamba2_state_dtype() -> Mamba2StateDType: + dtype_map = { + "float32": torch.float32, + "bfloat16": torch.bfloat16, + } + ssm_dtype = dtype_map[os.environ["SGLANG_MAMBA_SSM_DTYPE"]] + return Mamba2StateDType(conv=CONV_DTYPE, temporal=ssm_dtype) + + +@dataclass(kw_only=True, frozen=True) +class BaseLinearStateParams(ABC): + dtype: Mamba2StateDType = field(default_factory=mamba2_state_dtype) + layers: list[int] + + @property + def mamba_cache_per_req(self) -> int: + conv_numel = int( + np.sum([np.prod(conv_shape) for conv_shape in self.shape.conv]) + ) + + ssm_numel = int(np.prod(self.shape.temporal)) + return ( + conv_numel * self.dtype.conv.itemsize + + ssm_numel * self.dtype.temporal.itemsize + ) * len(self.layers) + + +@dataclass(kw_only=True, frozen=True) +class Mamba2StateShape: + conv: list[tuple[int, int]] + temporal: tuple[int, int, int] + + intermediate_size: int + conv_dim: int + ssm_state_size: int + num_heads: int + head_dim: int + state_size: int + conv_kernel: int + + @staticmethod + def create( + *, + tp_world_size: int, + intermediate_size: int, + n_groups: int, + num_heads: int, + head_dim: int, + state_size: int, + conv_kernel: int, + ) -> "Mamba2StateShape": + # if n_groups is not divisible by world_size, need to extend the shards + # to ensure all groups needed by a head is sharded along with it + if n_groups % tp_world_size != 0: + extra_groups = extra_groups_for_head_shards(n_groups, tp_world_size) + n_groups += extra_groups + # heads and n_groups are TP-ed + conv_dim = intermediate_size + 2 * n_groups * state_size + + # contiguous along 'dim' axis + conv_state_shape = divide(conv_dim, tp_world_size), conv_kernel - 1 + + # These are not TP-ed as they depend on A, dt_bias, D + # - they are typically small + # e.g., QWen3-Next: (32, 128, 128) + temporal_state_shape = (divide(num_heads, tp_world_size), head_dim, state_size) + return Mamba2StateShape( + conv=[conv_state_shape], + temporal=temporal_state_shape, + intermediate_size=intermediate_size, + conv_dim=conv_dim, + ssm_state_size=state_size, + num_heads=num_heads, + head_dim=head_dim, + state_size=state_size, + conv_kernel=conv_kernel, + ) + + +@dataclass(kw_only=True, frozen=True) +class Mamba2CacheParams(BaseLinearStateParams): + shape: Mamba2StateShape + + +@dataclass(kw_only=True, frozen=True) +class KimiLinearStateShape: + conv: List[tuple[int, int]] + temporal: tuple[int, int, int] + + num_heads: int + head_dim: int + num_k_heads: int + head_k_dim: int + conv_kernel: int + num_spec: int + + @staticmethod + def create( + *, + tp_world_size: int, + num_heads: int, + head_dim: int, + num_k_heads: Optional[int] = None, + head_k_dim: Optional[int] = None, + conv_kernel_size: int = 4, + num_spec: int = 0, + ) -> "KimiLinearStateShape": + if num_k_heads is None: + num_k_heads = num_heads + if head_k_dim is None: + head_k_dim = head_dim + + proj_size = num_heads * head_dim + proj_k_size = num_k_heads * head_k_dim + + conv_state_shape = (divide(proj_size, tp_world_size), conv_kernel_size - 1) + conv_state_k_shape = (divide(proj_k_size, tp_world_size), conv_kernel_size - 1) + temporal_state_shape = (divide(num_heads, tp_world_size), head_dim, head_dim) + + conv_state_shape = conv_state_shape[1], conv_state_shape[0] + conv_state_k_shape = conv_state_k_shape[1], conv_state_k_shape[0] + + return KimiLinearStateShape( + conv=[conv_state_shape, conv_state_k_shape, conv_state_k_shape], + temporal=temporal_state_shape, + num_heads=num_heads, + head_dim=head_dim, + num_k_heads=num_k_heads, + head_k_dim=head_k_dim, + conv_kernel=conv_kernel_size, + num_spec=num_spec, + ) + + +@dataclass(kw_only=True, frozen=True) +class KimiLinearCacheParams(BaseLinearStateParams): + shape: KimiLinearStateShape diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/model_config.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/model_config.py new file mode 100644 index 00000000..26dfbe5e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/model_config.py @@ -0,0 +1,1043 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import json +import logging +import math +import os +from enum import Enum, IntEnum, auto +from typing import Any, List, Optional, Set, Union + +import torch +from transformers import PretrainedConfig + +from sglang.srt.environ import envs +from sglang.srt.layers.quantization import QUANTIZATION_METHODS +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import is_hip, retry +from sglang.srt.utils.hf_transformers_utils import ( + get_config, + get_context_length, + get_generation_config, + get_hf_text_config, + get_sparse_attention_config, +) +from sglang.utils import is_in_ci + +logger = logging.getLogger(__name__) + + +class AttentionArch(IntEnum): + MLA = auto() + MHA = auto() + + +class ModelImpl(str, Enum): + AUTO = "auto" + SGLANG = "sglang" + TRANSFORMERS = "transformers" + MINDSPORE = "mindspore" + + +def is_deepseek_nsa(config: PretrainedConfig) -> bool: + return ( + config.architectures is not None + and config.architectures[0] + in [ + "DeepseekV3ForCausalLM", + "DeepseekV32ForCausalLM", + "DeepseekV3ForCausalLMNextN", + ] + and getattr(config, "index_topk", None) is not None + ) + + +def get_nsa_index_head_dim(config: PretrainedConfig) -> int: + assert is_deepseek_nsa(config) + return config.index_head_dim + + +def get_nsa_index_topk(config: PretrainedConfig) -> int: + assert is_deepseek_nsa(config) + return config.index_topk + + +def get_nsa_index_n_heads(config: PretrainedConfig) -> int: + assert is_deepseek_nsa(config) + return config.index_n_heads + + +class ModelConfig: + def __init__( + self, + model_path: str, + trust_remote_code: bool = True, + revision: Optional[str] = None, + context_length: Optional[int] = None, + model_override_args: str = "{}", + is_embedding: Optional[bool] = None, + enable_multimodal: Optional[bool] = None, + dtype: str = "auto", + quantization: Optional[str] = None, + override_config_file: Optional[str] = None, + is_draft_model: bool = False, + hybrid_kvcache_ratio: Optional[ + float + ] = None, # TODO: remove this, it is not a model config + model_impl: Union[str, ModelImpl] = ModelImpl.AUTO, + sampling_defaults: str = "openai", + quantize_and_serve: bool = False, + ) -> None: + # Parse args + self.model_path = model_path + self.revision = revision + self.quantization = quantization + self.is_draft_model = is_draft_model + self.model_impl = model_impl + self.sampling_defaults = sampling_defaults + self.quantize_and_serve = quantize_and_serve + + # Validate quantize_and_serve configuration + self._validate_quantize_and_serve_config() + + # Get hf config + self._maybe_pull_model_tokenizer_from_remote() + self.model_override_args = json.loads(model_override_args) + kwargs = {} + if override_config_file and override_config_file.strip(): + kwargs["_configuration_file"] = override_config_file.strip() + self.hf_config = get_config( + self.model_path, + trust_remote_code=trust_remote_code, + revision=revision, + model_override_args=self.model_override_args, + **kwargs, + ) + self.hf_text_config = get_hf_text_config(self.hf_config) + self.hf_generation_config = get_generation_config( + self.model_path, + trust_remote_code=trust_remote_code, + revision=revision, + **kwargs, + ) + + # Set enable_multimodal + if enable_multimodal is None: + mm_disabled_models = [ + "Gemma3ForConditionalGeneration", + "Llama4ForConditionalGeneration", + "Step3VLForConditionalGeneration", + ] + if self.hf_config.architectures[0] in mm_disabled_models: + enable_multimodal = False + logger.info( + f"Multimodal is disabled for {self.hf_config.model_type}. To enable it, set --enable-multimodal." + ) + else: + enable_multimodal = True + + # Config draft model + self._config_draft_model() + + # Check model type + self.attention_chunk_size = getattr( + self.hf_text_config, "attention_chunk_size", None + ) + self.is_hybrid_swa = is_hybrid_model( + self.hf_config.architectures, + hybrid_kvcache_ratio=hybrid_kvcache_ratio, + context_length=context_length, + attention_chunk_size=self.attention_chunk_size, + ) + if self.is_hybrid_swa is not None: + self.swa_attention_layer_ids, self.full_attention_layer_ids = ( + get_hybrid_layer_ids( + self.hf_config.architectures, self.hf_text_config.num_hidden_layers + ) + ) + self.is_generation = is_generation_model( + self.hf_config.architectures, is_embedding + ) + self.is_multimodal = enable_multimodal and is_multimodal_model( + self.hf_config.architectures + ) + self.is_multimodal_gen = enable_multimodal and is_multimodal_gen_model( + self.hf_config.architectures + ) + self.is_image_gen = enable_multimodal and is_image_gen_model( + self.hf_config.architectures + ) + self.is_audio_model = enable_multimodal and is_audio_model( + self.hf_config.architectures + ) + # TODO: requires further polishing + self.is_image_understandable_model = enable_multimodal and hasattr( + self.hf_config, "vision_config" + ) + self.is_audio_understandable_model = enable_multimodal and hasattr( + self.hf_config, "audio_config" + ) + + self.is_multimodal_chunked_prefill_supported = ( + enable_multimodal + and is_multimodal_chunked_prefill_supported(self.hf_config.architectures) + ) + self.is_encoder_decoder = is_encoder_decoder_model(self.hf_config.architectures) + self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype) + + # Derive context length and model shapes + self._derive_context_length(context_length) + self._derive_model_shapes() + + # Verify quantization + self._verify_quantization() + + # Verify dual-chunk attention config + self._verify_dual_chunk_attention_config() + + # Cache attributes + self.hf_eos_token_id = self._get_hf_eos_token_id() + + # multimodal + self.image_token_id = getattr( + self.hf_config, "image_token_id", None + ) or getattr(self.hf_config, "image_token_index", None) + + # matryoshka embeddings + self.matryoshka_dimensions = getattr( + self.hf_config, "matryoshka_dimensions", None + ) + self.is_matryoshka = self.matryoshka_dimensions or getattr( + self.hf_config, "is_matryoshka", False + ) + + @staticmethod + def from_server_args( + server_args: ServerArgs, + model_path: str = None, + model_revision: str = None, + **kwargs, + ): + return ModelConfig( + model_path=model_path or server_args.model_path, + trust_remote_code=server_args.trust_remote_code, + revision=model_revision or server_args.revision, + context_length=server_args.context_length, + model_override_args=server_args.json_model_override_args, + is_embedding=server_args.is_embedding, + enable_multimodal=server_args.enable_multimodal, + dtype=server_args.dtype, + quantization=server_args.quantization, + hybrid_kvcache_ratio=server_args.hybrid_kvcache_ratio, + model_impl=server_args.model_impl, + sampling_defaults=server_args.sampling_defaults, + quantize_and_serve=server_args.quantize_and_serve, + override_config_file=server_args.decrypted_config_file, + **kwargs, + ) + + def _config_draft_model(self): + is_draft_model = self.is_draft_model + + if ( + is_draft_model + and self.hf_config.architectures[0] == "DeepseekV3ForCausalLM" + ): + self.hf_config.architectures[0] = "DeepseekV3ForCausalLMNextN" + + if is_draft_model and self.hf_config.architectures[0] == "Glm4MoeForCausalLM": + self.hf_config.architectures[0] = "Glm4MoeForCausalLMNextN" + + if ( + is_draft_model + and self.hf_config.architectures[0] == "LongcatFlashForCausalLM" + ): + self.hf_config.architectures[0] = "LongcatFlashForCausalLMNextN" + self.hf_config.num_hidden_layers = self.hf_config.num_nextn_predict_layers + + if is_draft_model and self.hf_config.architectures[0] == "MiMoForCausalLM": + self.hf_config.architectures[0] = "MiMoMTP" + if is_draft_model and self.hf_config.architectures[0] in [ + "BailingMoeV2ForCausalLM", + "BailingMoeForCausalLM", + ]: + self.hf_config.architectures[0] = "BailingMoeForCausalLMNextN" + if ( + is_draft_model + and self.hf_config.architectures[0] == "Ernie4_5_MoeForCausalLM" + ): + self.hf_config.architectures[0] = "Ernie4_5_MoeForCausalLMMTP" + + if is_draft_model and self.hf_config.architectures[0] == "Qwen3NextForCausalLM": + self.hf_config.architectures[0] = "Qwen3NextForCausalLMMTP" + self.hf_config.num_nextn_predict_layers = 1 + + def _derive_context_length(self, context_length: int): + is_draft_model = self.is_draft_model + derived_context_len = get_context_length(self.hf_text_config) + + if context_length is not None: + if context_length > derived_context_len: + reason = "Target model's" if is_draft_model else "User-specified" + msg = ( + f"Warning: {reason} context_length ({context_length}) is greater than the derived context_length ({derived_context_len}). " + f"This may lead to incorrect model outputs or CUDA errors. Note that the derived context_length may differ from max_position_embeddings in the model's config." + ) + if ( + envs.SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN.get() + or is_in_ci() # FIXME: fix this special case + ): + logger.warning(msg) + self.context_len = context_length + if is_draft_model: + self.hf_text_config.max_position_embeddings = context_length + logger.warning( + f"Overriding the draft model's max_position_embeddings to {context_length}." + ) + else: + raise ValueError( + f"{msg} To allow overriding this maximum, set the env var SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1" + ) + else: + self.context_len = context_length + else: + self.context_len = derived_context_len + + # Transfer context_len to HuggingFace config so models can access it + self.hf_config.context_len = self.context_len + + def _derive_model_shapes(self): + # Unify the config keys for hf_text_config + self.head_dim = getattr( + self.hf_text_config, + "head_dim", + self.hf_text_config.hidden_size // self.hf_text_config.num_attention_heads, + ) + + # FIXME: temporary special judge for MLA architecture + if ( + "DeepseekV2ForCausalLM" in self.hf_config.architectures + or "DeepseekV32ForCausalLM" in self.hf_config.architectures + or "DeepseekV3ForCausalLM" in self.hf_config.architectures + or "DeepseekV3ForCausalLMNextN" in self.hf_config.architectures + or "LongcatFlashForCausalLM" in self.hf_config.architectures + or "LongcatFlashForCausalLMNextN" in self.hf_config.architectures + or "DotsVLMForCausalLM" in self.hf_config.architectures + ): + self.head_dim = 256 + self.attention_arch = AttentionArch.MLA + self.kv_lora_rank = self.hf_config.kv_lora_rank + self.qk_nope_head_dim = self.hf_config.qk_nope_head_dim + self.qk_rope_head_dim = self.hf_config.qk_rope_head_dim + self.v_head_dim = self.hf_config.v_head_dim + self.index_head_dim = ( + get_nsa_index_head_dim(self.hf_config) + if is_deepseek_nsa(self.hf_config) + else None + ) + + # Handle rope scaling with yarn + self.scaling = 1 / math.sqrt(self.qk_nope_head_dim + self.qk_rope_head_dim) + if self.hf_config.rope_scaling: + mscale_all_dim = self.hf_config.rope_scaling.get( + "mscale_all_dim", False + ) + scaling_factor = self.hf_config.rope_scaling["factor"] + mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim)) + self.scaling = self.scaling * mscale * mscale + + elif "MiniCPM3ForCausalLM" in self.hf_config.architectures: + self.head_dim = 128 + self.attention_arch = AttentionArch.MLA + self.kv_lora_rank = self.hf_config.kv_lora_rank + self.qk_rope_head_dim = self.hf_config.qk_rope_head_dim + elif "DeepseekVL2ForCausalLM" in self.hf_config.architectures and getattr( + self.hf_text_config, "use_mla", True + ): + self.head_dim = 256 + self.attention_arch = AttentionArch.MLA + self.kv_lora_rank = self.hf_text_config.kv_lora_rank + self.qk_rope_head_dim = self.hf_text_config.qk_rope_head_dim + elif "KimiVLForConditionalGeneration" in self.hf_config.architectures: + self.head_dim = 256 + self.attention_arch = AttentionArch.MLA + self.kv_lora_rank = self.hf_text_config.kv_lora_rank + self.qk_rope_head_dim = self.hf_text_config.qk_rope_head_dim + self.v_head_dim = self.hf_text_config.v_head_dim + self.qk_nope_head_dim = self.hf_text_config.qk_nope_head_dim + elif "KimiLinearForCausalLM" in self.hf_config.architectures: + self.head_dim = 72 + self.attention_arch = AttentionArch.MLA + self.kv_lora_rank = self.hf_config.kv_lora_rank + self.qk_rope_head_dim = self.hf_config.qk_rope_head_dim + self.v_head_dim = self.hf_config.v_head_dim + self.qk_nope_head_dim = self.hf_config.qk_nope_head_dim + else: + if ( + "MistralModel" in self.hf_config.architectures + or "MixtralForCausalLM" in self.hf_config.architectures + or "MistralForCausalLM" in self.hf_config.architectures + ): + if getattr(self, "head_dim", None) is None: + self.head_dim = ( + self.hf_config.hidden_size // self.hf_config.num_attention_heads + ) + # In transformers==4.52.3, the head_dim is null in MistralConfig + if ( + not hasattr(self.hf_text_config, "head_dim") + or self.hf_text_config.head_dim is None + ): + setattr(self.hf_text_config, "head_dim", self.head_dim) + + self.attention_arch = AttentionArch.MHA + + self.num_attention_heads = self.hf_text_config.num_attention_heads + self.num_key_value_heads = getattr( + self.hf_text_config, "num_key_value_heads", None + ) + + # for Dbrx and MPT models + if self.hf_config.model_type in ["dbrx", "mpt"]: + self.num_key_value_heads = getattr( + self.hf_config.attn_config, "kv_n_heads", None + ) + + if self.num_key_value_heads is None: + self.num_key_value_heads = self.num_attention_heads + self.hidden_size = self.hf_text_config.hidden_size + self.num_hidden_layers = self.hf_text_config.num_hidden_layers + self.num_attention_layers = self.num_hidden_layers + if "LongcatFlashForCausalLM" in self.hf_config.architectures: + self.num_attention_layers = self.num_hidden_layers * 2 + self.num_nextn_predict_layers = getattr( + self.hf_text_config, "num_nextn_predict_layers", None + ) + self.vocab_size = self.hf_text_config.vocab_size + + def get_total_num_attention_heads(self) -> int: + return self.num_attention_heads + + def get_num_attention_heads(self, tensor_parallel_size) -> int: + total_num_attention_heads = self.num_attention_heads + return max(1, total_num_attention_heads // tensor_parallel_size) + + # adapted from https://github.com/vllm-project/vllm/blob/main/vllm/config.py#L289 + def get_total_num_kv_heads(self) -> int: + """Returns the total number of KV heads.""" + # For GPTBigCode & Falcon: + # NOTE: for falcon, when new_decoder_architecture is True, the + # multi_query flag is ignored and we use n_head_kv for the number of + # KV heads. + falcon_model_types = ["falcon", "RefinedWeb", "RefinedWebModel"] + new_decoder_arch_falcon = ( + self.hf_config.model_type in falcon_model_types + and getattr(self.hf_config, "new_decoder_architecture", False) + ) + if not new_decoder_arch_falcon and getattr( + self.hf_text_config, "multi_query", False + ): + # Multi-query attention, only one KV head. + # Currently, tensor parallelism is not supported in this case. + return 1 + + # For DBRX and MPT + if self.hf_config.model_type in ["mpt"]: + if "kv_n_heads" in self.hf_config.attn_config: + return self.hf_config.attn_config["kv_n_heads"] + return self.hf_config.num_attention_heads + if self.hf_config.model_type in ["dbrx"]: + return getattr( + self.hf_config.attn_config, + "kv_n_heads", + self.hf_config.num_attention_heads, + ) + if self.hf_config.model_type in ["nemotron-nas"]: + nkvh = { + self.hf_config.num_attention_heads // block.attention.n_heads_in_group + for block in self.hf_config.block_configs + if not block.attention.no_op + } + if len(nkvh) == 0: + raise RuntimeError("Couldn't determine number of kv heads") + if len(nkvh) > 1: + raise ValueError( + "Variable GQA (VGQA) is not yet supported for nemotron-nas in sglang" + ) + return next(iter(nkvh)) + + attributes = [ + # For Falcon: + "n_head_kv", + "num_kv_heads", + # For LLaMA-2: + "num_key_value_heads", + # For ChatGLM: + "multi_query_group_num", + # For Step3 + "num_attention_groups", + ] + for attr in attributes: + num_kv_heads = getattr(self.hf_text_config, attr, None) + if num_kv_heads is not None: + return num_kv_heads + + # For non-grouped-query attention models, the number of KV heads is + # equal to the number of attention heads. + return self.hf_text_config.num_attention_heads + + def get_num_kv_heads(self, tensor_parallel_size) -> int: + """Returns the number of KV heads per GPU.""" + total_num_kv_heads = self.get_total_num_kv_heads() + # If tensor parallelism is used, we divide the number of KV heads by + # the tensor parallel size. We will replicate the KV heads in the + # case where the number of KV heads is smaller than the tensor + # parallel size so each GPU has at least one KV head. + return max(1, total_num_kv_heads // tensor_parallel_size) + + # adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/config.py + def _parse_quant_hf_config(self): + quant_cfg = getattr(self.hf_config, "quantization_config", None) + if quant_cfg is None: + # compressed-tensors uses a "compression_config" key + quant_cfg = getattr(self.hf_config, "compression_config", None) + if quant_cfg is None: + # check if is modelopt or mixed-precision model -- Both of them don't have corresponding field + # in hf `config.json` but has a standalone `hf_quant_config.json` in the root directory + # example: https://huggingface.co/nvidia/Llama-3.1-8B-Instruct-FP8/tree/main + # example: https://huggingface.co/Barrrrry/DeepSeek-R1-W4AFP8/tree/main + is_local = os.path.exists(self.model_path) + if not is_local: + import huggingface_hub + + try: + from huggingface_hub import HfApi, hf_hub_download + + hf_api = HfApi() + # Retry HF API call up to 3 times + file_exists = retry( + lambda: hf_api.file_exists( + self.model_path, "hf_quant_config.json" + ), + max_retry=2, + initial_delay=1.0, + max_delay=5.0, + ) + if file_exists: + # Download and parse the quantization config for remote models + quant_config_file = hf_hub_download( + repo_id=self.model_path, + filename="hf_quant_config.json", + revision=self.revision, + ) + with open(quant_config_file) as f: + quant_config_dict = json.load(f) + quant_cfg = self._parse_modelopt_quant_config(quant_config_dict) + except huggingface_hub.errors.OfflineModeIsEnabled: + logger.warning( + "Offline mode is enabled, skipping hf_quant_config.json check" + ) + except Exception as e: + logger.warning( + f"Failed to check hf_quant_config.json: {self.model_path} {e}" + ) + elif os.path.exists(os.path.join(self.model_path, "hf_quant_config.json")): + quant_config_file = os.path.join( + self.model_path, "hf_quant_config.json" + ) + with open(quant_config_file) as f: + quant_config_dict = json.load(f) + quant_cfg = self._parse_modelopt_quant_config(quant_config_dict) + return quant_cfg + + def _parse_modelopt_quant_config(self, quant_config_dict: dict) -> Optional[dict]: + """Parse ModelOpt quantization config and return the appropriate quant_method.""" + json_quant_configs = quant_config_dict["quantization"] + quant_algo = json_quant_configs.get("quant_algo", None) + + if quant_algo == "MIXED_PRECISION": + return {"quant_method": "w4afp8"} + elif quant_algo and ("FP4" in quant_algo or "NVFP4" in quant_algo): + return {"quant_method": "modelopt_fp4"} + elif quant_algo and "FP8" in quant_algo: + return {"quant_method": "modelopt_fp8"} + else: + return None + + def _is_already_quantized(self) -> bool: + """Check if the model is already quantized based on config files.""" + # Check for HuggingFace quantization config + from sglang.srt.utils import has_hf_quant_config + + return has_hf_quant_config(self.model_path) + + def _get_modelopt_quant_type(self) -> str: + """Extract ModelOpt quantization type from unified quantization flag.""" + if self.quantization == "modelopt_fp8": + return "fp8" + elif self.quantization == "modelopt_fp4": + return "nvfp4" + elif self.quantization == "modelopt": + # Auto-detect from model config + quant_cfg = self._parse_quant_hf_config() + if quant_cfg: + quant_method = quant_cfg.get("quant_method", "").lower() + if "fp4" in quant_method: + return "fp4" + elif "fp8" in quant_method: + return "fp8" + # Default to fp8 if can't detect + return "fp8" + else: + return "fp8" # Default fallback + + def _validate_quantize_and_serve_config(self): + """Validate quantize_and_serve configuration.""" + if not self.quantize_and_serve: + return + + # Check if ModelOpt quantization is specified + _MODELOPT_QUANTIZATION_METHODS = [ + "modelopt", + "modelopt_fp8", + "modelopt_fp4", + ] + modelopt_quantization_specified = ( + self.quantization in _MODELOPT_QUANTIZATION_METHODS + ) + + if not modelopt_quantization_specified: + raise ValueError( + "quantize_and_serve requires ModelOpt quantization (set with --quantization " + f"{{{', '.join(sorted(_MODELOPT_QUANTIZATION_METHODS))}}})" + ) + + # quantize_and_serve is disabled due to compatibility issues + raise NotImplementedError( + "quantize_and_serve functionality is currently disabled due to compatibility issues. " + "Please use the separate quantize-then-deploy workflow instead. " + "Step 1: Quantize and export model. " + "Step 2: Deploy the exported model." + ) + + # adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/config.py + def _verify_quantization(self) -> None: + supported_quantization = [*QUANTIZATION_METHODS] + rocm_supported_quantization = [ + "awq", + "gptq", + "fp8", + "compressed_tensors", + "compressed-tensors", + "fbgemm_fp8", + "w8a8_fp8", + "petit_nvfp4", + "quark", + "mxfp4", + "auto-round", + ] + optimized_quantization_methods = [ + "fp8", + "marlin", + "modelopt_fp8", + "modelopt_fp4", + "gptq_marlin_24", + "gptq_marlin", + "awq_marlin", + "fbgemm_fp8", + "compressed_tensors", + "compressed-tensors", + "experts_int8", + "w8a8_int8", + "w8a8_fp8", + "moe_wna16", + "qoq", + "w4afp8", + "petit_nvfp4", + "quark", + ] + compatible_quantization_methods = { + "modelopt_fp8": ["modelopt"], + "modelopt_fp4": ["modelopt"], + "petit_nvfp4": ["modelopt"], + "w8a8_int8": ["compressed-tensors", "compressed_tensors"], + "w8a8_fp8": ["compressed-tensors", "compressed_tensors"], + } + if self.quantization is not None: + self.quantization = self.quantization.lower() + + # Parse quantization method from the HF model config, if available. + quant_cfg = self._parse_quant_hf_config() + + if quant_cfg is not None: + quant_method = quant_cfg.get( + "quant_method", "" if not self.quantization else self.quantization + ).lower() + + # Detect which checkpoint is it + for _, method in QUANTIZATION_METHODS.items(): + quantization_override = method.override_quantization_method( + quant_cfg, self.quantization + ) + if quantization_override: + quant_method = quantization_override + self.quantization = quantization_override + break + + # Verify quantization configurations. + if self.quantization is None: + self.quantization = quant_method + elif self.quantization != quant_method: + if ( + self.quantization not in compatible_quantization_methods + or quant_method + not in compatible_quantization_methods[self.quantization] + ): + raise ValueError( + "Quantization method specified in the model config " + f"({quant_method}) does not match the quantization " + f"method specified in the `quantization` argument " + f"({self.quantization})." + ) + + if self.quantization is not None: + if self.quantization not in supported_quantization: + raise ValueError( + f"Unknown quantization method: {self.quantization}. Must " + f"be one of {supported_quantization}." + ) + if is_hip() and self.quantization not in rocm_supported_quantization: + raise ValueError( + f"{self.quantization} quantization is currently not " + f"supported in ROCm." + ) + if self.quantization not in optimized_quantization_methods: + logger.warning( + "%s quantization is not fully " + "optimized yet. The speed can be slower than " + "non-quantized models.", + self.quantization, + ) + + def _verify_dual_chunk_attention_config(self) -> None: + if hasattr(self.hf_config, "dual_chunk_attention_config"): + # Try loading the sparse attention config + sparse_attn_config = get_sparse_attention_config(self.model_path) + if not sparse_attn_config: + return + self.hf_config.dual_chunk_attention_config["sparse_attention_config"] = ( + sparse_attn_config + ) + if ( + "sparse_attention_enabled" + not in self.hf_config.dual_chunk_attention_config + ): + self.hf_config.dual_chunk_attention_config[ + "sparse_attention_enabled" + ] = True + + def _get_hf_eos_token_id(self) -> Optional[Set[int]]: + eos_ids = getattr(self.hf_config, "eos_token_id", None) + if eos_ids is not None: + # it can be either int or list of int + eos_ids = {eos_ids} if isinstance(eos_ids, int) else set(eos_ids) + if eos_ids is None: + eos_ids = set() + if self.hf_generation_config: + generation_eos_ids = getattr( + self.hf_generation_config, "eos_token_id", None + ) + if generation_eos_ids: + generation_eos_ids = ( + {generation_eos_ids} + if isinstance(generation_eos_ids, int) + else set(generation_eos_ids) + ) + eos_ids = eos_ids | generation_eos_ids + return eos_ids + + def get_default_sampling_params(self) -> dict[str, Any]: + """ + Get default sampling parameters from the model's generation config. + + This method returns non-default sampling parameters from the model's + generation_config.json when sampling_defaults is set to "model". + + Returns: + A dictionary containing the non-default sampling parameters. + """ + if self.sampling_defaults != "model": + return {} + + if self.hf_generation_config is None: + return {} + + config = self.hf_generation_config.to_dict() + + available_params = [ + "repetition_penalty", + "temperature", + "top_k", + "top_p", + "min_p", + ] + + default_sampling_params = { + p: config.get(p) for p in available_params if config.get(p) is not None + } + + return default_sampling_params + + def _maybe_pull_model_tokenizer_from_remote(self) -> None: + """ + Pull the model config files to a temporary + directory in case of remote. + + Args: + model: The model name or path. + + """ + from sglang.srt.connector import create_remote_connector + from sglang.srt.utils import is_remote_url + + if is_remote_url(self.model_path): + logger.info("Pulling model configs from remote...") + # BaseConnector implements __del__() to clean up the local dir. + # Since config files need to exist all the time, so we DO NOT use + # with statement to avoid closing the client. + client = create_remote_connector(self.model_path) + if is_remote_url(self.model_path): + client.pull_files(allow_pattern=["*config.json"]) + self.model_weights = self.model_path + self.model_path = client.get_local_dir() + + +# adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/config.py +_STR_DTYPE_TO_TORCH_DTYPE = { + "half": torch.float16, + "float16": torch.float16, + "float": torch.float32, + "float32": torch.float32, + "bfloat16": torch.bfloat16, +} + + +# adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/config.py +def _get_and_verify_dtype( + config: PretrainedConfig, + dtype: Union[str, torch.dtype], +) -> torch.dtype: + # NOTE: getattr(config, "torch_dtype", torch.float32) is not correct + # because config.torch_dtype can be None. + config_dtype = getattr(config, "dtype", None) + if isinstance(config_dtype, str): + config_dtype = _STR_DTYPE_TO_TORCH_DTYPE.get(config_dtype, None) + if config_dtype is None: + config_dtype = torch.float32 + + if isinstance(dtype, str): + dtype = dtype.lower() + if dtype == "auto": + if config_dtype == torch.float32: + if config.model_type.startswith("gemma"): + if config.model_type == "gemma": + gemma_version = "" + else: + gemma_version = config.model_type[5] + logger.info( + f"For Gemma {gemma_version}, we downcast float32 to bfloat16 instead " + "of float16 by default. Please specify `dtype` if you " + "want to use float16." + ) + torch_dtype = torch.bfloat16 + else: + # Following the common practice, we use float16 for float32 + # models. + torch_dtype = torch.float16 + else: + torch_dtype = config_dtype + else: + if dtype not in _STR_DTYPE_TO_TORCH_DTYPE: + raise ValueError(f"Unknown dtype: {dtype}") + torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype] + elif isinstance(dtype, torch.dtype): + torch_dtype = dtype + else: + raise ValueError(f"Unknown dtype: {dtype}") + + # Verify the dtype. + if torch_dtype != config_dtype: + if torch_dtype == torch.float32: + # Upcasting to float32 is allowed. + logger.info("Upcasting %s to %s.", config_dtype, torch_dtype) + pass + elif config_dtype == torch.float32: + # Downcasting from float32 to float16 or bfloat16 is allowed. + logger.info("Downcasting %s to %s.", config_dtype, torch_dtype) + pass + else: + # Casting between float16 and bfloat16 is allowed with a warning. + logger.warning("Casting %s to %s.", config_dtype, torch_dtype) + + return torch_dtype + + +def is_generation_model(model_architectures: List[str], is_embedding: bool = False): + # We have two ways to determine whether a model is a generative model. + # 1. Check the model architecture + # 2. check the `is_embedding` server args + + if ( + "LlamaEmbeddingModel" in model_architectures + or "MistralModel" in model_architectures + or "LlamaForSequenceClassification" in model_architectures + or "LlamaForSequenceClassificationWithNormal_Weights" in model_architectures + or "InternLM2ForRewardModel" in model_architectures + or "Qwen2ForRewardModel" in model_architectures + or "Qwen2ForSequenceClassification" in model_architectures + or "Qwen3ForSequenceClassification" in model_architectures + or "CLIPModel" in model_architectures + or "BertModel" in model_architectures + or "Contriever" in model_architectures + or "BertForSequenceClassification" in model_architectures + or "XLMRobertaModel" in model_architectures + or "XLMRobertaForSequenceClassification" in model_architectures + ): + return False + else: + return not is_embedding + + +multimodal_model_archs = [ + "CLIPModel", + "DeepseekVL2ForCausalLM", + "Gemma3ForConditionalGeneration", + "Gemma3nForConditionalGeneration", + "Glm4vForConditionalGeneration", + "Glm4vMoeForConditionalGeneration", + "Grok1VForCausalLM", + "Grok1AForCausalLM", + "LlavaLlamaForCausalLM", + "Llama4ForConditionalGeneration", + "LlavaMistralForCausalLM", + "LlavaQwenForCausalLM", + "LlavaForConditionalGeneration", + "LlavaVidForCausalLM", + "MiniCPMO", + "MiniCPMV", + "Mistral3ForConditionalGeneration", + "MultiModalityCausalLM", + "MllamaForConditionalGeneration", + "NemotronH_Nano_VL_V2", + "Qwen2AudioForConditionalGeneration", + "Qwen2VLForConditionalGeneration", + "Qwen2_5_VLForConditionalGeneration", + "Qwen3VLForConditionalGeneration", + "Qwen3VLMoeForConditionalGeneration", + "Qwen3OmniMoeForConditionalGeneration", + "KimiVLForConditionalGeneration", + "InternVLChatModel", + "InternS1ForConditionalGeneration", + "Phi4MMForCausalLM", + "Step3VLForConditionalGeneration", + "POINTSV15ChatModel", + "DotsVLMForCausalLM", + "DotsOCRForCausalLM", + "Sarashina2VisionForCausalLM", + "NVILAForConditionalGeneration", + "NVILALiteForConditionalGeneration", + "DeepseekOCRForCausalLM", + "JetVLMForConditionalGeneration", +] + +if envs.SGLANG_EXTERNAL_MM_MODEL_ARCH.value: + multimodal_model_archs.append(envs.SGLANG_EXTERNAL_MM_MODEL_ARCH.value) + + +def is_multimodal_model(model_architectures: List[str]): + if any( + multi_model_arch in model_architectures + for multi_model_arch in multimodal_model_archs + ): + return True + else: + return False + + +def is_multimodal_gen_model(model_architectures: List[str]): + return False + + +def is_image_gen_model(model_architectures: List[str]): + return False + + +def is_audio_model(model_architectures: List[str]): + return False + + +def is_encoder_decoder_model(model_architectures: List[str]): + return "MllamaForConditionalGeneration" in model_architectures + + +def is_multimodal_chunked_prefill_supported(model_architectures: List[str]): + """Check if chunked prefill is supported for a MultiModal model.""" + unsupported = [ + "Grok1VForCausalLM", + "Grok1AForCausalLM", + "LlavaLlamaForCausalLM", + "MllamaForConditionalGeneration", + "CLIPModel", + ] + if any(multi_model_arch in unsupported for multi_model_arch in model_architectures): + return False + else: + return True + + +def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float: + if scale <= 1: + return 1.0 + return 0.1 * mscale * math.log(scale) + 1.0 + + +def is_hybrid_model( + model_architectures: List[str], + hybrid_kvcache_ratio: Optional[float], + context_length: Optional[int], + attention_chunk_size: Optional[int], +): + if hybrid_kvcache_ratio is None: + return None + elif ( + hybrid_kvcache_ratio > 0 + and model_architectures[0] == "Llama4ForConditionalGeneration" + and context_length > attention_chunk_size + ): + return hybrid_kvcache_ratio + else: + return None + + +def get_hybrid_layer_ids(model_architectures: List[str], num_hidden_layers: int): + if "Llama4ForConditionalGeneration" in model_architectures: + swa_attention_layer_ids = [ + i for i in range(num_hidden_layers) if (i + 1) % 4 != 0 + ] + full_attention_layer_ids = [ + i for i in range(num_hidden_layers) if (i + 1) % 4 == 0 + ] + else: + swa_attention_layer_ids = None + full_attention_layer_ids = None + return swa_attention_layer_ids, full_attention_layer_ids diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/modelopt_config.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/modelopt_config.py new file mode 100644 index 00000000..911b4ce0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/modelopt_config.py @@ -0,0 +1,30 @@ +# Configuration for NVIDIA ModelOpt quantization integration +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class ModelOptConfig: + """Configuration for NVIDIA ModelOpt quantization operations. + + This configuration class holds parameters for ModelOpt quantization, + checkpoint management, and model export operations. + + Args: + quant: Quantization method/type (e.g., "fp8", "fp4") + checkpoint_restore_path: Path to restore ModelOpt checkpoint from + checkpoint_save_path: Path to save ModelOpt checkpoint to + export_path: Path to export quantized model in HuggingFace format + quantize_and_serve: Whether to quantize and serve in one step + """ + + quant: Optional[str] = None + checkpoint_restore_path: Optional[str] = None + checkpoint_save_path: Optional[str] = None + export_path: Optional[str] = None + quantize_and_serve: bool = False + + def __post_init__(self): + """Validate configuration after initialization.""" + # Add any validation logic if needed + pass diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/nano_nemotron_vl.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/nano_nemotron_vl.py new file mode 100644 index 00000000..09ab29ab --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/srt/configs/nano_nemotron_vl.py @@ -0,0 +1,114 @@ +# Copyright 2025 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Adapted from https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16/blob/cb5a65ff10232128389d882d805fa609427544f1/configuration.py + +from typing import Any + +from transformers.configuration_utils import PretrainedConfig + +from sglang.srt.configs.nemotron_h import NemotronHConfig +from sglang.srt.configs.radio import RadioConfig +from sglang.srt.multimodal.internvl_utils import IMAGENET_MEAN, IMAGENET_STD + + +def float_triplet(seq: Any): + a, b, c = tuple(seq) + assert ( + isinstance(a, float) and isinstance(b, float) and isinstance(c, float) + ), "expected three floats" + return a, b, c + + +class NemotronH_Nano_VL_V2_Config(PretrainedConfig): + model_type = "NemotronH_Nano_VL_V2" + is_composition = True + + def __init__( + self, + vision_config=None, + llm_config=None, + force_image_size: int = 512, + patch_size: int = 16, + downsample_ratio=0.5, + template=None, + ps_version="v2", + image_tag_type="internvl", + projector_hidden_size=4096, + vit_hidden_size=1280, + video_pruning_rate: float = 0.0, + video_context_token: str = "
) for a message. + """ + return jinja_env.from_string(_message_template).render( + role=message["role"], + content=message["content"], + variant=message.get("variant", None), + ) + + +jinja_env.globals["message_to_html"] = message_to_html + + +_report_template = """ + + + + + + {% if metrics %} +

Metrics

+ + + + + + + + + + {% for name, value in metrics.items() %} + + + + + {% endfor %} +
MetricValue
Score{{ score | float | round(3) }}
{{ name }}{{ value }}
+ {% endif %} +

Examples

+ {% for html in htmls %} + {{ html | safe }} +
+ {% endfor %} + + +""" + + +def make_report(eval_result: EvalResult) -> str: + """ + Create a standalone HTML report from an EvalResult. + """ + return jinja_env.from_string(_report_template).render( + score=eval_result.score, + metrics=eval_result.metrics, + htmls=eval_result.htmls, + ) + + +def make_report_from_example_htmls(htmls: List[str]): + """ + Create a standalone HTML report from a list of example htmls + """ + return jinja_env.from_string(_report_template).render( + score=None, metrics={}, htmls=htmls + ) + + +def download_dataset(path, url): + print(f"Downloading dataset {path} from {url}") + try: + response = requests.get(url, stream=True) + response.raise_for_status() + + total_size = int(response.headers.get("content-length", 0)) + block_size = 8192 + + with open(path, "wb") as f, tqdm( + desc="Downloading", + total=total_size, + unit="iB", + unit_scale=True, + unit_divisor=1024, + ) as progress_bar: + for data in response.iter_content(block_size): + size = f.write(data) + progress_bar.update(size) + + print(f"Dataset downloaded and saved to {path}") + except requests.RequestException as e: + raise Exception(f"Failed to download dataset: {e}") + + +def set_ulimit(target_soft_limit=65535): + resource_type = resource.RLIMIT_NOFILE + current_soft, current_hard = resource.getrlimit(resource_type) + + if current_soft < target_soft_limit: + try: + resource.setrlimit(resource_type, (target_soft_limit, current_hard)) + except ValueError as e: + print(f"Fail to set RLIMIT_NOFILE: {e}") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_gpqa.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_gpqa.py new file mode 100644 index 00000000..b39366ef --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_gpqa.py @@ -0,0 +1,94 @@ +# Adapted from https://github.com/openai/simple-evals/ + +""" +GPQA: A Graduate-Level Google-Proof Q&A Benchmark +David Rein, Betty Li Hou, Asa Cooper Stickland, Jackson Petty, Richard Yuanzhe Pang, Julien Dirani, Julian Michael, Samuel R. Bowman +https://arxiv.org/abs/2311.12022 +""" + +import random +import re +from typing import Optional + +import pandas + +from sglang.test import simple_eval_common as common +from sglang.test.simple_eval_common import ( + ANSWER_PATTERN_MULTICHOICE, + HTML_JINJA, + Eval, + EvalResult, + SamplerBase, + SingleEvalResult, + format_multichoice_question, +) + + +class GPQAEval(Eval): + def __init__( + self, + filename: str, + num_examples: Optional[int], + num_threads: int, + n_repeats: int = 1, + ): + df = pandas.read_csv(filename) + examples = [row.to_dict() for _, row in df.iterrows()] + rng = random.Random(0) + if num_examples: + assert n_repeats == 1, "n_repeats only supported for num_examples" + examples = rng.sample(examples, num_examples) + examples = examples * n_repeats + examples = [ + example | {"permutation": rng.sample(range(4), 4)} for example in examples + ] + self.examples = examples + self.n_repeats = n_repeats + self.num_threads = num_threads + + def __call__(self, sampler: SamplerBase) -> EvalResult: + def fn(row: dict): + choices = [ + row["Correct Answer"], + row["Incorrect Answer 1"], + row["Incorrect Answer 2"], + row["Incorrect Answer 3"], + ] + choices = [choices[i] for i in row["permutation"]] + correct_index = choices.index(row["Correct Answer"]) + correct_answer = "ABCD"[correct_index] + choices_dict = dict( + A=choices[0], + B=choices[1], + C=choices[2], + D=choices[3], + Question=row["Question"], + ) + prompt_messages = [ + sampler._pack_message( + content=format_multichoice_question(choices_dict), role="user" + ) + ] + response_text = sampler(prompt_messages) + if response_text is None: + response_text = "" + match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text) + extracted_answer = match.group(1) if match else None + score = 1.0 if extracted_answer == correct_answer else 0.0 + html = common.jinja_env.from_string(HTML_JINJA).render( + prompt_messages=prompt_messages, + next_message=dict(content=response_text, role="assistant"), + score=score, + correct_answer=correct_answer, + extracted_answer=extracted_answer, + ) + convo = prompt_messages + [dict(content=response_text, role="assistant")] + return SingleEvalResult( + html=html, + score=score, + convo=convo, + metrics={"chars": len(response_text)}, + ) + + results = common.map_with_progress(fn, self.examples, self.num_threads) + return common.aggregate_results(results) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_humaneval.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_humaneval.py new file mode 100644 index 00000000..aa77b4d9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_humaneval.py @@ -0,0 +1,131 @@ +# Adapted from https://github.com/openai/simple-evals/ + +""" +HumanEval: Evaluating Large Language Models Trained on Code +Mark Chen and Jerry Tworek and Heewoo Jun and Qiming Yuan and Henrique Ponde de Oliveira Pinto and Jared Kaplan and Harri Edwards and Yuri Burda and Nicholas Joseph and Greg Brockman and Alex Ray and Raul Puri and Gretchen Krueger and Michael Petrov and Heidy Khlaaf and Girish Sastry and Pamela Mishkin and Brooke Chan and Scott Gray and Nick Ryder and Mikhail Pavlov and Alethea Power and Lukasz Kaiser and Mohammad Bavarian and Clemens Winter and Philippe Tillet and Felipe Petroski Such and Dave Cummings and Matthias Plappert and Fotios Chantzis and Elizabeth Barnes and Ariel Herbert-Voss and William Hebgen Guss and Alex Nichol and Alex Paino and Nikolas Tezak and Jie Tang and Igor Babuschkin and Suchir Balaji and Shantanu Jain and William Saunders and Christopher Hesse and Andrew N. Carr and Jan Leike and Josh Achiam and Vedant Misra and Evan Morikawa and Alec Radford and Matthew Knight and Miles Brundage and Mira Murati and Katie Mayer and Peter Welinder and Bob McGrew and Dario Amodei and Sam McCandlish and Ilya Sutskever and Wojciech Zaremba +https://arxiv.org/abs/2107.03374 https://github.com/openai/human-eval/ +""" + +import random +import re +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Dict, List, Optional + +try: + from human_eval.data import read_problems + from human_eval.evaluation import estimate_pass_at_k + from human_eval.execution import check_correctness # , unsafe_execute +except (ImportError, ModuleNotFoundError): + print("\nPlease install human-eval at https://github.com/openai/human-eval.\n") + raise + +from sglang.test import simple_eval_common as common +from sglang.test.simple_eval_common import ( + HTML_JINJA, + Eval, + EvalResult, + SamplerBase, + SingleEvalResult, +) + + +def evaluate_functional_correctness( + sample: Dict[str, str], + completions: List[str], + n_workers: int = 4, + timeout: float = 3.0, +): + """ + Evaluates the functional correctness of generated samples, and writes + results to f"{sample_file}_results.jsonl.gz" + """ + + # Check the generated samples against test suites. + with ThreadPoolExecutor(max_workers=n_workers) as executor: + futures = [] + for i, completion in enumerate(completions): + args = (sample, completion, timeout, i) + future = executor.submit(check_correctness, *args) + futures.append(future) + results = [] + for future in as_completed(futures): + result = future.result() + results.append(result) + passed = [int(r["passed"]) for r in results] + return passed + + +class HumanEval(Eval): + def __init__( + self, + num_examples: Optional[int], + num_threads: int, + num_samples_per_task: int = 5, + ks_passes: List[int] = [1, 2, 5], + timeout: int = 120, + ): + self.seed = 0 + self.examples = read_problems() + self.examples = list(self.examples.values()) + + self._num_examples = num_examples + if self._num_examples: + self.examples = random.Random(self.seed).sample(self.examples, num_examples) + self._num_samples_per_task = num_samples_per_task + self._ks_passes = ks_passes + self._timeout = timeout + self._num_threads = num_threads + + def __call__(self, sampler: SamplerBase) -> EvalResult: + instruction = "Read the following function signature and docstring, and fully implement the function described. Your response should only contain the code for this function.\n" + + def find_code(completion): + completion = completion or "" + pattern = re.compile(r"```python\n(.*?)```", re.DOTALL) + matches = pattern.findall(completion) + extracted_answer = matches[0] if len(matches) >= 1 else completion + extracted_answer = extracted_answer[ + extracted_answer.find(":\n ") + 2 : + ] # remove signature + return extracted_answer + + def fn(sample: Dict[str, str]): + prompt_messages = [ + sampler._pack_message( + role="user", content=instruction + sample["prompt"] + ) + ] + completions = [ + find_code(sampler(prompt_messages)) + for _ in range(self._num_samples_per_task) + ] + results = evaluate_functional_correctness(sample, completions) + total = len(results) + correct = sum(results) + score = sum(results) / len(results) + html = common.jinja_env.from_string(HTML_JINJA).render( + prompt_messages=prompt_messages, + next_message=dict(content=completions[0], role="assistant"), + score=score, + correct_answer=[1] * len(results), + extracted_answer=results, + ) + convo = prompt_messages + [ + dict(content=completion, role="assistant") for completion in completions + ] + return SingleEvalResult( + html=html, + score=score, + convo=convo, + metrics={ + f"pass@{k}": estimate_pass_at_k([total], [correct], k) + # this will be aggregated so no need of .mean() + for k in self._ks_passes + if total >= k + }, + ) + + results = common.map_with_progress( + fn, self.examples, num_threads=self._num_threads + ) + return common.aggregate_results(results) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_longbench_v2.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_longbench_v2.py new file mode 100644 index 00000000..645b76e3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_longbench_v2.py @@ -0,0 +1,344 @@ +# Adapted from https://github.com/openai/simple-evals/ + +""" +LongBench v2: Towards Deeper Understanding and Reasoning on Realistic Long-Context Multitasks +Yushi Bai, Shangqing Tu, Jiajie Zhang, Hao Peng, Xiaozhi Wang, Xin Lv, Shulin Cao, Jiazheng Xu, Lei Hou, Yuxiao Dong, Jie Tang, Juanzi Li +https://arxiv.org/abs/2412.15204 +""" + +import csv +import json +import os +import re +from typing import Any, Dict, List, Optional + +from transformers import AutoTokenizer + +from sglang.test import simple_eval_common as common +from sglang.test.simple_eval_common import ( + ANSWER_PATTERN_MULTICHOICE, + HTML_JINJA, + Eval, + EvalResult, + SamplerBase, + SingleEvalResult, +) + +# LongBench-v2 task categories +TASK_CATEGORIES = { + "single_document_qa", + "multi_document_qa", + "long_in_context_learning", + "long_dialogue_history", + "code_repo_understanding", + "long_structured_data", +} + +DEFAULT_DATASET = "THUDM/LongBench-v2" +DEFAULT_DATASET_SPLIT = "train" + + +def format_longbench_v2_question(row: dict) -> str: + """Format a LongBench-v2 question using the official template.""" + context = row.get("context", "") + question = row.get("question", "") + + # Handle both standard format (A, B, C, D) and alternative format (choices list) + if "choices" in row: + choices = row["choices"] + choice_A = choices[0] if len(choices) > 0 else "" + choice_B = choices[1] if len(choices) > 1 else "" + choice_C = choices[2] if len(choices) > 2 else "" + choice_D = choices[3] if len(choices) > 3 else "" + else: + choice_A = row.get("A", row.get("choice_A", "")) + choice_B = row.get("B", row.get("choice_B", "")) + choice_C = row.get("C", row.get("choice_C", "")) + choice_D = row.get("D", row.get("choice_D", "")) + + # Official LongBench-v2 template + prompt = f""" +Please read the following text and answer the question below. + +{context.strip()} + + +What is the correct answer to this question: {question.strip()} +Choices: +(A) {choice_A.strip()} +(B) {choice_B.strip()} +(C) {choice_C.strip()} +(D) {choice_D.strip()} + +Format your response as follows: "The correct answer is (insert answer here)".""" + + return prompt + + +def extract_longbench_v2_answer(response: str) -> Optional[str]: + """Extract answer from model response using official LongBench-v2 method.""" + response = response.replace("*", "") + + # First try: "The correct answer is (A)" + match = re.search(r"The correct answer is \(([A-D])\)", response, re.IGNORECASE) + if match: + return match.group(1).upper() + + # Second try: "The correct answer is A" + match = re.search(r"The correct answer is ([A-D])", response, re.IGNORECASE) + if match: + return match.group(1).upper() + + # Fallback: Standard SGLang multichoice pattern + match = re.search(ANSWER_PATTERN_MULTICHOICE, response) + if match: + return match.group(1).upper() + + # Generic fallback when model says "answer is A" + match = re.search(r"answer\s+is\s*\(?([A-D])\)?", response, re.IGNORECASE) + if match: + return match.group(1).upper() + + return None + + +class LongBenchV2Eval(Eval): + """ + Evaluation utility for LongBench-v2 dataset. + + LongBench-v2 is designed to assess the ability of LLMs to handle long-context problems + requiring deep understanding and reasoning across real-world multitasks. + """ + + def __init__( + self, + model: str = None, + data_source: str = DEFAULT_DATASET, + num_examples: Optional[int] = None, + num_threads: int = 1, + n_repeats: int = 1, + categories: Optional[List[str]] = None, + max_context_length: Optional[int] = None, + min_context_length: Optional[int] = None, + ): + """ + Initialize LongBench-v2 evaluation. + + Args: + data_source: HuggingFace dataset name, local file path (CSV/JSON) + num_examples: Number of examples to evaluate (None for all) + num_threads: Number of threads for parallel processing + n_repeats: Number of times to repeat evaluation for error bars + categories: List of task categories to include (None for all) + max_context_length: Maximum context length in characters + min_context_length: Minimum context length in characters + """ + self.tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True) + self.min_context_length = min_context_length + self.max_context_length = max_context_length + # Load dataset based on data source type + examples = self._load_dataset(data_source) + + # Apply filtering + if categories: + examples = [ex for ex in examples if ex.get("category") in categories] + + # Sample examples if specified + if num_examples: + assert n_repeats == 1, "n_repeats only supported when not sampling examples" + examples = examples[: min(num_examples, len(examples))] + + # Repeat examples for multiple runs + examples = examples * n_repeats + + if not examples: + raise ValueError( + "No examples available for LongBench-v2 evaluation after filtering" + ) + + self.examples = examples + self.n_repeats = n_repeats + self.num_threads = num_threads + + print(f"Loaded {len(self.examples)} examples from LongBench-v2") + if categories: + print(f"Filtered to categories: {categories}") + if min_context_length or max_context_length: + print( + f"Context length filter: {min_context_length}-{max_context_length} characters" + ) + + def _load_dataset(self, data_source: str) -> List[Dict[str, Any]]: + """Load dataset from HuggingFace hub or local files.""" + + if not data_source: + data_source = DEFAULT_DATASET + + if os.path.exists(data_source): + raw_examples = self._load_local_file(data_source) + else: + raw_examples = self._load_hf_dataset(data_source) + + return [self._normalize_example(example) for example in raw_examples] + + def _load_local_file(self, path: str) -> List[Dict[str, Any]]: + """Load examples from a local CSV/JSON/JSONL file.""" + + suffix = os.path.splitext(path)[1].lower() + if suffix in {".json", ".jsonl"}: + with open(path, "r", encoding="utf-8") as fh: + if suffix == ".jsonl": + data = [json.loads(line) for line in fh if line.strip()] + else: + data = json.load(fh) + elif suffix == ".csv": + with open(path, "r", encoding="utf-8") as fh: + reader = csv.DictReader(fh) + data = list(reader) + else: + # Try JSON, then CSV as fallback + try: + with open(path, "r", encoding="utf-8") as fh: + data = json.load(fh) + except json.JSONDecodeError: + with open(path, "r", encoding="utf-8") as fh: + reader = csv.DictReader(fh) + data = list(reader) + + if isinstance(data, dict): + data = data.get("data", []) + + if not isinstance(data, list): + raise ValueError("Expected list of examples from local file") + + return data + + def _load_hf_dataset(self, identifier: str) -> List[Dict[str, Any]]: + """Load the dataset from HuggingFace Hub.""" + + parts = identifier.split(":", maxsplit=1) + dataset_name = parts[0] + split = parts[1] if len(parts) == 2 else DEFAULT_DATASET_SPLIT + + try: + from datasets import load_dataset # type: ignore + except ImportError as exc: + raise ImportError( + "Please install the 'datasets' package to load LongBench-v2 from HuggingFace: pip install datasets" + ) from exc + + dataset = load_dataset(dataset_name, split=split) + return [dict(row) for row in dataset] + + def _normalize_example(self, example: Dict[str, Any]) -> Dict[str, Any]: + """Ensure each example exposes the expected keys.""" + + normalized = dict(example) + + for letter in ["A", "B", "C", "D"]: + choice_key = f"choice_{letter}" + if letter not in normalized and choice_key in normalized: + normalized[letter] = normalized[choice_key] + + if "category" not in normalized and "domain" in normalized: + normalized["category"] = normalized["domain"] + + answer = normalized.get("answer") + if isinstance(answer, str): + normalized["answer"] = answer.strip().upper() + elif isinstance(answer, int) and 0 <= answer < 4: + normalized["answer"] = ["A", "B", "C", "D"][answer] + + return normalized + + def _check_context_length( + self, + formatted_question: str, + tokenizer: AutoTokenizer, + min_length: Optional[int], + max_length: Optional[int], + ) -> bool: + """Filter examples by context length measured in characters.""" + input_ids = tokenizer.encode(formatted_question) + context_length = len(input_ids) + + if min_length is not None and context_length < min_length: + return False + if max_length is not None and context_length > max_length: + return False + + return True + + def __call__(self, sampler: SamplerBase) -> EvalResult: + """Run the evaluation.""" + + def fn(row: dict): + # Format the question using official template + formatted_question = format_longbench_v2_question(row) + + if self.min_context_length or self.max_context_length: + if not self._check_context_length( + formatted_question, + self.tokenizer, + self.min_context_length, + self.max_context_length, + ): + # Skip this example + return None + + prompt_messages = [ + sampler._pack_message(content=formatted_question, role="user") + ] + + # Get model response + response_text = sampler(prompt_messages) + if response_text is None: + response_text = "" + + # Extract answer using official method + extracted_answer = extract_longbench_v2_answer(response_text) + + # Get correct answer + correct_answer = row.get("answer", "") + if isinstance(correct_answer, str): + correct_answer = correct_answer.strip().upper() + elif isinstance(correct_answer, int) and 0 <= correct_answer < 4: + correct_answer = ["A", "B", "C", "D"][correct_answer] + + # Calculate score + score = 1.0 if extracted_answer == correct_answer else 0.0 + + # Generate HTML report + html = common.jinja_env.from_string(HTML_JINJA).render( + prompt_messages=prompt_messages, + next_message=dict(content=response_text, role="assistant"), + score=score, + correct_answer=correct_answer, + extracted_answer=extracted_answer, + ) + + # Build conversation + convo = prompt_messages + [dict(content=response_text, role="assistant")] + + # Prepare metrics + metrics = {"chars": len(response_text)} + + # Add category-specific metrics + category = row.get("category", row.get("domain", "unknown")) + if category in TASK_CATEGORIES: + metrics[category] = score + + difficulty = row.get("difficulty") + if isinstance(difficulty, str) and difficulty: + metrics[f"difficulty_{difficulty.lower()}"] = score + + return SingleEvalResult( + html=html, + score=score, + convo=convo, + metrics=metrics, + ) + + # Run evaluation with progress tracking + results = common.map_with_progress(fn, self.examples, self.num_threads) + return common.aggregate_results(results) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_math.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_math.py new file mode 100644 index 00000000..37d4b120 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_math.py @@ -0,0 +1,74 @@ +# Adapted from https://github.com/openai/simple-evals/ + +""" +Measuring Mathematical Problem Solving With the MATH Dataset +Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt +https://arxiv.org/abs/2103.03874 +""" + +import random +import re +from typing import Optional + +import pandas + +from sglang.test import simple_eval_common as common +from sglang.test.simple_eval_common import ( + ANSWER_PATTERN, + HTML_JINJA, + Eval, + EvalResult, + SamplerBase, + SingleEvalResult, + check_equality, +) + +QUERY_TEMPLATE = """ +Solve the following math problem step by step. The last line of your response should be of the form Answer: $ANSWER (without quotes) where $ANSWER is the answer to the problem. + +{Question} + +Remember to put your answer on its own line after "Answer:", and you do not need to use a \\boxed command. +""".strip() + + +class MathEval(Eval): + def __init__( + self, + filename: str, + equality_checker: SamplerBase, + num_examples: Optional[int], + num_threads: int, + ): + df = pandas.read_csv(filename) + examples = [row.to_dict() for _, row in df.iterrows()] + if num_examples: + examples = random.Random(0).sample(examples, num_examples) + self.examples = examples + self.equality_checker = equality_checker + self.num_threads = num_threads + + def __call__(self, sampler: SamplerBase) -> EvalResult: + def fn(row: dict): + prompt_messages = [ + sampler._pack_message(content=QUERY_TEMPLATE.format(**row), role="user") + ] + response_text = sampler(prompt_messages) + response_text = response_text or "" + match = re.search(ANSWER_PATTERN, response_text) + extracted_answer = match.group(1) if match else None + score = float( + check_equality(self.equality_checker, row["Answer"], extracted_answer) + ) + html = common.jinja_env.from_string(HTML_JINJA).render( + prompt_messages=prompt_messages, + next_message=dict(content=response_text, role="assistant"), + score=score, + correct_answer=row["Answer"], + extracted_answer=extracted_answer, + ) + convo = prompt_messages + [dict(content=response_text, role="assistant")] + return SingleEvalResult(html=html, score=score, convo=convo) + + results = common.map_with_progress(fn, self.examples, self.num_threads) + return common.aggregate_results(results) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_mgsm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_mgsm.py new file mode 100644 index 00000000..0b0b72a2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_mgsm.py @@ -0,0 +1,203 @@ +# Adapted from https://github.com/openai/simple-evals/ + +""" +MGSM: Multilingual Grade School Math Benchmark (MGSM) is a benchmark of grade-school math problems. +Language Models are Multilingual Chain-of-Thought Reasoners +Freda Shi, Mirac Suzgun, Markus Freitag, Xuezhi Wang, Suraj Srivats, Soroush Vosoughi, Hyung Won Chung, Yi Tay, Sebastian Ruder, Denny Zhou, Dipanjan Das, Jason Wei +https://arxiv.org/abs/2210.03057 reference: https://github.com/google-research/url-nlp +""" + +import re +import urllib +from typing import Optional + +from sglang.test import simple_eval_common as common +from sglang.test.simple_eval_common import ( + HTML_JINJA, + Eval, + EvalResult, + SamplerBase, + SingleEvalResult, +) + +ALL_LANGUAGES = ["bn", "de", "en", "es", "fr", "ja", "ru", "sw", "te", "th", "zh"] +LATIN_LANGUAGES = ["de", "en", "es", "fr", "sw"] +NON_LATIN_LANGUAGES = ["bn", "ja", "ru", "te", "th", "zh"] + +LANG_TO_FPATH = { + "bn": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_bn.tsv", + "de": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_de.tsv", + "en": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_en.tsv", + "es": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_es.tsv", + "fr": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_fr.tsv", + "ja": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_ja.tsv", + "ru": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_ru.tsv", + "sw": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_sw.tsv", + "te": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_te.tsv", + "th": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_th.tsv", + "zh": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_zh.tsv", +} +LANG_TO_INSTRUCTIONS = { + "en": """Solve this math problem. Give the reasoning steps before giving the final answer on the last line by itself in the format of "Answer:". Do not add anything other than the integer answer after "Answer:". + +{input}""", + "bn": """এই গণিতের সমস্যাটি সমাধান করুন। চূড়ান্ত উত্তর দেওয়ার আগে যুক্তিসম্পন্ন পদক্ষেপ প্রদান করুন। চূড়ান্ত উত্তরটি একক সংখ্যা হিসাবে "উত্তর:" এর পরে শেষ লাইনে দিন। "উত্তর:" এর পরে অন্য কিছু যুক্ত করবেন না।. + +{input}""", + "de": """Löse dieses Mathematikproblem. Gib die Schritte zur Begründung an, bevor du die endgültige Antwort in der letzten Zeile alleine im Format "Antwort:" gibst. Füge nichts anderes als die ganzzahlige Antwort nach "Antwort:" hinzu. + +{input}""", + "es": """Resuelve este problema matemático. Proporciona los pasos de razonamiento antes de dar la respuesta final en la última línea por sí misma en el formato de "Respuesta:". No añadas nada más que la respuesta entera después de "Respuesta:". + +{input}""", + "fr": """Résolvez ce problème de mathématiques. Donnez les étapes de raisonnement avant de fournir la réponse finale sur la dernière ligne elle-même dans le format de "Réponse:". N'ajoutez rien d'autre que la réponse entière après "Réponse:". + +{input}""", + "ja": """の数学の問題を解いてください。最終的な答えを出す前に、解答の推論過程を記述してください。そして最後の行には "答え:" の形式で答えを記述し、その後には整数の答え以外何も追加しないでください。 + +{input}""", + "ru": """Решите эту математическую задачу. Объясните шаги рассуждения перед тем, как дать окончательный ответ в последней строке сам по себе в формате "Ответ:". Не добавляйте ничего, кроме целочисленного ответа после "Ответ:". + +{input}""", + "sw": """Suluhisha tatizo hili la hesabu. Toa hatua za mantiki kabla ya kutoa jibu la mwisho kwenye mstari wa mwisho peke yake katika muundo wa "Jibu:". Usiongeze chochote kingine isipokuwa jibu la integer baada ya "Jibu:". + +{input}""", + "te": """ఈ గణిత సమస్యను పరిష్కరించండి. చివరి సమాధానాన్ని ఇవ్వదానికి ముందు తర్కాత్మక అదుగులను ఇవ్వండి. చివరి పంక్తిలో మాత్రమే 'సమాధానం:' అనే ఆకారంలో చివరి సమాధానాద్ని ఇవ్వండి సమాధానం: తర్వాత పూర్ణాంక సమాధానానికి తప్పించి ఎదేనా చేర్చవద్దు. + +{input}""", + "th": """แก้ปัญหาคณิตศาสตร์นี้ ให้ให้ขั้นตอนการใช้เหตุผลก่อนที่จะให้คำตอบสุดท้ายในบรรทัดสุดท้ายโดยอยู่ในรูปแบบ "คำตอบ:" ไม่ควรเพิ่มอะไรนอกจากคำตอบที่เป็นจำนวนเต็มหลังจาก "คำตอบ:" + +{input}""", + "zh": """解决这个数学问题。在最后一行给出答案前，请提供推理步骤。最后一行应该以 "答案: " 的形式独立给出答案。在 "答案：" 后不要添加除整数答案之外的任何内容。 + +{input}""", +} + +LANG_TO_ANSWER_PREFIX = { + "en": "Answer", + "bn": "উত্তর", + "de": "Antwort", + "es": "Respuesta", + "fr": "Réponse", + "ja": "答え", + "ru": "Ответ", + "sw": "Jibu", + "te": "సమాధానం", + "th": "คำตอบ", + "zh": "答案", +} + + +def parse_answer(answer: str, answer_prefix: str) -> str: + if answer_prefix not in answer: + return "" + + answer_text = answer.split(answer_prefix)[-1].strip() + + # find all the numbers (including decimals) in the string + numbers = re.findall(r"\d+\.?\d*", answer_text.replace(",", "")) + + # return the first number (removing trailing decimal point if present), + # or an empty string if there were no numbers + return numbers[-1].rstrip(".") if numbers else "" + + +def score_mgsm(target: str, prediction: str) -> bool: + if "." in prediction: + prediction = prediction.rstrip("0").rstrip(".") + + target = target.replace(",", "") + prediction = prediction.replace(",", "") + + return target == prediction + + +def get_lang_examples(lang: str) -> list[dict[str, str]]: + fpath = LANG_TO_FPATH[lang] + examples = [] + with urllib.request.urlopen(fpath) as f: + for line in f.read().decode("utf-8").splitlines(): + inputs, targets = line.strip().split("\t") + if "." in targets: + raise ValueError(f"targets {targets} contains a decimal point.") + # targets = int(targets.replace(",", "")) + examples.append({"inputs": inputs, "targets": targets, "lang": lang}) + return examples + + +def get_all_examples() -> list[dict[str, str]]: + examples = [] + for lang in ALL_LANGUAGES: + if lang != "en": + continue + examples += get_lang_examples(lang) + return examples + + +class MGSMEval(Eval): + def __init__( + self, + num_examples_per_lang: int = 250, # restrict to a subset of the data for debugging + num_threads: int = 64, + languages: Optional[list[str]] = ALL_LANGUAGES, + ): + if languages is None: + languages = ALL_LANGUAGES + else: + for language in languages: + if language not in ALL_LANGUAGES: + raise ValueError( + f"language {language} is not a valid language. " + f"It should be one in {ALL_LANGUAGES}" + ) + self._languages = languages + self._num_examples_per_lang = num_examples_per_lang + self._num_threads = num_threads + + examples = [] + for lang in self._languages: + lang_examples = get_lang_examples(lang) + examples.extend(lang_examples[: self._num_examples_per_lang]) + self.examples = examples + + def __call__(self, sampler: SamplerBase) -> EvalResult: + def fn(example: dict[str, str]): + language = example["lang"] + latin_language = ( + "group_latin" if language in LATIN_LANGUAGES else "group_non_latin" + ) + correct_answer = example["targets"] + instructoin = LANG_TO_INSTRUCTIONS[language] + prompt_messages = [ + sampler._pack_message( + content=instructoin.format(input=example["inputs"]), role="user" + ) + ] + try: + response_text = sampler(prompt_messages) + except Exception as e: + response_text = "" + + answer_prefix = LANG_TO_ANSWER_PREFIX[language] + extracted_answer = parse_answer(response_text, answer_prefix) + + score = score_mgsm(correct_answer, extracted_answer) + html = common.jinja_env.from_string(HTML_JINJA).render( + prompt_messages=prompt_messages, + next_message=dict(content=response_text, role="assistant"), + score=score, + correct_answer=correct_answer, + extracted_answer=extracted_answer, + ) + convo = prompt_messages + [dict(content=response_text, role="assistant")] + return SingleEvalResult( + html=html, + score=score, + convo=convo, + metrics={language: score, latin_language: score}, + ) + + results = common.map_with_progress( + fn, self.examples, num_threads=self._num_threads + ) + return common.aggregate_results(results, default_stats=("mean", "std")) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_mmlu.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_mmlu.py new file mode 100644 index 00000000..a68dbb93 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_mmlu.py @@ -0,0 +1,122 @@ +# Adapted from https://github.com/openai/simple-evals/ + +""" +Measuring Massive Multitask Language Understanding +Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt +https://arxiv.org/abs/2009.03300 +""" + +import random +import re +from typing import Optional + +import pandas + +from sglang.test import simple_eval_common as common +from sglang.test.simple_eval_common import ( + ANSWER_PATTERN_MULTICHOICE, + HTML_JINJA, + Eval, + EvalResult, + SamplerBase, + SingleEvalResult, + format_multichoice_question, +) + +subject2category = { + "abstract_algebra": "stem", + "anatomy": "other", + "astronomy": "stem", + "business_ethics": "other", + "clinical_knowledge": "other", + "college_biology": "stem", + "college_chemistry": "stem", + "college_computer_science": "stem", + "college_mathematics": "stem", + "college_medicine": "other", + "college_physics": "stem", + "computer_security": "stem", + "conceptual_physics": "stem", + "econometrics": "social_sciences", + "electrical_engineering": "stem", + "elementary_mathematics": "stem", + "formal_logic": "humanities", + "global_facts": "other", + "high_school_biology": "stem", + "high_school_chemistry": "stem", + "high_school_computer_science": "stem", + "high_school_european_history": "humanities", + "high_school_geography": "social_sciences", + "high_school_government_and_politics": "social_sciences", + "high_school_macroeconomics": "social_sciences", + "high_school_mathematics": "stem", + "high_school_microeconomics": "social_sciences", + "high_school_physics": "stem", + "high_school_psychology": "social_sciences", + "high_school_statistics": "stem", + "high_school_us_history": "humanities", + "high_school_world_history": "humanities", + "human_aging": "other", + "human_sexuality": "social_sciences", + "international_law": "humanities", + "jurisprudence": "humanities", + "logical_fallacies": "humanities", + "machine_learning": "stem", + "management": "other", + "marketing": "other", + "medical_genetics": "other", + "miscellaneous": "other", + "moral_disputes": "humanities", + "moral_scenarios": "humanities", + "nutrition": "other", + "philosophy": "humanities", + "prehistory": "humanities", + "professional_accounting": "other", + "professional_law": "humanities", + "professional_medicine": "other", + "professional_psychology": "social_sciences", + "public_relations": "social_sciences", + "security_studies": "social_sciences", + "sociology": "social_sciences", + "us_foreign_policy": "social_sciences", + "virology": "other", + "world_religions": "humanities", +} + + +class MMLUEval(Eval): + def __init__(self, filename: str, num_examples: Optional[int], num_threads: int): + df = pandas.read_csv(filename) + examples = [row.to_dict() for _, row in df.iterrows()] + if num_examples: + examples = random.Random(0).sample(examples, num_examples) + self.examples = examples + self.num_threads = num_threads + + def __call__(self, sampler: SamplerBase) -> EvalResult: + def fn(row: dict): + prompt_messages = [ + sampler._pack_message( + content=format_multichoice_question(row), role="user" + ) + ] + response_text = sampler(prompt_messages) + response_text = response_text or "" + match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text) + extracted_answer = match.group(1) if match else None + score = 1.0 if extracted_answer == row["Answer"] else 0.0 + html = common.jinja_env.from_string(HTML_JINJA).render( + prompt_messages=prompt_messages, + next_message=dict(content=response_text, role="assistant"), + score=score, + correct_answer=row["Answer"], + extracted_answer=extracted_answer, + ) + convo = prompt_messages + [dict(content=response_text, role="assistant")] + category = subject2category.get(row["Subject"], "other") + return SingleEvalResult( + html=html, score=score, metrics={category: score}, convo=convo + ) + + results = common.map_with_progress(fn, self.examples, self.num_threads) + return common.aggregate_results(results) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_mmmu_vlm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_mmmu_vlm.py new file mode 100644 index 00000000..f13cfd68 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/simple_eval_mmmu_vlm.py @@ -0,0 +1,442 @@ +""" +MMMU evaluation for VLMs using the run_eval simple-evals interface. + +""" + +from __future__ import annotations + +import base64 +import io +from typing import List, Optional, Tuple + +from datasets import concatenate_datasets, load_dataset +from PIL import Image + +from sglang.test import simple_eval_common as common +from sglang.test.simple_eval_common import ( + HTML_JINJA, + Eval, + EvalResult, + SamplerBase, + SingleEvalResult, + map_with_progress, +) + + +class MMMUVLMEval(Eval): + DOMAIN_CAT2SUB_CAT = { + "Art and Design": ["Art", "Art_Theory", "Design", "Music"], + "Business": ["Accounting", "Economics", "Finance", "Manage", "Marketing"], + "Science": ["Biology", "Chemistry", "Geography", "Math", "Physics"], + "Health and Medicine": [ + "Basic_Medical_Science", + "Clinical_Medicine", + "Diagnostics_and_Laboratory_Medicine", + "Pharmacy", + "Public_Health", + ], + "Humanities and Social Science": [ + "History", + "Literature", + "Sociology", + "Psychology", + ], + "Tech and Engineering": [ + "Agriculture", + "Architecture_and_Engineering", + "Computer_Science", + "Electronics", + "Energy_and_Power", + "Materials", + "Mechanical_Engineering", + ], + } + + def __init__( + self, num_examples: Optional[int] = 100, num_threads: int = 32, seed: int = 42 + ): + """Create MMMU VLM eval (Math subset, 100 fixed samples by default).""" + self.num_examples = num_examples + self.num_threads = num_threads + self.seed = seed + # Prepare samples deterministically across all MMMU subjects (validation split) + self.samples = self._prepare_mmmu_samples(self.num_examples) + + @staticmethod + def _to_data_uri(image: Image.Image) -> str: + if image.mode == "RGBA": + image = image.convert("RGB") + buf = io.BytesIO() + image.save(buf, format="PNG") + b64 = base64.b64encode(buf.getvalue()).decode("utf-8") + return f"data:image/png;base64,{b64}" + + @staticmethod + def _build_mc_mapping(options: List[str]) -> Tuple[dict, List[str]]: + index2ans = {} + all_choices = [] + ch = ord("A") + for opt in options: + letter = chr(ch) + index2ans[letter] = opt + all_choices.append(letter) + ch += 1 + return index2ans, all_choices + + def _prepare_mmmu_samples(self, k: int) -> List[dict]: + # Subjects and domains copied from MMMU data_utils to categorize results + subjects: List[str] = [] + for subs in self.DOMAIN_CAT2SUB_CAT.values(): + subjects.extend(subs) + + # Load validation split of each subject + datasets = [] + for subj in subjects: + try: + d = load_dataset("MMMU/MMMU", subj, split="validation") + # attach subject info via transform + d = d.add_column("__subject__", [subj] * len(d)) + datasets.append(d) + except Exception: + continue + if not datasets: + raise RuntimeError("Failed to load MMMU datasets") + + merged = concatenate_datasets(datasets) + + # Deterministic selection: sort by id (fallback to subject+index) + def _key(idx): + ex = merged[idx] + return str(ex.get("id", f"{ex['__subject__']}:{idx}")) + + order = sorted(range(len(merged)), key=_key) + picked_indices = order[:k] + + samples: List[dict] = [] + for idx in picked_indices: + ex = merged[idx] + subject = ex["__subject__"] + image = ex.get("image_1") + if image is None or not hasattr(image, "convert"): + continue + data_uri = self._to_data_uri(image) + question = ex.get("question", "") + answer = ex.get("answer") + raw_options = ex.get("options") + question_type = "open" + index2ans = None + all_choices = None + options = None + if raw_options: + try: + options = ( + raw_options + if isinstance(raw_options, list) + else list(eval(raw_options)) + ) + if isinstance(options, list) and len(options) > 0: + index2ans, all_choices = self._build_mc_mapping(options) + question_type = "multiple-choice" + except Exception: + options = None + + # Build final textual prompt; include choices if MC + prompt_text = f"Question: {question}\n\n" + if options: + letters = [chr(ord("A") + i) for i in range(len(options))] + for letter, opt in zip(letters, options): + prompt_text += f"{letter}) {opt}\n" + prompt_text += "\nAnswer: " + + samples.append( + { + "id": ex.get("id", f"{subject}:{idx}"), + "final_input_prompt": prompt_text, + "image_data": data_uri, + "answer": answer, + "question_type": question_type, + "index2ans": index2ans, + "all_choices": all_choices, + "category": subject, + } + ) + + return samples + + @staticmethod + def _split_prompt_for_image(prompt: str) -> tuple[str, str]: + """Split a prompt containing an inline image tag into prefix and suffix. + + If no tag is present, treat the whole prompt as prefix and empty suffix. + """ + if "<" in prompt and ">" in prompt: + prefix = prompt.split("<")[0] + suffix = prompt.split(">", 1)[1] + return prefix, suffix + return prompt, "" + + @staticmethod + def build_chat_messages_from_prompt(prompt: str, image_data) -> List: + """Split a prompt containing an inline image tag into prefix and suffix. + + If no tag is present, treat the whole prompt as prefix and empty suffix. + """ + # Build a vision+text message for OpenAI-compatible API + prefix, suffix = MMMUVLMEval._split_prompt_for_image(prompt) + + content: List[dict] = [] + if prefix: + content.append({"type": "text", "text": prefix}) + content.append({"type": "image_url", "image_url": {"url": image_data}}) + if suffix: + content.append({"type": "text", "text": suffix}) + prompt_messages = [{"role": "user", "content": content}] + + return prompt_messages + + def __call__(self, sampler: SamplerBase) -> EvalResult: + def fn(sample: dict): + prompt = sample["final_input_prompt"] + image_data = sample["image_data"] + prompt_messages = MMMUVLMEval.build_chat_messages_from_prompt( + prompt, image_data + ) + + # Sample + response_text = sampler(prompt_messages) + response_text = response_text or "" + + # Parse and score + gold = sample["answer"] + if ( + sample["question_type"] == "multiple-choice" + and sample["all_choices"] + and sample["index2ans"] + ): + pred = _parse_multi_choice_response( + response_text, sample["all_choices"], sample["index2ans"] + ) + score = 1.0 if (gold is not None and pred == gold) else 0.0 + extracted_answer = pred + else: + parsed_list = _parse_open_response(response_text) + score = ( + 1.0 if (gold is not None and _eval_open(gold, parsed_list)) else 0.0 + ) + extracted_answer = ", ".join(map(str, parsed_list)) + + html_rendered = common.jinja_env.from_string(HTML_JINJA).render( + prompt_messages=prompt_messages, + next_message=dict(content=response_text, role="assistant"), + score=score, + correct_answer=gold, + extracted_answer=extracted_answer, + ) + + convo = prompt_messages + [dict(content=response_text, role="assistant")] + return SingleEvalResult( + html=html_rendered, + score=score, + metrics={"__category__": sample["category"]}, + convo=convo, + ) + + results = map_with_progress(fn, self.samples, self.num_threads) + + # Build category table and overall accuracy + # Gather per-sample correctness and category + per_cat_total: dict[str, int] = {} + per_cat_correct: dict[str, int] = {} + htmls = [] + convos = [] + scores: List[float] = [] + for r in results: + # __category__ stored under metrics + cat = r.metrics.get("__category__") if r.metrics else None + if cat is None: + cat = "Unknown" + per_cat_total[cat] = per_cat_total.get(cat, 0) + 1 + if r.score: + per_cat_correct[cat] = per_cat_correct.get(cat, 0) + 1 + htmls.append(r.html) + convos.append(r.convo) + if r.score is not None: + scores.append(r.score) + + evaluation_result = {} + for cat, tot in per_cat_total.items(): + corr = per_cat_correct.get(cat, 0) + acc = (corr / tot) if tot > 0 else 0.0 + evaluation_result[cat] = {"acc": round(acc, 3), "num_example": tot} + + printable_results = {} + # Domains first + for domain, cats in self.DOMAIN_CAT2SUB_CAT.items(): + acc_sum = 0.0 + num_sum = 0 + for cat in cats: + if cat in evaluation_result: + acc_sum += ( + evaluation_result[cat]["acc"] + * evaluation_result[cat]["num_example"] + ) + num_sum += evaluation_result[cat]["num_example"] + if num_sum > 0: + printable_results[f"Overall-{domain}"] = { + "num": num_sum, + "acc": round(acc_sum / num_sum, 3), + } + # add each sub-category row if present + for cat in cats: + if cat in evaluation_result: + printable_results[cat] = { + "num": evaluation_result[cat]["num_example"], + "acc": evaluation_result[cat]["acc"], + } + + # Overall + total_num = sum(v["num_example"] for v in evaluation_result.values()) + overall_acc = ( + sum(v["acc"] * v["num_example"] for v in evaluation_result.values()) + / total_num + if total_num > 0 + else 0.0 + ) + printable_results["Overall"] = {"num": total_num, "acc": round(overall_acc, 3)} + + # Build EvalResult + return EvalResult( + score=overall_acc, metrics=printable_results, htmls=htmls, convos=convos + ) + + +def _parse_multi_choice_response( + response: str, all_choices: List[str], index2ans: dict +) -> str: + # loosely adapted from benchmark mmmu eval + for char in [",", ".", "!", "?", ";", ":", "'"]: + response = response.strip(char) + response = " " + response + " " + + # Prefer explicit letter with bracket e.g. (A) + candidates: List[str] = [] + for choice in all_choices: + if f"({choice})" in response: + candidates.append(choice) + if not candidates: + for choice in all_choices: + if f" {choice} " in response: + candidates.append(choice) + if not candidates and len(response.split()) > 5: + # try match by option text + for idx, ans in index2ans.items(): + if ans and ans.lower() in response.lower(): + candidates.append(idx) + if not candidates: + # fallback to first choice + return all_choices[0] + if len(candidates) == 1: + return candidates[0] + # choose the last occurrence + starts = [] + for can in candidates: + pos = response.rfind(f"({can})") + if pos == -1: + pos = response.rfind(f" {can} ") + if pos == -1 and index2ans.get(can): + pos = response.lower().rfind(index2ans[can].lower()) + starts.append(pos) + return candidates[int(max(range(len(starts)), key=lambda i: starts[i]))] + + +def _check_is_number(s: str) -> bool: + try: + float(s.replace(",", "")) + return True + except Exception: + return False + + +def _normalize_str(s: str): + s = s.strip() + if _check_is_number(s): + s = s.replace(",", "") + try: + v = round(float(s), 2) + return [v] + except Exception: + return [s.lower()] + return [s.lower()] if len(s) > 1 else [" " + s, s + " "] + + +def _extract_numbers(s: str) -> List[str]: + import re as _re + + pattern_commas = r"-?\b\d{1,3}(?:,\d{3})+\b" + pattern_scientific = r"-?\d+(?:\.\d+)?[eE][+-]?\d+" + pattern_simple = r"-?(?:\d+\.\d+|\.\d+|\d+\b)(?![eE][+-]?\d+)(?![,\d])" + return ( + _re.findall(pattern_commas, s) + + _re.findall(pattern_scientific, s) + + _re.findall(pattern_simple, s) + ) + + +def _parse_open_response(response: str) -> List[str]: + import re as _re + + def get_key_subresponses(resp: str) -> List[str]: + resp = resp.strip().strip(".").lower() + subs = _re.split(r"\.\s(?=[A-Z])|\n", resp) + indicators = [ + "could be ", + "so ", + "is ", + "thus ", + "therefore ", + "final ", + "answer ", + "result ", + ] + keys = [] + for i, s in enumerate(subs): + cands = [*indicators] + if i == len(subs) - 1: + cands.append("=") + shortest = None + for ind in cands: + if ind in s: + part = s.split(ind)[-1].strip() + if not shortest or len(part) < len(shortest): + shortest = part + if shortest and shortest not in [":", ",", ".", "!", "?", ";", ":", "'"]: + keys.append(shortest) + return keys or [resp] + + key_resps = get_key_subresponses(response) + pred_list = key_resps.copy() + for r in key_resps: + pred_list.extend(_extract_numbers(r)) + out = [] + for x in pred_list: + out.extend(_normalize_str(x)) + # dedup + return list(dict.fromkeys(out)) + + +def _eval_open(gold, preds: List[str]) -> bool: + if isinstance(gold, list): + norm_answers = [] + for ans in gold: + norm_answers.extend(_normalize_str(ans)) + else: + norm_answers = _normalize_str(gold) + for p in preds: + if isinstance(p, str): + for na in norm_answers: + if isinstance(na, str) and na in p: + return True + else: + if p in norm_answers: + return True + return False diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/speculative/test_spec_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/speculative/test_spec_utils.py new file mode 100644 index 00000000..47085afd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/speculative/test_spec_utils.py @@ -0,0 +1,348 @@ +import unittest + +import numpy as np +import torch + +from sglang.srt.mem_cache.memory_pool import copy_all_layer_kv_cache_tiled +from sglang.srt.speculative.spec_utils import assign_draft_cache_locs +from sglang.srt.utils import next_power_of_2 + +BYTES_PER_TILE = 128 + + +class TestSpecUtils(unittest.TestCase): + + def setUp(self): + self.device = "cuda" if torch.cuda.is_available() else "cpu" + self.data_ptrs = torch.zeros(2, 1, dtype=torch.uint64, device=self.device) + self.k_cache = [ + torch.zeros((100, 1, 1), dtype=torch.float32, device=self.device) + ] + self.v_cache = [ + torch.zeros((100, 1, 1), dtype=torch.float32, device=self.device) + ] + self.k_cache[0][:11, 0, 0] = torch.tensor( + [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], + dtype=torch.float32, + device=self.device, + ) + self.v_cache[0][:11, 0, 0] = torch.tensor( + [-0.0, -0.1, -0.2, -0.3, -0.4, -0.5, -0.6, -0.7, -0.8, -0.9, -1.0], + dtype=torch.float32, + device=self.device, + ) + self.data_ptrs[0, 0] = self.k_cache[0].data_ptr() + self.data_ptrs[1, 0] = self.v_cache[0].data_ptr() + + self.data_strides = torch.tensor( + [ + np.prod(x.shape[1:]) * x.dtype.itemsize + for x in self.k_cache + self.v_cache + ], + device=self.device, + dtype=torch.int64, + ) + + def test_assign_draft_cache_locs_single_seq(self): + # Testing Setup: req_to_token starting from 4 + # 4,5,6,7,{8,9,10}, 8,9,10 is the last partial page, 3 tokens < page_size=4 + # next kv cache will be stored starting 11,12,13... + device = self.device + num_seqs = 1 + page_size = 4 + speculative_num_steps = 5 + topk = 8 + seq_lens_num = 7 + extend_lens_num = 61 # includes the duplicated last page + req_pool_indices = torch.arange(num_seqs, dtype=torch.int32, device=device) + req_to_token = torch.zeros((num_seqs, 100), dtype=torch.int32, device=device) + req_to_token[0, :seq_lens_num] = torch.tensor( + [4, 5, 6, 7, 8, 9, 10], device=device + ) + seq_lens = torch.tensor([seq_lens_num], dtype=torch.int32, device=device) + extend_lens = torch.tensor([extend_lens_num], dtype=torch.int32, device=device) + num_new_pages_per_topk = torch.tensor([2], dtype=torch.int32, device=device) + out_cache_loc = torch.arange(11, 11 + extend_lens_num, device=device) + last_page_lens = torch.tensor([3], dtype=torch.int32, device=device) + last_page_lens_cumsum = torch.cumsum(last_page_lens, dim=0) + duplicate_cache_len = last_page_lens.sum().item() * (topk - 1) + target_cache_loc = torch.zeros( + duplicate_cache_len, dtype=torch.int32, device=device + ) + source_cache_loc = torch.zeros( + duplicate_cache_len, dtype=torch.int32, device=device + ) + assign_draft_cache_locs[(num_seqs,)]( + req_pool_indices, + req_to_token, + seq_lens, + extend_lens, + num_new_pages_per_topk, + out_cache_loc, + source_cache_loc, + target_cache_loc, + last_page_lens_cumsum, + duplicate_cache_len, + req_to_token.shape[1], + topk, + speculative_num_steps, + page_size, + next_power_of_2(num_seqs), + next_power_of_2(speculative_num_steps + page_size), + ) + + out_cache_loc = out_cache_loc[: num_seqs * topk * speculative_num_steps] + expected_source_cache_loc = torch.tensor( + [8, 9, 10] * (topk - 1), device=device, dtype=torch.int32 + ) + assert torch.allclose(source_cache_loc, expected_source_cache_loc) + + copy_all_layer_kv_cache_tiled[(len(self.data_ptrs),)]( + self.data_ptrs, + self.data_strides, + target_cache_loc, + source_cache_loc, + len(target_cache_loc), + next_power_of_2(len(target_cache_loc)), + BYTES_PER_TILE, + ) + assert torch.allclose( + self.k_cache[0][16:19, 0, 0], + torch.tensor( + [0.8, 0.9, 1.0], + dtype=torch.float32, + device=device, + ), + ) + assert torch.allclose( + self.v_cache[0][16:19, 0, 0], + torch.tensor( + [-0.8, -0.9, -1.0], + dtype=torch.float32, + device=device, + ), + ) + + def test_assign_draft_cache_locs_multi_seq(self): + device = self.device + num_seqs = 3 + page_size = 4 + speculative_num_steps = 5 + topk = 8 + req_pool_indices = torch.arange(num_seqs, dtype=torch.int32, device=device) + req_to_token = torch.zeros((num_seqs, 100), dtype=torch.int32, device=device) + seq_lens = torch.tensor([8, 7, 5], dtype=torch.int32, device=device) + extend_lens = torch.tensor([64, 64, 64], dtype=torch.int32, device=device) + num_new_pages_per_topk = torch.tensor( + [2, 2, 2], dtype=torch.int32, device=device + ) + req_to_token = torch.zeros((num_seqs, 100), dtype=torch.int32, device=device) + req_to_token[0, :8] = torch.tensor([4, 5, 6, 7, 8, 9, 10, 11], device=device) + req_to_token[1, :7] = torch.tensor([4, 5, 6, 7, 8, 9, 10], device=device) + req_to_token[2, :5] = torch.tensor([4, 5, 6, 7, 8], device=device) + last_page_lens = torch.tensor([0, 3, 1], dtype=torch.int32, device=device) + last_page_lens_cumsum = torch.cumsum(last_page_lens, dim=0) + duplicate_cache_len = last_page_lens.sum().item() * (topk - 1) + out_cache_loc = torch.arange( + 12, 12 + torch.sum(extend_lens), dtype=torch.int32, device=device + ) + target_cache_loc = torch.zeros( + duplicate_cache_len, dtype=torch.int32, device=device + ) + source_cache_loc = torch.zeros( + duplicate_cache_len, dtype=torch.int32, device=device + ) + assign_draft_cache_locs[(num_seqs,)]( + req_pool_indices, + req_to_token, + seq_lens, + extend_lens, + num_new_pages_per_topk, + out_cache_loc, + source_cache_loc, + target_cache_loc, + last_page_lens_cumsum, + duplicate_cache_len, + req_to_token.shape[1], + topk, + speculative_num_steps, + page_size, + next_power_of_2(num_seqs), + next_power_of_2(speculative_num_steps + page_size), + ) + out_cache_loc = out_cache_loc[: num_seqs * topk * speculative_num_steps] + # fmt: off + expected_out_cache_loc = torch.tensor([ + 12, 13, 14, 15, 16, + 20, 21, 22, 23, 24, + 28, 29, 30, 31, 32, + 36, 37, 38, 39, 40, + 44, 45, 46, 47, 48, + 52, 53, 54, 55, 56, + 60, 61, 62, 63, 64, + 68, 69, 70, 71, 72, + 76, 77, 78, 79, 80, + 84, 85, 86, 87, 88, + 92, 93, 94, 95, 96, + 100, 101, 102, 103, 104, + 108, 109, 110, 111, 112, + 116, 117, 118, 119, 120, + 124, 125, 126, 127, 128, + 132, 133, 134, 135, 136, + 140, 141, 142, 143, 144, + 148, 149, 150, 151, 152, + 156, 157, 158, 159, 160, + 164, 165, 166, 167, 168, + 172, 173, 174, 175, 176, + 180, 181, 182, 183, 184, + 188, 189, 190, 191, 192, + 196, 197, 198, 199, 200 + ], device=device, dtype=torch.int32) + expected_source_cache_loc = torch.tensor([8, 9, 10] * 7 + [8] * 7, device=device, dtype=torch.int32) + expected_target_cache_loc = torch.tensor([ + 81, 82, 83, 89, 90, 91, 97, 98, 99, 105, 106, 107, 113, 114, + 115, 121, 122, 123, 129, 130, 131, 147, 155, 163, 171, 179, 187, 195 + ], device=device, dtype=torch.int32) + # fmt: on + assert torch.allclose(out_cache_loc, expected_out_cache_loc) + assert torch.allclose(source_cache_loc, expected_source_cache_loc) + assert torch.allclose(target_cache_loc, expected_target_cache_loc) + copy_all_layer_kv_cache_tiled[(len(self.data_ptrs),)]( + self.data_ptrs, + self.data_strides, + target_cache_loc, + source_cache_loc, + len(target_cache_loc), + next_power_of_2(len(target_cache_loc)), + BYTES_PER_TILE, + ) + assert torch.allclose( + self.k_cache[0][81:84, 0, 0], + torch.tensor( + [0.8, 0.9, 1.0], + dtype=torch.float32, + device=device, + ), + ) + assert torch.allclose( + self.v_cache[0][81:84, 0, 0], + torch.tensor( + [-0.8, -0.9, -1.0], + dtype=torch.float32, + device=device, + ), + ) + + def test_assign_draft_cache_locs_page_size_1(self): + # Test to make sure page_size=1 not affected + device = self.device + num_seqs = 1 + page_size = 1 + speculative_num_steps = 5 + topk = 8 + seq_lens_num = 7 + extend_lens_num = topk * speculative_num_steps + req_pool_indices = torch.arange(num_seqs, dtype=torch.int32, device=device) + req_to_token = torch.zeros((num_seqs, 100), dtype=torch.int32, device=device) + req_to_token[0, :seq_lens_num] = torch.tensor( + [4, 5, 6, 7, 8, 9, 10], device=device + ) + seq_lens = torch.tensor([seq_lens_num], dtype=torch.int32, device=device) + extend_lens = torch.tensor([extend_lens_num], dtype=torch.int32, device=device) + num_new_pages_per_topk = torch.tensor([2], dtype=torch.int32, device=device) + out_cache_loc = torch.arange(11, 11 + extend_lens_num, device=device) + last_page_lens = torch.tensor([3], dtype=torch.int32, device=device) + duplicate_cache_len = 0 + target_cache_loc = None + source_cache_loc = None + last_page_lens_cumsum = None + assign_draft_cache_locs[(num_seqs,)]( + req_pool_indices, + req_to_token, + seq_lens, + extend_lens, + num_new_pages_per_topk, + out_cache_loc, + source_cache_loc, + target_cache_loc, + last_page_lens_cumsum, + duplicate_cache_len, + req_to_token.shape[1], + topk, + speculative_num_steps, + page_size, + next_power_of_2(num_seqs), + next_power_of_2(speculative_num_steps + page_size), + ) + out_cache_loc = out_cache_loc[: num_seqs * topk * speculative_num_steps] + expected_out_cache_loc = torch.arange(11, 11 + extend_lens_num, device=device) + assert torch.allclose(out_cache_loc, expected_out_cache_loc) + + def test_assign_draft_cache_locs_page_size_gt_spec_steps(self): + device = self.device + num_seqs = 1 + page_size = 16 + speculative_num_steps = 4 + topk = 3 + seq_lens_num = 12 + pool_len = 256 + req_pool_indices = torch.arange(num_seqs, dtype=torch.int32, device=device) + req_to_token = torch.zeros( + (num_seqs, pool_len), dtype=torch.int32, device=device + ) + req_to_token[0, :seq_lens_num] = torch.arange( + seq_lens_num, dtype=torch.int32, device=device + ) + seq_lens = torch.tensor([seq_lens_num], dtype=torch.int32, device=device) + last_page_len = seq_lens_num % page_size + last_page_lens = torch.tensor([last_page_len], dtype=torch.int32, device=device) + last_page_lens_cumsum = torch.cumsum(last_page_lens, dim=0) + num_new_pages_per_topk_val = ( + last_page_len + speculative_num_steps + page_size - 1 + ) // page_size + num_new_pages_per_topk = torch.tensor( + [num_new_pages_per_topk_val], dtype=torch.int32, device=device + ) + extend_lens_num = num_new_pages_per_topk_val * page_size * topk + extend_lens = torch.tensor([extend_lens_num], dtype=torch.int32, device=device) + out_cache_loc = torch.arange( + 2000, 2000 + extend_lens_num, dtype=torch.int32, device=device + ) + duplicate_cache_len = last_page_lens.sum().item() * (topk - 1) + target_cache_loc = torch.zeros( + duplicate_cache_len, dtype=torch.int32, device=device + ) + source_cache_loc = torch.zeros( + duplicate_cache_len, dtype=torch.int32, device=device + ) + assign_draft_cache_locs[(num_seqs,)]( + req_pool_indices, + req_to_token, + seq_lens, + extend_lens, + num_new_pages_per_topk, + out_cache_loc, + source_cache_loc, + target_cache_loc, + last_page_lens_cumsum, + duplicate_cache_len, + req_to_token.shape[1], + topk, + speculative_num_steps, + page_size, + next_power_of_2(num_seqs), + next_power_of_2(speculative_num_steps + page_size), + ) + trimmed = out_cache_loc[: num_seqs * topk * speculative_num_steps] + expected = [] + for topk_id in range(topk): + start = seq_lens_num + topk_id * num_new_pages_per_topk_val * page_size + expected.append( + req_to_token[0, start : start + speculative_num_steps].clone() + ) + expected_out_cache_loc = torch.cat(expected) + assert torch.allclose(trimmed, expected_out_cache_loc) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_activation.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_activation.py new file mode 100644 index 00000000..dd5c668c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_activation.py @@ -0,0 +1,105 @@ +import itertools +import unittest + +import torch + +from sglang.srt.layers.activation import GeluAndMul, QuickGELU +from sglang.srt.utils import is_hip +from sglang.test.test_utils import CustomTestCase + +_is_hip = is_hip() + + +class TestGeluAndMul(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16] + NUM_TOKENS = [7, 83, 2048] + D = [512, 4096, 5120, 13824] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _run_gelu_and_mul_test(self, num_tokens, d, dtype, seed): + torch.manual_seed(seed) + + layer = GeluAndMul().to(dtype=dtype) + x = torch.randn(num_tokens, 2 * d, dtype=dtype) + + with torch.inference_mode(): + ref_out = layer.forward_native(x) + out = layer.forward_cuda(x) + + if dtype == torch.bfloat16: + atol = rtol = 1e-2 + else: + atol = rtol = 1e-3 + + self.assertTrue(torch.allclose(out, ref_out, atol=atol, rtol=rtol)) + + def test_gelu_and_mul(self): + for params in itertools.product( + self.NUM_TOKENS, + self.D, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + num_tokens=params[0], + d=params[1], + dtype=params[2], + seed=params[3], + ): + self._run_gelu_and_mul_test(*params) + + +class TestQuickGELU(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16] + NUM_TOKENS = [7, 83, 2048] # batch = sequence length + DIMS = [512, 4096, 5120, 13824] # all multiples of 16 bytes + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _run_gelu_quick_test(self, n_tok: int, dim: int, dtype: torch.dtype, seed: int): + torch.manual_seed(seed) + + layer = QuickGELU().to(dtype=dtype) + + x = torch.randn(n_tok, dim, dtype=dtype, device="cuda") + + with torch.inference_mode(): + ref = layer.forward_native(x) # x * sigmoid(1.702 * x), fp32 math + if _is_hip: + out = layer.forward_hip(x) # 128-bit vectorised kernel from sgl-kernel + else: + out = layer.forward_cuda(x) + + tol = 1e-2 if dtype is torch.bfloat16 else 1e-3 + self.assertTrue( + torch.allclose(out, ref, atol=tol, rtol=tol), + msg=f"Mismatch @ B={n_tok}, D={dim}, dtype={dtype}", + ) + print(f"Match @ B={n_tok}, D={dim}, dtype={dtype}") + + def test_quick_gelu(self): + for params in itertools.product( + self.NUM_TOKENS, self.DIMS, self.DTYPES, self.SEEDS + ): + with self.subTest( + num_tokens=params[0], + dim=params[1], + dtype=params[2], + seed=params[3], + ): + self._run_gelu_quick_test(*params) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_block_fp8.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_block_fp8.py new file mode 100644 index 00000000..2390489c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_block_fp8.py @@ -0,0 +1,660 @@ +import itertools +import unittest + +import torch + +from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import fused_moe +from sglang.srt.layers.moe.topk import TopKConfig, select_experts +from sglang.srt.layers.quantization.fp8_kernel import ( + per_tensor_quant_mla_fp8, + per_token_group_quant_fp8, + per_token_group_quant_mla_deep_gemm_masked_fp8, + static_quant_fp8, + w8a8_block_fp8_matmul, +) +from sglang.srt.layers.quantization.fp8_utils import input_to_float8 +from sglang.test.test_utils import CustomTestCase + +_is_cuda = torch.cuda.is_available() and torch.version.cuda + + +# For test +def native_per_token_group_quant_fp8( + x, group_size, eps=1e-10, dtype=torch.float8_e4m3fn +): + """Function to perform per-token-group quantization on an input tensor `x` using native torch. + + It converts the tensor values into float8 values and returns the + quantized tensor along with the scaling factor used for quantization. + Note that only `torch.float8_e4m3fn` is supported for now. + """ + assert ( + x.shape[-1] % group_size == 0 + ), "the last dimension of `x` cannot be divisible by `group_size`" + assert x.is_contiguous(), "`x` is not contiguous" + + finfo = torch.finfo(dtype) + fp8_min = finfo.min + fp8_max = finfo.max + + x_ = x.reshape(x.numel() // group_size, group_size) + amax = x_.abs().max(dim=-1, keepdim=True)[0].clamp(min=eps).to(torch.float32) + x_s = amax / fp8_max + x_q = (x_ / x_s).clamp(min=fp8_min, max=fp8_max).to(dtype) + x_q = x_q.reshape(x.shape) + x_s = x_s.reshape(x.shape[:-1] + (x.shape[-1] // group_size,)) + + return x_q, x_s + + +class TestPerTokenGroupQuantFP8(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16, torch.float32] + NUM_TOKENS = [7, 83, 2048] + D = [512, 4096, 5120, 13824] + GROUP_SIZE = [64, 128, 256, 512] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _per_token_group_quant_fp8(self, num_tokens, d, dtype, group_size, seed): + torch.manual_seed(seed) + + x = torch.rand(num_tokens, d, dtype=dtype) + + with torch.inference_mode(): + ref_out, ref_scale = native_per_token_group_quant_fp8(x, group_size) + out, scale = per_token_group_quant_fp8(x, group_size) + + self.assertTrue( + torch.allclose(out.to(torch.float32), ref_out.to(torch.float32), rtol=0.20) + ) + self.assertTrue(torch.allclose(scale, ref_scale)) + + def test_per_token_group_quant_fp8(self): + for params in itertools.product( + self.NUM_TOKENS, + self.D, + self.DTYPES, + self.GROUP_SIZE, + self.SEEDS, + ): + with self.subTest( + num_tokens=params[0], + d=params[1], + dtype=params[2], + group_size=params[3], + seed=params[4], + ): + self._per_token_group_quant_fp8(*params) + + +# For test +def native_static_quant_fp8(x, x_s, dtype=torch.float8_e4m3fn): + """Function to perform static quantization on an input tensor `x` using native torch. + + It converts the tensor values into float8 values and returns the + quantized tensor along with the scaling factor used for quantization. + """ + assert x.is_contiguous(), "`x` is not contiguous" + assert x_s.numel() == 1, "only supports per-tensor scale" + + finfo = torch.finfo(dtype) + fp8_min = finfo.min + fp8_max = finfo.max + + x_ = x.reshape(x.numel() // x.shape[-1], x.shape[-1]) + x_s_inv = 1.0 / x_s + x_q = (x_ * x_s_inv).clamp(min=fp8_min, max=fp8_max).to(dtype) + x_q = x_q.reshape(x.shape) + + return x_q, x_s + + +class TestStaticQuantFP8(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16, torch.float32] + NUM_TOKENS = [7, 83, 2048] + D = [512, 4096, 5120, 13824] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _static_quant_fp8(self, num_tokens, d, dtype, seed): + torch.manual_seed(seed) + + x = torch.rand(num_tokens, d, dtype=dtype) + fp8_max = torch.finfo(torch.float8_e4m3fn).max + x_s = x.max() / fp8_max + + with torch.inference_mode(): + ref_out, _ = native_static_quant_fp8(x, x_s) + out, _ = static_quant_fp8(x, x_s, repeat_scale=True) + + self.assertTrue( + torch.allclose(out.to(torch.float32), ref_out.to(torch.float32), rtol=0.50) + ) + + def test_static_quant_fp8(self): + for params in itertools.product( + self.NUM_TOKENS, + self.D, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + num_tokens=params[0], + d=params[1], + dtype=params[2], + seed=params[3], + ): + self._static_quant_fp8(*params) + + +class TestPerTensorQuantMlaFP8(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16, torch.float32] + NUM_TOKENS = [7, 83, 2048] + D = [512, 4096, 5120, 13824] + LAST_D_EXT = [1024, 0] + LAST_D = [512] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _per_tensor_quant_mla_fp8(self, num_tokens, d, last_d_ext, last_d, dtype, seed): + torch.manual_seed(seed) + + x = torch.rand( + (num_tokens, d // last_d, last_d + last_d_ext), + dtype=dtype, + ) + x_sub, _ = x.split([last_d, last_d_ext], dim=-1) + + with torch.inference_mode(): + ref_out, ref_s = input_to_float8(x_sub.transpose(0, 1)) + out, out_s = per_tensor_quant_mla_fp8(x_sub.transpose(0, 1)) + + self.assertTrue(out.is_contiguous()) + self.assertTrue( + torch.allclose(out.to(torch.float32), ref_out.to(torch.float32), rtol=0.50) + ) + self.assertTrue( + torch.allclose(out_s.to(torch.float32), ref_s.to(torch.float32)) + ) + + def test_per_tensor_quant_mla_fp8(self): + for params in itertools.product( + self.NUM_TOKENS, + self.D, + self.LAST_D_EXT, + self.LAST_D, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + num_tokens=params[0], + d=params[1], + last_d_ext=params[2], + last_d=params[3], + dtype=params[4], + seed=params[5], + ): + self._per_tensor_quant_mla_fp8(*params) + + +class TestPerTokenGroupQuantMlaDeepGemmMaskedFP8(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16, torch.float32] + B = [128] + NUM_TOKENS = [7, 83, 2048, 1024 * 16] + D = [512, 128] + GROUP_SIZE = [128] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _per_token_group_quant_mla_deep_gemm_masked_fp8( + self, b, num_tokens, d, dtype, group_size, seed + ): + torch.manual_seed(seed) + + x = torch.rand(b, num_tokens, d, dtype=dtype) + + with torch.inference_mode(): + ref_out, ref_scale = native_per_token_group_quant_fp8(x, group_size, 1e-12) + out, scale, _, _, _ = per_token_group_quant_mla_deep_gemm_masked_fp8( + x, group_size + ) + out = out[:, :num_tokens, :] + scale = scale[:, :num_tokens, :] + + self.assertTrue( + torch.allclose( + out.to(torch.float32), ref_out.to(torch.float32), rtol=0.20, atol=1e-2 + ) + ) + self.assertTrue(torch.allclose(scale, ref_scale)) + + def test_per_token_group_quant_mla_deep_gemm_masked_fp8(self): + for params in itertools.product( + self.B, + self.NUM_TOKENS, + self.D, + self.DTYPES, + self.GROUP_SIZE, + self.SEEDS, + ): + with self.subTest( + b=params[0], + num_tokens=params[1], + d=params[2], + dtype=params[3], + group_size=params[4], + seed=params[5], + ): + self._per_token_group_quant_mla_deep_gemm_masked_fp8(*params) + + +# For test +def native_w8a8_block_fp8_matmul(A, B, As, Bs, block_size, output_dtype=torch.float16): + """This function performs matrix multiplication with block-wise quantization using native torch. + + It takes two input tensors `A` and `B` with scales `As` and `Bs`. + The output is returned in the specified `output_dtype`. + """ + + A = A.to(torch.float32) + B = B.to(torch.float32) + assert A.shape[-1] == B.shape[-1] + assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2 + assert len(block_size) == 2 + block_n, block_k = block_size[0], block_size[1] + assert (A.shape[-1] + block_k - 1) // block_k == As.shape[-1] + assert A.shape[:-1] == As.shape[:-1] + + M = A.numel() // A.shape[-1] + N, K = B.shape + origin_C_shape = A.shape[:-1] + (N,) + A = A.reshape(M, A.shape[-1]) + As = As.reshape(M, As.shape[-1]) + n_tiles = (N + block_n - 1) // block_n + k_tiles = (K + block_k - 1) // block_k + assert n_tiles == Bs.shape[0] + assert k_tiles == Bs.shape[1] + + C_shape = (M, N) + C = torch.zeros(C_shape, dtype=torch.float32, device=A.device) + + A_tiles = [A[:, i * block_k : min((i + 1) * block_k, K)] for i in range(k_tiles)] + B_tiles = [ + [ + B[ + j * block_n : min((j + 1) * block_n, N), + i * block_k : min((i + 1) * block_k, K), + ] + for i in range(k_tiles) + ] + for j in range(n_tiles) + ] + C_tiles = [C[:, j * block_n : min((j + 1) * block_n, N)] for j in range(n_tiles)] + As_tiles = [As[:, i : i + 1] for i in range(k_tiles)] + + for i in range(k_tiles): + for j in range(n_tiles): + a = A_tiles[i] + b = B_tiles[j][i] + c = C_tiles[j] + s = As_tiles[i] * Bs[j][i] + c[:, :] += torch.matmul(a, b.t()) * s + + C = C.reshape(origin_C_shape).to(output_dtype) + return C + + +class TestW8A8BlockFP8Matmul(CustomTestCase): + + if not _is_cuda: + OUT_DTYPES = [torch.float32, torch.half, torch.bfloat16] + M = [1, 7, 83, 512, 2048] + NKs = [ + (N, K) + for N in [128, 512, 1024, 4096, 7748, 13824] + for K in [256, 4096, 5120, 3884, 13824] + ] + # BLOCK_SIZE = [[64, 64], [64, 128], [128, 64], [128, 128]] + BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + else: + # use practical shape in DeepSeek V3 for test + OUT_DTYPES = [torch.bfloat16] + M = [64, 128, 512, 1024, 4096] + NKs = [ + (2112, 7168), + (1536, 7168), + (3072, 1536), + (24576, 7168), + (4096, 512), + (7168, 2048), + (4608, 7168), + (512, 7168), + (7168, 2304), + (7168, 512), + ] + BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _w8a8_block_fp8_matmul(self, M, NK, block_size, out_dtype, seed): + N, K = NK + torch.manual_seed(seed) + # NOTE(HandH1998): to avoid overflow when out_dtype = torch.half + factor_for_scale = 1e-2 + fp8_info = torch.finfo(torch.float8_e4m3fn) + fp8_max, fp8_min = fp8_info.max, fp8_info.min + + A_fp32 = (torch.rand(M, K, dtype=torch.float32) - 0.5) * 2 * fp8_max + A_fp8 = A_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + B_fp32 = (torch.rand(N, K, dtype=torch.float32) - 0.5) * 2 * fp8_max + B_fp8 = B_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + block_n, block_k = block_size[0], block_size[1] + n_tiles = (N + block_n - 1) // block_n + k_tiles = (K + block_k - 1) // block_k + + As = torch.rand(M, k_tiles, dtype=torch.float32) * factor_for_scale + Bs = torch.rand(n_tiles, k_tiles, dtype=torch.float32) * factor_for_scale + + with torch.inference_mode(): + ref_out = native_w8a8_block_fp8_matmul( + A_fp8, B_fp8, As, Bs, block_size, out_dtype + ) + out = w8a8_block_fp8_matmul(A_fp8, B_fp8, As, Bs, block_size, out_dtype) + + self.assertTrue( + torch.mean(torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))) + / torch.mean(torch.abs(ref_out.to(torch.float32))) + < 0.001 + ) + + def test_w8a8_block_fp8_matmul(self): + for params in itertools.product( + self.M, + self.NKs, + self.BLOCK_SIZE, + self.OUT_DTYPES, + self.SEEDS, + ): + with self.subTest( + M=params[0], + NKs=params[1], + block_size=params[2], + out_dtype=params[3], + seed=params[4], + ): + self._w8a8_block_fp8_matmul(*params) + + +# For test +def torch_w8a8_block_fp8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape): + """This function performs fused moe with block-wise quantization using native torch.""" + + B, D = a.shape + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + + _, block_k = block_shape[0], block_shape[1] + a_q, a_s = native_per_token_group_quant_fp8(a, block_k) + # NOTE(HandH1998): Since "index_cuda" not implemented for 'Float8_e4m3fn', we need to cast `float8`` to `float32``. + a_q = a_q.to(torch.float32) + for i in range(w1.shape[0]): + mask = topk_ids == i + if mask.sum(): + inter_out = native_w8a8_block_fp8_matmul( + a_q[mask], w1[i], a_s[mask], w1_s[i], block_shape, output_dtype=a.dtype + ) + act_out = SiluAndMul().forward_native(inter_out) + act_out_q, act_out_s = native_per_token_group_quant_fp8(act_out, block_k) + act_out = act_out.to(torch.float32) + out[mask] = native_w8a8_block_fp8_matmul( + act_out_q, w2[i], act_out_s, w2_s[i], block_shape, output_dtype=a.dtype + ) + return ( + out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype) + ).sum(dim=1) + + +class TestW8A8BlockFP8FusedMoE(CustomTestCase): + DTYPES = [torch.float32, torch.half, torch.bfloat16] + M = [1, 33, 64, 222, 1024 * 128] + N = [128, 1024, 2048] + K = [256, 4096, 5120] + E = [8, 24] + TOP_KS = [2, 6] + BLOCK_SIZE = [[64, 64], [64, 128], [128, 64], [128, 128]] + # BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _w8a8_block_fp8_fused_moe(self, M, N, K, E, topk, block_size, dtype, seed): + torch.manual_seed(seed) + # NOTE(HandH1998): to avoid overflow when out_dtype = torch.half + factor_for_scale = 1e-2 + fp8_info = torch.finfo(torch.float8_e4m3fn) + fp8_max, fp8_min = fp8_info.max, fp8_info.min + + a = torch.randn((M, K), dtype=dtype) / 10 + + w1_fp32 = (torch.rand((E, 2 * N, K), dtype=torch.float32) - 0.5) * 2 * fp8_max + w1 = w1_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + w2_fp32 = (torch.rand((E, K, N), dtype=torch.float32) - 0.5) * 2 * fp8_max + w2 = w2_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + block_n, block_k = block_size[0], block_size[1] + n_tiles_w1 = (2 * N + block_n - 1) // block_n + n_tiles_w2 = (K + block_n - 1) // block_n + k_tiles_w1 = (K + block_k - 1) // block_k + k_tiles_w2 = (N + block_k - 1) // block_k + + w1_s = ( + torch.rand((E, n_tiles_w1, k_tiles_w1), dtype=torch.float32) + * factor_for_scale + ) + w2_s = ( + torch.rand((E, n_tiles_w2, k_tiles_w2), dtype=torch.float32) + * factor_for_scale + ) + + score = torch.randn((M, E), dtype=dtype) + + with torch.inference_mode(): + ref_out = torch_w8a8_block_fp8_moe( + a, w1, w2, w1_s, w2_s, score, topk, block_size + ) + topk_output = select_experts( + hidden_states=a, + router_logits=score, + topk_config=TopKConfig(top_k=topk, renormalize=False), + ) + out = fused_moe( + a, + w1, + w2, + topk_output, + use_fp8_w8a8=True, + w1_scale=w1_s, + w2_scale=w2_s, + block_shape=block_size, + ) + + self.assertTrue( + torch.mean(torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))) + / torch.mean(torch.abs(ref_out.to(torch.float32))) + < 0.02 + ) + + def test_w8a8_block_fp8_fused_moe(self): + for params in itertools.product( + self.M, + self.N, + self.K, + self.E, + self.TOP_KS, + self.BLOCK_SIZE, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + E=params[3], + topk=params[4], + block_size=params[5], + dtype=params[6], + seed=params[7], + ): + self._w8a8_block_fp8_fused_moe(*params) + + +# For test +def torch_w8a8_block_fp8_bmm(a, a_s, w, w_s, block_shape, out_dtype): + """This function performs bmm with block-wise quantization using native torch.""" + + B, N, _ = w.shape + _, M, _ = a.shape + out = torch.empty((B, M, N), dtype=out_dtype, device=a.device) + + for i in range(B): + out[i] = native_w8a8_block_fp8_matmul( + a[i], w[i], a_s[i], w_s[i], block_shape, output_dtype=out_dtype + ) + + return out + + +class TestW8A8BlockFP8BatchedDeepGemm(CustomTestCase): + DTYPES = [torch.bfloat16] + M = [1, 33, 64, 222, 8192] + N = [128, 512] + K = [128, 512] + BATCH = [128] + BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + try: + import deep_gemm # noqa: F401 + except ImportError: + raise unittest.SkipTest("DeepGEMM is not available") + torch.set_default_device("cuda") + + def _w8a8_block_fp8_batched_deep_gemm(self, M, N, K, B, block_size, dtype, seed): + torch.manual_seed(seed) + factor_for_scale = 1e-2 + fp8_info = torch.finfo(torch.float8_e4m3fn) + fp8_max, fp8_min = fp8_info.max, fp8_info.min + + a_fp32 = torch.randn((B, M, K), dtype=torch.float32) / 10 + a = a_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + w_fp32 = (torch.rand((B, N, K), dtype=torch.float32) - 0.5) * 2 * fp8_max + w = w_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + block_n, block_k = block_size[0], block_size[1] + n_tiles_w = (N + block_n - 1) // block_n + k_tiles_w = (K + block_k - 1) // block_k + + w_s = ( + torch.rand((B, n_tiles_w, k_tiles_w), dtype=torch.float32) + * factor_for_scale + ) + a_s = torch.rand((B, M, k_tiles_w), dtype=torch.float32) * factor_for_scale + + ae = a.new_empty(B, (M + 255) // 256 * 256, K) + ae_s = a_s.new_empty(B, (M + 255) // 256 * 256, k_tiles_w) + oe = torch.empty((B, (M + 255) // 256 * 256, N), dtype=dtype) + ae[:, :M, :] = a + ae_s[:, :M, :] = a_s + + masked_m = torch.full((B,), M, dtype=torch.int) + expected_m = M + lhs = ( + ae, + ae_s, + ) + rhs = ( + w, + w_s, + ) + + from deep_gemm import fp8_m_grouped_gemm_nt_masked + + with torch.inference_mode(): + ref_out = torch_w8a8_block_fp8_bmm(a, a_s, w, w_s, block_size, dtype) + fp8_m_grouped_gemm_nt_masked(lhs, rhs, oe, masked_m, expected_m) + out = oe[:, :M, :] + + self.assertTrue( + torch.mean(torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))) + / torch.mean(torch.abs(ref_out.to(torch.float32))) + < 0.0001 + ) + + def test_w8a8_block_fp8_batched_deep_gemm(self): + + for params in itertools.product( + self.M, + self.N, + self.K, + self.BATCH, + self.BLOCK_SIZE, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + B=params[3], + block_size=params[4], + dtype=params[5], + seed=params[6], + ): + self._w8a8_block_fp8_batched_deep_gemm(*params) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_block_fp8_deep_gemm_blackwell.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_block_fp8_deep_gemm_blackwell.py new file mode 100644 index 00000000..833c23e7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_block_fp8_deep_gemm_blackwell.py @@ -0,0 +1,251 @@ +import itertools +import unittest +from typing import List, Tuple + +import torch +from deep_gemm import fp8_gemm_nt + +from sglang.test.test_utils import CustomTestCase + +_is_cuda = torch.cuda.is_available() and torch.version.cuda + + +# Modify form DeepGEMM Blackwell +def ceil_div(x: int, y: int) -> int: + return (x + y - 1) // y + + +def align(x: int, y: int) -> int: + return ceil_div(x, y) * y + + +def per_token_group_quant_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 and x.size(1) % 128 == 0 + m, n = x.shape + x_view = x.view(m, -1, 128) + x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4) + sf = x_amax / 448.0 + return (x_view * (1.0 / sf.unsqueeze(2))).to(torch.float8_e4m3fn).view(m, n), sf + + +def per_block_quant_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 + m, n = x.shape + x_padded = torch.zeros( + (align(m, 128), align(n, 128)), dtype=x.dtype, device=x.device + ) + x_padded[:m, :n] = x + x_view = x_padded.view(-1, 128, x_padded.size(1) // 128, 128) + x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4) + sf = x_amax / 448.0 + x_scaled = (x_view * (1.0 / sf)).to(torch.float8_e4m3fn) + return x_scaled.view_as(x_padded)[:m, :n].contiguous(), sf.view( + x_view.size(0), x_view.size(2) + ) + + +def ceil_to_ue8m0(x: torch.Tensor): + assert x.view(-1).amax().item() > 0 + return torch.pow(2.0, torch.ceil(torch.log2(x.abs()))) + + +def per_token_group_quant_mxfp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 and x.size(1) % 128 == 0 + m, n = x.shape + x_view = x.view(m, -1, 128) + x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4) + sf = ceil_to_ue8m0(x_amax / 448.0) + return (x_view * (1.0 / sf.unsqueeze(2))).to(torch.float8_e4m3fn).view(m, n), sf + + +def per_block_quant_mxfp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 + m, n = x.shape + x_padded = torch.zeros( + (align(m, 128), align(n, 128)), dtype=x.dtype, device=x.device + ) + x_padded[:m, :n] = x + x_view = x_padded.view(-1, 128, x_padded.size(1) // 128, 128) + x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4) + sf = ceil_to_ue8m0(x_amax / 448.0) + x_scaled = (x_view * (1.0 / sf)).to(torch.float8_e4m3fn) + return x_scaled.view_as(x_padded)[:m, :n].contiguous(), sf.view( + x_view.size(0), x_view.size(2) + ) + + +# For test +def native_w8a8_block_fp8_matmul(A, B, As, Bs, block_size, output_dtype=torch.float16): + """This function performs matrix multiplication with block-wise quantization using native torch. + + It takes two input tensors `A` and `B` with scales `As` and `Bs`. + The output is returned in the specified `output_dtype`. + """ + + A = A.to(torch.float32) + B = B.to(torch.float32) + assert A.shape[-1] == B.shape[-1] + assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2 + assert len(block_size) == 2 + block_n, block_k = block_size[0], block_size[1] + assert (A.shape[-1] + block_k - 1) // block_k == As.shape[-1] + assert A.shape[:-1] == As.shape[:-1] + + M = A.numel() // A.shape[-1] + N, K = B.shape + origin_C_shape = A.shape[:-1] + (N,) + A = A.reshape(M, A.shape[-1]) + As = As.reshape(M, As.shape[-1]) + n_tiles = (N + block_n - 1) // block_n + k_tiles = (K + block_k - 1) // block_k + assert n_tiles == Bs.shape[0] + assert k_tiles == Bs.shape[1] + + C_shape = (M, N) + C = torch.zeros(C_shape, dtype=torch.float32, device=A.device) + + A_tiles = [A[:, i * block_k : min((i + 1) * block_k, K)] for i in range(k_tiles)] + B_tiles = [ + [ + B[ + j * block_n : min((j + 1) * block_n, N), + i * block_k : min((i + 1) * block_k, K), + ] + for i in range(k_tiles) + ] + for j in range(n_tiles) + ] + C_tiles = [C[:, j * block_n : min((j + 1) * block_n, N)] for j in range(n_tiles)] + As_tiles = [As[:, i : i + 1] for i in range(k_tiles)] + + for i in range(k_tiles): + for j in range(n_tiles): + a = A_tiles[i] + b = B_tiles[j][i] + c = C_tiles[j] + s = As_tiles[i] * Bs[j][i] + c[:, :] += torch.matmul(a, b.t()) * s + + C = C.reshape(origin_C_shape).to(output_dtype) + return C + + +def block_quant_dequant( + x_q_block: torch.Tensor, + x_s: torch.Tensor, + block_size: List[int], + dtype: torch.dtype, +) -> torch.Tensor: + """This function converts block-wise quantization to unquantized. + The inputs are block-wise quantization tensor `x_q_block`, block-wise quantization scale + and the block size. + The output is an unquantized tensor with dtype. + """ + block_n, block_k = block_size[0], block_size[1] + n, k = x_q_block.shape + n_tiles = (n + block_n - 1) // block_n + k_tiles = (k + block_k - 1) // block_k + assert n_tiles == x_s.shape[0] + assert k_tiles == x_s.shape[1] + + x_dq_block = torch.empty_like(x_q_block, dtype=dtype) + + for j in range(n_tiles): + for i in range(k_tiles): + x_q_block_tile = x_q_block[ + j * block_n : min((j + 1) * block_n, n), + i * block_k : min((i + 1) * block_k, k), + ] + x_dq_block_tile = x_dq_block[ + j * block_n : min((j + 1) * block_n, n), + i * block_k : min((i + 1) * block_k, k), + ] + x_dq_block_tile[:, :] = x_q_block_tile.to(torch.float32) * x_s[j][i] + + return x_dq_block + + +class TestDeepGemmBlackwell(CustomTestCase): + + if not _is_cuda: + OUT_DTYPES = [torch.float32, torch.half, torch.bfloat16] + M = [1, 7, 83, 512, 2048] + NKs = [ + (N, K) + for N in [128, 512, 1024, 4096, 7748, 13824] + for K in [256, 4096, 5120, 3884, 13824] + ] + # BLOCK_SIZE = [[64, 64], [64, 128], [128, 64], [128, 128]] + BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + else: + # use practical shape in DeepSeek V3 for test + OUT_DTYPES = [torch.bfloat16] + M = [64, 128, 512, 1024, 4096] + NKs = [ + (2112, 7168), + (1536, 7168), + # (3072, 1536), + # (24576, 7168), + # (4096, 512), + # (7168, 2048), + # (4608, 7168), + # (512, 7168), + # (7168, 2304), + # (7168, 512), + ] + BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _test_deep_gemm_blackwell(self, M, NK, block_size, out_dtype, seed): + N, K = NK + torch.manual_seed(seed) + + A = torch.empty((M, K), dtype=torch.bfloat16).normal_(0, 0.2) + B = torch.empty((N, K), dtype=torch.bfloat16).normal_(0, 0.2) + + A_q, A_s = per_token_group_quant_fp8(A) + B_q, B_s = per_block_quant_fp8(B) + + A_dq = block_quant_dequant(A_q, A_s, [1, block_size[1]], out_dtype) + B_dq = block_quant_dequant(B_q, B_s, block_size, out_dtype) + + A_qu = per_token_group_quant_mxfp8(A_dq) + B_qu = per_block_quant_mxfp8(B_dq) + out = None + + with torch.inference_mode(): + ref_out = native_w8a8_block_fp8_matmul( + A_qu[0], B_qu[0], A_qu[1], B_qu[1], block_size, out_dtype + ) + out = torch.empty_like(ref_out) + fp8_gemm_nt(A_qu, B_qu, out) + + torch.testing.assert_close(out, ref_out, atol=1e-1, rtol=1e-2) + + def test_deep_gemm_blackwell(self): + for params in itertools.product( + self.M, + self.NKs, + self.BLOCK_SIZE, + self.OUT_DTYPES, + self.SEEDS, + ): + with self.subTest( + M=params[0], + NKs=params[1], + block_size=params[2], + out_dtype=params[3], + seed=params[4], + ): + self._test_deep_gemm_blackwell(*params) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_custom_ops.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_custom_ops.py new file mode 100644 index 00000000..c07c95db --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_custom_ops.py @@ -0,0 +1,148 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/8ca7a71df787ad711ad3ac70a5bd2eb2bb398938/tests/quantization/test_fp8.py + +import pytest +import torch + +from sglang.srt.layers.quantization.fp8_kernel import is_fp8_fnuz, scaled_fp8_quant +from sglang.srt.utils import is_cuda, is_hip + +_is_cuda = is_cuda() +_is_hip = is_hip() +_is_fp8_fnuz = is_fp8_fnuz() +fp8_dtype = torch.float8_e4m3fnuz if _is_fp8_fnuz else torch.float8_e4m3fn + + +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +def test_scaled_fp8_quant_per_tensor(dtype) -> None: + + def quantize_ref_per_tensor(tensor, inv_scale): + # The reference implementation that fully aligns to + # the kernel being tested. + finfo = torch.finfo(fp8_dtype) + scale = inv_scale.reciprocal() + qweight = (tensor.to(torch.float32) * scale).clamp(min=finfo.min, max=finfo.max) + qweight = qweight.to(fp8_dtype) + return qweight + + def dequantize_per_tensor(tensor, inv_scale, dtype): + fake_qweight = tensor.to(dtype) + dq_weight = fake_qweight * inv_scale + return dq_weight + + # Note that we use a shape % 8 != 0 to cover edge cases, + # because scaled_fp8_quant is vectorized by 8. + x = (torch.randn(size=(11, 11), device="cuda") * 13).to(dtype) + + # Test Per Tensor Dynamic quantization + # scale = max(abs(x)) / FP8_E4M3_MAX + y, scale = scaled_fp8_quant(x, None) + ref_y = quantize_ref_per_tensor(x, scale) + torch.testing.assert_close(y, ref_y) + torch.testing.assert_close( + dequantize_per_tensor(y, scale, dtype), + dequantize_per_tensor(ref_y, scale, dtype), + ) + + # Test Per Tensor Static quantization + y, _ = scaled_fp8_quant(x, scale) + ref_y = quantize_ref_per_tensor(x, scale) + torch.testing.assert_close(y, ref_y) + torch.testing.assert_close( + dequantize_per_tensor(y, scale, dtype), + dequantize_per_tensor(ref_y, scale, dtype), + ) + + +if _is_cuda or _is_hip: + + @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) + def test_scaled_fp8_quant_per_token_dynamic(dtype) -> None: + def quantize_ref_per_token(tensor, inv_scale): + # The reference implementation that fully aligns to + # the kernel being tested. + finfo = torch.finfo(fp8_dtype) + scale = inv_scale.reciprocal() + qweight = (tensor.to(torch.float32) * scale).clamp( + min=finfo.min, max=finfo.max + ) + qweight = qweight.to(fp8_dtype) + return qweight + + def dequantize_per_token(tensor, inv_scale, dtype): + fake_qweight = tensor.to(dtype) + dq_weight = fake_qweight * inv_scale + return dq_weight + + # Note that we use a shape % 8 = 0, + # because per_token_quant_fp8 is vectorized by 8 elements. + x = (torch.randn(size=(11, 16), device="cuda") * 13).to(dtype) + + # Test Per Tensor Dynamic quantization + # scale = max(abs(x)) / FP8_E4M3_MAX + y, scale = scaled_fp8_quant(x, None, use_per_token_if_dynamic=True) + ref_y = quantize_ref_per_token(x, scale) + torch.testing.assert_close(y, ref_y) + torch.testing.assert_close( + dequantize_per_token(y, scale, dtype), + dequantize_per_token(ref_y, scale, dtype), + ) + + @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) + def test_scaled_fp8_quant_with_padding(dtype) -> None: + original_rows = 5 + x = (torch.randn(size=(original_rows, 16), device="cuda") * 13).to(dtype) + + padding_size = 10 + + # Test with dynamic quantization + y_dynamic, scale_dynamic = scaled_fp8_quant( + x, None, num_token_padding=padding_size + ) + + # Verify output shape has the padded size + assert y_dynamic.shape[0] == padding_size + assert y_dynamic.shape[1] == x.shape[1] + + # Verify that the actual data in the non-padded region is correctly quantized + y_without_padding, scale_without_padding = scaled_fp8_quant(x, None) + torch.testing.assert_close(y_dynamic[:original_rows], y_without_padding) + + # Test with static quantization + # First get a scale + _, scale = scaled_fp8_quant(x, None) + + # Then use it for static quantization with padding + y_static, _ = scaled_fp8_quant(x, scale, num_token_padding=padding_size) + + # Verify output shape has the padded size + assert y_static.shape[0] == padding_size + assert y_static.shape[1] == x.shape[1] + + # Verify that the actual data in the non-padded region is correctly quantized + y_static_without_padding, _ = scaled_fp8_quant(x, scale) + torch.testing.assert_close(y_static[:original_rows], y_static_without_padding) + + # Test with per-token dynamic quantization + y_per_token, scale_per_token = scaled_fp8_quant( + x, None, num_token_padding=padding_size, use_per_token_if_dynamic=True + ) + + # Verify output shape has the padded size + assert y_per_token.shape[0] == padding_size + assert y_per_token.shape[1] == x.shape[1] + + # Verify that the actual data in the non-padded region is correctly quantized + y_per_token_without_padding, scale_per_token_without_padding = scaled_fp8_quant( + x, None, use_per_token_if_dynamic=True + ) + torch.testing.assert_close( + y_per_token[:original_rows], y_per_token_without_padding + ) + torch.testing.assert_close( + scale_per_token[:original_rows], scale_per_token_without_padding + ) + + +if __name__ == "__main__": + # Run the specific test function directly + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_cutlass_moe.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_cutlass_moe.py new file mode 100755 index 00000000..4e4eee37 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_cutlass_moe.py @@ -0,0 +1,306 @@ +import argparse + +import torch +import triton # Added import +import triton.testing # Added import +from transformers import AutoConfig + +from sglang.srt.layers.moe.cutlass_moe import cutlass_fused_experts_fp8 +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import fused_experts +from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig +from sglang.srt.layers.moe.topk import StandardTopKOutput + + +# Copy from: https://github.com/deepseek-ai/DeepGEMM/blob/main/deep_gemm/utils.py +def calc_diff(x, y): + x, y = x.double(), y.double() + denominator = (x * x + y * y).sum() + sim = 2 * (x * y).sum() / denominator + return 1 - sim + + +def get_model_config(tp_size: int): + config = AutoConfig.from_pretrained( + "deepseek-ai/Deepseek-R1", trust_remote_code=True + ) + E = config.n_routed_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + + return { + "num_experts": E, + "topk": topk, + "hidden_size": config.hidden_size, + "shard_intermediate_size": shard_intermediate_size, + "dtype": config.dtype, + "block_shape": config.quantization_config["weight_block_size"], + } + + +def to_fp8(tensor: torch.Tensor) -> torch.Tensor: + """Converts tensor to FP8 E4M3, scaling values to fit the range.""" + finfo = torch.finfo(torch.float8_e4m3fn) + # Calculate max absolute value safely + max_val = torch.max(torch.abs(tensor)) + # Avoid division by zero if tensor is all zeros + if max_val == 0: + scale_factor = 1.0 + else: + # Scale factor to bring the max value to finfo.max + scale_factor = finfo.max / max_val + + # Apply scaling + scaled_tensor = tensor * scale_factor + + # Clamp and convert + fp8_tensor = scaled_tensor.clamp(min=finfo.min, max=finfo.max).to( + dtype=torch.float8_e4m3fn + ) + return fp8_tensor + + +def run_test(tp_size, batch_size, model_config, check=False): + print(f"\n--- Batch Size: {batch_size} ---") + torch.set_default_device("cuda") + torch.cuda.manual_seed_all(42) # For reproducible random numbers + + E = model_config["num_experts"] + topk = model_config["topk"] + H = model_config["hidden_size"] + I = model_config["shard_intermediate_size"] + block_shape = model_config["block_shape"] # Tuple (BLOCK_N, BLOCK_K) + dtype = model_config["dtype"] # e.g., torch.bfloat16 + + print( + f"Config: E={E}, topk={topk}, H={H}, I_shard={I}, dtype={dtype}, block_shape={block_shape}" + ) + + # --- Input Data --- + # Use bf16/fp16 for input activation based on model config + x = torch.randn((batch_size, H), device="cuda", dtype=dtype) + # --- Weights (Generate in higher precision, then convert to FP8) --- + # Generate weights suitable for FP8 conversion (e.g., scaled appropriately) + w1_hp = torch.randn((E, I, H), device="cuda", dtype=torch.float32) + w2_hp = torch.randn((E, H, I // 2), device="cuda", dtype=torch.float32) + + w1 = to_fp8(w1_hp) + w2 = to_fp8(w2_hp) + + # --- Scales for FP8 Weights --- + block_n, block_k = block_shape + # Calculate number of blocks needed + w1_blocks_dim1 = (I + block_n - 1) // block_n + w1_blocks_dim2 = (H + block_k - 1) // block_k + w2_blocks_dim1 = (H + block_n - 1) // block_n + w2_blocks_dim2 = (I // 2 + block_k - 1) // block_k + + # Scales are typically float32 or float16/bfloat16 + scale_dtype = torch.float32 # Or dtype if scales match model dtype + w1_scale = torch.full( + (E, w1_blocks_dim1, w1_blocks_dim2), 1, device="cuda", dtype=scale_dtype + ) # Avoid zero scales + w2_scale = torch.full( + (E, w2_blocks_dim1, w2_blocks_dim2), 1, device="cuda", dtype=scale_dtype + ) # Avoid zero scales + + # --- Routing Information --- + topk_weights = torch.softmax( + torch.rand(batch_size, topk, device="cuda", dtype=dtype), dim=-1 + ) + topk_ids = torch.randint(0, E, (batch_size, topk), dtype=torch.int32, device="cuda") + + a1_strides = torch.full((E,), H, dtype=torch.int64, device="cuda") + c1_strides = torch.full((E,), I, dtype=torch.int64, device="cuda") + a2_strides = torch.full((E,), I // 2, dtype=torch.int64, device="cuda") + c2_strides = torch.full((E,), H, dtype=torch.int64, device="cuda") + + workspace = torch.empty( + (7182 * 1024), device="cuda", dtype=torch.uint8 + ) # Allocate sufficient workspace + # Pointer arrays (often filled by the kernel or a prep step, but needed as args) + a_ptrs = torch.empty((E,), dtype=torch.int64, device="cuda") + b_ptrs = torch.empty((E,), dtype=torch.int64, device="cuda") + out_ptrs = torch.empty((E,), dtype=torch.int64, device="cuda") + a_scales_ptrs = torch.empty((E,), dtype=torch.int64, device="cuda") + b_scales_ptrs = torch.empty((E,), dtype=torch.int64, device="cuda") + expert_offsets = torch.empty((E + 1,), dtype=torch.int32, device="cuda") + problem_sizes1 = torch.empty((E, 3), dtype=torch.int32, device="cuda") + problem_sizes2 = torch.empty((E, 3), dtype=torch.int32, device="cuda") + + enable_es = (False, False) + if torch.cuda.get_device_name(torch.cuda.current_device()) == "NVIDIA H200": + enable_es = (False, True) + elif torch.cuda.get_device_name(torch.cuda.current_device()) == "NVIDIA H20": + enable_es = (True, True) + + # --- Lambdas for Benchmarking --- + cutlass_lambda = lambda: cutlass_fused_experts_fp8( + x, + w1.transpose(1, 2), # Transposed + w2.transpose(1, 2), # Transposed + w1_scale.transpose(1, 2), + w2_scale.transpose(1, 2), + topk_weights, + topk_ids, + a1_strides, + c1_strides, + a2_strides, + c2_strides, + workspace, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + expert_offsets, + problem_sizes1, + problem_sizes2, + enable_es=enable_es, + ) + + topk_output = StandardTopKOutput( + topk_weights=topk_weights, + topk_ids=topk_ids, + router_logits=torch.randn( + (batch_size, topk), device=topk_weights.device, dtype=dtype + ), + ) + + moe_runner_config = MoeRunnerConfig( + num_experts=E, + top_k=topk, + hidden_size=H, + intermediate_size_per_partition=I, + params_dtype=dtype, + activation="silu", + inplace=False, + ) + + # Note: Triton expects non-transposed weights + triton_lambda = lambda: fused_experts( + x, + w1, + w2, + topk_output, + moe_runner_config, + use_fp8_w8a8=True, + w1_scale=w1_scale, + w2_scale=w2_scale, + block_shape=block_shape, + ) + + # --- Warmup --- + print("Warming up...") + for _ in range(10): + _ = cutlass_lambda() + _ = triton_lambda() + torch.cuda.synchronize() + + # --- Benchmarking --- + quantiles = [0.5, 0.2, 0.8] + print(f"Benchmarking Cutlass fused_experts...") + cutlass_ms, cutlass_min, cutlass_max = triton.testing.do_bench_cudagraph( + cutlass_lambda, rep=1000, quantiles=quantiles + ) + + print(f"Benchmarking Triton fused_experts...") + triton_ms, triton_min, triton_max = triton.testing.do_bench_cudagraph( + triton_lambda, rep=1000, quantiles=quantiles + ) + print( + f"Cutlass fused_experts time: {cutlass_ms:.3f} ms (median) [{cutlass_min:.3f} - {cutlass_max:.3f}]" + ) + print( + f"Triton fused_experts time: {triton_ms:.3f} ms (median) [{triton_min:.3f} - {triton_max:.3f}]" + ) + + # --- Correctness Check --- + if check: + print("Running correctness check...") + with torch.no_grad(): + # Run CUTLASS version (requires transposed weights) + y_cutlass = cutlass_fused_experts_fp8( + x, + w1.transpose(1, 2), # Transposed + w2.transpose(1, 2), # Transposed + w1_scale.transpose(1, 2), + w2_scale.transpose(1, 2), + topk_weights, + topk_ids, + a1_strides, + c1_strides, + a2_strides, + c2_strides, + workspace, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + expert_offsets, + problem_sizes1, + problem_sizes2, + enable_es=enable_es, + ) + + # Run Triton version (requires original shape weights, use inplace=False) + y_triton = fused_experts( + x, + w1, # Original shape + w2, # Original shape + topk_output, + moe_runner_config, + use_fp8_w8a8=True, + w1_scale=w1_scale, + w2_scale=w2_scale, + block_shape=block_shape, + ) + + diff = calc_diff(y_cutlass, y_triton) + print(f"Diff: {diff:.6f}") + + # Tolerance might need adjustment based on FP8 specifics and kernel differences + # FP8 comparisons often require higher tolerance than FP16/BF16 + assert diff < 1e-4, f"Diff too high! {diff}" + print("Correctness check passed.") + + +def main(tp_size=8, batch_sizes=[1, 4, 8, 16, 32, 64, 128, 256, 512], check=False): + model_config = get_model_config(tp_size) + print("Model Config:", model_config) + for batch_size in batch_sizes: + run_test(tp_size, batch_size, model_config, check) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--tp-size", type=int, default=8, help="Tensor Parallel size") + parser.add_argument( + "--batch-sizes", + type=int, + nargs="+", + default=[ + 1, + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512, + 1024, + 2048, + 4096, + 8192, + ], # Adjusted default + help="List of batch sizes to test", + ) + parser.add_argument("--check", action="store_true", help="Enable check mode") + args = parser.parse_args() + + print(f"Running benchmarks with TP size: {args.tp_size}") + print(f"Testing batch sizes: {args.batch_sizes}") + + main(tp_size=args.tp_size, batch_sizes=args.batch_sizes, check=args.check) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_cutlass_w16a16_moe.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_cutlass_w16a16_moe.py new file mode 100644 index 00000000..f3dd0772 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_cutlass_w16a16_moe.py @@ -0,0 +1,118 @@ +# SPDX-License-Identifier: Apache-2.0 +import pytest +import torch +from flashinfer.fused_moe import cutlass_fused_moe as flashinfer_cutlass_fused_moe + +from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.moe.topk import TopKConfig, select_experts +from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler + +MNK_FACTORS = [ + (2, 1024, 1024), + (2, 1024, 1536), + (2, 3072, 1024), + (2, 3072, 1536), + (64, 1024, 1024), + (64, 1024, 1536), + (64, 3072, 1024), + (64, 2048, 1024), + (224, 1024, 1024), + (224, 1024, 1536), +] + + +# Reference implementation of torch_moe for unquantized weights +def torch_moe_reference(a, w13, w2, score, topk): + B, D = a.shape + + set_global_server_args_for_scheduler(ServerArgs(model_path="dummy")) + + # Flip w13 layout + dim = -2 + size = w13.size(dim) + assert size % 2 == 0, f"Expected even size in dim {dim}, got {size}" + half = size // 2 + # Reorder weight + w1, w3 = w13.split(half, dim=dim) + w13 = torch.cat([w3, w1], dim=dim).contiguous() + + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + + for i in range(w13.shape[0]): + mask = topk_ids == i + if mask.sum(): + out[mask] = SiluAndMul()(a[mask] @ w13[i].transpose(0, 1)) @ w2[ + i + ].transpose(0, 1) + + return ( + out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype) + ).sum(dim=1) + + +@pytest.mark.parametrize("m,n,k", MNK_FACTORS) +@pytest.mark.parametrize("e", [40, 64, 256]) +@pytest.mark.parametrize("topk", [1, 6, 8]) +@torch.inference_mode() +def test_flashinfer_bf16_cutlass_moe(m: int, n: int, k: int, e: int, topk: int): + """ + Test the bf16 cutlass moe API. + + Args: + m: number of tokens + n: intermediate size + k: hidden size + e: number of experts + topk: top-k experts per token + """ + torch.manual_seed(7) + + dtype = torch.bfloat16 + + # Create unquantized weights + a = torch.randn((m, k), device="cuda", dtype=dtype) / 10 + + # w13: fused gate_up projection [num_experts, 2*intermediate, hidden] + # FlashInfer CUTLASS expects [up, gate] layout + w13 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10 + + # w2: down projection [num_experts, hidden, intermediate] + w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10 + + # Generate router scores + score = torch.randn((m, e), device="cuda", dtype=dtype) + + # Get topk routing + topk_output = select_experts( + hidden_states=a, + router_logits=score, + topk_config=TopKConfig(top_k=topk, renormalize=False), + ) + topk_weights, topk_ids, _ = topk_output + + # Test: Call FlashInfer CUTLASS fused_moe (unquantized version) + test_output = flashinfer_cutlass_fused_moe( + input=a, + token_selected_experts=topk_ids, + token_final_scales=topk_weights, + fc1_expert_weights=w13, + fc2_expert_weights=w2, + output_dtype=dtype, + quant_scales=None, + )[0] + + # Reference: Torch implementation + torch_output = torch_moe_reference(a, w13, w2, score, topk) + + # Compare outputs + torch.testing.assert_close(torch_output, test_output, rtol=1e-2, atol=1e-2) + + +if __name__ == "__main__": + # Run a simple test case + test_flashinfer_bf16_cutlass_moe(224, 1024, 1024, 8, 2) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_cutlass_w4a8_moe.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_cutlass_w4a8_moe.py new file mode 100644 index 00000000..e75154ef --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_cutlass_w4a8_moe.py @@ -0,0 +1,285 @@ +# SPDX-License-Identifier: Apache-2.0 + +from typing import Optional + +import pytest +import torch + +from sglang.srt.layers.moe.cutlass_w4a8_moe import cutlass_w4a8_moe +from sglang.srt.layers.moe.topk import TopKConfig, select_experts + + +def pack_int4_values_to_int8(int4_values_interleaved: torch.Tensor) -> torch.Tensor: + if int4_values_interleaved.shape[-1] % 2 != 0: + raise ValueError( + "the last dim size of int4_values_interleaved tensor must be even." + ) + + input_tensor_int8 = int4_values_interleaved.to(torch.int8) + + low_nibbles = input_tensor_int8[..., 0::2] + high_nibbles = input_tensor_int8[..., 1::2] + + packed_tensor = (high_nibbles << 4) | (low_nibbles & 0x0F) + + return packed_tensor.to(torch.int8) + + +def pack_interleave(num_experts, ref_weight, ref_scale, alignment=4): + n, k = ref_weight.shape[1], ref_weight.shape[2] + + weight = pack_int4_values_to_int8(ref_weight.cpu()).cuda() + w_q = weight.view((num_experts, n, k // 2)).view(torch.int8) + w_q = w_q.contiguous() + + scale_interleaved = ref_scale.reshape( + ref_scale.shape[0], + ref_scale.shape[1], + (ref_scale.shape[2] // alignment), + alignment, + ) # [E, N, K/4, 4] + scale_interleaved = scale_interleaved.permute(0, 2, 1, 3) # [E, K/4, N, 4] + scale_interleaved = scale_interleaved.reshape( + ref_scale.shape[0], + ref_scale.shape[2] // alignment, + ref_scale.shape[1] * alignment, + ) # [E, K/4, N*4] + w_scale = scale_interleaved.contiguous() + + return w_q, w_scale + + +@pytest.mark.parametrize("M", [1, 2, 4, 8, 16]) +@pytest.mark.parametrize("N", [2048]) +@pytest.mark.parametrize("K", [7168]) +@pytest.mark.parametrize("E", [256]) +@pytest.mark.parametrize("tp_size", [8]) +@pytest.mark.parametrize("use_ep_moe", [True, False]) +@pytest.mark.parametrize("topk", [8]) +@pytest.mark.parametrize("group_size", [128]) +@pytest.mark.parametrize("dtype", [torch.bfloat16]) +def test_cutlass_w4a8_moe(M, N, K, E, tp_size, use_ep_moe, topk, group_size, dtype): + if use_ep_moe: + local_e = E // tp_size + else: # tp mode + local_e = E + N = N // tp_size + + debug = False + if debug: + a = torch.ones((M, K), dtype=dtype, device="cuda") * 0.001 + ref_weight_1 = torch.ones((local_e, N * 2, K), dtype=torch.int8, device="cuda") + ref_weight_2 = torch.ones((local_e, K, N), dtype=torch.int8, device="cuda") + a1_scale = torch.ones(1, dtype=torch.float32, device="cuda") + a2_scale = torch.ones(1, dtype=torch.float32, device="cuda") + scale_1 = torch.ones( + (local_e, N * 2, K // group_size), dtype=dtype, device="cuda" + ) + scale_2 = torch.ones((local_e, K, N // group_size), dtype=dtype, device="cuda") + else: + a = torch.randn(M, K, dtype=dtype, device="cuda") + ref_weight_1 = torch.randint( + -8, 8, (local_e, N * 2, K), dtype=torch.int8, device="cuda" + ) + ref_weight_2 = torch.randint( + -8, 8, (local_e, K, N), dtype=torch.int8, device="cuda" + ) + affine_coeff = 0.005 + a1_scale = torch.randn(1, dtype=torch.float32, device="cuda") + a2_scale = torch.randn(1, dtype=torch.float32, device="cuda") + scale_1 = ( + torch.randn(local_e, N * 2, K // group_size, dtype=dtype, device="cuda") + * affine_coeff + ) + scale_2 = ( + torch.randn(local_e, K, N // group_size, dtype=dtype, device="cuda") + * affine_coeff + ) + + w1_q, w1_scale = pack_interleave(local_e, ref_weight_1, scale_1) + if use_ep_moe: + w2_q, w2_scale = pack_interleave(local_e, ref_weight_2, scale_2) + else: + w2_q, w2_scale = pack_interleave(local_e, ref_weight_2, scale_2, 1) + + device = "cuda" + a_strides1 = torch.full((local_e, 3), K, device=device, dtype=torch.int64) + c_strides1 = torch.full((local_e, 3), 2 * N, device=device, dtype=torch.int64) + a_strides2 = torch.full((local_e, 3), N, device=device, dtype=torch.int64) + c_strides2 = torch.full((local_e, 3), K, device=device, dtype=torch.int64) + b_strides1 = a_strides1 + s_strides13 = c_strides1 + b_strides2 = a_strides2 + s_strides2 = c_strides2 + + score = torch.randn((M, E), dtype=dtype, device=device) + topk_output = select_experts( + hidden_states=a, + router_logits=score, + topk_config=TopKConfig(top_k=topk, renormalize=False), + ) + topk_weights, topk_ids, _ = topk_output + expert_map = torch.arange(E, dtype=torch.int32, device=device) + expert_map[local_e:] = -1 + + output = cutlass_moe( + a, + w1_q, + w2_q, + w1_scale, + w2_scale, + topk_weights, + topk_ids, + a_strides1, + b_strides1, + c_strides1, + a_strides2, + b_strides2, + c_strides2, + s_strides13, + s_strides2, + local_e, + a1_scale, + a2_scale, + expert_map, + ) + + ref_output = ref( + a, + local_e, + topk_weights, + topk_ids, + ref_weight_1, + ref_weight_2, + scale_1, + scale_2, + has_pre_quant=True, + has_alpha=True, + pre_quant_scale_1=a1_scale, + pre_quant_scale_2=a2_scale, + alpha_1=a1_scale, + alpha_2=a2_scale, + ) + + # compare + torch.cuda.synchronize() + + # compare final output + torch.testing.assert_close(output, ref_output, rtol=1e-2, atol=0.1) + print("SUCCESS: Final output tensors are close.") + + +def cutlass_moe( + a: torch.Tensor, + w1_q: torch.Tensor, + w2_q: torch.Tensor, + w1_scale: torch.Tensor, + w2_scale: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + a_strides1: torch.Tensor, + b_strides1: torch.Tensor, + c_strides1: torch.Tensor, + a_strides2: torch.Tensor, + b_strides2: torch.Tensor, + c_strides2: torch.Tensor, + s_strides13: torch.Tensor, + s_strides2: torch.Tensor, + num_local_experts: int, + a1_scale: Optional[torch.Tensor] = None, + a2_scale: Optional[torch.Tensor] = None, + expert_map: Optional[torch.Tensor] = None, + apply_router_weight_on_input: bool = False, +): + topk_ids = expert_map[topk_ids] + device = a.device + + expert_offsets = torch.empty( + (num_local_experts + 1), dtype=torch.int32, device=device + ) + problem_sizes1 = torch.empty( + (num_local_experts, 3), dtype=torch.int32, device=device + ) + problem_sizes2 = torch.empty( + (num_local_experts, 3), dtype=torch.int32, device=device + ) + return cutlass_w4a8_moe( + a, + w1_q, + w2_q, + w1_scale, + w2_scale, + topk_weights, + topk_ids, + a_strides1, + b_strides1, + c_strides1, + a_strides2, + b_strides2, + c_strides2, + s_strides13, + s_strides2, + expert_offsets, + problem_sizes1, + problem_sizes2, + a1_scale, + a2_scale, + apply_router_weight_on_input, + ) + + +def ref( + x: torch.Tensor, + num_experts: int, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + ref_weight_1: torch.Tensor, + ref_weight_2: torch.Tensor, + ref_weight_scale_1: torch.Tensor, + ref_weight_scale_2: torch.Tensor, + has_pre_quant: bool = False, + has_alpha: bool = False, + pre_quant_scale_1: Optional[torch.Tensor] = None, + pre_quant_scale_2: Optional[torch.Tensor] = None, + alpha_1: Optional[torch.Tensor] = None, + alpha_2: Optional[torch.Tensor] = None, +): + results = torch.zeros_like(x) + dtype = x.dtype + for e_idx in range(num_experts): + mask = topk_ids == e_idx + activated_tokens = mask.sum(1).bool() + act = x[activated_tokens, :] + if act.shape[0] == 0: + continue + final_scale = (topk_weights * mask).sum(1)[activated_tokens].unsqueeze(1) + + act = ( + torch.clamp((act / pre_quant_scale_1.float()), -448.0, 448.0) + .to(torch.float8_e4m3fn) + .to(dtype) + ) + w3_w1 = ref_weight_1[e_idx] + ref_w_scale_repeat = ( + ref_weight_scale_1[e_idx].repeat_interleave(128, dim=1).to(float) + ) + w3_w1 = (w3_w1.to(float) * ref_w_scale_repeat).to(dtype) + fc1 = ((torch.matmul(act, w3_w1.T)) * alpha_1).to(torch.float16) + + gate, fc1 = fc1.chunk(2, dim=-1) + fc1 = fc1 * torch.nn.functional.silu(gate) + act = torch.clamp((fc1 / pre_quant_scale_2.float()), -448.0, 448.0).to( + torch.float8_e4m3fn + ) + act = act.to(dtype) + + w2 = ref_weight_2[e_idx] + ref_w_scale_repeat = ( + ref_weight_scale_2[e_idx].repeat_interleave(128, dim=1).to(float) + ) + w2 = (w2.to(float) * ref_w_scale_repeat).to(dtype) + fc2 = (torch.matmul(act, w2.T) * alpha_2).to(torch.float16) + + results[activated_tokens, :] += (fc2 * final_scale).to(results.dtype) + + return results diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_deepep_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_deepep_utils.py new file mode 100644 index 00000000..aa15b5a0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_deepep_utils.py @@ -0,0 +1,219 @@ +# Copy from deepseek-ai/DeepEP/tests/test_utils.py + +import os +import sys +from typing import Optional + +import numpy as np +import torch +import torch.distributed as dist + + +def init_dist(local_rank: int, num_local_ranks: int): + # NOTES: you may rewrite this function with your own cluster settings + ip = os.getenv("MASTER_ADDR", "127.0.0.1") + port = int(os.getenv("MASTER_PORT", "8361")) + num_nodes = int(os.getenv("WORLD_SIZE", 1)) + node_rank = int(os.getenv("RANK", 0)) + assert (num_local_ranks < 8 and num_nodes == 1) or num_local_ranks == 8 + + dist.init_process_group( + backend="nccl", + init_method=f"tcp://{ip}:{port}", + world_size=num_nodes * num_local_ranks, + rank=node_rank * num_local_ranks + local_rank, + ) + torch.set_default_dtype(torch.bfloat16) + torch.set_default_device("cuda") + torch.cuda.set_device(local_rank) + + return ( + dist.get_rank(), + dist.get_world_size(), + dist.new_group(list(range(num_local_ranks * num_nodes))), + ) + + +def calc_diff(x: torch.Tensor, y: torch.Tensor): + x, y = x.double() + 1, y.double() + 1 + denominator = (x * x + y * y).sum() + sim = 2 * (x * y).sum() / denominator + return (1 - sim).item() + + +def per_token_cast_to_fp8(x: torch.Tensor): + assert x.dim() == 2 and x.size(1) % 128 == 0 + m, n = x.shape + x_view = x.view(m, -1, 128) + x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4) + return (x_view * (448.0 / x_amax.unsqueeze(2))).to(torch.float8_e4m3fn).view( + m, n + ), (x_amax / 448.0).view(m, -1) + + +def per_token_cast_back(x_fp8: torch.Tensor, x_scales: torch.Tensor): + x_fp32 = x_fp8.to(torch.float32).view(x_fp8.size(0), -1, 128) + x_scales = x_scales.view(x_fp8.size(0), -1, 1) + return (x_fp32 * x_scales).view(x_fp8.shape).to(torch.bfloat16) + + +def inplace_unique(x: torch.Tensor, num_slots: int): + assert x.dim() == 2 + mask = x < 0 + x_padded = x.masked_fill(mask, num_slots) + bin_count = torch.zeros((x.size(0), num_slots + 1), dtype=x.dtype, device=x.device) + bin_count.scatter_add_(1, x_padded, torch.ones_like(x_padded)) + bin_count = bin_count[:, :num_slots] + sorted_bin_count, sorted_bin_idx = torch.sort(bin_count, dim=-1, descending=True) + sorted_bin_idx.masked_fill_(sorted_bin_count == 0, -1) + sorted_bin_idx = torch.sort(sorted_bin_idx, descending=True, dim=-1).values + x[:, :].fill_(-1) + valid_len = min(num_slots, x.size(1)) + x[:, :valid_len] = sorted_bin_idx[:, :valid_len] + + +def create_grouped_scores( + scores: torch.Tensor, group_idx: torch.Tensor, num_groups: int +): + num_tokens, num_experts = scores.shape + scores = scores.view(num_tokens, num_groups, -1) + mask = torch.zeros((num_tokens, num_groups), dtype=torch.bool, device=scores.device) + mask = mask.scatter_(1, group_idx, True).unsqueeze(-1).expand_as(scores) + return (scores * mask).view(num_tokens, num_experts) + + +def bench(fn, num_warmups: int = 20, num_tests: int = 30, post_fn=None): + # Flush L2 cache with 256 MB data + torch.cuda.synchronize() + cache = torch.empty(int(256e6 // 4), dtype=torch.int, device="cuda") + + # Warmup + for _ in range(num_warmups): + fn() + + # Flush L2 + cache.zero_() + + # Testing + start_events = [torch.cuda.Event(enable_timing=True) for _ in range(num_tests)] + end_events = [torch.cuda.Event(enable_timing=True) for _ in range(num_tests)] + for i in range(num_tests): + # Record + start_events[i].record() + fn() + end_events[i].record() + if post_fn is not None: + post_fn() + torch.cuda.synchronize() + + times = np.array( + [s.elapsed_time(e) / 1e3 for s, e in zip(start_events, end_events)] + )[1:] + return np.average(times), np.min(times), np.max(times) + + +class empty_suppress: + def __enter__(self): + return self + + def __exit__(self, *_): + pass + + +class suppress_stdout_stderr: + def __enter__(self): + self.outnull_file = open(os.devnull, "w") + self.errnull_file = open(os.devnull, "w") + + self.old_stdout_fileno_undup = sys.stdout.fileno() + self.old_stderr_fileno_undup = sys.stderr.fileno() + + self.old_stdout_fileno = os.dup(sys.stdout.fileno()) + self.old_stderr_fileno = os.dup(sys.stderr.fileno()) + + self.old_stdout = sys.stdout + self.old_stderr = sys.stderr + + os.dup2(self.outnull_file.fileno(), self.old_stdout_fileno_undup) + os.dup2(self.errnull_file.fileno(), self.old_stderr_fileno_undup) + + sys.stdout = self.outnull_file + sys.stderr = self.errnull_file + return self + + def __exit__(self, *_): + sys.stdout = self.old_stdout + sys.stderr = self.old_stderr + + os.dup2(self.old_stdout_fileno, self.old_stdout_fileno_undup) + os.dup2(self.old_stderr_fileno, self.old_stderr_fileno_undup) + + os.close(self.old_stdout_fileno) + os.close(self.old_stderr_fileno) + + self.outnull_file.close() + self.errnull_file.close() + + +def bench_kineto( + fn, + kernel_names, + num_tests: int = 30, + suppress_kineto_output: bool = False, + trace_path: Optional[str] = None, + barrier_comm_profiling: bool = False, +): + # Profile + suppress = suppress_stdout_stderr if suppress_kineto_output else empty_suppress + with suppress(): + schedule = torch.profiler.schedule(wait=0, warmup=1, active=1, repeat=1) + with torch.profiler.profile( + activities=[torch.profiler.ProfilerActivity.CUDA], schedule=schedule + ) as prof: + for i in range(2): + # NOTES: use a large kernel and a barrier to eliminate the unbalanced CPU launch overhead + if barrier_comm_profiling: + lhs = torch.randn((8192, 8192), dtype=torch.float, device="cuda") + rhs = torch.randn((8192, 8192), dtype=torch.float, device="cuda") + lhs @ rhs + dist.all_reduce(torch.ones(1, dtype=torch.float, device="cuda")) + for _ in range(num_tests): + fn() + prof.step() + + # Parse the profiling table + assert isinstance(kernel_names, str) or isinstance(kernel_names, tuple) + is_tupled = isinstance(kernel_names, tuple) + prof_lines = ( + prof.key_averages() + .table(sort_by="cuda_time_total", max_name_column_width=100) + .split("\n") + ) + kernel_names = (kernel_names,) if isinstance(kernel_names, str) else kernel_names + assert all([isinstance(name, str) for name in kernel_names]) + for name in kernel_names: + assert ( + sum([name in line for line in prof_lines]) == 1 + ), f"Errors of the kernel {name} in the profiling table" + + # Save chrome traces + if trace_path is not None: + prof.export_chrome_trace(trace_path) + + # Return average kernel times + units = {"ms": 1e3, "us": 1e6} + kernel_times = [] + for name in kernel_names: + for line in prof_lines: + if name in line: + time_str = line.split()[-2] + for unit, scale in units.items(): + if unit in time_str: + kernel_times.append(float(time_str.replace(unit, "")) / scale) + break + break + return tuple(kernel_times) if is_tupled else kernel_times[0] + + +def hash_tensor(t: torch.Tensor): + return t.view(torch.int64).sum().item() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_deterministic.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_deterministic.py new file mode 100644 index 00000000..78bb9f5d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_deterministic.py @@ -0,0 +1,690 @@ +""" +Batch the same prompt in random batch sizes, and test if the results are consistent across different trials. + +Usage: +# Single mode: test determinism with varying batch sizes +python3 -m sglang.test.test_deterministic --n-trials 50 --test-mode single + +# Prefix mode: test with shared prefixes +python3 -m sglang.test.test_deterministic --n-start 1 --n-trials 50 --test-mode prefix + +# Radix Cache Consistency mode: test radix cache determinism (cached vs uncached prefill) +python3 -m sglang.test.test_deterministic --test-mode radix_cache +""" + +import argparse +import dataclasses +import json +import os +import random +from typing import Any, Dict, List, Optional + +import requests + +from sglang.profiler import run_profile + +PROMPT_1 = "Tell me about Richard Feynman: " +PROMPT_2 = "Generate 1000 random numbers. Go directly into it, don't say Sure and don't say here are numbers. Just start with a number." +dirpath = os.path.dirname(__file__) +with open(os.path.join(dirpath, "long_prompt.txt"), "r") as f: + LONG_PROMPT = f.read() + + +@dataclasses.dataclass +class BenchArgs: + host: str = "localhost" + port: int = 30000 + batch_size: int = 1 + temperature: float = 0.0 + sampling_seed: int = 42 + max_new_tokens: int = 100 + frequency_penalty: float = 0.0 + presence_penalty: float = 0.0 + return_logprob: bool = False + stream: bool = False + profile: bool = False + profile_steps: int = 3 + profile_by_stage: bool = False + test_mode: str = "single" + n_trials: int = 50 + n_start: int = 1 + + @staticmethod + def add_cli_args(parser: argparse.ArgumentParser): + parser.add_argument("--host", type=str, default=BenchArgs.host) + parser.add_argument("--port", type=int, default=BenchArgs.port) + parser.add_argument("--n-trials", type=int, default=BenchArgs.n_trials) + parser.add_argument("--n-start", type=int, default=BenchArgs.n_start) + parser.add_argument("--temperature", type=float, default=BenchArgs.temperature) + parser.add_argument( + "--sampling-seed", type=int, default=BenchArgs.sampling_seed + ) + parser.add_argument( + "--max-new-tokens", type=int, default=BenchArgs.max_new_tokens + ) + parser.add_argument( + "--frequency-penalty", type=float, default=BenchArgs.frequency_penalty + ) + parser.add_argument( + "--presence-penalty", type=float, default=BenchArgs.presence_penalty + ) + parser.add_argument("--return-logprob", action="store_true") + parser.add_argument("--stream", action="store_true") + parser.add_argument( + "--test-mode", + type=str, + default=BenchArgs.test_mode, + choices=[ + "single", + "prefix", + "radix_cache", + "p_vs_d", + ], + ) + parser.add_argument("--profile", action="store_true") + parser.add_argument( + "--profile-steps", type=int, default=BenchArgs.profile_steps + ) + parser.add_argument("--profile-by-stage", action="store_true") + + @classmethod + def from_cli_args(cls, args: argparse.Namespace): + attrs = [attr.name for attr in dataclasses.fields(cls)] + return cls(**{attr: getattr(args, attr) for attr in attrs}) + + +def send_single( + args, + profile: bool = False, + profile_steps: int = 3, + profile_by_stage: bool = False, + return_full_response: bool = False, + input_ids: List[int] = None, + prompt: List[str] = None, + max_new_tokens: int = None, + extra_params: Optional[Dict[str, Any]] = None, + pick_first_result: bool = True, +): + base_url = f"http://{args.host}:{args.port}" + + # Use input_ids if provided, otherwise use text prompts + if input_ids is not None: + assert prompt is None + json_data = { + "input_ids": input_ids, + "sampling_params": { + "temperature": args.temperature, + "max_new_tokens": ( + max_new_tokens + if max_new_tokens is not None + else args.max_new_tokens + ), + "frequency_penalty": args.frequency_penalty, + "presence_penalty": args.presence_penalty, + }, + "return_logprob": args.return_logprob, + "stream": args.stream, + **(extra_params or {}), + } + else: + assert input_ids is None + json_data = { + "text": prompt, + "sampling_params": { + "temperature": args.temperature, + "max_new_tokens": ( + max_new_tokens + if max_new_tokens is not None + else args.max_new_tokens + ), + "frequency_penalty": args.frequency_penalty, + "presence_penalty": args.presence_penalty, + }, + "return_logprob": args.return_logprob, + "stream": args.stream, + **(extra_params or {}), + } + + if args.sampling_seed is not None: + # sglang server cannot parse None value for sampling_seed + json_data["sampling_params"]["sampling_seed"] = args.sampling_seed + + if profile: + run_profile( + url=base_url, + num_steps=profile_steps, + activities=["CPU", "GPU"], + profile_by_stage=profile_by_stage, + ) + + response = requests.post( + f"{base_url}/generate", + json=json_data, + stream=args.stream, + ) + + if response.status_code != 200: + ret = response.json() + print(f"Error: {ret}") + return None + + if args.stream: + for chunk in response.iter_lines(decode_unicode=False): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]": + break + ret = json.loads(chunk[5:].strip("\n")) + else: + ret = response.json() + + if pick_first_result: + ret = ret[0] if isinstance(ret, list) else ret + + if return_full_response: + return ret + else: + return ret["text"] + + +def send_prefix( + args, batch_size: int, prompts: List[str], return_full_response: bool = False +): + requests.post(f"http://{args.host}:{args.port}/flush_cache") + + batch_data = [] + sampled_indices = [] + for _ in range(batch_size): + sampled_index = random.randint(0, len(prompts) - 1) + sampled_indices.append(sampled_index) + batch_data.append(prompts[sampled_index]) + + json_data = { + "text": batch_data, + "sampling_params": { + "temperature": args.temperature, + "max_new_tokens": args.max_new_tokens, + "frequency_penalty": args.frequency_penalty, + "presence_penalty": args.presence_penalty, + }, + "return_logprob": args.return_logprob, + "stream": args.stream, + } + + if args.sampling_seed is not None: + json_data["sampling_params"]["sampling_seed"] = args.sampling_seed + + response = requests.post( + f"http://{args.host}:{args.port}/generate", + json=json_data, + stream=args.stream, + ) + ret = response.json() + if response.status_code != 200: + print(ret) + return -1, -1, -1 + + if return_full_response: + # Return full responses grouped by prompt index + ret_dict = {i: [] for i in range(len(prompts))} + for i in range(batch_size): + ret_dict[sampled_indices[i]].append(ret[i]) + return ret_dict + else: + # Return only text grouped by prompt index + ret_dict = {i: [] for i in range(len(prompts))} + for i in range(batch_size): + ret_dict[sampled_indices[i]].append(ret[i]["text"]) + return ret_dict + + +def compare_logprobs(logprobs1, logprobs2, tolerance=0): + """Compare two logprobs sequences with a tolerance.""" + if len(logprobs1) != len(logprobs2): + return False, f"Length mismatch: {len(logprobs1)} vs {len(logprobs2)}" + + for i, (lp1, lp2) in enumerate(zip(logprobs1, logprobs2)): + # Each element is [logprob, token_id] + if lp1[1] != lp2[1]: + return False, f"Token ID mismatch at position {i}: {lp1[1]} vs {lp2[1]}" + if abs(lp1[0] - lp2[0]) > tolerance: + return ( + False, + f"Logprob mismatch at position {i}: {lp1[0]} vs {lp2[0]} (diff: {abs(lp1[0] - lp2[0])})", + ) + + return True, "Logprobs match" + + +def _test_mode_p_vs_d(args, batch_size): + print() + print(f"Execute: test p_vs_d {batch_size=}") + + random.seed(42) + args.return_logprob = True + query_extra_params = { + "logprob_start_len": 0, + "return_text_in_logprobs": True, + } + + def _create_prompts(): + ans = [PROMPT_1, PROMPT_2] + for i in range(batch_size - len(ans)): + end = random.randrange(1, 4096) + if random.random() < 0.5: + begin = 0 + else: + begin = random.randrange(0, end) + ans.append(LONG_PROMPT[begin:end]) + return ans[:batch_size] + + # warmup + flush + send_single(args, input_ids=[1] * 64, max_new_tokens=65, return_full_response=True) + requests.post(f"http://{args.host}:{args.port}/flush_cache") + + prompts = _create_prompts() + + resp_a = send_single( + args, + prompt=prompts, + max_new_tokens=args.max_new_tokens, + return_full_response=True, + pick_first_result=False, + extra_params=query_extra_params, + ) + info_a = _extract_ids_and_logprobs(resp_a) + + requests.post(f"http://{args.host}:{args.port}/flush_cache") + + resp_b = send_single( + args, + input_ids=[x["io"].token_ids for x in info_a], + max_new_tokens=1, + return_full_response=True, + pick_first_result=False, + extra_params=query_extra_params, + ) + info_b = _extract_ids_and_logprobs(resp_b) + + ans = [] + for i, (info_a_item, info_b_item) in enumerate(zip(info_a, info_b, strict=True)): + print(f"Compare sequence {i} in batch...") + correct = TokenIdsAndLogprobs.compare(info_a_item["io"], info_b_item["input"]) + ans.append(int(correct)) + + return ans + + +@dataclasses.dataclass +class TokenIdsAndLogprobs: + token_ids: List[int] + logprobs: List[float] + + def __add__(self, other): + return TokenIdsAndLogprobs( + token_ids=self.token_ids + other.token_ids, + logprobs=self.logprobs + other.logprobs, + ) + + @classmethod + def compare(cls, a: "TokenIdsAndLogprobs", b: "TokenIdsAndLogprobs"): + import numpy as np + + assert len(a.token_ids) == len(b.token_ids) + token_match = a.token_ids == b.token_ids + logprobs_match = a.logprobs == b.logprobs + + if token_match: + print(f"✅ Token match") + else: + print(f"❌ Token mismatch: {a.token_ids=} {b.token_ids=}") + + if logprobs_match: + print(f"✅ Logprobs match:", a.logprobs[:5]) + else: + print(f"❌ Logprobs mismatch") + # Only print first 5 elements for readability + n_show = 5 + a_show = a.logprobs[:n_show] + b_show = b.logprobs[:n_show] + print( + " A: ", + [f"{x:.10f}" if x is not None else "None" for x in a_show], + f"... ({len(a.logprobs)} total)" if len(a.logprobs) > n_show else "", + ) + print( + " B: ", + [f"{x:.10f}" if x is not None else "None" for x in b_show], + f"... ({len(b.logprobs)} total)" if len(b.logprobs) > n_show else "", + ) + diff = [ + abs(x - y) if x is not None else float("nan") + for x, y in zip(a.logprobs, b.logprobs) + ] + print( + " Diff:", + [f"{x:.10e}" for x in diff[:n_show]], + f"... ({len(diff)} total)" if len(diff) > n_show else "", + ) + + # Compute KL-divergence using K3 approximation + # KL(P||Q) ≈ (exp(log(P) - log(Q)) - 1) - (log(P) - log(Q)) + # This is based on selected token logprobs only + valid_pairs = [ + (lp_a, lp_b) + for lp_a, lp_b in zip(a.logprobs, b.logprobs) + if lp_a is not None and lp_b is not None + ] + if valid_pairs and token_match: + logprobs_a = np.array([lp for lp, _ in valid_pairs]) + logprobs_b = np.array([lp for _, lp in valid_pairs]) + + # K3 approximation: KL(A||B) ≈ (exp(logr) - 1) - logr, where logr = log_a - log_b + logr = logprobs_a - logprobs_b + kl_per_token = (np.exp(logr) - 1) - logr + kl_mean = np.mean(kl_per_token) + kl_max = np.max(kl_per_token) + + print(f" KL(A||B) mean: {kl_mean:.10e}") + print(f" KL(A||B) max : {kl_max:.10e}") + print(f" Mean absolute logprob diff: {np.mean(np.abs(logr)):.10e}") + + return token_match and logprobs_match + + +def _extract_ids_and_logprobs(responses): + def _extract_part(response, name): + token_ids, logprobs = [], [] + for item in response["meta_info"][name]: + logprob, token_id, text = item + token_ids.append(token_id) + logprobs.append(logprob) + return TokenIdsAndLogprobs(token_ids=token_ids, logprobs=logprobs) + + def _extract_one_response(response): + input = _extract_part(response, "input_token_logprobs") + output = _extract_part(response, "output_token_logprobs") + return dict(input=input, output=output, io=input + output) + + if not isinstance(responses, list): + responses = [responses] + return [_extract_one_response(x) for x in responses] + + +def test_deterministic(args): + if args.test_mode == "single": + # In single mode, we test the deterministic behavior by sending the same prompt in batch sizes ranging from 1 to n_trials. + texts = [] + for i in range(1, args.n_trials + 1): + batch_size = i + text = send_single(args, args.profile, prompt=[PROMPT_1] * batch_size) + text = text.replace("\n", " ") + print(f"Trial {i} with batch size {batch_size}: {text}") + texts.append(text) + print(f"Total samples: {len(texts)}, Unique samples: {len(set(texts))}") + return [len(set(texts))] + + elif args.test_mode == "prefix": + # In prefix mode, we create prompts from the same long prompt, with different lengths of common prefix. + len_prefix = [1, 511, 2048, 4097] + num_prompts = len(len_prefix) + outputs = {i: [] for i in range(4)} + prompts = [LONG_PROMPT[: len_prefix[i]] for i in range(4)] + + # If return_logprob is enabled, store full responses for comparison + if args.return_logprob: + full_responses = {i: [] for i in range(4)} + + for i in range(args.n_start, args.n_start + args.n_trials): + batch_size = i + ret_dict = send_prefix( + args, batch_size, prompts, return_full_response=args.return_logprob + ) + msg = f"Testing Trial {i} with batch size {batch_size}," + for i in range(num_prompts): + msg += f" # prefix length {len_prefix[i]}: {len(ret_dict[i])}," + print(msg) + for i in range(num_prompts): + if args.return_logprob: + # Store full response for logprob comparison + full_responses[i].extend(ret_dict[i]) + # Extract text for determinism check + outputs[i].extend([resp["text"] for resp in ret_dict[i]]) + else: + outputs[i].extend(ret_dict[i]) + + for i in range(num_prompts): + print( + f"Prompt {i} with prefix length {len_prefix[i]}: total samples: {len(outputs[i])}, Unique samples: {len(set(outputs[i]))}" + ) + + results = [] + for i in range(num_prompts): + results.append(len(set(outputs[i]))) + + # If logprobs are enabled, compare them across different batch sizes + if args.return_logprob: + print(f"\n{'='*60}") + print("Logprobs Comparison Across Batch Sizes") + print("=" * 60) + + logprob_results = [] + for prompt_idx in range(num_prompts): + print( + f"\nPrompt {prompt_idx} (prefix length {len_prefix[prompt_idx]}):" + ) + responses = full_responses[prompt_idx] + + if len(responses) < 2: + continue + + # Compare all responses against the first one + reference = responses[0] + all_match = True + mismatches = [] + + for j, resp in enumerate(responses[1:], start=1): + ref_logprobs = reference["meta_info"]["output_token_logprobs"] + resp_logprobs = resp["meta_info"]["output_token_logprobs"] + + match, msg = compare_logprobs(ref_logprobs, resp_logprobs) + + if not match: + print(f" ✗ Sample {j+1}: {msg}") + mismatches.append((j + 1, msg)) + all_match = False + + if all_match: + print(f" ✓ All {len(responses)} samples have identical logprobs") + logprob_results.append(1) + else: + print( + f" ✗ Found {len(mismatches)} mismatches out of {len(responses)} samples" + ) + logprob_results.append(0) + + print(f"\n{'='*60}") + if all(r == 1 for r in logprob_results): + print("✓✓✓ Logprobs are identical across all batch sizes! ✓✓✓") + else: + print("✗✗✗ Some logprobs differ across batch sizes! ✗✗✗") + + return results + + elif args.test_mode == "radix_cache": + # Radix mode requires logprobs to compare results + args.return_logprob = True + + print("\n=== Prefill Cache Consistency Test ===") + print( + "This test verifies prefill request produces consistent logprobs w/ and w/o cache.\n" + ) + + # We noticed that we cannot call flush cache before any request, otherwise it will hang. + warmup_response = send_single( + args, input_ids=[1] * 64, max_new_tokens=65, return_full_response=True + ) + + # Flush cache first to make sure there is no cache hit from previous tests + flush_response = requests.post(f"http://{args.host}:{args.port}/flush_cache") + + print(f"Step 1: Generating random 64 token IDs...") + # Use a reasonable token ID range (e.g., 1-50000 for most tokenizers) + # Avoid special tokens like 0 (padding), 1 (BOS), 2 (EOS) + # set seed for random.randint + random.seed(42) + initial_token_ids = [random.randint(100, 50000) for _ in range(64)] + + print(f"✓ Using {len(initial_token_ids)} initial tokens") + print(f" Initial token IDs: {initial_token_ids}") + + print( + f"\nStep 2: Generating 2 tokens from {len(initial_token_ids)} token prefix..." + ) + first_response = send_single( + args, + input_ids=initial_token_ids, + max_new_tokens=100, + return_full_response=True, + ) + first_output_text = first_response["text"] + first_output_token_ids = first_response["output_ids"] + first_output_logprobs = first_response["meta_info"]["output_token_logprobs"] + + expected_token_id = first_output_token_ids[-1] + expected_logprob = first_output_logprobs[-1][0] + + print(f"✓ Generated {len(first_output_token_ids)} tokens") + print(f' Output text: "{first_output_text}"') + + print( + f"\nStep 3: Generating with radix cache (164 tokens prefill, should hit > 128 tokens cache, based on page size)..." + ) + prefix_token_ids = initial_token_ids + first_output_token_ids[:-1] + print( + f" Prefix: {len(initial_token_ids)} initial + 64 generated = {len(prefix_token_ids)} tokens" + ) + print(f"Using Prompt: {prefix_token_ids}") + cached_response = send_single( + args, + input_ids=prefix_token_ids, + max_new_tokens=1, + return_full_response=True, + ) + cached_logprobs = cached_response["meta_info"]["output_token_logprobs"] + cached_token_data = cached_logprobs[0] + cached_logprob = cached_token_data[0] + cached_token_id = cached_token_data[1] + + print(f"✓ Generated with cache:") + print(f" Token ID: {cached_token_id}") + print(f" Logprob: {cached_logprob:.10f}") + + print(f"\nStep 4: Flushing cache...") + flush_response = requests.post(f"http://{args.host}:{args.port}/flush_cache") + + print( + f"\nStep 5: Generating without cache (same 164 tokens prefill, no cache)..." + ) + print(f"Using Prompt: {prefix_token_ids}") + + uncached_response = send_single( + args, + input_ids=prefix_token_ids, + max_new_tokens=1, + return_full_response=True, + ) + + uncached_logprobs = uncached_response["meta_info"]["output_token_logprobs"] + uncached_token_data = uncached_logprobs[0] + uncached_logprob = uncached_token_data[0] + uncached_token_id = uncached_token_data[1] + + print(f"✓ Generated without cache:") + print(f" Token ID: {uncached_token_id}") + print(f" Logprob: {uncached_logprob:.10f}") + + # Step 6: Compare results + print(f"\n{'='*60}") + print("Comparison 1: Decode (Request 1) vs Prefill with Cache (Request 2)") + print("=" * 60) + + # Compare first request (decode) vs second request (prefill with cache) + # We expect them to be different (different kernels) + decode_vs_prefill_token_match = expected_token_id == cached_token_id + decode_vs_prefill_logprob_match = expected_logprob == cached_logprob + + print( + f" Decode token (Request 1): ID={expected_token_id}, logprob={expected_logprob:.10f}" + ) + print( + f" Prefill w/ cache token (Request 2): ID={cached_token_id}, logprob={cached_logprob:.10f}" + ) + print( + f" Token ID match: {'✓ YES' if decode_vs_prefill_token_match else '✗ NO'}" + ) + print( + f" Logprob match: {'✓ YES' if decode_vs_prefill_logprob_match else '✗ NO'}" + ) + if not decode_vs_prefill_logprob_match: + diff = abs(expected_logprob - cached_logprob) + print(f" Logprob difference: {diff:.10e}") + print(f" Note: We expect these to be DIFFERENT (decode vs prefill kernels)") + + print(f"\n{'='*60}") + print( + "Comparison 2: Cached Prefill (Request 2) vs Uncached Prefill (Request 3)" + ) + print("=" * 60) + + # Main test: compare cached vs uncached prefill (should be identical) + token_match = cached_token_id == uncached_token_id + logprob_match = cached_logprob == uncached_logprob + + print( + f" Cached prefill token (Request 2): ID={cached_token_id}, logprob={cached_logprob:.10f}" + ) + print( + f" Uncached prefill token (Request 3): ID={uncached_token_id}, logprob={uncached_logprob:.10f}" + ) + print(f" Token ID match: {'✓ YES' if token_match else '✗ NO'}") + if not token_match: + print(f" Cached: {cached_token_id}") + print(f" Uncached: {uncached_token_id}") + + print(f" Logprob match: {'✓ YES' if logprob_match else '✗ NO'}") + if not logprob_match: + print(f" Cached: {cached_logprob:.10f}") + print(f" Uncached: {uncached_logprob:.10f}") + diff = abs(cached_logprob - uncached_logprob) + print(f" Difference: {diff:.10e}") + print(f" Note: We expect these to be IDENTICAL (both prefill kernels)") + + print(f"\n{'='*60}") + if token_match and logprob_match: + print("✓✓✓ TEST PASSED - Radix cache is consistent! ✓✓✓") + return [1] + else: + print("✗✗✗ TEST FAILED - Radix cache produces different results! ✗✗✗") + return [0] + + elif args.test_mode == "p_vs_d": + # TODO also extract other modes to functions + ans = [] + for i in range(1, args.n_trials + 1): + ans += _test_mode_p_vs_d(args, batch_size=i) + return ans + + else: + raise ValueError(f"Invalid test mode: {args.test_mode}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + BenchArgs.add_cli_args(parser) + args = parser.parse_args() + + if args.sampling_seed is None: + args.sampling_seed = 42 + + test_deterministic(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_deterministic_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_deterministic_utils.py new file mode 100644 index 00000000..46c9d5ec --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_deterministic_utils.py @@ -0,0 +1,74 @@ +import unittest + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_deterministic import BenchArgs, test_deterministic +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +DEFAULT_MODEL = "Qwen/Qwen3-8B" +COMMON_SERVER_ARGS = [ + "--trust-remote-code", + "--cuda-graph-max-bs", + "32", + "--enable-deterministic-inference", +] + + +class TestDeterministicBase(CustomTestCase): + @classmethod + def get_server_args(cls): + return COMMON_SERVER_ARGS + + @classmethod + def get_model(cls): + return DEFAULT_MODEL + + @classmethod + def setUpClass(cls): + cls.model = cls.get_model() + cls.base_url = DEFAULT_URL_FOR_TEST + if "--attention-backend" not in cls.get_server_args(): + raise unittest.SkipTest("Skip the base test class") + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=cls.get_server_args(), + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def _extract_host_and_port(self, url): + return url.split("://")[-1].split(":")[0], int(url.split(":")[-1]) + + def test_single(self): + args = BenchArgs() + url = DEFAULT_URL_FOR_TEST + args.host, args.port = self._extract_host_and_port(url) + args.test_mode = "single" + args.n_start = 10 + args.n_trials = 20 + results = test_deterministic(args) + args.temperature = 0.5 # test for deterministic sampling + for result in results: + assert result == 1 + + def test_prefix_with_logprobs(self): + args = BenchArgs() + url = DEFAULT_URL_FOR_TEST + args.host, args.port = self._extract_host_and_port(url) + args.test_mode = "prefix" + args.n_start = 10 + args.n_trials = 10 + args.temperature = 0.5 # test for deterministic sampling + args.return_logprob = True # Enable logprobs comparison + results = test_deterministic(args) + for result in results: + assert result == 1 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_disaggregation_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_disaggregation_utils.py new file mode 100644 index 00000000..e4396170 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_disaggregation_utils.py @@ -0,0 +1,158 @@ +import logging +import os +import time +import warnings +from urllib.parse import urlparse + +import requests + +from sglang.srt.environ import envs +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_with_error_check, +) + +logger = logging.getLogger(__name__) + + +class TestDisaggregationBase(CustomTestCase): + @classmethod + def setUpClass(cls): + parsed_url = urlparse(DEFAULT_URL_FOR_TEST) + cls.base_host = parsed_url.hostname + base_port = str(parsed_url.port) + cls.lb_port = base_port + cls.prefill_port = f"{int(base_port) + 100}" + cls.decode_port = f"{int(base_port) + 200}" + cls.prefill_url = f"http://{cls.base_host}:{cls.prefill_port}" + cls.decode_url = f"http://{cls.base_host}:{cls.decode_port}" + cls.lb_url = f"http://{cls.base_host}:{cls.lb_port}" + print(f"{cls.base_host=} {cls.lb_port=} {cls.prefill_port=} {cls.decode_port=}") + cls.process_lb, cls.process_decode, cls.process_prefill = None, None, None + + # config transfer backend and rdma devices + if is_in_ci(): + cls.transfer_backend = ["--disaggregation-transfer-backend", "mooncake"] + cls.rdma_devices = ["--disaggregation-ib-device", get_rdma_devices_args()] + else: + cls.transfer_backend = [ + "--disaggregation-transfer-backend", + envs.SGLANG_TEST_PD_DISAGG_BACKEND.get(), + ] + cls.rdma_devices = [ + "--disaggregation-ib-device", + envs.SGLANG_TEST_PD_DISAGG_DEVICES.get(), + ] + if cls.rdma_devices[1] is None: + cls.rdma_devices = [] + msg = "No RDMA devices specified for disaggregation test, using default settings." + warnings.warn(msg) + + @classmethod + def launch_lb(cls): + lb_command = [ + "python3", + "-m", + "sglang_router.launch_router", + "--pd-disaggregation", + "--mini-lb", # FIXME: remove this + "--prefill", + cls.prefill_url, + "--decode", + cls.decode_url, + "--host", + cls.base_host, + "--port", + cls.lb_port, + ] + print("Starting load balancer:", " ".join(lb_command)) + cls.process_lb = popen_with_error_check(lb_command) + cls.wait_server_ready(cls.lb_url + "/health") + + @classmethod + def wait_server_ready(cls, url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH): + start_time = time.perf_counter() + while True: + try: + response = requests.get(url) + if response.status_code == 200: + print(f"Server {url} is ready") + return + except Exception: + pass + + if time.perf_counter() - start_time > timeout: + raise RuntimeError(f"Server {url} failed to start in {timeout}s") + time.sleep(1) + + @classmethod + def tearDownClass(cls): + for process in [cls.process_lb, cls.process_decode, cls.process_prefill]: + if process: + try: + kill_process_tree(process.pid) + except Exception as e: + print(f"Error killing process {process.pid}: {e}") + + # wait for 5 seconds + time.sleep(5) + + +def get_rdma_devices_args(): + def _parse_list_env(var_name: str): + val = os.getenv(var_name) + if not val: + return None + items = [x.strip() for x in val.split(",") if x.strip()] + return items or None + + def _pick_default_pair(rdma_all_devices): + return [rdma_all_devices[0], rdma_all_devices[len(rdma_all_devices) // 2]] + + rdma_all_devices = _parse_list_env("SGLANG_CI_RDMA_ALL_DEVICES") or [ + f"mlx5_roce{i}" for i in range(8) + ] + logger.info("Resolved rdma_all_devices=%s", rdma_all_devices) + + n_rdma = len(rdma_all_devices) + + # 1. Get visible GPU indices + cuda_visible_devices = os.getenv("CUDA_VISIBLE_DEVICES") + if not cuda_visible_devices: + warnings.warn("CUDA_VISIBLE_DEVICES is not set. Using default RDMA devices.") + return ",".join(_pick_default_pair(rdma_all_devices)) + + try: + # Convert to list of integers (handling possible spaces and empty strings) + gpu_indices = [ + int(idx.strip()) for idx in cuda_visible_devices.split(",") if idx.strip() + ] + if not gpu_indices or len(gpu_indices) > 4: + return ",".join(_pick_default_pair(rdma_all_devices)) + except ValueError: + warnings.warn(f"Invalid CUDA_VISIBLE_DEVICES format: {cuda_visible_devices}") + return ",".join(_pick_default_pair(rdma_all_devices)) + + # 2. Calculate base RDMA index group (each group of 4 GPUs uses consecutive devices) + base_rdma_group = (min(gpu_indices) // 4) * 4 + for gpu_idx in gpu_indices: + if not (base_rdma_group <= gpu_idx < base_rdma_group + 4): + warnings.warn( + f"GPU index {gpu_idx} is outside expected group " + f"{base_rdma_group}-{base_rdma_group+3}" + ) + + # 3. Generate RDMA device names + rdma_devices = [] + for gpu_idx in gpu_indices: + nic_index = gpu_idx // (8 // n_rdma) + rdma_devices.append(rdma_all_devices[nic_index]) + + if not rdma_devices: + return ",".join(_pick_default_pair(rdma_all_devices)) + + return ",".join(rdma_devices) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_dynamic_grad_mode.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_dynamic_grad_mode.py new file mode 100644 index 00000000..078c3139 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_dynamic_grad_mode.py @@ -0,0 +1,58 @@ +import unittest + +import torch + +from sglang.srt.utils import DynamicGradMode +from sglang.test.test_utils import CustomTestCase + + +class TestDynamicGradMode(CustomTestCase): + def test_inference(self): + # Test inference_mode + DynamicGradMode.set_inference_mode(True) + + @DynamicGradMode() + def create_tensor_x(): + return torch.empty(0) + + X = create_tensor_x() + self.assertTrue(not X.requires_grad and X.is_inference()) + + def test_no_grad(self): + # Test no_grad + DynamicGradMode.set_inference_mode(False) + + @DynamicGradMode() + def create_tensor_y(): + return torch.empty(0) + + Y = create_tensor_y() + self.assertTrue(not Y.requires_grad and not Y.is_inference()) + + def test_nested_inference(self): + # Test no_grad nested inference_mode, inference_mode should has higher priority + DynamicGradMode.set_inference_mode(False) + + @DynamicGradMode() + def create_tensor_z(): + with torch.inference_mode(): + return torch.empty(0) + + Z = create_tensor_z() + self.assertTrue(not Z.requires_grad and Z.is_inference()) + + def test_nested_no_grad(self): + # Test inference_mode nested no_grad, inference_mode should has higher priority + DynamicGradMode.set_inference_mode(True) + + @DynamicGradMode() + def create_tensor_w(): + with torch.no_grad(): + return torch.empty(0) + + W = create_tensor_w() + self.assertTrue(not W.requires_grad and W.is_inference()) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_kvfp4_quant_dequant.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_kvfp4_quant_dequant.py new file mode 100755 index 00000000..122694a4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_kvfp4_quant_dequant.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 + +import time + +import numpy as np +import pytest +import torch + +from sglang.srt.layers.quantization.kvfp4_tensor import KVFP4QuantizeUtil + + +def calculate_accuracy_metrics( + original: torch.Tensor, reconstructed: torch.Tensor +) -> dict[str, float]: + """Calculate accuracy metrics between original and reconstructed tensors.""" + mse = torch.mean((original - reconstructed) ** 2).item() + mae = torch.mean(torch.abs(original - reconstructed)).item() + + # PSNR calculation + max_val = torch.max(torch.abs(original)).item() + psnr = 20 * np.log10(max_val / np.sqrt(mse)) if mse > 0 else float("inf") + + # Relative error + rel_error = torch.mean( + torch.abs(original - reconstructed) / (torch.abs(original) + 1e-8) + ).item() + + return {"MSE": mse, "MAE": mae, "PSNR": psnr, "Relative Error": rel_error} + + +def run_benchmark(m, n, k, num_runs=100) -> dict[str, dict[str, float]]: + """Run FP8 vs KVFP4 quantization benchmark and return metrics.""" + tensor_bf16 = torch.randn(m, n, k, dtype=torch.bfloat16, device="cuda") + + # --- FP8 --- + for _ in range(3): # warmup + _ = tensor_bf16 * 2 + torch.cuda.synchronize() + + start = time.time() + for _ in range(num_runs): + tensor_fp8 = tensor_bf16.to(torch.float8_e4m3fn) + torch.cuda.synchronize() + fp8_quant_time = (time.time() - start) / num_runs + + start = time.time() + for _ in range(num_runs): + tensor_fp8_dequant = tensor_fp8.to(torch.bfloat16) + torch.cuda.synchronize() + fp8_dequant_time = (time.time() - start) / num_runs + + fp8_metrics = calculate_accuracy_metrics(tensor_bf16, tensor_fp8_dequant) + + # --- KVFP4 --- + tensor_fp4, scale_factors = KVFP4QuantizeUtil.batched_quantize(tensor_bf16) + _ = KVFP4QuantizeUtil.batched_dequantize(tensor_fp4, scale_factors) + + start = time.time() + for _ in range(num_runs): + tensor_fp4, scale_factors = KVFP4QuantizeUtil.batched_quantize(tensor_bf16) + torch.cuda.synchronize() + fp4_quant_time = (time.time() - start) / num_runs + + start = time.time() + for _ in range(num_runs): + tensor_fp4_dequant = KVFP4QuantizeUtil.batched_dequantize( + tensor_fp4, scale_factors + ) + torch.cuda.synchronize() + fp4_dequant_time = (time.time() - start) / num_runs + + fp4_metrics = calculate_accuracy_metrics(tensor_bf16, tensor_fp4_dequant) + + return { + "fp8": { + "quant_time": fp8_quant_time, + "dequant_time": fp8_dequant_time, + **fp8_metrics, + }, + "fp4": { + "quant_time": fp4_quant_time, + "dequant_time": fp4_dequant_time, + **fp4_metrics, + }, + } + + +# default tensor shapes (m, n, k) +# [M, 1, 576]: DeepSeekR1-FP4 MLA +# [M, 8, 64]: gpt-oss-20b MHA +MNK_FACTORS = [ + (64, 1, 576), + (512, 1, 576), + (1024, 1, 576), + (4096, 1, 576), + (2868672, 1, 576), + (64, 8, 64), + (512, 8, 64), + (1024, 8, 64), + (4096, 8, 64), + (2868672, 8, 64), +] + + +@pytest.mark.parametrize("m,n,k", MNK_FACTORS) +def test_kvfp4_quant_dequant(m, n, k): + """Benchmark FP8 vs KVFP4 for predefined tensor shapes.""" + print(f"\n=== Running benchmark for tensor shape: [{m}, {n}, {k}] ===") + results = run_benchmark(m, n, k) + + print("FP8:", results["fp8"]) + print("FP4:", results["fp4"]) + + # Basic assertions to make sure metrics are reasonable + assert results["fp4"]["MSE"] < 1.0 + assert results["fp8"]["MSE"] < 1.0 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_layernorm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_layernorm.py new file mode 100644 index 00000000..299e5dcf --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_layernorm.py @@ -0,0 +1,185 @@ +import itertools +import unittest + +import torch + +from sglang.srt.layers.layernorm import GemmaRMSNorm, LayerNorm, RMSNorm +from sglang.test.test_utils import CustomTestCase + + +class TestRMSNorm(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16] + NUM_TOKENS = [7, 83, 4096] + HIDDEN_SIZES = [768, 769, 770, 771, 5120, 5124, 5125, 5126, 8192, 8199] + ADD_RESIDUAL = [False, True] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _run_rms_norm_test(self, num_tokens, hidden_size, add_residual, dtype, seed): + torch.manual_seed(seed) + + layer = RMSNorm(hidden_size).to(dtype=dtype) + layer.weight.data.normal_(mean=1.0, std=0.1) + scale = 1 / (2 * hidden_size) + x = torch.randn(num_tokens, hidden_size, dtype=dtype) * scale + residual = torch.randn_like(x) * scale if add_residual else None + + with torch.inference_mode(): + ref_out = layer.forward_native(x, residual) + out = layer(x, residual) + + if add_residual: + self.assertTrue(torch.allclose(out[0], ref_out[0], atol=1e-2, rtol=1e-2)) + self.assertTrue(torch.allclose(out[1], ref_out[1], atol=1e-2, rtol=1e-2)) + else: + self.assertTrue(torch.allclose(out, ref_out, atol=1e-2, rtol=1e-2)) + + def test_rms_norm(self): + for params in itertools.product( + self.NUM_TOKENS, + self.HIDDEN_SIZES, + self.ADD_RESIDUAL, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + num_tokens=params[0], + hidden_size=params[1], + add_residual=params[2], + dtype=params[3], + seed=params[4], + ): + self._run_rms_norm_test(*params) + + +class TestGemmaRMSNorm(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16] + NUM_TOKENS = [7, 83, 4096] + HIDDEN_SIZES = [768, 769, 770, 771, 5120, 5124, 5125, 5126, 8192, 8199] + ADD_RESIDUAL = [False, True] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _run_gemma_rms_norm_test( + self, num_tokens, hidden_size, add_residual, dtype, seed + ): + torch.manual_seed(seed) + + layer = GemmaRMSNorm(hidden_size).to(dtype=dtype) + layer.weight.data.normal_(mean=1.0, std=0.1) + scale = 1 / (2 * hidden_size) + x = torch.randn(num_tokens, hidden_size, dtype=dtype) * scale + residual = torch.randn_like(x) * scale if add_residual else None + + with torch.inference_mode(): + ref_out = layer.forward_native(x, residual) + out = layer(x, residual) + + if add_residual: + self.assertTrue(torch.allclose(out[0], ref_out[0], atol=1e-3, rtol=1e-3)) + self.assertTrue(torch.allclose(out[1], ref_out[1], atol=1e-3, rtol=1e-3)) + else: + self.assertTrue(torch.allclose(out, ref_out, atol=1e-3, rtol=1e-3)) + + def test_gemma_rms_norm(self): + for params in itertools.product( + self.NUM_TOKENS, + self.HIDDEN_SIZES, + self.ADD_RESIDUAL, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + num_tokens=params[0], + hidden_size=params[1], + add_residual=params[2], + dtype=params[3], + seed=params[4], + ): + self._run_gemma_rms_norm_test(*params) + + +class TestLayerNorm(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16] + PARAM_DTYPES = [torch.bfloat16, torch.float32] + NUM_TOKENS = [7, 83, 1024] + HIDDEN_SIZES = [128, 512, 1536, 5120, 5124, 5125, 5126, 7168] + USE_AFFINE = [False, True] + USE_BIAS = [False, True] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _run_layer_norm_test( + self, num_tokens, hidden_size, use_affine, use_bias, dtype, seed, param_dtype + ): + torch.manual_seed(seed) + + layer = LayerNorm( + hidden_size, elementwise_affine=use_affine, bias=use_bias, dtype=param_dtype + ) + if use_affine: + layer.weight.data.normal_(mean=1.0, std=0.1) + if use_bias: + layer.bias.data.normal_(mean=0.0, std=0.1) + + scale = 1 / (2 * hidden_size) + x = torch.randn(num_tokens, hidden_size, dtype=dtype) * scale + + with torch.inference_mode(): + ref_out = layer.forward_native(x) + out = layer(x) + + self.assertTrue(torch.allclose(out, ref_out, atol=1e-2, rtol=1e-3)) + + if ( + use_affine + and use_bias + and not (dtype == torch.bfloat16 and param_dtype == torch.float32) + ): + layer.dtype = torch.float32 + layer.weight.data = layer.weight.data.to(torch.float32) + layer.bias.data = layer.bias.data.to(torch.float32) + with torch.inference_mode(): + cuda_out = layer(x.to(torch.bfloat16)).to(x.dtype) + + self.assertTrue(torch.allclose(cuda_out, ref_out, atol=2e-2, rtol=1e-3)) + + def test_layer_norm(self): + for params in itertools.product( + self.NUM_TOKENS, + self.HIDDEN_SIZES, + self.USE_AFFINE, + self.USE_BIAS, + self.DTYPES, + self.SEEDS, + self.PARAM_DTYPES, + ): + with self.subTest( + num_tokens=params[0], + hidden_size=params[1], + use_affine=params[2], + use_bias=params[3], + dtype=params[4], + seed=params[5], + param_dtype=params[6], + ): + self._run_layer_norm_test(*params) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_marlin_moe.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_marlin_moe.py new file mode 100644 index 00000000..dd2497bb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_marlin_moe.py @@ -0,0 +1,288 @@ +from typing import Optional + +import pytest +import torch +from sgl_kernel.scalar_type import ScalarType, scalar_types + +from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.moe.fused_moe_triton.fused_marlin_moe import fused_marlin_moe +from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler +from sglang.test.test_marlin_utils import awq_marlin_quantize, marlin_quantize + +set_global_server_args_for_scheduler(object.__new__(ServerArgs)) + + +def stack_and_dev(tensors: list[torch.Tensor]): + dev = tensors[0].device + return torch.stack(tensors, dim=0).to(dev) + + +def torch_experts( + a: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weight: torch.Tensor, + topk_ids: torch.Tensor, + global_num_experts: int = -1, + expert_map: Optional[torch.Tensor] = None, + quant_dtype: Optional[torch.dtype] = None, + apply_router_weights_on_input: bool = False, +) -> torch.Tensor: + assert ( + global_num_experts == -1 + or (global_num_experts == w1.shape[0] and expert_map is None) + or (expert_map is not None and global_num_experts == expert_map.shape[0]) + ) + + M, K = a.shape + topk = topk_ids.shape[1] + print("quant_dtype", quant_dtype) + # exit(0) + if apply_router_weights_on_input: + assert topk == 1 + a = a * topk_weight.to(a.dtype) + + a = a.view(M, -1, K).repeat(1, topk, 1).reshape(-1, K) + + out = torch.zeros(M * topk, w2.shape[1], dtype=a.dtype, device=a.device) + + num_experts = w1.shape[0] + + topk_ids = topk_ids.view(-1) + if expert_map is not None: + topk_ids = expert_map[topk_ids] + + f32 = torch.float32 + + for i in range(num_experts): + mask = topk_ids == i + if mask.sum(): + if quant_dtype is None: + tmp1 = a[mask] @ w1[i].transpose(0, 1) + tmp2 = SiluAndMul()(tmp1) + out[mask] = tmp2 @ w2[i].transpose(0, 1) + + if apply_router_weights_on_input: + return out + else: + return ( + (out.view(M, -1, w2.shape[1]).to(f32) * topk_weight.view(M, -1, 1)) + .sum(dim=1) + .to(out.dtype) + ) + + +def torch_moe( + a: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + score: torch.Tensor, + topk: int, + global_num_experts: int = -1, + expert_map: Optional[torch.Tensor] = None, +) -> torch.Tensor: + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + return torch_experts( + a, w1, w2, topk_weight, topk_ids, global_num_experts, expert_map + ) + + +def marlin_moe_generate_valid_test_cases(): + import itertools + + m_list = [1, 123, 666] + n_list = [128, 1024] + k_list = [256, 2048] + e_list = [4, 12] + topk_list = [2, 3] + dtype_list = [torch.half, torch.bfloat16] + group_size_list = [128] + act_order_list = [True, False] + quant_type_list = [ + scalar_types.uint4, + scalar_types.uint4b8, + ] + is_k_full_list = [True, False] + + all_combinations = itertools.product( + m_list, + n_list, + k_list, + e_list, + topk_list, + dtype_list, + group_size_list, + act_order_list, + quant_type_list, + is_k_full_list, + ) + + def is_invalid( + m, n, k, e, topk, dtype, group_size, act_order, quant_type, is_k_full + ): + + # Filter act_order + if act_order: + if group_size in (-1, k, n): + return False + if quant_type not in [scalar_types.uint4b8]: + return False + elif not is_k_full: + return False + + return True + + cases = [] + for case in all_combinations: + if is_invalid(*case): + cases.append(case) + return cases + + +@pytest.mark.flaky(reruns=2) +@pytest.mark.parametrize( + ("m, n, k, e, topk, dtype, group_size," "act_order, quant_type, is_k_full"), + marlin_moe_generate_valid_test_cases(), +) +def test_fused_marlin_moe( + m: int, + n: int, + k: int, + e: int, + topk: int, + dtype: torch.dtype, + group_size: int, + act_order: bool, + quant_type: ScalarType, + is_k_full: bool, +): + if not torch.cuda.is_available(): + pytest.skip("CUDA device not available") + + torch.manual_seed(0) + + has_zp = quant_type in [scalar_types.uint4, scalar_types.uint8] + + # Filter act_order + if act_order: + if group_size == -1: + return + if group_size in (k, n): + return + if has_zp: + return + else: + if not is_k_full: + return + + a = torch.randn((m, k), device="cuda", dtype=dtype) / 10 + w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 20 + w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 20 + + e_map = None + + w_ref1_l = [] + qweight1_l = [] + scales1_l = [] + zeros1_l = [] + g_idx1_l = [] + sort_indices1_l = [] + + for i in range(w1.shape[0]): + if has_zp: + w_ref1, qweight1, scales1, zeros1 = awq_marlin_quantize( + w1[i].transpose(1, 0), quant_type, group_size + ) + + w_ref1_l.append(w_ref1.T) + qweight1_l.append(qweight1) + scales1_l.append(scales1) + zeros1_l.append(zeros1) + else: + test_perm = torch.randperm(k) + w_ref1, qweight1, scales1, g_idx1, sort_indices1, _ = marlin_quantize( + w1[i].transpose(1, 0), quant_type, group_size, act_order, test_perm + ) + + w_ref1_l.append(w_ref1.T) + qweight1_l.append(qweight1) + scales1_l.append(scales1) + g_idx1_l.append(g_idx1) + sort_indices1_l.append(sort_indices1) + + w_ref1 = stack_and_dev(w_ref1_l) + qweight1 = stack_and_dev(qweight1_l).contiguous() + scales1 = stack_and_dev(scales1_l) + g_idx1 = stack_and_dev(g_idx1_l) if g_idx1_l else None + zeros1 = stack_and_dev(zeros1_l) if zeros1_l else None + sort_indices1 = stack_and_dev(sort_indices1_l) if sort_indices1_l else None + + w_ref2_l = [] + qweight2_l = [] + scales2_l = [] + zeros2_l = [] + g_idx2_l = [] + sort_indices2_l = [] + + for i in range(w2.shape[0]): + if has_zp: + w_ref2, qweight2, scales2, zeros2 = awq_marlin_quantize( + w2[i].transpose(1, 0), quant_type, group_size + ) + + w_ref2_l.append(w_ref2.T) + qweight2_l.append(qweight2) + scales2_l.append(scales2) + zeros2_l.append(zeros2) + else: + test_perm = torch.randperm(n) + w_ref2, qweight2, scales2, g_idx2, sort_indices2, _ = marlin_quantize( + w2[i].transpose(1, 0), quant_type, group_size, act_order, test_perm + ) + + w_ref2_l.append(w_ref2.T) + qweight2_l.append(qweight2) + scales2_l.append(scales2) + g_idx2_l.append(g_idx2) + sort_indices2_l.append(sort_indices2) + + w_ref2 = stack_and_dev(w_ref2_l) + qweight2 = stack_and_dev(qweight2_l).contiguous() + scales2 = stack_and_dev(scales2_l) + g_idx2 = stack_and_dev(g_idx2_l) if g_idx2_l else None + zeros2 = stack_and_dev(zeros2_l) if zeros2_l else None + sort_indices2 = stack_and_dev(sort_indices2_l) if sort_indices2_l else None + + score = torch.randn((m, e), device="cuda", dtype=dtype) + from sglang.srt.layers.moe.topk import fused_topk_torch_native + + topk_weights, topk_ids = fused_topk_torch_native(a, score, topk, False) + + torch_output = torch_moe(a, w_ref1, w_ref2, score, topk, expert_map=e_map) + + marlin_output = fused_marlin_moe( + a, + qweight1, + qweight2, + scales1, + scales2, + score, + topk_weights, + topk_ids, + g_idx1=g_idx1, + g_idx2=g_idx2, + sort_indices1=sort_indices1, + sort_indices2=sort_indices2, + w1_zeros=zeros1, + w2_zeros=zeros2, + num_bits=4, + is_k_full=is_k_full, + ) + + torch.testing.assert_close(marlin_output, torch_output, atol=5e-2, rtol=0) + + +if __name__ == "__main__": + # Run the specific test function directly + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_marlin_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_marlin_utils.py new file mode 100644 index 00000000..0c059007 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_marlin_utils.py @@ -0,0 +1,171 @@ +""" +Adapted from +https://github.com/vllm-project/vllm/blob/020f58abcdea65302225663130d08fd8f4dd755a/vllm/model_executor/layers/quantization/utils/marlin_utils_test.py +""" + +# SPDX-License-Identifier: Apache-2.0 +"""Utility functions used for tests and benchmarks""" + +from typing import Optional + +import numpy as np +import torch +from sgl_kernel.scalar_type import ScalarType + +from sglang.srt.layers.quantization.marlin_utils import ( + GPTQ_MARLIN_TILE, + marlin_permute_scales, + marlin_zero_points, +) +from sglang.srt.layers.quantization.utils import ( + get_pack_factor, + gptq_quantize_weights, + quantize_weights, + sort_weights, +) + + +class MarlinWorkspace: + + def __init__(self, out_features, min_thread_n, max_parallel): + assert ( + out_features % min_thread_n == 0 + ), "out_features = {} is undivisible by min_thread_n = {}".format( + out_features, min_thread_n + ) + + max_workspace_size = (out_features // min_thread_n) * max_parallel + + self.scratch = torch.zeros(max_workspace_size, dtype=torch.int, device="cuda") + + +def marlin_permute_weights(q_w, size_k, size_n, perm, tile=GPTQ_MARLIN_TILE): + assert q_w.shape == (size_k, size_n) + assert size_k % tile == 0, f"size_k = {size_k}, tile = {tile}" + assert size_n % tile == 0, f"size_k = {size_n}, tile = {tile}" + + # Permute weights to 16x64 marlin tiles + q_w = q_w.reshape((size_k // tile, tile, size_n // tile, tile)) + q_w = q_w.permute((0, 2, 1, 3)) + q_w = q_w.reshape((size_k // tile, size_n * tile)) + + q_w = q_w.reshape((-1, perm.numel()))[:, perm].reshape(q_w.shape) + + return q_w + + +def marlin_weights(q_w, size_k, size_n, num_bits, perm): + # Permute + q_w = marlin_permute_weights(q_w, size_k, size_n, perm) + + # Pack + pack_factor = get_pack_factor(num_bits) + orig_device = q_w.device + + q_w = q_w.cpu().numpy().astype(np.uint32) + + q_packed = np.zeros((q_w.shape[0], q_w.shape[1] // pack_factor), dtype=np.uint32) + for i in range(pack_factor): + q_packed |= q_w[:, i::pack_factor] << num_bits * i + + q_packed = torch.from_numpy(q_packed.astype(np.int32)).to(orig_device) + + return q_packed + + +def get_weight_perm(num_bits: int): + perm_list: list[int] = [] + for i in range(32): + perm1: list[int] = [] + col = i // 4 + for block in [0, 1]: + for row in [ + 2 * (i % 4), + 2 * (i % 4) + 1, + 2 * (i % 4 + 4), + 2 * (i % 4 + 4) + 1, + ]: + perm1.append(16 * row + col + 8 * block) + for j in range(4): + perm_list.extend([p + 256 * j for p in perm1]) + + perm = np.array(perm_list) + + if num_bits == 4: + interleave = np.array([0, 2, 4, 6, 1, 3, 5, 7]) + elif num_bits == 8: + interleave = np.array([0, 2, 1, 3]) + else: + raise Exception("num_bits must be 4 or 8, got {}".format(num_bits)) + + perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel() + perm = torch.from_numpy(perm) + return perm + + +def marlin_quantize( + w: torch.Tensor, + quant_type: ScalarType, + group_size: int, + act_order: bool, + test_perm: Optional[torch.Tensor] = None, +): + size_k, size_n = w.shape + num_bits = quant_type.size_bits + + # Normalize group_size + if group_size == -1: + group_size = size_k + assert group_size <= size_k + + # Quantize (and apply act_order if provided) + w_ref, q_w, s, g_idx, rand_perm = gptq_quantize_weights( + w, quant_type, group_size, act_order, test_perm + ) + + # For act_order, sort the "weights" and "g_idx" so that group ids are + # increasing + sort_indices = torch.empty(0, dtype=torch.int, device=w.device) + if act_order: + q_w, g_idx, sort_indices = sort_weights(q_w, g_idx) + + # Reformat to marlin + weight_perm = get_weight_perm(num_bits) + marlin_q_w = marlin_weights(q_w, size_k, size_n, num_bits, weight_perm) + marlin_s = marlin_permute_scales(s, size_k, size_n, group_size) + + # Create result + res_list = [w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, rand_perm] + for i in range(len(res_list)): + res_list[i] = res_list[i].to(w.device) + + return res_list + + +def awq_marlin_quantize(w: torch.Tensor, quant_type: ScalarType, group_size: int): + size_k, size_n = w.shape + + # Normalize group_size + if group_size == -1: + group_size = size_k + assert group_size <= size_k + + # Detect num groups + assert size_k % group_size == 0 + num_groups = size_k // group_size + + # Quantize with zp + w_ref, q_w, s, zp = quantize_weights(w, quant_type, group_size, zero_points=True) + + # Reformat to marlin + weight_perm = get_weight_perm(quant_type.size_bits) + marlin_q_w = marlin_weights(q_w, size_k, size_n, quant_type.size_bits, weight_perm) + marlin_s = marlin_permute_scales(s, size_k, size_n, group_size) + marlin_zp = marlin_zero_points(zp, num_groups, size_n, quant_type.size_bits) + + # Create result + res_list = [w_ref, marlin_q_w, marlin_s, marlin_zp] + for i in range(len(res_list)): + res_list[i] = res_list[i].to(w.device) + + return res_list diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_programs.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_programs.py new file mode 100644 index 00000000..919d8128 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_programs.py @@ -0,0 +1,578 @@ +"""This file contains the SGL programs used for unit testing.""" + +import json +import re +import time + +import numpy as np + +import sglang as sgl +from sglang.srt.utils import is_hip +from sglang.utils import download_and_cache_file, read_jsonl + +_is_hip = is_hip() + + +def test_few_shot_qa(): + @sgl.function + def few_shot_qa(s, question): + s += "The following are questions with answers.\n\n" + s += "Q: What is the capital of France?\n" + s += "A: Paris\n" + s += "Q: What is the capital of Germany?\n" + s += "A: Berlin\n" + s += "Q: What is the capital of Italy?\n" + s += "A: Rome\n" + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n", temperature=0) + + ret = few_shot_qa.run(question="What is the capital of the United States?") + assert "washington" in ret["answer"].strip().lower(), f"answer: {ret['answer']}" + + rets = few_shot_qa.run_batch( + [ + {"question": "What is the capital of Japan?"}, + {"question": "What is the capital of the United Kingdom?"}, + {"question": "What is the capital city of China?"}, + ], + temperature=0.1, + ) + answers = [x["answer"].strip().lower() for x in rets] + assert answers == ["tokyo", "london", "beijing"], f"answers: {answers}" + + +def test_mt_bench(): + @sgl.function + def answer_mt_bench(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1")) + with s.user(): + s += question_2 + with s.assistant(): + s += sgl.gen("answer_2") + + question_1 = "Compose an engaging travel blog post about a recent trip to Hawaii, highlighting cultural experiences and must-see attractions." + question_2 = ( + "Rewrite your previous response. Start every sentence with the letter A." + ) + ret = answer_mt_bench.run( + question_1=question_1, question_2=question_2, temperature=0.7, max_new_tokens=64 + ) + assert len(ret.messages()) in [4, 5] + + +def test_select(check_answer): + @sgl.function + def true_or_false(s, statement): + s += "Determine whether the statement below is True, False, or Unknown.\n" + s += "Statement: The capital of France is Pairs.\n" + s += "Answer: True\n" + s += "Statement: " + statement + "\n" + s += "Answer:" + sgl.select("answer", ["True", "False", "Unknown"]) + + ret = true_or_false.run( + statement="The capital of Germany is Berlin.", + ) + if check_answer: + assert ret["answer"] == "True", ret.text() + else: + assert ret["answer"] in ["True", "False", "Unknown"] + + ret = true_or_false.run( + statement="The capital of Canada is Tokyo.", + ) + if check_answer: + assert ret["answer"] == "False", ret.text() + else: + assert ret["answer"] in ["True", "False", "Unknown"] + + ret = true_or_false.run( + statement="Purple is a better color than green.", + ) + if check_answer: + assert ret["answer"] == "Unknown", ret.text() + else: + assert ret["answer"] in ["True", "False", "Unknown"] + + +def test_decode_int(): + @sgl.function + def decode_int(s): + s += "The number of hours in a day is " + sgl.gen_int("hours") + "\n" + s += "The number of days in a year is " + sgl.gen_int("days") + "\n" + + ret = decode_int.run(temperature=0.1) + assert int(ret["hours"]) == 24, ret.text() + assert int(ret["days"]) == 365, ret.text() + + +def test_decode_json_regex(): + @sgl.function + def decode_json(s): + from sglang.lang.ir import REGEX_FLOAT, REGEX_INT, REGEX_STR + + s += "Generate a JSON object to describe the basic city information of Paris.\n" + s += "Here are the JSON object:\n" + + # NOTE: we recommend using dtype gen or whole regex string to control the output + + with s.var_scope("json_output"): + s += "{\n" + s += ' "name": ' + sgl.gen(regex=REGEX_STR) + ",\n" + s += ' "population": ' + sgl.gen(regex=REGEX_INT, stop=[" ", "\n"]) + ",\n" + s += ' "area": ' + sgl.gen(regex=REGEX_INT, stop=[" ", "\n"]) + ",\n" + s += ' "latitude": ' + sgl.gen(regex=REGEX_FLOAT, stop=[" ", "\n"]) + "\n" + s += "}" + + ret = decode_json.run(temperature=0.0) + try: + js_obj = json.loads(ret["json_output"]) + except json.decoder.JSONDecodeError: + print("JSONDecodeError", ret["json_output"]) + raise + assert isinstance(js_obj["name"], str) + assert isinstance(js_obj["population"], int) + + +def test_decode_json(): + @sgl.function + def decode_json(s): + s += "Generate a JSON object to describe the basic city information of Paris.\n" + + with s.var_scope("json_output"): + s += "{\n" + s += ' "name": ' + sgl.gen_string() + ",\n" + s += ' "population": ' + sgl.gen_int() + ",\n" + s += ' "area": ' + sgl.gen(dtype=int) + ",\n" + s += ' "country": ' + sgl.gen_string() + ",\n" + s += ' "timezone": ' + sgl.gen(dtype=str) + "\n" + s += "}" + + ret = decode_json.run(max_new_tokens=64) + try: + js_obj = json.loads(ret["json_output"]) + except json.decoder.JSONDecodeError: + print("JSONDecodeError", ret["json_output"]) + raise + assert isinstance(js_obj["name"], str) + assert isinstance(js_obj["population"], int) + + +def test_expert_answer(check_answer=True): + @sgl.function + def expert_answer(s, question): + s += "Question: " + question + "\n" + s += ( + "A good person to answer this question is" + + sgl.gen("expert", stop=[".", "\n"]) + + ".\n" + ) + s += ( + "For example," + + s["expert"] + + " would answer that " + + sgl.gen("answer", stop=".") + + "." + ) + + ret = expert_answer.run(question="What is the capital of France?", temperature=0.1) + + if check_answer: + assert "paris" in ret.text().lower(), f"Answer: {ret.text()}" + + +def test_tool_use(): + def calculate(expression): + return f"{eval(expression)}" + + @sgl.function + def tool_use(s, lhs, rhs): + s += "Please perform computations using a calculator. You can use calculate(expression) to get the results.\n" + s += "For example,\ncalculate(1+2)=3\ncalculate(3*4)=12\n" + s += "Question: What is the product of " + str(lhs) + " and " + str(rhs) + "?\n" + s += ( + "Answer: The answer is calculate(" + + sgl.gen("expression", stop=")") + + ") = " + ) + with s.var_scope("answer"): + s += calculate(s["expression"]) + + lhs, rhs = 257, 983 + ret = tool_use(lhs=lhs, rhs=rhs, temperature=0) + assert int(ret["answer"]) == lhs * rhs + + +def test_react(): + @sgl.function + def react(s, question): + s += """ +Question: Which country does the founder of Microsoft live in? +Thought 1: I need to search for the founder of Microsoft. +Action 1: Search [Founder of Microsoft]. +Observation 1: The founder of Microsoft is Bill Gates. +Thought 2: I need to search for the country where Bill Gates lives in. +Action 2: Search [Where does Bill Gates live]. +Observation 2: Bill Gates lives in the United States. +Thought 3: The answer is the United States. +Action 3: Finish [United States].\n +""" + + s += "Question: " + question + "\n" + + for i in range(1, 5): + s += f"Thought {i}:" + sgl.gen(stop=[".", "\n"]) + ".\n" + s += f"Action {i}: " + sgl.select(f"action_{i}", ["Search", "Finish"]) + + if s[f"action_{i}"] == "Search": + s += " [" + sgl.gen(stop="]") + "].\n" + s += f"Observation {i}:" + sgl.gen(stop=[".", "\n"]) + ".\n" + else: + s += " [" + sgl.gen("answer", stop="]") + "].\n" + break + + ret = react.run( + question="What country does the creator of Linux live in?", + temperature=0.1, + ) + answer = ret["answer"].lower() + assert "finland" in answer or "states" in answer + + +def test_parallel_decoding(): + max_tokens = 64 + fork_size = 5 + + @sgl.function + def parallel_decoding(s, topic): + s += "Act as a helpful assistant.\n" + s += "USER: Give some tips for " + topic + ".\n" + s += ( + "ASSISTANT: Okay. Here are " + + str(fork_size) + + " concise tips, each under 8 words:\n" + ) + + # Generate skeleton + for i in range(1, 1 + fork_size): + s += f"{i}." + sgl.gen(max_tokens=16, stop=[".", "\n"]) + ".\n" + + # Generate detailed tips + forks = s.fork(fork_size) + for i in range(fork_size): + forks[ + i + ] += f"Now, I expand tip {i+1} into a detailed paragraph:\nTip {i+1}:" + forks[i] += sgl.gen("detailed_tip", max_tokens, stop=["\n\n"]) + forks.join() + + # Concatenate tips and summarize + s += "Here are these tips with detailed explanation:\n" + for i in range(fork_size): + s += f"Tip {i+1}:" + forks[i]["detailed_tip"] + "\n" + + s += "\nIn summary," + sgl.gen("summary", max_tokens=512) + + ret = parallel_decoding.run(topic="writing a good blog post", temperature=0.3) + assert isinstance(ret["summary"], str) + + +def test_parallel_encoding(check_answer=True): + max_tokens = 64 + + @sgl.function + def parallel_encoding(s, question, context_0, context_1, context_2): + s += "USER: I will ask a question based on some statements.\n" + s += "ASSISTANT: Sure. I will give the answer.\n" + s += "USER: Please memorize these statements.\n" + + contexts = [context_0, context_1, context_2] + + forks = s.fork(len(contexts)) + forks += lambda i: f"Statement {i}: " + contexts[i] + "\n" + forks.join(mode="concate_and_append") + + s += "Now, please answer the following question. " "Do not list options." + s += "\nQuestion: " + question + "\n" + s += "ASSISTANT:" + sgl.gen("answer", max_tokens=max_tokens) + + ret = parallel_encoding.run( + question="Who is the father of Julian?", + context_0="Ethan is the father of Liam.", + context_1="Noah is the father of Julian.", + context_2="Oliver is the father of Carlos.", + temperature=0, + ) + answer = ret["answer"] + + if check_answer: + assert "Noah" in answer + + +def test_image_qa(): + @sgl.function + def image_qa(s, question): + s += sgl.user(sgl.image("example_image.png") + question) + s += sgl.assistant(sgl.gen("answer")) + + state = image_qa.run( + question="Please describe this image in simple words.", + temperature=0, + max_new_tokens=64, + ) + + assert ( + "taxi" in state.messages()[-1]["content"] + or "car" in state.messages()[-1]["content"] + ), f"{state.messages()[-1]['content']}" + + +def test_stream(): + @sgl.function + def qa(s, question): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question) + s += sgl.assistant(sgl.gen("answer")) + + ret = qa( + question="Compose an engaging travel blog post about a recent trip to Hawaii, highlighting cultural experiences and must-see attractions.", + stream=True, + ) + out = "" + for chunk in ret.text_iter(): + out += chunk + + ret = qa( + question="Compose an engaging travel blog post about a recent trip to Hawaii, highlighting cultural experiences and must-see attractions.", + stream=True, + ) + out = "" + for chunk in ret.text_iter("answer"): + out += chunk + + +def test_regex(): + regex = r"((25[0-5]|2[0-4]\d|[01]?\d\d?).){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)" + + @sgl.function + def regex_gen(s): + s += "Q: What is the IP address of the Google DNS servers?\n" + s += "A: " + sgl.gen( + "answer", + temperature=0, + regex=regex, + ) + + state = regex_gen.run() + answer = state["answer"] + assert re.match(regex, answer) + + +def test_dtype_gen(): + @sgl.function + def dtype_gen(s): + s += "Q: What is the full name of DNS?\n" + s += "A: The full names is " + sgl.gen("str_res", dtype=str, stop="\n") + "\n" + s += "Q: Which year was DNS invented?\n" + s += "A: " + sgl.gen("int_res", dtype=int) + "\n" + s += "Q: What is the value of pi?\n" + s += "A: " + sgl.gen("float_res", dtype=float) + "\n" + s += "Q: Is the sky blue?\n" + s += "A: " + sgl.gen("bool_res", dtype=bool) + "\n" + + state = dtype_gen.run() + + try: + state["int_res"] = int(state["int_res"]) + state["float_res"] = float(state["float_res"]) + state["bool_res"] = bool(state["bool_res"]) + # assert state["str_res"].startswith('"') and state["str_res"].endswith('"') + except ValueError: + print(state) + raise + + +def test_completion_speculative(): + @sgl.function(num_api_spec_tokens=64) + def gen_character_spec(s): + s += "Construct a character within the following format:\n" + s += "Name: Steve Jobs.\nBirthday: February 24, 1955.\nJob: Apple CEO.\n" + s += "\nPlease generate new Name, Birthday and Job.\n" + s += ( + "Name:" + + sgl.gen("name", stop="\n") + + "\nBirthday:" + + sgl.gen("birthday", stop="\n") + ) + s += "\nJob:" + sgl.gen("job", stop="\n") + "\n" + + @sgl.function + def gen_character_no_spec(s): + s += "Construct a character within the following format:\n" + s += "Name: Steve Jobs.\nBirthday: February 24, 1955.\nJob: Apple CEO.\n" + s += "\nPlease generate new Name, Birthday and Job.\n" + s += ( + "Name:" + + sgl.gen("name", stop="\n") + + "\nBirthday:" + + sgl.gen("birthday", stop="\n") + ) + s += "\nJob:" + sgl.gen("job", stop="\n") + "\n" + + token_usage = sgl.global_config.default_backend.token_usage + + token_usage.reset() + gen_character_spec().sync() + usage_with_spec = token_usage.prompt_tokens + + token_usage.reset() + gen_character_no_spec().sync() + usage_with_no_spec = token_usage.prompt_tokens + + assert ( + usage_with_spec < usage_with_no_spec + ), f"{usage_with_spec} vs {usage_with_no_spec}" + + +def test_chat_completion_speculative(): + @sgl.function(num_api_spec_tokens=256) + def gen_character_spec(s): + s += sgl.system("You are a helpful assistant.") + s += sgl.user("Construct a character within the following format:") + s += sgl.assistant( + "Name: Steve Jobs.\nBirthday: February 24, 1955.\nJob: Apple CEO.\n" + ) + s += sgl.user("Please generate new Name, Birthday and Job.\n") + s += sgl.assistant( + "Name:" + + sgl.gen("name", stop="\n") + + "\nBirthday:" + + sgl.gen("birthday", stop="\n") + + "\nJob:" + + sgl.gen("job", stop="\n") + ) + + gen_character_spec().sync() + + +def test_hellaswag_select(): + """Benchmark the accuracy of sgl.select on the HellaSwag dataset.""" + + def get_one_example(lines, i, include_answer): + ret = lines[i]["activity_label"] + ": " + lines[i]["ctx"] + " " + if include_answer: + ret += lines[i]["endings"][lines[i]["label"]] + return ret + + def get_few_shot_examples(lines, k): + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + # Read data + url = "https://raw.githubusercontent.com/rowanz/hellaswag/master/data/hellaswag_val.jsonl" + filename = download_and_cache_file(url) + lines = list(read_jsonl(filename)) + + # Construct prompts + num_questions = 200 + num_shots = 20 + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + choices = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + choices.append(lines[i]["endings"]) + labels.append(lines[i]["label"]) + arguments = [{"question": q, "choices": c} for q, c in zip(questions, choices)] + + ##################################### + ######### SGL Program Begin ######### + ##################################### + + import sglang as sgl + + @sgl.function + def few_shot_hellaswag(s, question, choices): + s += few_shot_examples + question + s += sgl.select("answer", choices=choices) + + ##################################### + ########## SGL Program End ########## + ##################################### + + # Run requests + tic = time.perf_counter() + rets = few_shot_hellaswag.run_batch( + arguments, + temperature=0, + num_threads=64, + progress_bar=True, + generator_style=False, + ) + preds = [] + for i, ret in enumerate(rets): + preds.append(choices[i].index(ret["answer"])) + latency = time.perf_counter() - tic + + # Compute accuracy + accuracy = np.mean(np.array(preds) == np.array(labels)) + + # Test generator style of run_batch + tic = time.perf_counter() + rets = few_shot_hellaswag.run_batch( + arguments, + temperature=0, + num_threads=64, + progress_bar=True, + generator_style=True, + ) + preds_gen = [] + for i, ret in enumerate(rets): + preds_gen.append(choices[i].index(ret["answer"])) + latency_gen = time.perf_counter() - tic + + # Compute accuracy + accuracy_gen = np.mean(np.array(preds_gen) == np.array(labels)) + print(f"{accuracy=}, {accuracy_gen=}") + assert np.abs(accuracy_gen - accuracy) < 0.1 + assert np.abs(latency_gen - latency) < 1 if not _is_hip else 2 + + return accuracy, latency + + +def test_gen_min_new_tokens(): + """ + Validate sgl.gen(min_tokens) functionality. + + The test asks a question where, without a min_tokens constraint, the generated answer is expected to be short. + By enforcing the min_tokens parameter, we ensure the generated answer has at least the specified number of tokens. + We verify that the number of tokens in the answer is >= the min_tokens threshold. + """ + import sglang as sgl + from sglang.srt.utils.hf_transformers_utils import get_tokenizer + + model_path = sgl.global_config.default_backend.endpoint.get_model_name() + MIN_TOKENS, MAX_TOKENS = 64, 128 + + @sgl.function + def convo_1(s): + s += sgl.user("What is the capital of the United States?") + s += sgl.assistant( + sgl.gen("answer", min_tokens=MIN_TOKENS, max_tokens=MAX_TOKENS) + ) + + def assert_min_tokens(tokenizer, text): + token_ids = tokenizer.encode(text) + assert ( + len(token_ids) >= MIN_TOKENS + ), f"Generated {len(token_ids)} tokens, min required: {MIN_TOKENS}. Text: {text}" + + tokenizer = get_tokenizer(model_path) + + state = convo_1.run() + assert_min_tokens(tokenizer, state["answer"]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_utils.py new file mode 100644 index 00000000..98b2e8a4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/test/test_utils.py @@ -0,0 +1,1919 @@ +"""Common utilities for testing and benchmarking""" + +import argparse +import asyncio +import copy +import json +import logging +import os +import random +import re +import subprocess +import sys +import threading +import time +import unittest +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime +from functools import partial, wraps +from io import BytesIO +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Awaitable, Callable, List, Optional, Tuple + +import aiohttp +import numpy as np +import requests +import torch +import torch.nn.functional as F +from PIL import Image + +from sglang.bench_serving import run_benchmark +from sglang.global_config import global_config +from sglang.srt.utils import ( + get_bool_env_var, + get_device, + is_port_available, + kill_process_tree, + retry, +) +from sglang.test.run_eval import run_eval +from sglang.utils import get_exception_traceback + +# General test models +DEFAULT_MODEL_NAME_FOR_TEST = "meta-llama/Llama-3.1-8B-Instruct" +DEFAULT_SMALL_MODEL_NAME_FOR_TEST = "meta-llama/Llama-3.2-1B-Instruct" +DEFAULT_SMALL_MODEL_NAME_FOR_TEST_BASE = "meta-llama/Llama-3.2-1B" +DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE = "Qwen/Qwen3-Reranker-0.6B" +DEFAULT_MOE_MODEL_NAME_FOR_TEST = "mistralai/Mixtral-8x7B-Instruct-v0.1" +DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_BASE = "Qwen/Qwen1.5-MoE-A2.7B" +DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_CHAT = "Qwen/Qwen1.5-MoE-A2.7B-Chat" + +# MLA test models +DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST = "Alibaba-NLP/gte-Qwen2-1.5B-instruct" +DEFAULT_SMALL_CROSS_ENCODER_MODEL_NAME_FOR_TEST = "cross-encoder/ms-marco-MiniLM-L6-v2" +DEFAULT_MLA_MODEL_NAME_FOR_TEST = "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct" +DEFAULT_MLA_FP8_MODEL_NAME_FOR_TEST = "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8" +DEFAULT_MODEL_NAME_FOR_TEST_MLA = "lmsys/sglang-ci-dsv3-test" +DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN = "lmsys/sglang-ci-dsv3-test-NextN" + +# NVFP4 models +DEFAULT_DEEPSEEK_NVFP4_MODEL_FOR_TEST = "nvidia/DeepSeek-V3-0324-FP4" +DEFAULT_MODEL_NAME_FOR_TEST_MOE_NVFP4 = "nvidia/Qwen3-30B-A3B-FP4" + +# FP8 models +DEFAULT_MODEL_NAME_FOR_TEST_FP8 = "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8" +DEFAULT_MODEL_NAME_FOR_ACCURACY_TEST_FP8 = "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8" +DEFAULT_MODEL_NAME_FOR_DYNAMIC_QUANT_ACCURACY_TEST_FP8 = ( + "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8-dynamic" +) +DEFAULT_MODEL_NAME_FOR_MODELOPT_QUANT_ACCURACY_TEST_FP8 = ( + "nvidia/Llama-3.1-8B-Instruct-FP8" +) +DEFAULT_MODEL_NAME_FOR_TEST_QWEN_FP8 = "Qwen/Qwen3-1.7B-FP8" +DEFAULT_MODEL_NAME_FOR_TEST_FP8_WITH_MOE = "gaunernst/DeepSeek-V2-Lite-Chat-FP8" + +# MXFP4 models +# Standard MXFP4 MoE test model +DEFAULT_MODEL_NAME_FOR_TEST_MXFP4_WITH_MOE = "openai/gpt-oss-20b" + +# W8A8 models +DEFAULT_MODEL_NAME_FOR_TEST_W8A8 = "RedHatAI/Llama-3.2-3B-quantized.w8a8" +DEFAULT_MODEL_NAME_FOR_TEST_W8A8_WITH_MOE = "nytopop/Qwen3-30B-A3B.w8a8" + +# INT4 models +DEFAULT_MODEL_NAME_FOR_TEST_AWQ_INT4 = ( + "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4" +) + +# EAGLE +DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST = "meta-llama/Llama-2-7b-chat-hf" +DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST = "lmsys/sglang-EAGLE-llama2-chat-7B" +DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST_EAGLE3 = "meta-llama/Llama-3.1-8B-Instruct" +DEFAULT_EAGLE_DP_ATTENTION_TARGET_MODEL_FOR_TEST = "Qwen/Qwen3-30B-A3B" +DEFAULT_EAGLE_DP_ATTENTION_DRAFT_MODEL_FOR_TEST = "Tengyunw/qwen3_30b_moe_eagle3" +DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3 = "lmsys/sglang-EAGLE3-LLaMA3.1-Instruct-8B" +DEFAULT_STANDALONE_SPECULATIVE_TARGET_MODEL_FOR_TEST = ( + "meta-llama/Llama-3.1-8B-Instruct" +) +DEFAULT_STANDALONE_SPECULATIVE_DRAFT_MODEL_FOR_TEST = "meta-llama/Llama-3.2-1B-Instruct" +DEFAULT_NGRAM_SPECULATIVE_TARGET_MODEL_FOR_TEST = "Qwen/Qwen2.5-Coder-7B-Instruct" + +# Other use cases +DEFAULT_AUTOROUND_MODEL_NAME_FOR_TEST = ( + "OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc", # auto_round:auto_gptq + "Intel/Qwen2-0.5B-Instruct-int4-sym-AutoRound", # auto_round:auto_awq +) +DEFAULT_MODEL_NAME_FOR_TEST_LOCAL_ATTENTION = ( + "meta-llama/Llama-4-Scout-17B-16E-Instruct" +) +DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST = "Alibaba-NLP/gte-Qwen2-1.5B-instruct" +DEFAULT_REASONING_MODEL_NAME_FOR_TEST = "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B" +DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST = "deepseek-ai/DeepSeek-V3-0324" +DEFAULT_AWQ_MOE_MODEL_NAME_FOR_TEST = ( + "hugging-quants/Mixtral-8x7B-Instruct-v0.1-AWQ-INT4" +) +DEFAULT_ENABLE_THINKING_MODEL_NAME_FOR_TEST = "Qwen/Qwen3-30B-A3B" +DEFAULT_DEEPSEEK_W4AFP8_MODEL_FOR_TEST = "Barrrrry/DeepSeek-R1-W4AFP8" + +# Nightly tests +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 = "meta-llama/Llama-3.1-8B-Instruct,mistralai/Mistral-7B-Instruct-v0.3,deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct,google/gemma-2-27b-it" +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2 = "meta-llama/Llama-3.1-70B-Instruct,mistralai/Mixtral-8x7B-Instruct-v0.1,Qwen/Qwen2-57B-A14B-Instruct" +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1 = "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8,neuralmagic/Mistral-7B-Instruct-v0.3-FP8,neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8,neuralmagic/gemma-2-2b-it-FP8" +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2 = "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8,neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8,neuralmagic/Qwen2-72B-Instruct-FP8,neuralmagic/Qwen2-57B-A14B-Instruct-FP8,neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8,zai-org/GLM-4.5-Air-FP8" +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_QUANT_TP1 = "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4,hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4,hugging-quants/Mixtral-8x7B-Instruct-v0.1-AWQ-INT4" +DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN = "Qwen/Qwen2.5-1.5B-Instruct" +DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST = "Qwen/Qwen2.5-VL-3B-Instruct" + +DEFAULT_IMAGE_URL = "https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true" +DEFAULT_VIDEO_URL = "https://raw.githubusercontent.com/EvolvingLMMs-Lab/sglang/dev/onevision_local/assets/jobs.mp4" + +DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 600 + + +def download_image_with_retry(image_url: str, max_retries: int = 3) -> Image.Image: + for i in range(max_retries): + try: + response = requests.get(image_url, timeout=30) + response.raise_for_status() + image = Image.open(BytesIO(response.content)) + image.load() + return image + except Exception as e: + if i == max_retries - 1: + raise RuntimeError( + f"Failed to download image after {max_retries} retries: {image_url}" + ) from e + time.sleep(2**i) + + +def is_in_ci(): + """Return whether it is in CI runner.""" + return get_bool_env_var("SGLANG_IS_IN_CI") + + +def is_in_amd_ci(): + """Return whether it is in an AMD CI runner.""" + return get_bool_env_var("SGLANG_IS_IN_CI_AMD") + + +def is_blackwell_system(): + """Return whether it is running on a Blackwell (B200) system.""" + return get_bool_env_var("IS_BLACKWELL") + + +def _use_cached_default_models(model_repo: str): + cache_dir = os.getenv("DEFAULT_MODEL_CACHE_DIR") + if cache_dir and model_repo: + model_path = os.path.join(cache_dir, model_repo) + if os.path.isdir(model_path): + return os.path.abspath(model_path) + return "" + + +if is_in_ci(): + DEFAULT_PORT_FOR_SRT_TEST_RUNNER = ( + 10000 + int(os.environ.get("CUDA_VISIBLE_DEVICES", "0")[0]) * 2000 + ) +else: + DEFAULT_PORT_FOR_SRT_TEST_RUNNER = ( + 20000 + int(os.environ.get("CUDA_VISIBLE_DEVICES", "0")[0]) * 1000 + ) +DEFAULT_URL_FOR_TEST = f"http://127.0.0.1:{DEFAULT_PORT_FOR_SRT_TEST_RUNNER + 1000}" + +if is_in_amd_ci(): + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 3000 + +if is_blackwell_system(): + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 3000 + + +def call_generate_lightllm(prompt, temperature, max_tokens, stop=None, url=None): + assert url is not None + + data = { + "inputs": prompt, + "parameters": { + "temperature": temperature, + "max_new_tokens": max_tokens, + "stop_sequences": stop, + }, + } + res = requests.post(url, json=data) + assert res.status_code == 200 + pred = res.json()["generated_text"][0] + return pred + + +def find_available_port(base_port: int): + port = base_port + random.randint(100, 1000) + while True: + if is_port_available(port): + return port + if port < 60000: + port += 42 + else: + port -= 43 + + +def call_generate_vllm(prompt, temperature, max_tokens, stop=None, n=1, url=None): + assert url is not None + + data = { + "prompt": prompt, + "temperature": temperature, + "max_tokens": max_tokens, + "stop": stop, + "n": n, + } + res = requests.post(url, json=data) + assert res.status_code == 200 + if n == 1: + pred = res.json()["text"][0][len(prompt) :] + else: + pred = [x[len(prompt) :] for x in res.json()["text"]] + return pred + + +def call_generate_outlines( + prompt, temperature, max_tokens, stop=None, regex=None, n=1, url=None +): + assert url is not None + + data = { + "prompt": prompt, + "temperature": temperature, + "max_tokens": max_tokens, + "stop": stop, + "regex": regex, + "n": n, + } + res = requests.post(url, json=data) + assert res.status_code == 200 + if n == 1: + pred = res.json()["text"][0][len(prompt) :] + else: + pred = [x[len(prompt) :] for x in res.json()["text"]] + return pred + + +def call_generate_srt_raw(prompt, temperature, max_tokens, stop=None, url=None): + assert url is not None + + data = { + "text": prompt, + "sampling_params": { + "temperature": temperature, + "max_new_tokens": max_tokens, + "stop": stop, + }, + } + res = requests.post(url, json=data) + assert res.status_code == 200 + obj = res.json() + pred = obj["text"] + return pred + + +def call_generate_guidance( + prompt, temperature, max_tokens, stop=None, n=1, regex=None, model=None +): + assert model is not None + from guidance import gen + + rets = [] + for _ in range(n): + out = ( + model + + prompt + + gen( + name="answer", + max_tokens=max_tokens, + temperature=temperature, + stop=stop, + regex=regex, + ) + ) + rets.append(out["answer"]) + return rets if n > 1 else rets[0] + + +def call_select_lightllm(context, choices, url=None): + assert url is not None + + scores = [] + for i in range(len(choices)): + data = { + "inputs": context + choices[i], + "parameters": { + "max_new_tokens": 1, + }, + } + res = requests.post(url, json=data) + assert res.status_code == 200 + scores.append(0) + return np.argmax(scores) + + +def call_select_vllm(context, choices, url=None): + assert url is not None + + scores = [] + for i in range(len(choices)): + data = { + "prompt": context + choices[i], + "max_tokens": 1, + "prompt_logprobs": 1, + } + res = requests.post(url, json=data) + assert res.status_code == 200 + scores.append(res.json().get("prompt_score", 0)) + return np.argmax(scores) + + """ + Modify vllm/entrypoints/api_server.py + + if final_output.prompt_logprobs is not None: + score = np.mean([prob[t_id] for t_id, prob in zip(final_output.prompt_token_ids[1:], final_output.prompt_logprobs[1:])]) + ret["prompt_score"] = score + """ + + +def call_select_guidance(context, choices, model=None): + assert model is not None + from guidance import select + + out = model + context + select(choices, name="answer") + return choices.index(out["answer"]) + + +def add_common_other_args_and_parse(parser: argparse.ArgumentParser): + parser.add_argument("--parallel", type=int, default=64) + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=None) + parser.add_argument( + "--backend", + type=str, + required=True, + choices=[ + "vllm", + "outlines", + "lightllm", + "gserver", + "guidance", + "srt-raw", + "llama.cpp", + ], + ) + parser.add_argument("--n-ctx", type=int, default=4096) + parser.add_argument( + "--model-path", type=str, default="meta-llama/Llama-2-7b-chat-hf" + ) + parser.add_argument("--result-file", type=str, default="result.jsonl") + args = parser.parse_args() + + if args.port is None: + default_port = { + "vllm": 21000, + "outlines": 21000, + "lightllm": 22000, + "srt-raw": 30000, + "gserver": 9988, + } + args.port = default_port.get(args.backend, None) + return args + + +def auto_config_device() -> str: + """Auto-config available device platform""" + + try: + device = get_device() + except (RuntimeError, ImportError) as e: + print(f"Warning: {e} - Falling back to CPU") + device = "cpu" + + return device + + +def add_common_sglang_args_and_parse(parser: argparse.ArgumentParser): + parser.add_argument("--parallel", type=int, default=64) + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=30000) + parser.add_argument("--backend", type=str, default="srt") + parser.add_argument( + "--device", + type=str, + default="auto", + choices=["auto", "cuda", "rocm", "cpu"], + help="Device type (auto/cuda/rocm/cpu). Auto will detect available platforms", + ) + parser.add_argument("--result-file", type=str, default="result.jsonl") + parser.add_argument("--raw-result-file", type=str) + args = parser.parse_args() + + return args + + +def select_sglang_backend(args: argparse.Namespace): + from sglang.lang.backend.openai import OpenAI + from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint + + if args.backend.startswith("srt"): + if args.backend == "srt-no-parallel": + global_config.enable_parallel_encoding = False + backend = RuntimeEndpoint(f"{args.host}:{args.port}") + elif args.backend.startswith("gpt-"): + backend = OpenAI(args.backend) + else: + raise ValueError(f"Invalid backend: {args.backend}") + return backend + + +def _get_call_generate(args: argparse.Namespace): + if args.backend == "lightllm": + return partial(call_generate_lightllm, url=f"{args.host}:{args.port}/generate") + elif args.backend == "vllm": + return partial(call_generate_vllm, url=f"{args.host}:{args.port}/generate") + elif args.backend == "srt-raw": + return partial(call_generate_srt_raw, url=f"{args.host}:{args.port}/generate") + elif args.backend == "outlines": + return partial(call_generate_outlines, url=f"{args.host}:{args.port}/generate") + elif args.backend == "guidance": + from guidance import models + + model = models.LlamaCpp(args.model_path, n_gpu_layers=-1, n_ctx=args.n_ctx) + call_generate = partial(call_generate_guidance, model=model) + call_generate("Hello,", 1.0, 8, ".") + return call_generate + else: + raise ValueError(f"Invalid backend: {args.backend}") + + +def _get_call_select(args: argparse.Namespace): + if args.backend == "lightllm": + return partial(call_select_lightllm, url=f"{args.host}:{args.port}/generate") + elif args.backend == "vllm": + return partial(call_select_vllm, url=f"{args.host}:{args.port}/generate") + elif args.backend == "guidance": + from guidance import models + + model = models.LlamaCpp(args.model_path, n_gpu_layers=-1, n_ctx=args.n_ctx) + call_select = partial(call_select_guidance, model=model) + + call_select("Hello,", ["world", "earth"]) + return call_select + else: + raise ValueError(f"Invalid backend: {args.backend}") + + +def get_call_generate(args: argparse.Namespace): + call_generate = _get_call_generate(args) + + def func(*args, **kwargs): + try: + return call_generate(*args, **kwargs) + except Exception: + print("Exception in call_generate:\n" + get_exception_traceback()) + raise + + return func + + +def get_call_select(args: argparse.Namespace): + call_select = _get_call_select(args) + + def func(*args, **kwargs): + try: + return call_select(*args, **kwargs) + except Exception: + print("Exception in call_select:\n" + get_exception_traceback()) + raise + + return func + + +def _get_default_models(): + import inspect + + current_module = inspect.getmodule(_get_default_models) + default_models = set() + for name, value in current_module.__dict__.items(): + if ( + isinstance(name, str) + and "DEFAULT_" in name + and "MODEL_" in name + and isinstance(value, str) + ): + if "," in value: + parts = [part.strip() for part in value.split(",")] + default_models.update(parts) + else: + default_models.add(value.strip()) + return json.dumps(list(default_models)) + + +def try_cached_model(model_repo: str): + model_dir = _use_cached_default_models(model_repo) + return model_dir if model_dir else model_repo + + +def popen_with_error_check(command: list[str], allow_exit: bool = False): + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + def _run_and_check(): + stdout, stderr = process.communicate() + + while process.poll() is None: + time.sleep(5) + + if not allow_exit or process.returncode != 0: + raise Exception( + f"{command} exited with code {process.returncode}\n{stdout=}\n{stderr=}" + ) + + t = threading.Thread(target=_run_and_check) + t.start() + return process + + +def popen_launch_server( + model: str, + base_url: str, + timeout: float, + api_key: Optional[str] = None, + other_args: Optional[list[str]] = None, + env: Optional[dict] = None, + return_stdout_stderr: Optional[tuple] = None, + device: str = "auto", + pd_separated: bool = False, + num_replicas: Optional[int] = None, +): + """Launch a server process with automatic device detection. + + Args: + device: Device type ("auto", "cuda", "rocm" or "cpu"). + If "auto", will detect available platforms automatically. + """ + other_args = other_args or [] + + # Auto-detect device if needed + if device == "auto": + device = auto_config_device() + other_args = list(other_args) + other_args += ["--device", str(device)] + + _, host, port = base_url.split(":") + host = host[2:] + + use_mixed_pd_engine = not pd_separated and num_replicas is not None + if pd_separated or use_mixed_pd_engine: + command = "sglang.launch_pd_server" + else: + command = "sglang.launch_server" + + command = [ + "python3", + "-m", + command, + "--model-path", + model, + *[str(x) for x in other_args], + ] + + if pd_separated or use_mixed_pd_engine: + command.extend( + [ + "--lb-host", + host, + "--lb-port", + port, + ] + ) + else: + command.extend( + [ + "--host", + host, + "--port", + port, + ] + ) + + if use_mixed_pd_engine: + command.extend( + [ + "--mixed", + "--num-replicas", + str(num_replicas), + ] + ) + + if api_key: + command += ["--api-key", api_key] + + print(f"command={' '.join(command)}") + + if return_stdout_stderr: + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + text=True, + bufsize=1, + ) + + def _dump(src, sinks): + for line in iter(src.readline, ""): + for sink in sinks: + sink.write(line) + sink.flush() + src.close() + + threading.Thread( + target=_dump, + args=(process.stdout, [return_stdout_stderr[0], sys.stdout]), + daemon=True, + ).start() + threading.Thread( + target=_dump, + args=(process.stderr, [return_stdout_stderr[1], sys.stderr]), + daemon=True, + ).start() + else: + process = subprocess.Popen(command, stdout=None, stderr=None, env=env) + + start_time = time.perf_counter() + with requests.Session() as session: + while time.perf_counter() - start_time < timeout: + return_code = process.poll() + if return_code is not None: + # Server failed to start (non-zero exit code) or crashed + raise Exception( + f"Server process exited with code {return_code}. " + "Check server logs for errors." + ) + + try: + headers = { + "Content-Type": "application/json; charset=utf-8", + "Authorization": f"Bearer {api_key}", + } + response = session.get( + f"{base_url}/health_generate", + headers=headers, + ) + if response.status_code == 200: + return process + except requests.RequestException: + pass + + return_code = process.poll() + if return_code is not None: + raise Exception( + f"Server unexpectedly exits ({return_code=}). Usually there will be error logs describing the cause far above this line." + ) + + time.sleep(10) + + kill_process_tree(process.pid) + raise TimeoutError("Server failed to start within the timeout period.") + + +def popen_launch_pd_server( + model: str, + base_url: str, + timeout: float, + api_key: Optional[str] = None, + other_args: list[str] = (), + env: Optional[dict] = None, +): + _, host, port = base_url.split(":") + host = host[2:] + + command = "sglang.launch_server" + + command = [ + "python3", + "-m", + command, + "--model-path", + model, + *[str(x) for x in other_args], + ] + + command.extend( + [ + "--host", + host, + "--port", + port, + ] + ) + + if api_key: + command += ["--api-key", api_key] + + print(f"command={' '.join(command)}") + + process = subprocess.Popen(command, stdout=None, stderr=None, env=env) + + return process + + +def get_similarities(vec1, vec2): + return F.cosine_similarity(torch.tensor(vec1), torch.tensor(vec2), dim=0) + + +def get_benchmark_args( + base_url="", + dataset_name="", + dataset_path="", + tokenizer="", + num_prompts=500, + sharegpt_output_len=None, + random_input_len=4096, + random_output_len=2048, + sharegpt_context_len=None, + request_rate=float("inf"), + disable_stream=False, + disable_ignore_eos=False, + seed: int = 0, + device="auto", + pd_separated: bool = False, + lora_name=None, + lora_request_distribution="uniform", + lora_zipf_alpha=1.5, +): + return SimpleNamespace( + backend="sglang", + base_url=base_url, + host=None, + port=None, + dataset_name=dataset_name, + dataset_path=dataset_path, + model=None, + tokenizer=tokenizer, + num_prompts=num_prompts, + sharegpt_output_len=sharegpt_output_len, + sharegpt_context_len=sharegpt_context_len, + random_input_len=random_input_len, + random_output_len=random_output_len, + random_range_ratio=0.0, + request_rate=request_rate, + multi=None, + output_file=None, + disable_tqdm=False, + disable_stream=disable_stream, + return_logprob=False, + seed=seed, + disable_ignore_eos=disable_ignore_eos, + extra_request_body=None, + apply_chat_template=False, + profile=None, + lora_name=lora_name, + lora_request_distribution=lora_request_distribution, + lora_zipf_alpha=lora_zipf_alpha, + prompt_suffix="", + device=device, + pd_separated=pd_separated, + ) + + +def run_bench_serving( + model, + num_prompts, + request_rate, + other_server_args, + dataset_name="random", + dataset_path="", + tokenizer=None, + random_input_len=4096, + random_output_len=2048, + sharegpt_context_len=None, + disable_stream=False, + disable_ignore_eos=False, + need_warmup=False, + seed: int = 0, + device="auto", + background_task: Optional[Callable[[str, asyncio.Event], Awaitable[None]]] = None, + lora_name: Optional[str] = None, +): + if device == "auto": + device = auto_config_device() + # Launch the server + base_url = DEFAULT_URL_FOR_TEST + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_server_args, + ) + + # Run benchmark + args = get_benchmark_args( + base_url=base_url, + dataset_name=dataset_name, + dataset_path=dataset_path, + tokenizer=tokenizer, + num_prompts=num_prompts, + random_input_len=random_input_len, + random_output_len=random_output_len, + sharegpt_context_len=sharegpt_context_len, + request_rate=request_rate, + disable_stream=disable_stream, + disable_ignore_eos=disable_ignore_eos, + seed=seed, + device=device, + lora_name=lora_name, + ) + + async def _run(): + if need_warmup: + warmup_args = copy.deepcopy(args) + warmup_args.num_prompts = 16 + await asyncio.to_thread(run_benchmark, warmup_args) + + start_event = asyncio.Event() + stop_event = asyncio.Event() + task_handle = ( + asyncio.create_task(background_task(base_url, start_event, stop_event)) + if background_task + else None + ) + + try: + start_event.set() + result = await asyncio.to_thread(run_benchmark, args) + finally: + if task_handle: + stop_event.set() + await task_handle + + return result + + try: + res = asyncio.run(_run()) + finally: + kill_process_tree(process.pid) + + assert res["completed"] == num_prompts + return res + + +async def _run_api_benchmark_requests( + base_url: str, + endpoint: str, + test_requests: List[dict], + num_requests: int, + response_validator: Callable[[dict], bool], +): + """ + Helper function to run API benchmark requests and collect metrics. + + Args: + base_url: The base URL of the server + endpoint: The API endpoint to test (e.g., "/v1/score", "/v1/embeddings") + test_requests: List of request payloads to send + num_requests: Total number of requests expected + response_validator: Function to validate if response contains expected data + + Returns: + Dictionary with benchmark metrics + """ + start_time = time.monotonic() + successful_requests = 0 + total_latency = 0 + latencies = [] + + async with aiohttp.ClientSession() as session: + for request_data in test_requests: + try: + request_start = time.monotonic() + async with session.post( + f"{base_url}{endpoint}", + json=request_data, + timeout=aiohttp.ClientTimeout(total=30), + ) as response: + if response.status == 200: + response_data = await response.json() + request_end = time.monotonic() + + if response_validator(response_data): + latency_ms = (request_end - request_start) * 1000 + latencies.append(latency_ms) + total_latency += latency_ms + successful_requests += 1 + except Exception: + continue + + end_time = time.monotonic() + total_time = end_time - start_time + + if successful_requests > 0: + throughput = successful_requests / total_time + avg_latency = total_latency / successful_requests + p95_latency = np.percentile(latencies, 95) if latencies else 0 + + return { + "completed": successful_requests, + "total_requests": num_requests, + "throughput": throughput, + "avg_latency_ms": avg_latency, + "p95_latency_ms": p95_latency, + "successful_requests": successful_requests, + } + else: + return { + "completed": 0, + "total_requests": num_requests, + "throughput": 0, + "avg_latency_ms": 0, + "p95_latency_ms": 0, + "successful_requests": 0, + } + + +def run_score_benchmark( + model, + num_requests=100, + batch_size=5, + other_server_args=None, + need_warmup=False, + device="auto", +): + """Score API benchmark function compatible with run_bench_serving pattern""" + if other_server_args is None: + other_server_args = [] + + if device == "auto": + device = auto_config_device() + + # Launch the server (consistent with run_bench_serving) + base_url = DEFAULT_URL_FOR_TEST + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_server_args, + ) + + async def _run_benchmark(): + # Load tokenizer for generating test data + from sglang.srt.utils.hf_transformers_utils import get_tokenizer + + tokenizer = get_tokenizer(model) + + # Score API configuration + score_query_tokens = 120 + score_item_tokens = 180 + score_label_token_ids = [9454, 2753] # Yes/No token IDs + special_token = "<|im_start|>" + + def generate_text_with_token_count(num_tokens): + """Generate text with precise token count using replicated token.""" + text = special_token * num_tokens + actual_tokens = len(tokenizer.encode(text, add_special_tokens=False)) + if actual_tokens != num_tokens: + text = special_token * ( + num_tokens + // len(tokenizer.encode(special_token, add_special_tokens=False)) + ) + return text + + if need_warmup: + warmup_data = { + "query": generate_text_with_token_count(score_query_tokens), + "items": [ + generate_text_with_token_count(score_item_tokens) for _ in range(3) + ], + "label_token_ids": score_label_token_ids, + "model": model, + "apply_softmax": True, + } + + async with aiohttp.ClientSession() as session: + try: + await session.post( + f"{base_url}/v1/score", + json=warmup_data, + timeout=aiohttp.ClientTimeout(total=30), + ) + except: + pass # Ignore warmup errors + + test_requests = [] + for i in range(num_requests): + query = generate_text_with_token_count(score_query_tokens) + items = [ + generate_text_with_token_count(score_item_tokens) + for _ in range(batch_size) + ] + + score_data = { + "query": query, + "items": items, + "label_token_ids": score_label_token_ids, + "model": model, + "apply_softmax": True, + } + test_requests.append(score_data) + + # Run benchmark requests using shared helper + return await _run_api_benchmark_requests( + base_url=base_url, + endpoint="/v1/score", + test_requests=test_requests, + num_requests=num_requests, + response_validator=lambda resp: "scores" in resp or "logprobs" in resp, + ) + + try: + res = asyncio.run(_run_benchmark()) + finally: + kill_process_tree(process.pid) + + assert res["completed"] == res["successful_requests"] + return res + + +def run_embeddings_benchmark( + model, + num_requests=100, + batch_size=1, + input_tokens=500, + other_server_args=None, + need_warmup=False, + device="auto", +): + """Embeddings API benchmark function compatible with run_bench_serving pattern""" + if other_server_args is None: + other_server_args = [] + + if device == "auto": + device = auto_config_device() + + # Add --is-embedding flag for embedding models + server_args = ["--is-embedding"] + other_server_args + + # Launch the server (consistent with run_bench_serving) + base_url = DEFAULT_URL_FOR_TEST + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=server_args, + ) + + async def _run_benchmark(): + + # Load tokenizer for generating test data + from sglang.srt.utils.hf_transformers_utils import get_tokenizer + + tokenizer = get_tokenizer(model) + + def generate_text_with_token_count(num_tokens): + """Generate text with precise token count using special tokens.""" + # Use a token that reliably produces 1 token + special_token = "<|im_start|>" + # Verify it's a single token + test_tokens = tokenizer.encode(special_token, add_special_tokens=False) + text = special_token * num_tokens + return text + + # Generate input text + input_text = generate_text_with_token_count(input_tokens) + + if need_warmup: + warmup_data = { + "input": input_text, + "model": model, + } + + async with aiohttp.ClientSession() as session: + try: + await session.post( + f"{base_url}/v1/embeddings", + json=warmup_data, + timeout=aiohttp.ClientTimeout(total=30), + ) + except: + pass # Ignore warmup errors + + test_requests = [] + for i in range(num_requests): + if batch_size == 1: + input_data = input_text + else: + input_data = [input_text for _ in range(batch_size)] + + embeddings_data = { + "input": input_data, + "model": model, + } + + test_requests.append(embeddings_data) + + # Run benchmark requests using shared helper + return await _run_api_benchmark_requests( + base_url=base_url, + endpoint="/v1/embeddings", + test_requests=test_requests, + num_requests=num_requests, + response_validator=lambda resp: "data" in resp, + ) + + try: + res = asyncio.run(_run_benchmark()) + finally: + kill_process_tree(process.pid) + + assert res["completed"] == res["successful_requests"] + return res + + +def run_bench_serving_multi( + model, + base_url, + other_server_args, + benchmark_args, + need_warmup=False, + pd_separated=False, +): + # Launch the server + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_server_args, + pd_separated=pd_separated, + ) + + # run benchmark for all + res_l = [] + try: + for args in benchmark_args: + if need_warmup: + warmup_args = copy.deepcopy(args) + warmup_args.num_prompts = 16 + run_benchmark(warmup_args) + + res = run_benchmark(args) + res_l.append((args, res)) + finally: + kill_process_tree(process.pid) + + return res_l + + +def run_bench_one_batch(model, other_args): + """Launch a offline process with automatic device detection. + + Args: + device: Device type ("auto", "cuda", "rocm" or "cpu"). + If "auto", will detect available platforms automatically. + """ + # Auto-detect device if needed + + device = auto_config_device() + print(f"Auto-configed device: {device}", flush=True) + other_args += ["--device", str(device)] + + command = [ + "python3", + "-m", + "sglang.bench_one_batch", + "--batch-size", + "1", + "--input", + "128", + "--output", + "8", + *[str(x) for x in other_args], + ] + if model is not None: + command += ["--model-path", model] + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + try: + stdout, stderr = process.communicate() + output = stdout.decode() + error = stderr.decode() + print(f"Output: {output}", flush=True) + print(f"Error: {error}", flush=True) + + # Return prefill_latency, decode_throughput, decode_latency + prefill_line = output.split("\n")[-9] + decode_line = output.split("\n")[-3] + pattern = ( + r"latency: (?P\d+\.\d+).*?throughput:\s*(?P\d+\.\d+)" + ) + match = re.search(pattern, prefill_line) + if match: + prefill_latency = float(match.group("latency")) + match = re.search(pattern, decode_line) + if match: + decode_latency = float(match.group("latency")) + decode_throughput = float(match.group("throughput")) + finally: + kill_process_tree(process.pid) + + return prefill_latency, decode_throughput, decode_latency + + +def run_bench_offline_throughput(model, other_args): + command = [ + "python3", + "-m", + "sglang.bench_offline_throughput", + "--num-prompts", + "1", + "--dataset-name", + "random", + "--random-input-len", + "256", + "--random-output-len", + "256", + "--model-path", + model, + *[str(x) for x in other_args], + ] + + print(f"command={' '.join(command)}") + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + try: + stdout, stderr = process.communicate() + output = stdout.decode(errors="backslashreplace") + error = stderr.decode(errors="backslashreplace") + print(f"Output: {output}", flush=True) + print(f"Error: {error}", flush=True) + + output_throughput = -1 + for line in output.split("\n"): + if "Last generation throughput (tok/s):" in line: + output_throughput = float(line.split(":")[-1]) + finally: + kill_process_tree(process.pid) + + return output_throughput + + +def run_bench_one_batch_server( + model, + base_url, + server_args, + bench_args, + other_server_args, + simulate_spec_acc_lens=None, +): + from sglang.bench_one_batch_server import run_benchmark + + if simulate_spec_acc_lens is not None: + env = {**os.environ, "SIMULATE_ACC_LEN": str(simulate_spec_acc_lens)} + else: + env = None + + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_server_args, + env=env, + ) + try: + run_benchmark(server_args=server_args, bench_args=bench_args) + finally: + kill_process_tree(process.pid) + + +def lcs(X, Y): + m = len(X) + n = len(Y) + L = [[0] * (n + 1) for _ in range(m + 1)] + + for i in range(m + 1): + for j in range(n + 1): + if i == 0 or j == 0: + L[i][j] = 0 + elif X[i - 1] == Y[j - 1]: + L[i][j] = L[i - 1][j - 1] + 1 + else: + L[i][j] = max(L[i - 1][j], L[i][j - 1]) + + return L[m][n] + + +def calculate_rouge_l(output_strs_list1, output_strs_list2): + """calculate the ROUGE-L score""" + rouge_l_scores = [] + + for s1, s2 in zip(output_strs_list1, output_strs_list2): + lcs_len = lcs(s1, s2) + precision = lcs_len / len(s1) if len(s1) > 0 else 0 + recall = lcs_len / len(s2) if len(s2) > 0 else 0 + if precision + recall > 0: + fmeasure = (2 * precision * recall) / (precision + recall) + else: + fmeasure = 0.0 + rouge_l_scores.append(fmeasure) + + return rouge_l_scores + + +STDERR_FILENAME = "/tmp/stderr.txt" +STDOUT_FILENAME = "/tmp/stdout.txt" + + +def read_output(output_lines: List[str], filename: str = STDERR_FILENAME): + """Print the output in real time with another thread.""" + while not os.path.exists(filename): + time.sleep(0.01) + + pt = 0 + while pt >= 0: + if pt > 0 and not os.path.exists(filename): + break + try: + lines = open(filename).readlines() + except FileNotFoundError: + print(f"{pt=}, {os.path.exists(filename)=}") + raise + for line in lines[pt:]: + print(line, end="", flush=True) + output_lines.append(line) + pt += 1 + time.sleep(0.1) + + +def run_and_check_memory_leak( + workload_func, + disable_radix_cache, + enable_mixed_chunk, + disable_overlap, + chunked_prefill_size, + assert_has_abort, +): + other_args = [ + "--chunked-prefill-size", + str(chunked_prefill_size), + "--log-level", + "debug", + ] + if disable_radix_cache: + other_args += ["--disable-radix-cache"] + if enable_mixed_chunk: + other_args += ["--enable-mixed-chunk"] + if disable_overlap: + other_args += ["--disable-overlap-schedule"] + + model = DEFAULT_MODEL_NAME_FOR_TEST + port = random.randint(4000, 5000) + base_url = f"http://127.0.0.1:{port}" + + # Create files and launch the server + stdout = open(STDOUT_FILENAME, "w") + stderr = open(STDERR_FILENAME, "w") + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + return_stdout_stderr=(stdout, stderr), + ) + + # Launch a thread to stream the output + output_lines = [] + t = threading.Thread(target=read_output, args=(output_lines,)) + t.start() + + # Run the workload + workload_func(base_url, model) + + # Clean up everything + kill_process_tree(process.pid) + stdout.close() + stderr.close() + if os.path.exists(STDOUT_FILENAME): + os.remove(STDOUT_FILENAME) + if os.path.exists(STDERR_FILENAME): + os.remove(STDERR_FILENAME) + kill_process_tree(process.pid) + t.join() + + # Assert success + has_new_server = False + has_leak = False + has_abort = False + for line in output_lines: + if "Uvicorn running" in line: + has_new_server = True + if "leak" in line: + has_leak = True + if "Abort" in line: + has_abort = True + + assert has_new_server + assert not has_leak + if assert_has_abort: + assert has_abort + + +def run_command_and_capture_output(command, env: Optional[dict] = None): + stdout = open(STDOUT_FILENAME, "w") + stderr = open(STDERR_FILENAME, "w") + process = subprocess.Popen( + command, stdout=stdout, stderr=stdout, env=env, text=True + ) + + # Launch a thread to stream the output + output_lines = [] + t = threading.Thread(target=read_output, args=(output_lines, STDOUT_FILENAME)) + t.start() + + # Join the process + process.wait() + + stdout.close() + stderr.close() + if os.path.exists(STDOUT_FILENAME): + os.remove(STDOUT_FILENAME) + if os.path.exists(STDERR_FILENAME): + os.remove(STDERR_FILENAME) + kill_process_tree(process.pid) + t.join() + + return output_lines + + +def run_mmlu_test( + disable_radix_cache=False, + enable_mixed_chunk=False, + disable_overlap=False, + chunked_prefill_size=32, +): + def workload_func(base_url, model): + # Run the eval + args = SimpleNamespace( + base_url=base_url, + model=model, + eval_name="mmlu", + num_examples=128, + num_threads=128, + ) + + try: + metrics = run_eval(args) + assert metrics["score"] >= 0.65, f"{metrics=}" + finally: + pass + + run_and_check_memory_leak( + workload_func, + disable_radix_cache, + enable_mixed_chunk, + disable_overlap, + chunked_prefill_size, + assert_has_abort=False, + ) + + +def run_mulit_request_test( + disable_radix_cache=False, + enable_mixed_chunk=False, + enable_overlap=False, + chunked_prefill_size=32, +): + def workload_func(base_url, model): + def run_one(_): + prompt = """ + System: You are a helpful assistant. + User: What is the capital of France? + Assistant: The capital of France is + """ + + response = requests.post( + f"{base_url}/generate", + json={ + "text": prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 8, + }, + }, + ) + ret = response.json() + + with ThreadPoolExecutor(2) as executor: + list(executor.map(run_one, list(range(4)))) + + run_and_check_memory_leak( + workload_func, + disable_radix_cache, + enable_mixed_chunk, + enable_overlap, + chunked_prefill_size, + assert_has_abort=False, + ) + + +def write_github_step_summary(content): + if not os.environ.get("GITHUB_STEP_SUMMARY"): + logging.warning("GITHUB_STEP_SUMMARY environment variable not set") + return + + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: + f.write(content) + + +def run_logprob_check(self: unittest.TestCase, arg: Tuple): + ( + input_len, + output_len, + temperature, + logprob_start_len, + return_logprob, + top_logprobs_num, + ) = arg + input_ids = list(range(input_len)) + + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": input_ids, + "sampling_params": { + "temperature": temperature, + "max_new_tokens": output_len, + "ignore_eos": True, + }, + "return_logprob": return_logprob, + "logprob_start_len": logprob_start_len, + "top_logprobs_num": top_logprobs_num, + }, + ) + response_json = response.json() + + res = response_json + self.assertEqual(res["meta_info"]["prompt_tokens"], input_len) + self.assertEqual(res["meta_info"]["completion_tokens"], output_len) + + # Test the number of tokens are correct + if return_logprob: + self.assertEqual( + len(res["meta_info"]["input_token_logprobs"]) + logprob_start_len, + res["meta_info"]["prompt_tokens"], + ) + self.assertEqual(len(res["meta_info"]["output_token_logprobs"]), output_len) + + if top_logprobs_num: + self.assertEqual( + len(res["meta_info"]["input_top_logprobs"]) + logprob_start_len, + res["meta_info"]["prompt_tokens"], + ) + self.assertEqual(len(res["meta_info"]["output_top_logprobs"]), output_len) + + for i in range(output_len): + self.assertEqual( + len(res["meta_info"]["output_top_logprobs"][i]), + top_logprobs_num, + ) + + # Test the top-1 tokens are the same as output tokens if temperature == 0 + if temperature == 0: + rank = 0 + while rank < len(res["meta_info"]["output_top_logprobs"][i]): + try: + self.assertListEqual( + res["meta_info"]["output_token_logprobs"][i], + res["meta_info"]["output_top_logprobs"][i][rank], + ) + break + except AssertionError: + # There's a tie. Allow the second item in this case. + if ( + res["meta_info"]["output_top_logprobs"][i][rank][0] + == res["meta_info"]["output_top_logprobs"][i][rank + 1][ + 0 + ] + ): + rank += 1 + else: + raise + + +def send_generate_requests(base_url: str, num_requests: int) -> List[str]: + """Sends generate request serially and returns status codes. Max concurrency is 1.""" + + def generate(): + prompt = """ + System: You are a helpful assistant. + User: What is the capital of France? + Assistant: The capital of France is + """ + response = requests.post( + f"{base_url}/generate", + json={ + "text": prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 500, + }, + }, + ) + return response.status_code + + return [generate() for _ in range(num_requests)] + + +async def send_concurrent_generate_requests( + base_url: str, num_requests: int +) -> List[str]: + """Sends generate request concurrently and returns status codes. Max concurrency is num_requests.""" + + async def async_generate(): + async with aiohttp.ClientSession() as session: + prompt = """ + System: You are a helpful assistant. + User: What is the capital of France? + Assistant: The capital of France is + """ + async with session.post( + f"{base_url}/generate", + json={ + "text": prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 500, + }, + }, + ) as response: + return response.status + + tasks = [asyncio.create_task(async_generate()) for _ in range(num_requests)] + return await asyncio.gather(*tasks) + + +async def send_concurrent_generate_requests_with_custom_params( + base_url: str, + custom_params: List[dict[str, Any]], +) -> Tuple[int, Any]: + """Sends generate request concurrently with custom parameters and returns status code and response json tuple. Max concurrency is num_requests.""" + + base_payload = { + "text": """ + System: You are a helpful assistant. + User: What is the capital of France? + Assistant: The capital of France is + """, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 500, + }, + } + + async def async_generate_with_priority(req): + async with aiohttp.ClientSession() as session: + async with session.post( + f"{base_url}/generate", + json=req, + ) as response: + resp_json = await response.json() + return (response.status, resp_json) + + tasks = [] + for c in custom_params: + req = base_payload.copy() + req.update(c) + tasks.append(asyncio.create_task(async_generate_with_priority(req))) + return await asyncio.gather(*tasks) + + +class CustomTestCase(unittest.TestCase): + def _callTestMethod(self, method): + max_retry = int( + os.environ.get("SGLANG_TEST_MAX_RETRY", "1" if is_in_ci() else "0") + ) + retry( + lambda: super(CustomTestCase, self)._callTestMethod(method), + max_retry=max_retry, + ) + + def setUp(self): + print(f"[CI Test Method] {self.__class__.__name__}.{self._testMethodName}") + + +def dump_bench_raw_result( + path: str, + states, + preds, + labels, +): + if not path: + return + + rows = [] + for i in range(len(states)): + state = states[i] + output = state["answer"] + prompt = _ensure_remove_suffix(state.text(), output) + rows.append( + dict( + prompt_id=i, + prompt=prompt, + output=output, + correct=bool(preds[i] == labels[i]), + ) + ) + + print(f"BenchRawResultDumper save results to {path}") + Path(path).write_text("\n".join(json.dumps(row) for row in rows)) + + +def _ensure_remove_suffix(text: str, suffix: str): + assert text.endswith(suffix) + return text.removesuffix(suffix) + + +class ModelLaunchSettings: + def __init__( + self, + model_path: str, + tp_size: int = 1, + extra_args: Optional[List[str]] = None, + env: Optional[dict] = None, + ): + self.model_path = model_path + self.tp_size = tp_size + self.extra_args = list(extra_args) if extra_args else [] + self.env = env + + if self.tp_size > 1 and "--tp" not in self.extra_args: + self.extra_args.extend(["--tp", str(self.tp_size)]) + + fixed_args = ["--enable-multimodal", "--trust-remote-code"] + for fixed_arg in fixed_args: + if fixed_arg not in self.extra_args: + self.extra_args.append(fixed_arg) + + +class ModelEvalMetrics: + def __init__(self, accuracy: float, eval_time: float): + self.accuracy = accuracy + self.eval_time = eval_time + + +def extract_trace_link_from_bench_one_batch_server_output(output: str) -> str: + match = re.search(r"\[Profile\]\((.*?)\)", output) + if match: + trace_link = match.group(1) + return trace_link + return None + + +def parse_models(model_string: str): + return [model.strip() for model in model_string.split(",") if model.strip()] + + +def check_evaluation_test_results( + results, + test_name, + model_accuracy_thresholds, + model_latency_thresholds=None, + model_count=None, +): + """ + results: list of tuple of (model_path, accuracy, latency) + """ + failed_models = [] + if model_latency_thresholds is not None: + summary = " | model | status | score | score_threshold | latency | latency_threshold | \n" + summary += "| ----- | ------ | ----- | --------------- | ------- | ----------------- | \n" + else: + summary = " | model | status | score | score_threshold | \n" + summary += "| ----- | ------ | ----- | --------------- | \n" + + results_dict = {res[0]: (res[1], res[2]) for res in results} + + for model, accuracy_threshold in sorted(model_accuracy_thresholds.items()): + latency_threshold = ( + model_latency_thresholds.get(model) + if model_latency_thresholds is not None + else 1e9 + ) + + if model in results_dict: + accuracy, latency = results_dict[model] + is_success = accuracy >= accuracy_threshold and latency <= latency_threshold + status_emoji = "✅" if is_success else "❌" + + if not is_success: + if accuracy < accuracy_threshold: + failed_models.append( + f"\nScore Check Failed: {model}\n" + f"Model {model} score ({accuracy:.4f}) is below threshold ({accuracy_threshold:.4f})" + ) + if latency > latency_threshold: + failed_models.append( + f"\nLatency Check Failed: {model}\n" + f"Model {model} latency ({latency:.4f}) is above threshold ({latency_threshold:.4f})" + ) + + if model_latency_thresholds is not None: + line = f"| {model} | {status_emoji} | {accuracy} | {accuracy_threshold} | {latency} | {latency_threshold}\n" + else: + line = ( + f"| {model} | {status_emoji} | {accuracy} | {accuracy_threshold}\n" + ) + else: + status_emoji = "❌" + failed_models.append(f"Model failed to launch or be evaluated: {model}") + if model_latency_thresholds is not None: + line = f"| {model} | {status_emoji} | N/A | {accuracy_threshold} | N/A | {latency_threshold}\n" + else: + line = f"| {model} | {status_emoji} | N/A | {accuracy_threshold}\n" + + summary += line + + print(summary) + + if is_in_ci(): + write_github_step_summary(f"## {test_name}\n{summary}") + + if failed_models: + print("Some models failed the evaluation.") + raise AssertionError("\n".join(failed_models)) + + +# Bench knobs for bench_one_batch_server (override by env) +def _parse_int_list_env(name: str, default_val: str): + val = os.environ.get(name, default_val) + return [int(x) for x in val.split(",") if x] + + +# Return filenames +def find_traces_under_path(path: str) -> List[str]: + results = [] + for _, dirs, files in os.walk(path): + for file in files: + if file.endswith(".trace.json.gz"): + results.append(f"{file}") + return results + + +def write_results_to_json(model, metrics, mode="a"): + result = { + "timestamp": datetime.now().isoformat(), + "model": model, + "metrics": metrics, + "score": metrics["score"], + } + + if "latency" in metrics: + result["latency"] = (metrics.get("latency"),) + + existing_results = [] + if mode == "a" and os.path.exists("results.json"): + try: + with open("results.json", "r") as f: + existing_results = json.load(f) + except json.JSONDecodeError: + existing_results = [] + + if isinstance(existing_results, list): + existing_results.append(result) + else: + existing_results = [result] + + with open("results.json", "w") as f: + json.dump(existing_results, f, indent=2) + + +def intel_amx_benchmark(extra_args=None, min_throughput=None): + def decorator(test_func): + @wraps(test_func) + def wrapper(self): + common_args = [ + "--attention-backend", + "intel_amx", + "--disable-radix", + "--trust-remote-code", + ] + full_args = common_args + (extra_args or []) + + model = test_func(self) + prefill_latency, decode_throughput, decode_latency = run_bench_one_batch( + model, full_args + ) + + print(f"{model=}") + print(f"{prefill_latency=}") + print(f"{decode_throughput=}") + print(f"{decode_latency=}") + + if is_in_ci() and min_throughput is not None: + self.assertGreater(decode_throughput, min_throughput) + + return wrapper + + return decorator diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/utils.py new file mode 100644 index 00000000..d378f22b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/utils.py @@ -0,0 +1,576 @@ +"""Common utilities""" + +import importlib +import json +import logging +import os +import random +import socket +import ssl +import subprocess +import sys +import time +import traceback +import urllib.request +import weakref +from collections import OrderedDict +from concurrent.futures import ThreadPoolExecutor +from functools import wraps +from io import BytesIO +from json import dumps +from typing import Any, Callable, List, Optional, Tuple, Type, Union + +import numpy as np +import pybase64 +import requests +from IPython.display import HTML, display +from pydantic import BaseModel +from tqdm import tqdm + +from sglang.srt.environ import envs + +logger = logging.getLogger(__name__) + + +def execute_once(func): + has_run = None + + @wraps(func) + def wrapper(*args, **kwargs): + nonlocal has_run + if not has_run: + func(*args, **kwargs) + has_run = True + + return wrapper + + +@execute_once +def info_once(message: str): + logger.info(message) + + +def convert_json_schema_to_str(json_schema: Union[dict, str, Type[BaseModel]]) -> str: + """Convert a JSON schema to a string. + Parameters + ---------- + json_schema + The JSON schema. + Returns + ------- + str + The JSON schema converted to a string. + Raises + ------ + ValueError + If the schema is not a dictionary, a string or a Pydantic class. + """ + if isinstance(json_schema, dict): + schema_str = json.dumps(json_schema) + elif isinstance(json_schema, str): + schema_str = json_schema + elif issubclass(json_schema, BaseModel): + schema_str = json.dumps(json_schema.model_json_schema()) + else: + raise ValueError( + f"Cannot parse schema {json_schema}. The schema must be either " + + "a Pydantic class, a dictionary or a string that contains the JSON " + + "schema specification" + ) + return schema_str + + +def get_exception_traceback(): + etype, value, tb = sys.exc_info() + err_str = "".join(traceback.format_exception(etype, value, tb)) + return err_str + + +def is_same_type(values: list): + """Return whether the elements in values are of the same type.""" + if len(values) <= 1: + return True + else: + t = type(values[0]) + return all(isinstance(v, t) for v in values[1:]) + + +def read_jsonl(filename: str): + """Read a JSONL file.""" + with open(filename) as fin: + for line in fin: + if line.startswith("#"): + continue + yield json.loads(line) + + +def dump_state_text(filename: str, states: list, mode: str = "w"): + """Dump program state in a text file.""" + from sglang.lang.interpreter import ProgramState + + with open(filename, mode) as fout: + for i, s in enumerate(states): + if isinstance(s, str): + pass + elif isinstance(s, ProgramState): + s = s.text() + else: + s = str(s) + + fout.write( + "=" * 40 + f" {i} " + "=" * 40 + "\n" + s + "\n" + "=" * 80 + "\n\n" + ) + + +class HttpResponse: + def __init__(self, resp): + self.resp = resp + + def json(self): + return json.loads(self.resp.read()) + + @property + def status_code(self): + return self.resp.status + + +def http_request( + url, + json=None, + stream=False, + api_key=None, + verify=None, + method: Optional[str] = None, +): + """A faster version of requests.post with low-level urllib API.""" + headers = {"Content-Type": "application/json; charset=utf-8"} + + # add the Authorization header if an api key is provided + if api_key is not None: + headers["Authorization"] = f"Bearer {api_key}" + + if stream: + return requests.post(url, json=json, stream=True, headers=headers) + else: + req = urllib.request.Request(url, headers=headers, method=method) + if json is None: + data = None + else: + data = bytes(dumps(json), encoding="utf-8") + + try: + if sys.version_info >= (3, 13): + # Python 3.13+: Use SSL context (cafile removed) + if verify and isinstance(verify, str): + context = ssl.create_default_context(cafile=verify) + else: + context = ssl.create_default_context() + resp = urllib.request.urlopen(req, data=data, context=context) + else: + resp = urllib.request.urlopen(req, data=data, cafile=verify) + return HttpResponse(resp) + except urllib.error.HTTPError as e: + return HttpResponse(e) + + +def encode_image_base64(image_path: Union[str, bytes]): + """Encode an image in base64.""" + if isinstance(image_path, str): + with open(image_path, "rb") as image_file: + data = image_file.read() + return pybase64.b64encode(data).decode("utf-8") + elif isinstance(image_path, bytes): + return pybase64.b64encode(image_path).decode("utf-8") + else: + # image_path is PIL.WebPImagePlugin.WebPImageFile + image = image_path + buffered = BytesIO() + image.save(buffered, format="PNG") + return pybase64.b64encode(buffered.getvalue()).decode("utf-8") + + +def encode_frame(frame): + import cv2 # pip install opencv-python-headless + from PIL import Image + + # Convert the frame to RGB (OpenCV uses BGR by default) + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + + # Convert the frame to PIL Image to easily convert to bytes + im_pil = Image.fromarray(frame) + + # Convert to bytes + buffered = BytesIO() + + # frame_format = str(os.getenv('FRAME_FORMAT', "JPEG")) + + im_pil.save(buffered, format="PNG") + + frame_bytes = buffered.getvalue() + + # Return the bytes of the frame + return frame_bytes + + +def encode_video_base64(video_path: str, num_frames: int = 16): + import cv2 # pip install opencv-python-headless + + cap = cv2.VideoCapture(video_path) + if not cap.isOpened(): + raise IOError(f"Could not open video file:{video_path}") + + total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) + print(f"target_frames: {num_frames}") + + frame_indices = np.linspace(0, total_frames - 1, num_frames, dtype=int) + + frames = [] + for _ in range(total_frames): + ret, frame = cap.read() + if ret: + frames.append(frame) + else: + # Handle the case where the frame could not be read + # print(f"Warning: Could not read frame at index {i}.") + pass + + cap.release() + + # Safely select frames based on frame_indices, avoiding IndexError + frames = [frames[i] for i in frame_indices if i < len(frames)] + + # If there are not enough frames, duplicate the last frame until we reach the target + while len(frames) < num_frames: + frames.append(frames[-1]) + + # Use ThreadPoolExecutor to process and encode frames in parallel + with ThreadPoolExecutor() as executor: + encoded_frames = list(executor.map(encode_frame, frames)) + + # encoded_frames = list(map(encode_frame, frames)) + + # Concatenate all frames bytes + video_bytes = b"".join(encoded_frames) + + # Encode the concatenated bytes to base64 + video_base64 = "video:" + pybase64.b64encode(video_bytes).decode("utf-8") + + return video_base64 + + +def _is_chinese_char(cp: int): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ( + (cp >= 0x4E00 and cp <= 0x9FFF) + or (cp >= 0x3400 and cp <= 0x4DBF) # + or (cp >= 0x20000 and cp <= 0x2A6DF) # + or (cp >= 0x2A700 and cp <= 0x2B73F) # + or (cp >= 0x2B740 and cp <= 0x2B81F) # + or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0xF900 and cp <= 0xFAFF) + or (cp >= 0x2F800 and cp <= 0x2FA1F) # + ): # + return True + + return False + + +def find_printable_text(text: str): + """Returns the longest printable substring of text that contains only entire words.""" + # Borrowed from https://github.com/huggingface/transformers/blob/061580c82c2db1de9139528243e105953793f7a2/src/transformers/generation/streamers.py#L99 + + # After the symbol for a new line, we flush the cache. + if text.endswith("\n"): + return text + # If the last token is a CJK character, we print the characters. + elif len(text) > 0 and _is_chinese_char(ord(text[-1])): + return text + # Otherwise if the penultimate token is a CJK character, we print the characters except for the last one. + elif len(text) > 1 and _is_chinese_char(ord(text[-2])): + return text[:-1] + # Otherwise, prints until the last space char (simple heuristic to avoid printing incomplete words, + # which may change with the subsequent token -- there are probably smarter ways to do this!) + else: + return text[: text.rfind(" ") + 1] + + +class LazyImport: + """Lazy import to make `import sglang` run faster.""" + + def __init__(self, module_name: str, class_name: str): + self.module_name = module_name + self.class_name = class_name + self._module = None + + def _load(self): + if self._module is None: + module = importlib.import_module(self.module_name) + self._module = getattr(module, self.class_name) + return self._module + + def __getattr__(self, name: str): + module = self._load() + return getattr(module, name) + + def __call__(self, *args, **kwargs): + module = self._load() + return module(*args, **kwargs) + + +def download_and_cache_file(url: str, filename: Optional[str] = None): + """Read and cache a file from a url.""" + if filename is None: + filename = os.path.join("/tmp", url.split("/")[-1]) + + # Check if the cache file already exists + if os.path.exists(filename): + return filename + + print(f"Downloading from {url} to {filename}") + + # Stream the response to show the progress bar + response = requests.get(url, stream=True) + response.raise_for_status() # Check for request errors + + # Total size of the file in bytes + total_size = int(response.headers.get("content-length", 0)) + chunk_size = 1024 # Download in chunks of 1KB + + # Use tqdm to display the progress bar + with open(filename, "wb") as f, tqdm( + desc=filename, + total=total_size, + unit="B", + unit_scale=True, + unit_divisor=1024, + ) as bar: + for chunk in response.iter_content(chunk_size=chunk_size): + f.write(chunk) + bar.update(len(chunk)) + + return filename + + +def is_in_ci() -> bool: + return envs.SGLANG_IS_IN_CI.get() + + +def print_highlight(html_content: str): + if is_in_ci(): + html_content = str(html_content).replace("\n", "
") + display(HTML(f"{html_content}")) + else: + print(html_content) + + +process_socket_map = weakref.WeakKeyDictionary() + + +def reserve_port(host, start=30000, end=40000): + """ + Reserve an available port by trying to bind a socket. + Returns a tuple (port, lock_socket) where `lock_socket` is kept open to hold the lock. + """ + candidates = list(range(start, end)) + random.shuffle(candidates) + + for port in candidates: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + # Attempt to bind to the port on localhost + sock.bind((host, port)) + return port, sock + except socket.error: + sock.close() # Failed to bind, try next port + continue + raise RuntimeError("No free port available.") + + +def release_port(lock_socket): + """ + Release the reserved port by closing the lock socket. + """ + try: + lock_socket.close() + except Exception as e: + print(f"Error closing socket: {e}") + + +def execute_shell_command(command: str) -> subprocess.Popen: + """ + Execute a shell command and return its process handle. + """ + command = command.replace("\\\n", " ").replace("\\", " ") + parts = command.split() + return subprocess.Popen(parts, text=True, stderr=subprocess.STDOUT) + + +def launch_server_cmd(command: str, host: str = "0.0.0.0", port: int = None): + """ + Launch the server using the given command. + If no port is specified, a free port is reserved. + """ + if port is None: + port, lock_socket = reserve_port(host) + else: + lock_socket = None + + full_command = f"{command} --port {port}" + process = execute_shell_command(full_command) + + if lock_socket is not None: + process_socket_map[process] = lock_socket + + return process, port + + +def terminate_process(process): + """ + Terminate the process and automatically release the reserved port. + """ + from sglang.srt.utils import kill_process_tree + + kill_process_tree(process.pid) + + lock_socket = process_socket_map.pop(process, None) + if lock_socket is not None: + release_port(lock_socket) + + +def wait_for_server(base_url: str, timeout: int = None) -> None: + """Wait for the server to be ready by polling the /v1/models endpoint. + + Args: + base_url: The base URL of the server + timeout: Maximum time to wait in seconds. None means wait forever. + """ + start_time = time.perf_counter() + while True: + try: + response = requests.get( + f"{base_url}/v1/models", + headers={"Authorization": "Bearer None"}, + ) + if response.status_code == 200: + time.sleep(5) + print_highlight( + """\n + NOTE: Typically, the server runs in a separate terminal. + In this notebook, we run the server and notebook code together, so their outputs are combined. + To improve clarity, the server logs are displayed in the original black color, while the notebook outputs are highlighted in blue. + To reduce the log length, we set the log level to warning for the server, the default log level is info. + We are running those notebooks in a CI environment, so the throughput is not representative of the actual performance. + """ + ) + break + + if timeout and time.perf_counter() - start_time > timeout: + raise TimeoutError("Server did not become ready within timeout period") + except requests.exceptions.RequestException: + time.sleep(1) + + +class TypeBasedDispatcher: + def __init__(self, mapping: List[Tuple[Type, Callable]]): + # Use dictionary for fast exact type matching, using OrderedDict(mapping) + # to maintains registration order + self._mapping = OrderedDict(mapping) + # MRO cache for inheritance-based matching + self._mro_cache = {} + self._fallback_fn = None + + def add_fallback_fn(self, fallback_fn: Callable): + self._fallback_fn = fallback_fn + + def __iadd__(self, other: "TypeBasedDispatcher"): + for ty, fn in other._mapping.items(): + if ty not in self._mapping: + self._mapping[ty] = fn + + self._mro_cache.clear() + return self + + def __call__(self, obj: Any): + obj_type = type(obj) + # 1. First try exact match(o(1)) + fn = self._mapping.get(obj_type) + if fn is not None: + return fn(obj) + + # 2. If exact match fails, check MRO cache + cached_fn = self._mro_cache.get(obj_type) + if cached_fn is not None: + return cached_fn(obj) + + # 3.search in registration order for compatible type(maintains origin behavior) + for ty, fn in self._mapping.items(): + if isinstance(obj, ty): + self._mro_cache[obj_type] = fn + return fn(obj) + + # 4. if no matching type found, cache this result + self._mro_cache[obj_type] = None + + if self._fallback_fn is not None: + return self._fallback_fn(obj) + raise ValueError(f"Invalid object: {obj}") + + +def trim_overlap(existing_text, new_chunk): + """ + Finds the largest suffix of 'existing_text' that is a prefix of 'new_chunk' + and removes that overlap from the start of 'new_chunk'. + """ + max_overlap = 0 + max_possible = min(len(existing_text), len(new_chunk)) + for i in range(max_possible, 0, -1): + if existing_text.endswith(new_chunk[:i]): + max_overlap = i + break + return new_chunk[max_overlap:] + + +def stream_and_merge(llm, prompt, sampling_params): + """ + 1) Streams the text, + 2) Removes chunk overlaps, + 3) Returns the merged text. + """ + final_text = "" + for chunk in llm.generate(prompt, sampling_params, stream=True): + chunk_text = chunk["text"] + cleaned_chunk = trim_overlap(final_text, chunk_text) + final_text += cleaned_chunk + return final_text + + +async def async_stream_and_merge(llm, prompt, sampling_params): + """ + Streams tokens asynchronously, removes chunk overlaps, + and yields the cleaned chunk in real time for printing. + """ + final_text = "" + generator = await llm.async_generate(prompt, sampling_params, stream=True) + async for chunk in generator: + chunk_text = chunk["text"] + cleaned_chunk = trim_overlap(final_text, chunk_text) + final_text += cleaned_chunk + yield cleaned_chunk # yield the non-overlapping portion + + +def resolve_obj_by_qualname(qualname: str) -> Any: + """ + Resolve an object by its fully qualified name. + """ + module_name, obj_name = qualname.rsplit(".", 1) + module = importlib.import_module(module_name) + return getattr(module, obj_name) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/version.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/version.py new file mode 100644 index 00000000..a779a442 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/python/sglang/version.py @@ -0,0 +1 @@ +__version__ = "0.5.6" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/check_vram_clear.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/check_vram_clear.sh new file mode 100755 index 00000000..51e5a915 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/check_vram_clear.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +check_vram_clear() { + local vram_threshold_percent=5 # Allow up to 5% VRAM usage + local memory_threshold_mb=500 # Allow up to 500MB memory usage + + if command -v rocm-smi >/dev/null 2>&1; then + echo "Checking ROCm GPU VRAM usage..." + # Check if any GPU has more than threshold VRAM allocated + local high_usage=$(rocm-smi --showmemuse | grep -E "GPU Memory Allocated \(VRAM%\): ([6-9]|[1-9][0-9]|100)") + if [ -n "$high_usage" ]; then + echo "ERROR: VRAM usage exceeds threshold (${vram_threshold_percent}%) on some GPUs:" + echo "$high_usage" + rocm-smi --showmemuse + return 1 + else + echo "✓ VRAM usage is within acceptable limits on all GPUs" + return 0 + fi + fi +} + +# If this script is run directly (not sourced), run the check +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + set -e + check_vram_clear +fi diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/amd_ci_exec.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/amd_ci_exec.sh new file mode 100755 index 00000000..6cfaa35b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/amd_ci_exec.sh @@ -0,0 +1,60 @@ +#!/bin/bash +set -euo pipefail + +# Detect GPU family from hostname (e.g., linux-mi35x-gpu-1-xxxxx-runner-zzzzz) +HOSTNAME_VALUE=$(hostname) +GPU_FAMILY="" + +# Host names look like: linux-mi35x-gpu-1-xxxxx-runner-zzzzz +if [[ "${HOSTNAME_VALUE}" =~ ^linux-(mi[0-9]+[a-z]*)-gpu-[0-9]+ ]]; then + GPU_FAMILY="${BASH_REMATCH[1]}" + echo "Detected GPU family from hostname: ${GPU_FAMILY}" +else + echo "Warning: could not parse GPU family from '${HOSTNAME_VALUE}'" +fi + +WORKDIR="/sglang-checkout/test/srt" +declare -A ENV_MAP=( + [SGLANG_IS_IN_CI_AMD]=1 + [SGLANG_IS_IN_CI]=1 + [SGLANG_USE_AITER]=1 +) + +# Conditionally add GPU_ARCHS only for mi35x +if [[ "${GPU_FAMILY}" == "mi35x" ]]; then + ENV_MAP[GPU_ARCHS]="gfx950" +fi + +# Parse -w/--workdir and -e ENV=VAL +while [[ $# -gt 0 ]]; do + case "$1" in + -w|--workdir) + WORKDIR="$2" + shift 2 + ;; + -e) + IFS="=" read -r key val <<< "$2" + ENV_MAP["$key"]="$val" + shift 2 + ;; + --) + shift + break + ;; + *) + break + ;; + esac +done + +# Build final ENV_ARGS +ENV_ARGS=() +for key in "${!ENV_MAP[@]}"; do + ENV_ARGS+=("-e" "$key=${ENV_MAP[$key]}") +done + +# Run docker exec +docker exec \ + -w "$WORKDIR" \ + "${ENV_ARGS[@]}" \ + ci_sglang "$@" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/amd_ci_install_dependency.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/amd_ci_install_dependency.sh new file mode 100755 index 00000000..43bf25ce --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/amd_ci_install_dependency.sh @@ -0,0 +1,81 @@ +#!/bin/bash +set -euo pipefail +HOSTNAME_VALUE=$(hostname) +GPU_ARCH="mi30x" # default + +# Host names look like: linux-mi35x-gpu-1-xxxxx-runner-zzzzz +if [[ "${HOSTNAME_VALUE}" =~ ^linux-(mi[0-9]+[a-z]*)-gpu-[0-9]+ ]]; then + GPU_ARCH="${BASH_REMATCH[1]}" + echo "Detected GPU architecture from hostname: ${GPU_ARCH}" +else + echo "Warning: could not parse GPU architecture from '${HOSTNAME_VALUE}', defaulting to ${GPU_ARCH}" +fi + +# Install the required dependencies in CI. +# Fix permissions on pip cache, ignore errors from concurrent access or missing temp files +docker exec ci_sglang chown -R root:root /sgl-data/pip-cache 2>/dev/null || true +docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache --upgrade pip +docker exec ci_sglang pip uninstall sgl-kernel -y || true +# Clear Python cache to ensure latest code is used +docker exec ci_sglang find /opt/venv -name "*.pyc" -delete || true +docker exec ci_sglang find /opt/venv -name "__pycache__" -type d -exec rm -rf {} + || true +docker exec -w /sglang-checkout/sgl-kernel ci_sglang bash -c "rm -f pyproject.toml && mv pyproject_rocm.toml pyproject.toml && python3 setup_rocm.py install" + +# Helper function to install with retries and fallback PyPI mirror +install_with_retry() { + local max_attempts=3 + local cmd="$@" + + for attempt in $(seq 1 $max_attempts); do + echo "Attempt $attempt/$max_attempts: $cmd" + if eval "$cmd"; then + echo "Success!" + return 0 + fi + + if [ $attempt -lt $max_attempts ]; then + echo "Failed, retrying in 5 seconds..." + sleep 5 + # Try with alternative PyPI index on retry + if [[ "$cmd" =~ "pip install" ]] && [ $attempt -eq 2 ]; then + cmd="$cmd --index-url https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com" + echo "Using fallback PyPI mirror: $cmd" + fi + fi + done + + echo "Failed after $max_attempts attempts" + return 1 +} + +case "${GPU_ARCH}" in + mi35x) + echo "Runner uses ${GPU_ARCH}; will fetch mi35x image." + docker exec ci_sglang rm -rf python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml + install_with_retry docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e "python[dev_hip]" --no-deps + # For lmms_evals evaluating MMMU + docker exec -w / ci_sglang git clone --branch v0.4.1 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git + install_with_retry docker exec -w /lmms-eval ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e . --no-deps + ;; + mi30x|mi300|mi325) + echo "Runner uses ${GPU_ARCH}; will fetch mi30x image." + docker exec ci_sglang rm -rf python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml + install_with_retry docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e "python[dev_hip]" + # For lmms_evals evaluating MMMU + docker exec -w / ci_sglang git clone --branch v0.4.1 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git + install_with_retry docker exec -w /lmms-eval ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e . + ;; + *) + echo "Runner architecture '${GPU_ARCH}' unrecognised;" >&2 + ;; +esac + +docker exec -w / ci_sglang git clone https://github.com/merrymercy/human-eval.git +install_with_retry docker exec -w /human-eval ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e . + +docker exec -w / ci_sglang mkdir -p /dummy-grok +mkdir -p dummy-grok && wget https://sharkpublic.blob.core.windows.net/sharkpublic/sglang/dummy_grok.json -O dummy-grok/config.json +docker cp ./dummy-grok ci_sglang:/ + +docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache huggingface_hub[hf_xet] +docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache pytest diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/amd_ci_start_container.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/amd_ci_start_container.sh new file mode 100755 index 00000000..cc4522d7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/amd_ci_start_container.sh @@ -0,0 +1,162 @@ +#!/bin/bash +set -euo pipefail + +# Get version from SGLang version.py file +SGLANG_VERSION_FILE="$(dirname "$0")/../../python/sglang/version.py" +SGLANG_VERSION="v0.5.5" # Default version, will be overridden if version.py is found + +TMP_VERSION_FILE=$(mktemp) +if git fetch --depth=1 origin main; then + if git show origin/main:python/sglang/version.py >"$TMP_VERSION_FILE" 2>/dev/null; then + VERSION_FROM_FILE="v$(cat "$SGLANG_VERSION_FILE" | cut -d'"' -f2)" + if [ -n "$VERSION_FROM_FILE" ]; then + SGLANG_VERSION="$VERSION_FROM_FILE" + echo "Using SGLang version from origin/main: $SGLANG_VERSION" + else + echo "Warning: Could not parse version from origin/main; using default $SGLANG_VERSION" >&2 + fi + else + echo "Warning: version.py not found on origin/main; using default $SGLANG_VERSION" >&2 + fi +else + echo "Warning: failed to fetch origin/main; using default $SGLANG_VERSION" >&2 +fi +rm -f "$TMP_VERSION_FILE" + + +# Default base tags (can be overridden by command line arguments) +ROCM_VERSION="rocm700" +DEFAULT_MI30X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi30x" +DEFAULT_MI35X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi35x" + +# Parse command line arguments +MI30X_BASE_TAG="${DEFAULT_MI30X_BASE_TAG}" +MI35X_BASE_TAG="${DEFAULT_MI35X_BASE_TAG}" + +while [[ $# -gt 0 ]]; do + case $1 in + --mi30x-base-tag) MI30X_BASE_TAG="$2"; shift 2;; + --mi35x-base-tag) MI35X_BASE_TAG="$2"; shift 2;; + -h|--help) + echo "Usage: $0 [--mi30x-base-tag TAG] [--mi35x-base-tag TAG]" + exit 0 + ;; + *) echo "Unknown option $1"; exit 1;; + esac +done + + + +# Detect GPU architecture from the Kubernetes runner hostname +HOSTNAME_VALUE=$(hostname) +GPU_ARCH="mi30x" # default + +# Host names look like: linux-mi35x-gpu-1-xxxxx-runner-zzzzz +if [[ "${HOSTNAME_VALUE}" =~ ^linux-(mi[0-9]+[a-z]*)-gpu-[0-9]+ ]]; then + GPU_ARCH="${BASH_REMATCH[1]}" + echo "Detected GPU architecture from hostname: ${GPU_ARCH}" +else + echo "Warning: could not parse GPU architecture from '${HOSTNAME_VALUE}', defaulting to ${GPU_ARCH}" +fi + +# Normalise / collapse architectures we don’t yet build specifically for +case "${GPU_ARCH}" in + mi35x) + echo "Runner uses ${GPU_ARCH}; will fetch mi35x image." + ;; + mi30x|mi300|mi325) + echo "Runner uses ${GPU_ARCH}; will fetch mi30x image." + GPU_ARCH="mi30x" + ;; + *) + echo "Runner architecture '${GPU_ARCH}' unrecognised; defaulting to mi30x image." >&2 + GPU_ARCH="mi30x" + ;; +esac + + +# Set up DEVICE_FLAG based on Kubernetes pod info +if [[ -f /etc/podinfo/gha-render-devices ]]; then + DEVICE_FLAG=$(cat /etc/podinfo/gha-render-devices) +else + DEVICE_FLAG="--device /dev/dri" +fi + + +# Find the latest image +find_latest_image() { + local gpu_arch=$1 + local base_tag days_back image_tag + + case "${gpu_arch}" in + mi30x) base_tag="${MI30X_BASE_TAG}" ;; + mi35x) base_tag="${MI35X_BASE_TAG}" ;; + *) echo "Error: unsupported GPU architecture '${gpu_arch}'" >&2; return 1 ;; + esac + + # First, check local cache + for days_back in {0..6}; do + image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)" + local local_image="rocm/sgl-dev:${image_tag}" + image_id=$(docker images -q "${local_image}") + if [[ -n "$image_id" ]]; then + echo "Found cached image locally: ${local_image}" >&2 + echo "${local_image}" + return 0 + fi + done + + # If not found locally, fall back to pulling from public registry + for days_back in {0..6}; do + image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)" + echo "Checking for image: rocm/sgl-dev:${image_tag}" >&2 + if docker manifest inspect "rocm/sgl-dev:${image_tag}" >/dev/null 2>&1; then + echo "Found available image: rocm/sgl-dev:${image_tag}" >&2 + echo "rocm/sgl-dev:${image_tag}" + return 0 + fi + done + + echo "No recent images found. Searching any cached local images matching ROCm+arch…" >&2 + local any_local + any_local=$(docker images --format '{{.Repository}}:{{.Tag}}' --filter "reference=rocm/sgl-dev:*${ROCM_VERSION}*${gpu_arch}*" | sort -r | head -n 1) + if [[ -n "$any_local" ]]; then + echo "Using cached fallback image: ${any_local}" >&2 + echo "${any_local}" + return 0 + fi + + echo "Error: no ${gpu_arch} image found in the last 7 days for base ${base_tag}" >&2 + echo "Using hard-coded fallback…" >&2 + if [[ "${gpu_arch}" == "mi35x" ]]; then + echo "rocm/sgl-dev:v0.5.5-rocm700-mi35x-20251110" + else + echo "rocm/sgl-dev:v0.5.5-rocm700-mi30x-20251110" + fi +} + +# Pull and run the latest image +IMAGE=$(find_latest_image "${GPU_ARCH}") +echo "Pulling Docker image: ${IMAGE}" +docker pull "${IMAGE}" + +CACHE_HOST=/home/runner/sgl-data +if [[ -d "$CACHE_HOST" ]]; then + CACHE_VOLUME="-v $CACHE_HOST:/sgl-data" +else + CACHE_VOLUME="" +fi + +echo "Launching container: ci_sglang" +docker run -dt --user root --device=/dev/kfd ${DEVICE_FLAG} \ + -v "${GITHUB_WORKSPACE:-$PWD}:/sglang-checkout" \ + $CACHE_VOLUME \ + --ipc=host --group-add video \ + --shm-size 32g \ + --cap-add=SYS_PTRACE \ + -e HF_TOKEN="${HF_TOKEN:-}" \ + -e HF_HOME=/sgl-data/hf-cache \ + --security-opt seccomp=unconfined \ + -w /sglang-checkout \ + --name ci_sglang \ + "${IMAGE}" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/ci_install_deepep.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/ci_install_deepep.sh new file mode 100755 index 00000000..738ac761 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/ci_install_deepep.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# Install the dependency in CI. +set -euxo pipefail + +bash scripts/ci/ci_install_dependency.sh + +export GDRCOPY_HOME=/usr/src/gdrdrv-2.5.1/ +export CUDA_HOME=/usr/local/cuda + +GRACE_BLACKWELL=${GRACE_BLACKWELL:-0} +# Detect architecture +ARCH=$(uname -m) +if [ "$ARCH" != "x86_64" ] && [ "$ARCH" != "aarch64" ]; then + echo "Unsupported architecture: $ARCH" + exit 1 +fi + +if python3 -c "import deep_ep" >/dev/null 2>&1; then + echo "deep_ep is already installed or importable. Skipping installation." + exit 0 +fi + +# Install system dependencies +apt install -y curl wget git sudo libibverbs-dev rdma-core infiniband-diags openssh-server perftest ibverbs-providers libibumad3 libibverbs1 libnl-3-200 libnl-route-3-200 librdmacm1 build-essential cmake + +# Install GDRCopy +rm -rf /opt/gdrcopy && mkdir -p /opt/gdrcopy +cd /opt/gdrcopy +git clone https://github.com/NVIDIA/gdrcopy.git . +git checkout v2.5.1 +apt update +apt install -y nvidia-dkms-580 +apt install -y build-essential devscripts debhelper fakeroot pkg-config dkms +apt install -y check libsubunit0 libsubunit-dev python3-venv +cd packages +CUDA=/usr/local/cuda ./build-deb-packages.sh +dpkg -i gdrdrv-dkms_*.deb +dpkg -i libgdrapi_*.deb +dpkg -i gdrcopy-tests_*.deb +dpkg -i gdrcopy_*.deb + +# Set up library paths based on architecture +LIB_PATH="/usr/lib/$ARCH-linux-gnu" +if [ ! -e "$LIB_PATH/libmlx5.so" ]; then + ln -s $LIB_PATH/libmlx5.so.1 $LIB_PATH/libmlx5.so +fi +apt-get update && apt-get install -y libfabric-dev + +# Install DeepEP +DEEPEP_DIR=/root/.cache/deepep +rm -rf ${DEEPEP_DIR} +if [ "$GRACE_BLACKWELL" = "1" ]; then + # We use Tom's DeepEP fork for GB200 for now, which supports fp4 dispatch. + GRACE_BLACKWELL_DEEPEP_BRANCH=gb200_blog_part_2 + git clone https://github.com/fzyzcjy/DeepEP.git ${DEEPEP_DIR} && \ + pushd ${DEEPEP_DIR} && \ + git checkout ${GRACE_BLACKWELL_DEEPEP_BRANCH} && \ + sed -i 's/#define NUM_CPU_TIMEOUT_SECS 100/#define NUM_CPU_TIMEOUT_SECS 1000/' csrc/kernels/configs.cuh && \ + popd +else + git clone https://github.com/deepseek-ai/DeepEP.git ${DEEPEP_DIR} && \ + pushd ${DEEPEP_DIR} && \ + git checkout 9af0e0d0e74f3577af1979c9b9e1ac2cad0104ee && \ + popd +fi + +cd ${DEEPEP_DIR} +if [ "$GRACE_BLACKWELL" = "1" ]; then + CUDA_VERSION=$(nvidia-smi | grep "CUDA Version" | head -n1 | awk '{print $9}') + if [ "$CUDA_VERSION" = "12.8" ]; then + CHOSEN_TORCH_CUDA_ARCH_LIST='10.0' + elif awk -v ver="$CUDA_VERSION" 'BEGIN {exit !(ver > 12.8)}'; then + CHOSEN_TORCH_CUDA_ARCH_LIST='10.0;10.3' + else + echo "Unsupported CUDA version for Grace Blackwell: $CUDA_VERSION" && exit 1 + fi && \ + if [ "${CUDA_VERSION%%.*}" = "13" ]; then \ + sed -i "/^ include_dirs = \['csrc\/'\]/a\ include_dirs.append('${CUDA_HOME}/include/cccl')" setup.py; \ + fi + TORCH_CUDA_ARCH_LIST="${CHOSEN_TORCH_CUDA_ARCH_LIST}" pip install --no-build-isolation . +else + python3 setup.py install +fi diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/ci_install_dependency.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/ci_install_dependency.sh new file mode 100755 index 00000000..d2370e18 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/ci_install_dependency.sh @@ -0,0 +1,149 @@ +#!/bin/bash +# Install the dependency in CI. +set -euxo pipefail + +IS_BLACKWELL=${IS_BLACKWELL:-0} +CU_VERSION="cu129" +OPTIONAL_DEPS="${1:-}" + +# Detect system architecture +ARCH=$(uname -m) +echo "Detected architecture: ${ARCH}" + +if [ "$CU_VERSION" = "cu130" ]; then + NVRTC_SPEC="nvidia-cuda-nvrtc" +else + NVRTC_SPEC="nvidia-cuda-nvrtc-cu12" +fi + +# Kill existing processes +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +bash "${SCRIPT_DIR}/../killall_sglang.sh" +echo "CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}" + +# Clear torch compilation cache +python3 -c 'import os, shutil, tempfile, getpass; cache_dir = os.environ.get("TORCHINDUCTOR_CACHE_DIR") or os.path.join(tempfile.gettempdir(), "torchinductor_" + getpass.getuser()); shutil.rmtree(cache_dir, ignore_errors=True)' + +# Install apt packages +apt install -y git libnuma-dev libssl-dev pkg-config + +# Check if protoc of correct architecture is already installed +if command -v protoc >/dev/null 2>&1; then + if protoc --version >/dev/null 2>&1; then + echo "protoc already installed: $(protoc --version)" + else + echo "protoc found but not runnable, reinstalling..." + INSTALL_PROTOC=1 + fi +else + INSTALL_PROTOC=1 +fi + +# Install protoc for router build (gRPC protobuf compilation) +if [ "${INSTALL_PROTOC:-0}" = "1" ]; then + echo "Installing protoc..." + if command -v apt-get &> /dev/null; then + # Ubuntu/Debian + apt-get update + apt-get install -y wget unzip gcc g++ perl make + elif command -v yum &> /dev/null; then + # RHEL/CentOS + yum update -y + yum install -y wget unzip gcc gcc-c++ perl-core make + fi + + cd /tmp + # Determine protoc architecture + if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then + PROTOC_ARCH="aarch_64" + else + PROTOC_ARCH="x86_64" + fi + PROTOC_ZIP="protoc-32.0-linux-${PROTOC_ARCH}.zip" + wget https://github.com/protocolbuffers/protobuf/releases/download/v32.0/${PROTOC_ZIP} + unzip -o ${PROTOC_ZIP} -d /usr/local + rm ${PROTOC_ZIP} + protoc --version + cd - +else + echo "protoc already installed: $(protoc --version)" +fi + +# Install uv +if [ "$IS_BLACKWELL" = "1" ]; then + # The blackwell CI runner has some issues with pip and uv, + # so we can only use pip with `--break-system-packages` + PIP_CMD="pip" + PIP_INSTALL_SUFFIX="--break-system-packages" + $PIP_CMD install --upgrade pip + + # Clean up existing installations + $PIP_CMD uninstall -y sgl-kernel sglang $PIP_INSTALL_SUFFIX || true + $PIP_CMD uninstall -y flashinfer-python flashinfer-cubin flashinfer-jit-cache $PIP_INSTALL_SUFFIX || true +else + # In normal cases, we use uv, which is much faster than pip. + pip install --upgrade pip + pip install uv + export UV_SYSTEM_PYTHON=true + + PIP_CMD="uv pip" + PIP_INSTALL_SUFFIX="--index-strategy unsafe-best-match --prerelease allow" + + # Clean up existing installations + $PIP_CMD uninstall sgl-kernel sglang || true + $PIP_CMD uninstall flashinfer-python flashinfer-cubin flashinfer-jit-cache || true +fi + +EXTRAS="dev" +if [ -n "$OPTIONAL_DEPS" ]; then + EXTRAS="dev,${OPTIONAL_DEPS}" +fi +echo "Installing python extras: [${EXTRAS}]" + +# Install the main package +$PIP_CMD install -e "python[${EXTRAS}]" --extra-index-url https://download.pytorch.org/whl/${CU_VERSION} $PIP_INSTALL_SUFFIX + +# Install router for pd-disagg test +$PIP_CMD install sglang-router $PIP_INSTALL_SUFFIX + +# Install sgl-kernel +SGL_KERNEL_VERSION_FROM_KERNEL=$(grep -Po '(?<=^version = ")[^"]*' sgl-kernel/pyproject.toml) +SGL_KERNEL_VERSION_FROM_SRT=$(grep -Po -m1 '(?<=sgl-kernel==)[0-9A-Za-z\.\-]+' python/pyproject.toml) +echo "SGL_KERNEL_VERSION_FROM_KERNEL=${SGL_KERNEL_VERSION_FROM_KERNEL} SGL_KERNEL_VERSION_FROM_SRT=${SGL_KERNEL_VERSION_FROM_SRT}" + +if [ "${CUSTOM_BUILD_SGL_KERNEL:-}" = "true" ]; then + ls -alh sgl-kernel/dist + # Determine wheel architecture + if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then + WHEEL_ARCH="aarch64" + else + WHEEL_ARCH="x86_64" + fi + $PIP_CMD install sgl-kernel/dist/sgl_kernel-${SGL_KERNEL_VERSION_FROM_KERNEL}-cp310-abi3-manylinux2014_${WHEEL_ARCH}.whl --force-reinstall $PIP_INSTALL_SUFFIX +else + $PIP_CMD install sgl-kernel==${SGL_KERNEL_VERSION_FROM_SRT} --force-reinstall $PIP_INSTALL_SUFFIX +fi + +# Show current packages +$PIP_CMD list + +$PIP_CMD install mooncake-transfer-engine==0.3.7.post2 "${NVRTC_SPEC}" py-spy scipy huggingface_hub[hf_xet] pytest $PIP_INSTALL_SUFFIX + +if [ "$IS_BLACKWELL" != "1" ]; then + # For lmms_evals evaluating MMMU + git clone --branch v0.5 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git + $PIP_CMD install -e lmms-eval/ $PIP_INSTALL_SUFFIX +fi + +# DeepEP depends on nvshmem 3.4.5 +$PIP_CMD install nvidia-nvshmem-cu12==3.4.5 --force-reinstall $PIP_INSTALL_SUFFIX + +# Cudnn with version less than 9.16.0.29 will cause performance regression on Conv3D kernel +$PIP_CMD install nvidia-cudnn-cu12==9.16.0.29 --force-reinstall $PIP_INSTALL_SUFFIX +$PIP_CMD uninstall xformers || true +# Show current packages +$PIP_CMD list +python3 -c "import torch; print(torch.version.cuda)" + +# Prepare the CI runner (cleanup HuggingFace cache, etc.) +bash "${SCRIPT_DIR}/prepare_runner.sh" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/ci_install_rust.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/ci_install_rust.sh new file mode 100755 index 00000000..7f67b820 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/ci_install_rust.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -euxo pipefail + +# Check if sudo is available +if command -v sudo >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y libssl-dev pkg-config protobuf-compiler +else + apt-get update + apt-get install -y libssl-dev pkg-config protobuf-compiler +fi + +# Install rustup (Rust installer and version manager) +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.90 + + +# Follow the installation prompts, then reload your shell +. "$HOME/.cargo/env" +source $HOME/.cargo/env + +# Verify installation +rustc --version +cargo --version +protoc --version diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/ci_start_disaggregation_servers.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/ci_start_disaggregation_servers.sh new file mode 100755 index 00000000..bbfdac9d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/ci_start_disaggregation_servers.sh @@ -0,0 +1,106 @@ +#!/bin/bash +set -euo pipefail + +# Optional: set DISAGG_READY_FILE to a filepath; when all servers are healthy, the script will +# create this file as a readiness signal (useful for CI to proceed to next steps). +DISAGG_READY_FILE="${DISAGG_READY_FILE:-}" + +MODEL_PATH="/raid/models/meta-llama/Llama-3.1-8B-Instruct" + +# Function to find the first available active IB device +find_active_ib_device() { + for device in mlx5_{0..11}; do + if ibv_devinfo $device >/dev/null 2>&1; then + state=$(ibv_devinfo $device | grep "state:" | head -1 | awk '{print $2}') + if [[ "$state" == "PORT_ACTIVE" ]]; then + echo "$device" + return 0 + fi + fi + done + echo "No active IB device found" >&2 + return 1 +} + +# Get the first available active IB device +DEVICE=$(find_active_ib_device) +echo "Using IB device: $DEVICE" + +# Launch prefill servers on GPU 0–3 +for i in {0..3}; do + PORT=$((30001 + i)) + BOOTSTRAP_PORT=$((9001 + i)) + HOST="127.0.0.$((i + 1))" + echo "Launching PREFILL server on GPU $i at $HOST:$PORT (bootstrap: $BOOTSTRAP_PORT)" + CUDA_VISIBLE_DEVICES=$i \ + python3 -m sglang.launch_server \ + --model-path "$MODEL_PATH" \ + --disaggregation-mode prefill \ + --host "$HOST" \ + --port "$PORT" \ + --disaggregation-ib-device "$DEVICE" \ + --disaggregation-bootstrap-port "$BOOTSTRAP_PORT" & +done + +# Launch decode servers on GPU 4–7 +for i in {4..7}; do + PORT=$((30001 + i)) + HOST="127.0.0.$((i + 1))" + echo "Launching DECODE server on GPU $i at $HOST:$PORT" + CUDA_VISIBLE_DEVICES=$i \ + python3 -m sglang.launch_server \ + --model-path "$MODEL_PATH" \ + --disaggregation-mode decode \ + --host "$HOST" \ + --port "$PORT" \ + --disaggregation-ib-device "$DEVICE" \ + --base-gpu-id 0 & +done + +# Wait for disaggregation servers to initialize +echo "Waiting for disaggregation servers to initialize..." + +# Health check with 5-minute timeout +TIMEOUT=300 +START_TIME=$(date +%s) + +echo "Checking health of all 8 servers..." +while true; do + CURRENT_TIME=$(date +%s) + ELAPSED=$((CURRENT_TIME - START_TIME)) + + if [ $ELAPSED -ge $TIMEOUT ]; then + echo "❌ Timeout: Servers did not become healthy within 5 minutes" + exit 1 + fi + + HEALTHY_COUNT=0 + # Check all 8 servers (127.0.0.1-8:30001-30008) + for i in {1..8}; do + if curl -s -f "http://127.0.0.$i:$((30000 + i))/health" >/dev/null 2>&1; then + HEALTHY_COUNT=$((HEALTHY_COUNT + 1)) + fi + done + + echo "Healthy servers: $HEALTHY_COUNT/8 (elapsed: ${ELAPSED}s)" + + if [ $HEALTHY_COUNT -eq 8 ]; then + echo "✅ All 8 servers are healthy!" + # Emit readiness signal file if requested + if [ -n "$DISAGG_READY_FILE" ]; then + echo "Creating readiness flag: $DISAGG_READY_FILE" + # Ensure parent dir exists; ignore errors + mkdir -p "$(dirname "$DISAGG_READY_FILE")" 2>/dev/null || true + touch "$DISAGG_READY_FILE" + fi + break + else + sleep 10 # Wait 10 seconds before next check + fi +done + +# Don't launch router here - just keep servers running +echo "✅ All disaggregation servers are ready and waiting for router connections" + +# Keep the script running +wait diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/cleanup_hf_cache.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/cleanup_hf_cache.py new file mode 100755 index 00000000..fd9038a1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/cleanup_hf_cache.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +""" +Clean up stale HuggingFace cache artifacts from previous failed downloads. + +This script removes incomplete marker files, temporary files, and lock files +from the HuggingFace cache directory. These artifacts can accumulate from +interrupted or failed downloads and may interfere with future downloads. +""" + +import os +import sys +from pathlib import Path +from typing import List + +try: + from huggingface_hub import constants + + HF_HUB_AVAILABLE = True +except ImportError: + print("Warning: huggingface_hub not available") + HF_HUB_AVAILABLE = False + + +def get_hf_cache_dir() -> str: + """Get the HuggingFace cache directory.""" + if HF_HUB_AVAILABLE: + return constants.HF_HUB_CACHE + + # Fallback to environment variable or default + hf_home = os.environ.get("HF_HOME", os.path.expanduser("~/.cache/huggingface")) + return os.path.join(hf_home, "hub") + + +def find_stale_artifacts(cache_dir: str) -> List[Path]: + """ + Find stale artifact files in the HuggingFace cache. + + Args: + cache_dir: HuggingFace cache directory + + Returns: + List of paths to stale artifact files + """ + cache_path = Path(cache_dir) + + if not cache_path.exists(): + return [] + + # Patterns for stale files to clean up + patterns = [ + "**/*.incomplete", # Incomplete download markers + "**/*.tmp", # Temporary files + "**/*.lock", # Lock files from interrupted downloads + ] + + stale_files = [] + for pattern in patterns: + stale_files.extend(cache_path.glob(pattern)) + + return stale_files + + +def cleanup_artifacts(artifacts: List[Path]) -> tuple[int, int]: + """ + Remove stale artifact files. + + Args: + artifacts: List of file paths to remove + + Returns: + Tuple of (successful_removals, failed_removals) + """ + successful = 0 + failed = 0 + + for file_path in artifacts: + try: + file_path.unlink() + print(f" Removed: {file_path}") + successful += 1 + except Exception as e: + print(f" Warning: Could not remove {file_path}: {e}") + failed += 1 + + return successful, failed + + +def main() -> int: + """ + Main cleanup logic. + + Returns: + Always returns 0 (cleanup is best-effort and should not fail CI) + """ + print("=" * 70) + print("HuggingFace Cache Cleanup") + print("=" * 70) + + # Get cache directory + cache_dir = get_hf_cache_dir() + print(f"Cache directory: {cache_dir}") + + if not os.path.exists(cache_dir): + print("Cache directory does not exist - nothing to clean") + return 0 + + print("-" * 70) + + # Find stale artifacts + print("Scanning for stale artifacts...") + stale_artifacts = find_stale_artifacts(cache_dir) + + if not stale_artifacts: + print("✓ No stale cache artifacts found") + return 0 + + # Clean up artifacts + print(f"Found {len(stale_artifacts)} stale artifact(s) to remove:") + successful, failed = cleanup_artifacts(stale_artifacts) + + print("-" * 70) + + # Summary + if failed > 0: + print(f"⚠ Cleaned up {successful} file(s), {failed} removal(s) failed") + else: + print(f"✓ Successfully cleaned up {successful} stale file(s)") + + # Always return 0 - cleanup failures should not fail CI + return 0 + + +if __name__ == "__main__": + try: + exit_code = main() + sys.exit(exit_code) + except KeyboardInterrupt: + print("\nInterrupted by user") + sys.exit(0) + except Exception as e: + print(f"ERROR: Unexpected error during cleanup: {e}") + import traceback + + traceback.print_exc() + # Still return 0 - cleanup failures should not fail CI + sys.exit(0) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/npu_ci_install_dependency.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/npu_ci_install_dependency.sh new file mode 100755 index 00000000..7cfdaaee --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/npu_ci_install_dependency.sh @@ -0,0 +1,66 @@ +#!/bin/bash +set -euo pipefail + +PIP_INSTALL="python3 -m pip install --no-cache-dir" +DEVICE_TYPE=$1 + + +# Install the required dependencies in CI. +apt update -y && apt install -y \ + build-essential \ + cmake \ + wget \ + curl \ + net-tools \ + zlib1g-dev \ + lld \ + clang \ + locales \ + ccache \ + ca-certificates +update-ca-certificates +${PIP_INSTALL} --upgrade pip +# Pin wheel to 0.45.1, REF: https://github.com/pypa/wheel/issues/662 +${PIP_INSTALL} wheel==0.45.1 pybind11 + + +### Install MemFabric +${PIP_INSTALL} mf-adapter==1.0.0 + + +### Install PyTorch and PTA +PYTORCH_VERSION="2.8.0" +TORCHVISION_VERSION="0.23.0" +${PIP_INSTALL} torch==${PYTORCH_VERSION} torchvision==${TORCHVISION_VERSION} --index-url https://download.pytorch.org/whl/cpu + +PTA_URL="https://sglang-ascend.obs.cn-east-3.myhuaweicloud.com/sglang/torch_npu/torch_npu-2.8.0.post2.dev20251113-cp311-cp311-manylinux_2_28_aarch64.whl" +${PIP_INSTALL} ${PTA_URL} + + +### Install Triton-Ascend +TRITON_ASCEND_URL="https://sglang-ascend.obs.cn-east-3.myhuaweicloud.com/sglang/triton_ascend/triton_ascend-3.2.0.dev2025112116-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl" +${PIP_INSTALL} ${TRITON_ASCEND_URL} + + +### Install BiSheng +BISHENG_NAME="Ascend-BiSheng-toolkit_aarch64_20251121.run" +BISHENG_URL="https://sglang-ascend.obs.cn-east-3.myhuaweicloud.com/sglang/triton_ascend/${BISHENG_NAME}" +wget -O "${BISHENG_NAME}" "${BISHENG_URL}" && chmod a+x "${BISHENG_NAME}" && "./${BISHENG_NAME}" --install && rm "${BISHENG_NAME}" + + +### Install sgl-kernel-npu +SGL_KERNEL_NPU_TAG="20251128" +git clone --depth 1 https://github.com/sgl-project/sgl-kernel-npu.git --branch ${SGL_KERNEL_NPU_TAG} +(cd sgl-kernel-npu && bash ./build.sh && ${PIP_INSTALL} output/deep_ep*.whl output/sgl_kernel_npu*.whl && cd "$(python3 -m pip show deep-ep | grep -E '^Location:' | awk '{print $2}')" && ln -s deep_ep/deep_ep_cpp*.so) + + +### Install CustomOps (TODO: to be removed once merged into sgl-kernel-npu) +wget https://sglang-ascend.obs.cn-east-3.myhuaweicloud.com/ops/CANN-custom_ops-8.2.0.0-$DEVICE_TYPE-linux.aarch64.run +chmod a+x ./CANN-custom_ops-8.2.0.0-$DEVICE_TYPE-linux.aarch64.run +./CANN-custom_ops-8.2.0.0-$DEVICE_TYPE-linux.aarch64.run --quiet --install-path=/usr/local/Ascend/ascend-toolkit/latest/opp +wget https://sglang-ascend.obs.cn-east-3.myhuaweicloud.com/ops/custom_ops-1.0.$DEVICE_TYPE-cp311-cp311-linux_aarch64.whl +pip install ./custom_ops-1.0.$DEVICE_TYPE-cp311-cp311-linux_aarch64.whl + +### Install SGLang +rm -rf python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml +${PIP_INSTALL} -v -e "python[srt_npu]" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/prepare_runner.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/prepare_runner.sh new file mode 100755 index 00000000..fe0bf440 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/prepare_runner.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Prepare the CI runner by cleaning up stale HuggingFace cache artifacts and validating models +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +echo "Preparing CI runner..." +echo "" + +# Clean up stale HuggingFace cache artifacts from previous failed downloads +python3 "${SCRIPT_DIR}/cleanup_hf_cache.py" +echo "" + +echo "CI runner preparation complete!" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/publish_traces.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/publish_traces.py new file mode 100644 index 00000000..e024c7bb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/publish_traces.py @@ -0,0 +1,382 @@ +""" +Publish performance traces to GitHub repository +""" + +import argparse +import base64 +import json +import os +import sys +import time +from urllib.error import HTTPError +from urllib.request import Request, urlopen + + +def make_github_request(url, token, method="GET", data=None): + """Make authenticated request to GitHub API""" + headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + # "User-Agent": "sglang-ci", + "X-GitHub-Api-Version": "2022-11-28", + } + + if data: + headers["Content-Type"] = "application/json" + data = json.dumps(data).encode("utf-8") + + req = Request(url, data=data, headers=headers, method=method) + + try: + with urlopen(req) as response: + return response.read().decode("utf-8") + except HTTPError as e: + print(f"GitHub API request failed: {e}") + try: + error_body = e.read().decode("utf-8") + print(f"Error response body: {error_body}") + e.error_body = error_body # Attach for later inspection + except Exception: + e.error_body = "" + raise + except Exception as e: + print(f"GitHub API request failed with a non-HTTP error: {e}") + raise + + +def verify_token_permissions(repo_owner, repo_name, token): + """Verify that the token has necessary permissions for the repository""" + print("Verifying token permissions...") + + # Check if we can access the repository + try: + url = f"https://api.github.com/repos/{repo_owner}/{repo_name}" + response = make_github_request(url, token) + repo_data = json.loads(response) + print(f"Repository access verified: {repo_data['full_name']}") + except Exception as e: + print(f"Failed to access repository: {e}") + return False + + # Check if we can read the repository contents + try: + url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/contents" + response = make_github_request(url, token) + print("Repository contents access verified") + except Exception as e: + print(f"Failed to access repository contents: {e}") + return False + + return True + + +def get_branch_sha(repo_owner, repo_name, branch, token): + """Get SHA of the branch head""" + url = ( + f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/refs/heads/{branch}" + ) + response = make_github_request(url, token) + data = json.loads(response) + return data["object"]["sha"] + + +def get_tree_sha(repo_owner, repo_name, commit_sha, token): + """Get tree SHA from commit""" + url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/commits/{commit_sha}" + response = make_github_request(url, token) + data = json.loads(response) + return data["tree"]["sha"] + + +def create_blob(repo_owner, repo_name, content, token, max_retries=3): + """Create a blob with file content""" + url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/blobs" + + # Encode content as base64 for GitHub API + content_b64 = base64.b64encode(content).decode("utf-8") + + data = {"content": content_b64, "encoding": "base64"} + + for attempt in range(max_retries): + try: + response = make_github_request(url, token, method="POST", data=data) + return json.loads(response)["sha"] + except Exception as e: + if attempt < max_retries - 1: + wait_time = 2**attempt # Exponential backoff: 1s, 2s, 4s + print( + f"Blob creation failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..." + ) + time.sleep(wait_time) + else: + raise + + +def create_tree(repo_owner, repo_name, base_tree_sha, files, token, max_retries=3): + """Create a new tree with files""" + url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/trees" + + tree_items = [] + for i, (file_path, content) in enumerate(files): + # Create blob first to get SHA + blob_sha = create_blob(repo_owner, repo_name, content, token) + tree_items.append( + { + "path": file_path, + "mode": "100644", + "type": "blob", + "sha": blob_sha, + } + ) + # Progress indicator for large uploads + if (i + 1) % 10 == 0 or (i + 1) == len(files): + print(f"Created {i + 1}/{len(files)} blobs...") + + data = {"base_tree": base_tree_sha, "tree": tree_items} + + for attempt in range(max_retries): + try: + response = make_github_request(url, token, method="POST", data=data) + return json.loads(response)["sha"] + except Exception as e: + if attempt < max_retries - 1: + wait_time = 2**attempt + print( + f"Tree creation failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..." + ) + time.sleep(wait_time) + else: + raise + + +def create_commit( + repo_owner, repo_name, tree_sha, parent_sha, message, token, max_retries=3 +): + """Create a new commit""" + url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/commits" + + data = {"tree": tree_sha, "parents": [parent_sha], "message": message} + + for attempt in range(max_retries): + try: + response = make_github_request(url, token, method="POST", data=data) + commit_sha = json.loads(response)["sha"] + + # Verify the commit was actually created + verify_url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/commits/{commit_sha}" + verify_response = make_github_request(verify_url, token) + verify_data = json.loads(verify_response) + if verify_data["sha"] != commit_sha: + raise Exception( + f"Commit verification failed: expected {commit_sha}, got {verify_data['sha']}" + ) + + return commit_sha + except Exception as e: + if attempt < max_retries - 1: + wait_time = 2**attempt + print( + f"Commit creation failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..." + ) + time.sleep(wait_time) + else: + raise + + +def update_branch_ref(repo_owner, repo_name, branch, commit_sha, token, max_retries=3): + """Update branch reference to point to new commit""" + url = ( + f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/refs/heads/{branch}" + ) + + data = {"sha": commit_sha} + + for attempt in range(max_retries): + try: + make_github_request(url, token, method="PATCH", data=data) + return + except HTTPError as e: + # Check if this is an "Object does not exist" error + is_object_not_exist = False + if hasattr(e, "error_body"): + try: + error_data = json.loads(e.error_body) + if "Object does not exist" in error_data.get("message", ""): + is_object_not_exist = True + except Exception: + pass + + if is_object_not_exist and attempt < max_retries - 1: + # This might be a transient consistency issue - wait and retry + wait_time = 2**attempt + print( + f"Branch update failed with 'Object does not exist' (attempt {attempt + 1}/{max_retries}), waiting {wait_time}s for consistency..." + ) + time.sleep(wait_time) + else: + raise + except Exception as e: + if attempt < max_retries - 1: + wait_time = 2**attempt + print( + f"Branch update failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..." + ) + time.sleep(wait_time) + else: + raise + + +def copy_trace_files(source_dir, target_base_path): + """Copy trace files and return list of files to upload""" + files_to_upload = [] + + if not os.path.exists(source_dir): + print(f"Warning: Traces directory {source_dir} does not exist") + return files_to_upload + + # Walk through source directory and find .json.gz files + for root, dirs, files in os.walk(source_dir): + for file in files: + if file.endswith(".json.gz"): + source_file = os.path.join(root, file) + # Calculate relative path from source_dir + rel_path = os.path.relpath(source_file, source_dir) + target_path = f"{target_base_path}/{rel_path}" + + # Read file content + with open(source_file, "rb") as f: + content = f.read() + + files_to_upload.append((target_path, content)) + + return files_to_upload + + +def publish_traces(traces_dir, run_id, run_number): + """Publish traces to GitHub repository in a single commit""" + # Get environment variables + token = os.getenv("GITHUB_TOKEN") + if not token: + print("Error: GITHUB_TOKEN environment variable not set") + sys.exit(1) + + # Repository configuration + repo_owner = "sglang-bot" + repo_name = "sglang-ci-data" + branch = "main" + target_base_path = f"traces/{run_id}" + + # Copy trace files + files_to_upload = copy_trace_files(traces_dir, target_base_path) + + if not files_to_upload: + print("No trace files found to upload") + return + + print(f"Found {len(files_to_upload)} files to upload") + + # Verify token permissions before proceeding + if not verify_token_permissions(repo_owner, repo_name, token): + print( + "Token permission verification failed. Please check the token permissions." + ) + sys.exit(1) + + max_retries = 5 + retry_delay = 5 # seconds + + for attempt in range(max_retries): + try: + # Get current branch head + branch_sha = get_branch_sha(repo_owner, repo_name, branch, token) + print(f"Current branch head: {branch_sha}") + + # Get current tree + tree_sha = get_tree_sha(repo_owner, repo_name, branch_sha, token) + print(f"Current tree SHA: {tree_sha}") + + # Create new tree with all files + new_tree_sha = create_tree( + repo_owner, repo_name, tree_sha, files_to_upload, token + ) + print(f"Created new tree: {new_tree_sha}") + + # Create commit + commit_message = f"Nightly traces for run {run_id} at {run_number} ({len(files_to_upload)} files)" + commit_sha = create_commit( + repo_owner, + repo_name, + new_tree_sha, + branch_sha, + commit_message, + token, + ) + print(f"Created commit: {commit_sha}") + + # Update branch reference + update_branch_ref(repo_owner, repo_name, branch, commit_sha, token) + print("Updated branch reference") + + print("Successfully published all traces in a single commit") + return + + except Exception as e: + # Check for retryable errors + is_retryable = False + error_type = "unknown" + + if hasattr(e, "error_body"): + if "Update is not a fast forward" in e.error_body: + is_retryable = True + error_type = "fast-forward conflict" + elif "Object does not exist" in e.error_body: + is_retryable = True + error_type = "object consistency" + + # Also retry on HTTP errors that might be transient + if isinstance(e, HTTPError) and e.code in [422, 500, 502, 503, 504]: + is_retryable = True + error_type = f"HTTP {e.code}" + + if is_retryable and attempt < max_retries - 1: + print( + f"Attempt {attempt + 1}/{max_retries} failed ({error_type}). Retrying in {retry_delay} seconds..." + ) + time.sleep(retry_delay) + else: + print(f"Failed to publish traces after {attempt + 1} attempts: {e}") + raise + + +def main(): + parser = argparse.ArgumentParser( + description="Publish performance traces to GitHub repository" + ) + parser.add_argument( + "--traces-dir", + type=str, + required=True, + help="Traces directory to publish", + ) + args = parser.parse_args() + + # Get environment variables + run_id = os.getenv("GITHUB_RUN_ID", "test") + run_number = os.getenv("GITHUB_RUN_NUMBER", "12345") + + if not run_id or not run_number: + print( + "Error: GITHUB_RUN_ID and GITHUB_RUN_NUMBER environment variables must be set" + ) + sys.exit(1) + + # Use traces directory + traces_dir = args.traces_dir + print(f"Processing traces from directory: {traces_dir}") + + # Publish traces + publish_traces(traces_dir, run_id, run_number) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/slash_command_handler.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/slash_command_handler.py new file mode 100644 index 00000000..9b0efa26 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci/slash_command_handler.py @@ -0,0 +1,283 @@ +import json +import os +import sys + +from github import Auth, Github + +# Configuration +PERMISSIONS_FILE_PATH = ".github/CI_PERMISSIONS.json" + + +def get_env_var(name): + val = os.getenv(name) + if not val: + print(f"Error: Environment variable {name} not set.") + sys.exit(1) + return val + + +def load_permissions(user_login): + """ + Reads the permissions JSON from the local file system and returns + the permissions dict for the specific user. + """ + try: + print(f"Loading permissions from {PERMISSIONS_FILE_PATH}...") + if not os.path.exists(PERMISSIONS_FILE_PATH): + print(f"Error: Permissions file not found at {PERMISSIONS_FILE_PATH}") + return None + + with open(PERMISSIONS_FILE_PATH, "r") as f: + data = json.load(f) + + user_perms = data.get(user_login) + + if not user_perms: + print(f"User '{user_login}' not found in permissions file.") + return None + + return user_perms + + except Exception as e: + print(f"Failed to load or parse permissions file: {e}") + sys.exit(1) + + +def handle_tag_run_ci(gh_repo, pr, comment, user_perms, react_on_success=True): + """ + Handles the /tag-run-ci-label command. + Returns True if action was taken, False otherwise. + """ + if not user_perms.get("can_tag_run_ci_label", False): + print("Permission denied: can_tag_run_ci_label is false.") + return False + + print("Permission granted. Adding 'run-ci' label.") + pr.add_to_labels("run-ci") + + if react_on_success: + comment.create_reaction("+1") + print("Label added and comment reacted.") + else: + print("Label added (reaction suppressed).") + + return True + + +def handle_rerun_failed_ci(gh_repo, pr, comment, user_perms, react_on_success=True): + """ + Handles the /rerun-failed-ci command. + Reruns workflows with 'failure' or 'skipped' conclusions. + Returns True if action was taken, False otherwise. + """ + if not user_perms.get("can_rerun_failed_ci", False): + print("Permission denied: can_rerun_failed_ci is false.") + return False + + print("Permission granted. Triggering rerun of failed or skipped workflows.") + + # Get the SHA of the latest commit in the PR + head_sha = pr.head.sha + print(f"Checking workflows for commit: {head_sha}") + + # List all workflow runs for this commit + runs = gh_repo.get_workflow_runs(head_sha=head_sha) + + rerun_count = 0 + for run in runs: + if run.status != "completed": + continue + + if run.conclusion == "failure": + # DEBUG + print(f"Rerunning failed workflow: {run.name} (ID: {run.id})") + try: + # Use rerun_failed_jobs for efficiency on failures + run.rerun_failed_jobs() + rerun_count += 1 + except Exception as e: + print(f"Failed to rerun workflow {run.id}: {e}") + + elif run.conclusion == "skipped": + print(f"Rerunning skipped workflow: {run.name} (ID: {run.id})") + try: + # Skipped workflows don't have 'failed jobs', so we use full rerun() + run.rerun() + rerun_count += 1 + except Exception as e: + print(f"Failed to rerun workflow {run.id}: {e}") + + if rerun_count > 0: + print(f"Triggered rerun for {rerun_count} workflows.") + if react_on_success: + comment.create_reaction("+1") + return True + else: + print("No failed or skipped workflows found to rerun.") + return False + + +def handle_rerun_stage( + gh_repo, pr, comment, user_perms, stage_name, react_on_success=True +): + """ + Handles the /rerun-stage command. + Triggers a workflow_dispatch to run only the specified stage, skipping dependencies. + Returns True if action was taken, False otherwise. + """ + if not user_perms.get("can_rerun_stage", False): + print("Permission denied: can_rerun_stage is false.") + return False + + if not stage_name: + print("Error: No stage name provided") + comment.create_reaction("confused") + comment.create_comment( + f"❌ Please specify a stage name: `/rerun-stage `\n\n" + f"Examples: `/rerun-stage unit-test-backend-4-gpu`, `/rerun-stage accuracy-test-1-gpu`" + ) + return False + + print(f"Permission granted. Triggering workflow_dispatch for stage '{stage_name}'.") + + # Valid stage names that support target_stage + valid_stages = [ + "stage-a-test-1", + "multimodal-gen-test-1-gpu", + "multimodal-gen-test-2-gpu", + "quantization-test", + "unit-test-backend-1-gpu", + "unit-test-backend-2-gpu", + "unit-test-backend-4-gpu", + "unit-test-backend-8-gpu-h200", + "unit-test-backend-8-gpu-h20", + "performance-test-1-gpu-part-1", + "performance-test-1-gpu-part-2", + "performance-test-1-gpu-part-3", + "performance-test-2-gpu", + "accuracy-test-1-gpu", + "accuracy-test-2-gpu", + "unit-test-deepep-4-gpu", + "unit-test-deepep-8-gpu", + "unit-test-backend-4-gpu-b200", + "unit-test-backend-4-gpu-gb200", + ] + + if stage_name not in valid_stages: + comment.create_reaction("confused") + comment.create_comment( + f"❌ Stage `{stage_name}` doesn't support isolated runs yet.\n\n" + f"Currently supported stages:\n" + + "\n".join(f"- `{s}`" for s in valid_stages) + + "\n\nOther stages will be added soon. For now, use `/rerun-failed-ci` for those stages." + ) + return False + + try: + # Get the PR Test workflow + workflows = gh_repo.get_workflows() + pr_test_workflow = None + for wf in workflows: + if wf.name == "PR Test": + pr_test_workflow = wf + break + + if not pr_test_workflow: + print("Error: PR Test workflow not found") + return False + + # Trigger workflow_dispatch on the PR's head branch + ref = pr.head.ref + print(f"Triggering workflow on branch: {ref}") + + success = pr_test_workflow.create_dispatch( + ref=ref, + inputs={"version": "release", "target_stage": stage_name}, + ) + + if success: + print(f"Successfully triggered workflow for stage '{stage_name}'") + if react_on_success: + comment.create_reaction("+1") + comment.create_comment( + f"✅ Triggered `{stage_name}` to run independently (skipping dependencies).\n\n" + f"Check the [Actions tab](https://github.com/{gh_repo.full_name}/actions) for progress." + ) + return True + else: + print("Failed to trigger workflow_dispatch") + return False + + except Exception as e: + print(f"Error triggering workflow_dispatch: {e}") + comment.create_reaction("confused") + comment.create_comment( + f"❌ Failed to trigger workflow: {str(e)}\n\n" + f"Please check the logs or contact maintainers." + ) + return False + + +def main(): + # 1. Load Environment Variables + token = get_env_var("GITHUB_TOKEN") + repo_name = get_env_var("REPO_FULL_NAME") + pr_number = int(get_env_var("PR_NUMBER")) + comment_id = int(get_env_var("COMMENT_ID")) + comment_body = get_env_var("COMMENT_BODY").strip() + user_login = get_env_var("USER_LOGIN") + + # 2. Load Permissions (Local Check) + user_perms = load_permissions(user_login) + + if not user_perms: + print(f"User {user_login} does not have any configured permissions. Exiting.") + return + + # 3. Initialize GitHub API with Auth + auth = Auth.Token(token) + g = Github(auth=auth) + + repo = g.get_repo(repo_name) + pr = repo.get_pull(pr_number) + comment = repo.get_issue(pr_number).get_comment(comment_id) + + # 4. Parse Command and Execute + first_line = comment_body.split("\n")[0].strip() + + if first_line.startswith("/tag-run-ci-label"): + handle_tag_run_ci(repo, pr, comment, user_perms) + + elif first_line.startswith("/rerun-failed-ci"): + handle_rerun_failed_ci(repo, pr, comment, user_perms) + + elif first_line.startswith("/tag-and-rerun-ci"): + # Perform both actions, but suppress individual reactions + print("Processing combined command: /tag-and-rerun-ci") + + tagged = handle_tag_run_ci( + repo, pr, comment, user_perms, react_on_success=False + ) + rerun = handle_rerun_failed_ci( + repo, pr, comment, user_perms, react_on_success=False + ) + + # If at least one action was successful, add the reaction here + if tagged or rerun: + comment.create_reaction("+1") + print("Combined command processed successfully; reaction added.") + else: + print("Combined command finished, but no actions were taken.") + + elif first_line.startswith("/rerun-stage"): + # Extract stage name from command + parts = first_line.split(maxsplit=1) + stage_name = parts[1].strip() if len(parts) > 1 else None + handle_rerun_stage(repo, pr, comment, user_perms, stage_name) + + else: + print(f"Unknown or ignored command: {first_line}") + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci_monitor/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci_monitor/README.md new file mode 100644 index 00000000..4c0f953d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci_monitor/README.md @@ -0,0 +1,334 @@ +# SGLang CI Monitor + +> **Note**: This README.md is primarily generated by Claude 4 with some manual adjustments. + +A comprehensive toolkit to analyze CI failures and performance trends for the SGLang project. This toolkit includes four main tools: + +1. **CI Analyzer** (`ci_analyzer.py`): Analyzes CI failures and provides detailed failure pattern analysis +2. **Performance Analyzer** (`ci_analyzer_perf.py`): Tracks performance metrics over time and generates trend charts +3. **Test Balance Analyzer** (`ci_analyzer_balance.py`): Analyzes test time gaps between elapsed and estimated times to help balance CI +4. **Failures Analyzer** (`ci_failures_analysis.py`): Tracks consecutive failures, identifies flaky jobs, and monitors runner health + +## Features + +### CI Analyzer (`ci_analyzer.py`) +- **Simple Analysis**: Analyze recent CI runs and identify failure patterns +- **Category Classification**: Automatically categorize failures by type (unit-test, performance, etc.) +- **Pattern Recognition**: Identify common failure patterns (timeouts, build failures, etc.) +- **CI Links**: Direct links to recent failed CI runs for detailed investigation +- **Last Success Tracking**: Track the last successful run for each failed job with PR information +- **JSON Export**: Export detailed analysis data to JSON format + +### Performance Analyzer (`ci_analyzer_perf.py`) +- **Performance Tracking**: Monitor performance metrics across CI runs over time +- **Automated Chart Generation**: Generate time-series charts for each performance metric +- **Multi-Test Support**: Track performance for all test types (throughput, latency, accuracy) +- **CSV Export**: Export performance data in structured CSV format +- **Trend Analysis**: Visualize performance trends with interactive charts +- **Comprehensive Metrics**: Track output throughput, E2E latency, TTFT, accept length, and more +- **Time-Based Sampling**: Intelligent sampling strategy to cover extended time periods (up to 30 days) with limited API calls + +### Test Balance Analyzer (`ci_analyzer_balance.py`) +- **Time Gap Analysis**: Identify GPU tests with large gaps between elapsed and estimated times +- **CI Balancing**: Help optimize CI by identifying tests that need time adjustments +- **Gap Tracking**: Track maximum time gaps for each test across multiple CI runs +- **PR Test Focus**: Only analyzes GPU jobs from pr-test.yml workflow (excludes AMD and other workflows) +- **Ranking System**: Sort tests by time gap severity to prioritize adjustments +- **CSV Export**: Export analysis results in CSV format for easy review +- **GitHub Integration**: Generate GitHub Actions summaries with recommendations + +### Failures Analyzer (`ci_failures_analysis.py`) +- **Consecutive Failure Tracking**: Identify jobs currently failing +- **Runner Health Monitoring**: Track runner failure rates and identify problematic infrastructure +- **Multi-Workflow Support**: Monitors PR Test (Nvidia), PR Test (AMD), and PR Test (Xeon) workflows +- **Queue Time Tracking**: Monitor average and P90 queue times per runner type +- **Alert System**: Automatic alerts for consecutive failures and runner problems +- **Instance Tracking**: Monitor specific runner instances for targeted remediation +- **Slack Notifications**: Send condensed alerts to Slack (top 3 jobs/runners by consecutive failures and failure rates) +- **GitHub Integration**: Generate comprehensive summaries with actionable recommendations +- **JSON Export**: Export detailed analysis data for further processing + +### Common Features +- **Automated Monitoring**: GitHub Actions workflow for continuous CI and performance monitoring + +## Installation + +### For CI Analyzer +No additional dependencies required beyond Python standard library and `requests`: + +```bash +pip install requests +``` + +### For Performance Analyzer +Additional dependencies required for chart generation: + +```bash +pip install requests matplotlib pandas +``` + +### For Test Balance Analyzer +No additional dependencies required beyond Python standard library and `requests`: + +```bash +pip install requests +``` + +## Usage + +### CI Analyzer + +#### Basic Usage + +```bash +# Replace YOUR_GITHUB_TOKEN with your actual token from https://github.com/settings/tokens +python ci_analyzer.py --token YOUR_GITHUB_TOKEN +``` + +#### Advanced Usage + +```bash +# Analyze last 1000 runs +python ci_analyzer.py --token YOUR_GITHUB_TOKEN --limit 1000 + +# Custom output file +python ci_analyzer.py --token YOUR_GITHUB_TOKEN --limit 500 --output my_analysis.json +``` + +### Performance Analyzer + +#### Basic Usage + +```bash +# Analyze performance trends from recent CI runs +python ci_analyzer_perf.py --token YOUR_GITHUB_TOKEN +``` + +#### Advanced Usage + +```bash +# Analyze last 1000 PR Test runs (auto-enables uniform sampling for ~30 days coverage) +python ci_analyzer_perf.py --token YOUR_GITHUB_TOKEN --limit 1000 + +# Custom output directory +python ci_analyzer_perf.py --token YOUR_GITHUB_TOKEN --limit 500 --output-dir my_performance_data + +# Use sampling with 500 runs (will use sequential mode since < 500 threshold) +python ci_analyzer_perf.py --token YOUR_GITHUB_TOKEN --limit 500 + +# Get ALL performance data within a specific date range (recommended for historical analysis) +python ci_analyzer_perf.py --token YOUR_GITHUB_TOKEN --start-date 2024-12-01 --end-date 2024-12-31 + +# Get complete data for the last week +python ci_analyzer_perf.py --token YOUR_GITHUB_TOKEN --start-date $(date -d '7 days ago' +%Y-%m-%d) --end-date $(date +%Y-%m-%d) + +# Upload results to GitHub repository for sharing +python ci_analyzer_perf.py --token YOUR_GITHUB_TOKEN --limit 1000 --upload-to-github +``` + +### Test Balance Analyzer + +#### Basic Usage + +```bash +# Analyze PR Test GPU job time gaps from recent CI runs +python ci_analyzer_balance.py --token YOUR_GITHUB_TOKEN +``` + +#### Advanced Usage + +```bash +# Analyze last 1000 PR Test GPU CI runs for comprehensive test balance analysis +python ci_analyzer_balance.py --token YOUR_GITHUB_TOKEN --limit 1000 + +# Custom output file +python ci_analyzer_balance.py --token YOUR_GITHUB_TOKEN --limit 500 --output my_balance_analysis.json +``` + +### Failures Analyzer + +#### Quick Start + +```bash +# Set token as environment variable (recommended for security) +export GITHUB_TOKEN="your_token_here" + +# Quick test with recent runs +python ci_failures_analysis.py --token $GITHUB_TOKEN --limit 50 --threshold 2 + +# Standard analysis (same as automated workflow) +python ci_failures_analysis.py --token $GITHUB_TOKEN --limit 300 --threshold 2 + +# Deep analysis +python ci_failures_analysis.py --token $GITHUB_TOKEN --limit 500 --threshold 3 +``` + +#### Monitored Workflows + +The Failures Analyzer monitors the following workflows: + +- **PR Test** - Nvidia GPU tests (self-hosted runners: 1-gpu-runner, 4-gpu-h100-runner, etc.) +- **PR Test (AMD)** - AMD GPU tests (AMD-specific runners) +- **PR Test (Xeon)** - Intel Xeon CPU tests (Xeon-specific runners) + +All three workflows are analyzed together, with runner statistics tracked separately by runner type. + +#### Slack Notifications + +The Failures Analyzer can send condensed alerts to Slack. See [SLACK_SETUP.md](SLACK_SETUP.md) for complete setup instructions. + +**What gets sent:** +- Top 3 jobs with consecutive failures +- Top 3 runners with consecutive failures +- Top 3 jobs with highest total failure rate +- Top 3 runners with highest total failure rate +- Queue time summary + +```bash +# Send Slack notification from analysis JSON +export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/YOUR/WEBHOOK/URL" +python slack_notifier.py --json ci_failure_analysis.json +``` + +#### Understanding the Output + +The script generates a **2-section report**: + +**Section 1: Currently Broken Jobs (Active Consecutive Failures)** +- Shows consecutive failure streaks +- These need immediate attention + +**Section 2: Runner Health Analysis** +- Shows which runners have high failure rates +- Includes queue time metrics (average and P90) +- Helps identify infrastructure vs code issues + +#### Alert Types + +**Job Alerts (Consecutive Failures):** +- Triggered when a job fails ≥ threshold times in a row +- Example: threshold=2, job fails 3 times → ALERT + +**Runner Alerts:** +- **Runner Health**: Runner has >30% failure rate with ≥2 different jobs failing +- **Runner Instance**: Specific instance has >50% failure rate with ≥3 jobs + +#### Output Files + +- **Console**: Human-readable 3-section report (always generated) +- **JSON**: Detailed data (optional, only if `--output` is specified) +- **GitHub Summary**: Markdown (automatically generated in GitHub Actions) + +**Important**: Make sure your GitHub token has `repo` and `workflow` permissions, otherwise you'll get 404 errors. + +## Data Collection Strategies + +The Performance Analyzer offers multiple strategies for collecting performance data to suit different analysis needs. + +### 1. Uniform Sampling Strategy + +**When to use**: Daily monitoring and trend analysis over extended periods. + +- **Automatically enabled** when `--limit >= 500` +- **Disabled** for smaller limits (< 500) to maintain backward compatibility + +#### How it works: +- Collects data uniformly across a 30-day period +- Ensures even time distribution of samples +- Provides consistent coverage for trend analysis + +#### Example with 1000 Runs: +- **Time Range**: Last 30 days +- **Distribution**: 1000 samples evenly distributed across the period +- **Coverage**: ~33 samples per day on average + +### 2. Date Range Collection + +**When to use**: Historical analysis, specific period investigation, or complete data collection. + +Use `--start-date` and `--end-date` parameters to get **ALL** CI runs within a specific time range. + +#### Features: +- **Complete Data**: Gets every CI run in the specified range (no sampling) +- **No Limit**: Ignores the `--limit` parameter +- **Flexible Range**: Specify any date range you need +- **Historical Analysis**: Perfect for investigating specific time periods + +#### Date Format: +- Use `YYYY-MM-DD` format (e.g., `2024-12-01`) +- Both parameters are optional: + - Only `--start-date`: Gets all runs from that date to now + - Only `--end-date`: Gets all runs from 30 days ago to that date + - Both: Gets all runs in the specified range + +### 3. Sequential Collection (Traditional) + +**When to use**: Quick checks or when you only need recent data. + +- **Default behavior** for `--limit < 500` +- Gets the most recent CI runs in chronological order +- Fast and simple for immediate analysis + +### Comparison + +| Strategy | Use Case | Time Coverage | Data Completeness | API Efficiency | +|----------|----------|---------------|-------------------|----------------| +| **Uniform Sampling** | Daily monitoring, trends | ~30 days | Sampled | High | +| **Date Range** | Historical analysis | Any range | Complete | Variable | +| **Sequential** | Quick checks | 3-4 days | Complete (recent) | High | + +### Benefits + +- **Flexible Analysis**: Choose the right strategy for your needs +- **Extended Coverage**: Up to 30 days with sampling, unlimited with date ranges +- **Complete Data**: Get every run in a specific period when needed +- **API Efficiency**: Optimized for different use patterns + +## Parameters + +### CI Analyzer Parameters + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `--token` | Required | GitHub Personal Access Token | +| `--limit` | 100 | Number of CI runs to analyze | +| `--output` | ci_analysis.json | Output JSON file for detailed data | + +### Performance Analyzer Parameters + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `--token` | Required | GitHub Personal Access Token | +| `--limit` | 100 | Number of PR Test runs to analyze (ignored when using date range) | +| `--output-dir` | performance_tables | Output directory for CSV tables and PNG charts | +| `--start-date` | None | Start date for date range query (YYYY-MM-DD format) | +| `--end-date` | None | End date for date range query (YYYY-MM-DD format) | +| `--upload-to-github` | False | Upload results to sglang-bot/sglang-ci-data repository | + +### Test Balance Analyzer Parameters + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `--token` | Required | GitHub Personal Access Token | +| `--limit` | 1000 | Number of CI runs to analyze | +| `--output` | test_balance_report.json | Output JSON file for detailed analysis data | + +### Failures Analyzer Parameters + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `--token` | Required | GitHub Personal Access Token | +| `--limit` | 500 | Number of workflow runs to analyze | +| `--threshold` | 3 | Alert threshold for consecutive failures | +| `--output` | None | Output JSON file (optional, only writes if specified) | + +## Getting GitHub Token + +1. Go to [GitHub Settings > Personal Access Tokens](https://github.com/settings/tokens) +2. Click "Generate new token" > "Generate new token (classic)" +3. **Important**: Select the following permissions: + - `repo` (Full control of private repositories) - **Required for accessing repository data** + - `workflow` (Update GitHub Action workflows) - **Required for reading CI/CD data** +4. Copy the generated token and use it as `YOUR_GITHUB_TOKEN` + +**Note**: Without the `repo` and `workflow` permissions, the tool will not be able to access CI run data and will return 404 errors. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci_monitor/ci_analyzer.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci_monitor/ci_analyzer.py new file mode 100755 index 00000000..150e4f4f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci_monitor/ci_analyzer.py @@ -0,0 +1,1206 @@ +#!/usr/bin/env python3 + +import argparse +import base64 +import json +import os +import re +import sys +import time +from collections import Counter, defaultdict +from datetime import datetime, timedelta +from typing import Dict, List, Optional + +import requests + + +class SGLangCIAnalyzer: + + def __init__(self, token: str): + self.token = token + self.base_url = "https://api.github.com" + self.repo = "sgl-project/sglang" + self.headers = { + "Authorization": f"token {token}", + "Accept": "application/vnd.github.v3+json", + "User-Agent": "SGLang-CI-Analyzer/1.0", + } + self.session = requests.Session() + self.session.headers.update(self.headers) + + # Nightly workflow files to monitor + self.nightly_workflows = [ + "nightly-test-nvidia.yml", + "nightly-test-amd.yml", + "nightly-test-intel.yml", + ] + + # Performance metric patterns for parsing logs + self.perf_patterns = { + "output_throughput": re.compile( + r"Output token throughput \(tok/s\):\s*([\d.]+)" + ), + "input_throughput": re.compile( + r"Input token throughput \(tok/s\):\s*([\d.]+)" + ), + "latency": re.compile(r"Median E2E Latency \(ms\):\s*([\d.]+)"), + "ttft": re.compile(r"Median TTFT \(ms\):\s*([\d.]+)"), + "accept_length": re.compile(r"Accept length:\s*([\d.]+)"), + "accuracy": re.compile(r"Accuracy:\s*([\d.]+)"), + "gsm8k_score": re.compile(r"GSM8K Score:\s*([\d.]+)"), + } + + # Historical data repository + self.data_repo = "sglang-bot/sglang-ci-data" + self.data_branch = "main" + + def get_recent_runs(self, limit: int = 100, branch: str = None) -> List[Dict]: + branch_info = f" from branch '{branch}'" if branch else "" + print(f"Fetching {limit} recent CI runs{branch_info}...") + + all_runs = [] + page = 1 + per_page = 100 + + while len(all_runs) < limit: + url = f"{self.base_url}/repos/{self.repo}/actions/runs" + params = {"per_page": min(per_page, limit - len(all_runs)), "page": page} + if branch: + params["branch"] = branch + + try: + response = self.session.get(url, params=params) + response.raise_for_status() + data = response.json() + + if not data.get("workflow_runs"): + break + + all_runs.extend(data["workflow_runs"]) + print(f"Fetched {len(all_runs)} runs so far...") + + if len(data["workflow_runs"]) < per_page: + break + + page += 1 + time.sleep(0.1) + + except requests.exceptions.RequestException as e: + print(f"Error fetching CI data: {e}") + break + + return all_runs[:limit] + + def analyze_ci_failures(self, runs: List[Dict]) -> Dict: + print( + "Analyzing CI failure data (pr-test.yml, quantization-test.yml, nightly-test.yml jobs only)..." + ) + + job_categories = { + "build": [ + "build-test", + "sgl-kernel-build-wheels", + ], + "unit-test": [ + "stage-a-test-1", + "unit-test-backend-1-gpu", + "unit-test-backend-2-gpu", + "unit-test-backend-4-gpu", + "unit-test-backend-8-gpu", + ], + "performance": [ + "performance-test-1-gpu-part-1", + "performance-test-1-gpu-part-2", + "performance-test-1-gpu-part-3", + "performance-test-2-gpu", + ], + "accuracy": [ + "accuracy-test-1-gpu", + "accuracy-test-2-gpu", + ], + "mla-test": [ + "sgl-kernel-mla-test", + ], + "deepep": [ + "unit-test-deepep-4-gpu", + "unit-test-deepep-8-gpu", + ], + "per-commit": [ + "per-commit-8-gpu-h20", + ], + "nightly": [ + # NVIDIA job names (nightly-test-nvidia.yml) + "nightly-test-general-1-gpu-runner", + "nightly-test-general-4-gpu-h100", + "nightly-test-general-8-gpu-h200", + "nightly-test-general-8-gpu-h20", + "nightly-test-text-accuracy-2-gpu-runner", + "nightly-test-text-perf-2-gpu-runner", + "nightly-test-vlm-accuracy-2-gpu-runner", + "nightly-test-vlm-perf-2-gpu-runner", + "nightly-test-perf-4-gpu-b200", + "nightly-test-perf-8-gpu-b200", + # AMD job names (nightly-test-amd.yml) + "nightly-test", # AMD uses this generic name with matrix + ], + "integration": [ + "run-all-notebooks", + "quantization-test", + "test-disaggregation", + ], + "b200": [ + "unit-test-backend-4-gpu-b200", + ], + "gb200": [ + "unit-test-backend-4-gpu-gb200", + ], + } + + stats = { + "total_runs": len(runs), + "failed_runs": 0, + "successful_runs": 0, + "cancelled_runs": 0, + "skipped_runs": 0, + "category_failures": defaultdict(int), + "job_failures": defaultdict(int), + "failure_patterns": defaultdict(int), + "job_failure_links": defaultdict( + list + ), # Store recent failure links for each job + "job_last_success": {}, # Store last successful run for each job + "performance_metrics": defaultdict( + lambda: defaultdict(list) + ), # Track performance metrics for nightly jobs + } + + total_runs = len(runs) + for i, run in enumerate(runs, 1): + if i % max(1, min(50, total_runs // 10)) == 0 or i == total_runs: + progress = (i / total_runs) * 100 + print(f"Progress: {i}/{total_runs} ({progress:.1f}%)") + + run_status = run.get("conclusion", "unknown") + workflow_name = run.get("name", "Unknown") + run_id = run.get("id") + run_number = run.get("run_number") + created_at = run.get("created_at") + + if run_status == "failure": + stats["failed_runs"] += 1 + elif run_status == "success": + stats["successful_runs"] += 1 + elif run_status == "cancelled": + stats["cancelled_runs"] += 1 + elif run_status == "skipped": + stats["skipped_runs"] += 1 + + jobs = self._get_job_details(run_id) + run_url = f"https://github.com/{self.repo}/actions/runs/{run_id}" + pr_info = self._get_pr_info(run) + + for job in jobs: + job_name = job.get("name", "Unknown") + job_conclusion = job.get("conclusion", "unknown") + + target_jobs = [ + "check-changes", + "sgl-kernel-build-wheels", + "sgl-kernel-unit-test", + "sgl-kernel-mla-test", + "sgl-kernel-benchmark-test", + "stage-a-test-1", + "unit-test-backend-1-gpu", + "unit-test-backend-2-gpu", + "unit-test-backend-4-gpu", + "unit-test-backend-8-gpu-h200", + "unit-test-backend-8-gpu-h20", + "performance-test-1-gpu-part-1", + "performance-test-1-gpu-part-2", + "performance-test-1-gpu-part-3", + "performance-test-2-gpu", + "accuracy-test-1-gpu", + "accuracy-test-2-gpu", + "unit-test-deepep-4-gpu", + "unit-test-deepep-8-gpu", + "unit-test-backend-8-gpu-deepseek-v32", + "unit-test-backend-4-gpu-b200", + "unit-test-backend-4-gpu-gb200", + "quantization-test", + # NVIDIA job names (nightly-test-nvidia.yml) + "nightly-test-general-1-gpu-runner", + "nightly-test-general-4-gpu-h100", + "nightly-test-general-8-gpu-h200", + "nightly-test-general-8-gpu-h20", + "nightly-test-text-accuracy-2-gpu-runner", + "nightly-test-text-perf-2-gpu-runner", + "nightly-test-vlm-accuracy-2-gpu-runner", + "nightly-test-vlm-perf-2-gpu-runner", + "nightly-test-perf-4-gpu-b200", + "nightly-test-perf-8-gpu-b200", + # AMD job names (nightly-test-amd.yml) + "nightly-test", + ] + + if job_name in target_jobs: + if job_conclusion == "success": + stats["job_last_success"][job_name] = { + "url": run_url, + "run_number": run_number, + "created_at": created_at, + "pr_info": pr_info, + } + + # Parse performance metrics from successful nightly jobs + if job_name in job_categories["nightly"] and ( + "perf" in job_name.lower() + or "accuracy" in job_name.lower() + or "eval" in job_name.lower() + ): + job_id = job.get("id") + logs = self.get_job_logs(job_id) + if logs: + metrics = self.parse_metrics_from_logs(logs, job_name) + for metric_name, values in metrics.items(): + if values: + for value in values: + stats["performance_metrics"][job_name][ + metric_name + ].append( + { + "value": value, + "timestamp": created_at, + "run_id": run_id, + "run_url": run_url, + } + ) + + elif job_conclusion == "failure": + stats["job_failures"][job_name] += 1 + + if len(stats["job_failure_links"][job_name]) < 3: + stats["job_failure_links"][job_name].append( + { + "url": run_url, + "run_number": run_number, + "created_at": created_at, + "pr_info": pr_info, + } + ) + + for category, jobs_list in job_categories.items(): + if any( + job_pattern in job_name for job_pattern in jobs_list + ): + stats["category_failures"][category] += 1 + break + + self._analyze_failure_pattern(job, stats) + + time.sleep(0.1) + + return stats + + def _get_job_details(self, run_id: int) -> List[Dict]: + url = f"{self.base_url}/repos/{self.repo}/actions/runs/{run_id}/jobs" + try: + response = self.session.get(url) + response.raise_for_status() + return response.json().get("jobs", []) + except: + return [] + + def _get_pr_info(self, run: Dict) -> Dict: + pr_info = { + "pr_number": None, + "author": run.get("head_commit", {}) + .get("author", {}) + .get("name", "Unknown"), + "head_sha": run.get("head_sha", ""), + "head_branch": run.get("head_branch", ""), + } + + pull_requests = run.get("pull_requests", []) + if pull_requests: + pr_info["pr_number"] = pull_requests[0].get("number") + + return pr_info + + def _analyze_failure_pattern(self, job: Dict, stats: Dict): + job_name = job.get("name", "") + steps = job.get("steps", []) + + for step in steps: + if step.get("conclusion") == "failure": + step_name = step.get("name", "") + + if "timeout" in step_name.lower(): + stats["failure_patterns"]["Timeout"] += 1 + elif "build" in step_name.lower() or "build" in job_name.lower(): + stats["failure_patterns"]["Build Failure"] += 1 + elif "install" in step_name.lower() or "dependency" in job_name.lower(): + stats["failure_patterns"]["Dependency Installation Failure"] += 1 + elif "unit" in job_name.lower() or "unit-test" in job_name.lower(): + stats["failure_patterns"]["Unit Test Failure"] += 1 + elif "performance" in job_name.lower() or "perf" in job_name.lower(): + stats["failure_patterns"]["Performance Test Failure"] += 1 + elif "accuracy" in job_name.lower(): + stats["failure_patterns"]["Accuracy Test Failure"] += 1 + elif "mla" in job_name.lower(): + stats["failure_patterns"]["MLA Test Failure"] += 1 + elif "deepep" in job_name.lower(): + stats["failure_patterns"]["DeepEP Test Failure"] += 1 + elif "nightly" in job_name.lower(): + stats["failure_patterns"]["Nightly Test Failure"] += 1 + elif "notebook" in job_name.lower(): + stats["failure_patterns"]["Notebook Test Failure"] += 1 + elif "disaggregation" in job_name.lower(): + stats["failure_patterns"]["Disaggregation Test Failure"] += 1 + elif "h20" in job_name.lower() or "h200" in job_name.lower(): + stats["failure_patterns"]["H20/H200 GPU Failure"] += 1 + elif "b200" in job_name.lower(): + stats["failure_patterns"]["B200 GPU Failure"] += 1 + elif "gpu" in job_name.lower(): + stats["failure_patterns"]["GPU Related Failure"] += 1 + else: + stats["failure_patterns"]["Other"] += 1 + + def generate_report(self, stats: Dict): + print("\n" + "=" * 60) + print("SGLang CI Analysis Report (Target Workflows Only)") + print("=" * 60) + + total = stats["total_runs"] + failed = stats["failed_runs"] + success = stats["successful_runs"] + cancelled = stats["cancelled_runs"] + skipped = stats["skipped_runs"] + success_rate = (success / total * 100) if total > 0 else 0 + + print(f"\nOverall Statistics:") + print(f" Total runs: {total}") + print(f" Successful: {success}") + print(f" Failed: {failed}") + print(f" Cancelled: {cancelled}") + print(f" Skipped: {skipped}") + print(f" Success rate: {success_rate:.1f}%") + + if stats["category_failures"]: + print(f"\nCategory Failure Statistics:") + for category, count in sorted( + stats["category_failures"].items(), key=lambda x: x[1], reverse=True + ): + print(f" {category}: {count} failures") + + if stats["job_failures"]: + print(f"\nMost Frequently Failed Jobs (Top 50):") + for i, (job, count) in enumerate( + sorted(stats["job_failures"].items(), key=lambda x: x[1], reverse=True)[ + :50 + ], + 1, + ): + print(f" {i:2d}. {job}: {count} times") + + if job in stats["job_last_success"]: + last_success = stats["job_last_success"][job] + success_date = datetime.fromisoformat( + last_success["created_at"].replace("Z", "+00:00") + ) + pr_info = last_success["pr_info"] + + pr_text = "" + if pr_info["pr_number"]: + pr_text = ( + f" (PR #{pr_info['pr_number']} by {pr_info['author']})" + ) + else: + pr_text = f" by {pr_info['author']}" + + print( + f" Last Success: Run #{last_success['run_number']} ({success_date.strftime('%Y-%m-%d %H:%M')}){pr_text}: {last_success['url']}" + ) + + if ( + job in stats["job_failure_links"] + and stats["job_failure_links"][job] + ): + print(" Recent Failures:") + for link_info in stats["job_failure_links"][job]: + created_at = datetime.fromisoformat( + link_info["created_at"].replace("Z", "+00:00") + ) + + pr_info = link_info.get("pr_info", {}) + pr_text = "" + if pr_info.get("pr_number"): + pr_text = f" (PR #{pr_info['pr_number']} by {pr_info.get('author', 'Unknown')})" + else: + pr_text = f" by {pr_info.get('author', 'Unknown')}" + + print( + f" - Run #{link_info['run_number']} ({created_at.strftime('%Y-%m-%d %H:%M')}){pr_text}: {link_info['url']}" + ) + + if stats["failure_patterns"]: + print(f"\nFailure Pattern Analysis:") + for pattern, count in sorted( + stats["failure_patterns"].items(), key=lambda x: x[1], reverse=True + ): + print(f" {pattern}: {count} times") + + print("\n" + "=" * 60) + + def save_detailed_report(self, stats: Dict, output_file: str = "ci_analysis.json"): + with open(output_file, "w", encoding="utf-8") as f: + json.dump(stats, f, ensure_ascii=False, indent=2) + print(f"\nDetailed report saved to: {output_file}") + + def generate_github_summary(self, stats: Dict): + try: + github_step_summary = os.environ.get("GITHUB_STEP_SUMMARY") + if not github_step_summary: + print("Not running in GitHub Actions, skipping summary generation") + return + + print("Generating GitHub Actions summary for CI Analysis...") + + summary_lines = [] + summary_lines.append("# SGLang CI Analysis Report (Target Workflows Only)") + summary_lines.append("") + + total = stats["total_runs"] + failed = stats["failed_runs"] + success = stats["successful_runs"] + cancelled = stats["cancelled_runs"] + skipped = stats["skipped_runs"] + success_rate = (success / total * 100) if total > 0 else 0 + + summary_lines.append("## Overall Statistics") + summary_lines.append("") + summary_lines.append("| Metric | Count | Percentage |") + summary_lines.append("|--------|-------|------------|") + summary_lines.append(f"| Total Runs | {total} | 100% |") + summary_lines.append( + f"| Successful | {success} | {success/total*100:.1f}% |" + ) + summary_lines.append(f"| Failed | {failed} | {failed/total*100:.1f}% |") + summary_lines.append( + f"| Cancelled | {cancelled} | {cancelled/total*100:.1f}% |" + ) + summary_lines.append(f"| Skipped | {skipped} | {skipped/total*100:.1f}% |") + summary_lines.append(f"| **Success Rate** | **{success_rate:.1f}%** | - |") + summary_lines.append("") + + if stats["category_failures"]: + summary_lines.append("## Category Failure Statistics") + summary_lines.append("") + summary_lines.append("| Category | Failures |") + summary_lines.append("|----------|----------|") + for category, count in sorted( + stats["category_failures"].items(), key=lambda x: x[1], reverse=True + ): + summary_lines.append(f"| {category} | {count} |") + summary_lines.append("") + + if stats["job_failures"]: + summary_lines.append("## Most Frequently Failed Jobs (Top 20)") + summary_lines.append("") + + top_failures = sorted( + stats["job_failures"].items(), key=lambda x: x[1], reverse=True + )[:20] + + for i, (job, count) in enumerate(top_failures, 1): + summary_lines.append(f"### {i}. `{job}` ({count} failures)") + summary_lines.append("") + + if job in stats["job_last_success"]: + last_success = stats["job_last_success"][job] + success_date = datetime.fromisoformat( + last_success["created_at"].replace("Z", "+00:00") + ) + pr_info = last_success["pr_info"] + + pr_text = "" + if pr_info["pr_number"]: + pr_text = ( + f" (PR #{pr_info['pr_number']} by {pr_info['author']})" + ) + else: + pr_text = f" by {pr_info['author']}" + + summary_lines.append( + f"**Last Success:** [Run #{last_success['run_number']}]({last_success['url']}) ({success_date.strftime('%Y-%m-%d %H:%M')}){pr_text}" + ) + summary_lines.append("") + + if ( + job in stats["job_failure_links"] + and stats["job_failure_links"][job] + ): + summary_lines.append("**Recent Failures:**") + for link_info in stats["job_failure_links"][job]: + created_at = datetime.fromisoformat( + link_info["created_at"].replace("Z", "+00:00") + ) + + pr_info = link_info.get("pr_info", {}) + pr_text = "" + if pr_info.get("pr_number"): + pr_text = f" (PR #{pr_info['pr_number']} by {pr_info.get('author', 'Unknown')})" + else: + pr_text = f" by {pr_info.get('author', 'Unknown')}" + + summary_lines.append( + f"- [Run #{link_info['run_number']}]({link_info['url']}) ({created_at.strftime('%Y-%m-%d %H:%M')}){pr_text}" + ) + summary_lines.append("") + + if stats["failure_patterns"]: + summary_lines.append("## Failure Pattern Analysis") + summary_lines.append("") + summary_lines.append("| Pattern | Count |") + summary_lines.append("|---------|-------|") + for pattern, count in sorted( + stats["failure_patterns"].items(), key=lambda x: x[1], reverse=True + ): + summary_lines.append(f"| {pattern} | {count} |") + summary_lines.append("") + + # Performance metrics section for nightly jobs + if stats.get("performance_metrics"): + summary_lines.append("## Nightly Test Performance Metrics") + summary_lines.append("") + summary_lines.append("| Job | Metric | Latest Value | Count | Trend |") + summary_lines.append("|-----|--------|--------------|-------|-------|") + + for job_name in sorted(stats["performance_metrics"].keys()): + job_metrics = stats["performance_metrics"][job_name] + for metric_name in sorted(job_metrics.keys()): + metric_data = job_metrics[metric_name] + if metric_data: + # Calculate average of recent values + values = [m["value"] for m in metric_data] + avg_value = sum(values) / len(values) + count = len(values) + + # Simple trend: compare first half vs second half + trend_indicator = "➡️" + if len(values) >= 4: + first_half = values[: len(values) // 2] + second_half = values[len(values) // 2 :] + first_avg = sum(first_half) / len(first_half) + second_avg = sum(second_half) / len(second_half) + + if first_avg > 0: + change_pct = ( + (second_avg - first_avg) / first_avg + ) * 100 + + # For throughput metrics, up is good + # For latency/ttft metrics, down is good + if "throughput" in metric_name.lower(): + if change_pct > 10: + trend_indicator = f"📈 +{change_pct:.1f}%" + elif change_pct < -10: + trend_indicator = f"⚠️ 📉 {change_pct:.1f}%" + else: + trend_indicator = f"➡️ {change_pct:+.1f}%" + elif ( + "latency" in metric_name.lower() + or "ttft" in metric_name.lower() + ): + if change_pct < -10: + trend_indicator = f"📈 {change_pct:.1f}%" + elif change_pct > 10: + trend_indicator = f"⚠️ 📉 +{change_pct:.1f}%" + else: + trend_indicator = f"➡️ {change_pct:+.1f}%" + else: + trend_indicator = f"➡️ {change_pct:+.1f}%" + + summary_lines.append( + f"| {job_name} | {metric_name} | {avg_value:.2f} | {count} | {trend_indicator} |" + ) + + summary_lines.append("") + + with open(github_step_summary, "w", encoding="utf-8") as f: + f.write("\n".join(summary_lines)) + f.write("\n\n---\n\n") + + print("GitHub Actions summary generated successfully") + + except Exception as e: + print(f"Failed to generate GitHub Actions summary: {e}") + + def get_nightly_runs(self, days: int = 2) -> List[Dict]: + """Get nightly test workflow runs from the last N days""" + print(f"Fetching nightly test runs from the last {days} days...") + + since_date = (datetime.now() - timedelta(days=days)).isoformat() + all_runs = [] + + for workflow_file in self.nightly_workflows: + print(f" Fetching from {workflow_file}...") + page = 1 + per_page = 10 # Nightly runs once per day, so 10 runs covers ~10 days max + workflow_runs = [] + max_runs_per_workflow = days * 5 # Allow up to 5 runs per day per workflow + + while len(workflow_runs) < max_runs_per_workflow: + url = f"{self.base_url}/repos/{self.repo}/actions/runs" + params = { + "workflow_id": workflow_file, + "per_page": per_page, + "page": page, + "created": f">={since_date}", + } + + try: + response = self.session.get(url, params=params) + response.raise_for_status() + data = response.json() + + if not data.get("workflow_runs"): + break + + runs = data["workflow_runs"] + workflow_runs.extend(runs) + + if len(runs) < per_page: + break + + page += 1 + time.sleep(0.1) + + except requests.exceptions.RequestException as e: + print(f" Warning: Error fetching from {workflow_file}: {e}") + break + + print(f" Fetched {len(workflow_runs)} runs from {workflow_file}") + all_runs.extend(workflow_runs) + + print(f"Total nightly runs fetched: {len(all_runs)}") + return all_runs + + def get_job_logs(self, job_id: int) -> Optional[str]: + """Get logs for a specific job""" + url = f"{self.base_url}/repos/{self.repo}/actions/jobs/{job_id}/logs" + try: + response = self.session.get(url) + response.raise_for_status() + return response.text + except requests.exceptions.RequestException as e: + print(f" Warning: Could not fetch logs for job {job_id}: {e}") + return None + + def parse_metrics_from_logs( + self, logs: str, job_name: str + ) -> Dict[str, List[float]]: + """Parse performance metrics from job logs""" + metrics = defaultdict(list) + + if not logs: + return metrics + + for line in logs.split("\n"): + for metric_name, pattern in self.perf_patterns.items(): + match = pattern.search(line) + if match: + try: + value = float(match.group(1)) + metrics[metric_name].append(value) + except (ValueError, IndexError): + continue + + return dict(metrics) + + def analyze_nightly_with_metrics(self, runs: List[Dict]) -> Dict: + """Analyze nightly test runs including performance metrics""" + print("Analyzing nightly test data with performance metrics...") + + # Get nightly job names from the existing job categories + nightly_jobs = [ + # NVIDIA job names (nightly-test-nvidia.yml) + "nightly-test-general-1-gpu-runner", + "nightly-test-general-4-gpu-h100", + "nightly-test-general-8-gpu-h200", + "nightly-test-general-8-gpu-h20", + "nightly-test-text-accuracy-2-gpu-runner", + "nightly-test-text-perf-2-gpu-runner", + "nightly-test-vlm-accuracy-2-gpu-runner", + "nightly-test-vlm-perf-2-gpu-runner", + "nightly-test-perf-4-gpu-b200", + "nightly-test-perf-8-gpu-b200", + # AMD job names (nightly-test-amd.yml) + "nightly-test", + # Intel job names (nightly-test-intel.yml) + "placeholder", + ] + + stats = { + "total_runs": len(runs), + "successful_runs": 0, + "failed_runs": 0, + "cancelled_runs": 0, + "job_stats": defaultdict( + lambda: { + "total": 0, + "success": 0, + "failure": 0, + "recent_failures": [], + "avg_duration_minutes": 0, + "durations": [], + "performance_metrics": defaultdict(list), + } + ), + "daily_stats": defaultdict( + lambda: { + "total": 0, + "success": 0, + "failure": 0, + } + ), + } + + for i, run in enumerate(runs, 1): + if i % 10 == 0: + print(f"Processed {i}/{len(runs)} runs...") + + run_status = run.get("conclusion", "unknown") + run_id = run.get("id") + run_number = run.get("run_number") + created_at = run.get("created_at") + run_url = f"https://github.com/{self.repo}/actions/runs/{run_id}" + + # Track daily stats + date_str = created_at.split("T")[0] if created_at else "unknown" + stats["daily_stats"][date_str]["total"] += 1 + + if run_status == "success": + stats["successful_runs"] += 1 + stats["daily_stats"][date_str]["success"] += 1 + elif run_status == "failure": + stats["failed_runs"] += 1 + stats["daily_stats"][date_str]["failure"] += 1 + elif run_status == "cancelled": + stats["cancelled_runs"] += 1 + + # Analyze individual jobs + jobs = self._get_job_details(run_id) + for job in jobs: + job_name = job.get("name", "Unknown") + job_conclusion = job.get("conclusion", "unknown") + job_id = job.get("id") + started_at = job.get("started_at") + completed_at = job.get("completed_at") + + # Only track nightly test jobs + if job_name not in nightly_jobs: + continue + + job_stat = stats["job_stats"][job_name] + job_stat["total"] += 1 + + if job_conclusion == "success": + job_stat["success"] += 1 + + # For successful performance/accuracy jobs, fetch metrics + if ( + "perf" in job_name.lower() + or "accuracy" in job_name.lower() + or "eval" in job_name.lower() + ): + logs = self.get_job_logs(job_id) + if logs: + metrics = self.parse_metrics_from_logs(logs, job_name) + for metric_name, values in metrics.items(): + if values: + job_stat["performance_metrics"][metric_name].extend( + [ + { + "value": v, + "timestamp": created_at, + "run_id": run_id, + "job_name": job_name, + } + for v in values + ] + ) + + elif job_conclusion == "failure": + job_stat["failure"] += 1 + + if len(job_stat["recent_failures"]) < 5: + job_stat["recent_failures"].append( + { + "run_url": run_url, + "run_number": run_number, + "created_at": created_at, + "job_url": job.get("html_url"), + } + ) + + # Track duration + if started_at and completed_at: + try: + start = datetime.fromisoformat( + started_at.replace("Z", "+00:00") + ) + end = datetime.fromisoformat( + completed_at.replace("Z", "+00:00") + ) + duration_minutes = (end - start).total_seconds() / 60 + job_stat["durations"].append(duration_minutes) + except: + pass + + time.sleep(0.1) + + # Calculate average durations + for job_name, job_stat in stats["job_stats"].items(): + if job_stat["durations"]: + job_stat["avg_duration_minutes"] = sum(job_stat["durations"]) / len( + job_stat["durations"] + ) + del job_stat["durations"] + + return stats + + def generate_nightly_report(self, stats: Dict, output_file: str = None): + """Generate a report for nightly test analysis""" + print("\n" + "=" * 80) + print("NIGHTLY TEST MONITOR REPORT") + print("=" * 80) + print(f"Report Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + print(f"Total Runs Analyzed: {stats['total_runs']}") + print( + f"Successful: {stats['successful_runs']} " + f"({stats['successful_runs']/max(1, stats['total_runs'])*100:.1f}%)" + ) + print( + f"Failed: {stats['failed_runs']} " + f"({stats['failed_runs']/max(1, stats['total_runs'])*100:.1f}%)" + ) + print(f"Cancelled: {stats['cancelled_runs']}") + print("=" * 80) + + # Daily trend + print("\nDAILY TRENDS:") + print("-" * 80) + daily_stats = sorted(stats["daily_stats"].items(), reverse=True)[:7] + for date, day_stats in daily_stats: + success_rate = (day_stats["success"] / max(1, day_stats["total"])) * 100 + print( + f"{date}: {day_stats['total']} runs, {day_stats['success']} success " + f"({success_rate:.1f}%), {day_stats['failure']} failed" + ) + + # Job statistics + print("\nJOB STATISTICS:") + print("-" * 80) + print( + f"{'Job Name':<50} {'Total':<8} {'Success':<8} {'Failed':<8} " + f"{'Rate':<8} {'Avg Duration'}" + ) + print("-" * 80) + + job_stats_sorted = sorted( + stats["job_stats"].items(), key=lambda x: x[1]["failure"], reverse=True + ) + + for job_name, job_stat in job_stats_sorted: + total = job_stat["total"] + success = job_stat["success"] + failure = job_stat["failure"] + success_rate = (success / max(1, total)) * 100 + avg_duration = job_stat["avg_duration_minutes"] + + print( + f"{job_name:<50} {total:<8} {success:<8} {failure:<8} " + f"{success_rate:>6.1f}% {avg_duration:>7.1f}m" + ) + + # Show performance metrics if available + if job_stat.get("performance_metrics"): + perf_metrics = job_stat["performance_metrics"] + print(f" Performance metrics:") + + for metric_name, metric_data in perf_metrics.items(): + if metric_data: + values = [m["value"] for m in metric_data] + avg_value = sum(values) / len(values) + print(f" - {metric_name}: {avg_value:.2f} (n={len(values)})") + + # Show recent failures + if job_stat["recent_failures"]: + print(f" Recent failures:") + for failure in job_stat["recent_failures"][:3]: + print(f" - Run #{failure['run_number']}: {failure['run_url']}") + + print("=" * 80) + + # Save to file if requested + if output_file: + with open(output_file, "w") as f: + json.dump(stats, f, indent=2, default=str) + print(f"\nDetailed stats saved to: {output_file}") + + def generate_nightly_github_summary(self, stats: Dict): + """Generate GitHub Actions summary for nightly test analysis""" + try: + github_step_summary = os.environ.get("GITHUB_STEP_SUMMARY") + if not github_step_summary: + print( + "Not running in GitHub Actions, skipping nightly summary generation" + ) + return + + print("Generating GitHub Actions summary for Nightly Analysis...") + + summary_lines = [] + summary_lines.append("# Nightly Test Monitor Report") + summary_lines.append("") + summary_lines.append( + f"**Report Generated:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" + ) + summary_lines.append("") + + # Overall statistics + total = stats["total_runs"] + success = stats["successful_runs"] + failed = stats["failed_runs"] + cancelled = stats["cancelled_runs"] + + summary_lines.append("## Overall Statistics") + summary_lines.append("") + summary_lines.append("| Metric | Count | Percentage |") + summary_lines.append("|--------|-------|------------|") + summary_lines.append(f"| Total Runs | {total} | 100% |") + summary_lines.append( + f"| Successful | {success} | {success/max(1,total)*100:.1f}% |" + ) + summary_lines.append( + f"| Failed | {failed} | {failed/max(1,total)*100:.1f}% |" + ) + summary_lines.append( + f"| Cancelled | {cancelled} | {cancelled/max(1,total)*100:.1f}% |" + ) + summary_lines.append("") + + # Daily trends + summary_lines.append("## Daily Trends") + summary_lines.append("") + summary_lines.append( + "| Date | Total Runs | Success | Failed | Success Rate |" + ) + summary_lines.append( + "|------|------------|---------|--------|--------------|" + ) + + daily_stats = sorted(stats["daily_stats"].items(), reverse=True)[:7] + for date, day_stats in daily_stats: + success_rate = (day_stats["success"] / max(1, day_stats["total"])) * 100 + summary_lines.append( + f"| {date} | {day_stats['total']} | {day_stats['success']} | " + f"{day_stats['failure']} | {success_rate:.1f}% |" + ) + summary_lines.append("") + + # Job statistics with performance metrics + if stats["job_stats"]: + summary_lines.append("## Job Statistics") + summary_lines.append("") + + job_stats_sorted = sorted( + stats["job_stats"].items(), + key=lambda x: x[1]["failure"], + reverse=True, + ) + + for job_name, job_stat in job_stats_sorted: + total_job = job_stat["total"] + success_job = job_stat["success"] + failure_job = job_stat["failure"] + success_rate_job = (success_job / max(1, total_job)) * 100 + avg_duration = job_stat["avg_duration_minutes"] + + summary_lines.append(f"### {job_name}") + summary_lines.append("") + summary_lines.append( + f"**Stats:** {total_job} runs | {success_job} success ({success_rate_job:.1f}%) | " + f"{failure_job} failed | Avg duration: {avg_duration:.1f}m" + ) + summary_lines.append("") + + # Performance metrics + if job_stat.get("performance_metrics"): + summary_lines.append("**Performance Metrics:**") + summary_lines.append("") + summary_lines.append("| Metric | Avg Value | Samples |") + summary_lines.append("|--------|-----------|---------|") + + for metric_name, metric_data in job_stat[ + "performance_metrics" + ].items(): + if metric_data: + values = [m["value"] for m in metric_data] + avg_value = sum(values) / len(values) + summary_lines.append( + f"| {metric_name} | {avg_value:.2f} | {len(values)} |" + ) + summary_lines.append("") + + # Recent failures + if job_stat["recent_failures"]: + summary_lines.append("**Recent Failures:**") + for failure in job_stat["recent_failures"][:3]: + summary_lines.append( + f"- [Run #{failure['run_number']}]({failure['run_url']})" + ) + summary_lines.append("") + + with open(github_step_summary, "a", encoding="utf-8") as f: + f.write("\n".join(summary_lines)) + f.write("\n\n---\n\n") + + print("GitHub Actions nightly summary generated successfully") + + except Exception as e: + print(f"Failed to generate nightly GitHub Actions summary: {e}") + + def detect_nightly_regressions(self, stats: Dict) -> List[Dict]: + """Detect regressions in nightly tests""" + regressions = [] + + for job_name, job_stat in stats["job_stats"].items(): + total = job_stat["total"] + failure = job_stat["failure"] + + if total > 0: + failure_rate = (failure / total) * 100 + + # Flag jobs with high failure rates + if failure_rate > 30: + regressions.append( + { + "job_name": job_name, + "type": "high_failure_rate", + "failure_rate": failure_rate, + "total_runs": total, + "failures": failure, + } + ) + + # Flag jobs with recent consecutive failures + recent_failures = len(job_stat["recent_failures"]) + if recent_failures >= 3: + regressions.append( + { + "job_name": job_name, + "type": "consecutive_failures", + "recent_failure_count": recent_failures, + } + ) + + if regressions: + print("\n" + "=" * 80) + print("REGRESSIONS DETECTED:") + print("=" * 80) + for regression in regressions: + print(f"\nJob: {regression['job_name']}") + if regression["type"] == "high_failure_rate": + print( + f" High failure rate: {regression['failure_rate']:.1f}% " + f"({regression['failures']}/{regression['total_runs']})" + ) + elif regression["type"] == "consecutive_failures": + print( + f" {regression['recent_failure_count']} recent consecutive failures" + ) + print("=" * 80) + + return regressions + + +def main(): + parser = argparse.ArgumentParser(description="SGLang CI Analyzer") + parser.add_argument("--token", required=True, help="GitHub Personal Access Token") + parser.add_argument( + "--mode", + choices=["ci", "nightly"], + default="ci", + help="Analysis mode: 'ci' for general CI analysis, 'nightly' for nightly test monitoring (default: ci)", + ) + parser.add_argument( + "--limit", + type=int, + default=100, + help="Number of runs to analyze (for ci mode, default: 100)", + ) + parser.add_argument( + "--days", + type=int, + default=2, + help="Number of days to analyze (for nightly mode, default: 2)", + ) + parser.add_argument( + "--output", + help="Output file for detailed stats (JSON)", + ) + parser.add_argument( + "--branch", + default=None, + help="Filter runs by branch (default: None - all branches). Specify branch name to filter.", + ) + + args = parser.parse_args() + + analyzer = SGLangCIAnalyzer(args.token) + + try: + if args.mode == "nightly": + # Nightly test monitoring mode + runs = analyzer.get_nightly_runs(days=args.days) + + if not runs: + print("No nightly test runs found in the specified time period.") + sys.exit(1) + + stats = analyzer.analyze_nightly_with_metrics(runs) + analyzer.generate_nightly_report(stats, args.output) + analyzer.generate_nightly_github_summary(stats) + regressions = analyzer.detect_nightly_regressions(stats) + + # Report regressions but don't stop the monitor + if regressions: + print("\n⚠️ Regressions detected - see report above") + else: + print("\n✓ No significant regressions detected") + sys.exit(0) + + else: + # Regular CI analysis mode + branch = args.branch if args.branch else None + runs = analyzer.get_recent_runs(args.limit, branch) + + if not runs: + print("No CI run data found") + return + + stats = analyzer.analyze_ci_failures(runs) + analyzer.generate_report(stats) + + output_file = args.output or "ci_analysis.json" + analyzer.save_detailed_report(stats, output_file) + analyzer.generate_github_summary(stats) + + except Exception as e: + print(f"Error during analysis: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci_monitor/ci_analyzer_balance.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci_monitor/ci_analyzer_balance.py new file mode 100755 index 00000000..e0779990 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci_monitor/ci_analyzer_balance.py @@ -0,0 +1,533 @@ +import argparse +import json +import os +import re +import sys +import time +from collections import defaultdict +from datetime import datetime +from typing import Dict, List, Optional, Tuple + +import requests + + +class SGLangTestBalanceAnalyzer: + + def __init__(self, token: str): + self.token = token + self.base_url = "https://api.github.com" + self.repo = "sgl-project/sglang" + self.headers = { + "Authorization": f"token {token}", + "Accept": "application/vnd.github.v3+json", + "User-Agent": "SGLang-Test-Balance-Analyzer/1.0", + } + self.session = requests.Session() + self.session.headers.update(self.headers) + + self.test_time_pattern = re.compile( + r"filename='([^']+)',\s*elapsed=(\d+),\s*estimated_time=(\d+)" + ) + + def get_recent_runs(self, limit: int = 1000) -> List[Dict]: + print(f"Fetching {limit} recent CI runs...") + + all_runs = [] + page = 1 + per_page = 100 + + while len(all_runs) < limit: + url = f"{self.base_url}/repos/{self.repo}/actions/runs" + params = {"per_page": min(per_page, limit - len(all_runs)), "page": page} + + try: + response = self.session.get(url, params=params) + response.raise_for_status() + data = response.json() + + if not data.get("workflow_runs"): + break + + all_runs.extend(data["workflow_runs"]) + print(f"Fetched {len(all_runs)} runs so far...") + + if len(data["workflow_runs"]) < per_page: + break + + page += 1 + time.sleep(0.1) + + except requests.exceptions.RequestException as e: + print(f"Error fetching CI data: {e}") + break + + return all_runs[:limit] + + def get_job_logs(self, run_id: int, job_name: str) -> Optional[str]: + try: + jobs_url = f"{self.base_url}/repos/{self.repo}/actions/runs/{run_id}/jobs" + response = self.session.get(jobs_url) + response.raise_for_status() + jobs_data = response.json() + + target_job = None + for job in jobs_data.get("jobs", []): + if job.get("name", "") == job_name: + target_job = job + break + + if not target_job: + return None + + logs_url = f"{self.base_url}/repos/{self.repo}/actions/jobs/{target_job['id']}/logs" + response = self.session.get(logs_url) + response.raise_for_status() + + return response.text + + except Exception as e: + if "404" not in str(e): + print(f"Failed to get job {job_name} logs: {e}") + return None + + def get_all_jobs_for_run(self, run_id: int) -> List[Dict]: + try: + jobs_url = f"{self.base_url}/repos/{self.repo}/actions/runs/{run_id}/jobs" + response = self.session.get(jobs_url) + response.raise_for_status() + jobs_data = response.json() + return jobs_data.get("jobs", []) + except Exception as e: + print(f"Failed to get jobs for run {run_id}: {e}") + return [] + + def get_job_logs_by_id(self, job_id: int) -> Optional[str]: + try: + logs_url = f"{self.base_url}/repos/{self.repo}/actions/jobs/{job_id}/logs" + response = self.session.get(logs_url) + response.raise_for_status() + return response.text + except Exception as e: + if "404" not in str(e): + print(f"Failed to get job {job_id} logs: {e}") + return None + + def parse_test_times(self, log_content: str) -> List[Dict]: + if not log_content: + return [] + + test_times = [] + matches = self.test_time_pattern.findall(log_content) + filtered_count = 0 + + for match in matches: + filename, elapsed_str, estimated_str = match + try: + elapsed = int(elapsed_str) + estimated = int(estimated_str) + gap = elapsed - estimated + + if self._is_abnormal_test_data( + elapsed, estimated, log_content, filename + ): + filtered_count += 1 + continue + + test_times.append( + { + "filename": filename, + "elapsed": elapsed, + "estimated": estimated, + "gap": gap, + } + ) + except ValueError: + continue + + return test_times + + def _is_abnormal_test_data( + self, elapsed: int, estimated: int, log_content: str, filename: str + ) -> bool: + + # To avoid collect retry data + if elapsed % estimated == 0: + return True + + return False + + def collect_test_balance_data(self, runs: List[Dict]) -> Dict[str, Dict]: + print("Starting test balance data collection...") + + test_gaps = defaultdict( + lambda: { + "max_gap": 0, + "max_elapsed": 0, + "max_estimated": 0, + "max_gap_run_info": {}, + "total_runs": 0, + "all_gaps": [], + } + ) + + total_tests_parsed = 0 + abnormal_tests_filtered = 0 + + target_job_prefixes = [ + "stage-a-test-1", + "unit-test-backend-1-gpu", + "unit-test-backend-2-gpu", + "unit-test-backend-4-gpu", + "unit-test-backend-8-gpu-h200", + "unit-test-backend-8-gpu-h20", + "unit-test-backend-4-gpu-b200", + "unit-test-backend-4-gpu-gb200", + "unit-test-deepep-4-gpu", + "unit-test-deepep-8-gpu", + "unit-test-backend-8-gpu-deepseek-v32", + "performance-test-1-gpu-part-1", + "performance-test-1-gpu-part-2", + "performance-test-1-gpu-part-3", + "performance-test-2-gpu", + "accuracy-test-1-gpu", + "accuracy-test-2-gpu", + ] + + total_runs = len(runs) + for i, run in enumerate(runs, 1): + if i % 10 == 0 or i == total_runs: + print(f"Processing run {i}/{total_runs}: #{run.get('run_number')}") + + workflow_name = run.get("name", "") + if "AMD" in workflow_name or "amd" in workflow_name.lower(): + continue + + run_info = { + "run_number": run.get("run_number"), + "created_at": run.get("created_at"), + "head_sha": run.get("head_sha", "")[:8], + "author": run.get("head_commit", {}) + .get("author", {}) + .get("name", "Unknown"), + "url": f"https://github.com/{self.repo}/actions/runs/{run.get('id')}", + } + + pull_requests = run.get("pull_requests", []) + if pull_requests: + run_info["pr_number"] = pull_requests[0].get("number") + + all_jobs = self.get_all_jobs_for_run(run.get("id")) + + for job in all_jobs: + job_name = job.get("name", "") + job_id = job.get("id") + + matches_prefix = False + for prefix in target_job_prefixes: + if job_name.startswith(prefix): + matches_prefix = True + break + + if not matches_prefix: + continue + + logs = self.get_job_logs_by_id(job_id) + if not logs: + continue + + test_times = self.parse_test_times(logs) + total_tests_parsed += len(test_times) + + for test_data in test_times: + filename = test_data["filename"] + elapsed = test_data["elapsed"] + estimated = test_data["estimated"] + gap = test_data["gap"] + + test_stats = test_gaps[filename] + test_stats["total_runs"] += 1 + test_stats["all_gaps"].append(gap) + + if gap > test_stats["max_gap"]: + test_stats["max_gap"] = gap + test_stats["max_elapsed"] = elapsed + test_stats["max_estimated"] = estimated + test_stats["max_gap_run_info"] = { + **run_info, + "job_name": job_name, + "job_url": f"https://github.com/{self.repo}/actions/runs/{run.get('id')}/job/{job_id}", + } + + time.sleep(0.1) + + return dict(test_gaps) + + def generate_balance_report( + self, test_data: Dict[str, Dict], output_file: str = "test_balance_report.json" + ): + print("\n" + "=" * 80) + print("SGLang Test Balance Analysis Report (PR Test GPU Jobs)") + print("=" * 80) + + sorted_tests = sorted( + test_data.items(), key=lambda x: x[1]["max_gap"], reverse=True + ) + + print(f"\nTotal tests analyzed: {len(sorted_tests)}") + print( + f"Tests with significant gaps (>100s): {len([t for t in sorted_tests if t[1]['max_gap'] > 100])}" + ) + print( + f"Tests with large gaps (>300s): {len([t for t in sorted_tests if t[1]['max_gap'] > 300])}" + ) + print( + f"Note: Abnormal test data (due to failures/retries) has been filtered out" + ) + + report_data = { + "summary": { + "total_tests": len(sorted_tests), + "tests_with_gaps_over_100s": len( + [t for t in sorted_tests if t[1]["max_gap"] > 100] + ), + "tests_with_gaps_over_300s": len( + [t for t in sorted_tests if t[1]["max_gap"] > 300] + ), + "analysis_timestamp": datetime.now().isoformat(), + }, + "test_balance_table": [], + } + + print(f"\nTop 50 PR Test GPU Jobs with Largest Time Gaps:") + print("-" * 100) + print( + f"{'Rank':<4} {'Test File':<40} {'Max Gap':<8} {'Max Elapsed':<12} {'Max Estimated':<15} {'Job Name':<25}" + ) + print("-" * 100) + + for i, (filename, stats) in enumerate(sorted_tests[:50], 1): + test_name = filename.split("/")[-1] if "/" in filename else filename + job_name = ( + stats["max_gap_run_info"].get("job_name", "Unknown") + if stats["max_gap_run_info"] + else "Unknown" + ) + + print( + f"{i:<4} {test_name:<40} {stats['max_gap']:<8} {stats['max_elapsed']:<12} {stats['max_estimated']:<15} {job_name:<25}" + ) + + report_data["test_balance_table"].append( + { + "rank": i, + "filename": filename, + "test_name": test_name, + "max_gap": stats["max_gap"], + "max_elapsed": stats["max_elapsed"], + "max_estimated": stats["max_estimated"], + "max_gap_run_info": stats["max_gap_run_info"], + "total_runs": stats["total_runs"], + } + ) + + with open(output_file, "w", encoding="utf-8") as f: + json.dump(report_data, f, ensure_ascii=False, indent=2) + print(f"\nDetailed report saved to: {output_file}") + + return report_data + + def generate_github_summary(self, report_data: Dict): + try: + github_step_summary = os.environ.get("GITHUB_STEP_SUMMARY") + if not github_step_summary: + print("Not running in GitHub Actions, skipping summary generation") + return + + print("Generating GitHub Actions summary for Test Balance Analysis...") + + summary_lines = [] + summary_lines.append( + "# SGLang Test Balance Analysis Report (PR Test GPU Jobs)" + ) + summary_lines.append("") + summary_lines.append( + f"**Analysis Timestamp:** {report_data['summary']['analysis_timestamp']}" + ) + summary_lines.append("") + + summary_lines.append("## Summary Statistics") + summary_lines.append("") + summary_lines.append("| Metric | Count |") + summary_lines.append("|--------|-------|") + summary_lines.append( + f"| Total Tests Analyzed | {report_data['summary']['total_tests']} |" + ) + summary_lines.append( + f"| Tests with Gaps > 100s | {report_data['summary']['tests_with_gaps_over_100s']} |" + ) + summary_lines.append( + f"| Tests with Gaps > 300s | {report_data['summary']['tests_with_gaps_over_300s']} |" + ) + summary_lines.append("") + + summary_lines.append("## Top 30 PR Test GPU Jobs with Largest Time Gaps") + summary_lines.append("") + summary_lines.append( + "| Rank | Test File | Max Gap (s) | Max Elapsed (s) | Max Estimated (s) | Job Name | Job Link | Total Runs |" + ) + summary_lines.append( + "|------|-----------|-------------|----------------|------------------|---------|----------|------------|" + ) + + for test in report_data["test_balance_table"][:30]: + test_name = test["test_name"] + if len(test_name) > 30: + test_name = test_name[:27] + "..." + + job_name = ( + test["max_gap_run_info"].get("job_name", "Unknown") + if test["max_gap_run_info"] + else "Unknown" + ) + job_url = ( + test["max_gap_run_info"].get("job_url", "") + if test["max_gap_run_info"] + else "" + ) + job_link = f"[{job_name}]({job_url})" if job_url else job_name + + summary_lines.append( + f"| {test['rank']} | `{test_name}` | {test['max_gap']} | {test['max_elapsed']} | {test['max_estimated']} | {job_name} | [{job_name}]({job_url}) | {test['total_runs']} |" + ) + + summary_lines.append("") + summary_lines.append("## Recommendations") + summary_lines.append("") + summary_lines.append( + "Based on the analysis above, consider adjusting estimated times for tests with large gaps:" + ) + summary_lines.append("") + + top_5_tests = report_data["test_balance_table"][:5] + for test in top_5_tests: + test_name = test["test_name"] + if len(test_name) > 40: + test_name = test_name[:37] + "..." + suggested_estimated = test["max_elapsed"] + 50 + summary_lines.append( + f"- **{test_name}**: Current max elapsed: {test['max_elapsed']}s, suggested estimated: {suggested_estimated}s" + ) + + summary_lines.append("") + summary_lines.append( + "Set estimated times to be slightly higher than the maximum observed elapsed time to avoid CI timeouts." + ) + + with open(github_step_summary, "w", encoding="utf-8") as f: + f.write("\n".join(summary_lines)) + + print("GitHub Actions summary generated successfully") + + except Exception as e: + print(f"Failed to generate GitHub Actions summary: {e}") + + def save_csv_report( + self, report_data: Dict, output_file: str = "test_balance_report.csv" + ): + import csv + + with open(output_file, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f) + + writer.writerow( + [ + "Rank", + "Test File", + "Test Name", + "Max Gap (s)", + "Max Elapsed (s)", + "Max Estimated (s)", + "Job Name", + "Max Gap Job URL", + "Total Runs", + ] + ) + + for test in report_data["test_balance_table"]: + max_job_url = ( + test["max_gap_run_info"].get("job_url", "") + if test["max_gap_run_info"] + else "" + ) + job_name = ( + test["max_gap_run_info"].get("job_name", "Unknown") + if test["max_gap_run_info"] + else "Unknown" + ) + + writer.writerow( + [ + test["rank"], + test["filename"], + test["test_name"], + test["max_gap"], + test["max_elapsed"], + test["max_estimated"], + job_name, + max_job_url, + test["total_runs"], + ] + ) + + print(f"CSV report saved to: {output_file}") + + +def main(): + parser = argparse.ArgumentParser(description="SGLang Test Balance Analyzer") + parser.add_argument("--token", required=True, help="GitHub Personal Access Token") + parser.add_argument( + "--limit", + type=int, + default=1000, + help="Number of runs to analyze (default: 1000)", + ) + parser.add_argument( + "--output", + default="test_balance_report.json", + help="Output file (default: test_balance_report.json)", + ) + + args = parser.parse_args() + + analyzer = SGLangTestBalanceAnalyzer(args.token) + + try: + runs = analyzer.get_recent_runs(args.limit) + + if not runs: + print("No CI run data found") + return + + test_data = analyzer.collect_test_balance_data(runs) + + if not test_data: + print("No test balance data found") + return + + report_data = analyzer.generate_balance_report(test_data, args.output) + + csv_output = args.output.replace(".json", ".csv") + analyzer.save_csv_report(report_data, csv_output) + + analyzer.generate_github_summary(report_data) + + except Exception as e: + print(f"Error during analysis: {e}") + import traceback + + traceback.print_exc() + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci_monitor/ci_analyzer_perf.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci_monitor/ci_analyzer_perf.py new file mode 100755 index 00000000..fa8822dd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci_monitor/ci_analyzer_perf.py @@ -0,0 +1,1375 @@ +#!/usr/bin/env python3 +""" +SGLang CI Performance Analyzer - Simplified Version +Collect performance data based on actual log format +""" + +import argparse +import base64 +import csv +import os +import re +import sys +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime +from typing import Dict, List, Optional + +import matplotlib.dates as mdates +import matplotlib.pyplot as plt +import pandas as pd +import requests +from matplotlib import rcParams + + +class SGLangPerfAnalyzer: + """SGLang CI Performance Analyzer""" + + def __init__(self, token: str): + self.token = token + self.base_url = "https://api.github.com" + self.repo = "sgl-project/sglang" + self.headers = { + "Authorization": f"token {token}", + "Accept": "application/vnd.github.v3+json", + "User-Agent": "SGLang-Perf-Analyzer/1.0", + } + self.session = requests.Session() + self.session.headers.update(self.headers) + + # Performance test job names + self.performance_jobs = [ + "performance-test-1-gpu-part-1", + "performance-test-1-gpu-part-2", + "performance-test-2-gpu", + ] + + # Strictly match tests and metrics shown in the images + self.target_tests_and_metrics = { + "performance-test-1-gpu-part-1": { + "test_bs1_default": ["output_throughput_token_s"], + "test_online_latency_default": ["median_e2e_latency_ms"], + "test_offline_throughput_default": ["output_throughput_token_s"], + "test_offline_throughput_non_stream_small_batch_size": [ + "output_throughput_token_s" + ], + "test_online_latency_eagle": ["median_e2e_latency_ms", "accept_length"], + "test_lora_online_latency": ["median_e2e_latency_ms", "median_ttft_ms"], + "test_lora_online_latency_with_concurrent_adapter_updates": [ + "median_e2e_latency_ms", + "median_ttft_ms", + ], + }, + "performance-test-1-gpu-part-2": { + "test_offline_throughput_without_radix_cache": [ + "output_throughput_token_s" + ], + "test_offline_throughput_with_triton_attention_backend": [ + "output_throughput_token_s" + ], + "test_offline_throughput_default_fp8": ["output_throughput_token_s"], + "test_vlm_offline_throughput": ["output_throughput_token_s"], + "test_vlm_online_latency": ["median_e2e_latency_ms"], + }, + "performance-test-2-gpu": { + "test_moe_tp2_bs1": ["output_throughput_token_s"], + "test_torch_compile_tp2_bs1": ["output_throughput_token_s"], + "test_moe_offline_throughput_default": ["output_throughput_token_s"], + "test_moe_offline_throughput_without_radix_cache": [ + "output_throughput_token_s" + ], + "test_pp_offline_throughput_default_decode": [ + "output_throughput_token_s" + ], + "test_pp_long_context_prefill": ["input_throughput_token_s"], + }, + } + + # Performance metric patterns - only keep metrics needed in images + self.perf_patterns = { + # Key metrics shown in images + "output_throughput_token_s": r"Output token throughput \(tok/s\):\s*([\d.]+)", + "Output_throughput_token_s": r"Output throughput:\s*([\d.]+)\s*token/s", + "median_e2e_latency_ms": r"Median E2E Latency \(ms\):\s*([\d.]+)", + "median_ttft_ms": r"Median TTFT \(ms\):\s*([\d.]+)", + "accept_length": r"Accept length:\s*([\d.]+)", + "input_throughput_token_s": r"Input token throughput \(tok/s\):\s*([\d.]+)", + } + + # Pre-compile regex patterns for better performance + self.compiled_patterns = { + name: re.compile(pattern, re.IGNORECASE) + for name, pattern in self.perf_patterns.items() + } + + # Pre-compile test pattern + self.test_pattern = re.compile( + r"python3 -m unittest (test_bench_\w+\.TestBench\w+\.test_\w+)" + ) + + # Setup matplotlib fonts and styles + self._setup_matplotlib() + + # GitHub data repository settings + self.data_repo = "sglang-bot/sglang-ci-data" + self.data_branch = "main" + + def _setup_matplotlib(self): + """Setup matplotlib fonts and styles""" + # Set fonts + rcParams["font.sans-serif"] = ["Arial", "DejaVu Sans", "Liberation Sans"] + rcParams["axes.unicode_minus"] = False # Fix minus sign display issue + + # Set chart styles + plt.style.use("default") + rcParams["figure.figsize"] = (12, 6) + rcParams["font.size"] = 10 + rcParams["axes.grid"] = True + rcParams["grid.alpha"] = 0.3 + + def get_recent_runs( + self, limit: int = 100, start_date: str = None, end_date: str = None + ) -> List[Dict]: + """Get recent CI run data with multiple collection strategies""" + + # If date range is specified, get all data in that range + if start_date or end_date: + return self._get_date_range_runs(start_date, end_date) + + print(f"Getting PR Test runs (limit: {limit})...") + + # Use sampling strategy if limit >= 500, otherwise use sequential + if limit >= 500: + print(f"Using uniform sampling for {limit} runs to cover ~30 days...") + return self._get_sampled_runs(limit) + else: + return self._get_sequential_runs(limit) + + def _get_sequential_runs(self, limit: int) -> List[Dict]: + """Original sequential method for smaller limits""" + print(f"Using sequential sampling for {limit} runs...") + + pr_test_runs = [] + page = 1 + per_page = 100 + + while len(pr_test_runs) < limit: + url = f"{self.base_url}/repos/{self.repo}/actions/runs" + params = {"per_page": per_page, "page": page} + + try: + response = self.session.get(url, params=params) + response.raise_for_status() + data = response.json() + + if not data.get("workflow_runs"): + break + + # Filter PR Test runs + current_pr_tests = [ + run for run in data["workflow_runs"] if run.get("name") == "PR Test" + ] + + # Add to result list, but not exceed limit + for run in current_pr_tests: + if len(pr_test_runs) < limit: + pr_test_runs.append(run) + else: + break + + print(f"Got {len(pr_test_runs)} PR test runs...") + + # Exit if no more data on this page or reached limit + if len(data["workflow_runs"]) < per_page or len(pr_test_runs) >= limit: + break + + page += 1 + time.sleep(0.1) # Avoid API rate limiting + + except requests.exceptions.RequestException as e: + print(f"Error getting CI data: {e}") + break + + return pr_test_runs + + def _get_sampled_runs(self, limit: int) -> List[Dict]: + """Uniform sampling method for 30-day coverage""" + from datetime import datetime, timedelta + + # Uniform sampling across 30 days + sampled_runs = self._sample_time_period(limit, days_back=30, uniform=True) + + print( + f"Sampled {len(sampled_runs)} runs from 30-day period (requested: {limit})" + ) + return sampled_runs + + def _sample_time_period( + self, + target_samples: int, + days_back: int, + skip_recent_days: int = 0, + uniform: bool = False, + ) -> List[Dict]: + """Sample runs from a specific time period""" + from datetime import datetime, timedelta + + # Calculate time range + end_time = datetime.utcnow() - timedelta(days=skip_recent_days) + start_time = end_time - timedelta(days=days_back - skip_recent_days) + + sampling_type = "uniform" if uniform else "systematic" + print( + f" {sampling_type.title()} sampling {target_samples} runs from {start_time.strftime('%Y-%m-%d')} to {end_time.strftime('%Y-%m-%d')}" + ) + + collected_runs = [] + page = 1 + per_page = 100 + total_in_period = 0 + + while True: + url = f"{self.base_url}/repos/{self.repo}/actions/runs" + params = {"per_page": per_page, "page": page} + + try: + response = self.session.get(url, params=params) + response.raise_for_status() + data = response.json() + + if not data.get("workflow_runs"): + break + + period_runs = [] + for run in data["workflow_runs"]: + if run.get("name") != "PR Test": + continue + + created_at = run.get("created_at", "") + if created_at: + try: + run_time = datetime.fromisoformat( + created_at.replace("Z", "+00:00") + ).replace(tzinfo=None) + if start_time <= run_time <= end_time: + period_runs.append(run) + total_in_period += 1 + except: + continue + + collected_runs.extend(period_runs) + + # Progress indicator every 5 pages + if page % 5 == 0: + print( + f" Page {page}: Found {total_in_period} runs in target period, collected {len(collected_runs)} total" + ) + + # Check if we've gone past our time window + if data["workflow_runs"]: + last_run_time_str = data["workflow_runs"][-1].get("created_at", "") + if last_run_time_str: + try: + last_run_time = datetime.fromisoformat( + last_run_time_str.replace("Z", "+00:00") + ).replace(tzinfo=None) + if last_run_time < start_time: + print(f" Reached time boundary at page {page}") + break + except: + pass + + if len(data["workflow_runs"]) < per_page: + break + + page += 1 + time.sleep(0.1) + + except requests.exceptions.RequestException as e: + print(f" Error getting data for time period: {e}") + break + + print( + f" Found {total_in_period} runs in time period, collected {len(collected_runs)} for sampling" + ) + + # Debug: Show time range of collected data + if collected_runs: + collected_runs_sorted = sorted( + collected_runs, key=lambda x: x.get("created_at", "") + ) + earliest = ( + collected_runs_sorted[0].get("created_at", "")[:10] + if collected_runs_sorted + else "N/A" + ) + latest = ( + collected_runs_sorted[-1].get("created_at", "")[:10] + if collected_runs_sorted + else "N/A" + ) + print(f" Collected data spans from {earliest} to {latest}") + + # Sample from collected runs + if len(collected_runs) <= target_samples: + return collected_runs + + if uniform: + # Uniform sampling: sort by time and select evenly distributed samples + collected_runs.sort(key=lambda x: x.get("created_at", "")) + step = len(collected_runs) / target_samples + sampled_runs = [] + + for i in range(target_samples): + index = int(i * step) + if index < len(collected_runs): + sampled_runs.append(collected_runs[index]) + else: + # Systematic sampling for even distribution + step = len(collected_runs) / target_samples + sampled_runs = [] + + for i in range(target_samples): + index = int(i * step) + if index < len(collected_runs): + sampled_runs.append(collected_runs[index]) + + print( + f" Sampled {len(sampled_runs)} runs from {len(collected_runs)} available" + ) + + # Debug: Show time range of sampled data + if sampled_runs: + sampled_runs_sorted = sorted( + sampled_runs, key=lambda x: x.get("created_at", "") + ) + earliest = ( + sampled_runs_sorted[0].get("created_at", "")[:10] + if sampled_runs_sorted + else "N/A" + ) + latest = ( + sampled_runs_sorted[-1].get("created_at", "")[:10] + if sampled_runs_sorted + else "N/A" + ) + print(f" Sampled data spans from {earliest} to {latest}") + + return sampled_runs + + def _get_date_range_runs( + self, start_date: str = None, end_date: str = None + ) -> List[Dict]: + """Get all CI runs within specified date range""" + from datetime import datetime, timedelta + + # Parse dates + if start_date: + try: + start_time = datetime.strptime(start_date, "%Y-%m-%d") + except ValueError: + raise ValueError( + f"Invalid start_date format. Use YYYY-MM-DD, got: {start_date}" + ) + else: + # Default to 30 days ago if no start date + start_time = datetime.utcnow() - timedelta(days=30) + + if end_date: + try: + end_time = datetime.strptime(end_date, "%Y-%m-%d") + timedelta( + days=1 + ) # Include the end date + except ValueError: + raise ValueError( + f"Invalid end_date format. Use YYYY-MM-DD, got: {end_date}" + ) + else: + # Default to now if no end date + end_time = datetime.utcnow() + + # Validate date range + if start_time >= end_time: + raise ValueError( + f"start_date ({start_date}) must be before end_date ({end_date})" + ) + + print( + f"Getting ALL CI runs from {start_time.strftime('%Y-%m-%d')} to {end_time.strftime('%Y-%m-%d')}" + ) + + collected_runs = [] + page = 1 + per_page = 100 + total_in_period = 0 + + while True: + url = f"{self.base_url}/repos/{self.repo}/actions/runs" + params = {"per_page": per_page, "page": page} + + try: + response = self.session.get(url, params=params) + response.raise_for_status() + data = response.json() + + if not data.get("workflow_runs"): + break + + # Filter runs in date range and PR Test runs + period_runs = [] + for run in data["workflow_runs"]: + if run.get("name") != "PR Test": + continue + + created_at = run.get("created_at", "") + if created_at: + try: + run_time = datetime.fromisoformat( + created_at.replace("Z", "+00:00") + ).replace(tzinfo=None) + if start_time <= run_time <= end_time: + period_runs.append(run) + total_in_period += 1 + except: + continue + + collected_runs.extend(period_runs) + + # Progress indicator every 5 pages + if page % 5 == 0: + print( + f" Page {page}: Found {total_in_period} runs in date range, collected {len(collected_runs)} total" + ) + + # Check if we've gone past our time window + if data["workflow_runs"]: + last_run_time_str = data["workflow_runs"][-1].get("created_at", "") + if last_run_time_str: + try: + last_run_time = datetime.fromisoformat( + last_run_time_str.replace("Z", "+00:00") + ).replace(tzinfo=None) + if last_run_time < start_time: + print(f" Reached time boundary at page {page}") + break + except: + pass + + if len(data["workflow_runs"]) < per_page: + break + + page += 1 + time.sleep(0.1) + + except requests.exceptions.RequestException as e: + print(f" Error getting data for date range: {e}") + break + + print( + f"Found {total_in_period} runs in date range {start_time.strftime('%Y-%m-%d')} to {end_time.strftime('%Y-%m-%d')}" + ) + + # Sort by creation time (newest first) + collected_runs.sort(key=lambda x: x.get("created_at", ""), reverse=True) + + return collected_runs + + def get_job_logs(self, run_id: int, job_name: str) -> Optional[str]: + """Get logs for specific job with early exit optimization""" + try: + # First get job list with pagination to ensure we get all jobs + jobs_url = f"{self.base_url}/repos/{self.repo}/actions/runs/{run_id}/jobs" + response = self.session.get(jobs_url, params={"per_page": 100}) + response.raise_for_status() + jobs_data = response.json() + + # Find matching job with early exit + target_job = None + for job in jobs_data.get("jobs", []): + if job_name in job.get("name", ""): + # Early exit if job failed or was skipped + if job.get("conclusion") not in ["success", "neutral"]: + return None + target_job = job + break + + if not target_job: + return None + + # Get logs + logs_url = f"{self.base_url}/repos/{self.repo}/actions/jobs/{target_job['id']}/logs" + response = self.session.get(logs_url) + response.raise_for_status() + + return response.text + + except Exception as e: + # Reduce verbose error logging for common failures + if "404" not in str(e): + print(f"Failed to get job {job_name} logs: {e}") + return None + + def get_all_job_logs_parallel(self, run_id: int) -> Dict[str, Optional[str]]: + """Get logs for all performance jobs in parallel""" + + def fetch_job_logs(job_name: str) -> tuple[str, Optional[str]]: + """Fetch logs for a single job""" + logs = self.get_job_logs(run_id, job_name) + return job_name, logs + + results = {} + with ThreadPoolExecutor( + max_workers=8 + ) as executor: # Increased concurrent requests + # Submit all job log requests + future_to_job = { + executor.submit(fetch_job_logs, job_name): job_name + for job_name in self.performance_jobs + } + + # Collect results as they complete + for future in as_completed(future_to_job): + job_name, logs = future.result() + results[job_name] = logs + + return results + + def parse_performance_data( + self, log_content: str, job_name: str + ) -> Dict[str, Dict[str, str]]: + """Parse specified performance data from logs""" + if not log_content: + return {} + + test_data = {} + + # Get target tests for current job + target_tests = self.target_tests_and_metrics.get(job_name, {}) + if not target_tests: + return test_data + + # Find all unittest tests using pre-compiled pattern + test_matches = self.test_pattern.findall(log_content) + + for test_match in test_matches: + test_name = test_match.split(".")[-1] # Extract test name + + # Only process target tests + if test_name not in target_tests: + continue + + # Find performance data after this test + test_section = self._extract_test_section(log_content, test_match) + if test_section: + # Only find metrics needed for this test + target_metrics = target_tests[test_name] + perf_data = {} + + for metric_name in target_metrics: + if metric_name in self.compiled_patterns: + compiled_pattern = self.compiled_patterns[metric_name] + matches = compiled_pattern.findall(test_section) + if matches: + perf_data[metric_name] = matches[-1] # Take the last match + + if perf_data: + test_data[test_name] = perf_data + + return test_data + + def _extract_test_section(self, log_content: str, test_pattern: str) -> str: + """Extract log section for specific test""" + lines = log_content.split("\n") + test_start = -1 + test_end = len(lines) + + # Find test start position + for i, line in enumerate(lines): + if test_pattern in line: + test_start = i + break + + if test_start == -1: + return "" + + # Find test end position (next test start or major separator) + for i in range(test_start + 1, len(lines)): + line = lines[i] + if ( + "python3 -m unittest" in line and "test_" in line + ) or "##[group]" in line: + test_end = i + break + + return "\n".join(lines[test_start:test_end]) + + def collect_performance_data(self, runs: List[Dict]) -> Dict[str, List[Dict]]: + """Collect all performance data""" + print("Starting performance data collection...") + + # Create data list for each test + all_test_data = {} + + total_runs = len(runs) + for i, run in enumerate(runs, 1): + if not isinstance(run, dict): + print(f" Warning: run #{i} is not a dict, skipping.") + continue + + run_info = { + "run_number": run.get("run_number"), + "created_at": run.get("created_at"), + "head_sha": (run.get("head_sha") or "")[:8], + "author": "Unknown", + "pr_number": None, + "url": f"https://github.com/{self.repo}/actions/runs/{run.get('id')}", + } + head_commit = run.get("head_commit", {}) + if isinstance(head_commit, dict): + run_info["author"] = head_commit.get("author", {}).get( + "name", "Unknown" + ) + + # Extract PR number + pull_requests = run.get("pull_requests", []) + if pull_requests: + run_info["pr_number"] = pull_requests[0].get("number") + + # Get all job logs in parallel + all_job_logs = self.get_all_job_logs_parallel(run.get("id")) + + # Process each performance test job + for job_name, logs in all_job_logs.items(): + if not logs: + continue + + # Parse performance data + test_results = self.parse_performance_data(logs, job_name) + + for test_name, perf_data in test_results.items(): + # Create full test name including job info + full_test_name = f"{job_name}_{test_name}" + + if full_test_name not in all_test_data: + all_test_data[full_test_name] = [] + + test_entry = {**run_info, **perf_data} + all_test_data[full_test_name].append(test_entry) + print( + f" Found {test_name} performance data: {list(perf_data.keys())}" + ) + + time.sleep(0.2) + return all_test_data + + def generate_performance_tables( + self, test_data: Dict[str, List[Dict]], output_dir: str = "performance_tables" + ): + """Generate performance data tables""" + print(f"Generating performance tables to directory: {output_dir}") + + # Create output directory structure + os.makedirs(output_dir, exist_ok=True) + + # Create subdirectory for each job + job_dirs = {} + for job_name in self.performance_jobs: + job_dir = os.path.join(output_dir, f"{job_name}_summary") + os.makedirs(job_dir, exist_ok=True) + job_dirs[job_name] = job_dir + + # Generate table for each test + for full_test_name, data_list in test_data.items(): + if not data_list: + continue + + # Determine which job this test belongs to + job_name = None + test_name = full_test_name + for job in self.performance_jobs: + if full_test_name.startswith(job): + job_name = job + test_name = full_test_name[len(job) + 1 :] # Remove job prefix + break + + if not job_name: + continue + + job_dir = job_dirs[job_name] + table_file = os.path.join(job_dir, f"{test_name}.csv") + + # Generate CSV table + self._write_csv_table(table_file, test_name, data_list) + + # Generate corresponding chart + print(f" Generating chart for {test_name}...") + self._generate_chart(table_file, test_name, data_list, job_dir) + + print("Performance tables and charts generation completed!") + + def _write_csv_table(self, file_path: str, test_name: str, data_list: List[Dict]): + """Write CSV table""" + if not data_list: + return + + # Get all possible columns + all_columns = set() + for entry in data_list: + all_columns.update(entry.keys()) + + # Define column order + base_columns = ["created_at", "run_number", "pr_number", "author", "head_sha"] + perf_columns = [col for col in all_columns if col not in base_columns + ["url"]] + columns = base_columns + sorted(perf_columns) + ["url"] + + with open(file_path, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f) + + # Write header + writer.writerow(columns) + + # Write data rows + for entry in sorted( + data_list, key=lambda x: x.get("created_at", ""), reverse=True + ): + row = [] + for col in columns: + value = entry.get(col, "") + if col == "created_at" and value: + # Format time to consistent format + try: + # Handle ISO 8601 format: "2025-09-26T11:16:40Z" + if "T" in value and "Z" in value: + dt = datetime.fromisoformat( + value.replace("Z", "+00:00") + ) + value = dt.strftime("%Y-%m-%d %H:%M") + # If already in desired format, keep it + elif len(value) == 16 and " " in value: + # Validate format + datetime.strptime(value, "%Y-%m-%d %H:%M") + else: + # Try to parse and reformat + dt = datetime.fromisoformat(value) + value = dt.strftime("%Y-%m-%d %H:%M") + except: + # If all parsing fails, keep original value + pass + elif col == "pr_number" and value: + value = f"#{value}" + row.append(str(value)) + writer.writerow(row) + + print(f" Generated table: {file_path} ({len(data_list)} records)") + + def _generate_chart( + self, csv_file_path: str, test_name: str, data_list: List[Dict], output_dir: str + ): + """Generate corresponding time series charts for tables""" + print( + f" Starting chart generation for {test_name} with {len(data_list)} data points" + ) + + if not data_list or len(data_list) < 2: + print( + f" Skipping chart for {test_name}: insufficient data ({len(data_list) if data_list else 0} records)" + ) + return + + try: + # Prepare data + timestamps = [] + metrics_data = {} + + # Get performance metric columns (exclude basic info columns) + base_columns = { + "created_at", + "run_number", + "pr_number", + "author", + "head_sha", + "url", + } + perf_metrics = [] + + for entry in data_list: + for key in entry.keys(): + if key not in base_columns and key not in perf_metrics: + perf_metrics.append(key) + + if not perf_metrics: + print( + f" Skipping chart for {test_name}: no performance metrics found" + ) + return + + print(f" Found performance metrics: {perf_metrics}") + + # Parse data + for entry in data_list: + # Parse time + try: + time_str = entry.get("created_at", "") + if time_str: + # Handle different time formats + timestamp = None + + # Try ISO 8601 format first (from GitHub API): "2025-09-26T11:16:40Z" + if "T" in time_str and "Z" in time_str: + try: + # Parse and convert to naive datetime (remove timezone info) + dt_with_tz = datetime.fromisoformat( + time_str.replace("Z", "+00:00") + ) + timestamp = dt_with_tz.replace(tzinfo=None) + except: + # Fallback for older Python versions + timestamp = datetime.strptime( + time_str, "%Y-%m-%dT%H:%M:%SZ" + ) + + # Try CSV format: "2025-09-26 08:43" + elif " " in time_str and len(time_str) == 16: + timestamp = datetime.strptime(time_str, "%Y-%m-%d %H:%M") + + # Try other common formats + else: + formats_to_try = [ + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%d", + ] + for fmt in formats_to_try: + try: + timestamp = datetime.strptime(time_str, fmt) + break + except: + continue + + if timestamp: + timestamps.append(timestamp) + + # Collect metric data + for metric in perf_metrics: + if metric not in metrics_data: + metrics_data[metric] = [] + + value = entry.get(metric, "") + try: + numeric_value = float(value) + metrics_data[metric].append(numeric_value) + except: + metrics_data[metric].append(None) + else: + print( + f" Failed to parse timestamp format: '{time_str}'" + ) + + except Exception as e: + print(f" Error processing entry: {e}") + continue + + if not timestamps: + print( + f" Skipping chart for {test_name}: no valid timestamps found" + ) + return + + print(f" Parsed {len(timestamps)} timestamps") + + # Sort by time + sorted_data = sorted( + zip(timestamps, *[metrics_data[m] for m in perf_metrics]) + ) + timestamps = [item[0] for item in sorted_data] + for i, metric in enumerate(perf_metrics): + metrics_data[metric] = [item[i + 1] for item in sorted_data] + + # Create chart for each metric + for metric in perf_metrics: + values = metrics_data[metric] + valid_data = [ + (t, v) for t, v in zip(timestamps, values) if v is not None + ] + + if len(valid_data) < 2: + print( + f" Skipping chart for {test_name}_{metric}: insufficient valid data ({len(valid_data)} points)" + ) + continue + + valid_timestamps, valid_values = zip(*valid_data) + + # Create chart + plt.figure(figsize=(12, 6)) + plt.plot( + valid_timestamps, + valid_values, + marker="o", + linewidth=2, + markersize=4, + ) + + # Set title and labels + title = f"{test_name} - {self._format_metric_name(metric)}" + plt.title(title, fontsize=14, fontweight="bold") + plt.xlabel("Time", fontsize=12) + plt.ylabel(self._get_metric_unit(metric), fontsize=12) + + # Format x-axis + plt.gca().xaxis.set_major_formatter(mdates.DateFormatter("%m-%d %H:%M")) + plt.gca().xaxis.set_major_locator( + mdates.HourLocator(interval=max(1, len(valid_timestamps) // 10)) + ) + plt.xticks(rotation=45) + + # Add grid + plt.grid(True, alpha=0.3) + + # Adjust layout + plt.tight_layout() + + # Save chart + chart_filename = f"{test_name}_{metric}.png" + chart_path = os.path.join(output_dir, chart_filename) + plt.savefig(chart_path, dpi=300, bbox_inches="tight") + plt.close() + + print(f" Generated chart: {chart_path}") + + except Exception as e: + print(f" Failed to generate chart for {test_name}: {e}") + import traceback + + traceback.print_exc() + + def _format_metric_name(self, metric: str) -> str: + """Format metric name for display""" + name_mapping = { + "output_throughput_token_s": "Output Throughput", + "median_e2e_latency_ms": "Median E2E Latency", + "median_ttft_ms": "Median TTFT", + "accept_length": "Accept Length", + "input_throughput_token_s": "Input Throughput", + } + return name_mapping.get(metric, metric) + + def _get_metric_unit(self, metric: str) -> str: + """Get metric unit""" + if "throughput" in metric and "token_s" in metric: + return "token/s" + elif "latency" in metric and "ms" in metric: + return "ms" + elif "accept_length" in metric: + return "length" + else: + return "value" + + def generate_summary_report(self, test_data: Dict[str, List[Dict]]): + """Generate summary report""" + print("\n" + "=" * 60) + print("SGLang CI Performance Data Collection Report") + print("=" * 60) + + total_tests = len([test for test, data in test_data.items() if data]) + total_records = sum(len(data) for data in test_data.values()) + + print(f"\nOverall Statistics:") + print(f" Number of tests collected: {total_tests}") + print(f" Total records: {total_records}") + + print(f"\nStatistics by job:") + for job_name in self.performance_jobs: + job_tests = [test for test in test_data.keys() if test.startswith(job_name)] + job_records = sum(len(test_data[test]) for test in job_tests) + print(f" {job_name}: {len(job_tests)} tests, {job_records} records") + + for test in job_tests: + data = test_data[test] + test_short_name = test[len(job_name) + 1 :] + print(f" - {test_short_name}: {len(data)} records") + + print("\n" + "=" * 60) + + def upload_file_to_github( + self, file_path: str, github_path: str, commit_message: str + ) -> bool: + """Upload a file to GitHub repository with retry logic""" + max_retries = 30 + retry_count = 0 + + while retry_count < max_retries: + try: + # Read file content + with open(file_path, "rb") as f: + content = f.read() + + # Encode content to base64 + content_encoded = base64.b64encode(content).decode("utf-8") + + # Check if file exists to get SHA + check_url = ( + f"{self.base_url}/repos/{self.data_repo}/contents/{github_path}" + ) + check_response = self.session.get(check_url) + + sha = None + if check_response.status_code == 200: + sha = check_response.json().get("sha") + + # Prepare upload data + upload_data = { + "message": commit_message, + "content": content_encoded, + "branch": self.data_branch, + } + + if sha: + upload_data["sha"] = sha + + # Upload file + response = self.session.put(check_url, json=upload_data) + + if response.status_code in [200, 201]: + print(f" ✅ Uploaded: {github_path}") + return True + elif response.status_code == 403: + retry_count += 1 + wait_time = min(2**retry_count, 30) + print( + f" ⚠️ Upload forbidden (403) for {github_path}, retrying in {wait_time}s... (attempt {retry_count}/{max_retries})" + ) + if retry_count >= max_retries: + print( + f" ❌ Failed to upload {github_path} after {max_retries} attempts (403 Forbidden)" + ) + return False + time.sleep(wait_time) + else: + response.raise_for_status() + + except requests.exceptions.RequestException as e: + retry_count += 1 + wait_time = min(2**retry_count, 30) + print( + f" ⚠️ Upload error for {github_path} (attempt {retry_count}/{max_retries}): {e}" + ) + if retry_count >= max_retries: + print( + f" ❌ Failed to upload {github_path} after {max_retries} attempts: {e}" + ) + return False + print(f" Retrying in {wait_time}s...") + time.sleep(wait_time) + except Exception as e: + print(f" ❌ Failed to upload {github_path}: {e}") + return False + + return False + + def upload_performance_data_to_github(self, output_dir: str): + """Upload performance_tables to GitHub with original structure""" + print("📤 Uploading performance data to GitHub...") + + # Check if target repository exists with retry logic + repo_url = f"{self.base_url}/repos/{self.data_repo}" + max_retries = 30 + retry_count = 0 + + print(f"🔍 Checking repository access to {self.data_repo}...") + + while retry_count < max_retries: + try: + repo_response = self.session.get(repo_url) + + if repo_response.status_code == 200: + print(f"✅ Repository {self.data_repo} is accessible") + break + elif repo_response.status_code == 404: + print( + f"❌ Repository {self.data_repo} does not exist or is not accessible" + ) + print(" Please ensure:") + print(" 1. The repository exists") + print(" 2. Your GitHub token has access to this repository") + print(" 3. Your token has 'contents:write' permission") + return + elif repo_response.status_code == 403: + retry_count += 1 + wait_time = min(2**retry_count, 60) # Exponential backoff, max 60s + print( + f"⚠️ Repository access forbidden (403), retrying in {wait_time}s... (attempt {retry_count}/{max_retries})" + ) + if retry_count >= max_retries: + print( + f"❌ Failed to access repository after {max_retries} attempts" + ) + print(" This might be due to:") + print(" 1. GitHub API rate limiting") + print(" 2. Token permissions issue") + print(" 3. Repository access restrictions") + return + time.sleep(wait_time) + else: + retry_count += 1 + wait_time = min(2**retry_count, 60) + print( + f"⚠️ Repository access failed with status {repo_response.status_code}, retrying in {wait_time}s... (attempt {retry_count}/{max_retries})" + ) + if retry_count >= max_retries: + print( + f"❌ Failed to access repository {self.data_repo} after {max_retries} attempts" + ) + return + time.sleep(wait_time) + + except Exception as e: + retry_count += 1 + wait_time = min(2**retry_count, 60) + print( + f"⚠️ Error checking repository (attempt {retry_count}/{max_retries}): {e}" + ) + if retry_count >= max_retries: + print( + f"❌ Failed to check repository after {max_retries} attempts: {e}" + ) + return + print(f" Retrying in {wait_time}s...") + time.sleep(wait_time) + + # Generate timestamp for this upload + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + + uploaded_count = 0 + + # Upload all files maintaining original structure + for root, dirs, files in os.walk(output_dir): + for file in files: + local_path = os.path.join(root, file) + + # Keep original directory structure + rel_path = os.path.relpath(local_path, output_dir) + github_path = f"performance_data/{timestamp}/{rel_path}".replace( + "\\", "/" + ) + + # Upload file + commit_msg = f"Add performance data: {rel_path} ({timestamp})" + if self.upload_file_to_github(local_path, github_path, commit_msg): + uploaded_count += 1 + + print(f"📤 Uploaded {uploaded_count} files to GitHub") + + # Print access info + base_url = f"https://github.com/{self.data_repo}/tree/{self.data_branch}/performance_data/{timestamp}" + print(f"🔗 View uploaded data at: {base_url}") + + # Generate GitHub Actions summary + self._generate_github_summary(output_dir, timestamp) + + def _generate_github_summary(self, output_dir: str, timestamp: str): + """Generate GitHub Actions summary with performance data""" + try: + # Check if running in GitHub Actions + github_step_summary = os.environ.get("GITHUB_STEP_SUMMARY") + if not github_step_summary: + print("ℹ️ Not running in GitHub Actions, skipping summary generation") + return + + print("📊 Generating GitHub Actions summary...") + + # Collect all CSV and PNG files + csv_files = [] + png_files = [] + + for root, dirs, files in os.walk(output_dir): + for file in files: + file_path = os.path.join(root, file) + rel_path = os.path.relpath(file_path, output_dir) + + if file.endswith(".csv"): + csv_files.append((file_path, rel_path)) + elif file.endswith(".png"): + png_files.append((file_path, rel_path)) + + # Sort files by job and test name + csv_files.sort(key=lambda x: x[1]) + png_files.sort(key=lambda x: x[1]) + + # Generate markdown summary + summary_lines = [] + summary_lines.append("# 📊 SGLang Performance Analysis Report") + summary_lines.append("") + summary_lines.append(f"**Analysis Timestamp:** {timestamp}") + summary_lines.append(f"**Total CSV Files:** {len(csv_files)}") + summary_lines.append(f"**Total Chart Files:** {len(png_files)}") + summary_lines.append("") + + # GitHub data repository link + base_url = f"https://github.com/{self.data_repo}/tree/{self.data_branch}/performance_data/{timestamp}" + summary_lines.append(f"🔗 **[View All Data on GitHub]({base_url})**") + summary_lines.append("") + + # Group by job + job_groups = {} + for csv_path, rel_path in csv_files: + # Extract job name from path: job_summary/test_name.csv + parts = rel_path.split("/") + if len(parts) >= 2: + job_name = parts[0].replace("_summary", "") + test_name = parts[1].replace(".csv", "") + + if job_name not in job_groups: + job_groups[job_name] = [] + job_groups[job_name].append((csv_path, test_name, rel_path)) + + # Generate summary for each job + for job_name in sorted(job_groups.keys()): + summary_lines.append(f"## 🚀 {job_name}") + summary_lines.append("") + + tests = job_groups[job_name] + tests.sort(key=lambda x: x[1]) # Sort by test name + + for csv_path, test_name, rel_path in tests: + summary_lines.append(f"### 📈 {test_name}") + + # Add CSV data preview + try: + with open(csv_path, "r", encoding="utf-8") as f: + lines = f.readlines() + if len(lines) > 1: # Has header and data + summary_lines.append("") + summary_lines.append("**Recent Performance Data:**") + summary_lines.append("") + + # Show header + header = lines[0].strip() + summary_lines.append( + f"| {' | '.join(header.split(','))} |" + ) + summary_lines.append( + f"| {' | '.join(['---'] * len(header.split(',')))} |" + ) + + # Show most recent 5 records (CSV is already sorted newest first) + data_lines = lines[1:] + for line in data_lines[ + :5 + ]: # Take first 5 lines (most recent) + if line.strip(): + summary_lines.append( + f"| {' | '.join(line.strip().split(','))} |" + ) + + summary_lines.append("") + except Exception as e: + summary_lines.append(f"*Error reading CSV data: {e}*") + summary_lines.append("") + + # Add chart image if exists + test_prefix = rel_path.replace(".csv", "") + matching_charts = [ + (png_path, png_rel) + for png_path, png_rel in png_files + if png_rel.startswith(test_prefix) + ] + + for png_path, chart_rel_path in matching_charts: + chart_url = f"https://github.com/{self.data_repo}/raw/{self.data_branch}/performance_data/{timestamp}/{chart_rel_path}" + # Extract metric name from filename: test_name_metric_name.png + filename = os.path.basename(chart_rel_path) + metric_name = filename.replace(f"{test_name}_", "").replace( + ".png", "" + ) + summary_lines.append( + f"**{self._format_metric_name(metric_name)} Trend:**" + ) + summary_lines.append("") + summary_lines.append( + f"![{test_name}_{metric_name}]({chart_url})" + ) + summary_lines.append("") + + summary_lines.append("---") + summary_lines.append("") + + # Write summary to GitHub Actions (append mode to preserve CI Analysis report) + with open(github_step_summary, "a", encoding="utf-8") as f: + f.write("\n".join(summary_lines)) + + print("✅ GitHub Actions summary generated successfully") + + except Exception as e: + print(f"❌ Failed to generate GitHub Actions summary: {e}") + import traceback + + traceback.print_exc() + + +def main(): + parser = argparse.ArgumentParser(description="SGLang CI Performance Analyzer") + parser.add_argument("--token", required=True, help="GitHub Personal Access Token") + parser.add_argument( + "--limit", + type=int, + default=100, + help="Number of runs to analyze (default: 100)", + ) + parser.add_argument( + "--output-dir", + default="performance_tables", + help="Output directory (default: performance_tables)", + ) + parser.add_argument( + "--upload-to-github", + action="store_true", + help="Upload results to sglang-bot/sglang-ci-data repository", + ) + parser.add_argument( + "--start-date", + type=str, + help="Start date for date range query (YYYY-MM-DD format). When specified with --end-date, gets ALL runs in range.", + ) + parser.add_argument( + "--end-date", + type=str, + help="End date for date range query (YYYY-MM-DD format). When specified with --start-date, gets ALL runs in range.", + ) + + args = parser.parse_args() + + # Create analyzer + analyzer = SGLangPerfAnalyzer(args.token) + + try: + # Get CI run data + runs = analyzer.get_recent_runs(args.limit, args.start_date, args.end_date) + + if not runs: + print("No CI run data found") + return + + # Collect performance data + test_data = analyzer.collect_performance_data(runs) + + # Generate performance tables + analyzer.generate_performance_tables(test_data, args.output_dir) + + # Upload to GitHub if requested + if args.upload_to_github: + analyzer.upload_performance_data_to_github(args.output_dir) + + # Generate summary report + analyzer.generate_summary_report(test_data) + + except Exception as e: + print(f"Error during analysis: {e}") + import traceback + + traceback.print_exc() + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci_monitor/ci_failures_analysis.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci_monitor/ci_failures_analysis.py new file mode 100644 index 00000000..d4cc7d41 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ci_monitor/ci_failures_analysis.py @@ -0,0 +1,2139 @@ +""" +SGLang CI Consecutive Failures Analyzer + +Monitors GitHub Actions workflows for consecutive test failures and runner issues. +Detects failure streaks, tracks job health, identifies problematic runners, and generates alerts. + +Features: +- Analyzes all jobs in PR Test workflow (excluding administrative jobs) +- Tracks consecutive failure streaks for each job +- Monitors runner health and failure rates +- Identifies whether failures are code-related or infrastructure-related +- Generates detailed reports with actionable recommendations + +Usage: + python ci_failures_analysis.py --token --limit 500 --threshold 3 +""" + +import argparse +import json +import os +import sys +import time +from collections import defaultdict +from datetime import datetime +from typing import Dict, List, Optional, Tuple + +import requests + + +class SGLangFailuresAnalyzer: + """Analyzes consecutive failures in GitHub Actions workflows.""" + + def __init__(self, token: str, alert_threshold: int = 3): + self.token = token + self.alert_threshold = alert_threshold + self.base_url = "https://api.github.com" + self.repo = "sgl-project/sglang" + self.headers = { + "Authorization": f"token {token}", + "Accept": "application/vnd.github.v3+json", + "User-Agent": "SGLang-Failures-Analyzer/1.0", + } + self.session = requests.Session() + self.session.headers.update(self.headers) + + # Target workflows to monitor + self.target_workflows = [ + "PR Test", # Nvidia GPU tests + "PR Test (AMD)", # AMD GPU tests + "PR Test (Xeon)", # Intel Xeon CPU tests + ] + + # Jobs to EXCLUDE from analysis (administrative/setup jobs, not actual tests) + self.excluded_jobs = [ + "check-changes", + "pr-test-finish", # Nvidia workflow teardown + "pr-test-amd-finish", # AMD workflow teardown + "call-gate", + "pr-gate", + ] + + def get_recent_runs(self, limit: int = 500) -> List[Dict]: + """ + Fetch recent workflow runs from GitHub API. + Keeps fetching until we have 'limit' runs from target workflows. + """ + print( + f"Fetching until we have {limit} runs from target workflows (PR Test, PR Test AMD, PR Test Xeon)..." + ) + + filtered_runs = [] + page = 1 + per_page = 100 + max_pages = 100 # Safety limit to prevent infinite loops (10,000 total runs) + + while len(filtered_runs) < limit and page <= max_pages: + url = f"{self.base_url}/repos/{self.repo}/actions/runs" + params = {"per_page": per_page, "page": page} + + try: + response = self.session.get(url, params=params, timeout=30) + response.raise_for_status() + data = response.json() + + if not data.get("workflow_runs"): + print("No more workflow runs available") + break + + # Filter this batch to target workflows + batch_filtered = [ + run + for run in data["workflow_runs"] + if run.get("name") in self.target_workflows + and run.get("status") == "completed" + ] + + filtered_runs.extend(batch_filtered) + print( + f"Fetched {len(filtered_runs)} target workflow runs so far (scanned page {page})..." + ) + + # If GitHub returned fewer than per_page, we've reached the end + if len(data["workflow_runs"]) < per_page: + print("Reached end of available workflow runs") + break + + page += 1 + time.sleep(0.1) + + except requests.exceptions.RequestException as e: + print(f"Error fetching workflow runs: {e}") + break + + if page > max_pages: + print( + f"Warning: Reached max pages limit ({max_pages}). Consider reducing --limit or increasing max_pages." + ) + + print(f"Collected {len(filtered_runs)} completed target workflow runs") + return filtered_runs[:limit] + + def get_jobs_for_run(self, run_id: int) -> List[Dict]: + """Get all jobs for a specific workflow run.""" + try: + url = f"{self.base_url}/repos/{self.repo}/actions/runs/{run_id}/jobs" + response = self.session.get(url, timeout=30) + response.raise_for_status() + data = response.json() + jobs = data.get("jobs", []) + return jobs + except requests.exceptions.RequestException as e: + print(f"Error fetching jobs for run {run_id}: {e}") + return [] + + def analyze_runner_health( + self, runs: List[Dict] + ) -> Tuple[Dict[str, Dict], Dict[str, Dict], Dict[str, Dict], Dict[str, Dict]]: + """ + Analyze runner health by tracking failures per runner and consecutive failure streaks. + + Returns: + Tuple of (runner_stats, runner_instance_data, runner_streak_data, runner_instance_streak_data) + - runner_stats: Overall stats per runner (failure rate, total jobs, etc.) + - runner_instance_data: Per-instance breakdown of failures + - runner_streak_data: Consecutive failure streaks per runner label + - runner_instance_streak_data: Consecutive failure streaks per runner instance + """ + print("\nAnalyzing runner health and consecutive failures...") + + # Sort runs by created_at (oldest first) + sorted_runs = sorted(runs, key=lambda x: x.get("created_at", "")) + + # Track runner statistics (overall) + runner_total_jobs: Dict[str, int] = defaultdict(int) + runner_failed_jobs: Dict[str, int] = defaultdict(int) + runner_job_failures: Dict[str, Dict[str, int]] = defaultdict( + lambda: defaultdict(int) + ) + runner_job_totals: Dict[str, Dict[str, int]] = defaultdict( + lambda: defaultdict(int) + ) + + # Track queue times per runner instance (can aggregate for runner labels if needed) + runner_instance_queue_times: Dict[str, List[float]] = defaultdict(list) + + # Track individual runner instances (runner_name + runner_id) + runner_instance_stats: Dict[str, Dict] = defaultdict( + lambda: {"total_jobs": 0, "failed_jobs": 0, "jobs_failed": defaultdict(int)} + ) + + # Track consecutive failures per runner (by labels) + runner_current_streak: Dict[str, int] = defaultdict(int) + runner_max_streak: Dict[str, int] = defaultdict(int) + runner_first_failure_in_streak: Dict[str, Optional[Dict]] = {} + runner_last_failure_in_streak: Dict[str, Optional[Dict]] = {} + runner_recovery_info: Dict[str, Optional[Dict]] = {} + runner_error_signatures: Dict[str, Dict[str, int]] = defaultdict( + lambda: defaultdict(int) + ) + + # Track consecutive failures per runner instance + runner_instance_current_streak: Dict[str, int] = defaultdict(int) + runner_instance_max_streak: Dict[str, int] = defaultdict(int) + runner_instance_first_failure: Dict[str, Optional[Dict]] = {} + runner_instance_last_failure: Dict[str, Optional[Dict]] = {} + runner_instance_recovery: Dict[str, Optional[Dict]] = {} + runner_instance_error_signatures: Dict[str, Dict[str, int]] = defaultdict( + lambda: defaultdict(int) + ) + + total_runs_processed = len(sorted_runs) + for i, run in enumerate(sorted_runs, 1): + if i % 50 == 0 or i == total_runs_processed: + print( + f"Processing run {i}/{total_runs_processed} for runner analysis: #{run.get('run_number')}" + ) + + head_commit = run.get("head_commit") or {} + run_info = { + "run_number": run.get("run_number"), + "run_id": run.get("id"), + "created_at": run.get("created_at"), + "head_sha": run.get("head_sha", "")[:8], + "author": head_commit.get("author", {}).get("name", "Unknown"), + "url": f"https://github.com/{self.repo}/actions/runs/{run.get('id')}", + } + + pull_requests = run.get("pull_requests", []) + if pull_requests: + run_info["pr_number"] = pull_requests[0].get("number") + + # Get jobs for this run + jobs = self.get_jobs_for_run(run.get("id")) + + # Track whether each runner had at least one failure in this run + runner_had_failure: Dict[str, bool] = defaultdict(bool) + runner_had_success: Dict[str, bool] = defaultdict(bool) + runner_instance_had_failure: Dict[str, bool] = defaultdict(bool) + runner_instance_had_success: Dict[str, bool] = defaultdict(bool) + # Track first failed job for each runner in this run (for linking) + runner_first_failed_job: Dict[str, Dict] = {} + runner_instance_first_failed_job: Dict[str, Dict] = {} + + for job in jobs: + job_name = job.get("name", "") + + # Skip excluded jobs (administrative/setup jobs) + if any( + job_name.startswith(excluded) for excluded in self.excluded_jobs + ): + continue + + # Extract runner information + # GitHub API might use different fields for runner info + runner_name = ( + job.get("runner_name") + or job.get("runner", {}).get("name") + or "unknown" + ) + runner_id = job.get("runner_id") or job.get("runner", {}).get("id") + + # Get runner labels (from runs-on field in workflow) + runner_labels = job.get("labels", []) + runner_labels_str = ( + ", ".join(runner_labels) if runner_labels else "unknown" + ) + + # Skip jobs without runner information (likely skipped/queued jobs) + if not runner_labels_str or runner_labels_str == "unknown": + continue + + # Track by runner labels (primary identifier) + # Use labels as the key since they're more informative than runner_name + runner_key = runner_labels_str + runner_total_jobs[runner_key] += 1 + runner_job_totals[runner_key][job_name] += 1 + + # Track by specific runner instance + if runner_id: + runner_instance_key = f"{runner_labels_str}_{runner_id}" + runner_instance_stats[runner_instance_key]["total_jobs"] += 1 + # Store runner name for reference + runner_instance_stats[runner_instance_key][ + "runner_name" + ] = runner_name + + # Calculate queue time (time from created to started) per instance + created_at = job.get("created_at") + started_at = job.get("started_at") + if created_at and started_at: + try: + from datetime import datetime + + created_time = datetime.fromisoformat( + created_at.replace("Z", "+00:00") + ) + started_time = datetime.fromisoformat( + started_at.replace("Z", "+00:00") + ) + queue_time_seconds = ( + started_time - created_time + ).total_seconds() + if queue_time_seconds >= 0: # Sanity check + runner_instance_queue_times[runner_instance_key].append( + queue_time_seconds + ) + except (ValueError, AttributeError): + pass # Skip if timestamp parsing fails + + conclusion = job.get("conclusion") + + if conclusion == "failure": + # Failure detected + runner_failed_jobs[runner_key] += 1 + runner_job_failures[runner_key][job_name] += 1 + runner_had_failure[runner_key] = True + + # Track first failed job for this runner in this run (for linking) + if runner_key not in runner_first_failed_job: + runner_first_failed_job[runner_key] = { + "job_id": job.get("id"), + "job_url": job.get("html_url", run_info["url"]), + "job_name": job_name, + } + + # Extract error signature for runner + error_signature = self._extract_error_signature(job) + if error_signature: + runner_error_signatures[runner_key][error_signature] += 1 + + if runner_id: + runner_instance_stats[runner_instance_key]["failed_jobs"] += 1 + runner_instance_stats[runner_instance_key]["jobs_failed"][ + job_name + ] += 1 + runner_instance_had_failure[runner_instance_key] = True + + # Track first failed job for this runner instance in this run + if runner_instance_key not in runner_instance_first_failed_job: + runner_instance_first_failed_job[runner_instance_key] = { + "job_id": job.get("id"), + "job_url": job.get("html_url", run_info["url"]), + "job_name": job_name, + } + + # Extract error signature for runner instance + if error_signature: + runner_instance_error_signatures[runner_instance_key][ + error_signature + ] += 1 + + elif conclusion == "success": + runner_had_success[runner_key] = True + if runner_id: + runner_instance_had_success[runner_instance_key] = True + + # Update consecutive failure streaks based on run-level results + # A runner is considered "failing" if it had at least one failure in the run + for runner_key in set( + list(runner_had_failure.keys()) + list(runner_had_success.keys()) + ): + if runner_had_failure[runner_key]: + runner_current_streak[runner_key] += 1 + failure_info = { + **run_info, + "runner_key": runner_key, + } + + # Include job URL if we have it + if runner_key in runner_first_failed_job: + failure_info.update(runner_first_failed_job[runner_key]) + + # Track if this is the first failure in a new streak + if runner_current_streak[runner_key] == 1: + runner_first_failure_in_streak[runner_key] = failure_info + # Always update last failure to the most recent one + runner_last_failure_in_streak[runner_key] = failure_info + + # Update max streak + if ( + runner_current_streak[runner_key] + > runner_max_streak[runner_key] + ): + runner_max_streak[runner_key] = runner_current_streak[ + runner_key + ] + + elif runner_had_success[runner_key]: + # Success - streak broken + if runner_current_streak[runner_key] > 0: + runner_recovery_info[runner_key] = { + **run_info, + "runner_key": runner_key, + "streak_length": runner_current_streak[runner_key], + } + + runner_current_streak[runner_key] = 0 + runner_first_failure_in_streak[runner_key] = None + runner_last_failure_in_streak[runner_key] = None + + # Update instance streaks + for runner_instance_key in set( + list(runner_instance_had_failure.keys()) + + list(runner_instance_had_success.keys()) + ): + if runner_instance_had_failure[runner_instance_key]: + runner_instance_current_streak[runner_instance_key] += 1 + + if runner_instance_current_streak[runner_instance_key] == 1: + failure_info = { + **run_info, + "runner_instance": runner_instance_key, + } + # Include job URL if we have it + if runner_instance_key in runner_instance_first_failed_job: + failure_info.update( + runner_instance_first_failed_job[runner_instance_key] + ) + runner_instance_first_failure[runner_instance_key] = ( + failure_info + ) + + # Always update last failure to the most recent one + failure_info = { + **run_info, + "runner_instance": runner_instance_key, + } + # Include job URL if we have it + if runner_instance_key in runner_instance_first_failed_job: + failure_info.update( + runner_instance_first_failed_job[runner_instance_key] + ) + runner_instance_last_failure[runner_instance_key] = failure_info + + if ( + runner_instance_current_streak[runner_instance_key] + > runner_instance_max_streak[runner_instance_key] + ): + runner_instance_max_streak[runner_instance_key] = ( + runner_instance_current_streak[runner_instance_key] + ) + + elif runner_instance_had_success[runner_instance_key]: + if runner_instance_current_streak[runner_instance_key] > 0: + runner_instance_recovery[runner_instance_key] = { + **run_info, + "runner_instance": runner_instance_key, + "streak_length": runner_instance_current_streak[ + runner_instance_key + ], + } + + runner_instance_current_streak[runner_instance_key] = 0 + runner_instance_first_failure[runner_instance_key] = None + runner_instance_last_failure[runner_instance_key] = None + + time.sleep(0.05) + + # Build final runner stats + runner_stats = {} + for runner_key in runner_total_jobs.keys(): + total = runner_total_jobs[runner_key] + failed = runner_failed_jobs[runner_key] + failure_rate = (failed / total * 100) if total > 0 else 0 + + # Calculate queue time statistics by aggregating from runner instances + # Find all instances that match this runner label + aggregated_queue_times = [] + for instance_key, queue_times in runner_instance_queue_times.items(): + # Extract the labels part from "labels_id" + instance_labels = ( + instance_key.rsplit("_", 1)[0] + if "_" in instance_key + else instance_key + ) + if instance_labels == runner_key: + aggregated_queue_times.extend(queue_times) + + avg_queue_time = ( + sum(aggregated_queue_times) / len(aggregated_queue_times) + if aggregated_queue_times + else 0 + ) + p90_queue_time = 0 + if aggregated_queue_times: + sorted_queue_times = sorted(aggregated_queue_times) + p90_index = int(len(sorted_queue_times) * 0.9) + p90_queue_time = ( + sorted_queue_times[p90_index] + if p90_index < len(sorted_queue_times) + else sorted_queue_times[-1] + ) + + runner_stats[runner_key] = { + "total_jobs": total, + "failed_jobs": failed, + "failure_rate": failure_rate, + "unique_jobs_with_failures": len(runner_job_failures[runner_key]), + "jobs_failed": dict(runner_job_failures[runner_key]), + "jobs_total": dict(runner_job_totals[runner_key]), + "avg_queue_time_seconds": avg_queue_time, + "p90_queue_time_seconds": p90_queue_time, + "queue_time_samples": len(aggregated_queue_times), + } + + # Convert runner instance stats to regular dicts with queue time stats + runner_instance_data = {} + for instance_key, stats in runner_instance_stats.items(): + # Calculate queue time statistics for this instance + queue_times = runner_instance_queue_times[instance_key] + avg_queue_time = sum(queue_times) / len(queue_times) if queue_times else 0 + p90_queue_time = 0 + if queue_times: + sorted_queue_times = sorted(queue_times) + p90_index = int(len(sorted_queue_times) * 0.9) + p90_queue_time = ( + sorted_queue_times[p90_index] + if p90_index < len(sorted_queue_times) + else sorted_queue_times[-1] + ) + + runner_instance_data[instance_key] = { + "total_jobs": stats["total_jobs"], + "failed_jobs": stats["failed_jobs"], + "failure_rate": ( + stats["failed_jobs"] / stats["total_jobs"] * 100 + if stats["total_jobs"] > 0 + else 0 + ), + "jobs_failed": dict(stats["jobs_failed"]), + "runner_name": stats.get("runner_name", "unknown"), + "avg_queue_time_seconds": avg_queue_time, + "p90_queue_time_seconds": p90_queue_time, + "queue_time_samples": len(queue_times), + } + + # Build runner streak data + runner_streak_data = {} + for runner_key in runner_total_jobs.keys(): + # Get top 3 error signatures for this runner + error_sigs = runner_error_signatures.get(runner_key, {}) + top_errors = sorted(error_sigs.items(), key=lambda x: x[1], reverse=True)[ + :3 + ] + + runner_streak_data[runner_key] = { + "current_streak": runner_current_streak[runner_key], + "max_streak": runner_max_streak[runner_key], + "total_failures": runner_failed_jobs[runner_key], + "total_jobs": runner_total_jobs[runner_key], + "failure_rate": ( + runner_failed_jobs[runner_key] / runner_total_jobs[runner_key] * 100 + if runner_total_jobs[runner_key] > 0 + else 0 + ), + "jobs_failed": dict(runner_job_failures[runner_key]), + "first_failure_in_streak": runner_first_failure_in_streak.get( + runner_key + ), + "last_failure_in_streak": runner_last_failure_in_streak.get(runner_key), + "recovery_info": runner_recovery_info.get(runner_key), + "top_error_signatures": top_errors, + } + + # Build runner instance streak data + runner_instance_streak_data = {} + for instance_key in runner_instance_stats.keys(): + # Get top 3 error signatures for this runner instance + error_sigs = runner_instance_error_signatures.get(instance_key, {}) + top_errors = sorted(error_sigs.items(), key=lambda x: x[1], reverse=True)[ + :3 + ] + + runner_instance_streak_data[instance_key] = { + "current_streak": runner_instance_current_streak[instance_key], + "max_streak": runner_instance_max_streak[instance_key], + "total_failures": runner_instance_stats[instance_key]["failed_jobs"], + "total_jobs": runner_instance_stats[instance_key]["total_jobs"], + "failure_rate": ( + runner_instance_stats[instance_key]["failed_jobs"] + / runner_instance_stats[instance_key]["total_jobs"] + * 100 + if runner_instance_stats[instance_key]["total_jobs"] > 0 + else 0 + ), + "runner_name": runner_instance_stats[instance_key].get( + "runner_name", "unknown" + ), + "jobs_failed": dict(runner_instance_stats[instance_key]["jobs_failed"]), + "first_failure_in_streak": runner_instance_first_failure.get( + instance_key + ), + "last_failure_in_streak": runner_instance_last_failure.get( + instance_key + ), + "recovery_info": runner_instance_recovery.get(instance_key), + "top_error_signatures": top_errors, + } + + return ( + runner_stats, + runner_instance_data, + runner_streak_data, + runner_instance_streak_data, + ) + + def _extract_error_signature(self, job: Dict) -> str: + """ + Extract error signature from a failed job. + + Returns a simplified error type string. + """ + # Check if job has steps with failures + steps = job.get("steps", []) + if not steps: + return "Unknown Error" + + # Look for failed steps + failed_steps = [s for s in steps if s.get("conclusion") == "failure"] + if not failed_steps: + return "Unknown Error" + + # Try to fetch and parse logs for the first failed step + first_failed_step = failed_steps[0] + step_number = first_failed_step.get("number") + + # Attempt to get detailed error from logs + if step_number is not None: + try: + job_id = job.get("id") + # Fetch logs for this specific step + log_url = ( + f"{self.base_url}/repos/{self.repo}/actions/jobs/{job_id}/logs" + ) + response = self.session.get(log_url, timeout=10) + + if response.status_code == 200: + log_text = response.text + + # Check for specific error patterns in logs (case-insensitive) + log_lower = log_text.lower() + + # CUDA/GPU Memory errors (most common for GPU clusters) + if ( + "cuda out of memory" in log_lower + or "cudaerror: out of memory" in log_lower + ): + return "CUDA OOM" + elif "out of memory" in log_lower and ( + "gpu" in log_lower or "device" in log_lower + ): + return "GPU OOM" + elif "out of memory" in log_lower and "cuda" not in log_lower: + return "Out of Memory" + + # CUDA/GPU device errors + if ( + "cuda error: device-side assert" in log_lower + or "device-side assert" in log_lower + ): + return "CUDA Device Assert" + elif ( + "cuda error: an illegal memory access" in log_lower + or "illegal memory access" in log_lower + ): + return "CUDA Illegal Memory Access" + elif "cuda error" in log_lower or "cudaerror" in log_lower: + return "CUDA Error" + elif "gpu" in log_lower and ( + "hang" in log_lower or "hung" in log_lower + ): + return "GPU Hang" + elif ( + "no cuda-capable device" in log_lower + or "cuda device count" in log_lower + and "0" in log_lower + ): + return "No GPU Available" + + # ROCm/AMD GPU errors + if ( + "hipoutofmemoryerror" in log_lower + or "hip out of memory" in log_lower + ): + return "ROCm OOM" + elif "hiperror" in log_lower or "rocm error" in log_lower: + return "ROCm/HIP Error" + + # NCCL/collective communication errors (multi-GPU) + if "nccl error" in log_lower or "ncclerror" in log_lower: + return "NCCL Error" + elif "timeout after" in log_lower and "nccl" in log_lower: + return "NCCL Timeout" + + # Process/system errors + if "killed" in log_lower and ( + "oom" in log_lower or "out of memory" in log_lower + ): + return "Process Killed (OOM)" + elif "killed" in log_lower or "sigkill" in log_lower: + return "Process Killed" + elif "segmentation fault" in log_lower or "sigsegv" in log_lower: + return "Segmentation Fault" + + # Timeout errors + if "timeout" in log_lower or "timed out" in log_lower: + return "Timeout" + + # Connection/network errors + if ( + "connection refused" in log_lower + or "connection reset" in log_lower + ): + return "Connection Error" + elif "ssh" in log_lower and ( + "failed" in log_lower or "error" in log_lower + ): + return "SSH Error" + + # Import/module errors + if "modulenotfounderror" in log_lower or "importerror" in log_lower: + return "Import Error" + + # Assertion errors + if "assertionerror" in log_lower: + return "Assertion Error" + + # Pytest-specific errors + if ( + "pytest" in log_lower + and "error" in log_lower + and "collection" in log_lower + ): + return "Pytest Collection Error" + + except Exception: + # If log fetching fails, fall back to step name analysis + pass + + # Fallback to step name analysis if we couldn't get logs or didn't find specific errors + step_name = first_failed_step.get("name", "Unknown Step") + + # Simplify common patterns based on step name + if "timeout" in step_name.lower(): + return "Timeout" + elif "setup" in step_name.lower() or "install" in step_name.lower(): + return "Setup/Installation Error" + elif "test" in step_name.lower(): + return f"Test Failure: {step_name[:50]}" + elif "build" in step_name.lower(): + return "Build Error" + else: + return f"Step Failed: {step_name[:50]}" + + def construct_cron_failures_on_main( + self, runs: List[Dict], overall_job_streak_data: Dict[str, Dict] + ) -> Tuple[Dict[str, Dict], Dict[str, int]]: + """ + Analyses consecutive failures for each job triggered by cron on main branch only. + Compares error signatures with overall data to detect if main-branch failures + have same or different error patterns than PR-triggered failures. + + Args: + runs: All workflow runs (will be filtered to cron-triggered only) + overall_job_streak_data: Overall job streak data (from all runs) for comparison + + Returns: + Tuple of (main_streak_data, job_current_streaks_main) + - main_streak_data: Same structure as job_streak_data, plus 'matches_overall_error' field + - job_current_streaks_main: Dict mapping job name to current streak count on main + """ + print( + "\nAnalyzing consecutive failures on main branch (cron-triggered runs only)..." + ) + + # Filter to only cron-triggered runs (scheduled runs) + # Scheduled/cron runs have event == 'schedule' + cron_runs = [run for run in runs if run.get("event") == "schedule"] + + print( + f"Found {len(cron_runs)} cron-triggered runs out of {len(runs)} total runs" + ) + + if not cron_runs: + print("No cron-triggered runs found") + return {}, {} + + # Reuse existing analyze_consecutive_failures on filtered runs + main_streak_data, job_current_streaks_main = self.analyze_consecutive_failures( + cron_runs + ) + + # Now add comparison with overall data for at-a-glance diagnostics + for job_name, main_data in main_streak_data.items(): + matches_overall_error = False + + if job_name in overall_job_streak_data: + main_top_errors = main_data.get("top_error_signatures", []) + overall_top_errors = overall_job_streak_data[job_name].get( + "top_error_signatures", [] + ) + + if main_top_errors and overall_top_errors: + # Check if the most common error on main matches the most common overall error + main_top_error = main_top_errors[0][0] + overall_top_error = overall_top_errors[0][0] + matches_overall_error = main_top_error == overall_top_error + + # Add comparison flag to the data + main_data["matches_overall_error"] = matches_overall_error + + return main_streak_data, job_current_streaks_main + + def analyze_consecutive_failures( + self, runs: List[Dict] + ) -> Tuple[Dict[str, Dict], Dict[str, int]]: + """ + Analyze consecutive failures for each job. + + Returns: + Tuple of (job_streak_data, job_current_streaks) + """ + print("\nAnalyzing consecutive failures...") + + # Sort runs by created_at (oldest first) to track streaks chronologically + sorted_runs = sorted(runs, key=lambda x: x.get("created_at", "")) + + # Track current streak for each job + job_current_streak: Dict[str, int] = defaultdict(int) + job_max_streak: Dict[str, int] = defaultdict(int) + job_total_failures: Dict[str, int] = defaultdict(int) + job_total_runs: Dict[str, int] = defaultdict(int) + job_first_failure_in_streak: Dict[str, Optional[Dict]] = {} + job_last_failure_in_streak: Dict[str, Optional[Dict]] = {} + job_recovery_info: Dict[str, Optional[Dict]] = {} + job_error_signatures: Dict[str, Dict[str, int]] = defaultdict( + lambda: defaultdict(int) + ) + + total_runs_processed = len(sorted_runs) + for i, run in enumerate(sorted_runs, 1): + if i % 50 == 0 or i == total_runs_processed: + print( + f"Processing run {i}/{total_runs_processed}: #{run.get('run_number')}" + ) + + head_commit = run.get("head_commit") or {} + run_info = { + "run_number": run.get("run_number"), + "run_id": run.get("id"), + "created_at": run.get("created_at"), + "head_sha": run.get("head_sha", "")[:8], + "author": head_commit.get("author", {}).get("name", "Unknown"), + "url": f"https://github.com/{self.repo}/actions/runs/{run.get('id')}", + } + + pull_requests = run.get("pull_requests", []) + if pull_requests: + run_info["pr_number"] = pull_requests[0].get("number") + + # Get jobs for this run + jobs = self.get_jobs_for_run(run.get("id")) + + for job in jobs: + job_name = job.get("name", "") + + # Skip excluded jobs (administrative/setup jobs) + if any( + job_name.startswith(excluded) for excluded in self.excluded_jobs + ): + continue + + job_total_runs[job_name] += 1 + conclusion = job.get("conclusion") + + if conclusion == "failure": + # Failure detected + job_total_failures[job_name] += 1 + job_current_streak[job_name] += 1 + + # Track if this is the first failure in a new streak + if job_current_streak[job_name] == 1: + job_first_failure_in_streak[job_name] = { + **run_info, + "job_name": job_name, + "job_id": job.get("id"), + "job_url": job.get("html_url", run_info["url"]), + "conclusion": conclusion, + } + + # Always update last failure to the most recent one + job_last_failure_in_streak[job_name] = { + **run_info, + "job_name": job_name, + "job_id": job.get("id"), + "job_url": job.get("html_url", run_info["url"]), + "conclusion": conclusion, + } + + # Extract error signature from job + error_signature = self._extract_error_signature(job) + if error_signature: + job_error_signatures[job_name][error_signature] += 1 + + # Update max streak + if job_current_streak[job_name] > job_max_streak[job_name]: + job_max_streak[job_name] = job_current_streak[job_name] + + elif conclusion == "success": + # Success - streak broken + if job_current_streak[job_name] > 0: + # Record recovery + job_recovery_info[job_name] = { + **run_info, + "job_name": job_name, + "streak_length": job_current_streak[job_name], + } + + job_current_streak[job_name] = 0 + job_first_failure_in_streak[job_name] = None + job_last_failure_in_streak[job_name] = None + + time.sleep(0.05) + + # Build final results + job_streak_data = {} + for job_name in job_current_streak.keys(): + # Get top 3 error signatures + error_sigs = job_error_signatures.get(job_name, {}) + top_errors = sorted(error_sigs.items(), key=lambda x: x[1], reverse=True)[ + :3 + ] + + job_streak_data[job_name] = { + "current_streak": job_current_streak[job_name], + "max_streak": job_max_streak[job_name], + "total_failures": job_total_failures[job_name], + "total_runs": job_total_runs[job_name], + "failure_rate": ( + job_total_failures[job_name] / job_total_runs[job_name] * 100 + if job_total_runs[job_name] > 0 + else 0 + ), + "first_failure_in_streak": job_first_failure_in_streak.get(job_name), + "last_failure_in_streak": job_last_failure_in_streak.get(job_name), + "recovery_info": job_recovery_info.get(job_name), + "top_error_signatures": top_errors, + } + + return job_streak_data, job_current_streak + + def detect_alerts( + self, + job_streak_data: Dict[str, Dict], + job_current_streaks: Dict[str, int], + runner_stats: Optional[Dict[str, Dict]] = None, + runner_instance_data: Optional[Dict[str, Dict]] = None, + runner_streak_data: Optional[Dict[str, Dict]] = None, + runner_instance_streak_data: Optional[Dict[str, Dict]] = None, + ) -> Tuple[List[Dict], List[Dict]]: + """ + Detect jobs and runners that need alerts based on thresholds. + + Returns: + Tuple of (job_alerts, runner_alerts) + """ + job_alerts = [] + + for job_name, data in job_streak_data.items(): + current_streak = data["current_streak"] + + # Alert condition: consecutive failures >= threshold + if current_streak >= self.alert_threshold: + job_alerts.append( + { + "job_name": job_name, + "current_streak": current_streak, + "max_streak": data["max_streak"], + "failure_rate": data["failure_rate"], + "first_failure": data["first_failure_in_streak"], + "last_failure": data["last_failure_in_streak"], + "top_error_signatures": data.get("top_error_signatures", []), + "alert_type": "consecutive_failures", + "severity": "high" if current_streak >= 5 else "medium", + } + ) + + # Detect runner alerts + runner_alerts = [] + + # Alert for runners with consecutive failures + if runner_streak_data: + for runner_labels, streak_data in runner_streak_data.items(): + if streak_data["current_streak"] >= self.alert_threshold: + runner_alerts.append( + { + "runner_labels": runner_labels, + "current_streak": streak_data["current_streak"], + "max_streak": streak_data["max_streak"], + "failure_rate": streak_data["failure_rate"], + "total_failures": streak_data["total_failures"], + "total_jobs": streak_data["total_jobs"], + "jobs_failed": streak_data.get("jobs_failed", {}), + "first_failure": streak_data["first_failure_in_streak"], + "last_failure": streak_data["last_failure_in_streak"], + "top_error_signatures": streak_data.get( + "top_error_signatures", [] + ), + "alert_type": "runner_consecutive_failures", + "severity": ( + "high" + if streak_data["current_streak"] >= 5 + else "medium" + ), + } + ) + + # Alert for runner instances with consecutive failures + if runner_instance_streak_data: + for instance_key, streak_data in runner_instance_streak_data.items(): + if streak_data["current_streak"] >= self.alert_threshold: + # Get queue time info from runner_instance_data + instance_data = runner_instance_data.get(instance_key, {}) + avg_queue = instance_data.get("avg_queue_time_seconds", 0) + + runner_alerts.append( + { + "runner_instance": instance_key, + "runner_name": streak_data.get("runner_name", "unknown"), + "current_streak": streak_data["current_streak"], + "max_streak": streak_data["max_streak"], + "failure_rate": streak_data["failure_rate"], + "total_failures": streak_data["total_failures"], + "total_jobs": streak_data["total_jobs"], + "jobs_failed": streak_data.get("jobs_failed", {}), + "first_failure": streak_data["first_failure_in_streak"], + "last_failure": streak_data["last_failure_in_streak"], + "top_error_signatures": streak_data.get( + "top_error_signatures", [] + ), + "avg_queue_time_seconds": avg_queue, + "alert_type": "runner_instance_consecutive_failures", + "severity": ( + "high" + if streak_data["current_streak"] >= 5 + else "medium" + ), + } + ) + + if runner_stats: + # Alert if runner has high failure rate (>30%) and multiple jobs failing + for runner_labels, stats in runner_stats.items(): + if ( + stats["failure_rate"] > 50 + and stats["unique_jobs_with_failures"] >= 3 + ): + runner_alerts.append( + { + "runner_labels": runner_labels, + "failure_rate": stats["failure_rate"], + "total_jobs": stats["total_jobs"], + "failed_jobs": stats["failed_jobs"], + "unique_jobs_with_failures": stats[ + "unique_jobs_with_failures" + ], + "alert_type": "runner_health", + "severity": ( + "high" if stats["failure_rate"] > 50 else "medium" + ), + } + ) + + # Check for specific runner instances with concerning patterns + if runner_instance_data: + for instance_key, stats in runner_instance_data.items(): + # Alert if a specific runner instance has >50% failure rate with >=3 jobs + if stats["failure_rate"] > 50 and stats["total_jobs"] >= 3: + runner_alerts.append( + { + "runner_instance": instance_key, + "runner_name": stats.get("runner_name", "unknown"), + "failure_rate": stats["failure_rate"], + "total_jobs": stats["total_jobs"], + "failed_jobs": stats["failed_jobs"], + "jobs_failed": stats["jobs_failed"], + "alert_type": "runner_instance_health", + "severity": "high", + } + ) + + return job_alerts, runner_alerts + + # print statements here mainly for local testing + def generate_failure_report( + self, + job_streak_data: Dict[str, Dict], + job_alerts: List[Dict], + runner_stats: Optional[Dict[str, Dict]] = None, + runner_instance_data: Optional[Dict[str, Dict]] = None, + runner_alerts: Optional[List[Dict]] = None, + runner_streak_data: Optional[Dict[str, Dict]] = None, + runner_instance_streak_data: Optional[Dict[str, Dict]] = None, + main_streak_data: Optional[Dict[str, Dict]] = None, + output_file: Optional[str] = None, + ): + """Generate detailed failure analysis report.""" + print("\n" + "=" * 80) + print("SGLang Consecutive Failures Analysis Report") + print("=" * 80) + + # Sort jobs by current streak (descending) + sorted_jobs = sorted( + job_streak_data.items(), + key=lambda x: (x[1]["current_streak"], x[1]["failure_rate"]), + reverse=True, + ) + + # Summary Statistics + print("\n## Summary Statistics") + print( + f"Total (unique) jobs analyzed across PR Test workflows: {len(sorted_jobs)}" + ) + print( + f"Jobs with Active Failure Streaks: {sum(1 for j in sorted_jobs if j[1]['current_streak'] > 0)}" + ) + print(f"Job Alerts Triggered: {len(job_alerts)}") + + # Add counter for main branch cron jobs + if main_streak_data: + main_jobs_count = len(main_streak_data) + main_jobs_with_streaks = sum( + 1 for j in main_streak_data.values() if j["current_streak"] > 0 + ) + print( + f"Jobs on Main Branch (cron-triggered): {main_jobs_count} ({main_jobs_with_streaks} with active streaks)" + ) + else: + print(f"Jobs on Main Branch (cron-triggered): 0 (no cron runs found)") + + if runner_stats: + print(f"Total Runners Analyzed: {len(runner_stats)}") + print( + f"Runner Alerts Triggered: {len(runner_alerts) if runner_alerts else 0}" + ) + + # Queue Time Summary + if runner_stats: + all_avg_queue_times = [] + all_p90_queue_times = [] + for stats in runner_stats.values(): + if stats["queue_time_samples"] > 0: + all_avg_queue_times.append(stats["avg_queue_time_seconds"]) + all_p90_queue_times.append(stats["p90_queue_time_seconds"]) + + if all_avg_queue_times: + overall_avg = sum(all_avg_queue_times) / len(all_avg_queue_times) + overall_p90 = sum(all_p90_queue_times) / len(all_p90_queue_times) + print("\n## Queue Time Summary") + print( + f"Average Queue Time (across all runners): {overall_avg / 60:.1f} minutes ({overall_avg:.0f}s)" + ) + print( + f"P90 Queue Time (across all runners): {overall_p90 / 60:.1f} minutes ({overall_p90:.0f}s)" + ) + + # ALERTS: Critical Consecutive Job Failures (streak >= 2) + if job_alerts: + # Filter alerts with streak >= 2 + filtered_job_alerts = [a for a in job_alerts if a["current_streak"] >= 2] + + if filtered_job_alerts: + print("\n" + "=" * 150) + print( + "## ALERTS: Critical Consecutive Job Failures (PR + Scheduled, streak >= 2)" + ) + print("=" * 150) + print( + f"\n{'Job Name':<40} {'Streak':<8} {'Max':<6} {'First Failure':<16} {'Last Failure':<16} {'Top Errors':<60}" + ) + print("-" * 150) + + for alert in sorted( + filtered_job_alerts, key=lambda x: x["current_streak"], reverse=True + ): + job_name = alert["job_name"] + display_name = ( + job_name if len(job_name) <= 38 else job_name[:35] + "..." + ) + + first_failure = alert.get("first_failure") + first_failure_str = ( + f"Run #{first_failure['run_number']}" + if first_failure + else "N/A" + ) + + last_failure = alert.get("last_failure") + last_failure_str = ( + f"Run #{last_failure['run_number']}" if last_failure else "N/A" + ) + + # Format top errors - don't truncate + top_errors = alert.get("top_error_signatures", []) + if top_errors: + error_display = ", ".join( + [f"{err[0]} ({err[1]})" for err in top_errors] + ) + else: + error_display = "N/A" + + print( + f"{display_name:<40} {alert['current_streak']:<8} {alert['max_streak']:<6} {first_failure_str:<16} {last_failure_str:<16} {error_display:<60}" + ) + else: + print("\n" + "=" * 100) + print( + "## ALERTS: Critical Consecutive Job Failures (PR + Scheduled, streak >= 2)" + ) + print("=" * 100) + print( + "\nNothing to display (no jobs with consecutive failure streak >= 2)" + ) + + # ALERTS: Runners with Issues (streak >= 2) + if runner_alerts: + # Only show consecutive failure alerts with streak >= 2, and only machine instances + instance_alerts = [ + a + for a in runner_alerts + if a["alert_type"] == "runner_instance_consecutive_failures" + and a.get("current_streak", 0) >= 2 + ] + + if instance_alerts: + print("\n" + "=" * 170) + print("## ALERTS: Runners with Issues (streak >= 2)") + print("=" * 170) + print("\n### Runner Consecutive Failures") + print( + f"\n{'Runner':<30} {'Str':<5} {'Max':<5} {'Fail%':<7} {'AvgQ':<7} {'First':<13} {'Last':<13} {'Top Errors':<45} {'Jobs Failed':<40}" + ) + print("-" * 170) + + for alert in sorted( + instance_alerts, + key=lambda x: x.get("current_streak", 0), + reverse=True, + ): + # Use the actual machine name instead of labels or instance key + runner_name = alert.get("runner_name", "unknown") + display_name = ( + runner_name + if len(runner_name) <= 28 + else runner_name[:25] + "..." + ) + + # Get all failed jobs - don't truncate + jobs_failed = alert.get("jobs_failed", {}) + top_jobs = sorted( + jobs_failed.items(), key=lambda x: x[1], reverse=True + ) + jobs_display = ( + ", ".join([f"{job} ({count})" for job, count in top_jobs]) + if top_jobs + else "N/A" + ) + + # Format queue time + avg_queue = alert.get("avg_queue_time_seconds", 0) + avg_queue_str = f"{avg_queue / 60:.1f}m" if avg_queue > 0 else "N/A" + + first_failure = alert.get("first_failure") + first_failure_str = ( + f"Run #{first_failure['run_number']}" + if first_failure + else "N/A" + ) + + last_failure = alert.get("last_failure") + last_failure_str = ( + f"Run #{last_failure['run_number']}" if last_failure else "N/A" + ) + + # Format top errors - don't truncate + top_errors = alert.get("top_error_signatures", []) + if top_errors: + error_display = ", ".join( + [f"{err[0]} ({err[1]})" for err in top_errors] + ) + else: + error_display = "N/A" + + print( + f"{display_name:<30} {alert['current_streak']:<5} {alert['max_streak']:<5} {alert['failure_rate']:>5.1f}% {avg_queue_str:<7} {first_failure_str:<13} {last_failure_str:<13} {error_display:<45} {jobs_display:<40}" + ) + else: + print("\n" + "=" * 100) + print("## ALERTS: Runners with Issues (streak >= 2)") + print("=" * 100) + print( + "\nNothing to display (no runners with consecutive failure streak >= 2)" + ) + + # Main Branch Health Section: Jobs failing on cron-triggered main branch runs + if main_streak_data: + # Sort by current streak (descending) + sorted_main_jobs = sorted( + main_streak_data.items(), + key=lambda x: (x[1]["current_streak"], x[1]["failure_rate"]), + reverse=True, + ) + + # Show only jobs with streak >= 2 + broken_main_jobs = [ + (name, data) + for name, data in sorted_main_jobs + if data["current_streak"] >= 2 + ] + + if broken_main_jobs: + print("\n" + "=" * 140) + print( + f"## MAIN BRANCH HEALTH: Failing Jobs on Scheduled Main Branch Runs ({len(broken_main_jobs)} jobs)" + ) + print("=" * 140) + print( + f"\n{'Job Name':<40} {'Streak':<8} {'Max':<6} {'First':<13} {'Last':<13} {'Top Errors':<50}" + ) + print("-" * 140) + for job_name, data in broken_main_jobs[:15]: + display_name = ( + job_name if len(job_name) <= 38 else job_name[:35] + "..." + ) + + # Get first and last failure info + first_failure = data.get("first_failure_in_streak") + first_failure_str = ( + f"Run #{first_failure['run_number']}" + if first_failure + else "N/A" + ) + + last_failure = data.get("last_failure_in_streak") + last_failure_str = ( + f"Run #{last_failure['run_number']}" if last_failure else "N/A" + ) + + # Format top errors - don't truncate + top_errors = data.get("top_error_signatures", []) + if top_errors: + error_display = ", ".join( + [f"{err[0]} ({err[1]})" for err in top_errors] + ) + else: + error_display = "N/A" + + print( + f"{display_name:<40} {data['current_streak']:<8} {data['max_streak']:<6} {first_failure_str:<13} {last_failure_str:<13} {error_display:<50}" + ) + else: + print("\n" + "=" * 100) + print("## MAIN BRANCH HEALTH: Scheduled Main Branch Runs") + print("=" * 100) + print( + "\n No consecutive failing jobs (streak >= 2) on main branch scheduled runs" + ) + + # Section 1: Currently Broken Jobs (streak >= 2) + broken_jobs = [ + (name, data) for name, data in sorted_jobs if data["current_streak"] >= 2 + ] + + if broken_jobs: + print("\n" + "=" * 140) + print( + "## Section 1: Top 15 Consecutively Failing Jobs (PR + Scheduled, streak >= 2)" + ) + print("=" * 140) + print( + f"\n{'Job Name':<40} {'Streak':<8} {'Max':<6} {'First':<13} {'Last':<13} {'Top Errors':<50}" + ) + print("-" * 140) + for job_name, data in broken_jobs[:20]: + display_name = ( + job_name if len(job_name) <= 38 else job_name[:35] + "..." + ) + + # Get first and last failure info + first_failure = data.get("first_failure_in_streak") + first_failure_str = ( + f"Run #{first_failure['run_number']}" if first_failure else "N/A" + ) + + last_failure = data.get("last_failure_in_streak") + last_failure_str = ( + f"Run #{last_failure['run_number']}" if last_failure else "N/A" + ) + + # Format top errors - don't truncate + top_errors = data.get("top_error_signatures", []) + if top_errors: + error_display = ", ".join( + [f"{err[0]} ({err[1]})" for err in top_errors] + ) + else: + error_display = "N/A" + + print( + f"{display_name:<40} {data['current_streak']:<8} {data['max_streak']:<6} {first_failure_str:<13} {last_failure_str:<13} {error_display:<50}" + ) + + # Section 2: Runner Health Analysis - Use machine names from runner instances (streak >= 2) + if runner_instance_data and runner_instance_streak_data: + # Combine instance stats with streak data and sort by consecutive failures first + combined_data = [] + for instance_key, stats in runner_instance_data.items(): + streak_data = runner_instance_streak_data.get(instance_key, {}) + combined_data.append( + { + "runner_name": stats.get("runner_name", "unknown"), + "instance_key": instance_key, + "current_streak": streak_data.get("current_streak", 0), + "max_streak": streak_data.get("max_streak", 0), + "failure_rate": stats["failure_rate"], + "total_jobs": stats["total_jobs"], + "unique_jobs": len(stats.get("jobs_failed", {})), + "avg_queue": stats.get("avg_queue_time_seconds", 0), + "p90_queue": stats.get("p90_queue_time_seconds", 0), + "queue_samples": stats.get("queue_time_samples", 0), + "first_failure": streak_data.get("first_failure_in_streak"), + "last_failure": streak_data.get("last_failure_in_streak"), + "top_error_signatures": streak_data.get( + "top_error_signatures", [] + ), + } + ) + + # Sort by current streak (descending), then max streak, then failure rate + sorted_runners = sorted( + combined_data, + key=lambda x: (x["current_streak"], x["max_streak"], x["failure_rate"]), + reverse=True, + ) + + # Only show runners with streak >= 2 + runners_with_issues = [ + r for r in sorted_runners if r["current_streak"] >= 2 + ] + + if runners_with_issues: + print("\n" + "=" * 160) + print( + "## Section 2: Top 15 Workers by Consecutive Failures (streak >= 2)" + ) + print("=" * 160) + print( + f"\n{'Machine Name':<30} {'Str':<5} {'Max':<5} {'Fail%':<7} {'AvgQ':<7} {'First':<13} {'Last':<13} {'Top Errors':<45} {'Total Jobs':<11} {'Unique Jobs':<12}" + ) + print("-" * 160) + + for runner_data in runners_with_issues[:15]: + # Truncate machine name if too long for display + display_name = ( + runner_data["runner_name"] + if len(runner_data["runner_name"]) <= 28 + else runner_data["runner_name"][:25] + "..." + ) + + # Format streaks + streak_str = str(runner_data["current_streak"]) + max_str = str(runner_data["max_streak"]) + + # Format queue time + avg_queue_str = ( + f"{runner_data['avg_queue'] / 60:.1f}m" + if runner_data["queue_samples"] > 0 + else "N/A" + ) + + # Get first and last failure info + first_failure = runner_data.get("first_failure") + first_failure_str = ( + f"Run #{first_failure['run_number']}" + if first_failure + else "N/A" + ) + + last_failure = runner_data.get("last_failure") + last_failure_str = ( + f"Run #{last_failure['run_number']}" if last_failure else "N/A" + ) + + # Format top errors - don't truncate + top_errors = runner_data.get("top_error_signatures", []) + if top_errors: + error_display = ", ".join( + [f"{err[0]} ({err[1]})" for err in top_errors] + ) + else: + error_display = "N/A" + + print( + f"{display_name:<30} {streak_str:<5} {max_str:<5} {runner_data['failure_rate']:>5.1f}% {avg_queue_str:<7} {first_failure_str:<13} {last_failure_str:<13} {error_display:<45} {runner_data['total_jobs']:<11} {runner_data['unique_jobs']:<12}" + ) + + # Build report data (always needed for GitHub summary) + # Calculate overall queue time for summary + overall_avg_queue = 0 + overall_p90_queue = 0 + if runner_stats: + all_avg_queue_times = [ + stats["avg_queue_time_seconds"] + for stats in runner_stats.values() + if stats["queue_time_samples"] > 0 + ] + all_p90_queue_times = [ + stats["p90_queue_time_seconds"] + for stats in runner_stats.values() + if stats["queue_time_samples"] > 0 + ] + if all_avg_queue_times: + overall_avg_queue = sum(all_avg_queue_times) / len(all_avg_queue_times) + overall_p90_queue = sum(all_p90_queue_times) / len(all_p90_queue_times) + + # Calculate main branch stats + main_jobs_count = len(main_streak_data) if main_streak_data else 0 + main_jobs_with_streaks = ( + sum(1 for j in main_streak_data.values() if j["current_streak"] > 0) + if main_streak_data + else 0 + ) + + report_data = { + "summary": { + "total_jobs": len(sorted_jobs), + "jobs_with_streaks": sum( + 1 for j in sorted_jobs if j[1]["current_streak"] > 0 + ), + "job_alerts_triggered": len(job_alerts), + "runner_alerts_triggered": len(runner_alerts) if runner_alerts else 0, + "total_runners": len(runner_stats) if runner_stats else 0, + "alert_threshold": self.alert_threshold, + "analysis_timestamp": datetime.now().isoformat(), + "avg_queue_time_seconds": overall_avg_queue, + "p90_queue_time_seconds": overall_p90_queue, + "main_jobs_count": main_jobs_count, + "main_jobs_with_streaks": main_jobs_with_streaks, + }, + "job_streak_data": { + job_name: { + **data, + # Convert datetime objects to strings for JSON serialization + "first_failure_in_streak": data["first_failure_in_streak"], + "recovery_info": data["recovery_info"], + } + for job_name, data in sorted_jobs + }, + "job_alerts": job_alerts, + "runner_stats": runner_stats if runner_stats else {}, + "runner_instance_data": ( + runner_instance_data if runner_instance_data else {} + ), + "runner_streak_data": runner_streak_data if runner_streak_data else {}, + "runner_instance_streak_data": ( + runner_instance_streak_data if runner_instance_streak_data else {} + ), + "runner_alerts": runner_alerts if runner_alerts else [], + "main_streak_data": main_streak_data if main_streak_data else {}, + } + + # Save to JSON only if output file is specified + if output_file: + with open(output_file, "w", encoding="utf-8") as f: + json.dump(report_data, f, ensure_ascii=False, indent=2) + print(f"\nDetailed report saved to: {output_file}") + + print("=" * 80) + + return report_data + + def generate_github_summary(self, report_data: Dict): + """Generate GitHub Actions Step Summary.""" + try: + github_step_summary = os.environ.get("GITHUB_STEP_SUMMARY") + if not github_step_summary: + print("Not running in GitHub Actions, skipping summary generation") + return + + print("Generating GitHub Actions summary...") + + summary_lines = [] + summary_lines.append("# SGLang Consecutive Failures Analysis") + summary_lines.append("") + summary_lines.append( + f"**Analysis Timestamp:** {report_data['summary']['analysis_timestamp']}" + ) + summary_lines.append( + f"**Alert Threshold:** {report_data['summary']['alert_threshold']} consecutive failures" + ) + summary_lines.append("") + + # Summary stats + summary_lines.append("## Summary Statistics") + summary_lines.append("") + summary_lines.append("| Metric | Count |") + summary_lines.append("|--------|-------|") + summary_lines.append( + f"| Total (unique) jobs analyzed across PR Test workflows | {report_data['summary']['total_jobs']} |" + ) + summary_lines.append( + f"| Jobs with Active Failure Streaks | {report_data['summary']['jobs_with_streaks']} |" + ) + summary_lines.append( + f"| Job Alerts Triggered | {report_data['summary']['job_alerts_triggered']} |" + ) + + # Add main branch job counter + main_jobs_count = report_data["summary"].get("main_jobs_count", 0) + main_jobs_with_streaks = report_data["summary"].get( + "main_jobs_with_streaks", 0 + ) + if main_jobs_count > 0: + summary_lines.append( + f"| Jobs on Main Branch (cron-triggered) | {main_jobs_count} ({main_jobs_with_streaks} with active streaks) |" + ) + else: + summary_lines.append( + f"| Jobs on Main Branch (cron-triggered) | 0 (no cron runs found) |" + ) + + summary_lines.append( + f"| Total Runners Analyzed | {report_data['summary']['total_runners']} |" + ) + summary_lines.append( + f"| Runner Alerts Triggered | {report_data['summary']['runner_alerts_triggered']} |" + ) + summary_lines.append("") + + # Queue Time Summary + if report_data.get("summary", {}).get("avg_queue_time_seconds") is not None: + summary_lines.append("## Queue Time Summary") + summary_lines.append("") + summary_lines.append("| Metric | Value |") + summary_lines.append("|--------|-------|") + avg_queue = report_data["summary"]["avg_queue_time_seconds"] + p90_queue = report_data["summary"]["p90_queue_time_seconds"] + summary_lines.append( + f"| Average Queue Time (across all runners) | {avg_queue / 60:.1f} minutes ({avg_queue:.0f}s) |" + ) + summary_lines.append( + f"| P90 Queue Time (across all runners) | {p90_queue / 60:.1f} minutes ({p90_queue:.0f}s) |" + ) + summary_lines.append("") + + # Job Alerts section (streak >= 2) + if report_data.get("job_alerts"): + # Filter alerts with streak >= 2 + filtered_job_alerts = [ + a for a in report_data["job_alerts"] if a["current_streak"] >= 2 + ] + + if filtered_job_alerts: + summary_lines.append( + "## Alerts: Critical Consecutive Job Failures (PR + Scheduled, streak >= 2)" + ) + summary_lines.append("") + summary_lines.append( + "| Job Name | Streak | Max | First Failure | Last Failure | Top Errors |" + ) + summary_lines.append( + "|----------|--------|-----|---------------|--------------|------------|" + ) + + for alert in sorted( + filtered_job_alerts, + key=lambda x: x["current_streak"], + reverse=True, + ): + job_name = alert["job_name"] + if len(job_name) > 35: + job_name = job_name[:32] + "..." + + first_failure = alert.get("first_failure") + if first_failure: + first_failure_str = f"[Run #{first_failure['run_number']}]({first_failure.get('job_url', first_failure['url'])})" + else: + first_failure_str = "N/A" + + last_failure = alert.get("last_failure") + if last_failure: + last_failure_str = f"[Run #{last_failure['run_number']}]({last_failure.get('job_url', last_failure['url'])})" + else: + last_failure_str = "N/A" + + # Format top errors as bullet list + top_errors = alert.get("top_error_signatures", []) + if top_errors: + error_str = "
".join( + [f"• {err[0]} ({err[1]})" for err in top_errors] + ) + else: + error_str = "N/A" + + summary_lines.append( + f"| `{job_name}` | {alert['current_streak']} | {alert['max_streak']} | " + f"{first_failure_str} | {last_failure_str} | {error_str} |" + ) + + summary_lines.append("") + else: + summary_lines.append( + "## Alerts: Critical Consecutive Job Failures (PR + Scheduled, streak >= 2)" + ) + summary_lines.append("") + summary_lines.append( + "Nothing to display (no jobs with consecutive failure streak >= 2)" + ) + summary_lines.append("") + + # Runner Alerts section (streak >= 2) + if report_data.get("runner_alerts"): + # Only show consecutive failure alerts with streak >= 2, and only machine instances + instance_alerts = [ + a + for a in report_data["runner_alerts"] + if a["alert_type"] == "runner_instance_consecutive_failures" + and a.get("current_streak", 0) >= 2 + ] + + if instance_alerts: + summary_lines.append("## Alerts: Workers with Issues (streak >= 2)") + summary_lines.append("") + summary_lines.append( + "| Runner | Streak | Max | Fail Rate | Avg Queue | First Failure | Last Failure | Top Errors | Jobs Failed |" + ) + summary_lines.append( + "|--------|--------|-----|-----------|-----------|---------------|--------------|------------|-------------|" + ) + + for alert in sorted( + instance_alerts, + key=lambda x: x.get("current_streak", 0), + reverse=True, + ): + # Use the actual machine name instead of labels or instance key + runner_name = alert.get("runner_name", "unknown") + if len(runner_name) > 28: + runner_name = runner_name[:25] + "..." + + # Get all failed jobs as bullet list + jobs_failed = alert.get("jobs_failed", {}) + top_jobs = sorted( + jobs_failed.items(), key=lambda x: x[1], reverse=True + ) + jobs_str = ( + "
".join( + [f"• {job} ({count})" for job, count in top_jobs] + ) + if top_jobs + else "N/A" + ) + + # Format queue time + avg_queue = alert.get("avg_queue_time_seconds", 0) + avg_queue_str = ( + f"{avg_queue / 60:.1f}m" if avg_queue > 0 else "N/A" + ) + + first_failure = alert.get("first_failure") + if first_failure: + first_failure_str = f"[Run #{first_failure['run_number']}]({first_failure.get('job_url', first_failure['url'])})" + else: + first_failure_str = "N/A" + + last_failure = alert.get("last_failure") + if last_failure: + last_failure_str = f"[Run #{last_failure['run_number']}]({last_failure.get('job_url', last_failure['url'])})" + else: + last_failure_str = "N/A" + + # Format top errors as bullet list + top_errors = alert.get("top_error_signatures", []) + if top_errors: + error_str = "
".join( + [f"• {err[0]} ({err[1]})" for err in top_errors] + ) + else: + error_str = "N/A" + + summary_lines.append( + f"| `{runner_name}` | {alert['current_streak']} | {alert['max_streak']} | " + f"{alert['failure_rate']:.1f}% | {avg_queue_str} | {first_failure_str} | {last_failure_str} | " + f"{error_str} | {jobs_str} |" + ) + + summary_lines.append("") + summary_lines.append("") + else: + summary_lines.append("## Alerts: Runners with Issues (streak >= 2)") + summary_lines.append("") + summary_lines.append( + "Nothing to display (no runners with consecutive failure streak >= 2)" + ) + summary_lines.append("") + summary_lines.append("") + + # Main Branch Health Section: Jobs failing on cron-triggered main branch runs + if report_data.get("main_streak_data"): + # Sort by current streak (descending) + sorted_main_jobs = sorted( + report_data["main_streak_data"].items(), + key=lambda x: (x[1]["current_streak"], x[1]["failure_rate"]), + reverse=True, + ) + + # Show only jobs with streak >= 2 + broken_main_jobs = [ + (name, data) + for name, data in sorted_main_jobs + if data["current_streak"] >= 2 + ] + + if broken_main_jobs: + summary_lines.append( + f"## Main Branch Health: Consecutive Failing Jobs on Scheduled Main Branch Runs (streak >= 2)" + ) + summary_lines.append("") + summary_lines.append( + "| Job Name | Streak | Max | First Failure | Last Failure | Top Errors |" + ) + summary_lines.append( + "|----------|--------|-----|---------------|--------------|------------|" + ) + for job_name, data in broken_main_jobs[:15]: + display_name = ( + job_name if len(job_name) <= 35 else job_name[:32] + "..." + ) + + # Get first and last failure info + first_failure = data.get("first_failure_in_streak") + if first_failure: + first_failure_str = f"[Run #{first_failure['run_number']}]({first_failure.get('job_url', first_failure['url'])})" + else: + first_failure_str = "N/A" + + last_failure = data.get("last_failure_in_streak") + if last_failure: + last_failure_str = f"[Run #{last_failure['run_number']}]({last_failure.get('job_url', last_failure['url'])})" + else: + last_failure_str = "N/A" + + # Format top errors as bullet list + top_errors = data.get("top_error_signatures", []) + if top_errors: + error_str = "
".join( + [f"• {err[0]} ({err[1]})" for err in top_errors] + ) + else: + error_str = "N/A" + + summary_lines.append( + f"| `{display_name}` | {data['current_streak']} | {data['max_streak']} | " + f"{first_failure_str} | {last_failure_str} | {error_str} |" + ) + + summary_lines.append("") + else: + summary_lines.append( + "## Main Branch Health: Scheduled Main Branch Runs" + ) + summary_lines.append("") + summary_lines.append( + "No consecutive failing jobs (streak >= 2) on main branch scheduled runs" + ) + summary_lines.append("") + + # Section 1: Currently Broken Jobs - Only show if there are broken jobs + sorted_jobs = sorted( + report_data["job_streak_data"].items(), + key=lambda x: (x[1]["current_streak"], x[1]["failure_rate"]), + reverse=True, + ) + + # Only show jobs with streak >= 2 + broken_jobs = [ + (name, data) + for name, data in sorted_jobs + if data["current_streak"] >= 2 + ] + + if broken_jobs: + summary_lines.append( + "## Section 1: Top 15 Consecutively Failing Jobs (PR + Scheduled, streak >= 2)" + ) + summary_lines.append("") + summary_lines.append( + "| Job Name | Streak | Max | First Failure | Last Failure | Top Errors |" + ) + summary_lines.append( + "|----------|--------|-----|---------------|--------------|------------|" + ) + for job_name, data in broken_jobs[:20]: + display_name = ( + job_name if len(job_name) <= 35 else job_name[:32] + "..." + ) + + # Get first and last failure info + first_failure = data.get("first_failure_in_streak") + if first_failure: + first_failure_str = f"[Run #{first_failure['run_number']}]({first_failure.get('job_url', first_failure['url'])})" + else: + first_failure_str = "N/A" + + last_failure = data.get("last_failure_in_streak") + if last_failure: + last_failure_str = f"[Run #{last_failure['run_number']}]({last_failure.get('job_url', last_failure['url'])})" + else: + last_failure_str = "N/A" + + # Format top errors as bullet list + top_errors = data.get("top_error_signatures", []) + if top_errors: + error_str = "
".join( + [f"• {err[0]} ({err[1]})" for err in top_errors] + ) + else: + error_str = "N/A" + + summary_lines.append( + f"| `{display_name}` | {data['current_streak']} | {data['max_streak']} | " + f"{first_failure_str} | {last_failure_str} | {error_str} |" + ) + + summary_lines.append("") + + # Section 2: Runner Health Analysis - Use machine names from runner instances + if report_data.get("runner_instance_data") and report_data.get( + "runner_instance_streak_data" + ): + # Combine instance stats with streak data and sort by consecutive failures first + combined_data = [] + for instance_key, stats in report_data["runner_instance_data"].items(): + streak_data = report_data["runner_instance_streak_data"].get( + instance_key, {} + ) + combined_data.append( + { + "runner_name": stats.get("runner_name", "unknown"), + "instance_key": instance_key, + "current_streak": streak_data.get("current_streak", 0), + "max_streak": streak_data.get("max_streak", 0), + "failure_rate": stats["failure_rate"], + "total_jobs": stats["total_jobs"], + "unique_jobs": len(stats.get("jobs_failed", {})), + "avg_queue": stats.get("avg_queue_time_seconds", 0), + "p90_queue": stats.get("p90_queue_time_seconds", 0), + "queue_samples": stats.get("queue_time_samples", 0), + "first_failure": streak_data.get("first_failure_in_streak"), + "last_failure": streak_data.get("last_failure_in_streak"), + "top_error_signatures": streak_data.get( + "top_error_signatures", [] + ), + } + ) + + # Sort by current streak (descending), then max streak, then failure rate + sorted_runners = sorted( + combined_data, + key=lambda x: ( + x["current_streak"], + x["max_streak"], + x["failure_rate"], + ), + reverse=True, + ) + + # Only show runners with streak >= 2 + runners_with_issues = [ + r for r in sorted_runners if r["current_streak"] >= 2 + ] + + if runners_with_issues: + summary_lines.append( + "## Section 2: Top 15 Consecutively Failing Workers (streak >= 2)" + ) + summary_lines.append("") + summary_lines.append( + "| Machine Name | Streak | Max | Fail Rate | Avg Queue | First Failure | Last Failure | Top Errors | Total Jobs | Unique Jobs |" + ) + summary_lines.append( + "|--------------|--------|-----|-----------|-----------|---------------|--------------|------------|------------|-------------|" + ) + + for runner_data in runners_with_issues[:15]: + display_name = ( + runner_data["runner_name"] + if len(runner_data["runner_name"]) <= 28 + else runner_data["runner_name"][:25] + "..." + ) + + # Format streaks + streak_str = str(runner_data["current_streak"]) + max_str = str(runner_data["max_streak"]) + + # Format queue time + avg_queue_str = ( + f"{runner_data['avg_queue'] / 60:.1f}m" + if runner_data["queue_samples"] > 0 + else "N/A" + ) + + # Get first and last failure info + first_failure = runner_data.get("first_failure") + if first_failure: + first_failure_str = f"[Run #{first_failure['run_number']}]({first_failure.get('job_url', first_failure['url'])})" + else: + first_failure_str = "N/A" + + last_failure = runner_data.get("last_failure") + if last_failure: + last_failure_str = f"[Run #{last_failure['run_number']}]({last_failure.get('job_url', last_failure['url'])})" + else: + last_failure_str = "N/A" + + # Format top errors as bullet list + top_errors = runner_data.get("top_error_signatures", []) + if top_errors: + error_str = "
".join( + [f"• {err[0]} ({err[1]})" for err in top_errors] + ) + else: + error_str = "N/A" + + summary_lines.append( + f"| `{display_name}` | {streak_str} | {max_str} | {runner_data['failure_rate']:.1f}% | " + f"{avg_queue_str} | {first_failure_str} | {last_failure_str} | {error_str} | " + f"{runner_data['total_jobs']} | {runner_data['unique_jobs']} |" + ) + + summary_lines.append("") + + # Write summary + with open(github_step_summary, "a", encoding="utf-8") as f: + f.write("\n".join(summary_lines)) + + print("GitHub Actions summary generated successfully") + + except Exception as e: + print(f"Failed to generate GitHub Actions summary: {e}") + import traceback + + traceback.print_exc() + + +def main(): + parser = argparse.ArgumentParser(description="SGLang Consecutive Failures Analyzer") + parser.add_argument("--token", required=True, help="GitHub Personal Access Token") + parser.add_argument( + "--limit", + type=int, + default=1000, + help="Number of workflow runs to analyze across all monitored workflows (default: 1000)", + ) + parser.add_argument( + "--threshold", + type=int, + default=3, + help="Alert threshold for consecutive failures (default: 3)", + ) + parser.add_argument( + "--output", + default=None, + help="Output JSON file (optional, only writes if specified)", + ) + + args = parser.parse_args() + + analyzer = SGLangFailuresAnalyzer(args.token, alert_threshold=args.threshold) + + try: + # Fetch recent runs + runs = analyzer.get_recent_runs(args.limit) + + if not runs: + print("No workflow runs found") + return + + # Analyze consecutive failures + job_streak_data, job_current_streaks = analyzer.analyze_consecutive_failures( + runs + ) + + if not job_streak_data: + print("No job data found") + return + + # Skip aggregation to show individual job shards + print(f"\nTotal jobs (including shards): {len(job_streak_data)}") + + # Analyze consecutive failures on main branch (cron-triggered only) + main_streak_data, main_current_streaks = ( + analyzer.construct_cron_failures_on_main(runs, job_streak_data) + ) + + # Analyze runner health and consecutive failures + ( + runner_stats, + runner_instance_data, + runner_streak_data, + runner_instance_streak_data, + ) = analyzer.analyze_runner_health(runs) + + # Detect alerts + job_alerts, runner_alerts = analyzer.detect_alerts( + job_streak_data, + job_current_streaks, + runner_stats, + runner_instance_data, + runner_streak_data, + runner_instance_streak_data, + ) + + # Generate report + report_data = analyzer.generate_failure_report( + job_streak_data, + job_alerts, + runner_stats, + runner_instance_data, + runner_alerts, + runner_streak_data, + runner_instance_streak_data, + main_streak_data, + args.output, + ) + + # Generate GitHub Actions summary + analyzer.generate_github_summary(report_data) + + # Exit with error code if alerts triggered + total_alerts = len(job_alerts) + len(runner_alerts) + if total_alerts > 0: + print( + f"\n!!!!! {len(job_alerts)} job alert(s) and {len(runner_alerts)} runner alert(s) triggered!" + ) + sys.exit(0) # Don't fail the workflow, just report + else: + print("\n No alerts triggered") + + except Exception as e: + print(f"Error during analysis: {e}") + import traceback + + traceback.print_exc() + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/code_sync/copy_from_oss.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/code_sync/copy_from_oss.py new file mode 100644 index 00000000..7517dff9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/code_sync/copy_from_oss.py @@ -0,0 +1,299 @@ +""" +Sync code from OSS repo to the local repo and open a PR if changes exist. + +NOTE: +1. You need to execute this script in the git root folder. +2. A GH_TOKEN environment variable is required to create the pull request. + - see also https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens + +This script will: +1. Clone the sgl-project/sglang repository (or use a local copy). +2. Sync specified files and directories using rsync. +3. Check if the sync operation resulted in any changes. +4. If there are changes: + a. Create a new branch. + b. Commit and push the changes. + c. Open a pull request using the GitHub CLI (gh). + +Usage: +# Run the full sync and PR creation process +python3 scripts/copy_from_oss.py + +# Perform a dry run without making any actual changes +python3 scripts/copy_from_oss.py --dry-run + +# Use a local directory as the source instead of cloning +python3 scripts/copy_from_oss.py --local-dir ~/projects/sglang +""" + +import argparse +import datetime +import os +import shutil +import subprocess +import tempfile + +# --- Configuration Begin --- +# List of folders and files to copy from the OSS repo. +# Changes outside these paths will be ignored. +folder_names = [ + "3rdparty", + "assets", + "benchmark", + "docker", + "docs", + "examples", + "python/sglang/lang", + "python/sglang/jit_kernel", + "python/sglang/srt", + "python/sglang/test", + "python/sglang/utils.py", + "python/sglang/README.md", + "sgl-kernel", + "test/manual", + "test/nightly", + "test/registered", + "test/srt", + "test/README.md", + "README.md", +] + +private_repo = "your-org/sglang-private-repo" +# --- Configuration End --- + + +def write_github_step_summary(content): + if not os.environ.get("GITHUB_STEP_SUMMARY"): + return + + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: + f.write(content) + + +def check_dependencies(): + """Check for required command-line tools.""" + if not shutil.which("git"): + raise EnvironmentError("git is not installed or not in PATH.") + if not shutil.which("gh"): + raise EnvironmentError("GitHub CLI (gh) is not installed or not in PATH.") + print("✅ All dependencies (git, gh) are available.") + + +def checkout_main(dry_run): + """Checkout to the main branch.""" + commands = [ + "git checkout main", + "git reset --hard", + ] + for cmd in commands: + print(f"Run: {cmd}") + if not dry_run: + try: + subprocess.run(cmd, shell=True, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + print(f"Git command failed: {e.stderr.decode()}") + raise + print("✅ Checkout the main branch.") + + +def get_source_folder(args): + """ + Prepare the source repository, either by cloning from GitHub or using a local directory. + Returns the path to the source repo root, a temporary directory path (if created), + and the short commit hash. + """ + temp_dir = None + if args.local_dir: + oss_root = os.path.expanduser(args.local_dir) + if not os.path.exists(oss_root): + raise FileNotFoundError( + f"Specified local directory {oss_root} does not exist." + ) + print(f"Using local directory as the source: {oss_root}") + else: + temp_dir = tempfile.mkdtemp() + oss_root = temp_dir + print(f"Created temporary directory: {oss_root}") + + repo_url = "https://github.com/sgl-project/sglang.git" + try: + subprocess.run( + [ + "git", + "clone", + "--single-branch", + "--branch", + "main", + repo_url, + temp_dir, + ], + check=True, + capture_output=True, + ) + print(f"Successfully cloned repository to {temp_dir}") + except subprocess.CalledProcessError as e: + print(f"Error cloning repository: {e.stderr.decode()}") + raise + + commit_hash = subprocess.run( + ["git", "-C", oss_root, "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip()[:8] + print(f"✅ Get source OSS code at commit: {commit_hash}") + return oss_root, temp_dir, commit_hash + + +def sync_directories(oss_root, folder_names, dry_run): + """Sync specified directories from oss_root to current working directory.""" + rsync_commands = [] + for folder_name in folder_names: + target_name = f"{oss_root}/{folder_name}" + src_name = "./" + "/".join(folder_name.split("/")[:-1]) + cmd = f"rsync -r --delete {target_name} {src_name}" + rsync_commands.append(cmd) + + for cmd in rsync_commands: + try: + print(f"Run: {cmd}") + if not dry_run: + subprocess.run(cmd, shell=True, check=True) + except subprocess.CalledProcessError as e: + print(f"Error executing command '{cmd}': {e}") + raise + print(f"✅ Sync all folders.") + + +def check_for_changes(): + """Check if there are any uncommitted git changes.""" + # This command exits with 1 if there are changes, 0 otherwise. + result = subprocess.run(["git", "diff", "--quiet"]) + return result.returncode != 0 + + +def create_and_push_branch(branch_name, commit_message, dry_run): + """Create a new branch, commit all changes, and push to origin.""" + commands = [ + f"git checkout -b {branch_name}", + "git config user.name 'github-actions[bot]'", + "git config user.email 'github-actions[bot]@users.noreply.github.com'", + "git add .", + f"git commit -m '{commit_message}'", + f"git push origin {branch_name} --force", + ] + print("\nCreating and pushing git branch...") + for cmd in commands: + print(f"Run: {cmd}") + if not dry_run: + try: + subprocess.run(cmd, shell=True, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + print(f"Git command failed: {e.stderr.decode()}") + raise + + +def create_pull_request(branch_name, title, body, dry_run): + """Create a pull request using the GitHub CLI.""" + gh_token = os.getenv("GH_TOKEN") + if not gh_token: + print( + "\n⚠️ Warning: GH_TOKEN environment variable not set. Skipping PR creation." + ) + if not dry_run: + return + + print("\nCreating pull request...") + command = [ + "gh", + "pr", + "create", + "--base", + "main", + "--head", + branch_name, + "--repo", + private_repo, + "--title", + title, + "--body", + body, + ] + print(f"Run: {' '.join(command)}") + if not dry_run: + env = os.environ.copy() + env["GH_TOKEN"] = gh_token + try: + result = subprocess.run( + command, check=True, capture_output=True, text=True, env=env + ) + pr_url = result.stdout.strip() + msg = f"✅ Successfully created pull request: {pr_url}" + print(msg) + write_github_step_summary(msg) + except subprocess.CalledProcessError as e: + print(f"Error creating pull request: {e.stderr}") + raise + + +def main(): + parser = argparse.ArgumentParser( + description="Copy code from OSS and open a PR if changes are detected." + ) + parser.add_argument( + "--local-dir", + type=str, + help="Path to local SGLang directory to use instead of cloning from GitHub.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Dry run the script without executing git, rsync, or gh commands.", + ) + args = parser.parse_args() + + check_dependencies() + checkout_main(args.dry_run) + + oss_root, temp_dir, oss_commit = get_source_folder(args) + + try: + # Sync directories + sync_directories(oss_root, folder_names, args.dry_run) + + # Check for changes and create PR if necessary + if not check_for_changes(): + msg = "😴 No changes detected. The code is already in sync." + print(msg) + write_github_step_summary(msg) + return + + print("✅ Changes detected. Proceeding to create a PR.") + + current_date = datetime.datetime.now().strftime("%Y%m%d") + branch_name = f"copy-from-oss-{oss_commit}-{current_date}" + commit_message = f"Copy OSS code from {oss_commit} on {current_date}" + pr_title = ( + f"[Automated PR] Copy OSS code from commit {oss_commit} on {current_date}" + ) + pr_body = ( + f"Copy OSS code from https://github.com/sgl-project/sglang/commit/{oss_commit} on {current_date}." + "\n\n---\n\n" + "*This is an automated PR created by scripts/copy_from_oss.py.*" + ) + + create_and_push_branch(branch_name, commit_message, args.dry_run) + create_pull_request(branch_name, pr_title, pr_body, args.dry_run) + + finally: + # Remove temporary directory if it was created + if temp_dir: + try: + shutil.rmtree(temp_dir) + print(f"\nRemoved temporary directory: {temp_dir}") + except OSError as e: + print(f"Error removing temporary directory {temp_dir}: {e}") + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/code_sync/copy_to_oss.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/code_sync/copy_to_oss.py new file mode 100644 index 00000000..74d3a66c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/code_sync/copy_to_oss.py @@ -0,0 +1,432 @@ +""" +Sync a specific commit from the local private repo to the OSS upstream and open a PR. + +NOTE: +1. You need to execute this script in the git root folder. +2. A GH_TOKEN environment variable is required to create the pull request. + - see also https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens + +This script will: +1. Take a commit hash as an argument (or use the latest commit by default). +2. Create a patch for that commit. +3. Filter the patch to only include changes in specified directories. +4. Clone the sgl-project/sglang repository. +5. Create a new branch in the OSS repo. +6. Apply the filtered patch, commit, and force push. +7. Open a pull request to the OSS repo using the GitHub CLI (gh). + +Usage: +# Sync the latest commit from the current branch +python3 scripts/copy_to_oss.py + +# Run the full sync and PR creation process for a given commit +python3 scripts/copy_to_oss.py --commit + +# Perform a dry run without making any actual changes +python3 scripts/copy_to_oss.py --commit --dry-run +""" + +import argparse +import datetime +import os +import shutil +import subprocess +import tempfile + +# --- Configuration Begin --- +# List of folders and files to copy to the OSS repo. +# Changes outside these paths will be ignored. +folder_names = [ + "3rdparty", + "assets", + "benchmark", + "docker", + "docs", + "examples", + "python/sglang/lang", + "python/sglang/jit_kernel", + "python/sglang/srt", + "python/sglang/test", + "python/sglang/utils.py", + "python/sglang/README.md", + "sgl-kernel", + "test/manual", + "test/nightly", + "test/registered", + "test/srt", + "test/README.md", + "README.md", +] + +# --- Configuration End --- + + +def write_github_step_summary(content): + if not os.environ.get("GITHUB_STEP_SUMMARY"): + return + + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: + f.write(content) + + +def get_commit_info(commit_ref): + """ + Retrieves the hash and message of a specific commit. + + Args: + commit_ref (str): The commit hash, tag, or branch to inspect (e.g., 'HEAD'). + + Returns: + A tuple containing the (commit_hash, commit_message), + or (None, None) if an error occurs. + """ + try: + # Use a custom format to get the hash (%H) and the full message (%B) + # separated by a null character for safe parsing. + command = ["git", "log", "-1", f"--pretty=%H%x00%B", commit_ref] + result = subprocess.run( + command, capture_output=True, text=True, check=True, encoding="utf-8" + ) + + # Split the output by the null character separator + commit_hash, commit_message = result.stdout.strip().split("\x00", 1) + return commit_hash, commit_message + + except FileNotFoundError: + print("❌ Error: 'git' command not found. Is Git installed and in your PATH?") + except subprocess.CalledProcessError as e: + print(f"❌ Error getting commit info for '{commit_ref}': {e.stderr.strip()}") + print( + "Hint: Make sure you are running this from within a Git repository and the commit exists." + ) + + return None, None + + +def check_dependencies(): + """Check for required command-line tools.""" + if not shutil.which("git"): + raise EnvironmentError("git is not installed or not in PATH.") + if not shutil.which("gh"): + raise EnvironmentError("GitHub CLI (gh) is not installed or not in PATH.") + print("✅ All dependencies (git, gh) are available.") + + +def create_filtered_patch(commit_hash, dry_run): + """ + Create a patch file for the given commit, containing only changes + to files and directories specified in `folder_names`. + """ + print(f"Creating a filtered patch for commit {commit_hash}") + + try: + # Get the list of all files changed in the commit + changed_files_raw = subprocess.run( + ["git", "diff-tree", "--no-commit-id", "--name-only", "-r", commit_hash], + capture_output=True, + text=True, + check=True, + ).stdout + changed_files = changed_files_raw.strip().split("\n") + + # Filter the list of files + relevant_files = [ + f for f in changed_files if any(f.startswith(path) for path in folder_names) + ] + + if not relevant_files: + msg = "\n😴 No relevant file changes found in this commit. Exiting." + print(msg) + write_github_step_summary(msg) + return None, None + + print("Found relevant changes in the following files:") + for f in relevant_files: + print(f" - {f}") + + # Create a patch containing only the changes for the relevant files + patch_command = [ + "git", + "format-patch", + "--stdout", + f"{commit_hash}^..{commit_hash}", + "--", + ] + relevant_files + + print(f"Run: {' '.join(patch_command)}") + + patch_content = subprocess.run( + patch_command, capture_output=True, text=True, check=True + ).stdout + + # Save the patch to a temporary file + patch_file = tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".patch", encoding="utf-8" + ) + patch_file.write(patch_content) + patch_file.close() + + print(f"✅ Filtered patch created successfully at: {patch_file.name}") + return patch_file.name, relevant_files + + except subprocess.CalledProcessError as e: + print(f"Error creating patch: {e.stderr}") + raise + + +def get_oss_repo(dry_run): + """ + Clones the OSS repository into a temporary directory. + Returns the path to the repo root and the temp directory itself. + """ + gh_token = os.getenv("GH_TOKEN") + if not gh_token: + print("⚠️ Warning: GH_TOKEN environment variable not set. Skipping PR creation.") + if not dry_run: + return + + temp_dir = tempfile.mkdtemp() + oss_root = os.path.join(temp_dir, "sglang") + print(f"\nCreated temporary directory for OSS repo: {temp_dir}") + + repo_url = f"https://{gh_token}@github.com/sgl-project/sglang.git" + command = ["git", "clone", "--branch", "main", repo_url, oss_root] + + print(f"Run: {' '.join(command)}") + if not dry_run: + try: + subprocess.run(command, check=True, capture_output=True) + print(f"✅ Successfully cloned repository to {oss_root}") + except subprocess.CalledProcessError as e: + print(f"Error cloning repository: {e.stderr.decode()}") + shutil.rmtree(temp_dir) + raise + + return oss_root, temp_dir + + +def apply_patch_and_push(oss_root, patch_file, branch_name, commit_message, dry_run): + """ + In the OSS repo, create a branch, apply the patch, commit, and push. + """ + print("\nApplying patch and pushing to OSS repo...") + + original_cwd = os.getcwd() + if not dry_run: + os.chdir(oss_root) + + try: + # Define commands as lists to avoid shell injection issues + commands_to_run = [ + ["git", "checkout", "-b", branch_name], + ["git", "apply", patch_file], + ["git", "config", "user.name", "github-actions[bot]"], + [ + "git", + "config", + "user.email", + "github-actions[bot]@users.noreply.github.com", + ], + ["git", "add", "."], + ] + + for cmd_list in commands_to_run: + print(f"Run: {' '.join(cmd_list)}") + if not dry_run: + subprocess.run(cmd_list, check=True, capture_output=True, text=True) + + # Handle commit separately to pass multi-line message safely via stdin + commit_cmd = ["git", "commit", "-F", "-"] + print(f"Run: {' '.join(commit_cmd)}") + if not dry_run: + print(f"Commit Message:\n---\n{commit_message}\n---") + subprocess.run( + commit_cmd, + input=commit_message, + text=True, + check=True, + capture_output=True, + ) + + # Push the changes + push_cmd = ["git", "push", "origin", branch_name, "--force"] + print(f"Run: {' '.join(push_cmd)}") + if not dry_run: + subprocess.run(push_cmd, check=True, capture_output=True, text=True) + + except subprocess.CalledProcessError as e: + print(f"Git command failed: {e.stderr}") + raise + finally: + if not dry_run: + os.chdir(original_cwd) + + print("✅ Branch created, patch applied, and pushed successfully.") + + +def create_pull_request(oss_root, branch_name, title, body, dry_run): + """Create a pull request in the OSS repo using the GitHub CLI.""" + gh_token = os.getenv("GH_TOKEN") + if not gh_token: + print("⚠️ Warning: GH_TOKEN environment variable not set. Skipping PR creation.") + if not dry_run: + return + + print("\nCreating pull request...") + command = [ + "gh", + "pr", + "create", + "--base", + "main", + "--head", + branch_name, + "--repo", + "sgl-project/sglang", + "--title", + title, + "--body", + body, + ] + + print(f"Run: {' '.join(command)}") + if not dry_run: + env = os.environ.copy() + env["GH_TOKEN"] = gh_token + try: + result = subprocess.run( + command, + check=True, + capture_output=True, + text=True, + env=env, + cwd=oss_root, + ) + msg = f"✅ Successfully created pull request: {result.stdout.strip()}" + print(msg) + write_github_step_summary(msg) + except subprocess.CalledProcessError as e: + print(f"Error creating pull request: {e.stderr}") + # Check if a PR already exists + if "A pull request for" in e.stderr and "already exists" in e.stderr: + print("ℹ️ A PR for this branch likely already exists.") + else: + raise + + +def get_commit_author(commit_hash): + """Get the author name and email of a commit.""" + try: + author_name = subprocess.run( + ["git", "show", "-s", "--format=%an", commit_hash], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + author_email = subprocess.run( + ["git", "show", "-s", "--format=%ae", commit_hash], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + return author_name, author_email + except subprocess.CalledProcessError as e: + print(f"Error getting commit author for {commit_hash}: {e.stderr}") + raise + + +def main(): + parser = argparse.ArgumentParser( + description="Copy a commit from the private repo to OSS and open a PR." + ) + parser.add_argument( + "--commit", + type=str, + default="LAST", + help="The commit hash to sync. Defaults to 'LAST' to use the latest commit.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Dry run the script without executing git, rsync, or gh commands.", + ) + args = parser.parse_args() + + check_dependencies() + + commit_ref = "HEAD" if args.commit == "LAST" else args.commit + commit_hash, original_commit_message = get_commit_info(commit_ref) + + if not commit_hash: + return # Exit if we couldn't get commit info + + # Display the details of the commit being processed + if args.commit == "LAST": + summary = ( + f"\nℹ️ No commit specified. Using the last commit:\n" + f" - **Hash:** `{commit_hash}`\n" + f" - **Message:** {original_commit_message}\n\n" + ) + else: + summary = ( + f"\nℹ️ Using specified commit:\n" + f" - **Hash:** `{commit_hash}`\n" + f" - **Message:** {original_commit_message}\n\n" + ) + print(summary) + write_github_step_summary(summary) + + short_hash = commit_hash[:8] + + patch_file = None + temp_dir = None + try: + # 1. Create a filtered patch from the local repo + patch_file, relevant_files = create_filtered_patch(commit_hash, args.dry_run) + if not patch_file: + return + + # 2. Get the OSS repo + oss_root, temp_dir = get_oss_repo(args.dry_run) + + # 3. Get original commit author for the co-author line + author_name, author_email = get_commit_author(commit_hash) + + # 4. Prepare content for the commit and PR based on changed files + file_list_str = "\n".join([f"- {f}" for f in relevant_files]) + filename_list_str = ", ".join([f.split("/")[-1] for f in relevant_files]) + if len(filename_list_str) > 40: + filename_list_str = filename_list_str[:40] + "..." + current_date = datetime.datetime.now().strftime("%Y%m%d") + pr_title = f"[Auto Sync] Update {filename_list_str} ({current_date})" + pr_body = ( + f"Sync changes from commit `{short_hash}`.\n\n" + f"**Files Changed:**\n{file_list_str}\n\n" + f"Author: {author_name} <{author_email}>" + f"\n\n---\n\n" + f"*This is an automated PR created by scripts/copy_from_oss.py.*" + ) + + # 5. Create branch, apply patch, and push + branch_name = f"sync-{short_hash}-{current_date}" + co_author_line = f"Co-authored-by: {author_name} <{author_email}>" + commit_message = f"{pr_title}\n\n{co_author_line}" + apply_patch_and_push( + oss_root, patch_file, branch_name, commit_message, args.dry_run + ) + + # 6. Create Pull Request + create_pull_request(oss_root, branch_name, pr_title, pr_body, args.dry_run) + + finally: + # Cleanup temporary files + if patch_file and os.path.exists(patch_file): + os.remove(patch_file) + print(f"\nRemoved temporary patch file: {patch_file}") + if temp_dir and os.path.exists(temp_dir): + shutil.rmtree(temp_dir) + print(f"Removed temporary directory: {temp_dir}") + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/code_sync/guideline.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/code_sync/guideline.md new file mode 100644 index 00000000..52f08eb4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/code_sync/guideline.md @@ -0,0 +1,27 @@ +### Sync Code Between OSS and Private Fork + +You can use the following principles and tools to sync the code between a private fork and the OSS repo [sgl-project/sglang](https://github.com/sgl-project/sglang/tree/main). +It learns from [Copybara](https://github.com/google/copybara), a tool used at Google for maintaining open-source code synchronization. + +## Principals + +- The core folders (e.g., `python/sglang/srt`) are 100% mirrored between the private fork and OSS repo. +- The OSS repo is the single source of truth. If one commit changes `python/sglang/srt` in the private repo, the change should be synced to the OSS repo as soon as possible with the action B below. +- The common code (e.g., base classes, well-known techniques in the industry without private secrets) goes to `python/sglang/srt`. The private-specific code (e.g., with private-specific features, confidential info) goes to `python/sglang/private` . +- Anytime you want to make private changes to a file or class under `python/sglang/srt`, duplicate the file and move it under `python/sglang/private`. You can achieve code reuse by importing and inheriting. + +## How to sync the code bidirectionally +### Action A: Copy code from OSS to private + +- We can run this action: [Open A PR to Copy Code From OSS](https://github.com/sgl-project/sglang/tree/main/.github/workflows/open-pr-copy-from-oss.yml) + - It opens a PR to copy all files under certain folders (e.g., `python/sglang/srt` , `test/srt` , `sgl-kernel` ) from the OSS main branch to the private fork. + - Since the OSS repo is the single source of truth, this action copies files and overwrites any changes in the private fork. To prevent the private changes from being overwritten, you need to ensure all private changes are merged into the OSS repo before running this action. +- This action will be run automatically every day and can also be triggered manually. + +### Action B: Copy diff from private to OSS + +- We can run this action: [Open A PR to Copy Code To OSS](https://github.com/sgl-project/sglang/tree/main/.github/workflows/open-pr-copy-to-oss.yml) + - It opens a PR to apply the diff of one specific commit of the private fork to the OSS main branch. It will only pick the changes under certain folders (e.g., `python/sglang/srt` , `test/srt` , `sgl-kernel` ) and ignore changes under private folders (e.g., `python/sglang/private` ) + - For example, you can have a PR that changes both `python/sglang/srt` and `python/sglang/private/srt`. Once you merge the PR into the private repo, `python/sglang/srt` becomes desynced between the two repos. You need to run this action on your merge commit immediately to open a PR to send your diff to the OSS repo. Then, we need to merge the OSS PR as soon as possible. Once your OSS PR is merged, we can run action A again. + - Action A copies files directly, but Action B applies diff. This is because OSS is the source of truth; action A can just copy files. Action B cannot copy, so it uses diff instead. +- This action currently needs a manual trigger in order to prevent incidental code leaks. One can also consider making it automatic. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/code_sync/install_github_cli.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/code_sync/install_github_cli.sh new file mode 100755 index 00000000..2ef1db02 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/code_sync/install_github_cli.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Check if gh is installed before attempting to install it +if ! command -v gh &> /dev/null +then +echo "GitHub CLI not found. Installing now..." +(type -p wget >/dev/null || ( apt update && apt install wget -y)) \ +&& mkdir -p -m 755 /etc/apt/keyrings \ +&& out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \ +&& cat $out | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ +&& chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ +&& mkdir -p -m 755 /etc/apt/sources.list.d \ +&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ +&& apt update \ +&& apt install gh -y +else +echo "GitHub CLI is already installed. Skipping installation." +fi diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/convert_otel_2_perfetto.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/convert_otel_2_perfetto.py new file mode 100644 index 00000000..42f1127b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/convert_otel_2_perfetto.py @@ -0,0 +1,375 @@ +import argparse +import bisect +import json +import time +from collections import defaultdict +from pathlib import Path +from typing import Any, Dict, Iterable, List, Tuple + +parser = argparse.ArgumentParser( + description="Convert SGLang OTEL trace files to Perfetto format.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) +parser.add_argument( + "-i", + "--input", + dest="input_file", + required=True, + type=str, + help="Path to the input OTEL trace file (JSON or JSONL format).", +) +parser.add_argument( + "-o", + "--output", + dest="output_file", + type=str, + default="sglang_trace_perfetto.json", + help="Path to the output Perfetto JSON file.", +) +parser.add_argument( + "-f", "--torch-file", dest="torch_file", help="specify torch profile file" +) + +args = parser.parse_args() + +perfetto_data = None +if args.torch_file: + with open(args.torch_file, "r", encoding="utf-8") as file: + perfetto_data = json.load(file) + baseline = perfetto_data["baseTimeNanoseconds"] +else: + baseline = 0 + + +def id_generator(): + i = 0 + while True: + yield i + i += 1 + + +relation_id_gen = id_generator() + + +class SpanLayoutContainer: + def __init__(self): + self.intervals = [] + + def check_overlap(self, start, end): + idx = bisect.bisect_left(self.intervals, (start, float("-inf"))) + + if idx > 0: + prev_start, prev_end = self.intervals[idx - 1] + if prev_end > start: + return True + + if idx < len(self.intervals): + next_start, next_end = self.intervals[idx] + if next_start < end: + return True + return False + + def insert_span(self, start, end): + bisect.insort_left(self.intervals, (start, end)) + + +def new_metadata_level1(name: str, pid): + return { + "name": "process_name", + "ph": "M", + "pid": pid, + "args": {"name": name}, + } + + +def new_metadata_level2(name: str, pid, slot_seq): + return { + "name": "thread_name", + "ph": "M", + "pid": pid, + "tid": slot_seq, + "args": {"name": name}, + } + + +def __find_line(graph, trans_graph_status, slot_meta_data, pid, start, end): + if pid in trans_graph_status: + line = trans_graph_status[pid] + if start == end: + return line + # check conflict + if not graph[pid][line].check_overlap(start, end): + return line + + if pid not in graph: + line = 1 + graph[pid] = {line: SpanLayoutContainer()} + trans_graph_status[pid] = line + slot_meta_data.append(new_metadata_level2("slot", pid, line)) + return line + + for line in graph[pid]: + if not graph[pid][line].check_overlap(start, end): + trans_graph_status[pid] = line + return line + + new_line = len(graph[pid]) + 1 + graph[pid][new_line] = SpanLayoutContainer() + trans_graph_status[pid] = new_line + slot_meta_data.append(new_metadata_level2("slot", pid, new_line)) + return new_line + + +OtelSpan = Dict[str, Any] + + +def load_otel_data(path: str | Path): + p = Path(path) + with p.open("rt", encoding="utf-8") as f: + first = f.read(1) + f.seek(0) + if first == "[": + data = json.load(f) # JSON array + else: + data = [json.loads(line) for line in f if line.strip()] # JSONL + return data + + +def extract_all_otel_spans(otel_data): + otel_spans = [] + for line_data in otel_data: + for resource_spans in line_data["resourceSpans"]: + for scope_spans in resource_spans["scopeSpans"]: + for span in scope_spans["spans"]: + if "attributes" in span: + attributes_dict = { + attr.get("key"): next( + iter(attr.get("value", {}).values()), None + ) + for attr in span["attributes"] + } + span["attributes"] = attributes_dict + else: + span["attributes"] = {} + otel_spans.append(span) + return otel_spans + + +def build_otel_span_tree(otel_spans): + span_id_map = {span["spanId"]: span for span in otel_spans} + for span in otel_spans: + span["child"] = [] + + bootstrap_room_spans = [] + + for span in otel_spans: + span_id = span["spanId"] + parent_span_id = span.get("parentSpanId", "") + if parent_span_id == "": + # check if root span is a request span + attrs = span.get("attributes", {}) + bootstrap_room_spans.append(span) + elif parent_span_id in span_id_map: + parent_span = span_id_map[parent_span_id] + parent_span["child"].append(span) + + link_spans = [] + if "links" in span: + for link in span["links"]: + link_span = span_id_map.get(link["spanId"]) + if link_span: + link_spans.append(link_span) + span["links"] = link_spans + + return bootstrap_room_spans + + +def generate_perfetto_span(otel_bootstrap_room_spans, thread_meta_data): + for bootstrap_room_span in otel_bootstrap_room_spans: + bootstrap_room = bootstrap_room_span["attributes"]["bootstrap_room"] + bootstrap_room_span["spans"] = [] + + for node_req_span in bootstrap_room_span["child"]: + rid = node_req_span["attributes"]["rid"] + + for thread_span in node_req_span["child"]: + pid = int(thread_span["attributes"]["pid"]) + thread_name = f'{thread_span["attributes"]["host_id"][:8]}:{thread_span["attributes"]["thread_label"]}' + if "tp_rank" in thread_span["attributes"]: + thread_name += f"-TP{thread_span['attributes']['tp_rank']}" + + if pid not in thread_meta_data: + thread_meta_data[pid] = new_metadata_level1(thread_name, pid) + + for span in thread_span["child"]: + span["attributes"]["bootstrap_room"] = bootstrap_room + span["attributes"]["rid"] = rid + span["host_id"] = thread_span["attributes"]["host_id"] + span["pid"] = pid + + span["startTimeUnixNano"] = int(span["startTimeUnixNano"]) + span["endTimeUnixNano"] = int(span["endTimeUnixNano"]) + ts = span["startTimeUnixNano"] + dur = span["endTimeUnixNano"] - ts + + perfetto_span = { + "ph": "X", + "name": span.get("name", "unknown"), + "cat": "sglang", + "ts": (ts - baseline) / 1000.0, + "dur": (dur - 1000) / 1000.0, + "pid": pid, + "tid": 0, + "args": span["attributes"], + } + + span["perfetto_span"] = perfetto_span + bootstrap_room_span["spans"].append(span) + + +def generate_perfetto_span_layout(otel_bootstrap_room_spans, slot_meta_data): + for bootstrap_room_span in otel_bootstrap_room_spans: + bootstrap_room_span["spans"] = sorted( + bootstrap_room_span["spans"], key=lambda x: int(x["startTimeUnixNano"]) + ) + + otel_bootstrap_room_spans = sorted( + otel_bootstrap_room_spans, key=lambda x: int(x["spans"][0]["startTimeUnixNano"]) + ) + graph = {} + for bootstrap_room_span in otel_bootstrap_room_spans: + req_thread_status = {} + for span in bootstrap_room_span["spans"]: + line = __find_line( + graph, + req_thread_status, + slot_meta_data, + span["perfetto_span"]["pid"], + span["startTimeUnixNano"], + span["endTimeUnixNano"], + ) + graph[span["perfetto_span"]["pid"]][line].insert_span( + span["startTimeUnixNano"], span["endTimeUnixNano"] + ) + span["perfetto_span"]["tid"] = line + + +def generate_perfetto_events(otel_bootstrap_room_spans): + for bootstrap_room_span in otel_bootstrap_room_spans: + for span in bootstrap_room_span["spans"]: + span["perfetto_events"] = [] + if "events" in span: + for event in span["events"]: + attributes_dict = { + attr.get("key"): next( + iter(attr.get("value", {}).values()), None + ) + for attr in event["attributes"] + } + perfetto_event = { + "ph": "i", + "cat": "sglang", + "ts": (int(event["timeUnixNano"]) - baseline) / 1000.0, + "pid": span["perfetto_span"]["pid"], + "tid": span["perfetto_span"]["tid"], + "name": event.get("name", "unknown"), + "args": attributes_dict, + } + + span["perfetto_events"].append(perfetto_event) + + +def generate_perfetto_links(otel_bootstrap_room_spans): + for bootstrap_room_span in otel_bootstrap_room_spans: + for span in bootstrap_room_span["spans"]: + span["perfetto_links"] = [] + if "links" in span: + for link_span in span["links"]: + if "correlation" in link_span["perfetto_span"]["args"]: + id = link_span["perfetto_span"]["args"]["correlation"] + else: + id = next(relation_id_gen) + link_span["perfetto_span"]["args"]["correlation"] = id + + perfetto_start_node = { + "ph": "s", + "id": id, + "pid": link_span["perfetto_span"]["pid"], + "tid": link_span["perfetto_span"]["tid"], + "ts": link_span["perfetto_span"]["ts"], + "cat": "ac2g", + "name": "ac2g", + } + + perfetto_end_node = { + "ph": "f", + "id": id, + "pid": span["perfetto_span"]["pid"], + "tid": span["perfetto_span"]["tid"], + "ts": span["perfetto_span"]["ts"], + "cat": "ac2g", + "name": "ac2g", + "bp": "e", + } + + span["perfetto_links"].append(perfetto_start_node) + span["perfetto_links"].append(perfetto_end_node) + + +def gather_all_perfetto_elems( + otel_bootstrap_room_spans, thread_meta_data, slot_meta_data +): + elems = [] + elems.extend(thread_meta_data.values()) + elems.extend(slot_meta_data) + for bootstrap_room_span in otel_bootstrap_room_spans: + for span in bootstrap_room_span["spans"]: + elems.append(span["perfetto_span"]) + elems.extend(span["perfetto_events"]) + elems.extend(span["perfetto_links"]) + + return elems + + +def write_json(perfetto_elems): + global perfetto_data + + if args.torch_file: + perfetto_data["traceEvents"].extend(perfetto_elems) + filered_data = [ + item + for item in perfetto_data["traceEvents"] + if item.get("cat") != "gpu_user_annotation" + ] + perfetto_data["traceEvents"] = filered_data + else: + perfetto_data = perfetto_elems + + with open(args.output_file, "w", encoding="utf-8") as file: + json.dump(perfetto_data, file, ensure_ascii=False, indent=4) + + +def main(): + start_time = time.time() + otel_data = load_otel_data(args.input_file) + otel_spans = extract_all_otel_spans(otel_data) + otel_bootstrap_room_spans = build_otel_span_tree(otel_spans) + thread_meta_data = {} + generate_perfetto_span(otel_bootstrap_room_spans, thread_meta_data) + slot_meta_data = [] + generate_perfetto_span_layout(otel_bootstrap_room_spans, slot_meta_data) + generate_perfetto_events(otel_bootstrap_room_spans) + generate_perfetto_links(otel_bootstrap_room_spans) + perfetto_elems = gather_all_perfetto_elems( + otel_bootstrap_room_spans, thread_meta_data, slot_meta_data + ) + write_json(perfetto_elems) + end_time = time.time() + execution_time = end_time - start_time + print(f"\nConversion finished successfully!") + print(f"Output written to: {args.output_file}") + print(f"Execution time: {execution_time * 1000:.4f} ms") + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ensure_vram_clear.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ensure_vram_clear.sh new file mode 100755 index 00000000..0dd72096 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/ensure_vram_clear.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +# Source the VRAM checking function +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/check_vram_clear.sh" + +ensure_vram_clear() { + local max_retries=3 + local retry_count=0 + + # Stop and remove any existing ci_sglang container + echo "Stopping any existing ci_sglang container..." + docker stop ci_sglang || true + docker rm ci_sglang || true + + # Log host information for debugging + echo "=== Host Information ===" + echo "Hostname: $(hostname)" + echo "Host IP: $(hostname -I 2>/dev/null || echo 'N/A')" + echo "Date: $(date)" + echo "Mode: rocm" + echo "========================" + echo "Running in ROCm mode" + + # Show initial GPU status + echo "=== Initial GPU Memory Status ===" + rocm-smi --showmemuse + echo "==================================" + + while [ $retry_count -lt $max_retries ]; do + echo "=== Cleanup Attempt $((retry_count + 1))/$max_retries ===" + + # Clean SGLang processes + echo "Killing SGLang processes..." + pgrep -f 'sglang::|sglang\.launch_server|sglang\.bench|sglang\.data_parallel|sglang\.srt' | xargs -r kill -9 || true + + if [ $retry_count -gt 0 ]; then + echo "Performing aggressive cleanup..." + # Kill all processes using KFD + rocm-smi --showpids 2>/dev/null | grep 'PID:' | awk '{print $2}' | xargs -r kill -9 2>/dev/null || true + # Wait a bit for cleanup to take effect + echo "Waiting 30 seconds for VRAM to clear..." + sleep 30 + fi + + # Check VRAM + echo "Checking VRAM status..." + if check_vram_clear; then + echo "✓ VRAM cleanup successful after $((retry_count + 1)) attempts" + return 0 + else + echo "✗ VRAM still not clear after attempt $((retry_count + 1))" + retry_count=$((retry_count + 1)) + fi + done + + # Failed after all retries + echo "=== FAILED: VRAM cleanup unsuccessful after $max_retries attempts ===" + echo "Final GPU status:" + timeout 30 rocm-smi --showmemuse || echo "rocm-smi timed out" + echo "Processes using GPU:" + rocm-smi --showpids 2>/dev/null | grep -q 'PID:' || echo "No processes found using /dev/kfd" + + # Print detailed information about suspicious processes + echo "=== Detailed Process Information ===" + if command -v rocm-smi >/dev/null 2>&1; then + # For AMD GPUs, get processes from rocm-smi --showpids + kfd_pids=$(rocm-smi --showpids 2>/dev/null | grep 'PID:' | awk '{print $2}' | sort -u) + if [ -n "$kfd_pids" ]; then + echo "Processes accessing /dev/kfd (AMD GPU device):" + for pid in $kfd_pids; do + if ps -p $pid -o pid,ppid,cmd --no-headers 2>/dev/null; then + echo " └─ Command line: $(ps -p $pid -o cmd --no-headers 2>/dev/null | head -1)" + else + echo " └─ PID $pid: Process not found or already terminated" + fi + done + else + echo "No processes found accessing /dev/kfd" + fi + fi + + # Check for any remaining sglang-related processes + echo "Checking for any remaining sglang-related processes:" + sglang_procs=$(pgrep -f 'sglang::|sglang\.launch_server|sglang\.bench|sglang\.data_parallel|sglang\.srt' 2>/dev/null) + if [ -n "$sglang_procs" ]; then + echo "Found sglang processes still running:" + for pid in $sglang_procs; do + ps -p $pid -o pid,ppid,cmd --no-headers 2>/dev/null || echo "PID $pid not found" + done + else + echo "No sglang-related processes found." + fi + + echo "==================================================================" + return 1 +} + +# If this script is run directly (not sourced), run the ensure function +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + set -e + ensure_vram_clear "$@" +fi diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/export_deepseek_nextn.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/export_deepseek_nextn.py new file mode 100644 index 00000000..5da0e4bc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/export_deepseek_nextn.py @@ -0,0 +1,115 @@ +""" +Export NextN layer for DeepSeek-V3/R1 model. The exported model can be used for speculative decoding. + +Usage: +python3 export_deepseek_nextn.py --input-dir /path/to/DeepSeek-V3 --output-dir /path/to/DeepSeek-V3-NextN +""" + +import argparse +import json +import os +import shutil + +from safetensors import safe_open +from safetensors.torch import save_file +from transformers import AutoConfig + + +def get_nextn_layer_id(config): + if not hasattr(config, "num_hidden_layers"): + raise ValueError("'num_hidden_layers' not found in model config.") + return config.num_hidden_layers + + +def update_and_save_config(config, output_dir): + new_config = config.to_dict() + new_config.update( + { + "num_hidden_layers": 1, + "architectures": ["DeepseekV3ForCausalLMNextN"], + } + ) + with open(os.path.join(output_dir, "config.json"), "w") as f: + json.dump(new_config, f, indent=2, ensure_ascii=False, sort_keys=True) + + +def copy_non_safetensors_files(input_dir, output_dir): + for filename in os.listdir(input_dir): + src_file_path = os.path.join(input_dir, filename) + if os.path.isfile(src_file_path) and not filename.endswith(".safetensors"): + dst_file_path = os.path.join(output_dir, filename) + shutil.copy2(src_file_path, dst_file_path) + print(f"All non-safetensors files have been copied to {output_dir}") + + +def export_nextn_layer_parameters(input_dir, output_dir, nextn_layer_id): + prefix = f"model.layers.{nextn_layer_id}" + output_path = os.path.join(output_dir, "nextn_layer_parameters.safetensors") + params = {} + for filename in os.listdir(input_dir): + if not filename.endswith(".safetensors"): + continue + + file_path = os.path.join(input_dir, filename) + print(f"Processing: {filename}") + + try: + with safe_open(file_path, framework="pt") as f: + matching_keys = [k for k in f.keys() if k.startswith(prefix)] + + if not matching_keys: + print(f" No parameters starting with '{prefix}' found") + continue + + for key in matching_keys: + if "embed_tokens" in key or "shared_head.head" in key: + continue + new_key = key.replace(prefix, "model.layers.0") + params[new_key] = f.get_tensor(key) + + except Exception as e: + print(f" Error processing {filename}: {str(e)}") + + if params: + print(f"Saving {len(params)} parameters to {output_path}") + save_file(params, output_path) + else: + print("No matching parameters found.") + + # Update safetensors index + index_path = os.path.join(output_dir, "model.safetensors.index.json") + print(f"Updating safetensors index to {index_path}") + index_data = {"weight_map": {}} + for key in params: + index_data["weight_map"][key] = "nextn_layer_parameters.safetensors" + with open(index_path, "w") as f: + json.dump(index_data, f, indent=4) + + print("All done.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Export NextN layer parameters for DeepSeek-V3/R1" + ) + parser.add_argument( + "--input-dir", + type=str, + required=True, + help="Input HF model directory.", + ) + parser.add_argument( + "--output-dir", + type=str, + required=True, + help="Output nextn model directory.", + ) + args = parser.parse_args() + + config = AutoConfig.from_pretrained(args.input_dir, trust_remote_code=True) + assert config.num_nextn_predict_layers == 1, "Only 1 nextn layer is supported." + nextn_layer_id = get_nextn_layer_id(config) + os.makedirs(args.output_dir, exist_ok=True) + copy_non_safetensors_files(args.input_dir, args.output_dir) + update_and_save_config(config, args.output_dir) + export_nextn_layer_parameters(args.input_dir, args.output_dir, nextn_layer_id) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/killall_sglang.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/killall_sglang.sh new file mode 100755 index 00000000..e637aaee --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/killall_sglang.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +if [ "$1" = "rocm" ]; then + echo "Running in ROCm mode" + + # Clean SGLang processes + pgrep -f 'sglang::|sglang\.launch_server|sglang\.bench|sglang\.data_parallel|sglang\.srt|sgl_diffusion::' | xargs -r kill -9 + +else + # Show current GPU status + nvidia-smi + + # Clean SGLang processes + pgrep -f 'sglang::|sglang\.launch_server|sglang\.bench|sglang\.data_parallel|sglang\.srt|sgl_diffusion::' | xargs -r kill -9 + + # Clean all GPU processes if any argument is provided + if [ $# -gt 0 ]; then + # Check if sudo is available + if command -v sudo >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y lsof + else + apt-get update + apt-get install -y lsof + fi + kill -9 $(nvidia-smi | sed -n '/Processes:/,$p' | grep " [0-9]" | awk '{print $5}') 2>/dev/null + lsof /dev/nvidia* | awk '{print $2}' | xargs kill -9 2>/dev/null + fi + + # Show GPU status after clean up + nvidia-smi +fi diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/bench_speculative.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/bench_speculative.py new file mode 100644 index 00000000..a67ed0c3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/bench_speculative.py @@ -0,0 +1,320 @@ +""" +Usage: +# single GPU +python3 bench_speculative.py --model-path meta-llama/Llama-2-7b-chat-hf --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B + +# multiple GPU +python3 bench_speculative.py --model-path deepseek-ai/DeepSeek-V3 --speculative-draft-model-path lmsys/DeepSeek-V3-NextN --tp-size 8 --trust-remote-code --batch-size 1 4 8 16 32 --steps 0 1 2 --topk 0 1 2 4 --num_draft_tokens 0 2 4 8 +""" + +import argparse +import asyncio +import json +import os +import time +from types import SimpleNamespace +from typing import List + +import numpy as np +import requests +from transformers import AutoTokenizer + +from sglang.bench_serving import ( + DatasetRow, + benchmark, + sample_mmmu_requests, + set_global_args, +) +from sglang.srt.server_args import ServerArgs +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + kill_process_tree, + popen_launch_server, +) + + +def node0_print(msg): + if server_args.node_rank == 0: + print(msg) + + +prompts = [ + "Human: Give me a fully functional FastAPI server. Show the full, long python code without stop.\n\nAssistant:", + "Human: Imagine you are an experienced Ethereum developer tasked with creating a smart contract for a blockchain messenger. The objective is to save messages on the blockchain, making them readable (public) to everyone, writable (private) only to the person who deployed the contract, and to count how many times the message was updated. Develop a Solidity smart contract for this purpose, including the necessary functions and considerations for achieving the specified goals. Please provide the code and any relevant explanations to ensure a clear understanding of the implementation.\n\nAssistant:", + "Human: Write a travel blog post to Hawaii.\n\nAssistant:", + "Human: I want you to act as an English translator, spelling corrector and improver. I will speak to you in any language and you will detect the language, translate it and answer in the corrected and improved version of my text, in English. I want you to replace my simplified A0-level words and sentences with more beautiful and elegant, upper level English words and sentences. Keep the meaning same, but make them more literary. My first sentence is 'istanbulu cok seviyom burada olmak cok guzel'. Answer in more than 5000 words.\n\nAssistant:", + "Human: I want you to act as a storyteller. You will come up with entertaining stories that are engaging, imaginative and captivating for the audience. It can be fairy tales, educational stories or any other type of stories which has the potential to capture people's attention and imagination. Depending on the target audience, you may choose specific themes or topics for your storytelling session e.g., if it’s children then you can talk about animals; If it’s adults then history-based tales might engage them better etc. Answer in more than 5000 words. My first request is 'I need an interesting story on perseverance.'\n\nAssistant:", + "Human: Solve x^2 = -1. Think step-by-step. Give me a long detailed explanation. \n\nAssistant:", + "Human: Tell me about the president of the USA in wikipedia style.\n\nAssistant:", + "Human: Hello? Who are you? Write code, math, and poem to explanin yourself.\n\nAssistant:", +] + + +class FakeTokenizer: + def encode(self, text: str, add_special_tokens: bool = False): + return [] + + +def send_one_batch(base_url, num_prompts, batch_size, processor, is_multimodal): + # format: (prompt, input_len, output len). We set input_len as a dummy value 0. + if is_multimodal: + backend = "sglang-oai-chat" + api_url = f"{base_url}/v1/chat/completions" + input_requests = sample_mmmu_requests( + num_prompts, + processor, + backend=backend, + fixed_output_len=512, + ) + tokenizer = processor.tokenizer + else: + padded_prompts = (prompts * ((num_prompts + len(prompts) - 1) // len(prompts)))[ + :num_prompts + ] + input_requests: List[DatasetRow] = [ + DatasetRow(p, 0, 512) for p in padded_prompts + ] + backend = "sglang" + api_url = f"{base_url}/generate" + tokenizer = processor + + # We need to set some dummy values in order to call `benchmark` below. + args = SimpleNamespace( + disable_ignore_eos=False, + disable_stream=False, + return_logprob=False, + backend=backend, + dataset_name="custom", + num_prompts=None, + sharegpt_output_len=None, + random_input_len=None, + random_output_len=None, + random_range_ratio=None, + output_file=None, + warmup_requests=1, + output_details=False, + ) + set_global_args(args) + + # Run benchmark + results = asyncio.run( + benchmark( + backend=backend, + api_url=api_url, + base_url=base_url, + model_id="default", + tokenizer=tokenizer, + input_requests=input_requests, + request_rate=float("inf"), + max_concurrency=batch_size, + disable_tqdm=False, + lora_names=None, + lora_request_distribution=None, + lora_zipf_alpha=None, + extra_request_body={}, + profile=None, + ) + ) + + assert results["completed"] == len(input_requests) + acc_length = results["accept_length"] or 1.0 + avg_output_token = results["total_output_tokens"] / results["completed"] + + server_info = requests.get(base_url + "/get_server_info").json() + # We use 20% percentile instead of median on purpose + step_time = np.percentile( + server_info["internal_states"][0]["step_time_dict"][str(batch_size)], 20 + ) + speed = 1 / step_time * acc_length + + return ( + round(acc_length, 3), + round(step_time, 5), + round(speed, 3), + avg_output_token, + ) + + +def main(args, server_args): + base_url = "http://127.0.0.1:20000" + + configs = [] + for batch_size in args.batch_size: + for steps in args.steps: + for topk in args.topk: + for num_draft_tokens in args.num_draft_tokens: + if steps * topk + 1 < num_draft_tokens: + continue + + if (steps == 0 or topk == 0 or num_draft_tokens == 0) and ( + steps + topk + num_draft_tokens != 0 + ): + # steps == 0 and topk == 0 and num_draft_tokens == 0 is a special case for non-speculative decoding. + continue + + configs.append((batch_size, steps, topk, num_draft_tokens)) + + for i in range(args.start, args.end or len(configs)): + batch_size, steps, topk, num_draft_tokens = configs[i] + + node0_print( + f"Start {i=}: {batch_size=}, {steps=}, {topk=}, {num_draft_tokens=}" + ) + + # Create an LLM. + if steps == 0: + other_args = [] + else: + other_args = [ + "--speculative-num-steps", + steps, + "--speculative-eagle-topk", + topk, + "--speculative-num-draft-tokens", + num_draft_tokens, + ] + if server_args.speculative_draft_model_path is not None: + other_args.extend( + [ + "--speculative-draft-model-path", + server_args.speculative_draft_model_path, + "--speculative-algorithm", + server_args.speculative_algorithm, + ] + ) + + other_args.extend( + [ + "--cuda-graph-max-bs", + batch_size, + "--mem-fraction-static", + server_args.mem_fraction_static, + "--tp-size", + server_args.tp_size, + "--max-running-requests", + batch_size, + ] + ) + + if server_args.trust_remote_code: + other_args.extend( + [ + "--trust-remote-code", + ] + ) + + if server_args.attention_backend: + other_args.extend( + [ + "--attention-backend", + server_args.attention_backend, + ] + ) + + if server_args.quantization: + other_args.extend( + [ + "--quantization", + server_args.quantization, + ] + ) + + process = popen_launch_server( + args.model_path, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + env={ + "SGLANG_RECORD_STEP_TIME": "1", + **os.environ, + }, + ) + + if args.is_multimodal: + from transformers import AutoProcessor + + processor = AutoProcessor.from_pretrained( + args.model_path, trust_remote_code=server_args.trust_remote_code + ) + else: + processor = AutoTokenizer.from_pretrained( + args.model_path, trust_remote_code=server_args.trust_remote_code + ) + + try: + # Warmup + send_one_batch( + base_url, batch_size, batch_size, processor, args.is_multimodal + ) + + # Benchmark + acc_length, step_time, speed, completion_tokens = send_one_batch( + base_url, + max(args.num_prompts, batch_size), + batch_size, + processor, + args.is_multimodal, + ) + finally: + kill_process_tree(process.pid) + + node0_print( + f"Finish {i=}: {batch_size=}, {steps=}, {topk=}, {num_draft_tokens=}, {speed=:.2f} token/s, step_time={step_time * 1000:.2f} ms" + ) + + record = { + "batch_size": batch_size, + "steps": steps, + "topk": topk, + "num_draft_tokens": num_draft_tokens, + "acc_length": acc_length, + "step_time": step_time, + "speed": speed, + "completion_tokens": completion_tokens, + } + + with open(args.output, "a") as fout: + fout.write(json.dumps(record) + "\n") + + # Wait for the server to shutdown + time.sleep(5) + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + parser.add_argument( + "--batch-size", + type=int, + nargs="+", + default=(1, 2, 4, 8, 16), + ) + parser.add_argument( + "--steps", + type=int, + nargs="+", + default=(0, 1, 3, 5, 7), # use (0, 1, 2, 3, 4) for large batch size + ) + parser.add_argument( + "--topk", + type=int, + nargs="+", + default=(0, 1, 2, 4, 8), + ) + parser.add_argument( + "--num_draft_tokens", + type=int, + nargs="+", + default=(0, 2, 4, 8, 16, 32), # use (0, 2, 4, 8) for large batch size + ) + parser.add_argument("--num-prompts", type=int, default=16) + parser.add_argument("--start", type=int, default=0) + parser.add_argument("--end", type=int) + parser.add_argument("--output", type=str, default="output.jsonl") + parser.add_argument("--is-multimodal", action="store_true", default=False) + args = parser.parse_args() + server_args: ServerArgs = ServerArgs.from_cli_args(args) + + main(args, server_args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/disaggregation/cli-logprob.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/disaggregation/cli-logprob.py new file mode 100644 index 00000000..4c69a055 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/disaggregation/cli-logprob.py @@ -0,0 +1,22 @@ +prompt = "The capital of france is " + +import json + +import requests + +response = requests.post( + "http://0.0.0.0:8000/generate", + json={ + "text": prompt, + "sampling_params": {"temperature": 0}, + "return_logprob": True, + "return_input_logprob": True, + "logprob_start_len": 0, + }, +) + +j = response.json() +input_logprobs = j["meta_info"]["input_token_logprobs"] +output_logprobs = j["meta_info"]["output_token_logprobs"] + +print(len(input_logprobs), len(output_logprobs)) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/disaggregation/cli-so.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/disaggregation/cli-so.py new file mode 100644 index 00000000..7ccafc7e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/disaggregation/cli-so.py @@ -0,0 +1,34 @@ +import json + +import requests + +port = 8000 + +json_schema = json.dumps( + { + "type": "object", + "properties": { + "name": {"type": "string", "pattern": "^[\\w]+$"}, + "population": {"type": "integer"}, + }, + "required": ["name", "population"], + } +) + +# JSON +response = requests.post( + f"http://localhost:{port}/generate", + json={ + "text": "Here is the information of the capital of France in the JSON format.\n", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 64, + "json_schema": json_schema, + }, + }, +) + +print(response.json()) + + +# python3 -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --trust-remote-code --disaggregation-mode prefill --tp 2 --disaggregation-ib-device mlx5_roce0,mlx5_roce1 --speculative-algorithm EAGLE --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B --speculative-num-steps 3 --speculative-eagle-topk 4 --speculative-num-draft-tokens 16 --cuda-graph-max-bs 8 --host 127.0.0.1 --port 8100 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/disaggregation/cli.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/disaggregation/cli.py new file mode 100644 index 00000000..721a6dd5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/disaggregation/cli.py @@ -0,0 +1,29 @@ +import json + +import requests + +prompt = """ +According to CNBC's Faber, the investors present on the call interpreted this statement as an indication of an upcoming funding round. While speculative, Faber believes the funding round could be as large as $25 billion, and bestow a valuation of between $150 billion and $200 billion on xAI. + +For the benefit of those who might not be aware, xAI recently acquired the social media platform X in an all-stock deal that valued the former at $80 billion and the latter at $33 billion, inclusive of $12 billion in liabilities. This meant that the deal bestowed a gross valuation of $45 billion on X before factoring in its debt load of $12 billion. + +Bear in mind that Elon Musk took X (then called Twitter) private back in 2022 in a $44 billion deal. Since then, Musk has managed to stem X's cash bleed, with the company reportedly generating $1.2 billion in adjusted EBITDA in 2024. + +According to the investors present on the call, xAI is currently generating around $1 billion in annual revenue. This contrasts sharply with the erstwhile muted expectations of many investors, who did not expect the startup to generate any material revenue this year. + +Elsewhere, Faber also alludes to the fact that xAI is already working on its next big training supercluster, officially dubbed the Colossus 2, which is expected to eventually house as many as 1 million NVIDIA GPUs at a cost of between $35 billion and $40 billion. + + +Even though xAI's Grok LLM is already largely comparable with OpenAI's cutting-edge models, the Colossus 2 would significantly up the ante, and could feasibly challenge OpenAI's apex position in the AI sphere. + +Give your honest take on the above text: +""" + +response = requests.post( + "http://0.0.0.0:8000/generate", + json={"text": prompt, "sampling_params": {"temperature": 0}}, +) + + +response_json = response.json() +print(response_json["text"]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/frontend_reasoning.ipynb b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/frontend_reasoning.ipynb new file mode 100644 index 00000000..fcdce25a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/frontend_reasoning.ipynb @@ -0,0 +1,241 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Launch A Server\n", + "\n", + "Launch the server with a reasoning model (Qwen 3.5-4B) and reasoning parser." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sglang import separate_reasoning, assistant_begin, assistant_end\n", + "from sglang import assistant, function, gen, system, user\n", + "from sglang import image\n", + "from sglang import RuntimeEndpoint, set_default_backend\n", + "from sglang.srt.utils import load_image\n", + "from sglang.test.test_utils import is_in_ci\n", + "from sglang.utils import print_highlight, terminate_process, wait_for_server\n", + "\n", + "\n", + "if is_in_ci():\n", + " from patch import launch_server_cmd\n", + "else:\n", + " from sglang.utils import launch_server_cmd\n", + "\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"python3 -m sglang.launch_server --model-path Qwen/Qwen3-4B --reasoning-parser qwen3 --host 0.0.0.0\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")\n", + "print(f\"Server started on http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Set the default backend. Note: you can set chat_template_name in RontimeEndpoint. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "set_default_backend(\n", + " RuntimeEndpoint(f\"http://localhost:{port}\", chat_template_name=\"qwen\")\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's start with a basic question-answering task. And see how the reasoning content is generated." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def basic_qa(s, question):\n", + " s += system(f\"You are a helpful assistant than can answer questions.\")\n", + " s += user(question)\n", + " s += assistant_begin()\n", + " s += gen(\"answer\", max_tokens=512)\n", + " s += assistant_end()\n", + "\n", + "\n", + "state = basic_qa(\"List 3 countries and their capitals.\")\n", + "print_highlight(state[\"answer\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With `separate_reasoning`, you can move the reasoning content to `{param_name}_reasoning_content` in the state." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def basic_qa_separate_reasoning(s, question):\n", + " s += system(f\"You are a helpful assistant than can answer questions.\")\n", + " s += user(question)\n", + " s += assistant_begin()\n", + " s += separate_reasoning(gen(\"answer\", max_tokens=512), model_type=\"qwen3\")\n", + " s += assistant_end()\n", + "\n", + "\n", + "reasoning_state = basic_qa_separate_reasoning(\"List 3 countries and their capitals.\")\n", + "print_highlight(reasoning_state.stream_executor.variable_event.keys())\n", + "print_highlight(\n", + " f\"\\nSeparated Reasoning Content:\\n{reasoning_state['answer_reasoning_content']}\"\n", + ")\n", + "\n", + "print_highlight(f\"\\n\\nContent:\\n{reasoning_state['answer']}\")\n", + "print_highlight(f\"\\n\\nMessages:\\n{reasoning_state.messages()[-1]}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`separate_reasoning` can also be used in multi-turn conversations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def multi_turn_qa(s):\n", + " s += system(f\"You are a helpful assistant than can answer questions.\")\n", + " s += user(\"Please give me a list of 3 countries and their capitals.\")\n", + " s += assistant(\n", + " separate_reasoning(gen(\"first_answer\", max_tokens=512), model_type=\"qwen3\")\n", + " )\n", + " s += user(\"Please give me another list of 3 countries and their capitals.\")\n", + " s += assistant(\n", + " separate_reasoning(gen(\"second_answer\", max_tokens=512), model_type=\"qwen3\")\n", + " )\n", + " return s\n", + "\n", + "\n", + "reasoning_state = multi_turn_qa()\n", + "print_highlight(f\"\\n\\nfirst_answer:\\n{reasoning_state['first_answer']}\")\n", + "print_highlight(\n", + " f\"\\n\\nfirst_answer_reasoning_content:\\n{reasoning_state['first_answer_reasoning_content']}\"\n", + ")\n", + "print_highlight(f\"\\n\\nsecond_answer:\\n{reasoning_state['second_answer']}\")\n", + "print_highlight(\n", + " f\"\\n\\nsecond_answer_reasoning_content:\\n{reasoning_state['second_answer_reasoning_content']}\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using No thinking as Qwen 3's advanced feature \n", + "\n", + "sglang separate_reasoning is particularly useful when combined with Qwen 3's advanced feature.\n", + "\n", + "[Qwen 3's advanced usages](https://qwenlm.github.io/blog/qwen3/#advanced-usages)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "reasoning_state = basic_qa_separate_reasoning(\n", + " \"List 3 countries and their capitals. /no_think\"\n", + ")\n", + "print_highlight(f\"Reasoning Content:\\n{reasoning_state['answer_reasoning_content']}\")\n", + "print_highlight(f\"Content:\\n{reasoning_state['answer']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`separate_reasoning` can also be used in regular expression generation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def regular_expression_gen(s):\n", + " s += user(\n", + " \"What is the IP address of the Google DNS servers? just provide the answer\"\n", + " )\n", + " s += assistant(\n", + " separate_reasoning(\n", + " gen(\n", + " \"answer\",\n", + " temperature=0,\n", + " regex=r\"((25[0-5]|2[0-4]\\d|[01]?\\d\\d?).){3}(25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\",\n", + " max_tokens=512,\n", + " ),\n", + " model_type=\"qwen3\",\n", + " ),\n", + " )\n", + "\n", + "\n", + "reasoning_state = regular_expression_gen()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print_highlight(f\"Answer:\\n{reasoning_state['answer']}\")\n", + "print_highlight(\n", + " f\"\\n\\nReasoning Content:\\n{reasoning_state['answer_reasoning_content']}\"\n", + ")" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/load_tokenizer.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/load_tokenizer.py new file mode 100644 index 00000000..6fccc256 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/load_tokenizer.py @@ -0,0 +1,14 @@ +import argparse +import code + +from sglang.srt.utils.hf_transformers_utils import get_tokenizer + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--name", type=str, default="meta-llama/Meta-Llama-3-8B-Instruct" + ) + args = parser.parse_args() + + t = get_tokenizer(args.name) + code.interact(local=locals()) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/long_context_example.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/long_context_example.py new file mode 100644 index 00000000..c5e035d2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/long_context_example.py @@ -0,0 +1,36 @@ +from urllib.request import urlopen + +from openai import OpenAI + +test_cases = { + "64k": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-1M/test-data/64k.txt", + "200k": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-1M/test-data/200k.txt", + "600k": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-1M/test-data/600k.txt", + "1m": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-1M/test-data/1m.txt", +} + +client = OpenAI(api_key="EMPTY", base_url="http://127.0.0.1:30000/v1") + +for name, url in test_cases.items(): + print(f"\n==== Running test case: {name} ====") + try: + with urlopen(url, timeout=10) as response: + prompt = response.read().decode("utf-8") + except Exception as e: + print(f"Failed to load prompt for {name}: {e}") + continue + + try: + response = client.chat.completions.create( + model="meta-llama/Llama-4-Scout-17B-16E-Instruct", + messages=[{"role": "user", "content": prompt}], + stream=True, + max_tokens=128, + temperature=0, + ) + + for chunk in response: + if chunk.choices and chunk.choices[0].delta.content is not None: + print(chunk.choices[0].delta.content, end="", flush=True) + except Exception as e: + print(f"\nError during completion for {name}: {e}") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/lora/analyzer.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/lora/analyzer.py new file mode 100644 index 00000000..15568fc1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/lora/analyzer.py @@ -0,0 +1,77 @@ +import glob +import json +import os +import re +import sys + +from tqdm import tqdm + +sys.path.append("../../") +from fix_corrupted_json import clean_json_file + +dirpath = "/Users/ying" +output_file_prefix = "analyzed_log" + +time = {} +tot_time = {} +size = {} + +os.system(f"rm {output_file_prefix}*") + +for dirname in glob.glob(os.path.join(dirpath, "trace*")): + print(dirname) + trace_name = dirname.split("/")[-1] + time[trace_name] = {} + size[trace_name] = {} + total_time = 0 + for filename in tqdm(glob.glob(os.path.join(dirname, "*.json"))): + step_name = filename.split("/")[-1].split(".")[0] + step_name = "_".join(step_name.split("_")[1:]) + if "prefill" not in filename and "decode" not in filename: + continue + + match = re.search(r"(prefill|decode)_step_(\d+)\.json", filename) + if match: + phase = match.group(1) + step = match.group(2) + else: + raise Exception(f"Cannot parse {filename}") + + try: + with open(filename, "r") as f: + trace = json.load(f) + except: + clean_json_file(filename, filename) + with open(filename, "r") as f: + trace = json.load(f) + + for event in trace["traceEvents"]: + name = event["name"] + if name in ["profile_prefill_step", "profile_decode_step"]: + dur = event["dur"] / 1e3 + time[trace_name][step_name] = dur + break + total_time += dur + + step = int(step_name.split("_")[-1]) + with open(os.path.join(dirname, f"size_{step}.json"), "r") as f: + size_info = json.load(f) + size[trace_name][step_name] = size_info["size"] + + tot_time[trace_name] = total_time + time[trace_name] = dict( + sorted(time[trace_name].items(), key=lambda x: int(x[0].split("_")[-1])) + ) + size[trace_name] = dict( + sorted(size[trace_name].items(), key=lambda x: int(x[0].split("_")[-1])) + ) + + with open(f"{output_file_prefix}_{trace_name}", "a") as f: + for k, v in time[trace_name].items(): + size_v = size[trace_name][k] + print(f"{k:>15}{v:10.2f}\t{size_v}") + f.write(f"{k:>15}{v:10.2f}\t{size_v}\n") + +with open(f"{output_file_prefix}_total_time", "w") as f: + print(tot_time) + json.dump(tot_time, f) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/lora/lora_hf_play.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/lora/lora_hf_play.py new file mode 100644 index 00000000..0abddd2c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/lora/lora_hf_play.py @@ -0,0 +1,62 @@ +import torch +from peft import PeftModel +from transformers import LlamaForCausalLM, LlamaTokenizer + +MODEL = "mistralai/Mistral-7B-Instruct-v0.3" +# ADAPTER = "winddude/wizardLM-LlaMA-LoRA-7B" +ADAPTER = "/home/ying/test_lora" +HF_TOKEN = "..." + + +prompt = """ +### Instruction: +Write a poem about the transformers Python library. +Mention the word "large language models" in that poem. +### Response: +The Transformers are large language models, +They're used to make predictions on text. +""" + + +tokenizer = LlamaTokenizer.from_pretrained(MODEL) + +base_model = LlamaForCausalLM.from_pretrained( + MODEL, + device_map="auto", + # load_in_8bit=True, + torch_dtype=torch.float16, + # use_auth_token=HF_TOKEN, +).cuda() + + +# base model generate +with torch.no_grad(): + output_tensors = base_model.generate( + input_ids=tokenizer(prompt, return_tensors="pt").input_ids.cuda(), + max_new_tokens=32, + do_sample=False, + )[0] + +output = tokenizer.decode(output_tensors, skip_special_tokens=True) +print("======= base output ========") +print(output) + + +# peft model generate +model = PeftModel.from_pretrained( + base_model, + ADAPTER, + torch_dtype=torch.float16, + is_trainable=False, +) + +with torch.no_grad(): + output_tensors = model.generate( + input_ids=tokenizer(prompt, return_tensors="pt").input_ids.cuda(), + max_new_tokens=32, + do_sample=False, + )[0] + +output = tokenizer.decode(output_tensors, skip_special_tokens=True) +print("======= peft output ========") +print(output) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/lora/lora_vllm_play.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/lora/lora_vllm_play.py new file mode 100644 index 00000000..4f77d8be --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/lora/lora_vllm_play.py @@ -0,0 +1,30 @@ +from vllm import LLM, SamplingParams +from vllm.lora.request import LoRARequest + +MODEL = "mistralai/Mistral-7B-Instruct-v0.3" +ADAPTER = "/home/ying/test_lora" +prompt = """ +### Instruction: +Write a poem about the transformers Python library. +Mention the word "large language models" in that poem. +### Response: +The Transformers are large language models, +They're used to make predictions on text. +""" + + +llm = LLM(model=MODEL, enable_lora=True) + +sampling_params = SamplingParams( + temperature=0, + max_tokens=32, +) + +prompts = [prompt] + +outputs = llm.generate( + prompts, sampling_params, lora_request=LoRARequest("test_lora", 1, ADAPTER) +) + +print(outputs[0].prompt) +print(outputs[0].outputs[0].text) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/reference_hf.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/reference_hf.py new file mode 100644 index 00000000..538c31f7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/reference_hf.py @@ -0,0 +1,197 @@ +""" +Usage: python3 scripts/playground/reference_hf.py --model-path MODEL_PATH --model-type {text,vlm} [--max-new-tokens NUM] [--dtype DTYPE] + --model-path MODEL_PATH: Path to model (default: TinyLlama/TinyLlama-1.1B-Chat-v0.4) + --model-type {text,vlm}: Model type, text or vlm (default: text) + --max-new-tokens NUM: Max new tokens to generate (default: 16) + --dtype DTYPE: Data type for computation (default: float16) +Note: '--model' is deprecated; use '--model-path'. Runs normal_text() for text, vlm_text_with_image() for vlm. + +Reference output: +========== Prompt 0 ========== +prefill logits (final) tensor([-8.3125, -7.1172, 3.3398, ..., -4.9531, -4.1328, -3.4141], + device='cuda:0') + The capital of France is Paris. +The capital of the United States is Washington, D.C. + +========== Prompt 1 ========== +prefill logits (final) tensor([-8.9062, -9.0156, 4.1484, ..., -4.9922, -4.4961, -4.0742], + device='cuda:0') + The capital of the United Kindom is London. +The capital of the United Kingdom is London. +The capital of + +========== Prompt 2 ========== +prefill logits (final) tensor([-9.6328, -9.0547, 4.0234, ..., -5.3047, -4.7148, -4.4609], + device='cuda:0') + Today is a sunny day and I like to go for a walk in the park. +I'm going to the +""" + +import argparse + +import requests +import torch +from PIL import Image +from transformers import ( + AutoModelForCausalLM, + AutoModelForImageTextToText, + AutoProcessor, +) + +from sglang.srt.utils.hf_transformers_utils import get_tokenizer + + +@torch.no_grad() +def vlm_text_with_image(args): + # Load the processor and model for ImageTextToText tasks + processor = AutoProcessor.from_pretrained(args.model_path, trust_remote_code=True) + model = AutoModelForImageTextToText.from_pretrained( + args.model_path, + torch_dtype=args.dtype, + low_cpu_mem_usage=True, + device_map="auto", + trust_remote_code=True, + ) + + torch.cuda.set_device(0) + + # List of image URLs to process + image_urls = [ + "https://github.com/haotian-liu/LLaVA/blob/1a91fc274d7c35a9b50b3cb29c4247ae5837ce39/images/llava_v1_5_radar.jpg?raw=true" + ] + + # Conversation template for the processor + conversation = [ + { + "role": "user", + "content": [ + { + "type": "image", + }, + {"type": "text", "text": "Describe this image."}, + ], + } + ] + + max_new_tokens = args.max_new_tokens + + for i, url in enumerate(image_urls): + # Load the image from the URL + image = Image.open(requests.get(url, stream=True).raw) + + # Apply the chat template to the text prompt + # Notice that not all processors support chat templates. + # LLaVA and QWen are two processors that support chat templates. + if not hasattr(processor, "apply_chat_template"): + raise ValueError("The processor does not support chat templates.") + text_prompt = processor.apply_chat_template( + conversation, add_generation_prompt=True + ) + + # Prepare inputs for the model + inputs = processor(text=[text_prompt], images=[image], return_tensors="pt").to( + "cuda:0" + ) + + # Generate output from the model + output_ids = model.generate( + **inputs, do_sample=False, max_new_tokens=max_new_tokens + ) + output_str = processor.decode(output_ids[0]) + + # Get the logits from the model's forward pass + outputs = model.forward(**inputs) + logits = outputs.logits[0, -1, :] + + print(f"\n========== Image {i} ==========") + print("prefill logits (final)", logits) + # TODO(gaocegege): The output contains numerous <|image_pad|> tokens, + # making it cluttered and difficult to read. + # These tokens should be removed or cleaned up for better readability. + print(output_str) + + +@torch.no_grad() +def normal_text(args): + t = get_tokenizer(args.model_path, trust_remote_code=True) + m = AutoModelForCausalLM.from_pretrained( + args.model_path, + torch_dtype=args.dtype, + low_cpu_mem_usage=True, + device_map="auto", + trust_remote_code=True, + ) + + prompts = [ + "The capital of France is", + "The capital of the United Kindom is", + "Today is a sunny day and I like", + ] + max_new_tokens = args.max_new_tokens + + torch.cuda.set_device(0) + + for i, p in enumerate(prompts): + if isinstance(p, str): + input_ids = t.encode(p, return_tensors="pt").to("cuda:0") + else: + input_ids = torch.tensor([p], device="cuda:0") + + output_ids = m.generate( + input_ids, do_sample=False, max_new_tokens=max_new_tokens + ) + output_str = t.decode(output_ids[0]) + + prefill_logits = m.forward(input_ids).logits[0][-1] + + print(f"\n========== Prompt {i} ==========") + print("prefill logits (final)", prefill_logits) + print(output_str) + + +@torch.no_grad() +def synthetic_tokens(args): + m = AutoModelForCausalLM.from_pretrained( + args.model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True + ) + m.cuda() + print(m) + + input_len = 256 + output_len = 8 + prompts = [list(range(5, 5 + input_len))] + + for p in prompts: + input_ids = p + for i in range(output_len + 1): + prefill_logits = m.forward(torch.tensor([input_ids], device="cuda")).logits[ + 0 + ][-1] + + if i == 0: + print("prefill logits", prefill_logits) + else: + print("decode", i - 1, prefill_logits) + + input_ids.append(torch.argmax(prefill_logits).item()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--model-path", + type=str, + default="TinyLlama/TinyLlama-1.1B-Chat-v0.4", + ) + parser.add_argument("--max-new-tokens", type=int, default=16) + + parser.add_argument("--dtype", type=str, default="float16") + + parser.add_argument("--model-type", type=str, default="text") + + args = parser.parse_args() + + if args.model_type == "vlm": + vlm_text_with_image(args) + else: + normal_text(args) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/replay_request_dump.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/replay_request_dump.py new file mode 100644 index 00000000..301cf948 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/replay_request_dump.py @@ -0,0 +1,151 @@ +""" +Usage: +# replay from a folder +python3 replay_request_dump.py --file-number 100 --parallel 512 --input-folder /data/lianmin/sglang_request_dump/grok-mini-0220-engine-5756f8f94-28bm6/ + +# replay from a single file +python3 replay_request_dump.py --parallel 512 --input-file /data/sglang_crash_dump/memx-cti-34-sr1.xpop.twttr.net/crash_dump_2025-06-04_20-13-18.pkl +""" + +import argparse +import glob +import json +import pickle +import time +from concurrent.futures import ThreadPoolExecutor +from dataclasses import asdict +from datetime import datetime + +import requests + +from sglang.bench_serving import set_ulimit +from sglang.utils import get_exception_traceback + + +def read_records(files): + records = [] + for f in files: + tmp = pickle.load(open(f, "rb")) + if isinstance(tmp, dict) and "requests" in tmp: + records.extend(tmp["requests"]) + else: + records.extend(tmp) + + return records + + +def run_one_request_internal(record): + (req, output, replay_init_time, start_time, end_time, idx) = record + time.sleep(max(0, (start_time - (time.time() - replay_init_time)) / args.speed)) + + if "completion_tokens" in output.get("meta_info", {}): + recorded_completion_tokens = output["meta_info"]["completion_tokens"] + else: + recorded_completion_tokens = "" + + json_data = asdict(req) + stream = json_data["stream"] + + if args.ignore_eos: + json_data["sampling_params"]["ignore_eos"] = True + if recorded_completion_tokens: + json_data["sampling_params"]["max_new_tokens"] = recorded_completion_tokens + + response = requests.post( + f"http://{args.host}:{args.port}/generate", + json=json_data, + stream=stream, + ) + + if stream: + for chunk in response.iter_lines(decode_unicode=False): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]": + break + ret = json.loads(chunk[5:].strip("\n")) + else: + ret = response.json() + + prompt_tokens = ret["meta_info"]["prompt_tokens"] + completion_tokens = ret["meta_info"]["completion_tokens"] + print( + f"{idx=}, {start_time=:.2f}, {prompt_tokens=}, " + f"{completion_tokens=}, {recorded_completion_tokens=}" + ) + + +def run_one_request(record): + # global success_ct, error_ct + + try: + run_one_request_internal(record) + # success_ct += 1 + except Exception: + # error_ct += 1 + traceback = get_exception_traceback() + print(f"Hit an exception: {traceback}") + + +def main(records): + if len(records) == 0: + return + + base_time = records[0][-2] + base_time_str = datetime.fromtimestamp(base_time).strftime("%y-%m-%d %H:%M:%S") + print(f"{base_time_str=}") + replay_init_time = time.time() + + for i in range(len(records)): + req, output, start_time, end_time = records[i] + start_time -= base_time + records[i] = (req, output, replay_init_time, start_time, end_time, i) + + with ThreadPoolExecutor(args.parallel) as executor: + executor.map(run_one_request, records) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="localhost") + parser.add_argument("--port", type=int, default=30000) + parser.add_argument( + "--input-folder", type=str, default=None, help="Folder containing pickle files" + ) + parser.add_argument( + "--input-file", type=str, default=None, help="Single pickle file to process" + ) + parser.add_argument("--file-number", type=int, default=1) + parser.add_argument("--req-number", type=int, default=1000000) + parser.add_argument("--req-start", type=int, default=0) + parser.add_argument("--parallel", type=int, default=512) + parser.add_argument("--idx", type=int, default=None) + parser.add_argument("--ignore-eos", action="store_true") + parser.add_argument("--speed", type=float, default=1) + args = parser.parse_args() + + set_ulimit() + + files = [] + if args.input_file: + files = [args.input_file] + if args.file_number > 1: + print("Warning: --file-number is ignored when --input-file is provided.") + elif args.input_folder: + files = glob.glob(f"{args.input_folder}/*.pkl") + files = files[: args.file_number] + else: + print("Error: Either --input-folder or --input-file must be provided.") + exit(1) + print(f"{files=}") + + records = read_records(files) + # Sort by the receive time, before filtering + records.sort(key=lambda x: x[-2]) + records = records[args.req_start :] + if args.idx: + records = [records[args.idx]] + print(f"testing {args.idx=}") + print(f"{records[0]}") + print(f"{len(records)=}") + main(records) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/router/test_tree.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/router/test_tree.py new file mode 100644 index 00000000..af41c738 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/router/test_tree.py @@ -0,0 +1,207 @@ +import random +import string +import time +import unittest +from typing import Dict, List, Tuple + +from tree import MultiTenantRadixTree + + +class TestMultiTenantRadixTree(unittest.TestCase): + def setUp(self): + self.tree = MultiTenantRadixTree() + + def test_insert_exact_match(self): + """Test 1: Basic insert and exact match operations""" + # Insert a single string for one tenant + self.tree.insert("hello", "tenant1") + matched, tenant = self.tree.prefix_match("hello") + self.assertEqual(matched, "hello") + self.assertEqual(tenant, "tenant1") + + # Insert same string for different tenant + self.tree.insert("hello", "tenant2") + matched, tenant = self.tree.prefix_match("hello") + self.assertIn(tenant, ["tenant1", "tenant2"]) + + # Insert different string for same tenant + self.tree.insert("world", "tenant1") + matched, tenant = self.tree.prefix_match("world") + self.assertEqual(matched, "world") + self.assertEqual(tenant, "tenant1") + + print(self.tree.pretty_print()) + + def test_insert_partial_match(self): + """Test 2: Insert with partial matching scenarios""" + # Test partial matches with common prefixes + self.tree.insert("hello", "tenant1") + print(self.tree.pretty_print()) + self.tree.insert("help", "tenant2") + print(self.tree.pretty_print()) + + # Match exact strings + matched, tenant = self.tree.prefix_match("hello") + self.assertEqual(matched, "hello") + self.assertEqual(tenant, "tenant1") + + matched, tenant = self.tree.prefix_match("help") + self.assertEqual(matched, "help") + self.assertEqual(tenant, "tenant2") + + # Match partial string + matched, tenant = self.tree.prefix_match("hel") + self.assertEqual(matched, "hel") + self.assertIn(tenant, ["tenant1", "tenant2"]) + + # Match longer string + matched, tenant = self.tree.prefix_match("hello_world") + self.assertEqual(matched, "hello") + self.assertEqual(tenant, "tenant1") + + def test_insert_edge_cases(self): + """Test 3: Edge cases for insert and match operations""" + # Empty string + self.tree.insert("", "tenant1") + matched, tenant = self.tree.prefix_match("") + self.assertEqual(matched, "") + self.assertEqual(tenant, "tenant1") + + # Single character + self.tree.insert("a", "tenant1") + matched, tenant = self.tree.prefix_match("a") + self.assertEqual(matched, "a") + self.assertEqual(tenant, "tenant1") + + # Very long string + long_str = "a" * 1000 + self.tree.insert(long_str, "tenant1") + matched, tenant = self.tree.prefix_match(long_str) + self.assertEqual(matched, long_str) + self.assertEqual(tenant, "tenant1") + + # Unicode characters + self.tree.insert("你好", "tenant1") + matched, tenant = self.tree.prefix_match("你好") + self.assertEqual(matched, "你好") + self.assertEqual(tenant, "tenant1") + + def test_simple_eviction(self): + """Test 4: Simple eviction scenarios + Tenant1: limit 10 chars + Tenant2: limit 5 chars + + Should demonstrate: + 1. Basic eviction when size limit exceeded + 2. Proper eviction based on last access time + 3. Verification that shared nodes remain intact for other tenants + """ + # Set up size limits + max_size = {"tenant1": 10, "tenant2": 5} + + # Insert strings for both tenants + self.tree.insert("hello", "tenant1") # size 5 + self.tree.insert("hello", "tenant2") # size 5 + self.tree.insert("world", "tenant2") # size 5, total for tenant2 = 10 + + # Verify initial sizes + sizes_before = self.tree.get_used_size_per_tenant() + self.assertEqual(sizes_before["tenant1"], 5) # "hello" = 5 + self.assertEqual(sizes_before["tenant2"], 10) # "hello" + "world" = 10 + + # Evict - should remove "hello" from tenant2 as it's the oldest + self.tree.evict_tenant_data(max_size) + + # Verify sizes after eviction + sizes_after = self.tree.get_used_size_per_tenant() + self.assertEqual(sizes_after["tenant1"], 5) # Should be unchanged + self.assertEqual(sizes_after["tenant2"], 5) # Only "world" remains + + # Verify "world" remains for tenant2 (was accessed more recently) + matched, tenant = self.tree.prefix_match("world") + self.assertEqual(matched, "world") + self.assertEqual(tenant, "tenant2") + + def test_medium_eviction(self): + """Test 5: Medium complexity eviction scenarios with shared prefixes + Tenant1: limit 10 chars + Tenant2: limit 7 chars (forces one string to be evicted) + + Tree structure after inserts: + └── 'h' [t1, t2] + ├── 'i' [t1, t2] # Oldest for t2 + └── 'e' [t1, t2] + ├── 'llo' [t1, t2] + └── 'y' [t2] # Newest for t2 + + Size calculations: + tenant1: "h"(1) + "i"(1) + "e"(1) + "llo"(3) = 6 chars + tenant2: "h"(1) + "i"(1) + "e"(1) + "llo"(3) + "y"(1) = 7 chars + + After eviction (tenant2 exceeds limit by 1 char): + "hi" should be removed from tenant2 as it's the oldest access + """ + max_size = { + "tenant1": 10, + "tenant2": 6, + } # tenant2 will need to evict one string + + # Create a tree with overlapping prefixes + self.tree.insert("hi", "tenant1") + self.tree.insert("hi", "tenant2") # OLDEST for t2 + + self.tree.insert("hello", "tenant1") + self.tree.insert("hello", "tenant2") + + self.tree.insert("hey", "tenant2") # NEWEST for t2 + + # Verify initial sizes + sizes_before = self.tree.get_used_size_per_tenant() + self.assertEqual(sizes_before["tenant1"], 6) # h(1) + i(1) + e(1) + llo(3) = 6 + self.assertEqual( + sizes_before["tenant2"], 7 + ) # h(1) + i(1) + e(1) + llo(3) + y(1) = 7 + + print("\nTree before eviction:") + print(self.tree.pretty_print()) + + # Evict - should remove "hi" from tenant2 as it's the oldest + self.tree.evict_tenant_data(max_size) + + print("\nTree after eviction:") + print(self.tree.pretty_print()) + + # Verify sizes after eviction + sizes_after = self.tree.get_used_size_per_tenant() + self.assertEqual(sizes_after["tenant1"], 6) # Should be unchanged + self.assertEqual(sizes_after["tenant2"], 6) # h(1) + e(1) + llo(3) + y(1) = 6 + + def test_advanced_eviction(self): + ... + # Create 4 tenants + # Each tenants keeps adding strings with shared prefixes to thousands usage + # Set a strict limit for each tenant to only 100 + # At the end, check whether all of the tenant is under 100 after eviction + + max_size = {"tenant1": 100, "tenant2": 100, "tenant3": 100, "tenant4": 100} + + prefixes = ["aqwefcisdf", "iajsdfkmade", "kjnzxcvewqe", "iejksduqasd"] + for i in range(100): + for j, prefix in enumerate(prefixes): + random_suffix = "".join(random.choices(string.ascii_letters, k=10)) + self.tree.insert(prefix + random_suffix, f"tenant{j+1}") + + sizes_before = self.tree.get_used_size_per_tenant() + print(sizes_before) + + self.tree.evict_tenant_data(max_size) + + sizes_after = self.tree.get_used_size_per_tenant() + print(sizes_after) + # ensure size_after is below max_size + for tenant, size in sizes_after.items(): + self.assertLessEqual(size, max_size[tenant]) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/router/tree.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/router/tree.py new file mode 100644 index 00000000..9cbfa7cf --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/playground/router/tree.py @@ -0,0 +1,292 @@ +import time +from collections import defaultdict +from typing import Dict, List + + +class Node: + def __init__(self): + self.children: Dict[str, Node] = dict() + # We choose to use text because most of the use cases are text-to-text, + # so we can save the tokenizing overhead. + self.text: str = "" + # Maps tenant_id to their last access timestamp + self.tenant_last_access_time: Dict[str, float] = dict() + self.parent = None + + +def shared_prefix_length(s1, s2): + min_length = min(len(s1), len(s2)) + for i in range(min_length): + if s1[i] != s2[i]: + return i + return min_length + + +class MultiTenantRadixTree: + """ + Python Reference of Rust implementation of MultiTenantRadixTree + + MultiTenantRadixTree is the overlap of multiple radix trees by different tenant + Each node in the tree can be owned by multiple tenants, allowing for efficient storage of common prefixes + while maintaining tenant isolation. + + Key concepts: + - Tenant: An entity that owns a subset of the stored strings + - Each node tracks which tenants have access to it via tenant_last_access_time + - The tree structure is shared, but queries can be filtered by tenant_id + """ + + def __init__(self): + self.root = Node() + + def insert(self, s: str, tenant_id: str) -> None: + """ + Insert string 's' and associate it with the given tenant_id. + + Args: + s: The string to insert + tenant_id: The identifier of the tenant who owns this string + """ + curr = self.root + curr_idx = 0 + curr.tenant_last_access_time[tenant_id] = time.time() + + while curr_idx < len(s): + matched_node = None + if s[curr_idx] in curr.children: + matched_node = curr.children[s[curr_idx]] + + if matched_node is None: + # No match => create a new node + new_node = Node() + new_node.text = s[curr_idx:] + new_node.parent = curr + + curr.children[s[curr_idx]] = new_node + curr_idx = len(s) + curr = new_node + curr.tenant_last_access_time[tenant_id] = time.time() + else: + shared_len = shared_prefix_length(s[curr_idx:], matched_node.text) + + # 1. If the matched text is shorter than the node text => split the node + if shared_len < len(matched_node.text): + # Split structure: [matched_node] => [new_node] -> [contracted_matched_node] + + matched_text = matched_node.text[:shared_len] + unmatched_text = matched_node.text[shared_len:] + + new_node = Node() + new_node.text = matched_text + new_node.children = {unmatched_text[0]: matched_node} + new_node.parent = curr + new_node.parent.children[matched_text[0]] = new_node + new_node.tenant_last_access_time = ( + matched_node.tenant_last_access_time.copy() + ) + + # Contract matched node + matched_node.text = unmatched_text + matched_node.parent = new_node + + curr_idx += shared_len + curr = new_node + curr.tenant_last_access_time[tenant_id] = time.time() + # 2. If the matched text is longer or equal to the node text => walk down the node + else: + curr_idx += shared_len + curr = matched_node + curr.tenant_last_access_time[tenant_id] = time.time() + + def prefix_match(self, s: str) -> tuple[str, int]: + """ + Match string 's' with multiple tenants' trees in one operation. + + Args: + s: The string to match + + Returns: + Tuple(str, int): The longest prefix of 's' that matches the tree and the first tenant_id that own the matched prefix + """ + curr = self.root + curr_idx = 0 + + ret_text = "" + ret_tenant = None + + while curr_idx < len(s): + matched_node = None + if s[curr_idx] in curr.children: + matched_node = curr.children[s[curr_idx]] + + if matched_node is None: + break + + shared_len = shared_prefix_length(s[curr_idx:], matched_node.text) + if shared_len == len(matched_node.text): + curr_idx += shared_len + curr = matched_node + else: + curr_idx += shared_len + curr = matched_node + break + + selected_tenant = list(curr.tenant_last_access_time.keys())[0] + + # traverse back to the root to update last access time for the selected tenant + while curr != self.root: + curr.tenant_last_access_time[selected_tenant] = time.time() + curr = curr.parent + + return s[:curr_idx], selected_tenant + + def evict_tenant_data(self, max_size_per_tenant: Dict[str, int]) -> None: + """ + Evict data for tenants that have exceeded their storage limits. + + Args: + max_size_per_tenant: Dictionary mapping tenant_id to their maximum allowed storage size + """ + + def leaf_of(node): + """ + If the node is a leaf for a tenant, add tenant_id to the return list + This will return list of tenant ids + If not a leaf for all tenants, return [] + """ + candidates = dict([(k, True) for k in node.tenant_last_access_time.keys()]) + + for n in node.children.values(): + for c in n.tenant_last_access_time.keys(): + candidates[c] = False + + return [k for k, v in candidates.items() if v] + + # maintain a heap with (time, tenant, node) as the value + import heapq + + # 1. traverse the tree to + # a. add all the leaves into a heap (a node with N tenants will be added N times into the heap) + # b. calculate the used size for each tenant + # do a dfs with stack + stack = [self.root] + pq = [] + used_size_per_tenant = defaultdict(int) + + while stack: + curr = stack.pop() + for t in curr.tenant_last_access_time.keys(): + used_size_per_tenant[t] += len(curr.text) + + for c in curr.children.values(): + stack.append(c) + + # if the node is a leaf for a tenant, add the tenant to the heap + tenants = leaf_of(curr) + for t in tenants: + heapq.heappush(pq, (curr.tenant_last_access_time[t], t, curr)) + + # 2. pop the heap + # a. if the tenant's used size is less than the limit, continue + # b. if the tenant's used size is greater than the limit, remove the leaf and update the used size, and add its parent to the heap + while len(pq) > 0: + time, tenant, node = heapq.heappop(pq) + if used_size_per_tenant[tenant] <= max_size_per_tenant[tenant]: + continue + + # remove the leaf + used_size_per_tenant[tenant] -= len(node.text) + del node.tenant_last_access_time[tenant] + # if no children and no tenants, remove the node + if len(node.children) == 0 and len(node.tenant_last_access_time) == 0: + del node.parent.children[node.text[0]] + + # add its parent to the heap + if tenant in leaf_of(node.parent): + heapq.heappush( + pq, + (node.parent.tenant_last_access_time[tenant], tenant, node.parent), + ) + + def get_used_size_per_tenant(self) -> Dict[str, int]: + """ + Calculate the used storage size for each tenant. + + Returns: + Dict[str, int]: A dictionary mapping tenant_id to their used storage size + """ + used_size_per_tenant = defaultdict(int) + + stack = [self.root] + while stack: + curr = stack.pop() + for t in curr.tenant_last_access_time.keys(): + used_size_per_tenant[t] += len(curr.text) + + for c in curr.children.values(): + stack.append(c) + + return used_size_per_tenant + + def remove_tenant(self, tenant_id: str) -> None: + """ + Remove all data associated with a specific tenant from the tree. + This operation maintains the integrity of the shared tree structure while + removing only the specified tenant's access information. + + Args: + tenant_id: The identifier of the tenant whose data should be removed + """ + # TODO: Implementation needed + pass + + def pretty_print(self) -> str: + """ + Returns a string representation of the tree showing the structure, tenant ownership, + and leaf status for each node. + + Returns: + str: A formatted string showing the tree hierarchy with tenant information + """ + + def _node_to_str(node: Node, prefix: str = "", is_last: bool = True) -> str: + # Current node representation + node_str = prefix + node_str += "└── " if is_last else "├── " + + # Add node text + node_str += f"'{node.text}' [" + + # Add tenant information including both timestamp and leaf status + tenant_info = [] + for tid, ts in node.tenant_last_access_time.items(): + time_str = ( + time.strftime("%H:%M:%S.", time.localtime(ts)) + + f"{(ts % 1):0.3f}"[2:] + ) + tenant_info.append(f"{tid} | {time_str}") + + node_str += ", ".join(tenant_info) + node_str += "]\n" + + # Handle children + children = list(node.children.items()) + for i, (char, child) in enumerate(children): + is_last_child = i == len(children) - 1 + # Adjust prefix for children based on whether this is the last child + new_prefix = prefix + (" " if is_last else "│ ") + node_str += _node_to_str(child, new_prefix, is_last_child) + + return node_str + + if not self.root.children: + return "Empty tree" + + # Start with root's children since root itself is just an empty node + result = "" + children = list(self.root.children.items()) + for i, (char, child) in enumerate(children): + is_last = i == len(children) - 1 + result += _node_to_str(child, "", is_last) + + return result diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/README.md new file mode 100644 index 00000000..e564cf1d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/README.md @@ -0,0 +1,94 @@ +# Release Scripts + +This directory contains scripts to automate version bumping for SGLang releases. + +## Scripts + +### `bump_sglang_version.py` +Updates SGLang version across all relevant files following the pattern from [PR #10468](https://github.com/sgl-project/sglang/pull/10468). + +**Usage:** +```bash +python scripts/release/bump_sglang_version.py 0.5.3rc0 +``` + +**Files updated:** +- `Makefile` +- `benchmark/deepseek_v3/README.md` +- `docker/rocm.Dockerfile` +- `docs/get_started/install.md` +- `docs/platforms/amd_gpu.md` +- `docs/platforms/ascend_npu.md` +- `python/pyproject.toml` +- `python/pyproject_other.toml` +- `python/sglang/version.py` + +### `bump_kernel_version.py` +Updates sgl-kernel version across all relevant files following the pattern from [PR #10732](https://github.com/sgl-project/sglang/pull/10732). + +**Usage:** +```bash +python scripts/release/bump_kernel_version.py 0.3.12 +``` + +**Files updated:** +- `sgl-kernel/pyproject.toml` +- `sgl-kernel/pyproject_cpu.toml` +- `sgl-kernel/pyproject_rocm.toml` +- `sgl-kernel/python/sgl_kernel/version.py` + +## Manual Testing Instructions + +### Test SGLang Version Bump + +1. **Run the script:** + ```bash + python scripts/release/bump_sglang_version.py 0.5.4rc0 + ``` + +2. **Verify changes with git diff:** + ```bash + git diff + ``` + +3. **Check specific files contain the new version:** + ```bash + grep -r "0.5.4rc0" python/sglang/version.py + grep -r "0.5.4rc0" python/pyproject.toml + grep -r "0.5.4rc0" docs/get_started/install.md + ``` + +4. **Reset changes (if testing):** + ```bash + git checkout . + ``` + +### Test Kernel Version Bump + +1. **Run the script:** + ```bash + python scripts/release/bump_kernel_version.py 0.3.13 + ``` + +2. **Verify changes with git diff:** + ```bash + git diff + ``` + +3. **Check specific files contain the new version:** + ```bash + grep -r "0.3.13" sgl-kernel/python/sgl_kernel/version.py + grep -r "0.3.13" sgl-kernel/pyproject.toml + ``` + +4. **Reset changes (if testing):** + ```bash + git checkout . + ``` + +## Version Format Validation + +- **SGLang versions:** `X.Y.Z` or `X.Y.ZrcN` (e.g., `0.5.3` or `0.5.3rc0`) +- **Kernel versions:** `X.Y.Z` (e.g., `0.3.12`) + +The scripts will validate the version format and exit with an error if invalid. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/bump_kernel_version.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/bump_kernel_version.py new file mode 100755 index 00000000..2ea471ae --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/bump_kernel_version.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 + +import argparse +from pathlib import Path + +from utils import bump_version + + +def main(): + parser = argparse.ArgumentParser( + description="Bump sgl-kernel version across all relevant files" + ) + parser.add_argument( + "new_version", + help="New version (e.g., 0.3.12, 0.3.11rc0, or 0.3.11.post1)", + ) + args = parser.parse_args() + + version_file = Path("sgl-kernel/python/sgl_kernel/version.py") + + files_to_update = [ + Path("sgl-kernel/pyproject.toml"), + Path("sgl-kernel/pyproject_cpu.toml"), + Path("sgl-kernel/pyproject_rocm.toml"), + Path("sgl-kernel/python/sgl_kernel/version.py"), + ] + + bump_version(args.new_version, version_file, files_to_update) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/bump_kernel_version_to_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/bump_kernel_version_to_sglang.py new file mode 100755 index 00000000..37cf674b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/bump_kernel_version_to_sglang.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +""" +Bump sgl-kernel version in SGLang files to match the version in sgl-kernel/pyproject.toml. +Updates: + - python/pyproject.toml + - python/sglang/srt/entrypoints/engine.py + - docker/Dockerfile +""" + +import re +import sys +from pathlib import Path + +try: + import tomllib # Python 3.11+ +except ImportError: + import tomli as tomllib # Fallback for older Python versions + + +def get_kernel_version_from_source() -> str: + """Extract version from sgl-kernel/pyproject.toml""" + pyproject_path = Path("sgl-kernel/pyproject.toml") + + if not pyproject_path.exists(): + print(f"Error: {pyproject_path} not found") + sys.exit(1) + + with open(pyproject_path, "rb") as f: + data = tomllib.load(f) + + version = data.get("project", {}).get("version") + if not version: + print("Error: Could not find version in sgl-kernel/pyproject.toml") + sys.exit(1) + + return version + + +def update_python_pyproject(new_version: str) -> bool: + """Update sgl-kernel version in python/pyproject.toml""" + pyproject_path = Path("python/pyproject.toml") + + if not pyproject_path.exists(): + print(f"Error: {pyproject_path} not found") + sys.exit(1) + + content = pyproject_path.read_text() + + # Replace "sgl-kernel==x.x.x" with new version + new_content = re.sub( + r'"sgl-kernel==[^"]+"', + f'"sgl-kernel=={new_version}"', + content, + ) + + if content == new_content: + print("No changes needed in python/pyproject.toml") + return False + + pyproject_path.write_text(new_content) + print(f"✓ Updated python/pyproject.toml to version {new_version}") + return True + + +def update_engine_py(new_version: str) -> bool: + """Update sgl-kernel version in python/sglang/srt/entrypoints/engine.py""" + engine_path = Path("python/sglang/srt/entrypoints/engine.py") + + if not engine_path.exists(): + print(f"Error: {engine_path} not found") + sys.exit(1) + + content = engine_path.read_text() + + # Replace version in assert_pkg_version("sgl-kernel", "version", ...) + new_content = re.sub( + r'(assert_pkg_version\s*\(\s*"sgl-kernel"\s*,\s*)"[^"]+"', + rf'\1"{new_version}"', + content, + ) + + if content == new_content: + print("No changes needed in engine.py") + return False + + engine_path.write_text(new_content) + print(f"✓ Updated engine.py to version {new_version}") + return True + + +def update_dockerfile(new_version: str) -> bool: + """Update SGL_KERNEL_VERSION in docker/Dockerfile""" + dockerfile_path = Path("docker/Dockerfile") + + if not dockerfile_path.exists(): + print(f"Error: {dockerfile_path} not found") + sys.exit(1) + + content = dockerfile_path.read_text() + + # Replace ARG SGL_KERNEL_VERSION=x.x.x with new version + new_content = re.sub( + r"^(ARG\s+SGL_KERNEL_VERSION=)(.+)$", + rf"\g<1>{new_version}", + content, + flags=re.MULTILINE, + ) + + if content == new_content: + print("No changes needed in Dockerfile") + return False + + dockerfile_path.write_text(new_content) + print(f"✓ Updated Dockerfile to version {new_version}") + return True + + +def main(): + kernel_version = get_kernel_version_from_source() + print(f"Bumping sgl-kernel version to: {kernel_version}\n") + + updated_files = [] + + if update_python_pyproject(kernel_version): + updated_files.append("python/pyproject.toml") + + if update_engine_py(kernel_version): + updated_files.append("python/sglang/srt/entrypoints/engine.py") + + if update_dockerfile(kernel_version): + updated_files.append("docker/Dockerfile") + + print() + if updated_files: + print(f"✓ Successfully updated {len(updated_files)} file(s):") + for file in updated_files: + print(f" - {file}") + else: + print("✓ All files already have the correct version") + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/bump_sglang_version.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/bump_sglang_version.py new file mode 100755 index 00000000..76d26383 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/bump_sglang_version.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +import argparse +from pathlib import Path + +from utils import bump_version + + +def main(): + parser = argparse.ArgumentParser( + description="Bump SGLang version across all relevant files" + ) + parser.add_argument( + "new_version", + help="New version (e.g., 0.5.4, 0.5.3rc0, or 0.5.3.post1)", + ) + args = parser.parse_args() + + version_file = Path("python/sglang/version.py") + + files_to_update = [ + Path("benchmark/deepseek_v3/README.md"), + Path("docker/Dockerfile"), + Path("docker/rocm.Dockerfile"), + Path("docs/get_started/install.md"), + Path("docs/platforms/amd_gpu.md"), + Path("docs/platforms/ascend_npu.md"), + Path("python/pyproject.toml"), + Path("python/pyproject_other.toml"), + Path("python/pyproject_cpu.toml"), + Path("python/pyproject_xpu.toml"), + Path("python/sglang/version.py"), + ] + + bump_version(args.new_version, version_file, files_to_update) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/check_kernel_version_to_sglang.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/check_kernel_version_to_sglang.py new file mode 100755 index 00000000..1d8f011f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/check_kernel_version_to_sglang.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +""" +Check if sgl-kernel version from sgl-kernel/pyproject.toml matches the versions +used in SGLang files (python/pyproject.toml, engine.py, and Dockerfile). +Sets GitHub Actions output variables to indicate if sync is needed. +""" + +import os +import re +import sys +from pathlib import Path + +try: + import tomllib # Python 3.11+ +except ImportError: + import tomli as tomllib # Fallback for older Python versions + + +def get_kernel_version_from_source() -> str: + """Extract version from sgl-kernel/pyproject.toml (line 11)""" + pyproject_path = Path("sgl-kernel/pyproject.toml") + + if not pyproject_path.exists(): + print(f"Error: {pyproject_path} not found") + sys.exit(1) + + with open(pyproject_path, "rb") as f: + data = tomllib.load(f) + + version = data.get("project", {}).get("version") + if not version: + print("Error: Could not find version in sgl-kernel/pyproject.toml") + sys.exit(1) + + return version + + +def get_kernel_version_from_python_pyproject() -> str: + """Extract sgl-kernel version from python/pyproject.toml""" + pyproject_path = Path("python/pyproject.toml") + + if not pyproject_path.exists(): + print(f"Error: {pyproject_path} not found") + sys.exit(1) + + content = pyproject_path.read_text() + + # Match "sgl-kernel==x.x.x" + match = re.search(r'"sgl-kernel==([^"]+)"', content) + if not match: + print("Error: Could not find sgl-kernel version in python/pyproject.toml") + sys.exit(1) + + return match.group(1) + + +def get_kernel_version_from_engine() -> str: + """Extract sgl-kernel version from python/sglang/srt/entrypoints/engine.py""" + engine_path = Path("python/sglang/srt/entrypoints/engine.py") + + if not engine_path.exists(): + print(f"Error: {engine_path} not found") + sys.exit(1) + + content = engine_path.read_text() + + # Find the assert_pkg_version call for sgl-kernel + # Look for the pattern: assert_pkg_version("sgl-kernel", "version", ...) + match = re.search( + r'assert_pkg_version\s*\(\s*"sgl-kernel"\s*,\s*"([^"]+)"', content + ) + if not match: + print("Error: Could not find sgl-kernel version in engine.py") + sys.exit(1) + + return match.group(1) + + +def get_kernel_version_from_dockerfile() -> str: + """Extract SGL_KERNEL_VERSION from docker/Dockerfile""" + dockerfile_path = Path("docker/Dockerfile") + + if not dockerfile_path.exists(): + print(f"Error: {dockerfile_path} not found") + sys.exit(1) + + content = dockerfile_path.read_text() + + # Match ARG SGL_KERNEL_VERSION=x.x.x + match = re.search(r"^ARG\s+SGL_KERNEL_VERSION=(.+)$", content, re.MULTILINE) + if not match: + print("Error: Could not find SGL_KERNEL_VERSION in Dockerfile") + sys.exit(1) + + return match.group(1).strip() + + +def main(): + kernel_version = get_kernel_version_from_source() + pyproject_version = get_kernel_version_from_python_pyproject() + engine_version = get_kernel_version_from_engine() + dockerfile_version = get_kernel_version_from_dockerfile() + + print(f"Kernel version in sgl-kernel/pyproject.toml: {kernel_version}") + print(f"Kernel version in python/pyproject.toml: {pyproject_version}") + print(f"Kernel version in engine.py: {engine_version}") + print(f"Kernel version in Dockerfile: {dockerfile_version}") + + # Check if any version differs from the source + needs_sync = ( + kernel_version != pyproject_version + or kernel_version != engine_version + or kernel_version != dockerfile_version + ) + + # Set GitHub Actions output + github_output = os.getenv("GITHUB_OUTPUT") + if github_output: + with open(github_output, "a") as f: + f.write(f"needs_sync={'true' if needs_sync else 'false'}\n") + f.write(f"kernel_version={kernel_version}\n") + + if needs_sync: + print(f"\n✓ Sync needed to version: {kernel_version}") + mismatches = [] + if kernel_version != pyproject_version: + mismatches.append( + f" - python/pyproject.toml: {pyproject_version} → {kernel_version}" + ) + if kernel_version != engine_version: + mismatches.append(f" - engine.py: {engine_version} → {kernel_version}") + if kernel_version != dockerfile_version: + mismatches.append( + f" - Dockerfile: {dockerfile_version} → {kernel_version}" + ) + + print("Changes needed:") + for mismatch in mismatches: + print(mismatch) + + sys.exit(0) + else: + print("\n✓ All versions are in sync, no action needed") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/commit_and_pr.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/commit_and_pr.sh new file mode 100755 index 00000000..b61ec6ab --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/commit_and_pr.sh @@ -0,0 +1,72 @@ +#!/bin/bash +set -e + +# Script to commit version bump changes and create a pull request +# Usage: commit_and_pr.sh +# +# Arguments: +# version_type: "SGLang" or "sgl-kernel" +# new_version: The new version number +# branch_name: The git branch name to push to + +VERSION_TYPE="$1" +NEW_VERSION="$2" +BRANCH_NAME="$3" + +if [ -z "$VERSION_TYPE" ] || [ -z "$NEW_VERSION" ] || [ -z "$BRANCH_NAME" ]; then + echo "Error: Missing required arguments" + echo "Usage: $0 " + exit 1 +fi + +# Get changed files and format them +echo "Getting changed files..." +FILES_LIST=$(git diff --name-only | sed 's/^/- /') +COMMIT_FILES=$(git diff --name-only | sed 's/^/ - /') + +# Commit changes +echo "Committing changes..." +git add -A +git commit -m "chore: bump ${VERSION_TYPE} version to ${NEW_VERSION} + +This commit updates the ${VERSION_TYPE} version across all relevant files: +${COMMIT_FILES} + +🤖 Generated with GitHub Actions" + +# Push changes +echo "Pushing to ${BRANCH_NAME}..." +git push origin "${BRANCH_NAME}" + +# Create pull request +echo "Creating pull request..." +PR_URL=$(gh pr create \ + --title "chore: bump ${VERSION_TYPE} version to ${NEW_VERSION}" \ + --body "## Summary + +This PR bumps the ${VERSION_TYPE} version to \`${NEW_VERSION}\` across all relevant files. + +## Files Updated +${FILES_LIST} + +🤖 Generated with GitHub Actions" \ + --base main \ + --head "${BRANCH_NAME}") + +echo "✓ Pull request created successfully" + +# Add GitHub Actions job summary +if [ -n "$GITHUB_STEP_SUMMARY" ]; then + cat >> "$GITHUB_STEP_SUMMARY" < +# +# Arguments: +# kernel_version: The kernel version being synced +# branch_name: The git branch name to push to + +KERNEL_VERSION="$1" +BRANCH_NAME="$2" + +if [ -z "$KERNEL_VERSION" ] || [ -z "$BRANCH_NAME" ]; then + echo "Error: Missing required arguments" + echo "Usage: $0 " + exit 1 +fi + +# Get changed files and format them +echo "Getting changed files..." +FILES_LIST=$(git diff --name-only | sed 's/^/- /') +COMMIT_FILES=$(git diff --name-only | sed 's/^/ - /') + +# Commit changes +echo "Committing changes..." +git add -A +git commit -m "chore: bump sgl-kernel version to ${KERNEL_VERSION} in SGLang + +This commit updates the sgl-kernel version across SGLang files to match +the version defined in sgl-kernel/pyproject.toml. + +Files updated: +${COMMIT_FILES} + +🤖 Generated with GitHub Actions" + +# Push changes +echo "Pushing to ${BRANCH_NAME}..." +git push origin "${BRANCH_NAME}" + +# Create pull request +echo "Creating pull request..." +PR_URL=$(gh pr create \ + --title "chore: bump sgl-kernel version to ${KERNEL_VERSION}" \ + --body "## Summary + +This PR bumps the \`sgl-kernel\` version to \`${KERNEL_VERSION}\` across SGLang files to match the version defined in \`sgl-kernel/pyproject.toml\`. + +**Kernel Version:** \`${KERNEL_VERSION}\` + +## Files Updated +${FILES_LIST} + +## Context + +The sgl-kernel version in \`sgl-kernel/pyproject.toml\` has been updated. This PR ensures that all SGLang files referencing the kernel version are updated accordingly: +- \`python/pyproject.toml\` - dependency specification +- \`python/sglang/srt/entrypoints/engine.py\` - version check +- \`docker/Dockerfile\` - Docker build argument + +🤖 Generated with GitHub Actions" \ + --base main \ + --head "${BRANCH_NAME}") + +echo "✓ Pull request created successfully" + +# Add GitHub Actions job summary +if [ -n "$GITHUB_STEP_SUMMARY" ]; then + cat >> "$GITHUB_STEP_SUMMARY" < stable of lower patch + self.assertEqual(compare_versions("0.5.4rc0", "0.5.3"), 1) + self.assertEqual(compare_versions("0.5.3.post1", "0.5.4rc0"), -1) + + def test_compare_versions_different_minor(self): + """Test comparing versions with different minor numbers.""" + self.assertEqual(compare_versions("0.4.9", "0.5.0"), -1) + self.assertEqual(compare_versions("0.5.0", "0.4.9"), 1) + + def test_compare_versions_different_major(self): + """Test comparing versions with different major numbers.""" + self.assertEqual(compare_versions("0.9.9", "1.0.0"), -1) + self.assertEqual(compare_versions("1.0.0", "0.9.9"), 1) + + def test_real_world_scenarios(self): + """Test real-world version bump scenarios.""" + # Scenario 1: RC progression + self.assertEqual(compare_versions("0.5.3rc0", "0.5.3rc1"), -1) + + # Scenario 2: RC to stable release + self.assertEqual(compare_versions("0.5.3rc2", "0.5.3"), -1) + + # Scenario 3: Stable to post-release hotfix + self.assertEqual(compare_versions("0.5.3", "0.5.3.post1"), -1) + + # Scenario 4: Post-release to next RC + self.assertEqual(compare_versions("0.5.3.post1", "0.5.4rc0"), -1) + + # Scenario 5: Next stable version + self.assertEqual(compare_versions("0.5.3", "0.5.4"), -1) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/utils.py new file mode 100644 index 00000000..da212734 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/release/utils.py @@ -0,0 +1,220 @@ +import re +import sys +from pathlib import Path +from typing import List, Tuple + +try: + import tomllib # Python 3.11+ +except ImportError: + import tomli as tomllib # Fallback for older Python versions + + +def normalize_version(version: str) -> str: + """Remove 'v' prefix from version string if present.""" + return version.lstrip("v") + + +def validate_version(version: str) -> bool: + """Validate version format: X.Y.Z, X.Y.Zrc0, or X.Y.Z.post1""" + pattern = r"^\d+\.\d+\.\d+(rc\d+|\.post\d+)?$" + return bool(re.match(pattern, version)) + + +def parse_version(version: str) -> Tuple[int, int, int, int, int]: + """ + Parse version string into comparable components. + + Returns: (major, minor, patch, pre_release, post_release) + - pre_release: -1000 + rc_number for rcN, 0 for stable (rc0 < rc1 < stable) + - post_release: N for .postN, 0 otherwise + + The pre_release field uses negative numbers to ensure RC versions come before + stable versions when tuples are compared. Python compares tuples element by + element, so (0, 5, 3, -1000, 0) < (0, 5, 3, 0, 0) ensures rc0 < stable. + + Examples: + - "0.5.3rc0" → (0, 5, 3, -1000, 0) # rc0 comes before stable + - "0.5.3rc1" → (0, 5, 3, -999, 0) # rc1 comes after rc0 + - "0.5.3" → (0, 5, 3, 0, 0) # stable version + - "0.5.3.post1" → (0, 5, 3, 0, 1) # post comes after stable + """ + # Match version components + match = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:rc(\d+)|\.post(\d+))?$", version) + if not match: + raise ValueError(f"Invalid version format: {version}") + + major, minor, patch, rc, post = match.groups() + major, minor, patch = int(major), int(minor), int(patch) + + if rc is not None: + # RC version: pre_release = -1000 + rc_number (ensures rc0 < rc1 < ... < stable) + return (major, minor, patch, -1000 + int(rc), 0) + elif post is not None: + # Post version: post_release = N + return (major, minor, patch, 0, int(post)) + else: + # Stable version + return (major, minor, patch, 0, 0) + + +def compare_versions(v1: str, v2: str) -> int: + """ + Compare two version strings following PEP 440 ordering. + + Returns: + - -1 if v1 < v2 + - 0 if v1 == v2 + - 1 if v1 > v2 + + Version ordering: X.Y.ZrcN < X.Y.Z < X.Y.Z.postN < X.Y.(Z+1) + """ + parsed_v1 = parse_version(v1) + parsed_v2 = parse_version(v2) + + if parsed_v1 < parsed_v2: + return -1 + elif parsed_v1 > parsed_v2: + return 1 + else: + return 0 + + +def get_repo_root() -> Path: + return Path(__file__).parent.parent.parent + + +def read_current_version(version_file: Path) -> str: + content = version_file.read_text() + match = re.search(r'__version__\s*=\s*["\']([^"\']+)["\']', content) + if not match: + raise ValueError(f"Could not find version in {version_file}") + return match.group(1) + + +def replace_in_file(file_path: Path, old_version: str, new_version: str) -> bool: + if not file_path.exists(): + print(f"Warning: {file_path} does not exist, skipping") + return False + + content = file_path.read_text() + + # For TOML files, parse and update only the [project] version field + if file_path.suffix == ".toml": + try: + # Parse TOML to verify structure + toml_data = tomllib.loads(content) + + # Check if [project] section exists and has version field + if "project" not in toml_data or "version" not in toml_data["project"]: + print( + f"Warning: {file_path} does not have [project] version field, skipping" + ) + return False + + # Use regex to replace only the version field in [project] section + # This pattern matches the version field that comes after [project] + # and before any other section marker + pattern = r'(\[project\].*?version\s*=\s*)["\']([^"\']+)["\']' + new_content = re.sub( + pattern, rf'\g<1>"{new_version}"', content, flags=re.DOTALL + ) + except Exception as e: + print(f"Warning: Failed to parse {file_path} as TOML: {e}") + print("Falling back to simple string replacement") + new_content = content.replace(old_version, new_version) + else: + # For non-TOML files, use simple string replacement + new_content = content.replace(old_version, new_version) + + if content == new_content: + print(f"No changes needed in {file_path}") + return False + + file_path.write_text(new_content) + print(f"✓ Updated {file_path}") + return True + + +def bump_version( + new_version: str, + version_file: Path, + files_to_update: List[Path], +) -> None: + # Normalize version (remove 'v' prefix if present) + new_version = normalize_version(new_version) + + if not validate_version(new_version): + print(f"Error: Invalid version format: {new_version}") + print("Expected format: X.Y.Z, X.Y.ZrcN, or X.Y.Z.postN") + print("Examples: 0.5.4, 0.5.3rc0, 0.5.3.post1") + sys.exit(1) + + repo_root = get_repo_root() + version_file_abs = repo_root / version_file + + if not version_file_abs.exists(): + print(f"Error: Version file {version_file_abs} does not exist") + sys.exit(1) + + old_version = read_current_version(version_file_abs) + print(f"Current version: {old_version}") + print(f"New version: {new_version}") + print() + + # Compare versions + comparison = compare_versions(new_version, old_version) + if comparison == 0: + print("Error: New version is the same as current version") + sys.exit(1) + elif comparison < 0: + print( + f"Error: New version ({new_version}) is older than current version ({old_version})" + ) + print("Version must be greater than the current version") + sys.exit(1) + + updated_count = 0 + for file_rel in files_to_update: + file_abs = repo_root / file_rel + if replace_in_file(file_abs, old_version, new_version): + updated_count += 1 + + print() + print(f"Successfully updated {updated_count} file(s)") + print(f"Version bumped from {old_version} to {new_version}") + + # Validate that all files now contain the new version + print("\nValidating version updates...") + failed_files = [] + for file_rel in files_to_update: + file_abs = repo_root / file_rel + if not file_abs.exists(): + print(f"Warning: File {file_rel} does not exist, skipping validation.") + continue + + content = file_abs.read_text() + + # For TOML files, use regex to specifically check the version field + if file_abs.suffix == ".toml": + # Match version field with optional quotes + pattern = r'version\s*=\s*["\']?' + re.escape(new_version) + r'["\']?' + if not re.search(pattern, content): + failed_files.append(file_rel) + print(f"✗ {file_rel} does not contain version {new_version}") + else: + print(f"✓ {file_rel} validated") + else: + # For non-TOML files, use simple string search + if new_version not in content: + failed_files.append(file_rel) + print(f"✗ {file_rel} does not contain version {new_version}") + else: + print(f"✓ {file_rel} validated") + + if failed_files: + print(f"\nError: {len(failed_files)} file(s) were not updated correctly:") + for file_rel in failed_files: + print(f" - {file_rel}") + sys.exit(1) + + print("\nAll files validated successfully!") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/sort_testcases_alphabetically.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/sort_testcases_alphabetically.py new file mode 100644 index 00000000..67700836 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/sort_testcases_alphabetically.py @@ -0,0 +1,27 @@ +""" +Sort the test case by name alphabetically for run_suite.py +""" + +from dataclasses import dataclass + + +@dataclass +class TestFile: + name: str + estimated_time: float = 60 + + +suites = {} + + +if __name__ == "__main__": + for key in suites: + cases = suites[key] + names = [x.name for x in cases] + names.sort() + + print(f' "{key}": [') + for name in names: + estimated_time = [x.estimated_time for x in cases if x.name == name][0] + print(f' TestFile("{name}", {estimated_time}),') + print(f" ],\n") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/update_kernel_whl_index.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/update_kernel_whl_index.py new file mode 100644 index 00000000..1b812a6a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/update_kernel_whl_index.py @@ -0,0 +1,53 @@ +# Reference: https://github.com/flashinfer-ai/flashinfer/blob/v0.2.0/scripts/update_whl_index.py + +import argparse +import hashlib +import pathlib +import re + +# All the CUDA versions that the wheels will cover +SUPPORTED_CUDA_VERSIONS = ["129", "130"] +DEFAULT_CUDA_VERSION = "129" + + +def check_wheel_cuda_version(path_name, target_cuda_version): + # For other CUDA versions, the wheel path name will contain the cuda version suffix, e.g. sgl_kernel-0.3.16.post5+cu130-cp310-abi3-manylinux2014_x86_64.whl + if target_cuda_version != DEFAULT_CUDA_VERSION: + return target_cuda_version in path_name + + # For the default CUDA version, the wheel path name will not contain any cuda version suffix, e.g. sgl_kernel-0.3.16.post5-cp310-abi3-manylinux2014_x86_64.whl + # So we need to check if the wheel path name contains any other cuda version suffix + for cuda_version in SUPPORTED_CUDA_VERSIONS: + if cuda_version != DEFAULT_CUDA_VERSION and cuda_version in path_name: + return False + return True + + +def update_wheel_index(cuda_version=DEFAULT_CUDA_VERSION): + index_dir = pathlib.Path(f"sgl-whl/cu{cuda_version}/sgl-kernel") + index_dir.mkdir(exist_ok=True) + base_url = "https://github.com/sgl-project/whl/releases/download" + + for path in sorted(pathlib.Path("sgl-kernel/dist").glob("*.whl")): + # Skip the wheel if mismatches the passed in cuda_version + if not check_wheel_cuda_version(path.name, cuda_version): + continue + with open(path, "rb") as f: + sha256 = hashlib.sha256(f.read()).hexdigest() + ver = re.findall( + r"sgl_kernel-([0-9.]+(?:\.post[0-9]+)?)(?:\+cu[0-9]+)?-", path.name + )[0] + full_url = f"{base_url}/v{ver}/{path.name}#sha256={sha256}" + with (index_dir / "index.html").open("a") as f: + f.write(f'{path.name}
\n') + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--cuda", type=str, default="118") + args = parser.parse_args() + update_wheel_index(args.cuda) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/version_branch_to_tag.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/version_branch_to_tag.sh new file mode 100755 index 00000000..9f587fb0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/scripts/version_branch_to_tag.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -euxo pipefail + +# This script is used for release. +# It tags all remote branches starting with 'v' with the same name as the branch, +# deletes the corresponding branches from the remote, and pushes the tags to the remote repository. + +git fetch origin --prune + +# List all branches starting with 'v' +branches=$(git branch -r | grep 'origin/v' | sed 's/origin\///') + +# Loop through each branch +for branch in $branches; do + echo "Processing branch: $branch" + + # Get the commit hash for the branch + commit_hash=$(git rev-parse origin/$branch) + + # Create a tag with the same name as the branch using the commit hash + git tag $branch $commit_hash + + # Delete the branch from the remote + git push origin --delete $branch +done + +# Push all tags to the remote repository +git push --tags + +echo "All branches starting with 'v' have been tagged, deleted from remote, and pushed to the remote repository." diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/.clang-format b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/.clang-format new file mode 100644 index 00000000..afbd654a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/.clang-format @@ -0,0 +1,15 @@ +BasedOnStyle: Google +IndentWidth: 2 +ColumnLimit: 120 +AllowShortFunctionsOnASingleLine: Empty +DerivePointerAlignment: false +PointerAlignment: Left +NamespaceIndentation: None +SortIncludes: true +AllowShortLoopsOnASingleLine: false +BinPackParameters: false # Prevents packing parameters in declarations +BinPackArguments: false # Prevents packing arguments in function calls +AlignAfterOpenBracket: AlwaysBreak # Forces a break after the opening parenthesis +AlignOperands: Align # Aligns arguments vertically +PenaltyBreakBeforeFirstCallParameter: 1 # Encourages breaking before the first argument +PenaltyReturnTypeOnItsOwnLine: 100 # Keeps return type with function name diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/CMakeLists.txt b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/CMakeLists.txt new file mode 100644 index 00000000..3dafcfb1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/CMakeLists.txt @@ -0,0 +1,598 @@ +cmake_minimum_required(VERSION 3.26 FATAL_ERROR) +project(sgl-kernel LANGUAGES CXX CUDA) + +# utils +include(${CMAKE_CURRENT_LIST_DIR}/cmake/utils.cmake) +include(FetchContent) + +# CMake +cmake_policy(SET CMP0169 OLD) +cmake_policy(SET CMP0177 NEW) +set(CMAKE_COLOR_DIAGNOSTICS ON) +set(CMAKE_VERBOSE_MAKEFILE ON CACHE BOOL "ON") +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(CMAKE_SHARED_LIBRARY_PREFIX "") + +# Python +find_package(Python COMPONENTS Interpreter Development.Module ${SKBUILD_SABI_COMPONENT} REQUIRED) + +# CXX +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3") + +# CUDA +enable_language(CUDA) +find_package(CUDAToolkit REQUIRED) +set_property(GLOBAL PROPERTY CUDA_SEPARABLE_COMPILATION ON) + +message(STATUS "Detected CUDA_VERSION=${CUDA_VERSION}") +if ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "13.0") + message("CUDA_VERSION ${CUDA_VERSION} >= 13.0") +elseif ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "12.8") + message("CUDA_VERSION ${CUDA_VERSION} >= 12.8") +elseif ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "12.4") + message("CUDA_VERSION ${CUDA_VERSION} >= 12.4") +elseif ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "12.1") + message("CUDA_VERSION ${CUDA_VERSION} >= 12.1") +elseif ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "11.8") + message("CUDA_VERSION ${CUDA_VERSION} >= 11.8") +endif() + +# Torch +find_package(Torch REQUIRED) +clear_cuda_arches(CMAKE_FLAG) + +# Third Party repos +# cutlass +FetchContent_Declare( + repo-cutlass + GIT_REPOSITORY https://github.com/NVIDIA/cutlass + GIT_TAG 57e3cfb47a2d9e0d46eb6335c3dc411498efa198 + GIT_SHALLOW OFF +) +FetchContent_Populate(repo-cutlass) + +# DeepGEMM +FetchContent_Declare( + repo-deepgemm + GIT_REPOSITORY https://github.com/sgl-project/DeepGEMM + GIT_TAG ffe2b6b97420a9f8c58268ca55755168e6e2f360 + GIT_SHALLOW OFF +) +FetchContent_Populate(repo-deepgemm) + +# fmt +FetchContent_Declare( + repo-fmt + GIT_REPOSITORY https://github.com/fmtlib/fmt + GIT_TAG 553ec11ec06fbe0beebfbb45f9dc3c9eabd83d28 + GIT_SHALLOW OFF +) +FetchContent_Populate(repo-fmt) + +# Triton kernel +FetchContent_Declare( + repo-triton + GIT_REPOSITORY "https://github.com/triton-lang/triton" + GIT_TAG v3.5.1 + GIT_SHALLOW OFF +) +FetchContent_Populate(repo-triton) + +# flashinfer +FetchContent_Declare( + repo-flashinfer + GIT_REPOSITORY https://github.com/flashinfer-ai/flashinfer.git + GIT_TAG bc29697ba20b7e6bdb728ded98f04788e16ee021 + GIT_SHALLOW OFF +) +FetchContent_Populate(repo-flashinfer) + +# flash-attention +FetchContent_Declare( + repo-flash-attention + GIT_REPOSITORY https://github.com/sgl-project/sgl-attn + GIT_TAG f20a52329482ddca4a627b2f028f88c2959ee299 + GIT_SHALLOW OFF +) +FetchContent_Populate(repo-flash-attention) + +# mscclpp +FetchContent_Declare( + repo-mscclpp + GIT_REPOSITORY https://github.com/microsoft/mscclpp.git + GIT_TAG 51eca89d20f0cfb3764ccd764338d7b22cd486a6 + GIT_SHALLOW OFF +) +FetchContent_Populate(repo-mscclpp) + +# fast-hadamard-transform +FetchContent_Declare( + repo-fast-hadamard-transform + GIT_REPOSITORY https://github.com/sgl-project/fast-hadamard-transform.git + GIT_TAG 48f3c13764dc2ec662ade842a4696a90a137f1bc + GIT_SHALLOW OFF +) +FetchContent_Populate(repo-fast-hadamard-transform) + +# ccache option +option(ENABLE_CCACHE "Whether to use ccache" ON) +find_program(CCACHE_FOUND ccache) +if(CCACHE_FOUND AND ENABLE_CCACHE AND DEFINED ENV{CCACHE_DIR}) + message(STATUS "Building with CCACHE enabled") + set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "ccache") + set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK "ccache") +endif() + +# Enable gencode below SM90 +option(ENABLE_BELOW_SM90 "Enable below SM90" ON) + +if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") + set(ENABLE_BELOW_SM90 OFF) + message(STATUS "For aarch64, disable gencode below SM90 by default") +endif() + +include_directories( + ${PROJECT_SOURCE_DIR}/include + ${PROJECT_SOURCE_DIR}/csrc +) + +set(SGL_KERNEL_CUDA_FLAGS + "-DNDEBUG" + "-DOPERATOR_NAMESPACE=sgl-kernel" + "-O3" + "-Xcompiler" + "-fPIC" + "-gencode=arch=compute_90,code=sm_90" + "-std=c++17" + "-DFLASHINFER_ENABLE_F16" + "-DCUTE_USE_PACKED_TUPLE=1" + "-DCUTLASS_ENABLE_TENSOR_CORE_MMA=1" + "-DCUTLASS_VERSIONS_GENERATED" + "-DCUTLASS_TEST_LEVEL=0" + "-DCUTLASS_TEST_ENABLE_CACHED_RESULTS=1" + "-DCUTLASS_DEBUG_TRACE_LEVEL=0" + "--expt-relaxed-constexpr" + "--expt-extended-lambda" + + # The following flag leads to the CMAKE_BUILD_PARALLEL_LEVEL breaking, + # it triggers OOM with low memory host. Extract the threads number to + # option named SGL_KERNEL_COMPILE_THREADS, default value 32. + # "--threads=32" + + # Supress warnings + "-Xcompiler=-Wno-clang-format-violations" + "-Xcompiler=-Wno-conversion" + "-Xcompiler=-Wno-deprecated-declarations" + "-Xcompiler=-Wno-terminate" + "-Xcompiler=-Wfatal-errors" + "-Xcompiler=-ftemplate-backtrace-limit=1" + "-Xcudafe=--diag_suppress=177" # variable was declared but never referenced + "-Xcudafe=--diag_suppress=2361" # invalid narrowing conversion from "char" to "signed char" + + # uncomment to debug + # "--ptxas-options=-v" + # "--ptxas-options=--verbose,--register-usage-level=10,--warn-on-local-memory-usage" +) + +set(SGL_KERNEL_COMPILE_THREADS 32 CACHE STRING "Set compilation threads, default 32") + +# When SGL_KERNEL_COMPILE_THREADS value is less than 1, set it to 1 +if (NOT SGL_KERNEL_COMPILE_THREADS MATCHES "^[0-9]+$") + message(FATAL_ERROR "SGL_KERNEL_COMPILE_THREADS must be an integer, but was set to '${SGL_KERNEL_COMPILE_THREADS}'.") +elseif (SGL_KERNEL_COMPILE_THREADS LESS 1) + message(STATUS "SGL_KERNEL_COMPILE_THREADS was set to a value less than 1. Using 1 instead.") + set(SGL_KERNEL_COMPILE_THREADS 1) +endif() + +list(APPEND SGL_KERNEL_CUDA_FLAGS + "--threads=${SGL_KERNEL_COMPILE_THREADS}" +) + +option(SGL_KERNEL_ENABLE_BF16 "Enable BF16" ON) +option(SGL_KERNEL_ENABLE_FP8 "Enable FP8" ON) +option(SGL_KERNEL_ENABLE_FP4 "Enable FP4" OFF) +option(SGL_KERNEL_ENABLE_FA3 "Enable FA3" OFF) +option(SGL_KERNEL_ENABLE_SM90A "Enable SM90A" OFF) +option(SGL_KERNEL_ENABLE_SM100A "Enable SM100A" OFF) + +if (SGL_KERNEL_ENABLE_BF16) + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-DFLASHINFER_ENABLE_BF16" + ) +endif() + +if (SGL_KERNEL_ENABLE_FP8) + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-DFLASHINFER_ENABLE_FP8" + "-DFLASHINFER_ENABLE_FP8_E4M3" + "-DFLASHINFER_ENABLE_FP8_E5M2" + ) +endif() + +if (ENABLE_BELOW_SM90) + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-gencode=arch=compute_80,code=sm_80" + "-gencode=arch=compute_89,code=sm_89" + ) + if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-gencode=arch=compute_87,code=sm_87" + ) + endif() + +endif() + +if ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "12.8" OR SGL_KERNEL_ENABLE_SM100A) + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-gencode=arch=compute_100a,code=sm_100a" + "-gencode=arch=compute_120a,code=sm_120a" + ) + # refer sm_121, sm_110 and sm_101 description https://github.com/pytorch/pytorch/pull/156176 + if ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "13.0") + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-gencode=arch=compute_103a,code=sm_103a" + "--compress-mode=size" + ) + if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-gencode=arch=compute_110a,code=sm_110a" + "-gencode=arch=compute_121a,code=sm_121a" + ) + endif() + else() + if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-gencode=arch=compute_101a,code=sm_101a" + ) + endif() + endif() +endif() + +if ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "12.4") + set(SGL_KERNEL_ENABLE_FA3 ON) + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-gencode=arch=compute_90a,code=sm_90a" + ) +endif() + +if ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "12.8" OR SGL_KERNEL_ENABLE_FP4) + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-DENABLE_NVFP4=1" + ) +endif() + +# All source files +# NOTE: Please sort the filenames alphabetically +set(SOURCES + "csrc/allreduce/custom_all_reduce.cu" + "csrc/allreduce/mscclpp_allreduce.cu" + "csrc/attention/cascade.cu" + "csrc/attention/cutlass_mla_kernel.cu" + "csrc/attention/merge_attn_states.cu" + "csrc/attention/vertical_slash_index.cu" + "csrc/common_extension.cc" + "csrc/elementwise/activation.cu" + "csrc/elementwise/cast.cu" + "csrc/elementwise/concat_mla.cu" + "csrc/elementwise/copy.cu" + "csrc/elementwise/fused_add_rms_norm_kernel.cu" + "csrc/elementwise/rope.cu" + "csrc/elementwise/topk.cu" + "csrc/expert_specialization/es_fp8_blockwise.cu" + + "csrc/gemm/awq_kernel.cu" + "csrc/gemm/bmm_fp8.cu" + "csrc/gemm/dsv3_fused_a_gemm.cu" + "csrc/gemm/dsv3_router_gemm_bf16_out.cu" + "csrc/gemm/dsv3_router_gemm_entry.cu" + "csrc/gemm/dsv3_router_gemm_float_out.cu" + "csrc/gemm/fp8_blockwise_gemm_kernel.cu" + "csrc/gemm/fp8_gemm_kernel.cu" + "csrc/gemm/int8_gemm_kernel.cu" + "csrc/gemm/nvfp4_expert_quant.cu" + "csrc/gemm/nvfp4_quant_entry.cu" + "csrc/gemm/nvfp4_quant_kernels.cu" + "csrc/gemm/nvfp4_scaled_mm_entry.cu" + "csrc/gemm/nvfp4_scaled_mm_kernels.cu" + "csrc/gemm/per_tensor_quant_fp8.cu" + "csrc/gemm/per_token_group_quant_8bit.cu" + "csrc/gemm/per_token_group_quant_8bit_v2.cu" + "csrc/gemm/per_token_quant_fp8.cu" + "csrc/gemm/qserve_w4a8_per_chn_gemm.cu" + "csrc/gemm/qserve_w4a8_per_group_gemm.cu" + "csrc/gemm/marlin/gptq_marlin.cu" + "csrc/gemm/marlin/gptq_marlin_repack.cu" + "csrc/gemm/marlin/awq_marlin_repack.cu" + "csrc/gemm/gptq/gptq_kernel.cu" + "csrc/grammar/apply_token_bitmask_inplace_cuda.cu" + + "csrc/kvcacheio/transfer.cu" + "csrc/mamba/causal_conv1d.cu" + "csrc/memory/store.cu" + "csrc/memory/weak_ref_tensor.cpp" + + "csrc/moe/cutlass_moe/w4a8/scaled_mm_entry.cu" + "csrc/moe/cutlass_moe/w4a8/w4a8_moe_data.cu" + "csrc/moe/cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cu" + "csrc/moe/marlin_moe_wna16/ops.cu" + "csrc/moe/moe_align_kernel.cu" + "csrc/moe/moe_fused_gate.cu" + "csrc/moe/fused_qknorm_rope_kernel.cu" + "csrc/moe/kimi_k2_moe_fused_gate.cu" + "csrc/moe/moe_sum.cu" + "csrc/moe/moe_sum_reduce.cu" + "csrc/moe/moe_topk_softmax_kernels.cu" + "csrc/moe/moe_topk_sigmoid_kernels.cu" + "csrc/moe/nvfp4_blockwise_moe.cu" + "csrc/moe/fp8_blockwise_moe_kernel.cu" + "csrc/moe/prepare_moe_input.cu" + + "csrc/quantization/gguf/gguf_kernel.cu" + "csrc/speculative/eagle_utils.cu" + "csrc/speculative/ngram_utils.cu" + "csrc/speculative/packbit.cu" + "csrc/speculative/speculative_sampling.cu" + + "${repo-flashinfer_SOURCE_DIR}/csrc/norm.cu" + "${repo-flashinfer_SOURCE_DIR}/csrc/renorm.cu" + "${repo-flashinfer_SOURCE_DIR}/csrc/sampling.cu" + + "${repo-fast-hadamard-transform_SOURCE_DIR}/csrc/fast_hadamard_transform_cuda.cu" + "${repo-fast-hadamard-transform_SOURCE_DIR}/csrc/fast_hadamard_transform.cpp" + + "${repo-flash-attention_SOURCE_DIR}/csrc/flash_attn/src/flash_fwd_sparse_hdim128_bf16_causal_sm80.cu" + "${repo-flash-attention_SOURCE_DIR}/csrc/flash_attn/src/flash_fwd_sparse_hdim128_bf16_sm80.cu" + "${repo-flash-attention_SOURCE_DIR}/csrc/flash_attn/src/flash_fwd_sparse_hdim128_fp16_causal_sm80.cu" + "${repo-flash-attention_SOURCE_DIR}/csrc/flash_attn/src/flash_fwd_sparse_hdim128_fp16_sm80.cu" + "${repo-flash-attention_SOURCE_DIR}/csrc/flash_attn/flash_sparse_api.cpp" +) + +set(INCLUDES + ${repo-cutlass_SOURCE_DIR}/include + ${repo-cutlass_SOURCE_DIR}/tools/util/include + ${repo-flashinfer_SOURCE_DIR}/include + ${repo-flashinfer_SOURCE_DIR}/csrc + ${repo-mscclpp_SOURCE_DIR}/include + ${repo-cutlass_SOURCE_DIR}/examples/77_blackwell_fmha + ${repo-cutlass_SOURCE_DIR}/examples/common + ${repo-flash-attention_SOURCE_DIR}/csrc/flash_attn/src +) + +# =========================== Common SM90 Build ============================= # +# Build SM90 library with fast math optimization (same namespace, different directory) +Python_add_library(common_ops_sm90_build MODULE USE_SABI ${SKBUILD_SABI_VERSION} WITH_SOABI ${SOURCES}) + +target_compile_options(common_ops_sm90_build PRIVATE + $<$:${SGL_KERNEL_CUDA_FLAGS} -use_fast_math> +) +target_include_directories(common_ops_sm90_build PRIVATE ${INCLUDES}) +# Set output name and separate build directory to avoid conflicts +set_target_properties(common_ops_sm90_build PROPERTIES + OUTPUT_NAME "common_ops" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/sm90" +) + +# =========================== Common SM100+ Build ============================= # +# Build SM100+ library with precise math (same namespace, different directory) +Python_add_library(common_ops_sm100_build MODULE USE_SABI ${SKBUILD_SABI_VERSION} WITH_SOABI ${SOURCES}) + +target_compile_options(common_ops_sm100_build PRIVATE + $<$:${SGL_KERNEL_CUDA_FLAGS}> +) +target_include_directories(common_ops_sm100_build PRIVATE ${INCLUDES}) +# Set output name and separate build directory to avoid conflicts +set_target_properties(common_ops_sm100_build PROPERTIES + OUTPUT_NAME "common_ops" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/sm100" +) + +find_package(Python3 COMPONENTS Interpreter REQUIRED) +execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import torch; print(int(torch._C._GLIBCXX_USE_CXX11_ABI))" + OUTPUT_VARIABLE TORCH_CXX11_ABI + OUTPUT_STRIP_TRAILING_WHITESPACE +) +if(TORCH_CXX11_ABI STREQUAL "0") + message(STATUS "Using old C++ ABI (-D_GLIBCXX_USE_CXX11_ABI=0)") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") +else() + message(STATUS "Using new C++11 ABI (-D_GLIBCXX_USE_CXX11_ABI=1)") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=1") + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=1") +endif() + +# mscclpp option +set(MSCCLPP_USE_CUDA ON) +set(MSCCLPP_BYPASS_GPU_CHECK ON) +set(MSCCLPP_BUILD_TESTS OFF) +add_subdirectory( + ${repo-mscclpp_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR}/mscclpp-build +) + +target_link_libraries(common_ops_sm90_build PRIVATE ${TORCH_LIBRARIES} c10 cuda cublas cublasLt mscclpp_static) +target_link_libraries(common_ops_sm100_build PRIVATE ${TORCH_LIBRARIES} c10 cuda cublas cublasLt mscclpp_static) + +# sparse flash attention +target_compile_definitions(common_ops_sm90_build PRIVATE + FLASHATTENTION_DISABLE_BACKWARD + FLASHATTENTION_DISABLE_DROPOUT + FLASHATTENTION_DISABLE_UNEVEN_K +) +target_compile_definitions(common_ops_sm100_build PRIVATE + FLASHATTENTION_DISABLE_BACKWARD + FLASHATTENTION_DISABLE_DROPOUT + FLASHATTENTION_DISABLE_UNEVEN_K +) + +# Install to different subdirectories +# CMake will find the built libraries in their respective LIBRARY_OUTPUT_DIRECTORY locations +# and install them to the specified destinations +install(TARGETS common_ops_sm90_build LIBRARY DESTINATION sgl_kernel/sm90) +install(TARGETS common_ops_sm100_build LIBRARY DESTINATION sgl_kernel/sm100) + +# ============================ Optional Install: FA3 ============================= # +# set flash-attention sources file +# Now FA3 support sm80/sm86/sm90 +if (SGL_KERNEL_ENABLE_FA3) + set(SGL_FLASH_KERNEL_CUDA_FLAGS + "-DNDEBUG" + "-DOPERATOR_NAMESPACE=sgl-kernel" + "-O3" + "-Xcompiler" + "-fPIC" + "-gencode=arch=compute_90a,code=sm_90a" + "-std=c++17" + "-DCUTE_USE_PACKED_TUPLE=1" + "-DCUTLASS_ENABLE_TENSOR_CORE_MMA=1" + "-DCUTLASS_VERSIONS_GENERATED" + "-DCUTLASS_TEST_LEVEL=0" + "-DCUTLASS_TEST_ENABLE_CACHED_RESULTS=1" + "-DCUTLASS_DEBUG_TRACE_LEVEL=0" + "--expt-relaxed-constexpr" + "--expt-extended-lambda" + "--use_fast_math" + "-Xcompiler=-Wconversion" + "-Xcompiler=-fno-strict-aliasing" + ) + + if (ENABLE_BELOW_SM90) + list(APPEND SGL_FLASH_KERNEL_CUDA_FLAGS + "-gencode=arch=compute_80,code=sm_80" + "-gencode=arch=compute_86,code=sm_86" + ) + # SM8X Logic + file(GLOB FA3_SM8X_GEN_SRCS + "${repo-flash-attention_SOURCE_DIR}/hopper/instantiations/flash_fwd_hdim*_sm80.cu") + endif() + + file(GLOB FA3_BF16_GEN_SRCS + "${repo-flash-attention_SOURCE_DIR}/hopper/instantiations/flash_fwd_hdimall_bf16*_sm90.cu") + file(GLOB FA3_BF16_GEN_SRCS_ + "${repo-flash-attention_SOURCE_DIR}/hopper/instantiations/flash_fwd_hdimdiff_bf16*_sm90.cu") + list(APPEND FA3_BF16_GEN_SRCS ${FA3_BF16_GEN_SRCS_}) + + # FP16 source files + file(GLOB FA3_FP16_GEN_SRCS + "${repo-flash-attention_SOURCE_DIR}/hopper/instantiations/flash_fwd_hdimall_fp16*_sm90.cu") + file(GLOB FA3_FP16_GEN_SRCS_ + "${repo-flash-attention_SOURCE_DIR}/hopper/instantiations/flash_fwd_hdimdiff_fp16*_sm90.cu") + list(APPEND FA3_FP16_GEN_SRCS ${FA3_FP16_GEN_SRCS_}) + + # FP8 source files + file(GLOB FA3_FP8_GEN_SRCS + "${repo-flash-attention_SOURCE_DIR}/hopper/instantiations/flash_fwd_hdimall_e4m3*_sm90.cu") + file(GLOB FA3_FP8_GEN_SRCS_ + "${repo-flash-attention_SOURCE_DIR}/hopper/instantiations/flash_fwd_hdimdiff_e4m3*_sm90.cu") + list(APPEND FA3_FP8_GEN_SRCS ${FA3_FP8_GEN_SRCS_}) + + set(FA3_GEN_SRCS ${FA3_BF16_GEN_SRCS} ${FA3_FP16_GEN_SRCS} ${FA3_FP8_GEN_SRCS} ${FA3_SM8X_GEN_SRCS}) + + set(FLASH_SOURCES + "csrc/flash_extension.cc" + "${repo-flash-attention_SOURCE_DIR}/hopper/flash_prepare_scheduler.cu" + "${repo-flash-attention_SOURCE_DIR}/hopper/flash_api.cpp" + "${repo-flash-attention_SOURCE_DIR}/hopper/flash_fwd_combine.cu" + "${FA3_GEN_SRCS}" + ) + + Python_add_library(flash_ops MODULE USE_SABI ${SKBUILD_SABI_VERSION} WITH_SOABI ${FLASH_SOURCES}) + + target_compile_options(flash_ops PRIVATE $<$:${SGL_FLASH_KERNEL_CUDA_FLAGS}>) + target_include_directories(flash_ops PRIVATE + ${repo-cutlass_SOURCE_DIR}/include + ${repo-cutlass_SOURCE_DIR}/tools/util/include + ${repo-flash-attention_SOURCE_DIR}/hopper + ) + target_link_libraries(flash_ops PRIVATE ${TORCH_LIBRARIES} c10 cuda) + + install(TARGETS flash_ops LIBRARY DESTINATION "sgl_kernel") + set(FLASH_OPS_COMPILE_DEFS + FLASHATTENTION_DISABLE_BACKWARD + FLASHATTENTION_DISABLE_DROPOUT + FLASHATTENTION_DISABLE_UNEVEN_K + FLASHATTENTION_VARLEN_ONLY + ) + + if(NOT ENABLE_BELOW_SM90) + list(APPEND FLASH_OPS_COMPILE_DEFS FLASHATTENTION_DISABLE_SM8x) + endif() + target_compile_definitions(flash_ops PRIVATE ${FLASH_OPS_COMPILE_DEFS}) +endif() + +# Build spatial_ops as a separate, optional extension for green contexts +set(SPATIAL_SOURCES + "csrc/spatial/greenctx_stream.cu" + "csrc/spatial_extension.cc" +) + +Python_add_library(spatial_ops MODULE USE_SABI ${SKBUILD_SABI_VERSION} WITH_SOABI ${SPATIAL_SOURCES}) +target_compile_options(spatial_ops PRIVATE $<$:${SGL_KERNEL_CUDA_FLAGS}>) +target_link_libraries(spatial_ops PRIVATE ${TORCH_LIBRARIES} c10 cuda) +install(TARGETS spatial_ops LIBRARY DESTINATION sgl_kernel) + +# ============================ Extra Install: FLashMLA ============================= # +include(${CMAKE_CURRENT_LIST_DIR}/cmake/flashmla.cmake) + +# ============================ Extra Install: DeepGEMM (JIT) ============================= # +# Create a separate library for DeepGEMM's Python API. +# This keeps its compilation isolated from the main common_ops. +set(DEEPGEMM_SOURCES + "${repo-deepgemm_SOURCE_DIR}/csrc/python_api.cpp" +) + +Python_add_library(deep_gemm_cpp MODULE USE_SABI ${SKBUILD_SABI_VERSION} WITH_SOABI ${DEEPGEMM_SOURCES}) + +# Link against necessary libraries, including nvrtc for JIT compilation. +target_link_libraries(deep_gemm_cpp PRIVATE ${TORCH_LIBRARIES} c10 cuda nvrtc mscclpp_static) + +# Add include directories needed by DeepGEMM. +target_include_directories(deep_gemm_cpp PRIVATE + ${repo-deepgemm_SOURCE_DIR}/deep_gemm/include + ${repo-cutlass_SOURCE_DIR}/include + ${repo-fmt_SOURCE_DIR}/include +) + +# Apply the same compile options as common_ops. +target_compile_options(deep_gemm_cpp PRIVATE $<$:${SGL_KERNEL_CUDA_FLAGS}>) + +# Create an empty __init__.py to make `deepgemm` a Python package. +file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/deepgemm_pkg_init.py "") +install( + FILES ${CMAKE_CURRENT_BINARY_DIR}/deepgemm_pkg_init.py + DESTINATION deep_gemm + RENAME __init__.py +) + +# Install the compiled DeepGEMM API library. +install(TARGETS deep_gemm_cpp LIBRARY DESTINATION deep_gemm) + +# Install the source files required by DeepGEMM for runtime JIT compilation. +install( + DIRECTORY ${repo-deepgemm_SOURCE_DIR}/deep_gemm/ + DESTINATION deep_gemm +) + +install(DIRECTORY "${repo-cutlass_SOURCE_DIR}/include/cute/" + DESTINATION "deep_gemm/include/cute") + +install(DIRECTORY "${repo-cutlass_SOURCE_DIR}/include/cutlass/" + DESTINATION "deep_gemm/include/cutlass") + +# ============================ Extra Install: triton kernels ============================= # +install(DIRECTORY "${repo-triton_SOURCE_DIR}/python/triton_kernels/triton_kernels/" + DESTINATION "triton_kernels" + PATTERN ".git*" EXCLUDE + PATTERN "__pycache__" EXCLUDE) + +# ============================ Extra Install: FA4 ============================= # +# TODO: find a better install condition. +if ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "12.8" OR SGL_KERNEL_ENABLE_SM100A) + # flash_attn/cute + install(DIRECTORY "${repo-flash-attention_SOURCE_DIR}/flash_attn/cute/" + DESTINATION "flash_attn/cute" + PATTERN ".git*" EXCLUDE + PATTERN "__pycache__" EXCLUDE) + endif() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/LICENSE b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/LICENSE new file mode 100644 index 00000000..9c422689 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2023-2024 SGLang Team + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/Makefile b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/Makefile new file mode 100644 index 00000000..8a81ae80 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/Makefile @@ -0,0 +1,69 @@ +.PHONY: help check-deps install-deps tree ln submodule install build clean rebuild test format update + +# Show help for each target +help: ## Show this help message + @echo "Available targets:" + @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' + +check-deps: ## Check and install required Python formatting dependencies + @command -v isort >/dev/null 2>&1 || (echo "Installing isort..." && pip install isort) + @command -v black >/dev/null 2>&1 || (echo "Installing black..." && pip install black) + +install-deps: ## Install Python formatting tools (isort and black) + pip install scikit-build-core isort black + +tree: ## Show project directory structure + @tree --prune -I "__pycache__|*.egg-info|*.so|build|3rdparty|dist" + +submodule: ## Initialize and update git submodules + @git submodule update --init --recursive + +ln: submodule ## Create compilation database + @rm -rf build && mkdir build && cd build && cmake .. -DCMAKE_EXPORT_COMPILE_COMMANDS=YES -DCMAKE_POLICY_VERSION_MINIMUM=3.5 + +install: submodule ## Install package in development mode + @pip install -e . --no-build-isolation + +build: install-deps submodule ## Build and install wheel package + @rm -rf dist/* || true && CMAKE_POLICY_VERSION_MINIMUM=3.5 MAX_JOBS=$(nproc) CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) uv build --wheel -Cbuild-dir=build . --verbose --color=always --no-build-isolation && pip3 install dist/*whl --force-reinstall --no-deps + +clean: ## Remove build artifacts + @rm -rf build dist *.egg-info + +rebuild: clean submodule build ## Clean and rebuild the project + @echo "Succeed to rebuild" + +test: ## Run all tests + @find tests -name "test_*.py" | xargs -n 1 python3 + +format: check-deps ## Format all source files + @echo "Formatting source files..." + @find csrc tests -name '*.cc' -o -name '*.cu' -o -name '*.cuh' -o -name '*.h' -o -name '*.hpp' | xargs clang-format -i + @find python tests -name '*.py' | xargs isort + @find python tests -name '*.py' | xargs black + @pre-commit run --all-files + +FILES_TO_UPDATE = python/sgl_kernel/version.py \ + pyproject.toml \ + pyproject_rocm.toml \ + pyproject_cpu.toml + +update: ## Update version numbers across project files. Usage: make update + @if [ -z "$(filter-out $@,$(MAKECMDGOALS))" ]; then \ + echo "Version required. Usage: make update "; \ + exit 1; \ + fi + @OLD_VERSION=$$(grep "version" python/sgl_kernel/version.py | cut -d '"' -f2); \ + NEW_VERSION=$(filter-out $@,$(MAKECMDGOALS)); \ + echo "Updating version from $$OLD_VERSION to $$NEW_VERSION"; \ + for file in $(FILES_TO_UPDATE); do \ + if [ "$(shell uname)" = "Darwin" ]; then \ + sed -i '' -e "s/$$OLD_VERSION/$$NEW_VERSION/g" $$file; \ + else \ + sed -i -e "s/$$OLD_VERSION/$$NEW_VERSION/g" $$file; \ + fi \ + done; \ + echo "Version update complete" + +%: + @: diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/README.md new file mode 100644 index 00000000..51fe3ad3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/README.md @@ -0,0 +1,107 @@ +# SGL Kernel + +[Kernel Library](https://github.com/sgl-project/sglang/tree/main/sgl-kernel) for SGLang + +
+ +[![License: Apache-2.0](https://img.shields.io/badge/License-Apache--2.0-blue.svg)](https://github.com/sgl-project/sglang/blob/main/LICENSE) +[![PyPI](https://img.shields.io/pypi/v/sgl-kernel)](https://pypi.org/project/sgl-kernel) + +
+ +SGL Kernel provides optimized compute primitives for the SGLang framework, enabling efficient inference for large language models and vision-language models through custom kernel operations. + +## Installation +Requires torch == 2.9.1 + +```bash +# Latest version +pip3 install sgl-kernel --upgrade +``` + +## Building from Source +Requires +- CMake ≥3.31, +- Python ≥3.10 +- scikit-build-core +- ninja(optional) + +### Use Makefile to build sgl-kernel + +```bash +make build +``` + +## Contribution + +### Steps to add a new kernel: + +1. Implement the kernel in [csrc](https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc) +2. Expose the interface in [include/sgl_kernel_ops.h](https://github.com/sgl-project/sglang/blob/main/sgl-kernel/include/sgl_kernel_ops.h) +3. Create torch extension in [csrc/common_extension.cc](https://github.com/sgl-project/sglang/blob/main/sgl-kernel/csrc/common_extension.cc) +4. Update [CMakeLists.txt](https://github.com/sgl-project/sglang/blob/main/sgl-kernel/CMakeLists.txt) to include new CUDA source +5. Expose Python interface in [python](https://github.com/sgl-project/sglang/blob/main/sgl-kernel/python/sgl_kernel) +6. Add test and benchmark + +### Development Tips + +1. When creating torch extensions, add the function definition with `m.def`, and device binding with `m.impl`: + +- How to write schema: [Schema reference](https://github.com/pytorch/pytorch/blob/main/aten/src/ATen/native/README.md#func) + + ```cpp + // We need def with schema here for torch.compile + m.def( + "bmm_fp8(Tensor A, Tensor B, Tensor! D, Tensor A_scale, Tensor B_scale, Tensor workspace_buffer, " + "int cublas_handle) -> ()"); + m.impl("bmm_fp8", torch::kCUDA, &bmm_fp8); + ``` + +### Adapting C++ Native Types for Torch Compatibility + +Third-party C++ libraries often use int and float, but PyTorch bindings require int64_t and double due to Python's type mapping. + +Use make_pytorch_shim from sgl_kernel_torch_shim.h to handle conversions automatically: + +```cpp + +// Add type conversion for int -> int64_t +template <> +struct pytorch_library_compatible_type { + using type = int64_t; + static int convert_from_type(int64_t arg) { + TORCH_CHECK(arg <= std::numeric_limits::max(), "value too large"); + TORCH_CHECK(arg >= std::numeric_limits::min(), "value too small"); + return arg; + } +}; +``` +```cpp +// Wrap your function +m.impl("fwd", torch::kCUDA, make_pytorch_shim(&mha_fwd)); +``` + +### Testing & Benchmarking + +1. Add pytest tests in [tests/](https://github.com/sgl-project/sglang/tree/main/sgl-kernel/tests), if you need to skip some test, please use `@pytest.mark.skipif` + +```python +@pytest.mark.skipif( + skip_condition, reason="Nvfp4 Requires compute capability of 10 or above." +) +``` + +2. Add benchmarks using [triton benchmark](https://triton-lang.org/main/python-api/generated/triton.testing.Benchmark.html) in [benchmark/](https://github.com/sgl-project/sglang/tree/main/sgl-kernel/benchmark) + + **We recommend using `triton.testing.do_bench_cudagraph` for kernel benchmarking**: + + Compared to `triton.testing.do_bench`, `do_bench_cudagraph` provides: + - Reduced CPU overhead impact for more accurate kernel performance measurements + - Incorporation of PDL (Programmatic Dependent Launch) effects into individual kernel results + - More realistic performance data on PDL-supported architectures (SM >= 90) + +3. Run test suite + +## FAQ +- Q: Segmentation fault with CUDA 12.6 +- A: Update ptxas to 12.8, reference: [segment fault error](https://github.com/Dao-AILab/flash-attention/issues/1453) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/THIRDPARTYNOTICES.txt b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/THIRDPARTYNOTICES.txt new file mode 100644 index 00000000..2a81411a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/THIRDPARTYNOTICES.txt @@ -0,0 +1,488 @@ +Notice for flashinfer-ai/flashinfer +------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +------------------------------------------------------------------------------------------------- +Some of the code in this project are adapted from other open-source projects with different +licenses. This product also bundles some third-party components under other open source licenses. +This section summarizes those components and their licenses. +See licenses/ for text of these licenses. + +BSD 3-Clause License +-------------------- + +include/flashinfer/attention/hopper/epilogue.cuh +include/flashinfer/attention/hopper/mainloop.cuh +include/flashinfer/attention/hopper/kernel_traits.cuh +include/flashinfer/attention/hopper/named_barrier.cuh +include/flashinfer/attention/hopper/tile_scheduler.cuh +include/flashinfer/attention/hopper/utils.cuh + +BSD 3-Clause "New" License +-------------------------- + +3rdparty/cutlass +include/flashinfer/attention/hopper/block_sparse_gather.cuh + +Notice for NVIDIA/TensorRT-LLM +------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Notice for deepseek-ai/DeepGEMM +------------------------------- + +MIT License + +Copyright (c) 2025 DeepSeek + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Notice for Dao-AILab/flash-attention +------------------------------- + +BSD 3-Clause License + +Copyright (c) 2022, the respective contributors, as shown by the AUTHORS file. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_activation.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_activation.py new file mode 100644 index 00000000..3caa5b93 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_activation.py @@ -0,0 +1,211 @@ +# Benchmarks SGLang kernels versus vLLM across +# (kernel, dtype, batch_size, seq_len, dim) and prints speed-up. +import argparse +import itertools +import os +import re +from typing import List, Tuple + +import sgl_kernel +import torch +import torch.nn.functional as F +import triton +import triton.testing +from sgl_kernel import gelu_and_mul, gelu_tanh_and_mul, silu_and_mul + +# Optional vLLM import +try: + from vllm import _custom_ops as vllm_ops + + VLLM_AVAILABLE = True +except ImportError: + vllm_ops = None + VLLM_AVAILABLE = False + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + +# gelu_quick is only available on HIP/ROCm platforms +try: + from sgl_kernel import gelu_quick + + GELU_QUICK_AVAILABLE = True +except ImportError: + GELU_QUICK_AVAILABLE = False + gelu_quick = None + +if VLLM_AVAILABLE and not hasattr(vllm_ops, "silu_and_mul"): + vllm_ops = torch.ops._C + + +def str2int_list(arg: str) -> List[int]: + if arg in ("", None): + return [] + if re.fullmatch(r"\d+(,\d+)*", arg.strip()) is None: + raise argparse.ArgumentTypeError(f"Bad int list: {arg}") + return [int(x) for x in arg.split(",")] + + +def calculate_diff( + kernel: str, dtype: torch.dtype, batch_size: int, seq_len: int, dim: int +) -> bool: + """Compare vLLM with SGLang for one shape.""" + device = torch.device("cuda") + + if not VLLM_AVAILABLE: + print( + f"[{kernel:14s} | {str(dtype):9s} | B={batch_size:3d} | " + f"L={seq_len:3d} | D={dim:5d}] ⚠️ vLLM not available, skipping comparison" + ) + return True + + # activation-only quick GELU + if kernel == "gelu_quick": + if not GELU_QUICK_AVAILABLE: + print( + f"[{kernel:14s} | {str(dtype):9s} | B={batch_size:3d} | " + f"L={seq_len:3d} | D={dim:5d}] ⚠️ not available on this platform" + ) + return True + x = torch.randn(batch_size, seq_len, dim, dtype=dtype, device=device) + ref_out = torch.zeros_like(x) + getattr(vllm_ops, kernel)(ref_out, x) + test_out = getattr(sgl_kernel, kernel)(x) + # fused activation x mul kernels + else: + x = torch.randn(batch_size, seq_len, 2 * dim, dtype=dtype, device=device) + ref_out = torch.zeros(batch_size, seq_len, dim, dtype=dtype, device=device) + getattr(vllm_ops, kernel)(ref_out, x) + test_out = getattr(sgl_kernel, kernel)(x) + + ok = torch.allclose(ref_out, test_out, rtol=1e-3, atol=1e-5) + tag = "✅ match" if ok else "❌ mismatch" + print( + f"[{kernel:14s} | {str(dtype):9s} | B={batch_size:3d} | " + f"L={seq_len:3d} | D={dim:5d}] {tag}" + ) + return ok + + +# CI environment uses simplified parameters for kernels and dtypes too +if IS_CI: + kernels = ["silu_and_mul"] # Only test one kernel in CI + dtypes = [torch.float16] # Only test one dtype in CI +else: + kernels = ["silu_and_mul", "gelu_and_mul", "gelu_tanh_and_mul"] + if GELU_QUICK_AVAILABLE: + kernels.append("gelu_quick") + dtypes = [torch.float16, torch.bfloat16] + + +def make_configs(bsizes: List[int], slens: List[int], dims_: List[int]) -> List[Tuple]: + return list(itertools.product(kernels, dtypes, bsizes, slens, dims_)) + + +# CI environment uses simplified parameters +if IS_CI: + default_batch_sizes = [1] # Single batch size for CI + default_seq_lens = [1] # Single sequence length for CI + default_dims = [1024] # Single dimension for CI +else: + default_batch_sizes = [2**i for i in range(0, 5, 2)] # 1,4,16 + default_seq_lens = [2**i for i in range(0, 8, 2)] # 1,4,16,64 + default_dims = [2**i for i in range(10, 15)] # 1024...16384 + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["kernel", "dtype", "batch_size", "seq_len", "dim"], + x_vals=[], + line_arg="provider", + line_vals=["vllm", "sglang", "speedup"], + line_names=["vLLM", "SGL Kernel", "Speed-up (x)"], + styles=[("blue", "-"), ("green", "-"), ("red", "--")], + ylabel="µs (median) or × (speed-up)", + plot_name="activation-performance", + args={}, + ) +) +def benchmark(kernel, dtype, batch_size, seq_len, dim, provider): + device = torch.device("cuda") + in_mult = 1 if kernel == "gelu_quick" else 2 + x = torch.randn(batch_size, seq_len, in_mult * dim, dtype=dtype, device=device) + y0 = torch.zeros(batch_size, seq_len, dim, dtype=dtype, device=device) + + if not VLLM_AVAILABLE and provider in ["vllm", "speedup"]: + # Skip vLLM-related benchmarks if vLLM is not available + return (0, 0, 0) + + if VLLM_AVAILABLE: + vllm_kernel = getattr(vllm_ops, kernel) + if kernel == "gelu_quick" and not GELU_QUICK_AVAILABLE: + # Skip benchmark for gelu_quick if not available + return (0, 0, 0) + sglang_kernel = getattr(sgl_kernel, kernel) + + def baseline(): + if VLLM_AVAILABLE: + tmp = y0.clone() + vllm_kernel(tmp, x) + return tmp + else: + return torch.zeros_like(y0) + + def sglang(): + return sglang_kernel(x) + + # timing helper + def timed(fn): + for _ in range(5): + fn() + torch.cuda.synchronize() + ms, qmin, qmax = triton.testing.do_bench_cudagraph( + fn, quantiles=[0.5, 0.2, 0.8] + ) + return 1000 * ms, 1000 * qmax, 1000 * qmin + + if provider == "vllm": + return timed(baseline) + if provider == "sglang": + return timed(sglang) + + # provider == "speedup" + t_ref, _, _ = timed(baseline) + t_sgl, _, _ = timed(sglang) + spd = t_ref / t_sgl if t_ref > 0 else 1.0 + return (spd, spd, spd) + + +if __name__ == "__main__": + p = argparse.ArgumentParser("Activation kernel benchmark") + p.add_argument("--batch_sizes", type=str2int_list, default=default_batch_sizes) + p.add_argument("--seq_lens", type=str2int_list, default=default_seq_lens) + p.add_argument("--dims", type=str2int_list, default=default_dims) + p.add_argument("--verify_only", action="store_true") + args = p.parse_args() + + # coerce lists + if isinstance(args.batch_sizes, str): + args.batch_sizes = str2int_list(args.batch_sizes) + if isinstance(args.seq_lens, str): + args.seq_lens = str2int_list(args.seq_lens) + if isinstance(args.dims, str): + args.dims = str2int_list(args.dims) + + # patch perf_report grid + benchmark_grid = make_configs(args.batch_sizes, args.seq_lens, args.dims) + if hasattr(benchmark, "benchmarks"): + benchmark.benchmarks.x_vals = benchmark_grid + else: + benchmark.benchmark.x_vals = benchmark_grid + + if args.verify_only: + # Test with the first available kernel + test_kernel = kernels[0] + ok = calculate_diff(test_kernel, torch.float16, 1, 1, args.dims[0]) + print("✅ sanity pass" if ok else "❌ mismatch") + else: + benchmark.run(print_data=True) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_awq_dequant.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_awq_dequant.py new file mode 100644 index 00000000..6bd03ab8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_awq_dequant.py @@ -0,0 +1,153 @@ +import itertools +import os +from typing import List, Tuple + +import torch +import triton +import triton.testing +from sgl_kernel import awq_dequantize + +# Optional vLLM import +try: + from vllm import _custom_ops as ops + + VLLM_AVAILABLE = True +except ImportError: + ops = None + VLLM_AVAILABLE = False + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + + +def vllm_awq_dequantize( + qweight: torch.Tensor, scales: torch.Tensor, qzeros: torch.Tensor +) -> Tuple[torch.Tensor, torch.Tensor]: + if not VLLM_AVAILABLE: + # Fallback to SGLang implementation + return sglang_awq_dequantize(qweight, scales, qzeros) + return ops.awq_dequantize(qweight, scales, qzeros, 0, 0, 0) + + +def sglang_awq_dequantize( + qweight: torch.Tensor, scales: torch.Tensor, qzeros: torch.Tensor +) -> Tuple[torch.Tensor, torch.Tensor]: + + return awq_dequantize(qweight, scales, qzeros) + + +def calculate_diff(qweight_row: int, qweight_col: int): + """Calculate difference between VLLM and SGLang implementations.""" + device = torch.device("cuda") + qweight = torch.randint( + 0, + torch.iinfo(torch.int32).max, + (qweight_row, qweight_col), + dtype=torch.int32, + device=device, + ) + group_size = qweight_row + scales_row = qweight_row // group_size + scales_col = qweight_col * 8 + scales = torch.rand(scales_row, scales_col, dtype=torch.float16, device=device) + qzeros = torch.randint( + 0, + torch.iinfo(torch.int32).max, + (scales_row, qweight_col), + dtype=torch.int32, + device=device, + ) + + if not VLLM_AVAILABLE: + print("⚠️ vLLM not available, skipping comparison") + return + + vllm_out = vllm_awq_dequantize(qweight, scales, qzeros) + sglang_out = sglang_awq_dequantize(qweight, scales, qzeros) + + output_diff = torch.abs(vllm_out.float() - sglang_out.float()).mean().item() + + if torch.allclose( + vllm_out.to(torch.float32), sglang_out.to(torch.float32), rtol=1e-3, atol=1e-5 + ): + print("✅ All implementations match") + else: + print("❌ Implementations differ") + + +# CI environment uses simplified parameters +if IS_CI: + qweight_row_range = [128] # Single row size for CI + qweight_cols_range = [16] # Single column size for CI +else: + qweight_row_range = [3584, 18944, 128, 256, 512, 1024] + qweight_cols_range = [448, 576, 4736, 16, 32, 64, 128] + +configs = list(itertools.product(qweight_row_range, qweight_cols_range)) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["qweight_row", "qweight_col"], + x_vals=configs, + line_arg="provider", + line_vals=["vllm", "sglang"] if VLLM_AVAILABLE else ["sglang"], + line_names=["VLLM", "SGL Kernel"] if VLLM_AVAILABLE else ["SGL Kernel"], + styles=[("blue", "-"), ("green", "-")] if VLLM_AVAILABLE else [("green", "-")], + ylabel="us", + plot_name="awq-dequantize-performance", + args={}, + ) +) +def benchmark(qweight_row, qweight_col, provider): + dtype = torch.float16 + device = torch.device("cuda") + qweight = torch.randint( + 0, + torch.iinfo(torch.int32).max, + (qweight_row, qweight_col), + dtype=torch.int32, + device=device, + ) + group_size = qweight_row + scales_row = qweight_row // group_size + scales_col = qweight_col * 8 + scales = torch.rand(scales_row, scales_col, dtype=torch.float16, device=device) + qzeros = torch.randint( + 0, + torch.iinfo(torch.int32).max, + (scales_row, qweight_col), + dtype=torch.int32, + device=device, + ) + + quantiles = [0.5, 0.2, 0.8] + + if provider == "vllm": + if not VLLM_AVAILABLE: + return (0, 0, 0) + fn = lambda: vllm_awq_dequantize( + qweight.clone(), scales.clone(), qzeros.clone() + ) + elif provider == "sglang": + fn = lambda: sglang_awq_dequantize( + qweight.clone(), scales.clone(), qzeros.clone() + ) + + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph(fn, quantiles=quantiles) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + +if __name__ == "__main__": + # Simplify for CI environment + if IS_CI: + qweight_row, qweight_col = 128, 16 # Smaller values for CI + else: + qweight_row, qweight_col = 3584, 448 + + calculate_diff(qweight_row=qweight_row, qweight_col=qweight_col) + benchmark.run(print_data=True) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_cutlass_mla.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_cutlass_mla.py new file mode 100644 index 00000000..6947f309 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_cutlass_mla.py @@ -0,0 +1,178 @@ +import argparse +import copy +import itertools +import os + +import torch +import triton +from sgl_kernel import cutlass_mla_decode, cutlass_mla_get_workspace_size + +from sglang.srt.utils import get_device_capability + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + +# CI environment uses simplified parameters +if IS_CI: + bs_range = [1] # Single batch size for CI + qlen_range = [64] # Single sequence length for CI +else: + bs_range = [1, 8, 32, 64, 128, 256] + qlen_range = [1, 64, 128, 256, 512, 1024, 2048, 4096, 8192] + +configs = list(itertools.product(bs_range, qlen_range)) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size", "seq_len"], + x_vals=configs, + x_log=False, + line_arg="provider", + line_vals=[ + "128 heads", + "64 heads", + "32 heads", + "16 heads", + ], + line_names=[ + "128 heads", + "64 heads", + "32 heads", + "16 heads", + ], + styles=[("green", "-"), ("green", "--"), ("blue", "-"), ("blue", "--")], + ylabel="GB/s", + plot_name="cutlass mla", + args={}, + ) +) +def benchmark(batch_size, seq_len, provider, block_size, num_kv_splits): + d = 576 + dn = 64 + dv = 512 + + h_q_map = { + "128": 128, + "64": 64, + "32": 32, + "16": 16, + } + parsed_h_q = next( + (value for key, value in h_q_map.items() if key in provider), None + ) + + if parsed_h_q is None: + raise ValueError(f"Unknown head configuration in provider: {provider}") + h_q = parsed_h_q + + seq_lens = torch.full((batch_size,), seq_len, dtype=torch.int32, device="cuda") + max_seq_len = seq_lens.max().item() + block_num = (max_seq_len + block_size - 1) // block_size + + # Pad block_num so that small blocks can be packed into full 128-sized CUTLASS tiles. + # One 128-wide tile can hold (128 // block_size) small blocks. + pack_factor = 128 // block_size + block_num = ((block_num + pack_factor - 1) // pack_factor) * pack_factor + + qn = ( + torch.randn(h_q, batch_size, d - dn, dtype=torch.bfloat16, device="cuda") + * 100.0 + ) + qr = torch.randn(batch_size, h_q, dn, dtype=torch.bfloat16, device="cuda") * 100.0 + block_table = torch.randint( + 0, + batch_size * block_num, + (batch_size, block_num), + dtype=torch.int32, + device="cuda", + ) + + kv_cache = torch.randn( + block_table.numel(), block_size, d, dtype=torch.bfloat16, device="cuda" + ) + + workspace_size = cutlass_mla_get_workspace_size( + block_num * block_size, batch_size, num_kv_splits=num_kv_splits + ) + workspace = torch.empty(workspace_size, device="cuda", dtype=torch.uint8) + + quantiles = [0.5, 0.2, 0.8] + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: cutlass_mla_decode( + qn.transpose(0, 1), + qr, + kv_cache, + seq_lens, + block_table, + workspace, + 1.44, + num_kv_splits, + ), + quantiles=quantiles, + ) + + q_size = qn.numel() * qn.element_size() + qr.numel() * qr.element_size() + + gbps = ( + lambda ms: ( + q_size + q_size * dv / d + kv_cache.numel() * kv_cache.element_size() + ) + * 1e-9 + / (ms * 1e-3) + ) + return gbps(ms), gbps(max_ms), gbps(min_ms) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--block-sizes", + nargs="+", + type=int, + default=[1, 32, 64, 128], + help="List of batch sizes", + ) + parser.add_argument( + "--num-kv-splits", + nargs="+", + type=int, + default=[-1], + help="List of batch sizes", + ) + args = parser.parse_args() + + # Skip in CI environment or unsupported architectures + if IS_CI: + major, minor = get_device_capability() + if major is None or major < 10: # Requires compute capability 10.0+ + print("Skipping Cutlass MLA benchmark in CI environment") + if major is not None: + print( + f"Cutlass MLA requires compute capability 10.0+, but found {major}.{minor}" + ) + else: + print("Could not determine device capability") + else: + for block_size in args.block_sizes: + for kv_split in args.num_kv_splits: + print(f"block_size={block_size}, num_kv_splits={kv_split}: ") + benchmark.run( + print_data=True, + block_size=block_size, + num_kv_splits=kv_split, + ) + print("Benchmark finished!") + else: + for block_size in args.block_sizes: + for kv_split in args.num_kv_splits: + print(f"block_size={block_size}, num_kv_splits={kv_split}: ") + benchmark.run( + print_data=True, + block_size=block_size, + num_kv_splits=kv_split, + ) + print("Benchmark finished!") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_dsv3_fused_a_gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_dsv3_fused_a_gemm.py new file mode 100644 index 00000000..bdf7f85d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_dsv3_fused_a_gemm.py @@ -0,0 +1,76 @@ +import argparse +import os + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + +import torch +import torch.nn.functional as F +import triton +import triton.testing +from sgl_kernel import dsv3_fused_a_gemm + +# CI environment uses simplified parameters +if IS_CI: + num_tokens_vals = [1] # Only test 1 value in CI + line_vals = ["sgl-kernel"] # Only test sgl-kernel implementation in CI +else: + num_tokens_vals = [i + 1 for i in range(16)] # Test 1-16 in full mode + line_vals = ["torch", "sgl-kernel"] + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["num_tokens"], + x_vals=num_tokens_vals, + x_log=False, + line_arg="impl", + line_vals=line_vals, + line_names=( + ["torch (bf16)", "dsv3_fused_a_gemm"] + if not IS_CI + else ["dsv3_fused_a_gemm"] + ), + styles=[("blue", "-"), ("orange", "-")] if not IS_CI else [("orange", "-")], + ylabel="TFLOPs", + plot_name="bf16 dsv3 fused a GEMM throughput", + args={}, + ) +) +def benchmark(num_tokens, impl): + kHdIn = 7168 + kHdOut = 2112 + M, K, N = num_tokens, kHdIn, kHdOut + + mat_a = torch.randn((M, K), dtype=torch.bfloat16, device="cuda").contiguous() + mat_b = torch.randn((N, K), dtype=torch.bfloat16, device="cuda").transpose(0, 1) + + quantiles = [0.5, 0.2, 0.8] + + if impl == "torch": + + def runner(): + F.linear(mat_a, mat_b.T) + + elif impl == "sgl-kernel": + + def runner(): + dsv3_fused_a_gemm(mat_a, mat_b) + + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph(runner, quantiles=quantiles) + + def tflops(t_ms): + flops = 2 * M * K * N + return flops / (t_ms * 1e-3) / 1e12 + + return tflops(ms), tflops(max_ms), tflops(min_ms) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + args = parser.parse_args() + + benchmark.run(print_data=True) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_dsv3_router_gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_dsv3_router_gemm.py new file mode 100644 index 00000000..2daee279 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_dsv3_router_gemm.py @@ -0,0 +1,154 @@ +import argparse +import os + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + +import torch +import torch.nn.functional as F +import triton +import triton.testing +from sgl_kernel import dsv3_router_gemm + +# CI environment uses simplified parameters +if IS_CI: + num_tokens_vals = [1] # Only test 1 value in CI + line_vals = ["sgl-kernel-256"] # Only test one implementation in CI +else: + num_tokens_vals = [i + 1 for i in range(16)] # Test 1-16 in full mode + line_vals = ["torch-256", "sgl-kernel-256", "torch-384", "sgl-kernel-384"] + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["num_tokens"], + x_vals=num_tokens_vals, + x_log=False, + line_arg="impl", + line_vals=line_vals, + line_names=( + [ + "torch-256", + "dsv3_router_gemm-256", + "torch-384", + "dsv3_router_gemm-384", + ] + if not IS_CI + else ["dsv3_router_gemm-256"] + ), + styles=( + [("blue", "-"), ("orange", "-"), ("green", "-"), ("red", "-")] + if not IS_CI + else [("orange", "-")] + ), + ylabel="TFLOPs", + plot_name="input-bf16-output-bf16 dsv3 router gemm throughput", + args={}, + ) +) +def benchmark_bf16_output(num_tokens, impl): + # M: num_tokens, K: hidden_dim, N: num_experts + M, K = num_tokens, 7168 + + if impl == "torch-256" or impl == "sgl-kernel-256": + N = 256 + elif impl == "torch-384" or impl == "sgl-kernel-384": + N = 384 + else: + raise ValueError(f"Unknown impl: {impl}") + + mat_a = torch.randn((M, K), dtype=torch.bfloat16, device="cuda").contiguous() + mat_b = torch.randn((N, K), dtype=torch.bfloat16, device="cuda").contiguous() + + quantiles = [0.5, 0.2, 0.8] + + if impl == "torch-256" or impl == "torch-384": + + def runner(): + F.linear(mat_a, mat_b) + + elif impl == "sgl-kernel-256" or impl == "sgl-kernel-384": + + def runner(): + dsv3_router_gemm(mat_a, mat_b, out_dtype=torch.bfloat16) + + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph(runner, quantiles=quantiles) + + def tflops(t_ms): + flops = 2 * M * K * N + return flops / (t_ms * 1e-3) / 1e12 + + return tflops(ms), tflops(max_ms), tflops(min_ms) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["num_tokens"], + x_vals=num_tokens_vals, + x_log=False, + line_arg="impl", + line_vals=line_vals, + line_names=( + [ + "torch-256", + "dsv3_router_gemm-256", + "torch-384", + "dsv3_router_gemm-384", + ] + if not IS_CI + else ["dsv3_router_gemm-256"] + ), + styles=( + [("blue", "-"), ("orange", "-"), ("green", "-"), ("red", "-")] + if not IS_CI + else [("orange", "-")] + ), + ylabel="TFLOPs", + plot_name="input-bf16-output-fp32 dsv3 router gemm throughput", + args={}, + ) +) +def benchmark_float_output(num_tokens, impl): + # M: num_tokens, K: hidden_dim, N: num_experts + M, K = num_tokens, 7168 + + if impl == "torch-256" or impl == "sgl-kernel-256": + N = 256 + elif impl == "torch-384" or impl == "sgl-kernel-384": + N = 384 + else: + raise ValueError(f"Unknown impl: {impl}") + + mat_a = torch.randn((M, K), dtype=torch.bfloat16, device="cuda").contiguous() + mat_b = torch.randn((N, K), dtype=torch.bfloat16, device="cuda").contiguous() + + quantiles = [0.5, 0.2, 0.8] + + if impl == "torch-256" or impl == "torch-384": + + def runner(): + F.linear(mat_a, mat_b).to(torch.float32) + + elif impl == "sgl-kernel-256" or impl == "sgl-kernel-384": + + def runner(): + dsv3_router_gemm(mat_a, mat_b, out_dtype=torch.float32) + + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph(runner, quantiles=quantiles) + + def tflops(t_ms): + flops = 2 * M * K * N + return flops / (t_ms * 1e-3) / 1e12 + + return tflops(ms), tflops(max_ms), tflops(min_ms) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + args = parser.parse_args() + + benchmark_bf16_output.run(print_data=True) + benchmark_float_output.run(print_data=True) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_es_fp8_blockwise_grouped_gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_es_fp8_blockwise_grouped_gemm.py new file mode 100644 index 00000000..7591c5dd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_es_fp8_blockwise_grouped_gemm.py @@ -0,0 +1,338 @@ +import argparse +import random +from dataclasses import dataclass +from typing import List, Tuple + +import numpy as np +import torch +from sgl_kernel import ( + es_fp8_blockwise_scaled_grouped_mm, + fp8_blockwise_scaled_grouped_mm, +) + +random.seed(28) + + +def ceil_div(x: int, y: int) -> int: + return (x + y - 1) // y + + +def per_token_cast_to_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 + m, n = x.shape + pad_size = (128 - (n % 128)) % 128 + x = torch.nn.functional.pad(x, (0, pad_size), value=0) if pad_size > 0 else x + x_view = x.view(m, -1, 128) + x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4) + fp8_data = (x_view * (448.0 / x_amax.unsqueeze(2))).to(torch.float8_e4m3fn) + return fp8_data.view(m, n + pad_size)[:, :n], (x_amax / 448.0).view(m, -1) + + +def per_block_cast_to_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 + m, n = x.shape + x_padded = torch.zeros( + (ceil_div(m, 128) * 128, ceil_div(n, 128) * 128), dtype=x.dtype, device=x.device + ) + x_padded[:m, :n] = x + x_view = x_padded.view(-1, 128, x_padded.size(1) // 128, 128) + x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4) + x_scaled = (x_view * (448.0 / x_amax)).to(torch.float8_e4m3fn) + return x_scaled.view_as(x_padded)[:m, :n].contiguous(), (x_amax / 448.0).view( + x_view.size(0), x_view.size(2) + ) + + +def create_unbalanced_expert_token_distribution(max_num_experts): + ratios = [random.random() for _ in range(max_num_experts)] + + def convert_to_tokens(ratio: float): + if ratio <= 0.7: + return random.randint(1, 32) + elif ratio > 0.7 and ratio <= 0.85: + return random.randint(32, 64) + elif ratio > 0.85 and ratio <= 0.95: + return random.randint(64, 128) + elif ratio > 0.95: + return random.randint(128, 1024) + else: + return 128 + + group_ms = [convert_to_tokens(ratio) for ratio in ratios] + return group_ms + + +group_ms = create_unbalanced_expert_token_distribution(8192) +# group_ms = [128 for _ in range(8192)] +# group_ms = [128 if i % 2 == 0 else 64 for i in range(8192)] + + +def bench_es( + n: int, + k: int, + num_groups: int, + num_warmup: int, + num_run: int, +) -> Tuple[float, int]: + device = "cuda" + alignment = 128 + n_g = ceil_div(n, alignment) * alignment + k_g = ceil_div(k, alignment) * alignment + out_dtype = torch.bfloat16 + + expert_offsets = torch.zeros((num_groups + 1), device=device, dtype=torch.int32) + problem_sizes = torch.zeros((num_groups, 3), device=device, dtype=torch.int32) + + a_tensors = [] + b_tensors = [] + a_scales_tensors = [] + b_scales_tensors = [] + if False: + print("Token Distributtion: ", group_ms[0:num_groups]) + print("Token Count: ", sum(group_ms[0:num_groups])) + for g in range(num_groups): + m_g = group_ms[g] + expert_offsets[g + 1] = expert_offsets[g] + m_g + problem_sizes[g][:] = torch.tensor([m_g, n_g, k_g], device=device) + + a_g, a_scale = per_token_cast_to_fp8(torch.randn((m_g, k_g), device=device)) + b_g, b_scale = per_block_cast_to_fp8(torch.randn((n_g, k_g), device=device).t()) + a_tensors.append(a_g) + b_tensors.append(b_g) + a_scales_tensors.append(a_scale) + b_scales_tensors.append(b_scale) + + a_stack = torch.empty( + (expert_offsets[-1], k_g), device=device, dtype=torch.float8_e4m3fn + ) + b_stack = torch.empty( + (num_groups, n_g, k_g), device=device, dtype=torch.float8_e4m3fn + ) + + for g in range(num_groups): + a_stack[expert_offsets[g] : expert_offsets[g + 1]] = a_tensors[g] + b_stack[g] = b_tensors[g].t() + b_stack = b_stack.transpose(1, 2) + + a_scale_stack = torch.empty( + (expert_offsets[-1], k_g // 128), device=device, dtype=torch.float32 + ) + b_scale_stack = torch.empty( + (num_groups, n_g // 128, k_g // 128), device=device, dtype=torch.float32 + ) + + for g in range(num_groups): + a_scale_stack[expert_offsets[g] : expert_offsets[g + 1]] = a_scales_tensors[g] + b_scale_stack[g] = b_scales_tensors[g].t() + b_scale_stack = b_scale_stack.transpose(1, 2) + + c_out = torch.empty((expert_offsets[-1], n_g), device=device, dtype=out_dtype) + a_strides = torch.full( + (num_groups,), a_stack.stride(0), device=device, dtype=torch.int64 + ) + d_strides = torch.full( + (num_groups,), c_out.stride(0), device=device, dtype=torch.int64 + ) + workspace = torch.empty((1024 * 1024 * 1024), device=device, dtype=torch.uint8) + + def run_cutlass(): + es_fp8_blockwise_scaled_grouped_mm( + c_out, + a_stack, + b_stack, + a_scale_stack, + b_scale_stack, + a_strides, + a_strides, + d_strides, + problem_sizes, + expert_offsets[:-1], + workspace, + ) + + run_cutlass() + # warmup + for _ in range(num_warmup): + run_cutlass() + torch.cuda.synchronize() + + # run + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + start_event.record() + for _ in range(num_run): + run_cutlass() + end_event.record() + end_event.synchronize() + torch.cuda.synchronize() + avg = start_event.elapsed_time(end_event) / num_run * 1000 # us + + return avg, expert_offsets[-1] + + +def bench_sgl( + n: int, + k: int, + num_groups: int, + num_warmup: int, + num_run: int, +) -> Tuple[float, int]: + device = "cuda" + alignment = 128 + n_g = ceil_div(n, alignment) * alignment + k_g = ceil_div(k, alignment) * alignment + out_dtype = torch.bfloat16 + + expert_offsets = torch.zeros((num_groups + 1), device=device, dtype=torch.int32) + problem_sizes = torch.zeros((num_groups, 3), device=device, dtype=torch.int32) + layout_sfa = torch.zeros((num_groups, 5), device=device, dtype=torch.int32) + layout_sfb = torch.zeros((num_groups, 5), device=device, dtype=torch.int32) + + a_tensors = [] + b_tensors = [] + a_scales_tensors = [] + b_scales_tensors = [] + + for g in range(num_groups): + m_g = group_ms[g] + expert_offsets[g + 1] = expert_offsets[g] + m_g + problem_sizes[g][:] = torch.tensor([m_g, n_g, k_g], device=device) + + a_g, a_scale = per_token_cast_to_fp8(torch.randn((m_g, k_g), device=device)) + b_g, b_scale = per_block_cast_to_fp8(torch.randn((n_g, k_g), device=device).t()) + a_tensors.append(a_g) + b_tensors.append(b_g) + a_scales_tensors.append(a_scale) + b_scales_tensors.append(b_scale) + + a_stack = torch.empty( + (expert_offsets[-1], k_g), device=device, dtype=torch.float8_e4m3fn + ) + b_stack = torch.empty( + (num_groups, n_g, k_g), device=device, dtype=torch.float8_e4m3fn + ) + + for g in range(num_groups): + a_stack[expert_offsets[g] : expert_offsets[g + 1]] = a_tensors[g] + b_stack[g] = b_tensors[g].t() + b_stack = b_stack.transpose(1, 2) + + a_scale_stack = torch.empty( + (expert_offsets[-1], k_g // 128), device=device, dtype=torch.float32 + ) + b_scale_stack = torch.empty( + (num_groups, n_g // 128, k_g // 128), device=device, dtype=torch.float32 + ) + + for g in range(num_groups): + a_scale_stack[expert_offsets[g] : expert_offsets[g + 1]] = a_scales_tensors[g] + b_scale_stack[g] = b_scales_tensors[g].t() + b_scale_stack = b_scale_stack.transpose(1, 2) + + c_out = torch.empty((expert_offsets[-1], n_g), device=device, dtype=out_dtype) + a_strides = torch.full( + (num_groups,), a_stack.stride(0), device=device, dtype=torch.int64 + ) + c_strides = torch.full( + (num_groups,), c_out.stride(0), device=device, dtype=torch.int64 + ) + workspace = torch.empty((1024 * 1024 * 1024), device=device, dtype=torch.uint8) + a_ptrs = torch.empty((num_groups,), device=device, dtype=torch.int64) + b_ptrs = torch.empty((num_groups,), device=device, dtype=torch.int64) + out_ptrs = torch.empty((num_groups,), device=device, dtype=torch.int64) + a_scales_ptrs = torch.empty((num_groups,), device=device, dtype=torch.int64) + b_scales_ptrs = torch.empty((num_groups,), device=device, dtype=torch.int64) + + def run_cutlass(): + fp8_blockwise_scaled_grouped_mm( + c_out, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a_stack, + b_stack, + a_scale_stack, + b_scale_stack, + a_strides, + a_strides, + c_strides, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets[:-1], + workspace, + ) + + # warmup + for _ in range(num_warmup): + run_cutlass() + torch.cuda.synchronize() + + # run + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + start_event.record() + for _ in range(num_run): + run_cutlass() + end_event.record() + end_event.synchronize() + torch.cuda.synchronize() + avg = start_event.elapsed_time(end_event) / num_run * 1000 # us + + return avg, expert_offsets[-1] + + +benchmark_kernels = {"es": bench_es, "sgl-kernel": bench_sgl} + + +@dataclass +class ShapeArg: + n: int + k: int + num_groups: int + + +def benchmark_one_shape( + shape_args: List[ShapeArg], + num_warmup: int, + num_run: int, +): + for shape in shape_args: + print(f"\nBenchmark: n={shape.n}, k={shape.k}, num_groups={shape.num_groups}") + for kernel_name, kernel_func in benchmark_kernels.items(): + average_time, m = kernel_func( + shape.n, + shape.k, + shape.num_groups, + num_warmup, + num_run, + ) + print(f"{kernel_name}: {average_time} us") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--num-warmup", type=int, default=3) + parser.add_argument("--num-run", type=int, default=20) + shape_args = [ + # Prefill, DeepSeek-R1, gateup, chunk_size = 4096, TP = 8 + ShapeArg(n=512, k=7168, num_groups=256), + # Prefill, DeepSeek-R1, down, chunk_size = 4096, TP = 8 + ShapeArg(n=7168, k=256, num_groups=256), + # Prefill, Qwen3-235B-A22B-FP8, gateup, TP = 4 + ShapeArg(n=768, k=4096, num_groups=128), + # Prefill, Qwen3-235B-A22B-FP8, down, TP = 4 + ShapeArg(n=4096, k=384, num_groups=128), + # Decode, DeepSeek-R1, gateup, bs = 128, EP = 8 + ShapeArg(n=4096, k=7168, num_groups=32), + # Decode, DeepSeek-R1, gateup, bs = 256, EP = 16 + ShapeArg(n=4096, k=7168, num_groups=16), + ] + args = parser.parse_args() + benchmark_one_shape(shape_args, args.num_warmup, args.num_run) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_fp4_gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_fp4_gemm.py new file mode 100755 index 00000000..0323fde2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_fp4_gemm.py @@ -0,0 +1,241 @@ +import argparse +import copy +import csv +import itertools +import os + +import pytest +import torch +import triton +from flashinfer import mm_fp4 +from sgl_kernel import cutlass_scaled_fp4_mm, scaled_fp4_quant + +from sglang.srt.utils import get_device_capability + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + +FLOAT4_E2M1_MAX = 6.0 +FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max + + +def get_weight_shapes(args): + models_tps = args.tp_sizes + + if models_tps == [4]: + return [[1024, 3584], [7168, 256], [7168, 2304], [9216, 3584]] + + if models_tps == [8]: + return [[512, 3584], [7168, 128], [7168, 1152], [4608, 3584]] + return [ + [1024, 3584], + [7168, 256], + [7168, 2304], + [9216, 3584], + [512, 3584], + [7168, 128], + [7168, 1152], + [4608, 3584], + ] + + +# CI environment uses simplified parameters +if IS_CI: + batch_sizes = [1, 8] # Simplified for CI +else: + batch_sizes = [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512, + 1024, + 2048, + 3072, + 4096, + 8192, + 16384, + ] + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=batch_sizes, + # x_vals = [64], + x_log=False, + line_arg="provider", + line_vals=["cutlass", "cudnn", "trtllm"], + line_names=["baseline cutlass fp4", "cudnn fp4", "trtllm fp4"], + styles=[("red", "solid"), ("blue", "solid"), ("green", "solid")], + ylabel="latency (ms)", + plot_name="fp4_gemm_benchmark", + args={}, + ) +) +def benchmark(batch_size, provider, N, K, dtype, correctness, csv_file): + M = batch_size + packed_k = K + K = 2 * packed_k + a_dtype = torch.randn((M, K), dtype=dtype, device="cuda") + b_dtype = torch.randn((N, K), dtype=dtype, device="cuda") + a_global_scale = ( + (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(a_dtype.flatten(), dim=-1) + ).to(torch.float32) + b_global_scale = ( + (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(b_dtype.flatten(), dim=-1) + ).to(torch.float32) + + alpha = 1.0 / (a_global_scale * b_global_scale) + a_fp4, a_scale_interleaved = scaled_fp4_quant(a_dtype, a_global_scale) + # print("a_fp4", a_fp4) + b_fp4, b_scale_interleaved = scaled_fp4_quant(b_dtype, b_global_scale) + res_fi = torch.empty((M, N), dtype=dtype, device="cuda") + + quantiles = [0.5, 0.2, 0.8] + if provider == "cutlass": + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: cutlass_scaled_fp4_mm( + a_fp4, b_fp4, a_scale_interleaved, b_scale_interleaved, alpha, dtype + ), + quantiles=quantiles, + ) + if provider == "cudnn": + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: mm_fp4( + a_fp4, + b_fp4.T, + a_scale_interleaved, + b_scale_interleaved.T, + alpha, + dtype, + res_fi, + ), + quantiles=quantiles, + ) + if provider == "trtllm": + a_scale_interleaved = a_scale_interleaved.to(torch.uint8) + b_scale_interleaved = b_scale_interleaved.to(torch.uint8) + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: mm_fp4( + a_fp4, + b_fp4.T, + a_scale_interleaved, + b_scale_interleaved.T, + alpha, + dtype, + res_fi, + backend="trtllm", + ), + quantiles=quantiles, + ) + if correctness: + res_cutlass = cutlass_scaled_fp4_mm( + a_fp4, b_fp4, a_scale_interleaved, b_scale_interleaved, alpha, dtype + ) + mm_fp4( + a_fp4, + b_fp4.T, + a_scale_interleaved, + b_scale_interleaved.T, + alpha, + dtype, + res_fi, + backend="cudnn", + ) + assert torch.allclose( + res_fi, res_cutlass, atol=1e-3, rtol=1e-3 + ), "cudnn fp4 doesn't match cutlass fp4" + mm_fp4( + a_fp4, + b_fp4.T, + a_scale_interleaved, + b_scale_interleaved.T, + alpha, + dtype, + res_fi, + backend="trtllm", + ) + assert torch.allclose( + res_fi, res_cutlass, atol=1e-3, rtol=1e-3 + ), "trtllm fp4 doesn't match cutlass fp4" + + if csv_file: + with open(csv_file, "a", newline="") as f: + writer = csv.writer(f) + writer.writerow([provider, M, N, K, ms]) + + return ms, min_ms, max_ms + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--tp-sizes", + nargs="+", + type=int, + default=[1], + help="List of tensor parallel sizes", + ) + parser.add_argument( + "--dtype", + type=torch.dtype, + default=torch.bfloat16, + help="Data type", + ) + parser.add_argument( + "--correctness", + action="store_true", + help="Check correctness", + ) + parser.add_argument( + "--csv", + type=str, + default="results_cutlass_cudnn.csv", + help="CSV file to save results", + ) + args = parser.parse_args() + + # Simplify for CI environment + if IS_CI: + args.tp_sizes = [args.tp_sizes[0]] # Use only first TP size + + if args.csv: + with open(args.csv, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(["provider", "m", "n", "k", "time_ms"]) + + # Check architecture compatibility - FP4 operations require sm100a/sm103a + major, minor = get_device_capability() + if major is None or major < 10: # Requires compute capability 10.0+ (sm100a/sm103a) + print("Skipping FP4 GEMM benchmark") + if major is not None: + print(f"FP4 operations require sm100a/sm103a, but found sm{major}{minor}") + else: + print("Could not determine device capability") + else: + NKs = get_weight_shapes(args) + + # Limit iterations in CI + if IS_CI: + NKs = NKs[:2] # Only test first 2 shapes in CI + + for N, K in NKs: + print(f"DeepSeek-R1-0528-FP4 N={N} K={K}: ") + benchmark.run( + print_data=True, + N=N, + K=K, + dtype=args.dtype, + correctness=args.correctness, + csv_file=args.csv, + ) + print("Benchmark finished!") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_fp8_blockwise_gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_fp8_blockwise_gemm.py new file mode 100644 index 00000000..70766df9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_fp8_blockwise_gemm.py @@ -0,0 +1,239 @@ +import argparse +import copy +import itertools +import os + +import deep_gemm +import torch +import triton +from deep_gemm.utils.layout import get_mn_major_tma_aligned_tensor +from sgl_kernel import fp8_blockwise_scaled_mm + +# Optional vLLM import +try: + from vllm._custom_ops import cutlass_scaled_mm as vllm_scaled_mm + + VLLM_AVAILABLE = True +except ImportError: + vllm_scaled_mm = None + VLLM_AVAILABLE = False + +from sglang.srt.layers.quantization.fp8_kernel import ( + w8a8_block_fp8_matmul_triton as w8a8_block_fp8_matmul, +) + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + + +def get_weight_shapes(args): + models_tps = list(itertools.product(args.models, args.tp_sizes)) + # NOTE(HandH1998): The weight shapes only works for DeepSeek-V3. Modify them, if you tune for another different model. + # cannot TP + total = [ + (512 + 64, 7168), + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (7168, 16384), + (7168, 18432), + ] + # N can TP + n_tp = [ + (18432 * 2, 7168), + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (24576, 1536), + (4096, 7168), + ] + # K can TP + k_tp = [(7168, 18432), (7168, 16384), (7168, 2048)] + # only support Deepseek-V3 + SUPPORT_MODEL = ["deepseek-ai/DeepSeek-V3"] + + weight_shapes = [] + for model, tp_size in models_tps: + assert model in SUPPORT_MODEL + for t in total: + new_t = [t[0], t[1], model] + weight_shapes.append(new_t) + for n_t in n_tp: + new_t = [n_t[0] // tp_size, n_t[1], model] + weight_shapes.append(new_t) + for k_t in k_tp: + new_t = [k_t[0], k_t[1] // tp_size, model] + weight_shapes.append(new_t) + return weight_shapes + + +def cdiv(a: int, b: int) -> int: + """Ceiling division.""" + return -(a // -b) + + +def fp8_gemm_deepgemm( + x_fp8: torch.Tensor, + x_scale: torch.Tensor, + y_fp8: torch.Tensor, + y_scale: torch.Tensor, + m: int, + n: int, + k: int, +): + """DeepGEMM implementation of FP8 GEMM""" + out = torch.empty((m, n), device="cuda", dtype=torch.bfloat16) + + # Run DeepGEMM kernel + deep_gemm.fp8_gemm_nt((x_fp8, x_scale), (y_fp8, y_scale), out) + return out + + +def scale_shape(shape, group_shape): + assert len(shape) == len(group_shape) + return tuple(cdiv(shape[i], group_shape[i]) for i in range(len(group_shape))) + + +# CI environment uses simplified parameters +if IS_CI: + batch_sizes = [1, 8] # Simplified for CI +else: + batch_sizes = [1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096] + +# Filter providers based on availability +available_providers = ["sgl-kernel"] +available_names = ["sgl-kernel"] +available_styles = [("orange", "-")] + +if VLLM_AVAILABLE: + available_providers.insert(0, "vllm") + available_names.insert(0, "vllm") + available_styles.insert(0, ("blue", "-")) + +available_providers.append("triton") +available_names.append("sglang triton") +available_styles.append(("red", "-")) + +# Add deepgemm if available +try: + import deep_gemm + + available_providers.append("deepgemm") + available_names.append("deepgemm") + available_styles.append(("yellow", "-")) +except ImportError: + pass + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=batch_sizes, + x_log=False, + line_arg="provider", + line_vals=available_providers, + line_names=available_names, + styles=available_styles, + ylabel="GB/s", + plot_name="fp8 blockwise scaled matmul", + args={}, + ) +) +def benchmark(batch_size, provider, N, K): + M = batch_size + fp8_info = torch.finfo(torch.float8_e4m3fn) + fp8_max, fp8_min = fp8_info.max, fp8_info.min + + a_fp32 = (torch.rand(M, K, dtype=torch.float32, device="cuda") - 0.5) * 2 * fp8_max + a_fp8 = a_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + b_fp32 = (torch.rand(N, K, dtype=torch.float32, device="cuda") - 0.5) * 2 * fp8_max + b_fp8 = b_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + scale_a_group_shape = (1, 128) + scale_b_group_shape = (128, 128) + scale_a_shape = scale_shape(a_fp8.shape, scale_a_group_shape) + scale_b_shape = scale_shape(b_fp8.shape, scale_b_group_shape) + + scale_a = torch.randn(scale_a_shape, device="cuda", dtype=torch.float32) + scale_b = torch.randn(scale_b_shape, device="cuda", dtype=torch.float32) + + quantiles = [0.5, 0.2, 0.8] + if provider == "sgl-kernel": + scale_a = scale_a.t().contiguous().t() + b_fp8, scale_b = b_fp8.t(), scale_b.t() + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: fp8_blockwise_scaled_mm( + a_fp8, b_fp8, scale_a, scale_b, torch.float16 + ), + quantiles=quantiles, + ) + elif provider == "vllm": + if not VLLM_AVAILABLE: + return (0, 0, 0) + scale_a = scale_a.t().contiguous().t() + b_fp8, scale_b = b_fp8.t(), scale_b.t() + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: vllm_scaled_mm(a_fp8, b_fp8, scale_a, scale_b, torch.float16), + quantiles=quantiles, + ) + elif provider == "triton": + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: w8a8_block_fp8_matmul( + a_fp8, b_fp8, scale_a, scale_b, [128, 128], torch.float16 + ), + quantiles=quantiles, + ) + if provider == "deepgemm": + scale_a_col_major = get_mn_major_tma_aligned_tensor(scale_a.clone()) + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: fp8_gemm_deepgemm( + a_fp8, scale_a_col_major, b_fp8, scale_b, M, N, K + ), + quantiles=quantiles, + ) + return ms * 1000, max_ms * 1000, min_ms * 1000 # convert to ms + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--models", + nargs="+", + type=str, + default=["deepseek-ai/DeepSeek-V3"], + help="List of models to benchmark", + ) + parser.add_argument( + "--tp-sizes", + nargs="+", + type=int, + default=[1], + help="List of tensor parallel sizes", + ) + args = parser.parse_args() + + # Simplify for CI environment + if IS_CI: + args.models = [args.models[0]] # Use only first model + args.tp_sizes = [args.tp_sizes[0]] # Use only first TP size + + NK_model_names = get_weight_shapes(args) + + # Limit iterations in CI + if IS_CI: + NK_model_names = NK_model_names[:2] # Only test first 2 shapes in CI + + for N, K, model_name in NK_model_names: + if N % 128 != 0 or K % 128 != 0: + print(f"Skip {N=}, {K=} now") + continue + print(f"{model_name} N={N} K={K}: ") + benchmark.run( + print_data=True, + N=N, + K=K, + ) + + print("Benchmark finished!") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_fp8_blockwise_group_gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_fp8_blockwise_group_gemm.py new file mode 100644 index 00000000..19e425b5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_fp8_blockwise_group_gemm.py @@ -0,0 +1,343 @@ +import argparse +import os + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) +import random +from dataclasses import dataclass +from typing import List, Tuple + +import deep_gemm +import torch +from sgl_kernel import fp8_blockwise_scaled_grouped_mm + + +def get_m_alignment_for_contiguous_layout(): + return 128 + + +def ceil_div(x: int, y: int) -> int: + return (x + y - 1) // y + + +def per_token_cast_to_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 + m, n = x.shape + pad_size = (128 - (n % 128)) % 128 + x = torch.nn.functional.pad(x, (0, pad_size), value=0) if pad_size > 0 else x + x_view = x.view(m, -1, 128) + x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4) + fp8_data = (x_view * (448.0 / x_amax.unsqueeze(2))).to(torch.float8_e4m3fn) + return fp8_data.view(m, n + pad_size)[:, :n], (x_amax / 448.0).view(m, -1) + + +def per_block_cast_to_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 + m, n = x.shape + x_padded = torch.zeros( + (ceil_div(m, 128) * 128, ceil_div(n, 128) * 128), dtype=x.dtype, device=x.device + ) + x_padded[:m, :n] = x + x_view = x_padded.view(-1, 128, x_padded.size(1) // 128, 128) + x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4) + x_scaled = (x_view * (448.0 / x_amax)).to(torch.float8_e4m3fn) + return x_scaled.view_as(x_padded)[:m, :n].contiguous(), (x_amax / 448.0).view( + x_view.size(0), x_view.size(2) + ) + + +def construct_contiguous_grouped( + num_groups: int, expected_m_per_group: int, k: int, n: int +) -> Tuple[ + int, + Tuple[torch.Tensor, torch.Tensor], + Tuple[torch.Tensor, torch.Tensor], + torch.Tensor, + torch.Tensor, + torch.Tensor, +]: + alignment = get_m_alignment_for_contiguous_layout() + group_ms = [int(expected_m_per_group) for _ in range(num_groups)] + m = sum([ceil_div(x, alignment) * alignment for x in group_ms]) + + x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) + y = torch.randn((num_groups, n, k), device="cuda", dtype=torch.bfloat16) + m_indices = torch.empty(m, device="cuda", dtype=torch.int32) + out = torch.empty((m, n), device="cuda", dtype=torch.bfloat16) + + start = 0 + for i, group_m in enumerate(group_ms): + actual_end = start + group_m + aligned_end = start + ceil_div(group_m, alignment) * alignment + m_indices[start:actual_end] = i + m_indices[actual_end:aligned_end] = -1 + start = aligned_end + + assert m % 4 == 0, f"TMA alignment error: {m}" + x_fp8 = per_token_cast_to_fp8(x) + y_fp8 = ( + torch.empty_like(y, dtype=torch.float8_e4m3fn), + torch.empty( + (num_groups, ceil_div(n, 128), k // 128), device="cuda", dtype=torch.float + ), + ) + for i in range(num_groups): + y_fp8[0][i], y_fp8[1][i] = per_block_cast_to_fp8(y[i]) + + return m, x_fp8, y_fp8, m_indices, out + + +def bench_deepgemm( + expected_m_per_group: int, + n: int, + k: int, + num_groups: int, + num_warmup: int, + num_run: int, +) -> Tuple[float, int]: + # construct tensors + m, x_fp8, y_fp8, m_indices, out = construct_contiguous_grouped( + num_groups, expected_m_per_group, k, n + ) + + def run_deepgemm(): + deep_gemm.m_grouped_fp8_gemm_nt_contiguous(x_fp8, y_fp8, out, m_indices) + + # warmup + for _ in range(num_warmup): + run_deepgemm() + torch.cuda.synchronize() + + # run + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + latencies: list[float] = [] + start_event.record() + for _ in range(num_run): + run_deepgemm() + end_event.record() + end_event.synchronize() + torch.cuda.synchronize() + avg = start_event.elapsed_time(end_event) / num_run * 1000 # us + + return avg, m + + +def bench_cutlass( + expected_m_per_group: int, + n: int, + k: int, + num_groups: int, + num_warmup: int, + num_run: int, +) -> Tuple[float, int]: + device = "cuda" + alignment = 16 + n_g = ceil_div(n, alignment) * alignment + k_g = ceil_div(k, alignment) * alignment + out_dtype = torch.bfloat16 + + expert_offsets = torch.zeros((num_groups + 1), device=device, dtype=torch.int32) + problem_sizes = torch.zeros((num_groups, 3), device=device, dtype=torch.int32) + layout_sfa = torch.zeros((num_groups, 5), device=device, dtype=torch.int32) + layout_sfb = torch.zeros((num_groups, 5), device=device, dtype=torch.int32) + + a_tensors = [] + b_tensors = [] + a_scales_tensors = [] + b_scales_tensors = [] + + # TODO(@TianQiLin666666): Unique group_ms in all bench function + group_ms = [ + alignment * ceil_div(int(expected_m_per_group), alignment) + for _ in range(num_groups) + ] + for g in range(num_groups): + m_g = group_ms[g] + expert_offsets[g + 1] = expert_offsets[g] + m_g + problem_sizes[g][:] = torch.tensor([m_g, n_g, k_g], device=device) + + a_g, a_scale = per_token_cast_to_fp8(torch.randn((m_g, k_g), device=device)) + b_g, b_scale = per_block_cast_to_fp8(torch.randn((n_g, k_g), device=device).t()) + a_tensors.append(a_g) + b_tensors.append(b_g) + a_scales_tensors.append(a_scale) + b_scales_tensors.append(b_scale) + + a_stack = torch.empty( + (expert_offsets[-1], k_g), device=device, dtype=torch.float8_e4m3fn + ) + b_stack = torch.empty( + (num_groups, n_g, k_g), device=device, dtype=torch.float8_e4m3fn + ) + + for g in range(num_groups): + a_stack[expert_offsets[g] : expert_offsets[g + 1]] = a_tensors[g] + b_stack[g] = b_tensors[g].t() + b_stack = b_stack.transpose(1, 2) + + a_scale_stack = torch.empty( + (expert_offsets[-1], k_g // 128), device=device, dtype=torch.float32 + ) + b_scale_stack = torch.empty( + (num_groups, n_g // 128, k_g // 128), device=device, dtype=torch.float32 + ) + + for g in range(num_groups): + a_scale_stack[expert_offsets[g] : expert_offsets[g + 1]] = a_scales_tensors[g] + b_scale_stack[g] = b_scales_tensors[g].t() + b_scale_stack = b_scale_stack.transpose(1, 2) + + c_out = torch.empty((expert_offsets[-1], n_g), device=device, dtype=out_dtype) + a_strides = torch.full( + (num_groups,), a_stack.stride(0), device=device, dtype=torch.int64 + ) + c_strides = torch.full( + (num_groups,), c_out.stride(0), device=device, dtype=torch.int64 + ) + workspace = torch.empty((1024 * 1024 * 1024), device=device, dtype=torch.uint8) + a_ptrs = torch.empty((num_groups,), device=device, dtype=torch.int64) + b_ptrs = torch.empty((num_groups,), device=device, dtype=torch.int64) + out_ptrs = torch.empty((num_groups,), device=device, dtype=torch.int64) + a_scales_ptrs = torch.empty((num_groups,), device=device, dtype=torch.int64) + b_scales_ptrs = torch.empty((num_groups,), device=device, dtype=torch.int64) + + def run_cutlass(): + fp8_blockwise_scaled_grouped_mm( + c_out, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a_stack, + b_stack, + a_scale_stack, + b_scale_stack, + a_strides, + a_strides, + c_strides, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets[:-1], + workspace, + ) + + # warmup + for _ in range(num_warmup): + run_cutlass() + torch.cuda.synchronize() + + # run + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + start_event.record() + for _ in range(num_run): + run_cutlass() + end_event.record() + end_event.synchronize() + torch.cuda.synchronize() + avg = start_event.elapsed_time(end_event) / num_run * 1000 # us + + return avg, expert_offsets[-1] + + +def bench_sglang_triton( + expected_m_per_group: int, + n: int, + k: int, + num_groups: int, + num_warmup: int, + num_run: int, +) -> Tuple[float, int]: + pass + + +benchmark_kernels = { + "deepgemm": bench_deepgemm, + "cutlass": bench_cutlass, + # "triton": bench_sglang_triton, +} + + +@dataclass +class ShapeArg: + expected_m_per_group: int + n: int + k: int + num_groups: int + + +def benchmark_one_shape( + shape_args: List[ShapeArg], + num_warmup: int, + num_run: int, +): + for shape in shape_args: + print( + f"\nBenchmark: expected_m_per_group={shape.expected_m_per_group}, n={shape.n}, k={shape.k}, num_groups={shape.num_groups}" + ) + for kernel_name, kernel_func in benchmark_kernels.items(): + average_time, m = kernel_func( + shape.expected_m_per_group, + shape.n, + shape.k, + shape.num_groups, + num_warmup, + num_run, + ) + print(f"{kernel_name}: {average_time} us") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--num-warmup", type=int, default=3) + parser.add_argument("--num-run", type=int, default=10) + + # CI environment uses simplified parameters + if IS_CI: + shape_args = [ + # Only test one simple shape in CI + ShapeArg(expected_m_per_group=128, n=512, k=7168, num_groups=256), + ] + else: + shape_args = [ + # Prefill, DeepSeek-R1, gateup, chunk_size = 4096, TP = 8 + ShapeArg(expected_m_per_group=128, n=512, k=7168, num_groups=256), + # Prefill, DeepSeek-R1, gateup, chunk_size = 8192, TP = 8 + ShapeArg(expected_m_per_group=256, n=512, k=7168, num_groups=256), + # Prefill, DeepSeek-R1, gateup, chunk_size = 8192, TP = 16 + ShapeArg(expected_m_per_group=256, n=256, k=7168, num_groups=256), + # Prefill, DeepSeek-R1, gateup, chunk_size = 16384, TP = 16 + ShapeArg(expected_m_per_group=512, n=256, k=7168, num_groups=256), + # Decode, DeepSeek-R1, gateup, bs = 32, TP = 8 + ShapeArg(expected_m_per_group=1, n=512, k=7168, num_groups=256), + # Decode, DeepSeek-R1, gateup, bs = 64, TP = 16 + ShapeArg(expected_m_per_group=2, n=256, k=7168, num_groups=256), + # Prefill, DeepSeek-R1, gateup, chunk_size = 8192, EP = 8 + ShapeArg(expected_m_per_group=256, n=4096, k=7168, num_groups=32), + # Prefill, DeepSeek-R1, gateup, chunk_size = 16384, EP = 16 + ShapeArg(expected_m_per_group=512, n=4096, k=7168, num_groups=16), + # Decode, DeepSeek-R1, gateup, bs = 128, EP = 8 + ShapeArg(expected_m_per_group=4, n=4096, k=7168, num_groups=32), + # Decode, DeepSeek-R1, gateup, bs = 256, EP = 16 + ShapeArg(expected_m_per_group=8, n=4096, k=7168, num_groups=16), + # Prefill, Qwen3-235B-A22B-FP8, gateup, chunk_size = 16384, TP = 4 + ShapeArg(expected_m_per_group=1024, n=768, k=4096, num_groups=128), + # Prefill, Qwen3-235B-A22B-FP8, down, chunk_size = 16384, TP = 4 + ShapeArg(expected_m_per_group=1024, n=4096, k=384, num_groups=128), + # Decode, Qwen3-235B-A22B-FP8, gateup, bs = 256, TP = 4 + ShapeArg(expected_m_per_group=16, n=768, k=4096, num_groups=128), + # Decode, Qwen3-235B-A22B-FP8, down, bs = 256, TP = 4 + ShapeArg(expected_m_per_group=16, n=4096, k=384, num_groups=128), + ] + args = parser.parse_args() + benchmark_one_shape(shape_args, args.num_warmup, args.num_run) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_fp8_gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_fp8_gemm.py new file mode 100644 index 00000000..a49f3b06 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_fp8_gemm.py @@ -0,0 +1,232 @@ +import argparse +import copy +import itertools +import os +from typing import Optional, Tuple + +import torch +import triton +from sgl_kernel import fp8_scaled_mm as sgl_scaled_mm +from sgl_kernel import sgl_per_tensor_quant_fp8 + +# Optional vLLM import +try: + from vllm._custom_ops import cutlass_scaled_mm as vllm_scaled_mm + from vllm._custom_ops import scaled_fp8_quant as vllm_scaled_fp8_quant + + VLLM_AVAILABLE = True +except ImportError: + vllm_scaled_mm = None + vllm_scaled_fp8_quant = None + VLLM_AVAILABLE = False + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + +# Weight Shapes are in the format +# ([K, N], TP_SPLIT_DIM) +# Example: +# A shape of ([14336, 4096], 0) indicates the following GEMM shape, +# - TP1 : K = 14336, N = 4096 +# - TP2 : K = 7168, N = 4096 +# A shape of ([4096, 6144], 1) indicates the following GEMM shape, +# - TP1 : K = 4096, N = 6144 +# - TP4 : K = 4096, N = 1536 + +# TP1 shapes +WEIGHT_SHAPES = { + "meta-llama/Llama-3.1-8B-Instruct": [ + ([4096, 6144], 1), + ([4096, 4096], 0), + ([4096, 28672], 1), + ([14336, 4096], 0), + ], + "meta-llama/Llama-3.3-70B-Instruct": [ + ([8192, 10240], 1), + ([8192, 8192], 0), + ([8192, 57344], 1), + ([28672, 8192], 0), + ], + "mistralai/Mistral-Large-Instruct-2407": [ + ([12288, 14336], 1), + ([12288, 12288], 0), + ([12288, 57344], 1), + ([28672, 12288], 0), + ], + "Qwen/Qwen2.5-7B-Instruct": [ + ([3584, 4608], 1), + ([3584, 3584], 0), + ([3584, 37888], 1), + ([18944, 3584], 0), + ], + "Qwen/Qwen2.5-32B-Instruct": [ + ([5120, 7168], 1), + ([5120, 5120], 0), + ([5120, 55296], 1), + ([27648, 5120], 0), + ], + "Qwen/Qwen2.5-72B-Instruct": [ + ([8192, 10240], 1), + ([8192, 8192], 0), + ([8192, 59136], 1), + ([29568, 8192], 0), + ], + "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": [ + ([2048, 3072], 1), + ([2048, 4096], 1), + ([2048, 2048], 0), + ([2048, 576], 0), + ([2048, 21888], 1), + ([10944, 2048], 0), + ([2048, 2816], 1), + ([1408, 2048], 0), + ], +} + + +def sglang_scaled_fp8_quant( + input: torch.Tensor, + scale: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + fp8_type_: torch.dtype = torch.float8_e4m3fn + output = torch.empty_like(input, device=input.device, dtype=fp8_type_) + is_static = True + if scale is None: + scale = torch.zeros(1, device=input.device, dtype=torch.float32) + is_static = False + sgl_per_tensor_quant_fp8(input, output, scale, is_static) + + return output, scale + + +# CI environment uses simplified parameters +if IS_CI: + batch_sizes = [1] # Single batch size for CI +else: + batch_sizes = [1, 16, 64, 128, 256, 512, 1024, 2048] + +# Filter line_vals based on vLLM availability +if VLLM_AVAILABLE: + line_vals = [ + "vllm-fp8-fp16", + "vllm-fp8-bf16", + "sglang-fp8-fp16", + "sglang-fp8-bf16", + ] + line_names = [ + "vllm-fp8-fp16", + "vllm-fp8-bf16", + "sglang-fp8-fp16", + "sglang-fp8-bf16", + ] + styles = [("green", "-"), ("green", "--"), ("blue", "-"), ("blue", "--")] +else: + line_vals = [ + "sglang-fp8-fp16", + "sglang-fp8-bf16", + ] + line_names = [ + "sglang-fp8-fp16", + "sglang-fp8-bf16", + ] + styles = [("blue", "-"), ("blue", "--")] + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=batch_sizes, + x_log=False, + line_arg="provider", + line_vals=line_vals, + line_names=line_names, + styles=styles, + ylabel="GB/s", + plot_name="fp8 scaled matmul", + args={}, + ) +) +def benchmark(batch_size, provider, N, K): + # M, N, K = batch_size, 4096, 8192 + M = batch_size + a = torch.ones((M, K), device="cuda") * 5.0 + b = torch.ones((N, K), device="cuda") * 5.0 + # vLLM expects scalar scales, while sglang can handle per-token scales + scale_a_scalar = torch.randn(1, device="cuda", dtype=torch.float32) + scale_b_scalar = torch.randn(1, device="cuda", dtype=torch.float32) + scale_a = torch.randn((M,), device="cuda", dtype=torch.float32) + scale_b = torch.randn((N,), device="cuda", dtype=torch.float32) + quantiles = [0.5, 0.2, 0.8] + + dtype = torch.float16 if "fp16" in provider else torch.bfloat16 + + if "vllm-fp8" in provider: + if not VLLM_AVAILABLE: + # Return zero if vLLM is not available + return (0, 0, 0) + a_fp8, scale_a_fp8 = vllm_scaled_fp8_quant(a, scale_a_scalar) + b_fp8, scale_b_fp8 = vllm_scaled_fp8_quant(b, scale_b_scalar) + b_fp8 = b_fp8.t() + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: vllm_scaled_mm(a_fp8, b_fp8, scale_a_fp8, scale_b_fp8, dtype), + quantiles=quantiles, + ) + elif "sglang-fp8" in provider: + a_fp8, scale_a_fp8 = sglang_scaled_fp8_quant(a, scale_a) + b_fp8, scale_b_fp8 = sglang_scaled_fp8_quant(b, scale_b) + b_fp8 = b_fp8.t() + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: sgl_scaled_mm( + a_fp8, b_fp8, scale_a_fp8, scale_b_fp8, dtype, bias=None + ), + quantiles=quantiles, + ) + + gbps = lambda ms: (2 * M * N * K + M * N) * a.element_size() * 1e-9 / (ms * 1e-3) + return gbps(ms), gbps(max_ms), gbps(min_ms) + + +def prepare_shapes(args): + KN_model_names = [] + models_tps = list(itertools.product(args.models, args.tp_sizes)) + for model, tp_size in models_tps: + assert model in WEIGHT_SHAPES + for KN, tp_split_dim in copy.deepcopy(WEIGHT_SHAPES[model]): + KN[tp_split_dim] = KN[tp_split_dim] // tp_size + KN.append(model) + KN_model_names.append(KN) + return KN_model_names + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--models", + nargs="+", + type=str, + default=["meta-llama/Llama-3.1-8B-Instruct"], + help="List of models to benchmark", + ) + parser.add_argument( + "--tp-sizes", + nargs="+", + type=int, + default=[1], + help="List of tensor parallel sizes", + ) + args = parser.parse_args() + + # Simplify for CI environment + if IS_CI: + args.models = [args.models[0]] # Use only first model + args.tp_sizes = [args.tp_sizes[0]] # Use only first TP size + + KN_model_names = prepare_shapes(args) + for K, N, model_name in KN_model_names: + print(f"{model_name} N={N} K={K}: ") + benchmark.run(print_data=True, N=N, K=K) + + print("Benchmark finished!") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_int8_gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_int8_gemm.py new file mode 100644 index 00000000..95f0f3bb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_int8_gemm.py @@ -0,0 +1,185 @@ +import argparse +import copy +import itertools +import os + +import torch +import triton +from sgl_kernel import int8_scaled_mm + +# Optional vLLM import +try: + from vllm._custom_ops import cutlass_scaled_mm as vllm_scaled_mm + + VLLM_AVAILABLE = True +except ImportError: + vllm_scaled_mm = None + VLLM_AVAILABLE = False + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + + +def to_int8(tensor: torch.Tensor) -> torch.Tensor: + return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8) + + +WEIGHT_SHAPES = { + "meta-llama/Llama-3.1-8B-Instruct": [ + ([4096, 6144], 1), + ([4096, 4096], 0), + ([4096, 28672], 1), + ([14336, 4096], 0), + ], + "meta-llama/Llama-3.3-70B-Instruct": [ + ([8192, 10240], 1), + ([8192, 8192], 0), + ([8192, 57344], 1), + ([28672, 8192], 0), + ], + "mistralai/Mistral-Large-Instruct-2407": [ + ([12288, 14336], 1), + ([12288, 12288], 0), + ([12288, 57344], 1), + ([28672, 12288], 0), + ], + "Qwen/Qwen2.5-7B-Instruct": [ + ([3584, 4608], 1), + ([3584, 3584], 0), + ([3584, 37888], 1), + ([18944, 3584], 0), + ], + "Qwen/Qwen2.5-32B-Instruct": [ + ([5120, 7168], 1), + ([5120, 5120], 0), + ([5120, 55296], 1), + ([27648, 5120], 0), + ], + "Qwen/Qwen2.5-72B-Instruct": [ + ([8192, 10240], 1), + ([8192, 8192], 0), + ([8192, 59136], 1), + ([29568, 8192], 0), + ], + "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": [ + ([2048, 3072], 1), + ([2048, 4096], 1), + ([2048, 2048], 0), + ([2048, 576], 0), + ([2048, 21888], 1), + ([10944, 2048], 0), + ([2048, 2816], 1), + ([1408, 2048], 0), + ], +} + + +# CI environment uses simplified parameters +if IS_CI: + batch_sizes = [1] # Single batch size for CI +else: + batch_sizes = [1, 16, 32, 64, 128, 256, 512, 1024, 2048] + +# Filter providers based on vLLM availability +if VLLM_AVAILABLE: + line_vals = ["vllm", "sgl-kernel"] + line_names = ["vllm int8 gemm", "sgl-kernel int8 gemm"] + styles = [("blue", "-"), ("orange", "-")] +else: + line_vals = ["sgl-kernel"] + line_names = ["sgl-kernel int8 gemm"] + styles = [("orange", "-")] + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=batch_sizes, + x_log=False, + line_arg="provider", + line_vals=line_vals, + line_names=line_names, + styles=styles, + ylabel="GB/s", + plot_name="int8 scaled matmul", + args={}, + ) +) +def benchmark(batch_size, provider, N, K): + M = batch_size + a = to_int8(torch.randn((M, K), device="cuda") * 5) + b = to_int8(torch.randn((N, K), device="cuda").t() * 5) + scale_a = torch.randn((M,), device="cuda", dtype=torch.float32) + scale_b = torch.randn((N,), device="cuda", dtype=torch.float32) + bias = torch.randn((N,), device="cuda", dtype=torch.float16) + + quantiles = [0.5, 0.2, 0.8] + if provider == "sgl-kernel": + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: int8_scaled_mm(a, b, scale_a, scale_b, torch.float16, bias), + quantiles=quantiles, + ) + elif provider == "vllm": + if not VLLM_AVAILABLE: + return (0, 0, 0) + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: vllm_scaled_mm(a, b, scale_a, scale_b, torch.float16, bias), + quantiles=quantiles, + ) + gbps = ( + lambda ms: ( + (2 * M * N * K - M * N) * a.element_size() + + (3 * M * N) * scale_a.element_size() + ) + * 1e-9 + / (ms * 1e-3) + ) + return gbps(ms), gbps(max_ms), gbps(min_ms) + + +def prepare_shapes(args): + KN_model_names = [] + models_tps = list(itertools.product(args.models, args.tp_sizes)) + for model, tp_size in models_tps: + assert model in WEIGHT_SHAPES + for KN, tp_split_dim in copy.deepcopy(WEIGHT_SHAPES[model]): + KN[tp_split_dim] = KN[tp_split_dim] // tp_size + KN.append(model) + KN_model_names.append(KN) + return KN_model_names + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--models", + nargs="+", + type=str, + default=["meta-llama/Llama-3.1-8B-Instruct"], + help="List of models to benchmark", + ) + parser.add_argument( + "--tp-sizes", + nargs="+", + type=int, + default=[1], + help="List of tensor parallel sizes", + ) + args = parser.parse_args() + + # Skip in CI environment due to architecture compatibility issues + if IS_CI: + print( + "Skipping INT8 GEMM benchmark in CI environment due to architecture compatibility issues" + ) + print("INT8 operations may not be supported on all GPU architectures") + else: + KN_model_names = prepare_shapes(args) + for K, N, model_name in KN_model_names: + print(f"{model_name} N={N} K={K}: ") + benchmark.run(print_data=True, N=N, K=K) + + print("Benchmark finished!") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_kimi_k2_moe_fused_gate.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_kimi_k2_moe_fused_gate.py new file mode 100644 index 00000000..78b75231 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_kimi_k2_moe_fused_gate.py @@ -0,0 +1,117 @@ +import itertools +import math +import os + +import torch +import triton +import triton.language as tl +from sgl_kernel import kimi_k2_moe_fused_gate + +from sglang.srt.layers.moe.topk import kimi_k2_biased_topk_impl + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + + +def kimi_k2_biased_topk_torch_compile(scores, bias, topk, routed_scaling_factor): + """Original torch.compile-based implementation""" + return kimi_k2_biased_topk_impl( + scores, + scores, + bias, + topk=topk, + renormalize=True, + routed_scaling_factor=routed_scaling_factor, + ) + + +def kimi_k2_biased_topk_fused_kernel(scores, bias, topk, routed_scaling_factor): + """Our fused CUDA kernel implementation""" + return kimi_k2_moe_fused_gate( + scores, + bias, + topk=topk, + renormalize=True, + routed_scaling_factor=routed_scaling_factor, + ) + + +# CI environment uses simplified parameters +if IS_CI: + seq_length_range = [5000] # Only test one sequence length in CI +else: + seq_length_range = [ + 1, + 8, + 16, + 32, + 64, + 128, + 256, + 512, + 1024, + 2048, + 4096, + 10000, + 15000, + 20000, + 25000, + 30000, + 35000, + 40000, + ] + +configs = [(sq,) for sq in seq_length_range] + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["seq_length"], + x_vals=[list(_) for _ in configs], + line_arg="provider", + line_vals=["torch_compile", "fused_kernel"], + line_names=["Torch Compile", "Fused Kernel"], + styles=[("blue", "-"), ("red", "-")], + ylabel="us", + plot_name="kimi-k2-moe-fused-gate-performance", + args={}, + ) +) +def benchmark(seq_length, provider): + dtype = torch.float32 + device = torch.device("cuda") + num_experts, topk = 384, 6 # Kimi K2 configuration + routed_scaling_factor = 2.872 # Kimi K2's routed scaling factor + + scores = torch.randn((seq_length, num_experts), device=device, dtype=dtype) + bias = torch.rand(num_experts, device=device, dtype=dtype) + + quantiles = [0.5, 0.2, 0.8] + + if provider == "torch_compile": + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: kimi_k2_biased_topk_torch_compile( + scores.clone(), bias.clone(), topk, routed_scaling_factor + ), + quantiles=quantiles, + ) + elif provider == "fused_kernel": + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: kimi_k2_biased_topk_fused_kernel( + scores.clone(), bias.clone(), topk, routed_scaling_factor + ), + quantiles=quantiles, + ) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + +if __name__ == "__main__": + print("=" * 80) + print("Benchmarking Kimi K2 MoE Fused Gate Performance") + print("=" * 80) + print("\nPerformance vs Sequence Length (384 experts, topk=6)") + benchmark.run(print_data=True, save_path=".") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_moe_align_block_size.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_moe_align_block_size.py new file mode 100644 index 00000000..5dc4a0e4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_moe_align_block_size.py @@ -0,0 +1,445 @@ +import argparse +import itertools +import os + +import torch +import triton +import triton.language as tl +from sgl_kernel import moe_align_block_size as sgl_moe_align_block_size + +try: + from vllm import _custom_ops as ops + + VLLM_AVAILABLE = True +except ImportError: + ops = None + VLLM_AVAILABLE = False + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + +USE_RANDOM_PERM = False + + +def ceil_div(a, b): + return (a + b - 1) // b + + +@triton.jit +def moe_align_block_size_stage1( + topk_ids_ptr, + tokens_cnts_ptr, + num_experts: tl.constexpr, + numel: tl.constexpr, + tokens_per_thread: tl.constexpr, +): + pid = tl.program_id(0) + start_idx = pid * tokens_per_thread + off_c = (pid + 1) * num_experts + + for i in range(tokens_per_thread): + if start_idx + i < numel: + idx = tl.load(topk_ids_ptr + start_idx + i) + token_cnt = tl.load(tokens_cnts_ptr + off_c + idx) + tl.store(tokens_cnts_ptr + off_c + idx, token_cnt + 1) + + +@triton.jit +def moe_align_block_size_stage2( + tokens_cnts_ptr, + num_experts: tl.constexpr, +): + pid = tl.program_id(0) + last_cnt = 0 + for i in range(1, num_experts + 1): + token_cnt = tl.load(tokens_cnts_ptr + i * num_experts + pid) + last_cnt = last_cnt + token_cnt + tl.store(tokens_cnts_ptr + i * num_experts + pid, last_cnt) + + +@triton.jit +def moe_align_block_size_stage3( + total_tokens_post_pad_ptr, + tokens_cnts_ptr, + cumsum_ptr, + num_experts: tl.constexpr, + block_size: tl.constexpr, +): + last_cumsum = 0 + off_cnt = num_experts * num_experts + for i in range(1, num_experts + 1): + token_cnt = tl.load(tokens_cnts_ptr + off_cnt + i - 1) + last_cumsum = last_cumsum + tl.cdiv(token_cnt, block_size) * block_size + tl.store(cumsum_ptr + i, last_cumsum) + tl.store(total_tokens_post_pad_ptr, last_cumsum) + + +@triton.jit +def moe_align_block_size_stage4( + topk_ids_ptr, + sorted_token_ids_ptr, + expert_ids_ptr, + tokens_cnts_ptr, + cumsum_ptr, + num_experts: tl.constexpr, + block_size: tl.constexpr, + numel: tl.constexpr, + tokens_per_thread: tl.constexpr, +): + pid = tl.program_id(0) + start_idx = tl.load(cumsum_ptr + pid) + end_idx = tl.load(cumsum_ptr + pid + 1) + + for i in range(start_idx, end_idx, block_size): + tl.store(expert_ids_ptr + i // block_size, pid) + + start_idx = pid * tokens_per_thread + off_t = pid * num_experts + + for i in range(start_idx, tl.minimum(start_idx + tokens_per_thread, numel)): + expert_id = tl.load(topk_ids_ptr + i) + token_cnt = tl.load(tokens_cnts_ptr + off_t + expert_id) + rank_post_pad = token_cnt + tl.load(cumsum_ptr + expert_id) + tl.store(sorted_token_ids_ptr + rank_post_pad, i) + tl.store(tokens_cnts_ptr + off_t + expert_id, token_cnt + 1) + + +def moe_align_block_size_triton( + topk_ids: torch.Tensor, + num_experts: int, + block_size: int, + sorted_token_ids: torch.Tensor, + expert_ids: torch.Tensor, + num_tokens_post_pad: torch.Tensor, +) -> None: + numel = topk_ids.numel() + grid = (num_experts,) + tokens_cnts = torch.zeros( + (num_experts + 1, num_experts), dtype=torch.int32, device=topk_ids.device + ) + cumsum = torch.zeros((num_experts + 1,), dtype=torch.int32, device=topk_ids.device) + tokens_per_thread = ceil_div(numel, num_experts) + + moe_align_block_size_stage1[grid]( + topk_ids, + tokens_cnts, + num_experts, + numel, + tokens_per_thread, + ) + moe_align_block_size_stage2[grid]( + tokens_cnts, + num_experts, + ) + moe_align_block_size_stage3[(1,)]( + num_tokens_post_pad, + tokens_cnts, + cumsum, + num_experts, + block_size, + ) + moe_align_block_size_stage4[grid]( + topk_ids, + sorted_token_ids, + expert_ids, + tokens_cnts, + cumsum, + num_experts, + block_size, + numel, + tokens_per_thread, + ) + + +def calculate_diff(num_tokens, num_experts=256, block_size=128, topk=8): + topk_ids = torch.stack( + [ + torch.randperm(num_experts, dtype=torch.int32, device="cuda")[:topk] + for _ in range(num_tokens) + ] + ) + + # SGL kernel uses dynamic padding optimization + max_num_tokens_padded_sgl = topk_ids.numel() + num_experts * (block_size - 1) + if topk_ids.numel() < num_experts + 1: + max_num_tokens_padded_sgl = topk_ids.numel() * block_size + sorted_ids_cuda = torch.empty( + (max_num_tokens_padded_sgl,), dtype=torch.int32, device=topk_ids.device + ) + sorted_ids_cuda.fill_(topk_ids.numel()) + max_num_m_blocks_sgl = max_num_tokens_padded_sgl // block_size + expert_ids_cuda = torch.zeros( + (max_num_m_blocks_sgl,), dtype=torch.int32, device=topk_ids.device + ) + num_tokens_post_pad_cuda = torch.empty( + (1), dtype=torch.int32, device=topk_ids.device + ) + cumsum_buffer = torch.zeros( + num_experts + 1, dtype=torch.int32, device=topk_ids.device + ) + + # Triton and vLLM use original padding calculation + max_num_tokens_padded_triton = topk_ids.numel() + num_experts * (block_size - 1) + max_num_m_blocks_triton = max_num_tokens_padded_triton // block_size + sorted_ids_triton = torch.empty( + (max_num_tokens_padded_triton,), dtype=torch.int32, device=topk_ids.device + ) + sorted_ids_triton.fill_(topk_ids.numel()) + expert_ids_triton = torch.zeros( + (max_num_m_blocks_triton,), dtype=torch.int32, device=topk_ids.device + ) + num_tokens_post_pad_triton = torch.empty_like(num_tokens_post_pad_cuda) + + sorted_ids_vllm = torch.empty_like(sorted_ids_triton) + sorted_ids_vllm.fill_(topk_ids.numel()) + expert_ids_vllm = torch.zeros_like(expert_ids_triton) + num_tokens_post_pad_vllm = torch.empty_like(num_tokens_post_pad_cuda) + + # compare the performance of cuda, triton and vllm implementation + sgl_moe_align_block_size( + topk_ids, + num_experts, + block_size, + sorted_ids_cuda, + expert_ids_cuda, + num_tokens_post_pad_cuda, + cumsum_buffer, + ) + moe_align_block_size_triton( + topk_ids, + num_experts, + block_size, + sorted_ids_triton, + expert_ids_triton, + num_tokens_post_pad_triton, + ) + + if VLLM_AVAILABLE: + try: + ops.moe_align_block_size( + topk_ids, + num_experts, + block_size, + sorted_ids_vllm, + expert_ids_vllm, + num_tokens_post_pad_vllm, + ) + print(f"✅ VLLM implementation works with {num_experts} experts!") + vllm_works = True + except Exception as e: + print(f"❌ VLLM implementation failed with {num_experts} experts: {e}") + vllm_works = False + else: + print("⚠️ vLLM not available, skipping vLLM test") + vllm_works = False + + if torch.allclose(expert_ids_cuda, expert_ids_triton) and torch.allclose( + num_tokens_post_pad_cuda, num_tokens_post_pad_triton + ): + print("✅ SGL and Triton implementations match") + else: + print("❌ SGL and Triton implementations do not match") + print("SGL expert_ids:", expert_ids_cuda) + print("Triton expert_ids:", expert_ids_triton) + print("SGL num_tokens_post_pad:", num_tokens_post_pad_cuda) + print("Triton num_tokens_post_pad:", num_tokens_post_pad_triton) + + if ( + vllm_works + and torch.allclose(expert_ids_cuda, expert_ids_vllm) + and torch.allclose(num_tokens_post_pad_cuda, num_tokens_post_pad_vllm) + ): + print("✅ SGL and VLLM implementations match") + else: + if not vllm_works: + print("⚠️ VLLM comparison skipped due to failure") + else: + print("❌ SGL and VLLM implementations do not match") + print("SGL expert_ids:", expert_ids_cuda) + print("VLLM expert_ids:", expert_ids_vllm) + print("SGL num_tokens_post_pad:", num_tokens_post_pad_cuda) + print("VLLM num_tokens_post_pad:", num_tokens_post_pad_vllm) + + +# Test range +num_tokens_range = [1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192] +num_experts_range = [8, 32, 64, 128, 256] +topk_range = [1, 2, 4, 8] + +configs = list(itertools.product(num_tokens_range, num_experts_range, topk_range)) + + +def get_topk_ids(num_tokens: int, num_experts: int, topk: int) -> torch.Tensor: + topk_ids = torch.zeros((num_tokens, topk), dtype=torch.int32, device="cuda") + for i in range(num_tokens): + topk_ids[i, :] = torch.randperm(num_experts, dtype=torch.int32, device="cuda")[ + :topk + ] + return topk_ids + + +def sgl_moe_align_block_size_with_empty( + topk_ids, + num_experts, + block_size, + sorted_ids, + expert_ids, + num_tokens_post_pad, + pad_sorted_token_ids=False, +): + if not pad_sorted_token_ids: + sorted_ids.fill_(topk_ids.numel()) + + cumsum_buffer = torch.empty( + num_experts + 1, dtype=torch.int32, device=topk_ids.device + ) + + sgl_moe_align_block_size( + topk_ids, + num_experts, + block_size, + sorted_ids.clone(), + expert_ids.clone(), + num_tokens_post_pad.clone(), + cumsum_buffer, + pad_sorted_token_ids, + ) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["num_tokens", "num_experts", "topk"], + x_vals=configs, + line_arg="provider", + line_vals=["sgl", "sgl_fusion", "triton"], + line_names=["SGL", "SGL Fusion", "Triton"], + styles=[("blue", "-"), ("red", "-"), ("green", "-")], + ylabel="us", + plot_name="moe-align-block-size-performance", + args={}, + ) +) +def benchmark(num_tokens, num_experts, topk, provider): + block_size = 128 + + if USE_RANDOM_PERM: + topk_ids = get_topk_ids(num_tokens, num_experts, topk) + else: + topk_ids = torch.randint( + 0, + num_experts, + (num_tokens, topk), + dtype=torch.int32, + device="cuda", + ) + + # Calculate max_num_tokens_padded based on provider + if provider == "sgl" or provider == "sgl_fusion": + # Apply dynamic padding optimization for SGL kernel + max_num_tokens_padded = topk_ids.numel() + num_experts * (block_size - 1) + if topk_ids.numel() < num_experts: + max_num_tokens_padded = topk_ids.numel() * block_size + else: # triton + # Use original padding calculation for Triton + max_num_tokens_padded = topk_ids.numel() + num_experts * (block_size - 1) + + # Create tensors + sorted_ids = torch.empty( + (max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device + ) + max_num_m_blocks = max_num_tokens_padded // block_size + expert_ids = torch.empty( + (max_num_m_blocks,), dtype=torch.int32, device=topk_ids.device + ) + num_tokens_post_pad = torch.empty((1), dtype=torch.int32, device=topk_ids.device) + + quantiles = [0.5, 0.2, 0.8] + if provider == "sgl": + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: sgl_moe_align_block_size_with_empty( + topk_ids, + num_experts, + block_size, + sorted_ids, + expert_ids, + num_tokens_post_pad, + ), + quantiles=quantiles, + ) + elif provider == "sgl_fusion": + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: sgl_moe_align_block_size_with_empty( + topk_ids, + num_experts, + block_size, + sorted_ids, + expert_ids, + num_tokens_post_pad, + pad_sorted_token_ids=True, + ), + quantiles=quantiles, + ) + elif provider == "triton": + sorted_ids.fill_(topk_ids.numel()) + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: moe_align_block_size_triton( + topk_ids, + num_experts, + block_size, + sorted_ids.clone(), + expert_ids.clone(), + num_tokens_post_pad.clone(), + ), + quantiles=quantiles, + ) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", + type=str, + default="./configs/benchmark_ops/moe_align_blocks/", + help="Path to save moe align benchmark results", + ) + parser.add_argument( + "--num_experts", + type=int, + default=256, + choices=[8, 16, 32, 64, 128, 256], + help="Number of experts for benchmark", + ) + parser.add_argument( + "--topk", + type=int, + default=8, + choices=[2, 4, 8], + help="Top-k value for benchmark", + ) + parser.add_argument( + "--skip_full_benchmark", + action="store_true", + help="Only run the calculate_diff function, skip full benchmarking", + ) + args = parser.parse_args() + + # Simplify for CI environment + if IS_CI: + num_tokens = 256 # Smaller for CI + num_experts = 8 # Smaller for CI + topk = 2 # Smaller for CI + else: + num_tokens = 1024 + num_experts = args.num_experts + topk = args.topk + + calculate_diff(num_tokens=num_tokens, num_experts=num_experts, topk=topk) + + if not args.skip_full_benchmark and not IS_CI: # Skip full benchmark in CI + print(f"\n📊 Running performance benchmark for {args.num_experts} experts...") + benchmark.run(print_data=True) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_moe_ep_post_reorder.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_moe_ep_post_reorder.py new file mode 100644 index 00000000..2a617d72 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_moe_ep_post_reorder.py @@ -0,0 +1,90 @@ +import os + +import torch + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) +import triton + +from sglang.srt.layers.moe.ep_moe.kernels import post_reorder_triton_kernel + +# CI environment uses simplified parameters +if IS_CI: + batch_sizes = [64, 128] # Only test 2 values in CI +else: + batch_sizes = [64, 128, 256, 512, 640, 768, 1024, 2048, 4096] + +configs = [(bs,) for bs in batch_sizes] + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=[list(_) for _ in configs], + line_arg="provider", + line_vals=["triton"], + line_names=["Triton Kernel"], + styles=[("orange", "-")], + ylabel="us", + plot_name="ep-moe-post-reorder-performance", + args={}, + ) +) +def benchmark(batch_size, provider): + dtype = torch.bfloat16 + device = torch.device("cuda") + hidden_size, topk, start_expert_id, end_expert_id, block_size = 4096, 8, 0, 255, 512 + + def alloc_tensors(): + down_output = torch.randn( + batch_size * topk, hidden_size, dtype=dtype, device=device + ) + output = torch.zeros(batch_size, hidden_size, dtype=dtype, device=device) + src2dst = torch.randint( + 0, batch_size * topk, (batch_size, topk), dtype=torch.int32, device=device + ) + topk_ids = torch.randint( + start_expert_id, + end_expert_id + 1, + (batch_size, topk), + dtype=torch.int32, + device=device, + ) + topk_weights = torch.rand(batch_size, topk, dtype=dtype, device=device) + return down_output, output, src2dst, topk_ids, topk_weights + + quantiles = [0.5, 0.2, 0.8] + + if provider == "triton": + d_out, out, s2d, tk_ids, tk_weights = alloc_tensors() + + def run_triton(): + post_reorder_triton_kernel[(batch_size,)]( + d_out.view(-1), + out.view(-1), + s2d.view(-1), + tk_ids.view(-1), + tk_weights.view(-1), + start_expert_id, + end_expert_id, + topk, + hidden_size, + 0, + block_size, + ) + + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + run_triton, quantiles=quantiles + ) + + else: + raise ValueError(f"Unknown provider: {provider}") + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + +if __name__ == "__main__": + benchmark.run(print_data=True) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_moe_fused_gate.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_moe_fused_gate.py new file mode 100644 index 00000000..cb5ac176 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_moe_fused_gate.py @@ -0,0 +1,89 @@ +import itertools +import math +import os + +import torch +import triton +import triton.language as tl +from sgl_kernel import moe_fused_gate + +from sglang.srt.layers.moe.topk import biased_grouped_topk + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + + +def biased_grouped_topk_org(scores, bias, num_expert_group, topk_group, topk): + return biased_grouped_topk( + scores, + scores, + bias, + topk=topk, + renormalize=True, + num_expert_group=num_expert_group, + topk_group=topk_group, + routed_scaling_factor=2.5, # DeepSeek-R1 : 2.5, Kimi K2: 2.872 + ) + + +def biased_grouped_topk_org_fuse_kernel( + scores, bias, num_expert_group, topk_group, topk +): + return moe_fused_gate(scores, bias, num_expert_group, topk_group, topk) + + +# CI environment uses simplified parameters +if IS_CI: + seq_length_range = [5000] # Only test one sequence length in CI +else: + seq_length_range = [5000, 10000, 15000, 20000, 25000, 30000, 35000, 40000] + +configs = [(sq,) for sq in seq_length_range] + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["seq_length"], + x_vals=[list(_) for _ in configs], + line_arg="provider", + line_vals=["original", "kernel"], + line_names=["Original", "SGL Kernel"], + styles=[("blue", "-"), ("red", "-")], + ylabel="us", + plot_name="moe-fused-gate-performance", + args={}, + ) +) +def benchmark(seq_length, provider): + dtype = torch.float32 + device = torch.device("cuda") + num_experts, num_expert_group, topk_group, topk = 256, 8, 4, 8 + + scores = torch.randn((seq_length, num_experts), device=device, dtype=dtype) + bias = torch.rand(num_experts, device=device, dtype=dtype) + + quantiles = [0.5, 0.2, 0.8] + + if provider == "original": + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: biased_grouped_topk_org( + scores.clone(), bias.clone(), num_expert_group, topk_group, topk + ), + quantiles=quantiles, + ) + elif provider == "kernel": + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: biased_grouped_topk_org_fuse_kernel( + scores.clone(), bias.clone(), num_expert_group, topk_group, topk + ), + quantiles=quantiles, + ) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + +if __name__ == "__main__": + benchmark.run(print_data=True) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_moe_topk_sigmoid.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_moe_topk_sigmoid.py new file mode 100644 index 00000000..d34e68b9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_moe_topk_sigmoid.py @@ -0,0 +1,171 @@ +import itertools +import os + +import pytest +import torch +import triton +from sgl_kernel import topk_sigmoid + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + + +def torch_topk_sigmoid_native( + gating_output: torch.Tensor, + topk: int, + renormalize: bool, + correction_bias: torch.Tensor = None, +): + scores = gating_output.sigmoid() + if correction_bias is not None: + n_routed_experts = gating_output.shape[-1] + scores_for_choice = scores.view( + -1, n_routed_experts + ) + correction_bias.unsqueeze(0) + _, topk_indices = torch.topk(scores_for_choice, k=topk, dim=-1) + topk_weights = scores.gather(1, topk_indices) + else: + topk_weights, topk_indices = torch.topk(scores, k=topk, dim=-1) + + if renormalize: + topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True) + + return topk_weights, topk_indices + + +def sglang_topk_sigmoid( + gating_output: torch.Tensor, + topk: int, + renormalize: bool, + correction_bias: torch.Tensor = None, +): + num_tokens, num_experts = gating_output.shape + + topk_weights = torch.empty((num_tokens, topk), dtype=torch.float32, device="cuda") + topk_indices = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda") + + topk_sigmoid( + topk_weights, + topk_indices, + gating_output, + renormalize=renormalize, + correction_bias=correction_bias, + ) + + return topk_weights, topk_indices + + +def get_topk_sigmoid_input(num_tokens, num_experts): + gating_output = torch.randn( + (num_tokens, num_experts), dtype=torch.float32, device="cuda" + ) + correction_bias = torch.randn((num_experts), dtype=torch.float32, device="cuda") + return gating_output, correction_bias + + +def calculate_diff(num_tokens, num_experts, topk): + gating_output, correction_bias = get_topk_sigmoid_input(num_tokens, num_experts) + + weights_torch, indices_torch = torch_topk_sigmoid_native( + gating_output.clone(), + topk, + True, + correction_bias.clone(), + ) + weights_sglang, indices_sglang = sglang_topk_sigmoid( + gating_output.clone(), + topk, + True, + correction_bias.clone(), + ) + + weights_diff = torch.abs(weights_torch - weights_sglang).mean().item() + indices_match = torch.equal(indices_torch, indices_sglang) + + if ( + torch.allclose(weights_torch, weights_sglang, atol=1e-3, rtol=1e-3) + and indices_match + ): + print("✅ Torch and SGLang topk_sigmoid implementations match") + else: + print( + f"❌ Implementations differ: Weights diff={weights_diff}, Indices match={indices_match}" + ) + + +# CI environment uses simplified parameters +if IS_CI: + num_tokens_range = [128] # Single value for CI + num_experts_range = [32] # Single value for CI + topk_range = [2] # Single value for CI +else: + num_tokens_range = [128, 512, 1024, 2048, 4096, 8192, 16384, 32768] + num_experts_range = [32, 64, 128, 256, 12, 512] + topk_range = [1, 2, 4, 8] + +configs = list(itertools.product(num_tokens_range, num_experts_range, topk_range)) + + +# Filter providers based on vLLM availability +line_vals = ["sglang", "torch"] +line_names = ["SGLang", "Torch"] +styles = [("blue", "-"), ("green", "-")] + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["num_tokens", "num_experts", "topk"], + x_vals=configs, + line_arg="provider", + line_vals=line_vals, + line_names=line_names, + styles=styles, + ylabel="Latency (us)", + plot_name="topk-sigmoid-performance", + args={}, + ) +) +def benchmark(num_tokens, num_experts, topk, provider): + gating_output, correction_bias = get_topk_sigmoid_input(num_tokens, num_experts) + + if provider == "torch" or provider == "torch1": + + def fn(): + return torch_topk_sigmoid_native( + gating_output, + topk, + True, + correction_bias, + ) + + elif provider == "sglang" or provider == "sglang1": + + def fn(): + return sglang_topk_sigmoid(gating_output, topk, True, correction_bias) + + quantiles = [0.5, 0.2, 0.8] + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph(fn, quantiles=quantiles) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + +if __name__ == "__main__": + # Simplify configs for CI environment + if IS_CI: + test_configs = [(20, 32, 2)] # Single config for CI + else: + test_configs = [ + (20, 256, 4), + (20, 256, 8), + (20, 12, 4), + (20, 12, 1), + (20, 512, 4), + (20, 512, 1), + ] + + for num_tokens, num_experts, topk in test_configs: + calculate_diff(num_tokens, num_experts, topk) + benchmark.run(print_data=True) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_moe_topk_softmax.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_moe_topk_softmax.py new file mode 100644 index 00000000..e065981b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_moe_topk_softmax.py @@ -0,0 +1,163 @@ +import itertools +import os + +import pytest +import torch +import triton +from sgl_kernel import topk_softmax + +# Optional vLLM import +try: + from vllm import _custom_ops as vllm_custom_ops + + VLLM_AVAILABLE = True +except ImportError: + vllm_custom_ops = None + VLLM_AVAILABLE = False + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + + +def vllm_topk_softmax(gating_output, topk): + if not VLLM_AVAILABLE: + # Fallback to SGLang implementation if vLLM is not available + return sglang_topk_softmax(gating_output, topk) + + num_tokens, num_experts = gating_output.shape + + topk_weights = torch.empty( + (num_tokens, topk), device=gating_output.device, dtype=torch.float32 + ) + topk_indices = torch.empty( + (num_tokens, topk), dtype=torch.int32, device=gating_output.device + ) + token_expert_indices = torch.empty( + (num_tokens, topk), dtype=torch.int32, device=gating_output.device + ) + torch.ops._moe_C.topk_softmax( + topk_weights, topk_indices, token_expert_indices, gating_output + ) + return topk_weights, topk_indices + + +def sglang_topk_softmax(gating_output, topk): + num_tokens, num_experts = gating_output.shape + + topk_weights = torch.empty( + (num_tokens, topk), device=gating_output.device, dtype=torch.float32 + ) + topk_indices = torch.empty( + (num_tokens, topk), dtype=torch.int32, device=gating_output.device + ) + + topk_softmax( + topk_weights=topk_weights, + topk_ids=topk_indices, + gating_output=gating_output, + ) + + return topk_weights, topk_indices + + +def calculate_diff(num_tokens, num_experts, topk): + gating_output = torch.randn( + (num_tokens, num_experts), device="cuda", dtype=torch.float32 + ) + weights_vllm, indices_vllm = vllm_topk_softmax(gating_output.clone(), topk) + weights_sglang, indices_sglang = sglang_topk_softmax(gating_output.clone(), topk) + + weights_diff = torch.abs(weights_vllm - weights_sglang).mean().item() + indices_match = torch.equal(indices_vllm, indices_sglang) + + if not VLLM_AVAILABLE: + print("⚠️ vLLM not available, skipping comparison") + return + + if ( + torch.allclose(weights_vllm, weights_sglang, atol=1e-3, rtol=1e-3) + and indices_match + ): + print("✅ VLLM and SGLang topk_softmax implementations match") + else: + print( + f"❌ Implementations differ: Weights diff={weights_diff}, Indices match={indices_match}" + ) + + +# CI environment uses simplified parameters +if IS_CI: + num_tokens_range = [128] # Single value for CI + num_experts_range = [32] # Single value for CI + topk_range = [2] # Single value for CI +else: + num_tokens_range = [128, 512, 1024, 2048, 4096, 8192, 16384, 32768] + num_experts_range = [32, 64, 128, 256, 12, 512] + topk_range = [1, 2, 4, 8] + +configs = list(itertools.product(num_tokens_range, num_experts_range, topk_range)) + + +# Filter providers based on vLLM availability +if VLLM_AVAILABLE: + line_vals = ["sglang", "vllm"] + line_names = ["SGLang", "VLLM"] + styles = [("blue", "-"), ("green", "-")] +else: + line_vals = ["sglang"] + line_names = ["SGLang"] + styles = [("blue", "-")] + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["num_tokens", "num_experts", "topk"], + x_vals=configs, + line_arg="provider", + line_vals=line_vals, + line_names=line_names, + styles=styles, + ylabel="Latency (us)", + plot_name="topk-softmax-performance", + args={}, + ) +) +def benchmark(num_tokens, num_experts, topk, provider): + + gating_output = torch.randn( + (num_tokens, num_experts), device="cuda", dtype=torch.float32 + ) + + if provider == "vllm" or provider == "vllm1": + if not VLLM_AVAILABLE: + return (0, 0, 0) + fn = lambda: vllm_topk_softmax(gating_output, topk) + elif provider == "sglang" or provider == "sglang1": + fn = lambda: sglang_topk_softmax(gating_output, topk) + + quantiles = [0.5, 0.2, 0.8] + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph(fn, quantiles=quantiles) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + +if __name__ == "__main__": + # Simplify configs for CI environment + if IS_CI: + test_configs = [(20, 32, 2)] # Single config for CI + else: + test_configs = [ + (20, 256, 4), + (20, 256, 8), + (20, 12, 4), + (20, 12, 1), + (20, 512, 4), + (20, 512, 1), + ] + + for num_tokens, num_experts, topk in test_configs: + calculate_diff(num_tokens, num_experts, topk) + benchmark.run(print_data=True) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_mrope.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_mrope.py new file mode 100644 index 00000000..ec0a1585 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_mrope.py @@ -0,0 +1,250 @@ +# Adapted from vLLM benchmark_mrope.py + +# This script benchmarks the mrope kernel (mainly for Qwen2VL and Qwen2.5VL models). +# It generates test data, runs benchmarks, and saves results to a CSV file. +# +# The CSV file (named with current date/time) contains these columns: +# model_name, tp_size, num_tokens, num_heads, num_kv_heads, head_dim, max_position, +# rope_theta, is_neox_style, rope_scaling, dtype, torch_mean, torch_median, torch_p99, +# torch_min, torch_max, triton_mean, triton_median, triton_p99, triton_min, triton_max, +# speedup +# +# == Usage Examples == +# +# Single model benchmark: +# python3 benchmark_mrope.py --model-name Qwen/Qwen2.5-VL-7B-Instruct --tp-size 8 \ +# --warmup-iter 10 --benchmark-iter 100 --dtype bfloat16 --seed 0 --num-tokens 1024 + +import argparse +import time +from typing import Any + +import numpy as np +import torch +from transformers import AutoConfig + +from sglang.srt.layers.rotary_embedding import get_rope + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + +def get_model_config(model_name: str): + """Get model configuration parameters""" + config = AutoConfig.from_pretrained(model_name, trust_remote_code=True) + return config + + +def generate_test_data( + num_tokens: int, + num_q_heads: int, + num_kv_heads: int, + head_size: int, + max_position_embeddings: int, + dtype: torch.dtype, + device: torch.device, +): + """Generate test data for given configuration.""" + # Create 2D positions (3, num_tokens) for multimodal case + positions = torch.randint( + 0, max_position_embeddings // 4, (3, num_tokens), device=device + ) + + # Create query and key tensors + query = torch.randn(num_tokens, num_q_heads * head_size, dtype=dtype, device=device) + key = torch.randn(num_tokens, num_kv_heads * head_size, dtype=dtype, device=device) + + return positions, query, key + + +def calculate_stats(times: list[float]) -> dict[str, float]: + """Calculate statistics from a list of times.""" + times_array = np.array(times) + return { + "mean": np.mean(times_array), + "median": np.median(times_array), + "p99": np.percentile(times_array, 99), + "min": np.min(times_array), + "max": np.max(times_array), + } + + +def benchmark_mrope( + model_name: str, + num_tokens: int, + head_dim: int, + tp_size: int, + num_heads: int, + num_kv_heads: int, + max_position: int = 8192, + rope_theta: float = 10000, + is_neox_style: bool = True, + rope_scaling: dict[str, Any] = None, + dtype: torch.dtype = torch.bfloat16, + seed: int = 0, + warmup_iter: int = 10, + benchmark_iter: int = 100, +): + torch.manual_seed(seed) + torch.set_default_device(device) + # the parameters to compute the q k v size based on tp_size + mrope_helper_class = get_rope( + head_size=head_dim, + rotary_dim=head_dim, + max_position=max_position, + base=rope_theta, + is_neox_style=is_neox_style, + rope_scaling=rope_scaling, + dtype=dtype, + ).to(device=device) + + print(80 * "=") + print( + f"Evaluating model: {model_name} " + f"with tp_size: {tp_size} " + f"and num_tokens: {num_tokens}, " + f"dtype: {dtype}" + ) + + # create q k v input tensors + # create rotary pos emb input tensors + positions, query, key = generate_test_data( + num_tokens, num_heads, num_kv_heads, head_dim, max_position, dtype, device + ) + + # Warm up + for _ in range(warmup_iter): + mrope_helper_class.forward_native( + positions, + query.clone(), + key.clone(), + ) + + mrope_helper_class.forward( + positions, + query.clone(), + key.clone(), + ) + + torch.cuda.synchronize() + + # Time reference implementation + torch_times = [] + for _ in range(benchmark_iter): + query_clone = query.clone() + key_clone = key.clone() + torch.cuda.synchronize() + start_time = time.time() + + mrope_helper_class.forward_native( + positions, + query_clone, + key_clone, + ) + + torch.cuda.synchronize() + torch_times.append(time.time() - start_time) + + # Time triton kernel implementation + triton_times = [] + for _ in range(benchmark_iter): + query_clone = query.clone() + key_clone = key.clone() + torch.cuda.synchronize() + start_time = time.time() + mrope_helper_class.forward( + positions, + query_clone, + key_clone, + ) + torch.cuda.synchronize() + triton_times.append(time.time() - start_time) + + # Calculate statistics + torch_stats = calculate_stats(torch_times) + triton_stats = calculate_stats(triton_times) + print(f"\nPerformance for config ({num_tokens}, {num_heads}, {num_kv_heads}):") + + print( + f"Torch implementation: " + f"mean={torch_stats['mean']:.8f}s, " + f"median={torch_stats['median']:.8f}s, " + f"p99={torch_stats['p99']:.8f}s" + ) + + print( + f"Triton implementation: " + f"mean={triton_stats['mean']:.8f}s, " + f"median={triton_stats['median']:.8f}s, " + f"p99={triton_stats['p99']:.8f}s" + ) + + print( + f"Triton Speedup over Torch: {torch_stats['mean'] / triton_stats['mean']:.8f}x" + ) + + return torch_stats, triton_stats + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Benchmark the rotary embedding kernels." + ) + parser.add_argument("--model-name", type=str, default="") + parser.add_argument("--tp-size", type=int, default=1) + parser.add_argument("--warmup-iter", type=int, default=10) + parser.add_argument("--benchmark-iter", type=int, default=100) + parser.add_argument("--dtype", type=str, choices=["bfloat16"], default="bfloat16") + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--num-tokens", type=int, nargs="+", required=False) + parser.add_argument("--trust-remote-code", action="store_true") + args = parser.parse_args() + print(args) + + model_tp_dict = {} + if args.model_name == "": + model_tp_dict = { + "Qwen/Qwen2-VL-2B-Instruct": [1], + "Qwen/Qwen2-VL-7B-Instruct": [1], + "Qwen/Qwen2-VL-72B-Instruct": [2, 4, 8], + "Qwen/Qwen2.5-VL-3B-Instruct": [1, 2, 4, 8], + "Qwen/Qwen2.5-VL-7B-Instruct": [1, 2, 4, 8], + "Qwen/Qwen2.5-VL-72B-Instruct": [2, 4, 8], + } + else: + model_tp_dict[args.model_name] = [args.tp_size] + + if args.num_tokens is None: + num_tokens_list = [2**i for i in range(0, 18)] + else: + num_tokens_list = args.num_tokens + + for model_name, tp_list in model_tp_dict.items(): + for tp_size in tp_list: + config = get_model_config(model_name) + # get the model config + total_num_kv_heads = config.num_key_value_heads + total_num_heads = config.num_attention_heads + num_heads = total_num_heads // tp_size + num_kv_heads = max(1, total_num_kv_heads // tp_size) + head_dim = config.hidden_size // total_num_heads + is_neox_style = True + rope_theta = config.rope_theta + max_position = config.max_position_embeddings + + for num_tokens in num_tokens_list: + benchmark_mrope( + model_name=model_name, + num_tokens=num_tokens, + head_dim=head_dim, + tp_size=tp_size, + num_heads=num_heads, + num_kv_heads=num_kv_heads, + max_position=max_position, + rope_theta=rope_theta, + is_neox_style=is_neox_style, + rope_scaling=config.rope_scaling, + dtype=getattr(torch, args.dtype), + seed=args.seed, + warmup_iter=args.warmup_iter, + benchmark_iter=args.benchmark_iter, + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_nvfp4_scaled_gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_nvfp4_scaled_gemm.py new file mode 100644 index 00000000..3867f609 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_nvfp4_scaled_gemm.py @@ -0,0 +1,192 @@ +import argparse +import copy +import itertools +import os + +import torch +import triton +from sgl_kernel import cutlass_scaled_fp4_mm, scaled_fp4_quant + +from sglang.srt.utils import get_device_capability + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + +FLOAT4_E2M1_MAX = 6.0 +FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max + +# Weight Shapes are in the format +# ([K, N], TP_SPLIT_DIM) +# Example: +# A shape of ([14336, 4096], 0) indicates the following GEMM shape, +# - TP1 : K = 14336, N = 4096 +# - TP2 : K = 7168, N = 4096 +# A shape of ([4096, 6144], 1) indicates the following GEMM shape, +# - TP1 : K = 4096, N = 6144 +# - TP4 : K = 4096, N = 1536 + +# TP1 shapes +WEIGHT_SHAPES = { + "meta-llama/Llama-3.1-8B-Instruct": [ + ([4096, 6144], 1), + ([4096, 4096], 0), + ([4096, 28672], 1), + ([14336, 4096], 0), + ], + "meta-llama/Llama-3.3-70B-Instruct": [ + ([8192, 10240], 1), + ([8192, 8192], 0), + ([8192, 57344], 1), + ([28672, 8192], 0), + ], + "mistralai/Mistral-Large-Instruct-2407": [ + ([12288, 14336], 1), + ([12288, 12288], 0), + ([12288, 57344], 1), + ([28672, 12288], 0), + ], + "Qwen/Qwen2.5-7B-Instruct": [ + ([3584, 4608], 1), + ([3584, 3584], 0), + ([3584, 37888], 1), + ([18944, 3584], 0), + ], + "Qwen/Qwen2.5-32B-Instruct": [ + ([5120, 7168], 1), + ([5120, 5120], 0), + ([5120, 55296], 1), + ([27648, 5120], 0), + ], + "Qwen/Qwen2.5-72B-Instruct": [ + ([8192, 10240], 1), + ([8192, 8192], 0), + ([8192, 59136], 1), + ([29568, 8192], 0), + ], + "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": [ + ([2048, 3072], 1), + ([2048, 4096], 1), + ([2048, 2048], 0), + ([2048, 576], 0), + ([2048, 21888], 1), + ([10944, 2048], 0), + ([2048, 2816], 1), + ([1408, 2048], 0), + ], +} + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=[1, 16, 64, 128, 256, 512, 1024, 2048], + x_log=False, + line_arg="provider", + line_vals=[ + "sglang-fp4-fp16", + "sglang-fp4-bf16", + ], + line_names=[ + "sglang-fp4-fp16", + "sglang-fp4-bf16", + ], + styles=[("green", "-"), ("blue", "-")], + ylabel="TFLOPS", + plot_name="fp4 block scaled matmul", + args={}, + ) +) +def benchmark(batch_size, provider, N, K): + # M, N, K = batch_size, 4096, 8192 + run_step = 100 + dtype = torch.float16 if "fp16" in provider else torch.bfloat16 + M = batch_size + a = torch.randn((M, K), dtype=dtype, device="cuda") + b = torch.randn((N, K), dtype=dtype, device="cuda") + a_global_scale = ( + (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(a.flatten(), dim=-1) + ).to(torch.float32) + b_global_scale = ( + (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(b.flatten(), dim=-1) + ).to(torch.float32) + alpha = 1.0 / (a_global_scale * b_global_scale) + a_fp4, a_scale_interleaved = scaled_fp4_quant(a, a_global_scale) + b_fp4, b_scale_interleaved = scaled_fp4_quant(b, b_global_scale) + + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + + # Bridging the gap between CPU and GPU + for _ in range(25): + c = a @ b.t() + # Warmup + for _ in range(5): + cutlass_scaled_fp4_mm( + a_fp4, b_fp4, a_scale_interleaved, b_scale_interleaved, alpha, dtype + ) + start_event.record() + for _ in range(run_step): + cutlass_scaled_fp4_mm( + a_fp4, b_fp4, a_scale_interleaved, b_scale_interleaved, alpha, dtype + ) + end_event.record() + end_event.synchronize() + torch.cuda.synchronize() + ms = start_event.elapsed_time(end_event) / run_step + + tflops = lambda ms: (2 * M * N * K) * 1e-9 / ms + return tflops(ms) + + +def prepare_shapes(args): + KN_model_names = [] + models_tps = list(itertools.product(args.models, args.tp_sizes)) + for model, tp_size in models_tps: + assert model in WEIGHT_SHAPES + for KN, tp_split_dim in copy.deepcopy(WEIGHT_SHAPES[model]): + KN[tp_split_dim] = KN[tp_split_dim] // tp_size + KN.append(model) + KN_model_names.append(KN) + return KN_model_names + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--models", + nargs="+", + type=str, + default=["meta-llama/Llama-3.1-8B-Instruct"], + help="List of models to benchmark", + ) + parser.add_argument( + "--tp-sizes", + nargs="+", + type=int, + default=[1], + help="List of tensor parallel sizes", + ) + args = parser.parse_args() + + # Check architecture compatibility - FP4 operations require sm100a/sm103a + major, minor = get_device_capability() + if major is None or major < 10: # Requires compute capability 10.0+ (sm100a/sm103a) + print("Skipping NVIDIA FP4 scaled GEMM benchmark") + if major is not None: + print(f"FP4 operations require sm100a/sm103a, but found sm{major}{minor}") + else: + print("Could not determine device capability") + else: + KN_model_names = prepare_shapes(args) + + # Limit iterations in CI + if IS_CI: + KN_model_names = KN_model_names[:2] # Only test first 2 shapes in CI + + for K, N, model_name in KN_model_names: + print(f"{model_name} N={N} K={K}: ") + benchmark.run(print_data=True, N=N, K=K) + print("Benchmark finished!") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_per_tensor_quant_fp8.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_per_tensor_quant_fp8.py new file mode 100644 index 00000000..ead9e9aa --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_per_tensor_quant_fp8.py @@ -0,0 +1,126 @@ +import itertools +import math +import os +from typing import Any, Dict, List, Optional, Tuple + +import numpy as np +import torch +import triton +import triton.testing +from sgl_kernel import sgl_per_tensor_quant_fp8 + +# Optional imports +try: + from vllm import _custom_ops as ops + + VLLM_AVAILABLE = True +except ImportError: + ops = None + VLLM_AVAILABLE = False + +from sglang.srt.utils import is_hip + +_is_hip = is_hip() + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + +fp8_type_ = torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn + + +def vllm_scaled_fp8_quant( + input: torch.Tensor, + scale: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + if not VLLM_AVAILABLE: + # Fallback to SGLang implementation + return sglang_scaled_fp8_quant(input, scale) + return ops.scaled_fp8_quant(input, scale) + + +def sglang_scaled_fp8_quant( + input: torch.Tensor, + scale: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + fp8_type_: torch.dtype = torch.float8_e4m3fn + output = torch.empty_like(input, device=input.device, dtype=fp8_type_) + is_static = True + if scale is None: + scale = torch.zeros(1, device=input.device, dtype=torch.float32) + is_static = False + sgl_per_tensor_quant_fp8(input, output, scale, is_static) + + return output, scale + + +def calculate_diff(batch_size: int, seq_len: int): + """Calculate difference between VLLM and SGLang implementations.""" + device = torch.device("cuda") + x = torch.rand((batch_size, seq_len), dtype=torch.float16, device=device) + + if not VLLM_AVAILABLE: + print("⚠️ vLLM not available, skipping comparison") + return + + vllm_out, vllm_scale = vllm_scaled_fp8_quant(x) + sglang_out, sglang_scale = sglang_scaled_fp8_quant(x) + + scale_diff = torch.abs(vllm_scale - sglang_scale).item() + output_diff = torch.abs(vllm_out.float() - sglang_out.float()).mean().item() + + if torch.allclose( + vllm_out.to(torch.float32), sglang_out.to(torch.float32), rtol=1e-3, atol=1e-5 + ) and torch.allclose(vllm_scale, sglang_scale, rtol=1e-3, atol=1e-5): + print("✅ All implementations match") + else: + print("❌ Implementations differ") + + +# CI environment uses simplified parameters +if IS_CI: + batch_size_range = [16] # Single batch size for CI + seq_len_range = [64] # Single sequence length for CI +else: + batch_size_range = [16, 32, 64, 128] + seq_len_range = [64, 128, 256, 512, 1024, 2048] + +configs = list(itertools.product(batch_size_range, seq_len_range)) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size", "seq_len"], + x_vals=configs, + line_arg="provider", + line_vals=["vllm", "sglang"], + line_names=["VLLM", "SGL Kernel"], + styles=[("blue", "-"), ("green", "-")], + ylabel="us", + plot_name="per-tensor-quant-fp8-performance", + args={}, + ) +) +def benchmark(batch_size, seq_len, provider): + dtype = torch.float16 + device = torch.device("cuda") + + x = torch.randn(batch_size * seq_len, 4096, device=device, dtype=dtype) + + quantiles = [0.5, 0.2, 0.8] + + if provider == "vllm": + fn = lambda: vllm_scaled_fp8_quant(x.clone()) + elif provider == "sglang": + fn = lambda: sglang_scaled_fp8_quant(x.clone()) + + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph(fn, quantiles=quantiles) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + +if __name__ == "__main__": + calculate_diff(batch_size=4, seq_len=4096) + benchmark.run(print_data=True) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_per_token_group_quant_8bit.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_per_token_group_quant_8bit.py new file mode 100644 index 00000000..04610b17 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_per_token_group_quant_8bit.py @@ -0,0 +1,246 @@ +import itertools +import os +import time +from functools import partial +from pathlib import Path + +import torch +import triton +from sgl_kernel.test_utils import create_per_token_group_quant_test_data + +from sglang.srt.layers.quantization.fp8_kernel import ( + create_per_token_group_quant_fp8_output_scale, +) +from sglang.srt.layers.quantization.fp8_kernel import ( + per_token_group_quant_8bit as triton_per_token_group_quant_8bit, +) +from sglang.srt.layers.quantization.fp8_kernel import sglang_per_token_group_quant_8bit +from sglang.srt.utils import is_hip +from sglang.srt.utils.bench_utils import bench_kineto + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + +_is_hip = is_hip() +fp8_type_ = torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn + + +mode_concentrated = IS_CI or (os.environ.get("SGLANG_BENCH_MODE", "") == "concentrated") + +if int(os.environ.get("SGLANG_NSYS_PROFILING", "0")): + configs = [ + [ + 768 * 8, + 2048, + 128, + 48, + fp8_type_, + dict( + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + fuse_silu_and_mul=True, + # masked_layout_mode=None, + masked_layout_mode="balanced", + # masked_layout_mode="extreme", + ), + ] + ] +elif mode_concentrated: + configs = list( + itertools.product( + [768], + [1536, 7168, 16384], + [128], + [None], + [fp8_type_], + [ + dict( + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + fuse_silu_and_mul=False, + masked_layout_mode=None, + ), + ], + ) + ) + list( + itertools.product( + [768 * 8], + [2048], + [128], + [48], + [fp8_type_], + [ + dict( + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + fuse_silu_and_mul=True, + masked_layout_mode=None, + ), + dict( + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + fuse_silu_and_mul=True, + masked_layout_mode="balanced", + ), + dict( + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + fuse_silu_and_mul=True, + masked_layout_mode="imbalanced", + ), + dict( + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + fuse_silu_and_mul=True, + masked_layout_mode="extreme", + ), + ], + ) + ) +else: + configs = list( + itertools.product( + [1, 4, 16, 64, 256, 768, 2048, 8192, 16384], + [1536, 7168, 16384], + [128], + [None], + [fp8_type_], + [ + dict( + column_major_scales=False, + scale_tma_aligned=False, + scale_ue8m0=False, + fuse_silu_and_mul=False, + masked_layout_mode=None, + ), + dict( + column_major_scales=True, + scale_tma_aligned=False, + scale_ue8m0=False, + fuse_silu_and_mul=False, + masked_layout_mode=None, + ), + dict( + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=False, + fuse_silu_and_mul=False, + masked_layout_mode=None, + ), + dict( + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + fuse_silu_and_mul=False, + masked_layout_mode=None, + ), + ], + ) + ) + list( + itertools.product( + [1 * 8, 4 * 8, 64 * 8, 256 * 8, 768 * 8], + [2048], + [128], + [8, 16, 32, 48], + [fp8_type_], + [ + dict( + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + fuse_silu_and_mul=True, + masked_layout_mode=None, + ), + dict( + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + fuse_silu_and_mul=True, + masked_layout_mode="balanced", + ), + dict( + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + fuse_silu_and_mul=True, + masked_layout_mode="imbalanced", + ), + dict( + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + fuse_silu_and_mul=True, + masked_layout_mode="extreme", + ), + ], + ) + ) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=[ + "num_tokens", + "hidden_dim", + "group_size", + "num_ranks", + "dst_dtype", + "flags", + ], + x_vals=configs, + line_arg="provider", + line_vals=["triton", "sglang"], + # Triton has multi kernels and we only report the time for the core one + line_names=["Triton (Inaccurate)", "SGL Kernel"], + styles=[("blue", "-"), ("green", "-")], + ylabel="us", + plot_name="per-token-group-quant-8bit-performance", + args={}, + ) +) +def benchmark( + num_tokens, hidden_dim, group_size, num_ranks, dst_dtype, flags, provider +): + print( + f"Testing: {num_tokens=} {hidden_dim=} {group_size=} {num_ranks=} {dst_dtype=} {flags=} {provider=}" + ) + + x, masked_m = create_per_token_group_quant_test_data( + num_tokens=num_tokens, hidden_dim=hidden_dim, num_ranks=num_ranks, flags=flags + ) + + fn, kernel_names = { + "triton": ( + triton_per_token_group_quant_8bit, + "_per_token_group_quant_8bit|_silu_and_mul_post_quant_kernel", + ), + "sglang": ( + partial(sglang_per_token_group_quant_8bit, enable_v2=True), + "per_token_group_quant_8bit_kernel", + ), + }[provider] + bench_fn = lambda: fn( + x=x, + masked_m=masked_m, + group_size=group_size, + dst_dtype=dst_dtype, + **{k: v for k, v in flags.items() if k not in ["masked_layout_mode"]}, + ) + + time_s = bench_kineto( + bench_fn, kernel_names=kernel_names, num_tests=300 if mode_concentrated else 30 + ) + return time_s * 1e6 + + +if __name__ == "__main__": + benchmark.run(print_data=True) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_per_token_quant_fp8.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_per_token_quant_fp8.py new file mode 100644 index 00000000..8db1869d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_per_token_quant_fp8.py @@ -0,0 +1,230 @@ +import itertools +import os +from typing import Optional, Tuple + +import torch +import triton +import triton.testing +from sgl_kernel import sgl_per_token_quant_fp8 + +# Optional vLLM import +try: + from vllm import _custom_ops as ops + + VLLM_AVAILABLE = True +except ImportError: + ops = None + VLLM_AVAILABLE = False + +from sglang.srt.utils import is_hip + +_is_hip = is_hip() + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + +fp8_type_ = torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn + +# Get correct FP8 E4M3 maximum value +if _is_hip: + FP8_E4M3_MAX = 224.0 # ROCM uses 224.0 +else: + # For CUDA, get the actual max value from the type + FP8_E4M3_MAX = float(torch.finfo(fp8_type_).max) + + +def torch_per_token_quant_fp8( + input: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Pure PyTorch reference implementation for per-token FP8 quantization.""" + device = input.device + dtype = input.dtype + + # Find max absolute value per token (row) - exactly like CUDA kernel + max_vals = torch.abs(input).max(dim=1)[0] # [num_tokens] + + # Calculate scale per token - exactly like CUDA kernel: scale = max_value / FP8_E4M3_MAX + scales = max_vals / FP8_E4M3_MAX # [num_tokens] + + # No special zero handling - directly compute 1.0 / scale like CUDA kernel + scale_inv = 1.0 / scales # [num_tokens] + + # Quantize: input * scale_inv, then clamp to FP8 range + quantized_float = input * scale_inv.unsqueeze(1) # Broadcast scale_inv + quantized_float = torch.clamp(quantized_float, -FP8_E4M3_MAX, FP8_E4M3_MAX) + + # Convert to FP8 - use more explicit conversion + quantized_fp8 = quantized_float.to(fp8_type_) + + return quantized_fp8, scales + + +def vllm_per_token_quant_fp8( + input: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + if not VLLM_AVAILABLE: + # Fallback to SGLang implementation + return sglang_per_token_quant_fp8(input) + return ops.scaled_fp8_quant(input, use_per_token_if_dynamic=True) + + +def sglang_per_token_quant_fp8( + input: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + scale = torch.zeros(input.size(0), device=input.device, dtype=torch.float32) + output = torch.empty_like(input, device=input.device, dtype=fp8_type_) + sgl_per_token_quant_fp8(input, output, scale) + + return output, scale + + +def calculate_diff(batch_size: int, seq_len: int, hidden_dim: int): + """Compare Torch reference, VLLM, and SGLang implementations.""" + device = torch.device("cuda") + x = torch.rand( + (batch_size * seq_len, hidden_dim), dtype=torch.float16, device=device + ) + + # Get all three implementations + torch_out, torch_scale = torch_per_token_quant_fp8(x) + vllm_out, vllm_scale = vllm_per_token_quant_fp8(x) + sglang_out, sglang_scale = sglang_per_token_quant_fp8(x) + + if not VLLM_AVAILABLE: + print("⚠️ vLLM not available, skipping vLLM comparison") + # Only compare Torch vs SGLang + torch_sglang_scale_diff = torch.abs(torch_scale - sglang_scale).mean().item() + torch_sglang_out_diff = ( + torch.abs(torch_out.float() - sglang_out.float()).mean().item() + ) + print(f"Scale difference (Torch vs SGLang): {torch_sglang_scale_diff:.8f}") + print(f"Output difference (Torch vs SGLang): {torch_sglang_out_diff:.8f}") + return + + print(f"\n=== Comparison for hidden_dim={hidden_dim} ===") + + # Compare scales + torch_vllm_scale_diff = torch.abs(torch_scale - vllm_scale).mean().item() + torch_sglang_scale_diff = torch.abs(torch_scale - sglang_scale).mean().item() + vllm_sglang_scale_diff = torch.abs(vllm_scale - sglang_scale).mean().item() + + print(f"Scale differences:") + print(f" Torch vs VLLM: {torch_vllm_scale_diff:.8f}") + print(f" Torch vs SGLang: {torch_sglang_scale_diff:.8f}") + print(f" VLLM vs SGLang: {vllm_sglang_scale_diff:.8f}") + + # Compare outputs + torch_vllm_out_diff = torch.abs(torch_out.float() - vllm_out.float()).mean().item() + torch_sglang_out_diff = ( + torch.abs(torch_out.float() - sglang_out.float()).mean().item() + ) + vllm_sglang_out_diff = ( + torch.abs(vllm_out.float() - sglang_out.float()).mean().item() + ) + + print(f"Output differences:") + print(f" Torch vs VLLM: {torch_vllm_out_diff:.8f}") + print(f" Torch vs SGLang: {torch_sglang_out_diff:.8f}") + print(f" VLLM vs SGLang: {vllm_sglang_out_diff:.8f}") + + # Check tolerances + rtol, atol = 1e-3, 1e-5 + + torch_vllm_match = torch.allclose( + torch_out.float(), vllm_out.float(), rtol=rtol, atol=atol + ) and torch.allclose(torch_scale, vllm_scale, rtol=rtol, atol=atol) + torch_sglang_match = torch.allclose( + torch_out.float(), sglang_out.float(), rtol=rtol, atol=atol + ) and torch.allclose(torch_scale, sglang_scale, rtol=rtol, atol=atol) + + if hidden_dim == 1368: + rtol = 1e-2 + # we found vllm sglang has diff when hidden dim is not dividable by 16 + # and we believe SGLang is closer to Torch implementation + + vllm_sglang_match = torch.allclose( + vllm_out.float(), sglang_out.float(), rtol=rtol, atol=atol + ) and torch.allclose(vllm_scale, sglang_scale, rtol=rtol, atol=atol) + + print(f"Matches (rtol={rtol}, atol={atol}):") + print(f" Torch vs VLLM: {'✅' if torch_vllm_match else '❌'}") + print(f" Torch vs SGLang: {'✅' if torch_sglang_match else '❌'}") + print(f" VLLM vs SGLang: {'✅' if vllm_sglang_match else '❌'}") + + +# CI environment uses simplified parameters +if IS_CI: + batch_size_range = [16] # Single batch size for CI + seq_len_range = [64] # Single sequence length for CI + hidden_dim_range = [2048] # Single hidden dimension for CI +else: + batch_size_range = [16, 32, 64, 128] + seq_len_range = [64, 128, 256, 512, 1024, 2048, 4096] + hidden_dim_range = [1368, 2048, 4096] + +configs = list(itertools.product(batch_size_range, seq_len_range, hidden_dim_range)) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size", "seq_len", "hidden_dim"], + x_vals=configs, + line_arg="provider", + line_vals=( + ["torch", "vllm", "sglang"] if VLLM_AVAILABLE else ["torch", "sglang"] + ), + line_names=( + ["Torch Reference", "VLLM", "SGL Kernel"] + if VLLM_AVAILABLE + else ["Torch Reference", "SGL Kernel"] + ), + styles=( + [("red", "-"), ("blue", "-"), ("green", "-")] + if VLLM_AVAILABLE + else [("red", "-"), ("green", "-")] + ), + ylabel="us", + plot_name="per-token-dynamic-quant-fp8-performance", + args={}, + ) +) +def benchmark_quantization(batch_size, seq_len, hidden_dim, provider): + dtype = torch.float16 + device = torch.device("cuda") + + x = torch.randn(batch_size * seq_len, hidden_dim, device=device, dtype=dtype) + + quantiles = [0.5, 0.2, 0.8] + + if provider == "torch": + fn = lambda: torch_per_token_quant_fp8(x.clone()) + elif provider == "vllm": + if not VLLM_AVAILABLE: + return (0, 0, 0) + fn = lambda: vllm_per_token_quant_fp8(x.clone()) + elif provider == "sglang": + fn = lambda: sglang_per_token_quant_fp8(x.clone()) + + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph(fn, quantiles=quantiles) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + +if __name__ == "__main__": + # Test various hidden dimensions for correctness - simplified for CI + if IS_CI: + test_dims = [2048] # Single dimension for CI + batch_size, seq_len = 4, 64 # Smaller values for CI + else: + test_dims = [1368, 2048, 4096] + batch_size, seq_len = 4, 4096 + + for dim in test_dims: + calculate_diff(batch_size=batch_size, seq_len=seq_len, hidden_dim=dim) + + print("\n" + "=" * 60) + print("Starting performance benchmark...") + benchmark_quantization.run(print_data=True) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_qserve_w4a8_gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_qserve_w4a8_gemm.py new file mode 100644 index 00000000..5827fa99 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_qserve_w4a8_gemm.py @@ -0,0 +1,216 @@ +import argparse +import copy +import itertools +import os + +import torch +import triton +from sgl_kernel import ( + int8_scaled_mm, + qserve_w4a8_per_chn_gemm, + qserve_w4a8_per_group_gemm, +) + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + + +def to_int8(tensor: torch.Tensor) -> torch.Tensor: + return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8) + + +WEIGHT_SHAPES = { + "meta-llama/Llama-3.1-8B-Instruct": [ + ([4096, 6144], 1), + ([4096, 4096], 0), + ([4096, 28672], 1), + ([14336, 4096], 0), + ], + "meta-llama/Llama-3.3-70B-Instruct": [ + ([8192, 10240], 1), + ([8192, 8192], 0), + ([8192, 57344], 1), + ([28672, 8192], 0), + ], + "mistralai/Mistral-Large-Instruct-2407": [ + ([12288, 14336], 1), + ([12288, 12288], 0), + ([12288, 57344], 1), + ([28672, 12288], 0), + ], + "Qwen/Qwen2.5-7B-Instruct": [ + ([3584, 4608], 1), + ([3584, 3584], 0), + ([3584, 37888], 1), + ([18944, 3584], 0), + ], + "Qwen/Qwen2.5-32B-Instruct": [ + ([5120, 7168], 1), + ([5120, 5120], 0), + ([5120, 55296], 1), + ([27648, 5120], 0), + ], + "Qwen/Qwen2.5-72B-Instruct": [ + ([8192, 10240], 1), + ([8192, 8192], 0), + ([8192, 59136], 1), + ([29568, 8192], 0), + ], + "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": [ + ([2048, 3072], 1), + ([2048, 4096], 1), + ([2048, 2048], 0), + ([2048, 576], 0), + ([2048, 21888], 1), + ([10944, 2048], 0), + ([2048, 2816], 1), + ([1408, 2048], 0), + ], +} + + +# CI environment uses simplified parameters +if IS_CI: + batch_sizes = [1, 16] # Simplified for CI +else: + batch_sizes = [1, 16, 32, 64, 128, 256, 512, 1024, 2048] + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=batch_sizes, + x_log=False, + line_arg="provider", + line_vals=["FP16", "W8A8", "Qserve_W4A8_Per_Channel", "Qserve_W4A8_Per_Group"], + line_names=["FP16", "W8A8", "Qserve_W4A8_Per_Channel", "Qserve_W4A8_Per_Group"], + styles=[("blue", "-"), ("orange", "-"), ("green", "-"), ("red", "-")], + ylabel="ms", + plot_name="FP16_vs_W8A8_vs_Qserve_W4A8_GEMM", + args={}, + ) +) +def benchmark(batch_size, provider, N, K): + M = batch_size + # For W8A8 + a = to_int8(torch.randn((M, K), device="cuda") * 5) + b = to_int8(torch.randn((N, K), device="cuda").t() * 5) + a_fp16 = a.to(torch.float16) + b_fp16 = b.to(torch.float16) + scale_a = torch.randn((M,), device="cuda", dtype=torch.float32) + scale_b = torch.randn((N,), device="cuda", dtype=torch.float32) + + # For Qserve W4A8 per channel + a_qserve_chn = a + # two int4s pack into one int8 + b_qserve_chn = to_int8(torch.randn((N, K // 2), device="cuda") * 5) + # b_qserve_chn = b.t().contiguous() + scale_a_qserve_chn = scale_a.to(torch.float16) + scale_b_qserve_chn = scale_b.to(torch.float16) + szero_b_qserve_chn = torch.randn((N,), device="cuda", dtype=torch.float16) + a_sum_qserve_chn = torch.randn((M,), device="cuda", dtype=torch.float16) + + # For Qserve W4A8 per group + group_size = 128 + assert K % group_size == 0, "K must be divisible by group_size" + a_qserve_group = a + # two int4s pack into one int8 + b_qserve_group = to_int8(torch.randn((N, K // 2), device="cuda") * 5) + # b_qserve_group = b.t().contiguous() + scale_a_qserve_group = scale_a.to(torch.float16) + scale_b_qserve_group = scale_b.to(torch.float16) + scale_i8_b_qserve_group = to_int8( + torch.randn((K // group_size, N), device="cuda", dtype=torch.float16) + ) + zero_i8_b_qserve_group = to_int8( + torch.randn((K // group_size, N), device="cuda", dtype=torch.float16) + ) + + quantiles = [0.5, 0.2, 0.8] + if provider == "FP16": + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: torch.matmul(a_fp16, b_fp16), + quantiles=quantiles, + ) + if provider == "W8A8": + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: int8_scaled_mm(a, b, scale_a, scale_b, torch.float16), + quantiles=quantiles, + ) + if provider == "Qserve_W4A8_Per_Channel": + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: qserve_w4a8_per_chn_gemm( + a_qserve_chn, + b_qserve_chn, + scale_b_qserve_chn, + scale_a_qserve_chn, + szero_b_qserve_chn, + a_sum_qserve_chn, + ), + quantiles=quantiles, + ) + if provider == "Qserve_W4A8_Per_Group": + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: qserve_w4a8_per_group_gemm( + a_qserve_group, + b_qserve_group, + zero_i8_b_qserve_group, + scale_i8_b_qserve_group, + scale_b_qserve_group, + scale_a_qserve_group, + ), + quantiles=quantiles, + ) + + return ms, max_ms, min_ms + + +def prepare_shapes(args): + KN_model_names = [] + models_tps = list(itertools.product(args.models, args.tp_sizes)) + for model, tp_size in models_tps: + assert model in WEIGHT_SHAPES + for KN, tp_split_dim in copy.deepcopy(WEIGHT_SHAPES[model]): + KN[tp_split_dim] = KN[tp_split_dim] // tp_size + KN.append(model) + KN_model_names.append(KN) + return KN_model_names + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--models", + nargs="+", + type=str, + default=["meta-llama/Llama-3.1-8B-Instruct"], + help="List of models to benchmark", + ) + parser.add_argument( + "--tp-sizes", + nargs="+", + type=int, + default=[1], + help="List of tensor parallel sizes", + ) + args = parser.parse_args() + + # Skip in CI environment + if IS_CI: + print("Skipping QServe W4A8 GEMM benchmark in CI environment") + print("QServe operations may have compatibility issues in CI") + else: + KN_model_names = prepare_shapes(args) + + for K, N, model_name in KN_model_names: + print(f"{model_name} N={N} K={K}: ") + benchmark.run( + print_data=True, + N=N, + K=K, + ) + + print("Benchmark finished!") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_rmsnorm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_rmsnorm.py new file mode 100644 index 00000000..d521ab05 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_rmsnorm.py @@ -0,0 +1,397 @@ +# Benchmarks SGLang RMSNorm kernels versus vLLM and FlashInfer across +# (batch_size, seq_len, hidden_size) and prints speed-up. +import argparse +import itertools +import os +import re +from typing import List, Optional, Tuple, Union + +import sgl_kernel +import torch +import torch.nn as nn +import triton +import triton.testing +from sgl_kernel.utils import is_arch_support_pdl + +# Optional imports +try: + from flashinfer.norm import fused_add_rmsnorm, rmsnorm + + FLASHINFER_AVAILABLE = True +except ImportError: + fused_add_rmsnorm = None + rmsnorm = None + FLASHINFER_AVAILABLE = False + +try: + from vllm import _custom_ops as vllm_ops + + VLLM_AVAILABLE = True +except ImportError: + vllm_ops = None + VLLM_AVAILABLE = False + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + + +def str2int_list(arg: str) -> List[int]: + if arg in ("", None): + return [] + if re.fullmatch(r"\d+(,\d+)*", arg.strip()) is None: + raise argparse.ArgumentTypeError(f"Bad int list: {arg}") + return [int(x) for x in arg.split(",")] + + +class HuggingFaceRMSNorm(nn.Module): + def __init__(self, hidden_size: int, eps: float = 1e-6) -> None: + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + orig_dtype = x.dtype + x = x.to(torch.float32) + if residual is not None: + x = x + residual.to(torch.float32) + residual = x.to(orig_dtype) + + variance = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + self.variance_epsilon) + x = x.to(orig_dtype) * self.weight + if residual is None: + return x + else: + return x, residual + + +def rmsnorm_naive( + x: torch.Tensor, + weight: torch.Tensor, + residual: Optional[torch.Tensor] = None, + eps: float = 1e-6, +): + naive_norm = HuggingFaceRMSNorm(x.shape[-1], eps=eps) + naive_norm.weight = nn.Parameter(weight) + naive_norm = naive_norm.to(x.device) + + orig_shape = x.shape + x = x.view(-1, x.shape[-1]) + if residual is not None: + residual = residual.view(-1, residual.shape[-1]) + + output = naive_norm(x, residual) + + if isinstance(output, tuple): + output = (output[0].view(orig_shape), output[1].view(orig_shape)) + else: + output = output.view(orig_shape) + return output + + +def rmsnorm_flashinfer( + x: torch.Tensor, + weight: torch.Tensor, + residual: Optional[torch.Tensor] = None, + eps: float = 1e-6, +): + if not FLASHINFER_AVAILABLE: + # Fallback to naive implementation if FlashInfer is not available + return rmsnorm_naive(x, weight, residual, eps) + + orig_shape = x.shape + x = x.view(-1, x.shape[-1]) + if residual is not None: + residual = residual.view(-1, residual.shape[-1]) + + if residual is not None: + fused_add_rmsnorm(x, residual, weight, eps) + output = (x, residual) + else: + output = rmsnorm(x, weight, eps) + + if isinstance(output, tuple): + output = (output[0].view(orig_shape), output[1].view(orig_shape)) + else: + output = output.view(orig_shape) + return output + + +def rmsnorm_vllm( + x: torch.Tensor, + weight: torch.Tensor, + residual: Optional[torch.Tensor] = None, + eps: float = 1e-6, +): + if not VLLM_AVAILABLE: + # Fallback to naive implementation if vLLM is not available + return rmsnorm_naive(x, weight, residual, eps) + + orig_shape = x.shape + x = x.view(-1, x.shape[-1]) + if residual is not None: + residual = residual.view(-1, residual.shape[-1]) + + if residual is not None: + vllm_ops.fused_add_rms_norm(x, residual, weight, eps) + output = (x, residual) + else: + out = torch.empty_like(x) + vllm_ops.rms_norm(out, x, weight, eps) + output = out + + if isinstance(output, tuple): + output = (output[0].view(orig_shape), output[1].view(orig_shape)) + else: + output = output.view(orig_shape) + return output + + +def rmsnorm_sglang( + x: torch.Tensor, + weight: torch.Tensor, + residual: Optional[torch.Tensor] = None, + eps: float = 1e-6, + enable_pdl: Optional[bool] = None, +): + orig_shape = x.shape + x = x.view(-1, x.shape[-1]) + if residual is not None: + residual = residual.view(-1, residual.shape[-1]) + + if enable_pdl is None: + enable_pdl = is_arch_support_pdl() + + if residual is not None: + sgl_kernel.fused_add_rmsnorm(x, residual, weight, eps, enable_pdl=enable_pdl) + output = (x, residual) + else: + out = torch.empty_like(x) + sgl_kernel.rmsnorm(x, weight, eps, out=out, enable_pdl=enable_pdl) + output = out + + if isinstance(output, tuple): + output = (output[0].view(orig_shape), output[1].view(orig_shape)) + else: + output = output.view(orig_shape) + return output + + +def calculate_diff(batch_size, seq_len, hidden_size, use_residual=True): + dtype = torch.bfloat16 + x = torch.randn(batch_size, seq_len, hidden_size, dtype=dtype, device="cuda") + weight = torch.ones(hidden_size, dtype=dtype, device="cuda") + residual = torch.randn_like(x) if use_residual else None + + output_naive = rmsnorm_naive( + x.clone(), weight, residual.clone() if residual is not None else None + ) + output_flashinfer = rmsnorm_flashinfer( + x.clone(), weight, residual.clone() if residual is not None else None + ) + output_vllm = rmsnorm_vllm( + x.clone(), weight, residual.clone() if residual is not None else None + ) + output_sglang = rmsnorm_sglang( + x.clone(), weight, residual.clone() if residual is not None else None + ) + + if use_residual: + output_naive = output_naive[0] + output_flashinfer = output_flashinfer[0] + output_vllm = output_vllm[0] + output_sglang = output_sglang[0] + + print(f"Naive output={output_naive}") + if FLASHINFER_AVAILABLE: + print(f"FlashInfer output={output_flashinfer}") + else: + print("FlashInfer not available, skipped") + if VLLM_AVAILABLE: + print(f"VLLM output={output_vllm}") + else: + print("vLLM not available, skipped") + print(f"SGLang output={output_sglang}") + + # Only compare available implementations + all_match = torch.allclose(output_naive, output_sglang, atol=1e-2, rtol=1e-2) + if FLASHINFER_AVAILABLE: + all_match = all_match and torch.allclose( + output_naive, output_flashinfer, atol=1e-2, rtol=1e-2 + ) + if VLLM_AVAILABLE: + all_match = all_match and torch.allclose( + output_naive, output_vllm, atol=1e-2, rtol=1e-2 + ) + + if all_match: + print("✅ All available implementations match") + else: + print("❌ Implementations differ") + + +# CI environment uses simplified parameters +if IS_CI: + default_batch_sizes = [1] # Single batch size for CI + default_seq_lens = [64] # Single sequence length for CI + default_hidden_sizes = [4096] # Single hidden size for CI +else: + default_batch_sizes = [2**i for i in range(0, 7, 2)] # 1, 4, 16, 64 + default_seq_lens = [2**i for i in range(6, 11, 1)] # 64, 128, 256, 512, 1024 + default_hidden_sizes = [32 * 128, 48 * 128] # 4096, 6144 + + +def make_configs(bsizes: List[int], slens: List[int], hsizes: List[int]) -> List[Tuple]: + return list(itertools.product(bsizes, slens, hsizes)) + + +# Filter providers based on availability +available_providers = ["huggingface", "sglang"] +available_names = ["HuggingFace", "SGL Kernel"] +available_styles = [("blue", "-"), ("orange", "-")] + +if FLASHINFER_AVAILABLE: + available_providers.insert(-1, "flashinfer") + available_names.insert(-1, "FlashInfer") + available_styles.insert(-1, ("green", "-")) + +if VLLM_AVAILABLE: + available_providers.insert(-1, "vllm") + available_names.insert(-1, "vLLM") + available_styles.insert(-1, ("red", "-")) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size", "seq_len", "hidden_size"], + x_vals=[], + line_arg="provider", + line_vals=available_providers, + line_names=available_names, + styles=available_styles, + ylabel="µs (median) or × (speed-up)", + plot_name="rmsnorm-performance", + args={}, + ) +) +def benchmark(batch_size, seq_len, hidden_size, provider, use_residual): + device = torch.device("cuda") + dtype = torch.bfloat16 + + x = torch.randn(batch_size, seq_len, hidden_size, dtype=dtype, device=device) + weight = torch.ones(hidden_size, dtype=dtype, device=device) + residual = torch.randn_like(x) if use_residual else None + + # timing helper + def timed(fn): + for _ in range(5): + fn() + torch.cuda.synchronize() + ms, qmin, qmax = triton.testing.do_bench_cudagraph( + fn, quantiles=[0.5, 0.2, 0.8] + ) + return 1000 * ms, 1000 * qmax, 1000 * qmin + + if provider == "huggingface": + return timed( + lambda: rmsnorm_naive( + x.clone(), + weight, + residual.clone() if residual is not None else None, + ) + ) + elif provider == "flashinfer": + if not FLASHINFER_AVAILABLE: + return (0, 0, 0) + return timed( + lambda: rmsnorm_flashinfer( + x.clone(), + weight, + residual.clone() if residual is not None else None, + ) + ) + elif provider == "vllm": + if not VLLM_AVAILABLE: + return (0, 0, 0) + return timed( + lambda: rmsnorm_vllm( + x.clone(), + weight, + residual.clone() if residual is not None else None, + ) + ) + elif provider == "sglang": + return timed( + lambda: rmsnorm_sglang( + x.clone(), + weight, + residual.clone() if residual is not None else None, + ) + ) + + # provider == "speedup" + if VLLM_AVAILABLE: + t_ref, _, _ = timed( + lambda: rmsnorm_vllm( + x.clone(), + weight, + residual.clone() if residual is not None else None, + ) + ) + else: + t_ref, _, _ = timed( + lambda: rmsnorm_naive( + x.clone(), + weight, + residual.clone() if residual is not None else None, + ) + ) + t_sgl, _, _ = timed( + lambda: rmsnorm_sglang( + x.clone(), + weight, + residual.clone() if residual is not None else None, + ) + ) + spd = t_ref / t_sgl if t_ref > 0 else 1.0 + return (spd, spd, spd) + + +if __name__ == "__main__": + p = argparse.ArgumentParser("RMSNorm kernel benchmark") + p.add_argument("--batch_sizes", type=str2int_list, default=default_batch_sizes) + p.add_argument("--seq_lens", type=str2int_list, default=default_seq_lens) + p.add_argument("--hidden_sizes", type=str2int_list, default=default_hidden_sizes) + p.add_argument( + "--use_residual", action="store_true", help="Whether to use residual connection" + ) + p.add_argument("--verify_only", action="store_true") + args = p.parse_args() + + # coerce lists + if isinstance(args.batch_sizes, str): + args.batch_sizes = str2int_list(args.batch_sizes) + if isinstance(args.seq_lens, str): + args.seq_lens = str2int_list(args.seq_lens) + if isinstance(args.hidden_sizes, str): + args.hidden_sizes = str2int_list(args.hidden_sizes) + + # patch perf_report grid + benchmark_grid = make_configs(args.batch_sizes, args.seq_lens, args.hidden_sizes) + if hasattr(benchmark, "benchmarks"): + benchmark.benchmarks.x_vals = benchmark_grid + else: + benchmark.benchmark.x_vals = benchmark_grid + + if args.verify_only: + ok = calculate_diff(4, 128, args.hidden_sizes[0], args.use_residual) + print("✅ sanity pass" if ok else "❌ mismatch") + else: + benchmark.run(print_data=True, use_residual=args.use_residual) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_rotary_embedding.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_rotary_embedding.py new file mode 100644 index 00000000..0cab8e65 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_rotary_embedding.py @@ -0,0 +1,111 @@ +import itertools +import os + +import torch +import triton +from sgl_kernel import FusedSetKVBufferArg +from sgl_kernel.testing.rotary_embedding import ( + FlashInferRotaryEmbedding, + MHATokenToKVPool, + RotaryEmbedding, + create_inputs, +) + +from sglang.srt.utils.bench_utils import bench_kineto + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + +# CI environment uses simplified parameters +if IS_CI: + batch_seq_configs = [(1, 1)] # Single config for CI + save_kv_configs = [False] # Single option for CI +else: + batch_seq_configs = [ + (1, 1), + (32, 1), + (128, 1), + (512, 1), + (2, 512), + (4, 4096), + ] + save_kv_configs = [False, True] + +configs = [ + (batch_size, seq_len, save_kv_cache) + for batch_size, seq_len in batch_seq_configs + for save_kv_cache in save_kv_configs +] + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size", "seq_len", "save_kv_cache"], + x_vals=configs, + line_arg="provider", + line_vals=["sglang"], + line_names=["SGL Kernel"], + styles=[("green", "-")], + ylabel="us", + plot_name="bench_rotary_embedding", + args={}, + ) +) +def benchmark(batch_size, seq_len, save_kv_cache, provider): + device = torch.device("cuda") + + num_q_heads = 32 + num_kv_heads = 8 + head_size = 64 + dtype = torch.bfloat16 + + config = dict( + head_size=head_size, + rotary_dim=64, + max_position_embeddings=4096, + base=8000, + is_neox_style=True, + dtype=dtype, + ) + rope_flashinfer = FlashInferRotaryEmbedding(**config).to(device) + pool_flashinfer = MHATokenToKVPool(head_num=num_kv_heads, head_dim=head_size) + + inputs = create_inputs( + head_size=head_size, + batch_size=batch_size, + seq_len=seq_len, + device=device, + dtype=dtype, + num_q_heads=num_q_heads, + num_kv_heads=num_kv_heads, + ) + + query_flashinfer, key_flashinfer = inputs["query"].clone(), inputs["key"].clone() + + bench_fn = lambda: rope_flashinfer.forward_cuda( + inputs["pos_ids"], + query_flashinfer, + key_flashinfer, + fused_set_kv_buffer_arg=( + FusedSetKVBufferArg( + value=inputs["value"], + k_buffer=pool_flashinfer.k_buffer[0].view(-1, num_kv_heads * head_size), + v_buffer=pool_flashinfer.v_buffer[0].view(-1, num_kv_heads * head_size), + k_scale=None, + v_scale=None, + cache_loc=inputs["out_cache_loc"], + ) + if save_kv_cache + else None + ), + ) + + time_s = bench_kineto(bench_fn, kernel_names="BatchQKApplyRotaryPosIds") + return time_s * 1e6 + + +if __name__ == "__main__": + benchmark.run(print_data=True) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_sum_scale.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_sum_scale.py new file mode 100644 index 00000000..ad9621ee --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_sum_scale.py @@ -0,0 +1,251 @@ +import os + +import torch +import triton +import triton.language as tl +from sgl_kernel import moe_sum_reduce as moe_sum_reduce_cuda +from triton.testing import do_bench + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + + +@triton.jit +def _moe_sum_reduce_kernel( + input_ptr, + input_stride_0, + input_stride_1, + input_stride_2, + output_ptr, + output_stride_0, + output_stride_1, + token_num: int, + topk_num: int, + hidden_dim: int, + routed_scaling_factor: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_DIM: tl.constexpr, + NUM_STAGE: tl.constexpr, +): + input_stride_0 = tl.cast(input_stride_0, dtype=tl.int64) + input_stride_1 = tl.cast(input_stride_1, dtype=tl.int64) + output_stride_0 = tl.cast(output_stride_0, dtype=tl.int64) + + token_block_id = tl.program_id(0) + dim_block_id = tl.program_id(1) + + offs_token = token_block_id * BLOCK_M + tl.arange(0, BLOCK_M) + offs_dim = dim_block_id * BLOCK_DIM + tl.arange(0, BLOCK_DIM) + + mask_token = offs_token < token_num + mask_dim = offs_dim < hidden_dim + + base_ptrs = input_ptr + offs_token[:, None] * input_stride_0 + offs_dim[None, :] + + accumulator = tl.zeros((BLOCK_M, BLOCK_DIM), dtype=tl.float32) + for i in tl.range(0, topk_num, num_stages=NUM_STAGE): + tile = tl.load( + base_ptrs + i * input_stride_1, + mask=mask_token[:, None] & mask_dim[None, :], + other=0.0, + ) + accumulator += tile.to(tl.float32) + accumulator *= routed_scaling_factor + + # -------- Write back -------- + store_ptrs = output_ptr + offs_token[:, None] * output_stride_0 + offs_dim[None, :] + tl.store( + store_ptrs, + accumulator.to(input_ptr.dtype.element_ty), + mask=mask_token[:, None] & mask_dim[None, :], + ) + + +# _moe_sum_reduce_kernel kernel modified from https://github.com/ModelTC/lightllm/blob/main/lightllm/common/fused_moe/moe_sum_reduce.py +def moe_sum_reduce_triton( + input: torch.Tensor, output: torch.Tensor, routed_scaling_factor: float +): + assert input.is_contiguous() + assert output.is_contiguous() + + token_num, topk_num, hidden_dim = input.shape + assert output.shape[0] == token_num and output.shape[1] == hidden_dim + + BLOCK_M = 1 + BLOCK_DIM = 2048 + NUM_STAGE = 1 + num_warps = 16 + + grid = ( + triton.cdiv(token_num, BLOCK_M), + triton.cdiv(hidden_dim, BLOCK_DIM), + ) + + _moe_sum_reduce_kernel[grid]( + input, + *input.stride(), + output, + *output.stride(), + token_num=token_num, + topk_num=topk_num, + hidden_dim=hidden_dim, + routed_scaling_factor=routed_scaling_factor, + BLOCK_M=BLOCK_M, + BLOCK_DIM=BLOCK_DIM, + NUM_STAGE=NUM_STAGE, + num_warps=num_warps, + ) + return + + +def compute_sum_scaled_baseline( + x: torch.Tensor, out: torch.Tensor, routed_scaling_factor: float +) -> torch.Tensor: + torch.sum(x, dim=1, out=out) + out.mul_(routed_scaling_factor) + return out + + +@torch.compile +def compute_sum_scaled_compiled( + x: torch.Tensor, out: torch.Tensor, routed_scaling_factor: float +) -> torch.Tensor: + torch.sum(x * routed_scaling_factor, dim=1, out=out) + return out + + +def get_benchmark(dtype=torch.bfloat16): + num_tokens_range = [2**i for i in range(0, 13)] + + @triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["num_tokens"], + x_vals=num_tokens_range, + line_arg="version", + line_vals=["baseline", "compiled", "triton", "cuda"], + line_names=["Original", "TorchCompile", "TritonKernel", "CudaKernel"], + styles=[("blue", "-"), ("green", "-"), ("red", "-"), ("yellow", "-")], + ylabel="us", + plot_name=f"sum_scaled_performance_{str(dtype).split('.')[-1]}", + args={}, + ) + ) + def benchmark(num_tokens, version): + topk = 9 + hidden_size = 4096 + dtype = torch.bfloat16 + scaling_factor = 0.3 + + x = torch.randn(num_tokens, topk, hidden_size, dtype=dtype, device="cuda") + out = torch.empty(num_tokens, hidden_size, dtype=dtype, device="cuda") + + # Warmup + for _ in range(3): + if version == "baseline": + compute_sum_scaled_baseline(x, out, scaling_factor) + elif version == "compiled": + compute_sum_scaled_compiled(x, out, scaling_factor) + elif version == "triton": + moe_sum_reduce_triton(x, out, scaling_factor) + else: + moe_sum_reduce_cuda(x, out, scaling_factor) + + # Benchmark + quantiles = [0.5, 0.2, 0.8] + if version == "baseline": + ms, min_ms, max_ms = do_bench( + lambda: compute_sum_scaled_baseline(x, out, scaling_factor), + quantiles=quantiles, + ) + elif version == "compiled": + ms, min_ms, max_ms = do_bench( + lambda: compute_sum_scaled_compiled(x, out, scaling_factor), + quantiles=quantiles, + ) + elif version == "triton": + ms, min_ms, max_ms = do_bench( + lambda: moe_sum_reduce_triton(x, out, scaling_factor), + quantiles=quantiles, + ) + else: + ms, min_ms, max_ms = do_bench( + lambda: moe_sum_reduce_cuda(x, out, scaling_factor), + quantiles=quantiles, + ) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + return benchmark + + +def verify_correctness(num_tokens=1024, dtype=torch.bfloat16): + x = torch.randn(num_tokens, 9, 4096, device="cuda", dtype=dtype) + scaling_factor = 0.3 + + out_baseline = torch.empty_like(x[:, 0]) + compute_sum_scaled_baseline(x, out_baseline, scaling_factor) + + out_compiled = torch.empty_like(out_baseline) + compute_sum_scaled_compiled(x, out_compiled, scaling_factor) + + out_cuda = torch.empty_like(out_baseline) + moe_sum_reduce_cuda(x, out_cuda, scaling_factor) + + triton_skipped = dtype == torch.float64 + if not triton_skipped: + out_triton = torch.empty_like(out_baseline) + moe_sum_reduce_triton(x, out_triton, scaling_factor) + + if dtype == torch.float64: + atol, rtol = 1e-12, 1e-12 + elif dtype == torch.float32: + atol, rtol = 1e-6, 1e-6 + else: # bfloat16 / float16 + atol, rtol = 1e-2, 1e-2 + + ok_compiled = torch.allclose(out_baseline, out_compiled, atol=atol, rtol=rtol) + ok_cuda = torch.allclose(out_baseline, out_cuda, atol=atol, rtol=rtol) + ok_triton = ( + True + if triton_skipped + else torch.allclose(out_baseline, out_triton, atol=atol, rtol=rtol) + ) + + if ok_compiled and ok_triton and ok_cuda: + msg = "✅ All implementations match" + if triton_skipped: + msg += " (Triton skipped for float64)" + print(msg) + else: + print("❌ Implementations differ") + print( + f"Baseline vs Compiled: {(out_baseline - out_compiled).abs().max().item()}" + ) + if not triton_skipped: + print( + f"Baseline vs Triton: {(out_baseline - out_triton).abs().max().item()}" + ) + print(f"Baseline vs Cuda: {(out_baseline - out_cuda).abs().max().item()}") + + +if __name__ == "__main__": + print("Running correctness verification for bfloat16...") + verify_correctness(dtype=torch.bfloat16) + + # CI environment uses simplified parameters + if not IS_CI: + print("Running correctness verification for float64...") + verify_correctness(dtype=torch.float64) + + print("Running correctness verification for float64...") + verify_correctness(dtype=torch.float64) + + print("\nRunning performance benchmark for bfloat16...") + benchmark = get_benchmark(dtype=torch.bfloat16) + benchmark.run( + print_data=True, + # save_path="./configs/benchmark_ops/sum_scaled/" + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_top_k_top_p_sampling.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_top_k_top_p_sampling.py new file mode 100644 index 00000000..278356c3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/benchmark/bench_top_k_top_p_sampling.py @@ -0,0 +1,147 @@ +import itertools +import os + +import sgl_kernel +import torch +import triton +import triton.testing + +# CI environment detection +IS_CI = ( + os.getenv("CI", "false").lower() == "true" + or os.getenv("GITHUB_ACTIONS", "false").lower() == "true" +) + + +def torch_top_k_top_p_joint_sampling_from_probs( + normalized_prob, top_k, top_p, eps=1e-4 +): + """Reference PyTorch implementation of joint top-k top-p sampling.""" + batch_size, vocab_size = normalized_prob.shape + samples = torch.empty(batch_size, dtype=torch.int64, device=normalized_prob.device) + + for i in range(batch_size): + p_val = top_p[i].item() + k_val = top_k[i].item() + + # top-p mask + sorted_prob, indices = torch.sort(normalized_prob[i], descending=False) + cdf = torch.cumsum(sorted_prob, dim=-1) + mask_top_p = torch.zeros( + vocab_size, dtype=torch.int32, device=normalized_prob.device + ) + mask_top_p.scatter_add_(0, indices, (cdf > (1 - p_val) - eps).int()) + + # top-k mask + sorted_prob_desc, _ = torch.sort(normalized_prob[i], descending=True) + pivot = sorted_prob_desc[k_val - 1] + mask_top_k = (normalized_prob[i] >= pivot).int() + + # joint mask + mask = torch.minimum(mask_top_p, mask_top_k).bool() + + # sample from masked probs + masked_probs = normalized_prob[i] * mask + masked_probs = masked_probs / masked_probs.sum() + idx = torch.multinomial(masked_probs, 1) + samples[i] = idx + + return samples + + +def calculate_diff(batch_size, vocab_size, p): + """Compare Torch reference and SGLang kernel for correctness.""" + torch.manual_seed(42) + if p == 0.1: + k = int(vocab_size * 0.5) + elif p == 0.5: + k = int(vocab_size * 0.1) + else: + raise ValueError("p not recognized") + + device = torch.device("cuda") + pre_norm_prob = torch.rand(batch_size, vocab_size, device=device) + normalized_prob = pre_norm_prob / pre_norm_prob.sum(dim=-1, keepdim=True) + + top_p_tensor = torch.full((batch_size,), p, device=device) + top_k_tensor = torch.full((batch_size,), k, device=device) + + torch_samples = torch_top_k_top_p_joint_sampling_from_probs( + normalized_prob, top_k_tensor, top_p_tensor + ) + sglang_samples = sgl_kernel.top_k_top_p_sampling_from_probs( + normalized_prob, top_k_tensor, top_p_tensor, filter_apply_order="joint" + ) + + +# parameter space - simplified for CI +if IS_CI: + batch_size_range = [16] # Single batch size for CI + vocab_size_range = [111] # Single vocab size for CI + p_range = [0.1] # Single p value for CI +else: + batch_size_range = [16, 64, 128] + vocab_size_range = [111, 32000] + p_range = [0.1, 0.5] + +configs = list(itertools.product(batch_size_range, vocab_size_range, p_range)) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size", "vocab_size", "p"], + x_vals=configs, + line_arg="provider", + line_vals=["torch", "sglang"], + line_names=["Torch Reference", "SGL Kernel"], + styles=[("red", "-"), ("green", "-")], + ylabel="us", + plot_name="top-k-top-p-joint-sampling-performance", + args={}, + ) +) +def benchmark_sampling(batch_size, vocab_size, p, provider): + torch.manual_seed(42) + if p == 0.1: + k = int(vocab_size * 0.5) + elif p == 0.5: + k = int(vocab_size * 0.1) + else: + raise ValueError("p not recognized") + + device = torch.device("cuda") + pre_norm_prob = torch.rand(batch_size, vocab_size, device=device) + normalized_prob = pre_norm_prob / pre_norm_prob.sum(dim=-1, keepdim=True) + top_p_tensor = torch.full((batch_size,), p, device=device) + top_k_tensor = torch.full((batch_size,), k, device=device) + + if provider == "torch": + fn = lambda: torch_top_k_top_p_joint_sampling_from_probs( + normalized_prob.clone(), top_k_tensor, top_p_tensor + ) + elif provider == "sglang": + fn = lambda: sgl_kernel.top_k_top_p_sampling_from_probs( + normalized_prob.clone(), + top_k_tensor, + top_p_tensor, + filter_apply_order="joint", + ) + + ms, min_ms, max_ms = triton.testing.do_bench(fn, quantiles=[0.5, 0.2, 0.8]) + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + +if __name__ == "__main__": + # Correctness check - simplified for CI + if IS_CI: + # Only test one configuration in CI + test_configs = [configs[0]] if configs else [(16, 111, 0.1)] + else: + test_configs = configs + + for cfg in test_configs: + calculate_diff(*cfg) + + print("\n" + "=" * 60) + print("Starting performance benchmark...") + benchmark_sampling.run(print_data=True) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/build.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/build.sh new file mode 100755 index 00000000..2807bada --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/build.sh @@ -0,0 +1,240 @@ +#!/bin/bash +set -ex + +PYTHON_VERSION=$1 +CUDA_VERSION=$2 +PYTHON_ROOT_PATH=/opt/python/cp${PYTHON_VERSION//.}-cp${PYTHON_VERSION//.} + +if [ -z "$3" ]; then + ARCH=$(uname -i) +else + ARCH=$3 +fi + +echo "ARCH: $ARCH" +if [ ${ARCH} = "aarch64" ]; then + LIBCUDA_ARCH="sbsa" + BUILDER_NAME="pytorch/manylinuxaarch64-builder" +else + LIBCUDA_ARCH=${ARCH} + BUILDER_NAME="pytorch/manylinux2_28-builder" +fi + +if [ ${CUDA_VERSION} = "13.0" ]; then + DOCKER_IMAGE="${BUILDER_NAME}:cuda${CUDA_VERSION}" + TORCH_INSTALL="pip install --no-cache-dir torch==2.9.1 --index-url https://download.pytorch.org/whl/cu130" +elif [ ${CUDA_VERSION} = "12.9" || ${CUDA_VERSION} = "12.8" ]; then + DOCKER_IMAGE="${BUILDER_NAME}:cuda${CUDA_VERSION}" + TORCH_INSTALL="pip install --no-cache-dir torch==2.9.1 --index-url https://download.pytorch.org/whl/cu128" +else + DOCKER_IMAGE="${BUILDER_NAME}:cuda${CUDA_VERSION}" + TORCH_INSTALL="pip install --no-cache-dir torch==2.9.1 --index-url https://download.pytorch.org/whl/cu126" +fi + +# Create cache directories for persistent build artifacts in home directory +# Using home directory to persist across workspace cleanups/checkouts +CACHE_DIR="${HOME}/.cache/sgl-kernel" +CMAKE_DOWNLOAD_CACHE="${CACHE_DIR}/cmake-downloads" +CCACHE_DIR="${CACHE_DIR}/ccache" + +mkdir -p "${CMAKE_DOWNLOAD_CACHE}" +mkdir -p "${CCACHE_DIR}" + +echo "===================================" +echo "Cache Configuration" +echo "===================================" +echo "CMake download cache: ${CMAKE_DOWNLOAD_CACHE}" +echo "ccache directory: ${CCACHE_DIR}" +echo "ccache enabled: ${USE_CCACHE:-1}" +echo "" + +docker run --rm \ + -v $(pwd):/sgl-kernel \ + -v ${CMAKE_DOWNLOAD_CACHE}:/cmake-downloads \ + -v ${CCACHE_DIR}:/ccache \ + -e ENABLE_CMAKE_PROFILE="${ENABLE_CMAKE_PROFILE:-}" \ + -e ENABLE_BUILD_PROFILE="${ENABLE_BUILD_PROFILE:-}" \ + -e USE_CCACHE="${USE_CCACHE:-1}" \ + ${DOCKER_IMAGE} \ + bash -c " + set -e + # Install CMake (version >= 3.26) - Robust Installation with caching + echo \"==================================\" + echo \"Installing CMake\" + echo \"==================================\" + export CMAKE_VERSION_MAJOR=3.31 + export CMAKE_VERSION_MINOR=1 + # Setting these flags to reduce OOM chance only on ARM + export CMAKE_BUILD_PARALLEL_LEVEL=$(( $(nproc)/3 < 48 ? $(nproc)/3 : 48 )) + if [ \"${ARCH}\" = \"aarch64\" ]; then + export CUDA_NVCC_FLAGS=\"-Xcudafe --threads=2\" + export MAKEFLAGS='-j2' + export CMAKE_BUILD_PARALLEL_LEVEL=2 + export NINJAFLAGS='-j2' + echo \"ARM detected: Using extra conservative settings (2 parallel jobs)\" + fi + + CMAKE_TARBALL=\"cmake-\${CMAKE_VERSION_MAJOR}.\${CMAKE_VERSION_MINOR}-linux-${ARCH}.tar.gz\" + + # Check if CMake is already cached + if [ -f \"/cmake-downloads/\${CMAKE_TARBALL}\" ]; then + echo \"Using cached CMake from /cmake-downloads/\${CMAKE_TARBALL}\" + cp /cmake-downloads/\${CMAKE_TARBALL} . + else + echo \"Downloading CMake from: https://cmake.org/files/v\${CMAKE_VERSION_MAJOR}/\${CMAKE_TARBALL}\" + wget https://cmake.org/files/v\${CMAKE_VERSION_MAJOR}/\${CMAKE_TARBALL} + # Cache the downloaded file + cp \${CMAKE_TARBALL} /cmake-downloads/ + fi + + tar -xzf \${CMAKE_TARBALL} + mv cmake-\${CMAKE_VERSION_MAJOR}.\${CMAKE_VERSION_MINOR}-linux-${ARCH} /opt/cmake + export PATH=/opt/cmake/bin:\$PATH + export LD_LIBRARY_PATH=/lib64:\$LD_LIBRARY_PATH + + # Debugging CMake + echo \"PATH: \$PATH\" + which cmake + cmake --version + + if [ \"${USE_CCACHE}\" = \"1\" ]; then + echo \"==================================\" + echo \"Installing and configuring ccache\" + echo \"==================================\" + + # Install ccache 4.12.1 from source for CUDA support (yum provides old 3.7.7) + echo \"Installing ccache 4.12.1 from source...\" + + # Install build dependencies + yum install -y gcc gcc-c++ make wget tar + + # Download and build ccache 4.12.1 + cd /tmp + wget -q https://github.com/ccache/ccache/releases/download/v4.12.1/ccache-4.12.1.tar.xz + tar -xf ccache-4.12.1.tar.xz + cd ccache-4.12.1 + + # Build and install (uses already-installed CMake 3.31) + mkdir build && cd build + /opt/cmake/bin/cmake -D CMAKE_BUILD_TYPE=Release -D CMAKE_INSTALL_PREFIX=/usr .. >/dev/null + make -j\$(nproc) >/dev/null + make install >/dev/null + + # Verify installation + ccache --version + echo \"ccache 4.12.1 installed successfully\" + cd /sgl-kernel + + # Configure ccache + export CCACHE_DIR=/ccache + export CCACHE_BASEDIR=/sgl-kernel + export CCACHE_MAXSIZE=10G + export CCACHE_COMPILERCHECK=content + export CCACHE_COMPRESS=true + export CCACHE_SLOPPINESS=file_macro,time_macros,include_file_mtime,include_file_ctime + + # Set up ccache as compiler launcher (don't use PATH to avoid -ccbin conflicts) + export CMAKE_C_COMPILER_LAUNCHER=ccache + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export CMAKE_CUDA_COMPILER_LAUNCHER=ccache + + # Show ccache stats before build + ccache -sV || true + echo \"\" + else + echo \"==================================\" + echo \"ccache disabled (USE_CCACHE=0)\" + echo \"==================================\" + echo \"\" + fi + + yum install numactl-devel -y --nogpgcheck && \ + yum install libibverbs -y --nogpgcheck && \ + ln -sv /usr/lib64/libibverbs.so.1 /usr/lib64/libibverbs.so && \ + ${PYTHON_ROOT_PATH}/bin/${TORCH_INSTALL} && \ + ${PYTHON_ROOT_PATH}/bin/pip install --no-cache-dir ninja setuptools==75.0.0 wheel==0.41.0 numpy uv scikit-build-core && \ + export FLASHINFER_CUDA_ARCH_LIST='8.0 8.9 9.0a 10.0a 12.0a' && \ + export CUDA_VERSION=${CUDA_VERSION} && \ + mkdir -p /usr/lib/${ARCH}-linux-gnu/ && \ + ln -s /usr/local/cuda-${CUDA_VERSION}/targets/${LIBCUDA_ARCH}-linux/lib/stubs/libcuda.so /usr/lib/${ARCH}-linux-gnu/libcuda.so && \ + export CPLUS_INCLUDE_PATH=/usr/local/cuda/include/cccl${CPLUS_INCLUDE_PATH:+:${CPLUS_INCLUDE_PATH}} && \ + export C_INCLUDE_PATH=/usr/local/cuda/include/cccl${C_INCLUDE_PATH:+:${C_INCLUDE_PATH}} && \ + + cd /sgl-kernel && \ + ls -la ${PYTHON_ROOT_PATH}/lib/python${PYTHON_VERSION}/site-packages/wheel/ && \ + + # Enable CMake profiling if requested + if [ -n \"${ENABLE_CMAKE_PROFILE}\" ]; then + echo \"CMake profiling enabled - will save to /sgl-kernel/cmake-profile.json\" + export CMAKE_ARGS=\"--profiling-output=/sgl-kernel/cmake-profile.json --profiling-format=google-trace\" + fi + + export NINJA_STATUS=\"[%f/%t %es] \" + # Enable Ninja build profiling if requested + if [ -n \"${ENABLE_BUILD_PROFILE}\" ]; then + echo \"Ninja build profiling enabled - will save to /sgl-kernel/build-trace.json\" + fi + + PYTHONPATH=${PYTHON_ROOT_PATH}/lib/python${PYTHON_VERSION}/site-packages ${PYTHON_ROOT_PATH}/bin/python -m uv build --wheel -Cbuild-dir=build . --color=always --no-build-isolation && \ + ./rename_wheels.sh + + # Show profile location if profiling was enabled + if [ -n \"${ENABLE_CMAKE_PROFILE}\" ] && [ -f /sgl-kernel/cmake-profile.json ]; then + echo \"\" + echo \"==================================\" + echo \"CMake Profile Generated\" + echo \"==================================\" + echo \"Profile saved to: cmake-profile.json\" + echo \"View in browser: chrome://tracing or edge://tracing\" + echo \"\" + fi + + # Generate Ninja build trace if profiling enabled + if [ -n \"${ENABLE_BUILD_PROFILE}\" ] && [ -f /sgl-kernel/build/.ninja_log ]; then + echo \"\" + echo \"==================================\" + echo \"Generating Ninja Build Trace\" + echo \"==================================\" + + # Download ninjatracing script from GitHub (using PR #39 branch for ninja log v7 support) + wget -q https://raw.githubusercontent.com/cradleapps/ninjatracing/084212eaf68f25c70579958a2ed67fb4ec2a9ca4/ninjatracing -O /tmp/ninjatracing || echo \"Note: Failed to download ninjatracing, skipping build trace\" + + # Convert .ninja_log to Chrome trace (JSON format) + if [ -f /tmp/ninjatracing ]; then + ${PYTHON_ROOT_PATH}/bin/python /tmp/ninjatracing /sgl-kernel/build/.ninja_log > /sgl-kernel/build-trace.json || true + + if [ -f /sgl-kernel/build-trace.json ]; then + # Compress the trace for smaller file size and faster loading + gzip -9 -k /sgl-kernel/build-trace.json 2>/dev/null || true + + echo \"Build trace saved to: build-trace.json\" + if [ -f /sgl-kernel/build-trace.json.gz ]; then + ORIGINAL_SIZE=\$(stat -f%z /sgl-kernel/build-trace.json 2>/dev/null || stat -c%s /sgl-kernel/build-trace.json) + COMPRESSED_SIZE=\$(stat -f%z /sgl-kernel/build-trace.json.gz 2>/dev/null || stat -c%s /sgl-kernel/build-trace.json.gz) + echo \"Compressed to: build-trace.json.gz (\${RATIO}% smaller)\" + fi + echo \"\" + echo \"View in browser:\" + echo \" - chrome://tracing (load JSON file)\" + echo \" - ui.perfetto.dev (recommended, supports .gz files)\" + echo \"\" + echo \"Shows:\" + echo \" - Compilation time per file\" + echo \" - Parallelism utilization\" + echo \" - Critical path (longest dependency chain)\" + echo \" - Where the 2-hour build time went\" + fi + fi + echo \"\" + fi + + # Show ccache statistics after build + if [ \"${USE_CCACHE}\" = \"1\" ]; then + echo \"\" + echo \"==================================\" + echo \"ccache Statistics\" + echo \"==================================\" + ccache -s + echo \"\" + fi + " diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/cmake/flashmla.cmake b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/cmake/flashmla.cmake new file mode 100644 index 00000000..c17266af --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/cmake/flashmla.cmake @@ -0,0 +1,67 @@ +include(FetchContent) + +# flash_mla +FetchContent_Declare( + repo-flashmla + GIT_REPOSITORY https://github.com/sgl-project/FlashMLA + GIT_TAG be055fb7df0090fde45f08e9cb5b8b4c0272da73 + GIT_SHALLOW OFF +) +FetchContent_Populate(repo-flashmla) + +set(FLASHMLA_CUDA_FLAGS + "--expt-relaxed-constexpr" + "--expt-extended-lambda" + "--use_fast_math" + + "-Xcudafe=--diag_suppress=177" # variable was declared but never referenced +) + +# The FlashMLA kernels only work on hopper and require CUDA 12.4 or later. +# Only build FlashMLA kernels if we are building for something compatible with +# sm90a +if(${CUDA_VERSION} VERSION_GREATER 12.4) + list(APPEND FLASHMLA_CUDA_FLAGS + "-gencode=arch=compute_90a,code=sm_90a" + ) +endif() +if(${CUDA_VERSION} VERSION_GREATER 12.8) + list(APPEND FLASHMLA_CUDA_FLAGS + "-gencode=arch=compute_100a,code=sm_100a" + ) +endif() + + +set(FlashMLA_SOURCES + "csrc/flashmla_extension.cc" + ${repo-flashmla_SOURCE_DIR}/csrc/python_api.cpp + ${repo-flashmla_SOURCE_DIR}/csrc/smxx/get_mla_metadata.cu + ${repo-flashmla_SOURCE_DIR}/csrc/smxx/mla_combine.cu + ${repo-flashmla_SOURCE_DIR}/csrc/sm90/decode/dense/splitkv_mla.cu + ${repo-flashmla_SOURCE_DIR}/csrc/sm90/decode/sparse_fp8/splitkv_mla.cu + ${repo-flashmla_SOURCE_DIR}/csrc/sm90/prefill/sparse/fwd.cu + ${repo-flashmla_SOURCE_DIR}/csrc/sm100/decode/sparse_fp8/splitkv_mla.cu + ${repo-flashmla_SOURCE_DIR}/csrc/sm100/prefill/dense/fmha_cutlass_fwd_sm100.cu + ${repo-flashmla_SOURCE_DIR}/csrc/sm100/prefill/dense/fmha_cutlass_bwd_sm100.cu + ${repo-flashmla_SOURCE_DIR}/csrc/sm100/prefill/sparse/fwd.cu + + ${repo-flashmla_SOURCE_DIR}/csrc/extension/sm90/dense_fp8/dense_fp8_python_api.cpp + ${repo-flashmla_SOURCE_DIR}/csrc/extension/sm90/dense_fp8/flash_fwd_mla_fp8_sm90.cu + ${repo-flashmla_SOURCE_DIR}/csrc/extension/sm90/dense_fp8/flash_fwd_mla_metadata.cu +) + +Python_add_library(flashmla_ops MODULE USE_SABI ${SKBUILD_SABI_VERSION} WITH_SOABI ${FlashMLA_SOURCES}) +target_compile_options(flashmla_ops PRIVATE $<$:${FLASHMLA_CUDA_FLAGS}>) +target_include_directories(flashmla_ops PRIVATE + ${repo-flashmla_SOURCE_DIR}/csrc + ${repo-flashmla_SOURCE_DIR}/csrc/sm90 + ${repo-flashmla_SOURCE_DIR}/csrc/extension/sm90/dense_fp8/ + ${repo-flashmla_SOURCE_DIR}/csrc/cutlass/include + ${repo-flashmla_SOURCE_DIR}/csrc/cutlass/tools/util/include +) + +target_link_libraries(flashmla_ops PRIVATE ${TORCH_LIBRARIES} c10 cuda) + +install(TARGETS flashmla_ops LIBRARY DESTINATION "sgl_kernel") + +target_compile_definitions(flashmla_ops PRIVATE) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/cmake/utils.cmake b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/cmake/utils.cmake new file mode 100644 index 00000000..8d676e47 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/cmake/utils.cmake @@ -0,0 +1,19 @@ +# Adapt from: https://github.com/neuralmagic/vllm-flash-attention/blob/main/cmake/utils.cmake +# +# Clear all `-gencode` flags from `CMAKE_CUDA_FLAGS` and store them in +# `CUDA_ARCH_FLAGS`. +# +# Example: +# CMAKE_CUDA_FLAGS="-Wall -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75" +# clear_cuda_arches(CUDA_ARCH_FLAGS) +# CUDA_ARCH_FLAGS="-gencode arch=compute_70,code=sm_70;-gencode arch=compute_75,code=sm_75" +# CMAKE_CUDA_FLAGS="-Wall" +# +macro(clear_cuda_arches CUDA_ARCH_FLAGS) + # Extract all `-gencode` flags from `CMAKE_CUDA_FLAGS` + string(REGEX MATCHALL "-gencode arch=[^ ]+" CUDA_ARCH_FLAGS "${CMAKE_CUDA_FLAGS}") + + # Remove all `-gencode` flags from `CMAKE_CUDA_FLAGS` since they will be modified + # and passed back via the `CUDA_ARCHITECTURES` property. + string(REGEX REPLACE "-gencode arch=[^ ]+ *" "" CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS}") +endmacro() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/custom_all_reduce.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/custom_all_reduce.cu new file mode 100644 index 00000000..a1f5ed53 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/custom_all_reduce.cu @@ -0,0 +1,137 @@ +// Adapted from: https://github.com/vllm-project/vllm/blob/v0.8.2/csrc/custom_all_reduce.cu +#include +#include +#include +#include + +#include "custom_all_reduce.cuh" + +// Fake pointer type, must match fptr_t type in ops.h. +// We use this type alias to indicate when pointers are passed in as int64_t. +using fptr_t = int64_t; +static_assert(sizeof(void*) == sizeof(fptr_t)); + +fptr_t +init_custom_ar(const std::vector& fake_ipc_ptrs, torch::Tensor& rank_data, int64_t rank, bool full_nvlink) { + int world_size = fake_ipc_ptrs.size(); + if (world_size > 8) throw std::invalid_argument("world size > 8 is not supported"); + if (world_size % 2 != 0) throw std::invalid_argument("Odd num gpus is not supported for now"); + if (rank < 0 || rank >= world_size) throw std::invalid_argument("invalid rank passed in"); + + sglang::Signal* ipc_ptrs[8]; + for (int i = 0; i < world_size; i++) { + ipc_ptrs[i] = reinterpret_cast(fake_ipc_ptrs[i]); + } + return (fptr_t) new sglang::CustomAllreduce( + ipc_ptrs, rank_data.data_ptr(), rank_data.numel(), rank, world_size, full_nvlink); +} + +/** + * Make sure tensor t's data lies completely within ((char)t.data_ptr()) + + * t.numel() * t.element_size(). This is slightly weaker than t.is_contiguous() + * because it allows transpose of contiguous slice (i.e. slicing the first + * dimension). Currently, we require this because stride information is not + * passed into the kernels and we treat input tensors as flat. + * + * Examples + * A = torch.zeros(3, 3, 3) + * 1. A: OK + * 2. A[1:]: OK + * 3. A.permute(2, 0, 1): OK + * 4. A[1:].permute(2, 0, 1): OK + * 5. A[None].expand(2, -1, -1, -1): Not OK + * 6. A[:, 1:, 1:]: Not OK + */ +bool _is_weak_contiguous(torch::Tensor& t) { + return t.is_contiguous() || + (t.storage().nbytes() - t.storage_offset() * t.element_size() == t.numel() * t.element_size()); +} + +/** + * Performs an out-of-place allreduce and stores result in out. + * + * If _reg_buffer is null, assumes inp.data_ptr() is already IPC-registered. + * Otherwise, _reg_buffer is assumed to be IPC-registered and inp is first + * copied into _reg_buffer. + */ +void all_reduce(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out, fptr_t _reg_buffer, int64_t reg_buffer_sz_bytes) { + auto fa = reinterpret_cast(_fa); + const at::cuda::OptionalCUDAGuard device_guard(device_of(inp)); + auto stream = c10::cuda::getCurrentCUDAStream().stream(); + + TORCH_CHECK_EQ(inp.scalar_type(), out.scalar_type()); + TORCH_CHECK_EQ(inp.numel(), out.numel()); + TORCH_CHECK(_is_weak_contiguous(out)); + TORCH_CHECK(_is_weak_contiguous(inp)); + auto input_size = inp.numel() * inp.element_size(); + auto reg_buffer = reinterpret_cast(_reg_buffer); + if (reg_buffer) { + TORCH_CHECK_LE(input_size, reg_buffer_sz_bytes); + AT_CUDA_CHECK(cudaMemcpyAsync(reg_buffer, inp.data_ptr(), input_size, cudaMemcpyDeviceToDevice, stream)); + } else { + reg_buffer = inp.data_ptr(); + } + switch (out.scalar_type()) { + case at::ScalarType::Float: { + fa->allreduce( + stream, reinterpret_cast(reg_buffer), reinterpret_cast(out.data_ptr()), out.numel()); + break; + } + case at::ScalarType::Half: { + fa->allreduce( + stream, reinterpret_cast(reg_buffer), reinterpret_cast(out.data_ptr()), out.numel()); + break; + } +#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)) + case at::ScalarType::BFloat16: { + fa->allreduce( + stream, + reinterpret_cast(reg_buffer), + reinterpret_cast(out.data_ptr()), + out.numel()); + break; + } +#endif + default: + throw std::runtime_error("custom allreduce only supports float32, float16 and bfloat16"); + } +} + +void dispose(fptr_t _fa) { + delete reinterpret_cast(_fa); +} + +int64_t meta_size() { + return sizeof(sglang::Signal); +} + +void register_buffer(fptr_t _fa, const std::vector& fake_ipc_ptrs) { + auto fa = reinterpret_cast(_fa); + TORCH_CHECK(fake_ipc_ptrs.size() == fa->world_size_); + void* ipc_ptrs[8]; + for (int i = 0; i < fake_ipc_ptrs.size(); i++) { + ipc_ptrs[i] = reinterpret_cast(fake_ipc_ptrs[i]); + } + fa->register_buffer(ipc_ptrs); +} + +// Use vector to represent byte data for python binding compatibility. +std::tuple, std::vector> get_graph_buffer_ipc_meta(fptr_t _fa) { + auto fa = reinterpret_cast(_fa); + auto [handle, offsets] = fa->get_graph_buffer_ipc_meta(); + std::vector bytes(handle.begin(), handle.end()); + return std::make_tuple(bytes, offsets); +} + +// Use vector to represent byte data for python binding compatibility. +void register_graph_buffers( + fptr_t _fa, const std::vector>& handles, const std::vector>& offsets) { + auto fa = reinterpret_cast(_fa); + std::vector bytes; + bytes.reserve(handles.size()); + for (int i = 0; i < handles.size(); i++) { + bytes.emplace_back(handles[i].begin(), handles[i].end()); + } + bytes.reserve(handles.size()); + fa->register_graph_buffers(bytes, offsets); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/custom_all_reduce.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/custom_all_reduce.cuh new file mode 100644 index 00000000..ec223bde --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/custom_all_reduce.cuh @@ -0,0 +1,489 @@ +// Adapted from https://github.com/vllm-project/vllm/blob/v0.8.2/csrc/custom_all_reduce.cuh +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "utils.h" + +namespace sglang { + +constexpr int kMaxBlocks = 36; +// Counter may overflow, but it's fine since unsigned int overflow is +// well-defined behavior. +using FlagType = uint32_t; +struct Signal { + alignas(128) FlagType self_counter[kMaxBlocks][8]; + // Two sets of peer counters are needed for two syncs. The reason is that + // it's possible for peer GPU block to arrive at the second sync point while + // the current GPU block haven't passed the first sync point. Thus, peer GPU + // may write counter+1 while current GPU is busy waiting for counter. We use + // alternating counter array to avoid this possibility. + alignas(128) FlagType peer_counter[2][kMaxBlocks][8]; +}; + +struct __align__(16) RankData { + const void* __restrict__ ptrs[8]; +}; + +struct __align__(16) RankSignals { + Signal* signals[8]; +}; + +// like std::array, but aligned +template +struct __align__(alignof(T) * sz) array_t { + T data[sz]; + using type = T; + static constexpr int size = sz; +}; + +// use packed type to maximize memory efficiency +// goal: generate ld.128 and st.128 instructions +template +struct packed_t { + // the (P)acked type for load/store + using P = array_t; + // the (A)ccumulator type for reduction + using A = array_t; +}; + +#define DINLINE __device__ __forceinline__ + +// scalar cast functions +DINLINE float upcast_s(half val) { + return __half2float(val); +} + +template +DINLINE T downcast_s(float val); +template <> +DINLINE half downcast_s(float val) { + return __float2half(val); +} + +// scalar add functions +// for some reason when compiling with Pytorch, the + operator for half and +// bfloat is disabled so we call the intrinsics directly +DINLINE half& assign_add(half& a, half b) { + a = __hadd(a, b); + return a; +} +DINLINE float& assign_add(float& a, float b) { + return a += b; +} + +#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)) +DINLINE float upcast_s(nv_bfloat16 val) { + return __bfloat162float(val); +} +template <> +DINLINE nv_bfloat16 downcast_s(float val) { + return __float2bfloat16(val); +} +DINLINE nv_bfloat16& assign_add(nv_bfloat16& a, nv_bfloat16 b) { + a = __hadd(a, b); + return a; +} +#endif + +template +DINLINE array_t& packed_assign_add(array_t& a, array_t b) { +#pragma unroll + for (int i = 0; i < N; i++) { + assign_add(a.data[i], b.data[i]); + } + return a; +} + +template +DINLINE array_t upcast(array_t val) { + if constexpr (std::is_same::value) { + return val; + } else { + array_t out; +#pragma unroll + for (int i = 0; i < N; i++) { + out.data[i] = upcast_s(val.data[i]); + } + return out; + } +} + +template +DINLINE O downcast(array_t val) { + if constexpr (std::is_same::value) { + return val; + } else { + O out; +#pragma unroll + for (int i = 0; i < O::size; i++) { + out.data[i] = downcast_s(val.data[i]); + } + return out; + } +} + +static DINLINE void st_flag_release(FlagType* flag_addr, FlagType flag) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700 + asm volatile("st.release.sys.global.u32 [%1], %0;" ::"r"(flag), "l"(flag_addr)); +#else + asm volatile("membar.sys; st.volatile.global.u32 [%1], %0;" ::"r"(flag), "l"(flag_addr)); +#endif +} + +static DINLINE FlagType ld_flag_acquire(FlagType* flag_addr) { + FlagType flag; +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700 + asm volatile("ld.acquire.sys.global.u32 %0, [%1];" : "=r"(flag) : "l"(flag_addr)); +#else + asm volatile("ld.volatile.global.u32 %0, [%1]; membar.gl;" : "=r"(flag) : "l"(flag_addr)); +#endif + return flag; +} + +static DINLINE void st_flag_volatile(FlagType* flag_addr, FlagType flag) { + asm volatile("st.volatile.global.u32 [%1], %0;" ::"r"(flag), "l"(flag_addr)); +} + +static DINLINE FlagType ld_flag_volatile(FlagType* flag_addr) { + FlagType flag; + asm volatile("ld.volatile.global.u32 %0, [%1];" : "=r"(flag) : "l"(flag_addr)); + return flag; +} + +// is_start: whether this is the very first synchronization barrier. +// need_fence: whether a memory fence is needed. If true, a release-acquire +// semantic is used to enforce memory access order before and after this +// barrier. +template +DINLINE void multi_gpu_barrier(const RankSignals& sg, Signal* self_sg, int rank) { + if constexpr (!is_start) __syncthreads(); + static_assert(!(is_start && need_fence)); // Start barrier shouldn't need fence. + if (threadIdx.x < ngpus) { + // Increment the counter. Technically we only need one counter, but we use + // multiple per block to eliminate the need to share the counter via smem. + auto val = self_sg->self_counter[blockIdx.x][threadIdx.x] += 1; + // Write the expected counter value to peer and wait for correct value from + // peer. + auto peer_counter_ptr = &sg.signals[threadIdx.x]->peer_counter[val % 2][blockIdx.x][rank]; + auto self_counter_ptr = &self_sg->peer_counter[val % 2][blockIdx.x][threadIdx.x]; + if constexpr (need_fence) { + st_flag_release(peer_counter_ptr, val); + while (ld_flag_acquire(self_counter_ptr) != val) + ; + } else { + st_flag_volatile(peer_counter_ptr, val); + while (ld_flag_volatile(self_counter_ptr) != val) + ; + } + } + if constexpr (is_start || need_fence) __syncthreads(); +} + +template +DINLINE P packed_reduce(const P* ptrs[], int idx) { + A tmp = upcast(ptrs[0][idx]); +#pragma unroll + for (int i = 1; i < ngpus; i++) { + packed_assign_add(tmp, upcast(ptrs[i][idx])); + } + return downcast

(tmp); +} + +template +__global__ void __launch_bounds__(512, 1) cross_device_reduce_1stage( + RankData* _dp, RankSignals sg, Signal* self_sg, T* __restrict__ result, int rank, int size) { + using P = typename packed_t::P; + using A = typename packed_t::A; + // note: we don't reorder the address so the accumulation order is the same + // for all ranks, ensuring bitwise identical results + auto dp = *_dp; + multi_gpu_barrier(sg, self_sg, rank); + // do the actual reduction + for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size; idx += gridDim.x * blockDim.x) { + ((P*)result)[idx] = packed_reduce((const P**)&dp.ptrs[0], idx); + } + multi_gpu_barrier(sg, self_sg, rank); +} + +template +DINLINE P* get_tmp_buf(Signal* sg) { + return (P*)(((Signal*)sg) + 1); +} + +template +__global__ void __launch_bounds__(512, 1) cross_device_reduce_2stage( + RankData* _dp, RankSignals sg, Signal* self_sg, T* __restrict__ result, int rank, int size) { + int tid = blockIdx.x * blockDim.x + threadIdx.x; + int stride = gridDim.x * blockDim.x; + using P = typename packed_t::P; + using A = typename packed_t::A; + int part = size / ngpus; + int start = rank * part; + int end = rank == ngpus - 1 ? size : start + part; + int largest_part = part + size % ngpus; + const P* ptrs[ngpus]; + P* tmps[ngpus]; +#pragma unroll + for (int i = 0; i < ngpus; i++) { + int target = (rank + i) % ngpus; + ptrs[i] = (const P*)_dp->ptrs[target]; + tmps[i] = get_tmp_buf

(sg.signals[target]); + } + auto tmp_out = tmps[0]; + multi_gpu_barrier(sg, self_sg, rank); + // stage 1: reduce scatter + for (int idx = start + tid; idx < end; idx += stride) { + tmp_out[idx - start] = packed_reduce(ptrs, idx); + } + multi_gpu_barrier(sg, self_sg, rank); + + // stage 2: allgather. Note: it's important to match the tid between + // the two stages, because visibility across devices is only guaranteed + // between threads that have the same tid. If thread i computes the sum of + // start + i in the first stage, then thread i also gathers start + i from all + // ranks. + for (int idx = tid; idx < largest_part; idx += stride) { +#pragma unroll + for (int i = 0; i < ngpus; i++) { + int gather_from_rank = ((rank + i) % ngpus); + if (gather_from_rank == ngpus - 1 || idx < part) { + int dst_idx = gather_from_rank * part + idx; + ((P*)result)[dst_idx] = tmps[i][idx]; + } + } + } +} + +using IPC_KEY = std::array; +static_assert(sizeof(IPC_KEY) == sizeof(cudaIpcMemHandle_t)); +static_assert(alignof(IPC_KEY) == alignof(cudaIpcMemHandle_t)); + +class CustomAllreduce { + public: + int rank_; + int world_size_; + bool full_nvlink_; + + RankSignals sg_; + // Stores an map from a pointer to its peer pointters from all ranks. + std::unordered_map buffers_; + Signal* self_sg_; + + // Stores rank data from all ranks. This is mainly for cuda graph purposes. + // For cuda graph to work, all kernel arguments must be fixed during graph + // capture time. However, the peer pointers are not known during graph capture + // time. Therefore, during capture, we increment the rank data pointer and use + // that as the argument to the kernel. The kernel arguments are stored in + // graph_unreg_buffers_. The actual peer pointers will be filled in at the + // memory pointed to by the pointers in graph_unreg_buffers_ when + // the IPC handles are exchanged between ranks. + // + // The overall process looks like this: + // 1. Graph capture. + // 2. Each rank obtains the IPC handles for each addresses used during cuda + // graph capture using get_graph_buffer_ipc_meta. + // 3. (In Python) all gather the IPC handles. + // 4. Obtain the peer pointers by opening the IPC handles, and store them in + // the rank data array at corresponding positions. + RankData *d_rank_data_base_, *d_rank_data_end_; + std::vector graph_unreg_buffers_; + // a map from IPC handles to opened IPC pointers + std::map ipc_handles_; + + /** + * Signals are an array of ipc-enabled buffers from all ranks. + * For each of the buffer, the layout is as follows: + * | -- sizeof(Signal) -- | ------ a few MB ----- | + * The first section is for allreduce synchronization, and the second section + * is for storing the intermediate results required by some allreduce algos. + * + * Note: this class does not own any device memory. Any required buffers + * are passed in from the constructor. + */ + CustomAllreduce( + Signal** signals, void* rank_data, size_t rank_data_sz, int rank, int world_size, bool full_nvlink = true) + : rank_(rank), + world_size_(world_size), + full_nvlink_(full_nvlink), + self_sg_(signals[rank]), + d_rank_data_base_(reinterpret_cast(rank_data)), + d_rank_data_end_(d_rank_data_base_ + rank_data_sz / sizeof(RankData)) { + for (int i = 0; i < world_size_; i++) { + sg_.signals[i] = signals[i]; + } + } + + char* open_ipc_handle(const void* ipc_handle) { + auto [it, new_handle] = ipc_handles_.insert({*((IPC_KEY*)ipc_handle), nullptr}); + if (new_handle) { + char* ipc_ptr; + CHECK_CUDA_SUCCESS(cudaIpcOpenMemHandle( + (void**)&ipc_ptr, *((const cudaIpcMemHandle_t*)ipc_handle), cudaIpcMemLazyEnablePeerAccess)); + it->second = ipc_ptr; + } + return it->second; + } + + std::pair> get_graph_buffer_ipc_meta() { + auto num_buffers = graph_unreg_buffers_.size(); + auto handle_sz = sizeof(cudaIpcMemHandle_t); + std::string handles(handle_sz * num_buffers, static_cast(0)); + std::vector offsets(num_buffers); + for (int i = 0; i < num_buffers; i++) { + auto ptr = graph_unreg_buffers_[i]; + void* base_ptr; + // note: must share the base address of each allocation, or we get wrong + // address + if (cuPointerGetAttribute(&base_ptr, CU_POINTER_ATTRIBUTE_RANGE_START_ADDR, (CUdeviceptr)ptr) != CUDA_SUCCESS) + throw std::runtime_error("failed to get pointer attr"); + CHECK_CUDA_SUCCESS(cudaIpcGetMemHandle((cudaIpcMemHandle_t*)&handles[i * handle_sz], base_ptr)); + offsets[i] = ((char*)ptr) - ((char*)base_ptr); + } + return std::make_pair(handles, offsets); + } + + void check_rank_data_capacity(size_t num = 1) { + if (d_rank_data_base_ + num > d_rank_data_end_) + throw std::runtime_error( + "Rank data buffer is overflowed by " + std::to_string(d_rank_data_base_ + num - d_rank_data_end_)); + } + + /** + * Register already-shared IPC pointers. + */ + void register_buffer(void** ptrs) { + check_rank_data_capacity(); + RankData data; + for (int i = 0; i < world_size_; i++) { + data.ptrs[i] = ptrs[i]; + } + auto d_data = d_rank_data_base_++; + CHECK_CUDA_SUCCESS(cudaMemcpy(d_data, &data, sizeof(RankData), cudaMemcpyHostToDevice)); + buffers_[ptrs[rank_]] = d_data; + } + + // Note: when registering graph buffers, we intentionally choose to not + // deduplicate the addresses. That means if the allocator reuses some + // addresses, they will be registered again. This is to account for the remote + // possibility of different allocation patterns between ranks. For example, + // rank 1 may get the same input address for the second allreduce, but rank 2 + // got a different address. IPC handles have internal reference counting + // mechanism so overhead should be small. + void + register_graph_buffers(const std::vector& handles, const std::vector>& offsets) { + auto num_buffers = graph_unreg_buffers_.size(); + check_rank_data_capacity(num_buffers); + std::vector rank_data(num_buffers); + for (int i = 0; i < num_buffers; i++) { + auto self_ptr = graph_unreg_buffers_[i]; + auto& rd = rank_data[i]; + for (int j = 0; j < world_size_; j++) { + if (j != rank_) { + char* handle = open_ipc_handle(&handles[j][i * sizeof(cudaIpcMemHandle_t)]); + handle += offsets[j][i]; + rd.ptrs[j] = handle; + } else { + rd.ptrs[j] = self_ptr; + } + } + } + CHECK_CUDA_SUCCESS( + cudaMemcpy(d_rank_data_base_, rank_data.data(), sizeof(RankData) * num_buffers, cudaMemcpyHostToDevice)); + d_rank_data_base_ += num_buffers; + graph_unreg_buffers_.clear(); + } + + /** + * Performs allreduce, assuming input has already been registered. + * + * Block and grid default configs are results after careful grid search. Using + * 36 blocks give the best or close to the best runtime on the devices I + * tried: A100, A10, A30, T4, V100. You'll notice that NCCL kernels also only + * take a small amount of SMs. Not quite sure the underlying reason, but my + * guess is that too many SMs will cause contention on NVLink bus. + */ + template + void allreduce(cudaStream_t stream, T* input, T* output, int size, int threads = 512, int block_limit = 36) { + auto d = packed_t::P::size; + if (size % d != 0) + throw std::runtime_error( + "custom allreduce currently requires input length to be multiple " + "of " + + std::to_string(d)); + if (block_limit > kMaxBlocks) + throw std::runtime_error( + "max supported block limit is " + std::to_string(kMaxBlocks) + ". Got " + std::to_string(block_limit)); + + RankData* ptrs; + cudaStreamCaptureStatus status; + CHECK_CUDA_SUCCESS(cudaStreamIsCapturing(stream, &status)); + if (status == cudaStreamCaptureStatusActive) { + ptrs = d_rank_data_base_ + graph_unreg_buffers_.size(); + graph_unreg_buffers_.push_back(input); + } else { + auto it = buffers_.find(input); + if (it == buffers_.end()) + throw std::runtime_error( + "buffer address " + std::to_string(reinterpret_cast(input)) + " is not registered!"); + ptrs = it->second; + } + + size /= d; + auto bytes = size * sizeof(typename packed_t::P); + int blocks = std::min(block_limit, (size + threads - 1) / threads); +#define KL(ngpus, name) name<<>>(ptrs, sg_, self_sg_, output, rank_, size); + // TODO(hanzhi713): Threshold is different for A100 and H100. + // Add per device threshold. +#define REDUCE_CASE(ngpus) \ + case ngpus: { \ + if (world_size_ == 2) { \ + KL(ngpus, cross_device_reduce_1stage); \ + } else if (full_nvlink_) { \ + if ((world_size_ <= 4 && bytes < 512 * 1024) || (world_size_ <= 8 && bytes < 256 * 1024)) { \ + KL(ngpus, cross_device_reduce_1stage); \ + } else { \ + KL(ngpus, cross_device_reduce_2stage); \ + } \ + } \ + break; \ + } + + switch (world_size_) { + REDUCE_CASE(2) + REDUCE_CASE(4) + REDUCE_CASE(6) + REDUCE_CASE(8) + default: + throw std::runtime_error( + "custom allreduce only supports num gpus in (2,4,6,8). Actual num " + "gpus = " + + std::to_string(world_size_)); + } +#undef REDUCE_CASE +#undef KL + } + + ~CustomAllreduce() { + for (auto [_, ptr] : ipc_handles_) { + CHECK_CUDA_SUCCESS(cudaIpcCloseMemHandle(ptr)); + } + } +}; +/** + * To inspect PTX/SASS, copy paste this header file to compiler explorer and add + a template instantiation: + * template void sglang::CustomAllreduce::allreduce(cudaStream_t, half *, + half *, int, int, int); +*/ +} // namespace sglang diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/custom_all_reduce.hip b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/custom_all_reduce.hip new file mode 100644 index 00000000..7a1a586d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/custom_all_reduce.hip @@ -0,0 +1,180 @@ +// !!! This is a file automatically generated by hipify!!! +#include +#include +#include +#include + +#include "custom_all_reduce_hip.cuh" + +// fake pointer type, must match fptr_t type in ops.h +using fptr_t = int64_t; +static_assert(sizeof(void*) == sizeof(fptr_t)); + +fptr_t init_custom_ar(torch::Tensor& meta, torch::Tensor& rank_data, + const std::vector& handles, + const std::vector& offsets, int64_t rank, + bool full_nvlink) { + int world_size = offsets.size(); + if (world_size > 8) + throw std::invalid_argument("world size > 8 is not supported"); + if (world_size % 2 != 0) + throw std::invalid_argument("Odd num gpus is not supported for now"); + if (world_size != handles.size()) + throw std::invalid_argument( + "handles length should equal to offsets length"); + if (rank < 0 || rank >= world_size) + throw std::invalid_argument("invalid rank passed in"); + + hipIpcMemHandle_t ipc_handles[8]; + for (int i = 0; i < world_size; i++) { + std::memcpy(&ipc_handles[i], handles[i].data(), sizeof(hipIpcMemHandle_t)); + } + return (fptr_t) new sglang::CustomAllreduce( + reinterpret_cast(meta.data_ptr()), rank_data.data_ptr(), + rank_data.numel(), ipc_handles, offsets, rank, full_nvlink); +} + +/** + * Make sure tensor t's data lies completely within ((char)t.data_ptr()) + + * t.numel() * t.element_size(). This is slightly weaker than t.is_contiguous() + * because it allows transpose of contiguous slice (i.e. slicing the first + * dimension). Currently, we require this because stride information is not + * passed into the kernels and we treat input tensors as flat. + * + * Examples + * A = torch.zeros(3, 3, 3) + * 1. A: OK + * 2. A[1:]: OK + * 3. A.permute(2, 0, 1): OK + * 4. A[1:].permute(2, 0, 1): OK + * 5. A[None].expand(2, -1, -1, -1): Not OK + * 6. A[:, 1:, 1:]: Not OK + */ +bool _is_weak_contiguous(torch::Tensor& t) { + return t.is_contiguous() || + (t.storage().nbytes() - t.storage_offset() * t.element_size() == + t.numel() * t.element_size()); +} + +void _all_reduce(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out, + hipStream_t stream) { + auto fa = reinterpret_cast(_fa); + TORCH_CHECK(_is_weak_contiguous(out)); + switch (out.scalar_type()) { + case at::ScalarType::Float: { + fa->allreduce(stream, reinterpret_cast(inp.data_ptr()), + reinterpret_cast(out.data_ptr()), + out.numel()); + break; + } + case at::ScalarType::Half: { + fa->allreduce(stream, reinterpret_cast(inp.data_ptr()), + reinterpret_cast(out.data_ptr()), out.numel()); + break; + } +#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)) + case at::ScalarType::BFloat16: { + fa->allreduce( + stream, reinterpret_cast(inp.data_ptr()), + reinterpret_cast(out.data_ptr()), out.numel()); + break; + } +#endif + default: + throw std::runtime_error( + "custom allreduce only supports float32, float16 and bfloat16"); + } +} + +void all_reduce_reg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out) { + const at::hip::OptionalHIPGuardMasqueradingAsCUDA device_guard(device_of(inp)); + auto stream = c10::hip::getCurrentHIPStreamMasqueradingAsCUDA().stream(); + TORCH_CHECK_EQ(inp.scalar_type(), out.scalar_type()); + TORCH_CHECK_EQ(inp.numel(), out.numel()); + _all_reduce(_fa, inp, out, stream); +} + +void all_reduce_unreg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& reg_buffer, + torch::Tensor& out) { + const at::hip::OptionalHIPGuardMasqueradingAsCUDA device_guard(device_of(inp)); + auto stream = c10::hip::getCurrentHIPStreamMasqueradingAsCUDA().stream(); + + auto input_size = inp.numel() * inp.element_size(); + TORCH_CHECK_EQ(inp.scalar_type(), out.scalar_type()); + TORCH_CHECK_EQ(inp.numel(), out.numel()); + TORCH_CHECK(input_size <= reg_buffer.numel() * reg_buffer.element_size(), + "registered buffer is too small to contain the input"); + AT_CUDA_CHECK(hipMemcpyAsync(reg_buffer.data_ptr(), inp.data_ptr(), + input_size, hipMemcpyDeviceToDevice, stream)); + _all_reduce(_fa, reg_buffer, out, stream); +} + +void dispose(fptr_t _fa) { + auto fa = reinterpret_cast(_fa); + delete fa; +} + +int64_t meta_size() { return sizeof(sglang::Signal); } + +void register_buffer(fptr_t _fa, torch::Tensor& t, + const std::vector& handles, + const std::vector& offsets) { + auto fa = reinterpret_cast(_fa); + fa->register_buffer(handles, offsets, t.data_ptr()); +} + +std::tuple> get_graph_buffer_ipc_meta( + fptr_t _fa) { + auto fa = reinterpret_cast(_fa); + auto [handle_bytes, offsets] = fa->get_graph_buffer_ipc_meta(); + auto options = + torch::TensorOptions().dtype(torch::kUInt8).device(torch::kCPU); + auto handles = + torch::empty({static_cast(handle_bytes.size())}, options); + std::memcpy(handles.data_ptr(), handle_bytes.data(), handle_bytes.size()); + return {handles, std::move(offsets)}; +} + +void register_graph_buffers(fptr_t _fa, const std::vector& handles, + const std::vector>& offsets) { + auto fa = reinterpret_cast(_fa); + fa->register_graph_buffers(handles, offsets); +} + +void free_meta_buffer(void* buffer) { CUDACHECK(hipFree(buffer)); } + +torch::Tensor get_meta_buffer_ipc_handle(torch::Tensor& inp) { + auto options = + torch::TensorOptions().dtype(torch::kUInt8).device(torch::kCPU); + auto data_handle = + torch::empty({static_cast(sizeof(hipIpcMemHandle_t))}, options); + CUDACHECK(hipIpcGetMemHandle((hipIpcMemHandle_t*)data_handle.data_ptr(), + inp.data_ptr())); + return data_handle; +} + +torch::Tensor allocate_meta_buffer(int64_t size) { + auto device_index = c10::hip::current_device(); + at::DeviceGuard device_guard(at::Device(at::DeviceType::CUDA, device_index)); + void* buffer; + hipStreamCaptureMode mode = hipStreamCaptureModeRelaxed; + auto stream = c10::hip::getCurrentHIPStreamMasqueradingAsCUDA().stream(); + AT_CUDA_CHECK(hipThreadExchangeStreamCaptureMode(&mode)); + AT_CUDA_CHECK( + hipExtMallocWithFlags((void**)&buffer, size, hipDeviceMallocUncached)); + AT_CUDA_CHECK(hipMemsetAsync(buffer, 0, size, stream)); + AT_CUDA_CHECK(hipStreamSynchronize(stream)); + AT_CUDA_CHECK(hipThreadExchangeStreamCaptureMode(&mode)); + auto options = torch::TensorOptions() + .dtype(torch::kI8) + .device(torch::kCUDA, device_index); + return torch::from_blob(buffer, {size}, free_meta_buffer, options); +} + +std::vector get_device_bdf(int dev) { + char busIdStr[] = "0000:00:00.0"; + std::vector bdf(sizeof(busIdStr), 0); + CUDACHECK(hipDeviceGetPCIBusId((char*)bdf.data(), sizeof(busIdStr), dev)); + bdf.resize(bdf.size() - 1); // remove trailing NULL + return bdf; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/custom_all_reduce_hip.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/custom_all_reduce_hip.cuh new file mode 100644 index 00000000..ff4d28d2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/custom_all_reduce_hip.cuh @@ -0,0 +1,582 @@ +// !!! This is a file automatically generated by hipify!!! +#pragma once + +#include +#ifdef USE_ROCM +#include +typedef __hip_bfloat16 nv_bfloat16; +#else +#include +#endif +#include +#include + +#include +#include +#include +#include +#include + +#define CUDACHECK(cmd) \ + do { \ + hipError_t e = cmd; \ + if (e != hipSuccess) { \ + printf("Failed: Cuda error %s:%d '%s'\n", __FILE__, __LINE__, hipGetErrorString(e)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +namespace sglang { + +constexpr int kMaxBlocks = 64; +// note: we don't want to use atomics for signals because peer atomics are no +// supported on PCIe links +struct Signal { + alignas(128) uint32_t start[kMaxBlocks][8]; + alignas(128) uint32_t end[kMaxBlocks][8]; + alignas(128) uint32_t _flag[kMaxBlocks]; // incremental flags for each rank +}; + +#ifdef USE_ROCM +struct __align__(16) RankData { + const void* ptrs[8]; +}; +#else +struct __align__(16) RankData { + const void* __restrict__ ptrs[8]; +}; +#endif + +struct __align__(16) RankSignals { +#ifndef USE_ROCM + volatile +#endif + Signal* signals[8]; +}; + +// like std::array, but aligned +template +struct __align__(alignof(T) * sz) array_t { + T data[sz]; + using type = T; + static constexpr int size = sz; +}; + +// use packed type to maximize memory efficiency +// goal: generate ld.128 and st.128 instructions +template +struct packed_t { + // the (P)acked type for load/store + using P = array_t; + // the (A)ccumulator type for reduction + using A = array_t; +}; + +#define DINLINE __device__ __forceinline__ + +// scalar cast functions +DINLINE float upcast_s(half val) { + return __half2float(val); +} + +template +DINLINE T downcast_s(float val); +template <> +DINLINE half downcast_s(float val) { + return __float2half(val); +} + +// scalar add functions +// for some reason when compiling with Pytorch, the + operator for half and +// bfloat is disabled so we call the intrinsics directly +DINLINE half& assign_add(half& a, half b) { + a = __hadd(a, b); + return a; +} +DINLINE float& assign_add(float& a, float b) { + return a += b; +} + +#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)) +DINLINE float upcast_s(nv_bfloat16 val) { + return __bfloat162float(val); +} +template <> +DINLINE nv_bfloat16 downcast_s(float val) { + return __float2bfloat16(val); +} +DINLINE nv_bfloat16& assign_add(nv_bfloat16& a, nv_bfloat16 b) { + a = __hadd(a, b); + return a; +} +#endif + +template +DINLINE array_t& packed_assign_add(array_t& a, array_t b) { +#pragma unroll + for (int i = 0; i < N; i++) { + assign_add(a.data[i], b.data[i]); + } + return a; +} + +template +DINLINE array_t upcast(array_t val) { + if constexpr (std::is_same::value) { + return val; + } else { + array_t out; +#pragma unroll + for (int i = 0; i < N; i++) { + out.data[i] = upcast_s(val.data[i]); + } + return out; + } +} + +template +DINLINE O downcast(array_t val) { + if constexpr (std::is_same::value) { + return val; + } else { + O out; +#pragma unroll + for (int i = 0; i < O::size; i++) { + out.data[i] = downcast_s(val.data[i]); + } + return out; + } +} + +// This function is meant to be used as the first synchronization in the all +// reduce kernel. Thus, it doesn't need to make any visibility guarantees for +// prior memory accesses. Note: volatile writes will not be reordered against +// other volatile writes. +template +DINLINE void start_sync( + const RankSignals& sg, +#ifndef USE_ROCM + volatile +#endif + Signal* self_sg, + int rank) { +#ifdef USE_ROCM + uint32_t flag = self_sg->_flag[blockIdx.x] + 1; + if (threadIdx.x < ngpus) { + // simultaneously write to the corresponding flag of all ranks. + // Latency = 1 p2p write + __scoped_atomic_store_n( + &sg.signals[threadIdx.x]->start[blockIdx.x][rank], flag, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + // wait until we got true from all ranks + while (__scoped_atomic_load_n(&self_sg->start[blockIdx.x][threadIdx.x], __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE) < + flag) + ; + } + __syncthreads(); + // use one thread to update flag + if (threadIdx.x == 0) self_sg->_flag[blockIdx.x] = flag; +#else + if (threadIdx.x < ngpus) { + // reset flag for next time + self_sg->end[blockIdx.x][threadIdx.x] = 0; + // simultaneously write to the corresponding flag of all ranks. + // Latency = 1 p2p write + sg.signals[threadIdx.x]->start[blockIdx.x][rank] = 1; + // wait until we got true from all ranks + while (!self_sg->start[blockIdx.x][threadIdx.x]) + ; + } + __syncthreads(); +#endif +} + +// This function is meant to be used as the second or the final synchronization +// barrier in the all reduce kernel. If it's the final synchronization barrier, +// we don't need to make any visibility guarantees for prior memory accesses. +template +DINLINE void end_sync( + const RankSignals& sg, +#ifndef USE_ROCM + volatile +#endif + Signal* self_sg, + int rank) { +#ifdef USE_ROCM + __syncthreads(); + // eliminate the case that prior writes are not visible after signals become + // visible. Note that I did not managed to make this happen through a lot of + // testing. Might be the case that hardware provides stronger guarantee than + // the memory model. + uint32_t flag = self_sg->_flag[blockIdx.x] + 1; + if (threadIdx.x < ngpus) { + // simultaneously write to the corresponding flag of all ranks. + // Latency = 1 p2p write + __scoped_atomic_store_n( + &sg.signals[threadIdx.x]->end[blockIdx.x][rank], + flag, + final_sync ? __ATOMIC_RELAXED : __ATOMIC_RELEASE, + __MEMORY_SCOPE_SYSTEM); + // wait until we got true from all ranks + while (__scoped_atomic_load_n( + &self_sg->end[blockIdx.x][threadIdx.x], + final_sync ? __ATOMIC_RELAXED : __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_DEVICE) < flag) + ; + } + __syncthreads(); + // use one thread to update flag + if (threadIdx.x == 0) self_sg->_flag[blockIdx.x] = flag; +#else + __syncthreads(); + // eliminate the case that prior writes are not visible after signals become + // visible. Note that I did not managed to make this happen through a lot of + // testing. Might be the case that hardware provides stronger guarantee than + // the memory model. + if constexpr (!final_sync) __threadfence_system(); + if (threadIdx.x < ngpus) { + // reset flag for next time + self_sg->start[blockIdx.x][threadIdx.x] = 0; + // simultaneously write to the corresponding flag of all ranks. + // Latency = 1 p2p write + sg.signals[threadIdx.x]->end[blockIdx.x][rank] = 1; + // wait until we got true from all ranks + while (!self_sg->end[blockIdx.x][threadIdx.x]) + ; + } + if constexpr (!final_sync) __syncthreads(); +#endif +} + +template +DINLINE P packed_reduce(const P* ptrs[], int idx) { + A tmp = upcast(ptrs[0][idx]); +#pragma unroll + for (int i = 1; i < ngpus; i++) { + packed_assign_add(tmp, upcast(ptrs[i][idx])); + } + return downcast

(tmp); +} + +template +__global__ void __launch_bounds__(512, 1) cross_device_reduce_1stage( + RankData* _dp, + RankSignals sg, +#ifndef USE_ROCM + volatile +#endif + Signal* self_sg, + T* __restrict__ result, + int rank, + int size) { + using P = typename packed_t::P; + using A = typename packed_t::A; + // note: we don't reorder the address so the accumulation order is the same + // for all ranks, ensuring bitwise identical results + auto dp = *_dp; + start_sync(sg, self_sg, rank); + // do the actual reduction + for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size; idx += gridDim.x * blockDim.x) { + ((P*)result)[idx] = packed_reduce((const P**)&dp.ptrs[0], idx); + } + end_sync(sg, self_sg, rank); +} + +template +#ifdef USE_ROCM +DINLINE P* get_tmp_buf(Signal* sg) { +#else +DINLINE P* get_tmp_buf(volatile Signal* sg) { +#endif + return (P*)(((Signal*)sg) + 1); +} + +template +__global__ void __launch_bounds__(512, 1) cross_device_reduce_2stage( + RankData* _dp, + RankSignals sg, +#ifndef USE_ROCM + volatile +#endif + Signal* self_sg, + T* __restrict__ result, + int rank, + int size) { + int tid = blockIdx.x * blockDim.x + threadIdx.x; + int stride = gridDim.x * blockDim.x; + using P = typename packed_t::P; + using A = typename packed_t::A; + int part = size / ngpus; + int start = rank * part; + int end = rank == ngpus - 1 ? size : start + part; + int largest_part = part + size % ngpus; + const P* ptrs[ngpus]; + P* tmps[ngpus]; +#pragma unroll + for (int i = 0; i < ngpus; i++) { + int target = (rank + i) % ngpus; + ptrs[i] = (const P*)_dp->ptrs[target]; + tmps[i] = get_tmp_buf

(sg.signals[target]); + } + auto tmp_out = tmps[0]; + start_sync(sg, self_sg, rank); + // stage 1: reduce scatter + for (int idx = start + tid; idx < end; idx += stride) { + tmp_out[idx - start] = packed_reduce(ptrs, idx); + } + end_sync(sg, self_sg, rank); + + // stage 2: allgather. Note: it's important to match the tid between + // the two stages, because visibility across devices is only guaranteed + // between threads that have the same tid. If thread i computes the sum of + // start + i in the first stage, then thread i also gathers start + i from all + // ranks. + for (int idx = tid; idx < largest_part; idx += stride) { +#pragma unroll + for (int i = 0; i < ngpus; i++) { + int gather_from_rank = ((rank + i) % ngpus); + if (gather_from_rank == ngpus - 1 || idx < part) { + int dst_idx = gather_from_rank * part + idx; + ((P*)result)[dst_idx] = tmps[i][idx]; + } + } + } +} + +using IPC_KEY = std::array; +static_assert(sizeof(IPC_KEY) == sizeof(hipIpcMemHandle_t)); +static_assert(alignof(IPC_KEY) == alignof(hipIpcMemHandle_t)); + +class CustomAllreduce { + public: + int rank_; + int world_size_; + bool full_nvlink_; + + // below are device pointers + RankSignals sg_; + std::unordered_map buffers_; + Signal* self_sg_; + + // stores the registered device pointers from all ranks + RankData *d_rank_data_base_, *d_rank_data_end_; + std::vector graph_unreg_buffers_; + // a map from IPC handles to opened IPC pointers + std::map ipc_handles_; + + /** + * meta is a pointer to device metadata and temporary buffer for allreduce. + * + * There's a total of sizeof(Signal) of prefix before the actual data, + * so meta + 1 points to actual temporary buffer. + * + * note: this class does not own any device memory. Any required buffers + * are passed in from the constructor + */ + CustomAllreduce( + Signal* meta, + void* rank_data, + size_t rank_data_sz, + const hipIpcMemHandle_t* handles, + const std::vector& offsets, + int rank, + bool full_nvlink = true) + : rank_(rank), + world_size_(offsets.size()), + full_nvlink_(full_nvlink), + self_sg_(meta), + d_rank_data_base_(reinterpret_cast(rank_data)), + d_rank_data_end_(d_rank_data_base_ + rank_data_sz / sizeof(RankData)) { + for (int i = 0; i < world_size_; i++) { + Signal* rank_sg; + if (i != rank_) { + char* handle = open_ipc_handle(&handles[i]); + handle += offsets[i]; + rank_sg = (Signal*)handle; + } else { + rank_sg = self_sg_; + } + sg_.signals[i] = rank_sg; + } + } + + char* open_ipc_handle(const void* ipc_handle) { + auto [it, new_handle] = ipc_handles_.insert({*((IPC_KEY*)ipc_handle), nullptr}); + if (new_handle) { + char* ipc_ptr; + CUDACHECK(hipIpcOpenMemHandle( + (void**)&ipc_ptr, *((const hipIpcMemHandle_t*)ipc_handle), hipIpcMemLazyEnablePeerAccess)); + it->second = ipc_ptr; + } + return it->second; + } + + std::pair, std::vector> get_graph_buffer_ipc_meta() { + auto num_buffers = graph_unreg_buffers_.size(); + auto handle_sz = sizeof(hipIpcMemHandle_t); + std::vector handles(handle_sz * num_buffers, 0); + std::vector offsets(num_buffers); + for (int i = 0; i < num_buffers; i++) { + auto ptr = graph_unreg_buffers_[i]; + void* base_ptr; + // note: must share the base address of each allocation, or we get wrong + // address + if (hipPointerGetAttribute( + &base_ptr, +#ifdef USE_ROCM + HIP_POINTER_ATTRIBUTE_RANGE_START_ADDR, +#else + CU_POINTER_ATTRIBUTE_RANGE_START_ADDR, +#endif + (hipDeviceptr_t)ptr) != hipSuccess) + throw std::runtime_error("failed to get pointer attr"); + CUDACHECK(hipIpcGetMemHandle((hipIpcMemHandle_t*)&handles[i * handle_sz], base_ptr)); + offsets[i] = ((char*)ptr) - ((char*)base_ptr); + } + return std::make_pair(handles, offsets); + } + + void check_rank_data_capacity(size_t num = 1) { + if (d_rank_data_base_ + num > d_rank_data_end_) + throw std::runtime_error( + "Rank data buffer is overflowed by " + std::to_string(d_rank_data_base_ + num - d_rank_data_end_)); + } + + void register_buffer(const std::vector& handles, const std::vector& offsets, void* self) { + check_rank_data_capacity(); + RankData data; + for (int i = 0; i < world_size_; i++) { + if (i != rank_) { + char* handle = open_ipc_handle(handles[i].data()); + handle += offsets[i]; + data.ptrs[i] = handle; + } else { + data.ptrs[i] = self; + } + } + auto d_data = d_rank_data_base_++; + CUDACHECK(hipMemcpy(d_data, &data, sizeof(RankData), hipMemcpyHostToDevice)); + buffers_[self] = d_data; + } + + // note: when registering graph buffers, we intentionally choose to not + // deduplicate the addresses. That means if the allocator reuses some + // addresses, they will be registered again. This is to account for the remote + // possibility of different allocation patterns between ranks. For example, + // rank 1 may get the same input address for the second allreduce, but rank 2 + // got a different address. IPC handles have internal reference counting + // mechanism so overhead should be small. + void + register_graph_buffers(const std::vector& handles, const std::vector>& offsets) { + auto num_buffers = graph_unreg_buffers_.size(); + check_rank_data_capacity(num_buffers); + std::vector rank_data(num_buffers); + for (int i = 0; i < num_buffers; i++) { + auto self_ptr = graph_unreg_buffers_[i]; + auto& rd = rank_data[i]; + for (int j = 0; j < world_size_; j++) { + if (j != rank_) { + char* handle = open_ipc_handle(&handles[j][i * sizeof(hipIpcMemHandle_t)]); + handle += offsets[j][i]; + rd.ptrs[j] = handle; + } else { + rd.ptrs[j] = self_ptr; + } + } + } + CUDACHECK(hipMemcpy(d_rank_data_base_, rank_data.data(), sizeof(RankData) * num_buffers, hipMemcpyHostToDevice)); + d_rank_data_base_ += num_buffers; + graph_unreg_buffers_.clear(); + } + + /** + * This is the result after careful grid search. Using 36 blocks give the best + * or close to the best runtime on the devices I tried: A100, A10, A30, T4, + * V100. You'll notice that NCCL kernels also only take a small amount of SMs. + * Not quite sure the underlying reason, but my guess is that too many SMs + * will cause contention on NVLink bus. + */ + template + void allreduce( + hipStream_t stream, + T* input, + T* output, + int size, +#ifndef USE_ROCM + int threads = 512, + int block_limit = 36){ +#else + int threads = 512, + int block_limit = 16) { +#endif + auto d = packed_t::P::size; + if (size % d != 0) + throw std::runtime_error( + "custom allreduce currently requires input length to be multiple " + "of " + + std::to_string(d)); + if (block_limit > kMaxBlocks) + throw std::runtime_error( + "max supported block limit is " + std::to_string(kMaxBlocks) + ". Got " + std::to_string(block_limit)); + + RankData* ptrs; + hipStreamCaptureStatus status; + CUDACHECK(hipStreamIsCapturing(stream, &status)); + if (status == hipStreamCaptureStatusActive) { + ptrs = d_rank_data_base_ + graph_unreg_buffers_.size(); + graph_unreg_buffers_.push_back(input); + } else { + auto it = buffers_.find(input); + if (it == buffers_.end()) + throw std::runtime_error( + "buffer address " + std::to_string(reinterpret_cast(input)) + " is not registered!"); + ptrs = it->second; + } + + size /= d; + auto bytes = size * sizeof(typename packed_t::P); + int blocks = ::min(block_limit, (size + threads - 1) / threads); +#define KL(ngpus, name) \ + hipLaunchKernelGGL( \ + (name), dim3(blocks), dim3(threads), 0, stream, ptrs, sg_, self_sg_, output, rank_, size); +#define REDUCE_CASE(ngpus) \ + case ngpus: { \ + if (world_size_ == 2) { \ + KL(ngpus, cross_device_reduce_1stage); \ + } else if (full_nvlink_) { \ + if ((world_size_ <= 4 && bytes < 512 * 1024) || (world_size_ <= 8 && bytes < 256 * 1024)) { \ + KL(ngpus, cross_device_reduce_1stage); \ + } else { \ + KL(ngpus, cross_device_reduce_2stage); \ + } \ + } \ + break; \ + } + + switch (world_size_) { + REDUCE_CASE(2) + REDUCE_CASE(4) + REDUCE_CASE(6) + REDUCE_CASE(8) + default: + throw std::runtime_error( + "custom allreduce only supports num gpus in (2,4,6,8). Actual num " + "gpus = " + + std::to_string(world_size_)); + } +#undef REDUCE_CASE +#undef KL +} + +~CustomAllreduce() { + for (auto [_, ptr] : ipc_handles_) { + CUDACHECK(hipIpcCloseMemHandle(ptr)); + } +} +}; // namespace sglang +/** + * To inspect PTX/SASS, copy paste this header file to compiler explorer and add + a template instantiation: + * template void sglang::CustomAllreduce::allreduce(hipStream_t, half *, + half *, int, int, int); +*/ +} // namespace sglang diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/mscclpp_allreduce.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/mscclpp_allreduce.cu new file mode 100644 index 00000000..d9cda22f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/mscclpp_allreduce.cu @@ -0,0 +1,140 @@ +#include +#include +#include +#include + +#include "mscclpp_allreduce.cuh" + +enum MscclContextSelection { + MSCCL1NODELL = 1, + MSCCL2NODELL = 2, +}; + +class MscclContext { + public: + MscclContextSelection selection_; + std::shared_ptr msccl_1nodeLL_context; + std::shared_ptr msccl_2nodeLL_context; + MscclContext(MscclContextSelection selection) : selection_(selection) {} + template + void allreduce( + cudaStream_t stream, T* input, T* output, const size_t input_numel, int threads = 512, int block_limit = 21) { + if (selection_ == MSCCL1NODELL) { + msccl_1nodeLL_context->allreduce(stream, input, output, input_numel, threads, block_limit); + } else if (selection_ == MSCCL2NODELL) { + msccl_2nodeLL_context->allreduce(stream, input, output, input_numel, threads, block_limit); + } + } +}; + +using fptr_t = int64_t; +static_assert(sizeof(void*) == sizeof(fptr_t)); + +torch::Tensor _unique_id2tensor(const mscclpp::UniqueId& unique_id) { + auto options = torch::TensorOptions().dtype(torch::kByte).device(torch::kCPU); + auto tensor = torch::empty({static_cast(unique_id.size())}, options); + std::memcpy(tensor.data_ptr(), unique_id.data(), unique_id.size()); + return tensor; +} + +// Function to convert vector of int32_t back to array of uint8_t +mscclpp::UniqueId _tensor2unique_id(const torch::Tensor& tensor) { + mscclpp::UniqueId unique_id; + std::memcpy(unique_id.data(), tensor.data_ptr(), unique_id.size()); + return unique_id; +} + +torch::Tensor mscclpp_generate_unique_id() { + mscclpp::UniqueId unique_id = mscclpp::TcpBootstrap::createUniqueId(); + return _unique_id2tensor(unique_id); +} + +fptr_t mscclpp_init_context( + const torch::Tensor& unique_id, + const int64_t rank, + const int64_t world_size, + torch::Tensor& scratch, + torch::Tensor& put_buffer, + const int64_t nranks_per_node, + const std::vector& rank_to_node, + const std::vector& rank_to_ib, + const int64_t context_selection) { + MscclContext* context_ptr = new MscclContext(static_cast(context_selection)); + mscclpp::UniqueId uid = _tensor2unique_id(unique_id); + if (context_selection == MSCCL1NODELL) { + void* scratch_ptr = reinterpret_cast(scratch.data_ptr()); + const size_t scratch_bytes = scratch.numel() * scratch.element_size(); + context_ptr->msccl_1nodeLL_context = std::make_shared( + uid, rank, world_size, scratch_ptr, scratch_bytes, nranks_per_node, rank_to_node, rank_to_ib); + } else if (context_selection == MSCCL2NODELL) { + void* scratch_ptr = reinterpret_cast(scratch.data_ptr()); + const size_t scratch_bytes = scratch.numel() * scratch.element_size(); + void* put_buffer_ptr = reinterpret_cast(put_buffer.data_ptr()); + const size_t put_buffer_bytes = put_buffer.numel() * put_buffer.element_size(); + context_ptr->msccl_2nodeLL_context = std::make_shared( + uid, + rank, + world_size, + scratch_ptr, + scratch_bytes, + put_buffer_ptr, + put_buffer_bytes, + nranks_per_node, + rank_to_node, + rank_to_ib); + } else { + throw std::runtime_error("invalid context selection"); + } + return (fptr_t)context_ptr; +} + +bool _mscclpp_is_weak_contiguous(torch::Tensor& t) { + return t.is_contiguous() || + (t.storage().nbytes() - t.storage_offset() * t.element_size() == t.numel() * t.element_size()); +} +void mscclpp_allreduce(fptr_t _context, torch::Tensor& inp, torch::Tensor& out, int64_t nthreads, int64_t nblocks) { + MscclContext* context = reinterpret_cast(_context); + const at::cuda::OptionalCUDAGuard device_guard(device_of(inp)); + auto stream = c10::cuda::getCurrentCUDAStream().stream(); + + TORCH_CHECK_EQ(inp.scalar_type(), out.scalar_type()); + TORCH_CHECK_EQ(inp.numel(), out.numel()); + TORCH_CHECK(_mscclpp_is_weak_contiguous(out)); + TORCH_CHECK(_mscclpp_is_weak_contiguous(inp)); + switch (out.scalar_type()) { + case at::ScalarType::Float: { + context->allreduce( + stream, + reinterpret_cast(inp.data_ptr()), + reinterpret_cast(out.data_ptr()), + inp.numel(), + nthreads, + nblocks); + break; + } + case at::ScalarType::Half: { + context->allreduce( + stream, + reinterpret_cast(inp.data_ptr()), + reinterpret_cast(out.data_ptr()), + inp.numel(), + nthreads, + nblocks); + break; + } +#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)) + case at::ScalarType::BFloat16: { + context->allreduce<__nv_bfloat16>( + stream, + reinterpret_cast<__nv_bfloat16*>(inp.data_ptr()), + reinterpret_cast<__nv_bfloat16*>(out.data_ptr()), + inp.numel(), + nthreads, + nblocks); + break; + } +#endif + default: + throw std::runtime_error("custom allreduce only supports float32, float16 and bfloat16"); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/mscclpp_allreduce.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/mscclpp_allreduce.cuh new file mode 100644 index 00000000..ba0bc33f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/mscclpp_allreduce.cuh @@ -0,0 +1,779 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +#pragma once +#ifdef USE_ROCM +#include +#else +#include +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +// comment this for test_mscclpp_allreduce.cu +#include "utils.h" + +namespace sglang { + +__device__ mscclpp::DeviceSyncer deviceSyncer; +__device__ mscclpp::DeviceSyncer allGatherDeviceSyncer; +__device__ mscclpp::DeviceSyncer reduceScatterDeviceSyncer; +__device__ mscclpp::DeviceSyncer ibDeviceSyncer; + +template +__forceinline__ __device__ To bit_cast(const From& src) { + static_assert(sizeof(To) == sizeof(From), "Size mismatch for bit_cast"); + + union { + From f; + To t; + } u; + u.f = src; + return u.t; +} + +template +__forceinline__ __device__ T add_elements(T a, T b) { + return a + b; +} + +template <> +__forceinline__ __device__ __half2 add_elements(__half2 a, __half2 b) { + return __hadd2(a, b); +} + +#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)) +template <> +__forceinline__ __device__ __nv_bfloat162 add_elements(__nv_bfloat162 a, __nv_bfloat162 b) { + return __hadd2(a, b); +} +#endif + +template +__forceinline__ __device__ int4 add_vectors_helper(int4 a, int4 b) { + int4 ret; + ret.w = bit_cast(add_elements(bit_cast(a.w), bit_cast(b.w))); + ret.x = bit_cast(add_elements(bit_cast(a.x), bit_cast(b.x))); + ret.y = bit_cast(add_elements(bit_cast(a.y), bit_cast(b.y))); + ret.z = bit_cast(add_elements(bit_cast(a.z), bit_cast(b.z))); + return ret; +} + +template +__forceinline__ __device__ int4 add_vectors(int4 a, int4 b) { + return add_vectors_helper(a, b); +} + +#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)) +template <> +__forceinline__ __device__ int4 add_vectors<__nv_bfloat16>(int4 a, int4 b) { + return add_vectors_helper<__nv_bfloat162>(a, b); +} +#endif + +template <> +__forceinline__ __device__ int4 add_vectors<__half>(int4 a, int4 b) { + return add_vectors_helper<__half2>(a, b); +} + +template +__forceinline__ __device__ uint2 add_vectors_helper(uint2 a, uint2 b) { + uint2 ret; + ret.x = bit_cast(add_elements(bit_cast(a.x), bit_cast(b.x))); + ret.y = bit_cast(add_elements(bit_cast(a.y), bit_cast(b.y))); + return ret; +} + +template +__forceinline__ __device__ uint2 add_vectors(uint2 a, uint2 b) { + return add_vectors_helper(a, b); +} + +#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)) +template <> +__forceinline__ __device__ uint2 add_vectors<__nv_bfloat16>(uint2 a, uint2 b) { + return add_vectors_helper<__nv_bfloat162>(a, b); +} +#endif + +template <> +__forceinline__ __device__ uint2 add_vectors<__half>(uint2 a, uint2 b) { + return add_vectors_helper<__half2>(a, b); +} + +template +__forceinline__ __device__ int add_vectors_helper(int a, int b) { + return bit_cast(add_elements(bit_cast(a), bit_cast(b))); +} + +template +__forceinline__ __device__ int add_vectors(int a, int b) { + return add_vectors_helper(a, b); +} + +#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)) +template <> +__forceinline__ __device__ int add_vectors<__nv_bfloat16>(int a, int b) { + return add_vectors_helper<__nv_bfloat162>(a, b); +} +#endif + +template <> +__forceinline__ __device__ int add_vectors<__half>(int a, int b) { + return add_vectors_helper<__half2>(a, b); +} + +// ------------------------------------------------------- +// allreduce_LL_1node using LLPacket, origin allreduce2 +// ------------------------------------------------------- + +__device__ uint64_t globalFlag = 1; + +template +__global__ void __launch_bounds__(1024, 1) allreduce_LL_1node( + mscclpp::MemoryChannelDeviceHandle* memChans, + TYPE* buff, + TYPE* scratch, + void* resultBuff, + int rank, + int worldSize, + size_t nelems) { + nelems = nelems / (sizeof(int) / sizeof(TYPE)); + // This version of allreduce only works for single nodes + const int nPeers = worldSize - 1; + const size_t nPkts = nelems / 2; + const int nelemsPerRank = nelems / worldSize; + const int nPktsPerRank = nelemsPerRank / 2; + // flag for packets. Initially 1 + const uint32_t flag = (uint32_t)globalFlag; + // thread block & channel info + const int nBlocksPerPeer = gridDim.x / nPeers; + const int localBlockIdx = blockIdx.x % nBlocksPerPeer; + const int peerIdx = blockIdx.x / nBlocksPerPeer; + const int remoteRank = peerIdx < rank ? peerIdx : peerIdx + 1; + mscclpp::MemoryChannelDeviceHandle memChan = memChans[peerIdx]; + const int tid = threadIdx.x + localBlockIdx * blockDim.x; + // double buffering + size_t scratchBaseOffset = (flag & 1) ? 0 : nPkts * sizeof(mscclpp::LLPacket); + void* scratchBuff = (void*)((char*)scratch + scratchBaseOffset); + size_t scratchOffset = scratchBaseOffset + rank * nPktsPerRank * sizeof(mscclpp::LLPacket); + size_t scratchResultOffset = + (flag & 1) ? 2 * nPkts * sizeof(mscclpp::LLPacket) : 3 * nPkts * sizeof(mscclpp::LLPacket); + size_t srcOffset = remoteRank * nelemsPerRank * sizeof(int); + uint2* src = (uint2*)((char*)buff + rank * nelemsPerRank * sizeof(int)); + uint2* dst = (uint2*)((char*)resultBuff + rank * nelemsPerRank * sizeof(int)); + + // step 1: write to scratch buffer + memChan.putPackets(scratchOffset, srcOffset, nelemsPerRank * sizeof(int), tid, blockDim.x * nBlocksPerPeer, flag); + // step 2: get data from scratch buffer, reduce data and write result to remote scratch buffer + for (int idx = threadIdx.x + blockIdx.x * blockDim.x; idx < nPktsPerRank; idx += blockDim.x * gridDim.x) { + uint2 data = make_uint2(0, 0); + for (int index = 0; index < nPeers; index++) { + const int remoteRank = index < rank ? index : index + 1; + mscclpp::LLPacket* dstPkt = (mscclpp::LLPacket*)scratchBuff + remoteRank * nPktsPerRank; + uint2 val = dstPkt[idx].read(flag); + data = add_vectors(val, data); + } + data = add_vectors(data, src[idx]); + dst[idx] = data; + + mscclpp::LLPacket packet; + packet.data1 = data.x; + packet.flag1 = flag; + packet.data2 = data.y; + packet.flag2 = flag; + size_t offset = scratchResultOffset / sizeof(mscclpp::LLPacket) + (idx + rank * nPktsPerRank); + for (int index = 0; index < nPeers; index++) { + memChans[index].write(offset, packet); + } + } + // step 3: get data result from scratch buffer + mscclpp::LLPacket* dstPkt = (mscclpp::LLPacket*)((char*)scratch + scratchResultOffset); + const int dstOffset = remoteRank * nPktsPerRank; + uint2* result = (uint2*)((char*)resultBuff + remoteRank * nelemsPerRank * sizeof(int)); + for (int idx = threadIdx.x + localBlockIdx * blockDim.x; idx < nPktsPerRank; idx += blockDim.x * nBlocksPerPeer) { + uint2 data = dstPkt[idx + dstOffset].read(flag); + result[idx].x = data.x; + result[idx].y = data.y; + } + if (threadIdx.x == 0 && blockIdx.x == 0) { + globalFlag += 1; + } +} + +// ------------------------------------------------------- +// allreduce_LL_2node using LLPacket, origin allreduce5 +// ------------------------------------------------------- + +template +__global__ void __launch_bounds__(1024, 1) allreduce_LL_2node( + mscclpp::MemoryChannelDeviceHandle* memChans, + mscclpp::PortChannelDeviceHandle* portChans, + TYPE* buff, + TYPE* scratch, + TYPE* putBuff, + TYPE* resultBuff, + int rank, + int nRanksPerNode, + int worldSize, + size_t nelems) { + nelems = nelems / (sizeof(int) / sizeof(TYPE)); + // This version of allreduce only works for single nodes + const int nPeersInNode = nRanksPerNode - 1; + const int nPkts = nelems / 2; + const int nelemsPerLocalRank = nelems / nRanksPerNode; + const int nPktsPerLocalRank = nelemsPerLocalRank / 2; + const int localRankId = rank % nRanksPerNode; + // flag for packets. Initially 1 + const uint32_t flag = (uint32_t)globalFlag; + // thread block & channel info + const int nBlocksPerPeer = gridDim.x / nPeersInNode; + const int localBlockIdx = blockIdx.x % nBlocksPerPeer; + const int peerIdx = blockIdx.x / nBlocksPerPeer; + const int remoteRankIdx = peerIdx < localRankId ? peerIdx : peerIdx + 1; + mscclpp::MemoryChannelDeviceHandle memChan = memChans[peerIdx]; + mscclpp::PortChannelDeviceHandle portChan = portChans[localRankId]; + const int tid = threadIdx.x + localBlockIdx * blockDim.x; + // double buffering + size_t scratchBaseOffset = (flag & 1) ? 0 : nPkts * sizeof(mscclpp::LLPacket); + size_t putBaseOffset = (flag & 1) ? 0 : nPktsPerLocalRank * sizeof(mscclpp::LLPacket); + void* scratchBuff = (void*)((char*)scratch + scratchBaseOffset); + size_t scratchOffset = scratchBaseOffset + localRankId * nPktsPerLocalRank * sizeof(mscclpp::LLPacket); + size_t scratchResultOffset = + (flag & 1) ? 2 * nPkts * sizeof(mscclpp::LLPacket) : 3 * nPkts * sizeof(mscclpp::LLPacket); + size_t srcOffset = remoteRankIdx * nelemsPerLocalRank * sizeof(int); + uint2* src = (uint2*)((char*)buff + localRankId * nelemsPerLocalRank * sizeof(int)); + uint2* dst = (uint2*)((char*)resultBuff + localRankId * nelemsPerLocalRank * sizeof(int)); + + // step 1: write to scratch buffer + if (nRanksPerNode > 1) { + memChan.putPackets( + scratchOffset, srcOffset, nelemsPerLocalRank * sizeof(int), tid, blockDim.x * nBlocksPerPeer, flag); + } + // step 2: get data from scratch buffer, do local reduce-scatter in each node. + mscclpp::LLPacket* putPkt = (mscclpp::LLPacket*)((char*)putBuff + putBaseOffset); + for (int idx = threadIdx.x + blockIdx.x * blockDim.x; idx < nPktsPerLocalRank; idx += blockDim.x * gridDim.x) { + uint2 data = make_uint2(0, 0); + for (int index = 0; index < nPeersInNode; index++) { + const int remoteRank = index < localRankId ? index : index + 1; + mscclpp::LLPacket* dstPkt = (mscclpp::LLPacket*)scratchBuff + remoteRank * nPktsPerLocalRank; + uint2 val = dstPkt[idx].read(flag); + data = add_vectors(val, data); + } + data = add_vectors(data, src[idx]); + putPkt[idx].write(data.x, data.y, flag); + dst[idx] = data; + } + deviceSyncer.sync(gridDim.x); + // step 3. send local reduced data to remote node. + if (threadIdx.x == 0 && blockIdx.x == 0) { + portChan.put(scratchOffset, putBaseOffset, nPktsPerLocalRank * sizeof(mscclpp::LLPacket)); + if ((flag & 63) == 0) { + portChan.flush(); + } + } + // step 4. try to read the data from scratch buffer and write to local peers + mscclpp::LLPacket* dstPkt = (mscclpp::LLPacket*)scratchBuff + localRankId * nPktsPerLocalRank; + for (int idx = threadIdx.x + blockIdx.x * blockDim.x; idx < nPktsPerLocalRank; idx += blockDim.x * gridDim.x) { + uint2 res = dst[idx]; + uint2 val = dstPkt[idx].read(flag); + res = add_vectors(res, val); + + mscclpp::LLPacket packet; + packet.data1 = res.x; + packet.flag1 = flag; + packet.data2 = res.y; + packet.flag2 = flag; + size_t offset = scratchResultOffset / sizeof(mscclpp::LLPacket) + (idx + localRankId * nPktsPerLocalRank); + for (int index = 0; index < nPeersInNode; index++) { + memChans[index].write(offset, packet); + } + dst[idx] = res; + } + + // step 5: get data result from scratch buffer + dstPkt = (mscclpp::LLPacket*)((char*)scratch + scratchResultOffset); + const int dstOffset = remoteRankIdx * nPktsPerLocalRank; + uint2* result = (uint2*)((char*)resultBuff + remoteRankIdx * nelemsPerLocalRank * sizeof(int)); + if (nRanksPerNode > 1) { + for (int idx = threadIdx.x + localBlockIdx * blockDim.x; idx < nPktsPerLocalRank; + idx += blockDim.x * nBlocksPerPeer) { + uint2 data = dstPkt[idx + dstOffset].read(flag); + result[idx] = data; + } + } + if (threadIdx.x == 0 && blockIdx.x == 0) { + globalFlag += 1; + } +} + +static const mscclpp::Transport IBs[] = { + mscclpp::Transport::IB0, + mscclpp::Transport::IB1, + mscclpp::Transport::IB2, + mscclpp::Transport::IB3, + mscclpp::Transport::IB4, + mscclpp::Transport::IB5, + mscclpp::Transport::IB6, + mscclpp::Transport::IB7}; + +class MscclCommGroup { + public: + std::shared_ptr comm_; + const size_t rank_; + const size_t world_size_; + const std::vector rank_to_node_; + const std::vector rank_to_ib_; + MscclCommGroup( + mscclpp::UniqueId unique_id, + const size_t rank, + const size_t world_size, + const std::vector& rank_to_node, + const std::vector& rank_to_ib) + : rank_(rank), world_size_(world_size), rank_to_node_(rank_to_node), rank_to_ib_(rank_to_ib) { + auto bootstrap = std::make_shared(rank, world_size); + bootstrap->initialize(unique_id); + comm_ = std::make_shared(bootstrap); + } + template + void allreduce(cudaStream_t stream, T* output, size_t input_numel, int threads = 512, int block_limit = 21) { + throw std::runtime_error("you should not call allreduce of a base context"); + } + bool is_same_node(int r1, int r2) { + return rank_to_node_[r1] == rank_to_node_[r2]; + } + + void make_connection( + std::unordered_map>& same_node_connections, + std::unordered_map>& cross_node_connections) { + same_node_connections.clear(); + cross_node_connections.clear(); + std::unordered_map>> conn_futures; + for (int r = 0; r < world_size_; ++r) { + if (r == rank_) continue; + mscclpp::Transport transport = is_same_node(r, rank_) ? mscclpp::Transport::CudaIpc : IBs[rank_to_ib_[r]]; + conn_futures.emplace(r, comm_->connectOnSetup(r, 0, transport)); + } + comm_->setup(); + for (int r = 0; r < world_size_; ++r) { + if (r == rank_) continue; + if (is_same_node(r, rank_)) { + same_node_connections.emplace(r, conn_futures[r].get()); + } else { + cross_node_connections.emplace(r, conn_futures[r].get()); + } + } + } + + void make_memory_channels_with_scratch( + void* tensor_ptr, + const size_t tensor_bytes, + void* scratch_ptr, + const size_t scratch_bytes, + const std::unordered_map>& connections, + std::unordered_map>& semaphores, + std::unordered_map& registered_memories, + std::unordered_map& channels) { + channels.clear(); + make_semaphores(connections, semaphores); + register_tensor_with_connections(scratch_ptr, scratch_bytes, connections, registered_memories); + for (const auto& [peer, _] : connections) { + channels.emplace( + peer, mscclpp::MemoryChannel(semaphores[peer], registered_memories[peer], tensor_ptr, scratch_ptr)); + } + } + void make_port_channels_with_scratch( + std::shared_ptr proxyService, + void* tensor_ptr, + const size_t tensor_bytes, + void* scratch_ptr, + const size_t scratch_bytes, + const std::unordered_map>& connections, + std::unordered_map>& semaphores, + std::unordered_map& registered_memories, + std::unordered_map& channels) { + channels.clear(); + make_semaphores(connections, semaphores); + + mscclpp::TransportFlags flags; + for (const auto& [_, conn] : connections) { + flags |= conn->transport(); + } + auto local_reg_memory = comm_->registerMemory(tensor_ptr, tensor_bytes, flags); + + register_tensor_with_connections(scratch_ptr, scratch_bytes, connections, registered_memories); + std::unordered_map semaphore_ids; + std::unordered_map memory_ids; + memory_ids[rank_] = proxyService->addMemory(local_reg_memory); + for (const auto& [peer, memory] : registered_memories) { + if (peer == rank_) continue; + memory_ids[peer] = proxyService->addMemory(memory); + } + for (const auto& [peer, semaphore] : semaphores) { + semaphore_ids[peer] = proxyService->addSemaphore(semaphore); + } + + for (const auto& [peer, _] : connections) { + channels.emplace(peer, proxyService->portChannel(semaphore_ids[peer], memory_ids[peer], memory_ids[rank_])); + } + } + + template + void make_semaphores( + const std::unordered_map>& connections, + std::unordered_map>& semaphores) { + semaphores.clear(); + for (const auto& [peer, conn] : connections) { + semaphores[peer] = std::make_shared(*comm_, conn); + } + comm_->setup(); + } + + void register_tensor_with_connections( + void* tensor_ptr, + size_t tensor_bytes, + const std::unordered_map>& connections, + std::unordered_map& registered_memories) { + registered_memories.clear(); + mscclpp::TransportFlags all_transports; + for (const auto& [_, connection] : connections) { + all_transports |= connection->transport(); + } + mscclpp::RegisteredMemory buf_reg_mem = comm_->registerMemory(tensor_ptr, tensor_bytes, all_transports); + registered_memories[rank_] = buf_reg_mem; + + std::unordered_map> remote_mem_futures; + for (const auto& [r, connection] : connections) { + comm_->sendMemoryOnSetup(buf_reg_mem, r, 0); + auto remoteMemory = comm_->recvMemoryOnSetup(r, 0); + remote_mem_futures.emplace(r, remoteMemory); + } + comm_->setup(); + for (auto& [r, mem_feature] : remote_mem_futures) { + registered_memories.emplace(r, mem_feature.get()); + } + } + + void make_device_memory_handle_base_on_new_ptr( + const std::unordered_map& old_memory_channels, + std::unordered_map& registered_sm_memories, + std::unordered_map>& memory_semaphores, + std::unordered_map& memory_channels, + mscclpp::GpuBuffer& device_memory_handle, + void* input, + void* scratch, + const cudaStream_t stream) { + memory_channels.clear(); + for (const auto& [peer, channel] : old_memory_channels) { + memory_channels.emplace( + peer, mscclpp::MemoryChannel(memory_semaphores[peer], registered_sm_memories[peer], input, scratch)); + } + std::vector memory_channels_list; + for (int r = 0; r < world_size_; r++) { + if (r == rank_) continue; + if (is_same_node(r, rank_)) { + memory_channels_list.push_back(memory_channels[r]); + } + } + std::vector memory_channel_handlers(memory_channels_list.size()); + std::transform( + memory_channels_list.begin(), + memory_channels_list.end(), + memory_channel_handlers.begin(), + [](const mscclpp::MemoryChannel& channel) { return channel.deviceHandle(); }); + mscclpp::gpuMemcpyAsync( + device_memory_handle.data(), + memory_channel_handlers.data(), + memory_channel_handlers.size(), + stream, + cudaMemcpyHostToDevice); + } +}; + +class Msccl1NodeLLcontext { + private: + std::shared_ptr comm_group_ = nullptr; + void* scratch_; + const size_t scratch_bytes_; + std::unordered_map> same_node_connections_; + std::unordered_map> cross_node_connections_; + + std::unordered_map registered_sm_memories_; + std::unordered_map> memory_semaphores_; + std::unordered_map memory_channels_; + mscclpp::GpuBuffer d_memHandles_; + std::unordered_map> input_ptr2memory_channels_; + std::unordered_map> input_ptr2d_memHandles_; + cudaStream_t h2d_stream; + const size_t nranks_per_node_; + + public: + Msccl1NodeLLcontext( + mscclpp::UniqueId unique_id, + const size_t rank, + const size_t world_size, + void* scratch, + const size_t scratch_bytes, + const size_t nranks_per_node, + const std::vector& rank_to_node, + const std::vector& rank_to_ib) + : scratch_(scratch), + scratch_bytes_(scratch_bytes), + nranks_per_node_(nranks_per_node), + d_memHandles_(nranks_per_node - 1) { + CHECK_CUDA_SUCCESS(cudaStreamCreateWithFlags(&h2d_stream, cudaStreamNonBlocking)); + comm_group_ = std::make_shared(unique_id, rank, world_size, rank_to_node, rank_to_ib); + comm_group_->make_connection(same_node_connections_, cross_node_connections_); + comm_group_->make_memory_channels_with_scratch( + scratch_, + scratch_bytes_, + scratch_, + scratch_bytes_, + same_node_connections_, + memory_semaphores_, + registered_sm_memories_, + memory_channels_); + std::vector memory_channels_list; + for (int r = 0; r < comm_group_->world_size_; r++) { + if (r == comm_group_->rank_) continue; + memory_channels_list.push_back(memory_channels_[r]); + } + std::vector memory_channel_handlers(memory_channels_list.size()); + std::transform( + memory_channels_list.begin(), + memory_channels_list.end(), + memory_channel_handlers.begin(), + [](const mscclpp::MemoryChannel& channel) { return channel.deviceHandle(); }); + mscclpp::gpuMemcpy( + d_memHandles_.data(), memory_channel_handlers.data(), memory_channel_handlers.size(), cudaMemcpyHostToDevice); + } + + ~Msccl1NodeLLcontext() { + CHECK_CUDA_SUCCESS(cudaStreamDestroy(h2d_stream)); + } + + template + void allreduce(cudaStream_t stream, T* input, T* output, size_t input_numel, int nthreads = 512, int nblocks = 21) { + dim3 nthrs(nthreads); + dim3 nblks(nblocks); + cudaStreamCaptureStatus capturing_status; + CHECK_CUDA_SUCCESS(cudaStreamIsCapturing(stream, &capturing_status)); + mscclpp::MemoryChannelDeviceHandle* memChans; + if (capturing_status != cudaStreamCaptureStatusActive) { + std::unordered_map memory_channels; + comm_group_->make_device_memory_handle_base_on_new_ptr( + memory_channels_, + registered_sm_memories_, + memory_semaphores_, + memory_channels, + d_memHandles_, + input, + scratch_, + h2d_stream); + CHECK_CUDA_SUCCESS(cudaStreamSynchronize(h2d_stream)); + memChans = d_memHandles_.data(); + } else { + void* input_void_ptr = reinterpret_cast(input); + if (input_ptr2d_memHandles_.find(input_void_ptr) == input_ptr2d_memHandles_.end()) { + std::unordered_map memory_channels; + mscclpp::GpuBuffer device_memory_handle(comm_group_->world_size_ - 1); + comm_group_->make_device_memory_handle_base_on_new_ptr( + memory_channels_, + registered_sm_memories_, + memory_semaphores_, + memory_channels, + device_memory_handle, + input, + scratch_, + h2d_stream); + input_ptr2memory_channels_.emplace(input_void_ptr, memory_channels); + input_ptr2d_memHandles_.emplace(input_void_ptr, device_memory_handle); + } + auto it = input_ptr2d_memHandles_.find(input_void_ptr); + memChans = it->second.data(); + } + allreduce_LL_1node<<>>( + memChans, (T*)input, (T*)scratch_, output, comm_group_->rank_, comm_group_->world_size_, input_numel); + + cudaError_t status = cudaGetLastError(); + if (status != cudaSuccess) { + printf("rank: %lu failed to launch allreduce_LL_1node: %s\n", comm_group_->rank_, cudaGetErrorString(status)); + } + } +}; + +class Msccl2NodeLLcontext { + private: + std::shared_ptr comm_group_ = nullptr; + void* scratch_; + const size_t scratch_bytes_; + void* put_buffer_; + const size_t put_buffer_bytes_; + std::unordered_map> same_node_connections_; + std::unordered_map> cross_node_connections_; + + std::unordered_map registered_sm_memories_; + std::unordered_map registered_port_memories_; + + std::unordered_map> memory_semaphores_; + std::unordered_map> port_semaphores_; + + std::unordered_map memory_channels_; + std::unordered_map port_channels_; + + mscclpp::GpuBuffer d_memHandles_; + mscclpp::GpuBuffer d_portHandles_; + + std::shared_ptr proxyService; + cudaStream_t h2d_stream; + const size_t nranks_per_node_; + + std::unordered_map> input_ptr2memory_channels_; + std::unordered_map> input_ptr2d_memHandles_; + + public: + Msccl2NodeLLcontext( + mscclpp::UniqueId unique_id, + const size_t rank, + const size_t world_size, + void* scratch, + const size_t scratch_bytes, + void* put_buffer, + const size_t put_buffer_bytes, + const size_t nranks_per_node, + const std::vector& rank_to_node, + const std::vector& rank_to_ib) + : scratch_(scratch), + scratch_bytes_(scratch_bytes), + put_buffer_(put_buffer), + put_buffer_bytes_(put_buffer_bytes), + nranks_per_node_(nranks_per_node), + d_memHandles_(nranks_per_node - 1), + d_portHandles_(world_size - nranks_per_node) { + CHECK_CUDA_SUCCESS(cudaStreamCreateWithFlags(&h2d_stream, cudaStreamNonBlocking)); + comm_group_ = std::make_shared(unique_id, rank, world_size, rank_to_node, rank_to_ib); + proxyService = std::make_shared(); + proxyService->startProxy(); + comm_group_->make_connection(same_node_connections_, cross_node_connections_); + comm_group_->make_memory_channels_with_scratch( + scratch_, + scratch_bytes_, + scratch_, + scratch_bytes_, + same_node_connections_, + memory_semaphores_, + registered_sm_memories_, + memory_channels_); + comm_group_->make_port_channels_with_scratch( + proxyService, + put_buffer_, + put_buffer_bytes_, + scratch_, + scratch_bytes_, + cross_node_connections_, + port_semaphores_, + registered_port_memories_, + port_channels_); + std::vector memory_channels_list; + std::vector port_channels_list; + for (int r = 0; r < comm_group_->world_size_; r++) { + if (r == comm_group_->rank_) continue; + if (comm_group_->is_same_node(r, comm_group_->rank_)) { + memory_channels_list.push_back(memory_channels_[r]); + } else { + port_channels_list.push_back(port_channels_[r]); + } + } + std::vector memory_channel_handlers(memory_channels_list.size()); + std::transform( + memory_channels_list.begin(), + memory_channels_list.end(), + memory_channel_handlers.begin(), + [](const mscclpp::MemoryChannel& channel) { return channel.deviceHandle(); }); + mscclpp::gpuMemcpy( + d_memHandles_.data(), memory_channel_handlers.data(), memory_channel_handlers.size(), cudaMemcpyHostToDevice); + + std::vector port_channel_handlers(port_channels_list.size()); + std::transform( + port_channels_list.begin(), + port_channels_list.end(), + port_channel_handlers.begin(), + [](const mscclpp::PortChannel& channel) { return channel.deviceHandle(); }); + mscclpp::gpuMemcpy( + d_portHandles_.data(), port_channel_handlers.data(), port_channel_handlers.size(), cudaMemcpyHostToDevice); + } + + ~Msccl2NodeLLcontext() { + CHECK_CUDA_SUCCESS(cudaStreamDestroy(h2d_stream)); + if (proxyService) { + proxyService->stopProxy(); + } + } + + template + void + allreduce(cudaStream_t stream, T* input, T* output, const size_t input_numel, int nthreads = 512, int nblocks = 21) { + dim3 nthrs(nthreads); + dim3 nblks(nblocks); + cudaStreamCaptureStatus capturing_status; + CHECK_CUDA_SUCCESS(cudaStreamIsCapturing(stream, &capturing_status)); + mscclpp::MemoryChannelDeviceHandle* memChans; + if (capturing_status != cudaStreamCaptureStatusActive) { + std::unordered_map memory_channels; + comm_group_->make_device_memory_handle_base_on_new_ptr( + memory_channels_, + registered_sm_memories_, + memory_semaphores_, + memory_channels, + d_memHandles_, + input, + scratch_, + h2d_stream); + CHECK_CUDA_SUCCESS(cudaStreamSynchronize(h2d_stream)); + memChans = d_memHandles_.data(); + } else { + void* input_void_ptr = reinterpret_cast(input); + if (input_ptr2d_memHandles_.find(input_void_ptr) == input_ptr2d_memHandles_.end()) { + std::unordered_map memory_channels; + mscclpp::GpuBuffer device_memory_handle(7); + comm_group_->make_device_memory_handle_base_on_new_ptr( + memory_channels_, + registered_sm_memories_, + memory_semaphores_, + memory_channels, + device_memory_handle, + input, + scratch_, + h2d_stream); + input_ptr2memory_channels_.emplace(input_void_ptr, memory_channels); + input_ptr2d_memHandles_.emplace(input_void_ptr, device_memory_handle); + } + auto it = input_ptr2d_memHandles_.find(input_void_ptr); + memChans = it->second.data(); + } + allreduce_LL_2node<<>>( + memChans, + d_portHandles_.data(), + (T*)input, + (T*)scratch_, + (T*)put_buffer_, + output, + comm_group_->rank_, + nranks_per_node_, + comm_group_->world_size_, + input_numel); + + cudaError_t status = cudaGetLastError(); + if (status != cudaSuccess) { + printf("rank: %lu failed to launch allreduce_LL_2node: %s\n", comm_group_->rank_, cudaGetErrorString(status)); + } + } +}; + +} // namespace sglang diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/quick_all_reduce.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/quick_all_reduce.cu new file mode 100644 index 00000000..757c05d2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/quick_all_reduce.cu @@ -0,0 +1,111 @@ +#include +#include +#include +#include + +#ifdef USE_ROCM + +#include "quick_all_reduce.h" + +quickreduce::fptr_t init_custom_qr(int64_t rank, int64_t world_size, std::optional qr_max_size) { + if (world_size > 8) throw std::invalid_argument("world size > 8 is not supported"); + if (world_size == 6) throw std::invalid_argument("world size == 6 is not supported"); + if (world_size % 2 != 0) throw std::invalid_argument("Odd num gpus is not supported for now"); + if (rank < 0 || rank >= world_size) throw std::invalid_argument("invalid rank passed in"); + quickreduce::DeviceComms* fptr = new quickreduce::DeviceComms(); + fptr->init(world_size, rank, qr_max_size); + return (quickreduce::fptr_t)fptr; +} + +void qr_destroy(quickreduce::fptr_t _fa) { + if (_fa) { + auto fa = reinterpret_cast(_fa); + fa->destroy(); + delete fa; + } +} + +torch::Tensor qr_get_handle(quickreduce::fptr_t _fa) { + auto fa = reinterpret_cast(_fa); + hipIpcMemHandle_t handle = fa->get_handle(); + auto options = torch::TensorOptions().dtype(torch::kUInt8).device(torch::kCPU); + auto data_handle = torch::empty({static_cast(sizeof(hipIpcMemHandle_t))}, options); + std::memcpy(data_handle.data_ptr(), &handle, sizeof(hipIpcMemHandle_t)); + return data_handle; +} + +void qr_open_handles(quickreduce::fptr_t _fa, const std::vector& handles) { + auto fa = reinterpret_cast(_fa); + std::vector ipc_handles; + ipc_handles.reserve(handles.size()); + for (auto& handle : handles) { + // Ensure the tensor is on the same device as the current device. + hipIpcMemHandle_t ipc_handle; + std::memcpy(&ipc_handle, handle.data_ptr(), sizeof(hipIpcMemHandle_t)); + ipc_handles.push_back(ipc_handle); + } + fa->open_ipc_handles(ipc_handles); +} + +void qr_all_reduce( + quickreduce::fptr_t _fa, torch::Tensor& inp, torch::Tensor& out, int64_t quant_level, bool cast_bf2half) { + auto fa = reinterpret_cast(_fa); + const at::cuda::OptionalCUDAGuard device_guard(device_of(inp)); + auto stream = at::cuda::getCurrentHIPStreamMasqueradingAsCUDA(); + + TORCH_CHECK_EQ(inp.scalar_type(), out.scalar_type()); + TORCH_CHECK_EQ(inp.numel(), out.numel()); + TORCH_CHECK_LE(out.numel(), fa->kMaxProblemSize); + if (out.scalar_type() == at::ScalarType::Half) { + fa->allreduce( + reinterpret_cast(inp.data_ptr()), + reinterpret_cast(out.data_ptr()), + out.numel(), + quant_level, + stream); + } else if (out.scalar_type() == at::ScalarType::BFloat16) { + if (cast_bf2half) { + fa->allreduce( + reinterpret_cast(inp.data_ptr()), + reinterpret_cast(out.data_ptr()), + out.numel(), + quant_level, + stream); + } else { + fa->allreduce( + reinterpret_cast(inp.data_ptr()), + reinterpret_cast(out.data_ptr()), + out.numel(), + quant_level, + stream); + } + } else { + throw std::runtime_error("quick allreduce only supports float16 and bfloat16"); + } +} + +int64_t qr_max_size() { + // The default is 2GB (2,147,483,648 bytes) + return static_cast(std::numeric_limits::max()) + 1; +} + +#define INSTANTIATE_FOR_WORLDSIZE(T, Codec, cast_bf2half) \ + template struct quickreduce::AllReduceTwoshot, cast_bf2half>; \ + template struct quickreduce::AllReduceTwoshot, cast_bf2half>; \ + template struct quickreduce::AllReduceTwoshot, cast_bf2half>; + +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecFP, false) +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ4, false) +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ6, false) +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ8, false) +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecFP, true) +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ4, true) +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ6, true) +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ8, true) + +INSTANTIATE_FOR_WORLDSIZE(half, quickreduce::CodecFP, false) +INSTANTIATE_FOR_WORLDSIZE(half, quickreduce::CodecQ4, false) +INSTANTIATE_FOR_WORLDSIZE(half, quickreduce::CodecQ6, false) +INSTANTIATE_FOR_WORLDSIZE(half, quickreduce::CodecQ8, false) + +#endif // USE_ROCM diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/quick_all_reduce.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/quick_all_reduce.cuh new file mode 100644 index 00000000..4032e666 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/quick_all_reduce.cuh @@ -0,0 +1,634 @@ +#pragma once + +#include + +#include "quick_all_reduce_base.h" + +namespace quickreduce { + +struct CodecBase { + const int thread; + const int rank; + const int group_leader; + __quickreduce_device_inline__ CodecBase(int thread, int rank) + : thread(thread), rank(rank), group_leader((threadIdx.x / kThreadGroupSize) * kThreadGroupSize) { + set_fp16_ovfl(true); + } +}; + +// Default full precision codec. +template +struct CodecFP : public CodecBase { + static constexpr int kWorldSize = world_size; + static constexpr int kRankAtoms = kAtoms / kWorldSize; + + // Codec tile size process by this workgroup. + // Each thread processes atoms of f16x8_t (16B). + static constexpr int kRankTransmittedTileSize = kBlockSize * kRankAtoms * sizeof(int32x4_t); + static_assert(kRankTransmittedTileSize % 16 == 0, "kRankTransmittedTileSize must be 16B aligned."); + + // Total tile size for the collective communication. + static constexpr int kTransmittedTileSize = kRankTransmittedTileSize * kWorldSize; + + __quickreduce_device_inline__ CodecFP(int thread, int rank) : CodecBase(thread, rank) {} + + __quickreduce_device_inline__ void send(int32x4_t* __restrict__ send_buffer, const int32x4_t* __restrict__ data) { + for (int i = 0; i < kRankAtoms; i++) { + __builtin_nontemporal_store(data[i], send_buffer + thread); + send_buffer += kAtomStride; + } + } + + __quickreduce_device_inline__ void recv(int32x4_t** __restrict__ recv_buffer, int32x4_t* __restrict__ data) { + for (int i = 0; i < kRankAtoms; i++) { + data[i] = __builtin_nontemporal_load(*recv_buffer + thread); + *recv_buffer += kAtomStride; + } + } +}; + +// Int4 symmetric quantization codec. +// We quantize the FP16 data to block-scaled Int4 in blocks of 4 * +// kThreadGroupSize. +template +struct CodecQ4 : public CodecBase { + static constexpr int kWorldSize = world_size; + + // Codec tile size process by this workgroup. + // Each threads processes a fragment of fp16x8_t (16B), + // into a int4x8_t (4B) and a fp16 scale shared among 32 values. + static constexpr int kRankAtoms = kAtoms / kWorldSize; + static constexpr int kRankTileStride = 1152; + static constexpr int kRankTileScaleOffset = 1024; + static constexpr int kRankTransmittedTileSize = kRankTileStride * kRankAtoms; + static_assert(kRankTransmittedTileSize % 16 == 0, "kRankTransmittedTileSize must be 16B aligned."); + + static constexpr int kRankBufferTileStride = kRankTileStride / sizeof(int32x4_t); + + // Total tile size for the collective communication. + static constexpr int kTransmittedTileSize = kRankTransmittedTileSize * kWorldSize; + + // Constants configuration + + // {-1/8.0h, -1/8.0h}, f16x2_t + static constexpr int kScaleFactor = std::is_same::value ? 0xB000B000 : 0xBE00BE00; + + // {1e-7, 1e-7}, f16x2_t + static constexpr int kScaleEpsilon = std::is_same::value ? 0x00010001 : 0x33D733D7; + + // {-8, -8}, f16x2_t + static constexpr int kRangeMin = std::is_same::value ? 0xC800C800 : 0xC100C100; + + // {+7, +7}, f16x2_t + static constexpr int kRangeMax = std::is_same::value ? 0x47004700 : 0x40E040E0; + + // {+8, +8}, int16x2_t + static constexpr int kRangeBias = 0x00080008; + + __quickreduce_device_inline__ CodecQ4(int thread, int rank) : CodecBase(thread, rank) {} + + __quickreduce_device_inline__ void send(int32x4_t* __restrict__ send_buffer, const int32x4_t* __restrict__ data) { + for (int k = 0; k < kRankAtoms; k++) { + int32x4_t const atom = data[k]; + + // Compute the absolute maximum of the atom in the thread group + // In 2 blocks of values, upper/lower halves of the f16x2_t + int wblockmax = group_abs_max(atom); + + // Derive scales + int decoding_scale; + int encoding_scale; + decoding_scale = packed_mul(wblockmax, kScaleFactor); + encoding_scale = packed_add(decoding_scale, kScaleEpsilon); + encoding_scale = packed_rcp(encoding_scale); + + // Apply scales to get quantized values + int32x4_t w; + for (int i = 0; i < 4; i++) { + w[i] = packed_mul(atom[i], encoding_scale); + w[i] = packed_max(w[i], kRangeMin); + w[i] = packed_min(w[i], kRangeMax); + } + + // Convert from f16x2_t to uint16x2_t + int32x4_t q; + { + int16_t* qi = reinterpret_cast(&q); + T* wh = reinterpret_cast(&w); + for (int i = 0; i < 8; i++) + qi[i] = (int16_t)rintf(T2float_cast(wh[i])); + + for (int i = 0; i < 4; i++) { + q[i] = packed_add(q[i], kRangeBias); + } + } + + // Pack 8 x q4 into int32_t + int qw = q[0] | (q[1] << 4) | (q[2] << 8) | (q[3] << 12); + + // Write quantized atom to send_buffer + // note: only the group leader stores the scale + uint8_t* atom_ptr = reinterpret_cast(send_buffer + k * kRankBufferTileStride); + int32_t* qw_ptr = reinterpret_cast(atom_ptr) + thread; + int* qs_ptr = reinterpret_cast(atom_ptr + kRankTileScaleOffset) + (thread / 8); + + __builtin_nontemporal_store(qw, qw_ptr); + if (threadIdx.x == group_leader) { + __builtin_nontemporal_store(decoding_scale, qs_ptr); + } + } + } + + __quickreduce_device_inline__ void recv(int32x4_t** __restrict__ recv_buffer, int32x4_t* __restrict__ data) { + for (int k = 0; k < kRankAtoms; k++) { + // Directly read quantized atom from recv_buffer + uint8_t* atom_ptr = reinterpret_cast(*recv_buffer); + int32_t* qw_ptr = reinterpret_cast(atom_ptr) + thread; + int* qs_ptr = reinterpret_cast(atom_ptr + kRankTileScaleOffset) + (thread / 8); + + int32_t qw = __builtin_nontemporal_load(qw_ptr); + int qs = __builtin_nontemporal_load(qs_ptr); + + *recv_buffer += kRankBufferTileStride; + + // Unpack q4 into f16x8_t + int32x4_t w; + { + static constexpr uint kMask000F = 0x000F000F; + static constexpr uint kHalf2_1024 = 0x64006400; // {1024.0, 1024.0}, fp16x2_t + static uint constexpr kHalf2_1032 = 0xE408E408; // {-1032.0, -1032.0}, fp16x2_t + + for (int i = 0; i < 4; i++) { + if constexpr (std::is_same::value) { + int32_t q4 = ((qw >> (i * 4)) & kMask000F) | kHalf2_1024; + w[i] = packed_add(q4, kHalf2_1032); + } else { + int32_t int16_2 = (qw >> (i * 4)) & kMask000F; + int16_t low = static_cast(int16_2 & 0xFFFF); + int16_t high = static_cast((int16_2 >> 16) & 0xFFFF); + nv_bfloat16 bf_low = __float2bfloat16(static_cast(low)); + nv_bfloat16 bf_high = __float2bfloat16(static_cast(high)); + nv_bfloat162 bf2 = __halves2bfloat162(bf_low, bf_high); + int32_t packed_bf16 = *reinterpret_cast(&bf2); + w[i] = packed_add(packed_bf16, kRangeMin); + } + } + } + + // Apply decoding scales + for (int i = 0; i < 4; i++) { + w[i] = packed_mul(w[i], qs); + } + + data[k] = w; + } + } +}; + +// Int6 symmetric quantization codec. +// We quantize the FP16 data to block-scaled Int6 in blocks of 4 * +// kThreadGroupSize. +template +struct CodecQ6 : public CodecBase { + static constexpr int kWorldSize = world_size; + + // Codec tile size process by this workgroup. + // Each threads processes a fragment of fp16x8_t (16B), + // into a int6x8_t (4B + 2B) and a fp16 scale shared among 32 values. + static constexpr int kRankAtoms = kAtoms / kWorldSize; + static constexpr int kRankTileStride = 1664; + static constexpr int kRankTileQ2Offset = 1024; + static constexpr int kRankTileScaleOffset = 1536; + static constexpr int kRankTransmittedTileSize = kRankTileStride * kRankAtoms; + static_assert(kRankTransmittedTileSize % 16 == 0, "kRankTransmittedTileSize must be 16B aligned."); + + static constexpr int kRankBufferTileStride = kRankTileStride / sizeof(int32x4_t); + + // Total tile size for the collective communication. + static constexpr int kTransmittedTileSize = kRankTransmittedTileSize * kWorldSize; + + // Constants configuration + + // {-1/32.0h, -1/32.0h}, fp16x2_t + static constexpr int kScaleFactor = std::is_same::value ? 0xA800A800 : 0xBD00BD00; + + // {1e-7, 1e-7}, fp16x2_t + static constexpr int kScaleEpsilon = std::is_same::value ? 0x00010001 : 0x33D733D7; + + // {-32, -32}, fp16x2_t + static constexpr int kRangeMin = std::is_same::value ? 0xD000D000 : 0xC200C200; + + // {+31, +31}, fp16x2_t + static constexpr int kRangeMax = std::is_same::value ? 0x4FC04FC0 : 0x41F841F8; + + // {+32, +32}, int16x2_t + static constexpr int kRangeBias = 0x00200020; + + __quickreduce_device_inline__ CodecQ6(int thread, int rank) : CodecBase(thread, rank) {} + + __quickreduce_device_inline__ void send(int32x4_t* __restrict__ send_buffer, const int32x4_t* __restrict__ data) { + for (int k = 0; k < kRankAtoms; k++) { + int32x4_t const atom = data[k]; + + // Compute the absolute maximum of the atom in the thread group + // In 2 blocks of values, upper/lower halves of the f16x2_t + int wblockmax = group_abs_max(atom); + + // Derive scales + int decoding_scale; + int encoding_scale; + decoding_scale = packed_mul(wblockmax, kScaleFactor); + encoding_scale = packed_add(decoding_scale, kScaleEpsilon); + encoding_scale = packed_rcp(encoding_scale); + + // Apply scales to get quantized values + int32x4_t w; + for (int i = 0; i < 4; i++) { + w[i] = packed_mul(atom[i], encoding_scale); + w[i] = packed_max(w[i], kRangeMin); + w[i] = packed_min(w[i], kRangeMax); + } + + // Convert from f16x2_t to uint16x2_t + int32x4_t q; + { + int16_t* qi = reinterpret_cast(&q); + T* wh = reinterpret_cast(&w); + for (int i = 0; i < 8; i++) + qi[i] = (int16_t)rintf(T2float_cast(wh[i])); + + for (int i = 0; i < 4; i++) { + q[i] = packed_add(q[i], kRangeBias); + } + } + + // Pack 8 x q6 into int32_t + int16_t + uint32_t q4w; + uint16_t q2w = 0; + q4w = (q[0] & 0x000F000F) | ((q[1] & 0x000F000F) << 4) | ((q[2] & 0x000F000F) << 8) | ((q[3] & 0x000F000F) << 12); + { + int16_t* tw = reinterpret_cast(&q); +#pragma unroll + for (int i = 0; i < 8; i++) { + q2w |= (tw[i] >> 4) << (i * 2); + } + } + // Write quantized atom to send_buffer + // note: only the group leader stores the scale + uint8_t* atom_ptr = reinterpret_cast(send_buffer + k * kRankBufferTileStride); + uint32_t* q4w_ptr = reinterpret_cast(atom_ptr) + thread; + uint16_t* q2w_ptr = reinterpret_cast(atom_ptr + kRankTileQ2Offset) + thread; + int* qs_ptr = reinterpret_cast(atom_ptr + kRankTileScaleOffset) + (thread / 8); + + __builtin_nontemporal_store(q4w, q4w_ptr); + __builtin_nontemporal_store(q2w, q2w_ptr); + if (threadIdx.x == group_leader) { + __builtin_nontemporal_store(decoding_scale, qs_ptr); + } + } + } + + __quickreduce_device_inline__ void recv(int32x4_t** __restrict__ recv_buffer, int32x4_t* __restrict__ data) { + for (int k = 0; k < kRankAtoms; k++) { + // Directly read quantized atom from recv_buffer + uint8_t* atom_ptr = reinterpret_cast(*recv_buffer); + uint32_t* q4w_ptr = reinterpret_cast(atom_ptr) + thread; + uint16_t* q2w_ptr = reinterpret_cast(atom_ptr + kRankTileQ2Offset) + thread; + int* qs_ptr = reinterpret_cast(atom_ptr + kRankTileScaleOffset) + (thread / 8); + + uint32_t q4w = __builtin_nontemporal_load(q4w_ptr); + uint16_t q2w = __builtin_nontemporal_load(q2w_ptr); + int qs = __builtin_nontemporal_load(qs_ptr); + + *recv_buffer += kRankBufferTileStride; + + // Unpack q6 into fp16x8_t + int32x4_t w; + { + static uint constexpr kMask000F = 0x000F000F; + static uint constexpr kHalf2_1024 = 0x64006400; // {1024.0, 1024.0}, fp16x2_t + static uint constexpr kHalf2_1056 = 0xE420E420; // {-1056.0, -1056.0}, fp16x2_t + +#pragma unroll + for (int i = 0; i < 4; i++) { + int32_t q4 = q4w & kMask000F; + int32_t q2 = (q2w & 0x3) | ((q2w & 0xC) << 14); + q4w >>= 4; + q2w >>= 4; + if constexpr (std::is_same::value) { + int32_t q6 = q4 | (q2 << 4) | kHalf2_1024; + asm volatile("v_pk_add_f16 %0, %1, %2" : "=v"(w[i]) : "v"(q6), "v"(kHalf2_1056)); + } else { + int32_t int16_2 = q4 | (q2 << 4); + int16_t low = static_cast(int16_2 & 0xFFFF); + int16_t high = static_cast((int16_2 >> 16) & 0xFFFF); + + nv_bfloat16 bf_low = __float2bfloat16(static_cast(low)); + nv_bfloat16 bf_high = __float2bfloat16(static_cast(high)); + nv_bfloat162 bf2 = __halves2bfloat162(bf_low, bf_high); + int32_t packed_bf16 = *reinterpret_cast(&bf2); + w[i] = packed_add(packed_bf16, kRangeMin); + } + } + } + + // Apply decoding scales + for (int i = 0; i < 4; i++) { + w[i] = packed_mul(w[i], qs); + } + + // That's pretty much it... + data[k] = w; + } + } +}; + +// Int8 symmetric quantization codec. +// We quantize the FP16 data to block-scaled Int8 in blocks of 4 * +// kThreadGroupSize. +template +struct CodecQ8 : public CodecBase { + static constexpr int kWorldSize = world_size; + + // Codec tile size process by this workgroup. + // Each threads processes a fragment of f16x8_t (16B), + // into a int8x8_t (8B) and a f16 scale shared among 32 values. + static constexpr int kRankAtoms = kAtoms / kWorldSize; + static constexpr int kRankTileStride = 2176; + static constexpr int kRankTileScaleOffset = 2048; + static constexpr int kRankTransmittedTileSize = kRankTileStride * kRankAtoms; + static_assert(kRankTransmittedTileSize % 16 == 0, "kRankTileSize must be 16B aligned."); + + static constexpr int kRankBufferTileStride = kRankTileStride / sizeof(int32x4_t); + + // Total tile size for the collective communication. + static constexpr int kTransmittedTileSize = kRankTransmittedTileSize * kWorldSize; + + // Constants configuration + + // {-1/128.0h, -1/128.0h}, f16x2_t + static constexpr int kScaleFactor = std::is_same::value ? 0xA000A000 : 0xBC00BC00; + + // {1e-7, 1e-7}, f16x2_t + static constexpr int kScaleEpsilon = std::is_same::value ? 0x00010001 : 0x33D733D7; + + // {-128, -128}, f16x2_t + static constexpr int kRangeMin = std::is_same::value ? 0xD800D800 : 0xC300C300; + // {+127, +127}, f16x2_t + static constexpr int kRangeMax = std::is_same::value ? 0x57F057F0 : 0x42FE42FE; + + // {+128, +128}, int16x2_t + static constexpr int kRangeBias = 0x00800080; + + __quickreduce_device_inline__ CodecQ8(int thread, int rank) : CodecBase(thread, rank) {} + + __quickreduce_device_inline__ void send(int32x4_t* __restrict__ send_buffer, int32x4_t const* __restrict__ data) { + for (int k = 0; k < kRankAtoms; k++) { + int32x4_t const atom = data[k]; + // Compute the absolute maximum of the atom in the thread group + // In 2 blocks of values, upper/lower halves of the f16x2_t + int wblockmax = group_abs_max(atom); + + // Derive scales + int decoding_scale; + int encoding_scale; + decoding_scale = packed_mul(wblockmax, kScaleFactor); + encoding_scale = packed_add(decoding_scale, kScaleEpsilon); + encoding_scale = packed_rcp(encoding_scale); + + // Apply scales to get quantized values + int32x4_t w; + for (int i = 0; i < 4; i++) { + w[i] = packed_mul(atom[i], encoding_scale); + w[i] = packed_max(w[i], kRangeMin); + w[i] = packed_min(w[i], kRangeMax); + } + + // Convert from f16x2_t to uint16x2_t + int32x4_t q; + { + int16_t* qi = reinterpret_cast(&q); + T* wh = reinterpret_cast(&w); + for (int i = 0; i < 8; i++) + qi[i] = (int16_t)rintf(T2float_cast(wh[i])); + + for (int i = 0; i < 4; i++) { + q[i] = packed_add(q[i], kRangeBias); + } + } + + // Pack 8 x q8 into int32x2_t + int32x2_t qw; + qw[0] = q[0] | (q[1] << 8); + qw[1] = q[2] | (q[3] << 8); + + // Write quantized atom to send_buffer + // note: only the group leader stores the scale + uint8_t* atom_ptr = reinterpret_cast(send_buffer + k * kRankBufferTileStride); + int32x2_t* qw_ptr = reinterpret_cast(atom_ptr) + thread; + int* qs_ptr = reinterpret_cast(atom_ptr + kRankTileScaleOffset) + (thread / 8); + + __builtin_nontemporal_store(qw, qw_ptr); + if (threadIdx.x == group_leader) { + __builtin_nontemporal_store(decoding_scale, qs_ptr); + } + } + } + + __quickreduce_device_inline__ void recv(int32x4_t** __restrict__ recv_buffer, int32x4_t* __restrict__ data) { + for (int k = 0; k < kRankAtoms; k++) { + // Directly read quantized atom from recv_buffer + uint8_t* atom_ptr = reinterpret_cast(*recv_buffer); + int32x2_t* qw_ptr = reinterpret_cast(atom_ptr) + thread; + int* qs_ptr = reinterpret_cast(atom_ptr + kRankTileScaleOffset) + (thread / 8); + + int32x2_t qw = __builtin_nontemporal_load(qw_ptr); + int qs = __builtin_nontemporal_load(qs_ptr); + + *recv_buffer += kRankBufferTileStride; + + // Unpack q8 into fp16x8_t + int32x4_t w; + { + static uint constexpr kMask00FF = 0x00FF00FF; + + // {1024.0, 1024.0}, fp16x2_t + static uint constexpr kHalf2_1024 = 0x64006400; + + // {-1152.0, -1152.0}, fp16x2_t + static uint constexpr kHalf2_1152 = 0xE480E480; + +#pragma unroll + for (int i = 0; i < 4; i++) { + if constexpr (std::is_same::value) { + int32_t q8 = ((qw[i / 2] >> ((i % 2) * 8)) & kMask00FF) | kHalf2_1024; + w[i] = packed_add(q8, kHalf2_1152); + } else { + int32_t int16_2 = (qw[i / 2] >> ((i % 2) * 8)) & kMask00FF; + int16_t low = static_cast(int16_2 & 0xFFFF); + int16_t high = static_cast((int16_2 >> 16) & 0xFFFF); + nv_bfloat16 bf_low = __float2bfloat16(static_cast(low)); + nv_bfloat16 bf_high = __float2bfloat16(static_cast(high)); + nv_bfloat162 bf2 = __halves2bfloat162(bf_low, bf_high); + int32_t packed_bf16 = *reinterpret_cast(&bf2); + w[i] = packed_add(packed_bf16, kRangeMin); + } + } + } + + // Apply decoding scales + for (int i = 0; i < 4; i++) { + w[i] = packed_mul(w[i], qs); + } + + data[k] = w; + } + } +}; + +// Twoshot All Reduce +template +struct AllReduceTwoshot { + static_assert(sizeof(T) == 2); + + static constexpr int kWorldSize = Codec::kWorldSize; + + __device__ static void + run(T const* __restrict__ input, + T* __restrict__ output, + uint32_t const N, // number of elements + int const block, // block index + int const rank, // rank index + uint8_t** __restrict__ buffer_list, // communication buffers + uint32_t const data_offset, // offset to start of the data buffer + uint32_t flag_color, + int64_t data_size_per_phase) { + // Topology + int thread = threadIdx.x + threadIdx.y * kWavefront; + uint8_t* rank_buffer = buffer_list[rank]; + Codec codec(thread, rank); + int block_id = blockIdx.x; + int grid_size = gridDim.x; + // -------------------------------------------------------- + // Read input into registers + int32x4_t tA[kAtoms]; + + BufferResource src_buffer(const_cast(input), N * sizeof(T)); + uint32_t src_offset = block * kTileSize + thread * sizeof(int32x4_t); + + for (int i = 0; i < kAtoms; i++) { + tA[i] = buffer_load_dwordx4(src_buffer.descriptor, src_offset, 0, 0); + src_offset += kAtomStride * sizeof(int32x4_t); + if constexpr (cast_bf2half) { + const nv_bfloat162* bf_buf = reinterpret_cast(&tA[i]); + half2 half_buf[4]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + float2 f = __bfloat1622float2(bf_buf[j]); + half_buf[j] = __float22half2_rn(f); + } + tA[i] = *reinterpret_cast(half_buf); + } + } + + // -------------------------------------------------------- + // Phase-1A: Write segment data into the communication buffer of the target + // rank responsible for this segment. + uint32_t comm_data0_offset = data_offset + block_id * Codec::kTransmittedTileSize; + uint32_t comm_data1_offset = data_size_per_phase + comm_data0_offset; + + uint32_t comm_flags0_offset = block_id * (kWorldSize * sizeof(uint32_t)); + uint32_t comm_flags1_offset = (data_offset / 2) + comm_flags0_offset; + + for (int r = 0; r < kWorldSize; r++) { + int32x4_t* send_buffer = + reinterpret_cast(buffer_list[r] + comm_data0_offset + rank * Codec::kRankTransmittedTileSize); + codec.send(send_buffer, &tA[r * Codec::kRankAtoms]); + } + + __syncthreads(); + if (thread < kWorldSize) { + int r = thread; + uint32_t* flag_ptr = reinterpret_cast(buffer_list[r] + comm_flags0_offset + rank * sizeof(uint32_t)); + set_sync_flag(flag_ptr, flag_color); + } + // -------------------------------------------------------- + // Phase-1B: Reduce the segment data from the communication buffers. + int32x4_t tR[Codec::kRankAtoms] = {}; + { + // Read the data from the communication buffer. + int32x4_t* recv_buffer = reinterpret_cast(rank_buffer + comm_data0_offset); + uint32_t* flag_ptr = reinterpret_cast(rank_buffer + comm_flags0_offset); + + for (int r = 0; r < kWorldSize; r++) { + // Wait for the flags to be set. + if (thread == 0) { + wait_sync_flag(&flag_ptr[r], flag_color); + } + __syncthreads(); + + // note: we reuse tA as temp buffer here + codec.recv(&recv_buffer, tA); + + for (int i = 0; i < Codec::kRankAtoms; i++) { + packed_assign_add(&tR[i], &tA[i]); + } + } + } + + // Phase-2: Write the reduced segment to every other rank + for (int r = 0; r < kWorldSize; r++) { + int32x4_t* send_buffer = + reinterpret_cast(buffer_list[r] + comm_data1_offset + rank * Codec::kRankTransmittedTileSize); + codec.send(send_buffer, tR); + } + + __syncthreads(); + if (thread < kWorldSize) { + int r = thread; + uint32_t* flag_ptr = reinterpret_cast(buffer_list[r] + comm_flags1_offset + rank * sizeof(uint32_t)); + set_sync_flag(flag_ptr, flag_color); + } + + // Phase-2: Read the gather segments from the rank's communication buffer. + { + // Read the data from the communication buffer. + int32x4_t* recv_buffer = reinterpret_cast(rank_buffer + comm_data1_offset); + uint32_t* flag_ptr = reinterpret_cast(rank_buffer + comm_flags1_offset); + + for (int r = 0; r < kWorldSize; r++) { + // Wait for the flags to be set. + if (thread == 0) { + wait_sync_flag(&flag_ptr[r], flag_color); + } + __syncthreads(); + + // Gather all reduced and final rank segments into tA. + codec.recv(&recv_buffer, &tA[r * Codec::kRankAtoms]); + } + } + + // -------------------------------------------------------- + // Write the result to output. + BufferResource dst_buffer(output, N * sizeof(T)); + uint32_t dst_offset = block * kTileSize + thread * sizeof(int32x4_t); + + for (int i = 0; i < kAtoms; i++) { + if constexpr (cast_bf2half) { + const half2* half_buf = reinterpret_cast(&tA[i]); + nv_bfloat162 bf16_buf[4]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + float2 f = __half22float2(half_buf[j]); + bf16_buf[j] = __float22bfloat162_rn(f); + } + buffer_store_dwordx4(*reinterpret_cast(bf16_buf), dst_buffer.descriptor, dst_offset, 0, 0); + } else { + buffer_store_dwordx4(tA[i], dst_buffer.descriptor, dst_offset, 0, 0); + } + dst_offset += kAtomStride * sizeof(int32x4_t); + } + } +}; + +} // namespace quickreduce diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/quick_all_reduce.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/quick_all_reduce.h new file mode 100644 index 00000000..5cf961b8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/quick_all_reduce.h @@ -0,0 +1,237 @@ +#pragma once + +#include + +#include + +#include "quick_all_reduce.cuh" + +#define HIP_CHECK(err) \ + do { \ + hipError_t err_ = (err); \ + if (err_ != hipSuccess) { \ + std::printf("HIP error %d at %s:%d. %s\n", err_, __FILE__, __LINE__, hipGetErrorString(err_)); \ + throw std::runtime_error("HIP error"); \ + } \ + } while (0) + +namespace quickreduce { +using fptr_t = int64_t; +static_assert(sizeof(void*) == sizeof(fptr_t)); + +template +__global__ __quickreduce_launch_bounds_two_shot__ static void allreduce_prototype_twoshot( + T const* A, + T* B, + uint32_t N, + uint32_t num_blocks, + int rank, + uint8_t** dbuffer_list, + uint32_t data_offset, + uint32_t flag_color, + int64_t data_size_per_phase) { + int block = blockIdx.x; + int grid = gridDim.x; + + while (block < num_blocks) { + AllReduceKernel::run(A, B, N, block, rank, dbuffer_list, data_offset, flag_color, data_size_per_phase); + block += grid; + flag_color++; + } +} + +#define TWOSHOT_DISPATCH(__codec) \ + if (world_size == 2) { \ + using LineCodec = __codec; \ + using AllReduceKernel = AllReduceTwoshot; \ + hipLaunchKernelGGL( \ + (allreduce_prototype_twoshot), \ + dim3(grid), \ + dim3(kBlockTwoShot), \ + 0, \ + stream, \ + A, \ + B, \ + N, \ + num_blocks, \ + rank, \ + dbuffer_list, \ + data_offset, \ + flag_color, \ + this->kMaxProblemSize); \ + } else if (world_size == 4) { \ + using LineCodec = __codec; \ + using AllReduceKernel = AllReduceTwoshot; \ + hipLaunchKernelGGL( \ + (allreduce_prototype_twoshot), \ + dim3(grid), \ + dim3(kBlockTwoShot), \ + 0, \ + stream, \ + A, \ + B, \ + N, \ + num_blocks, \ + rank, \ + dbuffer_list, \ + data_offset, \ + flag_color, \ + this->kMaxProblemSize); \ + } else if (world_size == 8) { \ + using LineCodec = __codec; \ + using AllReduceKernel = AllReduceTwoshot; \ + hipLaunchKernelGGL( \ + (allreduce_prototype_twoshot), \ + dim3(grid), \ + dim3(kBlockTwoShot), \ + 0, \ + stream, \ + A, \ + B, \ + N, \ + num_blocks, \ + rank, \ + dbuffer_list, \ + data_offset, \ + flag_color, \ + this->kMaxProblemSize); \ + } + +enum QuickReduceQuantLevel { + F16 = 0, + INT8 = 1, + INT6 = 2, + INT4 = 3, +}; + +struct DeviceComms { + // Max problem size is 2GB (in bytes) or half of uint32_t max value. + int64_t kMaxProblemSize = static_cast(std::numeric_limits::max()) + 1; + + // Max TP-8 + static int constexpr kMaxWorldSize = 8; + + bool initialized = false; + uint32_t flag_color = 1; + int world_size; + int rank; + + uint8_t* dbuffer; + uint8_t** dbuffer_list; + hipIpcMemHandle_t buffer_ipc_handle; + std::vector all_buffer_ipc_handles; + std::vector buffer_list; + uint32_t data_offset; + + DeviceComms() : initialized(false), world_size(1), rank(0) {} + ~DeviceComms() { + destroy(); + } + + void init(int world_size, int rank, std::optional max_problem_size = std::nullopt) { + destroy(); + this->world_size = world_size; + this->rank = rank; + if (max_problem_size.has_value() && max_problem_size.value() > 0) { + this->kMaxProblemSize = max_problem_size.value(); + } + // Allocate buffer size for worst case: F16 2-stage buffer. + uint32_t flags_buffer_size = 2 * world_size * kMaxNumBlocks * sizeof(uint32_t); + static int64_t data_buffer_size = 2 * this->kMaxProblemSize; + int64_t total_buffer_size = flags_buffer_size + data_buffer_size; + data_offset = flags_buffer_size; + HIP_CHECK(hipExtMallocWithFlags((void**)&dbuffer, total_buffer_size, hipDeviceMallocUncached)); + + // Clear the flags buffer. + HIP_CHECK(hipMemset(dbuffer, 0, flags_buffer_size)); + + // Device-side list of IPC buffers. + buffer_list.resize(world_size); + HIP_CHECK(hipMalloc(&dbuffer_list, world_size * sizeof(uint8_t*))); + + // Create IPC handles for rank's communication buffer. + all_buffer_ipc_handles.resize(world_size); + HIP_CHECK(hipIpcGetMemHandle(&buffer_ipc_handle, dbuffer)); + + initialized = true; + } + int get_world_size() { + return world_size; + } + int get_rank() { + return rank; + } + bool status() { + return initialized; + } + hipIpcMemHandle_t const get_handle() { + return buffer_ipc_handle; + } + + void destroy() { + if (initialized) { + for (int i = 0; i < world_size; i++) { + if (i != rank) { + HIP_CHECK(hipIpcCloseMemHandle(dbuffer_list[i])); + } + } + + HIP_CHECK(hipFree(dbuffer)); + HIP_CHECK(hipFree(dbuffer_list)); + + initialized = false; + } + } + + void open_ipc_handles(std::vector const& ipc_handles) { + assert(ipc_handles.size() == all_buffer_ipc_handles.size()); + for (int i = 0; i < world_size; i++) { + all_buffer_ipc_handles[i] = ipc_handles[i]; + } + + // Open device memory access to the IPC communication buffers. + // Note: For our own rank, we do not need to open a handle. + for (int i = 0; i < world_size; i++) { + if (i != rank) { + HIP_CHECK( + hipIpcOpenMemHandle((void**)&buffer_list[i], all_buffer_ipc_handles[i], hipIpcMemLazyEnablePeerAccess)); + } else { + buffer_list[i] = dbuffer; + } + } + + HIP_CHECK(hipMemcpy(dbuffer_list, buffer_list.data(), world_size * sizeof(uint8_t*), hipMemcpyHostToDevice)); + } + + template + void allreduce(T const* A, T* B, uint32_t N, int quant_level, hipStream_t stream) { + if (world_size != 2 && world_size != 4 && world_size != 8) { + throw std::runtime_error("All Reduce not supported for world_size = " + std::to_string(world_size)); + } + + // Configuration. + uint32_t msg_size = N * sizeof(T); + uint32_t num_blocks = divceil(msg_size, kTileSize); + uint32_t grid = min(kMaxNumBlocks, num_blocks); + auto quant_level_ = static_cast(quant_level); + switch (quant_level_) { + case QuickReduceQuantLevel::INT8: + TWOSHOT_DISPATCH(CodecQ8) + break; + case QuickReduceQuantLevel::INT6: + TWOSHOT_DISPATCH(CodecQ6) + break; + case QuickReduceQuantLevel::INT4: + TWOSHOT_DISPATCH(CodecQ4) + break; + default: + TWOSHOT_DISPATCH(CodecFP) + break; + } + HIP_CHECK(cudaGetLastError()); + // Rotate the flag color. + flag_color += divceil(N, grid); + } +}; + +} // namespace quickreduce diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/quick_all_reduce_base.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/quick_all_reduce_base.h new file mode 100644 index 00000000..759b28f3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/quick_all_reduce_base.h @@ -0,0 +1,318 @@ +#pragma once + +#include +#include +#include + +#include + +#define __quickreduce_device_inline__ __device__ __forceinline__ +#define __quickreduce_launch_bounds_two_shot__ __launch_bounds__(256, 4) +#define __quickreduce_launch_bounds_one_shot__ __launch_bounds__(512, 4) + +namespace quickreduce { + +typedef __hip_bfloat16 nv_bfloat16; +typedef __hip_bfloat162 nv_bfloat162; + +using int32x2_t = __attribute__((__vector_size__(2 * sizeof(int)))) int; +using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int; + +// Setup acquire-release semantics for vector memory reads (mubuf instruction) +// as per architecture. +#if defined(__gfx942__) +// CDNA3: Scope bits sc0, sc1 +#define MUBUF_ACQUIRE 16 +#define MUBUF_RELEASE 16 +#elif (defined(__gfx908__) || defined(__gfx90a__)) +// CDNA1 and CDNA2 - glc bit +#define MUBUF_ACQUIRE 1 +#define MUBUF_RELEASE 0 +#endif + +static constexpr int kNegOne = 0xBC00BC00; // {-1, -1}, fp16x2_t + +// Number of atoms (4xf16x2_t) processed by a single thread +static constexpr int kAtoms = 8; + +// We use a workgroup of 256 threads +static constexpr int kBlockSize = 256; +static constexpr int kAtomStride = kBlockSize; + +// Size and atom stride of source/destination data that the block will +// process. +// Workgroup scope = Tile = (256 threads x 8 atoms x 16B) +static constexpr int kTileSize = kBlockSize * kAtoms * sizeof(int32x4_t); + +// Max number of blocks. 304 CUs on MI300 +static constexpr int kMaxNumBlocks = 304 * 4; + +// Standard CDNA wavefront size. +static constexpr int kWavefront = 64; + +// 256 thread, 4 wavefronts. +static dim3 constexpr kBlockTwoShot = {kWavefront, kBlockSize / kWavefront, 1}; + +// Number of threads in a group for quantization +// It corresponds to 32 F16 elements in quantization block +static constexpr int kThreadGroupSize = 8; + +// Methods +__quickreduce_device_inline__ __host__ unsigned long divceil(unsigned long x, unsigned long y) { + return ((x + y - 1) / y); +} + +union BufferResource { + __quickreduce_device_inline__ constexpr BufferResource() : config(0x00020000U) {} + + __quickreduce_device_inline__ constexpr BufferResource(void* buffer_address, uint32_t buffer_size) + : address(buffer_address), range(buffer_size), config(0x00020000U) {} + + int32x4_t descriptor; + struct { + void* address; // 8B, out of which first 48b is address, and 16b is stride + // (unused) + uint32_t range; // Byte range for the buffer resource + uint32_t config; // Constant, DFMT=32b + }; +}; + +__quickreduce_device_inline__ static int32x4_t buffer_load_dwordx4( + int32x4_t srsrc, int32_t voffset, int32_t soffset, int32_t aux) __asm("llvm.amdgcn.raw.buffer.load.v4i32"); + +__quickreduce_device_inline__ static void +buffer_store_dwordx4(int32x4_t data, int32x4_t srsrc, int32_t voffset, int32_t soffset, int32_t aux) __asm( + "llvm.amdgcn.raw.buffer.store.v4i32"); + +__quickreduce_device_inline__ static void set_fp16_ovfl(bool const value) { +#if defined(__gfx942__) + if (value) { + asm volatile("s_setreg_imm32_b32 0xdc1, 1;" ::); + } else { + asm volatile("s_setreg_imm32_b32 0xdc1, 0;" ::); + } +#endif +} +union bf162_int_union { + int i; + nv_bfloat162 bf2; +}; + +template +__quickreduce_device_inline__ void packed_assign_add(int32x4_t* A, int32x4_t* B); + +template <> +__quickreduce_device_inline__ void packed_assign_add(int32x4_t* A, int32x4_t* B) { + int32x4_t& tR_fragment = A[0]; + int32x4_t& tA_fragment = B[0]; + + asm volatile("v_pk_add_f16 %0, %1, %2" : "=v"(tR_fragment[0]) : "v"(tR_fragment[0]), "v"(tA_fragment[0])); + asm volatile("v_pk_add_f16 %0, %1, %2" : "=v"(tR_fragment[1]) : "v"(tR_fragment[1]), "v"(tA_fragment[1])); + asm volatile("v_pk_add_f16 %0, %1, %2" : "=v"(tR_fragment[2]) : "v"(tR_fragment[2]), "v"(tA_fragment[2])); + asm volatile("v_pk_add_f16 %0, %1, %2" : "=v"(tR_fragment[3]) : "v"(tR_fragment[3]), "v"(tA_fragment[3])); +} + +template <> +__quickreduce_device_inline__ void packed_assign_add(int32x4_t* A, int32x4_t* B) { + nv_bfloat162* tA = reinterpret_cast(A); + nv_bfloat162* tB = reinterpret_cast(B); +#pragma unroll + for (int i = 0; i < 4; i++) { + tA[i] = __hadd2(tA[i], tB[i]); + } +} + +template +__quickreduce_device_inline__ int packed_max(int a, int b); + +template <> +__quickreduce_device_inline__ int packed_max(int a, int b) { + int result; + asm volatile("v_pk_max_f16 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); + return result; +} + +template <> +__quickreduce_device_inline__ int packed_max(int a, int b) { + bf162_int_union A, B, R; + A.i = a; + B.i = b; + R.bf2 = __hmax2(A.bf2, B.bf2); + return R.i; +} + +template +__quickreduce_device_inline__ int packed_min(int a, int b); + +template <> +__quickreduce_device_inline__ int packed_min(int a, int b) { + int result; + asm volatile("v_pk_min_f16 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); + return result; +} + +template <> +__quickreduce_device_inline__ int packed_min(int a, int b) { + bf162_int_union A, B, R; + A.i = a; + B.i = b; + R.bf2 = __hmin2(A.bf2, B.bf2); + return R.i; +} + +template +__quickreduce_device_inline__ int packed_abs_max(int a, int b); + +template <> +__quickreduce_device_inline__ int packed_abs_max(int a, int b) { + half2 wmaxh2 = __builtin_bit_cast(half2, a); + half2 wminh2 = __builtin_bit_cast(half2, b); + half2 wblockmaxh2; + + wblockmaxh2.x = __hgt(__habs(wmaxh2.x), __habs(wminh2.x)) ? wmaxh2.x : wminh2.x; + wblockmaxh2.y = __hgt(__habs(wmaxh2.y), __habs(wminh2.y)) ? wmaxh2.y : wminh2.y; + return __builtin_bit_cast(int, wblockmaxh2); +} + +template <> +__quickreduce_device_inline__ int packed_abs_max(int a, int b) { + bf162_int_union A, B, R; + A.i = a; + B.i = b; + R.bf2.x = __hgt(__habs(A.bf2.x), __habs(B.bf2.x)) ? A.bf2.x : B.bf2.x; + R.bf2.y = __hgt(__habs(A.bf2.y), __habs(B.bf2.y)) ? A.bf2.y : B.bf2.y; + return R.i; +} + +template +__quickreduce_device_inline__ int packed_add(int a, int b); + +template <> +__quickreduce_device_inline__ int packed_add(int a, int b) { + int result; + asm volatile("v_pk_add_f16 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); + return result; +} + +template <> +__quickreduce_device_inline__ int packed_add(int a, int b) { + bf162_int_union A, B, R; + A.i = a; + B.i = b; + R.bf2 = __hadd2(A.bf2, B.bf2); + return R.i; +} + +template <> +__quickreduce_device_inline__ int packed_add(int a, int b) { + int result; + asm volatile("v_pk_add_i16 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); + return result; +} + +template +__quickreduce_device_inline__ int packed_sub(int a, int b); + +template <> +__quickreduce_device_inline__ int packed_sub(int a, int b) { + int result; + + // MI300 lacks packed fp16 sub instruction. So we do -1 * min + max + asm volatile("v_pk_fma_f16 %0, %1, %2 %3" : "=v"(result) : "v"(kNegOne), "v"(b), "v"(a)); + return result; +} + +template <> +__quickreduce_device_inline__ int packed_sub(int a, int b) { + bf162_int_union A, B, R; + A.i = a; + B.i = b; + R.bf2 = __hsub2(A.bf2, B.bf2); + return R.i; +} + +template +__quickreduce_device_inline__ int packed_mul(int a, int b); + +template <> +__quickreduce_device_inline__ int packed_mul(int a, int b) { + int result; + asm volatile("v_pk_mul_f16 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); + return result; +} + +template <> +__quickreduce_device_inline__ int packed_mul(int a, int b) { + nv_bfloat162* tA = reinterpret_cast(&a); + nv_bfloat162* tB = reinterpret_cast(&b); + nv_bfloat162 tR = __hmul2(*tA, *tB); + return *(reinterpret_cast(&tR)); +} + +template +__quickreduce_device_inline__ int packed_rcp(int a); + +template <> +__quickreduce_device_inline__ int packed_rcp(int a) { + return __builtin_bit_cast(int, h2rcp(__builtin_bit_cast(half2, a))); +} + +template <> +__quickreduce_device_inline__ int packed_rcp(int a) { + bf162_int_union A, R; + A.i = a; + R.bf2 = h2rcp(A.bf2); + return R.i; +} + +// changes dtype +__quickreduce_device_inline__ float T2float_cast(half a) { + return __half2float(a); +} + +__quickreduce_device_inline__ float T2float_cast(nv_bfloat16 a) { + return __bfloat162float(a); +} + +template +__quickreduce_device_inline__ int group_abs_max(int32x4_t atom) { + const int group_leader = (threadIdx.x / kThreadGroupSize) * kThreadGroupSize; + + int wmax, wmin, wblockmax; + int a, b; + a = packed_max(atom[0], atom[1]); + b = packed_max(atom[2], atom[3]); + + wmax = packed_max(a, b); + + a = packed_min(atom[0], atom[1]); + b = packed_min(atom[2], atom[3]); + + wmin = packed_min(a, b); + + // Reduce the max among a group of threads + // Note: This is basically 2 blocks of values setup as the + // upper/lower halves of the f16x2_t + for (int i = 1; i < kThreadGroupSize; i <<= 1) { + int x = __shfl_down(wmax, i); + wmax = packed_max(wmax, x); + + int y = __shfl_down(wmin, i); + wmin = packed_min(wmin, y); + } + wblockmax = packed_abs_max(wmax, wmin); + // Share with the cohort + wblockmax = __shfl(wblockmax, group_leader); + return wblockmax; +} + +__quickreduce_device_inline__ void set_sync_flag(uint32_t* flag_ptr, uint32_t flag) { + __atomic_store_n(flag_ptr, flag, __ATOMIC_RELEASE); +} + +__quickreduce_device_inline__ void wait_sync_flag(uint32_t* flag_ptr, uint32_t flag) { + while (__atomic_load_n(flag_ptr, __ATOMIC_RELAXED) != flag) { + } +} + +} // namespace quickreduce diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/test_mscclpp_allreduce.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/test_mscclpp_allreduce.cu new file mode 100644 index 00000000..4ca0c573 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/allreduce/test_mscclpp_allreduce.cu @@ -0,0 +1,153 @@ +/* + * this file is used to test mscclpp_allreduce.cu using mpirun + * this file is adapted from https://github.com/flashinfer-ai/flashinfer/blob/v0.2.5/src/test_sum_all_reduce.cu +usage: +cd PATH-TO-THIS-FILE +export MPI_HOME=/usr/local/mpi +# export MPI_HOME=/opt/hpcx/ompi/ +export MSCCLPP_HOME=/workspace/test/mscclpp +nvcc -O2 -arch=native -std=c++17 test_mscclpp_allreduce.cu \ + -o test_mscclpp_allreduce -D_GLIBCXX_USE_CXX11_ABI=0 \ + -I${MSCCLPP_HOME}/include -L${MSCCLPP_HOME}/build -lmscclpp \ + -lnccl -I${MPI_HOME}/include -L${MPI_HOME}/lib -lmpi + +/opt/hpcx/ompi/bin/ +mpirun --allow-run-as-root -H 127.0.0.1:8 -np 8 \ + --map-by ppr:8:node \ + --mca btl_openib_warn_no_device_params_found 0 \ + --mca btl_tcp_if_include bond0 \ + --allow-run-as-root -np 8 \ + -x NCCL_RUNTIME_CONNECT=0 -x NCCL_IB_GID_INDEX=3 -x NCCL_DEBUG=WARN \ + -x LD_PRELOAD=${MSCCLPP_HOME}/build/libmscclpp.so ./test_mscclpp_allreduce + */ +#include +#include +#include +#include + +#ifndef CHECK_CUDA_SUCCESS +#define CHECK_CUDA_SUCCESS(cmd) \ + do { \ + cudaError_t e = cmd; \ + if (e != cudaSuccess) { \ + printf("Failed: Cuda error %s:%d '%s'\n", __FILE__, __LINE__, cudaGetErrorString(e)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) +#endif + +#include + +#include "mscclpp_allreduce.cuh" + +template +bool isclose(T a, T b, float rtol = 1e-5, float atol = 1e-8) { + return fabs(a - b) <= (atol + rtol * fabs(b)); +} + +int main(int argc, char* argv[]) { + // init mpi + MPI_Init(&argc, &argv); + printf("MPI Initialized.\n"); + int nranks, rank; + + // get work size and rank id + MPI_Comm_size(MPI_COMM_WORLD, &nranks); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + cudaSetDevice(rank); + printf("nranks: %d, rank: %d\n", nranks, rank); + + // init host and device buffers + using T = float; + using ReduceT = float; + const size_t num_elems = 2 * 1024 * 1024; + std::vector host_buf(num_elems); + for (uint32_t i = 0; i < num_elems; ++i) { + host_buf[i] = T(i + rank); + } + thrust::device_vector device_buf(host_buf); + const size_t buf_size_in_bytes = num_elems * sizeof(T); + std::vector host_result_buf(num_elems); + thrust::device_vector device_result_buf(host_result_buf); + + std::vector host_scratch_buf(num_elems * 8); + for (uint32_t i = 0; i < num_elems; ++i) { + host_scratch_buf[i] = 1; + } + thrust::device_vector device_scratch_buf(host_scratch_buf); + std::vector host_put_buf(num_elems); + thrust::device_vector device_put_buf(host_put_buf); + + mscclpp::UniqueId unique_id; + if (rank == 0) unique_id = mscclpp::TcpBootstrap::createUniqueId(); + MPI_Bcast(&unique_id, sizeof(unique_id), MPI_BYTE, 0, MPI_COMM_WORLD); + + std::vector rank_to_node(nranks); + std::vector rank_to_ib(nranks); + for (int i = 0; i < nranks; i++) { + rank_to_node[i] = i / 8; + rank_to_ib[i] = i % 8; + } + + cudaStream_t s; + CHECK_CUDA_SUCCESS(cudaStreamCreate(&s)); + CHECK_CUDA_SUCCESS(cudaStreamSynchronize(s)); + if (nranks == 8) { + auto context = std::make_shared( + unique_id, + rank, + nranks, + thrust::raw_pointer_cast(device_scratch_buf.data()), + buf_size_in_bytes * 8, + rank_to_node, + rank_to_ib); + printf("rank: %d, Msccl1NodeLLcontext setup.\n", rank); + MPI_Barrier(MPI_COMM_WORLD); + context->allreduce( + s, + thrust::raw_pointer_cast(device_buf.data()), + thrust::raw_pointer_cast(device_result_buf.data()), + device_buf.size()); + } else if (nranks == 16) { + // TODO: this branch is untested since there is something wrong with mpirun in my test machince + auto context = std::make_shared( + unique_id, + rank, + nranks, + thrust::raw_pointer_cast(device_scratch_buf.data()), + buf_size_in_bytes * 8, + thrust::raw_pointer_cast(device_put_buf.data()), + buf_size_in_bytes, + rank_to_node, + rank_to_ib); + printf("rank: %d, Msccl2NodeLLcontext setup.\n", rank); + MPI_Barrier(MPI_COMM_WORLD); + context->allreduce( + s, + thrust::raw_pointer_cast(device_buf.data()), + thrust::raw_pointer_cast(device_result_buf.data()), + device_buf.size()); + } + + // check result correctness + thrust::host_vector host_buf_result = device_result_buf; + size_t num_results_error_atol_1e_3_rtol_1e_3 = 0; + bool nan_detected = false; + + for (uint32_t i = 0; i < num_elems; ++i) { + T expected = T(i * nranks + (nranks - 1) * nranks / 2); + if (std::isnan(float(host_buf_result[i]))) { + nan_detected = true; + } + if (!isclose(float(host_buf_result[i]), float(expected), 1e-3, 1e-3)) { + num_results_error_atol_1e_3_rtol_1e_3++; + } + } + float result_accuracy = 1. - float(num_results_error_atol_1e_3_rtol_1e_3) / float(num_elems); + + printf("rank: %d, nan_detected: %d accuracy: %f\n", rank, nan_detected, result_accuracy); + + CHECK_CUDA_SUCCESS(cudaStreamDestroy(s)); + MPI_Finalize(); + return 0; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cascade.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cascade.cu new file mode 100644 index 00000000..9d49360d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cascade.cu @@ -0,0 +1,55 @@ +// Adapted from +// https://github.com/flashinfer-ai/flashinfer/blob/55576c626421b5ee7e7ebe74afd26465c8ae863f/csrc/cascade.cu + +#include +#include + +#include + +#include "pytorch_extension_utils.h" + +using namespace flashinfer; + +void merge_state( + at::Tensor v_a, at::Tensor s_a, at::Tensor v_b, at::Tensor s_b, at::Tensor v_merged, at::Tensor s_merged) { + CHECK_INPUT(v_a); + CHECK_INPUT(s_a); + CHECK_INPUT(v_b); + CHECK_INPUT(s_b); + auto device = v_a.device(); + CHECK_EQ(s_a.device(), device); + CHECK_EQ(v_b.device(), device); + CHECK_EQ(s_b.device(), device); + CHECK_DIM(3, v_a); + CHECK_DIM(2, s_a); + CHECK_DIM(3, v_b); + CHECK_DIM(2, s_b); + CHECK_SHAPE(v_a, v_b); + CHECK_SHAPE(s_a, s_b); + CHECK_EQ(v_a.size(0), s_a.size(0)); + CHECK_EQ(v_a.size(1), s_b.size(1)); + unsigned int seq_len = v_a.size(0); + unsigned int num_heads = v_a.size(1); + unsigned int head_dim = v_a.size(2); + + const c10::cuda::OptionalCUDAGuard device_guard(v_a.device()); + auto stream = at::cuda::getCurrentCUDAStream(); + + bool success = DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP16(v_a.scalar_type(), c_type, [&] { + cudaError_t status = MergeState( + static_cast(v_a.data_ptr()), + static_cast(s_a.data_ptr()), + static_cast(v_b.data_ptr()), + static_cast(s_b.data_ptr()), + static_cast(v_merged.data_ptr()), + static_cast(s_merged.data_ptr()), + seq_len, + num_heads, + head_dim, + stream); + TORCH_CHECK(status == cudaSuccess, "MergeState kernel launch failed: ", cudaGetErrorString(status)); + return true; + }); + + TORCH_CHECK(success, "MergeState kernel launch failed: unsupported data type"); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cutlass_mla_kernel.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cutlass_mla_kernel.cu new file mode 100644 index 00000000..a41779c1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cutlass_mla_kernel.cu @@ -0,0 +1,274 @@ +/* +Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include + +#include +#include + +#include "cutlass_sm100_mla/device/sm100_mla.hpp" +#include "cutlass_sm100_mla/kernel/sm100_mla_tile_scheduler.hpp" +#include "utils.h" + +// clang-format off +#if !defined(CUDA_VERSION) || CUDA_VERSION < 12040 +void cutlass_mla_decode( + torch::Tensor const& out, + torch::Tensor const& q_nope, + torch::Tensor const& q_pe, + torch::Tensor const& kv_c_and_k_pe_cache, + torch::Tensor const& seq_lens, + torch::Tensor const& page_table, + torch::Tensor const& workspace, + int64_t num_kv_splits) { + TORCH_CHECK(false, "CUDA version must be >= 12.4 for cutlass_mla_decode"); +} +int64_t cutlass_mla_get_workspace_size(int64_t max_seq_len, int64_t num_batches, int64_t sm_count, int64_t num_kv_splits) { + TORCH_CHECK(false, "CUDA version must be >= 12.4 for cutlass_mla_get_workspace_size"); +} +#else + +#define CUTLASS_CHECK(status) \ + { \ + cutlass::Status error = status; \ + TORCH_CHECK(error == cutlass::Status::kSuccess, cutlassGetStatusString(error)); \ + } + +using namespace cute; +using namespace cutlass::fmha::kernel; + +template +struct IsPersistent { + static const bool value = v; +}; + +template > +struct MlaSm100 { + using Element = T; + using ElementAcc = float; + using ElementOut = T; + + using TileShape = Shape<_128, _128, Shape<_512, _64>>; + using TileShapeH = cute::tuple_element_t<0, TileShape>; + using TileShapeD = cute::tuple_element_t<2, TileShape>; + + // H K (D_latent D_rope) B + using ProblemShape = cute::tuple; + + using StrideQ = cute::tuple; // H D B + using StrideK = cute::tuple; // K D B + using StrideO = StrideK; // H D B + using StrideLSE = cute::tuple<_1, int>; // H B + + using TileScheduler = + std::conditional_t; + + using FmhaKernel = cutlass::fmha::kernel::Sm100FmhaMlaKernelTmaWarpspecialized< + TileShape, + Element, + ElementAcc, + ElementOut, + ElementAcc, + TileScheduler, + /*kIsCpAsync=*/!IsPaged128>; + using Fmha = cutlass::fmha::device::MLA; +}; + +template +typename T::Fmha::Arguments args_from_options( + at::Tensor const& out, + at::Tensor const& q_nope, + at::Tensor const& q_pe, + at::Tensor const& kv_c_and_k_pe_cache, + at::Tensor const& seq_lens, + at::Tensor const& page_table, + double sm_scale, + int64_t num_kv_splits) { + cutlass::KernelHardwareInfo hw_info; + hw_info.device_id = q_nope.device().index(); + hw_info.sm_count = cutlass::KernelHardwareInfo::query_device_multiprocessor_count(hw_info.device_id); + + int batches = q_nope.size(0); + int page_count_per_seq = page_table.size(1); + int page_count_total = kv_c_and_k_pe_cache.size(0); + int page_size = kv_c_and_k_pe_cache.size(1); + int max_seq_len = page_size * page_count_per_seq; + using TileShapeH = typename T::TileShapeH; + using TileShapeD = typename T::TileShapeD; + auto problem_shape = cute::make_tuple(TileShapeH{}, max_seq_len, TileShapeD{}, batches); + + auto [H, K, D, B] = problem_shape; + auto [D_latent, D_rope] = D; + + float scale = float(sm_scale); + + using StrideQ = typename T::StrideQ; + using StrideK = typename T::StrideK; + using StrideO = typename T::StrideO; + using StrideLSE = typename T::StrideLSE; + + StrideQ stride_Q_nope = cute::make_tuple( + static_cast(q_nope.stride(1)), _1{}, static_cast(q_nope.stride(0))); + StrideQ stride_Q_pe = cute::make_tuple( + static_cast(q_pe.stride(1)), _1{}, static_cast(q_pe.stride(0))); + + StrideK stride_C = cute::make_tuple( + static_cast(0 + D_latent + D_rope), _1{}, static_cast(page_size * (D_latent + D_rope))); + StrideLSE stride_PT = cute::make_stride(_1{}, page_count_per_seq); + StrideLSE stride_LSE = cute::make_tuple(_1{}, 0 + H); + StrideO stride_O = cute::make_tuple(static_cast(0 + D_latent), _1{}, static_cast(0 + H * D_latent)); + + using Element = typename T::Element; + using ElementOut = typename T::ElementOut; + using ElementAcc = typename T::ElementAcc; + auto Q_nope_ptr = static_cast(q_nope.data_ptr()); + auto Q_pe_ptr = static_cast(q_pe.data_ptr()); + auto C_ptr = static_cast(kv_c_and_k_pe_cache.data_ptr()); + typename T::Fmha::Arguments arguments{ + problem_shape, + {scale, + Q_nope_ptr, + stride_Q_nope, + Q_pe_ptr, + stride_Q_pe, + C_ptr, + stride_C, + C_ptr + D_latent, + stride_C, + static_cast(seq_lens.data_ptr()), + static_cast(page_table.data_ptr()), + stride_PT, + page_count_total, + page_size}, + {static_cast(out.data_ptr()), stride_O, static_cast(nullptr), stride_LSE}, + hw_info, + // TODO(trevor-m): Change split_kv back to -1 when + // https://github.com/NVIDIA/cutlass/issues/2274 is fixed. Split_kv=1 will + // perform worse with larger context length and smaller batch sizes. + static_cast(num_kv_splits), // split_kv + nullptr, // is_var_split_kv + }; + // TODO(kaixih@nvidia): When split_kv=-1 and is_var_split_kv=false, we compute + // split_kv automatically based on batch size and sequence length to balance + // workload across available SMs. Consider using var_split_kv for manual + // control if needed. + T::Fmha::set_split_kv(arguments); + return arguments; +} + +template +void runMla( + at::Tensor const& out, + at::Tensor const& q_nope, + at::Tensor const& q_pe, + at::Tensor const& kv_c_and_k_pe_cache, + at::Tensor const& seq_lens, + at::Tensor const& page_table, + at::Tensor const& workspace, + double sm_scale, + int64_t num_kv_splits, + cudaStream_t stream) { + using MlaSm100Type = MlaSm100; + typename MlaSm100Type::Fmha fmha; + auto arguments = args_from_options(out, q_nope, q_pe, kv_c_and_k_pe_cache, seq_lens, page_table, sm_scale, num_kv_splits); + + CUTLASS_CHECK(fmha.can_implement(arguments)); + + CUTLASS_CHECK(fmha.initialize(arguments, workspace.data_ptr(), stream)); + + CUTLASS_CHECK(fmha.run(arguments, workspace.data_ptr(), stream)); +} + +#define DISPATCH_BOOL(expr, const_expr, ...) \ + [&]() -> bool { \ + if (expr) { \ + constexpr bool const_expr = true; \ + return __VA_ARGS__(); \ + } else { \ + constexpr bool const_expr = false; \ + return __VA_ARGS__(); \ + } \ + }() + +void cutlass_mla_decode( + torch::Tensor const& out, + torch::Tensor const& q_nope, + torch::Tensor const& q_pe, + torch::Tensor const& kv_c_and_k_pe_cache, + torch::Tensor const& seq_lens, + torch::Tensor const& page_table, + torch::Tensor const& workspace, + double sm_scale, + int64_t num_kv_splits) { + auto sm_version = getSMVersion(); + // On SM103a, half of the accuracy tests are failing. + TORCH_CHECK(sm_version == 100, "cutlass_mla_decode is only supported on compute capability 10.0, but found sm version ", sm_version); + + auto in_dtype = q_nope.dtype(); + at::cuda::CUDAGuard device_guard{(char)q_nope.get_device()}; + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(q_nope.get_device()); + const int page_size = kv_c_and_k_pe_cache.size(1); + + // NOTE(alcanderian): IsPersistent has bug with manual split_kv. + // Kernel will hang if batch is too large with large num_kv_splits. (for example bs=8, num_kv_splits=8) + // Maybe per batch split kv will fix this. + DISPATCH_BOOL(page_size == 128, IsPaged128, [&] { + DISPATCH_BOOL(num_kv_splits <= 1, NotManualSplitKV, [&] { + if (in_dtype == at::ScalarType::Half) { + runMla>( + out, q_nope, q_pe, kv_c_and_k_pe_cache, seq_lens, page_table, workspace, sm_scale, num_kv_splits, stream); + } else if (in_dtype == at::ScalarType::BFloat16) { + runMla>( + out, q_nope, q_pe, kv_c_and_k_pe_cache, seq_lens, page_table, workspace, sm_scale, num_kv_splits, stream); + } else if (in_dtype == at::ScalarType::Float8_e4m3fn) { + runMla>( + out, q_nope, q_pe, kv_c_and_k_pe_cache, seq_lens, page_table, workspace, sm_scale, num_kv_splits, stream); + } else { + TORCH_CHECK(false, "Unsupported input data type of MLA"); + } + return true; + }); + return true; + }); +} + +int64_t cutlass_mla_get_workspace_size(int64_t max_seq_len, int64_t num_batches, int64_t sm_count, int64_t num_kv_splits) { + // Workspace size depends on ElementAcc and ElementLSE (same as ElementAcc) + // which are float, so Element type here doesn't matter. + using MlaSm100Type = MlaSm100; + + // Get split kv. Requires problem shape and sm_count only. + typename MlaSm100Type::Fmha::Arguments arguments; + using TileShapeH = typename MlaSm100Type::TileShapeH; + using TileShapeD = typename MlaSm100Type::TileShapeD; + arguments.problem_shape = + cute::make_tuple(TileShapeH{}, static_cast(max_seq_len), TileShapeD{}, static_cast(num_batches)); + // Assumes device 0 when getting sm_count. + arguments.hw_info.sm_count = + sm_count <= 0 ? cutlass::KernelHardwareInfo::query_device_multiprocessor_count(/*device_id=*/0) : sm_count; + arguments.split_kv = static_cast(num_kv_splits); + MlaSm100Type::Fmha::set_split_kv(arguments); + + return MlaSm100Type::Fmha::get_workspace_size(arguments); +} + +#endif +// clang-format on diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cutlass_sm100_mla/device/sm100_mla.hpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cutlass_sm100_mla/device/sm100_mla.hpp new file mode 100644 index 00000000..dd4ed231 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cutlass_sm100_mla/device/sm100_mla.hpp @@ -0,0 +1,358 @@ +/*************************************************************************************************** + * Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! + \file + \brief An universal device layer for cutlass 3.x-style kernels. +*/ + +// clang-format off +#pragma once + +// common +#include "cutlass/cutlass.h" +#include "cutlass/device_kernel.h" + +#if !defined(__CUDACC_RTC__) +#include "cutlass/cluster_launch.hpp" +#include "cutlass/trace.h" +#endif // !defined(__CUDACC_RTC__) + +#include "../kernel/sm100_fmha_mla_tma_warpspecialized.hpp" +#include "../kernel/sm100_fmha_mla_reduction.hpp" + +//////////////////////////////////////////////////////////////////////////////// + +namespace cutlass::fmha::device { + +using namespace cute; +using namespace cutlass::fmha::kernel; + + +//////////////////////////////////////////////////////////////////////////////// +////////////////////////////// CUTLASS 3.x API ///////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// + +template< + class Kernel_ +> +class MLA { +public: + + using Kernel = Kernel_; + + using ReductionKernel = cutlass::fmha::kernel::Sm100FmhaMlaReductionKernel< + typename Kernel::ElementOut, + typename Kernel::ElementAcc, + typename Kernel::ElementAcc, + Kernel::TileShapeH::value, + Kernel::TileShapeL::value, + 256 /*Max split*/ + >; + + /// Argument structure: User API + using KernelArguments = typename Kernel::Arguments; + using ReductionArguments = typename ReductionKernel::Arguments; + + using Arguments = KernelArguments; + + /// Argument structure: Kernel API + using KernelParams = typename Kernel::Params; + using ReductionParams = typename ReductionKernel::Params; + struct Params { + KernelParams fmha_params; + ReductionParams reduction_params; + }; + +private: + + /// Kernel API parameters object + Params params_; + + bool is_initialized(bool set = false) { + static bool initialized = false; + if (set) initialized = true; + return initialized; + } + + static ReductionArguments to_reduction_args(Arguments const& args) { + auto [H, K, D, B] = args.problem_shape; + return ReductionArguments{ + nullptr, args.epilogue.ptr_o, nullptr, args.epilogue.ptr_lse, + args.mainloop.softmax_scale, B, args.split_kv, K, args.mainloop.ptr_seq, + args.ptr_split_kv, Kernel::TileShapeS::value + }; + } + +public: + + /// Access the Params structure + Params const& params() const { + return params_; + } + + static void set_split_kv (KernelArguments& args) { + if (args.split_kv >= 1) return; + auto [H, K, D, B] = args.problem_shape; + int sm_count = args.hw_info.sm_count; + int max_splits = ceil_div(K, 128); + int sms_per_batch = max(1, sm_count / B); + int split_heur = min(max_splits, sms_per_batch); + int waves = ceil_div(B * split_heur, sm_count); + int k_waves = ceil_div(max_splits, split_heur); + int split_wave_aware = ceil_div(max_splits, k_waves); + args.split_kv = split_wave_aware; + } + + /// Determines whether the GEMM can execute the given problem. + static Status + can_implement(Arguments const& args) { + if (! Kernel::can_implement(args)) { + return Status::kInvalid; + } + if (! ReductionKernel::can_implement(to_reduction_args(args))) { + return Status::kInvalid; + } + return Status::kSuccess; + } + + /// Gets the workspace size + static size_t + get_workspace_size(Arguments const& args) { + size_t workspace_bytes = 0; + workspace_bytes += Kernel::get_workspace_size(args); + workspace_bytes += ReductionKernel::get_workspace_size(to_reduction_args(args)); + return workspace_bytes; + } + + /// Computes the maximum number of active blocks per multiprocessor + static int maximum_active_blocks(int /* smem_capacity */ = -1) { + CUTLASS_TRACE_HOST("MLA::maximum_active_blocks()"); + int max_active_blocks = -1; + int smem_size = Kernel::SharedStorageSize; + + // first, account for dynamic smem capacity if needed + cudaError_t result; + if (smem_size >= (48 << 10)) { + CUTLASS_TRACE_HOST(" Setting smem size to " << smem_size); + result = cudaFuncSetAttribute( + device_kernel, + cudaFuncAttributeMaxDynamicSharedMemorySize, + smem_size); + if (cudaSuccess != result) { + result = cudaGetLastError(); // to clear the error bit + CUTLASS_TRACE_HOST( + " cudaFuncSetAttribute() returned error: " + << cudaGetErrorString(result)); + return -1; + } + } + + // query occupancy after setting smem size + result = cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &max_active_blocks, + device_kernel, + Kernel::MaxThreadsPerBlock, + smem_size); + + if (cudaSuccess != result) { + result = cudaGetLastError(); // to clear the error bit + CUTLASS_TRACE_HOST( + " cudaOccupancyMaxActiveBlocksPerMultiprocessor() returned error: " + << cudaGetErrorString(result)); + return -1; + } + + CUTLASS_TRACE_HOST(" max_active_blocks: " << max_active_blocks); + return max_active_blocks; + } + + /// Initializes GEMM state from arguments. + Status + initialize(Arguments const& args, void* workspace = nullptr, cudaStream_t stream = nullptr) { + CUTLASS_TRACE_HOST("MLA::initialize() - workspace " + << workspace << ", stream: " << (stream ? "non-null" : "null")); + + // Initialize the workspace + Status status = Kernel::initialize_workspace(args, workspace, stream); + if (status != Status::kSuccess) { + return status; + } + status = ReductionKernel::initialize_workspace(to_reduction_args(args), workspace, stream); + if (status != Status::kSuccess) { + return status; + } + KernelParams kernel_params = Kernel::to_underlying_arguments(args, workspace); + + ReductionArguments reduction_args = to_reduction_args(args); + if (reduction_args.split_kv > 1) { + reduction_args.ptr_oaccum = kernel_params.epilogue.ptr_o_acc; + reduction_args.ptr_lseaccum = kernel_params.epilogue.ptr_lse_acc; + } + ReductionParams reduction_params = ReductionKernel::to_underlying_arguments(reduction_args, workspace); + // Initialize the Params structure + params_ = Params {kernel_params, reduction_params}; + + if (is_initialized()) return Status::kSuccess; + + // account for dynamic smem capacity if needed + // no dynamic smem is needed for reduction kernel + int smem_size = Kernel::SharedStorageSize; + if (smem_size >= (48 << 10)) { + CUTLASS_TRACE_HOST(" Setting smem size to " << smem_size); + cudaError_t result = cudaFuncSetAttribute( + device_kernel, + cudaFuncAttributeMaxDynamicSharedMemorySize, + smem_size); + if (cudaSuccess != result) { + result = cudaGetLastError(); // to clear the error bit + CUTLASS_TRACE_HOST(" cudaFuncSetAttribute() returned error: " << cudaGetErrorString(result)); + return Status::kErrorInternal; + } + } + + is_initialized(true); + + return Status::kSuccess; + } + + /// Update API is preserved in 3.0, but does not guarantee a lightweight update of params. + Status + update(Arguments const& args, void* workspace = nullptr) { + CUTLASS_TRACE_HOST("MLA()::update() - workspace: " << workspace); + + size_t workspace_bytes = get_workspace_size(args); + if (workspace_bytes > 0 && nullptr == workspace) { + return Status::kErrorWorkspaceNull; + } + + auto fmha_params = Kernel::to_underlying_arguments(args, workspace); + + ReductionArguments reduction_args = to_reduction_args(args); + if (reduction_args.split_kv > 1) { + reduction_args.ptr_oaccum = fmha_params.epilogue.ptr_o_acc; + reduction_args.ptr_lseaccum = fmha_params.epilogue.ptr_lse_acc; + } + ReductionParams reduction_params = ReductionKernel::to_underlying_arguments(reduction_args, workspace); + // Initialize the Params structure + params_ = Params {fmha_params, reduction_params}; + + return Status::kSuccess; + } + + /// Primary run() entry point API that is static allowing users to create and manage their own params. + /// Supplied params struct must be construct by calling Kernel::to_underling_arguments() + static Status + run(Params& params, cudaStream_t stream = nullptr) { + CUTLASS_TRACE_HOST("MLA::run()"); + dim3 const block = Kernel::get_block_shape(); + dim3 const grid = Kernel::get_grid_shape(params.fmha_params); + + // configure smem size and carveout + int smem_size = Kernel::SharedStorageSize; + + Status launch_result; + // Use extended launch API only for mainloops that use it + if constexpr(Kernel::ArchTag::kMinComputeCapability >= 90) { + dim3 cluster(cute::size<0>(typename Kernel::ClusterShape{}), + cute::size<1>(typename Kernel::ClusterShape{}), + cute::size<2>(typename Kernel::ClusterShape{})); + void const* kernel = (void const*) device_kernel; + void* kernel_params[] = {¶ms.fmha_params}; + launch_result = ClusterLauncher::launch(grid, cluster, block, smem_size, stream, kernel, kernel_params); + } + else { + launch_result = Status::kSuccess; + device_kernel<<>>(params.fmha_params); + } + + cudaError_t result = cudaGetLastError(); + if (cudaSuccess != result or Status::kSuccess != launch_result) { + //return Status::kSuccess; + CUTLASS_TRACE_HOST(" Kernel launch failed. Reason: " << result); + return Status::kErrorInternal; + } + if (params.reduction_params.split_kv > 1) { + // launch reduction kernel + dim3 const block = ReductionKernel::get_block_shape(); + dim3 const grid = ReductionKernel::get_grid_shape(params.reduction_params); + device_kernel<<>>(params.reduction_params); + cudaError_t result = cudaGetLastError(); + if (cudaSuccess == result) { + return Status::kSuccess; + } + else { + CUTLASS_TRACE_HOST(" Kernel launch failed. Reason: " << result); + return Status::kErrorInternal; + } + } + else { + return Status::kSuccess; + } + } + + // + // Non-static launch overloads that first create and set the internal params struct of this kernel handle. + // + + /// Launches the kernel after first constructing Params internal state from supplied arguments. + Status + run(Arguments const& args, void* workspace = nullptr, cudaStream_t stream = nullptr) { + Status status = initialize(args, workspace, stream); + if (Status::kSuccess == status) { + status = run(params_, stream); + } + return status; + } + + /// Launches the kernel after first constructing Params internal state from supplied arguments. + Status + operator()(Arguments const& args, void* workspace = nullptr, cudaStream_t stream = nullptr) { + return run(args, workspace, stream); + } + + /// Overload that allows a user to re-launch the same kernel without updating internal params struct. + Status + run(cudaStream_t stream = nullptr) { + return run(params_, stream); + } + + /// Overload that allows a user to re-launch the same kernel without updating internal params struct. + Status + operator()(cudaStream_t stream = nullptr) { + return run(params_, stream); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass::fmha::device + +//////////////////////////////////////////////////////////////////////////////// diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_fmha_mla_reduction.hpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_fmha_mla_reduction.hpp new file mode 100644 index 00000000..b75870d0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_fmha_mla_reduction.hpp @@ -0,0 +1,198 @@ +/*************************************************************************************************** + * Copyright (c) 2024 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ + +// clang-format off +#pragma once + +#include "cutlass/cutlass.h" +#include "cutlass/arch/arch.h" +#include "cute/tensor.hpp" + +namespace cutlass::fmha::kernel { + +using namespace cute; +template< + class ElementOut, + class ElementAcc, + class ElementScale, + size_t kNumHeads, + size_t kHeadDimLatent, + int kMaxSplits +> +struct Sm100FmhaMlaReductionKernel { + + static const int SharedStorageSize = 0; + static const int MaxThreadsPerBlock = 128; + static const int MinBlocksPerMultiprocessor = 1; + + using ArchTag = cutlass::arch::Sm100; + + static_assert(kHeadDimLatent % MaxThreadsPerBlock == 0); + struct Arguments { + ElementAcc* ptr_oaccum = nullptr; + ElementOut* ptr_o = nullptr; + ElementAcc* ptr_lseaccum = nullptr; + ElementAcc* ptr_lse = nullptr; + ElementScale scale = 1.f; + int num_batches = 0; + int split_kv = -1; + int dim_k = -1; + int* ptr_seq = nullptr; + int* ptr_split_kv = nullptr; + int tile_shape_s = 128; + }; + using Params = Arguments; + + static Params to_underlying_arguments(Arguments const& args, void* workspace) { + return {args.ptr_oaccum, args.ptr_o, args.ptr_lseaccum, args.ptr_lse, + args.scale, args.num_batches, args.split_kv, args.dim_k, args.ptr_seq, + args.ptr_split_kv, args.tile_shape_s}; + } + + static size_t get_workspace_size(Arguments const& /*args*/) { + return 0; + } + + static Status initialize_workspace( + Arguments const& /*args*/, void* /*ws*/, cudaStream_t /*stream*/) { + return Status::kSuccess; + } + + static dim3 get_grid_shape(Params const& params) { + return dim3(kNumHeads, 1, params.num_batches); + } + + static dim3 get_block_shape() { + return dim3(MaxThreadsPerBlock, 1, 1); + } + + static bool can_implement(Arguments const& args) { + if (args.num_batches <= 0) return false; + if (args.split_kv <= 0) return false; + return true; + } + + CUTLASS_DEVICE void operator() (Params const& params, char* smem_raw) { + if (params.split_kv <= 1) return; + auto blk_coord = make_coord(blockIdx.x, _0{}, blockIdx.z); + + __shared__ ElementAcc sLseScale[kMaxSplits]; + const size_t offset_lseaccum = get<0>(blk_coord) + kNumHeads * params.split_kv * get<2>(blk_coord); + const size_t offset_lse = get<0>(blk_coord) + kNumHeads * get<2>(blk_coord); + + Tensor gLSEaccum = make_tensor(make_gmem_ptr(params.ptr_lseaccum + offset_lseaccum), + make_shape(params.split_kv), Stride>{}); + + Tensor gLSE = make_tensor(make_gmem_ptr(params.ptr_lse + offset_lse), + Shape<_1>{}, Stride<_1>{}); + + auto dim_k = params.ptr_seq == nullptr ? params.dim_k : params.ptr_seq[get<2>(blk_coord)]; + auto local_split_kv = params.ptr_split_kv == nullptr ? params.split_kv : params.ptr_split_kv[get<2>(blk_coord)]; + auto k_tile_total = ceil_div(dim_k, params.tile_shape_s); + auto k_tile_per_cta = ceil_div(k_tile_total, local_split_kv); + local_split_kv = ceil_div(k_tile_total, k_tile_per_cta); + + int warp_idx = cutlass::canonical_warp_idx_sync(); + if (warp_idx == 0) { + constexpr int kNLsePerThread = cute::ceil_div(kMaxSplits, 32); + + ElementAcc local_lse[kNLsePerThread]; + + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < kNLsePerThread; ++i) { + const int split = i * 32 + threadIdx.x; + local_lse[i] = split < local_split_kv ? gLSEaccum(split) : -std::numeric_limits::infinity(); + } + + ElementAcc lse_max = -std::numeric_limits::infinity(); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < kNLsePerThread; ++i) { + lse_max = max(lse_max, local_lse[i]); + } + CUTLASS_PRAGMA_UNROLL + for (int offset = 16; offset >= 1; offset /= 2) { + lse_max = max(lse_max, __shfl_xor_sync(0xffffffff, lse_max, offset)); + } + lse_max = lse_max == -std::numeric_limits::infinity() ? 0.0f : lse_max; // In case all local LSEs are -inf + lse_max = __shfl_sync(0xffffffff, lse_max, 0); + + ElementAcc sum_lse = 0; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < kNLsePerThread; ++i) { + sum_lse = sum_lse + expf(local_lse[i] - lse_max); + } + + CUTLASS_PRAGMA_UNROLL + for (int offset = 16; offset >= 1; offset /= 2) { + sum_lse = sum_lse + __shfl_xor_sync(0xffffffff, sum_lse, offset); + } + + sum_lse = __shfl_sync(0xffffffff, sum_lse, 0); + + ElementAcc global_lse = (sum_lse == 0.f || sum_lse != sum_lse) ? std::numeric_limits::infinity() : logf(sum_lse) + lse_max; + if (threadIdx.x == 0 and params.ptr_lse != nullptr) { + gLSE(0) = global_lse; + } + + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < kNLsePerThread; ++i) { + const int split = i * 32 + threadIdx.x; + if (split < local_split_kv) { + sLseScale[split] = expf(local_lse[i] - global_lse); + } + } + } + __syncthreads(); + + constexpr int Elements = kHeadDimLatent / MaxThreadsPerBlock; + const size_t offset_oaccum = kHeadDimLatent * params.split_kv * (get<0>(blk_coord) + kNumHeads * get<2>(blk_coord)); + Tensor gOaccum = make_tensor(make_gmem_ptr(params.ptr_oaccum + offset_oaccum), + Shape>{}, Stride<_1>{}); + ElementAcc local_val[Elements] = {0}; + for (int split = 0; split < local_split_kv; ++split) { + ElementAcc lse_scale = sLseScale[split]; + CUTLASS_PRAGMA_UNROLL + for(int i = 0; i < Elements; ++i) { + local_val[i] += lse_scale * gOaccum(threadIdx.x + MaxThreadsPerBlock * i); + } + gOaccum.data() = gOaccum.data() + kHeadDimLatent; + } + auto ptr_o_local = params.ptr_o + (get<0>(blk_coord) + get<2>(blk_coord) * kNumHeads) * kHeadDimLatent; + Tensor gO = make_tensor(make_gmem_ptr(ptr_o_local), Shape>{}, Stride<_1>{}); + + CUTLASS_PRAGMA_UNROLL + for(int i = 0; i < Elements; ++i) { + gO(threadIdx.x + MaxThreadsPerBlock * i) = static_cast(local_val[i]); + } + } +}; + +} // namespace cutlass::fmha::kernel diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_fmha_mla_tma_warpspecialized.hpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_fmha_mla_tma_warpspecialized.hpp new file mode 100644 index 00000000..9809db84 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_fmha_mla_tma_warpspecialized.hpp @@ -0,0 +1,2018 @@ +/*************************************************************************************************** + * Copyright (c) 2024 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ + +// clang-format off +#pragma once + +#include "cutlass/cutlass.h" + +#include "cute/tensor.hpp" +#include "cute/arch/simd_sm100.hpp" + +#include "cutlass/arch/arch.h" +#include "cutlass/arch/memory_sm80.h" +#include "cutlass/epilogue/thread/linear_combination.h" +#include "cutlass/gemm/collective/collective_builder.hpp" + +#include "gather_tensor.hpp" // from examples/common +#include "common/pow_2.hpp" + +namespace cutlass::fmha::kernel { + +using namespace cute; + +template< + class TileShape, + class Element_, + class ElementAcc_, + class ElementOut_, + class ElementLSE_, + class TileScheduler, +#ifdef CPASYNC + bool kIsCpAsync = true +#else + bool kIsCpAsync = false +#endif +> +struct Sm100FmhaMlaKernelTmaWarpspecialized { + + using Element = Element_; + using ElementAcc = ElementAcc_; + using ElementOut = ElementOut_; + using ElementLSE = ElementLSE_; + + // only 2Sm mode is supported + static const bool kIs2Sm = true; + static const int MaxThreadsPerBlock = 256; + static const int MinBlocksPerMultiprocessor = 1; + static const int TotalSNum = 2; + static const int TotalPNum = 2; + using ArchTag = cutlass::arch::Sm100; + + using ClusterShape = cute::conditional_t, Shape<_1, _1, _1>>; + + using TileShapeH = tuple_element_t<0, TileShape>; + using TileShapeS = tuple_element_t<1, TileShape>; + using TileShapeD = tuple_element_t<2, TileShape>; + + using TileShapeL = tuple_element_t<0, TileShapeD>; + using TileShapeR = tuple_element_t<1, TileShapeD>; + static_assert(TileShapeL{} % TileShapeR{} == 0, "Rope head dim must divide latent head dim"); + + using ProblemShape = Shape; + using TensorStride = Stride; + using TmemAllocator = cute::conditional_t; + + static_assert(TileShapeH{} == 128); + static const int kWarpsInN = kIs2Sm ? 2 : 1; + + static const int kNumComputeWarps = 4; + static const int kNumLoadWarps = kIsCpAsync ? 2 : 1; + + enum class WarpRole { + kMma = 0x1, kLoad = 0x2, kCompute = 0x3, kLoadPageTable = 0x4, kEmpty=0x0 + }; + + static const long long unsigned int kWarpAssignment = kIsCpAsync ? 0x4221'3333ull : 0x0021'3333ull; + + static CUTLASS_DEVICE WarpRole warp_idx_to_role(int warp_idx) { + return static_cast((kWarpAssignment >> (4 * warp_idx)) & 0xF); + } + + static const int Alignment = 128 / sizeof_bits_v; + static const int AlignmentOut = 128 / sizeof_bits_v; + + using TileShapeQK = Shape; + static const int StagesQK = 24 / sizeof(Element); // free parameter + static const int IterationsQKLatent = decltype(TileShapeL{} / get<2>(TileShapeQK{}))::value; + static const int IterationsQKRope = decltype(TileShapeR{} / get<2>(TileShapeQK{}))::value; + static const int IterationsQK = IterationsQKLatent + IterationsQKRope; + + using Schedule = cute::conditional_t; + using CollectiveMmaQK = typename cutlass::gemm::collective::CollectiveBuilder< + cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, + Element, TensorStride, Alignment, + Element, TensorStride, Alignment, + ElementAcc, + TileShapeQK, ClusterShape, cutlass::gemm::collective::StageCount, + Schedule>::CollectiveOp; + using TiledMmaQK = typename CollectiveMmaQK::TiledMma; + using CtaShapeQK = typename CollectiveMmaQK::CtaShape_MNK; + + // chosen for unified smem staging between K and V + using TileShapePV = Shape; + using TransposeTensorStride = decltype(select<1,0,2>(TensorStride{})); + static const int StagesPV = StagesQK; // not sure why, but must be at least two. check pipes + static const int IterationsPV_K = decltype(TileShapeS{} / get<2>(TileShapePV{}))::value; + static const int IterationsPV_N = decltype(TileShapeL{} / get<1>(TileShapePV{}))::value; + + using CollectiveMmaPV = typename cutlass::gemm::collective::CollectiveBuilder< + cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, + Element, TensorStride, Alignment, + Element, TransposeTensorStride, Alignment, + ElementAcc, + TileShapePV, ClusterShape, cutlass::gemm::collective::StageCount, + Schedule>::CollectiveOp; + using CtaShapePV = typename CollectiveMmaPV::CtaShape_MNK; + static_assert(std::is_same_v); + + using TiledMmaPV = typename CollectiveMmaPV::TiledMma; + + using AtomThrShapeMNK = typename CollectiveMmaQK::AtomThrShapeMNK; + static_assert(typename CollectiveMmaQK::AtomThrShapeMNK{} == typename CollectiveMmaPV::AtomThrShapeMNK{}, "schedule must match"); + + static const int StagesPageTable = kIsCpAsync ? StagesPV : 1; + + // pipelines from load to mma, PipelineTmaUmmaAsync, stages tbd + // use expect_tx for Q load + using PipelineLoadQK = cute::conditional_t, PipelineTmaUmmaAsync>; + using PipelineLoadPV = PipelineLoadQK; + // pipeline from mma (Q@K) to softmax, PipelineUmmaAsync, 2 stages + using PipelineS = PipelineUmmaAsync; + // pipeline from softmax (P) to mma (bmm2), PipelineUmmaAsync, 2 stages + using PipelineP = PipelineUmmaConsumerAsync; + // pipeline from mma to softmax (for rescale), PipelineUmmaAsync, 1 stage + using PipelineO = PipelineUmmaAsync<1, AtomThrShapeMNK>; + + using PipelinePT = PipelineAsync; + + struct PipelineStorage { + alignas(16) typename PipelineLoadQK::SharedStorage load_qk; + alignas(16) typename PipelineS::SharedStorage mma_s; + alignas(16) typename PipelineP::SharedStorage p_mma; + alignas(16) typename PipelineO::SharedStorage mma_o; + alignas(16) typename PipelinePT::SharedStorage load_page_table; + }; + + template + static CUTE_DEVICE constexpr auto unstageSmemLayout(Layout const& layout, Stages stages = {}) { + return composition(layout, make_tuple(_, _, _, make_layout(stages))); + } + + using SmemLayoutQ = decltype(unstageSmemLayout(typename CollectiveMmaQK::SmemLayoutA{}, Int{})); + using SmemLayoutKC = typename CollectiveMmaQK::SmemLayoutB; + using SmemLayoutVC = typename CollectiveMmaPV::SmemLayoutB; + using SmemLayoutP = decltype(unstageSmemLayout(typename CollectiveMmaPV::SmemLayoutA{}, make_shape(Int{}, _2{}))); + + static const int kBytesLoadQ = size(AtomThrShapeMNK{}) * cutlass::bits_to_bytes(cosize(take<0,3>(SmemLayoutQ{})) * cute::sizeof_bits_v); + static const int kBytesLoadKC = size(AtomThrShapeMNK{}) * cutlass::bits_to_bytes(cosize(take<0,3>(SmemLayoutKC{})) * cute::sizeof_bits_v); + static const int kBytesLoadVC = size(AtomThrShapeMNK{}) * cutlass::bits_to_bytes(cosize(take<0,3>(SmemLayoutVC{})) * cute::sizeof_bits_v); + // pre-condition for overlapped smem staging + static_assert(kBytesLoadKC == kBytesLoadVC); + static_assert(StagesQK == StagesPV); + + static const int kTransactionsBytesLoadQK = kBytesLoadKC; + static const int kTransactionsBytesLoadExtraQ = kBytesLoadQ; + static const int kTransactionsBytesLoadPV = kBytesLoadVC; + + static const int kNamedBarrierExchange = (int) cutlass::arch::ReservedNamedBarriers::TransformBarrier; + // This Named Barrier is introduced to solve Q tile loading overwritten issue when enable persistent + // tile scheduler for FP8 MLA. + static const int kNamedBarrierEpilogue = (int) cutlass::arch::ReservedNamedBarriers::EpilogueBarrier; + // + static const int kNamedBarrierTmemDealloc = (int) cutlass::arch::ReservedNamedBarriers::TmemAllocBarrier; + + enum class TmemAllocation : uint32_t { + kSizeS = TileShapeS::value / kWarpsInN, + // Overall + kSizeO = TileShapeL::value / kWarpsInN, + // Between accumulators we loop over + kSizeAccO = decltype(get<1>(TileShapePV{}))::value / kWarpsInN, + kNumS = TotalSNum, + kNumP = TotalPNum, + kNumO = 1, + kS0 = 0, + kS1 = kS0 + kSizeS, + kO0 = kS1 + kSizeS, + kTotal = kO0 + kSizeO + }; + + static_assert(static_cast(TmemAllocation::kTotal) <= TmemAllocator::Sm100TmemCapacityColumns, "using too much tmem"); + + struct TensorStorage { + // to communicate max and row_sum + cute::array smem_exchange; + cute::array smem_page_table; + alignas(2048) cute::array> smem_q; + union { + alignas(2048) cute::array> smem_kc; + alignas(2048) cute::array> smem_vc; + }; + alignas(2048) cute::array> smem_p; + }; + + struct SharedStorage { + PipelineStorage pipelines; + TensorStorage tensors; + uint32_t tmem_base_ptr; + }; + + static const int SharedStorageSize = sizeof(SharedStorage); + static_assert(SharedStorageSize <= cutlass::arch::sm100_smem_capacity_bytes, "using too much smem"); + + struct MainloopArguments { + ElementAcc softmax_scale; + + // all tensors strides are (num_heads or seqlen, head_dim, batch) + // head_dim stride is always 1 + Element* ptr_q_latent; + TensorStride stride_q_latent; + Element* ptr_q_rope; + TensorStride stride_q_rope; + + Element* ptr_c_latent; + TensorStride stride_c_latent; + Element* ptr_k_rope; + TensorStride stride_k_rope; + + // for paged attention, we interpret what was previously [batch, seqlen] + // as [page_count, page_size], and index according to page_table + int* ptr_seq = nullptr; + int* ptr_page_table = nullptr; + // page table is [batch, seqlen or similar] + Stride<_1, int> stride_page_table = {}; + int page_count = 0; + int page_size = TileShapeS{}; // powers of two if kIsCpAsync, otherwise TileShapeS + }; + + struct EpilogueArguments { + ElementOut* ptr_o = nullptr; + TensorStride stride_o; + ElementLSE* ptr_lse = nullptr; + Stride<_1, int> stride_lse; + ElementAcc output_scale = 1.0f; + }; + + struct Arguments { + // (num_heads=128, seqlen, (d_latent=512, d_rope=64), batch_count) + // for paged attention, seqlen is max seqlen + ProblemShape problem_shape; + MainloopArguments mainloop; + EpilogueArguments epilogue; + KernelHardwareInfo hw_info; + int split_kv = -1; + int* ptr_split_kv = nullptr; + }; + + using TmaLoadQLatent = typename CollectiveMmaQK::Params::TMA_A; + using TmaLoadQRope = typename CollectiveMmaQK::Params::TMA_A; + using TmaLoadCLatent = typename CollectiveMmaQK::Params::TMA_B; + using TmaLoadKRope = typename CollectiveMmaQK::Params::TMA_B; + using TmaLoadCLatentTranspose = typename CollectiveMmaPV::Params::TMA_B; + + struct MainloopParams { + TmaLoadQLatent tma_load_q_latent; + TmaLoadQRope tma_load_q_rope; + TmaLoadCLatent tma_load_c_latent; + TmaLoadKRope tma_load_k_rope; + TmaLoadCLatentTranspose tma_load_c_latent_transpose; + }; + + struct EpilogueParams { + ElementOut* ptr_o = nullptr; + ElementAcc* ptr_o_acc = nullptr; + TensorStride stride_o; + TensorStride stride_o_acc; + ElementLSE* ptr_lse = nullptr; + ElementLSE* ptr_lse_acc = nullptr; + Stride<_1, int> stride_lse; + Stride<_1, int> stride_lse_acc; + ElementAcc output_scale = 1.0f; + }; + + struct Params { + ProblemShape problem_shape; + MainloopArguments mainloop; + EpilogueParams epilogue; + MainloopParams mainloop_params; + typename TileScheduler::Params tile_scheduler; + int split_kv = -1; + int* ptr_split_kv = nullptr; + }; + + static Params to_underlying_arguments(Arguments const& args, void* workspace) { + //workspace = nullptr; // let's get an error if one of these needs workspace + + auto [H, K, D, B] = args.problem_shape; + auto [L, R] = D; + + int paged_B = B; + int paged_K = K; + if (args.mainloop.ptr_page_table != nullptr) { + paged_B = args.mainloop.page_count; + paged_K = args.mainloop.page_size; + } + + auto params_qk_latent = CollectiveMmaQK::to_underlying_arguments( + make_shape(H, K, L, B), + typename CollectiveMmaQK::Arguments { + args.mainloop.ptr_q_latent, args.mainloop.stride_q_latent, + args.mainloop.ptr_c_latent, args.mainloop.stride_c_latent, + }, nullptr); + + auto params_qk_latent_paged = CollectiveMmaQK::to_underlying_arguments( + make_shape(H, paged_K, L, paged_B), + typename CollectiveMmaQK::Arguments { + args.mainloop.ptr_q_latent, args.mainloop.stride_q_latent, + args.mainloop.ptr_c_latent, args.mainloop.stride_c_latent, + }, nullptr); + + auto params_qk_rope = CollectiveMmaQK::to_underlying_arguments( + make_shape(H, K, R, B), + typename CollectiveMmaQK::Arguments { + args.mainloop.ptr_q_rope, args.mainloop.stride_q_rope, + args.mainloop.ptr_k_rope, args.mainloop.stride_k_rope, + }, nullptr); + + auto params_qk_rope_paged = CollectiveMmaQK::to_underlying_arguments( + make_shape(H, paged_K, R, paged_B), + typename CollectiveMmaQK::Arguments { + args.mainloop.ptr_q_rope, args.mainloop.stride_q_rope, + args.mainloop.ptr_k_rope, args.mainloop.stride_k_rope, + }, nullptr); + + + auto stride_c_latent_transpose = select<1,0,2>(args.mainloop.stride_c_latent); + auto params_pv_latent = CollectiveMmaPV::to_underlying_arguments( + make_shape(H, L, paged_K, paged_B), + typename CollectiveMmaPV::Arguments { + args.mainloop.ptr_q_latent, args.mainloop.stride_q_latent, // dummy, never used + args.mainloop.ptr_c_latent, stride_c_latent_transpose, + }, nullptr); + + MainloopParams mainloop_params { + params_qk_latent.tma_load_a, + params_qk_rope.tma_load_a, + params_qk_latent_paged.tma_load_b, + params_qk_rope_paged.tma_load_b, + params_pv_latent.tma_load_b + }; + + EpilogueParams epilogue_params; + + epilogue_params.ptr_o = args.epilogue.ptr_o; + epilogue_params.stride_o = args.epilogue.stride_o; + epilogue_params.ptr_lse = args.epilogue.ptr_lse; + epilogue_params.stride_lse = args.epilogue.stride_lse; + epilogue_params.output_scale = args.epilogue.output_scale; + + if (args.split_kv > 1) { + ElementAcc* ptr_o_acc = reinterpret_cast(workspace); + ElementLSE* ptr_lse_acc = reinterpret_cast(ptr_o_acc + H * L * args.split_kv * B); + epilogue_params.ptr_o_acc = ptr_o_acc; + epilogue_params.ptr_lse_acc = ptr_lse_acc; + + epilogue_params.stride_o_acc = make_tuple(static_cast(0 + L) * args.split_kv, _1{}, static_cast(0 + H * L) * args.split_kv); + epilogue_params.stride_lse_acc = make_tuple(_1{}, (0 + H) * args.split_kv); + } + + return {args.problem_shape, args.mainloop, epilogue_params, mainloop_params, + TileScheduler::to_underlying_arguments(args.problem_shape, args.hw_info, ClusterShape{}, args.split_kv), args.split_kv, args.ptr_split_kv}; + } + + static size_t get_workspace_size(Arguments const& args) { + ProblemShape problem_shape = args.problem_shape; + auto [H, K, D, B] = problem_shape; + auto [D_latent, D_rope] = D; + auto split_kv = args.split_kv; + return (sizeof(ElementAcc) * D_latent + sizeof(ElementLSE)) * H * split_kv * B; + } + static Status initialize_workspace( + Arguments const& /*args*/, void* /*ws*/, cudaStream_t /*stream*/) { + return Status::kSuccess; + } + + static dim3 get_grid_shape(Params const& params) { + return TileScheduler::get_grid_shape(params.tile_scheduler); + } + + static dim3 get_block_shape() { + dim3 block(MaxThreadsPerBlock, 1, 1); + return block; + } + + static bool can_implement(Arguments const& args) { + if (kIsCpAsync) { + if ((args.mainloop.page_size & (args.mainloop.page_size - 1)) != 0) { + return false; + } + if (args.mainloop.page_size > TileShapeS{}) { + return false; + } + } + else { + if (args.mainloop.ptr_page_table != nullptr && args.mainloop.page_size != TileShapeS{}) { + return false; + } + } + if (get<0>(args.problem_shape) != 128) { + return false; + } + if (get<1>(args.problem_shape) <= 0) { + return false; + } + if (args.split_kv <= 0) { + return false; + } + return true; + } + + + CUTLASS_DEVICE void operator()(Params const& params, char* smem_raw) { + + TileScheduler tile_scheduler(params.tile_scheduler); + + int warp_idx = cutlass::canonical_warp_idx_sync(); + auto role = warp_idx_to_role(warp_idx); + uint32_t lane_predicate = cute::elect_one_sync(); + + uint32_t cta_rank_in_cluster = cute::block_rank_in_cluster(); + int cta_coord_v = cta_rank_in_cluster % size<0>(AtomThrShapeMNK{}); + bool is_mma_leader_cta = cta_coord_v == 0; + + if (role == WarpRole::kLoad && lane_predicate && ! kIsCpAsync) { + prefetch_tma_descriptor(params.mainloop_params.tma_load_q_latent.get_tma_descriptor()); + prefetch_tma_descriptor(params.mainloop_params.tma_load_c_latent.get_tma_descriptor()); + prefetch_tma_descriptor(params.mainloop_params.tma_load_q_rope.get_tma_descriptor()); + prefetch_tma_descriptor(params.mainloop_params.tma_load_k_rope.get_tma_descriptor()); + prefetch_tma_descriptor(params.mainloop_params.tma_load_c_latent_transpose.get_tma_descriptor()); + } + SharedStorage& shared_storage = *reinterpret_cast(smem_raw); + + typename PipelineLoadQK::Params pipeline_load_qk_params; + if (role == WarpRole::kLoad) { + pipeline_load_qk_params.role = PipelineLoadQK::ThreadCategory::Producer; + } + if (role == WarpRole::kMma) { + pipeline_load_qk_params.role = PipelineLoadQK::ThreadCategory::Consumer; + } + if constexpr (kIsCpAsync) { + // we can make our life easier by unconditionally loading blocks + // since we know it'll always be legal + pipeline_load_qk_params.producer_arv_count = kNumLoadWarps * cutlass::NumThreadsPerWarp * size(AtomThrShapeMNK{}); + } + else { + pipeline_load_qk_params.is_leader = lane_predicate && (role == WarpRole::kLoad) && is_mma_leader_cta; + pipeline_load_qk_params.transaction_bytes = kTransactionsBytesLoadQK; + } + pipeline_load_qk_params.initializing_warp = 0; + PipelineLoadQK pipeline_load_qk(shared_storage.pipelines.load_qk, pipeline_load_qk_params, + ClusterShape{}, /*barrier init*/ cute::true_type{}, /*mask calc*/cute::false_type{}); + + typename PipelineS::Params pipeline_mma_s_params; + if (role == WarpRole::kMma) { + pipeline_mma_s_params.role = PipelineS::ThreadCategory::Producer; + } + if (role == WarpRole::kCompute) { + pipeline_mma_s_params.role = PipelineS::ThreadCategory::Consumer; + } + pipeline_mma_s_params.consumer_arv_count = kNumComputeWarps * cutlass::NumThreadsPerWarp * size(AtomThrShapeMNK{}); + pipeline_mma_s_params.initializing_warp = 1; + PipelineS pipeline_mma_s( + shared_storage.pipelines.mma_s, + pipeline_mma_s_params, + ClusterShape{}, /*barrier init*/ cute::true_type{}, /*mask calc*/cute::false_type{}); + + typename PipelineP::Params pipeline_p_mma_params; + if (role == WarpRole::kMma) { + pipeline_p_mma_params.role = PipelineP::ThreadCategory::Consumer; + } + if (role == WarpRole::kCompute) { + pipeline_p_mma_params.role = PipelineP::ThreadCategory::Producer; + } + pipeline_p_mma_params.producer_arv_count = kNumComputeWarps * cutlass::NumThreadsPerWarp * size(AtomThrShapeMNK{}); + pipeline_p_mma_params.consumer_arv_count = 1; + pipeline_p_mma_params.initializing_warp = 2; + PipelineP pipeline_p_mma( + shared_storage.pipelines.p_mma, + pipeline_p_mma_params, + ClusterShape{}, /*barrier init*/ cute::true_type{}, /*mask calc*/cute::false_type{}); + + typename PipelineO::Params pipeline_mma_o_params; + if (role == WarpRole::kMma) { + pipeline_mma_o_params.role = PipelineO::ThreadCategory::Producer; + } + if (role == WarpRole::kCompute) { + pipeline_mma_o_params.role = PipelineO::ThreadCategory::Consumer; + } + pipeline_mma_o_params.consumer_arv_count = kNumComputeWarps * cutlass::NumThreadsPerWarp * size(AtomThrShapeMNK{}); + pipeline_mma_o_params.initializing_warp = 3; + PipelineO pipeline_mma_o( + shared_storage.pipelines.mma_o, + pipeline_mma_o_params, + ClusterShape{}, /*barrier init*/ cute::true_type{}, /*mask calc*/cute::false_type{}); + + typename PipelinePT::Params pipeline_pt_params; + if (role == WarpRole::kLoad) { + pipeline_pt_params.role = PipelinePT::ThreadCategory::Consumer; + } + if (role == WarpRole::kLoadPageTable) { + pipeline_pt_params.role = PipelinePT::ThreadCategory::Producer; + } + pipeline_pt_params.consumer_arv_count = kNumLoadWarps * cutlass::NumThreadsPerWarp; + pipeline_pt_params.producer_arv_count = cutlass::NumThreadsPerWarp; + pipeline_pt_params.initializing_warp = 4; + PipelinePT pipeline_page_table( + shared_storage.pipelines.load_page_table, + pipeline_pt_params); + + TmemAllocator tmem_allocator; + + pipeline_init_arrive_relaxed(size(ClusterShape{})); + + pipeline_load_qk.init_masks(ClusterShape{}); // do we need an update here for 2Sm? + pipeline_mma_s.init_masks(ClusterShape{}); + pipeline_p_mma.init_masks(ClusterShape{}); + pipeline_mma_o.init_masks(ClusterShape{}); + + typename PipelineLoadQK::PipelineState pipeline_load_qk_consumer_state; + typename PipelineLoadQK::PipelineState pipeline_load_qk_producer_state = cutlass::make_producer_start_state(); + + typename PipelineS::PipelineState pipeline_mma_s_consumer_state; + typename PipelineS::PipelineState pipeline_mma_s_producer_state = cutlass::make_producer_start_state(); + + typename PipelineP::PipelineState pipeline_p_mma_consumer_state; + typename PipelineP::PipelineState pipeline_p_mma_producer_state = cutlass::make_producer_start_state(); + + typename PipelineO::PipelineState pipeline_mma_o_consumer_state; + typename PipelineO::PipelineState pipeline_mma_o_producer_state = cutlass::make_producer_start_state(); + + typename PipelinePT::PipelineState pipeline_pt_consumer_state; + typename PipelinePT::PipelineState pipeline_pt_producer_state = cutlass::make_producer_start_state(); + + pipeline_init_wait(size(ClusterShape{})); + + if (role == WarpRole::kLoadPageTable) { + CUTLASS_PRAGMA_NO_UNROLL + for (; tile_scheduler.is_valid(); ++tile_scheduler) { + auto blk_coord = tile_scheduler.get_block_coord(); + auto problem_shape = params.problem_shape; + auto local_split_kv = params.split_kv; + if (params.mainloop.ptr_seq != nullptr) { + get<1>(problem_shape) = params.mainloop.ptr_seq[get<2>(blk_coord)]; + if (params.ptr_split_kv != nullptr) { + local_split_kv = params.ptr_split_kv[get<2>(blk_coord)]; + } + } + if (local_split_kv <= get<3>(blk_coord)) + continue; + load_page_table( + blk_coord, + problem_shape, + params.mainloop, + shared_storage.tensors, + pipeline_page_table, pipeline_pt_producer_state, + local_split_kv + ); + } + } + else if (role == WarpRole::kLoad) { + if constexpr (kIsCpAsync) { + CUTLASS_PRAGMA_NO_UNROLL + for (; tile_scheduler.is_valid(); ++tile_scheduler) { + auto blk_coord = tile_scheduler.get_block_coord(); + auto problem_shape = params.problem_shape; + auto local_split_kv = params.split_kv; + if (params.mainloop.ptr_seq != nullptr) { + get<1>(problem_shape) = params.mainloop.ptr_seq[get<2>(blk_coord)]; + if (params.ptr_split_kv != nullptr) { + local_split_kv = params.ptr_split_kv[get<2>(blk_coord)]; + } + } + if (local_split_kv <= get<3>(blk_coord)) + continue; + load_cpasync( + blk_coord, + problem_shape, + params.mainloop, + params.mainloop_params, + shared_storage.tensors, + pipeline_load_qk, pipeline_load_qk_producer_state, + local_split_kv, + /* must be shared pipe */ + pipeline_page_table, pipeline_pt_consumer_state + ); + cutlass::arch::NamedBarrier((kNumComputeWarps + kNumLoadWarps) * NumThreadsPerWarp, kNamedBarrierEpilogue).arrive_and_wait(); + } + } + else { + if (params.mainloop.ptr_page_table != nullptr) { + CUTLASS_PRAGMA_NO_UNROLL + for (; tile_scheduler.is_valid(); ++tile_scheduler) { + auto blk_coord = tile_scheduler.get_block_coord(); + auto problem_shape = params.problem_shape; + auto local_split_kv = params.split_kv; + if (params.mainloop.ptr_seq != nullptr) { + get<1>(problem_shape) = params.mainloop.ptr_seq[get<2>(blk_coord)]; + if (params.ptr_split_kv != nullptr) { + local_split_kv = params.ptr_split_kv[get<2>(blk_coord)]; + } + } + if (local_split_kv <= get<3>(blk_coord)) + continue; + load_tma( + blk_coord, + problem_shape, + params.mainloop, + params.mainloop_params, + shared_storage.tensors, + pipeline_load_qk, pipeline_load_qk_producer_state, + pipeline_load_qk, pipeline_load_qk_producer_state, + local_split_kv + ); + cutlass::arch::NamedBarrier((kNumComputeWarps + kNumLoadWarps) * NumThreadsPerWarp, kNamedBarrierEpilogue).arrive_and_wait(); + } + } + else { + CUTLASS_PRAGMA_NO_UNROLL + for (; tile_scheduler.is_valid(); ++tile_scheduler) { + auto blk_coord = tile_scheduler.get_block_coord(); + auto problem_shape = params.problem_shape; + auto local_split_kv = params.split_kv; + if (params.mainloop.ptr_seq != nullptr) { + get<1>(problem_shape) = params.mainloop.ptr_seq[get<2>(blk_coord)]; + if (params.ptr_split_kv != nullptr) { + local_split_kv = params.ptr_split_kv[get<2>(blk_coord)]; + } + } + if (local_split_kv <= get<3>(blk_coord)) + continue; + load_tma( + blk_coord, + problem_shape, + params.mainloop, + params.mainloop_params, + shared_storage.tensors, + pipeline_load_qk, pipeline_load_qk_producer_state, + pipeline_load_qk, pipeline_load_qk_producer_state, + local_split_kv + ); + cutlass::arch::NamedBarrier((kNumComputeWarps + kNumLoadWarps) * NumThreadsPerWarp, kNamedBarrierEpilogue).arrive_and_wait(); + } + } + } + } + else if (role == WarpRole::kMma) { + tmem_allocator.allocate(TmemAllocator::Sm100TmemCapacityColumns, &shared_storage.tmem_base_ptr); + __syncwarp(); + + if (is_mma_leader_cta) { + CUTLASS_PRAGMA_NO_UNROLL + for (; tile_scheduler.is_valid(); ++tile_scheduler) { + auto blk_coord = tile_scheduler.get_block_coord(); + auto problem_shape = params.problem_shape; + auto local_split_kv = params.split_kv; + if (params.mainloop.ptr_seq != nullptr) { + get<1>(problem_shape) = params.mainloop.ptr_seq[get<2>(blk_coord)]; + if (params.ptr_split_kv != nullptr) { + local_split_kv = params.ptr_split_kv[get<2>(blk_coord)]; + } + } + if (local_split_kv <= get<3>(blk_coord)) + continue; + mma(blk_coord, + problem_shape, + shared_storage.tensors, + pipeline_load_qk, pipeline_load_qk_consumer_state, + pipeline_load_qk, pipeline_load_qk_consumer_state, + pipeline_mma_s, pipeline_mma_s_producer_state, + pipeline_p_mma, pipeline_p_mma_consumer_state, + pipeline_mma_o, pipeline_mma_o_producer_state, + local_split_kv + ); + } + } + + //cutlass::arch::NamedBarrier((kNumComputeWarps + 1) * NumThreadsPerWarp, kNamedBarrierTmemDealloc).arrive_and_wait(); + + //uint32_t free_stage_ptr = shared_storage.tmem_base_ptr; + //tmem_allocator.free(free_stage_ptr, TmemAllocator::Sm100TmemCapacityColumns); + } + else if (role == WarpRole::kCompute) { + CUTLASS_PRAGMA_NO_UNROLL + for (; tile_scheduler.is_valid(); ++tile_scheduler) { + auto blk_coord = tile_scheduler.get_block_coord(); + auto problem_shape = params.problem_shape; + auto split_kv = params.split_kv; + auto local_split_kv = split_kv; + if (params.mainloop.ptr_seq != nullptr) { + get<1>(problem_shape) = params.mainloop.ptr_seq[get<2>(blk_coord)]; + if (params.ptr_split_kv != nullptr) { + local_split_kv = params.ptr_split_kv[get<2>(blk_coord)]; + } + } + if (local_split_kv <= get<3>(blk_coord)) + continue; + compute( + blk_coord, + problem_shape, + params.mainloop, // for softmax_scale + params.epilogue, + shared_storage.tensors, // for smem_comm + pipeline_mma_s, pipeline_mma_s_consumer_state, + pipeline_p_mma, pipeline_p_mma_producer_state, + pipeline_mma_o, pipeline_mma_o_consumer_state, + local_split_kv + ); + } + + //cutlass::arch::NamedBarrier((kNumComputeWarps + 1) * NumThreadsPerWarp, kNamedBarrierTmemDealloc).arrive(); + } + + cute::cluster_sync(); + cutlass::arch::NamedBarrier((kNumComputeWarps + 1) * NumThreadsPerWarp, kNamedBarrierTmemDealloc).arrive(); + if (role == WarpRole::kMma) { + uint32_t free_stage_ptr = shared_storage.tmem_base_ptr; + tmem_allocator.free(free_stage_ptr, TmemAllocator::Sm100TmemCapacityColumns); + } + } + + template + CUTLASS_DEVICE void load_page_table( + BlkCoord const& blk_coord, + ProblemShape const& problem_shape, + MainloopArguments const& mainloop_args, + TensorStorage& shared_tensors, + PipelinePT& pipeline_page_table, + typename PipelinePT::PipelineState& pipeline_pt_producer_state, int const& split_kv) { + + auto [H, K, D, B] = problem_shape; + int batch_coord = get<2>(blk_coord); + + auto mPT_l = make_tensor(make_gmem_ptr(mainloop_args.ptr_page_table), + make_shape(mainloop_args.page_count, B), + mainloop_args.stride_page_table); + auto mPT = mPT_l(_, batch_coord); + + int k_tile_total = ceil_div(K, TileShapeS{}); + int k_tile_per_cta = ceil_div(k_tile_total, split_kv); + int k_index = get<3>(blk_coord) * k_tile_per_cta; // lower limit + int k_tile_count = max(0, min(k_tile_total, k_index + k_tile_per_cta) - k_index); + if (k_tile_count == 0) { + return; + } + + auto page_size = Pow2{mainloop_args.page_size}; + auto pages_per_tile = Pow2{TileShapeS{} / page_size}; + int thread_idx = threadIdx.x % cutlass::NumThreadsPerWarp; + +#if 1 + for (; k_tile_count > 0; ++k_index, --k_tile_count) { + pipeline_page_table.producer_acquire(pipeline_pt_producer_state); + + // assume a single warp + + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < TileShapeS{}; i += cutlass::NumThreadsPerWarp) { + int idx = i + thread_idx; + bool guard = idx < pages_per_tile; + int smem_idx = pipeline_pt_producer_state.index() * TileShapeS::value + idx; + int pt_idx = pages_per_tile * k_index + idx; + + cutlass::arch::cp_async_zfill( + &shared_tensors.smem_page_table[smem_idx], &mPT(pt_idx), guard + ); + } + + pipeline_page_table.producer_commit(pipeline_pt_producer_state, cutlass::arch::cpasync_barrier_arrive); + ++pipeline_pt_producer_state; + } +#endif + } + + + struct Gather { + int& page_table_stage; + Pow2 pages_per_tile; + const int * __restrict__ smem_page_table; + + CUTLASS_DEVICE int operator()(int idx) const { + return smem_page_table[page_table_stage * TileShapeS::value + idx % pages_per_tile]; + } + + CUTLASS_DEVICE friend void print(Gather const&) { + printf(""); + } + + }; + + + template + CUTLASS_DEVICE void load_cpasync( + BlkCoord const& blk_coord, + ProblemShape const& problem_shape, + MainloopArguments const& mainloop_args, + MainloopParams const& mainloop_params, + TensorStorage& shared_tensors, + PipelineLoadQK& pipeline_load, + typename PipelineLoadQK::PipelineState& pipeline_load_producer_state, + int const& split_kv, + PipelinePT& pipeline_page_table, + typename PipelinePT::PipelineState& pipeline_pt_consumer_state) { + + auto [H, K, D, B] = problem_shape; + auto [D_latent, D_rope] = D; + + using X = Underscore; + + int k_tile_total = ceil_div(K, TileShapeS{}); + int k_tile_per_cta = ceil_div(k_tile_total, split_kv); + int k_index = get<3>(blk_coord) * k_tile_per_cta; // lower limit + int k_tile_count = max(0, min(k_tile_total, k_index + k_tile_per_cta) - k_index); + if (k_tile_count == 0) { + return; + } + + // partition all tensors + auto mQL = make_tensor(make_gmem_ptr(mainloop_args.ptr_q_latent), make_shape(H, D_latent, B), mainloop_args.stride_q_latent); + auto mQR = make_tensor(make_gmem_ptr(mainloop_args.ptr_q_rope), make_shape(H, D_rope, B), mainloop_args.stride_q_rope); + + int paged_B = mainloop_args.page_count; + auto paged_K = Pow2{mainloop_args.page_size}; + auto mPT_l = make_tensor(make_gmem_ptr(mainloop_args.ptr_page_table), make_shape(paged_B, B), mainloop_args.stride_page_table); + + int batch_coord = get<2>(blk_coord); + auto mPT = mPT_l(_, batch_coord); + + auto gQL = local_tile(mQL, TileShapeQK{}, make_coord(_,_,_), Step<_1, X, _1>{}); + auto gQR = local_tile(mQR, TileShapeQK{}, make_coord(_,_,_), Step<_1, X, _1>{}); + + ThrMMA cta_mma_qk = TiledMmaQK{}.get_slice(get<0>(blk_coord) % size(AtomThrShapeMNK{})); + ThrMMA cta_mma_pv = TiledMmaPV{}.get_slice(get<0>(blk_coord) % size(AtomThrShapeMNK{})); + + auto tSgQL = cta_mma_qk.partition_A(gQL); + auto tSgQR = cta_mma_qk.partition_A(gQR); + + Tensor sQ = make_tensor(make_smem_ptr(shared_tensors.smem_q.begin()), SmemLayoutQ{}); + Tensor sKC = make_tensor(make_smem_ptr(shared_tensors.smem_kc.begin()), SmemLayoutKC{}); + Tensor sVC = make_tensor(make_smem_ptr(shared_tensors.smem_vc.begin()), SmemLayoutVC{}); + + auto make_copy_for = [](auto sT) { + auto rT_a = sT.layout()(_, _, _, _0{}); + auto rT = make_ordered_layout(shape(rT_a), stride(rT_a)); + auto threads = Int{}; + auto values = Int{}; + return make_cotiled_copy( + Copy_Atom, Element>{}, + make_ordered_layout( + make_shape(threads, values), + make_stride(_1{}, _0{})), + rT); + }; + + // like cute::copy, but makes sure we do all page table lookups first + auto copy_split = [](auto atom, auto src, auto dst) { + auto src_v = group_modes<1, rank_v>(src); + auto dst_v = group_modes<1, rank_v>(dst); + + auto src_v_ptrs = make_tensor(size<1>(src_v)); + for (int i = 0; i < size<1>(src_v); i++) { + src_v_ptrs(i) = &src_v(_0{}, i); + } + + + for (int i = 0; i < size<1>(src_v); i++) { + auto src_v_i = make_tensor( + make_gmem_ptr(src_v_ptrs(i)), + make_shape(shape<0>(src_v)), + make_stride(make_stride(_1{}, _0{})) + ); + atom.call(src_v_i, dst_v(_, i)); + } + }; + + auto tiled_copy_q = make_copy_for(sQ); + auto tiled_copy_kc = make_copy_for(sKC); + auto tiled_copy_vc = make_copy_for(sVC); + + auto thr_copy_q = tiled_copy_q.get_thread_slice(threadIdx.x % (kNumLoadWarps * cutlass::NumThreadsPerWarp)); + auto thr_copy_kc = tiled_copy_kc.get_thread_slice(threadIdx.x % (kNumLoadWarps * cutlass::NumThreadsPerWarp)); + auto thr_copy_vc = tiled_copy_vc.get_thread_slice(threadIdx.x % (kNumLoadWarps * cutlass::NumThreadsPerWarp)); + + auto tQsQ = thr_copy_q.partition_D(sQ); + auto tQgQL = thr_copy_q.partition_S(tSgQL); + auto tQgQR = thr_copy_q.partition_S(tSgQR); + + auto tKCsKC = thr_copy_kc.partition_D(sKC); + auto tVCsVC = thr_copy_vc.partition_D(sVC); + + auto pipeline_pt_release_state = pipeline_pt_consumer_state; + + int page_table_stage = -1; + Pow2 pages_per_tile{TileShapeS{} / paged_K}; + const int * __restrict__ smem_page_table = shared_tensors.smem_page_table.begin(); + Gather gather{page_table_stage, pages_per_tile, smem_page_table}; + + auto mCL = make_tensor( + make_gmem_ptr(mainloop_args.ptr_c_latent), + ComposedLayout{ + make_layout( + make_shape(make_shape(paged_K, paged_B), _1{}), + make_stride(make_stride(get<0>(mainloop_args.stride_c_latent), example::CustomStride(gather, get<2>(mainloop_args.stride_c_latent))), get<1>(mainloop_args.stride_c_latent))), + make_coord(_0{}, _0{}), + make_identity_layout(make_shape(paged_K * paged_B, D_latent))}); + + auto mKR = make_tensor( + make_gmem_ptr(mainloop_args.ptr_k_rope), + ComposedLayout{ + make_layout( + make_shape(make_shape(paged_K, paged_B), _1{}), + make_stride(make_stride(get<0>(mainloop_args.stride_k_rope), example::CustomStride(gather, get<2>(mainloop_args.stride_k_rope))), get<1>(mainloop_args.stride_k_rope))), + make_coord(_0{}, _0{}), + make_identity_layout(make_shape(paged_K * paged_B, D_latent))}); + + auto mCLT = make_tensor( + make_gmem_ptr(mainloop_args.ptr_c_latent), + ComposedLayout{ + make_layout( + make_shape(_1{}, make_shape(paged_K, paged_B)), + make_stride(get<1>(mainloop_args.stride_c_latent), make_stride(get<0>(mainloop_args.stride_c_latent), example::CustomStride(gather, get<2>(mainloop_args.stride_c_latent))))), + make_coord(_0{}, _0{}), + make_identity_layout(make_shape(D_latent, paged_K * paged_B))}); + + auto gCL = local_tile(mCL, TileShapeQK{}, make_coord(_,_,_), Step{}); + auto gKR = local_tile(mKR, TileShapeQK{}, make_coord(_,_,_), Step{}); + auto gCLT = local_tile(mCLT, TileShapePV{}, make_coord(_,_,_), Step{}); + + auto tSgCL = cta_mma_qk.partition_B(gCL); + auto tSgKR = cta_mma_qk.partition_B(gKR); + auto tOgCLT = cta_mma_pv.partition_B(gCLT); + + auto tKCgCL = thr_copy_kc.partition_S(tSgCL); + auto tKCgKR = thr_copy_kc.partition_S(tSgKR); + auto tVCgCLT = thr_copy_vc.partition_S(tOgCLT); + + // latent is first in memory, so let's load it first always + // startup: alternate Q and K, set tx count appropriately, for k_idx = 0 + auto& pipeline_acquire_state = pipeline_load_producer_state; + auto pipeline_commit_state = pipeline_acquire_state; + int pipeline_offset = 0; + + for (int i = 0; i < StagesPV; i++) { + cutlass::arch::cp_async_fence(); + } + + auto load_stage = [&](auto fn) { + pipeline_load.producer_acquire(pipeline_acquire_state); + fn(pipeline_acquire_state.index()); + cutlass::arch::cp_async_fence(); + + ++pipeline_acquire_state; + ++pipeline_offset; + + if (pipeline_offset == StagesPV - 1) { + cutlass::arch::cp_async_wait(); + pipeline_load.producer_commit(pipeline_commit_state); + ++pipeline_commit_state; + --pipeline_offset; + } + }; + + pipeline_page_table.consumer_wait(pipeline_pt_consumer_state); + page_table_stage = pipeline_pt_consumer_state.index(); + ++pipeline_pt_consumer_state; + + // each Q/K tile consists of rope and latent + for (int i = 0; i < IterationsQKLatent; i++) { + load_stage([&](int index) { + cute::copy(tiled_copy_q, tQgQL(_, _, _, _, _0{}, i, batch_coord), tQsQ(_, _, _, _, i)); + copy_split(tiled_copy_kc, tKCgCL(_, _, _, _, k_index, i), tKCsKC(_, _, _, _, index)); + }); + } + + for (int i = 0; i < IterationsQKRope; i++) { + load_stage([&](int index) { + cute::copy(tiled_copy_q, tQgQR(_, _, _, _, _0{}, i, batch_coord), tQsQ(_, _, _, _, IterationsQKLatent + i)); + copy_split(tiled_copy_kc, tKCgKR(_, _, _, _, k_index, i), tKCsKC(_, _, _, _, index)); + }); + } + + k_index += 1; + k_tile_count -= 1; + + // assume k_tile_count >= 1 + // perform K+Q load here + CUTLASS_PRAGMA_NO_UNROLL + while (k_tile_count > 0) { + + pipeline_page_table.consumer_wait(pipeline_pt_consumer_state); + page_table_stage = pipeline_pt_consumer_state.index(); + ++pipeline_pt_consumer_state; + + for (int i = 0; i < IterationsQKLatent; i++) { + load_stage([&](int index) { + copy_split(tiled_copy_kc, tKCgCL(_, _, _, _, k_index, i), tKCsKC(_, _, _, _, index)); + }); + } + + for (int i = 0; i < IterationsQKRope; i++) { + load_stage([&](int index) { + copy_split(tiled_copy_kc, tKCgKR(_, _, _, _, k_index, i), tKCsKC(_, _, _, _, index)); + }); + } + + page_table_stage = pipeline_pt_release_state.index(); + + for (int i = 0; i < IterationsPV_K; i++) { + for (int j = 0; j < IterationsPV_N; j++) { + load_stage([&](int index) { + copy_split(tiled_copy_vc, tVCgCLT(_, _, _, _, j, IterationsPV_K * (k_index - 1) + i), tVCsVC(_, _, _, _, index)); + }); + } + } + + pipeline_page_table.consumer_release(pipeline_pt_release_state); + ++pipeline_pt_release_state; + + k_index += 1; + k_tile_count -= 1; + } + + page_table_stage = pipeline_pt_release_state.index(); + + for (int i = 0; i < IterationsPV_K; i++) { + for (int j = 0; j < IterationsPV_N; j++) { + load_stage([&](int index) { + copy_split(tiled_copy_vc, tVCgCLT(_, _, _, _, j, IterationsPV_K * (k_index - 1) + i), tVCsVC(_, _, _, _, index)); + }); + } + } + + pipeline_page_table.consumer_release(pipeline_pt_release_state); + ++pipeline_pt_release_state; + + while (pipeline_offset > 0) { + cutlass::arch::cp_async_fence(); + + cutlass::arch::cp_async_wait(); + pipeline_load.producer_commit(pipeline_commit_state); + ++pipeline_commit_state; + --pipeline_offset; + } + + cutlass::arch::cp_async_wait<0>(); + + } + + + template + CUTLASS_DEVICE void load_tma( + BlkCoord const& blk_coord, + ProblemShape const& problem_shape, + MainloopArguments const& mainloop_args, + MainloopParams const& mainloop_params, + TensorStorage& shared_tensors, + PipelineLoadQK& pipeline_load_qk, + typename PipelineLoadQK::PipelineState& pipeline_load_qk_producer_state, + PipelineLoadPV& pipeline_load_pv, + typename PipelineLoadPV::PipelineState& pipeline_load_pv_producer_state, + int const& split_kv) { + + auto [H, K, D, B] = problem_shape; + auto [D_latent, D_rope] = D; + + int k_tile_total = ceil_div(K, TileShapeS{}); + int k_tile_per_cta = ceil_div(k_tile_total, split_kv); + int k_index = get<3>(blk_coord) * k_tile_per_cta; // lower limit + int k_tile_count = max(0, min(k_tile_total, k_index + k_tile_per_cta) - k_index); + if (k_tile_count == 0) { + return; + } + + using X = Underscore; + + // partition all tensors + auto mQL = mainloop_params.tma_load_q_latent.get_tma_tensor(make_shape(H, D_latent, B)); + auto mQR = mainloop_params.tma_load_q_rope.get_tma_tensor(make_shape(H, D_rope, B)); + + int paged_B = B; + int paged_K = K; + if constexpr (kIsPaged) { + paged_B = mainloop_args.page_count; + paged_K = mainloop_args.page_size; + } + auto mPT_l = make_tensor(make_gmem_ptr(mainloop_args.ptr_page_table), make_shape(paged_B, B), mainloop_args.stride_page_table); + + auto mCL = mainloop_params.tma_load_c_latent.get_tma_tensor(make_shape(paged_K, D_latent, paged_B)); + auto mKR = mainloop_params.tma_load_k_rope.get_tma_tensor(make_shape(paged_K, D_rope, paged_B)); + + auto mCLT = mainloop_params.tma_load_c_latent_transpose.get_tma_tensor(make_shape(D_latent, paged_K, paged_B)); + + auto gQL = local_tile(mQL, TileShapeQK{}, make_coord(_,_,_), Step<_1, X, _1>{}); + auto gQR = local_tile(mQR, TileShapeQK{}, make_coord(_,_,_), Step<_1, X, _1>{}); + + auto gCL = local_tile(mCL, TileShapeQK{}, make_coord(_,_,_), Step{}); + auto gKR = local_tile(mKR, TileShapeQK{}, make_coord(_,_,_), Step{}); + auto gCLT = local_tile(mCLT, TileShapePV{}, make_coord(_,_,_), Step{}); + + ThrMMA cta_mma_qk = TiledMmaQK{}.get_slice(get<0>(blk_coord) % size(AtomThrShapeMNK{})); + ThrMMA cta_mma_pv = TiledMmaPV{}.get_slice(get<0>(blk_coord) % size(AtomThrShapeMNK{})); + + auto tSgQL = cta_mma_qk.partition_A(gQL); + auto tSgQR = cta_mma_qk.partition_A(gQR); + + auto tSgCL = cta_mma_qk.partition_B(gCL); + auto tSgKR = cta_mma_qk.partition_B(gKR); + + auto tOgCLT = cta_mma_pv.partition_B(gCLT); + + Tensor sQ = make_tensor(make_smem_ptr(shared_tensors.smem_q.begin()), SmemLayoutQ{}); + Tensor sKC = make_tensor(make_smem_ptr(shared_tensors.smem_kc.begin()), SmemLayoutKC{}); + Tensor sVC = make_tensor(make_smem_ptr(shared_tensors.smem_vc.begin()), SmemLayoutVC{}); + + auto [tQLgQL_mkl, tQsQ] = tma_partition( + mainloop_params.tma_load_q_latent, _0{}, make_layout(_1{}), + group_modes<0,3>(sQ), group_modes<0,3>(tSgQL)); + + auto [tQRgQR_mkl, tQsQ_ignore] = tma_partition( + mainloop_params.tma_load_q_rope, _0{}, make_layout(_1{}), + group_modes<0,3>(sQ), group_modes<0,3>(tSgQR)); + + auto [tCLgCL_nkl, tKCsKC] = tma_partition( + mainloop_params.tma_load_c_latent, _0{}, make_layout(_1{}), + group_modes<0,3>(sKC), group_modes<0,3>(tSgCL)); + + auto [tKRgKR_nkl, tKCsKC_ignore] = tma_partition( + mainloop_params.tma_load_k_rope, _0{}, make_layout(_1{}), + group_modes<0,3>(sKC), group_modes<0,3>(tSgKR)); + + auto [tCLTgCLT_nkl, tVCsVC] = tma_partition( + mainloop_params.tma_load_c_latent_transpose, _0{}, make_layout(_1{}), + group_modes<0,3>(sVC), group_modes<0,3>(tOgCLT)); + + uint16_t mcast_mask = 0; + + int batch_coord = get<2>(blk_coord); + Tensor tQLgQL = tQLgQL_mkl(_, _, _, batch_coord); + Tensor tQRgQR = tQRgQR_mkl(_, _, _, batch_coord); + + auto mPT = mPT_l(_, batch_coord); + + Tensor tCLgCL = tCLgCL_nkl(_, _, _, _); + Tensor tKRgKR = tKRgKR_nkl(_, _, _, _); + + // careful: stage and k are swapped here! + Tensor tCLTgCLT = tCLTgCLT_nkl(_, _, _, _); + + // latent is first in memory, so let's load it first always + // startup: alternate Q and K, set tx count appropriately, for k_idx = 0 + + // each Q/K tile consists of rope and latent + for (int i = 0; i < IterationsQKLatent; i++) { + pipeline_load_qk.producer_expect_transaction(pipeline_load_qk_producer_state, kTransactionsBytesLoadExtraQ); + pipeline_load_qk.producer_acquire(pipeline_load_qk_producer_state); + auto tma_barrier = pipeline_load_qk.producer_get_barrier(pipeline_load_qk_producer_state); + + if (cute::elect_one_sync()) { + // expect the extra bytes + // load_qk ql + cute::copy(mainloop_params.tma_load_q_latent.with(*tma_barrier, mcast_mask), tQLgQL(_, _0{}, i), tQsQ(_, i)); + // load_qk cl + if constexpr (kIsPaged) { + cute::copy( + mainloop_params.tma_load_c_latent.with(*tma_barrier, mcast_mask), + tCLgCL(_, _0{}, i, mPT(k_index)), + tKCsKC(_, pipeline_load_qk_producer_state.index()) + ); + } + else { + cute::copy( + mainloop_params.tma_load_c_latent.with(*tma_barrier, mcast_mask), + tCLgCL(_, k_index, i, batch_coord), + tKCsKC(_, pipeline_load_qk_producer_state.index())); + } + } + ++pipeline_load_qk_producer_state; + } + + for (int i = 0; i < IterationsQKRope; i++) { + pipeline_load_qk.producer_expect_transaction(pipeline_load_qk_producer_state, kTransactionsBytesLoadExtraQ); + pipeline_load_qk.producer_acquire(pipeline_load_qk_producer_state); + auto tma_barrier = pipeline_load_qk.producer_get_barrier(pipeline_load_qk_producer_state); + + if (cute::elect_one_sync()) { + // expect the extra bytes + // load_qk ql + cute::copy(mainloop_params.tma_load_q_rope.with(*tma_barrier, mcast_mask), tQRgQR(_, _0{}, i), tQsQ(_, i + IterationsQKLatent)); + // load_qk cl + if constexpr (kIsPaged) { + cute::copy( + mainloop_params.tma_load_k_rope.with(*tma_barrier, mcast_mask), + tKRgKR(_, _0{}, i, mPT(k_index)), + tKCsKC(_, pipeline_load_qk_producer_state.index()) + ); + } + else { + cute::copy( + mainloop_params.tma_load_k_rope.with(*tma_barrier, mcast_mask), + tKRgKR(_, k_index, i, batch_coord), + tKCsKC(_, pipeline_load_qk_producer_state.index())); + } + } + ++pipeline_load_qk_producer_state; + } + + k_index += 1; + k_tile_count -= 1; + + // assume k_tile_count >= 1 + // perform K+Q load here + CUTLASS_PRAGMA_NO_UNROLL + while (k_tile_count > 0) { + + // perform K load + for (int i = 0; i < IterationsQKLatent; i++) { + pipeline_load_qk.producer_acquire(pipeline_load_qk_producer_state); + auto tma_barrier = pipeline_load_qk.producer_get_barrier(pipeline_load_qk_producer_state); + + if (cute::elect_one_sync()) { + // load_qk cl + if constexpr (kIsPaged) { + cute::copy( + mainloop_params.tma_load_c_latent.with(*tma_barrier, mcast_mask), + tCLgCL(_, _0{}, i, mPT(k_index)), + tKCsKC(_, pipeline_load_qk_producer_state.index()) + ); + } + else { + cute::copy( + mainloop_params.tma_load_c_latent.with(*tma_barrier, mcast_mask), + tCLgCL(_, k_index, i, batch_coord), + tKCsKC(_, pipeline_load_qk_producer_state.index())); + } + } + ++pipeline_load_qk_producer_state; + } + + for (int i = 0; i < IterationsQKRope; i++) { + pipeline_load_qk.producer_acquire(pipeline_load_qk_producer_state); + auto tma_barrier = pipeline_load_qk.producer_get_barrier(pipeline_load_qk_producer_state); + + if (cute::elect_one_sync()) { + // load_qk cl + if constexpr (kIsPaged) { + cute::copy( + mainloop_params.tma_load_k_rope.with(*tma_barrier, mcast_mask), + tKRgKR(_, _0{}, i, mPT(k_index)), + tKCsKC(_, pipeline_load_qk_producer_state.index()) + ); + } + else { + cute::copy( + mainloop_params.tma_load_k_rope.with(*tma_barrier, mcast_mask), + tKRgKR(_, k_index, i, batch_coord), + tKCsKC(_, pipeline_load_qk_producer_state.index())); + } + } + ++pipeline_load_qk_producer_state; + } + + // prefetch next K load to keep busy while we transpose-load from cache + const int kPrefetchDistance = 1; + for (int i = 0; i < IterationsQKLatent; i++) { + if (cute::elect_one_sync()) { + if constexpr (kIsPaged) { + if (k_tile_count > kPrefetchDistance) { + cute::prefetch( + mainloop_params.tma_load_c_latent, + tCLgCL(_, _0{}, i, mPT(k_index + kPrefetchDistance)) + ); + } + } + else { + cute::prefetch( + mainloop_params.tma_load_c_latent, + tCLgCL(_, k_index + kPrefetchDistance, i, batch_coord) + ); + } + } + } + + for (int i = 0; i < IterationsQKRope; i++) { + if (cute::elect_one_sync()) { + if constexpr (kIsPaged) { + if (k_tile_count > kPrefetchDistance) { + cute::prefetch( + mainloop_params.tma_load_k_rope, + tKRgKR(_, _0{}, i, mPT(k_index + kPrefetchDistance)) + ); + } + } + else { + cute::prefetch( + mainloop_params.tma_load_k_rope, + tKRgKR(_, k_index + kPrefetchDistance, i, batch_coord) + ); + } + } + } + + // perform V load (k_idx - 1) + + for (int i = 0; i < IterationsPV_K; i++) { + for (int j = 0; j < IterationsPV_N; j++) { + pipeline_load_pv.producer_acquire(pipeline_load_pv_producer_state); + auto tma_barrier = pipeline_load_pv.producer_get_barrier(pipeline_load_pv_producer_state); + + if (cute::elect_one_sync()) { + // load_pv cl + // note the transpose in indices! + // note we are off-by-one on k_index + if constexpr (kIsPaged) { + cute::copy( + mainloop_params.tma_load_c_latent_transpose.with(*tma_barrier, mcast_mask, cute::TMA::CacheHintSm100::EVICT_FIRST), + tCLTgCLT(_, j, i, mPT(k_index - 1)), + tVCsVC(_, pipeline_load_pv_producer_state.index()) + ); + } + else { + cute::copy( + mainloop_params.tma_load_c_latent_transpose.with(*tma_barrier, mcast_mask, cute::TMA::CacheHintSm100::EVICT_FIRST), + tCLTgCLT(_, j, IterationsPV_K * (k_index - 1) + i, batch_coord), + tVCsVC(_, pipeline_load_pv_producer_state.index()) + ); + } + } + ++pipeline_load_pv_producer_state; + } + } + + k_index += 1; + k_tile_count -= 1; + } + + for (int i = 0; i < IterationsPV_K; i++) { + for (int j = 0; j < IterationsPV_N; j++) { + pipeline_load_pv.producer_acquire(pipeline_load_pv_producer_state); + auto tma_barrier = pipeline_load_pv.producer_get_barrier(pipeline_load_pv_producer_state); + + if (cute::elect_one_sync()) { + // load_pv cl + // note the transpose in indices + // note we are off-by-one on k_index + + if constexpr (kIsPaged) { + cute::copy( + mainloop_params.tma_load_c_latent_transpose.with(*tma_barrier, mcast_mask, cute::TMA::CacheHintSm100::EVICT_FIRST), + tCLTgCLT(_, j, i, mPT(k_index - 1)), + tVCsVC(_, pipeline_load_pv_producer_state.index()) + ); + } + else { + cute::copy( + mainloop_params.tma_load_c_latent_transpose.with(*tma_barrier, mcast_mask, cute::TMA::CacheHintSm100::EVICT_FIRST), + tCLTgCLT(_, j, IterationsPV_K * (k_index - 1) + i, batch_coord), + tVCsVC(_, pipeline_load_pv_producer_state.index()) + ); + } + } + ++pipeline_load_pv_producer_state; + } + } + } + + template + CUTLASS_DEVICE void mma( + BlkCoord const& blk_coord, + ProblemShape const& problem_shape, + TensorStorage& shared_tensors, + PipelineLoadQK& pipeline_load_qk, + typename PipelineLoadQK::PipelineState& pipeline_load_qk_consumer_state, + PipelineLoadPV& pipeline_load_pv, + typename PipelineLoadPV::PipelineState& pipeline_load_pv_consumer_state, + PipelineS& pipeline_mma_s, + typename PipelineS::PipelineState& pipeline_mma_s_producer_state, + PipelineP& pipeline_p_mma, + typename PipelineP::PipelineState& pipeline_p_mma_consumer_state, + PipelineO& pipeline_mma_o, + typename PipelineO::PipelineState& pipeline_mma_o_producer_state, + int const& split_kv) { + + auto [H, K, D, B] = problem_shape; + + int k_tile_total = ceil_div(K, TileShapeS{}); + int k_tile_per_cta = ceil_div(k_tile_total, split_kv); + int k_index = get<3>(blk_coord) * k_tile_per_cta; // lower limit + int k_tile_count = max(0, min(k_tile_total, k_index + k_tile_per_cta) - k_index); + if (k_tile_count == 0) { + return; + } + + // mma init + Tensor sQ = make_tensor(make_smem_ptr(shared_tensors.smem_q.begin()), SmemLayoutQ{}); + Tensor sKC = make_tensor(make_smem_ptr(shared_tensors.smem_kc.begin()), SmemLayoutKC{}); + Tensor sVC = make_tensor(make_smem_ptr(shared_tensors.smem_vc.begin()), SmemLayoutVC{}); + Tensor sP = make_tensor(make_smem_ptr((Element*) shared_tensors.smem_p.begin()), SmemLayoutP{}); + + Tensor tSrQ = TiledMmaQK::make_fragment_A(sQ); + Tensor tSrKC = TiledMmaQK::make_fragment_B(sKC); + Tensor tOrP = TiledMmaPV::make_fragment_A(sP); + Tensor tOrVC = TiledMmaPV::make_fragment_B(sVC); + + TiledMmaQK tiled_mma_qk; + TiledMmaPV tiled_mma_pv; + + Tensor tStS = partition_fragment_C(tiled_mma_qk, select<0,1>(TileShapeQK{})); + Tensor tOtO = partition_fragment_C(tiled_mma_pv, select<0,1>(TileShapePV{})); + + tiled_mma_pv.accumulate_ = UMMA::ScaleOut::Zero; + + pipeline_mma_s.producer_acquire(pipeline_mma_s_producer_state); + + // Mma S0 S1 O0 S2 O1 ... Sn On-1 On + // S0 ownership -- ----- -- -- + // S1 ownership -- ----- ---- + // O ownership -- -- ---- -- + + tiled_mma_qk.accumulate_ = UMMA::ScaleOut::Zero; + for (int i = 0; i < IterationsQK; i++) { + pipeline_load_qk.consumer_wait(pipeline_load_qk_consumer_state); + int read_stage = pipeline_load_qk_consumer_state.index(); + + tStS.data() = uint32_t(pipeline_mma_s_producer_state.index() == 0 ? TmemAllocation::kS0 : TmemAllocation::kS1); + + CUTLASS_PRAGMA_UNROLL + for (int k_block = 0; k_block < size<2>(tSrQ); ++k_block) { + cute::gemm(tiled_mma_qk, + tSrQ(_,_,k_block,i), + tSrKC(_,_,k_block,read_stage), + tStS); + tiled_mma_qk.accumulate_ = UMMA::ScaleOut::One; + } + + pipeline_load_qk.consumer_release(pipeline_load_qk_consumer_state); + ++pipeline_load_qk_consumer_state; + } + + pipeline_mma_s.producer_commit(pipeline_mma_s_producer_state); + ++pipeline_mma_s_producer_state; + + k_tile_count -= 1; + + CUTLASS_PRAGMA_NO_UNROLL + while (k_tile_count > 0) { + + pipeline_mma_s.producer_acquire(pipeline_mma_s_producer_state); + tiled_mma_qk.accumulate_ = UMMA::ScaleOut::Zero; + for (int i = 0; i < IterationsQK; i++) { + pipeline_load_qk.consumer_wait(pipeline_load_qk_consumer_state); + int read_stage = pipeline_load_qk_consumer_state.index(); + + tStS.data() = uint32_t(pipeline_mma_s_producer_state.index() == 0 ? TmemAllocation::kS0 : TmemAllocation::kS1); + + CUTLASS_PRAGMA_UNROLL + for (int k_block = 0; k_block < size<2>(tSrQ); ++k_block) { + cute::gemm(tiled_mma_qk, + tSrQ(_,_,k_block,i), + tSrKC(_,_,k_block,read_stage), + tStS); + tiled_mma_qk.accumulate_ = UMMA::ScaleOut::One; + } + + pipeline_load_qk.consumer_release(pipeline_load_qk_consumer_state); + ++pipeline_load_qk_consumer_state; + } + + pipeline_mma_s.producer_commit(pipeline_mma_s_producer_state); + ++pipeline_mma_s_producer_state; + + pipeline_mma_o.producer_acquire(pipeline_mma_o_producer_state); + pipeline_p_mma.consumer_wait(pipeline_p_mma_consumer_state); + + for (int i = 0; i < IterationsPV_K; i++) { + auto acc_flag = tiled_mma_pv.accumulate_; + for (int j = 0; j < IterationsPV_N; j++) { + pipeline_load_pv.consumer_wait(pipeline_load_pv_consumer_state); + + int read_stage = pipeline_load_pv_consumer_state.index(); + + tOtO.data() = uint32_t(TmemAllocation::kO0) + j * uint32_t(TmemAllocation::kSizeAccO); + tiled_mma_pv.accumulate_ = acc_flag; + + CUTLASS_PRAGMA_UNROLL + for (int k_block = 0; k_block < size<2>(tOrP); ++k_block) { + cute::gemm(tiled_mma_pv, + tOrP(_,_,k_block, make_coord(i, pipeline_p_mma_consumer_state.index())), + tOrVC(_,_,k_block,read_stage), + tOtO); + tiled_mma_pv.accumulate_ = UMMA::ScaleOut::One; + } + + pipeline_load_pv.consumer_release(pipeline_load_pv_consumer_state); + ++pipeline_load_pv_consumer_state; + } + } + + pipeline_p_mma.consumer_release(pipeline_p_mma_consumer_state); + ++pipeline_p_mma_consumer_state; + pipeline_mma_o.producer_commit(pipeline_mma_o_producer_state); + ++pipeline_mma_o_producer_state; + + --k_tile_count; + } + + pipeline_mma_o.producer_acquire(pipeline_mma_o_producer_state); + pipeline_p_mma.consumer_wait(pipeline_p_mma_consumer_state); + + for (int i = 0; i < IterationsPV_K; i++) { + auto acc_flag = tiled_mma_pv.accumulate_; + for (int j = 0; j < IterationsPV_N; j++) { + pipeline_load_pv.consumer_wait(pipeline_load_pv_consumer_state); + + int read_stage = pipeline_load_pv_consumer_state.index(); + + tOtO.data() = uint32_t(TmemAllocation::kO0) + j * uint32_t(TmemAllocation::kSizeAccO); + tiled_mma_pv.accumulate_ = acc_flag; + + CUTLASS_PRAGMA_UNROLL + for (int k_block = 0; k_block < size<2>(tOrP); ++k_block) { + cute::gemm(tiled_mma_pv, + tOrP(_,_,k_block, make_coord(i, pipeline_p_mma_consumer_state.index())), + tOrVC(_,_,k_block,read_stage), + tOtO); + tiled_mma_pv.accumulate_ = UMMA::ScaleOut::One; + } + + pipeline_load_pv.consumer_release(pipeline_load_pv_consumer_state); + ++pipeline_load_pv_consumer_state; + } + } + + pipeline_p_mma.consumer_release(pipeline_p_mma_consumer_state); + ++pipeline_p_mma_consumer_state; + pipeline_mma_o.producer_commit(pipeline_mma_o_producer_state); + ++pipeline_mma_o_producer_state; + } + + + template + CUTLASS_DEVICE void softmax( + IsLastTile const& is_last_tile, + ElementAcc& row_max, + ElementAcc& row_sum, + ElementAcc& correction_factor, + ProblemShape const& problem_shape, + MainloopArguments const& mainloop_args, + TensorStorage& shared_tensors, + int k_index, + uint32_t tmem_s, + int smem_p_index) { + + auto load_op = cute::SM100_TMEM_LOAD_32dp32b32x{}; + + TiledMmaQK tiled_mma_qk; + + Tensor tStS = partition_fragment_C(tiled_mma_qk, select<0,1>(TileShapeQK{})); + tStS.data() = tmem_s; + + CUTE_STATIC_ASSERT_V(shape<1>(tStS) == _1{}); + CUTE_STATIC_ASSERT_V(shape<2>(tStS) == _1{}); + Tensor tAcc = tStS(make_coord(_,_),_0{},_0{}); + + Tensor cS = make_identity_tensor(take<0,2>(CtaShapeQK{})); + + auto tiled_t2r = make_tmem_copy(load_op, tAcc); + auto thread_idx = threadIdx.x % size(tiled_t2r); + + auto thread_t2r = tiled_t2r.get_slice(thread_idx); + Tensor tTR_cS = thread_t2r.partition_D(cS); + Tensor tTR_rAcc = make_tensor(shape(tTR_cS)); + + Tensor tTR_rS_frag = make_tensor(shape(tTR_rAcc)); + const int AlignmentS = 4; + Tensor tTR_tAcc = thread_t2r.partition_S(tAcc); + Tensor tTR_rAcc_vec = recast>(tTR_rAcc); + Tensor tTR_rS_vec = recast>(tTR_rS_frag); + + // load s + copy(tiled_t2r, tTR_tAcc, tTR_rAcc); + + if (is_last_tile) { + for (int i = 0; i < size(tTR_rAcc); i++) { + if (get<1>(tTR_cS(i)) + TileShapeS{} * k_index >= get<1>(problem_shape)) { + tTR_rAcc(i) = -std::numeric_limits::infinity(); + } + } + } + + // max + ElementAcc row_max_new = row_max; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size(tTR_rAcc); i += 1) { + row_max_new = ::fmax(row_max_new, tTR_rAcc(i)); + } + + // for 2x2 dp, reduce here + if constexpr (kWarpsInN > 1) { + shared_tensors.smem_exchange[threadIdx.x] = row_max_new; + cutlass::arch::NamedBarrier(kNumComputeWarps*NumThreadsPerWarp, kNamedBarrierExchange).sync(); + // (64, 2) shape + int peer_index = (threadIdx.x + 64) % 128; + row_max_new = cutlass::max(row_max_new, shared_tensors.smem_exchange[peer_index]); + } + +#ifndef B2B + // find correction factor + ElementAcc softmax_scale_log2 = mainloop_args.softmax_scale * static_cast(M_LOG2E); + correction_factor = ::exp2f(softmax_scale_log2 * (row_max - row_max_new)); + row_max = row_max_new; + + // softmax + ElementAcc row_max_scale_log2 = row_max * softmax_scale_log2; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size(tTR_rAcc); i++) { + tTR_rAcc(i) = ::exp2f(softmax_scale_log2 * tTR_rAcc(i) - row_max_scale_log2); + } +#endif + + // quantize + cutlass::NumericArrayConverter epilogue_op; + + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size(tTR_rAcc_vec); i++) { + tTR_rS_vec(i) = epilogue_op(tTR_rAcc_vec(i)); + } + + Tensor sP = make_tensor(make_smem_ptr((Element*) shared_tensors.smem_p.begin()), SmemLayoutP{})(_, _, _, make_coord(_, smem_p_index)); + + Tensor tOcP = TiledMmaPV{}.get_slice(_0{}).partition_A(cS); + + // have a mapping for each thread to coord + // find identical mapping to coords for the MMA + auto l = make_ordered_layout(make_shape(make_shape(_64{}, _2{}), make_shape(_16{}, TileShapeS{} / _32{})), make_stride(make_stride(_0{}, _3{}), make_stride(_1{}, _2{}))); + auto sP_ = as_position_independent_swizzle_tensor(sP); + copy_aligned(tTR_rS_frag, sP_.compose(l)(threadIdx.x, _)); + + // sum + row_sum *= correction_factor; + + static_assert(cute::is_same_v); + auto tTR_rAcc_float2 = recast(tTR_rAcc); + auto sums = make_tensor(_4{}); + static_assert(size(tTR_rAcc_float2) % size(sums) == 0); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size(sums); i++) { + sums(i) = tTR_rAcc_float2(i); + } + CUTLASS_PRAGMA_UNROLL + for (int i = size(sums); i < size(tTR_rAcc_float2); i += size(sums)) { + CUTLASS_PRAGMA_UNROLL + for (int j = 0; j < size(sums); j++) { + cute::add(sums(j), sums(j), tTR_rAcc_float2(i + j)); + } + } + CUTLASS_PRAGMA_UNROLL + for (int i = 1; i < size(sums); i *= 2) { + CUTLASS_PRAGMA_UNROLL + for (int j = 0; j < size(sums); j += 2*i) { + cute::add(sums(j), sums(j), sums(j+i)); + } + } + row_sum += sums(0).x + sums(0).y; + } + + + CUTLASS_DEVICE void rescale( + ElementAcc correction_factor, + uint32_t tmem_o) { + + // for b2b gemm, do nothing +#ifndef B2B + auto load_op = cute::SM100_TMEM_LOAD_32dp32b32x{}; + auto store_op = TMEM::tmem_load_to_store(load_op); + + TiledMmaPV tiled_mma_pv; + + Tensor tOtO = partition_fragment_C(tiled_mma_pv, select<0,1>(TileShapePV{})); + tOtO.data() = tmem_o; + + CUTE_STATIC_ASSERT_V(shape<1>(tOtO) == _1{}); + CUTE_STATIC_ASSERT_V(shape<2>(tOtO) == _1{}); + Tensor tAcc = tOtO(make_coord(_,_),_0{},_0{}); + + auto cta_tiler_pv = take<0,2>(typename CollectiveMmaPV::CtaShape_MNK{}); + Tensor gO = make_tensor(make_gmem_ptr((ElementAcc*) nullptr), cta_tiler_pv, make_stride(0, 0)); + + auto tiled_t2r = make_tmem_copy(load_op, tAcc); + auto tiled_r2t = make_tmem_copy(store_op, tAcc); + auto thread_idx = threadIdx.x % size(tiled_t2r); + + auto thread_t2r = tiled_t2r.get_slice(thread_idx); + auto thread_r2t = tiled_r2t.get_slice(thread_idx); + Tensor tTR_gO = thread_t2r.partition_D(gO); + Tensor tTR_rAcc = make_tensor(shape(tTR_gO)); + + Tensor tTR_tAcc = thread_t2r.partition_S(tAcc); + + // load o + copy(tiled_t2r, tTR_tAcc, tTR_rAcc); + + // multiply by correction factor + float2 correction_factor_vec = make_float2(correction_factor, correction_factor); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size(tTR_rAcc); i += 2) { + float2 in = make_float2(tTR_rAcc(i + 0), tTR_rAcc(i + 1)); + float2 out; + cute::mul(out, in, correction_factor_vec); + tTR_rAcc(i + 0) = out.x; + tTR_rAcc(i + 1) = out.y; + } + + // store o + copy(tiled_r2t, tTR_rAcc, tTR_tAcc); +#endif + } + + + template + CUTLASS_DEVICE void epilogue( + ElementAcc& row_max, + ElementAcc& row_sum, + BlkCoord const& cta_coord, + ProblemShape const& problem_shape, + MainloopArguments const& mainloop_args, + EpilogueParams const& epilogue_args, + TensorStorage& shared_tensors, + uint32_t tmem_o, + int const& split_kv) { + + auto load_op = cute::SM100_TMEM_LOAD_32dp32b32x{}; + + TiledMmaPV tiled_mma_pv; + + Tensor tOtO = TiledMmaPV::make_fragment_C(partition_shape_C(TiledMmaPV{}, take<0, 2>(TileShapePV{}))); + tOtO.data() = tmem_o; + + CUTE_STATIC_ASSERT_V(shape<1>(tOtO) == _1{}); + CUTE_STATIC_ASSERT_V(shape<2>(tOtO) == _1{}); + Tensor tAcc = tOtO(make_coord(_,_),_0{},_0{}); + + auto [H, K, D, B] = problem_shape; + auto [D_latent, D_rope] = D; + if (epilogue_args.ptr_o_acc != nullptr) { + using ElementOutAcc = ElementAcc; + constexpr auto AlignmentOutAcc = 128 / cute::sizeof_bits_v; + Tensor mO = make_tensor(make_gmem_ptr(epilogue_args.ptr_o_acc + get<3>(cta_coord) * D_latent), make_shape(H, D_latent, B), epilogue_args.stride_o_acc); + auto cta_tiler_pv = take<0,2>(typename CollectiveMmaPV::CtaShape_MNK{}); + Tensor gO = local_tile(mO, cta_tiler_pv, take<0,3>(cta_coord)); + + auto tiled_t2r = make_tmem_copy(load_op, tAcc); + auto thread_idx = threadIdx.x % size(tiled_t2r); + + auto thread_t2r = tiled_t2r.get_slice(thread_idx); + Tensor tTR_gO = thread_t2r.partition_D(gO); + Tensor tTR_rAcc = make_tensor(shape(tTR_gO)); + + Tensor tTR_rO_frag = make_tensor(shape(tTR_rAcc)); + Tensor tTR_rO_src = recast>(coalesce(tTR_rO_frag)); + Tensor tR2G_rO_dst = recast>(coalesce(tTR_gO)); + Tensor tTR_tAcc = thread_t2r.partition_S(tAcc); + + copy(tiled_t2r, tTR_tAcc, tTR_rAcc); + + cutlass::epilogue::thread::LinearCombination epilogue_op({epilogue_args.output_scale / row_sum}); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size(tTR_rAcc); i++) { + tTR_rO_frag(i) = epilogue_op(tTR_rAcc(i)); + } + + copy(tTR_rO_src, tR2G_rO_dst); + +#ifndef B2B + + // compute LSE + ElementAcc lse = cutlass::fast_log(row_sum) + mainloop_args.softmax_scale * row_max; + + // store LSE + Tensor mLSE = make_tensor(make_gmem_ptr(epilogue_args.ptr_lse_acc + H * get<3>(cta_coord)), make_shape(H, B), epilogue_args.stride_lse_acc); + Tensor gLSE = local_tile(mLSE, append<3>(cta_tiler_pv, _1{}), take<0,3>(cta_coord), Step<_1, Underscore, _1>{}); + // for 2x2 dp, this must be conditional and the index is wrong + if (! kIs2Sm || (threadIdx.x < 64)) + { + gLSE(threadIdx.x) = lse; + } + #endif + } + else { + Tensor mO = make_tensor(make_gmem_ptr(epilogue_args.ptr_o), make_shape(H, D_latent, B), epilogue_args.stride_o); + auto cta_tiler_pv = take<0,2>(typename CollectiveMmaPV::CtaShape_MNK{}); + Tensor gO = local_tile(mO, cta_tiler_pv, take<0,3>(cta_coord)); + + auto tiled_t2r = make_tmem_copy(load_op, tAcc); + auto thread_idx = threadIdx.x % size(tiled_t2r); + + auto thread_t2r = tiled_t2r.get_slice(thread_idx); + Tensor tTR_gO = thread_t2r.partition_D(gO); + Tensor tTR_rAcc = make_tensor(shape(tTR_gO)); + + Tensor tTR_rO_frag = make_tensor(shape(tTR_rAcc)); + Tensor tTR_rO_src = recast>(coalesce(tTR_rO_frag)); + Tensor tR2G_rO_dst = recast>(coalesce(tTR_gO)); + Tensor tTR_tAcc = thread_t2r.partition_S(tAcc); + + copy(tiled_t2r, tTR_tAcc, tTR_rAcc); + + cutlass::epilogue::thread::LinearCombination epilogue_op({epilogue_args.output_scale / row_sum}); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size(tTR_rAcc); i++) { + tTR_rO_frag(i) = epilogue_op(tTR_rAcc(i)); + } + + copy(tTR_rO_src, tR2G_rO_dst); + +#ifndef B2B + if (epilogue_args.ptr_lse != nullptr) { + // compute LSE + ElementAcc lse = cutlass::fast_log(row_sum) + mainloop_args.softmax_scale * row_max; + + // store LSE + Tensor mLSE = make_tensor(make_gmem_ptr(epilogue_args.ptr_lse), make_shape(H, B), epilogue_args.stride_lse); + Tensor gLSE = local_tile(mLSE, append<3>(cta_tiler_pv, _1{}), take<0,3>(cta_coord), Step<_1, Underscore, _1>{}); + + // for 2x2 dp, this must be conditional and the index is wrong + if (! kIs2Sm || (threadIdx.x < 64)) + { + gLSE(threadIdx.x) = lse; + } + } +#endif + } + } + + + template + CUTLASS_DEVICE void compute( + CtaCoord const& cta_coord, + ProblemShape const& problem_shape, + MainloopArguments const& mainloop_args, + EpilogueParams const& epilogue_args, + TensorStorage& shared_tensors, + PipelineS& pipeline_mma_s, + typename PipelineS::PipelineState& pipeline_mma_s_consumer_state, + PipelineP& pipeline_p_mma, + typename PipelineP::PipelineState& pipeline_p_mma_producer_state, + PipelineO& pipeline_mma_o, + typename PipelineO::PipelineState& pipeline_mma_o_consumer_state, + int const& split_kv) { + + auto [H, K, D, B] = problem_shape; + + int k_tile_total = ceil_div(K, TileShapeS{}); + int k_tile_per_cta = ceil_div(k_tile_total, split_kv); + int k_index = get<3>(cta_coord) * k_tile_per_cta; // lower limit + int k_tile_count = max(0, min(k_tile_total, k_index + k_tile_per_cta) - k_index); + if (k_tile_count == 0) { + + // if we return early, we have to make sure we release the load warp + cutlass::arch::NamedBarrier( + (kNumComputeWarps + kNumLoadWarps) * NumThreadsPerWarp, + kNamedBarrierEpilogue + ).arrive(); + + return; + } + int k_index_final = k_tile_total - 1; + + ElementAcc row_max = -std::numeric_limits::infinity(); + ElementAcc row_sum = 0; + ElementAcc correction_factor = 1; + + pipeline_p_mma.producer_acquire(pipeline_p_mma_producer_state); + pipeline_mma_s.consumer_wait(pipeline_mma_s_consumer_state); + + auto dispatch_bool = [](bool b, auto fn) { + if (b) { + fn(cute::true_type{}); + } + else { + fn(cute::false_type{}); + } + }; + + // softmax s0 -> p0 + dispatch_bool(k_index == k_index_final, [&](auto is_last_tile) { + softmax( + is_last_tile, + row_max, row_sum, correction_factor, + problem_shape, mainloop_args, shared_tensors, k_index, + uint32_t(pipeline_mma_s_consumer_state.index() == 0 ? TmemAllocation::kS0 : TmemAllocation::kS1), + pipeline_p_mma_producer_state.index() + ); + }); + + k_index += 1; + + cutlass::arch::fence_view_async_tmem_load(); + cutlass::arch::fence_view_async_shared(); + pipeline_mma_s.consumer_release(pipeline_mma_s_consumer_state); + ++pipeline_mma_s_consumer_state; + pipeline_p_mma.producer_commit(pipeline_p_mma_producer_state); + ++pipeline_p_mma_producer_state; + + k_tile_count -= 1; + + CUTLASS_PRAGMA_NO_UNROLL + while (k_tile_count > 0) { + pipeline_p_mma.producer_acquire(pipeline_p_mma_producer_state); + pipeline_mma_s.consumer_wait(pipeline_mma_s_consumer_state); + + // softmax s1 -> p1 + dispatch_bool(k_index == k_index_final, [&](auto is_last_tile) { + softmax( + is_last_tile, + row_max, row_sum, correction_factor, + problem_shape, mainloop_args, shared_tensors, k_index, + uint32_t(pipeline_mma_s_consumer_state.index() == 0 ? TmemAllocation::kS0 : TmemAllocation::kS1), + pipeline_p_mma_producer_state.index() + ); + }); + + cutlass::arch::fence_view_async_tmem_load(); + cutlass::arch::fence_view_async_shared(); + pipeline_mma_s.consumer_release(pipeline_mma_s_consumer_state); + ++pipeline_mma_s_consumer_state; + pipeline_p_mma.producer_commit(pipeline_p_mma_producer_state); + ++pipeline_p_mma_producer_state; + + pipeline_mma_o.consumer_wait(pipeline_mma_o_consumer_state); + + // rescale + CUTLASS_PRAGMA_UNROLL + for (int j = 0; j < IterationsPV_N; j++) { + rescale(correction_factor, uint32_t(TmemAllocation::kO0) + j * uint32_t(TmemAllocation::kSizeAccO)); + } + + cutlass::arch::fence_view_async_tmem_store(); + pipeline_mma_o.consumer_release(pipeline_mma_o_consumer_state); + ++pipeline_mma_o_consumer_state; + + --k_tile_count; + k_index += 1; + } + + pipeline_mma_o.consumer_wait(pipeline_mma_o_consumer_state); + +#ifdef B2B + row_sum = 1; +#else + if constexpr (kWarpsInN > 1) { + // reduce row_sum if needed (for 2x2 dp) + shared_tensors.smem_exchange[threadIdx.x] = row_sum; + cutlass::arch::NamedBarrier(kNumComputeWarps*NumThreadsPerWarp, kNamedBarrierExchange).sync(); + // (64, 2) shape + int peer_index = (threadIdx.x + 64) % 128; + row_sum += shared_tensors.smem_exchange[peer_index]; + } +#endif + + cutlass::arch::NamedBarrier((kNumComputeWarps + kNumLoadWarps) * NumThreadsPerWarp, kNamedBarrierEpilogue).arrive(); + + // epilogue + CUTLASS_PRAGMA_UNROLL + for (int j = 0; j < IterationsPV_N; j++) { + epilogue( + row_max, row_sum, + replace<1>(cta_coord, j), problem_shape, + mainloop_args, epilogue_args, shared_tensors, + uint32_t(TmemAllocation::kO0) + j * uint32_t(TmemAllocation::kSizeAccO), split_kv + ); + } + + cutlass::arch::fence_view_async_tmem_load(); + pipeline_mma_o.consumer_release(pipeline_mma_o_consumer_state); + ++pipeline_mma_o_consumer_state; + } + +}; + +/////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass::fmha::kernel diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_mla_tile_scheduler.hpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_mla_tile_scheduler.hpp new file mode 100644 index 00000000..30389e79 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_mla_tile_scheduler.hpp @@ -0,0 +1,160 @@ +/*************************************************************************************************** + * Copyright (c) 2024 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ + +// clang-format off +#pragma once + +#include "cutlass/cutlass.h" +#include "cutlass/fast_math.h" +#include "cutlass/kernel_hardware_info.h" + +namespace cutlass::fmha::kernel { + +//////////////////////////////////////////////////////////////////////////////// + +struct Sm100MlaIndividualTileScheduler { + + struct Params { + dim3 grid; + }; + + bool valid_ = true; + + CUTLASS_DEVICE + Sm100MlaIndividualTileScheduler(Params const&) {} + + template + static Params to_underlying_arguments( + ProblemShape const& problem_shape, KernelHardwareInfo hw_info, + ClusterShape const& cluster_shape, int const& split_kv) { + using namespace cute; + dim3 grid(get<0>(cluster_shape), get<3>(problem_shape) /* Batch */, split_kv /*Maximum Split KV*/); + return Params{ grid }; + } + + static dim3 get_grid_shape(Params const& params) { + return params.grid; + } + + CUTLASS_DEVICE + bool is_valid() { + return valid_; + } + + CUTLASS_DEVICE + auto get_block_coord() { + using namespace cute; + return make_coord(blockIdx.x, _0{}, blockIdx.y, blockIdx.z); + } + + CUTLASS_DEVICE + Sm100MlaIndividualTileScheduler& operator++() { + valid_ = false; + return *this; + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct Sm100MlaPersistentTileScheduler { + + struct Params { + int num_blocks; + FastDivmod divmod_m_block; + FastDivmod divmod_b; + FastDivmod divmod_split_kv; + KernelHardwareInfo hw_info; + }; + + int block_idx = 0; + Params params; + + CUTLASS_DEVICE + Sm100MlaPersistentTileScheduler(Params const& params) : block_idx(blockIdx.x), params(params) {} + + template + static Params to_underlying_arguments( + ProblemShape const& problem_shape, KernelHardwareInfo hw_info, + ClusterShape const& cluster_shape, int const& split_kv) { + using namespace cute; + // Get SM count if needed, otherwise use user supplied SM count + int sm_count = hw_info.sm_count; + if (sm_count <= 1 || sm_count % size<0>(cluster_shape) != 0) { + CUTLASS_TRACE_HOST(" WARNING: Arguments do not include a valid SM count.\n" + " For optimal performance, populate the arguments KernelHardwareInfo struct with the SM count."); + sm_count = KernelHardwareInfo::query_device_multiprocessor_count(hw_info.device_id); + } + + CUTLASS_TRACE_HOST("to_underlying_arguments(): Setting persistent grid SM count to " << sm_count); + hw_info.sm_count = sm_count; + + int num_m_blocks = size<0>(cluster_shape); + int num_blocks = num_m_blocks * get<3>(problem_shape) /* Batch */; + num_blocks *= split_kv; /* Maximum Split KV*/ + + return Params { + num_blocks, + { num_m_blocks}, { get<3>(problem_shape) }, {split_kv}, + hw_info + }; + } + + static dim3 get_grid_shape(Params const& params) { + dim3 grid(std::min(params.num_blocks, params.hw_info.sm_count), 1, 1); + return grid; + } + + CUTLASS_DEVICE + bool is_valid() { + return block_idx < params.num_blocks; + } + + CUTLASS_DEVICE + auto get_block_coord() { + using namespace cute; + int block_decode = block_idx; + int m_block, bidb, n_split_kv; + params.divmod_m_block(block_decode, m_block, block_decode); + params.divmod_b(block_decode, bidb, block_decode); + params.divmod_split_kv(block_decode, n_split_kv, block_decode); + return make_coord(m_block, _0{}, bidb, n_split_kv); + } + + CUTLASS_DEVICE + Sm100MlaPersistentTileScheduler& operator++() { + block_idx += gridDim.x; + return *this; + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass::fmha::kernel diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/merge_attn_states.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/merge_attn_states.cu new file mode 100644 index 00000000..c719498b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/merge_attn_states.cu @@ -0,0 +1,204 @@ +#include +#include + +#include +#include + +#include "pytorch_extension_utils.h" + +// Helper functions to convert between different data types +// (float, half, bfloat16) for the merge attention states kernel. +inline __device__ float to_float(float u) { + return u; +} +inline __device__ float to_float(half u) { + return __half2float(u); +} +inline __device__ float to_float(__nv_bfloat16 u) { + return __bfloat162float(u); +} +inline __device__ void from_float(float& d, float s) { + d = s; +} +inline __device__ void from_float(half& d, float s) { + d = __float2half(s); +} +inline __device__ void from_float(__nv_bfloat16& d, float s) { + d = __float2bfloat16(s); +} + +// Implements section 2.2 of https://www.arxiv.org/pdf/2501.01005 +template +__global__ void merge_attn_states_kernel( + scalar_t* output, + float* output_lse, + const scalar_t* prefix_output, + const float* prefix_lse, + const scalar_t* suffix_output, + const float* suffix_lse, + const uint num_tokens, + const uint num_heads, + const uint head_size) { + using pack_128b_t = uint4; + const uint pack_size = 16 / sizeof(scalar_t); + const uint threads_per_head = head_size / pack_size; + + const uint global_idx = blockIdx.x * NUM_THREADS + threadIdx.x; + const uint token_head_threads = num_tokens * num_heads * threads_per_head; + + if (global_idx >= token_head_threads) return; + + // global_idx -> token_idx + head_idx + pack_idx + const uint token_head_idx = global_idx / threads_per_head; + const uint pack_idx = global_idx % threads_per_head; + + const uint token_idx = token_head_idx / num_heads; + const uint head_idx = token_head_idx % num_heads; + + const uint pack_offset = pack_idx * pack_size; // (0~15)*8, etc. + const uint head_offset = token_idx * num_heads * head_size + head_idx * head_size; + const scalar_t* prefix_head_ptr = prefix_output + head_offset; + const scalar_t* suffix_head_ptr = suffix_output + head_offset; + scalar_t* output_head_ptr = output + head_offset; + + // float p_lse = prefix_lse[head_idx * num_tokens + token_idx]; + // float s_lse = suffix_lse[head_idx * num_tokens + token_idx]; + float p_lse = prefix_lse[token_idx * num_heads + head_idx]; + float s_lse = suffix_lse[token_idx * num_heads + head_idx]; + p_lse = std::isinf(p_lse) ? -std::numeric_limits::infinity() : p_lse; + s_lse = std::isinf(s_lse) ? -std::numeric_limits::infinity() : s_lse; + + const float max_lse = fmaxf(p_lse, s_lse); + p_lse = p_lse - max_lse; + s_lse = s_lse - max_lse; + const float p_se = expf(p_lse); + const float s_se = expf(s_lse); + const float out_se = p_se + s_se; + const float p_scale = p_se / out_se; + const float s_scale = s_se / out_se; + + if (pack_offset < head_size) { + // Pack 128b load + pack_128b_t p_out_pack = reinterpret_cast(prefix_head_ptr)[pack_offset / pack_size]; + pack_128b_t s_out_pack = reinterpret_cast(suffix_head_ptr)[pack_offset / pack_size]; + pack_128b_t o_out_pack; + +#pragma unroll + for (uint i = 0; i < pack_size; ++i) { + // Always use float for FMA to keep high precision. + // half(uint16_t), bfloat16, float -> float. + const float p_out_f = to_float(reinterpret_cast(&p_out_pack)[i]); + const float s_out_f = to_float(reinterpret_cast(&s_out_pack)[i]); + // fma: a * b + c = p_out_f * p_scale + (s_out_f * s_scale) + const float o_out_f = p_out_f * p_scale + (s_out_f * s_scale); + // float -> half(uint16_t), bfloat16, float. + from_float(reinterpret_cast(&o_out_pack)[i], o_out_f); + } + + // Pack 128b storage + reinterpret_cast(output_head_ptr)[pack_offset / pack_size] = o_out_pack; + } + // We only need to write to output_lse once per head. + if (output_lse != nullptr && pack_idx == 0) { + float out_lse = logf(out_se) + max_lse; + output_lse[token_idx * num_heads + head_idx] = out_lse; + } +} + +// The following macro is used to dispatch the conversion function based on +// the output data type. The FN is a macro that calls a function with +// template. +#define DISPATCH_BY_SCALAR_DTYPE(scalar_dtype, fn) \ + { \ + if (scalar_dtype == at::ScalarType::Float) { \ + fn(float); \ + } else if (scalar_dtype == at::ScalarType::Half) { \ + fn(half); \ + } else if (scalar_dtype == at::ScalarType::BFloat16) { \ + fn(__nv_bfloat16); \ + } else { \ + TORCH_CHECK(false, "Unsupported data type of O: ", scalar_dtype); \ + } \ + } + +#define LAUNCH_MERGE_ATTN_STATES(scalar_t, NUM_THREADS) \ + { \ + merge_attn_states_kernel<<>>( \ + reinterpret_cast(output.data_ptr()), \ + reinterpret_cast(output_lse.data_ptr()), \ + reinterpret_cast(prefix_output.data_ptr()), \ + reinterpret_cast(prefix_lse.data_ptr()), \ + reinterpret_cast(suffix_output.data_ptr()), \ + reinterpret_cast(suffix_lse.data_ptr()), \ + num_tokens, \ + num_heads, \ + head_size); \ + } + +/*@brief Merges the attention states from prefix and suffix + * into the output tensor. NUM_TOKENS: n, NUM_HEADS: h, HEAD_SIZE: d + * + * @param output [n,h,d] The output tensor to store the merged attention states. + * @param output_lse [h,d] Optional tensor to store the log-sum-exp values. + * @param prefix_output [n,h,d] The prefix attention states. + * @param prefix_lse [n,h] The log-sum-exp values for the prefix attention + * states. + * @param suffix_output [n,h,d] The suffix attention states. + * @param suffix_lse [n,h] The log-sum-exp values for the suffix attention + * states. + */ +template +void merge_attn_states_launcher( + const at::Tensor& prefix_output, // [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] + const at::Tensor& prefix_lse, // [NUM_TOKENS, NUM_HEADS] + const at::Tensor& suffix_output, // [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] + const at::Tensor& suffix_lse, // [NUM_TOKENS, NUM_HEADS] + at::Tensor& output, // [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] + at::Tensor& output_lse // [NUM_TOKENS, NUM_HEADS] +) { + constexpr uint NUM_THREADS = 128; + const uint num_tokens = output.size(0); + const uint num_heads = output.size(1); + const uint head_size = output.size(2); + const uint pack_size = 16 / sizeof(scalar_t); + TORCH_CHECK(head_size % pack_size == 0, "headsize must be multiple of pack_size:", pack_size); + // Process one pack elements per thread. for float, the + // pack_size is 4 for half/bf16, the pack_size is 8. + const uint threads_per_head = head_size / pack_size; + const uint total_threads = num_tokens * num_heads * threads_per_head; + + dim3 block(NUM_THREADS); + dim3 grid((total_threads + NUM_THREADS - 1) / NUM_THREADS); + + const c10::cuda::OptionalCUDAGuard device_guard(prefix_output.device()); + auto stream = at::cuda::getCurrentCUDAStream(); + + LAUNCH_MERGE_ATTN_STATES(scalar_t, NUM_THREADS); +} + +#define CALL_MERGE_ATTN_STATES_LAUNCHER(scalar_t) \ + { merge_attn_states_launcher(v_a, s_a, v_b, s_b, v_merged, s_merged); } + +void merge_state_v2( + at::Tensor v_a, at::Tensor s_a, at::Tensor v_b, at::Tensor s_b, at::Tensor v_merged, at::Tensor s_merged) { + // Input tensors must be contiguous + CHECK_INPUT(v_a); // v_a prefix_output (seq_len, num_heads, head_dim) + CHECK_INPUT(s_a); // s_a prefix_lse (seq_len, num_heads) + CHECK_INPUT(v_b); // v_b suffix_output (seq_len, num_heads, head_dim) + CHECK_INPUT(s_b); // s_b suffix_lse (seq_len, num_heads) + // v_merged output (seq_len, num_heads, head_dim) + // s_merged output_lse (seq_len, num_heads) + auto device = v_a.device(); + CHECK_EQ(s_a.device(), device); + CHECK_EQ(v_b.device(), device); + CHECK_EQ(s_b.device(), device); + CHECK_DIM(3, v_a); + CHECK_DIM(2, s_a); + CHECK_DIM(3, v_b); + CHECK_DIM(2, s_b); + CHECK_SHAPE(v_a, v_b); + CHECK_SHAPE(s_a, s_b); + CHECK_EQ(v_a.size(0), s_a.size(0)); + CHECK_EQ(v_a.size(1), s_b.size(1)); + DISPATCH_BY_SCALAR_DTYPE(v_merged.dtype(), CALL_MERGE_ATTN_STATES_LAUNCHER); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/vertical_slash_index.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/vertical_slash_index.cu new file mode 100644 index 00000000..118f780d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/attention/vertical_slash_index.cu @@ -0,0 +1,462 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// This file is for blocksparse attention utils cuda kernel. + +#include +#include +#include +#include + +// Save the start index of each block in the given range into block_offset. +// Returns the updated block count. +__device__ int64_t save_blocks( + int* block_offset, + int64_t range_start, + int64_t range_end, + int64_t block_size, + int64_t input_block_count, + int64_t kv_seqlen) { + if (range_start >= kv_seqlen) { + return input_block_count; + } + if (range_end > kv_seqlen) { + range_end = kv_seqlen; + } + int64_t current_block_count = input_block_count; + for (int idx = range_start; idx < range_end; idx += block_size) { + block_offset[current_block_count++] = idx; + } + return current_block_count; +} + +// CUDA kernel: convert sparse vertical/slash indices to block/column offsets. +__global__ void convert_vertical_slash_indexes_kernel( + const int* q_seqlens, // [BATCH, ] + const int* kv_seqlens, // [BATCH, ] + const int* vertical_indexes, // [BATCH, N_HEADS, NNZ_V] + const int* slash_indexes, // [BATCH, N_HEADS, NNZ_S] + int* block_count, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M)] + int* block_offset, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M), NNZ_S] + int* column_count, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M)] + int* column_index, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M), NNZ_V] + int64_t N_HEADS, + int64_t N_ROWS, + int64_t BLOCK_SIZE_M, + int64_t BLOCK_SIZE_N, + int64_t NNZ_V, + int64_t NNZ_S, + bool causal // True for intra, False for succ +) { + const int batch_idx = blockIdx.y; + const int head_idx = blockIdx.x; + const int group_idx = blockIdx.z; + + int64_t q_seqlen = q_seqlens[batch_idx]; + int64_t kv_seqlen = kv_seqlens[batch_idx]; + int64_t block_idx_m = group_idx * blockDim.x + threadIdx.x; + int64_t start_m = block_idx_m * BLOCK_SIZE_M; + if (start_m >= q_seqlen) { + return; + } + int64_t end_m = start_m + BLOCK_SIZE_M; + vertical_indexes += (batch_idx * N_HEADS + head_idx) * NNZ_V; + slash_indexes += (batch_idx * N_HEADS + head_idx) * NNZ_S; + int64_t row_offset = (batch_idx * N_HEADS + head_idx) * N_ROWS + block_idx_m; + block_count += row_offset; + block_offset += row_offset * NNZ_S; + column_count += row_offset; + column_index += row_offset * NNZ_V; + + bool has_slash = true; + int64_t tmp_col_cnt = 0, tmp_blk_cnt = 0; + int64_t s = 0, v = 0; + int64_t v_idx = vertical_indexes[v++]; + int64_t s_idx = slash_indexes[s++]; + if (causal) { + while (s_idx >= end_m + (kv_seqlen - q_seqlen) && s < NNZ_S) { + s_idx = slash_indexes[s++]; + } + if (s_idx > end_m + (kv_seqlen - q_seqlen)) has_slash = false; + s_idx = max((kv_seqlen - q_seqlen) + end_m - s_idx, BLOCK_SIZE_M); + } else { + while (s_idx >= end_m + kv_seqlen && s < NNZ_S) { + s_idx = slash_indexes[s++]; + } + if (s_idx > end_m + kv_seqlen) has_slash = false; + s_idx = max(kv_seqlen + end_m - s_idx, BLOCK_SIZE_M); + } + + int64_t range_start = s_idx - BLOCK_SIZE_M, range_end = s_idx; + if (!has_slash) { + if (causal) { + range_start = (kv_seqlen - q_seqlen) + end_m; + range_end = (kv_seqlen - q_seqlen) + end_m + BLOCK_SIZE_N; + } else { + range_start = kv_seqlen; + range_end = kv_seqlen + BLOCK_SIZE_N; + } + } + + bool slash_finished = false; + while (1) { + if (v_idx < range_end) { + if (v_idx < range_start) { + column_index[tmp_col_cnt++] = v_idx; + } + if (v < NNZ_V) { + v_idx = vertical_indexes[v++]; + } else { + if (causal) + v_idx = end_m + BLOCK_SIZE_N + (kv_seqlen - q_seqlen); + else + v_idx = end_m + BLOCK_SIZE_N + kv_seqlen; + } + } else { + if ((s < NNZ_S && causal) || (s < NNZ_S && !causal && slash_indexes[s] >= start_m)) { + if (causal) + s_idx = max((kv_seqlen - q_seqlen) + end_m - slash_indexes[s++], BLOCK_SIZE_M); + else + s_idx = max(kv_seqlen + end_m - slash_indexes[s++], BLOCK_SIZE_M); + } else { + if (v == NNZ_V || (v_idx > range_start && causal)) { + // add the last vertical if no more slash + if (v == NNZ_V && !causal && v_idx < kv_seqlen) { + column_index[tmp_col_cnt++] = v_idx; + } + tmp_blk_cnt = save_blocks(block_offset, range_start, range_end, BLOCK_SIZE_N, tmp_blk_cnt, kv_seqlen); + break; + } else { + if (causal) { + range_start = (kv_seqlen - q_seqlen) + end_m; + range_end = (kv_seqlen - q_seqlen) + end_m + BLOCK_SIZE_N; + } else { + // if slash_finished but there are vertical left, save current + // blocks + tmp_blk_cnt = save_blocks(block_offset, range_start, range_end, BLOCK_SIZE_N, tmp_blk_cnt, kv_seqlen); + range_start = kv_seqlen; + range_end = kv_seqlen + BLOCK_SIZE_N; + } + slash_finished = true; + } + } + if (!slash_finished) { + if (s_idx > range_end + BLOCK_SIZE_M) { + tmp_blk_cnt = save_blocks(block_offset, range_start, range_end, BLOCK_SIZE_N, tmp_blk_cnt, kv_seqlen); + range_start = s_idx - BLOCK_SIZE_M; + range_end = s_idx; + } else if (s_idx > range_end) { + range_end += BLOCK_SIZE_M; + } + } + } + } + + block_count[0] = tmp_blk_cnt; + column_count[0] = tmp_col_cnt; +} + +// Host function: launches the kernel with 64 threads per block. +void convert_vertical_slash_indexes_64x64( + const int* q_seqlens, // [BATCH, ] + const int* kv_seqlens, // [BATCH, ] + const int* vertical_indexes, // [BATCH, N_HEADS, NNZ_V] + const int* slash_indexes, // [BATCH, N_HEADS, NNZ_S] + int* block_count, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M)] + int* block_offset, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M), NNZ_S] + int* column_count, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M)] + int* column_index, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M), NNZ_V] + int64_t BATCH_SIZE, + int64_t N_HEADS, + int64_t N_ROWS, + int64_t BLOCK_SIZE_M, + int64_t BLOCK_SIZE_N, + int64_t NNZ_V, + int64_t NNZ_S, + bool causal) { + const int N_THREADS = 64; + const dim3 dimBlock((int32_t)N_THREADS); + const dim3 dimGrid( + (int32_t)N_HEADS, (int32_t)BATCH_SIZE, ((int32_t)N_ROWS + (int32_t)N_THREADS - 1) / (int32_t)N_THREADS); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + convert_vertical_slash_indexes_kernel<<>>( + q_seqlens, + kv_seqlens, + vertical_indexes, + slash_indexes, + block_count, + block_offset, + column_count, + column_index, + N_HEADS, + N_ROWS, + BLOCK_SIZE_M, + BLOCK_SIZE_N, + NNZ_V, + NNZ_S, + causal); +} + +// Host function: prepares tensor pointers and launches the CUDA kernel. +void convert_vertical_slash_indexes( + torch::Tensor& block_count, // [BATCH, N_HEADS, NUM_ROWS] + torch::Tensor& block_offset, // [BATCH, N_HEADS, NUM_ROWS, NNZ_S] + torch::Tensor& column_count, // [BATCH, N_HEADS, NUM_ROWS] + torch::Tensor& column_index, // [BATCH, N_HEADS, NUM_ROWS, NNZ_V] + torch::Tensor q_seqlens, // [BATCH, ] + torch::Tensor kv_seqlens, // [BATCH, ] + torch::Tensor vertical_indexes, // [BATCH, N_HEADS, NNZ_V] + torch::Tensor slash_indexes, // [BATCH, N_HEADS, NNZ_S] + int64_t context_size, + int64_t block_size_M, + int64_t block_size_N, + bool causal) { + cudaSetDevice(q_seqlens.get_device()); + + int64_t batch_size = slash_indexes.size(0); + int64_t num_heads = slash_indexes.size(1); + int64_t nnz_slash = slash_indexes.size(2); + int64_t nnz_vertical = vertical_indexes.size(2); + int64_t num_rows = (context_size + block_size_M - 1) / block_size_M; + + convert_vertical_slash_indexes_64x64( + q_seqlens.data_ptr(), + kv_seqlens.data_ptr(), + vertical_indexes.data_ptr(), + slash_indexes.data_ptr(), + block_count.data_ptr(), + block_offset.data_ptr(), + column_count.data_ptr(), + column_index.data_ptr(), + batch_size, + num_heads, + num_rows, + block_size_M, + block_size_N, + nnz_vertical, + nnz_slash, + causal); +} + +// --- mergehead kernels --- // + +// Kernel: like above, but supports per-head variable NNZ_V/NNZ_S. +__global__ void convert_vertical_slash_indexes_kernel_mergehead( + const int* q_seqlens, // [BATCH, ] + const int* kv_seqlens, // [BATCH, ] + const int* vertical_indexes, // [BATCH, N_HEADS, NNZ_V] + const int* slash_indexes, // [BATCH, N_HEADS, NNZ_S] + const int* per_head_vertical_topkv, + const int* per_head_slash_topkv, + int* block_count, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M)] + int* block_offset, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M), NNZ_S] + int* column_count, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M)] + int* column_index, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M), NNZ_V] + int64_t N_HEADS, + int64_t N_ROWS, + int64_t BLOCK_SIZE_M, + int64_t BLOCK_SIZE_N, + int64_t NNZ_V, + int64_t NNZ_S, + bool causal // True for intra, False for succ +) { + const int batch_idx = blockIdx.y; + const int head_idx = blockIdx.x; + const int group_idx = blockIdx.z; + + int64_t q_seqlen = q_seqlens[batch_idx]; + int64_t kv_seqlen = kv_seqlens[batch_idx]; + int64_t block_idx_m = group_idx * blockDim.x + threadIdx.x; + int64_t start_m = block_idx_m * BLOCK_SIZE_M; + if (start_m >= q_seqlen) { + return; + } + int64_t end_m = start_m + BLOCK_SIZE_M; + vertical_indexes += (batch_idx * N_HEADS + head_idx) * NNZ_V; + slash_indexes += (batch_idx * N_HEADS + head_idx) * NNZ_S; + int64_t row_offset = (batch_idx * N_HEADS + head_idx) * N_ROWS + block_idx_m; + block_count += row_offset; + block_offset += row_offset * NNZ_S; + column_count += row_offset; + column_index += row_offset * NNZ_V; + + // MergeHead: each head has it's unique max topk NNZ_V，NNZ_S. (NNZ_V，NNZ_S + // above is buffer size, use to compute offset) + NNZ_S = per_head_slash_topkv[head_idx]; + NNZ_V = per_head_vertical_topkv[head_idx]; + + bool has_slash = true; + int64_t tmp_col_cnt = 0, tmp_blk_cnt = 0; + int64_t s = 0, v = 0; + int64_t v_idx = vertical_indexes[v++]; + int64_t s_idx = slash_indexes[s++]; + if (causal) { + while (s_idx >= end_m + (kv_seqlen - q_seqlen) && s < NNZ_S) { + s_idx = slash_indexes[s++]; + } + if (s_idx > end_m + (kv_seqlen - q_seqlen)) has_slash = false; + s_idx = max((kv_seqlen - q_seqlen) + end_m - s_idx, BLOCK_SIZE_M); + } else { + while (s_idx >= end_m + kv_seqlen && s < NNZ_S) { + s_idx = slash_indexes[s++]; + } + if (s_idx > end_m + kv_seqlen) has_slash = false; + s_idx = max(kv_seqlen + end_m - s_idx, BLOCK_SIZE_M); + } + + int64_t range_start = s_idx - BLOCK_SIZE_M, range_end = s_idx; + if (!has_slash) { + if (causal) { + range_start = (kv_seqlen - q_seqlen) + end_m; + range_end = (kv_seqlen - q_seqlen) + end_m + BLOCK_SIZE_N; + } else { + range_start = kv_seqlen; + range_end = kv_seqlen + BLOCK_SIZE_N; + } + } + + bool slash_finished = false; + while (1) { + if (v_idx < range_end) { + if (v_idx < range_start) { + column_index[tmp_col_cnt++] = v_idx; + } + if (v < NNZ_V) { + v_idx = vertical_indexes[v++]; + } else { + if (causal) + v_idx = end_m + BLOCK_SIZE_N + (kv_seqlen - q_seqlen); + else + v_idx = end_m + BLOCK_SIZE_N + kv_seqlen; + } + } else { + if ((s < NNZ_S && causal) || (s < NNZ_S && !causal && slash_indexes[s] >= start_m)) { + if (causal) + s_idx = max((kv_seqlen - q_seqlen) + end_m - slash_indexes[s++], BLOCK_SIZE_M); + else + s_idx = max(kv_seqlen + end_m - slash_indexes[s++], BLOCK_SIZE_M); + } else { + if (v == NNZ_V || (v_idx > range_start && causal)) { + // add the last vertical if no more slash + if (v == NNZ_V && !causal && v_idx < kv_seqlen) { + column_index[tmp_col_cnt++] = v_idx; + } + tmp_blk_cnt = save_blocks(block_offset, range_start, range_end, BLOCK_SIZE_N, tmp_blk_cnt, kv_seqlen); + break; + } else { + if (causal) { + range_start = (kv_seqlen - q_seqlen) + end_m; + range_end = (kv_seqlen - q_seqlen) + end_m + BLOCK_SIZE_N; + } else { + // if slash_finished but there are vertical left, save current + // blocks + tmp_blk_cnt = save_blocks(block_offset, range_start, range_end, BLOCK_SIZE_N, tmp_blk_cnt, kv_seqlen); + range_start = kv_seqlen; + range_end = kv_seqlen + BLOCK_SIZE_N; + } + slash_finished = true; + } + } + if (!slash_finished) { + if (s_idx > range_end + BLOCK_SIZE_M) { + tmp_blk_cnt = save_blocks(block_offset, range_start, range_end, BLOCK_SIZE_N, tmp_blk_cnt, kv_seqlen); + range_start = s_idx - BLOCK_SIZE_M; + range_end = s_idx; + } else if (s_idx > range_end) { + range_end += BLOCK_SIZE_M; + } + } + } + } + + block_count[0] = tmp_blk_cnt; + column_count[0] = tmp_col_cnt; +} + +// Launch the mergehead kernel with 64 threads per block. +void convert_vertical_slash_indexes_64x64_mergehead( + const int* q_seqlens, // [BATCH, ] + const int* kv_seqlens, // [BATCH, ] + const int* vertical_indexes, // [BATCH, N_HEADS, NNZ_V] + const int* slash_indexes, // [BATCH, N_HEADS, NNZ_S] + int* per_head_vertical_topkv, + int* per_head_slash_topkv, + int* block_count, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M)] + int* block_offset, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M), NNZ_S] + int* column_count, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M)] + int* column_index, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M), NNZ_V] + int64_t BATCH_SIZE, + int64_t N_HEADS, + int64_t N_ROWS, + int64_t BLOCK_SIZE_M, + int64_t BLOCK_SIZE_N, + int64_t NNZ_V, + int64_t NNZ_S, + bool causal) { + const int N_THREADS = 64; + const dim3 dimBlock(N_THREADS); + const dim3 dimGrid(N_HEADS, BATCH_SIZE, (N_ROWS + N_THREADS - 1) / N_THREADS); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + convert_vertical_slash_indexes_kernel_mergehead<<>>( + q_seqlens, + kv_seqlens, + vertical_indexes, + slash_indexes, + per_head_vertical_topkv, + per_head_slash_topkv, + block_count, + block_offset, + column_count, + column_index, + N_HEADS, + N_ROWS, + BLOCK_SIZE_M, + BLOCK_SIZE_N, + NNZ_V, + NNZ_S, + causal); +} + +// Host wrapper for mergehead kernel. +void convert_vertical_slash_indexes_mergehead( + torch::Tensor& block_count, // [BATCH, N_HEADS, NUM_ROWS] + torch::Tensor& block_offset, // [BATCH, N_HEADS, NUM_ROWS, NNZ_S] + torch::Tensor& column_count, // [BATCH, N_HEADS, NUM_ROWS] + torch::Tensor& column_index, // [BATCH, N_HEADS, NUM_ROWS, NNZ_V] + torch::Tensor q_seqlens, // [BATCH, ] + torch::Tensor kv_seqlens, // [BATCH, ] + torch::Tensor vertical_indexes, // [BATCH, N_HEADS, NNZ_V] + torch::Tensor slash_indexes, // [BATCH, N_HEADS, NNZ_S] + torch::Tensor vertical_indices_count, // [N_HEADS, ] + torch::Tensor slash_indices_count, + int64_t context_size, + int64_t block_size_M, + int64_t block_size_N, + bool causal) { + cudaSetDevice(q_seqlens.get_device()); + + int batch_size = slash_indexes.size(0); + int num_heads = slash_indexes.size(1); + int nnz_slash = slash_indexes.size(2); + int nnz_vertical = vertical_indexes.size(2); + int num_rows = (context_size + block_size_M - 1) / block_size_M; + + convert_vertical_slash_indexes_64x64_mergehead( + q_seqlens.data_ptr(), + kv_seqlens.data_ptr(), + vertical_indexes.data_ptr(), + slash_indexes.data_ptr(), + vertical_indices_count.data_ptr(), + slash_indices_count.data_ptr(), + block_count.data_ptr(), + block_offset.data_ptr(), + column_count.data_ptr(), + column_index.data_ptr(), + batch_size, + num_heads, + num_rows, + block_size_M, + block_size_N, + nnz_vertical, + nnz_slash, + causal); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/common_extension.cc b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/common_extension.cc new file mode 100644 index 00000000..1344675b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/common_extension.cc @@ -0,0 +1,599 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include +#include + +#include "sgl_kernel_ops.h" + +TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) { + /* + * From csrc/allreduce + */ + m.def("get_graph_buffer_ipc_meta", &get_graph_buffer_ipc_meta); + m.def("register_graph_buffers", ®ister_graph_buffers); + m.def("dispose", &dispose); + m.def("meta_size", &meta_size); + m.def("register_buffer", ®ister_buffer); + + m.def( + "init_custom_ar(int[] ipc_tensors, Tensor rank_data, " + "int rank, bool full_nvlink) -> int"); + m.impl("init_custom_ar", torch::kCUDA, &init_custom_ar); + + m.def( + "all_reduce(int fa, Tensor inp, Tensor! out, int reg_buffer, " + "int reg_buffer_sz_bytes) -> ()"); + m.impl("all_reduce", torch::kCUDA, &all_reduce); + + m.def("mscclpp_generate_unique_id", &mscclpp_generate_unique_id); + m.def( + "mscclpp_init_context(Tensor unique_id, int rank, int world_size, Tensor scratch, Tensor put_buffer, " + "int nranks_per_node, int[] rank_to_node, int[] rank_to_ib, int context_selection) -> int"); + m.impl("mscclpp_init_context", torch::kCUDA, &mscclpp_init_context); + + m.def("mscclpp_allreduce(int context, Tensor inp, Tensor! out, int nthreads, int nblocks) -> ()"); + m.impl("mscclpp_allreduce", torch::kCUDA, &mscclpp_allreduce); + + /* + * From csrc/attention + */ + m.def("merge_state(Tensor v_a, Tensor s_a, Tensor v_b, Tensor s_b, Tensor! v_merged, Tensor! s_merged) -> ()"); + m.impl("merge_state", torch::kCUDA, &merge_state); + m.def("merge_state_v2(Tensor v_a, Tensor s_a, Tensor v_b, Tensor s_b, Tensor! v_merged, Tensor! s_merged) -> ()"); + m.impl("merge_state_v2", torch::kCUDA, &merge_state_v2); + m.def( + "cutlass_mla_decode(Tensor! out, Tensor q_nope, Tensor q_pe, Tensor kv_c_and_k_pe_cache, Tensor seq_lens, Tensor " + "page_table, Tensor! workspace, float sm_scale, int num_kv_splits) -> ()"); + m.impl("cutlass_mla_decode", torch::kCUDA, &cutlass_mla_decode); + m.def("cutlass_mla_get_workspace_size", &cutlass_mla_get_workspace_size); + + /* + * From csrc/elementwise + */ + m.def("rmsnorm(Tensor! output, Tensor input, Tensor weight, float eps, bool enable_pdl) -> ()"); + m.impl("rmsnorm", torch::kCUDA, &rmsnorm); + + m.def("fused_add_rmsnorm(Tensor! input, Tensor! residual, Tensor weight, float eps, bool enable_pdl) -> ()"); + m.impl("fused_add_rmsnorm", torch::kCUDA, &sgl_fused_add_rmsnorm); + + m.def("gemma_rmsnorm(Tensor! output, Tensor input, Tensor weight, float eps, bool enable_pdl) -> ()"); + m.impl("gemma_rmsnorm", torch::kCUDA, &gemma_rmsnorm); + + m.def("gemma_fused_add_rmsnorm(Tensor! input, Tensor! residual, Tensor weight, float eps, bool enable_pdl) -> ()"); + m.impl("gemma_fused_add_rmsnorm", torch::kCUDA, &gemma_fused_add_rmsnorm); + + m.def("silu_and_mul(Tensor! out, Tensor input) -> ()"); + m.impl("silu_and_mul", torch::kCUDA, &silu_and_mul); + + m.def("gelu_tanh_and_mul(Tensor! out, Tensor input) -> ()"); + m.impl("gelu_tanh_and_mul", torch::kCUDA, &gelu_tanh_and_mul); + + m.def("gelu_and_mul(Tensor! out, Tensor input) -> ()"); + m.impl("gelu_and_mul", torch::kCUDA, &gelu_and_mul); + + m.def( + "apply_rope_pos_ids_cos_sin_cache(Tensor q, Tensor k, Tensor! q_rope, Tensor! k_rope, Tensor cos_sin_cache, " + "Tensor pos_ids, bool interleave, bool enable_pdl, " + "Tensor? v, Tensor!? k_buffer, Tensor!? v_buffer, Tensor? kv_cache_loc) -> ()"); + m.impl("apply_rope_pos_ids_cos_sin_cache", torch::kCUDA, &apply_rope_pos_ids_cos_sin_cache); + + m.def( + "downcast_fp8(Tensor k, Tensor v, Tensor k_out, Tensor v_out, Tensor k_scale, Tensor v_scale, Tensor loc, " + "int mult, int offset) -> ()"); + m.impl("downcast_fp8", torch::kCUDA, &downcast_fp8); + + m.def("copy_to_gpu_no_ce(Tensor input, Tensor! output) -> ()"); + m.impl("copy_to_gpu_no_ce", torch::kCUDA, ©_to_gpu_no_ce); + m.def("concat_mla_k(Tensor! k, Tensor k_nope, Tensor k_rope) -> ()"); + m.impl("concat_mla_k", torch::kCUDA, &concat_mla_k); + + m.def("concat_mla_absorb_q(Tensor a, Tensor b, Tensor! out) -> ()"); + m.impl("concat_mla_absorb_q", torch::kCUDA, &concat_mla_absorb_q); + + m.def("fast_topk(Tensor score, Tensor indices, Tensor lengths, Tensor? row_starts) -> ()"); + m.impl("fast_topk", torch::kCUDA, &fast_topk_interface); + m.def( + "fast_topk_transform_fused(Tensor score, Tensor lengths, Tensor dst_page_table, Tensor src_page_table, Tensor " + "cu_seqlens_q, Tensor? row_starts) -> ()"); + m.impl("fast_topk_transform_fused", torch::kCUDA, &fast_topk_transform_interface); + m.def( + "fast_topk_transform_ragged_fused(Tensor score, Tensor lengths, Tensor topk_indices_ragged, Tensor " + "topk_indices_offset, Tensor ? row_starts) -> ()"); + m.impl("fast_topk_transform_ragged_fused", torch::kCUDA, &fast_topk_transform_ragged_interface); + + /* + * From csrc/gemm + */ + m.def("awq_dequantize(Tensor qweight, Tensor scales, Tensor qzeros) -> Tensor"); + m.impl("awq_dequantize", torch::kCUDA, &awq_dequantize); + + m.def( + "int8_scaled_mm(Tensor mat_a, Tensor mat_b, Tensor scales_a, Tensor scales_b, ScalarType out_dtype, Tensor? " + "bias) -> Tensor"); + m.impl("int8_scaled_mm", torch::kCUDA, &int8_scaled_mm); + + m.def( + "fp8_scaled_mm(Tensor mat_a, Tensor mat_b, Tensor scales_a, Tensor scales_b, ScalarType out_dtype, Tensor? " + "bias) -> Tensor"); + m.impl("fp8_scaled_mm", torch::kCUDA, &fp8_scaled_mm); + + m.def( + "fp8_blockwise_scaled_mm(Tensor mat_a, Tensor mat_b, Tensor scales_a, Tensor scales_b, ScalarType out_dtype) -> " + "Tensor"); + m.impl("fp8_blockwise_scaled_mm", torch::kCUDA, &fp8_blockwise_scaled_mm); + + m.def( + "sgl_per_token_group_quant_8bit(Tensor input, Tensor output_q, Tensor output_s, int group_size," + " float eps, float fp8_min, float fp8_max, bool scale_ue8m0) -> ()"); + m.impl("sgl_per_token_group_quant_8bit", torch::kCUDA, &sgl_per_token_group_quant_8bit); + + m.def( + "sgl_per_token_group_quant_8bit_v2(Tensor input, Tensor output_q, Tensor output_s, int group_size," + " float eps, float fp8_min, float fp8_max, bool scale_ue8m0, bool fuse_silu_and_mul, Tensor? masked_m) -> ()"); + m.impl("sgl_per_token_group_quant_8bit_v2", torch::kCUDA, &sgl_per_token_group_quant_8bit_v2); + + m.def("sgl_per_tensor_quant_fp8(Tensor input, Tensor output_q, Tensor output_s, bool is_static) -> ()"); + m.impl("sgl_per_tensor_quant_fp8", torch::kCUDA, &sgl_per_tensor_quant_fp8); + + m.def("sgl_per_token_quant_fp8(Tensor input, Tensor output_q, Tensor output_s) -> ()"); + m.impl("sgl_per_token_quant_fp8", torch::kCUDA, &sgl_per_token_quant_fp8); + + m.def( + "cutlass_scaled_fp4_mm(Tensor! out, Tensor a, Tensor b," + " Tensor block_scale_a, Tensor block_scale_b," + " Tensor alpha) -> ()"); + m.impl("cutlass_scaled_fp4_mm", torch::kCUDA, &cutlass_scaled_fp4_mm); + + m.def( + "scaled_fp4_quant(Tensor! output, Tensor! input," + " Tensor! output_scale, Tensor! input_scale) -> ()"); + m.impl("scaled_fp4_quant", torch::kCUDA, &scaled_fp4_quant); + + m.def("dsv3_fused_a_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()"); + m.impl("dsv3_fused_a_gemm", torch::kCUDA, &dsv3_fused_a_gemm); + + // Compute NVFP4 experts quantization. + m.def( + "scaled_fp4_experts_quant(Tensor! output, Tensor! output_scale," + "Tensor input, Tensor input_global_scale, Tensor input_offset_by_experts," + "Tensor output_scale_offset_by_experts) -> ()"); + m.impl("scaled_fp4_experts_quant", torch::kCUDA, &scaled_fp4_experts_quant); + + m.def( + "silu_and_mul_scaled_fp4_experts_quant(Tensor! output, Tensor! output_scale," + "Tensor input, Tensor input_global_scale, Tensor mask, bool use_silu_and_mul) -> ()"); + m.impl("silu_and_mul_scaled_fp4_experts_quant", torch::kCUDA, &silu_and_mul_scaled_fp4_experts_quant); + + m.def( + "cutlass_fp4_group_mm(Tensor! output, Tensor a, Tensor b," + "Tensor a_blockscale, Tensor b_blockscale, Tensor alphas," + "Tensor ab_strides, Tensor c_strides, Tensor problem_sizes," + " Tensor expert_offsets, Tensor sf_offsets) -> ()"); + m.impl("cutlass_fp4_group_mm", torch::kCUDA, &cutlass_fp4_group_mm); + + m.def("dsv3_router_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()"); + m.impl("dsv3_router_gemm", torch::kCUDA, &dsv3_router_gemm); + + /* + * From csrc/gemm/gptq + */ + m.def( + "gptq_marlin_gemm(Tensor! a, Tensor? c_or_none," + "Tensor! b_q_weight, Tensor! b_scales, Tensor? global_scale_or_none," + "Tensor? b_zeros_or_none, Tensor? g_idx_or_none, Tensor? perm_or_none," + "Tensor! workspace, int b_q_type_id, int size_m, int size_n, int size_k," + "bool is_k_full, bool use_atomic_add, bool use_fp32_reduce, bool is_zp_float) -> Tensor"); + m.impl("gptq_marlin_gemm", torch::kCUDA, &gptq_marlin_gemm); + + m.def( + "gptq_gemm(Tensor a, Tensor b_q_weight, Tensor b_gptq_qzeros, Tensor b_gptq_scales, Tensor b_g_idx, bool " + "use_shuffle, int bit) -> Tensor"); + m.impl("gptq_gemm", torch::kCUDA, &gptq_gemm); + + m.def("gptq_shuffle(Tensor! q_weight, Tensor q_perm, int bit) -> ()"); + m.impl("gptq_shuffle", torch::kCUDA, &gptq_shuffle); + + m.def("gptq_marlin_repack(Tensor! b_q_weight, Tensor! perm, int size_k, int size_n, int num_bits) -> Tensor"); + m.impl("gptq_marlin_repack", torch::kCUDA, &gptq_marlin_repack); + + m.def("awq_marlin_repack(Tensor! b_q_weight, int size_k, int size_n, int num_bits) -> Tensor"); + m.impl("awq_marlin_repack", torch::kCUDA, &awq_marlin_repack); + + /* + * From csrc/moe + */ + m.def( + "moe_align_block_size(Tensor topk_ids, int num_experts, int block_size, Tensor! sorted_token_ids, Tensor! " + "experts_ids, Tensor! num_tokens_post_pad, Tensor! cumsum_buffer, bool " + "pad_sorted_token_ids) -> ()"); + m.impl("moe_align_block_size", torch::kCUDA, &moe_align_block_size); + + m.def( + "topk_softmax(Tensor! topk_weights, Tensor! topk_indices, Tensor gating_output, bool renormalize, float " + "moe_softcapping, Tensor? correction_bias) -> ()"); + m.impl("topk_softmax", torch::kCUDA, &topk_softmax); + + m.def( + "topk_sigmoid(Tensor! topk_weights, Tensor! topk_indices, Tensor gating_output, bool renormalize, Tensor? " + "correction_bias) -> ()"); + m.impl("topk_sigmoid", torch::kCUDA, &topk_sigmoid); + + m.def("moe_sum_reduce(Tensor input, Tensor output, float routed_scaling_factor) -> ()"); + m.impl("moe_sum_reduce", torch::kCUDA, &moe_sum_reduce); + + m.def("moe_sum(Tensor input, Tensor! output) -> ()"); + m.impl("moe_sum", torch::kCUDA, &moe_sum); + + m.def( + "moe_fused_gate(Tensor input, Tensor bias, int num_expert_group, int topk_group, int topk, int " + "num_fused_shared_experts, float routed_scaling_factor, bool apply_routed_scaling_factor_on_output) -> " + "(Tensor[])"); + m.impl("moe_fused_gate", torch::kCUDA, &moe_fused_gate); + + m.def( + "kimi_k2_moe_fused_gate(Tensor input, Tensor bias, int topk, bool renormalize, " + "float routed_scaling_factor, bool apply_routed_scaling_factor_on_output) -> " + "(Tensor[])"); + m.impl("kimi_k2_moe_fused_gate", torch::kCUDA, &kimi_k2_moe_fused_gate); + + m.def( + "fp8_blockwise_scaled_grouped_mm(Tensor output, Tensor a_ptrs, Tensor b_ptrs, Tensor out_ptrs, Tensor " + "a_scales_ptrs, Tensor b_scales_ptrs, Tensor a, Tensor b, Tensor scales_a, Tensor scales_b, Tensor " + "stride_a, Tensor stride_b, Tensor stride_c, Tensor layout_sfa, Tensor layout_sfb, Tensor problem_sizes, Tensor " + "expert_offsets, Tensor workspace) -> ()"); + m.impl("fp8_blockwise_scaled_grouped_mm", torch::kCUDA, &fp8_blockwise_scaled_grouped_mm); + + m.def( + "prepare_moe_input(Tensor topk_ids, Tensor expert_offsets, Tensor? blockscale_offsets, Tensor problem_sizes1," + " Tensor problem_sizes2, Tensor input_permutation, Tensor output_permutation, int num_experts, int n, int k) -> " + "()"); + m.impl("prepare_moe_input", torch::kCUDA, &prepare_moe_input); + + m.def("shuffle_rows(Tensor input, Tensor dst2src_map, Tensor output) -> ()"); + m.impl("shuffle_rows", torch::kCUDA, &shuffle_rows); + + m.def("apply_shuffle_mul_sum(Tensor input, Tensor output, Tensor permutation, Tensor? factors) -> ()"); + m.impl("apply_shuffle_mul_sum", torch::kCUDA, &apply_shuffle_mul_sum); + + m.def( + "fused_qk_norm_rope(Tensor! qkv, int num_heads_q, " + "int num_heads_k, int num_heads_v, int head_dim, float eps, " + "Tensor q_weight, Tensor k_weight, float base, " + "bool is_neox, Tensor position_ids, float factor, float low, float high, float attention_factor) -> ()"); + m.impl("fused_qk_norm_rope", torch::kCUDA, &fused_qk_norm_rope); + + /* + * From csrc/moe/cutlass_moe/w4a8 + */ + m.def( + "get_cutlass_w4a8_moe_mm_data(Tensor topk_ids, Tensor! expert_offsets, " + " Tensor! problem_sizes1, Tensor! problem_sizes2, " + " Tensor! input_permutation, " + " Tensor! output_permutation, int num_experts, " + " int n, int k) -> ()"); + m.impl("get_cutlass_w4a8_moe_mm_data", torch::kCUDA, &get_cutlass_w4a8_moe_mm_data); + + m.def( + "cutlass_w4a8_moe_mm(Tensor! d, Tensor a, Tensor b, " + " Tensor a_scales, Tensor b_scales, Tensor expert_offsets, " + " Tensor problem_sizes, Tensor a_strides, " + " Tensor b_strides, Tensor d_strides, Tensor s_strides," + " int chunk_size, int topk) -> ()"); + m.impl("cutlass_w4a8_moe_mm", torch::kCUDA, &cutlass_w4a8_moe_mm); + + /* + * From csrc/moe/marlin_moe_wna16 + */ + m.def( + "moe_wna16_marlin_gemm(Tensor! a, Tensor? c_or_none," + "Tensor! b_q_weight, Tensor? b_bias_or_none, Tensor! b_scales," + "Tensor? global_scale_or_none, Tensor? b_zeros_or_none," + "Tensor? g_idx_or_none, Tensor? perm_or_none, Tensor! workspace," + "Tensor sorted_token_ids," + "Tensor! expert_ids, Tensor! num_tokens_past_padded," + "Tensor! topk_weights, int moe_block_size, int top_k, " + "bool mul_topk_weights, bool is_ep, int b_q_type_id," + "int size_m, int size_n, int size_k," + "bool is_k_full, bool use_atomic_add," + "bool use_fp32_reduce, bool is_zp_float) -> Tensor"); + m.impl("moe_wna16_marlin_gemm", torch::kCUDA, &moe_wna16_marlin_gemm); + + /* + * From csrc/speculative + */ + m.def( + "tree_speculative_sampling_target_only(Tensor! predicts, Tensor! accept_index, Tensor! accept_token_num, " + "Tensor candidates, Tensor retrive_index, Tensor retrive_next_token, Tensor retrive_next_sibling, " + "Tensor uniform_samples, Tensor uniform_samples_for_final_sampling, Tensor target_probs, Tensor draft_probs, " + "float threshold_single, float threshold_acc, " + "bool deterministic) -> ()"); + m.impl("tree_speculative_sampling_target_only", torch::kCUDA, &tree_speculative_sampling_target_only); + + m.def( + "verify_tree_greedy(Tensor! predicts, Tensor! accept_index, Tensor! accept_token_num, " + "Tensor candidates, Tensor retrive_index, Tensor retrive_next_token, Tensor retrive_next_sibling, " + "Tensor target_predict) -> ()"); + m.impl("verify_tree_greedy", torch::kCUDA, &verify_tree_greedy); + + m.def( + "reconstruct_indices_from_tree_mask(Tensor tree_mask, Tensor verified_seq_len, Tensor positions, " + "Tensor retrive_index, Tensor retrive_next_token, Tensor retrive_next_sibling, " + "int batch_size, int draft_token_num) -> ()"); + m.impl("reconstruct_indices_from_tree_mask", torch::kCUDA, &reconstruct_indices_from_tree_mask); + + m.def( + "build_tree_kernel_efficient(Tensor parent_list, Tensor selected_index, Tensor verified_seq_len, " + "Tensor! tree_mask, Tensor! positions, Tensor! retrive_index, Tensor! retrive_next_token, " + "Tensor! retrive_next_sibling, int topk, int depth, int draft_token_num, int tree_mask_mode) -> " + "()"); + m.impl("build_tree_kernel_efficient", torch::kCUDA, &build_tree_kernel_efficient); + + m.def( + "segment_packbits(Tensor x, Tensor input_indptr, Tensor output_indptr, Tensor! y, int batch_size, " + "int cuda_stream) -> ()"); + m.impl("segment_packbits", torch::kCUDA, &segment_packbits); + + /* + * From csrc/kvcacheio + */ + m.def( + "transfer_kv_per_layer(Tensor src_k, Tensor dst_k, Tensor src_v, Tensor dst_v, Tensor src_indices, Tensor " + "dst_indices, int item_size, int block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_per_layer", torch::kCUDA, &transfer_kv_per_layer); + m.def( + "transfer_kv_per_layer_pf_lf(Tensor src_k, Tensor dst_k, Tensor src_v, Tensor dst_v, Tensor src_indices, Tensor " + "dst_indices, int layer_id, int item_size, int src_layout_dim, int block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_per_layer_pf_lf", torch::kCUDA, &transfer_kv_per_layer_pf_lf); + m.def( + "transfer_kv_per_layer_ph_lf(Tensor src_k, Tensor dst_k, Tensor src_v, Tensor dst_v, Tensor src_indices, Tensor " + "dst_indices, int layer_id, int item_size, int src_layout_dim, int page_size, int head_num, int block_quota, int " + "num_warps_per_block) -> ()"); + m.impl("transfer_kv_per_layer_ph_lf", torch::kCUDA, &transfer_kv_per_layer_ph_lf); + m.def( + "transfer_kv_all_layer(Tensor src_k_layers, Tensor dst_k_layers, Tensor src_v_layers, Tensor dst_v_layers, " + "Tensor src_indices, Tensor dst_indices, int item_size, int num_layers, int block_quota, int " + "num_warps_per_block) -> ()"); + m.impl("transfer_kv_all_layer", torch::kCUDA, &transfer_kv_all_layer); + m.def( + "transfer_kv_all_layer_lf_pf(Tensor src_k_layers, Tensor dst_k, Tensor src_v_layers, Tensor dst_v, " + "Tensor src_indices, Tensor dst_indices, int item_size, int dst_layout_dim, int num_layers, int block_quota, int " + "num_warps_per_block) -> ()"); + m.impl("transfer_kv_all_layer_lf_pf", torch::kCUDA, &transfer_kv_all_layer_lf_pf); + m.def( + "transfer_kv_all_layer_lf_ph(Tensor src_k_layers, Tensor dst_k, Tensor src_v_layers, Tensor dst_v, " + "Tensor src_indices, Tensor dst_indices, int item_size, int dst_layout_dim, int num_layers, int page_size, int " + "head_num, int block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_all_layer_lf_ph", torch::kCUDA, &transfer_kv_all_layer_lf_ph); + m.def( + "transfer_kv_per_layer_mla(Tensor src, Tensor dst, Tensor src_indices, Tensor dst_indices, int item_size, int " + "block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_per_layer_mla", torch::kCUDA, &transfer_kv_per_layer_mla); + m.def( + "transfer_kv_per_layer_mla_pf_lf(Tensor src, Tensor dst, Tensor src_indices, Tensor dst_indices, int layer_id, " + "int item_size, int src_layout_dim, int block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_per_layer_mla_pf_lf", torch::kCUDA, &transfer_kv_per_layer_mla_pf_lf); + m.def( + "transfer_kv_all_layer_mla(Tensor src_layers, Tensor dst_layers, Tensor src_indices, Tensor dst_indices, int " + "item_size, int num_layers, int block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_all_layer_mla", torch::kCUDA, &transfer_kv_all_layer_mla); + m.def( + "transfer_kv_all_layer_mla_lf_pf(Tensor src_layers, Tensor dst, Tensor src_indices, Tensor dst_indices, " + "int item_size, int dst_layout_dim, int num_layers, int block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_all_layer_mla_lf_pf", torch::kCUDA, &transfer_kv_all_layer_mla_lf_pf); + m.def( + "transfer_kv_direct(Tensor[] src_layers, Tensor[] dst_layers, Tensor src_indices, Tensor dst_indices, int " + "page_size) -> ()"); + m.impl("transfer_kv_direct", torch::kCUDA, &transfer_kv_direct); + m.def( + "transfer_kv_per_layer_direct_pf_lf(Tensor[] src_ptrs, Tensor[] dst_ptrs, Tensor src_indices, " + "Tensor dst_indices, int layer_id, int page_size)->() "); + m.impl("transfer_kv_per_layer_direct_pf_lf", torch::kCUDA, &transfer_kv_per_layer_direct_pf_lf); + m.def( + "transfer_kv_all_layer_direct_lf_pf(Tensor[] src_ptrs, Tensor[] dst_ptrs, Tensor src_indices, " + "Tensor dst_indices, int page_size) ->() "); + m.impl("transfer_kv_all_layer_direct_lf_pf", torch::kCUDA, &transfer_kv_all_layer_direct_lf_pf); + + /* + * From csrc/memory + */ + m.def("store_kv_cache(Tensor k_cache, Tensor v_cache, Tensor out_loc, Tensor k, Tensor v) -> ()"); + m.impl("store_kv_cache", &store_kv_cache); + + m.def("weak_ref_tensor(Tensor tensor) -> Tensor"); + m.impl("weak_ref_tensor", torch::kCUDA, &weak_ref_tensor); + + /* + * From FlashInfer + */ + m.def( + "bmm_fp8(Tensor A, Tensor B, Tensor! D, Tensor A_scale, Tensor B_scale, Tensor workspace_buffer, " + "int cublas_handle) -> ()", + {at::Tag::needs_fixed_stride_order}); + m.impl("bmm_fp8", torch::kCUDA, &bmm_fp8); + + m.def( + "min_p_sampling_from_probs(Tensor probs, Tensor output, Tensor? maybe_indices, Tensor? maybe_min_p_arr, float " + "min_p_val, bool deterministic, Generator? gen) -> ()"); + m.impl("min_p_sampling_from_probs", torch::kCUDA, &min_p_sampling_from_probs); + + m.def("top_k_renorm_probs(Tensor probs, Tensor! renorm_probs, Tensor? maybe_top_k_arr, int top_k_val) -> ()"); + m.impl("top_k_renorm_probs", torch::kCUDA, &top_k_renorm_probs); + + m.def("top_p_renorm_probs(Tensor probs, Tensor! renorm_probs, Tensor? maybe_top_p_arr, float top_p_val) -> ()"); + m.impl("top_p_renorm_probs", torch::kCUDA, &top_p_renorm_probs); + + m.def( + "top_p_sampling_from_probs(Tensor probs, Tensor output, Tensor? maybe_indices, Tensor? " + "maybe_top_p_arr, float top_p_val, bool deterministic, Generator? gen) -> ()"); + m.impl("top_p_sampling_from_probs", torch::kCUDA, &top_p_sampling_from_probs); + + m.def( + "top_k_top_p_sampling_from_probs(Tensor probs, Tensor output, Tensor? maybe_indices, Tensor? maybe_top_k_arr, " + "float top_k_val, Tensor? maybe_top_p_arr, float top_p_val, bool deterministic, Generator? gen) -> ()"); + m.impl("top_k_top_p_sampling_from_probs", torch::kCUDA, &top_k_top_p_sampling_from_probs); + + m.def("top_k_mask_logits(Tensor logits, Tensor mask_logits, Tensor? maybe_top_k_arr, int top_k_val) -> ()"); + m.impl("top_k_mask_logits", torch::kCUDA, &top_k_mask_logits); + + /* + * From Sparse Flash Attention + */ + m.def( + "fwd_sparse(Tensor! q, Tensor k, Tensor v, " + "Tensor block_count, Tensor block_offset, Tensor column_count, Tensor column_index, " + "Tensor!? out, Tensor? alibi_slopes, " + "float p_dropout, float softmax_scale, bool is_causal, " + "float softcap, bool return_softmax, Generator? gen)" + "-> Tensor[]"); + m.impl("fwd_sparse", torch::kCUDA, &flash::mha_fwd_sparse); + + m.def( + "varlen_fwd_sparse(Tensor! q, Tensor k, Tensor v, " + "Tensor block_count, Tensor block_offset, Tensor column_count, Tensor column_index, " + "Tensor!? out, Tensor cu_seqlens_q, " + "Tensor cu_seqlens_k, Tensor? seqused_k, Tensor? alibi_slopes, " + "int max_seqlen_q, int max_seqlen_k, float p_dropout, float softmax_scale, bool zero_tensors, " + "bool is_causal, float softcap, bool return_softmax, " + "Generator? gen) -> Tensor[]"); + m.impl("varlen_fwd_sparse", torch::kCUDA, &flash::mha_varlen_fwd_sparse); + + // Sparse Attention utils + m.def( + "convert_vertical_slash_indexes(" + " Tensor! block_count, Tensor! block_offset, " + " Tensor! column_count, Tensor! column_index, " + " Tensor q_seqlens, Tensor q_seqlens, " + " Tensor vertical_indexes, Tensor slash_indexes, " + " int context_size, int block_size_M, int block_size_N, " + " bool causal) -> ()"); + m.impl("convert_vertical_slash_indexes", torch::kCUDA, &convert_vertical_slash_indexes); + + m.def( + "convert_vertical_slash_indexes_mergehead(" + " Tensor! block_count, Tensor! block_offset, " + " Tensor! column_count, Tensor! column_index, " + " Tensor q_seqlens, Tensor q_seqlens, " + " Tensor vertical_indexes, Tensor slash_indexes, " + " Tensor vertical_indices_count, Tensor slash_indices_count, " + " int context_size, int block_size_M, int block_size_N, " + " bool causal) -> ()"); + m.impl("convert_vertical_slash_indexes_mergehead", torch::kCUDA, &convert_vertical_slash_indexes_mergehead); + + /* + * From csrc/grammar + */ + m.def("apply_token_bitmask_inplace_cuda(Tensor logits, Tensor bitmask, Tensor? indices=None) -> ()"); + m.impl("apply_token_bitmask_inplace_cuda", &ApplyTokenBitmaskInplace); + + /* + * From csrc/gemm (QServe) + */ + m.def( + "qserve_w4a8_per_chn_gemm(Tensor _in_feats, Tensor _kernel, Tensor _wscales, Tensor _ascales, Tensor _w_szs, " + "Tensor _a_ssums, Tensor! _out_feats) -> ()"); + m.impl("qserve_w4a8_per_chn_gemm", torch::kCUDA, &qserve_w4a8_per_chn_gemm); + + m.def( + "qserve_w4a8_per_group_gemm(Tensor _in_feats, Tensor _kernel, Tensor _zeros, Tensor _scales_i8, Tensor _wscales, " + "Tensor _ascales, Tensor! _out_feats) -> ()"); + m.impl("qserve_w4a8_per_group_gemm", torch::kCUDA, &qserve_w4a8_per_group_gemm); + + /* + * From csrc/quantization/gguf + */ + m.def( + "ggml_dequantize(Tensor W, int type, SymInt m, SymInt n, ScalarType? " + "dtype) -> Tensor"); + m.impl("ggml_dequantize", torch::kCUDA, &ggml_dequantize); + + m.def( + "ggml_mul_mat_vec_a8(Tensor W, Tensor X, int type, SymInt row) " + "-> Tensor"); + m.impl("ggml_mul_mat_vec_a8", torch::kCUDA, &ggml_mul_mat_vec_a8); + + m.def("ggml_mul_mat_a8(Tensor W, Tensor X, int type, SymInt row) -> Tensor"); + m.impl("ggml_mul_mat_a8", torch::kCUDA, &ggml_mul_mat_a8); + + m.def( + "ggml_moe_a8(Tensor X, Tensor W, " + "Tensor sorted_token_ids, Tensor expert_ids, Tensor " + "num_tokens_post_padded, " + "int type, SymInt row, SymInt top_k, SymInt tokens) -> Tensor"); + m.impl("ggml_moe_a8", torch::kCUDA, &ggml_moe_a8); + + m.def( + "ggml_moe_a8_vec(Tensor X, Tensor W, " + "Tensor topk_ids, int top_k, " + "int type, SymInt row, SymInt tokens) -> Tensor"); + m.impl("ggml_moe_a8_vec", torch::kCUDA, &ggml_moe_a8_vec); + + m.def("ggml_moe_get_block_size(int type) -> int"); + m.impl("ggml_moe_get_block_size", torch::kCUDA, &ggml_moe_get_block_size); + + /* + * From csrc/mamba + */ + m.def( + "causal_conv1d_update(Tensor! x," + "Tensor! conv_state," + "Tensor! weight," + "Tensor? bias_," + "bool silu_activation," + "Tensor? cache_seqlens_," + "Tensor? conv_state_indices," + "int pad_slot_id) -> ()"); + m.impl("causal_conv1d_update", torch::kCUDA, &causal_conv1d_update); + + m.def( + "causal_conv1d_fwd(Tensor! x, Tensor! weight," + "Tensor? bias_," + "Tensor!? conv_states," + "Tensor? query_start_loc," + "Tensor? cache_indices," + "Tensor? has_initial_state," + "bool silu_activation," + "int pad_slot_id) -> ()"); + m.impl("causal_conv1d_fwd", torch::kCUDA, &causal_conv1d_fwd); + + /* + * From csrc/expert_sepcialization + */ + m.def( + "es_fp8_blockwise_scaled_grouped_mm(Tensor output, Tensor a, Tensor b, Tensor scales_a, Tensor scales_b, Tensor " + "stride_a, Tensor stride_b, Tensor stride_d, Tensor problem_sizes, Tensor expert_offsets, Tensor workspace) -> " + "()"); + m.impl("es_fp8_blockwise_scaled_grouped_mm", &es_fp8_blockwise_scaled_grouped_mm); + + /* + * From fast-hadamard-transform + */ + m.def("fast_hadamard_transform(Tensor x, float scale) -> Tensor"); + m.impl("fast_hadamard_transform", torch::kCUDA, &fast_hadamard_transform); + + m.def("fast_hadamard_transform_12N(Tensor x, float scale) -> Tensor"); + m.impl("fast_hadamard_transform_12N", torch::kCUDA, &fast_hadamard_transform_12N); + + m.def("fast_hadamard_transform_20N(Tensor x, float scale) -> Tensor"); + m.impl("fast_hadamard_transform_20N", torch::kCUDA, &fast_hadamard_transform_20N); + + m.def("fast_hadamard_transform_28N(Tensor x, float scale) -> Tensor"); + m.impl("fast_hadamard_transform_28N", torch::kCUDA, &fast_hadamard_transform_28N); + + m.def("fast_hadamard_transform_40N(Tensor x, float scale) -> Tensor"); + m.impl("fast_hadamard_transform_40N", torch::kCUDA, &fast_hadamard_transform_40N); +} + +REGISTER_EXTENSION(common_ops) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/common_extension_rocm.cc b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/common_extension_rocm.cc new file mode 100644 index 00000000..9981e467 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/common_extension_rocm.cc @@ -0,0 +1,191 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "sgl_kernel_ops.h" + +TORCH_LIBRARY_EXPAND(sgl_kernel, m) { + /* + * From csrc/activation + */ + m.def("silu_and_mul(Tensor! out, Tensor input) -> ()"); + m.impl("silu_and_mul", torch::kCUDA, &silu_and_mul); + + m.def("gelu_tanh_and_mul(Tensor! out, Tensor input) -> ()"); + m.impl("gelu_tanh_and_mul", torch::kCUDA, &gelu_tanh_and_mul); + + m.def("gelu_and_mul(Tensor! out, Tensor input) -> ()"); + m.impl("gelu_and_mul", torch::kCUDA, &gelu_and_mul); + + m.def("gelu_quick(Tensor! out, Tensor input) -> ()"); + m.impl("gelu_quick", torch::kCUDA, &gelu_quick); + + /* + * From csrc/allreduce + */ + m.def( + "init_custom_ar(Tensor meta, Tensor rank_data, " + "str[] handles, int[] offsets, int rank, " + "bool full_nvlink) -> int"); + m.impl("init_custom_ar", torch::kCUDA, &init_custom_ar); + + m.def("all_reduce_reg(int fa, Tensor inp, Tensor! out) -> ()"); + m.impl("all_reduce_reg", torch::kCUDA, &all_reduce_reg); + + m.def( + "all_reduce_unreg(int fa, Tensor inp, Tensor reg_buffer, Tensor! out) -> " + "()"); + m.impl("all_reduce_unreg", torch::kCUDA, &all_reduce_unreg); + + m.def("dispose", &dispose); + + m.def("meta_size", &meta_size); + + m.def( + "register_buffer(int fa, Tensor t, str[] handles, " + "int[] offsets) -> ()"); + m.impl("register_buffer", torch::kCUDA, ®ister_buffer); + + m.def("get_graph_buffer_ipc_meta", &get_graph_buffer_ipc_meta); + m.def("register_graph_buffers", ®ister_graph_buffers); + + m.def("allocate_meta_buffer", &allocate_meta_buffer); + m.impl("allocate_meta_buffer", torch::kCUDA, &allocate_meta_buffer); + + m.def("get_meta_buffer_ipc_handle", &get_meta_buffer_ipc_handle); + m.impl("get_meta_buffer_ipc_handle", torch::kCPU, &get_meta_buffer_ipc_handle); + + // quick allreduce + m.def( + "qr_all_reduce(int fa, Tensor inp, Tensor out, int quant_level, bool " + "cast_bf2half) -> ()"); + m.impl("qr_all_reduce", torch::kCUDA, &qr_all_reduce); + + m.def("init_custom_qr", &init_custom_qr); + m.def("qr_destroy", &qr_destroy); + + m.def("qr_get_handle", &qr_get_handle); + + m.def("qr_open_handles(int _fa, Tensor[](b!) handles) -> ()"); + m.impl("qr_open_handles", torch::kCPU, &qr_open_handles); + + // Max input size in bytes + m.def("qr_max_size", &qr_max_size); + + /* + * From csrc/moe + */ + m.def( + "moe_align_block_size(Tensor topk_ids, int num_experts, int block_size, Tensor! sorted_token_ids, Tensor! " + "experts_ids, Tensor! num_tokens_post_pad, Tensor! cumsum_buffer, bool " + "pad_sorted_token_ids) -> ()"); + m.impl("moe_align_block_size", torch::kCUDA, &moe_align_block_size); + + m.def( + "topk_softmax(Tensor! topk_weights, Tensor! topk_indices, Tensor gating_output, bool renormalize, float " + "moe_softcapping, Tensor? correction_bias) -> ()"); + m.impl("topk_softmax", torch::kCUDA, &topk_softmax); + + m.def( + "topk_sigmoid(Tensor! topk_weights, Tensor! topk_indices, Tensor gating_output, bool renormalize, Tensor? " + "correction_bias) -> ()"); + m.impl("topk_sigmoid", torch::kCUDA, &topk_sigmoid); + + /* + * From csrc/speculative + */ + m.def( + "verify_tree_greedy(Tensor! predicts, Tensor! accept_index, Tensor! accept_token_num, " + "Tensor candidates, Tensor retrive_index, Tensor retrive_next_token, Tensor retrive_next_sibling, " + "Tensor target_predict) -> ()"); + m.impl("verify_tree_greedy", torch::kCUDA, &verify_tree_greedy); + + m.def( + "build_tree_kernel_efficient(Tensor parent_list, Tensor selected_index, Tensor verified_seq_len, " + "Tensor! tree_mask, Tensor! positions, Tensor! retrive_index, Tensor! retrive_next_token, " + "Tensor! retrive_next_sibling, int topk, int depth, int draft_token_num, int tree_mask_mode) -> " + "()"); + m.impl("build_tree_kernel_efficient", torch::kCUDA, &build_tree_kernel_efficient); + + /* + * From csrc/kvcacheio + */ + m.def( + "transfer_kv_per_layer(Tensor src_k, Tensor dst_k, Tensor src_v, Tensor dst_v, Tensor src_indices, Tensor " + "dst_indices, int item_size, int block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_per_layer", torch::kCUDA, &transfer_kv_per_layer); + m.def( + "transfer_kv_per_layer_pf_lf(Tensor src_k, Tensor dst_k, Tensor src_v, Tensor dst_v, Tensor src_indices, Tensor " + "dst_indices, int layer_id, int item_size, int src_layout_dim, int block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_per_layer_pf_lf", torch::kCUDA, &transfer_kv_per_layer_pf_lf); + m.def( + "transfer_kv_all_layer(Tensor src_k_layers, Tensor dst_k_layers, Tensor src_v_layers, Tensor dst_v_layers, " + "Tensor src_indices, Tensor dst_indices, int item_size, int num_layers, int block_quota, int " + "num_warps_per_block) -> ()"); + m.impl("transfer_kv_all_layer", torch::kCUDA, &transfer_kv_all_layer); + m.def( + "transfer_kv_all_layer_lf_pf(Tensor src_k_layers, Tensor dst_k, Tensor src_v_layers, Tensor dst_v, " + "Tensor src_indices, Tensor dst_indices, int item_size, int dst_layout_dim, int num_layers, int block_quota, int " + "num_warps_per_block) -> ()"); + m.impl("transfer_kv_all_layer_lf_pf", torch::kCUDA, &transfer_kv_all_layer_lf_pf); + m.def( + "transfer_kv_per_layer_mla(Tensor src, Tensor dst, Tensor src_indices, Tensor dst_indices, int item_size, int " + "block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_per_layer_mla", torch::kCUDA, &transfer_kv_per_layer_mla); + m.def( + "transfer_kv_per_layer_mla_pf_lf(Tensor src, Tensor dst, Tensor src_indices, Tensor dst_indices, int layer_id, " + "int item_size, int src_layout_dim, int block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_per_layer_mla_pf_lf", torch::kCUDA, &transfer_kv_per_layer_mla_pf_lf); + m.def( + "transfer_kv_all_layer_mla(Tensor src_layers, Tensor dst_layers, Tensor src_indices, Tensor dst_indices, int " + "item_size, int num_layers, int block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_all_layer_mla", torch::kCUDA, &transfer_kv_all_layer_mla); + m.def( + "transfer_kv_all_layer_mla_lf_pf(Tensor src_layers, Tensor dst, Tensor src_indices, Tensor dst_indices, " + "int item_size, int dst_layout_dim, int num_layers, int block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_all_layer_mla_lf_pf", torch::kCUDA, &transfer_kv_all_layer_mla_lf_pf); + m.def( + "transfer_kv_direct(Tensor[] src_layers, Tensor[] dst_layers, Tensor src_indices, Tensor dst_indices, int " + "page_size) -> ()"); + m.impl("transfer_kv_direct", torch::kCUDA, &transfer_kv_direct); + m.def( + "transfer_kv_per_layer_direct_pf_lf(Tensor[] src_ptrs, Tensor[] dst_ptrs, Tensor src_indices, " + "Tensor dst_indices, int layer_id, int page_size)->() "); + m.impl("transfer_kv_per_layer_direct_pf_lf", torch::kCUDA, &transfer_kv_per_layer_direct_pf_lf); + m.def( + "transfer_kv_all_layer_direct_lf_pf(Tensor[] src_ptrs, Tensor[] dst_ptrs, Tensor src_indices, " + "Tensor dst_indices, int page_size) ->() "); + m.impl("transfer_kv_all_layer_direct_lf_pf", torch::kCUDA, &transfer_kv_all_layer_direct_lf_pf); + m.def( + "transfer_kv_all_layer_lf_ph(Tensor src_k_layers, Tensor dst_k, Tensor src_v_layers, Tensor dst_v, " + "Tensor src_indices, Tensor dst_indices, int item_size, int dst_layout_dim, int num_layers, int page_size, int " + "head_num, int block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_all_layer_lf_ph", torch::kCUDA, &transfer_kv_all_layer_lf_ph); + m.def( + "transfer_kv_per_layer_ph_lf(Tensor src_k, Tensor dst_k, Tensor src_v, Tensor dst_v, Tensor src_indices, Tensor " + "dst_indices, int layer_id, int item_size, int src_layout_dim, int page_size, int head_num, int block_quota, int " + "num_warps_per_block) -> ()"); + m.impl("transfer_kv_per_layer_ph_lf", torch::kCUDA, &transfer_kv_per_layer_ph_lf); + + /* + * From csrc/grammar + */ + m.def("apply_token_bitmask_inplace_cuda(Tensor logits, Tensor bitmask, Tensor? indices=None) -> ()"); + m.impl("apply_token_bitmask_inplace_cuda", &ApplyTokenBitmaskInplace); +} + +REGISTER_EXTENSION(common_ops) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/CMakeLists.txt b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/CMakeLists.txt new file mode 100755 index 00000000..220ed3e2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/CMakeLists.txt @@ -0,0 +1,99 @@ +cmake_minimum_required(VERSION 3.18 FATAL_ERROR) +project(sgl_kernel) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +find_package(Python COMPONENTS Interpreter Development.Module ${SKBUILD_SABI_COMPONENT} REQUIRED) + +execute_process( + COMMAND ${Python_EXECUTABLE} + -c "import torch; print(torch.utils.cmake_prefix_path)" + OUTPUT_VARIABLE TORCH_PY_PREFIX + OUTPUT_STRIP_TRAILING_WHITESPACE +) + +message(STATUS ${TORCH_PY_PREFIX}) +list(APPEND CMAKE_PREFIX_PATH ${TORCH_PY_PREFIX}/Torch) +find_package(Torch REQUIRED) + +include_directories( + ${TORCH_INCLUDE_DIRS} + ${TORCH_INSTALL_PREFIX}/include + ${Python_INCLUDE_DIRS} + ${CMAKE_CURRENT_SOURCE_DIR}/../../csrc + ${CMAKE_CURRENT_SOURCE_DIR}/../../include +) + +# Platform-specific library directory +if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + set(PLAT_LIB_DIR "/usr/lib/x86_64-linux-gnu") + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64") + set(PLAT_LIB_DIR "/usr/lib/aarch64-linux-gnu") + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le|ppc64") + set(PLAT_LIB_DIR "/usr/lib/powerpc64le-linux-gnu") +else() + set(PLAT_LIB_DIR "/usr/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu") +endif() +link_directories(${PLAT_LIB_DIR}) + +# Conda library path support +if(DEFINED ENV{CONDA_PREFIX}) + set(CONDA_LIB_DIR "$ENV{CONDA_PREFIX}/lib") + message(STATUS "Using Conda lib dir: ${CONDA_LIB_DIR}") + link_directories(${CONDA_LIB_DIR}) + set(CONDA_INCLUDE_DIR "$ENV{CONDA_PREFIX}/include") + include_directories(${CONDA_INCLUDE_DIR}) + + # Look for libnuma in Conda's lib directory + find_library(NUMA_LIB numa HINTS "${CONDA_LIB_DIR}") + if(NUMA_LIB) + message(STATUS "Found libnuma: ${NUMA_LIB}") + else() + message(FATAL_ERROR "libnuma not found in Conda environment at ${CONDA_LIB_DIR}\n" + "Please install it using: conda install libnuma numactl\n") + endif() +else() + if(DEFINED ENV{VIRTUAL_ENV}) + set(VENV_LIB_DIR "$ENV{VIRTUAL_ENV}/lib") + message(STATUS "Using venv lib dir: ${VENV_LIB_DIR}") + link_directories(${VENV_LIB_DIR}) + set(VENV_INCLUDE_DIR "$ENV{VIRTUAL_ENV}/include") + include_directories(${VENV_INCLUDE_DIR}) + endif() + # Look for libnuma in system env paths + find_library(NUMA_LIB numa) + if(NUMA_LIB) + message(STATUS "Found libnuma: ${NUMA_LIB}") + else() + message(FATAL_ERROR "libnuma not found in system environment\n" + "Please install it using: apt-get install libnuma numactl\n") + endif() +endif() + +file(GLOB SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp") + +if(NOT DEFINED ENV{SGLANG_CPU_FP8_CVT_FTZ}) + set(ENV{SGLANG_CPU_FP8_CVT_FTZ} "1") +endif() + +if("$ENV{SGLANG_CPU_FP8_CVT_FTZ}" STREQUAL "1") + message(STATUS "Enabling macro: SGLANG_CPU_FP8_CVT_FTZ") + add_compile_definitions(SGLANG_CPU_FP8_CVT_FTZ) +endif() + +add_compile_options( + -O3 + -Wno-unknown-pragmas + -march=native + -fopenmp +) + +Python_add_library(common_ops MODULE USE_SABI ${SKBUILD_SABI_VERSION} WITH_SOABI ${SOURCES}) +target_link_libraries(common_ops PRIVATE ${TORCH_LIBRARIES} ${NUMA_LIB}) +target_include_directories(common_ops PRIVATE ${TORCH_INCLUDE_DIRS}) + +install(TARGETS common_ops + LIBRARY DESTINATION sgl_kernel +) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/activation.cpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/activation.cpp new file mode 100644 index 00000000..70756776 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/activation.cpp @@ -0,0 +1,135 @@ +#include "common.h" +#include "vec.h" + +namespace { + +template +void act_and_mul_kernel_impl( + scalar_t* __restrict__ output, + const scalar_t* __restrict__ input, + int64_t num_tokens, + int64_t dim, + const func_t& f, + const vec_func_t& vf) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + + constexpr int64_t kVecSize = bVec::size(); + at::parallel_for(0, num_tokens, 0, [&](int64_t begin, int64_t end) { + for (int64_t i = begin; i < end; ++i) { + // local ptrs + const scalar_t* __restrict__ input_ptr = input + i * 2 * dim; + const scalar_t* __restrict__ input_other_ptr = input_ptr + dim; + scalar_t* __restrict__ output_ptr = output + i * dim; + + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= dim - kVecSize; d += kVecSize) { + bVec x_bvec = bVec::loadu(input_ptr + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + bVec y_bvec = bVec::loadu(input_other_ptr + d); + fVec y_fvec0, y_fvec1; + std::tie(y_fvec0, y_fvec1) = at::vec::convert_to_float(y_bvec); + + x_fvec0 = vf(x_fvec0); + x_fvec1 = vf(x_fvec1); + + x_fvec0 = x_fvec0 * y_fvec0; + x_fvec1 = x_fvec1 * y_fvec1; + + x_bvec = convert_from_float_ext(x_fvec0, x_fvec1); + x_bvec.store(output_ptr + d); + } +#pragma GCC unroll 4 + for (; d < dim; ++d) { + float x_val = static_cast(input_ptr[d]); + float y_val = static_cast(input_other_ptr[d]); + output_ptr[d] = f(x_val) * y_val; + } + } + }); +} + +} // anonymous namespace + +// input : {num_tokens, 2 * d} +// output : {num_tokens, d} +at::Tensor silu_and_mul_cpu(at::Tensor& input) { + RECORD_FUNCTION("sgl-kernel::silu_and_mul_cpu", std::vector({input})); + auto sizes = input.sizes().vec(); + int64_t last_dim = input.ndimension() - 1; + int64_t d = sizes[last_dim] / 2; + sizes[last_dim] = d; + int64_t num_tokens = input.numel() / input.size(-1); + at::Tensor out = at::empty(sizes, input.options()); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(input.scalar_type(), "silu_and_mul", [&] { + using Vec = at::vec::Vectorized; + act_and_mul_kernel_impl( + out.data_ptr(), + input.data_ptr(), + num_tokens, + d, + [](float x) { return x / (1.f + std::exp(-x)); }, + [](Vec x) { return x / (Vec(1.f) + x.neg().exp()); }); + }); + return out; +} + +at::Tensor gelu_tanh_and_mul_cpu(const at::Tensor& input) { + RECORD_FUNCTION("sgl-kernel::gelu_tanh_and_mul_cpu", std::vector({input})); + auto sizes = input.sizes().vec(); + int64_t last_dim = input.ndimension() - 1; + int64_t d = sizes[last_dim] / 2; + sizes[last_dim] = d; + int64_t num_tokens = input.numel() / input.size(-1); + at::Tensor out = at::empty(sizes, input.options()); + const float sqrt_2_div_pi = std::sqrt(2.f / M_PI); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(input.scalar_type(), "gelu_tanh_and_mul", [&] { + using Vec = at::vec::Vectorized; + act_and_mul_kernel_impl( + out.data_ptr(), + input.data_ptr(), + num_tokens, + d, + [sqrt_2_div_pi](float x) { + float x3 = x * x * x; + float tanh_arg = sqrt_2_div_pi * (x + 0.044715f * x3); + return 0.5f * x * (1.f + std::tanh(tanh_arg)); + }, + [sqrt_2_div_pi](Vec x) { + Vec x3 = x * x * x; + Vec tanh_arg = Vec(sqrt_2_div_pi) * (x + Vec(0.044715f) * x3); + return Vec(0.5f) * x * (Vec(1.f) + tanh_arg.tanh()); + }); + }); + + return out; +} + +at::Tensor gelu_and_mul_cpu(const at::Tensor& input) { + RECORD_FUNCTION("sgl-kernel::gelu_and_mul_cpu", std::vector({input})); + auto sizes = input.sizes().vec(); + int64_t last_dim = input.ndimension() - 1; + int64_t d = sizes[last_dim] / 2; + sizes[last_dim] = d; + int64_t num_tokens = input.numel() / input.size(-1); + at::Tensor out = at::empty(sizes, input.options()); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(input.scalar_type(), "gelu_and_mul", [&] { + using Vec = at::vec::Vectorized; + const float inv_sqrt2 = 1.0f / std::sqrt(2.0f); + act_and_mul_kernel_impl( + out.data_ptr(), + input.data_ptr(), + num_tokens, + d, + [inv_sqrt2](float x) { return 0.5f * x * (1.f + std::erf(x * inv_sqrt2)); }, + [inv_sqrt2](Vec x) { return Vec(0.5f) * x * (Vec(1.f) + (x * Vec(inv_sqrt2)).erf()); }); + }); + + return out; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/bmm.cpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/bmm.cpp new file mode 100644 index 00000000..9e809a46 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/bmm.cpp @@ -0,0 +1,123 @@ +#include "common.h" +#include "gemm.h" +#include "vec.h" + +namespace { + +template +void bmm_kernel_impl( + scalar_t* __restrict__ out, + const scalar_t* __restrict__ mat1, + const scalar_t* __restrict__ mat2, + int64_t B, + int64_t M, + int64_t N, + int64_t K, + int64_t mat1_strideB, + int64_t mat1_strideM, + int64_t out_strideB, + int64_t out_strideM, + float scale = 0.f) { + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + + // mat2 contiguous in [B, N, K] + int64_t mat2_strideB = N * K; + int64_t mat2_strideN = K; + + const bool use_brgemm = can_use_brgemm(M); + + // parallel on [B, MB, NB] + at::parallel_for(0, B * MB * NB, 0, [&](int64_t begin, int64_t end) { + int64_t bs{0}, mb{0}, nb{0}; + data_index_init(begin, bs, B, mb, MB, nb, NB); + + // for brgemm, use float32 for accumulate + alignas(64) float Ctmp[BLOCK_M * BLOCK_N]; + + for (int i = begin; i < end; ++i) { + UNUSED(i); + int mb_start = mb * BLOCK_M; + int mb_size = std::min(M - mb_start, BLOCK_M); + int nb_start = nb * BLOCK_N; + int nb_size = std::min(N - nb_start, BLOCK_N); + + tinygemm_kernel( + /* A */ mat1 + bs * mat1_strideB + mb_start * mat1_strideM, + /* B */ mat2 + bs * mat2_strideB + nb_start * mat2_strideN /* nb * BLOCK_N * K */, + /* C */ out + bs * out_strideB + mb_start * out_strideM + nb_start, + /* Ctmp*/ Ctmp, + /* M */ mb_size, + /* N */ nb_size, + /* K */ K, + /* lda */ mat1_strideM, + /* ldb */ nb_size, + /* ldc */ out_strideM, + /* brg */ use_brgemm); + + // move to the next index + data_index_step(bs, B, mb, MB, nb, NB); + } + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); +} + +} // anonymous namespace + +// mat1 : [B, M, K] +// mat2 : [B, N, K] or [B, OC, IC] +// out : [B, M, N] +// scale: [] 0-dim tensor for per tensor quant +// +void bmm_cpu( + at::Tensor& out, at::Tensor& mat1, at::Tensor& mat2, bool is_vnni, const std::optional& scale) { + RECORD_FUNCTION("sgl-kernel::bmm_cpu", std::vector({out, mat1, mat2})); + + auto packed_w = is_vnni ? mat2 : convert_weight_packed(mat2); + + // input and out could be non-contiguous + // weight needs to be contiguous in [OC, IC] order + CHECK_LAST_DIM_CONTIGUOUS_INPUT(mat1); + CHECK_LAST_DIM_CONTIGUOUS_INPUT(out); + CHECK_INPUT(mat2); + CHECK_DIM(3, out); + CHECK_DIM(3, mat1); + CHECK_DIM(3, mat2); + + int64_t B = mat1.size(0); + int64_t M = mat1.size(1); + int64_t N = mat2.size(1); + int64_t K = mat1.size(2); + + TORCH_CHECK(!scale.has_value(), "bmm: do not support fp8 weight for now.") + TORCH_CHECK(N % 32 == 0, "tinygemm requires N to be 32x."); + + int64_t mat1_strideB = mat1.stride(0); + int64_t mat1_strideM = mat1.stride(1); + int64_t out_strideB = out.stride(0); + int64_t out_strideM = out.stride(1); + + // check shapes + TORCH_CHECK(mat2.size(0) == B && mat2.size(2) == K, "bmm: mat2 shape mismatch!"); + TORCH_CHECK(out.size(0) == B && out.size(1) == M, "bmm: out shape mismatch!"); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(mat1.scalar_type(), "bmm_kernel_impl", [&] { + bmm_kernel_impl( + out.data_ptr(), + mat1.data_ptr(), + packed_w.data_ptr(), + B, + M, + N, + K, + mat1_strideB, + mat1_strideM, + out_strideB, + out_strideM); + }); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/common.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/common.h new file mode 100644 index 00000000..0fb13260 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/common.h @@ -0,0 +1,324 @@ +#pragma once + +#include +#include +#include + +#if defined(_OPENMP) +#include +#endif + +namespace { + +// dispatch bool +#define AT_DISPATCH_BOOL(BOOL_V, BOOL_NAME, ...) \ + [&] { \ + if (BOOL_V) { \ + constexpr bool BOOL_NAME = true; \ + return __VA_ARGS__(); \ + } else { \ + constexpr bool BOOL_NAME = false; \ + return __VA_ARGS__(); \ + } \ + }() + +// dispatch: bfloat16, float16, int8_t, fp8_e4m3 +#define CPU_DISPATCH_PACKED_TYPES(TYPE, ...) \ + [&] { \ + switch (TYPE) { \ + case at::ScalarType::BFloat16: { \ + using packed_t = at::BFloat16; \ + return __VA_ARGS__(); \ + } \ + case at::ScalarType::Half: { \ + using packed_t = at::Half; \ + return __VA_ARGS__(); \ + } \ + case at::ScalarType::Char: { \ + using packed_t = int8_t; \ + return __VA_ARGS__(); \ + } \ + case at::ScalarType::Float8_e4m3fn: { \ + using packed_t = at::Float8_e4m3fn; \ + return __VA_ARGS__(); \ + } \ + default: \ + TORCH_CHECK(false, "Unsupported floating data type.\n"); \ + } \ + }() + +// dispatch with mixed dtypes (TYPE1, TYPE2): +// TYPE1: the primary dtype (input, output, weight); +// TYPE2: the secondary dtype (bias, etc.). +#define CPU_DISPATCH_REDUCED_FLOATING_TYPES_EXT(TYPE1, TYPE2, ...) \ + [&] { \ + if (TYPE2 == at::kFloat) { \ + switch (TYPE1) { \ + case at::ScalarType::BFloat16: { \ + using scalar_t = at::BFloat16; \ + using param_t = float; \ + return __VA_ARGS__(); \ + } \ + case at::ScalarType::Half: { \ + using scalar_t = at::Half; \ + using param_t = float; \ + return __VA_ARGS__(); \ + } \ + default: \ + TORCH_CHECK(false, "Unsupported floating data type.\n"); \ + } \ + } else { \ + TORCH_CHECK(TYPE1 == TYPE2); \ + switch (TYPE1) { \ + case at::ScalarType::BFloat16: { \ + using scalar_t = at::BFloat16; \ + using param_t = at::BFloat16; \ + return __VA_ARGS__(); \ + } \ + case at::ScalarType::Half: { \ + using scalar_t = at::Half; \ + using param_t = at::Half; \ + return __VA_ARGS__(); \ + } \ + default: \ + TORCH_CHECK(false, "Unsupported floating data type.\n"); \ + } \ + } \ + }() + +#define UNUSED(x) (void)(x) + +#define CHECK_CPU(x) TORCH_CHECK(x.device().type() == at::kCPU, #x " must be a CPU tensor") + +#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_LAST_DIM_CONTIGUOUS(x) \ + TORCH_CHECK(x.strides()[x.strides().size() - 1] == 1, #x "must be contiguous at last dimension") + +#define CHECK_INPUT(x) \ + CHECK_CPU(x); \ + CHECK_CONTIGUOUS(x) +#define CHECK_LAST_DIM_CONTIGUOUS_INPUT(x) \ + CHECK_CPU(x); \ + CHECK_LAST_DIM_CONTIGUOUS(x) + +#define CHECK_DIM(d, x) TORCH_CHECK(x.dim() == d, #x " must be a " #d "D tensor") + +#define CHECK_EQ(a, b) TORCH_CHECK((a) == (b), "CHECK_EQ(" #a ", " #b ") failed. ", a, " vs ", b) + +// [NB] Parallel Routines +// +// * at::parallel_for - applies for most of generic use cases, this will be compiled +// against openmp in default torch release. +// +// * parallel_for - same function as above, can choose payload partition scheme in +// balance211. +// +// * parallel_2d - parallel for 2 dimensions, used in GEMM, etc. +// this one will do payload balance across 2 dimensions. +// + +// grain size for each thread +constexpr int GRAIN_SIZE = 1024; + +template ::value, int>::type = 0> +inline T div_up(T x, T y) { + return (x + y - 1) / y; +} + +// you can only use at::get_thread_num() with at::parallel_for() +// as it is lazy initialized, otherwise it will always return 0. +inline int get_thread_num() { +#if defined(_OPENMP) + return omp_get_thread_num(); +#else + return 0; +#endif +} + +// balance payload across each thread +template +inline void balance211(T n, T nth, T ith, T& n_start, T& n_end) { +#if 0 + // onednn partition pattern + T& n_my = n_end; + if (nth <= 1 || n == 0) { + n_start = 0; + n_my = n; + } else { + T n1 = div_up(n, nth); + T n2 = n1 - 1; + T T1 = n - n2 * nth; + n_my = ith < T1 ? n1 : n2; + n_start = ith <= T1 ? ith*n1 : T1 * n1 + (ith - T1) * n2; + } + n_end += n_start; +#else + // pytorch aten partition pattern + T n_my = div_up(n, nth); + n_start = ith * n_my; + n_end = std::min(n_start + n_my, n); +#endif +} + +template +inline void parallel_for(int n, const func_t& f) { +#if defined(_OPENMP) +#pragma omp parallel + { + int nth = omp_get_num_threads(); + int ith = omp_get_thread_num(); + int tbegin, tend; + balance211(n, nth, ith, tbegin, tend); + f(tbegin, tend); + } +#else + f(0, n); +#endif +} + +// for 1d parallel, use `actual_nth` +// for 2d parallel, use even nths, e.g. 43->42 +int inline adjust_num_threads(int m) { + int actual_nth = at::get_num_threads(); + if (m == 1) { + return actual_nth; + } + return std::max(1, (actual_nth >> 1) * 2); +} + +template +inline void parallel_2d(int m, int n, const func_t& f) { + // make sure we have even num_threads + int nth = adjust_num_threads(m); + + // [NOTE] thread blocking: + // + // 1) prefer square block per thread + // 2) use even number of CPU cores + // 3) use all `num_threads` cores + // + // we have: + // TM * TN = T + // BM / TM = BN / TN + // then: + // TM = ((BM / BN) * T) ^ 0.5 + // + float r = float(m) / n; + int nth_m = std::ceil(std::sqrt(r * nth)); + int nth_n = 1; + for (; nth_m > 0; --nth_m) { + nth_n = nth / nth_m; + if (nth_m * nth_n == nth) { + break; + } + } + +#if defined(_OPENMP) +#pragma omp parallel num_threads(nth) + { + int ith = omp_get_thread_num(); + int ith_m = ith / nth_n; + int ith_n = ith % nth_n; + + int thread_block_m = div_up(m, nth_m); + int thread_block_n = div_up(n, nth_n); + + int begin_m = ith_m * thread_block_m; + int end_m = std::min(m, begin_m + thread_block_m); + int begin_n = ith_n * thread_block_n; + int end_n = std::min(n, begin_n + thread_block_n); + + f(begin_m, end_m, begin_n, end_n); + } +#else + f(0, m, 0, n); +#endif +} + +// limit max cache blocks +// when we need to do pre-unpack for weights, e.g. fp8 +#define MAX_CACHE_BLOCK_SIZE 4 + +template +inline int get_cache_blocks(int chunk_size) { + // L2 2MB and ratio of 50% + const int L2_size = 2048 * 1024 >> 1; + return std::max(1, int(L2_size / (chunk_size * sizeof(T)))); +} + +template <> +inline int get_cache_blocks(int chunk_size) { + // fp8 uses bf16 as accumulate type + int cache_block_size = get_cache_blocks(chunk_size); + return std::min(MAX_CACHE_BLOCK_SIZE, cache_block_size); +} + +// 2d sequential loop in range : [mb0, mb1), [nb0, nb1) +template +inline void loop_2d(int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1, int64_t chunk_size, const func_t& f) { + // get number of blocks for L2 in most inner loop + int64_t cache_blocks_nb = get_cache_blocks(chunk_size); + + // loop order: [NB / cache_blocks_nb, MB, cache_blocks_nb] + // TODO: implement reverse order of [MB / cache_blocks_mb, NB, cache_blocks_mb] + for (int64_t nbb = nb0; nbb < nb1; nbb += cache_blocks_nb) { + for (int64_t mb = mb0; mb < mb1; ++mb) { + for (int64_t nb = nbb; nb < std::min(nbb + cache_blocks_nb, nb1); ++nb) { + f(mb, nb, nb - nbb); + } + } + } +} + +// data indexing for dimension collapse +template +inline T data_index_init(T offset) { + return offset; +} + +template +inline T data_index_init(T offset, T& x, const T& X, Args&&... args) { + offset = data_index_init(offset, std::forward(args)...); + x = offset % X; + return offset / X; +} + +inline bool data_index_step() { + return true; +} + +template +inline bool data_index_step(T& x, const T& X, Args&&... args) { + if (data_index_step(std::forward(args)...)) { + x = ((x + 1) == X) ? 0 : (x + 1); + return x == 0; + } + return false; +} + +// forced unroll for perf critical path + +#if __has_attribute(always_inline) +#define ALWAYS_INLINE __attribute__((__always_inline__)) inline +#else +#define ALWAYS_INLINE inline +#endif + +template +struct Unroll { + template + ALWAYS_INLINE void operator()(const Func& f, Args... args) const { + Unroll{}(f, args...); + f(std::integral_constant{}, args...); + } +}; + +template <> +struct Unroll<1> { + template + ALWAYS_INLINE void operator()(const Func& f, Args... args) const { + f(std::integral_constant{}, args...); + } +}; + +} // anonymous namespace diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/decode.cpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/decode.cpp new file mode 100644 index 00000000..3de2708e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/decode.cpp @@ -0,0 +1,1744 @@ +#include "common.h" +#include "gemm.h" +#include "vec.h" + +namespace { + +// [NOTE] TODO list for this kernel: +// 1. tune the value for BLOCK_N +// 2. planning for {batches, num_heads, num_kv_splits} +// and use actual num_kv_splits for small seq length +// 3. try fast impl of `.tanh()` +// 4. provide amx kernel for index_gemm_kernel_nn when M = 16 +// + +#if defined(CPU_CAPABILITY_AVX512) +// key: from [N, 32] to [32/2, N, 2] +// val: from [N, 32] to [N/2, 32, 2] +template +inline void pack_vnni_Nx32( + scalar_t* __restrict__ dst0, + scalar_t* __restrict__ dst1, + const scalar_t* __restrict__ src, + const index_t* __restrict__ ind, + int N, + int ld_src, + int ld_dst0, + int ld_dst1, + bool convert_v) { + __m512i vinputs[16]; + int n = 0; + for (; n < N; ++n) { + vinputs[n] = _mm512_loadu_si512(src + ind[n] * ld_src); + } + // padding with zero to avoid uninitialized vectors + for (; n < 16; ++n) { + vinputs[n] = _mm512_set1_epi32(0); + } + + // pack value, skip 64 elems for deepseek + // handle 2 vectors at a time from [2, 32] to [32, 2] + if (convert_v) { + for (int n = 0; n < 16; n += 2) { + __m512i d0, d1; + std::tie(d0, d1) = transpose_2x32_16bit(vinputs[n], vinputs[n + 1]); + _mm512_storeu_si512(dst1 + (n >> 1) * ld_dst1 * 2, d0); + _mm512_storeu_si512(dst1 + (n >> 1) * ld_dst1 * 2 + 32, d1); + } + } + + // pack key + transpose_16x16_32bit(vinputs); + + const __mmask16 vmask = (1 << N) - 1; + for (int k = 0; k < 16; ++k) { + _mm512_mask_storeu_epi32(dst0 + k * ld_dst0 * 2, vmask, vinputs[k]); + } +} +#endif + +// [NOTE]: MLA vnni format conversion +// +// here we apply same strategy as `FlashMLA`: +// each kv_cache is loaded once and packed twice (L2 cache hit) +// +// * for key: from [N, K/2, 2] to [K/2, N, 2] +// * for value: from [N/2, 2, Kv] to [N/2, Kv, 2] +// +template +void pack_vnni( + scalar_t* __restrict__ dst0, + scalar_t* __restrict__ dst1, + const scalar_t* __restrict__ src, + const index_t* __restrict__ ind, + int N, + int K, + int Kv, + int ld_src, + int ld_dst0, + int ld_dst1) { +#if defined(CPU_CAPABILITY_AVX512) + const int NB = div_up(N, 16); + const int KB = K / 32; // no remainder + const int KBv = Kv / 32; // no remainder + + for (int nb = 0; nb < NB; ++nb) { + for (int kb = 0; kb < KB; ++kb) { + // handle 16x512bits each block + int nb_size = std::min(N - nb * 16, 16); + pack_vnni_Nx32( + /* dst0 */ dst0 + ((kb * 32) >> 1) * ld_dst0 * 2 + nb * 16 * 2, + /* dst1 */ dst1 + ((nb * 16) >> 1) * ld_dst1 * 2 + kb * 32 * 2, + /* src */ src + kb * 32, + /* ind */ ind + nb * 16, + /* N */ nb_size, + /* ld_src */ ld_src, + /* ld_dst0 */ ld_dst0, + /* ld_dst1 */ ld_dst1, + /* cvt_v */ kb < KBv); + } + } +#else + for (int n = 0; n < N; ++n) { + index_t index = ind[n]; + for (int k = 0; k < K / 2; ++k) { + for (int d = 0; d < 2; ++d) { + dst0[k * ld_dst0 * 2 + n * 2 + d] = src[index * ld_src + k * 2 + d]; + } + } + } + // from [N/2, 2, K] to [N/2, K, 2] + for (int n = 0; n < (N >> 1) * 2; n += 2) { + index_t index0 = ind[n + 0]; + index_t index1 = ind[n + 1]; + for (int k = 0; k < Kv; ++k) { + dst1[(n >> 1) * ld_dst1 * 2 + k * 2 + 0] = src[index0 * ld_src + k]; + dst1[(n >> 1) * ld_dst1 * 2 + k * 2 + 1] = src[index1 * ld_src + k]; + } + } + if (N % 2 != 0) { + index_t index = ind[N - 1]; + for (int k = 0; k < Kv; ++k) { + dst1[(N >> 1) * ld_dst1 * 2 + k * 2 + 0] = src[index * ld_src + k]; + dst1[(N >> 1) * ld_dst1 * 2 + k * 2 + 1] = 0; + } + } +#endif +} + +template +inline void fill_stub(scalar_t* __restrict__ out, float val, int64_t size) { + using Vec = at::vec::Vectorized; + constexpr int kVecSize = Vec::size(); + const Vec data_vec = Vec(static_cast(val)); + int64_t d = 0; +#pragma GCC unroll 4 + for (; d <= size - kVecSize; d += kVecSize) { + data_vec.store(out + d); + } + if (size - d > 0) { + data_vec.store(out + d, size - d); + } +} + +template +inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ acc, float s, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + const fVec s_fvec = fVec(s); + int64_t d = 0; +#pragma GCC unroll 4 + for (; d <= size - kVecSize; d += kVecSize) { + fVec a_fvec0 = fVec::loadu(acc + d) * s_fvec; + fVec a_fvec1 = fVec::loadu(acc + d + fVec::size()) * s_fvec; + bVec out_bvec = convert_from_float_ext(a_fvec0, a_fvec1); + out_bvec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(acc[d] * s); + } +} + +template +inline void copy_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ src, int64_t size) { + using bVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + int64_t d = 0; +#pragma GCC unroll 4 + for (; d <= size - kVecSize; d += kVecSize) { + bVec out_bvec = bVec::loadu(src + d); + out_bvec.store(out + d); + } + for (; d < size; ++d) { + out[d] = src[d]; + } +} + +template +inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ input) { + static_assert(BLOCK_N % 32 == 0); + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + + constexpr int COLS = BLOCK_N / 16; + auto store = [&](auto i) { + constexpr int col = i % COLS; + // for COLS = 2, 4 use 512bit store + if constexpr (col % 2 == 0) { + fVec a_fvec0 = fVec::loadu(input + col * 16); + fVec a_fvec1 = fVec::loadu(input + col * 16 + 16); + bVec out_bvec = convert_from_float_ext(a_fvec0, a_fvec1); + out_bvec.store(out + col * 16); + } + }; + Unroll{}(store); +} + +// GEMM handles query @ key (indexed) x scale +// A : [M, K] +// B : [N, K] indexed +// C : [M, N] +// +template +struct tinygemm_kernel_nt { + static inline void apply( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B, + float* __restrict__ C, + const index_t* __restrict__ indices, + float scale, + int64_t lda, + int64_t ldb, + int64_t ldc, + int64_t K, + int64_t max_tokens) { + for (int64_t m = 0; m < BLOCK_M; ++m) { + for (int64_t n = 0; n < BLOCK_N; ++n) { + float sum = 0.f; + int64_t b_idx = indices[n]; + TORCH_CHECK(b_idx < max_tokens, "token index out of scope!"); + for (int64_t k = 0; k < K; ++k) { + sum += scale * static_cast(A[m * lda + k]) * static_cast(B[b_idx * ldb + k]); + } + C[m * ldc + n] = sum; + } + } + } +}; + +#if defined(CPU_CAPABILITY_AVX512) +template +struct tinygemm_kernel_nt { + static inline void apply( + const at::BFloat16* __restrict__ A, + const at::BFloat16* __restrict__ B, + float* __restrict__ C, + const index_t* __restrict__ indices, + float scale, + int64_t lda, + int64_t ldb, + int64_t ldc, + int64_t K, + int64_t max_tokens) { + constexpr int ROWS = BLOCK_M; + constexpr int COLS = BLOCK_N; + + __m512bh va; + __m512bh vb[COLS]; + __m512 vc[ROWS * COLS]; + __m512 vscale = _mm512_set1_ps(scale); + + auto loadc = [&](auto i) { vc[i] = _mm512_setzero_ps(); }; + Unroll{}(loadc); + + // for main loop + auto compute = [&](auto i, int64_t k) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + va = (__m512bh)(_mm512_loadu_si512(A + row * lda + k)); + } + if constexpr (row == 0) { + if constexpr (col + 1 < COLS) { + int64_t b_idx_prefetch = indices[col + 1]; + _mm_prefetch(B + b_idx_prefetch * ldb + k, _MM_HINT_T0); + } + int64_t b_idx = indices[col]; + TORCH_CHECK(b_idx < max_tokens, "token index out of scope!"); + vb[col] = (__m512bh)(_mm512_loadu_si512(B + b_idx * ldb + k)); + } + vc[i] = _mm512_dpbf16_ps(vc[i], va, vb[col]); + }; + + // for remainder + auto compute2 = [&](auto i, int64_t k, __mmask32 mask) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + va = (__m512bh)(_mm512_maskz_loadu_epi16(mask, A + row * lda + k)); + } + if constexpr (row == 0) { + int64_t b_idx = indices[col]; + TORCH_CHECK(b_idx < max_tokens, "token index out of scope!"); + vb[col] = (__m512bh)(_mm512_maskz_loadu_epi16(mask, B + b_idx * ldb + k)); + } + vc[i] = _mm512_dpbf16_ps(vc[i], va, vb[col]); + }; + + int64_t k = 0; + for (; k <= K - 32; k += 32) { + Unroll{}(compute, k); + } + int64_t count = K - k; + if (count > 0) { + __mmask32 mask = (1ULL << count) - 1; + Unroll{}(compute2, k, mask); + } + + auto storec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + C[row * ldc + col] = _mm512_reduce_add_ps(_mm512_mul_ps(vc[i], vscale)); + }; + Unroll{}(storec); + } +}; +#endif + +#if defined(CPU_CAPABILITY_AVX512) +template +struct tinygemm_kernel_nt { + static inline void apply( + const at::Half* __restrict__ A, + const at::Half* __restrict__ B, + float* __restrict__ C, + const index_t* __restrict__ indices, + float scale, + int64_t lda, + int64_t ldb, + int64_t ldc, + int64_t K, + int64_t max_tokens) { + constexpr int ROWS = BLOCK_M; + constexpr int COLS = BLOCK_N; + + __m512 va0, va1; + __m512 vb0[COLS], vb1[COLS]; + __m512 vc[ROWS * COLS]; + __m512 vscale = _mm512_set1_ps(scale); + + auto loadc = [&](auto i) { vc[i] = _mm512_setzero_ps(); }; + Unroll{}(loadc); + + auto compute = [&](auto i, int64_t k) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + __m512i a16 = _mm512_loadu_si512((__m512i const*)(A + row * lda + k)); + va0 = CVT_FP16_TO_FP32(_mm512_extracti32x8_epi32(a16, 0)); + va1 = CVT_FP16_TO_FP32(_mm512_extracti32x8_epi32(a16, 1)); + } + + if constexpr (row == 0) { + int64_t b_idx = indices[col]; + TORCH_CHECK(b_idx < max_tokens, "token index out of scope!"); + __m512i b16 = _mm512_loadu_si512((__m512i const*)(B + b_idx * ldb + k)); + vb0[col] = CVT_FP16_TO_FP32(_mm512_extracti32x8_epi32(b16, 0)); + vb1[col] = CVT_FP16_TO_FP32(_mm512_extracti32x8_epi32(b16, 1)); + } + + vc[i] = _mm512_fmadd_ps(va0, vb0[col], _mm512_fmadd_ps(va1, vb1[col], vc[i])); + }; + + auto compute2 = [&](auto i, int64_t k, __mmask32 mask) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + __m512i a16 = _mm512_maskz_loadu_epi16(mask, (const void*)(A + row * lda + k)); + va0 = CVT_FP16_TO_FP32(_mm512_extracti32x8_epi32(a16, 0)); + va1 = CVT_FP16_TO_FP32(_mm512_extracti32x8_epi32(a16, 1)); + } + + if constexpr (row == 0) { + int64_t b_idx = indices[col]; + TORCH_CHECK(b_idx < max_tokens, "token index out of scope!"); + __m512i b16 = _mm512_maskz_loadu_epi16(mask, (const void*)(B + b_idx * ldb + k)); + vb0[col] = CVT_FP16_TO_FP32(_mm512_extracti32x8_epi32(b16, 0)); + vb1[col] = CVT_FP16_TO_FP32(_mm512_extracti32x8_epi32(b16, 1)); + } + + vc[i] = _mm512_fmadd_ps(va0, vb0[col], _mm512_fmadd_ps(va1, vb1[col], vc[i])); + }; + + int64_t k = 0; + for (; k <= K - 32; k += 32) { + Unroll{}(compute, k); + } + int64_t count = K - k; + if (count > 0) { + __mmask32 mask = (1ULL << count) - 1; + Unroll{}(compute2, k, mask); + } + + auto storec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + C[row * ldc + col] = _mm512_reduce_add_ps(_mm512_mul_ps(vc[i], vscale)); + }; + Unroll{}(storec); + } +}; +#endif + +#define LAUNCH_TINYGEMM_KERNEL_NT(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_nt::apply( \ + A + mb_start * lda, B, C + mb_start * ldc + nb_start, indices + nb_start, scale, lda, ldb, ldc, K, max_tokens); + +// this is used when N isn't multiple of 16, +// N corresponds to `head_size_v` which should be 16x +template +inline void tinygemm_kernel_nn_scalar( + const float* __restrict__ A, + const scalar_t* __restrict__ B, + float* __restrict__ C, + const index_t* __restrict__ indices, + const float* __restrict__ scale, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + int64_t max_tokens) { + for (int64_t m = 0; m < M; ++m) { + for (int64_t n = 0; n < N; ++n) { + C[m * ldc + n] *= scale[m]; + for (int64_t k = 0; k < K; ++k) { + int64_t b_idx = indices[k]; + TORCH_CHECK(b_idx < max_tokens, "token index out of scope!"); + C[m * ldc + n] += A[m * lda + k] * static_cast(B[b_idx * ldb + n]); + } + } + } +} + +// GEMM handles v' * scale + attn @ value (indexed) +// A : [M, K] +// B : [K, N] indexed +// C ：[M, N] +// +template +struct tinygemm_kernel_nn { + static inline void apply( + const float* __restrict__ A, + const scalar_t* __restrict__ B, + float* __restrict__ C, + const index_t* __restrict__ indices, + const float* __restrict__ scale, + int64_t lda, + int64_t ldb, + int64_t ldc, + int64_t K, + int64_t max_tokens) { + tinygemm_kernel_nn_scalar(A, B, C, indices, scale, BLOCK_M, BLOCK_N, K, lda, ldb, ldc, max_tokens); + } +}; + +#if defined(CPU_CAPABILITY_AVX512) +template +struct tinygemm_kernel_nn { + static inline void apply( + const float* __restrict__ A, + const at::BFloat16* __restrict__ B, + float* __restrict__ C, + const index_t* __restrict__ indices, + const float* __restrict__ scale, + int64_t lda, + int64_t ldb, + int64_t ldc, + int64_t K, + int64_t max_tokens) { + constexpr int ROWS = BLOCK_M; + constexpr int COLS = BLOCK_N / 16; + + __m512 va; + __m512 vb[COLS]; + __m512 vc[ROWS * COLS]; + __m512 vscale; + + auto loadc = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Warray-bounds" + if constexpr (col == 0) { + vscale = _mm512_set1_ps(scale[row]); + } +#pragma GCC diagnostic pop + vc[i] = _mm512_loadu_ps(C + row * ldc + col * 16); + vc[i] = _mm512_mul_ps(vc[i], vscale); + }; + Unroll{}(loadc); + + auto compute = [&](auto i, int64_t k) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + va = _mm512_set1_ps(A[row * lda + k]); + } + if constexpr (row == 0) { + if (k + 1 < K) { + int64_t b_idx_prefetch = indices[k + 1]; + _mm_prefetch(B + b_idx_prefetch * ldb + col * 16, _MM_HINT_T0); + } + int64_t b_idx = indices[k]; + TORCH_CHECK(b_idx < max_tokens, "token index out of scope!"); + + // for COLS = 2, 4, 6, 8 use 512 bit load + // for COLS = 1, 3, 5, 7 use 256 bit load + if constexpr (COLS % 2 == 0) { + if constexpr (col % 2 == 0) { + __m512i b16 = _mm512_loadu_si512(reinterpret_cast(B + b_idx * ldb + col * 16)); + vb[col + 0] = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32(b16, 0)); + vb[col + 1] = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32(b16, 1)); + } + } else { + __m256i b16 = _mm256_loadu_si256(reinterpret_cast(B + b_idx * ldb + col * 16)); + vb[col] = CVT_BF16_TO_FP32(b16); + } + } + vc[i] = _mm512_fmadd_ps(va, vb[col], vc[i]); + }; + + for (int64_t k = 0; k < K; ++k) { + Unroll{}(compute, k); + } + + auto storec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + _mm512_storeu_ps(C + row * ldc + col * 16, vc[i]); + }; + Unroll{}(storec); + } +}; +#endif + +#if defined(CPU_CAPABILITY_AVX512) +template +struct tinygemm_kernel_nn { + static inline void apply( + const float* __restrict__ A, + const at::Half* __restrict__ B, + float* __restrict__ C, + const index_t* __restrict__ indices, + const float* __restrict__ scale, + int64_t lda, + int64_t ldb, + int64_t ldc, + int64_t K, + int64_t max_tokens) { + constexpr int ROWS = BLOCK_M; + constexpr int COLS = BLOCK_N / 16; + + __m512 va; + __m512 vb[COLS]; + __m512 vc[ROWS * COLS]; + __m512 vscale; + + auto loadc = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Warray-bounds" + if constexpr (col == 0) { + vscale = _mm512_set1_ps(scale[row]); + } +#pragma GCC diagnostic pop + vc[i] = _mm512_loadu_ps(C + row * ldc + col * 16); + vc[i] = _mm512_mul_ps(vc[i], vscale); + }; + Unroll{}(loadc); + + auto compute = [&](auto i, int64_t k) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + va = _mm512_set1_ps(A[row * lda + k]); + } + if constexpr (row == 0) { + if (k + 1 < K) { + int64_t b_idx_prefetch = indices[k + 1]; + _mm_prefetch(B + b_idx_prefetch * ldb + col * 16, _MM_HINT_T0); + } + int64_t b_idx = indices[k]; + TORCH_CHECK(b_idx < max_tokens, "token index out of scope!"); + + // for COLS = 2, 4, 6, 8 use 512 bit load + // for COLS = 1, 3, 5, 7 use 256 bit load + if constexpr (COLS % 2 == 0) { + if constexpr (col % 2 == 0) { + __m512i b16 = _mm512_loadu_si512(reinterpret_cast(B + b_idx * ldb + col * 16)); + vb[col + 0] = CVT_FP16_TO_FP32(_mm512_extracti32x8_epi32(b16, 0)); + vb[col + 1] = CVT_FP16_TO_FP32(_mm512_extracti32x8_epi32(b16, 1)); + } + } else { + __m256i b16 = _mm256_loadu_si256(reinterpret_cast(B + b_idx * ldb + col * 16)); + vb[col] = CVT_FP16_TO_FP32(b16); + } + } + vc[i] = _mm512_fmadd_ps(va, vb[col], vc[i]); + }; + + for (int64_t k = 0; k < K; ++k) { + Unroll{}(compute, k); + } + + auto storec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + _mm512_storeu_ps(C + row * ldc + col * 16, vc[i]); + }; + Unroll{}(storec); + } +}; +#endif + +#define LAUNCH_TINYGEMM_KERNEL_NN(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_nn::apply( \ + A + mb_start * lda, \ + B + nb_start, \ + C + mb_start * ldc + nb_start, \ + indices, \ + scale + mb_start, \ + lda, \ + ldb, \ + ldc, \ + K, \ + max_tokens); + +template +void index_gemm_kernel_nt( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B, + float* __restrict__ C, + const index_t* __restrict__ indices, + float scale, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + int64_t max_tokens) { + // pattern: 1-8-8 + if (M == 1) { + constexpr int64_t BLOCK_N = 8; + const int64_t NB = div_up(N, BLOCK_N); + int64_t mb_start = 0, lda = 1, ldc = 1; + + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (nb_size) { + case 1: + LAUNCH_TINYGEMM_KERNEL_NT(1, 1); + break; + case 2: + LAUNCH_TINYGEMM_KERNEL_NT(1, 2); + break; + case 3: + LAUNCH_TINYGEMM_KERNEL_NT(1, 3); + break; + case 4: + LAUNCH_TINYGEMM_KERNEL_NT(1, 4); + break; + case 5: + LAUNCH_TINYGEMM_KERNEL_NT(1, 5); + break; + case 6: + LAUNCH_TINYGEMM_KERNEL_NT(1, 6); + break; + case 7: + LAUNCH_TINYGEMM_KERNEL_NT(1, 7); + break; + case 8: + LAUNCH_TINYGEMM_KERNEL_NT(1, 8); + break; + default: + TORCH_CHECK(false, "Unexpected block size, 1x", "nb_size"); + } + } + return; + } + + // default pattern: 1-6-24 + // FP16 pattern: 2-8-16 + constexpr int64_t BLOCK_M = 4; + constexpr int64_t BLOCK_N = std::is_same_v ? 4 : 6; + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + + for (int64_t mb = 0; mb < MB; ++mb) { + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(BLOCK_M, M - mb_start); + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (mb_size << 4 | nb_size) { + // mb_size = 1 + case 0x11: + LAUNCH_TINYGEMM_KERNEL_NT(1, 1); + break; + case 0x12: + LAUNCH_TINYGEMM_KERNEL_NT(1, 2); + break; + case 0x13: + LAUNCH_TINYGEMM_KERNEL_NT(1, 3); + break; + case 0x14: + LAUNCH_TINYGEMM_KERNEL_NT(1, 4); + break; + case 0x15: + LAUNCH_TINYGEMM_KERNEL_NT(1, 5); + break; + case 0x16: + LAUNCH_TINYGEMM_KERNEL_NT(1, 6); + break; + // mb_size = 2 + case 0x21: + LAUNCH_TINYGEMM_KERNEL_NT(2, 1); + break; + case 0x22: + LAUNCH_TINYGEMM_KERNEL_NT(2, 2); + break; + case 0x23: + LAUNCH_TINYGEMM_KERNEL_NT(2, 3); + break; + case 0x24: + LAUNCH_TINYGEMM_KERNEL_NT(2, 4); + break; + case 0x25: + LAUNCH_TINYGEMM_KERNEL_NT(2, 5); + break; + case 0x26: + LAUNCH_TINYGEMM_KERNEL_NT(2, 6); + break; + // mb_size = 3 + case 0x31: + LAUNCH_TINYGEMM_KERNEL_NT(3, 1); + break; + case 0x32: + LAUNCH_TINYGEMM_KERNEL_NT(3, 2); + break; + case 0x33: + LAUNCH_TINYGEMM_KERNEL_NT(3, 3); + break; + case 0x34: + LAUNCH_TINYGEMM_KERNEL_NT(3, 4); + break; + case 0x35: + LAUNCH_TINYGEMM_KERNEL_NT(3, 5); + break; + case 0x36: + LAUNCH_TINYGEMM_KERNEL_NT(3, 6); + break; + // mb_size = 4 + case 0x41: + LAUNCH_TINYGEMM_KERNEL_NT(4, 1); + break; + case 0x42: + LAUNCH_TINYGEMM_KERNEL_NT(4, 2); + break; + case 0x43: + LAUNCH_TINYGEMM_KERNEL_NT(4, 3); + break; + case 0x44: + LAUNCH_TINYGEMM_KERNEL_NT(4, 4); + break; + case 0x45: + LAUNCH_TINYGEMM_KERNEL_NT(4, 5); + break; + case 0x46: + LAUNCH_TINYGEMM_KERNEL_NT(4, 6); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size"); + } + } + } +} + +template +void index_gemm_kernel_nn( + const float* __restrict__ A, + const scalar_t* __restrict__ B, + float* __restrict__ C, + const index_t* __restrict__ indices, + float* __restrict__ scale, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + int64_t max_tokens) { + constexpr int kVecSize = 16; + if ((N & (kVecSize - 1)) != 0) { + tinygemm_kernel_nn_scalar(A, B, C, indices, scale, M, N, K, lda, ldb, ldc, max_tokens); + return; + } + + // pattern: 1-8-8 + if (M == 1) { + constexpr int64_t BLOCK_N = 8 * kVecSize; + const int64_t NB = div_up(N, BLOCK_N); + int64_t mb_start = 0, lda = 1, ldc = 1; + + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (nb_size >> 4) { + case 1: + LAUNCH_TINYGEMM_KERNEL_NN(1, 16); + break; + case 2: + LAUNCH_TINYGEMM_KERNEL_NN(1, 32); + break; + case 3: + LAUNCH_TINYGEMM_KERNEL_NN(1, 48); + break; + case 4: + LAUNCH_TINYGEMM_KERNEL_NN(1, 64); + break; + case 5: + LAUNCH_TINYGEMM_KERNEL_NN(1, 80); + break; + case 6: + LAUNCH_TINYGEMM_KERNEL_NN(1, 96); + break; + case 7: + LAUNCH_TINYGEMM_KERNEL_NN(1, 112); + break; + case 8: + LAUNCH_TINYGEMM_KERNEL_NN(1, 128); + break; + default: + TORCH_CHECK(false, "Unexpected block size, 1x", "nb_size"); + } + } + return; + } + + constexpr int64_t BLOCK_M = 4; + constexpr int64_t BLOCK_N = 6 * kVecSize; + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + + for (int64_t mb = 0; mb < MB; ++mb) { + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(BLOCK_M, M - mb_start); + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (mb_size << 4 | nb_size >> 4) { + // mb_size = 1 + case 0x11: + LAUNCH_TINYGEMM_KERNEL_NN(1, 16); + break; + case 0x12: + LAUNCH_TINYGEMM_KERNEL_NN(1, 32); + break; + case 0x13: + LAUNCH_TINYGEMM_KERNEL_NN(1, 48); + break; + case 0x14: + LAUNCH_TINYGEMM_KERNEL_NN(1, 64); + break; + case 0x15: + LAUNCH_TINYGEMM_KERNEL_NN(1, 80); + break; + case 0x16: + LAUNCH_TINYGEMM_KERNEL_NN(1, 96); + break; + // mb_size = 2 + case 0x21: + LAUNCH_TINYGEMM_KERNEL_NN(2, 16); + break; + case 0x22: + LAUNCH_TINYGEMM_KERNEL_NN(2, 32); + break; + case 0x23: + LAUNCH_TINYGEMM_KERNEL_NN(2, 48); + break; + case 0x24: + LAUNCH_TINYGEMM_KERNEL_NN(2, 64); + break; + case 0x25: + LAUNCH_TINYGEMM_KERNEL_NN(2, 80); + break; + case 0x26: + LAUNCH_TINYGEMM_KERNEL_NN(2, 96); + break; + // mb_size = 3 + case 0x31: + LAUNCH_TINYGEMM_KERNEL_NN(3, 16); + break; + case 0x32: + LAUNCH_TINYGEMM_KERNEL_NN(3, 32); + break; + case 0x33: + LAUNCH_TINYGEMM_KERNEL_NN(3, 48); + break; + case 0x34: + LAUNCH_TINYGEMM_KERNEL_NN(3, 64); + break; + case 0x35: + LAUNCH_TINYGEMM_KERNEL_NN(3, 80); + break; + case 0x36: + LAUNCH_TINYGEMM_KERNEL_NN(3, 96); + break; + // mb_size = 4 + case 0x41: + LAUNCH_TINYGEMM_KERNEL_NN(4, 16); + break; + case 0x42: + LAUNCH_TINYGEMM_KERNEL_NN(4, 32); + break; + case 0x43: + LAUNCH_TINYGEMM_KERNEL_NN(4, 48); + break; + case 0x44: + LAUNCH_TINYGEMM_KERNEL_NN(4, 64); + break; + case 0x45: + LAUNCH_TINYGEMM_KERNEL_NN(4, 80); + break; + case 0x46: + LAUNCH_TINYGEMM_KERNEL_NN(4, 96); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size"); + } + } + } +} + +template +void decode_set_kv_buffer( + scalar_t* __restrict__ k_buffer, + scalar_t* __restrict__ v_buffer, + const scalar_t* __restrict__ key, + const scalar_t* __restrict__ value, + const int64_t* __restrict__ loc, + int64_t batches, + int64_t num_heads_kv, + int64_t head_size, + int64_t head_size_v, + int64_t k_strideN, + int64_t k_strideH, + int64_t v_strideN, + int64_t v_strideH, + int64_t nk_strideN, + int64_t nk_strideH, + int64_t nv_strideN, + int64_t nv_strideH, + bool is_mla) { + at::parallel_for(0, batches * num_heads_kv, 0, [&](int64_t begin, int64_t end) { + int64_t bs{0}, head_kv_id{0}; + data_index_init(begin, bs, batches, head_kv_id, num_heads_kv); + + for (int64_t i = begin; i < end; i++) { + int64_t loc_val = loc[bs]; + scalar_t* k_buffer_ptr = k_buffer + loc_val * k_strideN + head_kv_id * k_strideH; + const scalar_t* new_key_ptr = key + bs * nk_strideN + head_kv_id * nk_strideH; + copy_stub(k_buffer_ptr, new_key_ptr, head_size); + if (!is_mla) { + scalar_t* v_buffer_ptr = v_buffer + loc_val * v_strideN + head_kv_id * v_strideH; + const scalar_t* new_value_ptr = value + bs * nv_strideN + head_kv_id * nv_strideH; + copy_stub(v_buffer_ptr, new_value_ptr, head_size_v); + } + + // move to the next index + data_index_step(bs, batches, head_kv_id, num_heads_kv); + } + }); +} + +template +void decode_accumulate_kv_splits( + scalar_t* __restrict__ output, + float* __restrict__ attn_logits, + int64_t batches, + int64_t num_heads, + int64_t head_size_v, + int64_t num_kv_splits, + int64_t l_stride1, + int64_t l_stride2) { + using Vec = at::vec::Vectorized; + + // parallel on [batches, num_heads] + at::parallel_for(0, batches * num_heads, 0, [&](int64_t begin, int64_t end) { + // NB: here we use logits[b][h][0] as acc, since + // for the first kv split (kv_id == 0): + // m_delta = std::exp(-inf) = 0 + // e_logic = std::exp(0) = 1 + // acc = acc * m_delta + tv * e_logic = tv + for (int64_t i = begin; i < end; ++i) { + float* __restrict__ acc = attn_logits + i * l_stride1; + + float s_prime = 0.f; + float m_prime = -std::numeric_limits::infinity(); + + // update acc with from each kv_split + for (int64_t kv_id = 0; kv_id < num_kv_splits; ++kv_id) { + float* __restrict__ tv = acc + kv_id * l_stride2; + const float tlogic = (acc + kv_id * l_stride2)[head_size_v]; + + float m_i = std::max(tlogic, m_prime); + float m_delta = std::exp(m_prime - m_i); + float e_logic = std::exp(tlogic - m_i); + if (kv_id != 0) { + at::vec::map2( + [m_delta, e_logic](Vec x, Vec y) { return x * Vec(m_delta) + y * Vec(e_logic); }, + acc, + acc, + tv, + head_size_v); + } + + s_prime = s_prime * m_delta + e_logic; + m_prime = m_i; + } + + copy_stub(output + i * head_size_v, acc, 1 / s_prime, head_size_v); + } + }); +} + +template +void decode_attention_kernel_impl( + scalar_t* __restrict__ output, + float* __restrict__ attn_logits, + const scalar_t* __restrict__ query, + const scalar_t* __restrict__ k_buffer, + const scalar_t* __restrict__ v_buffer, + const index_t* __restrict__ req_to_token, + const int64_t* __restrict__ req_pool_indices, + const int64_t* __restrict__ seq_lens, + int64_t batches, + int64_t num_heads, + int64_t head_size, + int64_t head_size_v, + int64_t num_kv_splits, + int64_t q_strideM, + int64_t q_strideH, + int64_t k_strideN, + int64_t k_strideH, + int64_t v_strideN, + int64_t v_strideH, + float scaling, + float logit_cap, + int64_t max_num_reqs, + int64_t max_context_len, + int64_t max_total_num_tokens) { + using Vec = at::vec::Vectorized; + + // strides + const int64_t l_stride1 = num_kv_splits * (head_size_v + 1); + const int64_t l_stride2 = head_size_v + 1; + + const bool has_logit_cap = logit_cap > 0; + float rlogit_cap = has_logit_cap ? 1 / logit_cap : 0.f; + + // parallel on [batches, num_heads, num_kv_splits] + at::parallel_for(0, batches * num_heads * num_kv_splits, 0, [&](int64_t begin, int64_t end) { + int64_t bs{0}, head_id{0}, kv_id{0}; + data_index_init(begin, bs, batches, head_id, num_heads, kv_id, num_kv_splits); + + // s_prime and s_delta + alignas(64) float s_i[BLOCK_N]; + float* __restrict__ s_delta = s_i; + + for (int64_t i = begin; i < end; ++i) { + // get query + const scalar_t* __restrict__ q_ptr = query + bs * q_strideM + head_id * q_strideH; + + // get key/value + int64_t seq_len_kv = seq_lens[bs]; + int64_t req_pool_id = req_pool_indices[bs]; + TORCH_CHECK(seq_len_kv <= max_context_len, "seq_len_kv out of scope!"); + TORCH_CHECK(req_pool_id < max_num_reqs, "req_pool_id out of scope!"); + + const int64_t SPLIT_SIZE = div_up(seq_len_kv, num_kv_splits); + const int64_t kv_start = kv_id * SPLIT_SIZE; + const int64_t kv_end = std::min(kv_start + SPLIT_SIZE, seq_len_kv); + + float m_prime = -std::numeric_limits::infinity(); + float s_prime = 0.f; + + // get v_prime, and init to zero + float* __restrict__ v_prime = attn_logits + i * (head_size_v + 1); + fill_stub(v_prime, 0.f, head_size_v); + + // loop over K and V sequence with BLOCK_N + for (int64_t n = kv_start; n < kv_end; n += BLOCK_N) { + int64_t n_size = std::min(BLOCK_N, kv_end - n); + + // calculate s_i <- scale * Q @ K + index_gemm_kernel_nt( + /* A */ q_ptr, + /* B */ k_buffer + head_id * k_strideH, + /* C */ s_i, + /* ind */ req_to_token + req_pool_id * max_context_len + n, + /* scl */ scaling, + /* M */ 1, + /* N */ n_size, + /* K */ head_size, + /* lda */ 1, + /* ldb */ k_strideN, + /* ldc */ 1, + /* mtt */ max_total_num_tokens); + + // TODO: `tanh` from torch uses sleef u10, going to be slow + if (has_logit_cap) { + at::vec::map( + [logit_cap, rlogit_cap](Vec x) { return Vec(logit_cap) * (x * Vec(rlogit_cap)).tanh(); }, + s_i, + s_i, + n_size); + } + + // m_i: max value per row + float m_i = at::vec::reduce_all([](Vec& x, Vec& y) { return at::vec::maximum(x, y); }, s_i, n_size); + m_i = std::max(m_i, m_prime); + + // m_delta <- exp(m' - m_i) + float m_delta = std::exp(m_prime - m_i); + + // s_delta <- exp(s_i - m_i) + at::vec::map([m_i](Vec x) { return (x - Vec(m_i)).exp_u20(); }, s_delta, s_i, n_size); + + // s' <- s' * m_delta + sum(s_delta) + s_prime *= m_delta; + s_prime += at::vec::reduce_all([](Vec& x, Vec& y) { return x + y; }, s_delta, n_size); + + m_prime = m_i; + + // calculate V' <- s_delta @ V + V' * m_delta + index_gemm_kernel_nn( + /* A */ s_delta, + /* B */ v_buffer + head_id * v_strideH, + /* C */ v_prime, + /* ind */ req_to_token + req_pool_id * max_context_len + n, + /* scl */ &m_delta, + /* M */ 1, + /* N */ head_size_v, + /* K */ n_size, + /* lda */ 1, + /* ldb */ v_strideN, + /* ldc */ 1, + /* mtt */ max_total_num_tokens); + } // loop with KV blocks + + // only update v' when kv_split_size > 0 + if (kv_end > kv_start) { + float s = 1 / s_prime; + at::vec::map([s](Vec out) { return out * Vec(s); }, v_prime, v_prime, head_size_v); + + v_prime[head_size_v] = m_prime + std::log(s_prime); + } + + // move to the next index + data_index_step(bs, batches, head_id, num_heads, kv_id, num_kv_splits); + } + }); + + decode_accumulate_kv_splits( + output, attn_logits, batches, num_heads, head_size_v, num_kv_splits, l_stride1, l_stride2); +} // MHA + +template +void decode_attention_mla_kernel_impl( + scalar_t* __restrict__ output, + float* __restrict__ attn_logits, + const scalar_t* __restrict__ query, + const scalar_t* __restrict__ k_buffer, + const scalar_t* __restrict__ v_buffer, + const index_t* __restrict__ req_to_token, + const int64_t* __restrict__ req_pool_indices, + const int64_t* __restrict__ seq_lens, + scalar_t* __restrict__ buffer, + int64_t batches, + int64_t num_heads, + int64_t head_size, + int64_t head_size_v, + int64_t num_kv_splits, + int64_t q_strideM, + int64_t q_strideH, + int64_t k_strideN, + int64_t k_strideH, + int64_t v_strideN, + int64_t v_strideH, + float scaling, + float logit_cap, + int64_t max_num_reqs, + int64_t max_context_len, + int64_t max_total_num_tokens, + int64_t buffer_size_per_thread) { + using Vec = at::vec::Vectorized; + + // block length for heads + const int64_t BLOCK_H = batches == 1 ? 6 : (batches > 16 ? 22 : 11); + + // strides + const int64_t l_stride0 = num_heads * num_kv_splits * (head_size_v + 1); + const int64_t l_stride1 = num_kv_splits * (head_size_v + 1); + const int64_t l_stride2 = head_size_v + 1; + + TORCH_CHECK(logit_cap == 0.f, "decode MLA: expect no logit_cap."); + + // partition the heads into blocks for parallel + const int64_t num_blocks = div_up(num_heads, BLOCK_H); + + // parallel on [batches, num_blocks, num_kv_splits] + at::parallel_for(0, batches * num_blocks * num_kv_splits, 0, [&](int64_t begin, int64_t end) { + int64_t bs{0}, block_id{0}, kv_id{0}; + data_index_init(begin, bs, batches, block_id, num_blocks, kv_id, num_kv_splits); + + int tid = at::get_thread_num(); + scalar_t* __restrict__ Btmp0 = buffer + tid * buffer_size_per_thread; + scalar_t* __restrict__ Btmp1 = Btmp0 + BLOCK_N * head_size; + + // init Btmp1 just once for each thread to prevent NaN + // Btmp0 is not needed as it computes full K every single time + fill_stub(Btmp1, 0.f, BLOCK_N * head_size_v); + + alignas(64) float s_i[BLOCK_H * BLOCK_N]; + float* __restrict__ s_delta = s_i; + alignas(64) scalar_t s_delta2[BLOCK_H * BLOCK_N]; + + alignas(64) float s_prime[BLOCK_H]; + alignas(64) float m_prime[BLOCK_H]; + alignas(64) float m_delta[BLOCK_H]; + + for (int64_t i = begin; i < end; ++i) { + const int64_t h_start = block_id * BLOCK_H; + const int64_t h_end = std::min(block_id * BLOCK_H + BLOCK_H, num_heads); + const int64_t h_size = h_end - h_start; + + // get query + const scalar_t* __restrict__ q_ptr = query + bs * q_strideM + h_start * q_strideH; + + int64_t seq_len_kv = seq_lens[bs]; + int64_t req_pool_id = req_pool_indices[bs]; + TORCH_CHECK(seq_len_kv <= max_context_len, "seq_len_kv out of scope!"); + TORCH_CHECK(req_pool_id < max_num_reqs, "req_pool_id out of scope!"); + + const int64_t SPLIT_SIZE = div_up(seq_len_kv, num_kv_splits); + const int64_t kv_start = kv_id * SPLIT_SIZE; + const int64_t kv_end = std::min(kv_start + SPLIT_SIZE, seq_len_kv); + + fill_stub(s_prime, 0.f, BLOCK_H); + fill_stub(m_prime, -std::numeric_limits::infinity(), BLOCK_H); + + // get v_prime, and init to zero + float* __restrict__ v_prime = attn_logits + bs * l_stride0 + h_start * l_stride1 + kv_id * l_stride2; + for (int64_t h = 0; h < h_size; ++h) { + fill_stub(v_prime + h * l_stride1, 0.f, head_size_v); + } + + // loop over K and V sequence with BLOCK_N + for (int64_t n = kv_start; n < kv_end; n += BLOCK_N) { + int64_t n_size = std::min(BLOCK_N, kv_end - n); + const int64_t padded_n_size = div_up(int(n_size), TILE_K) * TILE_K; + + // get key and pack + pack_vnni( + /* dst0 */ Btmp0, + /* dst1 */ Btmp1, + /* src */ k_buffer + /* head_kv_id */ 0 * k_strideH, + /* ind */ req_to_token + req_pool_id * max_context_len + n, + /* N */ n_size, + /* K */ head_size, + /* Kv */ head_size_v, + /* ld_src */ k_strideN, + /* ld_dst0 */ BLOCK_N, + /* ld_dst1 */ head_size_v); + + // calculate s_i <- Q @ K + at::native::cpublas::brgemm( + /* M */ h_size, + /* N */ n_size, + /* K */ head_size, + /* lda */ q_strideH, + /* ldb */ BLOCK_N, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ q_ptr, + /* B */ Btmp0, + /* C */ s_i); + + const Vec scale_vec = Vec(scaling); + for (int64_t h = 0; h < h_size; ++h) { + // s_i <- s_i * scale + at::vec::map( + [scale_vec](Vec x) { return x * scale_vec; }, s_i + h * BLOCK_N, s_i + h * BLOCK_N, n_size); + + // m_i: max value per row + float m_i = at::vec::reduce_all( + [](Vec& x, Vec& y) { return at::vec::maximum(x, y); }, s_i + h * BLOCK_N, n_size); + m_i = std::max(m_i, m_prime[h]); + + // m_delta <- exp(m' - m_i) + m_delta[h] = std::exp(m_prime[h] - m_i); + + // s_delta <- exp(s_i - m_i) + at::vec::map( + [m_i](Vec x) { return (x - Vec(m_i)).exp_u20(); }, s_delta + h * BLOCK_N, s_i + h * BLOCK_N, n_size); + + // s' <- s' * m_delta + sum(s_delta) + s_prime[h] *= m_delta[h]; + s_prime[h] += at::vec::reduce_all([](Vec& x, Vec& y) { return x + y; }, s_delta + h * BLOCK_N, n_size); + + m_prime[h] = m_i; + + // v' <- v' * m_delta + float scale_m = m_delta[h]; + at::vec::map( + [scale_m](Vec x) { return x * Vec(scale_m); }, + v_prime + h * l_stride1, + v_prime + h * l_stride1, + head_size_v); + + // pad s_delta with 0 first and then convert to scalar_t + fill_stub(s_delta + h * BLOCK_N + n_size, 0.f, padded_n_size - n_size); + copy_stub(s_delta2 + h * BLOCK_N, s_delta + h * BLOCK_N); + } + + // calculate V' <- s_delta @ V + V' + at::native::cpublas::brgemm( + /* M */ h_size, + /* N */ head_size_v, + /* K */ padded_n_size, // n_size + /* lda */ BLOCK_N, + /* ldb */ head_size_v, + /* ldc */ l_stride1, + /* add_C */ true, + /* A */ s_delta2, + /* B */ Btmp1, + /* C */ v_prime); + } // loop with KV blocks + + // only update v' when kv_split_size > 0 + if (kv_end > kv_start) { + for (int64_t h = 0; h < h_size; ++h) { + float s = 1 / s_prime[h]; + at::vec::map( + [s](Vec out) { return out * Vec(s); }, v_prime + h * l_stride1, v_prime + h * l_stride1, head_size_v); + (v_prime + h * l_stride1)[head_size_v] = m_prime[h] + std::log(s_prime[h]); + } + } + + // move to the next index + data_index_step(bs, batches, block_id, num_blocks, kv_id, num_kv_splits); + } + at::native::cpublas::brgemm_release(); + }); + + decode_accumulate_kv_splits( + output, attn_logits, batches, num_heads, head_size_v, num_kv_splits, l_stride1, l_stride2); +} // MLA + +template +void decode_attention_grouped_kernel_impl( + scalar_t* __restrict__ output, + float* __restrict__ attn_logits, + const scalar_t* __restrict__ query, + const scalar_t* __restrict__ k_buffer, + const scalar_t* __restrict__ v_buffer, + const index_t* __restrict__ req_to_token, + const int64_t* __restrict__ req_pool_indices, + const int64_t* __restrict__ seq_lens, + int64_t batches, + int64_t num_heads, + int64_t num_heads_kv, + int64_t head_size, + int64_t head_size_v, + int64_t num_kv_splits, + int64_t q_strideM, + int64_t q_strideH, + int64_t k_strideN, + int64_t k_strideH, + int64_t v_strideN, + int64_t v_strideH, + float scaling, + float logit_cap, + int64_t max_num_reqs, + int64_t max_context_len, + int64_t max_total_num_tokens) { + using Vec = at::vec::Vectorized; + + // block length for heads + // we parallel on [batches, divup(num_heads, BLOCK_H), num_kv_splits] + // use smaller BLOCK_H when batches is small to utilize all cores + constexpr int64_t kBLOCK_H = 16; + const int64_t BLOCK_H = std::min(4 * batches, kBLOCK_H); + + // strides + const int64_t l_stride0 = num_heads * num_kv_splits * (head_size_v + 1); + const int64_t l_stride1 = num_kv_splits * (head_size_v + 1); + const int64_t l_stride2 = head_size_v + 1; + + const bool has_logit_cap = logit_cap > 0; + float rlogit_cap = has_logit_cap ? 1 / logit_cap : 0.f; + + // partition the heads into blocks for parallel + const int64_t num_groups = num_heads / num_heads_kv; + const int64_t num_blocks = div_up(num_groups, BLOCK_H); + + // parallel on [batches, num_heads_kv, num_blocks, num_kv_splits] + at::parallel_for(0, batches * num_heads_kv * num_blocks * num_kv_splits, 0, [&](int64_t begin, int64_t end) { + int64_t bs{0}, head_kv_id{0}, block_id{0}, kv_id{0}; + data_index_init(begin, bs, batches, head_kv_id, num_heads_kv, block_id, num_blocks, kv_id, num_kv_splits); + + alignas(64) float s_i[BLOCK_H * BLOCK_N]; + float* __restrict__ s_delta = s_i; + + alignas(64) float s_prime[BLOCK_H]; + alignas(64) float m_prime[BLOCK_H]; + alignas(64) float m_delta[BLOCK_H]; + + for (int64_t i = begin; i < end; ++i) { + const int64_t h_start = head_kv_id * num_groups + block_id * BLOCK_H; + const int64_t h_end = head_kv_id * num_groups + std::min(block_id * BLOCK_H + BLOCK_H, num_groups); + const int64_t h_size = h_end - h_start; + + // get query + const scalar_t* __restrict__ q_ptr = query + bs * q_strideM + h_start * q_strideH; + + int64_t seq_len_kv = seq_lens[bs]; + int64_t req_pool_id = req_pool_indices[bs]; + TORCH_CHECK(seq_len_kv <= max_context_len, "seq_len_kv out of scope!"); + TORCH_CHECK(req_pool_id < max_num_reqs, "req_pool_id out of scope!"); + + const int64_t SPLIT_SIZE = div_up(seq_len_kv, num_kv_splits); + const int64_t kv_start = kv_id * SPLIT_SIZE; + const int64_t kv_end = std::min(kv_start + SPLIT_SIZE, seq_len_kv); + + fill_stub(s_prime, 0.f, BLOCK_H); + fill_stub(m_prime, -std::numeric_limits::infinity(), BLOCK_H); + + // get v_prime, and init to zero + float* __restrict__ v_prime = attn_logits + bs * l_stride0 + h_start * l_stride1 + kv_id * l_stride2; + for (int64_t h = 0; h < h_size; ++h) { + fill_stub(v_prime + h * l_stride1, 0.f, head_size_v); + } + + // loop over K and V sequence with BLOCK_N + for (int64_t n = kv_start; n < kv_end; n += BLOCK_N) { + int64_t n_size = std::min(BLOCK_N, kv_end - n); + + // calculate Q @ K + index_gemm_kernel_nt( + /* A */ q_ptr, + /* B */ k_buffer + head_kv_id * k_strideH, + /* C */ s_i, + /* ind */ req_to_token + req_pool_id * max_context_len + n, + /* scl */ scaling, + /* M */ h_size, + /* N */ n_size, + /* K */ head_size, + /* lda */ q_strideH, + /* ldb */ k_strideN, + /* ldc */ BLOCK_N, + /* mtt */ max_total_num_tokens); + + if (has_logit_cap) { + at::vec::map( + [logit_cap, rlogit_cap](Vec x) { return Vec(logit_cap) * (x * Vec(rlogit_cap)).tanh(); }, + s_i, + s_i, + BLOCK_H * BLOCK_N); + } + + // update the scaling coefficients + for (int64_t h = 0; h < h_size; ++h) { + // m_i: max value per row + float m_i = at::vec::reduce_all( + [](Vec& x, Vec& y) { return at::vec::maximum(x, y); }, s_i + h * BLOCK_N, n_size); + m_i = std::max(m_i, m_prime[h]); + + // m_delta <- exp(m' - m_i) + m_delta[h] = std::exp(m_prime[h] - m_i); + + // s_delta <- exp(s_i - m_i) + at::vec::map( + [m_i](Vec x) { return (x - Vec(m_i)).exp_u20(); }, s_delta + h * BLOCK_N, s_i + h * BLOCK_N, n_size); + + // s' <- s' * m_delta + sum(s_delta) + s_prime[h] *= m_delta[h]; + s_prime[h] += at::vec::reduce_all([](Vec& x, Vec& y) { return x + y; }, s_delta + h * BLOCK_N, n_size); + + m_prime[h] = m_i; + } + + // calculate V' <- s_delta @ V + V' * m_delta + index_gemm_kernel_nn( + /* A */ s_delta, + /* B */ v_buffer + head_kv_id * v_strideH, + /* C */ v_prime, + /* ind */ req_to_token + req_pool_id * max_context_len + n, + /* scl */ m_delta, + /* M */ h_size, + /* N */ head_size_v, + /* K */ n_size, + /* lda */ BLOCK_N, + /* ldb */ v_strideN, + /* ldc */ l_stride1, + /* mtt */ max_total_num_tokens); + } // loop with KV blocks + + // only update v' when kv_split_size > 0 + if (kv_end > kv_start) { + for (int64_t h = 0; h < h_size; ++h) { + float s = 1 / s_prime[h]; + at::vec::map( + [s](Vec out) { return out * Vec(s); }, v_prime + h * l_stride1, v_prime + h * l_stride1, head_size_v); + (v_prime + h * l_stride1)[head_size_v] = m_prime[h] + std::log(s_prime[h]); + } + } + + // move to the next index + data_index_step(bs, batches, head_kv_id, num_heads_kv, block_id, num_blocks, kv_id, num_kv_splits); + } + }); + + decode_accumulate_kv_splits( + output, attn_logits, batches, num_heads, head_size_v, num_kv_splits, l_stride1, l_stride2); +} // GQA/MQA + +} // anonymous namespace + +// query: [num_tokens, num_heads, head_size] +// output: [num_tokens, num_heads, head_size] +// k_buffer: [max_total_num_tokens, num_heads, head_size] +// v_buffer: [max_total_num_tokens, num_heads, head_size_v] +// attn_logits: [num_seqs, num_heads, num_kv_splits, head_size_v + 1] +// req_to_token: [max_num_reqs, max_context_len] int32 or int64 +// req_pool_indices: [num_seqs] int64 +// seq_lens: [num_seqs] int64 +// +void decode_attention_cpu( + at::Tensor& query, + at::Tensor& k_buffer, + at::Tensor& v_buffer, + at::Tensor& output, + at::Tensor& key, + at::Tensor& value, + at::Tensor& loc, + at::Tensor& attn_logits, + at::Tensor& req_to_token, + at::Tensor& req_pool_indices, + at::Tensor& seq_lens, + double sm_scale, + double logit_cap) { + RECORD_FUNCTION( + "sgl-kernel::decode_attention_cpu", + std::vector( + {query, output, k_buffer, v_buffer, attn_logits, req_to_token, req_pool_indices, seq_lens})); + + CHECK_LAST_DIM_CONTIGUOUS_INPUT(query); + CHECK_LAST_DIM_CONTIGUOUS_INPUT(k_buffer); + CHECK_LAST_DIM_CONTIGUOUS_INPUT(v_buffer); + // for MLA, key and value shares the same storage and value could be non-contiguous + CHECK_LAST_DIM_CONTIGUOUS_INPUT(key); + CHECK_LAST_DIM_CONTIGUOUS_INPUT(value); + CHECK_DIM(3, query); + CHECK_DIM(3, k_buffer); + CHECK_DIM(3, v_buffer); + CHECK_DIM(3, key); + CHECK_DIM(3, value); + CHECK_DIM(1, loc); + + int64_t num_seqs = seq_lens.size(0); + int64_t max_num_reqs = req_to_token.size(0); + int64_t max_context_len = req_to_token.size(1); + int64_t max_total_num_tokens = k_buffer.size(0); + + int64_t num_heads = query.size(1); + int64_t num_heads_kv = k_buffer.size(1); + int64_t head_size = query.size(2); + int64_t head_size_v = v_buffer.size(2); + + int64_t num_kv_splits = attn_logits.size(2); + + CHECK_EQ(loc.numel(), num_seqs); + CHECK_EQ(attn_logits.size(0), num_seqs); + CHECK_EQ(attn_logits.size(1), num_heads); + CHECK_EQ(attn_logits.size(3), head_size_v + 1); + CHECK_EQ(attn_logits.scalar_type(), at::kFloat); + + // strides for query + int64_t q_strideM = query.stride(0); + int64_t q_strideH = query.stride(1); + + // strides for k_buffer and v_buffer + int64_t k_strideN = k_buffer.stride(0); + int64_t k_strideH = k_buffer.stride(1); + int64_t v_strideN = v_buffer.stride(0); + int64_t v_strideH = v_buffer.stride(1); + // strides for new key and value + int64_t nk_strideN = key.stride(0); + int64_t nk_strideH = key.stride(1); + int64_t nv_strideN = value.stride(0); + int64_t nv_strideH = value.stride(1); + + // check index data types + const auto index_dtype = req_to_token.scalar_type(); + TORCH_CHECK( + index_dtype == at::kInt || index_dtype == at::kLong, + "decode: expect req_to_token to be int32 or int64, got ", + index_dtype); + TORCH_CHECK(seq_lens.scalar_type() == at::kLong, "decode: expect req_lens to be int64, got ", seq_lens.scalar_type()); + TORCH_CHECK( + req_pool_indices.scalar_type() == at::kLong, + "decode: expect req_pool_indices to be int64, got ", + req_pool_indices.scalar_type()); + + // check if we have MLA here + void* k_buffer_data = k_buffer.data_ptr(); + void* v_buffer_data = v_buffer.data_ptr(); + const bool is_mla = (k_buffer_data == v_buffer_data) && (num_heads_kv == 1) && (head_size == head_size_v + 64); + + // block length for k_buffer and v_buffer + constexpr int BLOCK_N = 256; + + // buffer for packing k_cache and v_cache + int num_threads = at::get_num_threads(); + int64_t size_per_thread = is_mla ? BLOCK_N * head_size + BLOCK_N * head_size_v : 0; + auto buffer = at::empty({num_threads, size_per_thread}, k_buffer.options()); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(query.scalar_type(), "decode_attention_kernel", [&] { + AT_DISPATCH_INDEX_TYPES(index_dtype, "decode_attention_indices", [&] { + // update the kv buffer + decode_set_kv_buffer( + (scalar_t*)k_buffer_data, + (scalar_t*)v_buffer_data, + key.data_ptr(), + value.data_ptr(), + loc.data_ptr(), + num_seqs, + num_heads_kv, + head_size, + head_size_v, + k_strideN, + k_strideH, + v_strideN, + v_strideH, + nk_strideN, + nk_strideH, + nv_strideN, + nv_strideH, + is_mla); + + if (num_heads == num_heads_kv) { + // MHA + decode_attention_kernel_impl( + output.data_ptr(), + attn_logits.data_ptr(), + query.data_ptr(), + (const scalar_t*)k_buffer_data, + (const scalar_t*)v_buffer_data, + req_to_token.data_ptr(), + req_pool_indices.data_ptr(), + seq_lens.data_ptr(), + num_seqs, + num_heads, + head_size, + head_size_v, + num_kv_splits, + q_strideM, + q_strideH, + k_strideN, + k_strideH, + v_strideN, + v_strideH, + sm_scale, + logit_cap, + max_num_reqs, + max_context_len, + max_total_num_tokens); + } else if (is_mla) { + // MLA + decode_attention_mla_kernel_impl( + output.data_ptr(), + attn_logits.data_ptr(), + query.data_ptr(), + (const scalar_t*)k_buffer_data, + (const scalar_t*)v_buffer_data, + req_to_token.data_ptr(), + req_pool_indices.data_ptr(), + seq_lens.data_ptr(), + buffer.data_ptr(), + num_seqs, + num_heads, + head_size, + head_size_v, + num_kv_splits, + q_strideM, + q_strideH, + k_strideN, + k_strideH, + v_strideN, + v_strideH, + sm_scale, + logit_cap, + max_num_reqs, + max_context_len, + max_total_num_tokens, + size_per_thread); + } else { + // GQA/MQA + decode_attention_grouped_kernel_impl( + output.data_ptr(), + attn_logits.data_ptr(), + query.data_ptr(), + (const scalar_t*)k_buffer_data, + (const scalar_t*)v_buffer_data, + req_to_token.data_ptr(), + req_pool_indices.data_ptr(), + seq_lens.data_ptr(), + num_seqs, + num_heads, + num_heads_kv, + head_size, + head_size_v, + num_kv_splits, + q_strideM, + q_strideH, + k_strideN, + k_strideH, + v_strideN, + v_strideH, + sm_scale, + logit_cap, + max_num_reqs, + max_context_len, + max_total_num_tokens); + } + }); + }); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/extend.cpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/extend.cpp new file mode 100644 index 00000000..3162ccea --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/extend.cpp @@ -0,0 +1,723 @@ +#include "common.h" +#include "gemm.h" +#include "vec.h" + +namespace { + +// [NOTE]: extend attention for CPU +// 1. tune BLOCK_M and BLOCK_N +// 2. can handle non-contiguous k_exttend and v_extend +// 3. computes attention for prefix and extend separately +// 4. TODO: vectorize `pack_vnni` and `pack_vnni2` +// + +template +inline index_t get_index(index_t* ind, int i) { + return (ind == nullptr) ? (index_t)i : ind[i]; +} + +#if defined(CPU_CAPABILITY_AVX512) +// key: from [N, 32] to [32/2, N, 2] +template +inline void pack_vnni_Nx32( + scalar_t* __restrict__ dst, + const scalar_t* __restrict__ src, + const index_t* __restrict__ ind, + int N, + int ld_src, + int ld_dst) { + __m512i vinputs[16]; + + int n = 0; + for (; n < N; ++n) { + index_t index = get_index(ind, n); + vinputs[n] = _mm512_loadu_si512(src + index * ld_src); + } + // padding with zero to avoid uninitialized vectors + for (; n < 16; ++n) { + vinputs[n] = _mm512_set1_epi32(0); + } + + // pack key + transpose_16x16_32bit(vinputs); + + const __mmask16 vmask = (1 << N) - 1; + for (int k = 0; k < 16; ++k) { + _mm512_mask_storeu_epi32(dst + k * ld_dst * 2, vmask, vinputs[k]); + } +} + +// value: from [K, 32] to [K/2, 32, 2] +template +inline void pack_vnni_Kx32( + scalar_t* __restrict__ dst, + const scalar_t* __restrict__ src, + const index_t* __restrict__ ind, + int K, + int ld_src, + int ld_dst) { + __m512i vinputs[2]; + + int k = 0; + for (; k < K; ++k) { + index_t index = get_index(ind, k); + vinputs[k] = _mm512_loadu_si512(src + index * ld_src); + } + // padding with zero to avoid uninitialized vectors + for (; k < 2; ++k) { + vinputs[k] = _mm512_set1_epi32(0); + } + + // pack value + __m512i d0, d1; + std::tie(d0, d1) = transpose_2x32_16bit(vinputs[0], vinputs[1]); + _mm512_storeu_si512(dst + 0 * ld_dst * 2, d0); + _mm512_storeu_si512(dst + 0 * ld_dst * 2 + 32, d1); +} +#endif + +// convert to vnni format +// from [N, K/2, 2] to [K/2, N, 2] for bfloat16 and float16 +template +void pack_vnni( + scalar_t* __restrict__ dst, + const scalar_t* __restrict__ src, + const index_t* __restrict__ ind, + int N, + int K, + int ld_src, + int ld_dst) { +#if defined(CPU_CAPABILITY_AVX512) + const int NB = div_up(N, 16); + const int KB = K / 32; // no remainder + const bool is_indexed = ind != nullptr; + + for (int nb = 0; nb < NB; ++nb) { + for (int kb = 0; kb < KB; ++kb) { + // handle 16x512bits each block + int nb_size = std::min(N - nb * 16, 16); + pack_vnni_Nx32( + /* dst */ dst + ((kb * 32) >> 1) * ld_dst * 2 + nb * 16 * 2, + /* src */ src + kb * 32 + (is_indexed ? 0 : nb * 16 * ld_src), + /* ind */ is_indexed ? ind + nb * 16 : nullptr, + /* N */ nb_size, + /* ld_src */ ld_src, + /* ld_dst */ ld_dst); + } + } +#else + for (int n = 0; n < N; ++n) { + index_t index = get_index(ind, n); + for (int k = 0; k < K / 2; ++k) { + for (int d = 0; d < 2; ++d) { + dst[k * ld_dst * 2 + n * 2 + d] = src[index * ld_src + k * 2 + d]; + } + } + } +#endif +} + +// convert to vnni format +// from [K/2, 2, N] to [K/2, N, 2] for bfloat16 and float16 +template +void pack_vnni2( + scalar_t* __restrict__ dst, + const scalar_t* __restrict__ src, + const index_t* __restrict__ ind, + int K, + int N, + int ld_src, + int ld_dst) { +#if defined(CPU_CAPABILITY_AVX512) + const int KB = div_up(K, 2); + const int NB = N / 32; // no remainder + const bool is_indexed = ind != nullptr; + + for (int kb = 0; kb < KB; ++kb) { + for (int nb = 0; nb < NB; ++nb) { + // handle 2x512bits each block + int kb_size = std::min(K - kb * 2, 2); + pack_vnni_Kx32( + /* dst */ dst + ((kb * 2) >> 1) * ld_dst * 2 + nb * 32 * 2, + /* src */ src + (is_indexed ? 0 : kb * 2 * ld_src) + nb * 32, + /* ind */ is_indexed ? ind + kb * 2 : nullptr, + /* K */ kb_size, + /* ld_src */ ld_src, + /* ld_dst */ ld_dst); + } + } +#else + int k = 0; + for (; k < (K >> 1) * 2; k += 2) { + index_t index0 = get_index(ind, k + 0); + index_t index1 = get_index(ind, k + 1); + for (int n = 0; n < N; ++n) { + dst[(k >> 1) * ld_dst * 2 + n * 2 + 0] = src[index0 * ld_src + n]; + dst[(k >> 1) * ld_dst * 2 + n * 2 + 1] = src[index1 * ld_src + n]; + } + } + if (K % 2 != 0) { + index_t index = get_index(ind, K - 1); + for (int n = 0; n < N; ++n) { + dst[(K >> 1) * ld_dst * 2 + n * 2 + 0] = src[index * ld_src + n]; + dst[(K >> 1) * ld_dst * 2 + n * 2 + 1] = 0; + } + k += 2; + } +#endif +} + +template +inline void fill_stub(scalar_t* __restrict__ out, float val, int size) { + using Vec = at::vec::Vectorized; + constexpr int kVecSize = Vec::size(); + const Vec data_vec = Vec(static_cast(val)); + int d = 0; +#pragma GCC unroll 4 + for (; d <= size - kVecSize; d += kVecSize) { + data_vec.store(out + d); + } + if (size - d > 0) { + data_vec.store(out + d, size - d); + } +} + +template +inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ input) { + static_assert(BLOCK_N % 32 == 0); + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + + constexpr int COLS = BLOCK_N / 16; + auto store = [&](auto i) { + constexpr int col = i % COLS; + // for COLS = 2, 4 use 512bit store + if constexpr (col % 2 == 0) { + fVec a_fvec0 = fVec::loadu(input + col * 16); + fVec a_fvec1 = fVec::loadu(input + col * 16 + 16); + bVec out_bvec = convert_from_float_ext(a_fvec0, a_fvec1); + out_bvec.store(out + col * 16); + } + }; + Unroll{}(store); +} + +template +inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ acc, float s, int size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + const fVec s_fvec = fVec(s); + int d = 0; +#pragma GCC unroll 4 + for (; d <= size - kVecSize; d += kVecSize) { + fVec a_fvec0 = fVec::loadu(acc + d) * s_fvec; + fVec a_fvec1 = fVec::loadu(acc + d + fVec::size()) * s_fvec; + bVec out_bvec = convert_from_float_ext(a_fvec0, a_fvec1); + out_bvec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(acc[d] * s); + } +} + +template +void extend_attention_kernel_impl( + scalar_t* __restrict__ o_extend, + const scalar_t* __restrict__ q_extend, + const scalar_t* __restrict__ k_extend, + const scalar_t* __restrict__ v_extend, + const scalar_t* __restrict__ k_buffer, + const scalar_t* __restrict__ v_buffer, + const index_t* __restrict__ req_to_token, + const int64_t* __restrict__ req_pool_indices, + const int64_t* __restrict__ seq_lens, + const index_t* __restrict__ extend_seq_lens, + const index_t* __restrict__ extend_start_loc, + const void* __restrict__ buffer, + int batches, + int num_heads, + int num_heads_kv, + int head_size, + int head_size_v, + int q_strideM, + int q_strideH, + int ke_strideN, + int ke_strideH, + int ve_strideN, + int ve_strideH, + int k_strideN, + int k_strideH, + int v_strideN, + int v_strideH, + float scaling, + float logit_cap, + int max_num_reqs, + int max_context_len, + int max_total_num_tokens, + int max_len_extend, + int buffer_size_per_thread, + bool is_prefix_skipped) { + using Vec = at::vec::Vectorized; + + // strides + const int o_strideM = num_heads * head_size_v; + const int o_strideH = head_size_v; + + // we use same buffer for packed key and value + const int ldb_tmp = std::max(head_size, head_size_v); + + const bool has_logit_cap = logit_cap > 0; + float rlogit_cap = has_logit_cap ? 1 / logit_cap : 0.f; + + const int num_groups = num_heads / num_heads_kv; + TORCH_CHECK(num_groups * num_heads_kv == num_heads); + + // number of blocks along M + int MB = div_up(max_len_extend, BLOCK_M); + + // parallel on [batches, num_heads, BM] + at::parallel_for(0, batches * num_heads * MB, 0, [&](int begin, int end) { + int bs{0}, head_id{0}, mb{0}; + data_index_init(begin, bs, batches, head_id, num_heads, mb, MB); + + int tid = at::get_thread_num(); + // s_i and s_delta: [BLOCK_M, BLOCK_N] + float* __restrict__ s_i = reinterpret_cast((char*)(buffer) + tid * buffer_size_per_thread); + float* __restrict__ s_delta = s_i; + + // v_prime: [BLOCK_M, head_size_v] + float* __restrict__ v_prime = s_i + BLOCK_M * BLOCK_N; + + // s_delta2: [BLOCK_M, BLOCK_N]; copy of s_delta in scalar_t + scalar_t* __restrict__ s_delta2 = reinterpret_cast(v_prime + BLOCK_N * head_size_v); + + // Btmp: [BLOCK_N, max(head_size, head_size_v)] + scalar_t* __restrict__ Btmp = s_delta2 + BLOCK_M * BLOCK_N; + + // init Btmp just once for each thread to prevent NaN + fill_stub(Btmp, 0.f, BLOCK_N * ldb_tmp); + + alignas(64) float s_prime[BLOCK_M]; + alignas(64) float m_prime[BLOCK_M]; + + for (int i = begin; i < end; ++i) { + // seq_len = prefix + extend + int head_kv_id = head_id / num_groups; + int seq_len = seq_lens[bs]; + int seq_len_extend = extend_seq_lens[bs]; + int seq_len_prefix = seq_len - seq_len_extend; + int seq_extend_start_loc = extend_start_loc[bs]; + + int req_pool_id = req_pool_indices[bs]; + TORCH_CHECK(seq_len_prefix >= 0, "prefix len < 0!"); + TORCH_CHECK(seq_len <= max_context_len, "seq_len out of scope!"); + TORCH_CHECK(req_pool_id < max_num_reqs, "req_pool_id out of scope!"); + + if (is_prefix_skipped) { + TORCH_CHECK(seq_len_prefix == 0, "extend attention: expect seq_len_prefix to be 0, got ", seq_len_prefix); + } + + // offset and size in MB + int m = mb * BLOCK_N; + int m_size = std::min(BLOCK_M, seq_len_extend - m); + + if (m_size <= 0) { + data_index_step(bs, batches, head_id, num_heads, mb, MB); + continue; + } + + // get query + const scalar_t* __restrict__ q_ptr = q_extend + (seq_extend_start_loc + m) * q_strideM + head_id * q_strideH; + + // init v', s' and m' + fill_stub(v_prime, 0.f, m_size * head_size_v); + fill_stub(s_prime, 0.f, m_size); + fill_stub(m_prime, -std::numeric_limits::infinity(), m_size); + + // stage 1: compute scores with prefix + for (int n = 0; n < seq_len_prefix; n += BLOCK_N) { + int n_size = std::min(BLOCK_N, seq_len_prefix - n); + + // `n_size` is K in 2nd gemm, pad to TILE_K; + const int padded_n_size = div_up(n_size, TILE_K) * TILE_K; + + // get key and pack + pack_vnni( + /* dst */ Btmp, + /* src */ k_buffer + head_kv_id * k_strideH, + /* ind */ req_to_token + req_pool_id * max_context_len + n, + /* N */ n_size, + /* K */ head_size, + /* ld_src */ k_strideN, + /* ld_dst */ BLOCK_N); + + // calculate s_i <- Q @ K + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ head_size, + /* lda */ q_strideM, + /* ldb */ BLOCK_N, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ q_ptr, + /* B */ Btmp, + /* C */ s_i); + + const Vec scale_vec = Vec(scaling); + for (int row = 0; row < m_size; ++row) { + // s_i <- s_i * scale + at::vec::map( + [scale_vec](Vec x) { return x * scale_vec; }, s_i + row * BLOCK_N, s_i + row * BLOCK_N, n_size); + + // TODO: `tanh` from torch uses sleef u10, going to be slow + if (has_logit_cap) { + at::vec::map( + [logit_cap, rlogit_cap](Vec x) { return Vec(logit_cap) * (x * Vec(rlogit_cap)).tanh(); }, + s_i + row * BLOCK_N, + s_i + row * BLOCK_N, + n_size); + } + + // m_i: max value per row + float m_i = at::vec::reduce_all( + [](Vec& x, Vec& y) { return at::vec::maximum(x, y); }, s_i + row * BLOCK_N, n_size); + m_i = std::max(m_i, m_prime[row]); + + // m_delta <- exp(m' - m_i) + float m_delta = std::exp(m_prime[row] - m_i); + + // s_delta <- exp(s_i - m_i) + at::vec::map( + [m_i](Vec x) { return (x - Vec(m_i)).exp_u20(); }, s_delta + row * BLOCK_N, s_i + row * BLOCK_N, n_size); + + // s' <- s' * m_delta + sum(s_delta) + s_prime[row] *= m_delta; + s_prime[row] += + at::vec::reduce_all([](Vec& x, Vec& y) { return x + y; }, s_delta + row * BLOCK_N, n_size); + + m_prime[row] = m_i; + + // v' <- v' * m_delta + at::vec::map( + [m_delta](Vec x) { return x * Vec(m_delta); }, + v_prime + row * head_size_v, + v_prime + row * head_size_v, + head_size_v); + + // pad s_delta with 0 first and then convert to scalar_t + fill_stub(s_delta + row * BLOCK_N + n_size, 0.f, padded_n_size - n_size); + copy_stub(s_delta2 + row * BLOCK_N, s_delta + row * BLOCK_N); + } + + // get value and pack + pack_vnni2( + /* dst */ Btmp, + /* src */ v_buffer + head_kv_id * v_strideH, + /* ind */ req_to_token + req_pool_id * max_context_len + n, + /* K */ n_size, + /* N */ head_size_v, + /* ld_src */ v_strideN, + /* ld_dst */ head_size_v); + + // calculate V' <- s_delta @ V + V' + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ head_size_v, + /* K */ padded_n_size, // n_size + /* lda */ BLOCK_N, + /* ldb */ head_size_v, + /* ldc */ head_size_v, + /* add_C */ true, + /* A */ s_delta2, + /* B */ Btmp, + /* C */ v_prime); + } // loop with seq_len_prefix + + // stage 2: compute the triangle part + int num_keys = std::min(seq_len_extend, m + BLOCK_M); + for (int n = 0; n < num_keys; n += BLOCK_N) { + int n_size = std::min(BLOCK_N, num_keys - n); + + // `n_size` is K in 2nd gemm, pad to TILE_K; + const int padded_n_size = div_up(n_size, TILE_K) * TILE_K; + + // get key and pack + pack_vnni( + /* dst */ Btmp, + /* src */ k_extend + (seq_extend_start_loc + n) * ke_strideN + head_kv_id * ke_strideH, + /* ind */ nullptr, + /* N */ n_size, + /* K */ head_size, + /* ld_src */ ke_strideN, + /* ld_dst */ BLOCK_N); + + // calculate s_i <- Q @ K + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ head_size, + /* lda */ q_strideM, + /* ldb */ BLOCK_N, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ q_ptr, + /* B */ Btmp, + /* C */ s_i); + + // apply causal mask + if (num_keys - n <= BLOCK_N) { + for (int row = 0; row < m_size; ++row) { + int last_col = m + row - n; + // fill [last_col + 1, n_size) to -inf + float* row_ptr = s_i + row * BLOCK_N; + fill_stub(row_ptr + last_col + 1, -std::numeric_limits::infinity(), n_size - last_col - 1); + } + } + + const Vec scale_vec = Vec(scaling); + for (int row = 0; row < m_size; ++row) { + // s_i <- s_i * scale + at::vec::map( + [scale_vec](Vec x) { return x * scale_vec; }, s_i + row * BLOCK_N, s_i + row * BLOCK_N, n_size); + + // TODO: `tanh` from torch uses sleef u10, going to be slow + if (has_logit_cap) { + at::vec::map( + [logit_cap, rlogit_cap](Vec x) { return Vec(logit_cap) * (x * Vec(rlogit_cap)).tanh(); }, + s_i + row * BLOCK_N, + s_i + row * BLOCK_N, + n_size); + } + + // m_i: max value per row + float m_i = at::vec::reduce_all( + [](Vec& x, Vec& y) { return at::vec::maximum(x, y); }, s_i + row * BLOCK_N, n_size); + m_i = std::max(m_i, m_prime[row]); + + // m_delta <- exp(m' - m_i) + float m_delta = std::exp(m_prime[row] - m_i); + + // s_delta <- exp(s_i - m_i) + at::vec::map( + [m_i](Vec x) { return (x - Vec(m_i)).exp_u20(); }, s_delta + row * BLOCK_N, s_i + row * BLOCK_N, n_size); + + // s' <- s' * m_delta + sum(s_delta) + s_prime[row] *= m_delta; + s_prime[row] += + at::vec::reduce_all([](Vec& x, Vec& y) { return x + y; }, s_delta + row * BLOCK_N, n_size); + + m_prime[row] = m_i; + + // v' <- v' * m_delta + at::vec::map( + [m_delta](Vec x) { return x * Vec(m_delta); }, + v_prime + row * head_size_v, + v_prime + row * head_size_v, + head_size_v); + + // pad s_delta with 0 first and then convert to scalar_t + fill_stub(s_delta + row * BLOCK_N + n_size, 0.f, padded_n_size - n_size); + copy_stub(s_delta2 + row * BLOCK_N, s_delta + row * BLOCK_N); + } + + // get value and pack + pack_vnni2( + /* dst */ Btmp, + /* src */ v_extend + (seq_extend_start_loc + n) * ve_strideN + head_kv_id * ve_strideH, + /* ind */ nullptr, + /* K */ n_size, + /* N */ head_size_v, + /* ld_src */ ve_strideN, + /* ld_dst */ head_size_v); + + // calculate V' <- s_delta @ V + V' + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ head_size_v, + /* K */ padded_n_size, // n_size + /* lda */ BLOCK_N, + /* ldb */ head_size_v, + /* ldc */ head_size_v, + /* add_C */ true, + /* A */ s_delta2, + /* B */ Btmp, + /* C */ v_prime); + } // loop with seq_len_extend + + scalar_t* __restrict__ out_ptr = o_extend + (seq_extend_start_loc + m) * o_strideM + head_id * o_strideH; + for (int row = 0; row < m_size; ++row) { + float s = 1 / s_prime[row]; + copy_stub(out_ptr + row * o_strideM, v_prime + row * head_size_v, s, head_size_v); + } + + // move to the next index + data_index_step(bs, batches, head_id, num_heads, mb, MB); + } + at::native::cpublas::brgemm_release(); + }); +} + +} // anonymous namespace + +// q_extend, k_extend, v_extend, o_extend: contiguous tensors +// k_buffer, v_buffer: (prefix + extend) tensors in mem_manager +// +// q_extend: [num_tokens, num_heads, head_size] +// k_extend: [num_extend_tokens, num_heads, head_size] +// v_extend: [num_extend_tokens, num_heads, head_size] +// o_extend: [num_tokens, num_heads, head_size] +// k_buffer: [max_total_num_tokens, num_heads, head_size] +// v_buffer: [max_total_num_tokens, num_heads, head_size] +// req_to_token: [max_num_reqs, max_context_len] int32 or int64 +// req_pool_indices: [num_seqs] int64 +// seq_lens: [num_seqs] int64 +// extend_seq_lens: [num_seqs] +// extend_start_loc: [num_seqs] +// +void extend_attention_cpu( + at::Tensor& q_extend, + at::Tensor& k_extend, + at::Tensor& v_extend, + at::Tensor& o_extend, + at::Tensor& k_buffer, + at::Tensor& v_buffer, + at::Tensor& req_to_token, + at::Tensor& req_pool_indices, + at::Tensor& seq_lens, + at::Tensor& extend_seq_lens, + at::Tensor& extend_start_loc, + int64_t max_len_extend, + double sm_scale, + double logit_cap) { + RECORD_FUNCTION( + "sgl-kernel::extend_attention_cpu", + std::vector( + {q_extend, + k_extend, + v_extend, + o_extend, + k_buffer, + v_buffer, + req_to_token, + req_pool_indices, + seq_lens, + extend_seq_lens, + extend_start_loc})); + + CHECK_LAST_DIM_CONTIGUOUS_INPUT(q_extend); + CHECK_INPUT(o_extend); + CHECK_LAST_DIM_CONTIGUOUS_INPUT(k_extend); + CHECK_LAST_DIM_CONTIGUOUS_INPUT(v_extend); + CHECK_LAST_DIM_CONTIGUOUS_INPUT(k_buffer); + CHECK_LAST_DIM_CONTIGUOUS_INPUT(v_buffer); + + int num_seqs = seq_lens.size(0); + int max_num_reqs = req_to_token.size(0); + int max_context_len = req_to_token.size(1); + int max_total_num_tokens = k_buffer.size(0); + + int num_heads = q_extend.size(1); + int num_heads_kv = k_extend.size(1); + int head_size = q_extend.size(2); + int head_size_v = v_extend.size(2); + + // strides for q_extend, k_extend and v_extend + int q_strideM = q_extend.stride(0); + int q_strideH = q_extend.stride(1); + int ke_strideN = k_extend.stride(0); + int ke_strideH = k_extend.stride(1); + int ve_strideN = v_extend.stride(0); + int ve_strideH = v_extend.stride(1); + + // strides for k_buffer and v_buffer + int k_strideN = k_buffer.stride(0); + int k_strideH = k_buffer.stride(1); + int v_strideN = v_buffer.stride(0); + int v_strideH = v_buffer.stride(1); + + // check sizes + CHECK_EQ(req_pool_indices.size(0), num_seqs); + CHECK_EQ(extend_seq_lens.size(0), num_seqs); + CHECK_EQ(extend_start_loc.size(0), num_seqs); + CHECK_EQ(v_extend.size(1), num_heads_kv); + CHECK_EQ(k_buffer.size(1), v_buffer.size(1)); + + // MLA will skip prefix part + const bool is_prefix_skipped = k_buffer.size(1) != num_heads_kv; + + // check index data types + const auto index_dtype = req_to_token.scalar_type(); + TORCH_CHECK( + index_dtype == at::kInt || index_dtype == at::kLong, + "extend: expect req_to_token to be int32 or int64, got ", + index_dtype); + TORCH_CHECK(seq_lens.scalar_type() == at::kLong, "extend: expect req_lens to be int64, got ", seq_lens.scalar_type()); + TORCH_CHECK( + req_pool_indices.scalar_type() == at::kLong, + "extend: expect req_pool_indices to be int64, got ", + req_pool_indices.scalar_type()); + TORCH_CHECK( + extend_seq_lens.scalar_type() == index_dtype && extend_start_loc.scalar_type() == index_dtype, + "extend: expect extend_seq_lens and extend_start_loc to have same dtype as req_to_token."); + + // D and DV need to be 32x as we transpose by 512-bit + TORCH_CHECK(head_size % 32 == 0, "invalid head_size ", head_size); + TORCH_CHECK(head_size_v % 32 == 0, "invalid head_size_v ", head_size_v); + + // block size for query seq length + constexpr int BLOCK_M = 32; + // block size for key/value seq length + constexpr int BLOCK_N = 32; + + const int size_per_thread = + /* s_i */ BLOCK_M * BLOCK_N * sizeof(float) + + /* v_prime */ BLOCK_M * head_size_v * sizeof(float) + + /* s_delta */ BLOCK_M * BLOCK_N * sizeof(uint16_t) + + /* Btmp */ BLOCK_N * std::max(head_size, head_size_v) * sizeof(uint16_t); + + int num_threads = at::get_num_threads(); + auto buffer = at::empty({num_threads, size_per_thread}, q_extend.options().dtype(at::kChar)); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(q_extend.scalar_type(), "extend_attention_kernel", [&] { + AT_DISPATCH_INDEX_TYPES(index_dtype, "extend_attention_indices", [&] { + extend_attention_kernel_impl( + o_extend.data_ptr(), + q_extend.data_ptr(), + k_extend.data_ptr(), + v_extend.data_ptr(), + k_buffer.data_ptr(), + v_buffer.data_ptr(), + req_to_token.data_ptr(), + req_pool_indices.data_ptr(), + seq_lens.data_ptr(), + extend_seq_lens.data_ptr(), + extend_start_loc.data_ptr(), + buffer.data_ptr(), + num_seqs, + num_heads, + num_heads_kv, + head_size, + head_size_v, + q_strideM, + q_strideH, + ke_strideN, + ke_strideH, + ve_strideN, + ve_strideH, + k_strideN, + k_strideH, + v_strideN, + v_strideH, + sm_scale, + logit_cap, + max_num_reqs, + max_context_len, + max_total_num_tokens, + max_len_extend, + size_per_thread, + is_prefix_skipped); + }); + }); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/gemm.cpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/gemm.cpp new file mode 100644 index 00000000..48655b9f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/gemm.cpp @@ -0,0 +1,525 @@ +#include "gemm.h" + +#include "common.h" +#include "vec.h" + +namespace { + +// packed layout: +// quants {N, K} int8_t +// comp {N} int32_t +template +inline void s8s8_compensation(int8_t* __restrict__ packed, int K) { +#if defined(CPU_CAPABILITY_AVX512) + constexpr int COLS = BLOCK_N / 16; + __m512i vcomp[COLS]; + + for (int col = 0; col < COLS; ++col) { + vcomp[col] = _mm512_setzero_si512(); + } + + const int64_t offset = BLOCK_N * K; + const __m512i off = _mm512_set1_epi8(static_cast(0x80)); + for (int k = 0; k < K / 4; ++k) { + for (int col = 0; col < COLS; ++col) { + __m512i vb = _mm512_loadu_si512((const __m512i*)(packed + k * BLOCK_N * 4 + col * 64)); + vcomp[col] = _mm512_dpbusd_epi32(vcomp[col], off, vb); + } + } + + for (int col = 0; col < COLS; ++col) { + _mm512_storeu_si512((__m512i*)(packed + offset + col * 64), vcomp[col]); + } +#else + TORCH_CHECK(false, "s8s8_compensation not implemented!"); +#endif +} + +// convert to vnni format +// from [N, K] to [K/2, N, 2] for bfloat16 and float16 +template +inline void pack_vnni(packed_t* __restrict__ packed, const packed_t* __restrict__ weight, int N, int K) { + const int VNNI_BLK = 2; + for (int n = 0; n < N; ++n) { + for (int k = 0; k < K / VNNI_BLK; ++k) { + for (int d = 0; d < VNNI_BLK; ++d) { + packed[k * N * VNNI_BLK + n * VNNI_BLK + d] = weight[n * K + k * VNNI_BLK + d]; + } + } + } +} + +template <> +inline void pack_vnni(int8_t* __restrict__ packed, const int8_t* __restrict__ weight, int N, int K) { + constexpr int BLOCK_N = block_size_n(); + TORCH_CHECK(N == BLOCK_N); + + const int VNNI_BLK = 4; + for (int n = 0; n < N; ++n) { + for (int k = 0; k < K / VNNI_BLK; ++k) { + for (int d = 0; d < VNNI_BLK; ++d) { + packed[k * N * VNNI_BLK + n * VNNI_BLK + d] = weight[n * K + k * VNNI_BLK + d]; + } + } + } + s8s8_compensation(packed, K); +} + +template +inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ input, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + fVec data0 = fVec::loadu(input + d); + fVec data1 = fVec::loadu(input + d + fVec::size()); + bVec out_vec = convert_from_float_ext(data0, data1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d]); + } +} + +template +inline void copy_add_stub( + scalar_t* __restrict__ out, const float* __restrict__ input, const float* __restrict__ bias, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + fVec data0 = fVec::loadu(input + d) + fVec::loadu(bias + d); + fVec data1 = fVec::loadu(input + d + fVec::size()) + fVec::loadu(bias + d + fVec::size()); + bVec out_vec = convert_from_float_ext(data0, data1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d] + bias[d]); + } +} + +template +struct tinygemm_kernel_nn { + static inline void apply( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B, + scalar_t* __restrict__ C, + const float* __restrict__ bias, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!"); + } +}; + +#if defined(CPU_CAPABILITY_AVX512) +template +struct tinygemm_kernel_nn { + static inline void apply( + const at::BFloat16* __restrict__ A, + const at::BFloat16* __restrict__ B, + at::BFloat16* __restrict__ C, + const float* __restrict__ bias, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + constexpr int ROWS = BLOCK_M; + constexpr int COLS = BLOCK_N / 16; + + // prefetch distance + constexpr int PREFETCH_SIZE_K = 0; + + __m512bh va; + __m512bh vb[COLS]; + __m512 vc[ROWS * COLS]; + + auto loadc = [&](auto i) { + constexpr int col = i % COLS; + if constexpr (has_bias) { + vc[i] = _mm512_loadu_ps(bias + col * 16); + } else { + vc[i] = _mm512_set1_ps(0.f); + } + }; + Unroll{}(loadc); + + const int64_t K2 = K >> 1; + const int64_t lda2 = lda >> 1; + const int64_t ldb2 = ldb; // ldb * 2 >> 1; + const float* a_ptr = reinterpret_cast(A); + const float* b_ptr = reinterpret_cast(B); + + auto compute = [&](auto i, int64_t k) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + va = (__m512bh)(_mm512_set1_ps(a_ptr[row * lda2 + k])); + } + if constexpr (row == 0) { + vb[col] = (__m512bh)(_mm512_loadu_si512(b_ptr + k * ldb2 + col * 16)); + if constexpr (PREFETCH_SIZE_K > 0) { + _mm_prefetch(b_ptr + (k + PREFETCH_SIZE_K) * ldb2 + col * 16, _MM_HINT_T0); + } + } + vc[i] = _mm512_dpbf16_ps(vc[i], va, vb[col]); + }; + for (int64_t k = 0; k < K2; ++k) { + Unroll{}(compute, k); + } + + auto storec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + // for COLS = 2, 4 use 512bit store + // for COLS = 1, 3 use 256bit store + if constexpr (COLS % 2 == 0) { + if constexpr (col % 2 == 0) { + _mm512_storeu_si512( + reinterpret_cast<__m512i*>((C + row * ldc + col * 16)), + (__m512i)(_mm512_cvtne2ps_pbh(vc[row * COLS + col + 1], vc[row * COLS + col]))); + } + } else { + _mm256_storeu_si256(reinterpret_cast<__m256i*>(C + row * ldc + col * 16), (__m256i)(_mm512_cvtneps_pbh(vc[i]))); + } + }; + Unroll{}(storec); + } +}; +#endif + +#define LAUNCH_TINYGEMM_KERNEL_NN(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_nn::apply( \ + A + mb_start * lda, \ + B + nb_start * 2, \ + C + mb_start * ldc + nb_start, \ + has_bias ? bias + nb_start : nullptr, \ + K, \ + lda, \ + ldb, \ + ldc); + +template +struct brgemm { + static inline void apply( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B, + scalar_t* __restrict__ C, + float* __restrict__ Ctmp, + const float* __restrict__ bias, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + constexpr int BLOCK_N = block_size_n(); + at::native::cpublas::brgemm(M, N, K, lda, ldb, BLOCK_N, /* add_C */ false, A, B, Ctmp); + + // copy from Ctmp to C + for (int64_t m = 0; m < M; ++m) { + if constexpr (has_bias) { + copy_add_stub(C + m * ldc, Ctmp + m * BLOCK_N, bias, N); + } else { + copy_stub(C + m * ldc, Ctmp + m * BLOCK_N, N); + } + } + } +}; + +template +void tinygemm_kernel( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B, + scalar_t* __restrict__ C, + float* __restrict__ Ctmp, + const float* __restrict__ bias, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg) { + if (brg) { + brgemm::apply(A, B, C, Ctmp, bias, M, N, K, lda, ldb, ldc); + return; + } + + // pattern: 1-4-16, N = 16, 32, 48, 64 + constexpr int64_t BLOCK_M = 4; + constexpr int64_t BLOCK_N = 64; + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + for (int mb = 0; mb < MB; ++mb) { + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(BLOCK_M, M - mb_start); + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (mb_size << 4 | nb_size >> 4) { + // mb_size = 1 + case 0x11: + LAUNCH_TINYGEMM_KERNEL_NN(1, 16); + break; + case 0x12: + LAUNCH_TINYGEMM_KERNEL_NN(1, 32); + break; + case 0x13: + LAUNCH_TINYGEMM_KERNEL_NN(1, 48); + break; + case 0x14: + LAUNCH_TINYGEMM_KERNEL_NN(1, 64); + break; + // mb_size = 2 + case 0x21: + LAUNCH_TINYGEMM_KERNEL_NN(2, 16); + break; + case 0x22: + LAUNCH_TINYGEMM_KERNEL_NN(2, 32); + break; + case 0x23: + LAUNCH_TINYGEMM_KERNEL_NN(2, 48); + break; + case 0x24: + LAUNCH_TINYGEMM_KERNEL_NN(2, 64); + break; + // mb_size = 3 + case 0x31: + LAUNCH_TINYGEMM_KERNEL_NN(3, 16); + break; + case 0x32: + LAUNCH_TINYGEMM_KERNEL_NN(3, 32); + break; + case 0x33: + LAUNCH_TINYGEMM_KERNEL_NN(3, 48); + break; + case 0x34: + LAUNCH_TINYGEMM_KERNEL_NN(3, 64); + break; + // mb_size = 4 + case 0x41: + LAUNCH_TINYGEMM_KERNEL_NN(4, 16); + break; + case 0x42: + LAUNCH_TINYGEMM_KERNEL_NN(4, 32); + break; + case 0x43: + LAUNCH_TINYGEMM_KERNEL_NN(4, 48); + break; + case 0x44: + LAUNCH_TINYGEMM_KERNEL_NN(4, 64); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, " x ", nb_size); + } + } + } +} + +template +void weight_packed_linear_kernel_impl( + scalar_t* __restrict__ out, + const scalar_t* __restrict__ mat1, + const scalar_t* __restrict__ mat2, + const float* __restrict__ bias, + int64_t M, + int64_t N, + int64_t K, + int64_t mat1_strideM, + int64_t out_strideM) { + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + + const bool use_brgemm = can_use_brgemm(M); + + // parallel on [MB, NB] + AT_DISPATCH_BOOL(bias != nullptr, has_bias, [&] { + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // for brgemm, use float32 for accumulate + alignas(64) float Ctmp[BLOCK_M * BLOCK_N]; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(M - mb_start, BLOCK_M); + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(N - nb_start, BLOCK_N); + + tinygemm_kernel( + /* A */ mat1 + mb_start * mat1_strideM, + /* B */ mat2 + nb_start * K /* nb * BLOCK_N * K */, + /* C */ out + mb_start * out_strideM + nb_start, + /* Ctmp*/ Ctmp, + /* bias*/ bias + nb_start, + /* M */ mb_size, + /* N */ nb_size, + /* K */ K, + /* lda */ mat1_strideM, + /* ldb */ nb_size, + /* ldc */ out_strideM, + /* brg */ use_brgemm); + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + }); +} + +} // anonymous namespace + +// tinygemm interface +template +void tinygemm_kernel( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B, + scalar_t* __restrict__ C, + float* __restrict__ Ctmp, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg) { + tinygemm_kernel(A, B, C, Ctmp, nullptr, M, N, K, lda, ldb, ldc, brg); +} + +#define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE) \ + template void tinygemm_kernel( \ + const TYPE* __restrict__ A, \ + const TYPE* __restrict__ B, \ + TYPE* __restrict__ C, \ + float* __restrict__ Ctmp, \ + int64_t M, \ + int64_t N, \ + int64_t K, \ + int64_t lda, \ + int64_t ldb, \ + int64_t ldc, \ + bool brg) + +INSTANTIATE_TINYGEMM_TEMPLATE(at::BFloat16); +INSTANTIATE_TINYGEMM_TEMPLATE(at::Half); + +at::Tensor convert_weight_packed(at::Tensor& weight) { + // for 3d moe weights + // weight : [E, OC, IC] + // w1 : [E, 2N, K] + // w2 : [E, K, N] + CHECK_INPUT(weight); + + const int64_t ndim = weight.ndimension(); + TORCH_CHECK(ndim == 2 || ndim == 3, "expect weight to be 2d or 3d, got ", ndim, "d tensor."); + const auto st = weight.scalar_type(); + const int64_t E = ndim == 3 ? weight.size(0) : 1; + const int64_t OC = ndim == 3 ? weight.size(1) : weight.size(0); + const int64_t IC = ndim == 3 ? weight.size(2) : weight.size(1); + + // we handle 2 TILE_N at a time. + TORCH_CHECK(OC % TILE_N == 0, "invalid weight out features ", OC); + TORCH_CHECK(IC % TILE_K == 0, "invalid weight input features ", IC); + + constexpr int64_t BLOCK_N = block_size_n(); + const int64_t NB = div_up(OC, BLOCK_N); + + // use phony sizes here [E, OC, IC], for each [E], [OC, IC] -> [IC / 2, OC, 2] + auto packed_weight = at::empty({}, weight.options()); + const int64_t stride = OC * IC; + + TORCH_CHECK( + st == at::kBFloat16 || st == at::kHalf || st == at::kChar || st == at::kFloat8_e4m3fn, + "expect weight to be bfloat16, float16, int8 or fp8_e4m3."); + + CPU_DISPATCH_PACKED_TYPES(st, [&] { + // adjust most inner dimension size + const int packed_row_size = get_row_size(IC); + auto sizes = weight.sizes().vec(); + sizes[ndim - 1] = packed_row_size; + packed_weight.resize_(sizes); + + const packed_t* w_data = weight.data_ptr(); + packed_t* packed_data = packed_weight.data_ptr(); + + // parallel on {E, NB} + at::parallel_for(0, E * NB, 0, [&](int64_t begin, int64_t end) { + int64_t e{0}, nb{0}; + data_index_init(begin, e, E, nb, NB); + + for (int64_t i = begin; i < end; ++i) { + UNUSED(i); + + int64_t n = nb * BLOCK_N; + int64_t n_size = std::min(BLOCK_N, OC - n); + pack_vnni( + packed_data + e * OC * packed_row_size + n * packed_row_size, w_data + e * stride + n * IC, n_size, IC); + + // move to the next index + data_index_step(e, E, nb, NB); + } + }); + }); + return packed_weight; +} + +// mat1 : [M, K] +// mat2 : [N, K] +// bias : [N] +// out : [M, N] +// +at::Tensor +weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, const std::optional& bias, bool is_vnni) { + RECORD_FUNCTION("sgl-kernel::weight_packed_linear", std::vector({mat1, mat2, bias})); + + auto packed_w = is_vnni ? mat2 : convert_weight_packed(mat2); + + CHECK_LAST_DIM_CONTIGUOUS_INPUT(mat1); + CHECK_INPUT(mat2); + + int64_t M = mat1.size(0); + int64_t N = mat2.size(0); + int64_t K = mat2.size(1); + CHECK_EQ(mat1.size(1), K); + CHECK_DIM(2, mat1); + CHECK_DIM(2, mat2); + + auto out = at::empty({M, N}, mat1.options()); + + // strides + int64_t mat1_strideM = mat1.stride(0); + int64_t out_strideM = out.stride(0); + + const bool has_bias = bias.has_value(); + const float* bias_data = nullptr; + if (has_bias) { + CHECK_EQ(bias.value().size(0), N); + bias_data = bias.value().data_ptr(); + } + + AT_DISPATCH_REDUCED_FLOATING_TYPES(mat1.scalar_type(), "weight_packed_linear_kernel_impl", [&] { + weight_packed_linear_kernel_impl( + out.data_ptr(), + mat1.data_ptr(), + packed_w.data_ptr(), + bias_data, + M, + N, + K, + mat1_strideM, + out_strideM); + }); + + return out; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/gemm.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/gemm.h new file mode 100644 index 00000000..6a16a298 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/gemm.h @@ -0,0 +1,202 @@ +#pragma once +#include + +#include "common.h" + +// amx-bf16 +#define TILE_M 16 +#define TILE_N 16 +#define TILE_K 32 + +// block size for AMX gemm +constexpr int block_size_m() { + return 2 * TILE_M; +} +constexpr int block_size_n() { + return 2 * TILE_N; +} + +// define threshold using brgemm (intel AMX) +template +inline bool can_use_brgemm(int M); +template <> +inline bool can_use_brgemm(int M) { + return M > 4; +} +template <> +inline bool can_use_brgemm(int M) { + return true; +} +// this requires PyTorch 2.7 or above +template <> +inline bool can_use_brgemm(int M) { + return M > 4; +} + +template <> +inline bool can_use_brgemm(int M) { + return M > 4; +} + +// work around compiler internal error +#define BLOCK_K 128 // 4 * TILE_K + +// adjust leading dimension size for K +template +inline int64_t get_row_size(int64_t K) { + return K; +} + +template <> +inline int64_t get_row_size(int64_t K) { + return K + sizeof(int32_t); +} + +inline int64_t get_row_size(int64_t K, bool use_int8_w8a8) { + return use_int8_w8a8 ? K + sizeof(int32_t) : K; +} + +// pack weight to vnni format +at::Tensor convert_weight_packed(at::Tensor& weight); + +// moe implementations for int8 w8a8 +template +void fused_experts_int8_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic1, + scalar_t* __restrict__ ic2, + uint8_t* __restrict__ A_tmp, + float* __restrict__ C_tmp, + uint8_t* __restrict__ Aq_tmp, + float* __restrict__ As_tmp, + const scalar_t* __restrict__ input, + const int8_t* __restrict__ packed_w1, + const int8_t* __restrict__ packed_w2, + const float* __restrict__ w1s, + const float* __restrict__ w2s, + const float* __restrict__ topk_weights, + const int32_t* __restrict__ sorted_ids, + const int32_t* __restrict__ expert_ids, + const int32_t* __restrict__ offsets, + int64_t M, + int64_t N, + int64_t K, + int64_t E, + int64_t topk, + int64_t num_tokens_post_pad); + +// moe implementations for fp8 w8a16 +template +void fused_experts_fp8_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic0, + scalar_t* __restrict__ ic1, + scalar_t* __restrict__ ic2, + scalar_t* __restrict__ A_tmp, + scalar_t* __restrict__ B_tmp, + float* __restrict__ C_tmp, + const scalar_t* __restrict__ input, + const at::Float8_e4m3fn* __restrict__ packed_w1, + const at::Float8_e4m3fn* __restrict__ packed_w2, + const float* __restrict__ w1s, + const float* __restrict__ w2s, + int64_t block_size_N, + int64_t block_size_K, + const float* __restrict__ topk_weights, + const int32_t* __restrict__ sorted_ids, + const int32_t* __restrict__ expert_ids, + const int32_t* __restrict__ offsets, + int64_t M, + int64_t N, + int64_t K, + int64_t E, + int64_t topk, + int64_t num_tokens_post_pad); + +// shared expert implementation for int8 w8a8 +template +void shared_expert_int8_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic1, + float* __restrict__ C_tmp, + uint8_t* __restrict__ Aq_tmp, + float* __restrict__ As_tmp, + const scalar_t* __restrict__ input, + const int8_t* __restrict__ packed_w1, + const int8_t* __restrict__ packed_w2, + const float* __restrict__ w1s, + const float* __restrict__ w2s, + const scalar_t* __restrict__ fused_experts_out, + float routed_scaling_factor, + int64_t M, + int64_t N, + int64_t K); + +template +void shared_expert_fp8_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic0, + scalar_t* __restrict__ ic1, + scalar_t* __restrict__ B_tmp, + float* __restrict__ C_tmp, + const scalar_t* __restrict__ input, + const at::Float8_e4m3fn* __restrict__ packed_w1, + const at::Float8_e4m3fn* __restrict__ packed_w2, + const float* __restrict__ w1s, + const float* __restrict__ w2s, + int64_t block_size_N, + int64_t block_size_K, + const scalar_t* __restrict__ fused_experts_out, + float routed_scaling_factor, + int64_t M, + int64_t N, + int64_t K); + +// tinygemm interface +template +void tinygemm_kernel( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B, + scalar_t* __restrict__ C, + float* __restrict__ Ctmp, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg); + +template +void tinygemm_kernel( + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B, + scalar_t* __restrict__ C, + int32_t* __restrict__ Ctmp, + const float* __restrict__ As, + const float* __restrict__ Bs, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg); + +template +void tinygemm_kernel( + const scalar_t* __restrict__ A, + const at::Float8_e4m3fn* __restrict__ B, + scalar_t* __restrict__ C, + scalar_t* __restrict__ Btmp, + float* __restrict__ Ctmp, + const float* __restrict__ scale, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg, + int64_t block_size_K, + bool do_unpack = true); diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/gemm_fp8.cpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/gemm_fp8.cpp new file mode 100644 index 00000000..008f8329 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/gemm_fp8.cpp @@ -0,0 +1,551 @@ +#include "common.h" +#include "gemm.h" +#include "vec.h" + +namespace { + +template +inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ input, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + fVec data0 = fVec::loadu(input + d); + fVec data1 = fVec::loadu(input + d + fVec::size()); + bVec out_vec = convert_from_float_ext(data0, data1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d]); + } +} + +template +inline void copy_add_stub( + scalar_t* __restrict__ out, const float* __restrict__ input, const float* __restrict__ bias, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + fVec data0 = fVec::loadu(input + d) + fVec::loadu(bias + d); + fVec data1 = fVec::loadu(input + d + fVec::size()) + fVec::loadu(bias + d + fVec::size()); + bVec out_vec = convert_from_float_ext(data0, data1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d] + bias[d]); + } +} + +inline void unpack_B( + at::BFloat16* __restrict__ Btmp, + const at::Float8_e4m3fn* __restrict__ packed_B, + int N, + int K, + int ldb, + int ldb_tmp, + float scale) { +#if defined(CPU_CAPABILITY_AVX512) + // [K/2, N, 2] + const int K2 = K >> 1; + const int ldb2 = ldb; // ldb * 2 >> 1; + const uint16_t* b_ptr = reinterpret_cast(packed_B); + const __m512 vd = _mm512_set1_ps(scale); + + constexpr int BLOCK_N = block_size_n(); + static_assert(BLOCK_N == 32); + + // prefetch distance + constexpr int PREFETCH_SIZE_K = 64; + +#pragma GCC unroll 4 + for (int k = 0; k < K2; ++k) { + __m512i b8 = _mm512_loadu_si512(b_ptr + k * ldb2); + if constexpr (PREFETCH_SIZE_K > 0) { + _mm_prefetch(b_ptr + (k + PREFETCH_SIZE_K) * ldb2, _MM_HINT_T0); + } + + __m256i b8_0 = _mm512_extracti32x8_epi32(b8, 0); + __m256i b8_1 = _mm512_extracti32x8_epi32(b8, 1); + + __m512bh bf16_0 = CVT_FP8_TO_BF16(b8_0); + __m512bh bf16_1 = CVT_FP8_TO_BF16(b8_1); + + // Apply scale + __m512 f0_lo = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32((__m512i)bf16_0, 0)); + __m512 f0_hi = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32((__m512i)bf16_0, 1)); + __m512 f1_lo = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32((__m512i)bf16_1, 0)); + __m512 f1_hi = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32((__m512i)bf16_1, 1)); + + f0_lo = _mm512_mul_ps(f0_lo, vd); + f0_hi = _mm512_mul_ps(f0_hi, vd); + f1_lo = _mm512_mul_ps(f1_lo, vd); + f1_hi = _mm512_mul_ps(f1_hi, vd); + + bf16_0 = _mm512_cvtne2ps_pbh(f0_hi, f0_lo); + bf16_1 = _mm512_cvtne2ps_pbh(f1_hi, f1_lo); + + _mm512_storeu_si512(Btmp + k * ldb_tmp * 2 + 0, (__m512i)bf16_0); + _mm512_storeu_si512(Btmp + k * ldb_tmp * 2 + 32, (__m512i)bf16_1); + } +#else + TORCH_CHECK(false, "unpack_B: scalar path not implemented!"); +#endif +} + +template +struct tinygemm_kernel_nn { + static inline void apply( + const scalar_t* __restrict__ A, + const packed_t* __restrict__ B, + scalar_t* __restrict__ C, + const float* __restrict__ bias, + const float* __restrict__ scale, + int K, + int lda, + int ldb, + int ldc, + int64_t block_size_K) { + TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!"); + } +}; + +#if defined(CPU_CAPABILITY_AVX512) +template +struct tinygemm_kernel_nn { + static inline void apply( + const at::BFloat16* __restrict__ A, + const at::Float8_e4m3fn* __restrict__ B, + at::BFloat16* __restrict__ C, + const float* __restrict__ bias, + const float* __restrict__ scale, + int K, + int lda, + int ldb, + int ldc, + int64_t block_size_K) { + constexpr int ROWS = BLOCK_M; + constexpr int COLS = BLOCK_N / 16; + + const int KB = div_up(K, BLOCK_K); + + // prefetch distance + constexpr int PREFETCH_SIZE_K = 64; + constexpr int PREFETCH_SIZE_KB = 1; + + __m512bh va; + __m512bh vb[COLS]; + __m512 vc[ROWS * COLS]; + __m512 vsum[ROWS * COLS]; + + // block quant scale + __m512 vscale; + + auto loadc = [&](auto i) { + constexpr int col = i % COLS; + if constexpr (has_bias) { + vc[i] = _mm512_loadu_ps(bias + col * 16); + } else { + vc[i] = _mm512_setzero_ps(); + } + }; + Unroll{}(loadc); + + const int lda2 = lda >> 1; + const int ldb2 = ldb; // ldb * 2 >> 1; + const float* a_ptr = reinterpret_cast(A); + const uint16_t* b_ptr = reinterpret_cast(B); + + auto compute = [&](auto i, int k) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + va = (__m512bh)(_mm512_set1_ps(a_ptr[row * lda2 + k])); + if constexpr (PREFETCH_SIZE_K > 0) { + _mm_prefetch(a_ptr + row * lda2 + k + PREFETCH_SIZE_K, _MM_HINT_T0); + } + } + if constexpr (row == 0) { + if constexpr (col % 2 == 0) { + __m512i b8 = _mm512_loadu_si512(b_ptr + k * ldb2 + col * 16); + if constexpr (PREFETCH_SIZE_K > 0) { + _mm_prefetch(b_ptr + (k + PREFETCH_SIZE_K) * ldb2 + col * 16, _MM_HINT_T0); + } + vb[col + 0] = CVT_FP8_TO_BF16(_mm512_extracti32x8_epi32(b8, 0)); + vb[col + 1] = CVT_FP8_TO_BF16(_mm512_extracti32x8_epi32(b8, 1)); + } + } + vsum[i] = _mm512_dpbf16_ps(vsum[i], va, vb[col]); + }; + + constexpr int BLOCK_K2 = BLOCK_K >> 1; + for (int kb = 0; kb < KB; ++kb) { + int kb_start = kb * BLOCK_K2; + int kb_end = std::min(K >> 1, kb_start + BLOCK_K2); + // 1. load scale vector + vscale = _mm512_set1_ps(scale[kb]); + if constexpr (PREFETCH_SIZE_KB > 0) { + _mm_prefetch(scale + kb + PREFETCH_SIZE_KB, _MM_HINT_T0); + } + // 2. zero vsum for each block + Unroll{}([&](auto i) { vsum[i] = _mm512_setzero_ps(); }); + // 3. accumulate across each block + for (int k = kb_start; k < kb_end; ++k) { + Unroll{}(compute, k); + } + // 4. apply scale + Unroll{}([&](auto i) { vc[i] = _mm512_fmadd_ps(vsum[i], vscale, vc[i]); }); + } + + auto storec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + // for COLS = 2,4 use 512bit store + if constexpr (col % 2 == 0) { + _mm512_storeu_si512( + reinterpret_cast<__m512i*>((C + row * ldc + col * 16)), + (__m512i)(_mm512_cvtne2ps_pbh(vc[row * COLS + col + 1], vc[row * COLS + col]))); + } + }; + Unroll{}(storec); + } +}; +#endif + +#define LAUNCH_TINYGEMM_KERNEL_NN(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_nn::apply( \ + A + mb_start * lda, \ + B + nb_start * 2, \ + C + mb_start * ldc + nb_start, \ + has_bias ? bias + nb_start : nullptr, \ + scale, \ + K, \ + lda, \ + ldb, \ + ldc, \ + block_size_K); + +template +struct brgemm { + static inline void apply( + const scalar_t* __restrict__ A, + const packed_t* __restrict__ B, + scalar_t* __restrict__ C, + scalar_t* __restrict__ Btmp, + float* __restrict__ Ctmp, + const float* __restrict__ bias, + const float* __restrict__ scale, + int M, + int N, + int K, + int lda, + int ldb, + int ldc, + bool do_unpack = true) { + TORCH_CHECK(false, "struct brgemm: primary template not implemented!"); + } +}; + +template +struct brgemm { + static inline void apply( + const at::BFloat16* __restrict__ A, + const at::Float8_e4m3fn* __restrict__ B, + at::BFloat16* __restrict__ C, + at::BFloat16* __restrict__ Btmp, + float* __restrict__ Ctmp, + const float* __restrict__ bias, + const float* __restrict__ scale, + int M, + int N, + int K, + int lda, + int ldb, + int ldc, + bool do_unpack = true) { + constexpr int BLOCK_N = block_size_n(); + + // [K, BLOCK_N] -> [K / 2, BLOCK_N * 2] + const int ldb_tmp = BLOCK_N; + + if (do_unpack) { + for (int k = 0; k < K; k += BLOCK_K) { + int kb_size = std::min(BLOCK_K, K - k); + + int idx = k >> 7; // k / BLOCK_K where BLOCK_K = 128 + unpack_B(Btmp + k * ldb_tmp, B + k * ldb, N, kb_size, ldb, ldb_tmp, scale[idx]); + } + } + + at::native::cpublas::brgemm(M, N, K, lda, ldb_tmp, BLOCK_N, /* add_C */ false, A, Btmp, Ctmp); + + // copy from Ctmp to C + for (int m = 0; m < M; ++m) { + if constexpr (has_bias) { + copy_add_stub(C + m * ldc, Ctmp + m * BLOCK_N, bias, N); + } else { + copy_stub(C + m * ldc, Ctmp + m * BLOCK_N, N); + } + } + } +}; + +template +void tinygemm_kernel( + const scalar_t* __restrict__ A, + const at::Float8_e4m3fn* __restrict__ B, + scalar_t* __restrict__ C, + scalar_t* __restrict__ Btmp, + float* __restrict__ Ctmp, + const float* __restrict__ scale, + const float* __restrict__ bias, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg, + int64_t block_size_K, + bool do_unpack = true) { + if (brg) { + brgemm::apply( + A, B, C, Btmp, Ctmp, bias, scale, M, N, K, lda, ldb, ldc, do_unpack); + return; + } + + // pattern: 1-4-16 + constexpr int64_t BLOCK_M = 4; + constexpr int64_t BLOCK_N = 64; + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + for (int mb = 0; mb < MB; ++mb) { + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(BLOCK_M, M - mb_start); + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (mb_size << 4 | nb_size >> 4) { + case 0x12: + LAUNCH_TINYGEMM_KERNEL_NN(1, 32); + break; + case 0x22: + LAUNCH_TINYGEMM_KERNEL_NN(2, 32); + break; + case 0x32: + LAUNCH_TINYGEMM_KERNEL_NN(3, 32); + break; + case 0x42: + LAUNCH_TINYGEMM_KERNEL_NN(4, 32); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size"); + } + } + } +} + +template +void fp8_scaled_mm_kernel_impl( + scalar_t* __restrict__ out, + const scalar_t* __restrict__ mat1, + const at::Float8_e4m3fn* __restrict__ mat2, + const float* __restrict__ scales2, + const float* __restrict__ bias, + scalar_t* __restrict__ buffer, + int64_t M, + int64_t N, + int64_t K, + int64_t mat1_strideM, + int64_t out_strideM, + int64_t block_size_N, + int64_t block_size_K, + int64_t buffer_size_per_thread) { + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + + const int64_t scale_size_K = div_up(K, block_size_K); + const int64_t blocks_n_per_group = block_size_N / BLOCK_N; + + const bool use_brgemm = can_use_brgemm(M); + + // parallel on [MB, NB] + AT_DISPATCH_BOOL(bias != nullptr, has_bias, [&] { + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + int tid = get_thread_num(); + scalar_t* __restrict__ Btmp = buffer + tid * buffer_size_per_thread; + float* __restrict__ Ctmp = (float*)((void*)(Btmp + MAX_CACHE_BLOCK_SIZE * BLOCK_N * K)); + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + const float* scale_ptr = scales2 + (nb / blocks_n_per_group) * scale_size_K; + + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(M - mb_start, BLOCK_M); + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(N - nb_start, BLOCK_N); + + // only do unpacking for the first row + bool do_unpack = (mb == mb0); + + tinygemm_kernel( + /* A */ mat1 + mb_start * mat1_strideM, + /* B */ mat2 + nb_start * K, // nb * BLOCK_N * K + /* C */ out + mb_start * out_strideM + nb_start, + /* Btmp */ Btmp + nb_offset * BLOCK_N * K, + /* Ctmp */ Ctmp, + /* scale */ scale_ptr, + /* bias */ bias + nb_start, + /* M */ mb_size, + /* N */ nb_size, + /* K */ K, + /* lda */ mat1_strideM, + /* ldb */ nb_size, + /* ldc */ out_strideM, + /* brg */ use_brgemm, + /* block_size_K */ block_size_K, + /* do_unpack */ do_unpack); + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + }); +} + +} // anonymous namespace + +// tinygemm interface +template +void tinygemm_kernel( + const scalar_t* __restrict__ A, + const at::Float8_e4m3fn* __restrict__ B, + scalar_t* __restrict__ C, + scalar_t* __restrict__ Btmp, + float* __restrict__ Ctmp, + const float* __restrict__ scale, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg, + int64_t block_size_K, + bool do_unpack) { + tinygemm_kernel( + A, B, C, Btmp, Ctmp, scale, nullptr, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack); +} + +#define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE) \ + template void tinygemm_kernel( \ + const TYPE* __restrict__ A, \ + const at::Float8_e4m3fn* __restrict__ B, \ + TYPE* __restrict__ C, \ + TYPE* __restrict__ Btmp, \ + float* __restrict__ Ctmp, \ + const float* __restrict__ scale, \ + int64_t M, \ + int64_t N, \ + int64_t K, \ + int64_t lda, \ + int64_t ldb, \ + int64_t ldc, \ + bool brg, \ + int64_t block_size_K, \ + bool do_unpack) + +INSTANTIATE_TINYGEMM_TEMPLATE(at::BFloat16); +INSTANTIATE_TINYGEMM_TEMPLATE(at::Half); + +at::Tensor fp8_scaled_mm_cpu( + at::Tensor& mat1, + at::Tensor& mat2, + at::Tensor& scales2, + std::vector block_size, + const std::optional& bias, + at::ScalarType out_dtype, + bool is_vnni) { + RECORD_FUNCTION("sgl-kernel::fp8_scaled_mm_cpu", std::vector({mat1, mat2, scales2, block_size, bias})); + + auto packed_w = is_vnni ? mat2 : convert_weight_packed(mat2); + + CHECK_LAST_DIM_CONTIGUOUS_INPUT(mat1); + CHECK_INPUT(mat2); + CHECK_INPUT(scales2); + TORCH_CHECK(scales2.scalar_type() == at::kFloat, "fp8_scaled_mm_cpu: expect scales2 to be float32."); + + int64_t M = mat1.size(0); + int64_t N = mat2.size(0); + int64_t K = mat2.size(1); + + CHECK_EQ(mat1.size(1), K); + CHECK_DIM(2, mat1); + CHECK_DIM(2, mat2); + + TORCH_CHECK(block_size.size() == 2, "fp8_scaled_mm_cpu: expect block_size.size() to be 2."); + + int64_t block_size_N = block_size[0]; + int64_t block_size_K = block_size[1]; + + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + TORCH_CHECK(block_size_N % BLOCK_N == 0, "fp8_scaled_mm_cpu: expect block_size_N to be multiples of BLOCK_N"); + TORCH_CHECK(block_size_K == BLOCK_K, "fp8_scaled_mm_cpu: expect block_size_K equals to BLOCK_K"); + CHECK_EQ(scales2.size(0), div_up(N, block_size_N)); + CHECK_EQ(scales2.size(1), div_up(K, block_size_K)); + + const auto st = mat1.scalar_type(); + TORCH_CHECK(st == at::kBFloat16 || st == at::kHalf, "fp8_scaled_mm_cpu: expect A to be bfloat16 or half."); + TORCH_CHECK(st == out_dtype, "fp8_scaled_mm_cpu: expect A has same dtype with out_dtype."); + TORCH_CHECK(mat2.scalar_type() == at::kFloat8_e4m3fn, "fp8_scaled_mm_cpu: expect mat2 to be fp8_e4m3."); + TORCH_CHECK(scales2.scalar_type() == at::kFloat, "fp8_scaled_mm_cpu: expect scales to be float32."); + auto out = at::empty({M, N}, mat1.options().dtype(out_dtype)); + + // strides + int64_t mat1_strideM = mat1.stride(0); + int64_t out_strideM = out.stride(0); + + const bool has_bias = bias.has_value(); + const float* bias_data = nullptr; + if (has_bias) { + CHECK_EQ(bias.value().size(0), N); + bias_data = bias.value().data_ptr(); + } + + // Btmp : [T, BLOCK_N * K] + // Ctmp : [T, BLOCK_M * BLOCK_N] + int num_threads = at::get_num_threads(); + int64_t size_per_thread = MAX_CACHE_BLOCK_SIZE * BLOCK_N * K + BLOCK_M * BLOCK_N * 2; + auto buffer = at::empty({num_threads, size_per_thread}, mat1.options()); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(out_dtype, "fp8_scaled_mm_kernel_impl", [&] { + fp8_scaled_mm_kernel_impl( + out.data_ptr(), + mat1.data_ptr(), + packed_w.data_ptr(), + scales2.data_ptr(), + bias_data, + buffer.data_ptr(), + M, + N, + K, + mat1_strideM, + out_strideM, + block_size_N, + block_size_K, + size_per_thread); + }); + + return out; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/gemm_int8.cpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/gemm_int8.cpp new file mode 100644 index 00000000..cb614660 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/gemm_int8.cpp @@ -0,0 +1,547 @@ +#include "common.h" +#include "gemm.h" +#include "vec.h" + +namespace { + +template +struct scale_C { + static inline void apply( + scalar_t* __restrict__ C, + const int32_t* __restrict__ Ctmp, + const int32_t* __restrict__ Bcomp, + const float* __restrict__ bias, + float As, + const float* __restrict__ Bs) { + TORCH_CHECK(false, "scale_C: scalar path not implemented!"); + } +}; + +#if defined(CPU_CAPABILITY_AVX512) +template +struct scale_C { + static inline void apply( + at::BFloat16* __restrict__ C, + const int32_t* __restrict__ Ctmp, + const int32_t* __restrict__ Bcomp, + const float* __restrict__ bias, + float As, + const float* __restrict__ Bs) { + constexpr int COLS = BLOCK_N / 16; + static_assert(COLS % 2 == 0); + + __m512 vc[COLS]; + __m512 vd0 = _mm512_set1_ps(As); + + auto compute = [&](auto col) { + __m512 vd1 = _mm512_loadu_ps(Bs + col * 16); + __m512i vcomp = _mm512_loadu_si512(Bcomp + col * 16); + __m512i vc32 = _mm512_loadu_si512(Ctmp + col * 16); + vc[col] = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc32, vcomp)); + if constexpr (has_bias) { + __m512 vbias = _mm512_loadu_ps(bias + col * 16); + vc[col] = _mm512_fmadd_ps(_mm512_mul_ps(vc[col], vd0), vd1, vbias); + } else { + vc[col] = _mm512_mul_ps(_mm512_mul_ps(vc[col], vd0), vd1); + } + }; + Unroll{}(compute); + + auto storec = [&](auto col) { + // for COLS = 2, 4 use 512bit store + if constexpr (col % 2 == 0) { + _mm512_storeu_si512( + reinterpret_cast<__m512i*>((C + col * 16)), (__m512i)(_mm512_cvtne2ps_pbh(vc[col + 1], vc[col + 0]))); + } + }; + Unroll{}(storec); + } +}; +#endif + +template +struct tinygemm_kernel_nn { + static inline void apply( + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B, + scalar_t* __restrict__ C, + const float* __restrict__ As, + const float* __restrict__ Bs, + const int32_t* __restrict__ Bcomp, + const float* __restrict__ bias, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!"); + } +}; + +#if defined(CPU_CAPABILITY_AVX512) +template +struct tinygemm_kernel_nn { + static inline void apply( + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B, + at::BFloat16* __restrict__ C, + const float* __restrict__ As, + const float* __restrict__ Bs, + const int32_t* __restrict__ Bcomp, + const float* __restrict__ bias, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + constexpr int ROWS = BLOCK_M; + constexpr int COLS = BLOCK_N / 16; + static_assert(COLS % 2 == 0); + + // prefetch distance + constexpr int PREFETCH_SIZE_K = 0; + + __m512i va; + __m512i vb[COLS]; + __m512i vc[ROWS * COLS]; + __m512i vcomp[COLS]; + __m512 vd0; + __m512 vd1[COLS]; + + // oops! 4x4 spills but we use 4x2 + __m512 vbias[COLS]; + + // [NOTE]: s8s8 igemm compensation in avx512-vnni + // + // avx512-vnni has no s8s8, so we need to change s8s8 to u8s8 with compensate: + // + // a * b = (a + 128) * b - 128 * b + // s s u s u s + // + // 1) 128 * b is pre-computed when packing B to vnni formats + // 2) a + 128 is fused when dynamically quantize A + // + auto loadc = [&](auto i) { vc[i] = _mm512_set1_epi32(0); }; + Unroll{}(loadc); + + const int64_t K4 = K >> 2; + const int64_t lda4 = lda >> 2; + const int64_t ldb4 = ldb; // ldb * 4 >> 2; + const int32_t* a_ptr = reinterpret_cast(A); + const int32_t* b_ptr = reinterpret_cast(B); + + auto compute = [&](auto i, int64_t k) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + va = _mm512_set1_epi32(a_ptr[row * lda4 + k]); + } + if constexpr (row == 0) { + vb[col] = _mm512_loadu_si512(b_ptr + k * ldb4 + col * 16); + if constexpr (PREFETCH_SIZE_K > 0) { + _mm_prefetch(b_ptr + (k + PREFETCH_SIZE_K) * ldb4 + col * 16, _MM_HINT_T0); + } + } + vc[i] = _mm512_dpbusd_epi32(vc[i], va, vb[col]); + }; + for (int64_t k = 0; k < K4; ++k) { + Unroll{}(compute, k); + } + + auto storec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + // load a scale + if constexpr (col == 0) { + vd0 = _mm512_set1_ps(As[row]); + } + // load b scale and vcomp per 2 vectors + // also load bias if any + if constexpr (row == 0) { + if constexpr (col % 2 == 0) { + vd1[col + 0] = _mm512_loadu_ps(Bs + col * 16); + vd1[col + 1] = _mm512_loadu_ps(Bs + col * 16 + 16); + vcomp[col + 0] = _mm512_loadu_si512(Bcomp + col * 16); + vcomp[col + 1] = _mm512_loadu_si512(Bcomp + col * 16 + 16); + if constexpr (has_bias) { + vbias[col + 0] = _mm512_loadu_ps(bias + col * 16); + vbias[col + 1] = _mm512_loadu_ps(bias + col * 16 + 16); + } + } + } + + // for COLS = 2, 4 use 512bit store + if constexpr (col % 2 == 0) { + __m512 vc0 = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc[row * COLS + col + 0], vcomp[col + 0])); + __m512 vc1 = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc[row * COLS + col + 1], vcomp[col + 1])); + if constexpr (has_bias) { + vc0 = _mm512_fmadd_ps(_mm512_mul_ps(vc0, vd0), vd1[col + 0], vbias[col + 0]); + vc1 = _mm512_fmadd_ps(_mm512_mul_ps(vc1, vd0), vd1[col + 1], vbias[col + 1]); + } else { + vc0 = _mm512_mul_ps(_mm512_mul_ps(vc0, vd0), vd1[col + 0]); + vc1 = _mm512_mul_ps(_mm512_mul_ps(vc1, vd0), vd1[col + 1]); + } + + _mm512_storeu_si512( + reinterpret_cast<__m512i*>((C + row * ldc + col * 16)), (__m512i)(_mm512_cvtne2ps_pbh(vc1, vc0))); + } + }; + Unroll{}(storec); + } +}; +#endif + +#define LAUNCH_TINYGEMM_KERNEL_NN(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_nn::apply( \ + A + mb_start * lda, \ + B + nb_start * 4, \ + C + mb_start * ldc + nb_start, \ + As + mb_start, \ + Bs + nb_start, \ + Bcomp + nb_start, \ + has_bias ? bias + nb_start : nullptr, \ + K, \ + lda, \ + ldb, \ + ldc); + +template +void tinygemm_kernel( + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B, + scalar_t* __restrict__ C, + int32_t* __restrict__ Ctmp, + const float* __restrict__ As, + const float* __restrict__ Bs, + const float* __restrict__ bias, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg) { + // B compensation + const int32_t* Bcomp = reinterpret_cast(B + block_size_n() * K); + + if (brg) { + constexpr int BLOCK_N = block_size_n(); + at::native::cpublas::brgemm(M, N, K, lda, ldb, BLOCK_N, /* add_C */ false, A, B, Ctmp); + + // apply compensation and scale + for (int64_t m = 0; m < M; ++m) { + scale_C::apply(C + m * ldc, Ctmp + m * BLOCK_N, Bcomp, bias, As[m], Bs); + } + return; + } + + // pattern: 1-4-16 + constexpr int64_t BLOCK_M = 4; + constexpr int64_t BLOCK_N = 64; + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + for (int64_t mb = 0; mb < MB; ++mb) { + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(BLOCK_M, M - mb_start); + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (mb_size << 4 | nb_size >> 4) { + // mb_size = 1 + case 0x12: + LAUNCH_TINYGEMM_KERNEL_NN(1, 32); + break; + case 0x14: + LAUNCH_TINYGEMM_KERNEL_NN(1, 64); + break; + // mb_size = 2 + case 0x22: + LAUNCH_TINYGEMM_KERNEL_NN(2, 32); + break; + case 0x24: + LAUNCH_TINYGEMM_KERNEL_NN(2, 64); + break; + // mb_size = 3 + case 0x32: + LAUNCH_TINYGEMM_KERNEL_NN(3, 32); + break; + case 0x34: + LAUNCH_TINYGEMM_KERNEL_NN(3, 64); + break; + // mb_size = 4 + case 0x42: + LAUNCH_TINYGEMM_KERNEL_NN(4, 32); + break; + case 0x44: + LAUNCH_TINYGEMM_KERNEL_NN(4, 64); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size"); + } + } + } +} + +template +void int8_scaled_mm_kernel_impl( + scalar_t* __restrict__ out, + const uint8_t* __restrict__ mat1, + const int8_t* __restrict__ mat2, + const float* __restrict__ scales1, + const float* __restrict__ scales2, + const float* __restrict__ bias, + int64_t M, + int64_t N, + int64_t K) { + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + + const bool use_brgemm = can_use_brgemm(M); + + // K + 4 after compensation + const int64_t packed_row_size = get_row_size(K); + + AT_DISPATCH_BOOL(bias != nullptr, has_bias, [&] { + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // for brgemm, use int32_t for accumulate + alignas(64) int32_t Ctmp[BLOCK_M * BLOCK_N]; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + int mb_start = mb * BLOCK_M; + int mb_size = std::min(M - mb_start, BLOCK_M); + int nb_start = nb * BLOCK_N; + int nb_size = std::min(N - nb_start, BLOCK_N); + + tinygemm_kernel( + /* A */ mat1 + mb_start * K, + /* B */ mat2 + nb_start * packed_row_size /* nb * BLOCK_N * (K + 4) */, + /* C */ out + mb_start * N + nb_start, + /* Ctmp*/ Ctmp, + /* As */ scales1 + mb_start, + /* Bs */ scales2 + nb_start, + /* bias*/ bias + nb_start, + /* M */ mb_size, + /* N */ nb_size, + /* K */ K, + /* lda */ K, + /* ldb */ nb_size, + /* ldc */ N, + /* brg */ use_brgemm); + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + }); +} + +} // anonymous namespace + +// tinygemm interface +template +void tinygemm_kernel( + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B, + scalar_t* __restrict__ C, + int32_t* __restrict__ Ctmp, + const float* __restrict__ As, + const float* __restrict__ Bs, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg) { + tinygemm_kernel(A, B, C, Ctmp, As, Bs, nullptr, M, N, K, lda, ldb, ldc, brg); +} + +#define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE) \ + template void tinygemm_kernel( \ + const uint8_t* __restrict__ A, \ + const int8_t* __restrict__ B, \ + TYPE* __restrict__ C, \ + int32_t* __restrict__ Ctmp, \ + const float* __restrict__ As, \ + const float* __restrict__ Bs, \ + int64_t M, \ + int64_t N, \ + int64_t K, \ + int64_t lda, \ + int64_t ldb, \ + int64_t ldc, \ + bool brg) + +INSTANTIATE_TINYGEMM_TEMPLATE(at::BFloat16); +INSTANTIATE_TINYGEMM_TEMPLATE(at::Half); + +std::tuple per_token_quant_int8_cpu(at::Tensor& A) { + RECORD_FUNCTION("sgl-kernel::per_token_quant_int8_cpu", std::vector({A})); + + CHECK_LAST_DIM_CONTIGUOUS_INPUT(A); + CHECK_DIM(2, A); + + int64_t M = A.size(0); + int64_t K = A.size(1); + int64_t lda = A.stride(0); + + const auto st = A.scalar_type(); + TORCH_CHECK(st == at::kBFloat16 || st == at::kHalf, "per_token_quant_int8: expect A to be bfloat16 or half."); + + auto Aq = at::empty({M, K}, A.options().dtype(at::kByte)); + auto As = at::empty({M}, A.options().dtype(at::kFloat)); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(st, "per_token_quant_int8", [&] { + uint8_t* __restrict__ Aq_data = Aq.data_ptr(); + float* __restrict__ As_data = As.data_ptr(); + const scalar_t* __restrict__ A_data = A.data_ptr(); + + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + quantize_row_int8(Aq_data + m * K, As_data[m], A_data + m * lda, K); + } + }); + }); + return std::make_tuple(Aq, As); +} + +// weight : static, per-channel, symmetric +// activation : dynamic, per-token, symmetric +// +// mat1 : [M, K] +// mat2 : [N, K] +// scales1 : [M] +// scales2 : [N] +// bias : [N] +// out : [M, N] +// +at::Tensor int8_scaled_mm_cpu( + at::Tensor& mat1, + at::Tensor& mat2, + at::Tensor& scales1, + at::Tensor& scales2, + const std::optional& bias, + at::ScalarType out_dtype, + bool is_vnni) { + RECORD_FUNCTION("sgl-kernel::int8_scaled_mm_cpu", std::vector({mat1, mat2, scales1, scales2, bias})); + + auto packed_w = is_vnni ? mat2 : convert_weight_packed(mat2); + + CHECK_INPUT(mat1); + CHECK_INPUT(mat2); + CHECK_INPUT(scales1); + CHECK_INPUT(scales2); + CHECK_DIM(2, mat1); + CHECK_DIM(2, mat2); + + int64_t M = mat1.size(0); + int64_t N = mat2.size(0); + int64_t K = mat1.size(1); + + // see [NOTE]: s8s8 igemm compensation in avx512-vnni + CHECK_EQ(mat2.size(1), (int64_t)(is_vnni ? K + sizeof(int32_t) : K)); + CHECK_EQ(scales1.numel(), M); + CHECK_EQ(scales2.numel(), N); + + TORCH_CHECK(mat1.scalar_type() == at::kByte, "int8_scaled_mm: expect mat1 to be uint8."); + TORCH_CHECK(mat2.scalar_type() == at::kChar, "int8_scaled_mm: expect mat2 to be int8."); + TORCH_CHECK( + scales1.scalar_type() == at::kFloat && scales2.scalar_type() == at::kFloat, + "int8_scaled_mm: expect scales to be float32."); + + auto out = at::empty({M, N}, mat1.options().dtype(out_dtype)); + + const bool has_bias = bias.has_value(); + const float* bias_data = nullptr; + if (has_bias) { + CHECK_EQ(bias.value().size(0), N); + bias_data = bias.value().data_ptr(); + } + + AT_DISPATCH_REDUCED_FLOATING_TYPES(out_dtype, "int8_scaled_mm_kernel_impl", [&] { + int8_scaled_mm_kernel_impl( + out.data_ptr(), + mat1.data_ptr(), + packed_w.data_ptr(), + scales1.data_ptr(), + scales2.data_ptr(), + bias_data, + M, + N, + K); + }); + return out; +} + +// fused `per_token_quant_int8_cpu` and `int8_scaled_mm_cpu` +at::Tensor int8_scaled_mm_with_quant( + at::Tensor& mat1, + at::Tensor& mat2, + at::Tensor& scales2, + const std::optional& bias, + at::ScalarType out_dtype, + bool is_vnni) { + RECORD_FUNCTION("sgl-kernel::int8_scaled_mm_cpu", std::vector({mat1, mat2, scales2, bias})); + + auto packed_w = is_vnni ? mat2 : convert_weight_packed(mat2); + + CHECK_LAST_DIM_CONTIGUOUS_INPUT(mat1); + CHECK_INPUT(mat2); + CHECK_INPUT(scales2); + CHECK_DIM(2, mat1); + CHECK_DIM(2, mat2); + + int64_t M = mat1.size(0); + int64_t N = mat2.size(0); + int64_t K = mat1.size(1); + int64_t lda = mat1.stride(0); + + // see [NOTE]: s8s8 igemm compensation in avx512-vnni + CHECK_EQ(mat2.size(1), (int64_t)(is_vnni ? K + sizeof(int32_t) : K)); + CHECK_EQ(scales2.numel(), N); + + const auto st = mat1.scalar_type(); + TORCH_CHECK(st == at::kBFloat16 || st == at::kHalf, "int8_scaled_mm_with_quant: expect A to be bfloat16 or half."); + TORCH_CHECK(st == out_dtype, "int8_scaled_mm_with_quant: expect A has same dtype with out_dtype."); + TORCH_CHECK(mat2.scalar_type() == at::kChar, "int8_scaled_mm_with_quant: expect mat2 to be int8."); + TORCH_CHECK(scales2.scalar_type() == at::kFloat, "int8_scaled_mm_with_quant: expect scales to be float32."); + + const int64_t buffer_size = M * K + M * sizeof(float); + auto buffer = at::empty({buffer_size}, mat1.options().dtype(at::kByte)); + auto out = at::empty({M, N}, mat1.options().dtype(out_dtype)); + + const bool has_bias = bias.has_value(); + const float* bias_data = nullptr; + if (has_bias) { + CHECK_EQ(bias.value().size(0), N); + bias_data = bias.value().data_ptr(); + } + + AT_DISPATCH_REDUCED_FLOATING_TYPES(out_dtype, "int8_scaled_mm_with_quant_kernel_impl", [&] { + uint8_t* __restrict__ Aq_data = buffer.data_ptr(); + float* __restrict__ As_data = (float*)((void*)(Aq_data + M * K)); + const scalar_t* __restrict__ A_data = mat1.data_ptr(); + + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + quantize_row_int8(Aq_data + m * K, As_data[m], A_data + m * lda, K); + } + }); + + int8_scaled_mm_kernel_impl( + out.data_ptr(), + Aq_data, + packed_w.data_ptr(), + As_data, + scales2.data_ptr(), + bias_data, + M, + N, + K); + }); + return out; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/interface.cpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/interface.cpp new file mode 100644 index 00000000..9057a50f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/interface.cpp @@ -0,0 +1,74 @@ +#include +#include + +#include "shm.h" + +// Communication settings +static int world_rank = -1; +static int world_size = -1; + +static bool is_initialized = false; + +static bool all_ranks_local_p = false; + +void initialize(int64_t size, int64_t rank) { + if (is_initialized) { + return; + } + + // Check whether all ranks is on the same physical machine. + // If true, we will use an SHM based low latency allreduce + + auto ls_string = std::getenv("LOCAL_SIZE"); + int ls = 0; + if (ls_string != NULL) { + ls = std::stoi(std::getenv("LOCAL_SIZE")); + } + + if (size >= 1 && size == ls) { + all_ranks_local_p = true; + } + + world_size = size; + world_rank = rank; + is_initialized = true; + + const char* addr_string = std::getenv("MASTER_ADDR"); + if (addr_string == NULL) { + addr_string = ""; + } + const char* port_string = std::getenv("MASTER_PORT"); + if (port_string == NULL) { + port_string = ""; + } + + if (all_ranks_local_p) { + shm_initialize(size, rank, addr_string, port_string); + } +} + +void shm_allreduce(torch::Tensor& data, int64_t op) { + RECORD_FUNCTION("sgl-kernel::shm_allreduce", std::vector({data})); + + TORCH_CHECK(op == c10d::ReduceOp::SUM, "Only torch.distributed.ReduceOp.SUM is supported"); + + auto numel = data.numel(); + int data_size = numel * data.element_size(); + all_reduce_outer_loop(data, numel, data_size); + + return; +} + +torch::Tensor shm_allgather(torch::Tensor& data, int64_t dim) { + RECORD_FUNCTION("sgl-kernel::shm_allgather", std::vector({data})); + + auto numel = data.numel(); + int data_size = numel * data.element_size(); + if (dim < 0) { + dim += data.dim(); + } + std::vector result_shape = data.sizes().vec(); + result_shape[dim] *= world_size; + torch::Tensor result_tensor = torch::empty(result_shape, data.options()); + return all_gather(result_tensor, data, dim, numel, data_size); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/moe.cpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/moe.cpp new file mode 100644 index 00000000..c3d66cec --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/moe.cpp @@ -0,0 +1,1322 @@ +#include "common.h" +#include "gemm.h" +#include "vec.h" + +namespace { + +// [NOTE]: Fused MoE kernel with AMX +// +// This file contains implementations for +// * `moe_align_block_size` +// * `fused_moe` +// +// The functionality is identical to triton kernel, excepts: +// * fuse silu_and_mul with gemm1, therefore this kernel +// allocates 2 intermediate_caches instead of 3 +// * add `offsets` in `moe_align_block_size` which keeps track +// of starting offset for each M block. this is for keeping +// output of silu_and_mul in sorted order, thus load_A for +// the 2nd gemm would be contiguous, therefore we can directly +// load A from intermediate_cache1. +// +// TODO: +// 1. tune BLOCK_M and BLOCK_N (BLOCK_N * K fit L2) +// 2. add prefetch for load A which is indexed access +// 3. abstract at::native::cpublas::brgemm with WoQ gemm (M = 1 & M != 1) +// + +template +inline void fill_stub(scalar_t* __restrict__ out, scalar_t val, int64_t size) { + using Vec = at::vec::Vectorized; + const Vec data_vec(val); + at::vec::map([data_vec](Vec out) { return out = data_vec; }, out, out, size); +} + +template +inline void copy_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t size) { + using Vec = at::vec::Vectorized; +// no remainder +#pragma GCC unroll 4 + for (int64_t d = 0; d < size; d += Vec::size()) { + Vec data = Vec::loadu(input + d); + data.store(out + d); + } +} + +template +inline void copy_mul_stub(scalar_t* __restrict__ out, const float* __restrict__ input, float weight, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + const fVec weight_vec = fVec(weight); + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + fVec data0 = fVec::loadu(input + d) * weight_vec; + fVec data1 = fVec::loadu(input + d + fVec::size()) * weight_vec; + bVec out_vec = convert_from_float_ext(data0, data1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d] * weight); + } +} + +// acc from [topk, K] to [K] +template +inline void sum_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t topk, int64_t K) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + if (topk == 1) { + // do copy for topk = 1 + copy_stub(out, input, K); + } else { + // do sum for topk != 1 + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= K - kVecSize; d += kVecSize) { + fVec sum_fvec0 = fVec(0.f); + fVec sum_fvec1 = fVec(0.f); + for (int t = 0; t < topk; ++t) { + bVec x_bvec = bVec::loadu(input + t * K + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + sum_fvec0 += x_fvec0; + sum_fvec1 += x_fvec1; + } + bVec out_bvec = convert_from_float_ext(sum_fvec0, sum_fvec1); + out_bvec.store(out + d); + } + for (; d < K; ++d) { + float sum_val = 0.f; + for (int t = 0; t < topk; ++t) { + sum_val += static_cast(input[t * K + d]); + } + out[d] = static_cast(sum_val); + } + } +} + +// out = input + input2 * scale +template +inline void add_mul_stub( + scalar_t* __restrict__ out, + const float* __restrict__ input, + const scalar_t* __restrict__ input2, + float scale, + int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + const fVec s_vec = fVec(scale); + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + fVec x0 = fVec::loadu(input + d); + fVec x1 = fVec::loadu(input + d + fVec::size()); + + bVec y_bvec = bVec::loadu(input2 + d); + fVec y0, y1; + std::tie(y0, y1) = at::vec::convert_to_float(y_bvec); + + x0 = x0 + y0 * s_vec; + x1 = x1 + y1 * s_vec; + bVec out_vec = convert_from_float_ext(x0, x1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d] + float(input2[d]) * scale); + } +} + +template +int moe_align_block_size( + int32_t* __restrict__ sorted_ids, + int32_t* __restrict__ expert_ids, + int32_t* __restrict__ topk_ids, + int32_t* __restrict__ total_cnts, + int32_t* __restrict__ cumsums, + int32_t* __restrict__ offsets, + int num_experts, + int numel, + int num_threads) { +#define T_INDEX(tt) total_cnts + (tt) * num_experts + + // accumulate count of expert ids locally + at::parallel_for(0, numel, 0, [&](int begin, int end) { + int tid = at::get_thread_num(); + int32_t* __restrict__ local_cnts = T_INDEX(tid + 1); + + for (int i = begin; i < end; ++i) { + local_cnts[topk_ids[i]]++; + } + }); + + using iVec = at::vec::Vectorized; + for (int t = 0; t < num_threads; ++t) { + at::vec::map2( + [](iVec x, iVec y) { return x + y; }, T_INDEX(t + 1), T_INDEX(t + 1), T_INDEX(t), num_experts); + } + + // the last row holds sums of each experts + int32_t* total_cnts_t_1 = T_INDEX(num_threads); + + cumsums[0] = 0; + for (int e = 0; e < num_experts; ++e) { + // accumulate `num_tokens_post_pad`, also as the expert offset + cumsums[e + 1] = cumsums[e] + div_up(total_cnts_t_1[e], BLOCK_M) * BLOCK_M; + + for (int k = cumsums[e]; k < cumsums[e + 1]; k += BLOCK_M) { + expert_ids[k / BLOCK_M] = e; + } + } + int num_tokens_post_pad = cumsums[num_experts]; + + at::parallel_for(0, numel, 0, [&](int begin, int end) { + int tid = at::get_thread_num(); + // thread tid offsets in `total_cnts` + int32_t* __restrict__ offsets = T_INDEX(tid); + + for (int i = begin; i < end; ++i) { + int32_t expert_id = topk_ids[i]; + int32_t b_offset = cumsums[expert_id]; + int32_t t_offset = offsets[expert_id]; + sorted_ids[b_offset + t_offset] = i; + offsets[expert_id]++; + } + }); + + // debug: the offset for thread t_1 should be identical to t_2 + int32_t* total_cnts_t_2 = T_INDEX(num_threads - 1); + for (int e = 0; e < num_experts; ++e) { + TORCH_CHECK(total_cnts_t_1[e] == total_cnts_t_2[e]); + } + + // padding value for sorted_ids: numel + auto sorted_id_size = [=](const int32_t* sorted_ids_ptr) { + for (int d = 0; d < BLOCK_M; ++d) { + if (sorted_ids_ptr[d] == numel) { + return d; + } + } + return BLOCK_M; + }; + + // offsets holds starting offset for each valida M blocks + // shape : [num_token_blocks + 1] + offsets[0] = 0; + const int num_token_blocks = num_tokens_post_pad / BLOCK_M; + at::parallel_for(0, num_token_blocks, GRAIN_SIZE / BLOCK_M, [&](int begin, int end) { + for (int mb = begin; mb < end; ++mb) { + offsets[mb + 1] = sorted_id_size(sorted_ids + mb * BLOCK_M); + } + }); + // TODO: do we need to vecterize this ? + for (int mb = 0; mb < num_token_blocks; ++mb) { + offsets[mb + 1] += offsets[mb]; + } + // debug: the last value of offsets should be `numel` + TORCH_CHECK(offsets[num_token_blocks] == numel); + + return num_tokens_post_pad; +} + +// silu : shape leading dimension +// input0 [m_size, BLOCK_N] BLOCK_N +// input1 [m_size, BLOCK_N] BLOCK_N +// output [M * topk, N] N +template +inline void silu_and_mul( + scalar_t* __restrict__ output, + const float* __restrict__ input0, // x: x0, x1 + const float* __restrict__ input1, // y: y0, y1 + int64_t m_size, + int64_t N) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + + const fVec one = fVec(1.f); + + // no remainder + for (int64_t m = 0; m < m_size; ++m) { + scalar_t* __restrict__ out = output + m * N; + const float* __restrict__ x = input0 + m * BLOCK_N; + const float* __restrict__ y = input1 + m * BLOCK_N; + + for (int64_t d = 0; d < BLOCK_N; d += bVec::size()) { + fVec x0 = fVec::loadu(x + d); + fVec x1 = fVec::loadu(x + d + fVec::size()); + fVec y0 = fVec::loadu(y + d); + fVec y1 = fVec::loadu(y + d + fVec::size()); + // silu + x0 = x0 / (one + x0.neg().exp_u20()); + x1 = x1 / (one + x1.neg().exp_u20()); + // mul + x0 = x0 * y0; + x1 = x1 * y1; + // convert + bVec out_vec = convert_from_float_ext(x0, x1); + out_vec.store(out + d); + } + } +} + +template +struct tinygemm_kernel_nn2 { + static inline void apply( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B0, + const scalar_t* __restrict__ B1, + scalar_t* __restrict__ C, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!"); + } +}; + +#if defined(CPU_CAPABILITY_AVX512) +template +struct tinygemm_kernel_nn2 { + static inline void apply( + const at::BFloat16* __restrict__ A, + const at::BFloat16* __restrict__ B0, + const at::BFloat16* __restrict__ B1, + at::BFloat16* __restrict__ C, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + constexpr int ROWS = BLOCK_M; + constexpr int COLS = BLOCK_N / 16; + + static_assert(COLS % 2 == 0); + + // prefetch distance + constexpr int PREFETCH_SIZE_K = 0; + + __m512bh va; + __m512bh vb0[COLS]; + __m512bh vb1[COLS]; + __m512 vc0[ROWS * COLS]; + __m512 vc1[ROWS * COLS]; + + auto loadc = [&](auto i) { + vc0[i] = _mm512_set1_ps(0.f); + vc1[i] = _mm512_set1_ps(0.f); + }; + Unroll{}(loadc); + + const int64_t K2 = K >> 1; + const int64_t lda2 = lda >> 1; + const int64_t ldb2 = ldb; // ldb * 2 >> 1; + const float* a_ptr = reinterpret_cast(A); + const float* b0_ptr = reinterpret_cast(B0); + const float* b1_ptr = reinterpret_cast(B1); + + auto compute = [&](auto i, int64_t k) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + va = (__m512bh)(_mm512_set1_ps(a_ptr[row * lda2 + k])); + } + if constexpr (row == 0) { + vb0[col] = (__m512bh)(_mm512_loadu_si512(b0_ptr + k * ldb2 + col * 16)); + vb1[col] = (__m512bh)(_mm512_loadu_si512(b1_ptr + k * ldb2 + col * 16)); + if constexpr (PREFETCH_SIZE_K > 0) { + _mm_prefetch(b0_ptr + (k + PREFETCH_SIZE_K) * ldb2 + col * 16, _MM_HINT_T0); + _mm_prefetch(b1_ptr + (k + PREFETCH_SIZE_K) * ldb2 + col * 16, _MM_HINT_T0); + } + } + vc0[i] = _mm512_dpbf16_ps(vc0[i], va, vb0[col]); + vc1[i] = _mm512_dpbf16_ps(vc1[i], va, vb1[col]); + }; + for (int64_t k = 0; k < K2; ++k) { + Unroll{}(compute, k); + } + + using Vec = at::vec::Vectorized; + const Vec one = Vec(1.f); + auto storec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + // for COLS = 2, 4 use 512bit store + if constexpr (col % 2 == 0) { + Vec x0 = vc0[row * COLS + col + 0]; + Vec x1 = vc0[row * COLS + col + 1]; + Vec y0 = vc1[row * COLS + col + 0]; + Vec y1 = vc1[row * COLS + col + 1]; + // silu + x0 = x0 / (one + x0.neg().exp_u20()); + x1 = x1 / (one + x1.neg().exp_u20()); + // mul + x0 = x0 * y0; + x1 = x1 * y1; + + _mm512_storeu_si512( + reinterpret_cast<__m512i*>((C + row * ldc + col * 16)), + (__m512i)(_mm512_cvtne2ps_pbh(__m512(x1), __m512(x0)))); + } + }; + Unroll{}(storec); + } +}; +#endif + +#define LAUNCH_TINYGEMM_KERNEL_NN(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_nn2::apply( \ + A + mb_start * lda, B0 + nb_start * 2, B1 + nb_start * 2, C + mb_start * ldc + nb_start, K, lda, ldb, ldc); + +template +void tinygemm_kernel( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B0, + const scalar_t* __restrict__ B1, + scalar_t* __restrict__ C, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + // pattern: 1-(2+2)-(8+8) + constexpr int64_t BLOCK_M = 4; + constexpr int64_t BLOCK_N = 32; + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + for (int mb = 0; mb < MB; ++mb) { + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(BLOCK_M, M - mb_start); + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (mb_size << 4 | nb_size >> 4) { + // mb_size = 1 + case 0x12: + LAUNCH_TINYGEMM_KERNEL_NN(1, 32); + break; + // mb_size = 2 + case 0x22: + LAUNCH_TINYGEMM_KERNEL_NN(2, 32); + break; + // mb_size = 3 + case 0x32: + LAUNCH_TINYGEMM_KERNEL_NN(3, 32); + break; + // mb_size = 4 + case 0x42: + LAUNCH_TINYGEMM_KERNEL_NN(4, 32); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size"); + } + } + } +} + +template +struct tinygemm_kernel_nn { + static inline void apply( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B, + float* __restrict__ C, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!"); + } +}; + +#if defined(CPU_CAPABILITY_AVX512) +template +struct tinygemm_kernel_nn { + static inline void apply( + const at::BFloat16* __restrict__ A, + const at::BFloat16* __restrict__ B, + float* __restrict__ C, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + constexpr int ROWS = BLOCK_M; + constexpr int COLS = BLOCK_N / 16; + + static_assert(COLS % 2 == 0); + + // prefetch distance + constexpr int PREFETCH_SIZE_K = 0; + + __m512bh va; + __m512bh vb[COLS]; + __m512 vc[ROWS * COLS]; + + auto loadc = [&](auto i) { vc[i] = _mm512_set1_ps(0.f); }; + Unroll{}(loadc); + + const int64_t K2 = K >> 1; + const int64_t lda2 = lda >> 1; + const int64_t ldb2 = ldb; // ldb * 2 >> 1; + const float* a_ptr = reinterpret_cast(A); + const float* b_ptr = reinterpret_cast(B); + + auto compute = [&](auto i, int64_t k) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + va = (__m512bh)(_mm512_set1_ps(a_ptr[row * lda2 + k])); + } + if constexpr (row == 0) { + vb[col] = (__m512bh)(_mm512_loadu_si512(b_ptr + k * ldb2 + col * 16)); + if constexpr (PREFETCH_SIZE_K > 0) { + _mm_prefetch(b_ptr + (k + PREFETCH_SIZE_K) * ldb2 + col * 16, _MM_HINT_T0); + } + } + vc[i] = _mm512_dpbf16_ps(vc[i], va, vb[col]); + }; + for (int64_t k = 0; k < K2; ++k) { + Unroll{}(compute, k); + } + + auto storec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + _mm512_storeu_ps(reinterpret_cast<__m512*>(C + row * ldc + col * 16), vc[i]); + }; + Unroll{}(storec); + } +}; +#endif + +#define LAUNCH_TINYGEMM_KERNEL_NN2(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_nn::apply( \ + A + mb_start * lda, B + nb_start * 2, C + mb_start * ldc + nb_start, K, lda, ldb, ldc); + +template +void tinygemm_kernel( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B, + float* __restrict__ C, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + // pattern: 1-2-8 + constexpr int64_t BLOCK_M = 4; + constexpr int64_t BLOCK_N = 32; + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + for (int mb = 0; mb < MB; ++mb) { + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(BLOCK_M, M - mb_start); + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (mb_size << 4 | nb_size >> 4) { + // mb_size = 1 + case 0x12: + LAUNCH_TINYGEMM_KERNEL_NN2(1, 32); + break; + // mb_size = 2 + case 0x22: + LAUNCH_TINYGEMM_KERNEL_NN2(2, 32); + break; + // mb_size = 3 + case 0x32: + LAUNCH_TINYGEMM_KERNEL_NN2(3, 32); + break; + // mb_size = 4 + case 0x42: + LAUNCH_TINYGEMM_KERNEL_NN2(4, 32); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size"); + } + } + } +} + +template +void fused_experts_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic1, + scalar_t* __restrict__ ic2, + scalar_t* __restrict__ A_tmp, + float* __restrict__ C_tmp, + const scalar_t* __restrict__ input, + const scalar_t* __restrict__ packed_w1, + const scalar_t* __restrict__ packed_w2, + const float* __restrict__ topk_weights, + const int32_t* __restrict__ sorted_ids, + const int32_t* __restrict__ expert_ids, + const int32_t* __restrict__ offsets, + int64_t M, + int64_t N, + int64_t K, + int64_t E, + int64_t topk, + int64_t num_tokens_post_pad) { + // handle 2 tiles per block + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + + // stage 1: intermediate_cache1 = silu(hidden_states @ w1) + const int64_t MB = div_up(num_tokens_post_pad, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + + // strides for w1: [E, 2N, K] + TORCH_CHECK(N % BLOCK_N == 0, "Fixme when N is not multiples of ", BLOCK_N); + + const int64_t stride_e = 2 * N * K; + const int64_t stride_n = K; + + int64_t avg_M = std::max(int64_t(1), M * topk / E); + const bool use_brgemm = can_use_brgemm(avg_M); + + // here we only parallel on half of 2N to fuse silu_and_mul with gemm + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // get local pointers + int tid = get_thread_num(); + scalar_t* __restrict__ A = A_tmp + tid * BLOCK_M * K; + float* __restrict__ C0 = C_tmp + tid * 2 * BLOCK_M * BLOCK_N; + float* __restrict__ C1 = C0 + BLOCK_M * BLOCK_N; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K * 2, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + // nb_upper from top half and nb_lower from bottom half + int64_t nb_upper = nb, nb_lower = nb + NB; + int64_t n_size = std::min(N - nb * BLOCK_N, BLOCK_N); + + // B shape [K, n_size] in vnni format + int32_t expert_id = expert_ids[mb]; + const scalar_t* __restrict__ B0 = packed_w1 + expert_id * stride_e + nb_upper * BLOCK_N * stride_n; + const scalar_t* __restrict__ B1 = packed_w1 + expert_id * stride_e + nb_lower * BLOCK_N * stride_n; + + // 1.a load A + const int32_t* A_ids = sorted_ids + mb * BLOCK_M; + int64_t m_size = offsets[mb + 1] - offsets[mb]; + + for (int64_t m = 0; m < m_size; ++m) { + int32_t index = A_ids[m] / topk; + copy_stub(A + m * K, input + index * K, K); + } + + if (use_brgemm) { + // 1.b gemm: C0 = A @ B0 + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B0, + /* C */ C0); + + // 1.c gemm: C1 = A @ B1 + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B1, + /* C */ C1); + + // 1.d silu and mul + const int64_t offset = offsets[mb]; + silu_and_mul(ic1 + offset * N + nb * BLOCK_N, C0, C1, m_size, N); + } else { + // fused 1.bcd: silu_and_mul(A @ B0, A @ B1) + const int64_t offset = offsets[mb]; + tinygemm_kernel( + /* A */ A, + /* B0 */ B0, + /* B1 */ B1, + /* C */ ic1 + offset * N + nb * BLOCK_N, + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ N); + } + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 2: intermediate_cache2 = intermediate_cache1 @ w2 + // w2 : [E, K, N] as [E, OC, IC] + const int64_t OC = K; // rename K as OC + const int64_t IC = N; // rename N as IC + const int64_t MB2 = MB; + const int64_t NB2 = div_up(OC, BLOCK_N); + const int64_t stride_e2 = OC * IC; + const int64_t stride_oc = IC; + + // parallel on [MB2, NB2] + parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // get local pointers + int tid = get_thread_num(); + // we won't be using C1 for gemm2 + float* __restrict__ C = C_tmp + tid * 2 * BLOCK_M * BLOCK_N; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + int64_t m_size = offsets[mb + 1] - offsets[mb]; + int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N); + + // A ptr from ic1 of [M * topk, N] in sorted order + // so as to avoid copy A to tmp buffer again + const scalar_t* __restrict__ A = ic1 + offsets[mb] * N; + const int32_t* A_ids = sorted_ids + mb * BLOCK_M; + + // B shape [IC, n_size] in vnni format + int32_t expert_id = expert_ids[mb]; + const scalar_t* __restrict__ B = packed_w2 + expert_id * stride_e2 + nb * BLOCK_N * stride_oc; + + // 2.a gemm: C = A @ B + if (use_brgemm) { + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B, + /* C */ C); + } else { + tinygemm_kernel( + /* A */ A, + /* B */ B, + /* C */ C, + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N); + } + + // 2.b copy from C to ic2 in original order + // and also mul topk_weights in float32 + for (int64_t m = 0; m < m_size; ++m) { + int32_t index = A_ids[m]; + float weight = topk_weights[index]; + copy_mul_stub(ic2 + index * K + nb * BLOCK_N, C + m * BLOCK_N, weight, n_size); + } + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 3: out = intermediate_cache2.sum(dim=1) + // from [M, topk, K] to [M, K] + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + sum_stub(output + m * K, ic2 + m * topk * K, topk, K); + } + }); +} + +template +void shared_expert_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic1, + float* __restrict__ C_tmp, + scalar_t* __restrict__ input, + const scalar_t* __restrict__ packed_w1, + const scalar_t* __restrict__ packed_w2, + const scalar_t* __restrict__ fused_experts_out, + float routed_scaling_factor, + int64_t M, + int64_t N, + int64_t K) { + // handle 2 tiles per block + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + + // stage 1: intermediate_cache1 = silu(hidden_states @ w1) + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + + TORCH_CHECK(N % BLOCK_N == 0, "Fixme when N is not multiples of ", BLOCK_N); + const int64_t stride_n = K; + + const bool use_brgemm = can_use_brgemm(M); + + // here we only parallel on half of 2N to fuse silu_and_mul with gemm + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // get local pointers + int tid = get_thread_num(); + float* __restrict__ C0 = C_tmp + tid * 2 * BLOCK_M * BLOCK_N; + float* __restrict__ C1 = C0 + BLOCK_M * BLOCK_N; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K * 2, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + // nb_upper from top half and nb_lower from bottom half + int64_t nb_upper = nb, nb_lower = nb + NB; + int64_t n_size = std::min(N - nb * BLOCK_N, BLOCK_N); + int64_t m_size = std::min(M - mb * BLOCK_M, BLOCK_M); + + // A shape [m_size, K] + const scalar_t* A = input + mb * BLOCK_M * K; + + // B shape [K, n_size] in vnni format + const scalar_t* __restrict__ B0 = packed_w1 + nb_upper * BLOCK_N * stride_n; + const scalar_t* __restrict__ B1 = packed_w1 + nb_lower * BLOCK_N * stride_n; + + if (use_brgemm) { + // 1.b gemm: C0 = A @ B0 + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B0, + /* C */ C0); + + // 1.c gemm: C1 = A @ B1 + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B1, + /* C */ C1); + + // 1.d silu and mul + silu_and_mul(ic1 + mb * BLOCK_M * N + nb * BLOCK_N, C0, C1, m_size, N); + } else { + // fused 1.bcd: silu_and_mul(A @ B0, A @ B1) + tinygemm_kernel( + /* A */ A, + /* B0 */ B0, + /* B1 */ B1, + /* C */ ic1 + mb * BLOCK_M * N + nb * BLOCK_N, + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ N); + } + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 2: output = intermediate_cache1 @ w2 + // w2 : [K, N] as [OC, IC] + const int64_t OC = K; // rename K as OC + const int64_t IC = N; // rename N as IC + const int64_t MB2 = MB; + const int64_t NB2 = div_up(OC, BLOCK_N); + const int64_t stride_oc = IC; + + // parallel on [MB2, NB2] + parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // get local pointers + int tid = get_thread_num(); + // we won't be using C1 for gemm2 + float* __restrict__ C = C_tmp + tid * 2 * BLOCK_M * BLOCK_N; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + int64_t m_size = std::min(M - mb * BLOCK_M, BLOCK_M); + int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N); + + // A shape [m_size, IC] + const scalar_t* __restrict__ A = ic1 + mb * BLOCK_M * N; + + // B shape [IC, n_size] in vnni format + const scalar_t* __restrict__ B = packed_w2 + nb * BLOCK_N * stride_oc; + + // 2.a gemm: C = A @ B + if (use_brgemm) { + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B, + /* C */ C); + } else { + tinygemm_kernel( + /* A */ A, + /* B */ B, + /* C */ C, + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N); + } + + // 2.b copy from C to output and add fused_experts_out + scalar_t* __restrict__ out = output + mb * BLOCK_M * K + nb * BLOCK_N; + const scalar_t* __restrict__ fused_out = fused_experts_out + mb * BLOCK_M * K + nb * BLOCK_N; + for (int64_t m = 0; m < m_size; ++m) { + add_mul_stub(out + m * K, C + m * BLOCK_N, fused_out + m * K, routed_scaling_factor, n_size); + } + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); +} + +} // anonymous namespace + +// common checks +static inline void check_moe_scales( + bool use_int8_w8a8, + bool use_fp8_w8a16, + const std::optional& w1_scale, + const std::optional& w2_scale, + const std::optional> block_size, + const std::optional& a1_scale, + const std::optional& a2_scale) { + if (use_int8_w8a8) { + TORCH_CHECK(w1_scale.has_value(), "missing w1_scale for int8 w8a8."); + TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for int8 w8a8."); + TORCH_CHECK(!a1_scale.has_value(), "static quantization for activation not supported."); + TORCH_CHECK(!a2_scale.has_value(), "static quantization for activation not supported."); + } + if (use_fp8_w8a16) { + TORCH_CHECK(w1_scale.has_value(), "missing w1_scale for fp8 w8a16."); + TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for fp8 w8a16."); + TORCH_CHECK(block_size.has_value(), "missing block_size for fp8 w8a16."); + TORCH_CHECK(block_size.value().size() == 2, "expect block_size.size() to be 2."); + } +} + +#define CHECK_MOE_SCALES_FP8(DIM0, DIM1) \ + auto w1s = w1_scale.value(); \ + auto w2s = w2_scale.value(); \ + auto block_size_val = block_size.value(); \ + int64_t block_size_N = block_size_val[0]; \ + int64_t block_size_K = block_size_val[1]; \ + TORCH_CHECK(w1s.size(DIM0) == div_up(2 * N, block_size_N)); \ + TORCH_CHECK(w1s.size(DIM1) == div_up(K, block_size_K)); \ + TORCH_CHECK(w2s.size(DIM0) == div_up(K, block_size_N)); \ + TORCH_CHECK(w2s.size(DIM1) == div_up(N, block_size_K)) + +// hidden_states: [M, K] +// w1: [E, 2N, K] +// w2: [E, K, N] +// topk_weights: [M, topk] +// topk_ids: [M, topk] (int32_t) +// +at::Tensor fused_experts_cpu( + at::Tensor& hidden_states, + at::Tensor& w1, + at::Tensor& w2, + at::Tensor& topk_weights, + at::Tensor& topk_ids, + bool inplace, + bool use_int8_w8a8, + bool use_fp8_w8a16, + const std::optional& w1_scale, + const std::optional& w2_scale, + const std::optional> block_size, + const std::optional& a1_scale, + const std::optional& a2_scale, + bool is_vnni) { + RECORD_FUNCTION( + "sgl-kernel::fused_experts_cpu", std::vector({hidden_states, w1, w2, topk_weights, topk_ids})); + + auto packed_w1 = is_vnni ? w1 : convert_weight_packed(w1); + auto packed_w2 = is_vnni ? w2 : convert_weight_packed(w2); + + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + + const auto st = hidden_states.scalar_type(); + CHECK_INPUT(hidden_states); + CHECK_INPUT(w1); + CHECK_INPUT(w2); + CHECK_EQ(topk_weights.sizes(), topk_ids.sizes()); + CHECK_DIM(2, hidden_states); + CHECK_DIM(3, w1); + CHECK_DIM(3, w2); + CHECK_DIM(2, topk_weights); + CHECK_DIM(2, topk_ids); + + CHECK_EQ(topk_ids.scalar_type(), at::kInt); + + // TODO: support topk_weights to be bf16 or fp16 in the kernel. + // The topk_weights of llama4 is computed via Llama4MoE:custom_routing_function and is bf16/fp16 + // while the kernel currently only supports it to be float32 + auto topk_weights_ = topk_weights.to(at::kFloat); + CHECK_EQ(topk_weights_.scalar_type(), at::kFloat); + + int64_t M = hidden_states.size(0); + int64_t K = hidden_states.size(1); + int64_t N = w1.size(1) / 2; + int64_t E = w1.size(0); + int64_t topk = topk_weights_.size(1); + + // we use int32_t compensation for int8 w8a8 + int64_t packed_K = get_row_size(K, use_int8_w8a8); + int64_t packed_N = get_row_size(N, use_int8_w8a8); + + // check weight shapes + CHECK_EQ(w2.size(0), E); + CHECK_EQ(w2.size(1), K); + CHECK_EQ(packed_w1.size(2), packed_K); + CHECK_EQ(packed_w2.size(2), packed_N); + + // check scales + check_moe_scales(use_int8_w8a8, use_fp8_w8a16, w1_scale, w2_scale, block_size, a1_scale, a2_scale); + + at::Tensor out_hidden_states = inplace ? hidden_states : at::empty_like(hidden_states); + + // NB: worst case is each expert holds a block with remainder of 1 + // 1. sorted_ids : [M * topk + E * (BLOCK_M - 1)] + // 2. expert_ids : [max_num_blocks] + // 3. total_cnts : [T + 1, E] + // 4. cumsums : [E + 1] + // 5. offsets : [max_num_blocks + 1] + // + int num_threads = at::get_num_threads(); + int64_t max_num_tokens_padded = M * topk + E * (BLOCK_M - 1); + int64_t max_num_blocks = div_up(max_num_tokens_padded, BLOCK_M); + auto buffer = at::empty( + {max_num_tokens_padded + max_num_blocks + (num_threads + 1) * E + (E + 1) + (max_num_blocks + 1)}, + topk_ids.options()); + + int32_t* __restrict__ sorted_ids = buffer.data_ptr(); + int32_t* __restrict__ expert_ids = sorted_ids + max_num_tokens_padded; + int32_t* __restrict__ total_cnts = expert_ids + max_num_blocks; + int32_t* __restrict__ cumsums = total_cnts + (num_threads + 1) * E; + int32_t* __restrict__ offsets = cumsums + (E + 1); + + // init sorted_ids with `numel` as the padding number + // init expert_ids with `num_experts` + int64_t numel = M * topk; + at::parallel_for(0, max_num_blocks, GRAIN_SIZE / BLOCK_M, [&](int64_t begin, int64_t end) { + int64_t m_start = begin * BLOCK_M; + int64_t m_size = std::min((end - begin) * BLOCK_M, max_num_tokens_padded - m_start); + fill_stub(sorted_ids + m_start, (int32_t)numel, m_size); + fill_stub(expert_ids + begin, (int32_t)E, end - begin); + }); + // zero total_cnts and cumsums + at::parallel_for(0, (num_threads + 1) * E + (E + 1), GRAIN_SIZE, [&](int64_t begin, int64_t end) { + fill_stub(total_cnts + begin, 0, end - begin); + }); + + // align experts index + int64_t num_tokens_post_pad = moe_align_block_size( + sorted_ids, expert_ids, topk_ids.data_ptr(), total_cnts, cumsums, offsets, E, numel, num_threads); + + // unlike triton kernel, we fuse silu with gemm1 so only need 2 intermediate_caches: + // 1. intermediate_cache1 : [M * topk, N] + // 2. intermediate_cache2 : [M * topk, K] + // 3. A_tmp : [T, BLOCK_M * K] + // 4. C_tmp : [T, 2 * BLOCK_M * BLOCK_N] + // + // for int8 w8a8: + // 5. Aq_tmp : [M, K] or [M * topk, N] + // 6. As_tmp : [M * topk] + // + // for fp8 w8a16: + // 7. intermediate_cache0 : [M * topk, 2N] + // 8. B_tmp : [T, MAX_CACHE_BLOCK_SIZE, BLOCK_N, std::max(K, N)] + // + int64_t buffer_size_nbytes = M * topk * N * 2 + M * topk * K * 2 + + num_threads * BLOCK_M * K * (use_int8_w8a8 ? 1 : 2) + + num_threads * 2 * BLOCK_M * BLOCK_N * sizeof(float); + + if (use_int8_w8a8) { + buffer_size_nbytes += std::max(M * K, M * topk * N) + M * topk * sizeof(float); + } + if (use_fp8_w8a16) { + buffer_size_nbytes += M * topk * 2 * N * 2 + num_threads * MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N) * 2; + } + + auto buffer2 = at::empty({buffer_size_nbytes}, hidden_states.options().dtype(at::kChar)); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(st, "fused_experts_kernel_impl", [&] { + scalar_t* __restrict__ intermediate_cache1 = (scalar_t*)((void*)(buffer2.data_ptr())); + scalar_t* __restrict__ intermediate_cache2 = intermediate_cache1 + M * topk * N; + + if (use_int8_w8a8) { + uint8_t* __restrict__ A_tmp = (uint8_t*)((void*)(intermediate_cache2 + M * topk * K)); + float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K)); + uint8_t* __restrict__ Aq_tmp = (uint8_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N)); + float* __restrict__ As_tmp = (float*)((void*)(Aq_tmp + std::max(M * K, M * topk * N))); + + auto w1s = w1_scale.value(); + auto w2s = w2_scale.value(); + TORCH_CHECK(w1s.numel() == E * 2 * N); + TORCH_CHECK(w2s.numel() == E * K); + + fused_experts_int8_kernel_impl( + out_hidden_states.data_ptr(), + intermediate_cache1, + intermediate_cache2, + A_tmp, + C_tmp, + Aq_tmp, + As_tmp, + hidden_states.data_ptr(), + packed_w1.data_ptr(), + packed_w2.data_ptr(), + w1s.data_ptr(), + w2s.data_ptr(), + topk_weights_.data_ptr(), + sorted_ids, + expert_ids, + offsets, + M, + N, + K, + E, + topk, + num_tokens_post_pad); + } else if (use_fp8_w8a16) { + // here we just ignore C_tmp as it is not used + scalar_t* __restrict__ A_tmp = (scalar_t*)((void*)(intermediate_cache2 + M * topk * K)); + float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K)); + scalar_t* __restrict__ intermediate_cache0 = (scalar_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N)); + scalar_t* __restrict__ B_tmp = (scalar_t*)((void*)(intermediate_cache0 + M * topk * 2 * N)); + + CHECK_MOE_SCALES_FP8(1, 2); + fused_experts_fp8_kernel_impl( + out_hidden_states.data_ptr(), + intermediate_cache0, + intermediate_cache1, + intermediate_cache2, + A_tmp, + B_tmp, + C_tmp, + hidden_states.data_ptr(), + packed_w1.data_ptr(), + packed_w2.data_ptr(), + w1s.data_ptr(), + w2s.data_ptr(), + block_size_N, + block_size_K, + topk_weights_.data_ptr(), + sorted_ids, + expert_ids, + offsets, + M, + N, + K, + E, + topk, + num_tokens_post_pad); + } else { + scalar_t* __restrict__ A_tmp = intermediate_cache2 + M * topk * K; + float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K)); + + fused_experts_kernel_impl( + out_hidden_states.data_ptr(), + intermediate_cache1, + intermediate_cache2, + A_tmp, + C_tmp, + hidden_states.data_ptr(), + packed_w1.data_ptr(), + packed_w2.data_ptr(), + topk_weights_.data_ptr(), + sorted_ids, + expert_ids, + offsets, + M, + N, + K, + E, + topk, + num_tokens_post_pad); + } + }); + return out_hidden_states; +} + +// shared expert kernel +// +// hidden_states: [M, K] +// w1: [2N, K] +// w2: [K, N] +// fused_experts_out +at::Tensor shared_expert_cpu( + at::Tensor& hidden_states, + at::Tensor& w1, + at::Tensor& w2, + at::Tensor& fused_experts_out, + double routed_scaling_factor, + bool inplace, + bool use_int8_w8a8, + bool use_fp8_w8a16, + const std::optional& w1_scale, + const std::optional& w2_scale, + const std::optional> block_size, + const std::optional& a1_scale, + const std::optional& a2_scale, + bool is_vnni) { + RECORD_FUNCTION("sgl-kernel::shared_expert_cpu", std::vector({hidden_states, w1, w2})); + + auto packed_w1 = is_vnni ? w1 : convert_weight_packed(w1); + auto packed_w2 = is_vnni ? w2 : convert_weight_packed(w2); + + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + + const auto st = hidden_states.scalar_type(); + CHECK_INPUT(hidden_states); + CHECK_INPUT(fused_experts_out); + CHECK_INPUT(w1); + CHECK_INPUT(w2); + CHECK_DIM(2, hidden_states); + CHECK_DIM(2, w1); + CHECK_DIM(2, w2); + CHECK_EQ(hidden_states.sizes(), fused_experts_out.sizes()); + CHECK_EQ(hidden_states.scalar_type(), st); + + int64_t M = hidden_states.size(0); + int64_t K = hidden_states.size(1); + int64_t N = w1.size(0) / 2; + + // we use int32_t compensation for int8 w8a8 + int64_t packed_K = get_row_size(K, use_int8_w8a8); + int64_t packed_N = get_row_size(N, use_int8_w8a8); + + // check weight shapes + CHECK_EQ(w2.size(0), K); + CHECK_EQ(packed_w1.size(1), packed_K); + CHECK_EQ(packed_w2.size(1), packed_N); + + // check scales + check_moe_scales(use_int8_w8a8, use_fp8_w8a16, w1_scale, w2_scale, block_size, a1_scale, a2_scale); + + at::Tensor out_hidden_states = inplace ? hidden_states : at::empty_like(hidden_states); + + // unlike triton kernel, we fuse silu with gemm1 so only need 2 intermediate_caches: + // 1. intermediate_cache1 : [M, N] + // 2. C_tmp : [T, 2 * BLOCK_M * BLOCK_N] + // + // for int8 w8a8: + // 3. Aq_tmp : [M, K] or [M, N] + // 4. As_tmp : [M] + // + // for fp8 w8a16: + // 5. intermediate_cache0 : [M, 2N] + // 6. B_tmp: [T, MAX_CACHE_BLOCK_SIZE, BLOCK_M, max(K, N)] + // + int num_threads = at::get_num_threads(); + int64_t buffer_size_nbytes = M * N * 2 + num_threads * 2 * BLOCK_M * BLOCK_N * sizeof(float); + + if (use_int8_w8a8) { + buffer_size_nbytes += std::max(M * K, M * N) + M * sizeof(float); + } + if (use_fp8_w8a16) { + buffer_size_nbytes += M * 2 * N * 2 + num_threads * MAX_CACHE_BLOCK_SIZE * BLOCK_M * std::max(K, N) * 2; + } + + auto buffer = at::empty({buffer_size_nbytes}, hidden_states.options().dtype(at::kChar)); + AT_DISPATCH_REDUCED_FLOATING_TYPES(st, "share_experts_kernel_impl", [&] { + scalar_t* __restrict__ intermediate_cache1 = (scalar_t*)((void*)(buffer.data_ptr())); + float* __restrict__ C_tmp = (float*)((void*)(intermediate_cache1 + M * N)); + + if (use_int8_w8a8) { + uint8_t* __restrict__ Aq_tmp = (uint8_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N)); + float* __restrict__ As_tmp = (float*)((void*)(Aq_tmp + std::max(M * K, M * N))); + + auto w1s = w1_scale.value(); + auto w2s = w2_scale.value(); + TORCH_CHECK(w1s.numel() == 2 * N); + TORCH_CHECK(w2s.numel() == K); + + shared_expert_int8_kernel_impl( + out_hidden_states.data_ptr(), + intermediate_cache1, + C_tmp, + Aq_tmp, + As_tmp, + hidden_states.data_ptr(), + packed_w1.data_ptr(), + packed_w2.data_ptr(), + w1s.data_ptr(), + w2s.data_ptr(), + fused_experts_out.data_ptr(), + routed_scaling_factor, + M, + N, + K); + } else if (use_fp8_w8a16) { + scalar_t* __restrict__ intermediate_cache0 = (scalar_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N)); + scalar_t* __restrict__ B_tmp = (scalar_t*)((void*)(intermediate_cache0 + M * 2 * N)); + + CHECK_MOE_SCALES_FP8(0, 1); + shared_expert_fp8_kernel_impl( + out_hidden_states.data_ptr(), + intermediate_cache0, + intermediate_cache1, + B_tmp, + C_tmp, + hidden_states.data_ptr(), + packed_w1.data_ptr(), + packed_w2.data_ptr(), + w1s.data_ptr(), + w2s.data_ptr(), + block_size_N, + block_size_K, + fused_experts_out.data_ptr(), + routed_scaling_factor, + M, + N, + K); + } else { + shared_expert_kernel_impl( + out_hidden_states.data_ptr(), + intermediate_cache1, + C_tmp, + hidden_states.data_ptr(), + packed_w1.data_ptr(), + packed_w2.data_ptr(), + fused_experts_out.data_ptr(), + routed_scaling_factor, + M, + N, + K); + } + }); + return out_hidden_states; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/moe_fp8.cpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/moe_fp8.cpp new file mode 100644 index 00000000..281c0089 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/moe_fp8.cpp @@ -0,0 +1,491 @@ +#include "common.h" +#include "gemm.h" +#include "vec.h" + +namespace { + +template +inline void copy_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t size) { + using Vec = at::vec::Vectorized; +// no remainder +#pragma GCC unroll 4 + for (int64_t d = 0; d < size; d += Vec::size()) { + Vec data = Vec::loadu(input + d); + data.store(out + d); + } +} + +template +inline void copy_mul_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, float weight, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + const fVec weight_vec = fVec(weight); + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + bVec x = bVec::loadu(input + d); + fVec x0, x1; + std::tie(x0, x1) = at::vec::convert_to_float(x); + x0 = x0 * weight_vec; + x1 = x1 * weight_vec; + bVec out_vec = convert_from_float_ext(x0, x1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d] * weight); + } +} + +// acc from [topk, K] to [K] +template +inline void sum_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t topk, int64_t K) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + if (topk == 1) { + // do copy for topk = 1 + copy_stub(out, input, K); + } else { + // do sum for topk != 1 + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= K - kVecSize; d += kVecSize) { + fVec sum_fvec0 = fVec(0.f); + fVec sum_fvec1 = fVec(0.f); + for (int t = 0; t < topk; ++t) { + bVec x_bvec = bVec::loadu(input + t * K + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + sum_fvec0 += x_fvec0; + sum_fvec1 += x_fvec1; + } + bVec out_bvec = convert_from_float_ext(sum_fvec0, sum_fvec1); + out_bvec.store(out + d); + } + for (; d < K; ++d) { + float sum_val = 0.f; + for (int t = 0; t < topk; ++t) { + sum_val += static_cast(input[t * K + d]); + } + out[d] = static_cast(sum_val); + } + } +} + +// out = input + input2 * scale +template +inline void add_mul_stub( + scalar_t* __restrict__ out, + const scalar_t* __restrict__ input, + const scalar_t* __restrict__ input2, + float scale, + int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + const fVec s_vec = fVec(scale); + + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + bVec x_bvec = bVec::loadu(input + d); + fVec x0, x1; + std::tie(x0, x1) = at::vec::convert_to_float(x_bvec); + + bVec y_bvec = bVec::loadu(input2 + d); + fVec y0, y1; + std::tie(y0, y1) = at::vec::convert_to_float(y_bvec); + + x0 = x0 + y0 * s_vec; + x1 = x1 + y1 * s_vec; + bVec out_vec = convert_from_float_ext(x0, x1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d] + float(input2[d]) * scale); + } +} + +template +inline void silu_and_mul_stub( + scalar_t* __restrict__ out, const scalar_t* __restrict__ input, const scalar_t* __restrict__ input2, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + const fVec one = fVec(1.f); + + // no remainder +#pragma GCC unroll 4 + for (int64_t d = 0; d < size; d += bVec::size()) { + bVec x = bVec::loadu(input + d); + fVec x0, x1; + std::tie(x0, x1) = at::vec::convert_to_float(x); + bVec y = bVec::loadu(input2 + d); + fVec y0, y1; + std::tie(y0, y1) = at::vec::convert_to_float(y); + x0 = x0 / (one + x0.neg().exp_u20()); + x1 = x1 / (one + x1.neg().exp_u20()); + x0 = x0 * y0; + x1 = x1 * y1; + bVec out_vec = convert_from_float_ext(x0, x1); + out_vec.store(out + d); + } +} + +} // anonymous namespace + +template +void fused_experts_fp8_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic0, + scalar_t* __restrict__ ic1, + scalar_t* __restrict__ ic2, + scalar_t* __restrict__ A_tmp, + scalar_t* __restrict__ B_tmp, + float* __restrict__ C_tmp, + const scalar_t* __restrict__ input, + const at::Float8_e4m3fn* __restrict__ packed_w1, + const at::Float8_e4m3fn* __restrict__ packed_w2, + const float* __restrict__ w1s, + const float* __restrict__ w2s, + int64_t block_size_N, + int64_t block_size_K, + const float* __restrict__ topk_weights, + const int32_t* __restrict__ sorted_ids, + const int32_t* __restrict__ expert_ids, + const int32_t* __restrict__ offsets, + int64_t M, + int64_t N, + int64_t K, + int64_t E, + int64_t topk, + int64_t num_tokens_post_pad) { + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + + // stage 1: intermediate_cache0 = hidden_states @ w1 + const int64_t MB = div_up(num_tokens_post_pad, BLOCK_M); + const int64_t NB = div_up(2 * N, BLOCK_N); + int64_t scale_size_N = div_up(2 * N, block_size_N); + int64_t scale_size_K = div_up(K, block_size_K); + int64_t blocks_n_per_group = block_size_N / BLOCK_N; + + const int64_t stride_e = 2 * N * K; + const int64_t stride_n = K; + + int64_t avg_M = std::max(int64_t(1), M * topk / E); + const bool use_brgemm = can_use_brgemm(avg_M); + + int64_t B_tmp_size_per_thread = MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N); + + // here we only parallel on half of 2N to fuse silu_and_mul with gemm + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // get local pointers + int tid = get_thread_num(); + scalar_t* __restrict__ A = A_tmp + tid * BLOCK_M * K; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + int64_t n_size = std::min(2 * N - nb * BLOCK_N, BLOCK_N); + + // B shape [K, n_size] in vnni format + int32_t expert_id = expert_ids[mb]; + const at::Float8_e4m3fn* __restrict__ B = packed_w1 + expert_id * stride_e + nb * BLOCK_N * stride_n; + const float* __restrict__ Bs = + w1s + expert_id * scale_size_N * scale_size_K + (nb / blocks_n_per_group) * scale_size_K; + + // do unpacking for the first row or a new expert + int32_t pre_expert_id = mb == 0 ? -1 : expert_ids[mb - 1]; + bool do_unpack = (mb == mb0) || (expert_id != pre_expert_id); + + // 1.a load A + const int32_t* A_ids = sorted_ids + mb * BLOCK_M; + int64_t m_size = offsets[mb + 1] - offsets[mb]; + + for (int64_t m = 0; m < m_size; ++m) { + int32_t index = A_ids[m] / topk; + copy_stub(A + m * K, input + index * K, K); + } + + const int64_t offset = offsets[mb]; + tinygemm_kernel( + /* A */ A, + /* B */ B, + /* C */ ic0 + offset * 2 * N + nb * BLOCK_N, + /* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * K, + /* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N, + /* scale */ Bs, + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ 2 * N, + /* brg */ use_brgemm, + /* block_size_K */ block_size_K, + /* do_unpack */ do_unpack); + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 1.5: intermediate_cache1 = silu(intermediate_cache0) + at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + silu_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, ic0 + m * 2 * N + N, N); + } + }); + + // stage 2: intermediate_cache2 = intermediate_cache1 @ w2 + // w2 : [E, K, N] as [E, OC, IC] + const int64_t OC = K; // rename K as OC + const int64_t IC = N; // rename N as IC + const int64_t MB2 = MB; + const int64_t NB2 = div_up(OC, BLOCK_N); + scale_size_N = div_up(K, block_size_N); + scale_size_K = div_up(N, block_size_K); + const int64_t stride_e2 = OC * IC; + const int64_t stride_oc = IC; + + // parallel on [MB2, NB2] + parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + int tid = get_thread_num(); + alignas(64) scalar_t C[BLOCK_M * BLOCK_K]; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + int64_t m_size = offsets[mb + 1] - offsets[mb]; + int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N); + + // A ptr from ic1 of [M * topk, N] in sorted order + // so as to avoid copy A to tmp buffer again + const scalar_t* __restrict__ A = ic1 + offsets[mb] * N; + const int32_t* A_ids = sorted_ids + mb * BLOCK_M; + + // B shape [IC, n_size] in vnni format + int32_t expert_id = expert_ids[mb]; + const at::Float8_e4m3fn* __restrict__ B = packed_w2 + expert_id * stride_e2 + nb * BLOCK_N * stride_oc; + const float* __restrict__ Bs = + w2s + expert_id * scale_size_N * scale_size_K + (nb / blocks_n_per_group) * scale_size_K; + + // do unpacking for the first row or a new expert + int32_t pre_expert_id = mb == 0 ? -1 : expert_ids[mb - 1]; + bool do_unpack = (mb == mb0) || (expert_id != pre_expert_id); + + tinygemm_kernel( + /* A */ A, + /* B */ B, + /* C */ C, + /* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * IC, + /* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N, + /* scale */ Bs, + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* brg */ use_brgemm, + /* block_size_K */ block_size_K, + /* do_unpack */ do_unpack); + + // 2.b copy from C to ic2 in original order + // and also mul topk_weights in float32 + for (int64_t m = 0; m < m_size; ++m) { + int32_t index = A_ids[m]; + float weight = topk_weights[index]; + copy_mul_stub(ic2 + index * K + nb * BLOCK_N, C + m * BLOCK_N, weight, n_size); + } + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 3: out = intermediate_cache2.sum(dim=1) + // from [M, topk, K] to [M, K] + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + sum_stub(output + m * K, ic2 + m * topk * K, topk, K); + } + }); +} + +#define INSTANTIATE_MOE_FP8_TEMPLATE(TYPE) \ + template void fused_experts_fp8_kernel_impl( \ + TYPE* __restrict__ output, \ + TYPE* __restrict__ ic0, \ + TYPE* __restrict__ ic1, \ + TYPE* __restrict__ ic2, \ + TYPE* __restrict__ A_tmp, \ + TYPE* __restrict__ B_tmp, \ + float* __restrict__ C_tmp, \ + const TYPE* __restrict__ input, \ + const at::Float8_e4m3fn* __restrict__ packed_w1, \ + const at::Float8_e4m3fn* __restrict__ packed_w2, \ + const float* __restrict__ w1s, \ + const float* __restrict__ w2s, \ + int64_t block_size_N, \ + int64_t block_size_K, \ + const float* __restrict__ topk_weights, \ + const int32_t* __restrict__ sorted_ids, \ + const int32_t* __restrict__ expert_ids, \ + const int32_t* __restrict__ offsets, \ + int64_t M, \ + int64_t N, \ + int64_t K, \ + int64_t E, \ + int64_t topk, \ + int64_t num_tokens_post_pad) + +INSTANTIATE_MOE_FP8_TEMPLATE(at::BFloat16); +INSTANTIATE_MOE_FP8_TEMPLATE(at::Half); + +template +void shared_expert_fp8_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic0, + scalar_t* __restrict__ ic1, + scalar_t* __restrict__ B_tmp, + float* __restrict__ C_tmp, + const scalar_t* __restrict__ input, + const at::Float8_e4m3fn* __restrict__ packed_w1, + const at::Float8_e4m3fn* __restrict__ packed_w2, + const float* __restrict__ w1s, + const float* __restrict__ w2s, + int64_t block_size_N, + int64_t block_size_K, + const scalar_t* __restrict__ fused_experts_out, + float routed_scaling_factor, + int64_t M, + int64_t N, + int64_t K) { + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + + // stage 1: intermediate_cache0 = hidden_states @ w1 + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(2 * N, BLOCK_N); + int64_t scale_size_K = div_up(K, block_size_K); + int64_t blocks_n_per_group = block_size_N / BLOCK_N; + + const bool use_brgemm = can_use_brgemm(M); + + int64_t B_tmp_size_per_thread = MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N); + + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + int tid = get_thread_num(); + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + int64_t m_size = std::min(M - mb * BLOCK_M, BLOCK_M); + int64_t n_size = std::min(2 * N - nb * BLOCK_N, BLOCK_N); + + // do unpacking for the first row + bool do_unpack = (mb == mb0); + + tinygemm_kernel( + /* A */ input + mb * BLOCK_M * K, + /* B */ packed_w1 + nb * BLOCK_N * K, + /* C */ ic0 + mb * BLOCK_M * 2 * N + nb * BLOCK_N, + /* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * K, + /* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N, + /* scale */ w1s + (nb / blocks_n_per_group) * scale_size_K, + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ 2 * N, + /* brg */ use_brgemm, + /* block_size_K */ block_size_K, + /* do_unpack */ do_unpack); + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 1.5: intermediate_cache1 = silu(intermediate_cache0) + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + silu_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, ic0 + m * 2 * N + N, N); + } + }); + + // stage 2: intermediate_cache2 = intermediate_cache1 @ w2 + // w2 : [K, N] as [OC, IC] + const int64_t OC = K; // rename K as OC + const int64_t IC = N; // rename N as IC + const int64_t MB2 = MB; + const int64_t NB2 = div_up(K, BLOCK_N); + scale_size_K = div_up(N, block_size_K); + + // parallel on [MB2, NB2] + parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + int tid = get_thread_num(); + alignas(64) scalar_t C[BLOCK_M * BLOCK_K]; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + int64_t m_size = std::min(M - mb * BLOCK_M, BLOCK_M); + int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N); + + // do unpacking for the first row + bool do_unpack = (mb == mb0); + + // 2.a gemm: C = A @ B + tinygemm_kernel( + /* A */ ic1 + mb * BLOCK_M * N, + /* B */ packed_w2 + nb * BLOCK_N * N, + /* C */ C, + /* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * IC, + /* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N, + /* scale */ w2s + (nb / blocks_n_per_group) * scale_size_K, + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* brg */ use_brgemm, + /* block_size_K */ block_size_K, + /* do_unpack */ do_unpack); + + // 2.b copy from C to output and add fused_experts_out + scalar_t* __restrict__ out = output + mb * BLOCK_M * K + nb * BLOCK_N; + const scalar_t* __restrict__ fused_out = fused_experts_out + mb * BLOCK_M * K + nb * BLOCK_N; + for (int64_t m = 0; m < m_size; ++m) { + add_mul_stub(out + m * K, C + m * BLOCK_N, fused_out + m * K, routed_scaling_factor, n_size); + } + }); + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } +} + +#define INSTANTIATE_SHARED_EXPERT_FP8_TEMPLATE(TYPE) \ + template void shared_expert_fp8_kernel_impl( \ + TYPE* __restrict__ output, \ + TYPE* __restrict__ ic0, \ + TYPE* __restrict__ ic1, \ + TYPE* __restrict__ B_tmp, \ + float* __restrict__ C_tmp, \ + const TYPE* __restrict__ input, \ + const at::Float8_e4m3fn* __restrict__ packed_w1, \ + const at::Float8_e4m3fn* __restrict__ packed_w2, \ + const float* __restrict__ w1s, \ + const float* __restrict__ w2s, \ + int64_t block_size_N, \ + int64_t block_size_K, \ + const TYPE* __restrict__ fused_experts_out, \ + float routed_scaling_factor, \ + int64_t M, \ + int64_t N, \ + int64_t K) + +INSTANTIATE_SHARED_EXPERT_FP8_TEMPLATE(at::BFloat16); +INSTANTIATE_SHARED_EXPERT_FP8_TEMPLATE(at::Half); diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/moe_int8.cpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/moe_int8.cpp new file mode 100644 index 00000000..8fbac902 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/moe_int8.cpp @@ -0,0 +1,1068 @@ +#include "common.h" +#include "gemm.h" +#include "vec.h" + +namespace { + +template +inline void copy_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t size) { + using Vec = at::vec::Vectorized; +// no remainder +#pragma GCC unroll 4 + for (int64_t d = 0; d < size; d += Vec::size()) { + Vec data = Vec::loadu(input + d); + data.store(out + d); + } +} + +template <> +inline void copy_stub(uint8_t* __restrict__ out, const uint8_t* __restrict__ input, int64_t size) { + // size might be 64x + 32 + std::memcpy(out, input, size * sizeof(uint8_t)); +} + +template +inline void copy_mul_stub(scalar_t* __restrict__ out, const float* __restrict__ input, float weight, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + const fVec weight_vec = fVec(weight); + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + fVec data0 = fVec::loadu(input + d) * weight_vec; + fVec data1 = fVec::loadu(input + d + fVec::size()) * weight_vec; + bVec out_vec = convert_from_float_ext(data0, data1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d] * weight); + } +} + +// acc from [topk, K] to [K] +template +inline void sum_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t topk, int64_t K) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + if (topk == 1) { + // do copy for topk = 1 + copy_stub(out, input, K); + } else { + // do sum for topk != 1 + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= K - kVecSize; d += kVecSize) { + fVec sum_fvec0 = fVec(0.f); + fVec sum_fvec1 = fVec(0.f); + for (int t = 0; t < topk; ++t) { + bVec x_bvec = bVec::loadu(input + t * K + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + sum_fvec0 += x_fvec0; + sum_fvec1 += x_fvec1; + } + bVec out_bvec = convert_from_float_ext(sum_fvec0, sum_fvec1); + out_bvec.store(out + d); + } + for (; d < K; ++d) { + float sum_val = 0.f; + for (int t = 0; t < topk; ++t) { + sum_val += static_cast(input[t * K + d]); + } + out[d] = static_cast(sum_val); + } + } +} + +// out = input + input2 * scale +template +inline void add_mul_stub( + scalar_t* __restrict__ out, + const float* __restrict__ input, + const scalar_t* __restrict__ input2, + float scale, + int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + const fVec s_vec = fVec(scale); + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + fVec x0 = fVec::loadu(input + d); + fVec x1 = fVec::loadu(input + d + fVec::size()); + + bVec y_bvec = bVec::loadu(input2 + d); + fVec y0, y1; + std::tie(y0, y1) = at::vec::convert_to_float(y_bvec); + + x0 = x0 + y0 * s_vec; + x1 = x1 + y1 * s_vec; + bVec out_vec = convert_from_float_ext(x0, x1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d] + float(input2[d]) * scale); + } +} + +template +inline void silu_and_mul( + scalar_t* __restrict__ C, + const int32_t* __restrict__ C0, // x: x0, x1 + const int32_t* __restrict__ C1, // y: y0, y1 + const float* __restrict__ As, + const float* __restrict__ Bs0, + const float* __restrict__ Bs1, + const int32_t* __restrict__ Bcomp0, + const int32_t* __restrict__ Bcomp1, + int64_t m_size, + int64_t N) { +#if defined(CPU_CAPABILITY_AVX512) + constexpr int COLS = BLOCK_N / 16; + static_assert(COLS % 2 == 0); + + __m512 vc0[COLS]; + __m512 vc1[COLS]; + __m512i vcomp0[COLS]; + __m512i vcomp1[COLS]; + __m512 vas; + __m512 vbs0[COLS]; + __m512 vbs1[COLS]; + + auto load_scale_and_comp = [&](auto col) { + vcomp0[col] = _mm512_loadu_si512(Bcomp0 + col * 16); + vcomp1[col] = _mm512_loadu_si512(Bcomp1 + col * 16); + vbs0[col] = _mm512_loadu_ps(Bs0 + col * 16); + vbs1[col] = _mm512_loadu_ps(Bs1 + col * 16); + }; + Unroll{}(load_scale_and_comp); + + auto scalec = [&](auto col, int64_t m) { + // update As + vas = _mm512_set1_ps(As[m]); + // C = As * (C - Bcomp) * Bs + __m512i vc32_0 = _mm512_loadu_si512(C0 + m * BLOCK_N + col * 16); + __m512i vc32_1 = _mm512_loadu_si512(C1 + m * BLOCK_N + col * 16); + vc0[col] = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc32_0, vcomp0[col])); + vc1[col] = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc32_1, vcomp1[col])); + vc0[col] = _mm512_mul_ps(_mm512_mul_ps(vc0[col], vas), vbs0[col]); + vc1[col] = _mm512_mul_ps(_mm512_mul_ps(vc1[col], vas), vbs1[col]); + }; + + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + const fVec one = fVec(1.f); + auto silu_and_mul = [&](auto col) { + fVec x = fVec(vc0[col]); + fVec y = fVec(vc1[col]); + x = x / (one + x.neg().exp_u20()); + vc0[col] = x * y; + }; + + auto storec = [&](auto col, int64_t m) { + if constexpr (col % 2 == 0) { + fVec x0 = fVec(vc0[col + 0]); + fVec x1 = fVec(vc0[col + 1]); + bVec out_vec = convert_from_float_ext(x0, x1); + out_vec.store(C + m * N + col * 16); + } + }; + + for (int64_t m = 0; m < m_size; ++m) { + Unroll{}(scalec, m); + Unroll{}(silu_and_mul); + Unroll{}(storec, m); + } +#else + TORCH_CHECK(false, "silu_and_mul: scalar path not implemented!"); +#endif +} + +template +inline void scale_C( + float* __restrict__ C, + const int32_t* __restrict__ Ctmp, + const float* __restrict__ As, + const float* __restrict__ Bs, + const int32_t* __restrict__ Bcomp, + int64_t m_size) { +#if defined(CPU_CAPABILITY_AVX512) + constexpr int COLS = BLOCK_N / 16; + static_assert(COLS % 2 == 0); + + __m512 vc[COLS]; + __m512i vcomp[COLS]; + __m512 vas; + __m512 vbs[COLS]; + + auto load_scale_and_comp = [&](auto col) { + vcomp[col] = _mm512_loadu_si512(Bcomp + col * 16); + vbs[col] = _mm512_loadu_ps(Bs + col * 16); + }; + Unroll{}(load_scale_and_comp); + + auto scalec = [&](auto col, int64_t m) { + // update As + vas = _mm512_set1_ps(As[m]); + // C = As * (C - Bcomp) * Bs + __m512i vc32 = _mm512_loadu_si512(Ctmp + m * BLOCK_N + col * 16); + vc[col] = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc32, vcomp[col])); + vc[col] = _mm512_mul_ps(_mm512_mul_ps(vc[col], vas), vbs[col]); + _mm512_storeu_ps(C + m * BLOCK_N + col * 16, vc[col]); + }; + + for (int64_t m = 0; m < m_size; ++m) { + Unroll{}(scalec, m); + } +#else + TORCH_CHECK(false, "scale_C: scalar path not implemented!"); +#endif +} + +/// gemm for w13 +template +struct tinygemm_kernel_vnni { + static inline void apply( + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B0, + const int8_t* __restrict__ B1, + scalar_t* __restrict__ C, + const float* __restrict__ As, + const float* __restrict__ Bs0, + const float* __restrict__ Bs1, + const int32_t* __restrict__ Bcomp0, + const int32_t* __restrict__ Bcomp1, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!"); + } +}; + +#if defined(CPU_CAPABILITY_AVX512) +template +struct tinygemm_kernel_vnni { + static inline void apply( + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B0, + const int8_t* __restrict__ B1, + at::BFloat16* __restrict__ C, + const float* __restrict__ As, + const float* __restrict__ Bs0, + const float* __restrict__ Bs1, + const int32_t* __restrict__ Bcomp0, + const int32_t* __restrict__ Bcomp1, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + constexpr int ROWS = BLOCK_M; + constexpr int COLS = BLOCK_N / 16; + static_assert(COLS % 2 == 0); + + __m512i va; + __m512i vb0[COLS]; + __m512i vb1[COLS]; + __m512i vc0[ROWS * COLS]; + __m512i vc1[ROWS * COLS]; + __m512i vcomp0[COLS]; + __m512i vcomp1[COLS]; + __m512 vas; + __m512 vbs0[COLS]; + __m512 vbs1[COLS]; + + auto loadc = [&](auto i) { + vc0[i] = _mm512_set1_epi32(0); + vc1[i] = _mm512_set1_epi32(0); + }; + Unroll{}(loadc); + + const int64_t K4 = K >> 2; + const int64_t lda4 = lda >> 2; + const int64_t ldb4 = ldb; // ldb * 4 >> 2; + const int32_t* a_ptr = reinterpret_cast(A); + const int32_t* b0_ptr = reinterpret_cast(B0); + const int32_t* b1_ptr = reinterpret_cast(B1); + + auto compute = [&](auto i, int64_t k) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + va = _mm512_set1_epi32(a_ptr[row * lda4 + k]); + } + if constexpr (row == 0) { + vb0[col] = _mm512_loadu_si512(b0_ptr + k * ldb4 + col * 16); + vb1[col] = _mm512_loadu_si512(b1_ptr + k * ldb4 + col * 16); + } + vc0[i] = _mm512_dpbusd_epi32(vc0[i], va, vb0[col]); + vc1[i] = _mm512_dpbusd_epi32(vc1[i], va, vb1[col]); + }; + for (int64_t k = 0; k < K4; ++k) { + Unroll{}(compute, k); + } + + auto scalec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + // load a scale + if constexpr (col == 0) { + vas = _mm512_set1_ps(As[row]); + } + // load b scale and vcomp + if constexpr (row == 0) { + vbs0[col] = _mm512_loadu_ps(Bs0 + col * 16); + vbs1[col] = _mm512_loadu_ps(Bs1 + col * 16); + vcomp0[col] = _mm512_loadu_si512(Bcomp0 + col * 16); + vcomp1[col] = _mm512_loadu_si512(Bcomp1 + col * 16); + } + __m512 c0 = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc0[i], vcomp0[col])); + __m512 c1 = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc1[i], vcomp1[col])); + vc0[i] = _mm512_castps_si512(_mm512_mul_ps(_mm512_mul_ps(c0, vas), vbs0[col])); + vc1[i] = _mm512_castps_si512(_mm512_mul_ps(_mm512_mul_ps(c1, vas), vbs1[col])); + }; + Unroll{}(scalec); + + using Vec = at::vec::Vectorized; + const Vec one = Vec(1.f); + auto storec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + // for COLS = 2, 4 use 512bit store + if constexpr (col % 2 == 0) { + Vec x0 = _mm512_castsi512_ps(vc0[row * COLS + col + 0]); + Vec x1 = _mm512_castsi512_ps(vc0[row * COLS + col + 1]); + Vec y0 = _mm512_castsi512_ps(vc1[row * COLS + col + 0]); + Vec y1 = _mm512_castsi512_ps(vc1[row * COLS + col + 1]); + // silu + x0 = x0 / (one + x0.neg().exp_u20()); + x1 = x1 / (one + x1.neg().exp_u20()); + // mul + x0 = x0 * y0; + x1 = x1 * y1; + + _mm512_storeu_si512( + reinterpret_cast<__m512i*>((C + row * ldc + col * 16)), + (__m512i)(_mm512_cvtne2ps_pbh(__m512(x1), __m512(x0)))); + } + }; + Unroll{}(storec); + } +}; +#endif + +#define LAUNCH_TINYGEMM_KERNEL_VNNI(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_vnni::apply( \ + A + mb_start * lda, \ + B0 + nb_start * 4, \ + B1 + nb_start * 4, \ + C + mb_start * ldc + nb_start, \ + As + mb_start, \ + Bs0 + nb_start, \ + Bs1 + nb_start, \ + Bcomp0 + nb_start, \ + Bcomp1 + nb_start, \ + K, \ + lda, \ + ldb, \ + ldc); + +template +void tinygemm_kernel( + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B0, + const int8_t* __restrict__ B1, + scalar_t* __restrict__ C, + const float* __restrict__ As, + const float* __restrict__ Bs0, + const float* __restrict__ Bs1, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + const int32_t* Bcomp0 = reinterpret_cast(B0 + block_size_n() * K); + const int32_t* Bcomp1 = reinterpret_cast(B1 + block_size_n() * K); + + // pattern: 1-(2+2)-(8+8) + constexpr int64_t BLOCK_M = 4; + constexpr int64_t BLOCK_N = 32; + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + for (int mb = 0; mb < MB; ++mb) { + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(BLOCK_M, M - mb_start); + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (mb_size << 4 | nb_size >> 4) { + case 0x12: + LAUNCH_TINYGEMM_KERNEL_VNNI(1, 32); + break; + case 0x22: + LAUNCH_TINYGEMM_KERNEL_VNNI(2, 32); + break; + case 0x32: + LAUNCH_TINYGEMM_KERNEL_VNNI(3, 32); + break; + case 0x42: + LAUNCH_TINYGEMM_KERNEL_VNNI(4, 32); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size"); + } + } + } +} + +/// gemm for w2 +template +struct tinygemm_kernel_vnni2 { + static inline void apply( + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B, + float* __restrict__ C, + const float* __restrict__ As, + const float* __restrict__ Bs, + const int32_t* __restrict__ Bcomp, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!"); + } +}; + +#if defined(CPU_CAPABILITY_AVX512) +template +struct tinygemm_kernel_vnni2 { + static inline void apply( + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B, + float* __restrict__ C, + const float* __restrict__ As, + const float* __restrict__ Bs, + const int32_t* __restrict__ Bcomp, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + constexpr int ROWS = BLOCK_M; + constexpr int COLS = BLOCK_N / 16; + static_assert(COLS % 2 == 0); + + __m512i va; + __m512i vb[COLS]; + __m512i vc[ROWS * COLS]; + __m512i vcomp[COLS]; + __m512 vas; + __m512 vbs[COLS]; + + auto loadc = [&](auto i) { vc[i] = _mm512_set1_epi32(0); }; + Unroll{}(loadc); + + const int64_t K4 = K >> 2; + const int64_t lda4 = lda >> 2; + const int64_t ldb4 = ldb; // ldb * 4 >> 2; + const int32_t* a_ptr = reinterpret_cast(A); + const int32_t* b_ptr = reinterpret_cast(B); + + auto compute = [&](auto i, int64_t k) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + va = _mm512_set1_epi32(a_ptr[row * lda4 + k]); + } + if constexpr (row == 0) { + vb[col] = _mm512_loadu_si512(b_ptr + k * ldb4 + col * 16); + } + vc[i] = _mm512_dpbusd_epi32(vc[i], va, vb[col]); + }; + for (int64_t k = 0; k < K4; ++k) { + Unroll{}(compute, k); + } + + auto storec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + // load a scale + if constexpr (col == 0) { + vas = _mm512_set1_ps(As[row]); + } + // load b scale and vcomp per 2 vectors + // also load bias if any + if constexpr (row == 0) { + if constexpr (col % 2 == 0) { + vbs[col + 0] = _mm512_loadu_ps(Bs + col * 16); + vbs[col + 1] = _mm512_loadu_ps(Bs + col * 16 + 16); + vcomp[col + 0] = _mm512_loadu_si512(Bcomp + col * 16); + vcomp[col + 1] = _mm512_loadu_si512(Bcomp + col * 16 + 16); + } + } + __m512 x = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc[i], vcomp[col])); + x = _mm512_mul_ps(_mm512_mul_ps(x, vas), vbs[col]); + _mm512_storeu_ps(reinterpret_cast<__m512*>(C + row * ldc + col * 16), x); + }; + Unroll{}(storec); + } +}; +#endif + +#define LAUNCH_TINYGEMM_KERNEL_VNNI2(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_vnni2::apply( \ + A + mb_start * lda, \ + B + nb_start * 4, \ + C + mb_start * ldc + nb_start, \ + As + mb_start, \ + Bs + nb_start, \ + Bcomp + nb_start, \ + K, \ + lda, \ + ldb, \ + ldc); + +template +void tinygemm_kernel( + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B, + float* __restrict__ C, + const float* __restrict__ As, + const float* __restrict__ Bs, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + // B compensation + const int32_t* Bcomp = reinterpret_cast(B + block_size_n() * K); + + // pattern: 1-4-16 + constexpr int64_t BLOCK_M = 4; + constexpr int64_t BLOCK_N = 64; + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + for (int64_t mb = 0; mb < MB; ++mb) { + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(BLOCK_M, M - mb_start); + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (mb_size << 4 | nb_size >> 4) { + case 0x12: + LAUNCH_TINYGEMM_KERNEL_VNNI2(1, 32); + break; + case 0x22: + LAUNCH_TINYGEMM_KERNEL_VNNI2(2, 32); + break; + case 0x32: + LAUNCH_TINYGEMM_KERNEL_VNNI2(3, 32); + break; + case 0x42: + LAUNCH_TINYGEMM_KERNEL_VNNI2(4, 32); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size"); + } + } + } +} + +} // anonymous namespace + +template +void fused_experts_int8_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic1, + scalar_t* __restrict__ ic2, + uint8_t* __restrict__ A_tmp, + float* __restrict__ C_tmp, + uint8_t* __restrict__ Aq_tmp, + float* __restrict__ As_tmp, + const scalar_t* __restrict__ input, + const int8_t* __restrict__ packed_w1, + const int8_t* __restrict__ packed_w2, + const float* __restrict__ w1s, + const float* __restrict__ w2s, + const float* __restrict__ topk_weights, + const int32_t* __restrict__ sorted_ids, + const int32_t* __restrict__ expert_ids, + const int32_t* __restrict__ offsets, + int64_t M, + int64_t N, + int64_t K, + int64_t E, + int64_t topk, + int64_t num_tokens_post_pad) { + // handle 2 tiles per block + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + + // stage 0: quantize input to uint8, [M, K] + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + quantize_row_int8(Aq_tmp + m * K, As_tmp[m], input + m * K, K); + } + }); + + // stage 1: intermediate_cache1 = silu(hidden_states @ w1) + const int64_t MB = div_up(num_tokens_post_pad, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + + // strides for w1: [E, 2N, K] + TORCH_CHECK(N % BLOCK_N == 0, "Fixme when N is not multiples of ", BLOCK_N); + + // K and N are packed for int8 + const int64_t packed_K = get_row_size(K); + const int64_t packed_N = get_row_size(N); + + const int64_t stride_e = 2 * N * packed_K; + const int64_t stride_n = packed_K; + + int64_t avg_M = std::max(int64_t(1), M * topk / E); + const bool use_brgemm = can_use_brgemm(avg_M); + + // here we only parallel on half of 2N to fuse silu_and_mul with gemm + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // get local pointers + int tid = get_thread_num(); + uint8_t* __restrict__ A = A_tmp + tid * BLOCK_M * K; + int32_t* __restrict__ C0 = reinterpret_cast(C_tmp) + tid * 2 * BLOCK_M * BLOCK_N; + int32_t* __restrict__ C1 = C0 + BLOCK_M * BLOCK_N; + + alignas(64) float As[BLOCK_M]; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K * 2, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + // nb_upper from top half and nb_lower from bottom half + int64_t nb_upper = nb, nb_lower = nb + NB; + int64_t n_size = std::min(N - nb * BLOCK_N, BLOCK_N); + + // B shape [K, n_size] in vnni format + int32_t expert_id = expert_ids[mb]; + const int8_t* __restrict__ B0 = packed_w1 + expert_id * stride_e + nb_upper * BLOCK_N * stride_n; + const int8_t* __restrict__ B1 = packed_w1 + expert_id * stride_e + nb_lower * BLOCK_N * stride_n; + const float* __restrict__ Bs0 = w1s + expert_id * 2 * N + nb_upper * BLOCK_N; + const float* __restrict__ Bs1 = w1s + expert_id * 2 * N + nb_lower * BLOCK_N; + + // 1.a load A + const int32_t* A_ids = sorted_ids + mb * BLOCK_M; + int64_t m_size = offsets[mb + 1] - offsets[mb]; + + for (int64_t m = 0; m < m_size; ++m) { + int32_t index = A_ids[m] / topk; + copy_stub(A + m * K, Aq_tmp + index * K, K); + As[m] = As_tmp[index]; + } + + if (use_brgemm) { + // 1.b gemm: C0 = A @ B0 + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B0, + /* C */ C0); + + // 1.c gemm: C1 = A @ B1 + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B1, + /* C */ C1); + + const int32_t* Bcomp0 = reinterpret_cast(B0 + block_size_n() * K); + const int32_t* Bcomp1 = reinterpret_cast(B1 + block_size_n() * K); + + // 1.d silu and mul + const int64_t offset = offsets[mb]; + silu_and_mul( + ic1 + offset * N + nb * BLOCK_N, C0, C1, As, Bs0, Bs1, Bcomp0, Bcomp1, m_size, N); + } else { + // fused 1.bcd: silu_and_mul(A @ B0, A @ B1) + const int64_t offset = offsets[mb]; + tinygemm_kernel( + /* A */ A, + /* B0 */ B0, + /* B1 */ B1, + /* C */ ic1 + offset * N + nb * BLOCK_N, + /* As */ As, + /* Bs0 */ Bs0, + /* Bs1 */ Bs1, + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ N); + } + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 1.5: quantize ic1 to uint8, [M * topk, N] + at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + quantize_row_int8(Aq_tmp + m * N, As_tmp[m], ic1 + m * N, N); + } + }); + + // stage 2: intermediate_cache2 = intermediate_cache1 @ w2 + // w2 : [E, K, N] as [E, OC, IC] + const int64_t OC = K; // rename K as OC + const int64_t IC = N; // rename N as IC + const int64_t MB2 = MB; + const int64_t NB2 = div_up(OC, BLOCK_N); + const int64_t stride_e2 = OC * packed_N; + const int64_t stride_oc = packed_N; + + // parallel on [MB2, NB2] + parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // get local pointers + int tid = get_thread_num(); + float* __restrict__ C = C_tmp + tid * 2 * BLOCK_M * BLOCK_N; + int32_t* __restrict__ C32 = reinterpret_cast(C + BLOCK_M * BLOCK_N); + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + int64_t m_size = offsets[mb + 1] - offsets[mb]; + int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N); + + // A ptr from ic1 of [M * topk, N] in sorted order + // so as to avoid copy A to tmp buffer again + const uint8_t* __restrict__ A = Aq_tmp + offsets[mb] * N; + const float* __restrict__ As = As_tmp + offsets[mb]; + const int32_t* A_ids = sorted_ids + mb * BLOCK_M; + + // B shape [IC, n_size] in vnni format + int32_t expert_id = expert_ids[mb]; + const int8_t* __restrict__ B = packed_w2 + expert_id * stride_e2 + nb * BLOCK_N * stride_oc; + const float* __restrict__ Bs = w2s + expert_id * K + nb * BLOCK_N; + + // 2.a gemm: C = A @ B + if (use_brgemm) { + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B, + /* C */ C32); + + // apply scales + const int32_t* Bcomp = reinterpret_cast(B + block_size_n() * IC); + scale_C(C, C32, As, Bs, Bcomp, m_size); + } else { + tinygemm_kernel( + /* A */ A, + /* B */ B, + /* C */ C, + /* As */ As, + /* Bs */ Bs, + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N); + } + + // 2.b copy from C to ic2 in original order + // and also mul topk_weights in float32 + for (int64_t m = 0; m < m_size; ++m) { + int32_t index = A_ids[m]; + float weight = topk_weights[index]; + copy_mul_stub(ic2 + index * K + nb * BLOCK_N, C + m * BLOCK_N, weight, n_size); + } + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 3: out = intermediate_cache2.sum(dim=1) + // from [M, topk, K] to [M, K] + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + sum_stub(output + m * K, ic2 + m * topk * K, topk, K); + } + }); +} + +#define INSTANTIATE_MOE_INT8_TEMPLATE(TYPE) \ + template void fused_experts_int8_kernel_impl( \ + TYPE* __restrict__ output, \ + TYPE* __restrict__ ic1, \ + TYPE* __restrict__ ic2, \ + uint8_t* __restrict__ A_tmp, \ + float* __restrict__ C_tmp, \ + uint8_t* __restrict__ Aq_tmp, \ + float* __restrict__ As_tmp, \ + const TYPE* __restrict__ input, \ + const int8_t* __restrict__ packed_w1, \ + const int8_t* __restrict__ packed_w2, \ + const float* __restrict__ w1s, \ + const float* __restrict__ w2s, \ + const float* __restrict__ topk_weights, \ + const int32_t* __restrict__ sorted_ids, \ + const int32_t* __restrict__ expert_ids, \ + const int32_t* __restrict__ offsets, \ + int64_t M, \ + int64_t N, \ + int64_t K, \ + int64_t E, \ + int64_t topk, \ + int64_t num_tokens_post_pad) + +INSTANTIATE_MOE_INT8_TEMPLATE(at::BFloat16); +INSTANTIATE_MOE_INT8_TEMPLATE(at::Half); + +template +void shared_expert_int8_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic1, + float* __restrict__ C_tmp, + uint8_t* __restrict__ Aq_tmp, + float* __restrict__ As_tmp, + const scalar_t* __restrict__ input, + const int8_t* __restrict__ packed_w1, + const int8_t* __restrict__ packed_w2, + const float* __restrict__ w1s, + const float* __restrict__ w2s, + const scalar_t* __restrict__ fused_experts_out, + float routed_scaling_factor, + int64_t M, + int64_t N, + int64_t K) { + // handle 2 tiles per block + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + + // stage 0: quantize input to uint8, [M, K] + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + quantize_row_int8(Aq_tmp + m * K, As_tmp[m], input + m * K, K); + } + }); + + // stage 1: intermediate_cache1 = silu(hidden_states @ w1) + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + + TORCH_CHECK(N % BLOCK_N == 0, "Fixme when N is not multiples of ", BLOCK_N); + + // K and N are packed for int8 + const int64_t packed_K = get_row_size(K); + const int64_t packed_N = get_row_size(N); + const int64_t stride_n = packed_K; + + const bool use_brgemm = can_use_brgemm(M); + + // here we only parallel on half of 2N to fuse silu_and_mul with gemm + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // get local pointers + int tid = get_thread_num(); + int32_t* __restrict__ C0 = reinterpret_cast(C_tmp) + tid * 2 * BLOCK_M * BLOCK_N; + int32_t* __restrict__ C1 = C0 + BLOCK_M * BLOCK_N; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K * 2, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + // nb_upper from top half and nb_lower from bottom half + int64_t nb_upper = nb, nb_lower = nb + NB; + int64_t n_size = std::min(N - nb * BLOCK_N, BLOCK_N); + int64_t m_size = std::min(M - mb * BLOCK_M, BLOCK_M); + + // A shape [m_size, K] + const uint8_t* A = Aq_tmp + mb * BLOCK_M * K; + const float* As = As_tmp + mb * BLOCK_M; + + // B shape [K, n_size] in vnni format + const int8_t* __restrict__ B0 = packed_w1 + nb_upper * BLOCK_N * stride_n; + const int8_t* __restrict__ B1 = packed_w1 + nb_lower * BLOCK_N * stride_n; + const float* __restrict__ Bs0 = w1s + nb_upper * BLOCK_N; + const float* __restrict__ Bs1 = w1s + nb_lower * BLOCK_N; + + if (use_brgemm) { + // 1.b gemm: C0 = A @ B0 + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B0, + /* C */ C0); + + // 1.c gemm: C1 = A @ B1 + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B1, + /* C */ C1); + + const int32_t* Bcomp0 = reinterpret_cast(B0 + block_size_n() * K); + const int32_t* Bcomp1 = reinterpret_cast(B1 + block_size_n() * K); + + // 1.d silu and mul + silu_and_mul( + ic1 + mb * BLOCK_M * N + nb * BLOCK_N, C0, C1, As, Bs0, Bs1, Bcomp0, Bcomp1, m_size, N); + } else { + // fused 1.bcd: silu_and_mul(A @ B0, A @ B1) + tinygemm_kernel( + /* A */ A, + /* B0 */ B0, + /* B1 */ B1, + /* C */ ic1 + mb * BLOCK_M * N + nb * BLOCK_N, + /* As */ As, + /* Bs0 */ Bs0, + /* Bs1 */ Bs1, + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ N); + } + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 1.5: quantize ic1 to uint8, [M * topk, N] + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + quantize_row_int8(Aq_tmp + m * N, As_tmp[m], ic1 + m * N, N); + } + }); + + // stage 2: intermediate_cache2 = intermediate_cache1 @ w2 + // w2 : [K, N] as [OC, IC] + const int64_t OC = K; // rename K as OC + const int64_t IC = N; // rename N as IC + const int64_t MB2 = MB; + const int64_t NB2 = div_up(OC, BLOCK_N); + const int64_t stride_oc = packed_N; + + // parallel on [MB2, NB2] + parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // get local pointers + int tid = get_thread_num(); + float* __restrict__ C = C_tmp + tid * 2 * BLOCK_M * BLOCK_N; + int32_t* __restrict__ C32 = reinterpret_cast(C + BLOCK_M * BLOCK_N); + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + int64_t m_size = std::min(M - mb * BLOCK_M, BLOCK_M); + int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N); + + // A shape [m_size, IC] + const uint8_t* __restrict__ A = Aq_tmp + mb * BLOCK_M * N; + const float* __restrict__ As = As_tmp + mb * BLOCK_M; + + // B shape [IC, n_size] in vnni format + const int8_t* __restrict__ B = packed_w2 + nb * BLOCK_N * stride_oc; + const float* __restrict__ Bs = w2s + nb * BLOCK_N; + + if (use_brgemm) { + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B, + /* C */ C32); + + // apply scales + const int32_t* Bcomp = reinterpret_cast(B + block_size_n() * IC); + scale_C(C, C32, As, Bs, Bcomp, m_size); + } else { + // 2.a gemm: C = A @ B + tinygemm_kernel( + /* A */ A, + /* B */ B, + /* C */ C, + /* As */ As, + /* Bs */ Bs, + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N); + } + + // 2.b copy from C to output and add fused_experts_out + scalar_t* __restrict__ out = output + mb * BLOCK_M * K + nb * BLOCK_N; + const scalar_t* __restrict__ fused_out = fused_experts_out + mb * BLOCK_M * K + nb * BLOCK_N; + for (int64_t m = 0; m < m_size; ++m) { + add_mul_stub(out + m * K, C + m * BLOCK_N, fused_out + m * K, routed_scaling_factor, n_size); + } + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); +} + +#define INSTANTIATE_SHARED_EXPERT_INT8_TEMPLATE(TYPE) \ + template void shared_expert_int8_kernel_impl( \ + TYPE* __restrict__ output, \ + TYPE* __restrict__ ic1, \ + float* __restrict__ C_tmp, \ + uint8_t* __restrict__ Aq_tmp, \ + float* __restrict__ As_tmp, \ + const TYPE* __restrict__ input, \ + const int8_t* __restrict__ packed_w1, \ + const int8_t* __restrict__ packed_w2, \ + const float* __restrict__ w1s, \ + const float* __restrict__ w2s, \ + const TYPE* __restrict__ fused_experts_out, \ + float routed_scaling_factor, \ + int64_t M, \ + int64_t N, \ + int64_t K) + +INSTANTIATE_SHARED_EXPERT_INT8_TEMPLATE(at::BFloat16); +INSTANTIATE_SHARED_EXPERT_INT8_TEMPLATE(at::Half); diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/norm.cpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/norm.cpp new file mode 100644 index 00000000..239ee1b2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/norm.cpp @@ -0,0 +1,417 @@ +#include "common.h" +#include "vec.h" + +namespace { + +// NB: avoid using `at::vec::map<>` on bfloat16 or half +// Llama4TextL2Norm +template +void l2norm_kernel_impl( + scalar_t* __restrict__ output, + const scalar_t* __restrict__ input, + int64_t batch_size, + int64_t hidden_size, + float eps = 1e-5) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + + constexpr int kVecSize = bVec::size(); + at::parallel_for(0, batch_size, 0, [&](int64_t begin, int64_t end) { + for (int64_t i = begin; i < end; ++i) { + // local ptrs + scalar_t* __restrict__ out_ptr = output + i * hidden_size; + const scalar_t* __restrict__ input_ptr = input + i * hidden_size; + + fVec sum_fvec = fVec(float(0)); + float sum_val = float(0); + + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= hidden_size - kVecSize; d += kVecSize) { + bVec x_bvec = bVec::loadu(input_ptr + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + sum_fvec += x_fvec0 * x_fvec0; + sum_fvec += x_fvec1 * x_fvec1; + } +#pragma GCC unroll 4 + for (; d < hidden_size; ++d) { + float x_val = static_cast(input_ptr[d]); + sum_val += x_val * x_val; + } + + sum_val += vec_reduce_sum(sum_fvec); + float rsqrt_var = float(1) / std::sqrt(sum_val / hidden_size + eps); + const fVec scale_fvec = fVec(rsqrt_var); + +#pragma GCC unroll 4 + for (d = 0; d <= hidden_size - kVecSize; d += kVecSize) { + bVec x_bvec = bVec::loadu(input_ptr + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + x_fvec0 = x_fvec0 * scale_fvec; + x_fvec1 = x_fvec1 * scale_fvec; + + bVec out_bvec = convert_from_float_ext(x_fvec0, x_fvec1); + out_bvec.store(out_ptr + d); + } +#pragma GCC unroll 4 + for (; d < hidden_size; ++d) { + float x_val = static_cast(input_ptr[d]); + out_ptr[d] = static_cast(x_val * rsqrt_var); + } + } + }); +} +template +void rmsnorm_kernel_impl( + scalar_t* __restrict__ output, + const scalar_t* __restrict__ input, + const scalar_t* __restrict__ weight, + int64_t batch_size, + int64_t hidden_size, + int64_t input_strideN, + float eps = 1e-5) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + + constexpr int kVecSize = bVec::size(); + at::parallel_for(0, batch_size, 0, [&](int64_t begin, int64_t end) { + for (int64_t i = begin; i < end; ++i) { + // local ptrs + scalar_t* __restrict__ out_ptr = output + i * hidden_size; + const scalar_t* __restrict__ input_ptr = input + i * input_strideN; + + fVec sum_fvec = fVec(float(0)); + float sum_val = float(0); + + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= hidden_size - kVecSize; d += kVecSize) { + bVec x_bvec = bVec::loadu(input_ptr + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + sum_fvec += x_fvec0 * x_fvec0; + sum_fvec += x_fvec1 * x_fvec1; + } +#pragma GCC unroll 4 + for (; d < hidden_size; ++d) { + float x_val = static_cast(input_ptr[d]); + sum_val += x_val * x_val; + } + + sum_val += vec_reduce_sum(sum_fvec); + float rsqrt_var = float(1) / std::sqrt(sum_val / hidden_size + eps); + const fVec scale_fvec = fVec(rsqrt_var); + +#pragma GCC unroll 4 + for (d = 0; d <= hidden_size - kVecSize; d += kVecSize) { + bVec x_bvec = bVec::loadu(input_ptr + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + bVec w_bvec = bVec::loadu(weight + d); + fVec w_fvec0, w_fvec1; + std::tie(w_fvec0, w_fvec1) = at::vec::convert_to_float(w_bvec); + + x_fvec0 = x_fvec0 * scale_fvec * w_fvec0; + x_fvec1 = x_fvec1 * scale_fvec * w_fvec1; + + bVec out_bvec = convert_from_float_ext(x_fvec0, x_fvec1); + out_bvec.store(out_ptr + d); + } +#pragma GCC unroll 4 + for (; d < hidden_size; ++d) { + float x_val = static_cast(input_ptr[d]); + float w_val = static_cast(weight[d]); + out_ptr[d] = static_cast(x_val * rsqrt_var * w_val); + } + } + }); +} + +template +void fused_add_rmsnorm_kernel_impl( + scalar_t* __restrict__ input, + scalar_t* __restrict__ residual, + const scalar_t* __restrict__ weight, + float* __restrict__ buffer, + int64_t batch_size, + int64_t hidden_size, + int64_t input_strideN, + float eps = 1e-5) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + + constexpr int kVecSize = bVec::size(); + at::parallel_for(0, batch_size, 0, [&](int64_t begin, int64_t end) { + int tid = at::get_thread_num(); + float* __restrict__ buffer_ptr = buffer + tid * hidden_size; + + for (int64_t i = begin; i < end; ++i) { + // local ptrs + scalar_t* __restrict__ input_ptr = input + i * input_strideN; + scalar_t* __restrict__ residual_ptr = residual + i * hidden_size; + + fVec sum_fvec = fVec(float(0)); + float sum_val = float(0); + + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= hidden_size - kVecSize; d += kVecSize) { + bVec x_bvec = bVec::loadu(input_ptr + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + bVec r_bvec = bVec::loadu(residual_ptr + d); + fVec r_fvec0, r_fvec1; + std::tie(r_fvec0, r_fvec1) = at::vec::convert_to_float(r_bvec); + + x_fvec0 += r_fvec0; + x_fvec1 += r_fvec1; + + bVec out_bvec = convert_from_float_ext(x_fvec0, x_fvec1); + out_bvec.store(residual_ptr + d); + + sum_fvec += x_fvec0 * x_fvec0; + sum_fvec += x_fvec1 * x_fvec1; + + x_fvec0.store(buffer_ptr + d); + x_fvec1.store(buffer_ptr + d + fVec::size()); + } +#pragma GCC unroll 4 + for (; d < hidden_size; ++d) { + float x_val = static_cast(input_ptr[d]); + float r_val = static_cast(residual_ptr[d]); + + x_val += r_val; + residual_ptr[d] = static_cast(x_val); + + sum_val += x_val * x_val; + buffer_ptr[d] = x_val; + } + + sum_val += vec_reduce_sum(sum_fvec); + float rsqrt_var = float(1) / std::sqrt(sum_val / hidden_size + eps); + const fVec scale_fvec = fVec(rsqrt_var); + +#pragma GCC unroll 4 + for (d = 0; d <= hidden_size - kVecSize; d += kVecSize) { + fVec x_fvec0 = fVec::loadu(buffer_ptr + d); + fVec x_fvec1 = fVec::loadu(buffer_ptr + d + fVec::size()); + + bVec w_bvec = bVec::loadu(weight + d); + fVec w_fvec0, w_fvec1; + std::tie(w_fvec0, w_fvec1) = at::vec::convert_to_float(w_bvec); + + x_fvec0 = x_fvec0 * scale_fvec * w_fvec0; + x_fvec1 = x_fvec1 * scale_fvec * w_fvec1; + bVec x_bvec = convert_from_float_ext(x_fvec0, x_fvec1); + x_bvec.store(input_ptr + d); + } +#pragma GCC unroll 4 + for (; d < hidden_size; ++d) { + float x_val = buffer_ptr[d] * rsqrt_var * static_cast(weight[d]); + input_ptr[d] = x_val; + } + } + }); +} + +template +void fused_rmsnorm_gated_kernel_impl( + scalar_t* __restrict__ output, + const scalar_t* __restrict__ input, + const scalar_t* __restrict__ weight, + const scalar_t* __restrict__ gate, + int64_t batch_size, + int64_t hidden_size, + int64_t input_strideN, + float eps = 1e-5) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + const fVec one = fVec(1.f); + + constexpr int kVecSize = bVec::size(); + at::parallel_for(0, batch_size, 0, [&](int64_t begin, int64_t end) { + for (int64_t i = begin; i < end; ++i) { + // local ptrs + scalar_t* __restrict__ out_ptr = output + i * hidden_size; + const scalar_t* __restrict__ input_ptr = input + i * input_strideN; + const scalar_t* __restrict__ gate_ptr = gate + i * hidden_size; + + fVec sum_fvec = fVec(float(0)); + float sum_val = float(0); + + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= hidden_size - kVecSize; d += kVecSize) { + bVec x_bvec = bVec::loadu(input_ptr + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + sum_fvec += x_fvec0 * x_fvec0; + sum_fvec += x_fvec1 * x_fvec1; + } +#pragma GCC unroll 4 + for (; d < hidden_size; ++d) { + float x_val = static_cast(input_ptr[d]); + sum_val += x_val * x_val; + } + + sum_val += vec_reduce_sum(sum_fvec); + float rsqrt_var = float(1) / std::sqrt(sum_val / hidden_size + eps); + const fVec scale_fvec = fVec(rsqrt_var); + +#pragma GCC unroll 4 + for (d = 0; d <= hidden_size - kVecSize; d += kVecSize) { + bVec x_bvec = bVec::loadu(input_ptr + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + bVec w_bvec = bVec::loadu(weight + d); + fVec w_fvec0, w_fvec1; + std::tie(w_fvec0, w_fvec1) = at::vec::convert_to_float(w_bvec); + + bVec g_bvec = bVec::loadu(gate_ptr + d); + fVec g_fvec0, g_fvec1; + std::tie(g_fvec0, g_fvec1) = at::vec::convert_to_float(g_bvec); + g_fvec0 = g_fvec0 / (one + g_fvec0.neg().exp_u20()); + g_fvec1 = g_fvec1 / (one + g_fvec1.neg().exp_u20()); + + x_fvec0 = x_fvec0 * scale_fvec * w_fvec0 * g_fvec0; + x_fvec1 = x_fvec1 * scale_fvec * w_fvec1 * g_fvec1; + + bVec out_bvec = convert_from_float_ext(x_fvec0, x_fvec1); + out_bvec.store(out_ptr + d); + } +#pragma GCC unroll 4 + for (; d < hidden_size; ++d) { + float x_val = static_cast(input_ptr[d]); + float w_val = static_cast(weight[d]); + float g_val = static_cast(gate_ptr[d]); + + out_ptr[d] = static_cast(x_val * rsqrt_var * w_val * g_val / (1.f + std::exp(-g_val))); + } + } + }); +} + +} // anonymous namespace + +// input : {batch_size, hidden_size} +at::Tensor l2norm_cpu(at::Tensor& input, double eps) { + RECORD_FUNCTION("sgl-kernel::l2norm_cpu", std::vector({input})); + + CHECK_INPUT(input); + CHECK_DIM(2, input); + int64_t batch_size = input.size(0); + int64_t hidden_size = input.size(1); + at::Tensor output = at::empty_like(input); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(input.scalar_type(), "l2norm_kernel", [&] { + l2norm_kernel_impl(output.data_ptr(), input.data_ptr(), batch_size, hidden_size, eps); + }); + return output; +} + +// input : {batch_size, hidden_size} +// weight: {hidden_size} +at::Tensor rmsnorm_cpu(at::Tensor& input, at::Tensor& weight, double eps) { + RECORD_FUNCTION("sgl-kernel::rmsnorm_cpu", std::vector({input, weight})); + + CHECK_LAST_DIM_CONTIGUOUS_INPUT(input); + CHECK_INPUT(weight); + CHECK_DIM(2, input); + CHECK_DIM(1, weight); + CHECK_EQ(input.size(1), weight.size(0)); + int64_t batch_size = input.size(0); + int64_t hidden_size = input.size(1); + at::Tensor output = at::empty_like(input); + int64_t input_strideN = input.stride(0); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(input.scalar_type(), "rmsnorm_kernel", [&] { + rmsnorm_kernel_impl( + output.data_ptr(), + input.data_ptr(), + weight.data_ptr(), + batch_size, + hidden_size, + input_strideN, + eps); + }); + return output; +} + +// input : {batch_size, hidden_size} +// weight: {hidden_size} +// gate: {batch_size, hidden_size} +at::Tensor fused_rmsnorm_gated_cpu(at::Tensor& input, at::Tensor& weight, at::Tensor& gate, double eps) { + RECORD_FUNCTION("sgl-kernel::fused_rmsnorm_gated_cpu", std::vector({input, weight, gate})); + + CHECK_LAST_DIM_CONTIGUOUS_INPUT(input); + CHECK_INPUT(weight); + CHECK_INPUT(gate); + CHECK_DIM(2, input); + CHECK_DIM(1, weight); + CHECK_DIM(2, gate); + CHECK_EQ(input.size(1), weight.size(0)); + int64_t batch_size = input.size(0); + int64_t hidden_size = input.size(1); + CHECK_EQ(input.size(0), gate.size(0)); + CHECK_EQ(input.size(1), gate.size(1)); + at::Tensor output = at::empty_like(input); + int64_t input_strideN = input.stride(0); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(input.scalar_type(), "fused_rmsnorm_gated_kernel", [&] { + fused_rmsnorm_gated_kernel_impl( + output.data_ptr(), + input.data_ptr(), + weight.data_ptr(), + gate.data_ptr(), + batch_size, + hidden_size, + input_strideN, + eps); + }); + return output; +} + +// input : {batch_size, hidden_size} +// residual: {batch_size, hidden_size} +// weight : {hidden_size} +void fused_add_rmsnorm_cpu(at::Tensor& input, at::Tensor& residual, at::Tensor& weight, double eps) { + RECORD_FUNCTION("sgl-kernel::fused_add_rmsnorm_cpu", std::vector({input, residual, weight})); + CHECK_LAST_DIM_CONTIGUOUS_INPUT(input); + CHECK_INPUT(residual); + CHECK_INPUT(weight); + CHECK_DIM(2, input); + CHECK_DIM(2, residual); + CHECK_DIM(1, weight); + CHECK_EQ(input.size(0), residual.size(0)); + CHECK_EQ(input.size(1), residual.size(1)); + CHECK_EQ(input.size(1), weight.size(0)); + int64_t batch_size = input.size(0); + int64_t hidden_size = input.size(1); + int64_t input_strideN = input.stride(0); + + // allocate temp buffer to store x in float32 per thread + // TODO: implement a singleton for context + int64_t num_threads = at::get_num_threads(); + at::Tensor buffer = at::empty({num_threads, hidden_size}, input.options().dtype(at::kFloat)); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(input.scalar_type(), "fused_add_rmsnorm_kernel", [&] { + fused_add_rmsnorm_kernel_impl( + input.data_ptr(), + residual.data_ptr(), + weight.data_ptr(), + buffer.data_ptr(), + batch_size, + hidden_size, + input_strideN, + eps); + }); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/numa_utils.cpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/numa_utils.cpp new file mode 100644 index 00000000..2699d0e2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/numa_utils.cpp @@ -0,0 +1,91 @@ +#include +#include +#include +#include +#include + +#include + +#include "common.h" + +std::string init_cpu_threads_env(const std::string& cpu_ids) { + bitmask* omp_cpu_mask = numa_parse_cpustring(cpu_ids.c_str()); + TORCH_CHECK(omp_cpu_mask->size > 0); + std::vector omp_cpu_ids; + omp_cpu_ids.reserve(omp_cpu_mask->size); + + constexpr int group_size = 8 * sizeof(*omp_cpu_mask->maskp); + + for (int offset = 0; offset < omp_cpu_mask->size; offset += group_size) { + unsigned long group_mask = omp_cpu_mask->maskp[offset / group_size]; + int i = 0; + while (group_mask) { + if (group_mask & 1) { + omp_cpu_ids.emplace_back(offset + i); + } + ++i; + group_mask >>= 1; + } + } + + // Memory node binding + if (numa_available() != -1) { + int mem_node_id = numa_node_of_cpu(omp_cpu_ids.front()); + bitmask* mask = numa_parse_nodestring(std::to_string(mem_node_id).c_str()); + bitmask* src_mask = numa_get_membind(); + + int pid = getpid(); + + // move all existing pages to the specified numa node. + *(src_mask->maskp) = *(src_mask->maskp) ^ *(mask->maskp); + int page_num = numa_migrate_pages(pid, src_mask, mask); + if (page_num == -1) { + TORCH_WARN(false, "numa_migrate_pages failed. errno: " + std::to_string(errno)); + } + + // restrict memory allocation node. + numa_set_membind(mask); + numa_set_strict(1); + } + + // OMP threads binding + omp_set_num_threads((int)omp_cpu_ids.size()); + at::set_num_threads((int)omp_cpu_ids.size()); + TORCH_CHECK_EQ(omp_cpu_ids.size(), at::get_num_threads()); + TORCH_CHECK_EQ(omp_cpu_ids.size(), omp_get_max_threads()); + + std::vector> thread_core_mapping; + thread_core_mapping.reserve(omp_cpu_ids.size()); + omp_lock_t writelock; + omp_init_lock(&writelock); + +#pragma omp parallel for schedule(static, 1) + for (size_t i = 0; i < omp_cpu_ids.size(); ++i) { + cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET(omp_cpu_ids[i], &mask); + int ret = sched_setaffinity(0, sizeof(cpu_set_t), &mask); + if (ret == -1) { + TORCH_CHECK(false, "sched_setaffinity failed. errno: " + std::to_string(errno)); + } + + omp_set_lock(&writelock); + thread_core_mapping.emplace_back(syscall(SYS_gettid), omp_cpu_ids[i]); + omp_unset_lock(&writelock); + } + + omp_destroy_lock(&writelock); + + numa_free_nodemask(omp_cpu_mask); + + std::stringstream ss; + ss << "OMP threads binding of Process " << getpid() << ":\n"; + std::sort( + thread_core_mapping.begin(), thread_core_mapping.end(), [](auto&& a, auto&& b) { return a.second < b.second; }); + for (auto&& item : thread_core_mapping) { + ss << "\t" + << "OMP tid: " << item.first << ", core " << item.second << "\n"; + } + + return ss.str(); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/qkv_proj.cpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/qkv_proj.cpp new file mode 100644 index 00000000..b3e2072e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/qkv_proj.cpp @@ -0,0 +1,701 @@ +#include "common.h" +#include "gemm.h" +#include "vec.h" + +namespace { + +// [NOTE]: Fused kernel for QKV projection with weight absorption and RoPE +// +// 1. `q_a_proj` and `kv_a_proj_with_mqa` fused into one gemm, +// otherwise we need to split IC for the 2nd gemm. +// 2. `q_a_layernorm` and `kv_a_layernorm` fused into one parallel loop. +// 3. k_input and v_input share the same storage, the torch API did +// this in `set_kv_buffer`. No additional memory movement. +// + +// [C0, C1] = A @ [B0, B1] +template +void segment_gemm_kernel_impl( + scalar_t* __restrict__ C0, + scalar_t* __restrict__ C1, + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B0, + const scalar_t* __restrict__ B1, + int64_t M, + int64_t N0, + int64_t N1, + int64_t K) { + // convert_weight_packed make sure N0 and N1 are 32x + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB0 = div_up(N0, BLOCK_N); + const int64_t NB1 = div_up(N1, BLOCK_N); + const int64_t NB = NB0 + NB1; + + const bool use_brgemm = can_use_brgemm(M); + + // parallel on [MB, NB0 + NB1] + at::parallel_for(0, MB * NB, 0, [&](int64_t begin, int64_t end) { + int64_t mb{0}, nb{0}; + data_index_init(begin, mb, MB, nb, NB); + + // for brgemm, use float32 for accumulate + alignas(64) float Ctmp[BLOCK_M * BLOCK_N]; + + for (int64_t i = begin; i < end; ++i) { + UNUSED(i); + int mb_start = mb * BLOCK_M; + int mb_size = std::min(M - mb_start, BLOCK_M); + int nb_start = nb * BLOCK_N; + int nb_size = BLOCK_N; + + const scalar_t* __restrict__ B = nb < NB0 ? B0 : B1; + scalar_t* __restrict__ C = nb < NB0 ? C0 : C1; + int64_t ldc = nb < NB0 ? N0 : N1; + int64_t local_nb_start = nb < NB0 ? nb_start : nb_start - N0; + + tinygemm_kernel( + /* A */ A + mb_start * K, + /* B */ B + local_nb_start * K /* nb * BLOCK_N * K */, + /* C */ C + mb_start * ldc + local_nb_start, + /* Ctmp*/ Ctmp, + /* M */ mb_size, + /* N */ nb_size, + /* K */ K, + /* lda */ K, + /* ldb */ nb_size, + /* ldc */ ldc, + /* brg */ use_brgemm); + + // move to the next index + data_index_step(mb, MB, nb, NB); + } + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); +} + +// [C0, C1] = A @ [B0, B1] +template +void segment_gemm_kernel_impl( + scalar_t* __restrict__ C0, + scalar_t* __restrict__ C1, + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B0, + const int8_t* __restrict__ B1, + const float* __restrict__ As, + const float* __restrict__ Bs0, + const float* __restrict__ Bs1, + int64_t M, + int64_t N0, + int64_t N1, + int64_t K) { + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB0 = div_up(N0, BLOCK_N); + const int64_t NB1 = div_up(N1, BLOCK_N); + const int64_t NB = NB0 + NB1; + + const bool use_brgemm = can_use_brgemm(M); + + // K + 4 after compensation + const int64_t packed_row_size = get_row_size(K); + + // parallel on [MB, NB0 + NB1] + at::parallel_for(0, MB * NB, 0, [&](int64_t begin, int64_t end) { + int64_t mb{0}, nb{0}; + data_index_init(begin, mb, MB, nb, NB); + + // for brgemm, use float32 for accumulate + alignas(64) int32_t Ctmp[BLOCK_M * BLOCK_N]; + + for (int64_t i = begin; i < end; ++i) { + UNUSED(i); + int mb_start = mb * BLOCK_M; + int mb_size = std::min(M - mb_start, BLOCK_M); + int nb_start = nb * BLOCK_N; + int nb_size = BLOCK_N; + + const int8_t* __restrict__ B = nb < NB0 ? B0 : B1; + const float* __restrict__ Bs = nb < NB0 ? Bs0 : Bs1; + scalar_t* __restrict__ C = nb < NB0 ? C0 : C1; + int64_t ldc = nb < NB0 ? N0 : N1; + int64_t local_nb_start = nb < NB0 ? nb_start : nb_start - N0; + + tinygemm_kernel( + /* A */ A + mb_start * K, + /* B */ B + local_nb_start * packed_row_size /* nb * BLOCK_N * (K + 4) */, + /* C */ C + mb_start * ldc + local_nb_start, + /* Ctmp*/ Ctmp, + /* As */ As + mb_start, + /* Bs */ Bs + local_nb_start, + /* M */ mb_size, + /* N */ nb_size, + /* K */ K, + /* lda */ K, + /* ldb */ nb_size, + /* ldc */ ldc, + /* brg */ use_brgemm); + + // move to the next index + data_index_step(mb, MB, nb, NB); + } + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); +} + +// [C0, C1] = A @ [B0, B1] +template +void segment_gemm_kernel_impl( + scalar_t* __restrict__ C0, + scalar_t* __restrict__ C1, + const scalar_t* __restrict__ A, + const at::Float8_e4m3fn* __restrict__ B0, + const at::Float8_e4m3fn* __restrict__ B1, + const float* __restrict__ Bs0, + const float* __restrict__ Bs1, + scalar_t* __restrict__ Btmp, + int64_t M, + int64_t N0, + int64_t N1, + int64_t K, + int64_t block_size_N, + int64_t block_size_K) { + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB0 = div_up(N0, BLOCK_N); + const int64_t NB1 = div_up(N1, BLOCK_N); + const int64_t NB = NB0 + NB1; + + const int64_t scale_size_K = div_up(K, block_size_K); + const int64_t blocks_n_per_group = block_size_N / BLOCK_N; + + const bool use_brgemm = can_use_brgemm(M); + + // parallel on [MB, NB0 + NB1] + at::parallel_for(0, MB * NB, 0, [&](int64_t begin, int64_t end) { + int64_t mb{0}, nb{0}; + data_index_init(begin, mb, MB, nb, NB); + + int tid = at::get_thread_num(); + // for brgemm, use float32 for accumulate + alignas(64) float Ctmp[BLOCK_M * BLOCK_N]; + + for (int64_t i = begin; i < end; ++i) { + UNUSED(i); + + int mb_start = mb * BLOCK_M; + int mb_size = std::min(M - mb_start, BLOCK_M); + int nb_start = nb * BLOCK_N; + int nb_size = BLOCK_N; + + const at::Float8_e4m3fn* __restrict__ B = nb < NB0 ? B0 : B1; + const float* __restrict__ Bs = nb < NB0 ? Bs0 : Bs1; + scalar_t* __restrict__ C = nb < NB0 ? C0 : C1; + int64_t ldc = nb < NB0 ? N0 : N1; + int64_t local_nb_start = nb < NB0 ? nb_start : nb_start - N0; + int64_t new_nb = nb < NB0 ? nb : nb - NB0; + + tinygemm_kernel( + /* A */ A + mb_start * K, + /* B */ B + local_nb_start * K /* nb * BLOCK_N * K */, + /* C */ C + mb_start * ldc + local_nb_start, + /* Btmp*/ Btmp + tid * BLOCK_N * K, + /* Ctmp*/ Ctmp, + /* Bs */ Bs + (new_nb / blocks_n_per_group) * scale_size_K, + /* M */ mb_size, + /* N */ nb_size, + /* K */ K, + /* lda */ K, + /* ldb */ nb_size, + /* ldc */ ldc, + /* brg */ use_brgemm, + /* block_size_K */ block_size_K); + + // move to the next index + data_index_step(mb, MB, nb, NB); + } + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); +} + +template +inline float reduce(const scalar_t* __restrict__ x, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + fVec sum_fvec = fVec(float(0)); + +// no remainder +#pragma GCC unroll 4 + for (int64_t d = 0; d < size; d += bVec::size()) { + bVec x_bvec = bVec::loadu(x + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + sum_fvec += x_fvec0 * x_fvec0; + sum_fvec += x_fvec1 * x_fvec1; + } + return vec_reduce_sum(sum_fvec); +} + +// map2 from aten functional doesn't have fast bf16->fp32 conversion +template +inline void map2(scalar_t* y, const scalar_t* x, const scalar_t* __restrict__ w, float scale, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + fVec scale_fvec = fVec(scale); + +// no remainder +#pragma GCC unroll 4 + for (int64_t d = 0; d < size; d += bVec::size()) { + bVec x_bvec = bVec::loadu(x + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + bVec w_bvec = bVec::loadu(w + d); + fVec w_fvec0, w_fvec1; + std::tie(w_fvec0, w_fvec1) = at::vec::convert_to_float(w_bvec); + x_fvec0 = x_fvec0 * scale_fvec * w_fvec0; + x_fvec1 = x_fvec1 * scale_fvec * w_fvec1; + bVec out_bvec = convert_from_float_ext(x_fvec0, x_fvec1); + out_bvec.store(y + d); + } +} + +template +void rms_norm_kernel_impl( + scalar_t* __restrict__ input0, + scalar_t* __restrict__ input1, + const scalar_t* __restrict__ weight0, + const scalar_t* __restrict__ weight1, + int64_t M, + int64_t N0, + int64_t N1, + int64_t stride1, + float eps = 1e-5) { + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + scalar_t* x0 = input0 + m * N0; + scalar_t* x1 = input1 + m * stride1; + float scale0 = reduce(x0, N0); + float scale1 = reduce(x1, N1); + scale0 = float(1) / std::sqrt(scale0 / N0 + eps); + scale1 = float(1) / std::sqrt(scale1 / N1 + eps); + map2(x0, x0, weight0, scale0, N0); + map2(x1, x1, weight1, scale1, N1); + } + }); +} + +template +inline void rotary(const scalar_t* input, scalar_t* out, const scalar_t* cos, const scalar_t* sin, int64_t size) { + TORCH_CHECK(false, "rotary scalar path not implemented."); +} + +#if defined(CPU_CAPABILITY_AVX512) +template <> +inline void rotary( + const at::BFloat16* input, at::BFloat16* out, const at::BFloat16* cos, const at::BFloat16* sin, int64_t size) { + // permute indices + const __m512i idx1 = _mm512_set_epi32(30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0); + const __m512i idx2 = _mm512_set_epi32(31, 29, 27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1); + const __m512i idy1 = _mm512_set_epi32(23, 7, 22, 6, 21, 5, 20, 4, 19, 3, 18, 2, 17, 1, 16, 0); + const __m512i idy2 = _mm512_set_epi32(31, 15, 30, 14, 29, 13, 28, 12, 27, 11, 26, 10, 25, 9, 24, 8); + +// rotary dim is 64, just 2 iters +#pragma GCC unroll 2 + for (int64_t d = 0; d < size; d += 32) { + int64_t d2 = d >> 1; + // load coefs + __m512 vcos = CVT_BF16_TO_FP32(_mm256_loadu_si256(reinterpret_cast(cos + d2))); + __m512 vsin = CVT_BF16_TO_FP32(_mm256_loadu_si256(reinterpret_cast(sin + d2))); + // load input + __m512i a16 = _mm512_loadu_si512(reinterpret_cast(input + d)); + __m512 a = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32(a16, 0)); + __m512 b = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32(a16, 1)); + // from [16, 2] to [2, 16] + __m512 in1 = _mm512_mask_permutex2var_ps(a, 0xffff, idx1, b); + __m512 in2 = _mm512_mask_permutex2var_ps(a, 0xffff, idx2, b); + // out1 = in1 * cos - in2 * sin; + // out2 = in2 * cos + in1 * sin + __m512 out1 = _mm512_sub_ps(_mm512_mul_ps(in1, vcos), _mm512_mul_ps(in2, vsin)); + __m512 out2 = _mm512_add_ps(_mm512_mul_ps(in2, vcos), _mm512_mul_ps(in1, vsin)); + // from [2, 16] to [16, 2] + a = _mm512_mask_permutex2var_ps(out1, 0xffff, idy1, out2); + b = _mm512_mask_permutex2var_ps(out1, 0xffff, idy2, out2); + + _mm512_storeu_si512(reinterpret_cast<__m512i*>((out + d)), (__m512i)(_mm512_cvtne2ps_pbh(b, a))); + } +} +#endif + +template +void rotary_emb_kernel_impl( + scalar_t* q_pe_out, + scalar_t* k_pe_out, + const scalar_t* q_pe, + const scalar_t* k_pe, + const int64_t* pos, + const scalar_t* cos_sin, + int64_t num_seqs, + int64_t num_heads, + int64_t rotary_dim, + int64_t q_strideB, + int64_t q_strideH, + int64_t k_strideB, + int64_t oq_strideB, + int64_t oq_strideH, + int64_t ok_strideB) { + TORCH_CHECK(rotary_dim % 32 == 0, "rotary_dim is not 32x."); + const int64_t rotary_offset = rotary_dim / 2; + + // parallel on [num_seqs, num_heads + 1] + // top [num_heads] handle q_pe and bottom [1] handle k_pe + at::parallel_for(0, num_seqs * (num_heads + 1), GRAIN_SIZE / rotary_dim, [&](int64_t begin, int64_t end) { + int64_t seq{0}, head_id{0}; + data_index_init(begin, seq, num_seqs, head_id, num_heads + 1); + + for (int64_t i = begin; i < end; ++i) { + UNUSED(i); + // get cos and sin cache ptr + int64_t index = pos[seq]; + const scalar_t* cos = cos_sin + index * rotary_dim; + const scalar_t* sin = cos + rotary_offset; + + const scalar_t* input = + (head_id < num_heads) ? q_pe + seq * q_strideB + head_id * q_strideH : k_pe + seq * k_strideB; + scalar_t* out = + (head_id < num_heads) ? q_pe_out + seq * oq_strideB + head_id * oq_strideH : k_pe_out + seq * ok_strideB; + rotary(input, out, cos, sin, rotary_dim); + + // move to the next index + data_index_step(seq, num_seqs, head_id, num_heads + 1); + } + }); +} + +} // anonymous namespace + +extern at::Tensor +weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, const std::optional& bias, bool is_vnni); + +extern at::Tensor int8_scaled_mm_with_quant( + at::Tensor& mat1, + at::Tensor& mat2, + at::Tensor& scales2, + const std::optional& bias, + at::ScalarType out_dtype, + bool is_vnni); + +extern void +bmm_cpu(at::Tensor& out, at::Tensor& mat1, at::Tensor& mat2, bool is_vnni, const std::optional& scale); + +extern at::Tensor fp8_scaled_mm_cpu( + at::Tensor& mat1, + at::Tensor& mat2, + at::Tensor& scales2, + std::vector block_size, + const std::optional& bias, + at::ScalarType out_dtype, + bool is_vnni); + +// NB: shapes in DeepDeek R1 +// +// hidden_states : [num_seqs, hidden_size] [1, 7168] +// q_a_proj_weight : [q_lora_rank, hidden_size] [1536, 7168] +// q_b_proj_weight : [num_heads * qk_head_dim, q_lora_rank] [4224, 1536] +// kv_a_proj_weight : [kv_lora_rank + qk_rope_head_dim, hidden_size] [576, 7168] +// w_kc : [num_heads, kv_lora_rank, qk_nope_head_dim] [22, 512, 128] +// q_a_layernorm_weight : [q_lora_rank] [1536] +// kv_a_layernorm_weight : [kv_lora_rank] [512] +// +std::tuple qkv_proj_with_rope( + at::Tensor& hidden_states, + at::Tensor& q_a_proj_weight, + at::Tensor& q_b_proj_weight, + at::Tensor& kv_a_proj_weight, + at::Tensor& w_kc, + at::Tensor& q_a_layernorm_weight, + at::Tensor& kv_a_layernorm_weight, + at::Tensor& positions, + at::Tensor& cos_sin_cache, + double eps, + bool use_int8_w8a8, + bool use_fp8_w8a16, + std::optional q_a_proj_scale, + std::optional q_b_proj_scale, + std::optional kv_a_proj_scale, + bool is_vnni, + std::optional> block_size) { + RECORD_FUNCTION( + "sgl-kernel::qkv_proj_with_rope", + std::vector({hidden_states, q_a_proj_weight, q_b_proj_weight, kv_a_proj_weight, w_kc})); + + const auto st = hidden_states.scalar_type(); + CHECK_INPUT(hidden_states); + CHECK_INPUT(positions); + CHECK_INPUT(cos_sin_cache); + CHECK_EQ(q_a_layernorm_weight.scalar_type(), st); + CHECK_EQ(kv_a_layernorm_weight.scalar_type(), st); + CHECK_EQ(positions.scalar_type(), at::kLong); + CHECK_EQ(cos_sin_cache.scalar_type(), st); + CHECK_DIM(2, hidden_states); + CHECK_DIM(3, w_kc); + CHECK_DIM(1, q_a_layernorm_weight); + CHECK_DIM(1, kv_a_layernorm_weight); + CHECK_DIM(1, positions); + CHECK_DIM(2, cos_sin_cache); + + // skip contiguous checks for weights, expect prepacked + TORCH_CHECK(is_vnni, "qkv_proj_with_rope: expect weights are prepacked!"); + + int64_t num_seqs = hidden_states.size(0); + int64_t hidden_size = hidden_states.size(1); + int64_t q_lora_rank = q_a_proj_weight.size(0); + int64_t num_heads = w_kc.size(0); + int64_t kv_lora_rank = w_kc.size(1); + int64_t qk_head_dim = q_b_proj_weight.size(0) / num_heads; + int64_t qk_nope_head_dim = w_kc.size(2); + int64_t qk_rope_head_dim = kv_a_proj_weight.size(0) - kv_lora_rank; + int64_t rotary_dim = cos_sin_cache.size(1); + + CHECK_EQ(positions.numel(), num_seqs); + CHECK_EQ(rotary_dim, qk_rope_head_dim); + CHECK_EQ(q_a_layernorm_weight.numel(), q_lora_rank); + CHECK_EQ(kv_a_layernorm_weight.numel(), kv_lora_rank); + + // check the packed dimension + CHECK_EQ(q_a_proj_weight.size(1), get_row_size(hidden_size, use_int8_w8a8)); + CHECK_EQ(q_b_proj_weight.size(1), get_row_size(q_lora_rank, use_int8_w8a8)); + CHECK_EQ(kv_a_proj_weight.size(1), get_row_size(hidden_size, use_int8_w8a8)); + + if (use_int8_w8a8) { + TORCH_CHECK(q_a_proj_scale.has_value(), "missing q_a_proj_scale for int8 w8a8."); + TORCH_CHECK(q_b_proj_scale.has_value(), "missing q_b_proj_scale for int8 w8a8."); + TORCH_CHECK(kv_a_proj_scale.has_value(), "missing kv_a_proj_scale for int8 w8a8."); + } + if (use_fp8_w8a16) { + TORCH_CHECK(q_a_proj_scale.has_value(), "missing q_a_proj_scale for fp8 w8a16."); + TORCH_CHECK(q_b_proj_scale.has_value(), "missing q_b_proj_scale for fp8 w8a16."); + TORCH_CHECK(kv_a_proj_scale.has_value(), "missing kv_a_proj_scale for fp8 w8a16."); + TORCH_CHECK(block_size.has_value(), "missing block_size for fp8 w8a16."); + TORCH_CHECK(block_size.value().size() == 2, "block_size should be 2D for fp8 w8a16."); + } + // outputs and temp buffer + const auto options = hidden_states.options(); + auto q_input = at::empty({num_seqs, num_heads, kv_lora_rank + qk_rope_head_dim}, options); + auto k_input = at::empty({num_seqs, 1, kv_lora_rank + qk_rope_head_dim}, options); + auto v_input = k_input.narrow(-1, 0, kv_lora_rank); + + // outputs of q_a_proj and q_b_proj + auto qa = at::empty({num_seqs, q_lora_rank}, options); + + // stage 1: q_a_proj and kv_a_proj + AT_DISPATCH_REDUCED_FLOATING_TYPES(st, "qkv_proj_kernel_impl", [&] { + if (use_int8_w8a8) { + auto q_a_proj_s = q_a_proj_scale.value(); + auto kv_a_proj_s = kv_a_proj_scale.value(); + TORCH_CHECK(q_a_proj_s.numel() == q_lora_rank); + TORCH_CHECK(kv_a_proj_s.numel() == kv_lora_rank + qk_rope_head_dim); + + auto buffer = at::empty({num_seqs * hidden_size + num_seqs * 4}, options.dtype(at::kByte)); + uint8_t* __restrict__ Aq_data = buffer.data_ptr(); + float* __restrict__ As_data = (float*)((void*)(Aq_data + num_seqs * hidden_size)); + const scalar_t* __restrict__ A_data = hidden_states.data_ptr(); + + at::parallel_for(0, num_seqs, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + quantize_row_int8(Aq_data + m * hidden_size, As_data[m], A_data + m * hidden_size, hidden_size); + } + }); + + segment_gemm_kernel_impl( + qa.data_ptr(), + k_input.data_ptr(), + Aq_data, + q_a_proj_weight.data_ptr(), + kv_a_proj_weight.data_ptr(), + As_data, + q_a_proj_s.data_ptr(), + kv_a_proj_s.data_ptr(), + num_seqs, + q_lora_rank, + kv_lora_rank + qk_rope_head_dim, + hidden_size); + } else if (use_fp8_w8a16) { + int64_t block_size_N = block_size.value()[0]; + int64_t block_size_K = block_size.value()[1]; + auto q_a_proj_s = q_a_proj_scale.value(); + auto kv_a_proj_s = kv_a_proj_scale.value(); + CHECK_EQ(q_a_proj_s.size(0), div_up(q_lora_rank, block_size_N)); + CHECK_EQ(q_a_proj_s.size(1), div_up(hidden_size, block_size_K)); + CHECK_EQ(kv_a_proj_s.size(0), div_up(kv_lora_rank + qk_rope_head_dim, block_size_N)); + CHECK_EQ(kv_a_proj_s.size(1), div_up(hidden_size, block_size_K)); + + const int BLOCK_N = block_size_n(); + const int num_threads = at::get_num_threads(); + auto buffer = at::empty({num_threads, BLOCK_N * hidden_size}, options); + segment_gemm_kernel_impl( + qa.data_ptr(), + k_input.data_ptr(), + hidden_states.data_ptr(), + q_a_proj_weight.data_ptr(), + kv_a_proj_weight.data_ptr(), + q_a_proj_s.data_ptr(), + kv_a_proj_s.data_ptr(), + buffer.data_ptr(), + num_seqs, + q_lora_rank, + kv_lora_rank + qk_rope_head_dim, + hidden_size, + block_size_N, + block_size_K); + } else { + segment_gemm_kernel_impl( + qa.data_ptr(), + k_input.data_ptr(), + hidden_states.data_ptr(), + q_a_proj_weight.data_ptr(), + kv_a_proj_weight.data_ptr(), + num_seqs, + q_lora_rank, + kv_lora_rank + qk_rope_head_dim, + hidden_size); + } + }); + + // stage 2: apply rmsnorm inplace + AT_DISPATCH_REDUCED_FLOATING_TYPES(st, "rms_norm_kernel_impl", [&] { + rms_norm_kernel_impl( + qa.data_ptr(), + v_input.data_ptr(), + q_a_layernorm_weight.data_ptr(), + kv_a_layernorm_weight.data_ptr(), + num_seqs, + q_lora_rank, + kv_lora_rank, + kv_lora_rank + qk_rope_head_dim, + eps); + }); + + // stage 3: q_b_proj + at::Tensor qb; + std::optional bias; + if (use_int8_w8a8) { + qb = int8_scaled_mm_with_quant(qa, q_b_proj_weight, q_b_proj_scale.value(), bias, at::kBFloat16, is_vnni); + } else if (use_fp8_w8a16) { + qb = fp8_scaled_mm_cpu( + qa, q_b_proj_weight, q_b_proj_scale.value(), block_size.value(), bias, at::kBFloat16, is_vnni); + } else { + qb = weight_packed_linear(qa, q_b_proj_weight, bias, is_vnni); + } + qb.as_strided_({num_seqs, num_heads, qk_head_dim}, {num_heads * qk_head_dim, qk_head_dim, 1}); + + // stage 4: bmm + std::optional scale; + auto q_nope = qb.narrow(2, 0, qk_nope_head_dim).transpose_(0, 1); + auto q_nope_out = q_input.narrow(2, 0, kv_lora_rank).transpose_(0, 1); + bmm_cpu(q_nope_out, q_nope, w_kc, is_vnni, scale); + + // stage 5: rope + AT_DISPATCH_REDUCED_FLOATING_TYPES(st, "rotary_emb_kernel_impl", [&] { + rotary_emb_kernel_impl( + q_input.data_ptr() + kv_lora_rank, + k_input.data_ptr() + kv_lora_rank, + qb.data_ptr() + qk_nope_head_dim, + k_input.data_ptr() + kv_lora_rank, + positions.data_ptr(), + cos_sin_cache.data_ptr(), + num_seqs, + num_heads, + rotary_dim, + num_heads * qk_head_dim, + qk_head_dim, + kv_lora_rank + qk_rope_head_dim, + num_heads * (kv_lora_rank + qk_rope_head_dim), + kv_lora_rank + qk_rope_head_dim, + kv_lora_rank + qk_rope_head_dim); + }); + + return std::make_tuple(q_input, k_input, v_input); +} + +std::tuple qkv_proj_with_rope_fused_weight( + at::Tensor& hidden_states, + at::Tensor& qkv_a_proj_weight, + at::Tensor& q_b_proj_weight, + at::Tensor& w_kc, + at::Tensor& q_a_layernorm_weight, + at::Tensor& kv_a_layernorm_weight, + at::Tensor& positions, + at::Tensor& cos_sin_cache, + double eps, + bool use_int8_w8a8, + bool use_fp8_w8a16, + std::optional qkv_a_proj_scale, + std::optional q_b_proj_scale, + bool is_vnni, + std::optional> block_size, + int64_t q_lora_rank, + int64_t kv_lora_rank, + int64_t qk_rope_head_dim) { + RECORD_FUNCTION( + "sgl-kernel::qkv_proj_with_rope_fused_weight", + std::vector({hidden_states, qkv_a_proj_weight, q_b_proj_weight, w_kc})); + + int64_t hidden_size = hidden_states.size(1); + CHECK_EQ(qkv_a_proj_weight.size(0), q_lora_rank + kv_lora_rank + qk_rope_head_dim); + CHECK_EQ(qkv_a_proj_weight.size(1), get_row_size(hidden_size, use_int8_w8a8)); + + std::vector weight_chunks = + at::split(qkv_a_proj_weight, {q_lora_rank, kv_lora_rank + qk_rope_head_dim}, 0); + at::Tensor q_a_proj_weight = weight_chunks[0]; + at::Tensor kv_a_proj_weight = weight_chunks[1]; + at::Tensor q_a_proj_s; + at::Tensor kv_a_proj_s; + + if (use_int8_w8a8) { + TORCH_CHECK(qkv_a_proj_scale.has_value(), "missing qkv_a_proj_scale for int8 w8a8."); + std::vector scale_chunks = + at::split(qkv_a_proj_scale.value(), {q_lora_rank, kv_lora_rank + qk_rope_head_dim}, 0); + q_a_proj_s = scale_chunks[0]; + kv_a_proj_s = scale_chunks[1]; + } + if (use_fp8_w8a16) { + TORCH_CHECK(qkv_a_proj_scale.has_value(), "missing qkv_a_proj_scale for fp8 w8a16."); + int64_t block_size_N = block_size.value()[0]; + int64_t q_a_proj_s_dim0 = div_up(q_lora_rank, block_size_N); + int64_t kv_a_proj_s_dim0 = div_up(kv_lora_rank + qk_rope_head_dim, block_size_N); + std::vector scale_chunks = at::split(qkv_a_proj_scale.value(), {q_a_proj_s_dim0, kv_a_proj_s_dim0}, 0); + q_a_proj_s = scale_chunks[0]; + kv_a_proj_s = scale_chunks[1]; + } + + return qkv_proj_with_rope( + hidden_states, + q_a_proj_weight, + q_b_proj_weight, + kv_a_proj_weight, + w_kc, + q_a_layernorm_weight, + kv_a_layernorm_weight, + positions, + cos_sin_cache, + eps, + use_int8_w8a8, + use_fp8_w8a16, + q_a_proj_s, + q_b_proj_scale, + kv_a_proj_s, + is_vnni, + block_size); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/rope.cpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/rope.cpp new file mode 100644 index 00000000..1c624946 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/rope.cpp @@ -0,0 +1,346 @@ +#include "common.h" +#include "vec.h" + +namespace { + +template +void rotary_embedding_3D_kernel_impl( + scalar_t* __restrict__ query_out, + scalar_t* __restrict__ key_out, + int64_t* __restrict__ positions, + scalar_t* __restrict__ query, + scalar_t* __restrict__ key, + scalar_t* __restrict__ cos_sin_cache, + int64_t num_tokens, + int64_t num_heads, + int64_t num_kv_heads, + int64_t head_size, + int64_t rotary_dim, + int64_t query_stride_s, + int64_t query_out_stride_s, + int64_t key_out_stride_s, + int64_t key_stride_s, + int64_t query_stride_h, + int64_t query_out_stride_h) { + int64_t HR = rotary_dim; + int64_t HK = rotary_dim; + int64_t COFF = HR / 2; + at::parallel_for(0, num_tokens * num_heads, GRAIN_SIZE / rotary_dim, [&](int64_t begin, int64_t end) { + int64_t seq{0}, head_id{0}; + data_index_init(begin, seq, num_tokens, head_id, num_heads); + for (int64_t i = begin; i < end; ++i) { + int64_t in_offset_q = seq * query_stride_s + head_id * query_stride_h; + int64_t out_offset_q = seq * query_out_stride_s + head_id * query_out_stride_h; + int64_t out_offset_k = seq * key_out_stride_s; + int64_t p = 0; + scalar_t* sin_start = nullptr; + scalar_t* cos_start = nullptr; + // step 0) get the rotary position embedding for the current position + p = positions[seq]; + sin_start = cos_sin_cache + p * HR + COFF; + cos_start = cos_sin_cache + p * HR; + // step 1) apply_rotary_pos_emb for the rotary_dim elements in every + // head of query/key + for (int64_t h = 0; h < rotary_dim; h += 2) { + scalar_t cos = cos_start[h >> 1]; + scalar_t sin = sin_start[h >> 1]; + scalar_t in1 = query[in_offset_q + h]; + scalar_t in2 = query[in_offset_q + h + 1]; + scalar_t out1 = in1 * cos - in2 * sin; + scalar_t out2 = in2 * cos + in1 * sin; + query_out[out_offset_q + h] = out1; + query_out[out_offset_q + h + 1] = out2; + } + for (int64_t h = 0; h < HK; h += 2) { + scalar_t cos = cos_start[h >> 1]; + scalar_t sin = sin_start[h >> 1]; + int64_t k_pe_offset = seq * key_stride_s; + scalar_t in1_k = key[k_pe_offset + h]; + scalar_t in2_k = key[k_pe_offset + h + 1]; + scalar_t out1_k = in1_k * cos - in2_k * sin; + scalar_t out2_k = in2_k * cos + in1_k * sin; + key_out[out_offset_k + h] = out1_k; + key_out[out_offset_k + h + 1] = out2_k; + } + // move to the next index + data_index_step(seq, num_tokens, head_id, num_heads); + } + }); +} + +template +void rotary_embedding_neox_2D_kernel_impl( + int64_t* __restrict__ positions, + scalar_t* __restrict__ query, + scalar_t* __restrict__ key, + scalar_t* __restrict__ cos_sin_cache, + int64_t rotary_dim, + int64_t query_stride_s, + int64_t key_stride_s, + int64_t num_heads, + int64_t num_kv_heads, + int64_t head_size, + int64_t num_tokens) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int64_t bVecSize = bVec::size(); + + int64_t embed_dim = rotary_dim / 2; + bool flag = (embed_dim % bVecSize == 0); + int64_t loop_upper = flag ? embed_dim : embed_dim - bVecSize; + + auto compute_loop = [&](int64_t token_head, scalar_t* cache_ptr, scalar_t* qk) { + int64_t j = 0; + for (; j < loop_upper; j += bVecSize) { + int64_t rot_offset = j; + int64_t x_index = rot_offset; + int64_t y_index = embed_dim + rot_offset; + + int64_t out_x = token_head + x_index; + int64_t out_y = token_head + y_index; + + bVec _cos = bVec::loadu(cache_ptr + x_index); + bVec _sin = bVec::loadu(cache_ptr + y_index); + + bVec _q_x = bVec::loadu(qk + out_x); + bVec _q_y = bVec::loadu(qk + out_y); + fVec _cos_0, _cos_1; + std::tie(_cos_0, _cos_1) = at::vec::convert_to_float(_cos); + fVec _sin_0, _sin_1; + std::tie(_sin_0, _sin_1) = at::vec::convert_to_float(_sin); + fVec _q_x_0, _q_x_1; + std::tie(_q_x_0, _q_x_1) = at::vec::convert_to_float(_q_x); + fVec _q_y_0, _q_y_1; + std::tie(_q_y_0, _q_y_1) = at::vec::convert_to_float(_q_y); + + auto out1_0 = _q_x_0 * _cos_0 - _q_y_0 * _sin_0; + auto out1_1 = _q_x_1 * _cos_1 - _q_y_1 * _sin_1; + auto out1 = convert_from_float_ext(out1_0, out1_1); + out1.store(qk + out_x); + + auto out2_0 = _q_y_0 * _cos_0 + _q_x_0 * _sin_0; + auto out2_1 = _q_y_1 * _cos_1 + _q_x_1 * _sin_1; + auto out2 = convert_from_float_ext(out2_0, out2_1); + out2.store(qk + out_y); + } + if (!flag) { + for (; j < embed_dim; ++j) { + int64_t x_index = j; + int64_t y_index = embed_dim + j; + + int64_t out_x = token_head + x_index; + int64_t out_y = token_head + y_index; + + float _cos = cache_ptr[x_index]; + float _sin = cache_ptr[y_index]; + + float _q_x = qk[out_x]; + float _q_y = qk[out_y]; + + qk[out_x] = _q_x * _cos - _q_y * _sin; + qk[out_y] = _q_y * _cos + _q_x * _sin; + } + } + }; + +#pragma omp parallel for + for (int64_t token_idx = 0; token_idx < num_tokens; ++token_idx) { + int64_t pos = positions[token_idx]; + scalar_t* cache_ptr = cos_sin_cache + pos * rotary_dim; + + for (int64_t i = 0; i < num_heads; ++i) { + int64_t head_idx = i; + int64_t token_head = token_idx * query_stride_s + head_idx * head_size; + compute_loop(token_head, cache_ptr, query); + } + + for (int64_t i = 0; i < num_kv_heads; ++i) { + int64_t head_idx = i; + int64_t token_head = token_idx * key_stride_s + head_idx * head_size; + compute_loop(token_head, cache_ptr, key); + } + } +} + +template +void rotary_embedding_2D_kernel_impl( + int64_t* __restrict__ positions, + scalar_t* __restrict__ query, + scalar_t* __restrict__ key, + scalar_t* __restrict__ cos_sin_cache, + int64_t rotary_dim, + int64_t query_stride_s, + int64_t key_stride_s, + int64_t num_heads, + int64_t num_kv_heads, + int64_t head_size, + int64_t num_tokens) { + int64_t embed_dim = rotary_dim / 2; + + at::parallel_for(0, num_tokens * num_heads, GRAIN_SIZE / rotary_dim, [&](int64_t begin, int64_t end) { + int64_t token_idx = {0}, i = {0}; + data_index_init(begin, token_idx, num_tokens, i, num_heads); + for ([[maybe_unused]] auto z : c10::irange(begin, end)) { + int64_t pos = positions[token_idx]; + scalar_t* cache_ptr = cos_sin_cache + pos * rotary_dim; + scalar_t* cos_cache_ptr = cache_ptr; + scalar_t* sin_cache_ptr = cache_ptr + embed_dim; + int64_t head_idx = i; + int64_t token_head = token_idx * query_stride_s + head_idx * head_size; + scalar_t* head_query = token_head + query; + for (int64_t j = 0; j < embed_dim; j += 1) { + int64_t rot_offset = j; + int64_t x_index = 2 * rot_offset; + int64_t y_index = 2 * rot_offset + 1; + + float cos = cos_cache_ptr[rot_offset]; + float sin = sin_cache_ptr[rot_offset]; + + float x = head_query[x_index]; + float y = head_query[y_index]; + + head_query[x_index] = x * cos - y * sin; + head_query[y_index] = y * cos + x * sin; + } + data_index_step(token_idx, num_tokens, i, num_heads); + } + }); + + at::parallel_for(0, num_tokens * num_kv_heads, GRAIN_SIZE / rotary_dim, [&](int64_t begin, int64_t end) { + int64_t token_idx{0}, i = {0}; + data_index_init(begin, token_idx, num_tokens, i, num_kv_heads); + for ([[maybe_unused]] auto z : c10::irange(begin, end)) { + int64_t pos = positions[token_idx]; + scalar_t* cache_ptr = cos_sin_cache + pos * rotary_dim; + scalar_t* cos_cache_ptr = cache_ptr; + scalar_t* sin_cache_ptr = cache_ptr + embed_dim; + int64_t head_idx = i; + int64_t token_head = token_idx * key_stride_s + head_idx * head_size; + scalar_t* head_key = key + token_head; + for (int64_t j = 0; j < embed_dim; j += 1) { + int64_t rot_offset = j; + int64_t x_index = 2 * rot_offset; + int64_t y_index = 2 * rot_offset + 1; + + float cos = cos_cache_ptr[rot_offset]; + float sin = sin_cache_ptr[rot_offset]; + + float x = head_key[x_index]; + float y = head_key[y_index]; + + head_key[x_index] = x * cos - y * sin; + head_key[y_index] = y * cos + x * sin; + } + data_index_step(token_idx, num_tokens, i, num_kv_heads); + } + }); +} + +} // namespace + +std::tuple rotary_embedding_cpu( + at::Tensor& positions, + at::Tensor& query, + at::Tensor& key, + int64_t head_size, + at::Tensor& cos_sin_cache, + bool is_neox) { + RECORD_FUNCTION("sgl-kernel::rotary_embedding_cpu", std::vector({query, key})); + CHECK_DIM(1, positions); + const auto input_dim = query.dim(); + const auto input_dtype = query.scalar_type(); + TORCH_CHECK( + input_dim == 2 || input_dim == 3, + " Query/Key must be 2D [num_tokens, num_heads*head_size] or 3D [num_tokens, num_heads, head_size] tensor"); + CHECK_DIM(2, cos_sin_cache); + CHECK_LAST_DIM_CONTIGUOUS_INPUT(query); + CHECK_LAST_DIM_CONTIGUOUS_INPUT(key); + + int64_t rotary_dim = cos_sin_cache.size(1); + if (input_dim == 3) { + // TODO: add support for head_dim != rotary_dim case when input_dim=3 + CHECK_EQ(query.size(-1), rotary_dim); + // TODO: add support for kv_head != 1 + CHECK_EQ(key.size(1), 1); + } + + int64_t num_tokens = positions.numel(); + CHECK_EQ(key.size(0), num_tokens); + CHECK_EQ(query.size(0), num_tokens); + + TORCH_CHECK(positions.scalar_type() == at::kLong, "expect positions to be int64, got ", positions.scalar_type()); + TORCH_CHECK(input_dtype == key.scalar_type(), "query and key must have the same data type"); + TORCH_CHECK(input_dtype == cos_sin_cache.scalar_type(), "query and cos_sin_cache must have the same data type"); + + int64_t num_heads = input_dim == 2 ? query.size(-1) / head_size : query.size(1); + int64_t num_kv_heads = input_dim == 2 ? key.size(-1) / head_size : key.size(1); + int64_t key_stride_s = key.stride(0); + int64_t query_stride_s = query.stride(0); + + // input stride of num head dim is meaningful only when input dim = 3 + int64_t query_stride_h = input_dim == 3 ? query.stride(1) : -1; + at::Tensor query_out = at::empty_like(query); + at::Tensor key_out = at::empty_like(key); + int64_t query_out_stride_s = query_out.stride(0); + int64_t key_out_stride_s = key_out.stride(0); + // output stride of num head dim is meaningful only when input dim = 3 + int64_t query_out_stride_h = input_dim == 3 ? query_out.stride(1) : -1; + + AT_DISPATCH_REDUCED_FLOATING_TYPES(input_dtype, "rotary_embedding_cpu", [&] { + if (input_dim == 2) { + if (is_neox) { + rotary_embedding_neox_2D_kernel_impl( + positions.data_ptr(), + query.data_ptr(), + key.data_ptr(), + cos_sin_cache.data_ptr(), + rotary_dim, + query_stride_s, + key_stride_s, + num_heads, + num_kv_heads, + head_size, + num_tokens); + } else { + rotary_embedding_2D_kernel_impl( + positions.data_ptr(), + query.data_ptr(), + key.data_ptr(), + cos_sin_cache.data_ptr(), + rotary_dim, + query_stride_s, + key_stride_s, + num_heads, + num_kv_heads, + head_size, + num_tokens); + } + query_out = query; + key_out = key; + + } else { + TORCH_CHECK( + is_neox == false, " Query/Key with 3D [num_tokens, num_heads, head_size] does not support neox rope yet"); + // TODO: add neox style support for rope impl with 3D inputs + rotary_embedding_3D_kernel_impl( + query_out.data_ptr(), + key_out.data_ptr(), + positions.data_ptr(), + query.data_ptr(), + key.data_ptr(), + cos_sin_cache.data_ptr(), + num_tokens, + num_heads, + num_kv_heads, + head_size, + rotary_dim, + query_stride_s, + query_out_stride_s, + key_out_stride_s, + key_stride_s, + query_stride_h, + query_out_stride_h); + } + }); + return std::make_tuple(query_out, key_out); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/shm.cpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/shm.cpp new file mode 100644 index 00000000..1bf65a9b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/shm.cpp @@ -0,0 +1,666 @@ +#include "shm.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// states for collectives +enum coll_state { + coll_begin = 0, + coll_allreduce_naive__copy_in_done, + coll_allreduce_naive__reduce_done, + // alternative state when allreduce is working on alternative buffer + // of the double buffer. + coll_alt1_allreduce_naive__copy_in_done, + coll_alt2_allreduce_naive__copy_in_done, + coll_alt1_allreduce_naive__reduce_done, + coll_allgather_naive__copy_in_done, + coll_alt1_allgather_naive__copy_in_done, + coll_alt2_allgather_naive__copy_in_done, +}; + +// SHM building blocks +struct SharedData { + const char* name; + int descriptor; + void* bytes; + size_t nbytes; +}; + +void shared_open(SharedData* data, const char* name, size_t nbytes) { + int d = shm_open(name, O_RDWR, S_IRUSR | S_IWUSR); + if (d != -1) { + void* bytes = mmap(NULL, nbytes, PROT_READ | PROT_WRITE, MAP_SHARED, d, 0); + data->name = name; + data->descriptor = d; + data->bytes = bytes; + data->nbytes = nbytes; + } else { + if (errno != ENOENT) { + // don't print if shm can not be found because we want to loop over from + // caller again until the other ranks created the shm + printf("shared_open %s failed, errno=%d\n", name, errno); + } + data->descriptor = -1; + } +} + +void shared_create(SharedData* data, const char* name, void* bytes, size_t nbytes) { + int d = shm_open(name, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR); + if (d != -1) { + nbytes = write(d, bytes, nbytes); + if (nbytes > 0) { + shared_open(data, name, nbytes); + } + } else { + printf("shared_create %s failed\n", name); + } +} + +static int world_size; + +// SHM based allreduce helper functions +// buffer that holds shm name +#define NAME_BUF_SIZE 1000 +#define MAX_BUF_SIZE 1048576 * 32 +#define NAIVE_ALLREDUCE_THRESHOLD 1048576 +#define SHM_BUFFER_NAME "deepspeed_allreduce_buffer" +struct allreduce_workspace { + enum coll_state states[2]; // idx=0 -- state for symmetric_naive_all_reduce + // idx=1 -- state for distributed_naive_all_reduce + // double buffer to avoid syncing between rounds + // offset=0 -- 2*NAIVE_ALLREDUCE_THRESHOLD : buffer for + // symmetric_naive_all_reduce after that : buffer for + // distributed_naive_all_reduce + char buffer[2 * NAIVE_ALLREDUCE_THRESHOLD + 2 * MAX_BUF_SIZE]; +}; + +#define BUFFER0_OFFSET(current_buffer) current_buffer* NAIVE_ALLREDUCE_THRESHOLD +#define BUFFER1_OFFSET(current_buffer) 2 * NAIVE_ALLREDUCE_THRESHOLD + current_buffer* MAX_BUF_SIZE + +struct allreduce_workspace** workspace; + +// buffer for small messages, double buffer +char** symmetric_buffer[2]; +// buffer for large messages, double buffer +char** distributed_buffer[2]; + +void wait_buffer_state_until_2(int index, enum coll_state state0, enum coll_state state1, int state_group) { + volatile enum coll_state* state_ptr = &(workspace[index]->states[state_group]); + + while (1) { + volatile enum coll_state cur_state = *state_ptr; + if (cur_state == state0 || cur_state == state1) break; + } +} + +__m512 cvt_bf16_to_fp32(const __m256i src) __attribute__((target("avx512bw"))); +inline __m512 cvt_bf16_to_fp32(const __m256i src) { + auto y = _mm512_cvtepu16_epi32(src); + return _mm512_castsi512_ps(_mm512_bslli_epi128(y, 2)); +} + +inline __m256i cvt_fp32_to_bf16(const __m512 src) __attribute__((target("avx512bw"))); +inline __m256i cvt_fp32_to_bf16(const __m512 src) { + __m512i value = _mm512_castps_si512(src); + __m512i nan = _mm512_set1_epi32(0xffff); + auto mask_value = _mm512_cmp_ps_mask(src, src, _CMP_ORD_Q); + __m512i ones = _mm512_set1_epi32(0x1); + __m512i vec_bias = _mm512_set1_epi32(0x7fff); + // uint32_t lsb = (input >> 16) & 1; + auto t_value = _mm512_and_si512(_mm512_srli_epi32(value, 16), ones); + // uint32_t rounding_bias = 0x7fff + lsb; + t_value = _mm512_add_epi32(t_value, vec_bias); + // input += rounding_bias; + t_value = _mm512_add_epi32(t_value, value); + // input = input >> 16; + t_value = _mm512_srli_epi32(t_value, 16); + // Check NaN before converting back to bf16 + t_value = _mm512_mask_blend_epi32(mask_value, nan, t_value); + return _mm512_cvtusepi32_epi16(t_value); +} + +__m512 cvt_fp16_to_fp32(const __m256i src) __attribute__((target("avx512bw"))); +inline __m512 cvt_fp16_to_fp32(const __m256i src) { + return _mm512_cvtph_ps(src); +} + +inline __m256i cvt_fp32_to_fp16(const __m512 src) __attribute__((target("avx512bw"))); +inline __m256i cvt_fp32_to_fp16(const __m512 src) { + return _mm512_cvtps_ph(src, (_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)); +} + +void reduce_bf16_buffers(int start_elements, int num_elements, char* to_buffer, char** buffers) + __attribute__((target("avx512bw"))); + +void reduce_fp16_buffers(int start_elements, int num_elements, char* to_buffer, char** buffers) + __attribute__((target("avx512bw"))); + +void reduce_fp32_buffers(int start_elements, int num_elements, char* to_buffer, char** buffers) + __attribute__((target("avx512bw"))); + +void reduce_all_buffers( + int start_elements, + int num_elements, + c10::ScalarType scalar_type, + int to_buffer_idx, + char* to_buffer, + char** buffers) { + switch (scalar_type) { + case c10::ScalarType::BFloat16: + reduce_bf16_buffers(start_elements, num_elements, to_buffer, buffers); + break; + case c10::ScalarType::Half: + reduce_fp16_buffers(start_elements, num_elements, to_buffer, buffers); + break; + case c10::ScalarType::Float: + reduce_fp32_buffers(start_elements, num_elements, to_buffer, buffers); + break; + default: + assert(!"Should not get here"); + } +} + +#define CVT_ADD_BF16(x) \ + do { \ + auto in##x##_val = cvt_bf16_to_fp32(_mm256_loadu_si256((__m256i*)(buffers[x] + i))); \ + inout_val = _mm512_add_ps(inout_val, in##x##_val); \ + } while (0) + +// Reduce functions down below use vectorized algorithm, the number of bytes +// processed each iteration depends on vector length. 256bit vector ==> 32 +// bytes, 512bit vector ==> 64 bytes If you change implementation of +// reduce_bf16_buffers, etc. , check whether this number needs to be changed +#define VECTOR_LENGTH_IN_BYTES 32 + +void reduce_bf16_buffers(int start_elements, int num_elements, char* to_buffer, char** buffers) { + const int element_size = 2; + const int vector_length = VECTOR_LENGTH_IN_BYTES / element_size; + int main_elements = num_elements - (num_elements % vector_length); + int remain_elements = num_elements % vector_length; + + // process aligned part +#pragma omp parallel for + for (int i = start_elements * element_size; i < (start_elements + main_elements) * element_size; + i += VECTOR_LENGTH_IN_BYTES) { + auto inout_val = cvt_bf16_to_fp32(_mm256_loadu_si256((__m256i*)(buffers[0] + i))); + switch (world_size) { + case 16: + CVT_ADD_BF16(15); + case 15: + CVT_ADD_BF16(14); + case 14: + CVT_ADD_BF16(13); + case 13: + CVT_ADD_BF16(12); + case 12: + CVT_ADD_BF16(11); + case 11: + CVT_ADD_BF16(10); + case 10: + CVT_ADD_BF16(9); + case 9: + CVT_ADD_BF16(8); + case 8: + CVT_ADD_BF16(7); + case 7: + CVT_ADD_BF16(6); + case 6: + CVT_ADD_BF16(5); + case 5: + CVT_ADD_BF16(4); + case 4: + CVT_ADD_BF16(3); + case 3: + CVT_ADD_BF16(2); + case 2: + CVT_ADD_BF16(1); + case 1: + break; + default: + for (int j = 1; j < world_size; j++) { + auto in_val = cvt_bf16_to_fp32(_mm256_loadu_si256((__m256i*)(buffers[j] + i))); + inout_val = _mm512_add_ps(inout_val, in_val); + } + } + _mm256_storeu_si256((__m256i*)(to_buffer + i), cvt_fp32_to_bf16(inout_val)); + } + + // process remaining part + int i = (start_elements + main_elements) * element_size; + while (remain_elements > 0) { + float val = 0.0f; + for (int j = 0; j < world_size; j++) { + val += *(at::BFloat16*)(buffers[j] + i); + } + *(at::BFloat16*)(to_buffer + i) = val; + remain_elements--; + i += element_size; + } +} + +#define CVT_ADD_FP16(x) \ + do { \ + auto in##x##_val = cvt_fp16_to_fp32(_mm256_loadu_si256((__m256i*)(buffers[x] + i))); \ + inout_val = _mm512_add_ps(inout_val, in##x##_val); \ + } while (0) + +void reduce_fp16_buffers(int start_elements, int num_elements, char* to_buffer, char** buffers) { + const int element_size = 2; + const int vector_length = VECTOR_LENGTH_IN_BYTES / element_size; + int main_elements = num_elements - (num_elements % vector_length); + int remain_elements = num_elements % vector_length; + + // process aligned part +#pragma omp parallel for + for (int i = start_elements * element_size; i < (start_elements + main_elements) * element_size; + i += VECTOR_LENGTH_IN_BYTES) { + auto inout_val = cvt_fp16_to_fp32(_mm256_loadu_si256((__m256i*)(buffers[0] + i))); + switch (world_size) { + case 16: + CVT_ADD_FP16(15); + case 15: + CVT_ADD_FP16(14); + case 14: + CVT_ADD_FP16(13); + case 13: + CVT_ADD_FP16(12); + case 12: + CVT_ADD_FP16(11); + case 11: + CVT_ADD_FP16(10); + case 10: + CVT_ADD_FP16(9); + case 9: + CVT_ADD_FP16(8); + case 8: + CVT_ADD_FP16(7); + case 7: + CVT_ADD_FP16(6); + case 6: + CVT_ADD_FP16(5); + case 5: + CVT_ADD_FP16(4); + case 4: + CVT_ADD_FP16(3); + case 3: + CVT_ADD_FP16(2); + case 2: + CVT_ADD_FP16(1); + case 1: + break; + default: + for (int j = 1; j < world_size; j++) { + auto in_val = cvt_fp16_to_fp32(_mm256_loadu_si256((__m256i*)(buffers[j] + i))); + inout_val = _mm512_add_ps(inout_val, in_val); + } + } + _mm256_storeu_si256((__m256i*)(to_buffer + i), cvt_fp32_to_fp16(inout_val)); + } + + // process remaining part + int i = (start_elements + main_elements) * element_size; + while (remain_elements > 0) { + float val = 0.0f; + for (int j = 0; j < world_size; j++) { + val += *(at::Half*)(buffers[j] + i); + } + *(at::Half*)(to_buffer + i) = val; + remain_elements--; + i += element_size; + } +} + +#define CVT_ADD_F32(x) \ + do { \ + auto in##x##_val = _mm256_loadu_ps((float*)(buffers[x] + i)); \ + inout_val = _mm256_add_ps(inout_val, in##x##_val); \ + } while (0) + +void reduce_fp32_buffers(int start_elements, int num_elements, char* to_buffer, char** buffers) { + const int element_size = 4; + const int vector_length = VECTOR_LENGTH_IN_BYTES / element_size; + int main_elements = num_elements - (num_elements % vector_length); + int remain_elements = num_elements % vector_length; + + // process aligned part +#pragma omp parallel for + for (int i = start_elements * element_size; i < (start_elements + main_elements) * element_size; + i += VECTOR_LENGTH_IN_BYTES) { + auto inout_val = _mm256_loadu_ps((float*)(buffers[0] + i)); + switch (world_size) { + case 16: + CVT_ADD_F32(15); + case 15: + CVT_ADD_F32(14); + case 14: + CVT_ADD_F32(13); + case 13: + CVT_ADD_F32(12); + case 12: + CVT_ADD_F32(11); + case 11: + CVT_ADD_F32(10); + case 10: + CVT_ADD_F32(9); + case 9: + CVT_ADD_F32(8); + case 8: + CVT_ADD_F32(7); + case 7: + CVT_ADD_F32(6); + case 6: + CVT_ADD_F32(5); + case 5: + CVT_ADD_F32(4); + case 4: + CVT_ADD_F32(3); + case 3: + CVT_ADD_F32(2); + case 2: + CVT_ADD_F32(1); + case 1: + break; + default: + for (int j = 1; j < world_size; j++) { + auto in_val = _mm256_loadu_ps((float*)(buffers[j] + i)); + inout_val = _mm256_add_ps(inout_val, in_val); + } + } + _mm256_storeu_ps((float*)(to_buffer + i), inout_val); + } + + // process remaining part + int i = (start_elements + main_elements) * element_size; + while (remain_elements > 0) { + float val = 0.0f; + for (int j = 0; j < world_size; j++) { + val += *(float*)(buffers[j] + i); + } + *(float*)(to_buffer + i) = val; + remain_elements--; + i += element_size; + } +} + +static bool is_initialized = false; +static int world_rank; + +void shm_initialize(int size, int rank, const char* addr_string, const char* port_string) { + if (is_initialized) { + return; + } + is_initialized = true; + + world_size = size; + world_rank = rank; + + char shm_name_prefix[NAME_BUF_SIZE]; + char shm_name[NAME_BUF_SIZE]; + snprintf(shm_name_prefix, NAME_BUF_SIZE, "%s_%d_%s_%s", SHM_BUFFER_NAME, getuid(), addr_string, port_string); + // create shared workspace for SHM based allreduce + SharedData allreduce_buffer; + // allocate workspace_buf for current rank + struct allreduce_workspace* workspace_buf; + struct allreduce_workspace* workspace_buf_other; + workspace_buf = (struct allreduce_workspace*)malloc(sizeof(struct allreduce_workspace)); + snprintf(shm_name, NAME_BUF_SIZE, "%.900s_%d", shm_name_prefix, rank); + shared_create(&allreduce_buffer, shm_name, workspace_buf, sizeof(struct allreduce_workspace)); + workspace_buf = (struct allreduce_workspace*)allreduce_buffer.bytes; + workspace_buf->states[0] = coll_alt2_allreduce_naive__copy_in_done; + workspace_buf->states[1] = coll_begin; + + // create the workspace pointer list + workspace = (struct allreduce_workspace**)malloc(size * sizeof(struct allreduce_workspace*)); + symmetric_buffer[0] = (char**)malloc(size * sizeof(char**)); + symmetric_buffer[1] = (char**)malloc(size * sizeof(char**)); + distributed_buffer[0] = (char**)malloc(size * sizeof(char**)); + distributed_buffer[1] = (char**)malloc(size * sizeof(char**)); + + // map shm of all ranks + for (int i = 0; i < size; i++) { + if (i != rank) { + snprintf(shm_name, NAME_BUF_SIZE, "%.900s_%d", shm_name_prefix, i); + // printf("open %s, %d\n", shm_name, rank); + do { + shared_open(&allreduce_buffer, shm_name, sizeof(struct allreduce_workspace)); + } while (allreduce_buffer.descriptor == -1 && errno == ENOENT); + workspace_buf_other = (struct allreduce_workspace*)allreduce_buffer.bytes; + workspace[i] = workspace_buf_other; + } else { + workspace[i] = workspace_buf; + } + symmetric_buffer[0][i] = workspace[i]->buffer + BUFFER0_OFFSET(0); + symmetric_buffer[1][i] = workspace[i]->buffer + BUFFER0_OFFSET(1); + distributed_buffer[0][i] = workspace[i]->buffer + BUFFER1_OFFSET(0); + distributed_buffer[1][i] = workspace[i]->buffer + BUFFER1_OFFSET(1); + } +} + +static void parallel_memcpy(void* to, void* from, size_t n_bytes) __attribute__((target("avx512bw"))); +static void parallel_memcpy(void* to, void* from, size_t n_bytes) { + auto aligned_bytes = n_bytes - (n_bytes % VECTOR_LENGTH_IN_BYTES); + // process aligned part +#pragma omp parallel for + for (size_t i = 0; i < aligned_bytes; i += VECTOR_LENGTH_IN_BYTES) { + auto val = _mm256_loadu_si256((__m256i*)((char*)from + i)); + _mm256_storeu_si256((__m256i*)((char*)to + i), val); + } + + // process remaining part + for (size_t i = aligned_bytes; i < n_bytes; i++) { + *((char*)to + i) = *((char*)from + i); + } +} + +#define positive_mod(num, mod) ((((num) % (mod)) + (mod)) % (mod)) +#define rank_mod(rank) positive_mod(rank, world_size) +size_t slice_size(size_t chunk_el, int slice_idx) { + size_t slice_size = chunk_el / world_size; + return slice_idx == world_size - 1 ? slice_size + (chunk_el % world_size) : slice_size; +} + +char* slice_data(char* data_ptr, size_t chunk_el, int el_size, int slice_idx) { + size_t slice_size = chunk_el / world_size; + size_t el_offset = slice_size * slice_idx; + return data_ptr + el_offset * el_size; +} + +size_t slice_el_start(size_t chunk_el, int slice_idx) { + size_t slice_size = chunk_el / world_size; + return slice_size * slice_idx; +} + +void symmetric_naive_all_reduce(char* data_ptr, c10::ScalarType scalar_type, size_t chunk_size, size_t chunk_el) { + const int state_group = 0; + static int current_buffer = 0; + static int state_idx = 0; + + // init states to case 0 to get rid of "maybe-uninitialized" warning. + enum coll_state copy_current = coll_allreduce_naive__copy_in_done; + enum coll_state copy_next = coll_alt1_allreduce_naive__copy_in_done; + + switch (state_idx) { + case 0: + copy_current = coll_allreduce_naive__copy_in_done; + copy_next = coll_alt1_allreduce_naive__copy_in_done; + break; + case 1: + copy_current = coll_alt1_allreduce_naive__copy_in_done; + copy_next = coll_alt2_allreduce_naive__copy_in_done; + break; + case 2: + copy_current = coll_alt2_allreduce_naive__copy_in_done; + copy_next = coll_allreduce_naive__copy_in_done; + break; + default: + assert(!"Should not get here."); + } + state_idx = (state_idx + 1) % 3; + + parallel_memcpy(symmetric_buffer[current_buffer][world_rank], data_ptr, chunk_size); + std::atomic_thread_fence(std::memory_order_release); + workspace[world_rank]->states[state_group] = copy_current; + + for (int i = 0; i < world_size; i++) { + // wait until the other rank copy the buffer + if (i != world_rank) { + wait_buffer_state_until_2(i, copy_current, copy_next, state_group); + } + } + + // each rank reduce the buffer independently so therre is no need for + // synchronization afterward + reduce_all_buffers(0, chunk_el, scalar_type, world_rank, data_ptr, symmetric_buffer[current_buffer]); + + // switch buffer + current_buffer = 1 - current_buffer; +} + +// naive allreduce distributed, each rank do naive reduce on its slice +void distributed_naive_reduce(char* data_ptr, c10::ScalarType scalar_type, size_t chunk_size, size_t chunk_el) { + const int state_group = 1; + static int current_buffer = 0; + static int state_idx = 0; + + // init states to case 0 to get rid of "maybe-uninitialized" warning. + enum coll_state copy_current = coll_allreduce_naive__copy_in_done; + enum coll_state reduce_current = coll_allreduce_naive__reduce_done; + enum coll_state copy_next = coll_alt1_allreduce_naive__copy_in_done; + + // similar to symmetric_naive_allreduce, but here we only need two sets of + // states, because distributed naive reduce has two barriers in the algorithm + switch (state_idx) { + case 0: + copy_current = coll_allreduce_naive__copy_in_done; + reduce_current = coll_allreduce_naive__reduce_done; + copy_next = coll_alt1_allreduce_naive__copy_in_done; + break; + case 1: + copy_current = coll_alt1_allreduce_naive__copy_in_done; + reduce_current = coll_alt1_allreduce_naive__reduce_done; + copy_next = coll_allreduce_naive__copy_in_done; + break; + default: + assert(!"Should not get here."); + } + state_idx = (state_idx + 1) % 2; + + int data_size = chunk_size / chunk_el; + parallel_memcpy(distributed_buffer[current_buffer][world_rank], data_ptr, chunk_size); + std::atomic_thread_fence(std::memory_order_release); + workspace[world_rank]->states[state_group] = copy_current; + + for (int i = 0; i < world_size; i++) { + // wait until all the other ranks copy the buffer + if (i != world_rank) wait_buffer_state_until_2(i, copy_current, reduce_current, state_group); + } + + // reduce scatter + reduce_all_buffers( + slice_el_start(chunk_el, world_rank), + slice_size(chunk_el, world_rank), + scalar_type, + world_rank, + distributed_buffer[current_buffer][world_rank], + distributed_buffer[current_buffer]); + std::atomic_thread_fence(std::memory_order_release); + workspace[world_rank]->states[state_group] = reduce_current; + + for (int i = 0; i < world_size; i++) { + // wait until all the other ranks reduce the buffer + if (i != world_rank) wait_buffer_state_until_2(i, reduce_current, copy_next, state_group); + } + + for (int i = 0; i < world_size; i++) { + int rank = (i + world_rank) % world_size; + parallel_memcpy( + slice_data(data_ptr, chunk_el, data_size, rank), + slice_data(distributed_buffer[current_buffer][rank], chunk_el, chunk_size / chunk_el, rank), + slice_size(chunk_el, rank) * data_size); + } + + current_buffer = 1 - current_buffer; +} + +void all_reduce_outer_loop(torch::Tensor& data, size_t numel, int data_size) { + for (int offset = 0; offset < data_size; offset += MAX_BUF_SIZE) { + auto data_ptr = ((char*)(data.data_ptr()) + offset); + size_t chunk_size = data_size - offset > MAX_BUF_SIZE ? MAX_BUF_SIZE : data_size - offset; + size_t chunk_el = chunk_size / (data_size / numel); + if (chunk_size < NAIVE_ALLREDUCE_THRESHOLD) { + symmetric_naive_all_reduce(data_ptr, data.scalar_type(), chunk_size, chunk_el); + } else { + distributed_naive_reduce(data_ptr, data.scalar_type(), chunk_size, chunk_el); + } + } +} + +void naive_all_gather(char* result_ptr, char* data_ptr, size_t res_stride, size_t chunk_size, size_t chunk_el) { + const int state_group = 1; + static int current_buffer = 0; + static int state_idx = 0; + + // init states to case 0 to get rid of "maybe-uninitialized" warning. + enum coll_state copy_current = coll_allgather_naive__copy_in_done; + enum coll_state copy_next = coll_alt1_allgather_naive__copy_in_done; + + switch (state_idx) { + case 0: + copy_current = coll_allgather_naive__copy_in_done; + copy_next = coll_alt1_allgather_naive__copy_in_done; + break; + case 1: + copy_current = coll_alt1_allgather_naive__copy_in_done; + copy_next = coll_alt2_allgather_naive__copy_in_done; + break; + case 2: + copy_current = coll_alt2_allgather_naive__copy_in_done; + copy_next = coll_allgather_naive__copy_in_done; + break; + default: + assert(!"Should not get here."); + } + state_idx = (state_idx + 1) % 3; + + parallel_memcpy(distributed_buffer[current_buffer][world_rank], data_ptr, chunk_size); + std::atomic_thread_fence(std::memory_order_release); + workspace[world_rank]->states[state_group] = copy_current; + + for (int i = 0; i < world_size; i++) { + // wait until all the other ranks copy the buffer + if (i != world_rank) wait_buffer_state_until_2(i, copy_current, copy_next, state_group); + } + for (int i = 0; i < world_size; i++) { + parallel_memcpy(result_ptr + i * res_stride, distributed_buffer[current_buffer][i], chunk_size); + } + current_buffer = 1 - current_buffer; +} + +torch::Tensor& all_gather(torch::Tensor& result, torch::Tensor& data, int dim, size_t numel, int data_size) { + size_t dim_el = data.stride(dim) * data.size(dim); + int dtype_size = data_size / numel; + size_t dim_size = dim_el * dtype_size; + int dim_count = data_size / dim_size; + auto data_ptr = (char*)(data.data_ptr()); + auto result_ptr = (char*)(result.data_ptr()); + for (int i = 0; i < dim_count; i++) { + for (size_t offset = 0; offset < dim_size; offset += MAX_BUF_SIZE) { + size_t chunk_size = dim_size - offset > MAX_BUF_SIZE ? MAX_BUF_SIZE : dim_size - offset; + size_t chunk_el = chunk_size / dtype_size; + naive_all_gather( + result_ptr + i * dim_size * world_size + offset, + data_ptr + i * dim_size + offset, + dim_size, + chunk_size, + chunk_el); + } + } + return result; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/shm.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/shm.h new file mode 100644 index 00000000..3e903972 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/shm.h @@ -0,0 +1,11 @@ +#include + +#include + +#ifndef __SHM_COLLECTIVES__ +#define __SHM_COLLECTIVES__ +#define VECTOR_LENGTH_IN_BYTES 32 +void shm_initialize(int size, int rank, const char* addr_string, const char* port_string); +void all_reduce_outer_loop(torch::Tensor& data, size_t numel, int data_size); +torch::Tensor& all_gather(torch::Tensor& result, torch::Tensor& data, int dim, size_t numel, int data_size); +#endif diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/topk.cpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/topk.cpp new file mode 100644 index 00000000..b4bdcd0b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/topk.cpp @@ -0,0 +1,665 @@ +#include "common.h" +#include "vec.h" + +namespace { + +template +inline void softmax(float* __restrict__ out, const scalar_t* __restrict__ input) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + + constexpr int kVecSize = bVec::size(); + + // step 1: get max + fVec max_fvec = fVec(-std::numeric_limits::infinity()); + if constexpr (SIZE < kVecSize) { + // SIZE = 1, 2, 4, 8, 16; only the top half is used + bVec x_bvec = bVec::loadu(input, SIZE); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + x_fvec0 = fVec::set(max_fvec, x_fvec0, SIZE); + max_fvec = at::vec::maximum(max_fvec, x_fvec0); + x_fvec0.store(out, SIZE); + } else { + for (int d = 0; d < SIZE; d += kVecSize) { + bVec x_bvec = bVec::loadu(input + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + max_fvec = at::vec::maximum(max_fvec, x_fvec0); + max_fvec = at::vec::maximum(max_fvec, x_fvec1); + x_fvec0.store(out + d); + x_fvec1.store(out + d + fVec::size()); + } + } + float max_val = vec_reduce_max(max_fvec); + max_fvec = fVec(max_val); + + // step 2: sum of (x - max).exp() + fVec sum_fvec = fVec(float(0)); + if constexpr (SIZE < fVec::size()) { + // SIZE = 1, 2, 4, 8 + fVec x_fvec = (fVec::loadu(out, SIZE) - max_fvec).exp_u20(); + x_fvec = fVec::set(sum_fvec, x_fvec, SIZE); + sum_fvec += x_fvec; + x_fvec.store(out, SIZE); + } else { + for (int d = 0; d < SIZE; d += fVec::size()) { + fVec x_fvec = (fVec::loadu(out + d) - max_fvec).exp_u20(); + sum_fvec += x_fvec; + x_fvec.store(out + d); + } + } + float sum_val = vec_reduce_sum(sum_fvec); + + // step 3: x * (1 / sum) + sum_fvec = fVec(1.f / sum_val); + if constexpr (SIZE < fVec::size()) { + // SIZE = 1, 2, 4, 8 + fVec out_fvec = fVec::loadu(out, SIZE) * sum_fvec; + out_fvec.store(out, SIZE); + } else { + for (int d = 0; d < SIZE; d += fVec::size()) { + fVec out_fvec = fVec::loadu(out + d) * sum_fvec; + out_fvec.store(out + d); + } + } +} + +template +void grouped_topk_kernel_impl( + float* __restrict__ topk_weights, + int32_t* __restrict__ topk_ids, + const scalar_t* __restrict__ gating_output, + int64_t num_tokens, + int64_t topk, + int64_t num_groups, + int64_t topk_group, + bool renormalize) { + const int64_t num_experts_per_group = NUM_EXPERTS / num_groups; + at::parallel_for(0, num_tokens, 0, [&](int64_t begin, int64_t end) { + alignas(64) float scores[NUM_EXPERTS]; + + using elem_t = std::pair; + std::vector queue(num_groups); + std::vector queue2(topk_group * num_experts_per_group); + + for (int64_t i = begin; i < end; ++i) { + // do softmax to get scores + softmax(scores, gating_output + i * NUM_EXPERTS); + + // find max score per group + for (int64_t g = 0; g < num_groups; ++g) { + float gmax = -std::numeric_limits::infinity(); + for (int64_t e = 0; e < num_experts_per_group; ++e) { + gmax = std::max(gmax, scores[g * num_experts_per_group + e]); + } + queue[g] = {gmax, g}; + } + + // find group topk + std::partial_sort( + queue.begin(), queue.begin() + topk_group, queue.end(), [](const elem_t& x, const elem_t& y) -> bool { + return x.first > y.first; + }); + + for (int64_t g = 0; g < topk_group; ++g) { + int32_t group_idx = queue[g].second; + for (int64_t e = 0; e < num_experts_per_group; ++e) { + int32_t expert_idx = group_idx * num_experts_per_group + e; + queue2[g * num_experts_per_group + e] = {scores[expert_idx], expert_idx}; + } + } + + // find global topk + std::partial_sort( + queue2.begin(), queue2.begin() + topk, queue2.end(), [](const elem_t& x, const elem_t& y) -> bool { + return x.first > y.first; + }); + + for (int64_t j = 0; j < topk; ++j) { + topk_weights[i * topk + j] = queue2[j].first; + topk_ids[i * topk + j] = queue2[j].second; + } + + if (renormalize) { + float sum = 0.f; + for (int64_t j = 0; j < topk; ++j) { + sum += topk_weights[i * topk + j]; + } + float scale = 1.f / sum; + for (int64_t j = 0; j < topk; ++j) { + topk_weights[i * topk + j] *= scale; + } + } + } + }); +} + +template +inline void sigmoid(float* __restrict__ out, const scalar_t* __restrict__ input) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + + const fVec one = fVec(1.f); + + constexpr int kVecSize = bVec::size(); + for (int d = 0; d < SIZE; d += kVecSize) { + bVec x_bvec = bVec::loadu(input + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + x_fvec0 = one / (one + x_fvec0.neg().exp_u20()); + x_fvec1 = one / (one + x_fvec1.neg().exp_u20()); + + x_fvec0.store(out + d); + x_fvec1.store(out + d + fVec::size()); + } +} + +template +void topk_sigmoid_kernel_impl( + float* __restrict__ topk_weights, + int32_t* __restrict__ topk_ids, + const scalar_t* __restrict__ gating_output, + int64_t num_tokens, + int64_t topk, + bool renormalize) { + using Vec = at::vec::Vectorized; + const int64_t num_experts_per_group = NUM_EXPERTS; + at::parallel_for(0, num_tokens, 0, [&](int64_t begin, int64_t end) { + alignas(64) float scores[NUM_EXPERTS]; + using elem_t = std::pair; + std::vector queue(num_experts_per_group); + + for (int64_t i = begin; i < end; ++i) { + at::vec::convert(gating_output + i * NUM_EXPERTS, scores, NUM_EXPERTS); + + float gmax = at::vec::reduce_all( + [](Vec& x, Vec& y) { return at::vec::maximum(x, y); }, scores, num_experts_per_group); + + // find position of first max, + // note that we may have multiple max values. + int first_max_idx = -1; + for (int64_t e = 0; e < num_experts_per_group; ++e) { + if (scores[e] == gmax) { + first_max_idx = e; + break; + } + } + + // scalar sigmoid + topk_weights[i] = 1.0 / (1.0 + exp(0.0 - gmax)); + topk_ids[i] = first_max_idx; + + if (renormalize) { + float sum = 0.f; + for (int64_t j = 0; j < topk; ++j) { + sum += topk_weights[i * topk + j]; + } + float scale = 1.f / sum; + for (int64_t j = 0; j < topk; ++j) { + topk_weights[i * topk + j] *= scale; + } + } + } + }); +} + +template +void topk_softmax_kernel_impl( + float* __restrict__ topk_weights, + int32_t* __restrict__ topk_ids, + const scalar_t* __restrict__ gating_output, + int64_t num_tokens, + int64_t topk, + bool renormalize) { + const int64_t num_experts_per_group = NUM_EXPERTS; + at::parallel_for(0, num_tokens, 0, [&](int64_t begin, int64_t end) { + alignas(64) float scores[NUM_EXPERTS]; + using elem_t = std::pair; + std::vector queue(num_experts_per_group); + + for (int64_t i = begin; i < end; ++i) { + softmax(scores, gating_output + i * NUM_EXPERTS); + + for (int64_t e = 0; e < num_experts_per_group; ++e) { + queue[e] = {scores[e], e}; + } + + std::partial_sort( + queue.begin(), + queue.begin() + num_experts_per_group, + queue.end(), + [](const elem_t& x, const elem_t& y) -> bool { return x.first > y.first; }); + + for (int64_t j = 0; j < topk; ++j) { + topk_weights[i * topk + j] = queue[j].first; + topk_ids[i * topk + j] = queue[j].second; + } + + if (renormalize) { + float sum = 0.f; + for (int64_t j = 0; j < topk; ++j) { + sum += topk_weights[i * topk + j]; + } + float scale = 1.f / sum; + for (int64_t j = 0; j < topk; ++j) { + topk_weights[i * topk + j] *= scale; + } + } + } + }); +} + +template +inline void +apply_bias(float* __restrict__ scores2, const float* __restrict__ scores, const param_t* __restrict__ bias) { + using fVec = at::vec::Vectorized; + using bVec = at::vec::Vectorized; + auto vec_size = bVec::size(); + int d = 0; + for (; d <= SIZE - vec_size; d += vec_size) { + fVec bias0, bias1, x0, x1; + std::tie(bias0, bias1) = load_float_vec2(bias + d); + std::tie(x0, x1) = load_float_vec2(scores + d); + x0 = x0 + bias0; + x1 = x1 + bias1; + x0.store(scores2 + d); + x1.store(scores2 + d + fVec::size()); + } + for (; d < SIZE; d++) { + scores2[d] = scores[d] + (float)bias[d]; + } +} + +template +void biased_grouped_topk_kernel_impl( + float* __restrict__ topk_weights, + int32_t* __restrict__ topk_ids, + const scalar_t* __restrict__ gating_output, + const param_t* __restrict__ bias, + int64_t num_tokens, + int64_t num_groups, + int64_t topk_group, + bool renormalize) { + using Vec = at::vec::Vectorized; + + const int64_t num_experts_per_group = NUM_EXPERTS / num_groups; + at::parallel_for(0, num_tokens, 0, [&](int64_t begin, int64_t end) { + // scores: sigmoid + alignas(64) float scores[NUM_EXPERTS]; + // scores for choice: sigmoid + bias + alignas(64) float scores2[NUM_EXPERTS]; + + using elem_t = std::pair; + std::vector queue(num_groups); + std::vector queue2(topk_group * num_experts_per_group); + + for (int64_t i = begin; i < end; ++i) { + // do sigmoid to get scores + sigmoid(scores, gating_output + i * NUM_EXPERTS); + + apply_bias(scores2, scores, bias); + + for (int64_t g = 0; g < num_groups; ++g) { + // find the max + float gmax = at::vec::reduce_all( + [](Vec& x, Vec& y) { return at::vec::maximum(x, y); }, + scores2 + g * num_experts_per_group, + num_experts_per_group); + + // find position of first max, + // note that we may have multiple max values. + int first_max_idx = -1; + for (int64_t e = 0; e < num_experts_per_group; ++e) { + if (scores2[g * num_experts_per_group + e] == gmax) { + first_max_idx = g * num_experts_per_group + e; + break; + } + } + + // find the 2nd max + scores2[first_max_idx] = -std::numeric_limits::infinity(); + float gmax2 = at::vec::reduce_all( + [](Vec& x, Vec& y) { return at::vec::maximum(x, y); }, + scores2 + g * num_experts_per_group, + num_experts_per_group); + // restore scores for choice + scores2[first_max_idx] = gmax; + + queue[g] = {gmax + gmax2, g}; + } + + // find group topk + std::partial_sort( + queue.begin(), queue.begin() + topk_group, queue.end(), [](const elem_t& x, const elem_t& y) -> bool { + return x.first > y.first; + }); + + for (int64_t g = 0; g < topk_group; ++g) { + int32_t group_idx = queue[g].second; + for (int64_t e = 0; e < num_experts_per_group; ++e) { + int32_t expert_idx = group_idx * num_experts_per_group + e; + queue2[g * num_experts_per_group + e] = {scores2[expert_idx], expert_idx}; + } + } + + // find global topk + std::partial_sort( + queue2.begin(), queue2.begin() + TOPK, queue2.end(), [](const elem_t& x, const elem_t& y) -> bool { + return x.first > y.first; + }); + + for (int j = 0; j < TOPK; ++j) { + int32_t index = queue2[j].second; + topk_ids[i * TOPK + j] = index; + topk_weights[i * TOPK + j] = scores[index]; + } + +#if defined(CPU_CAPABILITY_AVX512) + if (renormalize) { + __mmask16 mask = (1ULL << TOPK) - 1; + __m512 x = _mm512_maskz_loadu_ps(mask, topk_weights + i * TOPK); + float sum = _mm512_reduce_add_ps(x); + __m512 vscale = _mm512_set1_ps(1.f / sum); + __m512 y = _mm512_mul_ps(x, vscale); + _mm512_mask_storeu_ps(topk_weights + i * TOPK, mask, y); + } +#else + if (renormalize) { + float sum = 0.f; + for (int64_t j = 0; j < TOPK; ++j) { + sum += topk_weights[i * TOPK + j]; + } + float scale = 1.f / sum; + for (int64_t j = 0; j < TOPK; ++j) { + topk_weights[i * TOPK + j] *= scale; + } + } +#endif + } + }); +} + +#define LAUNCH_GROUPED_TOPK_KERNEL(NE) \ + grouped_topk_kernel_impl( \ + topk_weights.data_ptr(), \ + topk_ids.data_ptr(), \ + gating_output.data_ptr(), \ + num_tokens, \ + topk, \ + num_expert_group, \ + topk_group, \ + renormalize); + +#define LAUNCH_TOPK_SIGMOID_KERNEL(NE) \ + topk_sigmoid_kernel_impl( \ + topk_weights.data_ptr(), \ + topk_ids.data_ptr(), \ + gating_output.data_ptr(), \ + num_tokens, \ + topk, \ + renormalize); + +#define LAUNCH_TOPK_SOFTMAX_KERNEL(NE) \ + topk_softmax_kernel_impl( \ + topk_weights.data_ptr(), \ + topk_ids.data_ptr(), \ + gating_output.data_ptr(), \ + num_tokens, \ + topk, \ + renormalize); + +#define LAUNCH_BIASED_GROUPED_TOPK_KERNEL(NE, NTOPK) \ + biased_grouped_topk_kernel_impl( \ + topk_weights.data_ptr(), \ + topk_ids.data_ptr(), \ + gating_output.data_ptr(), \ + correction_bias.data_ptr(), \ + num_tokens, \ + num_expert_group, \ + topk_group, \ + renormalize); + +} // anonymous namespace + +std::tuple +topk_sigmoid_cpu(at::Tensor& hidden_states, at::Tensor& gating_output, int64_t topk, bool renormalize) { + RECORD_FUNCTION("sgl-kernel::topk_sigmoid_cpu", std::vector({hidden_states, gating_output})); + CHECK_INPUT(gating_output); + + const auto st = hidden_states.scalar_type(); + CHECK_EQ(gating_output.scalar_type(), st); + + int64_t num_tokens = hidden_states.size(0); + int64_t num_experts = gating_output.size(1); + TORCH_CHECK(gating_output.size(0) == num_tokens, "Number of tokens mismatch"); + TORCH_CHECK(topk == 1, "topk_sigmoid only supports topk=1 case"); + at::Tensor topk_weights = at::empty({num_tokens, topk}, hidden_states.options().dtype(at::kFloat)); + at::Tensor topk_ids = at::empty({num_tokens, topk}, hidden_states.options().dtype(at::kInt)); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(st, "topk_sigmoid_kernel", [&] { + switch (num_experts) { + case 1: + LAUNCH_TOPK_SIGMOID_KERNEL(1); + break; + case 2: + LAUNCH_TOPK_SIGMOID_KERNEL(2); + break; + case 4: + LAUNCH_TOPK_SIGMOID_KERNEL(4); + break; + case 8: + LAUNCH_TOPK_SIGMOID_KERNEL(8); + break; + case 16: + LAUNCH_TOPK_SIGMOID_KERNEL(16); + break; + case 32: + LAUNCH_TOPK_SIGMOID_KERNEL(32); + break; + case 64: + LAUNCH_TOPK_SIGMOID_KERNEL(64); + break; + case 128: + LAUNCH_TOPK_SIGMOID_KERNEL(128); + break; + case 160: + LAUNCH_TOPK_SIGMOID_KERNEL(160); + break; + case 256: + LAUNCH_TOPK_SIGMOID_KERNEL(256); + break; + default: + TORCH_CHECK(false, "Unexpected num_experts: ", num_experts); + } + }); + return std::make_tuple(topk_weights, topk_ids); +} + +std::tuple +topk_softmax_cpu(at::Tensor& hidden_states, at::Tensor& gating_output, int64_t topk, bool renormalize) { + RECORD_FUNCTION("sgl-kernel::topk_softmax_cpu", std::vector({hidden_states, gating_output})); + CHECK_INPUT(gating_output); + + const auto st = hidden_states.scalar_type(); + CHECK_EQ(gating_output.scalar_type(), st); + + int64_t num_tokens = hidden_states.size(0); + int64_t num_experts = gating_output.size(1); + TORCH_CHECK(gating_output.size(0) == num_tokens, "Number of tokens mismatch"); + + at::Tensor topk_weights = at::empty({num_tokens, topk}, hidden_states.options().dtype(at::kFloat)); + at::Tensor topk_ids = at::empty({num_tokens, topk}, hidden_states.options().dtype(at::kInt)); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(st, "topk_softmax_cpu", [&] { + switch (num_experts) { + case 1: + LAUNCH_TOPK_SOFTMAX_KERNEL(1); + break; + case 2: + LAUNCH_TOPK_SOFTMAX_KERNEL(2); + break; + case 4: + LAUNCH_TOPK_SOFTMAX_KERNEL(4); + break; + case 8: + LAUNCH_TOPK_SOFTMAX_KERNEL(8); + break; + case 16: + LAUNCH_TOPK_SOFTMAX_KERNEL(16); + break; + case 32: + LAUNCH_TOPK_SOFTMAX_KERNEL(32); + break; + case 64: + LAUNCH_TOPK_SOFTMAX_KERNEL(64); + break; + case 128: + LAUNCH_TOPK_SOFTMAX_KERNEL(128); + break; + case 160: + LAUNCH_TOPK_SOFTMAX_KERNEL(160); + break; + case 256: + LAUNCH_TOPK_SOFTMAX_KERNEL(256); + break; + default: + TORCH_CHECK(false, "Unexpected num_experts: ", num_experts); + } + }); + return std::make_tuple(topk_weights, topk_ids); +} + +// grouped topk for DeepSeek V2 +std::tuple grouped_topk_cpu( + at::Tensor& hidden_states, + at::Tensor& gating_output, + int64_t topk, + bool renormalize, + int64_t num_expert_group, + int64_t topk_group, + int64_t num_fused_shared_experts, + std::optional routed_scaling_factor, + std::optional num_token_non_padded) { + // TODO: Will support num_fused_shared_experts, routed_scaling_factor and num_token_non_padded. + // For now, we just check them as default value. + TORCH_CHECK( + num_fused_shared_experts == 0, + "num_fused_shared_experts must be 0 default value, got: ", + num_fused_shared_experts); + TORCH_CHECK( + !routed_scaling_factor.has_value() || routed_scaling_factor.value() == 1.0f, + "routed_scaling_factor must be None or 1.0f default value, got: ", + routed_scaling_factor.value()); + TORCH_CHECK( + !num_token_non_padded.has_value(), + "num_token_non_padded must be None default value, got: ", + num_token_non_padded.value()); + + RECORD_FUNCTION("sgl-kernel::grouped_topk_cpu", std::vector({hidden_states, gating_output})); + CHECK_INPUT(gating_output); + + const auto st = hidden_states.scalar_type(); + CHECK_EQ(gating_output.scalar_type(), st); + + int64_t num_tokens = hidden_states.size(0); + int64_t num_experts = gating_output.size(1); + TORCH_CHECK(gating_output.size(0) == num_tokens, "Number of tokens mismatch"); + at::Tensor topk_weights = at::empty({num_tokens, topk}, hidden_states.options().dtype(at::kFloat)); + at::Tensor topk_ids = at::empty({num_tokens, topk}, hidden_states.options().dtype(at::kInt)); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(st, "grouped_topk_kernel", [&] { + switch (num_experts) { + case 1: + LAUNCH_GROUPED_TOPK_KERNEL(1); + break; + case 2: + LAUNCH_GROUPED_TOPK_KERNEL(2); + break; + case 4: + LAUNCH_GROUPED_TOPK_KERNEL(4); + break; + case 8: + LAUNCH_GROUPED_TOPK_KERNEL(8); + break; + case 16: + LAUNCH_GROUPED_TOPK_KERNEL(16); + break; + case 32: + LAUNCH_GROUPED_TOPK_KERNEL(32); + break; + case 64: + LAUNCH_GROUPED_TOPK_KERNEL(64); + break; + case 128: + LAUNCH_GROUPED_TOPK_KERNEL(128); + break; + case 160: + LAUNCH_GROUPED_TOPK_KERNEL(160); + break; + case 256: + LAUNCH_GROUPED_TOPK_KERNEL(256); + break; + default: + TORCH_CHECK(false, "Unexpected num_experts: ", num_experts); + } + }); + return std::make_tuple(topk_weights, topk_ids); +} + +// biased grouped topk DeepSeek V3/R1 +std::tuple biased_grouped_topk_cpu( + at::Tensor& hidden_states, + at::Tensor& gating_output, + at::Tensor& correction_bias, + int64_t topk, + bool renormalize, + int64_t num_expert_group, + int64_t topk_group, + int64_t num_fused_shared_experts, + std::optional routed_scaling_factor, + std::optional num_token_non_padded) { + // TODO: Will support num_fused_shared_experts, routed_scaling_factor and num_token_non_padded. + // For now, we just check them as default value. + TORCH_CHECK( + num_fused_shared_experts == 0, + "num_fused_shared_experts must be 0 default value, got: ", + num_fused_shared_experts); + TORCH_CHECK( + !num_token_non_padded.has_value(), + "num_token_non_padded must be None default value, got: ", + num_token_non_padded.value()); + + RECORD_FUNCTION( + "sgl-kernel::biased_grouped_topk_cpu", std::vector({hidden_states, gating_output, correction_bias})); + + CHECK_INPUT(gating_output); + CHECK_INPUT(correction_bias); + + const auto st = hidden_states.scalar_type(); + CHECK_EQ(gating_output.scalar_type(), st); + + int64_t num_tokens = hidden_states.size(0); + int64_t num_experts = gating_output.size(1); + TORCH_CHECK(gating_output.size(0) == num_tokens, "Number of tokens mismatch"); + TORCH_CHECK(correction_bias.numel() == num_experts, "Bias shape mismatch"); + at::Tensor topk_weights = at::empty({num_tokens, topk}, hidden_states.options().dtype(at::kFloat)); + at::Tensor topk_ids = at::empty({num_tokens, topk}, hidden_states.options().dtype(at::kInt)); + + CPU_DISPATCH_REDUCED_FLOATING_TYPES_EXT(st, correction_bias.scalar_type(), "biased_grouped_topk_kernel", [&] { + TORCH_CHECK(topk == 8, "Unexpected topk: ", topk); + switch (num_experts) { + case 256: + LAUNCH_BIASED_GROUPED_TOPK_KERNEL(256, 8); + break; + case 384: + LAUNCH_BIASED_GROUPED_TOPK_KERNEL(384, 8); + break; + default: + TORCH_CHECK(false, "Unexpected num_experts: ", num_experts); + } + }); + return std::make_tuple(topk_weights, topk_ids); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/torch_extension_cpu.cpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/torch_extension_cpu.cpp new file mode 100644 index 00000000..e00bc319 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/torch_extension_cpu.cpp @@ -0,0 +1,378 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "sgl_kernel_ops.h" +#include "shm.h" + +// silu_and_mul +at::Tensor silu_and_mul_cpu(at::Tensor& input); + +// gelu_and_mul +at::Tensor gelu_tanh_and_mul_cpu(const at::Tensor& input); +at::Tensor gelu_and_mul_cpu(const at::Tensor& input); + +// l2norm +at::Tensor l2norm_cpu(at::Tensor& input, double eps); + +// rmsnorm +at::Tensor rmsnorm_cpu(at::Tensor& input, at::Tensor& weight, double eps); + +// qwen3_next_rmsnorm_gated +at::Tensor fused_rmsnorm_gated_cpu(at::Tensor& input, at::Tensor& weight, at::Tensor& gate, double eps); + +// fused_add_rmsnorm +void fused_add_rmsnorm_cpu(at::Tensor& input, at::Tensor& residual, at::Tensor& weight, double eps); + +// topk +std::tuple +topk_sigmoid_cpu(at::Tensor& hidden_states, at::Tensor& gating_output, int64_t topk, bool renormalize); +std::tuple +topk_softmax_cpu(at::Tensor& hidden_states, at::Tensor& gating_output, int64_t topk, bool renormalize); + +std::tuple grouped_topk_cpu( + at::Tensor& hidden_states, + at::Tensor& gating_output, + int64_t topk, + bool renormalize, + int64_t num_expert_group, + int64_t topk_group, + int64_t num_fused_shared_experts, + std::optional routed_scaling_factor, + std::optional num_token_non_padded); + +std::tuple biased_grouped_topk_cpu( + at::Tensor& hidden_states, + at::Tensor& gating_output, + at::Tensor& correction_bias, + int64_t topk, + bool renormalize, + int64_t num_expert_group, + int64_t topk_group, + int64_t num_fused_shared_experts, + std::optional routed_scaling_factor, + std::optional num_token_non_padded); + +// attention +void decode_attention_cpu( + at::Tensor& query, + at::Tensor& k_cache, + at::Tensor& v_cache, + at::Tensor& output, + at::Tensor& key, + at::Tensor& value, + at::Tensor& loc, + at::Tensor& attn_logits, + at::Tensor& req_to_token, + at::Tensor& req_pool_indices, + at::Tensor& seq_lens, + double sm_scale, + double logit_cap); + +void extend_attention_cpu( + at::Tensor& q_extend, + at::Tensor& k_extend, + at::Tensor& v_extend, + at::Tensor& o_extend, + at::Tensor& k_buffer, + at::Tensor& v_buffer, + at::Tensor& req_to_token, + at::Tensor& req_pool_indices, + at::Tensor& seq_lens, + at::Tensor& extend_seq_lens, + at::Tensor& extend_start_loc, + int64_t max_len_extend, + double sm_scale, + double logit_cap); + +// weight prepack +at::Tensor convert_weight_packed(at::Tensor& weight); + +// quant +std::tuple per_token_quant_int8_cpu(at::Tensor& A); + +// gemm +at::Tensor +weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, const std::optional& bias, bool is_vnni); + +// igemm +at::Tensor int8_scaled_mm_cpu( + at::Tensor& mat1, + at::Tensor& mat2, + at::Tensor& scales1, + at::Tensor& scales2, + const std::optional& bias, + at::ScalarType out_dtype, + bool is_vnni); + +// fp8 gemm +at::Tensor fp8_scaled_mm_cpu( + at::Tensor& mat1, + at::Tensor& mat2, + at::Tensor& scales2, + std::vector block_size, + const std::optional& bias, + at::ScalarType out_dtype, + bool is_vnni); + +// quant + igemm +at::Tensor int8_scaled_mm_with_quant( + at::Tensor& mat1, + at::Tensor& mat2, + at::Tensor& scales2, + const std::optional& bias, + at::ScalarType out_dtype, + bool is_vnni); + +// bmm +void bmm_cpu(at::Tensor& out, at::Tensor& mat1, at::Tensor& mat2, bool is_vnni, const std::optional& scale); + +// fused moe +at::Tensor fused_experts_cpu( + at::Tensor& hidden_states, + at::Tensor& w1, + at::Tensor& w2, + at::Tensor& topk_weights, + at::Tensor& topk_ids, + bool inplace, + bool use_int8_w8a8, + bool use_fp8_w8a16, + const std::optional& w1_scale, + const std::optional& w2_scale, + const std::optional> block_size, + const std::optional& a1_scale, + const std::optional& a2_scale, + bool is_vnni); + +at::Tensor shared_expert_cpu( + at::Tensor& hidden_states, + at::Tensor& w1, + at::Tensor& w2, + at::Tensor& fused_experts_out, + double routed_scaling_factor, + bool inplace, + bool use_int8_w8a8, + bool use_fp8_w8a16, + const std::optional& w1_scale, + const std::optional& w2_scale, + const std::optional> block_size, + const std::optional& a1_scale, + const std::optional& a2_scale, + bool is_vnni); + +// weight absorption +std::tuple qkv_proj_with_rope( + at::Tensor& hidden_states, + at::Tensor& q_a_proj_weight, + at::Tensor& q_b_proj_weight, + at::Tensor& kv_a_proj_weight, + at::Tensor& w_kc, + at::Tensor& q_a_layernorm_weight, + at::Tensor& kv_a_layernorm_weight, + at::Tensor& positions, + at::Tensor& cos_sin_cache, + double eps, + bool use_int8_w8a8, + bool use_fp8_w8a16, + std::optional q_a_proj_scale, + std::optional q_b_proj_scale, + std::optional kv_a_proj_scale, + bool is_vnni, + std::optional> block_size); + +std::tuple qkv_proj_with_rope_fused_weight( + at::Tensor& hidden_states, + at::Tensor& qkv_a_proj_weight, + at::Tensor& q_b_proj_weight, + at::Tensor& w_kc, + at::Tensor& q_a_layernorm_weight, + at::Tensor& kv_a_layernorm_weight, + at::Tensor& positions, + at::Tensor& cos_sin_cache, + double eps, + bool use_int8_w8a8, + bool use_fp8_w8a16, + std::optional qkv_a_proj_scale, + std::optional q_b_proj_scale, + bool is_vnni, + std::optional> block_size, + int64_t q_lora_rank, + int64_t kv_lora_rank, + int64_t qk_rope_head_dim); + +// shared memory init +void initialize(int64_t size, int64_t rank); + +// shared mmeory all_reduce +void shm_allreduce(at::Tensor& data, int64_t op); + +// shared memory all_gather +at::Tensor shm_allgather(at::Tensor& data, int64_t dim); + +// rope +std::tuple rotary_embedding_cpu( + at::Tensor& positions, + at::Tensor& query, + at::Tensor& key, + int64_t head_size, + at::Tensor& cos_sin_cache, + bool is_neox); + +// CPU and memory binding +std::string init_cpu_threads_env(const std::string& cpu_ids); + +TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) { + // activation + m.def("silu_and_mul_cpu(Tensor input) -> Tensor"); + m.impl("silu_and_mul_cpu", torch::kCPU, &silu_and_mul_cpu); + m.def("gelu_tanh_and_mul_cpu(Tensor input) -> Tensor"); + m.impl("gelu_tanh_and_mul_cpu", torch::kCPU, &gelu_tanh_and_mul_cpu); + m.def("gelu_and_mul_cpu(Tensor input) -> Tensor"); + m.impl("gelu_and_mul_cpu", torch::kCPU, &gelu_and_mul_cpu); + + // norm + m.def("rmsnorm_cpu(Tensor input, Tensor weight, float eps) -> Tensor"); + m.impl("rmsnorm_cpu", torch::kCPU, &rmsnorm_cpu); + m.def("l2norm_cpu(Tensor input, float eps) -> Tensor"); + m.impl("l2norm_cpu", torch::kCPU, &l2norm_cpu); + m.def("fused_rmsnorm_gated_cpu(Tensor input, Tensor weight, Tensor gate, float eps) -> Tensor"); + m.impl("fused_rmsnorm_gated_cpu", torch::kCPU, &fused_rmsnorm_gated_cpu); + m.def("fused_add_rmsnorm_cpu(Tensor(a!) input, Tensor residual, Tensor weight, float eps) -> ()"); + m.impl("fused_add_rmsnorm_cpu", torch::kCPU, &fused_add_rmsnorm_cpu); + + // topk + m.def("topk_sigmoid_cpu(Tensor hidden_states, Tensor gating_output, int topk, bool renormalize) -> (Tensor, Tensor)"); + m.impl("topk_sigmoid_cpu", torch::kCPU, &topk_sigmoid_cpu); + m.def("topk_softmax_cpu(Tensor hidden_states, Tensor gating_output, int topk, bool renormalize) -> (Tensor, Tensor)"); + m.impl("topk_softmax_cpu", torch::kCPU, &topk_softmax_cpu); + m.def( + "grouped_topk_cpu(Tensor hidden_states, Tensor gating_output, int topk, bool renormalize, int num_expert_group, " + "int topk_group, int num_fused_shared_experts, float? routed_scaling_factor, Tensor? num_token_non_padded) -> " + "(Tensor, Tensor)"); + m.impl("grouped_topk_cpu", torch::kCPU, &grouped_topk_cpu); + + // biased group topk + m.def( + "biased_grouped_topk_cpu(Tensor hidden_states, Tensor gating_output, Tensor correction_bias, int topk, bool " + "renormalize, int num_expert_group, int topk_group, int num_fused_shared_experts, float? routed_scaling_factor, " + "Tensor? num_token_non_padded) -> (Tensor, Tensor)"); + m.impl("biased_grouped_topk_cpu", torch::kCPU, &biased_grouped_topk_cpu); + + // decode + m.def( + "decode_attention_cpu(Tensor query, Tensor k_cache, Tensor v_cahce, Tensor(a!) output, Tensor key, Tensor value, " + "Tensor loc, Tensor attn_logits, Tensor req_to_token, Tensor req_pool_indices, Tensor seq_lens, float sm_scale, " + "float logit_cap) -> ()"); + m.impl("decode_attention_cpu", torch::kCPU, &decode_attention_cpu); + + // extend + m.def( + "extend_attention_cpu(Tensor q_extend, Tensor k_extend, Tensor v_extend, Tensor(a!) o_extend, Tensor k_buffer, " + "Tensor v_buffer, Tensor req_to_token, Tensor req_pool_indices, Tensor seq_lens, Tensor extend_seq_lens, Tensor " + "extend_start_loc, int max_len_extend, float sm_scale, float logit_cap) -> ()"); + m.impl("extend_attention_cpu", torch::kCPU, &extend_attention_cpu); + + // weight prepack + m.def("convert_weight_packed(Tensor weight) -> Tensor"); + m.impl("convert_weight_packed", torch::kCPU, &convert_weight_packed); + + // quant + m.def("per_token_quant_int8_cpu(Tensor A) -> (Tensor, Tensor)"); + m.impl("per_token_quant_int8_cpu", torch::kCPU, &per_token_quant_int8_cpu); + + // gemm + m.def("weight_packed_linear(Tensor mat1, Tensor mat2, Tensor? bias, bool is_vnni) -> Tensor"); + m.impl("weight_packed_linear", torch::kCPU, &weight_packed_linear); + + // igemm + m.def( + "int8_scaled_mm_cpu(Tensor mat1, Tensor mat2, Tensor scales1, Tensor scales2, Tensor? bias, ScalarType " + "out_dtype, bool is_vnni) -> Tensor"); + m.impl("int8_scaled_mm_cpu", torch::kCPU, &int8_scaled_mm_cpu); + + // fp8 gemm + m.def( + "fp8_scaled_mm_cpu(Tensor mat1, Tensor mat2, Tensor scales2, int[] block_size, Tensor? bias, ScalarType " + "out_dtype, bool is_vnni) -> Tensor"); + m.impl("fp8_scaled_mm_cpu", torch::kCPU, &fp8_scaled_mm_cpu); + + // quant + igemm + m.def( + "int8_scaled_mm_with_quant(Tensor mat1, Tensor mat2, Tensor scales2, Tensor? bias, ScalarType out_dtype, bool " + "is_vnni) -> Tensor"); + m.impl("int8_scaled_mm_with_quant", torch::kCPU, &int8_scaled_mm_with_quant); + + // bmm + m.def("bmm_cpu(Tensor(a!) out, Tensor mat1, Tensor mat2, bool is_vnni, Tensor? scale) -> ()"); + m.impl("bmm_cpu", torch::kCPU, &bmm_cpu); + + // moe + m.def( + "fused_experts_cpu(Tensor hidden_states, Tensor w1, Tensor w2, Tensor topk_weights, Tensor topk_ids, bool " + "inplace, bool use_int8_w8a8, bool use_fp8_w8a16, Tensor? w1_scale, Tensor? w2_scale, int[]? block_size, Tensor? " + "a1_scale, Tensor? a2_scale, bool " + "is_vnni) -> Tensor"); + m.impl("fused_experts_cpu", torch::kCPU, &fused_experts_cpu); + + // weight absorption + m.def( + "qkv_proj_with_rope(Tensor hidden_states, Tensor q_a_proj_weight, Tensor q_b_proj_weight, Tensor " + "kv_a_proj_weight, Tensor w_kc, Tensor q_a_layernorm_weight, Tensor kv_a_layernorm_weight, Tensor positions, " + "Tensor cos_sin_cache, float eps, bool use_int8_w8a8, bool use_fp8_w8a16, Tensor? q_a_proj_scale, Tensor? " + "q_b_proj_scale, Tensor? " + "kv_a_proj_scale, bool is_vnni, int[]? block_size) -> (Tensor, Tensor, Tensor)"); + m.impl("qkv_proj_with_rope", torch::kCPU, &qkv_proj_with_rope); + m.def( + "qkv_proj_with_rope_fused_weight(Tensor hidden_states, Tensor qkv_a_proj_weight, Tensor q_b_proj_weight, " + "Tensor w_kc, Tensor q_a_layernorm_weight, Tensor kv_a_layernorm_weight, Tensor positions, " + "Tensor cos_sin_cache, float eps, bool use_int8_w8a8, bool use_fp8_w8a16, Tensor? qkv_a_proj_scale, Tensor? " + "q_b_proj_scale," + "bool is_vnni, int[]? block_size, int q_lora_rank, int kv_lora_rank," + "int qk_rope_head_dim) -> (Tensor, Tensor, Tensor)"); + m.impl("qkv_proj_with_rope_fused_weight", torch::kCPU, &qkv_proj_with_rope_fused_weight); + + // shared expert + m.def( + "shared_expert_cpu(Tensor hidden_states, Tensor w1, Tensor w2, Tensor fused_experts_out, float " + "routed_scaling_factor, bool inplace, bool use_int8_w8a8, bool use_fp8_w8a16, Tensor? w1_scale, Tensor? " + "w2_scale, int[]? block_size, Tensor? a1_scale, Tensor? a2_scale, bool is_vnni) -> Tensor"); + m.impl("shared_expert_cpu", torch::kCPU, &shared_expert_cpu); + + // all reduce + m.def("initialize(int size, int rank) -> ()"); + m.def("shm_allreduce(Tensor(a!) data, int reduce_op) -> ()"); + m.impl("shm_allreduce", torch::kCPU, &shm_allreduce); + m.def("shm_allgather(Tensor data, int dim) -> Tensor"); + m.impl("shm_allgather", torch::kCPU, &shm_allgather); + + // rope + m.def( + "rotary_embedding_cpu(Tensor positions, Tensor query, Tensor key, int head_size, Tensor cos_sin_cache, " + "bool is_neox) -> (Tensor, Tensor)"); + m.impl("rotary_embedding_cpu", torch::kCPU, &rotary_embedding_cpu); + + // CPU and memory binding + m.def("init_cpu_threads_env(str cpu_ids) -> str"); +} + +TORCH_LIBRARY_IMPL(sgl_kernel, CatchAll, m) { + m.impl("init_cpu_threads_env", init_cpu_threads_env); + m.impl("initialize", &initialize); +} + +REGISTER_EXTENSION(common_ops) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/vec.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/vec.h new file mode 100644 index 00000000..d0ec53d8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cpu/vec.h @@ -0,0 +1,304 @@ +#pragma once + +#if defined(__AVX512F__) && defined(__AVX512BF16__) && defined(__AMX_BF16__) +#define CPU_CAPABILITY_AVX512 +#endif + +#include +#include + +namespace { + +using namespace at::vec; + +template , int> = 0> +inline Vectorized convert_from_float_ext(const Vectorized& a, const Vectorized& b) { + return at::vec::convert_from_float(a, b); +} + +// allow f16, bf16 +template , int> = 1> +inline std::tuple, Vectorized> load_float_vec2(const scalar_t* __restrict__ data) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + bVec x_vec = bVec::loadu(data); + fVec x0, x1; + std::tie(x0, x1) = at::vec::convert_to_float(x_vec); + return std::make_tuple(x0, x1); +} + +// allow f32 +inline std::tuple, Vectorized> load_float_vec2(const float* __restrict__ data) { + using fVec = at::vec::Vectorized; + fVec x0 = fVec::loadu(data); + fVec x1 = fVec::loadu(data + fVec::size()); + return std::make_tuple(x0, x1); +} + +#if defined(CPU_CAPABILITY_AVX512) + +// `at::vec::convert_from_float<>` from PyTorch doesn't have avx512-bf16 intrinsics +// use native instruction for bfloat16->float32 conversion +template <> +inline Vectorized +convert_from_float_ext(const Vectorized& a, const Vectorized& b) { + return (__m512i)(_mm512_cvtne2ps_pbh(__m512(b), __m512(a))); +} + +#define CVT_BF16_TO_FP32(a) _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(a), 16)) + +#define CVT_FP16_TO_FP32(a) _mm512_cvtph_ps(a) + +// this doesn't handle NaN. +inline __m512bh cvt_e4m3_bf16_intrinsic_no_nan(__m256i fp8_vec) { + const __m512i x = _mm512_cvtepu8_epi16(fp8_vec); + __m512i combined = _mm512_add_epi16(x, _mm512_set1_epi16(0x0780)); + combined = _mm512_slli_epi16(combined, 4); + combined = _mm512_and_si512(combined, _mm512_set1_epi16(0x87f0)); + combined = _mm512_add_epi16(combined, _mm512_set1_epi16(0x3c00)); + + const __mmask32 is_nonzero = _mm512_cmpneq_epi16_mask(x, _mm512_setzero_si512()); + return (__m512bh)_mm512_maskz_mov_epi16(is_nonzero, combined); +} + +inline __m512bh cvt_e4m3_bf16_intrinsic_without_denorm(__m256i fp8_vec) { + // The following conversion is without denorm behavior, that is to say, + // Max subnorm : S.0000.111 = 0.875 ∗ 2**(−6) + // Min subnorm : S.0000.001 = 2**(−9) + // 0.0019 ~ 0.0137 cannot be converted correctly. + __m512i x = _mm512_cvtepu8_epi16(fp8_vec); + auto mask = _mm512_cmpneq_epi16_mask( + _mm512_and_si512(x, _mm512_set1_epi16(127)), + _mm512_setzero_si512()); // mask = x & 0x7f + auto mask_nan = _mm512_cmpneq_epi16_mask( + _mm512_and_si512(x, _mm512_set1_epi16(127)), + _mm512_set1_epi16(127)); // mask_nan = x & 0x7f + auto mantissa = _mm512_slli_epi16(_mm512_and_si512(x, _mm512_set1_epi16(7)), 4); // mantissa = (x & 7) << 4 + auto exponent = _mm512_add_epi16( + _mm512_srli_epi16(_mm512_and_si512(x, _mm512_set1_epi16(120)), 3), + _mm512_set1_epi16(120)); // exponent = (((x >> 3) & 15) + 120) + auto nonsign = _mm512_maskz_mov_epi16(mask, _mm512_or_si512(mantissa, _mm512_slli_epi16(exponent, 7))); + nonsign = _mm512_mask_mov_epi16(_mm512_set1_epi16(0x7fff), mask_nan, nonsign); // deal with Nan + return (__m512bh)(_mm512_or_si512( + nonsign, + _mm512_slli_epi16( + _mm512_and_si512(x, _mm512_set1_epi16(128)), + 8))); // add sign (x & 128) << 8 +} + +inline __m512bh cvt_e4m3_bf16_intrinsic_with_denorm(__m256i fp8_vec) { + __m512i x = _mm512_cvtepu8_epi16(fp8_vec); + __m512i lg2mant = _mm512_mask_mov_epi16( + _mm512_mask_mov_epi16( + _mm512_setzero_si512(), _mm512_test_epi16_mask(x, _mm512_set1_epi16(2)), _mm512_set1_epi16(1)), + _mm512_test_epi16_mask(x, _mm512_set1_epi16(4)), + _mm512_set1_epi16(2)); + return (__m512bh)(_mm512_or_si512( + _mm512_maskz_mov_epi16( + _mm512_cmpneq_epi16_mask(_mm512_and_si512(x, _mm512_set1_epi16(127)), _mm512_setzero_si512()), + _mm512_mask_blend_epi16( + _mm512_test_epi16_mask(x, _mm512_set1_epi16(120)), + _mm512_or_si512( + _mm512_and_si512( + _mm512_sllv_epi16( + _mm512_and_si512(x, _mm512_set1_epi16(3)), _mm512_sub_epi16(_mm512_set1_epi16(7), lg2mant)), + _mm512_set1_epi16(0x007f)), + _mm512_slli_epi16(_mm512_add_epi16(lg2mant, _mm512_set1_epi16(118)), 7)), + _mm512_or_si512( + _mm512_slli_epi16(_mm512_and_si512(x, _mm512_set1_epi16(7)), 4), + _mm512_slli_epi16( + _mm512_add_epi16( + _mm512_srli_epi16(_mm512_and_si512(x, _mm512_set1_epi16(120)), 3), _mm512_set1_epi16(120)), + 7)))), + _mm512_slli_epi16(_mm512_and_si512(x, _mm512_set1_epi16(128)), 8))); +} + +inline __m512bh CVT_FP8_TO_BF16(__m256i a) { +#ifdef SGLANG_CPU_FP8_CVT_FTZ + return cvt_e4m3_bf16_intrinsic_no_nan(a); +#else + return cvt_e4m3_bf16_intrinsic_with_denorm(a); +#endif +} + +#endif + +// vector to scalar reduction +#if defined(CPU_CAPABILITY_AVX512) && 0 +inline float vec_reduce_sum(const Vectorized& a) { + return _mm512_reduce_add_ps(__m512(a)); +} + +inline float vec_reduce_max(const Vectorized& a) { + return _mm512_reduce_max_ps(__m512(a)); +} +#else +inline float vec_reduce_sum(const Vectorized& a) { + return vec_reduce_all([](Vectorized& x, Vectorized& y) { return x + y; }, a); +} + +inline float vec_reduce_max(const Vectorized& a) { + return vec_reduce_all([](Vectorized& x, Vectorized& y) { return maximum(x, y); }, a); +} +#endif + +// https://github.com/InternLM/lmdeploy/blob/086481ed84b59bee3b8e4274e5fc69620040c048/lmdeploy/pytorch/kernels/cuda/w8a8_triton_kernels.py#L282 +template +inline void +quantize_row_int8(uint8_t* __restrict__ Aq, float& As, const scalar_t* __restrict__ A, int64_t K, float eps = 1e-7) { + float amax = 0.f; // absolute max + for (int64_t k = 0; k < K; ++k) { + const float val = static_cast(A[k]); + amax = std::max(amax, std::abs(val)); + } + + amax = std::max(amax, eps); + const float scale = amax / 127; + const float inv_scale = 127 / amax; + + for (int64_t k = 0; k < K; ++k) { + const float val = static_cast(A[k]) * inv_scale; + Aq[k] = (uint8_t)(std::round(val)) + 128; + } + As = scale; +} + +#if defined(CPU_CAPABILITY_AVX512) +template <> +inline void quantize_row_int8( + uint8_t* __restrict__ Aq, float& As, const at::BFloat16* __restrict__ A, int64_t K, float eps) { + const __m512 signBit = _mm512_set1_ps(-0.0f); + const __m512i off = _mm512_set1_epi32(128); + + // K is 32x, no remainder + float amax = 0.f; + __m512 vamax0 = _mm512_set1_ps(0.f); + __m512 vamax1 = _mm512_set1_ps(0.f); + for (int64_t k = 0; k < K; k += 32) { + __m512i va = _mm512_loadu_si512((void*)(A + k)); + __m512 va0 = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32(va, 0)); + __m512 va1 = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32(va, 1)); + vamax0 = _mm512_max_ps(vamax0, _mm512_andnot_ps(signBit, va0)); + vamax1 = _mm512_max_ps(vamax1, _mm512_andnot_ps(signBit, va1)); + } + amax = _mm512_reduce_max_ps(_mm512_max_ps(vamax0, vamax1)); + amax = std::max(amax, eps); + const float scale = amax / 127; + const float inv_scale = 127 / amax; + const __m512 vd = _mm512_set1_ps(inv_scale); + + for (int64_t k = 0; k < K; k += 32) { + __m512i va = _mm512_loadu_si512((void*)(A + k)); + __m512 va0 = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32(va, 0)); + __m512 va1 = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32(va, 1)); + va0 = _mm512_mul_ps(va0, vd); + va1 = _mm512_mul_ps(va1, vd); + va0 = _mm512_roundscale_ps(va0, (_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)); + va1 = _mm512_roundscale_ps(va1, (_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)); + __m128i i0 = _mm512_cvtepi32_epi8(_mm512_add_epi32(_mm512_cvtps_epi32(va0), off)); + __m128i i1 = _mm512_cvtepi32_epi8(_mm512_add_epi32(_mm512_cvtps_epi32(va1), off)); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(Aq + k), _mm256_set_m128i(i1, i0)); + } + As = scale; +} +#endif + +// transpose utils +// taken from my PR in ggml: https://github.com/ggml-org/llama.cpp/pull/8998 +#if defined(CPU_CAPABILITY_AVX512) +inline void transpose_16x16_32bit(__m512i* v) { + __m512i v1[16]; + v1[0] = _mm512_unpacklo_epi32(v[0], v[1]); + v1[1] = _mm512_unpackhi_epi32(v[0], v[1]); + v1[2] = _mm512_unpacklo_epi32(v[2], v[3]); + v1[3] = _mm512_unpackhi_epi32(v[2], v[3]); + v1[4] = _mm512_unpacklo_epi32(v[4], v[5]); + v1[5] = _mm512_unpackhi_epi32(v[4], v[5]); + v1[6] = _mm512_unpacklo_epi32(v[6], v[7]); + v1[7] = _mm512_unpackhi_epi32(v[6], v[7]); + v1[8] = _mm512_unpacklo_epi32(v[8], v[9]); + v1[9] = _mm512_unpackhi_epi32(v[8], v[9]); + v1[10] = _mm512_unpacklo_epi32(v[10], v[11]); + v1[11] = _mm512_unpackhi_epi32(v[10], v[11]); + v1[12] = _mm512_unpacklo_epi32(v[12], v[13]); + v1[13] = _mm512_unpackhi_epi32(v[12], v[13]); + v1[14] = _mm512_unpacklo_epi32(v[14], v[15]); + v1[15] = _mm512_unpackhi_epi32(v[14], v[15]); + + v[0] = _mm512_unpacklo_epi64(v1[0], v1[2]); + v[1] = _mm512_unpackhi_epi64(v1[0], v1[2]); + v[2] = _mm512_unpacklo_epi64(v1[1], v1[3]); + v[3] = _mm512_unpackhi_epi64(v1[1], v1[3]); + v[4] = _mm512_unpacklo_epi64(v1[4], v1[6]); + v[5] = _mm512_unpackhi_epi64(v1[4], v1[6]); + v[6] = _mm512_unpacklo_epi64(v1[5], v1[7]); + v[7] = _mm512_unpackhi_epi64(v1[5], v1[7]); + v[8] = _mm512_unpacklo_epi64(v1[8], v1[10]); + v[9] = _mm512_unpackhi_epi64(v1[8], v1[10]); + v[10] = _mm512_unpacklo_epi64(v1[9], v1[11]); + v[11] = _mm512_unpackhi_epi64(v1[9], v1[11]); + v[12] = _mm512_unpacklo_epi64(v1[12], v1[14]); + v[13] = _mm512_unpackhi_epi64(v1[12], v1[14]); + v[14] = _mm512_unpacklo_epi64(v1[13], v1[15]); + v[15] = _mm512_unpackhi_epi64(v1[13], v1[15]); + + v1[0] = _mm512_shuffle_i32x4(v[0], v[4], 0x88); + v1[1] = _mm512_shuffle_i32x4(v[1], v[5], 0x88); + v1[2] = _mm512_shuffle_i32x4(v[2], v[6], 0x88); + v1[3] = _mm512_shuffle_i32x4(v[3], v[7], 0x88); + v1[4] = _mm512_shuffle_i32x4(v[0], v[4], 0xdd); + v1[5] = _mm512_shuffle_i32x4(v[1], v[5], 0xdd); + v1[6] = _mm512_shuffle_i32x4(v[2], v[6], 0xdd); + v1[7] = _mm512_shuffle_i32x4(v[3], v[7], 0xdd); + v1[8] = _mm512_shuffle_i32x4(v[8], v[12], 0x88); + v1[9] = _mm512_shuffle_i32x4(v[9], v[13], 0x88); + v1[10] = _mm512_shuffle_i32x4(v[10], v[14], 0x88); + v1[11] = _mm512_shuffle_i32x4(v[11], v[15], 0x88); + v1[12] = _mm512_shuffle_i32x4(v[8], v[12], 0xdd); + v1[13] = _mm512_shuffle_i32x4(v[9], v[13], 0xdd); + v1[14] = _mm512_shuffle_i32x4(v[10], v[14], 0xdd); + v1[15] = _mm512_shuffle_i32x4(v[11], v[15], 0xdd); + + v[0] = _mm512_shuffle_i32x4(v1[0], v1[8], 0x88); + v[1] = _mm512_shuffle_i32x4(v1[1], v1[9], 0x88); + v[2] = _mm512_shuffle_i32x4(v1[2], v1[10], 0x88); + v[3] = _mm512_shuffle_i32x4(v1[3], v1[11], 0x88); + v[4] = _mm512_shuffle_i32x4(v1[4], v1[12], 0x88); + v[5] = _mm512_shuffle_i32x4(v1[5], v1[13], 0x88); + v[6] = _mm512_shuffle_i32x4(v1[6], v1[14], 0x88); + v[7] = _mm512_shuffle_i32x4(v1[7], v1[15], 0x88); + v[8] = _mm512_shuffle_i32x4(v1[0], v1[8], 0xdd); + v[9] = _mm512_shuffle_i32x4(v1[1], v1[9], 0xdd); + v[10] = _mm512_shuffle_i32x4(v1[2], v1[10], 0xdd); + v[11] = _mm512_shuffle_i32x4(v1[3], v1[11], 0xdd); + v[12] = _mm512_shuffle_i32x4(v1[4], v1[12], 0xdd); + v[13] = _mm512_shuffle_i32x4(v1[5], v1[13], 0xdd); + v[14] = _mm512_shuffle_i32x4(v1[6], v1[14], 0xdd); + v[15] = _mm512_shuffle_i32x4(v1[7], v1[15], 0xdd); +} + +// remove warning : ignoring attributes on template argument ‘__m512i’ [-Wignored-attributes] +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wignored-attributes" + +// transpose from [2, 32] to [32, 2] +inline std::tuple<__m512i, __m512i> transpose_2x32_16bit(__m512i r0, __m512i r1) { + // r0: {a0, a1, ..., a31} + // r1: {b0, b1, ..., b31} + // + // d0: {a0, b0, ..., a15, b15} + // d1: {a16, b16, ..., a31, b31} + // + __m512i d0 = _mm512_unpacklo_epi16(r0, r1); + __m512i d1 = _mm512_unpackhi_epi16(r0, r1); + r0 = _mm512_shuffle_i32x4(d0, d1, 0x88); + r1 = _mm512_shuffle_i32x4(d0, d1, 0xdd); + d0 = _mm512_shuffle_i32x4(r0, r1, 0x88); + d1 = _mm512_shuffle_i32x4(r0, r1, 0xdd); + return std::make_tuple(d0, d1); +} +#pragma GCC diagnostic pop + +#endif + +} // anonymous namespace diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/common.hpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/common.hpp new file mode 100644 index 00000000..912ef8b1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/common.hpp @@ -0,0 +1,21 @@ +#pragma once + +#include "cuda_runtime.h" +#include "cutlass/cutlass.h" + +/** + * A wrapper for a kernel that is used to guard against compilation on + * architectures that will never use the kernel. The purpose of this is to + * reduce the size of the compiled binary. + * __CUDA_ARCH__ is not defined in host code, so this lets us smuggle the ifdef + * into code that will be executed on the device where it is defined. + */ +template +struct enable_sm90_or_later : Kernel { + template + CUTLASS_DEVICE void operator()(Args&&... args) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 900 + Kernel::operator()(std::forward(args)...); +#endif + } +}; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/detail/collective/mixed_input_utils.hpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/detail/collective/mixed_input_utils.hpp new file mode 100644 index 00000000..fcba4d40 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/detail/collective/mixed_input_utils.hpp @@ -0,0 +1,482 @@ +/* + * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "cute/arch/copy_sm90.hpp" +#include "cute/numeric/arithmetic_tuple.hpp" +#include "cute/util/type_traits.hpp" +#include "cutlass/cutlass.h" +#include "cutlass/numeric_conversion.h" + +///////////////////////////////////////////////////////////////////////////////////////////////// + +namespace cutlass::gemm::collective::detail { + +template +struct MixedGroupedGemmInputUtils { + private: + using KernelSchedule = typename Collective::KernelSchedule; + using ConversionMode = typename Collective::ConversionMode; + using SmemLayoutA = typename Collective::SmemLayoutA; + using SmemLayoutB = typename Collective::SmemLayoutB; + using SmemLayoutScale = typename Collective::SmemLayoutScale; + using SwappedElementA = typename Collective::SwappedElementA; + using SwappedElementB = typename Collective::SwappedElementB; + using RealSwappedElementA = typename Collective::RealSwappedElementA; + using RealSwappedElementB = typename Collective::RealSwappedElementB; + using ElementScale = typename Collective::ElementScale; + using ElementZero = typename Collective::ElementZero; + using SmemCopyAtomScale = typename Collective::SmemCopyAtomScale; + static constexpr auto KernelConversionMode = Collective::KernelConversionMode; + static constexpr auto ModeHasScales = Collective::ModeHasScales; + static constexpr auto UseScaleLookupTable = Collective::UseScaleLookupTable; + + public: + static constexpr auto elements_per_smem_scale() { + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + return 0; + } else if constexpr (ModeHasScales) { + return cute::cosize_v; + } else { + static_assert(cutlass::detail::dependent_false, "Type not handled in scale smem allocation."); + } + } + + static constexpr auto elements_per_smem_zero() { + if constexpr ( + KernelConversionMode == ConversionMode::DirectConvert || + KernelConversionMode == ConversionMode::ConvertAndScale) { + return 0; + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + return cute::cosize_v; + } else { + static_assert(cutlass::detail::dependent_false, "Type not handled in scale smem allocation."); + } + } + + // These methods use some the public members of the class. For that reason, we define them after the public section. + static constexpr uint32_t compute_tma_transaction_bytes_mk() { + return cutlass::bits_to_bytes( + size<0>(SmemLayoutA{}) * size<1>(SmemLayoutA{}) * static_cast(cute::sizeof_bits_v)); + } + + static constexpr uint32_t compute_tma_transaction_bytes_nk() { + return cutlass::bits_to_bytes( + size<0>(SmemLayoutB{}) * size<1>(SmemLayoutB{}) * static_cast(cute::sizeof_bits_v)); + } + + static constexpr uint32_t compute_tma_transaction_bytes_extra() { + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + return 0; + } else if constexpr (ModeHasScales) { + constexpr uint32_t scale_tx_bytes = cutlass::bits_to_bytes( + size<0>(SmemLayoutScale{}) * size<1>(SmemLayoutScale{}) * + static_cast(cute::sizeof_bits_v)); + static_assert(scale_tx_bytes % 128 == 0, "Each scale stage must be 128B aligned."); // required by TMA + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + return scale_tx_bytes; + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + // Scale and zero share smem layout + constexpr uint32_t zero_tx_bytes = cutlass::bits_to_bytes( + size<0>(SmemLayoutScale{}) * size<1>(SmemLayoutScale{}) * + static_cast(cute::sizeof_bits_v)); + static_assert(zero_tx_bytes % 128 == 0, "Each zero stage must be 128B aligned."); // required by TMA + return scale_tx_bytes + zero_tx_bytes; + } else { + static_assert( + cutlass::detail::dependent_false, "Type not handled in tma transaction bytes computation."); + } + } else { + static_assert( + cutlass::detail::dependent_false, "Type not handled in tma transaction bytes computation."); + } + } + + /// Utilities to copy A and extra inputs from smem to RF + template + CUTLASS_DEVICE static void copy_tensors_MK( + SmemTiledCopyA const& smem_tiled_copy_A, + TensorASmemView const& tCsA, + TensorACopyView& tCrA_copy_view, + cute::tuple const& partitioned_mma_extra_info, + cute::tuple const& tiled_copy_and_views, + int k_block, + int read_stage) { + copy(smem_tiled_copy_A, tCsA(_, _, k_block, read_stage), tCrA_copy_view(_, _, k_block)); + + if (k_block == 0) { + // We are starting a new k-tile so copy the scale + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + // nothing to do + } else if constexpr (ModeHasScales) { + auto smem_tiled_copy_S = cute::get<0>(tiled_copy_and_views); + auto tCrS_copy_view = cute::get<1>(tiled_copy_and_views); + auto tCsS = cute::get<0>(partitioned_mma_extra_info); + copy(smem_tiled_copy_S, tCsS(_, _, k_block, read_stage), tCrS_copy_view(_, _, k_block)); + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + // Nothing extra to do + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + auto tCsZ = cute::get<2>(partitioned_mma_extra_info); + auto tCrZ_copy_view = cute::get<2>(tiled_copy_and_views); + copy(smem_tiled_copy_S, tCsZ(_, _, k_block, read_stage), tCrZ_copy_view(_, _, k_block)); + } else { + static_assert( + cutlass::detail::dependent_false, "Conversion mode not handled in A -> RF path."); + } + } else { + static_assert(cutlass::detail::dependent_false, "Conversion mode not handled in A -> RF path."); + } + } + } + + // The core converter uses a lookup table to converts i4 -> 8 bit value. + template < + class EngineIn, + class LayoutIn, + class EngineOut, + class LayoutOut, + class EngineScale, + class LayoutScale> + CUTLASS_DEVICE static void lookup_table_convert( // Accept mutable temporaries + Tensor const& src, + Tensor&& dst, + Tensor const& scales_neg, + Tensor const& scales_pos) { + lookup_table_convert(src, dst, scales_neg, scales_pos); + } + template + CUTLASS_DEVICE static void lookup_table_convert( + Tensor const& src, + Tensor& dst, + Tensor const& scales_neg, + Tensor const& scales_pos) { + constexpr int N = cute::cosize(LayoutIn{}); + static_assert(N == 4 || N == 8); + static_assert(cosize(LayoutScale{}) <= N / 4, "at least 4 consecutive weights must share the same scale."); + using SrcArray = cutlass::Array; + using DstArray = cutlass::Array; + using RegArray = cutlass::AlignedArray; + + // View the input as reg + auto&& src_reg = cute::recast(src)(0); + auto&& r = cute::recast(dst)(0); + + // Determines if to get from the signed or unsigned candidates + static constexpr uint32_t immLut = (0xf0 & 0xcc) | 0xaa; + uint32_t sign; // ((reg & 0x88888888) | 0x64206420) >> 1 + asm volatile( + "{\n" + " lop3.b32 %0, %1, %2, %3, %4;\n" + "}\n" + : "=r"(sign) + : "r"(src_reg), "n"(0x88888888), "n"(0x64206420), "n"(immLut)); + sign = sign >> 1; + + // Ignore sign bit when indexing into LUT + uint32_t lut_idx = src_reg & 0x77777777; + Tensor scales_neg_ = cute::filter(scales_neg); + Tensor scales_pos_ = cute::filter(scales_pos); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < N / 4; ++i, lut_idx >>= 16, sign >>= 16) { + auto&& scale_neg_ = reinterpret_cast const&>(scales_neg_(i)); + auto&& scale_pos_ = reinterpret_cast const&>(scales_pos_(i)); + asm volatile( + "{\n" + " .reg .b32 pos, neg ;\n" + " prmt .b32 neg, %3, %4, %1 ;\n" + " prmt .b32 pos, %5, %6, %1 ;\n" + " prmt .b32 %0, pos, neg, %2 ;\n" + "}\n" + : "=r"(r[i]) + : "r"(lut_idx), "r"(sign), "r"(scale_neg_[0]), "r"(scale_neg_[1]), "r"(scale_pos_[0]), "r"(scale_pos_[1])); + } + } + + /// Utilities to dequantize A. + template + CUTLASS_DEVICE static void static_check_scale(Layout const& tensor) { + static_assert( + shape<0>(Layout{}) >= 4 && stride<0>(Layout{}) == 0, + "At least 4 adjacent weights in a thread must share the same scale."); + } + template + CUTLASS_DEVICE static void static_check_scale(Tensor const& tensor) { + static_check_scale(flatten(Layout{})); + } + + template + CUTLASS_DEVICE static void dequantize_A_kblock( + Tensor const& tCrA_load, + Tensor& tCrA_mma, + cute::tuple& partitioned_extra_info, + int const k_block) { + static_assert(is_rmem::value, "Input tensor for A conversion must come from registers"); + static_assert(is_rmem::value, "Output tensor for A conversion must come from registers"); + static_assert(cosize_v == cosize_v); + static_assert(size_v == cosize_v); + static_assert(size_v == cosize_v); + using SrcType = typename EngineIn::value_type; + using DstType = typename EngineOut::value_type; + + Tensor src = tCrA_load(_, _, k_block); + Tensor dst = tCrA_mma(_, _, k_block); + + CUTE_STATIC_ASSERT_V( + size(src(_, 0)) == cosize(src(_, 0).layout()), "The first mode of tensor src must be contiguous in memory"); + // try to make the size of the first mode equal to 32bit + int constexpr NumValPerSrcReg = cute::min(decltype(size(src(_, 0)))::value, ceil_div(32, sizeof_bits_v)); + Tensor src_vm = cute::group_modes<1, -1>(cute::zipped_divide(src, Int{})); + Tensor dst_vm = cute::group_modes<1, -1>(cute::zipped_divide(dst, Int{})); + + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size<1>(dst_vm); ++i) { + LayoutAwareConvert(src_vm(_, i), dst_vm(_, i)); + } + } else if constexpr (UseScaleLookupTable) { + constexpr int num_elements = decltype(size(src))::value; + static_assert( + is_same_v, + "Lookup table only supports int4 being the quant type now."); + static_assert(sizeof_bits_v == 64, "Lookup table only supports 8 8bit scale values now."); + static_assert( + num_elements % 4 == 0 && num_elements >= 4, "Lookup table requires a vector size of 4x when converting."); + + Tensor tCrS_neg = cute::get<1>(partitioned_extra_info); + auto&& tCrS_pos = cute::get<2>(partitioned_extra_info); // modification to its value is needed + Tensor scales_neg = tCrS_neg(_, _, k_block); + Tensor scales_pos = tCrS_pos(_, _, k_block); + CUTE_STATIC_ASSERT_V(cute::size(src) == cute::size(scales_neg)); + + static_check_scale(scales_neg); + static_check_scale(scales_pos); + Tensor scales_neg_vm = cute::group_modes<1, -1>(cute::zipped_divide(scales_neg, Int{})); + Tensor scales_pos_vm = cute::group_modes<1, -1>(cute::zipped_divide(scales_pos, Int{})); + + if (k_block == 0) { + Tensor scales_neg_vm_ = filter(scales_neg_vm); + Tensor scales_pos_vm_ = filter(scales_pos_vm); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size(scales_neg_vm_.layout()); ++i) { + auto&& scale_neg_ = reinterpret_cast const&>(scales_neg_vm_(i)); + auto&& scale_pos_ = reinterpret_cast&>(scales_pos_vm_(i)); + constexpr uint32_t immLut = (0xf0 & 0xcc) ^ 0xaa; + asm volatile( + "{\n" + " lop3 .b32 %0, %2, %4, %5, %6;\n" + " xor .b32 %1, %3, %5; \n" + "}\n" + : "=r"(scale_pos_[0]), "=r"(scale_pos_[1]) + : "r"(scale_neg_[0]), "r"(scale_neg_[1]), "n"(0xFFFFFF00), "n"(0x80808080), "n"(immLut)); + } + } + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size<1>(dst_vm); ++i) { + lookup_table_convert(src_vm(_, i), dst_vm(_, i), scales_neg_vm(_, i), scales_pos_vm(_, i)); + } + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + Tensor scales = cute::get<1>(partitioned_extra_info)(_, _, k_block); + CUTE_STATIC_ASSERT_V(size(src) == size(scales)); + Tensor scales_vm = cute::group_modes<1, -1>(cute::zipped_divide(scales, Int{})); + + if constexpr (is_same_v) { + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size<1>(dst_vm); ++i) { + LayoutAwareConvert(src_vm(_, i), dst_vm(_, i)); + CUTLASS_PRAGMA_UNROLL + for (int j = 0; j < size<0>(dst_vm); ++j) { + dst_vm(j, i) *= scales_vm(j, i); + } + } + } else { + auto stage = make_tensor_like(src_vm(_, 0)); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size<1>(dst_vm); ++i) { + LayoutAwareConvert(src_vm(_, i), stage); + CUTLASS_PRAGMA_UNROLL + for (int j = 0; j < size<0>(dst_vm); ++j) { + stage(j) *= scales_vm(j, i); + } + LayoutAwareConvert(stage, dst_vm(_, i)); + } + } + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + static_assert(is_same_v, "ElementScale and ElementZero must be the same."); + Tensor scales = cute::get<1>(partitioned_extra_info)(_, _, k_block); + Tensor zeros = cute::get<3>(partitioned_extra_info)(_, _, k_block); + CUTE_STATIC_ASSERT_V(size(src) == size(scales)); + CUTE_STATIC_ASSERT_V(size(src) == size(zeros)); + Tensor scales_vm = cute::group_modes<1, -1>(cute::zipped_divide(scales, Int{})); + Tensor zeros_vm = cute::group_modes<1, -1>(cute::zipped_divide(zeros, Int{})); + + if constexpr (is_same_v) { + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size<1>(dst_vm); ++i) { + LayoutAwareConvert(src_vm(_, i), dst_vm(_, i)); + CUTLASS_PRAGMA_UNROLL + for (int j = 0; j < size<0>(dst_vm); ++j) { + dst_vm(j, i) = dst_vm(j, i) * scales_vm(j, i) + zeros_vm(j, i); + } + } + } else { + auto stage = make_tensor_like(src_vm(_, 0)); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size<1>(dst_vm); ++i) { + LayoutAwareConvert(src_vm(_, i), stage); + CUTLASS_PRAGMA_UNROLL + for (int j = 0; j < size<0>(dst_vm); ++j) { + stage(j) = stage(j) * scales_vm(j, i) + zeros_vm(j, i); + } + LayoutAwareConvert(stage, dst_vm(_, i)); + } + } + } else { + static_assert(cutlass::detail::dependent_false, "No A data is loaded."); + } + } + + template + CUTLASS_DEVICE static void convert_A_kblock( + Tensor const& tCrA_load, Tensor& tCrA_mma, int const k_block) { + static_assert(is_rmem::value, "Input tensor for A conversion must come from registers"); + static_assert(is_rmem::value, "Output tensor for A conversion must come from registers"); + static_assert(cosize_v == cosize_v); + static_assert(size_v == cosize_v); + static_assert(size_v == cosize_v); + using SrcType = typename EngineIn::value_type; + + Tensor src = tCrA_load(_, _, k_block); + Tensor dst = tCrA_mma(_, _, k_block); + + CUTE_STATIC_ASSERT_V( + size(src(_, 0)) == cosize(src(_, 0).layout()), "The first mode of tensor src must be contiguous in memory"); + // try to make the size of the first mode equal to 32bit + int constexpr NumValPerSrcReg = cute::min(decltype(size(src(_, 0)))::value, ceil_div(32, sizeof_bits_v)); + Tensor src_vm = cute::group_modes<1, -1>(cute::zipped_divide(src, Int{})); + Tensor dst_vm = cute::group_modes<1, -1>(cute::zipped_divide(dst, Int{})); + + // KernelConversionMode == ConversionMode::DirectConvert + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size<1>(dst_vm); ++i) { + LayoutAwareConvert(src_vm(_, i), dst_vm(_, i)); + } + } + + /// Utilities for any additional inputs inside of the TMA load + template + CUTLASS_DEVICE static auto partition_extra_tma_inputs( + Params const& mainloop_params, + cute::tuple const& load_inputs, + TensorStorage& shared_tensors, + uint2 const& cluster_local_block_id, + int const m_coord, + int const l_coord) { + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + return cute::make_tuple(); + } else if constexpr (ModeHasScales) { + Tensor sS = + make_tensor(make_smem_ptr(shared_tensors.smem_scale.begin()), SmemLayoutScale{}); // (BLK_M,BLK_K,PIPE) + Tensor gS_mkl = get<2>(load_inputs); + auto block_tma_s = mainloop_params.tma_load_scale.get_slice(cluster_local_block_id.y); + Tensor gS = gS_mkl(_, _, m_coord, _, l_coord); // (BLK_M,BLK_K,k) + + Tensor tSgS = block_tma_s.partition_S(gS); // (TMA,TMA_M,TMA_K,k) + Tensor tSsS = block_tma_s.partition_D(sS); // (TMA,TMA_M,TMA_K,PIPE) + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + return cute::make_tuple(tSgS, tSsS); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + Tensor sZ = + make_tensor(make_smem_ptr(shared_tensors.smem_zero.begin()), SmemLayoutScale{}); // (BLK_M,BLK_K,PIPE) + Tensor gZ_mkl = get<3>(load_inputs); + auto block_tma_z = mainloop_params.tma_load_zero.get_slice(cluster_local_block_id.y); + Tensor gZ = gZ_mkl(_, _, m_coord, _, l_coord); // (BLK_M,BLK_K,k) + + Tensor tZgZ = block_tma_z.partition_S(gZ); // (TMA,TMA_M,TMA_K,k) + Tensor tZsZ = block_tma_z.partition_D(sZ); // (TMA,TMA_M,TMA_K,PIPE) + return cute::make_tuple(tSgS, tSsS, tZgZ, tZsZ); + } else { + static_assert( + cutlass::detail::dependent_false, "Conversion mode not handled for input partitioning."); + } + } else { + static_assert( + cutlass::detail::dependent_false, "Conversion mode not handled for input partitioning."); + } + } + + /// Utilities for partitioning extra inputs for loading from smem in the mainloop. + template + CUTLASS_DEVICE static auto + partition_extra_mma_info(ThreadMma const& mma_thread_slice, TensorStorage& shared_tensors) { + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + // nothing to do + return cute::make_tuple(); + } else if constexpr (UseScaleLookupTable) { + Tensor sS = + make_tensor(make_smem_ptr(shared_tensors.smem_scale.begin()), SmemLayoutScale{}); // (BLK_M,BLK_SCALE_K,PIPE) + Tensor tCsS = mma_thread_slice.partition_A(sS); + Tensor tCrS = make_tensor(mma_thread_slice.partition_fragment_A(sS(_, _, Int<0>{})).layout()); + + return cute::make_tuple(tCsS, tCrS); + } else if constexpr (ModeHasScales) { + Tensor sS = + make_tensor(make_smem_ptr(shared_tensors.smem_scale.begin()), SmemLayoutScale{}); // (BLK_M,BLK_SCALE_K,PIPE) + Tensor tCsS = mma_thread_slice.partition_A(sS); + Tensor tCrS = make_tensor(mma_thread_slice.partition_fragment_A(sS(_, _, Int<0>{})).layout()); + + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + return cute::make_tuple(tCsS, tCrS); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + Tensor sZ = make_tensor( + make_smem_ptr(shared_tensors.smem_zero.begin()), SmemLayoutScale{}); // (BLK_M,BLK_SCALE_K,PIPE) + Tensor tCsZ = mma_thread_slice.partition_A(sZ); + Tensor tCrZ = make_tensor(mma_thread_slice.partition_fragment_A(sZ(_, _, Int<0>{})).layout()); + return cute::make_tuple(tCsS, tCrS, tCsZ, tCrZ); + } else { + static_assert(cutlass::detail::dependent_false, "Conversion mode not handled in A -> RF path."); + } + } else { + static_assert(cutlass::detail::dependent_false, "Conversion mode not handled in A -> RF path."); + } + } + + /// Returns the tiled copy and copy views for the extra inputs. + template + CUTLASS_DEVICE static auto retile_extra_mma_info( + TiledMma const& tiled_mma, cute::tuple& partitioned_extra_info, int const warp_group_thread_idx) { + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + // nothing to do + return cute::make_tuple(); + } else if constexpr (ModeHasScales) { + auto smem_tiled_copy_S = make_tiled_copy_A(SmemCopyAtomScale{}, tiled_mma); + auto smem_thr_copy_S = smem_tiled_copy_S.get_thread_slice(warp_group_thread_idx); + Tensor tCrS_copy_view = smem_thr_copy_S.retile_D(cute::get<1>(partitioned_extra_info)); // (CPY,CPY_M,CPY_K) + + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + return cute::make_tuple(smem_tiled_copy_S, tCrS_copy_view); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + Tensor tCrZ_copy_view = smem_thr_copy_S.retile_D(cute::get<3>(partitioned_extra_info)); // (CPY,CPY_M,CPY_K) + return cute::make_tuple(smem_tiled_copy_S, tCrS_copy_view, tCrZ_copy_view); + } else { + static_assert(cutlass::detail::dependent_false, "Conversion mode not handled in A -> RF path."); + } + } else { + static_assert(cutlass::detail::dependent_false, "Conversion mode not handled in A -> RF path."); + } + } +}; + +} // namespace cutlass::gemm::collective::detail diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/epilogue/epilogue_per_row_per_col_scale.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/epilogue/epilogue_per_row_per_col_scale.h new file mode 100644 index 00000000..9f85bee2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/epilogue/epilogue_per_row_per_col_scale.h @@ -0,0 +1,309 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Adapted from +// https://github.com/NVIDIA/TensorRT-LLM/blob/be1788106245496872d18e702978e59b6bfd50e0/cpp/tensorrt_llm/cutlass_extensions/include/cutlass_extensions/epilogue/threadblock/epilogue_per_row_per_col_scale.h + +#pragma once + +#include +#include + +namespace cutlass { +namespace epilogue { +namespace threadblock { + +template < + typename ThreadblockShape_, + int ThreadCount, + typename ScaleTileIterator_, + typename OutputTileIterator_, + typename ElementAccumulator_, + typename ElementCompute_, + typename ElementwiseFunctor_, + bool UseMasking_ = false> +class EpilogueVisitorPerRowPerCol { + public: + using ThreadblockShape = ThreadblockShape_; + static int const kThreadCount = ThreadCount; + + using ScaleTileIterator = ScaleTileIterator_; + using OutputTileIterator = OutputTileIterator_; + using ElementwiseFunctor = ElementwiseFunctor_; + + static int const kIterations = OutputTileIterator::kIterations; + static int const kElementsPerAccess = OutputTileIterator::kElementsPerAccess; + + using ElementOutput = typename OutputTileIterator::Element; + using LayoutOutput = cutlass::layout::RowMajor; + using ElementAccumulator = ElementAccumulator_; + + using AlphaScaleElementType = typename ScaleTileIterator::Element; + + using ElementCompute = ElementCompute_; + using AccumulatorFragment = Array; + using ComputeFragment = Array; + using OutputVector = Array; + + static int const kThreadsPerRow = OutputTileIterator::ThreadMap::Detail::kAccessWidth; + static bool const kHasMultiStepsInRow = (OutputTileIterator::ThreadMap::Iterations::kColumn > 1); + + /// Argument structure + struct Arguments { + typename ElementwiseFunctor::Params elementwise; + int64_t batch_stride_alpha; + int64_t batch_stride_C; + int64_t batch_stride_D; + + // + // Methods + // + Arguments() : batch_stride_alpha(0), batch_stride_C(0), batch_stride_D(0) {} + + Arguments(typename ElementwiseFunctor::Params elementwise_) + : elementwise(elementwise_), batch_stride_alpha(0), batch_stride_C(0), batch_stride_D(0) {} + + Arguments( + typename ElementwiseFunctor::Params elementwise_, + int64_t batch_stride_alpha_, + int64_t batch_stride_C_, + int64_t batch_stride_D_) + : elementwise(elementwise_), + batch_stride_alpha(batch_stride_alpha_), + batch_stride_C(batch_stride_C_), + batch_stride_D(batch_stride_D_) {} + }; + + struct Params { + typename ElementwiseFunctor::Params elementwise; + int64_t batch_stride_alpha; + int64_t batch_stride_C; + int64_t batch_stride_D; + + // + // Methods + // + CUTLASS_HOST_DEVICE + Params() {} + + CUTLASS_HOST_DEVICE + Params(Arguments const& args) + : elementwise(args.elementwise), + batch_stride_alpha(args.batch_stride_alpha), + batch_stride_C(args.batch_stride_C), + batch_stride_D(args.batch_stride_D) {} + }; + + /// Shared storage + struct SharedStorage {}; + + private: + Params const& params_; + SharedStorage& shared_storage_; + MatrixCoord extent_; + MatrixCoord extent_real_; + ElementwiseFunctor elementwise_; + + bool const with_bias_; + bool const per_token_quant_; + bool const per_channel_quant_; + + AlphaScaleElementType* ptr_alpha_row_; + AlphaScaleElementType* ptr_alpha_col_; + ScaleTileIterator iterator_alpha_col_; + OutputTileIterator iterator_C_; + OutputTileIterator iterator_D_; + + AlphaScaleElementType element_alpha_row_ = 1.0f; + AlphaScaleElementType element_alpha_col_ = 1.0f; + typename ScaleTileIterator::Fragment fragment_alpha_col_; + typename OutputTileIterator::Fragment fragment_C_; + typename OutputTileIterator::Fragment fragment_D_; + + ElementAccumulator beta_; + + int column_offset_; + + MatrixCoord thread_offset_; + + public: + CUTLASS_DEVICE + EpilogueVisitorPerRowPerCol( + Params const& params, + SharedStorage& shared_storage, + cutlass::MatrixCoord const& problem_size, + int thread_idx, + int warp_idx, + int lane_idx, + typename ScaleTileIterator::Params params_alpha_col, + typename OutputTileIterator::Params params_C, + typename OutputTileIterator::Params params_D, + bool with_bias, + bool per_token_quant, + bool per_channel_quant, + AlphaScaleElementType* ptr_alpha_row, + AlphaScaleElementType* ptr_alpha_col, + typename OutputTileIterator::Element* ptr_C, + typename OutputTileIterator::Element* ptr_D, + cutlass::MatrixCoord const& threadblock_offset = cutlass::MatrixCoord(0, 0), + int column_offset = 0, + cutlass::MatrixCoord const& problem_size_real = cutlass::MatrixCoord(0, 0)) + : params_(params), + shared_storage_(shared_storage), + extent_(problem_size), + elementwise_(params.elementwise), + with_bias_(with_bias), + per_token_quant_(per_token_quant), + per_channel_quant_(per_channel_quant), + ptr_alpha_row_(ptr_alpha_row), + ptr_alpha_col_(ptr_alpha_col), + iterator_alpha_col_(params_alpha_col, ptr_alpha_col, problem_size, thread_idx, threadblock_offset), + iterator_C_(params_C, ptr_C, problem_size, thread_idx, threadblock_offset), + iterator_D_(params_D, ptr_D, problem_size, thread_idx, threadblock_offset), + extent_real_(problem_size_real) { + if (!per_channel_quant_ && (ptr_alpha_col_ != nullptr)) { + element_alpha_col_ = *ptr_alpha_col_; + } + + if (!per_token_quant_ && (ptr_alpha_row_ != nullptr)) { + element_alpha_row_ = *ptr_alpha_row_; + } + } + + /// Helper to indicate split-K behavior + CUTLASS_DEVICE + void set_k_partition( + int split_k_index, ///< Index of this threadblock within split-K partitioned scheme + int split_k_slices) { ///< Total number of split-K slices + } + + /// Called to set the batch index + CUTLASS_DEVICE + void set_batch_index(int batch_idx) { + iterator_alpha_col_.add_pointer_offset(batch_idx * params_.batch_stride_alpha); + iterator_C_.add_pointer_offset(batch_idx * params_.batch_stride_C); + iterator_D_.add_pointer_offset(batch_idx * params_.batch_stride_D); + } + + /// Called at the start of the epilogue just before iterating over accumulator slices + CUTLASS_DEVICE + void begin_epilogue() { + if (per_channel_quant_) { + iterator_alpha_col_.load(fragment_alpha_col_); + } + + if (with_bias_) { + iterator_C_.load(fragment_C_); + } + } + + /// Called at the start of one step before starting accumulator exchange + CUTLASS_DEVICE + void begin_step(int step_idx) { + fragment_D_.clear(); + } + + /// Called at the start of a row + CUTLASS_DEVICE + void begin_row(int row_idx) { + // load alpha_row in begin_step only when per token(row) scaling is used + if (per_token_quant_) { + int thread_offset_row = + iterator_D_.thread_start_row() + OutputTileIterator::ThreadMap::iteration_offset(row_idx).row(); + + arch::global_load( + element_alpha_row_, ptr_alpha_row_ + thread_offset_row, thread_offset_row < extent_.row()); + } + } + + /// Called after accumulators have been exchanged for each accumulator vector + CUTLASS_DEVICE + void visit(int iter_idx, int row_idx, int column_idx, int frag_idx, AccumulatorFragment const& accum) { + NumericArrayConverter source_converter; + + ComputeFragment result = source_converter(accum); + if (per_channel_quant_) { + ComputeFragment alpha_col = reinterpret_cast(&fragment_alpha_col_)[column_idx]; + result = per_token_channel_scale_accumulator_(result, alpha_col, element_alpha_row_); + } else { + result = per_token_scale_accumulator_(result, element_alpha_col_, element_alpha_row_); + } + + if (with_bias_) { + NumericArrayConverter bias_converter; + OutputVector bias = reinterpret_cast(&fragment_C_)[column_idx]; + result = bias_accumulator_(result, bias_converter(bias)); + } + + // Convert to the output + NumericArrayConverter output_converter; + OutputVector& output = reinterpret_cast(&fragment_D_)[frag_idx]; + output = output_converter(result); + } + + /// Called at the end of a row + CUTLASS_DEVICE + void end_row(int row_idx) {} + + /// Called after all accumulator elements have been visited + CUTLASS_DEVICE + void end_step(int step_idx) { + iterator_D_.store(fragment_D_); + ++iterator_D_; + } + + /// Called after all steps have been completed + CUTLASS_DEVICE + void end_epilogue() {} + + private: + CUTLASS_DEVICE + ComputeFragment per_token_channel_scale_accumulator_( + ComputeFragment const& accum, ComputeFragment const& scale_col, AlphaScaleElementType const& scale_row) { + ComputeFragment result; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < ComputeFragment::kElements; ++i) { + result[i] = accum[i] * (scale_col[i] * scale_row); + } + + return result; + } + + CUTLASS_DEVICE + ComputeFragment per_token_scale_accumulator_( + ComputeFragment const& accum, AlphaScaleElementType const& scale_col, AlphaScaleElementType const& scale_row) { + ComputeFragment result; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < ComputeFragment::kElements; ++i) { + result[i] = accum[i] * (scale_col * scale_row); + } + + return result; + } + + CUTLASS_DEVICE + ComputeFragment bias_accumulator_(ComputeFragment const& accum, ComputeFragment const& bias) { + ComputeFragment result; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < OutputVector::kElements; ++i) { + result[i] = accum[i] + bias[i]; + } + return result; + } +}; + +} // namespace threadblock +} // namespace epilogue +} // namespace cutlass diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/collective/builders/sm90_gmma_builder_mixed_input.inl b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/collective/builders/sm90_gmma_builder_mixed_input.inl new file mode 100644 index 00000000..db1fdf1e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/collective/builders/sm90_gmma_builder_mixed_input.inl @@ -0,0 +1,278 @@ +/* + * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "cute/arch/cluster_sm90.hpp" +#include "cute/tensor.hpp" +#include "cutlass/gemm/collective/builders/sm90_common.inl" +#include "cutlass/gemm/collective/collective_builder_decl.hpp" +#include "cutlass/gemm/collective/collective_mma_decl.hpp" +#include "cutlass/gemm/dispatch_policy.hpp" +#include "cutlass/pipeline/sm90_pipeline.hpp" + +// SM90 Collective Builders should be used only starting CUDA 12.0 +#if (__CUDACC_VER_MAJOR__ >= 12) +#define CUTLASS_SM90_COLLECTIVE_BUILDER_SUPPORTED +#endif + +///////////////////////////////////////////////////////////////////////////////////////////////// + +namespace cutlass::gemm::collective { + +///////////////////////////////////////////////////////////////////////////////////////////////// + +// GMMA_TMA_WS_RS +template < + class ElementA_, + class GmemLayoutATag_, + int AlignmentA, + class ElementB_, + class GmemLayoutBTag_, + int AlignmentB, + class ElementAccumulator, + class TileShape_MNK, + class ClusterShape_MNK, + class StageCountType, + class KernelScheduleType> +struct CollectiveBuilderMixedInput< + arch::Sm90, + arch::OpClassTensorOp, + ElementA_, + GmemLayoutATag_, + AlignmentA, + ElementB_, + GmemLayoutBTag_, + AlignmentB, + ElementAccumulator, + TileShape_MNK, + ClusterShape_MNK, + StageCountType, + KernelScheduleType, + cute::enable_if_t< + (cute::is_same_v || + cute::is_same_v || + cute::is_same_v || + cute::is_same_v || + cute::is_same_v) && + (detail::is_use_rmem_A() || + // ConvertAndScale and ConvertAndScaleWithZero + cute::is_tuple::value || cute::is_tuple::value || + // DirectConvert + sizeof_bits::value != sizeof_bits::value)>> { + private: + using ScaleA = detail::deduce_mixed_width_dtype_t<1, ElementA_>; + using ScaleB = detail::deduce_mixed_width_dtype_t<1, ElementB_>; + using ZeroA = detail::deduce_mixed_width_dtype_t<2, ElementA_>; + using ZeroB = detail::deduce_mixed_width_dtype_t<2, ElementB_>; + static constexpr bool NeitherIsTuple = !cute::is_tuple::value && !cute::is_tuple::value; + // Determine if mixed input types. + static constexpr bool IsMixedInput = cute::sizeof_bits_v> != + cute::sizeof_bits_v>; + static constexpr bool IsArrayOfPointersGemm = cute::is_any_of_v< + KernelScheduleType, + KernelPtrArrayTmaWarpSpecializedCooperative, + KernelPtrArrayTmaWarpSpecializedPingpong>; + static_assert(IsMixedInput || !IsArrayOfPointersGemm, "Only mixed input grouped RS GEMM is supported."); + + public: + using ElementA = detail::deduce_mixed_width_dtype_t<0, ElementA_>; + using ElementB = detail::deduce_mixed_width_dtype_t<0, ElementB_>; + + static_assert( + !IsMixedInput || (cute::is_tuple::value ^ cute::is_tuple::value || + (NeitherIsTuple && (sizeof_bits::value != sizeof_bits::value))), + "Either A OR B must be a tuple or the widths of A and B must be different."); + + static constexpr bool IsANarrow = sizeof_bits::value < sizeof_bits::value; + + template + static auto get_stride(T const& t) { + if constexpr (not cute::is_layout>::value) { + return t; + } else { + if constexpr (cute::is_pointer_v) { + return &cute::stride(*t); + } else { + return cute::stride(t); + } + } + } + + using GmemLayoutATag = decltype(get_stride(GmemLayoutATag_{})); + using GmemLayoutBTag = decltype(get_stride(GmemLayoutBTag_{})); + + using ElementPairA = + cute::conditional_t, ElementA_>; + using ElementPairB = + cute::conditional_t, ElementB_>; + + static constexpr bool IsATransformed = cute::is_tuple::value; + using ElementScale = cute::conditional_t; + using ElementZero = cute::conditional_t; + + static_assert(is_static::value); + static_assert(is_static::value); + static_assert( + detail::is_aligned(), + "Should meet TMA alignment requirement\n"); +#ifndef CUTLASS_SM90_COLLECTIVE_BUILDER_SUPPORTED + static_assert(cutlass::detail::dependent_false, "Unsupported Toolkit for SM90 Collective Builder\n"); +#endif + static constexpr cute::GMMA::Major GmmaMajorA = detail::gmma_rs_tag_to_major_A(); + static constexpr cute::GMMA::Major GmmaMajorB = detail::gmma_rs_tag_to_major_B(); + // If A is scaled, then we don't need to swap. Otherwise, we must ensure B goes to rmem and we must swap the + // operands. + static constexpr bool SwapAB = + IsMixedInput ? !IsATransformed : detail::is_swapAB(); + static constexpr bool IsWarpSpecializedTransposeB = + detail::is_warpspecialized_transpose_B(); + static_assert(!IsMixedInput || !IsWarpSpecializedTransposeB, "Mixed input GEMM does not support WS transpose B."); + + // When we relax the above assertion, we must handle setting the tile mma GmmaMajorB correctly. + static constexpr cute::GMMA::Major TiledMmaGmmaMajorB = SwapAB ? GmmaMajorA : GmmaMajorB; + + // For fp32 types, map to tf32 MMA value type. + using ElementAMma = cute::conditional_t, tfloat32_t, ElementA>; + using ElementBMma = cute::conditional_t, tfloat32_t, ElementB>; + + // Handle mixed dtypes and MMA. + using RealElementA = cute::conditional_t; + using RealElementB = cute::conditional_t; + using RealElementAMma = cute::conditional_t; + // Always the same for element B. + using RealElementBMma = RealElementB; + + static_assert( + !IsMixedInput || TiledMmaGmmaMajorB == GMMA::Major::K || sizeof_bits::value == 16, + "Mixed input GEMM does not support MN major layout except for 16bit"); + + using AtomLayoutMNK = cute::conditional_t< + cute::is_any_of_v< + KernelScheduleType, + KernelTmaWarpSpecializedCooperative, + KernelPtrArrayTmaWarpSpecializedCooperative>, + Layout>, + Layout>>; + + using TiledMma = decltype(cute::make_tiled_mma( + cute::GMMA::rs_op_selector< + RealElementAMma, + RealElementBMma, + ElementAccumulator, + TileShape_MNK, + GMMA::Major::K, + GMMA::Major::K>(), + AtomLayoutMNK{})); + + using GmemTiledCopyA = decltype(detail::sm90_cluster_shape_to_tma_atom(shape<1>(ClusterShape_MNK{}))); + using GmemTiledCopyB = decltype(detail::sm90_cluster_shape_to_tma_atom(shape<0>(ClusterShape_MNK{}))); + + using SmemLayoutAtomA = decltype(detail::rs_smem_selector< + GmmaMajorA, + ElementAMma, + decltype(cute::get<0>(TileShape_MNK{})), + decltype(cute::get<2>(TileShape_MNK{})), + IsWarpSpecializedTransposeB>()); + using SmemLayoutAtomB = decltype(detail::rs_smem_selector< + GmmaMajorB, + ElementBMma, + decltype(cute::get<1>(TileShape_MNK{})), + decltype(cute::get<2>(TileShape_MNK{})), + IsWarpSpecializedTransposeB>()); + + static constexpr size_t SmemAlignmentA = cutlass::detail::alignment_for_swizzle(SmemLayoutAtomA{}); + static constexpr size_t SmemAlignmentB = cutlass::detail::alignment_for_swizzle(SmemLayoutAtomB{}); + static constexpr int SmemAlignment = static_cast(cute::max(SmemAlignmentA, SmemAlignmentB)); + + // Handle mixed dtype array GEMM's size of tensor map storage. + static constexpr size_t TensorMapStorage = sizeof(cute::TmaDescriptor) * size_t(IsMixedInput) * 4; + static constexpr int KernelSmemCarveout = static_cast(TensorMapStorage); + static constexpr int Sm90ReducedSmemCapacityBytes = detail::sm90_smem_capacity_bytes - KernelSmemCarveout; + + static constexpr int PipelineStages = + IsMixedInput ? (IsArrayOfPointersGemm ? detail::compute_stage_count_or_override_single_affine_transformed_input< + Sm90ReducedSmemCapacityBytes, + RealElementA, + RealElementB, + ElementScale, + ElementZero, + TileShape_MNK, + StageCountType::bytes, + SmemAlignment>(StageCountType{}) + : detail::compute_stage_count_or_override_single_affine_transformed_input< + detail::sm90_smem_capacity_bytes, + RealElementA, + RealElementB, + ElementScale, + ElementZero, + TileShape_MNK, + StageCountType::bytes, + SmemAlignment>(StageCountType{})) + : detail::compute_stage_count_or_override< + detail::sm90_smem_capacity_bytes, + ElementAMma, + ElementBMma, + TileShape_MNK, + StageCountType::bytes, + SmemAlignment>(StageCountType{}); + + using DispatchPolicy = cute::conditional_t< + IsMixedInput, + cute::conditional_t< + IsArrayOfPointersGemm, + MainloopSm90ArrayTmaGmmaWarpSpecializedMixedInput, + MainloopSm90TmaGmmaRmemAWarpSpecializedMixedInput>, + MainloopSm90TmaGmmaRmemAWarpSpecialized>; + + using SmemCopyAtomA = cute::conditional_t>; + using SmemCopyAtomB = cute::conditional_t, void>; + + // We pack the scale data with the operand that will be optionally scaled and converted before MMA. + using StrideA = cute::conditional_t< + cute::is_layout>::value, + GmemLayoutATag_, + TagToStrideA_t>; + using StrideB = cute::conditional_t< + cute::is_layout>::value, + GmemLayoutBTag_, + TagToStrideB_t>; + + using CollectiveOp = CollectiveMmaArrayMixedInput< + DispatchPolicy, + TileShape_MNK, + ElementPairA, + StrideA, + ElementPairB, + StrideB, + TiledMma, + GmemTiledCopyA, + SmemLayoutAtomA, + SmemCopyAtomA, + cute::identity, + GmemTiledCopyB, + SmemLayoutAtomB, + SmemCopyAtomB, + cute::identity>; + + static_assert( + SmemAlignment == static_cast(cute::max(CollectiveOp::SmemAlignmentA, CollectiveOp::SmemAlignmentB))); +}; + +///////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass::gemm::collective + +///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/collective/collective_builder_mixed_input.hpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/collective/collective_builder_mixed_input.hpp new file mode 100644 index 00000000..8288ace9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/collective/collective_builder_mixed_input.hpp @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +///////////////////////////////////////////////////////////////////////////////////////////////// +#include "cutlass/gemm/collective/collective_builder.hpp" +#include "cutlass_extensions/gemm/collective/collective_mma_array_mixed_input.hpp" + +namespace cutlass::gemm::collective { + +///////////////////////////////////////////////////////////////////////////////////////////////// + +template < + class ArchTag, + class OpClass, + class ElementA, + class GmemLayoutA, + int AlignmentA, + class ElementB, + class GmemLayoutB, + int AlignmentB, + class ElementAccumulator, + class TileShape_MNK, + class ClusterShape_MNK, + class StageCountType, + class KernelScheduleType, + class Enable = void> +struct CollectiveBuilderMixedInput { + static_assert(sizeof(ElementA) == 0, "Could not build a collective for given parameters."); +}; + +///////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass::gemm::collective + +///////////////////////////////////////////////////////////////////////////////////////////////// + +#include "cutlass_extensions/gemm/collective/builders/sm90_gmma_builder_mixed_input.inl" +///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/collective/collective_mma_array_mixed_input.hpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/collective/collective_mma_array_mixed_input.hpp new file mode 100644 index 00000000..08afdffd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/collective/collective_mma_array_mixed_input.hpp @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "cutlass/detail/dependent_false.hpp" + +///////////////////////////////////////////////////////////////////////////////////////////////// + +namespace cutlass::gemm::collective { + +///////////////////////////////////////////////////////////////////////////////////////////////// + +template < + class DispatchPolicy, + class TileShape, + class ElementA, + class StrideA, + class ElementB, + class StrideB, + class TiledMma, + class GmemTiledCopyA, + class SmemLayoutAtomA, + class SmemCopyAtomA, + class TransformA, + class GmemTiledCopyB, + class SmemLayoutAtomB, + class SmemCopyAtomB, + class TransformB> +struct CollectiveMmaArrayMixedInput { + static_assert(cutlass::detail::dependent_false, "Could not find a mainloop specialization."); +}; + +///////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass::gemm::collective + +///////////////////////////////////////////////////////////////////////////////////////////////// + +#include "cutlass_extensions/gemm/collective/sm90_mma_array_tma_gmma_rs_warpspecialized_mixed_input_.hpp" +///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/collective/sm90_mma_array_tma_gmma_rs_warpspecialized_mixed_input_.hpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/collective/sm90_mma_array_tma_gmma_rs_warpspecialized_mixed_input_.hpp new file mode 100644 index 00000000..22b34479 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/collective/sm90_mma_array_tma_gmma_rs_warpspecialized_mixed_input_.hpp @@ -0,0 +1,1538 @@ +/*************************************************************************************************** + * Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +#pragma once + +#include "cute/algorithm/functional.hpp" +#include "cute/algorithm/gemm.hpp" +#include "cute/arch/cluster_sm90.hpp" +#include "cute/arch/copy_sm90.hpp" +#include "cute/atom/mma_atom.hpp" +#include "cute/numeric/arithmetic_tuple.hpp" +#include "cutlass/cuda_host_adapter.hpp" +#include "cutlass/cutlass.h" +#include "cutlass/gemm/dispatch_policy.hpp" +#include "cutlass/numeric_types.h" +#include "cutlass/pipeline/pipeline.hpp" +#include "cutlass/trace.h" +#include "cutlass_extensions/detail/collective/mixed_input_utils.hpp" + +#define GROUP_SIZE 128 + +///////////////////////////////////////////////////////////////////////////////////////////////// + +namespace cutlass::gemm::collective { +using namespace cute; + +///////////////////////////////////////////////////////////////////////////////////////////////// + +// WarpSpecialized Mainloop +template < + int Stages, + class ClusterShape, + class KernelSchedule_, + class TileShape_, + class ElementAOptionalTuple, + class StrideA_, + class ElementBOptionalTuple, + class StrideB_, + class TiledMma_, + class GmemTiledCopyA_, + class SmemLayoutAtomA_, + class SmemCopyAtomA_, + class TransformA_, + class GmemTiledCopyB_, + class SmemLayoutAtomB_, + class SmemCopyAtomB_, + class TransformB_> +struct CollectiveMmaArrayMixedInput< + MainloopSm90ArrayTmaGmmaWarpSpecializedMixedInput, + TileShape_, + ElementAOptionalTuple, + StrideA_, + ElementBOptionalTuple, + StrideB_, + TiledMma_, + GmemTiledCopyA_, + SmemLayoutAtomA_, + SmemCopyAtomA_, + TransformA_, + GmemTiledCopyB_, + SmemLayoutAtomB_, + SmemCopyAtomB_, + TransformB_> { + public: + enum class ConversionMode { DirectConvert, ConvertAndScale, ConvertAndScaleWithZero }; + + // + // Type Aliases + // + using DispatchPolicy = MainloopSm90ArrayTmaGmmaWarpSpecializedMixedInput; + using TileShape = TileShape_; + using KernelSchedule = KernelSchedule_; + + private: + template + friend struct detail::MixedGroupedGemmInputUtils; + using CollectiveType = CollectiveMma< + DispatchPolicy, + TileShape_, + ElementAOptionalTuple, + StrideA_, + ElementBOptionalTuple, + StrideB_, + TiledMma_, + GmemTiledCopyA_, + SmemLayoutAtomA_, + SmemCopyAtomA_, + TransformA_, + GmemTiledCopyB_, + SmemLayoutAtomB_, + SmemCopyAtomB_, + TransformB_>; + using Utils = detail::MixedGroupedGemmInputUtils; + + // + // Type Aliases + // + using ScaleA = detail::deduce_mixed_width_dtype_t<1, ElementAOptionalTuple>; + using ScaleB = detail::deduce_mixed_width_dtype_t<1, ElementBOptionalTuple>; + using ZeroA = detail::deduce_mixed_width_dtype_t<2, ElementAOptionalTuple>; + using ZeroB = detail::deduce_mixed_width_dtype_t<2, ElementBOptionalTuple>; + + public: + static_assert( + cute::is_tuple::value ^ cute::is_tuple::value, + "Either A OR B must be a tuple. It must take the from {ElementOperand, [ElementScale], [ElementZero]}. Inputs in " + "[] are optional."); + + using ElementA = detail::deduce_mixed_width_dtype_t<0, ElementAOptionalTuple>; + using ElementB = detail::deduce_mixed_width_dtype_t<0, ElementBOptionalTuple>; + static constexpr bool IsATransformed = cute::is_tuple::value; + using ElementScale = cute::conditional_t; + using ElementZero = cute::conditional_t; + // For cases where we can't have a void type, we can use this to allow the code to compile when the scale / zero is + // void. + using NonVoidElementScale = cute::conditional_t, float, ElementScale>; + using NonVoidElementZero = cute::conditional_t, float, ElementZero>; + + using StrideA = StrideA_; + using InternalStrideA = cute::remove_pointer_t; + using StrideB = StrideB_; + using InternalStrideB = cute::remove_pointer_t; + + using StrideScale = cute::Stride, int64_t, int64_t>; + using NonVoidStrideScale = + cute::conditional_t, cute::Stride<_1, int64_t, int64_t>, StrideScale>; + + static_assert( + (IsATransformed && (cutlass::gemm::detail::is_k_major() || is_layout::value || + is_layout::value)) || + (!IsATransformed && (cutlass::gemm::detail::is_k_major() || is_layout::value || + is_layout::value)), + "The transformed type must be K-major."); + + static_assert( + (IsATransformed && (sizeof(ElementB) == 2)) || (!IsATransformed && (sizeof(ElementA) == 2)) || + ((cutlass::gemm::detail::is_k_major() || is_layout::value || + is_layout::value) && + (cutlass::gemm::detail::is_k_major() || is_layout::value || + is_layout::value)), + "The unscaled element must be 2 bytes OR both inputs must be K-major"); + + static_assert( + cutlass::gemm::detail::is_mn_major(), + "Scale must be MN major [Col Major if A is scaled, Row Major if B is scaled]."); + + using CtaShape_MNK = decltype(shape_div(TileShape{}, ClusterShape{})); + using TiledMma = TiledMma_; + using ElementAccumulator = typename TiledMma::ValTypeC; + using GmemTiledCopyA = GmemTiledCopyA_; + using GmemTiledCopyB = GmemTiledCopyB_; + using GmemTiledCopyScale = cute::SM90_TMA_LOAD; + using SmemLayoutAtomA = SmemLayoutAtomA_; + using SmemLayoutAtomB = SmemLayoutAtomB_; + using SmemCopyAtomA = SmemCopyAtomA_; + using SmemCopyAtomB = SmemCopyAtomB_; + using SmemCopyAtomScale = Copy_Atom; + + // We must ensure the type to be scaled goes to RF + static constexpr bool SwapAB = !IsATransformed; + using SwappedStrideA = cute::conditional_t; + using SwappedStrideB = cute::conditional_t; + using InternalSwappedStrideA = cute::conditional_t; + using InternalSwappedStrideB = cute::conditional_t; + using SwappedSmemLayoutAtomA = cute::conditional_t; + using SwappedSmemLayoutAtomB = cute::conditional_t; + using SwappedSmemCopyAtomA = cute::conditional_t; + using SwappedSmemCopyAtomB = cute::conditional_t; + // TMA converts f32 input to tf32 when copying from GMEM to SMEM + // For all other types, cast to size equivalent uint type to avoid any rounding by TMA. + static constexpr bool ConvertF32toTF32A = cute::is_same_v; + static constexpr bool ConvertF32toTF32B = cute::is_same_v; + using ConvertedElementA = cute::conditional_t>>; + using ConvertedElementB = cute::conditional_t>>; + using RealSwappedElementA = cute::conditional_t; + using RealSwappedElementB = cute::conditional_t; + using SwappedElementA = cute::conditional_t; + using SwappedElementB = cute::conditional_t; + + using TransformA = TransformA_; + using TransformB = TransformB_; + using SwappedTransformA = cute::conditional_t; + using SwappedTransformB = cute::conditional_t; + using ArchTag = typename DispatchPolicy::ArchTag; + + static constexpr int IsSubbyteA = cute::sizeof_bits_v < 8; + using TmaElementA = cute::conditional_t; + using TmaElementScale = uint_bit_t>; // in case we have array. translating to uint + // to satisfy tma descriptor's specialization + + using MainloopPipeline = cutlass::PipelineTmaAsync; + using PipelineState = cutlass::PipelineState; + using PipelineParams = typename MainloopPipeline::Params; + + static constexpr int NumProducerThreadEvents = 1; + + using SmemLayoutAtomScale = Layout(SwappedSmemLayoutAtomA{})), cute::Int<1>>>; + using ScaleTileShape = decltype(make_shape(shape<0>(TileShape{}), shape<1>(SmemLayoutAtomScale{}))); + + static_assert(cute::rank(SwappedSmemLayoutAtomA{}) == 2, "SmemLayoutAtom must be rank 2 (M/N, K)"); + static_assert( + (size<0>(TileShape{}) % size<0>(SwappedSmemLayoutAtomA{})) == 0, "SmemLayoutAtom must evenly divide tile shape."); + static_assert( + (size<2>(TileShape{}) % size<1>(SwappedSmemLayoutAtomA{})) == 0, "SmemLayoutAtom must evenly divide tile shape."); + + static_assert(cute::rank(SwappedSmemLayoutAtomB{}) == 2, "SmemLayoutAtom must be rank 2 (M/N, K)"); + static_assert( + (size<1>(TileShape{}) % size<0>(SwappedSmemLayoutAtomB{})) == 0, "SmemLayoutAtom must evenly divide tile shape."); + static_assert( + (size<2>(TileShape{}) % size<1>(SwappedSmemLayoutAtomB{})) == 0, "SmemLayoutAtom must evenly divide tile shape."); + + static_assert(rank(SmemLayoutAtomScale{}) == 2, "SmemLayoutAtomScale must be rank 2"); + static_assert( + (size<0>(TileShape{}) % size<0>(SmemLayoutAtomScale{})) == 0, "SmemLayoutAtomScale must equal the tile shape."); + static_assert( + (size<2>(TileShape{}) % size<1>(SmemLayoutAtomScale{})) == 0, + "SmemLayoutAtomScale must evenly divide tile k shape."); + + /// Tile along modes in a way that maximizes the TMA box size. + using SmemLayoutA = decltype(detail::get_smem_layout( + SwappedSmemLayoutAtomA{}, select<0, 2>(TileShape{}), InternalSwappedStrideA{})); + using SmemLayoutB = decltype(detail::get_smem_layout( + SwappedSmemLayoutAtomB{}, select<1, 2>(TileShape{}), InternalSwappedStrideB{})); + + // It is assumed that the scales and zero-points share the same smem layout + using SmemLayoutScale = decltype(tile_to_shape( + SmemLayoutAtomScale{}, + make_shape(shape<0>(ScaleTileShape{}), shape<1>(ScaleTileShape{}), Int{}), + cute::conditional_t< + ::cutlass::gemm::detail::is_major<0, NonVoidStrideScale>(), + Step<_2, _1, _3>, + Step<_1, _2, _3>>{})); + + static_assert(DispatchPolicy::Stages >= 2, "Specialization requires Stages set to value 2 or more."); + static_assert( + not cute::is_base_of::value && + cute::is_base_of::value, + "MMA atom must source A from rmem and B operand from smem_desc for this mainloop."); + static_assert( + cute::is_same_v || cute::is_same_v, + "GmemTiledCopy - invalid SM90 TMA copy atom specified."); + static_assert( + cute::is_same_v || cute::is_same_v, + "GmemTiledCopy - invalid SM90 TMA copy atom specified."); + + // To relax them, we need to handle loading more than 1 row of scales for every main loop iteration. + // We must also handle updating the pipeline transaction bytes on the fly. + static_assert(size<1>(SmemLayoutAtomScale{}) == 1, "size<1>(SmemLayoutAtomScale) must be 1."); + + private: + static constexpr ConversionMode get_conversion_mode() { + if constexpr (cute::is_void_v) { + return ConversionMode::DirectConvert; + } else if constexpr (cute::is_void_v) { + return ConversionMode::ConvertAndScale; + } else { + return ConversionMode::ConvertAndScaleWithZero; + } + } + + public: + static constexpr ConversionMode KernelConversionMode = get_conversion_mode(); + static constexpr bool ModeHasScales = KernelConversionMode == ConversionMode::ConvertAndScale || + KernelConversionMode == ConversionMode::ConvertAndScaleWithZero; + static constexpr bool UseScaleLookupTable = + KernelConversionMode == ConversionMode::ConvertAndScale && cutlass::detail::is_Array_v; + static constexpr size_t SmemAlignmentA = cutlass::detail::alignment_for_swizzle(SmemLayoutA{}); + static constexpr size_t SmemAlignmentB = cutlass::detail::alignment_for_swizzle(SmemLayoutB{}); + static constexpr size_t SmemAlignmentScale = cute::max(SmemAlignmentA, SmemAlignmentB); + + static_assert(SmemAlignmentA >= 128 and SmemAlignmentB >= 128, "Require at least 128B alignment"); + + struct SharedStorage { + static constexpr int scale_elements = Utils::elements_per_smem_scale(); + static constexpr int zero_elements = Utils::elements_per_smem_zero(); + struct TensorStorage { + CUTE_ALIGNAS(SmemAlignmentA) cute::ArrayEngine> smem_A; + CUTE_ALIGNAS(SmemAlignmentB) cute::ArrayEngine> smem_B; + cute::ArrayEngine smem_scale; + cute::ArrayEngine smem_zero; + } tensors; + + struct TensorMapStorage { + cute::TmaDescriptor smem_tensormap_A; + cute::TmaDescriptor smem_tensormap_B; + cute::TmaDescriptor smem_tensormap_scale; + cute::TmaDescriptor smem_tensormap_zero; + }; + + using PipelineStorage = typename MainloopPipeline::SharedStorage; + PipelineStorage pipeline; + }; + using TensorStorage = typename SharedStorage::TensorStorage; + using TensorMapStorage = typename SharedStorage::TensorMapStorage; + using PipelineStorage = typename SharedStorage::PipelineStorage; + + static constexpr bool IsGroupedGemmKernel = !cute::is_same_v; + + // kernel Arguments + // Host side kernel arguments + struct Arguments { + ElementA const** ptr_A; + StrideA dA; + ElementB const** ptr_B; + StrideB dB; + ElementScale const** ptr_S = nullptr; + NonVoidStrideScale const* dS{}; + int chunk_size = 0; + ElementZero const** ptr_Z = nullptr; + }; + + // Device side kernel params + struct Params { + // Assumption: StrideA is congruent with Problem_MK + using LayoutA = + decltype(detail::get_gmem_layout(repeat_like(InternalSwappedStrideA{}, int32_t(0)), InternalSwappedStrideA{})); + using LayoutB = + decltype(detail::get_gmem_layout(repeat_like(InternalSwappedStrideB{}, int32_t(0)), InternalSwappedStrideB{})); + + using TMA_A = decltype(make_tma_copy( + GmemTiledCopyA{}, + make_tensor(detail::get_logical_ptr(static_cast(nullptr)), LayoutA{}), + SmemLayoutA{}(_, _, cute::Int<0>{}), + make_shape(shape<0>(TileShape{}), shape<2>(TileShape{})), + size<1>(ClusterShape{}))); // mcast along N mode for this M load, if any + // Assumption: StrideB is congruent with Problem_NK + using TMA_B = decltype(make_tma_copy( + GmemTiledCopyB{}, + make_tensor(detail::get_logical_ptr(static_cast(nullptr)), LayoutB{}), + SmemLayoutB{}(_, _, cute::Int<0>{}), + make_shape(shape<1>(TileShape{}), shape<2>(TileShape{})), + size<0>(ClusterShape{}))); // mcast along M mode for this N load, if any + + using TMA_Scale = decltype(make_tma_copy( + GmemTiledCopyScale{}, + make_tensor( + detail::get_logical_ptr(static_cast(nullptr)), + repeat_like(NonVoidStrideScale{}, int32_t(0)), + NonVoidStrideScale{}), + SmemLayoutScale{}(_, _, cute::Int<0>{}), + ScaleTileShape{}, + _1{})); // mcast along N mode for this M load, if any. Scale is ALWAYS loaded with A for RF kernel + + using TMA_Zero = decltype(make_tma_copy( + GmemTiledCopyScale{}, + make_tensor( + detail::get_logical_ptr(static_cast(nullptr)), + repeat_like(NonVoidStrideScale{}, int32_t(0)), + NonVoidStrideScale{}), + SmemLayoutScale{}(_, _, cute::Int<0>{}), + ScaleTileShape{}, + _1{})); // mcast along N mode for this M load, if any. Scale is ALWAYS loaded with A for RF kernel + + TMA_A tma_load_a; + TMA_B tma_load_b; + uint32_t tma_transaction_bytes = TmaTransactionBytes; + TMA_Scale tma_load_scale; + TMA_Zero tma_load_zero; + void* tensormaps; + SwappedElementA const** ptr_A; + SwappedStrideA ptr_dA; + SwappedElementB const** ptr_B; + SwappedStrideB ptr_dB; + NonVoidElementScale const** ptr_S; + NonVoidStrideScale const* dS; + NonVoidElementZero const** ptr_Z; + int64_t scale_k; + int chunk_size; + int reload_factor = (chunk_size + size<2>(TileShape{}) - 1) / size<2>(TileShape{}); + InternalSwappedStrideA dA; + InternalSwappedStrideB dB; + }; + + // + // Methods + // + + template + static constexpr Params to_underlying_arguments(ProblemShape problem_shapes, Arguments const& args, void* workspace) { + // These tensor shapes (only applicable for grouped gemm) and pointers are only used to create tensormap/tma desc. + // These will be replaced with correct values before the initial tma load. + auto init_shape = repeat_like(typename ProblemShape::UnderlyingProblemShape{}, int32_t(1)); + auto init_M = get<0>(init_shape); + auto init_N = get<1>(init_shape); + auto init_K = get<2>(init_shape); + + if constexpr (SwapAB) { + init_M = get<1>(init_shape); + init_N = get<0>(init_shape); + } + // Batches/Groups are managed by using appropriate pointers to input matrices + const uint32_t mock_L = 1; + SwappedElementA const* ptr_A_first_batch; + SwappedElementB const* ptr_B_first_batch; + SwappedStrideA ptr_dA; + SwappedStrideB ptr_dB; + InternalSwappedStrideA dA; + InternalSwappedStrideB dB; + + if constexpr (not SwapAB) { + ptr_A_first_batch = reinterpret_cast(args.ptr_A); + ptr_B_first_batch = reinterpret_cast(args.ptr_B); + } else { + ptr_A_first_batch = reinterpret_cast(args.ptr_B); + ptr_B_first_batch = reinterpret_cast(args.ptr_A); + } + + if constexpr (IsGroupedGemmKernel) { + // Strides for Grouped Gemm will be replaced prior to the first access regardless. + if constexpr (not SwapAB) { + ptr_dA = args.dA; + ptr_dB = args.dB; + } else { + ptr_dA = args.dB; + ptr_dB = args.dA; + } + dA = InternalSwappedStrideA{}; + if constexpr (is_layout::value) { + dA = make_layout( + transform_leaf( + dA.shape(), + [](auto x) { + if constexpr (not is_static_v) { + return static_cast(1); + } else { + return x; + } + }), + dA.stride()); + } + dB = InternalSwappedStrideB{}; + } else { + // Tensor shapes for Ptr-Array are initialized correctly only here. + auto problem_shape_MNK = problem_shapes.get_host_problem_shape(0); + init_M = get<0>(problem_shape_MNK); + init_N = get<1>(problem_shape_MNK); + init_K = get<2>(problem_shape_MNK); + + if constexpr (not SwapAB) { + dA = args.dA; + dB = args.dB; + } else { + dA = args.dB; + dB = args.dA; + } + ptr_dA = SwappedStrideA{}; + ptr_dB = SwappedStrideB{}; + } + Tensor tensor_a = make_tensor(ptr_A_first_batch, detail::get_gmem_layout(make_shape(init_M, init_K, mock_L), dA)); + Tensor tensor_b = make_tensor(ptr_B_first_batch, detail::get_gmem_layout(make_shape(init_N, init_K, mock_L), dB)); + + typename Params::TMA_A tma_load_a = make_tma_copy( + GmemTiledCopyA{}, + tensor_a, + SmemLayoutA{}(_, _, cute::Int<0>{}), + make_shape(shape<0>(TileShape{}), shape<2>(TileShape{})), + size<1>(ClusterShape{})); // mcast along N mode for this M load, if any + typename Params::TMA_B tma_load_b = make_tma_copy( + GmemTiledCopyB{}, + tensor_b, + SmemLayoutB{}(_, _, cute::Int<0>{}), + make_shape(shape<1>(TileShape{}), shape<2>(TileShape{})), + size<0>(ClusterShape{})); // mcast along M mode for this N load, if any + typename Params::TMA_Scale tma_load_scale{}; + typename Params::TMA_Zero tma_load_zero{}; + + void* tensormaps = workspace; + auto args_setup = + [&](auto ptr_A, auto ptr_B, int64_t scale_k = 0, int chunk_size = 0, int reload_factor = 1) -> Params { + return { + tma_load_a, + tma_load_b, + TmaTransactionBytes, + tma_load_scale, + tma_load_zero, + tensormaps, + reinterpret_cast(ptr_A), + ptr_dA, + reinterpret_cast(ptr_B), + ptr_dB, + reinterpret_cast(args.ptr_S), + args.dS, + reinterpret_cast(args.ptr_Z), + scale_k, + chunk_size, + reload_factor, + dA, + dB}; + }; + + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + return SwapAB ? args_setup(args.ptr_B, args.ptr_A) : args_setup(args.ptr_A, args.ptr_B); + } else if constexpr (ModeHasScales) { + auto fake_scale_k = 1; + ElementScale const* ptr_S = reinterpret_cast(args.ptr_S); + StrideScale dS{}; + Tensor tensor_scale = + make_tensor(detail::get_logical_ptr(ptr_S), make_layout(make_shape(init_M, fake_scale_k, mock_L), dS)); + tma_load_scale = make_tma_copy( + GmemTiledCopyScale{}, + tensor_scale, + SmemLayoutScale{}(_, _, cute::Int<0>{}), + ScaleTileShape{}, + _1{}); // mcast along N mode for this M load, if any + + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + return SwapAB ? args_setup( + args.ptr_B, + args.ptr_A, + fake_scale_k, + args.chunk_size, + (args.chunk_size + size<2>(TileShape{}) - 1) / size<2>(TileShape{})) + : args_setup( + args.ptr_A, + args.ptr_B, + fake_scale_k, + args.chunk_size, + (args.chunk_size + size<2>(TileShape{}) - 1) / size<2>(TileShape{})); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + ElementZero const* ptr_Z = reinterpret_cast(args.ptr_Z); + Tensor tensor_zero = + make_tensor(detail::get_logical_ptr(ptr_Z), make_layout(make_shape(init_M, fake_scale_k, mock_L), dS)); + tma_load_zero = make_tma_copy( + GmemTiledCopyScale{}, + tensor_zero, + SmemLayoutScale{}(_, _, cute::Int<0>{}), + ScaleTileShape{}, + _1{}); // mcast along N mode for this M load, if any + return SwapAB ? args_setup( + args.ptr_B, + args.ptr_A, + fake_scale_k, + args.chunk_size, + (args.chunk_size + size<2>(TileShape{}) - 1) / size<2>(TileShape{})) + : args_setup( + args.ptr_A, + args.ptr_B, + fake_scale_k, + args.chunk_size, + (args.chunk_size + size<2>(TileShape{}) - 1) / size<2>(TileShape{})); + + } else { + static_assert( + cutlass::detail::dependent_false, + "Conversion mode not handled in to_underlying_arguments."); + } + } else { + static_assert( + cutlass::detail::dependent_false, "Conversion mode not handled in to_underlying_arguments."); + } + } + + template + static size_t get_workspace_size(ProblemShape const& problem_shape, Arguments const& args, int sm_count) { + constexpr size_t SizeOfCuTensorMap = sizeof(cute::TmaDescriptor); + + // Calculating workspace size + auto calculate_workspace_size = [SizeOfCuTensorMap, sm_count](uint32_t num_input_tensors) { + return num_input_tensors * SizeOfCuTensorMap * sm_count; + }; + + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + // Allocate gmem space for input tensormaps per each SM, A tensormap copies followed by B tensormap copies + return calculate_workspace_size(2); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + // Allocate gmem space for input tensormaps per each SM, A tensormap copies followed by B tensormap copies, + // followed by scale tensormap copies + return calculate_workspace_size(3); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + // Allocate gmem space for input tensormaps per each SM, A tensormap copies followed by B tensormap copies, + // followed by scale and zeros tensormap copies + return calculate_workspace_size(4); + } else { + static_assert( + cutlass::detail::dependent_false, "Conversion mode not handled in get_workspace_size."); + } + } + + template + static cutlass::Status initialize_workspace( + ProblemShape const& problem_shape, + Arguments const& args, + void* workspace, + cudaStream_t stream, + CudaHostAdapter* cuda_adapter = nullptr) { + return cutlass::Status::kSuccess; + } + + template + CUTLASS_HOST_DEVICE static bool can_implement(ProblemShape problem_shapes, Arguments const& args) { + constexpr int tma_alignment_bits = 128; + constexpr int min_tma_aligned_elements_A = tma_alignment_bits / cutlass::sizeof_bits::value; + constexpr int min_tma_aligned_elements_B = tma_alignment_bits / cutlass::sizeof_bits::value; + + bool implementable = true; + if (problem_shapes.is_host_problem_shape_available()) { + // Check alignment for all problem sizes + for (int i = 0; i < problem_shapes.groups(); i++) { + auto problem_shape_MNKL = append<4>(problem_shapes.get_host_problem_shape(i), 1); + auto [M, N, K, L] = problem_shape_MNKL; + auto get_stride = [](auto stride) { + if constexpr (cute::is_pointer_v>) { + return *stride; + } else { + return stride; + } + }; + auto dA = get_stride(args.dA); + auto dB = get_stride(args.dB); + implementable = implementable && cutlass::detail::check_alignment( + detail::get_gmem_layout(cute::make_shape(M, K, L), dA)); + implementable = implementable && cutlass::detail::check_alignment( + detail::get_gmem_layout(cute::make_shape(N, K, L), dB)); + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + implementable = implementable && (args.ptr_S == nullptr); + implementable = implementable && (args.ptr_Z == nullptr); + } else if constexpr (ModeHasScales) { + const int scale_mn = SwapAB ? N : M; + const int scale_k = (K + args.chunk_size - 1) / args.chunk_size; + constexpr int min_tma_aligned_elements_scale = tma_alignment_bits / cutlass::sizeof_bits::value; + implementable = implementable && cutlass::detail::check_alignment( + cute::make_shape(scale_mn, scale_k, L), StrideScale{}); + implementable = implementable && (args.chunk_size == K || ((args.chunk_size % size<2>(TileShape{})) == 0)); + implementable = implementable && args.chunk_size != 0; + implementable = implementable && (args.ptr_S != nullptr); + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + implementable = implementable && (args.ptr_Z == nullptr); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + constexpr int min_tma_aligned_elements_zero = tma_alignment_bits / cutlass::sizeof_bits::value; + implementable = implementable && cutlass::detail::check_alignment( + cute::make_shape(scale_mn, scale_k, L), StrideScale{}); + implementable = implementable && (args.ptr_Z != nullptr); + } else { + static_assert( + cutlass::detail::dependent_false, "Conversion mode not handled in can_implement."); + } + } else { + static_assert( + cutlass::detail::dependent_false, "Conversion mode not handled in can_implement."); + } + } + } + + if (!implementable) { + CUTLASS_TRACE_HOST(" CAN IMPLEMENT: Problem Size doesn't meet the minimum alignment requirements for TMA.\n"); + } + return implementable; + } + + static constexpr int K_PIPE_MAX = DispatchPolicy::Stages; + static constexpr int K_PIPE_MMAS = 1; + static constexpr uint32_t TmaTransactionBytesMK = Utils::compute_tma_transaction_bytes_mk(); + static constexpr uint32_t TmaTransactionBytesNK = Utils::compute_tma_transaction_bytes_nk(); + static constexpr uint32_t TmaTransactionBytesExtra = Utils::compute_tma_transaction_bytes_extra(); + static constexpr uint32_t TmaTransactionBytes = + TmaTransactionBytesMK + TmaTransactionBytesNK + TmaTransactionBytesExtra; + + // Set up the data needed by this collective for load and mma. + // Returns a tuple of tensors. The collective and the kernel layer have the contract that the + // returned tuple must contain at least two elements, with the first two elements being: + // gA_mkl - The tma tensor, A after a local tile so it has shape (BLK_M,BLK_K,m,k,l) + // gB_nkl - The tma tensor, B after a local tile so it has shape (BLK_N,BLK_K,n,k,l) + // The rest of the tensors can be specified as needed by this collective. + template + CUTLASS_DEVICE auto load_init(ProblemShape_MNKL const& problem_shape_MNKL, Params const& mainloop_params) const { + using X = Underscore; + // Separate out problem shape for convenience + auto [M, N, K, L] = problem_shape_MNKL; + const int32_t mock_L = 1; + + // TMA requires special handling of strides to deal with coord codomain mapping + // Represent the full tensors -- get these from TMA + Tensor mA_mkl = mainloop_params.tma_load_a.get_tma_tensor( + shape(detail::get_gmem_layout(make_shape(M, K, mock_L), mainloop_params.dA))); // (m,k,l) + Tensor mB_nkl = mainloop_params.tma_load_b.get_tma_tensor( + shape(detail::get_gmem_layout(make_shape(N, K, mock_L), mainloop_params.dB))); // (n,k,l) + + // Make tiled views, defer the slice + Tensor gA_mkl = local_tile(mA_mkl, TileShape{}, make_coord(_, _, _), Step<_1, X, _1>{}); // (BLK_M,BLK_K,m,k,l) + Tensor gB_nkl = local_tile(mB_nkl, TileShape{}, make_coord(_, _, _), Step{}); // (BLK_N,BLK_K,n,k,l) + + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + return cute::make_tuple(gA_mkl, gB_nkl); + } else if constexpr (ModeHasScales) { + // The real scale_k that actually works + // auto scale_k = K / mainloop_params.chunk_size; + auto scale_k = K / GROUP_SIZE; + + Tensor mS_mkl = mainloop_params.tma_load_scale.get_tma_tensor(make_shape(M, scale_k, L)); // (m,scale_k,l) + Tensor gS_mkl = local_tile(mS_mkl, ScaleTileShape{}, make_coord(_, _)); // (BLK_M,BLK_Scale_K,m,scale_k,l) + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + return cute::make_tuple(gA_mkl, gB_nkl, gS_mkl); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + Tensor mZ_mkl = mainloop_params.tma_load_zero.get_tma_tensor(make_shape(M, scale_k, L)); // (m,scale_k,l) + Tensor gZ_mkl = local_tile(mZ_mkl, ScaleTileShape{}, make_coord(_, _)); // (BLK_M,BLK_Scale_K,m,scale_k,l) + return cute::make_tuple(gA_mkl, gB_nkl, gS_mkl, gZ_mkl); + } else { + static_assert(cutlass::detail::dependent_false, "Conversion mode not handled in load_init."); + } + } else { + static_assert(cutlass::detail::dependent_false, "Conversion mode not handled in load_init."); + } + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Perform a collective-scoped matrix multiply-accumulate + // Producer Perspective + template + CUTLASS_DEVICE void load( + Params const& mainloop_params, + MainloopPipeline pipeline, + PipelineState smem_pipe_write, + cute::tuple const& load_inputs, + cute::tuple const& input_tensormaps, + BlockCoord const& blk_coord, + KTileIterator k_tile_iter, + int k_tile_count, + int thread_idx, + uint32_t block_rank_in_cluster, + TensorStorage& shared_tensors) { + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + static_assert(sizeof...(Ts) == 2, "Direct convert needs two inputs"); + static_assert(sizeof...(TMs) == 2, "Direct convert needs two tensormaps"); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + static_assert(sizeof...(Ts) == 3, "Scaled convert needs three inputs"); + static_assert(sizeof...(TMs) == 3, "Scaled convert needs three tensormaps"); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + static_assert(sizeof...(Ts) == 4, "Scaled and zero convert needs four inputs"); + static_assert(sizeof...(TMs) == 4, "Scaled and zero convert needs four tensormaps"); + } else { + static_assert(cutlass::detail::dependent_false, "Conversion mode not handled in TMA load."); + } + + Tensor sA_ = make_tensor(make_smem_ptr(shared_tensors.smem_A.begin()), SmemLayoutA{}); // (BLK_M,BLK_K,PIPE) + Tensor sB_ = make_tensor(make_smem_ptr(shared_tensors.smem_B.begin()), SmemLayoutB{}); // (BLK_N,BLK_K,PIPE) + Tensor sA = as_position_independent_swizzle_tensor(sA_); // (BLK_M,BLK_K,PIPE) + Tensor sB = as_position_independent_swizzle_tensor(sB_); // (BLK_N,BLK_K,PIPE) + + // + // Prepare the TMA loads for A and B + // + + constexpr uint32_t cluster_shape_x = get<0>(typename DispatchPolicy::ClusterShape()); + uint2 cluster_local_block_id = {block_rank_in_cluster % cluster_shape_x, block_rank_in_cluster / cluster_shape_x}; + + Tensor gA_mkl = get<0>(load_inputs); + Tensor gB_nkl = get<1>(load_inputs); + + auto block_tma_a = mainloop_params.tma_load_a.get_slice(cluster_local_block_id.y); + auto block_tma_b = mainloop_params.tma_load_b.get_slice(cluster_local_block_id.x); + + // Partition the inputs based on the current block coordinates. + auto [m_coord, n_coord, k_coord, l_coord] = blk_coord; + Tensor gA = gA_mkl(_, _, m_coord, _, l_coord); // (BLK_M,BLK_K,k) + Tensor gB = gB_nkl(_, _, n_coord, _, l_coord); // (BLK_N,BLK_K,k) + + // Applies the mapping from block_tma_a + Tensor tAgA = block_tma_a.partition_S(gA); // (TMA,TMA_M,TMA_K,k) + Tensor tAsA = block_tma_a.partition_D(sA); // (TMA,TMA_M,TMA_K,PIPE) + + Tensor tBgB = block_tma_b.partition_S(gB); // (TMA,TMA_N,TMA_K,k) + Tensor tBsB = block_tma_b.partition_D(sB); // (TMA,TMA_N,TMA_K,PIPE) + + uint16_t mcast_mask_a = 0; + uint16_t mcast_mask_b = 0; + uint16_t mcast_mask_s = 0; + + // Issue TmaLoads + // Maps the tile -> block, value + if constexpr (cute::is_same_v) { + auto block_layout = Layout{}; // (m,n) -> block_id + for (int n = 0; n < size<1>(block_layout); ++n) { + mcast_mask_a |= (uint16_t(1) << block_layout(cluster_local_block_id.x, n, Int<0>{})); + } + } + + if constexpr (cute::is_same_v) { + auto block_layout = Layout{}; // (m,n) -> block_id + for (int m = 0; m < size<0>(block_layout); ++m) { + mcast_mask_b |= (uint16_t(1) << block_layout(m, cluster_local_block_id.y, Int<0>{})); + } + } + + auto extra_input_partitions = Utils::partition_extra_tma_inputs( + mainloop_params, load_inputs, shared_tensors, cluster_local_block_id, m_coord, l_coord); + + // Mainloop + CUTLASS_PRAGMA_NO_UNROLL + for (; k_tile_count > 0; --k_tile_count) { + // LOCK smem_pipe_write for _writing_ + pipeline.producer_acquire(smem_pipe_write); + + // + // Copy gmem to smem for *k_tile_iter + // + + using BarrierType = typename MainloopPipeline::ProducerBarrierType; + BarrierType* tma_barrier = pipeline.producer_get_barrier(smem_pipe_write); + + int write_stage = smem_pipe_write.index(); + if (cute::elect_one_sync()) { + copy( + mainloop_params.tma_load_a.with(get<0>(input_tensormaps), *tma_barrier, mcast_mask_a), + tAgA(_, _, _, *k_tile_iter), + tAsA(_, _, _, write_stage)); + copy( + mainloop_params.tma_load_b.with(get<1>(input_tensormaps), *tma_barrier, mcast_mask_b), + tBgB(_, _, _, *k_tile_iter), + tBsB(_, _, _, write_stage)); + } + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + // Nothing extra to do. + } else if constexpr (ModeHasScales) { + // scale copy + auto tSgS = get<0>(extra_input_partitions); + auto tSsS = get<1>(extra_input_partitions); + + // Temporary factor which will determine which k tile to reload from gmem. Needed so we don't modify tma + // transaction bytes on the fly. We must do a ceiling divide here to correctly handle with chunk_size == K. In + // that case, we don't require that K is a multiple of the threadblock tile K + const int scale_load_k = *k_tile_iter / 1; + // const int scale_load_k = *k_tile_iter / mainloop_params.reload_factor; // This will always be 0 when + // chunk_size == K. + if (cute::elect_one_sync()) { + copy( + mainloop_params.tma_load_scale.with(get<2>(input_tensormaps), *tma_barrier, mcast_mask_s), + tSgS(_, _, _, scale_load_k), + tSsS(_, _, _, write_stage)); + } + + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + // Nothing extra to do + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + // zero copy + auto tZgZ = get<2>(extra_input_partitions); + auto tZsZ = get<3>(extra_input_partitions); + if (cute::elect_one_sync()) { + copy( + mainloop_params.tma_load_zero.with(get<3>(input_tensormaps), *tma_barrier, mcast_mask_s), + tZgZ(_, _, _, scale_load_k), + tZsZ(_, _, _, write_stage)); + } + } else { + static_assert( + cutlass::detail::dependent_false, "Conversion mode not handled for TMA copy op."); + } + } else { + static_assert(cutlass::detail::dependent_false, "Conversion mode not handled for TMA copy op."); + } + ++k_tile_iter; + + // Advance smem_pipe_write + ++smem_pipe_write; + } + } + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Perform a Producer Epilogue to prevent early exit of blocks in a Cluster + CUTLASS_DEVICE void load_tail(MainloopPipeline pipeline, PipelineState smem_pipe_write) { + int lane_predicate = cute::elect_one_sync(); + + // Issue the epilogue waits + if (lane_predicate) { + // This helps avoid early exit of blocks in Cluster. + // Waits for all stages to either be released (all + // Consumer UNLOCKs), or if the stage was never used + // then it would just be acquired since the phase was + // still inverted from make_producer_start_state. + pipeline.producer_tail(smem_pipe_write); + } + } + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + /// Perform a collective-scoped matrix multiply-accumulate + /// Consumer Perspective + template + CUTLASS_DEVICE void + mma(MainloopPipeline pipeline, + PipelineState smem_pipe_read, + FrgTensorC& accum, + int k_tile_count, + int thread_idx, + TensorStorage& shared_tensors, + Params const& mainloop_params) { + static_assert(is_rmem::value, "C tensor must be rmem resident."); + static_assert(cute::rank(SmemLayoutA{}) == 3, "Smem layout must be rank 3."); + static_assert(cute::rank(SmemLayoutB{}) == 3, "Smem layout must be rank 3."); + static_assert(cute::rank(SwappedSmemLayoutAtomA{}) == 2, "SwappedSmemLayoutAtomA must be rank 2."); + static_assert(cute::rank(SwappedSmemLayoutAtomB{}) == 2, "SwappedSmemLayoutAtomB must be rank 2."); + static_assert( + !cute::is_void_v, + "SM90 GMMA mainloops must specify a non-void copy atom for smem sourced instructions."); + static_assert( + cute::is_void_v, + "SM90 GMMA mainloops cannot have a non-void copy atom for smem sourced instructions."); + + // Obtain warp index + int warp_idx = canonical_warp_idx_sync(); + [[maybe_unused]] int warp_group_thread_idx = thread_idx % 128; + + Tensor sA_ = make_tensor(make_smem_ptr(shared_tensors.smem_A.begin()), SmemLayoutA{}); // (BLK_M,BLK_K,PIPE) + Tensor sA = as_position_independent_swizzle_tensor(sA_); // (BLK_M,BLK_K,PIPE) + + Tensor sB = make_tensor(make_smem_ptr(shared_tensors.smem_B.begin()), SmemLayoutB{}); // (BLK_N,BLK_K,PIPE) + + // + // Define C accumulators and A/B partitioning + // + + // Layout of warp group to thread mapping + + static_assert( + stride<0>(typename TiledMma::BLayout{}) == 0 and + size<0>(typename TiledMma::BLayout{}) == NumThreadsPerWarpGroup, + "Stride of the first mode must be 0 and the size of the mode must be NumThreadsPerWarpGroup"); + + constexpr int MmaWarpGroups = size(TiledMma{}) / NumThreadsPerWarpGroup; + Layout warp_group_thread_layout = make_layout(Int{}, Int{}); + + int warp_group_idx = __shfl_sync(0xFFFFFFFF, thread_idx / NumThreadsPerWarpGroup, 0); + + TiledMma tiled_mma; + auto mma_thread_slice = tiled_mma.get_thread_slice(thread_idx); + Tensor tCsA = mma_thread_slice.partition_A(sA); + auto mma_warpgroup_slice = tiled_mma.get_slice(warp_group_thread_layout(warp_group_idx)); + + // Allocate fragments and descriptors + Tensor tCrA_mma = mma_thread_slice.partition_fragment_A(sA(_, _, Int<0>{})); // (MMA,MMA_M,MMA_K,PIPE) + Tensor tCrA_load = [&] { + if constexpr (not is_layout::value) { + // Make register tensor with MMA layout + return make_fragment_like(tCrA_mma); + } else { + // Make register tensor matching smem layout, converter will take care of de-swizzling + return make_tensor_like(tCsA(_, _, _, Int<0>{})); + } + }(); + Tensor tCsB = mma_warpgroup_slice.partition_B(sB); // (MMA,MMA_N,MMA_K,PIPE) + Tensor tCrB = mma_warpgroup_slice.make_fragment_B(tCsB); // (MMA,MMA_N,MMA_K,PIPE) + + // + // Copy Atom A retiling + // + auto smem_tiled_copy_A = make_tiled_copy_A(SwappedSmemCopyAtomA{}, tiled_mma); + auto smem_thr_copy_A = smem_tiled_copy_A.get_thread_slice(warp_group_thread_idx); + + Tensor tCrA_copy_view = smem_thr_copy_A.retile_D(tCrA_load); // (CPY,CPY_M,CPY_K) + + // Partition of thread -> shared and thread -> RF + auto partitioned_extra_info = Utils::partition_extra_mma_info(mma_thread_slice, shared_tensors); + auto copy_partitions_extra_info = + Utils::retile_extra_mma_info(tiled_mma, partitioned_extra_info, warp_group_thread_idx); + + CUTE_STATIC_ASSERT_V(size<1>(tCsA) == size<1>(tCrA_copy_view)); // CPY_M + CUTE_STATIC_ASSERT_V(size<2>(tCsA) == size<2>(tCrA_copy_view)); // CPY_K + CUTE_STATIC_ASSERT_V(size<1>(tCrA_mma) == size<1>(accum)); // MMA_M + CUTE_STATIC_ASSERT_V(size<1>(tCsB) == size<2>(accum)); // N + CUTE_STATIC_ASSERT_V(size<2>(tCsA) == size<2>(tCsB)); // K + CUTE_STATIC_ASSERT_V(size<3>(tCsA) == size<3>(tCsB)); // PIPE + CUTE_STATIC_ASSERT_V(Int{} == size<2>(sA)); // PIPE + CUTE_STATIC_ASSERT_V(Int{} == size<2>(sB)); // PIPE + + // + // PIPELINED MAIN LOOP + // + + // We release buffers to producer warps(dma load) with some mmas in flight + PipelineState smem_pipe_release = smem_pipe_read; + + multiply_add fma; + + constexpr int NumMMAsPerChunk = GROUP_SIZE / cute::get<0, 1>(tCsB.shape())(); + constexpr int NumChunksPerTileK = cute::size<1>(sA.shape())() / GROUP_SIZE; + cute::array intermediate_array; + + constexpr int K_BLOCK_MAX = size<2>(tCrA_load); + constexpr int K_WAIT_MAX = cute::min(K_BLOCK_MAX - 1, 7); + static_assert(K_BLOCK_MAX >= 4, "Consider increasing TileShapeK"); + + ConsumerToken barrier_token = {BarrierStatus::WaitAgain}; + // First k tile + { + barrier_token = pipeline.consumer_try_wait(smem_pipe_read); + pipeline.consumer_wait(smem_pipe_read, barrier_token); + + int read_stage = smem_pipe_read.index(); + + ++smem_pipe_read; + barrier_token = pipeline.consumer_try_wait(smem_pipe_read); + + // copy smem->rmem for A operand + + Utils::copy_tensors_MK( + smem_tiled_copy_A, tCsA, tCrA_copy_view, partitioned_extra_info, copy_partitions_extra_info, 0, read_stage); + if (K_BLOCK_MAX > 1) { + Utils::copy_tensors_MK( + smem_tiled_copy_A, tCsA, tCrA_copy_view, partitioned_extra_info, copy_partitions_extra_info, 1, read_stage); + } + + // src: tCrA_load, dst: tCrA_mma + Utils::convert_A_kblock(tCrA_load, tCrA_mma, 0); + + // Unroll the K mode manually to set scale D to 1 + CUTLASS_PRAGMA_UNROLL + for (int chunk_id = 0; chunk_id < NumChunksPerTileK; ++chunk_id) { + tiled_mma.accumulate_ = GMMA::ScaleOut::Zero; + + CUTLASS_PRAGMA_UNROLL + for (int mma_id = 0; mma_id < NumMMAsPerChunk; ++mma_id) { + int k_block = chunk_id * NumMMAsPerChunk + mma_id; + + warpgroup_arrive(); + + // (V,M) x (V,N) => (V,M,N) + cute::gemm(tiled_mma, tCrA_mma(_, _, k_block), tCrB(_, _, k_block, read_stage), intermediate_array[chunk_id]); + tiled_mma.accumulate_ = GMMA::ScaleOut::One; + + warpgroup_commit_batch(); + + if (k_block < K_BLOCK_MAX - 2) { + Utils::copy_tensors_MK( + smem_tiled_copy_A, + tCsA, + tCrA_copy_view, + partitioned_extra_info, + copy_partitions_extra_info, + k_block + 2, + read_stage); + } + if (k_block < K_BLOCK_MAX - 1) { + Utils::convert_A_kblock(tCrA_load, tCrA_mma, k_block + 1); + } + } + } + + warpgroup_wait<0>(); + + CUTLASS_PRAGMA_UNROLL + for (int chunk_id_ = 0; chunk_id_ < NumChunksPerTileK; ++chunk_id_) { + warpgroup_fence_operand(intermediate_array[chunk_id_]); + + // Apply the group-wise scaling + // tCrS ((4, _2, _2), MMA_M, _1) + // accum ((2, _2, _2), MMA_M, _1) + auto tCrS = cute::get<1>(partitioned_extra_info); + for (int mma_m = 0; mma_m < size<1>(accum); mma_m++) { + for (int m = 0; m < size<0, 1>(accum); m++) { + for (int n = 0; n < size<0, 2>(accum); n++) { + for (int e = 0; e < size<0, 0>(accum); e++) { + auto accum_coord = make_coord(make_tuple(e, m, n), mma_m, 0); + auto scale_coord = make_coord(make_tuple(0, m, 0), mma_m, 0); + + if (chunk_id_ == 0) { + accum(accum_coord) = + intermediate_array[chunk_id_](accum_coord) * static_cast(tCrS(scale_coord)[0]); + } else { + accum(accum_coord) = + fma(intermediate_array[chunk_id_](accum_coord), + static_cast(tCrS(scale_coord)[chunk_id_]), + accum(accum_coord)); + } + } + } + } + } + } + + --k_tile_count; + if (k_tile_count > 0) { + // Wait for K_BLOCK_MAX - 1 to be in flight to ensure that it is safe to overwrite the A registers for the first + // mma. + pipeline.consumer_wait(smem_pipe_read, barrier_token); + + Utils::copy_tensors_MK( + smem_tiled_copy_A, + tCsA, + tCrA_copy_view, + partitioned_extra_info, + copy_partitions_extra_info, + 0, + smem_pipe_read.index()); + + Utils::copy_tensors_MK( + smem_tiled_copy_A, + tCsA, + tCrA_copy_view, + partitioned_extra_info, + copy_partitions_extra_info, + 1, + smem_pipe_read.index()); + + Utils::convert_A_kblock(tCrA_load, tCrA_mma, 0); + } + } + + if (k_tile_count == 0) { + return; + } + + // Mainloop GMMAs + CUTLASS_PRAGMA_NO_UNROLL + for (; k_tile_count > 1; --k_tile_count) { + // + // Compute on k_tile + // + + int read_stage = smem_pipe_read.index(); + ++smem_pipe_read; + + // Unroll the K mode manually to set scale D to 1 + CUTLASS_PRAGMA_UNROLL + for (int chunk_id = 0; chunk_id < NumChunksPerTileK; ++chunk_id) { + tiled_mma.accumulate_ = GMMA::ScaleOut::Zero; + + CUTLASS_PRAGMA_UNROLL + for (int mma_id = 0; mma_id < NumMMAsPerChunk; ++mma_id) { + int k_block = chunk_id * NumMMAsPerChunk + mma_id; + + warpgroup_arrive(); + // (V,M) x (V,N) => (V,M,N) + cute::gemm(tiled_mma, tCrA_mma(_, _, k_block), tCrB(_, _, k_block, read_stage), intermediate_array[chunk_id]); + tiled_mma.accumulate_ = GMMA::ScaleOut::One; + warpgroup_commit_batch(); + + if (k_block == K_BLOCK_MAX - 1) { + pipeline.consumer_release(smem_pipe_release); // UNLOCK smem_pipe_release, done _computing_ on it + ++smem_pipe_release; + } + + if (k_block == 0) { + barrier_token = pipeline.consumer_try_wait(smem_pipe_read); + } + + if (k_block == K_BLOCK_MAX - 1) { + // The last k_block + + warpgroup_wait<0>(); + + CUTLASS_PRAGMA_UNROLL + for (int chunk_id_ = 0; chunk_id_ < NumChunksPerTileK; ++chunk_id_) { + warpgroup_fence_operand(intermediate_array[chunk_id_]); + + // Apply the group-wise scaling + auto tCrS = cute::get<1>(partitioned_extra_info); + for (int mma_m = 0; mma_m < size<1>(accum); mma_m++) { + for (int m = 0; m < size<0, 1>(accum); m++) { + for (int n = 0; n < size<0, 2>(accum); n++) { + for (int e = 0; e < size<0, 0>(accum); e++) { + auto accum_coord = make_coord(make_tuple(e, m, n), mma_m, 0); + auto scale_coord = make_coord(make_tuple(0, m, 0), mma_m, 0); + + accum(accum_coord) = + fma(intermediate_array[chunk_id_](accum_coord), + static_cast(tCrS(scale_coord)[chunk_id_]), + accum(accum_coord)); + } + } + } + } + } + + pipeline.consumer_wait(smem_pipe_read, barrier_token); + + // copy scales when passing k_block=0 + Utils::copy_tensors_MK( + smem_tiled_copy_A, + tCsA, + tCrA_copy_view, + partitioned_extra_info, + copy_partitions_extra_info, + 0, + smem_pipe_read.index()); + Utils::copy_tensors_MK( + smem_tiled_copy_A, + tCsA, + tCrA_copy_view, + partitioned_extra_info, + copy_partitions_extra_info, + 1, + smem_pipe_read.index()); + Utils::convert_A_kblock(tCrA_load, tCrA_mma, 0); + } else { + if (k_block < K_BLOCK_MAX - 2) { + Utils::copy_tensors_MK( + smem_tiled_copy_A, + tCsA, + tCrA_copy_view, + partitioned_extra_info, + copy_partitions_extra_info, + k_block + 2, + read_stage); + } + Utils::convert_A_kblock(tCrA_load, tCrA_mma, k_block + 1); + } + } + } + } + + { + // + // Last k tile + // + Tensor intermediate = make_fragment_like(accum); + + int read_stage = smem_pipe_read.index(); + + tiled_mma.accumulate_ = GMMA::ScaleOut::Zero; + + // Unroll the K mode manually to set scale D to 1 + CUTLASS_PRAGMA_UNROLL + for (int k_block = 0; k_block < K_BLOCK_MAX; ++k_block) { + warpgroup_arrive(); + // (V,M) x (V,N) => (V,M,N) + cute::gemm(tiled_mma, tCrA_mma(_, _, k_block), tCrB(_, _, k_block, read_stage), intermediate); + tiled_mma.accumulate_ = GMMA::ScaleOut::One; + warpgroup_commit_batch(); + + if (k_block == K_BLOCK_MAX - 1) { + // release prior barrier + pipeline.consumer_release(smem_pipe_release); // UNLOCK smem_pipe_release, done _computing_ on it + ++smem_pipe_release; + } + + if (k_block < K_BLOCK_MAX - 2) { + Utils::copy_tensors_MK( + smem_tiled_copy_A, + tCsA, + tCrA_copy_view, + partitioned_extra_info, + copy_partitions_extra_info, + k_block + 2, + read_stage); + } + if (k_block < K_BLOCK_MAX - 1) { + Utils::convert_A_kblock(tCrA_load, tCrA_mma, k_block + 1); + } + + if ((k_block + 1) % NumMMAsPerChunk == 0) { + tiled_mma.accumulate_ = GMMA::ScaleOut::Zero; + + warpgroup_wait<0>(); + warpgroup_fence_operand(intermediate); + + // Apply the group-wise scaling + auto tCrS = cute::get<1>(partitioned_extra_info); + for (int mma_m = 0; mma_m < size<1>(accum); mma_m++) { + for (int m = 0; m < size<0, 1>(accum); m++) { + for (int n = 0; n < size<0, 2>(accum); n++) { + for (int e = 0; e < size<0, 0>(accum); e++) { + auto accum_coord = make_coord(make_tuple(e, m, n), mma_m, 0); + auto scale_coord = make_coord(make_tuple(0, m, 0), mma_m, 0); + int scale_idx = k_block / NumMMAsPerChunk; + + accum(accum_coord) = fma( + intermediate(accum_coord), static_cast(tCrS(scale_coord)[scale_idx]), accum(accum_coord)); + } + } + } + } + } + } + } + } + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + /// Perform a Consumer Epilogue to release all buffers + CUTLASS_DEVICE void mma_tail(MainloopPipeline pipeline, PipelineState smem_pipe_release, int k_tile_count) { + // Prologue GMMAs + int prologue_mma_count = 1; + k_tile_count -= prologue_mma_count; + + smem_pipe_release.advance(k_tile_count); + + // Wait on all GMMAs to complete + // warpgroup_wait<0>(); + + for (int count = 0; count < prologue_mma_count; ++count) { + pipeline.consumer_release(smem_pipe_release); // UNLOCK smem_pipe_release, done _computing_ on it + ++smem_pipe_release; + } + } + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // + // Methods to perform different parts of TMA/Tensormap modifications + // + CUTLASS_DEVICE auto tensormaps_init( + Params const& mainloop_params, TensorMapStorage& shared_tensormaps, int32_t sm_count, int32_t sm_idx) { + cute::TmaDescriptor* gmem_tensormap = reinterpret_cast(mainloop_params.tensormaps); + + cute::TmaDescriptor* tma_desc_a = &gmem_tensormap[sm_idx]; + cute::TmaDescriptor* tma_desc_b = &gmem_tensormap[sm_idx + sm_count]; + cute::TmaDescriptor* tma_desc_scale = &gmem_tensormap[sm_idx + 2 * sm_count]; + cute::TmaDescriptor* tma_desc_zero = &gmem_tensormap[sm_idx + 3 * sm_count]; + + // Bringing tensormaps from params to smem for modification later + Tensor pA_tensormap = make_tensor(mainloop_params.tma_load_a.get_tma_descriptor(), Int<1>{}, Int<1>{}); + Tensor sA_tensormap = make_tensor(make_smem_ptr(&shared_tensormaps.smem_tensormap_A), Int<1>{}, Int<1>{}); + Tensor pB_tensormap = make_tensor(mainloop_params.tma_load_b.get_tma_descriptor(), Int<1>{}, Int<1>{}); + Tensor sB_tensormap = make_tensor(make_smem_ptr(&shared_tensormaps.smem_tensormap_B), Int<1>{}, Int<1>{}); + + if (cute::elect_one_sync()) { + copy(recast(pA_tensormap), recast(sA_tensormap)); + copy(recast(pB_tensormap), recast(sB_tensormap)); + } + + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + Tensor pS_tensormap = make_tensor(mainloop_params.tma_load_scale.get_tma_descriptor(), Int<1>{}, Int<1>{}); + Tensor sS_tensormap = make_tensor(make_smem_ptr(&shared_tensormaps.smem_tensormap_scale), Int<1>{}, Int<1>{}); + if (cute::elect_one_sync()) { + copy(recast(pS_tensormap), recast(sS_tensormap)); + } + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + Tensor pZ_tensormap = make_tensor(mainloop_params.tma_load_zero.get_tma_descriptor(), Int<1>{}, Int<1>{}); + Tensor sZ_tensormap = make_tensor(make_smem_ptr(&shared_tensormaps.smem_tensormap_zero), Int<1>{}, Int<1>{}); + if (cute::elect_one_sync()) { + copy(recast(pZ_tensormap), recast(sZ_tensormap)); + } + } else if constexpr (KernelConversionMode != ConversionMode::DirectConvert) { + static_assert( + cutlass::detail::dependent_false, "Conversion mode not handled in tensormaps_init."); + } + + __syncwarp(); + + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + return cute::make_tuple(tma_desc_a, tma_desc_b); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + return cute::make_tuple(tma_desc_a, tma_desc_b, tma_desc_scale); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + return cute::make_tuple(tma_desc_a, tma_desc_b, tma_desc_scale, tma_desc_zero); + } else { + static_assert( + cutlass::detail::dependent_false, "Conversion mode not handled in tensormaps_init."); + } + } + + // Replace address for the global tensor (to be done by single thread) + CUTLASS_DEVICE + void tensormaps_replace_global_address( + TensorMapStorage& shared_tensormaps, Params const& mainloop_params, int32_t next_batch) { + // Replacing global_address for the next batch + cute::tma_descriptor_replace_addr_in_shared_mem( + shared_tensormaps.smem_tensormap_A, mainloop_params.ptr_A[next_batch]); + cute::tma_descriptor_replace_addr_in_shared_mem( + shared_tensormaps.smem_tensormap_B, mainloop_params.ptr_B[next_batch]); + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + cute::tma_descriptor_replace_addr_in_shared_mem( + shared_tensormaps.smem_tensormap_scale, mainloop_params.ptr_S[next_batch]); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + cute::tma_descriptor_replace_addr_in_shared_mem( + shared_tensormaps.smem_tensormap_zero, mainloop_params.ptr_Z[next_batch]); + } else if constexpr (KernelConversionMode != ConversionMode::DirectConvert) { + static_assert( + cutlass::detail::dependent_false, + "Conversion mode not handled in tensormaps_replace_global_address."); + } + } + + // Replace dim and strides for the global tensor - used only for Grouped GEMM (to be done by single thread) + template + CUTLASS_DEVICE void tensormaps_replace_global_tensor_properties( + TensorMapStorage& shared_tensormaps, + Params const& mainloop_params, + int32_t next_group, + ProblemShape_MNKL problem_shape_mnkl) { + const uint32_t M = get<0>(problem_shape_mnkl); + const uint32_t N = get<1>(problem_shape_mnkl); + const uint32_t K = get<2>(problem_shape_mnkl); + + // Replace all dims for consistency + constexpr int MaxTensorRank = 5; + cute::array prob_shape_A = {1, 1, 1, 1, 1}; + cute::array prob_stride_A = {0, 0, 0, 0, 0}; + cute::array prob_shape_B = {1, 1, 1, 1, 1}; + cute::array prob_stride_B = {0, 0, 0, 0, 0}; + cute::array prob_shape_scale = {1, 1, 1, 1, 1}; + cute::array prob_stride_scale = {0, 0, 0, 0, 0}; + cute::array prob_shape_zero = {1, 1, 1, 1, 1}; + cute::array prob_stride_zero = {0, 0, 0, 0, 0}; + + SwappedElementA const* ptr_A = nullptr; + Tensor tensor_a = + make_tensor(ptr_A, detail::get_gmem_layout(make_shape(M, K, Int<1>{}), mainloop_params.ptr_dA[next_group])); + + SwappedElementB const* ptr_B = nullptr; + Tensor tensor_b = + make_tensor(ptr_B, detail::get_gmem_layout(make_shape(N, K, Int<1>{}), mainloop_params.ptr_dB[next_group])); + + cute::detail::fill_tma_gmem_shape_stride(mainloop_params.tma_load_a, tensor_a, prob_shape_A, prob_stride_A); + cute::detail::fill_tma_gmem_shape_stride(mainloop_params.tma_load_b, tensor_b, prob_shape_B, prob_stride_B); + + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + NonVoidElementScale const* ptr_S = nullptr; + // auto scale_k = K / mainloop_params.chunk_size; + auto scale_k = K / GROUP_SIZE; + Tensor tensor_scale = + make_tensor(detail::get_logical_ptr(ptr_S), make_shape(M, scale_k, Int<1>{}), mainloop_params.dS[next_group]); + cute::detail::fill_tma_gmem_shape_stride( + mainloop_params.tma_load_scale, tensor_scale, prob_shape_scale, prob_stride_scale); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + ElementZero const* ptr_Z = nullptr; + // auto scale_k = K / mainloop_params.chunk_size; + auto scale_k = K / GROUP_SIZE; + Tensor tensor_zero = + make_tensor(detail::get_logical_ptr(ptr_Z), make_shape(M, scale_k, Int<1>{}), mainloop_params.dS[next_group]); + cute::detail::fill_tma_gmem_shape_stride( + mainloop_params.tma_load_zero, tensor_zero, prob_shape_zero, prob_stride_zero); + } else if constexpr (KernelConversionMode != ConversionMode::DirectConvert) { + static_assert( + cutlass::detail::dependent_false, + "Conversion mode not handled in tensormaps_replace_global_tensor_properties."); + } + + // Convert strides to byte strides + for (uint64_t& stride : prob_stride_A) { + stride = (stride * sizeof_bits_v) / 8; + } + for (uint64_t& stride : prob_stride_B) { + stride = (stride * sizeof_bits_v) / 8; + } + for (uint64_t& stride : prob_stride_scale) { + stride = (stride * sizeof_bits_v) / 8; + } + for (uint64_t& stride : prob_stride_zero) { + stride = (stride * sizeof_bits_v) / 8; + } + + cute::tma_descriptor_replace_dims_strides_in_shared_mem( + shared_tensormaps.smem_tensormap_A, prob_shape_A, prob_stride_A); + cute::tma_descriptor_replace_dims_strides_in_shared_mem( + shared_tensormaps.smem_tensormap_B, prob_shape_B, prob_stride_B); + + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + cute::tma_descriptor_replace_dims_strides_in_shared_mem( + shared_tensormaps.smem_tensormap_scale, prob_shape_scale, prob_stride_scale); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + cute::tma_descriptor_replace_dims_strides_in_shared_mem( + shared_tensormaps.smem_tensormap_zero, prob_shape_zero, prob_stride_zero); + } else if constexpr (KernelConversionMode != ConversionMode::DirectConvert) { + static_assert( + cutlass::detail::dependent_false, + "Conversion mode not handled in tensormaps_replace_global_tensor_properties."); + } + } + + template + CUTLASS_DEVICE void tensormaps_perform_update( + TensorMapStorage& shared_tensormaps, + Params const& mainloop_params, + cute::tuple const& input_tensormaps, + ProblemShape_MNKL problem_shape_mnkl, + int32_t next_batch) { + if (cute::elect_one_sync()) { + // Replacing global_address for the next batch + tensormaps_replace_global_address(shared_tensormaps, mainloop_params, next_batch); + + if constexpr (IsGroupedGemmKernel) { + // Replacing global dims and strides for the next batch + tensormaps_replace_global_tensor_properties(shared_tensormaps, mainloop_params, next_batch, problem_shape_mnkl); + } + } + } + + template + CUTLASS_DEVICE void + tensormaps_cp_fence_release(TensorMapStorage& shared_tensormaps, cute::tuple const& input_tensormaps) { + if (cute::elect_one_sync()) { + cute::tma_desc_commit_group(); + cute::tma_desc_wait_group(); + } + // Entire warp must do this (i.e. it's aligned) + tma_descriptor_cp_fence_release(get<0>(input_tensormaps), shared_tensormaps.smem_tensormap_A); + tma_descriptor_cp_fence_release(get<1>(input_tensormaps), shared_tensormaps.smem_tensormap_B); + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + tma_descriptor_cp_fence_release(get<2>(input_tensormaps), shared_tensormaps.smem_tensormap_scale); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + tma_descriptor_cp_fence_release(get<3>(input_tensormaps), shared_tensormaps.smem_tensormap_zero); + } else if constexpr (KernelConversionMode != ConversionMode::DirectConvert) { + static_assert( + cutlass::detail::dependent_false, + "Conversion mode not handled in tensormaps_cp_fence_release."); + } + } + + // The entire warp must call this function collectively (that is, the instructions are aligned) + template + CUTLASS_DEVICE void tensormaps_fence_acquire(cute::tuple const& input_tensormaps) { + cute::tma_descriptor_fence_acquire(get<0>(input_tensormaps)); + cute::tma_descriptor_fence_acquire(get<1>(input_tensormaps)); + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + cute::tma_descriptor_fence_acquire(get<2>(input_tensormaps)); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + cute::tma_descriptor_fence_acquire(get<3>(input_tensormaps)); + } else if constexpr (KernelConversionMode != ConversionMode::DirectConvert) { + static_assert( + cutlass::detail::dependent_false, "Conversion mode not handled in tensormaps_fence_acquire."); + } + } + + template + CUTLASS_DEVICE InputTensors tensors_perform_update( + InputTensors const& input_tensors, + [[maybe_unused]] Params const& mainloop_params, + [[maybe_unused]] ProblemShape_MNKL problem_shape_mnkl, + [[maybe_unused]] int32_t next_batch) { + return input_tensors; + } +}; + +///////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass::gemm::collective + +///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/cutlass_gemm_caller.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/cutlass_gemm_caller.cuh new file mode 100644 index 00000000..737916d8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/cutlass_gemm_caller.cuh @@ -0,0 +1,62 @@ +// Adapted from +// https://github.com/vllm-project/vllm/blob/main/csrc/quantization/cutlass_w8a8/c3x/cutlass_gemm_caller.cuh + +#pragma once + +// clang-format will break include orders +// clang-format off +#include + +#include +#include + +#include "cutlass/cutlass.h" + +#include "cute/tensor.hpp" +#include "cute/atom/mma_atom.hpp" +#include "cutlass/numeric_types.h" + +#include "cutlass/gemm/device/gemm_universal_adapter.h" +#include "cutlass/gemm/kernel/gemm_universal.hpp" +#include "cutlass/epilogue/collective/collective_builder.hpp" +#include "cutlass/gemm/collective/collective_builder.hpp" +#include "cutlass/util/packed_stride.hpp" + +// clang-format on + +/** + * Helper function for checking CUTLASS errors + */ +#define CUTLASS_CHECK(status) \ + { \ + cutlass::Status error = status; \ + TORCH_CHECK(error == cutlass::Status::kSuccess, cutlassGetStatusString(error)); \ + } + +template +void cutlass_gemm_caller( + torch::Device device, + cute::Shape prob_shape, + typename GemmKernel::MainloopArguments mainloop_args, + typename GemmKernel::EpilogueArguments epilogue_args, + typename GemmKernel::TileSchedulerArguments scheduler = {}) { + cutlass::KernelHardwareInfo hw_info; + hw_info.device_id = c10::cuda::current_device(); + hw_info.sm_count = at::cuda::getCurrentDeviceProperties()->multiProcessorCount; + typename GemmKernel::Arguments args{ + cutlass::gemm::GemmUniversalMode::kGemm, prob_shape, mainloop_args, epilogue_args, hw_info, scheduler}; + + // Launch the CUTLASS GEMM kernel. + using GemmOp = cutlass::gemm::device::GemmUniversalAdapter; + GemmOp gemm_op; + CUTLASS_CHECK(gemm_op.can_implement(args)); + + size_t workspace_size = gemm_op.get_workspace_size(args); + auto const workspace_options = torch::TensorOptions().dtype(torch::kUInt8).device(device); + auto workspace = torch::empty(workspace_size, workspace_options); + + auto stream = at::cuda::getCurrentCUDAStream(device.index()); + + cutlass::Status status = gemm_op.run(args, workspace.data_ptr(), stream); + CUTLASS_CHECK(status); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/dispatch_policy.hpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/dispatch_policy.hpp new file mode 100644 index 00000000..6019e4a5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/dispatch_policy.hpp @@ -0,0 +1,38 @@ +// Adapted from https://github.com/vllm-project/vllm/blob/main/csrc/cutlass_extensions/gemm/dispatch_policy.hpp + +#pragma once + +#include "cutlass/gemm/dispatch_policy.hpp" + +namespace cutlass::gemm { + +////////////////////////////////////////////////////////////////////////////// + +// FP8 related policies (including Blocked Scaled Accumulation) +// `ScaleGranularityM` specifies scaling granularity along M, while zero-value +// `ScaleGranularityM` indicates that scaling granularity is +// `size<0>(TileShape_MNK{})` along M. +template +struct KernelTmaWarpSpecializedCooperativeFP8BlockScaledSubGroupMAccum : KernelTmaWarpSpecializedCooperative {}; + +// n-buffer in smem (Hopper TMA), pipelined with Hopper GMMA and TMA, Warp +// specialized dynamic schedule For FP8 kernels with Block Scaling +template < + int Stages_, + class ClusterShape_ = Shape<_1, _1, _1>, + class KernelSchedule = KernelTmaWarpSpecialized, + int ScaleGranularityM = 0 // `ScaleGranularityM` specifies scaling granularity along M, + // while zero-value `ScaleGranularityM` indicates that scaling + // granularity is `size<0>(TileShape_MNK{})` along M. + > +struct MainloopSm90TmaGmmaWarpSpecializedBlockScalingSubGroupMFP8 + : MainloopSm90TmaGmmaWarpSpecialized { + static_assert( + cute:: + is_same_v>, + "KernelSchedule must be one of the warp specialized policies"); +}; + +////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass::gemm diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/fp8_blockwise_gemm_sm90_dispatch.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/fp8_blockwise_gemm_sm90_dispatch.cuh new file mode 100644 index 00000000..05b70c4f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/fp8_blockwise_gemm_sm90_dispatch.cuh @@ -0,0 +1,197 @@ +// Adapted from +// https://github.com/vllm-project/vllm/blob/main/csrc/quantization/cutlass_w8a8/c3x/scaled_mm_blockwise_sm90_fp8_dispatch.cuh +#pragma once + +#include "cute/tensor.hpp" +#include "cutlass/cutlass.h" +#include "cutlass/epilogue/collective/collective_builder.hpp" +#include "cutlass/epilogue/dispatch_policy.hpp" +#include "cutlass/gemm/collective/collective_builder.hpp" +#include "cutlass/gemm/device/gemm_universal_adapter.h" +#include "cutlass/gemm/dispatch_policy.hpp" +#include "cutlass/gemm/kernel/gemm_universal.hpp" +#include "cutlass/gemm/kernel/tile_scheduler_params.h" +#include "cutlass/numeric_types.h" +#include "cutlass/tensor_ref.h" +#include "cutlass_extensions/common.hpp" +#include "cutlass_extensions/gemm/cutlass_gemm_caller.cuh" +#include "cutlass_extensions/gemm/dispatch_policy.hpp" + +using namespace cute; + +template < + typename SchedulerType, + typename OutType, + int GroupSizeM_, + int GroupSizeN_, + int GroupSizeK_, + int TileSizeM_ = 128, + class ClusterShape = Shape<_1, _2, _1>> +struct cutlass_3x_gemm_fp8_blockwise { + using GroupSizeM = Int; + using GroupSizeN = Int; + using GroupSizeK = Int; + using TileSizeM = Int; + + static_assert(TileSizeM_ % GroupSizeM_ == 0, "TileSizeM must be a multiple of GroupSizeM"); + + using ElementAB = cutlass::float_e4m3_t; + + // A matrix configuration + using ElementA = ElementAB; + using LayoutA = cutlass::layout::RowMajor; + static constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; + + // B matrix configuration + using ElementB = ElementAB; + using LayoutB = cutlass::layout::ColumnMajor; + static constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; + + // C/D matrix configuration + using ElementC = void; + using LayoutC = cutlass::layout::RowMajor; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + + using ElementD = OutType; + using LayoutD = cutlass::layout::RowMajor; + static constexpr int AlignmentD = AlignmentC; + + using ScaleTileShape = Shape<_1, _128, _128>; + using ScaleConfig = decltype(cutlass::detail::sm90_trivial_blockwise_scale_config(ScaleTileShape{})); + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); + + // Multiply-accumulate blocking/pipelining details + using ElementAccumulator = float; // Element type for internal accumulation + using ElementCompute = float; // Element type for compute + using TileShape = Shape; // Threadblock-level tile size + + using ArchTag = cutlass::arch::Sm90; + using OperatorClass = cutlass::arch::OpClassTensorOp; + using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecializedCooperative; + using EpilogueTileType = cutlass::epilogue::collective::EpilogueTileAuto; + using StoreEpilogueCompute = typename cutlass::epilogue::fusion::Sm90EVT; + + using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedCooperativeFP8Blockwise; + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + TileShape, + ClusterShape, + EpilogueTileType, + ElementAccumulator, + ElementCompute, + ElementC, + LayoutC, + AlignmentC, + ElementD, + LayoutD, + AlignmentD, + EpilogueSchedule, + StoreEpilogueCompute>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementA, + cute::tuple, + AlignmentA, + ElementB, + cute::tuple, + AlignmentB, + ElementAccumulator, + TileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + KernelSchedule>::CollectiveOp; + + using GemmKernel = cutlass::gemm::kernel::GemmUniversal< + Shape, // Indicates ProblemShape + CollectiveMainloop, + CollectiveEpilogue, + SchedulerType>; +}; + +template +void cutlass_gemm_caller_blockwise( + torch::Tensor& out, + torch::Tensor const& a, + torch::Tensor const& b, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales) { + using GemmKernel = typename Gemm::GemmKernel; + using ElementAB = typename Gemm::ElementAB; + using ElementA = ElementAB; + using ElementB = ElementAB; + using ElementD = typename Gemm::ElementD; + using ElementBlockScale = float; + + using ScaleTileShape = Shape<_1, _128, _128>; + using ScaleConfig = decltype(cutlass::detail::sm90_trivial_blockwise_scale_config(ScaleTileShape{})); + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); + + int m = a.size(0); + int k = a.size(1); + int n = b.size(1); + + auto a_ptr = static_cast(a.data_ptr()); + auto b_ptr = static_cast(b.data_ptr()); + + auto a_s_ptr = static_cast(a_scales.data_ptr()); + auto b_s_ptr = static_cast(b_scales.data_ptr()); + + using StrideA = typename GemmKernel::StrideA; + using StrideB = typename GemmKernel::StrideB; + using StrideD = typename GemmKernel::StrideD; + using StrideC = typename GemmKernel::StrideC; + + StrideA a_stride = cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(m, k, 1)); + StrideB b_stride = cutlass::make_cute_packed_stride(StrideB{}, cute::make_shape(n, k, 1)); + StrideC c_stride = cutlass::make_cute_packed_stride(StrideC{}, cute::make_shape(m, n, 1)); + LayoutSFA layout_sfa = ScaleConfig::tile_atom_to_shape_SFA(make_shape(m, n, k, 1)); + LayoutSFB layout_sfb = ScaleConfig::tile_atom_to_shape_SFB(make_shape(m, n, k, 1)); + + typename GemmKernel::MainloopArguments mainloop_args{ + a_ptr, a_stride, b_ptr, b_stride, a_s_ptr, layout_sfa, b_s_ptr, layout_sfb}; + auto c_ptr = static_cast(out.data_ptr()); + typename GemmKernel::EpilogueArguments epilogue_args{{}, c_ptr, c_stride, c_ptr, c_stride}; + + typename GemmKernel::TileSchedulerArguments scheduler; + + static constexpr bool UsesStreamKScheduler = + cute::is_same_v; + + if constexpr (UsesStreamKScheduler) { + using DecompositionMode = + typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90StreamKParams::DecompositionMode; + using ReductionMode = + typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90StreamKParams::ReductionMode; + + scheduler.decomposition_mode = DecompositionMode::StreamK; + scheduler.reduction_mode = ReductionMode::Nondeterministic; + } + + cutlass_gemm_caller(a.device(), {m, n, k, 1}, mainloop_args, epilogue_args, scheduler); +} + +template +void cutlass_gemm_blockwise_sm90_fp8_dispatch( + torch::Tensor& out, + torch::Tensor const& a, + torch::Tensor const& b, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales) { + auto k = a.size(1); + auto n = b.size(1); + + if (k > 3 * n) { + cutlass_gemm_caller_blockwise>( + out, a, b, a_scales, b_scales); + } else { + cutlass_gemm_caller_blockwise< + cutlass_3x_gemm_fp8_blockwise>( + out, a, b, a_scales, b_scales); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/gemm_universal_base_compat.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/gemm_universal_base_compat.h new file mode 100644 index 00000000..b58d8431 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/gemm_universal_base_compat.h @@ -0,0 +1,356 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Adapted from +// https://github.com/NVIDIA/TensorRT-LLM/blob/be1788106245496872d18e702978e59b6bfd50e0/cpp/tensorrt_llm/cutlass_extensions/include/cutlass_extensions/gemm/device/gemm_universal_base_compat.h +#pragma once + +#include +#include +#include + +//////////////////////////////////////////////////////////////////////////////// + +namespace cutlass { +namespace gemm { +namespace device { + +///////////////////////////////////////////////////////////////////////////////////////////////// + +/* + This is the device layer from CUTLASS 2.10 (SHA - cc85b64cf676c45f98a17e3a47c0aafcf817f088) + It is replicated here since we needed to duplicate kernel level APIs for mixed dtype GEMMs + and SmoothQuant. The newer device layer is not compatible with these older kernel level APIs. + + Note: While CUTLASS 3.x supports stream-k, none of the kernels in the extensions folder support + that feature at the moment. + */ + +template +class GemmUniversalBaseCompat { + public: + using GemmKernel = GemmKernel_; + using ThreadblockShape = typename GemmKernel::Mma::Shape; + + using ElementA = typename GemmKernel::ElementA; + using LayoutA = typename GemmKernel::LayoutA; + using TensorRefA = TensorRef; + static ComplexTransform const kTransformA = GemmKernel::kTransformA; + + using ElementB = typename GemmKernel::ElementB; + using LayoutB = typename GemmKernel::LayoutB; + using TensorRefB = TensorRef; + static ComplexTransform const kTransformB = GemmKernel::kTransformB; + + using ElementC = typename GemmKernel::ElementC; + using LayoutC = typename GemmKernel::LayoutC; + using TensorRefC = TensorRef; + using TensorRefD = TensorRef; + + using ElementAccumulator = typename GemmKernel::Mma::Policy::Operator::ElementC; + + using EpilogueOutputOp = typename GemmKernel::EpilogueOutputOp; + using ThreadblockSwizzle = typename GemmKernel::ThreadblockSwizzle; + using Operator = typename GemmKernel::Operator; + + /// Argument structure + using Arguments = typename GemmKernel::Arguments; + + protected: + /// Kernel parameters object + typename GemmKernel::Params params_; + + protected: + /// Private helper to obtain the grid dimensions with fix-up for split-K + static void get_grid_shape_(gemm::GemmCoord& grid_tiled_shape, int& gemm_k_size, Arguments const& args) { + // Determine grid shape + ThreadblockSwizzle threadblock_swizzle; + + grid_tiled_shape = threadblock_swizzle.get_tiled_shape( + args.problem_size, {ThreadblockShape::kM, ThreadblockShape::kN, ThreadblockShape::kK}, args.batch_count); + + gemm_k_size = args.problem_size.k(); + + if (args.mode == GemmUniversalMode::kGemm || args.mode == GemmUniversalMode::kGemmSplitKParallel) { + int const kAlignK = + const_max(const_max(128 / sizeof_bits::value, 128 / sizeof_bits::value), 1); + + gemm_k_size = round_up(ceil_div(args.problem_size.k(), args.batch_count), kAlignK); + + if (gemm_k_size) { + grid_tiled_shape.k() = ceil_div(args.problem_size.k(), gemm_k_size); + } + } + } + + public: + /// Constructs the GEMM. + GemmUniversalBaseCompat() {} + + /// Determines whether the GEMM can execute the given problem. + static Status can_implement(Arguments const& args) { + // Determine grid shape + cutlass::gemm::GemmCoord grid_tiled_shape; + int gemm_k_size = 0; + + get_grid_shape_(grid_tiled_shape, gemm_k_size, args); + + ThreadblockSwizzle threadblock_swizzle; + dim3 grid = threadblock_swizzle.get_grid_shape(grid_tiled_shape); + + uint32_t const kGridYZMax = ((1 << (sizeof(uint16_t) * 8)) - 1); + + if (!(grid.y <= kGridYZMax && grid.z <= kGridYZMax)) { + return Status::kErrorInvalidProblem; + } + + return GemmKernel::can_implement(args); + } + + /// Gets the workspace size + static size_t get_workspace_size(Arguments const& args) { + CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::get_workspace_size()"); + + size_t workspace_bytes = 0; + + // Determine grid shape + cutlass::gemm::GemmCoord grid_tiled_shape; + int gemm_k_size = 0; + + get_grid_shape_(grid_tiled_shape, gemm_k_size, args); + + if (args.mode == GemmUniversalMode::kGemmSplitKParallel) { + // Split-K parallel always requires a temporary workspace + workspace_bytes = sizeof(ElementC) * size_t(args.batch_stride_D) * size_t(grid_tiled_shape.k()); + } else if (args.mode == GemmUniversalMode::kGemm && grid_tiled_shape.k() > 1) { + // Serial split-K only requires a temporary workspace if the number of partitions along the + // GEMM K dimension is greater than one. + workspace_bytes = sizeof(int) * size_t(grid_tiled_shape.m()) * size_t(grid_tiled_shape.n()); + } + + CUTLASS_TRACE_HOST(" workspace_bytes: " << workspace_bytes); + + workspace_bytes += GemmKernel::get_extra_workspace_size(args, grid_tiled_shape); + + return workspace_bytes; + } + + /// Computes the grid shape + static dim3 get_grid_shape(Arguments const& args) { + CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::get_grid_shape()"); + + ThreadblockSwizzle threadblock_swizzle; + + cutlass::gemm::GemmCoord grid_tiled_shape; + int gemm_k_size = 0; + + get_grid_shape_(grid_tiled_shape, gemm_k_size, args); + dim3 result = threadblock_swizzle.get_grid_shape(grid_tiled_shape); + + CUTLASS_TRACE_HOST( + " grid_tiled_shape: " << grid_tiled_shape << "\n" + << " result = {" << result << "}"); + + return result; + } + + /// Computes the maximum number of active blocks per multiprocessor + static int maximum_active_blocks(int smem_capacity = -1) { + CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::maximum_active_blocks()"); + + int max_active_blocks = -1; + int smem_size = int(sizeof(typename GemmKernel::SharedStorage)); + + CUTLASS_TRACE_HOST(" smem_size: " << smem_size << " bytes"); + + if (smem_size <= (48 << 10)) { + cudaError_t result = cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &max_active_blocks, Kernel, GemmKernel::kThreadCount, smem_size); + + if (result == cudaSuccess) { + CUTLASS_TRACE_HOST(" max_active_blocks: " << max_active_blocks); + return max_active_blocks; + } + } else { + // Query assuming zero shared memory then compute occupancy limit based on SMEM + cudaError_t result = cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &max_active_blocks, Kernel, GemmKernel::kThreadCount, 0); + + if (result != cudaSuccess) { + CUTLASS_TRACE_HOST( + " cudaOccupancyMaxActiveBlocksPerMultiprocessor() returned error " << cudaGetErrorString(result)); + + return -1; + } + + if (smem_capacity < 0) { + int device_idx = 0; + result = cudaGetDevice(&device_idx); + + if (result != cudaSuccess) { + return -1; + } + + cudaDeviceProp properties; + result = cudaGetDeviceProperties(&properties, device_idx); + + if (result != cudaSuccess) { + return -1; + } + + smem_capacity = static_cast(properties.sharedMemPerMultiprocessor); + } + + int occupancy = std::min(max_active_blocks, smem_capacity / smem_size); + + CUTLASS_TRACE_HOST(" occupancy: " << occupancy); + + return occupancy; + } + + CUTLASS_TRACE_HOST(" returning internal error"); + + return -1; + } + + /// Initializes GEMM state from arguments. + Status initialize(Arguments const& args, void* workspace = nullptr, cudaStream_t stream = nullptr) { + CUTLASS_TRACE_HOST( + "GemmUniversalBaseCompat::initialize() - workspace " << workspace + << ", stream: " << (stream ? "non-null" : "null")); + + size_t workspace_bytes = get_workspace_size(args); + + CUTLASS_TRACE_HOST(" workspace_bytes: " << workspace_bytes); + + if (workspace_bytes) { + if (!workspace) { + CUTLASS_TRACE_HOST(" error: device workspace must not be null"); + + return Status::kErrorWorkspaceNull; + } + + if (args.mode == GemmUniversalMode::kGemm) { + CUTLASS_TRACE_HOST(" clearing device workspace"); + cudaError_t result = cudaMemsetAsync(workspace, 0, workspace_bytes, stream); + + if (result != cudaSuccess) { + CUTLASS_TRACE_HOST(" cudaMemsetAsync() returned error " << cudaGetErrorString(result)); + + return Status::kErrorInternal; + } + } + } + + // Get CUDA grid shape + cutlass::gemm::GemmCoord grid_tiled_shape; + int gemm_k_size = 0; + + get_grid_shape_(grid_tiled_shape, gemm_k_size, args); + + // Initialize the Params structure + params_ = typename GemmKernel::Params(args, grid_tiled_shape, gemm_k_size, static_cast(workspace)); + + // Specify shared memory capacity for kernel. + int smem_size = int(sizeof(typename GemmKernel::SharedStorage)); + + if (smem_size >= (48 << 10)) { + cudaError_t result = + cudaFuncSetAttribute(Kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size); + + if (result != cudaSuccess) { + return Status::kErrorInternal; + } + } + + return Status::kSuccess; + } + + /// Lightweight update given a subset of arguments + Status update(Arguments const& args, void* workspace = nullptr) { + CUTLASS_TRACE_HOST("GemmUniversalBaseCompat()::update() - workspace: " << workspace); + + size_t workspace_bytes = get_workspace_size(args); + + if (workspace_bytes && !workspace) { + return Status::kErrorWorkspaceNull; + } + + params_.update(args, workspace); + + return Status::kSuccess; + } + + /// Runs the kernel using initialized state. + Status run(cudaStream_t stream = nullptr) { + CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::run()"); + + // + // Configure grid and block dimensions + // + + ThreadblockSwizzle threadblock_swizzle; + + dim3 grid = threadblock_swizzle.get_grid_shape(params_.grid_tiled_shape); + dim3 block(GemmKernel::kThreadCount, 1, 1); + + int smem_size = int(sizeof(typename GemmKernel::SharedStorage)); + + // + // Launch kernel + // + + CUTLASS_TRACE_HOST(" grid: (" << grid << "), block: (" << block << "), SMEM: " << smem_size << " bytes"); + + // Launch + cutlass::Kernel<<>>(params_); + + // + // Query for errors + // + cudaError_t result = cudaGetLastError(); + + if (result != cudaSuccess) { + CUTLASS_TRACE_HOST(" grid launch failed with error " << cudaGetErrorString(result)); + return Status::kErrorInternal; + } + + return Status::kSuccess; + } + + /// Runs the kernel using initialized state. + Status operator()(cudaStream_t stream = nullptr) { + return run(stream); + } + + /// Runs the kernel using initialized state. + Status operator()(Arguments const& args, void* workspace = nullptr, cudaStream_t stream = nullptr) { + Status status = initialize(args, workspace, stream); + + if (status == Status::kSuccess) { + status = run(stream); + } + + return status; + } +}; + +///////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace device +} // namespace gemm +} // namespace cutlass + +///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/gemm_with_epilogue_visitor.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/gemm_with_epilogue_visitor.h new file mode 100644 index 00000000..905d11ba --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/cutlass_extensions/gemm/gemm_with_epilogue_visitor.h @@ -0,0 +1,492 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Adapted from +// https://github.com/NVIDIA/TensorRT-LLM/blob/be1788106245496872d18e702978e59b6bfd50e0/cpp/tensorrt_llm/cutlass_extensions/include/cutlass_extensions/gemm/kernel/gemm_with_epilogue_visitor.h + +#pragma once + +#include +#include +#include +#include +#include + +///////////////////////////////////////////////////////////////////////////////////////////////// + +namespace cutlass { +namespace gemm { +namespace kernel { + +///////////////////////////////////////////////////////////////////////////////////////////////// + +template < + typename Mma_, ///! Threadblock-scoped matrix multiply-accumulate + typename Epilogue_, ///! Epilogue + typename ThreadblockSwizzle_ ///! Threadblock swizzling function + > +struct GemmWithEpilogueVisitor { + public: + using Mma = Mma_; + using Epilogue = Epilogue_; + using EpilogueVisitor = typename Epilogue::Visitor; + using ThreadblockSwizzle = ThreadblockSwizzle_; + + using ElementA = typename Mma::IteratorA::Element; + using LayoutA = typename Mma::IteratorA::Layout; + using TensorRefA = TensorRef; + + using ElementB = typename Mma::IteratorB::Element; + using LayoutB = typename Mma::IteratorB::Layout; + using TensorRefB = TensorRef; + + using ElementCompute = typename EpilogueVisitor::ElementCompute; + using LayoutAlphaCol = cutlass::layout::RowMajor; + using LayoutAlphaRow = cutlass::layout::ColumnMajor; + using TensorRefAlphaCol = TensorRef; + using TensorRefAlphaRow = TensorRef; + + using ElementC = typename EpilogueVisitor::ElementOutput; + using LayoutC = typename Epilogue::Layout; + using TensorRefC = TensorRef; + + static ComplexTransform const kTransformA = Mma::kTransformA; + static ComplexTransform const kTransformB = Mma::kTransformB; + using Operator = typename Mma::Operator; + + using OperatorClass = typename Mma::Operator::OperatorClass; + using ThreadblockShape = typename Mma::Shape; + using WarpShape = typename Mma::Operator::Shape; + using InstructionShape = typename Mma::Policy::Operator::InstructionShape; + using ArchTag = typename Mma::ArchTag; + using EpilogueOutputOp = + typename Epilogue::Visitor::ElementwiseFunctor; // Define type so GemmUniversalBase doesn't complain + + static int const kStages = Mma::kStages; + static int const kAlignmentA = Mma::IteratorA::AccessType::kElements; + static int const kAlignmentB = Mma::IteratorB::AccessType::kElements; + static int const kAlignmentC = EpilogueVisitor::kElementsPerAccess; + + /// Warp count (concept: GemmShape) + using WarpCount = typename Mma::WarpCount; + static int const kThreadCount = 32 * WarpCount::kCount; + + /// Split-K preserves splits that are 128b aligned + static int const kSplitKAlignment = const_max(128 / sizeof_bits::value, 128 / sizeof_bits::value); + + // + // Structures + // + + /// Argument structure + struct Arguments { + // + // Data members + // + + GemmUniversalMode mode; + GemmCoord problem_size; + int batch_count; + + TensorRefA ref_A; + TensorRefB ref_B; + TensorRefAlphaCol ref_alpha_col; + TensorRefAlphaRow ref_alpha_row; + TensorRefC ref_C; + TensorRefC ref_D; + + int64_t batch_stride_A; + int64_t batch_stride_B; + int64_t batch_stride_D; + + typename EpilogueVisitor::Arguments epilogue_visitor; + + // + // Methods + // + + Arguments() : mode(GemmUniversalMode::kGemm), batch_count(1) {} + + /// constructs an arguments structure + Arguments( + GemmCoord problem_size_, + TensorRefA ref_A_, + TensorRefB ref_B_, + TensorRefAlphaCol ref_alpha_col_, + TensorRefAlphaRow ref_alpha_row_, + TensorRefC ref_C_, + TensorRefC ref_D_, + typename EpilogueVisitor::Arguments epilogue_visitor_) + : mode(GemmUniversalMode::kGemm), + problem_size(problem_size_), + batch_count(1), + ref_A(ref_A_), + ref_B(ref_B_), + ref_alpha_col(ref_alpha_col_), + ref_alpha_row(ref_alpha_row_), + ref_C(ref_C_), + ref_D(ref_D_), + batch_stride_A(0), + batch_stride_B(0), + batch_stride_D(0), + epilogue_visitor(epilogue_visitor_) {} + }; + + // + // Structure for precomputing values in host memory and passing to kernels + // + + /// Parameters structure + struct Params { + cutlass::gemm::GemmCoord problem_size; + cutlass::gemm::GemmCoord grid_tiled_shape; + int swizzle_log_tile; + + typename Mma::IteratorA::Params params_A; + typename Mma::IteratorB::Params params_B; + typename EpilogueVisitor::ScaleTileIterator::Params params_alpha_col; + typename EpilogueVisitor::ScaleTileIterator::Params params_alpha_row; + typename EpilogueVisitor::OutputTileIterator::Params params_C; + typename EpilogueVisitor::OutputTileIterator::Params params_D; + + GemmUniversalMode mode; + int batch_count; + int gemm_k_size; + + void* ptr_A; + void* ptr_B; + typename EpilogueVisitor::ScaleTileIterator::Element* ptr_alpha_col; + typename EpilogueVisitor::ScaleTileIterator::Element* ptr_alpha_row; + ElementC* ptr_C; + ElementC* ptr_D; + + int64_t batch_stride_A; + int64_t batch_stride_B; + + typename EpilogueVisitor::Params epilogue_visitor; + + // + // Methods + // + + CUTLASS_HOST_DEVICE + Params() + : swizzle_log_tile(0), + params_A(0), + params_B(0), + params_alpha_col(0), + params_C(0), + params_D(0), + batch_count(0), + gemm_k_size(0), + mode(cutlass::gemm::GemmUniversalMode::kGemm), + ptr_A(nullptr), + ptr_B(nullptr), + ptr_alpha_col(nullptr), + ptr_alpha_row(nullptr), + ptr_C(nullptr), + ptr_D(nullptr), + batch_stride_A(0), + batch_stride_B(0) {} + + Params(Arguments const& args, cutlass::gemm::GemmCoord const& grid_tiled_shape_, int gemm_k_size_, int* workspace_) + : problem_size(args.problem_size), + swizzle_log_tile(0), + params_A(args.ref_A.layout()), + params_B(args.ref_B.layout()), + params_alpha_col(args.ref_alpha_col.layout()), + params_alpha_row(args.ref_alpha_col.layout()), + params_C(args.ref_C.layout()), + params_D(args.ref_D.layout()), + mode(args.mode), + batch_count(args.batch_count), + gemm_k_size(args.problem_size.k()), + ptr_A(args.ref_A.data()), + ptr_B(args.ref_B.data()), + ptr_alpha_col(args.ref_alpha_col.data()), + ptr_alpha_row(args.ref_alpha_row.data()), + ptr_C(args.ref_C.data()), + ptr_D(args.ref_D.data()), + batch_stride_A(args.batch_stride_A), + batch_stride_B(args.batch_stride_B), + epilogue_visitor(args.epilogue_visitor) { + ThreadblockSwizzle threadblock_swizzle; + + grid_tiled_shape = threadblock_swizzle.get_tiled_shape( + args.problem_size, {ThreadblockShape::kM, ThreadblockShape::kN, ThreadblockShape::kK}, args.batch_count); + + if (args.mode == GemmUniversalMode::kGemm || args.mode == GemmUniversalMode::kGemmSplitKParallel) { + int const kAlignK = + const_max(const_max(128 / sizeof_bits::value, 128 / sizeof_bits::value), 1); + + gemm_k_size = round_up(ceil_div(args.problem_size.k(), args.batch_count), kAlignK); + + if (gemm_k_size) { + grid_tiled_shape.k() = ceil_div(args.problem_size.k(), gemm_k_size); + } + } + + swizzle_log_tile = threadblock_swizzle.get_log_tile(grid_tiled_shape); + } + }; + + /// Shared memory storage structure + union SharedStorage { + typename Mma::SharedStorage main_loop; + + struct { + typename Epilogue::SharedStorage epilogue; + typename EpilogueVisitor::SharedStorage visitor; + } epilogue; + }; + + public: + // + // Methods + // + + CUTLASS_DEVICE + GemmWithEpilogueVisitor() {} + + /// Determines whether kernel satisfies alignment + static Status can_implement(cutlass::gemm::GemmCoord const& problem_size) { + CUTLASS_TRACE_HOST("GemmWithEpilogueVisitor::can_implement()"); + + static int const kAlignmentA = Mma::IteratorA::AccessType::kElements; + static int const kAlignmentB = Mma::IteratorB::AccessType::kElements; + static int const kAlignmentC = EpilogueVisitor::OutputTileIterator::kElementsPerAccess; + + bool isAMisaligned = false; + bool isBMisaligned = false; + bool isCMisaligned = false; + + if (platform::is_same::value) { + isAMisaligned = problem_size.k() % kAlignmentA; + } else if (platform::is_same::value) { + isAMisaligned = problem_size.m() % kAlignmentA; + } else if ( + platform::is_same>::value || + platform::is_same>::value) { + isAMisaligned = problem_size.k() % kAlignmentA; + } + + if (platform::is_same::value) { + isBMisaligned = problem_size.n() % kAlignmentB; + } else if (platform::is_same::value) { + isBMisaligned = problem_size.k() % kAlignmentB; + } else if ( + platform::is_same>::value || + platform::is_same>::value) { + isBMisaligned = problem_size.k() % kAlignmentB; + } + + if (platform::is_same::value) { + isCMisaligned = problem_size.n() % kAlignmentC; + } else if (platform::is_same::value) { + isCMisaligned = problem_size.m() % kAlignmentC; + } else if ( + platform::is_same>::value || + platform::is_same>::value) { + isCMisaligned = problem_size.n() % kAlignmentC; + } + + if (isAMisaligned) { + CUTLASS_TRACE_HOST(" returning kErrorMisalignedOperand for A operand"); + return Status::kErrorMisalignedOperand; + } + + if (isBMisaligned) { + CUTLASS_TRACE_HOST(" returning kErrorMisalignedOperand for B operand"); + return Status::kErrorMisalignedOperand; + } + + if (isCMisaligned) { + CUTLASS_TRACE_HOST(" returning kErrorMisalignedOperand for C operand"); + return Status::kErrorMisalignedOperand; + } + + CUTLASS_TRACE_HOST(" returning kSuccess"); + + return Status::kSuccess; + } + + static Status can_implement(Arguments const& args) { + return can_implement(args.problem_size); + } + + static size_t get_extra_workspace_size(Arguments const& args, cutlass::gemm::GemmCoord const& grid_tiled_shape) { + return 0; + } + +#define SPLIT_K_ENABLED 1 + + /// Executes one GEMM + CUTLASS_DEVICE + void run_kernel_(Params const& params, SharedStorage& shared_storage) { + // Compute threadblock location + ThreadblockSwizzle threadblock_swizzle; + + cutlass::gemm::GemmCoord threadblock_tile_offset = threadblock_swizzle.get_tile_offset(params.swizzle_log_tile); + + // Early exit if CTA is out of range + if (params.grid_tiled_shape.m() <= threadblock_tile_offset.m() || + params.grid_tiled_shape.n() <= threadblock_tile_offset.n()) { + return; + } + + int offset_k = 0; + int problem_size_k = params.problem_size.k(); + + ElementA* ptr_A = static_cast(params.ptr_A); + ElementB* ptr_B = static_cast(params.ptr_B); + +#if SPLIT_K_ENABLED + // + // Fetch pointers based on mode. + // + if (params.mode == GemmUniversalMode::kGemm || params.mode == GemmUniversalMode::kGemmSplitKParallel) { + if (threadblock_tile_offset.k() + 1 < params.grid_tiled_shape.k()) { + problem_size_k = (threadblock_tile_offset.k() + 1) * params.gemm_k_size; + } + + offset_k = threadblock_tile_offset.k() * params.gemm_k_size; + } else if (params.mode == GemmUniversalMode::kBatched) { + ptr_A += threadblock_tile_offset.k() * params.batch_stride_A; + ptr_B += threadblock_tile_offset.k() * params.batch_stride_B; + } else if (params.mode == GemmUniversalMode::kArray) { + ptr_A = static_cast(params.ptr_A)[threadblock_tile_offset.k()]; + ptr_B = static_cast(params.ptr_B)[threadblock_tile_offset.k()]; + } +#endif + + // Compute initial location in logical coordinates + cutlass::MatrixCoord tb_offset_A{ + threadblock_tile_offset.m() * Mma::Shape::kM, + offset_k, + }; + + cutlass::MatrixCoord tb_offset_B{offset_k, threadblock_tile_offset.n() * Mma::Shape::kN}; + + // Compute position within threadblock + int thread_idx = threadIdx.x; + + // Construct iterators to A and B operands + typename Mma::IteratorA iterator_A( + params.params_A, ptr_A, {params.problem_size.m(), problem_size_k}, thread_idx, tb_offset_A); + + typename Mma::IteratorB iterator_B( + params.params_B, ptr_B, {problem_size_k, params.problem_size.n()}, thread_idx, tb_offset_B); + + // Broadcast the warp_id computed by lane 0 to ensure dependent code + // is compiled as warp-uniform. + int warp_idx = __shfl_sync(0xffffffff, threadIdx.x / 32, 0); + + int lane_idx = threadIdx.x % 32; + + // + // Main loop + // + + // Construct thread-scoped matrix multiply + Mma mma(shared_storage.main_loop, thread_idx, warp_idx, lane_idx); + + typename Mma::FragmentC accumulators; + + accumulators.clear(); + + // Compute threadblock-scoped matrix multiply-add + int gemm_k_iterations = (problem_size_k - offset_k + Mma::Shape::kK - 1) / Mma::Shape::kK; + + // Compute threadblock-scoped matrix multiply-add + mma(gemm_k_iterations, accumulators, iterator_A, iterator_B, accumulators); + + // + // Masked tile iterators constructed from members + // + + threadblock_tile_offset = threadblock_swizzle.get_tile_offset(params.swizzle_log_tile); + + // assume identity swizzle + MatrixCoord threadblock_offset( + threadblock_tile_offset.m() * Mma::Shape::kM, threadblock_tile_offset.n() * Mma::Shape::kN); + + int block_idx = threadblock_tile_offset.m() + threadblock_tile_offset.n() * params.grid_tiled_shape.m(); + + // + // Construct the epilogue visitor + // + + bool with_bias = true; + if (params.ptr_C == nullptr) { + with_bias = false; + } + + EpilogueVisitor epilogue_visitor( + params.epilogue_visitor, + shared_storage.epilogue.visitor, + params.problem_size.mn(), + thread_idx, + warp_idx, + lane_idx, + params.params_alpha_col, + params.params_C, + params.params_D, + with_bias, + true, + true, + params.ptr_alpha_row, + params.ptr_alpha_col, + params.ptr_C, + params.ptr_D, + threadblock_offset, + blockIdx.y * params.problem_size.m()); + + if (params.mode == GemmUniversalMode::kGemm) { + // Indicate which position in a serial reduction the output operator is currently updating + epilogue_visitor.set_k_partition(threadblock_tile_offset.k(), params.grid_tiled_shape.k()); + } else if (params.mode == GemmUniversalMode::kBatched || params.mode == GemmUniversalMode::kArray) { + epilogue_visitor.set_batch_index(threadblock_tile_offset.k()); + } + + // Construct the epilogue + Epilogue epilogue(shared_storage.epilogue.epilogue, thread_idx, warp_idx, lane_idx); + + // Execute the epilogue operator to update the destination tensor. + epilogue(epilogue_visitor, accumulators); + } + + template + CUTLASS_DEVICE void run_kernel(Params const& params, SharedStorage& shared_storage) { + if constexpr (platform::is_same::value) { + run_kernel_(params, shared_storage); + } else { + CUTLASS_NOT_IMPLEMENTED(); + } + } + + /// Executes one GEMM + CUTLASS_DEVICE + void operator()(Params const& params, SharedStorage& shared_storage) { + run_kernel(params, shared_storage); + } +}; + +///////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace kernel +} // namespace gemm +} // namespace cutlass + +///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/activation.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/activation.cu new file mode 100644 index 00000000..43617f87 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/activation.cu @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2024 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#ifndef USE_ROCM + +#include + +#include "utils.h" + +#else +#include "hip/hip_act_and_mul.cuh" +#endif + +// Adapted from flashinfer activation +// https://github.com/flashinfer-ai/flashinfer/blob/4e8eb1879f9c3ba6d75511e5893183bf8f289a62/csrc/activation.cu#L44 + +namespace detail { + +template +__device__ __forceinline__ float to_f32(const T& x) { +#if USE_ROCM + return castToFloat(x); +#else + return static_cast(x); +#endif +} + +template +__device__ __forceinline__ T from_f32(float f32) { +#if USE_ROCM + return castFromFloat(f32); +#else + return static_cast(f32); +#endif +} + +} // namespace detail + +template +__device__ __forceinline__ T silu(const T& x) { + float f32_val = detail::to_f32(x); + return detail::from_f32(f32_val / (1.0f + expf(-f32_val))); +} + +template +__device__ __forceinline__ T gelu(const T& x) { + constexpr float kAlpha = M_SQRT1_2; + float f32_val = detail::to_f32(x); + return detail::from_f32(f32_val * (0.5f * (1.0f + erf(f32_val * kAlpha)))); +} + +// gelu_quick(x) = x * torch.sigmoid(1.702 * x) +template +__device__ __forceinline__ T gelu_quick_act(const T& x) { + float f32_val = detail::to_f32(x); + return detail::from_f32(f32_val / (1.0f + expf(-f32_val * 1.702f))); +} + +template +__device__ __forceinline__ T gelu_tanh(const T& x) { + constexpr float kAlpha = 0.044715f; + constexpr float kBeta = 0.7978845608028654f; + float f32_val = detail::to_f32(x); + const float cdf = 0.5f * (1.0f + tanhf((kBeta * (f32_val + kAlpha * f32_val * f32_val * f32_val)))); + return detail::from_f32(f32_val * cdf); +} + +void silu_and_mul(at::Tensor& out, at::Tensor& input) { + int d = input.size(-1) / 2; + int64_t num_tokens = input.numel() / input.size(-1); + dim3 grid(num_tokens); + + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), c_type, [&] { + uint32_t vec_size = 16 / sizeof(c_type); + dim3 block(std::min(d / vec_size, 1024U)); +#if USE_ROCM + sgl_hip::activation::act_and_mul_kernel + <<>>(static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); +#else + flashinfer::activation::act_and_mul_kernel + <<>>(static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); +#endif + return true; + }); +} + +void gelu_tanh_and_mul(at::Tensor& out, at::Tensor& input) { + int d = input.size(-1) / 2; + int64_t num_tokens = input.numel() / input.size(-1); + dim3 grid(num_tokens); + + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), c_type, [&] { + uint32_t vec_size = 16 / sizeof(c_type); + dim3 block(std::min(d / vec_size, 1024U)); +#if USE_ROCM + sgl_hip::activation::act_and_mul_kernel + <<>>(static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); +#else + flashinfer::activation::act_and_mul_kernel + <<>>(static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); +#endif + return true; + }); +} + +void gelu_and_mul(at::Tensor& out, at::Tensor& input) { + int d = input.size(-1) / 2; + int64_t num_tokens = input.numel() / input.size(-1); + dim3 grid(num_tokens); + + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), c_type, [&] { + uint32_t vec_size = 16 / sizeof(c_type); + dim3 block(std::min(d / vec_size, 1024U)); +#if USE_ROCM + sgl_hip::activation::act_and_mul_kernel + <<>>(static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); +#else + flashinfer::activation::act_and_mul_kernel + <<>>(static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); +#endif + + return true; + }); +} + +#if USE_ROCM +void gelu_quick(at::Tensor& out, const at::Tensor& input) { + int d = input.size(-1); + int64_t num_tokens = input.numel() / input.size(-1); + dim3 grid(num_tokens); + + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), c_type, [&] { + uint32_t vec_size = 16 / sizeof(c_type); + dim3 block(std::min(d / vec_size, 1024U)); + sgl_hip::activation::act_only_kernel + <<>>(static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); + + return true; + }); +} +#endif diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/cast.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/cast.cu new file mode 100644 index 00000000..3ce8135d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/cast.cu @@ -0,0 +1,170 @@ +#include "pytorch_extension_utils.h" + +template +struct ConvertToFP8 { + static __device__ __nv_fp8_storage_t convert_to_fp8(T value) { + return 0; + } +}; + +template <> +struct ConvertToFP8<__nv_bfloat16> { + static __device__ __nv_fp8_storage_t convert_to_fp8(__nv_bfloat16 value) { + return __nv_cvt_bfloat16raw_to_fp8(value, __NV_SATFINITE, __NV_E4M3); + } +}; + +template <> +struct ConvertToFP8 { + static __device__ __nv_fp8_storage_t convert_to_fp8(half value) { + return __nv_cvt_halfraw_to_fp8(value, __NV_SATFINITE, __NV_E4M3); + } +}; + +template +struct ConvertFromFloat { + static __device__ T convert_from_float(float value) { + return 0; + } +}; + +template <> +struct ConvertFromFloat<__nv_bfloat16> { + static __device__ __nv_bfloat16 convert_from_float(float value) { + return __float2bfloat16(value); + } +}; + +template <> +struct ConvertFromFloat { + static __device__ half convert_from_float(float value) { + return __float2half(value); + } +}; + +template +__global__ void fused_downcast_kernel( + const T* cache_k, + const T* cache_v, + const float* k_scale, + const float* v_scale, + __nv_fp8_storage_t* output_k, + __nv_fp8_storage_t* output_v, + const int input_sl, + const int head, + const int dim, + const T max_fp8, + const T min_fp8, + const int64_t mult, + const int64_t offset, + const int64_t* loc) { + // TODO: change name + int token_idx = blockIdx.x; + int thread_idx = threadIdx.x; + int total_threads = blockDim.x; + + T k_scale_val = ConvertFromFloat::convert_from_float(k_scale[0]); + T v_scale_val = ConvertFromFloat::convert_from_float(v_scale[0]); + + T k_scale_inv = static_cast(1.f) / k_scale_val; + T v_scale_inv = static_cast(1.f) / v_scale_val; + + auto clamp = [&](T val) { return val > max_fp8 ? max_fp8 : (min_fp8 > val ? min_fp8 : val); }; + + if (token_idx < input_sl) { + int out_seq_idx = loc[token_idx]; + +#pragma unroll + for (int i = thread_idx; i < head * dim; i += total_threads) { + int in_idx = token_idx * head * dim + i; + int out_idx = (out_seq_idx * mult + offset) * head * dim + i; + + T k_val = cache_k[in_idx] * k_scale_inv; + k_val = clamp(k_val); + output_k[out_idx] = ConvertToFP8::convert_to_fp8(k_val); + + T v_val = cache_v[in_idx] * v_scale_inv; + v_val = clamp(v_val); + output_v[out_idx] = ConvertToFP8::convert_to_fp8(v_val); + } + } +} + +template +void downcast_fp8_impl( + at::Tensor& k, + at::Tensor& v, + at::Tensor& k_out, + at::Tensor& v_out, + at::Tensor& k_scale, + at::Tensor& v_scale, + at::Tensor& loc, + int64_t mult, + int64_t offset, + cudaStream_t stream) { + CHECK_INPUT(k); + CHECK_INPUT(v); + CHECK_INPUT(k_out); + CHECK_INPUT(v_out); + CHECK_INPUT(k_scale); + CHECK_INPUT(v_scale); + CHECK_INPUT(loc); + + int64_t input_sl = k.size(0); + int64_t head = k.size(1); + int64_t dim = k.size(2); + + dim3 grid(input_sl * head); + int vec_size = 8; + dim3 block(std::min(int(dim) / vec_size, 1024)); + + const T max_fp8 = static_cast(448.0f); + const T min_fp8 = static_cast(-448.0f); + + fused_downcast_kernel<<>>( + static_cast(k.data_ptr()), + static_cast(v.data_ptr()), + static_cast(k_scale.data_ptr()), + static_cast(v_scale.data_ptr()), + static_cast<__nv_fp8_storage_t*>(k_out.data_ptr()), + static_cast<__nv_fp8_storage_t*>(v_out.data_ptr()), + input_sl, + head, + dim, + max_fp8, + min_fp8, + mult, + offset, + static_cast(loc.data_ptr())); + + cudaError_t status = cudaGetLastError(); + TORCH_CHECK(status == cudaSuccess, "Kernel launch failed: " + std::string(cudaGetErrorString(status))); +} + +void downcast_fp8( + at::Tensor& k, + at::Tensor& v, + at::Tensor& k_out, + at::Tensor& v_out, + at::Tensor& k_scale, + at::Tensor& v_scale, + at::Tensor& loc, + int64_t mult, + int64_t offset) { + CHECK_INPUT(k); + CHECK_INPUT(v); + CHECK_INPUT(k_out); + CHECK_INPUT(v_out); + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + switch (k.scalar_type()) { + case at::ScalarType::BFloat16: + downcast_fp8_impl<__nv_bfloat16>(k, v, k_out, v_out, k_scale, v_scale, loc, mult, offset, stream); + break; + case at::ScalarType::Half: + downcast_fp8_impl<__half>(k, v, k_out, v_out, k_scale, v_scale, loc, mult, offset, stream); + break; + default: + TORCH_CHECK(false, "Unsupported input type for downcast_fp8. Expected bfloat16 or float16."); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/concat_mla.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/concat_mla.cu new file mode 100644 index 00000000..7d5b8595 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/concat_mla.cu @@ -0,0 +1,217 @@ +#include +#include +#include + +#include "pytorch_extension_utils.h" +#include "utils.cuh" + +constexpr int NUM_LOCAL_HEADS = 128; +constexpr int QK_NOPE_HEAD_DIM = 128; +constexpr int QK_ROPE_HEAD_DIM = 64; +constexpr int K_HEAD_DIM = QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM; + +constexpr int HEAD_CHUNK_SIZE = 16; +constexpr int NUM_HEAD_CHUNKS = NUM_LOCAL_HEADS / HEAD_CHUNK_SIZE; + +__global__ void concat_mla_k_kernel( + nv_bfloat16* __restrict__ k, + const nv_bfloat16* __restrict__ k_nope, + const nv_bfloat16* __restrict__ k_rope, + const int num_tokens, + const int64_t k_stride_0, + const int k_stride_1, + const int64_t k_nope_stride_0, + const int k_nope_stride_1, + const int64_t k_rope_stride_0) { + const int flat_warp_id = (blockIdx.x * blockDim.x + threadIdx.x) / 32; + const int token_id = flat_warp_id / NUM_HEAD_CHUNKS; + const int head_chunk_id = flat_warp_id % NUM_HEAD_CHUNKS; + const int lane_id = get_lane_id(); + if (token_id >= num_tokens) return; + + using NopeVec = int2; // 8B/thread，32 thread = 256B/row + using RopeVec = int; // 4B/thread，32 thread = 128B/row + static_assert(sizeof(NopeVec) * 32 == QK_NOPE_HEAD_DIM * sizeof(nv_bfloat16), "nope vec mismatch"); + static_assert(sizeof(RopeVec) * 32 == QK_ROPE_HEAD_DIM * sizeof(nv_bfloat16), "rope vec mismatch"); + + const int head_row0 = head_chunk_id * HEAD_CHUNK_SIZE; + + const int2* __restrict__ nope_src = + reinterpret_cast(k_nope + token_id * k_nope_stride_0 + head_row0 * k_nope_stride_1) + lane_id; + + int2* __restrict__ nope_dst = reinterpret_cast(k + token_id * k_stride_0 + head_row0 * k_stride_1) + lane_id; + + int* __restrict__ rope_dst = + reinterpret_cast(k + token_id * k_stride_0 + head_row0 * k_stride_1 + QK_NOPE_HEAD_DIM) + lane_id; + + const int nope_src_stride_v = (k_nope_stride_1 >> 2); // int2 covers 4 bf16 + const int nope_dst_stride_v = (k_stride_1 >> 2); + const int rope_dst_stride_v = (k_stride_1 >> 1); // int covers 2 bf16 + + const int* rope_base = reinterpret_cast(k_rope + token_id * k_rope_stride_0); + const RopeVec rope_val = ld_na_global_v1(rope_base + lane_id); + + prefetch_L2(nope_src); + NopeVec cur = ld_na_global_v2(nope_src); + +#pragma unroll + for (int i = 0; i < HEAD_CHUNK_SIZE; ++i) { + NopeVec next; + if (i + 1 < HEAD_CHUNK_SIZE) { + const int2* next_src = nope_src + nope_src_stride_v; + prefetch_L2(next_src); + next = ld_na_global_v2(next_src); + } + + st_na_global_v2(nope_dst, cur); + st_na_global_v1(rope_dst, rope_val); + + nope_src += nope_src_stride_v; + nope_dst += nope_dst_stride_v; + rope_dst += rope_dst_stride_v; + + cur = next; + } +} + +inline void check_tensor(const at::Tensor& t, int64_t shape0, int64_t shape1, int64_t shape2, c10::ScalarType dtype) { + TORCH_CHECK_EQ(t.dim(), 3); + TORCH_CHECK_EQ(t.size(0), shape0); + TORCH_CHECK_EQ(t.size(1), shape1); + TORCH_CHECK_EQ(t.size(2), shape2); + TORCH_CHECK_EQ(t.dtype(), dtype); + TORCH_CHECK(t.device().is_cuda()); + TORCH_CHECK_EQ(((int64_t)t.data_ptr()) % 16, 0); // alignment +} + +void concat_mla_k(at::Tensor k, at::Tensor k_nope, at::Tensor k_rope) { + const int num_tokens = k.size(0); + + check_tensor(k, num_tokens, NUM_LOCAL_HEADS, K_HEAD_DIM, at::kBFloat16); + check_tensor(k_nope, num_tokens, NUM_LOCAL_HEADS, QK_NOPE_HEAD_DIM, at::kBFloat16); + check_tensor(k_rope, num_tokens, 1, QK_ROPE_HEAD_DIM, at::kBFloat16); + TORCH_CHECK_EQ(k.stride(2), 1); + TORCH_CHECK_EQ(k_nope.stride(2), 1); + TORCH_CHECK_EQ(k_rope.stride(2), 1); + + const auto stream = at::cuda::getCurrentCUDAStream().stream(); + + constexpr int num_warps_per_block = 32; + const int grid_size = ceil_div(num_tokens * NUM_HEAD_CHUNKS, num_warps_per_block); + const int block_size = num_warps_per_block * 32; + + concat_mla_k_kernel<<>>( + reinterpret_cast(k.data_ptr()), + reinterpret_cast(k_nope.data_ptr()), + reinterpret_cast(k_rope.data_ptr()), + num_tokens, + k.stride(0), + k.stride(1), + k_nope.stride(0), + k_nope.stride(1), + k_rope.stride(0)); + cudaError_t err = cudaGetLastError(); + TORCH_CHECK(err == cudaSuccess, "CUDA kernel launch failed: ", cudaGetErrorString(err)); +} + +// ============================== concat_mla_absorb_q ============================== + +// TODO give a name prefix, also maybe refactor code above +constexpr int A_LAST_DIM = 512; +constexpr int B_LAST_DIM = 64; + +__global__ void concat_mla_absorb_q_kernel( + nv_bfloat16* a, + nv_bfloat16* b, + nv_bfloat16* out, + const int num_items, + const int dim_1, + const int64_t a_stride_0, + const int a_stride_1, + const int64_t b_stride_0, + const int b_stride_1, + const int64_t out_stride_0, + const int out_stride_1) { + const int flat_warp_id = (blockIdx.x * blockDim.x + threadIdx.x) / 32; + const int lane_id = get_lane_id(); + + const int idx_0 = flat_warp_id / dim_1; + const int idx_1 = flat_warp_id % dim_1; + + if (flat_warp_id >= num_items) { + return; + } + + using ABufType = int4; + constexpr int A_NUM_UNROLL = 2; + static_assert(sizeof(ABufType) * A_NUM_UNROLL == A_LAST_DIM * sizeof(a[0]) / 32); + ABufType a_buf[A_NUM_UNROLL]; + + using BBufType = int; + constexpr int B_NUM_UNROLL = 1; + static_assert(sizeof(BBufType) * B_NUM_UNROLL == B_LAST_DIM * sizeof(b[0]) / 32); + BBufType b_buf; + + { + const BBufType* base_addr = reinterpret_cast(b + idx_0 * b_stride_0 + idx_1 * b_stride_1); + b_buf = *(base_addr + lane_id); + } + +#pragma unroll + for (int i = 0; i < A_NUM_UNROLL; ++i) { + const ABufType* base_addr = reinterpret_cast(a + idx_0 * a_stride_0 + idx_1 * a_stride_1); + a_buf[i] = *(base_addr + i * 32 + lane_id); + } + + { + BBufType* base_addr = reinterpret_cast(out + idx_0 * out_stride_0 + idx_1 * out_stride_1 + A_LAST_DIM); + *(base_addr + lane_id) = b_buf; + } + +#pragma unroll + for (int i = 0; i < A_NUM_UNROLL; ++i) { + ABufType* base_addr = reinterpret_cast(out + idx_0 * out_stride_0 + idx_1 * out_stride_1); + *(base_addr + i * 32 + lane_id) = a_buf[i]; + } +} + +inline void check_tensor_concat_mla_absorb_q(const at::Tensor& t, int64_t shape2) { + TORCH_CHECK_EQ(t.dim(), 3); + TORCH_CHECK_EQ(t.size(2), shape2); + TORCH_CHECK_EQ(t.stride(2), 1); + TORCH_CHECK_EQ(t.dtype(), at::kBFloat16); + TORCH_CHECK(t.device().is_cuda()); + TORCH_CHECK_EQ(((int64_t)t.data_ptr()) % 16, 0); // alignment +} + +// TODO further optimize it later +void concat_mla_absorb_q(at::Tensor a, at::Tensor b, at::Tensor out) { + check_tensor_concat_mla_absorb_q(a, A_LAST_DIM); + check_tensor_concat_mla_absorb_q(b, B_LAST_DIM); + check_tensor_concat_mla_absorb_q(out, A_LAST_DIM + B_LAST_DIM); + + const auto stream = at::cuda::getCurrentCUDAStream().stream(); + + TORCH_CHECK_EQ(a.size(0) * a.size(1), b.size(0) * b.size(1)); + TORCH_CHECK_EQ(a.size(1), b.size(1)); + const int num_items = a.size(0) * a.size(1); + + constexpr int num_warps_per_block = 32; + const int grid_size = ceil_div(num_items, num_warps_per_block); + const int block_size = num_warps_per_block * 32; + + concat_mla_absorb_q_kernel<<>>( + reinterpret_cast(a.data_ptr()), + reinterpret_cast(b.data_ptr()), + reinterpret_cast(out.data_ptr()), + num_items, + a.size(1), + a.stride(0), + a.stride(1), + b.stride(0), + b.stride(1), + out.stride(0), + out.stride(1)); + cudaError_t err = cudaGetLastError(); + TORCH_CHECK(err == cudaSuccess, "CUDA kernel launch failed: ", cudaGetErrorString(err)); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/copy.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/copy.cu new file mode 100644 index 00000000..09719f51 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/copy.cu @@ -0,0 +1,58 @@ +#include +#include +#include + +#include + +template +struct InputArray { + int values[N]; +}; + +template +__global__ void copy_to_gpu_no_ce_kernel(const InputArray input_array, int* output) { + int idx = threadIdx.x + blockIdx.x * blockDim.x; + if (idx < N) { + output[idx] = input_array.values[idx]; + } +} + +template +void copy_to_gpu_no_ce_impl(const at::Tensor& input, at::Tensor& output) { + TORCH_CHECK(input.dim() == 1, "input must be 1-D"); + TORCH_CHECK(static_cast(input.numel()) == N, "input numel must equal template N"); + TORCH_CHECK(input.is_contiguous(), "input must be contiguous"); + TORCH_CHECK(input.dtype() == torch::kInt32, "input dtype must be int32"); + + TORCH_CHECK(output.dim() == 1, "output dim"); + TORCH_CHECK(static_cast(output.numel()) == N, "output size"); + TORCH_CHECK(output.is_contiguous(), "output contiguous"); + TORCH_CHECK(output.dtype() == torch::kInt32, "output dtype"); + + TORCH_CHECK(input.device().is_cpu(), "input must be a CPU tensor"); + TORCH_CHECK(output.device().is_cuda(), "output must be a CUDA tensor"); + + InputArray input_array; + const int* input_ptr = input.data_ptr(); + for (int i = 0; i < N; ++i) + input_array.values[i] = input_ptr[i]; + + // may use multi thread blocks if performance bottleneck + dim3 grid(1); + dim3 block(static_cast(input.numel())); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + copy_to_gpu_no_ce_kernel<<>>(input_array, output.data_ptr()); + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} + +void copy_to_gpu_no_ce(const at::Tensor& input, at::Tensor& output) { + int N = static_cast(input.numel()); + // Can use macro if there are more N needed + if (N == 72) { + copy_to_gpu_no_ce_impl<72>(input, output); + } else if (N == 64) { + copy_to_gpu_no_ce_impl<64>(input, output); + } else { + TORCH_CHECK(false, "unexpected N"); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/fused_add_rms_norm_kernel.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/fused_add_rms_norm_kernel.cu new file mode 100644 index 00000000..02d5b20c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/fused_add_rms_norm_kernel.cu @@ -0,0 +1,59 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include + +#include "utils.h" + +using namespace flashinfer; + +void sgl_fused_add_rmsnorm( + torch::Tensor input, torch::Tensor residual, torch::Tensor weight, double eps, bool enable_pdl) { + CHECK_INPUT(input); + CHECK_INPUT(residual); + CHECK_INPUT(weight); + auto device = input.device(); + CHECK_EQ(residual.device(), device); + CHECK_EQ(weight.device(), device); + CHECK_DIM(2, input); // input: (batch_size, hidden_size) + CHECK_DIM(2, residual); // residual: (batch_size, hidden_size) + CHECK_DIM(1, weight); // weight: (hidden_size) + CHECK_EQ(input.size(0), residual.size(0)); + CHECK_EQ(input.size(1), residual.size(1)); + CHECK_EQ(input.size(1), weight.size(0)); + unsigned int batch_size = input.size(0); + unsigned int hidden_size = input.size(1); + + cudaStream_t torch_current_stream = at::cuda::getCurrentCUDAStream(); + // support float16, bfloat16 and float32 + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), c_type, [&] { + cudaError_t status = norm::FusedAddRMSNorm( + static_cast(input.data_ptr()), + static_cast(residual.data_ptr()), + static_cast(weight.data_ptr()), + batch_size, + hidden_size, + input.stride(0), + residual.stride(0), + eps, + enable_pdl, + torch_current_stream); + TORCH_CHECK( + status == cudaSuccess, "FusedAddRMSNorm failed with error code " + std::string(cudaGetErrorString(status))); + return true; + }); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/pos_enc.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/pos_enc.cuh new file mode 100644 index 00000000..a2e4e2eb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/pos_enc.cuh @@ -0,0 +1,467 @@ +/* + * Copyright (c) 2023 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SGL_POS_ENC_CUH_ +#define SGL_POS_ENC_CUH_ + +#include // upstream + +namespace flashinfer { + +namespace kv_buffer_saver { + +template +__device__ __forceinline__ void prepare( + vec_t& v_vec, + IdType& kv_cache_offset, + DType* v, + IdType* kv_cache_loc, + uint32_t idx, + uint32_t tx, + uint32_t kv_head_idx, + size_t v_stride_n, + size_t v_stride_h) { + kv_cache_offset = kv_cache_loc[idx]; + + DType* v_ptr = v + get_elem_offset_impl(idx, kv_head_idx, 0, v_stride_n, v_stride_h); + v_vec.cast_load(v_ptr + tx * vec_size); +} + +template +__device__ __forceinline__ void save( + IdType& kv_cache_offset, + vec_t& k_vec, + vec_t& v_vec, + DType* k_buffer, + DType* v_buffer, + uint32_t idx, + uint32_t tx, + uint32_t kv_head_idx, + size_t k_buffer_stride_n, + size_t k_buffer_stride_h, + size_t v_buffer_stride_n, + size_t v_buffer_stride_h) { + DType* k_buffer_ptr = + k_buffer + get_elem_offset_impl(kv_cache_offset, kv_head_idx, 0, k_buffer_stride_n, k_buffer_stride_h); + DType* v_buffer_ptr = + v_buffer + get_elem_offset_impl(kv_cache_offset, kv_head_idx, 0, v_buffer_stride_n, v_buffer_stride_h); + k_vec.cast_store(k_buffer_ptr + tx * vec_size); + v_vec.cast_store(v_buffer_ptr + tx * vec_size); +} + +} // namespace kv_buffer_saver + +template < + bool save_kv_cache, + bool interleave, + uint32_t head_dim, + uint32_t vec_size, + uint32_t bdx, + typename DType, + typename IdType> +__global__ void BatchQKApplyRotaryPosIdsCosSinCacheEnhancedHeadParallelismKernel( + DType* q, + DType* k, + DType* v, + DType* q_rope, + DType* k_rope, + DType* k_buffer, + DType* v_buffer, + float* __restrict__ cos_sin_cache, + IdType* __restrict__ pos_ids, + uint32_t nnz, + uint32_t num_qo_heads, + uint32_t num_kv_heads, + uint32_t rotary_dim, + size_t q_stride_n, + size_t q_stride_h, + size_t k_stride_n, + size_t k_stride_h, + size_t v_stride_n, + size_t v_stride_h, + size_t q_rope_stride_n, + size_t q_rope_stride_h, + size_t k_rope_stride_n, + size_t k_rope_stride_h, + size_t k_buffer_stride_n, + size_t k_buffer_stride_h, + size_t v_buffer_stride_n, + size_t v_buffer_stride_h, + IdType* __restrict__ kv_cache_loc) { + uint32_t bx = blockIdx.x, tx = threadIdx.x, ty = threadIdx.y; + uint32_t by = blockIdx.y; + const uint32_t bdy = blockDim.y; + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + asm volatile("griddepcontrol.wait;"); +#endif + + vec_t cos, sin; + if (bx * bdy + ty < nnz) { + const uint32_t idx = bx * bdy + ty; + const IdType pos = pos_ids[idx]; + + const int half_rotary_dim = rotary_dim / 2; + + // 1. if interleave: + // - cos = cos_sin_cache[pos_id][tx * vec_size // 2] + // - sin = cos_sin_cache[pos_id][(rot_dim // 2) + tx * vec_size // 2] + // 2. if not interleave + // - cos = cos_cache[pos_id][(tx * vec_size) % (rot_dim // 2)] + // - sin = sin_cache[pos_id][(rot_dim // 2) + (tx * vec_size) % (rot_dim // 2)] + if (tx * vec_size < rotary_dim) { + int sin_offset = rotary_dim / 2; + int vec_idx; + if constexpr (interleave) { + vec_idx = (tx * vec_size) / 2; // Force integer division + } else { + vec_idx = (tx * vec_size) % half_rotary_dim; // Use half_rotary_dim + } + cos.load(cos_sin_cache + (pos * rotary_dim) + vec_idx); + sin.load(cos_sin_cache + (pos * rotary_dim) + (sin_offset + vec_idx)); + } + + if (by < num_qo_heads) { + uint32_t qo_head_idx = by; + DType* q_ptr = q + get_elem_offset_impl(idx, qo_head_idx, 0, q_stride_n, q_stride_h); + DType* q_rope_ptr = q_rope + get_elem_offset_impl(idx, qo_head_idx, 0, q_rope_stride_n, q_rope_stride_h); + vec_t q_vec; + if constexpr (interleave) { + q_vec = vec_apply_llama_rope_cos_sin_interleave_reuse_half(q_ptr, cos, sin, rotary_dim); + } else { + q_vec = vec_apply_llama_rope_cos_sin(q_ptr, cos, sin, rotary_dim); + } + q_vec.cast_store(q_rope_ptr + tx * vec_size); + } else { + uint32_t kv_head_idx = by - num_qo_heads; + DType* k_ptr = k + get_elem_offset_impl(idx, kv_head_idx, 0, k_stride_n, k_stride_h); + + DType* k_rope_ptr = k_rope + get_elem_offset_impl(idx, kv_head_idx, 0, k_rope_stride_n, k_rope_stride_h); + + vec_t v_vec; + IdType kv_cache_offset; + if constexpr (save_kv_cache) { + kv_buffer_saver::prepare( + v_vec, kv_cache_offset, v, kv_cache_loc, idx, tx, kv_head_idx, v_stride_n, v_stride_h); + } + + vec_t k_vec; + if constexpr (interleave) { + k_vec = vec_apply_llama_rope_cos_sin_interleave_reuse_half(k_ptr, cos, sin, rotary_dim); + } else { + k_vec = vec_apply_llama_rope_cos_sin(k_ptr, cos, sin, rotary_dim); + } + k_vec.cast_store(k_rope_ptr + tx * vec_size); + + if constexpr (save_kv_cache) { + kv_buffer_saver::save( + kv_cache_offset, + k_vec, + v_vec, + k_buffer, + v_buffer, + idx, + tx, + kv_head_idx, + k_buffer_stride_n, + k_buffer_stride_h, + v_buffer_stride_n, + v_buffer_stride_h); + } + } + } + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + asm volatile("griddepcontrol.launch_dependents;"); +#endif +} + +template < + bool save_kv_cache, + bool interleave, + uint32_t head_dim, + uint32_t vec_size, + uint32_t bdx, + typename DType, + typename IdType> +__global__ void BatchQKApplyRotaryPosIdsCosSinCacheEnhancedKernel( + DType* q, + DType* k, + DType* v, + DType* q_rope, + DType* k_rope, + DType* k_buffer, + DType* v_buffer, + float* __restrict__ cos_sin_cache, + IdType* __restrict__ pos_ids, + uint32_t nnz, + uint32_t num_qo_heads, + uint32_t num_kv_heads, + uint32_t rotary_dim, + size_t q_stride_n, + size_t q_stride_h, + size_t k_stride_n, + size_t k_stride_h, + size_t v_stride_n, + size_t v_stride_h, + size_t q_rope_stride_n, + size_t q_rope_stride_h, + size_t k_rope_stride_n, + size_t k_rope_stride_h, + size_t k_buffer_stride_n, + size_t k_buffer_stride_h, + size_t v_buffer_stride_n, + size_t v_buffer_stride_h, + IdType* __restrict__ kv_cache_loc) { + uint32_t bx = blockIdx.x, tx = threadIdx.x, ty = threadIdx.y; + const uint32_t bdy = blockDim.y; + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + asm volatile("griddepcontrol.wait;"); +#endif + + vec_t cos, sin; + if (bx * bdy + ty < nnz) { + const uint32_t idx = bx * bdy + ty; + const IdType pos = pos_ids[idx]; + const int half_rotary_dim = rotary_dim / 2; + + // 1. if interleave: + // - cos = cos_sin_cache[pos_id][tx * vec_size // 2] + // - sin = cos_sin_cache[pos_id][(rot_dim // 2) + tx * vec_size // 2] + // 2. if not interleave + // - cos = cos_cache[pos_id][(tx * vec_size) % (rot_dim // 2)] + // - sin = sin_cache[pos_id][(rot_dim // 2) + (tx * vec_size) % (rot_dim // 2)] + if (tx * vec_size < rotary_dim) { + int sin_offset = rotary_dim / 2; + int vec_idx; + if constexpr (interleave) { + vec_idx = (tx * vec_size) / 2; // Force integer division + } else { + vec_idx = (tx * vec_size) % half_rotary_dim; // Use half_rotary_dim + } + cos.load(cos_sin_cache + (pos * rotary_dim) + vec_idx); + sin.load(cos_sin_cache + (pos * rotary_dim) + (sin_offset + vec_idx)); + } + + // not to unroll the loop, because num head might be large and might lead to worse performance +#pragma unroll 1 + for (uint32_t qo_head_idx = 0; qo_head_idx < num_qo_heads; ++qo_head_idx) { + DType* q_ptr = q + get_elem_offset_impl(idx, qo_head_idx, 0, q_stride_n, q_stride_h); + DType* q_rope_ptr = q_rope + get_elem_offset_impl(idx, qo_head_idx, 0, q_rope_stride_n, q_rope_stride_h); + vec_t q_vec; + if constexpr (interleave) { + q_vec = vec_apply_llama_rope_cos_sin_interleave_reuse_half(q_ptr, cos, sin, rotary_dim); + } else { + q_vec = vec_apply_llama_rope_cos_sin(q_ptr, cos, sin, rotary_dim); + } + q_vec.cast_store(q_rope_ptr + tx * vec_size); + } + +#pragma unroll 1 + for (uint32_t kv_head_idx = 0; kv_head_idx < num_kv_heads; ++kv_head_idx) { + DType* k_ptr = k + get_elem_offset_impl(idx, kv_head_idx, 0, k_stride_n, k_stride_h); + + DType* k_rope_ptr = k_rope + get_elem_offset_impl(idx, kv_head_idx, 0, k_rope_stride_n, k_rope_stride_h); + + vec_t v_vec; + IdType kv_cache_offset; + if constexpr (save_kv_cache) { + kv_buffer_saver::prepare( + v_vec, kv_cache_offset, v, kv_cache_loc, idx, tx, kv_head_idx, v_stride_n, v_stride_h); + } + + vec_t k_vec; + if constexpr (interleave) { + k_vec = vec_apply_llama_rope_cos_sin_interleave_reuse_half(k_ptr, cos, sin, rotary_dim); + } else { + k_vec = vec_apply_llama_rope_cos_sin(k_ptr, cos, sin, rotary_dim); + } + k_vec.cast_store(k_rope_ptr + tx * vec_size); + + if constexpr (save_kv_cache) { + kv_buffer_saver::save( + kv_cache_offset, + k_vec, + v_vec, + k_buffer, + v_buffer, + idx, + tx, + kv_head_idx, + k_buffer_stride_n, + k_buffer_stride_h, + v_buffer_stride_n, + v_buffer_stride_h); + } + } + } + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + asm volatile("griddepcontrol.launch_dependents;"); +#endif +} + +#define DISPATCH_SAVE_KV_CACHE(save_kv_cache, SAVE_KV_CACHE, ...) \ + if (save_kv_cache) { \ + const bool SAVE_KV_CACHE = true; \ + __VA_ARGS__ \ + } else { \ + const bool SAVE_KV_CACHE = false; \ + __VA_ARGS__ \ + } + +template +cudaError_t BatchQKApplyRotaryPosIdsCosSinCacheEnhanced( + DType* q, + DType* k, + DType* v, + DType* q_rope, + DType* k_rope, + DType* k_buffer, + DType* v_buffer, + float* cos_sin_cache, + IdType* pos_ids, + uint32_t nnz, + uint32_t num_qo_heads, + uint32_t num_kv_heads, + uint32_t rotary_dim, + uint32_t head_dim, + size_t q_stride_n, + size_t q_stride_h, + size_t k_stride_n, + size_t k_stride_h, + size_t v_stride_n, + size_t v_stride_h, + size_t q_rope_stride_n, + size_t q_rope_stride_h, + size_t k_rope_stride_n, + size_t k_rope_stride_h, + size_t k_buffer_stride_n, + size_t k_buffer_stride_h, + size_t v_buffer_stride_n, + size_t v_buffer_stride_h, + IdType* kv_cache_loc, + bool interleave, + bool save_kv_cache, + bool enable_pdl, + cudaStream_t stream = nullptr) { + int dev_id = 0; + int num_sms = 0; + FLASHINFER_CUDA_CALL(cudaGetDevice(&dev_id)); + FLASHINFER_CUDA_CALL(cudaDeviceGetAttribute(&num_sms, cudaDevAttrMultiProcessorCount, dev_id)); + +#define LAUNCH_KERNEL_RAW(kernel_name) \ + do { \ + cudaLaunchConfig_t config = {}; \ + config.gridDim = nblks; \ + config.blockDim = nthrs; \ + config.dynamicSmemBytes = 0; \ + config.stream = stream; \ + cudaLaunchAttribute attrs[1] = {}; \ + attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; \ + attrs[0].val.programmaticStreamSerializationAllowed = enable_pdl; \ + config.numAttrs = 1; \ + config.attrs = attrs; \ + \ + FLASHINFER_CUDA_CALL(cudaLaunchKernelEx( \ + &config, \ + kernel_name, \ + q, \ + k, \ + v, \ + q_rope, \ + k_rope, \ + k_buffer, \ + v_buffer, \ + cos_sin_cache, \ + pos_ids, \ + nnz, \ + num_qo_heads, \ + num_kv_heads, \ + rotary_dim, \ + q_stride_n, \ + q_stride_h, \ + k_stride_n, \ + k_stride_h, \ + v_stride_n, \ + v_stride_h, \ + q_rope_stride_n, \ + q_rope_stride_h, \ + k_rope_stride_n, \ + k_rope_stride_h, \ + k_buffer_stride_n, \ + k_buffer_stride_h, \ + v_buffer_stride_n, \ + v_buffer_stride_h, \ + kv_cache_loc)); \ + } while (0) + + DISPATCH_SAVE_KV_CACHE(save_kv_cache, SAVE_KV_CACHE, { + DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { + DISPATCH_HEAD_DIM(head_dim, HEAD_DIM, { + // operate on 16 Bytes at a time + constexpr uint32_t vec_size = std::max(16 / sizeof(DType), HEAD_DIM / 32); + // how many threads needed per head_dim + constexpr uint32_t bdx = HEAD_DIM / vec_size; + // how many threads needed per block + uint32_t num_threads = std::max(128U, bdx); + // how many tokens can we process in a block + uint32_t bdy = num_threads / bdx; + // how many blocks needed to process all tokens + uint32_t nblks_x = (nnz + bdy - 1) / bdy; + + auto kernel_0 = BatchQKApplyRotaryPosIdsCosSinCacheEnhancedKernel< + SAVE_KV_CACHE, + INTERLEAVE, + HEAD_DIM, + vec_size, + bdx, + DType, + IdType>; + + int num_blocks_per_sm_0 = 0; + FLASHINFER_CUDA_CALL(cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &num_blocks_per_sm_0, kernel_0, num_threads, /*smem_size=*/0)); + uint32_t num_ctas_0 = num_blocks_per_sm_0 * num_sms; + + if ((nnz + bdy - 1) / bdy >= num_ctas_0) { + dim3 nblks(nblks_x); + dim3 nthrs(bdx, bdy); + LAUNCH_KERNEL_RAW(kernel_0); + } else { + dim3 nblks(nblks_x, num_qo_heads + num_kv_heads); + dim3 nthrs(bdx, bdy); + auto kernel_1 = BatchQKApplyRotaryPosIdsCosSinCacheEnhancedHeadParallelismKernel< + SAVE_KV_CACHE, + INTERLEAVE, + HEAD_DIM, + vec_size, + bdx, + DType, + IdType>; + LAUNCH_KERNEL_RAW(kernel_1); + } + }); + }); + }); +#undef LAUNCH_KERNEL_RAW + + return cudaSuccess; +} + +} // namespace flashinfer + +#endif // SGL_POS_ENC_CUH_ diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/rope.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/rope.cu new file mode 100644 index 00000000..b6abd1cc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/rope.cu @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2024 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include "pos_enc.cuh" +#include "utils.h" + +using namespace flashinfer; + +void apply_rope_pos_ids_cos_sin_cache( + at::Tensor q, + at::Tensor k, + at::Tensor q_rope, + at::Tensor k_rope, + at::Tensor cos_sin_cache, + at::Tensor pos_ids, + bool interleave, + bool enable_pdl, + const std::optional& v, + const std::optional& k_buffer, + const std::optional& v_buffer, + const std::optional& kv_cache_loc) { + CHECK_LAST_DIM_CONTIGUOUS(q); + CHECK_LAST_DIM_CONTIGUOUS(k); + + const bool save_kv_cache = v.has_value(); + if (save_kv_cache) { + TORCH_CHECK(v.has_value()); + TORCH_CHECK(k_buffer.has_value()); + TORCH_CHECK(v_buffer.has_value()); + TORCH_CHECK(kv_cache_loc.has_value()); + CHECK_LAST_DIM_CONTIGUOUS(v.value()); + CHECK_LAST_DIM_CONTIGUOUS(k_buffer.value()); + CHECK_LAST_DIM_CONTIGUOUS(v_buffer.value()); + CHECK_DIM(3, k_buffer.value()); // k_buffer: (nnz, H_K, D) + CHECK_DIM(3, v_buffer.value()); // v_buffer: (nnz, H_V, D) + CHECK_DIM(3, v.value()); // v: (nnz, H_V, D) + CHECK_DIM(1, kv_cache_loc.value()); // v: (n) + CHECK_INPUT(kv_cache_loc.value()); + } + size_t k_buffer_stride_n = save_kv_cache ? k_buffer->stride(0) : 0; + size_t k_buffer_stride_h = save_kv_cache ? k_buffer->stride(1) : 0; + size_t v_buffer_stride_n = save_kv_cache ? v_buffer->stride(0) : 0; + size_t v_buffer_stride_h = save_kv_cache ? v_buffer->stride(1) : 0; + size_t v_stride_n = save_kv_cache ? v->stride(0) : 0; + size_t v_stride_h = save_kv_cache ? v->stride(1) : 0; + auto kv_cache_loc_ptr = save_kv_cache ? static_cast(kv_cache_loc->data_ptr()) : nullptr; + + CHECK_INPUT(cos_sin_cache); + CHECK_INPUT(pos_ids); + auto device = q.device(); + CHECK_EQ(k.device(), device); + CHECK_EQ(cos_sin_cache.device(), device); + CHECK_EQ(pos_ids.device(), device); + CHECK_DIM(3, q); // q: (nnz, H_Q, D) + CHECK_DIM(3, k); // k: (nnz, H_K, D) + + // cos_sin_cache: (max_seq_len, R) + // First half of R is cos, second half is sin + CHECK_DIM(2, cos_sin_cache); + CHECK_EQ(q.size(0), k.size(0)); + CHECK_EQ(q.size(2), k.size(2)); + unsigned int rotary_dim = cos_sin_cache.size(1); + unsigned int num_qo_heads = q.size(1); + unsigned int num_kv_heads = k.size(1); + unsigned int head_dim = q.size(2); + unsigned int nnz = q.size(0); + size_t q_stride_n = q.stride(0); + size_t q_stride_h = q.stride(1); + size_t k_stride_n = k.stride(0); + size_t k_stride_h = k.stride(1); + + size_t q_rope_stride_n = q_rope.stride(0); + size_t q_rope_stride_h = q_rope.stride(1); + size_t k_rope_stride_n = k_rope.stride(0); + size_t k_rope_stride_h = k_rope.stride(1); + + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(q.scalar_type(), c_type, [&] { + // TODO temporarily only use `BatchQKApplyRotaryPosIdsCosSinCacheEnhanced` when save_kv_cache + // to avoid changing original code path; but this branch is feature-complete and should switch to this later + if (save_kv_cache) { + cudaError_t status = BatchQKApplyRotaryPosIdsCosSinCacheEnhanced( + static_cast(q.data_ptr()), + static_cast(k.data_ptr()), + save_kv_cache ? static_cast(v->data_ptr()) : nullptr, + static_cast(q_rope.data_ptr()), + static_cast(k_rope.data_ptr()), + save_kv_cache ? static_cast(k_buffer->data_ptr()) : nullptr, + save_kv_cache ? static_cast(v_buffer->data_ptr()) : nullptr, + static_cast(cos_sin_cache.data_ptr()), + static_cast(pos_ids.data_ptr()), + nnz, + num_qo_heads, + num_kv_heads, + rotary_dim, + head_dim, + q_stride_n, + q_stride_h, + k_stride_n, + k_stride_h, + v_stride_n, + v_stride_h, + q_rope_stride_n, + q_rope_stride_h, + k_rope_stride_n, + k_rope_stride_h, + k_buffer_stride_n, + k_buffer_stride_h, + v_buffer_stride_n, + v_buffer_stride_h, + kv_cache_loc_ptr, + interleave, + save_kv_cache, + enable_pdl, + stream); + TORCH_CHECK( + status == cudaSuccess, + "BatchQKApplyRotaryPosIdsCosSinCacheEnhanced failed with error code " + + std::string(cudaGetErrorString(status))); + } else { + TORCH_CHECK(!enable_pdl); + cudaError_t status = BatchQKApplyRotaryPosIdsCosSinCache( + static_cast(q.data_ptr()), + static_cast(k.data_ptr()), + static_cast(q_rope.data_ptr()), + static_cast(k_rope.data_ptr()), + static_cast(cos_sin_cache.data_ptr()), + static_cast(pos_ids.data_ptr()), + nnz, + num_qo_heads, + num_kv_heads, + rotary_dim, + head_dim, + q_stride_n, + q_stride_h, + k_stride_n, + k_stride_h, + q_rope_stride_n, + q_rope_stride_h, + k_rope_stride_n, + k_rope_stride_h, + interleave, + stream); + TORCH_CHECK( + status == cudaSuccess, + "BatchQKApplyRotaryPosIdsCosSinCache failed with error code " + std::string(cudaGetErrorString(status))); + } + return true; + }); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/topk.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/topk.cu new file mode 100644 index 00000000..066fe4be --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/topk.cu @@ -0,0 +1,522 @@ +/** + * @NOTE: This file is adapted from + * https://github.com/tile-ai/tilelang/blob/main/examples/deepseek_v32/topk_selector.py + * We: + * 1. adapt from tilelang to pure cuda + * 2. optimize the performance a little + * 3. fix the potential illegal memory access + */ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace { + +constexpr int TopK = 2048; +constexpr int kThreadsPerBlock = 1024; +constexpr size_t kSmem = 32 * 1024 * sizeof(uint32_t); // 128KB + +struct FastTopKParams { + const float* __restrict__ input; // [B, input_stride] + const int32_t* __restrict__ row_starts; // [B] + int32_t* __restrict__ indices; // [B, TopK] + int32_t* __restrict__ lengths; // [B] + int64_t input_stride; +}; + +// when length <= TopK, we can directly write the indices +__device__ void naive_topk_cuda(const float* __restrict__ score, int32_t* __restrict__ indice, int32_t length) { + const auto tid = threadIdx.x; + for (int i = tid; i < TopK; i += kThreadsPerBlock) { + indice[i] = (i < length) ? i : -1; + } +} + +// keep the first `length` entries, set others to -1 +__device__ void naive_topk_transform( + const float* __restrict__ score, + int32_t length, + int32_t* __restrict__ dst_page_table, + const int32_t* __restrict__ src_page_table) { + const auto tid = threadIdx.x; + for (auto i = tid; i < TopK; i += kThreadsPerBlock) { + dst_page_table[i] = (i < length) ? src_page_table[i] : -1; + } +} + +// keep the first `length` entries, set others to -1 +__device__ void naive_topk_transform_ragged( + const float* __restrict__ score, int32_t length, int32_t* __restrict__ topk_indices_ragged, int32_t offset) { + const auto tid = threadIdx.x; + for (auto i = tid; i < TopK; i += kThreadsPerBlock) { + topk_indices_ragged[i] = (i < length) ? static_cast(i) + offset : -1; + } +} + +__device__ __forceinline__ auto convert_to_uint8(float x) -> uint8_t { + __half h = __float2half_rn(x); + uint16_t bits = __half_as_ushort(h); + uint16_t key = (bits & 0x8000) ? static_cast(~bits) : static_cast(bits | 0x8000); + return static_cast(key >> 8); +} + +__device__ __forceinline__ auto convert_to_uint32(float x) -> uint32_t { + uint32_t bits = __float_as_uint(x); + return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u); +} + +__device__ void fast_topk_cuda_tl(const float* __restrict__ input, int* __restrict__ index, int row_start, int length) { + // An optimized topk kernel copied from tilelang kernel + // We assume length > TopK here, or it will crash + int topk = TopK; + constexpr auto BLOCK_SIZE = 1024; + constexpr auto RADIX = 256; + constexpr auto SMEM_INPUT_SIZE = kSmem / (2 * sizeof(int)); + + alignas(128) __shared__ int s_histogram_buf[2][RADIX + 128]; + alignas(128) __shared__ int s_counter; + alignas(128) __shared__ int s_threshold_bin_id; + alignas(128) __shared__ int s_num_input[2]; + + auto& s_histogram = s_histogram_buf[0]; + // allocate for two rounds + extern __shared__ int s_input_idx[][SMEM_INPUT_SIZE]; + + const int tx = threadIdx.x; + + // stage 1: 8bit coarse histogram + if (tx < RADIX + 1) s_histogram[tx] = 0; + __syncthreads(); + + for (int idx = tx; idx < length; idx += BLOCK_SIZE) { + const auto bin = convert_to_uint8(input[idx + row_start]); + ::atomicAdd(&s_histogram[bin], 1); + } + __syncthreads(); + + const auto run_cumsum = [&] { +#pragma unroll 8 + for (int i = 0; i < 8; ++i) { + static_assert(1 << 8 == RADIX); + if (C10_LIKELY(tx < RADIX)) { + const auto j = 1 << i; + const auto k = i & 1; + auto value = s_histogram_buf[k][tx]; + if (tx < RADIX - j) { + value += s_histogram_buf[k][tx + j]; + } + s_histogram_buf[k ^ 1][tx] = value; + } + __syncthreads(); + } + }; + + run_cumsum(); + if (tx < RADIX && s_histogram[tx] > topk && s_histogram[tx + 1] <= topk) { + s_threshold_bin_id = tx; + s_num_input[0] = 0; + s_counter = 0; + } + __syncthreads(); + + const auto threshold_bin = s_threshold_bin_id; + topk -= s_histogram[threshold_bin + 1]; + + if (topk == 0) { + for (int idx = tx; idx < length; idx += BLOCK_SIZE) { + const auto bin = static_cast(convert_to_uint8(input[idx + row_start])); + if (bin > threshold_bin) { + const auto pos = ::atomicAdd(&s_counter, 1); + index[pos] = idx; + } + } + __syncthreads(); + return; + } else { + __syncthreads(); + if (tx < RADIX + 1) { + s_histogram[tx] = 0; + } + __syncthreads(); + + for (int idx = tx; idx < length; idx += BLOCK_SIZE) { + const auto raw_input = input[idx + row_start]; + const auto bin = static_cast(convert_to_uint8(raw_input)); + if (bin > threshold_bin) { + const auto pos = ::atomicAdd(&s_counter, 1); + index[pos] = idx; + } else if (bin == threshold_bin) { + const auto pos = ::atomicAdd(&s_num_input[0], 1); + /// NOTE: (dark) fuse the histogram computation here + if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) { + s_input_idx[0][pos] = idx; + const auto bin = convert_to_uint32(raw_input); + const auto sub_bin = (bin >> 24) & 0xFF; + ::atomicAdd(&s_histogram[sub_bin], 1); + } + } + } + __syncthreads(); + } + + // stage 2: refine with 8bit radix passes +#pragma unroll 4 + for (int round = 0; round < 4; ++round) { + __shared__ int s_last_remain; + const auto r_idx = round % 2; + + // clip here to prevent overflow + const auto _raw_num_input = s_num_input[r_idx]; + const auto num_input = (_raw_num_input < int(SMEM_INPUT_SIZE)) ? _raw_num_input : int(SMEM_INPUT_SIZE); + + run_cumsum(); + if (tx < RADIX && s_histogram[tx] > topk && s_histogram[tx + 1] <= topk) { + s_threshold_bin_id = tx; + s_num_input[r_idx ^ 1] = 0; + s_last_remain = topk - s_histogram[tx + 1]; + } + __syncthreads(); + + const auto threshold_bin = s_threshold_bin_id; + topk -= s_histogram[threshold_bin + 1]; + + if (topk == 0) { + for (int i = tx; i < num_input; i += BLOCK_SIZE) { + const auto idx = s_input_idx[r_idx][i]; + const auto offset = 24 - round * 8; + const auto bin = (convert_to_uint32(input[idx + row_start]) >> offset) & 0xFF; + if (bin > threshold_bin) { + const auto pos = ::atomicAdd(&s_counter, 1); + index[pos] = idx; + } + } + __syncthreads(); + break; + } else { + __syncthreads(); + if (tx < RADIX + 1) { + s_histogram[tx] = 0; + } + __syncthreads(); + for (int i = tx; i < num_input; i += BLOCK_SIZE) { + const auto idx = s_input_idx[r_idx][i]; + const auto raw_input = input[idx + row_start]; + const auto offset = 24 - round * 8; + const auto bin = (convert_to_uint32(raw_input) >> offset) & 0xFF; + if (bin > threshold_bin) { + const auto pos = ::atomicAdd(&s_counter, 1); + index[pos] = idx; + } else if (bin == threshold_bin) { + if (round == 3) { + const auto pos = ::atomicAdd(&s_last_remain, -1); + if (pos > 0) { + index[TopK - pos] = idx; + } + } else { + const auto pos = ::atomicAdd(&s_num_input[r_idx ^ 1], 1); + if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) { + /// NOTE: (dark) fuse the histogram computation here + s_input_idx[r_idx ^ 1][pos] = idx; + const auto bin = convert_to_uint32(raw_input); + const auto sub_bin = (bin >> (offset - 8)) & 0xFF; + ::atomicAdd(&s_histogram[sub_bin], 1); + } + } + } + } + __syncthreads(); + } + } +} + +__global__ __launch_bounds__(kThreadsPerBlock) // topk + void topk_kernel(const FastTopKParams params) { + const auto& [input, row_starts, indices, lengths, input_stride] = params; + const auto bid = static_cast(blockIdx.x); + const auto row_start = row_starts == nullptr ? 0 : row_starts[bid]; + const auto length = lengths[bid]; + const auto indice = indices + bid * TopK; + const auto score = input + bid * input_stride; + if (length <= TopK) { + return naive_topk_cuda(score, indice, length); + } else { + return fast_topk_cuda_tl(score, indice, row_start, length); + } +} + +__global__ __launch_bounds__(kThreadsPerBlock) // decode + void topk_transform_decode_kernel( + const FastTopKParams params, + int32_t* __restrict__ dst_page_table, + const int32_t* __restrict__ src_page_table, + const int64_t src_stride) { + const auto& [input, _1, _2, lengths, input_stride] = params; + const auto bid = static_cast(blockIdx.x); + const auto tid = threadIdx.x; + const auto row_start = 0; + const auto length = lengths[bid]; + const auto src_page_entry = src_page_table + bid * src_stride; + const auto dst_page_entry = dst_page_table + bid * TopK; + const auto score = input + bid * input_stride; + if (length <= TopK) { + return naive_topk_transform(score, length, dst_page_entry, src_page_entry); + } else { + __shared__ int s_indices[TopK]; + fast_topk_cuda_tl(score, s_indices, row_start, length); + // copy src[s_indices] to dst, we manually unroll here + static_assert(TopK % kThreadsPerBlock == 0); + static_assert(TopK / kThreadsPerBlock == 2); + const auto idx_0 = tid; + const auto pos_0 = s_indices[idx_0]; + dst_page_entry[idx_0] = src_page_entry[pos_0]; + const auto idx_1 = tid + kThreadsPerBlock; + const auto pos_1 = s_indices[idx_1]; + dst_page_entry[idx_1] = src_page_entry[pos_1]; + } +} + +__global__ __launch_bounds__(kThreadsPerBlock) // prefill + void topk_transform_prefill_kernel( + const FastTopKParams params, + int32_t* __restrict__ dst_page_table, + const int32_t* __restrict__ src_page_table, + const int64_t src_stride, + const int32_t* __restrict__ cu_seqlens_q, + const int64_t prefill_bs) { + const auto& [input, row_starts, _, lengths, input_stride] = params; + const auto bid = static_cast(blockIdx.x); + const auto tid = threadIdx.x; + const auto length = lengths[bid]; + const auto row_start = row_starts == nullptr ? 0 : row_starts[bid]; + const auto dst_page_entry = dst_page_table + bid * TopK; + const auto score = input + bid * input_stride; + + /// NOTE: prefill bs is usually small, we can just use a simple loop here + /// We ensure that last cu_seqlens is equal to number of blocks launched + __shared__ const int32_t* s_src_page_entry; + if (C10_LIKELY(prefill_bs <= kThreadsPerBlock)) { + if (tid < prefill_bs) { + if (bid >= cu_seqlens_q[tid] && bid < cu_seqlens_q[tid + 1]) { + s_src_page_entry = src_page_table + tid * src_stride; + } + } + } else { + for (int64_t i = tid; i < prefill_bs; i += kThreadsPerBlock) { + if (bid >= cu_seqlens_q[i] && bid < cu_seqlens_q[i + 1]) { + s_src_page_entry = src_page_table + i * src_stride; + } + } + } + __syncthreads(); + const auto src_page_entry = s_src_page_entry; + + if (length <= TopK) { + return naive_topk_transform(score, length, dst_page_entry, src_page_entry); + } else { + __shared__ int s_indices[TopK]; + fast_topk_cuda_tl(score, s_indices, row_start, length); + // copy src[s_indices] to dst, we manually unroll here + static_assert(TopK % kThreadsPerBlock == 0); + static_assert(TopK / kThreadsPerBlock == 2); + const auto idx_0 = tid; + const auto pos_0 = s_indices[idx_0]; + dst_page_entry[idx_0] = src_page_entry[pos_0]; + const auto idx_1 = tid + kThreadsPerBlock; + const auto pos_1 = s_indices[idx_1]; + dst_page_entry[idx_1] = src_page_entry[pos_1]; + } +} + +__global__ __launch_bounds__(kThreadsPerBlock) // prefill, ragged kv + void topk_transform_prefill_ragged_kernel( + const FastTopKParams params, + int32_t* __restrict__ topk_indices_ragged, + const int32_t* __restrict__ topk_indices_offset) { + const auto& [input, row_starts, _, lengths, input_stride] = params; + const auto bid = static_cast(blockIdx.x); + const auto tid = threadIdx.x; + const auto row_start = row_starts == nullptr ? 0 : row_starts[bid]; + const auto length = lengths[bid]; + const auto dst_indices_entry = topk_indices_ragged + bid * TopK; + const auto score = input + bid * input_stride; + const auto offset = topk_indices_offset[bid]; + + if (length <= TopK) { + return naive_topk_transform_ragged(score, length, dst_indices_entry, offset); + } else { + __shared__ int s_indices[TopK]; + fast_topk_cuda_tl(score, s_indices, row_start, length); + // copy src[s_indices] to dst, we manually unroll here + static_assert(TopK % kThreadsPerBlock == 0); + static_assert(TopK / kThreadsPerBlock == 2); + const auto idx_0 = tid; + const auto pos_0 = s_indices[idx_0]; + dst_indices_entry[idx_0] = pos_0 + offset; + const auto idx_1 = tid + kThreadsPerBlock; + const auto pos_1 = s_indices[idx_1]; + dst_indices_entry[idx_1] = pos_1 + offset; + } +} + +auto get_params( + const at::Tensor& score, + const at::Tensor& lengths, + std::optional row_starts_opt = std::nullopt, + std::optional indices_opt = std::nullopt) -> FastTopKParams { + const auto B = score.size(0); + TORCH_CHECK(score.dim() == 2 && score.stride(1) == 1); + if (row_starts_opt.has_value()) { + const auto& row_starts = row_starts_opt.value(); + TORCH_CHECK(row_starts.dim() == 1); + TORCH_CHECK(row_starts.size(0) == B); + } + TORCH_CHECK(lengths.dim() == 1 && lengths.is_contiguous()); + TORCH_CHECK(lengths.size(0) == B); + int32_t* indices_data_ptr = nullptr; + if (indices_opt.has_value()) { + const auto& indices = indices_opt.value(); + TORCH_CHECK(indices.dim() == 2 && indices.is_contiguous()); + TORCH_CHECK(indices.size(0) == B); + TORCH_CHECK(indices.size(1) == TopK); + indices_data_ptr = indices.data_ptr(); + } + + return FastTopKParams{ + .input = score.data_ptr(), + .row_starts = row_starts_opt.has_value() ? row_starts_opt->data_ptr() : nullptr, + .indices = indices_data_ptr, + .lengths = lengths.data_ptr(), + .input_stride = score.stride(0), + }; +} + +template +void setup_kernel_smem_once() { + [[maybe_unused]] + static const auto result = + [] { return ::cudaFuncSetAttribute(f, ::cudaFuncAttributeMaxDynamicSharedMemorySize, max_dynamic_smem); }(); + TORCH_CHECK(result == cudaSuccess, "set_up_kernel_once failed:", ::cudaGetErrorString(result)); +} + +} // namespace + +#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor") + +void fast_topk_interface( + const at::Tensor& score, at::Tensor& indices, const at::Tensor& lengths, std::optional row_starts_opt) { + CHECK_CUDA(score); + CHECK_CUDA(indices); + if (row_starts_opt.has_value()) { + CHECK_CUDA(row_starts_opt.value()); + } + CHECK_CUDA(lengths); + const auto params = get_params(score, lengths, row_starts_opt, indices); + const auto B = score.size(0); + const auto stream = at::cuda::getCurrentCUDAStream().stream(); + const auto grid = dim3{static_cast(B)}; + const auto block = dim3{kThreadsPerBlock}; + setup_kernel_smem_once(); + topk_kernel<<>>(params); + const auto result = cudaGetLastError(); + TORCH_CHECK(result == cudaSuccess, "topk kernel failed:", ::cudaGetErrorString(result)); +} + +void fast_topk_transform_interface( + const at::Tensor& score, + const at::Tensor& lengths, + at::Tensor& dst_page_table, + const at::Tensor& src_page_table, + const at::Tensor& cu_seqlens_q, + std::optional row_starts_opt) { + CHECK_CUDA(score); + CHECK_CUDA(lengths); + CHECK_CUDA(dst_page_table); + CHECK_CUDA(src_page_table); + CHECK_CUDA(cu_seqlens_q); + if (row_starts_opt.has_value()) { + CHECK_CUDA(row_starts_opt.value()); + } + const auto params = get_params(score, lengths, row_starts_opt); + const auto B = score.size(0); + TORCH_CHECK(dst_page_table.dim() == 2 && dst_page_table.is_contiguous()); + TORCH_CHECK(src_page_table.dim() == 2 && src_page_table.stride(1) == 1); + TORCH_CHECK(cu_seqlens_q.dim() == 1 && cu_seqlens_q.is_contiguous()); + const auto prefill_bs = cu_seqlens_q.size(0) - 1; + TORCH_CHECK(dst_page_table.size(0) == B); + TORCH_CHECK(dst_page_table.size(1) == TopK); + TORCH_CHECK(src_page_table.size(0) == prefill_bs); + TORCH_CHECK(prefill_bs <= B); // prefill_bs should be smaller than expanded bs + + // launch kernel + const auto stream = at::cuda::getCurrentCUDAStream().stream(); + const auto grid = dim3{static_cast(B)}; + const auto block = dim3{kThreadsPerBlock}; + const auto src_stride = src_page_table.stride(0); + + // dispatch to decode or prefill + // extend and draft extend: row_starts_opt is not null, invokes the prefill kernel + // decode: row_starts_opt is null, invokes the decode kernel + // target verify: row_starts_opt is null, invokes the prefill kernel + const auto is_decode = !row_starts_opt.has_value() && prefill_bs == B; + if (is_decode) { + setup_kernel_smem_once(); + topk_transform_decode_kernel<<>>( + params, dst_page_table.data_ptr(), src_page_table.data_ptr(), src_stride); + } else { + setup_kernel_smem_once(); + topk_transform_prefill_kernel<<>>( + params, + dst_page_table.data_ptr(), + src_page_table.data_ptr(), + src_stride, + cu_seqlens_q.data_ptr(), + prefill_bs); + } + + const auto result = cudaGetLastError(); + TORCH_CHECK(result == cudaSuccess, "topk kernel failed:", ::cudaGetErrorString(result)); +} + +void fast_topk_transform_ragged_interface( + const at::Tensor& score, + const at::Tensor& lengths, + at::Tensor& topk_indices_ragged, + const at::Tensor& topk_indices_offset, + std::optional row_starts_opt) { + CHECK_CUDA(score); + CHECK_CUDA(lengths); + CHECK_CUDA(topk_indices_ragged); + CHECK_CUDA(topk_indices_offset); + if (row_starts_opt.has_value()) { + CHECK_CUDA(row_starts_opt.value()); + } + + const auto params = get_params(score, lengths, row_starts_opt); + const auto B = score.size(0); + TORCH_CHECK(topk_indices_ragged.dim() == 2 && topk_indices_ragged.is_contiguous()); + TORCH_CHECK(topk_indices_offset.dim() == 1); + + TORCH_CHECK(topk_indices_ragged.size(0) == B); + TORCH_CHECK(topk_indices_ragged.size(1) == TopK); + TORCH_CHECK(topk_indices_offset.size(0) == B); + + // launch kernel + const auto stream = at::cuda::getCurrentCUDAStream().stream(); + const auto grid = dim3{static_cast(B)}; + const auto block = dim3{kThreadsPerBlock}; + + setup_kernel_smem_once(); + topk_transform_prefill_ragged_kernel<<>>( + params, topk_indices_ragged.data_ptr(), topk_indices_offset.data_ptr()); + + const auto result = cudaGetLastError(); + TORCH_CHECK(result == cudaSuccess, "topk kernel failed:", ::cudaGetErrorString(result)); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/utils.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/utils.cuh new file mode 100644 index 00000000..3010a545 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/elementwise/utils.cuh @@ -0,0 +1,72 @@ +// Adapted from https://github.com/deepseek-ai/DeepEP/blob/main/csrc/kernels/utils.cuh + +#pragma once + +#include +#include + +#include + +__forceinline__ __device__ int get_lane_id() { + int lane_id; + asm("mov.s32 %0, %laneid;" : "=r"(lane_id)); + return lane_id; +} + +int ceil_div(int a, int b) { + return (a + b - 1) / b; +} + +__device__ __forceinline__ void st_na_global_v1(const int* ptr, int v) { + asm volatile("st.global.L1::no_allocate.s32 [%0], %1;" ::"l"(ptr), "r"(v) : "memory"); +} + +__device__ __forceinline__ void st_na_global_v2(const int2* ptr, const int2& v) { + asm volatile("st.global.L1::no_allocate.v2.s32 [%0], {%1, %2};" ::"l"(ptr), "r"(v.x), "r"(v.y) : "memory"); +} + +__device__ __forceinline__ void st_na_global_v4(const int4* ptr, const int4& v) { + asm volatile( + "st.global.L1::no_allocate.v4.s32 [%0], {%1, %2, %3, %4};" ::"l"(ptr), "r"(v.x), "r"(v.y), "r"(v.z), "r"(v.w) + : "memory"); +} + +__device__ __forceinline__ int ld_na_global_v1(const int* ptr) { + int r; +#ifdef USE_L2_HINT + asm volatile("ld.global.nc.L1::no_allocate.L2::128B.s32 %0, [%1];" : "=r"(r) : "l"(ptr)); +#else + asm volatile("ld.global.nc.L1::no_allocate.s32 %0, [%1];" : "=r"(r) : "l"(ptr)); +#endif + return r; +} + +__device__ __forceinline__ int2 ld_na_global_v2(const int2* ptr) { + int2 r; +#ifdef USE_L2_HINT + asm volatile("ld.global.nc.L1::no_allocate.L2::128B.v2.s32 {%0, %1}, [%2];" : "=r"(r.x), "=r"(r.y) : "l"(ptr)); +#else + asm volatile("ld.global.nc.L1::no_allocate.v2.s32 {%0, %1}, [%2];" : "=r"(r.x), "=r"(r.y) : "l"(ptr)); +#endif + return r; +} + +__device__ __forceinline__ int4 ld_na_global_v4(const int4* ptr) { + int4 r; +#ifdef USE_L2_HINT + asm volatile("ld.global.nc.L1::no_allocate.L2::128B.v4.s32 {%0, %1, %2, %3}, [%4];" + : "=r"(r.x), "=r"(r.y), "=r"(r.z), "=r"(r.w) + : "l"(ptr)); +#else + asm volatile("ld.global.nc.L1::no_allocate.v4.s32 {%0, %1, %2, %3}, [%4];" + : "=r"(r.x), "=r"(r.y), "=r"(r.z), "=r"(r.w) + : "l"(ptr)); +#endif + return r; +} + +__device__ __forceinline__ void prefetch_L2(const void* p) { +#if defined(ENABLE_L2_PREFETCH) + asm volatile("prefetch.global.L2 [%0];" ::"l"(p)); +#endif +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/expert_specialization/es_fp8_blockwise.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/expert_specialization/es_fp8_blockwise.cu new file mode 100644 index 00000000..f05209b3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/expert_specialization/es_fp8_blockwise.cu @@ -0,0 +1,167 @@ +#include + +#include + +#include "es_fp8_blockwise_launcher.cuh" + +/** + * @brief Performs blockwise grouped matrix multiplication on FP8 quantized inputs, + * with per-block scaling. + * + * This function dispatches to hardware-specific implementations (e.g., SM100 FP8) + * to compute: + * C_i = scale_a[i] * A_i * scale_b[i] * B_i + * for each expert group `i`, using input `problem_sizes` and `expert_offsets` + * to describe the individual matrix dimensions and their offsets. + * + * Input tensors A and B must be quantized to 8-bit formats and dequantized before multiplication. + * The output tensor is written with bfloat16 or half precision. + * + * @param output Output tensor (must be of type bfloat16 or half). + * @param a Input tensor A (must be kFloat8_e4m3fn). + * @param b Input tensor B (must be kFloat8_e4m3fn). + * @param scales_a Scaling factors for tensor A, float32 per expert group. + * @param scales_b Scaling factors for tensor B, float32 per expert group. + * @param stride_a Stride information for tensor A (int32). + * @param stride_b Stride information for tensor B (int32). + * @param stride_c Stride information for output tensor C (int32). + * @param problem_sizes 2D int32 tensor of shape (num_experts, 3), specifying (M, N, K) + * for each grouped matrix multiplication problem. + * @param expert_offsets 1D int32 tensor of size (num_experts), used to index into + * the grouped input tensors for dispatch. + */ +void es_fp8_blockwise_scaled_grouped_mm( + torch::Tensor& output, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Tensor& stride_a, + const torch::Tensor& stride_b, + const torch::Tensor& stride_d, + const torch::Tensor& problem_sizes, + const torch::Tensor& expert_offsets, + const torch::Tensor& workspace) { +#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED) && defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_SUPPORTED) + TORCH_CHECK(problem_sizes.dim() == 2, "problem_sizes must be 2D tensor"); + TORCH_CHECK(problem_sizes.size(1) == 3, "problem_sizes must have shape (num_experts, 3)"); + TORCH_CHECK( + problem_sizes.size(0) == expert_offsets.size(0), "Number of experts in problem_sizes must match expert_offsets"); + TORCH_CHECK(problem_sizes.dtype() == torch::kInt32, "problem_sizes must be int32"); + TORCH_CHECK(a.scalar_type() == torch::kFloat8_e4m3fn, "a must be kFloat8_e4m3fn"); + TORCH_CHECK(b.scalar_type() == torch::kFloat8_e4m3fn, "b must be kFloat8_e4m3fn"); + TORCH_CHECK( + output.scalar_type() == torch::kBFloat16 || output.scalar_type() == torch::kHalf, + "output must be bfloat16 or half"); + + int num_experts = (int)problem_sizes.size(0); + torch::TensorOptions options_int64 = torch::TensorOptions().dtype(torch::kInt64).device(a.device()); + torch::TensorOptions options_int32 = torch::TensorOptions().dtype(torch::kInt32).device(a.device()); + torch::Tensor out_ptrs = torch::empty(num_experts, options_int64); + torch::Tensor a_ptrs = torch::empty(num_experts, options_int64); + torch::Tensor b_ptrs = torch::empty(num_experts, options_int64); + torch::Tensor a_scales_ptrs = torch::empty(num_experts, options_int64); + torch::Tensor b_scales_ptrs = torch::empty(num_experts, options_int64); + + torch::Tensor layout_sfa = torch::empty({num_experts, 5}, options_int32); + torch::Tensor layout_sfb = torch::empty({num_experts, 5}, options_int32); + + torch::Tensor lm_problem_sizes = torch::empty({num_experts, 3}, options_int32); + torch::Tensor mm_problem_sizes = torch::empty({num_experts, 3}, options_int32); + torch::Tensor hm_problem_sizes = torch::empty({num_experts, 3}, options_int32); + + const std::string H20_device_type_str("NVIDIA H20"); + bool is_h20_device = std::string(at::cuda::getCurrentDeviceProperties()->name) == H20_device_type_str; + at::cuda::CUDAGuard device_guard{(char)a.get_device()}; + cudaStream_t stream = at::cuda::getCurrentCUDAStream(a.get_device()); + + if (output.dtype() == torch::kBFloat16) { + expert_specialization::es_sm90_fp8_blockwise_scaled_group_mm_pre_compute( + out_ptrs, + a_ptrs, + b_ptrs, + a_scales_ptrs, + b_scales_ptrs, + layout_sfa, + layout_sfb, + lm_problem_sizes, + mm_problem_sizes, + hm_problem_sizes, + output, + a, + b, + scales_a, + scales_b, + problem_sizes, + expert_offsets, + is_h20_device, + stream); + } else if (output.dtype() == torch::kFloat16) { + expert_specialization::es_sm90_fp8_blockwise_scaled_group_mm_pre_compute( + out_ptrs, + a_ptrs, + b_ptrs, + a_scales_ptrs, + b_scales_ptrs, + layout_sfa, + layout_sfb, + lm_problem_sizes, + mm_problem_sizes, + hm_problem_sizes, + output, + a, + b, + scales_a, + scales_b, + problem_sizes, + expert_offsets, + is_h20_device, + stream); + } else { + TORCH_CHECK(false, "Invalid output type (must be float16 or bfloat16)"); + } + + if (output.dtype() == torch::kBFloat16) { + expert_specialization::es_sm90_fp8_blockwise_scaled_group_mm_distpatch_out_dtype( + out_ptrs, + a_ptrs, + b_ptrs, + a_scales_ptrs, + b_scales_ptrs, + stride_a, + stride_b, + stride_d, + layout_sfa, + layout_sfb, + lm_problem_sizes, + mm_problem_sizes, + hm_problem_sizes, + workspace, + is_h20_device, + stream); + } else if (output.dtype() == torch::kFloat16) { + expert_specialization::es_sm90_fp8_blockwise_scaled_group_mm_distpatch_out_dtype( + out_ptrs, + a_ptrs, + b_ptrs, + a_scales_ptrs, + b_scales_ptrs, + stride_a, + stride_b, + stride_d, + layout_sfa, + layout_sfb, + lm_problem_sizes, + mm_problem_sizes, + hm_problem_sizes, + workspace, + is_h20_device, + stream); + } else { + TORCH_CHECK(false, "Invalid output type (must be float16 or bfloat16)"); + } +#else + TORCH_CHECK_NOT_IMPLEMENTED( + can_implement, "No implemented fp8_blockwise_scaled_grouped_mm for current compute capability: ", sm_version); +#endif +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/expert_specialization/es_fp8_blockwise_functor.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/expert_specialization/es_fp8_blockwise_functor.cuh new file mode 100644 index 00000000..db7f430f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/expert_specialization/es_fp8_blockwise_functor.cuh @@ -0,0 +1,268 @@ +#pragma once +#include + +#include + +#include "cute/tensor.hpp" +#include "es_fp8_blockwise_traits.cuh" + +namespace expert_specialization { + +using namespace cute; + +template +struct Fp8BlockwiseGroupedGemmOffsetFunctor { + // Input + int* expert_offsets{nullptr}; + // Base pointers + ElementAB* a_base{nullptr}; + ElementAB* b_base{nullptr}; + ElementD* out_base{nullptr}; + ElementSF* a_scales_base{nullptr}; + ElementSF* b_scales_base{nullptr}; + + // Output + // Pointer Array for A/B + ElementAB** a_offsets{nullptr}; + ElementAB** b_offsets{nullptr}; + ElementSF** a_scales_offsets{nullptr}; + ElementSF** b_scales_offsets{nullptr}; + ElementD** out_offsets{nullptr}; + + Fp8BlockwiseGroupedGemmOffsetFunctor() = default; + Fp8BlockwiseGroupedGemmOffsetFunctor( + int* _expert_offsets, + ElementAB* _a_base, + ElementAB* _b_base, + ElementD* _out_base, + ElementSF* _a_scales_base, + ElementSF* _b_scales_base, + ElementAB** _a_offsets, + ElementAB** _b_offsets, + ElementSF** _a_scales_offsets, + ElementSF** _b_scales_offsets, + ElementD** _out_offsets) + : expert_offsets(_expert_offsets), + a_base(_a_base), + b_base(_b_base), + out_base(_out_base), + a_scales_base(_a_scales_base), + b_scales_base(_b_scales_base), + a_offsets(_a_offsets), + b_offsets(_b_offsets), + a_scales_offsets(_a_scales_offsets), + b_scales_offsets(_b_scales_offsets), + out_offsets(_out_offsets) {} + + void CUTE_DEVICE operator()(int64_t expert_id, int m, int n, int k) { + int64_t expert_offset = static_cast(expert_offsets[expert_id]); + int64_t a_stride = 0; + int64_t b_stride = 0; + int64_t a_scale_stride = 0; + int64_t b_scale_stride = 0; + + a_stride = expert_offset * k; + b_stride = expert_id * k * n; + a_scale_stride = expert_offset * k / 128; + b_scale_stride = expert_id * k * n / 128 / 128; + + a_offsets[expert_id] = a_base + a_stride; + b_offsets[expert_id] = b_base + b_stride; + a_scales_offsets[expert_id] = a_scales_base + a_scale_stride; + b_scales_offsets[expert_id] = b_scales_base + b_scale_stride; + out_offsets[expert_id] = out_base + expert_offset * n; + } +}; + +template +struct Fp8BlockwiseGroupedGemmSFLayoutFunctor { + using ScaleConfig = typename PerfConfig::ScaleConfig; + using LayoutSFA = typename PerfConfig::LayoutSFA; + using LayoutSFB = typename PerfConfig::LayoutSFB; + LayoutSFA* layout_sfa_base{nullptr}; + LayoutSFB* layout_sfb_base{nullptr}; + + Fp8BlockwiseGroupedGemmSFLayoutFunctor() = default; + Fp8BlockwiseGroupedGemmSFLayoutFunctor(LayoutSFA* _layout_sfa_base, LayoutSFB* _layout_sfb_base) + : layout_sfa_base(_layout_sfa_base), layout_sfb_base(_layout_sfb_base) {} + + void CUTE_DEVICE operator()(int64_t expert_id, int m, int n, int k) { + LayoutSFA* layout_sfa_ptr = layout_sfa_base + expert_id; + LayoutSFB* layout_sfb_ptr = layout_sfb_base + expert_id; + *layout_sfa_ptr = ScaleConfig::tile_atom_to_shape_SFA(cute::make_shape(m, n, k, 1)); + *layout_sfb_ptr = ScaleConfig::tile_atom_to_shape_SFB(cute::make_shape(m, n, k, 1)); + } +}; + +// [Unused]: Specialization for Swap A/B +template <> +struct Fp8BlockwiseGroupedGemmSFLayoutFunctor { + using ScaleConfig = typename PerfConfigLowMH20::ScaleConfig; + using LayoutSFA = typename PerfConfigLowMH20::LayoutSFA; + using LayoutSFB = typename PerfConfigLowMH20::LayoutSFB; + LayoutSFA* layout_sfa_base{nullptr}; + LayoutSFB* layout_sfb_base{nullptr}; + + Fp8BlockwiseGroupedGemmSFLayoutFunctor() = default; + Fp8BlockwiseGroupedGemmSFLayoutFunctor(LayoutSFA* _layout_sfa_base, LayoutSFB* _layout_sfb_base) + : layout_sfa_base(_layout_sfa_base), layout_sfb_base(_layout_sfb_base) {} + + void CUTE_DEVICE operator()(int64_t expert_id, int m, int n, int k) { + LayoutSFA* layout_sfa_ptr = layout_sfa_base + expert_id; + LayoutSFB* layout_sfb_ptr = layout_sfb_base + expert_id; + *layout_sfa_ptr = ScaleConfig::tile_atom_to_shape_SFA(cute::make_shape(n, m, k, 1)); + *layout_sfb_ptr = ScaleConfig::tile_atom_to_shape_SFB(cute::make_shape(n, m, k, 1)); + } +}; + +template +struct Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor; + +template <> +struct Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor { + int* problem_sizes{nullptr}; + + Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor() = default; + Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor(int* _problem_sizes) : problem_sizes(_problem_sizes) {} + + void CUTE_DEVICE operator()(int64_t expert_id, int m, int n, int k) { + if (m < 64) { + // Swap A/B + problem_sizes[expert_id * 3 + 0] = n; + problem_sizes[expert_id * 3 + 1] = m; + problem_sizes[expert_id * 3 + 2] = k; + } else { + problem_sizes[expert_id * 3 + 0] = 0; + problem_sizes[expert_id * 3 + 1] = 0; + problem_sizes[expert_id * 3 + 2] = 0; + } + } +}; + +template <> +struct Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor { + int* problem_sizes{nullptr}; + + Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor() = default; + Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor(int* _problem_sizes) : problem_sizes(_problem_sizes) {} + + void CUTE_DEVICE operator()(int64_t expert_id, int m, int n, int k) { + if (m <= 32) { + // Swap A/B + problem_sizes[expert_id * 3 + 0] = n; + problem_sizes[expert_id * 3 + 1] = m; + problem_sizes[expert_id * 3 + 2] = k; + } else { + problem_sizes[expert_id * 3 + 0] = 0; + problem_sizes[expert_id * 3 + 1] = 0; + problem_sizes[expert_id * 3 + 2] = 0; + } + } +}; + +template <> +struct Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor { + int* problem_sizes{nullptr}; + + Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor() = default; + Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor(int* _problem_sizes) : problem_sizes(_problem_sizes) {} + + void CUTE_DEVICE operator()(int64_t expert_id, int m, int n, int k) { + if (m >= 64 && m < 128) { + problem_sizes[expert_id * 3 + 0] = m; + problem_sizes[expert_id * 3 + 1] = n; + problem_sizes[expert_id * 3 + 2] = k; + } else { + problem_sizes[expert_id * 3 + 0] = 0; + problem_sizes[expert_id * 3 + 1] = 0; + problem_sizes[expert_id * 3 + 2] = 0; + } + } +}; + +template <> +struct Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor { + int* problem_sizes{nullptr}; + + Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor() = default; + Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor(int* _problem_sizes) : problem_sizes(_problem_sizes) {} + + void CUTE_DEVICE operator()(int64_t expert_id, int m, int n, int k) { + if (m > 32 && m <= 64) { + problem_sizes[expert_id * 3 + 0] = n; + problem_sizes[expert_id * 3 + 1] = m; + problem_sizes[expert_id * 3 + 2] = k; + } else { + problem_sizes[expert_id * 3 + 0] = 0; + problem_sizes[expert_id * 3 + 1] = 0; + problem_sizes[expert_id * 3 + 2] = 0; + } + } +}; + +template <> +struct Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor { + int* problem_sizes{nullptr}; + + Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor() = default; + Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor(int* _problem_sizes) : problem_sizes(_problem_sizes) {} + + void CUTE_DEVICE operator()(int64_t expert_id, int m, int n, int k) { + if (m >= 128) { + problem_sizes[expert_id * 3 + 0] = m; + problem_sizes[expert_id * 3 + 1] = n; + problem_sizes[expert_id * 3 + 2] = k; + } else { + problem_sizes[expert_id * 3 + 0] = 0; + problem_sizes[expert_id * 3 + 1] = 0; + problem_sizes[expert_id * 3 + 2] = 0; + } + } +}; + +template <> +struct Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor { + int* problem_sizes{nullptr}; + + Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor() = default; + Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor(int* _problem_sizes) : problem_sizes(_problem_sizes) {} + + void CUTE_DEVICE operator()(int64_t expert_id, int m, int n, int k) { + if (m > 64) { + problem_sizes[expert_id * 3 + 0] = m; + problem_sizes[expert_id * 3 + 1] = n; + problem_sizes[expert_id * 3 + 2] = k; + } else { + problem_sizes[expert_id * 3 + 0] = 0; + problem_sizes[expert_id * 3 + 1] = 0; + problem_sizes[expert_id * 3 + 2] = 0; + } + } +}; + +template < + typename OffsetFunctor, + typename ScaleLayoutFunctor, + typename LowMProblemSizeFilterFunctor, + typename MiddleMProblemSizeFilterFunctor, + typename HighMProblemSizeFilterFunctor> +__global__ void groupedGemmPreComputeKernel( + int* problem_sizes, + OffsetFunctor offset_functor, + ScaleLayoutFunctor sf_functor, + LowMProblemSizeFilterFunctor lm_psf_functor, + MiddleMProblemSizeFilterFunctor mm_psf_functor, + HighMProblemSizeFilterFunctor hm_psf_functor) { + int64_t expert_id = static_cast(threadIdx.x); + int m = problem_sizes[expert_id * 3 + 0]; + int n = problem_sizes[expert_id * 3 + 1]; + int k = problem_sizes[expert_id * 3 + 2]; + + offset_functor(expert_id, m, n, k); + sf_functor(expert_id, m, n, k); + lm_psf_functor(expert_id, m, n, k); + mm_psf_functor(expert_id, m, n, k); + hm_psf_functor(expert_id, m, n, k); +} + +} // namespace expert_specialization diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/expert_specialization/es_fp8_blockwise_launcher.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/expert_specialization/es_fp8_blockwise_launcher.cuh new file mode 100644 index 00000000..3ed98821 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/expert_specialization/es_fp8_blockwise_launcher.cuh @@ -0,0 +1,284 @@ +#pragma once +#include +#include +#include + +#include +#include +#include + +#include "cute/tensor.hpp" +#include "cutlass/cutlass.h" +#include "es_fp8_blockwise_functor.cuh" + +namespace expert_specialization { + +using namespace cute; + +template +void es_sm90_fp8_blockwise_scaled_group_mm_pre_compute( + // Output + torch::Tensor& out_ptrs, + torch::Tensor& a_ptrs, + torch::Tensor& b_ptrs, + torch::Tensor& a_scales_ptrs, + torch::Tensor& b_scales_ptrs, + torch::Tensor& layout_sfa, + torch::Tensor& layout_sfb, + torch::Tensor& lm_problem_sizes, + torch::Tensor& mm_problem_sizes, + torch::Tensor& hm_problem_sizes, + // Input + torch::Tensor& out_tensors, + torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + torch::Tensor const& problem_sizes, + torch::Tensor const& expert_offsets, + bool is_h20_device, + cudaStream_t stream) { + TORCH_CHECK(a_tensors.dtype() == torch::kFloat8_e4m3fn); + TORCH_CHECK(b_tensors.dtype() == torch::kFloat8_e4m3fn); + TORCH_CHECK(a_scales.dtype() == torch::kFloat32); + TORCH_CHECK(b_scales.dtype() == torch::kFloat32); + + // Creat Scale Factor Layout Functor + using LayoutSFA = typename PerfConfigMiddleMH20::LayoutSFA; + using LayoutSFB = typename PerfConfigMiddleMH20::LayoutSFB; + struct Fp8BlockwiseGroupedGemmSFLayoutFunctor sf_layout( + reinterpret_cast(layout_sfa.data_ptr()), reinterpret_cast(layout_sfb.data_ptr())); + + int num_experts = (int)expert_offsets.size(0); + TORCH_CHECK(num_experts <= 1024, "Expert more than 1024"); // Max threads per block is 1024 + + struct Fp8BlockwiseGroupedGemmOffsetFunctor of( + static_cast(expert_offsets.data_ptr()), + static_cast(a_tensors.data_ptr()), + static_cast(b_tensors.data_ptr()), + static_cast(out_tensors.data_ptr()), + static_cast(a_scales.data_ptr()), + static_cast(b_scales.data_ptr()), + static_cast(a_ptrs.data_ptr()), + static_cast(b_ptrs.data_ptr()), + static_cast(a_scales_ptrs.data_ptr()), + static_cast(b_scales_ptrs.data_ptr()), + static_cast(out_ptrs.data_ptr())); + if (!is_h20_device) { + struct Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor lm_psf( + static_cast(lm_problem_sizes.data_ptr())); + struct Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor mm_psf( + static_cast(mm_problem_sizes.data_ptr())); + struct Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor hm_psf( + static_cast(hm_problem_sizes.data_ptr())); + groupedGemmPreComputeKernel<<<1, num_experts, 0, stream>>>( + static_cast(problem_sizes.data_ptr()), of, sf_layout, lm_psf, mm_psf, hm_psf); + } else { + struct Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor lm_psf( + static_cast(lm_problem_sizes.data_ptr())); + struct Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor mm_psf( + static_cast(mm_problem_sizes.data_ptr())); + struct Fp8BlockwiseGroupedGemmProblemSizeFilterFunctor hm_psf( + static_cast(hm_problem_sizes.data_ptr())); + groupedGemmPreComputeKernel<<<1, num_experts, 0, stream>>>( + static_cast(problem_sizes.data_ptr()), of, sf_layout, lm_psf, mm_psf, hm_psf); + } +} + +template +void launch_sm90_fp8_blockwise_scaled_group_mm( + torch::Tensor& out_ptrs, + const torch::Tensor& a_ptrs, + const torch::Tensor& b_ptrs, + const torch::Tensor& a_scales_ptrs, + const torch::Tensor& b_scales_ptrs, + const torch::Tensor& stride_a, + const torch::Tensor& stride_b, + const torch::Tensor& stride_d, + const torch::Tensor& layout_sfa, + const torch::Tensor& layout_sfb, + const torch::Tensor& problem_sizes, + const torch::Tensor& workspace, + cudaStream_t stream) { + using ElementA = typename GemmTraits::ElementA; + using StrideA = typename GemmTraits::StrideA; + using ElementB = typename GemmTraits::ElementB; + using StrideB = typename GemmTraits::StrideB; + using ElementAccumulator = typename GemmTraits::ElementAccumulator; + using LayoutSFA = typename GemmTraits::LayoutSFA; + using LayoutSFB = typename GemmTraits::LayoutSFB; + using ElementD = typename GemmTraits::ElementD; + using StrideD = typename GemmTraits::StrideD; + using UnderlyingProblemShape = typename GemmTraits::ProblemShape::UnderlyingProblemShape; + using Gemm = typename GemmTraits::Gemm; + using GemmKernel = typename GemmTraits::GemmKernel; + + int num_experts = (int)problem_sizes.size(0); + Gemm gemm_op; + + typename GemmKernel::MainloopArguments mainloop_args{ + static_cast(a_ptrs.data_ptr()), + static_cast(stride_a.data_ptr()), + static_cast(b_ptrs.data_ptr()), + static_cast(stride_b.data_ptr()), + static_cast(a_scales_ptrs.data_ptr()), + reinterpret_cast(layout_sfa.data_ptr()), + static_cast(b_scales_ptrs.data_ptr()), + reinterpret_cast(layout_sfb.data_ptr())}; + + cutlass::KernelHardwareInfo hw_info; + hw_info.device_id = c10::cuda::current_device(); + hw_info.sm_count = at::cuda::getCurrentDeviceProperties()->multiProcessorCount; + + typename GemmKernel::EpilogueArguments epilogue_args{ + {}, nullptr, nullptr, static_cast(out_ptrs.data_ptr()), static_cast(stride_d.data_ptr())}; + + UnderlyingProblemShape* problem_sizes_as_shapes = static_cast(problem_sizes.data_ptr()); + typename GemmKernel::Arguments args{ + cutlass::gemm::GemmUniversalMode::kGrouped, + {num_experts, problem_sizes_as_shapes, nullptr}, + mainloop_args, + epilogue_args, + hw_info}; + + auto can_implement_status = gemm_op.can_implement(args); + TORCH_CHECK(can_implement_status == cutlass::Status::kSuccess, "Failed to implement GEMM"); + + auto status = gemm_op.initialize(args, workspace.data_ptr(), stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, "Failed to initialize GEMM"); + + status = gemm_op.run(stream, nullptr, true); // Enable PDL + TORCH_CHECK(status == cutlass::Status::kSuccess, "Failed to run GEMM"); +} + +template +void es_sm90_fp8_blockwise_scaled_group_mm_distpatch_out_dtype( + torch::Tensor& out_ptrs, + const torch::Tensor& a_ptrs, + const torch::Tensor& b_ptrs, + const torch::Tensor& a_scales_ptrs, + const torch::Tensor& b_scales_ptrs, + const torch::Tensor& stride_a, + const torch::Tensor& stride_b, + const torch::Tensor& stride_d, + const torch::Tensor& layout_sfa, + const torch::Tensor& layout_sfb, + const torch::Tensor& lm_problem_sizes, + const torch::Tensor& mm_problem_sizes, + const torch::Tensor& hm_problem_sizes, + const torch::Tensor& workspace, + bool is_h20_device, + cudaStream_t stream) { + using LowMGemmH20Traits = + ExpertSpecializationSm90FP8BlockwiseGroupedGemmTraits; + using LowMGemmHx00Traits = + ExpertSpecializationSm90FP8BlockwiseGroupedGemmTraits; + using MiddleMGemmH20Traits = + ExpertSpecializationSm90FP8BlockwiseGroupedGemmTraits; + using MiddleMGemmHx00Traits = ExpertSpecializationSm90FP8BlockwiseGroupedGemmTraits< + OutType, + cutlass::layout::ColumnMajor, + PerfConfigMiddleMHx00>; + using HighMGemmH20Traits = + ExpertSpecializationSm90FP8BlockwiseGroupedGemmTraits; + using HighMGemmHx00Traits = + ExpertSpecializationSm90FP8BlockwiseGroupedGemmTraits; + + if (!is_h20_device) { + launch_sm90_fp8_blockwise_scaled_group_mm( + out_ptrs, + b_ptrs, + a_ptrs, + b_scales_ptrs, + a_scales_ptrs, + stride_b, + stride_a, + stride_d, + layout_sfb, + layout_sfa, + lm_problem_sizes, + workspace, + stream); + } else { + launch_sm90_fp8_blockwise_scaled_group_mm( + out_ptrs, + b_ptrs, + a_ptrs, + b_scales_ptrs, + a_scales_ptrs, + stride_b, + stride_a, + stride_d, + layout_sfb, + layout_sfa, + lm_problem_sizes, + workspace, + stream); + } + + if (!is_h20_device) { + launch_sm90_fp8_blockwise_scaled_group_mm( + out_ptrs, + b_ptrs, + a_ptrs, + b_scales_ptrs, + a_scales_ptrs, + stride_b, + stride_a, + stride_d, + layout_sfb, + layout_sfa, + mm_problem_sizes, + workspace, + stream); + } else { + launch_sm90_fp8_blockwise_scaled_group_mm( + out_ptrs, + a_ptrs, + b_ptrs, + a_scales_ptrs, + b_scales_ptrs, + stride_a, + stride_b, + stride_d, + layout_sfa, + layout_sfb, + mm_problem_sizes, + workspace, + stream); + } + + if (!is_h20_device) { + launch_sm90_fp8_blockwise_scaled_group_mm( + out_ptrs, + a_ptrs, + b_ptrs, + a_scales_ptrs, + b_scales_ptrs, + stride_a, + stride_b, + stride_d, + layout_sfa, + layout_sfb, + hm_problem_sizes, + workspace, + stream); + } else { + launch_sm90_fp8_blockwise_scaled_group_mm( + out_ptrs, + a_ptrs, + b_ptrs, + a_scales_ptrs, + b_scales_ptrs, + stride_a, + stride_b, + stride_d, + layout_sfa, + layout_sfb, + hm_problem_sizes, + workspace, + stream); + } +} + +} // namespace expert_specialization diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/expert_specialization/es_fp8_blockwise_traits.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/expert_specialization/es_fp8_blockwise_traits.cuh new file mode 100644 index 00000000..3bc7d929 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/expert_specialization/es_fp8_blockwise_traits.cuh @@ -0,0 +1,174 @@ +#pragma once + +// Misc +#include "cute/tensor.hpp" +#include "cutlass/arch/arch.h" +#include "cutlass/arch/mma.h" +#include "cutlass/cutlass.h" +#include "cutlass/detail/blockwise_scale_layout.hpp" +#include "cutlass/epilogue/dispatch_policy.hpp" +#include "cutlass/gemm/dispatch_policy.hpp" +#include "cutlass/gemm/group_array_problem_shape.hpp" +#include "cutlass/layout/layout.h" +#include "cutlass/numeric_conversion.h" +#include "cutlass/numeric_size.h" + +// Collective Builder +#include "cutlass/epilogue/collective/collective_builder.hpp" +#include "cutlass/epilogue/fusion/sm90_callbacks_tma_warpspecialized.hpp" +#include "cutlass/epilogue/thread/activation.h" +#include "cutlass/gemm/collective/collective_builder.hpp" + +// Integration +#include "cutlass/gemm/device/gemm_universal_adapter.h" +#include "cutlass/gemm/kernel/gemm_universal.hpp" + +namespace expert_specialization { + +using namespace cute; + +struct PerfConfigLowMH20 { + // Swap A/B + using ElementA = cutlass::float_e4m3_t; + using MmaTileShape = Shape<_128, _32, _128>; + using ClusterShape = Shape<_2, _1, _1>; + using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedPingpongFP8Blockwise; + using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecializedPingpong; + using ScaleConfig = + cutlass::detail::Sm90BlockwiseScaleConfig<128, 1, 128, cute::GMMA::Major::K, cute::GMMA::Major::K>; + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); +}; + +struct PerfConfigLowMHx00 { + // Swap A/B + using ElementA = cutlass::float_e4m3_t; + using MmaTileShape = Shape<_256, _32, _128>; + using ClusterShape = Shape<_2, _1, _1>; + using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperativeFP8Blockwise; + using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecializedCooperative; + using ScaleConfig = + cutlass::detail::Sm90BlockwiseScaleConfig<128, 1, 128, cute::GMMA::Major::K, cute::GMMA::Major::K>; + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); +}; + +struct PerfConfigMiddleMH20 { + using ElementA = cutlass::float_e4m3_t; + using MmaTileShape = Shape<_64, _128, _128>; + using ClusterShape = Shape<_1, _2, _1>; + using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedPingpongFP8Blockwise; + using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecializedPingpong; + using ScaleConfig = + cutlass::detail::Sm90BlockwiseScaleConfig<1, 128, 128, cute::GMMA::Major::K, cute::GMMA::Major::K>; + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); +}; + +struct PerfConfigMiddleMHx00 { + using ElementA = cutlass::float_e4m3_t; + using MmaTileShape = Shape<_256, _64, _128>; + using ClusterShape = Shape<_2, _1, _1>; + using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperativeFP8Blockwise; + using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecializedCooperative; + using ScaleConfig = + cutlass::detail::Sm90BlockwiseScaleConfig<128, 1, 128, cute::GMMA::Major::K, cute::GMMA::Major::K>; + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); +}; + +struct PerfConfigHighMH20 { + using ElementA = cutlass::float_e4m3_t; + using MmaTileShape = Shape<_64, _128, _128>; + using ClusterShape = Shape<_2, _1, _1>; + using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedPingpongFP8Blockwise; + using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecializedPingpong; + using ScaleConfig = + cutlass::detail::Sm90BlockwiseScaleConfig<1, 128, 128, cute::GMMA::Major::K, cute::GMMA::Major::K>; + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); +}; + +struct PerfConfigHighMHx00 { + using ElementA = cutlass::float_e4m3_t; + using MmaTileShape = Shape<_128, _128, _128>; + using ClusterShape = Shape<_1, _2, _1>; + using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperativeFP8Blockwise; + using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecializedCooperative; + using ScaleConfig = + cutlass::detail::Sm90BlockwiseScaleConfig<1, 128, 128, cute::GMMA::Major::K, cute::GMMA::Major::K>; + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); +}; + +template +struct ExpertSpecializationSm90FP8BlockwiseGroupedGemmTraits { + using ElementA = cutlass::float_e4m3_t; + using ElementB = cutlass::float_e4m3_t; + using ElementC = void; + using ElementD = OutType; + using ElementAccumulator = float; + using LayoutA = cutlass::layout::RowMajor; + using LayoutB = cutlass::layout::ColumnMajor; + using LayoutC = LayoutD; + using LayoutSFA = typename PerfConfig::LayoutSFA; + using LayoutSFB = typename PerfConfig::LayoutSFB; + using ProblemShape = cutlass::gemm::GroupProblemShape>; + + static constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentD = 128 / cutlass::sizeof_bits::value; + + using ArchTag = cutlass::arch::Sm90; + using OperatorClass = cutlass::arch::OpClassTensorOp; + static constexpr auto RoundStyle = cutlass::FloatRoundStyle::round_to_nearest; + using CustomEVTIdentity = // acc + cutlass::epilogue::fusion::Sm90EVT< + cutlass::epilogue::fusion:: + Sm90Compute, + cutlass::epilogue::fusion::Sm90AccFetch>; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + typename PerfConfig::MmaTileShape, + typename PerfConfig::ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementAccumulator, + ElementC, // Use void to avoid load Matrix C + LayoutC*, + AlignmentC, + ElementD, + LayoutD*, + AlignmentD, + typename PerfConfig::EpilogueSchedule, + CustomEVTIdentity>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementA, + cute::tuple, + AlignmentA, + ElementB, + cute::tuple, + AlignmentB, + ElementAccumulator, + typename PerfConfig::MmaTileShape, + typename PerfConfig::ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + typename PerfConfig::KernelSchedule>::CollectiveOp; + + using GemmKernel = cutlass::gemm::kernel::GemmUniversal; + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + using UnderlyingProblemShape = ProblemShape::UnderlyingProblemShape; + using StrideA = typename Gemm::GemmKernel::InternalStrideA; + using StrideB = typename Gemm::GemmKernel::InternalStrideB; + using StrideC = typename Gemm::GemmKernel::InternalStrideC; + using StrideD = typename Gemm::GemmKernel::InternalStrideD; +}; + +} // namespace expert_specialization diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/flash_extension.cc b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/flash_extension.cc new file mode 100644 index 00000000..df6024df --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/flash_extension.cc @@ -0,0 +1,66 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include +#include + +#include "sgl_flash_kernel_ops.h" + +TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) { + /* + * From flash-attention + */ + m.def( + "fwd(Tensor q," // (b, s_q, h, d) or (total_q, h, d) if there is cu_seqlens_q + " Tensor k," // (b_k, s_k, h_k, d) or (total_k, h_k, d) or paged + " Tensor v," // (b_k, s_k, h_k, dv) or (total_k, h_k, dv) or paged + " Tensor? k_new," // (b, s_k_new, h_k, d) or (total_k_new, h_k, d) + " Tensor? v_new," // (b, s_k_new, h_k, dv) or (total_k_new, h_k, dv) + " Tensor? q_v," // (b, s_q, h, dv) or (total_q_new, h, dv) + " Tensor? out," // (b, s_q, h, dv) or (total_q, h, dv) + " Tensor? cu_seqlens_q," // b+1 + " Tensor? cu_seqlens_k," // b+1 + " Tensor? cu_seqlens_k_new," // b+1 + " Tensor? seqused_q," // b + " Tensor? seqused_k," // b + " int? max_seqlen_q," + " int? max_seqlen_k," // TODO: check if needed + " Tensor? page_table," // (b_k, max_num_pages_per_seq) + " Tensor? kv_batch_idx," // b + " Tensor? leftpad_k," // b + " Tensor? rotary_cos," // seqlen_ro x (rotary_dim / 2) + " Tensor? rotary_sin," // seqlen_ro x (rotary_dim / 2) + " Tensor? seqlens_rotary," // b + " Tensor? q_descale," // (b, h_k) + " Tensor? k_descale," // (b, h_k) + " Tensor? v_descale," // (b, h_k) + " float? softmax_scale," // now optional + " bool is_causal," + " int window_size_left," + " int window_size_right," + " int attention_chunk," // NEW + " float softcap," // promoted to double in C++; schema float is fine + " bool is_rotary_interleaved," + " Tensor? scheduler_metadata," // (b + 1) + " int num_splits," + " bool? pack_gqa," + " int sm_margin," + " Tensor? sinks" + ") -> (Tensor, Tensor, Tensor, Tensor)"); // NEW return type: tuple of 4 tensors + + m.impl("fwd", torch::kCUDA, make_pytorch_shim(&mha_fwd)); +} + +REGISTER_EXTENSION(flash_ops) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/flashmla_extension.cc b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/flashmla_extension.cc new file mode 100644 index 00000000..8236c886 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/flashmla_extension.cc @@ -0,0 +1,55 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "sgl_kernel_ops.h" + +TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) { + /* + * From FlashMLA + */ + m.def( + "get_mla_decoding_metadata(Tensor seqlens_k, int num_q_tokens_per_head_k, int h_k, int? h_q, bool " + "is_fp8_kvcache, int? topk) -> Tensor[]"); + m.impl("get_mla_decoding_metadata", torch::kCUDA, &get_mla_decoding_metadata); + + m.def("get_mla_decoding_metadata_dense_fp8(Tensor seqlens_k, int num_heads_per_head_k, int num_heads_k) -> Tensor[]"); + m.impl("get_mla_decoding_metadata_dense_fp8", torch::kCUDA, &get_mla_decoding_metadata_dense_fp8); + + m.def( + "fwd_kvcache_mla(Tensor q, Tensor kv_cache, int head_size_v, Tensor seqlens_k, Tensor block_table, float " + "softmax_scale, bool is_causal, Tensor tile_scheduler_metadata, Tensor num_splits, bool is_fp8, Tensor? indices) " + "-> Tensor[]"); + m.impl("fwd_kvcache_mla", torch::kCUDA, &fwd_kvcache_mla); + + m.def( + "dense_prefill_fwd(Tensor workspace_buffer, Tensor q, Tensor k, Tensor v, Tensor cumulative_seqlen_q, Tensor " + "cumulative_seqlen_kv, Tensor o, Tensor lse, int mask_mode_code, float softmax_scale, int max_seqlen_q, int " + "max_seqlen_kv, bool is_varlen) -> ()"); + m.impl("dense_prefill_fwd", torch::kCUDA, &FMHACutlassSM100FwdRun); + + m.def("sparse_prefill_fwd(Tensor q, Tensor kv, Tensor indices, float sm_scale, int d_v) -> Tensor[]"); + m.impl("sparse_prefill_fwd", torch::kCUDA, &sparse_prefill_fwd); + + m.def( + "fwd_kvcache_mla_fp8(Tensor q, Tensor kcache, int head_size_v, Tensor seqlens_k, Tensor block_table, float " + "softmax_scale, bool is_causal, Tensor tile_scheduler_metadata, Tensor num_splits, Tensor? descale_q, Tensor? " + "descale_k) -> Tensor[]"); + m.impl("fwd_kvcache_mla_fp8", torch::kCUDA, &fwd_kvcache_mla_fp8); +} + +REGISTER_EXTENSION(flashmla_ops) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/awq_kernel.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/awq_kernel.cu new file mode 100644 index 00000000..eec93368 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/awq_kernel.cu @@ -0,0 +1,221 @@ +// Adapted from +// https://github.com/vllm-project/vllm/blob/eb59b5a6cba6727d3727c0372258db9002f687c1/csrc/quantization/awq/gemm_kernels.cu#L350 +#include +#include +#include +#include +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 +#include +#endif + +template +__device__ inline int lop3(int a, int b, int c) { + int res; + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n" : "=r"(res) : "r"(a), "r"(b), "r"(c), "n"(lut)); + return res; +} + +__device__ uint4 dequantize_s4_to_fp16x2(uint32_t const& source) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 750 + uint4 result; + + uint32_t* h = reinterpret_cast(&result); + uint32_t const i4s = reinterpret_cast(source); + + // First, we extract the i4s and construct an intermediate fp16 number. + static constexpr uint32_t immLut = (0xf0 & 0xcc) | 0xaa; + static constexpr uint32_t BOTTOM_MASK = 0x000f000f; + static constexpr uint32_t TOP_MASK = 0x00f000f0; + static constexpr uint32_t I4s_TO_F16s_MAGIC_NUM = 0x64006400; + + // Note that the entire sequence only requires 1 shift instruction. This is + // thanks to the register packing format and the fact that we force our + // integers to be unsigned, and account for this in the fp16 subtractions. In + // addition, I exploit the fact that sub and fma have the same throughput in + // order to convert elt_23 and elt_67 to fp16 without having to shift them to + // the bottom bits before hand. + + // Shift right by 8 to now consider elt_45 and elt_67. Issue first to hide RAW + // dependency if we issue immediately before required. + const uint32_t top_i4s = i4s >> 8; + // Extract elt_01 - (i4s & 0x000f000f) | 0x64006400 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n" + : "=r"(h[0]) + : "r"(i4s), "n"(BOTTOM_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut)); + // Extract elt_23 (i4s & 0x00f000f0) | 0x64006400 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n" + : "=r"(h[1]) + : "r"(i4s), "n"(TOP_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut)); + // Extract elt_45 (top_i4s & 0x000f000f) | 0x64006400 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n" + : "=r"(h[2]) + : "r"(top_i4s), "n"(BOTTOM_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut)); + // Extract elt_67 (top_i4s & 0x00f000f0) | 0x64006400 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n" + : "=r"(h[3]) + : "r"(top_i4s), "n"(TOP_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut)); + + // This is the half2 {1024, 1024} represented as an integer. + static constexpr uint32_t FP16_TOP_MAGIC_NUM = 0x64006400; + // This is the half2 {1 / 16, 1 / 16} represented as an integer. + static constexpr uint32_t ONE_SIXTEENTH = 0x2c002c00; + // This is the half2 {-64, -64} represented as an integer. + static constexpr uint32_t NEG_64 = 0xd400d400; + + // Finally, we construct the output numbers. + // Convert elt_01 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(h[0]) : "r"(h[0]), "r"(FP16_TOP_MAGIC_NUM)); + // Convert elt_23 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(h[1]) : "r"(h[1]), "r"(ONE_SIXTEENTH), "r"(NEG_64)); + // Convert elt_45 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(h[2]) : "r"(h[2]), "r"(FP16_TOP_MAGIC_NUM)); + // Convert elt_67 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(h[3]) : "r"(h[3]), "r"(ONE_SIXTEENTH), "r"(NEG_64)); + + return result; +#else + assert(false); + return {}; +#endif +} + +__device__ uint4 dequantize_s4_to_bf16x2(uint32_t const& source) { +#if CUDA_VERSION >= 12000 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 + uint4 result; + uint32_t* h = reinterpret_cast(&result); + uint32_t const i4s = source; + + // Define masks and constants + static constexpr uint32_t MASK = 0x000f000f; + static constexpr uint32_t EX = 0x43004300; + static constexpr uint32_t MUL = 0x3F803F80; + static constexpr uint32_t ADD = 0xC300C300; + + int lo0 = lop3<(0xf0 & 0xcc) | 0xaa>(i4s, MASK, EX); + int hi0 = lop3<(0xf0 & 0xcc) | 0xaa>(i4s >> 4, MASK, EX); + int lo1 = lop3<(0xf0 & 0xcc) | 0xaa>(i4s >> 8, MASK, EX); + int hi1 = lop3<(0xf0 & 0xcc) | 0xaa>(i4s >> 12, MASK, EX); + + nv_bfloat162* res = reinterpret_cast(h); + res[0] = __hfma2( + *reinterpret_cast(&lo0), + *reinterpret_cast(&MUL), + *reinterpret_cast(&ADD)); + res[1] = __hfma2( + *reinterpret_cast(&hi0), + *reinterpret_cast(&MUL), + *reinterpret_cast(&ADD)); + res[2] = __hfma2( + *reinterpret_cast(&lo1), + *reinterpret_cast(&MUL), + *reinterpret_cast(&ADD)); + res[3] = __hfma2( + *reinterpret_cast(&hi1), + *reinterpret_cast(&MUL), + *reinterpret_cast(&ADD)); + + return result; +#else + assert(false); + return {}; +#endif +#endif +} + +template +__global__ void __launch_bounds__(256) dequantize_weights( + int* __restrict__ qweight, + OutputT* __restrict__ scales, + int* __restrict__ qzeros, + OutputT* __restrict__ output, + int group_size, + int qweight_cols, + int qweight_rows) { +#if CUDA_VERSION >= 12000 + int col = blockIdx.x * blockDim.x + threadIdx.x; + int row = blockIdx.y * blockDim.y + threadIdx.y; + if (col >= qweight_cols || row >= qweight_rows) return; + + int group_idx = row / group_size; + int scale_offset = 8 * col + group_idx * qweight_cols * 8; + uint4 loaded_scale = *(uint4*)(scales + scale_offset); + + // Handle different data types + if constexpr (std::is_same::value) { + // FP16 path + uint4 zeros = dequantize_s4_to_fp16x2(qzeros[col + group_idx * qweight_cols]); + uint4 weight_fp16 = dequantize_s4_to_fp16x2(qweight[col + row * qweight_cols]); + + // Use PTX assembly for FP16 operations + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.x) : "r"(weight_fp16.x), "r"(zeros.x)); + asm volatile("mul.rn.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.x) : "r"(weight_fp16.x), "r"(loaded_scale.x)); + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.y) : "r"(weight_fp16.y), "r"(zeros.y)); + asm volatile("mul.rn.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.y) : "r"(weight_fp16.y), "r"(loaded_scale.y)); + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.z) : "r"(weight_fp16.z), "r"(zeros.z)); + asm volatile("mul.rn.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.z) : "r"(weight_fp16.z), "r"(loaded_scale.z)); + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.w) : "r"(weight_fp16.w), "r"(zeros.w)); + asm volatile("mul.rn.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.w) : "r"(weight_fp16.w), "r"(loaded_scale.w)); + + OutputT* output_ptr = output + 8 * col + 8 * row * qweight_cols; + *(uint4*)output_ptr = weight_fp16; + } else if constexpr (std::is_same::value) { + uint4 weight_raw = dequantize_s4_to_bf16x2(qweight[col + row * qweight_cols]); + uint4 zero_raw = dequantize_s4_to_bf16x2(qzeros[col + group_idx * qweight_cols]); + uint4 scale_raw = *reinterpret_cast(scales + scale_offset); + + // Vectorized processing (each uint4 contains 4 nv_bfloat162) + nv_bfloat162* weight_vec = reinterpret_cast(&weight_raw); + nv_bfloat162* zero_vec = reinterpret_cast(&zero_raw); + nv_bfloat162* scale_vec = reinterpret_cast(&scale_raw); + +// Single instruction dual-channel operation +#pragma unroll + for (int i = 0; i < 4; ++i) { // uint4 = 4 * nv_bfloat162 + weight_vec[i] = __hmul2(__hsub2(weight_vec[i], zero_vec[i]), scale_vec[i]); + } + + // Directly store to OutputT array (guaranteed contiguous memory) + OutputT* output_ptr = output + 8 * col + row * qweight_cols * 8; + static_assert(sizeof(uint4) == 8 * sizeof(OutputT), "Memory layout mismatch"); + *reinterpret_cast(output_ptr) = weight_raw; + } +#endif +} + +torch::Tensor awq_dequantize(torch::Tensor qweight, torch::Tensor scales, torch::Tensor qzeros) { + int qweight_rows = qweight.size(0); + int qweight_cols = qweight.size(1); + int group_size = qweight_rows / scales.size(0); + + int x_num_threads = 16; + int y_num_threads = 16; + int x_blocks = (qweight_cols + x_num_threads - 1) / x_num_threads; + int y_blocks = (qweight_rows + y_num_threads - 1) / y_num_threads; + + const at::cuda::OptionalCUDAGuard device_guard(device_of(qweight)); + + auto output_tensor_options = torch::TensorOptions().dtype(scales.dtype()).device(scales.device()); + at::Tensor output = torch::empty({qweight_rows, qweight_cols * 8}, output_tensor_options); + + auto _qweight = reinterpret_cast(qweight.data_ptr()); + auto _zeros = reinterpret_cast(qzeros.data_ptr()); + + dim3 num_blocks(x_blocks, y_blocks); + dim3 threads_per_block(x_num_threads, y_num_threads); + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + if (scales.scalar_type() == at::ScalarType::Half) { + auto _scales = reinterpret_cast(scales.data_ptr()); + auto _output = reinterpret_cast(output.data_ptr()); + dequantize_weights<<>>( + _qweight, _scales, _zeros, _output, group_size, qweight_cols, qweight_rows); + } else { + auto _scales = reinterpret_cast<__nv_bfloat16*>(scales.data_ptr()); + auto _output = reinterpret_cast<__nv_bfloat16*>(output.data_ptr()); + dequantize_weights<__nv_bfloat16><<>>( + _qweight, _scales, _zeros, _output, group_size, qweight_cols, qweight_rows); + } + + return output; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/bmm_fp8.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/bmm_fp8.cu new file mode 100644 index 00000000..cef85a7d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/bmm_fp8.cu @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2024 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +#include "pytorch_extension_utils.h" + +void bmm_fp8( + at::Tensor A, + at::Tensor B, + at::Tensor D, + at::Tensor A_scale, + at::Tensor B_scale, + at::Tensor workspace_buffer, + int64_t cublas_handle) { + TORCH_CHECK(A.is_cuda(), "A must be a CUDA tensor"); + TORCH_CHECK(B.is_cuda(), "B must be a CUDA tensor"); + TORCH_CHECK(D.is_cuda(), "D must be a CUDA tensor"); + TORCH_CHECK(A.dim() == 3, "Expected 3D tensor for A"); + TORCH_CHECK(B.dim() == 3, "Expected 3D tensor for B"); + TORCH_CHECK(D.dim() == 3, "Expected 3D tensor for D"); + TORCH_CHECK(A.size(0) == B.size(0) && A.size(0) == D.size(0), "Batch sizes must match"); + TORCH_CHECK(A.size(2) == B.size(1), "Incompatible matrix sizes"); + TORCH_CHECK(A.size(1) == D.size(1) && B.size(2) == D.size(2), "Result tensor has incorrect shape"); + + // PyTorch is row major by default. cuBLASLt is column major by default. + // We need row major D as expected. + // A ^ T * B = D, so D ^ T = B ^ T * A + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP8(B.scalar_type(), b_type, [&] { + return DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP8(A.scalar_type(), a_type, [&] { + return DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP16(D.scalar_type(), d_type, [&] { + auto batch_size = A.size(0); + auto m = A.size(1); + auto k = A.size(2); + auto n = B.size(2); + + auto lt_handle = reinterpret_cast(cublas_handle); + auto stream = at::cuda::getCurrentCUDAStream(); + + auto status = flashinfer::bmm_fp8::bmm_fp8_internal_cublaslt( + workspace_buffer.data_ptr(), + workspace_buffer.numel(), + static_cast(B.data_ptr()), + static_cast(A.data_ptr()), + static_cast(D.data_ptr()), + batch_size, + n, + m, + k, + static_cast(B_scale.data_ptr()), + static_cast(A_scale.data_ptr()), + lt_handle, + stream); + TORCH_CHECK( + status == CUBLAS_STATUS_SUCCESS, "bmm_fp8_internal_cublaslt failed: ", cublasGetStatusString(status)); + return true; + }); + }); + }); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/dsv3_fused_a_gemm.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/dsv3_fused_a_gemm.cu new file mode 100644 index 00000000..37aff1b9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/dsv3_fused_a_gemm.cu @@ -0,0 +1,673 @@ +/* + * Adapted from + * https://github.com/NVIDIA/TensorRT-LLM/blob/619709fc33bd5dc268f19d6a741fe7ed51c0f8f5/cpp/tensorrt_llm/kernels/dsv3MinLatencyKernels/dsv3FusedAGemm.cu + * + * Copyright (c) 2019-2024, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2021, NAVER Corp. Authored by CLOVA. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include "utils.h" + +using bf16_t = __nv_bfloat16; + +__device__ void hmma_16_8_16_f32acc_bf16ab( + float (&d_reg)[4], const bf16_t (&a_reg)[8], const bf16_t (&b_reg)[4], float const (&c_reg)[4]) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + uint32_t a0 = *reinterpret_cast(a_reg + 0); + uint32_t a1 = *reinterpret_cast(a_reg + 2); + uint32_t a2 = *reinterpret_cast(a_reg + 4); + uint32_t a3 = *reinterpret_cast(a_reg + 6); + uint32_t b0 = *reinterpret_cast(b_reg + 0); + uint32_t b1 = *reinterpret_cast(b_reg + 2); + asm volatile( + "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " + "{%0, %1, %2, %3}," + "{%4, %5, %6, %7}," + "{%8, %9}," + "{%10, %11, %12, %13};\n" + : "=f"(d_reg[0]), "=f"(d_reg[1]), "=f"(d_reg[2]), "=f"(d_reg[3]) + : "r"(a0), + "r"(a1), + "r"(a2), + "r"(a3), + "r"(b0), + "r"(b1), + "f"(d_reg[0]), + "f"(d_reg[1]), + "f"(d_reg[2]), + "f"(d_reg[3])); +#endif +} + +extern "C" { +__device__ uint32_t __nvvm_get_smem_pointer(void*); +} + +__device__ void ldgsts_128(void const* gPtr, void* sPtr, uint32_t pred) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + if (pred) { + uint32_t smemPtrAsUint32 = __nvvm_get_smem_pointer(sPtr); + asm volatile("cp.async.cg.shared.global.L2::128B [%0], [%1], %2;\n" ::"r"(smemPtrAsUint32), "l"(gPtr), "n"(16)); + } +#endif +} + +__device__ void ldsm_x4(void* smem_ptr, uint32_t* reg_ptr) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + asm volatile("ldmatrix.sync.aligned.x4.m8n8.shared.b16 {%0, %1, %2, %3}, [%4];\n" + : "=r"(reg_ptr[0]), "=r"(reg_ptr[1]), "=r"(reg_ptr[2]), "=r"(reg_ptr[3]) + : "r"(__nvvm_get_smem_pointer(smem_ptr))); +#endif +} + +template +__device__ int apply_swizzle_343_on_elem_row_col(int row_idx_, int col_idx_) { + uint32_t row_idx = *reinterpret_cast(&row_idx_); + uint32_t col_idx = *reinterpret_cast(&col_idx_); + row_idx = row_idx % 8; + row_idx = row_idx * (16 / sizeof(Type)); + col_idx = col_idx ^ row_idx; + return *reinterpret_cast(&col_idx); +} + +__device__ void initialize_barrier( + uint64_t* smem_barrier, // 64 bits user-manged barrier in smem + int thread_count = 1) // Thread count expected to arrive/wait on this barrier +{ +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + uint32_t smem_int_ptr = __nvvm_get_smem_pointer(smem_barrier); + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;\n" ::"r"(smem_int_ptr), "r"(thread_count)); +#endif +} + +// Barrier wait +__device__ void wait_barrier( + uint64_t* smem_barrier, // 64 bits user-manged barrier in smem + int phase_bit) // Current phase bit the barrier waiting to flip +{ +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + uint32_t smem_int_ptr = __nvvm_get_smem_pointer(smem_barrier); + asm volatile( + "{\n" + ".reg .pred P1;\n" + "LAB_WAIT:\n" + "mbarrier.try_wait.parity.shared::cta.b64 P1, [%0], %1;\n" + "@P1 bra DONE;\n" + "bra LAB_WAIT;\n" + "DONE:\n" + "}\n" ::"r"(smem_int_ptr), + "r"(phase_bit)); +#endif +} + +__device__ bool try_wait_barrier(uint64_t* smem_ptr, int phase_bit) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + uint32_t wait_complete; + uint32_t smem_int_ptr = __nvvm_get_smem_pointer(smem_ptr); + asm volatile( + "{\n\t" + ".reg .pred P1; \n\t" + "mbarrier.try_wait.parity.shared::cta.b64 P1, [%1], %2; \n\t" + "selp.b32 %0, 1, 0, P1; \n\t" + "}" + : "=r"(wait_complete) + : "r"(smem_int_ptr), "r"(phase_bit)); + return static_cast(wait_complete); +#endif + return false; +} + +// Barrier arrive +__device__ void arrive_barrier(uint64_t* smem_barrier) // 64 bits user-manged barrier in smem +{ +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + uint32_t smem_int_ptr = __nvvm_get_smem_pointer(smem_barrier); + asm volatile( + "{\n" + ".reg .b64 state; \n" + "mbarrier.arrive.shared::cta.b64 state, [%0];\n" + "}\n" ::"r"(smem_int_ptr)); +#endif +} + +__device__ void ldgsts_arrive(uint64_t* smem_barrier) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + uint32_t smem_int_ptr = __nvvm_get_smem_pointer(smem_barrier); + asm volatile("cp.async.mbarrier.arrive.noinc.shared.b64 [%0];" : : "r"(smem_int_ptr)); +#endif +} + +template +struct GmemLoaderA { + static constexpr int elem_bytes = 2; + static constexpr int vec_bytes = 16; + static constexpr int vec_elems = vec_bytes / elem_bytes; + static constexpr int thread_cnt = 64; + static_assert((tile_m * tile_k) % (vec_elems * thread_cnt) == 0); + static constexpr int a_inst_cnt_per_iter = (tile_m * tile_k) / (vec_elems * thread_cnt); + static_assert(gemm_k % tile_k == 0); + static constexpr int k_iter_cnt = gemm_k / tile_k; + + // Extra params to keep the order of k reduction... + static constexpr int mma_warp_cnt = 4; + static constexpr int per_mma_warp_k = tile_k / mma_warp_cnt; + static constexpr int k_each_chunk = gemm_k / mma_warp_cnt; + + private: + __device__ int k_project(int tile_k_idx) { + return (tile_k_idx / per_mma_warp_k * k_each_chunk) + (tile_k_idx % per_mma_warp_k); + } + + public: + __device__ GmemLoaderA(bf16_t const* gmem_a_local_, bf16_t* smem_a_, uint64_t* smem_barrier_) + : gmem_a(gmem_a_local_), smem_a(smem_a_), smem_barrier(smem_barrier_), local_tid(threadIdx.x % thread_cnt) {} + + __device__ void prepare() { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 +// swizzle, that's what we want. +#pragma unroll + for (int i = 0; i < a_inst_cnt_per_iter; i++) { + int linear_idx = local_tid * vec_elems + i * thread_cnt * vec_elems; + int m_idx = linear_idx / tile_k; + int k_idx = linear_idx % tile_k; + k_idx = apply_swizzle_343_on_elem_row_col(m_idx, k_idx); + a_smem_offsets[i] = m_idx * tile_k + k_idx; + } +#endif + } + + __device__ void issue_mainloop() { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 +#pragma unroll 1 + for (int loop_idx = 0; loop_idx < k_iter_cnt; loop_idx++) { + if (need_wait) { + wait_barrier(smem_barrier + 1 + stage_idx * 2, phase_bit); + } + int next_stage_idx = stage_idx + 1; + int next_phase_bit = next_stage_idx == stage_cnt ? phase_bit ^ 1 : phase_bit; + next_stage_idx = next_stage_idx == stage_cnt ? 0 : next_stage_idx; + if (loop_idx != k_iter_cnt - 1) { + need_wait = !try_wait_barrier(smem_barrier + 1 + next_stage_idx * 2, next_phase_bit); + } + +#pragma unroll + for (int i = 0; i < a_inst_cnt_per_iter; i++) { + int smem_offset = a_smem_offsets[i]; + bf16_t* smem_ptr_this_iter = smem_a + stage_idx * tile_m * tile_k + smem_offset; + int linear_idx = local_tid * vec_elems + i * thread_cnt * vec_elems; + int m_idx = linear_idx / tile_k; + int k_idx = linear_idx % tile_k; + int gmem_offset = m_idx * gemm_k + k_project(k_idx); + bf16_t const* gmem_ptr_this_iter = gmem_a + gmem_offset; + ldgsts_128(gmem_ptr_this_iter, smem_ptr_this_iter, true); + } + ldgsts_arrive(smem_barrier + stage_idx * 2); + + stage_idx = next_stage_idx; + phase_bit = next_phase_bit; + gmem_a += per_mma_warp_k; + } +#endif + } + + bf16_t const* gmem_a; + bf16_t* smem_a; + uint64_t* smem_barrier; + int local_tid; + int stage_idx = 0; + int phase_bit = 1; + bool need_wait = true; + + // per smem_stage, store with swizzle information + int a_smem_offsets[a_inst_cnt_per_iter]; +}; + +template +struct GmemLoaderB { + static constexpr int elem_bytes = 2; + static constexpr int vec_bytes = 16; + static constexpr int vec_elems = vec_bytes / elem_bytes; + static constexpr int thread_cnt = 64; + static_assert((tile_n * tile_k) % (vec_elems * thread_cnt) == 0); + static constexpr int b_inst_cnt_per_iter = (tile_n * tile_k) / (vec_elems * thread_cnt); + static_assert(gemm_k % tile_k == 0); + static constexpr int k_iter_cnt = gemm_k / tile_k; + + // Extra params to keep the order of k reduction... + static constexpr int mma_warp_cnt = 4; + static constexpr int per_mma_warp_k = tile_k / mma_warp_cnt; + static constexpr int k_each_chunk = gemm_k / mma_warp_cnt; + + private: + __device__ int k_project(int tile_k_idx) { + return (tile_k_idx / per_mma_warp_k * k_each_chunk) + (tile_k_idx % per_mma_warp_k); + } + + public: + __device__ GmemLoaderB(bf16_t const* gmem_b_local_, bf16_t* smem_b_, uint64_t* smem_barrier_, int gemm_n_) + : gmem_b(gmem_b_local_), + smem_b(smem_b_), + smem_barrier(smem_barrier_), + gemm_n(gemm_n_), + local_tid(threadIdx.x % thread_cnt) {} + + __device__ void prepare() { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 +// swizzle, that's what we want. +#pragma unroll + for (int i = 0; i < b_inst_cnt_per_iter; i++) { + int linear_idx = local_tid * vec_elems + i * thread_cnt * vec_elems; + int n_idx = linear_idx / tile_k; + int k_idx = linear_idx % tile_k; + k_idx = apply_swizzle_343_on_elem_row_col(n_idx, k_idx); + b_smem_offsets[i] = n_idx * tile_k + k_idx; + preds[i] = n_idx < gemm_n; + } +#endif + } + + __device__ void issue_mainloop() { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + asm volatile("griddepcontrol.wait;"); +#pragma unroll 1 + for (int loop_idx = 0; loop_idx < k_iter_cnt; loop_idx++) { + if (need_wait) { + wait_barrier(smem_barrier + 1 + stage_idx * 2, phase_bit); + } + int next_stage_idx = stage_idx + 1; + int next_phase_bit = next_stage_idx == stage_cnt ? phase_bit ^ 1 : phase_bit; + next_stage_idx = next_stage_idx == stage_cnt ? 0 : next_stage_idx; + if (loop_idx != k_iter_cnt - 1) { + need_wait = !try_wait_barrier(smem_barrier + 1 + next_stage_idx * 2, next_phase_bit); + } +#pragma unroll + for (int i = 0; i < b_inst_cnt_per_iter; i++) { + int smem_offset = b_smem_offsets[i]; + bf16_t* smem_ptr_this_iter = smem_b + stage_idx * tile_n * tile_k + smem_offset; + int linear_idx = local_tid * vec_elems + i * thread_cnt * vec_elems; + int n_idx = linear_idx / tile_k; + int k_idx = linear_idx % tile_k; + int gmem_offset = n_idx * gemm_k + k_project(k_idx); + bf16_t const* gmem_ptr_this_iter = gmem_b + gmem_offset; + ldgsts_128(gmem_ptr_this_iter, smem_ptr_this_iter, preds[i]); + } + ldgsts_arrive(smem_barrier + stage_idx * 2); + + stage_idx = next_stage_idx; + phase_bit = next_phase_bit; + gmem_b += per_mma_warp_k; + } +#endif + } + + bf16_t const* gmem_b; + bf16_t* smem_b; + uint64_t* smem_barrier; + int gemm_n; + int local_tid; + int stage_idx = 0; + int phase_bit = 1; + bool need_wait = true; + + // per smem_stage, store with swizzle information + int b_smem_offsets[b_inst_cnt_per_iter]; + uint32_t preds[b_inst_cnt_per_iter]; +}; + +template +struct MmaComputer { + static constexpr int elem_bytes = 2; + static constexpr int thread_cnt = 128; + static_assert(gemm_k % tile_k == 0); + static_assert(tile_k % (thread_cnt / 32) == 0); + static constexpr int per_warp_tile_k = tile_k / (thread_cnt / 32); + static constexpr int k_iter_cnt = gemm_k / tile_k; + static constexpr int k_phase_cnt = per_warp_tile_k / 16; + static constexpr int m_iter_cnt = (tile_m + 15) / 16; + static constexpr int n_iter_cnt = (tile_n + 7) / 8; // Possible to have non-1 n_iter_cnt for ab_swap m16 case. + static_assert(m_iter_cnt == 1); + static_assert(n_iter_cnt == 1 || n_iter_cnt == 2); + + __device__ MmaComputer( + bf16_t* gmem_c_local_, bf16_t* smem_a_, bf16_t* smem_b_, uint64_t* smem_barrier_, int warp_idx_, int gemm_n_) + : gmem_c(gmem_c_local_), + smem_a(smem_a_), + smem_b(smem_b_), + smem_barrier(smem_barrier_), + warp_idx(warp_idx_ - (thread_cnt / 32)), + gemm_n(gemm_n_) {} + + private: + __device__ constexpr int internal_b_atom_func(int tid) { + if constexpr (tile_n < 8) { + return (tid % tile_n) + ((tid % 8) / tile_n * 0) + tid / 8 * 8 * tile_n; + } else { + return (tid % 8) + ((tid % 32) / 8 * (tile_n * 8)); + } + } + + public: + __device__ void prepare() { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 +#pragma unroll + for (int i = 0; i < k_phase_cnt; i++) { + int linear_idx = (lane_idx % 16) + (lane_idx / 16) * 128 + i * 256; + int m_idx = linear_idx % tile_m; + int k_idx = linear_idx / tile_m + warp_k_offset_in_tile_k; + k_idx = apply_swizzle_343_on_elem_row_col(m_idx, k_idx); + a_smem_offsets[0][i] = m_idx * tile_k + k_idx; + } +#pragma unroll + for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) { +#pragma unroll + for (int i = 0; i < k_phase_cnt; i += 2) { // Special i+=2 for B. + int linear_idx = internal_b_atom_func(lane_idx) + i * tile_n * 16 + n_iter_idx * 8; + int n_idx = linear_idx % tile_n; + int k_idx = linear_idx / tile_n + warp_k_offset_in_tile_k; + k_idx = apply_swizzle_343_on_elem_row_col(n_idx, k_idx); + b_smem_offsets[n_iter_idx][i] = n_idx * tile_k + k_idx; + } + } +#endif + } + + __device__ void issue_mainloop() { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 +#pragma unroll 1 + for (int loop_idx = 0; loop_idx < k_iter_cnt; loop_idx++) { + wait_barrier(smem_barrier + 0 + stage_idx * 2, phase_bit); + +#pragma unroll + for (int i = 0; i < k_phase_cnt; i++) { + int smem_offset = a_smem_offsets[0][i]; + bf16_t* smem_ptr_this_iter = smem_a + stage_idx * tile_m * tile_k + smem_offset; + ldsm_x4(smem_ptr_this_iter, reinterpret_cast(a_reg[0][i])); + } + +#pragma unroll + for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) { +#pragma unroll + for (int i = 0; i < k_phase_cnt; i += 2) { + int smem_offset = b_smem_offsets[n_iter_idx][i]; + bf16_t* smem_ptr_this_iter = smem_b + stage_idx * tile_n * tile_k + smem_offset; + ldsm_x4(smem_ptr_this_iter, reinterpret_cast(b_reg[n_iter_idx][i])); + } + } + +#pragma unroll + for (int k_iter_idx = 0; k_iter_idx < k_phase_cnt; k_iter_idx++) { +#pragma unroll + for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) { + hmma_16_8_16_f32acc_bf16ab( + acc_reg[0][n_iter_idx], a_reg[0][k_iter_idx], b_reg[n_iter_idx][k_iter_idx], acc_reg[0][n_iter_idx]); + } + } + ::arrive_barrier(smem_barrier + 1 + stage_idx * 2); + stage_idx += 1; + phase_bit = stage_idx == stage_cnt ? phase_bit ^ 1 : phase_bit; + stage_idx = stage_idx == stage_cnt ? 0 : stage_idx; + } +#endif + } + + __device__ void epi() { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + asm volatile("bar.sync %0, %1;" : : "r"(1), "r"(thread_cnt)); + // reorganize the acc_reg + constexpr int thread_m = 2; + constexpr int thread_n = 2 * n_iter_cnt; + constexpr int cta_mma_n = n_iter_cnt * 8; + float acc_reg_reorg[thread_m][thread_n]; + + for (int i = 0; i < thread_m; i++) { + for (int j = 0; j < thread_n; j++) { + acc_reg_reorg[i][j] = acc_reg[0][j / 2][(j % 2) + (i * 2)]; + } + } + + // 4 x cosize(smem_c_layout) + float* smem_c = reinterpret_cast(smem_a); + // coord -> index + auto smem_c_index_func = [&](int m_idx, int n_idx) { + int group_rows = 32 / cta_mma_n; + int group_cnt = 2; + return (m_idx % group_rows * cta_mma_n) + (m_idx / group_rows * (32 + group_cnt)) + n_idx; + }; + constexpr int cosize_smem_c = ((tile_m * cta_mma_n) / 32) * (32 + 2); + +// This should be optimized to STS.64 but can not be STS.128 due to the bank index. +#pragma unroll + for (int m_idx_thread = 0; m_idx_thread < thread_m; m_idx_thread++) { +#pragma unroll + for (int n_idx_thread = 0; n_idx_thread < thread_n; n_idx_thread++) { + int m_idx = (lane_idx / 4) + m_idx_thread * 8; + int n_idx = ((lane_idx % 4) * 2) + (n_idx_thread % 2) + (n_idx_thread / 2) * 8; + smem_c[cosize_smem_c * warp_idx + smem_c_index_func(m_idx, n_idx)] = acc_reg_reorg[m_idx_thread][n_idx_thread]; + } + } + asm volatile("bar.sync %0, %1;" : : "r"(1), "r"(thread_cnt)); + + if (warp_idx == 0) { + constexpr int final_acc_reg_cnt = (tile_m * tile_n + 31) / 32; + float acc_final[final_acc_reg_cnt]{}; + +#pragma unroll + for (int reg_idx = 0; reg_idx < final_acc_reg_cnt; reg_idx++) { + int linear_idx = reg_idx * 32 + lane_idx; + int m_idx = linear_idx % tile_m; + int n_idx = linear_idx / tile_m; + acc_final[reg_idx] += smem_c[smem_c_index_func(m_idx, n_idx) + 0 * cosize_smem_c] + + smem_c[smem_c_index_func(m_idx, n_idx) + 1 * cosize_smem_c] + + smem_c[smem_c_index_func(m_idx, n_idx) + 2 * cosize_smem_c] + + smem_c[smem_c_index_func(m_idx, n_idx) + 3 * cosize_smem_c]; + } + +#pragma unroll + for (int reg_idx = 0; reg_idx < final_acc_reg_cnt; reg_idx++) { + int linear_idx = reg_idx * 32 + lane_idx; + int m_idx = linear_idx % tile_m; + int n_idx = linear_idx / tile_m; + if (m_idx < tile_m && n_idx < gemm_n) { + gmem_c[n_idx * gemm_m + m_idx] = acc_final[reg_idx]; + } + } + } +#endif + } + + bf16_t* gmem_c; + bf16_t* smem_a; + bf16_t* smem_b; + uint64_t* smem_barrier; + int warp_idx; + int gemm_n; + int stage_idx = 0; + int phase_bit = 0; + int lane_idx = threadIdx.x % 32; + int warp_k_offset_in_tile_k = warp_idx * per_warp_tile_k; + + int a_smem_offsets[m_iter_cnt][k_phase_cnt]; + int b_smem_offsets[n_iter_cnt][k_phase_cnt]; + + bf16_t a_reg[m_iter_cnt][k_phase_cnt][8]; + bf16_t b_reg[n_iter_cnt][k_phase_cnt][4]; + float acc_reg[m_iter_cnt][n_iter_cnt][4]{}; +}; + +// AB swapped, kernel is k-major, k-major, m-major +template +__global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel( + bf16_t* output, bf16_t const* mat_a, bf16_t const* mat_b, int gemm_n) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + constexpr int load_thread_cnt = 128; + constexpr int compute_thread_cnt = 128; + constexpr int thread_cnt = load_thread_cnt + compute_thread_cnt; + (void)thread_cnt; + static_assert(gemm_m % 16 == 0); + static_assert(gemm_k % tile_k == 0); + static_assert(gemm_m % tile_m == 0); + static_assert( + tile_k == 128 || tile_k == 256 || tile_k == 512 || + tile_k == 1024); // tile_k must be larger than 64 since 4 warp splitK. + static_assert(tile_m == 16); + constexpr int g2s_vec_bytes = 16; + constexpr int a_elem_bytes = 2; + constexpr int b_elem_bytes = 2; + // constexpr int c_elem_bytes = 2; + static_assert((tile_m * a_elem_bytes + tile_n * b_elem_bytes) * tile_k * stage_cnt <= 225 * 1024); + static_assert((tile_m * tile_k * a_elem_bytes) % (load_thread_cnt * g2s_vec_bytes) == 0); + static_assert((tile_n * tile_k * b_elem_bytes) % (load_thread_cnt * g2s_vec_bytes) == 0); + + extern __shared__ char smem[]; + uint64_t* smem_barrier = reinterpret_cast(smem); // producer,consumer; producer,consumer; ... + bf16_t* smem_a = reinterpret_cast(smem + (stage_cnt * 8 * 2 + 1024) / 1024 * 1024); + bf16_t* smem_b = smem_a + tile_m * tile_k * stage_cnt; + + int cta_m_idx = tile_m * blockIdx.x; + int cta_n_idx = tile_n * blockIdx.y; + bf16_t const* gmem_a_local = mat_a + cta_m_idx * gemm_k; + bf16_t const* gmem_b_local = mat_b + cta_n_idx * gemm_k; + bf16_t* gmem_c_local = output + cta_n_idx * gemm_m + cta_m_idx; + + int warp_idx = __shfl_sync(0xffffffff, threadIdx.x / 32, 0); + + if (warp_idx == 4) { + for (int i = 0; i < stage_cnt; i++) { + initialize_barrier(smem_barrier + i * 2 + 0, load_thread_cnt); // producer + initialize_barrier(smem_barrier + i * 2 + 1, compute_thread_cnt); // consumer + } + } + __syncthreads(); + + if (warp_idx < 2) { + GmemLoaderA a_loader(gmem_a_local, smem_a, smem_barrier); + a_loader.prepare(); + a_loader.issue_mainloop(); + } else if (warp_idx < 4) { + GmemLoaderB b_loader(gmem_b_local, smem_b, smem_barrier, gemm_n); + b_loader.prepare(); + b_loader.issue_mainloop(); + } else { + MmaComputer mma_computer( + gmem_c_local, smem_a, smem_b, smem_barrier, warp_idx, gemm_n); + mma_computer.prepare(); + mma_computer.issue_mainloop(); + mma_computer.epi(); + } + asm volatile("griddepcontrol.launch_dependents;"); +#endif +} + +template +void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens, cudaStream_t const stream) { + constexpr int gemm_m = kHdOut; // 2112 + int const gemm_n = num_tokens; // 16 + constexpr int gemm_k = kHdIn; // 7168 + constexpr int batch_size = 1; + std::swap(mat_a, mat_b); + constexpr int tile_m = 16; + constexpr int tile_n = kTileN; // 8 or 16 + constexpr int tile_k = std::max(256, 1024 / tile_n); // 256 + constexpr int max_stage_cnt = 1024 * 192 / ((tile_m + tile_n) * tile_k * sizeof(bf16_t)); + constexpr int k_iter_cnt = gemm_k / tile_k; + constexpr int stage_cnt = + k_iter_cnt > max_stage_cnt ? max_stage_cnt : k_iter_cnt; // possible tunable for smallK > 1 wave n. // 22 + int cta_m_cnt = gemm_m / tile_m; + int cta_n_cnt = (gemm_n + tile_n - 1) / tile_n; + constexpr int barrier_bytes = (stage_cnt * 16 + 1023) / 1024 * 1024; // 4096 + constexpr int smem_bytes = ((tile_m * 2 + tile_n * 2) * tile_k * stage_cnt + barrier_bytes + 1023) / 1024 * 1024; + + dim3 grid(cta_m_cnt, cta_n_cnt, 1); + dim3 block_size(256); + cudaLaunchConfig_t config; + config.gridDim = grid; + config.blockDim = block_size; + config.dynamicSmemBytes = smem_bytes; + config.stream = stream; + cudaLaunchAttribute attrs[1]; + attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attrs[0].val.programmaticStreamSerializationAllowed = getEnvEnablePDL(); + config.numAttrs = 1; + config.attrs = attrs; + if (smem_bytes >= (48 * 1024)) { + cudaFuncSetAttribute( + fused_a_gemm_kernel, + cudaFuncAttributeMaxDynamicSharedMemorySize, + smem_bytes); + } + cudaLaunchKernelEx( + &config, + fused_a_gemm_kernel, + output, + mat_a, + mat_b, + gemm_n); +} + +template void invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 8>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens, cudaStream_t); + +template void invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 16>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens, cudaStream_t); + +void dsv3_fused_a_gemm(torch::Tensor& output, torch::Tensor const& mat_a, torch::Tensor const& mat_b) { + TORCH_CHECK(mat_a.dim() == 2 && mat_b.dim() == 2 && output.dim() == 2); + int const num_tokens = mat_a.size(0); + int const hd_in = mat_a.size(1); + int const hd_out = mat_b.size(1); + + constexpr int kHdIn = 7168; + constexpr int kHdOut = 2112; + TORCH_CHECK(num_tokens >= 1 && num_tokens <= 16, "required 1 <= mat_a.shape[0] <= 16") + TORCH_CHECK(hd_in == kHdIn, "required mat_a.shape[1] == 7168") + TORCH_CHECK(hd_out == kHdOut, "required mat_b.shape[1] == 2112") + TORCH_CHECK(output.size(0) == num_tokens, "required output.shape[0] == mat_a.shape[0]") + TORCH_CHECK(output.size(1) == hd_out, "required output.shape[1] == mat_b.shape[1]") + + TORCH_CHECK(mat_a.stride(1) == 1, "mat_a must be a row major tensor"); // Row-major + TORCH_CHECK(output.stride(1) == 1, "output must be a row major tensor"); // Row-major + TORCH_CHECK(mat_b.stride(0) == 1, "mat_b must be a column major tensor"); // Column-major + + auto const data_type = mat_a.scalar_type(); + TORCH_CHECK( + mat_a.scalar_type() == torch::kBFloat16 && mat_b.scalar_type() == torch::kBFloat16, + "Only BFloat16 input dtype is supported") + TORCH_CHECK(output.scalar_type() == torch::kBFloat16, "Only BFloat16 output dtype is supported") + + auto const sm = getSMVersion(); + TORCH_CHECK(sm >= 90, "required CUDA ARCH >= SM_90"); + + auto stream = at::cuda::getCurrentCUDAStream(mat_a.get_device()); + if (num_tokens <= 8) { + invokeFusedAGemm<__nv_bfloat16, kHdIn, kHdOut, 8>( + reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), + num_tokens, + stream); + } else { + invokeFusedAGemm<__nv_bfloat16, kHdIn, kHdOut, 16>( + reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), + num_tokens, + stream); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/dsv3_router_gemm_bf16_out.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/dsv3_router_gemm_bf16_out.cu new file mode 100644 index 00000000..e613bd75 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/dsv3_router_gemm_bf16_out.cu @@ -0,0 +1,284 @@ +/* + * Adapted from + * https://github.com/NVIDIA/TensorRT-LLM/blob/main/cpp/tensorrt_llm/kernels/dsv3MinLatencyKernels/dsv3RouterGemm.cu + * https://github.com/NVIDIA/TensorRT-LLM/blob/main/cpp/tensorrt_llm/thop/dsv3RouterGemmOp.cpp + * + * Copyright (c) 2019-2023, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include "cuda_bf16.h" +#include "cuda_runtime.h" +#include "utils.h" + +// Custom FMA implementation using PTX assembly instructions +__device__ __forceinline__ void fma(float2& d, float2 const& a, float2 const& b, float2 const& c) { + asm volatile("fma.rn.f32x2 %0, %1, %2, %3;\n" + : "=l"(reinterpret_cast(d)) + : "l"(reinterpret_cast(a)), + "l"(reinterpret_cast(b)), + "l"(reinterpret_cast(c))); +} + +// Convert 8 bfloat16 values from a uint4 to float array - optimized conversion +template +__device__ __forceinline__ void bf16_uint4_to_float8(uint4 const& vec, float* dst) { + __nv_bfloat16* bf16_ptr = reinterpret_cast<__nv_bfloat16*>(const_cast(&vec)); + +#pragma unroll + for (int i = 0; i < VPT; i++) { + dst[i] = __bfloat162float(bf16_ptr[i]); + } +} + +template +__global__ +__launch_bounds__(128, 1) void router_gemm_kernel_bf16_output(__nv_bfloat16* out, T const* mat_a, T const* mat_b) { + // Each block handles one expert column + int const n_idx = blockIdx.x; + int const tid = threadIdx.x; + constexpr int kWarpSize = 32; + constexpr int kNumWarps = kBlockSize / kWarpSize; + // Constants for this kernel + constexpr int k_elems_per_k_iteration = VPT * kBlockSize; + constexpr int k_iterations = kHiddenDim / k_elems_per_k_iteration; // Total K iterations + + // Initialize accumulators for all M rows + float acc[kNumTokens] = {}; + + // Shared memory for warp-level reduction + __shared__ float sm_reduction[kNumTokens][kNumWarps]; // kNumWarps + + // B matrix is in column-major order, so we can directly load a column for the n_idx expert + T const* b_col = mat_b + n_idx * kHiddenDim; + + // Pre-compute k_base values for each iteration to help compiler optimize + // int k_bases[k_iterations]; + int k_bases[k_iterations]; +#pragma unroll + for (int ki = 0; ki < k_iterations; ki++) { + k_bases[ki] = ki * k_elems_per_k_iteration + tid * VPT; + } + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + asm volatile("griddepcontrol.wait;"); +#endif + + // Process the GEMM in chunks + for (int ki = 0; ki < k_iterations; ki++) { + int const k_base = k_bases[ki]; + + // Load B matrix values using vector load (8 bf16 values) + uint4 b_vec = *reinterpret_cast(b_col + k_base); + + // Convert B values to float + float b_float[VPT]; + bf16_uint4_to_float8(b_vec, b_float); + +// Process each token +#pragma unroll + for (int m_idx = 0; m_idx < kNumTokens; m_idx++) { + // Load both rows of A matrix using vector loads + uint4 a_vec = *reinterpret_cast(mat_a + (m_idx * kHiddenDim) + k_base); + + // Convert A values to float + float a_float[VPT]; + bf16_uint4_to_float8(a_vec, a_float); + +// Process elements in this chunk +#pragma unroll + for (int k = 0; k < VPT; k++) { + float a = a_float[k]; + float b = b_float[k]; + acc[m_idx] += a * b; + } + } + } + + // Perform warp-level reduction + int const warpSize = 32; + int const warpId = tid / warpSize; + int const laneId = tid % warpSize; + + // Register for warp-level reduction results + float warp_result[kNumTokens]; + +#pragma unroll + for (int m_idx = 0; m_idx < kNumTokens; m_idx++) { + warp_result[m_idx] = acc[m_idx]; + } + +// Perform warp-level reduction using optimized butterfly pattern +#pragma unroll + for (int m = 0; m < kNumTokens; m++) { + float sum = warp_result[m]; + + // Butterfly reduction pattern + sum += __shfl_xor_sync(0xffffffff, sum, 16); + sum += __shfl_xor_sync(0xffffffff, sum, 8); + sum += __shfl_xor_sync(0xffffffff, sum, 4); + sum += __shfl_xor_sync(0xffffffff, sum, 2); + sum += __shfl_xor_sync(0xffffffff, sum, 1); + + // Only the first thread in each warp stores to shared memory + if (laneId == 0) { + sm_reduction[m][warpId] = sum; + } + } + + __syncthreads(); + + // Final reduction across warps (only first thread) + if (tid == 0) { +#pragma unroll + for (int m = 0; m < kNumTokens; m++) { + float final_sum = 0.0f; + +// Sum across the kNumWarps +#pragma unroll + for (int w = 0; w < kNumWarps; w++) { + final_sum += sm_reduction[m][w]; + } + + // Write final result + out[m * kNumExperts + n_idx] = __float2bfloat16(final_sum); + } + } +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + asm volatile("griddepcontrol.launch_dependents;"); +#endif +} + +template +void invokeRouterGemmBf16Output(__nv_bfloat16* output, T const* mat_a, T const* mat_b, cudaStream_t stream) { + constexpr int VPT = 16 / sizeof(T); + constexpr int kBlockSize = 128; + cudaLaunchConfig_t config; + config.gridDim = kNumExperts; + config.blockDim = kBlockSize; + config.dynamicSmemBytes = 0; + config.stream = stream; + cudaLaunchAttribute attrs[1]; + attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attrs[0].val.programmaticStreamSerializationAllowed = getEnvEnablePDL(); + config.numAttrs = 1; + config.attrs = attrs; + cudaLaunchKernelEx( + &config, + router_gemm_kernel_bf16_output, + output, + mat_a, + mat_b); +} + +// Template instantiations for DEFAULT_NUM_EXPERTS experts +template void invokeRouterGemmBf16Output<__nv_bfloat16, 1, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 2, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 3, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 4, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 5, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 6, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 7, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 8, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 9, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 10, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 11, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 12, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 13, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 14, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 15, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 16, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +// Template instantiations for KIMI_K2_NUM_EXPERTS experts +template void invokeRouterGemmBf16Output<__nv_bfloat16, 1, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 2, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 3, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 4, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 5, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 6, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 7, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 8, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 9, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 10, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 11, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 12, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 13, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 14, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 15, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 16, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/dsv3_router_gemm_entry.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/dsv3_router_gemm_entry.cu new file mode 100644 index 00000000..4f09e6cf --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/dsv3_router_gemm_entry.cu @@ -0,0 +1,161 @@ +/* + * Adapted from + * https://github.com/NVIDIA/TensorRT-LLM/blob/main/cpp/tensorrt_llm/kernels/dsv3MinLatencyKernels/dsv3RouterGemm.cu + * https://github.com/NVIDIA/TensorRT-LLM/blob/main/cpp/tensorrt_llm/thop/dsv3RouterGemmOp.cpp + * + * Copyright (c) 2019-2023, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include "cuda_bf16.h" +#include "cuda_runtime.h" +#include "utils.h" + +static constexpr int DEFAULT_NUM_EXPERTS = 256; +static constexpr int KIMI_K2_NUM_EXPERTS = 384; +static constexpr int DEFAULT_HIDDEN_DIM = 7168; + +template +void invokeRouterGemmFloatOutput(float* output, T const* mat_a, T const* mat_b, cudaStream_t stream); + +template +void invokeRouterGemmBf16Output(__nv_bfloat16* output, T const* mat_a, T const* mat_b, cudaStream_t stream); + +template +struct LoopUnroller { + static void unroll_float_output( + int num_tokens, float* output, __nv_bfloat16 const* input, __nv_bfloat16 const* weights, cudaStream_t stream) { + if (num_tokens == kBegin) { + invokeRouterGemmFloatOutput<__nv_bfloat16, kBegin, kNumExperts, kHiddenDim>(output, input, weights, stream); + } else { + LoopUnroller::unroll_float_output( + num_tokens, output, input, weights, stream); + } + } + + static void unroll_bf16_output( + int num_tokens, + __nv_bfloat16* output, + __nv_bfloat16 const* input, + __nv_bfloat16 const* weights, + cudaStream_t stream) { + if (num_tokens == kBegin) { + invokeRouterGemmBf16Output<__nv_bfloat16, kBegin, kNumExperts, kHiddenDim>(output, input, weights, stream); + } else { + LoopUnroller::unroll_bf16_output( + num_tokens, output, input, weights, stream); + } + } +}; + +template +struct LoopUnroller { + static void unroll_float_output( + int num_tokens, float* output, __nv_bfloat16 const* input, __nv_bfloat16 const* weights, cudaStream_t stream) { + if (num_tokens == kEnd) { + invokeRouterGemmFloatOutput<__nv_bfloat16, kEnd, kNumExperts, kHiddenDim>(output, input, weights, stream); + } else { + throw std::invalid_argument("Invalid num_tokens, only supports 1 to 16"); + } + } + + static void unroll_bf16_output( + int num_tokens, + __nv_bfloat16* output, + __nv_bfloat16 const* input, + __nv_bfloat16 const* weights, + cudaStream_t stream) { + if (num_tokens == kEnd) { + invokeRouterGemmBf16Output<__nv_bfloat16, kEnd, kNumExperts, kHiddenDim>(output, input, weights, stream); + } else { + throw std::invalid_argument("Invalid num_tokens, only supports 1 to 16"); + } + } +}; + +void dsv3_router_gemm( + torch::Tensor& output, // [num_tokens, num_experts] + const torch::Tensor& mat_a, // [num_tokens, hidden_dim] + const torch::Tensor& mat_b // [num_experts, hidden_dim] +) { + TORCH_CHECK(output.dim() == 2 && mat_a.dim() == 2 && mat_b.dim() == 2); + + const int num_tokens = mat_a.size(0); + const int num_experts = mat_b.size(0); + const int hidden_dim = mat_a.size(1); + + TORCH_CHECK(mat_a.size(1) == mat_b.size(1), "mat_a and mat_b must have the same hidden_dim"); + TORCH_CHECK( + hidden_dim == DEFAULT_HIDDEN_DIM, + "Expected hidden_dim=", + DEFAULT_HIDDEN_DIM, + ", but got hidden_dim=", + hidden_dim); + TORCH_CHECK( + num_experts == DEFAULT_NUM_EXPERTS || num_experts == KIMI_K2_NUM_EXPERTS, + "Expected num_experts=", + DEFAULT_NUM_EXPERTS, + " or num_experts=", + KIMI_K2_NUM_EXPERTS, + ", but got num_experts=", + num_experts); + TORCH_CHECK( + num_tokens >= 1 && num_tokens <= 16, "currently num_tokens must be less than or equal to 16 for router_gemm"); + TORCH_CHECK(mat_a.dtype() == torch::kBFloat16, "mat_a must be bf16"); + TORCH_CHECK(mat_b.dtype() == torch::kBFloat16, "mat_b must be bf16"); + TORCH_CHECK( + output.dtype() == torch::kFloat32 || output.dtype() == torch::kBFloat16, "output must be float32 or bf16"); + + auto const sm = getSMVersion(); + TORCH_CHECK(sm >= 90, "required CUDA ARCH >= SM_90"); + + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + if (output.dtype() == torch::kFloat32) { + if (num_experts == DEFAULT_NUM_EXPERTS) { + LoopUnroller<1, 16, DEFAULT_NUM_EXPERTS, DEFAULT_HIDDEN_DIM>::unroll_float_output( + num_tokens, + reinterpret_cast(output.mutable_data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), + stream); + } else if (num_experts == KIMI_K2_NUM_EXPERTS) { + LoopUnroller<1, 16, KIMI_K2_NUM_EXPERTS, DEFAULT_HIDDEN_DIM>::unroll_float_output( + num_tokens, + reinterpret_cast(output.mutable_data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), + stream); + } + } else if (output.dtype() == torch::kBFloat16) { + if (num_experts == DEFAULT_NUM_EXPERTS) { + LoopUnroller<1, 16, DEFAULT_NUM_EXPERTS, DEFAULT_HIDDEN_DIM>::unroll_bf16_output( + num_tokens, + reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), + stream); + } else if (num_experts == KIMI_K2_NUM_EXPERTS) { + LoopUnroller<1, 16, KIMI_K2_NUM_EXPERTS, DEFAULT_HIDDEN_DIM>::unroll_bf16_output( + num_tokens, + reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), + stream); + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/dsv3_router_gemm_float_out.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/dsv3_router_gemm_float_out.cu new file mode 100644 index 00000000..88a364e2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/dsv3_router_gemm_float_out.cu @@ -0,0 +1,283 @@ +/* + * Adapted from + * https://github.com/NVIDIA/TensorRT-LLM/blob/main/cpp/tensorrt_llm/kernels/dsv3MinLatencyKernels/dsv3RouterGemm.cu + * https://github.com/NVIDIA/TensorRT-LLM/blob/main/cpp/tensorrt_llm/thop/dsv3RouterGemmOp.cpp + * + * Copyright (c) 2019-2023, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include "cuda_bf16.h" +#include "cuda_runtime.h" +#include "utils.h" + +// Custom FMA implementation using PTX assembly instructions +__device__ __forceinline__ void fma(float2& d, float2 const& a, float2 const& b, float2 const& c) { + asm volatile("fma.rn.f32x2 %0, %1, %2, %3;\n" + : "=l"(reinterpret_cast(d)) + : "l"(reinterpret_cast(a)), + "l"(reinterpret_cast(b)), + "l"(reinterpret_cast(c))); +} + +// Convert 8 bfloat16 values from a uint4 to float array - optimized conversion +template +__device__ __forceinline__ void bf16_uint4_to_float8(uint4 const& vec, float* dst) { + __nv_bfloat16* bf16_ptr = reinterpret_cast<__nv_bfloat16*>(const_cast(&vec)); + +#pragma unroll + for (int i = 0; i < VPT; i++) { + dst[i] = __bfloat162float(bf16_ptr[i]); + } +} + +template +__global__ __launch_bounds__(128, 1) void router_gemm_kernel_float_output(float* out, T const* mat_a, T const* mat_b) { + // Each block handles one expert column + int const n_idx = blockIdx.x; + int const tid = threadIdx.x; + constexpr int kWarpSize = 32; + constexpr int kNumWarps = kBlockSize / kWarpSize; + // Constants for this kernel + constexpr int k_elems_per_k_iteration = VPT * kBlockSize; + constexpr int k_iterations = kHiddenDim / k_elems_per_k_iteration; // Total K iterations + + // Initialize accumulators for all M rows + float acc[kNumTokens] = {}; + + // Shared memory for warp-level reduction + __shared__ float sm_reduction[kNumTokens][kNumWarps]; // kNumWarps + + // B matrix is in column-major order, so we can directly load a column for the n_idx expert + T const* b_col = mat_b + n_idx * kHiddenDim; + + // Pre-compute k_base values for each iteration to help compiler optimize + // int k_bases[k_iterations]; + int k_bases[k_iterations]; +#pragma unroll + for (int ki = 0; ki < k_iterations; ki++) { + k_bases[ki] = ki * k_elems_per_k_iteration + tid * VPT; + } + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + asm volatile("griddepcontrol.wait;"); +#endif + + // Process the GEMM in chunks + for (int ki = 0; ki < k_iterations; ki++) { + int const k_base = k_bases[ki]; + + // Load B matrix values using vector load (8 bf16 values) + uint4 b_vec = *reinterpret_cast(b_col + k_base); + + // Convert B values to float + float b_float[VPT]; + bf16_uint4_to_float8(b_vec, b_float); + +// Process each token +#pragma unroll + for (int m_idx = 0; m_idx < kNumTokens; m_idx++) { + // Load both rows of A matrix using vector loads + uint4 a_vec = *reinterpret_cast(mat_a + (m_idx * kHiddenDim) + k_base); + + // Convert A values to float + float a_float[VPT]; + bf16_uint4_to_float8(a_vec, a_float); + +// Process elements in this chunk +#pragma unroll + for (int k = 0; k < VPT; k++) { + float a = a_float[k]; + float b = b_float[k]; + acc[m_idx] += a * b; + } + } + } + + // Perform warp-level reduction + int const warpSize = 32; + int const warpId = tid / warpSize; + int const laneId = tid % warpSize; + + // Register for warp-level reduction results + float warp_result[kNumTokens]; + +#pragma unroll + for (int m_idx = 0; m_idx < kNumTokens; m_idx++) { + warp_result[m_idx] = acc[m_idx]; + } + +// Perform warp-level reduction using optimized butterfly pattern +#pragma unroll + for (int m = 0; m < kNumTokens; m++) { + float sum = warp_result[m]; + + // Butterfly reduction pattern + sum += __shfl_xor_sync(0xffffffff, sum, 16); + sum += __shfl_xor_sync(0xffffffff, sum, 8); + sum += __shfl_xor_sync(0xffffffff, sum, 4); + sum += __shfl_xor_sync(0xffffffff, sum, 2); + sum += __shfl_xor_sync(0xffffffff, sum, 1); + + // Only the first thread in each warp stores to shared memory + if (laneId == 0) { + sm_reduction[m][warpId] = sum; + } + } + + __syncthreads(); + + // Final reduction across warps (only first thread) + if (tid == 0) { +#pragma unroll + for (int m = 0; m < kNumTokens; m++) { + float final_sum = 0.0f; + +// Sum across the kNumWarps +#pragma unroll + for (int w = 0; w < kNumWarps; w++) { + final_sum += sm_reduction[m][w]; + } + + // Write final result + out[m * kNumExperts + n_idx] = final_sum; + } + } +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + asm volatile("griddepcontrol.launch_dependents;"); +#endif +} + +template +void invokeRouterGemmFloatOutput(float* output, T const* mat_a, T const* mat_b, cudaStream_t stream) { + constexpr int VPT = 16 / sizeof(T); + constexpr int kBlockSize = 128; + cudaLaunchConfig_t config; + config.gridDim = kNumExperts; + config.blockDim = kBlockSize; + config.dynamicSmemBytes = 0; + config.stream = stream; + cudaLaunchAttribute attrs[1]; + attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attrs[0].val.programmaticStreamSerializationAllowed = getEnvEnablePDL(); + config.numAttrs = 1; + config.attrs = attrs; + cudaLaunchKernelEx( + &config, + router_gemm_kernel_float_output, + output, + mat_a, + mat_b); +} + +// Template instantiations for DEFAULT_NUM_EXPERTS experts +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 1, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 2, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 3, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 4, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 5, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 6, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 7, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 8, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 9, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 10, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 11, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 12, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 13, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 14, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 15, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 16, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +// Template instantiations for KIMI_K2_NUM_EXPERTS experts +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 1, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 2, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 3, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 4, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 5, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 6, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 7, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 8, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 9, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 10, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 11, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 12, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 13, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 14, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 15, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 16, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/fp8_blockwise_gemm_kernel.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/fp8_blockwise_gemm_kernel.cu new file mode 100644 index 00000000..b8b23c42 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/fp8_blockwise_gemm_kernel.cu @@ -0,0 +1,465 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cutlass_extensions/gemm/cutlass_gemm_caller.cuh" +#include "cutlass_extensions/gemm/fp8_blockwise_gemm_sm90_dispatch.cuh" +#include "utils.h" + +using namespace cute; + +template < + typename OutType, + typename MmaTileShape, + typename PerSmTileShape, + typename EpilogueTileShape, + typename ScalesPerTile, + int TileSizeM_ = 128, + class ClusterShape = Shape<_1, _1, _1>> +void launch_sm100_fp8_blockwise_scaled_mm( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b) { + static constexpr int ScaleMsPerTile = size<0>(ScalesPerTile{}); + static constexpr int ScaleGranularityM = size<0>(MmaTileShape{}) / ScaleMsPerTile; + static constexpr int ScaleGranularityN = size<1>(MmaTileShape{}) / size<1>(ScalesPerTile{}); + static constexpr int ScaleGranularityK = size<2>(MmaTileShape{}) / size<2>(ScalesPerTile{}); + + using ElementAB = cutlass::float_e4m3_t; + using ElementA = ElementAB; + using ElementB = ElementAB; + using ElementC = void; + using ElementD = OutType; + using LayoutA = cutlass::layout::RowMajor; + using LayoutB = cutlass::layout::ColumnMajor; + using LayoutD = cutlass::layout::RowMajor; + using LayoutC = LayoutD; + // This means both SFA and SFB are column-major. + using ScaleConfig = cutlass::detail::Sm100BlockwiseScaleConfig< + ScaleGranularityM, + ScaleGranularityN, + ScaleGranularityK, + cute::UMMA::Major::MN, + cute::UMMA::Major::K>; + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); + + static constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentD = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentC = AlignmentD; + + using ElementAccumulator = float; + using ElementBlockScale = float; + using ElementCompute = float; + using ArchTag = cutlass::arch::Sm100; + using OperatorClass = cutlass::arch::OpClassTensorOp; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + cutlass::arch::OpClassTensorOp, + PerSmTileShape, + ClusterShape, + EpilogueTileShape, + ElementAccumulator, + ElementCompute, + ElementC, + LayoutC, + AlignmentC, + ElementD, + LayoutD, + AlignmentD, + cutlass::epilogue::TmaWarpSpecialized1Sm>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementA, + cute::tuple, + AlignmentA, + ElementB, + cute::tuple, + AlignmentB, + ElementAccumulator, + MmaTileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + cutlass::gemm::KernelTmaWarpSpecializedBlockwise1SmSm100>::CollectiveOp; + + using GemmKernel = cutlass::gemm::kernel::GemmUniversal< + Shape, + CollectiveMainloop, + CollectiveEpilogue, + cutlass::gemm::PersistentScheduler>; + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + + Gemm gemm_op; + + int m = a.size(0); + int k = a.size(1); + int n = b.size(1); + + auto a_ptr = static_cast(a.data_ptr()); + auto b_ptr = static_cast(b.data_ptr()); + auto scales_a_ptr = static_cast(scales_a.data_ptr()); + auto scales_b_ptr = static_cast(scales_b.data_ptr()); + auto c_ptr = static_cast(out.data_ptr()); + + using StrideA = typename GemmKernel::StrideA; + using StrideB = typename GemmKernel::StrideB; + using StrideD = typename GemmKernel::StrideD; + using StrideC = typename GemmKernel::StrideD; + + StrideA a_stride = cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(m, k, 1)); + StrideB b_stride = cutlass::make_cute_packed_stride(StrideB{}, cute::make_shape(n, k, 1)); + StrideC c_stride = cutlass::make_cute_packed_stride(StrideC{}, cute::make_shape(m, n, 1)); + LayoutSFA layout_SFA = ScaleConfig::tile_atom_to_shape_SFA(make_shape(m, n, k, 1)); + LayoutSFB layout_SFB = ScaleConfig::tile_atom_to_shape_SFB(make_shape(m, n, k, 1)); + + typename GemmKernel::MainloopArguments mainloop_args{ + a_ptr, a_stride, b_ptr, b_stride, scales_a_ptr, layout_SFA, scales_b_ptr, layout_SFB}; + + typename GemmKernel::EpilogueArguments epilogue_args{{}, c_ptr, c_stride, c_ptr, c_stride}; + epilogue_args.thread.alpha = 1.0f; + + typename GemmKernel::Arguments args = { + cutlass::gemm::GemmUniversalMode::kGemm, {m, n, k, 1}, mainloop_args, epilogue_args}; + + auto can_implement = gemm_op.can_implement(args); + TORCH_CHECK(can_implement == cutlass::Status::kSuccess, cutlassGetStatusString(can_implement)) + + size_t workspace_size = gemm_op.get_workspace_size(args); + cutlass::device_memory::allocation workspace(workspace_size); + + auto init_status = gemm_op.initialize(args, workspace.get()); + TORCH_CHECK(init_status == cutlass::Status::kSuccess, cutlassGetStatusString(init_status)); + + auto stream = at::cuda::getCurrentCUDAStream(a.get_device()); + + auto status = gemm_op.run(stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, cutlassGetStatusString(status)) +} + +template +void sm100_fp8_blockwise_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b) { + if (a.size(0) <= 128) { + using MmaTileShape = Shape<_64, _128, _128>; + using PerSmTileShape = Shape<_64, _128, _128>; + using EpilogueTileShape = Shape<_64, _64>; + using ScalesPerTile = Shape<_64, _1, _1>; + launch_sm100_fp8_blockwise_scaled_mm( + out, a, b, scales_a, scales_b); + } else { + using MmaTileShape = Shape<_128, _128, _128>; + using PerSmTileShape = Shape<_128, _128, _128>; + using EpilogueTileShape = Shape<_128, _64>; + using ScalesPerTile = Shape<_128, _1, _1>; + launch_sm100_fp8_blockwise_scaled_mm( + out, a, b, scales_a, scales_b); + } +} + +template < + typename OutType, + typename MmaTileShape, + typename PerSmTileShape, + typename EpilogueTileShape, + typename ScalesPerTile, + int TileSizeM_ = 128, + class ClusterShape = Shape<_1, _1, _1>> +void launch_sm120_fp8_blockwise_scaled_mm( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b) { + using ElementBlockScale = float; + + // A matrix configuration + using ElementA = cutlass::float_e4m3_t; // Element type for A matrix operand + using LayoutATag = cutlass::layout::RowMajor; // Layout type for A matrix operand + constexpr int AlignmentA = + 128 / cutlass::sizeof_bits::value; // Memory access granularity/alignment of A matrix in units of + // elements (up to 16 bytes) + + // B matrix configuration + using ElementB = cutlass::float_e4m3_t; // Element type for B matrix operand + using LayoutBTag = cutlass::layout::ColumnMajor; // Layout type for B matrix operand + constexpr int AlignmentB = + 128 / cutlass::sizeof_bits::value; // Memory access granularity/alignment of B matrix in units of + // elements (up to 16 bytes) + + // C/D matrix configuration + using ElementD = OutType; // Element type for D matrix operand + using ElementC = void; // Element type for C matrix operand + using LayoutCTag = cutlass::layout::RowMajor; // Layout type for C matrix operand + using LayoutDTag = cutlass::layout::RowMajor; // Layout type for D matrix operand + constexpr int AlignmentD = + 128 / cutlass::sizeof_bits::value; // Memory access granularity/alignment of C matrix in units of + // elements (up to 16 bytes) + constexpr int AlignmentC = + AlignmentD; // Memory access granularity/alignment of C matrix in units of elements (up to 16 bytes) + + // Kernel functional config + using ElementAccumulator = float; // Element type for internal accumulation + using ArchTag = cutlass::arch::Sm120; // Tag indicating the minimum SM that supports the intended feature + using OperatorClass = cutlass::arch::OpClassTensorOp; // Operator class tag - changed from OpClassBlockScaledTensorOp + + static constexpr int ScaleMsPerTile = size<0>(ScalesPerTile{}); + static constexpr int ScaleGranularityM = size<0>(MmaTileShape{}) / ScaleMsPerTile; + static constexpr int ScaleGranularityN = size<1>(MmaTileShape{}) / size<1>(ScalesPerTile{}); + static constexpr int ScaleGranularityK = size<2>(MmaTileShape{}) / size<2>(ScalesPerTile{}); + + using ScaleConfig = cutlass::detail::Sm120BlockwiseScaleConfig< + ScaleGranularityM, + ScaleGranularityN, + ScaleGranularityK, + cute::UMMA::Major::MN, + cute::UMMA::Major::K>; + // FP8 Block-wise scaling configuration + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); // Layout type for SFA matrix operand + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); // Layout type for SFB matrix operand + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + PerSmTileShape, + ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementAccumulator, + ElementC, + LayoutCTag, + AlignmentC, + ElementD, + LayoutDTag, + AlignmentD, + cutlass::epilogue::collective::EpilogueScheduleAuto // Epilogue schedule policy + >::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementA, + cute::tuple, + AlignmentA, + ElementB, + cute::tuple, + AlignmentB, + ElementAccumulator, + MmaTileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + cutlass::gemm::collective::KernelScheduleAuto // Kernel schedule policy. Auto defaults to cooperative kernel + // schedule + >::CollectiveOp; + + using GemmKernel = cutlass::gemm::kernel::GemmUniversal< + Shape, // Indicates ProblemShape + CollectiveMainloop, + CollectiveEpilogue, + void>; + + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + + Gemm gemm_op; + + int m = a.size(0); + int k = a.size(1); + int n = b.size(1); + + auto a_ptr = static_cast(a.data_ptr()); + auto b_ptr = static_cast(b.data_ptr()); + auto c_ptr = static_cast(out.data_ptr()); + + auto scales_a_ptr = static_cast(scales_a.data_ptr()); + auto scales_b_ptr = static_cast(scales_b.data_ptr()); + + using StrideA = typename Gemm::GemmKernel::StrideA; + using StrideB = typename Gemm::GemmKernel::StrideB; + using StrideD = typename Gemm::GemmKernel::StrideD; + using StrideC = typename Gemm::GemmKernel::StrideD; + + StrideA stride_a = cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(m, k, 1)); + StrideB stride_b = cutlass::make_cute_packed_stride(StrideB{}, cute::make_shape(n, k, 1)); + StrideC stride_c = cutlass::make_cute_packed_stride(StrideC{}, cute::make_shape(m, n, 1)); + LayoutSFA layout_SFA = ScaleConfig::tile_atom_to_shape_SFA(make_shape(m, n, k, 1)); + LayoutSFB layout_SFB = ScaleConfig::tile_atom_to_shape_SFB(make_shape(m, n, k, 1)); + + typename GemmKernel::MainloopArguments mainloop_args{ + a_ptr, stride_a, b_ptr, stride_b, scales_a_ptr, layout_SFA, scales_b_ptr, layout_SFB}; + + typename GemmKernel::EpilogueArguments epilogue_args{{}, c_ptr, stride_c, c_ptr, stride_c}; + epilogue_args.thread.alpha = 1.0f; + + typename Gemm::Arguments args = { + cutlass::gemm::GemmUniversalMode::kGemm, + {m, n, k, 1}, + mainloop_args, + epilogue_args, + }; + + auto can_implement = gemm_op.can_implement(args); + TORCH_CHECK(can_implement == cutlass::Status::kSuccess, cutlassGetStatusString(can_implement)) + + size_t workspace_size = gemm_op.get_workspace_size(args); + cutlass::device_memory::allocation workspace(workspace_size); + + auto init_status = gemm_op.initialize(args, workspace.get()); + TORCH_CHECK(init_status == cutlass::Status::kSuccess, cutlassGetStatusString(init_status)); + + auto stream = at::cuda::getCurrentCUDAStream(a.get_device()); + auto status = gemm_op.run(stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, cutlassGetStatusString(status)) +} + +template +void sm120_fp8_blockwise_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b) { + using MmaTileShape = Shape<_128, _128, _128>; + using PerSmTileShape = Shape<_128, _128, _128>; + using EpilogueTileShape = Shape<_128, _64>; + using ScalesPerTile = Shape<_128, _1, _1>; + launch_sm120_fp8_blockwise_scaled_mm( + out, a, b, scales_a, scales_b); +} + +torch::Tensor fp8_blockwise_scaled_mm( + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Dtype& out_dtype) { + TORCH_CHECK(mat_a.is_cuda(), "mat_a must be a CUDA tensor"); + TORCH_CHECK(mat_b.is_cuda(), "mat_b must be a CUDA tensor"); + TORCH_CHECK(mat_a.dim() == 2, "mat_a must be a 2D tensor"); + TORCH_CHECK(mat_b.dim() == 2, "mat_b must be a 2D tensor"); + TORCH_CHECK(mat_a.stride(1) == 1, "mat_a must be a row major tensor"); + TORCH_CHECK(mat_b.stride(0) == 1, "mat_b must be a column major tensor"); + TORCH_CHECK(mat_a.size(1) == mat_b.size(0), "mat_a and mat_b shapes cannot be multiplied"); + + TORCH_CHECK( + (mat_a.size(1) * mat_a.element_size()) % 16 == 0, "mat_a must be multiple of 16 bytes for memory alignment"); + TORCH_CHECK( + (mat_b.size(0) * mat_b.element_size()) % 16 == 0, "mat_b must be multiple of 16 bytes for memory alignment"); + TORCH_CHECK(mat_a.scalar_type() == torch::kFloat8_e4m3fn, "mat_a must be Float8_e4m3fn"); + TORCH_CHECK(mat_b.scalar_type() == torch::kFloat8_e4m3fn, "mat_b must be Float8_e4m3fn"); + TORCH_CHECK(out_dtype == torch::kHalf || out_dtype == torch::kBFloat16, "out_dtype must be Half or BFloat16"); + + auto is_contiguous_vector = [](const torch::Tensor& t) { + auto t_sizes = t.sizes(); + return t.is_contiguous() && + (t.dim() == 1 || (t.dim() == 2 && *std::min_element(t_sizes.begin(), t_sizes.end()) == 1)); + }; + + TORCH_CHECK(mat_a.size(0) == scales_a.size(0), "size of scales_a is not matched"); + TORCH_CHECK(mat_a.size(1) / 128 == scales_a.size(1), "size of scales_a is not matched"); + TORCH_CHECK(scales_a.stride(0) == 1 || is_contiguous_vector(scales_a), "scales_a must be M major"); + TORCH_CHECK(mat_b.size(0) / 128 == scales_b.size(0), "size of scales_b is not matched"); + TORCH_CHECK(mat_b.size(1) / 128 == scales_b.size(1), "size of scales_b is not matched"); + TORCH_CHECK(scales_b.stride(0) == 1 || is_contiguous_vector(scales_b), "scales_b must be K major"); + TORCH_CHECK(scales_a.scalar_type() == torch::kFloat32, "scales_a must be Float32"); + TORCH_CHECK(scales_b.scalar_type() == torch::kFloat32, "scales_b must be Float32"); + + torch::Tensor out = torch::empty({mat_a.size(0), mat_b.size(1)}, mat_a.options().dtype(out_dtype)); + TORCH_CHECK((out.size(1) * out.element_size()) % 16 == 0, "out must be multiple of 16 bytes for memory alignment"); + + auto sm_version = getSMVersion(); + + int64_t original_rows = mat_a.size(0); + torch::Tensor mat_a_padded = pad_tensor(mat_a, /*alignment=*/4); + torch::Tensor scales_a_padded = pad_tensor(scales_a, /*alignment=*/4, /*col_major=*/true); + torch::Tensor out_padded = torch::empty({mat_a_padded.size(0), mat_b.size(1)}, out.options()); + +#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED) +#if defined CUDA_VERSION && CUDA_VERSION >= 12000 + if (sm_version == 90) { + torch::Tensor scales_b_contiguous = scales_b.contiguous(); + if (out_dtype == torch::kBFloat16) { + cutlass_gemm_blockwise_sm90_fp8_dispatch( + out_padded, mat_a_padded, mat_b, scales_a_padded, scales_b_contiguous); + } else { + cutlass_gemm_blockwise_sm90_fp8_dispatch( + out_padded, mat_a_padded, mat_b, scales_a_padded, scales_b_contiguous); + } + return out_padded.slice(0, 0, original_rows); + } +#endif +#endif + +#if defined(CUTLASS_ARCH_MMA_SM100A_SUPPORTED) || defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) +#if defined CUDA_VERSION && CUDA_VERSION >= 12080 + if (sm_version == 100 +#if CUDA_VERSION >= 12090 + || sm_version == 103 +#endif + ) { + if (out_dtype == torch::kBFloat16) { + sm100_fp8_blockwise_dispatch_shape( + out_padded, mat_a_padded, mat_b, scales_a_padded, scales_b); + } else { + sm100_fp8_blockwise_dispatch_shape(out_padded, mat_a_padded, mat_b, scales_a_padded, scales_b); + } + return out_padded.slice(0, 0, original_rows); + } +#endif +#endif + +#if defined(CUTLASS_ARCH_MMA_SM120A_SUPPORTED) || defined(CUTLASS_ARCH_MMA_SM120_SUPPORTED) +#if defined(CUDA_VERSION) && CUDA_VERSION >= 12080 + if (sm_version == 120) { + if (out_dtype == torch::kBFloat16) { + sm120_fp8_blockwise_dispatch_shape( + out_padded, mat_a_padded, mat_b, scales_a_padded, scales_b); + } else { + sm120_fp8_blockwise_dispatch_shape(out_padded, mat_a_padded, mat_b, scales_a_padded, scales_b); + } + return out_padded.slice(0, 0, original_rows); + } +#endif +#endif + + TORCH_CHECK_NOT_IMPLEMENTED( + false, "No implemented fp8_blockwise_scaled_mm for current compute capability: ", sm_version); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/fp8_gemm_kernel.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/fp8_gemm_kernel.cu new file mode 100644 index 00000000..dea053e2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/fp8_gemm_kernel.cu @@ -0,0 +1,1532 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Adapted from +// https://github.com/NVIDIA/TensorRT-LLM/blob/v0.16.0/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_rowwise_gemm/fp8_rowwise_gemm_template.h +// https://github.com/NVIDIA/TensorRT-LLM/blob/v0.16.0/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_rowwise_gemm/fp8_rowwise_gemm_kernel_template_sm89.h +// https://github.com/NVIDIA/TensorRT-LLM/blob/v0.16.0/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_rowwise_gemm/fp8_rowwise_gemm_kernel_template_sm90.h + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "math.hpp" +#include "utils.h" + +using namespace cute; + +#if defined CUDA_VERSION && CUDA_VERSION >= 12040 +template < + typename ElementType, + typename OutElementType, + typename AccumElementType, + typename CtaShape, + typename WarpShape, + int Stages, + bool WithBias, + typename FP8MathOperator = cutlass::arch::OpMultiplyAdd, + template typename EpilogueVisitor = cutlass::epilogue::threadblock::Sm80EVT, + typename ThreadblockSwizzle = cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>> +struct DeviceGemmFp8RowwiseSm89 { + static_assert(std::is_same_v, "ElementType must be FP8(e4m3)"); + + using ElementA = ElementType; + using LayoutA = cutlass::layout::RowMajor; + static constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; + + using ElementB = ElementType; + using LayoutB = cutlass::layout::ColumnMajor; + static constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; + + using ElementC = OutElementType; + using LayoutC = cutlass::layout::RowMajor; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + + using ElementOutput = OutElementType; + using LayoutOutput = cutlass::layout::RowMajor; + static constexpr int AlignmentOutput = 128 / cutlass::sizeof_bits::value; + + using ElementAccumulator = AccumElementType; + using ElementComputeEpilogue = float; + using ArchTag = cutlass::arch::Sm89; + using OperatorClass = cutlass::arch::OpClassTensorOp; + + using InstructionShape = cutlass::gemm::GemmShape<16, 8, 32>; + // Number of epilogue stages in EVT + static constexpr int EVTEpilogueStages = 1; + + using OutputTileThreadMap = cutlass::epilogue::threadblock:: + OutputTileThreadLayout; + + // Definition of EVT + using accSrc = cutlass::epilogue::threadblock::VisitorAccFetch; + + using ComputeBScale = cutlass::epilogue::threadblock::VisitorCompute< + cutlass::multiplies, + ElementComputeEpilogue, + ElementComputeEpilogue, + cutlass::FloatRoundStyle::round_to_nearest>; + using bScaleSrc = cutlass::epilogue::threadblock:: + VisitorRowBroadcast>; + using EpilogueBScale = cutlass::epilogue::threadblock::Sm80EVT; + + using ComputeAScale = cutlass::epilogue::threadblock:: + VisitorCompute; + using aScaleSrc = cutlass::epilogue::threadblock:: + VisitorColBroadcast>; + using EpilogueAScale = cutlass::epilogue::threadblock::Sm80EVT; + + // With bias + using biasSrc = + cutlass::epilogue::threadblock::VisitorRowBroadcast>; + using ComputeAScaleWithBias = cutlass::epilogue::threadblock::VisitorCompute< + cutlass::multiply_add, + ElementC, + ElementComputeEpilogue, + cutlass::FloatRoundStyle::round_to_nearest>; + using EpilogueAScaleWithBias = + cutlass::epilogue::threadblock::Sm80EVT; + + using dTar = cutlass::epilogue::threadblock::VisitorAuxStore< + OutputTileThreadMap, + ElementC, + cutlass::FloatRoundStyle::round_to_nearest, + Stride>; + using EpilogueStore = typename cutlass::platform::conditional< + WithBias, + cutlass::epilogue::threadblock::Sm80EVT, + cutlass::epilogue::threadblock::Sm80EVT>::type; + + using EpilogueOp = EpilogueStore; + + using GemmKernel = typename cutlass::gemm::kernel::DefaultGemmWithVisitor< + ElementA, + LayoutA, + cutlass::ComplexTransform::kNone, + AlignmentA, + ElementB, + LayoutB, + cutlass::ComplexTransform::kNone, + AlignmentB, + ElementC, + LayoutC, + AlignmentC, + ElementAccumulator, + ElementComputeEpilogue, + OperatorClass, + ArchTag, + CtaShape, + WarpShape, + InstructionShape, + EpilogueOp, + ThreadblockSwizzle, + Stages, + FP8MathOperator, + EVTEpilogueStages>::GemmKernel; + + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; +}; + +template +typename Gemm::Arguments prepare_sm89_fp8_args( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + using ElementT = typename Gemm::ElementA; + using ElementOutput = typename Gemm::ElementD; + using ElementComputeEpilogue = float; + + int32_t m = a.size(0); + int32_t n = b.size(1); + int32_t k = a.size(1); + + int64_t lda = a.stride(0); + int64_t ldb = b.stride(1); + int64_t ldc = out.stride(0); + + ElementT const* ptr_a = reinterpret_cast(a.data_ptr()); + ElementT const* ptr_b = reinterpret_cast(b.data_ptr()); + ElementOutput const* ptr_bias = nullptr; + if constexpr (WithBias) { + TORCH_CHECK(bias.has_value()) + ptr_bias = reinterpret_cast(bias.value().data_ptr()); + } + ElementOutput* ptr_d = reinterpret_cast(out.data_ptr()); + ElementComputeEpilogue const* ptr_scales_a = reinterpret_cast(scales_a.data_ptr()); + ElementComputeEpilogue const* ptr_scales_b = reinterpret_cast(scales_b.data_ptr()); + + typename Gemm::Arguments args( + cutlass::gemm::GemmUniversalMode::kGemm, // Mode + {m, n, k}, // Problem size + 1, // Split-k factor + {}, // Epilogue args + ptr_a, // a pointer + ptr_b, // b pointer + nullptr, // c pointer (unused) + nullptr, // d pointer (unused) + m * k, // batch stride a (unused) + n * k, // batch stride b (unused) + m * n, // batch stride c (unused) + m * n, // batch stride d (unused) + lda, // stride a + ldb, // stride b + ldc, // stride c (unused) + ldc); // stride d (unused) + if constexpr (WithBias) { + args.epilogue = { + { + { + {}, // Accumulator + {ptr_scales_b, ElementComputeEpilogue(0), {_0{}, _1{}, _0{}}}, + {} // Multiplies + }, + {ptr_scales_a, ElementComputeEpilogue(0), {_1{}, _0{}, _0{}}}, + {ptr_bias, ElementOutput(0), {_0{}, _1{}, _0{}}}, + {} // Multiplies + }, + {ptr_d, {n, _1{}, _0{}}}}; + } else { + args.epilogue = { + { + { + {}, // Accumulator + {ptr_scales_b, ElementComputeEpilogue(0), {_0{}, _1{}, _0{}}}, + {} // Multiplies + }, + {ptr_scales_a, ElementComputeEpilogue(0), {_1{}, _0{}, _0{}}}, + {} // Multiplies + }, + {ptr_d, {n, _1{}, _0{}}}}; + } + + return args; +} + +template +void launch_sm89_fp8_scaled_mm( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + auto args = prepare_sm89_fp8_args(out, a, b, scales_a, scales_b, bias); + Gemm gemm_op; + + size_t workspace_size = gemm_op.get_workspace_size(args); + auto const workspace_options = torch::TensorOptions().dtype(torch::kUInt8).device(a.device()); + auto workspace = torch::empty(workspace_size, workspace_options); + auto stream = at::cuda::getCurrentCUDAStream(a.get_device()); + + auto can_implement = gemm_op.can_implement(args); + TORCH_CHECK(can_implement == cutlass::Status::kSuccess) + + auto status = gemm_op(args, workspace.data_ptr(), stream); + TORCH_CHECK(status == cutlass::Status::kSuccess) +} + +template +void sm89_fp8_dispatch_bias( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + using ElementInput = cutlass::float_e4m3_t; + using ElementOutput = OutType; + using AccumElementType = float; + if (bias) { + using Gemm = typename DeviceGemmFp8RowwiseSm89< + ElementInput, + ElementOutput, + AccumElementType, + CtaShape, + WarpShape, + Stages, + true>::Gemm; + return launch_sm89_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } else { + using Gemm = typename DeviceGemmFp8RowwiseSm89< + ElementInput, + ElementOutput, + AccumElementType, + CtaShape, + WarpShape, + Stages, + false>::Gemm; + return launch_sm89_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } +} + +template +void sm89_fp8_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + uint32_t const m = a.size(0); + uint32_t const n = out.size(1); + + if (m == 1) { + if (n <= 8192) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<16, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 7>(out, a, b, scales_a, scales_b, bias); + } else { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<32, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 5>(out, a, b, scales_a, scales_b, bias); + } + } else if (m <= 16) { + // M in (1, 16] + if (n <= 8192) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<16, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 4>(out, a, b, scales_a, scales_b, bias); + } else if (n <= 16384) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<32, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 5>(out, a, b, scales_a, scales_b, bias); + } else { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<16, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 7>(out, a, b, scales_a, scales_b, bias); + } + } else if (m <= 64) { + // M in (16, 64] + if (n <= 16384) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<32, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 7>(out, a, b, scales_a, scales_b, bias); + } else { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<16, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 7>(out, a, b, scales_a, scales_b, bias); + } + } else if (m <= 128) { + // M in (64, 128] + if (n <= 8192) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<64, 64, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + 4>(out, a, b, scales_a, scales_b, bias); + } else if (n <= 16384) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<64, 64, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + 5>(out, a, b, scales_a, scales_b, bias); + } else { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<32, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 5>(out, a, b, scales_a, scales_b, bias); + } + } else if (m <= 256) { + // M in (128, 256] + if (n <= 8192) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<128, 64, 64>, + cutlass::gemm::GemmShape<64, 32, 64>, + 5>(out, a, b, scales_a, scales_b, bias); + } else if (n <= 16384) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<64, 128, 64>, + cutlass::gemm::GemmShape<64, 32, 64>, + 7>(out, a, b, scales_a, scales_b, bias); + } else { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<128, 64, 128>, + cutlass::gemm::GemmShape<64, 32, 128>, + 4>(out, a, b, scales_a, scales_b, bias); + } + } else if (m <= 512) { + // M in (256, 512) + if (n <= 16384) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 32, 64>, + 2>(out, a, b, scales_a, scales_b, bias); + } else { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 32, 64>, + 4>(out, a, b, scales_a, scales_b, bias); + } + } else { + // M in (512, inf) + if (n <= 8192) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 32, 64>, + 3>(out, a, b, scales_a, scales_b, bias); + } else { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 32, 64>, + 2>(out, a, b, scales_a, scales_b, bias); + } + } +} +#endif + +#if defined CUDA_VERSION && CUDA_VERSION >= 12000 +template < + typename ElementType, + typename OutElementType, + typename AccumElementType, + typename CTAShape, + typename ClusterShape, + typename MainloopScheduleType, + typename EpilogueScheduleType, + typename TileSchedulerType = void, + bool WithBias = false> +struct DeviceGemmFp8RowwiseSm90 { + static_assert(std::is_same_v, "ElementType must be FP8(e4m3)"); + + // A matrix configuration + using ElementA = ElementType; // Element type for A matrix operand + using LayoutA = cutlass::layout::RowMajor; // Layout type for A matrix operand + static constexpr int AlignmentA = + 128 / cutlass::sizeof_bits::value; // Memory access granularity/alignment of A + // matrix in units of elements (up to 16 bytes) + + // B matrix configuration + using ElementB = ElementType; // Element type for B matrix operand + using LayoutB = cutlass::layout::ColumnMajor; // Layout type for B matrix operand + static constexpr int AlignmentB = + 128 / cutlass::sizeof_bits::value; // Memory access granularity/alignment of B + // matrix in units of elements (up to 16 bytes) + + // C/D matrix configuration + using ElementC = void; // Element type for C matrix operands + using LayoutC = cutlass::layout::RowMajor; // Layout type for C matrix operands + static constexpr int AlignmentC = + 128 / cutlass::sizeof_bits::value; // Memory access granularity/alignment of C matrices in + // units of elements (up to 16 bytes) + + // Output matrix configuration + using ElementOutput = OutElementType; // Element type for output matrix operands + using LayoutOutput = cutlass::layout::RowMajor; // Layout type for output matrix operands + static constexpr int AlignmentOutput = 128 / cutlass::sizeof_bits::value; + + // // Auxiliary matrix configuration and other fusion types + // using ElementBias = float; + + // Multiply-accumulate blocking/pipelining details + using ElementAccumulator = AccumElementType; // Element type for internal accumulation + using ElementCompute = float; // Element type for compute + using ElementComputeEpilogue = float; + using ArchTag = cutlass::arch::Sm90; // Tag indicating the minimum SM that supports the intended feature + using OperatorClass = cutlass::arch::OpClassTensorOp; // Operator class tag + using TileShape = CTAShape; // Threadblock-level tile size + + static constexpr bool PONG = false; + static constexpr bool FAST_ACCUM = true; + static constexpr bool USE_BIAS = false; + + using StageCountType = cutlass::gemm::collective::StageCountAuto; // Stage count maximized + // based on the tile size + using KernelSchedule = cutlass::gemm::collective::KernelScheduleAuto; // Kernel to launch based on the default + // setting in the Collective Builder + // Implement rowwise scaling epilogue. + using XScale = cutlass::epilogue::fusion::Sm90ColBroadcast< + 0, + TileShape, + ElementComputeEpilogue, + ElementComputeEpilogue, + cute::Stride, cute::Int<0>, cute::Int<0>>>; + + using WScale = cutlass::epilogue::fusion::Sm90RowBroadcast< + 0, + TileShape, + ElementComputeEpilogue, + ElementComputeEpilogue, + cute::Stride, cute::Int<1>, cute::Int<0>>>; + + using Bias = cutlass::epilogue::fusion::Sm90RowBroadcast< + 0, + TileShape, + ElementOutput, + ElementOutput, + cute::Stride, cute::Int<1>, cute::Int<0>>>; + + using Accum = cutlass::epilogue::fusion::Sm90AccFetch; + + using Compute0 = cutlass::epilogue::fusion::Sm90Compute< + cutlass::multiplies, + ElementComputeEpilogue, // First stage output type. + ElementComputeEpilogue, // First stage input types. + cutlass::FloatRoundStyle::round_to_nearest>; + + using EVTCompute0 = cutlass::epilogue::fusion::Sm90EVT; + + using Compute1 = cutlass::epilogue::fusion::Sm90Compute< + cutlass::multiplies, + ElementOutput, + ElementComputeEpilogue, // Second stage input types. + cutlass::FloatRoundStyle::round_to_nearest>; + + using EVTCompute1 = cutlass::epilogue::fusion::Sm90EVT; + + // With bias + using ComputeWithBias = cutlass::epilogue::fusion::Sm90Compute< + cutlass::multiply_add, + ElementOutput, + ElementComputeEpilogue, + cutlass::FloatRoundStyle::round_to_nearest>; + using EVTComputeWithBias = cutlass::epilogue::fusion::Sm90EVT; + + using EpilogueEVT = typename cutlass::platform::conditional::type; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + cutlass::arch::Sm90, + cutlass::arch::OpClassTensorOp, + TileShape, + ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementComputeEpilogue, + ElementC, + LayoutC, + AlignmentC, + ElementOutput, + LayoutOutput, + AlignmentOutput, + cutlass::epilogue::TmaWarpSpecialized, + EpilogueEVT>::CollectiveOp; + + using DefaultSchedule = cutlass::gemm::KernelTmaWarpSpecialized; + using PongSchedule = cutlass::gemm::KernelTmaWarpSpecializedPingpong; + using FastDefaultSchedule = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum; + using FastPongSchedule = cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum; + + using SlowAccum = DefaultSchedule; + using FastAccum = FastPongSchedule; // Default apply Pingpong + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementA, + LayoutA, + AlignmentA, + ElementB, + LayoutB, + AlignmentB, + ElementAccumulator, + TileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + MainloopScheduleType>::CollectiveOp; + + using GemmKernel = cutlass::gemm::kernel::GemmUniversal< + Shape, // Indicates ProblemShape + CollectiveMainloop, + CollectiveEpilogue, + TileSchedulerType>; + + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; +}; + +template +typename Gemm::Arguments prepare_sm90_fp8_args( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + using ElementT = typename Gemm::ElementA; + using ElementOutput = typename Gemm::ElementD; + using ElementComputeEpilogue = float; + using StrideA = typename Gemm::GemmKernel::StrideA; + using StrideB = typename Gemm::GemmKernel::StrideB; + using StrideC = typename Gemm::GemmKernel::StrideC; + using StrideD = typename Gemm::GemmKernel::StrideD; + + int32_t m = a.size(0); + int32_t n = b.size(1); + int32_t k = a.size(1); + ElementT const* ptr_a = reinterpret_cast(a.data_ptr()); + ElementT const* ptr_b = reinterpret_cast(b.data_ptr()); + ElementOutput const* ptr_bias = nullptr; + if constexpr (WithBias) { + TORCH_CHECK(bias.has_value()) + ptr_bias = reinterpret_cast(bias.value().data_ptr()); + } + ElementOutput* ptr_d = reinterpret_cast(out.data_ptr()); + ElementComputeEpilogue const* ptr_scales_a = reinterpret_cast(scales_a.data_ptr()); + ElementComputeEpilogue const* ptr_scales_b = reinterpret_cast(scales_b.data_ptr()); + + StrideA stride_a = cutlass::make_cute_packed_stride(StrideA{}, make_shape(m, k, 1)); + StrideB stride_b = cutlass::make_cute_packed_stride(StrideB{}, make_shape(n, k, 1)); + StrideC stride_c; + StrideD stride_d = cutlass::make_cute_packed_stride(StrideD{}, make_shape(m, n, 1)); + typename Gemm::Arguments args = { + cutlass::gemm::GemmUniversalMode::kGemm, + {m, n, k, 1}, + {ptr_a, stride_a, ptr_b, stride_b}, + {{}, // epilogue.thread + nullptr, + stride_c, + ptr_d, + stride_d}}; + if constexpr (WithBias) { + args.epilogue.thread = { + {ptr_scales_a}, + { + {ptr_scales_b}, + {}, // Accumulator + {} // Multiplies + }, + {ptr_bias}, + {}, // Multiplies + }; + } else { + args.epilogue.thread = { + {ptr_scales_a}, + { + {ptr_scales_b}, + {}, // Accumulator + {} // Multiplies + }, + {}, // Multiplies + }; + } + + return args; +} + +template +void launch_sm90_fp8_scaled_mm( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + auto args = prepare_sm90_fp8_args(out, a, b, scales_a, scales_b, bias); + Gemm gemm_op; + + size_t workspace_size = gemm_op.get_workspace_size(args); + auto const workspace_options = torch::TensorOptions().dtype(torch::kUInt8).device(a.device()); + auto workspace = torch::empty(workspace_size, workspace_options); + auto stream = at::cuda::getCurrentCUDAStream(a.get_device()); + + auto can_implement = gemm_op.can_implement(args); + TORCH_CHECK(can_implement == cutlass::Status::kSuccess) + + auto status = gemm_op.run(args, workspace.data_ptr(), stream); + + TORCH_CHECK(status == cutlass::Status::kSuccess) +} + +template < + typename OutType, + typename CTAShape, + typename ClusterShape, + typename MainloopScheduleType, + typename TileSchedulerType> +void sm90_fp8_dispatch_bias( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias, + bool fast_accum = true, + bool use_persistent = false) { + using ElementInput = cutlass::float_e4m3_t; + using ElementOutput = OutType; + using AccumElementType = float; + using EpilogueScheduleType = cutlass::epilogue::TmaWarpSpecialized; + + if (bias) { + using Gemm = typename DeviceGemmFp8RowwiseSm90< + ElementInput, + ElementOutput, + AccumElementType, + CTAShape, + ClusterShape, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + true>::Gemm; + return launch_sm90_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } else { + using Gemm = typename DeviceGemmFp8RowwiseSm90< + ElementInput, + ElementOutput, + AccumElementType, + CTAShape, + ClusterShape, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + false>::Gemm; + return launch_sm90_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } +} + +template +void sm90_fp8_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + uint32_t const m = a.size(0); + using FastPingpongScheduler = cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum; + using FastBasicScheduler = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum; + using PersistentTileScheduler = cutlass::gemm::PersistentScheduler; + using BasicTileScheduler = void; + if (m <= 1) { + return sm90_fp8_dispatch_bias< + OutType, + Shape<_64, _64, _128>, + Shape<_1, _8, _1>, + FastBasicScheduler, + BasicTileScheduler>(out, a, b, scales_a, scales_b, bias); + } + if (m <= 64) { + // m in [1, 64] + return sm90_fp8_dispatch_bias< + OutType, + Shape<_64, _64, _128>, + Shape<_1, _4, _1>, + FastPingpongScheduler, + PersistentTileScheduler>(out, a, b, scales_a, scales_b, bias); + } else if (m <= 256) { + // m in (64, 256] + return sm90_fp8_dispatch_bias< + OutType, + Shape<_64, _64, _128>, + Shape<_1, _1, _1>, + FastPingpongScheduler, + PersistentTileScheduler>(out, a, b, scales_a, scales_b, bias); + } else if (m <= 1024) { + // m in (256, 1024] + return sm90_fp8_dispatch_bias< + OutType, + Shape<_128, _128, _128>, + Shape<_1, _1, _1>, + FastPingpongScheduler, + PersistentTileScheduler>(out, a, b, scales_a, scales_b, bias); + } else { + // m in (1024, inf) + return sm90_fp8_dispatch_bias< + OutType, + Shape<_128, _128, _128>, + Shape<_2, _1, _1>, + FastPingpongScheduler, + PersistentTileScheduler>(out, a, b, scales_a, scales_b, bias); + } +} +#endif + +#if defined CUDA_VERSION && CUDA_VERSION >= 12080 +template < + typename ElementType, + typename OutElementType, + typename AccumElementType, + typename CTAShape, + typename ClusterShape, + typename MainloopScheduleType, + typename EpilogueScheduleType, + typename TileSchedulerType = void, + bool WithBias = false> +struct DeviceGemmFp8RowwiseSm100 { + static_assert(std::is_same_v, "ElementType must be FP8(e4m3)"); + using TileShape = CTAShape; + using Accum = cutlass::epilogue::fusion::Sm90AccFetch; + + using ElementComputeEpilogue = float; + using ScaleA = cutlass::epilogue::fusion::Sm90ColBroadcast< + 0, + TileShape, + ElementComputeEpilogue, + ElementComputeEpilogue, + cute::Stride, cute::Int<0>, cute::Int<0>>>; + + using ScaleB = cutlass::epilogue::fusion::Sm90RowBroadcast< + 0, + TileShape, + ElementComputeEpilogue, + ElementComputeEpilogue, + cute::Stride, cute::Int<1>, cute::Int<0>>>; + + using Bias = cutlass::epilogue::fusion::Sm90RowBroadcast< + 0, + TileShape, + OutElementType, + OutElementType, + cute::Stride, cute::Int<1>, cute::Int<0>>>; + + using Compute0 = cutlass::epilogue::fusion:: + Sm90Compute; + + using EVTCompute0 = cutlass::epilogue::fusion::Sm90EVT; + + using LayoutA = cutlass::layout::RowMajor; + static constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; + + using LayoutB = cutlass::layout::ColumnMajor; + static constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; + + using ElementC = void; + using LayoutC = cutlass::layout::RowMajor; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + + using LayoutD = cutlass::layout::RowMajor; + static constexpr int AlignmentD = AlignmentC; + + using Compute1MulAdd = cutlass::epilogue::fusion:: + Sm90Compute; + using Compute1Mul = cutlass::epilogue::fusion:: + Sm90Compute; + + using EVTCompute = typename std::conditional_t< + WithBias, + cutlass::epilogue::fusion::Sm90EVT, + cutlass::epilogue::fusion::Sm90EVT>; + using ArgumentType = typename EVTCompute::Arguments; + // MMA type + using ElementAccumulator = AccumElementType; + + // Epilogue types + using ElementCompute = float; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + cutlass::arch::Sm100, + cutlass::arch::OpClassTensorOp, + TileShape, + ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementCompute, + ElementC, + LayoutC, + AlignmentC, + OutElementType, + LayoutD, + AlignmentD, + EpilogueScheduleType, + EVTCompute>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + cutlass::arch::Sm100, + cutlass::arch::OpClassTensorOp, + ElementType, + LayoutA, + AlignmentA, + ElementType, + LayoutB, + AlignmentB, + ElementAccumulator, + TileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + MainloopScheduleType>::CollectiveOp; + using GemmKernel = + cutlass::gemm::kernel::GemmUniversal, CollectiveMainloop, CollectiveEpilogue, void>; + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + template + static auto args_from_tensor(torch::Tensor const& tensor) { + using Arguments = typename Descriptor::Arguments; + auto* data_ptr = static_cast(tensor.data_ptr()); + static_assert( + std::is_same_v || std::is_same_v || std::is_same_v); + return Arguments{data_ptr}; + } + + public: + static ArgumentType prepare_args( + torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + std::optional const& bias = std::nullopt) { + auto a_args = args_from_tensor(a_scales); + auto b_args = args_from_tensor(b_scales); + + typename EVTCompute0::Arguments evt0_args{b_args, {}, {}}; + + if constexpr (WithBias) { + auto bias_args = args_from_tensor(bias.value()); + return ArgumentType{a_args, evt0_args, bias_args, {}}; + } else { + return ArgumentType{a_args, evt0_args, {}}; + } + } +}; + +template +typename GemmType::Gemm::Arguments prepare_sm100_fp8_args( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + using Gemm = typename GemmType::Gemm; + using ElementT = typename Gemm::ElementA; + using ElementC = typename Gemm::ElementC; + using ElementOutput = typename Gemm::ElementD; + using ElementComputeEpilogue = float; + using GemmKernel = typename Gemm::GemmKernel; + + using StrideA = typename Gemm::GemmKernel::StrideA; + using StrideB = typename Gemm::GemmKernel::StrideB; + using StrideC = typename Gemm::GemmKernel::StrideC; + using StrideD = StrideC; + using StrideAux = StrideC; + + int32_t m = a.size(0); + int32_t n = b.size(1); + int32_t k = a.size(1); + + ElementT const* ptr_a = reinterpret_cast(a.data_ptr()); + ElementT const* ptr_b = reinterpret_cast(b.data_ptr()); + + StrideA stride_a = cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(m, k, 1)); + StrideB stride_b = cutlass::make_cute_packed_stride(StrideB{}, cute::make_shape(n, k, 1)); + StrideC stride_c = cutlass::make_cute_packed_stride(StrideC{}, cute::make_shape(m, n, 1)); + StrideD stride_d = cutlass::make_cute_packed_stride(StrideD{}, cute::make_shape(m, n, 1)); + StrideAux aux_stride = stride_d; + + typename GemmKernel::MainloopArguments mainloop_args{ptr_a, stride_a, ptr_b, stride_b}; + + typename GemmKernel::ProblemShape prob_shape = {m, n, k, 1}; + cutlass::KernelHardwareInfo hw_info; + typename GemmKernel::TileSchedulerArguments scheduler = {}; + + auto ptr_c = static_cast(out.data_ptr()); + + auto prepare_epilogue_args = [&](const c10::optional& bias = c10::nullopt) { + if constexpr (WithBias) { + TORCH_CHECK(bias.has_value(), "Bias tensor is required but not provided."); + return typename GemmKernel::EpilogueArguments{ + GemmType::prepare_args(scales_a, scales_b, bias.value()), ptr_c, stride_c, ptr_c, stride_d}; + } else { + return typename GemmKernel::EpilogueArguments{ + GemmType::prepare_args(scales_a, scales_b), ptr_c, stride_c, ptr_c, stride_d}; + } + }; + + typename GemmKernel::Arguments args{ + cutlass::gemm::GemmUniversalMode::kGemm, + prob_shape, + mainloop_args, + prepare_epilogue_args(bias), + hw_info, + scheduler}; + return args; +} + +template +void launch_sm100_fp8_scaled_mm( + torch::Tensor& out, + torch::Tensor const& a, + torch::Tensor const& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + auto args = prepare_sm100_fp8_args(out, a, b, scales_a, scales_b, bias); + + typename Gemm::Gemm gemm_op; + size_t workspace_size = gemm_op.get_workspace_size(args); + auto const workspace_options = torch::TensorOptions().dtype(torch::kUInt8).device(a.device()); + auto workspace = torch::empty(workspace_size, workspace_options); + auto stream = at::cuda::getCurrentCUDAStream(a.get_device()); + auto can_implement = gemm_op.can_implement(args); + TORCH_CHECK(can_implement == cutlass::Status::kSuccess) + auto status = gemm_op.run(args, workspace.data_ptr(), stream); + TORCH_CHECK(status == cutlass::Status::kSuccess) +} + +template +void sm100_fp8_dispatch_bias( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + using CTAShapeDefault = Shape<_256, _128, _64>; + using ClusterShapeDefault = Shape<_2, _2, _1>; + + using CTAShape256 = Shape<_128, _128, _128>; + using ClusterShape256 = Shape<_2, _1, _1>; + + using CTAShape64 = Shape<_64, _64, _128>; + using ClusterShape64 = Shape<_1, _1, _1>; + + using CTAShape16 = Shape<_64, _64, _128>; + using ClusterShape16 = Shape<_1, _4, _1>; + + using MainloopScheduleType = cutlass::gemm::collective::KernelScheduleAuto; + using EpilogueScheduleType = cutlass::epilogue::collective::EpilogueScheduleAuto; + using TileSchedulerType = void; + + using ElementInput = cutlass::float_e4m3_t; + using ElementOutput = OutType; + using AccumElementType = float; + + // Gemm type with bias + using BiasGemmDefault = DeviceGemmFp8RowwiseSm100< + ElementInput, + ElementOutput, + AccumElementType, + CTAShapeDefault, + ClusterShapeDefault, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + true>; + using BiasGemm256 = DeviceGemmFp8RowwiseSm100< + ElementInput, + ElementOutput, + AccumElementType, + CTAShape256, + ClusterShape256, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + true>; + using BiasGemm64 = DeviceGemmFp8RowwiseSm100< + ElementInput, + ElementOutput, + AccumElementType, + CTAShape64, + ClusterShape64, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + true>; + using BiasGemm16 = DeviceGemmFp8RowwiseSm100< + ElementInput, + ElementOutput, + AccumElementType, + CTAShape16, + ClusterShape16, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + true>; + + // Gemm type without bias + using GemmDefault = DeviceGemmFp8RowwiseSm100< + ElementInput, + ElementOutput, + AccumElementType, + CTAShapeDefault, + ClusterShapeDefault, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + false>; + using Gemm256 = DeviceGemmFp8RowwiseSm100< + ElementInput, + ElementOutput, + AccumElementType, + CTAShape256, + ClusterShape256, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + false>; + using Gemm64 = DeviceGemmFp8RowwiseSm100< + ElementInput, + ElementOutput, + AccumElementType, + CTAShape64, + ClusterShape64, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + false>; + using Gemm16 = DeviceGemmFp8RowwiseSm100< + ElementInput, + ElementOutput, + AccumElementType, + CTAShape16, + ClusterShape16, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + false>; + + // next power of 2 (minimum 16) + uint32_t const m = a.size(0); + uint32_t const mp2 = std::max(static_cast(16), next_pow_2(m)); + + if (bias) { + if (mp2 <= 16) { + // m in [1, 16] + return launch_sm100_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } else if (mp2 <= 64) { + // m in (16, 64] + return launch_sm100_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } else if (mp2 <= 256) { + // m in (64, 256] + return launch_sm100_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } else { + // m in (256, inf] + return launch_sm100_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } + } else { + if (mp2 <= 16) { + // m in [1, 16] + return launch_sm100_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } else if (mp2 <= 64) { + // m in (16, 64] + return launch_sm100_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } else if (mp2 <= 256) { + // m in (64, 256] + return launch_sm100_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } else { + return launch_sm100_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } + } +} + +template +void sm100_fp8_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + return sm100_fp8_dispatch_bias(out, a, b, scales_a, scales_b, bias); +} + +template < + typename ElementType, + typename OutElementType, + typename AccumElementType, + typename CTAShape, + typename ClusterShape, + typename MainloopScheduleType, + typename EpilogueScheduleType, + typename TileSchedulerType = void, + bool WithBias = false> +struct DeviceGemmFp8RowwiseSm120 { + static_assert(std::is_same_v, "ElementType must be FP8(e4m3)"); + using TileShape = CTAShape; + using Accum = cutlass::epilogue::fusion::Sm90AccFetch; + + using ElementComputeEpilogue = float; + using ScaleA = cutlass::epilogue::fusion::Sm90ColBroadcast< + 0, + TileShape, + ElementComputeEpilogue, + ElementComputeEpilogue, + cute::Stride, cute::Int<0>, cute::Int<0>>>; + + using ScaleB = cutlass::epilogue::fusion::Sm90RowBroadcast< + 0, + TileShape, + ElementComputeEpilogue, + ElementComputeEpilogue, + cute::Stride, cute::Int<1>, cute::Int<0>>>; + + using Bias = cutlass::epilogue::fusion::Sm90RowBroadcast< + 0, + TileShape, + OutElementType, + OutElementType, + cute::Stride, cute::Int<1>, cute::Int<0>>>; + + using Compute0 = cutlass::epilogue::fusion:: + Sm90Compute; + + using EVTCompute0 = cutlass::epilogue::fusion::Sm90EVT; + + using LayoutA = cutlass::layout::RowMajor; + static constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; + + using LayoutB = cutlass::layout::ColumnMajor; + static constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; + + using ElementC = void; + using LayoutC = cutlass::layout::RowMajor; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + + using LayoutD = cutlass::layout::RowMajor; + static constexpr int AlignmentD = AlignmentC; + + using Compute1MulAdd = cutlass::epilogue::fusion:: + Sm90Compute; + using Compute1Mul = cutlass::epilogue::fusion:: + Sm90Compute; + + using EVTCompute = typename std::conditional_t< + WithBias, + cutlass::epilogue::fusion::Sm90EVT, + cutlass::epilogue::fusion::Sm90EVT>; + using ArgumentType = typename EVTCompute::Arguments; + // MMA type + using ElementAccumulator = AccumElementType; + + // Epilogue types + using ElementCompute = float; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + cutlass::arch::Sm120, + cutlass::arch::OpClassTensorOp, + TileShape, + ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementCompute, + ElementC, + LayoutC, + AlignmentC, + OutElementType, + LayoutD, + AlignmentD, + EpilogueScheduleType, + EVTCompute>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + cutlass::arch::Sm120, + cutlass::arch::OpClassTensorOp, + ElementType, + LayoutA, + AlignmentA, + ElementType, + LayoutB, + AlignmentB, + ElementAccumulator, + TileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + MainloopScheduleType>::CollectiveOp; + using GemmKernel = + cutlass::gemm::kernel::GemmUniversal, CollectiveMainloop, CollectiveEpilogue, void>; + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + template + static auto args_from_tensor(torch::Tensor const& tensor) { + using Arguments = typename Descriptor::Arguments; + auto* data_ptr = static_cast(tensor.data_ptr()); + static_assert( + std::is_same_v || std::is_same_v || std::is_same_v); + return Arguments{data_ptr}; + } + + public: + static ArgumentType prepare_args( + torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + std::optional const& bias = std::nullopt) { + auto a_args = args_from_tensor(a_scales); + auto b_args = args_from_tensor(b_scales); + + typename EVTCompute0::Arguments evt0_args{b_args, {}, {}}; + + if constexpr (WithBias) { + auto bias_args = args_from_tensor(bias.value()); + return ArgumentType{a_args, evt0_args, bias_args, {}}; + } else { + return ArgumentType{a_args, evt0_args, {}}; + } + } +}; + +template +typename GemmType::Gemm::Arguments prepare_sm120_fp8_args( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + using Gemm = typename GemmType::Gemm; + using ElementT = typename Gemm::ElementA; + using ElementC = typename Gemm::ElementC; + using ElementOutput = typename Gemm::ElementD; + using ElementComputeEpilogue = float; + using GemmKernel = typename Gemm::GemmKernel; + + using StrideA = typename Gemm::GemmKernel::StrideA; + using StrideB = typename Gemm::GemmKernel::StrideB; + using StrideC = typename Gemm::GemmKernel::StrideC; + using StrideD = StrideC; + using StrideAux = StrideC; + + int32_t m = a.size(0); + int32_t n = b.size(1); + int32_t k = a.size(1); + + ElementT const* ptr_a = reinterpret_cast(a.data_ptr()); + ElementT const* ptr_b = reinterpret_cast(b.data_ptr()); + + StrideA stride_a = cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(m, k, 1)); + StrideB stride_b = cutlass::make_cute_packed_stride(StrideB{}, cute::make_shape(n, k, 1)); + StrideC stride_c = cutlass::make_cute_packed_stride(StrideC{}, cute::make_shape(m, n, 1)); + StrideD stride_d = cutlass::make_cute_packed_stride(StrideD{}, cute::make_shape(m, n, 1)); + StrideAux aux_stride = stride_d; + + typename GemmKernel::MainloopArguments mainloop_args{ptr_a, stride_a, ptr_b, stride_b}; + + typename GemmKernel::ProblemShape prob_shape = {m, n, k, 1}; + cutlass::KernelHardwareInfo hw_info; + typename GemmKernel::TileSchedulerArguments scheduler = {}; + + auto ptr_c = static_cast(out.data_ptr()); + + auto prepare_epilogue_args = [&](const c10::optional& bias = c10::nullopt) { + if constexpr (WithBias) { + TORCH_CHECK(bias.has_value(), "Bias tensor is required but not provided."); + return typename GemmKernel::EpilogueArguments{ + GemmType::prepare_args(scales_a, scales_b, bias.value()), ptr_c, stride_c, ptr_c, stride_d}; + } else { + return typename GemmKernel::EpilogueArguments{ + GemmType::prepare_args(scales_a, scales_b), ptr_c, stride_c, ptr_c, stride_d}; + } + }; + + typename GemmKernel::Arguments args{ + cutlass::gemm::GemmUniversalMode::kGemm, + prob_shape, + mainloop_args, + prepare_epilogue_args(bias), + hw_info, + scheduler}; + return args; +} + +template +void launch_sm120_fp8_scaled_mm( + torch::Tensor& out, + torch::Tensor const& a, + torch::Tensor const& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + auto args = prepare_sm120_fp8_args(out, a, b, scales_a, scales_b, bias); + + typename Gemm::Gemm gemm_op; + size_t workspace_size = gemm_op.get_workspace_size(args); + auto const workspace_options = torch::TensorOptions().dtype(torch::kUInt8).device(a.device()); + auto workspace = torch::empty(workspace_size, workspace_options); + auto stream = at::cuda::getCurrentCUDAStream(a.get_device()); + auto can_implement = gemm_op.can_implement(args); + TORCH_CHECK(can_implement == cutlass::Status::kSuccess) + auto status = gemm_op.run(args, workspace.data_ptr(), stream); + TORCH_CHECK(status == cutlass::Status::kSuccess) +} + +template +void sm120_fp8_dispatch_bias( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + using CTAShapeDefault = Shape<_128, _128, _128>; + using ClusterShapeDefault = Shape<_1, _1, _1>; + + using MainloopScheduleType = cutlass::gemm::collective::KernelScheduleAuto; + using EpilogueScheduleType = cutlass::epilogue::collective::EpilogueScheduleAuto; + using TileSchedulerType = void; + + using ElementInput = cutlass::float_e4m3_t; + using ElementOutput = OutType; + using AccumElementType = float; + + using BiasGemmDefault = DeviceGemmFp8RowwiseSm120< + ElementInput, + ElementOutput, + AccumElementType, + CTAShapeDefault, + ClusterShapeDefault, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + true>; + + using GemmDefault = DeviceGemmFp8RowwiseSm120< + ElementInput, + ElementOutput, + AccumElementType, + CTAShapeDefault, + ClusterShapeDefault, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + false>; + + if (bias) { + return launch_sm120_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } else { + return launch_sm120_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } +} + +template +void sm120_fp8_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + return sm120_fp8_dispatch_bias(out, a, b, scales_a, scales_b, bias); +} +#endif + +torch::Tensor fp8_scaled_mm( + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Dtype& out_dtype, + const c10::optional& bias) { + TORCH_CHECK(mat_a.is_cuda(), "mat_a must be a CUDA tensor"); + TORCH_CHECK(mat_b.is_cuda(), "mat_b must be a CUDA tensor"); + TORCH_CHECK(mat_a.dim() == 2, "mat_a must be a 2D tensor"); + TORCH_CHECK(mat_b.dim() == 2, "mat_b must be a 2D tensor"); + TORCH_CHECK(mat_a.stride(1) == 1, "mat_a must be a row major tensor"); + TORCH_CHECK(mat_b.stride(0) == 1, "mat_b must be a column major tensor"); + TORCH_CHECK(mat_a.size(1) == mat_b.size(0), "mat_a and mat_b shapes cannot be multiplied"); + + TORCH_CHECK( + (mat_a.size(1) * mat_a.element_size()) % 16 == 0, "mat_a must be multiple of 16 bytes for memory alignment"); + TORCH_CHECK( + (mat_b.size(0) * mat_b.element_size()) % 16 == 0, "mat_b must be multiple of 16 bytes for memory alignment"); + TORCH_CHECK(mat_a.scalar_type() == torch::kFloat8_e4m3fn, "mat_a must be Float8_e4m3fn"); + TORCH_CHECK(mat_b.scalar_type() == torch::kFloat8_e4m3fn, "mat_b must be Float8_e4m3fn"); + TORCH_CHECK(out_dtype == torch::kHalf || out_dtype == torch::kBFloat16, "out_dtype must be Half or BFloat16"); + + TORCH_CHECK(scales_a.numel() == mat_a.size(0), "size of scales_a is not matched"); + TORCH_CHECK(scales_b.numel() == mat_b.size(1), "size of scales_b is not matched"); + TORCH_CHECK(scales_a.is_contiguous(), "scales_a must be contiguous"); + TORCH_CHECK(scales_b.is_contiguous(), "scales_b msut be contiguous"); + TORCH_CHECK(scales_a.scalar_type() == torch::kFloat32, "scales_a must be Float32"); + TORCH_CHECK(scales_b.scalar_type() == torch::kFloat32, "scales_b must be Float32"); + + if (bias) { + TORCH_CHECK(bias->numel() == mat_b.size(1), "size of bias is not matched"); + TORCH_CHECK(bias->is_contiguous(), "bias must be contiguous"); + TORCH_CHECK(bias->dtype() == out_dtype, "bias dtype must match output dtype"); + } + + torch::Tensor out = torch::empty({mat_a.size(0), mat_b.size(1)}, mat_a.options().dtype(out_dtype)); + TORCH_CHECK((out.size(1) * out.element_size()) % 16 == 0, "out must be multiple of 16 bytes for memory alignment"); + + auto sm_version = getSMVersion(); + +#if defined CUDA_VERSION && CUDA_VERSION >= 12080 + if (sm_version >= 120) { + if (out_dtype == torch::kBFloat16) { + sm120_fp8_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + sm120_fp8_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } + return out; + } else if (sm_version >= 100) { + if (out_dtype == torch::kBFloat16) { + sm100_fp8_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + sm100_fp8_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } + return out; + } +#endif + +#if defined CUDA_VERSION && CUDA_VERSION >= 12000 + if (sm_version >= 90) { + if (out_dtype == torch::kBFloat16) { + sm90_fp8_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + sm90_fp8_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } + return out; + } +#endif + +#if defined CUDA_VERSION && CUDA_VERSION >= 12040 + if (sm_version == 89) { + if (out_dtype == torch::kBFloat16) { + sm89_fp8_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + sm89_fp8_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } + return out; + } +#endif + + TORCH_CHECK_NOT_IMPLEMENTED(false, "No implemented fp8_scaled_mm for current compute capability: ", sm_version); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/compat.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/compat.cuh new file mode 100644 index 00000000..506eeb76 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/compat.cuh @@ -0,0 +1,62 @@ +/* +Copied from https://github.com/turboderp/exllamav2 +*/ + +#ifndef _compat_cuh +#define _compat_cuh + +namespace sglang { +namespace gptq { +// atomicAdd for half types, to support CC < 7.x + +__device__ __forceinline__ void atomicAdd_half(half* address, half val) { + unsigned int* address_as_ui = (unsigned int*)((char*)address - ((size_t)address & 2)); + unsigned int old = *address_as_ui; + unsigned int assumed; + + do { + assumed = old; + __half_raw hsum; + hsum.x = (size_t)address & 2 ? (old >> 16) : (old & 0xffff); + half tmpres = __hadd(hsum, val); + hsum = __half_raw(tmpres); + old = (size_t)address & 2 ? (old & 0xffff) | (hsum.x << 16) : (old & 0xffff0000) | hsum.x; + old = atomicCAS(address_as_ui, assumed, old); + } while (assumed != old); +} + +// atomicAdd for half2 types + +__device__ __forceinline__ void atomicAdd_half2(half2* address, half2 val) { + unsigned int* address_as_ui = (unsigned int*)address; + unsigned int old = *address_as_ui; + unsigned int assumed; + do { + assumed = old; + half2 old_val = *((half2*)&old); + half2 new_val = __hadd2(old_val, val); + old = atomicCAS(address_as_ui, assumed, *((unsigned int*)&new_val)); + } while (assumed != old); +} + +// + +#if defined(__CUDA_ARCH__) || defined(USE_ROCM) +#if __CUDA_ARCH__ < 700 || defined(USE_ROCM) + +__device__ __forceinline__ void atomicAdd(half* address, half val) { + atomicAdd_half(address, val); +} + +#if __CUDA_ARCH__ < 600 || defined(USE_ROCM) +__device__ __forceinline__ void atomicAdd(half2* address, half2 val) { + atomicAdd_half2(address, val); +} +#endif + +#endif +#endif + +} // namespace gptq +} // namespace sglang +#endif diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/gptq_kernel.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/gptq_kernel.cu new file mode 100644 index 00000000..4dd5d8a2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/gptq_kernel.cu @@ -0,0 +1,1950 @@ +/* +Adapted from https://github.com/turboderp/exllamav2 and +https://github.com/qwopqwop200/GPTQ-for-LLaMa +*/ + +#include +#include +#include +#include +#include + +#include +#include + +#include "compat.cuh" +#include "matrix_view.cuh" +#include "qdq_2.cuh" +#include "qdq_3.cuh" +#include "qdq_4.cuh" +#include "qdq_8.cuh" + +namespace sglang { +namespace gptq { + +#define BLOCK_KN_SIZE 128 +#define BLOCK_M_SIZE_MAX 8 +#define MAX_GROUPS_IN_BLOCK (BLOCK_KN_SIZE / 32) +#define MAX_Q_GEMM_ROWS 50 +#define MAX_Q_GEMM_ROWS_8BIT 24 +#define MAX_ALT_GEMM_ROWS 8 +#define THREADS_X 32 +#define THREADS_Y 32 +#define DIVIDE(x, size) (((x) + (size) - 1) / (size)) + +#if defined(USE_ROCM) +#include +__host__ __forceinline__ hipblasStatus_t __compat_hipblasHgemm( + hipblasHandle_t handle, + hipblasOperation_t transA, + hipblasOperation_t transB, + int m, + int n, + int k, + const half* alpha, + const half* AP, + int lda, + const half* BP, + int ldb, + const half* beta, + half* CP, + int ldc) { + return hipblasHgemm( + handle, + transA, + transB, + m, + n, + k, + reinterpret_cast(alpha), + reinterpret_cast(AP), + lda, + reinterpret_cast(BP), + ldb, + reinterpret_cast(beta), + reinterpret_cast(CP), + ldc); +} +#define hipblasHgemm __compat_hipblasHgemm + +// Previous version of PyTorch were converting to rocBLAS instead of hipBLAS. +#define rocblas_operation_none HIPBLAS_OP_N +#define rocblas_hgemm __compat_hipblasHgemm +#endif + +__forceinline__ __device__ half2 dot22_8(half2 (&dq)[4], const half* a_ptr, const half2 g_result) { + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; +#pragma unroll + for (int i = 0; i < 4; i++) + result = __hfma2(dq[i], *a2_ptr++, result); + return __hadd2(result, g_result); +} + +__forceinline__ __device__ float dot22_8_f(half2 (&dq)[4], const half* a_ptr) { + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; +#pragma unroll + for (int i = 0; i < 4; i++) + result = __hfma2(dq[i], *a2_ptr++, result); + return __half2float(__low2half(result)) + __half2float(__high2half(result)); +} + +__forceinline__ __device__ half2 dot22_8(half2 (&dq)[4], const half* a_ptr, const half2 g_result, const half qs_h) { + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; +#pragma unroll + for (int i = 0; i < 4; i++) + result = __hfma2(dq[i], *a2_ptr++, result); + return __hfma2(result, __halves2half2(qs_h, qs_h), g_result); +} + +__forceinline__ __device__ half2 dot22_16(half2 (&dq)[8], const half* a_ptr, const half2 g_result, const half qs_h) { + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; +#pragma unroll + for (int i = 0; i < 8; i++) + result = __hfma2(dq[i], *a2_ptr++, result); + return __hfma2(result, __halves2half2(qs_h, qs_h), g_result); +} + +__forceinline__ __device__ half2 dot22_32(half2 (&dq)[16], const half* a_ptr, const half2 g_result, const half qs_h) { + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; +#pragma unroll + for (int i = 0; i < 16; i += 1) + result = __hfma2(dq[i], *a2_ptr++, result); + return __hfma2(result, __halves2half2(qs_h, qs_h), g_result); +} + +__forceinline__ __device__ float dot22_8_f(half2 (&dq)[4], const half* a_ptr, const float g_result, const float qs_f) { + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; +#pragma unroll + for (int i = 0; i < 4; i++) + result = __hfma2(dq[i], *a2_ptr++, result); + float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result)); + return fma(result_f, qs_f, g_result); +} + +__forceinline__ __device__ float dot22_16_f(half2 (&dq)[8], const half* a_ptr, const float g_result, const float qs_f) { + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; +#pragma unroll + for (int i = 0; i < 8; i++) + result = __hfma2(dq[i], *a2_ptr++, result); + float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result)); + return fma(result_f, qs_f, g_result); +} + +__forceinline__ __device__ float +dot22_32_f(half2 (&dq)[16], const half* a_ptr, const float g_result, const float qs_f) { + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; +#pragma unroll + for (int i = 0; i < 16; i += 1) + result = __hfma2(dq[i], *a2_ptr++, result); + float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result)); + return fma(result_f, qs_f, g_result); +} + +__forceinline__ __device__ half dot22_8_h(half2 (&dq)[4], const half* a_ptr, const half g_result, const half qs_h) { + // Use FP32 accumulator to avoid potential overflow since unscaled weights are + // in the range -128..127 + + float result = {}; +#pragma unroll + for (int i = 0; i < 4; i++) { + half2 w01 = dq[i]; + float w0 = __low2float(w01); + float w1 = __high2float(w01); + float x0 = __half2float(*a_ptr++); + float x1 = __half2float(*a_ptr++); + result = fma(w0, x0, result); + result = fma(w1, x1, result); + } + float qs = __half2float(qs_h); + result *= qs; + half result_h = __float2half_rn(result); + return __hadd(result_h, g_result); +} + +__forceinline__ __device__ half dot22_16_h(half2 (&dq)[8], const half* a_ptr, const half g_result, const half qs_h) { + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; +#pragma unroll + for (int i = 0; i < 8; i++) + result = __hfma2(dq[i], *a2_ptr++, result); + half result_h = __hadd(__low2half(result), __high2half(result)); + return __hfma(result_h, qs_h, g_result); +} + +__forceinline__ __device__ half dot22_32_h(half2 (&dq)[16], const half* a_ptr, const half g_result, const half qs_h) { + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; +#pragma unroll + for (int i = 0; i < 16; i += 1) + result = __hfma2(dq[i], *a2_ptr++, result); + half result_h = __hadd(__low2half(result), __high2half(result)); + return __hfma(result_h, qs_h, g_result); +} + +typedef void (*fp_gemm_half_q_half_gptq_kernel)( + const half*, + const uint32_t*, + const uint32_t*, + const half*, + half*, + const int, + const int, + const int, + const int, + const int*); + +template +__global__ void gemm_half_q_half_gptq_4bit_kernel( + const half* __restrict__ a, + const uint32_t* __restrict__ b_q_weight, + const uint32_t* __restrict__ b_gptq_qzeros, + const half* __restrict__ b_gptq_scales, + half* __restrict__ c, + const int size_m, + const int size_n, + const int size_k, + const int groups, + const int* __restrict__ b_q_perm) { + MatrixView_half a_(a, size_m, size_k); + MatrixView_half_rw c_(c, size_m, size_n); + MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n); + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n); + + auto t = threadIdx.x; + + // Block + auto offset_n = blockIdx.x * BLOCK_KN_SIZE * 4; + auto offset_m = blockIdx.y * m_count; + auto offset_k = blockIdx.z * BLOCK_KN_SIZE; + + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k); + + int n = offset_n + t * 4; + + // Preload block_a + __shared__ half block_a[m_count][BLOCK_KN_SIZE]; + + if (offset_k + t < end_k) { + for (int m = 0; m < m_count; ++m) { + const half* a_ptr = a_.item_ptr(offset_m + m, 0); + half* block_a_ptr = block_a[m]; + + half a0; + if (b_q_perm) + a0 = a_ptr[b_q_perm[offset_k + t]]; + else + a0 = a_ptr[offset_k + t]; + block_a_ptr[t] = a0; + } + } + + // Zero output + if (n >= size_n) return; + + if (blockIdx.z == 0) { + for (int m = 0; m < m_count; m++) + *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0; + } + + __syncthreads(); + + // Find initial group + int groupsize = size_k / groups; + int group = offset_k / groupsize; + int nextgroup = offset_k + groupsize; + + // a, b offset + int qk = offset_k / (32 / 4); + + const uint32_t* b_ptr = b_q_weight + qk * size_n + n; + const half* a_ptr = &block_a[0][0]; + int a_stride = BLOCK_KN_SIZE; + + // Initial group + int zeros[4]; + float scales[4]; + half2 z1z16[4][2]; + half2 y1y16[4][2]; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_f(scales, group, n); + dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]); + dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]); + dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]); + dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]); + + // Column result + float block_c[m_count][4] = {}; + + // Dequantize and multiply + int k = offset_k; + while (k < end_k) { + if (k == nextgroup) { + group++; + nextgroup += groupsize; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_f(scales, group, n); + dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]); + dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]); + dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]); + dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]); + } + +#pragma unroll + for (int j = 0; j < 4; j++) { + const int4* b_ptr4 = (int4*)b_ptr; + int4 load_int4 = *b_ptr4; + + half2 dq[4][4]; + dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n, false); + dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n, false); + dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n, false); + dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n, false); + +#pragma unroll + for (int m = 0; m < m_count; m++) { + block_c[m][0] = fma(dot22_8_f(dq[0], a_ptr + m * a_stride), scales[0], block_c[m][0]); + block_c[m][1] = fma(dot22_8_f(dq[1], a_ptr + m * a_stride), scales[1], block_c[m][1]); + block_c[m][2] = fma(dot22_8_f(dq[2], a_ptr + m * a_stride), scales[2], block_c[m][2]); + block_c[m][3] = fma(dot22_8_f(dq[3], a_ptr + m * a_stride), scales[3], block_c[m][3]); + } + + b_ptr += size_n; + a_ptr += 8; + } + + k += 32; + } + + for (int m = 0; m < m_count; m++) { + half2* out = (half2*)c_.item_ptr(offset_m + m, n); + half2 result01 = __halves2half2(__float2half_rn(block_c[m][0]), __float2half_rn(block_c[m][1])); + half2 result23 = __halves2half2(__float2half_rn(block_c[m][2]), __float2half_rn(block_c[m][3])); + atomicAdd(out, result01); + atomicAdd(out + 1, result23); + } +} + +template +__global__ void gemm_half_q_half_gptq_2bit_kernel( + const half* __restrict__ a, + const uint32_t* __restrict__ b_q_weight, + const uint32_t* __restrict__ b_gptq_qzeros, + const half* __restrict__ b_gptq_scales, + half* __restrict__ c, + const int size_m, + const int size_n, + const int size_k, + const int groups, + const int* __restrict__ b_q_perm) { + MatrixView_half a_(a, size_m, size_k); + MatrixView_half_rw c_(c, size_m, size_n); + MatrixView_q2_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n); + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n); + + auto t = threadIdx.x; + + // Block + auto offset_n = blockIdx.x * BLOCK_KN_SIZE * 4; + auto offset_m = blockIdx.y * m_count; + auto offset_k = blockIdx.z * BLOCK_KN_SIZE; + + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k); + + int n = offset_n + t * 4; + + // Preload block_a + __shared__ half block_a[m_count][BLOCK_KN_SIZE]; + + if (offset_k + t < end_k) { + for (int m = 0; m < m_count; ++m) { + const half* a_ptr = a_.item_ptr(offset_m + m, 0); + half* block_a_ptr = block_a[m]; + + half a0; + if (b_q_perm) + a0 = a_ptr[b_q_perm[offset_k + t]]; + else + a0 = a_ptr[offset_k + t]; + block_a_ptr[t] = a0; + } + } + + // Zero output + if (n >= size_n) return; + + if (blockIdx.z == 0) { + for (int m = 0; m < m_count; m++) + *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0; + } + + __syncthreads(); + + // Find initial group + int groupsize = size_k / groups; + int group = offset_k / groupsize; + int nextgroup = offset_k + groupsize; + + // a, b offset + int qk = offset_k / (32 / 2); + + const uint32_t* b_ptr = b_q_weight + qk * size_n + n; + const half* a_ptr = &block_a[0][0]; + int a_stride = BLOCK_KN_SIZE; + + // Initial group + int zeros[4]; + half scales[4]; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4(scales, group, n); + // Column result + half block_c[m_count][4] = {}; + + // Dequantize and multiply + int k = offset_k; + while (k < end_k) { + if (k == nextgroup) { + group++; + nextgroup += groupsize; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4(scales, group, n); + } + +#pragma unroll + for (int j = 0; j < 1; j++) { + const int4* b_ptr4 = (int4*)b_ptr; + int4 load_int4 = *b_ptr4; + + half2 dq[4][8]; + dequant_2bit_16(load_int4.x, dq[0], size_n, zeros[0] + 1); + dequant_2bit_16(load_int4.y, dq[1], size_n, zeros[1] + 1); + dequant_2bit_16(load_int4.z, dq[2], size_n, zeros[2] + 1); + dequant_2bit_16(load_int4.w, dq[3], size_n, zeros[3] + 1); + +#pragma unroll + for (int m = 0; m < m_count; m++) { + block_c[m][0] = dot22_16_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]); + block_c[m][1] = dot22_16_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]); + block_c[m][2] = dot22_16_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]); + block_c[m][3] = dot22_16_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]); + } + + b_ptr += size_n; + a_ptr += 16; + } + + k += 16; + } + + for (int m = 0; m < m_count; m++) { + half2* out = (half2*)c_.item_ptr(offset_m + m, n); + half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]); + half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]); + atomicAdd(out, result01); + atomicAdd(out + 1, result23); + } +} + +template +__global__ void gemm_half_q_half_gptq_3bit_kernel( + const half* __restrict__ a, + const uint32_t* __restrict__ b_q_weight, + const uint32_t* __restrict__ b_gptq_qzeros, + const half* __restrict__ b_gptq_scales, + half* __restrict__ c, + const int size_m, + const int size_n, + const int size_k, + const int groups, + const int* __restrict__ b_q_perm) { + MatrixView_half a_(a, size_m, size_k); + MatrixView_half_rw c_(c, size_m, size_n); + MatrixView_q3_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n); + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n); + + auto t = threadIdx.x; + + // Block + auto offset_n = blockIdx.x * BLOCK_KN_SIZE * 4; + auto offset_m = blockIdx.y * m_count; + auto offset_k = blockIdx.z * BLOCK_KN_SIZE; + + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k); + + int n = offset_n + t * 4; + + // Preload block_a + __shared__ half block_a[m_count][BLOCK_KN_SIZE]; + + if (offset_k + t < end_k) { + for (int m = 0; m < m_count; ++m) { + const half* a_ptr = a_.item_ptr(offset_m + m, 0); + half* block_a_ptr = block_a[m]; + + half a0; + if (b_q_perm) + a0 = a_ptr[b_q_perm[offset_k + t]]; + else + a0 = a_ptr[offset_k + t]; + block_a_ptr[t] = a0; + } + } + + // Zero output + if (n >= size_n) return; + + if (blockIdx.z == 0) { + for (int m = 0; m < m_count; m++) + *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0; + } + + __syncthreads(); + + // Find initial group + int groupsize = size_k / groups; + int group = offset_k / groupsize; + int nextgroup = offset_k + groupsize; + + // a, b offset + int qk = offset_k / 32 * 3; + + const uint32_t* b_ptr = b_q_weight + qk * size_n + n; + const half* a_ptr = &block_a[0][0]; + int a_stride = BLOCK_KN_SIZE; + + // Initial group + int zeros[4]; + half scales[4]; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4(scales, group, n); + // Column result + half block_c[m_count][4] = {}; + + // Dequantize and multiply + int k = offset_k; + while (k < end_k) { + if (k == nextgroup) { + group++; + nextgroup += groupsize; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4(scales, group, n); + } + +#pragma unroll + for (int j = 0; j < 1; j++) { + int4 load_int4[3]; + load_int4[0] = *((int4*)b_ptr); + b_ptr += size_n; + load_int4[1] = *((int4*)b_ptr); + b_ptr += size_n; + load_int4[2] = *((int4*)b_ptr); + b_ptr += size_n; + + half2 dq[4][16]; + dequant_3bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0], size_n, zeros[0] + 1); + dequant_3bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1], size_n, zeros[1] + 1); + dequant_3bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2], size_n, zeros[2] + 1); + dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3], size_n, zeros[3] + 1); + +#pragma unroll + for (int m = 0; m < m_count; m++) { + block_c[m][0] = dot22_32_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]); + block_c[m][1] = dot22_32_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]); + block_c[m][2] = dot22_32_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]); + block_c[m][3] = dot22_32_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]); + } + a_ptr += 32; + } + + k += 32; + } + + for (int m = 0; m < m_count; m++) { + half2* out = (half2*)c_.item_ptr(offset_m + m, n); + half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]); + half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]); + atomicAdd(out, result01); + atomicAdd(out + 1, result23); + } +} + +template +__global__ void gemm_half_q_half_gptq_8bit_kernel( + const half* __restrict__ a, + const uint32_t* __restrict__ b_q_weight, + const uint32_t* __restrict__ b_gptq_qzeros, + const half* __restrict__ b_gptq_scales, + half* __restrict__ c, + const int size_m, + const int size_n, + const int size_k, + const int groups, + const int* __restrict__ b_q_perm) { + MatrixView_half a_(a, size_m, size_k); + MatrixView_half_rw c_(c, size_m, size_n); + MatrixView_q8_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n); + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n); + + auto t = threadIdx.x; + + // Block + auto offset_n = blockIdx.x * BLOCK_KN_SIZE * 4; + auto offset_m = blockIdx.y * m_count; + auto offset_k = blockIdx.z * BLOCK_KN_SIZE; + + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k); + + int n = offset_n + t * 4; + + // Preload block_a + __shared__ half block_a[m_count][BLOCK_KN_SIZE]; + + if (offset_k + t < end_k) { + for (int m = 0; m < m_count; ++m) { + const half* a_ptr = a_.item_ptr(offset_m + m, 0); + half* block_a_ptr = block_a[m]; + + half a0; + if (b_q_perm) + a0 = a_ptr[b_q_perm[offset_k + t]]; + else + a0 = a_ptr[offset_k + t]; + block_a_ptr[t] = a0; + } + } + + // Zero output + if (n >= size_n) return; + + if (blockIdx.z == 0) { + for (int m = 0; m < m_count; m++) + *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0; + } + + __syncthreads(); + + // Find initial group + int groupsize = size_k / groups; + int group = offset_k / groupsize; + int nextgroup = offset_k + groupsize; + + // a, b offset + int qk = offset_k / (32 / 8); + + const uint32_t* b_ptr = b_q_weight + qk * size_n + n; + const half* a_ptr = &block_a[0][0]; + int a_stride = BLOCK_KN_SIZE; + + // Initial group + int zeros[4]; + half scales[4]; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4(scales, group, n); + // Column result + half block_c[m_count][4] = {}; + + // Dequantize and multiply + int k = offset_k; + while (k < end_k) { + if (k == nextgroup) { + group++; + nextgroup += groupsize; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4(scales, group, n); + } + +#pragma unroll + for (int j = 0; j < 4; j++) { + int4 load_int4[2]; + load_int4[0] = *((int4*)b_ptr); + b_ptr += size_n; + load_int4[1] = *((int4*)b_ptr); + b_ptr += size_n; + + half2 dq[4][4]; + dequant_8bit_8(load_int4[0].x, load_int4[1].x, dq[0], size_n, zeros[0] + 1); + dequant_8bit_8(load_int4[0].y, load_int4[1].y, dq[1], size_n, zeros[1] + 1); + dequant_8bit_8(load_int4[0].z, load_int4[1].z, dq[2], size_n, zeros[2] + 1); + dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n, zeros[3] + 1); + + for (int m = 0; m < m_count; m++) { + block_c[m][0] = dot22_8_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]); + block_c[m][1] = dot22_8_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]); + block_c[m][2] = dot22_8_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]); + block_c[m][3] = dot22_8_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]); + } + a_ptr += 8; + } + k += 32; + } + + for (int m = 0; m < m_count; m++) { + half2* out = (half2*)c_.item_ptr(offset_m + m, n); + half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]); + half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]); + atomicAdd(out, result01); + atomicAdd(out + 1, result23); + } +} + +fp_gemm_half_q_half_gptq_kernel pick_gemm_half_q_half_gptq_kernel(bool first_block, const int m_count, const int bit) { +#define SELECT_KERNEL(M_COUNT) \ + if (m_count == M_COUNT) { \ + if (bit == 2) return gemm_half_q_half_gptq_2bit_kernel; \ + if (bit == 3) return gemm_half_q_half_gptq_3bit_kernel; \ + if (bit == 4) return gemm_half_q_half_gptq_4bit_kernel; \ + if (bit == 8) return gemm_half_q_half_gptq_8bit_kernel; \ + } +#if BLOCK_M_SIZE_MAX >= 1 + SELECT_KERNEL(1); +#endif +#if BLOCK_M_SIZE_MAX >= 2 + SELECT_KERNEL(2); +#endif +#if BLOCK_M_SIZE_MAX >= 3 + SELECT_KERNEL(3); +#endif +#if BLOCK_M_SIZE_MAX >= 4 + SELECT_KERNEL(4); +#endif +#if BLOCK_M_SIZE_MAX >= 5 + SELECT_KERNEL(5); +#endif +#if BLOCK_M_SIZE_MAX >= 6 + SELECT_KERNEL(6); +#endif +#if BLOCK_M_SIZE_MAX >= 7 + SELECT_KERNEL(7); +#endif +#if BLOCK_M_SIZE_MAX >= 8 + SELECT_KERNEL(8); +#endif + return NULL; +} + +void gemm_half_q_half_cuda_part( + const half* a, + const uint32_t* b_q_weight, + const uint32_t* b_gptq_qzeros, + const half* b_gptq_scales, + const int* b_q_perm, + half* c, + int size_m, + int size_n, + int size_k, + int m_count, + int groups, + int bit) { + dim3 blockDim, gridDim; + blockDim.x = BLOCK_KN_SIZE; + blockDim.y = 1; + blockDim.z = 1; + gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE * 4); + gridDim.y = DIVIDE(size_m, m_count); + gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE); + + fp_gemm_half_q_half_gptq_kernel kernel = pick_gemm_half_q_half_gptq_kernel(true, m_count, bit); + + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + kernel<<>>( + a, b_q_weight, b_gptq_qzeros, b_gptq_scales, c, size_m, size_n, size_k, groups, b_q_perm); +} + +__global__ void reconstruct_exllama_8bit_kernel( + const uint32_t* __restrict__ b_q_weight, + const int* __restrict__ b_q_perm, + const uint32_t* __restrict__ b_gptq_qzeros, + const half* __restrict__ b_gptq_scales, + const int size_k, + const int size_n, + const int groups, + half* __restrict__ b) { + MatrixView_half_rw b_(b, size_k, size_n); + MatrixView_q8_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n); + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n); + + auto offset_k = BLOCK_KN_SIZE * blockIdx.y; + auto offset_n = BLOCK_KN_SIZE * blockIdx.x * 4; + + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k); + + // Preload remapping table + __shared__ int perm[BLOCK_KN_SIZE]; + auto t = threadIdx.x; + + if (b_q_perm) { + if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t]; + } + + // Column + int n = offset_n + t * 4; + if (n >= size_n) return; + + // Find initial group + int groupsize = size_k / groups; + int group = offset_k / groupsize; + int nextgroup = offset_k + groupsize; + + // b offset + int qk = offset_k / (32 / 8); + + const uint32_t* b_ptr = b_q_weight + qk * size_n + n; + + // Initial zeros/scale + int zeros[4]; + half2 scales[4]; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_h2(scales, group, n); + + __syncthreads(); + + int k = offset_k; + int lk = 0; + + while (k < end_k) { + if (k == nextgroup) { + group++; + nextgroup += groupsize; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_h2(scales, group, n); + } + + for (int p = 0; p < 4; p++) { + int4 load_int4[2]; + load_int4[0] = *((int4*)b_ptr); + b_ptr += size_n; + load_int4[1] = *((int4*)b_ptr); + b_ptr += size_n; + + half2 dq[4][4]; + dequant_8bit_8(load_int4[0].x, load_int4[1].x, dq[0], size_n, zeros[0] + 1); + dequant_8bit_8(load_int4[0].y, load_int4[1].y, dq[1], size_n, zeros[1] + 1); + dequant_8bit_8(load_int4[0].z, load_int4[1].z, dq[2], size_n, zeros[2] + 1); + dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n, zeros[3] + 1); + + // half* dqh = (half*)dq; + if (b_q_perm) { + for (int j = 0; j < 4; j++) { + for (int v = 0; v < 4; v++) + dq[v][j] = __hmul2(scales[v], dq[v][j]); + b_.set4( + perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j])); + b_.set4( + perm[lk++], + n, + __high2half(dq[0][j]), + __high2half(dq[1][j]), + __high2half(dq[2][j]), + __high2half(dq[3][j])); + } + } else { + for (int j = 0; j < 4; j++) { + for (int v = 0; v < 4; v++) + dq[v][j] = __hmul2(scales[v], dq[v][j]); + b_.set4( + offset_k + lk++, + n, + __low2half(dq[0][j]), + __low2half(dq[1][j]), + __low2half(dq[2][j]), + __low2half(dq[3][j])); + b_.set4( + offset_k + lk++, + n, + __high2half(dq[0][j]), + __high2half(dq[1][j]), + __high2half(dq[2][j]), + __high2half(dq[3][j])); + } + } + } + k += 32; + } +} + +__global__ void reconstruct_exllama_4bit_kernel( + const uint32_t* __restrict__ b_q_weight, + const int* __restrict__ b_q_perm, + const uint32_t* __restrict__ b_gptq_qzeros, + const half* __restrict__ b_gptq_scales, + const int size_k, + const int size_n, + const int groups, + half* __restrict__ b) { + MatrixView_half_rw b_(b, size_k, size_n); + MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n); + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n); + + auto offset_k = BLOCK_KN_SIZE * blockIdx.y; + auto offset_n = BLOCK_KN_SIZE * blockIdx.x * 4; + + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k); + + // Preload remapping table + __shared__ int perm[BLOCK_KN_SIZE]; + auto t = threadIdx.x; + + if (b_q_perm) { + if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t]; + } + + // Column + int n = offset_n + t * 4; + if (n >= size_n) return; + + // Find initial group + int groupsize = size_k / groups; + int group = offset_k / groupsize; + int nextgroup = offset_k + groupsize; + + // b offset + int qk = offset_k / (32 / 4); + + const uint32_t* b_ptr = b_q_weight + qk * size_n + n; + + // Initial zeros/scale + int zeros[4]; + half2 scales[4]; + half2 z1z16[4][2]; + half2 y1y16[4][2]; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_h2(scales, group, n); + dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]); + dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]); + dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]); + dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]); + + __syncthreads(); + + int k = offset_k; + int lk = 0; + + while (k < end_k) { + if (k == nextgroup) { + group++; + nextgroup += groupsize; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_h2(scales, group, n); + dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]); + dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]); + dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]); + dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]); + } + + for (int p = 0; p < 4; p++) { + half2 dq[4][4]; + const int4* b_ptr4 = (int4*)b_ptr; + int4 load_int4 = *b_ptr4; + + dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n, false); + dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n, false); + dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n, false); + dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n, false); + + b_ptr += size_n; + // half* dqh = (half*)dq; + if (b_q_perm) { + for (int j = 0; j < 4; j++) { + for (int v = 0; v < 4; v++) + dq[v][j] = __hmul2(scales[v], dq[v][j]); + b_.set4( + perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j])); + b_.set4( + perm[lk++], + n, + __high2half(dq[0][j]), + __high2half(dq[1][j]), + __high2half(dq[2][j]), + __high2half(dq[3][j])); + } + } else { + for (int j = 0; j < 4; j++) { + for (int v = 0; v < 4; v++) + dq[v][j] = __hmul2(scales[v], dq[v][j]); + b_.set4( + offset_k + lk++, + n, + __low2half(dq[0][j]), + __low2half(dq[1][j]), + __low2half(dq[2][j]), + __low2half(dq[3][j])); + b_.set4( + offset_k + lk++, + n, + __high2half(dq[0][j]), + __high2half(dq[1][j]), + __high2half(dq[2][j]), + __high2half(dq[3][j])); + } + } + } + k += 32; + } +} + +__global__ void reconstruct_exllama_3bit_kernel( + const uint32_t* __restrict__ b_q_weight, + const int* __restrict__ b_q_perm, + const uint32_t* __restrict__ b_gptq_qzeros, + const half* __restrict__ b_gptq_scales, + const int size_k, + const int size_n, + const int groups, + half* __restrict__ b) { + MatrixView_half_rw b_(b, size_k, size_n); + MatrixView_q3_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n); + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n); + + auto offset_k = BLOCK_KN_SIZE * blockIdx.y; + auto offset_n = BLOCK_KN_SIZE * blockIdx.x * 4; + + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k); + + // Preload remapping table + __shared__ int perm[BLOCK_KN_SIZE]; + auto t = threadIdx.x; + + if (b_q_perm) { + if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t]; + } + + // Column + int n = offset_n + t * 4; + if (n >= size_n) return; + + // Find initial group + int groupsize = size_k / groups; + int group = offset_k / groupsize; + int nextgroup = offset_k + groupsize; + + // b offset + int qk = offset_k / 32 * 3; + + const uint32_t* b_ptr = b_q_weight + qk * size_n + n; + + // Initial zeros/scale + int zeros[4]; + half2 scales[4]; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_h2(scales, group, n); + + __syncthreads(); + + int k = offset_k; + int lk = 0; + + while (k < end_k) { + if (k == nextgroup) { + group++; + nextgroup += groupsize; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_h2(scales, group, n); + } + + for (int p = 0; p < 1; p++) { + int4 load_int4[3]; + load_int4[0] = *((int4*)b_ptr); + b_ptr += size_n; + load_int4[1] = *((int4*)b_ptr); + b_ptr += size_n; + load_int4[2] = *((int4*)b_ptr); + b_ptr += size_n; + + half2 dq[4][16]; + dequant_3bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0], size_n, zeros[0] + 1); + dequant_3bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1], size_n, zeros[1] + 1); + dequant_3bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2], size_n, zeros[2] + 1); + dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3], size_n, zeros[3] + 1); + + if (b_q_perm) { + for (int j = 0; j < 16; j++) { + for (int v = 0; v < 4; v++) + dq[v][j] = __hmul2(scales[v], dq[v][j]); + b_.set4( + perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j])); + b_.set4( + perm[lk++], + n, + __high2half(dq[0][j]), + __high2half(dq[1][j]), + __high2half(dq[2][j]), + __high2half(dq[3][j])); + } + } else { + for (int j = 0; j < 16; j++) { + for (int v = 0; v < 4; v++) + dq[v][j] = __hmul2(scales[v], dq[v][j]); + b_.set4( + offset_k + lk++, + n, + __low2half(dq[0][j]), + __low2half(dq[1][j]), + __low2half(dq[2][j]), + __low2half(dq[3][j])); + b_.set4( + offset_k + lk++, + n, + __high2half(dq[0][j]), + __high2half(dq[1][j]), + __high2half(dq[2][j]), + __high2half(dq[3][j])); + } + } + } + k += 32; + } +} + +__global__ void reconstruct_exllama_2bit_kernel( + const uint32_t* __restrict__ b_q_weight, + const int* __restrict__ b_q_perm, + const uint32_t* __restrict__ b_gptq_qzeros, + const half* __restrict__ b_gptq_scales, + const int size_k, + const int size_n, + const int groups, + half* __restrict__ b) { + MatrixView_half_rw b_(b, size_k, size_n); + MatrixView_q2_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n); + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n); + + auto offset_k = BLOCK_KN_SIZE * blockIdx.y; + auto offset_n = BLOCK_KN_SIZE * blockIdx.x * 4; + + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k); + + // Preload remapping table + __shared__ int perm[BLOCK_KN_SIZE]; + auto t = threadIdx.x; + + if (b_q_perm) { + if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t]; + } + + // Column + int n = offset_n + t * 4; + if (n >= size_n) return; + + // Find initial group + int groupsize = size_k / groups; + int group = offset_k / groupsize; + int nextgroup = offset_k + groupsize; + + // b offset + int qk = offset_k / (32 / 2); + + const uint32_t* b_ptr = b_q_weight + qk * size_n + n; + + // Initial zeros/scale + int zeros[4]; + half2 scales[4]; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_h2(scales, group, n); + + __syncthreads(); + + int k = offset_k; + int lk = 0; + + while (k < end_k) { + if (k == nextgroup) { + group++; + nextgroup += groupsize; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_h2(scales, group, n); + } + + for (int p = 0; p < 2; p++) { + const int4* b_ptr4 = (int4*)b_ptr; + int4 load_int4 = *b_ptr4; + + half2 dq[4][8]; + dequant_2bit_16(load_int4.x, dq[0], size_n, zeros[0] + 1); + dequant_2bit_16(load_int4.y, dq[1], size_n, zeros[1] + 1); + dequant_2bit_16(load_int4.z, dq[2], size_n, zeros[2] + 1); + dequant_2bit_16(load_int4.w, dq[3], size_n, zeros[3] + 1); + + b_ptr += size_n; + // half* dqh = (half*)dq; + if (b_q_perm) { + for (int j = 0; j < 8; j++) { + for (int v = 0; v < 4; v++) + dq[v][j] = __hmul2(scales[v], dq[v][j]); + b_.set4( + perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j])); + b_.set4( + perm[lk++], + n, + __high2half(dq[0][j]), + __high2half(dq[1][j]), + __high2half(dq[2][j]), + __high2half(dq[3][j])); + } + } else { + for (int j = 0; j < 8; j++) { + for (int v = 0; v < 4; v++) + dq[v][j] = __hmul2(scales[v], dq[v][j]); + b_.set4( + offset_k + lk++, + n, + __low2half(dq[0][j]), + __low2half(dq[1][j]), + __low2half(dq[2][j]), + __low2half(dq[3][j])); + b_.set4( + offset_k + lk++, + n, + __high2half(dq[0][j]), + __high2half(dq[1][j]), + __high2half(dq[2][j]), + __high2half(dq[3][j])); + } + } + } + k += 32; + } +} + +void reconstruct_exllama( + const uint32_t* b_q_weight, + const uint32_t* b_gptq_qzeros, + const half* b_gptq_scales, + const int* b_q_perm, + half* out, + int height, + int width, + int groups, + int bit) { + dim3 blockDim, gridDim; + blockDim.x = BLOCK_KN_SIZE; + blockDim.y = 1; + gridDim.y = DIVIDE(height, BLOCK_KN_SIZE); + gridDim.x = DIVIDE(width, BLOCK_KN_SIZE); + + auto reconstruct_exllama_kernel = reconstruct_exllama_4bit_kernel; + if (bit == 2) { + reconstruct_exllama_kernel = reconstruct_exllama_2bit_kernel; + } else if (bit == 3) { + reconstruct_exllama_kernel = reconstruct_exllama_3bit_kernel; + } else if (bit == 8) { + reconstruct_exllama_kernel = reconstruct_exllama_8bit_kernel; + } + + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + reconstruct_exllama_kernel<<>>( + b_q_weight, b_q_perm, b_gptq_qzeros, b_gptq_scales, height, width, groups, out); +} + +__global__ void gemm_half_q_half_alt_4bit_kernel( + const half2* __restrict__ vec, + const uint32_t* __restrict__ mat, + half* __restrict__ mul, + const half* __restrict__ scales, + const uint32_t* __restrict__ zeros, + const int* __restrict__ g_idx, + int batch, + int height, + int width) { + int zero_width = width / 8; + int vec_height = height * 4; + const int blockwidth2 = BLOCK_KN_SIZE / 2; + auto b = blockIdx.y * BLOCK_M_SIZE_MAX; + int b_end = min(BLOCK_M_SIZE_MAX, batch - b); + auto h = BLOCK_KN_SIZE * blockIdx.z / 8; + int h_end = min(BLOCK_KN_SIZE / 8, height - h) * 4; + auto w = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x; + + __shared__ half2 blockvec[BLOCK_M_SIZE_MAX][blockwidth2]; + if (threadIdx.x < h_end) { + for (int m = 0; m < b_end; ++m) { + blockvec[m][threadIdx.x] = vec[(m + b) * vec_height + blockIdx.z * BLOCK_KN_SIZE / 2 + threadIdx.x]; + } + } + + __shared__ half2 deq2[256][8]; + auto val = threadIdx.x / 8; + auto off = threadIdx.x % 8; + for (; val < 256; val += BLOCK_KN_SIZE / 8) { + deq2[val][off] = __halves2half2(__int2half_rn(val & 0xF), __int2half_rn(val >> 4)); + } + + if (blockIdx.z == 0) { + for (int m = 0; m < b_end; m++) + mul[(b + m) * width + w] = __int2half_rn(0); + } + __syncthreads(); + + int i = width * h + w; + int g_h = h * 8; + int k = 0; + int z_w = w / 8; + int z_mod = (w % 8) * 4; + half2 res2; + half res[BLOCK_M_SIZE_MAX] = {}; + + unsigned int tmp; + while (k < h_end) { + tmp = mat[i]; + half2 scales_tmp[4]; + half2 zeros_tmp[4]; + for (int tmp_k = 0; tmp_k < 4; tmp_k++) { + int g = g_idx[g_h + (k + tmp_k) * 2]; + int g2 = g_idx[g_h + (k + tmp_k) * 2 + 1]; + half scale_f = scales[g * width + w]; + half scale_f2 = scales[g2 * width + w]; + half2 scale = __halves2half2(scale_f, scale_f2); + half2 zero = __halves2half2( + __hmul(scale_f, __int2half_rn(-((zeros[g * zero_width + z_w] >> z_mod) & 0xF) - 1)), + __hmul(scale_f2, __int2half_rn(-((zeros[g2 * zero_width + z_w] >> z_mod) & 0xF) - 1))); + scales_tmp[tmp_k] = scale; + zeros_tmp[tmp_k] = zero; + } + for (int m = 0; m < b_end; m++) { +#ifndef USE_ROCM + res2 = {}; +#else + res2.x = __half_as_ushort(__float2half(0)); + res2.y = __half_as_ushort(__float2half(0)); +#endif + res2 = __hfma2(__hfma2(deq2[(tmp >> 0) & 0xff][off], scales_tmp[0], zeros_tmp[0]), blockvec[m][k + 0], res2); + res2 = __hfma2(__hfma2(deq2[(tmp >> 8) & 0xff][off], scales_tmp[1], zeros_tmp[1]), blockvec[m][k + 1], res2); + res2 = __hfma2(__hfma2(deq2[(tmp >> 16) & 0xff][off], scales_tmp[2], zeros_tmp[2]), blockvec[m][k + 2], res2); + res2 = __hfma2(__hfma2(deq2[(tmp >> 24) & 0xff][off], scales_tmp[3], zeros_tmp[3]), blockvec[m][k + 3], res2); +#ifndef USE_ROCM + res[m] = __hadd(res[m], __hadd(res2.x, res2.y)); +#else + res[m] = __hadd(res[m], __hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y))); +#endif + } + i += width; + k += 4; + } + for (int m = 0; m < b_end; m++) { + atomicAdd(&mul[(b + m) * width + w], res[m]); + } +} + +__global__ void gemm_half_q_half_alt_8bit_kernel( + const half2* __restrict__ vec, + const uint32_t* __restrict__ mat, + half* __restrict__ mul, + const half* __restrict__ scales, + const uint32_t* __restrict__ zeros, + const int* __restrict__ g_idx, + int batch, + int height, + int width) { + int zero_width = width / 4; + int vec_height = height * 2; + const int blockwidth2 = BLOCK_KN_SIZE / 2; + auto b = blockIdx.y * BLOCK_M_SIZE_MAX; + int b_end = min(BLOCK_M_SIZE_MAX, batch - b); + auto h = BLOCK_KN_SIZE * blockIdx.z / 4; + int h_end = min(BLOCK_KN_SIZE / 4, height - h) * 2; + auto w = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x; + + __shared__ half2 blockvec[BLOCK_M_SIZE_MAX][blockwidth2]; + if (threadIdx.x < h_end) { + for (int m = 0; m < b_end; ++m) { + blockvec[m][threadIdx.x] = vec[(m + b) * vec_height + blockIdx.z * BLOCK_KN_SIZE / 2 + threadIdx.x]; + } + } + + if (blockIdx.z == 0) { + for (int m = 0; m < b_end; m++) + mul[(b + m) * width + w] = __int2half_rn(0); + } + __syncthreads(); + + int i = width * h + w; + int g_h = h * 4; + int k = 0; + int z_w = w / 4; + int z_mod = (w % 4) * 8; + half2 res2; + half res[BLOCK_M_SIZE_MAX] = {}; + + unsigned int tmp; + while (k < h_end) { + tmp = mat[i]; + half2 scales_tmp[2]; + half2 zeros_tmp[2]; + for (int tmp_k = 0; tmp_k < 2; tmp_k++) { + int g = g_idx[g_h + (k + tmp_k) * 2]; + int g2 = g_idx[g_h + (k + tmp_k) * 2 + 1]; + half scale_f = scales[g * width + w]; + half scale_f2 = scales[g2 * width + w]; + half2 scale = __halves2half2(scale_f, scale_f2); + half2 zero = __halves2half2( + __hmul(scale_f, __int2half_rn(-((zeros[g * zero_width + z_w] >> z_mod) & 0xff) - 1)), + __hmul(scale_f2, __int2half_rn(-((zeros[g2 * zero_width + z_w] >> z_mod) & 0xff) - 1))); + scales_tmp[tmp_k] = scale; + zeros_tmp[tmp_k] = zero; + } + for (int m = 0; m < b_end; m++) { +#ifndef USE_ROCM + res2 = {}; +#else + res2.x = __half_as_ushort(__float2half(0)); + res2.y = __half_as_ushort(__float2half(0)); +#endif + half2 v12 = __halves2half2(__int2half_rn(tmp & 0xFF), __int2half_rn((tmp >> 8) & 0xFF)); + res2 = __hfma2(__hfma2(v12, scales_tmp[0], zeros_tmp[0]), blockvec[m][k + 0], res2); + half2 v34 = __halves2half2(__int2half_rn((tmp >> 16) & 0xFF), __int2half_rn((tmp >> 24) & 0xFF)); + res2 = __hfma2(__hfma2(v34, scales_tmp[1], zeros_tmp[1]), blockvec[m][k + 1], res2); +#ifndef USE_ROCM + res[m] = __hadd(res[m], __hadd(res2.x, res2.y)); +#else + res[m] = __hadd(res[m], __hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y))); +#endif + } + i += width; + k += 2; + } + for (int m = 0; m < b_end; m++) { + atomicAdd(&mul[(b + m) * width + w], res[m]); + } +} + +void gemm_half_q_half_alt( + const half* a, + const uint32_t* b_q_weight, + const uint32_t* b_gptq_qzeros, + const half* b_gptq_scales, + const int* b_g_idx, + half* c, + int size_m, + int size_n, + int size_k, + int bit) { + dim3 blockDim, gridDim; + blockDim.x = BLOCK_KN_SIZE; + blockDim.y = 1; + blockDim.z = 1; + gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE); + gridDim.y = DIVIDE(size_m, BLOCK_M_SIZE_MAX); + gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE); + + auto kernel = gemm_half_q_half_alt_4bit_kernel; + if (bit == 8) { + kernel = gemm_half_q_half_alt_8bit_kernel; + } + + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + kernel<<>>( + (const half2*)a, b_q_weight, c, b_gptq_scales, b_gptq_qzeros, b_g_idx, size_m, size_k / 32 * bit, size_n); +} + +template +__global__ void reconstruct_gptq_kernel( + const uint32_t* __restrict__ w, + const half* __restrict__ w_scales, + const uint32_t* __restrict__ w_zeros, + const int* __restrict__ g_idx, + const int height, + const int width, + const int group, + half* __restrict__ out) { + // Start of block + + auto column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x; + auto row = blockIdx.y * 32 / bit; + if (column >= width) return; + + // Views + + MatrixView_half_rw out_(out, height, width); + MatrixView_half w_scales_(w_scales, group, width); + T w_zeros_(w_zeros, group, width); + + uint32_t w_read = w[blockIdx.y * width + column]; + half* out_ptr = out_.item_ptr(row, column); + +#pragma unroll + for (int s = 0; s < 32; s += bit) { + int group = g_idx[row + s / bit]; + half w_scale = w_scales_.item(group, column); + uint32_t w_zero = w_zeros_.item(group, column) + 1; + half w_item = __hmul(__int2half_rn((int)((w_read >> s) & ((1 << bit) - 1)) - w_zero), w_scale); + *out_ptr = w_item; + out_ptr += out_.width; + } +} + +__global__ void reconstruct_gptq_3bit_kernel( + const uint32_t* __restrict__ w, + const half* __restrict__ w_scales, + const uint32_t* __restrict__ w_zeros, + const int* __restrict__ g_idx, + const int height, + const int width, + const int group, + half* __restrict__ out) { + // Start of block + auto column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x; + auto row = blockIdx.y * 32; + if (column >= width) return; + + // Views + + MatrixView_half_rw out_(out, height, width); + MatrixView_half w_scales_(w_scales, group, width); + MatrixView_q3_row w_zeros_(w_zeros, group, width); + + uint32_t w1 = w[(blockIdx.y * 3) * width + column]; + uint32_t w2 = w[(blockIdx.y * 3 + 1) * width + column]; + uint32_t w3 = w[(blockIdx.y * 3 + 2) * width + column]; + half* out_ptr = out_.item_ptr(row, column); + +#pragma unroll + for (int i = 0; i < 32; i += 1) { + int group = g_idx[row + i]; + half w_scale = w_scales_.item(group, column); + uint32_t w_zero = w_zeros_.item(group, column) + 1; + int w_item; + if (i == 10) { + w_item = (w1 >> 30) | ((w2 << 2) & 0x4); + } else if (i == 21) { + w_item = (w2 >> 31) | ((w3 << 1) & 0x6); + } else if (i < 10) { + w_item = ((w1 >> (i * 3)) & 0x7); + } else if (i < 21) { + w_item = ((w2 >> (i * 3 - 32)) & 0x7); + } else { + w_item = ((w3 >> (i * 3 - 64)) & 0x7); + } + *out_ptr = __hmul(__int2half_rn(w_item - w_zero), w_scale); + out_ptr += out_.width; + } +} + +void reconstruct_gptq( + const uint32_t* b_q_weight, + const uint32_t* b_gptq_qzeros, + const half* b_gptq_scales, + const int* b_g_idx, + half* out, + int height, + int width, + int groups, + int bit) { + dim3 blockDim, gridDim; + blockDim.x = BLOCK_KN_SIZE; + blockDim.y = 1; + gridDim.y = DIVIDE(height, 32 / bit); + gridDim.x = DIVIDE(width, BLOCK_KN_SIZE); + + auto kernel = reconstruct_gptq_kernel; + if (bit == 2) { + kernel = reconstruct_gptq_kernel; + } else if (bit == 8) { + kernel = reconstruct_gptq_kernel; + } else if (bit == 3) { + kernel = reconstruct_gptq_3bit_kernel; + gridDim.y = DIVIDE(height, 32); + } + + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + kernel<<>>( + b_q_weight, b_gptq_scales, b_gptq_qzeros, b_g_idx, height, width, groups, out); +} + +void gemm_half_q_half_cuda( + cublasHandle_t cublas_handle, + const half* a, + const uint32_t* b_q_weight, + const uint32_t* b_gptq_qzeros, + const half* b_gptq_scales, + const int* b_g_idx, + half* c, + half* temp_dq, + int size_m, + int size_n, + int size_k, + int groups, + bool use_shuffle, + int bit) { + bool use_reconstruct; + if (use_shuffle) { + use_reconstruct = ((bit == 8 && size_m > MAX_Q_GEMM_ROWS_8BIT) || (bit != 8 && size_m > MAX_Q_GEMM_ROWS)); + } else { + // The 2/3-bit kernels are somehow slower than dequant + gemm baseline, so + // we disabled them for now. + use_reconstruct = (bit < 4 || size_m > MAX_ALT_GEMM_ROWS); + } + if (use_reconstruct) { + // Reconstruct FP16 matrix, then cuBLAS + if (use_shuffle) { + reconstruct_exllama(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx, temp_dq, size_k, size_n, groups, bit); + } else { + reconstruct_gptq(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx, temp_dq, size_k, size_n, groups, bit); + } + + const half alpha = __float2half(1.0f); + const half beta = __float2half(0.0f); + cublasHgemm( + cublas_handle, + CUBLAS_OP_N, + CUBLAS_OP_N, + size_n, + size_m, + size_k, + &alpha, + temp_dq, + size_n, + a, + size_k, + &beta, + c, + size_n); + } else if (use_shuffle) { + // Quantized matmul + int max_chunks = size_m / BLOCK_M_SIZE_MAX; + int last_chunk = max_chunks * BLOCK_M_SIZE_MAX; + int last_chunk_size = size_m - last_chunk; + + if (max_chunks) { + gemm_half_q_half_cuda_part( + a, + b_q_weight, + b_gptq_qzeros, + b_gptq_scales, + b_g_idx, + c, + last_chunk, + size_n, + size_k, + BLOCK_M_SIZE_MAX, + groups, + bit); + } + + if (last_chunk_size) { + gemm_half_q_half_cuda_part( + a + last_chunk * size_k, + b_q_weight, + b_gptq_qzeros, + b_gptq_scales, + b_g_idx, + c + last_chunk * size_n, + last_chunk_size, + size_n, + size_k, + last_chunk_size, + groups, + bit); + } + } else { + gemm_half_q_half_alt(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx, c, size_m, size_n, size_k, bit); + } +} + +__global__ void shuffle_4bit_kernel(uint32_t* __restrict__ b_q_weight, const int size_k, const int size_n) { + auto n = blockIdx.x * THREADS_X + threadIdx.x; + if (n >= size_n) return; + int k = 0; + uint32_t* b_ptr = b_q_weight + n; + while (k < size_k) { + shuffle_4bit_8(b_ptr, size_n); + b_ptr += 1 * size_n; + k += 8; + } +} + +__global__ void shuffle_8bit_kernel(uint32_t* __restrict__ b_q_weight, const int size_k, const int size_n) { + auto n = blockIdx.x * THREADS_X + threadIdx.x; + if (n >= size_n) return; + int k = 0; + uint32_t* b_ptr = b_q_weight + n; + while (k < size_k) { + shuffle_8bit_4(b_ptr, size_n); + b_ptr += 1 * size_n; + k += 4; + } +} + +__global__ void shuffle_2bit_kernel(uint32_t* __restrict__ b_q_weight, const int size_k, const int size_n) { + auto n = blockIdx.x * THREADS_X + threadIdx.x; + if (n >= size_n) return; + int k = 0; + uint32_t* b_ptr = b_q_weight + n; + while (k < size_k) { + shuffle_2bit_16(b_ptr, size_n); + b_ptr += 1 * size_n; + k += 16; + } +} + +__global__ void shuffle_3bit_kernel(uint32_t* __restrict__ b_q_weight, const int size_k, const int size_n) { + auto n = blockIdx.x * THREADS_X + threadIdx.x; + if (n >= size_n) return; + int k = 0; + uint32_t* b_ptr = b_q_weight + n; + while (k < size_k) { + shuffle_3bit_32(b_ptr, size_n); + b_ptr += 3 * size_n; + k += 32; + } +} + +__global__ void make_sequential_4bit_kernel( + const uint32_t* __restrict__ w, uint32_t* __restrict__ w_new, const int* __restrict__ q_perm, const int w_width) { + const uint64_t* w2 = (uint64_t*)w; + uint64_t* w_new2 = (uint64_t*)w_new; + int w2_stride = w_width >> 1; + auto w2_column = THREADS_X * blockIdx.x + threadIdx.x; + if (w2_column >= w2_stride) return; + auto w_new2_row = blockIdx.y; + int q_perm_idx = w_new2_row << 3; + uint64_t dst = 0; + +#pragma unroll + for (int i = 0; i < 8; i++) { + int source_row = q_perm[q_perm_idx++]; + + int w2_row = source_row >> 3; + int w2_subrow = source_row & 0x07; + int w2_row_shift = w2_subrow << 2; + int wnew2_row_shift = i << 2; + + uint64_t src = w2[w2_row * w2_stride + w2_column]; + src >>= w2_row_shift; + src &= 0x0000000f0000000f; + src <<= wnew2_row_shift; + dst |= src; + } + w_new2[w_new2_row * w2_stride + w2_column] = dst; +} + +__global__ void make_sequential_2bit_kernel( + const uint32_t* __restrict__ w, uint32_t* __restrict__ w_new, const int* __restrict__ q_perm, const int w_width) { + const uint64_t* w2 = (uint64_t*)w; + uint64_t* w_new2 = (uint64_t*)w_new; + int w2_stride = w_width >> 1; + auto w2_column = THREADS_X * blockIdx.x + threadIdx.x; + if (w2_column >= w2_stride) return; + auto w_new2_row = blockIdx.y; + int q_perm_idx = w_new2_row << 4; + uint64_t dst = 0; + +#pragma unroll + for (int i = 0; i < 16; i++) { + int source_row = q_perm[q_perm_idx++]; + + int w2_row = source_row >> 4; + int w2_subrow = source_row & 0x0f; + int w2_row_shift = w2_subrow << 1; + int wnew2_row_shift = i << 1; + + uint64_t src = w2[w2_row * w2_stride + w2_column]; + src >>= w2_row_shift; + src &= 0x0000000300000003; + src <<= wnew2_row_shift; + dst |= src; + } + w_new2[w_new2_row * w2_stride + w2_column] = dst; +} + +__global__ void make_sequential_3bit_kernel( + const uint32_t* __restrict__ w, uint32_t* __restrict__ w_new, const int* __restrict__ q_perm, const int w_width) { + auto w_column = THREADS_X * blockIdx.x + threadIdx.x; + if (w_column >= w_width) return; + auto w_new_row = blockIdx.y * 3; + auto q_perm_idx = blockIdx.y << 5; + uint32_t dst[3] = {0, 0, 0}; + +#pragma unroll + for (int i = 0; i < 32; i++) { + int source_row = q_perm[q_perm_idx++]; + int z_w = (source_row / 32) * 3; + int z_mod = source_row % 32; + int z_bit; + + if (z_mod != 10) { + if (z_mod != 21) { + z_bit = z_mod; + if (z_bit > 21) { + z_bit *= 3; + z_bit -= 64; + z_w += 2; + } else if (z_bit > 10) { + z_bit *= 3; + z_bit -= 32; + z_w += 1; + } else { + z_bit *= 3; + } + } else { + z_w += 1; + } + } + + uint64_t src; + if (z_mod == 10) { + src = (w[z_w * w_width + w_column] >> 30) | ((w[(z_w + 1) * w_width + w_column] << 2) & 0x4); + } else if (z_mod == 21) { + src = (w[z_w * w_width + w_column] >> 31) | ((w[(z_w + 1) * w_width + w_column] << 1) & 0x6); + } else { + src = w[z_w * w_width + w_column]; + src >>= z_bit; + src &= 0x07; + } + + z_w = 0; + if (i != 10) { + if (i != 21) { + z_bit = i; + if (z_bit > 21) { + z_bit *= 3; + z_bit -= 64; + z_w += 2; + } else if (z_bit > 10) { + z_bit *= 3; + z_bit -= 32; + z_w += 1; + } else { + z_bit *= 3; + } + } else { + z_w += 1; + } + } + if (i == 10) { + dst[z_w] |= (src & 0x03) << 30; + dst[z_w + 1] |= ((src & 0x4) >> 2); + } else if (i == 21) { + dst[z_w] |= (src & 0x01) << 31; + dst[z_w + 1] |= ((src & 0x6) >> 1); + } else { + dst[z_w] |= (src << z_bit); + } + } + w_new[w_new_row * w_width + w_column] = dst[0]; + w_new[(w_new_row + 1) * w_width + w_column] = dst[1]; + w_new[(w_new_row + 2) * w_width + w_column] = dst[2]; +} + +__global__ void make_sequential_8bit_kernel( + const uint32_t* __restrict__ w, uint32_t* __restrict__ w_new, const int* __restrict__ q_perm, const int w_width) { + const uint64_t* w2 = (uint64_t*)w; + uint64_t* w_new2 = (uint64_t*)w_new; + int w2_stride = w_width >> 1; + auto w2_column = THREADS_X * blockIdx.x + threadIdx.x; + if (w2_column >= w2_stride) return; + auto w_new2_row = blockIdx.y; + int q_perm_idx = w_new2_row << 2; + uint64_t dst = 0; + +#pragma unroll + for (int i = 0; i < 4; i++) { + int source_row = q_perm[q_perm_idx++]; + + int w2_row = source_row >> 2; + int w2_subrow = source_row & 0x03; + int w2_row_shift = w2_subrow << 3; + int wnew2_row_shift = i << 3; + + uint64_t src = w2[w2_row * w2_stride + w2_column]; + src >>= w2_row_shift; + src &= 0x000000ff000000ff; + src <<= wnew2_row_shift; + dst |= src; + } + w_new2[w_new2_row * w2_stride + w2_column] = dst; +} + +void shuffle_exllama_weight(uint32_t* q_weight, int* q_perm, int height, int width, int bit) { + if (q_perm) { + uint32_t* new_qweight = NULL; + cudaMalloc(&new_qweight, height / 32 * bit * width * sizeof(uint32_t)); + + dim3 blockDim, gridDim; + blockDim.x = THREADS_X; + blockDim.y = 1; + gridDim.x = DIVIDE(width, THREADS_X); + gridDim.y = height / 32 * bit; + + auto kernel = make_sequential_4bit_kernel; + if (bit == 2) { + kernel = make_sequential_2bit_kernel; + } else if (bit == 3) { + kernel = make_sequential_3bit_kernel; + gridDim.y = height / 32; + } else if (bit == 8) { + kernel = make_sequential_8bit_kernel; + } + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + kernel<<>>(q_weight, new_qweight, q_perm, width); + // Replace qweights + cudaMemcpyAsync(q_weight, new_qweight, height / 32 * bit * width * sizeof(uint32_t), cudaMemcpyDeviceToDevice); + // Cleanup + cudaDeviceSynchronize(); + cudaFree(new_qweight); + } + dim3 blockDim, gridDim; + blockDim.x = THREADS_X; + blockDim.y = 1; + gridDim.x = DIVIDE(width, THREADS_X); + gridDim.y = 1; + auto shuffle_kernel = shuffle_4bit_kernel; + if (bit == 2) { + shuffle_kernel = shuffle_2bit_kernel; + } else if (bit == 3) { + shuffle_kernel = shuffle_3bit_kernel; + } else if (bit == 8) { + shuffle_kernel = shuffle_8bit_kernel; + } + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + shuffle_kernel<<>>(q_weight, height, width); +} + +} // namespace gptq +} // namespace sglang + +torch::Tensor gptq_gemm( + torch::Tensor a, + torch::Tensor b_q_weight, + torch::Tensor b_gptq_qzeros, + torch::Tensor b_gptq_scales, + torch::Tensor b_g_idx, + bool use_shuffle, + int64_t bit) { + const at::cuda::OptionalCUDAGuard device_guard(device_of(a)); + auto options = torch::TensorOptions().dtype(a.dtype()).device(a.device()); + at::Tensor c = torch::empty({a.size(0), b_q_weight.size(1)}, options); + at::Tensor temp_dq = torch::empty({b_q_weight.size(0) * 32 / bit, b_q_weight.size(1)}, options); + + sglang::gptq::gemm_half_q_half_cuda( + at::cuda::getCurrentCUDABlasHandle(), + (const half*)a.data_ptr(), + (const uint32_t*)b_q_weight.data_ptr(), + (const uint32_t*)b_gptq_qzeros.data_ptr(), + (const half*)b_gptq_scales.data_ptr(), + b_g_idx.device().is_meta() ? NULL : (const int*)b_g_idx.data_ptr(), + (half*)c.data_ptr(), + (half*)temp_dq.data_ptr(), + c.size(0), // m + c.size(1), // n + a.size(1), // k + b_gptq_qzeros.size(0), // group number + use_shuffle, + bit); + return c; +} + +void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int64_t bit) { + const at::cuda::OptionalCUDAGuard device_guard(device_of(q_weight)); + sglang::gptq::shuffle_exllama_weight( + (uint32_t*)q_weight.data_ptr(), + q_perm.device().is_meta() || q_perm.numel() == 0 ? NULL : (int*)q_perm.data_ptr(), + q_weight.size(0) * 32 / bit, + q_weight.size(1), + bit); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/matrix_view.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/matrix_view.cuh new file mode 100644 index 00000000..3dfc8794 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/matrix_view.cuh @@ -0,0 +1,269 @@ +/* +Adapted from https://github.com/turboderp/exllamav2 and +https://github.com/turboderp/exllama +*/ + +#ifndef _matrix_view_cuh +#define _matrix_view_cuh + +#include +#include + +#include "qdq_util.cuh" + +namespace sglang { +namespace gptq { + +class MatrixView_half { + public: + const half* data; + const int height; + const int width; + + __device__ __forceinline__ MatrixView_half(const half* data, const int height, const int width) + : data(data), height(height), width(width) {} + + __device__ __forceinline__ half item(int row, int column) const { + return data[row * width + column]; + } + __device__ __forceinline__ half2 item_half2(int row, int column) const { + return ((half2*)data)[(row * width + column) / 2]; + } + __device__ __forceinline__ half2 item_half2half2(int row, int column) const { + return __half2half2(data[row * width + column]); + } + __device__ __forceinline__ const half* item_ptr(int row, int column) const { + return &data[row * width + column]; + } + + __device__ __forceinline__ void item4(half (&items)[4], int row, int column) const { + half2* ptr = (half2*)item_ptr(row, column); + half2 i01 = ptr[0]; + half2 i23 = ptr[1]; + items[0] = __low2half(i01); + items[1] = __high2half(i01); + items[2] = __low2half(i23); + items[3] = __high2half(i23); + } + __device__ __forceinline__ void item4_f(float (&items)[4], int row, int column) const { + half2* ptr = (half2*)item_ptr(row, column); + half2 i01 = ptr[0]; + half2 i23 = ptr[1]; + items[0] = __half2float(__low2half(i01)); + items[1] = __half2float(__high2half(i01)); + items[2] = __half2float(__low2half(i23)); + items[3] = __half2float(__high2half(i23)); + } + + __device__ __forceinline__ void item4_h2(half2 (&items)[4], int row, int column) const { + half2* ptr = (half2*)item_ptr(row, column); + half2 i01 = ptr[0]; + half2 i23 = ptr[1]; + items[0] = __half2half2(__low2half(i01)); + items[1] = __half2half2(__high2half(i01)); + items[2] = __half2half2(__low2half(i23)); + items[3] = __half2half2(__high2half(i23)); + } +}; + +class MatrixView_half_rw { + public: + half* data; + const int height; + const int width; + + __device__ __forceinline__ MatrixView_half_rw(half* data, const int height, const int width) + : data(data), height(height), width(width) {} + + __device__ __forceinline__ half item(int row, int column) const { + return data[row * width + column]; + } + __device__ __forceinline__ half2 item_half2(int row, int column) const { + return ((half2*)data)[(row * width + column) / 2]; + } + __device__ __forceinline__ half* item_ptr(int row, int column) { + return &data[row * width + column]; + } + __device__ __forceinline__ void set(int row, int column, half value) { + data[row * width + column] = value; + } + __device__ __forceinline__ void set_half2(int row, int column, half2 value) { + ((half2*)data)[(row * width + column) / 2] = value; + } + + __device__ __forceinline__ void set4(int row, int column, half v0, half v1, half v2, half v3) { + half2 v01 = __halves2half2(v0, v1); + half2 v23 = __halves2half2(v2, v3); + half2* ptr = (half2*)item_ptr(row, column); + ptr[0] = v01; + ptr[1] = v23; + } +}; + +class MatrixView_q4_row { + public: + const uint32_t* data; + const int height; + const int width; + + __device__ __forceinline__ MatrixView_q4_row(const uint32_t* data, const int height, const int width) + : data(data), height(height), width(width) {} + + __device__ __forceinline__ int item(int row, int column) const { + int shift = (column & 0x07) * 4; + return (data[row * width / 8 + column / 8] >> shift) & 0x0f; + } + + __device__ __forceinline__ void item2(int (&items)[2], int row, int column) const { + int shift = (column & 0x07) * 4; + uint32_t d = data[row * width / 8 + column / 8] >> shift; + items[0] = d & 0x0f; + items[1] = (d >> 4) & 0x0f; + } + + __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const { + int shift = (column & 0x07) * 4; + uint32_t d = data[row * width / 8 + column / 8] >> shift; + items[0] = d & 0x0f; + items[1] = (d >> 4) & 0x0f; + items[2] = (d >> 8) & 0x0f; + items[3] = (d >> 12) & 0x0f; + } +}; + +class MatrixView_q4_column { + public: + const uint32_t* data; + const int height; + const int width; + + __device__ __forceinline__ MatrixView_q4_column(const uint32_t* data, const int height, const int width) + : data(data), height(height), width(width) {} + + __device__ __forceinline__ int item(int row, int column) const { + int shift = (row & 0x07) * 4; + return (data[row / 8 * width + column] >> shift) & 0x0f; + } + + __device__ __forceinline__ uint32_t item_uint32_t(int row, int column) { + return data[row / 8 * width + column]; + } + __device__ __forceinline__ const uint32_t* item_uint32_ptr(int row, int column) { + return &data[row / 8 * width + column]; + } +}; + +class MatrixView_q2_row { + public: + const uint32_t* data; + const int height; + const int width; + + __device__ __forceinline__ MatrixView_q2_row(const uint32_t* data, const int height, const int width) + : data(data), height(height), width(width) {} + + __device__ __forceinline__ int item(int row, int column) const { + int shift = (column & 0x0f) * 2; + return (data[row * width / 16 + column / 16] >> shift) & 0x03; + } + + __device__ __forceinline__ void item2(int (&items)[2], int row, int column) const { + int shift = (column & 0x0f) * 2; + uint32_t d = data[row * width / 16 + column / 16] >> shift; + items[0] = d & 0x03; + items[1] = (d >> 2) & 0x03; + } + + __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const { + int shift = (column & 0x0f) * 2; + uint32_t d = data[row * width / 16 + column / 16] >> shift; + items[0] = d & 0x03; + items[1] = (d >> 2) & 0x03; + items[2] = (d >> 4) & 0x03; + items[3] = (d >> 6) & 0x03; + } +}; + +class MatrixView_q3_row { + public: + const uint32_t* data; + const int height; + const int width; + + __device__ __forceinline__ MatrixView_q3_row(const uint32_t* data, const int height, const int width) + : data(data), height(height), width(width) {} + + __device__ __forceinline__ int item(int row, int column) const { + int z_w = column * 3 / 32; + int z_mod = column & 0x1f; + + if (z_mod == 10) { + return (data[row * width * 3 / 32 + z_w] >> 30) | ((data[row * width * 3 / 32 + (z_w + 1)] << 2) & 0x4); + } else if (z_mod == 21) { + return (data[row * width * 3 / 32 + z_w] >> 31) | ((data[row * width * 3 / 32 + (z_w + 1)] << 1) & 0x6); + } else if (z_mod < 10) { + return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3)) & 0x07; + } else if (z_mod < 21) { + return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3 - 32)) & 0x07; + } else { + return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3 - 64)) & 0x07; + } + } + + __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const { + int shift = (column & 0x1f); + uint32_t d; + if (shift <= 4) { + d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3); + } else if (shift == 8) { + d = (data[row * width / 32 * 3 + column * 3 / 32] >> 24) | + ((data[row * width / 32 * 3 + column * 3 / 32 + 1] & 0x0f) << 8); + } else if (shift <= 16) { + d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3 - 32); + } else if (shift == 20) { + d = (data[row * width / 32 * 3 + column * 3 / 32] >> 28) | + ((data[row * width / 32 * 3 + column * 3 / 32 + 1] & 0xff) << 4); + } else { + d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3 - 64); + } + items[0] = d & 0x07; + items[1] = (d >> 3) & 0x07; + items[2] = (d >> 6) & 0x07; + items[3] = (d >> 9) & 0x07; + } +}; + +class MatrixView_q8_row { + public: + const uint32_t* data; + const int height; + const int width; + + __device__ __forceinline__ MatrixView_q8_row(const uint32_t* data, const int height, const int width) + : data(data), height(height), width(width) {} + + __device__ __forceinline__ int item(int row, int column) const { + int shift = (column & 0x03) * 8; + return (data[row * width / 4 + column / 4] >> shift) & 0xff; + } + + __device__ __forceinline__ void item2(int (&items)[2], int row, int column) const { + int shift = (column & 0x03) * 8; + uint32_t d = data[row * width / 4 + column / 4] >> shift; + items[0] = d & 0xff; + items[1] = (d >> 8) & 0xff; + } + + __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const { + int shift = (column & 0x03) * 2; + uint32_t d = data[row * width / 4 + column / 4] >> shift; + items[0] = d & 0xff; + items[1] = (d >> 8) & 0xff; + items[2] = (d >> 16) & 0xff; + items[3] = (d >> 24) & 0xff; + } +}; + +} // namespace gptq +} // namespace sglang +#endif diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/qdq_2.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/qdq_2.cuh new file mode 100644 index 00000000..4a75d7b5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/qdq_2.cuh @@ -0,0 +1,74 @@ +/* +Copied from https://github.com/turboderp/exllamav2 +*/ + +#ifndef _qdq_2_cuh +#define _qdq_2_cuh + +#include "qdq_util.cuh" + +namespace sglang { +namespace gptq { + +// Permutation: +// +// ffddbb99 77553311 eeccaa88 66442200 + +__forceinline__ __device__ void shuffle_2bit_16(uint32_t* q, int stride) { + uint32_t qa = q[0]; + uint32_t qb = 0; + +#pragma unroll + for (int i = 0; i < 8; i++) { + uint32_t qa0 = qa & 0x03; + uint32_t qa1 = (qa & 0x0c) >> 2; + qa >>= 4; + qb |= (qa1 << (i * 2 + 16)); + qb |= (qa0 << (i * 2)); + } + q[0] = qb; +} + +__forceinline__ __device__ void dequant_2bit_16(const uint32_t q_0, half2 (&dq)[8], int stride, const uint32_t zero) { + const uint32_t c0 = 0x64006400; + const half y4_ = __float2half_rn(1.0f / 4.0f); + const half y16_ = __float2half_rn(1.0f / 16.0f); + const half y64_ = __float2half_rn(1.0f / 64.0f); + const half2 y4 = __halves2half2(y4_, y4_); + const half2 y16 = __halves2half2(y16_, y16_); + const half2 y64 = __halves2half2(y64_, y64_); + + const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero); + const half z4_ = __hsub(__int2half_rn(-256), __int2half_rn(zero)); + const half z16_ = __hsub(__int2half_rn(-64), __int2half_rn(zero)); + const half z64_ = __hsub(__int2half_rn(-16), __int2half_rn(zero)); + const half2 z1 = __half2half2(z1_.as_half); + const half2 z4 = __half2half2(z4_); + const half2 z16 = __half2half2(z16_); + const half2 z64 = __half2half2(z64_); + + uint32_t qa = q_0; + half2_uint32 q0((qa & 0x00030003) | c0); // half2(q[ 0], q[ 1]) + 1024 + half2_uint32 q1((qa & 0x000c000c) | c0); // half2(q[ 2], q[ 3]) * 4 + 1024 + half2_uint32 q2((qa & 0x00300030) | c0); // half2(q[ 4], q[ 5]) * 16 + 1024 + half2_uint32 q3((qa & 0x00c000c0) | c0); // half2(q[ 6], q[ 7]) * 64 + 1024 + qa >>= 8; + half2_uint32 q4((qa & 0x00030003) | c0); // half2(q[ 8], q[ 8]) + 1024 + half2_uint32 q5((qa & 0x000c000c) | c0); // half2(q[10], q[11]) * 4 + 1024 + half2_uint32 q6((qa & 0x00300030) | c0); // half2(q[12], q[13]) * 16 + 1024 + half2_uint32 q7((qa & 0x00c000c0) | c0); // half2(q[14], q[15]) * 64 + 1024 + + dq[0] = __hadd2(q0.as_half2, z1); + dq[1] = __hfma2(q1.as_half2, y4, z4); + dq[2] = __hfma2(q2.as_half2, y16, z16); + dq[3] = __hfma2(q3.as_half2, y64, z64); + dq[4] = __hadd2(q4.as_half2, z1); + dq[5] = __hfma2(q5.as_half2, y4, z4); + dq[6] = __hfma2(q6.as_half2, y16, z16); + dq[7] = __hfma2(q7.as_half2, y64, z64); +} + +} // namespace gptq +} // namespace sglang + +#endif diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/qdq_3.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/qdq_3.cuh new file mode 100644 index 00000000..5996f342 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/qdq_3.cuh @@ -0,0 +1,146 @@ +#ifndef _qdq_3_cuh +#define _qdq_3_cuh + +#include "qdq_util.cuh" + +namespace sglang { +namespace gptq { +// Permutation: +// +// v9997775 55333111 u8886664 44222000 (u, v lsb) +// vjjjhhhf ffdddbbb uiiiggge eecccaaa +// vtttrrrp ppnnnlll usssqqqo oommmkkk + +__forceinline__ __device__ void shuffle_3bit_32(uint32_t* q, int stride) { + uint32_t qa = q[0 * stride]; + uint32_t qb = q[1 * stride]; + uint32_t qc = q[2 * stride]; + + // qa: aa999888 77766655 54443332 22111000 + // qb: lkkkjjji iihhhggg fffeeedd dcccbbba + // qc: vvvuuutt tsssrrrq qqpppooo nnnmmmll + + uint32_t qd = qc >> 26; + qc <<= 4; + qc |= qb >> 28; + qb <<= 2; + qb |= qa >> 30; + + // qa: ..999888 77766655 54443332 22111000 + // qb: ..jjjiii hhhgggff feeedddc ccbbbaaa + // qc: ..tttsss rrrqqqpp pooonnnm mmlllkkk + // qd: vvvuuu + + uint32_t za = 0; + uint32_t zb = 0; + uint32_t zc = 0; + + for (int i = 0; i < 5; i++) { + uint32_t t0 = qa & 0x07; + uint32_t t1 = (qa & 0x38) >> 3; + qa >>= 6; + za |= (t0 << (i * 3)); + za |= (t1 << (i * 3 + 16)); + } + for (int i = 0; i < 5; i++) { + uint32_t t0 = qb & 0x07; + uint32_t t1 = (qb & 0x38) >> 3; + qb >>= 6; + zb |= (t0 << (i * 3)); + zb |= (t1 << (i * 3 + 16)); + } + for (int i = 0; i < 5; i++) { + uint32_t t0 = qc & 0x07; + uint32_t t1 = (qc & 0x38) >> 3; + qc >>= 6; + zc |= (t0 << (i * 3)); + zc |= (t1 << (i * 3 + 16)); + } + + // za: 9997775 55333111 8886664 44222000 + // zb: jjjhhhf ffdddbbb iiiggge eecccaaa + // zc: tttrrrp ppnnnlll sssqqqo oommmkkk + // qd: vvvuuu + + za |= ((qd & 0x01) >> 0) << 15; + zb |= ((qd & 0x02) >> 1) << 15; + zc |= ((qd & 0x04) >> 2) << 15; + za |= ((qd & 0x08) >> 3) << 31; + zb |= ((qd & 0x10) >> 4) << 31; + zc |= ((qd & 0x20) >> 5) << 31; + + // za: v9997775 55333111 u8886664 44222000 (u, v lsb) + // zb: vjjjhhhf ffdddbbb uiiiggge eecccaaa + // zc: vtttrrrp ppnnnlll usssqqqo oommmkkk + + q[0 * stride] = za; + q[1 * stride] = zb; + q[2 * stride] = zc; +} + +__forceinline__ __device__ void dequant_3bit_32( + const uint32_t q_0, const uint32_t q_1, const uint32_t q_2, half2 (&dq)[16], int stride, const uint32_t zero) { + const uint32_t c0 = 0x64006400; + const half y8_ = __float2half_rn(1.0f / 8.0f); + const half y64_ = __float2half_rn(1.0f / 64.0f); + const half2 y8 = __halves2half2(y8_, y8_); + const half2 y64 = __halves2half2(y64_, y64_); + const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero); + const half z8_ = __hsub(__int2half_rn(-128), __int2half_rn(zero)); + const half z64_ = __hsub(__int2half_rn(-16), __int2half_rn(zero)); + const half2 z1 = __halves2half2(z1_.as_half, z1_.as_half); + const half2 z8 = __halves2half2(z8_, z8_); + const half2 z64 = __halves2half2(z64_, z64_); + + uint32_t qa = q_0; + uint32_t qb = q_1; + uint32_t qc = q_2; + + half2_uint32 q0((qa & 0x00070007) | c0); // half2(q[ 0], q[ 1]) + 1024 + half2_uint32 q1((qa & 0x00380038) | c0); // half2(q[ 2], q[ 3]) * 8 + 1024 + qa >>= 6; + half2_uint32 q2((qa & 0x00070007) | c0); // half2(q[ 4], q[ 5]) + 1024 + half2_uint32 q3((qa & 0x00380038) | c0); // half2(q[ 6], q[ 7]) * 8 + 1024 + half2_uint32 q4((qa & 0x01c001c0) | c0); // half2(q[ 8], q[ 9]) * 64 + 1024 + qa >>= 9; + qa &= 0x00010001; + half2_uint32 q5((qb & 0x00070007) | c0); // half2(q[10], q[11]) + 1024 + half2_uint32 q6((qb & 0x00380038) | c0); // half2(q[12], q[13]) * 8 + 1024 + qb >>= 6; + half2_uint32 q7((qb & 0x00070007) | c0); // half2(q[14], q[15]) + 1024 + half2_uint32 q8((qb & 0x00380038) | c0); // half2(q[16], q[17]) * 8 + 1024 + half2_uint32 q9((qb & 0x01c001c0) | c0); // half2(q[18], q[19]) * 64 + 1024 + qb >>= 8; + qb &= 0x00020002; + half2_uint32 q10((qc & 0x00070007) | c0); // half2(q[20], q[21]) + 1024 + half2_uint32 q11((qc & 0x00380038) | c0); // half2(q[22], q[23]) * 8 + 1024 + qc >>= 6; + half2_uint32 q12((qc & 0x00070007) | c0); // half2(q[24], q[25]) + 1024 + half2_uint32 q13((qc & 0x00380038) | c0); // half2(q[26], q[27]) * 8 + 1024 + half2_uint32 q14((qc & 0x01c001c0) | c0); // half2(q[28], q[29]) * 64 + 1024 + qc >>= 7; + qc &= 0x00040004; + half2_uint32 q15((qa | qb | qc) | c0); + + dq[0] = __hadd2(q0.as_half2, z1); + dq[1] = __hfma2(q1.as_half2, y8, z8); + dq[2] = __hadd2(q2.as_half2, z1); + dq[3] = __hfma2(q3.as_half2, y8, z8); + dq[4] = __hfma2(q4.as_half2, y64, z64); + dq[5] = __hadd2(q5.as_half2, z1); + dq[6] = __hfma2(q6.as_half2, y8, z8); + dq[7] = __hadd2(q7.as_half2, z1); + dq[8] = __hfma2(q8.as_half2, y8, z8); + dq[9] = __hfma2(q9.as_half2, y64, z64); + dq[10] = __hadd2(q10.as_half2, z1); + dq[11] = __hfma2(q11.as_half2, y8, z8); + dq[12] = __hadd2(q12.as_half2, z1); + dq[13] = __hfma2(q13.as_half2, y8, z8); + dq[14] = __hfma2(q14.as_half2, y64, z64); + dq[15] = __hadd2(q15.as_half2, z1); +} + +} // namespace gptq +} // namespace sglang + +#endif diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/qdq_4.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/qdq_4.cuh new file mode 100644 index 00000000..c96af471 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/qdq_4.cuh @@ -0,0 +1,114 @@ +/* +Copied from https://github.com/turboderp/exllamav2 +*/ + +#ifndef _qdq_4_cuh +#define _qdq_4_cuh + +#include "qdq_util.cuh" + +namespace sglang { +namespace gptq { +// Permutation: +// +// 77775555 33331111 66664444 22220000 + +__forceinline__ __device__ void shuffle_4bit_8(uint32_t* q, int stride) { + uint32_t qa = q[0]; + uint32_t qb = 0; + +#pragma unroll + for (int i = 0; i < 4; i++) { + uint32_t qa0 = qa & 0x0f; + uint32_t qa1 = (qa & 0xf0) >> 4; + qa >>= 8; + qb |= (qa1 << (i * 4 + 16)); + qb |= (qa0 << (i * 4)); + } + q[0] = qb; +} + +__forceinline__ __device__ void dequant_4bit_8(const uint32_t q_0, half2 (&dq)[4], int stride, const uint32_t zero) { + const uint32_t c0 = 0x64006400; + const half y16_ = __float2half_rn(1.0f / 16.0f); + const half2 y16 = __halves2half2(y16_, y16_); + const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero); + const half z16_ = __hsub(__int2half_rn(-64), __int2half_rn(zero)); + const half2 z1 = __half2half2(z1_.as_half); + const half2 z16 = __half2half2(z16_); + + uint32_t qa = q_0; + half2_uint32 q0((qa & 0x000f000f) | c0); // half2(q[ 0], q[ 1]) + 1024 + half2_uint32 q1((qa & 0x00f000f0) | c0); // half2(q[ 2], q[ 3]) * 16 + 1024 + qa >>= 8; + half2_uint32 q2((qa & 0x000f000f) | c0); // half2(q[ 4], q[ 5]) + 1024 + half2_uint32 q3((qa & 0x00f000f0) | c0); // half2(q[ 6], q[ 7]) * 16 + 1024 + + dq[0] = __hadd2(q0.as_half2, z1); + dq[1] = __hfma2(q1.as_half2, y16, z16); + dq[2] = __hadd2(q2.as_half2, z1); + dq[3] = __hfma2(q3.as_half2, y16, z16); +} + +__forceinline__ __device__ void +dequant_4bit_8_prep_zero_scale(const uint32_t zero, const half scale, half2 (&z1z16)[2], half2 (&y1y16)[2]) { + half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero); + half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero)); + + half2 scale2 = __half2half2(scale); + + z1z16[0] = __hmul2(scale2, __half2half2(z1.as_half)); + z1z16[1] = __hmul2(scale2, __half2half2(z16)); + + const half y1 = __float2half_rn(1.0f); + const half y16 = __float2half_rn(1.0f / 16.0f); + + y1y16[0] = __hmul2(scale2, __half2half2(y1)); + y1y16[1] = __hmul2(scale2, __half2half2(y16)); +} + +__forceinline__ __device__ void dequant_4bit_8_prep_zero(const uint32_t zero, half2 (&z1z16)[2], half2 (&y1y16)[2]) { + half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero); + half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero)); + + z1z16[0] = __half2half2(z1.as_half); + z1z16[1] = __half2half2(z16); + + const half y1 = __float2half_rn(1.0f); + const half y16 = __float2half_rn(1.0f / 16.0f); + + y1y16[0] = __half2half2(y1); + y1y16[1] = __half2half2(y16); +} + +__forceinline__ __device__ void +dequant_4bit_8_gptq(const uint32_t q_0, half2 (&dq)[4], half2 (&z1z16)[2], half2 (&y1y16)[2], int stride, bool scaled) { + const uint32_t c0 = 0x64006400; + + uint32_t qa = q_0; + half2_uint32 q0((qa & 0x000f000f) | c0); // half2( q[0] + 1024, q[1] + 1024 ) + half2_uint32 q1((qa & 0x00f000f0) | c0); // half2( q[2] * 16 + 1024, q[3] * 16 + 1024 ) + qa >>= 8; + half2_uint32 q2((qa & 0x000f000f) | c0); // half2( q[4] + 1024, q[5] + 1024 ) + half2_uint32 q3((qa & 0x00f000f0) | c0); // half2( q[6] * 16 + 1024, q[7] * 16 + 1024 ) + + if (scaled) { + dq[0] = __hfma2(q0.as_half2, y1y16[0], + z1z16[0]); // half2( q[0] * s - z * s, q[1] * s - z * s) + dq[1] = __hfma2(q1.as_half2, y1y16[1], + z1z16[1]); // half2( q[2] * s - z * s, q[3] * s - z * s) + dq[2] = __hfma2(q2.as_half2, y1y16[0], z1z16[0]); + dq[3] = __hfma2(q3.as_half2, y1y16[1], z1z16[1]); + } else { + dq[0] = __hadd2(q0.as_half2, z1z16[0]); // half2( q[0] - z, q[1] - z ) + dq[1] = __hfma2(q1.as_half2, y1y16[1], + z1z16[1]); // half2( q[2] - z, q[3] - z ) + dq[2] = __hadd2(q2.as_half2, z1z16[0]); // half2( q[4] - z, q[5] - z ) + dq[3] = __hfma2(q3.as_half2, y1y16[1], + z1z16[1]); // half2( q[6] - z, q[7] - z ) + } +} +} // namespace gptq +} // namespace sglang + +#endif diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/qdq_8.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/qdq_8.cuh new file mode 100644 index 00000000..c6a49d6d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/qdq_8.cuh @@ -0,0 +1,30 @@ +/* +Copied from https://github.com/turboderp/exllamav2 +*/ + +#ifndef _qdq_8_cuh +#define _qdq_8_cuh + +#include "qdq_util.cuh" + +namespace sglang { +namespace gptq { + +__forceinline__ __device__ void shuffle_8bit_4(uint32_t* q, int stride) {} + +__forceinline__ __device__ void +dequant_8bit_8(const uint32_t q_0, const uint32_t q_1, half2 (&dq)[4], int stride, const uint32_t zero) { + half dqh[8]; + for (int i = 0; i < 4; i++) + dqh[i] = dq_ns(exb(q_0, i * 8, 0xff), zero); + for (int i = 0; i < 4; i++) + dqh[i + 4] = dq_ns(exb(q_1, i * 8, 0xff), zero); + + for (int i = 0; i < 4; i++) + dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]); +} + +} // namespace gptq +} // namespace sglang + +#endif diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/qdq_util.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/qdq_util.cuh new file mode 100644 index 00000000..0977269c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/gptq/qdq_util.cuh @@ -0,0 +1,53 @@ +/* +Copied from https://github.com/turboderp/exllamav2 +*/ + +#ifndef _qdq_util_cuh +#define _qdq_util_cuh + +namespace sglang { +namespace gptq { + +union half2_uint32 { + uint32_t as_uint32; + half2 as_half2; + __device__ half2_uint32(uint32_t val) : as_uint32(val) {} + __device__ half2_uint32(half2 val) : as_half2(val) {} +}; + +union half_uint16 { + uint16_t as_uint16; + half as_half; + __device__ half_uint16(uint16_t val) : as_uint16(val) {} + __device__ half_uint16(half val) : as_half(val) {} +}; + +// Max_scale premultiplied by 1/256 + +__forceinline__ __device__ half dq_scale(const int qs, const half max_scale) { + int qs_i = qs + 1; + half qs_h = __int2half_rn(qs_i * qs_i); + qs_h = __hmul(qs_h, max_scale); + return qs_h; +} + +__forceinline__ __device__ half dq(const int q, const int qzero, const half scale) { + return __hmul(__int2half_rn(q - qzero), scale); +} + +__forceinline__ __device__ half dq_ns(const int q, const int qzero) { + // return __hsub(__int2half_rn(q), __int2half_rn(qzero)); + return __int2half_rn(q - qzero); +} + +__forceinline__ __device__ int exb(const uint32_t q, const int shift, const int mask) { + return (int)((q >> shift) & mask); +} + +__forceinline__ __device__ int exb(const uint32_t q1, const uint32_t q0, const int shift, const int mask) { + return (int)(__funnelshift_rc(q0, q1, shift) & mask); +} + +} // namespace gptq +} // namespace sglang +#endif diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/int8_gemm_kernel.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/int8_gemm_kernel.cu new file mode 100644 index 00000000..b47904cb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/int8_gemm_kernel.cu @@ -0,0 +1,747 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "cutlass_extensions/epilogue/epilogue_per_row_per_col_scale.h" +#include "cutlass_extensions/gemm/gemm_universal_base_compat.h" +#include "cutlass_extensions/gemm/gemm_with_epilogue_visitor.h" +#include "utils.h" + +using namespace cute; + +template < + typename ElementOutput, + typename ArchTag, + typename ThreadblockShape, + typename WarpShape, + typename InstructionShape, + int NumStages> +void cutlass_int8_scaled_mm( + torch::Tensor& out, + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + using ElementAccumulator = int32_t; + using ElementCompute = float; + using ElementInputA = int8_t; + using ElementInputB = int8_t; + + using OperatorClass = cutlass::arch::OpClassTensorOp; + using ThreadblockSwizzle = cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>; + + using DefaultGemmConf = cutlass::gemm::device:: + DefaultGemmConfiguration; + using EpilogueOutputOp = typename DefaultGemmConf::EpilogueOutputOp; + + using GemmKernel_ = typename cutlass::gemm::kernel::DefaultGemm< + ElementInputA, + cutlass::layout::RowMajor, + DefaultGemmConf::kAlignmentA, + ElementInputB, + cutlass::layout::ColumnMajor, + DefaultGemmConf::kAlignmentB, + ElementOutput, + cutlass::layout::RowMajor, + ElementAccumulator, + OperatorClass, + ArchTag, + ThreadblockShape, + WarpShape, + InstructionShape, + EpilogueOutputOp, + ThreadblockSwizzle, + NumStages, + true, + typename DefaultGemmConf::Operator>::GemmKernel; + + using AlphaColTileIterator = cutlass::epilogue::threadblock::PredicatedTileIterator< + cutlass::epilogue::threadblock::OutputTileOptimalThreadMap< + typename GemmKernel_::Epilogue::OutputTileIterator::ThreadMap::Shape, + typename GemmKernel_::Epilogue::OutputTileIterator::ThreadMap::Count, + GemmKernel_::Epilogue::OutputTileIterator::ThreadMap::kThreads, + GemmKernel_::Epilogue::OutputTileIterator::kElementsPerAccess, + cutlass::sizeof_bits::value>, + ElementCompute>; + + using EpilogueVisitor = typename cutlass::epilogue::threadblock::EpilogueVisitorPerRowPerCol< + ThreadblockShape, + GemmKernel_::kThreadCount, + AlphaColTileIterator, + typename GemmKernel_::Epilogue::OutputTileIterator, + ElementAccumulator, + ElementCompute, + EpilogueOutputOp>; + + using Epilogue = typename cutlass::epilogue::threadblock:: + EpilogueWithVisitorFromExistingEpilogue::Epilogue; + + using GemmKernel = + cutlass::gemm::kernel::GemmWithEpilogueVisitor; + + using Gemm = cutlass::gemm::device::GemmUniversalBaseCompat; + + Gemm gemm_op; + + int m = mat_a.size(0); + int k = mat_a.size(1); + int n = mat_b.size(1); + + auto a_ptr = static_cast(mat_a.data_ptr()); + auto b_ptr = static_cast(mat_b.data_ptr()); + auto o_ptr = static_cast(out.data_ptr()); + + auto a_s_ptr = static_cast(scales_a.data_ptr()); + auto b_s_ptr = static_cast(scales_b.data_ptr()); + + int64_t lda = mat_a.stride(0); + int64_t ldb = mat_b.stride(1); + int64_t ldd = out.stride(0); + + ElementOutput* bias_ptr = nullptr; + int64_t ldc = 0; + if (bias) { + bias_ptr = static_cast(bias->data_ptr()); + } + + typename EpilogueOutputOp::Params linearScalingParams; + typename EpilogueVisitor::Arguments visitor_args{linearScalingParams}; + + typename Gemm::Arguments args{ + {m, n, k}, {a_ptr, lda}, {b_ptr, ldb}, {b_s_ptr, 0}, {a_s_ptr, 0}, {bias_ptr, ldc}, {o_ptr, ldd}, visitor_args}; + + auto workspace = torch::empty( + gemm_op.get_workspace_size(args), torch::TensorOptions().dtype(torch::kUInt8).device(mat_a.device())); + + auto stream = at::cuda::getCurrentCUDAStream(mat_a.get_device()); + + auto can_implement = gemm_op.can_implement(args); + TORCH_CHECK( + can_implement == cutlass::Status::kSuccess, + "gemm cannot implement, error: ", + cutlassGetStatusString(can_implement)); + + auto status = gemm_op(args, workspace.data_ptr(), stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, "gemm executioin failed, error: ", cutlassGetStatusString(status)); +} + +template +void sm75_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + int m = mat_a.size(0); + if (m <= 32) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<32, 128, 64>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 2>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else if (m <= 64) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 128, 128>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 2>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else if (m <= 256) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<128, 128, 128>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 2>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 2>(out, mat_a, mat_b, scales_a, scales_b, bias); + } +} + +template +void sm80_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + int m = mat_a.size(0); + int n = mat_b.size(1); + if (m <= 16) { + if (n <= 4096) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<16, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + InstructionShape, + 6>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<16, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 32) { + if (n <= 4096) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<32, 64, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 6>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<32, 64, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 64) { + if (n <= 4096) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 64, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 128, 128>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 128 && n < 8192) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 128, 128>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } +} + +// Dispatch shape for sm89 (L40S, L20, RTX 4090), according to: +// https://github.com/vllm-project/vllm/blob/main/csrc/quantization/cutlass_w8a8/scaled_mm_c2x_sm89_int8_dispatch.cuh +template +void sm89_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + int m = mat_a.size(0); + int n = mat_b.size(1); + if (m <= 16) { + if (n <= 8192) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<16, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<16, 128, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + InstructionShape, + 4>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 32) { + if (n <= 8192) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<32, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<32, 128, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 4>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 64) { + if (n <= 8192) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 64, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 128, 128>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 3>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 128) { + if (n <= 8192) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 128, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 3>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else if (n <= 16384) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 64, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 256) { + if (n <= 4096) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 128, 128>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 3>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else if (n <= 8192) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else if (n <= 16384) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<256, 128, 64>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 3>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } +} + +template < + typename ElementOutput, + typename TileShape, + typename ClusterShape, + typename MainloopScheduleType, + bool WithBias> +void cutlass_int8_scaled_mm_sm90( + torch::Tensor& out, + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + using ArchTag = cutlass::arch::Sm90; + + using ElementAccumulator = int32_t; + using ElementCompute = float; + using ElementInputA = int8_t; + using ElementInputB = int8_t; + + static constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentOutput = 128 / cutlass::sizeof_bits::value; + + using OperatorClass = cutlass::arch::OpClassTensorOp; + + using EpilogueScheduleType = cutlass::epilogue::TmaWarpSpecialized; + using TileSchedulerType = cutlass::gemm::PersistentScheduler; + + using XScale = cutlass::epilogue::fusion:: + Sm90ColBroadcast<0, TileShape, ElementCompute, ElementCompute, Stride, Int<0>, Int<0>>>; + + using WScale = cutlass::epilogue::fusion:: + Sm90RowBroadcast<0, TileShape, ElementCompute, ElementCompute, Stride, Int<1>, Int<0>>>; + + using Bias = cutlass::epilogue::fusion:: + Sm90RowBroadcast<0, TileShape, ElementOutput, ElementOutput, Stride, Int<1>, Int<0>>>; + + using Accum = cutlass::epilogue::fusion::Sm90AccFetch; + + // Scale + using Compute0 = cutlass::epilogue::fusion:: + Sm90Compute; + + using EVTCompute0 = cutlass::epilogue::fusion::Sm90EVT; + + using Compute1 = cutlass::epilogue::fusion:: + Sm90Compute; + + using EVTCompute1 = cutlass::epilogue::fusion::Sm90EVT; + + // With bias + using ComputeWithBias = cutlass::epilogue::fusion:: + Sm90Compute; + using EVTComputeWithBias = cutlass::epilogue::fusion::Sm90EVT; + + using EpilogueEVT = typename cutlass::platform::conditional::type; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + TileShape, + ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementCompute, + ElementOutput, + cutlass::layout::RowMajor, + AlignmentC, + ElementOutput, + cutlass::layout::RowMajor, + AlignmentOutput, + EpilogueScheduleType, + EpilogueEVT>::CollectiveOp; + + using Stages = cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementInputA, + cutlass::layout::RowMajor, + AlignmentA, + ElementInputB, + cutlass::layout::ColumnMajor, + AlignmentB, + ElementAccumulator, + TileShape, + ClusterShape, + Stages, + MainloopScheduleType>::CollectiveOp; + + using GemmKernel = cutlass::gemm::kernel::GemmUniversal< + Shape, // Indicates ProblemShape + CollectiveMainloop, + CollectiveEpilogue, + TileSchedulerType>; + + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + + Gemm gemm_op; + + int m = mat_a.size(0); + int k = mat_a.size(1); + int n = mat_b.size(1); + + auto a_ptr = static_cast(mat_a.data_ptr()); + auto b_ptr = static_cast(mat_b.data_ptr()); + auto o_ptr = static_cast(out.data_ptr()); + + auto a_s_ptr = static_cast(scales_a.data_ptr()); + auto b_s_ptr = static_cast(scales_b.data_ptr()); + + using StrideA = typename Gemm::GemmKernel::StrideA; + using StrideB = typename Gemm::GemmKernel::StrideB; + using StrideC = typename Gemm::GemmKernel::StrideC; + using StrideD = typename Gemm::GemmKernel::StrideD; + + StrideA stride_a = cutlass::make_cute_packed_stride(StrideA{}, make_shape(m, k, 1)); + StrideB stride_b = cutlass::make_cute_packed_stride(StrideB{}, make_shape(n, k, 1)); + StrideC stride_c; + StrideD stride_d = cutlass::make_cute_packed_stride(StrideD{}, make_shape(m, n, 1)); + + typename Gemm::Arguments args = { + cutlass::gemm::GemmUniversalMode::kGemm, + {m, n, k, 1}, + {a_ptr, stride_a, b_ptr, stride_b}, + {{}, // epilogue.thread + nullptr, + stride_c, + o_ptr, + stride_d}}; + + if constexpr (WithBias) { + ElementOutput* bias_ptr = static_cast(bias->data_ptr()); + args.epilogue.thread = { + {a_s_ptr}, + {{b_s_ptr}, {}, {}}, + {bias_ptr}, + {}, + }; + } else { + args.epilogue.thread = { + {a_s_ptr}, + {{b_s_ptr}, {}, {}}, + {}, + }; + } + + auto workspace = torch::empty( + gemm_op.get_workspace_size(args), torch::TensorOptions().dtype(torch::kUInt8).device(mat_a.device())); + + auto stream = at::cuda::getCurrentCUDAStream(mat_a.get_device()); + + auto can_implement = gemm_op.can_implement(args); + TORCH_CHECK( + can_implement == cutlass::Status::kSuccess, + "gemm cannot implement, error: ", + cutlassGetStatusString(can_implement)); + + auto status = gemm_op(args, workspace.data_ptr(), stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, "gemm executioin failed, error: ", cutlassGetStatusString(status)); +} + +template +void sm90_dispatch_bias( + torch::Tensor& out, + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + if (bias) { + cutlass_int8_scaled_mm_sm90( + out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm_sm90( + out, mat_a, mat_b, scales_a, scales_b, bias); + } +} + +template +void sm90_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + int m = mat_a.size(0); + int n = mat_b.size(1); + if (m <= 32) { + if (n < 8192) { + return sm90_dispatch_bias< + ElementOutput, + Shape<_64, _64, _128>, + Shape<_1, _8, _1>, + cutlass::gemm::KernelTmaWarpSpecialized>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + return sm90_dispatch_bias< + ElementOutput, + Shape<_64, _128, _128>, + Shape<_1, _8, _1>, + cutlass::gemm::KernelTmaWarpSpecialized>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 64) { + if (n < 8192) { + return sm90_dispatch_bias< + ElementOutput, + Shape<_64, _64, _128>, + Shape<_1, _4, _1>, + cutlass::gemm::KernelTmaWarpSpecialized>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + return sm90_dispatch_bias< + ElementOutput, + Shape<_64, _64, _256>, + Shape<_1, _1, _1>, + cutlass::gemm::KernelTmaWarpSpecialized>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 128) { + if (n <= 4096) { + return sm90_dispatch_bias< + ElementOutput, + Shape<_64, _64, _128>, + Shape<_2, _1, _1>, + cutlass::gemm::KernelTmaWarpSpecialized>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + return sm90_dispatch_bias< + ElementOutput, + Shape<_64, _128, _128>, + Shape<_2, _1, _1>, + cutlass::gemm::KernelTmaWarpSpecialized>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else { + return sm90_dispatch_bias< + ElementOutput, + Shape<_128, _128, _128>, + Shape<_2, _1, _1>, + cutlass::gemm::KernelTmaWarpSpecializedPingpong>(out, mat_a, mat_b, scales_a, scales_b, bias); + } +} + +torch::Tensor int8_scaled_mm( + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Dtype& out_dtype, + const c10::optional& bias) { + TORCH_CHECK(mat_a.is_cuda(), "mat_a must be a CUDA tensor"); + TORCH_CHECK(mat_b.is_cuda(), "mat_b must be a CUDA tensor"); + TORCH_CHECK(mat_a.dim() == 2, "mat_a must be a 2D tensor"); + TORCH_CHECK(mat_b.dim() == 2, "mat_b must be a 2D tensor"); + TORCH_CHECK(mat_a.stride(1) == 1, "mat_a must be a row major tensor"); + TORCH_CHECK(mat_b.stride(0) == 1, "mat_b must be a column major tensor"); + TORCH_CHECK(mat_a.size(1) == mat_b.size(0), "mat_a and mat_b shapes cannot be multiplied"); + TORCH_CHECK(mat_a.size(1) % 16 == 0, "mat_a.size(1) must be multiple of 16 for memory alignment"); + TORCH_CHECK(mat_b.size(0) % 16 == 0, "mat_b.size(0) must be multiple of 16 for memory alignment"); + TORCH_CHECK(mat_b.size(1) % 8 == 0, "mat_b.size(1) must be multiple of 8 for memory alignment"); // out.stride(0) + TORCH_CHECK(mat_a.scalar_type() == torch::kInt8, "mat_a must be Int8"); + TORCH_CHECK(mat_b.scalar_type() == torch::kInt8, "mat_b must be Int8"); + TORCH_CHECK(out_dtype == torch::kHalf || out_dtype == torch::kBFloat16, "out_dtype must be Half or BFloat16"); + + TORCH_CHECK(scales_a.numel() == mat_a.size(0), "size of scales_a is not matched"); + TORCH_CHECK(scales_b.numel() == mat_b.size(1), "size of scales_b is not matched"); + TORCH_CHECK(scales_a.is_contiguous(), "scales_a must be contiguous"); + TORCH_CHECK(scales_b.is_contiguous(), "scales_b msut be contiguous"); + TORCH_CHECK(scales_a.scalar_type() == torch::kFloat32, "scales_a must be Float32"); + TORCH_CHECK(scales_b.scalar_type() == torch::kFloat32, "scales_b must be Float32"); + + if (bias) { + TORCH_CHECK(bias->numel() == mat_b.size(1), "size of bias is not matched"); + TORCH_CHECK(bias->is_contiguous(), "bias must be contiguous"); + TORCH_CHECK(bias->dtype() == out_dtype, "bias dtype must match output dtype"); + } + + torch::Tensor out = torch::empty({mat_a.size(0), mat_b.size(1)}, mat_a.options().dtype(out_dtype)); + + auto sm_version = getSMVersion(); + + if (sm_version >= 75 && sm_version < 80) { + TORCH_CHECK(out_dtype == torch::kHalf, "out_dtype must be Half for SM75"); + sm75_dispatch_shape>( + out, mat_a, mat_b, scales_a, scales_b, bias); + } else if (sm_version >= 80 && sm_version < 90) { + // sm86/sm89 has a much smaller shared memory size (100K) than sm80 (160K) + if (sm_version == 86 || sm_version == 89) { + if (out_dtype == torch::kBFloat16) { + sm89_dispatch_shape>( + out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + sm89_dispatch_shape>( + out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else { + if (out_dtype == torch::kBFloat16) { + sm80_dispatch_shape>( + out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + sm80_dispatch_shape>( + out, mat_a, mat_b, scales_a, scales_b, bias); + } + } + } else if (sm_version == 90) { +#if defined CUDA_VERSION && CUDA_VERSION >= 12000 + // cutlass 3.x + if (out_dtype == torch::kBFloat16) { + sm90_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + sm90_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } +#else + // fallback to cutlass 2.x + if (out_dtype == torch::kBFloat16) { + sm80_dispatch_shape>( + out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + sm80_dispatch_shape>( + out, mat_a, mat_b, scales_a, scales_b, bias); + } +#endif + } else { + TORCH_CHECK_NOT_IMPLEMENTED(false, "No implemented int8_scaled_mm for current compute capability."); + } + + return out; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/awq_marlin_repack.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/awq_marlin_repack.cu new file mode 100644 index 00000000..1c8de396 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/awq_marlin_repack.cu @@ -0,0 +1,253 @@ +#include "marlin.cuh" + +namespace marlin { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800 +template +__global__ void awq_marlin_repack_kernel( + uint32_t const* __restrict__ b_q_weight_ptr, uint32_t* __restrict__ out_ptr, int size_k, int size_n) { + return; +} +#else + +template +__global__ void awq_marlin_repack_kernel( + uint32_t const* __restrict__ b_q_weight_ptr, uint32_t* __restrict__ out_ptr, int size_k, int size_n) { + constexpr int pack_factor = 32 / num_bits; + + int k_tiles = size_k / tile_k_size; + int n_tiles = size_n / tile_n_size; + int block_k_tiles = div_ceil(k_tiles, gridDim.x); + + auto start_k_tile = blockIdx.x * block_k_tiles; + if (start_k_tile >= k_tiles) { + return; + } + + int finish_k_tile = min(start_k_tile + block_k_tiles, k_tiles); + + // Wait until the next thread tile has been loaded to shared memory. + auto wait_for_stage = [&]() { + // We only have `stages - 2` active fetches since we are double buffering + // and can only issue the next fetch when it is guaranteed that the previous + // shared memory load is fully complete (as it may otherwise be + // overwritten). + cp_async_wait(); + __syncthreads(); + }; + + extern __shared__ int4 sh[]; + + constexpr int tile_n_ints = tile_n_size / pack_factor; + + constexpr int stage_n_threads = tile_n_ints / 4; + constexpr int stage_k_threads = tile_k_size; + constexpr int stage_size = stage_k_threads * stage_n_threads; + + auto fetch_to_shared = [&](int pipe, int k_tile_id, int n_tile_id) { + if (n_tile_id >= n_tiles) { + cp_async_fence(); + return; + } + + int first_n = n_tile_id * tile_n_size; + int first_n_packed = first_n / pack_factor; + + int4* sh_ptr = sh + stage_size * pipe; + + if (threadIdx.x < stage_size) { + auto k_id = threadIdx.x / stage_n_threads; + auto n_id = threadIdx.x % stage_n_threads; + + int first_k = k_tile_id * tile_k_size; + + cp_async4( + &sh_ptr[k_id * stage_n_threads + n_id], + reinterpret_cast( + &(b_q_weight_ptr[(first_k + k_id) * (size_n / pack_factor) + first_n_packed + (n_id * 4)]))); + } + + cp_async_fence(); + }; + + auto repack_tile = [&](int pipe, int k_tile_id, int n_tile_id) { + if (n_tile_id >= n_tiles) { + return; + } + + auto warp_id = threadIdx.x / 32; + auto th_id = threadIdx.x % 32; + + if (warp_id >= 4) { + return; + } + + int tc_col = th_id / 4; + int tc_row = (th_id % 4) * 2; + + constexpr int tc_offsets[4] = {0, 1, 8, 9}; + + int cur_n = warp_id * 16 + tc_col; + int cur_n_packed = cur_n / pack_factor; + int cur_n_pos = cur_n % pack_factor; + + constexpr int sh_stride = tile_n_ints; + constexpr uint32_t mask = (1 << num_bits) - 1; + + int4* sh_stage_ptr = sh + stage_size * pipe; + uint32_t* sh_stage_int_ptr = reinterpret_cast(sh_stage_ptr); + + // Undo interleaving + int cur_n_pos_unpacked; + if constexpr (num_bits == 4) { + constexpr int undo_pack[8] = {0, 4, 1, 5, 2, 6, 3, 7}; + cur_n_pos_unpacked = undo_pack[cur_n_pos]; + } else { + constexpr int undo_pack[4] = {0, 2, 1, 3}; + cur_n_pos_unpacked = undo_pack[cur_n_pos]; + } + + uint32_t vals[8]; +#pragma unroll + for (int i = 0; i < 4; i++) { + int cur_elem = tc_row + tc_offsets[i]; + + int packed_src_0 = sh_stage_int_ptr[cur_n_packed + sh_stride * cur_elem]; + int packed_src_1 = sh_stage_int_ptr[cur_n_packed + (8 / pack_factor) + sh_stride * cur_elem]; + + vals[i] = (packed_src_0 >> (cur_n_pos_unpacked * num_bits)) & mask; + vals[4 + i] = (packed_src_1 >> (cur_n_pos_unpacked * num_bits)) & mask; + } + + constexpr int tile_size = tile_k_size * tile_n_size / pack_factor; + int out_offset = (k_tile_id * n_tiles + n_tile_id) * tile_size; + + // Result of: + // https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h + if constexpr (num_bits == 4) { + constexpr int pack_idx[8] = {0, 2, 4, 6, 1, 3, 5, 7}; + + uint32_t res = 0; +#pragma unroll + for (int i = 0; i < 8; i++) { + res |= vals[pack_idx[i]] << (i * 4); + } + + out_ptr[out_offset + th_id * 4 + warp_id] = res; + + } else { + constexpr int pack_idx[4] = {0, 2, 1, 3}; + + uint32_t res1 = 0; + uint32_t res2 = 0; +#pragma unroll + for (int i = 0; i < 4; i++) { + res1 |= vals[pack_idx[i]] << (i * 8); + res2 |= vals[4 + pack_idx[i]] << (i * 8); + } + + out_ptr[out_offset + th_id * 8 + (warp_id * 2) + 0] = res1; + out_ptr[out_offset + th_id * 8 + (warp_id * 2) + 1] = res2; + } + }; + + auto start_pipes = [&](int k_tile_id, int n_tile_id) { +#pragma unroll + for (int pipe = 0; pipe < repack_stages - 1; pipe++) { + fetch_to_shared(pipe, k_tile_id, n_tile_id + pipe); + } + + wait_for_stage(); + }; +#pragma unroll + for (int k_tile_id = start_k_tile; k_tile_id < finish_k_tile; k_tile_id++) { + int n_tile_id = 0; + + start_pipes(k_tile_id, n_tile_id); + + while (n_tile_id < n_tiles) { +#pragma unroll + for (int pipe = 0; pipe < repack_stages; pipe++) { + fetch_to_shared((pipe + repack_stages - 1) % repack_stages, k_tile_id, n_tile_id + pipe + repack_stages - 1); + repack_tile(pipe, k_tile_id, n_tile_id + pipe); + wait_for_stage(); + } + n_tile_id += repack_stages; + } + } +} +#endif +} // namespace marlin + +#define CALL_IF(NUM_BITS) \ + else if (num_bits == NUM_BITS) { \ + cudaFuncSetAttribute( \ + marlin::awq_marlin_repack_kernel, \ + cudaFuncAttributeMaxDynamicSharedMemorySize, \ + max_shared_mem); \ + marlin::awq_marlin_repack_kernel \ + <<>>(b_q_weight_ptr, out_ptr, size_k, size_n); \ + } + +torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k, int64_t size_n, int64_t num_bits) { + // Verify compatibility with marlin tile of 16x64 + TORCH_CHECK( + size_k % marlin::tile_k_size == 0, + "size_k = ", + size_k, + " is not divisible by tile_k_size = ", + marlin::tile_k_size); + TORCH_CHECK( + size_n % marlin::tile_n_size == 0, + "size_n = ", + size_n, + " is not divisible by tile_n_size = ", + marlin::tile_n_size); + + TORCH_CHECK(num_bits == 4 || num_bits == 8, "num_bits must be 4 or 8. Got = ", num_bits); + int const pack_factor = 32 / num_bits; + + // Verify B + TORCH_CHECK(b_q_weight.size(0) == size_k, "b_q_weight.size(0) = ", b_q_weight.size(0), " is not size_k = ", size_k); + TORCH_CHECK( + (size_n / pack_factor) == b_q_weight.size(1), + "Shape mismatch: b_q_weight.size(1) = ", + b_q_weight.size(1), + ", size_n = ", + size_n, + ", pack_factor = ", + pack_factor); + + // Verify device and strides + TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU"); + TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous"); + TORCH_CHECK(b_q_weight.dtype() == at::kInt, "b_q_weight type is not kInt"); + + // Alloc buffers + const at::cuda::OptionalCUDAGuard device_guard(device_of(b_q_weight)); + auto options = torch::TensorOptions().dtype(b_q_weight.dtype()).device(b_q_weight.device()); + torch::Tensor out = torch::empty({size_k / marlin::tile_size, size_n * marlin::tile_size / pack_factor}, options); + + // Get ptrs + uint32_t const* b_q_weight_ptr = reinterpret_cast(b_q_weight.data_ptr()); + uint32_t* out_ptr = reinterpret_cast(out.data_ptr()); + + // Get dev info + int dev = b_q_weight.get_device(); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(dev); + int blocks; + cudaDeviceGetAttribute(&blocks, cudaDevAttrMultiProcessorCount, dev); + + int max_shared_mem = 0; + cudaDeviceGetAttribute(&max_shared_mem, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev); + TORCH_CHECK(max_shared_mem > 0); + + if (false) { + } + CALL_IF(4) + CALL_IF(8) + else { + TORCH_CHECK(false, "Unsupported repack config: num_bits = ", num_bits); + } + + return out; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/dequant.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/dequant.h new file mode 100644 index 00000000..37893f42 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/dequant.h @@ -0,0 +1,504 @@ +/* +Fast Dequantization (Converting INT4/INT8/FP4/FP8 to FP16/BF16) + +The process of fast dequantization can be summarized as a combination +of bitwise operations and floating-point computations: + +weight =>(bit_op / bitwise operations)=> +f16_value =>(flop / floating-point computation)=> +dequantized_weight + +Since the dequantized weights typically require subtracting the zero point and +applying a scale factor, the floating-point computation step can be fused with +the zero-point subtraction and scaling operations. + +The following are the parts that need to be modified for the fused operation +of zero-point subtraction and scaling. + +## INT4 => FP16/BF16 or INT8 => FP16 + +The floating-point computation is `__hsub2` + +If has zero points: + + flop(bit_op(weight)) - flop(bit_op(zp)) + = sub(bit_op(weight), bias) - sub(bit_op(zp), bias) + = bit_op(weight) - bit_op(zp) + +so we don't need additional modification. + +If has float zero points: + + flop(bit_op(weight)) - fzp + = sub(bit_op(weight), bias) - fzp + = bit_op(weight) - (fzp + bias) + +where the `fzp + bias` can be computed at weight loading. But this +may have accuracy issue, so we should not use this in most cases. + +If has not zero points: + + scale(flop(bit_op(weight))) + = scale(sub(bit_op(weight), bias)) + = scale(bit_op(weight)) - scale(bias) + = fma(bit_op(weight), scale_factor, scale(bias)) + +where the `scale(bias)` can be cached. But this may have accuracy issue, +so we should not use this in most cases. + + +## INT8 => BF16 + +INT8 => BF16 is a special case, it use byte_perm instead of flop. +We cannot fused byte_perm with scaling. + + +## FP4/FP8 => FP16/BF16 + + scale(flop(bit_op(weight))) + = scale(mul(bit_op(weight), multiplier)) + = mul(bit_op(weight), scale_factor * multiplier) + +where `scale_factor * multiplier` can be computed at weight loading. + +*/ + +#include "marlin_dtypes.cuh" + +namespace MARLIN_NAMESPACE_NAME { + +#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800 +// Lookup-table based 3-input logical operation; explicitly used for +// dequantization as the compiler does not seem to automatically recognize it in +// all cases. +template +__device__ inline int lop3(int a, int b, int c) { + int res; + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n" : "=r"(res) : "r"(a), "r"(b), "r"(c), "n"(lut)); + return res; +} + +// Constructs destination register by taking bytes from 2 sources (based on +// mask) +template +__device__ inline uint32_t prmt(uint32_t a) { + uint32_t res; + asm volatile("prmt.b32 %0, %1, %2, %3;\n" : "=r"(res) : "r"(a), "n"(start_byte), "n"(mask)); + return res; +} + +template +__device__ inline void dequant(int q, scalar_t2* frag_b); + +// +// Efficiently dequantize 4bit values packed in an int32 value into a full +// B-fragment of 4 fp16 values. We mostly follow the strategy in the link below, +// with some small changes: +// - FP16: +// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L215-L287 +// - BF16: +// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L327-L385 +// +template <> +__device__ inline void dequant(int q, half2* frag_b) { + const int MASK = 0x000f000f; + const int EX = 0x64006400; + // Guarantee that the `(a & b) | c` operations are LOP3s. + int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX); + q >>= 4; + int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX); + + frag_b[0] = *reinterpret_cast(&lo); + frag_b[1] = *reinterpret_cast(&hi); +} + +template <> +__device__ inline void dequant(int q, half2* frag_b) { + const int LO = 0x000f000f; + const int HI = 0x00f000f0; + const int EX = 0x64006400; + // Guarantee that the `(a & b) | c` operations are LOP3s. + // clang-format off + int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX); + int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX); + // clang-format on + // We want signed int4 outputs, hence we fuse the `-8` symmetric zero point + // directly into `SUB` and `ADD`. + const int SUB = 0x64086408; + const int MUL = 0x2c002c00; + const int ADD = 0xd480d480; + frag_b[0] = __hsub2(*reinterpret_cast(&lo), *reinterpret_cast(&SUB)); + frag_b[1] = __hfma2( + *reinterpret_cast(&hi), *reinterpret_cast(&MUL), *reinterpret_cast(&ADD)); +} + +template <> +__device__ inline void dequant(int q, half2* frag_b) { + dequant(q, frag_b); +} + +template <> +__device__ inline void dequant(int q, half2* frag_b) { + const int LO = 0x000f000f; + const int HI = 0x00f000f0; + const int EX = 0x64006400; + // Guarantee that the `(a & b) | c` operations are LOP3s. + // clang-format off + int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX); + int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX); + // clang-format on + // We want signed int4 outputs, hence we fuse the `-8` symmetric zero point + // directly into `SUB` and `ADD`. + const int SUB = 0x64006400; + const int MUL = 0x2c002c00; + const int ADD = 0xd400d400; + frag_b[0] = __hsub2(*reinterpret_cast(&lo), *reinterpret_cast(&SUB)); + frag_b[1] = __hfma2( + *reinterpret_cast(&hi), *reinterpret_cast(&MUL), *reinterpret_cast(&ADD)); +} + +template <> +__device__ inline void dequant(int q, nv_bfloat162* frag_b) { + static constexpr uint32_t MASK = 0x000f000f; + static constexpr uint32_t EX = 0x43004300; + + // Guarantee that the `(a & b) | c` operations are LOP3s. + // clang-format off + int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX); + q >>= 4; + int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX); + // clang-format on + + frag_b[0] = *reinterpret_cast(&lo); + frag_b[1] = *reinterpret_cast(&hi); +} + +template <> +__device__ inline void dequant(int q, nv_bfloat162* frag_b) { + dequant(q, frag_b); + + static constexpr uint32_t SUB = 0x43084308; + + frag_b[0] = __hsub2(frag_b[0], *reinterpret_cast(&SUB)); + frag_b[1] = __hsub2(frag_b[1], *reinterpret_cast(&SUB)); +} + +template <> +__device__ inline void dequant(int q, nv_bfloat162* frag_b) { + dequant(q, frag_b); +} + +template <> +__device__ inline void dequant(int q, nv_bfloat162* frag_b) { + dequant(q, frag_b); + + static constexpr uint32_t SUB = 0x43004300; + + frag_b[0] = __hsub2(frag_b[0], *reinterpret_cast(&SUB)); + frag_b[1] = __hsub2(frag_b[1], *reinterpret_cast(&SUB)); +} + +// +// Fast Int8ToFp16/Int8ToBf16: Efficiently dequantize 8bit int values to fp16 or +// bf16 Reference: +// - FP16: +// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L53-L85 +// - BF16: +// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L125-L175 +// +template <> +__device__ inline void dequant(int q, half2* frag_b) { + static constexpr uint32_t mask_for_elt_01 = 0x5250; + static constexpr uint32_t mask_for_elt_23 = 0x5351; + static constexpr uint32_t start_byte_for_fp16 = 0x64646464; + + uint32_t lo = prmt(q); + uint32_t hi = prmt(q); + + frag_b[0] = *reinterpret_cast(&lo); + frag_b[1] = *reinterpret_cast(&hi); +} + +template <> +__device__ inline void dequant(int q, half2* frag_b) { + dequant(q, frag_b); + + static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64806480; + frag_b[0] = __hsub2(frag_b[0], *reinterpret_cast(&I8s_TO_F16s_MAGIC_NUM)); + frag_b[1] = __hsub2(frag_b[1], *reinterpret_cast(&I8s_TO_F16s_MAGIC_NUM)); +} + +template <> +__device__ inline void dequant(int q, half2* frag_b) { + dequant(q, frag_b); +} + +template <> +__device__ inline void dequant(int q, half2* frag_b) { + dequant(q, frag_b); + + static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64006400; + frag_b[0] = __hsub2(frag_b[0], *reinterpret_cast(&I8s_TO_F16s_MAGIC_NUM)); + frag_b[1] = __hsub2(frag_b[1], *reinterpret_cast(&I8s_TO_F16s_MAGIC_NUM)); +} + +template <> +__device__ inline void dequant(int q, nv_bfloat162* frag_b) { + float fp32_intermediates[4]; + uint32_t* fp32_intermediates_casted = reinterpret_cast(fp32_intermediates); + + static constexpr uint32_t fp32_base = 0x4B000000; + fp32_intermediates_casted[0] = __byte_perm(q, fp32_base, 0x7650); + fp32_intermediates_casted[1] = __byte_perm(q, fp32_base, 0x7652); + fp32_intermediates_casted[2] = __byte_perm(q, fp32_base, 0x7651); + fp32_intermediates_casted[3] = __byte_perm(q, fp32_base, 0x7653); + + fp32_intermediates[0] -= 8388736.f; + fp32_intermediates[1] -= 8388736.f; + fp32_intermediates[2] -= 8388736.f; + fp32_intermediates[3] -= 8388736.f; + + uint32_t* bf16_result_ptr = reinterpret_cast(frag_b); + bf16_result_ptr[0] = __byte_perm(fp32_intermediates_casted[0], fp32_intermediates_casted[1], 0x7632); + bf16_result_ptr[1] = __byte_perm(fp32_intermediates_casted[2], fp32_intermediates_casted[3], 0x7632); +} + +template <> +__device__ inline void dequant(int q, nv_bfloat162* frag_b) { + float fp32_intermediates[4]; + uint32_t* fp32_intermediates_casted = reinterpret_cast(fp32_intermediates); + + static constexpr uint32_t fp32_base = 0x4B000000; + fp32_intermediates_casted[0] = __byte_perm(q, fp32_base, 0x7650); + fp32_intermediates_casted[1] = __byte_perm(q, fp32_base, 0x7652); + fp32_intermediates_casted[2] = __byte_perm(q, fp32_base, 0x7651); + fp32_intermediates_casted[3] = __byte_perm(q, fp32_base, 0x7653); + + fp32_intermediates[0] -= 8388608.f; + fp32_intermediates[1] -= 8388608.f; + fp32_intermediates[2] -= 8388608.f; + fp32_intermediates[3] -= 8388608.f; + + uint32_t* bf16_result_ptr = reinterpret_cast(frag_b); + bf16_result_ptr[0] = __byte_perm(fp32_intermediates_casted[0], fp32_intermediates_casted[1], 0x7632); + bf16_result_ptr[1] = __byte_perm(fp32_intermediates_casted[2], fp32_intermediates_casted[3], 0x7632); +} + +template <> +__device__ inline void dequant(int q, half2* frag_b) { + // Constants for FP8 (E4M3) and FP16 formats + constexpr int FP8_EXPONENT = 4, FP16_EXPONENT = 5; + constexpr int RIGHT_SHIFT = FP16_EXPONENT - FP8_EXPONENT; + constexpr int MASK = 0x7F007F00; + + // Extract and shift FP8 values to FP16 format + int Out1 = (q & 0x80008000) | ((q & MASK) >> RIGHT_SHIFT); + q <<= 8; + int Out2 = (q & 0x80008000) | ((q & MASK) >> RIGHT_SHIFT); + + // Note: reverse indexing is intentional because weights are permuted + frag_b[1] = *reinterpret_cast(&Out1); + frag_b[0] = *reinterpret_cast(&Out2); +} + +template <> +__device__ inline void dequant(int q, half2* frag_b) { + dequant(q, frag_b); + + // Constants for FP8 (E4M3) and FP16 formats + constexpr int FP8_EXPONENT = 4, FP16_EXPONENT = 5; + + // Construct and apply exponent bias + constexpr int BIAS_OFFSET = (1 << (FP16_EXPONENT - 1)) - (1 << (FP8_EXPONENT - 1)); + const half2 bias_reg = __float2half2_rn(float(1 << BIAS_OFFSET)); + + // Convert to half2 and apply bias + frag_b[1] = __hmul2(frag_b[1], bias_reg); + frag_b[0] = __hmul2(frag_b[0], bias_reg); +} + +template <> +__device__ inline void dequant(int q, nv_bfloat162* frag_b) { + // Constants for FP8 (E4M3) and BF16 formats + constexpr int FP8_EXPONENT = 4, BF16_EXPONENT = 8; + constexpr int RIGHT_SHIFT = BF16_EXPONENT - FP8_EXPONENT; + + constexpr int MASK = 0x7F007F00; + + // Extract and shift FP8 values to BF16 format + int Out1 = (q & 0x80008000) | ((q & MASK) >> RIGHT_SHIFT); + q <<= 8; + int Out2 = (q & 0x80008000) | ((q & MASK) >> RIGHT_SHIFT); + + // Note: reverse indexing is intentional because weights are permuted + frag_b[1] = *reinterpret_cast(&Out1); + frag_b[0] = *reinterpret_cast(&Out2); +} + +template <> +__device__ inline void dequant(int q, nv_bfloat162* frag_b) { + dequant(q, frag_b); + + // Constants for FP8 (E4M3) and BF16 formats + constexpr int FP8_EXPONENT = 4, BF16_EXPONENT = 8; + + // Construct and apply exponent bias + constexpr int BIAS_OFFSET = (1 << (BF16_EXPONENT - 1)) - (1 << (FP8_EXPONENT - 1)); + // Add 127 (float exponent bias) to BIAS_OFFSET and shift to float exponent + // position + constexpr uint32_t BIAS = (BIAS_OFFSET + 127) << 23; + const nv_bfloat162 bias_reg = __float2bfloat162_rn(*reinterpret_cast(&BIAS)); + + // Convert to bfloat162 and apply bias + frag_b[1] = __hmul2(frag_b[1], bias_reg); + frag_b[0] = __hmul2(frag_b[0], bias_reg); +} + +template <> +__device__ inline void dequant(int q, half2* frag_b) { + // Constants for FP4 (E2M1) and FP16 formats + constexpr int FP4_EXPONENT = 2, FP16_EXPONENT = 5; + constexpr int RIGHT_SHIFT = FP16_EXPONENT - FP4_EXPONENT; + constexpr int MASK = 0x70007000; + + // Extract and shift FP4 values to FP16 format + int Out1 = (q & 0x80008000) | ((q & MASK) >> RIGHT_SHIFT); + q <<= 4; + int Out2 = (q & 0x80008000) | ((q & MASK) >> RIGHT_SHIFT); + + // Note: reverse indexing is intentional because weights are permuted + frag_b[1] = *reinterpret_cast(&Out1); + frag_b[0] = *reinterpret_cast(&Out2); +} + +template <> +__device__ inline void dequant(int q, half2* frag_b) { + dequant(q, frag_b); + + // Constants for FP4 (E2M1) and FP16 formats + constexpr int FP4_EXPONENT = 2, FP16_EXPONENT = 5; + + // Construct and apply exponent bias + constexpr int BIAS_OFFSET = (1 << (FP16_EXPONENT - 1)) - (1 << (FP4_EXPONENT - 1)); + const half2 bias_reg = __float2half2_rn(float(1 << BIAS_OFFSET)); + + // Convert to half2 and apply bias + frag_b[1] = __hmul2(frag_b[1], bias_reg); + frag_b[0] = __hmul2(frag_b[0], bias_reg); +} + +template <> +__device__ inline void dequant(int q, nv_bfloat162* frag_b) { + // Constants for FP4 (E2M1) and FP16 formats + constexpr int FP4_EXPONENT = 2, BF16_EXPONENT = 8; + constexpr int RIGHT_SHIFT = BF16_EXPONENT - FP4_EXPONENT; + constexpr int MASK = 0x70007000; + + // Extract and shift FP4 values to FP16 format + int Out1 = (q & 0x80008000) | ((q & MASK) >> RIGHT_SHIFT); + q <<= 4; + int Out2 = (q & 0x80008000) | ((q & MASK) >> RIGHT_SHIFT); + + // Note: reverse indexing is intentional because weights are permuted + frag_b[1] = *reinterpret_cast(&Out1); + frag_b[0] = *reinterpret_cast(&Out2); +} + +template <> +__device__ inline void dequant(int q, nv_bfloat162* frag_b) { + dequant(q, frag_b); + + // Constants for FP4 (E2M1) and BF16 formats + constexpr int FP4_EXPONENT = 2, BF16_EXPONENT = 8; + + // Construct and apply exponent bias + constexpr int BIAS_OFFSET = (1 << (BF16_EXPONENT - 1)) - (1 << (FP4_EXPONENT - 1)); + // Add 127 (float exponent bias) to BIAS_OFFSET and shift to float exponent + // position + constexpr uint32_t BIAS = (BIAS_OFFSET + 127) << 23; + const nv_bfloat162 bias_reg = __float2bfloat162_rn(*reinterpret_cast(&BIAS)); + + // Convert to half2 and apply bias + frag_b[1] = __hmul2(frag_b[1], bias_reg); + frag_b[0] = __hmul2(frag_b[0], bias_reg); +} + +template +__device__ inline void dequant_fp8_scales(int q, scalar_t2* frag_b); + +template <> +__device__ inline void dequant_fp8_scales(int q, half2* frag_b) { + int Out1 = (q & 0xFF00FF00) >> 1; + ; + q <<= 8; + int Out2 = (q & 0xFF00FF00) >> 1; + + // Note: reverse indexing is intentional because weights are permuted + frag_b[1] = *reinterpret_cast(&Out1); + frag_b[0] = *reinterpret_cast(&Out2); +}; + +template <> +__device__ inline void dequant_fp8_scales(int q, nv_bfloat162* frag_b) { + constexpr int FP8_EXPONENT = 4, BF16_EXPONENT = 8; + constexpr int RIGHT_SHIFT = BF16_EXPONENT - FP8_EXPONENT; + constexpr int MASK = 0x7F007F00; + + // Extract and shift FP8 values to BF16 format + int Out1 = ((q & 0x80008000) >> 1) | ((q & MASK) >> RIGHT_SHIFT); + q <<= 8; + int Out2 = ((q & 0x80008000) >> 1) | ((q & MASK) >> RIGHT_SHIFT); + + // Note: reverse indexing is intentional because weights are permuted + frag_b[1] = *reinterpret_cast(&Out1); + frag_b[0] = *reinterpret_cast(&Out2); +}; + +// New version with s_type_id parameter for marlin_moe_wna16_v2 +template +__device__ inline void dequant_fp8_scales(int q, scalar_t2* frag_b); + +template <> +__device__ inline void dequant_fp8_scales(int q, half2* frag_b) { + int Out1 = (q & 0xFF00FF00) >> 1; + ; + q <<= 8; + int Out2 = (q & 0xFF00FF00) >> 1; + + // Note: reverse indexing is intentional because weights are permuted + frag_b[1] = *reinterpret_cast(&Out1); + frag_b[0] = *reinterpret_cast(&Out2); +}; + +template <> +__device__ inline void dequant_fp8_scales(int q, nv_bfloat162* frag_b) { + constexpr int FP8_EXPONENT = 4, BF16_EXPONENT = 8; + constexpr int RIGHT_SHIFT = BF16_EXPONENT - FP8_EXPONENT; + constexpr int MASK = 0x7F007F00; + + // Extract and shift FP8 values to BF16 format + int Out1 = ((q & 0x80008000) >> 1) | ((q & MASK) >> RIGHT_SHIFT); + q <<= 8; + int Out2 = ((q & 0x80008000) >> 1) | ((q & MASK) >> RIGHT_SHIFT); + + // Note: reverse indexing is intentional because weights are permuted + frag_b[1] = *reinterpret_cast(&Out1); + frag_b[0] = *reinterpret_cast(&Out2); +} + +template <> +__device__ inline void dequant_fp8_scales(int q, nv_bfloat162* frag_b) { + // In this conversion, 2 ** -127 in FP8E8M0 would become 0 in BF16, + // but we assume that such a extreme value would not occur in real models. + int Out1 = (q & 0xFF00FF00) >> 1; + q <<= 7; + int Out2 = q & 0x7F807F80; + + // Note: reverse indexing is intentional because weights are permuted + frag_b[1] = *reinterpret_cast(&Out1); + frag_b[0] = *reinterpret_cast(&Out2); +} + +#endif + +} // namespace MARLIN_NAMESPACE_NAME diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/gptq_marlin.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/gptq_marlin.cu new file mode 100644 index 00000000..b3437fc4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/gptq_marlin.cu @@ -0,0 +1,1120 @@ +/* + * Modified by Neural Magic + * Copyright (C) Marlin.2024 Elias Frantar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Adapted from https://github.com/IST-DASLab/marlin + */ + +#ifndef MARLIN_NAMESPACE_NAME +#define MARLIN_NAMESPACE_NAME marlin +#endif + +#include "kernel.h" +#include "marlin_template.h" + +#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \ + static_assert( \ + std::is_same::value || std::is_same::value, \ + "only float16 and bfloat16 is supported"); + +namespace marlin { + +__global__ void MarlinDefault(MARLIN_KERNEL_PARAMS){}; + +using MarlinFuncPtr = void (*)(MARLIN_KERNEL_PARAMS); + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800 + +__global__ void permute_cols_kernel( + int4 const* __restrict__ a_int4_ptr, + int const* __restrict__ perm_int_ptr, + int4* __restrict__ out_int4_ptr, + int size_m, + int size_k, + int lda, + int block_rows) {} + +} // namespace marlin + +torch::Tensor gptq_marlin_gemm( + torch::Tensor& a, + std::optional c_or_none, + torch::Tensor& b_q_weight, + torch::Tensor& b_scales, + std::optional const& b_zeros_or_none, + std::optional const& g_idx_or_none, + std::optional const& perm_or_none, + torch::Tensor& workspace, + sglang::ScalarTypeId const& b_q_type_id, + int64_t size_m, + int64_t size_n, + int64_t size_k, + bool is_k_full, + bool use_atomic_add, + bool use_fp32_reduce, + bool is_zp_float) { + TORCH_CHECK_NOT_IMPLEMENTED(false, "marlin_gemm(..) requires CUDA_ARCH >= 8.0"); + return torch::empty({1, 1}); +} + +#else + +// For a given "a" of size [M,K] performs a permutation of the K columns based +// on the given "perm" indices. +__global__ void permute_cols_kernel( + int4 const* __restrict__ a_int4_ptr, + int const* __restrict__ perm_int_ptr, + int4* __restrict__ out_int4_ptr, + int size_m, + int size_k, + int lda, + int block_rows) { + auto start_row = block_rows * blockIdx.x; + int finish_row = start_row + block_rows; + if (finish_row > size_m) { + finish_row = size_m; + } + int cur_block_rows = finish_row - start_row; + + int input_row_stride = lda * sizeof(half) / 16; + int output_row_stride = size_k * sizeof(half) / 16; + + auto permute_row = [&](int row) { + int iters = size_k / default_threads; + int rest = size_k % default_threads; + + int input_offset = row * input_row_stride; + int output_offset = row * output_row_stride; + + half const* a_row_half = reinterpret_cast(a_int4_ptr + input_offset); + half* out_half = reinterpret_cast(out_int4_ptr + output_offset); + + int base_k = 0; + + for (int i = 0; i < iters; i++) { + auto cur_k = base_k + threadIdx.x; + int src_pos = perm_int_ptr[cur_k]; + + out_half[cur_k] = a_row_half[src_pos]; + + base_k += default_threads; + } + + if (rest) { + if (threadIdx.x < rest) { + auto cur_k = base_k + threadIdx.x; + int src_pos = perm_int_ptr[cur_k]; + + out_half[cur_k] = a_row_half[src_pos]; + } + } + }; + + for (int i = 0; i < cur_block_rows; i++) { + int cur_row = start_row + i; + if (cur_row < size_m) { + permute_row(cur_row); + } + } +} + +typedef struct { + int thread_k; + int thread_n; + int num_threads; +} thread_config_t; + +thread_config_t small_batch_thread_configs[] = { + // Ordered by priority + + // thread_k, thread_n, num_threads + {128, 128, 256}, + {64, 128, 128}, + {128, 64, 128}}; + +thread_config_t large_batch_thread_configs[] = { + // Ordered by priority + + // thread_k, thread_n, num_threads + {64, 256, 256}, + {64, 128, 128}, + {128, 64, 128}}; + +typedef struct { + int blocks_per_sm; + thread_config_t tb_cfg; +} exec_config_t; + +int get_scales_cache_size( + thread_config_t const& th_config, + int prob_m, + int prob_n, + int prob_k, + int num_bits, + int group_size, + bool has_act_order, + bool is_k_full) { + bool cache_scales_chunk = has_act_order && !is_k_full; + + int tb_n = th_config.thread_n; + int tb_k = th_config.thread_k; + + // Get max scale groups per thread-block + int tb_groups; + if (group_size == -1) { + tb_groups = 1; + } else if (group_size == 0) { + tb_groups = div_ceil(tb_k, 32); // Worst case is 32 group size + } else { + tb_groups = div_ceil(tb_k, group_size); + } + + if (cache_scales_chunk) { + int load_groups = tb_groups * pipe_stages * 2; // Chunk size is 2x pipeline over dim K + load_groups = max(load_groups, 32); // We load at least 32 scale groups + return load_groups * tb_n * 2; + } else { + int tb_scales = tb_groups * tb_n * 2; + + return tb_scales * pipe_stages; + } +} + +int get_kernel_cache_size( + thread_config_t const& th_config, + int thread_m_blocks, + int prob_m, + int prob_n, + int prob_k, + int num_bits, + int group_size, + bool has_act_order, + bool is_k_full, + int has_zp, + int is_zp_float) { + int pack_factor = 32 / num_bits; + + // Get B size + int tb_k = th_config.thread_k; + int tb_n = th_config.thread_n; + int tb_m = thread_m_blocks * 16; + int sh_a_size = pipe_stages * (tb_m * tb_k) * 2; + int sh_b_size = pipe_stages * (tb_k * tb_n / pack_factor) * 4; + int sh_red_size = tb_m * (tb_n + 8); + int sh_s_size = + get_scales_cache_size(th_config, prob_m, prob_n, prob_k, num_bits, group_size, has_act_order, is_k_full); + int sh_g_idx_size = has_act_order && !is_k_full ? pipe_stages * tb_k / 4 : 0; + int sh_zp_size = 0; + if (has_zp) { + if (is_zp_float) + sh_zp_size = sh_s_size; + else if (num_bits == 4) + sh_zp_size = sh_s_size / 4; + else if (num_bits == 8) + sh_zp_size = sh_s_size / 2; + } + + int total_size = max(sh_b_size, sh_red_size) + sh_a_size + sh_s_size + sh_zp_size + sh_g_idx_size; + + return total_size; +} + +bool is_valid_config( + thread_config_t const& th_config, + int thread_m_blocks, + int prob_m, + int prob_n, + int prob_k, + int num_bits, + int group_size, + bool has_act_order, + bool is_k_full, + int has_zp, + int is_zp_float, + int max_shared_mem) { + // Sanity + if (th_config.thread_k == -1 || th_config.thread_n == -1 || th_config.num_threads == -1) { + return false; + } + + // Verify K/N are divisible by thread K/N + if (prob_k % th_config.thread_k != 0 || prob_n % th_config.thread_n != 0) { + return false; + } + + // Verify min for thread K/N + if (th_config.thread_n < min_thread_n || th_config.thread_k < min_thread_k) { + return false; + } + + // num_threads must be at least 128 (= 4 warps) + if (th_config.num_threads < 128) { + return false; + } + + // Check that pipeline fits into cache + int cache_size = get_kernel_cache_size( + th_config, + thread_m_blocks, + prob_m, + prob_n, + prob_k, + num_bits, + group_size, + has_act_order, + is_k_full, + has_zp, + is_zp_float); + return cache_size <= max_shared_mem; +} + +#define _GET_IF( \ + W_TYPE, THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, M_BLOCK_SIZE_8, GROUP_BLOCKS, NUM_THREADS, IS_ZP_FLOAT) \ + else if ( \ + q_type == W_TYPE && thread_m_blocks == THREAD_M_BLOCKS && thread_n_blocks == THREAD_N_BLOCKS && \ + thread_k_blocks == THREAD_K_BLOCKS && m_block_size_8 == M_BLOCK_SIZE_8 && group_blocks == GROUP_BLOCKS && \ + num_threads == NUM_THREADS && is_zp_float == IS_ZP_FLOAT) { \ + kernel = Marlin< \ + scalar_t, \ + W_TYPE.id(), \ + NUM_THREADS, \ + THREAD_M_BLOCKS, \ + THREAD_N_BLOCKS, \ + THREAD_K_BLOCKS, \ + M_BLOCK_SIZE_8, \ + pipe_stages, \ + GROUP_BLOCKS, \ + IS_ZP_FLOAT>; \ + } + +// COMMON: cases for (group_blocks in [-1, 2, 4, 8] and is_zp_float == false) +// this is the most common cases +// BIGGROUP: cases for big group size (group_blocks in [-1, 8]) +// FZP: cases for float-zero-point (is_zp_float = true) +// ACT: cases for act order case (group_blocks == 0) +// FP4: cases for nvfp4(e2m1) (group_blocks == 1) +#define COMMON_GET_IF_M1(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 2, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 4, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 8, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) + +#define COMMON_GET_IF_M234(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) \ + \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) \ + \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) + +#define COMMON_GET_IF(W_TYPE) \ + COMMON_GET_IF_M1(W_TYPE, 8, 8, 256) \ + COMMON_GET_IF_M1(W_TYPE, 8, 4, 128) \ + COMMON_GET_IF_M1(W_TYPE, 4, 8, 128) \ + COMMON_GET_IF_M234(W_TYPE, 16, 4, 256) \ + COMMON_GET_IF_M234(W_TYPE, 8, 4, 128) \ + COMMON_GET_IF_M234(W_TYPE, 4, 8, 128) + +#define BIGGROUP_GET_IF_M1(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 8, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) + +#define BIGGROUP_GET_IF_M234(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) + +#define BIGGROUP_GET_IF(W_TYPE) \ + BIGGROUP_GET_IF_M1(W_TYPE, 8, 8, 256) \ + BIGGROUP_GET_IF_M1(W_TYPE, 8, 4, 128) \ + BIGGROUP_GET_IF_M1(W_TYPE, 4, 8, 128) \ + BIGGROUP_GET_IF_M234(W_TYPE, 16, 4, 256) \ + BIGGROUP_GET_IF_M234(W_TYPE, 8, 4, 128) \ + BIGGROUP_GET_IF_M234(W_TYPE, 4, 8, 128) + +#define FP4_GET_IF_M1(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 1, NUM_THREADS, false) + +#define FP4_GET_IF_M234(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 1, NUM_THREADS, false) + +#define FP4_GET_IF(W_TYPE) \ + FP4_GET_IF_M1(W_TYPE, 8, 8, 256) \ + FP4_GET_IF_M1(W_TYPE, 8, 4, 128) \ + FP4_GET_IF_M1(W_TYPE, 4, 8, 128) \ + FP4_GET_IF_M234(W_TYPE, 16, 4, 256) \ + FP4_GET_IF_M234(W_TYPE, 8, 4, 128) \ + FP4_GET_IF_M234(W_TYPE, 4, 8, 128) + +// We currently have 4-bit models only with group_blocks == 4 +#define FZP_GET_IF_M1(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 4, NUM_THREADS, true) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, true) + +#define FZP_GET_IF_M234(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, true) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, true) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, true) + +#define FZP_GET_IF(W_TYPE) \ + FZP_GET_IF_M1(W_TYPE, 8, 8, 256) \ + FZP_GET_IF_M1(W_TYPE, 8, 4, 128) \ + FZP_GET_IF_M1(W_TYPE, 4, 8, 128) \ + FZP_GET_IF_M234(W_TYPE, 16, 4, 256) \ + FZP_GET_IF_M234(W_TYPE, 8, 4, 128) \ + FZP_GET_IF_M234(W_TYPE, 4, 8, 128) + +// We currently have 4-bit models only with group_blocks == 4 +#define ACT_GET_IF_M1(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 0, NUM_THREADS, false) + +#define ACT_GET_IF_M234(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 0, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 0, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 0, NUM_THREADS, false) + +#define ACT_GET_IF(W_TYPE) \ + ACT_GET_IF_M1(W_TYPE, 8, 8, 256) \ + ACT_GET_IF_M1(W_TYPE, 8, 4, 128) \ + ACT_GET_IF_M1(W_TYPE, 4, 8, 128) \ + ACT_GET_IF_M234(W_TYPE, 16, 4, 256) \ + ACT_GET_IF_M234(W_TYPE, 8, 4, 128) \ + ACT_GET_IF_M234(W_TYPE, 4, 8, 128) + +template +MarlinFuncPtr get_marlin_kernel( + const sglang::ScalarType q_type, + int thread_m_blocks, + int thread_n_blocks, + int thread_k_blocks, + bool m_block_size_8, + bool has_act_order, + bool has_zp, + int group_blocks, + int num_threads, + bool is_zp_float) { + int num_bits = q_type.size_bits(); + auto kernel = MarlinDefault; + if (false) { + } + + COMMON_GET_IF(sglang::kU4) + COMMON_GET_IF(sglang::kU4B8) + COMMON_GET_IF(sglang::kU8B128) + + FP4_GET_IF(sglang::kFE2M1f) + + BIGGROUP_GET_IF(sglang::kFE4M3fn) + + ACT_GET_IF(sglang::kU4B8) + ACT_GET_IF(sglang::kU8B128) + + if (std::is_same::value) { + if (false) { + } + FZP_GET_IF(sglang::kU4) + } + + return kernel; +} + +template +exec_config_t determine_exec_config( + const sglang::ScalarType& q_type, + int prob_m, + int prob_n, + int prob_k, + int thread_m_blocks, + bool m_block_size_8, + int num_bits, + int group_size, + bool has_act_order, + bool is_k_full, + bool has_zp, + bool is_zp_float, + int max_shared_mem, + int sms) { + exec_config_t exec_cfg = exec_config_t{1, thread_config_t{-1, -1, -1}}; + thread_config_t* thread_configs = thread_m_blocks > 1 ? large_batch_thread_configs : small_batch_thread_configs; + int thread_configs_size = thread_m_blocks > 1 ? sizeof(large_batch_thread_configs) / sizeof(thread_config_t) + : sizeof(small_batch_thread_configs) / sizeof(thread_config_t); + + for (int i = 0; i < thread_configs_size; i++) { + thread_config_t th_config = thread_configs[i]; + + if (!is_valid_config( + th_config, + thread_m_blocks, + prob_m, + prob_n, + prob_k, + num_bits, + group_size, + has_act_order, + is_k_full, + has_zp, + is_zp_float, + max_shared_mem)) { + continue; + } + + int cache_size = get_kernel_cache_size( + th_config, + thread_m_blocks, + prob_m, + prob_n, + prob_k, + num_bits, + group_size, + has_act_order, + is_k_full, + has_zp, + is_zp_float); + + int group_blocks = 0; + if (!has_act_order) { + group_blocks = group_size == -1 ? -1 : group_size / 16; + } + + auto kernel = get_marlin_kernel( + q_type, + thread_m_blocks, + th_config.thread_n / 16, + th_config.thread_k / 16, + m_block_size_8, + has_act_order, + has_zp, + group_blocks, + th_config.num_threads, + is_zp_float); + + if (kernel == MarlinDefault) continue; + + // int m_tiles = div_ceil(prob_m, thread_m_blocks * 16); + // int n_tiles = prob_n / th_config.thread_n; + // int k_tiles = prob_k / th_config.thread_k; + + return {1, th_config}; + } + + return exec_cfg; +} + +template +void marlin_mm( + const void* A, + const void* B, + void* C, + void* C_tmp, + void* s, + void* s2, + void* zp, + void* g_idx, + void* perm, + void* a_tmp, + int prob_m, + int prob_n, + int prob_k, + int lda, + void* workspace, + sglang::ScalarType const& q_type, + bool has_act_order, + bool is_k_full, + bool has_zp, + int num_groups, + int group_size, + int dev, + cudaStream_t stream, + int thread_k_init, + int thread_n_init, + int sms, + bool use_atomic_add, + bool use_fp32_reduce, + bool is_zp_float) { + if (has_zp) { + TORCH_CHECK( + q_type == sglang::kU4 || q_type == sglang::kU8, + "q_type must be u4 or u8 when has_zp = True. Got = ", + q_type.str()); + } else { + TORCH_CHECK( + q_type == sglang::kU4B8 || q_type == sglang::kU8B128 || q_type == sglang::kFE4M3fn || q_type == sglang::kFE2M1f, + "q_type must be uint4b8, uint8b128, float8_e4m3fn or float4_e2m1f when " + "has_zp = False. Got = ", + q_type.str()); + } + + TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m, ", ", prob_n, ", ", prob_k, "]"); + + int group_blocks = 0; + if (has_act_order) { + if (is_k_full) { + TORCH_CHECK(group_size != -1); + group_blocks = group_size / 16; + TORCH_CHECK( + prob_k % group_blocks == 0, "prob_k = ", prob_k, " is not divisible by group_blocks = ", group_blocks); + } else { + TORCH_CHECK(group_size == 0); + group_blocks = 0; + } + } else { + if (group_size == -1) { + group_blocks = -1; + } else { + group_blocks = group_size / 16; + TORCH_CHECK( + prob_k % group_blocks == 0, "prob_k = ", prob_k, " is not divisible by group_blocks = ", group_blocks); + } + } + + int num_bits = q_type.size_bits(); + const int4* A_ptr = (const int4*)A; + const int4* B_ptr = (const int4*)B; + int4* C_ptr = (int4*)C; + int4* C_tmp_ptr = (int4*)C_tmp; + const int4* s_ptr = (const int4*)s; + const uint16_t* s2_ptr = (const uint16_t*)s2; + const int4* zp_ptr = (const int4*)zp; + const int* g_idx_ptr = (const int*)g_idx; + const int* perm_ptr = (const int*)perm; + int4* a_tmp_ptr = (int4*)a_tmp; + + int* locks = (int*)workspace; + + if (has_act_order) { + // Permute A columns + int block_rows = div_ceil(prob_m, sms); + // avoid ">>>" being formatted to "> > >" + // clang-format off + permute_cols_kernel<<>>( + A_ptr, perm_ptr, a_tmp_ptr, prob_m, prob_k, lda, block_rows); + // clang-format on + A_ptr = a_tmp_ptr; + lda = prob_k; + + // If we have a full K, then we can run the non-act-order version of Marlin + // (since the weight rows are reordered by increasing group ids, and by + // having a full K, we have full original groups) + if (is_k_full) has_act_order = false; + } + + int max_shared_mem = 0; + cudaDeviceGetAttribute(&max_shared_mem, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev); + TORCH_CHECK(max_shared_mem > 0); + + int max_par = 16; + if (prob_n <= 4096) max_par = 16 * 8; + int max_shared_mem_new = max_shared_mem; + int rest_m = prob_m; + int max_thread_m_blocks = 4; + while (rest_m) { + int par_count = rest_m / (max_thread_m_blocks * 16); + if (par_count > max_par) par_count = max_par; + int prob_m_split = par_count > 0 ? (par_count * (max_thread_m_blocks * 16)) : rest_m; + + int thread_k = thread_k_init; + int thread_n = thread_n_init; + + int thread_m_blocks = min(div_ceil(prob_m_split, 16), max_thread_m_blocks); + int m_block_size_8 = prob_m_split <= 8; + + // Set thread config + exec_config_t exec_cfg; + thread_config_t thread_tfg; + if (thread_k != -1 && thread_n != -1) { + thread_tfg = thread_config_t{thread_k, thread_n, default_threads}; + exec_cfg = exec_config_t{1, thread_tfg}; + TORCH_CHECK(prob_n % thread_n == 0, "prob_n = ", prob_n, " is not divisible by thread_n = ", thread_n); + TORCH_CHECK(prob_k % thread_k == 0, "prob_k = ", prob_k, " is not divisible by thread_k = ", thread_k); + } else { + // Auto config + exec_cfg = determine_exec_config( + q_type, + prob_m_split, + prob_n, + prob_k, + thread_m_blocks, + m_block_size_8, + num_bits, + group_size, + has_act_order, + is_k_full, + has_zp, + is_zp_float, + max_shared_mem, + sms); + thread_tfg = exec_cfg.tb_cfg; + if (thread_tfg.thread_k == -1 && max_thread_m_blocks > 1) { + max_thread_m_blocks--; + continue; + } + } + + int num_threads = thread_tfg.num_threads; + thread_k = thread_tfg.thread_k; + thread_n = thread_tfg.thread_n; + int blocks = sms * exec_cfg.blocks_per_sm; + if (exec_cfg.blocks_per_sm > 1) max_shared_mem_new = max_shared_mem / exec_cfg.blocks_per_sm - 1024; + + int thread_k_blocks = thread_k / 16; + int thread_n_blocks = thread_n / 16; + + TORCH_CHECK( + is_valid_config( + thread_tfg, + thread_m_blocks, + prob_m_split, + prob_n, + prob_k, + num_bits, + group_size, + has_act_order, + is_k_full, + has_zp, + is_zp_float, + max_shared_mem_new), + "Invalid thread config: thread_m_blocks = ", + thread_m_blocks, + ", thread_k = ", + thread_tfg.thread_k, + ", thread_n = ", + thread_tfg.thread_n, + ", num_threads = ", + thread_tfg.num_threads, + " for MKN = [", + prob_m, + ", ", + prob_k, + ", ", + prob_n, + "] and num_bits = ", + num_bits, + ", prob_m_split = ", + prob_m_split, + ", group_size = ", + group_size, + ", has_act_order = ", + has_act_order, + ", is_k_full = ", + is_k_full, + ", has_zp = ", + has_zp, + ", is_zp_float = ", + is_zp_float, + ", max_shared_mem_new = ", + max_shared_mem_new); + + auto kernel = get_marlin_kernel( + q_type, + thread_m_blocks, + thread_n_blocks, + thread_k_blocks, + m_block_size_8, + has_act_order, + has_zp, + group_blocks, + num_threads, + is_zp_float); + + if (kernel == MarlinDefault) { + TORCH_CHECK( + false, + "Unsupported shapes: MNK = [", + prob_m, + ", ", + prob_n, + ", ", + prob_k, + "]", + ", has_act_order = ", + has_act_order, + ", num_groups = ", + num_groups, + ", group_size = ", + group_size, + ", prob_m_split = ", + prob_m_split, + ", thread_m_blocks = ", + thread_m_blocks, + ", thread_n_blocks = ", + thread_n_blocks, + ", thread_k_blocks = ", + thread_k_blocks, + ", num_threads = ", + num_threads, + ", num_bits = ", + num_bits); + } + + cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem_new); + + bool part_use_atomic_add = use_atomic_add && div_ceil(prob_m_split, 64) * prob_n <= 2048; + + // avoid ">>>" being formatted to "> > >" + // clang-format off + kernel<<>>( + A_ptr, B_ptr, C_ptr, C_tmp_ptr, s_ptr, s2_ptr, zp_ptr, g_idx_ptr, num_groups, + prob_m_split, prob_n, prob_k, lda, locks, part_use_atomic_add, + use_fp32_reduce, max_shared_mem_new); + // clang-format on + + A_ptr += prob_m_split * (lda / 8); + C_ptr += prob_m_split * (prob_n / 8); + rest_m -= prob_m_split; + } +} + +} // namespace marlin + +torch::Tensor gptq_marlin_gemm( + torch::Tensor& a, + std::optional c_or_none, + torch::Tensor& b_q_weight, + torch::Tensor& b_scales, + std::optional const& global_scale_or_none, + std::optional const& b_zeros_or_none, + std::optional const& g_idx_or_none, + std::optional const& perm_or_none, + torch::Tensor& workspace, + sglang::ScalarTypeId const& b_q_type_id, + int64_t size_m, + int64_t size_n, + int64_t size_k, + bool is_k_full, + bool use_atomic_add, + bool use_fp32_reduce, + bool is_zp_float) { + sglang::ScalarType const b_q_type = sglang::ScalarType::from_id(b_q_type_id); + int pack_factor = 32 / b_q_type.size_bits(); + + // Verify A + TORCH_CHECK(a.size(0) == size_m, "Shape mismatch: a.size(0) = ", a.size(0), ", size_m = ", size_m); + TORCH_CHECK(a.size(1) == size_k, "Shape mismatch: a.size(1) = ", a.size(1), ", size_k = ", size_k); + + // Verify B + TORCH_CHECK( + size_k % MARLIN_NAMESPACE_NAME::tile_size == 0, + "size_k = ", + size_k, + " is not divisible by tile_size = ", + MARLIN_NAMESPACE_NAME::tile_size); + TORCH_CHECK( + (size_k / MARLIN_NAMESPACE_NAME::tile_size) == b_q_weight.size(0), + "Shape mismatch: b_q_weight.size(0) = ", + b_q_weight.size(0), + ", size_k = ", + size_k, + ", tile_size = ", + MARLIN_NAMESPACE_NAME::tile_size); + TORCH_CHECK( + b_q_weight.size(1) % MARLIN_NAMESPACE_NAME::tile_size == 0, + "b_q_weight.size(1) = ", + b_q_weight.size(1), + " is not divisible by tile_size = ", + MARLIN_NAMESPACE_NAME::tile_size); + int actual_size_n = (b_q_weight.size(1) / MARLIN_NAMESPACE_NAME::tile_size) * pack_factor; + TORCH_CHECK(size_n == actual_size_n, "size_n = ", size_n, ", actual_size_n = ", actual_size_n); + + // Verify device and strides + TORCH_CHECK(a.device().is_cuda(), "A is not on GPU"); + TORCH_CHECK(a.stride(1) == 1, "A.stride(1) is not 1"); + // We use int4 (16 bytes) to load A, so A must aligned to 16 bytes + TORCH_CHECK(a.stride(0) % 8 == 0, "A.stride(0) must divisible by 8"); + TORCH_CHECK(((uint64_t)a.data_ptr()) % 16 == 0, "A must aligned to 16 bytes"); + + TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU"); + TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous"); + + TORCH_CHECK(b_scales.device().is_cuda(), "b_scales is not on GPU"); + TORCH_CHECK(b_scales.is_contiguous(), "b_scales is not contiguous"); + + // thread_k: `k` size of a thread_tile in `weights` (can usually be left as + // auto -1) + int thread_k = -1; + // thread_n: `n` size of a thread_tile in `weights` (can usually be left as + // auto -1) + int thread_n = -1; + // sms: number of SMs to use for the kernel + int sms = -1; + cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, a.get_device()); + + // Alloc buffers + const at::cuda::OptionalCUDAGuard device_guard(device_of(a)); + auto options = torch::TensorOptions().dtype(a.dtype()).device(a.device()); + torch::Tensor c; + if (c_or_none.has_value()) { + c = c_or_none.value(); + TORCH_CHECK(c.device().is_cuda(), "c is not on GPU"); + TORCH_CHECK(c.is_contiguous(), "c is not contiguous"); + TORCH_CHECK(c.size(0) == size_m, "Shape mismatch: c.size(0) = ", c.size(0), ", size_m = ", size_m); + TORCH_CHECK(c.size(1) == size_n, "Shape mismatch: c.size(1) = ", c.size(1), ", size_n = ", size_n); + } else { + c = torch::empty({size_m, size_n}, options); + } + if (size_m == 0) return c; + + // Alloc C tmp buffer that is going to be used for the global reduce + torch::Tensor c_tmp; + auto options_fp32 = torch::TensorOptions().dtype(at::kFloat).device(a.device()); + if (use_fp32_reduce) { + int max_m_block_size = (size_m + 16 - 1) / 16 * 16; + max_m_block_size = min(max_m_block_size, 64); + int max_c_tmp_size = sms * max_m_block_size * MARLIN_NAMESPACE_NAME::max_thread_n; + c_tmp = torch::empty({max_c_tmp_size}, options_fp32); + } else { + c_tmp = torch::empty({0}, options_fp32); + } + + // Detect groupsize and act_order + int num_groups = -1; + int group_size = -1; + + int rank = b_scales.sizes().size(); + TORCH_CHECK(rank == 2, "b_scales rank = ", rank, " is not 2"); + TORCH_CHECK(b_scales.size(1) == size_n, "b_scales dim 1 = ", b_scales.size(1), " is not size_n = ", size_n); + num_groups = b_scales.size(0); + + torch::Tensor g_idx, perm, a_tmp; + if (g_idx_or_none.has_value() && perm_or_none.has_value()) { + g_idx = g_idx_or_none.value(); + perm = perm_or_none.value(); + + TORCH_CHECK(g_idx.device().is_cuda(), "g_idx is not on GPU"); + TORCH_CHECK(g_idx.is_contiguous(), "g_idx is not contiguous"); + TORCH_CHECK(perm.device().is_cuda(), "perm is not on GPU"); + TORCH_CHECK(perm.is_contiguous(), "perm is not contiguous"); + + // Verify g_idx and perm + TORCH_CHECK( + (g_idx.size(-1) == 0 && perm.size(-1) == 0) || (g_idx.size(-1) == size_k && perm.size(-1) == size_k), + "Unexpected g_idx.size(-1) = ", + g_idx.size(-1), + " and perm.size(-1) = ", + perm.size(-1), + ", where size_k = ", + size_k); + } else { + g_idx = torch::empty({0}, options); + perm = torch::empty({0}, options); + a_tmp = torch::empty({0}, options); + } + bool has_act_order = g_idx.size(-1) > 0 && perm.size(-1) > 0; + + if (has_act_order) { + a_tmp = torch::empty({size_m, size_k}, options); + if (is_k_full) { + TORCH_CHECK(num_groups > 1, "For act_order, num_groups must be > 1"); + TORCH_CHECK(size_k % num_groups == 0, "size_k = ", size_k, ", is not divisible by num_groups = ", num_groups); + group_size = size_k / num_groups; + } else { + group_size = 0; + } + + } else { + a_tmp = torch::empty({0}, options); + if (num_groups > 1) { + TORCH_CHECK( + size_k % num_groups == 0, "size_k = ", size_k, ", is not divisible by b_scales.size(0) = ", b_scales.size(0)); + group_size = size_k / num_groups; + } else { + group_size = -1; + } + } + + torch::Tensor global_scale; + if (global_scale_or_none.has_value()) { + global_scale = global_scale_or_none.value(); + TORCH_CHECK(b_q_type == sglang::kFE2M1f, "global_scale can only be used for float4_e2m1f."); + } else { + global_scale = torch::empty({0}, options); + TORCH_CHECK(!(b_q_type == sglang::kFE2M1f), "the global_scale parameter must be passed for float4_e2m1f."); + } + + torch::Tensor b_zeros; + if (b_zeros_or_none.has_value()) { + b_zeros = b_zeros_or_none.value(); + TORCH_CHECK(b_zeros.device().is_cuda(), "b_zeros is not on GPU"); + TORCH_CHECK(b_zeros.is_contiguous(), "b_zeros is not contiguous"); + } else { + b_zeros = torch::empty({0}, options); + } + bool has_zp = b_zeros.size(-1) > 0; + if (has_zp) { + TORCH_CHECK( + b_q_type == sglang::kU4 || b_q_type == sglang::kU8, + "b_q_type must be u4 or u8 when has_zp = True. Got = ", + b_q_type.str()); + } else { + TORCH_CHECK( + b_q_type == sglang::kU4B8 || b_q_type == sglang::kU8B128 || b_q_type == sglang::kFE4M3fn || + b_q_type == sglang::kFE2M1f, + "b_q_type must be uint4b8, uint8b128, float8_e4m3fn or " + "float4_e2m1f when " + "has_zp = False. Got = ", + b_q_type.str()); + } + + if (has_zp && is_zp_float) { + TORCH_CHECK( + a.scalar_type() == at::ScalarType::Half, + "Computation type must be float16 (half) when using float zero " + "points."); + } + + // Verify b_zeros + if (has_zp) { + int rank = b_zeros.sizes().size(); + TORCH_CHECK(rank == 2, "b_zeros rank = ", rank, " is not 2"); + if (is_zp_float) { + TORCH_CHECK(b_zeros.size(1) == size_n, "b_zeros dim 1 = ", b_zeros.size(1), " is not size_n = ", size_n); + TORCH_CHECK( + num_groups == b_zeros.size(0), "b_zeros dim 0 = ", b_zeros.size(0), " is not num_groups = ", num_groups); + TORCH_CHECK(num_groups != -1, "num_groups must be != -1"); + } else { + TORCH_CHECK( + b_zeros.size(0) == num_groups, "b_zeros dim 0 = ", b_zeros.size(0), " is not num_groups = ", num_groups); + TORCH_CHECK( + b_zeros.size(1) == size_n / pack_factor, + "b_zeros dim 1 = ", + b_zeros.size(1), + " is not size_n / pack_factor = ", + size_n / pack_factor); + } + } + + // Verify workspace size + TORCH_CHECK( + size_n % MARLIN_NAMESPACE_NAME::min_thread_n == 0, + "size_n = ", + size_n, + ", is not divisible by min_thread_n = ", + MARLIN_NAMESPACE_NAME::min_thread_n); + + int min_workspace_size = sms; + TORCH_CHECK( + workspace.numel() >= min_workspace_size, + "workspace.numel = ", + workspace.numel(), + " is below min_workspace_size = ", + min_workspace_size); + + int dev = a.get_device(); + if (a.scalar_type() == at::ScalarType::Half) { + void* scales_ptr; + if (b_q_type == sglang::kFE2M1f) { + scales_ptr = b_scales.data_ptr(); + } else { + scales_ptr = b_scales.data_ptr(); + } + + marlin::marlin_mm( + a.data_ptr(), + b_q_weight.data_ptr(), + c.data_ptr(), + c_tmp.data_ptr(), + scales_ptr, + global_scale.data_ptr(), + b_zeros.data_ptr(), + g_idx.data_ptr(), + perm.data_ptr(), + a_tmp.data_ptr(), + size_m, + size_n, + size_k, + a.stride(0), + workspace.data_ptr(), + b_q_type, + has_act_order, + is_k_full, + has_zp, + num_groups, + group_size, + dev, + at::cuda::getCurrentCUDAStream(dev), + thread_k, + thread_n, + sms, + use_atomic_add, + use_fp32_reduce, + is_zp_float); + } else if (a.scalar_type() == at::ScalarType::BFloat16) { + void* scales_ptr; + if (b_q_type == sglang::kFE2M1f) { + scales_ptr = b_scales.data_ptr(); + } else { + scales_ptr = b_scales.data_ptr(); + } + + marlin::marlin_mm( + a.data_ptr(), + b_q_weight.data_ptr(), + c.data_ptr(), + c_tmp.data_ptr(), + scales_ptr, + global_scale.data_ptr(), + b_zeros.data_ptr(), + g_idx.data_ptr(), + perm.data_ptr(), + a_tmp.data_ptr(), + size_m, + size_n, + size_k, + a.stride(0), + workspace.data_ptr(), + b_q_type, + has_act_order, + is_k_full, + has_zp, + num_groups, + group_size, + dev, + at::cuda::getCurrentCUDAStream(dev), + thread_k, + thread_n, + sms, + use_atomic_add, + use_fp32_reduce, + is_zp_float); + } else { + TORCH_CHECK(false, "gpt_marlin_gemm only supports bfloat16 and float16"); + } + + return c; +} + +#endif diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/gptq_marlin_repack.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/gptq_marlin_repack.cu new file mode 100644 index 00000000..3bdf8416 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/gptq_marlin_repack.cu @@ -0,0 +1,329 @@ +#include "marlin.cuh" + +namespace marlin { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800 +template +__global__ void gptq_marlin_repack_kernel( + uint32_t const* __restrict__ b_q_weight_ptr, + uint32_t const* __restrict__ perm_ptr, + uint32_t* __restrict__ out_ptr, + int size_k, + int size_n) { + return; +} +#else +template +__global__ void gptq_marlin_repack_kernel( + uint32_t const* __restrict__ b_q_weight_ptr, + uint32_t const* __restrict__ perm_ptr, + uint32_t* __restrict__ out_ptr, + int size_k, + int size_n) { + constexpr int pack_factor = 32 / num_bits; + + int k_tiles = size_k / tile_k_size; + int n_tiles = size_n / tile_n_size; + int block_k_tiles = div_ceil(k_tiles, gridDim.x); + + auto start_k_tile = blockIdx.x * block_k_tiles; + if (start_k_tile >= k_tiles) { + return; + } + + int finish_k_tile = min(start_k_tile + block_k_tiles, k_tiles); + + // Wait until the next thread tile has been loaded to shared memory. + auto wait_for_stage = [&]() { + // We only have `stages - 2` active fetches since we are double buffering + // and can only issue the next fetch when it is guaranteed that the previous + // shared memory load is fully complete (as it may otherwise be + // overwritten). + cp_async_wait(); + __syncthreads(); + }; + + extern __shared__ int4 sh[]; + + constexpr int perm_size = tile_k_size / 4; + + int4* sh_perm_ptr = sh; + int4* sh_pipe_ptr = sh_perm_ptr; + if constexpr (has_perm) { + sh_pipe_ptr += perm_size; + } + + constexpr int tile_ints = tile_k_size / pack_factor; + + constexpr int stage_n_threads = tile_n_size / 4; + constexpr int stage_k_threads = has_perm ? tile_k_size : tile_ints; + constexpr int stage_size = stage_k_threads * stage_n_threads; + + auto load_perm_to_shared = [&](int k_tile_id) { + int first_k_int4 = (k_tile_id * tile_k_size) / 4; + + int4 const* perm_int4_ptr = reinterpret_cast(perm_ptr); + + if (threadIdx.x < perm_size) { + sh_perm_ptr[threadIdx.x] = perm_int4_ptr[first_k_int4 + threadIdx.x]; + } + __syncthreads(); + }; + + auto fetch_to_shared = [&](int pipe, int k_tile_id, int n_tile_id) { + if (n_tile_id >= n_tiles) { + cp_async_fence(); + return; + } + + int first_n = n_tile_id * tile_n_size; + + int4* sh_ptr = sh_pipe_ptr + stage_size * pipe; + + if constexpr (has_perm) { + if (threadIdx.x < stage_size) { + auto k_id = threadIdx.x / stage_n_threads; + auto n_id = threadIdx.x % stage_n_threads; + + uint32_t const* sh_perm_int_ptr = reinterpret_cast(sh_perm_ptr); + + int src_k = sh_perm_int_ptr[k_id]; + int src_k_packed = src_k / pack_factor; + + cp_async4( + &sh_ptr[k_id * stage_n_threads + n_id], + reinterpret_cast(&(b_q_weight_ptr[src_k_packed * size_n + first_n + (n_id * 4)]))); + } + + } else { + if (threadIdx.x < stage_size) { + auto k_id = threadIdx.x / stage_n_threads; + auto n_id = threadIdx.x % stage_n_threads; + + int first_k = k_tile_id * tile_k_size; + int first_k_packed = first_k / pack_factor; + + cp_async4( + &sh_ptr[k_id * stage_n_threads + n_id], + reinterpret_cast(&(b_q_weight_ptr[(first_k_packed + k_id) * size_n + first_n + (n_id * 4)]))); + } + } + + cp_async_fence(); + }; + + auto repack_tile = [&](int pipe, int k_tile_id, int n_tile_id) { + if (n_tile_id >= n_tiles) { + return; + } + + auto warp_id = threadIdx.x / 32; + auto th_id = threadIdx.x % 32; + + if (warp_id >= 4) { + return; + } + + int tc_col = th_id / 4; + int tc_row = (th_id % 4) * 2; + + constexpr int tc_offsets[4] = {0, 1, 8, 9}; + + int cur_n = warp_id * 16 + tc_col; + + constexpr int sh_stride = 64; + constexpr uint32_t mask = (1 << num_bits) - 1; + + int4* sh_stage_ptr = sh_pipe_ptr + stage_size * pipe; + uint32_t* sh_stage_int_ptr = reinterpret_cast(sh_stage_ptr); + + uint32_t* sh_perm_int_ptr = reinterpret_cast(sh_perm_ptr); + + uint32_t vals[8]; + + if constexpr (has_perm) { + for (int i = 0; i < 4; i++) { + int k_idx = tc_row + tc_offsets[i]; + + uint32_t src_k = sh_perm_int_ptr[k_idx]; + uint32_t src_k_pos = src_k % pack_factor; + + uint32_t b1_val = sh_stage_int_ptr[k_idx * sh_stride + cur_n]; + uint32_t b1_cur_val = (b1_val >> (src_k_pos * num_bits)) & mask; + + uint32_t b2_val = sh_stage_int_ptr[k_idx * sh_stride + cur_n + 8]; + uint32_t b2_cur_val = (b2_val >> (src_k_pos * num_bits)) & mask; + + vals[i] = b1_cur_val; + vals[4 + i] = b2_cur_val; + } + + } else { + uint32_t b1_vals[tile_ints]; + uint32_t b2_vals[tile_ints]; + +#pragma unroll + for (int i = 0; i < tile_ints; i++) { + b1_vals[i] = sh_stage_int_ptr[cur_n + sh_stride * i]; + b2_vals[i] = sh_stage_int_ptr[cur_n + 8 + sh_stride * i]; + } + +#pragma unroll + for (int i = 0; i < 4; i++) { + int cur_elem = tc_row + tc_offsets[i]; + int cur_int = cur_elem / pack_factor; + int cur_pos = cur_elem % pack_factor; + + vals[i] = (b1_vals[cur_int] >> (cur_pos * num_bits)) & mask; + vals[4 + i] = (b2_vals[cur_int] >> (cur_pos * num_bits)) & mask; + } + } + + constexpr int tile_size = tile_k_size * tile_n_size / pack_factor; + int out_offset = (k_tile_id * n_tiles + n_tile_id) * tile_size; + + // Result of: + // https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h + if constexpr (num_bits == 4) { + constexpr int pack_idx[8] = {0, 2, 4, 6, 1, 3, 5, 7}; + + uint32_t res = 0; +#pragma unroll + for (int i = 0; i < 8; i++) { + res |= vals[pack_idx[i]] << (i * 4); + } + + out_ptr[out_offset + th_id * 4 + warp_id] = res; + + } else { + constexpr int pack_idx[4] = {0, 2, 1, 3}; + + uint32_t res1 = 0; + uint32_t res2 = 0; +#pragma unroll + for (int i = 0; i < 4; i++) { + res1 |= vals[pack_idx[i]] << (i * 8); + res2 |= vals[4 + pack_idx[i]] << (i * 8); + } + + out_ptr[out_offset + th_id * 8 + (warp_id * 2) + 0] = res1; + out_ptr[out_offset + th_id * 8 + (warp_id * 2) + 1] = res2; + } + }; + + auto start_pipes = [&](int k_tile_id, int n_tile_id) { +#pragma unroll + for (int pipe = 0; pipe < repack_stages - 1; pipe++) { + fetch_to_shared(pipe, k_tile_id, n_tile_id + pipe); + } + + wait_for_stage(); + }; +#pragma unroll + for (int k_tile_id = start_k_tile; k_tile_id < finish_k_tile; k_tile_id++) { + int n_tile_id = 0; + + if constexpr (has_perm) { + load_perm_to_shared(k_tile_id); + } + + start_pipes(k_tile_id, n_tile_id); + + while (n_tile_id < n_tiles) { +#pragma unroll + for (int pipe = 0; pipe < repack_stages; pipe++) { + fetch_to_shared((pipe + repack_stages - 1) % repack_stages, k_tile_id, n_tile_id + pipe + repack_stages - 1); + repack_tile(pipe, k_tile_id, n_tile_id + pipe); + wait_for_stage(); + } + n_tile_id += repack_stages; + } + } +} +#endif +} // namespace marlin + +#define CALL_IF(NUM_BITS, HAS_PERM) \ + else if (num_bits == NUM_BITS && has_perm == HAS_PERM) { \ + cudaFuncSetAttribute( \ + marlin::gptq_marlin_repack_kernel, \ + cudaFuncAttributeMaxDynamicSharedMemorySize, \ + max_shared_mem); \ + marlin::gptq_marlin_repack_kernel \ + <<>>( \ + b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n); \ + } + +torch::Tensor +gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm, int64_t size_k, int64_t size_n, int64_t num_bits) { + // Verify compatibility with marlin tile of 16x64 + TORCH_CHECK( + size_k % marlin::tile_k_size == 0, + "size_k = ", + size_k, + " is not divisible by tile_k_size = ", + marlin::tile_k_size); + TORCH_CHECK( + size_n % marlin::tile_n_size == 0, + "size_n = ", + size_n, + " is not divisible by tile_n_size = ", + marlin::tile_n_size); + + TORCH_CHECK(num_bits == 4 || num_bits == 8, "num_bits must be 4 or 8. Got = ", num_bits); + int const pack_factor = 32 / num_bits; + + // Verify B + TORCH_CHECK( + (size_k / pack_factor) == b_q_weight.size(0), + "Shape mismatch: b_q_weight.size(0) = ", + b_q_weight.size(0), + ", size_k = ", + size_k, + ", pack_factor = ", + pack_factor); + TORCH_CHECK(b_q_weight.size(1) == size_n, "b_q_weight.size(1) = ", b_q_weight.size(1), " is not size_n = ", size_n); + + // Verify device and strides + TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU"); + TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous"); + TORCH_CHECK(b_q_weight.dtype() == at::kInt, "b_q_weight type is not kInt"); + + TORCH_CHECK(perm.device().is_cuda(), "perm is not on GPU"); + TORCH_CHECK(perm.is_contiguous(), "perm is not contiguous"); + TORCH_CHECK(perm.dtype() == at::kInt, "perm type is not at::kInt"); + + // Alloc buffers + const at::cuda::OptionalCUDAGuard device_guard(device_of(b_q_weight)); + auto options = torch::TensorOptions().dtype(b_q_weight.dtype()).device(b_q_weight.device()); + torch::Tensor out = torch::empty({size_k / marlin::tile_size, size_n * marlin::tile_size / pack_factor}, options); + + // Detect if there is act_order + bool has_perm = perm.size(0) != 0; + + // Get ptrs + uint32_t const* b_q_weight_ptr = reinterpret_cast(b_q_weight.data_ptr()); + uint32_t const* perm_ptr = reinterpret_cast(perm.data_ptr()); + uint32_t* out_ptr = reinterpret_cast(out.data_ptr()); + + // Get dev info + int dev = b_q_weight.get_device(); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(dev); + int blocks; + cudaDeviceGetAttribute(&blocks, cudaDevAttrMultiProcessorCount, dev); + + int max_shared_mem = 0; + cudaDeviceGetAttribute(&max_shared_mem, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev); + TORCH_CHECK(max_shared_mem > 0); + + if (false) { + } + CALL_IF(4, false) + CALL_IF(4, true) + CALL_IF(8, false) + CALL_IF(8, true) + else { + TORCH_CHECK(false, "Unsupported repack config: num_bits = ", num_bits, ", has_perm = ", has_perm); + } + + return out; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/kernel.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/kernel.h new file mode 100644 index 00000000..72137b24 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/kernel.h @@ -0,0 +1,36 @@ + +#ifndef MARLIN_NAMESPACE_NAME +#define MARLIN_NAMESPACE_NAME marlin +#endif + +#include "marlin.cuh" +#include "marlin_dtypes.cuh" +#include "scalar_type.hpp" + +#define MARLIN_KERNEL_PARAMS \ + const int4 *__restrict__ A, const int4 *__restrict__ B, int4 *__restrict__ C, int4 *__restrict__ C_tmp, \ + const int4 *__restrict__ scales_ptr, const uint16_t *__restrict__ scale2_ptr, const int4 *__restrict__ zp_ptr, \ + const int *__restrict__ g_idx, int num_groups, int prob_m, int prob_n, int prob_k, int lda, int *locks, \ + bool use_atomic_add, bool use_fp32_reduce, int max_shared_mem + +namespace MARLIN_NAMESPACE_NAME { +template < + typename scalar_t, // compute dtype, half or nv_float16 + const sglang::ScalarTypeId w_type_id, // weight ScalarType id + const int threads, // number of threads in a threadblock + const int thread_m_blocks, // number of 16x16 blocks in the m + // dimension (batchsize) of the + // threadblock + const int thread_n_blocks, // same for n dimension (output) + const int thread_k_blocks, // same for k dimension (reduction) + const bool m_block_size_8, // whether m_block_size == 8 + // only works when thread_m_blocks == 1 + const int stages, // number of stages for the async global->shared + // fetch pipeline + const int group_blocks, // number of consecutive 16x16 blocks + // with a separate quantization scale + const bool is_zp_float // is zero point of float16 type? + > +__global__ void Marlin(MARLIN_KERNEL_PARAMS); + +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/marlin.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/marlin.cuh new file mode 100644 index 00000000..99903968 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/marlin.cuh @@ -0,0 +1,96 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#ifndef MARLIN_NAMESPACE_NAME +#define MARLIN_NAMESPACE_NAME marlin +#endif + +namespace MARLIN_NAMESPACE_NAME { +// Marlin params + +// 8 warps are a good choice since every SM has 4 schedulers and having more +// than 1 warp per schedule allows some more latency hiding. At the same time, +// we want relatively few warps to have many registers per warp and small tiles. +static constexpr int default_threads = 256; + +static constexpr int pipe_stages = 4; // 4 pipeline stages fit into shared memory + +static constexpr int min_thread_n = 64; +static constexpr int min_thread_k = 64; +static constexpr int max_thread_n = 256; + +static constexpr int tile_size = 16; +static constexpr int max_par = 16; + +// Repack params +static constexpr int repack_stages = 8; + +static constexpr int repack_threads = 256; + +static constexpr int tile_k_size = tile_size; +static constexpr int tile_n_size = tile_k_size * 4; + +// Helpers +template +struct Vec { + T elems[n]; + __device__ T& operator[](int i) { + return elems[i]; + } +}; + +using I4 = Vec; + +constexpr int div_ceil(int a, int b) { + return (a + b - 1) / b; +} + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800 +// No support for async +#else + +__device__ inline void cp_async4_pred(void* smem_ptr, const void* glob_ptr, bool pred = true) { + const int BYTES = 16; + uint32_t smem = static_cast(__cvta_generic_to_shared(smem_ptr)); + asm volatile( + "{\n" + " .reg .pred p;\n" + " setp.ne.b32 p, %0, 0;\n" + " @p cp.async.cg.shared.global [%1], [%2], %3;\n" + "}\n" ::"r"((int)pred), + "r"(smem), + "l"(glob_ptr), + "n"(BYTES)); +} + +__device__ inline void cp_async4(void* smem_ptr, const void* glob_ptr) { + const int BYTES = 16; + uint32_t smem = static_cast(__cvta_generic_to_shared(smem_ptr)); + asm volatile( + "{\n" + " cp.async.cg.shared.global [%0], [%1], %2;\n" + "}\n" ::"r"(smem), + "l"(glob_ptr), + "n"(BYTES)); +} + +__device__ inline void cp_async_fence() { + asm volatile("cp.async.commit_group;\n" ::); +} + +template +__device__ inline void cp_async_wait() { + asm volatile("cp.async.wait_group %0;\n" ::"n"(n)); +} + +#endif + +} // namespace MARLIN_NAMESPACE_NAME diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/marlin_dtypes.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/marlin_dtypes.cuh new file mode 100644 index 00000000..f1b821f1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/marlin_dtypes.cuh @@ -0,0 +1,82 @@ +#ifndef _data_types_cuh +#define _data_types_cuh +#include +#include + +#include "marlin.cuh" + +#ifndef MARLIN_NAMESPACE_NAME +#define MARLIN_NAMESPACE_NAME marlin +#endif + +namespace MARLIN_NAMESPACE_NAME { + +template +class ScalarType {}; + +template <> +class ScalarType { + public: + using scalar_t = half; + using scalar_t2 = half2; + + // Matrix fragments for tensor core instructions; their precise layout is + // documented here: + // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type + using FragA = Vec; + using FragB = Vec; + using FragC = Vec; + using FragS = Vec; + using FragZP = Vec; + + static __device__ float inline num2float(const half x) { + return __half2float(x); + } + + static __device__ half2 inline num2num2(const half x) { + return __half2half2(x); + } + + static __device__ half2 inline nums2num2(const half x1, const half x2) { + return __halves2half2(x1, x2); + } + + static __host__ __device__ half inline float2num(const float x) { + return __float2half(x); + } +}; + +template <> +class ScalarType { + public: + using scalar_t = nv_bfloat16; + using scalar_t2 = nv_bfloat162; + + using FragA = Vec; + using FragB = Vec; + using FragC = Vec; + using FragS = Vec; + using FragZP = Vec; + +#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800 + static __device__ float inline num2float(const nv_bfloat16 x) { + return __bfloat162float(x); + } + + static __device__ nv_bfloat162 inline num2num2(const nv_bfloat16 x) { + return __bfloat162bfloat162(x); + } + + static __device__ nv_bfloat162 inline nums2num2(const nv_bfloat16 x1, const nv_bfloat16 x2) { + return __halves2bfloat162(x1, x2); + } + + static __host__ __device__ nv_bfloat16 inline float2num(const float x) { + return __float2bfloat16(x); + } +#endif +}; + +} // namespace MARLIN_NAMESPACE_NAME + +#endif diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/marlin_template.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/marlin_template.h new file mode 100644 index 00000000..01eb3387 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/marlin/marlin_template.h @@ -0,0 +1,1629 @@ +/* + * Modified by Neural Magic + * Copyright (C) Marlin.2024 Elias Frantar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Adapted from https://github.com/IST-DASLab/marlin + */ +#ifndef MARLIN_NAMESPACE_NAME +#define MARLIN_NAMESPACE_NAME marlin +#endif + +#include "dequant.h" +#include "marlin.cuh" +#include "marlin_dtypes.cuh" +#include "scalar_type.hpp" + +#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \ + static_assert( \ + std::is_same::value || std::is_same::value, \ + "only float16 and bfloat16 is supported"); + +namespace MARLIN_NAMESPACE_NAME { + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800 + +template < + typename scalar_t, // compute dtype, half or nv_float16 + const sglang::ScalarTypeId w_type_id, // weight ScalarType id + const int threads, // number of threads in a threadblock + const int thread_m_blocks, // number of 16x16 blocks in the m + // dimension (batchsize) of the + // threadblock + const int thread_n_blocks, // same for n dimension (output) + const int thread_k_blocks, // same for k dimension (reduction) + const bool m_block_size_8, // whether m_block_size == 8 + // only works when thread_m_blocks == 1 + const int stages, // number of stages for the async global->shared + // fetch pipeline + const bool has_act_order, // whether act_order is enabled + const int group_blocks, // number of consecutive 16x16 blocks + // with a separate quantization scale + const bool is_zp_float // is zero point of float16 type? + > +__global__ void Marlin( + const int4* __restrict__ A, // fp16 input matrix of shape mxk + const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn + int4* __restrict__ C, // fp16 output buffer of shape mxn + int4* __restrict__ C_tmp, // fp32 tmp output buffer (for reduce) + const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape + // (k/groupsize)xn + const int* __restrict__ g_idx, // int32 group indices of shape k + int num_groups, // number of scale groups per output channel + int prob_m, // batch dimension m + int prob_n, // output dimension n + int prob_k, // reduction dimension k + int* locks, // extra global storage for barrier synchronization + bool use_fp32_reduce // whether to use fp32 global reduce +) {} + +} // namespace marlin + +#else + +// m16n8k16 tensor core mma instruction with fp16 inputs and fp32 +// output/accumulation. +template +__device__ inline void +mma(const typename ScalarType::FragA& a_frag, + const typename ScalarType::FragB& frag_b, + typename ScalarType::FragC& frag_c) { + const uint32_t* a = reinterpret_cast(&a_frag); + const uint32_t* b = reinterpret_cast(&frag_b); + float* c = reinterpret_cast(&frag_c); + if constexpr (std::is_same::value) { + asm volatile( + "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 " + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3]) + : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3])); + } else if constexpr (std::is_same::value) { + asm volatile( + "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3]) + : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3])); + } else { + STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t); + } +} + +template +__device__ inline void mma_trans( + const typename ScalarType::FragA& a_frag, + const typename ScalarType::FragB& frag_b, + const typename ScalarType::FragB& frag_b2, + typename ScalarType::FragC& frag_c) { + const uint32_t* a = reinterpret_cast(&a_frag); + const uint32_t* b = reinterpret_cast(&frag_b); + const uint32_t* b2 = reinterpret_cast(&frag_b2); + float* c = reinterpret_cast(&frag_c); + if constexpr (std::is_same::value) { + asm volatile( + "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 " + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3]) + : "r"(b[0]), + "r"(b2[0]), + "r"(b[1]), + "r"(b2[1]), + "r"(a[0]), + "r"(a[1]), + "f"(c[0]), + "f"(c[1]), + "f"(c[2]), + "f"(c[3])); + } else if constexpr (std::is_same::value) { + asm volatile( + "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3]) + : "r"(b[0]), + "r"(b2[0]), + "r"(b[1]), + "r"(b2[1]), + "r"(a[0]), + "r"(a[1]), + "f"(c[0]), + "f"(c[1]), + "f"(c[2]), + "f"(c[3])); + } else { + STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t); + } +} + +// Instruction for loading a full 16x16 matrix fragment of operand A from shared +// memory, directly in tensor core layout. +template +__device__ inline void ldsm(typename ScalarType::FragA& frag_a, const void* smem_ptr) { + uint32_t* a = reinterpret_cast(&frag_a); + uint32_t smem = static_cast(__cvta_generic_to_shared(smem_ptr)); + if constexpr (count == 4) { + asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n" + : "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3]) + : "r"(smem)); + } else if constexpr (count == 2) { + asm volatile("ldmatrix.sync.aligned.m8n8.x2.shared.b16 {%0,%1}, [%2];\n" : "=r"(a[0]), "=r"(a[1]) : "r"(smem)); + } else if constexpr (count == 1) { + asm volatile("ldmatrix.sync.aligned.m8n8.x1.shared.b16 {%0}, [%1];\n" : "=r"(a[0]) : "r"(smem)); + } else { + static_assert(count == 1 || count == 2 || count == 4, "invalid count"); + } +} + +// Multiply dequantized values by the corresponding quantization scale; used +// only for grouped quantization. +template +__device__ inline void +scale(typename ScalarType::FragB& frag_b, typename ScalarType::FragS& frag_s, int i) { + using scalar_t2 = typename ScalarType::scalar_t2; + scalar_t2 s = ScalarType::num2num2(reinterpret_cast(&frag_s)[i]); + frag_b[0] = __hmul2(frag_b[0], s); + frag_b[1] = __hmul2(frag_b[1], s); +} + +template +__device__ inline void scale_and_sub(typename ScalarType::FragB& frag_b, scalar_t s, scalar_t zp) { + using scalar_t2 = typename ScalarType::scalar_t2; + scalar_t2 s2 = ScalarType::num2num2(s); + scalar_t2 zp2 = ScalarType::num2num2(zp); + frag_b[0] = __hfma2(frag_b[0], s2, __hneg2(zp2)); + frag_b[1] = __hfma2(frag_b[1], s2, __hneg2(zp2)); +} + +template +__device__ inline void +sub_zp(typename ScalarType::FragB& frag_b, typename ScalarType::scalar_t2& frag_zp, int i) { + using scalar_t2 = typename ScalarType::scalar_t2; + scalar_t2 zp = ScalarType::num2num2(reinterpret_cast(&frag_zp)[i]); + frag_b[0] = __hsub2(frag_b[0], zp); + frag_b[1] = __hsub2(frag_b[1], zp); +} + +// Same as above, but for act_order (each K is multiplied individually) +template +__device__ inline void scale4( + typename ScalarType::FragB& frag_b, + typename ScalarType::FragS& frag_s_1, + typename ScalarType::FragS& frag_s_2, + typename ScalarType::FragS& frag_s_3, + typename ScalarType::FragS& frag_s_4, + int i) { + using scalar_t2 = typename ScalarType::scalar_t2; + scalar_t2 s_val_1_2; + s_val_1_2.x = reinterpret_cast(&frag_s_1)[i]; + s_val_1_2.y = reinterpret_cast(&frag_s_2)[i]; + + scalar_t2 s_val_3_4; + s_val_3_4.x = reinterpret_cast(&frag_s_3)[i]; + s_val_3_4.y = reinterpret_cast(&frag_s_4)[i]; + + frag_b[0] = __hmul2(frag_b[0], s_val_1_2); + frag_b[1] = __hmul2(frag_b[1], s_val_3_4); +} + +// Given 2 floats multiply by 2 scales (halves) +template +__device__ inline void scale_float(float* c, typename ScalarType::FragS& s) { + scalar_t* s_ptr = reinterpret_cast(&s); + c[0] = __fmul_rn(c[0], ScalarType::num2float(s_ptr[0])); + c[1] = __fmul_rn(c[1], ScalarType::num2float(s_ptr[1])); +} + +// Wait until barrier reaches `count`, then lock for current threadblock. +__device__ inline void barrier_acquire(int* lock, int count) { + if (threadIdx.x == 0) { + int state = -1; + do + // Guarantee that subsequent writes by this threadblock will be visible + // globally. + asm volatile("ld.global.acquire.gpu.b32 %0, [%1];\n" : "=r"(state) : "l"(lock)); + while (state != count); + } + __syncthreads(); +} + +// Release barrier and increment visitation count. +__device__ inline void barrier_release(int* lock, bool reset = false) { + __syncthreads(); + if (threadIdx.x == 0) { + if (reset) { + lock[0] = 0; + return; + } + int val = 1; + // Make sure that all writes since acquiring this barrier are visible + // globally, while releasing the barrier. + asm volatile("fence.acq_rel.gpu;\n"); + asm volatile("red.relaxed.gpu.global.add.s32 [%0], %1;\n" : : "l"(lock), "r"(val)); + } +} + +// Wait until value of lock to be negative, and then add 1 +__device__ inline void wait_negative_and_add(int* lock) { + if (threadIdx.x == 0) { + int state = 0; + do + // Guarantee that subsequent writes by this threadblock will be visible + // globally. + asm volatile("ld.global.acquire.gpu.b32 %0, [%1];\n" : "=r"(state) : "l"(lock)); + while (state >= 0); + atomicAdd(lock, 1); + } + __syncthreads(); +} + +template < + typename scalar_t, // compute dtype, half or nv_float16 + const sglang::ScalarTypeId w_type_id, // weight ScalarType id + const int threads, // number of threads in a threadblock + const int thread_m_blocks, // number of 16x16 blocks in the m + // dimension (batchsize) of the + // threadblock + const int thread_n_blocks, // same for n dimension (output) + const int thread_k_blocks, // same for k dimension (reduction) + const bool m_block_size_8, // whether m_block_size == 8 + // only works when thread_m_blocks == 1 + const int stages, // number of stages for the async global->shared + // fetch pipeline + const int group_blocks, // number of consecutive 16x16 blocks + // with a separate quantization scale + const bool is_zp_float // is zero point of float16 type? + > +__global__ void Marlin( + const int4* __restrict__ A, // fp16 input matrix of shape mxk + const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn + int4* __restrict__ C, // fp16 output buffer of shape mxn + int4* __restrict__ C_tmp, // fp32 tmp output buffer (for reduce) + const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape + // (k/groupsize)xn + const uint16_t* __restrict__ scale2_ptr, // fp16 global scale (for nvfp4 + // only) + const int4* __restrict__ zp_ptr, // 4bit packed zero-points of shape + // (k/groupsize)x(n/pack_factor) + const int* __restrict__ g_idx, // int32 group indices of shape k + int num_groups, // number of scale groups per output channel + int prob_m, // batch dimension m + int prob_n, // output dimension n + int prob_k, // reduction dimension k + int lda, // A.stride(0), equal to prob_k is A is contiguous + int* locks, // extra global storage for barrier synchronization + bool use_atomic_add, // whether to use atomic add to reduce + bool use_fp32_reduce, // whether to use fp32 global reduce + int max_shared_mem) { + // Each threadblock processes one "stripe" of the B matrix with (roughly) the + // same size, which might involve multiple column "slices" (of width 16 * + // `thread_n_blocks`). Stripes are defined as shown in the 3x3 matrix 5 SM + // example: + // 0 1 3 + // 0 2 3 + // 1 2 4 + // While this kind of partitioning makes things somewhat more complicated, it + // ensures good utilization of all SMs for many kinds of shape and GPU + // configurations, while requiring as few slow global cross-threadblock + // reductions as possible. + using Dtype = ScalarType; + using scalar_t2 = typename ScalarType::scalar_t2; + using FragA = typename ScalarType::FragA; + using FragB = typename ScalarType::FragB; + using FragC = typename ScalarType::FragC; + using FragS = typename ScalarType::FragS; + using FragZP = typename ScalarType::FragZP; + + static constexpr auto w_type = sglang::ScalarType::from_id(w_type_id); + constexpr bool has_zp = w_type == sglang::kU4 || w_type == sglang::kU8; + constexpr bool is_int_type = + w_type == sglang::kU4 || w_type == sglang::kU8 || w_type == sglang::kU4B8 || w_type == sglang::kU8B128; + // see comments of dequant.h for more details + constexpr bool dequant_skip_flop = !is_int_type || + has_zp && !is_zp_float && !std::is_same::value || + has_zp && !is_zp_float && !(w_type == sglang::kU8); + + scalar_t2 global_scale; + + if constexpr (w_type == sglang::kFE2M1f) { + uint16_t val = scale2_ptr[0]; + global_scale = Dtype::num2num2(*reinterpret_cast(&val)); + } + + constexpr bool has_act_order = group_blocks == 0; + constexpr int m_block_size = m_block_size_8 ? 8 : (16 * thread_m_blocks); + + constexpr int pack_factor = 32 / w_type.size_bits(); + static_assert(thread_m_blocks == 1 || !m_block_size_8); + + // For larger GEMMs we run multiple batchsize 64 versions in parallel for a + // better partitioning with less reductions + int parallel = 1; + if (prob_m > m_block_size) { + parallel = prob_m / m_block_size; + prob_m = m_block_size; + } + + int k_tiles = prob_k / 16 / thread_k_blocks; + int n_tiles = prob_n / 16 / thread_n_blocks; + int iters = div_ceil(k_tiles * n_tiles * parallel, gridDim.x); + + if constexpr (!has_act_order && group_blocks != -1) { + if (group_blocks >= thread_k_blocks) { + // Ensure that the number of tiles in each stripe is a multiple of the + // groupsize; this avoids an annoying special case where a stripe starts + // in the middle of group. + iters = (group_blocks / thread_k_blocks) * div_ceil(iters, (group_blocks / thread_k_blocks)); + } + } + + int slice_row = (iters * blockIdx.x) % k_tiles; + int slice_col_par = (iters * blockIdx.x) / k_tiles; + int slice_col = slice_col_par; + int slice_iters; // number of threadblock tiles in the current slice + int slice_count = 0; // total number of active threadblocks in the current slice + int slice_idx; // index of threadblock in current slice; numbered bottom to + // top + + int par_id = 0; + int locks_off = 0; + + // We can easily implement parallel problem execution by just remapping + // indices and advancing global pointers + if (slice_col_par >= n_tiles) { + A += (slice_col_par / n_tiles) * 16 * thread_m_blocks * lda / 8; + C += (slice_col_par / n_tiles) * 16 * thread_m_blocks * prob_n / 8; + slice_col = slice_col_par % n_tiles; + par_id = slice_col_par / n_tiles; + } + if (parallel * n_tiles >= gridDim.x) { + // when parallel * n_tiles >= sms + // then there are at most $sms$ conflict tile blocks + locks_off = blockIdx.x; + } else { + locks_off = (iters * blockIdx.x) / k_tiles - 1; + } + + // Compute all information about the current slice which is required for + // synchronization. + auto init_slice = [&](bool first_init = false) { + slice_iters = iters * (blockIdx.x + 1) - (k_tiles * slice_col_par + slice_row); + if (slice_iters < 0 || slice_col_par >= n_tiles * parallel) slice_iters = 0; + if (slice_iters == 0) return; + if (slice_row + slice_iters > k_tiles) slice_iters = k_tiles - slice_row; + slice_count = 1; + slice_idx = 0; + int col_first = iters * div_ceil(k_tiles * slice_col_par, iters); + if (col_first <= k_tiles * (slice_col_par + 1)) { + int col_off = col_first - k_tiles * slice_col_par; + slice_count = div_ceil(k_tiles - col_off, iters); + if (col_off > 0) slice_count++; + int delta_first = iters * blockIdx.x - col_first; + if (delta_first < 0 || (col_off == 0 && delta_first == 0)) + slice_idx = slice_count - 1; + else { + slice_idx = slice_count - 1 - delta_first / iters; + if (col_off > 0) slice_idx--; + } + } + if (parallel * n_tiles >= gridDim.x) { + if (slice_count > 1 && slice_idx == slice_count - 1) { + locks_off++; + } + } else { + locks_off++; + } + + if (first_init && use_atomic_add && slice_count > 1 && slice_idx == 0) { + constexpr int threads_per_m = 16 * thread_n_blocks / 8; + int m_per_thread = div_ceil(thread_m_blocks * 16, threads / threads_per_m); + if (m_block_size_8) m_per_thread = div_ceil(8, threads / threads_per_m); + for (int i = 0; i < m_per_thread; i++) { + int row = threads / threads_per_m * i + threadIdx.x / threads_per_m; + if (row < prob_m) { + int col = slice_col * 16 * thread_n_blocks / 8 + threadIdx.x % threads_per_m; + C[row * prob_n / 8 + col] = {0, 0, 0, 0}; + } + } + // After write zero to output, write a negative value to lock. + // Every SM that processes the same slice would wait for + // the negative value, and then atomicAdd 1 to it. + // After all SMs are processed, the lock value would back to 0 again. + __syncthreads(); + if (threadIdx.x == 0) locks[locks_off] = 1 - slice_count; + } + + if (slice_col == n_tiles) { + A += 16 * thread_m_blocks * lda / 8; + C += 16 * thread_m_blocks * prob_n / 8; + slice_col = 0; + par_id++; + } + }; + init_slice(true); + + // A sizes/strides + + // stride of the A matrix in global memory + int a_gl_stride = lda / 8; + // stride of an A matrix tile in shared memory + constexpr int a_sh_stride = 16 * thread_k_blocks / 8; + // delta between subsequent A tiles in global memory + constexpr int a_gl_rd_delta_o = 16 * thread_k_blocks / 8; + // between subsequent accesses within a tile + int a_gl_rd_delta_i = a_gl_stride * (threads / a_gl_rd_delta_o); + // between shared memory writes + constexpr int a_sh_wr_delta = a_sh_stride * (threads / a_gl_rd_delta_o); + // between shared memory tile reads + constexpr int a_sh_rd_delta_o = 2 * ((threads / 32) / (thread_n_blocks / 4)); + // within a shared memory tile + constexpr int a_sh_rd_delta_i = a_sh_stride * 16; + // overall size of a tile + constexpr int a_sh_stage = a_sh_stride * m_block_size; + // number of shared write iterations for a tile + constexpr int a_sh_wr_iters = div_ceil(a_sh_stage, a_sh_wr_delta); + + // B sizes/strides + int b_gl_stride = 16 * prob_n / (pack_factor * 4); + constexpr int b_sh_stride = ((thread_n_blocks * 16) * 16 / pack_factor) / 4; + constexpr int b_thread_vecs = w_type.size_bits() == 4 ? 1 : 2; + constexpr int b_sh_stride_threads = b_sh_stride / b_thread_vecs; + + int b_gl_rd_delta_o = b_gl_stride * thread_k_blocks; + int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride_threads); + constexpr int b_sh_wr_delta = threads * b_thread_vecs; + constexpr int b_sh_rd_delta = threads * b_thread_vecs; + constexpr int b_sh_stage = b_sh_stride * thread_k_blocks; + constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta; + + // Scale sizes/strides without act_order + int s_gl_stride = prob_n / 8; + constexpr int s_sh_stride = 16 * thread_n_blocks / 8; + constexpr int s_tb_groups = !has_act_order && group_blocks != -1 && group_blocks < thread_k_blocks + ? thread_k_blocks / group_blocks / (w_type == sglang::kFE2M1f ? 2 : 1) + : 1; + constexpr int s_sh_stage = s_tb_groups * s_sh_stride; + int s_gl_rd_delta = s_gl_stride; + + // Scale size/strides with act_order + constexpr int tb_k = 16 * thread_k_blocks; + constexpr int g_idx_stage = has_act_order ? (tb_k * sizeof(int)) / 16 : 0; + // constexpr int act_s_row_stride = 1; + // int act_s_col_stride = act_s_row_stride * num_groups; + constexpr int act_s_max_num_groups = 32; + int act_s_col_stride = 1; + int act_s_col_warp_stride = act_s_col_stride * 8; + + int tb_n_warps = thread_n_blocks / 4; + int act_s_col_tb_stride = act_s_col_warp_stride * tb_n_warps; + + // Zero-points sizes/strides + int zp_gl_stride = is_zp_float ? prob_n / 8 : (prob_n / pack_factor) / 4; + constexpr int zp_sh_stride = is_zp_float ? 16 * thread_n_blocks / 8 : ((16 * thread_n_blocks) / pack_factor) / 4; + constexpr int zp_tb_groups = s_tb_groups; + constexpr int zp_sh_stage = has_zp ? zp_tb_groups * zp_sh_stride : 0; + int zp_gl_rd_delta = zp_gl_stride; + + // Global A read index of current thread. + int a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) + (threadIdx.x % a_gl_rd_delta_o); + a_gl_rd += a_gl_rd_delta_o * slice_row; + // Shared write index of current thread. + int a_sh_wr = a_sh_stride * (threadIdx.x / a_gl_rd_delta_o) + (threadIdx.x % a_gl_rd_delta_o); + // Shared read index. + int a_sh_rd = a_sh_stride * ((threadIdx.x % 32) % (16 / (m_block_size_8 ? 2 : 1))) + + (threadIdx.x % 32) / (16 / (m_block_size_8 ? 2 : 1)); + a_sh_rd += 2 * ((threadIdx.x / 32) / (thread_n_blocks / 4)); + + int b_gl_rd = b_gl_stride * (threadIdx.x / b_sh_stride_threads) + (threadIdx.x % b_sh_stride_threads) * b_thread_vecs; + b_gl_rd += b_sh_stride * slice_col; + b_gl_rd += b_gl_rd_delta_o * slice_row; + auto b_sh_wr = threadIdx.x * b_thread_vecs; + auto b_sh_rd = threadIdx.x * b_thread_vecs; + + // For act_order + constexpr int k_iter_size = tb_k / b_sh_wr_iters; + int slice_k_start = tb_k * slice_row; + int slice_k_finish = slice_k_start + tb_k * slice_iters; + int slice_k_start_shared_fetch = slice_k_start; + int slice_n_offset = act_s_col_tb_stride * slice_col; + + // No act_order + int s_gl_rd; + if constexpr (!has_act_order) { + if constexpr (group_blocks == -1) { + s_gl_rd = s_sh_stride * slice_col + threadIdx.x; + } else { + s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) / (w_type == sglang::kFE2M1f ? 2 : 1) + + s_sh_stride * slice_col + threadIdx.x; + } + } + auto s_sh_wr = threadIdx.x; + bool s_sh_wr_pred = threadIdx.x < s_sh_stride; + + // Zero-points + int zp_gl_rd; + if constexpr (has_zp) { + if constexpr (group_blocks == -1) { + zp_gl_rd = zp_sh_stride * slice_col + threadIdx.x; + } else { + zp_gl_rd = zp_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) + zp_sh_stride * slice_col + threadIdx.x; + } + } + auto zp_sh_wr = threadIdx.x; + bool zp_sh_wr_pred = threadIdx.x < zp_sh_stride; + + // We use a different scale layout for grouped and column-wise quantization as + // we scale a `half2` tile in column-major layout in the former and in + // row-major in the latter case. + int s_sh_rd; + if constexpr (group_blocks != -1 && w_type == sglang::kFE2M1f) { + auto warp_id = threadIdx.x / 32; + int n_warps = thread_n_blocks / 4; + int warp_row = warp_id / n_warps; + + s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + (threadIdx.x % 32) / 4; + s_sh_rd = s_sh_rd * 2 + warp_row % 2; + + } else if constexpr (group_blocks != -1) + s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + (threadIdx.x % 32) / 4; + else if constexpr (group_blocks == -1 && (m_block_size_8 || (has_zp && !dequant_skip_flop))) + s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + (threadIdx.x % 32) / 8; + else + s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + (threadIdx.x % 32) % 4; + + // Zero-points have the same read layout as the scales + // (without column-wise case) + constexpr int num_col_threads = 8; + constexpr int num_row_threads = 4; + constexpr int num_ints_per_thread = 8 / pack_factor; + int zp_sh_rd; + if constexpr (has_zp) { + if constexpr (is_zp_float) { + if constexpr (group_blocks != -1) { + zp_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + (threadIdx.x % 32) / 4; + } + } else { + zp_sh_rd = num_ints_per_thread * num_col_threads * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + + num_ints_per_thread * ((threadIdx.x % 32) / num_row_threads); + } + } + + // Precompute which thread should not read memory in which iterations; this is + // needed if there are more threads than required for a certain tilesize or + // when the batchsize is not a multiple of 16. + bool a_sh_wr_pred[a_sh_wr_iters]; +#pragma unroll + for (int i = 0; i < a_sh_wr_iters; i++) + a_sh_wr_pred[i] = a_sh_wr_delta * i + a_sh_wr < a_sh_stride * prob_m; + + // To ensure that writing and reading A tiles to/from shared memory, the + // latter in fragment format, is fully bank conflict free, we need to use a + // rather fancy XOR-based layout. The key here is that neither reads nor + // writes of the 16-byte `int4` blocks of 8 consecutive threads involve the + // same shared memory banks. Further, it seems (based on NSight-Compute) that + // each warp must also write a consecutive memory segment? + auto transform_a = [&](int i) { + int row = i / a_gl_rd_delta_o; + return a_gl_rd_delta_o * row + (i % a_gl_rd_delta_o) ^ (row % 8); + }; + // Since the computation of this remapping is non-trivial and, due to our main + // loop unrolls, all shared memory accesses are static, we simply precompute + // both transformed reads and writes. + int a_sh_wr_trans[a_sh_wr_iters]; +#pragma unroll + for (int i = 0; i < a_sh_wr_iters; i++) + a_sh_wr_trans[i] = transform_a(a_sh_wr_delta * i + a_sh_wr); + int a_sh_rd_trans[b_sh_wr_iters][thread_m_blocks]; +#pragma unroll + for (int i = 0; i < b_sh_wr_iters; i++) { +#pragma unroll + for (int j = 0; j < thread_m_blocks; j++) + a_sh_rd_trans[i][j] = transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd); + } + + // Since B-accesses have non-constant stride they have to be computed at + // runtime; we break dependencies between subsequent accesses with a tile by + // maintining multiple pointers (we have enough registers), a tiny + // optimization. + const int4* B_ptr[b_sh_wr_iters]; +#pragma unroll + for (int i = 0; i < b_sh_wr_iters; i++) + B_ptr[i] = B + b_gl_rd_delta_i * i + b_gl_rd; + + extern __shared__ int4 sh[]; + // Shared memory storage for global fetch pipelines. + constexpr int sh_red_size = (2 * thread_n_blocks + 1) * 16 * thread_m_blocks; + constexpr int sh_b_size = stages * b_sh_stage; + int4* sh_b = sh; + int4* sh_red = sh; + int4* sh_g_idx = sh_b + (sh_red_size > sh_b_size ? sh_red_size : sh_b_size); + int4* sh_zp = sh_g_idx + (stages * g_idx_stage); + constexpr int sh_s_size = has_act_order ? (act_s_max_num_groups * s_sh_stride) : (stages * s_sh_stage); + int4* sh_s = sh_zp + (stages * zp_sh_stage); + // shared memory reused by reduction should be smaller than + // shared memory used by weight. + static_assert(thread_m_blocks * 16 * thread_n_blocks * 16 / 8 <= stages * b_sh_stage); + int4* sh_a = sh_s + sh_s_size; + // constexpr int shm_size_used = + // stages * (g_idx_stage + zp_sh_stage) + sh_s_size + + // (sh_red_size > sh_b_size ? sh_red_size : sh_b_size); + + // Register storage for double buffer of shared memory reads. + FragA frag_a[2][thread_m_blocks]; + I4 frag_b_quant[2][b_thread_vecs]; + FragC frag_c[thread_m_blocks][4][2]; + FragS frag_s[2][4]; // No act-order + FragS act_frag_s[2][4][4]; // For act-order + int frag_qzp[2][num_ints_per_thread]; // Zero-points + FragZP frag_zp; // Zero-points in fp16 + FragZP frag_zpf[2]; // Zero-points in fp16 in HQQ + + // Zero accumulators. + auto zero_accums = [&]() { +#pragma unroll + for (int i = 0; i < thread_m_blocks * 4 * 2 * 4; i++) + reinterpret_cast(frag_c)[i] = 0; + }; + + int sh_first_group_id = -1; + int sh_num_groups = -1; + + auto fetch_act_order_scales_to_shared = [&](bool is_async, int first_group_id, int last_group_id) { + sh_first_group_id = first_group_id; + sh_num_groups = last_group_id - first_group_id + 1; + + if (sh_num_groups > act_s_max_num_groups) { + sh_num_groups = act_s_max_num_groups; + } + + if (sh_first_group_id + sh_num_groups > num_groups) { + sh_num_groups = num_groups - sh_first_group_id; + } + + int row_offset = first_group_id * s_gl_stride; + + if (is_async) { + for (int i = 0; i < sh_num_groups; i++) { + if (threadIdx.x < s_sh_stride) { + cp_async4_pred( + &sh_s[(i * s_sh_stride) + threadIdx.x], + &scales_ptr[row_offset + (i * s_gl_stride) + slice_n_offset + threadIdx.x]); + } + } + } else { + for (int i = 0; i < sh_num_groups; i++) { + if (threadIdx.x < s_sh_stride) { + sh_s[(i * s_sh_stride) + threadIdx.x] = + scales_ptr[row_offset + (i * s_gl_stride) + slice_n_offset + threadIdx.x]; + } + } + } + }; + // Asynchronously fetch the next A, B and s tile from global to the next + // shared memory pipeline location. + auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true) { + if (pred) { + int4* sh_a_stage = sh_a + a_sh_stage * pipe; +#pragma unroll + for (int i = 0; i < a_sh_wr_iters; i++) { + cp_async4_pred( + &sh_a_stage[a_sh_wr_trans[i]], + &A[a_gl_rd_delta_i * i + a_gl_rd + a_gl_rd_delta_o * a_off], + a_sh_wr_pred[i]); + } + int4* sh_b_stage = sh_b + b_sh_stage * pipe; +#pragma unroll + for (int i = 0; i < b_sh_wr_iters; i++) { +#pragma unroll + for (int j = 0; j < b_thread_vecs; j++) { + cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr + j], B_ptr[i] + j); + } + + B_ptr[i] += b_gl_rd_delta_o; + } + + if constexpr (has_act_order) { + // Fetch g_idx thread-block portion + int full_pipe = a_off; + int cur_k = slice_k_start_shared_fetch + tb_k * full_pipe; + if (cur_k < prob_k && cur_k < slice_k_finish) { + int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe; + + int4 const* cur_g_idx_stage_ptr = reinterpret_cast(&g_idx[cur_k]); + + if (threadIdx.x < g_idx_stage) { + cp_async4_pred(&sh_g_idx_stage[threadIdx.x], &cur_g_idx_stage_ptr[threadIdx.x]); + } + } + } else { + if constexpr (group_blocks != -1) { + int4* sh_s_stage = sh_s + s_sh_stage * pipe; + + if constexpr (group_blocks >= thread_k_blocks) { + // Only fetch scales if this tile starts a new group + if (pipe % (group_blocks / thread_k_blocks) == 0) { + if (s_sh_wr_pred) { + cp_async4(&sh_s_stage[s_sh_wr], &scales_ptr[s_gl_rd]); + } + s_gl_rd += s_gl_rd_delta; + } + } else { + for (int i = 0; i < s_tb_groups; i++) { + if (s_sh_wr_pred) { + cp_async4(&sh_s_stage[i * s_sh_stride + s_sh_wr], &scales_ptr[s_gl_rd]); + } + s_gl_rd += s_gl_rd_delta; + } + } + } + + if constexpr (has_zp && group_blocks != -1) { + int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe; + + if constexpr (group_blocks >= thread_k_blocks) { + // Only fetch zero-points if this tile starts a new group + if (pipe % (group_blocks / thread_k_blocks) == 0) { + if (zp_sh_wr_pred) { + cp_async4(&sh_zp_stage[zp_sh_wr], &zp_ptr[zp_gl_rd]); + } + zp_gl_rd += zp_gl_rd_delta; + } + } else { + for (int i = 0; i < zp_tb_groups; i++) { + if (zp_sh_wr_pred) { + cp_async4(&sh_zp_stage[i * zp_sh_stride + zp_sh_wr], &zp_ptr[zp_gl_rd]); + } + zp_gl_rd += zp_gl_rd_delta; + } + } + } + } + } + // Insert a fence even when we are winding down the pipeline to ensure that + // waiting is also correct at this point. + cp_async_fence(); + }; + + auto fetch_col_zp_to_shared = [&]() { + if (zp_sh_wr_pred) { + cp_async4(&sh_zp[zp_sh_wr], &zp_ptr[zp_gl_rd]); + } + }; + + auto fetch_col_scale_to_shared = [&]() { + if (s_sh_wr_pred) { + cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]); + } + }; + + // Wait until the next thread tile has been loaded to shared memory. + auto wait_for_stage = [&]() { + // We only have `stages - 2` active fetches since we are double buffering + // and can only issue the next fetch when it is guaranteed that the previous + // shared memory load is fully complete (as it may otherwise be + // overwritten). + cp_async_wait(); + __syncthreads(); + }; + + // Load the next sub-tile from the current location in the shared memory pipe + // into the current register buffer. + auto fetch_to_registers = [&](int k, int pipe) { + int4* sh_a_stage = sh_a + a_sh_stage * pipe; +#pragma unroll + for (int i = 0; i < thread_m_blocks; i++) + ldsm(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]); + int4* sh_b_stage = sh_b + b_sh_stage * pipe; + +#pragma unroll + for (int i = 0; i < b_thread_vecs; i++) { + frag_b_quant[k % 2][i] = *reinterpret_cast(&sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd + i]); + } + }; + + bool is_same_group[stages]; + int same_group_id[stages]; + + auto init_same_group = [&](int pipe) { + if constexpr (!has_act_order) { + return; + } + + int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe; + int* sh_g_idx_int_ptr = reinterpret_cast(sh_g_idx_stage); + + int group_id_1 = sh_g_idx_int_ptr[0]; + int group_id_2 = sh_g_idx_int_ptr[tb_k - 1]; + + is_same_group[pipe] = group_id_1 == group_id_2; + same_group_id[pipe] = group_id_1; + }; + + auto fetch_scales_to_registers = [&](int k, int full_pipe) { + int pipe = full_pipe % stages; + + if constexpr (!has_act_order) { + // No act-order case + if constexpr (group_blocks == -1) { + // load only when starting a new slice + if (k == 0 && full_pipe == 0) { + reinterpret_cast(&frag_s)[0] = sh_s[s_sh_rd]; + reinterpret_cast(&frag_s)[1] = sh_s[s_sh_rd + 4]; + } + } else if constexpr (group_blocks != -1) { + if constexpr (group_blocks >= thread_k_blocks) { + if (k % b_sh_wr_iters == 0) { + int4* sh_s_stage = + sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) * (pipe / (group_blocks / thread_k_blocks))); + reinterpret_cast(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd]; + } else { + reinterpret_cast(&frag_s[1])[0] = reinterpret_cast(&frag_s[0])[0]; + } + } else { + auto warp_id = threadIdx.x / 32; + int n_warps = thread_n_blocks / 4; + + int warp_row = warp_id / n_warps; + + int cur_k = warp_row * 16; + cur_k += k_iter_size * (k % b_sh_wr_iters); + + int k_blocks = cur_k / 16; + int cur_group_id = k_blocks / (group_blocks * (w_type == sglang::kFE2M1f ? 2 : 1)); + + int4* sh_s_stage = sh_s + s_sh_stage * pipe; + + if constexpr (w_type_id != sglang::kFE2M1f.id()) { + reinterpret_cast(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd + cur_group_id * s_sh_stride]; + } else { + reinterpret_cast(&frag_s[k % 2])[0] = + reinterpret_cast(sh_s_stage)[s_sh_rd + cur_group_id * (2 * s_sh_stride)]; + } + } + } + + return; + } + + // Act-order case + + // Determine K of the "current" thread-block + int cur_k = slice_k_start + tb_k * full_pipe; + if (cur_k >= prob_k || cur_k >= slice_k_finish) { + return; + } + + // Reset (to current thread-block) since we read g_idx portion from the + // shared memory + cur_k = 0; + + // Progress to current iteration + cur_k += k_iter_size * (k % b_sh_wr_iters); + + // Determine "position" inside the thread-block (based on warp and + // thread-id) + auto warp_id = threadIdx.x / 32; + int n_warps = thread_n_blocks / 4; // Each warp processes 4 16-size tiles over N + + int warp_row = warp_id / n_warps; + int warp_col = warp_id % n_warps; + + cur_k += warp_row * 16; + + auto th_id = threadIdx.x % 32; + cur_k += (th_id % 4) * 2; // Due to tensor-core layout for fp16 B matrix + + int s_col_shift = + /*slice_n_offset +*/ (act_s_col_warp_stride * warp_col) + (th_id / 4) * act_s_col_stride; + + if (is_same_group[pipe]) { + if (k % 2 == 0) { + *(reinterpret_cast(&(act_frag_s[k % 2][0][0]))) = + sh_s[(same_group_id[pipe] - sh_first_group_id) * s_sh_stride + s_col_shift]; + } else { + *(reinterpret_cast(&(act_frag_s[k % 2][0][0]))) = + *(reinterpret_cast(&(act_frag_s[(k - 1) % 2][0][0]))); + } + + for (int i = 1; i < 4; i++) { + *(reinterpret_cast(&(act_frag_s[k % 2][i][0]))) = *(reinterpret_cast(&(act_frag_s[k % 2][0][0]))); + } + return; + } + + int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe; + int* sh_g_idx_int_ptr = reinterpret_cast(sh_g_idx_stage); + + constexpr int k_frag_offsets[4] = {0, 1, 8, 9}; // Tensor core offsets per thread + +#pragma unroll + for (int i = 0; i < 4; i++) { + int actual_k = cur_k + k_frag_offsets[i]; + + int group_id = sh_g_idx_int_ptr[actual_k]; + int rel_group_id = group_id - sh_first_group_id; + + *(reinterpret_cast(&(act_frag_s[k % 2][i][0]))) = sh_s[rel_group_id * s_sh_stride + s_col_shift]; + } + }; + + auto fetch_zp_to_registers = [&](int k, int full_pipe) { + // This code does not handle group_blocks == 0, + // which signifies act_order. + // has_zp implies AWQ, which doesn't have act_order, + static_assert(!has_zp || group_blocks != 0); + + if constexpr (has_zp && !is_zp_float) { + int pipe = full_pipe % stages; + + if constexpr (group_blocks == -1) { + // load only when starting a new slice + if (k == 0 && full_pipe == 0) { +#pragma unroll + for (int i = 0; i < num_ints_per_thread; i++) { + frag_qzp[k % 2][i] = (reinterpret_cast(sh_zp))[zp_sh_rd + i]; + } + } + + } else if constexpr (group_blocks >= thread_k_blocks) { + if (k % b_sh_wr_iters == 0) { + int4* sh_zp_stage = + sh_zp + zp_sh_stage * ((group_blocks / thread_k_blocks) * (pipe / (group_blocks / thread_k_blocks))); +#pragma unroll + for (int i = 0; i < num_ints_per_thread; i++) { + frag_qzp[k % 2][i] = (reinterpret_cast(sh_zp_stage))[zp_sh_rd + i]; + } + } + } else { + auto warp_id = threadIdx.x / 32; + int n_warps = thread_n_blocks / 4; + + int warp_row = warp_id / n_warps; + + int cur_k = warp_row * 16; + cur_k += k_iter_size * (k % b_sh_wr_iters); + + int k_blocks = cur_k / 16; + int cur_group_id = 0; + + // Suppress bogus and persistent divide-by-zero warning +#pragma nv_diagnostic push +#pragma nv_diag_suppress divide_by_zero + cur_group_id = k_blocks / group_blocks; +#pragma nv_diagnostic pop + + int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe; + + sh_zp_stage += cur_group_id * zp_sh_stride; + +#pragma unroll + for (int i = 0; i < num_ints_per_thread; i++) { + frag_qzp[k % 2][i] = (reinterpret_cast(sh_zp_stage))[zp_sh_rd + i]; + } + } + } + + else if constexpr (has_zp && is_zp_float) { + int pipe = full_pipe % stages; + + if constexpr (group_blocks != -1) { + if constexpr (group_blocks >= thread_k_blocks) { + if (k % b_sh_wr_iters == 0) { + int4* sh_zp_stage = + sh_zp + zp_sh_stage * ((group_blocks / thread_k_blocks) * (pipe / (group_blocks / thread_k_blocks))); + reinterpret_cast(&frag_zpf[k % 2])[0] = sh_zp_stage[zp_sh_rd]; + } + } else { + auto warp_id = threadIdx.x / 32; + int n_warps = thread_n_blocks / 4; + + int warp_row = warp_id / n_warps; + + int cur_k = warp_row * 16; + cur_k += k_iter_size * (k % b_sh_wr_iters); + + int k_blocks = cur_k / 16; + // Suppress bogus and persistent divide-by-zero warning +#pragma nv_diagnostic push +#pragma nv_diag_suppress divide_by_zero + int cur_group_id = k_blocks / group_blocks; +#pragma nv_diagnostic pop + + int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe; + + reinterpret_cast(&frag_zpf[k % 2])[0] = sh_zp_stage[zp_sh_rd + cur_group_id * zp_sh_stride]; + } + } + } + }; + + auto dequant_data = [&](int q, scalar_t2* frag_b_ptr) { + dequant(q, frag_b_ptr); + }; + + // Execute the actual tensor core matmul of a sub-tile. + bool is_first_matmul_in_slice = true; + auto matmul = [&](int k) { + int k2 = k % 2; + const bool is_new_zp = ((group_blocks != -1) && (group_blocks < thread_k_blocks || k == 0)) || + (group_blocks == -1 && is_first_matmul_in_slice); + if constexpr (has_zp && !is_zp_float) { + if (is_new_zp) { + if constexpr (group_blocks == -1) is_first_matmul_in_slice = false; + FragB frag_zp_0; + FragB frag_zp_1; + int zp_quant_0, zp_quant_1; + + if constexpr (w_type.size_bits() == 4) { + zp_quant_0 = frag_qzp[k2][0]; + zp_quant_1 = zp_quant_0 >> 8; + } else { + static_assert(w_type.size_bits() == 8); + zp_quant_0 = frag_qzp[k2][0]; + zp_quant_1 = frag_qzp[k2][1]; + } + + dequant_data(zp_quant_0, reinterpret_cast(&frag_zp)); + dequant_data(zp_quant_1, reinterpret_cast(&frag_zp) + 2); + } + } + if constexpr (!dequant_skip_flop && has_zp && is_zp_float) { + if (is_new_zp) { + reinterpret_cast(&frag_zp)[0] = reinterpret_cast(&frag_zpf[k2])[0]; + } + } + + if constexpr (w_type == sglang::kFE2M1f) { + int s_quant_0 = reinterpret_cast(frag_s[k2])[0]; + int s_quant_1 = reinterpret_cast(frag_s[k2])[1]; + + dequant_fp8_scales(s_quant_0, reinterpret_cast(&frag_s[k2])); + dequant_fp8_scales(s_quant_1, reinterpret_cast(&frag_s[k2]) + 2); + } + +// We have the m dimension as the inner loop in order to encourage overlapping +// dequantization and matmul operations. +#pragma unroll + for (int j = 0; j < 4; j++) { + FragB frag_b0; + FragB frag_b1; + int b_quant_0, b_quant_1; + + if constexpr (w_type_id == sglang::kFE2M1f.id()) { + b_quant_1 = frag_b_quant[k2][0][j]; + b_quant_0 = b_quant_1 << 8; + } else if constexpr (w_type.size_bits() == 4) { + b_quant_0 = frag_b_quant[k2][0][j]; + b_quant_1 = b_quant_0 >> 8; + } else { + static_assert(w_type.size_bits() == 8); + int* frag_b_quant_ptr = reinterpret_cast(frag_b_quant[k2]); + b_quant_0 = frag_b_quant_ptr[j * 2 + 0]; + b_quant_1 = frag_b_quant_ptr[j * 2 + 1]; + } + + dequant_data(b_quant_0, reinterpret_cast(&frag_b0)); + dequant_data(b_quant_1, reinterpret_cast(&frag_b1)); + + if constexpr (dequant_skip_flop && has_zp && !is_zp_float) { + sub_zp(frag_b0, frag_zp[j], 0); + sub_zp(frag_b1, frag_zp[j], 1); + } + + // Apply scale to frag_b0 + if constexpr (has_act_order) { + static_assert(group_blocks != -1); + scale4( + frag_b0, act_frag_s[k2][0][j], act_frag_s[k2][1][j], act_frag_s[k2][2][j], act_frag_s[k2][3][j], 0); + scale4( + frag_b1, act_frag_s[k2][0][j], act_frag_s[k2][1][j], act_frag_s[k2][2][j], act_frag_s[k2][3][j], 1); + } else if constexpr (!dequant_skip_flop && has_zp && !is_zp_float && group_blocks == -1) { + int idx = (threadIdx.x / 4) % 2; + scalar_t2 s2 = Dtype::nums2num2( + reinterpret_cast(&frag_s[j / 2][j % 2 * 2 + 0])[idx], + reinterpret_cast(&frag_s[j / 2][j % 2 * 2 + 1])[idx]); + if (is_new_zp) frag_zp[j] = __hmul2(frag_zp[j], s2); + scale_and_sub(frag_b0, s2.x, frag_zp[j].x); + scale_and_sub(frag_b1, s2.y, frag_zp[j].y); + } else if constexpr (!dequant_skip_flop && has_zp && group_blocks != -1) { + if (is_new_zp) frag_zp[j] = __hmul2(frag_zp[j], *reinterpret_cast(&frag_s[k2][j])); + scale_and_sub(frag_b0, frag_s[k2][j][0].x, frag_zp[j].x); + scale_and_sub(frag_b1, frag_s[k2][j][0].y, frag_zp[j].y); + } else if constexpr (group_blocks != -1) { + scale(frag_b0, frag_s[k2][j], 0); + scale(frag_b1, frag_s[k2][j], 1); + } + +#pragma unroll + for (int i = 0; i < thread_m_blocks; i++) { + if constexpr (m_block_size_8) { + mma_trans(frag_a[k2][i], frag_b0, frag_b1, frag_c[i][j][0]); + } else { + mma(frag_a[k2][i], frag_b0, frag_c[i][j][0]); + mma(frag_a[k2][i], frag_b1, frag_c[i][j][1]); + } + } + } + }; + + // Since we slice across the k dimension of a tile in order to increase the + // number of warps while keeping the n dimension of a tile reasonable, we have + // multiple warps that accumulate their partial sums of the same output + // location; which we have to reduce over in the end. We do in shared memory. + auto thread_block_reduce = [&]() { + constexpr int red_off = threads / b_sh_stride_threads / 2; + if (red_off >= 1) { + auto red_idx = threadIdx.x / b_sh_stride_threads; + constexpr int red_sh_stride = b_sh_stride_threads * 4 * 2; + constexpr int red_sh_delta = b_sh_stride_threads; + int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride_threads) + (threadIdx.x % b_sh_stride_threads); + + // Parallel logarithmic shared memory reduction. We make sure to avoid any + // unnecessary read or write iterations, e.g., for two warps we write only + // once by warp 1 and read only once by warp 0. + +#pragma unroll + for (int m_block = 0; m_block < thread_m_blocks; m_block++) { +#pragma unroll + for (int i = red_off; i > 0; i /= 2) { + if (i <= red_idx && red_idx < 2 * i) { +#pragma unroll + for (int j = 0; j < 4 * 2; j += (m_block_size_8 ? 2 : 1)) { + int red_sh_wr = red_sh_delta * j + (red_sh_rd - red_sh_stride * i); + if (i < red_off) { + float* c_rd = reinterpret_cast(&sh_red[red_sh_delta * j + red_sh_rd]); + float* c_wr = reinterpret_cast(&sh_red[red_sh_wr]); +#pragma unroll + for (int k = 0; k < 4; k++) + reinterpret_cast(frag_c)[4 * 2 * m_block + j][k] += c_rd[k] + c_wr[k]; + } + sh_red[red_sh_wr] = reinterpret_cast(&frag_c)[4 * 2 * m_block + j]; + } + } + __syncthreads(); + } + if (red_idx == 0) { +#pragma unroll + for (int i = 0; i < 4 * 2; i += (m_block_size_8 ? 2 : 1)) { + float* c_rd = reinterpret_cast(&sh_red[red_sh_delta * i + red_sh_rd]); +#pragma unroll + for (int j = 0; j < 4; j++) + reinterpret_cast(frag_c)[4 * 2 * m_block + i][j] += c_rd[j]; + } + } + __syncthreads(); + } + } + }; + + // Since multiple threadblocks may process parts of the same column slice, we + // finally have to globally reduce over the results. As the striped + // partitioning minimizes the number of such reductions and our outputs are + // usually rather small, we perform this reduction serially in L2 cache. + auto global_reduce_fp16 = [&](bool first = false, bool last = false) { + // We are very careful here to reduce directly in the output buffer to + // maximize L2 cache utilization in this step. To do this, we write out + // results in FP16 (but still reduce with FP32 compute). + constexpr int active_threads = 32 * thread_n_blocks / 4; + if (threadIdx.x < active_threads) { + int c_gl_stride = prob_n / 8; + int c_gl_wr_delta_o = 8 * c_gl_stride; + int c_gl_wr_delta_i = 4 * (active_threads / 32); + int c_gl_wr; + if constexpr (m_block_size_8) { + c_gl_wr = c_gl_stride * ((threadIdx.x % 4) * 2) + 4 * (threadIdx.x / 32) + (threadIdx.x % 32) / 8; + c_gl_wr += (2 * thread_n_blocks) * slice_col; + } else { + c_gl_wr = c_gl_stride * ((threadIdx.x % 32) / 4) + 4 * (threadIdx.x / 32) + threadIdx.x % 4; + c_gl_wr += (2 * thread_n_blocks) * slice_col; + } + constexpr int c_sh_wr_delta = active_threads; + auto c_sh_wr = threadIdx.x; + + int row = (threadIdx.x % 32) / 4; + + if (!first) { +// Interestingly, doing direct global accesses here really seems to mess up +// the compiler and lead to slowdowns, hence we also use async-copies even +// though these fetches are not actually asynchronous. +#pragma unroll + for (int i = 0; i < (m_block_size_8 ? 2 : thread_m_blocks * 4); i++) { + if constexpr (m_block_size_8) { + cp_async4_pred( + &sh_red[c_sh_wr + c_sh_wr_delta * i], + &C[c_gl_wr + i * c_gl_stride + (threadIdx.x % 8) / 4 * c_gl_wr_delta_i], + (threadIdx.x % 4) * 2 + i < prob_m); + } else { + cp_async4_pred( + &sh_red[c_sh_wr + c_sh_wr_delta * i], + &C[c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2)], + i < (thread_m_blocks - 1) * 4 || 8 * (i / 2) + row < prob_m); + } + } + cp_async_fence(); + cp_async_wait<0>(); + } + +#pragma unroll + for (int i = 0; i < (m_block_size_8 ? 2 : thread_m_blocks * 4); i++) { + bool mask = (!m_block_size_8) && (i < (thread_m_blocks - 1) * 4 || 8 * (i / 2) + row < prob_m) || + (m_block_size_8) && ((threadIdx.x % 4) * 2 + i < prob_m); + if (mask) { + if (!first) { + int4 c_red = sh_red[c_sh_wr + i * c_sh_wr_delta]; +#pragma unroll + for (int j = 0; j < 2 * 4; j++) { + int delta = 0; + if constexpr (m_block_size_8) { + delta = j % 2 == 1 ? -2 : 0; + } + reinterpret_cast(&frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4) + delta] += + Dtype::num2float(reinterpret_cast(&c_red)[j]); + } + } + if (!last) { + int4 c; +#pragma unroll + for (int j = 0; j < 2 * 4; j++) { + int delta = 0; + if constexpr (m_block_size_8) { + delta = j % 2 == 1 ? -2 : 0; + } + reinterpret_cast(&c)[j] = + Dtype::float2num(reinterpret_cast(&frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4) + delta]); + } + if constexpr (m_block_size_8) + C[c_gl_wr + i * c_gl_stride + (threadIdx.x % 8) / 4 * c_gl_wr_delta_i] = c; + else + C[c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2)] = c; + } + } + } + } + }; + + // Globally reduce over threadblocks that compute the same column block. + // We use a tmp C buffer to reduce in full fp32 precision. + auto global_reduce_fp32 = [&](bool first = false, bool last = false) { + constexpr int tb_m = thread_m_blocks * 16; + constexpr int tb_n = thread_n_blocks * 16; + + constexpr int c_size = tb_m * tb_n * sizeof(float) / 16; + + constexpr int active_threads = 32 * thread_n_blocks / 4; + bool is_th_active = threadIdx.x < active_threads; + + constexpr int num_floats = thread_m_blocks * 4 * 2 * 4; + constexpr int th_size = num_floats * sizeof(float) / 16; + + int c_cur_offset = locks_off * c_size; + + if (!is_th_active) { + return; + } + + if (!first) { + float* frag_c_ptr = reinterpret_cast(&frag_c); +#pragma unroll + for (int k = 0; k < th_size; k += (m_block_size_8 ? 2 : 1)) { + sh_red[threadIdx.x] = C_tmp[c_cur_offset + active_threads * k + threadIdx.x]; + + float* sh_c_ptr = reinterpret_cast(&sh_red[threadIdx.x]); +#pragma unroll + for (int f = 0; f < 4; f++) { + frag_c_ptr[k * 4 + f] += sh_c_ptr[f]; + } + } + } + + if (!last) { + int4* frag_c_ptr = reinterpret_cast(&frag_c); +#pragma unroll + for (int k = 0; k < th_size; k += (m_block_size_8 ? 2 : 1)) { + C_tmp[c_cur_offset + active_threads * k + threadIdx.x] = frag_c_ptr[k]; + } + } + }; + + // Write out the reduce final result in the correct layout. We only actually + // reshuffle matrix fragments in this step, the reduction above is performed + // in fragment layout. + auto write_result = [&]() { + int c_gl_stride = prob_n / 8; + constexpr int c_sh_stride = 2 * thread_n_blocks + 1; + int c_gl_wr_delta = c_gl_stride * (threads / (2 * thread_n_blocks)); + constexpr int c_sh_rd_delta = c_sh_stride * (threads / (2 * thread_n_blocks)); + + int c_gl_wr = c_gl_stride * (threadIdx.x / (2 * thread_n_blocks)) + (threadIdx.x % (2 * thread_n_blocks)); + c_gl_wr += (2 * thread_n_blocks) * slice_col; + int c_sh_wr; + if constexpr (m_block_size_8) { + c_sh_wr = (8 * c_sh_stride) * ((threadIdx.x % 32) % 4 * 2) + (threadIdx.x % 32) / 4; + c_sh_wr += 64 * (threadIdx.x / 32); + } else { + c_sh_wr = (4 * c_sh_stride) * ((threadIdx.x % 32) / 4) + (threadIdx.x % 32) % 4; + c_sh_wr += 32 * (threadIdx.x / 32); + } + + int c_sh_rd = c_sh_stride * (threadIdx.x / (2 * thread_n_blocks)) + (threadIdx.x % (2 * thread_n_blocks)); + + int c_gl_wr_end = c_gl_stride * prob_m; + // We first reorder in shared memory to guarantee the most efficient final + // global write patterns + auto write = [&](int idx, float c0, float c1, FragS& s) { + scalar_t2 res = Dtype::nums2num2(Dtype::float2num(c0), Dtype::float2num(c1)); + + // For per-column quantization we finally apply the scale here (only for + // 4-bit) + if constexpr ( + !has_act_order && group_blocks == -1 && w_type.size_bits() == 4 && (has_zp && dequant_skip_flop || !has_zp)) { + res = __hmul2(res, s[0]); + } + + if constexpr (w_type == sglang::kFE2M1f) { + res = __hmul2(res, global_scale); + } + + if constexpr (m_block_size_8) { + ((scalar_t*)sh_red)[idx] = res.x; + ((scalar_t*)sh_red)[idx + 8 * c_sh_stride] = res.y; + } else { + ((scalar_t2*)sh_red)[idx] = res; + } + }; + + if (threadIdx.x / 32 < thread_n_blocks / 4) { +#pragma unroll + for (int i = 0; i < thread_m_blocks; i++) { +#pragma unroll + for (int j = 0; j < 4; j++) { + if constexpr (m_block_size_8) { + int wr = c_sh_wr + 16 * j; + write(wr, frag_c[i][j][0][0], frag_c[i][j][0][1], frag_s[j / 2][2 * (j % 2) + 0]); + write(wr + 8, frag_c[i][j][0][2], frag_c[i][j][0][3], frag_s[j / 2][2 * (j % 2) + 1]); + } else { + int wr = c_sh_wr + 8 * j; + write( + wr + (4 * c_sh_stride) * 0 + 0, frag_c[i][j][0][0], frag_c[i][j][0][1], frag_s[j / 2][2 * (j % 2) + 0]); + write( + wr + (4 * c_sh_stride) * 8 + 0, frag_c[i][j][0][2], frag_c[i][j][0][3], frag_s[j / 2][2 * (j % 2) + 0]); + write( + wr + (4 * c_sh_stride) * 0 + 4, frag_c[i][j][1][0], frag_c[i][j][1][1], frag_s[j / 2][2 * (j % 2) + 1]); + write( + wr + (4 * c_sh_stride) * 8 + 4, frag_c[i][j][1][2], frag_c[i][j][1][3], frag_s[j / 2][2 * (j % 2) + 1]); + } + } + c_sh_wr += 16 * (4 * c_sh_stride); + } + } + __syncthreads(); + +#pragma unroll + for (int i = 0; i < div_ceil(16 * thread_m_blocks, threads / (2 * thread_n_blocks)); i++) { + if (c_gl_wr < c_gl_wr_end) { + if (use_atomic_add && slice_count > 1) { + scalar_t2* C_half2 = reinterpret_cast(&C[c_gl_wr]); + scalar_t2* sh_red_half2 = reinterpret_cast(&sh_red[c_sh_rd]); +#pragma unroll + for (int a = 0; a < 4; a++) { + atomicAdd(&C_half2[a], sh_red_half2[a]); + } + } else { + C[c_gl_wr] = sh_red[c_sh_rd]; + } + c_gl_wr += c_gl_wr_delta; + c_sh_rd += c_sh_rd_delta; + } + } + __syncthreads(); + }; + + // Start global fetch and register load pipelines. + auto start_pipes = [&]() { + +#pragma unroll + for (int i = 0; i < stages - 1; i++) { + if (has_act_order && i == 0) { + int last_g_idx = slice_k_start + stages * tb_k * 2; + if (last_g_idx >= prob_k) { + last_g_idx = prob_k - 1; + } + fetch_act_order_scales_to_shared(true, g_idx[slice_k_start], g_idx[last_g_idx]); + } + + if constexpr (has_zp && !is_zp_float && group_blocks == -1) { + if (i == 0) { + fetch_col_zp_to_shared(); + if constexpr (!dequant_skip_flop) { + fetch_col_scale_to_shared(); + } + } + } + fetch_to_shared(i, i, i < slice_iters); + } + + zero_accums(); + wait_for_stage(); + init_same_group(0); + fetch_to_registers(0, 0); + fetch_scales_to_registers(0, 0); + fetch_zp_to_registers(0, 0); + a_gl_rd += a_gl_rd_delta_o * (stages - 1); + if constexpr (has_act_order) { + slice_k_start_shared_fetch += tb_k * (stages - 1); + } + }; + if (slice_iters) { + start_pipes(); + } + + // Main loop. + while (slice_iters) { + // We unroll over both the global fetch and the register load pipeline to + // ensure all shared memory accesses are static. Note that both pipelines + // have even length meaning that the next iteration will always start at + // index 0. + +#pragma unroll + for (int pipe = 0; pipe < stages;) { +#pragma unroll + for (int k = 0; k < b_sh_wr_iters; k++) { + fetch_to_registers(k + 1, pipe % stages); + fetch_scales_to_registers(k + 1, pipe); + fetch_zp_to_registers(k + 1, pipe); + if (k == b_sh_wr_iters - 2) { + fetch_to_shared((pipe + stages - 1) % stages, pipe, slice_iters >= stages); + pipe++; + wait_for_stage(); + init_same_group(pipe % stages); + } + matmul(k); + } + slice_iters--; + if (slice_iters == 0) { + break; + } + } + + a_gl_rd += a_gl_rd_delta_o * stages; + + if constexpr (has_act_order) { + slice_k_start += tb_k * stages; + + if (slice_k_start < prob_k) { + slice_k_start_shared_fetch += tb_k * stages; + int first_group_id = g_idx[slice_k_start]; + int last_g_idx = slice_k_start + stages * tb_k * 2; + if (last_g_idx >= prob_k) { + last_g_idx = prob_k - 1; + } + int last_group_id = g_idx[last_g_idx]; + if (last_group_id >= sh_first_group_id + sh_num_groups) { + fetch_act_order_scales_to_shared(false, first_group_id, last_group_id); + __syncthreads(); + } + } + } + + // Process results and, if necessary, proceed to the next column slice. + // While this pattern may not be the most readable, other ways of writing + // the loop seemed to noticeably worse performance after compilation. + if (slice_iters == 0) { + cp_async_wait<0>(); + bool last = slice_idx == slice_count - 1; + // For per-column scales, we only fetch them here in the final step before + // write-out + if constexpr (!has_act_order && group_blocks == -1 && (has_zp && dequant_skip_flop || !has_zp)) { + if (w_type.size_bits() == 8 || (last || use_atomic_add)) { + if (s_sh_wr_pred) { + cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]); + } + cp_async_fence(); + } + } + + thread_block_reduce(); + if constexpr (!has_act_order && group_blocks == -1 && (has_zp && dequant_skip_flop || !has_zp)) { + if (w_type.size_bits() == 8 || (last || use_atomic_add)) { + cp_async_wait<0>(); + __syncthreads(); + if (threadIdx.x / 32 < thread_n_blocks / 4) { + reinterpret_cast(&frag_s)[0] = sh_s[s_sh_rd + 0]; + reinterpret_cast(&frag_s)[1] = sh_s[s_sh_rd + 4]; + if constexpr (m_block_size_8) { + int idx = (threadIdx.x / 4) % 2; + scalar_t2* frag_s_half2 = reinterpret_cast(frag_s); +#pragma unroll + for (int i = 0; i < 8; i++) { + frag_s_half2[i] = Dtype::num2num2(reinterpret_cast(&frag_s_half2[i])[idx]); + } + } + } + } + } + + // For 8-bit channelwise, we apply the scale before the global reduction + // that converts the fp32 results to fp16 (so that we avoid possible + // overflow in fp16) + if constexpr ( + !has_act_order && group_blocks == -1 && w_type.size_bits() == 8 && (has_zp && dequant_skip_flop || !has_zp)) { + if (threadIdx.x / 32 < thread_n_blocks / 4) { +#pragma unroll + for (int i = 0; i < thread_m_blocks; i++) { +#pragma unroll + for (int j = 0; j < 4; j++) { + scale_float(reinterpret_cast(&frag_c[i][j][0][0]), frag_s[j / 2][2 * (j % 2) + 0]); + scale_float( + reinterpret_cast(&frag_c[i][j][0][2]), frag_s[j / 2][2 * (j % 2) + (m_block_size_8 ? 1 : 0)]); + + if constexpr (!m_block_size_8) { + scale_float(reinterpret_cast(&frag_c[i][j][1][0]), frag_s[j / 2][2 * (j % 2) + 1]); + scale_float(reinterpret_cast(&frag_c[i][j][1][2]), frag_s[j / 2][2 * (j % 2) + 1]); + } + } + } + } + } + + if (slice_count > 1 && !use_atomic_add) { + // only globally reduce if there is more than one block in a slice + barrier_acquire(&locks[locks_off], slice_idx); + if (use_fp32_reduce) { + global_reduce_fp32(slice_idx == 0, last); + } else { + global_reduce_fp16(slice_idx == 0, last); + } + barrier_release(&locks[locks_off], last); + } + if (use_atomic_add && slice_count > 1 && slice_idx != 0) wait_negative_and_add(&locks[locks_off]); + if (last || use_atomic_add) + // only the last block in a slice actually writes the result + write_result(); + slice_row = 0; + slice_col_par++; + slice_col++; + is_first_matmul_in_slice = true; + init_slice(); + + if (slice_iters) { + a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) + (threadIdx.x % a_gl_rd_delta_o); +#pragma unroll + for (int i = 0; i < b_sh_wr_iters; i++) + B_ptr[i] += b_sh_stride - b_gl_rd_delta_o * k_tiles; + if (slice_col == 0) { +#pragma unroll + for (int i = 0; i < b_sh_wr_iters; i++) + B_ptr[i] -= b_gl_stride; + } + + // Update slice k/n for scales loading + if constexpr (has_act_order) { + slice_k_start = tb_k * slice_row; + slice_k_finish = slice_k_start + tb_k * slice_iters; + slice_k_start_shared_fetch = slice_k_start; + slice_n_offset = act_s_col_tb_stride * slice_col; + + } else { + s_gl_rd = s_sh_stride * slice_col + threadIdx.x; + zp_gl_rd = zp_sh_stride * slice_col + threadIdx.x; + } + + start_pipes(); + } + } + } +} + +} // namespace MARLIN_NAMESPACE_NAME + +#endif diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/math.hpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/math.hpp new file mode 100644 index 00000000..6764e1fd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/math.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + +inline constexpr uint32_t next_pow_2(uint32_t const num) { + if (num <= 1) return num; + return 1 << (CHAR_BIT * sizeof(num) - __builtin_clz(num - 1)); +} + +template +static inline constexpr auto div_ceil(A a, B b) { + return (a + b - 1) / b; +} + +// Round a down to the next multiple of b. The caller is responsible for making +// sure that b is non-zero +template +inline constexpr T round_to_previous_multiple_of(T a, T b) { + return a % b == 0 ? a : (a / b) * b; +} + +// Round a up to the next multiple of b. The caller is responsible for making +// sure that b is non-zero +template +inline constexpr T round_to_next_multiple_of(T a, T b) { + return a % b == 0 ? a : ((a / b) + 1) * b; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_expert_quant.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_expert_quant.cu new file mode 100644 index 00000000..5c9eeae8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_expert_quant.cu @@ -0,0 +1,728 @@ +#include +#include +#include +#include +#include + +#include "nvfp4_quant.cuh" +#include "utils.h" + +// Quantizes the provided PackedVec into the uint32_t output +template +__device__ uint32_t cvt_warp_fp16_to_fp4(PackedVec& vec, float SFScaleVal, uint8_t* SFout) { +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + // Get absolute maximum values among the local 8 values. + auto localMax = __habs2(vec.elts[0]); + +// Local maximum value. +#pragma unroll + for (int i = 1; i < CVT_FP4_ELTS_PER_THREAD / 2; i++) { + localMax = __hmax2(localMax, __habs2(vec.elts[i])); + } + + // Get the absolute maximum among all 16 values (two threads). + localMax = __hmax2(__shfl_xor_sync(uint32_t(-1), localMax, 1), localMax); + // Get the final absolute maximum values. + float vecMax = float(__hmax(localMax.x, localMax.y)); + + // Get the SF (max value of the vector / max value of e2m1). + // maximum value of e2m1 = 6.0. + // TODO: use half as compute data type. + float SFValue = SFScaleVal * (vecMax * reciprocal_approximate_ftz(6.0f)); + // 8 bits representation of the SF. + uint8_t fp8SFVal; + // Write the SF to global memory (STG.8). + if constexpr (UE8M0_SF) { + // Extract the 8 exponent bits from float32. + // float 32bits = 1 sign bit + 8 exponent bits + 23 mantissa bits. + uint32_t tmp = reinterpret_cast(SFValue) >> 23; + fp8SFVal = tmp & 0xff; + // Convert back to fp32. + reinterpret_cast(SFValue) = tmp << 23; + } else { + // Here SFValue is always positive, so E4M3 is the same as UE4M3. + __nv_fp8_e4m3 tmp = __nv_fp8_e4m3(SFValue); + reinterpret_cast<__nv_fp8_e4m3&>(fp8SFVal) = tmp; + // Convert back to fp32. + SFValue = float(tmp); + } + // Get the output scale. + // Recipe: final_scale = reciprocal(fp32(fp8(SFValue * SFScaleVal))) * + // reciprocal(SFScaleVal)) + float outputScale = + SFValue != 0 ? reciprocal_approximate_ftz(SFValue * reciprocal_approximate_ftz(SFScaleVal)) : 0.0f; + + if (SFout) { + // Write the SF to global memory (STG.8). + *SFout = fp8SFVal; + } + + // Convert the input to float. + float2 fp2Vals[CVT_FP4_ELTS_PER_THREAD / 2]; + +#pragma unroll + for (int i = 0; i < CVT_FP4_ELTS_PER_THREAD / 2; i++) { + if constexpr (std::is_same_v) { + fp2Vals[i] = __half22float2(vec.elts[i]); + } else { + fp2Vals[i] = __bfloat1622float2(vec.elts[i]); + } + fp2Vals[i].x *= outputScale; + fp2Vals[i].y *= outputScale; + } + + // Convert to e2m1 values. + uint32_t e2m1Vec = fp32_vec_to_e2m1(fp2Vals); + + // Write the e2m1 values to global memory. + return e2m1Vec; +#else + return 0; +#endif +} + +__device__ __forceinline__ float silu(const float& val) { + return val / (1.0f + __expf(-val)); +} + +template +inline __device__ void silu_and_mul(PackedVec& x_vec, const PackedVec& y_vec) { + float2 x[CVT_FP4_ELTS_PER_THREAD / 2]; + float2 y[CVT_FP4_ELTS_PER_THREAD / 2]; + +#pragma unroll + for (int i = 0; i < CVT_FP4_ELTS_PER_THREAD / 2; i++) { + if constexpr (std::is_same_v) { + x[i] = __half22float2(x_vec.elts[i]); + y[i] = __half22float2(y_vec.elts[i]); + x[i].x = silu(x[i].x) * y[i].x; + x[i].y = silu(x[i].y) * y[i].y; + x_vec.elts[i] = __float22half2_rn(x[i]); + } else { + x[i] = __bfloat1622float2(x_vec.elts[i]); + y[i] = __bfloat1622float2(y_vec.elts[i]); + x[i].x = silu(x[i].x) * y[i].x; + x[i].y = silu(x[i].y) * y[i].y; + x_vec.elts[i] = __float22bfloat162_rn(x[i]); + } + } +} + +// Use UE4M3 by default. +template +__global__ void +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) +__launch_bounds__(512, 4) cvt_fp16_to_fp4( +#else +cvt_fp16_to_fp4( +#endif + int32_t numRows, + int32_t numCols, + Type const* in, + float const* SFScale, + uint32_t* out, + uint32_t* SFout, + uint32_t* input_offset_by_experts, + uint32_t* output_scale_offset_by_experts, + int32_t* mask, + int n_experts, + bool low_latency) { +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + using PackedVec = PackedVec; + static constexpr int CVT_FP4_NUM_THREADS_PER_SF = (CVT_FP4_SF_VEC_SIZE / CVT_FP4_ELTS_PER_THREAD); + static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD, "Vec size is not matched."); + + // Input tensor row/col loops. + int tid = blockIdx.x * blockDim.x + threadIdx.x; + int colsPerRow = numCols / CVT_FP4_ELTS_PER_THREAD; + // TODO(kaixih@nvidia): For now, we assume mask is used together with + // silu_and_mal. Maybe we want a more general behavior of mask later. In the + // silu case, the input last dim doubles. + bool use_mask = mask != nullptr; + int actualColsPerRow = use_mask ? colsPerRow * 2 : colsPerRow; + + // Each global thread processes one element + for (int globalIdx = tid; globalIdx < numRows * colsPerRow; globalIdx += gridDim.x * blockDim.x) { + // Calculate which row and column this global thread should process + int rowIdx = globalIdx / colsPerRow; + int colIdx = globalIdx % colsPerRow; + + // Find index within the experts using different strategies based on expert + // count + int rowIdx_in_expert = 0; + int expert_idx = 0; + + if constexpr (SMALL_NUM_EXPERTS) { + for (int i = 0; i < n_experts; i++) { + uint32_t current_offset = __ldca(&input_offset_by_experts[i]); + uint32_t next_offset = __ldca(&input_offset_by_experts[i + 1]); + if (rowIdx >= current_offset && rowIdx < next_offset) { + rowIdx_in_expert = rowIdx - current_offset; + expert_idx = i; + break; + } + } + } else { + // Load input offsets into registers first, then do the computation. + // Local array size set to 17 because of register limit. + uint32_t local_offsets[17]; + for (int chunk_start = 0; chunk_start < n_experts; chunk_start += 16) { + *reinterpret_cast(local_offsets) = + __ldca(reinterpret_cast(&input_offset_by_experts[chunk_start])); + *reinterpret_cast(local_offsets + 4) = + __ldca(reinterpret_cast(&input_offset_by_experts[chunk_start + 4])); + *reinterpret_cast(local_offsets + 8) = + __ldca(reinterpret_cast(&input_offset_by_experts[chunk_start + 8])); + *reinterpret_cast(local_offsets + 12) = + __ldca(reinterpret_cast(&input_offset_by_experts[chunk_start + 12])); + local_offsets[16] = __ldca(&input_offset_by_experts[chunk_start + 16]); + +// Check against the 16 loaded offsets +#pragma unroll + for (int i = 0; i < 16; i++) { + if (rowIdx >= local_offsets[i] && rowIdx < local_offsets[i + 1]) { + rowIdx_in_expert = rowIdx - local_offsets[i]; + expert_idx = chunk_start + i; + break; + } + } + } + } + + // Early exit when using masks. + if (use_mask && rowIdx_in_expert >= mask[expert_idx]) { + continue; + } + + int64_t inOffset = rowIdx * actualColsPerRow + colIdx; + PackedVec in_vec = reinterpret_cast(in)[inOffset]; + if (use_mask) { + PackedVec in_vec_mul = reinterpret_cast(in)[inOffset + colsPerRow]; + silu_and_mul(in_vec, in_vec_mul); + } + + // Get the output tensor offset. + // Same as inOffset because 8 elements are packed into one uint32_t. + int64_t outOffset = rowIdx * colsPerRow + colIdx; + auto& out_pos = out[outOffset]; + + // Get the global scaling factor, which will be applied to the SF. + // Note SFScale is the same as next GEMM's alpha, which is + // (448.f / (Alpha_A / 6.f)). + float const SFScaleVal = SFScale == nullptr ? 1.0f : SFScale[expert_idx]; + + int factor = CVT_FP4_SF_VEC_SIZE * 4; + // The actual output_scales dim is computed from the padded numCols. + int32_t numCols_padded = (numCols + factor - 1) / factor * factor; + int numCols_SFout = numCols_padded / CVT_FP4_SF_VEC_SIZE / 4; + uint32_t* SFout_in_expert = SFout + output_scale_offset_by_experts[expert_idx] * numCols_SFout; + + auto sf_out = cvt_quant_to_fp4_get_sf_out_offset( + rowIdx_in_expert, colIdx, numCols, SFout_in_expert); + + out_pos = cvt_warp_fp16_to_fp4(in_vec, SFScaleVal, sf_out); + } +#endif +} + +// Use UE4M3 by default. +template +__global__ void +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) +__launch_bounds__(512, 4) cvt_fp16_to_fp4_expert( +#else +cvt_fp16_to_fp4_expert( +#endif + int32_t numRows, + int32_t numCols, + Type const* in, + float const* SFScale, + uint32_t* out, + uint32_t* SFout, + int32_t* mask, + bool use_silu_and_mul, + int n_experts) { +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + using PackedVec = PackedVec; + static constexpr int CVT_FP4_NUM_THREADS_PER_SF = (CVT_FP4_SF_VEC_SIZE / CVT_FP4_ELTS_PER_THREAD); + static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD, "Vec size is not matched."); + + // Input tensor row/col loops. + int tid = blockIdx.x * blockDim.x + threadIdx.x; + int stride = (gridDim.x * blockDim.x) / n_experts; + int remainder = (gridDim.x * blockDim.x) % n_experts; + int expert_idx; + int tid_in_expert; + int actual_stride; + if (remainder > 0) { + int bound = remainder * (stride + 1); + if (tid < bound) { + expert_idx = tid / (stride + 1); + tid_in_expert = tid % (stride + 1); + actual_stride = stride + 1; + } else { + expert_idx = remainder + (tid - bound) / stride; + tid_in_expert = (tid - bound) % stride; + actual_stride = stride; + } + } else { + expert_idx = tid / stride; + tid_in_expert = tid % stride; + actual_stride = stride; + } + int m = numRows / n_experts; + int padded_m = (m + (128 - 1)) / 128 * 128; + + int colsPerRow = numCols / CVT_FP4_ELTS_PER_THREAD; + // TODO(kaixih@nvidia): For now, we assume mask is used together with + // silu_and_mal. Maybe we want a more general behavior of mask later. In the + // silu case, the input last dim doubles. + bool use_mask = mask != nullptr; + int actualColsPerRow = use_silu_and_mul ? colsPerRow * 2 : colsPerRow; + + // Each global thread processes one element + for (int globalIdx = tid_in_expert + expert_idx * m * colsPerRow; globalIdx < (expert_idx + 1) * m * colsPerRow; + globalIdx += actual_stride) { + // Calculate which row and column this global thread should process + int rowIdx = globalIdx / colsPerRow; + int colIdx = globalIdx % colsPerRow; + + // Find index within the experts + int rowIdx_in_expert = rowIdx - expert_idx * m; + + // Early exit when using masks. + if (use_mask && rowIdx_in_expert >= mask[expert_idx]) { + break; + } + + int64_t inOffset = rowIdx * actualColsPerRow + colIdx; + PackedVec in_vec = reinterpret_cast(in)[inOffset]; + if (use_silu_and_mul) { + PackedVec in_vec_mul = reinterpret_cast(in)[inOffset + colsPerRow]; + silu_and_mul(in_vec, in_vec_mul); + } + + // Get the output tensor offset. + // Same as inOffset because 8 elements are packed into one uint32_t. + int64_t outOffset = rowIdx * colsPerRow + colIdx; + auto& out_pos = out[outOffset]; + + // Get the global scaling factor, which will be applied to the SF. + // Note SFScale is the same as next GEMM's alpha, which is + // (448.f / (Alpha_A / 6.f)). + float const SFScaleVal = SFScale == nullptr ? 1.0f : SFScale[expert_idx]; + + int factor = CVT_FP4_SF_VEC_SIZE * 4; + // The actual output_scales dim is computed from the padded numCols. + int32_t numCols_padded = (numCols + factor - 1) / factor * factor; + int numCols_SFout = numCols_padded / CVT_FP4_SF_VEC_SIZE / 4; + uint32_t* SFout_in_expert = SFout + expert_idx * padded_m * numCols_SFout; + + auto sf_out = cvt_quant_to_fp4_get_sf_out_offset( + rowIdx_in_expert, colIdx, numCols, SFout_in_expert); + + out_pos = cvt_warp_fp16_to_fp4(in_vec, SFScaleVal, sf_out); + } +#endif +} + +// Kernel for LARGE_M_TOPK = true (large m_topk optimized version) +template +__global__ void +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) +__launch_bounds__(1024, 4) cvt_fp16_to_fp4( +#else +cvt_fp16_to_fp4( +#endif + int32_t numRows, + int32_t numCols, + Type const* in, + float const* SFScale, + uint32_t* out, + uint32_t* SFout, + uint32_t* input_offset_by_experts, + uint32_t* output_scale_offset_by_experts, + int32_t* mask, + int n_experts) { +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + using PackedVec = PackedVec; + static constexpr int CVT_FP4_NUM_THREADS_PER_SF = (CVT_FP4_SF_VEC_SIZE / CVT_FP4_ELTS_PER_THREAD); + static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD, "Vec size is not matched."); + extern __shared__ uint32_t shared_input_offsets[]; + + // Load input offsets into shared memory. + // If n_experts is larger than 4, use vectorized int4 to save instructions. + // If n_experts is smaller than 4, read directly. + if constexpr (SMALL_NUM_EXPERTS) { + for (int i = threadIdx.x; i < n_experts + 1; i += blockDim.x) { + shared_input_offsets[i] = input_offset_by_experts[i]; + } + } else { + for (int i = threadIdx.x * 4; i < n_experts; i += blockDim.x * 4) { + *reinterpret_cast(&shared_input_offsets[i]) = *reinterpret_cast(&input_offset_by_experts[i]); + } + if (threadIdx.x == 0) { + shared_input_offsets[n_experts] = input_offset_by_experts[n_experts]; + } + } + + __syncthreads(); + + int tid = blockIdx.x * blockDim.x + threadIdx.x; + int colsPerRow = numCols / CVT_FP4_ELTS_PER_THREAD; + bool use_mask = mask != nullptr; + int actualColsPerRow = use_mask ? colsPerRow * 2 : colsPerRow; + + // Each global thread processes one element + for (int globalIdx = tid; globalIdx < numRows * colsPerRow; globalIdx += gridDim.x * blockDim.x) { + // Calculate which row and column this global thread should process + int rowIdx = globalIdx / colsPerRow; + int colIdx = globalIdx % colsPerRow; + + // Find expert using binary search for better performance with large m_topk + int rowIdx_in_expert = 0; + int expert_idx = 0; + + // Binary search through experts using shared memory + int left = 0, right = n_experts - 1; + while (left <= right) { + int mid = (left + right) / 2; + // Get offsets: shared_input_offsets[i] corresponds to + // input_offset_by_experts[i] + uint32_t mid_offset = shared_input_offsets[mid]; + uint32_t next_offset = shared_input_offsets[mid + 1]; + + if (rowIdx >= mid_offset && rowIdx < next_offset) { + rowIdx_in_expert = rowIdx - mid_offset; + expert_idx = mid; + break; + } else if (rowIdx < mid_offset) { + right = mid - 1; + } else { + left = mid + 1; + } + } + + if (use_mask && rowIdx_in_expert >= mask[expert_idx]) { + continue; + } + + int64_t inOffset = rowIdx * actualColsPerRow + colIdx; + + PackedVec in_vec = reinterpret_cast(in)[inOffset]; + if (use_mask) { + PackedVec in_vec_mul = reinterpret_cast(in)[inOffset + colsPerRow]; + silu_and_mul(in_vec, in_vec_mul); + } + + int64_t outOffset = rowIdx * colsPerRow + colIdx; + auto& out_pos = out[outOffset]; + + float const SFScaleVal = SFScale == nullptr ? 1.0f : SFScale[expert_idx]; + + int factor = CVT_FP4_SF_VEC_SIZE * 4; + int32_t numCols_padded = (numCols + factor - 1) / factor * factor; + int numCols_SFout = numCols_padded / CVT_FP4_SF_VEC_SIZE / 4; + uint32_t* SFout_in_expert = SFout + output_scale_offset_by_experts[expert_idx] * numCols_SFout; + + auto sf_out = cvt_quant_to_fp4_get_sf_out_offset( + rowIdx_in_expert, colIdx, numCols, SFout_in_expert); + + out_pos = cvt_warp_fp16_to_fp4(in_vec, SFScaleVal, sf_out); + } +#endif +} + +template +void quant_impl( + void* output, + void* output_scale, + void* input, + void* input_global_scale, + void* input_offset_by_experts, + void* output_scale_offset_by_experts, + void* mask, + bool use_silu_and_mul, + int m_topk, + int k, + int n_experts, + cudaStream_t stream) { + // TODO: this multiProcessorCount should be cached. + int device; + cudaGetDevice(&device); + int multiProcessorCount; + cudaDeviceGetAttribute(&multiProcessorCount, cudaDevAttrMultiProcessorCount, device); + + // Grid, Block size. + // Each thread converts 8 values. + int const workSizePerRow = k / ELTS_PER_THREAD; + int const totalWorkSize = m_topk * workSizePerRow; + dim3 block(std::min(workSizePerRow, 512)); + // Get number of blocks per SM (assume we can fully utilize the SM). + int const numBlocksPerSM = 2048 / block.x; + dim3 grid(std::min(static_cast((totalWorkSize + block.x - 1) / block.x), multiProcessorCount * numBlocksPerSM)); + while (grid.x <= multiProcessorCount && block.x > 64) { + grid.x *= 2; + block.x = (block.x + 1) / 2; + } + + // TODO(kaixih@nvidia): Should relax this to allow any grid size. + if (mask != nullptr) { + grid.x = (grid.x + n_experts - 1) / n_experts * n_experts; + cvt_fp16_to_fp4_expert<<>>( + m_topk, + k, + reinterpret_cast(input), + reinterpret_cast(input_global_scale), + reinterpret_cast(output), + reinterpret_cast(output_scale), + reinterpret_cast(mask), + use_silu_and_mul, + n_experts); + return; + } + + int const blockRepeat = (totalWorkSize + block.x * grid.x - 1) / (block.x * grid.x); + if (blockRepeat > 1) { + size_t shared_mem_size = (n_experts + 1) * sizeof(uint32_t); + if (n_experts >= 4) { + cvt_fp16_to_fp4<<>>( + m_topk, + k, + reinterpret_cast(input), + reinterpret_cast(input_global_scale), + reinterpret_cast(output), + reinterpret_cast(output_scale), + reinterpret_cast(input_offset_by_experts), + reinterpret_cast(output_scale_offset_by_experts), + reinterpret_cast(mask), + n_experts); + } else { + cvt_fp16_to_fp4<<>>( + m_topk, + k, + reinterpret_cast(input), + reinterpret_cast(input_global_scale), + reinterpret_cast(output), + reinterpret_cast(output_scale), + reinterpret_cast(input_offset_by_experts), + reinterpret_cast(output_scale_offset_by_experts), + reinterpret_cast(mask), + n_experts); + } + } else { + if (n_experts >= 16) { + cvt_fp16_to_fp4<<>>( + m_topk, + k, + reinterpret_cast(input), + reinterpret_cast(input_global_scale), + reinterpret_cast(output), + reinterpret_cast(output_scale), + reinterpret_cast(input_offset_by_experts), + reinterpret_cast(output_scale_offset_by_experts), + reinterpret_cast(mask), + n_experts, + /* bool low_latency */ true); + } else { + cvt_fp16_to_fp4<<>>( + m_topk, + k, + reinterpret_cast(input), + reinterpret_cast(input_global_scale), + reinterpret_cast(output), + reinterpret_cast(output_scale), + reinterpret_cast(input_offset_by_experts), + reinterpret_cast(output_scale_offset_by_experts), + reinterpret_cast(mask), + n_experts, + /* bool low_latency */ true); + } + } +} + +// Avoid redefinition warnings +#undef CHECK_CONTIGUOUS +#undef CHECK_TH_CUDA +#undef CHECK_INPUT + +/*Quantization entry for fp4 experts quantization*/ +#define CHECK_TH_CUDA(x, m) TORCH_CHECK(x.is_cuda(), m, "must be a CUDA tensor") +#define CHECK_CONTIGUOUS(x, m) TORCH_CHECK(x.is_contiguous(), m, "must be contiguous") +#define CHECK_INPUT(x, m) \ + CHECK_TH_CUDA(x, m); \ + CHECK_CONTIGUOUS(x, m); + +// constexpr auto FP8 = at::ScalarType::Float8_e4m3fn; +constexpr auto HALF = at::ScalarType::Half; +constexpr auto BF16 = at::ScalarType::BFloat16; +constexpr auto FLOAT = at::ScalarType::Float; +constexpr auto INT = at::ScalarType::Int; +constexpr auto UINT8 = at::ScalarType::Byte; + +void scaled_fp4_experts_quant_sm100a( + torch::Tensor& output, + torch::Tensor& output_scale, + torch::Tensor const& input, + torch::Tensor const& input_global_scale, + torch::Tensor const& input_offset_by_experts, + torch::Tensor const& output_scale_offset_by_experts) { + auto sm_version = getSMVersion(); + TORCH_CHECK(sm_version >= 100, "fp4_quant is only supported on sm100+"); + + CHECK_INPUT(output, "output must be a CUDA tensor"); + CHECK_INPUT(output_scale, "output_scale must be a CUDA tensor"); + CHECK_INPUT(input, "input must be a CUDA tensor"); + CHECK_INPUT(input_global_scale, "input_global_scale must be a CUDA tensor"); + CHECK_INPUT(input_offset_by_experts, "input_offset_by_experts must be a CUDA tensor"); + CHECK_INPUT(output_scale_offset_by_experts, "output_scale_offset_by_experts must be a CUDA tensor"); + + TORCH_CHECK(output.dim() == 2); + TORCH_CHECK(output_scale.dim() == 2); + TORCH_CHECK(input.dim() == 2); + TORCH_CHECK(input_global_scale.dim() == 1); + TORCH_CHECK(input_offset_by_experts.dim() == 1); + TORCH_CHECK(output_scale_offset_by_experts.dim() == 1); + + TORCH_CHECK(input.scalar_type() == HALF || input.scalar_type() == BF16); + TORCH_CHECK(input_global_scale.scalar_type() == FLOAT); + TORCH_CHECK(input_offset_by_experts.scalar_type() == INT); + TORCH_CHECK(output_scale_offset_by_experts.scalar_type() == INT); + // output is uint8 (two nvfp4 values are packed into one uint8) + // output_scale is int32 (four fp8 values are packed into one int32) + TORCH_CHECK(output.scalar_type() == UINT8); + TORCH_CHECK(output_scale.scalar_type() == INT); + + const int BLOCK_SIZE = 16; + auto m_topk = input.size(0); + auto k = input.size(1); + TORCH_CHECK(k % BLOCK_SIZE == 0, "k must be a multiple of 16"); + auto n_experts = input_global_scale.size(0); + TORCH_CHECK(input_offset_by_experts.size(0) == n_experts + 1); + TORCH_CHECK(output_scale_offset_by_experts.size(0) == n_experts + 1); + TORCH_CHECK(output.size(0) == m_topk); + TORCH_CHECK(output.size(1) == k / 2); + int scales_k = k / BLOCK_SIZE; + // 4 means the swizzle requirement by nvidia nvfp4. + int padded_k = (scales_k + (4 - 1)) / 4 * 4; + // 4 means 4 fp8 values are packed into one int32 + TORCH_CHECK(output_scale.size(1) * 4 == padded_k); + + auto in_dtype = input.dtype(); + at::cuda::CUDAGuard device_guard{(char)input.get_device()}; + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(input.get_device()); + if (in_dtype == at::ScalarType::Half) { + quant_impl( + output.data_ptr(), + output_scale.data_ptr(), + input.data_ptr(), + input_global_scale.data_ptr(), + input_offset_by_experts.data_ptr(), + output_scale_offset_by_experts.data_ptr(), + nullptr, // mask + false, // use_silu_and_mul + m_topk, + k, + n_experts, + stream); + } else if (in_dtype == at::ScalarType::BFloat16) { + quant_impl<__nv_bfloat16>( + output.data_ptr(), + output_scale.data_ptr(), + input.data_ptr(), + input_global_scale.data_ptr(), + input_offset_by_experts.data_ptr(), + output_scale_offset_by_experts.data_ptr(), + nullptr, // mask + false, // use_silu_and_mul + m_topk, + k, + n_experts, + stream); + } else { + TORCH_CHECK(false, "Expected input data type to be half or bfloat16"); + } +} + +void silu_and_mul_scaled_fp4_experts_quant_sm100a( + torch::Tensor& output, + torch::Tensor& output_scale, + torch::Tensor const& input, + torch::Tensor const& input_global_scale, + torch::Tensor const& mask, + bool use_silu_and_mul) { + auto sm_version = getSMVersion(); + TORCH_CHECK(sm_version >= 100, "fp4_quant is only supported on sm100+"); + + CHECK_INPUT(output, "output must be a CUDA tensor"); + CHECK_INPUT(output_scale, "output_scale must be a CUDA tensor"); + CHECK_INPUT(input, "input must be a CUDA tensor"); + CHECK_INPUT(input_global_scale, "input_global_scale must be a CUDA tensor"); + CHECK_INPUT(mask, "mask must be a CUDA tensor"); + + TORCH_CHECK(output.dim() == 2); + TORCH_CHECK(output_scale.dim() == 2); + TORCH_CHECK(input.dim() == 2); + TORCH_CHECK(input_global_scale.dim() == 1); + + TORCH_CHECK(input.scalar_type() == HALF || input.scalar_type() == BF16); + TORCH_CHECK(input_global_scale.scalar_type() == FLOAT); + TORCH_CHECK(mask.scalar_type() == INT); + // output is uint8 (two nvfp4 values are packed into one uint8) + // output_scale is int32 (four fp8 values are packed into one int32) + TORCH_CHECK(output.scalar_type() == UINT8); + TORCH_CHECK(output_scale.scalar_type() == INT); + + const int BLOCK_SIZE = 16; + auto m_topk = input.size(0); + auto k_by_2 = input.size(1); + auto k = k_by_2; + if (use_silu_and_mul) { + TORCH_CHECK(k_by_2 % 2 == 0, "k must be a multiple of 2"); + k = k_by_2 / 2; + } + auto n_experts = input_global_scale.size(0); + TORCH_CHECK(mask.size(0) == n_experts); + TORCH_CHECK(output.size(0) == m_topk); + TORCH_CHECK(output.size(1) == k / 2); + int scales_k = k / BLOCK_SIZE; + // 4 means the swizzle requirement by nvidia nvfp4. + int padded_k = (scales_k + (4 - 1)) / 4 * 4; + // 4 means 4 fp8 values are packed into one int32 + TORCH_CHECK(output_scale.size(1) * 4 == padded_k); + + auto in_dtype = input.dtype(); + at::cuda::CUDAGuard device_guard{(char)input.get_device()}; + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(input.get_device()); + if (in_dtype == at::ScalarType::Half) { + quant_impl( + output.data_ptr(), + output_scale.data_ptr(), + input.data_ptr(), + input_global_scale.data_ptr(), + nullptr, // input_offset_by_experts + nullptr, // output_scale_offset_by_experts + mask.data_ptr(), + use_silu_and_mul, + m_topk, + k, + n_experts, + stream); + } else if (in_dtype == at::ScalarType::BFloat16) { + quant_impl<__nv_bfloat16>( + output.data_ptr(), + output_scale.data_ptr(), + input.data_ptr(), + input_global_scale.data_ptr(), + nullptr, // input_offset_by_experts + nullptr, // output_scale_offset_by_experts + mask.data_ptr(), + use_silu_and_mul, + m_topk, + k, + n_experts, + stream); + } else { + TORCH_CHECK(false, "Expected input data type to be half or bfloat16"); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_quant.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_quant.cuh new file mode 100644 index 00000000..bc7c612a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_quant.cuh @@ -0,0 +1,182 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +// Get type2 from type or vice versa (applied to half and bfloat16) +template +struct TypeConverter { + using Type = half2; +}; // keep for generality + +template <> +struct TypeConverter { + using Type = half; +}; + +template <> +struct TypeConverter { + using Type = half2; +}; + +template <> +struct TypeConverter<__nv_bfloat162> { + using Type = __nv_bfloat16; +}; + +template <> +struct TypeConverter<__nv_bfloat16> { + using Type = __nv_bfloat162; +}; + +#define ELTS_PER_THREAD 8 + +constexpr int CVT_FP4_ELTS_PER_THREAD = 8; +constexpr int CVT_FP4_SF_VEC_SIZE = 16; + +// Convert 8 float32 values into 8 e2m1 values (represented as one uint32_t). +inline __device__ uint32_t fp32_vec_to_e2m1(float (&array)[8]) { + // PTX instructions used here requires >= sm100f. +#if CUTLASS_ARCH_MMA_SM100A_ENABLED || CUTLASS_ARCH_MMA_SM103A_ENABLED || CUTLASS_ARCH_MMA_SM120A_ENABLED || \ + (defined(__CUDA_ARCH_FAMILY_SPECIFIC__) && (__CUDA_ARCH_FAMILY_SPECIFIC__ > 1000)) + uint32_t val; + asm volatile( + "{\n" + ".reg .b8 byte0;\n" + ".reg .b8 byte1;\n" + ".reg .b8 byte2;\n" + ".reg .b8 byte3;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte0, %2, %1;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte1, %4, %3;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte2, %6, %5;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte3, %8, %7;\n" + "mov.b32 %0, {byte0, byte1, byte2, byte3};\n" + "}" + : "=r"(val) + : "f"(array[0]), + "f"(array[1]), + "f"(array[2]), + "f"(array[3]), + "f"(array[4]), + "f"(array[5]), + "f"(array[6]), + "f"(array[7])); + return val; +#else + printf("fp32_vec_to_e2m1 is not supported on this architecture\n"); + __trap(); + return 0; +#endif +} + +// Convert 4 float2 values into 8 e2m1 values (represented as one uint32_t). +inline __device__ uint32_t fp32_vec_to_e2m1(float2 (&array)[4]) { + // PTX instructions used here requires >= sm100f. +#if CUTLASS_ARCH_MMA_SM100A_ENABLED || CUTLASS_ARCH_MMA_SM103A_ENABLED || CUTLASS_ARCH_MMA_SM120A_ENABLED || \ + (defined(__CUDA_ARCH_FAMILY_SPECIFIC__) && (__CUDA_ARCH_FAMILY_SPECIFIC__ > 1000)) + uint32_t val; + asm volatile( + "{\n" + ".reg .b8 byte0;\n" + ".reg .b8 byte1;\n" + ".reg .b8 byte2;\n" + ".reg .b8 byte3;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte0, %2, %1;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte1, %4, %3;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte2, %6, %5;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte3, %8, %7;\n" + "mov.b32 %0, {byte0, byte1, byte2, byte3};\n" + "}" + : "=r"(val) + : "f"(array[0].x), + "f"(array[0].y), + "f"(array[1].x), + "f"(array[1].y), + "f"(array[2].x), + "f"(array[2].y), + "f"(array[3].x), + "f"(array[3].y)); + return val; +#else + printf("fp32_vec_to_e2m1 is not supported on this architecture\n"); + __trap(); + return 0; +#endif +} + +// Fast reciprocal. +inline __device__ float reciprocal_approximate_ftz(float a) { + float b; + asm volatile("rcp.approx.ftz.f32 %0, %1;\n" : "=f"(b) : "f"(a)); + return b; +} + +template +__device__ uint8_t* cvt_quant_to_fp4_get_sf_out_offset(int rowIdx, int colIdx, int numCols, SFType* SFout) { +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + static_assert(CVT_FP4_NUM_THREADS_PER_SF == 1 || CVT_FP4_NUM_THREADS_PER_SF == 2); + + // One pair of threads write one SF to global memory. + // TODO: stage through smem for packed STG.32 + // is it better than STG.8 from 4 threads ? + if (threadIdx.x % CVT_FP4_NUM_THREADS_PER_SF == 0) { + // SF vector index (16 elements share one SF in the K dimension). + int32_t kIdx = colIdx / CVT_FP4_NUM_THREADS_PER_SF; + int32_t mIdx = rowIdx; + + // SF layout [numMTiles, numKTiles, 32 (mTile), 4 (mTile), 4(kTile)] + // --> index [mTileIdx, kTileIdx, outerMIdx, innerMIdx, innerKIdx] + + int32_t mTileIdx = mIdx / (32 * 4); + // SF vector size 16. + int factor = CVT_FP4_SF_VEC_SIZE * 4; + int32_t numKTiles = (numCols + factor - 1) / factor; + int64_t mTileStride = numKTiles * 32 * 4 * 4; + + int32_t kTileIdx = (kIdx / 4); + int64_t kTileStride = 32 * 4 * 4; + + // M tile layout [32, 4] is column-major. + int32_t outerMIdx = (mIdx % 32); + int64_t outerMStride = 4 * 4; + + int32_t innerMIdx = (mIdx % (32 * 4)) / 32; + int64_t innerMStride = 4; + + int32_t innerKIdx = (kIdx % 4); + int64_t innerKStride = 1; + + // Compute the global offset. + int64_t SFOffset = mTileIdx * mTileStride + kTileIdx * kTileStride + outerMIdx * outerMStride + + innerMIdx * innerMStride + innerKIdx * innerKStride; + + return reinterpret_cast(SFout) + SFOffset; + } +#endif + return nullptr; +} + +// Define a 16 bytes packed data type. +template +struct PackedVec { + typename TypeConverter::Type elts[4]; +}; + +template <> +struct PackedVec<__nv_fp8_e4m3> { + __nv_fp8x2_e4m3 elts[8]; +}; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_quant_entry.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_quant_entry.cu new file mode 100644 index 00000000..67044d01 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_quant_entry.cu @@ -0,0 +1,77 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#if defined ENABLE_NVFP4 && ENABLE_NVFP4 +void scaled_fp4_quant_sm100a_sm120a( + torch::Tensor const& output, + torch::Tensor const& input, + torch::Tensor const& output_sf, + torch::Tensor const& input_sf); + +void scaled_fp4_experts_quant_sm100a( + torch::Tensor& output, + torch::Tensor& output_scale, + torch::Tensor const& input, + torch::Tensor const& input_global_scale, + torch::Tensor const& input_offset_by_experts, + torch::Tensor const& output_scale_offset_by_experts); + +void silu_and_mul_scaled_fp4_experts_quant_sm100a( + torch::Tensor& output, + torch::Tensor& output_scale, + torch::Tensor const& input, + torch::Tensor const& input_global_scale, + torch::Tensor const& mask, + bool use_silu_and_mul); + +#endif + +void scaled_fp4_quant( + torch::Tensor& output, torch::Tensor const& input, torch::Tensor& output_sf, torch::Tensor const& input_sf) { +#if defined ENABLE_NVFP4 && ENABLE_NVFP4 + return scaled_fp4_quant_sm100a_sm120a(output, input, output_sf, input_sf); +#endif + TORCH_CHECK_NOT_IMPLEMENTED(false, "No compiled nvfp4 quantization"); +} + +void scaled_fp4_experts_quant( + torch::Tensor& output, + torch::Tensor& output_scale, + torch::Tensor const& input, + torch::Tensor const& input_global_scale, + torch::Tensor const& input_offset_by_experts, + torch::Tensor const& output_scale_offset_by_experts) { +#if defined ENABLE_NVFP4 && ENABLE_NVFP4 + return scaled_fp4_experts_quant_sm100a( + output, output_scale, input, input_global_scale, input_offset_by_experts, output_scale_offset_by_experts); +#endif + TORCH_CHECK_NOT_IMPLEMENTED(false, "No compiled nvfp4 experts quantization kernel"); +} + +void silu_and_mul_scaled_fp4_experts_quant( + torch::Tensor& output, + torch::Tensor& output_scale, + torch::Tensor const& input, + torch::Tensor const& input_global_scale, + torch::Tensor const& mask, + bool use_silu_and_mul) { +#if defined ENABLE_NVFP4 && ENABLE_NVFP4 + return silu_and_mul_scaled_fp4_experts_quant_sm100a( + output, output_scale, input, input_global_scale, mask, use_silu_and_mul); +#endif + TORCH_CHECK_NOT_IMPLEMENTED(false, "No compiled nvfp4 experts quantization kernel"); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_quant_kernels.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_quant_kernels.cu new file mode 100644 index 00000000..bb99d76c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_quant_kernels.cu @@ -0,0 +1,242 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include + +#include "nvfp4_quant.cuh" +#include "utils.h" + +// Quantizes the provided PackedVec into the uint32_t output +template +__device__ uint32_t cvt_warp_fp16_to_fp4(PackedVec& vec, float SFScaleVal, uint8_t* SFout) { +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + // Get absolute maximum values among the local 8 values. + auto localMax = __habs2(vec.elts[0]); + +// Local maximum value. +#pragma unroll + for (int i = 1; i < CVT_FP4_ELTS_PER_THREAD / 2; i++) { + localMax = __hmax2(localMax, __habs2(vec.elts[i])); + } + + // Get the absolute maximum among all 16 values (two threads). + localMax = __hmax2(__shfl_xor_sync(uint32_t(-1), localMax, 1), localMax); + // Get the final absolute maximum values. + float vecMax = float(__hmax(localMax.x, localMax.y)); + + // Get the SF (max value of the vector / max value of e2m1). + // maximum value of e2m1 = 6.0. + // TODO: use half as compute data type. + float SFValue = SFScaleVal * (vecMax * reciprocal_approximate_ftz(6.0f)); + // 8 bits representation of the SF. + uint8_t fp8SFVal; + // Write the SF to global memory (STG.8). + if constexpr (UE8M0_SF) { + __nv_fp8_e8m0 tmp; + tmp.__x = __nv_cvt_float_to_e8m0(SFValue, __NV_SATFINITE, cudaRoundPosInf); + SFValue = static_cast(tmp); + fp8SFVal = tmp.__x; + } else { + // Here SFValue is always positive, so E4M3 is the same as UE4M3. + __nv_fp8_e4m3 tmp = __nv_fp8_e4m3(SFValue); + fp8SFVal = tmp.__x; + SFValue = static_cast(tmp); + } + // Get the output scale. + // Recipe: final_scale = reciprocal(fp32(fp8(SFValue * SFScaleVal))) * + // reciprocal(SFScaleVal)) + float outputScale = + SFValue != 0 ? reciprocal_approximate_ftz(SFValue * reciprocal_approximate_ftz(SFScaleVal)) : 0.0f; + + if (SFout) { + // Write the SF to global memory (STG.8). + *SFout = fp8SFVal; + } + + // Convert the input to float. + float2 fp2Vals[CVT_FP4_ELTS_PER_THREAD / 2]; + +#pragma unroll + for (int i = 0; i < CVT_FP4_ELTS_PER_THREAD / 2; i++) { + if constexpr (std::is_same_v) { + fp2Vals[i] = __half22float2(vec.elts[i]); + } else { + fp2Vals[i] = __bfloat1622float2(vec.elts[i]); + } + fp2Vals[i].x *= outputScale; + fp2Vals[i].y *= outputScale; + } + + // Convert to e2m1 values. + uint32_t e2m1Vec = fp32_vec_to_e2m1(fp2Vals); + + // Write the e2m1 values to global memory. + return e2m1Vec; +#else + return 0; +#endif +} + +// Use UE4M3 by default. +template +__global__ void +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) +__launch_bounds__(512, 4) cvt_fp16_to_fp4( +#else +cvt_fp16_to_fp4( +#endif + int32_t numRows, int32_t numCols, Type const* in, float const* SFScale, uint32_t* out, uint32_t* SFout) { +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + using PackedVec = PackedVec; + static constexpr int CVT_FP4_NUM_THREADS_PER_SF = (CVT_FP4_SF_VEC_SIZE / CVT_FP4_ELTS_PER_THREAD); + static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD, "Vec size is not matched."); + + // Get the global scaling factor, which will be applied to the SF. + // Note SFScale is the same as next GEMM's alpha, which is + // (448.f / (Alpha_A / 6.f)). + float const SFScaleVal = SFScale == nullptr ? 1.0f : SFScale[0]; + + // Input tensor row/col loops. + for (int rowIdx = blockIdx.x; rowIdx < numRows; rowIdx += gridDim.x) { + for (int colIdx = threadIdx.x; colIdx < numCols / CVT_FP4_ELTS_PER_THREAD; colIdx += blockDim.x) { + int64_t inOffset = rowIdx * (numCols / CVT_FP4_ELTS_PER_THREAD) + colIdx; + PackedVec in_vec = reinterpret_cast(in)[inOffset]; + // Get the output tensor offset. + // Same as inOffset because 8 elements are packed into one uint32_t. + int64_t outOffset = inOffset; + auto& out_pos = out[outOffset]; + + auto sf_out = + cvt_quant_to_fp4_get_sf_out_offset(rowIdx, colIdx, numCols, SFout); + + out_pos = cvt_warp_fp16_to_fp4(in_vec, SFScaleVal, sf_out); + } + } +#endif +} + +template +void invokeFP4Quantization( + int m, + int n, + T const* input, + float const* SFScale, + int64_t* output, + int32_t* SFOuput, + bool useUE8M0, + int multiProcessorCount, + cudaStream_t stream) { + // Grid, Block size. + // Each thread converts 8 values. + dim3 block(std::min(int(n / ELTS_PER_THREAD), 512)); + // Get number of blocks per SM (assume we can fully utilize the SM). + int const numBlocksPerSM = 2048 / block.x; + dim3 grid(std::min(int(m), multiProcessorCount * numBlocksPerSM)); + + // Launch the cvt kernel. + if (useUE8M0) { + cvt_fp16_to_fp4<<>>( + m, n, input, SFScale, reinterpret_cast(output), reinterpret_cast(SFOuput)); + } else { + cvt_fp16_to_fp4<<>>( + m, n, input, SFScale, reinterpret_cast(output), reinterpret_cast(SFOuput)); + } +} + +// Instantiate the function. +template void invokeFP4Quantization( + int m, + int n, + half const* input, + float const* SFScale, + int64_t* output, + int32_t* SFOuput, + bool useUE8M0, + int multiProcessorCount, + cudaStream_t stream); + +template void invokeFP4Quantization( + int m, + int n, + __nv_bfloat16 const* input, + float const* SFScale, + int64_t* output, + int32_t* SFOuput, + bool useUE8M0, + int multiProcessorCount, + cudaStream_t stream); + +inline int getMultiProcessorCount() { + static int multi_processor_count = []() { + int device_id = 0; + int count = 0; + + // Get the current CUDA device ID + CHECK_CUDA_SUCCESS(cudaGetDevice(&device_id)); + + // Get the number of multiprocessors for the current device + CHECK_CUDA_SUCCESS(cudaDeviceGetAttribute(&count, cudaDevAttrMultiProcessorCount, device_id)); + + return count; // Initialize the static variable + }(); + + return multi_processor_count; // Return the cached value on subsequent calls +} + +void scaled_fp4_quant_sm100a_sm120a( + torch::Tensor const& output, + torch::Tensor const& input, + torch::Tensor const& output_sf, + torch::Tensor const& input_sf) { + auto sm_version = getSMVersion(); + TORCH_CHECK(sm_version >= 100, "fp4_quant is only supported on sm100+"); + + int32_t m = input.size(0); + int32_t n = input.size(1); + + TORCH_CHECK(n % 16 == 0, "The N dimension must be multiple of 16."); + + int multiProcessorCount = getMultiProcessorCount(); + + auto input_sf_ptr = static_cast(input_sf.data_ptr()); + auto sf_out = static_cast(output_sf.data_ptr()); + auto output_ptr = static_cast(output.data_ptr()); + at::cuda::CUDAGuard device_guard{(char)input.get_device()}; + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(input.get_device()); + + // We don't support e8m0 scales at this moment. + bool useUE8M0 = false; + + switch (input.scalar_type()) { + case torch::kHalf: { + auto input_ptr = reinterpret_cast(input.data_ptr()); + invokeFP4Quantization(m, n, input_ptr, input_sf_ptr, output_ptr, sf_out, useUE8M0, multiProcessorCount, stream); + break; + } + case torch::kBFloat16: { + auto input_ptr = reinterpret_cast<__nv_bfloat16 const*>(input.data_ptr()); + invokeFP4Quantization(m, n, input_ptr, input_sf_ptr, output_ptr, sf_out, useUE8M0, multiProcessorCount, stream); + break; + } + default: { + std::cerr << "Observing: " << input.scalar_type() << " for the input datatype which is invalid"; + throw std::runtime_error("Unsupported input data type for quantize_to_fp4."); + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_scaled_mm_entry.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_scaled_mm_entry.cu new file mode 100644 index 00000000..3a050bbc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_scaled_mm_entry.cu @@ -0,0 +1,64 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#if defined ENABLE_NVFP4 && ENABLE_NVFP4 +void cutlass_scaled_fp4_mm_sm100a_sm120a( + torch::Tensor& D, + torch::Tensor const& A, + torch::Tensor const& B, + torch::Tensor const& A_sf, + torch::Tensor const& B_sf, + torch::Tensor const& alpha); + +// SM120 specific dispatch functions +void cutlass_fp4_bf16_gemm_dispatch_sm120( + torch::Tensor& D, + torch::Tensor const& A, + torch::Tensor const& B, + torch::Tensor const& A_sf, + torch::Tensor const& B_sf, + torch::Tensor const& alpha, + int m, + int n, + int k, + cudaStream_t stream); + +void cutlass_fp4_f16_gemm_dispatch_sm120( + torch::Tensor& D, + torch::Tensor const& A, + torch::Tensor const& B, + torch::Tensor const& A_sf, + torch::Tensor const& B_sf, + torch::Tensor const& alpha, + int m, + int n, + int k, + cudaStream_t stream); +#endif + +void cutlass_scaled_fp4_mm( + torch::Tensor& D, + torch::Tensor const& A, + torch::Tensor const& B, + torch::Tensor const& A_sf, + torch::Tensor const& B_sf, + torch::Tensor const& alpha) { +#if defined ENABLE_NVFP4 && ENABLE_NVFP4 + return cutlass_scaled_fp4_mm_sm100a_sm120a(D, A, B, A_sf, B_sf, alpha); +#endif + TORCH_CHECK_NOT_IMPLEMENTED(false, "No compiled nvfp4 mm kernel."); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_scaled_mm_kernels.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_scaled_mm_kernels.cu new file mode 100644 index 00000000..40d320ac --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/nvfp4_scaled_mm_kernels.cu @@ -0,0 +1,687 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "utils.h" + +// clang-format off +#include "cutlass/cutlass.h" +#include "cutlass/gemm/collective/collective_builder.hpp" +#include "cutlass/epilogue/collective/collective_builder.hpp" +#include "cutlass/gemm/device/gemm_universal_adapter.h" +#include "cutlass/gemm/kernel/gemm_universal.hpp" +#include "cutlass/util/packed_stride.hpp" +// clang-format on + +/** + * Helper function for checking CUTLASS errors + */ +#define CUTLASS_CHECK(status) \ + { \ + cutlass::Status error = status; \ + TORCH_CHECK(error == cutlass::Status::kSuccess, cutlassGetStatusString(error)); \ + } + +using namespace cute; + +// Helper function for next power of 2 +inline uint32_t next_pow_2(uint32_t x) { + if (x == 0) return 1; + x--; + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + x |= x >> 8; + x |= x >> 16; + return x + 1; +} + +#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) || defined(CUTLASS_ARCH_MMA_SM120_SUPPORTED) || \ + defined(CUTLASS_ARCH_MMA_SM121_SUPPORTED) +// Config(half_t/bfloat16_t) for M <= 128 +template +struct KernelConfigM128 { + using OutputType = T; + using MmaTileShape = Shape<_128, _256, _256>; + using ClusterShape = Shape; + using EpilogueTile = Shape<_128, _64>; // Avoid register spilling + using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecialized1Sm; + using MainloopSchedule = cutlass::gemm::KernelTmaWarpSpecialized1SmNvf4Sm100; + const static dim3 preferred_cluster; + const static dim3 fallback_cluster; +}; +template +const dim3 KernelConfigM128::preferred_cluster(1, 4, 1); +template +const dim3 KernelConfigM128::fallback_cluster(1, 2, 1); + +// Config(half_t/bfloat16_t) for M <= 256 +template +struct KernelConfigM256 { + using OutputType = T; + using MmaTileShape = Shape<_256, _256, _256>; + using ClusterShape = Shape; + using EpilogueTile = Shape<_128, _64>; // Avoid register spilling + using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecialized2Sm; + using MainloopSchedule = cutlass::gemm::KernelTmaWarpSpecialized2SmNvf4Sm100; + const static dim3 preferred_cluster; + const static dim3 fallback_cluster; +}; +template +const dim3 KernelConfigM256::preferred_cluster(2, 4, 1); +template +const dim3 KernelConfigM256::fallback_cluster(2, 1, 1); + +// Default config(half_t/bfloat16_t) for M > 256 +template +struct KernelConfigDefault { + using OutputType = T; + using MmaTileShape = Shape<_256, _256, _256>; + using ClusterShape = Shape; + using EpilogueTile = Shape<_128, _64>; // Avoid register spilling + using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecialized2Sm; + using MainloopSchedule = cutlass::gemm::KernelTmaWarpSpecialized2SmNvf4Sm100; + const static dim3 preferred_cluster; + const static dim3 fallback_cluster; +}; +template +const dim3 KernelConfigDefault::preferred_cluster(4, 4, 1); +template +const dim3 KernelConfigDefault::fallback_cluster(2, 1, 1); + +struct KernelConfigFp32 { + using OutputType = float; + using MmaTileShape = Shape<_128, _128, _256>; + using ClusterShape = Shape; + using EpilogueTile = cutlass::epilogue::collective::EpilogueTileAuto; + using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecialized1Sm; + using MainloopSchedule = cutlass::gemm::KernelTmaWarpSpecialized1SmNvf4Sm100; + const static dim3 preferred_cluster; + const static dim3 fallback_cluster; +}; +const dim3 KernelConfigFp32::preferred_cluster = dim3(1, 4, 1); +const dim3 KernelConfigFp32::fallback_cluster = dim3(1, 2, 1); + +// SM120 specific configurations +struct sm120_fp4_config_M256 { + using ClusterShape = Shape<_1, _1, _1>; + using MmaTileShape = Shape<_128, _128, _128>; + using PerSmTileShape_MNK = Shape<_128, _128, _128>; +}; + +struct sm120_fp4_config_default { + using ClusterShape = Shape<_1, _1, _1>; + using MmaTileShape = Shape<_256, _128, _128>; + using PerSmTileShape_MNK = Shape<_256, _128, _128>; +}; + +template +struct Fp4GemmSm100 { + using Config = KernelConfig; // For generating args + using OutputType = typename KernelConfig::OutputType; + // A matrix configuration + using ElementA = cutlass::nv_float4_t; + using LayoutATag = cutlass::layout::RowMajor; + static constexpr int AlignmentA = 32; + + // B matrix configuration + using ElementB = cutlass::nv_float4_t; + using LayoutBTag = cutlass::layout::ColumnMajor; + static constexpr int AlignmentB = 32; + + // C/D matrix configuration + using ElementD = OutputType; + using ElementC = OutputType; + using LayoutCTag = cutlass::layout::RowMajor; + using LayoutDTag = cutlass::layout::RowMajor; + static constexpr int AlignmentD = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + // Kernel functional config + using ElementAccumulator = float; + using ArchTag = cutlass::arch::Sm100; + using OperatorClass = cutlass::arch::OpClassBlockScaledTensorOp; + + // Kernel Perf config + using MmaTileShape = typename KernelConfig::MmaTileShape; + using ClusterShape = typename KernelConfig::ClusterShape; + using EpilogueTile = typename KernelConfig::EpilogueTile; + using EpilogueSchedule = typename KernelConfig::EpilogueSchedule; + using MainloopSchedule = typename KernelConfig::MainloopSchedule; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + MmaTileShape, + ClusterShape, + EpilogueTile, + ElementAccumulator, + ElementAccumulator, + void, + LayoutCTag, + AlignmentC, + ElementD, + LayoutDTag, + AlignmentD, + EpilogueSchedule, + cutlass::epilogue::fusion::LinearCombination>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementA, + LayoutATag, + AlignmentA, + ElementB, + LayoutBTag, + AlignmentB, + ElementAccumulator, + MmaTileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + MainloopSchedule>::CollectiveOp; + + using GemmKernel = + cutlass::gemm::kernel::GemmUniversal, CollectiveMainloop, CollectiveEpilogue, void>; + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + using StrideA = typename Gemm::GemmKernel::StrideA; + using LayoutA = decltype(cute::make_layout(make_shape(0, 0, 0), StrideA{})); + using LayoutSFA = typename Gemm::GemmKernel::CollectiveMainloop::LayoutSFA; + using StrideB = typename Gemm::GemmKernel::StrideB; + using LayoutB = decltype(cute::make_layout(make_shape(0, 0, 0), StrideB{})); + using LayoutSFB = typename Gemm::GemmKernel::CollectiveMainloop::LayoutSFB; + using StrideC = typename Gemm::GemmKernel::StrideC; + using LayoutC = decltype(cute::make_layout(make_shape(0, 0, 0), StrideC{})); + using StrideD = typename Gemm::GemmKernel::StrideD; + using LayoutD = decltype(cute::make_layout(make_shape(0, 0, 0), StrideD{})); +}; + +// SM120 specific GEMM template +template +struct Fp4GemmSm120 { + using ElementA = cutlass::nv_float4_t; + using LayoutATag = cutlass::layout::RowMajor; + static constexpr int AlignmentA = 32; + + using ElementB = cutlass::nv_float4_t; + using LayoutBTag = cutlass::layout::ColumnMajor; + static constexpr int AlignmentB = 32; + + using ElementD = OutType; + using ElementC = OutType; + using LayoutCTag = cutlass::layout::RowMajor; + using LayoutDTag = cutlass::layout::RowMajor; + static constexpr int AlignmentD = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + + using ElementAccumulator = float; + using ArchTag = cutlass::arch::Sm120; + using OperatorClass = cutlass::arch::OpClassBlockScaledTensorOp; + + using MmaTileShape = typename Config::MmaTileShape; + using ClusterShape = typename Config::ClusterShape; + using PerSmTileShape_MNK = typename Config::PerSmTileShape_MNK; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + PerSmTileShape_MNK, + ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementAccumulator, + ElementC, + LayoutCTag, + AlignmentC, + ElementD, + LayoutDTag, + AlignmentD, + cutlass::epilogue::collective::EpilogueScheduleAuto>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementA, + LayoutATag, + AlignmentA, + ElementB, + LayoutBTag, + AlignmentB, + ElementAccumulator, + MmaTileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + cutlass::gemm::collective::KernelScheduleAuto>::CollectiveOp; + + using GemmKernel = + cutlass::gemm::kernel::GemmUniversal, CollectiveMainloop, CollectiveEpilogue, void>; + + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; +}; + +template +typename T::Gemm::Arguments args_from_options( + at::Tensor& D, + at::Tensor const& A, + at::Tensor const& B, + at::Tensor const& A_sf, + at::Tensor const& B_sf, + at::Tensor const& alpha, + int64_t M, + int64_t N, + int64_t K) { + using ElementA = typename T::Gemm::ElementA; + using ElementB = typename T::Gemm::ElementB; + using ElementSFA = cutlass::float_ue4m3_t; + using ElementSFB = cutlass::float_ue4m3_t; + using ElementD = typename T::Gemm::ElementD; + using ElementCompute = float; + using StrideA = typename T::StrideA; + using StrideB = typename T::StrideB; + using StrideD = typename T::StrideD; + using Sm1xxBlkScaledConfig = typename T::Gemm::GemmKernel::CollectiveMainloop::Sm1xxBlkScaledConfig; + + int m = static_cast(M); + int n = static_cast(N); + int k = static_cast(K); + auto stride_A = cutlass::make_cute_packed_stride(StrideA{}, {m, k, 1}); + auto stride_B = cutlass::make_cute_packed_stride(StrideB{}, {n, k, 1}); + auto stride_D = cutlass::make_cute_packed_stride(StrideD{}, {m, n, 1}); + + auto layout_SFA = Sm1xxBlkScaledConfig::tile_atom_to_shape_SFA(cute::make_shape(m, n, k, 1)); + auto layout_SFB = Sm1xxBlkScaledConfig::tile_atom_to_shape_SFB(cute::make_shape(m, n, k, 1)); + + typename T::Gemm::Arguments arguments{ + cutlass::gemm::GemmUniversalMode::kGemm, + {m, n, k, 1}, + {// Mainloop arguments + static_cast(A.data_ptr()), + stride_A, + static_cast(B.data_ptr()), + stride_B, + static_cast(A_sf.data_ptr()), + layout_SFA, + static_cast(B_sf.data_ptr()), + layout_SFB}, + { // Epilogue arguments + {}, // epilogue.thread + nullptr, + stride_D, + static_cast(D.data_ptr()), + stride_D}}; + auto& fusion_args = arguments.epilogue.thread; + fusion_args.alpha_ptr = static_cast(alpha.data_ptr()); + using KernelConfig = typename T::Config; + arguments.hw_info.cluster_shape = KernelConfig::preferred_cluster; + arguments.hw_info.cluster_shape_fallback = KernelConfig::fallback_cluster; + return arguments; +} + +template +void runGemm( + at::Tensor& D, + at::Tensor const& A, + at::Tensor const& B, + at::Tensor const& A_sf, + at::Tensor const& B_sf, + at::Tensor const& alpha, + int64_t m, + int64_t n, + int64_t k, + cudaStream_t stream) { + typename T::Gemm gemm; + auto arguments = args_from_options(D, A, B, A_sf, B_sf, alpha, m, n, k); + + size_t workspace_size = T::Gemm::get_workspace_size(arguments); + auto const workspace_options = torch::TensorOptions().dtype(torch::kUInt8).device(A.device()); + auto workspace = torch::empty(workspace_size, workspace_options); + + CUTLASS_CHECK(gemm.can_implement(arguments)); + + CUTLASS_CHECK(gemm.initialize(arguments, workspace.data_ptr(), stream)); + + CUTLASS_CHECK(gemm.run(arguments, workspace.data_ptr(), stream)); +} + +// SM120 specific args_from_options function +template +typename Gemm::Arguments args_from_options_sm120( + at::Tensor& D, + at::Tensor const& A, + at::Tensor const& B, + at::Tensor const& A_sf, + at::Tensor const& B_sf, + torch::Tensor const& alpha, + int M, + int N, + int K) { + using ElementA = typename Gemm::ElementA; + using ElementB = typename Gemm::ElementB; + using ElementD = typename Gemm::ElementD; + using ElementSFA = cutlass::float_ue4m3_t; + using ElementSFB = cutlass::float_ue4m3_t; + using ElementCompute = float; + + using StrideA = typename Gemm::GemmKernel::StrideA; + using StrideB = typename Gemm::GemmKernel::StrideB; + using StrideC = typename Gemm::GemmKernel::StrideC; + using StrideD = typename Gemm::GemmKernel::StrideD; + + using Sm1xxBlkScaledConfig = typename Gemm::GemmKernel::CollectiveMainloop::Sm1xxBlkScaledConfig; + + auto stride_A = cutlass::make_cute_packed_stride(StrideA{}, {M, K, 1}); + auto stride_B = cutlass::make_cute_packed_stride(StrideB{}, {N, K, 1}); + auto stride_D = cutlass::make_cute_packed_stride(StrideD{}, {M, N, 1}); + + auto layout_SFA = Sm1xxBlkScaledConfig::tile_atom_to_shape_SFA(cute::make_shape(M, N, K, 1)); + auto layout_SFB = Sm1xxBlkScaledConfig::tile_atom_to_shape_SFB(cute::make_shape(M, N, K, 1)); + + typename Gemm::Arguments arguments{ + cutlass::gemm::GemmUniversalMode::kGemm, + {M, N, K, 1}, + {static_cast(A.data_ptr()), + stride_A, + static_cast(B.data_ptr()), + stride_B, + static_cast(A_sf.data_ptr()), + layout_SFA, + static_cast(B_sf.data_ptr()), + layout_SFB}, + {{}, static_cast(D.data_ptr()), stride_D, static_cast(D.data_ptr()), stride_D}}; + auto& fusion_args = arguments.epilogue.thread; + fusion_args.alpha_ptr = static_cast(alpha.data_ptr()); + + return arguments; +} + +// SM120 specific runGemm function +template +void runGemmSm120( + at::Tensor& D, + at::Tensor const& A, + at::Tensor const& B, + at::Tensor const& A_sf, + at::Tensor const& B_sf, + torch::Tensor const& alpha, + int M, + int N, + int K, + cudaStream_t stream) { + Gemm gemm; + + auto arguments = args_from_options_sm120(D, A, B, A_sf, B_sf, alpha, M, N, K); + + size_t workspace_size = Gemm::get_workspace_size(arguments); + auto const workspace_options = torch::TensorOptions().dtype(torch::kUInt8).device(A.device()); + auto workspace = torch::empty(workspace_size, workspace_options); + + CUTLASS_CHECK(gemm.can_implement(arguments)); + + CUTLASS_CHECK(gemm.initialize(arguments, workspace.data_ptr(), stream)); + + CUTLASS_CHECK(gemm.run(arguments, workspace.data_ptr(), stream)); +} + +// Dispatch function to select appropriate config based on M +template +void cutlassFp4GemmDispatch( + torch::Tensor& D, + torch::Tensor const& A, + torch::Tensor const& B, + torch::Tensor const& A_sf, + torch::Tensor const& B_sf, + torch::Tensor const& alpha, + int64_t m, + int64_t n, + int64_t k, + cudaStream_t stream) { + if (m <= 128) { + // m in [1, 128] + runGemm>>(D, A, B, A_sf, B_sf, alpha, m, n, k, stream); + } else if (m <= 256) { + // m in (128, 256] + runGemm>>(D, A, B, A_sf, B_sf, alpha, m, n, k, stream); + } else { + // m in (256, inf) + runGemm>>(D, A, B, A_sf, B_sf, alpha, m, n, k, stream); + } +} + +// Dispatch function to select appropriate config based on M +template <> +void cutlassFp4GemmDispatch( + torch::Tensor& D, + torch::Tensor const& A, + torch::Tensor const& B, + torch::Tensor const& A_sf, + torch::Tensor const& B_sf, + torch::Tensor const& alpha, + int64_t m, + int64_t n, + int64_t k, + cudaStream_t stream) { + runGemm>(D, A, B, A_sf, B_sf, alpha, m, n, k, stream); +} + +// SM120 specific dispatch functions +void cutlass_fp4_bf16_gemm_dispatch_sm120( + torch::Tensor& D, + torch::Tensor const& A, + torch::Tensor const& B, + torch::Tensor const& A_sf, + torch::Tensor const& B_sf, + torch::Tensor const& alpha, + int m, + int n, + int k, + cudaStream_t stream) { + uint32_t const mp2 = std::max(static_cast(16), next_pow_2(m)); + if (mp2 <= 256) { + runGemmSm120::Gemm>( + D, A, B, A_sf, B_sf, alpha, m, n, k, stream); + } else { + runGemmSm120::Gemm>( + D, A, B, A_sf, B_sf, alpha, m, n, k, stream); + } +} + +void cutlass_fp4_f16_gemm_dispatch_sm120( + torch::Tensor& D, + torch::Tensor const& A, + torch::Tensor const& B, + torch::Tensor const& A_sf, + torch::Tensor const& B_sf, + torch::Tensor const& alpha, + int m, + int n, + int k, + cudaStream_t stream) { + uint32_t const mp2 = std::max(static_cast(16), next_pow_2(m)); + if (mp2 <= 256) { + runGemmSm120::Gemm>( + D, A, B, A_sf, B_sf, alpha, m, n, k, stream); + } else { + runGemmSm120::Gemm>( + D, A, B, A_sf, B_sf, alpha, m, n, k, stream); + } +} + +#else +template +void cutlassFp4GemmDispatch( + at::Tensor& D, + at::Tensor const& A, + at::Tensor const& B, + at::Tensor const& A_sf, + at::Tensor const& B_sf, + at::Tensor const& alpha, + int64_t m, + int64_t n, + int64_t k, + cudaStream_t stream) { + TORCH_CHECK( + false, + "Unsupported CUTLASS version. Set VLLM_CUTLASS_SRC_DIR to " + "a CUTLASS 3.8 source directory to enable support."); +} +#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) || defined(CUTLASS_ARCH_MMA_SM120_SUPPORTED) || + // defined(CUTLASS_ARCH_MMA_SM121_SUPPORTED) + +// Undefine macros from utils.h to redefine with custom signatures +#undef CHECK_CONTIGUOUS +#undef CHECK_INPUT + +#define CHECK_TYPE(x, st, m) TORCH_CHECK(x.scalar_type() == st, "Inconsistency of Tensor type:", m) +#define CHECK_TH_CUDA(x, m) TORCH_CHECK(x.is_cuda(), m, "must be a CUDA tensor") +#define CHECK_CONTIGUOUS(x, m) TORCH_CHECK(x.is_contiguous(), m, "must be contiguous") +#define CHECK_INPUT(x, st, m) \ + CHECK_TH_CUDA(x, m); \ + CHECK_CONTIGUOUS(x, m); \ + CHECK_TYPE(x, st, m) + +constexpr auto FLOAT4_E2M1X2 = at::ScalarType::Byte; +constexpr auto SF_DTYPE = at::ScalarType::Float8_e4m3fn; + +void cutlass_scaled_fp4_mm_sm100a_sm120a( + torch::Tensor& D, + torch::Tensor const& A, + torch::Tensor const& B, + torch::Tensor const& A_sf, + torch::Tensor const& B_sf, + torch::Tensor const& alpha) { + CHECK_INPUT(A, FLOAT4_E2M1X2, "a"); + CHECK_INPUT(B, FLOAT4_E2M1X2, "b"); + + CHECK_INPUT(A_sf, SF_DTYPE, "scale_a"); + CHECK_INPUT(B_sf, SF_DTYPE, "scale_b"); + + CHECK_INPUT(alpha, at::ScalarType::Float, "alpha"); + + TORCH_CHECK(A.dim() == 2, "a must be a matrix"); + TORCH_CHECK(B.dim() == 2, "b must be a matrix"); + TORCH_CHECK( + A.size(1) == B.size(1), + "a and b shapes cannot be multiplied (", + A.size(0), + "x", + A.size(1), + " and ", + B.size(0), + "x", + B.size(1), + ")"); + + auto const m = A.size(0); + auto const n = B.size(0); + auto const k = A.size(1) * 2; + + constexpr int alignment = 32; + TORCH_CHECK( + k % alignment == 0, + "Expected k to be divisible by ", + alignment, + ", but got a shape: (", + A.size(0), + "x", + A.size(1), + "), k: ", + k, + "."); + TORCH_CHECK( + n % alignment == 0, + "Expected n to be divisible by ", + alignment, + ", but got b shape: (", + B.size(0), + "x", + B.size(1), + ")."); + + auto round_up = [](int x, int y) { return (x + y - 1) / y * y; }; + int rounded_m = round_up(m, 128); + int rounded_n = round_up(n, 128); + // Since k is divisible by 32 (alignment), k / 16 is guaranteed to be an + // integer. + int rounded_k = round_up(k / 16, 4); + + TORCH_CHECK(A_sf.dim() == 2, "scale_a must be a matrix"); + TORCH_CHECK(B_sf.dim() == 2, "scale_b must be a matrix"); + TORCH_CHECK( + A_sf.size(1) == B_sf.size(1), + "scale_a and scale_b shapes cannot be multiplied (", + A_sf.size(0), + "x", + A_sf.size(1), + " and ", + B_sf.size(0), + "x", + B_sf.size(1), + ")"); + TORCH_CHECK( + A_sf.size(0) == rounded_m && A_sf.size(1) == rounded_k, + "scale_a must be padded and swizzled to a shape (", + rounded_m, + "x", + rounded_k, + "), but got a shape (", + A_sf.size(0), + "x", + A_sf.size(1), + ")"); + TORCH_CHECK( + B_sf.size(0) == rounded_n && B_sf.size(1) == rounded_k, + "scale_b must be padded and swizzled to a shape (", + rounded_n, + "x", + rounded_k, + "), but got a shape (", + B_sf.size(0), + "x", + B_sf.size(1), + ")"); + + auto out_dtype = D.dtype(); + at::cuda::CUDAGuard device_guard{(char)A.get_device()}; + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(A.get_device()); + + // Check SM version and dispatch accordingly + auto sm_version = getSMVersion(); + + if (sm_version == 120) { + // Use SM120 specific dispatch + if (out_dtype == at::ScalarType::Half) { + cutlass_fp4_f16_gemm_dispatch_sm120(D, A, B, A_sf, B_sf, alpha, m, n, k, stream); + } else if (out_dtype == at::ScalarType::BFloat16) { + cutlass_fp4_bf16_gemm_dispatch_sm120(D, A, B, A_sf, B_sf, alpha, m, n, k, stream); + } else { + TORCH_CHECK(false, "Unsupported output data type of nvfp4 mm sm120 (", out_dtype, ")"); + } + } else { + // Use SM100 dispatch for other architectures + if (out_dtype == at::ScalarType::Half) { + cutlassFp4GemmDispatch(D, A, B, A_sf, B_sf, alpha, m, n, k, stream); + } else if (out_dtype == at::ScalarType::BFloat16) { + cutlassFp4GemmDispatch(D, A, B, A_sf, B_sf, alpha, m, n, k, stream); + } else if (out_dtype == at::ScalarType::Float) { + cutlassFp4GemmDispatch(D, A, B, A_sf, B_sf, alpha, m, n, k, stream); + } else { + TORCH_CHECK(false, "Unsupported output data type of nvfp4 mm"); + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/per_tensor_quant_fp8.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/per_tensor_quant_fp8.cu new file mode 100644 index 00000000..7afff779 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/per_tensor_quant_fp8.cu @@ -0,0 +1,123 @@ +#include +#include + +#include +#include +#include + +#include "utils.h" + +template +__global__ void +per_tensor_absmax_kernel(const T* __restrict__ input, float* __restrict__ output_s, const int64_t num_elements) { + float max_value = 0.0f; + unsigned int tid = threadIdx.x; + unsigned int gid = blockIdx.x * blockDim.x + threadIdx.x; + const int grid_size = blockDim.x * gridDim.x; + + constexpr uint32_t vec_size = 16 / sizeof(T); + using vec_t = flashinfer::vec_t; + + const int32_t num_vec_elems = num_elements / vec_size; + + for (int32_t i = gid; i < num_vec_elems; i += grid_size) { + vec_t input_vec; + input_vec.cast_load(input + i * vec_size); + +#pragma unroll + for (uint32_t j = 0; j < vec_size; ++j) { + float val = static_cast(input_vec[j]); + max_value = fmaxf(max_value, fabsf(val)); + } + } + + const int32_t remaining_start = num_vec_elems * vec_size; + for (int32_t idx = remaining_start + gid; idx < num_elements; idx += grid_size) { + float val = static_cast(input[idx]); + max_value = fmaxf(max_value, fabsf(val)); + } + + max_value = blockReduceMax(max_value); + + if (tid == 0) { + atomicMaxFloat(output_s, max_value / FP8_E4M3_MAX); + } +} + +template +__global__ void per_tensor_quant_fp8_kernel( + const T* __restrict__ input, + DST_DTYPE* __restrict__ output, + const float* __restrict__ scale, + const int64_t num_elements) { + const int gid = blockIdx.x * blockDim.x + threadIdx.x; + const int grid_size = blockDim.x * gridDim.x; + const float scale_val = 1.0f / (*scale); + + // We want to store 128 bits of data at a time. 16 = 128 / 8 bits + // Load is already vectorized, so 16 elements work for T. + const uint32_t VEC_SIZE = 16; + using vec_t = flashinfer::vec_t; + + const int32_t num_vec_elems = num_elements / VEC_SIZE; + + for (int32_t i = gid; i < num_vec_elems; i += grid_size) { + vec_t input_vec; + input_vec.cast_load(input + i * VEC_SIZE); + + DST_DTYPE output_arr[VEC_SIZE]; +#pragma unroll + for (uint32_t j = 0; j < VEC_SIZE; ++j) { + float val = fmax(fmin(static_cast(input_vec[j]) * scale_val, FP8_E4M3_MAX), -FP8_E4M3_MAX); +#if !defined(USE_ROCM) || defined(HIP_FP8_TYPE_E4M3) + output_arr[j] = static_cast(val); +#else + output_arr[j] = c10::Float8_e4m3fnuz( + __hip_cvt_float_to_fp8(val, fp8::fp8_type::__default_saturation, fp8::fp8_type::__default_interpret), + c10::Float8_e4m3fnuz::from_bits()); +#endif + } + *(uint4*)(output + i * VEC_SIZE) = *(uint4*)output_arr; + } + + const int32_t remaining_start = num_vec_elems * VEC_SIZE; + for (int32_t idx = remaining_start + gid; idx < num_elements; idx += grid_size) { + float val = fmax(-FP8_E4M3_MAX, fmin(static_cast(input[idx]) * scale_val, FP8_E4M3_MAX)); +#if !defined(USE_ROCM) || defined(HIP_FP8_TYPE_E4M3) + output[idx] = static_cast(val); +#else + output[idx] = c10::Float8_e4m3fnuz( + __hip_cvt_float_to_fp8(val, fp8::fp8_type::__default_saturation, fp8::fp8_type::__default_interpret), + c10::Float8_e4m3fnuz::from_bits()); +#endif + } +} + +void sgl_per_tensor_quant_fp8(torch::Tensor input, torch::Tensor output_q, torch::Tensor output_s, bool is_static) { + CHECK_INPUT(input); + CHECK_INPUT(output_q); + CHECK_INPUT(output_s); + + const int block_size = 256; + const int num_elements = input.numel(); + const int num_blocks = min((num_elements + block_size - 1) / block_size, 1024); + + dim3 grid(num_blocks); + dim3 block(block_size); + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), scalar_t, [&] { + if (is_static == false) { + per_tensor_absmax_kernel<<>>( + static_cast(input.data_ptr()), static_cast(output_s.data_ptr()), num_elements); + } + + per_tensor_quant_fp8_kernel<<>>( + static_cast(input.data_ptr()), + static_cast<__nv_fp8_e4m3*>(output_q.data_ptr()), + static_cast(output_s.data_ptr()), + num_elements); + return true; + }); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/per_token_group_quant_8bit.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/per_token_group_quant_8bit.cu new file mode 100644 index 00000000..4489aab2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/per_token_group_quant_8bit.cu @@ -0,0 +1,217 @@ +#include +#include + +#include +#include + +#include "utils.h" + +__device__ __forceinline__ float GroupReduceMax(float val, const int tid) { + unsigned mask = threadIdx.x % 32 >= 16 ? 0xffff0000 : 0x0000ffff; + + val = fmaxf(val, __shfl_xor_sync(mask, val, 8)); + val = fmaxf(val, __shfl_xor_sync(mask, val, 4)); + val = fmaxf(val, __shfl_xor_sync(mask, val, 2)); + val = fmaxf(val, __shfl_xor_sync(mask, val, 1)); + return val; +} + +template < + typename T, + typename DST_DTYPE, + bool IS_COLUMN_MAJOR = false, + bool SCALE_UE8M0 = false, + typename scale_packed_t = std::conditional_t> +__global__ void per_token_group_quant_8bit_kernel( + const T* __restrict__ input, + void* __restrict__ output_q, + scale_packed_t* __restrict__ output_s, + const int group_size, + const int num_groups, + const int groups_per_block, + const float eps, + const float min_8bit, + const float max_8bit, + const int num_groups_per_row = 0, + const int scale_stride = 0) { + const int threads_per_group = 16; + const int64_t local_group_id = threadIdx.x / threads_per_group; + const int lane_id = threadIdx.x % threads_per_group; + + const int64_t block_group_id = blockIdx.x * groups_per_block; + const int64_t global_group_id = block_group_id + local_group_id; + const int64_t block_group_offset = global_group_id * group_size; + + float local_absmax = eps; + + using scale_element_t = std::conditional_t; + static_assert(sizeof(scale_packed_t) % sizeof(scale_element_t) == 0); + + const T* group_input = input + block_group_offset; + DST_DTYPE* group_output = static_cast(output_q) + block_group_offset; + scale_element_t* scale_output; + + if constexpr (IS_COLUMN_MAJOR) { + const int num_elems_per_pack = static_cast(sizeof(scale_packed_t) / sizeof(scale_element_t)); + const int row_idx = global_group_id / num_groups_per_row; + const int col_idx_unpacked = global_group_id % num_groups_per_row; + const int col_idx = col_idx_unpacked / num_elems_per_pack; + const int pack_idx = col_idx_unpacked % num_elems_per_pack; + scale_output = reinterpret_cast(output_s) + + (col_idx * scale_stride * num_elems_per_pack + row_idx * num_elems_per_pack + pack_idx); + } else { + static_assert(!SCALE_UE8M0); + scale_output = output_s + global_group_id; + } + + constexpr uint32_t vec_size = 16 / sizeof(T); + using vec_t = flashinfer::vec_t; + + const int32_t num_vec_elems = group_size / vec_size; + + for (int32_t i = lane_id; i < num_vec_elems; i += 16) { + vec_t input_vec; + input_vec.cast_load(group_input + i * vec_size); + +#pragma unroll + for (uint32_t j = 0; j < vec_size; ++j) { + float val = static_cast(input_vec[j]); + float abs_val = fabsf(val); + local_absmax = fmaxf(local_absmax, abs_val); + } + } + + local_absmax = GroupReduceMax(local_absmax, lane_id); + + float y_s = local_absmax / max_8bit; + if constexpr (SCALE_UE8M0) { + y_s = exp2f(ceilf(log2f(fmaxf(y_s, 1e-10f)))); + } + + // TODO can optimize + scale_element_t y_s_quant; + if constexpr (SCALE_UE8M0) { + y_s_quant = (uint8_t)(((int)log2f(y_s)) + 127); + } else { + y_s_quant = y_s; + } + + if (lane_id == 0) { + *scale_output = y_s_quant; + } + + for (int32_t i = lane_id; i < num_vec_elems; i += 16) { + vec_t input_vec; + input_vec.cast_load(group_input + i * vec_size); + +#pragma unroll + for (uint32_t j = 0; j < vec_size; ++j) { + float val = static_cast(input_vec[j]); + float q_val = fminf(fmaxf(val / y_s, min_8bit), max_8bit); + group_output[i * vec_size + j] = DST_DTYPE(q_val); + } + } +} + +void sgl_per_token_group_quant_8bit( + torch::Tensor input, + torch::Tensor output_q, + torch::Tensor output_s, + int64_t group_size, + double eps, + double min_8bit, + double max_8bit, + bool scale_ue8m0) { + CHECK_INPUT(input); + CHECK_INPUT(output_q); + + const int num_groups = input.numel() / group_size; + + CHECK_EQ(input.numel() % group_size, 0); + CHECK_EQ(output_s.dim(), 2); + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + constexpr int THREADS_PER_GROUP = 16; + + int groups_per_block = 1; + + if (num_groups % 16 == 0) { + groups_per_block = 16; + } else if (num_groups % 8 == 0) { + groups_per_block = 8; + } else if (num_groups % 4 == 0) { + groups_per_block = 4; + } else if (num_groups % 2 == 0) { + groups_per_block = 2; + } + + auto dst_type = output_q.scalar_type(); + const int num_blocks = num_groups / groups_per_block; + const int num_threads = groups_per_block * THREADS_PER_GROUP; + + const bool is_column_major = output_s.stride(0) < output_s.stride(1); + const int hidden_dim = input.size(input.dim() - 1); + const int num_groups_per_row = hidden_dim / group_size; + const int scale_stride = output_s.stride(1); + +#define LAUNCH_KERNEL(T, DST_DTYPE) \ + do { \ + dim3 grid(num_blocks); \ + dim3 block(num_threads); \ + if (is_column_major) { \ + if (scale_ue8m0) { \ + per_token_group_quant_8bit_kernel<<>>( \ + static_cast(input.data_ptr()), \ + output_q.data_ptr(), \ + static_cast(output_s.data_ptr()), \ + group_size, \ + num_groups, \ + groups_per_block, \ + (float)eps, \ + (float)min_8bit, \ + (float)max_8bit, \ + num_groups_per_row, \ + scale_stride); \ + } else { \ + per_token_group_quant_8bit_kernel<<>>( \ + static_cast(input.data_ptr()), \ + output_q.data_ptr(), \ + static_cast(output_s.data_ptr()), \ + group_size, \ + num_groups, \ + groups_per_block, \ + (float)eps, \ + (float)min_8bit, \ + (float)max_8bit, \ + num_groups_per_row, \ + scale_stride); \ + } \ + } else { \ + assert(!scale_ue8m0); \ + per_token_group_quant_8bit_kernel<<>>( \ + static_cast(input.data_ptr()), \ + output_q.data_ptr(), \ + static_cast(output_s.data_ptr()), \ + group_size, \ + num_groups, \ + groups_per_block, \ + (float)eps, \ + (float)min_8bit, \ + (float)max_8bit); \ + } \ + } while (0) + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), scalar_t, [&] { + if (dst_type == at::ScalarType::Char) { + LAUNCH_KERNEL(scalar_t, int8_t); + return true; + } else if (dst_type == at::ScalarType::Float8_e4m3fn) { + LAUNCH_KERNEL(scalar_t, __nv_fp8_e4m3); + return true; + } + return false; + }); + +#undef LAUNCH_KERNEL +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/per_token_group_quant_8bit_v2.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/per_token_group_quant_8bit_v2.cu new file mode 100644 index 00000000..53903e6a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/per_token_group_quant_8bit_v2.cu @@ -0,0 +1,514 @@ +#include +#include + +#include +#include + +#include "utils.h" + +template +__device__ __forceinline__ float GroupReduceMax(float val, const int tid) { + unsigned mask = 0xffffffff; + + static_assert( + (THREADS_PER_SUBWARP & (THREADS_PER_SUBWARP - 1)) == 0 && THREADS_PER_SUBWARP <= 16 && THREADS_PER_SUBWARP >= 1, + "THREADS_PER_SUBWARP must be 1, 2, 4, 8, or 16"); + + if constexpr (THREADS_PER_SUBWARP >= 16) { + val = fmaxf(val, __shfl_xor_sync(mask, val, 8)); + } + if constexpr (THREADS_PER_SUBWARP >= 8) { + val = fmaxf(val, __shfl_xor_sync(mask, val, 4)); + } + if constexpr (THREADS_PER_SUBWARP >= 4) { + val = fmaxf(val, __shfl_xor_sync(mask, val, 2)); + } + if constexpr (THREADS_PER_SUBWARP >= 2) { + val = fmaxf(val, __shfl_xor_sync(mask, val, 1)); + } + return val; +} + +__device__ __forceinline__ float silu(const float& val) { +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + float half = 0.5f * val; + float t = __tanhf(half); + return half * (1.0f + t); +#else + return val / (1.0f + __expf(-val)); +#endif +} + +__device__ float2 fmul2_rn(float2 a, float2 b) { +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + return __fmul2_rn(a, b); +#else + float2 result; + result.x = a.x * b.x; + result.y = a.y * b.y; + return result; +#endif +} + +// Copied and modified from DeepEP +__forceinline__ __device__ float fast_pow2(int x) { + // We can ensure `-126 <= x and x <= 127` + uint32_t bits_x = (x + 127) << 23; + return *reinterpret_cast(&bits_x); +} + +// Copied and modified from DeepEP +__forceinline__ __device__ int fast_log2_ceil(float x) { + auto bits_x = *reinterpret_cast(&x); + auto exp_x = (bits_x >> 23) & 0xff; + auto man_bits = bits_x & ((1 << 23) - 1); + return exp_x - 127 + (man_bits != 0); +} + +// Copied and modified from DeepEP +template +__forceinline__ __device__ void calculate_fp8_scales(float amax, float& scale, float& scale_inv) { + constexpr float MAX_8BIT_INV = 1.0f / dtype_info::MAX; + if constexpr (ROUND_SCALE) { + auto exp_scale_inv = fast_log2_ceil(amax * MAX_8BIT_INV); + scale = fast_pow2(-exp_scale_inv); + scale_inv = fast_pow2(exp_scale_inv); + } else { + scale_inv = amax * MAX_8BIT_INV; + scale = dtype_info::MAX / amax; + } +} + +// Copied and modified from DeepEP +template > +__forceinline__ __device__ OUT_DTYPE_T extract_required_scale_format(float value) { + if constexpr (SCALE_UE8M0) { + return static_cast((*reinterpret_cast(&value)) >> 23); + } else { + return value; + } +} + +__device__ __forceinline__ void st_global(const int4* ptr, const int4& value) { + asm volatile( + "st.global.v4.s32 [%0], {%1, %2, %3, %4};" ::"l"(ptr), "r"(value.x), "r"(value.y), "r"(value.z), "r"(value.w)); +} + +__device__ __forceinline__ int4 ld_global_nc(const int4* ptr) { + int4 ret; + asm volatile("ld.global.nc.v4.s32 {%0, %1, %2, %3}, [%4];" + : "=r"(ret.x), "=r"(ret.y), "=r"(ret.z), "=r"(ret.w) + : "l"(ptr)); + return ret; +} + +template +struct DtypeInfo; + +template <> +struct DtypeInfo { + static constexpr float MIN = -128; + static constexpr float MAX = 127; +}; + +template <> +struct DtypeInfo { + static constexpr float MIN = -448; + static constexpr float MAX = 448; +}; + +template +__device__ __forceinline__ int compute_input_group_start_offset( + int expert_idx, + int token_idx, + int hidden_dim_group_idx, + int hidden_size, + int num_tokens_per_expert, + int group_size) { + return expert_idx * num_tokens_per_expert * hidden_size * (FUSE_SILU_AND_MUL ? 2 : 1) + + token_idx * hidden_size * (FUSE_SILU_AND_MUL ? 2 : 1) + hidden_dim_group_idx * group_size; +} + +constexpr float LOCAL_ABSMAX_ABS = 1e-10; +constexpr uint32_t INPUT_PRIMARY_VEC_NUM_BYTES = 32; + +struct NaiveScheduler { + static void compute_exec_config( + int threads_per_subwarp, + int num_local_experts, + int hidden_dim_num_groups, + int num_groups, + int& subwarps_per_block, + dim3& grid, + dim3& block) { + subwarps_per_block = ([=]() -> int { + if (num_groups % 16 == 0) { + return 16; + } else if (num_groups % 8 == 0) { + return 8; + } else if (num_groups % 4 == 0) { + return 4; + } else if (num_groups % 2 == 0) { + return 2; + } + return 1; + })(); + grid = dim3(num_groups / subwarps_per_block); + block = dim3(subwarps_per_block * threads_per_subwarp); + } + + template + __device__ __forceinline__ static void execute( + const int subwarps_per_block, + const int hidden_dim_num_groups, + const int32_t* masked_m, + const int num_tokens_per_expert, + FUNC fn) { + constexpr int expert_idx = 0; + + const int64_t subwarp_id = threadIdx.x / THREADS_PER_SUBWARP; + const int lane_id = threadIdx.x % THREADS_PER_SUBWARP; + + const int64_t block_group_id = blockIdx.x * subwarps_per_block; + const int64_t group_id = block_group_id + subwarp_id; + + int64_t input_group_start_offset; + if constexpr (!FUSE_SILU_AND_MUL) { + input_group_start_offset = group_id * GROUP_SIZE; + } + + const int token_idx = group_id / hidden_dim_num_groups; + // At the hidden_size dimension, we are handling idx-th group + const int hidden_dim_group_idx = group_id % hidden_dim_num_groups; + + if constexpr (FUSE_SILU_AND_MUL) { + const int hidden_size = hidden_dim_num_groups * GROUP_SIZE; + input_group_start_offset = compute_input_group_start_offset( + expert_idx, token_idx, hidden_dim_group_idx, hidden_size, num_tokens_per_expert, GROUP_SIZE); + } + + fn(expert_idx, token_idx, hidden_dim_group_idx, lane_id, input_group_start_offset); + } +}; + +struct MaskedLayoutScheduler { + // TODO can be dynamically determined (which may be good when num rank is small) + static constexpr int TOKEN_DIM_BLOCK_NUM_PER_EXPERT = 1024; + static constexpr int SUBWARPS_PER_BLOCK = 16; + + static void compute_exec_config( + int threads_per_subwarp, + int num_local_experts, + int hidden_dim_num_groups, + int num_groups, + int& subwarps_per_block, + dim3& grid, + dim3& block) { + subwarps_per_block = SUBWARPS_PER_BLOCK; + TORCH_CHECK(hidden_dim_num_groups % subwarps_per_block == 0); + grid = dim3(hidden_dim_num_groups / subwarps_per_block, TOKEN_DIM_BLOCK_NUM_PER_EXPERT, num_local_experts); + block = dim3(subwarps_per_block * threads_per_subwarp); + } + + template + __device__ __forceinline__ static void execute( + const int subwarps_per_block, + const int hidden_dim_num_groups, + const int32_t* masked_m, + const int num_tokens_per_expert, + FUNC fn) { + const int64_t subwarp_id = threadIdx.x / THREADS_PER_SUBWARP; + const int lane_id = threadIdx.x % THREADS_PER_SUBWARP; + + const int expert_idx = blockIdx.z; + const int token_idx_start = blockIdx.y; + + const int64_t hidden_dim_group_idx = blockIdx.x * SUBWARPS_PER_BLOCK + subwarp_id; + + const int curr_expert_token_num = masked_m[expert_idx]; + + for (int token_idx = token_idx_start; token_idx < curr_expert_token_num; + token_idx += TOKEN_DIM_BLOCK_NUM_PER_EXPERT) { + const int hidden_size = hidden_dim_num_groups * GROUP_SIZE; + const int64_t input_group_start_offset = compute_input_group_start_offset( + expert_idx, token_idx, hidden_dim_group_idx, hidden_size, num_tokens_per_expert, GROUP_SIZE); + fn(expert_idx, token_idx, hidden_dim_group_idx, lane_id, input_group_start_offset); + } + } +}; + +template < + typename SCHEDULER, + int GROUP_SIZE, + int THREADS_PER_SUBWARP, + typename T, + typename DST_DTYPE, + bool IS_COLUMN_MAJOR = false, + bool SCALE_UE8M0 = false, + bool FUSE_SILU_AND_MUL = false, + typename scale_packed_t = std::conditional_t> +__global__ void per_token_group_quant_8bit_kernel( + const T* __restrict__ input, + DST_DTYPE* __restrict__ output_q, + scale_packed_t* __restrict__ output_s, + const int32_t* __restrict__ masked_m, + const int subwarps_per_block, + const int hidden_dim_num_groups, + // TODO can this be removed? + const int scale_expert_stride, + const int scale_hidden_stride, + const int num_tokens_per_expert) { + using dst_dtype_info = DtypeInfo; + using scale_element_t = std::conditional_t; + static_assert(sizeof(scale_packed_t) % sizeof(scale_element_t) == 0); + + SCHEDULER::execute( + subwarps_per_block, + hidden_dim_num_groups, + masked_m, + num_tokens_per_expert, + [&](const int expert_idx, + const int token_idx, + const int hidden_dim_group_idx, + const int lane_id, + const int input_group_start_offset) { + constexpr uint32_t INPUT_PRIMARY_VEC_SIZE = INPUT_PRIMARY_VEC_NUM_BYTES / sizeof(T); + constexpr uint32_t INPUT_PRIMARY_INT4_SIZE = INPUT_PRIMARY_VEC_NUM_BYTES / sizeof(int4); + + const int offset_num_groups = expert_idx * num_tokens_per_expert * hidden_dim_num_groups + + token_idx * hidden_dim_num_groups + hidden_dim_group_idx; + + int4 input_primary_int4[INPUT_PRIMARY_INT4_SIZE]; + T* input_primary_vec = reinterpret_cast(input_primary_int4); + static_assert(sizeof(input_primary_vec[0]) * INPUT_PRIMARY_VEC_SIZE == sizeof(input_primary_int4)); + + int4 input_secondary_int4[INPUT_PRIMARY_INT4_SIZE]; + T* input_secondary_vec = reinterpret_cast(input_secondary_int4); + static_assert(sizeof(input_secondary_vec[0]) * INPUT_PRIMARY_VEC_SIZE == sizeof(input_secondary_int4)); + +#pragma unroll + for (uint32_t j = 0; j < INPUT_PRIMARY_INT4_SIZE; ++j) { + input_primary_int4[j] = ld_global_nc( + reinterpret_cast(input + input_group_start_offset + lane_id * INPUT_PRIMARY_VEC_SIZE) + j); + } + if constexpr (FUSE_SILU_AND_MUL) { + const int secondary_offset = hidden_dim_num_groups * GROUP_SIZE; +#pragma unroll + for (uint32_t j = 0; j < INPUT_PRIMARY_INT4_SIZE; ++j) { + input_secondary_int4[j] = ld_global_nc( + reinterpret_cast( + input + input_group_start_offset + lane_id * INPUT_PRIMARY_VEC_SIZE + secondary_offset) + + j); + } + } + + constexpr int num_elems_per_pack = static_cast(sizeof(scale_packed_t) / sizeof(scale_element_t)); + scale_element_t* scale_output; + if constexpr (IS_COLUMN_MAJOR) { + constexpr int scale_token_stride = 1; + + const int hidden_idx_packed = hidden_dim_group_idx / num_elems_per_pack; + const int pack_idx = hidden_dim_group_idx % num_elems_per_pack; + scale_output = reinterpret_cast(output_s) + + (expert_idx * scale_expert_stride * num_elems_per_pack + + hidden_idx_packed * scale_hidden_stride * num_elems_per_pack + + token_idx * scale_token_stride * num_elems_per_pack + pack_idx); + } else { + static_assert(!SCALE_UE8M0); + scale_output = output_s + offset_num_groups; + } + + // can speed up if too slow + if constexpr (IS_COLUMN_MAJOR and SCALE_UE8M0) { + const int remainder_num_groups = hidden_dim_num_groups % num_elems_per_pack; + if ((remainder_num_groups != 0) and (hidden_dim_group_idx == hidden_dim_num_groups - 1) and + (lane_id < num_elems_per_pack - remainder_num_groups)) { + const int shift = 1 + lane_id; + *(scale_output + shift) = 0; + } + } + + float local_absmax = LOCAL_ABSMAX_ABS; + +#pragma unroll + for (uint32_t j = 0; j < INPUT_PRIMARY_VEC_SIZE; ++j) { + float val; + if constexpr (FUSE_SILU_AND_MUL) { + // TODO maybe vectorize + T val_lowprec = static_cast(silu(static_cast(input_primary_vec[j]))) * input_secondary_vec[j]; + val = static_cast(val_lowprec); + input_primary_vec[j] = val_lowprec; + } else { + val = static_cast(input_primary_vec[j]); + } + + float abs_val = fabsf(val); + local_absmax = fmaxf(local_absmax, abs_val); + } + + local_absmax = GroupReduceMax(local_absmax, lane_id); + + float y_scale, y_scale_inv; + calculate_fp8_scales(local_absmax, y_scale, y_scale_inv); + float2 y_scale_repeated = {y_scale, y_scale}; + + if (lane_id == 0) { + *scale_output = extract_required_scale_format(y_scale_inv); + } + + int4 output_buf; + static_assert(sizeof(output_buf) == INPUT_PRIMARY_VEC_SIZE * sizeof(DST_DTYPE)); + + if constexpr (std::is_same_v) { + const auto output_buf_ptr = reinterpret_cast<__nv_fp8x2_storage_t*>(&output_buf); + static_assert(sizeof(output_buf) == INPUT_PRIMARY_VEC_SIZE / 2 * sizeof(__nv_fp8x2_storage_t)); + static_assert(INPUT_PRIMARY_VEC_SIZE % 2 == 0); + +#pragma unroll + for (uint32_t j = 0; j < INPUT_PRIMARY_VEC_SIZE; j += 2) { + float2 inputx2 = {static_cast(input_primary_vec[j]), static_cast(input_primary_vec[j + 1])}; + float2 outputx2 = fmul2_rn(inputx2, y_scale_repeated); + + outputx2.x = fminf(fmaxf(outputx2.x, dst_dtype_info::MIN), dst_dtype_info::MAX); + outputx2.y = fminf(fmaxf(outputx2.y, dst_dtype_info::MIN), dst_dtype_info::MAX); + + output_buf_ptr[j / 2] = __nv_cvt_float2_to_fp8x2(outputx2, __NV_SATFINITE, __NV_E4M3); + } + } else { + const auto output_buf_ptr = reinterpret_cast(&output_buf); + +#pragma unroll + for (uint32_t j = 0; j < INPUT_PRIMARY_VEC_SIZE; ++j) { + float val = static_cast(input_primary_vec[j]); + float q_val = fminf(fmaxf(val * y_scale, dst_dtype_info::MIN), dst_dtype_info::MAX); + output_buf_ptr[j] = DST_DTYPE(q_val); + } + } + + st_global( + reinterpret_cast(output_q + offset_num_groups * GROUP_SIZE + lane_id * INPUT_PRIMARY_VEC_SIZE), + output_buf); + }); +} + +void sgl_per_token_group_quant_8bit_v2( + // vanilla: (num_tokens, hidden_size) + // fuse_silu_and_mul: (num_tokens, hidden_size * 2) + // fuse_silu_and_mul + masked_layout: (num_experts, num_tokens-with-padding, hidden_size * 2) + torch::Tensor input, + torch::Tensor output_q, + torch::Tensor output_s, + int64_t group_size, + double eps, + double min_8bit, + double max_8bit, + bool scale_ue8m0, + bool fuse_silu_and_mul, + const std::optional& masked_m) { + CHECK_INPUT(input); + CHECK_INPUT(output_q); + TORCH_CHECK(input.numel() > 0); + + TORCH_CHECK(std::abs(LOCAL_ABSMAX_ABS - eps) < 1e-13); + + CHECK_EQ(input.numel() % group_size, 0); + const int num_groups = static_cast(input.numel()) / group_size / (fuse_silu_and_mul ? 2 : 1); + + const bool masked_layout = masked_m.has_value(); + TORCH_CHECK(output_s.dim() == (masked_layout ? 3 : 2)); + + const int num_local_experts = masked_layout ? input.size(0) : 1; + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + auto dst_type = output_q.scalar_type(); + + const bool is_column_major = output_s.stride(-2) < output_s.stride(-1); + const int hidden_dim_num_groups = static_cast(output_q.size(-1)) / group_size; + const int num_tokens_per_expert = static_cast(output_q.size(-2)); + const int scale_expert_stride = masked_layout ? static_cast(output_s.stride(0)) : 0; + const int scale_hidden_stride = static_cast(output_s.stride(-1)); + +#define LAUNCH_KERNEL_INNER(SCHEDULER, GROUP_SIZE, THREADS_PER_SUBWARP, T, DST_DTYPE, output_s_dtype, ...) \ + do { \ + int subwarps_per_block; \ + dim3 grid, block; \ + SCHEDULER::compute_exec_config( \ + THREADS_PER_SUBWARP, num_local_experts, hidden_dim_num_groups, num_groups, subwarps_per_block, grid, block); \ + \ + per_token_group_quant_8bit_kernel \ + <<>>( \ + static_cast(input.data_ptr()), \ + static_cast(output_q.data_ptr()), \ + static_cast(output_s.data_ptr()), \ + static_cast(masked_m.has_value() ? masked_m->data_ptr() : 0), \ + subwarps_per_block, \ + hidden_dim_num_groups, \ + scale_expert_stride, \ + scale_hidden_stride, \ + num_tokens_per_expert); \ + } while (0) + +#define LAUNCH_KERNEL(GROUP_SIZE, T, DST_DTYPE) \ + do { \ + constexpr int THREADS_PER_SUBWARP = GROUP_SIZE / 16; \ + TORCH_CHECK(THREADS_PER_SUBWARP* INPUT_PRIMARY_VEC_NUM_BYTES == group_size * sizeof(T)); \ + \ + using dst_dtype_info = DtypeInfo; \ + CHECK_EQ(dst_dtype_info::MIN, min_8bit); \ + CHECK_EQ(dst_dtype_info::MAX, max_8bit); \ + \ + if (is_column_major) { \ + if (scale_ue8m0) { \ + if (fuse_silu_and_mul) { \ + if (masked_layout) { \ + LAUNCH_KERNEL_INNER( \ + MaskedLayoutScheduler, GROUP_SIZE, THREADS_PER_SUBWARP, T, DST_DTYPE, uint32_t, true, true, true); \ + } else { \ + LAUNCH_KERNEL_INNER( \ + NaiveScheduler, GROUP_SIZE, THREADS_PER_SUBWARP, T, DST_DTYPE, uint32_t, true, true, true); \ + } \ + } else { \ + LAUNCH_KERNEL_INNER(NaiveScheduler, GROUP_SIZE, THREADS_PER_SUBWARP, T, DST_DTYPE, uint32_t, true, true); \ + } \ + } else { \ + LAUNCH_KERNEL_INNER(NaiveScheduler, GROUP_SIZE, THREADS_PER_SUBWARP, T, DST_DTYPE, float, true); \ + } \ + } else { \ + LAUNCH_KERNEL_INNER(NaiveScheduler, GROUP_SIZE, THREADS_PER_SUBWARP, T, DST_DTYPE, float, false); \ + } \ + } while (0) + +#define LAUNCH_KERNEL_OUTER(...) \ + switch (group_size) { \ + case 16: \ + LAUNCH_KERNEL(16, __VA_ARGS__); \ + break; \ + case 32: \ + LAUNCH_KERNEL(32, __VA_ARGS__); \ + break; \ + case 64: \ + LAUNCH_KERNEL(64, __VA_ARGS__); \ + break; \ + case 128: \ + LAUNCH_KERNEL(128, __VA_ARGS__); \ + break; \ + default: \ + TORCH_CHECK(false, "Unsupported group_size"); \ + } \ + while (0) + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP16(input.scalar_type(), scalar_t, [&] { + if (dst_type == at::ScalarType::Char) { + LAUNCH_KERNEL_OUTER(scalar_t, int8_t); + return true; + } else if (dst_type == at::ScalarType::Float8_e4m3fn) { + LAUNCH_KERNEL_OUTER(scalar_t, c10::Float8_e4m3fn); + return true; + } + return false; + }); + +#undef LAUNCH_KERNEL +#undef LAUNCH_KERNEL_INNER +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/per_token_quant_fp8.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/per_token_quant_fp8.cu new file mode 100644 index 00000000..e7bc3fab --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/per_token_quant_fp8.cu @@ -0,0 +1,242 @@ +#include + +#include +#include + +#include "utils.h" + +static constexpr int kWarpSize = 32; + +// --------------------------------------------------------------------------- +// 1. Warp‑local, no shared memory +// • One warp handles one token. +// • Eight tokens per 256‑thread CTA. +// --------------------------------------------------------------------------- +template +__global__ void per_token_quant_fp8_kernel( + const T* __restrict__ input, + DST_DTYPE* __restrict__ output_q, + float* __restrict__ output_s, + const int64_t hidden_dim, + const int64_t num_tokens) { + const int warp_id = threadIdx.x / kWarpSize; // 0‑7 (8 warps) + const int lane_id = threadIdx.x & (kWarpSize - 1); // 0‑31 + const int token_id = blockIdx.x * kTokensPerCTA + warp_id; + if (token_id >= num_tokens) return; + + // Global tensors for this token + const T* token_input = input + token_id * hidden_dim; + DST_DTYPE* token_output = output_q + token_id * hidden_dim; + float* token_scale = output_s + token_id; + + // + // Pass-1: Perform a warp reduce to find the max_value of a token's hidden_dim + // + float max_value = 0.f; + using vec_t = flashinfer::vec_t; + const int32_t num_vec_elems = hidden_dim / kVecSize; + + for (int32_t i = lane_id; i < num_vec_elems; i += kWarpSize) { + vec_t input_vec; + input_vec.cast_load(token_input + i * kVecSize); + +#pragma unroll + for (uint32_t j = 0; j < kVecSize; ++j) { + max_value = fmaxf(max_value, fabsf(static_cast(input_vec[j]))); + } + } + + float warp_max = warpReduceMax(max_value); + + __shared__ float scale; + scale = warp_max / FP8_E4M3_MAX; + // Broadcast scale + if (lane_id == 0) { + token_scale[0] = scale; + } + float scale_inv = (scale == 0.f) ? 0.f : 1.0f / scale; + + // + // Pass-2: quantize and write back + // + for (int i = lane_id; i < num_vec_elems; i += kWarpSize) { + vec_t input_vec; + input_vec.cast_load(token_input + i * kVecSize); + DST_DTYPE output_arr[kVecSize]; +#pragma unroll + for (uint32_t j = 0; j < kVecSize; ++j) { + float val = static_cast(input_vec[j]) * scale_inv; + val = fmaxf(fminf(val, FP8_E4M3_MAX), -FP8_E4M3_MAX); +#if !defined(USE_ROCM) || defined(HIP_FP8_TYPE_E4M3) + output_arr[j] = static_cast(val); +#else + output_arr[j] = c10::Float8_e4m3fnuz( + __hip_cvt_float_to_fp8(val, fp8::fp8_type::__default_saturation, fp8::fp8_type::__default_interpret), + c10::Float8_e4m3fnuz::from_bits()); +#endif + } + if constexpr (kVecSize == 16) { + *(uint4*)(token_output + i * kVecSize) = *(uint4*)output_arr; + } else { + // Use element-wise copy for vector size 8 to ensure correctness + for (int k = 0; k < kVecSize; ++k) { + token_output[i * kVecSize + k] = output_arr[k]; + } + } + } +} + +// --------------------------------------------------------------------------- +// 2. Baseline kernel (1 token / CTA, CUB block reduce) +// --------------------------------------------------------------------------- +template +__global__ void per_token_quant_fp8_small_batch_kernel( + const T* __restrict__ input, + DST_DTYPE* __restrict__ output_q, + float* __restrict__ output_s, + const int64_t hidden_dim, + const int64_t num_tokens) { + const int token_idx = blockIdx.x; + if (token_idx >= num_tokens) return; + + const int tid = threadIdx.x; + const int block_dim = blockDim.x; + + const T* token_input = input + token_idx * hidden_dim; + DST_DTYPE* token_output = output_q + token_idx * hidden_dim; + + float max_value = 0.0f; + + // Use template parameter for vector size + using vec_t = flashinfer::vec_t; + const int32_t num_vec_elems = hidden_dim / kVecSize; + + // Find max using vectorized loads + for (int32_t i = tid; i < num_vec_elems; i += block_dim) { + vec_t input_vec; + input_vec.cast_load(token_input + i * kVecSize); + +#pragma unroll + for (uint32_t j = 0; j < kVecSize; ++j) { + float val = static_cast(input_vec[j]); + max_value = fmaxf(max_value, fabsf(val)); + } + } + + max_value = blockReduceMax(max_value); + + __shared__ float scale; + if (tid == 0) { + scale = max_value / FP8_E4M3_MAX; + output_s[token_idx] = scale; + } + __syncthreads(); + + const float scale_inv = 1.0f / scale; + + // Quantize using vectorized loads + for (int32_t i = tid; i < num_vec_elems; i += block_dim) { + vec_t input_vec; + input_vec.cast_load(token_input + i * kVecSize); + + DST_DTYPE output_arr[kVecSize]; +#pragma unroll + for (uint32_t j = 0; j < kVecSize; ++j) { + float val = fmaxf(fminf(static_cast(input_vec[j]) * scale_inv, FP8_E4M3_MAX), -FP8_E4M3_MAX); +#if !defined(USE_ROCM) || defined(HIP_FP8_TYPE_E4M3) + output_arr[j] = static_cast(val); +#else + output_arr[j] = c10::Float8_e4m3fnuz( + __hip_cvt_float_to_fp8(val, fp8::fp8_type::__default_saturation, fp8::fp8_type::__default_interpret), + c10::Float8_e4m3fnuz::from_bits()); +#endif + } + + if constexpr (kVecSize == 16) { + *(uint4*)(token_output + i * kVecSize) = *(uint4*)output_arr; + } else { + // Use element-wise copy for vector size 8 to ensure correctness + for (int k = 0; k < kVecSize; ++k) { + token_output[i * kVecSize + k] = output_arr[k]; + } + } + } +} + +void sgl_per_token_quant_fp8(torch::Tensor input, torch::Tensor output_q, torch::Tensor output_s) { + CHECK_INPUT(input); + CHECK_INPUT(output_q); + CHECK_INPUT(output_s); + const auto input_sizes = input.sizes(); + const int64_t num_tokens = input_sizes[0]; + const int64_t hidden_dim = input_sizes[1]; + TORCH_CHECK(hidden_dim % 4 == 0, "Hidden dimension must be divisible by 4, but got ", hidden_dim); + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + const int sm_count = at::cuda::getCurrentDeviceProperties()->multiProcessorCount; + const int TOKENS_PER_CTA = 8; + const bool use_warp_kernel = (num_tokens >= sm_count * 2 * TOKENS_PER_CTA); + const bool use_vec16 = (hidden_dim % 16 == 0); + const bool use_vec8 = (hidden_dim % 8 == 0); + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), scalar_t, [&] { + if (use_warp_kernel) { + // -------- warp‑local --------------------------------------------------- + constexpr int THREADS = TOKENS_PER_CTA * kWarpSize; // 256 + dim3 grid((num_tokens + TOKENS_PER_CTA - 1) / TOKENS_PER_CTA); + dim3 block(THREADS); + + if (use_vec16) { + per_token_quant_fp8_kernel<<>>( + static_cast(input.data_ptr()), + static_cast<__nv_fp8_e4m3*>(output_q.data_ptr()), + static_cast(output_s.data_ptr()), + hidden_dim, + num_tokens); + } else if (use_vec8) { + per_token_quant_fp8_kernel<<>>( + static_cast(input.data_ptr()), + static_cast<__nv_fp8_e4m3*>(output_q.data_ptr()), + static_cast(output_s.data_ptr()), + hidden_dim, + num_tokens); + } else { + per_token_quant_fp8_kernel<<>>( + static_cast(input.data_ptr()), + static_cast<__nv_fp8_e4m3*>(output_q.data_ptr()), + static_cast(output_s.data_ptr()), + hidden_dim, + num_tokens); + } + } else { + // -------- baseline ----------------------------------------------------- + constexpr int THREADS = 256; + dim3 grid(num_tokens); + dim3 block(THREADS); + + if (use_vec16) { + per_token_quant_fp8_small_batch_kernel<<>>( + static_cast(input.data_ptr()), + static_cast<__nv_fp8_e4m3*>(output_q.data_ptr()), + static_cast(output_s.data_ptr()), + hidden_dim, + num_tokens); + } else if (use_vec8) { + per_token_quant_fp8_small_batch_kernel<<>>( + static_cast(input.data_ptr()), + static_cast<__nv_fp8_e4m3*>(output_q.data_ptr()), + static_cast(output_s.data_ptr()), + hidden_dim, + num_tokens); + } else { + per_token_quant_fp8_small_batch_kernel<<>>( + static_cast(input.data_ptr()), + static_cast<__nv_fp8_e4m3*>(output_q.data_ptr()), + static_cast(output_s.data_ptr()), + hidden_dim, + num_tokens); + } + } + return true; + }); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/qserve_w4a8_per_chn_gemm.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/qserve_w4a8_per_chn_gemm.cu new file mode 100644 index 00000000..79180737 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/qserve_w4a8_per_chn_gemm.cu @@ -0,0 +1,710 @@ +// Implemented by Haotian Tang and Shang Yang. +// @article{lin2024qserve, +// title={QServe: W4A8KV4 Quantization and System Co-design for Efficient LLM Serving}, +// author={Lin*, Yujun and Tang*, Haotian and Yang*, Shang and Zhang, Zhekai and Xiao, Guangxuan and Gan, Chuang and +// Han, Song}, journal={arXiv preprint arXiv:2405.04532}, year={2024} +// } +// @article{yang2025lserve, +// title={LServe: Efficient Long-sequence LLM Serving with Unified Sparse Attention}, +// author={Yang*, Shang and Guo*, Junxian and Tang, Haotian and Hu, Qinghao and Xiao, Guangxuan and Tang, Jiaming and +// Lin, Yujun and Liu, Zhijian and Lu, Yao and Han, Song}, year={2025} +// } + +// Adapted from https://github.com/mit-han-lab/omniserve/blob/main/kernels/csrc/qgemm/w4a8_per_chn/gemm_cuda.cu + +#include +#include +#include +#include + +#include "utils.h" + +#define OP_M 16 +#define OP_N 8 +#define OP_K 32 +#define INTRIN_M 16 +#define INTRIN_N 16 +#define INTRIN_K 32 +#define WARP_SIZE 32 +#define SMEM_PAD_A 0 +#define SMEM_PAD_B 0 +#define PACK_SIZE 16 +#if (__CUDACC_VER_MAJOR__ >= 11) && (__CUDACC_VER_MINOR__ >= 4) +#define L2_CACHEHINT(size) ".L2::" #size "B" +#else +#define L2_CACHEHINT(size) +#endif + +#define KERNEL_LAUNCH_CODE \ + constexpr int NUM_WARPS = (CTA_M / WARP_M) * (CTA_N / WARP_N) * (CTA_K / WARP_K); \ + constexpr int SCALES_SMEM_SIZE = (G >= CTA_K) ? (CTA_N * STAGES * 2) : (CTA_N * (CTA_K / G) * STAGES * 2); \ + constexpr int kSmemByteSize = \ + ((CTA_M * (CTA_K + SMEM_PAD_A) + CTA_N * (CTA_K + SMEM_PAD_B) / 2) * STAGES + SCALES_SMEM_SIZE) * \ + sizeof(int8_t); \ + if (kSmemByteSize >= 99 * 1024) { \ + printf( \ + "This kernel requires %d Bytes of shared memory, which exceeds " \ + "device limit.\n", \ + kSmemByteSize); \ + return; \ + } \ + int num_blocks_m = (num_out_feats + CTA_M - 1) / CTA_M; \ + int num_blocks_n = num_out_channels / CTA_N / 1; \ + const int log_tile = get_log_tile<8>((num_out_feats + CTA_M - 1) / CTA_M); \ + const int tile_shift = 1 << log_tile; \ + dim3 num_blocks(num_blocks_n* tile_shift, (num_blocks_m + tile_shift - 1) / tile_shift); \ + dim3 threads_per_block(WARP_SIZE, NUM_WARPS); \ + auto kernel_func = dense_kernel0; \ + cudaFuncSetAttribute(kernel_func, cudaFuncAttributeMaxDynamicSharedMemorySize, kSmemByteSize); \ + kernel_func<<>>( \ + in_feats, kernel, wscales, ascales, w_szs, a_ssums, out_feats, num_in_feats, num_out_channels, num_in_channels); + +template +__inline__ __host__ __device__ int get_log_tile(int n) { + if (N >= 8 && n >= 6) + return 3; + else if (N >= 4 && n >= 3) + return 2; + else if (N >= 2 && n >= 2) + return 1; + else + return 0; +} + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 +__inline__ __device__ uint2 get_block_idx_mapping(int blockIdx_x, int blockIdx_y, int log_tile) { + return make_uint2((blockIdx_x >> log_tile), (blockIdx_y << log_tile) + ((blockIdx_x) & ((1 << (log_tile)) - 1))); +} + +__inline__ __device__ uint32_t cast_smem_ptr_to_uint(void const* const ptr) { + uint32_t smem_int_ptr; + + asm("{.reg .u64 smem_ptr; cvta.to.shared.u64 smem_ptr, %1; cvt.u32.u64 %0, " + "smem_ptr; }\n" + : "=r"(smem_int_ptr) + : "l"(ptr)); + + return smem_int_ptr; +} + +__inline__ __device__ void ldmatrix_m8n8_x4_b16(int8_t* shared_warp, int ax0_0, uint32_t addr) { + __asm__ __volatile__( + "ldmatrix.sync.aligned.m8n8.x4.shared.b16" + "{%0, %1, %2, %3}, [%4];" + : "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[0]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[1]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[2]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[3]) + : "r"(addr)); +} + +__inline__ __device__ void ldmatrix_m8n8_x4_trans_b16(int8_t* shared_warp, int ax0_0, uint32_t addr) { + __asm__ __volatile__( + "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16" + "{%0, %1, %2, %3}, [%4];" + : "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[0]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[1]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[2]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[3]) + : "r"(addr)); +} + +// function from lmdeploy +__inline__ __device__ void cp_async_cg_A(uint32_t smem_int_ptr, const uint4* __restrict__ src, bool mask) { + const int cp_size = 16; + asm volatile("{" + " .reg .pred p;" + " setp.ne.b32 p, %0, 0;" + " @p cp.async.cg.shared.global" L2_CACHEHINT(128) " [%1], [%2], %3;" + "}" ::"r"((int)mask), + "r"(smem_int_ptr), + "l"(src), + "n"(cp_size)); +} + +__device__ __inline__ void mma_m16n8k32(void* C_warp, void* A_shared_warp, void* B_shared_warp) { + __asm__ __volatile__( + "mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32" + "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};" + : "=r"(((int*)C_warp)[0]), "=r"(((int*)C_warp)[1]), "=r"(((int*)C_warp)[2]), "=r"(((int*)C_warp)[3]) + : "r"(((unsigned*)A_shared_warp)[0]), + "r"(((unsigned*)A_shared_warp)[1]), + "r"(((unsigned*)A_shared_warp)[2]), + "r"(((unsigned*)A_shared_warp)[3]), + "r"(((unsigned*)B_shared_warp)[0]), + "r"(((unsigned*)B_shared_warp)[1]), + "r"(((int*)C_warp)[0]), + "r"(((int*)C_warp)[1]), + "r"(((int*)C_warp)[2]), + "r"(((int*)C_warp)[3])); +} + +template +__device__ __inline__ void global_to_share_one_stage_A( + int8_t* src, + int8_t* dst, + int global_ncols, + int cta_offset_m, + int cta_offset_n, + int global_iter_k, + int shared_iter_k, + bool mask, + bool* preds) { + constexpr int total_global_iters = (CTA_M * CTA_K) / PACK_SIZE / CTA_SIZE; + constexpr int partial_global_iters = total_global_iters / SHARED_K_ITERS; + constexpr int cta_step_m_or_n = (CTA_SIZE * PACK_SIZE) / CTA_K; + constexpr int warp_step_m_or_n = (WARP_SIZE * PACK_SIZE) / CTA_K; + constexpr int threads_per_row = CTA_K / PACK_SIZE; + constexpr int kSmemCol = CTA_K + SMEM_PAD_A; + int8_t* dst_hoisted = dst; + int8_t* src_hoisted = src + global_iter_k * CTA_K; + + if (mask) { +#pragma unroll + for (int _global_iter = 0; _global_iter < partial_global_iters; ++_global_iter) { + int global_iter = shared_iter_k * partial_global_iters + _global_iter; + + void* dst_ptr = (void*)(dst_hoisted + global_iter * cta_step_m_or_n * kSmemCol); + uint4* src_ptr = (uint4*)(src_hoisted + global_iter * cta_step_m_or_n * global_ncols); + // *dst_ptr = *src_ptr; + if constexpr (STAGES > 1) { + uint32_t addr = cast_smem_ptr_to_uint(dst_ptr); + cp_async_cg_A(addr, src_ptr, preds[global_iter]); + } else { + if (preds[global_iter]) *(uint4*)dst_ptr = *src_ptr; + } + } + } +} + +template +__device__ __inline__ void global_to_share_one_stage_B( + int8_t* src, + int8_t* dst, + int global_ncols, + int cta_offset_m, + int cta_offset_n, + int global_iter_k, + int shared_iter_k, + bool mask) { + constexpr int total_global_iters = (CTA_N * CTA_K) / 32 / CTA_SIZE; + constexpr int NUM_WARPS = CTA_SIZE / WARP_SIZE; + constexpr int warps_per_row = CTA_K / 32; + constexpr int cta_step_m_or_n = NUM_WARPS / warps_per_row; + constexpr int kSmemCol = CTA_K; + int8_t* dst_hoisted = dst; + int8_t* src_hoisted = src + global_iter_k * CTA_K * PACK_SIZE; + +#pragma unroll + for (int global_iter = 0; global_iter < total_global_iters; ++global_iter) { + void* dst_ptr = (void*)(dst_hoisted + global_iter * cta_step_m_or_n * kSmemCol * PACK_SIZE); + uint4* src_ptr = (uint4*)(src_hoisted + global_iter * cta_step_m_or_n * global_ncols * PACK_SIZE); + if constexpr (STAGES > 1) { + uint32_t addr = cast_smem_ptr_to_uint(dst_ptr); + cp_async_cg_A(addr, src_ptr, mask); + } else { + if (mask) *(uint4*)dst_ptr = *src_ptr; + } + } +} + +template +__device__ __inline__ void global_to_share_one_stage_zeros( + int8_t* src, + int8_t* dst, + int global_ncols, + int cta_offset_m, + int cta_offset_n, + int global_iter_k, + int shared_iter_k, + bool mask) { + constexpr int threads_needed = CTA_N / PACK_SIZE / 1; + constexpr int threads_used = threads_needed < CTA_SIZE ? threads_needed : CTA_SIZE; + constexpr int total_global_iters = CTA_N / PACK_SIZE / threads_used; + constexpr int threads_per_row = CTA_N / PACK_SIZE; + constexpr int kSmemCol = CTA_N; + bool local_mask = mask & (threadIdx.y * WARP_SIZE + threadIdx.x < threads_used); + int g_idx = global_iter_k * CTA_K / G; + + void* dst_ptr = (void*)(dst + (threadIdx.x % threads_per_row) * PACK_SIZE); + uint4* src_ptr = (uint4*)(src + g_idx * global_ncols + cta_offset_n + (threadIdx.x % threads_per_row) * PACK_SIZE); + if (STAGES > 1) { + uint32_t addr = cast_smem_ptr_to_uint(dst_ptr); + cp_async_cg_A(addr, src_ptr, local_mask); + } else { + if (local_mask) { + *(uint4*)dst_ptr = *src_ptr; + } + } +} + +template +__device__ __inline__ void +share_to_reg_one_stage_A(int8_t* src, int8_t* dst, int warp_offset_m, int warp_offset_n, int k_0_1, int shared_iters) { + constexpr int kSmemCol = CTA_K + SMEM_PAD_A; + int ld_col = (k_0_1 * INTRIN_K + (threadIdx.x / 16) * 16) / PACK_SIZE; + + for (int shared_iter = 0; shared_iter < shared_iters; ++shared_iter) { + int ld_row = warp_offset_m + shared_iter * INTRIN_M + (threadIdx.x % 16); + int ld_col_swizzled = ld_col ^ (ld_row / 2) & 3; + void* addr_ptr = (void*)(src + ld_row * kSmemCol + ld_col_swizzled * PACK_SIZE); + uint32_t addr = cast_smem_ptr_to_uint(addr_ptr); + ldmatrix_m8n8_x4_b16(dst, shared_iter, addr); + } +} + +template +__device__ __inline__ void share_to_reg_one_stage_B( + int8_t* src, + int8_t* dst, + int8_t* zeros, + int8_t* scales_i8, + int warp_offset_m, + int warp_offset_n, + int k_0_0, + int k_0_1, + int shared_iters) { + constexpr int kSmemCol = CTA_K + SMEM_PAD_B; +#pragma unroll + for (int shared_iter = 0; shared_iter < shared_iters; ++shared_iter) { + uint4 loaded = + *((uint4*)(src) + warp_offset_n / 32 * kSmemCol + shared_iter * 32 / 32 * kSmemCol + k_0_1 * INTRIN_K + + threadIdx.x); + + auto ptr = (uint32_t*)dst + shared_iter * 8; + ptr[0] = loaded.x & 0x0F0F0F0F; + ptr[4] = (loaded.x & 0xF0F0F0F0) >> 4; + ptr[2] = loaded.y & 0x0F0F0F0F; + ptr[6] = (loaded.y & 0xF0F0F0F0) >> 4; + ptr[1] = loaded.z & 0x0F0F0F0F; + ptr[5] = (loaded.z & 0xF0F0F0F0) >> 4; + ptr[3] = loaded.w & 0x0F0F0F0F; + ptr[7] = (loaded.w & 0xF0F0F0F0) >> 4; + } +} + +template +__global__ void dense_kernel0( + int8_t* __restrict__ A, + int8_t* __restrict__ B, + half2* __restrict__ wscales, + half* __restrict__ ascales, + half2* __restrict__ w_szs, + half* __restrict__ a_ssums, + half* __restrict__ C, + int M, + int64_t N, + int64_t K) { + constexpr int SPLITK = 1; + constexpr int NUM_WARPS_MN = CTA_M / WARP_M * CTA_N / WARP_N; + constexpr int NUM_WARPS = NUM_WARPS_MN * CTA_K / WARP_K; + constexpr int CTA_SIZE = NUM_WARPS * WARP_SIZE; + constexpr int CTA_SIZE_MN = NUM_WARPS_MN * WARP_SIZE; + constexpr int SLICES = CTA_K / WARP_K; + int num_blocks_n = (N + CTA_N - 1) / CTA_N; + int num_blocks_m = (M + CTA_M - 1) / CTA_M; + + int blockIdx_n = blockIdx.x; + int blockIdx_m = blockIdx.y; + const int log_tile = get_log_tile<8>((M + CTA_M - 1) / CTA_M); + const uint2 block_idx_mapping = get_block_idx_mapping(blockIdx_n, blockIdx_m, log_tile); + blockIdx_n = block_idx_mapping.x; + blockIdx_m = block_idx_mapping.y; + + int C_warp[CTA_M * CTA_N / CTA_SIZE_MN]; + constexpr int kSmemPadKA = CTA_K + SMEM_PAD_A; + constexpr int kSmemPadKB = CTA_K + SMEM_PAD_B; + constexpr int kSmemSizeAPerStage = CTA_M * kSmemPadKA; + constexpr int kSmemSizeBPerStage = CTA_N * kSmemPadKB / 2; + constexpr int kSmemSizeA = kSmemSizeAPerStage * STAGES; + constexpr int kSmemSizeB = kSmemSizeBPerStage * STAGES; + + constexpr int scales_load_interval = G >= CTA_K ? G / CTA_K : 1; + constexpr int scales_per_load = G < CTA_K ? CTA_K / G : 1; + constexpr int kSmemSizeScales = CTA_N * STAGES; + + extern __shared__ int8_t mem_shared[]; + int8_t* A_shared = mem_shared; + + int8_t* B_shared = mem_shared + kSmemSizeA; + int8_t* zeros_shared = mem_shared + kSmemSizeA + kSmemSizeB; + int8_t* scales_i8_shared = mem_shared + kSmemSizeA + kSmemSizeB + kSmemSizeScales; + + int8_t A_shared_warp_[2][WARP_M * WARP_K / WARP_SIZE]; + int8_t B_shared_warp_[2][WARP_N * WARP_K / WARP_SIZE]; + constexpr int A_total_global_iters = (CTA_M * CTA_K) / PACK_SIZE / CTA_SIZE; + constexpr int B_total_global_iters = (CTA_N * CTA_K) / PACK_SIZE / CTA_SIZE; + constexpr int A_src_step_m = (CTA_SIZE * PACK_SIZE) / CTA_K; + constexpr int A_warp_step_m = (WARP_SIZE * PACK_SIZE) / CTA_K; + constexpr int A_threads_per_row = CTA_K / PACK_SIZE; + + constexpr int B_warps_per_row = CTA_K / 32; + constexpr int B_src_step_n = NUM_WARPS / B_warps_per_row; + + int cta_offset_m = blockIdx_m * CTA_M; + int cta_offset_n = blockIdx_n * CTA_N; + int warp_mn = threadIdx.y % NUM_WARPS_MN; + int slice_id = threadIdx.y / NUM_WARPS_MN; + int warp_offset_m = (warp_mn % (CTA_M / WARP_M)) * WARP_M; + int warp_offset_n = (warp_mn / (CTA_M / WARP_M)) * WARP_N; + int warp_offset_k = slice_id * WARP_K; + + for (int i = 0; i < CTA_M * CTA_N / CTA_SIZE_MN; i++) + C_warp[i] = 0; + + int gemm_iters = (K + CTA_K - 1) / CTA_K; + int k_0_0_ld = 0; + int k_0_0 = 0; + constexpr int prologue_stages = STAGES == 1 ? 1 : STAGES - 1; + int A_hoisted_row = threadIdx.y * A_warp_step_m + (threadIdx.x / A_threads_per_row); + int A_hoisted_col = (threadIdx.x % A_threads_per_row); + int A_hoisted_col_swizzled = A_hoisted_col ^ (A_hoisted_row / 2) & 3; + + int8_t* A_shared_hoisted = A_shared + A_hoisted_row * kSmemPadKA + A_hoisted_col_swizzled * PACK_SIZE; + int8_t* B_shared_hoisted = B_shared + (threadIdx.y % B_warps_per_row) * 32 * PACK_SIZE + + (threadIdx.y / B_warps_per_row) * kSmemPadKB * PACK_SIZE + threadIdx.x * PACK_SIZE; + int8_t* A_hoisted = A + cta_offset_m * K + A_hoisted_row * K + A_hoisted_col * PACK_SIZE; + int8_t* B_hoisted = B + cta_offset_n / 32 * K * PACK_SIZE + (threadIdx.y % B_warps_per_row) * 32 * PACK_SIZE + + (threadIdx.y / B_warps_per_row) * K * PACK_SIZE + threadIdx.x * PACK_SIZE; + + bool A_g2s_preds[A_total_global_iters]; +#pragma unroll + for (int i = 0; i < A_total_global_iters; i++) { + A_g2s_preds[i] = (cta_offset_m + A_hoisted_row + i * A_src_step_m) < M; + } + + int* C_shared = reinterpret_cast(mem_shared); + +#pragma unroll + for (k_0_0_ld = 0; k_0_0_ld < prologue_stages; ++k_0_0_ld) { + global_to_share_one_stage_A( + A_hoisted, + A_shared_hoisted + k_0_0_ld * kSmemSizeAPerStage, + K, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + 0, + true, + A_g2s_preds); + global_to_share_one_stage_B( + B_hoisted, B_shared_hoisted + k_0_0_ld * kSmemSizeBPerStage, K, cta_offset_m, cta_offset_n, k_0_0_ld, 0, true); + + if constexpr (STAGES > 1) __pipeline_commit(); + } + if constexpr (STAGES > 1) __pipeline_wait_prior(STAGES - 2); + __syncthreads(); + + share_to_reg_one_stage_A( + A_shared + warp_offset_k, A_shared_warp_[0], warp_offset_m, warp_offset_n, 0, WARP_M / INTRIN_M); + share_to_reg_one_stage_B( + B_shared + warp_offset_k * PACK_SIZE, + B_shared_warp_[0], + zeros_shared, + scales_i8_shared, + warp_offset_m, + warp_offset_n, + 0, + 0, + WARP_N / 32); + constexpr int SHARED_K_ITERS = WARP_K / INTRIN_K; + + for (; k_0_0 < gemm_iters; ++k_0_0, ++k_0_0_ld) { + int ld_stage = k_0_0_ld % STAGES; + int compute_stage = k_0_0 % STAGES; + int8_t* A_shared_this_compute_stage; + int8_t* B_shared_this_compute_stage; + int8_t* zeros_shared_this_compute_stage; + int8_t* scales_i8_shared_this_compute_stage; + + for (int iter_k = 0; iter_k < SHARED_K_ITERS; ++iter_k) { + A_shared_this_compute_stage = A_shared + compute_stage * kSmemSizeAPerStage + warp_offset_k; + B_shared_this_compute_stage = B_shared + compute_stage * kSmemSizeBPerStage + warp_offset_k * PACK_SIZE; + zeros_shared_this_compute_stage = zeros_shared + (compute_stage)*CTA_N; + scales_i8_shared_this_compute_stage = scales_i8_shared + (compute_stage)*CTA_N; + + share_to_reg_one_stage_A( + A_shared_this_compute_stage, + A_shared_warp_[(iter_k + 1) % 2], + warp_offset_m, + warp_offset_n, + (iter_k + 1) % SHARED_K_ITERS, + WARP_M / INTRIN_M); + share_to_reg_one_stage_B( + B_shared_this_compute_stage, + B_shared_warp_[(iter_k + 1) % 2], + zeros_shared_this_compute_stage, + scales_i8_shared_this_compute_stage, + warp_offset_m, + warp_offset_n, + k_0_0 + (iter_k == SHARED_K_ITERS - 1), + (iter_k + 1) % SHARED_K_ITERS, + WARP_N / 32); + int8_t* A_shared_warp = A_shared_warp_[iter_k % 2]; + int8_t* B_shared_warp = B_shared_warp_[iter_k % 2]; + + for (int j_0_4 = 0; j_0_4 < WARP_N / INTRIN_N; ++j_0_4) { + for (int i_0_3 = 0; i_0_3 < WARP_M / INTRIN_M; ++i_0_3) { + mma_m16n8k32( + (void*)(C_warp + i_0_3 * WARP_N / INTRIN_N * 8 + j_0_4 * 8), + (void*)(A_shared_warp + i_0_3 * 16), + (void*)(B_shared_warp + j_0_4 * 16)); + mma_m16n8k32( + (void*)(C_warp + i_0_3 * WARP_N / INTRIN_N * 8 + j_0_4 * 8 + 4), + (void*)(A_shared_warp + i_0_3 * 16), + (void*)(B_shared_warp + j_0_4 * 16 + 8)); + } + } + + if (iter_k < SHARED_K_ITERS - 1) { + if constexpr (STAGES == 1) __syncthreads(); + global_to_share_one_stage_A( + A_hoisted, + A_shared_hoisted + ld_stage * kSmemSizeAPerStage, + K, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + iter_k, + k_0_0_ld < gemm_iters, + A_g2s_preds); + global_to_share_one_stage_B( + B_hoisted, + B_shared_hoisted + ld_stage * kSmemSizeBPerStage, + K, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + iter_k, + k_0_0_ld < gemm_iters); + } + + if (iter_k == SHARED_K_ITERS - 2) { + if constexpr (STAGES == 1 && SHARED_K_ITERS > 2) { + __syncthreads(); + } + global_to_share_one_stage_A( + A_hoisted, + A_shared_hoisted + ld_stage * kSmemSizeAPerStage, + K, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + iter_k + 1, + k_0_0_ld < gemm_iters, + A_g2s_preds); + global_to_share_one_stage_B( + B_hoisted, + B_shared_hoisted + ld_stage * kSmemSizeBPerStage, + K, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + iter_k + 1, + k_0_0_ld < gemm_iters); + if constexpr (STAGES > 1) { + __pipeline_commit(); + __pipeline_wait_prior(STAGES - 2); + } + compute_stage = (k_0_0 + 1) % STAGES; + __syncthreads(); + } + } + } + + __pipeline_commit(); + __pipeline_wait_prior(0); + __syncthreads(); + + if constexpr (SLICES > 1) { +#pragma unroll + for (int z = 0; z < SLICES; ++z) { + if (slice_id == z) { +#pragma unroll + for (int ax0_0_1 = 0; ax0_0_1 < WARP_M / INTRIN_M; ++ax0_0_1) { +#pragma unroll + for (int ax1_0_1 = 0; ax1_0_1 < WARP_N / INTRIN_N; ++ax1_0_1) { +#pragma unroll + for (int local_id = 0; local_id < OP_M * 16 / WARP_SIZE; ++local_id) { + if (z > 0) { + C_warp[ax0_0_1 * WARP_N / INTRIN_N * 8 + ax1_0_1 * 8 + local_id] += C_shared + [warp_offset_m * CTA_N + ax0_0_1 * OP_M * CTA_N + warp_offset_n + ax1_0_1 * 16 + + ((local_id % 4) / 2 * 8 + (threadIdx.x / 4)) * CTA_N + (local_id / 4) * 8 + (local_id % 2) + + (threadIdx.x % 4) * 2]; + } + C_shared + [warp_offset_m * CTA_N + ax0_0_1 * OP_M * CTA_N + warp_offset_n + ax1_0_1 * 16 + + ((local_id % 4) / 2 * 8 + (threadIdx.x / 4)) * CTA_N + (local_id / 4) * 8 + (local_id % 2) + + (threadIdx.x % 4) * 2] = C_warp[ax0_0_1 * WARP_N / INTRIN_N * 8 + ax1_0_1 * 8 + local_id]; + }; + } + } + } + __syncthreads(); + } + if (slice_id == 0) { +#pragma unroll + for (int ax0_0_1 = 0; ax0_0_1 < WARP_M / INTRIN_M; ++ax0_0_1) { +#pragma unroll + for (int ax1_0_1 = 0; ax1_0_1 < WARP_N / INTRIN_N; ++ax1_0_1) { +#pragma unroll + for (int local_id = 0; local_id < OP_M * 16 / WARP_SIZE; ++local_id) { + C_warp[ax0_0_1 * WARP_N / INTRIN_N * 8 + ax1_0_1 * 8 + local_id] = C_shared + [warp_offset_m * CTA_N + ax0_0_1 * OP_M * CTA_N + warp_offset_n + ax1_0_1 * 16 + + ((local_id % 4) / 2 * 8 + (threadIdx.x / 4)) * CTA_N + (local_id / 4) * 8 + (local_id % 2) + + (threadIdx.x % 4) * 2]; + }; + } + } + } + } + + int row_wb_thd = cta_offset_m + warp_offset_m + (threadIdx.x / 4); + int col_wb_thd = cta_offset_n + warp_offset_n + (threadIdx.x % 4) * 2; + if (slice_id == 0) { + for (int ax0_0_1 = 0; ax0_0_1 < WARP_M / INTRIN_M; ++ax0_0_1) { + int row_wb_1 = row_wb_thd + ax0_0_1 * OP_M; + for (int ax1_0_1 = 0; ax1_0_1 < WARP_N / INTRIN_N; ++ax1_0_1) { + int col_wb_1 = col_wb_thd + ax1_0_1 * 16; + int* C_warp_local = C_warp + ax0_0_1 * WARP_N / INTRIN_N * 8 + ax1_0_1 * 8; + for (int local_id = 0; local_id < OP_M * 16 / WARP_SIZE; local_id += 2) { + int row_wb = row_wb_1 + (local_id % 4) / 2 * 8; + if (row_wb < M) { + int col_wb = col_wb_1 + (local_id / 4) * 8 + (local_id % 2); + float2 wscale = __half22float2(*(wscales + col_wb / 2)); + float2 w_sz = __half22float2(*(w_szs + col_wb / 2)); + float ascale = __half2float(ascales[row_wb]); + float a_ssum = __half2float(a_ssums[row_wb]); + float2 psums = + make_float2(__int2float_rn(C_warp_local[local_id]), __int2float_rn(C_warp_local[local_id + 1])); + psums.x = psums.x * wscale.x * ascale - w_sz.x * a_ssum; + psums.y = psums.y * wscale.y * ascale - w_sz.y * a_ssum; + *reinterpret_cast(C + row_wb * N + col_wb) = __float22half2_rn(psums); + } + }; + } + } + } +} +#else +template +__global__ void dense_kernel0( + int8_t* __restrict__ A, + int8_t* __restrict__ B, + half2* __restrict__ wscales, + half* __restrict__ ascales, + half2* __restrict__ w_szs, + half* __restrict__ a_ssums, + half* __restrict__ C, + int M, + int64_t N, + int64_t K) { + // Not implemented for SM < 800 + assert(false); + return; +} +#endif + +void qserve_w4a8_per_chn_gemm( + const torch::Tensor& _in_feats, + const torch::Tensor& _kernel, + const torch::Tensor& _wscales, + const torch::Tensor& _ascales, + const torch::Tensor& _w_szs, + const torch::Tensor& _a_ssums, + torch::Tensor& _out_feats) { + // Check input tensor + TORCH_CHECK(_in_feats.is_cuda(), "_in_feats must be a CUDA tensor"); + TORCH_CHECK(_in_feats.dim() == 2, "_in_feats must be a 2D tensor"); + TORCH_CHECK(_in_feats.is_contiguous(), "_in_feats must be contiguous"); + TORCH_CHECK(_in_feats.scalar_type() == torch::kInt8, "_in_feats must be int8"); + // Check kernel tensor + TORCH_CHECK(_kernel.is_cuda(), "_kernel must be a CUDA tensor"); + TORCH_CHECK(_kernel.dim() == 2, "_kernel must be a 2D tensor"); + TORCH_CHECK(_kernel.is_contiguous(), "_kernel must be contiguous"); + TORCH_CHECK(_kernel.scalar_type() == torch::kInt8, "_kernel must be int8"); + // Check output tensor + TORCH_CHECK(_out_feats.is_cuda(), "_out_feats must be a CUDA tensor"); + TORCH_CHECK(_out_feats.is_contiguous(), "_out_feats must be contiguous"); + TORCH_CHECK(_out_feats.scalar_type() == torch::kHalf, "_out_feats must be half"); + + int num_in_feats = _in_feats.size(0); + int num_in_channels = _in_feats.size(1); + int num_out_feats = _out_feats.size(-2); + int num_out_channels = _out_feats.size(-1); + + // Check matmul shape + TORCH_CHECK(num_out_channels == _kernel.size(0), "num_out_channels must be equal to _kernel.size(0)"); + TORCH_CHECK(num_in_feats == num_out_feats, "num_in_feats must be equal to num_out_feats"); + + // Check _ascales + TORCH_CHECK(_ascales.is_cuda(), "_ascales must be a CUDA tensor"); + TORCH_CHECK(_ascales.is_contiguous(), "_ascales must be contiguous"); + TORCH_CHECK(_ascales.scalar_type() == torch::kHalf, "_ascales must be half"); + TORCH_CHECK(_ascales.numel() == num_in_feats, "_ascales must have num_in_feats elements"); + + // Check _wscales + TORCH_CHECK(_wscales.is_cuda(), "_wscales must be a CUDA tensor"); + TORCH_CHECK(_wscales.is_contiguous(), "_wscales must be contiguous"); + TORCH_CHECK(_wscales.scalar_type() == torch::kHalf, "_wscales must be half"); + TORCH_CHECK(_wscales.numel() == num_out_channels, "_wscales must have num_out_channels elements"); + + // Check _w_szs + TORCH_CHECK(_w_szs.is_cuda(), "_w_szs must be a CUDA tensor"); + TORCH_CHECK(_w_szs.is_contiguous(), "_w_szs must be contiguous"); + TORCH_CHECK(_w_szs.scalar_type() == torch::kHalf, "_w_szs must be half"); + TORCH_CHECK(_w_szs.numel() == num_out_channels, "_w_szs must have num_out_channels elements"); + + // Check _a_ssums + TORCH_CHECK(_a_ssums.is_cuda(), "_a_ssums must be a CUDA tensor"); + TORCH_CHECK(_a_ssums.is_contiguous(), "_a_ssums must be contiguous"); + TORCH_CHECK(_a_ssums.scalar_type() == torch::kHalf, "_a_ssums must be half"); + TORCH_CHECK(_a_ssums.numel() == num_in_feats, "_a_ssums must have num_in_feats elements"); + + auto in_feats = reinterpret_cast(_in_feats.data_ptr()); + auto kernel = reinterpret_cast(_kernel.data_ptr()); + auto w_szs = reinterpret_cast(_w_szs.data_ptr()); + auto a_ssums = reinterpret_cast(_a_ssums.data_ptr()); + auto wscales = reinterpret_cast(_wscales.data_ptr()); + auto ascales = reinterpret_cast(_ascales.data_ptr()); + auto out_feats = reinterpret_cast(_out_feats.data_ptr()); + auto stream = at::cuda::getCurrentCUDAStream(_in_feats.get_device()); + + auto sm_version = getSMVersion(); + if (sm_version >= 80) { + constexpr int G = 128; + + if (num_out_feats > 256) { + constexpr int CTA_M = 128; + constexpr int CTA_N = 128; + constexpr int CTA_K = 64; + constexpr int WARP_M = 64; + constexpr int WARP_N = 32; + constexpr int WARP_K = 64; + constexpr int STAGES = 3; + KERNEL_LAUNCH_CODE + } else if (num_out_feats >= 128) { + constexpr int CTA_M = 64; + constexpr int CTA_N = 64; + constexpr int CTA_K = 64; + constexpr int WARP_M = 32; + constexpr int WARP_N = 32; + constexpr int WARP_K = 64; + constexpr int STAGES = 4; + KERNEL_LAUNCH_CODE + } else { + constexpr int CTA_M = 32; + constexpr int CTA_N = 64; + constexpr int CTA_K = 128; + constexpr int WARP_M = 32; + constexpr int WARP_N = 32; + constexpr int WARP_K = 64; + constexpr int STAGES = 3; + KERNEL_LAUNCH_CODE + } + } else { + TORCH_CHECK_NOT_IMPLEMENTED( + false, "No implemented qserve_w4a8_per_chn_gemm for current compute capability: ", sm_version); + } + return; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/qserve_w4a8_per_group_gemm.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/qserve_w4a8_per_group_gemm.cu new file mode 100644 index 00000000..a99a203e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/gemm/qserve_w4a8_per_group_gemm.cu @@ -0,0 +1,795 @@ +// Implemented by Haotian Tang and Shang Yang. +// @article{lin2024qserve, +// title={QServe: W4A8KV4 Quantization and System Co-design for Efficient LLM Serving}, +// author={Lin*, Yujun and Tang*, Haotian and Yang*, Shang and Zhang, Zhekai and Xiao, Guangxuan and Gan, Chuang and +// Han, Song}, journal={arXiv preprint arXiv:2405.04532}, year={2024} +// } +// @article{yang2025lserve, +// title={LServe: Efficient Long-sequence LLM Serving with Unified Sparse Attention}, +// author={Yang*, Shang and Guo*, Junxian and Tang, Haotian and Hu, Qinghao and Xiao, Guangxuan and Tang, Jiaming and +// Lin, Yujun and Liu, Zhijian and Lu, Yao and Han, Song}, year={2025} +// } + +// Adapted from https://github.com/mit-han-lab/omniserve/blob/main/kernels/csrc/qgemm/w4a8_per_group/gemm_cuda.cu + +#include +#include +#include +#include + +#include "utils.h" + +#define OP_M 16 +#define OP_N 8 +#define OP_K 32 +#define INTRIN_M 16 +#define INTRIN_N 16 +#define INTRIN_K 32 +#define WARP_SIZE 32 +#define SMEM_PAD_A 0 +#define SMEM_PAD_B 0 +#define PACK_SIZE 16 +#if (__CUDACC_VER_MAJOR__ >= 11) && (__CUDACC_VER_MINOR__ >= 4) +#define L2_CACHEHINT(size) ".L2::" #size "B" +#else +#define L2_CACHEHINT(size) +#endif + +#define KERNEL_LAUNCH_CODE \ + constexpr int NUM_WARPS = (CTA_M / WARP_M) * (CTA_N / WARP_N) * (CTA_K / WARP_K); \ + constexpr int SCALES_SMEM_SIZE = (G >= CTA_K) ? (CTA_N * STAGES * 2) : (CTA_N * (CTA_K / G) * STAGES * 2); \ + constexpr int kSmemByteSize = \ + ((CTA_M * (CTA_K + SMEM_PAD_A) + CTA_N * (CTA_K + SMEM_PAD_B) / 2) * STAGES + SCALES_SMEM_SIZE) * \ + sizeof(int8_t); \ + if (kSmemByteSize >= 99 * 1024) { \ + printf( \ + "This kernel requires %d Bytes of shared memory, which exceeds " \ + "device limit.\n", \ + kSmemByteSize); \ + return; \ + } \ + int num_blocks_m = (num_out_feats + CTA_M - 1) / CTA_M; \ + int num_blocks_n = num_out_channels / CTA_N / 1; \ + const int log_tile = get_log_tile<8>((num_out_feats + CTA_M - 1) / CTA_M); \ + const int tile_shift = 1 << log_tile; \ + dim3 num_blocks(num_blocks_n* tile_shift, (num_blocks_m + tile_shift - 1) / tile_shift); \ + dim3 threads_per_block(WARP_SIZE, NUM_WARPS); \ + auto kernel_func = dense_kernel0; \ + cudaFuncSetAttribute(kernel_func, cudaFuncAttributeMaxDynamicSharedMemorySize, kSmemByteSize); \ + kernel_func<<>>( \ + in_feats, \ + kernel, \ + zeros, \ + scales_i8, \ + wscales, \ + ascales, \ + out_feats, \ + num_in_feats, \ + num_out_channels, \ + num_in_channels); + +template +__inline__ __host__ __device__ int get_log_tile(int n) { + if (N >= 8 && n >= 6) + return 3; + else if (N >= 4 && n >= 3) + return 2; + else if (N >= 2 && n >= 2) + return 1; + else + return 0; +} + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 +__inline__ __device__ uint2 get_block_idx_mapping(int blockIdx_x, int blockIdx_y, int log_tile) { + return make_uint2((blockIdx_x >> log_tile), (blockIdx_y << log_tile) + ((blockIdx_x) & ((1 << (log_tile)) - 1))); +} + +__inline__ __device__ uint32_t cast_smem_ptr_to_uint(void const* const ptr) { + uint32_t smem_int_ptr; + + asm("{.reg .u64 smem_ptr; cvta.to.shared.u64 smem_ptr, %1; cvt.u32.u64 %0, " + "smem_ptr; }\n" + : "=r"(smem_int_ptr) + : "l"(ptr)); + + return smem_int_ptr; +} + +__inline__ __device__ void ldmatrix_m8n8_x4_b16(int8_t* shared_warp, int ax0_0, uint32_t addr) { + __asm__ __volatile__( + "ldmatrix.sync.aligned.m8n8.x4.shared.b16" + "{%0, %1, %2, %3}, [%4];" + : "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[0]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[1]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[2]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[3]) + : "r"(addr)); +} + +__inline__ __device__ void ldmatrix_m8n8_x4_trans_b16(int8_t* shared_warp, int ax0_0, uint32_t addr) { + __asm__ __volatile__( + "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16" + "{%0, %1, %2, %3}, [%4];" + : "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[0]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[1]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[2]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[3]) + : "r"(addr)); +} + +// function from lmdeploy +__inline__ __device__ void cp_async_cg_A(uint32_t smem_int_ptr, const uint4* __restrict__ src, bool mask) { + const int cp_size = 16; + asm volatile("{" + " .reg .pred p;" + " setp.ne.b32 p, %0, 0;" + " @p cp.async.cg.shared.global" L2_CACHEHINT(128) " [%1], [%2], %3;" + "}" ::"r"((int)mask), + "r"(smem_int_ptr), + "l"(src), + "n"(cp_size)); +} + +__device__ __inline__ void mma_m16n8k32(void* C_warp, void* A_shared_warp, void* B_shared_warp) { + __asm__ __volatile__( + "mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32" + "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};" + : "=r"(((int*)C_warp)[0]), "=r"(((int*)C_warp)[1]), "=r"(((int*)C_warp)[2]), "=r"(((int*)C_warp)[3]) + : "r"(((unsigned*)A_shared_warp)[0]), + "r"(((unsigned*)A_shared_warp)[1]), + "r"(((unsigned*)A_shared_warp)[2]), + "r"(((unsigned*)A_shared_warp)[3]), + "r"(((unsigned*)B_shared_warp)[0]), + "r"(((unsigned*)B_shared_warp)[1]), + "r"(((int*)C_warp)[0]), + "r"(((int*)C_warp)[1]), + "r"(((int*)C_warp)[2]), + "r"(((int*)C_warp)[3])); +} + +template +__device__ __inline__ void global_to_share_one_stage_A( + int8_t* src, + int8_t* dst, + int global_ncols, + int cta_offset_m, + int cta_offset_n, + int global_iter_k, + int shared_iter_k, + bool mask, + bool* preds) { + constexpr int total_global_iters = (CTA_M * CTA_K) / PACK_SIZE / CTA_SIZE; + constexpr int partial_global_iters = total_global_iters / SHARED_K_ITERS; + constexpr int cta_step_m_or_n = (CTA_SIZE * PACK_SIZE) / CTA_K; + constexpr int warp_step_m_or_n = (WARP_SIZE * PACK_SIZE) / CTA_K; + constexpr int threads_per_row = CTA_K / PACK_SIZE; + constexpr int kSmemCol = CTA_K + SMEM_PAD_A; + int8_t* dst_hoisted = dst; + int8_t* src_hoisted = src + global_iter_k * CTA_K; + + if (mask) { +#pragma unroll + for (int _global_iter = 0; _global_iter < partial_global_iters; ++_global_iter) { + int global_iter = shared_iter_k * partial_global_iters + _global_iter; + void* dst_ptr = (void*)(dst_hoisted + global_iter * cta_step_m_or_n * kSmemCol); + uint4* src_ptr = (uint4*)(src_hoisted + global_iter * cta_step_m_or_n * global_ncols); + if constexpr (STAGES > 1) { + uint32_t addr = cast_smem_ptr_to_uint(dst_ptr); + cp_async_cg_A(addr, src_ptr, preds[global_iter]); + } else { + if (preds[global_iter]) *(uint4*)dst_ptr = *src_ptr; + } + } + } +} + +template +__device__ __inline__ void global_to_share_one_stage_B( + int8_t* src, + int8_t* dst, + int global_ncols, + int cta_offset_m, + int cta_offset_n, + int global_iter_k, + int shared_iter_k, + bool mask) { + constexpr int total_global_iters = (CTA_N * CTA_K) / 32 / CTA_SIZE; + constexpr int NUM_WARPS = CTA_SIZE / WARP_SIZE; + constexpr int warps_per_row = CTA_K / 32; + constexpr int cta_step_m_or_n = NUM_WARPS / warps_per_row; + constexpr int kSmemCol = CTA_K; + int8_t* dst_hoisted = dst; + int8_t* src_hoisted = src + global_iter_k * CTA_K * PACK_SIZE; + +#pragma unroll + for (int global_iter = 0; global_iter < total_global_iters; ++global_iter) { + void* dst_ptr = (void*)(dst_hoisted + global_iter * cta_step_m_or_n * kSmemCol * PACK_SIZE); + uint4* src_ptr = (uint4*)(src_hoisted + global_iter * cta_step_m_or_n * global_ncols * PACK_SIZE); + if constexpr (STAGES > 1) { + uint32_t addr = cast_smem_ptr_to_uint(dst_ptr); + cp_async_cg_A(addr, src_ptr, mask); + } else { + if (mask) *(uint4*)dst_ptr = *src_ptr; + } + } +} + +template +__device__ __inline__ void global_to_share_one_stage_zeros( + int8_t* src, + int8_t* dst, + int global_ncols, + int cta_offset_m, + int cta_offset_n, + int global_iter_k, + int shared_iter_k, + bool mask) { + constexpr int threads_needed = CTA_N / PACK_SIZE / 1; + constexpr int threads_used = threads_needed < CTA_SIZE ? threads_needed : CTA_SIZE; + constexpr int total_global_iters = CTA_N / PACK_SIZE / threads_used; + constexpr int threads_per_row = CTA_N / PACK_SIZE; + constexpr int kSmemCol = CTA_N; + bool local_mask = mask & (threadIdx.y * WARP_SIZE + threadIdx.x < threads_used); + int g_idx = global_iter_k * CTA_K / G; + + void* dst_ptr = (void*)(dst + (threadIdx.x % threads_per_row) * PACK_SIZE); + uint4* src_ptr = (uint4*)(src + g_idx * global_ncols + cta_offset_n + (threadIdx.x % threads_per_row) * PACK_SIZE); + if (STAGES > 1) { + uint32_t addr = cast_smem_ptr_to_uint(dst_ptr); + cp_async_cg_A(addr, src_ptr, local_mask); + } else { + if (local_mask) { + *(uint4*)dst_ptr = *src_ptr; + } + } +} + +template +__device__ __inline__ void +share_to_reg_one_stage_A(int8_t* src, int8_t* dst, int warp_offset_m, int warp_offset_n, int k_0_1, int shared_iters) { + constexpr int kSmemCol = CTA_K + SMEM_PAD_A; + int ld_col = (k_0_1 * INTRIN_K + (threadIdx.x / 16) * 16) / PACK_SIZE; + + for (int shared_iter = 0; shared_iter < shared_iters; ++shared_iter) { + int ld_row = warp_offset_m + shared_iter * INTRIN_M + (threadIdx.x % 16); + int ld_col_swizzled = ld_col ^ (ld_row / 2) & 3; + void* addr_ptr = (void*)(src + ld_row * kSmemCol + ld_col_swizzled * PACK_SIZE); + uint32_t addr = cast_smem_ptr_to_uint(addr_ptr); + ldmatrix_m8n8_x4_b16(dst, shared_iter, addr); + } +} + +template +__device__ __inline__ void share_to_reg_one_stage_B( + int8_t* src, + int8_t* dst, + int8_t* zeros, + int8_t* scales_i8, + int warp_offset_m, + int warp_offset_n, + int k_0_0, + int k_0_1, + int shared_iters) { + constexpr int kSmemCol = CTA_K + SMEM_PAD_B; +#pragma unroll + for (int shared_iter = 0; shared_iter < shared_iters; ++shared_iter) { + uint4 loaded = + *((uint4*)(src) + warp_offset_n / 32 * kSmemCol + shared_iter * 32 / 32 * kSmemCol + k_0_1 * INTRIN_K + + threadIdx.x); + uint32_t loaded_0 = loaded.x & 0x0F0F0F0F; + uint32_t loaded_4 = (loaded.x & 0xF0F0F0F0) >> 4; + uint32_t loaded_2 = loaded.y & 0x0F0F0F0F; + uint32_t loaded_6 = (loaded.y & 0xF0F0F0F0) >> 4; + uint32_t loaded_1 = loaded.z & 0x0F0F0F0F; + uint32_t loaded_5 = (loaded.z & 0xF0F0F0F0) >> 4; + uint32_t loaded_3 = loaded.w & 0x0F0F0F0F; + uint32_t loaded_7 = (loaded.w & 0xF0F0F0F0) >> 4; + + auto ptr = (uint32_t*)dst + shared_iter * 8; + int scales_zeros_offset = warp_offset_n + (threadIdx.x / 4) * 4 + shared_iter * 32; + uint32_t packed_scales = *reinterpret_cast(scales_i8 + scales_zeros_offset); + uint32_t packed_zeros = *reinterpret_cast(zeros + scales_zeros_offset); + + uint32_t scale_0 = packed_scales & 0xFF; + uint32_t zero_point_0 = __byte_perm(packed_zeros, 0, 0x00000000); + uint32_t ptr_0 = loaded_0 * scale_0; + uint32_t ptr_1 = loaded_1 * scale_0; + ptr[0] = __vadd4(ptr_0, zero_point_0); + ptr[1] = __vadd4(ptr_1, zero_point_0); + + uint32_t scale_1 = (packed_scales & 0xFF00) >> 8; + uint32_t zero_point_1 = __byte_perm(packed_zeros, 0, 0x00001111); + uint32_t ptr_2 = loaded_2 * scale_1; + uint32_t ptr_3 = loaded_3 * scale_1; + ptr[2] = __vadd4(ptr_2, zero_point_1); + ptr[3] = __vadd4(ptr_3, zero_point_1); + + uint32_t scale_2 = (packed_scales & 0xFF0000) >> 16; + uint32_t zero_point_2 = __byte_perm(packed_zeros, 0, 0x00002222); + uint32_t ptr_4 = loaded_4 * scale_2; + uint32_t ptr_5 = loaded_5 * scale_2; + ptr[4] = __vadd4(ptr_4, zero_point_2); + ptr[5] = __vadd4(ptr_5, zero_point_2); + + uint32_t scale_3 = (packed_scales & 0xFF000000) >> 24; + uint32_t zero_point_3 = __byte_perm(packed_zeros, 0, 0x00003333); + uint32_t ptr_6 = loaded_6 * scale_3; + uint32_t ptr_7 = loaded_7 * scale_3; + ptr[6] = __vadd4(ptr_6, zero_point_3); + ptr[7] = __vadd4(ptr_7, zero_point_3); + } +} + +template +__global__ void dense_kernel0( + int8_t* __restrict__ A, + int8_t* __restrict__ B, + int8_t* __restrict__ zeros, + int8_t* __restrict__ scales_i8, + half2* __restrict__ wscales, + half* __restrict__ ascales, + half* __restrict__ C, + int M, + int64_t N, + int64_t K) { + constexpr int SPLITK = 1; + constexpr int NUM_WARPS_MN = CTA_M / WARP_M * CTA_N / WARP_N; + constexpr int NUM_WARPS = NUM_WARPS_MN * CTA_K / WARP_K; + constexpr int CTA_SIZE = NUM_WARPS * WARP_SIZE; + constexpr int CTA_SIZE_MN = NUM_WARPS_MN * WARP_SIZE; + constexpr int SLICES = CTA_K / WARP_K; + int num_blocks_n = (N + CTA_N - 1) / CTA_N; + int num_blocks_m = (M + CTA_M - 1) / CTA_M; + + int blockIdx_n = blockIdx.x; + int blockIdx_m = blockIdx.y; + const int log_tile = get_log_tile<8>((M + CTA_M - 1) / CTA_M); + const uint2 block_idx_mapping = get_block_idx_mapping(blockIdx_n, blockIdx_m, log_tile); + blockIdx_n = block_idx_mapping.x; + blockIdx_m = block_idx_mapping.y; + + int C_warp[CTA_M * CTA_N / CTA_SIZE_MN]; + constexpr int kSmemPadKA = CTA_K + SMEM_PAD_A; + constexpr int kSmemPadKB = CTA_K + SMEM_PAD_B; + constexpr int kSmemSizeAPerStage = CTA_M * kSmemPadKA; + constexpr int kSmemSizeBPerStage = CTA_N * kSmemPadKB / 2; + constexpr int kSmemSizeA = kSmemSizeAPerStage * STAGES; + constexpr int kSmemSizeB = kSmemSizeBPerStage * STAGES; + + constexpr int scales_load_interval = G >= CTA_K ? G / CTA_K : 1; + constexpr int scales_per_load = G < CTA_K ? CTA_K / G : 1; + constexpr int kSmemSizeScales = CTA_N * STAGES; + + extern __shared__ int8_t mem_shared[]; + int8_t* A_shared = mem_shared; + + int8_t* B_shared = mem_shared + kSmemSizeA; + int8_t* zeros_shared = mem_shared + kSmemSizeA + kSmemSizeB; + int8_t* scales_i8_shared = mem_shared + kSmemSizeA + kSmemSizeB + kSmemSizeScales; + + int8_t A_shared_warp_[2][WARP_M * WARP_K / WARP_SIZE]; + int8_t B_shared_warp_[2][WARP_N * WARP_K / WARP_SIZE]; + constexpr int A_total_global_iters = (CTA_M * CTA_K) / PACK_SIZE / CTA_SIZE; + constexpr int B_total_global_iters = (CTA_N * CTA_K) / PACK_SIZE / CTA_SIZE; + constexpr int A_src_step_m = (CTA_SIZE * PACK_SIZE) / CTA_K; + constexpr int A_warp_step_m = (WARP_SIZE * PACK_SIZE) / CTA_K; + constexpr int A_threads_per_row = CTA_K / PACK_SIZE; + + constexpr int B_warps_per_row = CTA_K / 32; + constexpr int B_src_step_n = NUM_WARPS / B_warps_per_row; + + int cta_offset_m = blockIdx_m * CTA_M; + int cta_offset_n = blockIdx_n * CTA_N; + int warp_mn = threadIdx.y % NUM_WARPS_MN; + int slice_id = threadIdx.y / NUM_WARPS_MN; + int warp_offset_m = (warp_mn % (CTA_M / WARP_M)) * WARP_M; + int warp_offset_n = (warp_mn / (CTA_M / WARP_M)) * WARP_N; + int warp_offset_k = slice_id * WARP_K; + + for (int i = 0; i < CTA_M * CTA_N / CTA_SIZE_MN; i++) + C_warp[i] = 0; + + int gemm_iters = (K + CTA_K - 1) / CTA_K; + + int k_0_0_ld = 0; + int k_0_0 = 0; + constexpr int prologue_stages = STAGES == 1 ? 1 : STAGES - 1; + int A_hoisted_row = threadIdx.y * A_warp_step_m + (threadIdx.x / A_threads_per_row); + int A_hoisted_col = (threadIdx.x % A_threads_per_row); + int A_hoisted_col_swizzled = A_hoisted_col ^ (A_hoisted_row / 2) & 3; + + int8_t* A_shared_hoisted = A_shared + A_hoisted_row * kSmemPadKA + A_hoisted_col_swizzled * PACK_SIZE; + int8_t* B_shared_hoisted = B_shared + (threadIdx.y % B_warps_per_row) * 32 * PACK_SIZE + + (threadIdx.y / B_warps_per_row) * kSmemPadKB * PACK_SIZE + threadIdx.x * PACK_SIZE; + int8_t* A_hoisted = A + cta_offset_m * K + A_hoisted_row * K + A_hoisted_col * PACK_SIZE; + int8_t* B_hoisted = B + cta_offset_n / 32 * K * PACK_SIZE + (threadIdx.y % B_warps_per_row) * 32 * PACK_SIZE + + (threadIdx.y / B_warps_per_row) * K * PACK_SIZE + threadIdx.x * PACK_SIZE; + + bool A_g2s_preds[A_total_global_iters]; +#pragma unroll + for (int i = 0; i < A_total_global_iters; i++) { + A_g2s_preds[i] = (cta_offset_m + A_hoisted_row + i * A_src_step_m) < M; + } + + int* C_shared = reinterpret_cast(mem_shared); + +#pragma unroll + for (k_0_0_ld = 0; k_0_0_ld < prologue_stages; ++k_0_0_ld) { + global_to_share_one_stage_A( + A_hoisted, + A_shared_hoisted + k_0_0_ld * kSmemSizeAPerStage, + K, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + 0, + true, + A_g2s_preds); + global_to_share_one_stage_B( + B_hoisted, B_shared_hoisted + k_0_0_ld * kSmemSizeBPerStage, K, cta_offset_m, cta_offset_n, k_0_0_ld, 0, true); + global_to_share_one_stage_zeros( + zeros, zeros_shared + (k_0_0_ld)*CTA_N, N, cta_offset_m, cta_offset_n, k_0_0_ld, 0, k_0_0_ld < gemm_iters); + global_to_share_one_stage_zeros( + scales_i8, + scales_i8_shared + (k_0_0_ld)*CTA_N, + N, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + 0, + k_0_0_ld < gemm_iters); + + if constexpr (STAGES > 1) __pipeline_commit(); + } + if constexpr (STAGES > 1) __pipeline_wait_prior(STAGES - 2); + __syncthreads(); + + share_to_reg_one_stage_A( + A_shared + warp_offset_k, A_shared_warp_[0], warp_offset_m, warp_offset_n, 0, WARP_M / INTRIN_M); + share_to_reg_one_stage_B( + B_shared + warp_offset_k * PACK_SIZE, + B_shared_warp_[0], + zeros_shared, + scales_i8_shared, + warp_offset_m, + warp_offset_n, + 0, + 0, + WARP_N / 32); + constexpr int SHARED_K_ITERS = WARP_K / INTRIN_K; + + for (; k_0_0 < gemm_iters; ++k_0_0, ++k_0_0_ld) { + int ld_stage = k_0_0_ld % STAGES; + int compute_stage = k_0_0 % STAGES; + int8_t* A_shared_this_compute_stage; + int8_t* B_shared_this_compute_stage; + int8_t* zeros_shared_this_compute_stage; + int8_t* scales_i8_shared_this_compute_stage; + + for (int iter_k = 0; iter_k < SHARED_K_ITERS; ++iter_k) { + A_shared_this_compute_stage = A_shared + compute_stage * kSmemSizeAPerStage + warp_offset_k; + B_shared_this_compute_stage = B_shared + compute_stage * kSmemSizeBPerStage + warp_offset_k * PACK_SIZE; + zeros_shared_this_compute_stage = zeros_shared + (compute_stage)*CTA_N; + scales_i8_shared_this_compute_stage = scales_i8_shared + (compute_stage)*CTA_N; + + share_to_reg_one_stage_A( + A_shared_this_compute_stage, + A_shared_warp_[(iter_k + 1) % 2], + warp_offset_m, + warp_offset_n, + (iter_k + 1) % SHARED_K_ITERS, + WARP_M / INTRIN_M); + share_to_reg_one_stage_B( + B_shared_this_compute_stage, + B_shared_warp_[(iter_k + 1) % 2], + zeros_shared_this_compute_stage, + scales_i8_shared_this_compute_stage, + warp_offset_m, + warp_offset_n, + k_0_0 + (iter_k == SHARED_K_ITERS - 1), + (iter_k + 1) % SHARED_K_ITERS, + WARP_N / 32); + int8_t* A_shared_warp = A_shared_warp_[iter_k % 2]; + int8_t* B_shared_warp = B_shared_warp_[iter_k % 2]; + + for (int j_0_4 = 0; j_0_4 < WARP_N / INTRIN_N; ++j_0_4) { + for (int i_0_3 = 0; i_0_3 < WARP_M / INTRIN_M; ++i_0_3) { + mma_m16n8k32( + (void*)(C_warp + i_0_3 * WARP_N / INTRIN_N * 8 + j_0_4 * 8), + (void*)(A_shared_warp + i_0_3 * 16), + (void*)(B_shared_warp + j_0_4 * 16)); + mma_m16n8k32( + (void*)(C_warp + i_0_3 * WARP_N / INTRIN_N * 8 + j_0_4 * 8 + 4), + (void*)(A_shared_warp + i_0_3 * 16), + (void*)(B_shared_warp + j_0_4 * 16 + 8)); + } + } + + if (iter_k < SHARED_K_ITERS - 1) { + if constexpr (STAGES == 1) __syncthreads(); + global_to_share_one_stage_A( + A_hoisted, + A_shared_hoisted + ld_stage * kSmemSizeAPerStage, + K, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + iter_k, + k_0_0_ld < gemm_iters, + A_g2s_preds); + global_to_share_one_stage_B( + B_hoisted, + B_shared_hoisted + ld_stage * kSmemSizeBPerStage, + K, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + iter_k, + k_0_0_ld < gemm_iters); + } + + if (iter_k == SHARED_K_ITERS - 2) { + if constexpr (STAGES == 1 && SHARED_K_ITERS > 2) { + __syncthreads(); + } + global_to_share_one_stage_A( + A_hoisted, + A_shared_hoisted + ld_stage * kSmemSizeAPerStage, + K, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + iter_k + 1, + k_0_0_ld < gemm_iters, + A_g2s_preds); + global_to_share_one_stage_B( + B_hoisted, + B_shared_hoisted + ld_stage * kSmemSizeBPerStage, + K, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + iter_k + 1, + k_0_0_ld < gemm_iters); + global_to_share_one_stage_zeros( + zeros, + zeros_shared + (ld_stage)*CTA_N, + N, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + iter_k, + k_0_0_ld < gemm_iters); + global_to_share_one_stage_zeros( + scales_i8, + scales_i8_shared + (ld_stage)*CTA_N, + N, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + iter_k, + k_0_0_ld < gemm_iters); + if constexpr (STAGES > 1) { + __pipeline_commit(); + __pipeline_wait_prior(STAGES - 2); + } + compute_stage = (k_0_0 + 1) % STAGES; + __syncthreads(); + } + } + } + __pipeline_commit(); + __pipeline_wait_prior(0); + __syncthreads(); + + if constexpr (SLICES > 1) { +#pragma unroll + for (int z = 0; z < SLICES; ++z) { + if (slice_id == z) { +#pragma unroll + for (int ax0_0_1 = 0; ax0_0_1 < WARP_M / INTRIN_M; ++ax0_0_1) { +#pragma unroll + for (int ax1_0_1 = 0; ax1_0_1 < WARP_N / INTRIN_N; ++ax1_0_1) { +#pragma unroll + for (int local_id = 0; local_id < OP_M * 16 / WARP_SIZE; ++local_id) { + if (z > 0) { + C_warp[ax0_0_1 * WARP_N / INTRIN_N * 8 + ax1_0_1 * 8 + local_id] += C_shared + [warp_offset_m * CTA_N + ax0_0_1 * OP_M * CTA_N + warp_offset_n + ax1_0_1 * 16 + + ((local_id % 4) / 2 * 8 + (threadIdx.x / 4)) * CTA_N + (local_id / 4) * 8 + (local_id % 2) + + (threadIdx.x % 4) * 2]; + } + C_shared + [warp_offset_m * CTA_N + ax0_0_1 * OP_M * CTA_N + warp_offset_n + ax1_0_1 * 16 + + ((local_id % 4) / 2 * 8 + (threadIdx.x / 4)) * CTA_N + (local_id / 4) * 8 + (local_id % 2) + + (threadIdx.x % 4) * 2] = C_warp[ax0_0_1 * WARP_N / INTRIN_N * 8 + ax1_0_1 * 8 + local_id]; + }; + } + } + } + __syncthreads(); + } + if (slice_id == 0) { +#pragma unroll + for (int ax0_0_1 = 0; ax0_0_1 < WARP_M / INTRIN_M; ++ax0_0_1) { +#pragma unroll + for (int ax1_0_1 = 0; ax1_0_1 < WARP_N / INTRIN_N; ++ax1_0_1) { +#pragma unroll + for (int local_id = 0; local_id < OP_M * 16 / WARP_SIZE; ++local_id) { + C_warp[ax0_0_1 * WARP_N / INTRIN_N * 8 + ax1_0_1 * 8 + local_id] = C_shared + [warp_offset_m * CTA_N + ax0_0_1 * OP_M * CTA_N + warp_offset_n + ax1_0_1 * 16 + + ((local_id % 4) / 2 * 8 + (threadIdx.x / 4)) * CTA_N + (local_id / 4) * 8 + (local_id % 2) + + (threadIdx.x % 4) * 2]; + }; + } + } + } + } + + int row_wb_thd = cta_offset_m + warp_offset_m + (threadIdx.x / 4); + int col_wb_thd = cta_offset_n + warp_offset_n + (threadIdx.x % 4) * 2; + if (slice_id == 0) { + for (int ax0_0_1 = 0; ax0_0_1 < WARP_M / INTRIN_M; ++ax0_0_1) { + int row_wb_1 = row_wb_thd + ax0_0_1 * OP_M; + for (int ax1_0_1 = 0; ax1_0_1 < WARP_N / INTRIN_N; ++ax1_0_1) { + int col_wb_1 = col_wb_thd + ax1_0_1 * 16; + int* C_warp_local = C_warp + ax0_0_1 * WARP_N / INTRIN_N * 8 + ax1_0_1 * 8; + for (int local_id = 0; local_id < OP_M * 16 / WARP_SIZE; local_id += 2) { + int row_wb = row_wb_1 + (local_id % 4) / 2 * 8; + if (row_wb < M) { + int col_wb = col_wb_1 + (local_id / 4) * 8 + (local_id % 2); + float2 wscale = __half22float2(*(wscales + col_wb / 2)); + float ascale = __half2float(ascales[row_wb]); + float2 psums = + make_float2(__int2float_rn(C_warp_local[local_id]), __int2float_rn(C_warp_local[local_id + 1])); + psums.x *= wscale.x * ascale; + psums.y *= wscale.y * ascale; + *reinterpret_cast(C + row_wb * N + col_wb) = __float22half2_rn(psums); + } + }; + } + } + } +} +#else +template +__global__ void dense_kernel0( + int8_t* __restrict__ A, + int8_t* __restrict__ B, + int8_t* __restrict__ zeros, + int8_t* __restrict__ scales_i8, + half2* __restrict__ wscales, + half* __restrict__ ascales, + half* __restrict__ C, + int M, + int64_t N, + int64_t K) { + // Not implemented for SM < 800 + assert(false); + return; +} +#endif + +void qserve_w4a8_per_group_gemm( + const torch::Tensor& _in_feats, + const torch::Tensor& _kernel, + const torch::Tensor& _zeros, + const torch::Tensor& _scales_i8, + const torch::Tensor& _wscales, + const torch::Tensor& _ascales, + torch::Tensor& _out_feats) { + // Check input tensor + TORCH_CHECK(_in_feats.is_cuda(), "_in_feats must be a CUDA tensor"); + TORCH_CHECK(_in_feats.dim() == 2, "_in_feats must be a 2D tensor"); + TORCH_CHECK(_in_feats.is_contiguous(), "_in_feats must be contiguous"); + TORCH_CHECK(_in_feats.scalar_type() == torch::kInt8, "_in_feats must be int8"); + // Check kernel tensor + TORCH_CHECK(_kernel.is_cuda(), "_kernel must be a CUDA tensor"); + TORCH_CHECK(_kernel.dim() == 2, "_kernel must be a 2D tensor"); + TORCH_CHECK(_kernel.is_contiguous(), "_kernel must be contiguous"); + TORCH_CHECK(_kernel.scalar_type() == torch::kInt8, "_kernel must be int8"); + // Check output tensor + TORCH_CHECK(_out_feats.is_cuda(), "_out_feats must be a CUDA tensor"); + TORCH_CHECK(_out_feats.is_contiguous(), "_out_feats must be contiguous"); + TORCH_CHECK(_out_feats.scalar_type() == torch::kHalf, "_out_feats must be half"); + + int num_in_feats = _in_feats.size(0); + int num_in_channels = _in_feats.size(1); + int num_out_feats = _out_feats.size(-2); + int num_out_channels = _out_feats.size(-1); + + // Check matmul shape + TORCH_CHECK(num_out_channels == _kernel.size(0), "num_out_channels must be equal to _kernel.size(0)"); + TORCH_CHECK(num_in_feats == num_out_feats, "num_in_feats must be equal to num_out_feats"); + + // Check _ascales + TORCH_CHECK(_ascales.is_cuda(), "_ascales must be a CUDA tensor"); + TORCH_CHECK(_ascales.is_contiguous(), "_ascales must be contiguous"); + TORCH_CHECK(_ascales.scalar_type() == torch::kHalf, "_ascales must be half"); + TORCH_CHECK(_ascales.numel() == num_in_feats, "_ascales must have num_in_feats elements"); + + // Check _wscales + TORCH_CHECK(_wscales.is_cuda(), "_wscales must be a CUDA tensor"); + TORCH_CHECK(_wscales.is_contiguous(), "_wscales must be contiguous"); + TORCH_CHECK(_wscales.scalar_type() == torch::kHalf, "_wscales must be half"); + TORCH_CHECK(_wscales.numel() == num_out_channels, "_wscales must have num_out_channels elements"); + + // Check _scales_i8 + TORCH_CHECK(_scales_i8.is_cuda(), "_scales_i8 must be a CUDA tensor"); + TORCH_CHECK(_scales_i8.dim() == 2, "_scales_i8 must be a 2D tensor"); + TORCH_CHECK(_scales_i8.is_contiguous(), "_scales_i8 must be contiguous"); + TORCH_CHECK(_scales_i8.scalar_type() == torch::kInt8, "_scales_i8 must be int8"); + TORCH_CHECK(num_in_channels % _scales_i8.size(0) == 0, "num_in_channels must be divisible by _scales_i8.size(0)"); + TORCH_CHECK(num_out_channels == _scales_i8.size(1), "num_out_channels must be equal to _scales_i8.size(1)"); + + // Check _zeros + TORCH_CHECK(_zeros.is_cuda(), "_zeros must be a CUDA tensor"); + TORCH_CHECK(_zeros.dim() == 2, "_zeros must be a 2D tensor"); + TORCH_CHECK(_zeros.is_contiguous(), "_zeros must be contiguous"); + TORCH_CHECK(_zeros.scalar_type() == torch::kInt8, "_zeros must be int8"); + TORCH_CHECK(num_in_channels % _zeros.size(0) == 0, "num_in_channels must be divisible by _zeros.size(0)"); + TORCH_CHECK(num_out_channels == _zeros.size(1), "num_out_channels must be equal to _zeros.size(1)"); + + // Check group size + auto group_size = num_in_channels / _scales_i8.size(0); + TORCH_CHECK(group_size == 128, "group_size must be 128"); + + auto in_feats = reinterpret_cast(_in_feats.data_ptr()); + auto kernel = reinterpret_cast(_kernel.data_ptr()); + auto zeros = reinterpret_cast(_zeros.data_ptr()); + auto scales_i8 = reinterpret_cast(_scales_i8.data_ptr()); + auto wscales = reinterpret_cast(_wscales.data_ptr()); + auto ascales = reinterpret_cast(_ascales.data_ptr()); + // auto options = + // torch::TensorOptions().dtype(torch::kHalf).device(_in_feats.device()); + auto out_feats = reinterpret_cast(_out_feats.data_ptr()); + auto stream = at::cuda::getCurrentCUDAStream(_in_feats.get_device()); + auto sm_version = getSMVersion(); + if (sm_version >= 80) { + constexpr int G = 128; + + if (num_out_feats > 128) { + constexpr int CTA_M = 128; + constexpr int CTA_N = 64; + constexpr int CTA_K = 64; + constexpr int WARP_M = 64; + constexpr int WARP_N = 32; + constexpr int WARP_K = 64; + constexpr int STAGES = 4; + KERNEL_LAUNCH_CODE + } else if (num_out_feats >= 128) { + if (num_in_channels <= 4096) { + constexpr int CTA_M = 64; + constexpr int CTA_N = 64; + constexpr int CTA_K = 64; + constexpr int WARP_M = 32; + constexpr int WARP_N = 32; + constexpr int WARP_K = 64; + constexpr int STAGES = 4; + KERNEL_LAUNCH_CODE + } else { + constexpr int CTA_M = 64; + constexpr int CTA_N = 64; + constexpr int CTA_K = 128; + constexpr int WARP_M = 32; + constexpr int WARP_N = 32; + constexpr int WARP_K = 64; + constexpr int STAGES = 3; + KERNEL_LAUNCH_CODE + } + } else { + constexpr int CTA_M = 32; + constexpr int CTA_N = 64; + constexpr int CTA_K = 128; + constexpr int WARP_M = 32; + constexpr int WARP_N = 32; + constexpr int WARP_K = 64; + constexpr int STAGES = 3; + KERNEL_LAUNCH_CODE + } + } else { + TORCH_CHECK_NOT_IMPLEMENTED( + false, "No implemented qserve_w4a8_per_group_gemm for current compute capability: ", sm_version); + } + return; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/grammar/apply_token_bitmask_inplace_cuda.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/grammar/apply_token_bitmask_inplace_cuda.cu new file mode 100644 index 00000000..84b67855 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/grammar/apply_token_bitmask_inplace_cuda.cu @@ -0,0 +1,271 @@ +// Adapted from +// https://github.com/mlc-ai/xgrammar/blob/v0.1.18/python/xgrammar/kernels/apply_token_bitmask_inplace_cuda.cu + +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// clang-format off +#include +#include +#include +#include +#include + + +#if !defined(USE_ROCM) && (!defined(CUDA_VERSION) || CUDA_VERSION < 12040) +void ApplyTokenBitmaskInplace(at::Tensor logits, at::Tensor bitmask, at::optional indices = at::nullopt) { + TORCH_CHECK(false, "CUDA version must be >= 12.4 for ApplyTokenBitmaskInplace"); +} +#else + +#ifndef CUDART_INF_FP16 +#ifndef USE_ROCM +#define CUDART_INF_FP16 __ushort_as_half((unsigned short)0x7C00U) +#endif +#endif + +#ifndef CUDART_INF_BF16 +#ifndef USE_ROCM +#define CUDART_INF_BF16 __ushort_as_bfloat16((unsigned short)0x7F80U) +#endif +#endif + +constexpr int32_t BITS_PER_BLOCK = 32; +constexpr int32_t THREADS_PER_THREAD_BLOCK = 256; + +template +__device__ T NegativeInfinity() { + return -INFINITY; +} + +template <> +__device__ __half NegativeInfinity<__half>() { +#ifdef USE_ROCM + return __float2half(-INFINITY); +#else + return -CUDART_INF_FP16; +#endif +} + +template <> +__device__ __nv_bfloat16 NegativeInfinity<__nv_bfloat16>() { +#ifdef USE_ROCM + return __nv_bfloat16(-INFINITY); +#else + return -CUDART_INF_BF16; +#endif +} + +template +__device__ PackedT PackedNegativeInfinity() { + constexpr int kAlignment = sizeof(PackedT) / sizeof(T); + T packed[kAlignment]; +#pragma unroll + for (int i = 0; i < kAlignment; i++) { + packed[i] = NegativeInfinity(); + } + return *reinterpret_cast(packed); +} + +template +__global__ void __launch_bounds__(THREADS_PER_THREAD_BLOCK) LogitsBitmaskKernel( + T* __restrict__ logits, + const int32_t* __restrict__ bitmask, + const int32_t* __restrict__ indices, + int32_t vocab_size, + int32_t logits_stride, + int32_t bitmask_stride) { + constexpr int kAlignment = sizeof(PackedT) / sizeof(T); + constexpr uint32_t kPackedMask = (1 << kAlignment) - 1; + + const int batch_idx = (indices == nullptr) ? blockIdx.y : indices[blockIdx.y]; + + const int block_offset = blockIdx.x * THREADS_PER_THREAD_BLOCK * kBitsPerThread; + T* logits_gmem_ptr = logits + batch_idx * logits_stride + block_offset; + const int32_t* bitmask_gmem_ptr = bitmask + batch_idx * bitmask_stride + block_offset / BITS_PER_BLOCK; + const int bitmask_inner_idx = threadIdx.x % (BITS_PER_BLOCK / kAlignment); + T logits_reg[kAlignment]; + +#pragma unroll + for (int offset = threadIdx.x * kAlignment; offset < THREADS_PER_THREAD_BLOCK * kBitsPerThread; + offset += THREADS_PER_THREAD_BLOCK * kAlignment) { + if (block_offset + offset >= vocab_size) { + break; + } + + const uint32_t bitmask_val = + (~bitmask_gmem_ptr[offset / BITS_PER_BLOCK] >> (bitmask_inner_idx * kAlignment)) & kPackedMask; + + if (bitmask_val == 0) { + continue; + } + + if (bitmask_val == kPackedMask) { + *reinterpret_cast(logits_gmem_ptr + offset) = PackedNegativeInfinity(); + continue; + } + + *reinterpret_cast(logits_reg) = *reinterpret_cast(logits_gmem_ptr + offset); +#pragma unroll + for (int i = 0; i < kAlignment; i++) { + if (((bitmask_val >> i) & 1)) { + logits_reg[i] = NegativeInfinity(); + } + } + *reinterpret_cast(logits_gmem_ptr + offset) = *reinterpret_cast(logits_reg); + } +} + +template ::value>> +constexpr auto CeilDiv(T numerator, T denominator) { + return (numerator + denominator - 1) / denominator; +} + +template +void ApplyTokenBitmaskInplaceDispatchToBitsPerThread( + T* __restrict__ logits, + const int32_t* __restrict__ bitmask, + const int32_t* __restrict__ indices, + int32_t vocab_size, + int32_t logits_stride, + int32_t bitmask_stride, + int32_t num_rows) { + constexpr int kAlignment = sizeof(PackedT) / sizeof(T); + const int32_t num_blocks_per_row = CeilDiv(2048 / THREADS_PER_THREAD_BLOCK * 128, num_rows); + const int32_t num_bits_per_thread = CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * num_blocks_per_row); + + const dim3 block(THREADS_PER_THREAD_BLOCK); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + + if (num_bits_per_thread <= 4 && kAlignment <= 4) { + const dim3 grid(CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * 4), num_rows); + LogitsBitmaskKernel + <<>>(logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride); + } else if (num_bits_per_thread <= 8 && kAlignment <= 8) { + const dim3 grid(CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * 8), num_rows); + LogitsBitmaskKernel + <<>>(logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride); + } else if (num_bits_per_thread <= 16 && kAlignment <= 16) { + const dim3 grid(CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * 16), num_rows); + LogitsBitmaskKernel + <<>>(logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride); + } else { + const dim3 grid(CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * 32), num_rows); + LogitsBitmaskKernel + <<>>(logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride); + } +} + +template +void ApplyTokenBitmaskInplaceDispatchToPackedT( + T* __restrict__ logits, + const int32_t* __restrict__ bitmask, + const int32_t* __restrict__ indices, + int32_t vocab_size, + int32_t logits_stride, + int32_t bitmask_stride, + int32_t num_rows) { + if (logits_stride % (sizeof(float4) / sizeof(T)) == 0) { + ApplyTokenBitmaskInplaceDispatchToBitsPerThread( + logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride, num_rows); + } else { + ApplyTokenBitmaskInplaceDispatchToBitsPerThread( + logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride, num_rows); + } +} + +void ApplyTokenBitmaskInplace(at::Tensor logits, at::Tensor bitmask, at::optional indices = at::nullopt) { + TORCH_CHECK(logits.is_cuda(), "logits must be a CUDA tensor."); + TORCH_CHECK(logits.is_contiguous(), "logits must be contiguous."); + TORCH_CHECK(logits.dim() == 1 || logits.dim() == 2, "logits must be a 1D or 2D tensor."); + std::pair logits_shape = + logits.dim() == 2 ? std::make_pair(static_cast(logits.size(0)), static_cast(logits.size(1))) + : std::make_pair(1, static_cast(logits.size(0))); + + TORCH_CHECK(bitmask.is_cuda(), "bitmask must be a CUDA tensor."); + TORCH_CHECK(bitmask.is_contiguous(), "bitmask must be contiguous."); + TORCH_CHECK(bitmask.dim() == 1 || bitmask.dim() == 2, "bitmask must be a 1D or 2D tensor."); + std::pair bitmask_shape = + bitmask.dim() == 2 ? std::make_pair(static_cast(bitmask.size(0)), static_cast(bitmask.size(1))) + : std::make_pair(1, static_cast(bitmask.size(0))); + + TORCH_CHECK(bitmask.dtype() == torch::kInt32, "bitmask must be of type int32."); + + TORCH_CHECK( + (logits_shape.second + BITS_PER_BLOCK - 1) / BITS_PER_BLOCK >= bitmask_shape.second, + "The provided logits's vocab size should be no less than the bitmask's vocab size " + "(converted from bitmask size). But got vocab size ", + logits_shape.second, + " vs bitmask size ", + bitmask_shape.second); + + int vocab_size = std::min(logits_shape.second, bitmask_shape.second * BITS_PER_BLOCK); + + int32_t num_rows = logits_shape.first; + int32_t* indices_ptr = nullptr; + if (indices) { + TORCH_CHECK(indices->is_cuda(), "indices must be a CUDA tensor."); + TORCH_CHECK(indices->is_contiguous(), "indices must be contiguous."); + TORCH_CHECK(indices->dim() == 1, "indices must be a 1D tensor."); + TORCH_CHECK(indices->dtype() == torch::kInt32, "indices must be of type int32."); + num_rows = indices->size(0); + indices_ptr = indices->data_ptr(); + } else { + TORCH_CHECK(logits_shape.first == bitmask_shape.first, "logits and bitmask must have the same batch size."); + } + + switch (logits.scalar_type()) { + case torch::kFloat32: { + ApplyTokenBitmaskInplaceDispatchToPackedT( + logits.data_ptr(), + bitmask.data_ptr(), + indices_ptr, + vocab_size, + logits_shape.second, + bitmask_shape.second, + num_rows); + break; + } + case torch::kFloat16: { + ApplyTokenBitmaskInplaceDispatchToPackedT( + reinterpret_cast<__half*>(logits.data_ptr()), + bitmask.data_ptr(), + indices_ptr, + vocab_size, + logits_shape.second, + bitmask_shape.second, + num_rows); + break; + } + case torch::kBFloat16: { + ApplyTokenBitmaskInplaceDispatchToPackedT( + reinterpret_cast<__nv_bfloat16*>(logits.data_ptr()), + bitmask.data_ptr(), + indices_ptr, + vocab_size, + logits_shape.second, + bitmask_shape.second, + num_rows); + break; + } + default: + TORCH_CHECK(false, "logits dtype must be float, half or bfloat16."); + break; + } +} +#endif +// clang-format on diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/kvcacheio/transfer.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/kvcacheio/transfer.cu new file mode 100644 index 00000000..c1f37dfc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/kvcacheio/transfer.cu @@ -0,0 +1,807 @@ +#include +#include +#include + +#include + +#ifndef USE_ROCM +#define WARP_SIZE 32 +#include "pytorch_extension_utils.h" +#else +#include "pytorch_extension_utils_rocm.h" +#include "utils.h" // WARP_SIZE +#endif + +__device__ __forceinline__ void +transfer_item_warp(int32_t lane_id, const void* src_addr, void* dst_addr, int64_t item_size_bytes) { + const uint64_t* __restrict__ src = static_cast(src_addr); + uint64_t* __restrict__ dst = static_cast(dst_addr); + const int total_chunks = item_size_bytes / sizeof(uint64_t); + +#pragma unroll + for (int j = lane_id; j < total_chunks; j += WARP_SIZE) { +#ifndef USE_ROCM + uint64_t tmp; + asm volatile("ld.global.nc.b64 %0,[%1];" : "=l"(tmp) : "l"(src + j) : "memory"); + asm volatile("st.global.cg.b64 [%0],%1;" ::"l"(dst + j), "l"(tmp) : "memory"); + +#else + uint64_t tmp = __builtin_nontemporal_load(src + j); + __builtin_nontemporal_store(tmp, dst + j); +#endif + } +} + +template +__device__ __forceinline__ T* get_global_offset_lf( + T* base, + const uintptr_t* __restrict__ /*unused*/, + int64_t layer_id, + int64_t layer_dim, + int64_t page_id, + int64_t item_size_bytes) { + // layer first + return base + layer_id * layer_dim + page_id * item_size_bytes; +} + +template +__device__ __forceinline__ T* get_global_offset_pf( + T* base, + const uintptr_t* __restrict__ /*unused*/, + int64_t layer_id, + int64_t page_dim, + int64_t page_id, + int64_t item_size_bytes) { + // page first + return base + page_id * page_dim + layer_id * item_size_bytes; +} + +// get offset from layer base table when layers are not contiguous +template +__device__ __forceinline__ T* get_global_offset_lf_tbl( + T* /*unused*/, + const uintptr_t* __restrict__ layer_base_tbl, + int64_t layer_id, + int64_t /*unused*/, + int64_t page_id, + int64_t item_size_bytes) { + return reinterpret_cast(layer_base_tbl[layer_id]) + page_id * item_size_bytes; +} + +template +__device__ __forceinline__ T* get_global_offset_per_head_lf( + T* base, + const uintptr_t* __restrict__ /*unused*/, + int64_t layer_id, + int64_t layer_dim, + int64_t page_id, + int64_t item_size_bytes, + int64_t head_id, + int64_t head_num, + int64_t /*unused*/) { + // layer first offset func per head + return base + layer_id * layer_dim + page_id * item_size_bytes + item_size_bytes / head_num * head_id; +} + +template +__device__ __forceinline__ T* get_global_offset_per_head_lf_tbl( + T* /*unused*/, + const uintptr_t* __restrict__ layer_base_tbl, + int64_t layer_id, + int64_t /*unused*/, + int64_t page_id, + int64_t item_size_bytes, + int64_t head_id, + int64_t head_num, + int64_t /*unused*/) { + return reinterpret_cast(layer_base_tbl[layer_id]) + page_id * item_size_bytes + + item_size_bytes / head_num * head_id; +} + +template +__device__ __forceinline__ T* get_global_offset_ph( + T* base, + const uintptr_t* __restrict__ /*unused*/, + int64_t layer_id, + int64_t page_dim, + int64_t page_id, + int64_t item_size_bytes, + int64_t head_id, + int64_t head_num, + int64_t page_size) { + // page head layout: [page_num, head_num, page_size, layer_num, head_dim] + return base + page_id / page_size * page_size * page_dim + // page_num dimension offset + page_dim / head_num * head_id * page_size + // head_num dimension offset + page_id % page_size * page_dim / head_num + // page_size dimension offset + layer_id * item_size_bytes / head_num; // layer_num dimension offset +} + +template +__global__ void transfer_page_head_kernel_impl( + const void* __restrict__ src_k, + void* __restrict__ dst_k, + const void* __restrict__ src_v, + void* __restrict__ dst_v, + const int64_t* __restrict__ src_indices, + const int64_t* __restrict__ dst_indices, + int64_t start_layer_id, + int64_t num_layers_to_process, + int64_t num_items, + int64_t items_per_warp, + int64_t item_size_bytes, + int64_t src_layout_dim, + int64_t dst_layout_dim, + const uintptr_t* __restrict__ src_k_layer_tbl, + const uintptr_t* __restrict__ dst_k_layer_tbl, + const uintptr_t* __restrict__ src_v_layer_tbl, + const uintptr_t* __restrict__ dst_v_layer_tbl, + const int64_t page_size, + const int64_t head_num) { + int32_t tid = blockIdx.x * blockDim.x + threadIdx.x; + int32_t lane_id = tid % WARP_SIZE; + int32_t warp_id = tid / WARP_SIZE; + const int64_t head_size_bytes = item_size_bytes / head_num; + + for (int i = 0; i < items_per_warp; ++i) { + int64_t item_id = warp_id * items_per_warp + i; + if (item_id >= num_items) { + break; + } + const int64_t src_page_id = src_indices[item_id]; + const int64_t dst_page_id = dst_indices[item_id]; + + // Loop over layers if necessary + for (int64_t layer_id = start_layer_id; layer_id < start_layer_id + num_layers_to_process; ++layer_id) { + // For page head layout, the cache of each head in the token is discontinuous, need to loop + for (int64_t head_id = 0; head_id < head_num; ++head_id) { + const char* src_k_ptr = SrcOffsetFn( + static_cast(src_k), + src_k_layer_tbl, + layer_id, + src_layout_dim, + src_page_id, + item_size_bytes, + head_id, + head_num, + page_size); + char* dst_k_ptr = DstOffsetFn( + static_cast(dst_k), + dst_k_layer_tbl, + layer_id, + dst_layout_dim, + dst_page_id, + item_size_bytes, + head_id, + head_num, + page_size); + transfer_item_warp(lane_id, src_k_ptr, dst_k_ptr, head_size_bytes); + + const char* src_v_ptr = SrcOffsetFn( + static_cast(src_v), + src_v_layer_tbl, + layer_id, + src_layout_dim, + src_page_id, + item_size_bytes, + head_id, + head_num, + page_size); + char* dst_v_ptr = DstOffsetFn( + static_cast(dst_v), + dst_v_layer_tbl, + layer_id, + dst_layout_dim, + dst_page_id, + item_size_bytes, + head_id, + head_num, + page_size); + transfer_item_warp(lane_id, src_v_ptr, dst_v_ptr, head_size_bytes); + } + } + } +} + +template +__global__ void transfer_kernel_impl( + const void* __restrict__ src_k, + void* __restrict__ dst_k, + const void* __restrict__ src_v, + void* __restrict__ dst_v, + const int64_t* __restrict__ src_indices, + const int64_t* __restrict__ dst_indices, + int64_t start_layer_id, + int64_t num_layers_to_process, + int64_t num_items, + int64_t items_per_warp, + int64_t item_size_bytes, + int64_t src_layout_dim, + int64_t dst_layout_dim, + const uintptr_t* __restrict__ src_k_layer_tbl, + const uintptr_t* __restrict__ dst_k_layer_tbl, + const uintptr_t* __restrict__ src_v_layer_tbl, + const uintptr_t* __restrict__ dst_v_layer_tbl) { + int32_t tid = blockIdx.x * blockDim.x + threadIdx.x; + int32_t lane_id = tid % WARP_SIZE; + int32_t warp_id = tid / WARP_SIZE; + + for (int i = 0; i < items_per_warp; ++i) { + int64_t item_id = warp_id * items_per_warp + i; + if (item_id >= num_items) { + break; + } + const int64_t src_page_id = src_indices[item_id]; + const int64_t dst_page_id = dst_indices[item_id]; + + // Loop over layers if necessary + for (int64_t layer_id = start_layer_id; layer_id < start_layer_id + num_layers_to_process; ++layer_id) { + const char* src_ptr = SrcOffsetFn( + static_cast(src_k), src_k_layer_tbl, layer_id, src_layout_dim, src_page_id, item_size_bytes); + char* dst_ptr = DstOffsetFn( + static_cast(dst_k), dst_k_layer_tbl, layer_id, dst_layout_dim, dst_page_id, item_size_bytes); + transfer_item_warp(lane_id, src_ptr, dst_ptr, item_size_bytes); + + if constexpr (!IsMLA) { + const char* src_v_ptr = SrcOffsetFn( + static_cast(src_v), src_v_layer_tbl, layer_id, src_layout_dim, src_page_id, item_size_bytes); + char* dst_v_ptr = DstOffsetFn( + static_cast(dst_v), dst_v_layer_tbl, layer_id, dst_layout_dim, dst_page_id, item_size_bytes); + transfer_item_warp(lane_id, src_v_ptr, dst_v_ptr, item_size_bytes); + } + } + } +} + +template +void transfer_kv_launcher( + const at::Tensor& src_k, + at::Tensor& dst_k, + const at::Tensor& src_v, + at::Tensor& dst_v, + const at::Tensor& src_indices, + const at::Tensor& dst_indices, + int64_t start_layer_id, + int64_t num_layers_to_process, + int64_t item_size, + int64_t src_layout_dim, + int64_t dst_layout_dim, + const at::Tensor& src_k_layers, + const at::Tensor& dst_k_layers, + const at::Tensor& src_v_layers, + const at::Tensor& dst_v_layers, + int64_t block_quota, + int64_t num_warps_per_block, + const int64_t page_size = 16, + const int64_t head_num = 1) { + TORCH_CHECK(src_indices.is_cuda(), "Source indices must be a CUDA tensor"); + TORCH_CHECK(dst_indices.is_cuda(), "Destination indices must be a CUDA tensor"); + TORCH_CHECK(src_indices.scalar_type() == at::kLong, "Source indices must be of type long"); + TORCH_CHECK(dst_indices.scalar_type() == at::kLong, "Destination indices must be of type long"); + TORCH_CHECK(src_indices.numel() == dst_indices.numel(), "Source and destination indices must have the same length"); + TORCH_CHECK(item_size % 8 == 0, "Item byte size must be divisible by 8"); + + auto div_up = [](int64_t x, int64_t y) { return (x + y - 1) / y; }; + const int64_t num_items = src_indices.numel(); + const int64_t items_per_warp = div_up(num_items, block_quota * num_warps_per_block); + const int32_t num_blocks = div_up(num_items, items_per_warp * num_warps_per_block); + dim3 grid_dim(num_blocks, 1, 1); + const int32_t threads_per_block = num_warps_per_block * WARP_SIZE; + + const void* src_k_ptr = src_k.defined() ? src_k.data_ptr() : nullptr; + void* dst_k_ptr = dst_k.defined() ? dst_k.data_ptr() : nullptr; + const void* src_v_ptr = IsMLA || !src_v.defined() ? nullptr : src_v.data_ptr(); + void* dst_v_ptr = IsMLA || !dst_v.defined() ? nullptr : dst_v.data_ptr(); + const uintptr_t* src_k_tbl_ptr = src_k_layers.defined() ? src_k_layers.data_ptr() : nullptr; + const uintptr_t* dst_k_tbl_ptr = dst_k_layers.defined() ? dst_k_layers.data_ptr() : nullptr; + const uintptr_t* src_v_tbl_ptr = IsMLA || !src_v_layers.defined() ? nullptr : src_v_layers.data_ptr(); + const uintptr_t* dst_v_tbl_ptr = IsMLA || !dst_v_layers.defined() ? nullptr : dst_v_layers.data_ptr(); + + cudaStream_t torch_current_stream = at::cuda::getCurrentCUDAStream(); + if constexpr (PageHeadLayout) { + transfer_page_head_kernel_impl<<>>( + src_k_ptr, + dst_k_ptr, + src_v_ptr, + dst_v_ptr, + src_indices.data_ptr(), + dst_indices.data_ptr(), + start_layer_id, + num_layers_to_process, + num_items, + items_per_warp, + item_size, + src_layout_dim, + dst_layout_dim, + src_k_tbl_ptr, + dst_k_tbl_ptr, + src_v_tbl_ptr, + dst_v_tbl_ptr, + page_size, + head_num); + } else { + transfer_kernel_impl<<>>( + src_k_ptr, + dst_k_ptr, + src_v_ptr, + dst_v_ptr, + src_indices.data_ptr(), + dst_indices.data_ptr(), + start_layer_id, + num_layers_to_process, + num_items, + items_per_warp, + item_size, + src_layout_dim, + dst_layout_dim, + src_k_tbl_ptr, + dst_k_tbl_ptr, + src_v_tbl_ptr, + dst_v_tbl_ptr); + } + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} + +void transfer_kv_per_layer( + const at::Tensor src_k, + at::Tensor dst_k, + const at::Tensor src_v, + at::Tensor dst_v, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t block_quota, + int64_t num_warps_per_block) { + at::Tensor empty; + transfer_kv_launcher, get_global_offset_lf, false>( + src_k, + dst_k, + src_v, + dst_v, + src_indices, + dst_indices, + 0, + 1, + item_size, + 0, + 0, + empty, + empty, + empty, + empty, + block_quota, + num_warps_per_block); +} + +void transfer_kv_per_layer_pf_lf( + const at::Tensor src_k, + at::Tensor dst_k, + const at::Tensor src_v, + at::Tensor dst_v, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t layer_id, + int64_t item_size, + int64_t src_layout_dim, + int64_t block_quota, + int64_t num_warps_per_block) { + at::Tensor empty; + transfer_kv_launcher, get_global_offset_lf, false>( + src_k, + dst_k, + src_v, + dst_v, + src_indices, + dst_indices, + layer_id, + 1, + item_size, + src_layout_dim, + 0, + empty, + empty, + empty, + empty, + block_quota, + num_warps_per_block); +} + +void transfer_kv_per_layer_ph_lf( + const at::Tensor src_k, + at::Tensor dst_k, + const at::Tensor src_v, + at::Tensor dst_v, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t layer_id, + int64_t item_size, + int64_t src_layout_dim, + int64_t page_size, + int64_t head_num, + int64_t block_quota, + int64_t num_warps_per_block) { + at::Tensor empty; + transfer_kv_launcher, get_global_offset_per_head_lf, false, true>( + src_k, + dst_k, + src_v, + dst_v, + src_indices, + dst_indices, + layer_id, + 1, + item_size, + src_layout_dim, + 0, + empty, + empty, + empty, + empty, + block_quota, + num_warps_per_block, + page_size, + head_num); +} + +void transfer_kv_all_layer( + const at::Tensor src_k_layers, + const at::Tensor dst_k_layers, + const at::Tensor src_v_layers, + const at::Tensor dst_v_layers, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t num_layers, + int64_t block_quota, + int64_t num_warps_per_block) { + TORCH_CHECK(num_layers == src_k_layers.size(0), "Number of layers in source k tensor does not match num_layers"); + at::Tensor empty; + transfer_kv_launcher, get_global_offset_lf_tbl, false>( + empty, + empty, + empty, + empty, + src_indices, + dst_indices, + 0, + num_layers, + item_size, + 0, + 0, + src_k_layers, + dst_k_layers, + src_v_layers, + dst_v_layers, + block_quota, + num_warps_per_block); +} + +void transfer_kv_all_layer_lf_pf( + const at::Tensor src_k_layers, + at::Tensor dst_k, + const at::Tensor src_v_layers, + at::Tensor dst_v, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t dst_layout_dim, + int64_t num_layers, + int64_t block_quota, + int64_t num_warps_per_block) { + TORCH_CHECK(num_layers == src_k_layers.size(0), "Number of layers in source k tensor does not match num_layers"); + at::Tensor empty; + transfer_kv_launcher, get_global_offset_pf, false>( + empty, + dst_k, + empty, + dst_v, + src_indices, + dst_indices, + 0, + num_layers, + item_size, + 0, + dst_layout_dim, + src_k_layers, + empty, + src_v_layers, + empty, + block_quota, + num_warps_per_block); +} + +void transfer_kv_all_layer_lf_ph( + const at::Tensor src_k_layers, + at::Tensor dst_k, + const at::Tensor src_v_layers, + at::Tensor dst_v, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t dst_layout_dim, + int64_t num_layers, + int64_t page_size, + int64_t head_num, + int64_t block_quota, + int64_t num_warps_per_block) { + TORCH_CHECK(num_layers == src_k_layers.size(0), "Number of layers in source k tensor does not match num_layers"); + at::Tensor empty; + transfer_kv_launcher, get_global_offset_ph, false, true>( + empty, + dst_k, + empty, + dst_v, + src_indices, + dst_indices, + 0, + num_layers, + item_size, + 0, + dst_layout_dim, + src_k_layers, + empty, + src_v_layers, + empty, + block_quota, + num_warps_per_block, + page_size, + head_num); +} + +void transfer_kv_per_layer_mla( + const at::Tensor src, + at::Tensor dst, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t block_quota, + int64_t num_warps_per_block) { + at::Tensor empty; + transfer_kv_launcher, get_global_offset_lf, true>( + src, + dst, + empty, + empty, + src_indices, + dst_indices, + 0, + 1, + item_size, + 0, + 0, + empty, + empty, + empty, + empty, + block_quota, + num_warps_per_block); +} + +void transfer_kv_per_layer_mla_pf_lf( + const at::Tensor src, + at::Tensor dst, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t layer_id, + int64_t item_size, + int64_t src_layout_dim, + int64_t block_quota, + int64_t num_warps_per_block) { + at::Tensor empty; + transfer_kv_launcher, get_global_offset_lf, true>( + src, + dst, + empty, + empty, + src_indices, + dst_indices, + layer_id, + 1, + item_size, + src_layout_dim, + 0, + empty, + empty, + empty, + empty, + block_quota, + num_warps_per_block); +} + +void transfer_kv_all_layer_mla( + const at::Tensor src_layers, + const at::Tensor dst_layers, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t num_layers, + int64_t block_quota, + int64_t num_warps_per_block) { + TORCH_CHECK(num_layers == src_layers.size(0), "Number of layers in source tensor does not match num_layers"); + at::Tensor empty; + transfer_kv_launcher, get_global_offset_lf_tbl, true>( + empty, + empty, + empty, + empty, + src_indices, + dst_indices, + 0, + num_layers, + item_size, + 0, + 0, + src_layers, + dst_layers, + empty, + empty, + block_quota, + num_warps_per_block); +} + +void transfer_kv_all_layer_mla_lf_pf( + const at::Tensor src_layers, + at::Tensor dst, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t dst_layout_dim, + int64_t num_layers, + int64_t block_quota, + int64_t num_warps_per_block) { + TORCH_CHECK(num_layers == src_layers.size(0), "Number of layers in source tensor does not match num_layers"); + at::Tensor empty; + transfer_kv_launcher, get_global_offset_pf, true>( + empty, + dst, + empty, + empty, + src_indices, + dst_indices, + 0, + num_layers, + item_size, + 0, + dst_layout_dim, + src_layers, + empty, + empty, + empty, + block_quota, + num_warps_per_block); +} + +inline void transfer_page_direct( + const at::Tensor src_buffer, + at::Tensor dst_buffer, + int64_t src_page_index, + int64_t dst_page_index, + int64_t page_size) { + dst_buffer.slice(0, dst_page_index, dst_page_index + page_size) + .copy_( + src_buffer.slice(0, src_page_index, src_page_index + page_size), + /* non_blocking= */ true); +} + +void transfer_kv_direct( + const std::vector& src_layers, + std::vector dst_layers, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t page_size) { + TORCH_CHECK( + src_layers.size() == dst_layers.size(), "Source and destination layers must have the same number of layers"); + TORCH_CHECK(src_indices.numel() == dst_indices.numel(), "Source and destination indices must have the same length"); + TORCH_CHECK(page_size > 0, "Page size must be positive"); + TORCH_CHECK(src_indices.numel() % page_size == 0, "Source indices size must be divisible by page size"); + + auto src_indices_cpu = src_indices.cpu(); + auto dst_indices_cpu = dst_indices.cpu(); + + const auto num_indices = src_indices_cpu.numel(); + const int64_t num_layers = src_layers.size(); + int64_t* src_indices_ptr = src_indices_cpu.data_ptr(); + int64_t* dst_indices_ptr = dst_indices_cpu.data_ptr(); + + int64_t start_index = 0; + int64_t end_index = 0; + + for (int64_t i = 0; i < num_indices; ++i) { + if (i < num_indices - 1) { + auto src_diff = src_indices_ptr[i + 1] - src_indices_ptr[i]; + auto dst_diff = dst_indices_ptr[i + 1] - dst_indices_ptr[i]; + + if (src_diff == 1 && dst_diff == 1) { + continue; + } + end_index = i + 1; + } else { // last batch + end_index = num_indices; + } + auto src_index = src_indices_ptr[start_index]; + auto dst_index = dst_indices_ptr[start_index]; + auto num_tokens = end_index - start_index; + + for (int64_t j = 0; j < num_layers; ++j) { + transfer_page_direct(src_layers[j], dst_layers[j], src_index, dst_index, num_tokens); + } + start_index = end_index; + } +} + +template +inline void transfer_kv_page_first_direct_impl( + const std::vector& src_ptrs, + std::vector dst_ptrs, + const at::Tensor& src_indices, + const at::Tensor& dst_indices, + int64_t start_layer_id, + int64_t page_size) { + TORCH_CHECK(src_indices.numel() == dst_indices.numel(), "Source and destination indices must have the same length"); + TORCH_CHECK(page_size > 0, "Page size must be positive"); + TORCH_CHECK(src_indices.numel() % page_size == 0, "Source indices size must be divisible by page size"); + + auto src_indices_cpu = src_indices.cpu(); + auto dst_indices_cpu = dst_indices.cpu(); + const int64_t num_pages = src_indices_cpu.size(0) / page_size; + + if constexpr (IsLf2Pf) { + const bool is_mla = dst_ptrs.size() == 1; + const int64_t num_layers = is_mla ? src_ptrs.size() : src_ptrs.size() / 2; + + for (const auto i : c10::irange(num_pages)) { + auto s_index = src_indices_cpu[i * page_size].item(); + auto d_index = dst_indices_cpu[i * page_size].item() / page_size; + for (int64_t j = 0; j < num_layers; ++j) { + transfer_page_direct( + src_ptrs[j], dst_ptrs[0].select(0, d_index).select(0, start_layer_id + j), s_index, 0, page_size); + if (!is_mla) { + transfer_page_direct( + src_ptrs[j + num_layers], + dst_ptrs[1].select(0, d_index).select(0, start_layer_id + j), + s_index, + 0, + page_size); + } + } + } + } else { + const bool is_mla = src_ptrs.size() == 1; + const int64_t num_layers = is_mla ? dst_ptrs.size() : dst_ptrs.size() / 2; + + for (const auto i : c10::irange(num_pages)) { + auto s_index = src_indices_cpu[i * page_size].item() / page_size; + auto d_index = dst_indices_cpu[i * page_size].item(); + for (int64_t j = 0; j < num_layers; ++j) { + transfer_page_direct( + src_ptrs[0].select(0, s_index).select(0, start_layer_id + j), dst_ptrs[j], 0, d_index, page_size); + if (!is_mla) { + transfer_page_direct( + src_ptrs[1].select(0, s_index).select(0, start_layer_id + j), + dst_ptrs[j + num_layers], + 0, + d_index, + page_size); + } + } + } + } +} + +void transfer_kv_per_layer_direct_pf_lf( + const std::vector& src_ptrs, + std::vector dst_ptrs, + const at::Tensor& src_indices, + const at::Tensor& dst_indices, + int64_t layer_id, + int64_t page_size) { + transfer_kv_page_first_direct_impl(src_ptrs, dst_ptrs, src_indices, dst_indices, layer_id, page_size); +} + +void transfer_kv_all_layer_direct_lf_pf( + const std::vector& src_ptrs, + std::vector dst_ptrs, + const at::Tensor& src_indices, + const at::Tensor& dst_indices, + int64_t page_size) { + transfer_kv_page_first_direct_impl(src_ptrs, dst_ptrs, src_indices, dst_indices, 0, page_size); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/mamba/causal_conv1d.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/mamba/causal_conv1d.cu new file mode 100644 index 00000000..f31479fd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/mamba/causal_conv1d.cu @@ -0,0 +1,669 @@ +// clang-format off +// adapted from https://github.com/Dao-AILab/causal-conv1d/blob/main/csrc/causal_conv1d_fwd.cu +// and https://github.com/Dao-AILab/causal-conv1d/blob/main/csrc/causal_conv1d_update.cu +#include +#include +#include + +#include "causal_conv1d.h" +#include +#include +#include // For C10_CUDA_CHECK and C10_CUDA_KERNEL_LAUNCH_CHECK + +#include +#include + +#define BOOL_SWITCH(COND, CONST_NAME, ...) \ + [&] { \ + if (COND) { \ + static constexpr bool CONST_NAME = true; \ + return __VA_ARGS__(); \ + } else { \ + static constexpr bool CONST_NAME = false; \ + return __VA_ARGS__(); \ + } \ + }() + +#define CHECK_SHAPE(x, ...) TORCH_CHECK(x.sizes() == torch::IntArrayRef({__VA_ARGS__}), #x " must have shape (" #__VA_ARGS__ ")") + +#define DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(ITYPE, NAME, ...) \ + if (ITYPE == at::ScalarType::Half) { \ + using input_t = at::Half; \ + using weight_t = at::Half; \ + __VA_ARGS__(); \ + } else if (ITYPE == at::ScalarType::BFloat16) { \ + using input_t = at::BFloat16; \ + using weight_t = at::BFloat16; \ + __VA_ARGS__(); \ + } else if (ITYPE == at::ScalarType::Float) { \ + using input_t = float; \ + using weight_t = float; \ + __VA_ARGS__(); \ + } else { \ + AT_ERROR(#NAME, " not implemented for input type '", toString(ITYPE), "'"); \ + } + + +template +void causal_conv1d_fwd_cuda(ConvParamsBase ¶ms, cudaStream_t stream); + +template +void causal_conv1d_update_cuda(ConvParamsBase ¶ms, cudaStream_t stream); + +void set_conv_params_fwd(ConvParamsBase ¶ms, + // sizes + const size_t batch, + const size_t dim, + const size_t seqlen, + const size_t width, + // device pointers + const at::Tensor x, + const at::Tensor weight, + const at::Tensor out, + const std::optional& bias, + bool silu_activation, + int64_t pad_slot_id, + const std::optional& query_start_loc = std::nullopt, + const std::optional& cache_indices = std::nullopt, + const std::optional& has_initial_state = std::nullopt) { + + // Reset the parameters + memset(¶ms, 0, sizeof(params)); + + params.batch = batch; + params.dim = dim; + params.seqlen = seqlen; + params.width = width; + params.pad_slot_id = pad_slot_id; + + params.silu_activation = silu_activation; + + // Set the pointers and strides. + params.x_ptr = x.data_ptr(); + params.weight_ptr = weight.data_ptr(); + params.bias_ptr = bias.has_value() ? bias.value().data_ptr() : nullptr; + params.out_ptr = out.data_ptr(); + // All stride are in elements, not bytes. + params.query_start_loc_ptr = query_start_loc.has_value() ? query_start_loc.value().data_ptr() : nullptr; + params.cache_indices_ptr = cache_indices.has_value() ? cache_indices.value().data_ptr() : nullptr; + params.has_initial_state_ptr = has_initial_state.has_value() ? has_initial_state.value().data_ptr() : nullptr; + const bool varlen = params.query_start_loc_ptr != nullptr; + params.x_batch_stride = x.stride(varlen ? 1 : 0); + params.x_c_stride = x.stride(varlen ? 0 : 1); + params.x_l_stride = x.stride(varlen ? 1 : -1); + params.weight_c_stride = weight.stride(0); + params.weight_width_stride = weight.stride(1); + params.out_batch_stride = out.stride(varlen ? 1 : 0); + params.out_c_stride = out.stride(varlen ? 0 : 1); + params.out_l_stride = out.stride(varlen ? 1 : -1); +} + + +void causal_conv1d_fwd(const at::Tensor &x, const at::Tensor &weight, + const std::optional &bias_, + const std::optional &conv_states, + const std::optional &query_start_loc, + const std::optional &cache_indices, + const std::optional &has_initial_state, + bool silu_activation, + // used to identify padding entries if cache_indices provided + // in case of padding, the kernel will return early + int64_t pad_slot_id) { + auto input_type = x.scalar_type(); + auto weight_type = weight.scalar_type(); + TORCH_CHECK(input_type == at::ScalarType::Float || input_type == at::ScalarType::Half || input_type == at::ScalarType::BFloat16); + TORCH_CHECK(weight_type == at::ScalarType::Float || weight_type == at::ScalarType::Half || weight_type == at::ScalarType::BFloat16); + + TORCH_CHECK(x.is_cuda()); + TORCH_CHECK(weight.is_cuda()); + + const bool varlen = query_start_loc.has_value() ? true : false; + const auto sizes = x.sizes(); + const int batch_size = varlen ? query_start_loc.value().sizes()[0] - 1 : sizes[0]; + const int dim = varlen ? sizes[0] : sizes[1]; + const int seqlen = varlen ? sizes[1] : sizes[2]; + const int width = weight.size(-1); + if (varlen){ + CHECK_SHAPE(x, dim, seqlen); + } + else { + CHECK_SHAPE(x, batch_size, dim, seqlen); + } + CHECK_SHAPE(weight, dim, width); + + + + if (bias_.has_value()) { + auto bias = bias_.value(); + TORCH_CHECK(bias.scalar_type() == weight_type); + TORCH_CHECK(bias.is_cuda()); + TORCH_CHECK(bias.stride(-1) == 1); + CHECK_SHAPE(bias, dim); + } + + + if (has_initial_state.has_value()) { + auto has_initial_state_ = has_initial_state.value(); + TORCH_CHECK(has_initial_state_.scalar_type() == at::ScalarType::Bool); + TORCH_CHECK(has_initial_state_.is_cuda()); + CHECK_SHAPE(has_initial_state_, batch_size); + } + + + if (query_start_loc.has_value()) { + auto query_start_loc_ = query_start_loc.value(); + TORCH_CHECK(query_start_loc_.scalar_type() == at::ScalarType::Int); + TORCH_CHECK(query_start_loc_.is_cuda()); + } + + + if (cache_indices.has_value()) { + auto cache_indices_ = cache_indices.value(); + TORCH_CHECK(cache_indices_.scalar_type() == at::ScalarType::Int); + TORCH_CHECK(cache_indices_.is_cuda()); + CHECK_SHAPE(cache_indices_, batch_size); + } + + at::Tensor out = x; + + ConvParamsBase params; + set_conv_params_fwd(params, batch_size, dim, seqlen, width, x, weight, out, + bias_, + silu_activation, + pad_slot_id, + query_start_loc, + cache_indices, + has_initial_state + ); + + if (conv_states.has_value()) { + auto conv_states_ = conv_states.value(); + TORCH_CHECK(conv_states_.scalar_type() == input_type); + TORCH_CHECK(conv_states_.is_cuda()); + params.conv_states_ptr = conv_states_.data_ptr(); + params.conv_states_batch_stride = conv_states_.stride(0); + params.conv_states_c_stride = conv_states_.stride(-2); + params.conv_states_l_stride = conv_states_.stride(-1); + } else { + params.conv_states_ptr = nullptr; + } + + // Otherwise the kernel will be launched from cuda:0 device + // Cast to char to avoid compiler warning about narrowing + at::cuda::CUDAGuard device_guard{(char)x.get_device()}; + auto stream = at::cuda::getCurrentCUDAStream().stream(); + DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(x.scalar_type(), "causal_conv1d_fwd", [&] { + causal_conv1d_fwd_cuda(params, stream); + }); +} + + +void causal_conv1d_update(const at::Tensor &x, + const at::Tensor &conv_state, + const at::Tensor &weight, + const std::optional &bias_, + bool silu_activation, + const std::optional &cache_seqlens_, + const std::optional &conv_state_indices_, + // used to identify padding entries if cache_indices provided + // in case of padding, the kernel will return early + int64_t pad_slot_id) { + auto input_type = x.scalar_type(); + auto weight_type = weight.scalar_type(); + TORCH_CHECK(input_type == at::ScalarType::Float || input_type == at::ScalarType::Half || input_type == at::ScalarType::BFloat16); + TORCH_CHECK(weight_type == at::ScalarType::Float || weight_type == at::ScalarType::Half || weight_type == at::ScalarType::BFloat16); + TORCH_CHECK(weight_type == input_type, "weight type must equal to input type, other variations are disabled due to binary size limitations"); + TORCH_CHECK(conv_state.scalar_type() == input_type); + + TORCH_CHECK(x.is_cuda()); + TORCH_CHECK(conv_state.is_cuda()); + TORCH_CHECK(weight.is_cuda()); + + const auto sizes = x.sizes(); + const int batch_size = sizes[0]; + const int dim = sizes[1]; + const int seqlen = sizes[2]; + const int width = weight.size(-1); + const int conv_state_len = conv_state.size(2); + TORCH_CHECK(conv_state_len >= width - 1); + + CHECK_SHAPE(x, batch_size, dim, seqlen); + CHECK_SHAPE(weight, dim, width); + + TORCH_CHECK(width >= 2 && width <= 4, "causal_conv1d only supports width between 2 and 4"); + + if (bias_.has_value()) { + auto bias = bias_.value(); + TORCH_CHECK(bias.scalar_type() == weight_type); + TORCH_CHECK(bias.is_cuda()); + TORCH_CHECK(bias.stride(-1) == 1); + CHECK_SHAPE(bias, dim); + } + + at::Tensor out = x; + + ConvParamsBase params; + set_conv_params_fwd(params, batch_size, dim, seqlen, width, x, weight, out, + bias_, + silu_activation, + pad_slot_id); + params.conv_state_ptr = conv_state.data_ptr(); + params.conv_state_len = conv_state_len; + // All stride are in elements, not bytes. + params.conv_state_batch_stride = conv_state.stride(0); + params.conv_state_c_stride = conv_state.stride(1); + params.conv_state_l_stride = conv_state.stride(2); + + if (cache_seqlens_.has_value()) { + auto cache_seqlens = cache_seqlens_.value(); + TORCH_CHECK(cache_seqlens.scalar_type() == torch::kInt32); + TORCH_CHECK(cache_seqlens.is_cuda()); + TORCH_CHECK(cache_seqlens.stride(-1) == 1); + CHECK_SHAPE(cache_seqlens, batch_size); + params.cache_seqlens = cache_seqlens.data_ptr(); + } else { + params.cache_seqlens = nullptr; + } + + if (conv_state_indices_.has_value()) { + auto conv_state_indices = conv_state_indices_.value(); + TORCH_CHECK(conv_state_indices.scalar_type() == torch::kInt32) + TORCH_CHECK(conv_state_indices.is_cuda()); + TORCH_CHECK(conv_state_indices.stride(0) == 1) + CHECK_SHAPE(conv_state_indices, batch_size); + + int conv_state_entries = conv_state.size(0); + CHECK_SHAPE(conv_state, conv_state_entries, dim, conv_state_len); + + params.conv_state_indices_ptr = conv_state_indices.data_ptr(); + } else { + CHECK_SHAPE(conv_state, batch_size, dim, conv_state_len); + params.conv_state_indices_ptr = nullptr; + } + + // Otherwise the kernel will be launched from cuda:0 device + // Cast to char to avoid compiler warning about narrowing + at::cuda::CUDAGuard device_guard{(char)x.get_device()}; + auto stream = at::cuda::getCurrentCUDAStream().stream(); + DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(x.scalar_type(), "causal_conv1d_update", [&] { + causal_conv1d_update_cuda(params, stream); + }); +} + +template +struct Causal_conv1d_fwd_kernel_traits { + using input_t = input_t_; + using weight_t = weight_t_; + static constexpr int kNThreads = kNThreads_; + static constexpr int kWidth = kWidth_; + static constexpr int kNBytes = sizeof(input_t); + static_assert(kNBytes == 2 || kNBytes == 4); + static constexpr int kNElts = kNBytes == 4 ? 4 : 8; + static_assert(kWidth <= kNElts); + static constexpr bool kIsVecLoad = kIsVecLoad_; + using vec_t = typename BytesToType::Type; + using BlockLoadT = cub::BlockLoad; + using BlockLoadVecT = cub::BlockLoad; + using BlockStoreT = cub::BlockStore; + using BlockStoreVecT = cub::BlockStore; + static constexpr int kSmemIOSize = kIsVecLoad + ? 0 + : custom_max({sizeof(typename BlockLoadT::TempStorage), sizeof(typename BlockStoreT::TempStorage)}); + static constexpr int kSmemExchangeSize = kNThreads * kNBytes * kNElts; + static constexpr int kSmemSize = kSmemIOSize + kSmemExchangeSize; +}; + +template +__global__ __launch_bounds__(Ktraits::kNThreads) +void causal_conv1d_fwd_kernel(ConvParamsBase params) { + constexpr int kWidth = Ktraits::kWidth; + constexpr int kNThreads = Ktraits::kNThreads; + constexpr int kNElts = Ktraits::kNElts; + constexpr bool kIsVecLoad = Ktraits::kIsVecLoad; + using input_t = typename Ktraits::input_t; + using vec_t = typename Ktraits::vec_t; + using weight_t = typename Ktraits::weight_t; + + // Shared memory. + extern __shared__ char smem_[]; + auto& smem_load = reinterpret_cast(smem_); + auto& smem_load_vec = reinterpret_cast(smem_); + auto& smem_store = reinterpret_cast(smem_); + auto& smem_store_vec = reinterpret_cast(smem_); + vec_t *smem_exchange = reinterpret_cast(smem_ + Ktraits::kSmemIOSize); + + const bool kVarlen = params.query_start_loc_ptr != nullptr; + const int tidx = threadIdx.x; + const int batch_id = blockIdx.x; + const int channel_id = blockIdx.y; + const int *query_start_loc = kVarlen ? reinterpret_cast(params.query_start_loc_ptr) : nullptr; + const int sequence_start_index = kVarlen ? query_start_loc[batch_id] : batch_id; + const int seqlen = kVarlen ? query_start_loc[batch_id + 1] - sequence_start_index : params.seqlen; + + input_t *x = reinterpret_cast(params.x_ptr) + sequence_start_index * params.x_batch_stride + + channel_id * params.x_c_stride; + weight_t *weight = reinterpret_cast(params.weight_ptr) + channel_id * params.weight_c_stride; + input_t *out = reinterpret_cast(params.out_ptr) + sequence_start_index * params.out_batch_stride + + channel_id * params.out_c_stride; + float bias_val = params.bias_ptr == nullptr ? 0.f : float(reinterpret_cast(params.bias_ptr)[channel_id]); + + bool has_initial_state = params.has_initial_state_ptr == nullptr ? false + : reinterpret_cast(params.has_initial_state_ptr)[batch_id]; + + int* cache_indices = params.cache_indices_ptr == nullptr ? nullptr + : reinterpret_cast(params.cache_indices_ptr); + int cache_index = cache_indices == nullptr ? batch_id : cache_indices[batch_id]; + // cache_index == params.pad_slot_id is defined as padding, so we exit early + if (cache_index == params.pad_slot_id){ + return; + } + input_t *conv_states = params.conv_states_ptr == nullptr ? nullptr + : reinterpret_cast(params.conv_states_ptr) + cache_index * params.conv_states_batch_stride + channel_id * params.conv_states_c_stride; + + // Thread 0 will load the last elements of the previous chunk, so we initialize those to 0. + if (tidx == 0) { + input_t initial_state[kNElts] = {0}; + if (has_initial_state) { + #pragma unroll + for (int w = 0; w < kWidth - 1; ++w){ initial_state[kNElts - 1 - (kWidth - 2) + w ] = conv_states[w]; } + } + smem_exchange[kNThreads - 1] = reinterpret_cast(initial_state)[0]; + } + + float weight_vals[kWidth]; + #pragma unroll + for (int i = 0; i < kWidth; ++i) { weight_vals[i] = float(weight[i * params.weight_width_stride]); } + + constexpr int kChunkSize = kNThreads * kNElts; + const int n_chunks = (seqlen + kChunkSize - 1) / kChunkSize; + for (int chunk = 0; chunk < n_chunks; ++chunk) { + input_t x_vals_load[2 * kNElts] = {0}; + if constexpr(kIsVecLoad) { + typename Ktraits::BlockLoadVecT(smem_load_vec).Load(reinterpret_cast(x), *reinterpret_cast(&x_vals_load[kNElts]), (seqlen - chunk * kChunkSize) / kNElts); + } else { + __syncthreads(); + typename Ktraits::BlockLoadT(smem_load).Load(x, *reinterpret_cast(&x_vals_load[kNElts]), seqlen - chunk * kChunkSize); + } + x += kChunkSize; + __syncthreads(); + // Thread kNThreads - 1 don't write yet, so that thread 0 can read + // the last elements of the previous chunk. + if (tidx < kNThreads - 1) { smem_exchange[tidx] = reinterpret_cast(x_vals_load)[1]; } + __syncthreads(); + reinterpret_cast(x_vals_load)[0] = smem_exchange[tidx > 0 ? tidx - 1 : kNThreads - 1]; + __syncthreads(); + // Now thread kNThreads - 1 can write the last elements of the current chunk. + if (tidx == kNThreads - 1) { smem_exchange[tidx] = reinterpret_cast(x_vals_load)[1]; } + + float x_vals[2 * kNElts]; + #pragma unroll + for (int i = 0; i < 2 * kNElts; ++i) { x_vals[i] = float(x_vals_load[i]); } + + float out_vals[kNElts]; + #pragma unroll + for (int i = 0; i < kNElts; ++i) { + out_vals[i] = bias_val; + #pragma unroll + for (int w = 0; w < kWidth; ++w) { + out_vals[i] += weight_vals[w] * x_vals[kNElts + i - (kWidth - w - 1)]; + } + } + + if (params.silu_activation) { + #pragma unroll + for (int i = 0; i < kNElts; ++i) { + out_vals[i] = out_vals[i] / (1 + expf(-out_vals[i])); + } + } + + input_t out_vals_store[kNElts]; + #pragma unroll + for (int i = 0; i < kNElts; ++i) { out_vals_store[i] = out_vals[i]; } + if constexpr(kIsVecLoad) { + typename Ktraits::BlockStoreVecT(smem_store_vec).Store(reinterpret_cast(out), reinterpret_cast(out_vals_store), (seqlen - chunk * kChunkSize) / kNElts); + } else { + typename Ktraits::BlockStoreT(smem_store).Store(out, out_vals_store, seqlen - chunk * kChunkSize); + } + out += kChunkSize; + + int final_state_position = ((seqlen - (kWidth - 1)) - (n_chunks - 1) * kChunkSize); + // in case the final state is separated between the last "smem_exchange" and + // and the one before it (chunk = n_chunks - 1 and chunk = n_chunks - 2), + // (which occurs when `final_state_position` is a non-positivie index) + // we load the correct data from smem_exchange from both chunks, the last chunk iteration and the one before it + if (conv_states != nullptr && final_state_position < 0 && seqlen > kWidth){ + input_t vals_load[kNElts] = {0}; + if ((chunk == n_chunks - 2) && (tidx == kNThreads - 1)){ + // chunk = n_chunks - 2, a segment of the final state sits in the last index + reinterpret_cast(vals_load)[0] = smem_exchange[kNThreads - 1]; + #pragma unroll + for (int w = 0; w < -final_state_position; ++w){ + conv_states[w] = vals_load[kNElts + final_state_position + w]; + } + } + if ((chunk == n_chunks - 1) && tidx == 0){ + // chunk = n_chunks - 1, the second segment of the final state first positions + reinterpret_cast(vals_load)[0] = smem_exchange[0]; + for (int w = -final_state_position; w < kWidth - 1; ++w){ + conv_states[w] = vals_load[w + final_state_position]; + } + return; + } + } + } + // Final state is stored in the smem_exchange last token slot, + // in case seqlen < kWidth, we would need to take the final state from the + // initial state which is stored in conv_states + // in case seqlen > kWidth, we would need to load the last kWidth - 1 data + // and load it into conv_state accordingly + int last_thread = ((seqlen - (kWidth - 1)) - (n_chunks - 1) * kChunkSize) / kNElts; + if (conv_states != nullptr && tidx == last_thread) { + input_t x_vals_load[kNElts * 2] = {0}; + // in case we are on the first kWidth tokens + if (last_thread == 0 && seqlen < kWidth){ + // Need to take the initial state + reinterpret_cast(x_vals_load)[0] = smem_exchange[0]; + const int offset = seqlen - (kWidth - 1); + #pragma unroll + for (int w = 0; w < kWidth - 1; ++w){ + // pad the existing state + if ((w - seqlen) >= 0 && has_initial_state) { conv_states[w - seqlen] = conv_states[w]; } + else if ((w - seqlen) >= 0 && !has_initial_state) { conv_states[w - seqlen] = input_t(0.0f); } + } + #pragma unroll + for (int w = 0; w < kWidth - 1; ++w){ + if (offset + w >= 0) + conv_states[w] = x_vals_load[offset + w ]; + } + } + else { + // in case the final state is in between the threads data + const int offset = ((seqlen - (kWidth - 1)) % (kNElts)); + if ((offset + kWidth - 2) >= kNElts && (last_thread + 1 < kNThreads)){ + // In case last_thread == kNThreads - 1, accessing last_thread + 1 will result in a + // illegal access error on H100. + // Therefore, we access last_thread + 1, only if the final state data sits there + reinterpret_cast(x_vals_load)[1] = smem_exchange[last_thread + 1]; + } + reinterpret_cast(x_vals_load)[0] = smem_exchange[last_thread]; + #pragma unroll + for (int w = 0; w < kWidth - 1; ++w){ + conv_states[w] = x_vals_load[offset + w ]; + } + } + + } +} + + +template +void causal_conv1d_fwd_launch(ConvParamsBase ¶ms, cudaStream_t stream) { + static constexpr int kNElts = sizeof(input_t) == 4 ? 4 : 8; + const bool kVarlen = params.query_start_loc_ptr != nullptr; + BOOL_SWITCH(params.seqlen % kNElts == 0 && !kVarlen, kIsVecLoad, [&] { + using Ktraits = Causal_conv1d_fwd_kernel_traits; + constexpr int kSmemSize = Ktraits::kSmemSize; + dim3 grid(params.batch, params.dim); + + auto kernel = &causal_conv1d_fwd_kernel; + + if (kSmemSize >= 48 * 1024) { + #ifndef USE_ROCM + C10_CUDA_CHECK(cudaFuncSetAttribute( + kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, kSmemSize)); + #else + // There is a slight signature discrepancy in HIP and CUDA "FuncSetAttribute" function. + C10_CUDA_CHECK(cudaFuncSetAttribute( + (void *) kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, kSmemSize)); + std::cerr << "Warning (causal_conv1d fwd launch): attempting to set maxDynamicSharedMemorySize on an AMD GPU which is currently a non-op (in ROCm versions <= 6.1). This might lead to undefined behavior. \n" << std::endl; + #endif + } + kernel<<>>(params); + + C10_CUDA_KERNEL_LAUNCH_CHECK(); + }); +} + +template +void causal_conv1d_fwd_cuda(ConvParamsBase ¶ms, cudaStream_t stream) { + if (params.width == 2) { + causal_conv1d_fwd_launch<128, 2, input_t, weight_t>(params, stream); + } else if (params.width == 3) { + causal_conv1d_fwd_launch<128, 3, input_t, weight_t>(params, stream); + } else if (params.width == 4) { + causal_conv1d_fwd_launch<128, 4, input_t, weight_t>(params, stream); + } +} + + +template void causal_conv1d_fwd_cuda(ConvParamsBase ¶ms, cudaStream_t stream); +template void causal_conv1d_fwd_cuda(ConvParamsBase ¶ms, cudaStream_t stream); +template void causal_conv1d_fwd_cuda(ConvParamsBase ¶ms, cudaStream_t stream); + + + + +template +struct Causal_conv1d_update_kernel_traits { + using input_t = input_t_; + using weight_t = weight_t_; + static constexpr int kNThreads = kNThreads_; + static constexpr int kWidth = kWidth_; + static constexpr int kNBytes = sizeof(input_t); + static_assert(kNBytes == 2 || kNBytes == 4); +}; + +template +__global__ __launch_bounds__(Ktraits::kNThreads) +void causal_conv1d_update_kernel(ConvParamsBase params) { + constexpr int kWidth = Ktraits::kWidth; + constexpr int kNThreads = Ktraits::kNThreads; + using input_t = typename Ktraits::input_t; + using weight_t = typename Ktraits::weight_t; + + const int tidx = threadIdx.x; + const int batch_id = blockIdx.x; + const int channel_id = blockIdx.y * kNThreads + tidx; + if (channel_id >= params.dim) return; + + input_t *x = reinterpret_cast(params.x_ptr) + batch_id * params.x_batch_stride + + channel_id * params.x_c_stride; + + // If params.conv_state_batch_indices is set, then the conv state is gathered from the conv state tensor + // along the batch axis. Otherwise, the conv state coordinate is the same as the batch id. + const int conv_state_batch_coord = params.conv_state_indices_ptr == nullptr + ? batch_id + : params.conv_state_indices_ptr[batch_id]; + // conv_state_batch_coord == params.pad_slot_id is defined as padding so we exit early + if (conv_state_batch_coord == params.pad_slot_id){ + return; + } + input_t *conv_state = reinterpret_cast(params.conv_state_ptr) + + conv_state_batch_coord * params.conv_state_batch_stride + + channel_id * params.conv_state_c_stride; + + weight_t *weight = reinterpret_cast(params.weight_ptr) + channel_id * params.weight_c_stride; + input_t *out = reinterpret_cast(params.out_ptr) + batch_id * params.out_batch_stride + + channel_id * params.out_c_stride; + float bias_val = params.bias_ptr == nullptr ? 0.f : float(reinterpret_cast(params.bias_ptr)[channel_id]); + + int state_len = params.conv_state_len; + int advance_len = params.seqlen; + int cache_seqlen = kIsCircularBuffer ? params.cache_seqlens[batch_id] % state_len : 0; + int update_idx = cache_seqlen - (kWidth - 1); + update_idx = update_idx < 0 ? update_idx + state_len : update_idx; + + float weight_vals[kWidth] = {0}; + #pragma unroll + for (int i = 0; i < kWidth; ++i) { weight_vals[i] = float(weight[i * params.weight_width_stride]); } + + float x_vals[kWidth] = {0}; + if constexpr (!kIsCircularBuffer) { + #pragma unroll 2 + for (int i = 0; i < state_len - advance_len - (kWidth - 1); ++i) { + conv_state[i * params.conv_state_l_stride] = conv_state[(i + advance_len) * params.conv_state_l_stride]; + } + #pragma unroll + for (int i = 0; i < kWidth - 1; ++i) { + input_t state_val = conv_state[(state_len - (kWidth - 1) + i) * params.conv_state_l_stride]; + if (i < advance_len + (kWidth - 1) && state_len - advance_len - (kWidth - 1) + i >= 0) { + conv_state[(state_len - advance_len - (kWidth - 1) + i) * params.conv_state_l_stride] = state_val; + } + x_vals[i] = float(state_val); + } + } else { + #pragma unroll + for (int i = 0; i < kWidth - 1; ++i, update_idx = update_idx + 1 >= state_len ? update_idx + 1 - state_len : update_idx + 1) { + input_t state_val = conv_state[update_idx * params.conv_state_l_stride]; + x_vals[i] = float(state_val); + } + } + #pragma unroll 2 + for (int i = 0; i < params.seqlen; ++i) { + input_t x_val = x[i * params.x_l_stride]; + if constexpr (!kIsCircularBuffer) { + if (i < advance_len && state_len - advance_len + i >= 0) { + conv_state[(state_len - advance_len + i) * params.conv_state_l_stride] = x_val; + } + } else { + conv_state[update_idx * params.conv_state_l_stride] = x_val; + ++update_idx; + update_idx = update_idx >= state_len ? update_idx - state_len : update_idx; + } + x_vals[kWidth - 1] = float(x_val); + float out_val = bias_val; + #pragma unroll + for (int j = 0; j < kWidth; ++j) { out_val += weight_vals[j] * x_vals[j]; } + if (params.silu_activation) { out_val = out_val / (1 + expf(-out_val)); } + out[i * params.out_l_stride] = input_t(out_val); + // Shift the input buffer by 1 + #pragma unroll + for (int i = 0; i < kWidth - 1; ++i) { x_vals[i] = x_vals[i + 1]; } + } +} + +template +void causal_conv1d_update_launch(ConvParamsBase ¶ms, cudaStream_t stream) { + using Ktraits = Causal_conv1d_update_kernel_traits; + dim3 grid(params.batch, (params.dim + kNThreads - 1) / kNThreads); + auto kernel = params.cache_seqlens == nullptr + ? &causal_conv1d_update_kernel + : &causal_conv1d_update_kernel; + kernel<<>>(params); + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} + +template +void causal_conv1d_update_cuda(ConvParamsBase ¶ms, cudaStream_t stream) { + if (params.width == 2) { + causal_conv1d_update_launch<64, 2, input_t, weight_t>(params, stream); + } else if (params.width == 3) { + causal_conv1d_update_launch<64, 3, input_t, weight_t>(params, stream); + } else if (params.width == 4) { + causal_conv1d_update_launch<64, 4, input_t, weight_t>(params, stream); + } +} + +template void causal_conv1d_update_cuda(ConvParamsBase ¶ms, cudaStream_t stream); +template void causal_conv1d_update_cuda(ConvParamsBase ¶ms, cudaStream_t stream); +template void causal_conv1d_update_cuda(ConvParamsBase ¶ms, cudaStream_t stream); diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/mamba/causal_conv1d.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/mamba/causal_conv1d.h new file mode 100644 index 00000000..33f8e743 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/mamba/causal_conv1d.h @@ -0,0 +1,159 @@ +/****************************************************************************** + * Copyright (c) 2024, Tri Dao. + ******************************************************************************/ +// clang-format off +// adapted from https://github.com/Dao-AILab/causal-conv1d/blob/main/csrc/causal_conv1d.h +#pragma once + +#include +#include +//////////////////////////////////////////////////////////////////////////////////////////////////// + +struct ConvParamsBase { + using index_t = uint32_t; + + int batch, dim, seqlen, width; + int64_t pad_slot_id; + bool silu_activation; + + index_t x_batch_stride; + index_t x_c_stride; + index_t x_l_stride; + index_t weight_c_stride; + index_t weight_width_stride; + index_t out_batch_stride; + index_t out_c_stride; + index_t out_l_stride; + + int conv_state_len; + index_t conv_state_batch_stride; + index_t conv_state_c_stride; + index_t conv_state_l_stride; + + // Common data pointers. + void *__restrict__ x_ptr; + void *__restrict__ weight_ptr; + void *__restrict__ bias_ptr; + void *__restrict__ out_ptr; + + void *__restrict__ conv_state_ptr; + void *__restrict__ query_start_loc_ptr; + void *__restrict__ has_initial_state_ptr; + void *__restrict__ cache_indices_ptr; + int32_t *__restrict__ cache_seqlens; + + // For the continuous batching case. Makes it so that the mamba state for + // the current batch doesn't need to be a contiguous tensor. + int32_t *__restrict__ conv_state_indices_ptr; + + void *__restrict__ seq_idx_ptr; + + // No __restrict__ since initial_states could be the same as final_states. + void * initial_states_ptr; + index_t initial_states_batch_stride; + index_t initial_states_l_stride; + index_t initial_states_c_stride; + + void * final_states_ptr; + index_t final_states_batch_stride; + index_t final_states_l_stride; + index_t final_states_c_stride; + + void * conv_states_ptr; + index_t conv_states_batch_stride; + index_t conv_states_l_stride; + index_t conv_states_c_stride; +}; + + +#ifndef USE_ROCM + #include + + template + __device__ inline T shuffle_xor(T val, int offset) { + return __shfl_xor_sync(uint32_t(-1), val, offset); + } + + constexpr size_t custom_max(std::initializer_list ilist) + { + return std::max(ilist); + } + + template + constexpr T constexpr_min(T a, T b) { + return std::min(a, b); + } + +#else + #include + + template + __device__ inline T shuffle_xor(T val, int offset) { + return __shfl_xor(val, offset); + } + constexpr size_t custom_max(std::initializer_list ilist) + { + return *std::max_element(ilist.begin(), ilist.end()); + } + + template + constexpr T constexpr_min(T a, T b) { + return a < b ? a : b; + } +#endif + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template struct BytesToType {}; + +template<> struct BytesToType<16> { + using Type = uint4; + static_assert(sizeof(Type) == 16); +}; + +template<> struct BytesToType<8> { + using Type = uint64_t; + static_assert(sizeof(Type) == 8); +}; + +template<> struct BytesToType<4> { + using Type = uint32_t; + static_assert(sizeof(Type) == 4); +}; + +template<> struct BytesToType<2> { + using Type = uint16_t; + static_assert(sizeof(Type) == 2); +}; + +template<> struct BytesToType<1> { + using Type = uint8_t; + static_assert(sizeof(Type) == 1); +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct SumOp { +__device__ inline T operator()(T const & x, T const & y) { return x + y; } +}; + +template +struct Allreduce { + static_assert(THREADS == 32 || THREADS == 16 || THREADS == 8 || THREADS == 4); + template + static __device__ inline T run(T x, Operator &op) { + constexpr int OFFSET = THREADS / 2; + x = op(x, __shfl_xor_sync(uint32_t(-1), x, OFFSET)); + return Allreduce::run(x, op); + } +}; + +template<> +struct Allreduce<2> { +template +static __device__ inline T run(T x, Operator &op) { + x = op(x, __shfl_xor_sync(uint32_t(-1), x, 1)); + return x; +} +}; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/memory/store.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/memory/store.cu new file mode 100644 index 00000000..c6dd97eb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/memory/store.cu @@ -0,0 +1,147 @@ +#include +#include +#include +#include + +#include +#include +#include + +namespace { + +using std::size_t; +using std::uint64_t; + +// Each warp will process 256 bytes per loop iteration +template +__global__ void store_kv_cache_256x1( + uint64_t* __restrict__ k_cache, + uint64_t* __restrict__ v_cache, + const T* __restrict__ out_loc, + const size_t length, + const uint64_t* __restrict__ k, + const uint64_t* __restrict__ v, + const size_t kv_cache_stride, + const size_t kv_input_stride, + const size_t num_items) { + const auto idx = blockIdx.x * blockDim.x + threadIdx.x; + const auto warp_id = idx / 32; + const auto lane_id = idx % 32; + if (warp_id >= length) return; + const auto offset = out_loc[warp_id]; + const auto k_dst = k_cache + offset * kv_cache_stride; + const auto v_dst = v_cache + offset * kv_cache_stride; + const auto k_src = k + warp_id * kv_input_stride; + const auto v_src = v + warp_id * kv_input_stride; + for (size_t i = 0; i < num_items; ++i) { + k_dst[lane_id + i * 32] = k_src[lane_id + i * 32]; + v_dst[lane_id + i * 32] = v_src[lane_id + i * 32]; + } +} + +// Each warp will process 128 bytes per loop iteration +template +__global__ void store_kv_cache_128x2( + uint64_t* __restrict__ k_cache, + uint64_t* __restrict__ v_cache, + const T* __restrict__ out_loc, + const size_t length, + const uint64_t* __restrict__ k, + const uint64_t* __restrict__ v, + const size_t kv_cache_stride, + const size_t kv_input_stride, + const size_t num_items) { + const auto idx = blockIdx.x * blockDim.x + threadIdx.x; + const auto warp_id = idx / 32; + const auto lane_id = idx % 32; + if (warp_id >= length) return; + const auto offset = out_loc[warp_id]; + const auto copy_k = lane_id < 16; + const auto copy_id = lane_id % 16; + const auto cache = copy_k ? k_cache : v_cache; + const auto input = copy_k ? k : v; + const auto dst = cache + offset * kv_cache_stride; + const auto src = input + warp_id * kv_input_stride; + for (size_t i = 0; i < num_items; ++i) { + dst[copy_id + i * 16] = src[copy_id + i * 16]; + } +} + +} // namespace + +auto store_kv_cache(at::Tensor k_cache, at::Tensor v_cache, at::Tensor out_loc, at::Tensor k, at::Tensor v) -> void { + const auto max_tokens = k_cache.size(0); + const auto num_tokens = out_loc.size(0); + k_cache = k_cache.view({max_tokens, -1}); + v_cache = v_cache.view({max_tokens, -1}); + k = k.view({num_tokens, -1}); + v = v.view({num_tokens, -1}); + + TORCH_CHECK( + k_cache.is_cuda() && v_cache.is_cuda() && out_loc.is_cuda() && k.is_cuda() && v.is_cuda(), + "All tensors must be CUDA tensors"); + TORCH_CHECK(k_cache.sizes() == v_cache.sizes(), "k_cache and v_cache must have the same size"); + TORCH_CHECK(k_cache.strides() == v_cache.strides(), "k_cache and v_cache must have the same strides"); + TORCH_CHECK(k.sizes() == v.sizes(), "k and v must have the same size"); + TORCH_CHECK(k.strides() == v.strides(), "k and v must have the same strides"); + TORCH_CHECK(k.stride(-1) == 1 && k_cache.stride(-1) == 1, "k and k_cache must be contiguous in head."); + TORCH_CHECK(k.size(-1) == k_cache.size(-1), "k and k_cache must have the same head size"); + TORCH_CHECK(out_loc.dim() == 1 && out_loc.is_contiguous(), "out_loc must be a 1D contiguous tensor"); + static_assert(sizeof(uint64_t) == 8, "uint64_t must be 8 bytes, our code assumes that"); + + const auto length = out_loc.size(0); + const auto elem_size = k.element_size(); + const auto size_bytes = elem_size * k.size(-1); + const auto kv_cache_stride_bytes = elem_size * k_cache.stride(-2); + const auto kv_input_stride_bytes = elem_size * k.stride(-2); + const auto kv_cache_stride = kv_cache_stride_bytes / 8; + const auto kv_input_stride = kv_input_stride_bytes / 8; + + const auto k_cache_ptr = static_cast(k_cache.data_ptr()); + const auto v_cache_ptr = static_cast(v_cache.data_ptr()); + const auto k_ptr = static_cast(k.data_ptr()); + const auto v_ptr = static_cast(v.data_ptr()); + const auto num_threads = 256; + const auto num_warps = num_threads / 32; + const auto num_blocks = (length + num_warps - 1) / num_warps; + const auto stream = at::cuda::getCurrentCUDAStream(); + + AT_DISPATCH_INTEGRAL_TYPES(out_loc.scalar_type(), "store_kv_cache", [&] { + if constexpr (!std::is_same_v && !std::is_same_v) { + // do not instantiate the kernel if out_loc is not int32 or int64 + TORCH_CHECK(false, "out_loc must be of type int32 or int64, got: ", out_loc.scalar_type()); + } else { + if (size_bytes % 256 == 0) { + const auto items_per_warp = size_bytes / 256; + store_kv_cache_256x1<<>>( + k_cache_ptr, + v_cache_ptr, + out_loc.data_ptr(), + length, + k_ptr, + v_ptr, + kv_cache_stride, + kv_input_stride, + items_per_warp); + } else if (size_bytes % 128 == 0) { + const auto items_per_warp = size_bytes / 128; + store_kv_cache_128x2<<>>( + k_cache_ptr, + v_cache_ptr, + out_loc.data_ptr(), + length, + k_ptr, + v_ptr, + kv_cache_stride, + kv_input_stride, + items_per_warp); + } else { + TORCH_CHECK( + false, + "The last dimension size bytes of k and v must be" + " divisible by 128 at least, got: ", + size_bytes); + } + } + }); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/memory/weak_ref_tensor.cpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/memory/weak_ref_tensor.cpp new file mode 100644 index 00000000..6d7f8c4e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/memory/weak_ref_tensor.cpp @@ -0,0 +1,35 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Adapted from: https://github.com/vllm-project/vllm/blob/main/csrc/ops.h + +#include +#include + +#include + +at::Tensor weak_ref_tensor(const at::Tensor& tensor) { + TORCH_CHECK(tensor.is_cuda(), "weak_ref_tensor expects a CUDA tensor"); + + void* data_ptr = tensor.data_ptr(); + std::vector sizes = tensor.sizes().vec(); + std::vector strides = tensor.strides().vec(); + + auto options = tensor.options(); + + auto new_tensor = at::from_blob(data_ptr, sizes, strides, options); + + return new_tensor; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/scaled_mm_entry.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/scaled_mm_entry.cu new file mode 100644 index 00000000..28ed28c1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/scaled_mm_entry.cu @@ -0,0 +1,91 @@ +#include +#include +#include + +int32_t get_sm_version_num() { + int32_t major_capability, minor_capability; + cudaDeviceGetAttribute(&major_capability, cudaDevAttrComputeCapabilityMajor, 0); + cudaDeviceGetAttribute(&minor_capability, cudaDevAttrComputeCapabilityMinor, 0); + int32_t version_num = major_capability * 10 + minor_capability; + return version_num; +} + +void cutlass_w4a8_moe_mm_sm90( + torch::Tensor& d_tensors, + torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + torch::Tensor const& expert_offsets, + torch::Tensor const& problem_sizes, + torch::Tensor const& a_strides, + torch::Tensor const& b_strides, + torch::Tensor const& d_strides, + torch::Tensor const& s_strides, + int64_t chunk_size, + int64_t topk); + +void get_cutlass_w4a8_moe_mm_data_caller( + const torch::Tensor& topk_ids, + torch::Tensor& expert_offsets, + torch::Tensor& problem_sizes1, + torch::Tensor& problem_sizes2, + torch::Tensor& input_permutation, + torch::Tensor& output_permutation, + const int64_t num_experts, + const int64_t n, + const int64_t k); + +void cutlass_w4a8_moe_mm( + torch::Tensor& d_tensors, + torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + torch::Tensor const& expert_offsets, + torch::Tensor const& problem_sizes, + torch::Tensor const& a_strides, + torch::Tensor const& b_strides, + torch::Tensor const& d_strides, + torch::Tensor const& s_strides, + int64_t chunk_size, + int64_t topk) { + cutlass_w4a8_moe_mm_sm90( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size, + topk); + return; +} + +void get_cutlass_w4a8_moe_mm_data( + const torch::Tensor& topk_ids, + torch::Tensor& expert_offsets, + torch::Tensor& problem_sizes1, + torch::Tensor& problem_sizes2, + torch::Tensor& input_permutation, + torch::Tensor& output_permutation, + const int64_t num_experts, + const int64_t n, + const int64_t k) { + get_cutlass_w4a8_moe_mm_data_caller( + topk_ids, + expert_offsets, + problem_sizes1, + problem_sizes2, + input_permutation, + output_permutation, + num_experts, + n, + k); + return; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_get_group_starts.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_get_group_starts.cuh new file mode 100644 index 00000000..8f29ec37 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_get_group_starts.cuh @@ -0,0 +1,158 @@ +#pragma once + +#include +#include +#include + +#include "cutlass/bfloat16.h" +#include "cutlass/float8.h" + +template +__global__ void int4_fp8_get_group_gemm_starts( + int32_t* expert_offsets, + ElementA** a_offsets, + ElementB** b_offsets, + ElementC** out_offsets, + ElementAccumulator** a_scales_offsets, + cutlass::bfloat16_t** b_scales_offsets, + ElementA* a_base_as_int, + ElementB* b_base_as_int, + ElementC* out_base_as_int, + ElementAccumulator* a_scales_base_as_int, + cutlass::bfloat16_t* b_scales_base_as_int, + int64_t n, + int64_t k, + bool per_act_token, + bool per_out_ch) { + int expert_id = threadIdx.x; + int32_t expert_offset = expert_offsets[expert_id]; + + a_offsets[expert_id] = a_base_as_int + expert_offset * k; + b_offsets[expert_id] = b_base_as_int + expert_id * k * n / 2; + out_offsets[expert_id] = out_base_as_int + expert_offset * n; + a_scales_offsets[expert_id] = a_scales_base_as_int + (per_act_token ? expert_offset : 0); + b_scales_offsets[expert_id] = b_scales_base_as_int + (per_out_ch ? expert_id * n * k / 128 : expert_id); +} + +template +__global__ void int4_fp8_get_group_gemm_starts_3d( + ElementA** a_offsets, + ElementB** b_offsets, + ElementC** out_offsets, + ElementAccumulator** a_scales_offsets, + cutlass::bfloat16_t** b_scales_offsets, + ElementA* a_base_as_int, + ElementB* b_base_as_int, + ElementC* out_base_as_int, + ElementAccumulator* a_scales_base_as_int, + cutlass::bfloat16_t* b_scales_base_as_int, + int64_t n, + int64_t m, + int64_t k, + bool per_act_token, + bool per_out_ch, + int num_experts) { + int expert_id = blockIdx.x * blockDim.x + threadIdx.x; + if (expert_id >= num_experts) return; + + int64_t a_offset = expert_id * m * k; + int64_t b_offset = expert_id * k * n / 2; + int64_t out_offset = expert_id * m * n; + int64_t a_scales_offset = 0; + int64_t b_scales_offset = per_out_ch ? expert_id * n * 4 * k / 512 : expert_id; + + a_offsets[expert_id] = a_base_as_int + a_offset; + b_offsets[expert_id] = b_base_as_int + b_offset; + out_offsets[expert_id] = out_base_as_int + out_offset; + a_scales_offsets[expert_id] = a_scales_base_as_int + a_scales_offset; + b_scales_offsets[expert_id] = b_scales_base_as_int + b_scales_offset; +} + +#define __CALL_W4A8_GET_STARTS_KERNEL(TENSOR_C_TYPE, C_TYPE) \ + else if (out_tensors.dtype() == TENSOR_C_TYPE) { \ + int4_fp8_get_group_gemm_starts \ + <<<1, num_experts, 0, stream>>>( \ + static_cast(expert_offsets.data_ptr()), \ + static_cast(a_ptrs.data_ptr()), \ + static_cast(b_ptrs.data_ptr()), \ + static_cast(out_ptrs.data_ptr()), \ + static_cast(a_scales_ptrs.data_ptr()), \ + static_cast(b_scales_ptrs.data_ptr()), \ + static_cast(a_tensors.data_ptr()), \ + static_cast(b_tensors.data_ptr()), \ + static_cast(out_tensors.data_ptr()), \ + static_cast(a_scales.data_ptr()), \ + static_cast(b_scales.data_ptr()), \ + out_tensors.size(1), \ + a_tensors.size(1), \ + per_act_token, \ + per_out_ch); \ + } + +#define __CALL_W4A8_GET_STARTS_KERNEL_3D(TENSOR_C_TYPE, C_TYPE) \ + else if (out_tensors.dtype() == TENSOR_C_TYPE) { \ + int4_fp8_get_group_gemm_starts_3d \ + <<<1, num_experts, 0, stream>>>( \ + static_cast(a_ptrs.data_ptr()), \ + static_cast(b_ptrs.data_ptr()), \ + static_cast(out_ptrs.data_ptr()), \ + static_cast(a_scales_ptrs.data_ptr()), \ + static_cast(b_scales_ptrs.data_ptr()), \ + static_cast(a_tensors.data_ptr()), \ + static_cast(b_tensors.data_ptr()), \ + static_cast(out_tensors.data_ptr()), \ + static_cast(a_scales.data_ptr()), \ + static_cast(b_scales.data_ptr()), \ + out_tensors.size(2), \ + a_tensors.size(1), \ + a_tensors.size(2), \ + per_act_token, \ + per_out_ch, \ + num_experts); \ + } + +namespace { + +void run_int4_fp8_get_group_gemm_starts( + torch::Tensor const& expert_offsets, + torch::Tensor& a_ptrs, + torch::Tensor& b_ptrs, + torch::Tensor& out_ptrs, + torch::Tensor& a_scales_ptrs, + torch::Tensor& b_scales_ptrs, + torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, + torch::Tensor& out_tensors, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales) { + TORCH_CHECK(a_tensors.dtype() == torch::kFloat8_e4m3fn); + TORCH_CHECK(b_tensors.dtype() == torch::kInt8); + TORCH_CHECK(a_scales.dtype() == torch::kFloat32); + TORCH_CHECK(b_scales.dtype() == torch::kBFloat16); + + int num_experts = static_cast(expert_offsets.size(0)); + bool per_act_token = a_scales.numel() != 1; + bool per_out_ch = b_scales.numel() != num_experts; + + auto stream = at::cuda::getCurrentCUDAStream(expert_offsets.device().index()); + + if (a_tensors.dim() == 3) { + if (false) { + } + __CALL_W4A8_GET_STARTS_KERNEL_3D(torch::kBFloat16, cutlass::bfloat16_t) + __CALL_W4A8_GET_STARTS_KERNEL_3D(torch::kFloat16, half) + else { + TORCH_CHECK(false, "Invalid output type (must be float16 or bfloat16)"); + } + } else { + if (false) { + } + __CALL_W4A8_GET_STARTS_KERNEL(torch::kBFloat16, cutlass::bfloat16_t) + __CALL_W4A8_GET_STARTS_KERNEL(torch::kFloat16, half) + else { + TORCH_CHECK(false, "Invalid output type (must be float16 or bfloat16)"); + } + } +} + +} // namespace diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cu new file mode 100644 index 00000000..bd63d2ee --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cu @@ -0,0 +1,386 @@ +#include +#include +#include + +#include + +#include "cutlass/cutlass.h" +#include "w4a8_grouped_mm_c3x.cuh" + +using namespace cute; + +namespace { + +enum class Sched { PP, CO }; + +template +struct SM90W4A8Config { + using KernelSchedule = std::conditional_t< + S == Sched::PP, + cutlass::gemm::KernelPtrArrayTmaWarpSpecializedPingpong, + cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperative>; + + using EpilogueSchedule = std::conditional_t< + S == Sched::PP, + cutlass::epilogue::PtrArrayTmaWarpSpecializedPingpong, + cutlass::epilogue::PtrArrayTmaWarpSpecializedCooperative>; + + using TileShape = cute::Shape, cute::Int, cute::Int>; + using ClusterShape = cute::Shape, cute::Int, cute::Int>; + using Cutlass3xW4A8Gemm = cutlass_3x_w4a8_group_gemm; +}; + +template +using SM90_PP = SM90W4A8Config; + +template +using SM90_CO = SM90W4A8Config; + +template +inline void invoke_gemm( + torch::Tensor& d_tensors, + torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + torch::Tensor const& expert_offsets, + torch::Tensor const& problem_sizes, + torch::Tensor const& a_strides, + torch::Tensor const& b_strides, + torch::Tensor const& d_strides, + torch::Tensor const& s_strides, + int64_t chunk_size) { + using GemmT = typename Config::Cutlass3xW4A8Gemm; + cutlass_w4a8_group_gemm_caller( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); +} + +void dispatch_w4a8_moe_mm_sm90( + torch::Tensor& d_tensors, + torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + torch::Tensor const& expert_offsets, + torch::Tensor const& problem_sizes, + torch::Tensor const& a_strides, + torch::Tensor const& b_strides, + torch::Tensor const& d_strides, + torch::Tensor const& s_strides, + int64_t chunk_size, + int64_t topk) { + uint32_t const m = a_tensors.size(0) / topk; + uint32_t const n = d_tensors.size(1); + uint32_t const k = a_tensors.size(1); + + if (n == 4096 && k == 7168) { + // group gemm 1 + if (m <= 4) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else if (m <= 16) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else if (m <= 256) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else if (m <= 1024) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } + } else if (n == 7168 && k == 2048) { + // group gemm 2 + if (m <= 8) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else if (m <= 512) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } + } else if (n == 512 && k == 7168) { + // group gemm 1 for tp + if (m <= 4) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else if (m <= 16) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else if (m <= 256) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else if (m <= 1024) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } + } else if (n == 7168 && k == 256) { + // group gemm 2 for tp + if (m <= 8) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else if (m <= 512) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } + } else { + if (k % 512 == 0) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } + } +} + +} // namespace + +void cutlass_w4a8_moe_mm_sm90( + torch::Tensor& d_tensors, + torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + torch::Tensor const& expert_offsets, + torch::Tensor const& problem_sizes, + torch::Tensor const& a_strides, + torch::Tensor const& b_strides, + torch::Tensor const& d_strides, + torch::Tensor const& s_strides, + int64_t chunk_size, + int64_t topk) { + dispatch_w4a8_moe_mm_sm90( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size, + topk); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cuh new file mode 100644 index 00000000..5afd1c34 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cuh @@ -0,0 +1,279 @@ +#pragma once + +/** + * @file w4a8_grouped_mm_c3x.cuh + * @brief Implementation of grouped GEMM operation with int4 and fp8 mixed + * precision + * + * This file implements a grouped GEMM operation that multiplies FP8 matrices + * (A) with quantized INT4 matrices (B), applying per-block scaling factors. + * The implementation is optimized for NVIDIA Hopper GPUs, leveraging Tensor + * Cores for mixed precision arithmetic. + * + * Key features: + * - Supports grouped GEMM operations with multiple experts + * - Uses FP8 (e4m3) for matrix A + * - Uses INT4 quantization for matrix B with per-block scaling + * - Implements preprocessing for INT4 encoding and scale packing + * - Optimized for Hopper architecture with Tensor Core operations + */ + +#include +#include +#include +#include + +#include "cutlass/cutlass.h" +#include "cutlass/epilogue/collective/collective_builder.hpp" +#include "cutlass/gemm/device/gemm_universal_adapter.h" +#include "cutlass/gemm/dispatch_policy.hpp" +#include "cutlass/gemm/group_array_problem_shape.hpp" +#include "cutlass/gemm/kernel/gemm_universal.hpp" +#include "cutlass_extensions/gemm/collective/collective_builder_mixed_input.hpp" +#include "w4a8_get_group_starts.cuh" + +using namespace cute; + +namespace { + +// Type definitions +using MmaType = cutlass::float_e4m3_t; // FP8 e4m3 type +using QuantType = cutlass::int4b_t; // 4-bit integer type +using ElementAccumulator = float; // Accumulator type +using ElementScale = cutlass::bfloat16_t; // Scale type +using ElementC = cutlass::bfloat16_t; // Output type +using ElementD = ElementC; // Output type +using ProblemShape = cutlass::gemm::GroupProblemShape>; + +// Architecture-specific configurations +using ArchTag = cutlass::arch::Sm90; +using OperatorClass = cutlass::arch::OpClassTensorOp; +// constexpr int TileShapeK = 512; +// using TileShape = Shape<_128, _32, cute::Int>; +// using ClusterShape = Shape<_1, _1, _1>; + +// Layout configurations +using LayoutA = cutlass::layout::RowMajor; +using LayoutB = cutlass::layout::ColumnMajor; +using LayoutC = cutlass::layout::RowMajor; +using LayoutD = LayoutC; + +// Transposed layouts +using LayoutA_Transpose = typename cutlass::layout::LayoutTranspose::type; +using LayoutB_Transpose = typename cutlass::layout::LayoutTranspose::type; +using LayoutC_Transpose = typename cutlass::layout::LayoutTranspose::type; +using LayoutD_Transpose = typename cutlass::layout::LayoutTranspose::type; + +// Alignments +static constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; +static constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; +static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; +static constexpr int AlignmentD = 128 / cutlass::sizeof_bits::value; + +template +struct cutlass_3x_w4a8_group_gemm { + static constexpr int GroupSize = 128; + static constexpr int PackedScalesNum = get<2>(TileShape{}) / GroupSize; + using ElementScalePacked = cutlass::Array; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + TileShape, + ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementAccumulator, + ElementC, + LayoutC_Transpose*, + AlignmentC, + ElementD, + LayoutD_Transpose*, + AlignmentD, + EpilogueSchedule>::CollectiveOp; + + using CollectiveMainloopScaleOnly = typename cutlass::gemm::collective::CollectiveBuilderMixedInput< + ArchTag, + OperatorClass, + cute::tuple, + LayoutB_Transpose*, + AlignmentB, + MmaType, + LayoutA_Transpose*, + AlignmentA, + ElementAccumulator, + TileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + KernelSchedule>::CollectiveOp; + + // Define the final kernel and GEMM operation types + using GemmKernelScaleOnly = + cutlass::gemm::kernel::GemmUniversal; + + using GemmScaleOnly = cutlass::gemm::device::GemmUniversalAdapter; + + using StrideA = cute::remove_pointer_t>; + using StrideB = cute::remove_pointer_t>; + using StrideC = typename GemmKernelScaleOnly::InternalStrideC; + using StrideD = typename GemmKernelScaleOnly::InternalStrideD; + using StrideS = typename CollectiveMainloopScaleOnly::StrideScale; +}; + +/** + * @brief Main function to run int4 * fp8 grouped GEMM from PyTorch + * + * This function performs multiple GEMM operations in parallel where each + * operation multiplies an FP8 matrix (A) with a quantized INT4 matrix (B), + * applying per-channel scaling factors. It's designed for efficient execution + * on NVIDIA Hopper GPUs, leveraging Tensor Cores for optimal performance with + * mixed precision arithmetic. + * + * The function includes preprocessing steps for both INT4 tensors and scale + * factors to ensure optimal performance and correct operation. + * + * @param d_tensors Output tensor D with shape [total_m, total_n] + * @param a_tensors Tensor containing all A matrices (fp8_e4m3) with shape + * [total_m, K] + * @param b_tensors Tensor containing all B matrices (int4 packed as int8) with + * shape [E, N, K/2] + * @param a_scales Tensor containing A matrix scale factors + * @param b_scales Tensor containing B matrix scale factors with shape [E, + * K//512, N*4] + * @param expert_offsets Tensor containing expert offsets for determining group + * boundaries (int32) + * @param problem_sizes Tensor containing problem sizes with shape [num_experts, + * 3] (M, N, K for each group) (int32) + * @param a_strides Stride information for A tensors + * @param b_strides Stride information for B tensors + * @param d_strides Stride information for D tensors + * @param s_strides Stride information for scale tensors + * @param chunk_size Size of each chunk for scales (K / number of scale chunks) + */ +// template +template +void cutlass_w4a8_group_gemm_caller( + torch::Tensor& d_tensors, + torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + torch::Tensor const& expert_offsets, + torch::Tensor const& problem_sizes, + torch::Tensor const& a_strides, + torch::Tensor const& b_strides, + torch::Tensor const& d_strides, + torch::Tensor const& s_strides, + int64_t chunk_size) { + // using Gemm = cutlass_3x_w4a8_group_gemm; + using Args = typename Gemm::GemmScaleOnly::Arguments; + + int num_experts = static_cast(expert_offsets.size(0)); + bool per_act_token = a_scales.numel() != 1; + bool per_out_ch = b_scales.numel() != num_experts; + + // Check inputs + TORCH_CHECK(a_tensors.dim() == 2 or a_tensors.dim() == 3, "A tensor must be 2D/3D"); + TORCH_CHECK(b_tensors.dim() == 3, "B tensor must be 3D [E, N, K/2]"); + TORCH_CHECK(b_scales.dim() == 3, "Scale tensor must be 3D [E, K//512, N*4]"); + TORCH_CHECK(a_scales.dim() == 1, "A Scale tensor must be 1D [1]"); + TORCH_CHECK(expert_offsets.dim() == 1, "expert_offsets must be a 1D tensor"); + TORCH_CHECK(problem_sizes.dim() == 2, "problem_sizes must be 2D tensor"); + + // Check tensor shapes + TORCH_CHECK(problem_sizes.size(0) == num_experts, "problem_sizes must have num_experts rows"); + TORCH_CHECK(problem_sizes.size(1) == 3, "problem_sizes must have 3 columns (N, M, K)"); + TORCH_CHECK(b_tensors.size(0) == num_experts, "B tensor first dimension must match number of groups"); + TORCH_CHECK(b_scales.size(0) == num_experts, "Scale tensor first dimension must match number of groups"); + TORCH_CHECK( + b_tensors.size(2) * 2 == a_tensors.size(1) or b_tensors.size(2) * 2 == a_tensors.size(2), + "B tensor K/2 dimension must match A tensor K dimension"); + + // Check tensor types + TORCH_CHECK(a_tensors.scalar_type() == torch::kFloat8_e4m3fn, "A tensor must be fp8 (float_e4m3_t) type"); + TORCH_CHECK(b_tensors.scalar_type() == torch::kInt8, "B tensor must contain packed int4 values (stored as int8)"); + TORCH_CHECK(expert_offsets.scalar_type() == torch::kInt32, "Expert offsets must be int32 type"); + TORCH_CHECK(problem_sizes.scalar_type() == torch::kInt32, "Problem sizes must be int32 type"); + + auto stream = at::cuda::getCurrentCUDAStream(a_tensors.device().index()); + auto options_int = torch::TensorOptions().dtype(torch::kInt64).device(a_tensors.device()); + + torch::Tensor a_ptrs = torch::empty(num_experts, options_int); + torch::Tensor b_ptrs = torch::empty(num_experts, options_int); + torch::Tensor out_ptrs = torch::empty(num_experts, options_int); + torch::Tensor a_scales_ptrs = torch::empty(num_experts, options_int); + torch::Tensor b_scales_ptrs = torch::empty(num_experts, options_int); + + cutlass::KernelHardwareInfo hw_info; + hw_info.device_id = a_tensors.device().index(); + hw_info.sm_count = cutlass::KernelHardwareInfo::query_device_multiprocessor_count(hw_info.device_id); + + Args arguments; + decltype(arguments.epilogue.thread) fusion_args; + fusion_args.alpha = 0; + fusion_args.beta = 0; + fusion_args.alpha_ptr = a_scales.data_ptr(); + ; + fusion_args.beta_ptr = nullptr; + fusion_args.alpha_ptr_array = nullptr; + fusion_args.beta_ptr_array = nullptr; + fusion_args.dAlpha = {cute::_0{}, cute::_0{}, 0}; + fusion_args.dBeta = {cute::_0{}, cute::_0{}, 0}; + + ProblemShape::UnderlyingProblemShape* problem_sizes_as_shapes = + static_cast(problem_sizes.data_ptr()); + + run_int4_fp8_get_group_gemm_starts( + expert_offsets, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a_tensors, + b_tensors, + d_tensors, + a_scales, + b_scales); + + arguments = Args{ + cutlass::gemm::GemmUniversalMode::kGrouped, + {num_experts, problem_sizes_as_shapes, nullptr}, + {static_cast(b_ptrs.data_ptr()), + static_cast(b_strides.data_ptr()), + static_cast(a_ptrs.data_ptr()), + static_cast(a_strides.data_ptr()), + static_cast(b_scales_ptrs.data_ptr()), + static_cast(s_strides.data_ptr()), + static_cast(chunk_size)}, + {fusion_args, + nullptr, + nullptr, + static_cast(out_ptrs.data_ptr()), + static_cast(d_strides.data_ptr())}, + hw_info}; + + // Instantiate and run GEMM + typename Gemm::GemmScaleOnly gemm; + size_t workspace_size = Gemm::GemmScaleOnly::get_workspace_size(arguments); + auto const workspace_options = torch::TensorOptions().dtype(torch::kUInt8).device(a_tensors.device()); + auto workspace = torch::empty(workspace_size, workspace_options); + + cutlass::Status status = gemm.can_implement(arguments); + if (status != cutlass::Status::kSuccess) { + TORCH_CHECK(false, "GEMM implementation not supported"); + } + + status = gemm.initialize(arguments, workspace.data_ptr(), stream); + if (status != cutlass::Status::kSuccess) { + TORCH_CHECK(false, "GEMM initialization failed"); + } + + status = gemm.run(stream); + if (status != cutlass::Status::kSuccess) { + TORCH_CHECK(false, "GEMM execution failed"); + } +} + +} // namespace diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_moe_data.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_moe_data.cu new file mode 100644 index 00000000..f30f7b02 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_moe_data.cu @@ -0,0 +1,79 @@ +#include +#include +#include + +#include + +constexpr uint64_t THREADS_PER_EXPERT = 512; + +__global__ void compute_problem_sizes_w4a8( + const int32_t* __restrict__ topk_ids, + int32_t* problem_sizes1, + int32_t* problem_sizes2, + int32_t* atomic_buffer, + const int topk_length, + const int n, + const int k) { + int expert_id = blockIdx.x; + + int occurrences = 0; + for (int i = threadIdx.x; i < topk_length; i += THREADS_PER_EXPERT) { + occurrences += (topk_ids[i] == expert_id); + } + atomicAdd(&atomic_buffer[expert_id], occurrences); + __syncthreads(); + + if (threadIdx.x == 0) { + int final_occurrences = atomic_buffer[expert_id]; + problem_sizes1[expert_id * 3] = 2 * n; + problem_sizes1[expert_id * 3 + 1] = final_occurrences; + problem_sizes1[expert_id * 3 + 2] = k; + problem_sizes2[expert_id * 3] = k; + problem_sizes2[expert_id * 3 + 1] = final_occurrences; + problem_sizes2[expert_id * 3 + 2] = n; + } +} + +__global__ void compute_expert_offsets_w4a8( + const int32_t* __restrict__ problem_sizes1, + int32_t* expert_offsets, + int32_t* atomic_buffer, + const int num_experts) { + int32_t tot_offset = 0; + expert_offsets[0] = 0; + for (int i = 0; i < num_experts; ++i) { + atomic_buffer[i] = tot_offset; + tot_offset += problem_sizes1[i * 3 + 1]; + expert_offsets[i + 1] = tot_offset; + } +} + +void get_cutlass_w4a8_moe_mm_data_caller( + const torch::Tensor& topk_ids, + torch::Tensor& expert_offsets, + torch::Tensor& problem_sizes1, + torch::Tensor& problem_sizes2, + torch::Tensor& input_permutation, + torch::Tensor& output_permutation, + const int64_t num_experts, + const int64_t n, + const int64_t k) { + auto stream = at::cuda::getCurrentCUDAStream(topk_ids.device().index()); + auto options_int32 = torch::TensorOptions().dtype(torch::kInt32).device(topk_ids.device()); + torch::Tensor atomic_buffer = torch::zeros(num_experts, options_int32); + + int num_threads = min(THREADS_PER_EXPERT, topk_ids.numel()); + compute_problem_sizes_w4a8<<>>( + static_cast(topk_ids.data_ptr()), + static_cast(problem_sizes1.data_ptr()), + static_cast(problem_sizes2.data_ptr()), + static_cast(atomic_buffer.data_ptr()), + topk_ids.numel(), + n, + k); + compute_expert_offsets_w4a8<<<1, 1, 0, stream>>>( + static_cast(problem_sizes1.data_ptr()), + static_cast(expert_offsets.data_ptr()), + static_cast(atomic_buffer.data_ptr()), + num_experts); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe_helper.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe_helper.cu new file mode 100644 index 00000000..576ad233 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/cutlass_moe_helper.cu @@ -0,0 +1,142 @@ +#pragma once + +#include +#include +#include + +#include "cutlass/bfloat16.h" +#include "cutlass/float8.h" + +template < + typename ElementAB, + typename ElementC, + typename ElementAccumulator, + typename LayoutSFA, + typename LayoutSFB, + typename ScaleConfig> +__global__ void get_group_gemm_starts( + int32_t* expert_offsets, + ElementAB** a_offsets, + ElementAB** b_offsets, + ElementC** out_offsets, + ElementAccumulator** a_scales_offsets, + ElementAccumulator** b_scales_offsets, + ElementAB* a_base_as_int, + ElementAB* b_base_as_int, + ElementC* out_base_as_int, + ElementAccumulator* a_scales_base_as_int, + ElementAccumulator* b_scales_base_as_int, + LayoutSFA* layout_sfa_base_as_int, + LayoutSFB* layout_sfb_base_as_int, + int* problem_sizes, + int* problem_sizes_transpose, + bool transpose = false) { + int64_t expert_id = static_cast(threadIdx.x); + + if (expert_id >= gridDim.x * blockDim.x) { + return; + } + + int m = problem_sizes[expert_id * 3]; + int n = problem_sizes[expert_id * 3 + 1]; + int k = problem_sizes[expert_id * 3 + 2]; + if (transpose) { + problem_sizes_transpose[expert_id * 3] = n; + problem_sizes_transpose[expert_id * 3 + 1] = m; + problem_sizes_transpose[expert_id * 3 + 2] = k; + } + + int64_t expert_offset = static_cast(expert_offsets[expert_id]); + int64_t a_stride = 0; + int64_t b_stride = 0; + int64_t a_scale_stride = 0; + int64_t b_scale_stride = 0; + if (!transpose) { + a_stride = expert_offset * k; + b_stride = expert_id * k * n; + a_scale_stride = expert_offset * k / 128; + b_scale_stride = expert_id * k * n / 128 / 128; + } else { + a_stride = expert_id * k * n; + b_stride = expert_offset * k; + a_scale_stride = expert_id * k * n / 128 / 128; + b_scale_stride = expert_offset * k / 128; + } + a_offsets[expert_id] = a_base_as_int + a_stride; + b_offsets[expert_id] = b_base_as_int + b_stride; + out_offsets[expert_id] = out_base_as_int + expert_offset * n; + a_scales_offsets[expert_id] = a_scales_base_as_int + a_scale_stride; + b_scales_offsets[expert_id] = b_scales_base_as_int + b_scale_stride; + + LayoutSFA* layout_sfa_ptr = layout_sfa_base_as_int + expert_id; + LayoutSFB* layout_sfb_ptr = layout_sfb_base_as_int + expert_id; + + if (!transpose) { + *layout_sfa_ptr = ScaleConfig::tile_atom_to_shape_SFA(cute::make_shape(m, n, k, 1)); + *layout_sfb_ptr = ScaleConfig::tile_atom_to_shape_SFB(cute::make_shape(m, n, k, 1)); + } else { + *layout_sfa_ptr = ScaleConfig::tile_atom_to_shape_SFA(cute::make_shape(n, m, k, 1)); + *layout_sfb_ptr = ScaleConfig::tile_atom_to_shape_SFB(cute::make_shape(n, m, k, 1)); + } +} + +#define __CALL_GET_STARTS_KERNEL(TENSOR_C_TYPE, C_TYPE, LayoutSFA, LayoutSFB, ScaleConfig) \ + else if (out_tensors.dtype() == TENSOR_C_TYPE) { \ + get_group_gemm_starts \ + <<<1, num_experts, 0, stream>>>( \ + static_cast(expert_offsets.data_ptr()), \ + static_cast(a_ptrs.data_ptr()), \ + static_cast(b_ptrs.data_ptr()), \ + static_cast(out_ptrs.data_ptr()), \ + static_cast(a_scales_ptrs.data_ptr()), \ + static_cast(b_scales_ptrs.data_ptr()), \ + static_cast(a_tensors.data_ptr()), \ + static_cast(b_tensors.data_ptr()), \ + static_cast(out_tensors.data_ptr()), \ + static_cast(a_scales.data_ptr()), \ + static_cast(b_scales.data_ptr()), \ + reinterpret_cast(layout_sfa.data_ptr()), \ + reinterpret_cast(layout_sfb.data_ptr()), \ + static_cast(problem_sizes.data_ptr()), \ + static_cast(problem_sizes_transpose.data_ptr()), \ + transpose); \ + } + +namespace { +template +void run_get_group_gemm_starts( + torch::Tensor const& expert_offsets, + torch::Tensor& a_ptrs, + torch::Tensor& b_ptrs, + torch::Tensor& out_ptrs, + torch::Tensor& a_scales_ptrs, + torch::Tensor& b_scales_ptrs, + torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, + torch::Tensor& out_tensors, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + torch::Tensor const& layout_sfa, + torch::Tensor const& layout_sfb, + torch::Tensor const& problem_sizes, + torch::Tensor& problem_sizes_transpose, + bool transpose = false) { + TORCH_CHECK(a_tensors.dtype() == torch::kFloat8_e4m3fn); + TORCH_CHECK(b_tensors.dtype() == torch::kFloat8_e4m3fn); + TORCH_CHECK(a_scales.dtype() == torch::kFloat32); + TORCH_CHECK(b_scales.dtype() == torch::kFloat32); + TORCH_CHECK(out_tensors.size(1) % 128 == 0 or out_tensors.size(0) % 128 == 0); + TORCH_CHECK(a_tensors.size(1) % 128 == 0 or a_tensors.size(0) % 128 == 0); + + int num_experts = (int)expert_offsets.size(0); + auto stream = at::cuda::getCurrentCUDAStream(a_tensors.device().index()); + + if (false) { + } + __CALL_GET_STARTS_KERNEL(torch::kBFloat16, cutlass::bfloat16_t, LayoutSFA, LayoutSFB, ScaleConfig) + __CALL_GET_STARTS_KERNEL(torch::kFloat16, half, LayoutSFA, LayoutSFB, ScaleConfig) + else { + TORCH_CHECK(false, "Invalid output type (must be float16 or bfloat16)"); + } +} +} // namespace diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/fp8_blockwise_moe_kernel.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/fp8_blockwise_moe_kernel.cu new file mode 100644 index 00000000..e6a2ccbb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/fp8_blockwise_moe_kernel.cu @@ -0,0 +1,810 @@ +#include +#include +#include +#include + +#include "cute/tensor.hpp" +#include "cutlass/cutlass.h" +#include "cutlass/epilogue/collective/collective_builder.hpp" +#include "cutlass/epilogue/collective/default_epilogue.hpp" +#include "cutlass/epilogue/dispatch_policy.hpp" +#include "cutlass/epilogue/thread/activation.h" +#include "cutlass/epilogue/thread/linear_combination.h" +#include "cutlass/gemm/collective/collective_builder.hpp" +#include "cutlass/gemm/device/gemm_universal_adapter.h" +#include "cutlass/gemm/dispatch_policy.hpp" +#include "cutlass/gemm/group_array_problem_shape.hpp" +#include "cutlass/gemm/kernel/gemm_universal.hpp" +#include "cutlass/gemm/kernel/tile_scheduler_params.h" +#include "cutlass/tensor_ref.h" +#include "cutlass/util/command_line.h" +#include "cutlass/util/distribution.h" +#include "cutlass/util/host_tensor.h" +#include "cutlass/util/packed_stride.hpp" +#include "cutlass/util/reference/device/gemm.h" +#include "cutlass/util/reference/device/tensor_compare.h" +#include "cutlass/util/tensor_view_io.h" +#include "cutlass_moe_helper.cu" +#include "utils.h" + +using namespace cute; + +using ProblemShape = cutlass::gemm::GroupProblemShape>; + +template +void launch_sm90_fp8_blockwise_scaled_group_mm( + torch::Tensor& out_ptrs, + const torch::Tensor& a_ptrs, + const torch::Tensor& b_ptrs, + const torch::Tensor& a_scales_ptrs, + const torch::Tensor& b_scales_ptrs, + const torch::Tensor& stride_a, + const torch::Tensor& stride_b, + const torch::Tensor& stride_c, + const torch::Tensor& layout_sfa, + const torch::Tensor& layout_sfb, + const torch::Tensor& problem_sizes, + const torch::Tensor& expert_offsets, + const torch::Tensor& workspace) { + using ElementA = cutlass::float_e4m3_t; + using ElementB = cutlass::float_e4m3_t; + using ElementC = void; + using ElementD = OutType; + using ElementAccumulator = float; + using LayoutA = cutlass::layout::RowMajor; + using LayoutB = cutlass::layout::ColumnMajor; + using LayoutC = LayoutD; + + static constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + + using ArchTag = cutlass::arch::Sm90; + using OperatorClass = cutlass::arch::OpClassTensorOp; + static constexpr auto RoundStyle = cutlass::FloatRoundStyle::round_to_nearest; + using CustomEVTIdentity = // acc + cutlass::epilogue::fusion::Sm90EVT< + cutlass::epilogue::fusion:: + Sm90Compute, + cutlass::epilogue::fusion::Sm90AccFetch>; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + typename ScheduleConfig::MmaTileShape, + typename ScheduleConfig::ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementAccumulator, + ElementC, // Use void to avoid load Matrix C + LayoutC*, + AlignmentC, + ElementD, + LayoutC*, + AlignmentC, + typename ScheduleConfig::EpilogueSchedule, + CustomEVTIdentity>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementA, + cute::tuple, + AlignmentA, + ElementB, + cute::tuple, + AlignmentB, + ElementAccumulator, + typename ScheduleConfig::MmaTileShape, + typename ScheduleConfig::ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + typename ScheduleConfig::KernelSchedule>::CollectiveOp; + + using GemmKernel = cutlass::gemm::kernel::GemmUniversal; + + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + using UnderlyingProblemShape = ProblemShape::UnderlyingProblemShape; + using StrideA = typename Gemm::GemmKernel::InternalStrideA; + using StrideB = typename Gemm::GemmKernel::InternalStrideB; + using StrideC = typename Gemm::GemmKernel::InternalStrideC; + using StrideD = typename Gemm::GemmKernel::InternalStrideD; + + int num_experts = (int)expert_offsets.size(0); + Gemm gemm_op; + + typename GemmKernel::MainloopArguments mainloop_args{ + static_cast(a_ptrs.data_ptr()), + static_cast(stride_a.data_ptr()), + static_cast(b_ptrs.data_ptr()), + static_cast(stride_b.data_ptr()), + static_cast(a_scales_ptrs.data_ptr()), + reinterpret_cast(layout_sfa.data_ptr()), + static_cast(b_scales_ptrs.data_ptr()), + reinterpret_cast(layout_sfb.data_ptr())}; + + cutlass::KernelHardwareInfo hw_info; + hw_info.device_id = c10::cuda::current_device(); + hw_info.sm_count = at::cuda::getCurrentDeviceProperties()->multiProcessorCount; + + typename GemmKernel::EpilogueArguments epilogue_args{ + {}, + nullptr, + static_cast(stride_c.data_ptr()), + static_cast(out_ptrs.data_ptr()), + static_cast(stride_c.data_ptr())}; + + UnderlyingProblemShape* problem_sizes_as_shapes = static_cast(problem_sizes.data_ptr()); + typename GemmKernel::Arguments args{ + cutlass::gemm::GemmUniversalMode::kGrouped, + {num_experts, problem_sizes_as_shapes, nullptr}, + mainloop_args, + epilogue_args, + hw_info}; + + at::cuda::CUDAGuard device_guard{(char)a_ptrs.get_device()}; + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(a_ptrs.get_device()); + + auto can_implement_status = gemm_op.can_implement(args); + TORCH_CHECK(can_implement_status == cutlass::Status::kSuccess, "Failed to implement GEMM"); + + auto status = gemm_op.initialize(args, workspace.data_ptr(), stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, "Failed to initialize GEMM"); + + status = gemm_op.run(stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, "Failed to run GEMM"); +} + +template +void launch_sm100_fp8_blockwise_scaled_group_mm( + torch::Tensor& out_ptrs, + const torch::Tensor& a_ptrs, + const torch::Tensor& b_ptrs, + const torch::Tensor& a_scales_ptrs, + const torch::Tensor& b_scales_ptrs, + const torch::Tensor& stride_a, + const torch::Tensor& stride_b, + const torch::Tensor& stride_c, + const torch::Tensor& layout_sfa, + const torch::Tensor& layout_sfb, + const torch::Tensor& problem_sizes, + const torch::Tensor& expert_offsets, + const torch::Tensor& workspace) { + using ProblemShape = cutlass::gemm::GroupProblemShape>; + using ElementA = cutlass::float_e4m3_t; + using ElementB = cutlass::float_e4m3_t; + using ElementC = OutType; + using ElementD = ElementC; + using ElementAccumulator = float; + using LayoutA = cutlass::layout::RowMajor; + using LayoutB = cutlass::layout::ColumnMajor; + using LayoutC = LayoutD; + + static constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + + using ArchTag = cutlass::arch::Sm100; + using OperatorClass = cutlass::arch::OpClassTensorOp; + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + typename ScheduleConfig::MmaTileShape, + typename ScheduleConfig::ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementAccumulator, + void, + LayoutC*, + AlignmentC, + ElementD, + LayoutC*, + AlignmentC, + typename ScheduleConfig::EpilogueSchedule>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementA, + cute::tuple, + AlignmentA, + ElementB, + cute::tuple, + AlignmentB, + ElementAccumulator, + typename ScheduleConfig::MmaTileShape, + typename ScheduleConfig::ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + typename ScheduleConfig::KernelSchedule>::CollectiveOp; + + using GemmKernel = cutlass::gemm::kernel::GemmUniversal; + + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + using UnderlyingProblemShape = ProblemShape::UnderlyingProblemShape; + using StrideA = typename Gemm::GemmKernel::InternalStrideA; + using StrideB = typename Gemm::GemmKernel::InternalStrideB; + using StrideC = typename Gemm::GemmKernel::InternalStrideC; + using StrideD = typename Gemm::GemmKernel::InternalStrideD; + + int num_experts = (int)expert_offsets.size(0); + // Create an instance of the GEMM + Gemm gemm_op; + + typename GemmKernel::MainloopArguments mainloop_args{ + static_cast(a_ptrs.data_ptr()), + static_cast(stride_a.data_ptr()), + static_cast(b_ptrs.data_ptr()), + static_cast(stride_b.data_ptr()), + static_cast(a_scales_ptrs.data_ptr()), + reinterpret_cast(layout_sfa.data_ptr()), + static_cast(b_scales_ptrs.data_ptr()), + reinterpret_cast(layout_sfb.data_ptr())}; + + cutlass::KernelHardwareInfo hw_info; + + hw_info.device_id = 0; + // sm_count is the number of SMs on the current device, since we only support SM100 blackwell, so we set it to 148 + hw_info.sm_count = 148; + typename GemmKernel::EpilogueArguments epilogue_args{ + {}, + nullptr, + static_cast(stride_c.data_ptr()), + static_cast(out_ptrs.data_ptr()), + static_cast(stride_c.data_ptr())}; + + UnderlyingProblemShape* problem_sizes_as_shapes = static_cast(problem_sizes.data_ptr()); + typename GemmKernel::Arguments args{ + cutlass::gemm::GemmUniversalMode::kGrouped, + {num_experts, problem_sizes_as_shapes, nullptr}, + mainloop_args, + epilogue_args, + hw_info}; + + at::cuda::CUDAGuard device_guard{(char)a_ptrs.get_device()}; + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(a_ptrs.get_device()); + + auto can_implement_status = gemm_op.can_implement(args); + TORCH_CHECK(can_implement_status == cutlass::Status::kSuccess, "Failed to implement GEMM"); + + auto status = gemm_op.initialize(args, workspace.data_ptr(), stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, "Failed to initialize GEMM"); + + status = gemm_op.run(stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, "Failed to run GEMM"); +} + +template +void sm100_fp8_blockwise_group_mm_dispatch_shape( + torch::Tensor& output, + torch::Tensor& a_ptrs, + torch::Tensor& b_ptrs, + torch::Tensor& out_ptrs, + torch::Tensor& a_scales_ptrs, + torch::Tensor& b_scales_ptrs, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Tensor& stride_a, + const torch::Tensor& stride_b, + const torch::Tensor& stride_c, + const torch::Tensor& layout_sfa, + const torch::Tensor& layout_sfb, + const torch::Tensor& problem_sizes, + const torch::Tensor& expert_offsets, + const torch::Tensor& workspace) { + // Check the first matrix size to decide on the configuration + // Assuming all matrices in the group have similar size characteristics + // bool use_small_config = a[0].size(0) <= 128; + struct MmaConfig1 { + using ElementA = cutlass::float_e4m3_t; + using MmaTileShape = Shape<_256, _32, _128>; + using ClusterShape = Shape<_2, _1, _1>; // Layout type for SFB matrix operand + using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedBlockwise2SmSm100; + using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecialized2Sm; + using ScaleConfig = + cutlass::detail::Sm100BlockwiseScaleConfig<128, 1, 128, cute::UMMA::Major::K, cute::UMMA::Major::K>; + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); + }; + struct MmaConfig2 { + using ElementA = cutlass::float_e4m3_t; + using MmaTileShape = Shape<_128, _128, _128>; + using ClusterShape = Shape<_1, _1, _1>; // Layout type for SFB matrix operand + using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedBlockwise1SmSm100; + using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecialized1Sm; + using ScaleConfig = + cutlass::detail::Sm100BlockwiseScaleConfig<1, 128, 128, cute::UMMA::Major::K, cute::UMMA::Major::K>; + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); + }; + struct MmaConfig3 { + using ElementA = cutlass::float_e4m3_t; + using MmaTileShape = Shape<_64, _128, _128>; + using ClusterShape = Shape<_1, _1, _1>; // Layout type for SFB matrix operand + using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedBlockwise1SmSm100; + using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecialized1Sm; + using ScaleConfig = + cutlass::detail::Sm100BlockwiseScaleConfig<1, 128, 128, cute::UMMA::Major::K, cute::UMMA::Major::K>; + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); + }; + int num_experts = (int)expert_offsets.size(0); + torch::TensorOptions options_int = torch::TensorOptions().dtype(torch::kInt64).device(a.device()); + torch::Tensor problem_sizes_transpose = torch::empty(num_experts * 3, options_int); + torch::Tensor output_t = output.t(); + torch::Tensor a_t = a.t(); + torch::Tensor b_t = b.transpose(1, 2); + torch::Tensor scales_a_t = scales_a.t(); + torch::Tensor scales_b_t = scales_b.transpose(1, 2); + + if (a.size(0) <= 2048 && a.size(1) >= 2048) { + run_get_group_gemm_starts( + expert_offsets, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + b_t, + a_t, + output_t, + scales_b_t, + scales_a_t, + layout_sfa, + layout_sfb, + problem_sizes, + problem_sizes_transpose, + true); + launch_sm100_fp8_blockwise_scaled_group_mm( + out_ptrs, + a_ptrs, + b_ptrs, + a_scales_ptrs, + b_scales_ptrs, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes_transpose, + expert_offsets, + workspace); + output = output_t.t(); + } else if (a.size(0) > 2048 && a.size(1) >= 2048) { + run_get_group_gemm_starts( + expert_offsets, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a, + b, + output, + scales_a, + scales_b, + layout_sfa, + layout_sfb, + problem_sizes, + problem_sizes_transpose); + launch_sm100_fp8_blockwise_scaled_group_mm( + out_ptrs, + a_ptrs, + b_ptrs, + a_scales_ptrs, + b_scales_ptrs, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets, + workspace); + } else { + run_get_group_gemm_starts( + expert_offsets, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a, + b, + output, + scales_a, + scales_b, + layout_sfa, + layout_sfb, + problem_sizes, + problem_sizes_transpose); + launch_sm100_fp8_blockwise_scaled_group_mm( + out_ptrs, + a_ptrs, + b_ptrs, + a_scales_ptrs, + b_scales_ptrs, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets, + workspace); + } +} + +template +void sm90_fp8_blockwise_group_mm_dispatch_shape( + torch::Tensor& output, + torch::Tensor& a_ptrs, + torch::Tensor& b_ptrs, + torch::Tensor& out_ptrs, + torch::Tensor& a_scales_ptrs, + torch::Tensor& b_scales_ptrs, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Tensor& stride_a, + const torch::Tensor& stride_b, + const torch::Tensor& stride_c, + const torch::Tensor& layout_sfa, + const torch::Tensor& layout_sfb, + const torch::Tensor& problem_sizes, + const torch::Tensor& expert_offsets, + const torch::Tensor& workspace) { + struct MmaConfigSmallM { + // Swap A/B + using ElementA = cutlass::float_e4m3_t; + using MmaTileShape = Shape<_128, _32, _128>; + using ClusterShape = Shape<_2, _1, _1>; + // TODO: Check Pingpong or Cooperative + using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedPingpongFP8Blockwise; + using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecializedPingpong; + using ScaleConfig = + cutlass::detail::Sm90BlockwiseScaleConfig<128, 1, 128, cute::GMMA::Major::K, cute::GMMA::Major::K>; + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); + }; + + struct MmaConfigH20LargeK { + using ElementA = cutlass::float_e4m3_t; + using MmaTileShape = Shape<_64, _128, _128>; + using ClusterShape = Shape<_2, _1, _1>; + using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedPingpongFP8Blockwise; + using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecializedPingpong; + using ScaleConfig = + cutlass::detail::Sm90BlockwiseScaleConfig<1, 128, 128, cute::GMMA::Major::K, cute::GMMA::Major::K>; + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); + }; + + struct MmaConfigHx00AndH20SmallK { + using ElementA = cutlass::float_e4m3_t; + using MmaTileShape = Shape<_128, _128, _128>; + using ClusterShape = Shape<_1, _2, _1>; + using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperativeFP8Blockwise; + using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecializedCooperative; + using ScaleConfig = + cutlass::detail::Sm90BlockwiseScaleConfig<1, 128, 128, cute::GMMA::Major::K, cute::GMMA::Major::K>; + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); + }; + + int num_experts = (int)expert_offsets.size(0); + torch::TensorOptions options_int = torch::TensorOptions().dtype(torch::kInt64).device(a.device()); + torch::Tensor problem_sizes_transpose = torch::empty(num_experts * 3, options_int); + torch::Tensor output_t = output.t(); + torch::Tensor a_t = a.t(); + torch::Tensor b_t = b.transpose(1, 2); + torch::Tensor scales_a_t = scales_a.t(); + torch::Tensor scales_b_t = scales_b.transpose(1, 2); + + const std::string H20_device_type_str("NVIDIA H20"); + bool is_h20_device = std::string(at::cuda::getCurrentDeviceProperties()->name) == H20_device_type_str; + + if (a.size(0) <= 2048) { + run_get_group_gemm_starts( + expert_offsets, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + b_t, + a_t, + output_t, + scales_b_t, + scales_a_t, + layout_sfa, + layout_sfb, + problem_sizes, + problem_sizes_transpose, + true); + launch_sm90_fp8_blockwise_scaled_group_mm( + out_ptrs, + a_ptrs, + b_ptrs, + a_scales_ptrs, + b_scales_ptrs, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes_transpose, + expert_offsets, + workspace); + output = output_t.t(); + } else { + if (is_h20_device && a.size(1) > 128) { + // For H20 with K > 128, use Pingpong Schedule + run_get_group_gemm_starts< + MmaConfigH20LargeK::LayoutSFA, + MmaConfigH20LargeK::LayoutSFB, + MmaConfigH20LargeK::ScaleConfig>( + expert_offsets, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a, + b, + output, + scales_a, + scales_b, + layout_sfa, + layout_sfb, + problem_sizes, + problem_sizes_transpose); + launch_sm90_fp8_blockwise_scaled_group_mm( + out_ptrs, + a_ptrs, + b_ptrs, + a_scales_ptrs, + b_scales_ptrs, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets, + workspace); + } else { + // For H20 with K <= 128, and H100 & H200 & H800, use Cooperative Schedule + run_get_group_gemm_starts< + MmaConfigHx00AndH20SmallK::LayoutSFA, + MmaConfigHx00AndH20SmallK::LayoutSFB, + MmaConfigHx00AndH20SmallK::ScaleConfig>( + expert_offsets, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a, + b, + output, + scales_a, + scales_b, + layout_sfa, + layout_sfb, + problem_sizes, + problem_sizes_transpose); + launch_sm90_fp8_blockwise_scaled_group_mm( + out_ptrs, + a_ptrs, + b_ptrs, + a_scales_ptrs, + b_scales_ptrs, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets, + workspace); + } + } +} + +/** + * @brief Performs blockwise grouped matrix multiplication on FP8 quantized inputs, + * with per-block scaling. + * + * This function dispatches to hardware-specific implementations (e.g., SM100 FP8) + * to compute: + * C_i = scale_a[i] * A_i * scale_b[i] * B_i + * for each expert group `i`, using input `problem_sizes` and `expert_offsets` + * to describe the individual matrix dimensions and their offsets. + * + * Input tensors A and B must be quantized to 8-bit formats and dequantized before multiplication. + * The output tensor is written with bfloat16 or half precision. + * + * @param output Output tensor (must be of type bfloat16 or half). + * @param a Input tensor A (must be kFloat8_e4m3fn). + * @param b Input tensor B (must be kFloat8_e4m3fn). + * @param scales_a Scaling factors for tensor A, float32 per expert group. + * @param scales_b Scaling factors for tensor B, float32 per expert group. + * @param stride_a Stride information for tensor A (int32). + * @param stride_b Stride information for tensor B (int32). + * @param stride_c Stride information for output tensor C (int32). + * @param layout_sfa Layout descriptor for A (int32), e.g., row-major/column-major. + * @param layout_sfb Layout descriptor for B (int32). + * @param problem_sizes 2D int32 tensor of shape (num_experts, 3), specifying (M, N, K) + * for each grouped matrix multiplication problem. + * @param expert_offsets 1D int32 tensor of size (num_experts), used to index into + * the grouped input tensors for dispatch. + * @note Performance Optimization: + * If the batch size (a.size(0)) is smaller than 512, the implementation + * will internally transpose input matrices to align with the optimal memory access + * pattern for better GPU efficiency. This transformation is done within the kernel. + */ +void fp8_blockwise_scaled_grouped_mm( + torch::Tensor& output, + torch::Tensor& a_ptrs, + torch::Tensor& b_ptrs, + torch::Tensor& out_ptrs, + torch::Tensor& a_scales_ptrs, + torch::Tensor& b_scales_ptrs, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Tensor& stride_a, + const torch::Tensor& stride_b, + const torch::Tensor& stride_c, + const torch::Tensor& layout_sfa, + const torch::Tensor& layout_sfb, + const torch::Tensor& problem_sizes, + const torch::Tensor& expert_offsets, + const torch::Tensor& workspace) { + TORCH_CHECK(problem_sizes.dim() == 2, "problem_sizes must be 2D tensor"); + TORCH_CHECK(problem_sizes.size(1) == 3, "problem_sizes must have shape (num_experts, 3)"); + TORCH_CHECK( + problem_sizes.size(0) == expert_offsets.size(0), "Number of experts in problem_sizes must match expert_offsets"); + TORCH_CHECK(problem_sizes.dtype() == torch::kInt32, "problem_sizes must be int32"); + TORCH_CHECK(a.scalar_type() == torch::kFloat8_e4m3fn, "a must be kFloat8_e4m3fn"); + TORCH_CHECK(b.scalar_type() == torch::kFloat8_e4m3fn, "b must be kFloat8_e4m3fn"); + TORCH_CHECK( + output.scalar_type() == torch::kBFloat16 || output.scalar_type() == torch::kHalf, + "output must be bfloat16 or half"); + TORCH_CHECK(scales_a.scalar_type() == torch::kFloat32, "scales_a must be float32"); + TORCH_CHECK(scales_b.scalar_type() == torch::kFloat32, "scales_b must be float32"); + TORCH_CHECK(stride_a.scalar_type() == torch::kInt64, "stride_a must be int64"); + TORCH_CHECK(stride_b.scalar_type() == torch::kInt64, "stride_b must be int64"); + TORCH_CHECK(stride_c.scalar_type() == torch::kInt64, "stride_c must be int64"); + TORCH_CHECK(layout_sfa.scalar_type() == torch::kInt32, "layout_sfa must be int32"); + TORCH_CHECK(layout_sfb.scalar_type() == torch::kInt32, "layout_sfb must be int32"); + TORCH_CHECK(expert_offsets.scalar_type() == torch::kInt32, "expert_offsets must be int32"); + + TORCH_CHECK(output.dim() == 2, "output must be 2D tensor"); + TORCH_CHECK(a.dim() == 2, "a must be 2D tensor"); + TORCH_CHECK(b.dim() == 3, "b must be 3D tensor"); + TORCH_CHECK(scales_a.dim() == 2, "scales_a must be 2D tensor"); + TORCH_CHECK(scales_b.dim() == 3, "scales_b must be 3D tensor"); + TORCH_CHECK(stride_a.dim() == 1, "stride_a must be 1D tensor"); + TORCH_CHECK(stride_b.dim() == 1, "stride_b must be 1D tensor"); + TORCH_CHECK(stride_c.dim() == 1, "stride_c must be 1D tensor"); + TORCH_CHECK(layout_sfa.dim() == 2, "layout_sfa must be 1D tensor"); + TORCH_CHECK(layout_sfb.dim() == 2, "layout_sfb must be 1D tensor"); + TORCH_CHECK(a_ptrs.dim() == 1, "a_ptrs must be 1D tensor"); + TORCH_CHECK(b_ptrs.dim() == 1, "b_ptrs must be 1D tensor"); + TORCH_CHECK(out_ptrs.dim() == 1, "out_ptrs must be 1D tensor"); + TORCH_CHECK(a_scales_ptrs.dim() == 1, "a_scales_ptrs must be 1D tensor"); + TORCH_CHECK(b_scales_ptrs.dim() == 1, "b_scales_ptrs must be 1D tensor"); + TORCH_CHECK(expert_offsets.dim() == 1, "expert_offsets must be 1D tensor"); + TORCH_CHECK(workspace.dim() == 1, "workspace must be 1D tensor"); + + bool can_implement = false; + auto sm_version = getSMVersion(); + +#if defined(CUTLASS_ARCH_MMA_SM100A_SUPPORTED) || defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) +#if defined CUDA_VERSION && CUDA_VERSION >= 12080 + if (sm_version == 100 +#if CUDA_VERSION >= 12090 + || sm_version == 103 +#endif + ) { + if (output.scalar_type() == torch::kBFloat16) { + sm100_fp8_blockwise_group_mm_dispatch_shape( + output, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a, + b, + scales_a, + scales_b, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets, + workspace); + } else { + sm100_fp8_blockwise_group_mm_dispatch_shape( + output, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a, + b, + scales_a, + scales_b, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets, + workspace); + } + can_implement = true; + } +#endif +#endif + +#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED) && defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_SUPPORTED) + if (sm_version == 90) { + if (output.scalar_type() == torch::kBFloat16) { + sm90_fp8_blockwise_group_mm_dispatch_shape( + output, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a, + b, + scales_a, + scales_b, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets, + workspace); + } else { + sm90_fp8_blockwise_group_mm_dispatch_shape( + output, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a, + b, + scales_a, + scales_b, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets, + workspace); + } + can_implement = true; + } +#endif + TORCH_CHECK_NOT_IMPLEMENTED( + can_implement, "No implemented fp8_blockwise_scaled_grouped_mm for current compute capability: ", sm_version); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/fused_qknorm_rope_kernel.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/fused_qknorm_rope_kernel.cu new file mode 100644 index 00000000..0b6e52fc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/fused_qknorm_rope_kernel.cu @@ -0,0 +1,408 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// Adapted from +// https://github.com/NVIDIA/TensorRT-LLM/blob/main/cpp/tensorrt_llm/kernels/fusedQKNormRopeKernel.cu + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#define CHECK_TYPE(x, st) \ + TORCH_CHECK(x.scalar_type() == st, #x " dtype is ", x.scalar_type(), ", while ", st, " is expected") +#define CHECK_TH_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor") +#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_INPUT(x, st) \ + CHECK_TH_CUDA(x); \ + CHECK_CONTIGUOUS(x); \ + CHECK_TYPE(x, st) + +#define FINAL_MASK 0xffffffff + +namespace tensorrt_llm::common { +template +struct packed_as; + +// Specialization for packed_as used in this kernel. +template <> +struct packed_as { + using type = uint; +}; + +template <> +struct packed_as { + using type = uint2; +}; + +template <> +struct packed_as { + using type = uint4; +}; + +template +__inline__ __device__ T warpReduceSum(T val) { +#pragma unroll + for (int mask = 16; mask > 0; mask >>= 1) + val += __shfl_xor_sync(FINAL_MASK, val, mask, + 32); //__shfl_sync bf16 return float when sm < 80 + return val; +} + +template +inline __device__ __host__ T divUp(T m, T n) { + return (m + n - 1) / n; +} + +} // namespace tensorrt_llm::common +namespace tensorrt_llm::kernels { + +__device__ inline float compute_freq_yarn(float base, int head_dim, int half_dim, float factor, float low, float high) { + float freq = powf(base, -2.0f * half_dim / static_cast(head_dim)); + + if (factor != 1.0f) { + float inv_freq_extrapolation = freq; + float inv_freq_interpolation = freq / factor; + + float high_adj = high; + if (fabsf(low - high_adj) <= 1e-6f) { + high_adj += 0.001f; + } + + float linear_func = (static_cast(half_dim) - low) / (high_adj - low); + float ramp_func = fminf(fmaxf(linear_func, 0.0f), 1.0f); + float inv_freq_extrapolation_factor = 1.0f - ramp_func; + + freq = inv_freq_interpolation * (1.0f - inv_freq_extrapolation_factor) + + inv_freq_extrapolation * inv_freq_extrapolation_factor; + } + + return freq; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +// Perform per-head QK Norm and RoPE in a single kernel. +// head_dim: the dimension of each head +// interleave: interleave=!is_neox. +template +__global__ void fusedQKNormRopeKernel( + __nv_bfloat16* qkv, // Combined QKV tensor [num_tokens, (num_heads_q+num_heads_k+num_heads_v)*head_dim] + int const num_heads_q, // Number of query heads + int const num_heads_k, // Number of key heads + int const num_heads_v, // Number of value heads + float const eps, // Epsilon for RMS normalization + __nv_bfloat16 const* q_weight, // RMSNorm weights for query + __nv_bfloat16 const* k_weight, // RMSNorm weights for key + float const base, // Base for RoPE computation + int const* position_ids, // Position IDs for RoPE + int const num_tokens, // Number of tokens + // parameters for yarn + float factor, // factor in rope_scaling in config.json. When it is not 1.0, it means the model is using yarn. + float low, // threshold for high frequency + float high, // threshold for low frequency + float attention_factor // attention_factor applied on cos and sin +) { + int const warpsPerBlock = blockDim.x / 32; + int const warpId = threadIdx.x / 32; + int const laneId = threadIdx.x % 32; + + // Calculate global warp index to determine which head/token this warp processes + int const globalWarpIdx = blockIdx.x * warpsPerBlock + warpId; + + // Total number of attention heads (Q and K) + int const total_qk_heads = num_heads_q + num_heads_k; + + // Determine which token and head type (Q or K) this warp processes + int const tokenIdx = globalWarpIdx / total_qk_heads; + int const localHeadIdx = globalWarpIdx % total_qk_heads; + + // Skip if this warp is assigned beyond the number of tokens + if (tokenIdx >= num_tokens) return; + + bool const isQ = localHeadIdx < num_heads_q; + int const headIdx = isQ ? localHeadIdx : localHeadIdx - num_heads_q; + int const num_heads = num_heads_q + num_heads_k + num_heads_v; + static_assert( + head_dim % (32 * 2) == 0, + "head_dim must be divisible by 64 (each warp processes one head, and each thread gets even number of " + "elements)"); + constexpr int numElemsPerThread = head_dim / 32; + float elements[numElemsPerThread]; + constexpr int elemSizeBytes = numElemsPerThread * sizeof(__nv_bfloat16); + static_assert(elemSizeBytes % 4 == 0, "numSizeBytes must be a multiple of 4"); + constexpr int vecSize = elemSizeBytes / 4; // Use packed_as to perform loading/saving. + using vec_T = typename tensorrt_llm::common::packed_as::type; + + int offsetWarp; // Offset for the warp + if (isQ) { + // Q segment: token offset + head offset within Q segment + offsetWarp = tokenIdx * num_heads * head_dim + headIdx * head_dim; + } else { + // K segment: token offset + entire Q segment + head offset within K segment + offsetWarp = tokenIdx * num_heads * head_dim + num_heads_q * head_dim + headIdx * head_dim; + } + int offsetThread = offsetWarp + laneId * numElemsPerThread; + + // Sum of squares for RMSNorm + float sumOfSquares = 0.0f; + + // Load. + { + vec_T vec = *reinterpret_cast(&qkv[offsetThread]); + for (int i = 0; i < vecSize; i++) { + float2 vals = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(reinterpret_cast(&vec) + i)); + sumOfSquares += vals.x * vals.x; + sumOfSquares += vals.y * vals.y; + elements[2 * i] = vals.x; + elements[2 * i + 1] = vals.y; + } + } + + // Reduce sum across warp using the utility function + sumOfSquares = tensorrt_llm::common::warpReduceSum(sumOfSquares); + + // Compute RMS normalization factor + float rms_rcp = rsqrtf(sumOfSquares / static_cast(head_dim) + eps); + + // Normalize elements + for (int i = 0; i < numElemsPerThread; i++) { + int dim = laneId * numElemsPerThread + i; + float weight = isQ ? __bfloat162float(q_weight[dim]) : __bfloat162float(k_weight[dim]); + elements[i] *= rms_rcp * weight; + } + // Apply RoPE to normalized elements + float elements2[numElemsPerThread]; // Additional buffer required for RoPE. + float cos_vals[numElemsPerThread]; + float sin_vals[numElemsPerThread]; + float pos_id = static_cast(position_ids[tokenIdx]); + + if constexpr (interleave) { + // Perform interleaving. Fill cos_vals and sin_vals. + for (int i = 0; i < numElemsPerThread; i++) { + if (i % 2 == 0) { + elements2[i] = -elements[i + 1]; + } else { + elements2[i] = elements[i - 1]; + } + int dim_idx = laneId * numElemsPerThread + i; + int half_dim = dim_idx / 2; + float freq = compute_freq_yarn(base, head_dim, half_dim, factor, low, high); + float theta = pos_id * freq; + __sincosf(theta, &sin_vals[i], &cos_vals[i]); + } + } else { + // Before data exchange with in warp, we need to sync. + __syncwarp(); + // Get the data from the other half of the warp. Fill cos_vals and sin_vals. + for (int i = 0; i < numElemsPerThread; i++) { + elements2[i] = __shfl_xor_sync(0xffffffff, elements[i], 16); + if (laneId < 16) { + elements2[i] = -elements2[i]; + } + int dim_idx = laneId * numElemsPerThread + i; + dim_idx = (dim_idx * 2) % head_dim; + int half_dim = dim_idx / 2; + float freq = compute_freq_yarn(base, head_dim, half_dim, factor, low, high); + float theta = pos_id * freq; + __sincosf(theta, &sin_vals[i], &cos_vals[i]); + } + // __shfl_xor_sync does not provide memfence. Need to sync again. + __syncwarp(); + } + for (int i = 0; i < numElemsPerThread; i++) { + elements[i] = (elements[i] * cos_vals[i] + elements2[i] * sin_vals[i]) * attention_factor; + } + // Store. + { + vec_T vec; + for (int i = 0; i < vecSize; i++) { + __nv_bfloat162 vals = __float22bfloat162_rn(make_float2(elements[2 * i], elements[2 * i + 1])); + reinterpret_cast<__nv_bfloat162&>(*(reinterpret_cast(&vec) + i)) = vals; + } + vec_T* outputPtr = reinterpret_cast(&qkv[offsetThread]); + *outputPtr = vec; + } +} +// Borrowed from +// https://github.com/flashinfer-ai/flashinfer/blob/8125d079a43e9a0ba463a4ed1b639cefd084cec9/include/flashinfer/pos_enc.cuh#L568 +#define DISPATCH_INTERLEAVE(interleave, INTERLEAVE, ...) \ + if (interleave) { \ + const bool INTERLEAVE = true; \ + __VA_ARGS__ \ + } else { \ + const bool INTERLEAVE = false; \ + __VA_ARGS__ \ + } +void launchFusedQKNormRope( + void* qkv, + int const num_tokens, + int const num_heads_q, + int const num_heads_k, + int const num_heads_v, + int const head_dim, + float const eps, + void const* q_weight, + void const* k_weight, + float const base, + bool const interleave, + int const* position_ids, + float factor, + float low, + float high, + float attention_factor, + cudaStream_t stream) { + constexpr int blockSize = 256; + int const warpsPerBlock = blockSize / 32; + int const totalQKHeads = num_heads_q + num_heads_k; + int const totalWarps = num_tokens * totalQKHeads; + int const gridSize = common::divUp(totalWarps, warpsPerBlock); + dim3 gridDim(gridSize); + dim3 blockDim(blockSize); + // Head dimensions should be a multiple of 64 + // Add more cases as needed + switch (head_dim) { + case 64: + DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { + fusedQKNormRopeKernel<64, INTERLEAVE><<>>( + reinterpret_cast<__nv_bfloat16*>(qkv), + num_heads_q, + num_heads_k, + num_heads_v, + eps, + reinterpret_cast<__nv_bfloat16 const*>(q_weight), + reinterpret_cast<__nv_bfloat16 const*>(k_weight), + base, + position_ids, + num_tokens, + factor, + low, + high, + attention_factor); + }); + break; + case 128: + DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { + fusedQKNormRopeKernel<128, INTERLEAVE><<>>( + reinterpret_cast<__nv_bfloat16*>(qkv), + num_heads_q, + num_heads_k, + num_heads_v, + eps, + reinterpret_cast<__nv_bfloat16 const*>(q_weight), + reinterpret_cast<__nv_bfloat16 const*>(k_weight), + base, + position_ids, + num_tokens, + factor, + low, + high, + attention_factor); + }); + break; + case 256: + DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { + fusedQKNormRopeKernel<256, INTERLEAVE><<>>( + reinterpret_cast<__nv_bfloat16*>(qkv), + num_heads_q, + num_heads_k, + num_heads_v, + eps, + reinterpret_cast<__nv_bfloat16 const*>(q_weight), + reinterpret_cast<__nv_bfloat16 const*>(k_weight), + base, + position_ids, + num_tokens, + factor, + low, + high, + attention_factor); + }); + break; + default: + TORCH_CHECK(false, "Unsupported head dimension for fusedQKNormRope: ", head_dim); + } +} +} // namespace tensorrt_llm::kernels + +// Function for fused QK Norm and RoPE +// This operator applies RMS normalization and RoPE to Q and K tensors in a single CUDA kernel. +// The OP performs operations in-place on the input qkv tensor. +void fused_qk_norm_rope( + torch::Tensor& qkv, // Combined QKV tensor [num_tokens, (num_heads_q+num_heads_k+num_heads_v)*head_dim] + int64_t num_heads_q, // Number of query heads + int64_t num_heads_k, // Number of key heads + int64_t num_heads_v, // Number of value heads + int64_t head_dim, // Dimension per head + double eps, // Epsilon for RMS normalization + torch::Tensor& q_weight, // RMSNorm weights for query [head_dim] + torch::Tensor& k_weight, // RMSNorm weights for key [head_dim] + double base, // Base for RoPE computation + bool is_neox, // Whether RoPE is applied in Neox style + torch::Tensor& position_ids, // Position IDs for RoPE [num_tokens] + // parameters for yarn + double factor, // factor in rope_scaling in config.json. When it is not 1.0, it means the model is using yarn. + double low, // threshold for high frequency + double high, // threshold for low frequency + double attention_factor // attention_factor applied on cos and sin +) { + // Input validation + TORCH_CHECK(qkv.dim() == 2, "QKV tensor must be 2D: [num_tokens, (num_heads_q+num_heads_k+num_heads_v)*head_dim]"); + TORCH_CHECK(position_ids.dim() == 1, "Position IDs must be 1D: [num_tokens]"); + TORCH_CHECK(q_weight.dim() == 1, "Query weights must be 1D: [head_dim]"); + TORCH_CHECK(k_weight.dim() == 1, "Key weights must be 1D: [head_dim]"); + TORCH_CHECK(q_weight.size(0) == head_dim, "Query weights size must match head dimension"); + TORCH_CHECK(k_weight.size(0) == head_dim, "Key weights size must match head dimension"); + + CHECK_INPUT(qkv, torch::kBFloat16); + CHECK_INPUT(position_ids, torch::kInt32); + CHECK_INPUT(q_weight, torch::kBFloat16); + CHECK_INPUT(k_weight, torch::kBFloat16); + + int64_t num_tokens = qkv.size(0); + TORCH_CHECK(position_ids.size(0) == num_tokens, "Number of tokens in position_ids must match QKV"); + + int64_t total_heads = num_heads_q + num_heads_k + num_heads_v; + TORCH_CHECK( + qkv.size(1) == total_heads * head_dim, "QKV tensor size must match total number of heads and head dimension"); + + auto stream = at::cuda::getCurrentCUDAStream(qkv.get_device()); + + tensorrt_llm::kernels::launchFusedQKNormRope( + reinterpret_cast<__nv_bfloat16*>(qkv.data_ptr()), + static_cast(num_tokens), + static_cast(num_heads_q), + static_cast(num_heads_k), + static_cast(num_heads_v), + static_cast(head_dim), + static_cast(eps), + reinterpret_cast<__nv_bfloat16*>(q_weight.data_ptr()), + reinterpret_cast<__nv_bfloat16*>(k_weight.data_ptr()), + static_cast(base), + !is_neox, // interleave + reinterpret_cast(position_ids.data_ptr()), + static_cast(factor), + static_cast(low), + static_cast(high), + static_cast(attention_factor), + stream); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/kimi_k2_moe_fused_gate.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/kimi_k2_moe_fused_gate.cu new file mode 100644 index 00000000..f674e010 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/kimi_k2_moe_fused_gate.cu @@ -0,0 +1,311 @@ +#include +#include +#include + +#include + +// Kimi K2 specific constants +static constexpr int WARP_SIZE = 32; +static constexpr int WARPS_PER_CTA = 6; +static constexpr int NUM_EXPERTS = 384; +static constexpr int VPT = 12; // 384 / 32 = 12 + +// Small token optimization constants +static constexpr int SMALL_TOKEN_THRESHOLD = 512; +static constexpr int WARPS_PER_TOKEN_SMALL = 12; // Use 12 warps per token for small batches +static constexpr int THREADS_PER_BLOCK_SMALL = WARPS_PER_TOKEN_SMALL * WARP_SIZE; // 384 threads + +// Vectorization constants (used by large token kernel) +static constexpr int VEC_SIZE = 4; // Use float4 for vectorized loads + +// Small token optimized kernel: Each warp independently finds top-k, then merge, using warp-level topk +__global__ void kimi_k2_moe_fused_gate_kernel_small_token( + float* input, + float* bias, + float* output_ptr, + int32_t* indices_ptr, + int64_t num_rows, + int64_t topk, + bool renormalize, + double routed_scaling_factor, + bool apply_routed_scaling_factor_on_output) { + int64_t row_idx = blockIdx.x; + if (row_idx >= num_rows) return; + + int tid = threadIdx.x; + int warp_id = tid / WARP_SIZE; + int lane_id = tid % WARP_SIZE; + + // Shared memory: biased scores and original scores + __shared__ float shared_scores[NUM_EXPERTS]; + __shared__ float shared_original_scores[NUM_EXPERTS]; + // For storing selected top-k indices and values + __shared__ int selected_experts[8]; // Up to topk=6, I use 8 for alignment + __shared__ float selected_vals[8]; + // For warp-level reduction + __shared__ float warp_maxs[WARPS_PER_TOKEN_SMALL]; + __shared__ int warp_experts[WARPS_PER_TOKEN_SMALL]; + + // Load data: all 384 threads load one expert each + if (tid < NUM_EXPERTS) { + float input_val = input[row_idx * NUM_EXPERTS + tid]; + float bias_val = bias[tid]; + float sigmoid_val = 1.0f / (1.0f + expf(-input_val)); + float biased_val = sigmoid_val + bias_val; + shared_scores[tid] = biased_val; + shared_original_scores[tid] = sigmoid_val; + } + + __syncthreads(); + + // Find top-k using iterative selection, each iteration finds the next maximum + for (int k = 0; k < topk; k++) { + // Each thread holds one expert's value + float my_val = (tid < NUM_EXPERTS) ? shared_scores[tid] : -FLT_MAX; + int my_expert = tid; + + // Use warp-level reduction first + float warp_max_val = my_val; + int warp_max_expert = my_expert; + +#pragma unroll + for (int offset = 16; offset > 0; offset /= 2) { + float other_val = __shfl_down_sync(0xFFFFFFFF, warp_max_val, offset); + int other_expert = __shfl_down_sync(0xFFFFFFFF, warp_max_expert, offset); + if (other_val > warp_max_val) { + warp_max_val = other_val; + warp_max_expert = other_expert; + } + } + + // Warp leaders write to shared memory + if (lane_id == 0) { + warp_maxs[warp_id] = warp_max_val; + warp_experts[warp_id] = warp_max_expert; + } + + __syncthreads(); + + // Final reduction among warps (done by first warp) + if (warp_id == 0) { + float final_max = (lane_id < WARPS_PER_TOKEN_SMALL) ? warp_maxs[lane_id] : -FLT_MAX; + int final_expert = (lane_id < WARPS_PER_TOKEN_SMALL) ? warp_experts[lane_id] : -1; + +#pragma unroll + for (int offset = 16; offset > 0; offset /= 2) { + float other_val = __shfl_down_sync(0xFFFFFFFF, final_max, offset); + int other_expert = __shfl_down_sync(0xFFFFFFFF, final_expert, offset); + if (other_val > final_max) { + final_max = other_val; + final_expert = other_expert; + } + } + + if (lane_id == 0) { + selected_experts[k] = final_expert; + selected_vals[k] = final_max; + } + } + + __syncthreads(); + + // Mark the selected expert as used for next iteration + // All threads can read from selected_experts[k] + int selected = selected_experts[k]; + if (tid == selected) { + shared_scores[tid] = -FLT_MAX; + } + + __syncthreads(); + } + + // Write output (done by thread 0) + if (tid == 0) { + for (int k = 0; k < topk; k++) { + int expert_id = selected_experts[k]; + if (expert_id >= 0 && expert_id < NUM_EXPERTS) { + output_ptr[row_idx * topk + k] = shared_original_scores[expert_id]; + indices_ptr[row_idx * topk + k] = expert_id; + } + } + + // Renormalization + if (renormalize) { + float sum = 0.0f; + for (int k = 0; k < topk; k++) { + sum += output_ptr[row_idx * topk + k]; + } + + if (sum > 0.0f) { + for (int k = 0; k < topk; k++) { + int64_t idx = row_idx * topk + k; + output_ptr[idx] /= sum; + if (apply_routed_scaling_factor_on_output) { + output_ptr[idx] *= static_cast(routed_scaling_factor); + } + } + } + } + } +} + +// Large token kernel: Original implementation with vectorized loads +__global__ void kimi_k2_moe_fused_gate_kernel( + float* input, + float* bias, + float* output_ptr, + int32_t* indices_ptr, + int64_t num_rows, + int64_t topk, + bool renormalize, + double routed_scaling_factor, + bool apply_routed_scaling_factor_on_output) { + int64_t row_idx = blockIdx.x * WARPS_PER_CTA + threadIdx.y; + if (row_idx >= num_rows) return; + + int lane_id = threadIdx.x; + int warp_id = threadIdx.y; + + __shared__ float shared_scores[NUM_EXPERTS * WARPS_PER_CTA]; + __shared__ float shared_original_scores[NUM_EXPERTS * WARPS_PER_CTA]; + + float* warp_scores = shared_scores + warp_id * NUM_EXPERTS; + float* warp_original_scores = shared_original_scores + warp_id * NUM_EXPERTS; + + // Vectorized loading: each lane loads multiple float4 chunks + // VPT = 12, so we load 12/4 = 3 float4 per lane + const int VEC_PER_LANE = VPT / VEC_SIZE; // 3 + float4* input_vec = reinterpret_cast(input + row_idx * NUM_EXPERTS); + float4* bias_vec = reinterpret_cast(bias); + +#pragma unroll + for (int i = 0; i < VEC_PER_LANE; i++) { + int vec_idx = lane_id * VEC_PER_LANE + i; + float4 input_val = input_vec[vec_idx]; + float4 bias_val = bias_vec[vec_idx]; + +#pragma unroll + for (int j = 0; j < VEC_SIZE; j++) { + int expert = vec_idx * VEC_SIZE + j; + float inp = ((float*)&input_val)[j]; + float b = ((float*)&bias_val)[j]; + float sigmoid_val = 1.0f / (1.0f + expf(-inp)); + float biased_val = sigmoid_val + b; + warp_scores[expert] = biased_val; + warp_original_scores[expert] = sigmoid_val; + } + } + + __syncthreads(); + + for (int k = 0; k < topk; k++) { + float max_val = -FLT_MAX; + int max_expert = -1; + + for (int expert = lane_id; expert < NUM_EXPERTS; expert += WARP_SIZE) { + if (warp_scores[expert] > max_val) { + max_val = warp_scores[expert]; + max_expert = expert; + } + } + + for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) { + float other_val = __shfl_down_sync(0xFFFFFFFF, max_val, offset); + int other_expert = __shfl_down_sync(0xFFFFFFFF, max_expert, offset); + + if (other_val > max_val || (other_val == max_val && other_expert < max_expert)) { + max_val = other_val; + max_expert = other_expert; + } + } + + if (lane_id == 0 && max_expert != -1) { + int64_t output_idx = row_idx * topk + k; + output_ptr[output_idx] = warp_original_scores[max_expert]; + indices_ptr[output_idx] = max_expert; + warp_scores[max_expert] = -FLT_MAX; + } + + __syncwarp(); + } + + __syncthreads(); + + if (renormalize && lane_id == 0) { + float sum = 0.0f; + for (int k = 0; k < topk; k++) { + sum += output_ptr[row_idx * topk + k]; + } + + if (sum > 0.0f) { + for (int k = 0; k < topk; k++) { + int64_t idx = row_idx * topk + k; + output_ptr[idx] /= sum; + if (apply_routed_scaling_factor_on_output) { + output_ptr[idx] *= static_cast(routed_scaling_factor); + } + } + } + } +} + +std::vector kimi_k2_moe_fused_gate( + at::Tensor& input, + at::Tensor& bias, + int64_t topk, + bool renormalize, + double routed_scaling_factor, + bool apply_routed_scaling_factor_on_output) { + int64_t num_rows = input.size(0); + int32_t num_experts = input.size(1); + + // Assert: Only support 384 experts + TORCH_CHECK(num_experts == 384, "kimi_k2_moe_fused_gate only supports 384 experts, but got ", num_experts); + TORCH_CHECK(input.dtype() == bias.dtype(), "input and bias should have the same dtype"); + + auto options = torch::TensorOptions().dtype(torch::kFloat32).device(torch::kCUDA); + auto output = torch::empty({num_rows, topk}, options); + auto indices = torch::empty({num_rows, topk}, options.dtype(torch::kInt32)); + + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + // Only support float32 + TORCH_CHECK(input.scalar_type() == at::kFloat, "kimi_k2_moe_fused_gate only supports float32 input"); + TORCH_CHECK(bias.scalar_type() == at::kFloat, "kimi_k2_moe_fused_gate only supports float32 bias"); + + bool use_small_token_kernel = num_rows <= SMALL_TOKEN_THRESHOLD; + + if (use_small_token_kernel) { + // Small token kernel: Each block handles 1 token with multiple warps collaborating + int64_t num_blocks = num_rows; + dim3 block_dim(THREADS_PER_BLOCK_SMALL); + + kimi_k2_moe_fused_gate_kernel_small_token<<>>( + input.data_ptr(), + bias.data_ptr(), + output.data_ptr(), + indices.data_ptr(), + num_rows, + topk, + renormalize, + routed_scaling_factor, + apply_routed_scaling_factor_on_output); + } else { + // Large token kernel: Original implementation + int64_t num_blocks = (num_rows + WARPS_PER_CTA - 1) / WARPS_PER_CTA; + dim3 block_dim(WARP_SIZE, WARPS_PER_CTA); + + kimi_k2_moe_fused_gate_kernel<<>>( + input.data_ptr(), + bias.data_ptr(), + output.data_ptr(), + indices.data_ptr(), + num_rows, + topk, + renormalize, + routed_scaling_factor, + apply_routed_scaling_factor_on_output); + } + + return {output, indices}; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/generate_kernels.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/generate_kernels.py new file mode 100644 index 00000000..dea951f7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/generate_kernels.py @@ -0,0 +1,158 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import glob +import itertools +import os +import subprocess + +import jinja2 + +FILE_HEAD = """ +// auto generated by generate.py +// clang-format off +#pragma once + +namespace MARLIN_NAMESPACE_NAME { +""".strip() + +TEMPLATE = ( + "template __global__ void Marlin<" + "{{scalar_t}}, " + "{{w_type_id}}, " + "{{s_type_id}}, " + "{{threads}}, " + "{{thread_m_blocks}}, " + "{{thread_n_blocks}}, " + "{{thread_k_blocks}}, " + "{{'true' if m_block_size_8 else 'false'}}, " + "{{stages}}, " + "{{group_blocks}}, " + "{{'true' if is_zp_float else 'false'}}>" + "( MARLIN_KERNEL_PARAMS );" +) + +KERNEL_FILE_TEMPLATE = ( + "// auto generated by generate.py\n" + "// clang-format off\n" + "#pragma once\n\n" + "{% for kernel_file in kernel_files %}" + '#include "{{ kernel_file }}"\n' + "{% endfor %}" +) + +KERNEL_FILE_NAME = "kernel_marlin.cuh" + +# int8 with zero point case (sglang::kU8) is also supported, +# we don't add it to reduce wheel size. +# Only keep the most commonly used types to reduce compilation time +SCALAR_TYPES = [ + "sglang::kU4", + "sglang::kU4B8", + "sglang::kU8B128", +] +THREAD_CONFIGS = [(128, 128, 256), (64, 256, 256)] + +THREAD_M_BLOCKS = [0.5, 1, 2, 4] +# group_blocks: +# = 0 : act order case +# = -1 : channelwise quantization +# > 0 : group_size=16*group_blocks +GROUP_BLOCKS = [0, -1, 2, 4, 8] +DTYPES = ["fp16", "bf16"] + + +def remove_old_kernels(): + for filename in glob.glob(os.path.dirname(__file__) + "/kernel_*.cuh"): + subprocess.call(["rm", "-f", filename]) + + +def generate_new_kernels(): + kernel_files = set() + for scalar_type, dtype in itertools.product(SCALAR_TYPES, DTYPES): + all_template_str_list = [] + + for group_blocks, m_blocks, thread_configs in itertools.product( + GROUP_BLOCKS, THREAD_M_BLOCKS, THREAD_CONFIGS + ): + # act order case only support gptq-int4 and gptq-int8 + if group_blocks == 0 and scalar_type not in [ + "sglang::kU4B8", + "sglang::kU8B128", + ]: + continue + if thread_configs[2] == 256: + # for small batch (m_blocks == 1), we only need (128, 128, 256) + # for large batch (m_blocks > 1), we only need (64, 256, 256) + if m_blocks <= 1 and thread_configs[0] != 128: + continue + if m_blocks > 1 and thread_configs[0] != 64: + continue + + # we only support channelwise quantization and group_size == 128 + # for fp8 + if scalar_type == "sglang::kFE4M3fn" and group_blocks not in [-1, 8]: + continue + # nvfp4 only supports group_size == 16 + # mxfp4 only supports group_size == 32 + if scalar_type == "sglang::kFE2M1f" and group_blocks not in [1, 2]: + continue + # other quantization methods don't support group_size = 16 + if scalar_type != "sglang::kFE2M1f" and group_blocks == 1: + continue + + k_blocks = thread_configs[0] // 16 + n_blocks = thread_configs[1] // 16 + threads = thread_configs[2] + + c_dtype = "half" if dtype == "fp16" else "nv_bfloat16" + + if scalar_type == "sglang::kFE2M1f" and group_blocks == 1: + s_type = "sglang::kFE4M3fn" + elif scalar_type == "sglang::kFE2M1f" and group_blocks == 2: + s_type = "sglang::kFE8M0fnu" + if dtype == "fp16": + # we cannot safely dequantize e8m0 to fp16, so skip this + continue + elif dtype == "fp16": + s_type = "sglang::kFloat16" + elif dtype == "bf16": + s_type = "sglang::kBFloat16" + + template_str = jinja2.Template(TEMPLATE).render( + scalar_t=c_dtype, + w_type_id=scalar_type + ".id()", + s_type_id=s_type + ".id()", + threads=threads, + thread_m_blocks=max(m_blocks, 1), + thread_n_blocks=n_blocks, + thread_k_blocks=k_blocks, + m_block_size_8=m_blocks == 0.5, + stages="pipe_stages", + group_blocks=group_blocks, + is_zp_float=False, + ) + + all_template_str_list.append(template_str) + + file_content = FILE_HEAD + "\n\n" + file_content += "\n\n".join(all_template_str_list) + "\n\n}\n" + # Remove "sglang::" prefix (8 chars) from scalar_type for filename + filename = f"kernel_{dtype}_{scalar_type[8:].lower()}.cuh" + + with open(os.path.join(os.path.dirname(__file__), filename), "w") as f: + f.write(file_content) + kernel_files.add(filename) + + kernel_files = list(kernel_files) + kernel_files.sort() + + file_content = jinja2.Template(KERNEL_FILE_TEMPLATE).render( + kernel_files=kernel_files + ) + with open(os.path.join(os.path.dirname(__file__), KERNEL_FILE_NAME), "w") as f: + f.write(file_content) + + +if __name__ == "__main__": + remove_old_kernels() + generate_new_kernels() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel.h new file mode 100644 index 00000000..f3f0bf03 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel.h @@ -0,0 +1,40 @@ + +#ifndef MARLIN_NAMESPACE_NAME +#define MARLIN_NAMESPACE_NAME marlin_moe_wna16_v2 +#endif + +#include "gemm/marlin/marlin.cuh" +#include "gemm/marlin/marlin_dtypes.cuh" +#include "scalar_type.hpp" + +#define MARLIN_KERNEL_PARAMS \ + const int4 *__restrict__ A, const int4 *__restrict__ B, int4 *__restrict__ C, int4 *__restrict__ C_tmp, \ + const int4 *__restrict__ b_bias_ptr, const int4 *__restrict__ scales_ptr, \ + const uint16_t *__restrict__ scale2_ptr, const int4 *__restrict__ zp_ptr, const int *__restrict__ g_idx, \ + const int32_t *__restrict__ sorted_token_ids_ptr, const int32_t *__restrict__ expert_ids_ptr, \ + const int32_t *__restrict__ num_tokens_past_padded_ptr, const float *__restrict__ topk_weights_ptr, int top_k, \ + bool mul_topk_weights, bool is_ep, int num_groups, int prob_m, int prob_n, int prob_k, int *locks, \ + bool has_bias, bool use_atomic_add, bool use_fp32_reduce, int max_shared_mem + +namespace MARLIN_NAMESPACE_NAME { +template < + typename scalar_t, // compute dtype, half or nv_float16 + const sglang::ScalarTypeId w_type_id, // weight ScalarType id + const sglang::ScalarTypeId s_type_id, // weight scale ScalarType id + const int threads, // number of threads in a threadblock + const int thread_m_blocks, // number of 16x16 blocks in the m + // dimension (batchsize) of the + // threadblock + const int thread_n_blocks, // same for n dimension (output) + const int thread_k_blocks, // same for k dimension (reduction) + const bool m_block_size_8, // whether m_block_size == 8 + // only works when thread_m_blocks == 1 + const int stages, // number of stages for the async global->shared + // fetch pipeline + const int group_blocks, // number of consecutive 16x16 blocks + // with a separate quantization scale + const bool is_zp_float // is zero point of float16 type? + > +__global__ void Marlin(MARLIN_KERNEL_PARAMS); + +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku4.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku4.cuh new file mode 100644 index 00000000..51619bb5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku4.cuh @@ -0,0 +1,39 @@ +// auto generated by generate.py +// clang-format off +#pragma once + +namespace MARLIN_NAMESPACE_NAME { + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku4b8.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku4b8.cuh new file mode 100644 index 00000000..e192eb56 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku4b8.cuh @@ -0,0 +1,47 @@ +// auto generated by generate.py +// clang-format off +#pragma once + +namespace MARLIN_NAMESPACE_NAME { + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku8b128.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku8b128.cuh new file mode 100644 index 00000000..789d6c5f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku8b128.cuh @@ -0,0 +1,47 @@ +// auto generated by generate.py +// clang-format off +#pragma once + +namespace MARLIN_NAMESPACE_NAME { + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku4.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku4.cuh new file mode 100644 index 00000000..f6913103 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku4.cuh @@ -0,0 +1,39 @@ +// auto generated by generate.py +// clang-format off +#pragma once + +namespace MARLIN_NAMESPACE_NAME { + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku4b8.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku4b8.cuh new file mode 100644 index 00000000..b9611adc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku4b8.cuh @@ -0,0 +1,47 @@ +// auto generated by generate.py +// clang-format off +#pragma once + +namespace MARLIN_NAMESPACE_NAME { + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku8b128.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku8b128.cuh new file mode 100644 index 00000000..1fbe1cf4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku8b128.cuh @@ -0,0 +1,47 @@ +// auto generated by generate.py +// clang-format off +#pragma once + +namespace MARLIN_NAMESPACE_NAME { + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_marlin.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_marlin.cuh new file mode 100644 index 00000000..bb828dc5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_marlin.cuh @@ -0,0 +1,10 @@ +// auto generated by generate.py +// clang-format off +#pragma once + +#include "kernel_bf16_ku4.cuh" +#include "kernel_bf16_ku4b8.cuh" +#include "kernel_bf16_ku8b128.cuh" +#include "kernel_fp16_ku4.cuh" +#include "kernel_fp16_ku4b8.cuh" +#include "kernel_fp16_ku8b128.cuh" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/marlin_template.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/marlin_template.h new file mode 100644 index 00000000..1ba99787 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/marlin_template.h @@ -0,0 +1,1899 @@ +/* + * Modified by Neural Magic + * Copyright (C) Marlin.2024 Elias Frantar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Adapted from https://github.com/IST-DASLab/marlin + */ + +#ifndef MARLIN_NAMESPACE_NAME +#define MARLIN_NAMESPACE_NAME marlin_moe_wna16 +#endif + +#include "gemm/marlin/dequant.h" +#include "gemm/marlin/marlin.cuh" +#include "gemm/marlin/marlin_dtypes.cuh" +#include "scalar_type.hpp" + +#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \ + static_assert( \ + std::is_same::value || std::is_same::value, \ + "only float16 and bfloat16 is supported"); + +namespace MARLIN_NAMESPACE_NAME { + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800 + +template < + typename scalar_t, // compute dtype, half or nv_float16 + const sglang::ScalarTypeId w_type_id, // weight ScalarType id + const int threads, // number of threads in a threadblock + const int thread_m_blocks, // number of 16x16 blocks in the m + // dimension (batchsize) of the + // threadblock + const int thread_n_blocks, // same for n dimension (output) + const int thread_k_blocks, // same for k dimension (reduction) + const bool m_block_size_8, // whether m_block_size == 8 + // only works when thread_m_blocks == 1 + const int stages, // number of stages for the async global->shared + // fetch pipeline + const int group_blocks, // number of consecutive 16x16 blocks + // with a separate quantization scale + const bool is_zp_float // is zero point of float16 type? + > +__global__ void Marlin( + const int4* __restrict__ A, // fp16 input matrix of shape mxk + const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn + int4* __restrict__ C, // fp16 output buffer of shape mxn + int4* __restrict__ C_tmp, // fp32 tmp output buffer (for reduce) + const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape + // (k/groupsize)xn + const int4* __restrict__ zp_ptr, // 4bit packed zero-points of shape + // (k/groupsize)x(n/pack_factor) + const int* __restrict__ g_idx, // int32 group indices of shape k + const int32_t* __restrict__ sorted_token_ids_ptr, // moe sorted_ids + const int32_t* __restrict__ expert_ids_ptr, // moe expert ids + const int32_t* __restrict__ num_tokens_past_padded_ptr, // moe num tokens + const float* __restrict__ topk_weights_ptr, // moe top weights + int top_k, // num of experts per token + bool mul_topk_weights, // mul topk weights or not + bool is_ep, // expert parallelism + int num_groups, // number of scale groups per output channel + int prob_m, // batch dimension m + int prob_n, // output dimension n + int prob_k, // reduction dimension k + int* locks, // extra global storage for barrier synchronization + bool use_atomic_add, // whether to use atomic add to reduce + bool use_fp32_reduce, // whether to use fp32 global reduce + int max_shared_mem) {} + +} // namespace MARLIN_NAMESPACE_NAME + +#else + +// m16n8k16 tensor core mma instruction with fp16 inputs and fp32 +// output/accumulation. +template +__device__ inline void +mma(const typename ScalarType::FragA& a_frag, + const typename ScalarType::FragB& frag_b, + typename ScalarType::FragC& frag_c) { + const uint32_t* a = reinterpret_cast(&a_frag); + const uint32_t* b = reinterpret_cast(&frag_b); + float* c = reinterpret_cast(&frag_c); + if constexpr (std::is_same::value) { + asm volatile( + "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 " + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3]) + : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3])); + } else if constexpr (std::is_same::value) { + asm volatile( + "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3]) + : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3])); + } else { + STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t); + } +} + +template +__device__ inline void mma_trans( + const typename ScalarType::FragA& a_frag, + const typename ScalarType::FragB& frag_b, + const typename ScalarType::FragB& frag_b2, + typename ScalarType::FragC& frag_c) { + const uint32_t* a = reinterpret_cast(&a_frag); + const uint32_t* b = reinterpret_cast(&frag_b); + const uint32_t* b2 = reinterpret_cast(&frag_b2); + float* c = reinterpret_cast(&frag_c); + if constexpr (std::is_same::value) { + asm volatile( + "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 " + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3]) + : "r"(b[0]), + "r"(b2[0]), + "r"(b[1]), + "r"(b2[1]), + "r"(a[0]), + "r"(a[1]), + "f"(c[0]), + "f"(c[1]), + "f"(c[2]), + "f"(c[3])); + } else if constexpr (std::is_same::value) { + asm volatile( + "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3]) + : "r"(b[0]), + "r"(b2[0]), + "r"(b[1]), + "r"(b2[1]), + "r"(a[0]), + "r"(a[1]), + "f"(c[0]), + "f"(c[1]), + "f"(c[2]), + "f"(c[3])); + } else { + STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t); + } +} + +// Instruction for loading a full 16x16 matrix fragment of operand A from shared +// memory, directly in tensor core layout. +template +__device__ inline void ldsm(typename ScalarType::FragA& frag_a, const void* smem_ptr) { + uint32_t* a = reinterpret_cast(&frag_a); + uint32_t smem = static_cast(__cvta_generic_to_shared(smem_ptr)); + if constexpr (count == 4) { + asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n" + : "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3]) + : "r"(smem)); + } else if constexpr (count == 2) { + asm volatile("ldmatrix.sync.aligned.m8n8.x2.shared.b16 {%0,%1}, [%2];\n" : "=r"(a[0]), "=r"(a[1]) : "r"(smem)); + } else if constexpr (count == 1) { + asm volatile("ldmatrix.sync.aligned.m8n8.x1.shared.b16 {%0}, [%1];\n" : "=r"(a[0]) : "r"(smem)); + } else { + static_assert(count == 1 || count == 2 || count == 4, "invalid count"); + } +} + +// Multiply dequantized values by the corresponding quantization scale; used +// only for grouped quantization. +template +__device__ inline void +scale(typename ScalarType::FragB& frag_b, typename ScalarType::FragS& frag_s, int i) { + using scalar_t2 = typename ScalarType::scalar_t2; + scalar_t2 s = ScalarType::num2num2(reinterpret_cast(&frag_s)[i]); + frag_b[0] = __hmul2(frag_b[0], s); + frag_b[1] = __hmul2(frag_b[1], s); +} + +template +__device__ inline void scale_and_sub(typename ScalarType::FragB& frag_b, scalar_t s, scalar_t zp) { + using scalar_t2 = typename ScalarType::scalar_t2; + scalar_t2 s2 = ScalarType::num2num2(s); + scalar_t2 zp2 = ScalarType::num2num2(zp); + frag_b[0] = __hfma2(frag_b[0], s2, __hneg2(zp2)); + frag_b[1] = __hfma2(frag_b[1], s2, __hneg2(zp2)); +} + +template +__device__ inline void +sub_zp(typename ScalarType::FragB& frag_b, typename ScalarType::scalar_t2& frag_zp, int i) { + using scalar_t2 = typename ScalarType::scalar_t2; + scalar_t2 zp = ScalarType::num2num2(reinterpret_cast(&frag_zp)[i]); + frag_b[0] = __hsub2(frag_b[0], zp); + frag_b[1] = __hsub2(frag_b[1], zp); +} + +// Same as above, but for act_order (each K is multiplied individually) +template +__device__ inline void scale4( + typename ScalarType::FragB& frag_b, + typename ScalarType::FragS& frag_s_1, + typename ScalarType::FragS& frag_s_2, + typename ScalarType::FragS& frag_s_3, + typename ScalarType::FragS& frag_s_4, + int i) { + using scalar_t2 = typename ScalarType::scalar_t2; + scalar_t2 s_val_1_2; + s_val_1_2.x = reinterpret_cast(&frag_s_1)[i]; + s_val_1_2.y = reinterpret_cast(&frag_s_2)[i]; + + scalar_t2 s_val_3_4; + s_val_3_4.x = reinterpret_cast(&frag_s_3)[i]; + s_val_3_4.y = reinterpret_cast(&frag_s_4)[i]; + + frag_b[0] = __hmul2(frag_b[0], s_val_1_2); + frag_b[1] = __hmul2(frag_b[1], s_val_3_4); +} + +// Given 2 floats multiply by 2 scales (halves) +template +__device__ inline void scale_float(float* c, typename ScalarType::FragS& s) { + scalar_t* s_ptr = reinterpret_cast(&s); + c[0] = __fmul_rn(c[0], ScalarType::num2float(s_ptr[0])); + c[1] = __fmul_rn(c[1], ScalarType::num2float(s_ptr[1])); +} + +// Wait until barrier reaches `count`, then lock for current threadblock. +__device__ inline void barrier_acquire(int* lock, int count) { + if (threadIdx.x == 0) { + int state = -1; + do + // Guarantee that subsequent writes by this threadblock will be visible + // globally. + asm volatile("ld.global.acquire.gpu.b32 %0, [%1];\n" : "=r"(state) : "l"(lock)); + while (state != count); + } + __syncthreads(); +} + +// Release barrier and increment visitation count. +__device__ inline void barrier_release(int* lock, bool reset = false) { + __syncthreads(); + if (threadIdx.x == 0) { + if (reset) { + lock[0] = 0; + return; + } + int val = 1; + // Make sure that all writes since acquiring this barrier are visible + // globally, while releasing the barrier. + asm volatile("fence.acq_rel.gpu;\n"); + asm volatile("red.relaxed.gpu.global.add.s32 [%0], %1;\n" : : "l"(lock), "r"(val)); + } +} + +// Wait until value of lock to be negative, and then add 1 +__device__ inline void wait_negative_and_add(int* lock) { + if (threadIdx.x == 0) { + int state = 0; + do + // Guarantee that subsequent writes by this threadblock will be visible + // globally. + asm volatile("ld.global.acquire.gpu.b32 %0, [%1];\n" : "=r"(state) : "l"(lock)); + while (state >= 0); + atomicAdd(lock, 1); + } + __syncthreads(); +} + +template < + typename scalar_t, // compute dtype, half or nv_float16 + const sglang::ScalarTypeId w_type_id, // weight ScalarType id + const sglang::ScalarTypeId s_type_id, // weight scale ScalarType id + const int threads, // number of threads in a threadblock + const int thread_m_blocks, // number of 16x16 blocks in the m + // dimension (batchsize) of the + // threadblock + const int thread_n_blocks, // same for n dimension (output) + const int thread_k_blocks, // same for k dimension (reduction) + const bool m_block_size_8, // whether m_block_size == 8 + // only works when thread_m_blocks == 1 + const int stages, // number of stages for the async global->shared + // fetch pipeline + const int group_blocks, // number of consecutive 16x16 blocks + // with a separate quantization scale + const bool is_zp_float // is zero point of float16 type? + > +__global__ void Marlin( + const int4* __restrict__ A, // fp16 input matrix of shape mxk + const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn + int4* __restrict__ C, // fp16 output buffer of shape mxn + int4* __restrict__ C_tmp, // fp32 tmp output buffer (for reduce) + const int4* __restrict__ b_bias_ptr, + const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape + // (k/groupsize)xn + const uint16_t* __restrict__ scale2_ptr, // fp16 global scale (for nvfp4 + // only) + const int4* __restrict__ zp_ptr, // 4bit packed zero-points of shape + // (k/groupsize)x(n/pack_factor) + const int* __restrict__ g_idx, // int32 group indices of shape k + const int32_t* __restrict__ sorted_token_ids_ptr, // moe sorted_ids + const int32_t* __restrict__ expert_ids_ptr, // moe expert ids + const int32_t* __restrict__ num_tokens_past_padded_ptr, // moe num tokens + const float* __restrict__ topk_weights_ptr, // moe top weights + int top_k, // num of experts per token + bool mul_topk_weights, // mul topk weights or not + bool is_ep, // expert parallelism + int num_groups, // number of scale groups per output channel + int prob_m, // batch dimension m + int prob_n, // output dimension n + int prob_k, // reduction dimension k + int* locks, // extra global storage for barrier synchronization + bool has_bias, + bool use_atomic_add, // whether to use atomic add to reduce + bool use_fp32_reduce, // whether to use fp32 global reduce + int max_shared_mem) { + // Each threadblock processes one "stripe" of the B matrix with (roughly) the + // same size, which might involve multiple column "slices" (of width 16 * + // `thread_n_blocks`). Stripes are defined as shown in the 3x3 matrix 5 SM + // example: + // 0 1 3 + // 0 2 3 + // 1 2 4 + // While this kind of partitioning makes things somewhat more complicated, it + // ensures good utilization of all SMs for many kinds of shape and GPU + // configurations, while requiring as few slow global cross-threadblock + // reductions as possible. + using Dtype = ScalarType; + using scalar_t2 = typename ScalarType::scalar_t2; + using FragA = typename ScalarType::FragA; + using FragB = typename ScalarType::FragB; + using FragC = typename ScalarType::FragC; + using FragS = typename ScalarType::FragS; + using FragZP = typename ScalarType::FragZP; + + extern __shared__ int4 sh[]; + static constexpr auto w_type = sglang::ScalarType::from_id(w_type_id); + static constexpr auto s_type = sglang::ScalarType::from_id(s_type_id); + if constexpr (w_type == sglang::kFE2M1f) { + static_assert(s_type == sglang::kFE4M3fn && group_blocks == 1 || s_type == sglang::kFE8M0fnu && group_blocks == 2); + } else if constexpr (std::is_same::value) { + static_assert(s_type == sglang::kBFloat16); + } else if constexpr (std::is_same::value) { + static_assert(s_type == sglang::kFloat16); + } + + constexpr bool has_zp = w_type == sglang::kU4 || w_type == sglang::kU8; + constexpr bool is_int_type = + w_type == sglang::kU4 || w_type == sglang::kU8 || w_type == sglang::kU4B8 || w_type == sglang::kU8B128; + // see comments of dequant.h for more details + constexpr bool dequant_skip_flop = w_type == sglang::kFE4M3fn || + w_type == sglang::kFE2M1f && s_type == sglang::kFE4M3fn || + has_zp && !is_zp_float && !std::is_same::value || + has_zp && !is_zp_float && !(w_type == sglang::kU8); + + scalar_t2 global_scale; + + constexpr bool has_act_order = group_blocks == 0; + + constexpr int pack_factor = 32 / w_type.size_bits(); + static_assert(thread_m_blocks == 1 || !m_block_size_8); + constexpr int moe_block_size = m_block_size_8 ? 8 : (16 * thread_m_blocks); + const int group_size = (!has_act_order && group_blocks == -1) ? prob_k : prob_k / num_groups; + const int scales_expert_stride = prob_n * prob_k / group_size / (w_type == sglang::kFE2M1f ? 16 : 8); + const int zp_expert_stride = + is_zp_float ? prob_n * prob_k / group_size / 8 : prob_n * prob_k / group_size / (pack_factor * 4); + const int b_bias_expert_stride = prob_n / 8; + + // parallel: num valid moe blocks + int num_tokens_past_padded = num_tokens_past_padded_ptr[0]; + int parallel = num_tokens_past_padded / moe_block_size; + int num_valid_blocks = parallel; + if (is_ep) { + for (int i = 0; i < parallel; i++) { + if (expert_ids_ptr[i] == -1) num_valid_blocks--; + } + } + int num_invalid_blocks = parallel - num_valid_blocks; + parallel = num_valid_blocks; + + int k_tiles = prob_k / 16 / thread_k_blocks; + int n_tiles = prob_n / 16 / thread_n_blocks; + int iters = div_ceil(k_tiles * n_tiles * parallel, gridDim.x); + + if constexpr (!has_act_order && group_blocks != -1) { + if (group_blocks >= thread_k_blocks) { + // Ensure that the number of tiles in each stripe is a multiple of the + // groupsize; this avoids an annoying special case where a stripe starts + // in the middle of group. + iters = (group_blocks / thread_k_blocks) * div_ceil(iters, (group_blocks / thread_k_blocks)); + } + } + + int slice_row = (iters * blockIdx.x) % k_tiles; + int slice_col_par = (iters * blockIdx.x) / k_tiles; + int slice_col = slice_col_par; + int slice_iters; // number of threadblock tiles in the current slice + int slice_count = 0; // total number of active threadblocks in the current slice + int slice_idx; // index of threadblock in current slice; numbered bottom to + // top + + int par_id = 0; + int block_id = -1; + int64_t expert_id = 0; // use int64 to avoid computation result overflow + int old_expert_id = 0; + int64_t B_expert_off = 0; + + int4* sh_block_sorted_ids_int4 = sh; + int4* sh_rd_block_sorted_ids_int4 = sh_block_sorted_ids_int4 + moe_block_size / 4; + int4* sh_block_topk_weights_int4 = sh_rd_block_sorted_ids_int4 + moe_block_size / 4; + // sh_block_topk_weights_int4 only need (moe_block_size / 4); + // but we pad to align to 256 bytes + int4* sh_new = sh_block_topk_weights_int4 + moe_block_size / 2 + moe_block_size; + int32_t* sh_block_sorted_ids = reinterpret_cast(sh_block_sorted_ids_int4); + int32_t* sh_rd_block_sorted_ids = reinterpret_cast(sh_rd_block_sorted_ids_int4); + scalar_t2* sh_block_topk_weights = reinterpret_cast(sh_block_topk_weights_int4); + + int32_t block_num_valid_tokens = 0; + int32_t locks_off = 0; + + // We can easily implement parallel problem execution by just remapping + // indices and advancing global pointers + if (slice_col_par >= n_tiles) { + slice_col = slice_col_par % n_tiles; + par_id = slice_col_par / n_tiles; + } + if (parallel * n_tiles >= gridDim.x) { + // when parallel * n_tiles >= sms + // then there are at most $sms$ conflict tile blocks + locks_off = blockIdx.x; + } else { + locks_off = (iters * blockIdx.x) / k_tiles - 1; + } + + // read moe block data given block_id + // block_sorted_ids / block_num_valid_tokens / block_topk_weights + auto read_moe_block_data = [&](int block_id) { + block_num_valid_tokens = moe_block_size; +#pragma unroll + for (int i = 0; i < moe_block_size / 4; i++) { + int4 sorted_token_ids_int4 = + reinterpret_cast(sorted_token_ids_ptr)[block_id * moe_block_size / 4 + i]; + int* sorted_token_ids = reinterpret_cast(&sorted_token_ids_int4); +#pragma unroll + for (int j = 0; j < 4; j++) { + if (sorted_token_ids[j] >= prob_m * top_k) { + block_num_valid_tokens = i * 4 + j; + break; + } + } + if (block_num_valid_tokens != moe_block_size) break; + } + + __syncthreads(); + int tid4 = threadIdx.x / 4; + if (threadIdx.x % 4 == 0 && threadIdx.x < block_num_valid_tokens) { + sh_block_sorted_ids_int4[tid4] = + reinterpret_cast(sorted_token_ids_ptr)[block_id * moe_block_size / 4 + tid4]; + +#pragma unroll + for (int i = 0; i < 4; i++) + sh_rd_block_sorted_ids[tid4 * 4 + i] = sh_block_sorted_ids[tid4 * 4 + i] / top_k; + + if (mul_topk_weights) { +#pragma unroll + for (int i = 0; i < 4; i++) { + int idx = tid4 * 4 + i; + // idx = idx < block_num_valid_tokens ? idx : 0; + if (idx < block_num_valid_tokens) { + if constexpr (w_type == sglang::kFE2M1f && s_type == sglang::kFE4M3fn) { + sh_block_topk_weights[idx] = + __hmul2(global_scale, Dtype::num2num2(Dtype::float2num(topk_weights_ptr[sh_block_sorted_ids[idx]]))); + } else { + sh_block_topk_weights[idx] = + Dtype::num2num2(Dtype::float2num(topk_weights_ptr[sh_block_sorted_ids[idx]])); + } + } + } + } + } + __syncthreads(); + }; + + // when move to next moe block, find the next block_id and expert_id + // and then read moe block data + auto update_next_moe_block_data = [&]() { + if (par_id >= parallel) return; + + old_expert_id = expert_id; + if (num_invalid_blocks > 0) { + int skip_count = block_id == -1 ? par_id : 0; + block_id++; + for (int i = block_id; i < num_tokens_past_padded / moe_block_size; i++) { + expert_id = expert_ids_ptr[i]; + if (expert_id != -1) { + if (skip_count == 0) { + block_id = i; + break; + }; + skip_count--; + }; + } + } else { + block_id = par_id; + expert_id = expert_ids_ptr[block_id]; + } + + if constexpr (w_type == sglang::kFE2M1f && s_type == sglang::kFE4M3fn) { + uint16_t val = scale2_ptr[expert_id]; + global_scale = Dtype::num2num2(*reinterpret_cast(&val)); + } + + B_expert_off = expert_id * prob_n * prob_k / (pack_factor * 4); + scales_ptr += (expert_id - old_expert_id) * scales_expert_stride; + if constexpr (has_zp) { + zp_ptr += (expert_id - old_expert_id) * zp_expert_stride; + } + if constexpr (has_act_order) { + g_idx += (expert_id - old_expert_id) * prob_k; + } + if (has_bias) { + b_bias_ptr += (expert_id - old_expert_id) * b_bias_expert_stride; + } + + read_moe_block_data(block_id); + }; + + // Compute all information about the current slice which is required for + // synchronization. + auto init_slice = [&](bool first_init = false) { + slice_iters = iters * (blockIdx.x + 1) - (k_tiles * slice_col_par + slice_row); + if (slice_iters < 0 || slice_col_par >= n_tiles * parallel) slice_iters = 0; + if (slice_iters == 0) return; + if (slice_row + slice_iters > k_tiles) slice_iters = k_tiles - slice_row; + slice_count = 1; + slice_idx = 0; + int col_first = iters * div_ceil(k_tiles * slice_col_par, iters); + if (col_first <= k_tiles * (slice_col_par + 1)) { + int col_off = col_first - k_tiles * slice_col_par; + slice_count = div_ceil(k_tiles - col_off, iters); + if (col_off > 0) slice_count++; + int delta_first = iters * blockIdx.x - col_first; + if (delta_first < 0 || (col_off == 0 && delta_first == 0)) + slice_idx = slice_count - 1; + else { + slice_idx = slice_count - 1 - delta_first / iters; + if (col_off > 0) slice_idx--; + } + } + if (parallel * n_tiles >= gridDim.x) { + if (slice_count > 1 && slice_idx == slice_count - 1) { + locks_off++; + } + } else { + locks_off++; + } + + if (first_init && use_atomic_add && slice_count > 1 && slice_idx == 0) { + constexpr int threads_per_m = 16 * thread_n_blocks / 8; + int m_per_thread = div_ceil(block_num_valid_tokens, threads / threads_per_m); + for (int i = 0; i < m_per_thread; i++) { + int row = threads / threads_per_m * i + threadIdx.x / threads_per_m; + if (row < block_num_valid_tokens) { + int64_t sorted_row = sh_block_sorted_ids[row]; + int col = slice_col * 16 * thread_n_blocks / 8 + threadIdx.x % threads_per_m; + C[sorted_row * prob_n / 8 + col] = {0, 0, 0, 0}; + } + } + // After write zero to output, write a negative value to lock. + // Every SM that processes the same slice would wait for + // the negative value, and then atomicAdd 1 to it. + // After all SMs are processed, the lock value would back to 0 again. + __syncthreads(); + if (threadIdx.x == 0) locks[locks_off] = 1 - slice_count; + } + + if (slice_col == n_tiles) { + slice_col = 0; + par_id++; + update_next_moe_block_data(); + } + }; + + update_next_moe_block_data(); + init_slice(true); + + // A sizes/strides + + // stride of the A matrix in global memory + int a_gl_stride = prob_k / 8; + // stride of an A matrix tile in shared memory + constexpr int a_sh_stride = 16 * thread_k_blocks / 8; + // delta between subsequent A tiles in global memory + constexpr int a_gl_rd_delta_o = 16 * thread_k_blocks / 8; + // between subsequent accesses within a tile + int a_gl_rd_delta_i = a_gl_stride * (threads / a_gl_rd_delta_o); + // between shared memory writes + constexpr int a_sh_wr_delta = a_sh_stride * (threads / a_gl_rd_delta_o); + // between shared memory tile reads + constexpr int a_sh_rd_delta_o = 2 * ((threads / 32) / (thread_n_blocks / 4)); + // within a shared memory tile + constexpr int a_sh_rd_delta_i = a_sh_stride * 16; + // overall size of a tile + constexpr int a_sh_stage = a_sh_stride * (16 * thread_m_blocks); + // number of shared write iterations for a tile + constexpr int a_sh_wr_iters = div_ceil(a_sh_stage, a_sh_wr_delta); + + // B sizes/strides + int b_gl_stride = 16 * prob_n / (pack_factor * 4); + constexpr int b_sh_stride = ((thread_n_blocks * 16) * 16 / pack_factor) / 4; + constexpr int b_thread_vecs = w_type.size_bits() == 4 ? 1 : 2; + constexpr int b_sh_stride_threads = b_sh_stride / b_thread_vecs; + + int b_gl_rd_delta_o = b_gl_stride * thread_k_blocks; + int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride_threads); + constexpr int b_sh_wr_delta = threads * b_thread_vecs; + constexpr int b_sh_rd_delta = threads * b_thread_vecs; + constexpr int b_sh_stage = b_sh_stride * thread_k_blocks; + constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta; + + // Scale sizes/strides without act_order + int s_gl_stride = prob_n / 8; + constexpr int s_sh_stride = 16 * thread_n_blocks / 8; + constexpr int s_tb_groups = !has_act_order && group_blocks != -1 && group_blocks < thread_k_blocks + ? thread_k_blocks / group_blocks / (w_type == sglang::kFE2M1f ? 2 : 1) + : 1; + constexpr int s_sh_stage = s_tb_groups * s_sh_stride; + int s_gl_rd_delta = s_gl_stride; + + // Scale size/strides with act_order + constexpr int tb_k = 16 * thread_k_blocks; + constexpr int g_idx_stage = has_act_order ? (tb_k * sizeof(int)) / 16 : 0; + // constexpr int act_s_row_stride = 1; + // int act_s_col_stride = act_s_row_stride * num_groups; + constexpr int act_s_max_num_groups = 32; + int act_s_col_stride = 1; + int act_s_col_warp_stride = act_s_col_stride * 8; + int tb_n_warps = thread_n_blocks / 4; + int act_s_col_tb_stride = act_s_col_warp_stride * tb_n_warps; + + // Zero-points sizes/strides + int zp_gl_stride = is_zp_float ? prob_n / 8 : (prob_n / pack_factor) / 4; + constexpr int zp_sh_stride = is_zp_float ? 16 * thread_n_blocks / 8 : ((16 * thread_n_blocks) / pack_factor) / 4; + constexpr int zp_tb_groups = s_tb_groups; + constexpr int zp_sh_stage = has_zp ? zp_tb_groups * zp_sh_stride : 0; + int zp_gl_rd_delta = zp_gl_stride; + + // Global A read index of current thread. + int a_gl_rd_row = threadIdx.x / a_gl_rd_delta_o; + int a_gl_rd_col = a_gl_rd_delta_o * slice_row + threadIdx.x % a_gl_rd_delta_o; + + // Shared write index of current thread. + int a_sh_wr = a_sh_stride * (threadIdx.x / a_gl_rd_delta_o) + (threadIdx.x % a_gl_rd_delta_o); + // Shared read index. + int a_sh_rd = a_sh_stride * ((threadIdx.x % 32) % (16 / (m_block_size_8 ? 2 : 1))) + + (threadIdx.x % 32) / (16 / (m_block_size_8 ? 2 : 1)); + a_sh_rd += 2 * ((threadIdx.x / 32) / (thread_n_blocks / 4)); + + int b_gl_rd = b_gl_stride * (threadIdx.x / b_sh_stride_threads) + (threadIdx.x % b_sh_stride_threads) * b_thread_vecs; + b_gl_rd += b_sh_stride * slice_col; + b_gl_rd += b_gl_rd_delta_o * slice_row; + auto b_sh_wr = threadIdx.x * b_thread_vecs; + auto b_sh_rd = threadIdx.x * b_thread_vecs; + + // For act_order + constexpr int k_iter_size = tb_k / b_sh_wr_iters; + int slice_k_start = tb_k * slice_row; + int slice_k_finish = slice_k_start + tb_k * slice_iters; + int slice_k_start_shared_fetch = slice_k_start; + int slice_n_offset = act_s_col_tb_stride * slice_col; + + // No act_order + int s_gl_rd; + if constexpr (!has_act_order) { + if constexpr (group_blocks == -1) { + s_gl_rd = s_sh_stride * slice_col + threadIdx.x; + } else { + s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) / (w_type == sglang::kFE2M1f ? 2 : 1) + + s_sh_stride * slice_col + threadIdx.x; + } + } + auto s_sh_wr = threadIdx.x; + bool s_sh_wr_pred = threadIdx.x < s_sh_stride; + + // Zero-points + int zp_gl_rd; + if constexpr (has_zp) { + if constexpr (group_blocks == -1) { + zp_gl_rd = zp_sh_stride * slice_col + threadIdx.x; + } else { + zp_gl_rd = zp_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) + zp_sh_stride * slice_col + threadIdx.x; + } + } + auto zp_sh_wr = threadIdx.x; + bool zp_sh_wr_pred = threadIdx.x < zp_sh_stride; + + // We use a different scale layout for grouped and column-wise quantization as + // we scale a `half2` tile in column-major layout in the former and in + // row-major in the latter case. + int s_sh_rd; + if constexpr (group_blocks != -1 && w_type == sglang::kFE2M1f) { + auto warp_id = threadIdx.x / 32; + int n_warps = thread_n_blocks / 4; + int warp_row = warp_id / n_warps; + + s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + (threadIdx.x % 32) / 4; + s_sh_rd = s_sh_rd * 2 + (warp_row / group_blocks) % 2; + + } else if constexpr (group_blocks != -1) + s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + (threadIdx.x % 32) / 4; + else if constexpr (group_blocks == -1 && (m_block_size_8 || (has_zp && !dequant_skip_flop))) + s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + (threadIdx.x % 32) / 8; + else + s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + (threadIdx.x % 32) % 4; + + int bias_sh_rd; + if constexpr (m_block_size_8) { + bias_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + (threadIdx.x % 32) / 8; + } else { + bias_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + (threadIdx.x % 32) % 4; + } + + int bias_sh_wr = threadIdx.x; + int bias_gl_rd = (thread_n_blocks * 16 / 8) * slice_col + threadIdx.x; + + // Zero-points have the same read layout as the scales + // (without column-wise case) + constexpr int num_col_threads = 8; + constexpr int num_row_threads = 4; + constexpr int num_ints_per_thread = 8 / pack_factor; + int zp_sh_rd; + if constexpr (has_zp) { + if constexpr (is_zp_float) { + if constexpr (group_blocks != -1) { + zp_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + (threadIdx.x % 32) / 4; + } + } else { + zp_sh_rd = num_ints_per_thread * num_col_threads * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + + num_ints_per_thread * ((threadIdx.x % 32) / num_row_threads); + } + } + + // To ensure that writing and reading A tiles to/from shared memory, the + // latter in fragment format, is fully bank conflict free, we need to use a + // rather fancy XOR-based layout. The key here is that neither reads nor + // writes of the 16-byte `int4` blocks of 8 consecutive threads involve the + // same shared memory banks. Further, it seems (based on NSight-Compute) that + // each warp must also write a consecutive memory segment? + auto transform_a = [&](int i) { + int row = i / a_gl_rd_delta_o; + return a_gl_rd_delta_o * row + (i % a_gl_rd_delta_o) ^ (row % 8); + }; + // Since the computation of this remapping is non-trivial and, due to our main + // loop unrolls, all shared memory accesses are static, we simply precompute + // both transformed reads and writes. + int a_sh_wr_trans[a_sh_wr_iters]; +#pragma unroll + for (int i = 0; i < a_sh_wr_iters; i++) + a_sh_wr_trans[i] = transform_a(a_sh_wr_delta * i + a_sh_wr); + int a_sh_rd_trans[b_sh_wr_iters][thread_m_blocks]; +#pragma unroll + for (int i = 0; i < b_sh_wr_iters; i++) { +#pragma unroll + for (int j = 0; j < thread_m_blocks; j++) + a_sh_rd_trans[i][j] = transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd); + } + + // Since B-accesses have non-constant stride they have to be computed at + // runtime; we break dependencies between subsequent accesses with a tile by + // maintining multiple pointers (we have enough registers), a tiny + // optimization. + const int4* B_ptr[b_sh_wr_iters]; +#pragma unroll + for (int i = 0; i < b_sh_wr_iters; i++) + B_ptr[i] = B + b_gl_rd_delta_i * i + b_gl_rd; + + // Shared memory storage for global fetch pipelines. + constexpr int sh_red_size = (2 * thread_n_blocks + 1) * 16 * thread_m_blocks; + constexpr int sh_b_size = stages * b_sh_stage; + int4* sh_b = sh_new; + int4* sh_red = sh_new; + + constexpr int sh_size_b_red_min = (sh_red_size < sh_b_size ? sh_red_size : sh_b_size); + constexpr int sh_size_b_red_max = (sh_red_size > sh_b_size ? sh_red_size : sh_b_size); + constexpr int sh_bias_size = (thread_n_blocks * 16 / 8); + constexpr int sh_b_red_bias_size = + sh_size_b_red_max > (sh_size_b_red_min + sh_bias_size) ? sh_size_b_red_max : (sh_size_b_red_min + sh_bias_size); + + int4* sh_bias = sh_new + sh_size_b_red_min; + int4* sh_g_idx = sh_new + sh_b_red_bias_size; + int4* sh_zp = sh_g_idx + (stages * g_idx_stage); + constexpr int sh_s_size = has_act_order ? (act_s_max_num_groups * s_sh_stride) : (stages * s_sh_stage); + int4* sh_s = sh_zp + (stages * zp_sh_stage); + // shared memory reused by reduction should be smaller than + // shared memory used by weight. + static_assert(thread_m_blocks * 16 * thread_n_blocks * 16 / 8 <= stages * b_sh_stage); + int4* sh_a = sh_s + sh_s_size; + constexpr int shm_size_used = moe_block_size + stages * (g_idx_stage + zp_sh_stage) + sh_s_size + sh_b_red_bias_size; + + // all remaining shared memory is used to cache A (input) + // sh_a_max_row is at least ` stages * 16 * thread_m_blocks ` + int sh_a_max_row = ((max_shared_mem - 1024) / 16 - shm_size_used) / (thread_k_blocks * 2); + + // Register storage for double buffer of shared memory reads. + FragA frag_a[2][thread_m_blocks]; + I4 frag_b_quant[2][b_thread_vecs]; + FragC frag_c[thread_m_blocks][4][2]; + FragS frag_s[2][4]; // No act-order + FragS frag_bias[2][4]; + FragS act_frag_s[2][4][4]; // For act-order + int frag_qzp[2][num_ints_per_thread]; // Zero-points + FragZP frag_zp; // Zero-points in fp16 + FragZP frag_zpf[2]; // Zero-points in fp16 in HQQ + + // Zero accumulators. + auto zero_accums = [&]() { +#pragma unroll + for (int i = 0; i < thread_m_blocks * 4 * 2 * 4; i++) + reinterpret_cast(frag_c)[i] = 0; + }; + + int sh_first_group_id = -1; + int sh_num_groups = -1; + + auto fetch_act_order_scales_to_shared = [&](bool is_async, int first_group_id, int last_group_id) { + sh_first_group_id = first_group_id; + sh_num_groups = last_group_id - first_group_id + 1; + + if (sh_num_groups > act_s_max_num_groups) { + sh_num_groups = act_s_max_num_groups; + } + + if (sh_first_group_id + sh_num_groups > num_groups) { + sh_num_groups = num_groups - sh_first_group_id; + } + + int row_offset = first_group_id * s_gl_stride; + + if (is_async) { + for (int i = 0; i < sh_num_groups; i++) { + if (threadIdx.x < s_sh_stride) { + cp_async4_pred( + &sh_s[(i * s_sh_stride) + threadIdx.x], + &scales_ptr[row_offset + (i * s_gl_stride) + slice_n_offset + threadIdx.x]); + } + } + } else { + for (int i = 0; i < sh_num_groups; i++) { + if (threadIdx.x < s_sh_stride) { + sh_s[(i * s_sh_stride) + threadIdx.x] = + scales_ptr[row_offset + (i * s_gl_stride) + slice_n_offset + threadIdx.x]; + } + } + } + }; + + // Asynchronously fetch the next A, B and s tile from global to the next + // shared memory pipeline location. + bool should_load_a = true; + int max_num_stage_groups = ((sh_a_max_row - moe_block_size) / moe_block_size + 1) / stages; + max_num_stage_groups = max(max_num_stage_groups, 1); + auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true, int pipe_a = 0) { + if (pred) { + if (should_load_a) { + int4* sh_a_stage = sh_a + moe_block_size * a_sh_stride * pipe_a; +#pragma unroll + for (int i = 0; i < a_sh_wr_iters; i++) { + int row = a_gl_rd_delta_i / a_gl_stride * i + a_gl_rd_row; + int64_t sorted_row = 0; + if (!m_block_size_8 || row < 8) sorted_row = sh_rd_block_sorted_ids[row]; + int64_t true_idx = sorted_row * a_gl_stride + a_gl_rd_col + a_gl_rd_delta_o * a_off; + cp_async4_pred(&sh_a_stage[a_sh_wr_trans[i]], &A[true_idx], row < block_num_valid_tokens); + } + } + + int4* sh_b_stage = sh_b + b_sh_stage * pipe; +#pragma unroll + for (int i = 0; i < b_sh_wr_iters; i++) { +#pragma unroll + for (int j = 0; j < b_thread_vecs; j++) { + cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr + j], B_ptr[i] + j + B_expert_off); + } + + B_ptr[i] += b_gl_rd_delta_o; + } + + if constexpr (has_act_order) { + // Fetch g_idx thread-block portion + int full_pipe = a_off; + int cur_k = slice_k_start_shared_fetch + tb_k * full_pipe; + if (cur_k < prob_k && cur_k < slice_k_finish) { + int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe; + + int4 const* cur_g_idx_stage_ptr = reinterpret_cast(&g_idx[cur_k]); + + if (threadIdx.x < g_idx_stage) { + cp_async4_pred(&sh_g_idx_stage[threadIdx.x], &cur_g_idx_stage_ptr[threadIdx.x]); + } + } + } else { + if constexpr (group_blocks != -1) { + int4* sh_s_stage = sh_s + s_sh_stage * pipe; + + if constexpr (group_blocks >= thread_k_blocks) { + // Only fetch scales if this tile starts a new group + if (pipe % (group_blocks / thread_k_blocks) == 0) { + if (s_sh_wr_pred) { + cp_async4(&sh_s_stage[s_sh_wr], &scales_ptr[s_gl_rd]); + } + s_gl_rd += s_gl_rd_delta; + } + } else { + for (int i = 0; i < s_tb_groups; i++) { + if (s_sh_wr_pred) { + cp_async4(&sh_s_stage[i * s_sh_stride + s_sh_wr], &scales_ptr[s_gl_rd]); + } + s_gl_rd += s_gl_rd_delta; + } + } + } + + if constexpr (has_zp && group_blocks != -1) { + int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe; + + if constexpr (group_blocks >= thread_k_blocks) { + // Only fetch zero-points if this tile starts a new group + if (pipe % (group_blocks / thread_k_blocks) == 0) { + if (zp_sh_wr_pred) { + cp_async4(&sh_zp_stage[zp_sh_wr], &zp_ptr[zp_gl_rd]); + } + zp_gl_rd += zp_gl_rd_delta; + } + } else { + for (int i = 0; i < zp_tb_groups; i++) { + if (zp_sh_wr_pred) { + cp_async4(&sh_zp_stage[i * zp_sh_stride + zp_sh_wr], &zp_ptr[zp_gl_rd]); + } + zp_gl_rd += zp_gl_rd_delta; + } + } + } + } + } + // Insert a fence even when we are winding down the pipeline to ensure that + // waiting is also correct at this point. + cp_async_fence(); + }; + + auto fetch_col_zp_to_shared = [&]() { + if (zp_sh_wr_pred) { + cp_async4(&sh_zp[zp_sh_wr], &zp_ptr[zp_gl_rd]); + } + }; + + auto fetch_col_scale_to_shared = [&]() { + if (s_sh_wr_pred) { + cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]); + } + }; + + // Wait until the next thread tile has been loaded to shared memory. + auto wait_for_stage = [&]() { + // We only have `stages - 2` active fetches since we are double buffering + // and can only issue the next fetch when it is guaranteed that the previous + // shared memory load is fully complete (as it may otherwise be + // overwritten). + cp_async_wait(); + __syncthreads(); + }; + + // Load the next sub-tile from the current location in the shared memory pipe + // into the current register buffer. + auto fetch_to_registers = [&](int k, int pipe, int pipe_a = 0) { + int4* sh_a_stage = sh_a + moe_block_size * a_sh_stride * pipe_a; +#pragma unroll + for (int i = 0; i < thread_m_blocks; i++) + ldsm(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]); + int4* sh_b_stage = sh_b + b_sh_stage * pipe; + +#pragma unroll + for (int i = 0; i < b_thread_vecs; i++) { + frag_b_quant[k % 2][i] = *reinterpret_cast(&sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd + i]); + } + }; + + bool is_same_group[stages]; + int same_group_id[stages]; + + auto init_same_group = [&](int pipe) { + if constexpr (!has_act_order) { + return; + } + + int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe; + int* sh_g_idx_int_ptr = reinterpret_cast(sh_g_idx_stage); + + int group_id_1 = sh_g_idx_int_ptr[0]; + int group_id_2 = sh_g_idx_int_ptr[tb_k - 1]; + + is_same_group[pipe] = group_id_1 == group_id_2; + same_group_id[pipe] = group_id_1; + }; + + auto fetch_scales_to_registers = [&](int k, int full_pipe) { + int pipe = full_pipe % stages; + + if constexpr (!has_act_order) { + // No act-order case + if constexpr (group_blocks == -1) { + // load only when starting a new slice + if (k == 0 && full_pipe == 0) { + reinterpret_cast(&frag_s)[0] = sh_s[s_sh_rd]; + reinterpret_cast(&frag_s)[1] = sh_s[s_sh_rd + 4]; + } + } else if constexpr (group_blocks != -1) { + if constexpr (group_blocks >= thread_k_blocks) { + if (k % b_sh_wr_iters == 0) { + int4* sh_s_stage = + sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) * (pipe / (group_blocks / thread_k_blocks))); + reinterpret_cast(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd]; + } else { + reinterpret_cast(&frag_s[1])[0] = reinterpret_cast(&frag_s[0])[0]; + } + } else { + auto warp_id = threadIdx.x / 32; + int n_warps = thread_n_blocks / 4; + + int warp_row = warp_id / n_warps; + + int cur_k = warp_row * 16; + cur_k += k_iter_size * (k % b_sh_wr_iters); + + int k_blocks = cur_k / 16; + int cur_group_id = k_blocks / (group_blocks * (w_type == sglang::kFE2M1f ? 2 : 1)); + + int4* sh_s_stage = sh_s + s_sh_stage * pipe; + + if constexpr (w_type_id != sglang::kFE2M1f.id()) { + reinterpret_cast(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd + cur_group_id * s_sh_stride]; + } else if constexpr (group_blocks == 1 || thread_k_blocks > 4) { + reinterpret_cast(&frag_s[k % 2])[0] = + reinterpret_cast(sh_s_stage)[s_sh_rd + cur_group_id * (2 * s_sh_stride)]; + } else { + reinterpret_cast(&frag_s[k % 2])[0] = + reinterpret_cast(sh_s_stage)[s_sh_rd + cur_group_id * (2 * s_sh_stride) + k % 2]; + } + } + } + + return; + } + + // Act-order case + + // Determine K of the "current" thread-block + int cur_k = slice_k_start + tb_k * full_pipe; + if (cur_k >= prob_k || cur_k >= slice_k_finish) { + return; + } + + // Reset (to current thread-block) since we read g_idx portion from the + // shared memory + cur_k = 0; + + // Progress to current iteration + cur_k += k_iter_size * (k % b_sh_wr_iters); + + // Determine "position" inside the thread-block (based on warp and + // thread-id) + auto warp_id = threadIdx.x / 32; + int n_warps = thread_n_blocks / 4; // Each warp processes 4 16-size tiles over N + + int warp_row = warp_id / n_warps; + int warp_col = warp_id % n_warps; + + cur_k += warp_row * 16; + + auto th_id = threadIdx.x % 32; + cur_k += (th_id % 4) * 2; // Due to tensor-core layout for fp16 B matrix + + int s_col_shift = + /*slice_n_offset +*/ (act_s_col_warp_stride * warp_col) + (th_id / 4) * act_s_col_stride; + + if (is_same_group[pipe]) { + if (k % 2 == 0) { + *(reinterpret_cast(&(act_frag_s[k % 2][0][0]))) = + sh_s[(same_group_id[pipe] - sh_first_group_id) * s_sh_stride + s_col_shift]; + } else { + *(reinterpret_cast(&(act_frag_s[k % 2][0][0]))) = + *(reinterpret_cast(&(act_frag_s[(k - 1) % 2][0][0]))); + } + + for (int i = 1; i < 4; i++) { + *(reinterpret_cast(&(act_frag_s[k % 2][i][0]))) = *(reinterpret_cast(&(act_frag_s[k % 2][0][0]))); + } + return; + } + + int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe; + int* sh_g_idx_int_ptr = reinterpret_cast(sh_g_idx_stage); + + constexpr int k_frag_offsets[4] = {0, 1, 8, 9}; // Tensor core offsets per thread + +#pragma unroll + for (int i = 0; i < 4; i++) { + int actual_k = cur_k + k_frag_offsets[i]; + + int group_id = sh_g_idx_int_ptr[actual_k]; + int rel_group_id = group_id - sh_first_group_id; + + *(reinterpret_cast(&(act_frag_s[k % 2][i][0]))) = sh_s[rel_group_id * s_sh_stride + s_col_shift]; + } + }; + + auto fetch_zp_to_registers = [&](int k, int full_pipe) { + // This code does not handle group_blocks == 0, + // which signifies act_order. + // has_zp implies AWQ, which doesn't have act_order, + static_assert(!has_zp || group_blocks != 0); + + if constexpr (has_zp && !is_zp_float) { + int pipe = full_pipe % stages; + + if constexpr (group_blocks == -1) { + // load only when starting a new slice + if (k == 0 && full_pipe == 0) { +#pragma unroll + for (int i = 0; i < num_ints_per_thread; i++) { + frag_qzp[k % 2][i] = (reinterpret_cast(sh_zp))[zp_sh_rd + i]; + } + } + + } else if constexpr (group_blocks >= thread_k_blocks) { + if (k % b_sh_wr_iters == 0) { + int4* sh_zp_stage = + sh_zp + zp_sh_stage * ((group_blocks / thread_k_blocks) * (pipe / (group_blocks / thread_k_blocks))); +#pragma unroll + for (int i = 0; i < num_ints_per_thread; i++) { + frag_qzp[k % 2][i] = (reinterpret_cast(sh_zp_stage))[zp_sh_rd + i]; + } + } + } else { + auto warp_id = threadIdx.x / 32; + int n_warps = thread_n_blocks / 4; + + int warp_row = warp_id / n_warps; + + int cur_k = warp_row * 16; + cur_k += k_iter_size * (k % b_sh_wr_iters); + + int k_blocks = cur_k / 16; + int cur_group_id = 0; + + // Suppress bogus and persistent divide-by-zero warning +#pragma nv_diagnostic push +#pragma nv_diag_suppress divide_by_zero + cur_group_id = k_blocks / group_blocks; +#pragma nv_diagnostic pop + + int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe; + + sh_zp_stage += cur_group_id * zp_sh_stride; + +#pragma unroll + for (int i = 0; i < num_ints_per_thread; i++) { + frag_qzp[k % 2][i] = (reinterpret_cast(sh_zp_stage))[zp_sh_rd + i]; + } + } + } + + else if constexpr (has_zp && is_zp_float) { + int pipe = full_pipe % stages; + + if constexpr (group_blocks != -1) { + if constexpr (group_blocks >= thread_k_blocks) { + if (k % b_sh_wr_iters == 0) { + int4* sh_zp_stage = + sh_zp + zp_sh_stage * ((group_blocks / thread_k_blocks) * (pipe / (group_blocks / thread_k_blocks))); + reinterpret_cast(&frag_zpf[k % 2])[0] = sh_zp_stage[zp_sh_rd]; + } + } else { + auto warp_id = threadIdx.x / 32; + int n_warps = thread_n_blocks / 4; + + int warp_row = warp_id / n_warps; + + int cur_k = warp_row * 16; + cur_k += k_iter_size * (k % b_sh_wr_iters); + + int k_blocks = cur_k / 16; + // Suppress bogus and persistent divide-by-zero warning +#pragma nv_diagnostic push +#pragma nv_diag_suppress divide_by_zero + int cur_group_id = k_blocks / group_blocks; +#pragma nv_diagnostic pop + + int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe; + + reinterpret_cast(&frag_zpf[k % 2])[0] = sh_zp_stage[zp_sh_rd + cur_group_id * zp_sh_stride]; + } + } + } + }; + + auto dequant_data = [&](int q, scalar_t2* frag_b_ptr) { + dequant(q, frag_b_ptr); + }; + + // Execute the actual tensor core matmul of a sub-tile. + bool is_first_matmul_in_slice = true; + auto matmul = [&](int k) { + int k2 = k % 2; + const bool is_new_zp = ((group_blocks != -1) && (group_blocks < thread_k_blocks || k == 0)) || + (group_blocks == -1 && is_first_matmul_in_slice); + if constexpr (has_zp && !is_zp_float) { + if (is_new_zp) { + if constexpr (group_blocks == -1) is_first_matmul_in_slice = false; + int zp_quant_0, zp_quant_1; + + if constexpr (w_type.size_bits() == 4) { + zp_quant_0 = frag_qzp[k2][0]; + zp_quant_1 = zp_quant_0 >> 8; + } else { + static_assert(w_type.size_bits() == 8); + zp_quant_0 = frag_qzp[k2][0]; + zp_quant_1 = frag_qzp[k2][1]; + } + + dequant_data(zp_quant_0, reinterpret_cast(&frag_zp)); + dequant_data(zp_quant_1, reinterpret_cast(&frag_zp) + 2); + } + } + if constexpr (!dequant_skip_flop && has_zp && is_zp_float) { + if (is_new_zp) { + reinterpret_cast(&frag_zp)[0] = reinterpret_cast(&frag_zpf[k2])[0]; + } + } + + // Commented out FP4/FP8 scale dequantization since we don't generate + // kFE2M1f kernels to reduce compilation time + // if constexpr (w_type == sglang::kFE2M1f) { + // int s_quant_0 = reinterpret_cast(frag_s[k2])[0]; + // int s_quant_1 = reinterpret_cast(frag_s[k2])[1]; + // + // dequant_fp8_scales( + // s_quant_0, reinterpret_cast(&frag_s[k2])); + // dequant_fp8_scales( + // s_quant_1, reinterpret_cast(&frag_s[k2]) + 2); + // } + +// We have the m dimension as the inner loop in order to encourage overlapping +// dequantization and matmul operations. +#pragma unroll + for (int j = 0; j < 4; j++) { + FragB frag_b0; + FragB frag_b1; + int b_quant_0, b_quant_1; + + if constexpr (w_type_id == sglang::kFE2M1f.id()) { + b_quant_1 = frag_b_quant[k2][0][j]; + b_quant_0 = b_quant_1 << 8; + } else if constexpr (w_type.size_bits() == 4) { + b_quant_0 = frag_b_quant[k2][0][j]; + b_quant_1 = b_quant_0 >> 8; + } else { + static_assert(w_type.size_bits() == 8); + int* frag_b_quant_ptr = reinterpret_cast(frag_b_quant[k2]); + b_quant_0 = frag_b_quant_ptr[j * 2 + 0]; + b_quant_1 = frag_b_quant_ptr[j * 2 + 1]; + } + + dequant_data(b_quant_0, reinterpret_cast(&frag_b0)); + dequant_data(b_quant_1, reinterpret_cast(&frag_b1)); + + if constexpr (dequant_skip_flop && has_zp && !is_zp_float) { + sub_zp(frag_b0, frag_zp[j], 0); + sub_zp(frag_b1, frag_zp[j], 1); + } + + // Apply scale to frag_b0 + if constexpr (has_act_order) { + static_assert(group_blocks != -1); + scale4( + frag_b0, act_frag_s[k2][0][j], act_frag_s[k2][1][j], act_frag_s[k2][2][j], act_frag_s[k2][3][j], 0); + scale4( + frag_b1, act_frag_s[k2][0][j], act_frag_s[k2][1][j], act_frag_s[k2][2][j], act_frag_s[k2][3][j], 1); + } else if constexpr (!dequant_skip_flop && has_zp && !is_zp_float && group_blocks == -1) { + int idx = (threadIdx.x / 4) % 2; + scalar_t2 s2 = Dtype::nums2num2( + reinterpret_cast(&frag_s[j / 2][j % 2 * 2 + 0])[idx], + reinterpret_cast(&frag_s[j / 2][j % 2 * 2 + 1])[idx]); + if (is_new_zp) frag_zp[j] = __hmul2(frag_zp[j], s2); + scale_and_sub(frag_b0, s2.x, frag_zp[j].x); + scale_and_sub(frag_b1, s2.y, frag_zp[j].y); + } else if constexpr (!dequant_skip_flop && has_zp && group_blocks != -1) { + if (is_new_zp) frag_zp[j] = __hmul2(frag_zp[j], *reinterpret_cast(&frag_s[k2][j])); + scale_and_sub(frag_b0, frag_s[k2][j][0].x, frag_zp[j].x); + scale_and_sub(frag_b1, frag_s[k2][j][0].y, frag_zp[j].y); + } else if constexpr (group_blocks != -1) { + scale(frag_b0, frag_s[k2][j], 0); + scale(frag_b1, frag_s[k2][j], 1); + } + +#pragma unroll + for (int i = 0; i < thread_m_blocks; i++) { + if constexpr (m_block_size_8) { + mma_trans(frag_a[k2][i], frag_b0, frag_b1, frag_c[i][j][0]); + } else { + mma(frag_a[k2][i], frag_b0, frag_c[i][j][0]); + mma(frag_a[k2][i], frag_b1, frag_c[i][j][1]); + } + } + } + }; + + // Since we slice across the k dimension of a tile in order to increase the + // number of warps while keeping the n dimension of a tile reasonable, we have + // multiple warps that accumulate their partial sums of the same output + // location; which we have to reduce over in the end. We do in shared memory. + auto thread_block_reduce = [&]() { + constexpr int red_off = threads / b_sh_stride_threads / 2; + if (red_off >= 1) { + auto red_idx = threadIdx.x / b_sh_stride_threads; + constexpr int red_sh_stride = b_sh_stride_threads * 4 * 2; + constexpr int red_sh_delta = b_sh_stride_threads; + int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride_threads) + (threadIdx.x % b_sh_stride_threads); + + // Parallel logarithmic shared memory reduction. We make sure to avoid any + // unnecessary read or write iterations, e.g., for two warps we write only + // once by warp 1 and read only once by warp 0. + +#pragma unroll + for (int m_block = 0; m_block < thread_m_blocks; m_block++) { +#pragma unroll + for (int i = red_off; i > 0; i /= 2) { + if (i <= red_idx && red_idx < 2 * i) { +#pragma unroll + for (int j = 0; j < 4 * 2; j += (m_block_size_8 ? 2 : 1)) { + int red_sh_wr = red_sh_delta * j + (red_sh_rd - red_sh_stride * i); + if (i < red_off) { + float* c_rd = reinterpret_cast(&sh_red[red_sh_delta * j + red_sh_rd]); + float* c_wr = reinterpret_cast(&sh_red[red_sh_wr]); +#pragma unroll + for (int k = 0; k < 4; k++) + reinterpret_cast(frag_c)[4 * 2 * m_block + j][k] += c_rd[k] + c_wr[k]; + } + sh_red[red_sh_wr] = reinterpret_cast(&frag_c)[4 * 2 * m_block + j]; + } + } + __syncthreads(); + } + if (red_idx == 0) { +#pragma unroll + for (int i = 0; i < 4 * 2; i += (m_block_size_8 ? 2 : 1)) { + float* c_rd = reinterpret_cast(&sh_red[red_sh_delta * i + red_sh_rd]); +#pragma unroll + for (int j = 0; j < 4; j++) + reinterpret_cast(frag_c)[4 * 2 * m_block + i][j] += c_rd[j]; + } + } + __syncthreads(); + } + } + }; + + // Since multiple threadblocks may process parts of the same column slice, we + // finally have to globally reduce over the results. As the striped + // partitioning minimizes the number of such reductions and our outputs are + // usually rather small, we perform this reduction serially in L2 cache. + auto global_reduce_fp16 = [&](bool first = false, bool last = false) { + // We are very careful here to reduce directly in the output buffer to + // maximize L2 cache utilization in this step. To do this, we write out + // results in FP16 (but still reduce with FP32 compute). + constexpr int active_threads = 32 * thread_n_blocks / 4; + bool is_th_active = threadIdx.x < active_threads; + if (!is_th_active) { + return; + } + + int c_gl_stride = prob_n / 8; + int c_gl_wr_delta_o = 8 * c_gl_stride; + int c_gl_wr_delta_i = 4 * (active_threads / 32); + int c_gl_wr; + if constexpr (m_block_size_8) { + c_gl_wr = c_gl_stride * ((threadIdx.x % 4) * 2) + 4 * (threadIdx.x / 32) + (threadIdx.x % 32) / 8; + c_gl_wr += (2 * thread_n_blocks) * slice_col; + } else { + c_gl_wr = c_gl_stride * ((threadIdx.x % 32) / 4) + 4 * (threadIdx.x / 32) + threadIdx.x % 4; + c_gl_wr += (2 * thread_n_blocks) * slice_col; + } + constexpr int c_sh_wr_delta = active_threads; + int c_sh_wr = threadIdx.x; + + if (!first) { + +#pragma unroll + for (int i = 0; i < (m_block_size_8 ? 2 : thread_m_blocks * 4); i++) { + int c_idx; + if constexpr (m_block_size_8) + c_idx = c_gl_wr + i * c_gl_stride + (threadIdx.x % 8) / 4 * c_gl_wr_delta_i; + else + c_idx = c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2); + if (c_idx / c_gl_stride < block_num_valid_tokens) { + int64_t sorted_row = sh_block_sorted_ids[c_idx / c_gl_stride]; + int64_t true_idx = sorted_row * c_gl_stride + c_idx % c_gl_stride; + sh_red[c_sh_wr + c_sh_wr_delta * i] = C[true_idx]; + } + } + } + +#pragma unroll + for (int i = 0; i < (m_block_size_8 ? 2 : thread_m_blocks * 4); i++) { + if (!first) { + int4 c_red = sh_red[c_sh_wr + i * c_sh_wr_delta]; +#pragma unroll + for (int j = 0; j < 2 * 4; j++) { + int delta = 0; + if constexpr (m_block_size_8) { + delta = j % 2 == 1 ? -2 : 0; + } + reinterpret_cast(&frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4) + delta] += + Dtype::num2float(reinterpret_cast(&c_red)[j]); + } + } + if (!last) { + int4 c; +#pragma unroll + for (int j = 0; j < 2 * 4; j++) { + int delta = 0; + if constexpr (m_block_size_8) { + delta = j % 2 == 1 ? -2 : 0; + } + reinterpret_cast(&c)[j] = + Dtype::float2num(reinterpret_cast(&frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4) + delta]); + } + + int c_idx; + if constexpr (m_block_size_8) + c_idx = c_gl_wr + i * c_gl_stride + (threadIdx.x % 8) / 4 * c_gl_wr_delta_i; + else + c_idx = c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2); + if (c_idx / c_gl_stride < block_num_valid_tokens) { + int64_t sorted_row = sh_block_sorted_ids[c_idx / c_gl_stride]; + int64_t true_idx = sorted_row * c_gl_stride + c_idx % c_gl_stride; + C[true_idx] = c; + } + } + } + }; + + // Globally reduce over threadblocks that compute the same column block. + // We use a tmp C buffer to reduce in full fp32 precision. + auto global_reduce_fp32 = [&](bool first = false, bool last = false) { + constexpr int tb_m = thread_m_blocks * 16; + constexpr int tb_n = thread_n_blocks * 16; + + constexpr int c_size = tb_m * tb_n * sizeof(float) / 16; + + constexpr int active_threads = 32 * thread_n_blocks / 4; + bool is_th_active = threadIdx.x < active_threads; + + constexpr int num_floats = thread_m_blocks * 4 * 2 * 4; + constexpr int th_size = num_floats * sizeof(float) / 16; + + int c_cur_offset = locks_off * c_size; + + if (!is_th_active) { + return; + } + + if (!first) { + float* frag_c_ptr = reinterpret_cast(&frag_c); +#pragma unroll + for (int k = 0; k < th_size; k++) { + if constexpr (m_block_size_8) { + if (k % 2) continue; + } else { + if (k / 8 * 16 + (threadIdx.x % 32) / 4 >= block_num_valid_tokens) continue; + } + + sh_red[threadIdx.x] = C_tmp[c_cur_offset + active_threads * k + threadIdx.x]; + + float* sh_c_ptr = reinterpret_cast(&sh_red[threadIdx.x]); +#pragma unroll + for (int f = 0; f < 4; f++) { + frag_c_ptr[k * 4 + f] += sh_c_ptr[f]; + } + } + } + + if (!last) { + int4* frag_c_ptr = reinterpret_cast(&frag_c); +#pragma unroll + for (int k = 0; k < th_size; k++) { + if constexpr (m_block_size_8) { + if (k % 2) continue; + } else { + if (k / 8 * 16 + (threadIdx.x % 32) / 4 >= block_num_valid_tokens) continue; + } + + C_tmp[c_cur_offset + active_threads * k + threadIdx.x] = frag_c_ptr[k]; + } + } + }; + + // Write out the reduce final result in the correct layout. We only actually + // reshuffle matrix fragments in this step, the reduction above is performed + // in fragment layout. + auto write_result = [&](bool last) { + int c_gl_stride = prob_n / 8; + constexpr int c_sh_stride = 2 * thread_n_blocks + 1; + int c_gl_wr_delta = c_gl_stride * (threads / (2 * thread_n_blocks)); + constexpr int c_sh_rd_delta = c_sh_stride * (threads / (2 * thread_n_blocks)); + + int c_gl_wr = c_gl_stride * (threadIdx.x / (2 * thread_n_blocks)) + (threadIdx.x % (2 * thread_n_blocks)); + c_gl_wr += (2 * thread_n_blocks) * slice_col; + int c_sh_wr; + if constexpr (m_block_size_8) { + c_sh_wr = (8 * c_sh_stride) * ((threadIdx.x % 32) % 4 * 2) + (threadIdx.x % 32) / 4; + c_sh_wr += 64 * (threadIdx.x / 32); + } else { + c_sh_wr = (4 * c_sh_stride) * ((threadIdx.x % 32) / 4) + (threadIdx.x % 32) % 4; + c_sh_wr += 32 * (threadIdx.x / 32); + } + + int c_sh_rd = c_sh_stride * (threadIdx.x / (2 * thread_n_blocks)) + (threadIdx.x % (2 * thread_n_blocks)); + + // We first reorder in shared memory to guarantee the most efficient final + // global write patterns + auto write = [&](int idx, float c0, float c1, FragS& s, FragS& b_bias) { + scalar_t2 res = Dtype::nums2num2(Dtype::float2num(c0), Dtype::float2num(c1)); + + // For per-column quantization we finally apply the scale here (only for + // 4-bit) + if constexpr ( + !has_act_order && group_blocks == -1 && w_type.size_bits() == 4 && (has_zp && dequant_skip_flop || !has_zp)) { + scalar_t2 tmp_scale = s[0]; + if constexpr (m_block_size_8) { + tmp_scale = Dtype::num2num2(reinterpret_cast(&s[0])[(threadIdx.x % 8) / 4]); + } + res = __hmul2(res, tmp_scale); + } + + if constexpr (w_type == sglang::kFE2M1f && s_type == sglang::kFE4M3fn) { + if (!mul_topk_weights) { + res = __hmul2(res, global_scale); + } + } + if (has_bias && last) { + scalar_t2 tmp_bias = b_bias[0]; + if constexpr (m_block_size_8) { + tmp_bias = Dtype::num2num2(reinterpret_cast(&b_bias[0])[(threadIdx.x % 8) / 4]); + } + res = __hadd2(res, tmp_bias); + } + + if constexpr (m_block_size_8) { + ((scalar_t*)sh_red)[idx] = res.x; + ((scalar_t*)sh_red)[idx + 8 * c_sh_stride] = res.y; + } else { + ((scalar_t2*)sh_red)[idx] = res; + } + }; + + if (threadIdx.x / 32 < thread_n_blocks / 4) { +#pragma unroll + for (int i = 0; i < thread_m_blocks; i++) { +#pragma unroll + for (int j = 0; j < 4; j++) { + if constexpr (m_block_size_8) { + int wr = c_sh_wr + 16 * j; + write( + wr, + frag_c[i][j][0][0], + frag_c[i][j][0][1], + frag_s[j / 2][2 * (j % 2) + 0], + frag_bias[j / 2][2 * (j % 2) + 0]); + write( + wr + 8, + frag_c[i][j][0][2], + frag_c[i][j][0][3], + frag_s[j / 2][2 * (j % 2) + 1], + frag_bias[j / 2][2 * (j % 2) + 1]); + } else { + int wr = c_sh_wr + 8 * j; + write( + wr + (4 * c_sh_stride) * 0 + 0, + frag_c[i][j][0][0], + frag_c[i][j][0][1], + frag_s[j / 2][2 * (j % 2) + 0], + frag_bias[j / 2][2 * (j % 2) + 0]); + write( + wr + (4 * c_sh_stride) * 8 + 0, + frag_c[i][j][0][2], + frag_c[i][j][0][3], + frag_s[j / 2][2 * (j % 2) + 0], + frag_bias[j / 2][2 * (j % 2) + 0]); + write( + wr + (4 * c_sh_stride) * 0 + 4, + frag_c[i][j][1][0], + frag_c[i][j][1][1], + frag_s[j / 2][2 * (j % 2) + 1], + frag_bias[j / 2][2 * (j % 2) + 1]); + write( + wr + (4 * c_sh_stride) * 8 + 4, + frag_c[i][j][1][2], + frag_c[i][j][1][3], + frag_s[j / 2][2 * (j % 2) + 1], + frag_bias[j / 2][2 * (j % 2) + 1]); + } + } + c_sh_wr += 16 * (4 * c_sh_stride); + } + } + __syncthreads(); + +#pragma unroll + for (int i = 0; i < div_ceil(16 * thread_m_blocks, threads / (2 * thread_n_blocks)); i++) { + int row = c_gl_wr / c_gl_stride; + if (row < block_num_valid_tokens) { + int64_t sorted_row = sh_block_sorted_ids[row]; + int64_t true_idx = sorted_row * c_gl_stride + c_gl_wr % c_gl_stride; + scalar_t2 topk_weight_score; + if (mul_topk_weights) topk_weight_score = sh_block_topk_weights[row]; + if (use_atomic_add && slice_count > 1 || mul_topk_weights) { + scalar_t2* C_half2 = reinterpret_cast(&C[true_idx]); + scalar_t2* sh_red_half2 = reinterpret_cast(&sh_red[c_sh_rd]); +#pragma unroll + for (int a = 0; a < 4; a++) { + scalar_t2 res = sh_red_half2[a]; + if (mul_topk_weights) { + res = __hmul2(res, topk_weight_score); + } + + if (use_atomic_add && slice_count > 1) { + atomicAdd(&C_half2[a], res); + } else { + C_half2[a] = res; + }; + } + } else { + C[true_idx] = sh_red[c_sh_rd]; + } + c_gl_wr += c_gl_wr_delta; + c_sh_rd += c_sh_rd_delta; + } + } + __syncthreads(); + }; + + // Start global fetch and register load pipelines. + auto start_pipes = [&]() { + +#pragma unroll + for (int i = 0; i < stages - 1; i++) { + if (has_act_order && i == 0) { + int last_g_idx = slice_k_start + stages * tb_k * 2; + if (last_g_idx >= prob_k) { + last_g_idx = prob_k - 1; + } + fetch_act_order_scales_to_shared(true, g_idx[slice_k_start], g_idx[last_g_idx]); + } + + if constexpr (has_zp && !is_zp_float && group_blocks == -1) { + if (i == 0) { + fetch_col_zp_to_shared(); + if constexpr (!dequant_skip_flop) { + fetch_col_scale_to_shared(); + } + } + } + fetch_to_shared(i, i, i < slice_iters, i); + } + + zero_accums(); + wait_for_stage(); + init_same_group(0); + fetch_to_registers(0, 0); + fetch_scales_to_registers(0, 0); + fetch_zp_to_registers(0, 0); + a_gl_rd_col += a_gl_rd_delta_o * (stages - 1); + if constexpr (has_act_order) { + slice_k_start_shared_fetch += tb_k * (stages - 1); + } + }; + if (slice_iters) { + start_pipes(); + } + + // Main loop. + while (slice_iters) { + // We unroll over both the global fetch and the register load pipeline to + // ensure all shared memory accesses are static. Note that both pipelines + // have even length meaning that the next iteration will always start at + // index 0. + + for (int stage_group_id = 0; stage_group_id < max_num_stage_groups; stage_group_id++) { +#pragma unroll + for (int pipe = 0; pipe < stages;) { +#pragma unroll + for (int k = 0; k < b_sh_wr_iters; k++) { + int idx = (pipe >= stages && stage_group_id == max_num_stage_groups - 1) ? (pipe - stages) + : (pipe + stage_group_id * stages); + fetch_to_registers(k + 1, pipe % stages, idx); + fetch_scales_to_registers(k + 1, pipe); + fetch_zp_to_registers(k + 1, pipe); + if (k == b_sh_wr_iters - 2) { + int idx = (pipe >= 1 && stage_group_id == max_num_stage_groups - 1) + ? (pipe - 1) + : (pipe + (stage_group_id + 1) * stages - 1); + fetch_to_shared((pipe + stages - 1) % stages, pipe, slice_iters >= stages, idx); + pipe++; + wait_for_stage(); + init_same_group(pipe % stages); + } + matmul(k); + } + slice_iters--; + if (slice_iters == 0) { + break; + } + } + + a_gl_rd_col += a_gl_rd_delta_o * stages; + + if constexpr (has_act_order) { + slice_k_start += tb_k * stages; + + if (slice_k_start < prob_k) { + slice_k_start_shared_fetch += tb_k * stages; + int first_group_id = g_idx[slice_k_start]; + int last_g_idx = slice_k_start + stages * tb_k * 2; + if (last_g_idx >= prob_k) { + last_g_idx = prob_k - 1; + } + int last_group_id = g_idx[last_g_idx]; + if (last_group_id >= sh_first_group_id + sh_num_groups) { + fetch_act_order_scales_to_shared(false, first_group_id, last_group_id); + __syncthreads(); + } + } + } + if (slice_iters == 0) { + break; + } + } + + // Process results and, if necessary, proceed to the next column slice. + // While this pattern may not be the most readable, other ways of writing + // the loop seemed to noticeably worse performance after compilation. + if (slice_iters == 0) { + cp_async_wait<0>(); + bool last = slice_idx == slice_count - 1; + // For per-column scales, we only fetch them here in the final step before + // write-out + if constexpr (!has_act_order && group_blocks == -1 && (has_zp && dequant_skip_flop || !has_zp)) { + if (w_type.size_bits() == 8 || (last || use_atomic_add)) { + if (s_sh_wr_pred) { + cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]); + } + cp_async_fence(); + } + } + + thread_block_reduce(); + + if (has_bias && last) { + __syncthreads(); + cp_async4_pred(&sh_bias[bias_sh_wr], &b_bias_ptr[bias_gl_rd], threadIdx.x < 16 * thread_n_blocks / 8); + cp_async_fence(); + } + + if constexpr (!has_act_order && group_blocks == -1 && (has_zp && dequant_skip_flop || !has_zp)) { + if (w_type.size_bits() == 8 || (last || use_atomic_add)) { + cp_async_wait<0>(); + __syncthreads(); + if (threadIdx.x / 32 < thread_n_blocks / 4) { + reinterpret_cast(&frag_s)[0] = sh_s[s_sh_rd + 0]; + reinterpret_cast(&frag_s)[1] = sh_s[s_sh_rd + 4]; + if constexpr (m_block_size_8) { + int idx = (threadIdx.x / 4) % 2; + scalar_t2* frag_s_half2 = reinterpret_cast(frag_s); +#pragma unroll + for (int i = 0; i < 8; i++) { + frag_s_half2[i] = Dtype::num2num2(reinterpret_cast(&frag_s_half2[i])[idx]); + } + } + } + } + } + + // For 8-bit channelwise, we apply the scale before the global reduction + // that converts the fp32 results to fp16 (so that we avoid possible + // overflow in fp16) + if constexpr ( + !has_act_order && group_blocks == -1 && w_type.size_bits() == 8 && (has_zp && dequant_skip_flop || !has_zp)) { + if (threadIdx.x / 32 < thread_n_blocks / 4) { +#pragma unroll + for (int i = 0; i < thread_m_blocks; i++) { +#pragma unroll + for (int j = 0; j < 4; j++) { + scale_float(reinterpret_cast(&frag_c[i][j][0][0]), frag_s[j / 2][2 * (j % 2) + 0]); + scale_float( + reinterpret_cast(&frag_c[i][j][0][2]), frag_s[j / 2][2 * (j % 2) + (m_block_size_8 ? 1 : 0)]); + + if constexpr (!m_block_size_8) { + scale_float(reinterpret_cast(&frag_c[i][j][1][0]), frag_s[j / 2][2 * (j % 2) + 1]); + scale_float(reinterpret_cast(&frag_c[i][j][1][2]), frag_s[j / 2][2 * (j % 2) + 1]); + } + } + } + } + } + + if (slice_count > 1 && !use_atomic_add) { + // only globally reduce if there is more than one block in a slice + barrier_acquire(&locks[locks_off], slice_idx); + if (use_fp32_reduce) { + global_reduce_fp32(slice_idx == 0, last); + } else { + global_reduce_fp16(slice_idx == 0, last); + } + barrier_release(&locks[locks_off], last); + } + + if (has_bias && last) { + cp_async_wait<0>(); + __syncthreads(); + reinterpret_cast(&frag_bias)[0] = sh_bias[bias_sh_rd]; + reinterpret_cast(&frag_bias)[1] = sh_bias[bias_sh_rd + 4]; + __syncthreads(); + } + + if (use_atomic_add && slice_count > 1 && slice_idx != 0) wait_negative_and_add(&locks[locks_off]); + if (last || use_atomic_add) + // only the last block in a slice actually writes the result + write_result(last); + int old_slice_row = slice_row; + slice_row = 0; + slice_col_par++; + slice_col++; + is_first_matmul_in_slice = true; + init_slice(); + + // Should we load A matrix in next slice? + // `slice_col == 0`: when move to a new moe block + // `old_slice_row > 0`: + // when the last slice is not starting from k_index == 0 + // (only happen when it is the first slice of a threadblock) + // `prob_k > thread_k_blocks * 16 * stages * max_num_stage_groups`: + // when the required shared memory size is larger than + // the remaining shared memory + if (slice_col == 0 || old_slice_row || prob_k > thread_k_blocks * 16 * stages * max_num_stage_groups) { + should_load_a = true; + } else { + should_load_a = false; + } + + if (slice_iters) { + a_gl_rd_col = (threadIdx.x % a_gl_rd_delta_o); +#pragma unroll + for (int i = 0; i < b_sh_wr_iters; i++) + B_ptr[i] += b_sh_stride - b_gl_rd_delta_o * k_tiles; + if (slice_col == 0) { +#pragma unroll + for (int i = 0; i < b_sh_wr_iters; i++) + B_ptr[i] -= b_gl_stride; + } + + bias_gl_rd = (thread_n_blocks * 16 / 8) * slice_col + threadIdx.x; + // Update slice k/n for scales loading + if constexpr (has_act_order) { + slice_k_start = tb_k * slice_row; + slice_k_finish = slice_k_start + tb_k * slice_iters; + slice_k_start_shared_fetch = slice_k_start; + slice_n_offset = act_s_col_tb_stride * slice_col; + } else { + s_gl_rd = s_sh_stride * slice_col + threadIdx.x; + zp_gl_rd = zp_sh_stride * slice_col + threadIdx.x; + } + start_pipes(); + } + } + } +} + +} // namespace MARLIN_NAMESPACE_NAME + +#endif diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/ops.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/ops.cu new file mode 100644 index 00000000..57334663 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/marlin_moe_wna16/ops.cu @@ -0,0 +1,1237 @@ +/* + * Modified by Neural Magic + * Copyright (C) Marlin.2024 Elias Frantar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Adapted from https://github.com/IST-DASLab/marlin + */ + +#ifndef MARLIN_NAMESPACE_NAME +#define MARLIN_NAMESPACE_NAME marlin_moe_wna16 +#endif + +#include "kernel.h" +#include "kernel_marlin.cuh" +#include "marlin_template.h" + +#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \ + static_assert( \ + std::is_same::value || std::is_same::value, \ + "only float16 and bfloat16 is supported"); + +namespace MARLIN_NAMESPACE_NAME { + +__global__ void MarlinDefault(MARLIN_KERNEL_PARAMS){}; + +using MarlinFuncPtr = void (*)(MARLIN_KERNEL_PARAMS); + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800 + +template +__global__ void permute_cols_kernel( + int4 const* __restrict__ a_int4_ptr, + int const* __restrict__ perm_int_ptr, + int4* __restrict__ out_int4_ptr, + const int32_t* __restrict__ sorted_token_ids_ptr, + const int32_t* __restrict__ expert_ids_ptr, + const int32_t* __restrict__ num_tokens_past_padded_ptr, + int size_m, + int size_k, + int top_k) {}; + +} // namespace marlin + +torch::Tensor moe_wna16_marlin_gemm( + torch::Tensor& a, + std::optional c_or_none, + torch::Tensor& b_q_weight, + std::optional const& b_bias_or_none, + torch::Tensor& b_scales, + std::optional const& global_scale_or_none, + std::optional const& b_zeros_or_none, + std::optional const& g_idx_or_none, + std::optional const& perm_or_none, + torch::Tensor& workspace, + torch::Tensor& sorted_token_ids, + torch::Tensor& expert_ids, + torch::Tensor& num_tokens_past_padded, + torch::Tensor& topk_weights, + int64_t moe_block_size, + int64_t top_k, + bool mul_topk_weights, + bool is_ep, + sglang::ScalarTypeId const& b_q_type_id, + int64_t size_m, + int64_t size_n, + int64_t size_k, + bool is_k_full, + bool use_atomic_add, + bool use_fp32_reduce, + bool is_zp_float) { + TORCH_CHECK_NOT_IMPLEMENTED(false, "marlin_gemm(..) requires CUDA_ARCH >= 8.0"); + return torch::empty({1, 1}); +} + +#else + +// For a given "a" of size [M,K] performs a permutation of the K columns based +// on the given "perm" indices. +template +__global__ void permute_cols_kernel( + int4 const* __restrict__ a_int4_ptr, + int const* __restrict__ perm_int_ptr, + int4* __restrict__ out_int4_ptr, + const int32_t* __restrict__ sorted_token_ids_ptr, + const int32_t* __restrict__ expert_ids_ptr, + const int32_t* __restrict__ num_tokens_past_padded_ptr, + int size_m, + int size_k, + int top_k) { + int num_tokens_past_padded = num_tokens_past_padded_ptr[0]; + int num_moe_blocks = div_ceil(num_tokens_past_padded, moe_block_size); + int32_t block_sorted_ids[moe_block_size]; + int block_num_valid_tokens = 0; + int64_t old_expert_id = 0; + int64_t expert_id = 0; + int row_stride = size_k * sizeof(half) / 16; + + auto read_moe_block_data = [&](int block_id) { + block_num_valid_tokens = moe_block_size; + int4* tmp_block_sorted_ids = reinterpret_cast(block_sorted_ids); + for (int i = 0; i < moe_block_size / 4; i++) { + tmp_block_sorted_ids[i] = ((int4*)sorted_token_ids_ptr)[block_id * moe_block_size / 4 + i]; + } + for (int i = 0; i < moe_block_size; i++) { + if (block_sorted_ids[i] >= size_m * top_k) { + block_num_valid_tokens = i; + break; + }; + } + }; + + auto permute_row = [&](int row) { + int iters = size_k / default_threads; + int rest = size_k % default_threads; + + int in_offset = (row / top_k) * row_stride; + int out_offset = row * row_stride; + + half const* a_row_half = reinterpret_cast(a_int4_ptr + in_offset); + half* out_half = reinterpret_cast(out_int4_ptr + out_offset); + + int base_k = 0; + + for (int i = 0; i < iters; i++) { + auto cur_k = base_k + threadIdx.x; + int src_pos = perm_int_ptr[cur_k]; + + out_half[cur_k] = a_row_half[src_pos]; + + base_k += default_threads; + } + + if (rest) { + if (threadIdx.x < rest) { + auto cur_k = base_k + threadIdx.x; + int src_pos = perm_int_ptr[cur_k]; + + out_half[cur_k] = a_row_half[src_pos]; + } + } + }; + + for (int index = blockIdx.x; index < num_moe_blocks; index += gridDim.x) { + old_expert_id = expert_id; + int tmp_expert_id = expert_ids_ptr[index]; + if (tmp_expert_id == -1) continue; + expert_id = tmp_expert_id; + perm_int_ptr += (expert_id - old_expert_id) * size_k; + read_moe_block_data(index); + + for (int i = 0; i < block_num_valid_tokens; i++) + permute_row(block_sorted_ids[i]); + } +} + +typedef struct { + int thread_k; + int thread_n; + int num_threads; +} thread_config_t; + +thread_config_t small_batch_thread_configs[] = { + // Ordered by priority + + // thread_k, thread_n, num_threads + {128, 128, 256}, + {64, 128, 128}}; + +thread_config_t large_batch_thread_configs[] = { + // Ordered by priority + + // thread_k, thread_n, num_threads + {64, 256, 256}, + {64, 128, 128}}; + +typedef struct { + int blocks_per_sm; + thread_config_t tb_cfg; +} exec_config_t; + +int get_scales_cache_size( + thread_config_t const& th_config, + int prob_m, + int prob_n, + int prob_k, + int num_bits, + int group_size, + bool has_act_order, + bool is_k_full) { + bool cache_scales_chunk = has_act_order && !is_k_full; + + int tb_n = th_config.thread_n; + int tb_k = th_config.thread_k; + + // Get max scale groups per thread-block + int tb_groups; + if (group_size == -1) { + tb_groups = 1; + } else if (group_size == 0) { + tb_groups = div_ceil(tb_k, 32); // Worst case is 32 group size + } else { + tb_groups = div_ceil(tb_k, group_size); + } + + if (cache_scales_chunk) { + int load_groups = tb_groups * pipe_stages * 2; // Chunk size is 2x pipeline over dim K + load_groups = max(load_groups, 32); // We load at least 32 scale groups + return load_groups * tb_n * 2; + } else { + int tb_scales = tb_groups * tb_n * 2; + + return tb_scales * pipe_stages; + } +} + +int get_kernel_cache_size( + thread_config_t const& th_config, + bool m_block_size_8, + int thread_m_blocks, + int prob_m, + int prob_n, + int prob_k, + int num_bits, + int group_size, + bool has_act_order, + bool is_k_full, + int has_zp, + int is_zp_float) { + int pack_factor = 32 / num_bits; + + // Get B size + int tb_k = th_config.thread_k; + int tb_n = th_config.thread_n; + int tb_m = thread_m_blocks * 16; + + // shm size for block_sorted_ids/rd_block_sorted_ids/block_topk_weights + // both of them requires tb_m * 4 bytes (tb_m * int32 or tb_m * float32) + int sh_block_meta_size = tb_m * 4; + int sh_a_size = pipe_stages * (tb_m * tb_k) * 2; + int sh_b_size = pipe_stages * (tb_k * tb_n / pack_factor) * 4; + int sh_red_size = tb_m * (tb_n + 8) * 2; + int sh_bias_size = tb_n * 2; + int tmp_size = (sh_b_size > sh_red_size ? sh_red_size : sh_b_size) + sh_bias_size; + tmp_size = max(max(sh_b_size, sh_red_size), tmp_size); + + int sh_s_size = + get_scales_cache_size(th_config, prob_m, prob_n, prob_k, num_bits, group_size, has_act_order, is_k_full); + int sh_g_idx_size = has_act_order && !is_k_full ? pipe_stages * tb_k / 4 : 0; + int sh_zp_size = 0; + if (has_zp) { + if (is_zp_float) + sh_zp_size = sh_s_size; + else if (num_bits == 4) + sh_zp_size = sh_s_size / 4; + else if (num_bits == 8) + sh_zp_size = sh_s_size / 2; + } + + int total_size = tmp_size + sh_a_size + sh_s_size + sh_zp_size + sh_g_idx_size + sh_block_meta_size; + + return total_size; +} + +bool is_valid_config( + thread_config_t const& th_config, + bool m_block_size_8, + int thread_m_blocks, + int prob_m, + int prob_n, + int prob_k, + int num_bits, + int group_size, + bool has_act_order, + bool is_k_full, + int has_zp, + int is_zp_float, + int max_shared_mem) { + // Sanity + if (th_config.thread_k == -1 || th_config.thread_n == -1 || th_config.num_threads == -1) { + return false; + } + + // Verify K/N are divisible by thread K/N + if (prob_k % th_config.thread_k != 0 || prob_n % th_config.thread_n != 0) { + return false; + } + + // Verify min for thread K/N + if (th_config.thread_n < min_thread_n || th_config.thread_k < min_thread_k) { + return false; + } + + // num_threads must be at least 128 (= 4 warps) + if (th_config.num_threads < 128) { + return false; + } + + // Check that pipeline fits into cache + int cache_size = get_kernel_cache_size( + th_config, + m_block_size_8, + thread_m_blocks, + prob_m, + prob_n, + prob_k, + num_bits, + group_size, + has_act_order, + is_k_full, + has_zp, + is_zp_float); + return cache_size + 512 <= max_shared_mem; +} + +#define _GET_IF( \ + W_TYPE, THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, M_BLOCK_SIZE_8, GROUP_BLOCKS, NUM_THREADS, IS_ZP_FLOAT) \ + else if ( \ + q_type == W_TYPE && thread_m_blocks == THREAD_M_BLOCKS && thread_n_blocks == THREAD_N_BLOCKS && \ + thread_k_blocks == THREAD_K_BLOCKS && m_block_size_8 == M_BLOCK_SIZE_8 && group_blocks == GROUP_BLOCKS && \ + num_threads == NUM_THREADS && is_zp_float == IS_ZP_FLOAT) { \ + constexpr auto S_TYPE = W_TYPE == sglang::kFE2M1f \ + ? (GROUP_BLOCKS == 1 ? sglang::kFE4M3fn : sglang::kFE8M0fnu) \ + : (std::is_same::value ? sglang::kFloat16 : sglang::kBFloat16); \ + kernel = Marlin< \ + scalar_t, \ + W_TYPE.id(), \ + S_TYPE.id(), \ + NUM_THREADS, \ + THREAD_M_BLOCKS, \ + THREAD_N_BLOCKS, \ + THREAD_K_BLOCKS, \ + M_BLOCK_SIZE_8, \ + pipe_stages, \ + GROUP_BLOCKS, \ + IS_ZP_FLOAT>; \ + } + +// COMMON: cases for (group_blocks in [-1, 2, 4, 8] and is_zp_float == false) +// this is the most common cases +// BIGGROUP: cases for big group size (group_blocks in [-1, 8]) +// FZP: cases for float-zero-point (is_zp_float = true) +// ACT: cases for act order case (group_blocks == 0) +// FP4: cases for nvfp4(e2m1) (group_blocks == 1) +#define COMMON_GET_IF_M1(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 2, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 4, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 8, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) + +#define COMMON_GET_IF_M234(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) \ + \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) \ + \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) + +#define COMMON_GET_IF(W_TYPE) \ + COMMON_GET_IF_M1(W_TYPE, 8, 8, 256) \ + COMMON_GET_IF_M1(W_TYPE, 8, 4, 128) \ + COMMON_GET_IF_M234(W_TYPE, 16, 4, 256) \ + COMMON_GET_IF_M234(W_TYPE, 8, 4, 128) + +#define BIGGROUP_GET_IF_M1(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 8, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) + +#define BIGGROUP_GET_IF_M234(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) + +#define BIGGROUP_GET_IF(W_TYPE) \ + BIGGROUP_GET_IF_M1(W_TYPE, 8, 8, 256) \ + BIGGROUP_GET_IF_M1(W_TYPE, 8, 4, 128) \ + BIGGROUP_GET_IF_M234(W_TYPE, 16, 4, 256) \ + BIGGROUP_GET_IF_M234(W_TYPE, 8, 4, 128) + +#define NVFP4_GET_IF_M1(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 1, NUM_THREADS, false) + +#define NVFP4_GET_IF_M234(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 1, NUM_THREADS, false) + +#define NVFP4_GET_IF(W_TYPE) \ + NVFP4_GET_IF_M1(W_TYPE, 8, 8, 256) \ + NVFP4_GET_IF_M1(W_TYPE, 8, 4, 128) \ + NVFP4_GET_IF_M234(W_TYPE, 16, 4, 256) \ + NVFP4_GET_IF_M234(W_TYPE, 8, 4, 128) + +#define MXFP4_GET_IF_M1(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 2, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS, false) + +#define MXFP4_GET_IF_M234(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS, false) + +#define MXFP4_GET_IF(W_TYPE) \ + MXFP4_GET_IF_M1(W_TYPE, 8, 8, 256) \ + MXFP4_GET_IF_M1(W_TYPE, 8, 4, 128) \ + MXFP4_GET_IF_M234(W_TYPE, 16, 4, 256) \ + MXFP4_GET_IF_M234(W_TYPE, 8, 4, 128) + +// We currently have 4-bit models only with group_blocks == 4 +#define FZP_GET_IF_M1(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 4, NUM_THREADS, true) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, true) + +#define FZP_GET_IF_M234(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, true) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, true) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, true) + +#define FZP_GET_IF(W_TYPE) \ + FZP_GET_IF_M1(W_TYPE, 8, 8, 256) \ + FZP_GET_IF_M1(W_TYPE, 8, 4, 128) \ + FZP_GET_IF_M234(W_TYPE, 16, 4, 256) \ + FZP_GET_IF_M234(W_TYPE, 8, 4, 128) + +// We currently have 4-bit models only with group_blocks == 4 +#define ACT_GET_IF_M1(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 0, NUM_THREADS, false) + +#define ACT_GET_IF_M234(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 0, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 0, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 0, NUM_THREADS, false) + +#define ACT_GET_IF(W_TYPE) \ + ACT_GET_IF_M1(W_TYPE, 8, 8, 256) \ + ACT_GET_IF_M1(W_TYPE, 8, 4, 128) \ + ACT_GET_IF_M234(W_TYPE, 16, 4, 256) \ + ACT_GET_IF_M234(W_TYPE, 8, 4, 128) + +template +MarlinFuncPtr get_marlin_kernel( + const sglang::ScalarType q_type, + int thread_m_blocks, + int thread_n_blocks, + int thread_k_blocks, + bool m_block_size_8, + bool has_act_order, + bool has_zp, + int group_blocks, + int num_threads, + bool is_zp_float) { + int num_bits = q_type.size_bits(); + auto kernel = MarlinDefault; + if (false) { + } + + COMMON_GET_IF(sglang::kU4) + COMMON_GET_IF(sglang::kU4B8) + COMMON_GET_IF(sglang::kU8B128) + + NVFP4_GET_IF(sglang::kFE2M1f) + + BIGGROUP_GET_IF(sglang::kFE4M3fn) + + ACT_GET_IF(sglang::kU4B8) + ACT_GET_IF(sglang::kU8B128) + if (std::is_same::value) { + if (false) { + } + MXFP4_GET_IF(sglang::kFE2M1f) + } + + return kernel; +} + +template +exec_config_t determine_exec_config( + const sglang::ScalarType& q_type, + int prob_m, + int prob_n, + int prob_k, + int thread_m_blocks, + bool m_block_size_8, + int num_bits, + int group_size, + bool has_act_order, + bool is_k_full, + bool has_zp, + bool is_zp_float, + int max_shared_mem) { + exec_config_t exec_cfg = exec_config_t{1, thread_config_t{-1, -1, -1}}; + thread_config_t* thread_configs = thread_m_blocks > 1 ? large_batch_thread_configs : small_batch_thread_configs; + int thread_configs_size = thread_m_blocks > 1 ? sizeof(large_batch_thread_configs) / sizeof(thread_config_t) + : sizeof(small_batch_thread_configs) / sizeof(thread_config_t); + + int count = 0; + constexpr int device_max_reg_size = 255 * 1024; + for (int i = 0; i < thread_configs_size; i++) { + thread_config_t th_config = thread_configs[i]; + + if (!is_valid_config( + th_config, + m_block_size_8, + thread_m_blocks, + prob_m, + prob_n, + prob_k, + num_bits, + group_size, + has_act_order, + is_k_full, + has_zp, + is_zp_float, + max_shared_mem)) { + continue; + } + + int cache_size = get_kernel_cache_size( + th_config, + m_block_size_8, + thread_m_blocks, + prob_m, + prob_n, + prob_k, + num_bits, + group_size, + has_act_order, + is_k_full, + has_zp, + is_zp_float); + + int group_blocks = 0; + if (!has_act_order) { + group_blocks = group_size == -1 ? -1 : (group_size / 16); + } + + auto kernel = get_marlin_kernel( + q_type, + thread_m_blocks, + th_config.thread_n / 16, + th_config.thread_k / 16, + m_block_size_8, + has_act_order, + has_zp, + group_blocks, + th_config.num_threads, + is_zp_float); + + if (kernel == MarlinDefault) continue; + + if (thread_m_blocks > 1) { + exec_cfg = {1, th_config}; + break; + } else { + cudaFuncAttributes attr; + cudaFuncGetAttributes(&attr, kernel); + int reg_size = max(attr.numRegs, 1) * th_config.num_threads * 4; + int allow_count = min(device_max_reg_size / reg_size, max_shared_mem / (cache_size + 1024)); + allow_count = max(min(allow_count, 4), 1); + if (allow_count > count) { + count = allow_count; + exec_cfg = {count, th_config}; + }; + } + } + + return exec_cfg; +} + +template +void marlin_mm( + const void* A, + const void* B, + void* C, + void* C_tmp, + void* b_bias, + void* s, + void* s2, + void* zp, + void* g_idx, + void* perm, + void* a_tmp, + void* sorted_token_ids, + void* expert_ids, + void* num_tokens_past_padded, + void* topk_weights, + int moe_block_size, + int top_k, + bool mul_topk_weights, + bool is_ep, + int prob_m, + int prob_n, + int prob_k, + void* workspace, + sglang::ScalarType const& q_type, + bool has_bias, + bool has_act_order, + bool is_k_full, + bool has_zp, + int num_groups, + int group_size, + int dev, + cudaStream_t stream, + int thread_k, + int thread_n, + int sms, + bool use_atomic_add, + bool use_fp32_reduce, + bool is_zp_float) { + int thread_m_blocks = div_ceil(moe_block_size, 16); + bool m_block_size_8 = moe_block_size == 8; + + if (has_zp) { + TORCH_CHECK( + q_type == sglang::kU4 || q_type == sglang::kU8, + "q_type must be u4 or u8 when has_zp = True. Got = ", + q_type.str()); + } else { + TORCH_CHECK( + q_type == sglang::kU4B8 || q_type == sglang::kU8B128 || q_type == sglang::kFE4M3fn || q_type == sglang::kFE2M1f, + "q_type must be uint4b8, uint8b128, float8_e4m3fn or float4_e2m1f when " + "has_zp = False. Got = ", + q_type.str()); + } + + TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m, ", ", prob_n, ", ", prob_k, "]"); + + int group_blocks = 0; + if (has_act_order) { + if (is_k_full) { + TORCH_CHECK(group_size != -1); + group_blocks = group_size / 16; + TORCH_CHECK( + prob_k % group_blocks == 0, "prob_k = ", prob_k, " is not divisible by group_blocks = ", group_blocks); + } else { + TORCH_CHECK(group_size == 0); + group_blocks = 0; + } + } else { + if (group_size == -1) { + group_blocks = -1; + } else { + group_blocks = group_size / 16; + TORCH_CHECK( + prob_k % group_blocks == 0, "prob_k = ", prob_k, " is not divisible by group_blocks = ", group_blocks); + } + } + + int num_bits = q_type.size_bits(); + const int4* A_ptr = (const int4*)A; + const int4* B_ptr = (const int4*)B; + int4* C_ptr = (int4*)C; + int4* C_tmp_ptr = (int4*)C_tmp; + const int4* bias_ptr = (const int4*)b_bias; + const int4* s_ptr = (const int4*)s; + const uint16_t* s2_ptr = (const uint16_t*)s2; + const int4* zp_ptr = (const int4*)zp; + const int* g_idx_ptr = (const int*)g_idx; + const int* perm_ptr = (const int*)perm; + int4* a_tmp_ptr = (int4*)a_tmp; + const int32_t* sorted_token_ids_ptr = (const int32_t*)sorted_token_ids; + const int32_t* expert_ids_ptr = (const int32_t*)expert_ids; + const int32_t* num_tokens_past_padded_ptr = (const int32_t*)num_tokens_past_padded; + const float* topk_weights_ptr = (const float*)topk_weights; + int* locks = (int*)workspace; + + if (has_act_order) { + // Permute A columns + auto kernel = permute_cols_kernel<8>; + if (moe_block_size == 8) { + } else if (moe_block_size == 16) + kernel = permute_cols_kernel<16>; + else if (moe_block_size == 32) + kernel = permute_cols_kernel<32>; + else if (moe_block_size == 48) + kernel = permute_cols_kernel<48>; + else if (moe_block_size == 64) + kernel = permute_cols_kernel<64>; + else + TORCH_CHECK(false, "unsupported moe_block_size ", moe_block_size); + + // avoid ">>>" being formatted to "> > >" + // clang-format off + kernel<<>>( + A_ptr, perm_ptr, a_tmp_ptr, sorted_token_ids_ptr, expert_ids_ptr, + num_tokens_past_padded_ptr, prob_m, prob_k, top_k); + // clang-format on + A_ptr = a_tmp_ptr; + prob_m = prob_m * top_k; + top_k = 1; + + // If we have a full K, then we can run the non-act-order version of Marlin + // (since the weight rows are reordered by increasing group ids, and by + // having a full K, we have full original groups) + if (is_k_full) has_act_order = false; + } + + int max_shared_mem = 0; + cudaDeviceGetAttribute(&max_shared_mem, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev); + TORCH_CHECK(max_shared_mem > 0); + + // Set thread config + exec_config_t exec_cfg; + thread_config_t thread_tfg; + if (thread_k != -1 && thread_n != -1) { + thread_tfg = thread_config_t{thread_k, thread_n, default_threads}; + exec_cfg = exec_config_t{1, thread_tfg}; + TORCH_CHECK(prob_n % thread_n == 0, "prob_n = ", prob_n, " is not divisible by thread_n = ", thread_n); + TORCH_CHECK(prob_k % thread_k == 0, "prob_k = ", prob_k, " is not divisible by thread_k = ", thread_k); + } else { + // Auto config + exec_cfg = determine_exec_config( + q_type, + prob_m, + prob_n, + prob_k, + thread_m_blocks, + m_block_size_8, + num_bits, + group_size, + has_act_order, + is_k_full, + has_zp, + is_zp_float, + max_shared_mem); + thread_tfg = exec_cfg.tb_cfg; + } + + int num_threads = thread_tfg.num_threads; + thread_k = thread_tfg.thread_k; + thread_n = thread_tfg.thread_n; + int blocks = sms * exec_cfg.blocks_per_sm; + if (exec_cfg.blocks_per_sm > 1) max_shared_mem = max_shared_mem / exec_cfg.blocks_per_sm - 1024; + + int thread_k_blocks = thread_k / 16; + int thread_n_blocks = thread_n / 16; + + TORCH_CHECK( + is_valid_config( + thread_tfg, + m_block_size_8, + thread_m_blocks, + prob_m, + prob_n, + prob_k, + num_bits, + group_size, + has_act_order, + is_k_full, + has_zp, + is_zp_float, + max_shared_mem), + "Invalid thread config: thread_m_blocks = ", + thread_m_blocks, + ", thread_k = ", + thread_tfg.thread_k, + ", thread_n = ", + thread_tfg.thread_n, + ", num_threads = ", + thread_tfg.num_threads, + " for MKN = [", + prob_m, + ", ", + prob_k, + ", ", + prob_n, + "] and num_bits = ", + num_bits, + ", group_size = ", + group_size, + ", has_act_order = ", + has_act_order, + ", is_k_full = ", + is_k_full, + ", has_zp = ", + has_zp, + ", is_zp_float = ", + is_zp_float, + ", max_shared_mem = ", + max_shared_mem); + + auto kernel = get_marlin_kernel( + q_type, + thread_m_blocks, + thread_n_blocks, + thread_k_blocks, + m_block_size_8, + has_act_order, + has_zp, + group_blocks, + num_threads, + is_zp_float); + + if (kernel == MarlinDefault) { + TORCH_CHECK( + false, + "Unsupported shapes: MNK = [", + prob_m, + ", ", + prob_n, + ", ", + prob_k, + "]", + ", has_act_order = ", + has_act_order, + ", num_groups = ", + num_groups, + ", group_size = ", + group_size, + ", thread_m_blocks = ", + thread_m_blocks, + ", thread_n_blocks = ", + thread_n_blocks, + ", thread_k_blocks = ", + thread_k_blocks, + ", num_bits = ", + num_bits); + } + + cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); + // avoid ">>>" being formatted to "> > >" + // clang-format off + kernel<<>>( + A_ptr, B_ptr, C_ptr, C_tmp_ptr, bias_ptr, s_ptr, s2_ptr, zp_ptr, g_idx_ptr, + sorted_token_ids_ptr, expert_ids_ptr, num_tokens_past_padded_ptr, + topk_weights_ptr, top_k, mul_topk_weights, is_ep, num_groups, prob_m, + prob_n, prob_k, locks, has_bias, use_atomic_add, use_fp32_reduce, max_shared_mem); + // clang-format on +} + +} // namespace MARLIN_NAMESPACE_NAME + +torch::Tensor moe_wna16_marlin_gemm( + torch::Tensor& a, + std::optional const& c_or_none, + torch::Tensor& b_q_weight, + std::optional const& b_bias_or_none, + torch::Tensor& b_scales, + std::optional const& global_scale_or_none, + std::optional const& b_zeros_or_none, + std::optional const& g_idx_or_none, + std::optional const& perm_or_none, + torch::Tensor& workspace, + torch::Tensor& sorted_token_ids, + torch::Tensor& expert_ids, + torch::Tensor& num_tokens_past_padded, + torch::Tensor& topk_weights, + int64_t moe_block_size, + int64_t top_k, + bool mul_topk_weights, + bool is_ep, + sglang::ScalarTypeId const& b_q_type_id, + int64_t size_m, + int64_t size_n, + int64_t size_k, + bool is_k_full, + bool use_atomic_add, + bool use_fp32_reduce, + bool is_zp_float) { + sglang::ScalarType const b_q_type = sglang::ScalarType::from_id(b_q_type_id); + int pack_factor = 32 / b_q_type.size_bits(); + + if (moe_block_size != 8) { + TORCH_CHECK(moe_block_size % 16 == 0, "unsupported moe_block_size=", moe_block_size); + TORCH_CHECK(moe_block_size >= 16 && moe_block_size <= 64, "unsupported moe_block_size=", moe_block_size); + } + + // Verify A + TORCH_CHECK(a.size(0) == size_m, "Shape mismatch: a.size(0) = ", a.size(0), ", size_m = ", size_m); + TORCH_CHECK(a.size(1) == size_k, "Shape mismatch: a.size(1) = ", a.size(1), ", size_k = ", size_k); + + // Verify B + TORCH_CHECK( + size_k % MARLIN_NAMESPACE_NAME::tile_size == 0, + "size_k = ", + size_k, + " is not divisible by tile_size = ", + MARLIN_NAMESPACE_NAME::tile_size); + TORCH_CHECK( + (size_k / MARLIN_NAMESPACE_NAME::tile_size) == b_q_weight.size(1), + "Shape mismatch: b_q_weight.size(1) = ", + b_q_weight.size(1), + ", size_k = ", + size_k, + ", tile_size = ", + MARLIN_NAMESPACE_NAME::tile_size); + TORCH_CHECK( + b_q_weight.size(2) % MARLIN_NAMESPACE_NAME::tile_size == 0, + "b_q_weight.size(2) = ", + b_q_weight.size(2), + " is not divisible by tile_size = ", + MARLIN_NAMESPACE_NAME::tile_size); + int actual_size_n = (b_q_weight.size(2) / MARLIN_NAMESPACE_NAME::tile_size) * pack_factor; + TORCH_CHECK(size_n == actual_size_n, "size_n = ", size_n, ", actual_size_n = ", actual_size_n); + + // Verify device and strides + TORCH_CHECK(a.device().is_cuda(), "A is not on GPU"); + TORCH_CHECK(a.is_contiguous(), "A is not contiguous"); + + TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU"); + TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous"); + + TORCH_CHECK(b_scales.device().is_cuda(), "b_scales is not on GPU"); + TORCH_CHECK(b_scales.is_contiguous(), "b_scales is not contiguous"); + + // thread_k: `k` size of a thread_tile in `weights` (can usually be left as + // auto -1) + int thread_k = -1; + // thread_n: `n` size of a thread_tile in `weights` (can usually be left as + // auto -1) + int thread_n = -1; + // sms: number of SMs to use for the kernel + int sms = -1; + cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, a.get_device()); + + // Alloc buffers + const at::cuda::OptionalCUDAGuard device_guard(device_of(a)); + auto options = torch::TensorOptions().dtype(a.dtype()).device(a.device()); + torch::Tensor c; + if (c_or_none.has_value()) { + c = c_or_none.value(); + TORCH_CHECK(c.device().is_cuda(), "c is not on GPU"); + TORCH_CHECK(c.is_contiguous(), "c is not contiguous"); + TORCH_CHECK( + c.size(0) == size_m * top_k, "Shape mismatch: c.size(0) = ", c.size(0), ", size_m * topk = ", size_m * top_k); + TORCH_CHECK(c.size(1) == size_n, "Shape mismatch: c.size(1) = ", c.size(1), ", size_n = ", size_n); + } else { + c = torch::empty({size_m * top_k, size_n}, options); + } + + // Alloc C tmp buffer that is going to be used for the global reduce + torch::Tensor c_tmp; + auto options_fp32 = torch::TensorOptions().dtype(at::kFloat).device(a.device()); + if (use_fp32_reduce && !use_atomic_add) { + // max num of threadblocks is sms * 4 + long max_c_tmp_size = min( + (long)size_n * sorted_token_ids.size(0), (long)sms * 4 * moe_block_size * MARLIN_NAMESPACE_NAME::max_thread_n); + if (moe_block_size == 8) max_c_tmp_size *= 2; + c_tmp = torch::empty({max_c_tmp_size}, options_fp32); + } else { + c_tmp = torch::empty({0}, options_fp32); + } + + // Detect groupsize and act_order + int num_groups = -1; + int group_size = -1; + + int rank = b_scales.sizes().size(); + TORCH_CHECK(rank == 3, "b_scales rank = ", rank, " is not 3"); + TORCH_CHECK(b_scales.size(2) == size_n, "b_scales dim 2 = ", b_scales.size(2), " is not size_n = ", size_n); + num_groups = b_scales.size(1); + + torch::Tensor g_idx, perm, a_tmp; + if (g_idx_or_none.has_value() && perm_or_none.has_value()) { + g_idx = g_idx_or_none.value(); + perm = perm_or_none.value(); + + TORCH_CHECK(g_idx.device().is_cuda(), "g_idx is not on GPU"); + TORCH_CHECK(g_idx.is_contiguous(), "g_idx is not contiguous"); + TORCH_CHECK(perm.device().is_cuda(), "perm is not on GPU"); + TORCH_CHECK(perm.is_contiguous(), "perm is not contiguous"); + + // Verify g_idx and perm + TORCH_CHECK( + (g_idx.size(-1) == 0 && perm.size(-1) == 0) || (g_idx.size(-1) == size_k && perm.size(-1) == size_k), + "Unexpected g_idx.size(-1) = ", + g_idx.size(-1), + " and perm.size(-1) = ", + perm.size(-1), + ", where size_k = ", + size_k); + } else { + g_idx = torch::empty({0}, options); + perm = torch::empty({0}, options); + a_tmp = torch::empty({0}, options); + } + bool has_act_order = g_idx.size(-1) > 0 && perm.size(-1) > 0; + + if (has_act_order) { + a_tmp = torch::empty({size_m * top_k, size_k}, options); + if (is_k_full) { + TORCH_CHECK(num_groups > 1, "For act_order, num_groups must be > 1"); + TORCH_CHECK(size_k % num_groups == 0, "size_k = ", size_k, ", is not divisible by num_groups = ", num_groups); + group_size = size_k / num_groups; + } else { + group_size = 0; + } + + } else { + a_tmp = torch::empty({0}, options); + if (num_groups > 1) { + TORCH_CHECK( + size_k % num_groups == 0, "size_k = ", size_k, ", is not divisible by b_scales.size(1) = ", b_scales.size(1)); + group_size = size_k / num_groups; + } else { + group_size = -1; + } + } + + torch::Tensor global_scale; + if (global_scale_or_none.has_value()) { + global_scale = global_scale_or_none.value(); + TORCH_CHECK(b_q_type == sglang::kFE2M1f && group_size == 16, "global_scale can only be used for nvfp4 format."); + } else { + global_scale = torch::empty({0}, options); + TORCH_CHECK( + !(b_q_type == sglang::kFE2M1f && group_size == 16), + "the global_scale parameter must be passed for nvfp4 format."); + } + + bool has_bias = b_bias_or_none.has_value(); + torch::Tensor b_bias; + if (has_bias) { + b_bias = b_bias_or_none.value(); + TORCH_CHECK(b_bias.device().is_cuda(), "b_bias is not on GPU"); + TORCH_CHECK(b_bias.is_contiguous(), "b_bias is not contiguous"); + TORCH_CHECK(b_bias.size(1) == size_n, "b_bias.size(0) != size_n"); + TORCH_CHECK(b_bias.stride(1) == 1, "b_bias.stride(1) != 1"); + } else { + b_bias = torch::empty({0}, options); + } + + torch::Tensor b_zeros; + if (b_zeros_or_none.has_value()) { + b_zeros = b_zeros_or_none.value(); + TORCH_CHECK(b_zeros.device().is_cuda(), "b_zeros is not on GPU"); + TORCH_CHECK(b_zeros.is_contiguous(), "b_zeros is not contiguous"); + } else { + b_zeros = torch::empty({0}, options); + } + bool has_zp = b_zeros.size(-1) > 0; + if (has_zp) { + TORCH_CHECK( + b_q_type == sglang::kU4 || b_q_type == sglang::kU8, + "b_q_type must be u4 or u8 when has_zp = True. Got = ", + b_q_type.str()); + } else { + TORCH_CHECK( + b_q_type == sglang::kU4B8 || b_q_type == sglang::kU8B128 || b_q_type == sglang::kFE4M3fn || + b_q_type == sglang::kFE2M1f, + "b_q_type must be uint4b8, uint8b128, float8_e4m3fn or " + "float4_e2m1f when " + "has_zp = False. Got = ", + b_q_type.str()); + } + + if (has_zp && is_zp_float) { + TORCH_CHECK( + a.scalar_type() == at::ScalarType::Half, + "Computation type must be float16 (half) when using float zero " + "points."); + } + + // Verify b_zeros + if (has_zp) { + int rank = b_zeros.sizes().size(); + TORCH_CHECK(rank == 3, "b_zeros rank = ", rank, " is not 3"); + if (is_zp_float) { + TORCH_CHECK(b_zeros.size(2) == size_n, "b_zeros dim 2 = ", b_zeros.size(2), " is not size_n = ", size_n); + TORCH_CHECK( + num_groups == b_zeros.size(1), "b_zeros dim 1 = ", b_zeros.size(1), " is not num_groups = ", num_groups); + TORCH_CHECK(num_groups != -1, "num_groups must be != -1"); + } else { + TORCH_CHECK( + b_zeros.size(1) == num_groups, "b_zeros dim 1 = ", b_zeros.size(1), " is not num_groups = ", num_groups); + TORCH_CHECK( + b_zeros.size(2) == size_n / pack_factor, + "b_zeros dim 2 = ", + b_zeros.size(2), + " is not size_n / pack_factor = ", + size_n / pack_factor); + } + } + + // Verify workspace size + TORCH_CHECK( + size_n % MARLIN_NAMESPACE_NAME::min_thread_n == 0, + "size_n = ", + size_n, + ", is not divisible by min_thread_n = ", + MARLIN_NAMESPACE_NAME::min_thread_n); + + int max_n_tiles = size_n / MARLIN_NAMESPACE_NAME::min_thread_n; + int min_workspace_size = min(max_n_tiles * (int)(sorted_token_ids.size(0) / moe_block_size), sms * 4); + TORCH_CHECK( + workspace.numel() >= min_workspace_size, + "workspace.numel = ", + workspace.numel(), + " is below min_workspace_size = ", + min_workspace_size); + + int dev = a.get_device(); + if (a.scalar_type() == at::ScalarType::Half) { + void* scales_ptr; + if (b_q_type == sglang::kFE2M1f) { + if (group_size == 16) + scales_ptr = b_scales.data_ptr(); + else if (group_size == 32) + scales_ptr = b_scales.data_ptr(); + else + TORCH_CHECK(false, "float4_e2m1f only supports group_size == 16 (NVFP4) ", "and group_size == 32 (MXFP4)"); + } else { + scales_ptr = b_scales.data_ptr(); + } + + MARLIN_NAMESPACE_NAME::marlin_mm( + a.data_ptr(), + b_q_weight.data_ptr(), + c.data_ptr(), + c_tmp.data_ptr(), + b_bias.data_ptr(), + scales_ptr, + global_scale.data_ptr(), + b_zeros.data_ptr(), + g_idx.data_ptr(), + perm.data_ptr(), + a_tmp.data_ptr(), + sorted_token_ids.data_ptr(), + expert_ids.data_ptr(), + num_tokens_past_padded.data_ptr(), + topk_weights.data_ptr(), + moe_block_size, + top_k, + mul_topk_weights, + is_ep, + size_m, + size_n, + size_k, + workspace.data_ptr(), + b_q_type, + has_bias, + has_act_order, + is_k_full, + has_zp, + num_groups, + group_size, + dev, + at::cuda::getCurrentCUDAStream(dev), + thread_k, + thread_n, + sms, + use_atomic_add, + use_fp32_reduce, + is_zp_float); + } else if (a.scalar_type() == at::ScalarType::BFloat16) { + void* scales_ptr; + if (b_q_type == sglang::kFE2M1f) { + if (group_size == 16) + scales_ptr = b_scales.data_ptr(); + else if (group_size == 32) + scales_ptr = b_scales.data_ptr(); + else + TORCH_CHECK(false, "float4_e2m1f only supports group_size == 16 (NVFP4) ", "and group_size == 32 (MXFP4)"); + } else { + scales_ptr = b_scales.data_ptr(); + } + + MARLIN_NAMESPACE_NAME::marlin_mm( + a.data_ptr(), + b_q_weight.data_ptr(), + c.data_ptr(), + c_tmp.data_ptr(), + b_bias.data_ptr(), + scales_ptr, + global_scale.data_ptr(), + b_zeros.data_ptr(), + g_idx.data_ptr(), + perm.data_ptr(), + a_tmp.data_ptr(), + sorted_token_ids.data_ptr(), + expert_ids.data_ptr(), + num_tokens_past_padded.data_ptr(), + topk_weights.data_ptr(), + moe_block_size, + top_k, + mul_topk_weights, + is_ep, + size_m, + size_n, + size_k, + workspace.data_ptr(), + b_q_type, + has_bias, + has_act_order, + is_k_full, + has_zp, + num_groups, + group_size, + dev, + at::cuda::getCurrentCUDAStream(dev), + thread_k, + thread_n, + sms, + use_atomic_add, + use_fp32_reduce, + is_zp_float); + } else { + TORCH_CHECK(false, "moe_wna16_marlin_gemm only supports bfloat16 and float16"); + } + + return c; +} + +#endif + +// Registration is done in common_extension.cc for v2 version diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_align_kernel.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_align_kernel.cu new file mode 100644 index 00000000..b7bad43d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_align_kernel.cu @@ -0,0 +1,383 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include + +#include "utils.h" + +#define VEC_SIZE 4 +using Vec = int4; + +template +__global__ void count_and_sort_expert_tokens_kernel( + const scalar_t* __restrict__ topk_ids, + int32_t* __restrict__ sorted_token_ids, + int32_t* __restrict__ cumsum_buffer, + size_t numel) { + const size_t tid = blockIdx.x * blockDim.x + threadIdx.x; + const size_t stride = blockDim.x * gridDim.x; + + for (size_t i = tid; i < numel; i += stride) { + int32_t expert_id = topk_ids[i] + 1; + int32_t rank_post_pad = atomicAdd(&cumsum_buffer[expert_id], 1); + sorted_token_ids[rank_post_pad] = i; + } +} + +#ifdef __CUDA_ARCH__ +__device__ __forceinline__ int warp_exclusive_scan(int v, unsigned mask = 0xffffffffu) { + int original = v; +#pragma unroll + for (int offset = 1; offset < WARP_SIZE; offset <<= 1) { + int n = __shfl_up_sync(mask, v, offset); + if ((threadIdx.x & (WARP_SIZE - 1)) >= offset) v += n; + } + return v - original; +} +#endif + +template +__global__ void moe_align_block_size_kernel( + const scalar_t* __restrict__ topk_ids, + int32_t* __restrict__ sorted_token_ids, + int32_t* __restrict__ expert_ids, + int32_t* __restrict__ total_tokens_post_pad, + int32_t num_experts, + int32_t block_size, + size_t numel, + int32_t* __restrict__ cumsum, + bool pad_sorted_token_ids, + const int32_t scan_size, + int32_t max_num_tokens_padded) { + // Use a separate thread block to populate sorted_token_ids + if (blockIdx.x == 1) { + if (pad_sorted_token_ids) { + Vec fill_vec; + fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel; + int32_t total_vecs = (max_num_tokens_padded + VEC_SIZE - 1) / VEC_SIZE; + Vec* out_ptr = reinterpret_cast(sorted_token_ids); + for (int32_t i = threadIdx.x; i < total_vecs; i += blockDim.x) { + out_ptr[i] = fill_vec; + } + } + return; + } + + extern __shared__ int32_t smem[]; + int32_t* shared_counts = smem; // [num_experts] + int32_t* prefix = shared_counts + num_experts; // [num_experts + 1] + int32_t* scan_buf = prefix + num_experts + 1; // [scan_size] + __shared__ int32_t s_total_tokens_post_pad; + + const size_t tid = threadIdx.x; + const size_t stride = blockDim.x; + + if (tid < num_experts) { + shared_counts[tid] = 0; + } + + __syncthreads(); + + for (size_t i = tid; i < numel; i += stride) { + int expert_id = topk_ids[i] + 1; + atomicAdd(&shared_counts[expert_id], 1); + } + + __syncthreads(); + + int32_t padded_count = 0; + if (tid < num_experts) { + int32_t count = shared_counts[tid]; + padded_count = (count + block_size - 1) / block_size * block_size; + scan_buf[tid] = padded_count; + } + +#ifndef __CUDA_ARCH__ // HIP + + if (tid >= num_experts && tid < scan_size) { + scan_buf[tid] = 0; + } + + __syncthreads(); + + // Blelloch scan + int offset = 1; +#pragma unroll + for (int d = scan_size >> 1; d > 0; d >>= 1) { + if (tid < d) { + int ai = offset * (2 * tid + 1) - 1; + int bi = offset * (2 * tid + 2) - 1; + scan_buf[bi] += scan_buf[ai]; + } + offset <<= 1; + __syncthreads(); + } + + // down-sweep + if (tid == 0) { + prefix[num_experts] = scan_buf[scan_size - 1]; + scan_buf[scan_size - 1] = 0; + } + __syncthreads(); + +#pragma unroll + for (int d = 1; d < scan_size; d <<= 1) { + offset >>= 1; + if (tid < d) { + int ai = offset * (2 * tid + 1) - 1; + int bi = offset * (2 * tid + 2) - 1; + if (bi < scan_size) { + int temp = scan_buf[ai]; + scan_buf[ai] = scan_buf[bi]; + scan_buf[bi] += temp; + } + } + __syncthreads(); + } + + if (tid < num_experts) { + prefix[tid] = scan_buf[tid]; + } + + if (tid == 0) { + s_total_tokens_post_pad = prefix[num_experts]; + *total_tokens_post_pad = s_total_tokens_post_pad; + } + __syncthreads(); + +#else // CUDA + + // Intra warp prefix sum + int32_t* warp_sums = scan_buf + scan_size; // [<= 32] + const int warp_id = tid / WARP_SIZE; + const int lane_id = tid & (WARP_SIZE - 1); + const int num_warps_for_scan = (scan_size + WARP_SIZE - 1) / WARP_SIZE; + const int warp_sum = warp_exclusive_scan(padded_count) + padded_count; + if (lane_id == WARP_SIZE - 1) warp_sums[warp_id] = warp_sum; + __syncthreads(); + + // warp0 accumulate all the block's prefix sum + if (tid < WARP_SIZE) { + int val = (tid < num_warps_for_scan) ? warp_sums[tid] : 0; + int incl = warp_exclusive_scan(val) + val; + warp_sums[tid] = incl; + } + __syncthreads(); + + // Every thread obtains the whole block's sum + if (tid == 0) { + prefix[num_experts] = warp_sums[num_warps_for_scan - 1]; + s_total_tokens_post_pad = prefix[num_experts]; + *total_tokens_post_pad = s_total_tokens_post_pad; + } + __syncthreads(); + + // Fill 0 to scan_buf extended area (tid >= num_expert) + if (tid >= num_experts && tid < scan_size) scan_buf[tid] = 0; + __syncthreads(); + + // Perform 2 level exclusive-prefix-sum to scan_buf + int v = (tid < scan_size) ? scan_buf[tid] : 0; + int pre = warp_exclusive_scan(v); + if (lane_id == WARP_SIZE - 1) warp_sums[warp_id] = pre + v; + __syncthreads(); + + if (warp_id == 0) { + int val = (lane_id < num_warps_for_scan) ? warp_sums[lane_id] : 0; + warp_sums[lane_id] = warp_exclusive_scan(val); + } + __syncthreads(); + + int offset = warp_sums[warp_id]; + if (tid < scan_size) scan_buf[tid] = pre + offset; + __syncthreads(); + + // Write prefix[0..num_experts - 1] and cumsum + if (tid < num_experts) prefix[tid] = scan_buf[tid]; +#endif + + if (tid <= num_experts) { + cumsum[tid] = prefix[tid]; + } + // fill expert_ids + const int32_t num_blocks = s_total_tokens_post_pad / block_size; + for (int32_t i = tid; i < num_blocks; i += stride) { + int32_t block_start = i * block_size; + int left = 0, right = num_experts; + while (left < right) { + int mid = (left + right) >> 1; + if (prefix[mid] <= block_start) { + left = mid + 1; + } else { + right = mid; + } + } + expert_ids[i] = left - 2; + } +} + +template +__global__ void moe_align_block_size_small_batch_expert_kernel( + const scalar_t* __restrict__ topk_ids, + int32_t* __restrict__ sorted_token_ids, + int32_t* __restrict__ expert_ids, + int32_t* __restrict__ total_tokens_post_pad, + int32_t num_experts, + int32_t block_size, + size_t numel, + bool pad_sorted_token_ids, + int32_t max_num_tokens_padded) { + // Adapted from + // https://github.com/vllm-project/vllm/pull/29642/files#diff-5647b1413f4ae9aacba904eca8f8a8aee9079321eadff4c10101a2c6962dcc53R226 + // Use an additional group of threads to fill sorted_token_ids. + // Since the kernel will use sorted_token_ids afterward, + // we fill sorted_token_ids within the same threadblock to make + // synchronization easier. + if (threadIdx.x < fill_threads) { + // Initialize sorted_token_ids with numel + if (pad_sorted_token_ids) { + for (int32_t it = threadIdx.x; it < max_num_tokens_padded; it += fill_threads) { + sorted_token_ids[it] = numel; + } + } + // Three __syncthreads() corresponding to the other threads + __syncthreads(); + __syncthreads(); + __syncthreads(); + return; + } + + const size_t tid = threadIdx.x - fill_threads; + const size_t stride = blockDim.x - fill_threads; + + extern __shared__ int32_t shared_mem[]; + int32_t* cumsum = shared_mem; + int32_t* tokens_cnts = (int32_t*)(shared_mem + num_experts + 1); + + for (int i = 0; i < num_experts; ++i) { + tokens_cnts[(tid + 1) * num_experts + i] = 0; + } + + for (size_t i = tid; i < numel; i += stride) { + int32_t expert_id = topk_ids[i] + 1; + ++tokens_cnts[(tid + 1) * num_experts + expert_id]; + } + + __syncthreads(); + + if (tid < num_experts) { + tokens_cnts[tid] = 0; + for (int i = 1; i <= stride; ++i) { + tokens_cnts[i * num_experts + tid] += tokens_cnts[(i - 1) * num_experts + tid]; + } + } + + __syncthreads(); + + if (tid == 0) { + cumsum[0] = 0; + for (int i = 1; i <= num_experts; ++i) { + cumsum[i] = cumsum[i - 1] + CEILDIV(tokens_cnts[stride * num_experts + i - 1], block_size) * block_size; + } + *total_tokens_post_pad = static_cast(cumsum[num_experts]); + } + + __syncthreads(); + + if (tid < num_experts) { + for (int i = cumsum[tid]; i < cumsum[tid + 1]; i += block_size) { + expert_ids[i / block_size] = tid - 1; + } + } + + for (size_t i = tid; i < numel; i += stride) { + int32_t expert_id = topk_ids[i] + 1; + int32_t rank_post_pad = tokens_cnts[tid * num_experts + expert_id] + cumsum[expert_id]; + sorted_token_ids[rank_post_pad] = i; + ++tokens_cnts[tid * num_experts + expert_id]; + } +} + +void moe_align_block_size( + torch::Tensor topk_ids, + int64_t num_experts, + int64_t block_size, + torch::Tensor sorted_token_ids, + torch::Tensor experts_ids, + torch::Tensor num_tokens_post_pad, + torch::Tensor cumsum_buffer, + bool pad_sorted_token_ids) { + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + int threads = 1024; + threads = ((threads + WARP_SIZE - 1) / WARP_SIZE) * WARP_SIZE; + + int64_t max_num_tokens_padded = sorted_token_ids.size(0); + + DISPATCH_INTEGRAL_TYPES(topk_ids.scalar_type(), "moe_align_block_size_kernel", [&] { + bool small_batch_expert_mode = (topk_ids.numel() < 1024) && (num_experts <= 64); + + if (small_batch_expert_mode) { + const int32_t threads = max((int32_t)num_experts, WARP_SIZE); + constexpr int32_t fill_threads = 256; + const int32_t shared_mem_size = ((threads + 1) * num_experts + (num_experts + 1)) * sizeof(int32_t); + + auto small_batch_expert_kernel = moe_align_block_size_small_batch_expert_kernel; + small_batch_expert_kernel<<<1, fill_threads + threads, shared_mem_size, stream>>>( + topk_ids.data_ptr(), + sorted_token_ids.data_ptr(), + experts_ids.data_ptr(), + num_tokens_post_pad.data_ptr(), + num_experts, + block_size, + topk_ids.numel(), + pad_sorted_token_ids, + max_num_tokens_padded); + } else { + auto align_kernel = moe_align_block_size_kernel; + + const size_t scan_size = next_pow2(num_experts); + const size_t shared_mem_size = (num_experts + (num_experts + 1) + scan_size + WARP_SIZE) * sizeof(int32_t); + align_kernel<<<2, threads, shared_mem_size, stream>>>( + topk_ids.data_ptr(), + sorted_token_ids.data_ptr(), + experts_ids.data_ptr(), + num_tokens_post_pad.data_ptr(), + num_experts, + block_size, + topk_ids.numel(), + cumsum_buffer.data_ptr(), + pad_sorted_token_ids, + scan_size, + max_num_tokens_padded); + + const int block_threads = std::min(256, (int)threads); + const int num_blocks = (topk_ids.numel() + block_threads - 1) / block_threads; + const int max_blocks = 65535; + const int actual_blocks = std::min(num_blocks, max_blocks); + + auto sort_kernel = count_and_sort_expert_tokens_kernel; + sort_kernel<<>>( + topk_ids.data_ptr(), + sorted_token_ids.data_ptr(), + cumsum_buffer.data_ptr(), + topk_ids.numel()); + } + }); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_fused_gate.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_fused_gate.cu new file mode 100644 index 00000000..1f70a23d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_fused_gate.cu @@ -0,0 +1,523 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +template +using AlignedArray = cutlass::AlignedArray; +using bfloat16_t = cutlass::bfloat16_t; +using float16_t = cutlass::half_t; +using float32_t = float; + +// QQ NOTE: to handle the case for at::Half, error: more than one operator ">" matches these operands: built-in operator +// "arithmetic > arithmetic" function "operator>(const __half &, const __half &)" +template +__device__ inline bool cmp_gt(const T& a, const T& b) { + if constexpr (std::is_same::value) { + // at::Half (or float16_t in our native case) causes ambiguity, so we cast to float. + return static_cast(a) > static_cast(b); + } else { + // For types like float, at::BFloat16, or cutlass::half_t / cutlass::bfloat16_t, assume operator> works as expected. + return a > b; + } +} + +template +__device__ inline bool cmp_eq(const T& a, const T& b) { + if constexpr (std::is_same::value) { + return static_cast(a) == static_cast(b); + } else { + return a == b; + } +} + +// Fixed constants common to both dynamic and static template versions: +static constexpr int WARP_SIZE = 32; +static constexpr int WARPS_PER_CTA = 6; +static constexpr int MAX_VPT = 32; // maximum VPT we support, > params.VPT = num_expert / num_expert_group + +// Create an alias for Array using AlignedArray +template +using Array = AlignedArray; +// QQ: NOTE expression must have a constant value, this has to be > params.VPT +template +using AccessType = AlignedArray; + +template +__device__ void moe_fused_gate_impl( + void* input, + void* bias, + float* output_ptr, + int32_t* indices_ptr, + int64_t num_rows, + int64_t topk_group, + int64_t topk, + int64_t num_fused_shared_experts, + double routed_scaling_factor, + bool apply_routed_scaling_factor_on_output, + Params params) { + int tidx = threadIdx.x; + int64_t thread_row = + blockIdx.x * params.ROWS_PER_CTA + threadIdx.y * params.ROWS_PER_WARP + tidx / params.THREADS_PER_ROW; + if (thread_row >= num_rows) { + return; + } + + // Calculate topk_excluding_share_expert_fusion from topk + int64_t topk_excluding_share_expert_fusion = topk - num_fused_shared_experts; + + // Cast pointers to type T: + auto* input_ptr = reinterpret_cast(input); + auto* bias_ptr = reinterpret_cast(bias); + auto* thread_row_ptr = input_ptr + thread_row * params.NUM_EXPERTS; + + int thread_group_idx = tidx % params.THREADS_PER_ROW; + int first_elt_read_by_thread = thread_group_idx * params.VPT; + + // Create local arrays for the row chunk and bias chunk and then reinterpret the address of row_chunk as a pointer to + // AccessType. + T* thread_read_ptr = thread_row_ptr + first_elt_read_by_thread; + Array row_chunk; + AccessType const* vec_thread_read_ptr = reinterpret_cast const*>(thread_read_ptr); + + T* bias_thread_read_ptr = bias_ptr + first_elt_read_by_thread; + Array bias_chunk; + AccessType const* vec_bias_thread_read_ptr = reinterpret_cast const*>(bias_thread_read_ptr); + +// QQ NOTE: doing the follow will be slower than loop assign and more importantly +// have misaligned address issue when params.VPT < 8 and mismatch with MAX_VPT +// AccessType* row_chunk_vec_ptr = reinterpret_cast*>(&row_chunk); +// row_chunk_vec_ptr[0] = vec_thread_read_ptr[0]; +#pragma unroll + for (int ii = 0; ii < params.VPT; ++ii) { + row_chunk[ii] = vec_thread_read_ptr[0][ii]; + bias_chunk[ii] = vec_bias_thread_read_ptr[0][ii]; + } + + __syncthreads(); + +////////////////////// Sigmoid ////////////////////// +#pragma unroll + for (int ii = 0; ii < params.VPT; ++ii) { + row_chunk[ii] = static_cast(1.0f / (1.0f + expf(-float(row_chunk[ii])))); + } + __syncthreads(); + +////////////////////// Add Bias ////////////////////// +#pragma unroll + for (int ii = 0; ii < params.VPT; ++ii) { + bias_chunk[ii] = row_chunk[ii] + bias_chunk[ii]; + } + +////////////////////// Exclude Groups ////////////////////// +#pragma unroll + for (int k_idx = 0; k_idx < params.THREADS_PER_ROW - topk_group; + ++k_idx) { // QQ NOTE Here params.THREADS_PER_ROW = num_expert_group + int expert = first_elt_read_by_thread; + // local argmax + T max_val = static_cast(-FLT_MAX); + T max_val_second = static_cast(-FLT_MAX); +#pragma unroll + for (int ii = 0; ii < params.VPT; ++ii) { + T val = bias_chunk[ii]; + + if (cmp_gt(val, max_val)) { + max_val_second = max_val; + max_val = val; + } else if (cmp_gt(val, max_val_second)) { + max_val_second = val; + } + } + + // QQ NOTE: currently fixed to pick top2 sigmoid weight value in each expert group and sum them as the group weight + // to select expert groups + T max_sum = max_val + max_val_second; + +// argmin reduce +#pragma unroll + for (int mask = params.THREADS_PER_ROW / 2; mask > 0; mask /= 2) { + T other_max_sum = + static_cast(__shfl_xor_sync(0xFFFFFFFF, static_cast(max_sum), mask, params.THREADS_PER_ROW)); + int other_expert = __shfl_xor_sync(0xFFFFFFFF, expert, mask, params.THREADS_PER_ROW); + + // higher indices win + if (cmp_gt(max_sum, other_max_sum) || (cmp_eq(other_max_sum, max_sum) && other_expert > expert)) { + max_sum = other_max_sum; + expert = other_expert; + } + } + + // clear the max value in the thread + if (k_idx < params.THREADS_PER_ROW - topk_group) { + int const thread_to_clear_in_group = expert / params.VPT; + + if (thread_group_idx == thread_to_clear_in_group) { +#pragma unroll + for (int ii = 0; ii < params.VPT; ++ii) { + bias_chunk[ii] = static_cast(FLT_MAX); + } + } + } + } + + __syncthreads(); + + ////////////////////// Topk ////////////////////// + float output_sum = 0.0f; + for (int k_idx = 0; k_idx < topk_excluding_share_expert_fusion; ++k_idx) { + // local argmax + T max_val = bias_chunk[0]; + int expert = first_elt_read_by_thread; + + if (!cmp_eq(max_val, static_cast(FLT_MAX))) { +#pragma unroll + for (int ii = 1; ii < params.VPT; ++ii) { + T val = bias_chunk[ii]; + if (cmp_gt(val, max_val)) { + max_val = val; + expert = first_elt_read_by_thread + ii; + } + } + } else { + max_val = static_cast(-FLT_MAX); + } + + // argmax reduce +#pragma unroll + for (int mask = params.THREADS_PER_ROW / 2; mask > 0; mask /= 2) { + T other_max = + static_cast(__shfl_xor_sync(0xFFFFFFFF, static_cast(max_val), mask, params.THREADS_PER_ROW)); + int other_expert = __shfl_xor_sync(0xFFFFFFFF, expert, mask, params.THREADS_PER_ROW); + + // lower indices to win + if (cmp_gt(other_max, max_val) || (cmp_eq(other_max, max_val) && other_expert < expert)) { + max_val = other_max; + expert = other_expert; + } + } + + int thread_to_clear_in_group = expert / params.VPT; + int64_t idx = topk * thread_row + k_idx; + + if (thread_group_idx == thread_to_clear_in_group) { + int expert_to_clear_in_thread = expert % params.VPT; + + // clear the max value in the thread + bias_chunk[expert_to_clear_in_thread] = static_cast(-FLT_MAX); + + // store output + output_ptr[idx] = static_cast(row_chunk[expert_to_clear_in_thread]); + indices_ptr[idx] = static_cast(expert); + } + + // accumulate sum for all elements + if (thread_group_idx == 0) { + output_sum += output_ptr[idx]; + } + + __syncthreads(); + } + + if (thread_group_idx == 0 && num_fused_shared_experts > 0) { + int64_t last_idx = topk * thread_row + topk_excluding_share_expert_fusion; + int64_t expert_offset = 0; + indices_ptr[last_idx] = static_cast(params.NUM_EXPERTS + expert_offset); + + // Set the weight to the sum of all weights divided by routed_scaling_factor + output_ptr[last_idx] = output_sum / routed_scaling_factor; + + if (num_fused_shared_experts > 1) { + for (int i = 1; i < num_fused_shared_experts; ++i) { + ++last_idx; + ++expert_offset; + indices_ptr[last_idx] = static_cast(params.NUM_EXPERTS + expert_offset); + // Set the weight to the sum of all weights divided by routed_scaling_factor + output_ptr[last_idx] = output_sum / routed_scaling_factor; + } + } + } + __syncthreads(); + + ////////////////////// Rescale Output ////////////////////// + if (thread_group_idx == 0) { +#pragma unroll + for (int ii = 0; ii < topk; ++ii) { + int64_t const idx = topk * thread_row + ii; + output_ptr[idx] = output_ptr[idx] / output_sum; + if (apply_routed_scaling_factor_on_output) { + output_ptr[idx] *= routed_scaling_factor; + } + } + } +} + +//------------------------------------------------------------------------------ +// Templated Kernel Version (using compile-time constants) +//------------------------------------------------------------------------------ +template +struct KernelParams { + static constexpr int VPT = VPT_; + static constexpr int NUM_EXPERTS = NUM_EXPERTS_; + static constexpr int THREADS_PER_ROW = THREADS_PER_ROW_; + static constexpr int ROWS_PER_WARP = ROWS_PER_WARP_; + static constexpr int ROWS_PER_CTA = ROWS_PER_CTA_; + static constexpr int WARPS_PER_CTA = WARPS_PER_CTA_; +}; + +template < + typename T, + int VPT, + int NUM_EXPERTS, + int THREADS_PER_ROW, + int ROWS_PER_WARP, + int ROWS_PER_CTA, + int WARPS_PER_CTA> +__global__ void moe_fused_gate_kernel( + void* input, + void* bias, + float* output_ptr, + int32_t* indices_ptr, + int64_t num_rows, + int64_t topk_group, + int64_t topk, + int64_t num_fused_shared_experts, + double routed_scaling_factor, + bool apply_routed_scaling_factor_on_output) { + KernelParams params; + moe_fused_gate_impl( + input, + bias, + output_ptr, + indices_ptr, + num_rows, + topk_group, + topk, + num_fused_shared_experts, + routed_scaling_factor, + apply_routed_scaling_factor_on_output, + params); +} + +// Macro to compute compile-time constants and launch the kernel. +#define LAUNCH_MOE_GATE_CONFIG(T, EXPERTS, EXPERT_GROUP) \ + do { \ + constexpr int VPT = (EXPERTS) / (EXPERT_GROUP); \ + /* If EXPERT_GROUP > WARP_SIZE, fall back to 1 row per warp */ \ + constexpr int ROWS_PER_WARP = ((EXPERT_GROUP) <= WARP_SIZE) ? (WARP_SIZE / (EXPERT_GROUP)) : 1; \ + constexpr int ROWS_PER_CTA = WARPS_PER_CTA * ROWS_PER_WARP; \ + moe_fused_gate_kernel \ + <<>>( \ + input.data_ptr(), \ + bias.data_ptr(), \ + output.data_ptr(), \ + indices.data_ptr(), \ + num_rows, \ + topk_group, \ + topk, \ + num_fused_shared_experts, \ + routed_scaling_factor, \ + apply_routed_scaling_factor_on_output); \ + dispatched = true; \ + } while (0) + +//------------------------------------------------------------------------------ +// Dynamic Kernel Version (parameters computed at runtime) +//------------------------------------------------------------------------------ +struct KernelParamsDynamic { + int VPT; + int NUM_EXPERTS; + int THREADS_PER_ROW; + int ROWS_PER_WARP; + int ROWS_PER_CTA; + int WARPS_PER_CTA; +}; + +template +__global__ void moe_fused_gate_kernel_dynamic( + void* input, + void* bias, + float* output_ptr, + int32_t* indices_ptr, + int64_t num_rows, + int64_t num_experts, + int64_t num_expert_group, + int64_t topk_group, + int64_t topk, + int64_t num_fused_shared_experts, + double routed_scaling_factor, + bool apply_routed_scaling_factor_on_output) { + KernelParamsDynamic params; + params.NUM_EXPERTS = num_experts; // e.g, for deepseek v3, this is 256 + params.VPT = num_experts / num_expert_group; // e.g., for deepseek v3, this is 256 / 8 = 32 + params.THREADS_PER_ROW = num_expert_group; // fixed as num_expert_group, e.g., for deepseek v3, this is 8 + params.WARPS_PER_CTA = WARPS_PER_CTA; // fixed as 6 + params.ROWS_PER_WARP = std::max(1, WARP_SIZE / num_expert_group); // WARP_SIZE is fixed as 32 + params.ROWS_PER_CTA = params.WARPS_PER_CTA * params.ROWS_PER_WARP; + + moe_fused_gate_impl( + input, + bias, + output_ptr, + indices_ptr, + num_rows, + topk_group, + topk, + num_fused_shared_experts, + routed_scaling_factor, + apply_routed_scaling_factor_on_output, + params); +} + +//------------------------------------------------------------------------------ +// Host Launcher Function +//------------------------------------------------------------------------------ +std::vector moe_fused_gate( + at::Tensor& input, + at::Tensor& bias, + int64_t num_expert_group, + int64_t topk_group, + int64_t topk, + int64_t num_fused_shared_experts, + double routed_scaling_factor, + bool apply_routed_scaling_factor_on_output) { + TORCH_CHECK(input.dtype() == bias.dtype(), "input and bias should have the same dtype"); + + int64_t num_rows = input.size(0); + int32_t num_experts = input.size(1); + auto options = torch::TensorOptions().dtype(torch::kFloat32).device(torch::kCUDA); + auto output = torch::empty({num_rows, topk}, options); + auto indices = torch::empty({num_rows, topk}, options.dtype(torch::kInt32)); + + // Compute grid dimensions based on runtime value for num_expert_group. + int64_t rows_per_warp = std::max(1, WARP_SIZE / num_expert_group); + int64_t num_warps = (num_rows + rows_per_warp - 1) / rows_per_warp; + int64_t num_blocks = (num_warps + WARPS_PER_CTA - 1) / WARPS_PER_CTA; + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + dim3 block_dim(WARP_SIZE, WARPS_PER_CTA); + + // Check 1: Ensure that num_experts is a power of 2. + TORCH_CHECK((num_experts & (num_experts - 1)) == 0, "num_experts must be a power of 2, but got ", num_experts); + + // Check 2: Ensure that num_experts is divisible by num_expert_group. (this also means num_expert_group is power of 2) + TORCH_CHECK( + num_experts % num_expert_group == 0, + "num_experts must be divisible by num_expert_group, but got ", + num_experts, + " / ", + num_expert_group); + + int computed_vpt = num_experts / num_expert_group; + // Check 3: Ensure that num_experts/num_expert_group does not exceed MAX_VPT=32. Maximum VPT indicate max value per + // threads we can process. + TORCH_CHECK( + computed_vpt <= MAX_VPT, + "Per group experts: num_experts / num_expert_group = (", + computed_vpt, + ") exceeds the maximum supported (", + MAX_VPT, + ")"); + + // Dispatch to templated kernel for known compile-time configurations. + // We currently only support for: + // Case 1: 256 experts, with 8 or 16 groups. + // Case 2: 128 experts, with 4 or 8 groups. + // Case 3: other cases, require 8 <= num_experts / num_expert_group <= 32 + bool dispatched = false; + switch (num_experts) { + case 256: + if (num_expert_group == 8) + // This is deepseek v3 case. Here VPT = 256/8 = 32, ROWS_PER_WARP = 32/8 = 4, ROWS_PER_CTA = 6 * 4 = 24. + if (input.scalar_type() == at::kBFloat16) { + LAUNCH_MOE_GATE_CONFIG(bfloat16_t, 256, 8); + } else if (input.scalar_type() == at::kHalf) { + LAUNCH_MOE_GATE_CONFIG(float16_t, 256, 8); + } else if (input.scalar_type() == at::kFloat) { + LAUNCH_MOE_GATE_CONFIG(float32_t, 256, 8); + } else if (num_expert_group == 16) + // Here VPT = 256/16 = 16, ROWS_PER_WARP = 32/16 = 2, ROWS_PER_CTA = 6 * 2 = 12. + if (input.scalar_type() == at::kBFloat16) { + LAUNCH_MOE_GATE_CONFIG(bfloat16_t, 256, 16); + } else if (input.scalar_type() == at::kHalf) { + LAUNCH_MOE_GATE_CONFIG(float16_t, 256, 16); + } else if (input.scalar_type() == at::kFloat) { + LAUNCH_MOE_GATE_CONFIG(float32_t, 256, 16); + } + break; + case 128: + if (num_expert_group == 4) + // VPT = 128/4 = 32, ROWS_PER_WARP = 32/16 = 2, ROWS_PER_CTA = 6 * 2 = 12. + if (input.scalar_type() == at::kBFloat16) { + LAUNCH_MOE_GATE_CONFIG(bfloat16_t, 128, 4); + } else if (input.scalar_type() == at::kHalf) { + LAUNCH_MOE_GATE_CONFIG(float16_t, 128, 4); + } else if (input.scalar_type() == at::kFloat) { + LAUNCH_MOE_GATE_CONFIG(float32_t, 128, 4); + } else if (num_expert_group == 8) + // VPT = 128/8 = 16, ROWS_PER_WARP = 32/8 = 4, ROWS_PER_CTA = 6 * 4 = 24. + if (input.scalar_type() == at::kBFloat16) { + LAUNCH_MOE_GATE_CONFIG(bfloat16_t, 128, 8); + } else if (input.scalar_type() == at::kHalf) { + LAUNCH_MOE_GATE_CONFIG(float16_t, 128, 8); + } else if (input.scalar_type() == at::kFloat) { + LAUNCH_MOE_GATE_CONFIG(float32_t, 128, 8); + } + break; + default: + break; + } + if (!dispatched) { + // Fallback to the dynamic kernel if none of the supported combinations match. + // currently only support num_experts / num_expert_group <= 32 for dynamic kernels + if (input.scalar_type() == at::kBFloat16) { + moe_fused_gate_kernel_dynamic<<>>( + input.data_ptr(), + bias.data_ptr(), + output.data_ptr(), + indices.data_ptr(), + num_rows, + num_experts, + num_expert_group, + topk_group, + topk, + num_fused_shared_experts, + routed_scaling_factor, + apply_routed_scaling_factor_on_output); + } else if (input.scalar_type() == at::kHalf) { + moe_fused_gate_kernel_dynamic<<>>( + input.data_ptr(), + bias.data_ptr(), + output.data_ptr(), + indices.data_ptr(), + num_rows, + num_experts, + num_expert_group, + topk_group, + topk, + num_fused_shared_experts, + routed_scaling_factor, + apply_routed_scaling_factor_on_output); + } else if (input.scalar_type() == at::kFloat) { + moe_fused_gate_kernel_dynamic<<>>( + input.data_ptr(), + bias.data_ptr(), + output.data_ptr(), + indices.data_ptr(), + num_rows, + num_experts, + num_expert_group, + topk_group, + topk, + num_fused_shared_experts, + routed_scaling_factor, + apply_routed_scaling_factor_on_output); + } else { + TORCH_CHECK(false, "Unsupported data type for moe_fused_gate"); + } + } + return {output, indices}; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_sum.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_sum.cu new file mode 100644 index 00000000..af8cacf1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_sum.cu @@ -0,0 +1,66 @@ +#include +#include +#include +#include + +#include +#include + +#include "utils.h" + +template +__global__ void moe_sum_kernel( + scalar_t* __restrict__ out, // [..., d] + const scalar_t* __restrict__ input, // [..., topk, d] + const int d) { + const int64_t token_idx = blockIdx.x; + for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) { + scalar_t x = 0.0; +#pragma unroll + for (int k = 0; k < TOPK; ++k) { + x += SGLANG_LDG(&input[token_idx * TOPK * d + k * d + idx]); + } + out[token_idx * d + idx] = x; + } +} + +void moe_sum( + torch::Tensor& input, // [num_tokens, topk, hidden_size] + torch::Tensor& output) // [num_tokens, hidden_size] +{ + const int hidden_size = input.size(-1); + const auto num_tokens = output.numel() / hidden_size; + const int topk = input.size(1); + + dim3 grid(num_tokens); + dim3 block(std::min(hidden_size, 1024)); + const at::cuda::OptionalCUDAGuard device_guard(device_of(output)); + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + switch (topk) { + case 2: + DISPATCH_FLOAT_TYPES(input.scalar_type(), "moe_sum_kernel", [&] { + moe_sum_kernel + <<>>(output.data_ptr(), input.data_ptr(), hidden_size); + }); + break; + + case 3: + DISPATCH_FLOAT_TYPES(input.scalar_type(), "moe_sum_kernel", [&] { + moe_sum_kernel + <<>>(output.data_ptr(), input.data_ptr(), hidden_size); + }); + break; + + case 4: + DISPATCH_FLOAT_TYPES(input.scalar_type(), "moe_sum_kernel", [&] { + moe_sum_kernel + <<>>(output.data_ptr(), input.data_ptr(), hidden_size); + }); + break; + + default: + at::sum_out(output, input, 1); + break; + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_sum_reduce.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_sum_reduce.cu new file mode 100644 index 00000000..791ce620 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_sum_reduce.cu @@ -0,0 +1,403 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "cutlass/array.h" +#include "utils.h" + +template +using opmath_t = at::opmath_type; + +template +__device__ __forceinline__ opmath_t to_acc(T x) { + return static_cast>(x); +} + +template +__device__ __forceinline__ T from_acc(opmath_t x) { + return static_cast(x); +} + +template <> +__device__ __forceinline__ opmath_t to_acc(at::Half x) { + return __half2float(__nv_half(x)); +} +template <> +__device__ __forceinline__ at::Half from_acc(opmath_t x) { + return __float2half_rn(x); +} + +template <> +__device__ __forceinline__ opmath_t to_acc(at::BFloat16 x) { + return __bfloat162float(__nv_bfloat16(x)); +} +template <> +__device__ __forceinline__ at::BFloat16 from_acc(opmath_t x) { + return __float2bfloat16_rn(x); +} + +template +__device__ __forceinline__ T ldg_cg(const T* p) { + return __ldg(p); +} + +union Pack16B { + uint4 v; + __nv_bfloat16 u16[8]; +}; + +template +__global__ void moe_sum_reduce_warp_per_token_vec_kernel( + const at::BFloat16* __restrict__ x, + at::BFloat16* __restrict__ y, + const int64_t token_num, + const int64_t hidden_dim, + const int64_t topk_num, + const int64_t stride_token, // in elements + const int64_t stride_topk, // in elements + const int64_t out_stride_token, // in elements + const float scale) { + constexpr int VEC = 16; + constexpr int PACKS = VEC / 8; + + const int warp_id = threadIdx.x / 32; + const int lane = threadIdx.x % 32; + const int64_t t = (int64_t)blockIdx.y * WARPS_PER_BLOCK + warp_id; + if (t >= token_num) return; + + const int64_t n_chunks = hidden_dim / VEC; + + for (int64_t chunk = (int64_t)blockIdx.x * 32 + lane; chunk < n_chunks; chunk += (int64_t)gridDim.x * 32) { + const int64_t d = chunk * VEC; + const int64_t base = t * stride_token + d; + + float acc[VEC]; +#pragma unroll + for (int i = 0; i < VEC; ++i) + acc[i] = 0.f; + +#pragma unroll + for (int k = 0; k < topk_num; ++k) { +#pragma unroll + for (int p = 0; p < PACKS; ++p) { + const int64_t offset = base + (int64_t)k * stride_topk + p * 8; + Pack16B pack = {ldg_cg(reinterpret_cast(x + offset))}; + +#pragma unroll + for (int i = 0; i < 8; ++i) { + acc[p * 8 + i] += __bfloat162float(pack.u16[i]); + } + } + } + +#pragma unroll + for (int i = 0; i < VEC; ++i) + acc[i] *= scale; + +#pragma unroll + for (int p = 0; p < PACKS; ++p) { + Pack16B outp; +#pragma unroll + for (int i = 0; i < 8; ++i) { + outp.u16[i] = __float2bfloat16_rn(acc[p * 8 + i]); + } + const int64_t dst = t * out_stride_token + d + p * 8; + *reinterpret_cast(y + dst) = outp.v; + } + } +} + +template +__global__ void moe_sum_reduce_kernel_warp_token_topk( + const scalar_t* __restrict__ x, + scalar_t* __restrict__ y, + const int64_t token_num, + const int64_t hidden_dim, + const int64_t stride_token, + const int64_t stride_topk, + const int64_t out_stride_token, + const opmath_t scale) { + const int warp_id = threadIdx.x / 32; + const int lane = threadIdx.x % 32; + const int64_t t = (int64_t)blockIdx.y * WARPS_PER_BLOCK + warp_id; + if (t >= token_num) return; + + for (int64_t d = (int64_t)blockIdx.x * 32 + lane; d < hidden_dim; d += (int64_t)gridDim.x * 32) { + opmath_t acc = opmath_t(0); + const int64_t base = t * stride_token + d; + +#pragma unroll + for (int k = 0; k < TOPK; ++k) { + acc += to_acc(x[base + (int64_t)k * stride_topk]); + } + acc *= scale; + y[t * out_stride_token + d] = from_acc(acc); + } +} + +template +__global__ void moe_sum_reduce_kernel( + const scalar_t* __restrict__ x, + scalar_t* __restrict__ y, + const int64_t token_num, + const int64_t hidden_dim, + const int64_t stride_token, + const int64_t stride_topk, + const int64_t out_stride_token, + const opmath_t scale) { + for (int t = blockIdx.y; t < token_num; t += gridDim.y) { + for (int d = blockIdx.x * blockDim.x + threadIdx.x; d < hidden_dim; d += blockDim.x * gridDim.x) { + const int64_t base = t * stride_token + d; + opmath_t acc = opmath_t(0); + +#pragma unroll + for (int k = 0; k < TOPK; ++k) { + acc += to_acc(x[base + (int64_t)k * stride_topk]); + } + + acc *= scale; + y[t * out_stride_token + d] = from_acc(acc); + } + } +} + +// -------------------- general-topk fallback kernels -------------------- +// small-token +template +__global__ void moe_sum_reduce_kernel_general( + const scalar_t* __restrict__ x, + scalar_t* __restrict__ y, + const int64_t token_num, + const int64_t hidden_dim, + const int64_t stride_token, + const int64_t stride_topk, + const int64_t out_stride_token, + const int topk_num, + const opmath_t scale) { + for (int t = blockIdx.y; t < token_num; t += gridDim.y) { + for (int d = blockIdx.x * blockDim.x + threadIdx.x; d < hidden_dim; d += blockDim.x * gridDim.x) { + const int64_t base = t * stride_token + d; + opmath_t acc = opmath_t(0); +#pragma unroll 1 + for (int k = 0; k < topk_num; ++k) { + acc += to_acc(x[base + (int64_t)k * stride_topk]); + } + acc *= scale; + y[t * out_stride_token + d] = from_acc(acc); + } + } +} + +// warp-per-token +template +__global__ void moe_sum_reduce_kernel_warp_token_general( + const scalar_t* __restrict__ x, + scalar_t* __restrict__ y, + const int64_t token_num, + const int64_t hidden_dim, + const int64_t stride_token, + const int64_t stride_topk, + const int64_t out_stride_token, + const int topk_num, + const opmath_t scale) { + const int warp_id = threadIdx.x / 32; + const int lane = threadIdx.x % 32; + const int64_t t = (int64_t)blockIdx.y * WARPS_PER_BLOCK + warp_id; + if (t >= token_num) return; + + for (int64_t d = (int64_t)blockIdx.x * 32 + lane; d < hidden_dim; d += (int64_t)gridDim.x * 32) { + opmath_t acc = opmath_t(0); + const int64_t base = t * stride_token + d; +#pragma unroll 1 + for (int k = 0; k < topk_num; ++k) { + acc += to_acc(x[base + (int64_t)k * stride_topk]); + } + acc *= scale; + y[t * out_stride_token + d] = from_acc(acc); + } +} + +void moe_sum_reduce(at::Tensor& input, at::Tensor& output, double routed_scaling_factor) { + TORCH_CHECK(input.is_cuda(), "input must be CUDA tensor"); + TORCH_CHECK(output.is_cuda(), "output must be CUDA tensor"); + TORCH_CHECK(input.dim() == 3, "input must be a 3D tensor like [token_num, topk_num, hidden_dim]"); + TORCH_CHECK(output.dim() == 2, "output must be [token_num, hidden_dim]"); + TORCH_CHECK(input.size(0) == output.size(0), "token dim mismatch"); + TORCH_CHECK(input.size(2) == output.size(1), "hidden_dim mismatch"); + + TORCH_CHECK(input.is_contiguous(), "expect input to be contiguous"); + TORCH_CHECK(output.is_contiguous(), "expect output to be contiguous"); + + const int64_t token_num = input.size(0); + const int64_t topk_num = input.size(1); + const int64_t hidden_dim = input.size(2); + + const int64_t in_stride_token = input.stride(0); + const int64_t in_stride_topk = input.stride(1); + const int64_t out_stride_token = output.stride(0); + + auto stream = at::cuda::getCurrentCUDAStream(); + + const bool fast_bf16_vec_ok = (input.scalar_type() == at::kBFloat16) && (token_num > 256) && (hidden_dim % 8 == 0); + + // Fast path for bf16 vectorize + if (fast_bf16_vec_ok) { + constexpr int WARPS_PER_BLOCK = 8; + constexpr int THREADS = WARPS_PER_BLOCK * 32; + + const int64_t n_chunks = hidden_dim / 8; + int64_t grid_x = (n_chunks + 32 - 1) / 32; + if (grid_x > 65535) grid_x = 65535; + + int64_t grid_y = (token_num + WARPS_PER_BLOCK - 1) / WARPS_PER_BLOCK; + if (grid_y > 65535) grid_y = 65535; + + dim3 block(THREADS); + dim3 grid(static_cast(grid_x), static_cast(grid_y)); + + auto stream = at::cuda::getCurrentCUDAStream(); + + const float scale = static_cast(routed_scaling_factor); + moe_sum_reduce_warp_per_token_vec_kernel<<>>( + reinterpret_cast(input.data_ptr()), + reinterpret_cast(output.data_ptr()), + token_num, + hidden_dim, + topk_num, + in_stride_token, + in_stride_topk, + out_stride_token, + scale); + + TORCH_CHECK(cudaGetLastError() == cudaSuccess, "moe_sum_reduce CUDA kernel (bf16 vec) launch failed"); + return; + } + + const bool per_token_use_one_warp = (token_num > 128); + + if (!per_token_use_one_warp) { + // ---------- small-token ---------- + const int block_size = 256; + int64_t grid_x = (hidden_dim + block_size - 1) / block_size; + grid_x = grid_x > 65535 ? 65535 : grid_x; + int64_t grid_y = token_num < 65535 ? token_num : 65535; + + dim3 block(block_size); + dim3 grid(static_cast(grid_x), static_cast(grid_y)); + +#define LAUNCH_SMALL_TOKEN_KERNEL(TOPK) \ + moe_sum_reduce_kernel<<>>( \ + input.data_ptr(), \ + output.data_ptr(), \ + token_num, \ + hidden_dim, \ + in_stride_token, \ + in_stride_topk, \ + out_stride_token, \ + scale); + + AT_DISPATCH_FLOATING_TYPES_AND2( + at::kHalf, at::kBFloat16, input.scalar_type(), "moe_sum_reduce_cuda_small_token", [&] { + using scalar_t_ = scalar_t; + using acc_t_ = opmath_t; + const acc_t_ scale = static_cast(routed_scaling_factor); + + switch (topk_num) { + case 2: + LAUNCH_SMALL_TOKEN_KERNEL(2); + break; + case 4: + LAUNCH_SMALL_TOKEN_KERNEL(4); + break; + case 8: + LAUNCH_SMALL_TOKEN_KERNEL(8); + break; + case 9: + LAUNCH_SMALL_TOKEN_KERNEL(9); + break; + default: // launch general kernel + moe_sum_reduce_kernel_general<<>>( + input.data_ptr(), + output.data_ptr(), + token_num, + hidden_dim, + in_stride_token, + in_stride_topk, + out_stride_token, + static_cast(topk_num), + scale); + } + }); +#undef LAUNCH_SMALL_TOKEN_KERNEL + + TORCH_CHECK(cudaGetLastError() == cudaSuccess, "moe_sum_reduce CUDA kernel (small-token) launch failed"); + + } else { + // ---------- warp-per-token ---------- + constexpr int WARPS_PER_BLOCK = 4; + constexpr int THREADS = WARPS_PER_BLOCK * 32; + + int64_t gx = (hidden_dim + 32 - 1) / 32; + gx = gx > 65535 ? 65535 : gx; + + int64_t gy = (token_num + WARPS_PER_BLOCK - 1) / WARPS_PER_BLOCK; + gy = gy > 65535 ? 65535 : gy; + + dim3 block(THREADS); + dim3 grid(static_cast(gx), static_cast(gy)); + +#define LAUNCH_WARP_PER_TOKEN_KERNEL(TOPK) \ + moe_sum_reduce_kernel_warp_token_topk<<>>( \ + input.data_ptr(), \ + output.data_ptr(), \ + token_num, \ + hidden_dim, \ + in_stride_token, \ + in_stride_topk, \ + out_stride_token, \ + scale); + + AT_DISPATCH_FLOATING_TYPES_AND2( + at::kHalf, at::kBFloat16, input.scalar_type(), "moe_sum_reduce_cuda_large_token", [&] { + using scalar_t_ = scalar_t; + using acc_t_ = opmath_t; + const acc_t_ scale = static_cast(routed_scaling_factor); + + switch (topk_num) { + case 2: + LAUNCH_WARP_PER_TOKEN_KERNEL(2); + break; + case 4: + LAUNCH_WARP_PER_TOKEN_KERNEL(4); + break; + case 8: + LAUNCH_WARP_PER_TOKEN_KERNEL(8); + break; + case 9: + LAUNCH_WARP_PER_TOKEN_KERNEL(9); + break; + default: // launch general kernel + moe_sum_reduce_kernel_warp_token_general<<>>( + input.data_ptr(), + output.data_ptr(), + token_num, + hidden_dim, + in_stride_token, + in_stride_topk, + out_stride_token, + static_cast(topk_num), + scale); + } + }); +#undef LAUNCH_WARP_PER_TOKEN_KERNEL + + TORCH_CHECK(cudaGetLastError() == cudaSuccess, "moe_sum_reduce CUDA kernel (warp-token) launch failed"); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_topk_sigmoid_kernels.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_topk_sigmoid_kernels.cu new file mode 100644 index 00000000..81c584b1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_topk_sigmoid_kernels.cu @@ -0,0 +1,592 @@ +// Adapt from https://github.com/vllm-project/vllm/blob/v0.7.3/csrc/moe/topk_softmax_kernels.cu +// which is originally adapted from +// https://github.com/NVIDIA/TensorRT-LLM/blob/v0.7.1/cpp/tensorrt_llm/kernels/mixtureOfExperts/moe_kernels.cu +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#ifndef USE_ROCM +#include +#include +#include +#else +#include +#include +#endif + +#include "utils.h" + +#define MAX(a, b) ((a) > (b) ? (a) : (b)) +#define MIN(a, b) ((a) < (b) ? (a) : (b)) + +// Define reduction operators based on CUDA version +// CUDA 13 (12.9+) deprecated cub::Max/Min in favor of cuda::maximum/minimum +#if CUDA_VERSION >= 12090 +using MaxReduceOp = cuda::maximum<>; +using MinReduceOp = cuda::minimum<>; +#else +using MaxReduceOp = cub::Max; +using MinReduceOp = cub::Min; +#endif + +/// Aligned array type +template < + typename T, + /// Number of elements in the array + int N, + /// Alignment requirement in bytes + int Alignment = sizeof(T) * N> +class alignas(Alignment) AlignedArray { + T data[N]; +}; + +// ========================== Util functions to convert types ========================== +template +__device__ float convert_to_float(T x) { + if constexpr (std::is_same_v) { + return __half2float(x); + } else if constexpr (std::is_same_v) { + return __bfloat162float(x); + } else if constexpr (std::is_same_v) { + return x; + } else { + return static_cast(x); + } +} + +// ====================== Sigmoid things =============================== +// We have our own implementation of sigmoid here so we can support transposing the output +// in the sigmoid kernel when we extend this module to support expert-choice routing. +template +__launch_bounds__(TPB) __global__ void moeSigmoid( + const T* input, const bool* finished, float* output, const int num_cols, const float* correction_bias) { + const int thread_row_offset = blockIdx.x * num_cols; + + // Don't touch finished rows. + if ((finished != nullptr) && finished[blockIdx.x]) { + return; + } + + // First pass: Apply transformation, find max, and write transformed values to output + for (int ii = threadIdx.x; ii < num_cols; ii += TPB) { + const int idx = thread_row_offset + ii; + float val = convert_to_float(input[idx]); + + val = 1.0f / (1.0f + expf(-val)); + + // Apply correction bias if provided + if (correction_bias != nullptr) { + val = val + correction_bias[ii]; + } + + output[idx] = val; // Store transformed value + } +} + +template +__launch_bounds__(TPB) __global__ void moeTopK( + const float* inputs_after_sigmoid, + const bool* finished, + float* output, + int* indices, + const int num_experts, + const int k, + const int start_expert, + const int end_expert, + const bool renormalize, + const float* correction_bias) { + using cub_kvp = cub::KeyValuePair; + using BlockReduce = cub::BlockReduce; + __shared__ typename BlockReduce::TempStorage tmpStorage; + + cub_kvp thread_kvp; + cub::ArgMax arg_max; + + const int block_row = blockIdx.x; + + const bool row_is_active = finished ? !finished[block_row] : true; + const int thread_read_offset = blockIdx.x * num_experts; + float row_sum_for_renormalize = 0; + for (int k_idx = 0; k_idx < k; ++k_idx) { + thread_kvp.key = 0; + thread_kvp.value = -1.f; // This is OK because inputs are probabilities + + cub_kvp inp_kvp; + for (int expert = threadIdx.x; expert < num_experts; expert += TPB) { + const int idx = thread_read_offset + expert; + inp_kvp.key = expert; + inp_kvp.value = inputs_after_sigmoid[idx]; + + for (int prior_k = 0; prior_k < k_idx; ++prior_k) { + const int prior_winning_expert = indices[k * block_row + prior_k]; + + if (prior_winning_expert == expert) { + inp_kvp = thread_kvp; + } + } + + thread_kvp = arg_max(inp_kvp, thread_kvp); + } + + const cub_kvp result_kvp = BlockReduce(tmpStorage).Reduce(thread_kvp, arg_max); + if (threadIdx.x == 0) { + // Ignore experts the node isn't responsible for with expert parallelism + const int expert = result_kvp.key; + const bool node_uses_expert = expert >= start_expert && expert < end_expert; + const bool should_process_row = row_is_active && node_uses_expert; + + const int idx = k * block_row + k_idx; + float val = result_kvp.value; + if (correction_bias != nullptr) { + val -= correction_bias[expert]; + } + output[idx] = val; + indices[idx] = should_process_row ? (expert - start_expert) : num_experts; + assert(indices[idx] >= 0); + row_sum_for_renormalize += val; + } + __syncthreads(); + } + + if (renormalize && threadIdx.x == 0) { + float row_sum_for_renormalize_inv = 1.f / row_sum_for_renormalize; + for (int k_idx = 0; k_idx < k; ++k_idx) { + const int idx = k * block_row + k_idx; + output[idx] = output[idx] * row_sum_for_renormalize_inv; + } + } +} + +// ====================== TopK sigmoid things =============================== + +/* + A Top-K gating sigmoid written to exploit when the number of experts in the MoE layers + are a small power of 2. This allows us to cleanly share the rows among the threads in + a single warp and eliminate communication between warps (so no need to use shared mem). + + It fuses the sigmoid, max and argmax into a single kernel. + + Limitations: + 1) This implementation is intended for when the number of experts is a small power of 2. + 2) This implementation assumes k is small, but will work for any k. +*/ + +template +__launch_bounds__(WARPS_PER_CTA* WARP_SIZE) __global__ void topkGatingSigmoid( + const T* input, + const bool* finished, + float* output, + const int num_rows, + int* indices, + const int k, + const int start_expert, + const int end_expert, + const bool renormalize, + const float* correction_bias) { + // We begin by enforcing compile time assertions and setting up compile time constants. + static_assert(VPT == (VPT & -VPT), "VPT must be power of 2"); + static_assert(NUM_EXPERTS == (NUM_EXPERTS & -NUM_EXPERTS), "NUM_EXPERTS must be power of 2"); + static_assert(BYTES_PER_LDG == (BYTES_PER_LDG & -BYTES_PER_LDG), "BYTES_PER_LDG must be power of 2"); + static_assert(BYTES_PER_LDG <= 16, "BYTES_PER_LDG must be leq 16"); + + // Number of bytes each thread pulls in per load + static constexpr int ELTS_PER_LDG = BYTES_PER_LDG / sizeof(T); + static constexpr int ELTS_PER_ROW = NUM_EXPERTS; + static constexpr int THREADS_PER_ROW = ELTS_PER_ROW / VPT; + static constexpr int LDG_PER_THREAD = VPT / ELTS_PER_LDG; + + // Restrictions based on previous section. + static_assert(VPT % ELTS_PER_LDG == 0, "The elements per thread must be a multiple of the elements per ldg"); + static_assert(WARP_SIZE % THREADS_PER_ROW == 0, "The threads per row must cleanly divide the threads per warp"); + static_assert(THREADS_PER_ROW == (THREADS_PER_ROW & -THREADS_PER_ROW), "THREADS_PER_ROW must be power of 2"); + static_assert(THREADS_PER_ROW <= WARP_SIZE, "THREADS_PER_ROW can be at most warp size"); + + // We have NUM_EXPERTS elements per row. We specialize for small #experts + static constexpr int ELTS_PER_WARP = WARP_SIZE * VPT; + static constexpr int ROWS_PER_WARP = ELTS_PER_WARP / ELTS_PER_ROW; + static constexpr int ROWS_PER_CTA = WARPS_PER_CTA * ROWS_PER_WARP; + + // Restrictions for previous section. + static_assert(ELTS_PER_WARP % ELTS_PER_ROW == 0, "The elts per row must cleanly divide the total elt per warp"); + + // ===================== From this point, we finally start computing run-time variables. ======================== + + // Compute CTA and warp rows. We pack multiple rows into a single warp, and a block contains WARPS_PER_CTA warps. + // This, each block processes a chunk of rows. We start by computing the start row for each block. + const int cta_base_row = blockIdx.x * ROWS_PER_CTA; + + // Now, using the base row per thread block, we compute the base row per warp. + const int warp_base_row = cta_base_row + threadIdx.y * ROWS_PER_WARP; + + // The threads in a warp are split into sub-groups that will work on a row. + // We compute row offset for each thread sub-group + const int thread_row_in_warp = threadIdx.x / THREADS_PER_ROW; + const int thread_row = warp_base_row + thread_row_in_warp; + + // Threads with indices out of bounds should early exit here. + if (thread_row >= num_rows) { + return; + } + const bool row_is_active = finished ? !finished[thread_row] : true; + + // We finally start setting up the read pointers for each thread. First, each thread jumps to the start of the + // row it will read. + const T* thread_row_ptr = input + thread_row * ELTS_PER_ROW; + + // Now, we compute the group each thread belong to in order to determine the first column to start loads. + const int thread_group_idx = threadIdx.x % THREADS_PER_ROW; + const int first_elt_read_by_thread = thread_group_idx * ELTS_PER_LDG; + const T* thread_read_ptr = thread_row_ptr + first_elt_read_by_thread; + + // Determine the pointer type to use to read in the data depending on the BYTES_PER_LDG template param. In theory, + // this can support all powers of 2 up to 16. + // NOTE(woosuk): The original implementation uses CUTLASS aligned array here. + // We defined our own aligned array and use it here to avoid the dependency on CUTLASS. + using AccessType = AlignedArray; + + // Finally, we pull in the data from global mem + T row_chunk_temp[VPT]; + AccessType* row_chunk_vec_ptr = reinterpret_cast(&row_chunk_temp); + const AccessType* vec_thread_read_ptr = reinterpret_cast(thread_read_ptr); +#pragma unroll + // Note(Byron): interleaved loads to achieve better memory coalescing + // | thread[0] | thread[1] | thread[2] | thread[3] | thread[0] | thread[1] | thread[2] | thread[3] | ... + for (int ii = 0; ii < LDG_PER_THREAD; ++ii) { + row_chunk_vec_ptr[ii] = vec_thread_read_ptr[ii * THREADS_PER_ROW]; + } + + float row_chunk[VPT]; +#pragma unroll + // Note(Byron): upcast logits to float32 + for (int ii = 0; ii < VPT; ++ii) { + float val = convert_to_float(row_chunk_temp[ii]); + val = 1.0f / (1.0f + expf(-val)); + // Apply correction bias if provided + if (correction_bias != nullptr) { + /* + LDG is interleaved + |thread0 LDG| |thread1 LDG| |thread0 LDG| |thread1 LDG| + |--------- group0 --------| |----------group1 --------| + ^ local2 + */ + const int group_id = ii / ELTS_PER_LDG; + const int local_id = ii % ELTS_PER_LDG; + const int expert_idx = first_elt_read_by_thread + group_id * THREADS_PER_ROW * ELTS_PER_LDG + local_id; + val = val + correction_bias[expert_idx]; + } + + row_chunk[ii] = val; + } + + // Now, row_chunk contains the sigmoid of the row chunk. Now, I want to find the topk elements in each row, along + // with the max index. + int start_col = first_elt_read_by_thread; + static constexpr int COLS_PER_GROUP_LDG = ELTS_PER_LDG * THREADS_PER_ROW; + + float row_sum_for_renormalize = 0; + + for (int k_idx = 0; k_idx < k; ++k_idx) { + // First, each thread does the local argmax + float max_val = row_chunk[0]; + int expert = start_col; +#pragma unroll + for (int ldg = 0, col = start_col; ldg < LDG_PER_THREAD; ++ldg, col += COLS_PER_GROUP_LDG) { +#pragma unroll + for (int ii = 0; ii < ELTS_PER_LDG; ++ii) { + float val = row_chunk[ldg * ELTS_PER_LDG + ii]; + + // No check on the experts here since columns with the smallest index are processed first and only + // updated if > (not >=) + if (val > max_val) { + max_val = val; + expert = col + ii; + } + } + } + +// Now, we perform the argmax reduce. We use the butterfly pattern so threads reach consensus about the max. +// This will be useful for K > 1 so that the threads can agree on "who" had the max value. That thread can +// then blank out their max with -inf and the warp can run more iterations... +#pragma unroll + for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2) { + float other_max = SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, max_val, mask, THREADS_PER_ROW); + int other_expert = SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, expert, mask, THREADS_PER_ROW); + + // We want lower indices to "win" in every thread so we break ties this way + if (other_max > max_val || (other_max == max_val && other_expert < expert)) { + max_val = other_max; + expert = other_expert; + } + } + + // Write the max for this k iteration to global memory. + if (thread_group_idx == 0) { + // Add a guard to ignore experts not included by this node + const bool node_uses_expert = expert >= start_expert && expert < end_expert; + const bool should_process_row = row_is_active && node_uses_expert; + + // The lead thread from each sub-group will write out the final results to global memory. (This will be a + // single) thread per row of the input/output matrices. + const int idx = k * thread_row + k_idx; + if (correction_bias != nullptr) { + max_val -= correction_bias[expert]; + } + output[idx] = max_val; + indices[idx] = should_process_row ? (expert - start_expert) : NUM_EXPERTS; + row_sum_for_renormalize += max_val; + } + + // Finally, we clear the value in the thread with the current max if there is another iteration to run. + if (k_idx + 1 < k) { + const int ldg_group_for_expert = expert / COLS_PER_GROUP_LDG; + const int thread_to_clear_in_group = (expert / ELTS_PER_LDG) % THREADS_PER_ROW; + + // Only the thread in the group which produced the max will reset the "winning" value to -inf. + if (thread_group_idx == thread_to_clear_in_group) { + const int offset_for_expert = expert % ELTS_PER_LDG; + // Safe to set to any negative value since row_chunk values must be between 0 and 1. + row_chunk[ldg_group_for_expert * ELTS_PER_LDG + offset_for_expert] = -10000.f; + } + } + } + + // Fuse renormalization of topk_weights into this kernel + if (renormalize && thread_group_idx == 0) { + float row_sum_for_renormalize_inv = 1.f / row_sum_for_renormalize; +#pragma unroll + for (int k_idx = 0; k_idx < k; ++k_idx) { + const int idx = k * thread_row + k_idx; + output[idx] = output[idx] * row_sum_for_renormalize_inv; + } + } +} + +namespace detail { +// Constructs some constants needed to partition the work across threads at compile time. +template +struct TopkConstants { + static constexpr int ELTS_PER_LDG = BYTES_PER_LDG / sizeof(T); + static_assert(EXPERTS / (ELTS_PER_LDG * WARP_SIZE) == 0 || EXPERTS % (ELTS_PER_LDG * WARP_SIZE) == 0, ""); + static constexpr int VECs_PER_THREAD = MAX(1, EXPERTS / (ELTS_PER_LDG * WARP_SIZE)); + static constexpr int VPT = VECs_PER_THREAD * ELTS_PER_LDG; + static constexpr int THREADS_PER_ROW = EXPERTS / VPT; + static constexpr int ROWS_PER_WARP = WARP_SIZE / THREADS_PER_ROW; +}; +} // namespace detail + +template +void topkGatingSigmoidLauncherHelper( + const T* input, + const bool* finished, + float* output, + int* indices, + const int num_rows, + const int k, + const int start_expert, + const int end_expert, + const bool renormalize, + const float* correction_bias, + cudaStream_t stream) { + static constexpr std::size_t MAX_BYTES_PER_LDG = 16; + + static constexpr int BYTES_PER_LDG = MIN(MAX_BYTES_PER_LDG, sizeof(T) * EXPERTS); + using Constants = detail::TopkConstants; + static constexpr int VPT = Constants::VPT; + static constexpr int ROWS_PER_WARP = Constants::ROWS_PER_WARP; + const int num_warps = (num_rows + ROWS_PER_WARP - 1) / ROWS_PER_WARP; + const int num_blocks = (num_warps + WARPS_PER_TB - 1) / WARPS_PER_TB; + + dim3 block_dim(WARP_SIZE, WARPS_PER_TB); + topkGatingSigmoid<<>>( + input, finished, output, num_rows, indices, k, start_expert, end_expert, renormalize, correction_bias); +} + +#define LAUNCH_SIGMOID(TYPE, NUM_EXPERTS, WARPS_PER_TB) \ + topkGatingSigmoidLauncherHelper( \ + gating_output, \ + nullptr, \ + topk_weights, \ + topk_indices, \ + num_tokens, \ + topk, \ + 0, \ + num_experts, \ + renormalize, \ + correction_bias, \ + stream); + +template +void topkGatingSigmoidKernelLauncher( + const T* gating_output, + float* topk_weights, + int* topk_indices, + float* sigmoid_workspace, + const int num_tokens, + const int num_experts, + const int topk, + const bool renormalize, + const float* correction_bias, + cudaStream_t stream) { + static constexpr int WARPS_PER_TB = 4; + switch (num_experts) { + case 1: + LAUNCH_SIGMOID(T, 1, WARPS_PER_TB); + break; + case 2: + LAUNCH_SIGMOID(T, 2, WARPS_PER_TB); + break; + case 4: + LAUNCH_SIGMOID(T, 4, WARPS_PER_TB); + break; + case 8: + LAUNCH_SIGMOID(T, 8, WARPS_PER_TB); + break; + case 16: + LAUNCH_SIGMOID(T, 16, WARPS_PER_TB); + break; + case 32: + LAUNCH_SIGMOID(T, 32, WARPS_PER_TB); + break; + case 64: + LAUNCH_SIGMOID(T, 64, WARPS_PER_TB); + break; + case 128: + LAUNCH_SIGMOID(T, 128, WARPS_PER_TB); + break; + case 256: + LAUNCH_SIGMOID(T, 256, WARPS_PER_TB); + break; + default: { + TORCH_CHECK( + sigmoid_workspace != nullptr, + "sigmoid_workspace must be provided for num_experts that are not a power of 2."); + static constexpr int TPB = 256; + moeSigmoid + <<>>(gating_output, nullptr, sigmoid_workspace, num_experts, correction_bias); + moeTopK<<>>( + sigmoid_workspace, + nullptr, + topk_weights, + topk_indices, + num_experts, + topk, + 0, + num_experts, + renormalize, + correction_bias); + } + } +} + +void topk_sigmoid( + torch::Tensor& topk_weights, // [num_tokens, topk] + torch::Tensor& topk_indices, // [num_tokens, topk] + torch::Tensor& gating_output, // [num_tokens, num_experts] + const bool renormalize, + const c10::optional& correction_bias) { + // Check data type + TORCH_CHECK( + gating_output.scalar_type() == at::ScalarType::Float || gating_output.scalar_type() == at::ScalarType::Half || + gating_output.scalar_type() == at::ScalarType::BFloat16, + "gating_output must be float32, float16, or bfloat16"); + + // Check dimensions + TORCH_CHECK(gating_output.dim() == 2, "gating_output must be 2D tensor [num_tokens, num_experts]"); + TORCH_CHECK(topk_weights.dim() == 2, "topk_weights must be 2D tensor [num_tokens, topk]"); + TORCH_CHECK(topk_indices.dim() == 2, "topk_indices must be 2D tensor [num_tokens, topk]"); + + // Check shapes + TORCH_CHECK( + gating_output.size(0) == topk_weights.size(0), + "First dimension of topk_weights must match num_tokens in gating_output"); + TORCH_CHECK( + gating_output.size(0) == topk_indices.size(0), + "First dimension of topk_indices must match num_tokens in gating_output"); + TORCH_CHECK( + topk_weights.size(-1) == topk_indices.size(-1), + "Second dimension of topk_indices must match topk in topk_weights"); + TORCH_CHECK(topk_weights.size(-1) <= gating_output.size(-1), "topk must be less than or equal to num_experts"); + + const int num_experts = static_cast(gating_output.size(-1)); + const int num_tokens = static_cast(gating_output.size(0)); + const int topk = static_cast(topk_weights.size(-1)); + + const bool is_pow_2 = (num_experts != 0) && ((num_experts & (num_experts - 1)) == 0); + const bool needs_workspace = !is_pow_2 || num_experts > 256; + const int64_t workspace_size = needs_workspace ? num_tokens * num_experts : 0; + + const at::cuda::OptionalCUDAGuard device_guard(device_of(gating_output)); + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + torch::Tensor sigmoid_workspace = + torch::empty({workspace_size}, gating_output.options().dtype(at::ScalarType::Float)); + + const at::ScalarType dtype = gating_output.scalar_type(); + + // Validate correction_bias if provided - must always be float32 + const float* bias_ptr = nullptr; + if (correction_bias.has_value()) { + const torch::Tensor& bias_tensor = correction_bias.value(); + TORCH_CHECK(bias_tensor.dim() == 1, "correction_bias must be 1D tensor [num_experts]"); + TORCH_CHECK(bias_tensor.size(0) == num_experts, "correction_bias size must match num_experts"); + TORCH_CHECK( + bias_tensor.scalar_type() == at::ScalarType::Float, + "correction_bias must be float32, got ", + bias_tensor.scalar_type()); + bias_ptr = bias_tensor.data_ptr(); + } + + if (dtype == at::ScalarType::Float) { + topkGatingSigmoidKernelLauncher( + gating_output.data_ptr(), + topk_weights.data_ptr(), + topk_indices.data_ptr(), + sigmoid_workspace.data_ptr(), + num_tokens, + num_experts, + topk, + renormalize, + bias_ptr, + stream); + } else if (dtype == at::ScalarType::Half) { + topkGatingSigmoidKernelLauncher<__half>( + reinterpret_cast(gating_output.data_ptr()), + topk_weights.data_ptr(), + topk_indices.data_ptr(), + sigmoid_workspace.data_ptr(), + num_tokens, + num_experts, + topk, + renormalize, + bias_ptr, + stream); + } else if (dtype == at::ScalarType::BFloat16) { + topkGatingSigmoidKernelLauncher<__nv_bfloat16>( + reinterpret_cast(gating_output.data_ptr()), + topk_weights.data_ptr(), + topk_indices.data_ptr(), + sigmoid_workspace.data_ptr(), + num_tokens, + num_experts, + topk, + renormalize, + bias_ptr, + stream); + } else { + TORCH_CHECK(false, "Unsupported gating_output dtype: ", dtype); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_topk_softmax_kernels.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_topk_softmax_kernels.cu new file mode 100644 index 00000000..7afed8c8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/moe_topk_softmax_kernels.cu @@ -0,0 +1,704 @@ +// Adapt from https://github.com/vllm-project/vllm/blob/v0.7.3/csrc/moe/topk_softmax_kernels.cu +// which is originally adapted from +// https://github.com/NVIDIA/TensorRT-LLM/blob/v0.7.1/cpp/tensorrt_llm/kernels/mixtureOfExperts/moe_kernels.cu +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#ifndef USE_ROCM +#include +#include +#include +#else +#include +#include +#endif + +#include "utils.h" + +#define MAX(a, b) ((a) > (b) ? (a) : (b)) +#define MIN(a, b) ((a) < (b) ? (a) : (b)) + +// Define reduction operators based on CUDA version +// CUDA 13 (12.9+) deprecated cub::Max/Min in favor of cuda::maximum/minimum +#if CUDA_VERSION >= 12090 +using MaxReduceOp = cuda::maximum<>; +using MinReduceOp = cuda::minimum<>; +#else +using MaxReduceOp = cub::Max; +using MinReduceOp = cub::Min; +#endif + +/// Aligned array type +template < + typename T, + /// Number of elements in the array + int N, + /// Alignment requirement in bytes + int Alignment = sizeof(T) * N> +class alignas(Alignment) AlignedArray { + T data[N]; +}; + +// ========================== Util functions to convert types ========================== +template +__device__ float convert_to_float(T x) { + if constexpr (std::is_same_v) { + return __half2float(x); + } else if constexpr (std::is_same_v) { + return __bfloat162float(x); + } else if constexpr (std::is_same_v) { + return x; + } else { + return static_cast(x); + } +} + +// ====================== Softmax things =============================== +// We have our own implementation of softmax here so we can support transposing the output +// in the softmax kernel when we extend this module to support expert-choice routing. +template +__launch_bounds__(TPB) __global__ void moeSoftmax( + const T* input, + const bool* finished, + float* output, + const int num_cols, + const float moe_softcapping, + const float* correction_bias) { + using BlockReduce = cub::BlockReduce; + __shared__ typename BlockReduce::TempStorage tmpStorage; + + __shared__ float normalizing_factor; + __shared__ float float_max; + + const int thread_row_offset = blockIdx.x * num_cols; + + float threadData(-FLT_MAX); + + // Don't touch finished rows. + if ((finished != nullptr) && finished[blockIdx.x]) { + return; + } + + // First pass: Apply transformation, find max, and write transformed values to output + for (int ii = threadIdx.x; ii < num_cols; ii += TPB) { + const int idx = thread_row_offset + ii; + float val = convert_to_float(input[idx]); + + // Apply tanh softcapping if enabled + if (moe_softcapping != 0.0f) { + val = tanhf(val / moe_softcapping) * moe_softcapping; + } + + // Apply correction bias if provided + if (correction_bias != nullptr) { + val = val + correction_bias[ii]; + } + + output[idx] = val; // Store transformed value + threadData = max(val, threadData); + } + + const float maxElem = BlockReduce(tmpStorage).Reduce(threadData, MaxReduceOp()); + + if (threadIdx.x == 0) { + float_max = maxElem; + } + __syncthreads(); + + // Second pass: Compute sum using transformed values from output + threadData = 0; + for (int ii = threadIdx.x; ii < num_cols; ii += TPB) { + const int idx = thread_row_offset + ii; + threadData += exp((output[idx] - float_max)); + } + + const auto Z = BlockReduce(tmpStorage).Sum(threadData); + + if (threadIdx.x == 0) { + normalizing_factor = 1.f / Z; + } + __syncthreads(); + + // Third pass: Compute final softmax using transformed values from output + for (int ii = threadIdx.x; ii < num_cols; ii += TPB) { + const int idx = thread_row_offset + ii; + const float softmax_val = exp((output[idx] - float_max)) * normalizing_factor; + output[idx] = softmax_val; + } +} + +template +__launch_bounds__(TPB) __global__ void moeTopK( + const float* inputs_after_softmax, + const bool* finished, + float* output, + int* indices, + const int num_experts, + const int k, + const int start_expert, + const int end_expert, + const bool renormalize) { + using cub_kvp = cub::KeyValuePair; + using BlockReduce = cub::BlockReduce; + __shared__ typename BlockReduce::TempStorage tmpStorage; + + cub_kvp thread_kvp; + cub::ArgMax arg_max; + + const int block_row = blockIdx.x; + + const bool row_is_active = finished ? !finished[block_row] : true; + const int thread_read_offset = blockIdx.x * num_experts; + float row_sum_for_renormalize = 0; + for (int k_idx = 0; k_idx < k; ++k_idx) { + thread_kvp.key = 0; + thread_kvp.value = -1.f; // This is OK because inputs are probabilities + + cub_kvp inp_kvp; + for (int expert = threadIdx.x; expert < num_experts; expert += TPB) { + const int idx = thread_read_offset + expert; + inp_kvp.key = expert; + inp_kvp.value = inputs_after_softmax[idx]; + + for (int prior_k = 0; prior_k < k_idx; ++prior_k) { + const int prior_winning_expert = indices[k * block_row + prior_k]; + + if (prior_winning_expert == expert) { + inp_kvp = thread_kvp; + } + } + + thread_kvp = arg_max(inp_kvp, thread_kvp); + } + + const cub_kvp result_kvp = BlockReduce(tmpStorage).Reduce(thread_kvp, arg_max); + if (threadIdx.x == 0) { + // Ignore experts the node isn't responsible for with expert parallelism + const int expert = result_kvp.key; + const bool node_uses_expert = expert >= start_expert && expert < end_expert; + const bool should_process_row = row_is_active && node_uses_expert; + + const int idx = k * block_row + k_idx; + output[idx] = result_kvp.value; + indices[idx] = should_process_row ? (expert - start_expert) : num_experts; + assert(indices[idx] >= 0); + row_sum_for_renormalize += result_kvp.value; + } + __syncthreads(); + } + + if (renormalize && threadIdx.x == 0) { + float row_sum_for_renormalize_inv = 1.f / row_sum_for_renormalize; + for (int k_idx = 0; k_idx < k; ++k_idx) { + const int idx = k * block_row + k_idx; + output[idx] = output[idx] * row_sum_for_renormalize_inv; + } + } +} + +// ====================== TopK softmax things =============================== + +/* + A Top-K gating softmax written to exploit when the number of experts in the MoE layers + are a small power of 2. This allows us to cleanly share the rows among the threads in + a single warp and eliminate communication between warps (so no need to use shared mem). + + It fuses the softmax, max and argmax into a single kernel. + + Limitations: + 1) This implementation is intended for when the number of experts is a small power of 2. + 2) This implementation assumes k is small, but will work for any k. +*/ + +template +__launch_bounds__(WARPS_PER_CTA* WARP_SIZE) __global__ void topkGatingSoftmax( + const T* input, + const bool* finished, + float* output, + const int num_rows, + int* indices, + const int k, + const int start_expert, + const int end_expert, + const bool renormalize, + const float moe_softcapping, + const float* correction_bias) { + // We begin by enforcing compile time assertions and setting up compile time constants. + static_assert(VPT == (VPT & -VPT), "VPT must be power of 2"); + static_assert(NUM_EXPERTS == (NUM_EXPERTS & -NUM_EXPERTS), "NUM_EXPERTS must be power of 2"); + static_assert(BYTES_PER_LDG == (BYTES_PER_LDG & -BYTES_PER_LDG), "BYTES_PER_LDG must be power of 2"); + static_assert(BYTES_PER_LDG <= 16, "BYTES_PER_LDG must be leq 16"); + + // Number of bytes each thread pulls in per load + static constexpr int ELTS_PER_LDG = BYTES_PER_LDG / sizeof(T); + static constexpr int ELTS_PER_ROW = NUM_EXPERTS; + static constexpr int THREADS_PER_ROW = ELTS_PER_ROW / VPT; + static constexpr int LDG_PER_THREAD = VPT / ELTS_PER_LDG; + + // Restrictions based on previous section. + static_assert(VPT % ELTS_PER_LDG == 0, "The elements per thread must be a multiple of the elements per ldg"); + static_assert(WARP_SIZE % THREADS_PER_ROW == 0, "The threads per row must cleanly divide the threads per warp"); + static_assert(THREADS_PER_ROW == (THREADS_PER_ROW & -THREADS_PER_ROW), "THREADS_PER_ROW must be power of 2"); + static_assert(THREADS_PER_ROW <= WARP_SIZE, "THREADS_PER_ROW can be at most warp size"); + + // We have NUM_EXPERTS elements per row. We specialize for small #experts + static constexpr int ELTS_PER_WARP = WARP_SIZE * VPT; + static constexpr int ROWS_PER_WARP = ELTS_PER_WARP / ELTS_PER_ROW; + static constexpr int ROWS_PER_CTA = WARPS_PER_CTA * ROWS_PER_WARP; + + // Restrictions for previous section. + static_assert(ELTS_PER_WARP % ELTS_PER_ROW == 0, "The elts per row must cleanly divide the total elt per warp"); + + // ===================== From this point, we finally start computing run-time variables. ======================== + + // Compute CTA and warp rows. We pack multiple rows into a single warp, and a block contains WARPS_PER_CTA warps. + // This, each block processes a chunk of rows. We start by computing the start row for each block. + const int cta_base_row = blockIdx.x * ROWS_PER_CTA; + + // Now, using the base row per thread block, we compute the base row per warp. + const int warp_base_row = cta_base_row + threadIdx.y * ROWS_PER_WARP; + + // The threads in a warp are split into sub-groups that will work on a row. + // We compute row offset for each thread sub-group + const int thread_row_in_warp = threadIdx.x / THREADS_PER_ROW; + const int thread_row = warp_base_row + thread_row_in_warp; + + // Threads with indices out of bounds should early exit here. + if (thread_row >= num_rows) { + return; + } + const bool row_is_active = finished ? !finished[thread_row] : true; + + // We finally start setting up the read pointers for each thread. First, each thread jumps to the start of the + // row it will read. + const T* thread_row_ptr = input + thread_row * ELTS_PER_ROW; + + // Now, we compute the group each thread belong to in order to determine the first column to start loads. + const int thread_group_idx = threadIdx.x % THREADS_PER_ROW; + const int first_elt_read_by_thread = thread_group_idx * ELTS_PER_LDG; + const T* thread_read_ptr = thread_row_ptr + first_elt_read_by_thread; + + // Determine the pointer type to use to read in the data depending on the BYTES_PER_LDG template param. In theory, + // this can support all powers of 2 up to 16. + // NOTE(woosuk): The original implementation uses CUTLASS aligned array here. + // We defined our own aligned array and use it here to avoid the dependency on CUTLASS. + using AccessType = AlignedArray; + + // Finally, we pull in the data from global mem + T row_chunk_temp[VPT]; + AccessType* row_chunk_vec_ptr = reinterpret_cast(&row_chunk_temp); + const AccessType* vec_thread_read_ptr = reinterpret_cast(thread_read_ptr); +#pragma unroll + // Note(Byron): interleaved loads to achieve better memory coalescing + // | thread[0] | thread[1] | thread[2] | thread[3] | thread[0] | thread[1] | thread[2] | thread[3] | ... + for (int ii = 0; ii < LDG_PER_THREAD; ++ii) { + row_chunk_vec_ptr[ii] = vec_thread_read_ptr[ii * THREADS_PER_ROW]; + } + + float row_chunk[VPT]; +#pragma unroll + // Note(Byron): upcast logits to float32 + for (int ii = 0; ii < VPT; ++ii) { + row_chunk[ii] = convert_to_float(row_chunk_temp[ii]); + } + + // Apply tanh softcapping and correction bias + if (moe_softcapping != 0.0f || correction_bias != nullptr) { +#pragma unroll + for (int ii = 0; ii < VPT; ++ii) { + float val = row_chunk[ii]; + + // Apply tanh softcapping if enabled + if (moe_softcapping != 0.0f) { + val = tanhf(val / moe_softcapping) * moe_softcapping; + } + + // Apply correction bias if provided + if (correction_bias != nullptr) { + /* + LDG is interleaved + |thread0 LDG| |thread1 LDG| |thread0 LDG| |thread1 LDG| + |--------- group0 --------| |----------group1 --------| + ^ local2 + */ + const int group_id = ii / ELTS_PER_LDG; + const int local_id = ii % ELTS_PER_LDG; + const int expert_idx = first_elt_read_by_thread + group_id * THREADS_PER_ROW * ELTS_PER_LDG + local_id; + val = val + correction_bias[expert_idx]; + } + + row_chunk[ii] = val; + } + } + + // First, we perform a max reduce within the thread. We can do the max in fp16 safely (I think) and just + // convert to float afterwards for the exp + sum reduction. + float thread_max = row_chunk[0]; +#pragma unroll + for (int ii = 1; ii < VPT; ++ii) { + thread_max = max(thread_max, row_chunk[ii]); + } + + /*********************************/ + /********* Softmax Begin *********/ + /*********************************/ + +// Now, we find the max within the thread group and distribute among the threads. We use a butterfly reduce. +// lane id: 0-31 within a warp +#pragma unroll + for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2) { + // butterfly reduce with (lane id ^ mask) + thread_max = max(thread_max, SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, thread_max, mask, THREADS_PER_ROW)); + } + + // From this point, thread max in all the threads have the max within the row. + // Now, we subtract the max from each element in the thread and take the exp. We also compute the thread local sum. + float row_sum = 0; +#pragma unroll + for (int ii = 0; ii < VPT; ++ii) { + row_chunk[ii] = expf(row_chunk[ii] - thread_max); + row_sum += row_chunk[ii]; + } + +// Now, we perform the sum reduce within each thread group. Similar to the max reduce, we use a bufferfly pattern. +#pragma unroll + for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2) { + row_sum += SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, row_sum, mask, THREADS_PER_ROW); + } + + // From this point, all threads have the max and the sum for their rows in the thread_max and thread_sum variables + // respectively. Finally, we can scale the rows for the softmax. Technically, for top-k gating we don't need to + // compute the entire softmax row. We can likely look at the maxes and only compute for the top-k values in the row. + // However, this kernel will likely not be a bottle neck and it seems better to closer match torch and find the + // argmax after computing the softmax. + const float reciprocal_row_sum = 1.f / row_sum; + +#pragma unroll + for (int ii = 0; ii < VPT; ++ii) { + row_chunk[ii] = row_chunk[ii] * reciprocal_row_sum; + } + /*******************************/ + /********* Softmax End *********/ + /*******************************/ + + // Now, softmax_res contains the softmax of the row chunk. Now, I want to find the topk elements in each row, along + // with the max index. + int start_col = first_elt_read_by_thread; + static constexpr int COLS_PER_GROUP_LDG = ELTS_PER_LDG * THREADS_PER_ROW; + + float row_sum_for_renormalize = 0; + + for (int k_idx = 0; k_idx < k; ++k_idx) { + // First, each thread does the local argmax + float max_val = row_chunk[0]; + int expert = start_col; +#pragma unroll + for (int ldg = 0, col = start_col; ldg < LDG_PER_THREAD; ++ldg, col += COLS_PER_GROUP_LDG) { +#pragma unroll + for (int ii = 0; ii < ELTS_PER_LDG; ++ii) { + float val = row_chunk[ldg * ELTS_PER_LDG + ii]; + + // No check on the experts here since columns with the smallest index are processed first and only + // updated if > (not >=) + if (val > max_val) { + max_val = val; + expert = col + ii; + } + } + } + +// Now, we perform the argmax reduce. We use the butterfly pattern so threads reach consensus about the max. +// This will be useful for K > 1 so that the threads can agree on "who" had the max value. That thread can +// then blank out their max with -inf and the warp can run more iterations... +#pragma unroll + for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2) { + float other_max = SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, max_val, mask, THREADS_PER_ROW); + int other_expert = SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, expert, mask, THREADS_PER_ROW); + + // We want lower indices to "win" in every thread so we break ties this way + if (other_max > max_val || (other_max == max_val && other_expert < expert)) { + max_val = other_max; + expert = other_expert; + } + } + + // Write the max for this k iteration to global memory. + if (thread_group_idx == 0) { + // Add a guard to ignore experts not included by this node + const bool node_uses_expert = expert >= start_expert && expert < end_expert; + const bool should_process_row = row_is_active && node_uses_expert; + + // The lead thread from each sub-group will write out the final results to global memory. (This will be a + // single) thread per row of the input/output matrices. + const int idx = k * thread_row + k_idx; + output[idx] = max_val; + indices[idx] = should_process_row ? (expert - start_expert) : NUM_EXPERTS; + row_sum_for_renormalize += max_val; + } + + // Finally, we clear the value in the thread with the current max if there is another iteration to run. + if (k_idx + 1 < k) { + const int ldg_group_for_expert = expert / COLS_PER_GROUP_LDG; + const int thread_to_clear_in_group = (expert / ELTS_PER_LDG) % THREADS_PER_ROW; + + // Only the thread in the group which produced the max will reset the "winning" value to -inf. + if (thread_group_idx == thread_to_clear_in_group) { + const int offset_for_expert = expert % ELTS_PER_LDG; + // Safe to set to any negative value since row_chunk values must be between 0 and 1. + row_chunk[ldg_group_for_expert * ELTS_PER_LDG + offset_for_expert] = -10000.f; + } + } + } + + // Fuse renormalization of topk_weights into this kernel + if (renormalize && thread_group_idx == 0) { + float row_sum_for_renormalize_inv = 1.f / row_sum_for_renormalize; +#pragma unroll + for (int k_idx = 0; k_idx < k; ++k_idx) { + const int idx = k * thread_row + k_idx; + output[idx] = output[idx] * row_sum_for_renormalize_inv; + } + } +} + +namespace detail { +// Constructs some constants needed to partition the work across threads at compile time. +template +struct TopkConstants { + static constexpr int ELTS_PER_LDG = BYTES_PER_LDG / sizeof(T); + static_assert(EXPERTS / (ELTS_PER_LDG * WARP_SIZE) == 0 || EXPERTS % (ELTS_PER_LDG * WARP_SIZE) == 0, ""); + static constexpr int VECs_PER_THREAD = MAX(1, EXPERTS / (ELTS_PER_LDG * WARP_SIZE)); + static constexpr int VPT = VECs_PER_THREAD * ELTS_PER_LDG; + static constexpr int THREADS_PER_ROW = EXPERTS / VPT; + static constexpr int ROWS_PER_WARP = WARP_SIZE / THREADS_PER_ROW; +}; +} // namespace detail + +template +void topkGatingSoftmaxLauncherHelper( + const T* input, + const bool* finished, + float* output, + int* indices, + const int num_rows, + const int k, + const int start_expert, + const int end_expert, + const bool renormalize, + const float moe_softcapping, + const float* correction_bias, + cudaStream_t stream) { + static constexpr std::size_t MAX_BYTES_PER_LDG = 16; + + static constexpr int BYTES_PER_LDG = MIN(MAX_BYTES_PER_LDG, sizeof(T) * EXPERTS); + using Constants = detail::TopkConstants; + static constexpr int VPT = Constants::VPT; + static constexpr int ROWS_PER_WARP = Constants::ROWS_PER_WARP; + const int num_warps = (num_rows + ROWS_PER_WARP - 1) / ROWS_PER_WARP; + const int num_blocks = (num_warps + WARPS_PER_TB - 1) / WARPS_PER_TB; + + dim3 block_dim(WARP_SIZE, WARPS_PER_TB); + topkGatingSoftmax<<>>( + input, + finished, + output, + num_rows, + indices, + k, + start_expert, + end_expert, + renormalize, + moe_softcapping, + correction_bias); +} + +#define LAUNCH_SOFTMAX(TYPE, NUM_EXPERTS, WARPS_PER_TB) \ + topkGatingSoftmaxLauncherHelper( \ + gating_output, \ + nullptr, \ + topk_weights, \ + topk_indices, \ + num_tokens, \ + topk, \ + 0, \ + num_experts, \ + renormalize, \ + moe_softcapping, \ + correction_bias, \ + stream); + +template +void topkGatingSoftmaxKernelLauncher( + const T* gating_output, + float* topk_weights, + int* topk_indices, + float* softmax_workspace, + const int num_tokens, + const int num_experts, + const int topk, + const bool renormalize, + const float moe_softcapping, + const float* correction_bias, + cudaStream_t stream) { + static constexpr int WARPS_PER_TB = 4; + switch (num_experts) { + case 1: + LAUNCH_SOFTMAX(T, 1, WARPS_PER_TB); + break; + case 2: + LAUNCH_SOFTMAX(T, 2, WARPS_PER_TB); + break; + case 4: + LAUNCH_SOFTMAX(T, 4, WARPS_PER_TB); + break; + case 8: + LAUNCH_SOFTMAX(T, 8, WARPS_PER_TB); + break; + case 16: + LAUNCH_SOFTMAX(T, 16, WARPS_PER_TB); + break; + case 32: + LAUNCH_SOFTMAX(T, 32, WARPS_PER_TB); + break; + case 64: + LAUNCH_SOFTMAX(T, 64, WARPS_PER_TB); + break; + case 128: + LAUNCH_SOFTMAX(T, 128, WARPS_PER_TB); + break; + case 256: + LAUNCH_SOFTMAX(T, 256, WARPS_PER_TB); + break; + default: { + TORCH_CHECK( + softmax_workspace != nullptr, + "softmax_workspace must be provided for num_experts that are not a power of 2."); + static constexpr int TPB = 256; + moeSoftmax<<>>( + gating_output, nullptr, softmax_workspace, num_experts, moe_softcapping, correction_bias); + moeTopK<<>>( + softmax_workspace, nullptr, topk_weights, topk_indices, num_experts, topk, 0, num_experts, renormalize); + } + } +} + +void topk_softmax( + torch::Tensor& topk_weights, // [num_tokens, topk] + torch::Tensor& topk_indices, // [num_tokens, topk] + torch::Tensor& gating_output, // [num_tokens, num_experts] + const bool renormalize, + const double moe_softcapping, + const c10::optional& correction_bias) { + // Check data type + TORCH_CHECK( + gating_output.scalar_type() == at::ScalarType::Float || gating_output.scalar_type() == at::ScalarType::Half || + gating_output.scalar_type() == at::ScalarType::BFloat16, + "gating_output must be float32, float16, or bfloat16"); + + // Check dimensions + TORCH_CHECK(gating_output.dim() == 2, "gating_output must be 2D tensor [num_tokens, num_experts]"); + TORCH_CHECK(topk_weights.dim() == 2, "topk_weights must be 2D tensor [num_tokens, topk]"); + TORCH_CHECK(topk_indices.dim() == 2, "topk_indices must be 2D tensor [num_tokens, topk]"); + + // Check shapes + TORCH_CHECK( + gating_output.size(0) == topk_weights.size(0), + "First dimension of topk_weights must match num_tokens in gating_output"); + TORCH_CHECK( + gating_output.size(0) == topk_indices.size(0), + "First dimension of topk_indices must match num_tokens in gating_output"); + TORCH_CHECK( + topk_weights.size(-1) == topk_indices.size(-1), + "Second dimension of topk_indices must match topk in topk_weights"); + TORCH_CHECK(topk_weights.size(-1) <= gating_output.size(-1), "topk must be less than or equal to num_experts"); + + const int num_experts = static_cast(gating_output.size(-1)); + const int num_tokens = static_cast(gating_output.size(0)); + const int topk = static_cast(topk_weights.size(-1)); + + const bool is_pow_2 = (num_experts != 0) && ((num_experts & (num_experts - 1)) == 0); + const bool needs_workspace = !is_pow_2 || num_experts > 256; + const int64_t workspace_size = needs_workspace ? num_tokens * num_experts : 0; + + const at::cuda::OptionalCUDAGuard device_guard(device_of(gating_output)); + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + torch::Tensor softmax_workspace = + torch::empty({workspace_size}, gating_output.options().dtype(at::ScalarType::Float)); + + const at::ScalarType dtype = gating_output.scalar_type(); + + // Validate correction_bias if provided - must always be float32 + const float* bias_ptr = nullptr; + if (correction_bias.has_value()) { + const torch::Tensor& bias_tensor = correction_bias.value(); + TORCH_CHECK(bias_tensor.dim() == 1, "correction_bias must be 1D tensor [num_experts]"); + TORCH_CHECK(bias_tensor.size(0) == num_experts, "correction_bias size must match num_experts"); + TORCH_CHECK( + bias_tensor.scalar_type() == at::ScalarType::Float, + "correction_bias must be float32, got ", + bias_tensor.scalar_type()); + bias_ptr = bias_tensor.data_ptr(); + } + + // Cast moe_softcapping from double to float for CUDA kernels + const float moe_softcapping_f = static_cast(moe_softcapping); + + if (dtype == at::ScalarType::Float) { + topkGatingSoftmaxKernelLauncher( + gating_output.data_ptr(), + topk_weights.data_ptr(), + topk_indices.data_ptr(), + softmax_workspace.data_ptr(), + num_tokens, + num_experts, + topk, + renormalize, + moe_softcapping_f, + bias_ptr, + stream); + } else if (dtype == at::ScalarType::Half) { + topkGatingSoftmaxKernelLauncher<__half>( + reinterpret_cast(gating_output.data_ptr()), + topk_weights.data_ptr(), + topk_indices.data_ptr(), + softmax_workspace.data_ptr(), + num_tokens, + num_experts, + topk, + renormalize, + moe_softcapping_f, + bias_ptr, + stream); + } else if (dtype == at::ScalarType::BFloat16) { + topkGatingSoftmaxKernelLauncher<__nv_bfloat16>( + reinterpret_cast(gating_output.data_ptr()), + topk_weights.data_ptr(), + topk_indices.data_ptr(), + softmax_workspace.data_ptr(), + num_tokens, + num_experts, + topk, + renormalize, + moe_softcapping_f, + bias_ptr, + stream); + } else { + TORCH_CHECK(false, "Unsupported gating_output dtype: ", dtype); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/nvfp4_blockwise_moe.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/nvfp4_blockwise_moe.cu new file mode 100644 index 00000000..6dbfb7bf --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/nvfp4_blockwise_moe.cu @@ -0,0 +1,698 @@ +#include +#include +#include +#include +#include + +#include + +#include "cute/tensor.hpp" +#include "cutlass/epilogue/collective/collective_builder.hpp" +#include "cutlass/epilogue/collective/default_epilogue.hpp" +#include "cutlass/epilogue/thread/linear_combination.h" +#include "cutlass/gemm/collective/collective_builder.hpp" +#include "cutlass/gemm/device/gemm_universal_adapter.h" +#include "cutlass/gemm/dispatch_policy.hpp" +#include "cutlass/gemm/group_array_problem_shape.hpp" +#include "cutlass/gemm/kernel/gemm_universal.hpp" +#include "cutlass/tensor_ref.h" +#include "cutlass/util/command_line.h" +#include "cutlass/util/distribution.h" +#include "cutlass/util/host_tensor.h" +#include "cutlass/util/packed_stride.hpp" +#include "cutlass/util/reference/device/gemm.h" +#include "cutlass/util/reference/device/tensor_compare.h" +#include "cutlass/util/reference/host/gett.hpp" +#include "cutlass/util/reference/host/tensor_compare.h" +#include "cutlass/util/reference/host/tensor_fill.h" +#include "cutlass/util/reference/host/tensor_norm.h" +#include "cutlass/util/tensor_view_io.h" +#include "utils.h" + +using namespace cute; + +template < + typename ElementAB, + typename ElementC, + typename ElementSF, + typename ElementAccumulator, + typename LayoutSFA, + typename LayoutSFB, + typename ScaleConfig> +__global__ void __get_group_gemm_starts( + ElementAB** a_offsets, + ElementAB** b_offsets, + ElementC** out_offsets, + ElementSF** a_scales_offsets, + ElementSF** b_scales_offsets, + ElementAccumulator** alpha_offsets, + LayoutSFA* layout_sfa_base_as_int, + LayoutSFB* layout_sfb_base_as_int, + ElementAB* a_base_as_int, + ElementAB* b_base_as_int, + ElementC* out_base_as_int, + ElementSF* a_scales_base_as_int, + ElementSF* b_scales_base_as_int, + ElementAccumulator* alphas_base_as_int, + const int32_t* expert_offsets, + const int32_t* sf_offsets, + const int32_t* problem_sizes_as_shapes, + const int K, + const int N) { + int64_t expert_id = threadIdx.x; + if (expert_id >= gridDim.x * blockDim.x) { + return; + } + // Originally int32_t but upcasting to int64_t to avoid overflow + // during offset calculations + int64_t expert_offset = static_cast(expert_offsets[expert_id]); + int64_t sf_offset = static_cast(sf_offsets[expert_id]); + // size for block in block scale. + int64_t group_size = 16; + int64_t m = static_cast(problem_sizes_as_shapes[expert_id * 3]); + int64_t n = static_cast(problem_sizes_as_shapes[expert_id * 3 + 1]); + int64_t k = static_cast(problem_sizes_as_shapes[expert_id * 3 + 2]); + assert((m >= 0 && n == N && k == K && k % 2 == 0) && "unexpected problem sizes"); + + int64_t half_k = static_cast(k / 2); + int64_t group_k = static_cast(k / group_size); + // Shape of A as uint8/byte = [M, K // 2] + // Shape of B as uint8/byte = [E, N, K // 2] + a_offsets[expert_id] = a_base_as_int + expert_offset * half_k; + + b_offsets[expert_id] = b_base_as_int + expert_id * n * half_k; + // Shape of C = [M, N] + out_offsets[expert_id] = out_base_as_int + expert_offset * n; + // Shape of a_scale = [sum(sf_sizes), K // group_size] + a_scales_offsets[expert_id] = a_scales_base_as_int + sf_offset * group_k; + + assert((reinterpret_cast(a_scales_offsets[expert_id]) % 128) == 0 && "TMA requires 128-byte alignment"); + + // Shape of B scale = [E, N, K // group_size] + b_scales_offsets[expert_id] = b_scales_base_as_int + expert_id * n * group_k; + assert((reinterpret_cast(b_scales_offsets[expert_id]) % 128) == 0 && "TMA requires 128-byte alignment"); + // Shape of alpha = [E] + alpha_offsets[expert_id] = alphas_base_as_int + expert_id; + + LayoutSFA* layout_sfa_ptr = layout_sfa_base_as_int + expert_id; + LayoutSFB* layout_sfb_ptr = layout_sfb_base_as_int + expert_id; + + *layout_sfa_ptr = ScaleConfig::tile_atom_to_shape_SFA( + cute::make_shape(static_cast(m), static_cast(n), static_cast(k), 1)); + *layout_sfb_ptr = ScaleConfig::tile_atom_to_shape_SFB( + cute::make_shape(static_cast(m), static_cast(n), static_cast(k), 1)); +} + +#define __CALL_GET_STARTS_KERNEL_BLOCKSCALE( \ + ELEMENT_AB_TYPE, SF_TYPE, TENSOR_C_TYPE, C_TYPE, LayoutSFA, LayoutSFB, ScaleConfig) \ + else if (out_tensors.dtype() == TENSOR_C_TYPE) { \ + __get_group_gemm_starts \ + <<<1, num_experts, 0, stream>>>( \ + static_cast(a_starts.data_ptr()), \ + static_cast(b_starts.data_ptr()), \ + static_cast(out_starts.data_ptr()), \ + static_cast(a_scales_starts.data_ptr()), \ + static_cast(b_scales_starts.data_ptr()), \ + static_cast(alpha_starts.data_ptr()), \ + reinterpret_cast(layout_sfa.data_ptr()), \ + reinterpret_cast(layout_sfb.data_ptr()), \ + static_cast(a_tensors.data_ptr()), \ + static_cast(b_tensors.data_ptr()), \ + static_cast(out_tensors.data_ptr()), \ + static_cast(a_scales.data_ptr()), \ + static_cast(b_scales.data_ptr()), \ + static_cast(alphas.data_ptr()), \ + static_cast(expert_offsets.data_ptr()), \ + static_cast(sf_offsets.data_ptr()), \ + static_cast(problem_sizes.data_ptr()), \ + K, \ + N); \ + } + +template +void run_get_group_gemm_starts( + const torch::Tensor& a_starts, + const torch::Tensor& b_starts, + const torch::Tensor& out_starts, + const torch::Tensor& a_scales_starts, + const torch::Tensor& b_scales_starts, + const torch::Tensor& alpha_starts, + const torch::Tensor& layout_sfa, + const torch::Tensor& layout_sfb, + /*these are used for their base addresses*/ + torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, + torch::Tensor const& out_tensors, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + torch::Tensor const& alphas, + torch::Tensor const& expert_offsets, + torch::Tensor const& sf_offsets, + torch::Tensor const& problem_sizes, + int M, + int N, + int K) { + int num_experts = (int)expert_offsets.size(0); + auto stream = at::cuda::getCurrentCUDAStream(a_tensors.device().index()); + + TORCH_CHECK(out_tensors.size(1) == N, "Output tensor shape doesn't match expected shape"); + TORCH_CHECK( + K / 2 == b_tensors.size(2), + "b_tensors(dim = 2) and a_tensors(dim = 1) trailing" + " dimension must match"); + if (false) { + } + //(ELEMENT_AB_TYPE, BS_TYPE, TENSOR_C_TYPE, C_TYPE, LayoutSFA, LayoutSFB, + // ScaleConfig) + __CALL_GET_STARTS_KERNEL_BLOCKSCALE( + cutlass::float_e2m1_t, + cutlass::float_ue4m3_t, + torch::kBFloat16, + cutlass::bfloat16_t, + LayoutSFA, + LayoutSFB, + ScaleConfig) + __CALL_GET_STARTS_KERNEL_BLOCKSCALE( + cutlass::float_e2m1_t, cutlass::float_ue4m3_t, torch::kFloat16, half, LayoutSFA, LayoutSFB, ScaleConfig) + else { + TORCH_CHECK(false, "Invalid output type (must be float16 or bfloat16)"); + } +} + +void run_fp4_blockwise_scaled_group_mm_sm120( + torch::Tensor& output, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& a_blockscale, + const torch::Tensor& b_blockscales, + const torch::Tensor& alphas, + const torch::Tensor& ab_strides, + const torch::Tensor& c_strides, + const torch::Tensor& problem_sizes, + const torch::Tensor& expert_offsets, + const torch::Tensor& sf_offsets, + int M, + int N, + int K) { + using ProblemShape = cutlass::gemm::GroupProblemShape>; + using ElementType = cutlass::float_e2m1_t; + using ElementSFType = cutlass::float_ue4m3_t; + using ElementA = cutlass::nv_float4_t; + using ElementB = cutlass::nv_float4_t; + + using ElementC = cutlass::bfloat16_t; + using ElementD = cutlass::bfloat16_t; + using ElementAccumulator = float; + // Layout definitions + using LayoutA = cutlass::layout::RowMajor; + using LayoutB = cutlass::layout::ColumnMajor; + using LayoutC = cutlass::layout::RowMajor; + using LayoutD = cutlass::layout::RowMajor; + + // Alignment constraints + static constexpr int AlignmentA = 32; + static constexpr int AlignmentB = 32; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentD = 128 / cutlass::sizeof_bits::value; + + // Architecture definitions + using ArchTag = cutlass::arch::Sm120; + using OperatorClass = cutlass::arch::OpClassBlockScaledTensorOp; + using StageCountType = cutlass::gemm::collective::StageCountAuto; + using ThreadBlockShape = Shape<_128, _128, _128>; + // on the tile size + + using ClusterShape = Shape<_1, _1, _1>; + + using FusionOperation = + cutlass::epilogue::fusion::LinearCombination; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ThreadBlockShape, + ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementAccumulator, + ElementC, + LayoutC*, + AlignmentC, + ElementD, + LayoutC*, + AlignmentD, + cutlass::epilogue::collective::EpilogueScheduleAuto, + FusionOperation>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementA, + LayoutA*, + AlignmentA, + ElementB, + LayoutB*, + AlignmentB, + ElementAccumulator, + ThreadBlockShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + cutlass::gemm::KernelPtrArrayTmaWarpSpecializedPingpong>::CollectiveOp; + + using GemmKernel = cutlass::gemm::kernel::GemmUniversal; + + using Gemm1SM = cutlass::gemm::device::GemmUniversalAdapter; + using Gemm = Gemm1SM; + using StrideA = typename Gemm::GemmKernel::InternalStrideA; + using StrideB = typename Gemm::GemmKernel::InternalStrideB; + using StrideC = typename Gemm::GemmKernel::InternalStrideC; + using StrideD = typename Gemm::GemmKernel::InternalStrideD; + + using LayoutSFA = typename Gemm::GemmKernel::CollectiveMainloop::InternalLayoutSFA; + using LayoutSFB = typename Gemm::GemmKernel::CollectiveMainloop::InternalLayoutSFB; + using ScaleConfig = typename Gemm::GemmKernel::CollectiveMainloop::Sm1xxBlkScaledConfig; + + using UnderlyingProblemShape = ProblemShape::UnderlyingProblemShape; + int num_experts = static_cast(expert_offsets.size(0)); + auto options_int = torch::TensorOptions().dtype(torch::kInt64).device(a.device()); + + torch::Tensor a_ptrs = torch::empty(num_experts, options_int); + torch::Tensor b_ptrs = torch::empty(num_experts, options_int); + torch::Tensor out_ptrs = torch::empty(num_experts, options_int); + torch::Tensor a_scales_ptrs = torch::empty(num_experts, options_int); + torch::Tensor b_scales_ptrs = torch::empty(num_experts, options_int); + torch::Tensor alpha_ptrs = torch::empty(num_experts, options_int); + torch::Tensor layout_sfa = torch::empty({num_experts, 5}, options_int); + torch::Tensor layout_sfb = torch::empty({num_experts, 5}, options_int); + + run_get_group_gemm_starts( + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + alpha_ptrs, + layout_sfa, + layout_sfb, + a, + b, + output, + a_blockscale, + b_blockscales, + alphas, + expert_offsets, + sf_offsets, + problem_sizes, + M, + N, + K); + + // Create an instance of the GEMM + Gemm gemm_op; + + // Initialize problem_sizes_as_shapes correctly + UnderlyingProblemShape* problem_sizes_as_shapes = static_cast(problem_sizes.data_ptr()); + + // Set the Scheduler info + cutlass::KernelHardwareInfo hw_info; + + using RasterOrderOptions = cutlass::gemm::kernel::detail::RasterOrderOptions; + typename Gemm::GemmKernel::TileSchedulerArguments scheduler; + scheduler.raster_order = RasterOrderOptions::AlongM; + hw_info.device_id = a.get_device(); + static std::unordered_map cached_sm_counts; + if (cached_sm_counts.find(hw_info.device_id) == cached_sm_counts.end()) { + cached_sm_counts[hw_info.device_id] = + cutlass::KernelHardwareInfo::query_device_multiprocessor_count(hw_info.device_id); + } + hw_info.sm_count = min(cached_sm_counts[hw_info.device_id], INT_MAX); + + // Mainloop Arguments + typename GemmKernel::MainloopArguments mainloop_args{ + static_cast(a_ptrs.data_ptr()), + static_cast(ab_strides.data_ptr()), + static_cast(b_ptrs.data_ptr()), + static_cast(ab_strides.data_ptr()), + static_cast(a_scales_ptrs.data_ptr()), + reinterpret_cast(layout_sfa.data_ptr()), + static_cast(b_scales_ptrs.data_ptr()), + reinterpret_cast(layout_sfb.data_ptr())}; + + // Epilogue Arguments + typename GemmKernel::EpilogueArguments epilogue_args{ + {}, // epilogue.thread + nullptr, + static_cast(c_strides.data_ptr()), + static_cast(out_ptrs.data_ptr()), + static_cast(c_strides.data_ptr())}; + auto& fusion_args = epilogue_args.thread; + fusion_args.alpha_ptr_array = reinterpret_cast(alpha_ptrs.data_ptr()); + fusion_args.dAlpha = {_0{}, _0{}, 1}; + fusion_args.beta = 0.0f; + + // Gemm Arguments + typename GemmKernel::Arguments args{ + cutlass::gemm::GemmUniversalMode::kGrouped, + {num_experts, problem_sizes_as_shapes, nullptr}, + mainloop_args, + epilogue_args, + hw_info, + scheduler}; + + size_t workspace_size = Gemm::get_workspace_size(args); + auto const workspace_options = torch::TensorOptions().dtype(torch::kUInt8).device(a.device()); + auto workspace = torch::empty(workspace_size, workspace_options); + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(a.get_device()); + + auto can_implement_status = gemm_op.can_implement(args); + TORCH_CHECK(can_implement_status == cutlass::Status::kSuccess, "Failed to implement GEMM"); + + // Run the GEMM + auto status = gemm_op.initialize(args, workspace.data_ptr()); + TORCH_CHECK(status == cutlass::Status::kSuccess, "Failed to initialize GEMM"); + + status = gemm_op.run(args, workspace.data_ptr(), stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, "Failed to run GEMM"); +} + +template +void run_fp4_blockwise_scaled_group_mm_sm100( + torch::Tensor& output, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& a_blockscale, + const torch::Tensor& b_blockscales, + const torch::Tensor& alphas, + const torch::Tensor& ab_strides, + const torch::Tensor& c_strides, + const torch::Tensor& problem_sizes, + const torch::Tensor& expert_offsets, + const torch::Tensor& sf_offsets, + int M, + int N, + int K) { + using ProblemShape = cutlass::gemm::GroupProblemShape>; + using ElementType = cutlass::float_e2m1_t; + using ElementSFType = cutlass::float_ue4m3_t; + using ElementA = cutlass::nv_float4_t; + using ElementB = cutlass::nv_float4_t; + + using ElementC = OutType; + using ElementD = ElementC; + using ElementAccumulator = float; + // Layout definitions + using LayoutA = cutlass::layout::RowMajor; + using LayoutB = cutlass::layout::ColumnMajor; + using LayoutC = cutlass::layout::RowMajor; + using LayoutD = LayoutC; + + // Alignment constraints + static constexpr int AlignmentA = 32; + static constexpr int AlignmentB = 32; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentD = 128 / cutlass::sizeof_bits::value; + + // Architecture definitions + using ArchTag = cutlass::arch::Sm100; + using EpilogueOperatorClass = cutlass::arch::OpClassTensorOp; // Epilogue Operator class tag + using MainloopOperatorClass = cutlass::arch::OpClassBlockScaledTensorOp; // Mainloop Operator class tag + using StageCountType = cutlass::gemm::collective::StageCountAuto; // Stage count maximized based + // on the tile size + + using ClusterShape = Shape<_1, _1, _1>; + struct MMA1SMConfig { + using MmaTileShape = Shape<_128, _128, _128>; + using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100; // Kernel to launch + using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecialized1Sm; // Epilogue to launch + }; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + EpilogueOperatorClass, + typename MMA1SMConfig::MmaTileShape, + ClusterShape, + Shape<_128, _64>, + ElementAccumulator, + ElementAccumulator, + ElementC, + LayoutC*, + AlignmentC, + ElementD, + LayoutC*, + AlignmentD, + typename MMA1SMConfig::EpilogueSchedule>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + MainloopOperatorClass, + ElementA, + LayoutA*, + AlignmentA, + ElementB, + LayoutB*, + AlignmentB, + ElementAccumulator, + typename MMA1SMConfig::MmaTileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + typename MMA1SMConfig::KernelSchedule>::CollectiveOp; + + using GemmKernel = cutlass::gemm::kernel::GemmUniversal; + + using Gemm1SM = cutlass::gemm::device::GemmUniversalAdapter; + using Gemm = Gemm1SM; + using StrideA = typename Gemm::GemmKernel::InternalStrideA; + using StrideB = typename Gemm::GemmKernel::InternalStrideB; + using StrideC = typename Gemm::GemmKernel::InternalStrideC; + using StrideD = typename Gemm::GemmKernel::InternalStrideD; + + using LayoutSFA = typename Gemm::GemmKernel::CollectiveMainloop::InternalLayoutSFA; + using LayoutSFB = typename Gemm::GemmKernel::CollectiveMainloop::InternalLayoutSFB; + using ScaleConfig = typename Gemm::GemmKernel::CollectiveMainloop::Sm1xxBlkScaledConfig; + + using UnderlyingProblemShape = ProblemShape::UnderlyingProblemShape; + int num_experts = static_cast(expert_offsets.size(0)); + auto options_int = torch::TensorOptions().dtype(torch::kInt64).device(a.device()); + + torch::Tensor a_ptrs = torch::empty(num_experts, options_int); + torch::Tensor b_ptrs = torch::empty(num_experts, options_int); + torch::Tensor out_ptrs = torch::empty(num_experts, options_int); + torch::Tensor a_scales_ptrs = torch::empty(num_experts, options_int); + torch::Tensor b_scales_ptrs = torch::empty(num_experts, options_int); + torch::Tensor alpha_ptrs = torch::empty(num_experts, options_int); + torch::Tensor layout_sfa = torch::empty({num_experts, 5}, options_int); + torch::Tensor layout_sfb = torch::empty({num_experts, 5}, options_int); + + run_get_group_gemm_starts( + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + alpha_ptrs, + layout_sfa, + layout_sfb, + a, + b, + output, + a_blockscale, + b_blockscales, + alphas, + expert_offsets, + sf_offsets, + problem_sizes, + M, + N, + K); + + // Create an instance of the GEMM + Gemm gemm_op; + + // Initialize problem_sizes_as_shapes correctly + UnderlyingProblemShape* problem_sizes_as_shapes = static_cast(problem_sizes.data_ptr()); + + // Set the Scheduler info + cutlass::KernelHardwareInfo hw_info; + using RasterOrderOptions = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm100GroupParams< + typename ProblemShape::UnderlyingProblemShape>::RasterOrderOptions; + typename Gemm::GemmKernel::TileSchedulerArguments scheduler; + scheduler.raster_order = RasterOrderOptions::AlongM; + hw_info.device_id = a.get_device(); + static std::unordered_map cached_sm_counts; + if (cached_sm_counts.find(hw_info.device_id) == cached_sm_counts.end()) { + cached_sm_counts[hw_info.device_id] = + cutlass::KernelHardwareInfo::query_device_multiprocessor_count(hw_info.device_id); + } + hw_info.sm_count = min(cached_sm_counts[hw_info.device_id], INT_MAX); + + // Mainloop Arguments + typename GemmKernel::MainloopArguments mainloop_args{ + static_cast(a_ptrs.data_ptr()), + static_cast(ab_strides.data_ptr()), + static_cast(b_ptrs.data_ptr()), + static_cast(ab_strides.data_ptr()), + static_cast(a_scales_ptrs.data_ptr()), + reinterpret_cast(layout_sfa.data_ptr()), + static_cast(b_scales_ptrs.data_ptr()), + reinterpret_cast(layout_sfb.data_ptr())}; + + // Epilogue Arguments + typename GemmKernel::EpilogueArguments epilogue_args{ + {}, // epilogue.thread + nullptr, + static_cast(c_strides.data_ptr()), + static_cast(out_ptrs.data_ptr()), + static_cast(c_strides.data_ptr())}; + auto& fusion_args = epilogue_args.thread; + fusion_args.alpha_ptr_array = reinterpret_cast(alpha_ptrs.data_ptr()); + fusion_args.dAlpha = {_0{}, _0{}, 1}; + + // Gemm Arguments + typename GemmKernel::Arguments args{ + cutlass::gemm::GemmUniversalMode::kGrouped, + {num_experts, problem_sizes_as_shapes, nullptr}, + mainloop_args, + epilogue_args, + hw_info, + scheduler}; + + size_t workspace_size = Gemm::get_workspace_size(args); + auto const workspace_options = torch::TensorOptions().dtype(torch::kUInt8).device(a.device()); + auto workspace = torch::empty(workspace_size, workspace_options); + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(a.get_device()); + + auto can_implement_status = gemm_op.can_implement(args); + TORCH_CHECK(can_implement_status == cutlass::Status::kSuccess, "Failed to implement GEMM"); + + // Run the GEMM + auto status = gemm_op.initialize(args, workspace.data_ptr()); + TORCH_CHECK(status == cutlass::Status::kSuccess, "Failed to initialize GEMM"); + + status = gemm_op.run(args, workspace.data_ptr(), stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, "Failed to run GEMM"); +} + +// Undefine macros from utils.h to redefine with custom signatures +#undef CHECK_CONTIGUOUS +#undef CHECK_INPUT + +#define CHECK_TYPE(x, st, m) TORCH_CHECK(x.scalar_type() == st, ": Inconsistency of Tensor type:", m) +#define CHECK_TH_CUDA(x, m) TORCH_CHECK(x.is_cuda(), m, ": must be a CUDA tensor.") +#define CHECK_CONTIGUOUS(x, m) TORCH_CHECK(x.is_contiguous(), m, ": must be contiguous.") +#define CHECK_INPUT(x, st, m) \ + CHECK_TH_CUDA(x, m); \ + CHECK_CONTIGUOUS(x, m); \ + CHECK_TYPE(x, st, m) + +void cutlass_fp4_group_mm( + torch::Tensor& output, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& a_blockscale, + const torch::Tensor& b_blockscales, + const torch::Tensor& alphas, + const torch::Tensor& ab_strides, + const torch::Tensor& c_strides, + const torch::Tensor& problem_sizes, + const torch::Tensor& expert_offsets, + const torch::Tensor& sf_offsets) { +#if defined ENABLE_NVFP4 && ENABLE_NVFP4 + + constexpr auto FLOAT4_E2M1X2 = at::ScalarType::Byte; + constexpr auto SF_DTYPE = at::ScalarType::Float8_e4m3fn; + // Input validation + CHECK_INPUT(a, FLOAT4_E2M1X2, "a"); + CHECK_INPUT(b, FLOAT4_E2M1X2, "b"); + CHECK_INPUT(a_blockscale, SF_DTYPE, "a_blockscale"); + CHECK_INPUT(b_blockscales, SF_DTYPE, "b_blockscales"); + CHECK_INPUT(alphas, at::ScalarType::Float, "alphas"); + + TORCH_CHECK( + a_blockscale.dim() == 2, + "expected a_blockscale to be of shape [num_experts, rounded_m," + " k // group_size], observed rank: ", + a_blockscale.dim()) + TORCH_CHECK( + b_blockscales.dim() == 3, + "expected b_blockscale to be of shape: " + " [num_experts, n, k // group_size], observed rank: ", + b_blockscales.dim()) + TORCH_CHECK(problem_sizes.dim() == 2, "problem_sizes must be a 2D tensor"); + TORCH_CHECK(problem_sizes.size(1) == 3, "problem_sizes must have the shape (num_experts, 3)"); + TORCH_CHECK( + problem_sizes.size(0) == expert_offsets.size(0), "Number of experts in problem_sizes must match expert_offsets"); + TORCH_CHECK(problem_sizes.dtype() == torch::kInt32, "problem_sizes must be int32."); + + int M = static_cast(a.size(0)); + int N = static_cast(b.size(1)); + int E = static_cast(b.size(0)); + int K = static_cast(2 * b.size(2)); + + auto sm_version = getSMVersion(); + if (sm_version == 100 || sm_version == 103) { + if (output.scalar_type() == torch::kBFloat16) { + run_fp4_blockwise_scaled_group_mm_sm100( + output, + a, + b, + a_blockscale, + b_blockscales, + alphas, + ab_strides, + c_strides, + problem_sizes, + expert_offsets, + sf_offsets, + M, + N, + K); + } else { + run_fp4_blockwise_scaled_group_mm_sm100( + output, + a, + b, + a_blockscale, + b_blockscales, + alphas, + ab_strides, + c_strides, + problem_sizes, + expert_offsets, + sf_offsets, + M, + N, + K); + } + } else if (sm_version == 120) { + if (output.scalar_type() == torch::kBFloat16) { + run_fp4_blockwise_scaled_group_mm_sm120( + output, + a, + b, + a_blockscale, + b_blockscales, + alphas, + ab_strides, + c_strides, + problem_sizes, + expert_offsets, + sf_offsets, + M, + N, + K); + } else { + std::cout << "run_fp4_blockwise_scaled_group_mm_sm120 half no implementation" << std::endl; + } + } else { + TORCH_CHECK_NOT_IMPLEMENTED(false, "Unsupported SM version: " + std::to_string(sm_version)); + } +#else + TORCH_CHECK_NOT_IMPLEMENTED( + false, + "No compiled cutlass_fp4_group_mm kernel, sgl-kernel must " + "be compiled with ENABLE_NVFP4 for SM100+ and CUDA " + "12.8 or above."); +#endif +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/prepare_moe_input.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/prepare_moe_input.cu new file mode 100644 index 00000000..46fff064 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/moe/prepare_moe_input.cu @@ -0,0 +1,392 @@ +#include +#include +#include + +#include +#include + +#include "cutlass/array.h" +#include "utils.h" + +constexpr uint64_t THREADS_PER_EXPERT = 512; + +__global__ void compute_problem_sizes( + const int* __restrict__ topk_ids, + int32_t* problem_sizes1, + int32_t* problem_sizes2, + int32_t* atomic_buffer, + const int64_t topk_length, + const int64_t n, + const int64_t k) { + int expert_id = blockIdx.x; + + int occurrences = 0; + for (int i = threadIdx.x; i < topk_length; i += THREADS_PER_EXPERT) { + occurrences += (topk_ids[i] == expert_id); + } + atomicAdd(&atomic_buffer[expert_id], occurrences); + __syncthreads(); + + if (threadIdx.x == 0) { + int final_occurrences = atomic_buffer[expert_id]; + problem_sizes1[expert_id * 3] = final_occurrences; + problem_sizes1[expert_id * 3 + 1] = static_cast(2 * n); + problem_sizes1[expert_id * 3 + 2] = static_cast(k); + problem_sizes2[expert_id * 3] = final_occurrences; + problem_sizes2[expert_id * 3 + 1] = static_cast(k); + problem_sizes2[expert_id * 3 + 2] = static_cast(n); + } +} + +__global__ void compute_expert_offsets( + const int32_t* __restrict__ problem_sizes1, + int32_t* expert_offsets, + int32_t* atomic_buffer, + const int64_t num_experts) { + int32_t tot_offset = 0; + expert_offsets[0] = 0; + for (int i = 0; i < num_experts; ++i) { + atomic_buffer[i] = tot_offset; + tot_offset += problem_sizes1[i * 3]; + expert_offsets[i + 1] = tot_offset; + } +} + +__global__ void compute_expert_blockscale_offsets( + const int32_t* __restrict__ problem_sizes1, + int32_t* expert_offsets, + int32_t* blockscale_offsets, + int32_t* atomic_buffer, + const int64_t num_experts) { + int32_t tot_offset = 0; + int32_t tot_rounded_offset = 0; + expert_offsets[0] = 0; + blockscale_offsets[0] = 0; + for (int i = 0; i < num_experts; ++i) { + atomic_buffer[i] = tot_offset; + int num_tokens = problem_sizes1[i * 3]; + int rounded_num_tokens = (num_tokens + (128 - 1)) / 128 * 128; + tot_offset += num_tokens; + tot_rounded_offset += rounded_num_tokens; + expert_offsets[i + 1] = tot_offset; + blockscale_offsets[i + 1] = tot_rounded_offset; + } +} + +__global__ void compute_arg_sorts( + const int32_t* __restrict__ topk_ids, + int32_t* input_permutation, + int32_t* output_permutation, + int32_t* atomic_buffer, + const int64_t topk_length, + const int64_t topk) { + int expert_id = blockIdx.x; + + for (int i = threadIdx.x; i < topk_length; i += THREADS_PER_EXPERT) { + if (topk_ids[i] == expert_id) { + int start = atomicAdd(&atomic_buffer[expert_id], 1); + input_permutation[start] = i / topk; + output_permutation[i] = start; + } + } +} + +void get_moe_prepare_input_caller( + const torch::Tensor& topk_ids, + torch::Tensor& expert_offsets, + const std::optional& blockscale_offsets, + torch::Tensor& problem_sizes1, + torch::Tensor& problem_sizes2, + torch::Tensor& input_permutation, + torch::Tensor& output_permutation, + const int64_t num_experts, + const int64_t n, + const int64_t k) { + auto stream = at::cuda::getCurrentCUDAStream(topk_ids.device().index()); + auto options_int32 = torch::TensorOptions().dtype(torch::kInt32).device(topk_ids.device()); + torch::Tensor atomic_buffer = torch::zeros(num_experts, options_int32); + + uint32_t num_threads = static_cast(min(THREADS_PER_EXPERT, topk_ids.numel())); + uint32_t num_blocks = static_cast(num_experts); + + compute_problem_sizes<<>>( + static_cast(topk_ids.data_ptr()), + static_cast(problem_sizes1.data_ptr()), + static_cast(problem_sizes2.data_ptr()), + static_cast(atomic_buffer.data_ptr()), + topk_ids.numel(), + n, + k); + if (blockscale_offsets.has_value()) { + compute_expert_blockscale_offsets<<<1, 1, 0, stream>>>( + static_cast(problem_sizes1.data_ptr()), + static_cast(expert_offsets.data_ptr()), + static_cast(blockscale_offsets.value().data_ptr()), + static_cast(atomic_buffer.data_ptr()), + num_experts); + } else { + compute_expert_offsets<<<1, 1, 0, stream>>>( + static_cast(problem_sizes1.data_ptr()), + static_cast(expert_offsets.data_ptr()), + static_cast(atomic_buffer.data_ptr()), + num_experts); + } + compute_arg_sorts<<>>( + static_cast(topk_ids.data_ptr()), + static_cast(input_permutation.data_ptr()), + static_cast(output_permutation.data_ptr()), + static_cast(atomic_buffer.data_ptr()), + topk_ids.numel(), + topk_ids.size(1)); +} + +void prepare_moe_input( + const torch::Tensor& topk_ids, + torch::Tensor& expert_offsets, + const std::optional& blockscale_offsets, + torch::Tensor& problem_sizes1, + torch::Tensor& problem_sizes2, + torch::Tensor& input_permutation, + torch::Tensor& output_permutation, + const int64_t num_experts, + const int64_t n, + const int64_t k) { + TORCH_CHECK(topk_ids.dtype() == torch::kInt32); + get_moe_prepare_input_caller( + topk_ids, + expert_offsets, + blockscale_offsets, + problem_sizes1, + problem_sizes2, + input_permutation, + output_permutation, + num_experts, + n, + k); + return; +} + +template +__global__ void shuffleRowsKernel( + const T* input, + const int32_t* dst2src_map, + T* output, + int64_t num_src_rows, + int64_t num_dst_rows, + int64_t num_cols) { + int64_t dest_row_idx = blockIdx.x; + int64_t const source_row_idx = dst2src_map[dest_row_idx]; + + if (blockIdx.x < num_dst_rows) { + // Load 128-bits per thread + constexpr uint64_t ELEM_PER_THREAD = 128 / sizeof(T) / 8; + using DataElem = cutlass::Array; + + // Duplicate and permute rows + auto const* source_row_ptr = reinterpret_cast(input + source_row_idx * num_cols); + auto* dest_row_ptr = reinterpret_cast(output + dest_row_idx * num_cols); + + auto const start_offset = threadIdx.x; + auto const stride = blockDim.x; + auto const num_elems_in_col = num_cols / ELEM_PER_THREAD; + + for (auto elem_index = start_offset; elem_index < num_elems_in_col; elem_index += stride) { + dest_row_ptr[elem_index] = source_row_ptr[elem_index]; + } + } +} + +#define DECLARE_SHUFFLE_ROWS(T) \ + __global__ void shuffleRowsKernel( \ + const T* input, \ + const int32_t* dst2src_map, \ + T* output, \ + int64_t num_src_rows, \ + int64_t num_dest_rows, \ + int64_t num_cols); + +DECLARE_SHUFFLE_ROWS(float); +DECLARE_SHUFFLE_ROWS(half); +DECLARE_SHUFFLE_ROWS(__nv_bfloat16); +DECLARE_SHUFFLE_ROWS(__nv_fp8_e4m3); +DECLARE_SHUFFLE_ROWS(uint8_t); + +#define SHUFFLE_ROWS(T) \ + shuffleRowsKernel<<>>( \ + reinterpret_cast(input), \ + static_cast(dst2src_map.data_ptr()), \ + reinterpret_cast(output), \ + num_src_rows, \ + num_dst_rows, \ + num_cols) + +#define DTYPE_DISPATCH_CASE(T, CUDA_T) \ + case T: \ + SHUFFLE_ROWS(CUDA_T); \ + break; + +void shuffle_rows_caller( + const torch::Tensor& input_tensor, const torch::Tensor& dst2src_map, torch::Tensor& output_tensor) { + TORCH_CHECK( + input_tensor.scalar_type() == output_tensor.scalar_type(), + "Input and output tensors must have the same data type"); + auto stream = at::cuda::getCurrentCUDAStream().stream(); + uint32_t blocks = static_cast(output_tensor.size(0)); + uint32_t threads = 256; + int64_t num_dst_rows = output_tensor.size(0); + int64_t num_src_rows = input_tensor.size(0); + int64_t num_cols = input_tensor.size(1); + const void* input = input_tensor.data_ptr(); + void* output = output_tensor.data_ptr(); + switch (input_tensor.scalar_type()) { + DTYPE_DISPATCH_CASE(torch::kFloat16, half); + DTYPE_DISPATCH_CASE(torch::kBFloat16, __nv_bfloat16); + DTYPE_DISPATCH_CASE(torch::kFloat32, float); + DTYPE_DISPATCH_CASE(torch::kFloat8_e4m3fn, __nv_fp8_e4m3); + DTYPE_DISPATCH_CASE(torch::kUInt8, uint8_t); + default: + TORCH_CHECK(false, "[moe replicate input] data type dispatch fail!"); + } + return; +} + +void shuffle_rows(const torch::Tensor& input_tensor, const torch::Tensor& dst2src_map, torch::Tensor& output_tensor) { + shuffle_rows_caller(input_tensor, dst2src_map, output_tensor); + return; +} + +template +__global__ void apply_shuffle_mul_sum_kernel( + const scalar_t* __restrict__ input_tensor, // [m * topk, k] (expert-major layout) + scalar_t* __restrict__ output_tensor, // [m, k] (token-major layout) + const int32_t* __restrict__ permutation, // [m * topk] (c_map: token-major-idx -> expert-major-idx) + int m, + int topk, + int row_stride, + const scalar_t* __restrict__ factors) // [m * topk] (topk_weights, token-major layout) +{ + int i = blockIdx.x; + if (i >= m) { + return; + } + + constexpr uint32_t vec_size = 16 / sizeof(scalar_t); + using t = float; + using vec_t = flashinfer::vec_t; + int thread_idx = threadIdx.x; + int stride = blockDim.x; + + for (int d_vec_idx = thread_idx; d_vec_idx < row_stride / vec_size; d_vec_idx += stride) { + int d = d_vec_idx * vec_size; + vec_t sum_vec; + sum_vec.fill(0.0f); + + for (int j = 0; j < topk; ++j) { + int token_major_idx = i * topk + j; + int src_row = permutation[token_major_idx]; + + vec_t val_vec; + val_vec.cast_load(input_tensor + src_row * row_stride + d); + + t factor = 1.0; + if (factors != nullptr) { + factor = factors[token_major_idx]; + } + +#pragma unroll + for (int k = 0; k < vec_size; ++k) { + sum_vec[k] += factor * val_vec[k]; + } + } + sum_vec.cast_store(output_tensor + i * row_stride + d); + } + + // remainder part + int remainder_start = (row_stride / vec_size) * vec_size; + for (int d = remainder_start + thread_idx; d < row_stride; d += stride) { + t sum_val = 0.0; + for (int j = 0; j < topk; ++j) { + int token_major_idx = i * topk + j; + int src_row = permutation[token_major_idx]; + t val = input_tensor[src_row * row_stride + d]; + + t factor = 1.0; + if (factors != nullptr) { + factor = factors[token_major_idx]; + } + sum_val += factor * val; + } + output_tensor[i * row_stride + d] = sum_val; + } +} + +void get_apply_shuffle_mul_sum_caller( + const torch::Tensor& input_tensor, // [m * topk, row_stride], bf16/f16 + torch::Tensor& output_tensor, // [m, row_stride], bf16/f16 + const torch::Tensor& permutation, // [m * topk], int32 + const std::optional& factors_opt) // optional [m * topk], bf16/f16 +{ + TORCH_CHECK(input_tensor.dim() == 2, "input_tensor must be 2D [m * topk, row_stride]"); + TORCH_CHECK(output_tensor.dim() == 2, "output_tensor must be 2D [m, row_stride]"); + TORCH_CHECK(permutation.dim() == 1, "permutation must be 1D [m * topk]"); + + int m = output_tensor.size(0); + int topk = int(permutation.size(0) / m); + int row_stride = output_tensor.size(1); + + TORCH_CHECK(permutation.size(0) == m * topk, "permutation size must match m * topk"); + + auto scalar_type = output_tensor.scalar_type(); + uint32_t vec_size = 16 / sizeof(scalar_type); + auto blockDim = std::min(row_stride / vec_size, 1024U); + dim3 block(blockDim); + + dim3 grid(m); // blockIdx.x = j, blockIdx.y = i + auto stream = at::cuda::getCurrentCUDAStream(input_tensor.device().index()); + + const int32_t* perm_ptr = permutation.data_ptr(); + + void* factors_ptr = nullptr; + if (factors_opt.has_value()) { + TORCH_CHECK(factors_opt->dtype() == output_tensor.dtype(), "Factors must match output dtype"); + TORCH_CHECK(factors_opt->numel() == m * topk, "Factors must have shape [m * topk]"); + factors_ptr = factors_opt->data_ptr(); + } + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(output_tensor.scalar_type(), scalar_t, [&] { + apply_shuffle_mul_sum_kernel<<>>( + static_cast(input_tensor.data_ptr()), + static_cast(output_tensor.data_ptr()), + perm_ptr, + m, + topk, + row_stride, + static_cast(factors_ptr)); + return true; + }); +} + +/** + * @brief Applies a permutation-based shuffle, element-wise multiplication, and reduction over the second dimension. + * + * This function performs the equivalent of the following PyTorch expression: + * + * (c2[c_map].view(m, topk, k) * topk_weights.view(m, topk, 1).to(out_dtype)).sum(dim=1) + * + * Specifically: + * - `input` is shuffled using the `permutation` tensor. + * - The shuffled tensor is reshaped and multiplied element-wise with `factors` (e.g., top-k weights). + * - The result is summed along dimension 1 (the top-k dimension), and stored in `output`. + * + * @param input Input tensor of shape (m * topk, k), representing c2. + * @param output Output tensor of shape (m, k), where the final reduced results are stored. + * @param permutation Index tensor (e.g., c_map) that maps positions in `input` to shuffled layout. + * @param factors Optional scaling factors (e.g., top-k weights), shape (m * topk) or (m, topk). + */ +void apply_shuffle_mul_sum( + const torch::Tensor& input, + torch::Tensor& output, + const torch::Tensor& permutation, + const std::optional& factors) { + get_apply_shuffle_mul_sum_caller(input, output, permutation, factors); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/dequantize.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/dequantize.cuh new file mode 100644 index 00000000..57cb1a27 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/dequantize.cuh @@ -0,0 +1,583 @@ +// copied from +// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/dequantize.cuh +// copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/convert.cu +// Dequant functions +static __device__ __forceinline__ void dequantize_q4_0(const void* vx, const int ib, const int iqs, dfloat2& v) { + const block_q4_0* x = (const block_q4_0*)vx; + + const dfloat d = x[ib].d; + + const int vui = x[ib].qs[iqs]; + + v.x = __int2half_rn(vui & 0xF); + v.y = __int2half_rn(vui >> 4); + + v = __hsub2(v, __floats2half2_rn(8.0f, 8.0f)); + v = __hmul2(v, {d, d}); +} + +static __device__ __forceinline__ void dequantize_q4_1(const void* vx, const int ib, const int iqs, dfloat2& v) { + const block_q4_1* x = (const block_q4_1*)vx; + + const dfloat d = __low2half(x[ib].dm); + const dfloat m = __high2half(x[ib].dm); + + const int vui = x[ib].qs[iqs]; + + v.x = __int2half_rn(vui & 0xF); + v.y = __int2half_rn(vui >> 4); + + v = __hmul2(v, {d, d}); + v = __hadd2(v, {m, m}); +} + +static __device__ __forceinline__ void dequantize_q5_0(const void* vx, const int ib, const int iqs, dfloat2& v) { + const block_q5_0* x = (const block_q5_0*)vx; + + const dfloat d = x[ib].d; + + uint32_t qh; + memcpy(&qh, x[ib].qh, sizeof(qh)); + + const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10; + const int xh_1 = ((qh >> (iqs + 12))) & 0x10; + + v.x = __int2half_rn((x[ib].qs[iqs] & 0xf) | xh_0); + v.y = __int2half_rn((x[ib].qs[iqs] >> 4) | xh_1); + + v = __hsub2(v, __floats2half2_rn(16.0f, 16.0f)); + v = __hmul2(v, {d, d}); +} + +static __device__ __forceinline__ void dequantize_q5_1(const void* vx, const int ib, const int iqs, dfloat2& v) { + const block_q5_1* x = (const block_q5_1*)vx; + + const dfloat d = __low2half(x[ib].dm); + const dfloat m = __high2half(x[ib].dm); + + uint32_t qh; + memcpy(&qh, x[ib].qh, sizeof(qh)); + + const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10; + const int xh_1 = ((qh >> (iqs + 12))) & 0x10; + + v.x = __int2half_rn((x[ib].qs[iqs] & 0xf) | xh_0); + v.y = __int2half_rn((x[ib].qs[iqs] >> 4) | xh_1); + + v = __hmul2(v, {d, d}); + v = __hadd2(v, {m, m}); +} + +static __device__ __forceinline__ void dequantize_q8_0(const void* vx, const int ib, const int iqs, dfloat2& v) { + const block_q8_0* x = (const block_q8_0*)vx; + + const dfloat d = x[ib].d; + + v.x = __int2half_rn(x[ib].qs[iqs + 0]); + v.y = __int2half_rn(x[ib].qs[iqs + 1]); + + v = __hmul2(v, {d, d}); +} + +template +static __global__ void dequantize_block(const void* __restrict__ vx, dst_t* __restrict__ y, const int k) { + const int i = 2 * (blockDim.x * blockIdx.x + threadIdx.x); + + if (i >= k) { + return; + } + + const int ib = i / qk; // block index + const int iqs = (i % qk) / qr; // quant index + const int iybs = i - i % qk; // y block start index + const int y_offset = qr == 1 ? 1 : qk / 2; + + // dequantize + dfloat2 v; + dequantize_kernel(vx, ib, iqs, v); + + y[iybs + iqs + 0] = convert_from_half(v.x); + y[iybs + iqs + y_offset] = convert_from_half(v.y); +} + +template +static __global__ void dequantize_block_q2_K(const void* __restrict__ vx, dst_t* __restrict__ yy) { + const auto i = blockIdx.x; + const block_q2_K* x = (const block_q2_K*)vx; + + const auto tid = threadIdx.x; + const int n = tid / 32; + const int l = tid - 32 * n; + const int is = 8 * n + l / 16; + + const uint8_t q = x[i].qs[32 * n + l]; + dst_t* y = yy + i * QK_K + 128 * n; + + half dall = __low2half(x[i].dm); + half dmin = __high2half(x[i].dm); + y[l + 0] = convert_from_half(__hsub( + __hmul(dall, __int2half_rn((x[i].scales[is + 0] & 0xF) * ((q >> 0) & 3))), + __hmul(dmin, __int2half_rn(x[i].scales[is + 0] >> 4)))); + y[l + 32] = convert_from_half(__hsub( + __hmul(dall, __int2half_rn((x[i].scales[is + 2] & 0xF) * ((q >> 2) & 3))), + __hmul(dmin, __int2half_rn(x[i].scales[is + 2] >> 4)))); + y[l + 64] = convert_from_half(__hsub( + __hmul(dall, __int2half_rn((x[i].scales[is + 4] & 0xF) * ((q >> 4) & 3))), + __hmul(dmin, __int2half_rn(x[i].scales[is + 4] >> 4)))); + y[l + 96] = convert_from_half(__hsub( + __hmul(dall, __int2half_rn((x[i].scales[is + 6] & 0xF) * ((q >> 6) & 3))), + __hmul(dmin, __int2half_rn(x[i].scales[is + 6] >> 4)))); +} + +template +static __global__ void dequantize_block_q3_K(const void* __restrict__ vx, dst_t* __restrict__ yy) { + const auto i = blockIdx.x; + const block_q3_K* x = (const block_q3_K*)vx; + + const auto r = threadIdx.x / 4; + const int tid = r / 2; + const int is0 = r % 2; + const int l0 = 16 * is0 + 4 * (threadIdx.x % 4); + const int n = tid / 4; + const int j = tid - 4 * n; + + uint8_t m = 1 << (4 * n + j); + int is = 8 * n + 2 * j + is0; + int shift = 2 * j; + + int8_t us = is < 4 ? (x[i].scales[is - 0] & 0xF) | (((x[i].scales[is + 8] >> 0) & 3) << 4) + : is < 8 ? (x[i].scales[is - 0] & 0xF) | (((x[i].scales[is + 4] >> 2) & 3) << 4) + : is < 12 ? (x[i].scales[is - 8] >> 4) | (((x[i].scales[is + 0] >> 4) & 3) << 4) + : (x[i].scales[is - 8] >> 4) | (((x[i].scales[is - 4] >> 6) & 3) << 4); + half d_all = x[i].d; + half dl = __hmul(d_all, __int2half_rn(us - 32)); + + dst_t* y = yy + i * QK_K + 128 * n + 32 * j; + const uint8_t* q = x[i].qs + 32 * n; + const uint8_t* hm = x[i].hmask; + + for (int l = l0; l < l0 + 4; ++l) { + y[l] = convert_from_half(__hmul(dl, __int2half_rn((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4)))); + } +} + +static inline __device__ void get_scale_min_k4(int j, const uint8_t* q, uint8_t& d, uint8_t& m) { + if (j < 4) { + d = q[j] & 63; + m = q[j + 4] & 63; + } else { + d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4); + m = (q[j + 4] >> 4) | ((q[j - 0] >> 6) << 4); + } +} + +template +static __global__ void dequantize_block_q4_K(const void* __restrict__ vx, dst_t* __restrict__ yy) { + const block_q4_K* x = (const block_q4_K*)vx; + + const auto i = blockIdx.x; + + // assume 32 threads + const auto tid = threadIdx.x; + const int il = tid / 8; + const int ir = tid % 8; + const int is = 2 * il; + const int n = 4; + + dst_t* y = yy + i * QK_K + 64 * il + n * ir; + + const half dall = __low2half(x[i].dm); + const half dmin = __high2half(x[i].dm); + + const uint8_t* q = x[i].qs + 32 * il + n * ir; + + uint8_t sc, m; + get_scale_min_k4(is + 0, x[i].scales, sc, m); + const half d1 = __hmul(dall, __int2half_rn(sc)); + const half m1 = __hmul(dmin, __int2half_rn(m)); + get_scale_min_k4(is + 1, x[i].scales, sc, m); + const half d2 = __hmul(dall, __int2half_rn(sc)); + const half m2 = __hmul(dmin, __int2half_rn(m)); + for (int l = 0; l < n; ++l) { + y[l + 0] = convert_from_half(__hsub(__hmul(d1, __int2half_rn(q[l] & 0xF)), m1)); + y[l + 32] = convert_from_half(__hsub(__hmul(d2, __int2half_rn(q[l] >> 4)), m2)); + } +} + +template +static __global__ void dequantize_block_q5_K(const void* __restrict__ vx, dst_t* __restrict__ yy) { + const block_q5_K* x = (const block_q5_K*)vx; + + const auto i = blockIdx.x; + + // assume 64 threads - this is very slightly better than the one below + const auto tid = threadIdx.x; + const int il = tid / 16; // il is in 0...3 + const int ir = tid % 16; // ir is in 0...15 + const int is = 2 * il; // is is in 0...6 + + dst_t* y = yy + i * QK_K + 64 * il + 2 * ir; + + const half dall = __low2half(x[i].dm); + const half dmin = __high2half(x[i].dm); + + const uint8_t* ql = x[i].qs + 32 * il + 2 * ir; + const uint8_t* qh = x[i].qh + 2 * ir; + + uint8_t sc, m; + get_scale_min_k4(is + 0, x[i].scales, sc, m); + const half d1 = __hmul(dall, __int2half_rn(sc)); + const half m1 = __hmul(dmin, __int2half_rn(m)); + get_scale_min_k4(is + 1, x[i].scales, sc, m); + const half d2 = __hmul(dall, __int2half_rn(sc)); + const half m2 = __hmul(dmin, __int2half_rn(m)); + + uint8_t hm = 1 << (2 * il); + y[0] = convert_from_half(__hsub(__hmul(d1, __int2half_rn((ql[0] & 0xF) + (qh[0] & hm ? 16 : 0))), m1)); + y[1] = convert_from_half(__hsub(__hmul(d1, __int2half_rn((ql[1] & 0xF) + (qh[1] & hm ? 16 : 0))), m1)); + hm <<= 1; + y[32] = convert_from_half(__hsub(__hmul(d2, __int2half_rn((ql[0] >> 4) + (qh[0] & hm ? 16 : 0))), m2)); + y[33] = convert_from_half(__hsub(__hmul(d2, __int2half_rn((ql[1] >> 4) + (qh[1] & hm ? 16 : 0))), m2)); +} + +template +static __global__ void dequantize_block_q6_K(const void* __restrict__ vx, dst_t* __restrict__ yy) { + const block_q6_K* x = (const block_q6_K*)vx; + + const auto i = blockIdx.x; + + // assume 64 threads - this is very slightly better than the one below + const auto tid = threadIdx.x; + const int ip = tid / 32; // ip is 0 or 1 + const int il = tid - 32 * ip; // 0...32 + const int is = 8 * ip + il / 16; + + dst_t* y = yy + i * QK_K + 128 * ip + il; + + const half d = x[i].d; + + const uint8_t* ql = x[i].ql + 64 * ip + il; + const uint8_t qh = x[i].qh[32 * ip + il]; + const int8_t* sc = x[i].scales + is; + + y[0] = convert_from_half( + __hmul(d, __int2half_rn(sc[0] * ((int8_t)((ql[0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32)))); + y[32] = convert_from_half( + __hmul(d, __int2half_rn(sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32)))); + y[64] = convert_from_half( + __hmul(d, __int2half_rn(sc[4] * ((int8_t)((ql[0] >> 4) | (((qh >> 4) & 3) << 4)) - 32)))); + y[96] = convert_from_half( + __hmul(d, __int2half_rn(sc[6] * ((int8_t)((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32)))); +} + +template +static __global__ void dequantize_block_iq2_xxs(const void* __restrict__ vx, dst_t* __restrict__ yy) { + const auto i = blockIdx.x; + const block_iq2_xxs* x = (const block_iq2_xxs*)vx; + + const auto tid = threadIdx.x; + const int il = tid / 8; // 0...3 + const int ib = tid % 8; // 0...7 + dst_t* y = yy + i * QK_K + 32 * ib + 8 * il; + const uint16_t* q2 = x[i].qs + 4 * ib; + const uint8_t* aux8 = (const uint8_t*)q2; + const uint8_t* grid = (const uint8_t*)(iq2xxs_grid + aux8[il]); + const uint32_t aux32 = q2[2] | (q2[3] << 16); + const float d = __half2float(x[i].d) * (0.5f + (aux32 >> 28)) * 0.25f; + const uint8_t signs = ksigns_iq2xs[(aux32 >> 7 * il) & 127]; + for (int j = 0; j < 8; ++j) + y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); +} + +template +static __global__ void dequantize_block_iq2_xs(const void* __restrict__ vx, dst_t* __restrict__ yy) { + const auto i = blockIdx.x; + const block_iq2_xs* x = (const block_iq2_xs*)vx; + + const auto tid = threadIdx.x; + const int il = tid / 8; // 0...3 + const int ib = tid % 8; // 0...7 + dst_t* y = yy + i * QK_K + 32 * ib + 8 * il; + const uint16_t* q2 = x[i].qs + 4 * ib; + const uint8_t* grid = (const uint8_t*)(iq2xs_grid + (q2[il] & 511)); + const float d = __half2float(x[i].d) * (0.5f + ((x[i].scales[ib] >> 4 * (il / 2)) & 0xf)) * 0.25f; + const uint8_t signs = ksigns_iq2xs[q2[il] >> 9]; + for (int j = 0; j < 8; ++j) + y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); +} + +template +static __global__ void dequantize_block_iq2_s(const void* __restrict__ vx, dst_t* __restrict__ yy) { + const auto i = blockIdx.x; + const block_iq2_s* x = (const block_iq2_s*)vx; + + const auto tid = threadIdx.x; + const int il = tid / 8; // 0...3 + const int ib = tid % 8; // 0...7 + dst_t* y = yy + i * QK_K + 32 * ib + 8 * il; + const uint8_t* grid = (const uint8_t*)(iq2s_grid + (x[i].qs[4 * ib + il] | ((x[i].qh[ib] << (8 - 2 * il)) & 0x300))); + const float d = __half2float(x[i].d) * (0.5f + ((x[i].scales[ib] >> 4 * (il / 2)) & 0xf)) * 0.25f; + const uint8_t signs = x[i].qs[QK_K / 8 + 4 * ib + il]; + for (int j = 0; j < 8; ++j) + y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); +} + +template +static __global__ void dequantize_block_iq3_xxs(const void* __restrict__ vx, dst_t* __restrict__ yy) { + const auto i = blockIdx.x; + const block_iq3_xxs* x = (const block_iq3_xxs*)vx; + + const auto tid = threadIdx.x; + const int il = tid / 8; // 0...3 + const int ib = tid % 8; // 0...7 + dst_t* y = yy + i * QK_K + 32 * ib + 8 * il; + const uint8_t* q3 = x[i].qs + 8 * ib; + const uint16_t* gas = (const uint16_t*)(x[i].qs + QK_K / 4) + 2 * ib; + const uint8_t* grid1 = (const uint8_t*)(iq3xxs_grid + q3[2 * il + 0]); + const uint8_t* grid2 = (const uint8_t*)(iq3xxs_grid + q3[2 * il + 1]); + const uint32_t aux32 = gas[0] | (gas[1] << 16); + const float d = __half2float(x[i].d) * (0.5f + (aux32 >> 28)) * 0.5f; + const uint8_t signs = ksigns_iq2xs[(aux32 >> 7 * il) & 127]; + for (int j = 0; j < 4; ++j) { + y[j + 0] = d * grid1[j] * (signs & kmask_iq2xs[j + 0] ? -1.f : 1.f); + y[j + 4] = d * grid2[j] * (signs & kmask_iq2xs[j + 4] ? -1.f : 1.f); + } +} + +template +static __global__ void dequantize_block_iq3_s(const void* __restrict__ vx, dst_t* __restrict__ yy) { + const auto i = blockIdx.x; + const block_iq3_s* x = (const block_iq3_s*)vx; + + const auto tid = threadIdx.x; + const int il = tid / 8; // 0...3 + const int ib = tid % 8; // 0...7 + dst_t* y = yy + i * QK_K + 32 * ib + 8 * il; + const uint8_t* qs = x[i].qs + 8 * ib; + const uint8_t* grid1 = (const uint8_t*)(iq3xs_grid + (qs[2 * il + 0] | ((x[i].qh[ib] << (8 - 2 * il)) & 256))); + const uint8_t* grid2 = (const uint8_t*)(iq3xs_grid + (qs[2 * il + 1] | ((x[i].qh[ib] << (7 - 2 * il)) & 256))); + const float d = __half2float(x[i].d) * (0.5f + ((x[i].scales[ib / 2] >> 4 * (ib % 2)) & 0xf)) * 0.5f; + const uint8_t signs = x[i].signs[4 * ib + il]; + for (int j = 0; j < 4; ++j) { + y[j + 0] = d * grid1[j] * (signs & kmask_iq2xs[j + 0] ? -1.f : 1.f); + y[j + 4] = d * grid2[j] * (signs & kmask_iq2xs[j + 4] ? -1.f : 1.f); + } +} + +template +static __global__ void dequantize_block_iq1_s(const void* __restrict__ vx, dst_t* __restrict__ yy) { + const int64_t i = blockIdx.x; + const block_iq1_s* x = (const block_iq1_s*)vx; + + const int64_t tid = threadIdx.x; + const int64_t il = tid / 8; // 0...3 + const int64_t ib = tid % 8; // 0...7 + dst_t* y = yy + i * QK_K + 32 * ib + 8 * il; + const float delta = x[i].qh[ib] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA; + const float d = __half2float(x[i].d) * (2 * ((x[i].qh[ib] >> 12) & 7) + 1); + uint32_t grid32[2]; + const int8_t* q = (const int8_t*)grid32; + grid32[0] = iq1s_grid_gpu[x[i].qs[4 * ib + il] | (((x[i].qh[ib] >> 3 * il) & 7) << 8)]; + grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f; + grid32[0] &= 0x0f0f0f0f; + for (int j = 0; j < 8; ++j) { + y[j] = d * (q[j] + delta); + } +} + +template +static __global__ void dequantize_block_iq1_m(const void* __restrict__ vx, dst_t* __restrict__ yy) { + const int64_t i = blockIdx.x; + const block_iq1_m* x = (const block_iq1_m*)vx; + + const int64_t tid = threadIdx.x; + const int64_t il = tid / 8; // 0...3 + const int64_t ib = tid % 8; // 0...7 + dst_t* y = yy + i * QK_K + 32 * ib + 8 * il; + const uint16_t* sc = (const uint16_t*)x[i].scales; + iq1m_scale_t scale; + scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); + const int64_t ib16 = 2 * ib + il / 2; // sc[ib16/4] >> 3*(ib16%4) -> sc[ib/2] >> 3*((2*ib+il/2)%4); + const float d = __half2float(scale.f16) * (2 * ((sc[ib16 / 4] >> 3 * (ib16 % 4)) & 0x7) + 1); + const float delta = x[i].qh[2 * ib + il / 2] & (0x08 << 4 * (il % 2)) ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA; + uint32_t grid32[2]; + const int8_t* q = (const int8_t*)grid32; + grid32[0] = iq1s_grid_gpu[x[i].qs[4 * ib + il] | (((x[i].qh[2 * ib + il / 2] >> 4 * (il % 2)) & 7) << 8)]; + grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f; + grid32[0] &= 0x0f0f0f0f; + for (int j = 0; j < 8; ++j) { + y[j] = d * (q[j] + delta); + } +} + +template +static __global__ void dequantize_block_iq4_nl(const void* __restrict__ vx, dst_t* __restrict__ yy) { + const auto i = blockIdx.x; + const block_iq4_nl* x = (const block_iq4_nl*)vx + i * (QK_K / QK4_NL); + + const auto tid = threadIdx.x; + const int il = tid / 8; // 0...3 + const int ib = tid % 8; // 0...7 + dst_t* y = yy + i * QK_K + 32 * ib + 4 * il; + const uint8_t* q4 = x[ib].qs + 4 * il; + const float d = __half2float(x[ib].d); + for (int j = 0; j < 4; ++j) { + y[j + 0] = d * kvalues_iq4nl[q4[j] & 0xf]; + y[j + 16] = d * kvalues_iq4nl[q4[j] >> 4]; + } +} + +template +static __global__ void dequantize_block_iq4_xs(const void* __restrict__ vx, dst_t* __restrict__ yy) { + const auto i = blockIdx.x; + const block_iq4_xs* x = (const block_iq4_xs*)vx; + + const auto tid = threadIdx.x; + const int il = tid / 8; // 0...3 + const int ib = tid % 8; // 0...7 + dst_t* y = yy + i * QK_K + 32 * ib + 4 * il; + const uint8_t* q4 = x[i].qs + 16 * ib + 4 * il; + const float d = __half2float(x[i].d) * + ((((x[i].scales_l[ib / 2] >> 4 * (ib % 2)) & 0xf) | (((x[i].scales_h >> 2 * ib) & 3) << 4)) - 32); + for (int j = 0; j < 4; ++j) { + y[j + 0] = d * kvalues_iq4nl[q4[j] & 0xf]; + y[j + 16] = d * kvalues_iq4nl[q4[j] >> 4]; + } +} + +template +static void +dequantize_block_cuda(const void* __restrict__ vx, dst_t* __restrict__ y, const int k, cudaStream_t stream) { + const int num_blocks = (k + 2 * CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / (2 * CUDA_DEQUANTIZE_BLOCK_SIZE); + dequantize_block<<>>(vx, y, k); +} + +template +static void dequantize_row_q2_K_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_q2_K<<>>(vx, y); +} + +template +static void dequantize_row_q3_K_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_q3_K<<>>(vx, y); +} + +template +static void dequantize_row_q4_K_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_q4_K<<>>(vx, y); +} + +template +static void dequantize_row_q5_K_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_q5_K<<>>(vx, y); +} + +template +static void dequantize_row_q6_K_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_q6_K<<>>(vx, y); +} + +template +static void dequantize_row_iq2_xxs_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_iq2_xxs<<>>(vx, y); +} + +template +static void dequantize_row_iq2_xs_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_iq2_xs<<>>(vx, y); +} + +template +static void dequantize_row_iq2_s_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_iq2_s<<>>(vx, y); +} + +template +static void dequantize_row_iq3_xxs_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_iq3_xxs<<>>(vx, y); +} + +template +static void dequantize_row_iq3_s_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_iq3_s<<>>(vx, y); +} + +template +static void dequantize_row_iq1_s_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_iq1_s<<>>(vx, y); +} + +template +static void dequantize_row_iq1_m_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_iq1_m<<>>(vx, y); +} + +template +static void dequantize_row_iq4_nl_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) { + const int nb = (k + QK_K - 1) / QK_K; + dequantize_block_iq4_nl<<>>(vx, y); +} + +template +static void dequantize_row_iq4_xs_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) { + const int nb = (k + QK_K - 1) / QK_K; + dequantize_block_iq4_xs<<>>(vx, y); +} + +template +static to_cuda_ggml_t ggml_get_to_cuda(int64_t type) { + switch (type) { + case 2: + return dequantize_block_cuda; + case 3: + return dequantize_block_cuda; + case 6: + return dequantize_block_cuda; + case 7: + return dequantize_block_cuda; + case 8: + return dequantize_block_cuda; + case 10: + return dequantize_row_q2_K_cuda; + case 11: + return dequantize_row_q3_K_cuda; + case 12: + return dequantize_row_q4_K_cuda; + case 13: + return dequantize_row_q5_K_cuda; + case 14: + return dequantize_row_q6_K_cuda; + case 16: + return dequantize_row_iq2_xxs_cuda; + case 17: + return dequantize_row_iq2_xs_cuda; + case 18: + return dequantize_row_iq3_xxs_cuda; + case 19: + return dequantize_row_iq1_s_cuda; + case 20: + return dequantize_row_iq4_nl_cuda; + case 21: + return dequantize_row_iq3_s_cuda; + case 22: + return dequantize_row_iq2_s_cuda; + case 23: + return dequantize_row_iq4_xs_cuda; + case 29: + return dequantize_row_iq1_m_cuda; + default: + return nullptr; + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/ggml-common.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/ggml-common.h new file mode 100644 index 00000000..f6fbe57a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/ggml-common.h @@ -0,0 +1,1029 @@ +// adapted from +// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/ggml-common.h +// copied from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-common.h +#define QK_K 256 +#define K_QUANTS_PER_ITERATION 2 +#define WARP_SIZE_GGUF 32 +#define K_SCALE_SIZE 12 +#define CUDA_DEQUANTIZE_BLOCK_SIZE 256 +#define CUDA_QUANTIZE_BLOCK_SIZE 256 +#define GGML_CUDA_DMMV_X 32 +#define GGML_CUDA_MMV_Y 1 + +// Data Structures +// QK = number of values after dequantization +// QR = QK / number of values before dequantization +// QI = number of 32 bit integers before dequantization + +#define QK4_0 32 +#define QR4_0 2 +#define QI4_0 (QK4_0 / (4 * QR4_0)) +typedef struct { + half d; // delta + uint8_t qs[QK4_0 / 2]; // nibbles / quants +} block_q4_0; + +#define QK4_1 32 +#define QR4_1 2 +#define QI4_1 (QK4_1 / (4 * QR4_1)) +typedef struct { + half2 dm; // dm.x = delta, dm.y = min + uint8_t qs[QK4_1 / 2]; // nibbles / quants +} block_q4_1; + +#define QK5_0 32 +#define QR5_0 2 +#define QI5_0 (QK5_0 / (4 * QR5_0)) +typedef struct { + half d; // delta + uint8_t qh[4]; // 5-th bit of quants + uint8_t qs[QK5_0 / 2]; // nibbles / quants +} block_q5_0; + +#define QK5_1 32 +#define QR5_1 2 +#define QI5_1 (QK5_1 / (4 * QR5_1)) +typedef struct { + half2 dm; // dm.x = delta, dm.y = min + uint8_t qh[4]; // 5-th bit of quants + uint8_t qs[QK5_1 / 2]; // nibbles / quants +} block_q5_1; + +#define QK8_0 32 +#define QR8_0 1 +#define QI8_0 (QK8_0 / (4 * QR8_0)) +typedef struct { + half d; // delta + int8_t qs[QK8_0]; // quants +} block_q8_0; + +#define QK8_1 32 +#define QR8_1 1 +#define QI8_1 (QK8_1 / (4 * QR8_1)) +typedef struct { + half2 ds; // ds.x = delta, ds.y = sum + int8_t qs[QK8_0]; // quants +} block_q8_1; + +#define QR2_K 4 +#define QI2_K (QK_K / (4 * QR2_K)) +typedef struct { + uint8_t scales[QK_K / 16]; // scales and mins, quantized with 4 bits + uint8_t qs[QK_K / 4]; // quants + half2 dm; // super-block scale for quantized scales/mins +} block_q2_K; + +#define QR3_K 4 +#define QI3_K (QK_K / (4 * QR3_K)) +typedef struct { + uint8_t hmask[QK_K / 8]; // quants - high bit + uint8_t qs[QK_K / 4]; // quants - low 2 bits + uint8_t scales[K_SCALE_SIZE]; // scales, quantized with 6 bits + half d; // super-block scale +} block_q3_K; + +#define QR4_K 2 +#define QI4_K (QK_K / (4 * QR4_K)) +typedef struct { + half2 dm; // super-block scale for quantized scales/mins + uint8_t scales[3 * QK_K / 64]; // scales, quantized with 6 bits + uint8_t qs[QK_K / 2]; // 4--bit quants +} block_q4_K; + +#define QR5_K 2 +#define QI5_K (QK_K / (4 * QR5_K)) +typedef struct { + half2 dm; // super-block scale for quantized scales/mins + uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits + uint8_t qh[QK_K / 8]; // quants, high bit + uint8_t qs[QK_K / 2]; // quants, low 4 bits +} block_q5_K; + +#define QR6_K 2 +#define QI6_K (QK_K / (4 * QR6_K)) +typedef struct { + uint8_t ql[QK_K / 2]; // quants, lower 4 bits + uint8_t qh[QK_K / 4]; // quants, upper 2 bits + int8_t scales[QK_K / 16]; // scales + half d; // delta +} block_q6_K; + +#define QR2_XXS 8 +#define QI2_XXS (QK_K / (4 * QR2_XXS)) +typedef struct { + half d; + uint16_t qs[QK_K / 8]; +} block_iq2_xxs; + +#define QR2_XS 8 +#define QI2_XS (QK_K / (4 * QR2_XS)) +typedef struct { + half d; + uint16_t qs[QK_K / 8]; + uint8_t scales[QK_K / 32]; +} block_iq2_xs; + +#define QR2_S 8 +#define QI2_S (QK_K / (4 * QR2_S)) +typedef struct { + half d; + uint8_t qs[QK_K / 4]; + uint8_t qh[QK_K / 32]; + uint8_t scales[QK_K / 32]; +} block_iq2_s; + +#define QR3_XXS 8 +#define QI3_XXS (QK_K / (4 * QR3_XXS)) +typedef struct { + half d; + uint8_t qs[3 * (QK_K / 8)]; +} block_iq3_xxs; + +#define QR3_XS 8 +#define QI3_XS (QK_K / (4 * QR3_XS)) +#define IQ3S_N_SCALE QK_K / 64 +typedef struct { + half d; + uint8_t qs[QK_K / 4]; + uint8_t qh[QK_K / 32]; + uint8_t signs[QK_K / 8]; + uint8_t scales[IQ3S_N_SCALE]; +} block_iq3_s; + +// 1.5625 bpw +#define QR1_S 8 +#define QI1_S (QK_K / (4 * QR1_S)) +typedef struct { + half d; + uint8_t qs[QK_K / 8]; + uint16_t qh[QK_K / 32]; +} block_iq1_s; + +// 1.75 bpw +#define QR1_M 8 +#define QI1_M (QK_K / (4 * QR1_M)) +typedef struct { + uint8_t qs[QK_K / 8]; // grid index, low 8 bits + uint8_t qh[QK_K / 16]; // grid index, high 3 bits + grid shift bit (for two groups of 8) + uint8_t scales[QK_K / 32]; // 3-bit block scales (4-bit if QK_K == 64) +} block_iq1_m; + +// Used by IQ1_M quants +typedef union { + half f16; + uint16_t u16; +} iq1m_scale_t; + +#define QK4_NL 32 +#define QR4_NL 2 +#define QI4_NL (QK4_NL / (4 * QR4_NL)) +typedef struct { + half d; + uint8_t qs[QK4_NL / 2]; +} block_iq4_nl; + +#define QR4_XS 8 +#define QI4_XS (QK_K / (4 * QR4_XS)) +typedef struct { + half d; + uint16_t scales_h; + uint8_t scales_l[QK_K / 64]; + uint8_t qs[QK_K / 2]; +} block_iq4_xs; + +static const __device__ uint64_t iq2xxs_grid[256] = { + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, 0x0808080808082b2b, + 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b2b08, + 0x08080808082b2b2b, 0x0808080819080819, 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08, + 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, 0x080808082b082b2b, + 0x080808082b2b082b, 0x0808081908080819, 0x0808081908081908, 0x0808081908190808, 0x0808081908191919, + 0x0808081919080808, 0x080808192b081908, 0x080808192b192b08, 0x0808082b08080808, 0x0808082b0808082b, + 0x0808082b082b082b, 0x0808082b2b08082b, 0x0808190808080819, 0x0808190808081908, 0x0808190808190808, + 0x08081908082b0819, 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819082b08, + 0x08081908192b0808, 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x080819082b2b1908, + 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08, 0x08081919082b0808, 0x080819191908192b, + 0x08081919192b2b19, 0x080819192b080808, 0x080819192b190819, 0x0808192b08082b19, 0x0808192b08190808, + 0x0808192b19080808, 0x0808192b2b081908, 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919, + 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, 0x08082b0819080819, 0x08082b0819081908, + 0x08082b0819190808, 0x08082b081919082b, 0x08082b082b082b08, 0x08082b1908081908, 0x08082b1919080808, + 0x08082b2b0808082b, 0x08082b2b08191908, 0x0819080808080819, 0x0819080808081908, 0x0819080808190808, + 0x08190808082b0819, 0x0819080819080808, 0x08190808192b0808, 0x081908082b081908, 0x081908082b190808, + 0x081908082b191919, 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808, 0x0819081919190808, + 0x0819081919192b2b, 0x081908192b080808, 0x0819082b082b1908, 0x0819082b19081919, 0x0819190808080808, + 0x0819190808082b08, 0x08191908082b0808, 0x08191908082b1919, 0x0819190819082b19, 0x081919082b080808, + 0x0819191908192b08, 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, 0x08192b0808080819, + 0x08192b0808081908, 0x08192b0808190808, 0x08192b0819080808, 0x08192b082b080819, 0x08192b1908080808, + 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819, 0x082b080808080808, 0x082b08080808082b, + 0x082b080808082b2b, 0x082b080819081908, 0x082b0808192b0819, 0x082b08082b080808, 0x082b08082b08082b, + 0x082b0819082b2b19, 0x082b081919082b08, 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819, + 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, 0x082b19081919192b, 0x082b191908080808, + 0x082b191919080819, 0x082b1919192b1908, 0x082b192b2b190808, 0x082b2b0808082b08, 0x082b2b08082b0808, + 0x082b2b082b191908, 0x082b2b2b19081908, 0x1908080808080819, 0x1908080808081908, 0x1908080808190808, + 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, 0x1908080819082b08, + 0x190808081919192b, 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808, + 0x1908081908080808, 0x19080819082b0808, 0x19080819192b0819, 0x190808192b080808, 0x190808192b081919, + 0x1908082b08080819, 0x1908082b08190808, 0x1908082b19082b08, 0x1908082b1919192b, 0x1908082b192b2b08, + 0x1908190808080808, 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, 0x190819082b192b19, + 0x190819190819082b, 0x19081919082b1908, 0x1908192b08080808, 0x19082b0808080819, 0x19082b0808081908, + 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b1908080808, 0x19082b1919192b08, + 0x19082b19192b0819, 0x19082b192b08082b, 0x19082b2b19081919, 0x19082b2b2b190808, 0x1919080808080808, + 0x1919080808082b08, 0x1919080808190819, 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808, + 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, 0x191908192b2b1908, 0x1919082b2b190819, + 0x191919082b190808, 0x191919082b19082b, 0x1919191908082b2b, 0x1919192b08080819, 0x1919192b19191908, + 0x19192b0808080808, 0x19192b0808190819, 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808, + 0x19192b2b08082b08, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, 0x192b0808192b2b08, + 0x192b081908080808, 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808, 0x192b190808080808, + 0x192b190808081919, 0x192b191908190808, 0x192b19190819082b, 0x192b19192b081908, 0x192b2b081908082b, + 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808082b2b, 0x2b08080819080819, 0x2b0808082b08082b, + 0x2b08081908081908, 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, 0x2b08190808080819, + 0x2b08190808081908, 0x2b08190808190808, 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, + 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908, 0x2b08192b08082b19, 0x2b08192b19080808, + 0x2b08192b192b0808, 0x2b082b080808082b, 0x2b082b1908081908, 0x2b082b2b08190819, 0x2b19080808081908, + 0x2b19080808190808, 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b, + 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, 0x2b191908082b082b, 0x2b19190819081908, + 0x2b19191919190819, 0x2b192b082b080819, 0x2b192b19082b0808, 0x2b2b08080808082b, 0x2b2b080819190808, + 0x2b2b08082b081919, 0x2b2b081908082b19, 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, + 0x2b2b2b1908081908, +}; + +static const __device__ uint64_t iq2xs_grid[512] = { + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, 0x0808080808082b2b, + 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, 0x0808080808192b19, 0x08080808082b0808, + 0x08080808082b082b, 0x08080808082b1919, 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, + 0x080808081908192b, 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919, + 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, + 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b192b19, + 0x080808082b2b0808, 0x0808081908080819, 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, + 0x0808081908190808, 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b, + 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, + 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908, 0x08080819192b0808, 0x08080819192b2b08, + 0x080808192b080819, 0x080808192b081908, 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, + 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808, + 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919, 0x0808082b2b080808, + 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b, 0x0808190808082b19, + 0x0808190808190808, 0x080819080819082b, 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, + 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08, + 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808, 0x080819082b080819, + 0x080819082b081908, 0x080819082b190808, 0x0808191908080808, 0x080819190808082b, 0x0808191908081919, + 0x0808191908082b08, 0x0808191908190819, 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, + 0x0808191919081908, 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819, + 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808, 0x0808192b1908082b, + 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b, 0x08082b0808081919, 0x08082b0808082b08, + 0x08082b0808082b2b, 0x08082b0808190819, 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, + 0x08082b0819080819, 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808, + 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908, 0x08082b1908190808, + 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19, 0x08082b2b08080808, 0x08082b2b082b0808, + 0x08082b2b082b2b08, 0x08082b2b2b19192b, 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, + 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919, + 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808, 0x081908081908082b, + 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x08190808192b0808, + 0x08190808192b2b2b, 0x081908082b080819, 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, + 0x081908190808082b, 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908, + 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808, 0x081908192b080808, + 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819, 0x0819082b08081908, 0x0819082b0808192b, + 0x0819082b08190808, 0x0819082b19080808, 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, + 0x0819190808081919, 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808, + 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808, 0x08191908192b1908, + 0x081919082b080808, 0x0819191908080819, 0x0819191908081908, 0x0819191908190808, 0x0819191919080808, + 0x0819192b08080808, 0x0819192b08191908, 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, + 0x08192b0808190808, 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b, + 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819, 0x08192b2b2b2b2b19, + 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08, 0x082b080808082b2b, + 0x082b080808190819, 0x082b080808191908, 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, + 0x082b080819190808, 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908, + 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919, 0x082b082b08080808, + 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908, + 0x082b190808190808, 0x082b1908082b2b19, 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, + 0x082b19191919082b, 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b, + 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808, 0x082b2b0819191919, + 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08, 0x082b2b192b190808, 0x082b2b2b08082b08, + 0x082b2b2b082b0808, 0x082b2b2b2b08082b, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, + 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b, + 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, + 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b, 0x1908080819190819, + 0x1908080819191908, 0x19080808192b0808, 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, + 0x190808082b190808, 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08, + 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819, 0x1908081919081908, + 0x1908081919190808, 0x190808192b080808, 0x190808192b081919, 0x190808192b2b082b, 0x1908082b08080819, + 0x1908082b08081908, 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, + 0x1908190808080808, 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819, + 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819, 0x1908190819081908, + 0x1908190819190808, 0x190819082b080808, 0x190819082b191908, 0x1908191908080819, 0x1908191908081908, + 0x1908191908190808, 0x19081919082b1908, 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, + 0x1908192b08082b2b, 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908, + 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908, 0x19082b08192b082b, + 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908, 0x19082b1919190808, 0x19082b19192b2b19, + 0x19082b2b08081908, 0x1919080808080808, 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, + 0x1919080808190819, 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819, + 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819, 0x1919081908081908, + 0x1919081908190808, 0x1919081908191919, 0x1919081919080808, 0x191908191908082b, 0x1919082b08080808, + 0x1919082b19081908, 0x1919082b2b2b2b2b, 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, + 0x19191908082b0819, 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819, + 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08, 0x1919192b082b0819, + 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808, 0x19192b0808191908, 0x19192b0819080819, + 0x19192b0819190808, 0x19192b082b192b19, 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, + 0x19192b2b2b081919, 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, + 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19, 0x192b081908080808, + 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b, 0x192b190808080808, 0x192b19080819192b, + 0x192b191908190808, 0x192b191919080808, 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, + 0x192b2b08192b2b2b, 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b, + 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819, + 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b, 0x2b08080819080819, 0x2b08080819081908, + 0x2b08080819190808, 0x2b0808082b080808, 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, + 0x2b08081908080819, 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808, + 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808, 0x2b08082b2b080808, + 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08, 0x2b08190808080819, 0x2b08190808081908, + 0x2b08190808190808, 0x2b0819080819082b, 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, + 0x2b0819082b082b19, 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08, + 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919, 0x2b082b0819192b2b, + 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08, 0x2b082b190808192b, 0x2b082b2b082b082b, + 0x2b082b2b2b080808, 0x2b082b2b2b082b08, 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, + 0x2b19080808081908, 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908, + 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b, 0x2b19082b2b082b19, + 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908, 0x2b19190819190808, 0x2b19190819192b08, + 0x2b191919082b2b19, 0x2b1919192b190808, 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, + 0x2b192b082b2b192b, 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808, + 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808, 0x2b2b0808082b2b2b, + 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19, 0x2b2b08192b2b192b, 0x2b2b082b08080808, + 0x2b2b082b0808082b, 0x2b2b082b08082b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, + 0x2b2b190819080808, 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b, + 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808, 0x2b2b2b082b2b2b08, + 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, + 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b, +}; + +static const __device__ uint64_t iq2s_grid[1024] = { + 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, 0x0808080808082b2b, + 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, 0x0808080808192b19, 0x08080808082b0808, + 0x08080808082b082b, 0x08080808082b1919, 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, + 0x080808081908192b, 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919, + 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b, 0x08080808192b2b19, + 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819, + 0x080808082b191908, 0x080808082b2b0808, 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, + 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b, + 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, + 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908, + 0x080808191919192b, 0x0808081919192b19, 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, + 0x080808192b080819, 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919, + 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919, + 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808, 0x0808082b082b2b2b, + 0x0808082b19080819, 0x0808082b19081908, 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, + 0x0808082b19191919, 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908, + 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b, 0x0808190808082b19, + 0x0808190808190808, 0x080819080819082b, 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, + 0x08081908082b1908, 0x08081908082b192b, 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, + 0x0808190819081919, 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908, + 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b, 0x08081908192b1919, + 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b, 0x080819082b082b19, 0x080819082b190808, + 0x080819082b191919, 0x080819082b192b08, 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, + 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819, + 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808, 0x08081919082b1919, + 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908, 0x080819191908192b, 0x0808191919082b19, + 0x0808191919190808, 0x080819191919082b, 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, + 0x08081919192b1908, 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08, + 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819, 0x0808192b08081908, + 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808, 0x0808192b08191919, 0x0808192b19080808, + 0x0808192b19081919, 0x0808192b19082b08, 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, + 0x0808192b2b080819, 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b, + 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908, 0x08082b080819192b, + 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b08082b2b2b, 0x08082b0819080819, + 0x08082b0819081908, 0x08082b081908192b, 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, + 0x08082b0819191919, 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808, + 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908, + 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919, 0x08082b1908192b08, 0x08082b19082b0819, + 0x08082b1919080808, 0x08082b1919081919, 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, + 0x08082b19192b0808, 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819, + 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b, 0x08082b2b19190808, + 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b, 0x0819080808082b19, + 0x0819080808190808, 0x081908080819082b, 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, + 0x08190808082b1908, 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919, + 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b, 0x0819080819192b19, + 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919, 0x08190808192b2b08, 0x081908082b080819, + 0x081908082b081908, 0x081908082b08192b, 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, + 0x081908082b2b0819, 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919, + 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908, 0x081908190819192b, + 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b, 0x08190819082b1919, 0x08190819082b2b08, + 0x0819081919080819, 0x0819081919081908, 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, + 0x081908191919082b, 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908, + 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08, 0x081908192b190819, + 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908, 0x0819082b08082b19, 0x0819082b08190808, + 0x0819082b08191919, 0x0819082b082b0819, 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, + 0x0819082b19190819, 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808, + 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08, 0x0819190808190819, + 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19, 0x08191908082b0808, 0x08191908082b1919, + 0x08191908082b2b08, 0x0819190819080819, 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, + 0x0819190819190808, 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819, + 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919, 0x081919082b082b08, + 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808, 0x0819191908080819, 0x0819191908081908, + 0x081919190808192b, 0x0819191908082b19, 0x0819191908190808, 0x081919190819082b, 0x0819191908191919, + 0x0819191908192b08, 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b, + 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908, 0x08191919192b0808, + 0x081919192b080819, 0x081919192b081908, 0x081919192b190808, 0x0819192b08080808, 0x0819192b08081919, + 0x0819192b08082b08, 0x0819192b08190819, 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, + 0x0819192b19081908, 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819, + 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808, 0x08192b0808191919, + 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808, 0x08192b081908082b, 0x08192b0819081919, + 0x08192b0819082b08, 0x08192b0819190819, 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, + 0x08192b082b081908, 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08, + 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819, 0x08192b1919081908, + 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b, 0x08192b2b08081908, 0x08192b2b08190808, + 0x08192b2b19080808, 0x08192b2b1919192b, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, + 0x082b080808082b08, 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19, + 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819, 0x082b080819081908, + 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919, 0x082b0808192b1908, 0x082b08082b080808, + 0x082b08082b082b2b, 0x082b08082b191908, 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, + 0x082b081908190808, 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808, + 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908, 0x082b0819192b0808, + 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808, 0x082b082b08080808, 0x082b082b08082b2b, + 0x082b082b082b082b, 0x082b082b082b2b08, 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, + 0x082b082b2b082b08, 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908, + 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919, 0x082b190808192b08, + 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808, 0x082b19081908082b, 0x082b190819081919, + 0x082b190819082b08, 0x082b190819190819, 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, + 0x082b19082b081908, 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08, + 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819, 0x082b191919081908, + 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808, 0x082b192b08080819, 0x082b192b08081908, + 0x082b192b08190808, 0x082b192b19080808, 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, + 0x082b2b0808190819, 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808, + 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908, 0x082b2b1908190808, + 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b, 0x082b2b2b192b1908, 0x082b2b2b2b082b08, + 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, + 0x1908080808190808, 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b, + 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808, 0x190808081908082b, + 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, + 0x190808081919192b, 0x1908080819192b19, 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, + 0x190808082b080819, 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08, + 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, + 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908, 0x190808190819192b, 0x1908081908192b19, + 0x19080819082b0808, 0x19080819082b082b, 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, + 0x190808191908192b, 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919, + 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808, 0x190808192b08082b, + 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819, 0x190808192b191908, 0x190808192b2b0808, + 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, + 0x1908082b08192b08, 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08, + 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819, 0x1908082b2b081908, + 0x1908190808080808, 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808082b2b, + 0x1908190808190819, 0x1908190808191908, 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, + 0x19081908082b082b, 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908, + 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b, 0x1908190819191919, + 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908, 0x190819082b080808, 0x190819082b08082b, + 0x190819082b081919, 0x190819082b082b08, 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, + 0x1908191908080819, 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808, + 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819, 0x19081919082b1908, + 0x1908191919080808, 0x190819191908082b, 0x1908191919081919, 0x1908191919082b08, 0x1908191919190819, + 0x1908191919191908, 0x19081919192b0808, 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, + 0x190819192b190808, 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08, + 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819, 0x1908192b19081908, + 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919, 0x19082b0808080819, 0x19082b0808081908, + 0x19082b0808082b19, 0x19082b0808190808, 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, + 0x19082b08082b0819, 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919, + 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808, 0x19082b082b081908, + 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b, 0x19082b1908081919, 0x19082b1908082b08, + 0x19082b1908190819, 0x19082b1908191908, 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, + 0x19082b1919190808, 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908, + 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b, 0x1919080808081919, + 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908, 0x191908080819192b, 0x1919080808192b19, + 0x19190808082b0808, 0x19190808082b082b, 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, + 0x1919080819081908, 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b, + 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908, 0x191908082b080808, + 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08, 0x191908082b190819, 0x191908082b191908, + 0x1919081908080819, 0x1919081908081908, 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, + 0x191908190819082b, 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908, + 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08, 0x1919081919190819, + 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819, 0x191908192b081908, 0x191908192b190808, + 0x1919082b08080808, 0x1919082b08081919, 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, + 0x1919082b082b0808, 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19, + 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b, 0x1919190808082b19, + 0x1919190808190808, 0x191919080819082b, 0x1919190808191919, 0x1919190808192b08, 0x19191908082b0819, + 0x19191908082b1908, 0x1919190819080808, 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, + 0x1919190819190819, 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908, + 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919, 0x1919191908082b08, + 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808, 0x1919191919080819, 0x1919191919081908, + 0x1919191919190808, 0x191919192b080808, 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, + 0x1919192b082b192b, 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919, + 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808, 0x19192b0819080819, + 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b, 0x19192b082b080808, 0x19192b1908080819, + 0x19192b1908081908, 0x19192b1908190808, 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, + 0x19192b2b2b081919, 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b, + 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08, 0x192b0808082b0819, + 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919, 0x192b080819082b08, 0x192b080819190819, + 0x192b080819191908, 0x192b0808192b0808, 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, + 0x192b08190808082b, 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908, + 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808, 0x192b08192b080808, + 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808, 0x192b082b19080808, 0x192b082b1919192b, + 0x192b082b2b2b0819, 0x192b190808080808, 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, + 0x192b190808191908, 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808, + 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808, 0x192b191919080808, + 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b, 0x192b192b08080808, 0x192b192b2b191908, + 0x192b2b0808080819, 0x192b2b0808081908, 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, + 0x192b2b1908080808, 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808, + 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819, 0x2b08080808191908, + 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919, 0x2b08080819080819, 0x2b08080819081908, + 0x2b08080819190808, 0x2b0808081919082b, 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, + 0x2b0808082b080808, 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819, + 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b, 0x2b08081908191919, + 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908, 0x2b08081919080808, 0x2b0808191908082b, + 0x2b08081919081919, 0x2b08081919082b08, 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, + 0x2b0808192b081908, 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919, + 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819, 0x2b08082b19081908, + 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908, 0x2b0819080808192b, 0x2b08190808082b19, + 0x2b08190808190808, 0x2b0819080819082b, 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, + 0x2b08190819080808, 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819, + 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908, 0x2b0819082b190808, + 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919, 0x2b08191908082b08, 0x2b08191908190819, + 0x2b08191908191908, 0x2b081919082b0808, 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, + 0x2b0819192b080808, 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808, + 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919, 0x2b082b0808190819, + 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908, 0x2b082b0819190808, 0x2b082b082b2b082b, + 0x2b082b1908080819, 0x2b082b1908081908, 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, + 0x2b082b2b19192b08, 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819, + 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b, 0x2b19080808191919, + 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908081908082b, 0x2b19080819081919, + 0x2b19080819082b08, 0x2b19080819190819, 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, + 0x2b1908082b081908, 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819, + 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808, 0x2b19081919192b2b, + 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808, 0x2b19082b19080808, 0x2b19082b2b2b192b, + 0x2b19190808080808, 0x2b1919080808082b, 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, + 0x2b19190808191908, 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808, + 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908, 0x2b19191908190808, + 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819, 0x2b19192b08080808, 0x2b19192b1908192b, + 0x2b19192b192b1908, 0x2b192b0808080819, 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, + 0x2b192b0819080808, 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b, + 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b, 0x2b2b080808191908, + 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819, 0x2b2b080819081908, 0x2b2b080819190808, + 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b, 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, + 0x2b2b082b08082b2b, 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b, + 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819, 0x2b2b190808081908, + 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19, 0x2b2b19082b2b1908, 0x2b2b191908080808, + 0x2b2b191908192b19, 0x2b2b192b19190819, 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, + 0x2b2b2b1919191908, 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808, + 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b, +}; + +static const __device__ uint32_t iq3xxs_grid[256] = { + 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414, 0x04041c0c, + 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14, 0x040c140c, 0x040c142c, + 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404, 0x04140414, 0x04140424, 0x04140c0c, + 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e, 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, + 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c, 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, + 0x04243e1c, 0x04243e2c, 0x042c040c, 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, + 0x043e0c24, 0x043e0c34, 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, + 0x0c04141c, 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c, + 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04, 0x0c143e14, + 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c, 0x0c24042c, 0x0c242c04, + 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414, 0x0c3e2404, 0x14040404, 0x14040414, + 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434, 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, + 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c, 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, + 0x14140414, 0x14140c0c, 0x14140c3e, 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, + 0x141c0c04, 0x141c0c24, 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, + 0x142c3e24, 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c, + 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c, 0x1c0c2424, + 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14, 0x1c1c0c0c, 0x1c1c1c1c, + 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414, 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, + 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e, 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, + 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404, 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, + 0x24242c0c, 0x24243424, 0x242c142c, 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, + 0x2c040c14, 0x2c04240c, 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, + 0x2c143e14, 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c, + 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c, 0x340c340c, + 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14, 0x34341c1c, 0x343e041c, + 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14, 0x3e042c14, 0x3e0c1434, 0x3e0c2404, + 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c, 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, + 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04, +}; + +static const __device__ uint32_t iq3xs_grid[512] = { + 0x04040404, 0x0404040c, 0x04040414, 0x0404042c, 0x0404043e, 0x04040c04, 0x04040c0c, 0x04040c14, 0x04040c24, + 0x04040c34, 0x04041404, 0x0404140c, 0x0404142c, 0x04041c1c, 0x04042404, 0x04042414, 0x0404242c, 0x0404243e, + 0x04042c0c, 0x04042c1c, 0x04043404, 0x04043414, 0x04043e0c, 0x04043e24, 0x04043e3e, 0x040c0404, 0x040c040c, + 0x040c0414, 0x040c0424, 0x040c0c04, 0x040c0c0c, 0x040c0c2c, 0x040c1404, 0x040c141c, 0x040c143e, 0x040c1c0c, + 0x040c1c2c, 0x040c2424, 0x040c340c, 0x040c342c, 0x040c3e14, 0x04140404, 0x0414040c, 0x0414042c, 0x0414043e, + 0x04140c04, 0x04140c1c, 0x04140c34, 0x0414140c, 0x0414142c, 0x04141c04, 0x04141c24, 0x04142414, 0x0414242c, + 0x0414243e, 0x04142c0c, 0x04142c1c, 0x04143e04, 0x04143e1c, 0x041c041c, 0x041c0c0c, 0x041c0c2c, 0x041c1404, + 0x041c1414, 0x041c1c0c, 0x041c1c1c, 0x041c1c34, 0x041c2424, 0x041c2c04, 0x041c2c14, 0x041c343e, 0x041c3e0c, + 0x041c3e2c, 0x04240404, 0x04240c1c, 0x04240c3e, 0x0424140c, 0x04241424, 0x04241c14, 0x04242404, 0x0424241c, + 0x04242c0c, 0x04243e04, 0x042c0414, 0x042c0424, 0x042c1404, 0x042c1414, 0x042c1434, 0x042c1c1c, 0x042c240c, + 0x042c242c, 0x042c243e, 0x042c3434, 0x042c3e1c, 0x04340434, 0x04340c0c, 0x04340c1c, 0x04341c0c, 0x04342c14, + 0x04343e0c, 0x043e0404, 0x043e0414, 0x043e0424, 0x043e1404, 0x043e1414, 0x043e1434, 0x043e1c1c, 0x043e2c04, + 0x043e2c24, 0x0c040404, 0x0c04040c, 0x0c040414, 0x0c040424, 0x0c040c04, 0x0c040c0c, 0x0c040c1c, 0x0c040c2c, + 0x0c040c3e, 0x0c041404, 0x0c041414, 0x0c041c0c, 0x0c041c24, 0x0c041c34, 0x0c042c24, 0x0c042c34, 0x0c04340c, + 0x0c043e14, 0x0c0c0404, 0x0c0c040c, 0x0c0c041c, 0x0c0c0434, 0x0c0c0c04, 0x0c0c0c24, 0x0c0c140c, 0x0c0c1c04, + 0x0c0c1c1c, 0x0c0c240c, 0x0c0c2c04, 0x0c0c2c14, 0x0c0c3e04, 0x0c0c3e34, 0x0c140404, 0x0c140c14, 0x0c140c2c, + 0x0c140c3e, 0x0c141404, 0x0c141424, 0x0c141c14, 0x0c142404, 0x0c14241c, 0x0c142c2c, 0x0c143404, 0x0c143e14, + 0x0c1c040c, 0x0c1c0424, 0x0c1c043e, 0x0c1c0c04, 0x0c1c0c1c, 0x0c1c140c, 0x0c1c143e, 0x0c1c1c04, 0x0c1c1c24, + 0x0c1c240c, 0x0c1c3414, 0x0c1c3e04, 0x0c24041c, 0x0c24042c, 0x0c240c14, 0x0c240c24, 0x0c241c0c, 0x0c241c1c, + 0x0c242414, 0x0c242434, 0x0c242c04, 0x0c242c24, 0x0c2c040c, 0x0c2c0c04, 0x0c2c0c1c, 0x0c2c140c, 0x0c2c1c04, + 0x0c2c1c14, 0x0c2c2c0c, 0x0c341404, 0x0c341424, 0x0c34143e, 0x0c342424, 0x0c342434, 0x0c3e040c, 0x0c3e041c, + 0x0c3e0c04, 0x0c3e0c14, 0x0c3e140c, 0x0c3e1c2c, 0x0c3e240c, 0x0c3e3414, 0x0c3e3e04, 0x14040404, 0x1404040c, + 0x1404041c, 0x1404042c, 0x1404043e, 0x14040c04, 0x14040c14, 0x14040c24, 0x14040c34, 0x1404140c, 0x1404141c, + 0x1404143e, 0x14041c04, 0x14041c14, 0x1404240c, 0x1404241c, 0x1404242c, 0x14042c04, 0x14042c14, 0x1404343e, + 0x14043e04, 0x14043e1c, 0x14043e2c, 0x140c0404, 0x140c0414, 0x140c0c04, 0x140c0c1c, 0x140c0c3e, 0x140c1414, + 0x140c142c, 0x140c1c0c, 0x140c1c24, 0x140c2414, 0x140c2c0c, 0x1414040c, 0x14140424, 0x1414043e, 0x1414140c, + 0x1414141c, 0x14141c04, 0x14141c3e, 0x1414240c, 0x14142c1c, 0x14142c3e, 0x14143e0c, 0x14143e24, 0x141c0404, + 0x141c0414, 0x141c042c, 0x141c0c0c, 0x141c1414, 0x141c1424, 0x141c1c0c, 0x141c1c1c, 0x141c2414, 0x141c2c04, + 0x141c3434, 0x1424040c, 0x1424043e, 0x14241404, 0x1424141c, 0x14241c14, 0x14241c2c, 0x1424240c, 0x14243e14, + 0x14243e2c, 0x142c0424, 0x142c0c0c, 0x142c1414, 0x142c1c3e, 0x142c2404, 0x142c2c1c, 0x142c3e04, 0x14340404, + 0x14340414, 0x1434043e, 0x1434140c, 0x14342c2c, 0x1434340c, 0x143e042c, 0x143e0c0c, 0x143e1434, 0x143e1c04, + 0x143e241c, 0x143e2c04, 0x1c040414, 0x1c040c0c, 0x1c040c1c, 0x1c040c2c, 0x1c040c3e, 0x1c041414, 0x1c041c0c, + 0x1c041c1c, 0x1c041c2c, 0x1c042414, 0x1c042424, 0x1c04243e, 0x1c042c0c, 0x1c04341c, 0x1c043e0c, 0x1c0c040c, + 0x1c0c041c, 0x1c0c042c, 0x1c0c0c24, 0x1c0c140c, 0x1c0c141c, 0x1c0c2404, 0x1c0c3404, 0x1c0c3e14, 0x1c0c3e34, + 0x1c140404, 0x1c140c14, 0x1c141404, 0x1c141c14, 0x1c141c24, 0x1c142c04, 0x1c1c040c, 0x1c1c0c04, 0x1c1c0c24, + 0x1c1c140c, 0x1c1c141c, 0x1c1c143e, 0x1c1c1c04, 0x1c1c240c, 0x1c1c241c, 0x1c1c243e, 0x1c1c2c2c, 0x1c1c3e1c, + 0x1c24041c, 0x1c240c0c, 0x1c240c34, 0x1c241414, 0x1c241c0c, 0x1c242c14, 0x1c243404, 0x1c243424, 0x1c2c040c, + 0x1c2c0c04, 0x1c2c0c14, 0x1c2c142c, 0x1c2c1c14, 0x1c2c2424, 0x1c2c2c34, 0x1c2c3e1c, 0x1c340c34, 0x1c34240c, + 0x1c3e040c, 0x1c3e041c, 0x1c3e1404, 0x1c3e1414, 0x1c3e1c2c, 0x24040404, 0x24040424, 0x24040c14, 0x24041404, + 0x24041424, 0x2404143e, 0x24041c14, 0x2404240c, 0x24042c04, 0x24043e04, 0x240c0414, 0x240c043e, 0x240c0c0c, + 0x240c0c1c, 0x240c1414, 0x240c1c04, 0x240c1c2c, 0x240c241c, 0x240c2c0c, 0x240c2c2c, 0x2414040c, 0x2414041c, + 0x24140c04, 0x24140c2c, 0x2414140c, 0x24141c1c, 0x24142404, 0x24142c3e, 0x24143414, 0x24143e04, 0x241c0424, + 0x241c0c0c, 0x241c0c1c, 0x241c1404, 0x241c1414, 0x241c1c0c, 0x241c1c2c, 0x24240404, 0x24240414, 0x24241424, + 0x24241c3e, 0x24242404, 0x24243e0c, 0x242c042c, 0x242c043e, 0x242c140c, 0x242c3414, 0x24340c1c, 0x24341c24, + 0x24343404, 0x243e0c04, 0x243e0c2c, 0x243e1c04, 0x243e241c, 0x243e2c0c, 0x2c040414, 0x2c040c04, 0x2c040c24, + 0x2c041414, 0x2c042404, 0x2c042424, 0x2c04243e, 0x2c042c14, 0x2c043434, 0x2c043e24, 0x2c0c040c, 0x2c0c041c, + 0x2c0c042c, 0x2c0c0c14, 0x2c0c140c, 0x2c0c1c14, 0x2c0c3e14, 0x2c140404, 0x2c140c0c, 0x2c14141c, 0x2c141c04, + 0x2c141c34, 0x2c142c1c, 0x2c1c0414, 0x2c1c043e, 0x2c1c0c04, 0x2c1c143e, 0x2c1c2424, 0x2c1c2c0c, 0x2c1c342c, + 0x2c1c3e1c, 0x2c24040c, 0x2c240424, 0x2c241404, 0x2c241c14, 0x2c242434, 0x2c2c0c14, 0x2c2c1434, 0x2c2c2c0c, + 0x2c2c2c1c, 0x2c342414, 0x2c3e0414, 0x2c3e0424, 0x2c3e1414, 0x34040c0c, 0x34040c1c, 0x34040c2c, 0x34041c0c, + 0x34041c1c, 0x34043404, 0x340c0404, 0x340c1404, 0x340c143e, 0x340c3424, 0x34140c14, 0x34141c24, 0x34142414, + 0x34142c2c, 0x34143414, 0x34143e04, 0x341c0404, 0x341c0c24, 0x341c140c, 0x341c2404, 0x3424142c, 0x3424241c, + 0x34243414, 0x342c0404, 0x342c041c, 0x342c1c24, 0x342c3404, 0x3434042c, 0x34342404, 0x343e0c0c, 0x343e0c1c, + 0x3e040404, 0x3e040424, 0x3e04043e, 0x3e041404, 0x3e041414, 0x3e041c34, 0x3e042404, 0x3e042c24, 0x3e043414, + 0x3e0c0414, 0x3e0c0c0c, 0x3e0c1424, 0x3e0c241c, 0x3e0c242c, 0x3e14040c, 0x3e140424, 0x3e140c04, 0x3e140c34, + 0x3e14140c, 0x3e141c04, 0x3e142c0c, 0x3e1c0414, 0x3e1c1c14, 0x3e1c1c2c, 0x3e1c2c1c, 0x3e24040c, 0x3e24042c, + 0x3e240c1c, 0x3e241404, 0x3e242c04, 0x3e2c1414, 0x3e2c2414, 0x3e340414, 0x3e341c0c, 0x3e3e0404, +}; + +#define IQ1S_DELTA 0.125f +#define IQ1M_DELTA 0.125f +static const __device__ uint64_t iq1s_grid_gpu[2048] = { + 0x00000000, 0x00000002, 0x00000101, 0x00000200, 0x00000202, 0x00010001, 0x00010101, 0x00020000, 0x00020002, + 0x00020200, 0x00020202, 0x01000101, 0x01010001, 0x01010100, 0x01010102, 0x01020101, 0x02000000, 0x02000002, + 0x02000200, 0x02000202, 0x02010101, 0x02020000, 0x02020002, 0x02020200, 0x02020202, 0x00000110, 0x00000111, + 0x00010011, 0x00010110, 0x00010112, 0x00010211, 0x00010212, 0x00020111, 0x01000011, 0x01000112, 0x01000211, + 0x01010012, 0x01010111, 0x01010212, 0x01020011, 0x01020110, 0x01020112, 0x01020210, 0x02000111, 0x02010011, + 0x02010110, 0x02010112, 0x02020111, 0x00000020, 0x00000022, 0x00000220, 0x00000222, 0x00010121, 0x00020020, + 0x00020022, 0x00020220, 0x00020222, 0x01000121, 0x01010021, 0x01010221, 0x01020120, 0x01020221, 0x02000020, + 0x02000022, 0x02000220, 0x02000222, 0x02010021, 0x02010121, 0x02010221, 0x02020020, 0x02020022, 0x02020220, + 0x02020222, 0x00011001, 0x00011100, 0x00011102, 0x00021101, 0x01001001, 0x01001201, 0x01011101, 0x01011202, + 0x01021100, 0x01021101, 0x02011001, 0x02011201, 0x02021101, 0x00001011, 0x00001110, 0x00001111, 0x00001112, + 0x00011111, 0x00011210, 0x00011212, 0x00021211, 0x01001010, 0x01001111, 0x01001212, 0x01011010, 0x01011011, + 0x01011110, 0x01011111, 0x01011112, 0x01011211, 0x01021010, 0x01021012, 0x01021111, 0x01021210, 0x01021212, + 0x02001011, 0x02011011, 0x02011111, 0x02011210, 0x02011212, 0x02021011, 0x02021110, 0x02021111, 0x02021112, + 0x02021211, 0x00011120, 0x00011221, 0x01001021, 0x01001120, 0x01011020, 0x01011022, 0x01011121, 0x01011220, + 0x01021020, 0x01021021, 0x01021122, 0x01021221, 0x02001121, 0x02011021, 0x02011120, 0x02011221, 0x00002000, + 0x00002002, 0x00002200, 0x00002202, 0x00012101, 0x00022000, 0x00022002, 0x00022200, 0x00022202, 0x01002101, + 0x01012001, 0x01012102, 0x01022101, 0x02002000, 0x02002002, 0x02002200, 0x02002202, 0x02012101, 0x02022000, + 0x02022002, 0x02022200, 0x02022202, 0x00002111, 0x00012011, 0x00012110, 0x00012211, 0x00022110, 0x00022111, + 0x01002011, 0x01012010, 0x01012011, 0x01012111, 0x01022011, 0x01022110, 0x01022211, 0x02012011, 0x02012110, + 0x02012112, 0x02012211, 0x02022111, 0x00002020, 0x00002022, 0x00002220, 0x00002222, 0x00012121, 0x00022020, + 0x00022022, 0x00022220, 0x00022222, 0x01002121, 0x01012021, 0x01012221, 0x01022021, 0x01022121, 0x02002020, + 0x02002022, 0x02002121, 0x02002220, 0x02002222, 0x02012121, 0x02022020, 0x02022022, 0x02022220, 0x02022222, + 0x00110000, 0x00110001, 0x00110100, 0x00110201, 0x00120100, 0x00120101, 0x01100001, 0x01100100, 0x01110000, + 0x01110101, 0x01110200, 0x01120001, 0x01120100, 0x01120101, 0x01120201, 0x02110001, 0x02110100, 0x02110102, + 0x02120001, 0x02120101, 0x00100011, 0x00100110, 0x00100112, 0x00100211, 0x00110010, 0x00110012, 0x00110111, + 0x00110210, 0x00120011, 0x00120110, 0x00120211, 0x01100111, 0x01100212, 0x01110010, 0x01110011, 0x01110012, + 0x01110110, 0x01110111, 0x01110112, 0x01110211, 0x01120010, 0x01120111, 0x02100110, 0x02110012, 0x02110111, + 0x02120011, 0x02120110, 0x00110021, 0x00110120, 0x00110122, 0x00120121, 0x01100020, 0x01100122, 0x01100221, + 0x01110022, 0x01110121, 0x01110220, 0x01110222, 0x01120120, 0x01120122, 0x02100121, 0x02110021, 0x02110120, + 0x02110122, 0x02120121, 0x00101001, 0x00101102, 0x00101201, 0x00111100, 0x00111101, 0x00111200, 0x00111201, + 0x00121001, 0x00121102, 0x01101001, 0x01101101, 0x01101102, 0x01101200, 0x01101202, 0x01111001, 0x01111100, + 0x01111101, 0x01111102, 0x01111201, 0x01121002, 0x01121101, 0x01121200, 0x02101100, 0x02101201, 0x02111000, + 0x02111100, 0x02111101, 0x02111200, 0x02111201, 0x02111202, 0x02121001, 0x02121100, 0x02121101, 0x02121201, + 0x00101012, 0x00101111, 0x00101212, 0x00111011, 0x00111110, 0x00111111, 0x00111112, 0x00111211, 0x00121010, + 0x00121012, 0x00121111, 0x00121210, 0x00121212, 0x01101011, 0x01101110, 0x01101111, 0x01101112, 0x01111011, + 0x01111012, 0x01111110, 0x01111111, 0x01111112, 0x01111211, 0x01111212, 0x01121011, 0x01121110, 0x01121111, + 0x01121112, 0x01121211, 0x02101010, 0x02101012, 0x02101110, 0x02101111, 0x02101210, 0x02101212, 0x02111010, + 0x02111011, 0x02111110, 0x02111111, 0x02111112, 0x02111211, 0x02111212, 0x02121010, 0x02121012, 0x02121111, + 0x00101021, 0x00101120, 0x00101121, 0x00101122, 0x00111121, 0x00111122, 0x00111220, 0x00111222, 0x00121021, + 0x00121122, 0x01101020, 0x01101022, 0x01101120, 0x01101121, 0x01101220, 0x01101222, 0x01111021, 0x01111121, + 0x01111122, 0x01111220, 0x01111221, 0x01121021, 0x01121120, 0x01121121, 0x01121220, 0x01121221, 0x01121222, + 0x02101122, 0x02101222, 0x02111022, 0x02111121, 0x02121120, 0x02121221, 0x00112001, 0x00112102, 0x00122101, + 0x01102001, 0x01102100, 0x01102102, 0x01102201, 0x01112000, 0x01112101, 0x01112200, 0x01112202, 0x01122000, + 0x01122001, 0x01122100, 0x01122102, 0x01122201, 0x02102101, 0x02112001, 0x02112100, 0x02122101, 0x00112010, + 0x00112012, 0x00112111, 0x00112212, 0x00122011, 0x00122111, 0x01102012, 0x01102110, 0x01102111, 0x01102210, + 0x01112011, 0x01112110, 0x01112111, 0x01112112, 0x01112211, 0x01112212, 0x01122010, 0x01122111, 0x01122212, + 0x02102211, 0x02112011, 0x02112012, 0x02112111, 0x02112210, 0x02122011, 0x02122112, 0x02122211, 0x00102221, + 0x00112122, 0x00122120, 0x00122122, 0x01102120, 0x01102122, 0x01102221, 0x01112020, 0x01112022, 0x01112121, + 0x01112220, 0x01122021, 0x01122122, 0x01122221, 0x02102121, 0x02112021, 0x02112122, 0x02112222, 0x00200000, + 0x00200002, 0x00200200, 0x00200202, 0x00210101, 0x00220000, 0x00220002, 0x00220101, 0x00220200, 0x00220202, + 0x01200101, 0x01210001, 0x01210201, 0x01220001, 0x01220101, 0x02200000, 0x02200002, 0x02200200, 0x02200202, + 0x02210101, 0x02220000, 0x02220002, 0x02220101, 0x02220200, 0x02220202, 0x00200111, 0x00210011, 0x00210110, + 0x00210211, 0x00220111, 0x01200012, 0x01200110, 0x01200211, 0x01210111, 0x01210210, 0x01210212, 0x01220011, + 0x01220110, 0x01220111, 0x01220112, 0x02200111, 0x02210010, 0x02210112, 0x02210211, 0x02220111, 0x00200021, + 0x00200220, 0x00200222, 0x00210021, 0x00210121, 0x00220020, 0x00220022, 0x00220220, 0x00220222, 0x01200121, + 0x01210021, 0x01210122, 0x01210221, 0x01220121, 0x02200021, 0x02200220, 0x02200222, 0x02210021, 0x02210121, + 0x02220020, 0x02220022, 0x02220220, 0x02220222, 0x00201101, 0x00211100, 0x00211102, 0x00211201, 0x00221101, + 0x01201100, 0x01201101, 0x01201102, 0x01201201, 0x01211002, 0x01211101, 0x01211200, 0x01211202, 0x01221102, + 0x02201101, 0x02211001, 0x02211100, 0x02211201, 0x02221001, 0x02221101, 0x00201211, 0x00211111, 0x00221011, + 0x00221211, 0x01201010, 0x01201111, 0x01201210, 0x01211011, 0x01211110, 0x01211111, 0x01211211, 0x01221012, + 0x01221111, 0x01221210, 0x02201211, 0x02211010, 0x02211110, 0x02211111, 0x02211210, 0x02211212, 0x02221011, + 0x02221110, 0x02221112, 0x02221211, 0x00201121, 0x00211020, 0x00211022, 0x00211221, 0x00221121, 0x01201021, + 0x01201221, 0x01211121, 0x01221020, 0x01221021, 0x01221221, 0x02201120, 0x02201122, 0x02211020, 0x02211222, + 0x00202000, 0x00202002, 0x00202200, 0x00202202, 0x00212101, 0x00222000, 0x00222002, 0x00222200, 0x00222202, + 0x01202101, 0x01212001, 0x01212100, 0x01222101, 0x02202000, 0x02202002, 0x02202200, 0x02202202, 0x02222000, + 0x02222002, 0x02222200, 0x02222202, 0x00202211, 0x00212011, 0x00212110, 0x00212211, 0x00222111, 0x01202112, + 0x01202211, 0x01212012, 0x01212111, 0x01222011, 0x01222110, 0x01222112, 0x01222211, 0x02202111, 0x02212010, + 0x02212112, 0x02212211, 0x02222110, 0x02222111, 0x00202020, 0x00202022, 0x00202220, 0x00202222, 0x00222020, + 0x00222022, 0x00222220, 0x00222222, 0x01202121, 0x01212021, 0x01212122, 0x01212221, 0x01222121, 0x02202020, + 0x02202022, 0x02202220, 0x02202222, 0x02212121, 0x02222020, 0x02222022, 0x02222220, 0x02222222, 0x10000101, + 0x10010001, 0x10010102, 0x10020101, 0x11000201, 0x11010002, 0x11010101, 0x11010200, 0x11010202, 0x11020001, + 0x11020100, 0x11020102, 0x12010100, 0x12010201, 0x12020001, 0x12020102, 0x10000010, 0x10000011, 0x10000110, + 0x10000112, 0x10000211, 0x10010012, 0x10010111, 0x10010112, 0x10010210, 0x10010212, 0x10020011, 0x10020112, + 0x10020211, 0x11000111, 0x11000210, 0x11000212, 0x11010011, 0x11010110, 0x11010111, 0x11010112, 0x11010211, + 0x11010212, 0x11020111, 0x11020210, 0x11020212, 0x12000011, 0x12000110, 0x12000112, 0x12010010, 0x12010012, + 0x12010111, 0x12020010, 0x12020011, 0x12020012, 0x10000121, 0x10010021, 0x10010120, 0x10010122, 0x10020121, + 0x11000021, 0x11010022, 0x11010121, 0x11010222, 0x11020120, 0x11020221, 0x12000221, 0x12010120, 0x12020121, + 0x10001001, 0x10011101, 0x10011201, 0x10021201, 0x11001101, 0x11001200, 0x11001202, 0x11011001, 0x11011100, + 0x11011101, 0x11011102, 0x11021001, 0x11021002, 0x11021101, 0x11021200, 0x11021202, 0x12001001, 0x12001102, + 0x12001201, 0x12011000, 0x12011002, 0x12011101, 0x12021000, 0x12021001, 0x12021201, 0x10001011, 0x10001012, + 0x10001111, 0x10001212, 0x10011011, 0x10011110, 0x10011111, 0x10011112, 0x10011211, 0x10021010, 0x10021111, + 0x10021212, 0x11001011, 0x11001110, 0x11001111, 0x11001112, 0x11001211, 0x11011010, 0x11011011, 0x11011110, + 0x11011111, 0x11011112, 0x11011210, 0x11011211, 0x11021011, 0x11021110, 0x11021111, 0x11021112, 0x11021211, + 0x12001012, 0x12001110, 0x12001111, 0x12001210, 0x12011011, 0x12011110, 0x12011111, 0x12011112, 0x12011211, + 0x12011212, 0x12021111, 0x12021210, 0x12021212, 0x10001021, 0x10001121, 0x10001221, 0x10011120, 0x10011121, + 0x10011220, 0x10011222, 0x10021021, 0x10021120, 0x10021221, 0x11001020, 0x11001022, 0x11001121, 0x11001220, + 0x11011020, 0x11011021, 0x11011022, 0x11011121, 0x11011122, 0x11011221, 0x11021022, 0x11021121, 0x11021220, + 0x12001021, 0x12001121, 0x12001222, 0x12011120, 0x12011121, 0x12021021, 0x12021120, 0x12021122, 0x10002101, + 0x10012001, 0x10012101, 0x10012202, 0x10022101, 0x11002002, 0x11002201, 0x11012000, 0x11012101, 0x11012200, + 0x11022001, 0x11022100, 0x11022102, 0x11022201, 0x12002101, 0x12012001, 0x12012100, 0x12012102, 0x12012201, + 0x12022101, 0x10002011, 0x10002111, 0x10002112, 0x10002212, 0x10012010, 0x10012110, 0x10012111, 0x10012210, + 0x10022011, 0x10022110, 0x10022112, 0x11002010, 0x11002111, 0x11002212, 0x11012011, 0x11012012, 0x11012110, + 0x11012111, 0x11012112, 0x11012211, 0x11022010, 0x11022012, 0x11022111, 0x11022112, 0x11022212, 0x12002112, + 0x12002211, 0x12012012, 0x12012111, 0x12012112, 0x12012210, 0x12022011, 0x12022110, 0x12022112, 0x12022211, + 0x10012122, 0x11002120, 0x11002122, 0x11002221, 0x11012121, 0x11012220, 0x11012222, 0x11022120, 0x11022221, + 0x12012120, 0x12022121, 0x10100001, 0x10100100, 0x10100101, 0x10100102, 0x10100201, 0x10110002, 0x10110101, + 0x10110202, 0x10120001, 0x10120100, 0x10120201, 0x11100000, 0x11100101, 0x11100200, 0x11110001, 0x11110100, + 0x11110101, 0x11110102, 0x11110201, 0x11120101, 0x11120200, 0x12100102, 0x12100201, 0x12110101, 0x12110200, + 0x12120000, 0x12120001, 0x12120102, 0x12120201, 0x10100111, 0x10100210, 0x10100211, 0x10100212, 0x10110011, + 0x10110110, 0x10110111, 0x10110112, 0x10110210, 0x10110211, 0x10120010, 0x10120111, 0x10120112, 0x10120210, + 0x10120212, 0x11100011, 0x11100110, 0x11100111, 0x11100112, 0x11100211, 0x11110010, 0x11110011, 0x11110012, + 0x11110110, 0x11110111, 0x11110112, 0x11110210, 0x11110211, 0x11110212, 0x11120011, 0x11120110, 0x11120111, + 0x11120112, 0x11120211, 0x12100012, 0x12100111, 0x12110011, 0x12110110, 0x12110111, 0x12110112, 0x12110211, + 0x12120010, 0x12120111, 0x12120212, 0x10100021, 0x10100122, 0x10110022, 0x10110121, 0x10110222, 0x10120021, + 0x10120120, 0x11100022, 0x11100121, 0x11100222, 0x11110021, 0x11110120, 0x11110121, 0x11110122, 0x11110221, + 0x11120022, 0x11120121, 0x12100121, 0x12110020, 0x12110022, 0x12110121, 0x12110221, 0x12110222, 0x12120120, + 0x10101100, 0x10101101, 0x10111001, 0x10111100, 0x10111101, 0x10111102, 0x10111200, 0x10111201, 0x10121001, + 0x10121101, 0x10121200, 0x10121202, 0x11101001, 0x11101100, 0x11101101, 0x11101102, 0x11101201, 0x11101202, + 0x11111000, 0x11111001, 0x11111100, 0x11111101, 0x11111102, 0x11111200, 0x11111201, 0x11111202, 0x11121001, + 0x11121002, 0x11121100, 0x11121101, 0x11121102, 0x11121201, 0x12101000, 0x12101200, 0x12101202, 0x12111001, + 0x12111100, 0x12111101, 0x12111102, 0x12111201, 0x12121001, 0x12121100, 0x12121101, 0x12121202, 0x10101011, + 0x10101012, 0x10101110, 0x10101111, 0x10101112, 0x10101211, 0x10111010, 0x10111011, 0x10111012, 0x10111110, + 0x10111111, 0x10111112, 0x10111211, 0x10111212, 0x10121011, 0x10121110, 0x10121111, 0x10121112, 0x10121211, + 0x11101010, 0x11101011, 0x11101012, 0x11101110, 0x11101111, 0x11101112, 0x11101210, 0x11101211, 0x11111010, + 0x11111011, 0x11111012, 0x11111110, 0x11111111, 0x11111112, 0x11111210, 0x11111211, 0x11111212, 0x11121010, + 0x11121011, 0x11121110, 0x11121111, 0x11121112, 0x11121210, 0x11121211, 0x11121212, 0x12101011, 0x12101110, + 0x12101111, 0x12101211, 0x12101212, 0x12111010, 0x12111011, 0x12111110, 0x12111111, 0x12111112, 0x12111210, + 0x12111211, 0x12121011, 0x12121110, 0x12121111, 0x12121112, 0x12121211, 0x10101020, 0x10101021, 0x10101022, + 0x10101120, 0x10101122, 0x10101220, 0x10101221, 0x10111021, 0x10111120, 0x10111121, 0x10111220, 0x10111221, + 0x10121020, 0x10121021, 0x10121022, 0x10121120, 0x10121121, 0x10121122, 0x10121220, 0x10121221, 0x11101021, + 0x11101121, 0x11101122, 0x11101220, 0x11101221, 0x11101222, 0x11111020, 0x11111021, 0x11111022, 0x11111120, + 0x11111121, 0x11111122, 0x11111220, 0x11111221, 0x11111222, 0x11121021, 0x11121120, 0x11121121, 0x11121221, + 0x12101022, 0x12101121, 0x12101122, 0x12101220, 0x12101221, 0x12101222, 0x12111021, 0x12111121, 0x12111222, + 0x12121022, 0x12121121, 0x12121122, 0x12121220, 0x12121221, 0x10102100, 0x10102101, 0x10102102, 0x10102201, + 0x10112000, 0x10112101, 0x10112200, 0x10122001, 0x10122202, 0x11102101, 0x11102200, 0x11102202, 0x11112001, + 0x11112100, 0x11112101, 0x11112102, 0x11112200, 0x11112201, 0x11122000, 0x11122002, 0x11122100, 0x11122101, + 0x12102002, 0x12102201, 0x12112000, 0x12112002, 0x12112101, 0x12112200, 0x12122001, 0x12122201, 0x10102011, + 0x10102012, 0x10102111, 0x10102212, 0x10112011, 0x10112110, 0x10112111, 0x10112112, 0x10112211, 0x10122111, + 0x11102011, 0x11102110, 0x11102111, 0x11102112, 0x11102211, 0x11112010, 0x11112011, 0x11112012, 0x11112110, + 0x11112111, 0x11112112, 0x11112210, 0x11112211, 0x11112212, 0x11122011, 0x11122110, 0x11122111, 0x11122112, + 0x11122211, 0x12102011, 0x12102111, 0x12102211, 0x12112011, 0x12112110, 0x12112111, 0x12112112, 0x12112210, + 0x12112211, 0x12122111, 0x10102120, 0x10102220, 0x10112121, 0x10112222, 0x10122020, 0x10122121, 0x10122122, + 0x10122221, 0x11102121, 0x11102220, 0x11102221, 0x11112021, 0x11112121, 0x11112122, 0x11112220, 0x11112221, + 0x11122022, 0x11122121, 0x11122220, 0x11122222, 0x12102021, 0x12102222, 0x12112022, 0x12112121, 0x12112122, + 0x12112220, 0x12112222, 0x12122021, 0x10200101, 0x10210100, 0x10210102, 0x10210201, 0x10220101, 0x11200100, + 0x11210000, 0x11210101, 0x11210102, 0x11210200, 0x11210202, 0x11220001, 0x11220100, 0x11220102, 0x11220201, + 0x12200001, 0x12210102, 0x12220101, 0x10200011, 0x10200110, 0x10200112, 0x10200211, 0x10210012, 0x10210111, + 0x10220011, 0x10220012, 0x10220112, 0x10220211, 0x11200111, 0x11200211, 0x11210011, 0x11210111, 0x11210112, + 0x11210211, 0x11220111, 0x11220112, 0x11220212, 0x12200110, 0x12200212, 0x12210012, 0x12210111, 0x12220011, + 0x12220112, 0x12220211, 0x10210021, 0x10210122, 0x10210221, 0x11200020, 0x11200021, 0x11200122, 0x11210121, + 0x11210122, 0x11210220, 0x11220020, 0x12200121, 0x12210021, 0x12210122, 0x12220121, 0x10211001, 0x10211002, + 0x10211101, 0x10211102, 0x10211202, 0x10221001, 0x10221102, 0x10221201, 0x11201000, 0x11201002, 0x11201101, + 0x11201200, 0x11201202, 0x11211001, 0x11211100, 0x11211101, 0x11211102, 0x11211201, 0x11211202, 0x11221000, + 0x11221002, 0x11221101, 0x12201100, 0x12201101, 0x12201201, 0x12211000, 0x12211002, 0x12211100, 0x12211101, + 0x12211102, 0x12211200, 0x12211202, 0x12221001, 0x12221100, 0x12221201, 0x10201111, 0x10201210, 0x10201212, + 0x10211011, 0x10211111, 0x10211112, 0x10211211, 0x11201110, 0x11201111, 0x11201112, 0x11201211, 0x11211010, + 0x11211011, 0x11211110, 0x11211111, 0x11211112, 0x11211211, 0x11221011, 0x11221110, 0x11221111, 0x11221112, + 0x11221211, 0x12201112, 0x12201211, 0x12201212, 0x12211011, 0x12211111, 0x12211112, 0x12211211, 0x12211212, + 0x12221012, 0x12221111, 0x12221112, 0x12221210, 0x10201022, 0x10201221, 0x10211121, 0x10221020, 0x10221122, + 0x10221220, 0x10221221, 0x11201020, 0x11201121, 0x11201220, 0x11201222, 0x11211021, 0x11211120, 0x11211121, + 0x11211122, 0x11211220, 0x11211222, 0x11221020, 0x11221121, 0x11221220, 0x12201020, 0x12201022, 0x12201121, + 0x12201222, 0x12211120, 0x12211122, 0x12211220, 0x12211221, 0x12221020, 0x12221120, 0x12221122, 0x12221222, + 0x10212102, 0x10212201, 0x10222101, 0x11202001, 0x11212002, 0x11212101, 0x11212202, 0x11222001, 0x11222201, + 0x12202101, 0x12212001, 0x12212200, 0x12222102, 0x10202011, 0x10202110, 0x10212010, 0x10212111, 0x10222011, + 0x10222110, 0x10222112, 0x10222211, 0x11202010, 0x11202011, 0x11202111, 0x11202112, 0x11202210, 0x11212011, + 0x11212110, 0x11212111, 0x11212112, 0x11212211, 0x11222010, 0x11222111, 0x11222212, 0x12202012, 0x12202110, + 0x12202212, 0x12212111, 0x12222011, 0x12222110, 0x12222111, 0x12222211, 0x10212021, 0x10212122, 0x10212220, + 0x11202021, 0x11202120, 0x11202221, 0x11212020, 0x11212121, 0x11212220, 0x11212222, 0x11222120, 0x11222121, + 0x11222221, 0x12202122, 0x12212120, 0x12212220, 0x12212222, 0x12222122, 0x20000000, 0x20000002, 0x20000200, + 0x20000202, 0x20020000, 0x20020002, 0x20020200, 0x20020202, 0x21000101, 0x21010000, 0x21010001, 0x21010100, + 0x21010102, 0x21010201, 0x21020101, 0x22000000, 0x22000002, 0x22000200, 0x22000202, 0x22010101, 0x22020000, + 0x22020002, 0x22020200, 0x22020202, 0x20000111, 0x20010011, 0x20010110, 0x20010112, 0x20010211, 0x20020111, + 0x21000011, 0x21000110, 0x21000211, 0x21010010, 0x21010012, 0x21010111, 0x21010112, 0x21010210, 0x21010211, + 0x21020110, 0x21020112, 0x21020211, 0x22000111, 0x22000211, 0x22010110, 0x22010112, 0x22010211, 0x22020111, + 0x20000020, 0x20000022, 0x20000220, 0x20000222, 0x20010121, 0x20020020, 0x20020022, 0x20020220, 0x20020222, + 0x21010021, 0x21010120, 0x21010221, 0x21020121, 0x22000020, 0x22000022, 0x22000220, 0x22000222, 0x22010121, + 0x22020020, 0x22020022, 0x22020220, 0x22020222, 0x20011100, 0x20011201, 0x21001001, 0x21001100, 0x21011001, + 0x21011101, 0x21011202, 0x21021001, 0x21021100, 0x21021201, 0x22011100, 0x22011201, 0x20001011, 0x20001211, + 0x20011012, 0x20011111, 0x20011212, 0x20021112, 0x20021211, 0x21001010, 0x21001011, 0x21001111, 0x21001210, + 0x21011011, 0x21011110, 0x21011111, 0x21011112, 0x21011211, 0x21011212, 0x21021111, 0x21021112, 0x21021210, + 0x21021212, 0x22001011, 0x22001110, 0x22001112, 0x22001211, 0x22011010, 0x22011012, 0x22011111, 0x22011210, + 0x22021112, 0x20011021, 0x20011122, 0x20011221, 0x20021121, 0x21001021, 0x21001120, 0x21001221, 0x21001222, + 0x21011020, 0x21011121, 0x21011221, 0x21011222, 0x21021021, 0x21021122, 0x21021222, 0x22001121, 0x22011021, + 0x22011222, 0x22021120, 0x20002000, 0x20002002, 0x20002200, 0x20002202, 0x20012101, 0x20022000, 0x20022002, + 0x20022200, 0x20022202, 0x21002001, 0x21002101, 0x21012001, 0x21012100, 0x21012201, 0x21022101, 0x21022201, + 0x22002000, 0x22002002, 0x22002200, 0x22002202, 0x22012101, 0x22022000, 0x22022002, 0x22022200, 0x22022202, + 0x20002111, 0x20002112, 0x20012011, 0x20012110, 0x20012112, 0x20022111, 0x21002011, 0x21002110, 0x21002112, + 0x21002211, 0x21012010, 0x21012012, 0x21012111, 0x21012212, 0x21022011, 0x21022110, 0x22002111, 0x22012112, + 0x22012211, 0x22022111, 0x20002020, 0x20002022, 0x20002220, 0x20002222, 0x20012121, 0x20022020, 0x20022022, + 0x20022220, 0x20022222, 0x21002121, 0x21012021, 0x21012120, 0x21012122, 0x22002020, 0x22002022, 0x22002220, + 0x22002222, 0x22012121, 0x22022020, 0x22022022, 0x22022220, 0x22022222, 0x20100101, 0x20110001, 0x20110102, + 0x20110200, 0x20110201, 0x20120101, 0x21100001, 0x21100102, 0x21100201, 0x21110101, 0x21110200, 0x21110202, + 0x21120201, 0x21120202, 0x22100101, 0x22110001, 0x22110100, 0x22110102, 0x22110201, 0x22120101, 0x20100011, + 0x20100110, 0x20100112, 0x20100211, 0x20110010, 0x20110111, 0x20110210, 0x20110212, 0x20120011, 0x20120110, + 0x20120112, 0x20120211, 0x21100010, 0x21100111, 0x21110010, 0x21110011, 0x21110110, 0x21110111, 0x21110112, + 0x21110211, 0x21120012, 0x21120111, 0x22100110, 0x22100112, 0x22110012, 0x22110111, 0x22110210, 0x22120011, + 0x22120110, 0x22120112, 0x22120211, 0x20100121, 0x20110021, 0x20110120, 0x20110221, 0x20120121, 0x21100120, + 0x21100122, 0x21100221, 0x21110020, 0x21110022, 0x21110121, 0x21110220, 0x21120122, 0x21120221, 0x22100121, + 0x22110120, 0x22110122, 0x22120221, 0x20101001, 0x20101100, 0x20101102, 0x20111000, 0x20111101, 0x20111200, + 0x20121102, 0x21101000, 0x21101202, 0x21111001, 0x21111100, 0x21111101, 0x21111102, 0x21111200, 0x21111201, + 0x21121000, 0x21121001, 0x21121002, 0x21121101, 0x22101100, 0x22101102, 0x22111002, 0x22111100, 0x22111101, + 0x22111200, 0x22121001, 0x22121201, 0x20101010, 0x20101111, 0x20101210, 0x20101212, 0x20111010, 0x20111011, + 0x20111110, 0x20111111, 0x20111112, 0x20111211, 0x20121011, 0x20121111, 0x20121211, 0x20121212, 0x21101011, + 0x21101110, 0x21101111, 0x21101112, 0x21101211, 0x21111010, 0x21111011, 0x21111012, 0x21111110, 0x21111111, + 0x21111112, 0x21111210, 0x21111211, 0x21111212, 0x21121011, 0x21121110, 0x21121111, 0x21121112, 0x21121211, + 0x22101011, 0x22101111, 0x22101210, 0x22111011, 0x22111012, 0x22111110, 0x22111111, 0x22111112, 0x22111211, + 0x22111212, 0x22121010, 0x22121012, 0x22121111, 0x22121210, 0x22121212, 0x20101021, 0x20101120, 0x20111020, + 0x20111121, 0x20111221, 0x20121020, 0x20121122, 0x20121221, 0x21101121, 0x21101220, 0x21101221, 0x21111021, + 0x21111022, 0x21111121, 0x21111122, 0x21111221, 0x21121121, 0x21121220, 0x22101022, 0x22101120, 0x22101221, + 0x22101222, 0x22111022, 0x22111120, 0x22111121, 0x22121120, 0x22121122, 0x22121221, 0x20102101, 0x20112102, + 0x20112201, 0x20122101, 0x21102001, 0x21102102, 0x21112000, 0x21112002, 0x21112101, 0x21112102, 0x21112202, + 0x21122100, 0x21122101, 0x22102101, 0x22112001, 0x22112102, 0x22112201, 0x22122101, 0x20102110, 0x20102112, + 0x20102211, 0x20112010, 0x20112012, 0x20112111, 0x20112210, 0x20112212, 0x20122010, 0x20122011, 0x20122110, + 0x20122112, 0x21102010, 0x21102012, 0x21102111, 0x21102210, 0x21102212, 0x21112011, 0x21112110, 0x21112111, + 0x21112112, 0x21112211, 0x21122012, 0x21122111, 0x21122112, 0x21122212, 0x22102011, 0x22102110, 0x22112010, + 0x22112012, 0x22112111, 0x22112212, 0x22122011, 0x22122112, 0x20102121, 0x20112121, 0x20122121, 0x21102120, + 0x21102122, 0x21102221, 0x21112020, 0x21112121, 0x21112220, 0x21122021, 0x22102121, 0x22112021, 0x22112120, + 0x22112121, 0x22112122, 0x20200000, 0x20200002, 0x20200200, 0x20200202, 0x20210101, 0x20220000, 0x20220002, + 0x20220200, 0x20220202, 0x21200101, 0x21210001, 0x21210100, 0x21210102, 0x21210201, 0x22200000, 0x22200002, + 0x22200200, 0x22200202, 0x22210101, 0x22220000, 0x22220002, 0x22220200, 0x22220202, 0x20200111, 0x20200211, + 0x20210011, 0x20210110, 0x20210112, 0x20210211, 0x20210212, 0x21200112, 0x21200211, 0x21210011, 0x21210111, + 0x21210210, 0x21210212, 0x21220011, 0x21220110, 0x22200111, 0x22210010, 0x22210012, 0x22210112, 0x22210211, + 0x20200022, 0x20200220, 0x20200222, 0x20210020, 0x20210221, 0x20220022, 0x20220220, 0x20220222, 0x21200121, + 0x21210021, 0x21210122, 0x21210221, 0x21220121, 0x22200020, 0x22200022, 0x22200220, 0x22200222, 0x22210121, + 0x22220020, 0x22220022, 0x22220220, 0x22220222, 0x20211201, 0x20221101, 0x21201001, 0x21201100, 0x21211000, + 0x21211100, 0x21211101, 0x21211200, 0x21211202, 0x21221001, 0x21221101, 0x21221102, 0x21221200, 0x21221201, + 0x22201101, 0x20201112, 0x20201211, 0x20211010, 0x20211012, 0x20211111, 0x20211210, 0x20221112, 0x20221211, + 0x21201012, 0x21201111, 0x21211011, 0x21211110, 0x21211111, 0x21211112, 0x21211211, 0x21221111, 0x21221212, + 0x22201011, 0x22201110, 0x22201111, 0x22201112, 0x22201211, 0x22211012, 0x22211111, 0x22211210, 0x20201121, + 0x20211021, 0x20211122, 0x20211222, 0x20221021, 0x20221121, 0x21201120, 0x21201122, 0x21201222, 0x21211022, + 0x21211121, 0x21211122, 0x21211220, 0x21221020, 0x21221022, 0x22201122, 0x22211020, 0x22211121, 0x22211122, + 0x22211221, 0x22221021, 0x22221120, 0x22221122, 0x20202000, 0x20202002, 0x20202200, 0x20202202, 0x20222000, + 0x20222002, 0x20222200, 0x20222202, 0x21212001, 0x21212100, 0x21212102, 0x21212201, 0x22202000, 0x22202002, + 0x22202200, 0x22202202, 0x22212101, 0x22222000, 0x22222002, 0x22222200, 0x22222202, 0x20202111, 0x20212110, + 0x20212211, 0x20222011, 0x20222111, 0x21202011, 0x21212010, 0x21212111, 0x21212212, 0x21222011, 0x21222112, + 0x21222211, 0x22212010, 0x22212112, 0x20202020, 0x20202022, 0x20202220, 0x20202222, 0x20222020, 0x20222022, + 0x20222220, 0x20222222, 0x21212021, 0x21212120, 0x21212122, 0x22202020, 0x22202022, 0x22202220, 0x22202222, + 0x22212121, 0x22222020, 0x22222022, 0x22222220, 0x22222222, +}; + +static const __device__ uint8_t ksigns_iq2xs[128] = { + 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15, 144, 17, 18, 147, 20, 149, + 150, 23, 24, 153, 154, 27, 156, 29, 30, 159, 160, 33, 34, 163, 36, 165, 166, 39, 40, 169, 170, 43, + 172, 45, 46, 175, 48, 177, 178, 51, 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63, 192, 65, + 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, 78, 207, 80, 209, 210, 83, 212, 85, 86, 215, + 216, 89, 90, 219, 92, 221, 222, 95, 96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237, + 238, 111, 240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255, +}; + +static const __device__ uint64_t ksigns64[128] = { + 0x0000000000000000, 0xff000000000000ff, 0xff0000000000ff00, 0x000000000000ffff, 0xff00000000ff0000, + 0x0000000000ff00ff, 0x0000000000ffff00, 0xff00000000ffffff, 0xff000000ff000000, 0x00000000ff0000ff, + 0x00000000ff00ff00, 0xff000000ff00ffff, 0x00000000ffff0000, 0xff000000ffff00ff, 0xff000000ffffff00, + 0x00000000ffffffff, 0xff0000ff00000000, 0x000000ff000000ff, 0x000000ff0000ff00, 0xff0000ff0000ffff, + 0x000000ff00ff0000, 0xff0000ff00ff00ff, 0xff0000ff00ffff00, 0x000000ff00ffffff, 0x000000ffff000000, + 0xff0000ffff0000ff, 0xff0000ffff00ff00, 0x000000ffff00ffff, 0xff0000ffffff0000, 0x000000ffffff00ff, + 0x000000ffffffff00, 0xff0000ffffffffff, 0xff00ff0000000000, 0x0000ff00000000ff, 0x0000ff000000ff00, + 0xff00ff000000ffff, 0x0000ff0000ff0000, 0xff00ff0000ff00ff, 0xff00ff0000ffff00, 0x0000ff0000ffffff, + 0x0000ff00ff000000, 0xff00ff00ff0000ff, 0xff00ff00ff00ff00, 0x0000ff00ff00ffff, 0xff00ff00ffff0000, + 0x0000ff00ffff00ff, 0x0000ff00ffffff00, 0xff00ff00ffffffff, 0x0000ffff00000000, 0xff00ffff000000ff, + 0xff00ffff0000ff00, 0x0000ffff0000ffff, 0xff00ffff00ff0000, 0x0000ffff00ff00ff, 0x0000ffff00ffff00, + 0xff00ffff00ffffff, 0xff00ffffff000000, 0x0000ffffff0000ff, 0x0000ffffff00ff00, 0xff00ffffff00ffff, + 0x0000ffffffff0000, 0xff00ffffffff00ff, 0xff00ffffffffff00, 0x0000ffffffffffff, 0xffff000000000000, + 0x00ff0000000000ff, 0x00ff00000000ff00, 0xffff00000000ffff, 0x00ff000000ff0000, 0xffff000000ff00ff, + 0xffff000000ffff00, 0x00ff000000ffffff, 0x00ff0000ff000000, 0xffff0000ff0000ff, 0xffff0000ff00ff00, + 0x00ff0000ff00ffff, 0xffff0000ffff0000, 0x00ff0000ffff00ff, 0x00ff0000ffffff00, 0xffff0000ffffffff, + 0x00ff00ff00000000, 0xffff00ff000000ff, 0xffff00ff0000ff00, 0x00ff00ff0000ffff, 0xffff00ff00ff0000, + 0x00ff00ff00ff00ff, 0x00ff00ff00ffff00, 0xffff00ff00ffffff, 0xffff00ffff000000, 0x00ff00ffff0000ff, + 0x00ff00ffff00ff00, 0xffff00ffff00ffff, 0x00ff00ffffff0000, 0xffff00ffffff00ff, 0xffff00ffffffff00, + 0x00ff00ffffffffff, 0x00ffff0000000000, 0xffffff00000000ff, 0xffffff000000ff00, 0x00ffff000000ffff, + 0xffffff0000ff0000, 0x00ffff0000ff00ff, 0x00ffff0000ffff00, 0xffffff0000ffffff, 0xffffff00ff000000, + 0x00ffff00ff0000ff, 0x00ffff00ff00ff00, 0xffffff00ff00ffff, 0x00ffff00ffff0000, 0xffffff00ffff00ff, + 0xffffff00ffffff00, 0x00ffff00ffffffff, 0xffffffff00000000, 0x00ffffff000000ff, 0x00ffffff0000ff00, + 0xffffffff0000ffff, 0x00ffffff00ff0000, 0xffffffff00ff00ff, 0xffffffff00ffff00, 0x00ffffff00ffffff, + 0x00ffffffff000000, 0xffffffffff0000ff, 0xffffffffff00ff00, 0x00ffffffff00ffff, 0xffffffffffff0000, + 0x00ffffffffff00ff, 0x00ffffffffffff00, 0xffffffffffffffff, +}; + +static const __device__ uint8_t kmask_iq2xs[8] = {1, 2, 4, 8, 16, 32, 64, 128}; +static const __device__ int8_t kvalues_iq4nl[16] = { + -127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113}; + +typedef half dfloat; // dequantize float +typedef half2 dfloat2; +typedef void (*dequantize_kernel_t)(const void* vx, const int ib, const int iqs, dfloat2& v); +template +using to_cuda_ggml_t = void (*)(const void* __restrict__ x, dst_t* __restrict__ y, int k, cudaStream_t stream); +typedef float (*vec_dot_q_cuda_t)(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs); +typedef void (*allocate_tiles_cuda_t)(int** x_ql, half2** x_dm, int** x_qh, int** x_sc); +typedef void (*load_tiles_cuda_t)( + const void* __restrict__ vx, + int* __restrict__ x_ql, + half2* __restrict__ x_dm, + int* __restrict__ x_qh, + int* __restrict__ x_sc, + const int& i_offset, + const int& i_max, + const int& k, + const int& blocks_per_row); +typedef float (*vec_dot_q_mul_mat_cuda_t)( + const int* __restrict__ x_ql, + const half2* __restrict__ x_dm, + const int* __restrict__ x_qh, + const int* __restrict__ x_sc, + const int* __restrict__ y_qs, + const half2* __restrict__ y_ms, + const int& i, + const int& j, + const int& k); + +// Utility function + +template +static __device__ __forceinline__ dst_t convert_from_half(half val) { + return val; +} + +template <> +__device__ __forceinline__ c10::BFloat16 convert_from_half(half val) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 + return __float2bfloat16(__half2float(val)); +#else + return __half2float(val); +#endif // defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 +} + +template <> +__device__ __forceinline__ float convert_from_half(half val) { + return __half2float(val); +} + +#if defined(USE_ROCM) + +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + +typedef int8_t int8x4_t __attribute__((ext_vector_type(4))); +static __device__ __forceinline__ int __vsubss4(const int a, const int b) { + const int8x4_t va = reinterpret_cast(a); + const int8x4_t vb = reinterpret_cast(b); +#if __has_builtin(__builtin_elementwise_sub_sat) + const int8x4_t c = __builtin_elementwise_sub_sat(va, vb); + return reinterpret_cast(c); +#else + int8x4_t c; + int16_t tmp; +#pragma unroll + for (int i = 0; i < 4; i++) { + tmp = va[i] - vb[i]; + if (tmp > std::numeric_limits::max()) tmp = std::numeric_limits::max(); + if (tmp < std::numeric_limits::min()) tmp = std::numeric_limits::min(); + c[i] = tmp; + } + return reinterpret_cast(c); +#endif // __has_builtin(__builtin_elementwise_sub_sat) +} + +static __device__ __forceinline__ int __dp4a(const int a, const int b, int c) { +#if __has_builtin(__builtin_amdgcn_sdot4) + c = __builtin_amdgcn_sdot4(a, b, c, false); +#else + const int8x4_t va = reinterpret_cast(a); + const int8x4_t vb = reinterpret_cast(b); + c += va[0] * vb[0] + va[1] * vb[1] + va[2] * vb[2] + va[3] * vb[3]; +#endif + return c; +} + +static __device__ __forceinline__ uint32_t __vcmpeq4(const uint32_t a, const uint32_t b) { + uint32_t neq = a ^ b; + return !(neq & 0xff000000) * 0xff000000 | !(neq & 0x00ff0000) * 0x00ff0000 | !(neq & 0x0000ff00) * 0x0000ff00 | + !(neq & 0x000000ff) * 0x000000ff; +} + +static __device__ __forceinline__ uint32_t __vsub4(const uint32_t a, const uint32_t b) { + return (static_cast(((a & 0xff000000) >> 24) - ((b & 0xff000000) >> 24)) << 24) + + (static_cast(((a & 0x00ff0000) >> 16) - ((b & 0x00ff0000) >> 16)) << 16) + + (static_cast(((a & 0x0000ff00) >> 8) - ((b & 0x0000ff00) >> 8)) << 8) + + (static_cast(((a & 0x000000ff) >> 0) - ((b & 0x000000ff) >> 0)) << 0); +} +#endif // defined(USE_ROCM) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/gguf_kernel.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/gguf_kernel.cu new file mode 100644 index 00000000..1a4955f5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/gguf_kernel.cu @@ -0,0 +1,836 @@ +// Adatped from +// https://github.com/vllm-project/vllm/blob/755ed7b05be4743237d3339c4ff8c22bcaae04f4/csrc/quantization/gguf/gguf_kernel.cu +#include +#include +#include +#include + +// dont use clang-format here, it breaks the include order +// clang-format off +#include "utils.h" + +#include "ggml-common.h" +#include "vecdotq.cuh" +#include "dequantize.cuh" +#include "mmvq.cuh" +#include "mmq.cuh" +#include "moe.cuh" +#include "moe_vec.cuh" +// clang-format off + +// Q8 gemv +template +static __global__ void +quantize_q8_1(const scalar_t* __restrict__ x, void* __restrict__ vy, const int kx, const int kx_padded) { + const auto ix = blockDim.x * blockIdx.x + threadIdx.x; + if (ix >= kx_padded) { + return; + } + const auto iy = blockDim.y * blockIdx.y + threadIdx.y; + const int i_padded = iy * kx_padded + ix; + + block_q8_1* y = (block_q8_1*)vy; + + const int ib = i_padded / QK8_1; // block index + const int iqs = i_padded % QK8_1; // quant index + + const float xi = ix < kx ? static_cast(x[iy * kx + ix]) : 0.0f; + float amax = fabsf(xi); + float sum = xi; + +#pragma unroll + for (int mask = 16; mask > 0; mask >>= 1) { + amax = fmaxf(amax, SGLANG_SHFL_XOR_SYNC_WIDTH(uint32_t(-1), amax, mask, 32)); + sum += SGLANG_SHFL_XOR_SYNC_WIDTH(uint32_t(-1), sum, mask, 32); + } + + const float d = amax / 127; + const int8_t q = amax == 0.0f ? 0 : roundf(xi / d); + + y[ib].qs[iqs] = q; + + if (iqs > 0) { + return; + } + + y[ib].ds.x = __float2half(d); + y[ib].ds.y = __float2half(sum); +} + +template +static void quantize_row_q8_1_cuda(const scalar_t* x, void* vy, const int kx, const int ky, cudaStream_t stream) { + const int64_t kx_padded = (kx + 512 - 1) / 512 * 512; + const int block_num_x = (kx_padded + CUDA_QUANTIZE_BLOCK_SIZE - 1) / CUDA_QUANTIZE_BLOCK_SIZE; + constexpr int MAX_BLOCK_SIZE = 65535; + for (int off = 0; off < ky; off += MAX_BLOCK_SIZE) { + const int num_blocks_y = std::min(ky, off + MAX_BLOCK_SIZE) - off; + const dim3 num_blocks(block_num_x, num_blocks_y, 1); + const dim3 block_size(CUDA_DEQUANTIZE_BLOCK_SIZE, 1, 1); + quantize_q8_1<<>>( + &x[off * kx], (int32_t*)vy + off * (kx_padded / 32 * 9), kx, kx_padded); + } +} + +torch::Tensor ggml_dequantize( + torch::Tensor W, // quant weight + int64_t type, + int64_t m, + int64_t n, + std::optional const& dtype) { + const at::cuda::OptionalCUDAGuard device_guard(device_of(W)); + auto dtype_ = dtype.value_or(torch::kFloat16); + auto options = torch::TensorOptions().dtype(dtype_).device(W.device()); + at::Tensor DW = torch::empty({m, n}, options); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + + DISPATCH_FLOAT_TYPES(DW.scalar_type(), "ggml_dequantize", [&] { + auto to_cuda = ggml_get_to_cuda(type); + to_cuda((void*)W.data_ptr(), (scalar_t*)DW.data_ptr(), m * n, stream); + }); + + return DW; +} + +torch::Tensor ggml_mul_mat_vec_a8( + torch::Tensor W, // quant weight + torch::Tensor X, // input + int64_t type, + int64_t row) { + int col = X.sizes()[1]; + int vecs = X.sizes()[0]; + const int padded = (col + 512 - 1) / 512 * 512; + const at::cuda::OptionalCUDAGuard device_guard(device_of(X)); + auto options = torch::TensorOptions().dtype(X.dtype()).device(W.device()); + at::Tensor Y = torch::empty({vecs, row}, options); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + options = torch::TensorOptions().dtype(torch::kInt32).device(W.device()); + at::Tensor quant_X = torch::empty({vecs, padded / 32 * 9}, options); + DISPATCH_FLOAT_TYPES(X.scalar_type(), "ggml_mul_mat_vec_a8", [&] { + quantize_row_q8_1_cuda((scalar_t*)X.data_ptr(), (void*)quant_X.data_ptr(), col, vecs, stream); + switch (type) { + case 2: + mul_mat_vec_q4_0_q8_1_cuda( + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream); + break; + case 3: + mul_mat_vec_q4_1_q8_1_cuda( + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream); + break; + case 6: + mul_mat_vec_q5_0_q8_1_cuda( + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream); + break; + case 7: + mul_mat_vec_q5_1_q8_1_cuda( + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream); + break; + case 8: + mul_mat_vec_q8_0_q8_1_cuda( + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream); + break; + case 10: + mul_mat_vec_q2_K_q8_1_cuda( + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream); + break; + case 11: + mul_mat_vec_q3_K_q8_1_cuda( + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream); + break; + case 12: + mul_mat_vec_q4_K_q8_1_cuda( + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream); + break; + case 13: + mul_mat_vec_q5_K_q8_1_cuda( + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream); + break; + case 14: + mul_mat_vec_q6_K_q8_1_cuda( + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream); + break; + case 16: + mul_mat_vec_iq2_xxs_q8_1_cuda( + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream); + break; + case 17: + mul_mat_vec_iq2_xs_q8_1_cuda( + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream); + break; + case 18: + mul_mat_vec_iq3_xxs_q8_1_cuda( + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream); + break; + case 19: + mul_mat_vec_iq1_s_q8_1_cuda( + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream); + break; + case 20: + mul_mat_vec_iq4_nl_q8_1_cuda( + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream); + break; + case 21: + mul_mat_vec_iq3_s_q8_1_cuda( + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream); + break; + case 22: + mul_mat_vec_iq2_s_q8_1_cuda( + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream); + break; + case 23: + mul_mat_vec_iq4_xs_q8_1_cuda( + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream); + break; + case 29: + mul_mat_vec_iq1_m_q8_1_cuda( + (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream); + break; + } + }); + return Y; +} + +torch::Tensor ggml_mul_mat_a8( + torch::Tensor W, // quant weight + torch::Tensor X, // input + int64_t type, + int64_t row) { + int col = X.sizes()[1]; + int padded = (col + 512 - 1) / 512 * 512; + int batch = X.sizes()[0]; + const at::cuda::OptionalCUDAGuard device_guard(device_of(X)); + auto options = torch::TensorOptions().dtype(X.dtype()).device(W.device()); + at::Tensor Y = torch::empty({batch, row}, options); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + options = torch::TensorOptions().dtype(torch::kInt32).device(W.device()); + at::Tensor quant_X = torch::empty({batch, padded / 32 * 9}, options); + DISPATCH_FLOAT_TYPES(X.scalar_type(), "ggml_mul_mat_a8", [&] { + quantize_row_q8_1_cuda((scalar_t*)X.data_ptr(), (void*)quant_X.data_ptr(), col, batch, stream); + + switch (type) { + case 2: + ggml_mul_mat_q4_0_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + col, + row, + batch, + padded, + row, + stream); + break; + case 3: + ggml_mul_mat_q4_1_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + col, + row, + batch, + padded, + row, + stream); + break; + case 6: + ggml_mul_mat_q5_0_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + col, + row, + batch, + padded, + row, + stream); + break; + case 7: + ggml_mul_mat_q5_1_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + col, + row, + batch, + padded, + row, + stream); + break; + case 8: + ggml_mul_mat_q8_0_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + col, + row, + batch, + padded, + row, + stream); + break; + case 10: + ggml_mul_mat_q2_K_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + col, + row, + batch, + padded, + row, + stream); + break; + case 11: + ggml_mul_mat_q3_K_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + col, + row, + batch, + padded, + row, + stream); + break; + case 12: + ggml_mul_mat_q4_K_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + col, + row, + batch, + padded, + row, + stream); + break; + case 13: + ggml_mul_mat_q5_K_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + col, + row, + batch, + padded, + row, + stream); + break; + case 14: + ggml_mul_mat_q6_K_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + col, + row, + batch, + padded, + row, + stream); + break; + } + }); + return Y; +} + +torch::Tensor ggml_moe_a8( + torch::Tensor X, // input + torch::Tensor W, // expert weights + torch::Tensor sorted_token_ids, + torch::Tensor expert_ids, + torch::Tensor num_tokens_post_padded, + int64_t type, + int64_t row, + int64_t top_k, + int64_t tokens) { + int col = X.sizes()[1]; + int padded = (col + 512 - 1) / 512 * 512; + const at::cuda::OptionalCUDAGuard device_guard(device_of(X)); + auto options = torch::TensorOptions().dtype(X.dtype()).device(W.device()); + at::Tensor Y = torch::empty({tokens * top_k, row}, options); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + options = torch::TensorOptions().dtype(torch::kInt32).device(W.device()); + at::Tensor quant_X = torch::empty({tokens, padded / 32 * 9}, options); + DISPATCH_FLOAT_TYPES(X.scalar_type(), "ggml_moe_a8", [&] { + quantize_row_q8_1_cuda((scalar_t*)X.data_ptr(), (void*)quant_X.data_ptr(), col, tokens, stream); + switch (type) { + case 2: + ggml_moe_q4_0_q8_1_cuda( + (void*)quant_X.data_ptr(), + (void*)W.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)sorted_token_ids.data_ptr(), + (int*)expert_ids.data_ptr(), + (int*)num_tokens_post_padded.data_ptr(), + W.stride(0), + col, + row, + tokens, + padded, + row, + top_k, + sorted_token_ids.sizes()[0], + stream); + break; + case 3: + ggml_moe_q4_1_q8_1_cuda( + (void*)quant_X.data_ptr(), + (void*)W.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)sorted_token_ids.data_ptr(), + (int*)expert_ids.data_ptr(), + (int*)num_tokens_post_padded.data_ptr(), + W.stride(0), + col, + row, + tokens, + padded, + row, + top_k, + sorted_token_ids.sizes()[0], + stream); + break; + case 6: + ggml_moe_q5_0_q8_1_cuda( + (void*)quant_X.data_ptr(), + (void*)W.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)sorted_token_ids.data_ptr(), + (int*)expert_ids.data_ptr(), + (int*)num_tokens_post_padded.data_ptr(), + W.stride(0), + col, + row, + tokens, + padded, + row, + top_k, + sorted_token_ids.sizes()[0], + stream); + break; + case 7: + ggml_moe_q5_1_q8_1_cuda( + (void*)quant_X.data_ptr(), + (void*)W.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)sorted_token_ids.data_ptr(), + (int*)expert_ids.data_ptr(), + (int*)num_tokens_post_padded.data_ptr(), + W.stride(0), + col, + row, + tokens, + padded, + row, + top_k, + sorted_token_ids.sizes()[0], + stream); + break; + case 8: + ggml_moe_q8_0_q8_1_cuda( + (void*)quant_X.data_ptr(), + (void*)W.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)sorted_token_ids.data_ptr(), + (int*)expert_ids.data_ptr(), + (int*)num_tokens_post_padded.data_ptr(), + W.stride(0), + col, + row, + tokens, + padded, + row, + top_k, + sorted_token_ids.sizes()[0], + stream); + break; + case 10: + ggml_moe_q2_K_q8_1_cuda( + (void*)quant_X.data_ptr(), + (void*)W.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)sorted_token_ids.data_ptr(), + (int*)expert_ids.data_ptr(), + (int*)num_tokens_post_padded.data_ptr(), + W.stride(0), + col, + row, + tokens, + padded, + row, + top_k, + sorted_token_ids.sizes()[0], + stream); + break; + case 11: + ggml_moe_q3_K_q8_1_cuda( + (void*)quant_X.data_ptr(), + (void*)W.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)sorted_token_ids.data_ptr(), + (int*)expert_ids.data_ptr(), + (int*)num_tokens_post_padded.data_ptr(), + W.stride(0), + col, + row, + tokens, + padded, + row, + top_k, + sorted_token_ids.sizes()[0], + stream); + break; + case 12: + ggml_moe_q4_K_q8_1_cuda( + (void*)quant_X.data_ptr(), + (void*)W.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)sorted_token_ids.data_ptr(), + (int*)expert_ids.data_ptr(), + (int*)num_tokens_post_padded.data_ptr(), + W.stride(0), + col, + row, + tokens, + padded, + row, + top_k, + sorted_token_ids.sizes()[0], + stream); + break; + case 13: + ggml_moe_q5_K_q8_1_cuda( + (void*)quant_X.data_ptr(), + (void*)W.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)sorted_token_ids.data_ptr(), + (int*)expert_ids.data_ptr(), + (int*)num_tokens_post_padded.data_ptr(), + W.stride(0), + col, + row, + tokens, + padded, + row, + top_k, + sorted_token_ids.sizes()[0], + stream); + break; + case 14: + ggml_moe_q6_K_q8_1_cuda( + (void*)quant_X.data_ptr(), + (void*)W.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)sorted_token_ids.data_ptr(), + (int*)expert_ids.data_ptr(), + (int*)num_tokens_post_padded.data_ptr(), + W.stride(0), + col, + row, + tokens, + padded, + row, + top_k, + sorted_token_ids.sizes()[0], + stream); + break; + } + }); + return Y; +} + +torch::Tensor ggml_moe_a8_vec( + torch::Tensor X, // input + torch::Tensor W, // expert weights + torch::Tensor topk_ids, + int64_t top_k, + int64_t type, + int64_t row, + int64_t tokens) { + int col = X.sizes()[1]; + const int padded = (col + 512 - 1) / 512 * 512; + const at::cuda::OptionalCUDAGuard device_guard(device_of(X)); + auto options = torch::TensorOptions().dtype(X.dtype()).device(W.device()); + at::Tensor Y = torch::zeros({tokens * top_k, row}, options); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + options = torch::TensorOptions().dtype(torch::kInt32).device(W.device()); + at::Tensor quant_X = torch::empty({tokens, padded / 32 * 9}, options); + DISPATCH_FLOAT_TYPES(X.scalar_type(), "ggml_moe_vec_a8", [&] { + quantize_row_q8_1_cuda((scalar_t*)X.data_ptr(), (void*)quant_X.data_ptr(), col, tokens, stream); + switch (type) { + case 2: + moe_vec_q4_0_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)topk_ids.data_ptr(), + top_k, + tokens, + col, + row, + quant_X.stride(0), + stream); + break; + case 3: + moe_vec_q4_1_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)topk_ids.data_ptr(), + top_k, + tokens, + col, + row, + quant_X.stride(0), + stream); + break; + case 6: + moe_vec_q5_0_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)topk_ids.data_ptr(), + top_k, + tokens, + col, + row, + quant_X.stride(0), + stream); + break; + case 7: + moe_vec_q5_1_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)topk_ids.data_ptr(), + top_k, + tokens, + col, + row, + quant_X.stride(0), + stream); + break; + case 8: + moe_vec_q8_0_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)topk_ids.data_ptr(), + top_k, + tokens, + col, + row, + quant_X.stride(0), + stream); + break; + case 10: + moe_vec_q2_K_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)topk_ids.data_ptr(), + top_k, + tokens, + col, + row, + quant_X.stride(0), + stream); + break; + case 11: + moe_vec_q3_K_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)topk_ids.data_ptr(), + top_k, + tokens, + col, + row, + quant_X.stride(0), + stream); + break; + case 12: + moe_vec_q4_K_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)topk_ids.data_ptr(), + top_k, + tokens, + col, + row, + quant_X.stride(0), + stream); + break; + case 13: + moe_vec_q5_K_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)topk_ids.data_ptr(), + top_k, + tokens, + col, + row, + quant_X.stride(0), + stream); + break; + case 14: + moe_vec_q6_K_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)topk_ids.data_ptr(), + top_k, + tokens, + col, + row, + quant_X.stride(0), + stream); + break; + case 16: + moe_vec_iq2_xxs_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)topk_ids.data_ptr(), + top_k, + tokens, + col, + row, + quant_X.stride(0), + stream); + break; + case 17: + moe_vec_iq2_xs_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)topk_ids.data_ptr(), + top_k, + tokens, + col, + row, + quant_X.stride(0), + stream); + break; + case 18: + moe_vec_iq3_xxs_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)topk_ids.data_ptr(), + top_k, + tokens, + col, + row, + quant_X.stride(0), + stream); + break; + case 19: + moe_vec_iq1_s_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)topk_ids.data_ptr(), + top_k, + tokens, + col, + row, + quant_X.stride(0), + stream); + break; + case 20: + moe_vec_iq4_nl_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)topk_ids.data_ptr(), + top_k, + tokens, + col, + row, + quant_X.stride(0), + stream); + break; + case 21: + moe_vec_iq3_s_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)topk_ids.data_ptr(), + top_k, + tokens, + col, + row, + quant_X.stride(0), + stream); + break; + case 22: + moe_vec_iq2_s_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)topk_ids.data_ptr(), + top_k, + tokens, + col, + row, + quant_X.stride(0), + stream); + break; + case 23: + moe_vec_iq4_xs_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)topk_ids.data_ptr(), + top_k, + tokens, + col, + row, + quant_X.stride(0), + stream); + break; + case 29: + moe_vec_iq1_m_q8_1_cuda( + (void*)W.data_ptr(), + (void*)quant_X.data_ptr(), + (scalar_t*)Y.data_ptr(), + (int*)topk_ids.data_ptr(), + top_k, + tokens, + col, + row, + quant_X.stride(0), + stream); + break; + } + }); + return Y; +} + +int64_t ggml_moe_get_block_size(int64_t type) { + switch (type) { + case 2: + return MOE_X_Q4_0; + case 3: + return MOE_X_Q4_1; + case 6: + return MOE_X_Q5_0; + case 7: + return MOE_X_Q5_1; + case 8: + return MOE_X_Q8_0; + case 10: + return MOE_X_Q2_K; + case 11: + return MOE_X_Q3_K; + case 12: + return MOE_X_Q4_K; + case 13: + return MOE_X_Q5_K; + case 14: + return MOE_X_Q6_K; + } + return 0; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/mmq.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/mmq.cuh new file mode 100644 index 00000000..9838c14f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/mmq.cuh @@ -0,0 +1,881 @@ +// copied from +// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/mmq.cuh +// copied from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmq.cu +template < + typename scalar_t, + int qk, + int qr, + int qi, + bool need_sum, + typename block_q_t, + int mmq_x, + int mmq_y, + int nwarps, + allocate_tiles_cuda_t allocate_tiles, + load_tiles_cuda_t load_tiles, + int vdr, + vec_dot_q_mul_mat_cuda_t vec_dot> +static __device__ __forceinline__ void mul_mat_q( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst) { + const block_q_t* x = (const block_q_t*)vx; + const block_q8_1* y = (const block_q8_1*)vy; + + const int blocks_per_row_x = ncols_x / qk; + const int blocks_per_col_y = nrows_y / QK8_1; + const int blocks_per_warp = WARP_SIZE_GGUF / qi; + + const int& ncols_dst = ncols_y; + + const auto row_dst_0 = blockIdx.x * mmq_y; + const int& row_x_0 = row_dst_0; + + const auto col_dst_0 = blockIdx.y * mmq_x; + const int& col_y_0 = col_dst_0; + + int* tile_x_ql = nullptr; + half2* tile_x_dm = nullptr; + int* tile_x_qh = nullptr; + int* tile_x_sc = nullptr; + + allocate_tiles(&tile_x_ql, &tile_x_dm, &tile_x_qh, &tile_x_sc); + + __shared__ int tile_y_qs[mmq_x * WARP_SIZE_GGUF]; + __shared__ half2 tile_y_ds[mmq_x * WARP_SIZE_GGUF / QI8_1]; + + float sum[mmq_y / WARP_SIZE_GGUF][mmq_x / nwarps] = {{0.0f}}; + + for (int ib0 = 0; ib0 < blocks_per_row_x; ib0 += blocks_per_warp) { + load_tiles( + x + row_x_0 * blocks_per_row_x + ib0, + tile_x_ql, + tile_x_dm, + tile_x_qh, + tile_x_sc, + threadIdx.y, + nrows_x - row_x_0 - 1, + threadIdx.x, + blocks_per_row_x); + +#pragma unroll + for (int ir = 0; ir < qr && ib0 + ir * blocks_per_warp / qr < blocks_per_row_x; ++ir) { + const auto kqs = ir * WARP_SIZE_GGUF + threadIdx.x; + const int kbxd = kqs / QI8_1; + +#pragma unroll + for (int i = 0; i < mmq_x; i += nwarps) { + const int col_y_eff = min(col_y_0 + threadIdx.y + i, ncols_y - 1); // to prevent out-of-bounds memory accesses + const block_q8_1* by0 = &y[col_y_eff * blocks_per_col_y + ib0 * (qk / QK8_1) + kbxd]; + const int index_y = (threadIdx.y + i) * WARP_SIZE_GGUF + kqs % WARP_SIZE_GGUF; + tile_y_qs[index_y] = get_int_from_int8_aligned(by0->qs, threadIdx.x % QI8_1); + } + +#pragma unroll + for (int ids0 = 0; ids0 < mmq_x; ids0 += nwarps * QI8_1) { + const int ids = (ids0 + threadIdx.y * QI8_1 + threadIdx.x / (WARP_SIZE_GGUF / QI8_1)) % mmq_x; + const auto kby = threadIdx.x % (WARP_SIZE_GGUF / QI8_1); + const int col_y_eff = min(col_y_0 + ids, ncols_y - 1); + + // if the sum is not needed it's faster to transform the scale to f32 ahead of time + const half2* dsi_src = + &y[col_y_eff * blocks_per_col_y + ib0 * (qk / QK8_1) + ir * (WARP_SIZE_GGUF / QI8_1) + kby].ds; + half2* dsi_dst = &tile_y_ds[ids * (WARP_SIZE_GGUF / QI8_1) + kby]; + if (need_sum) { + *dsi_dst = *dsi_src; + } else { + float* dfi_dst = (float*)dsi_dst; + *dfi_dst = __low2float(*dsi_src); + } + } + + __syncthreads(); + + // #pragma unroll // unrolling this loop causes too much register pressure + for (int k = ir * WARP_SIZE_GGUF / qr; k < (ir + 1) * WARP_SIZE_GGUF / qr; k += vdr) { +#pragma unroll + for (int j = 0; j < mmq_x; j += nwarps) { +#pragma unroll + for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) { + sum[i / WARP_SIZE_GGUF][j / nwarps] += vec_dot( + tile_x_ql, tile_x_dm, tile_x_qh, tile_x_sc, tile_y_qs, tile_y_ds, threadIdx.x + i, threadIdx.y + j, k); + } + } + } + __syncthreads(); + } + } + +#pragma unroll + for (int j = 0; j < mmq_x; j += nwarps) { + const auto col_dst = col_dst_0 + j + threadIdx.y; + if (col_dst >= ncols_dst) { + return; + } + +#pragma unroll + for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) { + const auto row_dst = row_dst_0 + threadIdx.x + i; + if (row_dst >= nrows_dst) { + continue; + } + dst[col_dst * nrows_dst + row_dst] = sum[i / WARP_SIZE_GGUF][j / nwarps]; + } + } +} + +#if defined(USE_ROCM) +#define MMQ_X_Q4_0 64 +#define MMQ_Y_Q4_0 128 +#define NWARPS_Q4_0 8 +#else +#define MMQ_X_Q4_0 4 +#define MMQ_Y_Q4_0 32 +#define NWARPS_Q4_0 4 +#endif + +template +static __global__ void +#if defined(USE_ROCM) +__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_0, 2) +#endif + mul_mat_q4_0( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst) { + const int mmq_x = MMQ_X_Q4_0; + const int mmq_y = MMQ_Y_Q4_0; + const int nwarps = NWARPS_Q4_0; + + mul_mat_q< + scalar_t, + QK4_0, + QR4_0, + QI4_0, + true, + block_q4_0, + mmq_x, + mmq_y, + nwarps, + allocate_tiles_q4_0, + load_tiles_q4_0, + VDR_Q4_0_Q8_1_MMQ, + vec_dot_q4_0_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); +} + +template +static void ggml_mul_mat_q4_0_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + cudaStream_t stream) { + int mmq_x = MMQ_X_Q4_0; + int mmq_y = MMQ_Y_Q4_0; + int nwarps = NWARPS_Q4_0; + + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; + const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x; + const dim3 block_nums(block_num_x, block_num_y, 1); + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1); + + if (nrows_x % mmq_y == 0) { + const bool need_check = false; + mul_mat_q4_0 + <<>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); + } else { + const bool need_check = true; + mul_mat_q4_0 + <<>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); + } +} + +#if defined(USE_ROCM) +#define MMQ_X_Q4_1 64 +#define MMQ_Y_Q4_1 128 +#define NWARPS_Q4_1 8 +#else +#define MMQ_X_Q4_1 4 +#define MMQ_Y_Q4_1 32 +#define NWARPS_Q4_1 4 +#endif + +template +static __global__ void +#if defined(USE_ROCM) +__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_1, 2) +#endif + mul_mat_q4_1( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst) { + const int mmq_x = MMQ_X_Q4_1; + const int mmq_y = MMQ_Y_Q4_1; + const int nwarps = NWARPS_Q4_1; + + mul_mat_q< + scalar_t, + QK4_1, + QR4_1, + QI4_1, + true, + block_q4_1, + mmq_x, + mmq_y, + nwarps, + allocate_tiles_q4_1, + load_tiles_q4_1, + VDR_Q4_1_Q8_1_MMQ, + vec_dot_q4_1_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); +} + +template +static void ggml_mul_mat_q4_1_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + cudaStream_t stream) { + int mmq_x = MMQ_X_Q4_1; + int mmq_y = MMQ_Y_Q4_1; + int nwarps = NWARPS_Q4_1; + + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; + const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x; + const dim3 block_nums(block_num_x, block_num_y, 1); + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1); + + if (nrows_x % mmq_y == 0) { + const bool need_check = false; + mul_mat_q4_1 + <<>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); + } else { + const bool need_check = true; + mul_mat_q4_1 + <<>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); + } +} + +#if defined(USE_ROCM) +#define MMQ_X_Q5_0 64 +#define MMQ_Y_Q5_0 128 +#define NWARPS_Q5_0 8 +#else +#define MMQ_X_Q5_0 4 +#define MMQ_Y_Q5_0 32 +#define NWARPS_Q5_0 4 +#endif + +template +static __global__ void +#if defined(USE_ROCM) +__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_0, 2) +#endif + mul_mat_q5_0( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst) { + const int mmq_x = MMQ_X_Q5_0; + const int mmq_y = MMQ_Y_Q5_0; + const int nwarps = NWARPS_Q5_0; + + mul_mat_q< + scalar_t, + QK5_0, + QR5_0, + QI5_0, + false, + block_q5_0, + mmq_x, + mmq_y, + nwarps, + allocate_tiles_q5_0, + load_tiles_q5_0, + VDR_Q5_0_Q8_1_MMQ, + vec_dot_q5_0_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); +} + +template +static void ggml_mul_mat_q5_0_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + cudaStream_t stream) { + const int mmq_x = MMQ_X_Q5_0; + const int mmq_y = MMQ_Y_Q5_0; + const int nwarps = NWARPS_Q5_0; + + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; + const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x; + const dim3 block_nums(block_num_x, block_num_y, 1); + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1); + + if (nrows_x % mmq_y == 0) { + const bool need_check = false; + mul_mat_q5_0 + <<>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); + } else { + const bool need_check = true; + mul_mat_q5_0 + <<>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); + } +} + +#if defined(USE_ROCM) +#define MMQ_X_Q5_1 64 +#define MMQ_Y_Q5_1 128 +#define NWARPS_Q5_1 8 +#else +#define MMQ_X_Q5_1 4 +#define MMQ_Y_Q5_1 32 +#define NWARPS_Q5_1 4 +#endif + +template +static __global__ void +#if defined(USE_ROCM) +__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_1, 2) +#endif + mul_mat_q5_1( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst) { + const int mmq_x = MMQ_X_Q5_1; + const int mmq_y = MMQ_Y_Q5_1; + const int nwarps = NWARPS_Q5_1; + + mul_mat_q< + scalar_t, + QK5_1, + QR5_1, + QI5_1, + true, + block_q5_1, + mmq_x, + mmq_y, + nwarps, + allocate_tiles_q5_1, + load_tiles_q5_1, + VDR_Q5_1_Q8_1_MMQ, + vec_dot_q5_1_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); +} + +template +static void ggml_mul_mat_q5_1_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + cudaStream_t stream) { + const int mmq_x = MMQ_X_Q5_1; + const int mmq_y = MMQ_Y_Q5_1; + const int nwarps = NWARPS_Q5_1; + + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; + const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x; + const dim3 block_nums(block_num_x, block_num_y, 1); + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1); + + if (nrows_x % mmq_y == 0) { + const bool need_check = false; + mul_mat_q5_1 + <<>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); + } else { + const bool need_check = true; + mul_mat_q5_1 + <<>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); + } +} + +#if defined(USE_ROCM) +#define MMQ_X_Q8_0 64 +#define MMQ_Y_Q8_0 128 +#define NWARPS_Q8_0 8 +#else +#define MMQ_X_Q8_0 4 +#define MMQ_Y_Q8_0 32 +#define NWARPS_Q8_0 4 +#endif + +template +static __global__ void +#if defined(USE_ROCM) +__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q8_0, 2) +#endif + mul_mat_q8_0( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst) { + const int mmq_x = MMQ_X_Q8_0; + const int mmq_y = MMQ_Y_Q8_0; + const int nwarps = NWARPS_Q8_0; + + mul_mat_q< + scalar_t, + QK8_0, + QR8_0, + QI8_0, + false, + block_q8_0, + mmq_x, + mmq_y, + nwarps, + allocate_tiles_q8_0, + load_tiles_q8_0, + VDR_Q8_0_Q8_1_MMQ, + vec_dot_q8_0_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); +} + +template +static void ggml_mul_mat_q8_0_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + cudaStream_t stream) { + const int mmq_x = MMQ_X_Q8_0; + const int mmq_y = MMQ_Y_Q8_0; + const int nwarps = NWARPS_Q8_0; + + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; + const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x; + const dim3 block_nums(block_num_x, block_num_y, 1); + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1); + + if (nrows_x % mmq_y == 0) { + const bool need_check = false; + mul_mat_q8_0 + <<>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); + } else { + const bool need_check = true; + mul_mat_q8_0 + <<>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); + } +} + +#if defined(USE_ROCM) +#define MMQ_X_Q2_K 64 +#define MMQ_Y_Q2_K 128 +#define NWARPS_Q2_K 8 +#else +#define MMQ_X_Q2_K 4 +#define MMQ_Y_Q2_K 32 +#define NWARPS_Q2_K 4 +#endif + +template +static __global__ void +#if defined(USE_ROCM) +__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q2_K, 2) +#endif + mul_mat_q2_K( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst) { + const int mmq_x = MMQ_X_Q2_K; + const int mmq_y = MMQ_Y_Q2_K; + const int nwarps = NWARPS_Q2_K; + + mul_mat_q< + scalar_t, + QK_K, + QR2_K, + QI2_K, + false, + block_q2_K, + mmq_x, + mmq_y, + nwarps, + allocate_tiles_q2_K, + load_tiles_q2_K, + VDR_Q2_K_Q8_1_MMQ, + vec_dot_q2_K_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); +} + +template +static void ggml_mul_mat_q2_K_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + cudaStream_t stream) { + const int mmq_x = MMQ_X_Q2_K; + const int mmq_y = MMQ_Y_Q2_K; + const int nwarps = NWARPS_Q2_K; + + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; + const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x; + const dim3 block_nums(block_num_x, block_num_y, 1); + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1); + + if (nrows_x % mmq_y == 0) { + const bool need_check = false; + mul_mat_q2_K + <<>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); + } else { + const bool need_check = true; + mul_mat_q2_K + <<>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); + } +} + +#if defined(USE_ROCM) +#define MMQ_X_Q3_K 64 +#define MMQ_Y_Q3_K 128 +#define NWARPS_Q3_K 8 +#else +#define MMQ_X_Q3_K 4 +#define MMQ_Y_Q3_K 32 +#define NWARPS_Q3_K 4 +#endif + +template +static __global__ void +#if defined(USE_ROCM) +__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q3_K, 2) +#endif + mul_mat_q3_K( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst) { + + const int mmq_x = MMQ_X_Q3_K; + const int mmq_y = MMQ_Y_Q3_K; + const int nwarps = NWARPS_Q3_K; + + mul_mat_q< + scalar_t, + QK_K, + QR3_K, + QI3_K, + false, + block_q3_K, + mmq_x, + mmq_y, + nwarps, + allocate_tiles_q3_K, + load_tiles_q3_K, + VDR_Q3_K_Q8_1_MMQ, + vec_dot_q3_K_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); +} + +template +static void ggml_mul_mat_q3_K_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + cudaStream_t stream) { + const int mmq_x = MMQ_X_Q3_K; + const int mmq_y = MMQ_Y_Q3_K; + const int nwarps = NWARPS_Q3_K; + + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; + const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x; + const dim3 block_nums(block_num_x, block_num_y, 1); + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1); + + if (nrows_x % mmq_y == 0) { + const bool need_check = false; + mul_mat_q3_K + <<>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); + } else { + const bool need_check = true; + mul_mat_q3_K + <<>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); + } +} + +#if defined(USE_ROCM) +#define MMQ_X_Q4_K 64 +#define MMQ_Y_Q4_K 128 +#define NWARPS_Q4_K 8 +#else +#define MMQ_X_Q4_K 4 +#define MMQ_Y_Q4_K 32 +#define NWARPS_Q4_K 4 +#endif + +template +static __global__ void +#if defined(USE_ROCM) +__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_K, 2) +#endif + mul_mat_q4_K( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst) { + const int mmq_x = MMQ_X_Q4_K; + const int mmq_y = MMQ_Y_Q4_K; + const int nwarps = NWARPS_Q4_K; + + mul_mat_q< + scalar_t, + QK_K, + QR4_K, + QI4_K, + true, + block_q4_K, + mmq_x, + mmq_y, + nwarps, + allocate_tiles_q4_K, + load_tiles_q4_K, + VDR_Q4_K_Q8_1_MMQ, + vec_dot_q4_K_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); +} + +template +static void ggml_mul_mat_q4_K_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + cudaStream_t stream) { + const int mmq_x = MMQ_X_Q4_K; + const int mmq_y = MMQ_Y_Q4_K; + const int nwarps = NWARPS_Q4_K; + + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; + const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x; + const dim3 block_nums(block_num_x, block_num_y, 1); + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1); + + if (nrows_x % mmq_y == 0) { + const bool need_check = false; + mul_mat_q4_K + <<>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); + } else { + const bool need_check = true; + mul_mat_q4_K + <<>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); + } +} + +#if defined(USE_ROCM) +#define MMQ_X_Q5_K 64 +#define MMQ_Y_Q5_K 128 +#define NWARPS_Q5_K 8 +#else +#define MMQ_X_Q5_K 4 +#define MMQ_Y_Q5_K 32 +#define NWARPS_Q5_K 4 +#endif + +template +static __global__ void +#if defined(USE_ROCM) +__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_K, 2) +#endif + mul_mat_q5_K( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst) { + const int mmq_x = MMQ_X_Q5_K; + const int mmq_y = MMQ_Y_Q5_K; + const int nwarps = NWARPS_Q5_K; + + mul_mat_q< + scalar_t, + QK_K, + QR5_K, + QI5_K, + true, + block_q5_K, + mmq_x, + mmq_y, + nwarps, + allocate_tiles_q5_K, + load_tiles_q5_K, + VDR_Q5_K_Q8_1_MMQ, + vec_dot_q5_K_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); +} + +template +static void ggml_mul_mat_q5_K_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + cudaStream_t stream) { + const int mmq_x = MMQ_X_Q5_K; + const int mmq_y = MMQ_Y_Q5_K; + const int nwarps = NWARPS_Q5_K; + + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; + const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x; + const dim3 block_nums(block_num_x, block_num_y, 1); + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1); + + if (nrows_x % mmq_y == 0) { + const bool need_check = false; + mul_mat_q5_K + <<>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); + } else { + const bool need_check = true; + mul_mat_q5_K + <<>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); + } +} + +#if defined(USE_ROCM) +#define MMQ_X_Q6_K 64 +#define MMQ_Y_Q6_K 128 +#define NWARPS_Q6_K 8 +#else +#define MMQ_X_Q6_K 4 +#define MMQ_Y_Q6_K 32 +#define NWARPS_Q6_K 4 +#endif + +template +static __global__ void +#if defined(USE_ROCM) +__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q6_K, 2) +#endif + mul_mat_q6_K( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst) { + const int mmq_x = MMQ_X_Q6_K; + const int mmq_y = MMQ_Y_Q6_K; + const int nwarps = NWARPS_Q6_K; + + mul_mat_q< + scalar_t, + QK_K, + QR6_K, + QI6_K, + false, + block_q6_K, + mmq_x, + mmq_y, + nwarps, + allocate_tiles_q6_K, + load_tiles_q6_K, + VDR_Q6_K_Q8_1_MMQ, + vec_dot_q6_K_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); +} + +template +static void ggml_mul_mat_q6_K_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + cudaStream_t stream) { + const int mmq_x = MMQ_X_Q6_K; + const int mmq_y = MMQ_Y_Q6_K; + const int nwarps = NWARPS_Q6_K; + + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; + const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x; + const dim3 block_nums(block_num_x, block_num_y, 1); + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1); + + if (nrows_x % mmq_y == 0) { + const bool need_check = false; + mul_mat_q6_K + <<>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); + } else { + const bool need_check = true; + mul_mat_q6_K + <<>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/mmvq.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/mmvq.cuh new file mode 100644 index 00000000..7331731a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/mmvq.cuh @@ -0,0 +1,352 @@ +// copied from +// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/mmvq.cuh +// copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmvq.cu +template +static __global__ void mul_mat_vec_q( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int ncols, + const int nrows, + const int nvecs) { + const auto row = blockIdx.x * blockDim.y + threadIdx.y; + const auto vec = blockIdx.y; + + if (row >= nrows || vec >= nvecs) { + return; + } + + const int blocks_per_row = ncols / qk; + const int blocks_per_warp = vdr * WARP_SIZE / qi; + const int nrows_y = (ncols + 512 - 1) / 512 * 512; + + // partial sum for each thread + float tmp = 0.0f; + + const block_q_t* x = (const block_q_t*)vx; + const block_q8_1* y = (const block_q8_1*)vy; + + for (auto i = threadIdx.x / (qi / vdr); i < blocks_per_row; i += blocks_per_warp) { + const int ibx = row * blocks_per_row + i; // x block index + + const int iby = vec * (nrows_y / QK8_1) + i * (qk / QK8_1); // y block index that aligns with ibx + + const int iqs = vdr * (threadIdx.x % (qi / vdr)); // x block quant index when casting the quants to int + + tmp += vec_dot_q_cuda(&x[ibx], &y[iby], iqs); + } + + // sum up partial sums and write back result +#pragma unroll + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { + tmp += SGLANG_SHFL_XOR_SYNC(uint32_t(-1), tmp, mask); + } + + if (threadIdx.x == 0) { + dst[vec * nrows + row] = tmp; + } +} + +template +static void mul_mat_vec_q4_0_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols, + const int nrows, + const int nvecs, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, nvecs, 1); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + mul_mat_vec_q + <<>>(vx, vy, dst, ncols, nrows, nvecs); +} + +template +static void mul_mat_vec_q4_1_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols, + const int nrows, + const int nvecs, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, nvecs, 1); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + mul_mat_vec_q + <<>>(vx, vy, dst, ncols, nrows, nvecs); +} + +template +static void mul_mat_vec_q5_0_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols, + const int nrows, + const int nvecs, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, nvecs, 1); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + mul_mat_vec_q + <<>>(vx, vy, dst, ncols, nrows, nvecs); +} + +template +static void mul_mat_vec_q5_1_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols, + const int nrows, + const int nvecs, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, nvecs, 1); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + mul_mat_vec_q + <<>>(vx, vy, dst, ncols, nrows, nvecs); +} + +template +static void mul_mat_vec_q8_0_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols, + const int nrows, + const int nvecs, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, nvecs, 1); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + mul_mat_vec_q + <<>>(vx, vy, dst, ncols, nrows, nvecs); +} + +template +static void mul_mat_vec_q2_K_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols, + const int nrows, + const int nvecs, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, nvecs, 1); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + mul_mat_vec_q + <<>>(vx, vy, dst, ncols, nrows, nvecs); +} + +template +static void mul_mat_vec_q3_K_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols, + const int nrows, + const int nvecs, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, nvecs, 1); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + mul_mat_vec_q + <<>>(vx, vy, dst, ncols, nrows, nvecs); +} + +template +static void mul_mat_vec_q4_K_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols, + const int nrows, + const int nvecs, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, nvecs, 1); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + mul_mat_vec_q + <<>>(vx, vy, dst, ncols, nrows, nvecs); +} + +template +static void mul_mat_vec_q5_K_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols, + const int nrows, + const int nvecs, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, nvecs, 1); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + mul_mat_vec_q + <<>>(vx, vy, dst, ncols, nrows, nvecs); +} + +template +static void mul_mat_vec_q6_K_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols, + const int nrows, + const int nvecs, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, nvecs, 1); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + mul_mat_vec_q + <<>>(vx, vy, dst, ncols, nrows, nvecs); +} + +template +static void mul_mat_vec_iq2_xxs_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols, + const int nrows, + const int nvecs, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, nvecs, 1); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + mul_mat_vec_q + <<>>(vx, vy, dst, ncols, nrows, nvecs); +} + +template +static void mul_mat_vec_iq2_xs_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols, + const int nrows, + const int nvecs, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, nvecs, 1); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + mul_mat_vec_q + <<>>(vx, vy, dst, ncols, nrows, nvecs); +} + +template +static void mul_mat_vec_iq2_s_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols, + const int nrows, + const int nvecs, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, nvecs, 1); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + mul_mat_vec_q + <<>>(vx, vy, dst, ncols, nrows, nvecs); +} + +template +static void mul_mat_vec_iq3_xxs_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols, + const int nrows, + const int nvecs, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, nvecs, 1); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + mul_mat_vec_q + <<>>(vx, vy, dst, ncols, nrows, nvecs); +} + +template +static void mul_mat_vec_iq1_s_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols, + const int nrows, + const int nvecs, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, nvecs, 1); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + mul_mat_vec_q + <<>>(vx, vy, dst, ncols, nrows, nvecs); +} + +template +static void mul_mat_vec_iq1_m_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols, + const int nrows, + const int nvecs, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, nvecs, 1); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + mul_mat_vec_q + <<>>(vx, vy, dst, ncols, nrows, nvecs); +} + +template +static void mul_mat_vec_iq4_nl_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols, + const int nrows, + const int nvecs, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, nvecs, 1); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + mul_mat_vec_q + <<>>(vx, vy, dst, ncols, nrows, nvecs); +} + +template +static void mul_mat_vec_iq4_xs_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols, + const int nrows, + const int nvecs, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, nvecs, 1); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + mul_mat_vec_q + <<>>(vx, vy, dst, ncols, nrows, nvecs); +} + +template +static void mul_mat_vec_iq3_s_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int ncols, + const int nrows, + const int nvecs, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, nvecs, 1); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + mul_mat_vec_q + <<>>(vx, vy, dst, ncols, nrows, nvecs); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/moe.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/moe.cuh new file mode 100644 index 00000000..91e434fa --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/moe.cuh @@ -0,0 +1,1379 @@ +// copied from +// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/moe.cuh +#include + +/* Adapted from ./csrc/quantization/gguf/mmq.cuh + */ +template < + typename scalar_t, + int qk, + int qr, + int qi, + bool need_sum, + typename block_q_t, + int mmq_x, + int mmq_y, + int nwarps, + allocate_tiles_cuda_t allocate_tiles, + load_tiles_cuda_t load_tiles, + int vdr, + vec_dot_q_mul_mat_cuda_t vec_dot> +static __device__ __forceinline__ void moe_q( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int* __restrict__ sorted_token_ids, + const int* __restrict__ expert_ids, + const int* __restrict__ num_tokens_post_padded, + const int exp_stride, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + const int top_k) { + const int blocks_per_row_x = ncols_x / qk; + const int blocks_per_col_y = nrows_y / QK8_1; + const int blocks_per_warp = WARP_SIZE_GGUF / qi; + + const int ncols_dst = ncols_y * top_k; + + const auto row_dst_0 = blockIdx.x * mmq_y; + const int& row_x_0 = row_dst_0; + + const auto col_dst_0 = blockIdx.y * mmq_x; + + int token_offs[mmq_x / nwarps]; + for (int i = 0; i < mmq_x; i += nwarps) { + token_offs[i / nwarps] = sorted_token_ids[col_dst_0 + threadIdx.y + i]; + } + + const int exp_idx = expert_ids[blockIdx.y]; + if (exp_idx > 255 || exp_idx < 0) return; + if (blockIdx.y * mmq_x > num_tokens_post_padded[0]) return; + + const block_q_t* x = (const block_q_t*)((char*)vx + exp_idx * exp_stride); + const block_q8_1* y = (const block_q8_1*)(vy); + + int* tile_x_ql = nullptr; + half2* tile_x_dm = nullptr; + int* tile_x_qh = nullptr; + int* tile_x_sc = nullptr; + + allocate_tiles(&tile_x_ql, &tile_x_dm, &tile_x_qh, &tile_x_sc); + + __shared__ int tile_y_qs[mmq_x * WARP_SIZE_GGUF]; + __shared__ half2 tile_y_ds[mmq_x * WARP_SIZE_GGUF / QI8_1]; + + float sum[mmq_y / WARP_SIZE_GGUF][mmq_x / nwarps] = {{0.0f}}; + + for (int ib0 = 0; ib0 < blocks_per_row_x; ib0 += blocks_per_warp) { + load_tiles( + x + row_x_0 * blocks_per_row_x + ib0, + tile_x_ql, + tile_x_dm, + tile_x_qh, + tile_x_sc, + threadIdx.y, + nrows_x - row_x_0 - 1, + threadIdx.x, + blocks_per_row_x); + + const int n_per_r = ((qk * blocks_per_warp) / qr); +#pragma unroll + for (int ir = 0; ir < qr && ib0 * qk + ir * n_per_r < ncols_x; ++ir) { + const auto kqs = ir * WARP_SIZE_GGUF + threadIdx.x; + const int kbxd = kqs / QI8_1; + +#pragma unroll + for (int i = 0; i < mmq_x; i += nwarps) { + const int col_y_eff = token_offs[i / nwarps] / top_k; + const int block_x = ib0 * (qk / QK8_1) + kbxd; + if (col_y_eff < ncols_y && block_x < blocks_per_col_y) { + const block_q8_1* by0 = &y[col_y_eff * blocks_per_col_y + block_x]; + const int index_y = (threadIdx.y + i) * WARP_SIZE_GGUF + kqs % WARP_SIZE_GGUF; + tile_y_qs[index_y] = get_int_from_int8_aligned(by0->qs, threadIdx.x % QI8_1); + } + } + + if (threadIdx.x < n_per_r / QK8_1) { + const auto kby = threadIdx.x % (WARP_SIZE_GGUF / QI8_1); + const int col_y_eff = token_offs[threadIdx.y] / top_k; + const int block_x = ib0 * (qk / QK8_1) + ir * (WARP_SIZE_GGUF / QI8_1) + kby; + + if (col_y_eff < ncols_y && block_x < blocks_per_col_y) { + const half2* dsi_src = &y[col_y_eff * blocks_per_col_y + block_x].ds; + half2* dsi_dst = &tile_y_ds[threadIdx.y * (WARP_SIZE_GGUF / QI8_1) + kby]; + + if (need_sum) { + *dsi_dst = *dsi_src; + } else { + float* dfi_dst = (float*)dsi_dst; + *dfi_dst = __low2float(*dsi_src); + } + } + } + __syncthreads(); + + // #pragma unroll // unrolling this loop causes too much register pressure + for (int k = ir * WARP_SIZE_GGUF / qr; k < (ir + 1) * WARP_SIZE_GGUF / qr; k += vdr) { +#pragma unroll + for (int j = 0; j < mmq_x; j += nwarps) { +#pragma unroll + for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) { + sum[i / WARP_SIZE_GGUF][j / nwarps] += vec_dot( + tile_x_ql, tile_x_dm, tile_x_qh, tile_x_sc, tile_y_qs, tile_y_ds, threadIdx.x + i, threadIdx.y + j, k); + } + } + } + __syncthreads(); + } + } + +#pragma unroll + for (int j = 0; j < mmq_x; j += nwarps) { + const int col_dst = token_offs[j / nwarps]; + if (col_dst >= ncols_dst) { + return; + } + +#pragma unroll + for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) { + const auto row_dst = row_dst_0 + threadIdx.x + i; + if (row_dst >= nrows_dst) { + continue; + } + dst[col_dst * nrows_dst + row_dst] = sum[i / WARP_SIZE_GGUF][j / nwarps]; + } + } +} + +#if defined(USE_ROCM) +#define MOE_X_Q4_0 8 +#define MOE_Y_Q4_0 128 +#define NWARPS_Q4_0 8 +#else +#define MOE_X_Q4_0 4 +#define MOE_Y_Q4_0 32 +#define NWARPS_Q4_0 4 +#endif + +template +static __global__ void +#if defined(USE_ROCM) +__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_0, 2) +#endif + moe_q4_0( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int* sorted_token_ids, + const int* expert_ids, + const int* num_tokens_post_padded, + const int exp_stride, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + const int top_k) { + const int mmq_x = MOE_X_Q4_0; + const int mmq_y = MOE_Y_Q4_0; + const int nwarps = NWARPS_Q4_0; + + moe_q< + scalar_t, + QK4_0, + QR4_0, + QI4_0, + true, + block_q4_0, + mmq_x, + mmq_y, + nwarps, + allocate_tiles_q4_0, + load_tiles_q4_0, + VDR_Q4_0_Q8_1_MMQ, + vec_dot_q4_0_q8_1_mul_mat>( + vx, + vy, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); +} + +template +static void ggml_moe_q4_0_q8_1_cuda( + const void* inp, + const void* w, + scalar_t* dst, + const int* sorted_token_ids, + const int* expert_ids, + const int* num_tokens_post_padded, + const int exp_stride, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + const int top_k, + const int tokens_post_padded, + cudaStream_t stream) { + int mmq_x = MOE_X_Q4_0; + int mmq_y = MOE_Y_Q4_0; + int nwarps = NWARPS_Q4_0; + + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; + const int block_num_y = (tokens_post_padded) / mmq_x; + const dim3 block_nums(block_num_x, block_num_y, 1); + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1); + + if (nrows_x % mmq_y == 0) { + constexpr bool need_check = false; + moe_q4_0<<>>( + w, + inp, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); + } else { + constexpr bool need_check = true; + moe_q4_0<<>>( + w, + inp, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); + } +} + +#if defined(USE_ROCM) +#define MOE_X_Q4_1 8 +#define MOE_Y_Q4_1 128 +#define NWARPS_Q4_1 8 +#else +#define MOE_X_Q4_1 4 +#define MOE_Y_Q4_1 32 +#define NWARPS_Q4_1 4 +#endif + +template +static __global__ void +#if defined(USE_ROCM) +__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_1, 2) +#endif + moe_q4_1( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int* sorted_token_ids, + const int* expert_ids, + const int* num_tokens_post_padded, + const int exp_stride, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + const int top_k) { + const int mmq_x = MOE_X_Q4_1; + const int mmq_y = MOE_Y_Q4_1; + const int nwarps = NWARPS_Q4_1; + + moe_q< + scalar_t, + QK4_1, + QR4_1, + QI4_1, + true, + block_q4_1, + mmq_x, + mmq_y, + nwarps, + allocate_tiles_q4_1, + load_tiles_q4_1, + VDR_Q4_1_Q8_1_MMQ, + vec_dot_q4_1_q8_1_mul_mat>( + vx, + vy, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); +} + +template +static void ggml_moe_q4_1_q8_1_cuda( + const void* inp, + const void* w, + scalar_t* dst, + const int* sorted_token_ids, + const int* expert_ids, + const int* num_tokens_post_padded, + const int exp_stride, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + const int top_k, + const int tokens_post_padded, + cudaStream_t stream) { + int mmq_x = MOE_X_Q4_1; + int mmq_y = MOE_Y_Q4_1; + int nwarps = NWARPS_Q4_1; + + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; + const int block_num_y = (tokens_post_padded) / mmq_x; + const dim3 block_nums(block_num_x, block_num_y, 1); + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1); + + if (nrows_x % mmq_y == 0) { + constexpr bool need_check = false; + moe_q4_1<<>>( + w, + inp, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); + } else { + constexpr bool need_check = true; + moe_q4_1<<>>( + w, + inp, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); + } +} + +#if defined(USE_ROCM) +#define MOE_X_Q5_0 8 +#define MOE_Y_Q5_0 128 +#define NWARPS_Q5_0 8 +#else +#define MOE_X_Q5_0 4 +#define MOE_Y_Q5_0 32 +#define NWARPS_Q5_0 4 +#endif + +template +static __global__ void +#if defined(USE_ROCM) +__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_0, 2) +#endif + moe_q5_0( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int* sorted_token_ids, + const int* expert_ids, + const int* num_tokens_post_padded, + const int exp_stride, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + const int top_k) { + const int mmq_x = MOE_X_Q5_0; + const int mmq_y = MOE_Y_Q5_0; + const int nwarps = NWARPS_Q5_0; + + moe_q< + scalar_t, + QK5_0, + QR5_0, + QI5_0, + false, + block_q5_0, + mmq_x, + mmq_y, + nwarps, + allocate_tiles_q5_0, + load_tiles_q5_0, + VDR_Q5_0_Q8_1_MMQ, + vec_dot_q5_0_q8_1_mul_mat>( + vx, + vy, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); +} + +template +static void ggml_moe_q5_0_q8_1_cuda( + const void* inp, + const void* w, + scalar_t* dst, + const int* sorted_token_ids, + const int* expert_ids, + const int* num_tokens_post_padded, + const int exp_stride, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + const int top_k, + const int tokens_post_padded, + cudaStream_t stream) { + const int mmq_x = MOE_X_Q5_0; + const int mmq_y = MOE_Y_Q5_0; + const int nwarps = NWARPS_Q5_0; + + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; + const int block_num_y = (tokens_post_padded) / mmq_x; + const dim3 block_nums(block_num_x, block_num_y, 1); + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1); + + if (nrows_x % mmq_y == 0) { + constexpr bool need_check = false; + moe_q5_0<<>>( + w, + inp, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); + } else { + constexpr bool need_check = true; + moe_q5_0<<>>( + w, + inp, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); + } +} + +#if defined(USE_ROCM) +#define MOE_X_Q5_1 8 +#define MOE_Y_Q5_1 128 +#define NWARPS_Q5_1 8 +#else +#define MOE_X_Q5_1 4 +#define MOE_Y_Q5_1 32 +#define NWARPS_Q5_1 4 +#endif + +template +static __global__ void +#if defined(USE_ROCM) +__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_1, 2) +#endif + moe_q5_1( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int* sorted_token_ids, + const int* expert_ids, + const int* num_tokens_post_padded, + const int exp_stride, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + const int top_k) { + const int mmq_x = MOE_X_Q5_1; + const int mmq_y = MOE_Y_Q5_1; + const int nwarps = NWARPS_Q5_1; + + moe_q< + scalar_t, + QK5_1, + QR5_1, + QI5_1, + true, + block_q5_1, + mmq_x, + mmq_y, + nwarps, + allocate_tiles_q5_1, + load_tiles_q5_1, + VDR_Q5_1_Q8_1_MMQ, + vec_dot_q5_1_q8_1_mul_mat>( + vx, + vy, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); +} + +template +static void ggml_moe_q5_1_q8_1_cuda( + const void* inp, + const void* w, + scalar_t* dst, + const int* sorted_token_ids, + const int* expert_ids, + const int* num_tokens_post_padded, + const int exp_stride, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + const int top_k, + const int tokens_post_padded, + cudaStream_t stream) { + const int mmq_x = MOE_X_Q5_1; + const int mmq_y = MOE_Y_Q5_1; + const int nwarps = NWARPS_Q5_1; + + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; + const int block_num_y = (tokens_post_padded) / mmq_x; + const dim3 block_nums(block_num_x, block_num_y, 1); + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1); + + if (nrows_x % mmq_y == 0) { + constexpr bool need_check = false; + moe_q5_1<<>>( + w, + inp, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); + } else { + constexpr bool need_check = true; + moe_q5_1<<>>( + w, + inp, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); + } +} + +#if defined(USE_ROCM) +#define MOE_X_Q8_0 8 +#define MOE_Y_Q8_0 128 +#define NWARPS_Q8_0 8 +#else +#define MOE_X_Q8_0 4 +#define MOE_Y_Q8_0 32 +#define NWARPS_Q8_0 4 +#endif + +template +static __global__ void +#if defined(USE_ROCM) +__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q8_0, 2) +#endif + moe_q8_0( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int* sorted_token_ids, + const int* expert_ids, + const int* num_tokens_post_padded, + const int exp_stride, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + const int top_k) { + const int mmq_x = MOE_X_Q8_0; + const int mmq_y = MOE_Y_Q8_0; + const int nwarps = NWARPS_Q8_0; + + moe_q< + scalar_t, + QK8_0, + QR8_0, + QI8_0, + false, + block_q8_0, + mmq_x, + mmq_y, + nwarps, + allocate_tiles_q8_0, + load_tiles_q8_0, + VDR_Q8_0_Q8_1_MMQ, + vec_dot_q8_0_q8_1_mul_mat>( + vx, + vy, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); +} + +template +static void ggml_moe_q8_0_q8_1_cuda( + const void* inp, + const void* w, + scalar_t* dst, + const int* sorted_token_ids, + const int* expert_ids, + const int* num_tokens_post_padded, + const int exp_stride, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + const int top_k, + const int tokens_post_padded, + cudaStream_t stream) { + const int mmq_x = MOE_X_Q8_0; + const int mmq_y = MOE_Y_Q8_0; + const int nwarps = NWARPS_Q8_0; + + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; + const int block_num_y = (tokens_post_padded) / mmq_x; + const dim3 block_nums(block_num_x, block_num_y, 1); + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1); + + if (nrows_x % mmq_y == 0) { + constexpr bool need_check = false; + moe_q8_0<<>>( + w, + inp, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); + } else { + constexpr bool need_check = true; + moe_q8_0<<>>( + w, + inp, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); + } +} + +#if defined(USE_ROCM) +#define MOE_X_Q2_K 8 +#define MOE_Y_Q2_K 128 +#define NWARPS_Q2_K 8 +#else +#define MOE_X_Q2_K 4 +#define MOE_Y_Q2_K 32 +#define NWARPS_Q2_K 4 +#endif + +template +static __global__ void +#if defined(USE_ROCM) +__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q2_K, 2) +#endif + moe_q2_K( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int* sorted_token_ids, + const int* expert_ids, + const int* num_tokens_post_padded, + const int exp_stride, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + const int top_k) { + const int mmq_x = MOE_X_Q2_K; + const int mmq_y = MOE_Y_Q2_K; + const int nwarps = NWARPS_Q2_K; + + moe_q< + scalar_t, + QK_K, + QR2_K, + QI2_K, + false, + block_q2_K, + mmq_x, + mmq_y, + nwarps, + allocate_tiles_q2_K, + load_tiles_q2_K, + VDR_Q2_K_Q8_1_MMQ, + vec_dot_q2_K_q8_1_mul_mat>( + vx, + vy, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); +} + +template +static void ggml_moe_q2_K_q8_1_cuda( + const void* inp, + const void* w, + scalar_t* dst, + const int* sorted_token_ids, + const int* expert_ids, + const int* num_tokens_post_padded, + const int exp_stride, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + const int top_k, + const int tokens_post_padded, + cudaStream_t stream) { + const int mmq_x = MOE_X_Q2_K; + const int mmq_y = MOE_Y_Q2_K; + const int nwarps = NWARPS_Q2_K; + + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; + const int block_num_y = (tokens_post_padded) / mmq_x; + const dim3 block_nums(block_num_x, block_num_y, 1); + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1); + + if (nrows_x % mmq_y == 0) { + constexpr bool need_check = false; + moe_q2_K<<>>( + w, + inp, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); + } else { + constexpr bool need_check = true; + moe_q2_K<<>>( + w, + inp, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); + } +} + +#if defined(USE_ROCM) +#define MOE_X_Q3_K 8 +#define MOE_Y_Q3_K 128 +#define NWARPS_Q3_K 8 +#else +#define MOE_X_Q3_K 4 +#define MOE_Y_Q3_K 32 +#define NWARPS_Q3_K 4 +#endif + +template +static __global__ void +#if defined(USE_ROCM) +__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q3_K, 2) +#endif + moe_q3_K( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int* sorted_token_ids, + const int* expert_ids, + const int* num_tokens_post_padded, + const int exp_stride, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + const int top_k) { + + const int mmq_x = MOE_X_Q3_K; + const int mmq_y = MOE_Y_Q3_K; + const int nwarps = NWARPS_Q3_K; + + moe_q< + scalar_t, + QK_K, + QR3_K, + QI3_K, + false, + block_q3_K, + mmq_x, + mmq_y, + nwarps, + allocate_tiles_q3_K, + load_tiles_q3_K, + VDR_Q3_K_Q8_1_MMQ, + vec_dot_q3_K_q8_1_mul_mat>( + vx, + vy, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); +} +template +static void ggml_moe_q3_K_q8_1_cuda( + const void* inp, + const void* w, + scalar_t* dst, + const int* sorted_token_ids, + const int* expert_ids, + const int* num_tokens_post_padded, + const int exp_stride, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + const int top_k, + const int tokens_post_padded, + cudaStream_t stream) { + const int mmq_x = MOE_X_Q3_K; + const int mmq_y = MOE_Y_Q3_K; + const int nwarps = NWARPS_Q3_K; + + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; + const int block_num_y = (tokens_post_padded) / mmq_x; + const dim3 block_nums(block_num_x, block_num_y, 1); + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1); + + if (nrows_x % mmq_y == 0) { + constexpr bool need_check = false; + moe_q3_K<<>>( + w, + inp, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); + } else { + constexpr bool need_check = true; + moe_q3_K<<>>( + w, + inp, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); + } +} + +#if defined(USE_ROCM) +#define MOE_X_Q4_K 8 +#define MOE_Y_Q4_K 128 +#define NWARPS_Q4_K 8 +#else +#define MOE_X_Q4_K 4 +#define MOE_Y_Q4_K 32 +#define NWARPS_Q4_K 4 +#endif + +template +static __global__ void +#if defined(USE_ROCM) +__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_K, 2) +#endif + moe_q4_K( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int* sorted_token_ids, + const int* expert_ids, + const int* num_tokens_post_padded, + const int exp_stride, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + const int top_k) { + const int mmq_x = MOE_X_Q4_K; + const int mmq_y = MOE_Y_Q4_K; + const int nwarps = NWARPS_Q4_K; + + moe_q< + scalar_t, + QK_K, + QR4_K, + QI4_K, + true, + block_q4_K, + mmq_x, + mmq_y, + nwarps, + allocate_tiles_q4_K, + load_tiles_q4_K, + VDR_Q4_K_Q8_1_MMQ, + vec_dot_q4_K_q8_1_mul_mat>( + vx, + vy, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); +} + +template +static void ggml_moe_q4_K_q8_1_cuda( + const void* inp, + const void* w, + scalar_t* dst, + const int* sorted_token_ids, + const int* expert_ids, + const int* num_tokens_post_padded, + const int exp_stride, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + const int top_k, + const int tokens_post_padded, + cudaStream_t stream) { + const int mmq_x = MOE_X_Q4_K; + const int mmq_y = MOE_Y_Q4_K; + const int nwarps = NWARPS_Q4_K; + + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; + const int block_num_y = (tokens_post_padded) / mmq_x; + const dim3 block_nums(block_num_x, block_num_y, 1); + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1); + + if (nrows_x % mmq_y == 0) { + constexpr bool need_check = false; + moe_q4_K<<>>( + w, + inp, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); + } else { + constexpr bool need_check = true; + moe_q4_K<<>>( + w, + inp, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); + } +} + +#if defined(USE_ROCM) +#define MOE_X_Q5_K 8 +#define MOE_Y_Q5_K 128 +#define NWARPS_Q5_K 8 +#else +#define MOE_X_Q5_K 4 +#define MOE_Y_Q5_K 32 +#define NWARPS_Q5_K 4 +#endif + +template +static __global__ void +#if defined(USE_ROCM) +__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_K, 2) +#endif + moe_q5_K( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int* sorted_token_ids, + const int* expert_ids, + const int* num_tokens_post_padded, + const int exp_stride, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + const int top_k) { + const int mmq_x = MOE_X_Q5_K; + const int mmq_y = MOE_Y_Q5_K; + const int nwarps = NWARPS_Q5_K; + + moe_q< + scalar_t, + QK_K, + QR5_K, + QI5_K, + true, + block_q5_K, + mmq_x, + mmq_y, + nwarps, + allocate_tiles_q5_K, + load_tiles_q5_K, + VDR_Q5_K_Q8_1_MMQ, + vec_dot_q5_K_q8_1_mul_mat>( + vx, + vy, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); +} + +template +static void ggml_moe_q5_K_q8_1_cuda( + const void* inp, + const void* w, + scalar_t* dst, + const int* sorted_token_ids, + const int* expert_ids, + const int* num_tokens_post_padded, + const int exp_stride, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + const int top_k, + const int tokens_post_padded, + cudaStream_t stream) { + const int mmq_x = MOE_X_Q5_K; + const int mmq_y = MOE_Y_Q5_K; + const int nwarps = NWARPS_Q5_K; + + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; + const int block_num_y = (tokens_post_padded) / mmq_x; + const dim3 block_nums(block_num_x, block_num_y, 1); + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1); + + if (nrows_x % mmq_y == 0) { + constexpr bool need_check = false; + moe_q5_K<<>>( + w, + inp, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); + } else { + constexpr bool need_check = true; + moe_q5_K<<>>( + w, + inp, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); + } +} + +#if defined(USE_ROCM) +#define MOE_X_Q6_K 8 +#define MOE_Y_Q6_K 128 +#define NWARPS_Q6_K 8 +#else +#define MOE_X_Q6_K 4 +#define MOE_Y_Q6_K 32 +#define NWARPS_Q6_K 4 +#endif + +template +static __global__ void +#if defined(USE_ROCM) +__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q6_K, 2) +#endif + moe_q6_K( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int* sorted_token_ids, + const int* expert_ids, + const int* num_tokens_post_padded, + const int exp_stride, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + const int top_k) { + const int mmq_x = MOE_X_Q6_K; + const int mmq_y = MOE_Y_Q6_K; + const int nwarps = NWARPS_Q6_K; + + moe_q< + scalar_t, + QK_K, + QR6_K, + QI6_K, + false, + block_q6_K, + mmq_x, + mmq_y, + nwarps, + allocate_tiles_q6_K, + load_tiles_q6_K, + VDR_Q6_K_Q8_1_MMQ, + vec_dot_q6_K_q8_1_mul_mat>( + vx, + vy, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); +} + +template +static void ggml_moe_q6_K_q8_1_cuda( + const void* inp, + const void* w, + scalar_t* dst, + const int* sorted_token_ids, + const int* expert_ids, + const int* num_tokens_post_padded, + const int exp_stride, + const int ncols_x, + const int nrows_x, + const int ncols_y, + const int nrows_y, + const int nrows_dst, + const int top_k, + const int tokens_post_padded, + cudaStream_t stream) { + const int mmq_x = MOE_X_Q6_K; + const int mmq_y = MOE_Y_Q6_K; + const int nwarps = NWARPS_Q6_K; + + const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; + const int block_num_y = (tokens_post_padded) / mmq_x; + const dim3 block_nums(block_num_x, block_num_y, 1); + const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1); + + if (nrows_x % mmq_y == 0) { + constexpr bool need_check = false; + moe_q6_K<<>>( + w, + inp, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); + } else { + constexpr bool need_check = true; + moe_q6_K<<>>( + w, + inp, + dst, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + exp_stride, + ncols_x, + nrows_x, + ncols_y, + nrows_y, + nrows_dst, + top_k); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/moe_vec.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/moe_vec.cuh new file mode 100644 index 00000000..8cef9e08 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/moe_vec.cuh @@ -0,0 +1,413 @@ +// copied from +// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/moe_vec.cuh +// copied and adapted from +// https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmvq.cu +template +static __global__ void moe_vec_q( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int* topk_ids, + const int topk, + const int ncols, + const int nrows, + const int token_stride) { + const auto row = blockIdx.x * blockDim.y + threadIdx.y; + + const auto token = blockIdx.z / topk; + const auto expert = (topk_ids)[blockIdx.z]; + + if (row >= nrows) { + return; + } + + const int blocks_per_row = ncols / qk; + const int blocks_per_warp = vdr * WARP_SIZE / qi; + + // partial sum for each thread + float tmp = 0.0f; + + const block_q_t* x = ((const block_q_t*)vx) + expert * nrows * blocks_per_row; + const block_q8_1* y = (const block_q8_1*)(((const int*)vy) + token * token_stride); + + for (auto i = threadIdx.x / (qi / vdr); i < blocks_per_row; i += blocks_per_warp) { + const int ibx = row * blocks_per_row + i; // x block index + + const int iby = i * (qk / QK8_1); // y block index that aligns with ibx + + const int iqs = vdr * (threadIdx.x % (qi / vdr)); // x block quant index when casting the quants to int + + tmp += vec_dot_q_cuda(&x[ibx], &y[iby], iqs); + } + + // sum up partial sums and write back result +#pragma unroll + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { + tmp += SGLANG_SHFL_XOR_SYNC(uint32_t(-1), tmp, mask); + } + + if (threadIdx.x == 0) { + dst[blockIdx.z * nrows + row] = tmp; + } +} + +template +static void moe_vec_q4_0_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int* topk_ids, + const int top_k, + const int tokens, + const int ncols, + const int nrows, + const int token_stride, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, 1, tokens * top_k); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + moe_vec_q + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); +} + +template +static void moe_vec_q4_1_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int* topk_ids, + const int top_k, + const int tokens, + const int ncols, + const int nrows, + const int token_stride, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, 1, tokens * top_k); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + moe_vec_q + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); +} + +template +static void moe_vec_q5_0_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int* topk_ids, + const int top_k, + const int tokens, + const int ncols, + const int nrows, + const int token_stride, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, 1, tokens * top_k); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + moe_vec_q + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); +} + +template +static void moe_vec_q5_1_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int* topk_ids, + const int top_k, + const int tokens, + const int ncols, + const int nrows, + const int token_stride, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, 1, tokens * top_k); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + moe_vec_q + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); +} + +template +static void moe_vec_q8_0_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int* topk_ids, + const int top_k, + const int tokens, + const int ncols, + const int nrows, + const int token_stride, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, 1, tokens * top_k); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + moe_vec_q + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); +} + +template +static void moe_vec_q2_K_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int* topk_ids, + const int top_k, + const int tokens, + const int ncols, + const int nrows, + const int token_stride, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, 1, tokens * top_k); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + moe_vec_q + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); +} + +template +static void moe_vec_q3_K_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int* topk_ids, + const int top_k, + const int tokens, + const int ncols, + const int nrows, + const int token_stride, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, 1, tokens * top_k); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + moe_vec_q + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); +} + +template +static void moe_vec_q4_K_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int* topk_ids, + const int top_k, + const int tokens, + const int ncols, + const int nrows, + const int token_stride, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, 1, tokens * top_k); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + moe_vec_q + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); +} + +template +static void moe_vec_q5_K_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int* topk_ids, + const int top_k, + const int tokens, + const int ncols, + const int nrows, + const int token_stride, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, 1, tokens * top_k); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + moe_vec_q + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); +} + +template +static void moe_vec_q6_K_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int* topk_ids, + const int top_k, + const int tokens, + const int ncols, + const int nrows, + const int token_stride, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, 1, tokens * top_k); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + moe_vec_q + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); +} + +template +static void moe_vec_iq2_xxs_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int* topk_ids, + const int top_k, + const int tokens, + const int ncols, + const int nrows, + const int token_stride, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, 1, tokens * top_k); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + moe_vec_q + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); +} + +template +static void moe_vec_iq2_xs_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int* topk_ids, + const int top_k, + const int tokens, + const int ncols, + const int nrows, + const int token_stride, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, 1, tokens * top_k); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + moe_vec_q + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); +} + +template +static void moe_vec_iq2_s_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int* topk_ids, + const int top_k, + const int tokens, + const int ncols, + const int nrows, + const int token_stride, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, 1, tokens * top_k); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + moe_vec_q + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); +} + +template +static void moe_vec_iq3_xxs_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int* topk_ids, + const int top_k, + const int tokens, + const int ncols, + const int nrows, + const int token_stride, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, 1, tokens * top_k); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + moe_vec_q + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); +} + +template +static void moe_vec_iq1_s_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int* topk_ids, + const int top_k, + const int tokens, + const int ncols, + const int nrows, + const int token_stride, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, 1, tokens * top_k); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + moe_vec_q + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); +} + +template +static void moe_vec_iq1_m_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int* topk_ids, + const int top_k, + const int tokens, + const int ncols, + const int nrows, + const int token_stride, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, 1, tokens * top_k); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + moe_vec_q + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); +} + +template +static void moe_vec_iq4_nl_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int* topk_ids, + const int top_k, + const int tokens, + const int ncols, + const int nrows, + const int token_stride, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, 1, tokens * top_k); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + moe_vec_q + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); +} + +template +static void moe_vec_iq4_xs_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int* topk_ids, + const int top_k, + const int tokens, + const int ncols, + const int nrows, + const int token_stride, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, 1, tokens * top_k); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + moe_vec_q + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); +} + +template +static void moe_vec_iq3_s_q8_1_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int* topk_ids, + const int top_k, + const int tokens, + const int ncols, + const int nrows, + const int token_stride, + cudaStream_t stream) { + const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + const dim3 block_nums(block_num_y, 1, tokens * top_k); + const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); + moe_vec_q + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/vecdotq.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/vecdotq.cuh new file mode 100644 index 00000000..933c2aa5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/quantization/gguf/vecdotq.cuh @@ -0,0 +1,2037 @@ +// copied from +// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/vecdotq.cuh +// copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/vecdotq.cuh +// and https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmq.cu +static __device__ __forceinline__ int get_int_b2(const void* x, const int& i32) { + const uint16_t* x16 = (const uint16_t*)x; // assume at least 2 byte alignment + + int x32 = x16[2 * i32 + 0] << 0; + x32 |= x16[2 * i32 + 1] << 16; + + return x32; +} + +static __device__ __forceinline__ int get_int_b4(const void* x, const int& i32) { + return ((const int*)x)[i32]; // assume at least 4 byte alignment +} + +static __device__ __forceinline__ int get_int_from_int8(const int8_t* x8, const int& i32) { + const uint16_t* x16 = (const uint16_t*)(x8 + sizeof(int) * i32); // assume at least 2 byte alignment + int x32 = 0; + x32 |= x16[0] << 0; + x32 |= x16[1] << 16; + return x32; +} + +static __device__ __forceinline__ int get_int_from_uint8(const uint8_t* x8, const int& i32) { + const uint16_t* x16 = (const uint16_t*)(x8 + sizeof(int) * i32); // assume at least 2 byte alignment + int x32 = 0; + x32 |= x16[0] << 0; + x32 |= x16[1] << 16; + return x32; +} + +static __device__ __forceinline__ int get_int_from_int8_aligned(const int8_t* x8, const int& i32) { + return *((const int*)(x8 + sizeof(int) * i32)); // assume at least 4 byte alignment +} + +static __device__ __forceinline__ int get_int_from_uint8_aligned(const uint8_t* x8, const int& i32) { + return *((const int*)(x8 + sizeof(int) * i32)); // assume at least 4 byte alignment +} + +// VDR = vec dot ratio, how many contiguous integers each thread processes when the vec dot kernel is called +// MMVQ = mul_mat_vec_q, MMQ = mul_mat_q + +#define VDR_Q4_0_Q8_1_MMVQ 2 +#define VDR_Q4_0_Q8_1_MMQ 4 + +template +static __device__ __forceinline__ float +vec_dot_q4_0_q8_1_impl(const int* v, const int* u, const float& d4, const half2& ds8) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM + int sumi = 0; + +#pragma unroll + for (int i = 0; i < vdr; ++i) { + const int vi0 = (v[i] >> 0) & 0x0F0F0F0F; + const int vi1 = (v[i] >> 4) & 0x0F0F0F0F; + + // SIMD dot product of quantized values + sumi = __dp4a(vi0, u[2 * i + 0], sumi); + sumi = __dp4a(vi1, u[2 * i + 1], sumi); + } + + const float2 ds8f = __half22float2(ds8); + + // second part effectively subtracts 8 from each quant value + return d4 * (sumi * ds8f.x - (8 * vdr / QI4_0) * ds8f.y); +#endif +} + +#define VDR_Q4_1_Q8_1_MMVQ 2 +#define VDR_Q4_1_Q8_1_MMQ 4 + +template +static __device__ __forceinline__ float +vec_dot_q4_1_q8_1_impl(const int* v, const int* u, const half2& dm4, const half2& ds8) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM + int sumi = 0; + +#pragma unroll + for (int i = 0; i < vdr; ++i) { + const int vi0 = (v[i] >> 0) & 0x0F0F0F0F; + const int vi1 = (v[i] >> 4) & 0x0F0F0F0F; + + // SIMD dot product of quantized values + sumi = __dp4a(vi0, u[2 * i + 0], sumi); + sumi = __dp4a(vi1, u[2 * i + 1], sumi); + } + + const float2 tmp = __half22float2(__hmul2(dm4, ds8)); + const float d4d8 = tmp.x; + const float m4s8 = tmp.y; + + // scale second part of sum by QI8_1/(vdr * QR4_1) to compensate for multiple threads adding it + return sumi * d4d8 + m4s8 / (QI8_1 / (vdr * QR4_1)); +#endif +} + +#define VDR_Q5_0_Q8_1_MMVQ 2 +#define VDR_Q5_0_Q8_1_MMQ 4 + +template +static __device__ __forceinline__ float +vec_dot_q5_0_q8_1_impl(const int* vl, const int* vh, const int* u, const float& d5, const half2& ds8) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM + int sumi = 0; + +#pragma unroll + for (int i = 0; i < vdr; ++i) { + int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits + vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4 + vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12 + vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20 + vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28 + sumi = __dp4a(vi0, u[2 * i + 0], sumi); // SIMD dot product of quantized values + + int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits + vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4 + vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12 + vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20 + vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28 + sumi = __dp4a(vi1, u[2 * i + 1], sumi); // SIMD dot product of quantized values + } + + const float2 ds8f = __half22float2(ds8); + + // second part effectively subtracts 16 from each quant value + return d5 * (sumi * ds8f.x - (16 * vdr / QI5_0) * ds8f.y); +#endif +} + +#define VDR_Q5_1_Q8_1_MMVQ 2 +#define VDR_Q5_1_Q8_1_MMQ 4 + +template +static __device__ __forceinline__ float +vec_dot_q5_1_q8_1_impl(const int* vl, const int* vh, const int* u, const half2& dm5, const half2& ds8) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM + int sumi = 0; + +#pragma unroll + for (int i = 0; i < vdr; ++i) { + int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits + vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4 + vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12 + vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20 + vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28 + sumi = __dp4a(vi0, u[2 * i + 0], sumi); // SIMD dot product of quantized values + + int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits + vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4 + vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12 + vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20 + vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28 + sumi = __dp4a(vi1, u[2 * i + 1], sumi); // SIMD dot product of quantized values + } + + const float2 tmp = __half22float2(__hmul2(dm5, ds8)); + const float d5d8 = tmp.x; + const float m5s8 = tmp.y; + + // scale second part of sum by QI5_1 / vdr to compensate for multiple threads adding it + return sumi * d5d8 + m5s8 / (QI5_1 / vdr); +#endif +} + +#define VDR_Q8_0_Q8_1_MMVQ 2 +#define VDR_Q8_0_Q8_1_MMQ 8 + +template +static __device__ __forceinline__ float +vec_dot_q8_0_q8_1_impl(const int* v, const int* u, const float& d8_0, const float& d8_1) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM + int sumi = 0; + +#pragma unroll + for (int i = 0; i < vdr; ++i) { + // SIMD dot product of quantized values + sumi = __dp4a(v[i], u[i], sumi); + } + return d8_0 * d8_1 * sumi; +#endif +} + +template +static __device__ __forceinline__ float +vec_dot_q8_1_q8_1_impl(const int* v, const int* u, const half2& dm8, const half2& ds8) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM + + int sumi = 0; + +#pragma unroll + for (int i = 0; i < vdr; ++i) { + // SIMD dot product of quantized values + sumi = __dp4a(v[i], u[i], sumi); + } + + const float2 tmp = __half22float2(__hmul2(dm8, ds8)); + const float d8d8 = tmp.x; + const float m8s8 = tmp.y; + + // scale second part of sum by QI8_1/ vdr to compensate for multiple threads adding it + return sumi * d8d8 + m8s8 / (QI8_1 / vdr); +#endif +} + +#define VDR_Q2_K_Q8_1_MMVQ 1 +#define VDR_Q2_K_Q8_1_MMQ 2 + +// contiguous v/x values +static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq( + const int& v, + const int* __restrict__ u, + const uint8_t* __restrict__ scales, + const half2& dm2, + const float* __restrict__ d8) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM + float sumf_d = 0.0f; + float sumf_m = 0.0f; + +#pragma unroll + for (int i = 0; i < QR2_K; ++i) { + const int sc = scales[2 * i]; + + const int vi = (v >> (2 * i)) & 0x03030303; + + sumf_d += d8[i] * (__dp4a(vi, u[i], 0) * (sc & 0xF)); // SIMD dot product + + // fill int with 4x m + int m = sc >> 4; + m |= m << 8; + m |= m << 16; + sumf_m += d8[i] * __dp4a(m, u[i], 0); // multiply constant q2_K part with sum of q8_1 values + } + + const float2 dm2f = __half22float2(dm2); + + return dm2f.x * sumf_d - dm2f.y * sumf_m; +#endif +} + +static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq( + const int* __restrict__ v, + const int* __restrict__ u, + const uint8_t* __restrict__ scales, + const half2& dm2, + const float& d8) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM + int sumi_d = 0; + int sumi_m = 0; + +#pragma unroll + for (int i0 = 0; i0 < QI8_1; i0 += QI8_1 / 2) { + int sumi_d_sc = 0; + + const int sc = scales[i0 / (QI8_1 / 2)]; + + // fill int with 4x m + int m = sc >> 4; + m |= m << 8; + m |= m << 16; + +#pragma unroll + for (int i = i0; i < i0 + QI8_1 / 2; ++i) { + sumi_d_sc = __dp4a(v[i], u[i], sumi_d_sc); // SIMD dot product + sumi_m = __dp4a(m, u[i], sumi_m); // multiply sum of q8_1 values with m + } + + sumi_d += sumi_d_sc * (sc & 0xF); + } + + const float2 dm2f = __half22float2(dm2); + + return d8 * (dm2f.x * sumi_d - dm2f.y * sumi_m); +#endif +} + +#define VDR_Q3_K_Q8_1_MMVQ 1 +#define VDR_Q3_K_Q8_1_MMQ 2 + +// contiguous v/x values +static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq( + const int& vl, + const int& vh, + const int* __restrict__ u, + const uint8_t* __restrict__ scales, + const int& scale_offset, + const float& d3, + const float* __restrict__ d8) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM + + float sumf = 0.0f; + +#pragma unroll + for (int i = 0; i < QR3_K; ++i) { + const int isc = scale_offset + 2 * i; + + const int isc_low = isc % (QK_K / 32); + const int sc_shift_low = 4 * (isc / (QK_K / 32)); + const int sc_low = (scales[isc_low] >> sc_shift_low) & 0xF; + + const int isc_high = isc % (QK_K / 64); + const int sc_shift_high = 2 * (isc / (QK_K / 64)); + const int sc_high = ((scales[(QK_K / 32) + isc_high] >> sc_shift_high) & 3) << 4; + + const int sc = (sc_low | sc_high) - 32; + + const int vil = (vl >> (2 * i)) & 0x03030303; + + const int vih = ((vh >> i) << 2) & 0x04040404; + + const int vi = __vsubss4(vil, vih); + + sumf += d8[i] * (__dp4a(vi, u[i], 0) * sc); // SIMD dot product + } + + return d3 * sumf; +#endif +} + +static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq( + const int* __restrict__ v, + const int* __restrict__ u, + const int8_t* __restrict__ scales, + const float& d3, + const float& d8) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM + int sumi = 0; + +#pragma unroll + for (int i0 = 0; i0 < QR3_K * VDR_Q3_K_Q8_1_MMQ; i0 += QI8_1 / 2) { + int sumi_sc = 0; + + for (int i = i0; i < i0 + QI8_1 / 2; ++i) { + sumi_sc = __dp4a(v[i], u[i], sumi_sc); // SIMD dot product + } + + sumi += sumi_sc * scales[i0 / (QI8_1 / 2)]; + } + + return d3 * d8 * sumi; +#endif +} + +#define VDR_Q4_K_Q8_1_MMVQ 2 +#define VDR_Q4_K_Q8_1_MMQ 8 + +// contiguous v/x values +static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq( + const int* __restrict__ v, + const int* __restrict__ u, + const uint8_t* __restrict__ sc, + const uint8_t* __restrict__ m, + const half2& dm4, + const float* __restrict__ d8) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM + + float sumf_d = 0.0f; + float sumf_m = 0.0f; + +#pragma unroll + for (int i = 0; i < QR4_K; ++i) { + const int v0i = (v[0] >> (4 * i)) & 0x0F0F0F0F; + const int v1i = (v[1] >> (4 * i)) & 0x0F0F0F0F; + + const int dot1 = __dp4a(v1i, u[2 * i + 1], __dp4a(v0i, u[2 * i + 0], 0)); // SIMD dot product + const int dot2 = __dp4a(0x01010101, u[2 * i + 1], __dp4a(0x01010101, u[2 * i + 0], 0)); // sum of u + + sumf_d += d8[i] * (dot1 * sc[i]); + sumf_m += d8[i] * (dot2 * m[i]); // multiply constant part of q4_K with sum of q8_1 values + } + + const float2 dm4f = __half22float2(dm4); + return dm4f.x * sumf_d - dm4f.y * sumf_m; +#endif +} + +static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_mmq( + const int* __restrict__ v, + const int* __restrict__ u, + const uint8_t* __restrict__ sc, + const uint8_t* __restrict__ m, + const half2& dm4, + const half2* __restrict__ ds8) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM + float sumf_d = 0.0f; + float sumf_m = 0.0f; + +#pragma unroll + for (int i = 0; i < QR4_K * VDR_Q4_K_Q8_1_MMQ / QI8_1; ++i) { + int sumi_d = 0; + +#pragma unroll + for (int j = 0; j < QI8_1; ++j) { + sumi_d = __dp4a((v[j] >> (4 * i)) & 0x0F0F0F0F, u[i * QI8_1 + j], sumi_d); // SIMD dot product + } + + const float2 ds8f = __half22float2(ds8[i]); + + sumf_d += ds8f.x * (sc[i] * sumi_d); + sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val + } + + const float2 dm4f = __half22float2(dm4); + + return dm4f.x * sumf_d - dm4f.y * sumf_m; +#endif +} + +#define VDR_Q5_K_Q8_1_MMVQ 2 +#define VDR_Q5_K_Q8_1_MMQ 8 + +static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_vmmq( + const int* __restrict__ vl, + const int* __restrict__ vh, + const int* __restrict__ u, + const uint8_t* __restrict__ sc, + const uint8_t* __restrict__ m, + const half2& dm5, + const float* __restrict__ d8) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM + + float sumf_d = 0.0f; + float sumf_m = 0.0f; + +#pragma unroll + for (int i = 0; i < QR5_K; ++i) { + const int vl0i = (vl[0] >> (4 * i)) & 0x0F0F0F0F; + const int vl1i = (vl[1] >> (4 * i)) & 0x0F0F0F0F; + + const int vh0i = ((vh[0] >> i) << 4) & 0x10101010; + const int vh1i = ((vh[1] >> i) << 4) & 0x10101010; + + const int v0i = vl0i | vh0i; + const int v1i = vl1i | vh1i; + + const int dot1 = __dp4a(v0i, u[2 * i + 0], __dp4a(v1i, u[2 * i + 1], 0)); // SIMD dot product + const int dot2 = __dp4a(0x01010101, u[2 * i + 0], __dp4a(0x01010101, u[2 * i + 1], 0)); // sum of u + + sumf_d += d8[i] * (dot1 * sc[i]); + sumf_m += d8[i] * (dot2 * m[i]); + } + + const float2 dm5f = __half22float2(dm5); + return dm5f.x * sumf_d - dm5f.y * sumf_m; +#endif +} + +static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq( + const int* __restrict__ v, + const int* __restrict__ u, + const uint8_t* __restrict__ sc, + const uint8_t* __restrict__ m, + const half2& dm4, + const half2* __restrict__ ds8) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM + float sumf_d = 0.0f; + float sumf_m = 0.0f; + +#pragma unroll + for (int i = 0; i < QR5_K * VDR_Q5_K_Q8_1_MMQ / QI8_1; ++i) { + int sumi_d = 0; + +#pragma unroll + for (int j = 0; j < QI8_1; ++j) { + sumi_d = __dp4a(v[i * QI8_1 + j], u[i * QI8_1 + j], sumi_d); // SIMD dot product + } + + const float2 ds8f = __half22float2(ds8[i]); + + sumf_d += ds8f.x * (sc[i] * sumi_d); + sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val + } + + const float2 dm4f = __half22float2(dm4); + + return dm4f.x * sumf_d - dm4f.y * sumf_m; +#endif +} + +#define VDR_Q6_K_Q8_1_MMVQ 1 +#define VDR_Q6_K_Q8_1_MMQ 8 + +// contiguous v/x values +static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq( + const int& vl, + const int& vh, + const int* __restrict__ u, + const int8_t* __restrict__ scales, + const float& d, + const float* __restrict__ d8) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM + float sumf = 0.0f; + +#pragma unroll + for (int i = 0; i < QR6_K; ++i) { + const int sc = scales[4 * i]; + const int vil = (vl >> (4 * i)) & 0x0F0F0F0F; + const int vih = ((vh >> (4 * i)) << 4) & 0x30303030; + const int vi = __vsubss4((vil | vih), 0x20202020); // vi = (vil | vih) - 32 + + sumf += d8[i] * (__dp4a(vi, u[i], 0) * sc); // SIMD dot product + } + + return d * sumf; +#endif +} + +static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmq( + const int* __restrict__ v, + const int* __restrict__ u, + const int8_t* __restrict__ sc, + const float& d6, + const float* __restrict__ d8) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM + float sumf_d = 0.0f; + +#pragma unroll + for (int i0 = 0; i0 < VDR_Q6_K_Q8_1_MMQ; i0 += 4) { + int2 sumi_d = {0, 0}; // 2 q6_K scales per q8_1 scale + +#pragma unroll + for (int i = i0; i < i0 + 2; ++i) { + sumi_d.x = __dp4a(v[2 * i + 0], u[2 * i + 0], sumi_d.x); // SIMD dot product + sumi_d.x = __dp4a(v[2 * i + 1], u[2 * i + 1], sumi_d.x); // SIMD dot product + + sumi_d.y = __dp4a(v[2 * i + 4], u[2 * i + 4], sumi_d.y); // SIMD dot product + sumi_d.y = __dp4a(v[2 * i + 5], u[2 * i + 5], sumi_d.y); // SIMD dot product + } + + sumf_d += d8[i0 / 4] * (sc[i0 / 2 + 0] * sumi_d.x + sc[i0 / 2 + 1] * sumi_d.y); + } + + return d6 * sumf_d; +#endif +} + +static __device__ __forceinline__ float +vec_dot_q4_0_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) { + const block_q4_0* bq4_0 = (const block_q4_0*)vbq; + + int v[VDR_Q4_0_Q8_1_MMVQ]; + int u[2 * VDR_Q4_0_Q8_1_MMVQ]; + +#pragma unroll + for (int i = 0; i < VDR_Q4_0_Q8_1_MMVQ; ++i) { + v[i] = get_int_from_uint8(bq4_0->qs, iqs + i); + u[2 * i + 0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i); + u[2 * i + 1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI4_0); + } + + return vec_dot_q4_0_q8_1_impl(v, u, __half2float(bq4_0->d), bq8_1->ds); +} + +template +static __device__ __forceinline__ void allocate_tiles_q4_0(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) { + __shared__ int tile_x_qs[mmq_y * (WARP_SIZE_GGUF) + mmq_y]; + __shared__ float tile_x_d[mmq_y * (WARP_SIZE_GGUF / QI4_0) + mmq_y / QI4_0]; + *x_ql = tile_x_qs; + *x_dm = (half2*)tile_x_d; +} + +template +static __device__ __forceinline__ void load_tiles_q4_0( + const void* __restrict__ vx, + int* __restrict__ x_ql, + half2* __restrict__ x_dm, + int* __restrict__ x_qh, + int* __restrict__ x_sc, + const int& i_offset, + const int& i_max, + const int& k, + const int& blocks_per_row) { + const int kbx = k / QI4_0; + const int kqsx = k % QI4_0; + + const block_q4_0* bx0 = (const block_q4_0*)vx; + float* x_dmf = (float*)x_dm; + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps) { + int i = i0 + i_offset; + if (need_check) { + i = min(i, i_max); + } + const block_q4_0* bxi = bx0 + i * blocks_per_row + kbx; + x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8(bxi->qs, kqsx); + // x_dmf[i * (WARP_SIZE_GGUF/QI4_0) + i / QI4_0 + kbx] = bxi->d; + } + + const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI4_0; + const int kbxd = k % blocks_per_tile_x_row; + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_0) { + int i = i0 + i_offset * QI4_0 + k / blocks_per_tile_x_row; + if (need_check) { + i = min(i, i_max); + } + const block_q4_0* bxi = bx0 + i * blocks_per_row + kbxd; + x_dmf[i * (WARP_SIZE_GGUF / QI4_0) + i / QI4_0 + kbxd] = __half2float(bxi->d); + } +} + +static __device__ __forceinline__ float vec_dot_q4_0_q8_1_mul_mat( + const int* __restrict__ x_ql, + const half2* __restrict__ x_dm, + const int* __restrict__ x_qh, + const int* __restrict__ x_sc, + const int* __restrict__ y_qs, + const half2* __restrict__ y_ds, + const int& i, + const int& j, + const int& k) { + (void)x_qh; + (void)x_sc; + + const int kyqs = k % (QI8_1 / 2) + QI8_1 * (k / (QI8_1 / 2)); + const float* x_dmf = (const float*)x_dm; + + int u[2 * VDR_Q4_0_Q8_1_MMQ]; + +#pragma unroll + for (int l = 0; l < VDR_Q4_0_Q8_1_MMQ; ++l) { + u[2 * l + 0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF]; + u[2 * l + 1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI4_0) % WARP_SIZE_GGUF]; + } + + return vec_dot_q4_0_q8_1_impl( + &x_ql[i * (WARP_SIZE_GGUF + 1) + k], + u, + x_dmf[i * (WARP_SIZE_GGUF / QI4_0) + i / QI4_0 + k / QI4_0], + y_ds[j * (WARP_SIZE_GGUF / QI8_1) + (2 * k / QI8_1) % (WARP_SIZE_GGUF / QI8_1)]); +} + +static __device__ __forceinline__ float +vec_dot_q4_1_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) { + const block_q4_1* bq4_1 = (const block_q4_1*)vbq; + + int v[VDR_Q4_1_Q8_1_MMVQ]; + int u[2 * VDR_Q4_1_Q8_1_MMVQ]; + +#pragma unroll + for (int i = 0; i < VDR_Q4_1_Q8_1_MMVQ; ++i) { + v[i] = get_int_from_uint8_aligned(bq4_1->qs, iqs + i); + u[2 * i + 0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i); + u[2 * i + 1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI4_1); + } + + return vec_dot_q4_1_q8_1_impl(v, u, bq4_1->dm, bq8_1->ds); +} + +template +static __device__ __forceinline__ void allocate_tiles_q4_1(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) { + __shared__ int tile_x_qs[mmq_y * (WARP_SIZE_GGUF) + +mmq_y]; + __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF / QI4_1) + mmq_y / QI4_1]; + *x_ql = tile_x_qs; + *x_dm = tile_x_dm; +} + +template +static __device__ __forceinline__ void load_tiles_q4_1( + const void* __restrict__ vx, + int* __restrict__ x_ql, + half2* __restrict__ x_dm, + int* __restrict__ x_qh, + int* __restrict__ x_sc, + const int& i_offset, + const int& i_max, + const int& k, + const int& blocks_per_row) { + const int kbx = k / QI4_1; + const int kqsx = k % QI4_1; + + const block_q4_1* bx0 = (const block_q4_1*)vx; + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps) { + int i = i0 + i_offset; + if (need_check) { + i = min(i, i_max); + } + const block_q4_1* bxi = bx0 + i * blocks_per_row + kbx; + x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx); + } + + const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI4_1; + const int kbxd = k % blocks_per_tile_x_row; + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_1) { + int i = i0 + i_offset * QI4_1 + k / blocks_per_tile_x_row; + if (need_check) { + i = min(i, i_max); + } + const block_q4_1* bxi = bx0 + i * blocks_per_row + kbxd; + x_dm[i * (WARP_SIZE_GGUF / QI4_1) + i / QI4_1 + kbxd] = bxi->dm; + } +} + +static __device__ __forceinline__ float vec_dot_q4_1_q8_1_mul_mat( + const int* __restrict__ x_ql, + const half2* __restrict__ x_dm, + const int* __restrict__ x_qh, + const int* __restrict__ x_sc, + const int* __restrict__ y_qs, + const half2* __restrict__ y_ds, + const int& i, + const int& j, + const int& k) { + const int kyqs = k % (QI8_1 / 2) + QI8_1 * (k / (QI8_1 / 2)); + + int u[2 * VDR_Q4_1_Q8_1_MMQ]; + +#pragma unroll + for (int l = 0; l < VDR_Q4_1_Q8_1_MMQ; ++l) { + u[2 * l + 0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF]; + u[2 * l + 1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI4_1) % WARP_SIZE_GGUF]; + } + + return vec_dot_q4_1_q8_1_impl( + &x_ql[i * (WARP_SIZE_GGUF + 1) + k], + u, + x_dm[i * (WARP_SIZE_GGUF / QI4_1) + i / QI4_1 + k / QI4_1], + y_ds[j * (WARP_SIZE_GGUF / QI8_1) + (2 * k / QI8_1) % (WARP_SIZE_GGUF / QI8_1)]); +} + +static __device__ __forceinline__ float +vec_dot_q5_0_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) { + const block_q5_0* bq5_0 = (const block_q5_0*)vbq; + + int vl[VDR_Q5_0_Q8_1_MMVQ]; + int vh[VDR_Q5_0_Q8_1_MMVQ]; + int u[2 * VDR_Q5_0_Q8_1_MMVQ]; + +#pragma unroll + for (int i = 0; i < VDR_Q5_0_Q8_1_MMVQ; ++i) { + vl[i] = get_int_from_uint8(bq5_0->qs, iqs + i); + vh[i] = get_int_from_uint8(bq5_0->qh, 0) >> (4 * (iqs + i)); + u[2 * i + 0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i); + u[2 * i + 1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI5_0); + } + + return vec_dot_q5_0_q8_1_impl(vl, vh, u, __half2float(bq5_0->d), bq8_1->ds); +} + +template +static __device__ __forceinline__ void allocate_tiles_q5_0(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) { + __shared__ int tile_x_ql[mmq_y * (2 * WARP_SIZE_GGUF) + mmq_y]; + __shared__ float tile_x_d[mmq_y * (WARP_SIZE_GGUF / QI5_0) + mmq_y / QI5_0]; + + *x_ql = tile_x_ql; + *x_dm = (half2*)tile_x_d; +} + +template +static __device__ __forceinline__ void load_tiles_q5_0( + const void* __restrict__ vx, + int* __restrict__ x_ql, + half2* __restrict__ x_dm, + int* __restrict__ x_qh, + int* __restrict__ x_sc, + const int& i_offset, + const int& i_max, + const int& k, + const int& blocks_per_row) { + const int kbx = k / QI5_0; + const int kqsx = k % QI5_0; + + const block_q5_0* bx0 = (const block_q5_0*)vx; + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps) { + int i = i0 + i_offset; + + if (need_check) { + i = min(i, i_max); + } + const block_q5_0* bxi = bx0 + i * blocks_per_row + kbx; + const int ql = get_int_from_uint8(bxi->qs, kqsx); + const int qh = get_int_from_uint8(bxi->qh, 0) >> (4 * (k % QI5_0)); + + int qs0 = (ql >> 0) & 0x0F0F0F0F; + qs0 |= (qh << 4) & 0x00000010; // 0 -> 4 + qs0 |= (qh << 11) & 0x00001000; // 1 -> 12 + qs0 |= (qh << 18) & 0x00100000; // 2 -> 20 + qs0 |= (qh << 25) & 0x10000000; // 3 -> 28 + qs0 = __vsubss4(qs0, 0x10101010); // subtract 16 + + x_ql[i * (2 * WARP_SIZE_GGUF + 1) + 2 * k + 0] = qs0; + + int qs1 = (ql >> 4) & 0x0F0F0F0F; + qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4 + qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12 + qs1 |= (qh << 2) & 0x00100000; // 18 -> 20 + qs1 |= (qh << 9) & 0x10000000; // 19 -> 28 + qs1 = __vsubss4(qs1, 0x10101010); // subtract 16 + + x_ql[i * (2 * WARP_SIZE_GGUF + 1) + 2 * k + 1] = qs1; + } + + const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI5_0; + const int kbxd = k % blocks_per_tile_x_row; + float* x_dmf = (float*)x_dm; + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_0) { + int i = i0 + i_offset * QI5_0 + k / blocks_per_tile_x_row; + + if (need_check) { + i = min(i, i_max); + } + + const block_q5_0* bxi = bx0 + i * blocks_per_row + kbxd; + x_dmf[i * (WARP_SIZE_GGUF / QI5_0) + i / QI5_0 + kbxd] = __half2float(bxi->d); + } +} + +static __device__ __forceinline__ float vec_dot_q5_0_q8_1_mul_mat( + const int* __restrict__ x_ql, + const half2* __restrict__ x_dm, + const int* __restrict__ x_qh, + const int* __restrict__ x_sc, + const int* __restrict__ y_qs, + const half2* __restrict__ y_ds, + const int& i, + const int& j, + const int& k) { + const int kyqs = k % (QI8_1 / 2) + QI8_1 * (k / (QI8_1 / 2)); + const int index_bx = i * (WARP_SIZE_GGUF / QI5_0) + i / QI5_0 + k / QI5_0; + const float* x_dmf = (const float*)x_dm; + const float* y_df = (const float*)y_ds; + + int u[2 * VDR_Q5_0_Q8_1_MMQ]; + +#pragma unroll + for (int l = 0; l < VDR_Q5_0_Q8_1_MMQ; ++l) { + u[2 * l + 0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF]; + u[2 * l + 1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI5_0) % WARP_SIZE_GGUF]; + } + + return vec_dot_q8_0_q8_1_impl( + &x_ql[i * (2 * WARP_SIZE_GGUF + 1) + 2 * k], + u, + x_dmf[index_bx], + y_df[j * (WARP_SIZE_GGUF / QI8_1) + (2 * k / QI8_1) % (WARP_SIZE_GGUF / QI8_1)]); +} + +static __device__ __forceinline__ float +vec_dot_q5_1_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) { + const block_q5_1* bq5_1 = (const block_q5_1*)vbq; + + int vl[VDR_Q5_1_Q8_1_MMVQ]; + int vh[VDR_Q5_1_Q8_1_MMVQ]; + int u[2 * VDR_Q5_1_Q8_1_MMVQ]; + +#pragma unroll + for (int i = 0; i < VDR_Q5_1_Q8_1_MMVQ; ++i) { + vl[i] = get_int_from_uint8_aligned(bq5_1->qs, iqs + i); + vh[i] = get_int_from_uint8_aligned(bq5_1->qh, 0) >> (4 * (iqs + i)); + u[2 * i + 0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i); + u[2 * i + 1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI5_1); + } + + return vec_dot_q5_1_q8_1_impl(vl, vh, u, bq5_1->dm, bq8_1->ds); +} + +template +static __device__ __forceinline__ void allocate_tiles_q5_1(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) { + __shared__ int tile_x_ql[mmq_y * (2 * WARP_SIZE_GGUF) + mmq_y]; + __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF / QI5_1) + mmq_y / QI5_1]; + + *x_ql = tile_x_ql; + *x_dm = tile_x_dm; +} + +template +static __device__ __forceinline__ void load_tiles_q5_1( + const void* __restrict__ vx, + int* __restrict__ x_ql, + half2* __restrict__ x_dm, + int* __restrict__ x_qh, + int* __restrict__ x_sc, + const int& i_offset, + const int& i_max, + const int& k, + const int& blocks_per_row) { + const int kbx = k / QI5_1; + const int kqsx = k % QI5_1; + + const block_q5_1* bx0 = (const block_q5_1*)vx; + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps) { + int i = i0 + i_offset; + + if (need_check) { + i = min(i, i_max); + } + + const block_q5_1* bxi = bx0 + i * blocks_per_row + kbx; + + const int ql = get_int_from_uint8_aligned(bxi->qs, kqsx); + const int qh = get_int_from_uint8_aligned(bxi->qh, 0) >> (4 * (k % QI5_1)); + + int qs0 = (ql >> 0) & 0x0F0F0F0F; + qs0 |= (qh << 4) & 0x00000010; // 0 -> 4 + qs0 |= (qh << 11) & 0x00001000; // 1 -> 12 + qs0 |= (qh << 18) & 0x00100000; // 2 -> 20 + qs0 |= (qh << 25) & 0x10000000; // 3 -> 28 + + x_ql[i * (2 * WARP_SIZE_GGUF + 1) + 2 * k + 0] = qs0; + + int qs1 = (ql >> 4) & 0x0F0F0F0F; + qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4 + qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12 + qs1 |= (qh << 2) & 0x00100000; // 18 -> 20 + qs1 |= (qh << 9) & 0x10000000; // 19 -> 28 + + x_ql[i * (2 * WARP_SIZE_GGUF + 1) + 2 * k + 1] = qs1; + } + + const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI5_1; + const int kbxd = k % blocks_per_tile_x_row; + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_1) { + int i = i0 + i_offset * QI5_1 + k / blocks_per_tile_x_row; + + if (need_check) { + i = min(i, i_max); + } + + const block_q5_1* bxi = bx0 + i * blocks_per_row + kbxd; + + x_dm[i * (WARP_SIZE_GGUF / QI5_1) + i / QI5_1 + kbxd] = bxi->dm; + } +} + +static __device__ __forceinline__ float vec_dot_q5_1_q8_1_mul_mat( + const int* __restrict__ x_ql, + const half2* __restrict__ x_dm, + const int* __restrict__ x_qh, + const int* __restrict__ x_sc, + const int* __restrict__ y_qs, + const half2* __restrict__ y_ds, + const int& i, + const int& j, + const int& k) { + const int kyqs = k % (QI8_1 / 2) + QI8_1 * (k / (QI8_1 / 2)); + const int index_bx = i * (WARP_SIZE_GGUF / QI5_1) + +i / QI5_1 + k / QI5_1; + + int u[2 * VDR_Q5_1_Q8_1_MMQ]; + +#pragma unroll + for (int l = 0; l < VDR_Q5_1_Q8_1_MMQ; ++l) { + u[2 * l + 0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF]; + u[2 * l + 1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI5_1) % WARP_SIZE_GGUF]; + } + + return vec_dot_q8_1_q8_1_impl( + &x_ql[i * (2 * WARP_SIZE_GGUF + 1) + 2 * k], + u, + x_dm[index_bx], + y_ds[j * (WARP_SIZE_GGUF / QI8_1) + (2 * k / QI8_1) % (WARP_SIZE_GGUF / QI8_1)]); +} + +static __device__ __forceinline__ float +vec_dot_q8_0_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) { + const block_q8_0* bq8_0 = (const block_q8_0*)vbq; + + int v[VDR_Q8_0_Q8_1_MMVQ]; + int u[VDR_Q8_0_Q8_1_MMVQ]; + +#pragma unroll + for (int i = 0; i < VDR_Q8_0_Q8_1_MMVQ; ++i) { + v[i] = get_int_from_int8(bq8_0->qs, iqs + i); + u[i] = get_int_from_int8_aligned(bq8_1->qs, iqs + i); + } + + return vec_dot_q8_0_q8_1_impl(v, u, __half2float(bq8_0->d), __low2float(bq8_1->ds)); +} + +template +static __device__ __forceinline__ void allocate_tiles_q8_0(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) { + __shared__ int tile_x_qs[mmq_y * (WARP_SIZE_GGUF) + mmq_y]; + __shared__ float tile_x_d[mmq_y * (WARP_SIZE_GGUF / QI8_0) + mmq_y / QI8_0]; + + *x_ql = tile_x_qs; + *x_dm = (half2*)tile_x_d; +} + +template +static __device__ __forceinline__ void load_tiles_q8_0( + const void* __restrict__ vx, + int* __restrict__ x_ql, + half2* __restrict__ x_dm, + int* __restrict__ x_qh, + int* __restrict__ x_sc, + const int& i_offset, + const int& i_max, + const int& k, + const int& blocks_per_row) { + const int kbx = k / QI8_0; + const int kqsx = k % QI8_0; + float* x_dmf = (float*)x_dm; + + const block_q8_0* bx0 = (const block_q8_0*)vx; + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps) { + int i = i0 + i_offset; + + if (need_check) { + i = min(i, i_max); + } + const block_q8_0* bxi = bx0 + i * blocks_per_row + kbx; + x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_int8(bxi->qs, kqsx); + } + + const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI8_0; + const int kbxd = k % blocks_per_tile_x_row; + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI8_0) { + int i = i0 + i_offset * QI8_0 + k / blocks_per_tile_x_row; + + if (need_check) { + i = min(i, i_max); + } + const block_q8_0* bxi = bx0 + i * blocks_per_row + kbxd; + x_dmf[i * (WARP_SIZE_GGUF / QI8_0) + i / QI8_0 + kbxd] = __half2float(bxi->d); + } +} + +static __device__ __forceinline__ float vec_dot_q8_0_q8_1_mul_mat( + const int* __restrict__ x_ql, + const half2* __restrict__ x_dm, + const int* __restrict__ x_qh, + const int* __restrict__ x_sc, + const int* __restrict__ y_qs, + const half2* __restrict__ y_ds, + const int& i, + const int& j, + const int& k) { + const float* x_dmf = (const float*)x_dm; + const float* y_df = (const float*)y_ds; + + return vec_dot_q8_0_q8_1_impl( + &x_ql[i * (WARP_SIZE_GGUF + 1) + k], + &y_qs[j * WARP_SIZE_GGUF + k], + x_dmf[i * (WARP_SIZE_GGUF / QI8_0) + i / QI8_0 + k / QI8_0], + y_df[j * (WARP_SIZE_GGUF / QI8_1) + k / QI8_1]); +} + +static __device__ __forceinline__ float +vec_dot_q2_K_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) { + const block_q2_K* bq2_K = (const block_q2_K*)vbq; + + const int bq8_offset = QR2_K * (iqs / QI8_1); + const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1 / 2); + + const uint8_t* scales = bq2_K->scales + scale_offset; + + const int v = get_int_from_uint8_aligned(bq2_K->qs, iqs); + int u[QR2_K]; + float d8[QR2_K]; + +#pragma unroll + for (int i = 0; i < QR2_K; ++i) { + u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1); + d8[i] = __low2float(bq8_1[bq8_offset + i].ds); + } + + return vec_dot_q2_K_q8_1_impl_mmvq(v, u, scales, bq2_K->dm, d8); +} + +template +static __device__ __forceinline__ void allocate_tiles_q2_K(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) { + __shared__ int tile_x_ql[mmq_y * (WARP_SIZE_GGUF) + mmq_y]; + __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF / QI2_K) + mmq_y / QI2_K]; + __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF / 4) + mmq_y / 4]; + + *x_ql = tile_x_ql; + *x_dm = tile_x_dm; + *x_sc = tile_x_sc; +} + +template +static __device__ __forceinline__ void load_tiles_q2_K( + const void* __restrict__ vx, + int* __restrict__ x_ql, + half2* __restrict__ x_dm, + int* __restrict__ x_qh, + int* __restrict__ x_sc, + const int& i_offset, + const int& i_max, + const int& k, + const int& blocks_per_row) { + const int kbx = k / QI2_K; + const int kqsx = k % QI2_K; + + const block_q2_K* bx0 = (const block_q2_K*)vx; + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps) { + int i = i0 + i_offset; + + if (need_check) { + i = min(i, i_max); + } + const block_q2_K* bxi = bx0 + i * blocks_per_row + kbx; + x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx); + } + + const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI2_K; + const int kbxd = k % blocks_per_tile_x_row; + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI2_K) { + int i = (i0 + i_offset * QI2_K + k / blocks_per_tile_x_row) % mmq_y; + + if (need_check) { + i = min(i, i_max); + } + const block_q2_K* bxi = bx0 + i * blocks_per_row + kbxd; + x_dm[i * (WARP_SIZE_GGUF / QI2_K) + i / QI2_K + kbxd] = bxi->dm; + } + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 4) { + int i = i0 + i_offset * 4 + k / (WARP_SIZE_GGUF / 4); + + if (need_check) { + i = min(i, i_max); + } + const block_q2_K* bxi = bx0 + i * blocks_per_row + (k % (WARP_SIZE_GGUF / 4)) / (QI2_K / 4); + x_sc[i * (WARP_SIZE_GGUF / 4) + i / 4 + k % (WARP_SIZE_GGUF / 4)] = + get_int_from_uint8_aligned(bxi->scales, k % (QI2_K / 4)); + } +} + +static __device__ __forceinline__ float vec_dot_q2_K_q8_1_mul_mat( + const int* __restrict__ x_ql, + const half2* __restrict__ x_dm, + const int* __restrict__ x_qh, + const int* __restrict__ x_sc, + const int* __restrict__ y_qs, + const half2* __restrict__ y_ds, + const int& i, + const int& j, + const int& k) { + const int kbx = k / QI2_K; + const int ky = (k % QI2_K) * QR2_K; + const float* y_df = (const float*)y_ds; + + int v[QR2_K * VDR_Q2_K_Q8_1_MMQ]; + + const int kqsx = i * (WARP_SIZE_GGUF + 1) + kbx * QI2_K + (QI2_K / 2) * (ky / (2 * QI2_K)) + ky % (QI2_K / 2); + const int shift = 2 * ((ky % (2 * QI2_K)) / (QI2_K / 2)); + +#pragma unroll + for (int l = 0; l < QR2_K * VDR_Q2_K_Q8_1_MMQ; ++l) { + v[l] = (x_ql[kqsx + l] >> shift) & 0x03030303; + } + + const uint8_t* scales = ((const uint8_t*)&x_sc[i * (WARP_SIZE_GGUF / 4) + i / 4 + kbx * 4]) + ky / 4; + + const int index_y = j * WARP_SIZE_GGUF + (QR2_K * k) % WARP_SIZE_GGUF; + return vec_dot_q2_K_q8_1_impl_mmq( + v, &y_qs[index_y], scales, x_dm[i * (WARP_SIZE_GGUF / QI2_K) + i / QI2_K + kbx], y_df[index_y / QI8_1]); +} + +static __device__ __forceinline__ float +vec_dot_q3_K_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) { + const block_q3_K* bq3_K = (const block_q3_K*)vbq; + + const int bq8_offset = QR3_K * (iqs / (QI3_K / 2)); + const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1 / 2); + + const float d = __half2float(bq3_K->d); + + const int vl = get_int_from_uint8(bq3_K->qs, iqs); + + // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted + const int vh = ~get_int_from_uint8(bq3_K->hmask, iqs % (QI3_K / 2)) >> bq8_offset; + + int u[QR3_K]; + float d8[QR3_K]; + +#pragma unroll + for (int i = 0; i < QR3_K; ++i) { + u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1); + d8[i] = __low2float(bq8_1[bq8_offset + i].ds); + } + + return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8); +} + +template +static __device__ __forceinline__ void allocate_tiles_q3_K(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) { + __shared__ int tile_x_ql[mmq_y * (WARP_SIZE_GGUF) + mmq_y]; + __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF / QI3_K) + mmq_y / QI3_K]; + __shared__ int tile_x_qh[mmq_y * (WARP_SIZE_GGUF / 2) + mmq_y / 2]; + __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF / 4) + mmq_y / 4]; + + *x_ql = tile_x_ql; + *x_dm = tile_x_dm; + *x_qh = tile_x_qh; + *x_sc = tile_x_sc; +} + +template +static __device__ __forceinline__ void load_tiles_q3_K( + const void* __restrict__ vx, + int* __restrict__ x_ql, + half2* __restrict__ x_dm, + int* __restrict__ x_qh, + int* __restrict__ x_sc, + const int& i_offset, + const int& i_max, + const int& k, + const int& blocks_per_row) { + const int kbx = k / QI3_K; + const int kqsx = k % QI3_K; + + const block_q3_K* bx0 = (const block_q3_K*)vx; + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps) { + int i = i0 + i_offset; + if (need_check) { + i = min(i, i_max); + } + const block_q3_K* bxi = bx0 + i * blocks_per_row + kbx; + x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8(bxi->qs, kqsx); + } + + const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI3_K; + const int kbxd = k % blocks_per_tile_x_row; + float* x_dmf = (float*)x_dm; + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI3_K) { + int i = (i0 + i_offset * QI3_K + k / blocks_per_tile_x_row) % mmq_y; + if (need_check) { + i = min(i, i_max); + } + const block_q3_K* bxi = bx0 + i * blocks_per_row + kbxd; + x_dmf[i * (WARP_SIZE_GGUF / QI3_K) + i / QI3_K + kbxd] = __half2float(bxi->d); + } + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 2) { + int i = i0 + i_offset * 2 + k / (WARP_SIZE_GGUF / 2); + if (need_check) { + i = min(i, i_max); + } + const block_q3_K* bxi = bx0 + i * blocks_per_row + (k % (WARP_SIZE_GGUF / 2)) / (QI3_K / 2); + // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted + x_qh[i * (WARP_SIZE_GGUF / 2) + i / 2 + k % (WARP_SIZE_GGUF / 2)] = + ~get_int_from_uint8(bxi->hmask, k % (QI3_K / 2)); + } + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 4) { + int i = i0 + i_offset * 4 + k / (WARP_SIZE_GGUF / 4); + if (need_check) { + i = min(i, i_max); + } + const block_q3_K* bxi = bx0 + i * blocks_per_row + (k % (WARP_SIZE_GGUF / 4)) / (QI3_K / 4); + + const int ksc = k % (QI3_K / 4); + + const int ksc_low = ksc % (QI3_K / 8); + const int shift_low = 4 * (ksc / (QI3_K / 8)); + const int sc_low = (get_int_from_uint8(bxi->scales, ksc_low) >> shift_low) & 0x0F0F0F0F; + + const int ksc_high = QI3_K / 8; + const int shift_high = 2 * ksc; + const int sc_high = ((get_int_from_uint8(bxi->scales, ksc_high) >> shift_high) << 4) & 0x30303030; + + const int sc = __vsubss4(sc_low | sc_high, 0x20202020); + + x_sc[i * (WARP_SIZE_GGUF / 4) + i / 4 + k % (WARP_SIZE_GGUF / 4)] = sc; + } +} + +static __device__ __forceinline__ float vec_dot_q3_K_q8_1_mul_mat( + const int* __restrict__ x_ql, + const half2* __restrict__ x_dm, + const int* __restrict__ x_qh, + const int* __restrict__ x_sc, + const int* __restrict__ y_qs, + const half2* __restrict__ y_ds, + const int& i, + const int& j, + const int& k) { + const int kbx = k / QI3_K; + const int ky = (k % QI3_K) * QR3_K; + const float* x_dmf = (const float*)x_dm; + const float* y_df = (const float*)y_ds; + + const int8_t* scales = ((const int8_t*)(x_sc + i * (WARP_SIZE_GGUF / 4) + i / 4 + kbx * 4)) + ky / 4; + + int v[QR3_K * VDR_Q3_K_Q8_1_MMQ]; + +#pragma unroll + for (int l = 0; l < QR3_K * VDR_Q3_K_Q8_1_MMQ; ++l) { + const int kqsx = i * (WARP_SIZE_GGUF + 1) + kbx * QI3_K + (QI3_K / 2) * (ky / (2 * QI3_K)) + ky % (QI3_K / 2); + const int shift = 2 * ((ky % 32) / 8); + const int vll = (x_ql[kqsx + l] >> shift) & 0x03030303; + + const int vh = x_qh[i * (WARP_SIZE_GGUF / 2) + i / 2 + kbx * (QI3_K / 2) + (ky + l) % 8] >> ((ky + l) / 8); + const int vlh = (vh << 2) & 0x04040404; + + v[l] = __vsubss4(vll, vlh); + } + + const int index_y = j * WARP_SIZE_GGUF + (k * QR3_K) % WARP_SIZE_GGUF; + return vec_dot_q3_K_q8_1_impl_mmq( + v, &y_qs[index_y], scales, x_dmf[i * (WARP_SIZE_GGUF / QI3_K) + i / QI3_K + kbx], y_df[index_y / QI8_1]); +} + +static __device__ __forceinline__ float +vec_dot_q4_K_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) { + const block_q4_K* bq4_K = (const block_q4_K*)vbq; + + int v[2]; + int u[2 * QR4_K]; + float d8[QR4_K]; + + // iqs is in 0,2..30. bq8_offset = iqs/4 -> bq8_offset = 0, 2, 4, 6 + const int bq8_offset = QR4_K * ((iqs / 2) / (QI8_1 / 2)); + + // iqs = 0....3 -> bq8_offset = 0, want q4_offset = 0, 4, 8, 12 + // iqs = 4....7 -> bq8_offset = 2, want q4_offset = 32, 36, 40, 44 + // iqs = 8...11 -> bq8_offset = 4, want q4_offset = 64, 68, 72, 76 + // iqs = 12..15 -> bq8_offset = 6, want q4_offset = 96, 100, 104, 108 + + const int* q4 = (const int*)(bq4_K->qs + 16 * bq8_offset + 4 * ((iqs / 2) % 4)); + v[0] = q4[0]; + v[1] = q4[4]; + + const uint16_t* scales = (const uint16_t*)bq4_K->scales; + uint16_t aux[2]; + const int j = bq8_offset / 2; + if (j < 2) { + aux[0] = scales[j + 0] & 0x3f3f; + aux[1] = scales[j + 2] & 0x3f3f; + } else { + aux[0] = ((scales[j + 2] >> 0) & 0x0f0f) | ((scales[j - 2] & 0xc0c0) >> 2); + aux[1] = ((scales[j + 2] >> 4) & 0x0f0f) | ((scales[j - 0] & 0xc0c0) >> 2); + } + const uint8_t* sc = (const uint8_t*)aux; + const uint8_t* m = sc + 2; + + for (int i = 0; i < QR4_K; ++i) { + const block_q8_1* bq8i = bq8_1 + bq8_offset + i; + d8[i] = __low2float(bq8i->ds); + + const int* q8 = (const int*)bq8i->qs + ((iqs / 2) % 4); + u[2 * i + 0] = q8[0]; + u[2 * i + 1] = q8[4]; + } + + return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, bq4_K->dm, d8); +} + +template +static __device__ __forceinline__ void allocate_tiles_q4_K(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) { + __shared__ int tile_x_ql[mmq_y * (WARP_SIZE_GGUF) + mmq_y]; + __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF / QI4_K) + mmq_y / QI4_K]; + __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF / 8) + mmq_y / 8]; + + *x_ql = tile_x_ql; + *x_dm = tile_x_dm; + *x_sc = tile_x_sc; +} + +template +static __device__ __forceinline__ void load_tiles_q4_K( + const void* __restrict__ vx, + int* __restrict__ x_ql, + half2* __restrict__ x_dm, + int* __restrict__ x_qh, + int* __restrict__ x_sc, + const int& i_offset, + const int& i_max, + const int& k, + const int& blocks_per_row) { + const int kbx = k / QI4_K; // == 0 if QK_K == 256 + const int kqsx = k % QI4_K; // == k if QK_K == 256 + + const block_q4_K* bx0 = (const block_q4_K*)vx; + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps) { + int i = i0 + i_offset; + + if (need_check) { + i = min(i, i_max); + } + const block_q4_K* bxi = bx0 + i * blocks_per_row + kbx; + x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx); + } + + const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI4_K; // == 1 if QK_K == 256 + const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256 + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_K) { + int i = (i0 + i_offset * QI4_K + k / blocks_per_tile_x_row) % mmq_y; + if (need_check) { + i = min(i, i_max); + } + const block_q4_K* bxi = bx0 + i * blocks_per_row + kbxd; + x_dm[i * (WARP_SIZE_GGUF / QI4_K) + i / QI4_K + kbxd] = bxi->dm; + } + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) { + int i = (i0 + i_offset * 8 + k / (WARP_SIZE_GGUF / 8)) % mmq_y; + + if (need_check) { + i = min(i, i_max); + } + + const block_q4_K* bxi = bx0 + i * blocks_per_row + (k % (WARP_SIZE_GGUF / 8)) / (QI4_K / 8); + + const int* scales = (const int*)bxi->scales; + + const int ksc = k % (WARP_SIZE_GGUF / 8); + // scale arrangement after the following two lines: sc0,...,sc3, sc4,...,sc7, m0,...,m3, m4,...,m8 + int scales8 = (scales[(ksc % 2) + (ksc != 0)] >> (4 * (ksc & (ksc / 2)))) & 0x0F0F0F0F; // lower 4 bits + scales8 |= (scales[ksc / 2] >> (2 * (ksc % 2))) & 0x30303030; // upper 2 bits + + x_sc[i * (WARP_SIZE_GGUF / 8) + i / 8 + ksc] = scales8; + } +} + +static __device__ __forceinline__ float vec_dot_q4_K_q8_1_mul_mat( + const int* __restrict__ x_ql, + const half2* __restrict__ x_dm, + const int* __restrict__ x_qh, + const int* __restrict__ x_sc, + const int* __restrict__ y_qs, + const half2* __restrict__ y_ds, + const int& i, + const int& j, + const int& k) { + (void)x_qh; + + const uint8_t* sc = ((const uint8_t*)&x_sc[i * (WARP_SIZE_GGUF / 8) + i / 8 + k / 16]) + 2 * ((k % 16) / 8); + + const int index_y = j * WARP_SIZE_GGUF + (QR4_K * k) % WARP_SIZE_GGUF; + return vec_dot_q4_K_q8_1_impl_mmq( + &x_ql[i * (WARP_SIZE_GGUF + 1) + k], + &y_qs[index_y], + sc, + sc + 8, + x_dm[i * (WARP_SIZE_GGUF / QI4_K) + i / QI4_K], + &y_ds[index_y / QI8_1]); +} + +static __device__ __forceinline__ float +vec_dot_q5_K_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) { + const block_q5_K* bq5_K = (const block_q5_K*)vbq; + + int vl[2]; + int vh[2]; + int u[2 * QR5_K]; + float d8[QR5_K]; + + const int bq8_offset = QR5_K * ((iqs / 2) / (QI8_1 / 2)); + const int* ql = (const int*)(bq5_K->qs + 16 * bq8_offset + 4 * ((iqs / 2) % 4)); + const int* qh = (const int*)(bq5_K->qh + 4 * ((iqs / 2) % 4)); + + vl[0] = ql[0]; + vl[1] = ql[4]; + + vh[0] = qh[0] >> bq8_offset; + vh[1] = qh[4] >> bq8_offset; + + const uint16_t* scales = (const uint16_t*)bq5_K->scales; + uint16_t aux[2]; + const int j = bq8_offset / 2; + if (j < 2) { + aux[0] = scales[j + 0] & 0x3f3f; + aux[1] = scales[j + 2] & 0x3f3f; + } else { + aux[0] = ((scales[j + 2] >> 0) & 0x0f0f) | ((scales[j - 2] & 0xc0c0) >> 2); + aux[1] = ((scales[j + 2] >> 4) & 0x0f0f) | ((scales[j - 0] & 0xc0c0) >> 2); + } + const uint8_t* sc = (const uint8_t*)aux; + const uint8_t* m = sc + 2; + +#pragma unroll + for (int i = 0; i < QR5_K; ++i) { + const block_q8_1* bq8i = bq8_1 + bq8_offset + i; + d8[i] = __low2float(bq8i->ds); + + const int* q8 = (const int*)bq8i->qs + ((iqs / 2) % 4); + u[2 * i + 0] = q8[0]; + u[2 * i + 1] = q8[4]; + } + + return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, bq5_K->dm, d8); +} + +template +static __device__ __forceinline__ void allocate_tiles_q5_K(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) { + __shared__ int tile_x_ql[mmq_y * (2 * WARP_SIZE_GGUF) + mmq_y]; + __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF / QI5_K) + mmq_y / QI5_K]; + __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF / 8) + mmq_y / 8]; + + *x_ql = tile_x_ql; + *x_dm = tile_x_dm; + *x_sc = tile_x_sc; +} + +template +static __device__ __forceinline__ void load_tiles_q5_K( + const void* __restrict__ vx, + int* __restrict__ x_ql, + half2* __restrict__ x_dm, + int* __restrict__ x_qh, + int* __restrict__ x_sc, + const int& i_offset, + const int& i_max, + const int& k, + const int& blocks_per_row) { + const int kbx = k / QI5_K; // == 0 if QK_K == 256 + const int kqsx = k % QI5_K; // == k if QK_K == 256 + + const block_q5_K* bx0 = (const block_q5_K*)vx; + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps) { + int i = i0 + i_offset; + + if (need_check) { + i = min(i, i_max); + } + + const block_q5_K* bxi = bx0 + i * blocks_per_row + kbx; + const int ky = QR5_K * kqsx; + + const int ql = get_int_from_uint8_aligned(bxi->qs, kqsx); + const int ql0 = (ql >> 0) & 0x0F0F0F0F; + const int ql1 = (ql >> 4) & 0x0F0F0F0F; + + const int qh = get_int_from_uint8_aligned(bxi->qh, kqsx % (QI5_K / 4)); + const int qh0 = ((qh >> (2 * (kqsx / (QI5_K / 4)) + 0)) << 4) & 0x10101010; + const int qh1 = ((qh >> (2 * (kqsx / (QI5_K / 4)) + 1)) << 4) & 0x10101010; + + const int kq0 = ky - ky % (QI5_K / 2) + k % (QI5_K / 4) + 0; + const int kq1 = ky - ky % (QI5_K / 2) + k % (QI5_K / 4) + (QI5_K / 4); + + x_ql[i * (2 * WARP_SIZE_GGUF + 1) + kq0] = ql0 | qh0; + x_ql[i * (2 * WARP_SIZE_GGUF + 1) + kq1] = ql1 | qh1; + } + + const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI5_K; // == 1 if QK_K == 256 + const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256 + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_K) { + int i = (i0 + i_offset * QI5_K + k / blocks_per_tile_x_row) % mmq_y; + + if (need_check) { + i = min(i, i_max); + } + + const block_q5_K* bxi = bx0 + i * blocks_per_row + kbxd; + x_dm[i * (WARP_SIZE_GGUF / QI5_K) + i / QI5_K + kbxd] = bxi->dm; + } + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) { + int i = (i0 + i_offset * 8 + k / (WARP_SIZE_GGUF / 8)) % mmq_y; + + if (need_check) { + i = min(i, i_max); + } + + const block_q5_K* bxi = bx0 + i * blocks_per_row + (k % (WARP_SIZE_GGUF / 8)) / (QI5_K / 8); + + const int* scales = (const int*)bxi->scales; + + const int ksc = k % (WARP_SIZE_GGUF / 8); + + // scale arrangement after the following two lines: sc0,...,sc3, sc4,...,sc7, m0,...,m3, m4,...,m8 + int scales8 = (scales[(ksc % 2) + (ksc != 0)] >> (4 * (ksc & (ksc / 2)))) & 0x0F0F0F0F; // lower 4 bits + scales8 |= (scales[ksc / 2] >> (2 * (ksc % 2))) & 0x30303030; // upper 2 bits + + x_sc[i * (WARP_SIZE_GGUF / 8) + i / 8 + ksc] = scales8; + } +} + +static __device__ __forceinline__ float vec_dot_q5_K_q8_1_mul_mat( + const int* __restrict__ x_ql, + const half2* __restrict__ x_dm, + const int* __restrict__ x_qh, + const int* __restrict__ x_sc, + const int* __restrict__ y_qs, + const half2* __restrict__ y_ds, + const int& i, + const int& j, + const int& k) { + const uint8_t* sc = ((const uint8_t*)&x_sc[i * (WARP_SIZE_GGUF / 8) + i / 8 + k / 16]) + 2 * ((k % 16) / 8); + + const int index_x = i * (QR5_K * WARP_SIZE_GGUF + 1) + QR5_K * k; + const int index_y = j * WARP_SIZE_GGUF + (QR5_K * k) % WARP_SIZE_GGUF; + return vec_dot_q5_K_q8_1_impl_mmq( + &x_ql[index_x], + &y_qs[index_y], + sc, + sc + 8, + x_dm[i * (WARP_SIZE_GGUF / QI5_K) + i / QI5_K], + &y_ds[index_y / QI8_1]); +} + +static __device__ __forceinline__ float +vec_dot_q6_K_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) { + const block_q6_K* bq6_K = (const block_q6_K*)vbq; + + const int bq8_offset = 2 * QR6_K * (iqs / (QI6_K / 2)) + (iqs % (QI6_K / 2)) / (QI6_K / 4); + const int scale_offset = (QI6_K / 4) * (iqs / (QI6_K / 2)) + (iqs % (QI6_K / 2)) / (QI6_K / 8); + const int vh_shift = 2 * ((iqs % (QI6_K / 2)) / (QI6_K / 4)); + + const int vl = get_int_from_uint8(bq6_K->ql, iqs); + const int vh = get_int_from_uint8(bq6_K->qh, (QI6_K / 4) * (iqs / (QI6_K / 2)) + iqs % (QI6_K / 4)) >> vh_shift; + + const int8_t* scales = bq6_K->scales + scale_offset; + + int u[QR6_K]; + float d8[QR6_K]; + +#pragma unroll + for (int i = 0; i < QR6_K; ++i) { + u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + 2 * i].qs, iqs % QI8_1); + d8[i] = __low2float(bq8_1[bq8_offset + 2 * i].ds); + } + + return vec_dot_q6_K_q8_1_impl_mmvq(vl, vh, u, scales, __half2float(bq6_K->d), d8); +} + +template +static __device__ __forceinline__ void allocate_tiles_q6_K(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) { + __shared__ int tile_x_ql[mmq_y * (2 * WARP_SIZE_GGUF) + mmq_y]; + __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF / QI6_K) + mmq_y / QI6_K]; + __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF / 8) + mmq_y / 8]; + + *x_ql = tile_x_ql; + *x_dm = tile_x_dm; + *x_sc = tile_x_sc; +} + +template +static __device__ __forceinline__ void load_tiles_q6_K( + const void* __restrict__ vx, + int* __restrict__ x_ql, + half2* __restrict__ x_dm, + int* __restrict__ x_qh, + int* __restrict__ x_sc, + const int& i_offset, + const int& i_max, + const int& k, + const int& blocks_per_row) { + const int kbx = k / QI6_K; // == 0 if QK_K == 256 + const int kqsx = k % QI6_K; // == k if QK_K == 256 + + const block_q6_K* bx0 = (const block_q6_K*)vx; + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps) { + int i = i0 + i_offset; + + if (need_check) { + i = min(i, i_max); + } + + const block_q6_K* bxi = bx0 + i * blocks_per_row + kbx; + const int ky = QR6_K * kqsx; + + const int ql = get_int_from_uint8(bxi->ql, kqsx); + const int ql0 = (ql >> 0) & 0x0F0F0F0F; + const int ql1 = (ql >> 4) & 0x0F0F0F0F; + + const int qh = get_int_from_uint8(bxi->qh, (QI6_K / 4) * (kqsx / (QI6_K / 2)) + kqsx % (QI6_K / 4)); + const int qh0 = ((qh >> (2 * ((kqsx % (QI6_K / 2)) / (QI6_K / 4)))) << 4) & 0x30303030; + const int qh1 = (qh >> (2 * ((kqsx % (QI6_K / 2)) / (QI6_K / 4)))) & 0x30303030; + + const int kq0 = ky - ky % QI6_K + k % (QI6_K / 2) + 0; + const int kq1 = ky - ky % QI6_K + k % (QI6_K / 2) + (QI6_K / 2); + + x_ql[i * (2 * WARP_SIZE_GGUF + 1) + kq0] = __vsubss4(ql0 | qh0, 0x20202020); + x_ql[i * (2 * WARP_SIZE_GGUF + 1) + kq1] = __vsubss4(ql1 | qh1, 0x20202020); + } + + const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI6_K; // == 1 if QK_K == 256 + const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256 + float* x_dmf = (float*)x_dm; + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI6_K) { + int i = (i0 + i_offset * QI6_K + k / blocks_per_tile_x_row) % mmq_y; + + if (need_check) { + i = min(i, i_max); + } + + const block_q6_K* bxi = bx0 + i * blocks_per_row + kbxd; + + x_dmf[i * (WARP_SIZE_GGUF / QI6_K) + i / QI6_K + kbxd] = __half2float(bxi->d); + } + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) { + int i = (i0 + i_offset * 8 + k / (WARP_SIZE_GGUF / 8)) % mmq_y; + + if (need_check) { + i = min(i, i_max); + } + + const block_q6_K* bxi = bx0 + i * blocks_per_row + (k % (WARP_SIZE_GGUF / 8)) / 4; + + x_sc[i * (WARP_SIZE_GGUF / 8) + i / 8 + k % (WARP_SIZE_GGUF / 8)] = get_int_from_int8(bxi->scales, k % (QI6_K / 8)); + } +} + +static __device__ __forceinline__ float vec_dot_q6_K_q8_1_mul_mat( + const int* __restrict__ x_ql, + const half2* __restrict__ x_dm, + const int* __restrict__ x_qh, + const int* __restrict__ x_sc, + const int* __restrict__ y_qs, + const half2* __restrict__ y_ds, + const int& i, + const int& j, + const int& k) { + const float* x_dmf = (const float*)x_dm; + const float* y_df = (const float*)y_ds; + + const int8_t* sc = ((const int8_t*)&x_sc[i * (WARP_SIZE_GGUF / 8) + i / 8 + k / 8]); + + const int index_x = i * (QR6_K * WARP_SIZE_GGUF + 1) + QR6_K * k; + const int index_y = j * WARP_SIZE_GGUF + (QR6_K * k) % WARP_SIZE_GGUF; + return vec_dot_q6_K_q8_1_impl_mmq( + &x_ql[index_x], &y_qs[index_y], sc, x_dmf[i * (WARP_SIZE_GGUF / QI6_K) + i / QI6_K], &y_df[index_y / QI8_1]); +} + +static __device__ __forceinline__ float +vec_dot_iq2_xxs_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) { + const block_iq2_xxs* bq2 = (const block_iq2_xxs*)vbq; + + const int ib32 = iqs; + const uint16_t* q2 = bq2->qs + 4 * ib32; + const uint8_t* aux8 = (const uint8_t*)q2; + const int8_t* q8 = bq8_1[ib32].qs; + uint32_t aux32 = q2[2] | (q2[3] << 16); + int sumi = 0; + for (int l = 0; l < 4; ++l) { + const uint8_t* grid = (const uint8_t*)(iq2xxs_grid + aux8[l]); + const uint8_t signs = ksigns_iq2xs[aux32 & 127]; + for (int j = 0; j < 8; ++j) { + sumi += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1); + } + q8 += 8; + aux32 >>= 7; + } + const float d = __half2float(bq2->d) * (0.5f + aux32) * __half2float(bq8_1[ib32].ds.x) * 0.25f; + return d * sumi; +} + +static __device__ __forceinline__ float +vec_dot_iq2_xs_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) { + const block_iq2_xs* bq2 = (const block_iq2_xs*)vbq; + + const int ib32 = iqs; + const uint16_t* q2 = bq2->qs + 4 * ib32; + const int8_t* q8 = bq8_1[ib32].qs; + const uint8_t ls1 = bq2->scales[ib32] & 0xf; + const uint8_t ls2 = bq2->scales[ib32] >> 4; + int sumi1 = 0; + for (int l = 0; l < 2; ++l) { + const uint8_t* grid = (const uint8_t*)(iq2xs_grid + (q2[l] & 511)); + const uint8_t signs = ksigns_iq2xs[q2[l] >> 9]; + for (int j = 0; j < 8; ++j) { + sumi1 += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1); + } + q8 += 8; + } + int sumi2 = 0; + for (int l = 2; l < 4; ++l) { + const uint8_t* grid = (const uint8_t*)(iq2xs_grid + (q2[l] & 511)); + const uint8_t signs = ksigns_iq2xs[q2[l] >> 9]; + for (int j = 0; j < 8; ++j) { + sumi2 += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1); + } + q8 += 8; + } + const float d = __half2float(bq2->d) * __half2float(bq8_1[ib32].ds.x) * 0.25f; + return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2); +} + +static __device__ __forceinline__ float +vec_dot_iq2_s_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM + const block_iq2_s* bq2 = (const block_iq2_s*)vbq; + + const int ib32 = iqs; + const int8_t* q8 = bq8_1[ib32].qs; + const uint8_t* signs = bq2->qs + QK_K / 8 + 4 * ib32; + const uint8_t ls1 = bq2->scales[ib32] & 0xf; + const uint8_t ls2 = bq2->scales[ib32] >> 4; + int sumi1 = 0; + for (int l = 0; l < 2; ++l) { + const uint32_t* grid = + (const uint32_t*)(iq2s_grid + (bq2->qs[4 * ib32 + l] | ((bq2->qh[ib32] << (8 - 2 * l)) & 0x300))); + const uint32_t signs0 = __vcmpeq4(((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201); + const uint32_t signs1 = __vcmpeq4(((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201); + const int grid_l = __vsub4(grid[0] ^ signs0, signs0); + const int grid_h = __vsub4(grid[1] ^ signs1, signs1); + sumi1 = __dp4a(grid_l, *((const int*)q8 + 0), sumi1); + sumi1 = __dp4a(grid_h, *((const int*)q8 + 1), sumi1); + q8 += 8; + } + int sumi2 = 0; + for (int l = 2; l < 4; ++l) { + const uint32_t* grid = + (const uint32_t*)(iq2s_grid + (bq2->qs[4 * ib32 + l] | ((bq2->qh[ib32] << (8 - 2 * l)) & 0x300))); + const uint32_t signs0 = __vcmpeq4(((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201); + const uint32_t signs1 = __vcmpeq4(((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201); + const int grid_l = __vsub4(grid[0] ^ signs0, signs0); + const int grid_h = __vsub4(grid[1] ^ signs1, signs1); + sumi2 = __dp4a(grid_l, *((const int*)q8 + 0), sumi2); + sumi2 = __dp4a(grid_h, *((const int*)q8 + 1), sumi2); + q8 += 8; + } + const float d = __half2float(bq2->d) * __low2float(bq8_1[ib32].ds) * 0.25f; + return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2); +#endif +} + +static __device__ __forceinline__ float +vec_dot_iq3_xxs_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM + const block_iq3_xxs* bq2 = (const block_iq3_xxs*)vbq; + + const int ib32 = iqs; + const uint8_t* q3 = bq2->qs + 8 * ib32; + const uint16_t* gas = (const uint16_t*)(bq2->qs + QK_K / 4) + 2 * ib32; + const int8_t* q8 = bq8_1[ib32].qs; + uint32_t aux32 = gas[0] | (gas[1] << 16); + int sumi = 0; + for (int l = 0; l < 4; ++l) { + const uint32_t* grid1 = iq3xxs_grid + q3[2 * l + 0]; + const uint32_t* grid2 = iq3xxs_grid + q3[2 * l + 1]; + const uint32_t* signs = (const uint32_t*)(ksigns64 + (aux32 & 127)); + const int grid_l = __vsub4(grid1[0] ^ signs[0], signs[0]); + const int grid_h = __vsub4(grid2[0] ^ signs[1], signs[1]); + sumi = __dp4a(grid_l, *((int*)q8 + 0), sumi); + sumi = __dp4a(grid_h, *((int*)q8 + 1), sumi); + q8 += 8; + aux32 >>= 7; + } + const float d = __half2float(bq2->d) * (0.5f + aux32) * __low2float(bq8_1[ib32].ds) * 0.5f; + return d * sumi; +#endif +} + +static __device__ __forceinline__ float +vec_dot_iq3_s_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM + const block_iq3_s* bq2 = (const block_iq3_s*)vbq; + + const int ib32 = iqs; + const uint8_t* qs = bq2->qs + 8 * ib32; + const int8_t* q8 = bq8_1[ib32].qs; + int sumi = 0; + for (int l = 0; l < 4; ++l) { + const uint32_t* grid1 = iq3xs_grid + (qs[2 * l + 0] | ((bq2->qh[ib32] << (8 - 2 * l)) & 256)); + const uint32_t* grid2 = iq3xs_grid + (qs[2 * l + 1] | ((bq2->qh[ib32] << (7 - 2 * l)) & 256)); + uint32_t signs0 = __vcmpeq4(((bq2->signs[4 * ib32 + l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201); + uint32_t signs1 = __vcmpeq4(((bq2->signs[4 * ib32 + l] >> 4) * 0x01010101) & 0x08040201, 0x08040201); + const int grid_l = __vsub4(grid1[0] ^ signs0, signs0); + const int grid_h = __vsub4(grid2[0] ^ signs1, signs1); + sumi = __dp4a(grid_l, *((int*)q8 + 0), sumi); + sumi = __dp4a(grid_h, *((int*)q8 + 1), sumi); + q8 += 8; + } + const float d = __half2float(bq2->d) * (0.5f + ((bq2->scales[ib32 / 2] >> 4 * (ib32 % 2)) & 0xf)) * + __low2float(bq8_1[ib32].ds) * 0.5f; + return d * sumi; +#endif +} + +static __device__ __forceinline__ float +vec_dot_iq1_s_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM + const block_iq1_s* bq1 = (const block_iq1_s*)vbq; + + const int qs_packed = get_int_b2(bq1->qs, iqs); + const uint8_t* qs = (const uint8_t*)&qs_packed; + + const int qh = bq1->qh[iqs]; + + int sumi = 0; +#pragma unroll + for (int l0 = 0; l0 < 8; l0 += 2) { + const int grid = iq1s_grid_gpu[qs[l0 / 2] | (((qh >> 3 * (l0 / 2)) & 0x07) << 8)]; + + const int grid0 = (grid >> 0) & 0x0F0F0F0F; + const int grid1 = (grid >> 4) & 0x0F0F0F0F; + + const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0); + const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1); + + sumi = __dp4a(grid0, u0, sumi); + sumi = __dp4a(grid1, u1, sumi); + } + + const float d1q = __half2float(bq1->d) * (((qh >> 11) & 0x0E) + 1); + const float delta = -1.0f + IQ1S_DELTA - (qh & 0x8000) * (2.0f * IQ1S_DELTA / 0x8000); + const float2 ds = __half22float2(bq8_1[iqs].ds); + return d1q * (ds.x * sumi + ds.y * delta); +#endif +} + +static __device__ __forceinline__ float +vec_dot_iq1_m_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM + + const block_iq1_m* bq1 = (const block_iq1_m*)vbq; + + const int qs_packed = get_int_b4(bq1->qs, iqs); + const uint8_t* qs = (const uint8_t*)&qs_packed; + + int sumi[2] = {0}; + float sumf[2] = {0.0f}; +#pragma unroll + for (int l0 = 0; l0 < 8; l0 += 2) { + const int qhl = bq1->qh[2 * iqs + l0 / 4] >> (4 * ((l0 / 2) % 2)); + + const int grid = iq1s_grid_gpu[qs[l0 / 2] | ((qhl & 0x07) << 8)]; + + const int grid0 = (grid >> 0) & 0x0F0F0F0F; + const int grid1 = (grid >> 4) & 0x0F0F0F0F; + + const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0); + const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1); + + sumi[l0 / 4] = __dp4a(grid0, u0, sumi[l0 / 4]); + sumi[l0 / 4] = __dp4a(grid1, u1, sumi[l0 / 4]); + + const float delta = -1.0f + IQ1M_DELTA - (qhl & 0x08) * (2.0f * IQ1M_DELTA / 0x08); + int sumy = 0; + sumy = __dp4a(u0, 0x01010101, sumy); + sumy = __dp4a(u1, 0x01010101, sumy); + sumf[l0 / 4] += delta * sumy; + } + + const uint16_t* sc = (const uint16_t*)bq1->scales; + + iq1m_scale_t scale; + scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00F0) | ((sc[2] >> 4) & 0x0F00) | (sc[3] & 0xF000); + const float d = __half2float(scale.f16) * __low2float(bq8_1[iqs].ds); + + const int tmp = sc[iqs / 2] >> (6 * (iqs % 2)); + const int sc0 = 2 * ((tmp >> 0) & 0x07) + 1; + const int sc1 = 2 * ((tmp >> 3) & 0x07) + 1; + return d * ((sumi[0] + sumf[0]) * sc0 + (sumi[1] + sumf[1]) * sc1); +#endif +} + +static __device__ __forceinline__ void +get_int_from_table_16(const uint32_t& q4, const uint8_t* values, int& val1, int& val2) { + uint32_t aux32; + const uint8_t* q8 = (const uint8_t*)&aux32; + aux32 = q4 & 0x0f0f0f0f; + uint16_t v1 = values[q8[0]] | (values[q8[1]] << 8); + uint16_t v2 = values[q8[2]] | (values[q8[3]] << 8); + val1 = v1 | (v2 << 16); + aux32 = (q4 >> 4) & 0x0f0f0f0f; + v1 = values[q8[0]] | (values[q8[1]] << 8); + v2 = values[q8[2]] | (values[q8[3]] << 8); + val2 = v1 | (v2 << 16); +} + +static __device__ __forceinline__ float +vec_dot_iq4_nl_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM + + const block_iq4_nl* bq = (const block_iq4_nl*)vbq; + + const uint16_t* q4 = (const uint16_t*)bq->qs + 2 * iqs; + const int32_t* q8 = (const int32_t*)bq8_1->qs + iqs; + + const uint8_t* values = (const uint8_t*)kvalues_iq4nl; + + int v1, v2; + int sumi1 = 0, sumi2 = 0; + for (int l = 0; l < VDR_Q4_0_Q8_1_MMVQ; ++l) { + const uint32_t aux = q4[2 * l] | (q4[2 * l + 1] << 16); + get_int_from_table_16(aux, values, v1, v2); + sumi1 = __dp4a(v1, q8[l + 0], sumi1); + sumi2 = __dp4a(v2, q8[l + 4], sumi2); + } + const float d = __half2float(bq->d) * __low2float(bq8_1->ds); + return d * (sumi1 + sumi2); +#endif +} + +static __device__ __forceinline__ float +vec_dot_iq4_xs_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM + const block_iq4_xs* bq4 = (const block_iq4_xs*)vbq; + const uint8_t* values = (const uint8_t*)kvalues_iq4nl; + + // iqs is 0...7 + const int ib32 = iqs; + const int32_t* q8 = (const int*)bq8_1[ib32].qs; + const uint32_t* q4 = (const uint32_t*)bq4->qs + 4 * ib32; + const int8_t ls = ((bq4->scales_l[ib32 / 2] >> 4 * (ib32 % 2)) & 0xf) | (((bq4->scales_h >> 2 * ib32) & 3) << 4); + const float d = __half2float(bq4->d) * (ls - 32) * __low2float(bq8_1[ib32].ds); + int v1, v2; + int sumi1 = 0, sumi2 = 0; + for (int j = 0; j < 4; ++j) { + get_int_from_table_16(q4[j], values, v1, v2); + sumi1 = __dp4a(v1, q8[j + 0], sumi1); + sumi2 = __dp4a(v2, q8[j + 4], sumi2); + } + return d * (sumi1 + sumi2); +#endif +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/spatial/cuda_utils.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/spatial/cuda_utils.h new file mode 100644 index 00000000..65132975 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/spatial/cuda_utils.h @@ -0,0 +1,46 @@ +#include +#include + +#include + +#define CUDA_RT(call) \ + do { \ + cudaError_t _status = (call); \ + if (_status != cudaSuccess) { \ + std::cerr << "ERROR: CUDA RT call \"" << #call << "\" in line " << __LINE__ << " of file " << __FILE__ \ + << " failed with " << cudaGetErrorString(_status) << std::endl; \ + TORCH_CHECK( \ + false, \ + c10::str( \ + "ERROR: CUDA RT call \"", \ + #call, \ + "\" in line ", \ + __LINE__, \ + " of file ", \ + __FILE__, \ + " failed with ", \ + cudaGetErrorString(_status))); \ + } \ + } while (0) + +#define CUDA_DRV(call) \ + do { \ + CUresult _status = (call); \ + if (_status != CUDA_SUCCESS) { \ + const char* err_str; \ + cuGetErrorString(_status, &err_str); \ + std::cerr << "ERROR: CUDA DRV call \"" << #call << "\" in line " << __LINE__ << " of file " << __FILE__ \ + << " failed with " << err_str << std::endl; \ + TORCH_CHECK( \ + false, \ + c10::str( \ + "ERROR: CUDA DRV call \"", \ + #call, \ + "\" in line ", \ + __LINE__, \ + " of file ", \ + __FILE__, \ + " failed with ", \ + err_str)); \ + } \ + } while (0) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/spatial/greenctx_stream.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/spatial/greenctx_stream.cu new file mode 100644 index 00000000..0366565e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/spatial/greenctx_stream.cu @@ -0,0 +1,98 @@ +// Documentation: https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__GREEN__CONTEXTS.html +#include + +#include + +#include "cuda_utils.h" +#include "greenctx_stream.h" + +static int CUDA_DRIVER_VERSION; + +using PFN_cuGreenCtxStreamCreate = CUresult(CUDAAPI*)(CUstream*, CUgreenCtx, unsigned int, int); + +auto probe_cuGreenCtxStreamCreate() -> PFN_cuGreenCtxStreamCreate { + static PFN_cuGreenCtxStreamCreate pfn = nullptr; + CUDA_DRV(cuGetProcAddress("cuGreenCtxStreamCreate", reinterpret_cast(&pfn), CUDA_DRIVER_VERSION, 0, nullptr)); + return pfn; +} + +static std::vector create_greenctx_stream_fallback(CUgreenCtx gctx[2]) { + CUstream streamA, streamB; + CUcontext ctx; + + CUDA_DRV(cuCtxFromGreenCtx(&ctx, gctx[0])); + CUDA_DRV(cuCtxPushCurrent(ctx)); + CUDA_DRV(cuStreamCreate(&streamA, CU_STREAM_NON_BLOCKING)); + CUDA_DRV(cuCtxPopCurrent(nullptr)); + + CUDA_DRV(cuCtxFromGreenCtx(&ctx, gctx[1])); + CUDA_DRV(cuCtxPushCurrent(ctx)); + CUDA_DRV(cuStreamCreate(&streamB, CU_STREAM_NON_BLOCKING)); + CUDA_DRV(cuCtxPopCurrent(nullptr)); + + return {(int64_t)streamA, (int64_t)streamB}; +} + +inline void destroy_green_context(CUgreenCtx gctx) { + if (!gctx) return; + CUDA_DRV(cuGreenCtxDestroy(gctx)); +} + +static std::vector create_greenctx_stream_direct_dynamic(CUgreenCtx gctx[2]) { + // This symbol is introduced in CUDA 12.5 + const static auto pfn = probe_cuGreenCtxStreamCreate(); + if (!pfn) { + TORCH_WARN("cuGreenCtxStreamCreate(cuda>=12.5) is not available, using fallback"); + return create_greenctx_stream_fallback(gctx); + } + + CUstream streamA, streamB; + CUDA_DRV(pfn(&streamA, gctx[0], CU_STREAM_NON_BLOCKING, 0)); + CUDA_DRV(pfn(&streamB, gctx[1], CU_STREAM_NON_BLOCKING, 0)); + + return {(int64_t)streamA, (int64_t)streamB}; +} + +std::vector create_greenctx_stream_by_value(int64_t smA, int64_t smB, int64_t device) { + CUDA_DRV(cuDriverGetVersion(&CUDA_DRIVER_VERSION)); + + CUgreenCtx gctx[3]; + CUdevResourceDesc desc[3]; + CUdevResource input; + CUdevResource resources[4]; + + TORCH_CHECK(smA > 0 && smB > 0, "SM counts must be positive"); + + CUDA_DRV(cuDeviceGetDevResource((CUdevice)device, &input, CU_DEV_RESOURCE_TYPE_SM)); + + const unsigned minCount = static_cast(smA + smB); + const unsigned minCountA = static_cast(smA); + TORCH_CHECK(minCount <= input.sm.smCount, "Not enough SMs available for the requested configuration"); + + unsigned nbGroups = 1; + CUDA_DRV(cuDevSmResourceSplitByCount(&resources[2], &nbGroups, &input, &resources[3], 0, minCount)); + CUDA_DRV(cuDevResourceGenerateDesc(&desc[2], &resources[2], 1)); + CUDA_DRV(cuGreenCtxCreate(&gctx[2], desc[2], (CUdevice)device, CU_GREEN_CTX_DEFAULT_STREAM)); + CUDA_DRV(cuGreenCtxGetDevResource(gctx[2], &input, CU_DEV_RESOURCE_TYPE_SM)); + nbGroups = 1; + CUDA_DRV(cuDevSmResourceSplitByCount(&resources[0], &nbGroups, &input, &resources[1], 0, minCountA)); + CUDA_DRV(cuDevResourceGenerateDesc(&desc[0], &resources[0], 1)); + CUDA_DRV(cuGreenCtxCreate(&gctx[0], desc[0], (CUdevice)device, CU_GREEN_CTX_DEFAULT_STREAM)); + CUDA_DRV(cuDevResourceGenerateDesc(&desc[1], &resources[1], 1)); + CUDA_DRV(cuGreenCtxCreate(&gctx[1], desc[1], (CUdevice)device, CU_GREEN_CTX_DEFAULT_STREAM)); + + const int smCountA = resources[0].sm.smCount; + const int smCountB = resources[1].sm.smCount; + + std::vector streams = create_greenctx_stream_direct_dynamic(gctx); + + destroy_green_context(gctx[2]); + + std::vector vec = { + streams[0], // streamA + streams[1], // streamB + (int64_t)smCountA, + (int64_t)smCountB}; + + return vec; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/spatial/greenctx_stream.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/spatial/greenctx_stream.h new file mode 100644 index 00000000..2577e9f2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/spatial/greenctx_stream.h @@ -0,0 +1,2 @@ +#include +std::vector create_greenctx_stream_by_value(int64_t smA, int64_t smB, int64_t device); diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/spatial_extension.cc b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/spatial_extension.cc new file mode 100644 index 00000000..27833cd7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/spatial_extension.cc @@ -0,0 +1,29 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "sgl_kernel_ops.h" + +TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) { + /* + * From csrc/spatial + */ + m.def("create_greenctx_stream_by_value(int smA, int smB, int device) -> int[]"); + m.impl("create_greenctx_stream_by_value", &create_greenctx_stream_by_value); +} + +REGISTER_EXTENSION(spatial_ops) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/speculative/eagle_utils.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/speculative/eagle_utils.cu new file mode 100644 index 00000000..e8e30632 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/speculative/eagle_utils.cu @@ -0,0 +1,407 @@ +/* + * Copyright (c) 2025 by SGLang team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#ifndef USE_ROCM +#include "pytorch_extension_utils.h" +#else +#include "pytorch_extension_utils_rocm.h" +#endif + +typedef enum { FULL_MASK = 0, QLEN_ONLY = 1, QLEN_ONLY_BITPACKING = 2 } TreeMaskMode; + +// parent_list [bs, topk * (depth - 1) + 1)] +// selected_index [bs, draft_token_num - 1] +// verified_seq_len [bs] +// tree_mask [draft_token*(seq_len[0]+draft_token) | draft_token*(seq_len[1]+draft_token) | ..] = +// [sum(verified_seq_len)*draft_token+bs*draft_token*draft_token] positions [bs * draft_token] retrive_index [b, +// draft_token] retrive_next_token [b, draft_token] retrive_next_sibling [b, draft_token] +__global__ void build_tree_efficient( + int64_t* parent_list, + int64_t* selected_index, + int64_t* verified_seq_len, + bool* tree_mask, + int64_t* positions, + int64_t* retrive_index, + int64_t* retrive_next_token, + int64_t* retrive_next_sibling, + int topk, + int depth, + int draft_token_num, + int tree_mask_mode) { + int bid = blockIdx.x; + int tid = threadIdx.x; + + if (tid >= draft_token_num) { + return; + } + int seq_tree_idx = draft_token_num * draft_token_num * bid; + for (int i = 0; i < bid; i++) { + seq_tree_idx += verified_seq_len[i] * draft_token_num; + } + int seq_len = verified_seq_len[bid]; + int token_tree_idx; + if (tree_mask_mode == FULL_MASK) { + token_tree_idx = seq_tree_idx + (seq_len + draft_token_num) * tid + seq_len + 1; + } else { + token_tree_idx = draft_token_num * draft_token_num * bid + draft_token_num * tid + 1; + } + tree_mask[token_tree_idx - 1] = true; + for (int i = 0; i < draft_token_num - 1; i++) { + tree_mask[token_tree_idx + i] = false; + } + + int position = 0; + if (tid == 0) { + positions[bid * draft_token_num] = seq_len; + + int retrive_index_offset = bid * draft_token_num; + for (int i = draft_token_num - 1; i > 0; --i) { + int current_token_idx = retrive_index_offset + i; + retrive_index[bid * draft_token_num + i] = current_token_idx; + int parent_tb_idx = selected_index[bid * (draft_token_num - 1) + i - 1] / topk; + int parent_position = 0; + if (parent_tb_idx > 0) { + int parent_token_idx = parent_list[bid * (topk * (depth - 1) + 1) + parent_tb_idx]; + for (; parent_position < draft_token_num; ++parent_position) { + if (selected_index[bid * (draft_token_num - 1) + parent_position] == parent_token_idx) { + ++parent_position; + break; + } + } + } + if (parent_position == draft_token_num) { + printf( + "WARNING: invalid eagle tree!!! Detected a token with no parent token selected. " + "Please check if the logprob has nan. The token will be ignored to keep proceeding.\n"); + continue; + } + + if (retrive_next_token[bid * draft_token_num + parent_position] == -1) { + retrive_next_token[bid * draft_token_num + parent_position] = i; + } else { + int origin_next_token = retrive_next_token[bid * draft_token_num + parent_position]; + retrive_next_token[bid * draft_token_num + parent_position] = i; + retrive_next_sibling[bid * draft_token_num + i] = origin_next_token; + } + } + retrive_index[bid * draft_token_num] = bid * draft_token_num; + } else { + int cur_position = tid - 1; + while (true) { + position += 1; + tree_mask[token_tree_idx + cur_position] = true; + int parent_tb_idx = selected_index[bid * (draft_token_num - 1) + cur_position] / topk; + if (parent_tb_idx == 0) { + break; + } + + int token_idx = parent_list[bid * (topk * (depth - 1) + 1) + parent_tb_idx]; + for (cur_position = 0; cur_position < draft_token_num; ++cur_position) { + if (selected_index[bid * (draft_token_num - 1) + cur_position] == token_idx) { + break; + } + } + } + positions[bid * draft_token_num + tid] = position + seq_len; + } +} + +// parent_list [bs, topk * (depth - 1) + 1)] +// selected_index [bs, draft_token_num - 1] +// verified_seq_len [bs] +// tree_mask: [draft_token*num_bytes_per_item | .. ] = [bs*draft_token*num_bytes_per_item] +// positions [bs * draft_token] +// retrive_index [bs, draft_token] +// retrive_next_token [bs, draft_token] +// retrive_next_sibling [bs, draft_token] +__global__ void build_tree_efficient_partial_packed( + int64_t* parent_list, + int64_t* selected_index, + int64_t* verified_seq_len, + uint8_t* tree_mask, + int64_t* positions, + int64_t* retrive_index, + int64_t* retrive_next_token, + int64_t* retrive_next_sibling, + int topk, + int depth, + int draft_token_num, + size_t num_bytes_per_item) { + int bid = blockIdx.x; + int tid = threadIdx.x; + + if (tid >= draft_token_num) { + return; + } + int seq_len = verified_seq_len[bid]; + int token_tree_idx = (bid * draft_token_num + tid) * num_bytes_per_item; + tree_mask[token_tree_idx] = 1; // little endian + + int position = 0; + if (tid == 0) { + positions[bid * draft_token_num] = seq_len; + + int retrive_index_offset = bid * draft_token_num; + for (int i = draft_token_num - 1; i > 0; --i) { + int current_token_idx = retrive_index_offset + i; + retrive_index[bid * draft_token_num + i] = current_token_idx; + int parent_tb_idx = selected_index[bid * (draft_token_num - 1) + i - 1] / topk; + int parent_position = 0; + if (parent_tb_idx > 0) { + int parent_token_idx = parent_list[bid * (topk * (depth - 1) + 1) + parent_tb_idx]; + for (; parent_position < draft_token_num; ++parent_position) { + if (selected_index[bid * (draft_token_num - 1) + parent_position] == parent_token_idx) { + ++parent_position; + break; + } + } + } + if (parent_position == draft_token_num) { + printf( + "WARNING: invalid eagle tree!!! Detected a token with no parent token selected. " + "Please check if the logprob has nan. The token will be ignored to keep proceeding.\n"); + continue; + } + + if (retrive_next_token[bid * draft_token_num + parent_position] == -1) { + retrive_next_token[bid * draft_token_num + parent_position] = i; + } else { + int origin_next_token = retrive_next_token[bid * draft_token_num + parent_position]; + retrive_next_token[bid * draft_token_num + parent_position] = i; + retrive_next_sibling[bid * draft_token_num + i] = origin_next_token; + } + } + retrive_index[bid * draft_token_num] = bid * draft_token_num; + } else { + int cur_position = tid - 1; + while (true) { + position += 1; + int byte_idx = (cur_position + 1) / 8; + int bit_idx = (cur_position + 1) % 8; + tree_mask[token_tree_idx + byte_idx] |= (1 << bit_idx); + int parent_tb_idx = selected_index[bid * (draft_token_num - 1) + cur_position] / topk; + if (parent_tb_idx == 0) { + break; + } + + int token_idx = parent_list[bid * (topk * (depth - 1) + 1) + parent_tb_idx]; + for (cur_position = 0; cur_position < draft_token_num; ++cur_position) { + if (selected_index[bid * (draft_token_num - 1) + cur_position] == token_idx) { + break; + } + } + } + positions[bid * draft_token_num + tid] = position + seq_len; + } +} + +void build_tree_kernel_efficient( + at::Tensor parent_list, + at::Tensor selected_index, + at::Tensor verified_seq_len, + at::Tensor tree_mask, + at::Tensor positions, + at::Tensor retrive_index, + at::Tensor retrive_next_token, + at::Tensor retrive_next_sibling, + int64_t topk, + int64_t depth, + int64_t draft_token_num, + int64_t tree_mask_mode) { + // TODO (ying) check shape + // TODO (ying) check type + int bs = parent_list.size(0); + dim3 grid(bs); + dim3 block(draft_token_num); + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + if (tree_mask_mode == QLEN_ONLY_BITPACKING) { + size_t num_bytes_per_item = 1; + if (draft_token_num > 16) { + num_bytes_per_item = 4; + } else if (draft_token_num > 8) { + num_bytes_per_item = 2; + } + build_tree_efficient_partial_packed<<>>( + static_cast(parent_list.data_ptr()), + static_cast(selected_index.data_ptr()), + static_cast(verified_seq_len.data_ptr()), + static_cast(tree_mask.data_ptr()), + static_cast(positions.data_ptr()), + static_cast(retrive_index.data_ptr()), + static_cast(retrive_next_token.data_ptr()), + static_cast(retrive_next_sibling.data_ptr()), + int32_t(topk), + int32_t(depth), + int32_t(draft_token_num), + num_bytes_per_item); + } else { + build_tree_efficient<<>>( + static_cast(parent_list.data_ptr()), + static_cast(selected_index.data_ptr()), + static_cast(verified_seq_len.data_ptr()), + static_cast(tree_mask.data_ptr()), + static_cast(positions.data_ptr()), + static_cast(retrive_index.data_ptr()), + static_cast(retrive_next_token.data_ptr()), + static_cast(retrive_next_sibling.data_ptr()), + int32_t(topk), + int32_t(depth), + int32_t(draft_token_num), + int32_t(tree_mask_mode)); + } +} + +template +__global__ void VerifyTreeGreedy( + IdType* predicts, + IdType* accept_index, + IdType* accept_token_num, // mutable + IdType2* candidates, + IdType2* retrive_index, + IdType2* retrive_next_token, + IdType2* retrive_next_sibling, + IdType2* target_predict, + uint32_t batch_size, + uint32_t num_speculative_tokens, + uint32_t num_draft_tokens) { + uint32_t bx = blockIdx.x; + + IdType2 last_accepted_retrive_idx = retrive_index[bx * num_draft_tokens]; + accept_index[bx * num_speculative_tokens] = last_accepted_retrive_idx; + uint32_t num_accepted_tokens = 0; + IdType2 cur_index = 0; + + for (uint32_t j = 1; j < num_speculative_tokens; ++j) { + cur_index = retrive_next_token[bx * num_draft_tokens + cur_index]; + while (cur_index != -1) { + IdType2 draft_index = retrive_index[bx * num_draft_tokens + cur_index]; + IdType2 draft_token_id = candidates[bx * num_draft_tokens + cur_index]; + IdType2 target_token_id = target_predict[last_accepted_retrive_idx]; + + if (draft_token_id == target_token_id) { + // accept token + predicts[last_accepted_retrive_idx] = target_token_id; + ++num_accepted_tokens; + accept_index[bx * num_speculative_tokens + num_accepted_tokens] = draft_index; + last_accepted_retrive_idx = draft_index; + break; + } else { + cur_index = retrive_next_sibling[bx * num_draft_tokens + cur_index]; + } + } + if (cur_index == -1) break; + } + accept_token_num[bx] = num_accepted_tokens; + predicts[last_accepted_retrive_idx] = target_predict[last_accepted_retrive_idx]; +} + +// predicts: [tot_num_draft_tokens] +// accept_index: [bs, num_spec_step] +// accept_token_num: [bs] +// candidates: [bs, num_draft_tokens] +// retrive_index: [bs, num_draft_tokens] +// retrive_next_token: [bs, num_draft_tokens] +// retrive_next_sibling: [bs, num_draft_tokens] +// target_predict: [bs, num_draft_tokens] +void verify_tree_greedy( + at::Tensor predicts, + at::Tensor accept_index, + at::Tensor accept_token_num, // mutable + at::Tensor candidates, + at::Tensor retrive_index, + at::Tensor retrive_next_token, + at::Tensor retrive_next_sibling, + at::Tensor target_predict) { + CHECK_INPUT(candidates); + CHECK_INPUT(retrive_index); + CHECK_INPUT(retrive_next_token); + CHECK_INPUT(retrive_next_sibling); + CHECK_INPUT(target_predict); + auto device = target_predict.device(); + CHECK_EQ(candidates.device(), device); + CHECK_EQ(retrive_index.device(), device); + CHECK_EQ(retrive_next_token.device(), device); + CHECK_EQ(retrive_next_sibling.device(), device); + CHECK_EQ(target_predict.device(), device); + CHECK_DIM(1, predicts); + CHECK_DIM(2, accept_index); + CHECK_DIM(1, accept_token_num); + CHECK_DIM(2, candidates); + CHECK_DIM(2, retrive_index); + CHECK_DIM(2, retrive_next_token); + CHECK_DIM(2, retrive_next_sibling); + CHECK_DIM(2, target_predict); + unsigned int batch_size = candidates.size(0); + unsigned int num_spec_step = accept_index.size(1); + unsigned int num_draft_tokens = candidates.size(1); + CHECK_EQ(batch_size, accept_index.size(0)); + CHECK_EQ(batch_size, accept_token_num.size(0)); + CHECK_EQ(batch_size, retrive_index.size(0)); + CHECK_EQ(batch_size, retrive_next_token.size(0)); + CHECK_EQ(batch_size, retrive_next_sibling.size(0)); + CHECK_EQ(batch_size, target_predict.size(0)); + CHECK_EQ(num_draft_tokens, retrive_index.size(1)); + CHECK_EQ(num_draft_tokens, retrive_next_token.size(1)); + CHECK_EQ(num_draft_tokens, retrive_next_sibling.size(1)); + CHECK_EQ(num_draft_tokens, target_predict.size(1)); + CHECK_EQ(batch_size, accept_index.size(0)); + CHECK_EQ(batch_size, accept_token_num.size(0)); + if (predicts.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'predicts' to be of type int (torch.int32)."); + } + if (accept_index.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'accept_index' to be of type int (torch.int32)."); + } + if (accept_token_num.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'accept_token_num' to be of type int (torch.int32)."); + } + if (candidates.scalar_type() != at::kLong) { + throw std::runtime_error("Expected 'candidates' to be of type long (torch.int64)."); + } + if (retrive_index.scalar_type() != at::kLong) { + throw std::runtime_error("Expected 'retrive_index' to be of type long (torch.int64)."); + } + if (retrive_next_token.scalar_type() != at::kLong) { + throw std::runtime_error("Expected 'retrive_next_token' to be of type long (torch.int64)."); + } + if (retrive_next_sibling.scalar_type() != at::kLong) { + throw std::runtime_error("Expected 'retrive_next_sibling' to be of type long (torch.int64)."); + } + if (target_predict.scalar_type() != at::kLong) { + throw std::runtime_error("Expected 'target_predict' to be of type long (torch.int64)."); + } + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + dim3 grid(batch_size); + dim3 block(1); + + VerifyTreeGreedy<<>>( + static_cast(predicts.data_ptr()), + static_cast(accept_index.data_ptr()), + static_cast(accept_token_num.data_ptr()), + static_cast(candidates.data_ptr()), + static_cast(retrive_index.data_ptr()), + static_cast(retrive_next_token.data_ptr()), + static_cast(retrive_next_sibling.data_ptr()), + static_cast(target_predict.data_ptr()), + batch_size, + num_spec_step, + num_draft_tokens); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/speculative/ngram_utils.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/speculative/ngram_utils.cu new file mode 100644 index 00000000..b5105422 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/speculative/ngram_utils.cu @@ -0,0 +1,105 @@ +#include +#include + +#ifndef USE_ROCM +#include "pytorch_extension_utils.h" +#else +#include "pytorch_extension_utils_rocm.h" +#endif + +// tree_mask: [bs * draft_token_num * draft_token_num] +// verified_seq_len: [bs] +// positions: [bs * draft_token_num] +// retrive_index: [bs, draft_token_num] +// retrive_next_token: [bs, draft_token_num] +// retrive_next_sibling: [bs, draft_token_num] +__global__ void reconstructIndicesFromTreeMask( + bool* tree_mask, + int64_t* verified_seq_len, + int64_t* positions, + int64_t* retrive_index, + int64_t* retrive_next_token, + int64_t* retrive_next_sibling, + int batch_size, + int draft_token_num) { + int bid = blockIdx.x; + int tid = threadIdx.x; + + if (bid >= batch_size || tid >= draft_token_num) { + return; + } + int base_offset = draft_token_num * draft_token_num; + // token_idx: [bid * draft_token_num, (bid + 1) * draft_token_num) + int token_idx = bid * draft_token_num; + // tree_mask_idx: [bid * base_offset, (bid + 1) * base_offset) + int tree_mask_offset = bid * base_offset; + + int depth = 0; + int parent_idx = -1; + + for (int i = tid - 1, start_idx = tree_mask_offset + tid * draft_token_num; i >= 0; i--) { + if (tree_mask[start_idx + i]) { + depth++; + if (parent_idx == -1) { + parent_idx = i; + } + } + } + retrive_index[token_idx + tid] = token_idx + tid; + positions[token_idx + tid] = depth + verified_seq_len[bid]; + + int next_token_idx = -1; + for (int i = tid + 1; i < draft_token_num; i++) { + if (tree_mask[tree_mask_offset + i * draft_token_num + tid]) { + next_token_idx = i; + break; + } + } + retrive_next_token[token_idx + tid] = next_token_idx; + + int next_sibling_idx = -1; + if (parent_idx != -1) { + for (int i = tid + 1; i < draft_token_num; i++) { + int start_idx = tree_mask_offset + i * draft_token_num + parent_idx; + if (tree_mask[start_idx]) { + bool is_sibling = true; + int end_idx = tree_mask_offset + i * draft_token_num + i; + for (int j = start_idx + 1; j < end_idx; ++j) { + if (tree_mask[j]) { + is_sibling = false; + break; + } + } + if (is_sibling) { + next_sibling_idx = i; + break; + } + } + } + } + retrive_next_sibling[token_idx + tid] = next_sibling_idx; +} + +void reconstruct_indices_from_tree_mask( + at::Tensor tree_mask, + at::Tensor verified_seq_len, + at::Tensor positions, + at::Tensor retrive_index, + at::Tensor retrive_next_token, + at::Tensor retrive_next_sibling, + int64_t batch_size, + int64_t draft_token_num) { + dim3 grid(batch_size); + dim3 block(draft_token_num); + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + reconstructIndicesFromTreeMask<<>>( + static_cast(tree_mask.data_ptr()), + static_cast(verified_seq_len.data_ptr()), + static_cast(positions.data_ptr()), + static_cast(retrive_index.data_ptr()), + static_cast(retrive_next_token.data_ptr()), + static_cast(retrive_next_sibling.data_ptr()), + int(batch_size), + int(draft_token_num)); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/speculative/packbit.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/speculative/packbit.cu new file mode 100644 index 00000000..1decc3de --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/speculative/packbit.cu @@ -0,0 +1,51 @@ +// This is only a plugin used for flashinfer 0.1.6. The new version does not need it. +/* + * Copyright (c) 2025 by SGLang team. + * Copyright (c) 2025 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "pytorch_extension_utils.h" + +using namespace flashinfer; + +// bitorder = "little" +void segment_packbits( + at::Tensor x, + at::Tensor input_indptr, + at::Tensor output_indptr, + at::Tensor y, + int64_t batch_size, + int64_t cuda_stream) { + CHECK_INPUT(x); + CHECK_INPUT(input_indptr); + CHECK_INPUT(output_indptr); + auto device = x.device(); + CHECK_EQ(input_indptr.device(), device); + CHECK_EQ(output_indptr.device(), device); + CHECK_EQ(y.device(), device); + CHECK_GE(output_indptr.size(0), batch_size + 1); + + cudaStream_t stream = reinterpret_cast(cuda_stream); + cudaError_t status = quantization::SegmentPackBits( + static_cast(x.data_ptr()), + static_cast(y.data_ptr()), + static_cast(input_indptr.data_ptr()), + static_cast(output_indptr.data_ptr()), + batch_size, + quantization::BitOrder::kLittle, + stream); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/speculative/speculative_sampling.cu b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/speculative/speculative_sampling.cu new file mode 100644 index 00000000..7a1400f8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/speculative/speculative_sampling.cu @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2025 by SGLang team. + * Copyright (c) 2025 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pytorch_extension_utils.h" +#include "speculative_sampling.cuh" + +using namespace flashinfer; + +// predicts: [tot_num_draft_tokens] +// accept_index: [bs, num_spec_step] +// accept_token_num: [bs] +// candidates: [bs, num_draft_tokens] +// retrive_index: [bs, num_draft_tokens] +// retrive_next_token: [bs, num_draft_tokens] +// retrive_next_sibling: [bs, num_draft_tokens] +// uniform_samples: [bs, num_draft_tokens] +// target_probs: [bs, num_draft_tokens, vocab_size] +void tree_speculative_sampling_target_only( + at::Tensor predicts, + at::Tensor accept_index, + at::Tensor accept_token_num, // mutable + at::Tensor candidates, + at::Tensor retrive_index, + at::Tensor retrive_next_token, + at::Tensor retrive_next_sibling, + at::Tensor uniform_samples, + at::Tensor uniform_samples_for_final_sampling, + at::Tensor target_probs, + at::Tensor draft_probs, + double threshold_single, + double threshold_acc, + bool deterministic = true) { + CHECK_INPUT(candidates); + CHECK_INPUT(retrive_index); + CHECK_INPUT(retrive_next_token); + CHECK_INPUT(retrive_next_sibling); + CHECK_INPUT(uniform_samples); + CHECK_INPUT(uniform_samples_for_final_sampling); + CHECK_INPUT(target_probs); + auto device = target_probs.device(); + CHECK_EQ(candidates.device(), device); + CHECK_EQ(retrive_index.device(), device); + CHECK_EQ(retrive_next_token.device(), device); + CHECK_EQ(retrive_next_sibling.device(), device); + CHECK_EQ(uniform_samples.device(), device); + CHECK_EQ(uniform_samples_for_final_sampling.device(), device); + CHECK_EQ(target_probs.device(), device); + CHECK_DIM(1, predicts); + CHECK_DIM(2, accept_index); + CHECK_DIM(1, accept_token_num); + CHECK_DIM(2, candidates); + CHECK_DIM(2, retrive_index); + CHECK_DIM(2, retrive_next_token); + CHECK_DIM(2, retrive_next_sibling); + CHECK_DIM(2, uniform_samples); + CHECK_DIM(3, target_probs); + CHECK_DIM(3, draft_probs); + unsigned int batch_size = uniform_samples.size(0); + unsigned int num_spec_step = accept_index.size(1); + unsigned int num_draft_tokens = candidates.size(1); + unsigned int vocab_size = target_probs.size(2); + CHECK_EQ(batch_size, candidates.size(0)); + CHECK_EQ(batch_size, retrive_index.size(0)); + CHECK_EQ(batch_size, retrive_next_token.size(0)); + CHECK_EQ(batch_size, retrive_next_sibling.size(0)); + CHECK_EQ(batch_size, target_probs.size(0)); + CHECK_EQ(num_draft_tokens, retrive_index.size(1)); + CHECK_EQ(num_draft_tokens, retrive_next_token.size(1)); + CHECK_EQ(num_draft_tokens, retrive_next_sibling.size(1)); + CHECK_EQ(num_draft_tokens, uniform_samples.size(1)); + CHECK_EQ(num_draft_tokens, target_probs.size(1)); + CHECK_EQ(vocab_size, target_probs.size(2)); + CHECK_EQ(batch_size, accept_index.size(0)); + CHECK_EQ(batch_size, accept_token_num.size(0)); + if (predicts.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'predicts' to be of type int (torch.int32)."); + } + if (accept_index.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'accept_index' to be of type int (torch.int32)."); + } + if (accept_token_num.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'accept_token_num' to be of type int (torch.int32)."); + } + if (candidates.scalar_type() != at::kLong) { + throw std::runtime_error("Expected 'candidates' to be of type long (torch.int64)."); + } + if (retrive_index.scalar_type() != at::kLong) { + throw std::runtime_error("Expected 'retrive_index' to be of type long (torch.int64)."); + } + if (retrive_next_token.scalar_type() != at::kLong) { + throw std::runtime_error("Expected 'retrive_next_token' to be of type long (torch.int64)."); + } + if (retrive_next_sibling.scalar_type() != at::kLong) { + throw std::runtime_error("Expected 'retrive_next_sibling' to be of type long (torch.int64)."); + } + if (uniform_samples.scalar_type() != at::kFloat) { + throw std::runtime_error("Expected 'uniform_samples' to be of type float (torch.float32)."); + } + if (uniform_samples_for_final_sampling.scalar_type() != at::kFloat) { + throw std::runtime_error("Expected 'uniform_samples_for_final_sampling' to be of type float (torch.float32)."); + } + if (target_probs.scalar_type() != at::kFloat) { + throw std::runtime_error("Expected 'target_probs' to be of type float (torch.float32)."); + } + if (draft_probs.scalar_type() != at::kFloat) { + throw std::runtime_error("Expected 'target_probs' to be of type float (torch.float32)."); + } + CHECK_GE(threshold_single, 0); + CHECK_GE(1, threshold_single); + CHECK_GE(threshold_acc, 0); + CHECK_GE(1, threshold_acc); + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + cudaError_t status = sampling::TreeSpeculativeSamplingTargetOnly( + static_cast(predicts.data_ptr()), + static_cast(accept_index.data_ptr()), + static_cast(accept_token_num.data_ptr()), + static_cast(candidates.data_ptr()), + static_cast(retrive_index.data_ptr()), + static_cast(retrive_next_token.data_ptr()), + static_cast(retrive_next_sibling.data_ptr()), + static_cast(uniform_samples.data_ptr()), + static_cast(uniform_samples_for_final_sampling.data_ptr()), + static_cast(target_probs.data_ptr()), + static_cast(draft_probs.data_ptr()), + batch_size, + num_spec_step, + num_draft_tokens, + vocab_size, + static_cast(threshold_single), + static_cast(threshold_acc), + deterministic, + stream); + + TORCH_CHECK( + status == cudaSuccess, + "TreeSpeculativeSamplingTargetOnly failed with error code " + std::string(cudaGetErrorString(status))); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/speculative/speculative_sampling.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/speculative/speculative_sampling.cuh new file mode 100644 index 00000000..59f18bc2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/csrc/speculative/speculative_sampling.cuh @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2025 by SGLang team. + * Copyright (c) 2024-2025 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SPECULATIVE_SAMPLING_CUH_ +#define SPECULATIVE_SAMPLING_CUH_ + +#include + +#include + +namespace flashinfer { + +namespace sampling { + +using namespace cub; + +template < + uint32_t BLOCK_THREADS, + BlockScanAlgorithm SCAN_ALGORITHM, + BlockReduceAlgorithm REDUCE_ALGORITHM, + uint32_t VEC_SIZE, + bool DETERMINISTIC, + typename DType, + typename IdType, + typename IdType2> +__global__ void TreeSpeculativeSamplingTargetOnly( + IdType* predicts, // mutable + IdType* accept_index, // mutable + IdType* accept_token_num, // mutable + IdType2* candidates, + IdType2* retrive_index, + IdType2* retrive_next_token, + IdType2* retrive_next_sibling, + DType* uniform_samples, + DType* uniform_samples_for_final_sampling, + DType* target_probs, + DType* draft_probs, + uint32_t batch_size, + uint32_t num_speculative_tokens, + uint32_t num_draft_tokens, + uint32_t d, + DType threshold_single, + DType threshold_acc) { + const uint32_t bx = blockIdx.x, tx = threadIdx.x; + + extern __shared__ __align__(alignof(SamplingTempStorage)) + uint8_t smem_sampling[]; + auto& temp_storage = + reinterpret_cast&>(smem_sampling); + + DType prob_acc = 0.0; + uint32_t cur_prob_offset = bx * num_draft_tokens * d; + DType coin = uniform_samples[bx * num_draft_tokens]; + IdType2 last_accepted_retrive_idx = retrive_index[bx * num_draft_tokens]; + accept_index[bx * num_speculative_tokens] = last_accepted_retrive_idx; + uint32_t num_accepted_tokens = 0; + IdType2 cur_index = 0; + + for (uint32_t j = 1; j < num_speculative_tokens; ++j) { + cur_index = retrive_next_token[bx * num_draft_tokens + cur_index]; + while (cur_index != -1) { + IdType2 draft_index = retrive_index[bx * num_draft_tokens + cur_index]; + IdType2 draft_token_id = candidates[bx * num_draft_tokens + cur_index]; + DType target_prob_single = target_probs[cur_prob_offset + draft_token_id]; + prob_acc += target_prob_single; + + if (coin <= prob_acc / threshold_acc || target_prob_single >= threshold_single) { + // accept token + prob_acc = 0.; + cur_prob_offset = (bx * num_draft_tokens + cur_index) * d; + coin = uniform_samples[bx * num_draft_tokens + cur_index]; + predicts[last_accepted_retrive_idx] = draft_token_id; + ++num_accepted_tokens; + accept_index[bx * num_speculative_tokens + num_accepted_tokens] = draft_index; + last_accepted_retrive_idx = draft_index; + break; + } else { + // FIXME: leverage draft probs + draft_probs[cur_prob_offset + draft_token_id] = target_probs[cur_prob_offset + draft_token_id]; + cur_index = retrive_next_sibling[bx * num_draft_tokens + cur_index]; + } + } + if (cur_index == -1) break; + } + accept_token_num[bx] = num_accepted_tokens; + + // we need a different coin for the final sampling + coin = uniform_samples_for_final_sampling[bx]; + + // sample from relu(target_probs - draft_probs) + DType sum_relu_q_minus_p(0); + vec_t q_vec, p_vec; + DType relu_q_minus_p[VEC_SIZE]; + for (uint32_t i = 0; i < ceil_div(d, BLOCK_THREADS * VEC_SIZE); ++i) { + q_vec.fill(DType(0)); + p_vec.fill(DType(0)); + if ((i * BLOCK_THREADS + tx) * VEC_SIZE < d) { + q_vec.load(target_probs + cur_prob_offset + i * BLOCK_THREADS * VEC_SIZE + tx * VEC_SIZE); + if (num_accepted_tokens != num_speculative_tokens - 1) { + // there is no draft_probs for the bonus token + p_vec.load(draft_probs + cur_prob_offset + i * BLOCK_THREADS * VEC_SIZE + tx * VEC_SIZE); + } + } +#pragma unroll + for (uint32_t j = 0; j < VEC_SIZE; ++j) { + relu_q_minus_p[j] = max(q_vec[j] - p_vec[j], DType(0)); + } + sum_relu_q_minus_p += BlockReduce(temp_storage.block_prim.reduce) + .Sum(relu_q_minus_p); + __syncthreads(); + } + if (tx == 0) { + temp_storage.block_aggregate.value = sum_relu_q_minus_p; + } + // init the first rejected token to (d - 1) + temp_storage.sampled_id = d - 1; + __syncthreads(); + sum_relu_q_minus_p = temp_storage.block_aggregate.value; + DType u = coin * sum_relu_q_minus_p; + + DType aggregate_relu_q_minus_p(0); + for (uint32_t i = 0; i < ceil_div(d, BLOCK_THREADS * VEC_SIZE); ++i) { + q_vec.fill(DType(0)); + p_vec.fill(DType(0)); + if ((i * BLOCK_THREADS + tx) * VEC_SIZE < d) { + q_vec.load(target_probs + cur_prob_offset + i * BLOCK_THREADS * VEC_SIZE + tx * VEC_SIZE); + if (num_accepted_tokens != num_speculative_tokens - 1) { + // there is no draft_probs for the bonus token + p_vec.load(draft_probs + cur_prob_offset + i * BLOCK_THREADS * VEC_SIZE + tx * VEC_SIZE); + } + } + + vec_t relu_q_minus_p_vec; +#pragma unroll + for (uint32_t j = 0; j < VEC_SIZE; ++j) { + relu_q_minus_p_vec[j] = max(q_vec[j] - p_vec[j], DType(0)); + } + + DeviceSamplingFromProb( + i, d, [&](DType x) { return x > 0; }, u, relu_q_minus_p_vec, aggregate_relu_q_minus_p, &temp_storage); + if (aggregate_relu_q_minus_p > u) { + break; + } + } + __syncthreads(); + // set the first rejected token + predicts[last_accepted_retrive_idx] = temp_storage.sampled_id; + // value at not used indices are undefined +} + +template +cudaError_t TreeSpeculativeSamplingTargetOnly( + IdType* predicts, // mutable + IdType* output_token_ids, // mutable + IdType* output_accepted_token_num, // mutable + IdType2* candidates, + IdType2* retrive_index, + IdType2* retrive_next_token, + IdType2* retrive_next_sibling, + DType* uniform_samples, + DType* uniform_samples_for_final_sampling, + DType* target_probs, + DType* draft_probs, + uint32_t batch_size, + uint32_t num_speculative_tokens, + uint32_t num_draft_tokens, + uint32_t d, + DType threshold_single = 1, + DType threshold_acc = 1, + bool deterministic = true, + cudaStream_t stream = 0) { + constexpr uint32_t BLOCK_THREADS = 1024; + const uint32_t vec_size = std::gcd(16 / sizeof(DType), d); + + const uint32_t smem_size = sizeof(SamplingTempStorage); + dim3 nblks(batch_size); + dim3 nthrs(BLOCK_THREADS); + float capped_threshold_acc = fmaxf(threshold_acc, 1e-9f); + void* args[] = { + &predicts, + &output_token_ids, + &output_accepted_token_num, + &candidates, + &retrive_index, + &retrive_next_token, + &retrive_next_sibling, + &uniform_samples, + &uniform_samples_for_final_sampling, + &target_probs, + &draft_probs, + &batch_size, + &num_speculative_tokens, + &num_draft_tokens, + &d, + &threshold_single, + &capped_threshold_acc}; + DISPATCH_ALIGNED_VEC_SIZE( + vec_size, VEC_SIZE, {DISPATCH_DETERMINISTIC(deterministic, DETERMINISTIC, { + auto kernel = TreeSpeculativeSamplingTargetOnly< + BLOCK_THREADS, + SCAN_ALGO, + REDUCE_ALGO, + VEC_SIZE, + DETERMINISTIC, + DType, + IdType, + IdType2>; + FLASHINFER_CUDA_CALL(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + FLASHINFER_CUDA_CALL(cudaLaunchKernel((void*)kernel, nblks, nthrs, args, smem_size, stream)); + })}); + return cudaSuccess; +} + +} // namespace sampling + +} // namespace flashinfer + +#endif // SPECULATIVE_SAMPLING_CUH_ diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/hip_act_and_mul.cuh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/hip_act_and_mul.cuh new file mode 100644 index 00000000..ddb1b702 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/hip_act_and_mul.cuh @@ -0,0 +1,87 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#pragma once + +#include "utils.h" + +#define kBitsToLoad 128 +#define kBytesToLoad (kBitsToLoad / 8) + +// Adapted from +// [flashinfer::activation::act_and_mul_kernel](https://github.com/flashinfer-ai/flashinfer/blob/4e8eb1879f9c3ba6d75511e5893183bf8f289a62/include/flashinfer/activation.cuh#L29) + +namespace sgl_hip { +namespace activation { + +template +__global__ void act_and_mul_kernel(T* __restrict__ out, const T* __restrict__ input, const int d) { + constexpr uint32_t vec_size = kBytesToLoad / sizeof(T); + const int64_t token_idx = blockIdx.x; + const int64_t thread_idx = threadIdx.x; + const int64_t stride = blockDim.x; + const int64_t offset = token_idx * 2 * d; + +#pragma unroll 1 + for (uint32_t idx = thread_idx; idx < d / vec_size; idx += stride) { + sgl_hip::vec_t x_vec, y_vec, out_vec; + x_vec.cast_load(input + offset + idx * vec_size); + y_vec.cast_load(input + offset + d + idx * vec_size); +#pragma unroll + for (uint32_t i = 0; i < vec_size; ++i) { + out_vec[i] = Activation(x_vec[i]) * y_vec[i]; + } + out_vec.cast_store(out + token_idx * d + idx * vec_size); + } + + const int64_t remaining_offset = d - d % (stride * vec_size); + // process the remaining elements +#pragma unroll 1 + for (int64_t idx = thread_idx; idx < d % (stride * vec_size); idx += stride) { + T x = input[offset + remaining_offset + idx], y = input[offset + remaining_offset + d + idx]; + out[token_idx * d + remaining_offset + idx] = Activation(x) * y; + } +} + +template +__global__ void act_only_kernel(T* __restrict__ out, const T* __restrict__ input, const int d) { + constexpr uint32_t vec_size = kBytesToLoad / sizeof(T); + const int64_t token_idx = blockIdx.x; + const int64_t thread_idx = threadIdx.x; + const int64_t stride = blockDim.x; + const int64_t offset = token_idx * d; + +#pragma unroll 1 + for (uint32_t idx = thread_idx; idx < d / vec_size; idx += stride) { + sgl_hip::vec_t x_vec, y_vec, out_vec; + x_vec.cast_load(input + offset + idx * vec_size); +#pragma unroll + for (uint32_t i = 0; i < vec_size; ++i) { + out_vec[i] = Activation(x_vec[i]); + } + out_vec.cast_store(out + token_idx * d + idx * vec_size); + } + + const int64_t remaining_offset = d - d % (stride * vec_size); + // process the remaining elements +#pragma unroll 1 + for (int64_t idx = thread_idx; idx < d % (stride * vec_size); idx += stride) { + T x = input[offset + remaining_offset + idx]; + out[token_idx * d + remaining_offset + idx] = Activation(x); + } +} + +} // namespace activation +} // namespace sgl_hip diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/hip_math_def.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/hip_math_def.h new file mode 100644 index 00000000..356ed953 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/hip_math_def.h @@ -0,0 +1,94 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#pragma once + +#ifdef USE_ROCM + +#include +#include +#include + +// Adapted from flashinfer-rocm [PR#491](https://github.com/flashinfer-ai/flashinfer/pull/491) + +namespace amdgpu { + +template +__forceinline__ __device__ T shfl_xor_sync(unsigned mask, T var, int laneMask, int width = warpSize); + +template +__forceinline__ __device__ destDtype cast(srcDtype val); + +// specialization +template <> +__forceinline__ __device__ float shfl_xor_sync(unsigned mask, float var, int laneMask, int width) { + return __shfl_xor(var, laneMask, width); +} + +template <> +__forceinline__ __device__ int shfl_xor_sync(unsigned mask, int var, int laneMask, int width) { + return __shfl_xor(var, laneMask, width); +} + +template <> +__forceinline__ __device__ float cast(float val) { + return val; +} + +template <> +__forceinline__ __device__ float cast(__half val) { + return __half2float(val); +} + +template <> +__forceinline__ __device__ float cast(__hip_bfloat16 val) { + return __bfloat162float(val); +} + +template <> +__forceinline__ __device__ __half cast(float fval) { + return __float2half(fval); +} + +template <> +__forceinline__ __device__ __hip_bfloat16 cast(float fval) { + return __float2bfloat16(fval); +} + +} // namespace amdgpu + +template +__forceinline__ __device__ T __shfl_xor_sync(unsigned mask, T var, int laneMask, int width = warpSize) { + return amdgpu::shfl_xor_sync(mask, var, laneMask, width); +} + +template +__device__ __forceinline__ float castToFloat(srcDtype val) { + return amdgpu::cast(val); +} + +template +__device__ __forceinline__ dstDtype castFromFloat(float val) { + return amdgpu::cast(val); +} + +// operator overload to support flashinfer +__host__ __device__ __forceinline__ __half operator*(const __half& x, const __half& y) { + __half h_x = x; + __half h_y = y; + return __hmul(h_x, h_y); +} + +#endif diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/hip_vec_dtypes.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/hip_vec_dtypes.h new file mode 100644 index 00000000..a68a6986 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/hip_vec_dtypes.h @@ -0,0 +1,101 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#pragma once + +#if USE_ROCM + +#include +#include +#include + +// Adapted from flashinfer-rocm [PR#491](https://github.com/flashinfer-ai/flashinfer/pull/491)d + +#define SGL_HIP_INLINE inline __attribute__((always_inline)) __device__ + +namespace sgl_hip { + +template +struct vec_t; + +template +SGL_HIP_INLINE void cast_load_impl(vec_t& dst, const srcDtype* src); + +template +SGL_HIP_INLINE void cast_store_impl(dstDtype* dst_ptr, const vec_t& src); + +template +struct vec_t { + SGL_HIP_INLINE float_t& operator[](size_t i); + SGL_HIP_INLINE const float_t& operator[](size_t i) const; + SGL_HIP_INLINE float_t* ptr(); + + SGL_HIP_INLINE void load(const float_t* ptr); + SGL_HIP_INLINE void store(float_t* ptr) const; + + template + SGL_HIP_INLINE void cast_from(const vec_t& src); + template + SGL_HIP_INLINE void cast_load(const T* ptr); + template + SGL_HIP_INLINE void cast_store(T* ptr) const; +}; + +} // namespace sgl_hip + +// **** impl ***** + +namespace sgl_hip { + +template +SGL_HIP_INLINE void cast_load_impl(vec_t& dst, const srcDtype* src_ptr) { + if constexpr (std::is_same::value) { + dst.load(src_ptr); + } else { + vec_t tmp; + tmp.load(src_ptr); + dst.cast_from(tmp); + } +} + +template +SGL_HIP_INLINE void cast_store_impl(dstDtype* dst_ptr, const vec_t& src) { + if constexpr (std::is_same::value) { + src.store(dst_ptr); + } else { + vec_t tmp; + tmp.cast_from(src); + tmp.store(dst_ptr); + } +} + +template +template +SGL_HIP_INLINE void vec_t::cast_load(const T* ptr) { + cast_load_impl(*this, ptr); +} + +template +template +SGL_HIP_INLINE void vec_t::cast_store(T* ptr) const { + cast_store_impl(ptr, *this); +} + +} // namespace sgl_hip + +#include "impl/hip_vec_bf16_impl.h" +#include "impl/hip_vec_fp32_impl.h" +#include "impl/hip_vec_half_impl.h" +#endif diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/impl/hip_vec_bf16_impl.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/impl/hip_vec_bf16_impl.h new file mode 100644 index 00000000..b783f3f4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/impl/hip_vec_bf16_impl.h @@ -0,0 +1,177 @@ +#pragma once + +#if USE_ROCM + +#include +#include + +// Adapted from flashinfer-rocm [PR#491](https://github.com/flashinfer-ai/flashinfer/pull/491) + +using nv_bfloat16 = __hip_bfloat16; +using nv_bfloat162 = __hip_bfloat162; + +__BF16_HOST_DEVICE_STATIC__ __hip_bfloat162 make_bfloat162(const __hip_bfloat16 x, const __hip_bfloat16 y) { + __hip_bfloat162 t; + t.x = x; + t.y = y; + return t; +} + +namespace sgl_hip { + +// nv_bfloat16 x 1 +template <> +struct vec_t { + nv_bfloat16 data; + SGL_HIP_INLINE nv_bfloat16& operator[](size_t i) { + return ((nv_bfloat16*)(&data))[i]; + } + SGL_HIP_INLINE const nv_bfloat16& operator[](size_t i) const { + return ((const nv_bfloat16*)(&data))[i]; + } + SGL_HIP_INLINE nv_bfloat16* ptr() { + return reinterpret_cast(&data); + } + SGL_HIP_INLINE void load(const nv_bfloat16* ptr); + SGL_HIP_INLINE void store(nv_bfloat16* ptr) const; + template + SGL_HIP_INLINE void cast_from(const vec_t& src) { + cast_from_impl(*this, src); + } + template + SGL_HIP_INLINE void cast_load(const T* ptr) { + cast_load_impl(*this, ptr); + } + template + SGL_HIP_INLINE void cast_store(T* ptr) const { + cast_store_impl(ptr, *this); + } +}; + +SGL_HIP_INLINE void vec_t::load(const nv_bfloat16* ptr) { + data = *ptr; +} + +SGL_HIP_INLINE void vec_t::store(nv_bfloat16* ptr) const { + *ptr = data; +} + +// nv_bfloat16 x 2 +template <> +struct vec_t { + nv_bfloat162 data; + + SGL_HIP_INLINE nv_bfloat16& operator[](size_t i) { + return ((nv_bfloat16*)(&data))[i]; + } + SGL_HIP_INLINE const nv_bfloat16& operator[](size_t i) const { + return ((const nv_bfloat16*)(&data))[i]; + } + SGL_HIP_INLINE nv_bfloat16* ptr() { + return reinterpret_cast(&data); + } + SGL_HIP_INLINE void load(const nv_bfloat16* ptr); + SGL_HIP_INLINE void store(nv_bfloat16* ptr) const; + template + SGL_HIP_INLINE void cast_from(const vec_t& src) { + cast_from_impl(*this, src); + } + template + SGL_HIP_INLINE void cast_load(const T* ptr) { + cast_load_impl(*this, ptr); + } + template + SGL_HIP_INLINE void cast_store(T* ptr) const { + cast_store_impl(ptr, *this); + } +}; + +SGL_HIP_INLINE void vec_t::load(const nv_bfloat16* ptr) { + data = *((nv_bfloat162*)ptr); +} + +SGL_HIP_INLINE void vec_t::store(nv_bfloat16* ptr) const { + *((nv_bfloat162*)ptr) = data; +} + +template <> +struct vec_t { + uint2 data; + + SGL_HIP_INLINE nv_bfloat16& operator[](size_t i) { + return ((nv_bfloat16*)(&data))[i]; + } + SGL_HIP_INLINE const nv_bfloat16& operator[](size_t i) const { + return ((const nv_bfloat16*)(&data))[i]; + } + SGL_HIP_INLINE nv_bfloat16* ptr() { + return reinterpret_cast(&data); + } + SGL_HIP_INLINE void load(const nv_bfloat16* ptr); + SGL_HIP_INLINE void store(nv_bfloat16* ptr) const; + template + SGL_HIP_INLINE void cast_from(const vec_t& src) { + cast_from_impl(*this, src); + } + template + SGL_HIP_INLINE void cast_load(const T* ptr) { + cast_load_impl(*this, ptr); + } + template + SGL_HIP_INLINE void cast_store(T* ptr) const { + cast_store_impl(ptr, *this); + } +}; + +SGL_HIP_INLINE void vec_t::load(const nv_bfloat16* ptr) { + data = *((uint2*)ptr); +} + +SGL_HIP_INLINE void vec_t::store(nv_bfloat16* ptr) const { + *((uint2*)ptr) = data; +} + +// nv_bfloat16 x 8 or more + +template +struct vec_t { + uint4 data[vec_size / 8]; + + SGL_HIP_INLINE nv_bfloat16& operator[](size_t i) { + return ((nv_bfloat16*)data)[i]; + } + SGL_HIP_INLINE const nv_bfloat16& operator[](size_t i) const { + return ((const nv_bfloat16*)data)[i]; + } + SGL_HIP_INLINE nv_bfloat16* ptr() { + return reinterpret_cast(&data); + } + SGL_HIP_INLINE void load(const nv_bfloat16* ptr) { +#pragma unoll + for (size_t i = 0; i < vec_size / 8; ++i) { + data[i] = ((uint4*)ptr)[i]; + } + } + SGL_HIP_INLINE void store(nv_bfloat16* ptr) const { +#pragma unoll + for (size_t i = 0; i < vec_size / 8; ++i) { + ((uint4*)ptr)[i] = data[i]; + } + } + template + SGL_HIP_INLINE void cast_from(const vec_t& src) { + cast_from_impl(*this, src); + } + template + SGL_HIP_INLINE void cast_load(const T* ptr) { + cast_load_impl(*this, ptr); + } + template + SGL_HIP_INLINE void cast_store(T* ptr) const { + cast_store_impl(ptr, *this); + } +}; + +} // namespace sgl_hip + +#endif diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/impl/hip_vec_fp32_impl.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/impl/hip_vec_fp32_impl.h new file mode 100644 index 00000000..97cba632 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/impl/hip_vec_fp32_impl.h @@ -0,0 +1,129 @@ +#pragma once + +#if USE_ROCM + +#include + +// Adapted from flashinfer-rocm [PR#491](https://github.com/flashinfer-ai/flashinfer/pull/491) + +namespace sgl_hip { + +template <> +struct vec_t { + float data; + + SGL_HIP_INLINE float& operator[](size_t i) { + return ((float*)(&data))[i]; + } + SGL_HIP_INLINE const float& operator[](size_t i) const { + return ((const float*)(&data))[i]; + } + SGL_HIP_INLINE float* ptr() { + return reinterpret_cast(&data); + } + SGL_HIP_INLINE void load(const float* ptr); + SGL_HIP_INLINE void store(float* ptr) const; + template + SGL_HIP_INLINE void cast_from(const vec_t& src) { + cast_from_impl(*this, src); + } + template + SGL_HIP_INLINE void cast_load(const T* ptr) { + cast_load_impl(*this, ptr); + } + template + SGL_HIP_INLINE void cast_store(T* ptr) const { + cast_store_impl(ptr, *this); + } +}; + +SGL_HIP_INLINE void vec_t::load(const float* ptr) { + data = *ptr; +} + +SGL_HIP_INLINE void vec_t::store(float* ptr) const { + *ptr = data; +} + +// float x 2 + +template <> +struct vec_t { + float2 data; + + SGL_HIP_INLINE float& operator[](size_t i) { + return ((float*)(&data))[i]; + } + SGL_HIP_INLINE const float& operator[](size_t i) const { + return ((const float*)(&data))[i]; + } + SGL_HIP_INLINE float* ptr() { + return reinterpret_cast(&data); + } + SGL_HIP_INLINE void load(const float* ptr); + SGL_HIP_INLINE void store(float* ptr) const; + template + SGL_HIP_INLINE void cast_from(const vec_t& src) { + cast_from_impl(*this, src); + } + template + SGL_HIP_INLINE void cast_load(const T* ptr) { + cast_load_impl(*this, ptr); + } + template + SGL_HIP_INLINE void cast_store(T* ptr) const { + cast_store_impl(ptr, *this); + } +}; + +SGL_HIP_INLINE void vec_t::load(const float* ptr) { + data = *((float2*)ptr); +} + +SGL_HIP_INLINE void vec_t::store(float* ptr) const { + *((float2*)ptr) = data; +} + +// float x 4 or more +template +struct vec_t { + float4 data[vec_size / 4]; + + SGL_HIP_INLINE float& operator[](size_t i) { + return ((float*)(data))[i]; + } + SGL_HIP_INLINE const float& operator[](size_t i) const { + return ((const float*)(data))[i]; + } + SGL_HIP_INLINE float* ptr() { + return reinterpret_cast(&data); + } + SGL_HIP_INLINE void load(const float* ptr) { +#pragma unroll + for (size_t i = 0; i < vec_size / 4; ++i) { + data[i] = ((float4*)ptr)[i]; + } + } + SGL_HIP_INLINE void store(float* ptr) const { +#pragma unroll + for (size_t i = 0; i < vec_size / 4; ++i) { + ((float4*)ptr)[i] = data[i]; + } + } + template + SGL_HIP_INLINE void cast_from(const vec_t& src) { + cast_from_impl(*this, src); + } + template + SGL_HIP_INLINE void cast_load(const T* ptr) { + cast_load_impl(*this, ptr); + } + template + SGL_HIP_INLINE void cast_store(T* ptr) const { + cast_store_impl(ptr, *this); + } +}; + +} // namespace sgl_hip + +#endif diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/impl/hip_vec_half_impl.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/impl/hip_vec_half_impl.h new file mode 100644 index 00000000..767b9c62 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/hip/impl/hip_vec_half_impl.h @@ -0,0 +1,172 @@ +#pragma once + +#if USE_ROCM + +#include +#include + +// Adapted from flashinfer-rocm [PR#491](https://github.com/flashinfer-ai/flashinfer/pull/491) + +using half = __half; +using half2 = __half2; + +namespace sgl_hip { + +// half x 1 +template <> +struct vec_t { + half data; + + SGL_HIP_INLINE half& operator[](size_t i) { + return ((half*)(&data))[i]; + } + SGL_HIP_INLINE const half& operator[](size_t i) const { + return ((const half*)(&data))[i]; + } + SGL_HIP_INLINE half* ptr() { + return reinterpret_cast(&data); + } + SGL_HIP_INLINE void load(const half* ptr); + SGL_HIP_INLINE void store(half* ptr) const; + template + SGL_HIP_INLINE void cast_from(const vec_t& src) { + cast_from_impl(*this, src); + } + template + SGL_HIP_INLINE void cast_load(const T* ptr) { + cast_load_impl(*this, ptr); + } + template + SGL_HIP_INLINE void cast_store(T* ptr) const { + cast_store_impl(ptr, *this); + } +}; + +SGL_HIP_INLINE void vec_t::load(const half* ptr) { + data = *ptr; +} + +SGL_HIP_INLINE void vec_t::store(half* ptr) const { + *ptr = data; +} + +// half x 2 +template <> +struct vec_t { + half2 data; + + SGL_HIP_INLINE half& operator[](size_t i) { + return ((half*)(&data))[i]; + } + SGL_HIP_INLINE const half& operator[](size_t i) const { + return ((const half*)(&data))[i]; + } + SGL_HIP_INLINE half* ptr() { + return reinterpret_cast(&data); + } + SGL_HIP_INLINE void load(const half* ptr); + SGL_HIP_INLINE void store(half* ptr) const; + template + SGL_HIP_INLINE void cast_from(const vec_t& src) { + cast_from_impl(*this, src); + } + template + SGL_HIP_INLINE void cast_load(const T* ptr) { + cast_load_impl(*this, ptr); + } + template + SGL_HIP_INLINE void cast_store(T* ptr) const { + cast_store_impl(ptr, *this); + } +}; + +SGL_HIP_INLINE void vec_t::load(const half* ptr) { + data = *((half2*)ptr); +} + +SGL_HIP_INLINE void vec_t::store(half* ptr) const { + *((half2*)ptr) = data; +} + +// half x 4 + +template <> +struct vec_t { + uint2 data; + + SGL_HIP_INLINE half& operator[](size_t i) { + return ((half*)(&data))[i]; + } + SGL_HIP_INLINE const half& operator[](size_t i) const { + return ((const half*)(&data))[i]; + } + SGL_HIP_INLINE half* ptr() { + return reinterpret_cast(&data); + } + SGL_HIP_INLINE void load(const half* ptr); + SGL_HIP_INLINE void store(half* ptr) const; + template + SGL_HIP_INLINE void cast_from(const vec_t& src) { + cast_from_impl(*this, src); + } + template + SGL_HIP_INLINE void cast_load(const T* ptr) { + cast_load_impl(*this, ptr); + } + template + SGL_HIP_INLINE void cast_store(T* ptr) const { + cast_store_impl(ptr, *this); + } +}; + +SGL_HIP_INLINE void vec_t::load(const half* ptr) { + data = *((uint2*)ptr); +} + +SGL_HIP_INLINE void vec_t::store(half* ptr) const { + *((uint2*)ptr) = data; +} + +// half x 8 or more + +template +struct vec_t { + uint4 data[vec_size / 8]; + + SGL_HIP_INLINE half& operator[](size_t i) { + return ((half*)data)[i]; + } + SGL_HIP_INLINE const half& operator[](size_t i) const { + return ((const half*)data)[i]; + } + SGL_HIP_INLINE half* ptr() { + return reinterpret_cast(&data); + } + SGL_HIP_INLINE void load(const half* ptr) { +#pragma unroll + for (size_t i = 0; i < vec_size / 8; ++i) { + data[i] = ((uint4*)ptr)[i]; + } + } + SGL_HIP_INLINE void store(half* ptr) const { +#pragma unroll + for (size_t i = 0; i < vec_size / 8; ++i) { + ((uint4*)ptr)[i] = data[i]; + } + } + template + SGL_HIP_INLINE void cast_from(const vec_t& src) { + cast_from_impl(*this, src); + } + template + SGL_HIP_INLINE void cast_load(const T* ptr) { + cast_load_impl(*this, ptr); + } + template + SGL_HIP_INLINE void cast_store(T* ptr) const { + cast_store_impl(ptr, *this); + } +}; + +} // namespace sgl_hip +#endif diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/pytorch_extension_utils_rocm.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/pytorch_extension_utils_rocm.h new file mode 100644 index 00000000..fa5fd129 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/pytorch_extension_utils_rocm.h @@ -0,0 +1,20 @@ +#include + +#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor") + +#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_LAST_DIM_CONTIGUOUS(x) \ + TORCH_CHECK(x.strides()[x.strides().size() - 1] == 1, #x "must be contiguous at last dimension") + +#define CHECK_INPUT(x) \ + CHECK_CUDA(x); \ + CHECK_CONTIGUOUS(x) +#define CHECK_LAST_DIM_CONTIGUOUS_INPUT(x) \ + CHECK_CUDA(x); \ + CHECK_LAST_DIM_CONTIGUOUS(x) + +#define CHECK_DIM(d, x) TORCH_CHECK(x.dim() == d, #x " must be a " #d "D tensor") + +#define CHECK_EQ(a, b) TORCH_CHECK((a) == (b), "CHECK_EQ(" #a ", " #b ") failed. ", a, " vs ", b) + +#define CHECK_GE(a, b) TORCH_CHECK((a) >= (b), "CHECK_GE(" #a ", " #b ") failed. ", a, " vs ", b) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/scalar_type.hpp b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/scalar_type.hpp new file mode 100644 index 00000000..3ed1467b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/scalar_type.hpp @@ -0,0 +1,331 @@ +#pragma once + +// For TORCH_CHECK +#include + +namespace sglang { + +// +// ScalarType can represent a wide range of floating point and integer types, +// in particular it can be used to represent sub-byte data types (something +// that torch.dtype currently does not support). +// +// The type definitions on the Python side can be found in: vllm/scalar_type.py +// these type definitions should be kept up to date with any Python API changes +// here. +// +class ScalarType { + public: + enum NanRepr : uint8_t { + NAN_NONE = 0, // nans are not supported + NAN_IEEE_754 = 1, // nans are: exp all 1s, mantissa not all 0s + NAN_EXTD_RANGE_MAX_MIN = 2, // nans are: exp all 1s, mantissa all 1s + + NAN_REPR_ID_MAX + }; + + constexpr ScalarType( + uint8_t exponent, + uint8_t mantissa, + bool signed_, + int32_t bias, + bool finite_values_only = false, + NanRepr nan_repr = NAN_IEEE_754) + : exponent(exponent), + mantissa(mantissa), + signed_(signed_), + bias(bias), + finite_values_only(finite_values_only), + nan_repr(nan_repr) {}; + + static constexpr ScalarType int_(uint8_t size_bits, int32_t bias = 0) { + return ScalarType(0, size_bits - 1, true, bias); + } + + static constexpr ScalarType uint(uint8_t size_bits, int32_t bias = 0) { + return ScalarType(0, size_bits, false, bias); + } + + // IEEE 754 compliant floating point type + static constexpr ScalarType float_IEEE754(uint8_t exponent, uint8_t mantissa) { + TORCH_CHECK(mantissa > 0 && exponent > 0); + return ScalarType(exponent, mantissa, true, 0, false, NAN_IEEE_754); + } + + // IEEE 754 non-compliant floating point type + static constexpr ScalarType float_(uint8_t exponent, uint8_t mantissa, bool finite_values_only, NanRepr nan_repr) { + TORCH_CHECK(nan_repr < NAN_REPR_ID_MAX, "Invalid NanRepr"); + TORCH_CHECK(mantissa > 0 && exponent > 0); + TORCH_CHECK( + nan_repr != NAN_IEEE_754, + "use `float_IEEE754` constructor for floating point types that " + "follow IEEE 754 conventions"); + return ScalarType(exponent, mantissa, true, 0, finite_values_only, nan_repr); + } + + uint8_t const exponent; // size of the exponent field (0 for integer types) + uint8_t const mantissa; // size of the mantissa field (size of the integer + // excluding the sign bit for integer types) + bool const signed_; // flag if the type supports negative numbers (i.e. has a + // sign bit) + int32_t const bias; // stored values equal value + bias, + // used for quantized type + + // Extra Floating point info + bool const finite_values_only; // i.e. no +/-inf if true + NanRepr const nan_repr; // how NaNs are represented + // (not applicable for integer types) + + using Id = int64_t; + + private: + // Field size in id + template + static constexpr size_t member_id_field_width() { + using T = std::decay_t; + return std::is_same_v ? 1 : sizeof(T) * 8; + } + + template + static constexpr auto reduce_members_helper(Fn f, Init val, Member member, Rest... rest) { + auto new_val = f(val, member); + if constexpr (sizeof...(rest) > 0) { + return reduce_members_helper(f, new_val, rest...); + } else { + return new_val; + }; + } + + template + constexpr auto reduce_members(Fn f, Init init) const { + // Should be in constructor order for `from_id` + return reduce_members_helper(f, init, exponent, mantissa, signed_, bias, finite_values_only, nan_repr); + }; + + template + static constexpr auto reduce_member_types(Fn f, Init init) { + constexpr auto dummy_type = ScalarType(0, 0, false, 0, false, NAN_NONE); + return dummy_type.reduce_members(f, init); + }; + + static constexpr auto id_size_bits() { + return reduce_member_types( + [](int acc, auto member) -> int { return acc + member_id_field_width(); }, 0); + } + + public: + // unique id for this scalar type that can be computed at compile time for + // c++17 template specialization this is not needed once we migrate to + // c++20 and can pass literal classes as template parameters + constexpr Id id() const { + static_assert(id_size_bits() <= sizeof(Id) * 8, "ScalarType id is too large to be stored"); + + auto or_and_advance = [](std::pair result, auto member) -> std::pair { + auto [id, bit_offset] = result; + auto constexpr bits = member_id_field_width(); + return {id | (int64_t(member) & ((uint64_t(1) << bits) - 1)) << bit_offset, bit_offset + bits}; + }; + return reduce_members(or_and_advance, std::pair{}).first; + } + + // create a ScalarType from an id, for c++17 template specialization, + // this is not needed once we migrate to c++20 and can pass literal + // classes as template parameters + static constexpr ScalarType from_id(Id id) { + auto extract_and_advance = [id](auto result, auto member) { + using T = decltype(member); + auto [tuple, bit_offset] = result; + auto constexpr bits = member_id_field_width(); + auto extracted_val = static_cast((int64_t(id) >> bit_offset) & ((uint64_t(1) << bits) - 1)); + auto new_tuple = std::tuple_cat(tuple, std::make_tuple(extracted_val)); + return std::pair{new_tuple, bit_offset + bits}; + }; + + auto [tuple_args, _] = reduce_member_types(extract_and_advance, std::pair, int>{}); + return std::apply([](auto... args) { return ScalarType(args...); }, tuple_args); + } + + constexpr int64_t size_bits() const { + return mantissa + exponent + is_signed(); + } + constexpr bool is_signed() const { + return signed_; + } + constexpr bool is_integer() const { + return exponent == 0; + } + constexpr bool is_floating_point() const { + return exponent > 0; + } + constexpr bool is_ieee_754() const { + return is_floating_point() && finite_values_only == false && nan_repr == NAN_IEEE_754; + } + constexpr bool has_nans() const { + return is_floating_point() && nan_repr != NAN_NONE; + } + constexpr bool has_infs() const { + return is_floating_point() && finite_values_only == false; + } + constexpr bool has_bias() const { + return bias != 0; + } + + private: + double _floating_point_max() const { + TORCH_CHECK(mantissa <= 52 && exponent <= 11, "Cannot represent max/min as a double for type ", str()); + + uint64_t max_mantissa = (uint64_t(1) << mantissa) - 1; + if (nan_repr == NAN_EXTD_RANGE_MAX_MIN) { + max_mantissa -= 1; + } + + uint64_t max_exponent = (uint64_t(1) << exponent) - 2; + if (nan_repr == NAN_EXTD_RANGE_MAX_MIN || nan_repr == NAN_NONE) { + TORCH_CHECK(exponent < 11, "Cannot represent max/min as a double for type ", str()); + max_exponent += 1; + } + + // adjust the exponent to match that of a double + // for now we assume the exponent bias is the standard 2^(e-1) -1, (where e + // is the exponent bits), there is some precedent for non-standard biases, + // example `float8_e4m3b11fnuz` here: https://github.com/jax-ml/ml_dtypes + // but to avoid premature over complication we are just assuming the + // standard exponent bias until there is a need to support non-standard + // biases + uint64_t exponent_bias = (uint64_t(1) << (exponent - 1)) - 1; + uint64_t exponent_bias_double = (uint64_t(1) << 10) - 1; // double e = 11 + + uint64_t max_exponent_double = max_exponent - exponent_bias + exponent_bias_double; + + // shift the mantissa into the position for a double and + // the exponent + uint64_t double_raw = (max_mantissa << (52 - mantissa)) | (max_exponent_double << 52); + + return *reinterpret_cast(&double_raw); + } + + constexpr std::variant _raw_max() const { + if (is_floating_point()) { + return {_floating_point_max()}; + } else { + TORCH_CHECK(size_bits() < 64 || size_bits() == 64 && is_signed(), "Cannot represent max as a int64_t"); + return {(int64_t(1) << mantissa) - 1}; + } + } + + constexpr std::variant _raw_min() const { + if (is_floating_point()) { + TORCH_CHECK(is_signed(), "We currently assume all floating point types are signed"); + constexpr uint64_t sign_bit_double = (uint64_t(1) << 63); + + double max = _floating_point_max(); + uint64_t max_raw = *reinterpret_cast(&max); + uint64_t min_raw = max_raw | sign_bit_double; + return {*reinterpret_cast(&min_raw)}; + } else { + TORCH_CHECK(!is_signed() || size_bits() <= 64, "Cannot represent min as a int64_t"); + if (is_signed()) { + // set the top bit to 1 (i.e. INT64_MIN) and the rest to 0 + // then perform an arithmetic shift right to set all the bits above + // (size_bits() - 1) to 1 + return {INT64_MIN >> (64 - size_bits())}; + } else { + return {int64_t(0)}; + } + } + } + + public: + // Max representable value for this scalar type. + // (accounting for bias if there is one) + constexpr std::variant max() const { + return std::visit([this](auto x) -> std::variant { return {x - bias}; }, _raw_max()); + } + + // Min representable value for this scalar type. + // (accounting for bias if there is one) + constexpr std::variant min() const { + return std::visit([this](auto x) -> std::variant { return {x - bias}; }, _raw_min()); + } + + std::string str() const { + /* naming generally follows: https://github.com/jax-ml/ml_dtypes + * for floating point types (leading f) the scheme is: + * `float_em[flags]` + * flags: + * - no-flags: means it follows IEEE 754 conventions + * - f: means finite values only (no infinities) + * - n: means nans are supported (non-standard encoding) + * for integer types the scheme is: + * `[u]int[b]` + * - if bias is not present it means its zero + */ + if (is_floating_point()) { + auto ret = + "float" + std::to_string(size_bits()) + "_e" + std::to_string(exponent) + "m" + std::to_string(mantissa); + if (!is_ieee_754()) { + if (finite_values_only) { + ret += "f"; + } + if (nan_repr != NAN_NONE) { + ret += "n"; + } + } + return ret; + } else { + auto ret = ((is_signed()) ? "int" : "uint") + std::to_string(size_bits()); + if (has_bias()) { + ret += "b" + std::to_string(bias); + } + return ret; + } + } + + constexpr bool operator==(ScalarType const& other) const { + return mantissa == other.mantissa && exponent == other.exponent && bias == other.bias && signed_ == other.signed_ && + finite_values_only == other.finite_values_only && nan_repr == other.nan_repr; + } +}; + +using ScalarTypeId = ScalarType::Id; + +// "rust style" names generally following: +// https://github.com/pytorch/pytorch/blob/6d9f74f0af54751311f0dd71f7e5c01a93260ab3/torch/csrc/api/include/torch/types.h#L60-L70 +static inline constexpr auto kS4 = ScalarType::int_(4); +static inline constexpr auto kU4 = ScalarType::uint(4); +static inline constexpr auto kU4B8 = ScalarType::uint(4, 8); +static inline constexpr auto kS8 = ScalarType::int_(8); +static inline constexpr auto kU8 = ScalarType::uint(8); +static inline constexpr auto kU8B128 = ScalarType::uint(8, 128); + +static inline constexpr auto kFE2M1f = ScalarType::float_(2, 1, true, ScalarType::NAN_NONE); +static inline constexpr auto kFE3M2f = ScalarType::float_(3, 2, true, ScalarType::NAN_NONE); +static inline constexpr auto kFE4M3fn = ScalarType::float_(4, 3, true, ScalarType::NAN_EXTD_RANGE_MAX_MIN); +static inline constexpr auto kFE8M0fnu = ScalarType(8, 0, false, 0, true, ScalarType::NAN_EXTD_RANGE_MAX_MIN); +static inline constexpr auto kFE5M2 = ScalarType::float_IEEE754(5, 2); +static inline constexpr auto kFE8M7 = ScalarType::float_IEEE754(8, 7); +static inline constexpr auto kFE5M10 = ScalarType::float_IEEE754(5, 10); + +// Fixed width style names, generally following: +// https://github.com/pytorch/pytorch/blob/6d9f74f0af54751311f0dd71f7e5c01a93260ab3/torch/csrc/api/include/torch/types.h#L47-L57 +static inline constexpr auto kInt4 = kS4; +static inline constexpr auto kUint4 = kU4; +static inline constexpr auto kUint4b8 = kU4B8; +static inline constexpr auto kInt8 = kS8; +static inline constexpr auto kUint8 = kU8; +static inline constexpr auto kUint8b128 = kU8B128; + +static inline constexpr auto kFloat4_e2m1f = kFE2M1f; +static inline constexpr auto kFloat6_e3m2f = kFE3M2f; +static inline constexpr auto kFloat8_e4m3fn = kFE4M3fn; +static inline constexpr auto kFloat8_e5m2 = kFE5M2; +static inline constexpr auto kFloat16_e8m7 = kFE8M7; +static inline constexpr auto kFloat16_e5m10 = kFE5M10; + +// colloquial names +static inline constexpr auto kHalf = kFE5M10; +static inline constexpr auto kFloat16 = kHalf; +static inline constexpr auto kBFloat16 = kFE8M7; + +static inline constexpr auto kFloat16Id = kFloat16.id(); +}; // namespace sglang diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/sgl_flash_kernel_ops.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/sgl_flash_kernel_ops.h new file mode 100644 index 00000000..b36af6b6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/sgl_flash_kernel_ops.h @@ -0,0 +1,85 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include "sgl_kernel_torch_shim.h" + +#define TORCH_LIBRARY_EXPAND(NAME, MODULE) TORCH_LIBRARY(NAME, MODULE) + +#define _CONCAT(A, B) A##B +#define CONCAT(A, B) _CONCAT(A, B) + +#define _STRINGIFY(A) #A +#define STRINGIFY(A) _STRINGIFY(A) + +#define REGISTER_EXTENSION(NAME) \ + PyMODINIT_FUNC CONCAT(PyInit_, NAME)() { \ + static struct PyModuleDef module = {PyModuleDef_HEAD_INIT, STRINGIFY(NAME), nullptr, 0, nullptr}; \ + return PyModule_Create(&module); \ + } + +/* + * From flash-attention + */ +std::tuple mha_fwd( + at::Tensor q, // (b, s_q, h, d) or (total_q, h, d) if there is cu_seqlens_q + at::Tensor k, // (b_k, s_k, h_k, d) or (total_k, h_k, d) if there is cu_seqlens_k or (num_pages, page_size, + // h_k, d) if there is page_table. + at::Tensor v, // (b_k, s_k, h_k, dv) or (total_k, h_k, dv) if there is cu_seqlens_k or (num_pages, + // page_size, h_k, dv) if there is page_table. + std::optional k_new_, // (b, s_k_new, h_k, d) or (total_k_new, h_k, d) if there is cu_seqlens_k_new + std::optional v_new_, // (b, s_k_new, h_k, dv) or (total_k_new, h_k, dv) if there is cu_seqlens_k_new + std::optional q_v_, // (b, s_q, h, dv) or (total_q_new, h, dv) if there is cu_seqlens_q + std::optional out_, // (b, s_q, h, dv) or (total_q, h, dv) if there is cu_seqlens_q + std::optional cu_seqlens_q_, // b+1 + std::optional cu_seqlens_k_, // b+1 + std::optional cu_seqlens_k_new_, // b+1 + std::optional + seqused_q_, // b. If given, only this many elements of each batch element's queries and outputs are used. + std::optional + seqused_k_, // b. If given, only this many elements of each batch element's keys are used. + std::optional max_seqlen_q_, + // TODO: check if we need max_seqlen_k + std::optional max_seqlen_k_, + std::optional page_table_, // (b_k, max_num_pages_per_seq) + std::optional kv_batch_idx_, // b. indices to index into the KV cache + std::optional leftpad_k_, // b + std::optional rotary_cos_, // seqlen_ro x (rotary_dim / 2) + std::optional rotary_sin_, // seqlen_ro x (rotary_dim / 2) + std::optional seqlens_rotary_, // b + std::optional q_descale_, // (b, h_k), not (b, h) + std::optional k_descale_, // (b, h_k) + std::optional v_descale_, // (b, h_k) + std::optional softmax_scale_, + bool is_causal, + int64_t window_size_left, + int64_t window_size_right, + int64_t attention_chunk, + double softcap, + bool is_rotary_interleaved, // if true, rotary combines indices 0 & 1, else indices 0 & rotary_dim / 2 + std::optional scheduler_metadata_, // (b + 1) + int64_t num_splits, + std::optional pack_gqa_, + int64_t sm_margin, + std::optional& sinks_); // (h) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/sgl_kernel_ops.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/sgl_kernel_ops.h new file mode 100644 index 00000000..552519fa --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/sgl_kernel_ops.h @@ -0,0 +1,981 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "scalar_type.hpp" + +#define _CONCAT(A, B) A##B +#define CONCAT(A, B) _CONCAT(A, B) + +#define _STRINGIFY(A) #A +#define STRINGIFY(A) _STRINGIFY(A) + +#define TORCH_LIBRARY_EXPAND(NAME, MODULE) TORCH_LIBRARY(NAME, MODULE) + +#define REGISTER_EXTENSION(NAME) \ + PyMODINIT_FUNC CONCAT(PyInit_, NAME)() { \ + static struct PyModuleDef module = {PyModuleDef_HEAD_INIT, STRINGIFY(NAME), nullptr, 0, nullptr}; \ + return PyModule_Create(&module); \ + } + +using fptr_t = int64_t; + +/* + * From csrc/allreduce + */ +#ifdef USE_ROCM +// ROCM custom allreduce +fptr_t init_custom_ar( + torch::Tensor& meta, + torch::Tensor& rank_data, + const std::vector& handles, + const std::vector& offsets, + int64_t rank, + bool full_nvlink); +void all_reduce_reg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out); +void all_reduce_unreg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& reg_buffer, torch::Tensor& out); +void dispose(fptr_t _fa); +int64_t meta_size(); +void register_buffer( + fptr_t _fa, torch::Tensor& t, const std::vector& handles, const std::vector& offsets); +std::tuple> get_graph_buffer_ipc_meta(fptr_t _fa); +void register_graph_buffers( + fptr_t _fa, const std::vector& handles, const std::vector>& offsets); +torch::Tensor allocate_meta_buffer(int64_t size); +torch::Tensor get_meta_buffer_ipc_handle(torch::Tensor& inp); +// quick allreduce +fptr_t init_custom_qr(int64_t rank, int64_t world_size, std::optional qr_max_size = std::nullopt); +void qr_destroy(fptr_t _fa); +torch::Tensor qr_get_handle(fptr_t _fa); +void qr_open_handles(fptr_t _fa, const std::vector& handles); +void qr_all_reduce(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out, int64_t quant_level, bool cast_bf2half = false); +int64_t qr_max_size(); +#else +// custom allreduce +fptr_t +init_custom_ar(const std::vector& fake_ipc_ptrs, torch::Tensor& rank_data, int64_t rank, bool full_nvlink); +void dispose(fptr_t _fa); +int64_t meta_size(); +void all_reduce(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out, fptr_t _reg_buffer, int64_t reg_buffer_sz_bytes); +std::tuple, std::vector> get_graph_buffer_ipc_meta(fptr_t _fa); +void register_buffer(fptr_t _fa, const std::vector& fake_ipc_ptrs); +void register_graph_buffers( + fptr_t _fa, const std::vector>& handles, const std::vector>& offsets); + +// mscclpp +torch::Tensor mscclpp_generate_unique_id(); +fptr_t mscclpp_init_context( + const torch::Tensor& unique_id, + const int64_t rank, + const int64_t world_size, + torch::Tensor& scratch, + torch::Tensor& put_buffer, + const int64_t nranks_per_node, + const std::vector& rank_to_node, + const std::vector& rank_to_ib, + const int64_t context_selection); +void mscclpp_allreduce(fptr_t _context, torch::Tensor& inp, torch::Tensor& out, int64_t nthreads, int64_t nblocks); +#endif + +/* + * From csrc/attention + */ +void merge_state( + at::Tensor v_a, at::Tensor s_a, at::Tensor v_b, at::Tensor s_b, at::Tensor v_merged, at::Tensor s_merged); +void merge_state_v2( + at::Tensor v_a, at::Tensor s_a, at::Tensor v_b, at::Tensor s_b, at::Tensor v_merged, at::Tensor s_merged); +void cutlass_mla_decode( + torch::Tensor const& out, + torch::Tensor const& q_nope, + torch::Tensor const& q_pe, + torch::Tensor const& kv_c_and_k_pe_cache, + torch::Tensor const& seq_lens, + torch::Tensor const& page_table, + torch::Tensor const& workspace, + double sm_scale, + int64_t num_kv_splits = 1 /* Set to 1 to avoid cuda_graph issue by default. */); +int64_t cutlass_mla_get_workspace_size( + int64_t max_seq_len, + int64_t num_batches, + int64_t sm_count = 0, + int64_t num_kv_splits = 1 /* Set to 1 to avoid cuda_graph issue by default. */); + +/* + * From csrc/elementwise + */ +void rmsnorm(at::Tensor& output, at::Tensor& input, at::Tensor& weight, double eps, bool enable_pdl); +void sgl_fused_add_rmsnorm( + torch::Tensor input, torch::Tensor residual, torch::Tensor weight, double eps, bool enable_pdl); +void gemma_rmsnorm(at::Tensor& output, at::Tensor& input, at::Tensor& weight, double eps, bool enable_pdl); +void gemma_fused_add_rmsnorm(at::Tensor& input, at::Tensor& residual, at::Tensor& weight, double eps, bool enable_pdl); +void silu_and_mul(at::Tensor& out, at::Tensor& input); +void gelu_tanh_and_mul(at::Tensor& out, at::Tensor& input); +void gelu_and_mul(at::Tensor& out, at::Tensor& input); + +void apply_rope_pos_ids_cos_sin_cache( + at::Tensor q, + at::Tensor k, + at::Tensor q_rope, + at::Tensor k_rope, + at::Tensor cos_sin_cache, + at::Tensor pos_ids, + bool interleave, + bool enable_pdl, + const std::optional& v, + const std::optional& k_buffer, + const std::optional& v_buffer, + const std::optional& kv_cache_loc); + +void downcast_fp8( + at::Tensor& k, + at::Tensor& v, + at::Tensor& k_out, + at::Tensor& v_out, + at::Tensor& k_scale, + at::Tensor& v_scale, + at::Tensor& loc, + int64_t mult, + int64_t offset); + +void copy_to_gpu_no_ce(const at::Tensor& input, at::Tensor& output); +void concat_mla_k(torch::Tensor k, torch::Tensor k_nope, torch::Tensor k_rope); +void concat_mla_absorb_q(at::Tensor a, at::Tensor b, at::Tensor out); + +void fast_topk_interface( + const at::Tensor& score, + at::Tensor& indices, + const at::Tensor& lengths, + std::optional row_starts_opt = std::nullopt); +void fast_topk_transform_interface( + const at::Tensor& score, + const at::Tensor& lengths, + at::Tensor& dst_page_table, + const at::Tensor& src_page_table, + const at::Tensor& cu_seqlens_q, + std::optional row_starts_opt = std::nullopt); +void fast_topk_transform_ragged_interface( + const at::Tensor& score, + const at::Tensor& lengths, + at::Tensor& topk_indices_ragged, + const at::Tensor& topk_indices_offset, + std::optional row_starts_opt = std::nullopt); + +#ifdef USE_ROCM +void gelu_quick(at::Tensor& out, const at::Tensor& input); +#endif + +/* + * From csrc/gemm + */ +torch::Tensor awq_dequantize(torch::Tensor qweight, torch::Tensor scales, torch::Tensor qzeros); +void cutlass_scaled_fp4_mm( + torch::Tensor& D, + torch::Tensor const& A, + torch::Tensor const& B, + torch::Tensor const& A_sf, + torch::Tensor const& B_sf, + torch::Tensor const& alpha); +torch::Tensor int8_scaled_mm( + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Dtype& out_dtype, + const c10::optional& bias); +torch::Tensor fp8_scaled_mm( + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Dtype& out_dtype, + const c10::optional& bias); +torch::Tensor fp8_blockwise_scaled_mm( + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Dtype& out_dtype); +void scaled_fp4_quant( + torch::Tensor& output, torch::Tensor const& input, torch::Tensor& output_scale, torch::Tensor const& input_scale); +void sgl_per_token_group_quant_8bit( + at::Tensor input, + at::Tensor output_q, + at::Tensor output_s, + int64_t group_size, + double eps, + double fp8_min, + double fp8_max, + bool scale_ue8m0); +void sgl_per_token_group_quant_8bit_v2( + at::Tensor input, + at::Tensor output_q, + at::Tensor output_s, + int64_t group_size, + double eps, + double min_8bit, + double max_8bit, + bool scale_ue8m0, + bool fuse_silu_and_mul, + const std::optional& masked_m); +void sgl_per_tensor_quant_fp8(at::Tensor input, at::Tensor output_q, at::Tensor output_s, bool is_static); +void sgl_per_token_quant_fp8(at::Tensor input, at::Tensor output_q, at::Tensor output_s); +void bmm_fp8( + at::Tensor A, + at::Tensor B, + at::Tensor D, + at::Tensor A_scale, + at::Tensor B_scale, + at::Tensor workspace_buffer, + int64_t cublas_handle); +void dsv3_router_gemm(torch::Tensor& output, const torch::Tensor& mat_a, const torch::Tensor& mat_b); +void dsv3_fused_a_gemm(torch::Tensor& output, torch::Tensor const& mat_a, torch::Tensor const& mat_b); + +torch::Tensor gptq_marlin_gemm( + torch::Tensor& a, + std::optional c_or_none, + torch::Tensor& b_q_weight, + torch::Tensor& b_scales, + std::optional const& global_scale_or_none, + std::optional const& b_zeros_or_none, + std::optional const& g_idx_or_none, + std::optional const& perm_or_none, + torch::Tensor& workspace, + sglang::ScalarTypeId const& b_q_type_id, + int64_t size_m, + int64_t size_n, + int64_t size_k, + bool is_k_full, + bool use_atomic_add, + bool use_fp32_reduce, + bool is_zp_float); + +torch::Tensor gptq_gemm( + torch::Tensor a, + torch::Tensor b_q_weight, + torch::Tensor b_gptq_qzeros, + torch::Tensor b_gptq_scales, + torch::Tensor b_g_idx, + bool use_shuffle, + int64_t bit); + +void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int64_t bit); + +torch::Tensor +gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm, int64_t size_k, int64_t size_n, int64_t num_bits); + +torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k, int64_t size_n, int64_t num_bits); + +/* + * From csrc/moe + */ +void moe_align_block_size( + torch::Tensor topk_ids, + int64_t num_experts, + int64_t block_size, + torch::Tensor sorted_token_ids, + torch::Tensor experts_ids, + torch::Tensor num_tokens_post_pad, + torch::Tensor cumsum_buffer, + bool pad_sorted_token_ids); + +void topk_softmax( + torch::Tensor& topk_weights, + torch::Tensor& topk_indices, + torch::Tensor& gating_output, + bool renormalize, + double moe_softcapping, + const c10::optional& correction_bias); + +void topk_sigmoid( + torch::Tensor& topk_weights, + torch::Tensor& topk_indices, + torch::Tensor& gating_output, + bool renormalize, + const c10::optional& correction_bias); + +void moe_sum_reduce(at::Tensor& input, at::Tensor& output, double routed_scaling_factor); + +void moe_sum(torch::Tensor& input, torch::Tensor& output); + +std::vector moe_fused_gate( + at::Tensor& input, + at::Tensor& bias, + int64_t num_expert_group, + int64_t topk_group, + int64_t topk, + int64_t num_fused_shared_experts, + double routed_scaling_factor, + bool apply_routed_scaling_factor_on_output); + +std::vector kimi_k2_moe_fused_gate( + at::Tensor& input, + at::Tensor& bias, + int64_t topk, + bool renormalize, + double routed_scaling_factor, + bool apply_routed_scaling_factor_on_output); + +void fp8_blockwise_scaled_grouped_mm( + torch::Tensor& output, + torch::Tensor& a_ptrs, + torch::Tensor& b_ptrs, + torch::Tensor& out_ptrs, + torch::Tensor& a_scales_ptrs, + torch::Tensor& b_scales_ptrs, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Tensor& stride_a, + const torch::Tensor& stride_b, + const torch::Tensor& stride_c, + const torch::Tensor& layout_sfa, + const torch::Tensor& layout_sfb, + const torch::Tensor& problem_sizes, + const torch::Tensor& expert_offsets, + const torch::Tensor& workspace); + +void prepare_moe_input( + const torch::Tensor& topk_ids, + torch::Tensor& expert_offsets, + const std::optional& blockscale_offsets, + torch::Tensor& problem_sizes1, + torch::Tensor& problem_sizes2, + torch::Tensor& input_permutation, + torch::Tensor& output_permutation, + const int64_t num_experts, + const int64_t n, + const int64_t k); + +void shuffle_rows(const torch::Tensor& input_tensor, const torch::Tensor& dst2src_map, torch::Tensor& output_tensor); + +void apply_shuffle_mul_sum( + const torch::Tensor& input, + torch::Tensor& output, + const torch::Tensor& permutation, + const std::optional& factors); + +void fused_qk_norm_rope( + torch::Tensor& qkv, + int64_t num_heads_q, + int64_t num_heads_k, + int64_t num_heads_v, + int64_t head_dim, + double eps, + torch::Tensor& q_weight, + torch::Tensor& k_weight, + double base, + bool is_neox, + torch::Tensor& position_ids, + double factor, + double low, + double high, + double attention_factor); + +void cutlass_fp4_group_mm( + torch::Tensor& output, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& a_blockscale, + const torch::Tensor& b_blockscales, + const torch::Tensor& alphas, + const torch::Tensor& ab_strides, + const torch::Tensor& c_strides, + const torch::Tensor& problem_sizes, + const torch::Tensor& expert_offsets, + const torch::Tensor& sf_offsets); + +void scaled_fp4_experts_quant( + torch::Tensor& output, + torch::Tensor& output_scale, + torch::Tensor const& input, + torch::Tensor const& input_global_scale, + torch::Tensor const& input_offset_by_experts, + torch::Tensor const& output_scale_offset_by_experts); + +void silu_and_mul_scaled_fp4_experts_quant( + torch::Tensor& output, + torch::Tensor& output_scale, + torch::Tensor const& input, + torch::Tensor const& input_global_scale, + torch::Tensor const& mask, + bool use_silu_and_mul); + +/* + * From csrc/moe/cutlass_moe/w4a8 + */ +void get_cutlass_w4a8_moe_mm_data( + const torch::Tensor& topk_ids, + torch::Tensor& expert_offsets, + torch::Tensor& problem_sizes1, + torch::Tensor& problem_sizes2, + torch::Tensor& input_permutation, + torch::Tensor& output_permutation, + const int64_t num_experts, + const int64_t n, + const int64_t k); + +void cutlass_w4a8_moe_mm( + torch::Tensor& d_tensors, + torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + torch::Tensor const& expert_offsets, + torch::Tensor const& problem_sizes, + torch::Tensor const& a_strides, + torch::Tensor const& b_strides, + torch::Tensor const& d_strides, + torch::Tensor const& s_strides, + int64_t chunk_size, + int64_t topk); +/* + * From csrc/moe/marlin_moe_wna16 + */ +torch::Tensor moe_wna16_marlin_gemm( + torch::Tensor& a, + std::optional const& c_or_none, + torch::Tensor& b_q_weight, + std::optional const& b_bias_or_none, + torch::Tensor& b_scales, + std::optional const& global_scale_or_none, + std::optional const& b_zeros_or_none, + std::optional const& g_idx_or_none, + std::optional const& perm_or_none, + torch::Tensor& workspace, + torch::Tensor& sorted_token_ids, + torch::Tensor& expert_ids, + torch::Tensor& num_tokens_past_padded, + torch::Tensor& topk_weights, + int64_t moe_block_size, + int64_t top_k, + bool mul_topk_weights, + bool is_ep, + sglang::ScalarTypeId const& b_q_type_id, + int64_t size_m, + int64_t size_n, + int64_t size_k, + bool is_k_full, + bool use_atomic_add, + bool use_fp32_reduce, + bool is_zp_float); + +/* + * From csrc/speculative + */ +void tree_speculative_sampling_target_only( + at::Tensor predicts, // mutable + at::Tensor accept_index, // mutable + at::Tensor accept_token_num, // mutable + at::Tensor candidates, + at::Tensor retrive_index, + at::Tensor retrive_next_token, + at::Tensor retrive_next_sibling, + at::Tensor uniform_samples, + at::Tensor uniform_samples_for_final_sampling, + at::Tensor target_probs, + at::Tensor draft_probs, + double threshold_single = 1, + double threshold_acc = 1, + bool deterministic = true); + +void verify_tree_greedy( + at::Tensor predicts, // mutable + at::Tensor accept_index, // mutable + at::Tensor accept_token_num, // mutable + at::Tensor candidates, + at::Tensor retrive_index, + at::Tensor retrive_next_token, + at::Tensor retrive_next_sibling, + at::Tensor target_predict); + +void reconstruct_indices_from_tree_mask( + at::Tensor tree_mask, + at::Tensor verified_seq_len, + at::Tensor positions, // mutable + at::Tensor retrive_index, // mutable + at::Tensor retrive_next_token, // mutable + at::Tensor retrive_next_sibling, // mutable + int64_t batch_size, + int64_t draft_token_num); + +void build_tree_kernel_efficient( + at::Tensor parent_list, + at::Tensor selected_index, + at::Tensor verified_seq_len, + at::Tensor tree_mask, + at::Tensor positions, + at::Tensor retrive_index, + at::Tensor retrive_next_token, + at::Tensor retrive_next_sibling, + int64_t topk, + int64_t depth, + int64_t draft_token_num, + int64_t tree_mask_mode); + +void segment_packbits( + at::Tensor x, + at::Tensor input_indptr, + at::Tensor output_indptr, + at::Tensor y, + int64_t batch_size, + int64_t cuda_stream = 0); + +/* + * From csrc/kvcacheio + */ +void transfer_kv_per_layer( + const at::Tensor src_k, + at::Tensor dst_k, + const at::Tensor src_v, + at::Tensor dst_v, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t block_quota, + int64_t num_warps_per_block); + +void transfer_kv_per_layer_pf_lf( + const at::Tensor src_k, + at::Tensor dst_k, + const at::Tensor src_v, + at::Tensor dst_v, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t layer_id, + int64_t item_size, + int64_t src_layout_dim, + int64_t block_quota, + int64_t num_warps_per_block); + +void transfer_kv_per_layer_ph_lf( + const at::Tensor src_k, + at::Tensor dst_k, + const at::Tensor src_v, + at::Tensor dst_v, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t layer_id, + int64_t item_size, + int64_t src_layout_dim, + int64_t page_size, + int64_t head_num, + int64_t block_quota, + int64_t num_warps_per_block); + +void transfer_kv_all_layer( + const at::Tensor src_k_layers, + const at::Tensor dst_k_layers, + const at::Tensor src_v_layers, + const at::Tensor dst_v_layers, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t num_layers, + int64_t block_quota, + int64_t num_warps_per_block); + +void transfer_kv_all_layer_lf_pf( + const at::Tensor src_k_layers, + at::Tensor dst_k, + const at::Tensor src_v_layers, + at::Tensor dst_v, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t dst_layout_dim, + int64_t num_layers, + int64_t block_quota, + int64_t num_warps_per_block); + +void transfer_kv_all_layer_lf_ph( + const at::Tensor src_k_layers, + at::Tensor dst_k, + const at::Tensor src_v_layers, + at::Tensor dst_v, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t dst_layout_dim, + int64_t num_layers, + int64_t page_size, + int64_t head_num, + int64_t block_quota, + int64_t num_warps_per_block); + +void transfer_kv_per_layer_mla( + const at::Tensor src, + at::Tensor dst, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t block_quota, + int64_t num_warps_per_block); + +void transfer_kv_per_layer_mla_pf_lf( + const at::Tensor src, + at::Tensor dst, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t layer_id, + int64_t item_size, + int64_t src_layout_dim, + int64_t block_quota, + int64_t num_warps_per_block); + +void transfer_kv_all_layer_mla( + const at::Tensor src_layers, + const at::Tensor dst_layers, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t num_layers, + int64_t block_quota, + int64_t num_warps_per_block); + +void transfer_kv_all_layer_mla_lf_pf( + const at::Tensor src_layers, + at::Tensor dst, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t dst_layout_dim, + int64_t num_layers, + int64_t block_quota, + int64_t num_warps_per_block); + +void transfer_kv_direct( + const std::vector& src_layers, + std::vector dst_layers, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t page_size); + +void transfer_kv_per_layer_direct_pf_lf( + const std::vector& src_ptrs, + std::vector dst_ptrs, + const at::Tensor& src_indices, + const at::Tensor& dst_indices, + int64_t layer_id, + int64_t page_size); + +void transfer_kv_all_layer_direct_lf_pf( + const std::vector& src_ptrs, + std::vector dst_ptrs, + const at::Tensor& src_indices, + const at::Tensor& dst_indices, + int64_t page_size); + +/* + * From csrc/memory + */ +at::Tensor weak_ref_tensor(const at::Tensor& tensor); +void store_kv_cache(at::Tensor k_cache, at::Tensor v_cache, at::Tensor out_loc, at::Tensor k, at::Tensor v); + +/* + * From FlashInfer + */ +void min_p_sampling_from_probs( + at::Tensor probs, + at::Tensor output, + std::optional maybe_indices, + std::optional maybe_min_p_arr, + double min_p_val, + bool deterministic, + std::optional gen); + +void top_k_renorm_probs( + at::Tensor probs, at::Tensor renorm_probs, std::optional maybe_top_k_arr, int64_t top_k_val); + +void top_p_renorm_probs( + at::Tensor probs, at::Tensor renorm_probs, std::optional maybe_top_p_arr, double top_p_val); + +void top_k_top_p_sampling_from_probs( + at::Tensor probs, + at::Tensor output, + std::optional maybe_indices, + std::optional maybe_top_k_arr, + double top_k_val, + std::optional maybe_top_p_arr, + double top_p_val, + bool deterministic, + std::optional gen); + +void top_p_sampling_from_probs( + at::Tensor probs, + at::Tensor output, + std::optional maybe_indices, + std::optional maybe_top_p_arr, + double top_p_val, + bool deterministic, + std::optional gen); + +void top_k_mask_logits( + at::Tensor logits, at::Tensor mask_logits, std::optional maybe_top_k_arr, int64_t top_k_val); + +namespace flash { +/* + * From fa2 sparse + */ +std::vector mha_fwd_sparse( + at::Tensor& q, // batch_size x seqlen_q x num_heads x head_size + const at::Tensor& k, // batch_size x seqlen_k x num_heads_k x head_size + const at::Tensor& v, // batch_size x seqlen_k x num_heads_k x head_size + const at::Tensor& block_count, + const at::Tensor& block_offset, + const at::Tensor& column_count, + const at::Tensor& column_index, + const std::optional& out_, // batch_size x seqlen_q x num_heads x head_size + const std::optional& alibi_slopes_, // num_heads or batch_size x num_heads + const double p_dropout, + const double softmax_scale, + bool is_causal, + const double softcap, + const bool return_softmax, + std::optional gen_); + +std::vector mha_varlen_fwd_sparse( + at::Tensor& q, // total_q x num_heads x head_size, total_q := \sum_{i=0}^{b} s_i + const at::Tensor& k, // total_k x num_heads_k x head_size, total_k := \sum_{i=0}^{b} s_i. + const at::Tensor& v, // total_k x num_heads_k x head_size, total_k := \sum_{i=0}^{b} s_i. + const at::Tensor& block_count, + const at::Tensor& block_offset, + const at::Tensor& column_count, + const at::Tensor& column_index, + const c10::optional& out_, // total_q x num_heads x head_size, total_k := \sum_{i=0}^{b} s_i + const at::Tensor& cu_seqlens_q, // b+1 + const at::Tensor& cu_seqlens_k, // b+1 + const c10::optional& + seqused_k, // b. If given, only this many elements of each batch element's keys are used. + const c10::optional& alibi_slopes_, // num_heads or b x num_heads + int64_t max_seqlen_q, + const int64_t max_seqlen_k, + const double p_dropout, + const double softmax_scale, + const bool zero_tensors, + bool is_causal, + const double softcap, + const bool return_softmax, + c10::optional gen_); +} // namespace flash + +void convert_vertical_slash_indexes( + torch::Tensor& block_count, // [BATCH, N_HEADS, NUM_ROWS] + torch::Tensor& block_offset, // [BATCH, N_HEADS, NUM_ROWS, NNZ_S] + torch::Tensor& column_count, // [BATCH, N_HEADS, NUM_ROWS] + torch::Tensor& column_index, // [BATCH, N_HEADS, NUM_ROWS, NNZ_V] + torch::Tensor q_seqlens, // [BATCH, ] + torch::Tensor kv_seqlens, // [BATCH, ] + torch::Tensor vertical_indexes, // [BATCH, N_HEADS, NNZ_V] + torch::Tensor slash_indexes, // [BATCH, N_HEADS, NNZ_S] + int64_t context_size, + int64_t block_size_M, + int64_t block_size_N, + bool causal); + +void convert_vertical_slash_indexes_mergehead( + torch::Tensor& block_count, // [BATCH, N_HEADS, NUM_ROWS] + torch::Tensor& block_offset, // [BATCH, N_HEADS, NUM_ROWS, NNZ_S] + torch::Tensor& column_count, // [BATCH, N_HEADS, NUM_ROWS] + torch::Tensor& column_index, // [BATCH, N_HEADS, NUM_ROWS, NNZ_V] + torch::Tensor q_seqlens, // [BATCH, ] + torch::Tensor kv_seqlens, // [BATCH, ] + torch::Tensor vertical_indexes, // [BATCH, N_HEADS, NNZ_V] + torch::Tensor slash_indexes, // [BATCH, N_HEADS, NNZ_S] + torch::Tensor vertical_indices_count, // [N_HEADS, ] + torch::Tensor slash_indices_count, + int64_t context_size, + int64_t block_size_M, + int64_t block_size_N, + bool causal); + +/* + * From csrc/grammar + */ +void ApplyTokenBitmaskInplace(at::Tensor logits, at::Tensor bitmask, at::optional indices = at::nullopt); + +/* + * From csrc/gemm (QServe) + */ +void qserve_w4a8_per_chn_gemm( + const torch::Tensor& _in_feats, + const torch::Tensor& _kernel, + const torch::Tensor& _wscales, + const torch::Tensor& _ascales, + const torch::Tensor& _w_szs, + const torch::Tensor& _a_ssums, + torch::Tensor& _out_feats); + +void qserve_w4a8_per_group_gemm( + const torch::Tensor& _in_feats, + const torch::Tensor& _kernel, + const torch::Tensor& _zeros, + const torch::Tensor& _scales_i8, + const torch::Tensor& _wscales, + const torch::Tensor& _ascales, + torch::Tensor& _out_feats); + +/* + * From csrc/quantization/gguf + */ +torch::Tensor +ggml_dequantize(torch::Tensor W, int64_t type, int64_t m, int64_t n, std::optional const& dtype); + +torch::Tensor ggml_mul_mat_vec_a8(torch::Tensor W, torch::Tensor X, int64_t type, int64_t row); + +torch::Tensor ggml_mul_mat_a8(torch::Tensor W, torch::Tensor X, int64_t type, int64_t row); + +torch::Tensor ggml_moe_a8( + torch::Tensor X, + torch::Tensor W, + torch::Tensor sorted_token_ids, + torch::Tensor expert_ids, + torch::Tensor num_tokens_post_padded, + int64_t type, + int64_t row, + int64_t top_k, + int64_t tokens); + +torch::Tensor ggml_moe_a8_vec( + torch::Tensor X, torch::Tensor W, torch::Tensor topk_ids, int64_t top_k, int64_t type, int64_t row, int64_t tokens); + +int64_t ggml_moe_get_block_size(int64_t type); + +/* + * From csrc/spatial + */ +std::vector create_greenctx_stream_by_value(int64_t smA, int64_t smB, int64_t device); + +/* + * From csrc/mamba + */ +void causal_conv1d_update( + const at::Tensor& x, + const at::Tensor& conv_state, + const at::Tensor& weight, + const std::optional& bias_, + bool silu_activation, + const std::optional& cache_seqlens_, + const std::optional& conv_state_indices_, + int64_t pad_slot_id); + +void causal_conv1d_fwd( + const at::Tensor& x, + const at::Tensor& weight, + const std::optional& bias_, + const std::optional& conv_states, + const std::optional& query_start_loc, + const std::optional& cache_indices, + const std::optional& has_initial_state, + bool silu_activation, + int64_t pad_slot_id); + +/* + * From csrc/expert_specialization + */ +void es_fp8_blockwise_scaled_grouped_mm( + torch::Tensor& output, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Tensor& stride_a, + const torch::Tensor& stride_b, + const torch::Tensor& stride_d, + const torch::Tensor& problem_sizes, + const torch::Tensor& expert_offsets, + const torch::Tensor& workspace); + +/* + * From fast-hadamard-transform + */ +torch::Tensor fast_hadamard_transform(torch::Tensor& x, double scale); +torch::Tensor fast_hadamard_transform_12N(torch::Tensor& x, double scale); +torch::Tensor fast_hadamard_transform_20N(torch::Tensor& x, double scale); +torch::Tensor fast_hadamard_transform_28N(torch::Tensor& x, double scale); +torch::Tensor fast_hadamard_transform_40N(torch::Tensor& x, double scale); + +/* + * From flashmla + */ +std::vector get_mla_decoding_metadata( + at::Tensor& seqlens_k, + const int64_t num_q_tokens_per_head_k, + const int64_t h_k, + const std::optional h_q, + const bool is_fp8_kvcache, + const std::optional topk); + +std::vector fwd_kvcache_mla( + at::Tensor& q, // batch_size x seqlen_q x num_heads x head_size + const at::Tensor& kcache, // num_blocks x page_block_size x num_heads_k x head_size (when is_fp8 is False) or + // num_blocks x num_heads_k x (page_block_size*656) (when is_fp8 is True) + const int64_t head_size_v, + const at::Tensor& seqlens_k, // batch_size + const at::Tensor& block_table, // batch_size x max_num_blocks_per_seq + const double softmax_scale, + bool is_causal, + const at::Tensor& tile_scheduler_metadata, // num_sm_parts x TileSchedulerMetaDataSize + const at::Tensor& num_splits, // batch_size + 1 + const bool& is_fp8, + const std::optional& indices // None, or batch_size x seqlen_q x topk +); + +void FMHACutlassSM100FwdRun( + at::Tensor workspace_buffer, + at::Tensor q, + at::Tensor k, + at::Tensor v, + at::Tensor cumulative_seqlen_q, + at::Tensor cumulative_seqlen_kv, + at::Tensor o, + at::Tensor lse, + int64_t mask_mode_code, + double softmax_scale, + int64_t max_seqlen_q, + int64_t max_seqlen_kv, + bool is_varlen); + +std::vector +sparse_prefill_fwd(const at::Tensor& q, const at::Tensor& kv, const at::Tensor& indices, double sm_scale, int64_t d_v); + +std::vector fwd_kvcache_mla_fp8( + at::Tensor& q, // batch_size x seqlen_q x num_heads x head_size + const at::Tensor& kcache, // num_blocks x page_block_size x num_heads_k x head_size (when is_fp8 is False) or + // num_blocks x num_heads_k x (page_block_size*656) (when is_fp8 is True) + const int64_t head_size_v, + const at::Tensor& seqlens_k, // batch_size + const at::Tensor& block_table, // batch_size x max_num_blocks_per_seq + const double softmax_scale, + bool is_causal, + const at::Tensor& tile_scheduler_metadata, // num_sm_parts x TileSchedulerMetaDataSize + const at::Tensor& num_splits, // batch_size + 1 + const std::optional& descale_q, // None or batch_size + const std::optional& descale_k // None or batch_size +); + +std::vector get_mla_decoding_metadata_dense_fp8( + at::Tensor& seqlens_k, const int64_t num_heads_per_head_k, const int64_t num_heads_k); diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/sgl_kernel_torch_shim.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/sgl_kernel_torch_shim.h new file mode 100644 index 00000000..5a26f907 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/sgl_kernel_torch_shim.h @@ -0,0 +1,122 @@ +/*Adapt from: +https://github.com/neuralmagic/vllm-flash-attention/blob/90eacc1af2a7c3de62ea249e929ed5faccf38954/csrc/common/pytorch_shim.h + Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#pragma once + +#include + +/** + * Unfortunately, the type signatures of the flash_attn ops are not compatible + * with the PyTorch library bindings. To get around that we use + * `make_pytorch_shim` which creates a lambda that exposes the API using + * PyTorch compatible types to the types, then converts them to the types + * expected by the flash_attn ops. This shims allows us to make minimal changes + * to `flash_api.cpp` making it easier to synchronize with upstream changes. + * + * The `pytorch_library_compatible_type` struct is used to map from the + * flash_attn ops types to a PyTorch library compatible one. The main issues is + * that the following types are not support by PyTorch library bindings: + * - `int` + * - `float` + * - `std::optional &` + * - `std::optional &` + * So we convert them to (respectively): + * - `int64_t` + * - `double` + * - `const std::optional&` + * - `const std::optional&` + */ + +template +struct pytorch_library_compatible_type { + using type = T; + static T convert_from_type(T arg) { + return arg; + } +}; + +template +using pytorch_library_compatible_type_t = typename pytorch_library_compatible_type::type; + +template +T convert_from_pytorch_compatible_type(pytorch_library_compatible_type_t arg) { + return pytorch_library_compatible_type::convert_from_type(arg); +} + +// Map `c10::optional &` -> `const c10::optional&` +// (NOTE: this is bit unsafe but non of the ops in flash_attn mutate +// the optional container) +template +struct pytorch_library_compatible_type&> { + using type = const c10::optional&; + static c10::optional& convert_from_type(const c10::optional& arg) { + return const_cast&>(arg); + } +}; + +// Map `c10::optional` -> +// `c10::optional>` +// (NOTE: tested for `c10::optional` -> `c10::optional`) +template +struct pytorch_library_compatible_type> { + using type = c10::optional>; + static c10::optional> convert_from_type(c10::optional arg) { + return arg; + } +}; + +// Map `c10::optional&` -> `const c10::optional&` +template <> +struct pytorch_library_compatible_type&> { + using type = const c10::optional&; + static c10::optional& convert_from_type(const c10::optional& arg) { + return const_cast&>(reinterpret_cast&>(arg)); + } +}; + +// Map `int` -> `int64_t` +template <> +struct pytorch_library_compatible_type { + using type = int64_t; + static int convert_from_type(int64_t arg) { + TORCH_CHECK(arg <= std::numeric_limits::max(), "int64_t value is too large to be converted to int"); + TORCH_CHECK(arg >= std::numeric_limits::min(), "int64_t value is too small to be converted to int"); + return arg; + } +}; + +// Map `float` -> `double` +template <> +struct pytorch_library_compatible_type { + using type = double; + static float convert_from_type(double arg) { + TORCH_CHECK( + std::abs(arg) <= std::numeric_limits::max(), "double value is too large to be converted to float"); + return arg; + } +}; + +// +// Shim Utils +// + +template +auto make_pytorch_shim(Ret (*fun)(Args... args)) { + return [fun](pytorch_library_compatible_type_t... args) { + return fun(convert_from_pytorch_compatible_type(args)...); + }; +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/utils.h b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/utils.h new file mode 100644 index 00000000..c5a517ce --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/include/utils.h @@ -0,0 +1,469 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#pragma once + +#include +#include +#include + +#ifdef USE_ROCM +#include +#endif + +#ifdef USE_ROCM +// Adapted from flashinfer-rocm [PR#491](https://github.com/flashinfer-ai/flashinfer/pull/491) +#define _DISPATCH_CASE_F16(c_type, ...) \ + case at::ScalarType::Half: { \ + using c_type = __half; \ + return __VA_ARGS__(); \ + } + +#define _DISPATCH_CASE_BF16(c_type, ...) \ + case at::ScalarType::BFloat16: { \ + using c_type = __hip_bfloat16; \ + return __VA_ARGS__(); \ + } +#endif // USE_ROCM + +#ifndef USE_ROCM +// Adapt from FlashInfer +#ifdef FLASHINFER_ENABLE_F16 +#define _DISPATCH_CASE_F16(c_type, ...) \ + case at::ScalarType::Half: { \ + using c_type = nv_half; \ + return __VA_ARGS__(); \ + } +#else +#define _DISPATCH_CASE_F16(c_type, ...) +#endif // FLASHINFER_ENABLE_F16 + +#ifdef FLASHINFER_ENABLE_BF16 +#define _DISPATCH_CASE_BF16(c_type, ...) \ + case at::ScalarType::BFloat16: { \ + using c_type = nv_bfloat16; \ + return __VA_ARGS__(); \ + } +#else +#define _DISPATCH_CASE_BF16(c_type, ...) +#endif // FLASHINFER_ENABLE_BF16 + +#ifdef FLASHINFER_ENABLE_FP8_E4M3 +#define _DISPATCH_CASE_FP8_E4M3(c_type, ...) \ + case at::ScalarType::Float8_e4m3fn: { \ + using c_type = __nv_fp8_e4m3; \ + return __VA_ARGS__(); \ + } +#else +#define _DISPATCH_CASE_FP8_E4M3(c_type, ...) +#endif // FLASHINFER_ENABLE_FP8_E4M3 + +#ifdef FLASHINFER_ENABLE_FP8_E5M2 +#define _DISPATCH_CASE_FP8_E5M2(c_type, ...) \ + case at::ScalarType::Float8_e5m2: { \ + using c_type = __nv_fp8_e5m2; \ + return __VA_ARGS__(); \ + } +#else +#define _DISPATCH_CASE_FP8_E5M2(c_type, ...) +#endif // FLASHINFER_ENABLE_FP8_E5M2 + +#define DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP16(pytorch_dtype, c_type, ...) \ + [&]() -> bool { \ + switch (pytorch_dtype) { \ + _DISPATCH_CASE_F16(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_BF16(c_type, __VA_ARGS__) \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch data type " << pytorch_dtype; \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() + +#define DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP8(pytorch_dtype, c_type, ...) \ + [&]() -> bool { \ + switch (pytorch_dtype) { \ + _DISPATCH_CASE_FP8_E4M3(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_FP8_E5M2(c_type, __VA_ARGS__) \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch fp8 data type " << pytorch_dtype; \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() + +#define DISPATCH_PYTORCH_DTYPE_TO_CTYPE(pytorch_dtype, c_type, ...) \ + [&]() -> bool { \ + switch (pytorch_dtype) { \ + _DISPATCH_CASE_F16(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_BF16(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_FP8_E4M3(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_FP8_E5M2(c_type, __VA_ARGS__) \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch data type " << pytorch_dtype; \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() + +#define _DISPATCH_SWITCH(var_name, cond, ...) \ + [&]() -> bool { \ + switch (cond) { \ + __VA_ARGS__ \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch " var_name " " << int(cond); \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() + +#define _DISPATCH_SWITCH_U16x2(var1_name, var2_name, cond1, cond2, ...) \ + [&]() -> bool { \ + switch (pack_u16(cond1, cond2)) { \ + __VA_ARGS__ \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch (" var1_name ", " var2_name "): (" << int(cond1) << ", " \ + << int(cond2) << ")"; \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() + +#define _DISPATCH_CASE(case_expr, case_var, ...) \ + case case_expr: { \ + constexpr auto case_var = case_expr; \ + return __VA_ARGS__(); \ + } + +#define _DISPATCH_CASE_U16x2(case_expr1, case_expr2, case_var1, case_var2, ...) \ + case pack_u16(case_expr1, case_expr2): { \ + constexpr auto case_var1 = case_expr1; \ + constexpr auto case_var2 = case_expr2; \ + return __VA_ARGS__(); \ + } + +#define DISPATCH_BOOL(expr, const_expr, ...) \ + [&]() -> bool { \ + if (expr) { \ + constexpr bool const_expr = true; \ + return __VA_ARGS__(); \ + } else { \ + constexpr bool const_expr = false; \ + return __VA_ARGS__(); \ + } \ + }() + +inline void check_shape(const at::Tensor& a, const at::Tensor& b, const char* a_name, const char* b_name) { + TORCH_CHECK(a.dim() == b.dim(), a_name, ".dim() != ", b_name, ".dim(). ", a.dim(), " vs ", b.dim()); + for (int i = 0; i < a.dim(); ++i) { + TORCH_CHECK(a.size(i) == b.size(i), a_name, ".size(", i, ") != ", b_name, ".size(", i, ")"); + } +} + +inline constexpr uint32_t pack_u16(uint16_t a, uint16_t b) { + return (uint32_t(a) << 16) | uint32_t(b); +} + +#define CHECK_GQA_HEAD_DIVISIBLE(num_qo_heads, num_kv_heads) \ + TORCH_CHECK( \ + num_qo_heads % num_kv_heads == 0, \ + "num_qo_heads(", \ + num_qo_heads, \ + ") must be divisible by num_kv_heads(", \ + num_kv_heads, \ + ")") + +#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor") + +#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_LAST_DIM_CONTIGUOUS(x) \ + TORCH_CHECK(x.strides()[x.strides().size() - 1] == 1, #x "must be contiguous at last dimension") + +#define CHECK_INPUT(x) \ + CHECK_CUDA(x); \ + CHECK_CONTIGUOUS(x) +#define CHECK_LAST_DIM_CONTIGUOUS_INPUT(x) \ + CHECK_CUDA(x); \ + CHECK_LAST_DIM_CONTIGUOUS(x) + +#define CHECK_DIM(d, x) TORCH_CHECK(x.dim() == d, #x " must be a " #d "D tensor") + +#define CHECK_SHAPE(a, b) check_shape(a, b, #a, #b) + +#define CHECK_EQ(a, b) TORCH_CHECK((a) == (b), "CHECK_EQ(" #a ", " #b ") failed. ", a, " vs ", b) + +#define CHECK_GE(a, b) TORCH_CHECK((a) >= (b), "CHECK_GE(" #a ", " #b ") failed. ", a, " vs ", b) + +inline bool is_float8_tensor(const at::Tensor& tensor) { + return tensor.scalar_type() == at::ScalarType::Float8_e4m3fn || tensor.scalar_type() == at::ScalarType::Float8_e5m2; +} +#endif // USE_ROCM + +struct cuda_error : public std::runtime_error { + /** + * @brief Constructs a `cuda_error` object with the given `message`. + * + * @param message The error char array used to construct `cuda_error` + */ + cuda_error(const char* message) : std::runtime_error(message) {} + /** + * @brief Constructs a `cuda_error` object with the given `message` string. + * + * @param message The `std::string` used to construct `cuda_error` + */ + cuda_error(std::string const& message) : cuda_error{message.c_str()} {} +}; + +#define CHECK_CUDA_SUCCESS(cmd) \ + do { \ + cudaError_t e = cmd; \ + if (e != cudaSuccess) { \ + std::stringstream _message; \ + auto s = cudaGetErrorString(e); \ + _message << std::string(s) + "\n" << __FILE__ << ':' << __LINE__; \ + throw cuda_error(_message.str()); \ + } \ + } while (0) + +#define CHECK_IS_CUDA(x) TORCH_CHECK(x.device().is_cuda(), #x " must be a CUDA tensor") +#define CHECK_IS_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_CUDA_INPUT(x) \ + CHECK_IS_CUDA(x); \ + CHECK_IS_CONTIGUOUS(x) + +inline int getSMVersion() { + int device{-1}; + CHECK_CUDA_SUCCESS(cudaGetDevice(&device)); + int sm_major = 0; + int sm_minor = 0; + CHECK_CUDA_SUCCESS(cudaDeviceGetAttribute(&sm_major, cudaDevAttrComputeCapabilityMajor, device)); + CHECK_CUDA_SUCCESS(cudaDeviceGetAttribute(&sm_minor, cudaDevAttrComputeCapabilityMinor, device)); + return sm_major * 10 + sm_minor; +} + +inline bool isDeviceType(const std::string& device_type) { + int deviceCount; + CHECK_CUDA_SUCCESS(cudaGetDeviceCount(&deviceCount)); + + int device_id = -1; + if (deviceCount >= 1) { + CHECK_CUDA_SUCCESS(cudaGetDevice(&device_id)); + } else { + return false; + } + + cudaDeviceProp prop; + CHECK_CUDA_SUCCESS(cudaGetDeviceProperties(&prop, device_id)); + if (device_type == std::string(prop.name)) { + return true; + } + return false; +} + +inline bool getBoolEnv(char const* name) { + char const* env = std::getenv(name); + return env && env[0] == '1' && env[1] == '\0'; +} + +inline bool getEnvEnablePDL() { + static std::once_flag flag; + static bool enablePDL = false; + std::call_once(flag, [&]() { + if (getSMVersion() >= 90) { + // PDL will be enabled by setting the env variables `TRTLLM_ENABLE_PDL` to `1` + enablePDL = getBoolEnv("TRTLLM_ENABLE_PDL"); + } + }); + return enablePDL; +} + +// SGLANG_SHFL_XOR_* adapted from https://github.com/vllm-project/vllm/blob/v0.7.3/csrc/cuda_compat.h#L19-L28 +#ifndef USE_ROCM +#define SGLANG_SHFL_XOR_SYNC(mask, var, lane_mask) __shfl_xor_sync((mask), (var), (lane_mask)) +#define SGLANG_SHFL_XOR_SYNC_WIDTH(mask, var, lane_mask, width) __shfl_xor_sync((mask), (var), (lane_mask), (width)) +#else +#define SGLANG_SHFL_XOR_SYNC(mask, var, lane_mask) __shfl_xor((var), (lane_mask)) +#define SGLANG_SHFL_XOR_SYNC_WIDTH(mask, var, lane_mask, width) __shfl_xor((var), (lane_mask), (width)) +#endif + +#define DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(pytorch_dtype, c_type, ...) \ + [&]() -> bool { \ + switch (pytorch_dtype) { \ + case at::ScalarType::Float: { \ + using c_type = float; \ + return __VA_ARGS__(); \ + } \ + _DISPATCH_CASE_F16(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_BF16(c_type, __VA_ARGS__) \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch data type " << pytorch_dtype; \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() + +#define DISPATCH_CASE_INTEGRAL_TYPES(...) \ + AT_DISPATCH_CASE(at::ScalarType::Byte, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Char, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Short, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Int, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Long, __VA_ARGS__) + +#define DISPATCH_INTEGRAL_TYPES(TYPE, NAME, ...) \ + AT_DISPATCH_SWITCH(TYPE, NAME, DISPATCH_CASE_INTEGRAL_TYPES(__VA_ARGS__)) + +#define DISPATCH_CASE_FLOAT_TYPES(...) \ + AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) + +#define DISPATCH_FLOAT_TYPES(TYPE, NAME, ...) AT_DISPATCH_SWITCH(TYPE, NAME, DISPATCH_CASE_FLOAT_TYPES(__VA_ARGS__)) + +#define CEILDIV(x, y) (((x) + (y) - 1) / (y)) + +#ifndef USE_ROCM +#define WARP_SIZE 32 +#else +#if defined(__GFX9__) || !defined(__HIP_DEVICE_COMPILE__) +#define WARP_SIZE 64 +#else +#define WARP_SIZE 32 +#endif +#endif + +#ifdef USE_ROCM + +#include "hip/hip_math_def.h" +#include "hip/hip_vec_dtypes.h" + +#else + +template +__device__ __forceinline__ float castToFloat(srcDtype val) { + return static_cast(val); +} + +template +__device__ __forceinline__ dstDtype castFromFloat(float val) { + return static_cast(val); +} + +#endif + +// add FP8 support +#ifndef USE_ROCM +#include +using FP8_TYPE = c10::Float8_e4m3fn; +C10_HOST_DEVICE constexpr auto FP8_E4M3_MAX = std::numeric_limits::max(); +#else // USE_ROCM +#if HIP_FP8_TYPE_FNUZ +#include +using FP8_TYPE = c10::Float8_e4m3fnuz; +constexpr auto FP8_E4M3_MAX = 224.0f; +#else +#if HIP_FP8_TYPE_E4M3 +#include +using FP8_TYPE = c10::Float8_e4m3fn; +C10_HOST_DEVICE constexpr auto FP8_E4M3_MAX = std::numeric_limits::max(); +#else +#error "fp8 is not supported in this processor (arch < gfx942)." +#endif // HIP_FP8_TYPE_E4M3 +#endif // HIP_FP8_TYPE_FNUZ +#endif // USE_ROCM + +#define FULL_MASK 0xffffffff + +__device__ __forceinline__ float atomicMaxFloat(float* addr, float value) { +#ifndef USE_ROCM + float old; + old = (value >= 0) ? __int_as_float(atomicMax((int*)addr, __float_as_int(value))) + : __uint_as_float(atomicMin((unsigned int*)addr, __float_as_uint(value))); + return old; +#else + int* addr_as_i = (int*)addr; + int old = *addr_as_i, assumed; + do { + assumed = old; + old = atomicCAS(addr_as_i, assumed, __float_as_int(fmaxf(value, __int_as_float(assumed)))); + } while (assumed != old); + return __int_as_float(old); +#endif +} + +__device__ __forceinline__ float warpReduceMax(float value) { + value = fmaxf(value, __shfl_xor_sync(FULL_MASK, value, 16)); + value = fmaxf(value, __shfl_xor_sync(FULL_MASK, value, 8)); + value = fmaxf(value, __shfl_xor_sync(FULL_MASK, value, 4)); + value = fmaxf(value, __shfl_xor_sync(FULL_MASK, value, 2)); + value = fmaxf(value, __shfl_xor_sync(FULL_MASK, value, 1)); + return value; +} + +__device__ __forceinline__ float blockReduceMax(float value) { + static __shared__ float warpLevelMaxs[WARP_SIZE]; + const int laneId = threadIdx.x % WARP_SIZE; + const int warpId = threadIdx.x / WARP_SIZE; + + value = warpReduceMax(value); + + if (laneId == 0) warpLevelMaxs[warpId] = value; + __syncthreads(); + + value = (threadIdx.x < blockDim.x / WARP_SIZE) ? warpLevelMaxs[laneId] : 0; + if (warpId == 0) value = warpReduceMax(value); + + return value; +} + +// Pads to a multiple of `alignment` rows. +inline torch::Tensor pad_tensor(const torch::Tensor& tensor, int64_t alignment = 4, bool is_column_major = false) { + int64_t rows = tensor.size(0); + int64_t cols = tensor.size(1); + int64_t pad_rows = (alignment - (rows % alignment)) % alignment; // Compute padding size + + if (pad_rows == 0) { + return tensor; // Already aligned + } + + torch::Tensor padding = torch::zeros({pad_rows, cols}, tensor.options()); + torch::Tensor tensor_padded = torch::cat({tensor, padding}, 0); // Pad along rows + + // Ensure column-major layout + if (is_column_major) { + return tensor_padded.t().contiguous().t(); + } + return tensor_padded; +} + +// Get the next power of 2 of a number +inline uint32_t next_pow2(uint32_t x) noexcept { + if (x <= 1) return 1; + return 1u << (32 - __builtin_clz(x - 1)); +} + +/* + * LDG Support + */ +#ifndef USE_ROCM +#define SGLANG_LDG(arg) __ldg(arg) +#else +#define SGLANG_LDG(arg) *(arg) +#endif diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/kernel-runner-setup.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/kernel-runner-setup.sh new file mode 100755 index 00000000..d7411d1e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/kernel-runner-setup.sh @@ -0,0 +1,150 @@ +#!/bin/bash +set -e + +CUDA_VERSIONS="${1:-12-8,12-9}" + +echo "===================================" +echo "Installing Docker..." +echo "===================================" + +# Add Docker's official GPG key: +sudo apt-get update +sudo apt-get install -y ca-certificates curl +sudo install -m 0755 -d /etc/apt/keyrings +sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc +sudo chmod a+r /etc/apt/keyrings/docker.asc + +# Add the repository to Apt sources: +echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt-get update + +sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + +# Add current user to docker group +sudo usermod -aG docker $USER + +echo "Docker installed successfully!" +echo "Note: You need to log out and log back in for docker group membership to take effect" +echo "" + +# Detect architecture for Docker image selection +ARCH=$(uname -m) + +if [ "$ARCH" = "x86_64" ]; then + BUILDER_NAME="pytorch/manylinux2_28-builder" +elif [ "$ARCH" = "aarch64" ]; then + BUILDER_NAME="pytorch/manylinuxaarch64-builder" +else + echo "Unsupported architecture: $ARCH" + exit 1 +fi + +# Pull Docker images for the specified CUDA versions +echo "===================================" +echo "Pulling Docker Images..." +echo "===================================" +echo "Architecture: ${ARCH}" +echo "Builder: ${BUILDER_NAME}" + +# Parse CUDA versions and pull corresponding Docker images +IFS=',' read -ra CUDA_VERSION_ARRAY <<< "$CUDA_VERSIONS" + +# Convert CUDA versions from format "12-8" to "12.8" and pull images +for CUDA_VERSION in "${CUDA_VERSION_ARRAY[@]}"; do + # Trim whitespace + CUDA_VERSION=$(echo "$CUDA_VERSION" | xargs) + + # Convert format: 12-8 -> 12.8 + CUDA_VERSION_DOTTED=$(echo "$CUDA_VERSION" | tr '-' '.') + + DOCKER_IMAGE="${BUILDER_NAME}:cuda${CUDA_VERSION_DOTTED}" + + echo "" + echo "Pulling ${DOCKER_IMAGE}..." + + # Use newgrp to ensure docker commands work (user was just added to docker group) + if sg docker -c "docker pull ${DOCKER_IMAGE}"; then + echo "✓ Successfully pulled ${DOCKER_IMAGE}" + else + echo "✗ Failed to pull ${DOCKER_IMAGE}" + echo " You may need to log out and log back in for docker group to take effect" + fi +done + +echo "" +echo "Docker images pulled successfully!" +echo "" + +# Auto-detect Ubuntu version +if command -v lsb_release &> /dev/null; then + UBUNTU_VERSION=$(lsb_release -rs | tr -d '.') +else + UBUNTU_VERSION=$(. /etc/os-release && echo $VERSION_ID | tr -d '.') +fi + +# Set CUDA architecture (ARCH already detected above for Docker images) +if [ "$ARCH" = "x86_64" ]; then + CUDA_ARCH="x86_64" +elif [ "$ARCH" = "aarch64" ]; then + CUDA_ARCH="sbsa" +else + echo "Unsupported architecture: $ARCH" + exit 1 +fi + +echo "===================================" +echo "System Information:" +echo "===================================" +echo "Ubuntu Version: ${UBUNTU_VERSION}" +echo "Architecture: ${ARCH}" +echo "CUDA Architecture: ${CUDA_ARCH}" +echo "" + +# Install CUDA keyring (only need to do this once) +echo "===================================" +echo "Installing CUDA keyring..." +echo "===================================" +KEYRING_URL="https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/${CUDA_ARCH}/cuda-keyring_1.1-1_all.deb" +wget -q $KEYRING_URL -O cuda-keyring.deb +sudo dpkg -i cuda-keyring.deb +sudo apt-get update +rm cuda-keyring.deb +echo "CUDA keyring installed successfully!" +echo "" + +# Split CUDA versions and install each one +IFS=',' read -ra CUDA_VERSION_ARRAY <<< "$CUDA_VERSIONS" + +echo "===================================" +echo "Installing CUDA Toolkits..." +echo "===================================" +echo "Versions to install: ${CUDA_VERSIONS}" +echo "" + +for CUDA_VERSION in "${CUDA_VERSION_ARRAY[@]}"; do + # Trim whitespace + CUDA_VERSION=$(echo "$CUDA_VERSION" | xargs) + + echo "-----------------------------------" + echo "Installing CUDA Toolkit ${CUDA_VERSION}..." + echo "-----------------------------------" + + if sudo apt-get install -y cuda-toolkit-${CUDA_VERSION}; then + echo "✓ CUDA Toolkit ${CUDA_VERSION} installed successfully!" + else + echo "✗ Failed to install CUDA Toolkit ${CUDA_VERSION}" + echo " This might be due to an invalid version or repository issue" + fi + echo "" +done + +echo "===================================" +echo "Installation Summary" +echo "===================================" +echo "Installed CUDA versions:" +ls -d /usr/local/cuda-* 2>/dev/null || echo "No CUDA installations found in /usr/local/" +echo "" +echo "Setup complete!" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/pyproject.toml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/pyproject.toml new file mode 100644 index 00000000..a8956856 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/pyproject.toml @@ -0,0 +1,42 @@ +[build-system] +requires = [ + "scikit-build-core>=0.10", + "torch>=2.8.0", + "wheel", +] +build-backend = "scikit_build_core.build" + +[project] +name = "sgl-kernel" +version = "0.3.18.post2" +authors = [ + { name="Yineng Zhang", email="me@zhyncs.com" }, +] +description = "Kernel Library for SGLang" +readme = "README.md" +requires-python = ">=3.10" +license = { file = "LICENSE" } +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Environment :: GPU :: NVIDIA CUDA" +] +dependencies = [] + +[project.urls] +"Homepage" = "https://github.com/sgl-project/sglang/tree/main/sgl-kernel" +"Bug Tracker" = "https://github.com/sgl-project/sglang/issues" + +[tool.wheel] +exclude = [ + "dist*", + "tests*", +] + +[tool.scikit-build] +cmake.build-type = "Release" +minimum-version = "build-system.requires" + +wheel.py-api = "cp310" +wheel.license-files = [] +wheel.packages = ["python/sgl_kernel"] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/pyproject_cpu.toml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/pyproject_cpu.toml new file mode 100644 index 00000000..40e1576d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/pyproject_cpu.toml @@ -0,0 +1,35 @@ +[build-system] +requires = [ + "scikit-build-core>=0.10", + "torch>=2.7.1", + "wheel", +] +build-backend = "scikit_build_core.build" + +[project] +name = "sgl-kernel" +version = "0.3.18.post2" +description = "Kernel Library for SGLang" +readme = "README.md" +requires-python = ">=3.10" +license = { file = "LICENSE" } +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License" +] +dependencies = [] + +[project.urls] +"Homepage" = "https://github.com/sgl-project/sglang/tree/main/sgl-kernel" +"Bug Tracker" = "https://github.com/sgl-project/sglang/issues" + +[tool.wheel] +exclude = [ + "dist*", + "tests*", +] + +[tool.scikit-build] +cmake.source-dir = "csrc/cpu" +cmake.build-type = "Release" +minimum-version = "build-system.requires" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/pyproject_rocm.toml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/pyproject_rocm.toml new file mode 100644 index 00000000..b5b00a56 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/pyproject_rocm.toml @@ -0,0 +1,32 @@ +[build-system] +requires = [ + "setuptools>=75.0", + "scikit-build-core>=0.10", + "torch>=2.8.0", + "wheel", +] +build-backend = "setuptools.build_meta" + +[project] +name = "sgl-kernel" +version = "0.3.18.post2" +description = "Kernel Library for SGLang" +readme = "README.md" +requires-python = ">=3.10" +license = { file = "LICENSE" } +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Environment :: GPU :: NVIDIA CUDA" +] +dependencies = [] + +[project.urls] +"Homepage" = "https://github.com/sgl-project/sglang/tree/main/sgl-kernel" +"Bug Tracker" = "https://github.com/sgl-project/sglang/issues" + +[tool.wheel] +exclude = [ + "dist*", + "tests*", +] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/__init__.py new file mode 100644 index 00000000..40cfd37f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/__init__.py @@ -0,0 +1,142 @@ +import torch +from sgl_kernel.load_utils import _load_architecture_specific_ops, _preload_cuda_library + +# Initialize the ops library based on current GPU +common_ops = _load_architecture_specific_ops() + +# Preload the CUDA library to avoid the issue of libcudart.so.12 not found +if torch.version.cuda is not None: + _preload_cuda_library() + + +from sgl_kernel.allreduce import * +from sgl_kernel.attention import ( + cutlass_mla_decode, + cutlass_mla_get_workspace_size, + merge_state, + merge_state_v2, +) +from sgl_kernel.cutlass_moe import cutlass_w4a8_moe_mm, get_cutlass_w4a8_moe_mm_data +from sgl_kernel.elementwise import ( + FusedSetKVBufferArg, + apply_rope_with_cos_sin_cache_inplace, + concat_mla_absorb_q, + concat_mla_k, + copy_to_gpu_no_ce, + downcast_fp8, + fused_add_rmsnorm, + gelu_and_mul, + gelu_tanh_and_mul, + gemma_fused_add_rmsnorm, + gemma_rmsnorm, + rmsnorm, + silu_and_mul, +) +from sgl_kernel.expert_specialization import es_fp8_blockwise_scaled_grouped_mm +from sgl_kernel.fused_moe import moe_wna16_marlin_gemm +from sgl_kernel.gemm import ( + awq_dequantize, + bmm_fp8, + cutlass_scaled_fp4_mm, + dsv3_fused_a_gemm, + dsv3_router_gemm, + fp8_blockwise_scaled_mm, + fp8_scaled_mm, + gptq_gemm, + gptq_marlin_gemm, + gptq_shuffle, + int8_scaled_mm, + qserve_w4a8_per_chn_gemm, + qserve_w4a8_per_group_gemm, + scaled_fp4_experts_quant, + scaled_fp4_grouped_quant, + scaled_fp4_quant, + sgl_per_tensor_quant_fp8, + sgl_per_token_group_quant_8bit, + sgl_per_token_group_quant_fp8, + sgl_per_token_group_quant_int8, + sgl_per_token_quant_fp8, + shuffle_rows, + silu_and_mul_scaled_fp4_grouped_quant, +) +from sgl_kernel.grammar import apply_token_bitmask_inplace_cuda +from sgl_kernel.hadamard import ( + hadamard_transform, + hadamard_transform_12n, + hadamard_transform_20n, + hadamard_transform_28n, + hadamard_transform_40n, +) +from sgl_kernel.kvcacheio import ( + transfer_kv_all_layer, + transfer_kv_all_layer_mla, + transfer_kv_per_layer, + transfer_kv_per_layer_mla, +) +from sgl_kernel.mamba import causal_conv1d_fwd, causal_conv1d_update +from sgl_kernel.marlin import ( + awq_marlin_moe_repack, + awq_marlin_repack, + gptq_marlin_repack, +) +from sgl_kernel.memory import set_kv_buffer_kernel, weak_ref_tensor +from sgl_kernel.moe import ( + apply_shuffle_mul_sum, + cutlass_fp4_group_mm, + fp8_blockwise_scaled_grouped_mm, + fused_qk_norm_rope, + kimi_k2_moe_fused_gate, + moe_align_block_size, + moe_fused_gate, + moe_sum, + moe_sum_reduce, + prepare_moe_input, + topk_sigmoid, + topk_softmax, +) +from sgl_kernel.quantization import ( + ggml_dequantize, + ggml_moe_a8, + ggml_moe_a8_vec, + ggml_moe_get_block_size, + ggml_mul_mat_a8, + ggml_mul_mat_vec_a8, +) +from sgl_kernel.sampling import ( + min_p_sampling_from_probs, + top_k_mask_logits, + top_k_renorm_prob, + top_k_top_p_sampling_from_logits, + top_k_top_p_sampling_from_probs, + top_p_renorm_prob, + top_p_sampling_from_probs, +) +from sgl_kernel.speculative import ( + build_tree_kernel_efficient, + reconstruct_indices_from_tree_mask, + segment_packbits, + tree_speculative_sampling_target_only, + verify_tree_greedy, +) +from sgl_kernel.top_k import ( + fast_topk, + fast_topk_transform_fused, + fast_topk_transform_ragged_fused, + fast_topk_v2, +) +from sgl_kernel.version import __version__ + +if torch.version.hip is not None: + from sgl_kernel.elementwise import gelu_quick + + +def create_greenctx_stream_by_value(*args, **kwargs): + from sgl_kernel.spatial import create_greenctx_stream_by_value as _impl + + return _impl(*args, **kwargs) + + +def get_sm_available(*args, **kwargs): + from sgl_kernel.spatial import get_sm_available as _impl + + return _impl(*args, **kwargs) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/_fa4_interface.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/_fa4_interface.py new file mode 100644 index 00000000..5411cfe1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/_fa4_interface.py @@ -0,0 +1,562 @@ +# Adapted from https://github.com/Dao-AILab/flash-attention/blob/54d8aa6751fc9d5f0357854079261913d5df1f9d/flash_attn/cute/interface.py + +# Copyright (c) 2025, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao. +# [2025-10-14] Version in Cute-DSL, for Hopper and Blackwell. You'd need to install nvidia-cutlass-dsl==4.2.1. + + +import copy +import gc +import logging +import math +import os +from typing import Callable, Optional, Tuple + +logger = logging.getLogger(__name__) + + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +import torch +from cutlass.cute.runtime import from_dlpack +from flash_attn.cute import utils +from flash_attn.cute.flash_fwd import FlashAttentionForwardSm90 +from flash_attn.cute.flash_fwd_sm100 import FlashAttentionForwardSm100 + + +def maybe_contiguous(x): + return x.contiguous() if x is not None and x.stride(-1) != 1 else x + + +torch2cute_dtype_map = { + torch.float16: cutlass.Float16, + torch.bfloat16: cutlass.BFloat16, + torch.float32: cutlass.Float32, +} + + +def _flash_attn_fwd( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + cu_seqlens_q: Optional[torch.Tensor] = None, + cu_seqlens_k: Optional[torch.Tensor] = None, + seqused_q: Optional[torch.Tensor] = None, + seqused_k: Optional[torch.Tensor] = None, + page_table: Optional[torch.Tensor] = None, + softmax_scale: Optional[float] = None, + causal: bool = False, + softcap: Optional[float] = None, + window_size_left: Optional[int] = None, + window_size_right: Optional[int] = None, + learnable_sink: Optional[torch.Tensor] = None, + # m_block_size: int = 128, + # n_block_size: int = 64, + # num_threads: int = 128, + m_block_size: int = 128, + n_block_size: int = 128, + num_threads: int = 384, + pack_gqa: Optional[bool] = None, + _compute_capability: Optional[int] = None, + score_mod: Callable | None = None, + return_lse: bool = False, + out: Optional[torch.Tensor] = None, + lse: Optional[torch.Tensor] = None, + buffers: Optional[list[torch.Tensor]] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + q, k, v = [maybe_contiguous(t) for t in (q, k, v)] + num_head, head_dim = q.shape[-2:] + if cu_seqlens_q is None: + batch_size, seqlen_q = q.shape[:2] + total_q = batch_size * seqlen_q + else: + batch_size = cu_seqlens_q.shape[0] - 1 + seqlen_q = None + total_q = q.shape[0] + if page_table is not None: + assert cu_seqlens_k is None, "page_table is not supported with cu_seqlens_k" + assert page_table.dtype == torch.int32, "page_table must be int32" + assert ( + page_table.stride(-1) == 1 + ), "page_table must be contiguous in the last dimension" + max_num_pages_per_seq = page_table.shape[1] + assert page_table.shape == (batch_size, max_num_pages_per_seq) + num_pages, page_size = k.shape[:2] + seqlen_k = num_pages * page_size + else: + num_pages, page_size = None, None + seqlen_k = k.shape[-3] + num_head_kv = k.shape[-2] + head_dim_v = v.shape[-1] + if cu_seqlens_k is None: + if page_table is None: + assert k.shape == (batch_size, seqlen_k, num_head_kv, head_dim) + assert v.shape == (batch_size, seqlen_k, num_head_kv, head_dim_v) + else: + assert k.shape == (num_pages, page_size, num_head_kv, head_dim) + assert v.shape == (num_pages, page_size, num_head_kv, head_dim_v) + else: + assert k.shape == (seqlen_k, num_head_kv, head_dim) + assert v.shape == (seqlen_k, num_head_kv, head_dim_v) + assert cu_seqlens_k.shape == ( + batch_size + 1, + ), "cu_seqlens_k must have shape (batch_size + 1,)" + if cu_seqlens_q is not None: + assert cu_seqlens_q.shape == ( + batch_size + 1, + ), "cu_seqlens_q must have shape (batch_size + 1,)" + assert seqused_q is None or seqused_q.shape == ( + batch_size, + ), "seqused_q must have shape (batch_size,)" + assert seqused_k is None or seqused_k.shape == ( + batch_size, + ), "seqused_k must have shape (batch_size,)" + assert q.dtype in [ + torch.float16, + torch.bfloat16, + ], "inputs must be float16 or bfloat16" + assert q.dtype == k.dtype == v.dtype, "inputs must have the same dtype" + for t in [cu_seqlens_q, cu_seqlens_k, seqused_q, seqused_k]: + if t is not None: + assert ( + t.dtype == torch.int32 + ), "cu_seqlens_q, cu_seqlens_k, seqused_q, seqused_k must be int32" + assert ( + t.stride(0) == 1 + ), "cu_seqlens_q, cu_seqlens_k, seqused_q, seqused_k must be contiguous" + if learnable_sink is not None: + assert learnable_sink.shape == (num_head,) + assert learnable_sink.dtype == torch.bfloat16, "learnable_sink must be bfloat16" + assert all( + t is None or t.is_cuda + for t in ( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + seqused_q, + seqused_k, + page_table, + learnable_sink, + ) + ), "inputs must be on CUDA device" + assert num_head % num_head_kv == 0, "num_head must be divisible by num_head_kv" + assert head_dim <= 256, "head_dim must be less than or equal to 256" + alignment = 16 // q.element_size() + assert head_dim % alignment == 0, f"head_dim must be divisible by {alignment}" + assert head_dim_v % alignment == 0, f"head_dim_v must be divisible by {alignment}" + if softmax_scale is None: + softmax_scale = 1.0 / math.sqrt(head_dim) + if softcap == 0.0: + softcap = None + qhead_per_kvhead = num_head // num_head_kv + if pack_gqa is None: + pack_gqa = qhead_per_kvhead > 1 + + out_torch_dtype = q.dtype + device = q.device + q_batch_seqlen_shape = ( + (batch_size, seqlen_q) if cu_seqlens_q is None else (total_q,) + ) + lse_shape = ( + (batch_size, num_head, seqlen_q) + if cu_seqlens_q is None + else (num_head, total_q) + ) + requires_grad = q.requires_grad or k.requires_grad or v.requires_grad + + if out is None: + out = torch.empty( + *q_batch_seqlen_shape, + num_head, + head_dim_v, + dtype=out_torch_dtype, + device=device, + ) + else: + expected_out_shape = (*q_batch_seqlen_shape, num_head, head_dim_v) + assert ( + out.shape == expected_out_shape + ), f"out tensor shape {out.shape} does not match expected shape {expected_out_shape}" + assert ( + out.dtype == out_torch_dtype + ), f"out tensor dtype {out.dtype} does not match expected dtype {out_torch_dtype}" + assert ( + out.device == device + ), f"out tensor device {out.device} does not match input device {device}" + assert out.is_cuda, "out tensor must be on CUDA device" + + if lse is None: + lse = ( + torch.empty(lse_shape, dtype=torch.float32, device=device) + if requires_grad or return_lse + else None + ) + elif lse is not None: + assert ( + lse.shape == lse_shape + ), f"lse tensor shape {lse.shape} does not match expected shape {lse_shape}" + assert ( + lse.dtype == torch.float32 + ), f"lse tensor dtype {lse.dtype} does not match expected dtype torch.float32" + assert ( + lse.device == device + ), f"lse tensor device {lse.device} does not match input device {device}" + assert lse.is_cuda, "lse tensor must be on CUDA device" + + dtype = torch2cute_dtype_map[q.dtype] + q_tensor, k_tensor, v_tensor, o_tensor = [ + from_dlpack(t.detach(), assumed_align=16).mark_layout_dynamic( + leading_dim=t.ndim - 1 + ) + for t in (q, k, v, out) + ] + lse_tensor = ( + from_dlpack(lse.detach(), assumed_align=4).mark_layout_dynamic( + leading_dim=lse.ndim - 1 + ) + if lse is not None + else None + ) + ( + cu_seqlens_q_tensor, + cu_seqlens_k_tensor, + seqused_q_tensor, + seqused_k_tensor, + learnable_sink_tensor, + ) = [ + ( + from_dlpack(t.detach(), assumed_align=4).mark_layout_dynamic(leading_dim=0) + if t is not None + else None + ) + for t in (cu_seqlens_q, cu_seqlens_k, seqused_q, seqused_k, learnable_sink) + ] + page_table_tensor = ( + from_dlpack(page_table.detach(), assumed_align=4).mark_layout_dynamic( + leading_dim=1 + ) + if page_table is not None + else None + ) + if causal: + window_size_right = 0 + local = window_size_left is not None or window_size_right is not None + if window_size_left is not None or window_size_right is not None: + if window_size_left is None and window_size_right == 0: + causal, local = True, False + else: + causal, local = False, True + compute_capability = ( + torch.cuda.get_device_capability()[0] + if _compute_capability is None + else _compute_capability + ) + assert compute_capability in [ + 9, + 10, + ], "Unsupported compute capability. Supported: 9.x, 10.x" + current_stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream) + + if compute_capability == 9: # TODO: tune block size according to hdim + # Perf heuristic from upstream: hdim=128, noncausal, non-local benefits from larger n_block + if head_dim == head_dim_v == 128 and not causal and not local: + n_block_size = 192 + if compute_capability == 10: + # TODO: fix the varlen case + if ( + pack_gqa + and (128 % qhead_per_kvhead != 0) + or (cu_seqlens_q is not None or seqused_q is not None) + ): + pack_gqa = False + + if softcap is not None: + assert score_mod is None, "softcap and score_mod cannot be used together" + score_mod = utils.create_softcap_scoremod(softcap) + + if score_mod is not None: + is_varlen = ( + cu_seqlens_q is not None + or cu_seqlens_k is not None + or seqused_q is not None + or seqused_k is not None + ) + if is_varlen: + raise NotImplementedError( + "score_mod with buffers is not yet supported for varlen sequences. This will be fixed in a future PR." + ) + + cute_buffers = None + if buffers is not None: + cute_buffers = [from_dlpack(buf) for buf in buffers] + + compile_key = ( + dtype, + head_dim, + head_dim_v, + qhead_per_kvhead, + causal, + utils.hash_callable(score_mod) if score_mod is not None else None, + buffers is not None, + lse is None, + cu_seqlens_q is None, + cu_seqlens_k is None, + seqused_q is None, + seqused_k is None, + page_table is not None, + window_size_left is not None, + window_size_right is not None, + learnable_sink is not None, + m_block_size, + n_block_size, + num_threads, + pack_gqa, + compute_capability, + ) + if compile_key not in _flash_attn_fwd.compile_cache: + if compute_capability == 9: + assert page_table is None, "paged KV not supported on SM 9.0" + # fa_fwd = FlashAttentionForwardSm80( + fa_fwd = FlashAttentionForwardSm90( + dtype, + head_dim, + head_dim_v, + qhead_per_kvhead, + is_causal=causal, + is_local=local, + pack_gqa=pack_gqa, + tile_m=m_block_size, + tile_n=n_block_size, + # num_stages=1, + num_stages=2, + num_threads=num_threads, + Q_in_regs=False, + score_mod=score_mod, + has_buffers=buffers is not None, + ) + elif compute_capability == 10: + assert page_size in [ + None, + 128, + ], "Only page_size=128 is supported for paged KV on SM 10.0" + fa_fwd = FlashAttentionForwardSm100( + head_dim, + head_dim_v, + qhead_per_kvhead=qhead_per_kvhead, + is_causal=causal, + is_local=local, + pack_gqa=pack_gqa, + is_persistent=not causal + and not local + and cu_seqlens_q is None + and seqused_q is None, + score_mod=score_mod, + has_buffers=buffers is not None, + ) + else: + raise ValueError( + f"Unsupported compute capability: {compute_capability}. Supported: 9.x, 10.x" + ) + # TODO: check @can_implement + # TODO caching for buffers; cute_buffers + _flash_attn_fwd.compile_cache[compile_key] = cute.compile( + fa_fwd, + q_tensor, + k_tensor, + v_tensor, + o_tensor, + lse_tensor, + softmax_scale, + current_stream, + cu_seqlens_q_tensor, + cu_seqlens_k_tensor, + seqused_q_tensor, + seqused_k_tensor, + page_table_tensor, + window_size_left, + window_size_right, + learnable_sink_tensor, + cute_buffers, + ) + _flash_attn_fwd.compile_cache[compile_key]( + q_tensor, + k_tensor, + v_tensor, + o_tensor, + lse_tensor, + softmax_scale, + current_stream, + cu_seqlens_q_tensor, + cu_seqlens_k_tensor, + seqused_q_tensor, + seqused_k_tensor, + page_table_tensor, + window_size_left, + window_size_right, + learnable_sink_tensor, + cute_buffers, + ) + return out, lse + + +_flash_attn_fwd.compile_cache = {} + + +def warmup_flash_attn(f): + """ + Decorator for flash_attn_varlen_func: + - On first call, run several warmup passes with different flag combinations: + * return_softmax_lse in {False, True} + * global noncausal (window_size=(None,None)) + * causal (window_size=(None,0)) + * local sliding window (window_size=(64,64)) + * optionally pack_gqa=True if qheads > kvheads and allowed + - No score_mod / softcap (not supported for varlen yet) + - Executes sequentially to minimize peak GPU mem + - Does not modify user tensors (clones) + """ + disable_warmup = os.getenv("SGLANG_DISABLE_FA4_WARMUP", "").lower() in ( + "1", + "true", + "yes", + "on", + ) + if disable_warmup: + return f + + done = False + + def _clone_args(args, kwargs): + """Clone tensor arguments to avoid sharing storage; deepcopy for others.""" + + def maybe_clone(x): + if isinstance(x, torch.Tensor): + return x.detach().clone() # detach to avoid autograd edges + return copy.deepcopy(x) + + return tuple(maybe_clone(a) for a in args), { + k: maybe_clone(v) for k, v in kwargs.items() + } + + def _infer_heads(args, kwargs): + """Infer q and kv head counts from arguments.""" + # Expect signature: (q, k, v, cu_seqlens_q, cu_seqlens_k, ...) + q = args[0] if len(args) > 0 else kwargs.get("q") + k = args[1] if len(args) > 1 else kwargs.get("k") + try: + qh = int(q.shape[-2]) + kvh = int(k.shape[-2]) + return qh, kvh + except Exception: + return None, None + + def _run_warmups(args, kwargs): + """Run warmup calls sequentially and release memory after each.""" + base_args, base_kwargs = _clone_args(args, kwargs) + + qh, kvh = _infer_heads(base_args, base_kwargs) + can_pack_gqa = ( + qh is not None and kvh is not None and qh % kvh == 0 and qh // kvh > 1 + ) + has_page_table = ( + "page_table" in base_kwargs and base_kwargs["page_table"] is not None + ) + + # Window presets covering global, causal, and local + window_presets = [ + (None, None), # global noncausal + (None, 0), # causal + (64, 64), # local sliding window + ] + + lse_flags = [False, True] + + # Base combo list + combos = [] + for ws in window_presets: + for return_lse_flag in lse_flags: + combos.append(dict(window_size=ws, return_softmax_lse=return_lse_flag)) + + # Optionally add a pack_gqa=True variant (FA4 may disable it internally for some varlen shapes/SMs) + if can_pack_gqa: + for ws in window_presets: + combos.append( + dict(window_size=ws, return_softmax_lse=False, pack_gqa=True) + ) + + # If page_table is present, warm one combo with it (page_table in compile key for SM100) + if has_page_table: + combos.append(dict(window_size=(None, None), return_softmax_lse=False)) + + # Run sequentially + for combo in combos: + wa, wk = _clone_args(base_args, base_kwargs) + # Keep user-provided softcap/score_mod OUT (varlen+score_mod unsupported) + wk.pop("score_mod", None) + if "softcap" in wk and wk["softcap"]: + wk["softcap"] = 0.0 + # Apply combo + wk.update(combo) + with torch.cuda.stream(torch.cuda.current_stream()): + try: + f(*wa, **wk) + except Exception as e: + # Some combos can be invalid for specific head dims / arch. Ignore and continue. + logger.debug("Warmup combo skipped: %s", e) + del wa, wk + torch.cuda.empty_cache() + gc.collect() + + def wrapper(*args, **kwargs): + nonlocal done + if not done: + logger.info( + "Running FA4 warmup (global/causal/local, LSE on/off, optional GQA pack)..." + ) + _run_warmups(args, kwargs) + done = True + return f(*args, **kwargs) + + return wrapper + + +@warmup_flash_attn +def flash_attn_varlen_func( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + cu_seqlens_q: Optional[torch.Tensor] = None, + cu_seqlens_k: Optional[torch.Tensor] = None, + seqused_q: Optional[torch.Tensor] = None, + seqused_k: Optional[torch.Tensor] = None, + page_table: Optional[torch.Tensor] = None, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[Optional[int], Optional[int]] = (None, None), + learnable_sink: Optional[torch.Tensor] = None, + softcap: float = 0.0, + pack_gqa: Optional[bool] = None, + return_softmax_lse: Optional[bool] = False, +) -> Tuple[torch.Tensor, torch.Tensor]: + out, lse = _flash_attn_fwd( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + seqused_q, + seqused_k, + page_table=page_table, + softmax_scale=softmax_scale, + causal=causal, + window_size_left=window_size[0], + window_size_right=window_size[1], + learnable_sink=learnable_sink, + softcap=softcap, + pack_gqa=pack_gqa, + return_lse=return_softmax_lse, + ) + + return (out, lse) if return_softmax_lse else out diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/allreduce.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/allreduce.py new file mode 100644 index 00000000..544fc1d7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/allreduce.py @@ -0,0 +1,173 @@ +from typing import List, Optional, Tuple + +import torch + +if torch.version.hip is not None: + # ROCM custom allreduce + def init_custom_ar( + meta: torch.Tensor, + rank_data: torch.Tensor, + handles: List[str], + offsets: List[int], + rank: int, + full_nvlink: bool, + ) -> int: + return torch.ops.sgl_kernel.init_custom_ar.default( + meta, rank_data, handles, offsets, rank, full_nvlink + ) + + def all_reduce_reg(fa: int, inp: torch.Tensor, out: torch.Tensor) -> None: + torch.ops.sgl_kernel.all_reduce_reg.default(fa, inp, out) + + def all_reduce_unreg( + fa: int, inp: torch.Tensor, reg_buffer: torch.Tensor, out: torch.Tensor + ) -> None: + torch.ops.sgl_kernel.all_reduce_unreg.default(fa, inp, reg_buffer, out) + + def dispose(fa: int) -> None: + torch.ops.sgl_kernel.dispose.default(fa) + + def meta_size() -> int: + return torch.ops.sgl_kernel.meta_size.default() + + def register_buffer( + fa: int, t: torch.Tensor, handles: List[str], offsets: List[int] + ) -> None: + return torch.ops.sgl_kernel.register_buffer.default(fa, t, handles, offsets) + + def get_graph_buffer_ipc_meta(fa: int) -> Tuple[torch.Tensor, List[int]]: + return torch.ops.sgl_kernel.get_graph_buffer_ipc_meta.default(fa) + + def register_graph_buffers( + fa: int, handles: List[str], offsets: List[List[int]] + ) -> None: + torch.ops.sgl_kernel.register_graph_buffers.default(fa, handles, offsets) + + def allocate_meta_buffer(size: int) -> torch.Tensor: + return torch.ops.sgl_kernel.allocate_meta_buffer.default(size) + + def get_meta_buffer_ipc_handle(inp: torch.Tensor) -> torch.Tensor: + return torch.ops.sgl_kernel.get_meta_buffer_ipc_handle.default(inp) + + # ROCM quick allreduce + def init_custom_qr( + rank: int, world_size: int, qr_max_size: Optional[int] = None + ) -> int: + return torch.ops.sgl_kernel.init_custom_qr.default( + world_size, rank, qr_max_size + ) + + def qr_get_handle(fa: int) -> torch.Tensor: + return torch.ops.sgl_kernel.qr_get_handle.default(fa) + + def qr_open_handles(fa: int, handles: list[torch.Tensor]) -> None: + torch.ops.sgl_kernel.qr_open_handles.default(fa, handles) + + def qr_all_reduce( + fa: int, + profile: int, + inp: torch.Tensor, + out: torch.Tensor, + cast_bf162half: bool, + ) -> None: + torch.ops.sgl_kernel.qr_all_reduce.default( + fa, profile, inp, out, cast_bf162half + ) + + def qr_destroy(fa: int) -> None: + torch.ops.sgl_kernel.qr_destroy.default(fa) + + def qr_max_size() -> int: + return torch.ops.sgl_kernel.qr_max_size.default() + + # mscclpp + def mscclpp_generate_unique_id() -> bytes: + raise NotImplementedError() + + def mscclpp_init_context( + unique_id: bytes, + rank: int, + world_size: int, + scratch: torch.Tensor, + put_buffer: torch.Tensor, + nranks_per_node: int, + rank_to_node: List[int], + rank_to_ib: List[int], + context_selection: int, + ) -> int: + raise NotImplementedError() + + def mscclpp_allreduce( + context: int, inp: torch.Tensor, out: torch.Tensor, nthreads: int, nblocks: int + ) -> None: + raise NotImplementedError() + +else: + + def init_custom_ar( + ipc_tensors: List[int], rank_data: torch.Tensor, rank: int, full_nvlink: bool + ) -> int: + return torch.ops.sgl_kernel.init_custom_ar.default( + ipc_tensors, rank_data, rank, full_nvlink + ) + + def dispose(fa: int) -> None: + torch.ops.sgl_kernel.dispose.default(fa) + + def all_reduce( + fa: int, + inp: torch.Tensor, + out: torch.Tensor, + reg_buffer: int, + reg_buffer_sz_bytes: int, + ) -> None: + torch.ops.sgl_kernel.all_reduce.default( + fa, inp, out, reg_buffer, reg_buffer_sz_bytes + ) + + def get_graph_buffer_ipc_meta(fa) -> Tuple[List[int], List[int]]: + return torch.ops.sgl_kernel.get_graph_buffer_ipc_meta.default(fa) + + def register_buffer(fa: int, fake_ipc_ptrs: List[int]) -> None: + return torch.ops.sgl_kernel.register_buffer.default(fa, fake_ipc_ptrs) + + def register_graph_buffers( + fa: int, handles: List[List[int]], offsets: List[List[int]] + ) -> None: + torch.ops.sgl_kernel.register_graph_buffers.default(fa, handles, offsets) + + def meta_size() -> int: + return torch.ops.sgl_kernel.meta_size.default() + + def mscclpp_generate_unique_id() -> torch.Tensor: + return torch.ops.sgl_kernel.mscclpp_generate_unique_id.default() + + def mscclpp_init_context( + unique_id: torch.Tensor, + rank: int, + world_size: int, + scratch: torch.Tensor, + put_buffer: torch.Tensor, + nranks_per_node: int, + rank_to_node: List[int], + rank_to_ib: List[int], + context_selection: int, + ) -> int: + return torch.ops.sgl_kernel.mscclpp_init_context.default( + unique_id, + rank, + world_size, + scratch, + put_buffer, + nranks_per_node, + rank_to_node, + rank_to_ib, + context_selection, + ) + + def mscclpp_allreduce( + context: int, inp: torch.Tensor, out: torch.Tensor, nthreads: int, nblocks: int + ) -> None: + torch.ops.sgl_kernel.mscclpp_allreduce.default( + context, inp, out, nthreads, nblocks + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/attention.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/attention.py new file mode 100644 index 00000000..44dd6111 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/attention.py @@ -0,0 +1,132 @@ +from typing import Optional, Tuple + +import torch + + +def merge_state( + v_a: torch.Tensor, + s_a: torch.Tensor, + v_b: torch.Tensor, + s_b: torch.Tensor, + v_merged: Optional[torch.Tensor] = None, + s_merged: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + s_a = s_a.to(torch.float32) + s_b = s_b.to(torch.float32) + # Avoid creating new tensors if they are already provided + if v_merged is None: + v_merged = torch.empty_like(v_a) + if s_merged is None: + s_merged = torch.empty_like(s_a) + torch.ops.sgl_kernel.merge_state.default(v_a, s_a, v_b, s_b, v_merged, s_merged) + return v_merged, s_merged + + +def merge_state_v2( + v_a: torch.Tensor, + s_a: torch.Tensor, + v_b: torch.Tensor, + s_b: torch.Tensor, + v_merged: Optional[torch.Tensor] = None, + s_merged: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + s_a = s_a.to(torch.float32) + s_b = s_b.to(torch.float32) + # TODO(DefTruth): Currently, the custom merge_attn_states kernel + # does not support the FP8 data type and non - CUDA devices. + # It may be necessary to fall back to using the Triton kernel. + + # Avoid creating new tensors if they are already provided + if v_merged is None: + v_merged = torch.empty_like(v_a) + if s_merged is None: + s_merged = torch.empty_like(s_a) + torch.ops.sgl_kernel.merge_state_v2.default(v_a, s_a, v_b, s_b, v_merged, s_merged) + return v_merged, s_merged + + +def cutlass_mla_decode( + q_nope: torch.Tensor, + q_pe: torch.Tensor, + kv_c_and_k_pe_cache: torch.Tensor, + seq_lens: torch.Tensor, + page_table: torch.Tensor, + workspace: torch.Tensor, + sm_scale: float, + num_kv_splits: int = 1, # Set to 1 to avoid cuda_graph issue by default. +) -> torch.Tensor: + assert q_nope.ndim == 3, f"q_nope must be a 3D tensor, but got {q_nope.ndim}" + assert q_pe.ndim == 3, f"q_pe must be a 3D tensor, but got {q_pe.ndim}" + assert ( + kv_c_and_k_pe_cache.ndim == 3 + ), f"kv_c_and_k_pe_cache must be a 3D tensor, but got {kv_c_and_k_pe_cache.ndim}" + + B_q, H, D_q_nope = q_nope.shape + B_q_2, H_2, D_q_pe = q_pe.shape + assert (B_q == B_q_2) and (H == H_2) + + _, PAGE_SIZE, D_ckv = kv_c_and_k_pe_cache.shape + + D_latent = 512 + D_rope = 64 + assert D_q_nope == D_latent + assert D_q_pe == D_rope + assert D_ckv == D_latent + D_rope + + MAX_HEADS = 128 + assert H <= MAX_HEADS, f"H must be <= {MAX_HEADS}, but got {H}" + if H < MAX_HEADS: + q_nope_padded = q_nope.new_empty((B_q, MAX_HEADS, D_q_nope)) + q_nope_padded[:, :H] = q_nope + q_nope = q_nope_padded + + q_pe_padded = q_pe.new_empty((B_q, MAX_HEADS, D_q_pe)) + q_pe_padded[:, :H] = q_pe + q_pe = q_pe_padded + + assert len(page_table.shape) == 2 + B_block_table, block_num = page_table.shape + assert B_block_table == B_q + assert block_num > 0, f"block num must be greater than 0, got {block_num}" + assert block_num % (128 / PAGE_SIZE) == 0 + + # TODO(kaixih@nvidia): support fp8 + assert q_nope.dtype in ( + torch.float16, + torch.bfloat16, + ), f"q_nope.dtype needs to be fp16 or bf16 but got {q_nope.dtype}." + assert q_nope.dtype == q_pe.dtype == kv_c_and_k_pe_cache.dtype + assert ( + seq_lens.dtype == torch.int32 + ), f"seq_lens.dtype needs to be int32 but got {seq_lens.dtype}." + assert ( + page_table.dtype == torch.int32 + ), f"page_table.dtype needs to be int32 but got {page_table.dtype}." + + out = q_nope.new_empty((B_q, MAX_HEADS, D_latent)) + + torch.ops.sgl_kernel.cutlass_mla_decode.default( + out, + q_nope, + q_pe, + kv_c_and_k_pe_cache, + seq_lens, + page_table, + workspace, + sm_scale, + num_kv_splits, + ) + return out[:, :H].contiguous() + + +def cutlass_mla_get_workspace_size( + max_seq_len: int, + num_batches: int, + sm_count: int = 0, + num_kv_splits: int = 1, # Set to 1 to avoid cuda_graph issue by default. +) -> int: + assert max_seq_len > 0, f"max_seq_len must be greater than 0, got {max_seq_len}" + assert num_batches > 0, f"num_batches must be greater than 0, got {num_batches}" + return torch.ops.sgl_kernel.cutlass_mla_get_workspace_size.default( + max_seq_len, num_batches, sm_count, num_kv_splits + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/cutlass_moe.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/cutlass_moe.py new file mode 100644 index 00000000..52256bda --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/cutlass_moe.py @@ -0,0 +1,112 @@ +import torch + + +def get_cutlass_w4a8_moe_mm_data( + topk_ids: torch.Tensor, + expert_offsets: torch.Tensor, + problem_sizes1: torch.Tensor, + problem_sizes2: torch.Tensor, + input_permutation: torch.Tensor, + output_permutation: torch.Tensor, + num_experts: int, + n: int, + k: int, +): + """ + Prepare data necessary to perform CUTLASS grouped matrix multiplications + used in CUTLASS-based fused MoE. + + The function takes in topk_ids (token-expert mapping) and uses it to + compute: + - expert_offsets: Indices that mark at which token index each expert begins + its computation after the input is sorted with + input_permutation. The number of tokens computed with + expert E is expert_offsets[E + 1] - expert_offsets[E] + - problem_sizes1, problem_sizes2: MxNxK sizes of each expert's + multiplication in two grouped MMs used in + the fused MoE operation. + - input_permutation: Permutation that must be used to shuffle the input + before executing the MMs. + - output_permutation: Permutation that must be used to shuffle the output + after executing the MMs. + """ + torch.ops.sgl_kernel.get_cutlass_w4a8_moe_mm_data.default( + topk_ids, + expert_offsets, + problem_sizes1, + problem_sizes2, + input_permutation, + output_permutation, + num_experts, + n, + k, + ) + + +def cutlass_w4a8_moe_mm( + d: torch.Tensor, + a: torch.Tensor, + b: torch.Tensor, + a_scales: torch.Tensor, + b_scales: torch.Tensor, + experts_offsets: torch.tensor, + problem_sizes: torch.tensor, + a_strides: torch.tensor, + b_strides: torch.tensor, + d_strides: torch.tensor, + s_strides: torch.tensor, + chunk_size: int = 128, + topk: int = 8, +): + """ + Perform grouped matrix multiplication between int4 weights and fp8 activations. + + This function executes multiple GEMM operations in parallel, which is useful for + scenarios like Mixture of Experts (MoE) where different inputs go through different + experts. The implementation leverages NVIDIA Hopper architecture features for + optimal performance with quantized weights. + + Args: + d: Output matrices of shape [total_m, total_n] + a: Activation matrices in FP8 (float_e4m3_t) format + Each tensor should be of shape [total_m, K] in row-major layout + b: Weight matrices in packed int4 format + Each tensor should be of shape [E, N, K//2] in column-major layout + where each byte contains two 4-bit integers + a_scales: Scale factors for the inputs + b_scales: Scale factors for the quantized weights + Each tensor should be of shape [E, K//512, N*8] + experts_offsets: Tensor containing expert offsets for determining group boundaries + problem_sizes: with shape [num_experts, 3] (M, N, K for each group) (int32) + a_strides: Strides information for A matrices + b_strides: Strides information for B matrices + d_strides: Strides information for D matrices + s_strides: Strides information for b_scales matrices + chunk_size: Number of elements each scale value applies to (K//512), default to 128 + + Requirements: + - All tensors must be on a CUDA device + - Requires an NVIDIA Hopper GPU (H100) + - A tensors must be in float8_e4m3fn format + - B tensors must contain packed int4 values (stored as int8) + + Note: + The function computes: D = (A * (B * scales)) + for each group of tensors in parallel + """ + + torch.ops.sgl_kernel.cutlass_w4a8_moe_mm.default( + d, + a, + b, + a_scales, + b_scales, + experts_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size, + topk, + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/elementwise.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/elementwise.py new file mode 100644 index 00000000..5684800b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/elementwise.py @@ -0,0 +1,393 @@ +from dataclasses import dataclass +from typing import List, Optional + +import torch +from sgl_kernel.utils import is_arch_support_pdl + + +# These implementations extensively draw from and build upon the FlashInfer project https://github.com/flashinfer-ai/flashinfer +# Kudos to @yzh119 +def rmsnorm( + input: torch.Tensor, + weight: torch.Tensor, + eps: float = 1e-6, + out: Optional[torch.Tensor] = None, + enable_pdl: Optional[bool] = None, +) -> torch.Tensor: + r"""Root mean square normalization. + + ``out[i] = (input[i] / RMS(input)) * weight[i]`` + + Parameters + ---------- + input: torch.Tensor + Input tensor, shape (batch_size, hidden_size). + weight: torch.Tensor + Weight tensor, shape (hidden_size,). + eps: float + Epsilon for numerical stability. + out: Optional[torch.Tensor] + The output tensor, if specified, the kernel will update this tensor inplace. + enable_pdl: Optional[bool] + Whether to enable `programmatic dependent launch + `_ + If None, will be automatically enabled on Hopper architecture. + + Returns + ------- + output: torch.Tensor + Normalized tensor, shape (batch_size, hidden_size). + """ + if out is None: + out = torch.empty_like(input) + if enable_pdl is None: + enable_pdl = is_arch_support_pdl() + torch.ops.sgl_kernel.rmsnorm.default(out, input, weight, eps, enable_pdl) + return out + + +def fused_add_rmsnorm( + input: torch.Tensor, + residual: torch.Tensor, + weight: torch.Tensor, + eps: float = 1e-6, + enable_pdl: Optional[bool] = None, +) -> None: + r"""Fused add root mean square normalization. + + Step 1: + ``residual[i] += input[i]`` + + Step 2: + ``input[i] = (residual[i] / RMS(residual)) * weight[i]`` + + Parameters + ---------- + input: torch.Tensor + Input tensor, shape (batch_size, hidden_size). + residual: torch.Tensor + Residual tensor, shape (batch_size, hidden_size). + weight: torch.Tensor + Weight tensor, shape (hidden_size,). + eps: float + Epsilon for numerical stability. + enable_pdl: Optional[bool] + Whether to enable `programmatic dependent launch + `_ + If None, will be automatically enabled on Hopper architecture. + """ + if enable_pdl is None: + enable_pdl = is_arch_support_pdl() + torch.ops.sgl_kernel.fused_add_rmsnorm.default( + input, residual, weight, eps, enable_pdl + ) + + +def gemma_rmsnorm( + input: torch.Tensor, + weight: torch.Tensor, + eps: float = 1e-6, + out: Optional[torch.Tensor] = None, + enable_pdl: Optional[bool] = None, +) -> torch.Tensor: + r"""Gemma-style root mean square normalization. + + ``out[i] = (input[i] / RMS(input)) * (weight[i] + 1)`` + + Parameters + ---------- + input: torch.Tensor + Input tensor, shape (batch_size, hidden_size). + weight: torch.Tensor + Weight tensor, shape (hidden_size,). + eps: float + Epsilon for numerical stability. + out: Optional[torch.Tensor] + The output tensor, if specified, the kernel will update this tensor inplace. + enable_pdl: Optional[bool] + Whether to enable `programmatic dependent launch + `_ + If None, will be automatically enabled on Hopper architecture. + + Returns + ------- + output: torch.Tensor + Gemma Normalized tensor, shape (batch_size, hidden_size). + """ + if out is None: + out = torch.empty_like(input) + if enable_pdl is None: + enable_pdl = is_arch_support_pdl() + torch.ops.sgl_kernel.gemma_rmsnorm.default(out, input, weight, eps, enable_pdl) + return out + + +def gemma_fused_add_rmsnorm( + input: torch.Tensor, + residual: torch.Tensor, + weight: torch.Tensor, + eps: float = 1e-6, + enable_pdl: Optional[bool] = None, +) -> None: + r"""Gemma-style fused add root mean square normalization. + + Step 1: + ``residual[i] += input[i]`` + + Step 2: + ``input[i] = (residual[i] / RMS(residual)) * (weight + 1)`` + + Parameters + ---------- + input: torch.Tensor + Input tensor, shape (batch_size, hidden_size). + residual: torch.Tensor + Residual tensor, shape (batch_size, hidden_size). + weight: torch.Tensor + Weight tensor, shape (hidden_size,). + eps: float + Epsilon for numerical stability. + enable_pdl: Optional[bool] + Whether to enable `programmatic dependent launch + `_ + If None, will be automatically enabled on Hopper architecture. + """ + if enable_pdl is None: + enable_pdl = is_arch_support_pdl() + torch.ops.sgl_kernel.gemma_fused_add_rmsnorm.default( + input, residual, weight, eps, enable_pdl + ) + + +def _check_shape(input: torch.Tensor, output: torch.Tensor) -> None: + assert input.ndim == output.ndim, f"{input.ndim} != {output.ndim}" + assert ( + input.shape[:-1] == output.shape[:-1] + ), f"{input.shape[:-1]} != {output.shape[:-1]}" + assert ( + input.shape[-1] == 2 * output.shape[-1] + ), f"{input.shape[-1]} != {2 * output.shape[-1]}" + + +def silu_and_mul(input: torch.Tensor, out: torch.Tensor = None) -> torch.Tensor: + if input.shape[-1] * input.dtype.itemsize % 16 != 0: + raise ValueError("The pointers must be multiple of 16 bytes.") + if out is not None: + _check_shape(input, out) + else: + out = torch.empty( + input.shape[:-1] + (input.shape[-1] // 2,), + device=input.device, + dtype=input.dtype, + ) + torch.ops.sgl_kernel.silu_and_mul.default(out, input) + return out + + +def gelu_tanh_and_mul(input: torch.Tensor, out: torch.Tensor = None) -> torch.Tensor: + if input.shape[-1] * input.dtype.itemsize % 16 != 0: + raise ValueError("The pointers must be multiple of 16 bytes.") + if out is not None: + _check_shape(input, out) + else: + out = torch.empty( + input.shape[:-1] + (input.shape[-1] // 2,), + device=input.device, + dtype=input.dtype, + ) + torch.ops.sgl_kernel.gelu_tanh_and_mul.default(out, input) + return out + + +def gelu_and_mul(input: torch.Tensor, out: torch.Tensor = None) -> torch.Tensor: + if input.shape[-1] * input.dtype.itemsize % 16 != 0: + raise ValueError("The pointers must be multiple of 16 bytes.") + if out is not None: + _check_shape(input, out) + else: + out = torch.empty( + input.shape[:-1] + (input.shape[-1] // 2,), + device=input.device, + dtype=input.dtype, + ) + torch.ops.sgl_kernel.gelu_and_mul.default(out, input) + return out + + +if torch.version.hip is not None: + + def gelu_quick(input: torch.Tensor, out: torch.Tensor = None) -> torch.Tensor: + """ + Quick-GELU: y = x * sigmoid(1.702 * x) + + The CUDA/HIP kernel uses 128-bit (16-byte) vector loads & stores, + so the last-dimension byte length must be a multiple of 16 bytes. + """ + if input.shape[-1] * input.dtype.itemsize % 16 != 0: + raise ValueError( + f"The last dimension ({input.shape[-1]}) x itemsize " + f"({input.dtype.itemsize}) must be a multiple of 16 bytes." + ) + + if out is not None: + assert input.shape == out.shape, f"{input.shape} != {out.shape}" + else: + out = torch.empty_like(input) + + torch.ops.sgl_kernel.gelu_quick(out, input) + return out + + +@dataclass +class FusedSetKVBufferArg: + """ + value : Optional[torch.Tensor] + Value tensor, shape: ``(nnz, num_v_heads * head_size)``. + k_buffer : Optional[torch.Tensor] + Buffer for keys, shape: ``(nnz, num_k_heads * head_size)``. + v_buffer : Optional[torch.Tensor] + Buffer for values, shape: ``(nnz, num_v_heads * head_size)``. + k_scale : Optional[float] + Scale factor for keys. + v_scale : Optional[float] + Scale factor for values. + cache_loc : Optional[torch.Tensor] + Cache location tensor, used for indexing kv cache. + """ + + value: torch.Tensor + k_buffer: torch.Tensor + v_buffer: torch.Tensor + k_scale: Optional[float] + v_scale: Optional[float] + cache_loc: torch.Tensor + + +def _view_3d(x, head_size): + return x.view(x.shape[0], -1, head_size) + + +def apply_rope_with_cos_sin_cache_inplace( + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + head_size: int, + cos_sin_cache: torch.Tensor, + is_neox: bool = True, + fused_set_kv_buffer_arg: Optional[FusedSetKVBufferArg] = None, + enable_pdl: Optional[bool] = None, +) -> None: + r""" + Apply rotary embedding to keys and queries with precomputed cos/sin values. + This is designed to be compatible with the SGL/vLLM implementation. + The result is inplace applied to the input tensors. + + Parameters + ---------- + positions : torch.Tensor + Position indices, shape: ``(nnz)``. + query : torch.Tensor + Query tensor, shape: ``(nnz, num_q_heads * head_size)``. + key : torch.Tensor + Key tensor, shape: ``(nnz, num_k_heads * head_size)``. + cos_sin_cache : torch.Tensor + Cosine and Sine cache tensor, shape: ``(max_seq_len, rotary_dim)``. + Cosine is the first half and Sine is the second half on rotary_dim. + is_neox : bool + Whether to use Neox style RoPE, default: ``True``. + + * If ``True``, the last dimension of the query/key tensor is not interleaved, i.e., + we rotate the first half dimensions ``([..., :head_dim//2])`` and the second half + dimensions ``([..., head_dim//2:])``. + + * If ``False``, the last dimension of the query/key tensor is interleaved, i.e., + we rotate the even dimensions ``([..., ::2])`` and odd dimensions ``([..., 1::2])``. + fused_set_kv_buffer_arg : FusedSetKVBufferArg + Fuse the set-kv-buffer operation into this kernel + + Note + ---- + The rotary dimension is determined by the cosine cache and sine cache. + """ + if cos_sin_cache.dtype != torch.float32: + raise ValueError("cos_sin_cache should be float32") + + if enable_pdl is None: + # the non-fused branch does not yet support PDL, but after we switch to our impl for that branch it will + enable_pdl = is_arch_support_pdl() and (fused_set_kv_buffer_arg is not None) + + if (a := fused_set_kv_buffer_arg) is not None: + assert a.k_scale is None, "k_scale is not yet supported" + assert a.v_scale is None, "v_scale is not yet supported" + assert a.cache_loc.dtype == torch.int64, f"{a.cache_loc.dtype=}" + + torch.ops.sgl_kernel.apply_rope_pos_ids_cos_sin_cache.default( + _view_3d(query, head_size), + _view_3d(key, head_size), + _view_3d(query, head_size), + _view_3d(key, head_size), + cos_sin_cache, + positions.long(), + (not is_neox), + enable_pdl, + ( + _view_3d(fused_set_kv_buffer_arg.value, head_size) + if fused_set_kv_buffer_arg is not None + else None + ), + ( + _view_3d(fused_set_kv_buffer_arg.k_buffer, head_size) + if fused_set_kv_buffer_arg is not None + else None + ), + ( + _view_3d(fused_set_kv_buffer_arg.v_buffer, head_size) + if fused_set_kv_buffer_arg is not None + else None + ), + ( + fused_set_kv_buffer_arg.cache_loc + if fused_set_kv_buffer_arg is not None + else None + ), + ) + + +def downcast_fp8( + k: torch.Tensor, + v: torch.Tensor, + k_out: torch.Tensor, + v_out: torch.Tensor, + k_scale: torch.Tensor, + v_scale: torch.Tensor, + loc: torch.Tensor, + mult: int = 1, + offset: int = 0, +) -> None: + torch.ops.sgl_kernel.downcast_fp8( + k, v, k_out, v_out, k_scale, v_scale, loc, mult, offset + ) + + +def copy_to_gpu_no_ce(input: torch.Tensor, output: torch.Tensor): + torch.ops.sgl_kernel.copy_to_gpu_no_ce(input, output) + + +def concat_mla_k( + k: torch.Tensor, + k_nope: torch.Tensor, + k_rope: torch.Tensor, +): + torch.ops.sgl_kernel.concat_mla_k(k, k_nope, k_rope) + + +def concat_mla_absorb_q( + a: torch.Tensor, + b: torch.Tensor, +): + *batch_dims, _ = a.shape + out = torch.empty( + (*batch_dims, a.shape[-1] + b.shape[-1]), device=a.device, dtype=a.dtype + ) + torch.ops.sgl_kernel.concat_mla_absorb_q(a, b, out) + return out diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/expert_specialization.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/expert_specialization.py new file mode 100644 index 00000000..8b76a682 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/expert_specialization.py @@ -0,0 +1,29 @@ +import torch + + +def es_fp8_blockwise_scaled_grouped_mm( + output, + a, + b, + scales_a, + scales_b, + stride_a, + stride_b, + stride_d, + problem_sizes, + expert_offsets, + workspace, +): + torch.ops.sgl_kernel.es_fp8_blockwise_scaled_grouped_mm.default( + output, + a, + b, + scales_a, + scales_b, + stride_a, + stride_b, + stride_d, + problem_sizes, + expert_offsets, + workspace, + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/flash_attn.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/flash_attn.py new file mode 100644 index 00000000..c3ffbc54 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/flash_attn.py @@ -0,0 +1,369 @@ +from functools import lru_cache +from typing import Optional, Union + +import torch + +try: + from sgl_kernel import flash_ops +except: + raise ImportError( + "Can not import FA3 in sgl_kernel. Please check your installation." + ) + +try: + from ._fa4_interface import flash_attn_varlen_func as flash_attn_varlen_func_v4 +except ImportError: + flash_attn_varlen_func_v4 = None + + +@lru_cache(maxsize=1) +def is_fa3_supported(device=None) -> bool: + # There some fa3 FYI + # FA3 can fail without a enough shared memory for a some shapes, such as higher + # hidden_dim or some special cases. + # Right now, fa3 is supported for sm80/sm87 and sm86/sm89. The main different + # Between sm80/sm87 and sm86/sm89 is the shared memory size. you can follow the link below for more information + # https://docs.nvidia.com/cuda/cuda-c-programming-guide/#shared-memory-8-x + # And for sgl-kernel right now, we can build fa3 on sm80/sm86/sm89/sm90a. + # That means if you use A100/A*0/L20/L40/L40s/4090 you can use fa3. + return (torch.version.cuda >= "12.3") and ( + torch.cuda.get_device_capability(device)[0] == 9 + or torch.cuda.get_device_capability(device)[0] == 8 + ) + + +def maybe_contiguous(x): + return x.contiguous() if x is not None and x.stride(-1) != 1 else x + + +def flash_attn_with_kvcache( + q, + k_cache, + v_cache, + k=None, + v=None, + qv=None, + rotary_cos=None, + rotary_sin=None, + cache_seqlens: Optional[Union[int, torch.Tensor]] = None, + cache_batch_idx: Optional[torch.Tensor] = None, + cache_leftpad: Optional[torch.Tensor] = None, + page_table: Optional[torch.Tensor] = None, + cu_seqlens_q: Optional[torch.Tensor] = None, + cu_seqlens_k_new: Optional[torch.Tensor] = None, + max_seqlen_q: Optional[int] = None, + rotary_seqlens: Optional[torch.Tensor] = None, + q_descale: Optional[torch.Tensor] = None, + k_descale: Optional[torch.Tensor] = None, + v_descale: Optional[torch.Tensor] = None, + softmax_scale=None, + causal=False, + window_size=(-1, -1), # -1 means infinite context window + attention_chunk: Optional[int] = None, + softcap=0.0, # 0.0 means deactivated + rotary_interleaved=True, + scheduler_metadata=None, + num_splits=0, # Can be tuned for speed + pack_gqa=None, # Can be tuned for speed + sm_margin=0, # Can be tuned if some SMs are used for communication + return_softmax_lse=False, + sinks=None, + ver=3, +): + """ + If k and v are not None, k_cache and v_cache will be updated *inplace* with the new values from + k and v. This is useful for incremental decoding: you can pass in the cached keys/values from + the previous step, and update them with the new keys/values from the current step, and do + attention with the updated cache, all in 1 kernel. + + If you pass in k / v, you must make sure that the cache is large enough to hold the new values. + For example, the KV cache could be pre-allocated with the max sequence length, and you can use + cache_seqlens to keep track of the current sequence lengths of each sequence in the batch. + + Also apply rotary embedding if rotary_cos and rotary_sin are passed in. The key @k will be + rotated by rotary_cos and rotary_sin at indices cache_seqlens, cache_seqlens + 1, etc. + If causal or local (i.e., window_size != (-1, -1)), the query @q will be rotated by rotary_cos + and rotary_sin at indices cache_seqlens, cache_seqlens + 1, etc. + If not causal and not local, the query @q will be rotated by rotary_cos and rotary_sin at + indices cache_seqlens only (i.e. we consider all tokens in @q to be at position cache_seqlens). + + See tests/test_flash_attn.py::test_flash_attn_kvcache for examples of how to use this function. + + Supports multi-query and grouped-query attention (MQA/GQA) by passing in KV with fewer heads + than Q. Note that the number of heads in Q must be divisible by the number of heads in KV. + For example, if Q has 6 heads and K, V have 2 heads, head 0, 1, 2 of Q will attention to head + 0 of K, V, and head 3, 4, 5 of Q will attention to head 1 of K, V. + + If causal=True, the causal mask is aligned to the bottom right corner of the attention matrix. + For example, if seqlen_q = 2 and seqlen_k = 5, the causal mask (1 = keep, 0 = masked out) is: + 1 1 1 1 0 + 1 1 1 1 1 + If seqlen_q = 5 and seqlen_k = 2, the causal mask is: + 0 0 + 0 0 + 0 0 + 1 0 + 1 1 + If the row of the mask is all zero, the output will be zero. + + If window_size != (-1, -1), implements sliding window local attention. Query at position i + will only attend to keys between + [i + seqlen_k - seqlen_q - window_size[0], i + seqlen_k - seqlen_q + window_size[1]] inclusive. + + Note: Does not support backward pass. + + Arguments: + q: (batch_size, seqlen, nheads, headdim) + k_cache: (batch_size_cache, seqlen_cache, nheads_k, headdim) if there's no page_table, + or (num_blocks, page_block_size, nheads_k, headdim) if there's a page_table (i.e. paged KV cache) + page_block_size must be a multiple of 256. + v_cache: (batch_size_cache, seqlen_cache, nheads_k, headdim_v) if there's no page_table, + or (num_blocks, page_block_size, nheads_k, headdim_v) if there's a page_table (i.e. paged KV cache) + k [optional]: (batch_size, seqlen_new, nheads_k, headdim). If not None, we concatenate + k with k_cache, starting at the indices specified by cache_seqlens. + v [optional]: (batch_size, seqlen_new, nheads_k, headdim_v). Similar to k. + qv [optional]: (batch_size, seqlen, nheads, headdim_v) + rotary_cos [optional]: (seqlen_ro, rotary_dim / 2). If not None, we apply rotary embedding + to k and q. Only applicable if k and v are passed in. rotary_dim must be divisible by 16. + rotary_sin [optional]: (seqlen_ro, rotary_dim / 2). Similar to rotary_cos. + cache_seqlens: int, or (batch_size,), dtype torch.int32. The sequence lengths of the + KV cache. + cache_batch_idx: (batch_size,), dtype torch.int32. The indices used to index into the KV cache. + If None, we assume that the batch indices are [0, 1, 2, ..., batch_size - 1]. + If the indices are not distinct, and k and v are provided, the values updated in the cache + might come from any of the duplicate indices. + cache_leftpad: (batch_size,), dtype torch.int32. The index that the KV cache starts. If None, assume 0. + page_table [optional]: (batch_size, max_num_blocks_per_seq), dtype torch.int32. + softmax_scale: float. The scaling of QK^T before applying softmax. + Default to 1 / sqrt(headdim). + causal: bool. Whether to apply causal attention mask (e.g., for auto-regressive modeling). + window_size: (left, right). If not (-1, -1), implements sliding window local attention. + attention_chunk: Optional[int]. If not None, splits the query into chunks of this size to save memory. + softcap: float. Anything > 0 activates softcapping attention. + rotary_interleaved: bool. Only applicable if rotary_cos and rotary_sin are passed in. + If True, rotary embedding will combine dimensions 0 & 1, 2 & 3, etc. If False, + rotary embedding will combine dimensions 0 & rotary_dim / 2, 1 & rotary_dim / 2 + 1 + (i.e. GPT-NeoX style). + num_splits: int. If > 1, split the key/value into this many chunks along the sequence. + If num_splits == 1, we don't split the key/value. If num_splits == 0, we use a heuristic + to automatically determine the number of splits. + Don't change this unless you know what you are doing. + return_softmax_lse: bool. Whether to return the logsumexp of the attention scores. + + Return: + out: (batch_size, seqlen, nheads, headdim). + softmax_lse [optional, if return_softmax_lse=True]: (batch_size, nheads, seqlen). The + logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax + normalization factor). + """ + if ver == 4: + assert ( + flash_attn_varlen_func_v4 is not None + ), "FA4 is not available, please check your installation." + # Using `(-1, -1)` as no sliding window causes correctness issues for FA4. + assert ( + k is None and v is None + ), "FA4 does not support updating KV cache in-place." + assert ( + rotary_cos is None and rotary_sin is None and rotary_seqlens is None + ), "FA4 does not support rotary embedding." + assert ( + cache_batch_idx is None and cache_leftpad is None + ), "FA4 does not support non-consecutive batch indices or left padding." + assert ( + q_descale is None and k_descale is None and v_descale is None + ), "FA4 does not support descale." + + if window_size == (-1, -1): + window_size = (None, None) + return flash_attn_varlen_func_v4( + q=q, + k=k_cache, + v=v_cache, + cu_seqlens_q=cu_seqlens_q, + seqused_k=cache_seqlens, + softmax_scale=softmax_scale, + causal=causal, + window_size=window_size, + softcap=softcap, + pack_gqa=pack_gqa, + return_softmax_lse=return_softmax_lse, + learnable_sink=sinks, + page_table=page_table, + ) + + assert k_cache.stride(-1) == 1, "k_cache must have contiguous last dimension" + assert v_cache.stride(-1) == 1, "v_cache must have contiguous last dimension" + if softmax_scale is None: + softmax_scale = (q.shape[-1] + (qv.shape[-1] if qv is not None else 0)) ** ( + -0.5 + ) + if cache_seqlens is not None and isinstance(cache_seqlens, int): + cache_seqlens = torch.full( + (k_cache.shape[0],), cache_seqlens, dtype=torch.int32, device=k_cache.device + ) + cache_seqlens = maybe_contiguous(cache_seqlens) + + q, k_cache, k, v = [maybe_contiguous(x) for x in (q, k_cache, k, v)] + v_cache = ( + v_cache.contiguous() + if v_cache.stride(-1) != 1 and v_cache.stride(-3) != 1 + else v_cache + ) + cu_seqlens_q, cu_seqlens_k_new = [ + maybe_contiguous(x) for x in (cu_seqlens_q, cu_seqlens_k_new) + ] + page_table, cache_batch_idx, cache_leftpad = [ + maybe_contiguous(x) for x in (page_table, cache_batch_idx, cache_leftpad) + ] + rotary_cos, rotary_sin = [maybe_contiguous(x) for x in (rotary_cos, rotary_sin)] + rotary_seqlens = maybe_contiguous(rotary_seqlens) + attention_chunk = 0 if attention_chunk is None else int(attention_chunk) + + out, softmax_lse, *rest = torch.ops.sgl_kernel.fwd.default( + q, + k_cache, + v_cache, + k, + v, + qv, + None, # out + cu_seqlens_q, + None, # cu_seqlens_k + cu_seqlens_k_new, + None, # seqused_q + cache_seqlens, + max_seqlen_q, + None, # max_seqlen_k + page_table, + cache_batch_idx, + cache_leftpad, + rotary_cos, + rotary_sin, + rotary_seqlens, + q_descale, + k_descale, + v_descale, + softmax_scale, + causal, + window_size[0], + window_size[1], + attention_chunk, + softcap, + rotary_interleaved, + scheduler_metadata, + num_splits, + pack_gqa, + sm_margin, + sinks, + ) + # return (out, softmax_lse) if return_softmax_lse else out + return (out, softmax_lse, *rest) if return_softmax_lse else out + + +def flash_attn_varlen_func( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q=None, + max_seqlen_k=None, + seqused_q=None, + seqused_k=None, + page_table=None, + softmax_scale=None, + causal=False, + qv=None, + q_descale=None, + k_descale=None, + v_descale=None, + window_size=(-1, -1), + attention_chunk=0, + softcap=0.0, + num_splits=1, + pack_gqa=None, + sm_margin=0, + return_softmax_lse=False, + sinks=None, + ver=3, +): + if ver == 4: + assert ( + flash_attn_varlen_func_v4 is not None + ), "FA4 is not available, please check your installation." + # Using `(-1, -1)` as no sliding window causes correctness issues for FA4. + if window_size == (-1, -1): + window_size = (None, None) + return flash_attn_varlen_func_v4( + q, + k, + v, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + seqused_q=seqused_q, + seqused_k=seqused_k, + page_table=page_table, + softmax_scale=softmax_scale, + causal=causal, + window_size=window_size, + softcap=softcap, + pack_gqa=pack_gqa, + learnable_sink=sinks, + return_softmax_lse=return_softmax_lse, + ) + + if not is_fa3_supported(): + raise NotImplementedError( + "flash_attn at sgl-kernel is only supported on sm90 and above" + ) + + # FA3 requires max_seqlen_q and max_seqlen_k + if max_seqlen_q is None or max_seqlen_k is None: + raise ValueError("max_seqlen_q and max_seqlen_k are required for FA3") + + if softmax_scale is None: + softmax_scale = (q.shape[-1] + (qv.shape[-1] if qv is not None else 0)) ** ( + -0.5 + ) + attention_chunk = 0 if attention_chunk is None else int(attention_chunk) + + out, softmax_lse, *rest = torch.ops.sgl_kernel.fwd.default( + q, + k, + v, + None, # k_new + None, # v_new + qv, # qv + None, # out + cu_seqlens_q, + cu_seqlens_k, + None, # cu_seqlens_k_new + seqused_q, + seqused_k, + max_seqlen_q, + max_seqlen_k, + None, # page_table, + None, # kv_batch_idx + None, # leftpad_k + None, # rotary cos + None, # rotary sin + None, # seqlens_rotary + q_descale, + k_descale, + v_descale, + softmax_scale, + causal, + window_size[0], + window_size[1], + attention_chunk, + softcap, + is_rotary_interleaved=False, + scheduler_metadata=None, + num_splits=num_splits, + pack_gqa=pack_gqa, + sm_margin=sm_margin, + sinks=sinks, + ) + + return (out, softmax_lse, *rest) if return_softmax_lse else out diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/flash_mla.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/flash_mla.py new file mode 100644 index 00000000..144ddc31 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/flash_mla.py @@ -0,0 +1,155 @@ +from typing import Optional, Tuple + +import torch + +try: + from sgl_kernel import flashmla_ops # triggers TORCH extension registration +except Exception as _e: + _flashmla_import_error = _e +else: + _flashmla_import_error = None + +_IMPORT_ERROR = ImportError( + "Failed to load sgl_kernel.flashmla_ops extension. Ensure CUDA Driver >= 12.4" +) + + +def get_mla_metadata( + cache_seqlens: torch.Tensor, + num_q_tokens_per_head_k: int, + num_heads_k: int, + num_heads_q: Optional[int] = None, + is_fp8_kvcache: bool = False, + topk: Optional[int] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Arguments: + cache_seqlens: (batch_size), dtype torch.int32. + num_q_tokens_per_head_k: Equals to num_q_tokens_per_q_seq * num_heads_q // num_heads_k. + num_heads_k: The number of k heads. + num_heads_q: The number of q heads. This argument is optional when sparse attention is not enabled + is_fp8_kvcache: Whether the k_cache and v_cache are in fp8 format. + topk: If not None, sparse attention will be enabled, and only tokens in the `indices` array passed to `flash_mla_with_kvcache_sm90` will be attended to. + + Returns: + tile_scheduler_metadata: (num_sm_parts, TileSchedulerMetaDataSize), dtype torch.int32. + num_splits: (batch_size + 1), dtype torch.int32. + """ + if is_fp8_kvcache and topk is None: + return torch.ops.sgl_kernel.get_mla_decoding_metadata_dense_fp8.default( + cache_seqlens, + num_q_tokens_per_head_k, + num_heads_k, + ) + return torch.ops.sgl_kernel.get_mla_decoding_metadata.default( + cache_seqlens, + num_q_tokens_per_head_k, + num_heads_k, + num_heads_q, + is_fp8_kvcache, + topk, + ) + + +def flash_mla_with_kvcache( + q: torch.Tensor, + k_cache: torch.Tensor, + block_table: torch.Tensor, + cache_seqlens: torch.Tensor, + head_dim_v: int, + tile_scheduler_metadata: torch.Tensor, + num_splits: torch.Tensor, + softmax_scale: Optional[float] = None, + causal: bool = False, + descale_q: torch.Tensor | None = None, + descale_k: torch.Tensor | None = None, + is_fp8_kvcache: bool = False, + indices: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Arguments: + q: (batch_size, seq_len_q, num_heads_q, head_dim). + k_cache: (num_blocks, page_block_size, num_heads_k, head_dim). + block_table: (batch_size, max_num_blocks_per_seq), torch.int32. + cache_seqlens: (batch_size), torch.int32. + head_dim_v: Head dimension of v. + tile_scheduler_metadata: (num_sm_parts, TileSchedulerMetaDataSize), torch.int32, returned by get_mla_metadata. + num_splits: (batch_size + 1), torch.int32, returned by get_mla_metadata. + softmax_scale: float. The scale of QK^T before applying softmax. Default to 1 / sqrt(head_dim). + causal: bool. Whether to apply causal attention mask. + descale_q: (batch_size), torch.float32. Descaling factors for Q, used for fp8 quantization. + descale_k: (batch_size), torch.float32. Descaling factors for K, used for fp8 quantization. + is_fp8_kvcache: bool. Whether the k_cache and v_cache are in fp8 format. For the format of FP8 KV cache, please refer to README.md + indices: (batch_size, seq_len_q, topk), torch.int32. If not None, sparse attention will be enabled, and only tokens in the `indices` array will be attended to. Invalid indices should be set to -1 or numbers >= total_seq_len_kv. For details about how to set up `indices`, please refer to README.md. + + Returns: + out: (batch_size, seq_len_q, num_heads_q, head_dim_v). + softmax_lse: (batch_size, num_heads_q, seq_len_q), torch.float32. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + if indices is not None: + assert causal == False, "causal must be `false` if sparse attention is enabled." + assert (descale_q is None) == ( + descale_k is None + ), "descale_q and descale_k should be both None or both not None" + + if indices is None and q.element_size() == 1: + out, softmax_lse = torch.ops.sgl_kernel.fwd_kvcache_mla_fp8.default( + q, + k_cache, + head_dim_v, + cache_seqlens, + block_table, + softmax_scale, + causal, + tile_scheduler_metadata, + num_splits, + descale_q, + descale_k, + ) + else: + out, softmax_lse = torch.ops.sgl_kernel.fwd_kvcache_mla.default( + q, + k_cache, + head_dim_v, + cache_seqlens, + block_table, + softmax_scale, + causal, + tile_scheduler_metadata, + num_splits, + is_fp8_kvcache, + indices, + ) + return out, softmax_lse + + +def flash_mla_sparse_fwd( + q: torch.Tensor, + kv: torch.Tensor, + indices: torch.Tensor, + sm_scale: float, + d_v: int = 512, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Sparse attention prefill kernel + + Args: + q: [s_q, h_q, d_qk], bfloat16 + kv: [s_kv, h_kv, d_qk], bfloat16 + indices: [s_q, h_kv, topk], int32. Invalid indices should be set to -1 or numbers >= s_kv + sm_scale: float + d_v: The dimension of value vectors. Can only be 512 + + Returns: + (output, max_logits, lse) + About the definition of output, max_logits and lse, please refer to README.md + - output: [s_q, h_q, d_v], bfloat16 + - max_logits: [s_q, h_q], float + - lse: [s_q, h_q], float, 2-based log-sum-exp + """ + results = torch.ops.sgl_kernel.sparse_prefill_fwd.default( + q, kv, indices, sm_scale, d_v + ) + return results diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/fused_moe.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/fused_moe.py new file mode 100644 index 00000000..15f3a2be --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/fused_moe.py @@ -0,0 +1,61 @@ +from typing import Optional + +import torch + + +def moe_wna16_marlin_gemm( + a: torch.Tensor, + c_or_none: Optional[torch.Tensor], + b_q_weight: torch.Tensor, + b_bias_or_none: Optional[torch.Tensor], + b_scales: torch.Tensor, + global_scale_or_none: Optional[torch.Tensor], + b_zeros_or_none: Optional[torch.Tensor], + g_idx_or_none: Optional[torch.Tensor], + perm_or_none: Optional[torch.Tensor], + workspace: torch.Tensor, + sorted_token_ids: torch.Tensor, + expert_ids: torch.Tensor, + num_tokens_post_padded: torch.Tensor, + topk_weights: torch.Tensor, + moe_block_size: int, + top_k: int, + mul_topk_weights: bool, + is_ep: bool, + b_q_type_id: int, + size_m: int, + size_n: int, + size_k: int, + is_k_full: bool, + use_atomic_add: bool, + use_fp32_reduce: bool, + is_zp_float: bool, +): + return torch.ops.sgl_kernel.moe_wna16_marlin_gemm.default( + a, + c_or_none, + b_q_weight, + b_bias_or_none, + b_scales, + global_scale_or_none, + b_zeros_or_none, + g_idx_or_none, + perm_or_none, + workspace, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + topk_weights, + moe_block_size=moe_block_size, + top_k=top_k, + mul_topk_weights=mul_topk_weights, + is_ep=is_ep, + b_q_type_id=b_q_type_id, + size_m=size_m, + size_n=size_n, + size_k=size_k, + is_k_full=is_k_full, + use_atomic_add=use_atomic_add, + use_fp32_reduce=use_fp32_reduce, + is_zp_float=is_zp_float, + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/gemm.py new file mode 100644 index 00000000..4e23ebdd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/gemm.py @@ -0,0 +1,564 @@ +from typing import Optional, Tuple + +import torch +from sgl_kernel.scalar_type import ScalarType +from sgl_kernel.utils import _get_cache_buf + + +def awq_dequantize( + qweight: torch.Tensor, scales: torch.Tensor, qzeros: torch.Tensor +) -> torch.ByteTensor: + return torch.ops.sgl_kernel.awq_dequantize.default(qweight, scales, qzeros) + + +def int8_scaled_mm(mat_a, mat_b, scales_a, scales_b, out_dtype, bias=None): + return torch.ops.sgl_kernel.int8_scaled_mm.default( + mat_a, + mat_b, + scales_a, + scales_b, + out_dtype, + bias, + ) + + +def fp8_blockwise_scaled_mm(mat_a, mat_b, scales_a, scales_b, out_dtype): + return torch.ops.sgl_kernel.fp8_blockwise_scaled_mm.default( + mat_a, + mat_b, + scales_a, + scales_b, + out_dtype, + ) + + +def fp8_scaled_mm(mat_a, mat_b, scales_a, scales_b, out_dtype, bias=None): + return torch.ops.sgl_kernel.fp8_scaled_mm.default( + mat_a, + mat_b, + scales_a, + scales_b, + out_dtype, + bias, + ) + + +def _bmm_fp8_internal( + workspace_buffer: torch.Tensor, + A: torch.Tensor, + B: torch.Tensor, + D: torch.Tensor, + A_scale: torch.Tensor, + B_scale: torch.Tensor, +) -> None: + cublas_handle = torch.cuda.current_blas_handle() + torch.ops.sgl_kernel.bmm_fp8.default( + A, + B, + D, + A_scale, + B_scale, + workspace_buffer, + cublas_handle, + ) + + +def bmm_fp8( + A: torch.Tensor, + B: torch.Tensor, + A_scale: torch.Tensor, + B_scale: torch.Tensor, + dtype: torch.dtype, + out: Optional[torch.Tensor] = None, +) -> torch.Tensor: + if out is None: + out = torch.empty( + (A.shape[0], A.shape[1], B.shape[2]), + device=A.device, + dtype=dtype, + ) + workspace_buffer = _get_cache_buf("bmm_fp8_workspace", 32 * 1024 * 1024, A.device) + _bmm_fp8_internal(workspace_buffer, A, B, out, A_scale, B_scale) + return out + + +def dsv3_fused_a_gemm( + mat_a: torch.Tensor, + mat_b: torch.Tensor, + output: Optional[torch.Tensor] = None, +) -> torch.Tensor: + if output is None: + output = torch.empty( + (mat_a.shape[0], mat_b.shape[1]), + device=mat_a.device, + dtype=mat_a.dtype, + ) + torch.ops.sgl_kernel.dsv3_fused_a_gemm.default(output, mat_a, mat_b) + return output + + +def sgl_per_token_group_quant_8bit( + input: torch.Tensor, + output_q: torch.Tensor, + output_s: torch.Tensor, + group_size: int, + eps: float, + fp8_min: float, + fp8_max: float, + scale_ue8m0: bool = False, + fuse_silu_and_mul: bool = False, + masked_m: Optional[torch.Tensor] = None, + enable_v2: Optional[bool] = None, +) -> None: + if enable_v2 is None: + from sglang.srt.utils import get_bool_env_var + + enable_v2 = get_bool_env_var("SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2") + + if enable_v2: + return torch.ops.sgl_kernel.sgl_per_token_group_quant_8bit_v2.default( + input, + output_q, + output_s, + group_size, + eps, + fp8_min, + fp8_max, + scale_ue8m0, + fuse_silu_and_mul, + masked_m, + ) + + assert not fuse_silu_and_mul, "only v2 support fuse_silu_and_mul" + assert masked_m is None, "only v2 support masked_m" + torch.ops.sgl_kernel.sgl_per_token_group_quant_8bit.default( + input, output_q, output_s, group_size, eps, fp8_min, fp8_max, scale_ue8m0 + ) + + +# For legacy usage +sgl_per_token_group_quant_fp8 = sgl_per_token_group_quant_8bit +sgl_per_token_group_quant_int8 = sgl_per_token_group_quant_8bit + + +def sgl_per_tensor_quant_fp8( + input: torch.Tensor, + output_q: torch.Tensor, + output_s: torch.Tensor, + is_static: bool, +) -> None: + torch.ops.sgl_kernel.sgl_per_tensor_quant_fp8.default( + input, output_q, output_s, is_static + ) + + +def sgl_per_token_quant_fp8( + input: torch.Tensor, + output_q: torch.Tensor, + output_s: torch.Tensor, +) -> None: + torch.ops.sgl_kernel.sgl_per_token_quant_fp8.default(input, output_q, output_s) + + +def cutlass_scaled_fp4_mm( + a: torch.Tensor, + b: torch.Tensor, + block_scale_a: torch.Tensor, + block_scale_b: torch.Tensor, + alpha: torch.Tensor, + out_dtype: torch.dtype, +) -> torch.Tensor: + assert a.ndim == 2 and b.ndim == 2 + m, n = a.shape[0], b.shape[0] + out = torch.empty((m, n), dtype=out_dtype, device=a.device) + torch.ops.sgl_kernel.cutlass_scaled_fp4_mm.default( + out, a, b, block_scale_a, block_scale_b, alpha + ) + return out + + +def scaled_fp4_quant( + input: torch.Tensor, input_global_scale: torch.Tensor +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Quantize input tensor to FP4 and return quantized tensor and scale. + + This function quantizes the last dimension of the given tensor `input`. For + every 16 consecutive elements, a single dynamically computed scaling factor + is shared. This scaling factor is quantized using the `input_global_scale` + and is stored in a swizzled layout (see + https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-scale-factor-b-layout-4x). + + Args: + input: The input tensor to be quantized to FP4 + input_global_scale: A scalar scaling factor for the entire tensor. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: The output tensor in FP4 but every + two values are packed into a uint8 and float8_e4m3 scaling factors + in a sizzled layout. + """ + assert input.ndim >= 1, f"input.ndim needs to be >= 1, but got {input.ndim}." + other_dims = 1 if input.ndim == 1 else -1 + input = input.reshape(other_dims, input.shape[-1]) + m, n = input.shape + block_size = 16 + device = input.device + + assert n % block_size == 0, f"last dim has to be multiple of 16, but got {n}." + assert input.dtype in ( + torch.float16, + torch.bfloat16, + ), f"input.dtype needs to be fp16 or bf16 but got {input.dtype}." + + # Two fp4 values will be packed into an uint8. + output = torch.empty((m, n // 2), device=device, dtype=torch.uint8) + + # We use the rounded values to store the swizzled values. Then, the scaling + # factors in float8_e4m3fn are packed into an int32 for every 4 values. + rounded_m = ((m + 128 - 1) // 128) * 128 + scale_n = n // block_size + rounded_n = ((scale_n + 4 - 1) // 4) * 4 + # padded part should be zeroed out + if rounded_n > scale_n: + output_scale = torch.zeros( + (rounded_m, rounded_n // 4), device=device, dtype=torch.int32 + ) + else: + output_scale = torch.empty( + (rounded_m, rounded_n // 4), device=device, dtype=torch.int32 + ) + + torch.ops.sgl_kernel.scaled_fp4_quant.default( + output, input, output_scale, input_global_scale + ) + output_scale = output_scale.view(torch.float8_e4m3fn) + return output, output_scale + + +def qserve_w4a8_per_chn_gemm( + in_feats: torch.Tensor, + kernel: torch.Tensor, + wscales: torch.Tensor, + ascales: torch.Tensor, + w_szs: torch.Tensor, + a_ssums: torch.Tensor, + out_feats: Optional[torch.Tensor] = None, +) -> torch.Tensor: + if out_feats is None: + # NOTE(HandH1998): qserve_w4a8_per_chn_gemm only supports out dtype=torch.float16 now + out_feats = torch.empty( + (in_feats.shape[0], kernel.shape[0]), + device=in_feats.device, + dtype=torch.float16, + ) + torch.ops.sgl_kernel.qserve_w4a8_per_chn_gemm.default( + in_feats, kernel, wscales, ascales, w_szs, a_ssums, out_feats + ) + return out_feats + + +def qserve_w4a8_per_group_gemm( + in_feats: torch.Tensor, + kernel: torch.Tensor, + zeros: torch.Tensor, + scales_i8: torch.Tensor, + wscales: torch.Tensor, + ascales: torch.Tensor, + out_feats: Optional[torch.Tensor] = None, +) -> torch.Tensor: + if out_feats is None: + # NOTE(HandH1998): qserve_w4a8_per_group_gemm only supports out dtype=torch.float16 now + out_feats = torch.empty( + (in_feats.shape[0], kernel.shape[0]), + device=in_feats.device, + dtype=torch.float16, + ) + torch.ops.sgl_kernel.qserve_w4a8_per_group_gemm.default( + in_feats, kernel, zeros, scales_i8, wscales, ascales, out_feats + ) + return out_feats + + +def dsv3_router_gemm( + hidden_states: torch.Tensor, + router_weights: torch.Tensor, + out_dtype: torch.dtype = torch.bfloat16, +) -> torch.Tensor: + output = torch.empty( + hidden_states.shape[0], + router_weights.shape[0], + device=hidden_states.device, + dtype=out_dtype, + ) + torch.ops.sgl_kernel.dsv3_router_gemm( + output, + hidden_states, + router_weights, + ) + return output + + +def shuffle_rows(input_tensor, dst2src_map, output_tensor_shape): + output_tensor = torch.empty( + output_tensor_shape, + device=input_tensor.device, + dtype=input_tensor.dtype, + ) + torch.ops.sgl_kernel.shuffle_rows.default(input_tensor, dst2src_map, output_tensor) + return output_tensor + + +def scaled_fp4_grouped_quant( + input_tensor: torch.Tensor, + input_global_scale: torch.Tensor, + mask: torch.Tensor, +): + """ + Quantize input tensor to FP4 and return quantized tensor and scale, for + grouped gemm inputs (e.g., grouped_gemm_nt_masked for flashinfer). + Args: + input: The input tensor to be quantized to FP4, with shape (l, m, k) + l is number of groups, m is number of tokens per group, k is number of features. + input_global_scale: A scalar scaling factor for the entire tensor, with + shape (l,). + Outputs: + output: The quantized tensor in FP4, with shape (m, k // 2, l) but the physical + layout is (l, m, k // 2). `// 2` is because two fp4 values are packed into + an uint8. + output_scales: The blockscale tensor in FP8-E4M3, with shape (32, 4, rm, 4, rk, l) + but the physical layout is (l, rm, rk, 32, 4, 4). + Note: + For the shape of output_scales, `32 * 4 * rm` is a padded m to nearest multiple of 128. + `4 * rk` is a padded `k // 16` to nearest multiple of 4. These layout constants are + required by the NVIDIA Blackwell MMA operations. + """ + device = input_tensor.device + l, m, k = input_tensor.shape + sf_vec_size = 16 + assert k % sf_vec_size == 0, f"k must be multiple of 16, but got {k}." + + scale_k = k // sf_vec_size + padded_k = (scale_k + (4 - 1)) // 4 * 4 + padded_k_int32 = padded_k // 4 + padded_m = (m + (128 - 1)) // 128 * 128 + output = torch.empty(l, m, k // 2, device=device, dtype=torch.uint8) + output_scales = torch.empty( + l, padded_m, padded_k_int32, device=device, dtype=torch.int32 + ) + + torch.ops.sgl_kernel.silu_and_mul_scaled_fp4_experts_quant.default( + output.view(l * m, k // 2), + output_scales.view(l * padded_m, padded_k_int32), + input_tensor.view(l * m, k), + input_global_scale, + mask, + use_silu_and_mul=False, + ) + # The physical layout of the output is (l, m, k // 2), but we want to return a + # logical layout (m, k // 2, l) required by the flashinfer masked group gemm. + output = output.permute(1, 2, 0) + # The physical layout of the output scales is already swizzled as (l, rm, rk, 32, 4, 4), a + # requirement for the flashinfer masked group gemm, where rm=m/128 and rk=k/4. The logic + # layout is (32, 4, rm, 4, rk, l). + output_scales = output_scales.view(torch.float8_e4m3fn).view( + l, padded_m // 128, padded_k // 4, 32, 4, 4 + ) + output_scales = output_scales.permute(3, 4, 1, 5, 2, 0) + return output, output_scales + + +def silu_and_mul_scaled_fp4_grouped_quant( + input_tensor: torch.Tensor, + input_global_scale: torch.Tensor, + mask: torch.Tensor, +): + """ + Quantize input tensor to FP4 and return quantized tensor and scale, for + grouped gemm inputs (e.g., grouped_gemm_nt_masked for flashinfer). + Args: + input: The input tensor to be quantized to FP4, with shape (l, m, k * 2) + l is number of groups, m is number of tokens per group, k is number of features. + input_global_scale: A scalar scaling factor for the entire tensor, with + shape (l,). + mask: The mask tensor, with shape (l,) + Outputs: + output: The quantized tensor in FP4, with shape (m, k // 2, l) but the physical + layout is (l, m, k // 2). `// 2` is because two fp4 values are packed into + an uint8. + output_scales: The blockscale tensor in FP8-E4M3, with shape (32, 4, rm, 4, rk, l) + but the physical layout is (l, rm, rk, 32, 4, 4). + Note: + For the shape of output_scales, `32 * 4 * rm` is a padded m to nearest multiple of 128. + `4 * rk` is a padded `k // 16` to nearest multiple of 4. These layout constants are + required by the NVIDIA Blackwell MMA operations. + """ + device = input_tensor.device + l, m, k_by_2 = input_tensor.shape + k = k_by_2 // 2 + sf_vec_size = 16 + assert k % sf_vec_size == 0, f"k must be multiple of 16, but got {k}." + + scale_k = k // sf_vec_size + padded_k = (scale_k + (4 - 1)) // 4 * 4 + padded_k_int32 = padded_k // 4 + padded_m = (m + (128 - 1)) // 128 * 128 + output = torch.empty(l, m, k // 2, device=device, dtype=torch.uint8) + output_scales = torch.empty( + l, padded_m, padded_k_int32, device=device, dtype=torch.int32 + ) + + torch.ops.sgl_kernel.silu_and_mul_scaled_fp4_experts_quant.default( + output.view(l * m, k // 2), + output_scales.view(l * padded_m, padded_k_int32), + input_tensor.view(l * m, k_by_2), + input_global_scale, + mask, + use_silu_and_mul=True, + ) + # The physical layout of the output is (l, m, k // 2), but we want to return a + # logical layout (m, k // 2, l) required by the flashinfer masked group gemm. + output = output.permute(1, 2, 0) + # The physical layout of the output scales is already swizzled as (l, rm, rk, 32, 4, 4), a + # requirement for the flashinfer masked group gemm, where rm=m/128 and rk=k/4. The logic + # layout is (32, 4, rm, 4, rk, l). + output_scales = output_scales.view(torch.float8_e4m3fn).view( + l, padded_m // 128, padded_k // 4, 32, 4, 4 + ) + output_scales = output_scales.permute(3, 4, 1, 5, 2, 0) + return output, output_scales + + +def scaled_fp4_experts_quant( + input_tensor: torch.Tensor, + input_global_scale: torch.Tensor, + expert_offsets: torch.Tensor, + blockscale_offsets: torch.Tensor, + topk: int, + expert_map: Optional[torch.Tensor] = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """ + Quantize input tensor to FP4 and return quantized tensor and scale, for + packed MoE Inputs. + Args: + input: The input tensor to be quantized to FP4 + expert_map: The expert map tensor + input_global_scale: A scalar scaling factor for the entire tensor. + expert_offsets: The expert offsets tensor + blockscale_offsets: The blockscale offsets tensor + Outputs: + output: The quantized tensor in FP4 + output_scales: The blockscale tensor in FP8-E4M3 + """ + assert ( + input_tensor.ndim == 2 + ), f"input.ndim needs to be == 2, but got {input_tensor.ndim}." + if expert_map is not None: + (m, k) = input_tensor.shape + output_tensor_shape = (m * topk, k) + input_tensor = shuffle_rows(input_tensor, expert_map, output_tensor_shape) + m_numtopk, k = input_tensor.shape + # Control the maximum number of tokens per expert supported by the + # NVFP4 MoE Expert Quantization. This is used to prevent the kernel + # from running out of memory. This value can also be increased to support + # larger models. + import os + + MAX_TOKENS_PER_EXPERT = int(os.environ.get("MODELOPT_MAX_TOKENS_PER_EXPERT", 65536)) + assert m_numtopk <= MAX_TOKENS_PER_EXPERT * topk, ( + f"m_numtopk must be less than MAX_TOKENS_PER_EXPERT(" + f"{MAX_TOKENS_PER_EXPERT})" + f" for cutlass_moe_fp4, observed m_numtopk = {m_numtopk}. Use" + f" MODELOPT_MAX_TOKENS_PER_EXPERT to set this value." + ) + scales_k = k // 16 + padded_k = (scales_k + (4 - 1)) // 4 + + # output is uint8 and packed fp4 values + output = torch.empty( + m_numtopk, k // 2, device=input_tensor.device, dtype=torch.uint8 + ) + # padded part should be zeroed out + if padded_k > scales_k: + output_scales = torch.zeros( + MAX_TOKENS_PER_EXPERT * topk, + padded_k, + dtype=torch.int32, + device=input_tensor.device, + ) + else: + output_scales = torch.empty( + MAX_TOKENS_PER_EXPERT * topk, + padded_k, + dtype=torch.int32, + device=input_tensor.device, + ) + torch.ops.sgl_kernel.scaled_fp4_experts_quant.default( + output, + output_scales, + input_tensor, + input_global_scale, + expert_offsets, + blockscale_offsets, + ) + output_scales = output_scales.view(torch.float8_e4m3fn) + return output, output_scales + + +# GPTQ kernels +def gptq_marlin_gemm( + a: torch.Tensor, + c: Optional[torch.Tensor], + b_q_weight: torch.Tensor, + b_scales: torch.Tensor, + global_scale: Optional[torch.Tensor], + b_zeros: Optional[torch.Tensor], + g_idx: Optional[torch.Tensor], + perm: Optional[torch.Tensor], + workspace: torch.Tensor, + b_q_type: ScalarType, + size_m: int, + size_n: int, + size_k: int, + is_k_full: bool = True, + use_atomic_add: bool = False, + use_fp32_reduce: bool = False, + is_zp_float: bool = False, +) -> torch.Tensor: + return torch.ops.sgl_kernel.gptq_marlin_gemm( + a, + c, + b_q_weight, + b_scales, + global_scale, + b_zeros, + g_idx, + perm, + workspace, + b_q_type.id, + size_m, + size_n, + size_k, + is_k_full, + use_atomic_add, + use_fp32_reduce, + is_zp_float, + ) + + +def gptq_gemm( + a: torch.Tensor, + b_q_weight: torch.Tensor, + b_gptq_qzeros: torch.Tensor, + b_gptq_scales: torch.Tensor, + b_g_idx: torch.Tensor, + use_shuffle: bool, + bit: int, +) -> torch.Tensor: + return torch.ops.sgl_kernel.gptq_gemm( + a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx, use_shuffle, bit + ) + + +def gptq_shuffle(q_weight: torch.Tensor, q_perm: torch.Tensor, bit: int) -> None: + torch.torch.ops.sgl_kernel.gptq_shuffle(q_weight, q_perm, bit) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/grammar.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/grammar.py new file mode 100644 index 00000000..971f94bc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/grammar.py @@ -0,0 +1,15 @@ +from typing import List, Optional, Union + +import torch + + +def apply_token_bitmask_inplace_cuda( + logits: torch.Tensor, + bitmask: torch.Tensor, + indices: Optional[Union[List[int], torch.Tensor]] = None, +) -> None: + if isinstance(indices, list): + indices = torch.tensor(indices, dtype=torch.int32, device=logits.device) + if indices is not None: + indices = indices.to(logits.device) + torch.ops.sgl_kernel.apply_token_bitmask_inplace_cuda(logits, bitmask, indices) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/hadamard.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/hadamard.py new file mode 100644 index 00000000..102c540f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/hadamard.py @@ -0,0 +1,21 @@ +import torch + + +def hadamard_transform(x: torch.Tensor, scale: float = 1.0) -> torch.Tensor: + return torch.ops.sgl_kernel.fast_hadamard_transform.default(x, scale) + + +def hadamard_transform_12n(x: torch.Tensor, scale: float = 1.0) -> torch.Tensor: + return torch.ops.sgl_kernel.fast_hadamard_transform_12N.default(x, scale) + + +def hadamard_transform_20n(x: torch.Tensor, scale: float = 1.0) -> torch.Tensor: + return torch.ops.sgl_kernel.fast_hadamard_transform_20N.default(x, scale) + + +def hadamard_transform_28n(x: torch.Tensor, scale: float = 1.0) -> torch.Tensor: + return torch.ops.sgl_kernel.fast_hadamard_transform_28N.default(x, scale) + + +def hadamard_transform_40n(x: torch.Tensor, scale: float = 1.0) -> torch.Tensor: + return torch.ops.sgl_kernel.fast_hadamard_transform_40N.default(x, scale) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/kvcacheio.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/kvcacheio.py new file mode 100644 index 00000000..908a3efa --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/kvcacheio.py @@ -0,0 +1,307 @@ +from typing import List + +import torch + + +def is_hip() -> bool: + return torch.version.hip is not None + + +_is_hip = is_hip() + + +def transfer_kv_per_layer( + src_k: torch.Tensor, + dst_k: torch.Tensor, + src_v: torch.Tensor, + dst_v: torch.Tensor, + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + item_size: int, + block_quota: int = 2, + num_warps_per_block: int = 16 if _is_hip else 32, +): + torch.ops.sgl_kernel.transfer_kv_per_layer.default( + src_k, + dst_k, + src_v, + dst_v, + src_indices, + dst_indices, + item_size, + block_quota, + num_warps_per_block, + ) + + +def transfer_kv_per_layer_pf_lf( + src_k: torch.Tensor, + dst_k: torch.Tensor, + src_v: torch.Tensor, + dst_v: torch.Tensor, + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + layer_id: int, + item_size: int, + src_layout_dim: int, + block_quota: int = 2, + num_warps_per_block: int = 16 if _is_hip else 32, +): + torch.ops.sgl_kernel.transfer_kv_per_layer_pf_lf.default( + src_k, + dst_k, + src_v, + dst_v, + src_indices, + dst_indices, + layer_id, + item_size, + src_layout_dim, + block_quota, + num_warps_per_block, + ) + + +def transfer_kv_per_layer_ph_lf( + src_k: torch.Tensor, + dst_k: torch.Tensor, + src_v: torch.Tensor, + dst_v: torch.Tensor, + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + layer_id: int, + item_size: int, + src_layout_dim: int, + page_size: int, + head_num: int, + block_quota: int = 2, + num_warps_per_block: int = 16 if _is_hip else 32, +): + torch.ops.sgl_kernel.transfer_kv_per_layer_ph_lf.default( + src_k, + dst_k, + src_v, + dst_v, + src_indices, + dst_indices, + layer_id, + item_size, + src_layout_dim, + page_size, + head_num, + block_quota, + num_warps_per_block, + ) + + +def transfer_kv_all_layer( + src_k_layers: torch.Tensor, + dst_k_layers: torch.Tensor, + src_v_layers: torch.Tensor, + dst_v_layers: torch.Tensor, + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + item_size: int, + num_layers: int, + block_quota: int = 2, + num_warps_per_block: int = 16 if _is_hip else 32, +): + torch.ops.sgl_kernel.transfer_kv_all_layer.default( + src_k_layers, + dst_k_layers, + src_v_layers, + dst_v_layers, + src_indices, + dst_indices, + item_size, + num_layers, + block_quota, + num_warps_per_block, + ) + + +def transfer_kv_all_layer_lf_pf( + src_k_layers: torch.Tensor, + dst_k: torch.Tensor, + src_v_layers: torch.Tensor, + dst_v: torch.Tensor, + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + item_size: int, + dst_layout_dim: int, + num_layers: int, + block_quota: int = 2, + num_warps_per_block: int = 16 if _is_hip else 32, +): + torch.ops.sgl_kernel.transfer_kv_all_layer_lf_pf.default( + src_k_layers, + dst_k, + src_v_layers, + dst_v, + src_indices, + dst_indices, + item_size, + dst_layout_dim, + num_layers, + block_quota, + num_warps_per_block, + ) + + +def transfer_kv_all_layer_lf_ph( + src_k_layers: torch.Tensor, + dst_k: torch.Tensor, + src_v_layers: torch.Tensor, + dst_v: torch.Tensor, + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + item_size: int, + dst_layout_dim: int, + num_layers: int, + page_size: int, + head_num: int, + block_quota: int = 2, + num_warps_per_block: int = 16 if _is_hip else 32, +): + torch.ops.sgl_kernel.transfer_kv_all_layer_lf_ph.default( + src_k_layers, + dst_k, + src_v_layers, + dst_v, + src_indices, + dst_indices, + item_size, + dst_layout_dim, + num_layers, + page_size, + head_num, + block_quota, + num_warps_per_block, + ) + + +def transfer_kv_direct( + src_layers: List[torch.Tensor], + dst_layers: List[torch.Tensor], + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + page_size: int, +): + torch.ops.sgl_kernel.transfer_kv_direct.default( + src_layers, dst_layers, src_indices, dst_indices, page_size + ) + + +def transfer_kv_per_layer_direct_pf_lf( + src_ptrs: List[torch.Tensor], + dst_ptrs: List[torch.Tensor], + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + layer_id: int, + page_size: int, +): + torch.ops.sgl_kernel.transfer_kv_per_layer_direct_pf_lf.default( + src_ptrs, dst_ptrs, src_indices, dst_indices, layer_id, page_size + ) + + +def transfer_kv_all_layer_direct_lf_pf( + src_ptrs: List[torch.Tensor], + dst_ptrs: List[torch.Tensor], + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + page_size: int, +): + torch.ops.sgl_kernel.transfer_kv_all_layer_direct_lf_pf.default( + src_ptrs, dst_ptrs, src_indices, dst_indices, page_size + ) + + +def transfer_kv_per_layer_mla( + src: torch.Tensor, + dst: torch.Tensor, + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + item_size: int, + block_quota: int = 2, + num_warps_per_block: int = 16 if _is_hip else 32, +): + torch.ops.sgl_kernel.transfer_kv_per_layer_mla.default( + src, + dst, + src_indices, + dst_indices, + item_size, + block_quota, + num_warps_per_block, + ) + + +def transfer_kv_per_layer_mla_pf_lf( + src: torch.Tensor, + dst: torch.Tensor, + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + layer_id: int, + item_size: int, + src_layout_dim: int, + block_quota: int = 2, + num_warps_per_block: int = 16 if _is_hip else 32, +): + torch.ops.sgl_kernel.transfer_kv_per_layer_mla_pf_lf.default( + src, + dst, + src_indices, + dst_indices, + layer_id, + item_size, + src_layout_dim, + block_quota, + num_warps_per_block, + ) + + +def transfer_kv_all_layer_mla( + src_layers: torch.Tensor, + dst_layers: torch.Tensor, + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + item_size: int, + num_layers: int, + block_quota: int = 2, + num_warps_per_block: int = 16 if _is_hip else 32, +): + torch.ops.sgl_kernel.transfer_kv_all_layer_mla.default( + src_layers, + dst_layers, + src_indices, + dst_indices, + item_size, + num_layers, + block_quota, + num_warps_per_block, + ) + + +def transfer_kv_all_layer_mla_lf_pf( + src_layers: torch.Tensor, + dst: torch.Tensor, + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + item_size: int, + dst_layout_dim: int, + num_layers: int, + block_quota: int = 2, + num_warps_per_block: int = 16 if _is_hip else 32, +): + torch.ops.sgl_kernel.transfer_kv_all_layer_mla_lf_pf.default( + src_layers, + dst, + src_indices, + dst_indices, + item_size, + dst_layout_dim, + num_layers, + block_quota, + num_warps_per_block, + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/load_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/load_utils.py new file mode 100644 index 00000000..d0b18d3f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/load_utils.py @@ -0,0 +1,232 @@ +import ctypes +import glob +import importlib.util +import logging +import os +import shutil +from pathlib import Path +from typing import List + +import torch + +logger = logging.getLogger(__name__) + + +def _get_compute_capability(): + """Get the compute capability of the current GPU.""" + if not torch.cuda.is_available(): + return None + + # Get the current device + device = torch.cuda.current_device() + properties = torch.cuda.get_device_properties(device) + + # Return as integer (major * 10 + minor) + return properties.major * 10 + properties.minor + + +def _filter_compiled_extensions(file_list): + """Filter and prioritize compiled extensions over Python source files.""" + compiled_extensions = [".so", ".pyd", ".dll"] # Common compiled extension suffixes + compiled_files = [] + other_files = [] + + for file_path in file_list: + path = Path(file_path) + # Check if it's a compiled extension (including complex names like .abi3.so, .cpython-312.so) + if any( + str(path).endswith(ext) or ext in str(path) for ext in compiled_extensions + ): + compiled_files.append(file_path) + else: + other_files.append(file_path) + + # Return compiled files first, then others + return compiled_files + other_files + + +def _load_architecture_specific_ops(): + """Load the appropriate common_ops library based on GPU architecture.""" + compute_capability = _get_compute_capability() + logger.debug( + f"[sgl_kernel] GPU Detection: compute_capability = {compute_capability}" + ) + + # Get the directory where sgl_kernel is installed + sgl_kernel_dir = Path(__file__).parent + logger.debug(f"[sgl_kernel] sgl_kernel directory: {sgl_kernel_dir}") + + # Determine which version to load based on GPU architecture + if compute_capability == 90: + ops_subdir = "sm90" + variant_name = "SM90 (Hopper/H100 with fast math optimization)" + elif compute_capability is not None: + ops_subdir = "sm100" + variant_name = f"SM{compute_capability} (precise math for compatibility)" + else: + ops_subdir = "sm100" + variant_name = "CPU/No GPU detected (using precise math)" + + # Look for the compiled module with any valid extension + + ops_pattern = str(sgl_kernel_dir / ops_subdir / "common_ops.*") + raw_matching_files = glob.glob(ops_pattern) + matching_files = _filter_compiled_extensions(raw_matching_files) + + logger.debug(f"[sgl_kernel] Attempting to load {variant_name}") + logger.debug(f"[sgl_kernel] Looking for library matching pattern: {ops_pattern}") + logger.debug(f"[sgl_kernel] Found files: {raw_matching_files}") + logger.debug(f"[sgl_kernel] Prioritized files: {matching_files}") + + previous_import_errors: List[Exception] = [] + + # Try to load from the architecture-specific directory + if matching_files: + ops_path = Path(matching_files[0]) # Use the first prioritized file + logger.debug(f"[sgl_kernel] Found architecture-specific library: {ops_path}") + try: + # Load the module from specific path using importlib + spec = importlib.util.spec_from_file_location("common_ops", str(ops_path)) + if spec is None: + raise ImportError(f"Could not create module spec for {ops_path}") + + common_ops = importlib.util.module_from_spec(spec) + if spec.loader is None: + raise ImportError(f"Module spec has no loader for {ops_path}") + + logger.debug(f"[sgl_kernel] Loading module from {ops_path}...") + spec.loader.exec_module(common_ops) + logger.debug(f"[sgl_kernel] ✓ Successfully loaded {variant_name}") + logger.debug(f"[sgl_kernel] ✓ Module file: {common_ops.__file__}") + return common_ops + + except Exception as e: + previous_import_errors.append(e) + logger.debug( + f"[sgl_kernel] ✗ Failed to load from {ops_path}: {type(e).__name__}: {e}" + ) + # Continue to fallback + else: + logger.debug( + f"[sgl_kernel] ✗ Architecture-specific library not found matching pattern: {ops_pattern}" + ) + + # Try alternative directory (in case installation structure differs) + alt_pattern = str(sgl_kernel_dir / "common_ops.*") + raw_alt_files = glob.glob(alt_pattern) + alt_matching_files = _filter_compiled_extensions(raw_alt_files) + logger.debug(f"[sgl_kernel] Attempting fallback: looking for pattern {alt_pattern}") + logger.debug(f"[sgl_kernel] Found fallback files: {raw_alt_files}") + logger.debug(f"[sgl_kernel] Prioritized fallback files: {alt_matching_files}") + + if alt_matching_files: + alt_path = Path(alt_matching_files[0]) # Use the first prioritized file + logger.debug(f"[sgl_kernel] Found fallback library: {alt_path}") + try: + spec = importlib.util.spec_from_file_location("common_ops", str(alt_path)) + if spec is None: + raise ImportError(f"Could not create module spec for {alt_path}") + + common_ops = importlib.util.module_from_spec(spec) + if spec.loader is None: + raise ImportError(f"Module spec has no loader for {alt_path}") + + logger.debug(f"[sgl_kernel] Loading fallback module from {alt_path}...") + spec.loader.exec_module(common_ops) + logger.debug(f"[sgl_kernel] ✓ Successfully loaded fallback library") + logger.debug(f"[sgl_kernel] ✓ Module file: {common_ops.__file__}") + return common_ops + + except Exception as e: + previous_import_errors.append(e) + logger.debug( + f"[sgl_kernel] ✗ Failed to load fallback from {alt_path}: {type(e).__name__}: {e}" + ) + else: + logger.debug( + f"[sgl_kernel] ✗ Fallback library not found matching pattern: {alt_pattern}" + ) + + # Final attempt: try standard Python import (for backward compatibility) + logger.debug( + f"[sgl_kernel] Final attempt: trying standard Python import 'common_ops'" + ) + try: + import common_ops + + logger.debug(f"[sgl_kernel] ✓ Successfully imported via standard Python import") + logger.debug(f"[sgl_kernel] ✓ Module file: {common_ops.__file__}") + return common_ops + except ImportError as e: + previous_import_errors.append(e) + logger.debug(f"[sgl_kernel] ✗ Standard Python import failed: {e}") + + attempt_error_msg = "\n".join( + f"- {type(err).__name__}: {err}" for err in previous_import_errors + ) + + # All attempts failed + error_msg = f""" +[sgl_kernel] CRITICAL: Could not load any common_ops library! + +Attempted locations: +1. Architecture-specific pattern: {ops_pattern} - found files: {matching_files} +2. Fallback pattern: {alt_pattern} - found files: {alt_matching_files} +3. Standard Python import: common_ops - failed + +GPU Info: +- Compute capability: {compute_capability} +- Expected variant: {variant_name} + +Please ensure sgl_kernel is properly installed with: +pip install --upgrade sgl_kernel + +Error details from previous import attempts: +{attempt_error_msg} +""" + logger.debug(error_msg) + raise ImportError(error_msg) + + +# copy & modify from torch/utils/cpp_extension.py +def _find_cuda_home(): + """Find the CUDA install path.""" + # Guess #1 + cuda_home = os.environ.get("CUDA_HOME") or os.environ.get("CUDA_PATH") + if cuda_home is None: + # Guess #2 + nvcc_path = shutil.which("nvcc") + if nvcc_path is not None: + cuda_home = os.path.dirname(os.path.dirname(nvcc_path)) + else: + # Guess #3 + cuda_home = "/usr/local/cuda" + return cuda_home + + +def _preload_cuda_library(): + """Preload the CUDA runtime library to help avoid 'libcudart.so.12 not found' issues.""" + cuda_home = Path(_find_cuda_home()) + + candidate_dirs = [ + cuda_home / "lib", + cuda_home / "lib64", + Path("/usr/lib/x86_64-linux-gnu"), + Path("/usr/lib/aarch64-linux-gnu"), + Path("/usr/lib64"), + Path("/usr/lib"), + ] + + for base in candidate_dirs: + candidate = base / "libcudart.so.12" + if candidate.exists(): + try: + cuda_runtime_lib = candidate.resolve() + ctypes.CDLL(str(cuda_runtime_lib), mode=ctypes.RTLD_GLOBAL) + logger.debug(f"Preloaded CUDA runtime under {cuda_runtime_lib}") + return + except Exception as e: + logger.debug(f"Failed to load {candidate}: {e}") + continue + + logger.debug("[sgl_kernel] Could not preload CUDA runtime library") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/mamba.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/mamba.py new file mode 100644 index 00000000..85aa5b94 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/mamba.py @@ -0,0 +1,50 @@ +from typing import Optional + +import torch + + +# mamba +def causal_conv1d_fwd( + x: torch.Tensor, + weight: torch.Tensor, + bias_: Optional[torch.Tensor], + conv_states: Optional[torch.Tensor], + query_start_loc: Optional[torch.Tensor], + cache_indices: Optional[torch.Tensor], + has_initial_state: Optional[torch.Tensor], + silu_activation: bool, + pad_slot_id: int, +): + torch.ops.sgl_kernel.causal_conv1d_fwd( + x, + weight, + bias_, + conv_states, + query_start_loc, + cache_indices, + has_initial_state, + silu_activation, + pad_slot_id, + ) + + +def causal_conv1d_update( + x: torch.Tensor, + conv_state: torch.Tensor, + weight: torch.Tensor, + bias_: Optional[torch.Tensor], + silu_activation: bool, + cache_seqlens: Optional[torch.Tensor], + conv_state_indices: Optional[torch.Tensor], + pad_slot_id: int, +): + torch.ops.sgl_kernel.causal_conv1d_update( + x, + conv_state, + weight, + bias_, + silu_activation, + cache_seqlens, + conv_state_indices, + pad_slot_id, + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/marlin.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/marlin.py new file mode 100644 index 00000000..823d3fd4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/marlin.py @@ -0,0 +1,44 @@ +import torch + + +def gptq_marlin_repack( + b_q_weight, + perm, + size_k, + size_n, + num_bits, +) -> torch.Tensor: + return torch.ops.sgl_kernel.gptq_marlin_repack( + b_q_weight, + perm, + size_k, + size_n, + num_bits, + ) + + +def awq_marlin_repack( + b_q_weight: torch.Tensor, size_k: int, size_n: int, num_bits: int +) -> torch.Tensor: + return torch.ops.sgl_kernel.awq_marlin_repack(b_q_weight, size_k, size_n, num_bits) + + +def awq_marlin_moe_repack( + b_q_weight: torch.Tensor, + perm: torch.Tensor, + size_k: int, + size_n: int, + num_bits: int, +) -> torch.Tensor: + num_experts = b_q_weight.shape[0] + assert size_k % 16 == 0 + output = torch.empty( + (num_experts, size_k // 16, size_n * (num_bits // 2)), + device=b_q_weight.device, + dtype=b_q_weight.dtype, + ) + for e in range(num_experts): + output[e] = torch.ops.sgl_kernel.awq_marlin_repack( + b_q_weight[e], size_k, size_n, num_bits + ) + return output diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/memory.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/memory.py new file mode 100644 index 00000000..9ff4f957 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/memory.py @@ -0,0 +1,26 @@ +import torch + + +def set_kv_buffer_kernel( + k_cache: torch.Tensor, + v_cache: torch.Tensor, + loc: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + fallback: bool = False, +): + try: + if fallback: + raise RuntimeError("Fallback to torch implementation") + torch.ops.sgl_kernel.store_kv_cache(k_cache, v_cache, loc, k, v) + except RuntimeError: # ok, fallback to torch implementation + k_cache[loc] = k + v_cache[loc] = v + + +def weak_ref_tensor(tensor): + return ( + torch.ops.sgl_kernel.weak_ref_tensor(tensor) + if isinstance(tensor, torch.Tensor) + else tensor + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/moe.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/moe.py new file mode 100755 index 00000000..42c20ca5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/moe.py @@ -0,0 +1,334 @@ +from typing import Any, Dict, Optional + +import torch + + +def moe_align_block_size( + topk_ids, + num_experts, + block_size, + sorted_token_ids, + experts_ids, + num_tokens_post_pad, + cumsum_buffer, + pad_sorted_token_ids=False, +): + torch.ops.sgl_kernel.moe_align_block_size.default( + topk_ids, + num_experts, + block_size, + sorted_token_ids, + experts_ids, + num_tokens_post_pad, + cumsum_buffer, + pad_sorted_token_ids, + ) + + +def topk_softmax( + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + gating_output: torch.Tensor, + renormalize: bool = False, + moe_softcapping: float = 0.0, + correction_bias: Optional[torch.Tensor] = None, +) -> None: + """ + Compute top-k softmax for MoE routing. + + Args: + topk_weights: Output tensor for top-k weights [num_tokens, topk] + topk_ids: Output tensor for top-k expert indices [num_tokens, topk] + gating_output: Gating logits [num_tokens, num_experts] + renormalize: Whether to renormalize the top-k weights + moe_softcapping: Tanh softcapping value (0.0 to disable) + correction_bias: Per-expert bias correction [num_experts], must be float32 if provided + """ + torch.ops.sgl_kernel.topk_softmax.default( + topk_weights, + topk_ids, + gating_output, + renormalize, + moe_softcapping, + correction_bias, + ) + + +def topk_sigmoid( + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + gating_output: torch.Tensor, + renormalize: bool = False, + correction_bias: Optional[torch.Tensor] = None, +) -> None: + """ + Compute top-k sigmoid for MoE routing. + + Args: + topk_weights: Output tensor for top-k weights [num_tokens, topk] + topk_ids: Output tensor for top-k expert indices [num_tokens, topk] + gating_output: Gating logits [num_tokens, num_experts] + renormalize: Whether to renormalize the top-k weights + correction_bias: Per-expert bias correction [num_experts], must be float32 if provided + """ + torch.ops.sgl_kernel.topk_sigmoid.default( + topk_weights, + topk_ids, + gating_output, + renormalize, + correction_bias, + ) + + +def moe_sum_reduce( + input_tensor, + output_tensor, + routed_scaling_factor=0, +): + torch.ops.sgl_kernel.moe_sum_reduce.default( + input_tensor, + output_tensor, + routed_scaling_factor, + ) + + +def moe_sum( + input_tensor: torch.Tensor, + output_tensor: torch.Tensor, +): + torch.ops.sgl_kernel.moe_sum.default( + input_tensor, + output_tensor, + ) + + +def moe_fused_gate( + input_tensor, + bias, + num_expert_group, + topk_group, + topk, + num_fused_shared_experts=0, + routed_scaling_factor=0, + apply_routed_scaling_factor_on_output=False, +): + # This fused kernel function is used to select topk expert in a hierarchical 2-layer fashion + # it split group of expert into num_expert_group, and use top2 expert weight sum in each group + # as the group weight to select expert groups and then select topk experts within the selected groups + # the #experts is decided by the input tensor shape and we currently only support power of 2 #experts + # and #experts should be divisible by num_expert_group. #expert/num_expert_group <= 32 is limited for now. + # for non-supported case, we suggest to use the biased_grouped_topk func in sglang.srt.layers.moe.topk + # num_fused_shared_experts: if > 0, the last several experts will be + # replaced with shared experts. the shared experts will be divided by the + # routed_scaling_factor - this is intended to cancel out later when routed+shared + # output is scaled so that shared experts are not scaled. + # routed_scaling_factor: if > 0, the experts will be scaled by this factor + # apply_routed_scaling_factor_on_output: if true, output will be + # scaled by the routed_scaling_factor + return torch.ops.sgl_kernel.moe_fused_gate.default( + input_tensor, + bias, + num_expert_group, + topk_group, + topk, + num_fused_shared_experts, + routed_scaling_factor, + apply_routed_scaling_factor_on_output, + ) + + +def kimi_k2_moe_fused_gate( + input_tensor, + bias, + topk, + renormalize=True, + routed_scaling_factor=1.0, + apply_routed_scaling_factor_on_output=False, +): + """ + Simplified fused kernel for Kimi K2 model (num_expert_group=1). + This kernel removes the grouped topk logic since all experts belong to a single group. + + Args: + input_tensor: Gating output tensor [num_tokens, num_experts] + bias: Correction bias tensor [num_experts] + topk: Number of experts to select per token + renormalize: Whether to renormalize the topk weights + routed_scaling_factor: Scaling factor for expert weights + apply_routed_scaling_factor_on_output: If true, apply scaling factor to output + + Returns: + Tuple of (topk_weights, topk_ids) + - topk_weights: [num_tokens, topk] float32 tensor + - topk_ids: [num_tokens, topk] int32 tensor + """ + return torch.ops.sgl_kernel.kimi_k2_moe_fused_gate.default( + input_tensor, + bias, + topk, + renormalize, + routed_scaling_factor, + apply_routed_scaling_factor_on_output, + ) + + +def fp8_blockwise_scaled_grouped_mm( + output, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a, + b, + scales_a, + scales_b, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets, + workspace, +): + torch.ops.sgl_kernel.fp8_blockwise_scaled_grouped_mm.default( + output, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a, + b, + scales_a, + scales_b, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets, + workspace, + ) + + +def prepare_moe_input( + topk_ids, + expert_offsets, + problem_sizes1, + problem_sizes2, + input_permutation, + output_permutation, + num_experts, + n, + k, + blockscale_offsets: Optional[torch.Tensor] = None, +): + torch.ops.sgl_kernel.prepare_moe_input.default( + topk_ids, + expert_offsets, + blockscale_offsets, + problem_sizes1, + problem_sizes2, + input_permutation, + output_permutation, + num_experts, + n, + k, + ) + + +def apply_shuffle_mul_sum( + input, + output, + permutation, + factors, +): + torch.ops.sgl_kernel.apply_shuffle_mul_sum.default( + input, output, permutation, factors + ) + + +def fused_qk_norm_rope( + qkv: torch.Tensor, + num_heads_q: int, + num_heads_k: int, + num_heads_v: int, + head_dim: int, + eps: float, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + base: float, + is_neox: bool, + position_ids: torch.Tensor, + factor: float, + low: float, + high: float, + attention_factor: float, +) -> None: + torch.ops.sgl_kernel.fused_qk_norm_rope( + qkv, + num_heads_q, + num_heads_k, + num_heads_v, + head_dim, + eps, + q_weight, + k_weight, + base, + is_neox, + position_ids, + factor, + low, + high, + attention_factor, + ) + + +def cutlass_fp4_group_mm( + a_fp4, + b_fp4, + a_blockscale, + b_blockscale, + alphas, + out_dtype, + device, + params: Dict[str, Any], +): + """ + An FP4 Blockscaled Group Gemm that takes in a_tensors, b_tensors and runs + the gemms for each combination based on the specified problem sizes. + + This is used as the MoE gemm during NVFP4 Quantized FusedMoE forward. + - a/b_tensors: the NVFP4 a_ptrs and b_ptrs tensors which are quantized + input and expert weights. + - a_/b_scales: The blockscales in FP8-E4M3 precision + - ab_strides/c_strides: Strides for the a/b tensors between rows. + - expert_offsets/sf_offsets: Indices that mark at which token index + each expert begins its computation. The number of tokens + computed with expert E is expert_offsets[E + 1] - + expert_offsets[E] And the sf_size per expert is + sf_offset[E+1] - sf_offset[E] + - problem_sizes: MxNxK sizes of each expert's multiplication in two grouped + MMs used in the fused MoE operation. + """ + m_topk = a_fp4.shape[0] + n = b_fp4.shape[1] + c_shape = (m_topk, n) + c = torch.empty(c_shape, device=device, dtype=out_dtype) + torch.ops.sgl_kernel.cutlass_fp4_group_mm.default( + c, + a_fp4, + b_fp4, + a_blockscale, + b_blockscale, + alphas, + params["ab_strides"], + params["c_strides"], + params["problem_sizes"], + params["expert_offsets"], + params["blockscale_offsets"], + ) + return c.to(dtype=out_dtype) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/quantization/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/quantization/__init__.py new file mode 100644 index 00000000..0ad12c64 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/quantization/__init__.py @@ -0,0 +1,8 @@ +from .gguf import ( + ggml_dequantize, + ggml_moe_a8, + ggml_moe_a8_vec, + ggml_moe_get_block_size, + ggml_mul_mat_a8, + ggml_mul_mat_vec_a8, +) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/quantization/gguf.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/quantization/gguf.py new file mode 100644 index 00000000..994bc21d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/quantization/gguf.py @@ -0,0 +1,62 @@ +import torch + + +def ggml_dequantize( + weight: torch.Tensor, quant_type: int, M: int, N: int, dtype: torch.dtype +): + assert M > 0 and N > 0, "GGUF weight Input shape must be of positive dimensions" + return torch.ops.sgl_kernel.ggml_dequantize.default(weight, quant_type, M, N, dtype) + + +def ggml_mul_mat_vec_a8( + weight: torch.Tensor, x: torch.Tensor, quant_type: int, row: int +) -> torch.Tensor: + return torch.ops.sgl_kernel.ggml_mul_mat_vec_a8.default(weight, x, quant_type, row) + + +def ggml_mul_mat_a8( + weight: torch.Tensor, x: torch.Tensor, quant_type: int, row: int +) -> torch.Tensor: + return torch.ops.sgl_kernel.ggml_mul_mat_a8.default(weight, x, quant_type, row) + + +def ggml_moe_a8( + input: torch.Tensor, + weight: torch.Tensor, + sorted_token_ids: torch.Tensor, + expert_ids: torch.Tensor, + num_token_post_padded: torch.Tensor, + type: int, + row: int, + topk: int, + tokens: int, +) -> torch.Tensor: + return torch.ops.sgl_kernel.ggml_moe_a8.default( + input, + weight, + sorted_token_ids, + expert_ids, + num_token_post_padded, + type, + row, + topk, + tokens, + ) + + +def ggml_moe_a8_vec( + input: torch.Tensor, + weight: torch.Tensor, + topk_ids: torch.Tensor, + top_k: int, + type: int, + row: int, + tokens: int, +) -> torch.Tensor: + return torch.ops.sgl_kernel.ggml_moe_a8_vec.default( + input, weight, topk_ids, top_k, type, row, tokens + ) + + +def ggml_moe_get_block_size(type: int) -> int: + return torch.ops.sgl_kernel.ggml_moe_get_block_size.default(type) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/sampling.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/sampling.py new file mode 100644 index 00000000..4ee6f24d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/sampling.py @@ -0,0 +1,543 @@ +from typing import Optional, Union + +import torch +from sgl_kernel.utils import _to_tensor_scalar_tuple + + +def _top_k_renorm_probs_internal( + probs: torch.Tensor, + maybe_top_k_arr: Optional[torch.Tensor], + top_k_val: int, +) -> torch.Tensor: + probs = probs.float() + maybe_top_k_arr = maybe_top_k_arr.int() if maybe_top_k_arr is not None else None + renorm_probs = torch.empty_like(probs) + torch.ops.sgl_kernel.top_k_renorm_probs.default( + probs, renorm_probs, maybe_top_k_arr, top_k_val + ) + return renorm_probs + + +def top_k_renorm_probs( + probs: torch.Tensor, + top_k: Union[torch.Tensor, int], +) -> torch.Tensor: + r"""Adapt from https://github.com/flashinfer-ai/flashinfer/flashinfer/sampling.py + Fused GPU kernel for renormalizing probabilities by top-k thresholding. + + Parameters + ---------- + probs: torch.Tensor + Probabilities, shape ``(batch_size, num_classes)``. + top_k: Union[torch.Tensor, int] + Either a scalar or a tensor of shape ``(batch_size,)``, representing the top-k threshold for for + for re-normalizing probabilities, should be in ``(0, num_classes)``. + If a scalar, the same threshold is used for all requests. + If a tensor, each request has its own threshold. + We keep the top-k probabilities, set the rest to zero, and renormalize the probabilities. + + Returns + ------- + renorm_probs: torch.Tensor + Renormalized probabilities, shape ``(batch_size, num_classes)``. + + Note + ---- + This combination of ``top_k_renorm_probs`` and ``sampling_from_probs`` should be equivalent to + ``top_k_sampling_from_probs``. + """ + return _top_k_renorm_probs_internal(probs, *_to_tensor_scalar_tuple(top_k)) + + +top_k_renorm_prob = top_k_renorm_probs + + +def _top_p_renorm_probs_internal( + probs: torch.Tensor, + maybe_top_p_arr: Optional[torch.Tensor], + top_p_val: float, +) -> torch.Tensor: + probs = probs.float() + maybe_top_p_arr = maybe_top_p_arr.float() if maybe_top_p_arr is not None else None + renorm_probs = torch.empty_like(probs) + torch.ops.sgl_kernel.top_p_renorm_probs.default( + probs, renorm_probs, maybe_top_p_arr, top_p_val + ) + return renorm_probs + + +def top_p_renorm_probs( + probs: torch.Tensor, + top_p: Union[torch.Tensor, float], +) -> torch.Tensor: + r"""Adapt from https://github.com/flashinfer-ai/flashinfer/flashinfer/sampling.py + Fused GPU kernel for renormalizing probabilities by top-p thresholding. + + Parameters + ---------- + probs: torch.Tensor + Probabilities, shape ``(batch_size, num_classes)``. + top_p: Union[torch.Tensor, float] + Either a scalar or a tensor of shape ``(batch_size,)``, representing the top-p threshold for for + re-normalizing probabilities, should be in ``(0, 1)``. + If a scalar, the same threshold is used for all requests. + If a tensor, each request has its own threshold. + We mask out the probabilities less than `threshold` where the cumulative sum + of ``probs[probs >= threshold]`` is `top_p`, and renormalize the probabilities. + + Returns + ------- + renorm_probs: torch.Tensor + Renormalized probabilities, shape ``(batch_size, num_classes)``. + + Note + ---- + This combination of ``top_p_renorm_probs`` and ``sampling_from_probs`` should be equivalent to + ``top_p_sampling_from_probs``. + + """ + return _top_p_renorm_probs_internal(probs, *_to_tensor_scalar_tuple(top_p)) + + +top_p_renorm_prob = top_p_renorm_probs + + +def _top_p_sampling_from_probs_internal( + probs: torch.Tensor, + indices: Optional[torch.Tensor], + maybe_top_p_arr: Optional[torch.Tensor], + top_p_val: float, + deterministic: bool, + generator: Optional[torch.Generator], +) -> torch.Tensor: + with probs.device as device: + probs = probs.float() + maybe_top_p_arr = ( + maybe_top_p_arr.float() if maybe_top_p_arr is not None else None + ) + samples = torch.empty(probs.size(0), dtype=torch.int32, device=device) + torch.ops.sgl_kernel.top_p_sampling_from_probs.default( + probs, + samples, + indices, + maybe_top_p_arr, + top_p_val, + deterministic, + generator, + ) + return samples + + +def top_p_sampling_from_probs( + probs: torch.Tensor, + top_p: Union[torch.Tensor, float], + indices: Optional[torch.Tensor] = None, + deterministic: bool = True, + generator: Optional[torch.Generator] = None, + check_nan: bool = False, +) -> torch.Tensor: + r"""Adapt from https://github.com/flashinfer-ai/flashinfer/flashinfer/sampling.py + Fused GPU kernel for top-p sampling (nucleus sampling) from probabilities, + this operator implements GPU-based rejection sampling without explicit sorting. + Check the `blog post `_ for more details. + + The multiple rounds of rejection sampling are implemented in a single CUDA kernel, + which is more efficient than the naive implementation that launches a series of kernels. + + Parameters + ---------- + probs: torch.Tensor + Probabilities for sampling. When indices is not provided, shape should be ``(batch_size, num_classes)`` + and the i-th output will be sampled from the i-th row of probabilities. When indices is provided, + shape should be ``(unique_batch_size, num_classes)`` where unique_batch_size is the number of unique + probability distributions. + top_p: Union[torch.Tensor, float] + Either a scalar or a tensor of shape ``(batch_size,)``, representing the threshold for top-p sampling. + If a scalar, the same threshold is used for all requests. + If a tensor, each request has its own threshold. + indices: Optional[torch.Tensor] + Optional indices tensor of shape ``(batch_size,)`` that maps each output to a row in probs. + For example, if indices[i] = j, then the i-th output will be sampled from probs[j]. + This allows reusing the same probability distribution for multiple outputs. + If indices is not provided, the i-th output will be sampled from the i-th row of probs. + deterministic: bool + Whether to use deterministic kernel implementation, default is ``True``. + generator: Optional[torch.Generator] + A random number generator for the operation. + check_nan: bool + Whether to check nan in :attr:`probs`, default is ``False``. + + Returns + ------- + samples: torch.Tensor + Sampled categories, shape ``(batch_size,)``. + + Note + ---- + This function expects float32 inputs, and the output is int32. + + """ + if check_nan: + if torch.any(torch.isnan(probs)): + raise ValueError("Input probs contains NaN.") + return _top_p_sampling_from_probs_internal( + probs, indices, *_to_tensor_scalar_tuple(top_p), deterministic, generator + ) + + +def _top_k_top_p_sampling_from_probs_internal( + probs: torch.Tensor, + indices: Optional[torch.Tensor], + maybe_top_k_arr: Optional[torch.Tensor], + top_k_val: int, + maybe_top_p_arr: Optional[torch.Tensor], + top_p_val: float, + deterministic: bool, + generator: Optional[torch.Generator], +) -> torch.Tensor: + with probs.device as device: + probs = probs.float() + maybe_top_k_arr = maybe_top_k_arr.int() if maybe_top_k_arr is not None else None + maybe_top_p_arr = ( + maybe_top_p_arr.float() if maybe_top_p_arr is not None else None + ) + samples = torch.empty(probs.size(0), dtype=torch.int32, device=device) + torch.ops.sgl_kernel.top_k_top_p_sampling_from_probs.default( + probs, + samples, + indices, + maybe_top_k_arr, + top_k_val, + maybe_top_p_arr, + top_p_val, + deterministic, + generator, + ) + return samples + + +def top_k_top_p_sampling_from_probs( + probs: torch.Tensor, + top_k: Union[torch.Tensor, int], + top_p: Union[torch.Tensor, float], + indices: Optional[torch.Tensor] = None, + filter_apply_order: str = "top_k_first", + deterministic: bool = True, + generator: Optional[torch.Generator] = None, + check_nan: bool = False, +) -> torch.Tensor: + r"""Adapt from https://github.com/flashinfer-ai/flashinfer/flashinfer/sampling.py + Fused GPU kernel for top-k and top-p sampling from probabilities, + + this operator implements GPU-based rejection sampling without explicit sorting. + Check the `blog post `_ for more details. + + The multiple rounds of rejection sampling are implemented in a single CUDA kernel, + which is more efficient than the naive implementation that launches a series of kernels. + + Parameters + ---------- + probs: torch.Tensor + Probabilities for sampling. When indices is not provided, shape should be ``(batch_size, num_classes)`` + and the i-th output will be sampled from the i-th row of probabilities. When indices is provided, + shape should be ``(unique_batch_size, num_classes)`` where unique_batch_size is the number of unique + probability distributions. + top_k: Union[torch.Tensor, int] + Either a scalar or a tensor of shape ``(batch_size,)``, representing the threshold for top-k sampling. + If a scalar, the same threshold is used for all requests. + If a tensor, each request has its own threshold. + top_p: Union[torch.Tensor, float] + Either a scalar or a tensor of shape ``(batch_size,)``, representing the threshold for top-p sampling. + If a scalar, the same threshold is used for all requests. + If a tensor, each request has its own threshold. + indices: Optional[torch.Tensor] + Optional indices tensor of shape ``(batch_size,)`` that maps each output to a row in probs. + For example, if indices[i] = j, then the i-th output will be sampled from probs[j]. + This allows reusing the same probability distribution for multiple outputs. + If indices is not provided, the i-th output will be sampled from the i-th row of probs. + filter_apply_order: str + The order of applying top-k and top-p sampling, should be either ``"top_k_first"`` or ``"joint"``. + If ``"top_k_first"``, we first apply top-k filter, then apply top-p sampling on the top-k results. + If ``"joint"``, we apply top-k and top-p filter simultaneously in each round. Default is ``"top_k_first"``. + deterministic: bool + Whether to use deterministic kernel implementation, default is ``True``. + generator: Optional[torch.Generator] + A random number generator for the operation. + check_nan: bool + Whether to check nan in :attr:`probs`, default is ``False``. + + Returns + ------- + samples: torch.Tensor + Sampled categories, shape ``(batch_size,)``. + + Note + ---- + This function expects float32 inputs, and the output is int32. + + """ + if filter_apply_order == "top_k_first": + renorm_probs = top_k_renorm_probs(probs, top_k) + return top_p_sampling_from_probs( + renorm_probs, + top_p, + indices, + deterministic, + check_nan=check_nan, + generator=generator, + ) + elif filter_apply_order == "joint": + if check_nan: + if torch.any(torch.isnan(probs)): + raise ValueError("Input probs contains NaN.") + return _top_k_top_p_sampling_from_probs_internal( + probs, + indices, + *_to_tensor_scalar_tuple(top_k), + *_to_tensor_scalar_tuple(top_p), + deterministic, + generator, + ) + else: + raise ValueError(f"Invalid filter_apply_order: {filter_apply_order}") + + +def _min_p_sampling_from_probs_internal( + probs: torch.Tensor, + indices: Optional[torch.Tensor], + maybe_min_p_arr: Optional[torch.Tensor], + min_p_val: float, + deterministic: bool, + generator: Optional[torch.Generator], +) -> torch.Tensor: + with probs.device as device: + probs = probs.float() + maybe_min_p_arr = ( + maybe_min_p_arr.float() if maybe_min_p_arr is not None else None + ) + samples = torch.empty(probs.size(0), dtype=torch.int32, device=device) + torch.ops.sgl_kernel.min_p_sampling_from_probs.default( + probs, + samples, + indices, + maybe_min_p_arr, + min_p_val, + deterministic, + generator, + ) + return samples + + +def min_p_sampling_from_probs( + probs: torch.Tensor, + min_p: Union[torch.Tensor, float], + indices: Optional[torch.Tensor] = None, + deterministic: bool = True, + generator: Optional[torch.Generator] = None, + check_nan: bool = False, +) -> torch.Tensor: + r"""Adapt from https://github.com/flashinfer-ai/flashinfer/flashinfer/sampling.py + Fused GPU kernel for `min_p sampling `_ from probabilities, + + this operator implements GPU-based rejection sampling without explicit sorting. + Check the `blog post `_ for more details. + + The multiple rounds of rejection sampling are implemented in a single CUDA kernel, + which is more efficient than the naive implementation that launches a series of kernels. + + Parameters + ---------- + probs: torch.Tensor + Probabilities for sampling. When indices is not provided, shape should be ``(batch_size, num_classes)`` + and the i-th output will be sampled from the i-th row of probabilities. When indices is provided, + shape should be ``(unique_batch_size, num_classes)`` where unique_batch_size is the number of unique + probability distributions. + min_p: Union[torch.Tensor, float] + Either a scalar or a tensor of shape ``(batch_size,)``, representing the threshold for min-p sampling. + If a scalar, the same threshold is used for all requests. + If a tensor, each request has its own threshold. + indices: Optional[torch.Tensor] + Optional indices tensor of shape ``(batch_size,)`` that maps each output to a row in probs. + For example, if indices[i] = j, then the i-th output will be sampled from probs[j]. + This allows reusing the same probability distribution for multiple outputs. + If indices is not provided, the i-th output will be sampled from the i-th row of probs. + deterministic: bool + Whether to use deterministic kernel implementation, default is ``True``. + generator: Optional[torch.Generator] + A random number generator for the operation. + check_nan: bool + Whether to check nan in :attr:`probs`, default is ``False``. + + Returns + ------- + samples: torch.Tensor + Sampled categories, shape ``(batch_size,)``. + + Note + ---- + This function expects float32 inputs, and the output is int32. + """ + if check_nan: + if torch.any(torch.isnan(probs)): + raise ValueError("Input probs contains NaN.") + return _min_p_sampling_from_probs_internal( + probs, indices, *_to_tensor_scalar_tuple(min_p), deterministic, generator + ) + + +def _top_k_mask_logits_internal( + logits: torch.Tensor, + maybe_top_k_arr: Optional[torch.Tensor], + top_k_val: int, +) -> torch.Tensor: + logits = logits.float() + maybe_top_k_arr = maybe_top_k_arr.int() if maybe_top_k_arr is not None else None + mask_logits = torch.empty_like(logits) + torch.ops.sgl_kernel.top_k_mask_logits.default( + logits, mask_logits, maybe_top_k_arr, top_k_val + ) + return mask_logits + + +def top_k_mask_logits( + logits: torch.Tensor, + top_k: Union[torch.Tensor, int], +) -> torch.Tensor: + r"""Adapt from https://github.com/flashinfer-ai/flashinfer/flashinfer/sampling.py + Fused GPU kernel for masking logits by top-k thresholding. + + Parameters + ---------- + logits: torch.Tensor + Logits before softmax, shape ``(batch_size, num_classes)``. + top_k: Union[torch.Tensor, int] + Either a scalar or a tensor of shape ``(batch_size,)``, representing the top-k threshold for for + for masking logits, should be in ``(0, num_classes)``. + If a scalar, the same threshold is used for all requests. + If a tensor, each request has its own threshold. + We keep the top-k logits, set the rest to negative infinity. + + Returns + ------- + masked_logits: torch.Tensor + Masked logits, shape ``(batch_size, num_classes)``. + + Examples + -------- + + >>> import torch + >>> import flashinfer + >>> torch.manual_seed(42) + >>> batch_size = 4 + >>> vocab_size = 5 + >>> top_k = 3 + >>> logits = torch.randn(batch_size, vocab_size).to(0) + >>> logits + tensor([[ 1.9269, 1.4873, 0.9007, -2.1055, -0.7581], + [ 1.0783, 0.8008, 1.6806, 0.3559, -0.6866], + [-0.4934, 0.2415, -0.2316, 0.0418, -0.2516], + [ 0.8599, -0.3097, -0.3957, 0.8034, -0.6216]], device='cuda:0') + >>> masked_logits = flashinfer.sampling.top_k_mask_logits(logits, top_k) + >>> masked_logits + tensor([[ 1.9269, 1.4873, 0.9007, -inf, -inf], + [ 1.0783, 0.8008, 1.6806, -inf, -inf], + [ -inf, 0.2415, -0.2316, 0.0418, -inf], + [ 0.8599, -0.3097, -inf, 0.8034, -inf]], device='cuda:0') + + Note + ---- + The combination of ``top_k_mask_logits`` and ``softmax`` should be equivalent to ``top_k_renorm_probs``. + + See Also + -------- + top_k_renorm_probs + """ + return _top_k_mask_logits_internal(logits, *_to_tensor_scalar_tuple(top_k)) + + +def top_k_top_p_sampling_from_logits( + logits: torch.Tensor, + top_k: Union[torch.Tensor, int], + top_p: Union[torch.Tensor, float], + indices: Optional[torch.Tensor] = None, + filter_apply_order: str = "top_k_first", + deterministic: bool = True, + generator: Optional[torch.Generator] = None, + check_nan: bool = False, +) -> torch.Tensor: + r"""Adapt from https://github.com/flashinfer-ai/flashinfer/flashinfer/sampling.py + Fused GPU kernel for top-k and top-p sampling from probabilities, + + this operator implements GPU-based rejection sampling without explicit sorting. + Check the `blog post `_ for more details. + + The multiple rounds of rejection sampling are implemented in a single CUDA kernel, + which is more efficient than the naive implementation that launches a series of kernels. + + Parameters + ---------- + logits: torch.Tensor + Pre-softmax logits for sampling. When indices is not provided, shape should be ``(batch_size, num_classes)`` + and the i-th output will be sampled from the i-th row of logits. When indices is provided, + shape should be ``(unique_batch_size, num_classes)`` where unique_batch_size is the number of unique + probability distributions. + top_k: Union[torch.Tensor, int] + Either a scalar or a tensor of shape ``(batch_size,)``, representing the threshold for top-k sampling. + If a scalar, the same threshold is used for all requests. + If a tensor, each request has its own threshold. + top_p: Union[torch.Tensor, float] + Either a scalar or a tensor of shape ``(batch_size,)``, representing the threshold for top-p sampling. + If a scalar, the same threshold is used for all requests. + If a tensor, each request has its own threshold. + indices: Optional[torch.Tensor] + Optional indices tensor of shape ``(batch_size,)`` that maps each output to a row in probs. + For example, if indices[i] = j, then the i-th output will be sampled from probs[j]. + This allows reusing the same probability distribution for multiple outputs. + If indices is not provided, the i-th output will be sampled from the i-th row of probs. + filter_apply_order: str + The order of applying top-k and top-p sampling, should be either ``"top_k_first"`` or ``"joint"``. + If ``"top_k_first"``, we first apply top-k filter, then apply top-p sampling on the top-k results. + If ``"joint"``, we apply top-k and top-p filter simultaneously in each round. Default is ``"top_k_first"``. + deterministic: bool + Whether to use deterministic kernel implementation, default is ``True``. + generator: Optional[torch.Generator] + A random number generator for the operation. + check_nan: bool + Whether to check nan in :attr:`probs`, default is ``False``. + + Returns + ------- + samples: torch.Tensor + Sampled categories, shape ``(batch_size,)``. + + Note + ---- + This function expects float32 inputs, and the output is int32. + + """ + if filter_apply_order == "top_k_first": + masked_logits = top_k_mask_logits(logits, top_k) + probs = torch.softmax(masked_logits, dim=-1) + return top_p_sampling_from_probs( + probs, + top_p, + indices, + deterministic, + check_nan=check_nan, + generator=generator, + ) + elif filter_apply_order == "joint": + probs = torch.softmax(logits, dim=-1) + if check_nan: + if torch.any(torch.isnan(probs)): + raise ValueError("Input probs contains NaN.") + return _top_k_top_p_sampling_from_probs_internal( + probs, + indices, + *_to_tensor_scalar_tuple(top_k), + *_to_tensor_scalar_tuple(top_p), + deterministic, + generator, + ) + else: + raise ValueError(f"Invalid filter_apply_order: {filter_apply_order}") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/scalar_type.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/scalar_type.py new file mode 100644 index 00000000..571c1dca --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/scalar_type.py @@ -0,0 +1,352 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import functools +import struct +from dataclasses import dataclass +from enum import Enum +from typing import Optional, Union + +_SCALAR_TYPES_ID_MAP = {} + + +# Mirrors enum in `core/scalar_type.hpp` +class NanRepr(Enum): + NONE = 0 # nans are not supported + IEEE_754 = 1 # nans are: Exp all 1s, mantissa not all 0s + EXTD_RANGE_MAX_MIN = 2 # nans are: Exp all 1s, mantissa all 1s + + +# This ScalarType class is a parallel implementation of the C++ ScalarType +# class found in csrc/core/scalar_type.hpp. These two classes should be kept +# in sync until the inductor fully supports custom C++ classes. +@dataclass(frozen=True) +class ScalarType: + """ + ScalarType can represent a wide range of floating point and integer + types, in particular it can be used to represent sub-byte data types + (something that torch.dtype currently does not support). It is also + capable of representing types with a bias, i.e.: + `stored_value = value + bias`, + this is useful for quantized types (e.g. standard GPTQ 4bit uses a bias + of 8). The implementation for this class can be found in + csrc/core/scalar_type.hpp, these type signatures should be kept in sync + with that file. + """ + + exponent: int + """ + Number of bits in the exponent if this is a floating point type + (zero if this an integer type) + """ + + mantissa: int + """ + Number of bits in the mantissa if this is a floating point type, + or the number bits representing an integer excluding the sign bit if + this an integer type. + """ + + signed: bool + "If the type is signed (i.e. has a sign bit)" + + bias: int + """ + bias used to encode the values in this scalar type + (value = stored_value - bias, default 0) for example if we store the + type as an unsigned integer with a bias of 128 then the value 0 will be + stored as 128 and -1 will be stored as 127 and 1 will be stored as 129. + """ + + _finite_values_only: bool = False + """ + Private: if infs are supported, used `has_infs()` instead. + """ + + nan_repr: NanRepr = NanRepr.IEEE_754 + """ + How NaNs are represent in this scalar type, returns NanRepr value. + (not applicable for integer types) + """ + + def _floating_point_max_int(self) -> int: + assert ( + self.mantissa <= 52 and self.exponent <= 11 + ), f"Cannot represent max/min as a double for type {self.__str__()}" + + max_mantissa = (1 << self.mantissa) - 1 + if self.nan_repr == NanRepr.EXTD_RANGE_MAX_MIN: + max_mantissa = max_mantissa - 1 + + max_exponent = (1 << self.exponent) - 2 + if self.nan_repr == NanRepr.EXTD_RANGE_MAX_MIN or self.nan_repr == NanRepr.NONE: + assert ( + self.exponent < 11 + ), f"Cannot represent max/min as a double for type {self.__str__()}" + max_exponent = max_exponent + 1 + + # adjust the exponent to match that of a double + # for now we assume the exponent bias is the standard 2^(e-1) -1, (where + # e is the exponent bits), there is some precedent for non-standard + # biases, example `float8_e4m3b11fnuz` here: + # https://github.com/jax-ml/ml_dtypes but to avoid premature over + # complication we are just assuming the standard exponent bias until + # there is a need to support non-standard biases + exponent_bias = (1 << (self.exponent - 1)) - 1 + exponent_bias_double = (1 << 10) - 1 # double e = 11 + + max_exponent_double = max_exponent - exponent_bias + exponent_bias_double + + # shift the mantissa and exponent into the proper positions for an + # IEEE double and bitwise-or them together. + return (max_mantissa << (52 - self.mantissa)) | (max_exponent_double << 52) + + def _floating_point_max(self) -> float: + double_raw = self._floating_point_max_int() + return struct.unpack("!d", struct.pack("!Q", double_raw))[0] + + def _raw_max(self) -> Union[int, float]: + if self.is_floating_point(): + return self._floating_point_max() + else: + assert ( + self.size_bits < 64 or self.size_bits == 64 and self.is_signed() + ), "Cannot represent max as an int" + return (1 << self.mantissa) - 1 + + def _raw_min(self) -> Union[int, float]: + if self.is_floating_point(): + assert ( + self.is_signed() + ), "We currently assume all floating point types are signed" + sign_bit_double = 1 << 63 + + max_raw = self._floating_point_max_int() + min_raw = max_raw | sign_bit_double + return struct.unpack("!d", struct.pack("!Q", min_raw))[0] + else: + assert ( + not self.is_signed() or self.size_bits <= 64 + ), "Cannot represent min as a int64_t" + + if self.is_signed(): + return -(1 << (self.size_bits - 1)) + else: + return 0 + + @functools.cached_property + def id(self) -> int: + """ + Convert the ScalarType to an int which can be passed to pytorch custom + ops. This layout of the int must be kept in sync with the C++ + ScalarType's from_id method. + """ + val = 0 + offset = 0 + + def or_and_advance(member, bit_width): + nonlocal val + nonlocal offset + bit_mask = (1 << bit_width) - 1 + val = val | (int(member) & bit_mask) << offset + offset = offset + bit_width + + or_and_advance(self.exponent, 8) + or_and_advance(self.mantissa, 8) + or_and_advance(self.signed, 1) + or_and_advance(self.bias, 32) + or_and_advance(self._finite_values_only, 1) + or_and_advance(self.nan_repr.value, 8) + + assert offset <= 64, f"ScalarType fields too big {offset} to fit into an int64" + + _SCALAR_TYPES_ID_MAP[val] = self + + return val + + @property + def size_bits(self) -> int: + return self.exponent + self.mantissa + int(self.signed) + + def min(self) -> Union[int, float]: + """ + Min representable value for this scalar type. + (accounting for bias if there is one) + """ + return self._raw_min() - self.bias + + def max(self) -> Union[int, float]: + """ + Max representable value for this scalar type. + (accounting for bias if there is one) + """ + return self._raw_max() - self.bias + + def is_signed(self) -> bool: + """ + If the type is signed (i.e. has a sign bit), same as `signed` + added for consistency with: + https://pytorch.org/docs/stable/generated/torch.Tensor.is_signed.html + """ + return self.signed + + def is_floating_point(self) -> bool: + "If the type is a floating point type" + return self.exponent != 0 + + def is_integer(self) -> bool: + "If the type is an integer type" + return self.exponent == 0 + + def has_bias(self) -> bool: + "If the type has a non-zero bias" + return self.bias != 0 + + def has_infs(self) -> bool: + "If the type is floating point and supports infinity" + return not self._finite_values_only + + def has_nans(self) -> bool: + return self.nan_repr != NanRepr.NONE + + def is_ieee_754(self) -> bool: + """ + If the type is a floating point type that follows IEEE 754 + conventions + """ + return self.nan_repr == NanRepr.IEEE_754 and not self._finite_values_only + + def __str__(self) -> str: + """ + naming generally follows: https://github.com/jax-ml/ml_dtypes + for floating point types (leading f) the scheme is: + `float_em[flags]` + flags: + - no-flags: means it follows IEEE 754 conventions + - f: means finite values only (no infinities) + - n: means nans are supported (non-standard encoding) + for integer types the scheme is: + `[u]int[b]` + - if bias is not present it means its zero + """ + if self.is_floating_point(): + ret = ( + "float" + + str(self.size_bits) + + "_e" + + str(self.exponent) + + "m" + + str(self.mantissa) + ) + + if not self.is_ieee_754(): + if self._finite_values_only: + ret = ret + "f" + if self.nan_repr != NanRepr.NONE: + ret = ret + "n" + + return ret + else: + ret = ("int" if self.is_signed() else "uint") + str(self.size_bits) + if self.has_bias(): + ret = ret + "b" + str(self.bias) + return ret + + def __repr__(self) -> str: + return "ScalarType." + self.__str__() + + # __len__ needs to be defined (and has to throw TypeError) for pytorch's + # opcheck to work. + def __len__(self) -> int: + raise TypeError + + # + # Convenience Constructors + # + + @classmethod + def int_(cls, size_bits: int, bias: Optional[int]) -> "ScalarType": + "Create a signed integer scalar type (size_bits includes sign-bit)." + ret = cls(0, size_bits - 1, True, bias if bias else 0) + ret.id # noqa B018: make sure the id is cached + return ret + + @classmethod + def uint(cls, size_bits: int, bias: Optional[int]) -> "ScalarType": + """Create a unsigned integer scalar type.""" + ret = cls(0, size_bits, False, bias if bias else 0) + ret.id # noqa B018: make sure the id is cached + return ret + + @classmethod + def float_IEEE754(cls, exponent: int, mantissa: int) -> "ScalarType": + """ + Create a standard floating point type + (i.e. follows IEEE 754 conventions). + """ + assert mantissa > 0 and exponent > 0 + ret = cls(exponent, mantissa, True, 0) + ret.id # noqa B018: make sure the id is cached + return ret + + @classmethod + def float_( + cls, exponent: int, mantissa: int, finite_values_only: bool, nan_repr: NanRepr + ) -> "ScalarType": + """ + Create a non-standard floating point type + (i.e. does not follow IEEE 754 conventions). + """ + assert mantissa > 0 and exponent > 0 + assert nan_repr != NanRepr.IEEE_754, ( + "use `float_IEEE754` constructor for floating point types that " + "follow IEEE 754 conventions" + ) + ret = cls(exponent, mantissa, True, 0, finite_values_only, nan_repr) + ret.id # noqa B018: make sure the id is cached + return ret + + @classmethod + def from_id(cls, scalar_type_id: int): + if scalar_type_id not in _SCALAR_TYPES_ID_MAP: + raise ValueError(f"scalar_type_id {scalar_type_id} doesn't exists.") + return _SCALAR_TYPES_ID_MAP[scalar_type_id] + + +# naming generally follows: https://github.com/jax-ml/ml_dtypes +# for floating point types (leading f) the scheme is: +# `float_em[flags]` +# flags: +# - no-flags: means it follows IEEE 754 conventions +# - f: means finite values only (no infinities) +# - n: means nans are supported (non-standard encoding) +# for integer types the scheme is: +# `[u]int[b]` +# - if bias is not present it means its zero + + +class scalar_types: + int4 = ScalarType.int_(4, None) + uint4 = ScalarType.uint(4, None) + int8 = ScalarType.int_(8, None) + uint8 = ScalarType.uint(8, None) + float8_e4m3fn = ScalarType.float_(4, 3, True, NanRepr.EXTD_RANGE_MAX_MIN) + float8_e5m2 = ScalarType.float_IEEE754(5, 2) + float16_e8m7 = ScalarType.float_IEEE754(8, 7) + float16_e5m10 = ScalarType.float_IEEE754(5, 10) + + # fp6, https://github.com/usyd-fsalab/fp6_llm/tree/main + float6_e3m2f = ScalarType.float_(3, 2, True, NanRepr.NONE) + + # fp4, https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf + float4_e2m1f = ScalarType.float_(2, 1, True, NanRepr.NONE) + + # "gptq" types + uint2b2 = ScalarType.uint(2, 2) + uint3b4 = ScalarType.uint(3, 4) + uint4b8 = ScalarType.uint(4, 8) + uint8b128 = ScalarType.uint(8, 128) + + # colloquial names + bfloat16 = float16_e8m7 + float16 = float16_e5m10 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/sparse_flash_attn.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/sparse_flash_attn.py new file mode 100644 index 00000000..29b2f040 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/sparse_flash_attn.py @@ -0,0 +1,293 @@ +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn as nn + + +def maybe_contiguous(x): + return x.contiguous() if x is not None and x.stride(-1) != 1 else x + + +# Sparse attention utils +def convert_vertical_slash_indexes( + q_seqlens: torch.Tensor, # [BATCH, ] + kv_seqlens: torch.Tensor, # [BATCH, ] + vertical_indexes: torch.Tensor, # [BATCH, N_HEADS, NNZ_V] + slash_indexes: torch.Tensor, # [BATCH, N_HEADS, NNZ_S] + context_size: int, + block_size_M: int, + block_size_N: int, + causal: bool = True, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + batch_size = slash_indexes.size(0) + num_heads = slash_indexes.size(1) + nnz_slash = slash_indexes.size(2) + nnz_vertical = vertical_indexes.size(2) + num_rows = (context_size + block_size_M - 1) // block_size_M + + block_count = torch.zeros( + batch_size, num_heads, num_rows, dtype=q_seqlens.dtype, device=q_seqlens.device + ) + block_offset = torch.zeros( + batch_size, + num_heads, + num_rows, + nnz_slash, + dtype=q_seqlens.dtype, + device=q_seqlens.device, + ) + column_count = torch.zeros( + batch_size, num_heads, num_rows, dtype=q_seqlens.dtype, device=q_seqlens.device + ) + column_index = torch.zeros( + batch_size, + num_heads, + num_rows, + nnz_vertical, + dtype=q_seqlens.dtype, + device=q_seqlens.device, + ) + + torch.ops.sgl_kernel.convert_vertical_slash_indexes.default( + block_count, + block_offset, + column_count, + column_index, + q_seqlens, + kv_seqlens, + vertical_indexes, + slash_indexes, + context_size, + block_size_M, + block_size_N, + causal, + ) + return block_count, block_offset, column_count, column_index + + +def convert_vertical_slash_indexes_mergehead( + q_seqlens: torch.Tensor, # [BATCH, ] + kv_seqlens: torch.Tensor, # [BATCH, ] + vertical_indexes: torch.Tensor, # [BATCH, N_HEADS, NNZ_V] + slash_indexes: torch.Tensor, # [BATCH, N_HEADS, NNZ_S] + # [N_HEADS] : different head use different number of indices + vertical_indices_count: torch.Tensor, + slash_indices_count: torch.Tensor, + context_size: int, + block_size_M: int, + block_size_N: int, + causal: bool = True, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + batch_size = slash_indexes.size(0) + num_heads = slash_indexes.size(1) + nnz_slash = slash_indexes.size(2) + nnz_vertical = vertical_indexes.size(2) + num_rows = (context_size + block_size_M - 1) // block_size_M + + block_count = torch.empty( + batch_size, num_heads, num_rows, dtype=q_seqlens.dtype, device=q_seqlens.device + ) + block_offset = torch.empty( + batch_size, + num_heads, + num_rows, + nnz_slash, + dtype=q_seqlens.dtype, + device=q_seqlens.device, + ) + column_count = torch.empty( + batch_size, num_heads, num_rows, dtype=q_seqlens.dtype, device=q_seqlens.device + ) + column_index = torch.empty( + batch_size, + num_heads, + num_rows, + nnz_vertical, + dtype=q_seqlens.dtype, + device=q_seqlens.device, + ) + + torch.ops.sgl_kernel.convert_vertical_slash_indexes_mergehead.default( + block_count, + block_offset, + column_count, + column_index, + q_seqlens, + kv_seqlens, + vertical_indexes, + slash_indexes, + vertical_indices_count, + slash_indices_count, + context_size, + block_size_M, + block_size_N, + causal, + ) + return block_count, block_offset, column_count, column_index + + +def sparse_attn_func( + q, + k, + v, + block_count, + block_offset, + column_count, + column_index, + dropout_p=0.0, + softmax_scale=None, + causal=False, + softcap=0.0, # 0.0 means deactivated + alibi_slopes=None, + deterministic=False, + return_attn_probs=False, + *, + return_softmax_lse=False, + out=None, +): + """Compute attention with vertical and slash sparsity patterns. + Most Arguments are the same with the flash_attn_func interface, except for 4 extra args: + block_count and block_offset for slash sparsity patterns, and + column_count and column_index for vertical sparsity patterns. + For more details please refer to Appendix C.4.2 of paper https://arxiv.org/abs/2407.02490. + + Arguments: + q: (batch_size, seqlen, nheads, headdim) + k: (batch_size, seqlen, nheads_k, headdim) + v: (batch_size, seqlen, nheads_k, headdim) + block_count: (batch_size, nheads, cdiv(seqlen, BLOCK_M)) + block_offset: (batch_size, nheads, cdiv(seqlen, BLOCK_M), NNZ_S) + column_count: (batch_size, nheads, cdiv(seqlen, BLOCK_M)) + column_index: (batch_size, nheads, cdiv(seqlen, BLOCK_M), NNZ_V) + dropout_p: float. Dropout probability. + softmax_scale: float. The scaling of QK^T before applying softmax. + Default to 1 / sqrt(headdim). + causal: bool. Whether to apply causal attention mask (e.g., for auto-regressive modeling). + alibi_slopes: (nheads,) or (batch_size, nheads), fp32. A bias of + (-alibi_slope * |i + seqlen_k - seqlen_q - j|) + is added to the attention score of query i and key j. + deterministic: bool. Whether to use the deterministic implementation of the backward pass, + which is slightly slower and uses more memory. The forward pass is always deterministic. + return_attn_probs: bool. Whether to return the attention probabilities. This option is for + testing only. The returned probabilities are not guaranteed to be correct + (they might not have the right scaling). + Return: + out: (batch_size, seqlen, nheads, headdim). + softmax_lse [optional, if return_softmax_lse=True]: (batch_size, nheads, seqlen). The + logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax + normalization factor). + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + + q, k, v = [maybe_contiguous(x) for x in (q, k, v)] + out, softmax_lse = torch.ops.sgl_kernel.fwd_sparse.default( + q, + k, + v, + block_count, + block_offset, + column_count, + column_index, + out, + alibi_slopes, + dropout_p, + softmax_scale, + causal, + softcap, + return_attn_probs and dropout_p > 0, + None, + ) + return (out, softmax_lse) if return_softmax_lse else out + + +def sparse_attn_varlen_func( + q, + k, + v, + block_count, + block_offset, + column_count, + column_index, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + dropout_p=0.0, + softmax_scale=None, + causal=False, + softcap=0.0, # 0.0 means deactivated + alibi_slopes=None, + deterministic=False, + return_attn_probs=False, + *, + return_softmax_lse=False, + out=None, +): + """Compute attention with vertical and slash sparsity patterns. + Most Arguments are the same with the flash_attn_varlen_func interface, except for 4 extra args: + block_count and block_offset for slash sparsity patterns, and + column_count and column_index for vertical sparsity patterns. + For more details please refer to Appendix C.4.2 of paper https://arxiv.org/abs/2407.02490. + + Arguments: + q: (total_q, nheads, headdim), where total_q = total number of query tokens in the batch. + k: (total_k, nheads_k, headdim), where total_k = total number of key tokens in the batch. + v: (total_k, nheads_k, headdim), where total_k = total number of key tokens in the batch. + block_count: (batch_size, nheads, cdiv(seqlen, BLOCK_M)) + block_offset: (batch_size, nheads, cdiv(seqlen, BLOCK_M), NNZ_S) + column_count: (batch_size, nheads, cdiv(seqlen, BLOCK_M)) + column_index: (batch_size, nheads, cdiv(seqlen, BLOCK_M), NNZ_V) + cu_seqlens_q: (batch_size + 1,), dtype torch.int32. The cumulative sequence lengths + of the sequences in the batch, used to index into q. + cu_seqlens_k: (batch_size + 1,), dtype torch.int32. The cumulative sequence lengths + of the sequences in the batch, used to index into kv. + max_seqlen_q: int. Maximum query sequence length in the batch. + max_seqlen_k: int. Maximum key sequence length in the batch. + dropout_p: float. Dropout probability. + softmax_scale: float. The scaling of QK^T before applying softmax. + Default to 1 / sqrt(headdim). + causal: bool. Whether to apply causal attention mask (e.g., for auto-regressive modeling). + softcap: float. Anything > 0 activates softcapping attention. + alibi_slopes: (nheads,) or (batch_size, nheads), fp32. A bias of + (-alibi_slope * |i + seqlen_k - seqlen_q - j|) + is added to the attention score of query i and key j. + deterministic: bool. Whether to use the deterministic implementation of the backward pass, + which is slightly slower and uses more memory. The forward pass is always deterministic. + return_attn_probs: bool. Whether to return the attention probabilities. This option is for + testing only. The returned probabilities are not guaranteed to be correct + (they might not have the right scaling). + Return: + out: (total, nheads, headdim). + softmax_lse [optional, if return_softmax_lse=True]: (nheads, total_q_seqlen). The + logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax + normalization factor). + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + + q, k, v = [maybe_contiguous(x) for x in (q, k, v)] + out, softmax_lse = torch.ops.sgl_kernel.varlen_fwd_sparse.default( + q, + k, + v, + block_count, + block_offset, + column_count, + column_index, + out, + cu_seqlens_q, + cu_seqlens_k, + None, + alibi_slopes, + max_seqlen_q, + max_seqlen_k, + dropout_p, + softmax_scale, + False, + causal, + softcap, + return_attn_probs and dropout_p > 0, + None, + ) + return (out, softmax_lse) if return_softmax_lse else out diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/spatial.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/spatial.py new file mode 100644 index 00000000..201ec7b7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/spatial.py @@ -0,0 +1,63 @@ +import torch +from torch.cuda.streams import ExternalStream + +try: + from . import spatial_ops # triggers TORCH extension registration +except Exception as _e: + _spatial_import_error = _e +else: + _spatial_import_error = None + +_IMPORT_ERROR = ImportError( + "Failed to load sgl_kernel.spatial_ops extension. Ensure CUDA Driver >= 12.4" +) + + +def create_greenctx_stream_by_value( + SM_a: int, SM_b: int, device_id: int = None +) -> tuple[ExternalStream, ExternalStream]: + """ + Create two streams for greenctx. + Args: + sm_A (int): The SM of stream A. + sm_B (int): The weight of stream B. + device_id (int): The device id. + Returns: + tuple[ExternalStream, ExternalStream]: The two streams. + """ + if _spatial_import_error is not None: + raise _IMPORT_ERROR from _spatial_import_error + if device_id is None: + device_id = torch.cuda.current_device() + + res = torch.ops.sgl_kernel.create_greenctx_stream_by_value(SM_a, SM_b, device_id) + + stream_a = ExternalStream( + stream_ptr=res[0], device=torch.device(f"cuda:{device_id}") + ) + stream_b = ExternalStream( + stream_ptr=res[1], device=torch.device(f"cuda:{device_id}") + ) + + return stream_a, stream_b + + +def get_sm_available(device_id: int = None) -> int: + """ + Get the SMs available on the device. + Args: + device_id (int): The device id. + Returns: + int: The SMs available. + """ + if _spatial_import_error is not None: + raise _IMPORT_ERROR from _spatial_import_error + if device_id is None: + device_id = torch.cuda.current_device() + + device_props = torch.cuda.get_device_properties(device_id) + + # Get the number of Streaming Multiprocessors (SMs) + sm_count = device_props.multi_processor_count + + return sm_count diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/speculative.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/speculative.py new file mode 100644 index 00000000..17e42497 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/speculative.py @@ -0,0 +1,126 @@ +import torch + + +def tree_speculative_sampling_target_only( + predicts: torch.Tensor, # mutable + accept_index: torch.Tensor, # mutable + accept_token_num: torch.Tensor, # mutable + candidates: torch.Tensor, + retrive_index: torch.Tensor, + retrive_next_token: torch.Tensor, + retrive_next_sibling: torch.Tensor, + uniform_samples: torch.Tensor, + uniform_samples_for_final_sampling: torch.Tensor, + target_probs: torch.Tensor, + draft_probs: torch.Tensor, + threshold_single: float = 1.0, + threshold_acc: float = 1.0, + deterministic: bool = True, +) -> None: + torch.ops.sgl_kernel.tree_speculative_sampling_target_only.default( + predicts, + accept_index, + accept_token_num, + candidates, + retrive_index, + retrive_next_token, + retrive_next_sibling, + uniform_samples, + uniform_samples_for_final_sampling, + target_probs, + draft_probs, + threshold_single, + threshold_acc, + deterministic, + ) + + +def verify_tree_greedy( + predicts: torch.Tensor, # mutable + accept_index: torch.Tensor, # mutable + accept_token_num: torch.Tensor, # mutable + candidates: torch.Tensor, + retrive_index: torch.Tensor, + retrive_next_token: torch.Tensor, + retrive_next_sibling: torch.Tensor, + target_predict: torch.Tensor, +) -> None: + torch.ops.sgl_kernel.verify_tree_greedy.default( + predicts, + accept_index, + accept_token_num, + candidates, + retrive_index, + retrive_next_token, + retrive_next_sibling, + target_predict, + ) + + +def build_tree_kernel_efficient( + parent_list: torch.Tensor, + selected_index: torch.Tensor, + verified_seq_len: torch.Tensor, + tree_mask: torch.Tensor, + positions: torch.Tensor, + retrive_index: torch.Tensor, + retrive_next_token: torch.Tensor, + retrive_next_sibling: torch.Tensor, + topk: int, + depth: int, + draft_token_num: int, + tree_mask_mode: int, +) -> None: + torch.ops.sgl_kernel.build_tree_kernel_efficient.default( + parent_list, + selected_index, + verified_seq_len, + tree_mask, + positions, + retrive_index, + retrive_next_token, + retrive_next_sibling, + topk, + depth, + draft_token_num, + tree_mask_mode, + ) + + +def reconstruct_indices_from_tree_mask( + tree_mask: torch.Tensor, + verified_seq_len: torch.Tensor, + positions: torch.Tensor, + retrive_index: torch.Tensor, + retrive_next_token: torch.Tensor, + retrive_next_sibling: torch.Tensor, + batch_size: int, + draft_token_num: int, +) -> None: + torch.ops.sgl_kernel.reconstruct_indices_from_tree_mask.default( + tree_mask, + verified_seq_len, + positions, + retrive_index, + retrive_next_token, + retrive_next_sibling, + batch_size, + draft_token_num, + ) + + +def segment_packbits( + x: torch.Tensor, + input_indptr: torch.Tensor, + output_indptr: torch.Tensor, + y: torch.Tensor, + batch_size: int, +) -> None: + torch.ops.sgl_kernel.segment_packbits.default( + x, + input_indptr, + output_indptr, + y, + batch_size, + torch.cuda.current_stream().cuda_stream, + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/test_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/test_utils.py new file mode 100644 index 00000000..ede113fd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/test_utils.py @@ -0,0 +1,125 @@ +import torch + + +def create_per_token_group_quant_test_data(num_tokens, hidden_dim, num_ranks, flags): + device = torch.device("cuda") + dtype = torch.bfloat16 + + seed = num_tokens * 10000 + hidden_dim + gen_cpu = torch.Generator(device="cpu") + gen_cpu.manual_seed(seed) + gen_cuda = torch.Generator(device="cuda") + gen_cuda.manual_seed(seed) + + if flags["fuse_silu_and_mul"]: + effective_hidden_dim = hidden_dim * 2 + else: + effective_hidden_dim = hidden_dim + del hidden_dim + + if (masked_layout_mode := flags["masked_layout_mode"]) is not None: + num_max_dispatch_tokens_per_rank = 768 + num_global_experts = 288 + num_local_experts, remainder = divmod(num_global_experts, num_ranks) + assert remainder == 0 + + # mimic DeepEP low_latency_dispatch output + x = torch.randn( + num_local_experts, + num_max_dispatch_tokens_per_rank * num_ranks, + effective_hidden_dim, + device=device, + dtype=dtype, + generator=gen_cuda, + ) + + if masked_layout_mode == "balanced": + masked_m = _compute_balanced_split(num_tokens, num_local_experts) + elif masked_layout_mode == "imbalanced": + masked_m = _compute_imbalanced_split( + num_tokens, num_local_experts, gen_cpu=gen_cpu + ) + elif masked_layout_mode == "extreme": + masked_m = torch.tensor( + [num_tokens] + [0] * (num_local_experts - 1), dtype=torch.int + ) + else: + raise NotImplementedError + print(f"{masked_layout_mode=} {masked_m=} {x.shape=}") + + masked_m = masked_m.to(device) + + return x, masked_m + else: + x = torch.randn( + num_tokens, + effective_hidden_dim, + device=device, + dtype=dtype, + generator=gen_cuda, + ) + x[torch.randn(x.shape, device=device, generator=gen_cuda) < 0.001] *= 10 + return x, None + + +def _compute_balanced_split(total: int, arr_len: int): + base = total // arr_len + remainder = total % arr_len + ans = [base + 1 if i < remainder else base for i in range(arr_len)] + assert sum(ans) == total + return torch.tensor(ans, dtype=torch.int) + + +def _compute_imbalanced_split( + total: int, arr_len: int, gen_cpu, dtype=torch.int +) -> list[int]: + # can use `rand ** 2`, `rand ** 3`, etc, to change how imbalanced it is + noise_raw = torch.rand(arr_len, generator=gen_cpu) ** 3 + + noise = noise_raw / noise_raw.sum() + ans = (noise * total).round().to(dtype) + + diff = total - ans.sum().item() + while diff != 0: + idx = torch.randint(0, arr_len, (1,), generator=gen_cpu).item() + if diff > 0: + ans[idx] += 1 + diff -= 1 + elif diff < 0 and ans[idx] > 0: + ans[idx] -= 1 + diff += 1 + + assert sum(ans) == total + return ans + + +def assert_all_close_or_tiny_diff(a: torch.Tensor, b: torch.Tensor): + assert (a.shape == b.shape) and ( + a.dtype == b.dtype + ), f"{a.shape=} {b.shape=} {a.dtype=} {b.dtype=}" + numel = a.numel() + + if a.dtype == torch.float8_e4m3fn: + a_u8 = a.view(torch.uint8) + b_u8 = b.view(torch.uint8) + diff_u8 = (a_u8.to(torch.int16) - b_u8.to(torch.int16)).abs() + + count_diff_sign = ((a_u8 >= 0) & (b_u8 < 0)).sum().item() + count_tiny_diff = (diff_u8 == 1).sum().item() + count_large_diff = (diff_u8 >= 2).sum().item() + elif a.dtype == torch.int8: + diff = (a.to(torch.int16) - a.to(torch.int16)).abs() + count_diff_sign = ((a >= 0) & (b < 0)).sum().item() + count_tiny_diff = (diff == 1).sum().item() + count_large_diff = (diff >= 2).sum().item() + else: + raise NotImplementedError + + assert ( + (count_diff_sign == 0) + and (count_large_diff == 0) + and ( + (count_tiny_diff / numel < 0.005) + or ((count_tiny_diff / numel < 0.04) and (numel <= 4096)) + ) + ), f"{count_diff_sign=} {count_tiny_diff=} {count_large_diff=} {numel=} {a=} {b=}" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/testing/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/testing/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/testing/rotary_embedding.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/testing/rotary_embedding.py new file mode 100644 index 00000000..f1506479 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/testing/rotary_embedding.py @@ -0,0 +1,221 @@ +from typing import Optional, Tuple, Union + +import torch +from sgl_kernel import FusedSetKVBufferArg, apply_rope_with_cos_sin_cache_inplace + + +# vLLM torch native +def _apply_rotary_emb( + x: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + is_neox_style: bool, +) -> torch.Tensor: + """ + Args: + x: [num_tokens, num_heads, head_size] + cos: [num_tokens, head_size // 2] + sin: [num_tokens, head_size // 2] + is_neox_style: Whether to use the Neox-style or GPT-J-style rotary + positional embeddings. + """ + cos = cos.unsqueeze(-2).to(x.dtype) + sin = sin.unsqueeze(-2).to(x.dtype) + if is_neox_style: + x1, x2 = torch.chunk(x, 2, dim=-1) + else: + x1 = x[..., ::2] + x2 = x[..., 1::2] + o1 = x1 * cos - x2 * sin + o2 = x2 * cos + x1 * sin + if is_neox_style: + return torch.cat((o1, o2), dim=-1) + else: + return torch.stack((o1, o2), dim=-1).flatten(-2) + + +class RotaryEmbedding(torch.nn.Module): + # Reference: https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/rotary_embedding.py + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + dtype: torch.dtype, + ) -> None: + super().__init__() + self.head_size = head_size + self.rotary_dim = rotary_dim + self.max_position_embeddings = max_position_embeddings + self.base = base + self.is_neox_style = is_neox_style + self.dtype = dtype + + cache = self._compute_cos_sin_cache() + self.cos_sin_cache: torch.Tensor + self.register_buffer("cos_sin_cache", cache, persistent=False) + + def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor: + inv_freq = 1.0 / ( + base + ** ( + torch.arange(0, self.rotary_dim, 2, dtype=torch.float) / self.rotary_dim + ) + ) + return inv_freq + + def _compute_cos_sin_cache(self) -> torch.Tensor: + """Compute the cos and sin cache.""" + inv_freq = self._compute_inv_freq(self.base) + t = torch.arange(self.max_position_embeddings, dtype=torch.float) + + freqs = torch.einsum("i,j -> ij", t, inv_freq) + cos = freqs.cos() + sin = freqs.sin() + cache = torch.cat((cos, sin), dim=-1) + return cache + + def forward_native( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + offsets: Optional[torch.Tensor] = None, + fused_set_kv_buffer_arg: Optional[FusedSetKVBufferArg] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """A PyTorch-native implementation of forward().""" + assert ( + fused_set_kv_buffer_arg is None + ), "fused_set_kv_buffer_arg is not supported for native implementation" + + if offsets is not None: + positions = positions + offsets + + positions = positions.flatten() + num_tokens = positions.shape[0] + cos_sin = self.cos_sin_cache.index_select(0, positions) + + # Modification: float32 is required for the rotary embedding to work correctly + query = query.to(torch.float32) + key = key.to(torch.float32) + + cos, sin = cos_sin.chunk(2, dim=-1) + + query_shape = query.shape + query = query.view(num_tokens, -1, self.head_size) + query_rot = query[..., : self.rotary_dim] + query_pass = query[..., self.rotary_dim :] + query_rot = _apply_rotary_emb(query_rot, cos, sin, self.is_neox_style) + query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) + + key_shape = key.shape + key = key.view(num_tokens, -1, self.head_size) + key_rot = key[..., : self.rotary_dim] + key_pass = key[..., self.rotary_dim :] + key_rot = _apply_rotary_emb(key_rot, cos, sin, self.is_neox_style) + key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) + + # Modification: convert to the correct dtype + query = query.to(self.dtype) + key = key.to(self.dtype) + return query, key + + +class FlashInferRotaryEmbedding(RotaryEmbedding): + def forward_cuda( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + offsets: Optional[torch.Tensor] = None, + fused_set_kv_buffer_arg: Optional[FusedSetKVBufferArg] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + + apply_rope_with_cos_sin_cache_inplace( + positions=positions, + query=query, + key=key, + fused_set_kv_buffer_arg=fused_set_kv_buffer_arg, + head_size=self.head_size, + cos_sin_cache=self.cos_sin_cache, + is_neox=self.is_neox_style, + ) + + return query, key + + +class MHATokenToKVPool: + KV_POOL_SIZE = 16384 + + def __init__( + self, + head_num: int, + head_dim: int, + ): + self.head_num = head_num + self.head_dim = head_dim + self.size = MHATokenToKVPool.KV_POOL_SIZE + self.page_size = 1 + self.store_dtype = torch.bfloat16 + self.device = "cuda" + self.layer_num = 1 + self.start_layer = 0 + self._create_buffers() + + def _create_buffers(self): + self.k_buffer = [ + torch.zeros( + (self.size + self.page_size, self.head_num, self.head_dim), + dtype=self.store_dtype, + device=self.device, + ) + for _ in range(self.layer_num) + ] + self.v_buffer = [ + torch.zeros( + (self.size + self.page_size, self.head_num, self.head_dim), + dtype=self.store_dtype, + device=self.device, + ) + for _ in range(self.layer_num) + ] + + def set_kv_buffer( + self, + loc: torch.Tensor, + cache_k: torch.Tensor, + cache_v: torch.Tensor, + ): + layer_id = 0 + self.k_buffer[layer_id - self.start_layer][loc] = cache_k + self.v_buffer[layer_id - self.start_layer][loc] = cache_v + + +def create_inputs( + head_size: int, + batch_size: int, + seq_len: int, + device, + dtype: torch.dtype, + num_q_heads: int, + num_kv_heads: int, +): + pos_ids = torch.arange(seq_len, device=device).repeat(batch_size) + query = torch.randn( + batch_size * seq_len, num_q_heads * head_size, dtype=dtype, device=device + ) + key = torch.randn( + batch_size * seq_len, num_kv_heads * head_size, dtype=dtype, device=device + ) + value = torch.randn( + batch_size * seq_len, num_kv_heads * head_size, dtype=dtype, device=device + ) + out_cache_loc = torch.randperm( + MHATokenToKVPool.KV_POOL_SIZE, dtype=torch.int64, device=device + )[: batch_size * seq_len].clone() + + return dict( + pos_ids=pos_ids, query=query, key=key, value=value, out_cache_loc=out_cache_loc + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/top_k.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/top_k.py new file mode 100644 index 00000000..77fd5e5e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/top_k.py @@ -0,0 +1,117 @@ +from typing import Optional + +import torch + + +def fast_topk(values, topk, dim): + if topk == 1: + # Use max along the specified dimension to get both value and index + return torch.max(values, dim=dim, keepdim=True) + else: + # Use topk for efficiency with larger k values + # TODO: implement faster cuda kernels for large vocab sizes + return torch.topk(values, topk, dim=dim) + + +def fast_topk_v2( + score: torch.Tensor, + lengths: torch.Tensor, + topk: int, + row_starts: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """ + Get the topk indices of the score tensor. + Args: + score: The score tensor of shape (B, L). The score tensor is the logits + between the query and the key whose layout is either ragged or paged. + row_starts is only required when the key is ragged. + lengths: The lengths tensor of shape (B) + topk: The number of topk indices to get + row_starts: The start index of each row in the score tensor of shape (B). + For each row i, topk only applies to section [row_starts[i], row_starts[i] + lengths[i]] + of the score tensor. + Returns: + The topk indices tensor of shape (B, topk) + """ + assert ( + topk == 2048 + ), "fast_topk_v2 is only optimized for deepseek v3.2 model, where topk=2048" + assert score.dim() == 2 + topk_indices = score.new_empty((score.size(0), topk), dtype=torch.int32) + torch.ops.sgl_kernel.fast_topk(score, topk_indices, lengths, row_starts) + return topk_indices + + +def fast_topk_transform_fused( + score: torch.Tensor, + lengths: torch.Tensor, + page_table_size_1: torch.Tensor, # NOTE: page size should be 1 + cu_seqlens_q: torch.Tensor, + topk: int, + row_starts: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """ + Get the topk indices of the score tensor and then transform the topk indices + to indices to the page table (page_size = 1) + Args: + score: The score tensor of shape (B, L). The score tensor is the logits + between the query and the key whose layout is either ragged or paged. + row_starts is only required when the key is ragged. + lengths: The lengths tensor of shape (B) + page_table_size_1: The page table tensor of shape (Batch, topk) + cu_seqlens_q: The cumulative sequence lengths tensor of shape (Batch + 1) + topk: The number of topk indices to get + row_starts: The start index of each row in the score tensor of shape (B). + For each row i, topk only applies to section [row_starts[i], row_starts[i] + lengths[i]] + of the score tensor. It's only used for cases where the key is + ragged, i.e. during extend and draft extend. + Returns: + The topk indices tensor of shape (B, topk) + """ + assert ( + topk == 2048 + ), "fast_topk_transform_fused is only optimized for deepseek v3.2 model, where topk=2048" + assert score.dim() == 2 + src_page_table = page_table_size_1 + dst_page_table = score.new_empty((score.shape[0], topk), dtype=torch.int32) + torch.ops.sgl_kernel.fast_topk_transform_fused( + score, lengths, dst_page_table, src_page_table, cu_seqlens_q, row_starts + ) + return dst_page_table + + +def fast_topk_transform_ragged_fused( + score: torch.Tensor, + lengths: torch.Tensor, + topk_indices_offset: torch.Tensor, # ragged kv + topk: int, + row_starts: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """ + Get the topk indices of the score tensor and then transform the topk indices to + indices to ragged kv (non-paged). This function is only used for extend, + not including draft extend. + Args: + score: The score tensor of shape (B, L). The score tensor is the logits + between the query and the key which can be ragged or paged. + row_starts is only required when the key is ragged. + lengths: The lengths tensor of shape (B) + topk_indices_offset: The offset of topk indices in ragged kv of shape (B) + topk: The number of topk indices to get + row_starts: The start index of each row in the score tensor of shape (B). + For each row i, topk only applies to section [row_starts[i], row_starts[i] + lengths[i]] + of the score tensor. It can be None if only the fast path is triggered, + in the case of all values in lengths <= topk (not checked in the kernel, + guaranteed by the caller). + Returns: + The topk indices tensor of shape (B, topk) + """ + assert ( + topk == 2048 + ), "fast_topk_transform_ragged_fused is only optimized for deepseek v3.2 model, where topk=2048" + assert score.dim() == 2 + topk_indices_ragged = score.new_empty((score.shape[0], topk), dtype=torch.int32) + torch.ops.sgl_kernel.fast_topk_transform_ragged_fused( + score, lengths, topk_indices_ragged, topk_indices_offset, row_starts + ) + return topk_indices_ragged diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/utils.py new file mode 100644 index 00000000..d03476ef --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/utils.py @@ -0,0 +1,45 @@ +# Copyright 2025 SGLang Team. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import functools +from typing import Dict, Tuple + +import torch + +_cache_buf: Dict[Tuple[str, torch.device], torch.Tensor] = {} + + +def _get_cache_buf(name: str, bytes: int, device: torch.device) -> torch.Tensor: + key = (name, device) + buf = _cache_buf.get(key) + if buf is None: + buf = torch.empty(bytes, dtype=torch.uint8, device=device) + _cache_buf[key] = buf + return buf + + +def _to_tensor_scalar_tuple(x): + if isinstance(x, torch.Tensor): + return (x, 0) + else: + return (None, x) + + +@functools.lru_cache(maxsize=1) +def is_arch_support_pdl() -> bool: + # Hopper arch's compute capability == 9.0 + device = torch.cuda.current_device() + major, minor = torch.cuda.get_device_capability(device) + return major >= 9 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/version.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/version.py new file mode 100644 index 00000000..69edc1cf --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/python/sgl_kernel/version.py @@ -0,0 +1 @@ +__version__ = "0.3.18.post2" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/rename_wheels.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/rename_wheels.sh new file mode 100755 index 00000000..915f069e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/rename_wheels.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -ex + +WHEEL_DIR="dist" + +wheel_files=($WHEEL_DIR/*.whl) +for wheel in "${wheel_files[@]}"; do + intermediate_wheel="${wheel/linux/manylinux2014}" + + # Extract the current python version from the wheel name + if [[ $intermediate_wheel =~ -cp([0-9]+)- ]]; then + cp_version="${BASH_REMATCH[1]}" + else + echo "Could not extract Python version from wheel name: $intermediate_wheel" + continue + fi + + # Detect CUDA version and add appropriate suffix + if ls /usr/local/ | grep -q "12.4"; then + new_wheel="${intermediate_wheel/-cp${cp_version}/+cu124-cp${cp_version}}" + elif ls /usr/local/ | grep -q "12.8"; then + new_wheel="${intermediate_wheel/-cp${cp_version}/+cu128-cp${cp_version}}" + elif ls /usr/local/ | grep -q "13.0"; then + new_wheel="${intermediate_wheel/-cp${cp_version}/+cu130-cp${cp_version}}" + else + new_wheel="$intermediate_wheel" + fi + + if [[ "$wheel" != "$new_wheel" ]]; then + echo "Renaming $wheel to $new_wheel" + mv -- "$wheel" "$new_wheel" + fi +done +echo "Wheel renaming completed." diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/setup_rocm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/setup_rocm.py new file mode 100644 index 00000000..8cdb7c69 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/setup_rocm.py @@ -0,0 +1,117 @@ +# Copyright 2025 SGLang Team. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import os +import platform +import sys +from pathlib import Path + +import torch +from setuptools import find_packages, setup +from torch.utils.cpp_extension import BuildExtension, CUDAExtension + +root = Path(__file__).parent.resolve() +arch = platform.machine().lower() + + +def _get_version(): + with open(root / "pyproject.toml") as f: + for line in f: + if line.startswith("version"): + return line.split("=")[1].strip().strip('"') + + +operator_namespace = "sgl_kernel" +include_dirs = [ + root / "include", + root / "include" / "impl", + root / "csrc", +] + +sources = [ + "csrc/allreduce/custom_all_reduce.hip", + "csrc/allreduce/quick_all_reduce.cu", + "csrc/common_extension_rocm.cc", + "csrc/elementwise/activation.cu", + "csrc/grammar/apply_token_bitmask_inplace_cuda.cu", + "csrc/moe/moe_align_kernel.cu", + "csrc/moe/moe_topk_softmax_kernels.cu", + "csrc/moe/moe_topk_sigmoid_kernels.cu", + "csrc/speculative/eagle_utils.cu", + "csrc/kvcacheio/transfer.cu", +] + +cxx_flags = ["-O3"] +libraries = ["hiprtc", "amdhip64", "c10", "torch", "torch_python"] +extra_link_args = ["-Wl,-rpath,$ORIGIN/../../torch/lib", f"-L/usr/lib/{arch}-linux-gnu"] + +default_target = "gfx942" +amdgpu_target = os.environ.get("AMDGPU_TARGET", default_target) + +if torch.cuda.is_available(): + try: + amdgpu_target = torch.cuda.get_device_properties(0).gcnArchName.split(":")[0] + except Exception as e: + print(f"Warning: Failed to detect GPU properties: {e}") +else: + print(f"Warning: torch.cuda not available. Using default target: {amdgpu_target}") + +if amdgpu_target not in ["gfx942", "gfx950"]: + print( + f"Warning: Unsupported GPU architecture detected '{amdgpu_target}'. Expected 'gfx942' or 'gfx950'." + ) + sys.exit(1) + +fp8_macro = ( + "-DHIP_FP8_TYPE_FNUZ" if amdgpu_target == "gfx942" else "-DHIP_FP8_TYPE_E4M3" +) + +hipcc_flags = [ + "-DNDEBUG", + f"-DOPERATOR_NAMESPACE={operator_namespace}", + "-O3", + "-Xcompiler", + "-fPIC", + "-std=c++17", + f"--amdgpu-target={amdgpu_target}", + "-DENABLE_BF16", + "-DENABLE_FP8", + fp8_macro, +] + +ext_modules = [ + CUDAExtension( + name="sgl_kernel.common_ops", + sources=sources, + include_dirs=include_dirs, + extra_compile_args={ + "nvcc": hipcc_flags, + "cxx": cxx_flags, + }, + libraries=libraries, + extra_link_args=extra_link_args, + py_limited_api=False, + ), +] + +setup( + name="sgl-kernel", + version=_get_version(), + packages=find_packages(where="python"), + package_dir={"": "python"}, + ext_modules=ext_modules, + cmdclass={"build_ext": BuildExtension.with_options(use_ninja=True)}, + options={"bdist_wheel": {"py_limited_api": "cp39"}}, +) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/conftest.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/conftest.py new file mode 100644 index 00000000..4aae7ffb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/conftest.py @@ -0,0 +1,14 @@ +import pytest +import torch + + +# This fixture ensures the torch defaults don't get left in modified states between +# tests (e.g., when a test fails before restoring the original value), which +# can cause subsequent tests to fail. +@pytest.fixture(autouse=True) +def reset_torch_defaults(): + orig_default_device = torch.get_default_device() + orig_default_dtype = torch.get_default_dtype() + yield + torch.set_default_dtype(orig_default_dtype) + torch.set_default_device(orig_default_device) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/spatial/test_greenctx_stream.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/spatial/test_greenctx_stream.py new file mode 100644 index 00000000..c57bc336 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/spatial/test_greenctx_stream.py @@ -0,0 +1,25 @@ +import pytest +import torch +import torch.nn.functional as F +from sgl_kernel import create_greenctx_stream_by_value, get_sm_available + + +def test_green_ctx(): + A = torch.randn(5120, 5120).cuda() + B = torch.randn(5120, 5120).cuda() + C = torch.matmul(A, B) + sm_counts = get_sm_available(0) + stream_group = create_greenctx_stream_by_value(sm_counts // 2, sm_counts // 2, 0) + with torch.cuda.stream(stream_group[0]): + for _ in range(100): + result_0 = torch.matmul(A, B) + with torch.cuda.stream(stream_group[1]): + for _ in range(100): + result_1 = torch.matmul(A, B) + torch.cuda.synchronize() + assert torch.allclose(result_0, C) + assert torch.allclose(result_1, C) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/speculative/test_eagle_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/speculative/test_eagle_utils.py new file mode 100644 index 00000000..50335538 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/speculative/test_eagle_utils.py @@ -0,0 +1,87 @@ +import pytest +import torch +import torch.nn.functional as F +from sgl_kernel import verify_tree_greedy + + +def test_verify_tree_greedy(): + candidates = torch.tensor( + [ + [0, 1, 2, 3, 4, 5], + [7, 8, 9, 10, 11, 12], + ], + dtype=torch.int64, + device="cuda", + ) + retrive_index = torch.tensor( + [ + [0, 1, 2, 3, 4, 5], + [6, 7, 8, 9, 10, 11], + ], + dtype=torch.int64, + device="cuda", + ) + retrive_next_token = torch.tensor( + [ + [1, 2, -1, 4, 5, -1], + [4, 2, 3, -1, 5, -1], + ], + dtype=torch.int64, + device="cuda", + ) + retrive_next_sibling = torch.tensor( + [ + [-1, 3, -1, -1, -1, -1], + [-1, -1, -1, -1, 1, -1], + ], + dtype=torch.int64, + device="cuda", + ) + + target_logits = torch.full((2, 6, 20), 1, dtype=torch.float32, device="cuda") + target_logits[0, 0, 3] = 10 + target_logits[0, 3, 4] = 10 + target_logits[0, 4, 5] = 10 + target_logits[1, 0, 11] = 10 + target_logits[1, 4, 12] = 10 + for i in range(target_logits.shape[0]): + for j in range(target_logits.shape[1]): + if torch.max(target_logits[i][j]) < 10: + target_logits[i][j][18] = 10 + + target_predict = torch.argmax(target_logits, dim=-1) + predict_shape = (12,) + + bs = candidates.shape[0] + num_spec_step = 4 + + predicts = torch.full( + predict_shape, -1, dtype=torch.int32, device="cuda" + ) # mutable + accept_index = torch.full( + (bs, num_spec_step), -1, dtype=torch.int32, device="cuda" + ) # mutable + accept_token_num = torch.full((bs,), 0, dtype=torch.int32, device="cuda") # mutable + + verify_tree_greedy( + predicts=predicts, + accept_index=accept_index, + accept_token_num=accept_token_num, + candidates=candidates, + retrive_index=retrive_index, + retrive_next_token=retrive_next_token, + retrive_next_sibling=retrive_next_sibling, + target_predict=target_predict, + ) + + # Check the expected output. + assert predicts.tolist() == [3, -1, -1, 4, 5, 18, 11, -1, -1, -1, 12, 18] + assert accept_index.tolist() == [ + [0, 3, 4, 5], + [6, 10, 11, -1], + ] + assert accept_token_num.tolist() == [3, 2] + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/speculative/test_ngram_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/speculative/test_ngram_utils.py new file mode 100644 index 00000000..29bf89f9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/speculative/test_ngram_utils.py @@ -0,0 +1,76 @@ +import pytest +import torch +import torch.nn.functional as F +from sgl_kernel import reconstruct_indices_from_tree_mask + + +def test_reconstruct_indices_from_tree_mask(): + bs = 1 + num_branch_token = 4 + seq_lens = torch.tensor([12], device="cuda", dtype=torch.int64) + + retrive_index = torch.full( + (bs, num_branch_token), -1, device="cuda", dtype=torch.int64 + ) + retrive_next_token = torch.full( + (bs, num_branch_token), -1, device="cuda", dtype=torch.int64 + ) + retrive_next_sibling = torch.full( + (bs, num_branch_token), -1, device="cuda", dtype=torch.int64 + ) + positions = torch.empty((bs * num_branch_token), device="cuda", dtype=torch.int64) + + tree_mask = torch.tensor( + [ + 1, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 1, + ], + device="cuda", + dtype=torch.int32, + ).to(torch.bool) + + reconstruct_indices_from_tree_mask( + tree_mask, + seq_lens, + positions, # mutable + retrive_index, # mutable + retrive_next_token, # mutable + retrive_next_sibling, # mutable + bs, + num_branch_token, + ) + # print(f"debug: \n\n{tree_mask=}, {retrive_index=}, {retrive_next_token=}, {retrive_next_sibling=}, {positions=}\n\n") + assert retrive_index.tolist() == [ + [0, 1, 2, 3], + ], f"{retrive_index=}" + assert retrive_next_token.tolist() == [ + [1, -1, 3, -1], + ], f"{retrive_next_token=}" + assert retrive_next_sibling.tolist() == [ + [-1, 2, -1, -1], + ], f"{retrive_next_sibling=}" + assert positions.tolist() == [ + 12, + 13, + 13, + 14, + ], f"{positions=}" + + +if __name__ == "__main__": + test_reconstruct_indices_from_tree_mask() + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/speculative/test_speculative_sampling.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/speculative/test_speculative_sampling.py new file mode 100644 index 00000000..a9b59bb2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/speculative/test_speculative_sampling.py @@ -0,0 +1,129 @@ +import pytest +import torch +import torch.nn.functional as F +from sgl_kernel import tree_speculative_sampling_target_only + +test_cases = [ + ( + 1, + 1, + [3, -1, -1, 4, 5, 18, 11, -1, -1, -1, 12, 18], + [[0, 3, 4, 5], [6, 10, 11, -1]], + [3, 2], + ), + ( + 0, # threshold_single + 0, # threshold_acc + [1, 2, 18, -1, -1, -1, 11, -1, -1, -1, 12, 18], + [[0, 1, 2, -1], [6, 10, 11, -1]], + [2, 2], + ), +] + + +@pytest.mark.parametrize( + "threshold_single, threshold_acc, expected_predicts, expected_accept_index, expected_accept_token_num", + test_cases, +) +def test_tree_speculative_sampling_target_only( + threshold_single, + threshold_acc, + expected_predicts, + expected_accept_index, + expected_accept_token_num, +): + """ + Tests the tree_speculative_sampling_target_only function using Pytest parameterization. + """ + device = "cuda" + + candidates = torch.tensor( + [ + [0, 1, 2, 3, 4, 5], + [7, 8, 9, 10, 11, 12], + ], + dtype=torch.int64, + device=device, + ) + retrive_index = torch.tensor( + [ + [0, 1, 2, 3, 4, 5], + [6, 7, 8, 9, 10, 11], + ], + dtype=torch.int64, + device=device, + ) + retrive_next_token = torch.tensor( + [ + [1, 2, -1, 4, 5, -1], + [4, 2, 3, -1, 5, -1], + ], + dtype=torch.int64, + device=device, + ) + retrive_next_sibling = torch.tensor( + [ + [-1, 3, -1, -1, -1, -1], + [-1, -1, -1, -1, 1, -1], + ], + dtype=torch.int64, + device=device, + ) + + target_logits = torch.full((2, 6, 20), 1, dtype=torch.float32, device=device) + target_logits[0, 0, 3] = 10 + target_logits[0, 3, 4] = 10 + target_logits[0, 4, 5] = 10 + target_logits[1, 0, 11] = 10 + target_logits[1, 4, 12] = 10 + + for i in range(target_logits.shape[0]): + for j in range(target_logits.shape[1]): + if torch.max(target_logits[i, j]) < 10: + target_logits[i, j, 18] = 10 + + temperatures = torch.tensor([0.01, 0.01], dtype=torch.float32, device=device) + bs, num_draft_tokens = candidates.shape + num_spec_step = len(expected_accept_index[0]) + predict_shape = (len(expected_predicts),) + + predicts = torch.full(predict_shape, -1, dtype=torch.int32, device=device) + accept_index = torch.full((bs, num_spec_step), -1, dtype=torch.int32, device=device) + accept_token_num = torch.full((bs,), 0, dtype=torch.int32, device=device) + + expanded_temperature = temperatures.unsqueeze(1).unsqueeze(1) + target_probs = F.softmax(target_logits / expanded_temperature, dim=-1) + draft_probs = torch.full_like(target_probs, 0, dtype=torch.float32, device=device) + coins = torch.rand(bs, num_draft_tokens, device=device, dtype=torch.float32) + coins_for_final_sampling = torch.rand(bs, device=device).to(torch.float32) + + tree_speculative_sampling_target_only( + predicts=predicts, + accept_index=accept_index, + accept_token_num=accept_token_num, + candidates=candidates, + retrive_index=retrive_index, + retrive_next_token=retrive_next_token, + retrive_next_sibling=retrive_next_sibling, + uniform_samples=coins, + uniform_samples_for_final_sampling=coins_for_final_sampling, + target_probs=target_probs, + draft_probs=draft_probs, + threshold_single=threshold_single, + threshold_acc=threshold_acc, + deterministic=True, + ) + + assert ( + predicts.tolist() == expected_predicts + ), f"Predicts mismatch for thresholds ({threshold_single}, {threshold_acc})" + assert ( + accept_index.tolist() == expected_accept_index + ), f"Accept index mismatch for thresholds ({threshold_single}, {threshold_acc})" + assert ( + accept_token_num.tolist() == expected_accept_token_num + ), f"Accept token num mismatch for thresholds ({threshold_single}, {threshold_acc})" + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_activation.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_activation.py new file mode 100644 index 00000000..43593441 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_activation.py @@ -0,0 +1,39 @@ +# Adapted from https://github.com/flashinfer-ai/flashinfer/blob/4e8eb1879f9c3ba6d75511e5893183bf8f289a62/tests/test_activation.py + +import pytest +import sgl_kernel +import torch + + +@pytest.mark.parametrize("dim", [128, 256, 512, 2048, 4096, 11008, 16384]) +@pytest.mark.parametrize("batch_size", [1, 2, 4, 8, 16]) +@pytest.mark.parametrize("seq_len", [1, 2, 4, 8, 16, 32, 64, 128, 512]) +def test_fused_silu_mul(dim, batch_size, seq_len): + x = torch.randn(batch_size, seq_len, 2 * dim).to(0).to(torch.float16) + y_ref = x[..., dim:] * torch.nn.functional.silu(x[..., :dim]) + y = sgl_kernel.silu_and_mul(x) + torch.testing.assert_close(y_ref, y, rtol=1e-3, atol=1e-3) + + +@pytest.mark.parametrize("dim", [128, 256, 512, 2048, 4096, 11008, 16384]) +@pytest.mark.parametrize("batch_size", [1, 2, 4, 8, 16]) +@pytest.mark.parametrize("seq_len", [1, 2, 4, 8, 16, 32, 64, 128, 512]) +def test_fused_gelu_tanh_mul(dim, batch_size, seq_len): + x = torch.randn(batch_size, seq_len, 2 * dim).to(0).to(torch.float16) + y_ref = x[..., dim:] * torch.nn.functional.gelu(x[..., :dim], approximate="tanh") + y = sgl_kernel.gelu_tanh_and_mul(x) + torch.testing.assert_close(y_ref, y, rtol=1e-3, atol=1e-3) + + +@pytest.mark.parametrize("dim", [128, 256, 512, 2048, 4096, 11008, 16384]) +@pytest.mark.parametrize("batch_size", [1, 2, 4, 8, 16]) +@pytest.mark.parametrize("seq_len", [1, 2, 4, 8, 16, 32, 64, 128, 512]) +def test_fused_gelu_mul(dim, batch_size, seq_len): + x = torch.randn(batch_size, seq_len, 2 * dim).to(0).to(torch.float16) + y_ref = x[..., dim:] * torch.nn.functional.gelu(x[..., :dim], approximate="none") + y = sgl_kernel.gelu_and_mul(x) + torch.testing.assert_close(y_ref, y, rtol=1e-3, atol=1e-3) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_apply_token_bitmask_inplace.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_apply_token_bitmask_inplace.py new file mode 100644 index 00000000..48047913 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_apply_token_bitmask_inplace.py @@ -0,0 +1,23 @@ +import pytest +import torch +from sgl_kernel import apply_token_bitmask_inplace_cuda + + +def test_apply_token_bitmask_inplace_kernel(): + neginf = float("-inf") + bool_mask = torch.tensor([0, 1, 0, 1, 0, 1, 0, 1, 0, 1], dtype=torch.bool) + logits = torch.tensor( + [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0], dtype=torch.float32 + ) + expected = torch.where(bool_mask, logits, neginf) + + logits_gpu = logits.to("cuda") + bitmask = torch.tensor([0b1010101010], dtype=torch.int32).to("cuda") + apply_token_bitmask_inplace_cuda(logits_gpu, bitmask) + torch.cuda.synchronize() + torch.testing.assert_close(logits_gpu, expected.to("cuda")) + + +if __name__ == "__main__": + test_apply_token_bitmask_inplace_kernel() + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_awq_dequant.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_awq_dequant.py new file mode 100644 index 00000000..da68e88d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_awq_dequant.py @@ -0,0 +1,115 @@ +import itertools +from typing import Optional, Tuple + +import pytest +import torch +from sgl_kernel import awq_dequantize + + +def reverse_awq_order(t: torch.Tensor): + bits = 4 + AWQ_REVERSE_ORDER = [0, 4, 1, 5, 2, 6, 3, 7] + reverse_order_tensor = torch.arange( + t.shape[-1], + dtype=torch.int32, + device=t.device, + ) + reverse_order_tensor = reverse_order_tensor.view(-1, 32 // bits) + reverse_order_tensor = reverse_order_tensor[:, AWQ_REVERSE_ORDER] + reverse_order_tensor = reverse_order_tensor.view(-1) + + t = t[:, reverse_order_tensor] & 0xF + return t + + +# qweights - [R , C // 8], int32 +# scales - [R // G, C ], float16 +# zeros - [R // G, C // 8], int32 +def awq_dequantize_torch( + qweight: torch.Tensor, scales: torch.Tensor, qzeros: torch.Tensor, group_size: int +) -> torch.Tensor: + + if group_size == -1: + group_size = qweight.shape[0] + + bits = 4 + shifts = torch.arange(0, 32, bits, device=qzeros.device) + + iweights = torch.bitwise_right_shift(qweight[:, :, None], shifts[None, None, :]).to( + torch.int8 + ) + + iweights = iweights.view(iweights.shape[0], -1) + + zeros = torch.bitwise_right_shift(qzeros[:, :, None], shifts[None, None, :]).to( + torch.int8 + ) + zeros = zeros.view(qzeros.shape[0], -1) + zeros = reverse_awq_order(zeros) + + iweights = reverse_awq_order(iweights) + + iweights = torch.bitwise_and(iweights, (2**bits) - 1) + zeros = torch.bitwise_and(zeros, (2**bits) - 1) + + scales = scales.repeat_interleave(group_size, dim=0) + zeros = zeros.repeat_interleave(group_size, dim=0) + return (iweights - zeros) * scales + + +def sglang_awq_dequantize( + qweight: torch.Tensor, scales: torch.Tensor, qzeros: torch.Tensor +) -> torch.Tensor: + return awq_dequantize(qweight, scales, qzeros) + + +@pytest.mark.parametrize( + "qweight_row,qweight_col,is_bf16_act", + list( + itertools.product( + [3584, 18944, 128, 256, 512, 1024, 1536], + [448, 576, 4736, 16, 32, 64, 128, 72], + [True, False], + ) + ), +) +def test_awq_dequant_compare_implementations( + qweight_row: int, qweight_col: int, is_bf16_act: bool +): + device = torch.device("cuda") + qweight = torch.randint( + 0, + torch.iinfo(torch.int32).max, + (qweight_row, qweight_col), + dtype=torch.int32, + device=device, + ) + group_size = qweight_row + scales_row = qweight_row // group_size + scales_col = qweight_col * 8 + + if is_bf16_act: + scales = torch.rand(scales_row, scales_col, dtype=torch.bfloat16, device=device) + else: + scales = torch.rand(scales_row, scales_col, dtype=torch.float16, device=device) + + qzeros = torch.randint( + 0, + torch.iinfo(torch.int32).max, + (scales_row, qweight_col), + dtype=torch.int32, + device=device, + ) + + # Run both implementations + torch_out = awq_dequantize_torch(qweight, scales, qzeros, group_size) + sglang_out = sglang_awq_dequantize(qweight, scales, qzeros) + + # Compare results + torch.testing.assert_close( + torch_out.to(torch.float32), sglang_out.to(torch.float32), rtol=1e-3, atol=1e-5 + ) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_bmm_fp8.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_bmm_fp8.py new file mode 100644 index 00000000..e0be9289 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_bmm_fp8.py @@ -0,0 +1,43 @@ +# Adapted from https://github.com/flashinfer-ai/flashinfer/blob/4e8eb1879f9c3ba6d75511e5893183bf8f289a62/tests/test_bmm_fp8.py + +import pytest +import torch +import torch.nn.functional as F +from sgl_kernel import bmm_fp8 + + +def to_float8(x, dtype=torch.float8_e4m3fn): + finfo = torch.finfo(dtype) + min_val, max_val = x.aminmax() + amax = torch.maximum(min_val.abs(), max_val.abs()).clamp(min=1e-12) + scale = finfo.max / amax + x_scl_sat = (x * scale).clamp(min=finfo.min, max=finfo.max) + return x_scl_sat.to(dtype), scale.float().reciprocal() + + +@pytest.mark.parametrize("input_dtype", [torch.float8_e4m3fn, torch.float8_e5m2]) +@pytest.mark.parametrize("mat2_dtype", [torch.float8_e4m3fn, torch.float8_e5m2]) +@pytest.mark.parametrize("res_dtype", [torch.bfloat16, torch.float16]) +def test_bmm_fp8(input_dtype, mat2_dtype, res_dtype): + if input_dtype == torch.float8_e5m2 and mat2_dtype == torch.float8_e5m2: + pytest.skip("Invalid combination: both input and mat2 are e5m2") + + input = torch.randn([16, 48, 64], device="cuda", dtype=torch.bfloat16) + input_fp8, input_inv_s = to_float8(input, dtype=input_dtype) + + # mat2 row major -> column major + mat2 = torch.randn([16, 80, 64], device="cuda", dtype=torch.bfloat16).transpose( + -2, -1 + ) + mat2_fp8, mat2_inv_s = to_float8(mat2, dtype=mat2_dtype) + + res = torch.empty([16, 48, 80], device="cuda", dtype=res_dtype) + bmm_fp8(input_fp8, mat2_fp8, input_inv_s, mat2_inv_s, res_dtype, res) + + reference = torch.bmm(input, mat2) + cos_sim = F.cosine_similarity(reference.reshape(-1), res.reshape(-1), dim=0) + assert cos_sim > 0.99 + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_causal_conv1d.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_causal_conv1d.py new file mode 100644 index 00000000..a10e1f45 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_causal_conv1d.py @@ -0,0 +1,489 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/main/tests/kernels/mamba/test_causal_conv1d.py +from typing import Optional + +import torch +from sgl_kernel import causal_conv1d_fwd +from sgl_kernel import causal_conv1d_update as causal_conv1d_update_kernel + +PAD_SLOT_ID = -1 + + +def causal_conv1d_fn( + x: torch.Tensor, + weight: torch.Tensor, + bias: Optional[torch.Tensor] = None, + query_start_loc: Optional[torch.Tensor] = None, + cache_indices: Optional[torch.Tensor] = None, + has_initial_state: Optional[torch.Tensor] = None, + conv_states: Optional[torch.Tensor] = None, + activation: Optional[str] = "silu", + pad_slot_id: int = PAD_SLOT_ID, +): + """ + x: (batch, dim, seqlen) or (dim,cu_seq_len) for varlen + sequences are concatenated from left to right for varlen + weight: (dim, width) + bias: (dim,) + query_start_loc: (batch + 1) int32 + The cumulative sequence lengths of the sequences in + the batch, used to index into sequence. prepended by 0. + for example: query_start_loc = torch.Tensor([0,10,16,17]), + x.shape=(dim,17) + cache_indices: (batch) int32 + indicates the corresponding state index, + like so: conv_state = conv_states[cache_indices[batch_id]] + has_initial_state: (batch) bool + indicates whether should the kernel take the current state as initial + state for the calculations + conv_states: (...,dim,width - 1) itype + updated inplace if provided + activation: either None or "silu" or "swish" + pad_slot_id: int + if cache_indices is passed, lets the kernel identify padded + entries that will not be processed, + for example: cache_indices = [pad_slot_id, 1, 20, pad_slot_id] + in this case, the kernel will not process entries at + indices 0 and 3 + out: (batch, dim, seqlen) + """ + if activation not in [None, "silu", "swish"]: + raise NotImplementedError("activation must be None, silu, or swish") + if x.stride(-1) != 1: + x = x.contiguous() + bias = bias.contiguous() if bias is not None else None + + causal_conv1d_fwd( + x, + weight, + bias, + conv_states, + query_start_loc, + cache_indices, + has_initial_state, + activation in ["silu", "swish"], + pad_slot_id, + ) + return x + + +def causal_conv1d_update( + x: torch.Tensor, + conv_state: torch.Tensor, + weight: torch.Tensor, + bias: Optional[torch.Tensor] = None, + activation: Optional[str] = None, + cache_seqlens: Optional[torch.Tensor] = None, + conv_state_indices: Optional[torch.Tensor] = None, + pad_slot_id: int = PAD_SLOT_ID, +): + """ + x: (batch, dim) or (batch, dim, seqlen) + conv_state: (batch, dim, state_len), where state_len >= width - 1 + weight: (dim, width) + bias: (dim,) + cache_seqlens: (batch,), dtype int32. + If not None, the conv_state is treated as a circular buffer. + The conv_state will be updated by copying x to the conv_state + starting at the index + @cache_seqlens % state_len. + conv_state_indices: (batch,), dtype int32 + If not None, the conv_state is a larger tensor along the batch dim, + and we are selecting the batch coords specified by conv_state_indices. + Useful for a continuous batching scenario. + pad_slot_id: int + if cache_indices is passed, lets the kernel identify padded + entries that will not be processed, + for example: cache_indices = [pad_slot_id, 1 ,20 ,pad_slot_id] + in this case, the kernel will not process entries at + indices 0 and 3 + out: (batch, dim) or (batch, dim, seqlen) + """ + if activation not in [None, "silu", "swish"]: + raise NotImplementedError( + f"activation must be None, silu, or swish, actual: {activation}" + ) + activation_val = activation in ["silu", "swish"] + unsqueeze = x.dim() == 2 + if unsqueeze: + x = x.unsqueeze(-1) + causal_conv1d_update_kernel( + x, + conv_state, + weight, + bias, + activation_val, + cache_seqlens, + conv_state_indices, + pad_slot_id, + ) + if unsqueeze: + x = x.squeeze(-1) + return x + + +# SPDX-License-Identifier: Apache-2.0 + +from typing import Optional + +import pytest +import torch +import torch.nn.functional as F + + +def causal_conv1d_ref( + x: torch.Tensor, + weight: torch.Tensor, + bias: Optional[torch.Tensor] = None, + initial_states: Optional[torch.Tensor] = None, + return_final_states: bool = False, + final_states_out: Optional[torch.Tensor] = None, + activation: Optional[str] = "silu", +): + """ + x: (batch, dim, seqlen) + weight: (dim, width) + bias: (dim,) + initial_states: (batch, dim, width - 1) + final_states_out: (batch, dim, width - 1) + + out: (batch, dim, seqlen) + """ + if activation not in [None, "silu", "swish"]: + raise NotImplementedError("activation must be None, silu, or swish") + dtype_in = x.dtype + x = x.to(weight.dtype) + seqlen = x.shape[-1] + dim, width = weight.shape + if initial_states is None: + out = F.conv1d(x, weight.unsqueeze(1), bias, padding=width - 1, groups=dim) + else: + x = torch.cat([initial_states, x], dim=-1) + out = F.conv1d(x, weight.unsqueeze(1), bias, padding=0, groups=dim) + out = out[..., :seqlen] + if return_final_states: + final_states = F.pad(x, (width - 1 - x.shape[-1], 0)).to( + dtype_in + ) # (batch, dim, width - 1) + if final_states_out is not None: + final_states_out.copy_(final_states) + else: + final_states_out = final_states + out = (out if activation is None else F.silu(out)).to(dtype=dtype_in) + return (out, None) if not return_final_states else (out, final_states_out) + + +def causal_conv1d_update_ref( + x, conv_state, weight, bias=None, activation=None, cache_seqlens=None +): + """ + x: (batch, dim) or (batch, dim, seqlen) + conv_state: (batch, dim, state_len), where state_len >= width - 1 + weight: (dim, width) + bias: (dim,) + cache_seqlens: (batch,), dtype int32. + If not None, the conv_state is treated as a circular buffer. + The conv_state will be updated by copying x to the + conv_state starting at the index + @cache_seqlens % state_len before performing the convolution. + + out: (batch, dim) or (batch, dim, seqlen) + """ + if activation not in [None, "silu", "swish"]: + raise NotImplementedError("activation must be None, silu, or swish") + dtype_in = x.dtype + unsqueeze = x.dim() == 2 + if unsqueeze: + x = x.unsqueeze(-1) + batch, dim, seqlen = x.shape + width = weight.shape[1] + state_len = conv_state.shape[-1] + assert conv_state.shape == (batch, dim, state_len) + assert weight.shape == (dim, width) + if cache_seqlens is None: + x_new = torch.cat([conv_state, x], dim=-1).to( + weight.dtype + ) # (batch, dim, state_len + seqlen) + conv_state.copy_(x_new[:, :, -state_len:]) + else: + width_idx = torch.arange( + -(width - 1), 0, dtype=torch.long, device=x.device + ).unsqueeze(0) + cache_seqlens.unsqueeze(1) + width_idx = ( + torch.remainder(width_idx, state_len).unsqueeze(1).expand(-1, dim, -1) + ) + x_new = torch.cat([conv_state.gather(2, width_idx), x], dim=-1).to(weight.dtype) + copy_idx = torch.arange(seqlen, dtype=torch.long, device=x.device).unsqueeze( + 0 + ) + cache_seqlens.unsqueeze(1) + copy_idx = torch.remainder(copy_idx, state_len).unsqueeze(1).expand(-1, dim, -1) + conv_state.scatter_(2, copy_idx, x) + out = F.conv1d(x_new, weight.unsqueeze(1), bias, padding=0, groups=dim)[ + :, :, -seqlen: + ] + if unsqueeze: + out = out.squeeze(-1) + return (out if activation is None else F.silu(out)).to(dtype=dtype_in) + + +@pytest.mark.parametrize("itype", [torch.bfloat16, torch.float]) +@pytest.mark.parametrize("silu_activation", [True]) +@pytest.mark.parametrize("has_bias", [True]) +@pytest.mark.parametrize("has_initial_state", [True, False]) +@pytest.mark.parametrize("width", [4]) +@pytest.mark.parametrize( + "seqlen", [1, 8, 16, 32, 64, 128, 256, 512, 784, 1024, 1025, 2048, 4096] +) +@pytest.mark.parametrize("dim", [64]) +@pytest.mark.parametrize("batch", [1]) +def test_causal_conv1d( + batch, dim, seqlen, width, has_bias, silu_activation, has_initial_state, itype +): + device = "cuda" + rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3) + if itype == torch.bfloat16: + rtol, atol = 1e-2, 5e-2 + x = torch.randn(batch, dim, seqlen, device=device, dtype=itype).contiguous() + + weight = torch.randn(dim, width, device=device, dtype=itype) + bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None + if has_initial_state: + initial_states = torch.randn(batch, dim, width - 1, device=device, dtype=itype) + has_initial_state_tensor = torch.ones(batch, dtype=torch.bool, device=x.device) + else: + initial_states = None + has_initial_state_tensor = None + x_ref = x.clone() + weight_ref = weight.clone() + bias_ref = bias.clone() if bias is not None else None + initial_states_ref = initial_states.clone() if initial_states is not None else None + activation = None if not silu_activation else "silu" + out = causal_conv1d_fn( + x, + weight, + bias, + activation=activation, + conv_states=initial_states, + has_initial_state=has_initial_state_tensor, + ) + out_ref, final_states_ref = causal_conv1d_ref( + x_ref, + weight_ref, + bias_ref, + initial_states=initial_states_ref, + return_final_states=True, + activation=activation, + ) + if has_initial_state: + assert initial_states is not None and final_states_ref is not None + assert torch.allclose(initial_states, final_states_ref, rtol=rtol, atol=atol) + assert torch.allclose(out, out_ref, rtol=rtol, atol=atol) + + +@pytest.mark.parametrize("itype", [torch.bfloat16]) +@pytest.mark.parametrize("silu_activation", [False, True]) +@pytest.mark.parametrize("has_bias", [False, True]) +@pytest.mark.parametrize("seqlen", [1]) +@pytest.mark.parametrize("width", [4]) +@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096]) +def test_causal_conv1d_update(dim, width, seqlen, has_bias, silu_activation, itype): + device = "cuda" + rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3) + if itype == torch.bfloat16: + rtol, atol = 1e-2, 5e-2 + + batch = 2 + x = torch.randn(batch, dim, seqlen, device=device, dtype=itype) + x_ref = x.clone() + conv_state = torch.randn(batch, dim, width - 1, device=device, dtype=itype) + + weight = torch.randn(dim, width, device=device, dtype=itype) + bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None + conv_state_ref = conv_state.detach().clone() + activation = None if not silu_activation else "silu" + out = causal_conv1d_update(x, conv_state, weight, bias, activation=activation) + out_ref = causal_conv1d_update_ref( + x_ref, conv_state_ref, weight, bias, activation=activation + ) + + assert torch.equal(conv_state, conv_state_ref) + assert torch.allclose(out, out_ref, rtol=rtol, atol=atol) + + +@pytest.mark.parametrize("itype", [torch.float32, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("silu_activation", [False, True]) +@pytest.mark.parametrize("has_bias", [False, True]) +@pytest.mark.parametrize("seqlen", [1, 4, 5]) +@pytest.mark.parametrize("width", [2, 3, 4]) +@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096]) +# tests correctness in case subset of the sequences are padded +@pytest.mark.parametrize("with_padding", [True, False]) +def test_causal_conv1d_update_with_batch_gather( + with_padding, dim, width, seqlen, has_bias, silu_activation, itype +): + device = "cuda" + rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3) + if itype == torch.bfloat16: + rtol, atol = 1e-2, 5e-2 + + batch_size = 3 + padding = 5 if with_padding else 0 + padded_batch_size = batch_size + padding + total_entries = 10 * batch_size + + x = torch.randn(padded_batch_size, dim, 1, device=device, dtype=itype) + x_ref = x.clone() + + conv_state_indices = torch.randperm(total_entries)[:batch_size].to( + dtype=torch.int32, device=device + ) + unused_states_bool = torch.ones(total_entries, dtype=torch.bool, device=device) + unused_states_bool[conv_state_indices] = False + padded_state_indices = torch.concat( + [ + conv_state_indices, + torch.as_tensor([PAD_SLOT_ID] * padding, dtype=torch.int32, device=device), + ], + dim=0, + ) + conv_state = torch.randn(total_entries, dim, width - 1, device=device, dtype=itype) + conv_state_for_padding_test = conv_state.clone() + + weight = torch.randn(dim, width, device=device, dtype=itype) + bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None + conv_state_ref = conv_state[conv_state_indices, :].detach().clone() + activation = None if not silu_activation else "silu" + out = causal_conv1d_update( + x, + conv_state, + weight, + bias, + activation=activation, + conv_state_indices=padded_state_indices, + pad_slot_id=PAD_SLOT_ID, + ) + out_ref = causal_conv1d_update_ref( + x_ref[:batch_size], conv_state_ref, weight, bias, activation=activation + ) + + assert torch.equal(conv_state[conv_state_indices, :], conv_state_ref) + assert torch.allclose(out[:batch_size], out_ref, rtol=rtol, atol=atol) + assert torch.equal( + conv_state[unused_states_bool], conv_state_for_padding_test[unused_states_bool] + ) + + +@pytest.mark.parametrize("itype", [torch.bfloat16]) +@pytest.mark.parametrize("silu_activation", [True]) +@pytest.mark.parametrize("has_bias", [True]) +@pytest.mark.parametrize("width", [4]) +@pytest.mark.parametrize( + "seqlen", [8, 16, 32, 64, 128, 256, 512, 784, 1024, 2048, 2049, 4096] +) +@pytest.mark.parametrize("dim", [64, 4096]) +# tests correctness in case subset of the sequences are padded +@pytest.mark.parametrize("with_padding", [True, False]) +def test_causal_conv1d_varlen( + with_padding, dim, seqlen, width, has_bias, silu_activation, itype +): + device = "cuda" + torch.cuda.empty_cache() + rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3) + if itype == torch.bfloat16: + rtol, atol = 1e-2, 5e-2 + + seqlens = [] + batch_size = 4 + if seqlen < 10: + batch_size = 1 + padding = 3 if with_padding else 0 + padded_batch_size = batch_size + padding + nsplits = padded_batch_size - 1 + + eos_pos = torch.randperm(seqlen - 1)[:nsplits].sort().values + seqlens.append( + torch.diff( + torch.cat([torch.tensor([-1]), eos_pos, torch.tensor([seqlen - 1])]) + ).tolist() + ) + assert sum(seqlens[-1]) == seqlen + assert all(s > 0 for s in seqlens[-1]) + + total_entries = batch_size * 10 + cumsum = torch.cumsum(torch.tensor(seqlens[0]), dim=0).to(torch.int32) + cumsum = torch.concat([torch.tensor([0], dtype=torch.int32), cumsum], dim=0) + x = torch.randn(1, 4096 + dim + 64, seqlen, device=device, dtype=itype)[ + :, 4096 : 4096 + dim, : + ] + weight = torch.randn(dim, width, device=device, dtype=itype) + bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None + x_ref = x.clone() + weight_ref = weight.clone() + bias_ref = bias.clone() if bias is not None else None + activation = None if not silu_activation else "silu" + final_states = torch.randn( + total_entries, dim, width - 1, device=x.device, dtype=x.dtype + ) + final_states_ref = final_states.clone() + has_initial_states = torch.randint( + 0, 2, (cumsum.shape[0] - 1,), dtype=torch.bool, device=x.device + ) + state_indices = torch.randperm(total_entries, dtype=torch.int32, device=x.device)[ + :batch_size + ] + padded_state_indices = torch.concat( + [ + state_indices, + torch.as_tensor([PAD_SLOT_ID] * padding, dtype=torch.int32, device=device), + ], + dim=-1, + ) + + out = causal_conv1d_fn( + x.squeeze(0), + weight, + bias, + cumsum.cuda(), + padded_state_indices, + has_initial_states, + final_states, + activation, + PAD_SLOT_ID, + ) + out_ref = [] + out_ref_b = [] + + splits = [torch.split(var, seqlens[0], dim=-1) for var in (x_ref)] + for i in range(len(seqlens[0])): + x_s = [v[i].unsqueeze(0) for v in splits][0] + if padded_state_indices[i] == PAD_SLOT_ID: + continue + out_ref_b.append( + causal_conv1d_ref( + x_s, + weight_ref, + bias_ref, + activation=activation, + return_final_states=True, + final_states_out=final_states_ref[padded_state_indices[i]].unsqueeze(0), + initial_states=( + final_states_ref[padded_state_indices[i]].unsqueeze(0) + if has_initial_states[i] + else None + ), + ) + ) + out_ref.append(torch.cat([t[0] for t in out_ref_b], dim=2)) + out_ref_tensor = torch.cat(out_ref, dim=0) + + unpadded_out = out[:, : out_ref_tensor.shape[-1]] + assert torch.allclose(unpadded_out, out_ref_tensor, rtol=rtol, atol=atol) + assert torch.allclose( + final_states[state_indices], + final_states_ref[state_indices], + rtol=rtol, + atol=atol, + ) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_copy.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_copy.py new file mode 100644 index 00000000..70ed864c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_copy.py @@ -0,0 +1,16 @@ +import pytest +import sgl_kernel +import torch +from sgl_kernel.elementwise import copy_to_gpu_no_ce + + +@pytest.mark.parametrize("size", [64, 72]) +def test_copy_to_gpu_no_ce(size): + tensor_cpu = torch.randint(0, 1000000, (size,), dtype=torch.int32, device="cpu") + tensor_gpu = torch.empty_like(tensor_cpu, device="cuda") + copy_to_gpu_no_ce(tensor_cpu, tensor_gpu) + assert torch.all(tensor_cpu.cuda() == tensor_gpu) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_custom_allreduce.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_custom_allreduce.py new file mode 100644 index 00000000..d729392c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_custom_allreduce.py @@ -0,0 +1,185 @@ +import ctypes +import multiprocessing as mp +import random +import socket +import unittest +from typing import Any, List, Optional + +import sgl_kernel.allreduce as custom_ops +import torch +import torch.distributed as dist +from torch.distributed import ProcessGroup + +from sglang.srt.distributed.device_communicators.cuda_wrapper import CudaRTLibrary + + +def _run_correctness_worker(world_size, rank, distributed_init_port, test_sizes): + device = torch.device(f"cuda:{rank}") + torch.cuda.set_device(device) + distributed_init_method = f"tcp://localhost:{distributed_init_port}" + dist.init_process_group( + backend="nccl", + init_method=distributed_init_method, + rank=rank, + world_size=world_size, + ) + group = dist.group.WORLD + + try: + device = torch.device(f"cuda:{rank}") + max_size = 8192 * 1024 + meta_ptrs = TestCustomAllReduce.create_shared_buffer( + custom_ops.meta_size() + max_size, group=group + ) + + rank_data = torch.empty(8 * 1024 * 1024, dtype=torch.uint8, device=device) + buffer_ptrs = TestCustomAllReduce.create_shared_buffer(max_size, group=group) + + custom_ptr = custom_ops.init_custom_ar(meta_ptrs, rank_data, rank, True) + custom_ops.register_buffer(custom_ptr, buffer_ptrs) + + test_loop = 10 + for sz in test_sizes: + for dtype in [torch.float32, torch.float16, torch.bfloat16]: + for _ in range(test_loop): + inp1 = torch.randint(1, 16, (sz,), dtype=dtype, device=device) + inp1_ref = inp1.clone() + out1 = torch.empty_like(inp1) + + custom_ops.all_reduce( + custom_ptr, inp1, out1, buffer_ptrs[rank], max_size + ) + + dist.all_reduce(inp1_ref, group=group) + + torch.testing.assert_close(out1, inp1_ref) + + finally: + dist.barrier(group=group) + if custom_ptr is not None: + custom_ops.dispose(custom_ptr) + if buffer_ptrs: + TestCustomAllReduce.free_shared_buffer(buffer_ptrs, group) + if meta_ptrs: + TestCustomAllReduce.free_shared_buffer(meta_ptrs, group) + + dist.destroy_process_group(group=group) + + +def get_open_port() -> int: + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + except OSError: + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: + s.bind(("::1", 0)) + return s.getsockname()[1] + + +def multi_process_parallel( + world_size: int, test_target: Any, target_args: tuple = () +) -> None: + mp.set_start_method("spawn", force=True) + + procs = [] + distributed_init_port = get_open_port() + for i in range(world_size): + proc_args = (world_size, i, distributed_init_port) + target_args + proc = mp.Process(target=test_target, args=proc_args, name=f"Worker-{i}") + proc.start() + procs.append(proc) + + for i in range(world_size): + procs[i].join() + assert ( + procs[i].exitcode == 0 + ), f"Process {i} failed with exit code {procs[i].exitcode}" + + +class TestCustomAllReduce(unittest.TestCase): + test_sizes = [ + 512, + 2560, + 4096, + 5120, + 7680, + 32768, + 262144, + 524288, + 1048576, + 2097152, + ] + world_sizes = [2, 4, 8] + + @staticmethod + def create_shared_buffer( + size_in_bytes: int, group: Optional[ProcessGroup] = None + ) -> List[int]: + lib = CudaRTLibrary() + pointer = lib.cudaMalloc(size_in_bytes) + handle = lib.cudaIpcGetMemHandle(pointer) + if group is None: + group = dist.group.WORLD + world_size = dist.get_world_size(group=group) + rank = dist.get_rank(group=group) + + handle_bytes = ctypes.string_at(ctypes.addressof(handle), ctypes.sizeof(handle)) + input_tensor = torch.ByteTensor(list(handle_bytes)).to(f"cuda:{rank}") + gathered_tensors = [torch.empty_like(input_tensor) for _ in range(world_size)] + dist.all_gather(gathered_tensors, input_tensor, group=group) + + handles = [] + handle_type = type(handle) + for tensor in gathered_tensors: + bytes_list = tensor.cpu().tolist() + bytes_data = bytes(bytes_list) + handle_obj = handle_type() + ctypes.memmove(ctypes.addressof(handle_obj), bytes_data, len(bytes_data)) + handles.append(handle_obj) + + pointers: List[int] = [] + for i, h in enumerate(handles): + if i == rank: + pointers.append(pointer.value) + else: + try: + opened_ptr = lib.cudaIpcOpenMemHandle(h) + pointers.append(opened_ptr.value) + except Exception as e: + print(f"Rank {rank}: Failed to open IPC handle from rank {i}: {e}") + raise + + dist.barrier(group=group) + return pointers + + @staticmethod + def free_shared_buffer( + pointers: List[int], group: Optional[ProcessGroup] = None + ) -> None: + if group is None: + group = dist.group.WORLD + rank = dist.get_rank(group=group) + lib = CudaRTLibrary() + if pointers and len(pointers) > rank and pointers[rank] is not None: + lib.cudaFree(ctypes.c_void_p(pointers[rank])) + dist.barrier(group=group) + + def test_correctness(self): + for world_size in self.world_sizes: + available_gpus = torch.cuda.device_count() + if world_size > available_gpus: + print( + f"Skipping world_size={world_size}, requires {world_size} GPUs, found {available_gpus}" + ) + continue + + print(f"Running test for world_size={world_size}") + multi_process_parallel( + world_size, _run_correctness_worker, target_args=(self.test_sizes,) + ) + print(f"custom allreduce tp = {world_size}: OK") + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_cutlass_mla.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_cutlass_mla.py new file mode 100644 index 00000000..71de8327 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_cutlass_mla.py @@ -0,0 +1,104 @@ +import pytest +import torch +import torch.nn.functional as F +from sgl_kernel import cutlass_mla_decode, cutlass_mla_get_workspace_size +from torch import Tensor + +# Disable tests on SM103 until the accuracy issues are fixed. +if torch.cuda.get_device_capability() != (10, 0): + pytest.skip( + reason="Cutlass MLA Requires compute capability of 10.", + allow_module_level=True, + ) + + +def ref_mla( + out: Tensor, # (bs, num_heads, v_head_dim) + query: Tensor, # (bs, num_heads, head_dim) + kv_cache: Tensor, # (num_blocks, block_size, head_dim) + scale: float, + block_tables: Tensor, # (bs, max_num_blocks) + seq_lens: Tensor, # (bs,) +): + bs, num_heads, v_head_dim = out.shape + head_dim = query.shape[2] + + for i in range(bs): + # gather and flatten KV-cache + kv = kv_cache[block_tables[i]] # (max_num_blocks, block_size, head_dim) + kv = kv.view(1, -1, head_dim)[:, : seq_lens[i]] # (1, seq_len, head_dim) + v = kv[:, :, :v_head_dim] + + q = query[i].view(num_heads, 1, head_dim) + o = F.scaled_dot_product_attention(q, kv, v, scale=scale, enable_gqa=True) + out[i] = o.view(num_heads, v_head_dim) + + return out + + +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +@pytest.mark.parametrize("mean_seq_len", [128, 1024, 4096]) +@pytest.mark.parametrize("bs", [1, 2, 4]) +@pytest.mark.parametrize("varlen", [False, True]) +@pytest.mark.parametrize("block_size", [1, 16, 64, 128]) +@pytest.mark.parametrize("num_heads", [16, 32, 64, 128]) +@pytest.mark.parametrize("num_kv_splits", [-1, 1]) +def test_cutlass_mla_decode( + dtype: torch.dtype, + mean_seq_len: int, + bs: int, + varlen: bool, + block_size: int, + num_heads: int, + num_kv_splits: int, +): + torch.set_default_dtype(dtype) + torch.set_default_device("cuda") + torch.manual_seed(42) + + d = 576 + h_q = num_heads + dv = 512 + + q_nope_dim = 128 + q_pe_dim = 64 + scale = (q_nope_dim + q_pe_dim) ** (-0.5) + if varlen: + seq_lens = torch.empty(bs).normal_(mean_seq_len, mean_seq_len / 2) + seq_lens = seq_lens.clip(2).to(torch.int32) + else: + seq_lens = torch.full((bs,), mean_seq_len, dtype=torch.int32) + max_seq_len = seq_lens.max().item() + block_num = (max_seq_len + block_size - 1) // block_size + + # Pad block_num so that small blocks can be packed into full 128-sized CUTLASS tiles. + # One 128-wide tile can hold (128 // block_size) small blocks. + pack_factor = 128 // block_size + block_num = ((block_num + pack_factor - 1) // pack_factor) * pack_factor + + # Lager q values to detect split kv error + q = torch.randn(bs, h_q, d) * 100.0 + block_table = torch.randint(0, bs * block_num, (bs, block_num), dtype=torch.int32) + + kv_cache = torch.randn(block_table.numel(), block_size, d) + + workspace_size = cutlass_mla_get_workspace_size( + block_num * block_size, bs, num_kv_splits=num_kv_splits + ) + workspace = torch.empty(workspace_size, device="cuda", dtype=torch.uint8) + + q_nope = torch.empty((h_q, bs, dv)).transpose(0, 1) + q_nope.copy_(q[:, :, :dv]) + q_pe = q[:, :, dv:].clone() + + out_ref = q.new_zeros(bs, h_q, dv) + ref_mla(out_ref, q, kv_cache, scale, block_table, seq_lens) + out = cutlass_mla_decode( + q_nope, q_pe, kv_cache, seq_lens, block_table, workspace, scale, num_kv_splits + ) + + torch.testing.assert_close(out, out_ref, atol=1e-2, rtol=1e-2) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_cutlass_w4a8_moe_mm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_cutlass_w4a8_moe_mm.py new file mode 100644 index 00000000..7acba566 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_cutlass_w4a8_moe_mm.py @@ -0,0 +1,283 @@ +import pytest +import torch +from sgl_kernel import cutlass_w4a8_moe_mm, sgl_per_tensor_quant_fp8 +from utils import is_hopper + + +def pack_int4_values_to_int8(int4_values_interleaved: torch.Tensor) -> torch.Tensor: + if int4_values_interleaved.shape[-1] % 2 != 0: + raise ValueError( + "the last dim size of int4_values_interleaved tensor must be even." + ) + + input_tensor_int8 = int4_values_interleaved.to(torch.int8) + + low_nibbles = input_tensor_int8[..., 0::2] + high_nibbles = input_tensor_int8[..., 1::2] + + packed_tensor = (high_nibbles << 4) | (low_nibbles & 0x0F) + + return packed_tensor.to(torch.int8) + + +def pack_interleave(num_experts, ref_weight, ref_scale): + n, k = ref_weight.shape[1], ref_weight.shape[2] + + weight = pack_int4_values_to_int8(ref_weight.cpu()).cuda() + w_q = weight.view((num_experts, n, k // 2)).view(torch.int8) + w_q = w_q.contiguous() + + alignment = 4 if k % 512 == 0 else 1 + scale_interleaved = ref_scale.reshape( + ref_scale.shape[0], + ref_scale.shape[1], + (ref_scale.shape[2] // alignment), + alignment, + ) # [E, N, K/4, 4] + scale_interleaved = scale_interleaved.permute(0, 2, 1, 3) # [E, K/4, N, 4] + scale_interleaved = scale_interleaved.reshape( + ref_scale.shape[0], + ref_scale.shape[2] // alignment, + ref_scale.shape[1] * alignment, + ) # [E, K/4, N*4] + w_scale = scale_interleaved.contiguous() + + return w_q, w_scale + + +@pytest.mark.skipif( + not is_hopper(), + reason="cutlass_w4a8_moe_mm is only supported on sm90", +) +@pytest.mark.parametrize("batch_size", [1, 2, 4, 8, 16]) +def test_int4_fp8_grouped_gemm_single_expert(batch_size): + # Test parameters + num_experts = 1 + m = batch_size # batch size + k = 512 # input dimension + n = 1024 # output dimension + torch.manual_seed(0) + dtype = torch.bfloat16 + device = "cuda" + debug = False + + print(f"\nTesting with batch_size={batch_size}") + + # Create input tensors with ones + if debug: + a = torch.ones(m, k, dtype=torch.bfloat16, device=device) + ref_w = torch.ones(num_experts, n, k, dtype=torch.int8, device=device) + ref_w_scale = torch.ones(num_experts, n, k // 128, dtype=dtype, device=device) + else: + a = torch.randn(m, k, dtype=dtype, device=device) + ref_w = torch.randint( + -8, 8, (num_experts, n, k), dtype=torch.int8, device=device + ) + affine_coeff = 0.005 + ref_w_scale = ( + torch.randn(num_experts, n, k // 128, dtype=dtype, device=device) + * affine_coeff + ) + + w, w_scale = pack_interleave(num_experts, ref_w, ref_w_scale) + + # Create expert offsets and problem sizes + expert_offsets = torch.tensor([0, m], dtype=torch.int32, device=device) + problem_sizes = torch.tensor([[n, m, k]], dtype=torch.int32, device=device) + + a_strides = torch.full((num_experts, 3), k, device=device, dtype=torch.int64) + c_strides = torch.full((num_experts, 3), n, device=device, dtype=torch.int64) + b_strides = a_strides + s_strides = c_strides + + # Quantize input + a_q, a_scale = _per_tensor_quant_fp8(a) + + # Create output tensor + c = torch.empty((m, n), dtype=torch.bfloat16, device=device) + cutlass_w4a8_moe_mm( + c, + a_q, + w, + a_scale, + w_scale, + expert_offsets[:-1], + problem_sizes, + a_strides, + b_strides, + c_strides, + s_strides, + 128, + 8, + ) + c = c.to(dtype) + + # Reference implementation + experts_selection_result = torch.full((m,), 0) + c_ref = ref_grouped_gemm( + c, a_q, a_scale, ref_w, ref_w_scale, num_experts, experts_selection_result + ) + + # Compare results + try: + torch.testing.assert_close(c, c_ref, rtol=1e-2, atol=0.1) + except AssertionError as e: + # torch.set_printoptions(threshold=10_000) + print(f" FAILURE: tensors are NOT close.") + print(f" Ref tensor: {c_ref.flatten()}") + print(f" Cutlass tensor: {c.flatten()}") + print( + f" Max absolute difference: {torch.max(torch.abs(c.to(c_ref.dtype) - c_ref))}" + ) + print( + f" Mean absolute difference: {torch.mean(torch.abs(c.to(c_ref.dtype) - c_ref))}" + ) + print(f" AssertionError: {e}") + raise + + +def _per_tensor_quant_fp8( + x: torch.Tensor, + dtype: torch.dtype = torch.float8_e4m3fn, +): + assert x.is_contiguous(), "`x` is not contiguous" + + x_q = torch.empty_like(x, device=x.device, dtype=dtype) + x_s = torch.empty( + 1, + device=x.device, + dtype=torch.float32, + ) + sgl_per_tensor_quant_fp8(x, x_q, x_s, is_static=False) + return x_q, x_s + + +@pytest.mark.skipif( + not is_hopper(), + reason="cutlass_w4a8_moe_mm is only supported on sm90", +) +@pytest.mark.parametrize("batch_size", [2, 4, 8, 16, 32]) +@pytest.mark.parametrize("k", [256, 512, 1024, 2048, 4096, 7168]) +@pytest.mark.parametrize("n", [256, 512, 1024, 2048, 7168]) +@pytest.mark.parametrize("num_experts", [2, 4, 6, 8]) +def test_int4_fp8_grouped_gemm_multi_experts(batch_size, k, n, num_experts): + torch.manual_seed(0) + dtype = torch.bfloat16 + device = "cuda" + debug = False + + print( + f"\nTesting with batch_size={batch_size}, k={k}, n={n}, num_experts={num_experts}" + ) + + if debug: + a = torch.ones(batch_size, k, dtype=torch.bfloat16, device=device) + ref_w = torch.ones(num_experts, n, k, dtype=torch.int8, device=device) + ref_w_scale = torch.ones(num_experts, n, k // 128, dtype=dtype, device=device) + else: + a = torch.randn(batch_size, k, dtype=dtype, device=device) + ref_w = torch.randint( + -8, 8, (num_experts, n, k), dtype=torch.int8, device=device + ) + affine_coeff = 0.005 + ref_w_scale = ( + torch.randn(num_experts, n, k // 128, dtype=dtype, device=device) + * affine_coeff + ) + + w, w_scale = pack_interleave(num_experts, ref_w, ref_w_scale) + + # random select experts + experts_selection_result = torch.randint( + 0, num_experts, (batch_size,), device=device + ) + permutation = torch.argsort(experts_selection_result) + expert_token_counts = torch.bincount( + experts_selection_result, minlength=num_experts + ) + + # Create problem sizes and offsets for active experts + problem_sizes = [] + for i in range(num_experts): + problem_sizes.append([n, expert_token_counts[i].item(), k]) + problem_sizes = torch.tensor(problem_sizes, dtype=torch.int32, device=device) + + expert_offsets = [] + offset = 0 + for i in range(num_experts): + expert_offsets.append(offset) + offset += problem_sizes[i][1].item() + expert_offsets = torch.tensor(expert_offsets, dtype=torch.int32, device=device) + + # Permute input and quantize + a_q, a_scale = _per_tensor_quant_fp8(a) + a_q_perm = a_q[permutation] + + # Create stride tensors + a_strides = torch.full((num_experts, 3), k, device=device, dtype=torch.int64) + c_strides = torch.full((num_experts, 3), n, device=device, dtype=torch.int64) + b_strides = a_strides + s_strides = c_strides + + c_perm = torch.empty((batch_size, n), dtype=torch.bfloat16, device=device) + cutlass_w4a8_moe_mm( + c_perm, + a_q_perm, + w, + a_scale, + w_scale, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + c_strides, + s_strides, + 128, + 8, + ) + + # Un-permute the result + c = torch.empty_like(c_perm) + c[permutation] = c_perm + c = c.to(dtype) + + c_ref = ref_grouped_gemm( + c, a_q, a_scale, ref_w, ref_w_scale, num_experts, experts_selection_result + ) + + # Compare results + try: + torch.testing.assert_close(c, c_ref, rtol=1e-2, atol=0.1) + except AssertionError as e: + print(f" FAILURE: tensors are NOT close.") + print( + f" Max absolute difference: {torch.max(torch.abs(c.to(c_ref.dtype) - c_ref))}" + ) + print( + f" Mean absolute difference: {torch.mean(torch.abs(c.to(c_ref.dtype) - c_ref))}" + ) + print(f" AssertionError: {e}") + raise + + +def ref_grouped_gemm( + c, a_q, a_scale, w, w_scale, num_experts, experts_selection_result +): + dtype = torch.bfloat16 + c_ref = torch.zeros_like(c) + for i in range(num_experts): + token_idx = torch.where(experts_selection_result == i)[0] + if len(token_idx) == 0: + continue + a = a_q[token_idx] + + ref_w_scale_repeat = w_scale[i].repeat_interleave(128, dim=1).to(torch.float32) + ref_w = w[i].to(torch.float32) * ref_w_scale_repeat + c = torch.matmul(a.to(torch.float32), ref_w.t()) * a_scale + c_ref[token_idx] = c.to(dtype) + + return c_ref + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_dsv3_fused_a_gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_dsv3_fused_a_gemm.py new file mode 100644 index 00000000..914af95e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_dsv3_fused_a_gemm.py @@ -0,0 +1,32 @@ +import pytest +import torch +import torch.nn.functional as F +from sgl_kernel import dsv3_fused_a_gemm + + +@pytest.mark.parametrize("num_tokens", [i + 1 for i in range(16)]) +def test_dsv3_fused_a_gemm(num_tokens): + kHdIn = 7168 + kHdOut = 2112 + + mat_a = torch.randn( + (num_tokens, kHdIn), dtype=torch.bfloat16, device="cuda" + ).contiguous() + mat_b = torch.randn((kHdOut, kHdIn), dtype=torch.bfloat16, device="cuda").transpose( + 0, 1 + ) + output = torch.empty( + (num_tokens, kHdOut), dtype=torch.bfloat16, device="cuda" + ).contiguous() + + ref = F.linear(mat_a, mat_b.T) + + output = dsv3_fused_a_gemm(mat_a, mat_b) + + assert torch.allclose( + output, ref, rtol=1e-2, atol=1e-3 + ), "Fused GEMM output mismatch with torch.nn.functional.linear reference" + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_dsv3_router_gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_dsv3_router_gemm.py new file mode 100644 index 00000000..575769d6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_dsv3_router_gemm.py @@ -0,0 +1,35 @@ +import pytest +import torch +import torch.nn.functional as F +from sgl_kernel import dsv3_router_gemm + + +@pytest.mark.parametrize("num_tokens", [i + 1 for i in range(16)]) +@pytest.mark.parametrize("num_experts", [256, 384]) +def test_dsv3_router_gemm(num_tokens, num_experts): + hidden_dim = 7168 + + mat_a = torch.randn( + (num_tokens, hidden_dim), dtype=torch.bfloat16, device="cuda" + ).contiguous() + mat_b = torch.randn( + (num_experts, hidden_dim), dtype=torch.bfloat16, device="cuda" + ).contiguous() + + bf16_ref = F.linear(mat_a, mat_b) + float_ref = bf16_ref.to(torch.float32) + + bf16_output = dsv3_router_gemm(mat_a, mat_b, out_dtype=torch.bfloat16) + float_output = dsv3_router_gemm(mat_a, mat_b, out_dtype=torch.float32) + + assert torch.allclose( + bf16_output, bf16_ref, rtol=1e-2, atol=1e-3 + ), "Router GEMM output in bf16 dtype mismatch with torch.nn.functional.linear reference" + + assert torch.allclose( + float_output, float_ref, rtol=1e-2, atol=1e-3 + ), "Router GEMM output in float32 dtype mismatch with torch.nn.functional.linear reference" + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_es_fp8_blockwise_moe.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_es_fp8_blockwise_moe.py new file mode 100644 index 00000000..cd5bd6d6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_es_fp8_blockwise_moe.py @@ -0,0 +1,205 @@ +import random +from typing import Tuple + +import pytest +import torch +from sgl_kernel import es_fp8_blockwise_scaled_grouped_mm + + +def cdiv(a: int, b: int) -> int: + return -(a // -b) + + +def scale_shape(shape, group_shape): + return tuple(cdiv(shape[i], group_shape[i]) for i in range(len(group_shape))) + + +def to_fp8(tensor: torch.Tensor) -> torch.Tensor: + finfo = torch.finfo(torch.float8_e4m3fn) + return torch.round(tensor.clamp(min=finfo.min, max=finfo.max)).to( + dtype=torch.float8_e4m3fn + ) + + +# Copy from: https://github.com/deepseek-ai/DeepGEMM/blob/main/deep_gemm/utils.py +def calc_diff(x, y): + x, y = x.double(), y.double() + denominator = (x * x + y * y).sum() + sim = 2 * (x * y).sum() / denominator + return 1 - sim + + +def ceil_div(x: int, y: int) -> int: + return (x + y - 1) // y + + +def per_token_cast_to_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 + m, n = x.shape + pad_size = (128 - (n % 128)) % 128 + x = torch.nn.functional.pad(x, (0, pad_size), value=0) if pad_size > 0 else x + x_view = x.view(m, -1, 128) + x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4) + fp8_data = (x_view * (448.0 / x_amax.unsqueeze(2))).to(torch.float8_e4m3fn) + return fp8_data.view(m, n + pad_size)[:, :n], (x_amax / 448.0).view(m, -1) + + +def per_block_cast_to_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 + m, n = x.shape + x_padded = torch.zeros( + (ceil_div(m, 128) * 128, ceil_div(n, 128) * 128), dtype=x.dtype, device=x.device + ) + x_padded[:m, :n] = x + x_view = x_padded.view(-1, 128, x_padded.size(1) // 128, 128) + x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4) + x_scaled = (x_view * (448.0 / x_amax)).to(torch.float8_e4m3fn) + return x_scaled.view_as(x_padded)[:m, :n].contiguous(), (x_amax / 448.0).view( + x_view.size(0), x_view.size(2) + ) + + +def baseline_scaled_mm( + a: torch.Tensor, + b: torch.Tensor, + scale_a: torch.Tensor, + scale_b: torch.Tensor, + out_dtype: type[torch.dtype], +) -> torch.Tensor: + + def group_broadcast(t, shape): + for i, s in enumerate(shape): + if t.shape[i] != s and t.shape[i] != 1: + assert s % t.shape[i] == 0 + t = ( + t.unsqueeze(i + 1) + .expand(*t.shape[: i + 1], s // t.shape[i], *t.shape[i + 1 :]) + .flatten(i, i + 1) + ) + return t + + scale_a = group_broadcast(scale_a, a.shape) + scale_b = group_broadcast(scale_b, b.shape) + + return torch.mm( + (scale_a * a.to(dtype=torch.float32)), (scale_b * b.to(dtype=torch.float32)) + ).to(out_dtype) + + +def is_sm100_supported(device=None) -> bool: + return (torch.cuda.get_device_capability(device)[0] == 10) and ( + torch.version.cuda >= "12.8" + ) + + +def is_sm90_supported(device=None) -> bool: + return (torch.cuda.get_device_capability(device)[0] == 9) and ( + torch.version.cuda >= "12.3" + ) + + +@pytest.mark.skipif( + not is_sm90_supported(), + reason="es_fp8_blockwise_scaled_grouped_mm at sgl-kernel is only supported on sm90", +) +@pytest.mark.parametrize("num_experts", [8, 16, 32, 64, 128]) +@pytest.mark.parametrize("out_dtype", [torch.half, torch.bfloat16]) +def test_fp8_blockwise_scaled_grouped_mm(num_experts, out_dtype): + device = "cuda" + alignment = 128 + n_g = random.randint(1, 64) * alignment + k_g = random.randint(1, 64) * alignment + + expert_offsets = torch.zeros((num_experts + 1), device=device, dtype=torch.int32) + problem_sizes = torch.zeros((num_experts, 3), device=device, dtype=torch.int32) + a_tensors = [] + b_tensors = [] + a_scales_tensors = [] + b_scales_tensors = [] + baseline_tensors = [] + + for g in range(num_experts): + m_g = random.randint(1, 256) + expert_offsets[g + 1] = expert_offsets[g] + m_g + problem_sizes[g][:] = torch.tensor([m_g, n_g, k_g], device=device) + + a = torch.randn((m_g, k_g), device=device, dtype=out_dtype) # (M, K):(K, 1) + b = torch.randn((n_g, k_g), device=device, dtype=out_dtype).t() # (K, N):(1, K) + + a_g, a_scale = per_token_cast_to_fp8( + a + ) # ag -- (M, K):(K, 1), a_scale() -- (M, k):(k, 1) + b_g, b_scale = per_block_cast_to_fp8( + b + ) # bg -- (K, N):(N, 1), b_scale() -- (k, n):(n, 1) + a_tensors.append(a_g) + b_tensors.append(b_g) + a_scales_tensors.append(a_scale) + b_scales_tensors.append(b_scale) + + baseline = torch.mm(a, b) + baseline_tensors.append(baseline) + a_stack = torch.empty( + (expert_offsets[-1], k_g), device=device, dtype=torch.float8_e4m3fn + ) + b_stack = torch.empty( + (num_experts, n_g, k_g), device=device, dtype=torch.float8_e4m3fn + ) + a_scale_stack = torch.empty( + (expert_offsets[-1], (k_g // 128)), device=device, dtype=torch.float32 + ) + b_scale_stack = torch.empty( + (num_experts, n_g // 128, k_g // 128), device=device, dtype=torch.float32 + ) + + for g in range(num_experts): + # Matrix A is Row-Major. + a_stack[expert_offsets[g] : expert_offsets[g + 1], :] = a_tensors[ + g + ] # a_stack[expert_offsets[g] : expert_offsets[g + 1], :] -- (M, K):(K, 1) + b_stack[g] = b_tensors[g].t() # b_stack[g] -- (N, K):(K, 1) + + # We need K-Major scale factor + a_scale_stack[expert_offsets[g] : expert_offsets[g + 1], :] = a_scales_tensors[ + g + ] + b_scale_stack[g] = b_scales_tensors[ + g + ].t() # b_scale_stack[g] -- (k, n):(n, 1), we need transpose & contiguous later + b_stack = b_stack.transpose(1, 2) # Transpose Matrix B to Column-Major. + b_scale_stack = b_scale_stack.transpose(1, 2) + workspace = torch.empty((1024 * 1024 * 1024), device=device, dtype=torch.uint8) + c_out = torch.empty((expert_offsets[-1], n_g), device=device, dtype=out_dtype) + a_strides = torch.full( + (num_experts,), a_stack.stride(0), device=device, dtype=torch.int64 + ) + d_strides = torch.full( + (num_experts,), c_out.stride(0), device=device, dtype=torch.int64 + ) + + es_fp8_blockwise_scaled_grouped_mm( + c_out, + a_stack, + b_stack, + a_scale_stack, + b_scale_stack, + a_strides, + a_strides, + d_strides, + problem_sizes, + expert_offsets[:-1], + workspace, + ) + + for g in range(num_experts): + baseline = baseline_tensors[g] + actual = c_out[expert_offsets[g] : expert_offsets[g + 1]] + diff = calc_diff(actual, baseline) + assert diff < 0.001 + print( + f"m_g={baseline.shape[0]} n_g={n_g} k_g={k_g} num_experts={num_experts}, out_dtype={out_dtype}, diff={diff:.5f}: OK" + ) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_flash_attention.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_flash_attention.py new file mode 100644 index 00000000..159390e5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_flash_attention.py @@ -0,0 +1,1368 @@ +# Adapted from https://github.com/Dao-AILab/flash-attention/blob/main/hopper/test_flash_attn.py +import itertools +import math +from typing import Optional + +import pytest +import torch +import torch.nn.functional as F +from einops import rearrange, repeat + +apply_rotary_emb = None + + +def is_hopper(): + # Only Hopper supports different V headdim + return torch.cuda.get_device_properties(0).major == 9 + + +def is_fa3_supported(device=None) -> bool: + # There some fa3 FYI + # FA3 can fail without a enough shared memory for a some shapes, such as higher + # hidden_dim or some special cases. + # Right now, fa3 is supported for sm80/sm87 and sm86/sm89. The main different + # Between sm80/sm87 and sm86/sm89 is the shared memory size. you can follow the link below for more information + # https://docs.nvidia.com/cuda/cuda-c-programming-guide/#shared-memory-8-x + # And for sgl-kernel right now, we can build fa3 on sm80/sm86/sm89/sm90a. + # That means if you use A100/A*0/L20/L40/L40s/4090 you can use fa3. + return (torch.version.cuda >= "12.3") and ( + torch.cuda.get_device_capability(device)[0] == 9 + or torch.cuda.get_device_capability(device)[0] == 8 + ) + + +DISABLE_BACKWARD = True +# For CI test, we close them to True. +# DISABLE_SPLIT = os.getenv("FLASH_ATTENTION_DISABLE_SPLIT", "FALSE") == "TRUE" +# DISABLE_PAGEDKV = os.getenv("FLASH_ATTENTION_DISABLE_PAGEDKV", "FALSE") == "TRUE" +# DISABLE_APPENDKV = os.getenv("FLASH_ATTENTION_DISABLE_APPENDKV", "FALSE") == "TRUE" +# DISABLE_LOCAL = os.getenv("FLASH_ATTENTION_DISABLE_LOCAL", "FALSE") == "TRUE" +# DISABLE_SOFTCAP = os.getenv("FLASH_ATTENTION_DISABLE_SOFTCAP", "FALSE") == "TRUE" +# DISABLE_PACKGQA = os.getenv("FLASH_ATTENTION_DISABLE_PACKGQA", "FALSE") == "TRUE" +# DISABLE_FP16 = os.getenv("FLASH_ATTENTION_DISABLE_FP16", "FALSE") == "TRUE" +# DISABLE_FP8 = ( +# os.getenv("FLASH_ATTENTION_DISABLE_FP8", "FALSE") == "TRUE" +# or torch.cuda.get_device_capability("cuda")[0] < 9 +# ) + +DISABLE_SPLIT = False +DISABLE_PAGEDKV = True +DISABLE_APPENDKV = False +DISABLE_LOCAL = False +DISABLE_SOFTCAP = True +DISABLE_PACKGQA = False +DISABLE_FP16 = True +DISABLE_FP8 = True + + +# Adapted from https://github.com/Dao-AILab/flash-attention/blob/main/hopper/padding.py +def unpad_input(hidden_states, attention_mask, unused_mask=None): + """ + Arguments: + hidden_states: (batch, seqlen, ...) + attention_mask: (batch, seqlen), bool / int, 1 means valid and 0 means not valid. + unused_mask: (batch, seqlen), bool / int, 1 means the element is allocated but unused. + Return: + hidden_states: (total_nnz, ...), where total_nnz = number of tokens selected in attention_mask + unused_mask. + indices: (total_nnz), the indices of masked tokens from the flattened input sequence. + cu_seqlens: (batch + 1), the cumulative sequence lengths, used to index into hidden_states. + max_seqlen_in_batch: int + seqused: (batch), returns the number of tokens selected in attention_mask + unused_mask. + """ + all_masks = ( + (attention_mask + unused_mask) if unused_mask is not None else attention_mask + ) + seqlens_in_batch = all_masks.sum(dim=-1, dtype=torch.int32) + used_seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32) + indices = torch.nonzero(all_masks.flatten(), as_tuple=False).flatten() + max_seqlen_in_batch = seqlens_in_batch.max().item() + cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0)) + # TD [2022-03-04] We don't want to index with a bool mask, because Pytorch will expand the + # bool mask, then call nonzero to get the indices, then index with those. The indices is @dim + # times larger than it needs to be, wasting memory. It's faster and more memory-efficient to + # index with integer indices. + return ( + rearrange(hidden_states, "b s ... -> (b s) ...")[indices], + indices, + cu_seqlens, + max_seqlen_in_batch, + used_seqlens_in_batch, + ) + + +def generate_random_padding_mask( + max_seqlen, batch_size, device, mode="random", zero_lengths=False +): + assert mode in ["full", "random", "third"] + if mode == "full": + lengths = torch.full( + (batch_size, 1), max_seqlen, device=device, dtype=torch.int32 + ) + elif mode == "random": + lengths = torch.randint( + max(0 if zero_lengths else 1, max_seqlen - 20), + max_seqlen + 1, + (batch_size, 1), + device=device, + ) + elif mode == "third": + lengths = torch.randint( + max_seqlen // 3, max_seqlen + 1, (batch_size, 1), device=device + ) + + if zero_lengths: + # Generate zero-lengths every 5 batches and the last batch. + for i in range(batch_size): + if i % 5 == 0: + lengths[i] = 0 + lengths[-1] = 0 + padding_mask = ( + repeat(torch.arange(max_seqlen, device=device), "s -> b s", b=batch_size) + < lengths + ) + return padding_mask + + +def pad_input(hidden_states, indices, batch, seqlen): + """ + Arguments: + hidden_states: (total_nnz, ...), where total_nnz = number of tokens in selected in attention_mask. + indices: (total_nnz), the indices that represent the non-masked tokens of the original padded input sequence. + batch: int, batch size for the padded sequence. + seqlen: int, maximum sequence length for the padded sequence. + Return: + hidden_states: (batch, seqlen, ...) + """ + dim = hidden_states.shape[1:] + output = torch.zeros( + (batch * seqlen), *dim, device=hidden_states.device, dtype=hidden_states.dtype + ) + output[indices] = hidden_states + return rearrange(output, "(b s) ... -> b s ...", b=batch) + + +def construct_local_mask( + seqlen_q, + seqlen_k, + window_size=(-1, -1), # -1 means infinite window size + sink_token_length=0, + query_padding_mask=None, + key_padding_mask=None, + key_leftpad=None, + device=None, +): + row_idx = rearrange( + torch.arange(seqlen_q, device=device, dtype=torch.long), "s -> s 1" + ) + col_idx = torch.arange(seqlen_k, device=device, dtype=torch.long) + if key_leftpad is not None: + key_leftpad = rearrange(key_leftpad, "b -> b 1 1 1") + col_idx = repeat(col_idx, "s -> b 1 1 s", b=key_leftpad.shape[0]) + col_idx = torch.where(col_idx >= key_leftpad, col_idx - key_leftpad, 2**32) + sk = ( + seqlen_k + if key_padding_mask is None + else rearrange(key_padding_mask.sum(-1), "b -> b 1 1 1") + ) + sq = ( + seqlen_q + if query_padding_mask is None + else rearrange(query_padding_mask.sum(-1), "b -> b 1 1 1") + ) + if window_size[0] < 0: + return col_idx > row_idx + sk - sq + window_size[1] + else: + sk = torch.full_like(col_idx, seqlen_k) if key_padding_mask is None else sk + return torch.logical_or( + col_idx > torch.minimum(row_idx + sk - sq + window_size[1], sk), + torch.logical_and( + col_idx < row_idx + sk - sq - window_size[0], + col_idx >= sink_token_length, + ), + ) + + +def attention_ref( + q, + k, + v, + query_padding_mask=None, + key_padding_mask=None, + key_leftpad=None, + attn_bias=None, + dropout_p=0.0, + dropout_mask=None, + causal=False, + qv=None, + q_descale=None, + k_descale=None, + v_descale=None, + window_size=(-1, -1), # -1 means infinite window size + sink_token_length=0, + sinks: Optional[torch.Tensor] = None, + softcap=0.0, + upcast=True, + reorder_ops=False, + intermediate_dtype=None, +): + """ + Arguments: + q: (batch_size, seqlen_q, nheads, head_dim) + k: (batch_size, seqlen_k, nheads, head_dim) + v: (batch_size, seqlen_k, nheads, head_dim_v) + qv: (batch_size, seqlen_q, nheads, head_dim_v) + query_padding_mask: (batch_size, seqlen_q) + key_padding_mask: (batch_size, seqlen_k) + attn_bias: broadcastable to (batch_size, nheads, seqlen_q, seqlen_k) + dropout_p: float + dropout_mask: (batch_size, nheads, seqlen_q, seqlen_k) + causal: whether to apply causal masking + upcast: whether to cast all inputs to fp32, do all computation in fp32, then cast + output back to fp16/bf16. + reorder_ops: whether to change the order of operations (scaling k instead of scaling k, etc.) + without changing the math. This is to estimate the numerical error from operation + reordering. + Output: + output: (batch_size, seqlen_q, nheads, head_dim_v) + attention: (batch_size, nheads, seqlen_q, seqlen_k), softmax after dropout + """ + if causal: + window_size = (window_size[0], 0) + dtype_og = q.dtype + if upcast: + q, k, v = q.float(), k.float(), v.float() + qv = qv.float() if qv is not None else None + if q_descale is not None: + q_descale = repeat(q_descale, "b h -> b 1 (h g) 1", g=q.shape[2] // k.shape[2]) + q = (q.float() * q_descale).to(q.dtype) + qv = (qv.float() * q_descale).to(qv.dtype) if qv is not None else None + if k_descale is not None: + k = (k.float() * rearrange(k_descale, "b h -> b 1 h 1")).to(dtype=k.dtype) + if v_descale is not None: + v = (v.float() * rearrange(v_descale, "b h -> b 1 h 1")).to(dtype=v.dtype) + seqlen_q, seqlen_k = q.shape[1], k.shape[1] + k = repeat(k, "b s h d -> b s (h g) d", g=q.shape[2] // k.shape[2]) + v = repeat(v, "b s h d -> b s (h g) d", g=q.shape[2] // v.shape[2]) + d = q.shape[-1] + dv = v.shape[-1] + softmax_scale = 1.0 / math.sqrt(d if qv is None else d + dv) + if not reorder_ops: + scores = torch.einsum("bthd,bshd->bhts", q * softmax_scale, k) + else: + scores = torch.einsum("bthd,bshd->bhts", q, k * softmax_scale) + if qv is not None: + scores = scores + torch.einsum("bthd,bshd->bhts", qv * softmax_scale, v) + if softcap > 0: + scores = torch.tanh(scores / softcap) * softcap + if key_padding_mask is not None: + scores.masked_fill_( + rearrange(~key_padding_mask, "b s -> b 1 1 s"), float("-inf") + ) + if window_size[0] >= 0 or window_size[1] >= 0: + local_mask = construct_local_mask( + seqlen_q, + seqlen_k, + window_size, + sink_token_length, + query_padding_mask, + key_padding_mask, + key_leftpad=key_leftpad, + device=q.device, + ) + scores.masked_fill_(local_mask, float("-inf")) + if attn_bias is not None: + scores = scores + attn_bias + if sinks is None: + attention = torch.softmax(scores, dim=-1).to(v.dtype) + else: + scores_fp32 = scores.to(torch.float32) + logits_max = torch.amax(scores_fp32, dim=-1, keepdim=True) + sinks = rearrange(sinks, "h -> h 1 1") + logits_or_sinks_max = torch.maximum(sinks, logits_max) + unnormalized_scores = torch.exp(scores_fp32 - logits_or_sinks_max) + normalizer = unnormalized_scores.sum(dim=-1, keepdim=True) + torch.exp( + sinks - logits_or_sinks_max + ) + attention = (unnormalized_scores / normalizer).to(v.dtype) + # We want to mask here so that the attention matrix doesn't have any NaNs + # Otherwise we'll get NaN in dV + if query_padding_mask is not None: + attention = attention.masked_fill( + rearrange(~query_padding_mask, "b s -> b 1 s 1"), 0.0 + ) + # Without this we might get NaN in dv + if key_padding_mask is not None: + attention = attention.masked_fill( + rearrange(~key_padding_mask, "b s -> b 1 1 s"), 0.0 + ) + # Some rows might be completely masked out so we fill them with zero instead of NaN + if window_size[0] >= 0 or window_size[1] >= 0: + attention = attention.masked_fill( + torch.all(local_mask, dim=-1, keepdim=True), 0.0 + ) + dropout_scaling = 1.0 / (1 - dropout_p) + # attention_drop = attention.masked_fill(~dropout_mask, 0.0) * dropout_scaling + # output = torch.einsum('bhts,bshd->bthd', attention_drop , v) + if dropout_mask is not None: + attention_drop = attention.masked_fill(~dropout_mask, 0.0) + else: + attention_drop = attention + if intermediate_dtype is not None: + attention_drop = attention_drop.to(intermediate_dtype).to(attention_drop.dtype) + output = torch.einsum("bhts,bshd->bthd", attention_drop, v * dropout_scaling) + if query_padding_mask is not None: + output.masked_fill_(rearrange(~query_padding_mask, "b s -> b s 1 1"), 0.0) + return output.to(dtype=dtype_og), attention.to(dtype=dtype_og) + + +def generate_qkv( + q, + k, + v, + query_padding_mask=None, + key_padding_mask=None, + kvpacked=False, + qkvpacked=False, + add_unused_qkv=False, + query_unused_mask=None, + key_unused_mask=None, +): + """ + Arguments: + q: (batch_size, seqlen_q, nheads, d) + k: (batch_size, seqlen_k, nheads_k, d) + v: (batch_size, seqlen_k, nheads_k, d) + query_padding_mask: (batch_size, seqlen), bool + key_padding_mask: (batch_size, seqlen), bool + """ + assert not (kvpacked and qkvpacked) + batch_size, seqlen_q, nheads, d = q.shape + _, seqlen_k, nheads_k, _ = k.shape + assert k.shape == (batch_size, seqlen_k, nheads_k, d) + assert v.shape == (batch_size, seqlen_k, nheads_k, d) + if query_unused_mask is not None or key_unused_mask is not None: + assert not kvpacked + assert not qkvpacked + + if query_padding_mask is not None: + q_unpad, indices_q, cu_seqlens_q, max_seqlen_q, seqused_q = unpad_input( + q, + query_padding_mask, + query_unused_mask, + ) + output_pad_fn = lambda output_unpad: pad_input( + output_unpad, indices_q, batch_size, seqlen_q + ) + else: + q_unpad = rearrange(q, "b s h d -> (b s) h d") + cu_seqlens_q = torch.arange( + 0, + (batch_size + 1) * seqlen_q, + step=seqlen_q, + dtype=torch.int32, + device=q_unpad.device, + ) + seqused_q = None + max_seqlen_q = seqlen_q + output_pad_fn = lambda output_unpad: rearrange( + output_unpad, "(b s) h d -> b s h d", b=batch_size + ) + + if key_padding_mask is not None: + k_unpad, indices_k, cu_seqlens_k, max_seqlen_k, seqused_k = unpad_input( + k, key_padding_mask, key_unused_mask + ) + v_unpad, _, _, _, _ = unpad_input(v, key_padding_mask, key_unused_mask) + else: + k_unpad = rearrange(k, "b s h d -> (b s) h d") + v_unpad = rearrange(v, "b s h d -> (b s) h d") + cu_seqlens_k = torch.arange( + 0, + (batch_size + 1) * seqlen_k, + step=seqlen_k, + dtype=torch.int32, + device=k_unpad.device, + ) + seqused_k = None + max_seqlen_k = seqlen_k + + if qkvpacked: + assert (query_padding_mask == key_padding_mask).all() + assert nheads == nheads_k + qkv_unpad = torch.stack([q_unpad, k_unpad, v_unpad], dim=1) + qkv = torch.stack([q, k, v], dim=2) + if query_padding_mask is not None: + dqkv_pad_fn = lambda dqkv_unpad: pad_input( + dqkv_unpad, indices_q, batch_size, seqlen_q + ) + else: + dqkv_pad_fn = lambda dqkv_unpad: rearrange( + dqkv_unpad, "(b s) t h d -> b s t h d", b=batch_size + ) + return ( + qkv_unpad.detach().requires_grad_(), + cu_seqlens_q, + max_seqlen_q, + qkv.detach().requires_grad_(), + output_pad_fn, + dqkv_pad_fn, + ) + elif kvpacked: + kv_unpad = torch.stack([k_unpad, v_unpad], dim=1) + kv = torch.stack([k, v], dim=2) + dq_pad_fn = output_pad_fn + if key_padding_mask is not None: + dkv_pad_fn = lambda dkv_unpad: pad_input( + dkv_unpad, indices_k, batch_size, seqlen_k + ) + else: + dkv_pad_fn = lambda dkv_unpad: rearrange( + dkv_unpad, "(b s) t h d -> b s t h d", b=batch_size + ) + return ( + q_unpad.detach().requires_grad_(), + kv_unpad.detach().requires_grad_(), + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + q.detach().requires_grad_(), + kv.detach().requires_grad_(), + output_pad_fn, + dq_pad_fn, + dkv_pad_fn, + ) + else: + dq_pad_fn = output_pad_fn + if key_padding_mask is not None: + dk_pad_fn = lambda dk_unpad: pad_input( + dk_unpad, indices_k, batch_size, seqlen_k + ) + else: + dk_pad_fn = lambda dk_unpad: rearrange( + dk_unpad, "(b s) h d -> b s h d", b=batch_size + ) + return ( + q_unpad.detach().requires_grad_(), + k_unpad.detach().requires_grad_(), + v_unpad.detach().requires_grad_(), + cu_seqlens_q, + cu_seqlens_k, + seqused_q, + seqused_k, + max_seqlen_q, + max_seqlen_k, + q.detach().requires_grad_(), + k.detach().requires_grad_(), + v.detach().requires_grad_(), + output_pad_fn, + dq_pad_fn, + dk_pad_fn, + ) + + +@pytest.mark.skipif( + not is_fa3_supported(), + reason="flash_attn at sgl-kernel is only supported on sm90 or sm80", +) +# @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float8_e4m3fn]) +@pytest.mark.parametrize( + "dtype", [torch.bfloat16] + ([torch.float8_e4m3fn] if not DISABLE_FP8 else []) +) +# @pytest.mark.parametrize("dtype", [torch.bfloat16]) +# @pytest.mark.parametrize("dtype", [torch.float8_e4m3fn]) +@pytest.mark.parametrize("mha_type", ["mha", "mqa", "gqa"]) +# @pytest.mark.parametrize("mha_type", ["mha"]) +@pytest.mark.parametrize("has_sink", [False, True]) +# @pytest.mark.parametrize("has_sink", [False]) +@pytest.mark.parametrize("new_kv", [False] + ([True] if not DISABLE_APPENDKV else [])) +# @pytest.mark.parametrize("new_kv", [True]) +# @pytest.mark.parametrize( +# "causal,local", +# [(False, False), (True, False)] + ([(False, True)] if not DISABLE_LOCAL else []), +# ) +# @pytest.mark.parametrize("causal,local", [(False, False), (True, False)]) +@pytest.mark.parametrize("causal,local", [(False, False)]) +@pytest.mark.parametrize( + "seqlen_new_eq_seqlen_q", [True, False] if not DISABLE_APPENDKV else [True] +) +# @pytest.mark.parametrize("seqlen_new_eq_seqlen_q", [True]) +# @pytest.mark.parametrize("has_rotary_seqlens", [False, True]) +@pytest.mark.parametrize("has_rotary_seqlens", [False]) +@pytest.mark.parametrize( + "rotary_interleaved", [False, True] if not DISABLE_APPENDKV else [False] +) +# @pytest.mark.parametrize("rotary_interleaved", [True]) +@pytest.mark.parametrize( + "rotary_fraction", + ( + [0.0, 0.5, 1.0] + if (not DISABLE_APPENDKV) and (apply_rotary_emb is not None) + else [0.0] + ), +) +# @pytest.mark.parametrize("rotary_fraction", [0.0]) +@pytest.mark.parametrize( + "page_size", [None] + ([1, 4, 128] if not DISABLE_PAGEDKV else []) +) +# @pytest.mark.parametrize("page_size", [None]) +# @pytest.mark.parametrize("has_leftpad", [False, True]) +@pytest.mark.parametrize("has_leftpad", [False]) +# @pytest.mark.parametrize("has_batch_idx", [False, True]) +@pytest.mark.parametrize("has_batch_idx", [False]) +# @pytest.mark.parametrize("varlen_q", [False, True]) +@pytest.mark.parametrize("varlen_q", [False]) +# @pytest.mark.parametrize("d", [32, 59, 64, 80, 128, 256]) +# @pytest.mark.parametrize("d", [32, 64, 96, 128, 160, 192, 224, 256]) +# @pytest.mark.parametrize('d', [32, 40, 64, 80, 96, 128, 160, 192]) +# @pytest.mark.parametrize('d', [56, 80]) +@pytest.mark.parametrize("d", [64]) +# @pytest.mark.parametrize("d", [192]) +@pytest.mark.parametrize( + "seqlen_q,seqlen_k", + [ + (1, 128), + (1, 339), + (3, 1024), + (64, 800), + (64, 256), + (3, 799), + (64, 2048), + (16, 20000), + # (1, 128 * 1024), + # (16, 128 * 1024), + (128, 128), + (256, 512), # To test appending KV with more than 1 block + (2048, 3577), # Enough tile to test persistent scheduler + ], +) +# @pytest.mark.parametrize('seqlen_q,seqlen_k', [(256, 128)]) +def test_flash_attn_kvcache( + seqlen_q, + seqlen_k, + d, + varlen_q, + has_batch_idx, + has_leftpad, + page_size, + rotary_fraction, + rotary_interleaved, + has_rotary_seqlens, + seqlen_new_eq_seqlen_q, + causal, + local, + new_kv, + mha_type, + dtype, + has_sink, +): + from sgl_kernel.flash_attn import flash_attn_with_kvcache + + if page_size is not None and seqlen_k % page_size != 0: + pytest.skip() + if seqlen_q > seqlen_k and new_kv: + pytest.skip() + if not new_kv and rotary_fraction > 0.0: + pytest.skip() + if rotary_fraction == 0.0 and has_rotary_seqlens: + pytest.skip() + device = "cuda" + # set seed + torch.random.manual_seed(0) + batch_size = 5 + # batch_size = 1 + batch_size_cache = batch_size if not has_batch_idx else batch_size * 2 + nheads = 6 + # nheads = 1 + # rotary_dim must be a multiple of 16, and must be <= d + rotary_dim = math.floor(int(rotary_fraction * d) / 16) * 16 + nheads_k = nheads if mha_type == "mha" else (1 if mha_type == "mqa" else 3) + assert nheads % nheads_k == 0 + dtype_ref = torch.bfloat16 if dtype == torch.float8_e4m3fn else dtype + dv_vals = [128, d] if d > 128 and d <= 192 else ([256, 512, d] if d <= 64 else [d]) + + if has_sink: + sinks = torch.randn(nheads, dtype=torch.bfloat16, device=device) + else: + sinks = None + + if dtype == torch.float8_e4m3fn or not is_hopper(): + # for fp8 and ampere arch, we not support v head dim != qk head dim + dv_vals = [d] + for dv in dv_vals: + has_qv = d == 64 and dv >= 256 + q = ( + torch.randn(batch_size, seqlen_q, nheads, d, device=device, dtype=dtype_ref) + .to(dtype) + .to(dtype_ref) + ) + if has_qv: + qv = ( + torch.randn( + batch_size, seqlen_q, nheads, dv, device=device, dtype=dtype_ref + ) + .to(dtype) + .to(dtype_ref) + ) + else: + qv = None + if varlen_q: + query_padding_mask = generate_random_padding_mask( + seqlen_q, batch_size, device, mode="random" + ) + q_unpad, indices_q, cu_seqlens_q, max_seqlen_q, *rest = unpad_input( + q, query_padding_mask + ) + output_pad_fn = lambda output_unpad: pad_input( + output_unpad, indices_q, batch_size, seqlen_q + ) + qv_unpad = ( + rearrange(qv, "b s ... -> (b s) ...")[indices_q] if has_qv else None + ) + else: + query_padding_mask = None + q_unpad = q + qv_unpad = qv + cu_seqlens_q, max_seqlen_q = None, None + # Put window_size after QKV randn so that window_size changes from test to test + window_size = (-1, -1) if not local else torch.randint(0, seqlen_k, (2,)) + + seqlen_new = ( + seqlen_q + if seqlen_new_eq_seqlen_q + else torch.randint(1, seqlen_q + 1, (1,)).item() + ) + cu_seqlens_k_new = None + key_new_padding_mask = None + if new_kv: + k = ( + torch.randn( + batch_size, seqlen_new, nheads_k, d, device=device, dtype=dtype_ref + ) + .to(dtype) + .to(dtype_ref) + ) + v = ( + torch.randn( + batch_size, seqlen_new, nheads_k, dv, device=device, dtype=dtype_ref + ) + .to(dtype) + .to(dtype_ref) + ) + if varlen_q: # k & v are also varlen + key_new_padding_mask = generate_random_padding_mask( + seqlen_new, batch_size, device, mode="random" + ) + k_unpad, indices_k, cu_seqlens_k_new, *rest = unpad_input( + k, key_new_padding_mask + ) + v_unpad, *rest = unpad_input(v, key_new_padding_mask) + else: + k_unpad, v_unpad = k, v + else: + k, v, k_unpad, v_unpad = None, None, None, None + if page_size is None: + k_cache = ( + torch.randn( + batch_size_cache, + seqlen_k, + nheads_k, + d, + device=device, + dtype=dtype_ref, + ) + .to(dtype) + .to(dtype_ref) + ) + v_cache = ( + torch.randn( + batch_size_cache, + seqlen_k, + nheads_k, + dv, + device=device, + dtype=dtype_ref, + ) + .to(dtype) + .to(dtype_ref) + ) + page_table = None + else: + ( + k_cache, + v_cache, + page_table, + k_cache_paged, + v_cache_paged, + num_blocks, + ) = _generate_block_kvcache( + seqlen_k, + page_size, + batch_size_cache, + nheads_k, + d, + dv, + device, + dtype, + dtype_ref, + ) + cache_seqlens = torch.randint( + 0 if new_kv else 1, + # If we don't use seqlen_q in the case of causal and rotary, cos/sin won't be long enough + ( + ( + seqlen_k + - (seqlen_q if (causal or local) and rotary_dim > 1 else seqlen_new) + + 1 + ) + if new_kv + else (seqlen_k + 1) + ), + (batch_size,), + dtype=torch.int32, + device=device, + ) + if has_leftpad: + cache_leftpad = torch.cat( + [ + ( + torch.randint( + 0, + cache_seqlens[i].item(), + (1,), + dtype=torch.int32, + device=device, + ) + if cache_seqlens[i].item() > 0 + else torch.zeros(1, dtype=torch.int32, device=device) + ) + for i in range(batch_size) + ] + ) + else: + cache_leftpad = None + if has_batch_idx: + cache_batch_idx = torch.randperm( + batch_size_cache, dtype=torch.int32, device=device + )[:batch_size] + else: + cache_batch_idx = None + arange = rearrange(torch.arange(seqlen_k, device=device), "s -> 1 s") + cache_seqlens_expanded = rearrange(cache_seqlens, "b -> b 1") + if not new_kv: + key_padding_mask = arange < cache_seqlens_expanded + else: + k_new_seqlens = ( + key_new_padding_mask.sum(-1, keepdims=True) if varlen_q else seqlen_new + ) + key_padding_mask = arange < cache_seqlens_expanded + k_new_seqlens + if has_leftpad: + key_padding_mask = torch.logical_and( + key_padding_mask, + arange >= cache_leftpad.unsqueeze(-1).expand(-1, seqlen_k), + ) + # cache_seqlens = torch.tensor([64], dtype=torch.int32, device=device) + rotary_seqlens = cache_seqlens if not has_rotary_seqlens else cache_seqlens // 2 + if rotary_dim > 0: + angle = ( + torch.rand( + seqlen_k if page_size is None else num_blocks * page_size, + rotary_dim // 2, + device=device, + ) + * 2 + * math.pi + ) + cos = torch.cos(angle).to(dtype=dtype_ref).to(dtype).to(dtype_ref) + sin = torch.sin(angle).to(dtype=dtype_ref).to(dtype).to(dtype_ref) + if causal or local: + q_ro = apply_rotary_emb( + q, + cos, + sin, + seqlen_offsets=rotary_seqlens, + interleaved=rotary_interleaved, + ) + else: + q_ro = rearrange( + apply_rotary_emb( + rearrange(q, "b s h d -> b 1 (s h) d"), + cos, + sin, + seqlen_offsets=rotary_seqlens, + interleaved=rotary_interleaved, + ), + "b 1 (s h) d -> b s h d", + s=seqlen_q, + ) + # q_ro = q + k_ro = apply_rotary_emb( + k, + cos, + sin, + seqlen_offsets=rotary_seqlens, + interleaved=rotary_interleaved, + ) + else: + cos, sin = None, None + q_ro, k_ro = q, k + # k_cache[:, 64:] = -1 + k_cache_ref = ( + k_cache if not has_batch_idx else k_cache[cache_batch_idx] + ).clone() + v_cache_ref = ( + v_cache if not has_batch_idx else v_cache[cache_batch_idx] + ).clone() + if new_kv: + update_mask = torch.logical_and( + cache_seqlens_expanded <= arange, + arange < cache_seqlens_expanded + k_new_seqlens, + ) + k_to_update = rearrange(k_ro, "b s ... -> (b s) ...") + v_to_update = rearrange(v, "b s ... -> (b s) ...") + if varlen_q: + k_to_update = k_to_update[indices_k] + v_to_update = v_to_update[indices_k] + k_cache_ref[update_mask] = k_to_update + v_cache_ref[update_mask] = v_to_update + k_cache_rep = repeat( + k_cache_ref, "b s h d -> b s (h g) d", g=nheads // nheads_k + ) + v_cache_rep = repeat( + v_cache_ref, "b s h d -> b s (h g) d", g=nheads // nheads_k + ) + out_ref, _ = attention_ref( + q_ro, + k_cache_rep, + v_cache_rep, + query_padding_mask, + key_padding_mask, + causal=causal, + qv=qv, + window_size=window_size, + key_leftpad=cache_leftpad, + sinks=sinks, + ) + out_pt, _ = attention_ref( + q_ro, + k_cache_rep, + v_cache_rep, + query_padding_mask, + key_padding_mask, + causal=causal, + qv=qv, + window_size=window_size, + upcast=False, + reorder_ops=True, + key_leftpad=cache_leftpad, + intermediate_dtype=dtype if dtype == torch.float8_e4m3fn else None, + sinks=sinks, + ) + q = q.to(dtype) + q_unpad = q_unpad.to(dtype) if varlen_q else None + k_cache = k_cache.to(dtype) + v_cache = v_cache.to(dtype) + k_cache_paged = k_cache_paged.to(dtype) if page_size is not None else None + v_cache_paged = v_cache_paged.to(dtype) if page_size is not None else None + k = k.to(dtype) if k is not None else None + v = v.to(dtype) if v is not None else None + k_unpad = k_unpad.to(dtype) if k_unpad is not None else None + v_unpad = v_unpad.to(dtype) if v_unpad is not None else None + qv = qv.to(dtype) if qv is not None else None + qv_unpad = qv_unpad.to(dtype) if (varlen_q and qv is not None) else None + cos = cos.to(dtype) if cos is not None else None + sin = sin.to(dtype) if sin is not None else None + k_cache_saved = k_cache.clone() if page_size is None else k_cache_paged.clone() + v_cache_saved = v_cache.clone() if page_size is None else v_cache_paged.clone() + num_splits_vals = [1, 0] if not DISABLE_SPLIT else [1] + precompute_metadata_vals = [False] + for num_splits, precompute_metadata in itertools.product( + num_splits_vals, precompute_metadata_vals + ): + scheduler_metadata = None + # Repeat to test metadata reuse + for _ in range(1 if not precompute_metadata else 2): + if page_size is None: + k_cache.copy_(k_cache_saved) + v_cache.copy_(v_cache_saved) + else: + k_cache_paged.copy_(k_cache_saved) + v_cache_paged.copy_(v_cache_saved) + out, lse, *rest = flash_attn_with_kvcache( + q if not varlen_q else q_unpad, + k_cache if page_size is None else k_cache_paged, + v_cache if page_size is None else v_cache_paged, + k if not new_kv or not varlen_q else k_unpad, + v if not new_kv or not varlen_q else v_unpad, + qv=qv if not varlen_q else qv_unpad, + rotary_cos=cos, + rotary_sin=sin, + cache_seqlens=cache_seqlens, + cache_batch_idx=cache_batch_idx, + cache_leftpad=cache_leftpad, + page_table=page_table, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k_new=cu_seqlens_k_new, + max_seqlen_q=max_seqlen_q, + rotary_seqlens=rotary_seqlens, + causal=causal, + window_size=window_size, + rotary_interleaved=rotary_interleaved, + scheduler_metadata=scheduler_metadata, + num_splits=num_splits, + return_softmax_lse=True, + sinks=sinks, + ) + if varlen_q: + out = output_pad_fn(out) + # out = flash_attn_with_kvcache( + # q, k_cache, v_cache, cache_seqlens=cache_seqlens, causal=causal, window_size=window_size + # ) + # out = flash_attn_with_kvcache(q, k_cache, v_cache, causal=causal, window_size=window_size) + # qk = torch.einsum("bqhd,bkhd->bhqk", q, k_cache_ref) + # m = qk.amax(-1, keepdim=True) + # s_tmp = torch.exp((qk - m) / math.sqrt(d)) + # o1 = torch.einsum('bhst,bthd->bshd', s_tmp, v_cache_ref) + # lse_ref = torch.logsumexp(qk / math.sqrt(d), -1) + # probs = torch.softmax(qk, dim=-1) + print(f"Output max diff: {(out - out_ref).abs().max().item()}") + print(f"Output mean diff: {(out - out_ref).abs().mean().item()}") + print(f"Pytorch max diff: {(out_pt - out_ref).abs().max().item()}") + print(f"Pytorch mean diff: {(out_pt - out_ref).abs().mean().item()}") + # breakpoint() + + # Check that FlashAttention's numerical error is at most twice the numerical error + # of a Pytorch implementation. + if new_kv: + if page_size is None: + k_cache_select = ( + k_cache.to(dtype_ref) + if not has_batch_idx + else k_cache.to(dtype_ref)[cache_batch_idx] + ) + v_cache_select = ( + v_cache.to(dtype_ref) + if not has_batch_idx + else v_cache.to(dtype_ref)[cache_batch_idx] + ) + else: + k_cache_select = rearrange( + k_cache_paged.to(dtype_ref)[ + ( + page_table + if not has_batch_idx + else page_table[cache_batch_idx] + ).flatten() + ], + "(b nblocks) block_size ... -> b (nblocks block_size) ...", + b=batch_size, + )[:, :seqlen_k].to(dtype_ref) + v_cache_select = rearrange( + v_cache_paged.to(dtype_ref)[ + ( + page_table + if not has_batch_idx + else page_table[cache_batch_idx] + ).flatten() + ], + "(b nblocks) block_size ... -> b (nblocks block_size) ...", + b=batch_size, + )[:, :seqlen_k].to(dtype_ref) + k_cache_ref = k_cache_ref.to(dtype).to(dtype_ref) + v_cache_ref = v_cache_ref.to(dtype).to(dtype_ref) + if dtype is not torch.float8_e4m3fn: + assert torch.equal(v_cache_select, v_cache_ref) + else: + assert torch.allclose( + v_cache_select, v_cache_ref, rtol=1e-3, atol=1e-3 + ) + # breakpoint() + # if rotary_dim == 0 and dtype is not torch.float8_e4m3fn: + if rotary_dim == 0: + assert torch.equal(k_cache_select, k_cache_ref) + else: + # if not torch.allclose(k_cache_select, k_cache_ref, rtol=1e-3, atol=1e-3): + # breakpoint() + if dtype is not torch.float8_e4m3fn: + assert torch.allclose( + k_cache_select, k_cache_ref, rtol=1e-3, atol=1e-3 + ) + else: + assert torch.allclose( + k_cache_select, k_cache_ref, rtol=1e-1, atol=1e-1 + ) + mult = 4 if dtype == torch.float8_e4m3fn else 2 + assert (out - out_ref).abs().max().item() <= mult * ( + out_pt - out_ref + ).abs().max().item() + 1e-5 + mult_mean = 3 if dtype == torch.float8_e4m3fn else 1.5 + assert (out - out_ref).abs().mean().item() <= mult_mean * ( + out_pt - out_ref + ).abs().mean().item() + + +def _generate_block_kvcache( + seqlen_k, page_size, batch_size, nheads_k, d, dv, device, dtype, dtype_ref +): + num_blocks = math.ceil(seqlen_k / page_size) * batch_size * 3 + k_cache_paged = ( + torch.randn(num_blocks, page_size, nheads_k, d, device=device, dtype=dtype_ref) + .to(dtype) + .to(dtype_ref) + ) + v_cache_paged = ( + torch.randn(num_blocks, page_size, nheads_k, dv, device=device, dtype=dtype_ref) + .to(dtype) + .to(dtype_ref) + ) + page_table = rearrange( + torch.randperm(num_blocks, dtype=torch.int32, device=device), + "(b nblocks) -> b nblocks", + b=batch_size, + ) + k_cache = rearrange( + k_cache_paged[page_table.flatten()], + "(b nblocks) block_size ... -> b (nblocks block_size) ...", + b=batch_size, + )[:, :seqlen_k] + v_cache = rearrange( + v_cache_paged[page_table.flatten()], + "(b nblocks) block_size ... -> b (nblocks block_size) ...", + b=batch_size, + )[:, :seqlen_k] + return k_cache, v_cache, page_table, k_cache_paged, v_cache_paged, num_blocks + + +@pytest.mark.skipif( + not is_fa3_supported(), + reason="flash_attn at sgl-kernel is only supported on sm90 or sm80", +) +# @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float8_e4m3fn]) +@pytest.mark.parametrize( + "dtype", [torch.bfloat16] + ([torch.float8_e4m3fn] if not DISABLE_FP8 else []) +) +# @pytest.mark.parametrize("dtype", [torch.bfloat16]) +# @pytest.mark.parametrize("dtype", [torch.float8_e4m3fn]) +@pytest.mark.parametrize("mha_type", ["mha", "mqa", "gqa"]) +# @pytest.mark.parametrize("mha_type", ["mha"]) +@pytest.mark.parametrize("has_sink", [False, True]) +# @pytest.mark.parametrize("has_sink", [False]) +# @pytest.mark.parametrize("has_qv", [False, True]) +@pytest.mark.parametrize("has_qv", [False]) +# @pytest.mark.parametrize("deterministic", [False, True]) +@pytest.mark.parametrize("deterministic", [False]) +@pytest.mark.parametrize("softcap", [0.0] + ([15.0] if not DISABLE_SOFTCAP else [])) +# @pytest.mark.parametrize("softcap", [0.0]) +@pytest.mark.parametrize("local", [False]) +# @pytest.mark.parametrize("local", [False]) +@pytest.mark.parametrize("causal", [False, True]) +# @pytest.mark.parametrize("causal", [False]) +@pytest.mark.parametrize("add_unused_qkv", [False, True]) +# @pytest.mark.parametrize("add_unused_qkv", [True]) +# @pytest.mark.parametrize("d", [32, 64, 96, 128, 160, 192, 224, 256]) +# @pytest.mark.parametrize('d', [32, 40, 64, 80, 96, 128, 160, 192, 256]) +# @pytest.mark.parametrize('d', [32, 64, 96, 128, 160, 192]) +# @pytest.mark.parametrize('d', [56, 80]) +# @pytest.mark.parametrize('d', [32, 40, 64, 80, 96, 128]) +# @pytest.mark.parametrize("d", [64, 96, 128]) +# @pytest.mark.parametrize("d", COMPILED_HDIMS) +@pytest.mark.parametrize("d", [128]) +@pytest.mark.parametrize( + "seqlen_q,seqlen_k", + [ + (1, 1), + (1, 3), + (2, 1), + (511, 1), + (3, 513), + (64, 128), + (128, 128), + (256, 256), + (113, 203), + (128, 217), + (113, 211), + (108, 256), + (256, 512), + (307, 256), + (640, 128), + (512, 256), + (1024, 1024), + (1023, 1024), + (1024, 1023), + (2048, 2048), + ], +) +def test_flash_attn_varlen_output( + seqlen_q, + seqlen_k, + d, + add_unused_qkv, + causal, + local, + softcap, + deterministic, + has_qv, + mha_type, + dtype, + has_sink, +): + from sgl_kernel.flash_attn import flash_attn_varlen_func + + device = "cuda" + # set seed + torch.random.manual_seed(seqlen_q + seqlen_k + d + int(causal) * 2 + int(local)) + # batch_size = 40 + # nheads = 16 + batch_size = 9 if seqlen_q <= 2048 else 2 + nheads = 6 + # batch_size = 2 + # nheads = 1 + nheads_kv = nheads if mha_type == "mha" else (2 if mha_type == "gqa" else 1) + dtype_ref = torch.bfloat16 if dtype == torch.float8_e4m3fn else dtype + dv_vals = [128, d] if d > 128 and d <= 192 else ([256, 512, d] if d <= 64 else [d]) + if dtype == torch.float8_e4m3fn: + dv_vals = [d] + for dv in dv_vals: + q_ref = torch.randn( + batch_size, seqlen_q, nheads, d, device=device, dtype=dtype_ref + ) + if softcap > 0.0: + # Ensure the values of qk are at least within softcap range. + q_ref = (q_ref * softcap / 4).detach().requires_grad_() + q_ref = q_ref.to(dtype).to(dtype_ref).requires_grad_() + k_ref = ( + torch.randn( + batch_size, seqlen_k, nheads_kv, d, device=device, dtype=dtype_ref + ) + .to(dtype) + .to(dtype_ref) + .requires_grad_() + ) + v_ref = ( + torch.randn( + batch_size, seqlen_k, nheads_kv, dv, device=device, dtype=dtype_ref + ) + .to(dtype) + .to(dtype_ref) + .requires_grad_() + ) + if has_qv: + qv_ref = ( + torch.randn( + batch_size, seqlen_q, nheads, dv, device=device, dtype=dtype_ref + ) + .to(dtype) + .to(dtype_ref) + ) + else: + qv_ref = None + # Put window_size after QKV randn so that window_size changes from test to test + window_size = (-1, -1) if not local else torch.randint(0, seqlen_k, (2,)) + + if has_sink: + sinks = torch.randn(nheads, dtype=torch.bfloat16, device=device) + else: + sinks = None + + if dtype == torch.float8_e4m3fn: + q_descale, k_descale, v_descale = [ + torch.rand(batch_size, nheads_kv, device=device, dtype=torch.float32) + * 2 + for _ in range(3) + ] + else: + q_descale, k_descale, v_descale = None, None, None + q, k, v = [x.detach().requires_grad_() for x in (q_ref, k_ref, v_ref)] + qv = qv_ref.detach() if has_qv else None + query_padding_mask = generate_random_padding_mask( + seqlen_q, batch_size, device, mode="random", zero_lengths=False + ) + key_padding_mask = generate_random_padding_mask( + seqlen_k, batch_size, device, mode="random", zero_lengths=True + ) + + def _gen_unused_masks(padding_mask, add_unused, max_seq_len, bs, device): + if add_unused: + another_mask = generate_random_padding_mask(max_seq_len, bs, device) + attn_mask = torch.logical_and(padding_mask, another_mask) + unused_mask = torch.logical_xor( + torch.logical_or(padding_mask, another_mask), attn_mask + ) + else: + attn_mask = padding_mask + unused_mask = None + return attn_mask, unused_mask + + query_padding_mask, query_unused_mask = _gen_unused_masks( + query_padding_mask, add_unused_qkv, seqlen_q, batch_size, q.device + ) + key_padding_mask, key_unused_mask = _gen_unused_masks( + key_padding_mask, add_unused_qkv, seqlen_k, batch_size, k.device + ) + + ( + q_unpad, + k_unpad, + v_unpad, + cu_seqlens_q, + cu_seqlens_k, + seqused_q, + seqused_k, + max_seqlen_q, + max_seqlen_k, + q, + k, + v, + output_pad_fn, + dq_pad_fn, + dk_pad_fn, + ) = generate_qkv( + q, + k, + v, + query_padding_mask, + key_padding_mask, + kvpacked=False, + query_unused_mask=query_unused_mask, + key_unused_mask=key_unused_mask, + ) + q_unpad, k_unpad, v_unpad = [ + x.detach().to(dtype).requires_grad_() for x in (q_unpad, k_unpad, v_unpad) + ] + out_ref, attn_ref = attention_ref( + q_ref, + k_ref, + v_ref, + query_padding_mask, + key_padding_mask, + causal=causal, + qv=qv_ref, + q_descale=q_descale, + k_descale=k_descale, + v_descale=v_descale, + window_size=window_size, + softcap=softcap, + sinks=sinks, + ) + out_pt, attn_pt = attention_ref( + q_ref, + k_ref, + v_ref, + query_padding_mask, + key_padding_mask, + causal=causal, + qv=qv_ref, + q_descale=q_descale, + k_descale=k_descale, + v_descale=v_descale, + window_size=window_size, + softcap=softcap, + upcast=False, + reorder_ops=True, + intermediate_dtype=dtype if dtype == torch.float8_e4m3fn else None, + sinks=sinks, + ) + + print(f"Pytorch max diff: {(out_pt - out_ref).abs().max().item()}") + print(f"Pytorch mean diff: {(out_pt - out_ref).abs().mean().item()}") + + if query_unused_mask is not None: + q_zero_masking = rearrange(query_unused_mask, "b s -> b s 1 1") + + # Numerical error if we just do any arithmetic on out_ref + fwd_atol = 2 * (out_ref + 0.3 - 0.3 - out_ref).abs().max().item() + rtol = 2 if softcap == 0.0 else 3 + + pack_gqa_vals = [False, True] if not DISABLE_PACKGQA else [False] + num_splits_vals = [1, 3] if not DISABLE_SPLIT else [1] + for pack_gqa, num_splits in itertools.product(pack_gqa_vals, num_splits_vals): + out_unpad, lse, *rest = flash_attn_varlen_func( + q_unpad, + k_unpad, + v_unpad, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + seqused_q=seqused_q, + seqused_k=seqused_k, + causal=causal, + q_descale=q_descale, + k_descale=k_descale, + v_descale=v_descale, + window_size=window_size, + softcap=softcap, + return_softmax_lse=True, + sinks=sinks, + ) + out = output_pad_fn(out_unpad) + if query_unused_mask is not None: + out.masked_fill_(q_zero_masking, 0.0) + print(f"Output max diff: {(out - out_ref).abs().max().item()}") + print(f"Output mean diff: {(out - out_ref).abs().mean().item()}") + + # Check that FlashAttention's numerical error is at most 3x the numerical error + # of a Pytorch implementation. + assert (out - out_ref).abs().max().item() <= rtol * ( + out_pt - out_ref + ).abs().max().item() + fwd_atol + + if not DISABLE_BACKWARD and dtype != torch.float8_e4m3fn and not has_qv: + g_unpad = torch.randn_like(out_unpad) + do_o = ((g_unpad.float() * out_unpad.float()).sum(-1)).transpose(-1, -2) + dq_unpad, dk_unpad, dv_unpad = torch.autograd.grad( + out_unpad, (q_unpad, k_unpad, v_unpad), g_unpad + ) + dq = dq_pad_fn(dq_unpad) + dk = dk_pad_fn(dk_unpad) + dv = dk_pad_fn(dv_unpad) + if key_unused_mask is not None: + k_zero_masking = rearrange(key_unused_mask, "b s -> b s 1 1") + dk.masked_fill_(k_zero_masking, 0.0) + dv.masked_fill_(k_zero_masking, 0.0) + if query_unused_mask is not None: + dq.masked_fill_(q_zero_masking, 0.0) + # print(f"dO_O max diff: {(softmax_d - do_o).abs().max().item()}") + # assert (softmax_d - do_o).abs().max().item() <= 1e-5 + # assert dq_accum.abs().max().item() == 0.0 + g = output_pad_fn(g_unpad) + + # dq, dk, dv = torch.autograd.grad(out, (q, k, v), g) + dq_ref, dk_ref, dv_ref = torch.autograd.grad(out_ref, (q_ref, k_ref, v_ref), g) + dq_pt, dk_pt, dv_pt = torch.autograd.grad(out_pt, (q_ref, k_ref, v_ref), g) + print(f"dQ max diff: {(dq - dq_ref).abs().max().item()}") + print(f"dK max diff: {(dk - dk_ref).abs().max().item()}") + print(f"dV max diff: {(dv - dv_ref).abs().max().item()}") + print(f"dQ mean diff: {(dq - dq_ref).abs().mean().item()}") + print(f"dK mean diff: {(dk - dk_ref).abs().mean().item()}") + print(f"dV mean diff: {(dv - dv_ref).abs().mean().item()}") + print(f"dQ Pytorch max diff: {(dq_pt - dq_ref).abs().max().item()}") + print(f"dK Pytorch max diff: {(dk_pt - dk_ref).abs().max().item()}") + print(f"dV Pytorch max diff: {(dv_pt - dv_ref).abs().max().item()}") + print(f"dQ Pytorch mean diff: {(dq_pt - dq_ref).abs().mean().item()}") + print(f"dK Pytorch mean diff: {(dk_pt - dk_ref).abs().mean().item()}") + print(f"dV Pytorch mean diff: {(dv_pt - dv_ref).abs().mean().item()}") + + if not DISABLE_BACKWARD and dtype != torch.float8_e4m3fn and not has_qv: + dq_atol = 2 * (dq_ref + 0.3 - 0.3 - dq_ref).abs().max().item() + ( + 0 if softcap == 0 else 3e-4 + ) + assert (dq - dq_ref).abs().max().item() <= rtol * ( + dq_pt - dq_ref + ).abs().max().item() + dq_atol + dk_atol = 2 * (dk_ref + 0.3 - 0.3 - dk_ref).abs().max().item() + ( + 0 if softcap == 0 else 3e-4 + ) + assert (dk - dk_ref).abs().max().item() <= rtol * ( + dk_pt - dk_ref + ).abs().max().item() + dk_atol + dv_atol = 2 * (dv_ref + 0.3 - 0.3 - dv_ref).abs().max().item() + ( + 0 if softcap == 0 else 3e-4 + ) + assert (dv - dv_ref).abs().max().item() <= rtol * ( + dv_pt - dv_ref + ).abs().max().item() + dv_atol + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_flash_attention_4.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_flash_attention_4.py new file mode 100644 index 00000000..2296d71a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_flash_attention_4.py @@ -0,0 +1,1458 @@ +# Adapted from https://github.com/Dao-AILab/flash-attention/blob/8ecf128f683266735ba68e3c106ff67a2611886e/tests/cute/test_flash_attn.py + +# Copyright (c) 2025, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao. + +import itertools +import math +from functools import partial + +import pytest +import torch +import torch.nn.functional as F +from einops import rearrange, repeat + +try: + from flash_attn.layers.rotary import apply_rotary_emb +except ImportError: + apply_rotary_emb = None + +from sgl_kernel.flash_attn import flash_attn_varlen_func, flash_attn_with_kvcache +from sgl_kernel.testing.rotary_embedding import _apply_rotary_emb as apply_rotary_emb + +# from utils import is_hopper # Not used in this test + +# Force sgl_kernel.flash_attn wrappers to use FA4 (Cute-DSL) implementations. +# The wrappers accept a superset of args; for FA4, extra args are ignored. +flash_attn_varlen_func = partial(flash_attn_varlen_func, ver=4) +flash_attn_with_kvcache = partial(flash_attn_with_kvcache, ver=4) + +# Skip this test on Hopper machine +skip_condition = torch.cuda.get_device_capability() < (10, 0) + + +def unpad_input(hidden_states, attention_mask, unused_mask=None): + """ + Arguments: + hidden_states: (batch, seqlen, ...) + attention_mask: (batch, seqlen), bool / int, 1 means valid and 0 means not valid. + unused_mask: (batch, seqlen), bool / int, 1 means the element is allocated but unused. + Return: + hidden_states: (total_nnz, ...), where total_nnz = number of tokens selected in attention_mask + unused_mask. + indices: (total_nnz), the indices of masked tokens from the flattened input sequence. + cu_seqlens: (batch + 1), the cumulative sequence lengths, used to index into hidden_states. + max_seqlen_in_batch: int + seqused: (batch), returns the number of tokens selected in attention_mask + unused_mask. + """ + all_masks = ( + (attention_mask + unused_mask) if unused_mask is not None else attention_mask + ) + seqlens_in_batch = all_masks.sum(dim=-1, dtype=torch.int32) + used_seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32) + indices = torch.nonzero(all_masks.flatten(), as_tuple=False).flatten() + max_seqlen_in_batch = seqlens_in_batch.max().item() + cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0)) + # TD [2022-03-04] We don't want to index with a bool mask, because Pytorch will expand the + # bool mask, then call nonzero to get the indices, then index with those. The indices is @dim + # times larger than it needs to be, wasting memory. It's faster and more memory-efficient to + # index with integer indices. + return ( + rearrange(hidden_states, "b s ... -> (b s) ...")[indices], + indices, + cu_seqlens, + max_seqlen_in_batch, + used_seqlens_in_batch, + ) + + +def pad_input(hidden_states, indices, batch, seqlen): + """ + Arguments: + hidden_states: (total_nnz, ...), where total_nnz = number of tokens in selected in attention_mask. + indices: (total_nnz), the indices that represent the non-masked tokens of the original padded input sequence. + batch: int, batch size for the padded sequence. + seqlen: int, maximum sequence length for the padded sequence. + Return: + hidden_states: (batch, seqlen, ...) + """ + dim = hidden_states.shape[1:] + output = torch.zeros( + (batch * seqlen), *dim, device=hidden_states.device, dtype=hidden_states.dtype + ) + output[indices] = hidden_states + return rearrange(output, "(b s) ... -> b s ...", b=batch) + + +def generate_random_padding_mask( + max_seqlen, batch_size, device, mode="random", zero_lengths=False +): + assert mode in ["full", "random", "third"] + if mode == "full": + lengths = torch.full( + (batch_size, 1), max_seqlen, device=device, dtype=torch.int32 + ) + elif mode == "random": + lengths = torch.randint( + max(0 if zero_lengths else 1, max_seqlen - 20), + max_seqlen + 1, + (batch_size, 1), + device=device, + ) + elif mode == "third": + lengths = torch.randint( + max_seqlen // 3, max_seqlen + 1, (batch_size, 1), device=device + ) + else: + # This should never happen due to the assertion above, but for linter + lengths = torch.full( + (batch_size, 1), max_seqlen, device=device, dtype=torch.int32 + ) + + if zero_lengths: + # Generate zero-lengths every 5 batches and the last batch. + for i in range(batch_size): + if i % 5 == 0: + lengths[i] = 0 + lengths[-1] = 0 + padding_mask = ( + repeat(torch.arange(max_seqlen, device=device), "s -> b s", b=batch_size) + < lengths + ) + return padding_mask + + +def generate_qkv( + q, + k, + v, + query_padding_mask=None, + key_padding_mask=None, + qv=None, + kvpacked=False, + qkvpacked=False, + query_unused_mask=None, + key_unused_mask=None, +): + """ + Arguments: + q: (batch_size, seqlen_q, nheads, d) + k: (batch_size, seqlen_k, nheads_k, d) + v: (batch_size, seqlen_k, nheads_k, d_v) + query_padding_mask: (batch_size, seqlen), bool + key_padding_mask: (batch_size, seqlen), bool + """ + assert not (kvpacked and qkvpacked) + batch_size, seqlen_q, nheads, d = q.shape + d_v = v.shape[-1] + _, seqlen_k, nheads_k, _ = k.shape + assert k.shape == (batch_size, seqlen_k, nheads_k, d) + assert v.shape == (batch_size, seqlen_k, nheads_k, d_v) + if query_unused_mask is not None or key_unused_mask is not None: + assert not kvpacked + assert not qkvpacked + + if query_padding_mask is not None: + q_unpad, indices_q, cu_seqlens_q, max_seqlen_q, seqused_q = unpad_input( + q, query_padding_mask, query_unused_mask + ) + output_pad_fn = lambda output_unpad: pad_input( + output_unpad, indices_q, batch_size, seqlen_q + ) + qv_unpad = ( + rearrange(qv, "b s ... -> (b s) ...")[indices_q] if qv is not None else None + ) + else: + q_unpad = rearrange(q, "b s h d -> (b s) h d") + cu_seqlens_q = torch.arange( + 0, + (batch_size + 1) * seqlen_q, + step=seqlen_q, + dtype=torch.int32, + device=q_unpad.device, + ) + seqused_q = None + max_seqlen_q = seqlen_q + output_pad_fn = lambda output_unpad: rearrange( + output_unpad, "(b s) h d -> b s h d", b=batch_size + ) + qv_unpad = rearrange(qv, "b s ... -> (b s) ...") if qv is not None else None + + if key_padding_mask is not None: + k_unpad, indices_k, cu_seqlens_k, max_seqlen_k, seqused_k = unpad_input( + k, key_padding_mask, key_unused_mask + ) + v_unpad, *rest = unpad_input(v, key_padding_mask, key_unused_mask) + else: + k_unpad = rearrange(k, "b s h d -> (b s) h d") + v_unpad = rearrange(v, "b s h d -> (b s) h d") + cu_seqlens_k = torch.arange( + 0, + (batch_size + 1) * seqlen_k, + step=seqlen_k, + dtype=torch.int32, + device=k_unpad.device, + ) + seqused_k = None + max_seqlen_k = seqlen_k + + if qkvpacked: + assert (query_padding_mask == key_padding_mask).all() + assert nheads == nheads_k + qkv_unpad = torch.stack([q_unpad, k_unpad, v_unpad], dim=1) + qkv = torch.stack([q, k, v], dim=2) + if query_padding_mask is not None: + dqkv_pad_fn = lambda dqkv_unpad: pad_input( + dqkv_unpad, indices_q, batch_size, seqlen_q + ) + else: + dqkv_pad_fn = lambda dqkv_unpad: rearrange( + dqkv_unpad, "(b s) t h d -> b s t h d", b=batch_size + ) + return ( + qkv_unpad.detach().requires_grad_(), + cu_seqlens_q, + max_seqlen_q, + qkv.detach().requires_grad_(), + output_pad_fn, + dqkv_pad_fn, + ) + elif kvpacked: + kv_unpad = torch.stack([k_unpad, v_unpad], dim=1) + kv = torch.stack([k, v], dim=2) + dq_pad_fn = output_pad_fn + if key_padding_mask is not None: + dkv_pad_fn = lambda dkv_unpad: pad_input( + dkv_unpad, indices_k, batch_size, seqlen_k + ) + else: + dkv_pad_fn = lambda dkv_unpad: rearrange( + dkv_unpad, "(b s) t h d -> b s t h d", b=batch_size + ) + return ( + q_unpad.detach().requires_grad_(), + kv_unpad.detach().requires_grad_(), + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + q.detach().requires_grad_(), + kv.detach().requires_grad_(), + output_pad_fn, + dq_pad_fn, + dkv_pad_fn, + ) + else: + dq_pad_fn = output_pad_fn + if key_padding_mask is not None: + dk_pad_fn = lambda dk_unpad: pad_input( + dk_unpad, indices_k, batch_size, seqlen_k + ) + else: + dk_pad_fn = lambda dk_unpad: rearrange( + dk_unpad, "(b s) h d -> b s h d", b=batch_size + ) + return ( + q_unpad.detach().requires_grad_(), + k_unpad.detach().requires_grad_(), + v_unpad.detach().requires_grad_(), + qv_unpad.detach() if qv is not None else None, + cu_seqlens_q, + cu_seqlens_k, + seqused_q, + seqused_k, + max_seqlen_q, + max_seqlen_k, + q.detach().requires_grad_(), + k.detach().requires_grad_(), + v.detach().requires_grad_(), + qv.detach() if qv is not None else None, + output_pad_fn, + dq_pad_fn, + dk_pad_fn, + ) + + +def construct_local_mask( + seqlen_q, + seqlen_k, + window_size=(None, None), + sink_token_length=0, + query_padding_mask=None, + key_padding_mask=None, + key_leftpad=None, + device=None, +): + row_idx = rearrange( + torch.arange(seqlen_q, device=device, dtype=torch.long), "s -> s 1" + ) + col_idx = torch.arange(seqlen_k, device=device, dtype=torch.long) + if key_leftpad is not None: + key_leftpad = rearrange(key_leftpad, "b -> b 1 1 1") + col_idx = repeat(col_idx, "s -> b 1 1 s", b=key_leftpad.shape[0]) + col_idx = torch.where(col_idx >= key_leftpad, col_idx - key_leftpad, 2**32) + sk = ( + seqlen_k + if key_padding_mask is None + else rearrange(key_padding_mask.sum(-1), "b -> b 1 1 1") + ) + sq = ( + seqlen_q + if query_padding_mask is None + else rearrange(query_padding_mask.sum(-1), "b -> b 1 1 1") + ) + if window_size[0] is None: + return col_idx > row_idx + sk - sq + window_size[1] + else: + sk = torch.full_like(col_idx, seqlen_k) if key_padding_mask is None else sk + return torch.logical_or( + col_idx > torch.minimum(row_idx + sk - sq + window_size[1], sk), + torch.logical_and( + col_idx < row_idx + sk - sq - window_size[0], + col_idx >= sink_token_length, + ), + ) + + +def construct_chunk_mask( + seqlen_q, + seqlen_k, + attention_chunk, + query_padding_mask=None, + key_padding_mask=None, + key_leftpad=None, + device=None, +): + row_idx = rearrange( + torch.arange(seqlen_q, device=device, dtype=torch.long), "s -> s 1" + ) + col_idx = torch.arange(seqlen_k, device=device, dtype=torch.long) + if key_leftpad is not None: + key_leftpad = rearrange(key_leftpad, "b -> b 1 1 1") + col_idx = repeat(col_idx, "s -> b 1 1 s", b=key_leftpad.shape[0]) + col_idx = torch.where(col_idx >= key_leftpad, col_idx - key_leftpad, 2**32) + sk = ( + seqlen_k + if key_padding_mask is None + else rearrange(key_padding_mask.sum(-1), "b -> b 1 1 1") + ) + sq = ( + seqlen_q + if query_padding_mask is None + else rearrange(query_padding_mask.sum(-1), "b -> b 1 1 1") + ) + sk = torch.full_like(col_idx, seqlen_k) if key_padding_mask is None else sk + # Subtract remainder instead of divide and then multiply to take care of negative values + col_limit_left_chunk = row_idx + sk - sq - (row_idx + sk - sq) % attention_chunk + return torch.logical_or( + col_idx < col_limit_left_chunk, + col_idx >= col_limit_left_chunk + attention_chunk, + ) + + +def attention_ref( + q, + k, + v, + query_padding_mask=None, + key_padding_mask=None, + key_leftpad=None, + attn_bias=None, + dropout_p=0.0, + dropout_mask=None, + causal=False, + qv=None, + q_descale=None, + k_descale=None, + v_descale=None, + window_size=(None, None), + attention_chunk=0, + sink_token_length=0, + learnable_sink=None, + softcap=0.0, + upcast=True, + reorder_ops=False, + intermediate_dtype=None, +): + """ + Arguments: + q: (batch_size, seqlen_q, nheads, head_dim) + k: (batch_size, seqlen_k, nheads, head_dim) + v: (batch_size, seqlen_k, nheads, head_dim_v) + qv: (batch_size, seqlen_q, nheads, head_dim_v) + query_padding_mask: (batch_size, seqlen_q) + key_padding_mask: (batch_size, seqlen_k) + attn_bias: broadcastable to (batch_size, nheads, seqlen_q, seqlen_k) + dropout_p: float + dropout_mask: (batch_size, nheads, seqlen_q, seqlen_k) + causal: whether to apply causal masking + upcast: whether to cast all inputs to fp32, do all computation in fp32, then cast + output back to fp16/bf16. + reorder_ops: whether to change the order of operations (scaling k instead of scaling k, etc.) + without changing the math. This is to estimate the numerical error from operation + reordering. + Output: + output: (batch_size, seqlen_q, nheads, head_dim_v) + attention: (batch_size, nheads, seqlen_q, seqlen_k), softmax after dropout + """ + if causal: + window_size = (window_size[0], 0) + dtype_og = q.dtype + if upcast: + q, k, v = q.float(), k.float(), v.float() + qv = qv.float() if qv is not None else None + if q_descale is not None: + q_descale = repeat(q_descale, "b h -> b 1 (h g) 1", g=q.shape[2] // k.shape[2]) + q = (q.float() * q_descale).to(q.dtype) + qv = (qv.float() * q_descale).to(qv.dtype) if qv is not None else None + if k_descale is not None: + k = (k.float() * rearrange(k_descale, "b h -> b 1 h 1")).to(dtype=k.dtype) + if v_descale is not None: + v = (v.float() * rearrange(v_descale, "b h -> b 1 h 1")).to(dtype=v.dtype) + seqlen_q, seqlen_k = q.shape[1], k.shape[1] + k = repeat(k, "b s h d -> b s (h g) d", g=q.shape[2] // k.shape[2]) + v = repeat(v, "b s h d -> b s (h g) d", g=q.shape[2] // v.shape[2]) + d = q.shape[-1] + dv = v.shape[-1] + softmax_scale = 1.0 / math.sqrt(d if qv is None else d + dv) + if not reorder_ops: + scores = torch.einsum("bthd,bshd->bhts", q * softmax_scale, k) + else: + scores = torch.einsum("bthd,bshd->bhts", q, k * softmax_scale) + if qv is not None: + scores = scores + torch.einsum("bthd,bshd->bhts", qv * softmax_scale, v) + if softcap > 0: + scores = torch.tanh(scores / softcap) * softcap + if key_padding_mask is not None: + scores.masked_fill_( + rearrange(~key_padding_mask, "b s -> b 1 1 s"), float("-inf") + ) + local_mask = None + if window_size[0] is not None or window_size[1] is not None: + local_mask = construct_local_mask( + seqlen_q, + seqlen_k, + window_size, + sink_token_length, + query_padding_mask, + key_padding_mask, + key_leftpad=key_leftpad, + device=q.device, + ) + if attention_chunk > 0: + chunk_mask = construct_chunk_mask( + seqlen_q, + seqlen_k, + attention_chunk, + query_padding_mask, + key_padding_mask, + key_leftpad=key_leftpad, + device=q.device, + ) + local_mask = ( + torch.logical_or(local_mask, chunk_mask) + if local_mask is not None + else chunk_mask + ) + if local_mask is not None: + scores.masked_fill_(local_mask, float("-inf")) + if attn_bias is not None: + scores = scores + attn_bias + if learnable_sink is None: + attention = torch.softmax(scores, dim=-1).to(v.dtype) + else: + scores_fp32 = scores.to(torch.float32) + logits_max = torch.amax(scores_fp32, dim=-1, keepdim=True) + learnable_sink = rearrange(learnable_sink, "h -> h 1 1") + logits_or_sinks_max = torch.maximum(learnable_sink, logits_max) + unnormalized_scores = torch.exp(scores_fp32 - logits_or_sinks_max) + normalizer = unnormalized_scores.sum(dim=-1, keepdim=True) + torch.exp( + learnable_sink - logits_or_sinks_max + ) + attention = (unnormalized_scores / normalizer).to(v.dtype) + # We want to mask here so that the attention matrix doesn't have any NaNs + # Otherwise we'll get NaN in dV + if query_padding_mask is not None: + attention = attention.masked_fill( + rearrange(~query_padding_mask, "b s -> b 1 s 1"), 0.0 + ) + # Without this we might get NaN in dv + if key_padding_mask is not None: + attention = attention.masked_fill( + rearrange(~key_padding_mask, "b s -> b 1 1 s"), 0.0 + ) + # Some rows might be completely masked out so we fill them with zero instead of NaN + if local_mask is not None: + attention = attention.masked_fill( + torch.all(local_mask, dim=-1, keepdim=True), 0.0 + ) + dropout_scaling = 1.0 / (1 - dropout_p) + # attention_drop = attention.masked_fill(~dropout_mask, 0.0) * dropout_scaling + # output = torch.einsum('bhts,bshd->bthd', attention_drop , v) + if dropout_mask is not None: + attention_drop = attention.masked_fill(~dropout_mask, 0.0) + else: + attention_drop = attention + if intermediate_dtype is not None: + attention_drop = attention_drop.to(intermediate_dtype).to(attention_drop.dtype) + output = torch.einsum("bhts,bshd->bthd", attention_drop, v * dropout_scaling) + if query_padding_mask is not None: + output.masked_fill_(rearrange(~query_padding_mask, "b s -> b s 1 1"), 0.0) + return output.to(dtype=dtype_og), attention.to(dtype=dtype_og) + + +@pytest.mark.skipif( + skip_condition, reason="FA4 Requires compute capability of 10 or above." +) +# @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float8_e4m3fn]) +@pytest.mark.parametrize("dtype", [torch.bfloat16]) +@pytest.mark.parametrize("mha_type", ["mha", "mqa", "gqa"]) +# @pytest.mark.parametrize("mha_type", ["mqa"]) +@pytest.mark.parametrize("has_learnable_sink", [False, True]) +# @pytest.mark.parametrize("has_learnable_sink", [False]) +# @pytest.mark.parametrize("has_qv", [False, True]) +@pytest.mark.parametrize("has_qv", [False]) +# @pytest.mark.parametrize("deterministic", [False, True]) +@pytest.mark.parametrize("deterministic", [False]) +# @pytest.mark.parametrize("softcap", [0.0, 15.0]) +@pytest.mark.parametrize("softcap", [0.0]) +# @pytest.mark.parametrize("local", [False, True]) +@pytest.mark.parametrize("local", [False]) +@pytest.mark.parametrize("causal", [False, True]) +# @pytest.mark.parametrize("causal", [False]) +# @pytest.mark.parametrize("add_unused_qkv", [False, True]) +@pytest.mark.parametrize("add_unused_qkv", [False]) +# @pytest.mark.parametrize("d", [32, 64, 96, 128, 160, 192, 224, 256]) +# @pytest.mark.parametrize('d', [32, 40, 64, 80, 96, 128, 160, 192, 256]) +# @pytest.mark.parametrize('d', [32, 64, 96, 128, 160, 192]) +# @pytest.mark.parametrize('d', [56, 80]) +# @pytest.mark.parametrize('d', [32, 40, 64, 80, 96, 128]) +# @pytest.mark.parametrize("d", [64, 96, 128]) +@pytest.mark.parametrize("d", [128, 192]) +# @pytest.mark.parametrize("d", [192]) +@pytest.mark.parametrize( + "seqlen_q,seqlen_k", + [ + # (1, 1), + # (1, 3), + # (2, 1), + (511, 1), + (3, 513), + (64, 128), + (128, 128), + (256, 256), + # (113, 203), + # (128, 217), + # (113, 211), + # (108, 256), + # (256, 512), + (307, 256), + (640, 128), + (512, 256), + (1024, 1024), + (1023, 1024), + (1024, 1023), + (2048, 2048), + ], +) +def test_flash_attn_varlen_output( + seqlen_q, + seqlen_k, + d, + add_unused_qkv, + causal, + local, + softcap, + deterministic, + has_qv, + has_learnable_sink, + mha_type, + dtype, +): + if ( + causal or local + ): # Right now we only support causal attention with seqlen_k == seqlen_q + seqlen_k = seqlen_q + device = "cuda" + # set seed + torch.random.manual_seed(seqlen_q + seqlen_k + d + int(causal) * 2 + int(local)) + batch_size = 49 if seqlen_q <= 1024 else 7 + nheads = 6 + # batch_size = 1 + # nheads = 1 + nheads_kv = nheads if mha_type == "mha" else (3 if mha_type == "gqa" else 1) + dtype_ref = torch.bfloat16 if dtype == torch.float8_e4m3fn else dtype + # dv_vals = [128, d] if d > 128 and d <= 192 else ([256, 512, d] if d <= 64 else [d]) + dv_vals = [128] if d == 192 else ([d] if d != 128 else [64, d]) + if dtype == torch.float8_e4m3fn: + dv_vals = [d] + # attention_chunk_vals = [torch.randint(1, seqlen_k * 2, (1,)).item(), 0] if seqlen_q <= seqlen_k else [0] + attention_chunk_vals = [0] + for dv, attention_chunk in itertools.product(dv_vals, attention_chunk_vals): + q_ref = torch.randn( + batch_size, seqlen_q, nheads, d, device=device, dtype=dtype_ref + ) + if softcap > 0.0: + # Ensure the values of qk are at least within softcap range. + q_ref = (q_ref * softcap / 4).detach().requires_grad_() + q_ref = q_ref.to(dtype).to(dtype_ref).requires_grad_() + k_ref = ( + torch.randn( + batch_size, seqlen_k, nheads_kv, d, device=device, dtype=dtype_ref + ) + .to(dtype) + .to(dtype_ref) + .requires_grad_() + ) + v_ref = ( + torch.randn( + batch_size, seqlen_k, nheads_kv, dv, device=device, dtype=dtype_ref + ) + .to(dtype) + .to(dtype_ref) + .requires_grad_() + ) + if has_qv: + qv_ref = ( + torch.randn( + batch_size, seqlen_q, nheads, dv, device=device, dtype=dtype_ref + ) + .to(dtype) + .to(dtype_ref) + ) + else: + qv_ref = None + # Put window_size after QKV randn so that window_size changes from test to test + window_size = ( + (None, None) if not local else torch.randint(0, seqlen_k, (2,)).tolist() + ) + if has_learnable_sink: + learnable_sink = torch.randn(nheads, dtype=torch.bfloat16, device=device) + else: + learnable_sink = None + if dtype == torch.float8_e4m3fn: + q_descale, k_descale, v_descale = [ + torch.rand(batch_size, nheads_kv, device=device, dtype=torch.float32) + * 2 + for _ in range(3) + ] + else: + q_descale, k_descale, v_descale = None, None, None + q, k, v = [x.detach().requires_grad_() for x in (q_ref, k_ref, v_ref)] + qv = qv_ref.detach() if has_qv else None + query_padding_mask = generate_random_padding_mask( + seqlen_q, batch_size, device, mode="random", zero_lengths=False + ) + # TODO: test zero_lengths + key_padding_mask = generate_random_padding_mask( + # seqlen_k, batch_size, device, mode="random", zero_lengths=True + seqlen_k, + batch_size, + device, + mode="random", + zero_lengths=False, + ) + + def _gen_unused_masks(padding_mask, add_unused, max_seq_len, bs, device): + if add_unused: + another_mask = generate_random_padding_mask(max_seq_len, bs, device) + attn_mask = torch.logical_and(padding_mask, another_mask) + unused_mask = torch.logical_xor( + torch.logical_or(padding_mask, another_mask), attn_mask + ) + else: + attn_mask = padding_mask + unused_mask = None + return attn_mask, unused_mask + + query_padding_mask, query_unused_mask = _gen_unused_masks( + query_padding_mask, add_unused_qkv, seqlen_q, batch_size, q.device + ) + # query_padding_mask[:] = True + # query_unused_mask = None + key_padding_mask, key_unused_mask = _gen_unused_masks( + key_padding_mask, add_unused_qkv, seqlen_k, batch_size, k.device + ) + + if causal or local: + key_padding_mask = query_padding_mask + + result = generate_qkv( + q, + k, + v, + query_padding_mask, + key_padding_mask, + qv=qv, + kvpacked=False, + query_unused_mask=query_unused_mask, + key_unused_mask=key_unused_mask, + ) + ( + q_unpad, # 0 + k_unpad, # 1 + v_unpad, # 2 + qv_unpad, # 3 + cu_seqlens_q, # 4 + cu_seqlens_k, # 5 + seqused_q, # 6 + seqused_k, # 7 + max_seqlen_q, # 8 + max_seqlen_k, # 9 + q, # 10 + k, # 11 + v, # 12 + qv, # 13 + output_pad_fn, # 14 + dq_pad_fn, # 15 + dk_pad_fn, # 16 + ) = result + q_unpad, k_unpad, v_unpad = [ + x.detach().to(dtype).requires_grad_() for x in (q_unpad, k_unpad, v_unpad) + ] + out_ref, attn_ref = attention_ref( + q_ref, + k_ref, + v_ref, + query_padding_mask, + key_padding_mask, + causal=causal, + qv=qv_ref, + q_descale=q_descale, + k_descale=k_descale, + v_descale=v_descale, + window_size=window_size, + attention_chunk=attention_chunk, + learnable_sink=learnable_sink, + softcap=softcap, + ) + out_pt, attn_pt = attention_ref( + q_ref, + k_ref, + v_ref, + query_padding_mask, + key_padding_mask, + causal=causal, + qv=qv_ref, + q_descale=q_descale, + k_descale=k_descale, + v_descale=v_descale, + window_size=window_size, + attention_chunk=attention_chunk, + learnable_sink=learnable_sink, + softcap=softcap, + upcast=False, + reorder_ops=True, + intermediate_dtype=dtype if dtype == torch.float8_e4m3fn else None, + ) + + print(f"Pytorch max diff: {(out_pt - out_ref).abs().max().item()}") + print(f"Pytorch mean diff: {(out_pt - out_ref).abs().mean().item()}") + + if query_unused_mask is not None: + q_zero_masking = rearrange(query_unused_mask, "b s -> b s 1 1") + + # Numerical error if we just do any arithmetic on out_ref + fwd_atol = 2 * (out_ref + 0.3 - 0.3 - out_ref).abs().max().item() + rtol = 2 if softcap == 0.0 else 3 + + pack_gqa_vals = [False, True, None] + # num_splits_vals = [1, 3] + num_splits_vals = [1] + for pack_gqa, num_splits in itertools.product(pack_gqa_vals, num_splits_vals): + out_unpad, lse = flash_attn_varlen_func( + q_unpad, + k_unpad, + v_unpad, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + # max_seqlen_q and max_seqlen_k not needed for FA4 + seqused_q=seqused_q, + seqused_k=seqused_k, + causal=causal, + window_size=window_size, + softcap=softcap, + sinks=learnable_sink, # FA4 uses learnable_sink, not sinks + pack_gqa=pack_gqa, + return_softmax_lse=True, + ver=4, # Use FA4 + ) + out = output_pad_fn(out_unpad) + if query_unused_mask is not None: + out.masked_fill_(q_zero_masking, 0.0) + print(f"Output max diff: {(out - out_ref).abs().max().item()}") + print(f"Output mean diff: {(out - out_ref).abs().mean().item()}") + # if not causal: + # print(f"LSE max diff: {(lse - lse_ref).abs().max().item()}") + # breakpoint() + + # Check that FlashAttention's numerical error is at most 3x the numerical error + # of a Pytorch implementation. + assert (out - out_ref).abs().max().item() <= rtol * ( + out_pt - out_ref + ).abs().max().item() + fwd_atol + + if ( + dtype != torch.float8_e4m3fn + and not has_qv + and not dv > 256 + and not attention_chunk != 0 + and dv == d + and not has_learnable_sink + and False + ): + g_unpad = torch.randn_like(out_unpad) + do_o = ((g_unpad.float() * out_unpad.float()).sum(-1)).transpose(-1, -2) + # import flash_attn_3_cuda + # dq_unpad, dk_unpad, dv_unpad, softmax_d, dq_accum, lse_log2 = flash_attn_3_cuda.bwd_varlen( + # g_unpad, + # q_unpad, + # k_unpad, + # v_unpad, + # out_unpad, + # lse, + # None, + # None, + # None, + # cu_seqlens_q, + # cu_seqlens_k, + # None, None, + # max_seqlen_q, + # max_seqlen_k, + # d ** (-0.5), + # causal, + # window_size[0], window_size[1], + # softcap, + # deterministic, + # 0, # sm_margin + # ) + dq_unpad, dk_unpad, dv_unpad = torch.autograd.grad( + out_unpad, (q_unpad, k_unpad, v_unpad), g_unpad + ) + dq = dq_pad_fn(dq_unpad) + dk = dk_pad_fn(dk_unpad) + dv = dk_pad_fn(dv_unpad) + if key_unused_mask is not None: + k_zero_masking = rearrange(key_unused_mask, "b s -> b s 1 1") + dk.masked_fill_(k_zero_masking, 0.0) + dv.masked_fill_(k_zero_masking, 0.0) + if query_unused_mask is not None: + dq.masked_fill_(q_zero_masking, 0.0) + # print(f"dO_O max diff: {(softmax_d - do_o).abs().max().item()}") + # assert (softmax_d - do_o).abs().max().item() <= 1e-5 + # assert dq_accum.abs().max().item() == 0.0 + g = output_pad_fn(g_unpad) + + # qk = torch.einsum('bthd,bshd->bhts', q / (d ** 0.5), k).float() + # qk = torch.masked_fill(qk, rearrange(~key_padding_mask, "b s -> b 1 1 s"), float("-inf")) + # dS = torch.einsum('bthd,bshd->bhts', g.float(), v.float()) + # P = torch.softmax(qk, -1) + # dP = P * (dS - (g.float() * out.float()).sum(-1).transpose(1, 2).unsqueeze(-1)) + # dQ = torch.einsum('bhts,bshd->bthd', dP, k.float()) + # dV = torch.einsum('bhts,bthd->bshd', P, g.float()) + # dK = torch.einsum('bhts,bthd->bshd', dP, q.float()) + + # dq, dk, dv = torch.autograd.grad(out, (q, k, v), g) + dq_ref, dk_ref, dv_ref = torch.autograd.grad( + out_ref, (q_ref, k_ref, v_ref), g + ) + dq_pt, dk_pt, dv_pt = torch.autograd.grad(out_pt, (q_ref, k_ref, v_ref), g) + print(f"dQ max diff: {(dq - dq_ref).abs().max().item()}") + print(f"dK max diff: {(dk - dk_ref).abs().max().item()}") + print(f"dV max diff: {(dv - dv_ref).abs().max().item()}") + print(f"dQ mean diff: {(dq - dq_ref).abs().mean().item()}") + print(f"dK mean diff: {(dk - dk_ref).abs().mean().item()}") + print(f"dV mean diff: {(dv - dv_ref).abs().mean().item()}") + print(f"dQ Pytorch max diff: {(dq_pt - dq_ref).abs().max().item()}") + print(f"dK Pytorch max diff: {(dk_pt - dk_ref).abs().max().item()}") + print(f"dV Pytorch max diff: {(dv_pt - dv_ref).abs().max().item()}") + print(f"dQ Pytorch mean diff: {(dq_pt - dq_ref).abs().mean().item()}") + print(f"dK Pytorch mean diff: {(dk_pt - dk_ref).abs().mean().item()}") + print(f"dV Pytorch mean diff: {(dv_pt - dv_ref).abs().mean().item()}") + # breakpoint() + dq_atol = 2 * (dq_ref + 0.3 - 0.3 - dq_ref).abs().max().item() + ( + 0 if softcap == 0 else 3e-4 + ) + assert (dq - dq_ref).abs().max().item() <= rtol * ( + dq_pt - dq_ref + ).abs().max().item() + dq_atol + dk_atol = 2 * (dk_ref + 0.3 - 0.3 - dk_ref).abs().max().item() + ( + 0 if softcap == 0 else 3e-4 + ) + assert (dk - dk_ref).abs().max().item() <= rtol * ( + dk_pt - dk_ref + ).abs().max().item() + dk_atol + dv_atol = 2 * (dv_ref + 0.3 - 0.3 - dv_ref).abs().max().item() + ( + 0 if softcap == 0 else 3e-4 + ) + assert (dv - dv_ref).abs().max().item() <= rtol * ( + dv_pt - dv_ref + ).abs().max().item() + dv_atol + + +@pytest.mark.skipif( + skip_condition, reason="FA4 Requires compute capability of 10 or above." +) +# @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float8_e4m3fn]) +@pytest.mark.parametrize("dtype", [torch.bfloat16]) +# @pytest.mark.parametrize("dtype", [torch.float8_e4m3fn]) +@pytest.mark.parametrize("mha_type", ["mha", "mqa", "gqa"]) +# @pytest.mark.parametrize("mha_type", ["mha"]) +@pytest.mark.parametrize("has_learnable_sink", [False, True]) +# @pytest.mark.parametrize("has_learnable_sink", [False]) +# @pytest.mark.parametrize("new_kv", [False, True]) +@pytest.mark.parametrize("new_kv", [False]) +# @pytest.mark.parametrize("local", [False, True]) +@pytest.mark.parametrize("local", [False]) +# @pytest.mark.parametrize("causal", [False, True]) +@pytest.mark.parametrize("causal", [True]) +# @pytest.mark.parametrize("seqlen_new_eq_seqlen_q", [True, False]) +@pytest.mark.parametrize("seqlen_new_eq_seqlen_q", [False]) +# @pytest.mark.parametrize("has_rotary_seqlens", [False, True]) +@pytest.mark.parametrize("has_rotary_seqlens", [False]) +# @pytest.mark.parametrize("rotary_interleaved", [False, True]) +@pytest.mark.parametrize("rotary_interleaved", [True]) +# @pytest.mark.parametrize("rotary_fraction", [0.0, 0.5, 1.0]) +@pytest.mark.parametrize("rotary_fraction", [0.0]) +# @pytest.mark.parametrize("page_size", [None] + ([1, 4, 128])) +# @pytest.mark.parametrize("page_size", [None, 128]) +@pytest.mark.parametrize("page_size", [128]) +# @pytest.mark.parametrize("has_leftpad", [False, True]) +@pytest.mark.parametrize("has_leftpad", [False]) +# @pytest.mark.parametrize("has_batch_idx", [False, True]) +@pytest.mark.parametrize("has_batch_idx", [False]) +# @pytest.mark.parametrize("varlen_q", [False, True]) +@pytest.mark.parametrize("varlen_q", [False]) +# @pytest.mark.parametrize("d", [32, 59, 64, 80, 128, 256]) +# @pytest.mark.parametrize("d", [32, 64, 96, 128, 160, 192, 224, 256]) +# @pytest.mark.parametrize('d', [32, 40, 64, 80, 96, 128, 160, 192]) +# @pytest.mark.parametrize('d', [56, 80]) +# @pytest.mark.parametrize("d", [128]) +@pytest.mark.parametrize("d", [64]) +# @pytest.mark.parametrize("d", [192]) +@pytest.mark.parametrize( + "seqlen_q,seqlen_k", + [ + (1, 128), + (1, 339), + (3, 1024), + (64, 800), + (64, 256), + (3, 799), + (64, 2048), + (16, 20000), + # # (1, 128 * 1024), + # # (16, 128 * 1024), + # (128, 128), + # (256, 512), # To test appending KV with more than 1 block + # (2048, 3577), # Enough tile to test persistent scheduler + ], +) +# @pytest.mark.parametrize('seqlen_q,seqlen_k', [(256, 128)]) +def test_flash_attn_kvcache( + seqlen_q, + seqlen_k, + d, + varlen_q, + has_batch_idx, + has_leftpad, + page_size, + rotary_fraction, + rotary_interleaved, + has_rotary_seqlens, + seqlen_new_eq_seqlen_q, + causal, + local, + new_kv, + has_learnable_sink, + mha_type, + dtype, +): + if page_size is not None and seqlen_k % page_size != 0: + pytest.skip() + if seqlen_q > seqlen_k and new_kv: + pytest.skip() + if not new_kv and rotary_fraction > 0.0: + pytest.skip() + if rotary_fraction == 0.0 and has_rotary_seqlens: + pytest.skip() + device = "cuda" + # set seed + torch.random.manual_seed(0) + batch_size = 5 + # batch_size = 1 + batch_size_cache = batch_size if not has_batch_idx else batch_size * 2 + nheads = 6 + # nheads = 1 + # rotary_dim must be a multiple of 16, and must be <= d + rotary_dim = math.floor(int(rotary_fraction * d) / 16) * 16 + nheads_k = nheads if mha_type == "mha" else (1 if mha_type == "mqa" else 3) + assert nheads % nheads_k == 0 + dtype_ref = torch.bfloat16 if dtype == torch.float8_e4m3fn else dtype + # dv_vals = [128, d] if d > 128 and d <= 192 else ([256, 512, d] if d <= 64 else [d]) + dv_vals = [d] + if dtype == torch.float8_e4m3fn: + dv_vals = [d] + # attention_chunk_vals = [torch.randint(1, seqlen_k * 2, (1,)).item(), 0] if (causal or local) else [0] + attention_chunk_vals = [0] + for dv, attention_chunk in itertools.product(dv_vals, attention_chunk_vals): + # has_qv = d == 64 and dv >= 256 + has_qv = False + q = ( + torch.randn(batch_size, seqlen_q, nheads, d, device=device, dtype=dtype_ref) + .to(dtype) + .to(dtype_ref) + ) + if has_qv: + qv = ( + torch.randn( + batch_size, seqlen_q, nheads, dv, device=device, dtype=dtype_ref + ) + .to(dtype) + .to(dtype_ref) + ) + else: + qv = None + if varlen_q: + query_padding_mask = generate_random_padding_mask( + seqlen_q, batch_size, device, mode="random" + ) + q_unpad, indices_q, cu_seqlens_q, max_seqlen_q, *rest = unpad_input( + q, query_padding_mask + ) + output_pad_fn = lambda output_unpad: pad_input( + output_unpad, indices_q, batch_size, seqlen_q + ) + qv_unpad = ( + rearrange(qv, "b s ... -> (b s) ...")[indices_q] if has_qv else None + ) + else: + query_padding_mask = None + q_unpad = q + qv_unpad = qv + cu_seqlens_q, max_seqlen_q = None, None + # Put window_size after QKV randn so that window_size changes from test to test + window_size = ( + (None, None) if not local else torch.randint(0, seqlen_k, (2,)).tolist() + ) + if has_learnable_sink: + learnable_sink = torch.randn(nheads, dtype=torch.bfloat16, device=device) + else: + learnable_sink = None + + seqlen_new = ( + seqlen_q + if seqlen_new_eq_seqlen_q + else torch.randint(1, seqlen_q + 1, (1,)).item() + ) + cu_seqlens_k_new = None + key_new_padding_mask = None + if new_kv: + k = ( + torch.randn( + batch_size, seqlen_new, nheads_k, d, device=device, dtype=dtype_ref + ) + .to(dtype) + .to(dtype_ref) + ) + v = ( + torch.randn( + batch_size, seqlen_new, nheads_k, dv, device=device, dtype=dtype_ref + ) + .to(dtype) + .to(dtype_ref) + ) + if varlen_q: # k & v are also varlen + key_new_padding_mask = generate_random_padding_mask( + seqlen_new, batch_size, device, mode="random" + ) + k_unpad, indices_k, cu_seqlens_k_new, *rest = unpad_input( + k, key_new_padding_mask + ) + v_unpad, *rest = unpad_input(v, key_new_padding_mask) + else: + k_unpad, v_unpad = k, v + else: + k, v, k_unpad, v_unpad = None, None, None, None + if page_size is None: + k_cache = ( + torch.randn( + batch_size_cache, + seqlen_k, + nheads_k, + d, + device=device, + dtype=dtype_ref, + ) + .to(dtype) + .to(dtype_ref) + ) + v_cache = ( + torch.randn( + batch_size_cache, + seqlen_k, + nheads_k, + dv, + device=device, + dtype=dtype_ref, + ) + .to(dtype) + .to(dtype_ref) + ) + page_table = None + num_blocks = None + else: + ( + k_cache, + v_cache, + page_table, + k_cache_paged, + v_cache_paged, + num_blocks, + ) = _generate_block_kvcache( + seqlen_k, + page_size, + batch_size_cache, + nheads_k, + d, + dv, + device, + dtype, + dtype_ref, + ) + cache_seqlens = torch.randint( + 0 if new_kv else 1, + # If we don't use seqlen_q in the case of causal and rotary, cos/sin won't be long enough + ( + ( + seqlen_k + - (seqlen_q if (causal or local) and rotary_dim > 1 else seqlen_new) + + 1 + ) + if new_kv + else (seqlen_k + 1) + ), + (batch_size,), + dtype=torch.int32, + device=device, + ) + if has_leftpad: + cache_leftpad = torch.cat( + [ + ( + torch.randint( + 0, + cache_seqlens[i].item(), + (1,), + dtype=torch.int32, + device=device, + ) + if cache_seqlens[i].item() > 0 + else torch.zeros(1, dtype=torch.int32, device=device) + ) + for i in range(batch_size) + ] + ) + else: + cache_leftpad = None + if has_batch_idx: + cache_batch_idx = torch.randperm( + batch_size_cache, dtype=torch.int32, device=device + )[:batch_size] + else: + cache_batch_idx = None + arange = rearrange(torch.arange(seqlen_k, device=device), "s -> 1 s") + cache_seqlens_expanded = rearrange(cache_seqlens, "b -> b 1") + if not new_kv: + key_padding_mask = arange < cache_seqlens_expanded + else: + k_new_seqlens = ( + key_new_padding_mask.sum(-1, keepdims=True) if varlen_q else seqlen_new + ) + key_padding_mask = arange < cache_seqlens_expanded + k_new_seqlens + if has_leftpad: + key_padding_mask = torch.logical_and( + key_padding_mask, + arange >= cache_leftpad.unsqueeze(-1).expand(-1, seqlen_k), + ) + # cache_seqlens = torch.tensor([64], dtype=torch.int32, device=device) + rotary_seqlens = cache_seqlens if not has_rotary_seqlens else cache_seqlens // 2 + if rotary_dim > 0: + angle = ( + torch.rand( + seqlen_k if page_size is None else num_blocks * page_size, + rotary_dim // 2, + device=device, + ) + * 2 + * math.pi + ) + cos = torch.cos(angle).to(dtype=dtype_ref).to(dtype).to(dtype_ref) + sin = torch.sin(angle).to(dtype=dtype_ref).to(dtype).to(dtype_ref) + if causal or local: + q_ro = apply_rotary_emb( + q, + cos, + sin, + seqlen_offsets=rotary_seqlens, + interleaved=rotary_interleaved, + ) + else: + q_ro = rearrange( + apply_rotary_emb( + rearrange(q, "b s h d -> b 1 (s h) d"), + cos, + sin, + seqlen_offsets=rotary_seqlens, + interleaved=rotary_interleaved, + ), + "b 1 (s h) d -> b s h d", + s=seqlen_q, + ) + # q_ro = q + k_ro = apply_rotary_emb( + k, + cos, + sin, + seqlen_offsets=rotary_seqlens, + interleaved=rotary_interleaved, + ) + else: + cos, sin = None, None + q_ro, k_ro = q, k + # k_cache[:, 64:] = -1 + k_cache_ref = ( + k_cache if not has_batch_idx else k_cache[cache_batch_idx] + ).clone() + v_cache_ref = ( + v_cache if not has_batch_idx else v_cache[cache_batch_idx] + ).clone() + if new_kv: + update_mask = torch.logical_and( + cache_seqlens_expanded <= arange, + arange < cache_seqlens_expanded + k_new_seqlens, + ) + k_to_update = rearrange(k_ro, "b s ... -> (b s) ...") + v_to_update = rearrange(v, "b s ... -> (b s) ...") + if varlen_q: + k_to_update = k_to_update[indices_k] + v_to_update = v_to_update[indices_k] + k_cache_ref[update_mask] = k_to_update + v_cache_ref[update_mask] = v_to_update + k_cache_rep = repeat( + k_cache_ref, "b s h d -> b s (h g) d", g=nheads // nheads_k + ) + v_cache_rep = repeat( + v_cache_ref, "b s h d -> b s (h g) d", g=nheads // nheads_k + ) + out_ref, _ = attention_ref( + q_ro, + k_cache_rep, + v_cache_rep, + query_padding_mask, + key_padding_mask, + causal=causal, + qv=qv, + window_size=window_size, + learnable_sink=learnable_sink, + attention_chunk=attention_chunk, + key_leftpad=cache_leftpad, + ) + out_pt, _ = attention_ref( + q_ro, + k_cache_rep, + v_cache_rep, + query_padding_mask, + key_padding_mask, + causal=causal, + qv=qv, + window_size=window_size, + learnable_sink=learnable_sink, + attention_chunk=attention_chunk, + upcast=False, + reorder_ops=True, + key_leftpad=cache_leftpad, + intermediate_dtype=dtype if dtype == torch.float8_e4m3fn else None, + ) + q = q.to(dtype) + q_unpad = q_unpad.to(dtype) if varlen_q else None + k_cache = k_cache.to(dtype) + v_cache = v_cache.to(dtype) + k_cache_paged = k_cache_paged.to(dtype) if page_size is not None else None + v_cache_paged = v_cache_paged.to(dtype) if page_size is not None else None + k = k.to(dtype) if k is not None else None + v = v.to(dtype) if v is not None else None + k_unpad = k_unpad.to(dtype) if k_unpad is not None else None + v_unpad = v_unpad.to(dtype) if v_unpad is not None else None + qv = qv.to(dtype) if qv is not None else None + qv_unpad = qv_unpad.to(dtype) if (varlen_q and qv is not None) else None + cos = cos.to(dtype) if cos is not None else None + sin = sin.to(dtype) if sin is not None else None + k_cache_saved = k_cache.clone() if page_size is None else k_cache_paged.clone() + v_cache_saved = v_cache.clone() if page_size is None else v_cache_paged.clone() + # num_splits_vals = [1, 0] + num_splits_vals = [1] + # precompute_metadata_vals = [False, True] + precompute_metadata_vals = [False] + for num_splits, precompute_metadata in itertools.product( + num_splits_vals, precompute_metadata_vals + ): + # if precompute_metadata: + # scheduler_metadata = get_scheduler_metadata( + # batch_size, max_seqlen_q if varlen_q else seqlen_q, seqlen_k, nheads, nheads_k, d, + # cache_seqlens, q.dtype, headdim_v=dv, cu_seqlens_q=cu_seqlens_q, + # cu_seqlens_k_new=cu_seqlens_k_new, cache_leftpad=cache_leftpad, + # max_seqlen_k_new=seqlen_new, page_size=page_size, + # causal=causal, window_size=window_size, attention_chunk=attention_chunk, + # num_splits=num_splits + # ) + # else: + # scheduler_metadata = None + scheduler_metadata = None + # Repeat to test metadata reuse + for _ in range(1 if not precompute_metadata else 2): + if page_size is None: + k_cache.copy_(k_cache_saved) + v_cache.copy_(v_cache_saved) + else: + k_cache_paged.copy_(k_cache_saved) + v_cache_paged.copy_(v_cache_saved) + # For FA4, use flash_attn_varlen_func directly instead of flash_attn_with_kvcache + # This matches the pattern from the original FA4 test + out, lse = flash_attn_varlen_func( + q if not varlen_q else q_unpad, + k_cache if page_size is None else k_cache_paged, + v_cache if page_size is None else v_cache_paged, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=None, # FA4 doesn't use cu_seqlens_k for KV cache + # max_seqlen_q and max_seqlen_k not needed for FA4 + seqused_k=cache_seqlens, # Use cache_seqlens as seqused_k + page_table=page_table, + causal=causal, + window_size=window_size, + sinks=learnable_sink, # FA4 uses learnable_sink, not sinks + softcap=0.0, + pack_gqa=None, + return_softmax_lse=True, + ver=4, # Use FA4 + ) + if varlen_q: + out = output_pad_fn(out) + # out = flash_attn_with_kvcache( + # q, k_cache, v_cache, cache_seqlens=cache_seqlens, causal=causal, window_size=window_size + # ) + # out = flash_attn_with_kvcache(q, k_cache, v_cache, causal=causal, window_size=window_size) + # qk = torch.einsum("bqhd,bkhd->bhqk", q, k_cache_ref) + # m = qk.amax(-1, keepdim=True) + # s_tmp = torch.exp((qk - m) / math.sqrt(d)) + # o1 = torch.einsum('bhst,bthd->bshd', s_tmp, v_cache_ref) + # lse_ref = torch.logsumexp(qk / math.sqrt(d), -1) + # probs = torch.softmax(qk, dim=-1) + print(f"Output max diff: {(out - out_ref).abs().max().item()}") + print(f"Output mean diff: {(out - out_ref).abs().mean().item()}") + print(f"Pytorch max diff: {(out_pt - out_ref).abs().max().item()}") + print(f"Pytorch mean diff: {(out_pt - out_ref).abs().mean().item()}") + # breakpoint() + + # Check that FlashAttention's numerical error is at most twice the numerical error + # of a Pytorch implementation. + if new_kv: + if page_size is None: + k_cache_select = ( + k_cache.to(dtype_ref) + if not has_batch_idx + else k_cache.to(dtype_ref)[cache_batch_idx] + ) + v_cache_select = ( + v_cache.to(dtype_ref) + if not has_batch_idx + else v_cache.to(dtype_ref)[cache_batch_idx] + ) + else: + k_cache_select = rearrange( + k_cache_paged.to(dtype_ref)[ + ( + page_table + if not has_batch_idx + else page_table[cache_batch_idx] + ).flatten() + ], + "(b nblocks) block_size ... -> b (nblocks block_size) ...", + b=batch_size, + )[:, :seqlen_k].to(dtype_ref) + v_cache_select = rearrange( + v_cache_paged.to(dtype_ref)[ + ( + page_table + if not has_batch_idx + else page_table[cache_batch_idx] + ).flatten() + ], + "(b nblocks) block_size ... -> b (nblocks block_size) ...", + b=batch_size, + )[:, :seqlen_k].to(dtype_ref) + k_cache_ref = k_cache_ref.to(dtype).to(dtype_ref) + v_cache_ref = v_cache_ref.to(dtype).to(dtype_ref) + if dtype is not torch.float8_e4m3fn: + assert torch.equal(v_cache_select, v_cache_ref) + else: + assert torch.allclose( + v_cache_select, v_cache_ref, rtol=1e-3, atol=1e-3 + ) + # breakpoint() + # if rotary_dim == 0 and dtype is not torch.float8_e4m3fn: + if rotary_dim == 0: + assert torch.equal(k_cache_select, k_cache_ref) + else: + # if not torch.allclose(k_cache_select, k_cache_ref, rtol=1e-3, atol=1e-3): + # breakpoint() + if dtype is not torch.float8_e4m3fn: + assert torch.allclose( + k_cache_select, k_cache_ref, rtol=1e-3, atol=1e-3 + ) + else: + assert torch.allclose( + k_cache_select, k_cache_ref, rtol=1e-1, atol=1e-1 + ) + mult = 4 if dtype == torch.float8_e4m3fn else 2 + assert (out - out_ref).abs().max().item() <= mult * ( + out_pt - out_ref + ).abs().max().item() + 1e-5 + mult_mean = 3 if dtype == torch.float8_e4m3fn else 1.5 + assert (out - out_ref).abs().mean().item() <= mult_mean * ( + out_pt - out_ref + ).abs().mean().item() + + +def _generate_block_kvcache( + seqlen_k, page_size, batch_size, nheads_k, d, dv, device, dtype, dtype_ref +): + num_blocks = math.ceil(seqlen_k / page_size) * batch_size * 3 + k_cache_paged = ( + torch.randn(num_blocks, page_size, nheads_k, d, device=device, dtype=dtype_ref) + .to(dtype) + .to(dtype_ref) + ) + v_cache_paged = ( + torch.randn(num_blocks, page_size, nheads_k, dv, device=device, dtype=dtype_ref) + .to(dtype) + .to(dtype_ref) + ) + page_table = rearrange( + torch.randperm(num_blocks, dtype=torch.int32, device=device), + "(b nblocks) -> b nblocks", + b=batch_size, + ) + k_cache = rearrange( + k_cache_paged[page_table.flatten()], + "(b nblocks) block_size ... -> b (nblocks block_size) ...", + b=batch_size, + )[:, :seqlen_k] + v_cache = rearrange( + v_cache_paged[page_table.flatten()], + "(b nblocks) block_size ... -> b (nblocks block_size) ...", + b=batch_size, + )[:, :seqlen_k] + return k_cache, v_cache, page_table, k_cache_paged, v_cache_paged, num_blocks + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_flashmla.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_flashmla.py new file mode 100644 index 00000000..40da3ee4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_flashmla.py @@ -0,0 +1,662 @@ +import math +import random +from typing import Optional, Tuple + +import pytest +import torch +import triton +from sgl_kernel.flash_mla import ( + flash_mla_sparse_fwd, + flash_mla_with_kvcache, + get_mla_metadata, +) + +# ================ prefill usage ================ # +S_Q_PREFILL = [1, 62] +KV_TOPK_PREFILL = [ + # Regular shapes + (128, 128), + (256, 256), + (512, 512), + # Irregular shapes + (592, 128), + (1840, 256), + (1592, 384), + (1521, 512), + # Irregular shapes with OOB TopK + (95, 128), + (153, 256), + (114, 384), +] + +# ================= decode usage ================= # +B_DECODE = [1, 2, 6, 64] +S_Q_DECODE = [1, 2, 4] +S_K_DECODE = [20, 140, 4096] +IS_VARLEN = [False, True] +CAUSAL_TOPK = [(True, None), (False, None), (False, 128), (False, 2048)] +DTYPE = [torch.float16, torch.bfloat16] + + +def is_sm90_supported(device=None) -> bool: + return (torch.cuda.get_device_capability(device)[0] == 9) and ( + torch.version.cuda >= "12.3" + ) + + +def quantize_k_cache( + input_k_cache: torch.Tensor, # (num_blocks, block_size, h_k, d) + dv: int, + tile_size: int = 128, +) -> torch.Tensor: + """ + Quantize the k-cache + Return a tensor with shape (num_blocks, block_size, h_k, dv + 4(dv/tile_size) + t(d-dv)) of dtype uint8_t, where t = input_k_cache.element_size() + For more detail about the layout of K/V, please refer to comments in flash_mla_interface.py or README.md + """ + assert dv % tile_size == 0 + num_tiles = dv // tile_size + num_blocks, block_size, h_k, d = input_k_cache.shape + assert h_k == 1 + input_k_cache = input_k_cache.squeeze(2) # [num_blocks, block_size, d] + input_elem_size = input_k_cache.element_size() + + result = torch.empty( + (num_blocks, block_size, dv + num_tiles * 4 + input_elem_size * (d - dv)), + dtype=torch.float8_e4m3fn, + device=input_k_cache.device, + ) + result_k_nope_part = result[..., :dv] + result_k_scale_factor = result[..., dv : dv + num_tiles * 4].view(torch.float32) + result_k_rope_part = result[..., dv + num_tiles * 4 :].view(input_k_cache.dtype) + result_k_rope_part[:] = input_k_cache[..., dv:] + + for tile_idx in range(0, num_tiles): + cur_scale_factors_inv = ( + torch.abs( + input_k_cache[..., tile_idx * tile_size : (tile_idx + 1) * tile_size] + ) + .max(dim=-1) + .values + / 448.0 + ) # [num_blocks, block_size] + result_k_scale_factor[:, :, tile_idx] = cur_scale_factors_inv + + cur_scale_factors_inv.unsqueeze_(-1) # [num_blocks, block_size, 1] + cur_quantized_nope = ( + input_k_cache[ + ..., tile_idx * tile_size : (tile_idx + 1) * tile_size + ].float() + / cur_scale_factors_inv.float() + ).to(torch.float8_e4m3fn) + result_k_nope_part[..., tile_idx * tile_size : (tile_idx + 1) * tile_size] = ( + cur_quantized_nope + ) + + result = result.view(num_blocks, block_size, 1, -1) + return result + + +def dequantize_k_cache( + quant_k_cache: torch.Tensor, # (num_blocks, block_size, 1, bytes_per_token) + dv: int = 512, + tile_size: int = 128, + d: int = 576, +) -> torch.Tensor: + """ + De-quantize the k-cache + """ + assert dv % tile_size == 0 + num_tiles = dv // tile_size + num_blocks, block_size, h_k, _ = quant_k_cache.shape + assert h_k == 1 + result = torch.empty( + (num_blocks, block_size, d), dtype=torch.bfloat16, device=quant_k_cache.device + ) + + quant_k_cache = quant_k_cache.view(num_blocks, block_size, -1) + + input_nope = quant_k_cache[..., :dv] + input_scale = quant_k_cache[..., dv : dv + num_tiles * 4].view(torch.float32) + input_rope = quant_k_cache[..., dv + num_tiles * 4 :].view(torch.bfloat16) + result[..., dv:] = input_rope + + for tile_idx in range(0, num_tiles): + cur_nope = input_nope[ + ..., tile_idx * tile_size : (tile_idx + 1) * tile_size + ].to(torch.float32) + cur_scales = input_scale[..., tile_idx].unsqueeze(-1) + result[..., tile_idx * tile_size : (tile_idx + 1) * tile_size] = ( + cur_nope * cur_scales + ) + + result = result.view(num_blocks, block_size, 1, d) + return result + + +def cdiv(x: int, y: int): + return (x + y - 1) // y + + +def get_window_size(causal, window): + if window > 0: + window_size = (window - 1, 0) if causal else (window - 1, window - 1) + else: + window_size = (-1, -1) + return window_size + + +def get_attn_bias(s_q, s_k, causal, window): + attn_bias = torch.zeros(s_q, s_k, dtype=torch.float32, device="cuda") + if causal: + temp_mask = torch.ones(s_q, s_k, dtype=torch.bool, device="cuda").tril( + diagonal=s_k - s_q + ) + attn_bias.masked_fill_(temp_mask.logical_not(), float("-inf")) + if window > 0: + temp_mask = torch.ones(s_q, s_k, dtype=torch.bool, device="cuda").tril( + diagonal=s_k - s_q - window + ) + attn_bias.masked_fill_(temp_mask, float("-inf")) + temp_mask = torch.ones(s_q, s_k, dtype=torch.bool, device="cuda").tril( + diagonal=s_k - s_q + window - 1 + ) + attn_bias.masked_fill_(temp_mask.logical_not(), float("-inf")) + return attn_bias + + +def sdpa(query, key, value, attn_bias, softmax_scale=None): + query = query.float().transpose(-3, -2) + key = key.float().transpose(-3, -2) + value = value.float().transpose(-3, -2) + key = key.repeat_interleave(h // h_k, dim=-3) + value = value.repeat_interleave(h // h_k, dim=-3) + if softmax_scale is None: + softmax_scale = query.shape[-1] ** (-0.5) + attn_weight = (query @ key.transpose(-2, -1)) * softmax_scale + attn_weight += attn_bias + lse = attn_weight.logsumexp(dim=-1) + attn_weight = torch.softmax(attn_weight, dim=-1, dtype=torch.float32) + return attn_weight.to(query.dtype) @ value, lse + + +def sdpa_checkpoint(*args, **kwargs): + return checkpoint(sdpa, *args, use_reentrant=False, **kwargs) + + +def reference_torch_prefill( + s_q, s_kv, topk, indices, q, kv, sm_scale: float +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + def log2sumexp2(a: torch.Tensor, dim: int) -> torch.Tensor: + return torch.logsumexp(a * math.log(2), dim=dim) * math.log2(math.e) + + indices = indices[0, :, 0, :] # [s_q, topk] + invalid_indices_mask = (indices < 0) | (indices >= s_kv) + qs = q[0, :, :, :].float() # [s_q, h_q, d_qk] + kvs = kv[0, :, 0, :].float() # [s_kv, d_qk] + + kvs = torch.index_select( + kvs, 0, indices.masked_fill(invalid_indices_mask, 0).flatten() + ).view( + s_q, topk, 576 + ) # [s_q, topk, d_qk] + attn_score = qs @ kvs.transpose(1, 2) # [s_q, h_q, topk] + attn_score.masked_fill_(invalid_indices_mask.unsqueeze(1), float("-inf")) + attn_score *= sm_scale * math.log2(math.e) + max_logits = torch.max(attn_score, dim=-1)[0] # [s_q, h_q] + lse = log2sumexp2(attn_score, dim=-1) # [s_q, h_q] + attn_score = torch.exp2(attn_score - lse.unsqueeze(-1)) # [s_q, h_q, topk] + result = attn_score @ kvs[:, :, :512] + return (max_logits, lse, result) + + +def reference_torch_decode( + cache_seqlens: torch.Tensor, # [batch_size] + block_table: torch.Tensor, # [batch_size, ?] + q: torch.Tensor, # [batch_size, s_q, h_q, d] + blocked_k: torch.Tensor, # [?, block_size, h_kv, d] + dv: int, + is_causal: bool, + indices: Optional[torch.Tensor] = None, # [batch_size, s_q, topk] +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + A reference implementation in PyTorch + """ + + def get_topk_attn_mask(s_q: int, s_k: int, indices: torch.Tensor): + mask = torch.zeros(s_q, s_k, dtype=torch.bool, device="cuda") + for i in range(s_q): + cur_indices = indices[i] + valid_indices = cur_indices[cur_indices != -1] + mask[i, valid_indices] = True + return mask + + def scaled_dot_product_attention( + batch_idx: int, + query: torch.Tensor, # [h_q, s_q, d] + kv: torch.Tensor, # [h_kv, s_k, d] + dv: int, + is_causal, + indices: Optional[torch.Tensor], # [s_q, topk] + ) -> Tuple[torch.Tensor, torch.Tensor]: + h_q = query.size(0) + h_kv = kv.size(0) + s_q = query.shape[-2] + s_k = kv.shape[-2] + query = query.float() + kv = kv.float() + if h_kv != 1: + kv = kv.repeat_interleave(h_q // h_kv, dim=0) + kv[kv != kv] = 0.0 + attn_weight = query @ kv.transpose(-2, -1) # [h_q, s_q, s_k] + if (is_causal and query.size(1) > 1) or indices is not None: + mask = torch.ones(s_q, s_k, dtype=torch.bool, device="cuda") + if is_causal: + assert indices is None + mask = mask.tril(diagonal=s_k - s_q) + if indices is not None: + mask &= get_topk_attn_mask(s_q, s_k, indices) + attn_bias = torch.zeros(s_q, s_k, dtype=torch.float, device="cuda") + attn_bias.masked_fill_(mask.logical_not(), float("-inf")) + attn_weight += attn_bias.to(q.dtype) + attn_weight /= math.sqrt(query.size(-1)) + lse = attn_weight.logsumexp(dim=-1) # [h_q, s_q] + attn_weight = torch.softmax(attn_weight, dim=-1, dtype=torch.float32) + output = attn_weight @ kv[..., :dv] # [h_q, s_q, dv] + # Correct for q tokens which has no attendable k + lonely_q_mask = lse == float("-inf") + output[lonely_q_mask.unsqueeze(-1).broadcast_to(h_q, s_q, dv)] = 0.0 + lse[lonely_q_mask] = float("+inf") + + return output, lse + + b, s_q, h_q, d = q.size() + block_size = blocked_k.size(1) + h_kv = blocked_k.size(2) + cache_seqlens_cpu = cache_seqlens.cpu() + out_ref = torch.empty(b, s_q, h_q, dv, dtype=torch.float32, device="cuda") + lse_ref = torch.empty(b, h_q, s_q, dtype=torch.float32, device="cuda") + for i in range(b): + cur_len = cache_seqlens_cpu[i].item() + cur_num_blocks = cdiv(cur_len, block_size) + cur_block_indices = block_table[i][0:cur_num_blocks] + cur_kv = blocked_k[cur_block_indices].view(-1, h_kv, d)[:cur_len, ...] + cur_out, cur_lse = scaled_dot_product_attention( + i, + q[i].transpose(0, 1), + cur_kv.transpose(0, 1), + dv, + is_causal, + indices[i] if indices is not None else None, + ) + out_ref[i] = cur_out.transpose(0, 1) + lse_ref[i] = cur_lse + out_ref = out_ref.to(torch.bfloat16) + return out_ref, lse_ref + + +@pytest.mark.parametrize("s_q", S_Q_PREFILL) +@pytest.mark.parametrize("kv_topk", KV_TOPK_PREFILL) +@torch.inference_mode() +def test_flashmla_prefill( + s_q: int, + kv_topk: Tuple[int, int], +): + + torch.cuda.empty_cache() + + q = torch.randn((1, s_q, 128, 576), dtype=torch.bfloat16, device="cuda") / 10 + kv = torch.randn((1, kv_topk[0], 1, 576), dtype=torch.bfloat16, device="cuda") / 10 + + q.clamp_(-10, 10) + kv.clamp_(-10, 10) + + indices = torch.full( + (1, s_q, 1, kv_topk[1]), kv_topk[0], dtype=torch.int32, device="cuda" + ) + for s in range(s_q): + # NOTE We use the following method to generate indices so that most indices lies within [s_kv-20000, s_kv), which is more realistic for sparse attention + near_mask = ( + torch.randint(0, 32, (min(kv_topk[1], kv_topk[0]),), device="cuda") < 31 + ) + cur_indices = torch.randperm(kv_topk[0], device="cuda")[: kv_topk[1]] + cur_indices[near_mask] = torch.randint( + max(0, kv_topk[0] - 20000), + kv_topk[0] - 1, + (near_mask.sum().item(),), + device="cuda", + ) + if len(cur_indices) < kv_topk[1]: + cur_indices = torch.cat( + [ + cur_indices, + torch.full( + (kv_topk[1] - len(cur_indices),), 2147480000, device="cuda" + ), + ] + ) + cur_indices = cur_indices[torch.randperm(kv_topk[1], device="cuda")] + indices[0, s, 0] = cur_indices + indices = indices.to(q.device) + + sm_scale = 1 / math.sqrt(576) + torch.cuda.synchronize() + + ans_out, ans_max_logits, ans_lse = flash_mla_sparse_fwd( + q.squeeze(0), kv.squeeze(0), indices.squeeze(0), sm_scale=sm_scale + ) + + ans_out, ans_max_logits, ans_lse = ( + ans_out.float(), + ans_max_logits.float(), + ans_lse.float(), + ) + + torch.cuda.synchronize() + ref_max_logits, ref_lse, ref_out = reference_torch_prefill( + s_q, kv_topk[0], kv_topk[1], indices, q, kv, sm_scale + ) + torch.cuda.synchronize() + + torch.testing.assert_close(ans_out, ref_out, atol=8e-4, rtol=2.01 / 128) + torch.testing.assert_close( + ans_max_logits, + ref_max_logits, + atol=1e-6, + rtol=2.01 / 65536, + ) + torch.testing.assert_close(ans_lse, ref_lse, atol=1e-6, rtol=2.01 / 65536) + + +@pytest.mark.skipif(not is_sm90_supported(), reason="SM90 required for FP8 support") +@pytest.mark.parametrize("b", B_DECODE) +@pytest.mark.parametrize("s_q", S_Q_DECODE) +@pytest.mark.parametrize("s_k", S_K_DECODE) +@pytest.mark.parametrize("is_varlen", IS_VARLEN) +@pytest.mark.parametrize("causal_topk", CAUSAL_TOPK) +@pytest.mark.parametrize("dtype", DTYPE) +@torch.inference_mode() +def test_flash_mla_decode( + b: int, + s_q: int, + s_k: int, + is_varlen: bool, + causal_topk: Tuple[bool, Optional[int]], + dtype: torch.dtype, +): + d = 576 + dv = 512 + block_size = 64 + h_q = 128 + h_kv = 1 + is_causal = causal_topk[0] + topk = causal_topk[1] + + # Generating test data + torch.cuda.synchronize() + + cache_seqlens_cpu = torch.full((b,), s_k, dtype=torch.int32, device="cpu") + if is_varlen: + for i in range(b): + cache_seqlens_cpu[i] = max(random.normalvariate(s_k, s_k / 2), s_q) + + max_seqlen = cache_seqlens_cpu.max().item() + max_seqlen_pad = cdiv(max_seqlen, 256) * 256 + cache_seqlens = cache_seqlens_cpu.cuda() + + q = torch.randn(b, s_q, 128, d, dtype=torch.bfloat16, device="cuda") + q.clamp_(min=-1.0, max=1.0) + + block_table = torch.arange( + b * max_seqlen_pad // block_size, dtype=torch.int32, device="cuda" + ).view(b, max_seqlen_pad // block_size) + block_table = block_table.view(-1)[torch.randperm(block_table.numel())].view(b, -1) + blocked_k = ( + torch.randn( + block_table.numel(), + block_size, + h_kv, + d, + dtype=torch.bfloat16, + device="cuda", + ) + / 10 + ) + blocked_k.clamp_(min=-1.0, max=1.0) + + if topk is None: + for i in range(b): + cur_len = cache_seqlens_cpu[i].item() + cur_num_blocks = cdiv(cur_len, block_size) + blocked_k[block_table[i][cur_num_blocks:]] = float("nan") + if cur_len % block_size != 0: + blocked_k[block_table[i][cur_num_blocks - 1]][ + cur_len % block_size : + ] = float("nan") + block_table[i][cur_num_blocks:] = 2147480000 + abs_indices = None + indices_in_kvcache = None + else: + block_table_cpu = block_table.cpu() + abs_indices = torch.empty(b, s_q, topk, dtype=torch.int32, device="cpu") + indices_in_kvcache = torch.empty(b, s_q, topk, dtype=torch.int32, device="cpu") + for i in range(b): + # Generate indices + for j in range(s_q): + cur_abs_indices = torch.randperm( + int(cache_seqlens_cpu[i].item()), device="cpu" + )[:topk] + cur_blocked_indices = block_table_cpu[ + i, cur_abs_indices // block_size + ] * block_size + (cur_abs_indices % block_size) + if len(cur_abs_indices) < topk: + pad_len = topk - len(cur_abs_indices) + cur_abs_indices = torch.cat( + [cur_abs_indices, torch.full((pad_len,), -1, device="cpu")] + ) + cur_blocked_indices = torch.cat( + [cur_blocked_indices, torch.full((pad_len,), -1, device="cpu")] + ) + + # Mask KV + perm = torch.randperm(topk, device="cpu") + cur_abs_indices = cur_abs_indices[perm] + cur_blocked_indices = cur_blocked_indices[perm] + + abs_indices[i, j, :] = cur_abs_indices + indices_in_kvcache[i, j, :] = cur_blocked_indices + + # Mask nonused KV as NaN + all_indices = indices_in_kvcache.flatten().tolist() + all_indices = list(set(all_indices)) + if -1 in all_indices: + all_indices.remove(-1) + all_indices = torch.tensor(all_indices, dtype=torch.int32, device="cpu") + + blocked_k = blocked_k.view(-1, h_kv, d) + nonused_indices_mask = torch.ones( + blocked_k.size(0) * blocked_k.size(1), dtype=torch.bool, device="cpu" + ) + nonused_indices_mask[all_indices] = False + blocked_k[nonused_indices_mask, :, :] = float("nan") + blocked_k = blocked_k.view(-1, block_size, h_kv, d) + + abs_indices = abs_indices.to(q.device) + indices_in_kvcache = indices_in_kvcache.to(q.device) + + is_fp8 = topk is not None + if is_fp8: + # The quantization error may be too large to be distinguished from wrong kernels + # So we quantize and de-quantize kv-cache here to mitigate quantization error + blocked_k_quantized = quantize_k_cache(blocked_k, dv, 128) + blocked_k_dequantized = dequantize_k_cache(blocked_k_quantized) + blocked_k = blocked_k_dequantized + + # Get schedule metadata + torch.cuda.synchronize() + tile_scheduler_metadata, num_splits = get_mla_metadata( + cache_seqlens, s_q * h_q // h_kv, h_kv, h_q, is_fp8, topk + ) + torch.cuda.synchronize() + + out_ans, lse_ans = flash_mla_with_kvcache( + q, + blocked_k if not is_fp8 else blocked_k_quantized, # type: ignore + block_table, + cache_seqlens, + dv, + tile_scheduler_metadata, + num_splits, + causal=is_causal, + is_fp8_kvcache=is_fp8, + indices=indices_in_kvcache, + ) + + out_ref, lse_ref = reference_torch_decode( + cache_seqlens, block_table, q, blocked_k, dv, is_causal, abs_indices + ) + torch.testing.assert_close(out_ans, out_ref, atol=8e-4, rtol=2.01 / 128) + torch.testing.assert_close(lse_ans, lse_ref, atol=1e-6, rtol=8.01 / 65536) + + +@pytest.mark.skipif(not is_sm90_supported(), reason="SM90 required for FP8 support") +@pytest.mark.parametrize("b", [128]) +@pytest.mark.parametrize("s_q", [1, 2]) +@pytest.mark.parametrize("mean_sk", [4096, 8192, 16384]) +@pytest.mark.parametrize("h_q", [16, 32, 64, 128]) +@pytest.mark.parametrize("h_kv", [1]) +@pytest.mark.parametrize("d", [576]) +@pytest.mark.parametrize("dv", [512]) +@pytest.mark.parametrize("block_size", [64]) +@pytest.mark.parametrize("causal", [True]) +@pytest.mark.parametrize("varlen", [False, True]) +@pytest.mark.parametrize("torch_dtype", [torch.float8_e4m3fn]) +@torch.inference_mode() +def test_flash_mla_fp8( + b, s_q, mean_sk, h_q, h_kv, d, dv, block_size, causal, varlen, torch_dtype +): + device = torch.device("cuda:0") + init_dtype = torch.bfloat16 if torch_dtype == torch.float8_e4m3fn else torch_dtype + torch.set_default_dtype(init_dtype) + torch.set_default_device(device) + torch.cuda.set_device(device) + torch.manual_seed(0) + random.seed(0) + + use_fp8 = torch_dtype == torch.float8_e4m3fn + cache_seqlens = torch.full((b,), mean_sk, dtype=torch.int32) + if varlen: + for i in range(b): + cache_seqlens[i] = max(random.normalvariate(mean_sk, mean_sk / 2), s_q) + total_seqlens = cache_seqlens.sum().item() + max_seqlen = cache_seqlens.max().item() + max_seqlen_pad = triton.cdiv(max_seqlen, 256) * 256 + + q = torch.randn(b, s_q, h_q, d) + block_table = torch.arange( + b * max_seqlen_pad // block_size, dtype=torch.int32 + ).view(b, max_seqlen_pad // block_size) + blocked_k = torch.randn(block_table.numel(), block_size, h_kv, d) + for i in range(b): + blocked_k.view(b, max_seqlen_pad, h_kv, d)[i, cache_seqlens[i].item() :] = ( + float("nan") + ) + blocked_v = blocked_k[..., :dv] + + tile_scheduler_metadata, num_splits = get_mla_metadata( + cache_seqlens, s_q * h_q // h_kv, h_kv, is_fp8_kvcache=use_fp8 + ) + + init_dtype = q.dtype + if use_fp8: + fp8_dtype = torch.float8_e4m3fn + descale_q = torch.ones((1), dtype=torch.float32) + descale_k = torch.ones((1), dtype=torch.float32) + + q = q.to(fp8_dtype) + blocked_k = blocked_k.to(fp8_dtype) + blocked_v = blocked_v.to(fp8_dtype) + else: + descale_q = None + descale_k = None + + def flash_mla(): + return flash_mla_with_kvcache( + q, + blocked_k, + block_table, + cache_seqlens, + dv, + tile_scheduler_metadata, + num_splits, + causal=causal, + descale_q=descale_q, + descale_k=descale_k, + ) + + def scaled_dot_product_attention(query, key, value, is_causal=False): + query = query.float() + key = key.float() + value = value.float() + key = key.repeat_interleave(h_q // h_kv, dim=0) + value = value.repeat_interleave(h_q // h_kv, dim=0) + attn_weight = query @ key.transpose(-2, -1) / math.sqrt(query.size(-1)) + if is_causal: + s_q = query.shape[-2] + s_k = key.shape[-2] + attn_bias = torch.zeros(s_q, s_k, dtype=query.dtype) + temp_mask = torch.ones(s_q, s_k, dtype=torch.bool).tril(diagonal=s_k - s_q) + attn_bias.masked_fill_(temp_mask.logical_not(), float("-inf")) + attn_bias.to(query.dtype) + attn_weight += attn_bias + lse = attn_weight.logsumexp(dim=-1) + attn_weight = torch.softmax(attn_weight, dim=-1, dtype=torch.float32) + return attn_weight @ value, lse + + def ref_mla(): + q_ = (q.to(torch.float) * descale_q).to(init_dtype) if use_fp8 else q + blocked_k_ = ( + (blocked_k.to(torch.float) * descale_k).to(init_dtype) + if use_fp8 + else blocked_k + ) + blocked_v_ = ( + (blocked_v.to(torch.float) * descale_k).to(init_dtype) + if use_fp8 + else blocked_v + ) + out = torch.empty(b, s_q, h_q, dv, dtype=torch.float32) + lse = torch.empty(b, h_q, s_q, dtype=torch.float32) + for i in range(b): + begin = i * max_seqlen_pad + end = begin + cache_seqlens[i] + out_i, lse_i = scaled_dot_product_attention( + q_[i].transpose(0, 1), + blocked_k_.view(-1, h_kv, d)[begin:end].transpose(0, 1), + blocked_v_.view(-1, h_kv, dv)[begin:end].transpose(0, 1), + is_causal=causal, + ) + out[i] = out_i.transpose(0, 1) + lse[i] = lse_i + return out, lse + + def cal_diff( + x: torch.Tensor, y: torch.Tensor, name: str, use_fp8: bool = False + ) -> None: + x, y = x.double(), y.double() + cos_diff = 1 - 2 * (x * y).sum().item() / max( + (x * x + y * y).sum().item(), 1e-12 + ) + if use_fp8: + assert cos_diff < 1e-4 + else: + assert cos_diff < 1e-5 + + out_flash, lse_flash = flash_mla() + out_torch, lse_torch = ref_mla() + cal_diff(out_flash, out_torch, "out", use_fp8) + cal_diff(lse_flash, lse_torch, "lse") + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fp4_gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fp4_gemm.py new file mode 100644 index 00000000..47401618 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fp4_gemm.py @@ -0,0 +1,154 @@ +import pytest +import torch +from sgl_kernel import cutlass_scaled_fp4_mm, scaled_fp4_quant + +skip_condition = torch.cuda.get_device_capability() < (10, 0) + +DTYPES = [torch.float16, torch.bfloat16] +# m, n, k +SHAPES = [(128, 128, 64), (128, 128, 128), (256, 128, 64), (128, 256, 128)] +PAD_SHAPES = [(150, 128, 64), (128, 128, 96)] +SHAPES.extend(PAD_SHAPES) + +FLOAT4_E2M1_MAX = 6.0 +FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max + +kE2M1ToFloatArray = [ + 0.0, + 0.5, + 1.0, + 1.5, + 2.0, + 3.0, + 4.0, + 6.0, +] + + +def e2m1_to_fp32(int4_value): + signBit = int4_value & 0x8 + int4_absValue = int4_value & 0x7 + float_result = kE2M1ToFloatArray[int4_absValue] + if signBit: + float_result = -float_result + return float_result + + +def break_fp4_bytes(a, dtype): + assert a.dtype == torch.uint8 + m, n = a.shape + a = a.flatten() + # Get upper 4 bits + highHalfByte = (a & 0xF0) >> 4 + # Get lower 4 bits + lowHalfByte = a & 0x0F + fH = torch.tensor([e2m1_to_fp32(x) for x in highHalfByte]).to(a.device) + fL = torch.tensor([e2m1_to_fp32(x) for x in lowHalfByte]).to(a.device) + # [0xAB, 0xCD] -> [0xB, 0xA, 0xD, 0xC] + out = torch.stack((fL, fH), dim=-1).reshape(m, n * 2) + return out + + +def convert_swizzled_to_linear(a_sf_swizzled: torch.Tensor, m, k, block_size): + sf_m, sf_k = a_sf_swizzled.shape + m_tiles = (m + 128 - 1) // 128 + f = block_size * 4 + k_tiles = (k + f - 1) // f + tmp = torch.reshape(a_sf_swizzled, (1, m_tiles, k_tiles, 32, 4, 4)) + tmp = torch.permute(tmp, (0, 1, 4, 3, 2, 5)) + out = tmp.reshape(m_tiles * 128, k_tiles * f // block_size) + return out[0:m, 0:k] + + +def dequantize_to_dtype( + tensor_fp4, tensor_sf, global_scale, dtype, device, block_size=16 +): + """Dequantize the fp4 tensor back to high precision.""" + # Two fp4 values are packed into one uint8. + assert tensor_fp4.dtype == torch.uint8 + m, packed_k = tensor_fp4.shape + k = packed_k * 2 + tensor_f32 = break_fp4_bytes(tensor_fp4, dtype) + tensor_f32 = tensor_f32.reshape(m, k // block_size, block_size) + tensor_sf = tensor_sf.view(torch.float8_e4m3fn) + tensor_sf = convert_swizzled_to_linear(tensor_sf, m, k, block_size) + tensor_sf_dtype = tensor_sf.to(torch.float32) / global_scale + + # scale the tensor + out = (tensor_f32 * tensor_sf_dtype.unsqueeze(-1)).reshape(m, k) + return out + + +def get_ref_results( + a_fp4, + b_fp4, + a_sf, + b_sf, + a_global_scale, + b_global_scale, + m, + n, + dtype, + block_size, + device, +): + _, m_k = a_fp4.shape + _, n_k = b_fp4.shape + assert m_k == n_k + a_in_dtype = dequantize_to_dtype( + a_fp4, a_sf, a_global_scale, dtype=dtype, device=device, block_size=block_size + ) + b_in_dtype = dequantize_to_dtype( + b_fp4, b_sf, b_global_scale, dtype=dtype, device=device, block_size=block_size + ) + return torch.matmul(a_in_dtype, b_in_dtype.t()) + + +@pytest.mark.skipif( + skip_condition, reason="Nvfp4 Requires compute capability of 10 or above." +) +@pytest.mark.parametrize("dtype", DTYPES) +@pytest.mark.parametrize("shape", SHAPES) +@torch.inference_mode() +def test_nvfp4_gemm( + dtype: torch.dtype, + shape: tuple[int, int], +) -> None: + m, n, packed_k = shape + k = packed_k * 2 + block_size = 16 + a_dtype = torch.randn((m, k), dtype=dtype, device="cuda") + b_dtype = torch.randn((n, k), dtype=dtype, device="cuda") + + a_global_scale = ( + (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(a_dtype.flatten(), dim=-1) + ).to(torch.float32) + b_global_scale = ( + (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(b_dtype.flatten(), dim=-1) + ).to(torch.float32) + alpha = 1.0 / (a_global_scale * b_global_scale) + a_fp4, a_scale_interleaved = scaled_fp4_quant(a_dtype, a_global_scale) + b_fp4, b_scale_interleaved = scaled_fp4_quant(b_dtype, b_global_scale) + + expected_out = get_ref_results( + a_fp4, + b_fp4, + a_scale_interleaved, + b_scale_interleaved, + a_global_scale, + b_global_scale, + m, + n, + dtype, + block_size, + "cuda", + ) + out = cutlass_scaled_fp4_mm( + a_fp4, b_fp4, a_scale_interleaved, b_scale_interleaved, alpha, dtype + ) + + torch.testing.assert_close(out, expected_out.to(dtype=dtype), atol=1e-1, rtol=1e-1) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fp4_quantize.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fp4_quantize.py new file mode 100644 index 00000000..e29bac21 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fp4_quantize.py @@ -0,0 +1,260 @@ +import pytest +import torch +from flashinfer import ( + scaled_fp4_grouped_quantize, + silu_and_mul_scaled_nvfp4_experts_quantize, +) +from sgl_kernel import scaled_fp4_quant, silu_and_mul + +skip_condition = torch.cuda.get_device_capability() < (10, 0) + +DTYPES = [torch.float16, torch.bfloat16] +SHAPES = [(128, 64), (128, 128), (256, 64), (256, 128)] +PAD_SHAPES = [ + (90, 64), + (150, 64), + (128, 48), + (128, 80), + (150, 80), + (90, 48), + (90, 128), + (150, 128), + (150, 48), + (90, 80), +] + +FLOAT4_E2M1_MAX = 6.0 +FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max + +# E2M1 to float +# 0111 -> 6 +# 0110 -> 4 +# 0101 -> 3 +# 0100 -> 2 +# 0011 -> 1.5 +# 0010 -> 1 +# 0001 -> 0.5 +# 0000 -> 0 +E2M1_TO_FLOAT32 = [ + 0.0, + 0.5, + 1.0, + 1.5, + 2.0, + 3.0, + 4.0, + 6.0, + 0.0, + -0.5, + -1.0, + -1.5, + -2.0, + -3.0, + -4.0, + -6.0, +] +BLOCK_SIZE = 16 + + +def cast_from_fp4(x, m, n): + # The fp4 values are packed in uint8 as [v_1st | v_2nd] + v_2nd = x & 0xF + v_1st = (x >> 4) & 0xF + c = torch.stack((v_2nd, v_1st), dim=-1) + out = torch.tensor([E2M1_TO_FLOAT32[x] for x in c.flatten()]) + out = out.reshape(m, n).to(torch.float32) + return out + + +def cast_to_fp4(x): + sign = torch.sign(x) + x = torch.abs(x) + x[(x >= 0.0) & (x <= 0.25)] = 0.0 + x[(x > 0.25) & (x < 0.75)] = 0.5 + x[(x >= 0.75) & (x <= 1.25)] = 1.0 + x[(x > 1.25) & (x < 1.75)] = 1.5 + x[(x >= 1.75) & (x <= 2.5)] = 2.0 + x[(x > 2.5) & (x < 3.5)] = 3.0 + x[(x >= 3.5) & (x <= 5.0)] = 4.0 + x[x > 5.0] = 6.0 + return x * sign + + +def get_reciprocal(x): + if isinstance(x, torch.Tensor): + return torch.where(x == 0, torch.tensor(0.0, dtype=x.dtype), 1.0 / x) + elif isinstance(x, (float, int)): + return 0.0 if x == 0 else 1.0 / x + else: + raise TypeError("Input must be a float, int, or a torch.Tensor.") + + +def ref_nvfp4_quant(x, global_scale): + assert global_scale.dtype == torch.float32 + assert x.ndim == 2 + m, n = x.shape + x = torch.reshape(x, (m, n // BLOCK_SIZE, BLOCK_SIZE)) + vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32) + scale = global_scale * (vec_max * get_reciprocal(FLOAT4_E2M1_MAX)) + scale = scale.to(torch.float8_e4m3fn).to(torch.float32) + output_scale = get_reciprocal(scale * get_reciprocal(global_scale)) + + scaled_x = x.to(torch.float32) * output_scale + clipped_x = torch.clamp(scaled_x, -6.0, 6.0).reshape(m, n) + return cast_to_fp4(clipped_x), scale.squeeze(-1) + + +def recover_swizzled_scales(scale, m, n): + rounded_m = ((m + 128 - 1) // 128) * 128 + scale_n = n // BLOCK_SIZE + rounded_n = ((scale_n + 4 - 1) // 4) * 4 + # Recover the swizzled scaling factor to linear layout + tmp = torch.reshape(scale, (1, rounded_m // 128, rounded_n // 4, 32, 4, 4)) + tmp = torch.permute(tmp, (0, 1, 4, 3, 2, 5)) + result = torch.reshape(tmp, (rounded_m, rounded_n)).to(torch.float32) + return result[:m, :scale_n] + + +@pytest.mark.skipif( + skip_condition, reason="Nvfp4 Requires compute capability of 10 or above." +) +@pytest.mark.parametrize("dtype", DTYPES) +@pytest.mark.parametrize("shape", SHAPES) +@torch.inference_mode() +def test_quantize_to_fp4( + dtype: torch.dtype, + shape: tuple[int, int], +) -> None: + torch.manual_seed(42) + torch.set_default_device("cuda:0") + + m, n = shape + + x = torch.randn((m, n), dtype=dtype) + tensor_amax = torch.abs(x).max().to(torch.float32) + global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax + out_ref, scale_ref = ref_nvfp4_quant(x, global_scale) + + out, out_scale = scaled_fp4_quant(x, global_scale) + scale_ans = recover_swizzled_scales(out_scale, m, n) + out_ans = cast_from_fp4(out, m, n) + + torch.testing.assert_close(out_ans, out_ref) + torch.testing.assert_close(scale_ans, scale_ref) + + +@pytest.mark.skipif( + skip_condition, reason="Nvfp4 Requires compute capability of 10 or above." +) +@pytest.mark.parametrize("pad_shape", PAD_SHAPES) +@torch.inference_mode() +def test_quantize_to_fp4_padded(pad_shape: tuple[int, int]) -> None: + torch.manual_seed(42) + dtype = torch.float16 + torch.set_default_device("cuda:0") + + m, n = pad_shape + + x = torch.randn((m, n), dtype=dtype) + + tensor_amax = torch.abs(x).max().to(torch.float32) + global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax + out_ref, scale_ref = ref_nvfp4_quant(x, global_scale) + + out, out_scale = scaled_fp4_quant(x, global_scale) + + scale_ans = recover_swizzled_scales(out_scale, m, n) + out_ans = cast_from_fp4(out, m, n) + + torch.testing.assert_close(out_ans, out_ref) + torch.testing.assert_close(scale_ans, scale_ref) + + +@pytest.mark.skipif( + skip_condition, reason="Nvfp4 Requires compute capability of 10 or above." +) +@pytest.mark.parametrize("shape", [(2, 512, 2048), (2, 100, 128), (2, 128, 96)]) +def test_quantize_to_fp4_grouped(shape): + torch.manual_seed(42) + torch.set_default_device("cuda:0") + + l, m, k = shape + x = torch.randn((l, m, k), dtype=torch.bfloat16) + max_m = m // 2 + assert max_m <= m + mask = torch.randint(1, max_m, (l,), dtype=torch.int32) + tensor_amax = x.abs().amax(dim=(1, 2)).to(torch.float32) + x_sf_global = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax + output, output_scales = scaled_fp4_grouped_quantize( + x, + mask, + x_sf_global, + ) + # output in logical (m, k, l), but its physical layout is (l, m, k). + # So permute first to (l, m, k). + output = output.permute(2, 0, 1) + # output_scale in logical (32, 4, rm, 4, rk, l), but its physical layout is (l, rm, rk, 32, 4, 4). + # So permute first to (l, rm, rk, 32, 4, 4). + padded_m = ((m + 128 - 1) // 128) * 128 + output_scales = output_scales.permute(5, 2, 4, 0, 1, 3).view(l, padded_m, -1) + for i in range(l): + a_fp4, a_scale_interleaved = scaled_fp4_quant(x[i], x_sf_global[i]) + torch.testing.assert_close(a_fp4[: mask[i]], output[i][: mask[i]]) + # Recover swizzled scales to linear layout and drop padded values, so + # no extra checks on padding are needed. + scale_ref = recover_swizzled_scales(a_scale_interleaved, m, k) + scale_ans = recover_swizzled_scales(output_scales[i], m, k) + torch.testing.assert_close(scale_ref[: mask[i]], scale_ans[: mask[i]]) + + +@pytest.mark.skipif( + skip_condition, reason="Nvfp4 Requires compute capability of 10 or above." +) +@pytest.mark.parametrize("shape", [(32, 100, 2048), (32, 512, 2048), (6, 6144, 2048)]) +def test_silu_and_mul_quantize_to_fp4_grouped(shape): + torch.manual_seed(42) + torch.set_default_device("cuda:0") + + l, m, k = shape + x = torch.randn((l, m, k * 2), dtype=torch.bfloat16) + max_m = m // 2 + assert max_m <= m + mask = torch.randint(1, max_m, (l,), dtype=torch.int32) + + ref_y = silu_and_mul(x) + tensor_amax = ref_y.abs().amax(dim=(1, 2)).to(torch.float32) + y_sf_global = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax + ref_output, ref_output_scales = scaled_fp4_grouped_quantize( + ref_y, + mask, + y_sf_global, + ) + output, output_scales = silu_and_mul_scaled_nvfp4_experts_quantize( + x, + mask, + y_sf_global, + ) + + # output in logical (m, k, l), but its physical layout is (l, m, k). + # So permute first to (l, m, k). + output = output.permute(2, 0, 1) + ref_output = ref_output.permute(2, 0, 1) + + # output_scale in logical (32, 4, rm, 4, rk, l), but its physical layout is (l, rm, rk, 32, 4, 4). + # So permute first to (l, rm, rk, 32, 4, 4). + padded_m = ((m + 128 - 1) // 128) * 128 + output_scales = output_scales.permute(5, 2, 4, 0, 1, 3).view(l, padded_m, -1) + ref_output_scales = ref_output_scales.permute(5, 2, 4, 0, 1, 3).view( + l, padded_m, -1 + ) + + for i in range(l): + torch.testing.assert_close(ref_output[i, : mask[i]], output[i, : mask[i]]) + # We need to recover the swizzled scales to linear layout before applying mask slice. + scale_ref = recover_swizzled_scales(ref_output_scales[i], m, k) + scale_ans = recover_swizzled_scales(output_scales[i], m, k) + torch.testing.assert_close(scale_ref[: mask[i]], scale_ans[: mask[i]]) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fp8_blockwise_gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fp8_blockwise_gemm.py new file mode 100644 index 00000000..4c1dde33 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fp8_blockwise_gemm.py @@ -0,0 +1,93 @@ +import os +import random +from typing import Optional, Type + +import pytest +import torch +from sgl_kernel import fp8_blockwise_scaled_mm + + +def cdiv(a: int, b: int) -> int: + return -(a // -b) + + +def scale_shape(shape, group_shape): + assert len(shape) == len(group_shape) + return tuple(cdiv(shape[i], group_shape[i]) for i in range(len(group_shape))) + + +def baseline_scaled_mm( + a: torch.Tensor, + b: torch.Tensor, + scale_a: torch.Tensor, + scale_b: torch.Tensor, + out_dtype: Type[torch.dtype], + bias: Optional[torch.Tensor] = None, +) -> torch.Tensor: + # We treat N-dimensional group scaling as extended numpy-style broadcasting + # in numpy simply stretches dimensions with an extent of 1 to match the + # the target shape by repeating the data along that dimension (broadcasting) + # , we extend these semantics to say if the extent of a dimension in the + # source shape is not 1 and does not match the target shape we repeat each + # element along that dimension src_shape[dim] // target_shape[dim] times + # example if we have: + # a = [[1, 2], and target_shape = (2, 4) + # [3, 4]] + # then we would expand a to: + # a = [[1, 1, 2, 2], + # [3, 3, 4, 4]] + # NOTE this function this function does not explicitly broadcast dimensions + # with an extent of 1, since this can be done implicitly by pytorch + def group_broadcast(t, shape): + for i, s in enumerate(shape): + if t.shape[i] != s and t.shape[i] != 1: + assert s % t.shape[i] == 0 + t = ( + t.unsqueeze(i + 1) + .expand(*t.shape[: i + 1], s // t.shape[i], *t.shape[i + 1 :]) + .flatten(i, i + 1) + ) + return t + + scale_a = group_broadcast(scale_a, a.shape) + scale_b = group_broadcast(scale_b, b.shape) + output = torch.mm( + (scale_a * a.to(dtype=torch.float32)), (scale_b * b.to(dtype=torch.float32)) + ).to(out_dtype) + if bias is not None: + output = output + bias + return output + + +def _test_accuracy_once(M, N, K, out_dtype, device): + fp8_info = torch.finfo(torch.float8_e4m3fn) + fp8_max, fp8_min = fp8_info.max, fp8_info.min + a_fp32 = (torch.rand(M, K, dtype=torch.float32, device=device) - 0.5) * 2 * fp8_max + a_fp8 = a_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + b_fp32 = (torch.rand(N, K, dtype=torch.float32, device=device) - 0.5) * 2 * fp8_max + b_fp8 = b_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn).t() + scale_a_group_shape = (1, 128) + scale_b_group_shape = (128, 128) + scale_a_shape = scale_shape(a_fp8.shape, scale_a_group_shape) + scale_b_shape = scale_shape(b_fp8.shape, scale_b_group_shape) + scale_a = torch.randn(scale_a_shape, device=device, dtype=torch.float32) * 0.001 + scale_b = torch.randn(scale_b_shape, device=device, dtype=torch.float32) * 0.001 + scale_a = scale_a.t().contiguous().t() + scale_b = scale_b.t().contiguous().t() + o = baseline_scaled_mm(a_fp8, b_fp8, scale_a, scale_b, out_dtype) + o1 = fp8_blockwise_scaled_mm(a_fp8, b_fp8, scale_a, scale_b, out_dtype) + rtol = 0.02 + atol = 1 + torch.testing.assert_close(o, o1, rtol=rtol, atol=atol) + + +@pytest.mark.parametrize("M", [1, 3, 5, 127, 128, 512, 1024, 4096]) +@pytest.mark.parametrize("N", [128, 512, 1024, 4096, 8192, 14080]) +@pytest.mark.parametrize("K", [512, 1024, 4096, 8192, 14080, 16384]) +@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16]) +def test_accuracy(M, N, K, out_dtype): + _test_accuracy_once(M, N, K, out_dtype, "cuda") + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fp8_blockwise_moe.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fp8_blockwise_moe.py new file mode 100755 index 00000000..0488c094 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fp8_blockwise_moe.py @@ -0,0 +1,221 @@ +import random +from typing import Tuple + +import pytest +import torch +from sgl_kernel import fp8_blockwise_scaled_grouped_mm + + +def cdiv(a: int, b: int) -> int: + return -(a // -b) + + +def scale_shape(shape, group_shape): + return tuple(cdiv(shape[i], group_shape[i]) for i in range(len(group_shape))) + + +def to_fp8(tensor: torch.Tensor) -> torch.Tensor: + finfo = torch.finfo(torch.float8_e4m3fn) + return torch.round(tensor.clamp(min=finfo.min, max=finfo.max)).to( + dtype=torch.float8_e4m3fn + ) + + +# Copy from: https://github.com/deepseek-ai/DeepGEMM/blob/main/deep_gemm/utils.py +def calc_diff(x, y): + x, y = x.double(), y.double() + denominator = (x * x + y * y).sum() + sim = 2 * (x * y).sum() / denominator + return 1 - sim + + +def ceil_div(x: int, y: int) -> int: + return (x + y - 1) // y + + +def per_token_cast_to_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 + m, n = x.shape + pad_size = (128 - (n % 128)) % 128 + x = torch.nn.functional.pad(x, (0, pad_size), value=0) if pad_size > 0 else x + x_view = x.view(m, -1, 128) + x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4) + fp8_data = (x_view * (448.0 / x_amax.unsqueeze(2))).to(torch.float8_e4m3fn) + return fp8_data.view(m, n + pad_size)[:, :n], (x_amax / 448.0).view(m, -1) + + +def per_block_cast_to_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 + m, n = x.shape + x_padded = torch.zeros( + (ceil_div(m, 128) * 128, ceil_div(n, 128) * 128), dtype=x.dtype, device=x.device + ) + x_padded[:m, :n] = x + x_view = x_padded.view(-1, 128, x_padded.size(1) // 128, 128) + x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4) + x_scaled = (x_view * (448.0 / x_amax)).to(torch.float8_e4m3fn) + return x_scaled.view_as(x_padded)[:m, :n].contiguous(), (x_amax / 448.0).view( + x_view.size(0), x_view.size(2) + ) + + +def baseline_scaled_mm( + a: torch.Tensor, + b: torch.Tensor, + scale_a: torch.Tensor, + scale_b: torch.Tensor, + out_dtype: type[torch.dtype], +) -> torch.Tensor: + + def group_broadcast(t, shape): + for i, s in enumerate(shape): + if t.shape[i] != s and t.shape[i] != 1: + assert s % t.shape[i] == 0 + t = ( + t.unsqueeze(i + 1) + .expand(*t.shape[: i + 1], s // t.shape[i], *t.shape[i + 1 :]) + .flatten(i, i + 1) + ) + return t + + scale_a = group_broadcast(scale_a, a.shape) + scale_b = group_broadcast(scale_b, b.shape) + + return torch.mm( + (scale_a * a.to(dtype=torch.float32)), (scale_b * b.to(dtype=torch.float32)) + ).to(out_dtype) + + +def is_blackwell_supported(device=None) -> bool: + return (torch.cuda.get_device_capability(device)[0] in [10, 12]) and ( + torch.version.cuda >= "12.8" + ) + + +def is_sm90_supported(device=None) -> bool: + return (torch.cuda.get_device_capability(device)[0] == 9) and ( + torch.version.cuda >= "12.3" + ) + + +@pytest.mark.skipif( + not (is_blackwell_supported() or is_sm90_supported()), + reason="fp8_blockwise_scaled_grouped_mm at sgl-kernel is only supported on sm100 or sm90", +) +@pytest.mark.parametrize("num_experts", [8, 16, 32, 64, 128]) +@pytest.mark.parametrize("out_dtype", [torch.half, torch.bfloat16]) +def test_fp8_blockwise_scaled_grouped_mm(num_experts, out_dtype): + device = "cuda" + alignment = 128 + n_g = random.randint(1, 64) * 128 + k_g = random.randint(1, 64) * 128 + + expert_offsets = torch.zeros((num_experts + 1), device=device, dtype=torch.int32) + problem_sizes = torch.zeros((num_experts, 3), device=device, dtype=torch.int32) + layout_sfa = torch.zeros((num_experts, 5), device=device, dtype=torch.int32) + layout_sfb = torch.zeros((num_experts, 5), device=device, dtype=torch.int32) + + a_tensors = [] + b_tensors = [] + a_scales_tensors = [] + b_scales_tensors = [] + baseline_tensors = [] + + for g in range(num_experts): + m_g = random.randint(1, 256) + expert_offsets[g + 1] = expert_offsets[g] + m_g + problem_sizes[g][:] = torch.tensor([m_g, n_g, k_g], device=device) + + a = torch.randn((m_g, k_g), device=device, dtype=out_dtype) # (M, K):(K, 1) + b = torch.randn((n_g, k_g), device=device, dtype=out_dtype).t() # (K, N):(1, K) + + a_g, a_scale = per_token_cast_to_fp8( + a + ) # ag -- (M, K):(K, 1), a_scale() -- (M, k):(k, 1) + b_g, b_scale = per_block_cast_to_fp8( + b + ) # bg -- (K, N):(N, 1), b_scale() -- (k, n):(n, 1) + a_tensors.append(a_g) + b_tensors.append(b_g) + a_scales_tensors.append(a_scale) + b_scales_tensors.append(b_scale) + + baseline = torch.mm(a, b) + baseline_tensors.append(baseline) + a_stack = torch.empty( + (expert_offsets[-1], k_g), device=device, dtype=torch.float8_e4m3fn + ) + b_stack = torch.empty( + (num_experts, n_g, k_g), device=device, dtype=torch.float8_e4m3fn + ) + a_scale_stack = torch.empty( + (expert_offsets[-1], (k_g // 128)), device=device, dtype=torch.float32 + ) + b_scale_stack = torch.empty( + (num_experts, n_g // 128, k_g // 128), device=device, dtype=torch.float32 + ) + + for g in range(num_experts): + # Matrix A is Row-Major. + a_stack[expert_offsets[g] : expert_offsets[g + 1], :] = a_tensors[ + g + ] # a_stack[expert_offsets[g] : expert_offsets[g + 1], :] -- (M, K):(K, 1) + b_stack[g] = b_tensors[g].t() # b_stack[g] -- (N, K):(K, 1) + + # We need K-Major scale factor + a_scale_stack[expert_offsets[g] : expert_offsets[g + 1], :] = a_scales_tensors[ + g + ] + b_scale_stack[g] = b_scales_tensors[ + g + ].t() # b_scale_stack[g] -- (k, n):(n, 1), we need transpose & contiguous later + b_stack = b_stack.transpose(1, 2) # Transpose Matrix B to Column-Major. + b_scale_stack = b_scale_stack.transpose(1, 2) + + c_out = torch.empty((expert_offsets[-1], n_g), device=device, dtype=out_dtype) + a_strides = torch.full( + (num_experts,), a_stack.stride(0), device=device, dtype=torch.int64 + ) + c_strides = torch.full( + (num_experts,), c_out.stride(0), device=device, dtype=torch.int64 + ) + workspace = torch.empty((1024 * 1024 * 1024), device=device, dtype=torch.uint8) + a_ptrs = torch.empty((num_experts,), device=device, dtype=torch.int64) + b_ptrs = torch.empty((num_experts,), device=device, dtype=torch.int64) + out_ptrs = torch.empty((num_experts,), device=device, dtype=torch.int64) + a_scales_ptrs = torch.empty((num_experts,), device=device, dtype=torch.int64) + b_scales_ptrs = torch.empty((num_experts,), device=device, dtype=torch.int64) + + fp8_blockwise_scaled_grouped_mm( + c_out, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a_stack, + b_stack, + a_scale_stack, + b_scale_stack, + a_strides, + a_strides, + c_strides, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets[:-1], + workspace, + ) + + for g in range(num_experts): + baseline = baseline_tensors[g] + actual = c_out[expert_offsets[g] : expert_offsets[g + 1]] + diff = calc_diff(actual, baseline) + assert diff < 0.001 + print( + f"m_g={baseline.shape[0]} n_g={n_g} k_g={k_g} num_experts={num_experts}, out_dtype={out_dtype}, diff={diff:.5f}: OK" + ) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fp8_gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fp8_gemm.py new file mode 100644 index 00000000..e70e62af --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fp8_gemm.py @@ -0,0 +1,49 @@ +import pytest +import torch +from sgl_kernel import fp8_scaled_mm + + +def torch_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias): + o = torch.matmul(a.to(torch.float32), b.to(torch.float32)) + o = o.to(torch.float32) + temp1 = o * scale_a.view(-1, 1) + temp2 = temp1 * scale_b.view(1, -1) + final = temp2.to(out_dtype) + if bias is not None: + final = final + bias.view(1, -1) + return final + + +def _test_accuracy_once(M, N, K, with_bias, out_dtype, device): + fp8_info = torch.finfo(torch.float8_e4m3fn) + fp8_max, fp8_min = fp8_info.max, fp8_info.min + a_fp32 = (torch.rand(M, K, dtype=torch.float32, device=device) - 0.5) * 2 * fp8_max + a_fp8 = a_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + b_fp32 = (torch.rand(N, K, dtype=torch.float32, device=device) - 0.5) * 2 * fp8_max + b_fp8 = b_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + scale_a = torch.randn((M,), device=device, dtype=torch.float32) * 0.001 + scale_b = torch.randn((N,), device=device, dtype=torch.float32) * 0.001 + if with_bias: + bias = torch.randn((N,), device=device, dtype=out_dtype) + else: + bias = None + b_fp8 = b_fp8.t() + o = torch_scaled_mm(a_fp8, b_fp8, scale_a, scale_b, out_dtype, bias) + o1 = fp8_scaled_mm(a_fp8, b_fp8, scale_a, scale_b, out_dtype, bias) + rtol = 0.02 + atol = 1 + torch.testing.assert_close(o, o1, rtol=rtol, atol=atol) + print(f"M={M}, N={N}, K={K}, with_bias={with_bias}, out_dtype={out_dtype}: OK") + + +@pytest.mark.parametrize("M", [1, 128, 512, 1024, 4096]) +@pytest.mark.parametrize("N", [16, 128, 512, 1024, 4096]) +@pytest.mark.parametrize("K", [512, 1024, 4096, 8192, 16384]) +@pytest.mark.parametrize("with_bias", [True, False]) +@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16]) +def test_accuracy(M, N, K, with_bias, out_dtype): + _test_accuracy_once(M, N, K, with_bias, out_dtype, "cuda") + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fused_qk_norm_rope.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fused_qk_norm_rope.py new file mode 100644 index 00000000..05a74fac --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_fused_qk_norm_rope.py @@ -0,0 +1,225 @@ +import pytest +import torch +from sgl_kernel import fused_qk_norm_rope as sgl_fused_qk_norm_rope + +from sglang.srt.layers.layernorm import RMSNorm +from sglang.srt.layers.rotary_embedding import get_rope +from sglang.srt.server_args import ( + ServerArgs, + get_global_server_args, + set_global_server_args_for_scheduler, +) +from sglang.srt.utils import ( + cpu_has_amx_support, + is_cpu, + is_cuda, + is_hip, + is_npu, + is_xpu, +) + +_is_cuda = is_cuda() +_is_hip = is_hip() +_is_cpu = is_cpu() +_is_cpu_amx_available = cpu_has_amx_support() +_is_npu = is_npu() +_is_xpu = is_xpu() + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + +@torch.inference_mode() +def torch_ref_rms_norm_rope( + qkv, + num_heads_q, + num_heads_k, + num_heads_v, + head_dim, + eps, + q_weight, + k_weight, + base, + is_neox, + position_ids, +): + """ + PyTorch reference implementation of RMSNorm+RoPE for verification. + + Uses SGLang's own RMSNorm and RotaryEmbedding modules to ensure consistency + with the expected behavior of the fused kernel. + + Args: + qkv: Combined QKV tensor of shape [num_tokens, hidden_size] + num_heads_q: Number of query heads + num_heads_k: Number of key heads + num_heads_v: Number of value heads (unused for normalization/RoPE but needed for tensor splitting) + head_dim: Dimension of each head + eps: Epsilon value for RMS normalization + q_weight: RMSNorm weights for query [head_dim] + k_weight: RMSNorm weights for key [head_dim] + base: Base value for RoPE calculations + is_neox: Whether to use NeoX style RoPE + position_ids: Position IDs for RoPE of shape [num_tokens] + + Returns: + Combined tensor with Q and K parts normalized and RoPE applied + """ + # Get input shape information + num_tokens = qkv.shape[0] + hidden_size = qkv.shape[1] + + # Calculate dimensions for Q, K, V segments + q_size = num_heads_q * head_dim + k_size = num_heads_k * head_dim + v_size = num_heads_v * head_dim + + # Verify dimensions match + assert ( + hidden_size == q_size + k_size + v_size + ), f"Hidden size {hidden_size} doesn't match Q+K+V dimensions {q_size + k_size + v_size}" + + # Split the tensor into Q, K, V parts + q = qkv[:, :q_size] + k = qkv[:, q_size : q_size + k_size] + v = qkv[:, q_size + k_size :] + + rotary_emb = get_rope( + head_dim, + rotary_dim=head_dim, + max_position=8192, + base=10000, + is_neox_style=is_neox, + rope_scaling=None, + dual_chunk_attention_config=None, + ) + rotary_emb = rotary_emb.to(qkv.device) + + # Create and apply RMSNorm modules with custom weights + q_norm = RMSNorm(hidden_size=head_dim, eps=eps).to(qkv.device).to(qkv.dtype) + q_norm.weight.data.copy_(q_weight) + k_norm = RMSNorm(hidden_size=head_dim, eps=eps).to(qkv.device).to(qkv.dtype) + k_norm.weight.data.copy_(k_weight) + + q_by_head = q.reshape(-1, head_dim) + q_by_head = q_norm(q_by_head) + k_by_head = k.reshape(-1, head_dim) + k_by_head = k_norm(k_by_head) + q = q_by_head.view(q.shape) + k = k_by_head.view(k.shape) + + [q_rope, k_rope] = rotary_emb( + position_ids, + q, + k, + fused_set_kv_buffer_arg=None, + ) + + # Combine Q, K, V back together + result = torch.cat([q_rope, k_rope, v], dim=1) + + return result + + +head_dims = [64, 128] +# (Q heads, K heads, V heads) +num_heads_groups = [ + (16, 8, 8), # Qwen3-0.6B, Qwen3-1.7B + (32, 8, 8), # Qwen3-4B, Qwen3-8B, Qwen3-30B-A3B + (40, 8, 8), # Qwen3-14B + (64, 8, 8), # Qwen3-32B, Qwen3-235B-A22B +] +num_tokens_list = [1, 3, 8, 32, 256] +is_neox_list = [False, True] +dtypes = [torch.bfloat16] + + +@pytest.mark.skipif(not _is_cuda, reason="Skipping CUDA/ROCm only tests.") +@pytest.mark.parametrize("head_dim", head_dims) +@pytest.mark.parametrize("num_heads_group", num_heads_groups) +@pytest.mark.parametrize("num_tokens", num_tokens_list) +@pytest.mark.parametrize("is_neox", is_neox_list) +@pytest.mark.parametrize("dtype", dtypes) +def test_fused_qk_norm_rope(head_dim, num_heads_group, num_tokens, is_neox, dtype): + """ + Test the fused QK RMSNorm + RoPE operation with various configurations. + + This test verifies that the fused kernel correctly applies: + 1. RMSNorm to both query (Q) and key (K) portions of the QKV tensor + 2. Rotary Position Embeddings (RoPE) to the normalized Q and K + 3. Leaves the value (V) portion unchanged + + Args: + head_dim: Dimension of each attention head + num_heads_group: Tuple of (num_heads_q, num_heads_k, num_heads_v) + num_tokens: Number of tokens to process + dtype: Data type (float16 or bfloat16) + """ + set_global_server_args_for_scheduler(ServerArgs(model_path="dummy")) + device = "cuda" + torch_dtype = dtype + + # Unpack head counts + num_heads_q, num_heads_k, num_heads_v = num_heads_group + + # Calculate total hidden dimension + hidden_size = (num_heads_q + num_heads_k + num_heads_v) * head_dim + + # Generate random inputs directly as 2D [num_tokens, hidden_size] + torch.random.manual_seed(0) + qkv = torch.randn(num_tokens, hidden_size, dtype=torch_dtype, device=device) + qkv_copy = qkv.clone() + + # Generate position IDs with +100 offset to test decoding scenarios + position_ids = torch.arange(num_tokens, dtype=torch.int32, device=device) + 100 + + # Generate random weights for RMSNorm + q_weight = torch.randn(head_dim, dtype=torch_dtype, device=device) * 5.0 + k_weight = torch.randn(head_dim, dtype=torch_dtype, device=device) * 5.0 + + # Set RMSNorm and RoPE parameters + eps = 1e-5 + base = 10000.0 + + factor, low, high, attention_factor = 1.0, 0, 0, 1.0 + # Run the custom fusedQKNormRope operation + sgl_fused_qk_norm_rope( + qkv, + num_heads_q, + num_heads_k, + num_heads_v, + head_dim, + eps, + q_weight, + k_weight, + base, + is_neox, + position_ids, + factor, + low, + high, + attention_factor, + ) + output = qkv # This op is inplace + + # Compute reference output using TensorRT-LLM modules + ref_output = torch_ref_rms_norm_rope( + qkv_copy, + num_heads_q, + num_heads_k, + num_heads_v, + head_dim, + eps, + q_weight, + k_weight, + base, + is_neox, + position_ids, + ) + + # Compare outputs from custom kernel vs reference implementation + torch.testing.assert_close( + output, + ref_output, + rtol=5e-2, + atol=1e-1, + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_gguf.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_gguf.py new file mode 100644 index 00000000..3be5e6f3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_gguf.py @@ -0,0 +1,166 @@ +# SPDX-License-Identifier: Apache-2.0 + +import random +from pathlib import Path + +import numpy as np +import pytest +import torch +from gguf import GGMLQuantizationType, GGUFReader, ReaderTensor, dequantize +from huggingface_hub import snapshot_download +from sgl_kernel import ( + ggml_dequantize, + ggml_moe_a8, + ggml_moe_a8_vec, + ggml_moe_get_block_size, + ggml_mul_mat_a8, + ggml_mul_mat_vec_a8, +) + +GGUF_SAMPLE = snapshot_download("Isotr0py/test-gguf-sample") +GGUF_SAMPLE_MOE = snapshot_download("SzymonOzog/test-gguf-moe-sample") + + +def get_gguf_sample_tensors( + hidden_size: int, quant_type: GGMLQuantizationType +) -> list[ReaderTensor]: + sample_dir = GGUF_SAMPLE + filename = f"Quant_{quant_type.name}_{hidden_size}.gguf" + sample_file = Path(sample_dir) / filename + return GGUFReader(sample_file).tensors + + +def get_gguf_MoE_tensors( + hidden_size: int, quant_type: GGMLQuantizationType +) -> list[ReaderTensor]: + sample_dir = GGUF_SAMPLE_MOE + filename = f"Quant_{quant_type.name}_{hidden_size}.gguf" + sample_file = Path(sample_dir) / filename + return GGUFReader(sample_file).tensors + + +DTYPES = [torch.bfloat16] # [torch.half, torch.bfloat16, torch.float32] +# Hidden_size for testing, must match the sample file in HF repo, +# we have `hidden_size = 256, 1024` for test in HF repo currently. +HIDDEN_SIZES = [256, 1024] +NUM_TOKENS = [7, 2050] # Arbitrary values for testing +SEEDS = [0] +QUANT_TYPES = [ + # i-matrix + GGMLQuantizationType.IQ1_M, + GGMLQuantizationType.IQ1_S, + GGMLQuantizationType.IQ2_S, + GGMLQuantizationType.IQ2_XS, + GGMLQuantizationType.IQ3_S, + GGMLQuantizationType.IQ3_XXS, + GGMLQuantizationType.IQ4_NL, + GGMLQuantizationType.IQ4_XS, + # k-quants + GGMLQuantizationType.Q2_K, + GGMLQuantizationType.Q3_K, + GGMLQuantizationType.Q4_K, + GGMLQuantizationType.Q5_K, + GGMLQuantizationType.Q6_K, + # standard quantization + GGMLQuantizationType.Q4_0, + GGMLQuantizationType.Q5_0, + GGMLQuantizationType.Q8_0, +] + + +@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES) +@pytest.mark.parametrize("dtype", DTYPES) +@pytest.mark.parametrize("quant_type", QUANT_TYPES) +@torch.inference_mode() +def test_dequantize( + hidden_size: int, dtype: torch.dtype, quant_type: GGMLQuantizationType +): + tensors = get_gguf_sample_tensors(hidden_size, quant_type) + for tensor in tensors: + shape_str = tensor.name.split("_")[-1] + shape = map(int, shape_str.split("x")) + + ref_output = torch.tensor( + dequantize(tensor.data, quant_type), device="cuda" + ).to(dtype) + output = ggml_dequantize( + torch.tensor(tensor.data, device="cuda"), quant_type, *list(shape), dtype + ) + + torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=4e-2) + + +@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES) +@pytest.mark.parametrize("dtype", DTYPES) +@pytest.mark.parametrize("quant_type", QUANT_TYPES) +@torch.inference_mode() +def test_mmvq(hidden_size: int, dtype: torch.dtype, quant_type: GGMLQuantizationType): + + tensors = get_gguf_sample_tensors(hidden_size, quant_type) + x = torch.rand((1, hidden_size), dtype=dtype, device="cuda") + for tensor in tensors: + weight = torch.tensor(dequantize(tensor.data, quant_type), device="cuda").to( + dtype + ) + ref_output = x @ weight.T + + qweight = torch.tensor(tensor.data, device="cuda") + output = ggml_mul_mat_vec_a8(qweight, x, quant_type, qweight.shape[0]).to(dtype) + + # NOTE(FlamingoPg): There can be occasional errors, Loosen the granularity of gguf bf16 verification. + atols = {torch.half: 1, torch.bfloat16: 1.5, torch.float: 1} + rtols = {torch.half: 1e-1, torch.bfloat16: 3e1, torch.float: 1e-1} + + torch.testing.assert_close( + output, ref_output, atol=atols[dtype], rtol=rtols[dtype] + ) + + +@pytest.mark.parametrize("num_tokens", NUM_TOKENS) +@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES) +@pytest.mark.parametrize("dtype", DTYPES) +@pytest.mark.parametrize( + "quant_type", + [ + # k-quants + GGMLQuantizationType.Q2_K, + GGMLQuantizationType.Q3_K, + GGMLQuantizationType.Q4_K, + GGMLQuantizationType.Q5_K, + GGMLQuantizationType.Q6_K, + # standard quants + GGMLQuantizationType.Q4_0, + GGMLQuantizationType.Q5_0, + GGMLQuantizationType.Q8_0, + ], +) +@torch.inference_mode() +def test_mmq( + num_tokens: int, + hidden_size: int, + dtype: torch.dtype, + quant_type: GGMLQuantizationType, +): + + tensors = get_gguf_sample_tensors(hidden_size, quant_type) + x = torch.rand((num_tokens, hidden_size), dtype=dtype, device="cuda") + for tensor in tensors: + weight = torch.tensor(dequantize(tensor.data, quant_type), device="cuda").to( + dtype + ) + ref_output = x @ weight.T + + qweight = torch.tensor(tensor.data, device="cuda") + output = ggml_mul_mat_a8(qweight, x, quant_type, qweight.shape[0]) + atols = {torch.half: 1, torch.bfloat16: 1.5, torch.float: 1.2} + # test matrix has inputs centered around 0 and lower precision from + # bfloat16 tends to accumulate and can greatly inflate rtol + # since outputs are also very close to 0 + rtols = {torch.half: 1e-1, torch.bfloat16: 1e4, torch.float: 2e1} + torch.testing.assert_close( + output, ref_output, atol=atols[dtype], rtol=rtols[dtype] + ) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_gptq_kernel.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_gptq_kernel.py new file mode 100644 index 00000000..0ef4fad2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_gptq_kernel.py @@ -0,0 +1,131 @@ +import pytest +import torch +from sgl_kernel import gptq_gemm + +from sglang.srt.layers.quantization.utils import pack_cols, pack_rows + + +def torch_dequantize(q_weight, q_zeros, scales, g_idx, use_shuffle, bit, K, N): + assert bit == 4, "Reference dequantization only supports 4-bit" + group_size = K // scales.shape[0] + pack_factor = 32 // bit + + # unpack q_weight: (K//pack_factor, N) -> (K, N) + unpacked_q_weight = torch.empty( + q_weight.shape[0] * pack_factor, + q_weight.shape[1], + dtype=torch.uint8, + device=q_weight.device, + ) + for i in range(pack_factor): + unpacked_q_weight[i::pack_factor, :] = (q_weight >> (i * 4)) & 0x0F + + # unpack q_zeros: (num_groups, N//pack_factor) -> (num_groups, N) + unpacked_q_zeros = torch.empty( + q_zeros.shape[0], + q_zeros.shape[1] * pack_factor, + dtype=torch.uint8, + device=q_zeros.device, + ) + for i in range(pack_factor): + unpacked_q_zeros[:, i::pack_factor] = (q_zeros >> (i * 4)) & 0x0F + + unpacked_q_zeros += 1 + unpacked_q_zeros = unpacked_q_zeros.to(scales.dtype) + + scale_zeros = unpacked_q_zeros * scales # (num_groups, N) + + current_g_idx = torch.tensor( + [i // group_size for i in range(K)], dtype=torch.int32, device=q_weight.device + ) + + scale_mat = scales[current_g_idx] # (K, N) + scale_zeros_mat = scale_zeros[current_g_idx] # (K, N) + + # dequant: weight * scale - scale_zeros + dequantized_b = unpacked_q_weight.to(scales.dtype) * scale_mat - scale_zeros_mat + + return dequantized_b.reshape(K, N) + + +def torch_gptq_gemm( + a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx, use_shuffle, bit +): + K, N = a.shape[1], b_q_weight.shape[1] + + b_dequant = torch_dequantize( + b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx, use_shuffle, bit, K, N + ) + c = torch.matmul(a, b_dequant) + return c + + +def _test_gptq_gemm_once(M, N, K, bit, group_size, use_shuffle, dtype, device="cuda"): + + b_fp = torch.randn(K, N, dtype=dtype, device=device) + + assert K % group_size == 0, "K must be divisible by group_size" + num_groups = K // group_size + + if use_shuffle: + return + else: + g_idx = torch.tensor( + [i // group_size for i in range(K)], dtype=torch.int32, device=device + ) + b_shuffled = b_fp[g_idx] + + b_grouped = b_shuffled.reshape(num_groups, group_size, N) + + b_max = torch.max(b_grouped, dim=1, keepdim=True)[0] + b_min = torch.min(b_grouped, dim=1, keepdim=True)[0] + + scales = (b_max - b_min) / (2**bit - 1) + scales = scales.clamp(min=1e-6) + + zeros_float = (-b_min / scales).round() + + q_b = ( + (b_grouped / scales + zeros_float).round().clamp(0, 2**bit - 1).to(torch.uint8) + ) + + q_zeros_unpacked = zeros_float.to(torch.uint8) - 1 + + b_q_weight = pack_rows(q_b.reshape(K, N), bit, K, N) + + q_zeros_unpacked = q_zeros_unpacked.reshape(num_groups, N) + b_gptq_qzeros = pack_cols(q_zeros_unpacked, bit, num_groups, N) + b_gptq_scales = scales.squeeze(1) + + a = torch.randn(M, K, dtype=dtype, device=device) + + c_ref = torch_gptq_gemm( + a, b_q_weight, b_gptq_qzeros, b_gptq_scales, g_idx, use_shuffle, bit + ) + c_out = gptq_gemm( + a, b_q_weight, b_gptq_qzeros, b_gptq_scales, g_idx, use_shuffle, bit + ) + + rtol = 4e-2 + atol = 4e-2 + torch.testing.assert_close(c_ref, c_out, rtol=rtol, atol=atol) + print( + f"✅ Test passed: M={M}, N={N}, K={K}, bit={bit}, group_size={group_size}, use_shuffle={use_shuffle}, dtype={dtype}" + ) + + +@pytest.mark.parametrize("M", [1, 8, 128]) +@pytest.mark.parametrize("N", [2048, 4096]) +@pytest.mark.parametrize("K", [2048, 4096]) +@pytest.mark.parametrize("bit", [4]) +@pytest.mark.parametrize("group_size", [128]) +@pytest.mark.parametrize("use_shuffle", [False]) +@pytest.mark.parametrize("dtype", [torch.float16]) +def test_gptq_gemm(M, N, K, bit, group_size, use_shuffle, dtype): + if not torch.cuda.is_available(): + pytest.skip("CUDA not available") + _test_gptq_gemm_once(M, N, K, bit, group_size, use_shuffle, dtype, "cuda") + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_hadamard.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_hadamard.py new file mode 100644 index 00000000..5d1cd40e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_hadamard.py @@ -0,0 +1,78 @@ +import math + +import pytest +import torch +import torch.nn.functional as F +from einops import rearrange, repeat +from scipy.linalg import hadamard +from sgl_kernel import hadamard_transform + + +def hadamard_transform_ref(x, scale=1.0): + """ + x: (..., dim) + out: (..., dim) + """ + if hadamard is None: + raise ImportError("Please install scipy") + x_shape = x.shape + dim = x.shape[-1] + x = x.reshape(-1, dim) + log_dim = math.ceil(math.log2(dim)) + dim_padded = 2**log_dim + if dim != dim_padded: + x = F.pad(x, (0, dim_padded - dim)) + out = F.linear( + x, + torch.tensor(hadamard(dim_padded, dtype=float), dtype=x.dtype, device=x.device), + ) + out = out * scale + return out[..., :dim].reshape(*x_shape) + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize( + "dim", + [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 137, 1024, 2048, 4096, 8192, 16384, 32768], +) +def test_fast_hadamard_transform(dim, dtype): + device = "cuda" + + if dtype == torch.float32: + rtol, atol = 3e-4, 3e-3 + elif dtype == torch.bfloat16: + rtol, atol = 1e-2, 5e-2 + else: # float16 + rtol, atol = 3e-3, 5e-3 + + torch.random.manual_seed(0) + batch_size = 15 + + x = torch.randn(batch_size, dim, device=device, dtype=dtype) + x_ref = x.detach().clone().to(torch.float32) + x_pt = x.detach().clone() + + scale = 1 / math.sqrt(dim) + + out = hadamard_transform(x, scale=scale) + out_ref = hadamard_transform_ref(x_ref, scale=scale) + out_pt = hadamard_transform_ref(x_pt, scale=scale) + + torch.testing.assert_close( + out_pt.float(), + out_ref, + rtol=rtol, + atol=atol, + msg="Reference implementations mismatch", + ) + torch.testing.assert_close( + out.float(), + out_ref, + rtol=rtol, + atol=atol, + msg="fast_hadamard_transform output mismatch", + ) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_int8_gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_int8_gemm.py new file mode 100644 index 00000000..80f32cd0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_int8_gemm.py @@ -0,0 +1,48 @@ +import pytest +import torch +from sgl_kernel import int8_scaled_mm +from utils import is_sm10x + + +def to_int8(tensor: torch.Tensor) -> torch.Tensor: + return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8) + + +def torch_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias): + o = torch.matmul(a.to(torch.float32), b.to(torch.float32)) + if bias is not None: + o = o.to(torch.float32) * scale_a.view(-1, 1) * scale_b.view(1, -1) + bias + else: + o = o.to(torch.float32) * scale_a.view(-1, 1) * scale_b.view(1, -1) + return o.to(out_dtype) + + +def _test_accuracy_once(M, N, K, with_bias, out_dtype, device): + a = to_int8(torch.randn((M, K), device=device) * 5) + b = to_int8(torch.randn((N, K), device=device).t() * 5) + scale_a = torch.randn((M,), device="cuda", dtype=torch.float32) + scale_b = torch.randn((N,), device="cuda", dtype=torch.float32) + if with_bias: + bias = torch.randn((N,), device="cuda", dtype=out_dtype) * 10 + else: + bias = None + o = int8_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias) + o1 = torch_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias) + torch.testing.assert_close(o, o1) + + +@pytest.mark.skipif( + is_sm10x(), + reason="int8_scaled_mm is only supported on sm90 and lower", +) +@pytest.mark.parametrize("M", [1, 16, 32, 64, 128, 512, 1024, 4096, 8192]) +@pytest.mark.parametrize("N", [16, 128, 512, 1024, 4096, 8192, 16384]) +@pytest.mark.parametrize("K", [512, 1024, 4096, 8192, 16384]) +@pytest.mark.parametrize("with_bias", [True, False]) +@pytest.mark.parametrize("out_dtype", [torch.float16, torch.bfloat16]) +def test_accuracy(M, N, K, with_bias, out_dtype): + _test_accuracy_once(M, N, K, with_bias, out_dtype, "cuda") + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_kimi_k2_moe_fused_gate.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_kimi_k2_moe_fused_gate.py new file mode 100644 index 00000000..f96312a1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_kimi_k2_moe_fused_gate.py @@ -0,0 +1,124 @@ +import pytest +import torch +from sgl_kernel import kimi_k2_moe_fused_gate + +from sglang.srt.layers.moe.topk import kimi_k2_biased_topk_impl + + +@pytest.mark.parametrize( + "seq_length", + list(range(1, 10)) + + [16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536], +) +@pytest.mark.parametrize("topk", [6]) # Kimi K2 uses topk=6 +@pytest.mark.parametrize("dtype", [torch.float32]) +@pytest.mark.parametrize("apply_routed_scaling_factor_on_output", [False, True]) +def test_kimi_k2_moe_fused_gate( + seq_length, topk, dtype, apply_routed_scaling_factor_on_output +): + num_experts = 384 # Kimi K2: only support 384 experts + renormalize = True + routed_scaling_factor = 2.872 # Kimi K2's routed scaling factor + + torch.manual_seed(seq_length) + tensor = torch.rand((seq_length, num_experts), dtype=dtype, device="cuda") + scores = tensor.clone() + bias = torch.rand(num_experts, dtype=dtype, device="cuda") + + # Test our fused kernel + output, indices = kimi_k2_moe_fused_gate( + tensor, + bias, + topk=topk, + renormalize=renormalize, + routed_scaling_factor=routed_scaling_factor, + apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output, + ) + + # Reference implementation + ref_output, ref_indices = kimi_k2_biased_topk_impl( + scores, + scores, + bias, + topk=topk, + renormalize=renormalize, + routed_scaling_factor=routed_scaling_factor, + apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output, + ) + + # Check weights match (after sorting) + # Weights are the most important - they determine the actual MoE output + output_check = torch.allclose( + ref_output.sort()[0].to(torch.float32), + output.sort()[0].to(torch.float32), + rtol=1e-02, + atol=1e-03, + ) + + assert output_check, ( + f"Output mismatch at seq_length {seq_length}, dtype {dtype}, " + f"num_experts {num_experts}, topk {topk}, " + f"apply_routed_scaling_factor_on_output {apply_routed_scaling_factor_on_output}" + ) + + +@pytest.mark.parametrize("seq_length", [1024, 4096]) +@pytest.mark.parametrize("num_experts", [384]) +@pytest.mark.parametrize("topk", [6]) +def test_kimi_k2_specific_case(seq_length, num_experts, topk): + """Test specifically for Kimi K2 configuration: 384 experts, topk=6""" + dtype = torch.float32 + renormalize = True + routed_scaling_factor = 2.872 + + torch.manual_seed(42) + tensor = torch.rand((seq_length, num_experts), dtype=dtype, device="cuda") + scores = tensor.clone() + bias = torch.rand(num_experts, dtype=dtype, device="cuda") + + output, indices = kimi_k2_moe_fused_gate( + tensor, + bias, + topk=topk, + renormalize=renormalize, + routed_scaling_factor=routed_scaling_factor, + apply_routed_scaling_factor_on_output=False, + ) + + ref_output, ref_indices = kimi_k2_biased_topk_impl( + scores, + scores, + bias, + topk=topk, + renormalize=renormalize, + routed_scaling_factor=routed_scaling_factor, + apply_routed_scaling_factor_on_output=False, + ) + + # Verify output shapes + assert output.shape == (seq_length, topk) + assert indices.shape == (seq_length, topk) + assert output.dtype == torch.float32 + assert indices.dtype == torch.int32 + + # Verify weights are normalized (sum to 1 per token if renormalize=True) + if renormalize: + weight_sums = output.sum(dim=-1) + assert torch.allclose( + weight_sums, torch.ones_like(weight_sums), rtol=1e-3, atol=1e-4 + ) + + # Check weights match (after sorting) + # Weights are the most important - they determine the actual MoE output + output_check = torch.allclose( + ref_output.sort()[0].to(torch.float32), + output.sort()[0].to(torch.float32), + rtol=1e-02, + atol=1e-03, + ) + + assert output_check, f"Output mismatch for Kimi K2 specific case" + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_kvcacheio.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_kvcacheio.py new file mode 100644 index 00000000..5ba1c85a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_kvcacheio.py @@ -0,0 +1,694 @@ +import pytest +import torch +from sgl_kernel.kvcacheio import ( + transfer_kv_all_layer, + transfer_kv_all_layer_direct_lf_pf, + transfer_kv_all_layer_lf_ph, + transfer_kv_all_layer_mla, + transfer_kv_direct, + transfer_kv_per_layer, + transfer_kv_per_layer_direct_pf_lf, + transfer_kv_per_layer_mla, +) + +from sglang.srt.utils import is_hip + + +def ref_copy_with_indices(src_pool, dst_pool, src_indices, dst_indices): + dst_pool[dst_indices] = src_pool[src_indices].to(dst_pool.device) + + +def ref_copy_with_indices_pf_direct( + src_pool, dst_pool, src_indices, dst_indices, page_size, layer_id, lf_to_pf=False +): + if lf_to_pf: + for i in range(0, len(src_indices), page_size): + dst_pool[dst_indices[i] // page_size][layer_id] = src_pool[layer_id][ + src_indices[i : i + page_size] + ].to(dst_pool.device) + else: + for i in range(0, len(src_indices), page_size): + dst_pool[layer_id][dst_indices[i : i + page_size]] = src_pool[ + src_indices[i] // page_size + ][layer_id].to(dst_pool.device) + + +def ref_copy_with_indices_page_head( + src_pool, + dst_pool, + src_indices, + dst_indices, + page_size, + layer_id, + head_num, + lf_to_ph=False, +): + if lf_to_ph: + for head_id in range(head_num): + for i in range(0, len(src_indices)): + dst_pool[dst_indices[i] // page_size][head_id][ + dst_indices[i] % page_size + ][layer_id] = src_pool[layer_id][src_indices[i]][head_id].to( + dst_pool.device + ) + else: + for head_id in range(head_num): + for i in range(0, len(src_indices)): + dst_pool[layer_id][dst_indices[i]][head_id] = src_pool[ + src_indices[i] // page_size + ][head_id][src_indices[i] % page_size][layer_id].to(dst_pool.device) + + +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +@pytest.mark.parametrize("num_items_to_transfer", [1, 128, 1024]) +@pytest.mark.parametrize("page_size", [1, 16, 64]) +@pytest.mark.parametrize("item_size", [256]) +@pytest.mark.parametrize("total_items_in_pool", [10240]) +@pytest.mark.parametrize("is_mla", [False, True]) +@pytest.mark.parametrize("all_layers", [False, True]) +def test_transfer_kv( + dtype: torch.dtype, + num_items_to_transfer: int, + item_size: int, + page_size: int, + total_items_in_pool: int, + is_mla: bool, + all_layers: bool, +): + """ + Tests the per-layer transfer functions, treating tensors as memory pools. + """ + + original_dtype = torch.get_default_dtype() + torch.set_default_dtype(dtype) + device = "cuda" + torch.cuda.manual_seed(42) + + num_layers = 4 # A small number of layers for pool creation + + total_pages_in_pool = total_items_in_pool // page_size + num_pages_to_transfer = num_items_to_transfer // page_size + if num_pages_to_transfer == 0: + torch.set_default_dtype(original_dtype) + return + page_indices = torch.randperm(total_pages_in_pool, dtype=torch.int64) + src_indices_host = torch.cat( + [ + torch.arange(p * page_size, (p + 1) * page_size) + for p in page_indices[:num_pages_to_transfer] + ] + ) + src_indices_device = src_indices_host.to(device) + dst_indices_host = torch.cat( + [ + torch.arange(p * page_size, (p + 1) * page_size) + for p in page_indices[num_pages_to_transfer : 2 * num_pages_to_transfer] + ] + ) + dst_indices_device = dst_indices_host.to(device) + + # Prepare memory pools based on whether it's an MLA case. + if is_mla: + src_pool_host = torch.randn( + num_layers, total_items_in_pool, item_size + ).pin_memory() + dst_pool_ref = torch.zeros_like(src_pool_host).to(device) + dst_pool_kernel = torch.zeros_like(dst_pool_ref) + dst_pool_direct = torch.zeros_like(dst_pool_ref) + else: + src_k_pool = torch.randn( + num_layers, total_items_in_pool, item_size + ).pin_memory() + src_v_pool = torch.randn( + num_layers, total_items_in_pool, item_size + ).pin_memory() + dst_k_pool_ref = torch.zeros_like(src_k_pool).to(device) + dst_v_pool_ref = torch.zeros_like(src_v_pool).to(device) + dst_k_pool_kernel = torch.zeros_like(dst_k_pool_ref) + dst_v_pool_kernel = torch.zeros_like(dst_v_pool_ref) + dst_k_pool_direct = torch.zeros_like(dst_k_pool_ref) + dst_v_pool_direct = torch.zeros_like(dst_v_pool_ref) + + torch.cuda.synchronize() + + # We will test the per-layer function on the first layer (index 0) of the pool. + layer_idx_to_test = 0 + + if is_mla: + if not all_layers: + ref_copy_with_indices( + src_pool_host[layer_idx_to_test], + dst_pool_ref[layer_idx_to_test], + src_indices_host, + dst_indices_device, + ) + transfer_kv_per_layer_mla( + src_pool_host[layer_idx_to_test], + dst_pool_kernel[layer_idx_to_test], + src_indices_device, + dst_indices_device, + item_size=item_size * dtype.itemsize, + ) + transfer_kv_direct( + [src_pool_host[layer_idx_to_test]], + [dst_pool_direct[layer_idx_to_test]], + src_indices_host, + dst_indices_device, + page_size=page_size, + ) + else: + for layer_id in range(num_layers): + ref_copy_with_indices( + src_pool_host[layer_id], + dst_pool_ref[layer_id], + src_indices_host, + dst_indices_device, + ) + src_layers_device = torch.tensor( + [src_pool_host[layer_id].data_ptr() for layer_id in range(num_layers)], + dtype=torch.uint64, + device=device, + ) + dst_layers_device = torch.tensor( + [ + dst_pool_kernel[layer_id].data_ptr() + for layer_id in range(num_layers) + ], + dtype=torch.uint64, + device=device, + ) + transfer_kv_all_layer_mla( + src_layers_device, + dst_layers_device, + src_indices_device, + dst_indices_device, + item_size=item_size * dtype.itemsize, + num_layers=num_layers, + ) + transfer_kv_direct( + [src_pool_host[layer_id] for layer_id in range(num_layers)], + [dst_pool_direct[layer_id] for layer_id in range(num_layers)], + src_indices_host, + dst_indices_device, + page_size=page_size, + ) + torch.cuda.synchronize() + torch.testing.assert_close(dst_pool_kernel, dst_pool_ref) + torch.testing.assert_close(dst_pool_direct, dst_pool_ref) + else: + if not all_layers: + ref_copy_with_indices( + src_k_pool[layer_idx_to_test], + dst_k_pool_ref[layer_idx_to_test], + src_indices_host, + dst_indices_device, + ) + ref_copy_with_indices( + src_v_pool[layer_idx_to_test], + dst_v_pool_ref[layer_idx_to_test], + src_indices_host, + dst_indices_device, + ) + transfer_kv_per_layer( + src_k_pool[layer_idx_to_test], + dst_k_pool_kernel[layer_idx_to_test], + src_v_pool[layer_idx_to_test], + dst_v_pool_kernel[layer_idx_to_test], + src_indices_device, + dst_indices_device, + item_size=item_size * dtype.itemsize, + ) + transfer_kv_direct( + [src_k_pool[layer_idx_to_test], src_v_pool[layer_idx_to_test]], + [ + dst_k_pool_direct[layer_idx_to_test], + dst_v_pool_direct[layer_idx_to_test], + ], + src_indices_host, + dst_indices_device, + page_size=page_size, + ) + else: + for layer_id in range(num_layers): + ref_copy_with_indices( + src_k_pool[layer_id], + dst_k_pool_ref[layer_id], + src_indices_host, + dst_indices_device, + ) + ref_copy_with_indices( + src_v_pool[layer_id], + dst_v_pool_ref[layer_id], + src_indices_host, + dst_indices_device, + ) + + src_k_layers_device = torch.tensor( + [src_k_pool[layer_id].data_ptr() for layer_id in range(num_layers)], + dtype=torch.uint64, + device=device, + ) + src_v_layers_device = torch.tensor( + [src_v_pool[layer_id].data_ptr() for layer_id in range(num_layers)], + dtype=torch.uint64, + device=device, + ) + dst_k_layers_device = torch.tensor( + [ + dst_k_pool_kernel[layer_id].data_ptr() + for layer_id in range(num_layers) + ], + dtype=torch.uint64, + device=device, + ) + dst_v_layers_device = torch.tensor( + [ + dst_v_pool_kernel[layer_id].data_ptr() + for layer_id in range(num_layers) + ], + dtype=torch.uint64, + device=device, + ) + transfer_kv_all_layer( + src_k_layers_device, + dst_k_layers_device, + src_v_layers_device, + dst_v_layers_device, + src_indices_device, + dst_indices_device, + item_size=item_size * dtype.itemsize, + num_layers=num_layers, + ) + transfer_kv_direct( + [src_k_pool[layer_id] for layer_id in range(num_layers)] + + [src_v_pool[layer_id] for layer_id in range(num_layers)], + [dst_k_pool_direct[layer_id] for layer_id in range(num_layers)] + + [dst_v_pool_direct[layer_id] for layer_id in range(num_layers)], + src_indices_host, + dst_indices_device, + page_size=page_size, + ) + torch.cuda.synchronize() + torch.testing.assert_close(dst_k_pool_kernel, dst_k_pool_ref) + torch.testing.assert_close(dst_v_pool_kernel, dst_v_pool_ref) + torch.testing.assert_close(dst_k_pool_direct, dst_k_pool_ref) + torch.testing.assert_close(dst_v_pool_direct, dst_v_pool_ref) + + torch.set_default_dtype(original_dtype) + + +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +@pytest.mark.parametrize("num_items_to_transfer", [128, 1024, 8192]) +@pytest.mark.parametrize("page_size", [16, 64, 128]) +@pytest.mark.parametrize("item_size", [256]) +@pytest.mark.parametrize("total_items_in_pool", [20480]) +@pytest.mark.parametrize("is_mla", [False, True]) +@pytest.mark.parametrize("lf_to_pf", [False, True]) +def test_transfer_kv_pf_direct( + dtype: torch.dtype, + num_items_to_transfer: int, + item_size: int, + page_size: int, + total_items_in_pool: int, + is_mla: bool, + lf_to_pf: bool, +): + original_dtype = torch.get_default_dtype() + torch.set_default_dtype(dtype) + device = "cuda" + torch.cuda.manual_seed(42) + + num_layers = 4 + + total_pages_in_pool = total_items_in_pool // page_size + num_pages_to_transfer = num_items_to_transfer // page_size + if num_pages_to_transfer == 0: + torch.set_default_dtype(original_dtype) + return + page_indices = torch.randperm(total_pages_in_pool, dtype=torch.int64) + src_indices_host = torch.cat( + [ + torch.arange(p * page_size, (p + 1) * page_size) + for p in page_indices[:num_pages_to_transfer] + ] + ) + src_indices_device = src_indices_host.to(device) + dst_indices_host = torch.cat( + [ + torch.arange(p * page_size, (p + 1) * page_size) + for p in page_indices[num_pages_to_transfer : 2 * num_pages_to_transfer] + ] + ) + dst_indices_device = dst_indices_host.to(device) + + # We will test the per-layer function on the first layer (index 0) of the pool. + layer_idx_to_test = 0 + + if lf_to_pf: + if is_mla: + src_pool = torch.randn(num_layers, total_items_in_pool, item_size).to( + device + ) + src_pool_ptrs = [src_pool[i] for i in range(num_layers)] + dst_pool_ref = torch.zeros( + total_pages_in_pool, num_layers, page_size, item_size + ).pin_memory() + dst_pool_direct = torch.zeros_like(dst_pool_ref) + torch.cuda.synchronize() + + transfer_kv_all_layer_direct_lf_pf( + src_pool_ptrs, + [dst_pool_direct], + src_indices_host, + dst_indices_host, + page_size, + ) + for i in range(num_layers): + ref_copy_with_indices_pf_direct( + src_pool, + dst_pool_ref, + src_indices_device, + dst_indices_host, + page_size, + i, + lf_to_pf=True, + ) + torch.cuda.synchronize() + torch.testing.assert_close(dst_pool_direct, dst_pool_ref) + + else: + src_k_pool = torch.randn(num_layers, total_items_in_pool, item_size).to( + device + ) + src_k_pool_ptrs = [src_k_pool[i] for i in range(num_layers)] + src_v_pool = torch.randn(num_layers, total_items_in_pool, item_size).to( + device + ) + src_v_pool_ptrs = [src_v_pool[i] for i in range(num_layers)] + dst_k_pool_ref = torch.zeros( + total_pages_in_pool, num_layers, page_size, item_size + ).pin_memory() + dst_v_pool_ref = torch.zeros_like(dst_k_pool_ref) + dst_k_pool_direct = torch.zeros_like(dst_k_pool_ref) + dst_v_pool_direct = torch.zeros_like(dst_v_pool_ref) + torch.cuda.synchronize() + + transfer_kv_all_layer_direct_lf_pf( + src_k_pool_ptrs + src_v_pool_ptrs, + [dst_k_pool_direct, dst_v_pool_direct], + src_indices_host, + dst_indices_host, + page_size, + ) + for i in range(num_layers): + ref_copy_with_indices_pf_direct( + src_k_pool, + dst_k_pool_ref, + src_indices_device, + dst_indices_host, + page_size, + i, + lf_to_pf=True, + ) + ref_copy_with_indices_pf_direct( + src_v_pool, + dst_v_pool_ref, + src_indices_device, + dst_indices_host, + page_size, + i, + lf_to_pf=True, + ) + torch.cuda.synchronize() + torch.testing.assert_close(dst_k_pool_direct, dst_k_pool_ref) + torch.testing.assert_close(dst_v_pool_direct, dst_v_pool_ref) + else: + if is_mla: + src_pool = torch.randn( + total_pages_in_pool, num_layers, page_size, item_size + ).pin_memory() + + dst_pool_ref = torch.zeros(num_layers, total_items_in_pool, item_size).to( + device + ) + dst_pool_direct = torch.zeros_like(dst_pool_ref) + dst_pool_direct_ptrs = [dst_pool_direct[i] for i in range(num_layers)] + torch.cuda.synchronize() + + transfer_kv_per_layer_direct_pf_lf( + [src_pool], + [dst_pool_direct_ptrs[layer_idx_to_test]], + src_indices_host, + dst_indices_host, + layer_idx_to_test, + page_size, + ) + ref_copy_with_indices_pf_direct( + src_pool, + dst_pool_ref, + src_indices_host, + dst_indices_device, + page_size, + layer_idx_to_test, + lf_to_pf=False, + ) + torch.cuda.synchronize() + torch.testing.assert_close(dst_pool_direct, dst_pool_ref) + else: + src_k_pool = torch.randn( + total_pages_in_pool, num_layers, page_size, item_size + ).pin_memory() + src_v_pool = torch.randn( + total_pages_in_pool, num_layers, page_size, item_size + ).pin_memory() + + dst_k_pool_ref = torch.zeros(num_layers, total_items_in_pool, item_size).to( + device + ) + dst_k_pool_direct = torch.zeros_like(dst_k_pool_ref) + dst_k_pool_direct_ptrs = [dst_k_pool_direct[i] for i in range(num_layers)] + + dst_v_pool_ref = torch.zeros_like(dst_k_pool_ref) + dst_v_pool_direct = torch.zeros_like(dst_v_pool_ref) + dst_v_pool_direct_ptrs = [dst_v_pool_direct[i] for i in range(num_layers)] + torch.cuda.synchronize() + + transfer_kv_per_layer_direct_pf_lf( + [src_k_pool, src_v_pool], + [ + dst_k_pool_direct_ptrs[layer_idx_to_test], + dst_v_pool_direct_ptrs[layer_idx_to_test], + ], + src_indices_host, + dst_indices_host, + layer_idx_to_test, + page_size, + ) + + ref_copy_with_indices_pf_direct( + src_k_pool, + dst_k_pool_ref, + src_indices_host, + dst_indices_device, + page_size, + layer_idx_to_test, + lf_to_pf=False, + ) + ref_copy_with_indices_pf_direct( + src_v_pool, + dst_v_pool_ref, + src_indices_host, + dst_indices_device, + page_size, + layer_idx_to_test, + lf_to_pf=False, + ) + + torch.cuda.synchronize() + torch.testing.assert_close(dst_k_pool_direct, dst_k_pool_ref) + torch.testing.assert_close(dst_v_pool_direct, dst_v_pool_ref) + torch.set_default_dtype(original_dtype) + + +@pytest.mark.skipif(is_hip(), reason="HIP is not supported for this test") +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +@pytest.mark.parametrize("num_items_to_transfer", [256, 1024]) +@pytest.mark.parametrize("page_size", [16, 64, 128]) +@pytest.mark.parametrize("item_size", [1024]) +@pytest.mark.parametrize("head_num", [8, 16]) +@pytest.mark.parametrize("total_items_in_pool", [4096]) +@pytest.mark.parametrize("lf_to_ph", [False, True]) +def test_transfer_kv_page_head( + dtype: torch.dtype, + num_items_to_transfer: int, + page_size: int, + item_size: int, + head_num: int, + total_items_in_pool: int, + lf_to_ph: bool, +): + original_dtype = torch.get_default_dtype() + torch.set_default_dtype(dtype) + device = "cuda" + torch.cuda.manual_seed(42) + + num_layers = 4 + + total_pages_in_pool = total_items_in_pool // page_size + num_pages_to_transfer = num_items_to_transfer // page_size + if num_pages_to_transfer == 0: + torch.set_default_dtype(original_dtype) + return + + assert item_size % head_num == 0 + head_dim = item_size // head_num + + page_indices = torch.randperm(total_pages_in_pool, dtype=torch.int64) + src_indices_host = torch.cat( + [ + torch.arange(p * page_size, (p + 1) * page_size) + for p in page_indices[:num_pages_to_transfer] + ] + ) + src_indices_device = src_indices_host.to(device) + dst_indices_host = torch.cat( + [ + torch.arange(p * page_size, (p + 1) * page_size) + for p in page_indices[num_pages_to_transfer : 2 * num_pages_to_transfer] + ] + ) + dst_indices_device = dst_indices_host.to(device) + + # We will test the per-layer function on the first layer (index 0) of the pool. + layer_idx_to_test = 0 + + if lf_to_ph: + src_k_pool = torch.randn( + num_layers, total_items_in_pool, head_num, head_dim + ).to(device) + src_v_pool = torch.randn( + num_layers, total_items_in_pool, head_num, head_dim + ).to(device) + src_k_pool_ptrs = [src_k_pool[i] for i in range(num_layers)] + src_k_pool_ptrs = torch.tensor( + [x.data_ptr() for x in src_k_pool_ptrs], + dtype=torch.uint64, + device=device, + ) + src_v_pool_ptrs = [src_v_pool[i] for i in range(num_layers)] + src_v_pool_ptrs = torch.tensor( + [x.data_ptr() for x in src_v_pool_ptrs], + dtype=torch.uint64, + device=device, + ) + + dst_k_pool_ref = torch.zeros( + total_pages_in_pool, head_num, page_size, num_layers, head_dim + ).pin_memory() + dst_v_pool_ref = torch.zeros_like(dst_k_pool_ref).pin_memory() + + dst_k_pool_kernel = torch.zeros_like(dst_k_pool_ref).pin_memory() + dst_v_pool_kernel = torch.zeros_like(dst_v_pool_ref).pin_memory() + torch.cuda.synchronize() + + transfer_kv_all_layer_lf_ph( + src_k_pool_ptrs, + dst_k_pool_kernel, + src_v_pool_ptrs, + dst_v_pool_kernel, + src_indices_device, + dst_indices_device, + item_size * dtype.itemsize, + item_size * num_layers * dtype.itemsize, + num_layers, + page_size, + head_num, + ) + torch.cuda.synchronize() + + for i in range(num_layers): + ref_copy_with_indices_page_head( + src_k_pool, + dst_k_pool_ref, + src_indices_device, + dst_indices_host, + page_size, + i, + head_num, + lf_to_ph=True, + ) + ref_copy_with_indices_page_head( + src_v_pool, + dst_v_pool_ref, + src_indices_device, + dst_indices_host, + page_size, + i, + head_num, + lf_to_ph=True, + ) + torch.cuda.synchronize() + torch.testing.assert_close(dst_k_pool_kernel, dst_k_pool_ref) + torch.testing.assert_close(dst_v_pool_kernel, dst_v_pool_ref) + else: + from sgl_kernel.kvcacheio import transfer_kv_per_layer_ph_lf + + src_k_pool = torch.randn( + total_pages_in_pool, head_num, page_size, num_layers, head_dim + ).pin_memory() + src_v_pool = torch.randn( + total_pages_in_pool, head_num, page_size, num_layers, head_dim + ).pin_memory() + + dst_k_pool_ref = torch.zeros( + num_layers, total_items_in_pool, head_num, head_dim + ).to(device) + dst_v_pool_ref = torch.zeros_like(dst_k_pool_ref) + dst_k_pool_kernel = torch.zeros_like(dst_k_pool_ref) + dst_v_pool_kernel = torch.zeros_like(dst_v_pool_ref) + dst_k_pool_kernel_ptrs = [dst_k_pool_kernel[i] for i in range(num_layers)] + dst_v_pool_kernel_ptrs = [dst_v_pool_kernel[i] for i in range(num_layers)] + torch.cuda.synchronize() + + transfer_kv_per_layer_ph_lf( + src_k_pool, + dst_k_pool_kernel_ptrs[layer_idx_to_test], + src_v_pool, + dst_v_pool_kernel_ptrs[layer_idx_to_test], + src_indices_device, + dst_indices_device, + layer_idx_to_test, + item_size * dtype.itemsize, + item_size * num_layers * dtype.itemsize, + page_size, + head_num, + ) + + ref_copy_with_indices_page_head( + src_k_pool, + dst_k_pool_ref, + src_indices_host, + dst_indices_device, + page_size, + layer_idx_to_test, + head_num, + lf_to_ph=False, + ) + ref_copy_with_indices_page_head( + src_v_pool, + dst_v_pool_ref, + src_indices_host, + dst_indices_device, + page_size, + layer_idx_to_test, + head_num, + lf_to_ph=False, + ) + torch.cuda.synchronize() + torch.testing.assert_close(dst_k_pool_kernel, dst_k_pool_ref) + torch.testing.assert_close(dst_v_pool_kernel, dst_v_pool_ref) + torch.set_default_dtype(original_dtype) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_marlin_gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_marlin_gemm.py new file mode 100644 index 00000000..9c4d48a3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_marlin_gemm.py @@ -0,0 +1,121 @@ +import pytest +import torch +from sgl_kernel import gptq_marlin_gemm +from sgl_kernel.scalar_type import scalar_types + +from sglang.srt.layers.quantization.marlin_utils import marlin_make_workspace +from sglang.test.test_marlin_utils import awq_marlin_quantize, marlin_quantize + +MNK_FACTORS = [ + (1, 1, 1), + (1, 4, 8), + (1, 7, 5), + (13, 17, 67), + (26, 37, 13), + (67, 13, 11), + (257, 13, 11), + (658, 13, 11), +] + + +# uint4 for awq +# uint4b8 for gptq +@pytest.mark.parametrize("k_chunk", [128]) +@pytest.mark.parametrize("n_chunk", [64, 256]) +@pytest.mark.parametrize("quant_type", [scalar_types.uint4, scalar_types.uint4b8]) +@pytest.mark.parametrize("group_size", [-1, 32, 64, 128]) +@pytest.mark.parametrize("mnk_factors", MNK_FACTORS) +@pytest.mark.parametrize("act_order", [False, True]) +@pytest.mark.parametrize("is_k_full", [False, True]) +@pytest.mark.parametrize("use_atomic_add", [False, True]) +@pytest.mark.parametrize("use_fp32_reduce", [False, True]) +def test_gptq_marlin_gemm( + k_chunk, + n_chunk, + quant_type, + group_size, + mnk_factors, + act_order, + is_k_full, + use_atomic_add, + use_fp32_reduce, +): + m_factor, n_factor, k_factor = mnk_factors + has_zp = quant_type in [scalar_types.uint4, scalar_types.uint8] + + size_m = m_factor + size_k = k_chunk * k_factor + size_n = n_chunk * n_factor + + if act_order: + if group_size == -1: + return + if group_size == size_k: + return + if has_zp: + return + + if size_k % group_size != 0: + return + + a_input = torch.randn((size_m, size_k), dtype=torch.float16, device="cuda") + b_weight = torch.randn((size_k, size_n), dtype=torch.float16, device="cuda") + + if has_zp: + # AWQ style, unsigned + runtime zero-point + if group_size == 16: + return + w_ref, marlin_q_w, marlin_s, marlin_zp = awq_marlin_quantize( + b_weight, quant_type, group_size + ) + g_idx = None + sort_indices = None + marlin_s2 = None + else: + # GPTQ style, unsigned + symmetric bias + if group_size == 16: + return + w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize( + b_weight, quant_type, group_size, act_order + ) + marlin_zp = None + marlin_s2 = None + + workspace = marlin_make_workspace(w_ref.device) + + # marlin gemm + output = gptq_marlin_gemm( + a_input, + None, + marlin_q_w, + marlin_s, + marlin_s2, + marlin_zp, + g_idx, + sort_indices, + workspace, + quant_type, + a_input.shape[0], + b_weight.shape[1], + a_input.shape[1], + is_k_full=is_k_full, + use_atomic_add=use_atomic_add, + use_fp32_reduce=use_fp32_reduce, + is_zp_float=False, + ) + # ref gemm + output_ref = torch.matmul(a_input, w_ref) + + torch.cuda.synchronize() + + max_diff = torch.mean(torch.abs(output - output_ref)) / torch.mean( + torch.abs(output_ref) + ) + + assert max_diff < 0.04 + + +if __name__ == "__main__": + import subprocess + + subprocess.call(["pytest", "--tb=short", str(__file__)]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_marlin_repack.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_marlin_repack.py new file mode 100644 index 00000000..50809910 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_marlin_repack.py @@ -0,0 +1,148 @@ +import numpy as np +import pytest +import torch +from sgl_kernel import awq_marlin_repack, gptq_marlin_repack +from sgl_kernel.scalar_type import scalar_types + +from sglang.srt.layers.quantization.utils import ( + gptq_quantize_weights, + pack_cols, + pack_rows, + quantize_weights, + sort_weights, +) +from sglang.test.test_marlin_utils import get_weight_perm, marlin_weights + +GPTQ_MARLIN_TILE = 16 +MARLIN_K_CHUNKS = [128] +MARLIN_N_CHUNKS = [64, 256] + +MNK_FACTORS = [ + (1, 1, 1), + (1, 4, 8), + (1, 7, 5), + (13, 17, 67), + (26, 37, 13), + (67, 13, 11), + (257, 13, 11), + (658, 13, 11), +] + + +def awq_pack( + q_w: torch.Tensor, + num_bits: int, + size_k: int, + size_n: int, +): + assert q_w.shape == (size_k, size_n) + + # Interleave column dim (for the dequantize code) and pack it to int32 + if num_bits == 4: + interleave = np.array([0, 2, 4, 6, 1, 3, 5, 7]) + elif num_bits == 8: + interleave = np.array([0, 2, 1, 3]) + else: + raise Exception("num_bits must be 4 or 8, got {}".format(num_bits)) + + q_w = q_w.reshape((-1, len(interleave)))[:, interleave].ravel() + q_w = q_w.reshape((-1, size_n)).contiguous() + + return pack_cols(q_w, num_bits, size_k, size_n) + + +@pytest.mark.parametrize("num_bits", [4, 8]) +@pytest.mark.parametrize("k_tiles,n_tiles", [(1, 1), (2, 2)]) +@pytest.mark.parametrize("group_size", [16, 32]) +def test_awq_marlin_repack_correct(num_bits, k_tiles, n_tiles, group_size): + tile_k, tile_n = 16, 64 + size_k = k_tiles * tile_k + size_n = n_tiles * tile_n + pack_factor = 32 // num_bits + + b_weight = torch.randn((size_k, size_n), dtype=torch.float16, device="cuda") + + w_ref, q_w, s, zp = quantize_weights( + b_weight, scalar_types.uint4, group_size, zero_points=True + ) + + q_w_awq = awq_pack(q_w, num_bits, size_k, size_n) + + weight_perm = get_weight_perm(num_bits) + q_w_marlin = marlin_weights(q_w, size_k, size_n, num_bits, weight_perm) + + out_gpu = awq_marlin_repack(q_w_awq, size_k, size_n, num_bits) + assert out_gpu.is_cuda and out_gpu.dtype == torch.int32 + + expected_cols = size_n * tile_k // pack_factor + assert list(out_gpu.shape) == [size_k // tile_k, expected_cols] + + torch.cuda.synchronize() + + torch.testing.assert_close(out_gpu, q_w_marlin) + + +@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS) +@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS) +@pytest.mark.parametrize("quant_type", [scalar_types.uint4b8]) +@pytest.mark.parametrize("group_size", [-1, 32, 64, 128]) +@pytest.mark.parametrize("act_order", [False, True]) +@pytest.mark.parametrize("mnk_factors", MNK_FACTORS) +def test_gptq_marlin_repack( + k_chunk, n_chunk, quant_type, group_size, act_order, mnk_factors +): + m_factor, n_factor, k_factor = mnk_factors + + size_k = k_chunk * k_factor + size_n = n_chunk * n_factor + + # Filter act_order + if act_order: + if group_size == -1: + return + if group_size == size_k: + return + + # Normalize group_size + if group_size == -1: + group_size = size_k + assert group_size <= size_k + + if size_k % group_size != 0: + pytest.skip("size_k must be divisible by group_size") + + # Create input + b_weight = torch.randn((size_k, size_n), dtype=torch.float16, device="cuda") + + # Quantize (and apply act_order if provided) + w_ref, q_w, s, g_idx, rand_perm = gptq_quantize_weights( + b_weight, quant_type, group_size, act_order + ) + + q_w_gptq = pack_rows(q_w, quant_type.size_bits, size_k, size_n) + + # For act_order, sort the "weights" and "g_idx" so that group ids are + # increasing + sort_indices = torch.empty(0, dtype=torch.int, device=b_weight.device) + if act_order: + q_w, g_idx, sort_indices = sort_weights(q_w, g_idx) + + marlin_layout_perm = get_weight_perm(quant_type.size_bits) + q_w_marlin_ref = marlin_weights( + q_w, size_k, size_n, quant_type.size_bits, marlin_layout_perm + ) + + # Run Marlin repack GPU kernel + q_w_marlin = gptq_marlin_repack( + q_w_gptq, sort_indices, size_k, size_n, quant_type.size_bits + ) + + torch.cuda.synchronize() + + torch.testing.assert_close(q_w_marlin, q_w_marlin_ref) + + +if __name__ == "__main__": + import subprocess + + subprocess.call(["pytest", "--tb=short", str(__file__)]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_merge_state.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_merge_state.py new file mode 100644 index 00000000..70b9628d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_merge_state.py @@ -0,0 +1,142 @@ +# Adapted from https://github.com/flashinfer-ai/flashinfer/blob/55576c626421b5ee7e7ebe74afd26465c8ae863f/flashinfer/triton/kernels/cascade.py + +from typing import List + +import pytest +import torch +import triton +import triton.language as tl +from sgl_kernel import merge_state + + +def check_input(x: torch.Tensor): + assert x.is_cuda, f"{str(x)} must be a CUDA Tensor" + assert x.is_contiguous(), f"{str(x)} must be contiguous" + + +def check_dim(d, x: torch.Tensor): + assert x.dim() == d, f"{str(x)} must be a {d}D tensor" + + +def check_shape(a: torch.Tensor, b: torch.Tensor): + assert a.dim() == b.dim(), "tensors should have same dim" + for i in range(a.dim()): + assert a.size(i) == b.size( + i + ), f"tensors shape mismatch, {a.size()} and {b.size()}" + + +def check_device(tensors: List[torch.Tensor]): + device = tensors[0].device + for t in tensors: + assert ( + t.device == device + ), f"All tensors should be on the same device, but got {device} and {t.device}" + + +@triton.jit +def state_merge(o, m, d, other_o, other_m, other_d): + m_max = tl.maximum(m, other_m) + d = d * tl.exp2(m - m_max) + other_d * tl.exp2(other_m - m_max) + o = o * tl.exp2(m - m_max) + other_o * tl.exp2(other_m - m_max) + return o, m_max, d + + +@triton.jit +def state_normalize(o, m, d): + o = o / d + return o, m, d + + +@triton.jit +def state_get_lse(o, m, d): + return m + tl.log2(d) + + +@triton.jit +def merge_state_kernel( + v_a_ptr, + s_a_ptr, + v_b_ptr, + s_b_ptr, + v_merged_ptr, + s_merged_ptr, + num_heads, + head_dim, + bdx: tl.constexpr, + bdy: tl.constexpr, +): + pos = tl.program_id(axis=0) + for tx in tl.range(bdx): + for head_idx in tl.range(bdy): + s_a_val = tl.load(s_a_ptr + pos * num_heads + head_idx) + s_b_val = tl.load(s_b_ptr + pos * num_heads + head_idx) + + offsets = (pos * num_heads + head_idx) * head_dim + tx + v_a = tl.load(v_a_ptr + offsets) + v_b = tl.load(v_b_ptr + offsets) + + v_merged, s_max, d = state_merge( + o=v_a, m=s_a_val, d=1, other_o=v_b, other_m=s_b_val, other_d=1 + ) + v_merged, s_max, d = state_normalize(v_merged, s_max, d) + v_merged_offset = (pos * num_heads + head_idx) * head_dim + tx + tl.store(v_merged_ptr + v_merged_offset, v_merged) + + if s_merged_ptr: + tl.store( + s_merged_ptr + pos * num_heads + head_idx, + tl.log2(d) + s_max, + ) + + +def merge_state_triton( + v_a: torch.Tensor, s_a: torch.Tensor, v_b: torch.Tensor, s_b: torch.Tensor +): + check_input(v_a) + check_input(s_a) + check_input(v_b) + check_input(s_b) + check_device([v_a, s_a, v_b, s_b]) + check_dim(3, v_a) + check_dim(2, s_a) + check_dim(3, v_b) + check_dim(2, s_b) + check_shape(v_a, v_b) + check_shape(s_a, s_b) + assert v_a.size(0) == s_a.size(0) + assert v_a.size(1) == s_b.size(1) + s_a = s_a.to(torch.float32) + s_b = s_b.to(torch.float32) + seq_len = v_a.size(0) + num_heads = v_a.size(1) + head_dim = v_a.size(2) + v_merged = torch.empty_like(v_a).to(s_a.device) + s_merged = torch.empty((seq_len, num_heads)).to(s_a.device) + bdx = head_dim + bdy = num_heads + + merge_state_kernel[lambda meta: (seq_len,)]( + v_a, s_a, v_b, s_b, v_merged, s_merged, num_heads, head_dim, bdx=bdx, bdy=bdy + ) + + return v_merged, s_merged + + +@pytest.mark.parametrize("seq_len", [2048]) +@pytest.mark.parametrize("num_heads", [32]) +@pytest.mark.parametrize("head_dim", [128]) +def test_merge_state(seq_len, num_heads, head_dim): + va = torch.randn(seq_len, num_heads, head_dim).half().to("cuda:0") + sa = torch.randn(seq_len, num_heads, dtype=torch.float32).to("cuda:0") + vb = torch.randn(seq_len, num_heads, head_dim).half().to("cuda:0") + sb = torch.randn(seq_len, num_heads, dtype=torch.float32).to("cuda:0") + v_merged, s_merged = merge_state_triton(va, sa, vb, sb) + v_merged_std, s_merged_std = merge_state(va, sa, vb, sb) + + assert torch.allclose(v_merged, v_merged_std, atol=1e-2) + assert torch.allclose(s_merged, s_merged_std, atol=1e-2) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_merge_state_v2.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_merge_state_v2.py new file mode 100644 index 00000000..62326f75 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_merge_state_v2.py @@ -0,0 +1,400 @@ +from typing import Optional + +import pytest +import torch +import triton +import triton.language as tl +from sgl_kernel import merge_state, merge_state_v2 + + +@triton.jit +def merge_state_kernel( + output, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] v_merged + output_lse, # [NUM_TOKENS, NUM_HEADS] s_merged + prefix_output, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] v_a + prefix_lse, # [NUM_TOKENS, NUM_HEADS] s_a + suffix_output, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] v_b + suffix_lse, # [NUM_TOKENS, NUM_HEADS] s_b + HEAD_SIZE: tl.constexpr, + PADDED_HEAD_SIZE: tl.constexpr, + OUTPUT_LSE: tl.constexpr, +): + token_idx = tl.program_id(0) + num_tokens = tl.num_programs(0) + head_idx = tl.program_id(1) + num_heads = tl.num_programs(1) + + p_lse = tl.load(prefix_lse + token_idx * num_heads + head_idx) + s_lse = tl.load(suffix_lse + token_idx * num_heads + head_idx) + p_lse = float("-inf") if p_lse == float("inf") else p_lse + s_lse = float("-inf") if s_lse == float("inf") else s_lse + + max_lse = tl.maximum(p_lse, s_lse) + p_lse = p_lse - max_lse + s_lse = s_lse - max_lse + out_se = tl.exp(p_lse) + tl.exp(s_lse) + + if OUTPUT_LSE: + out_lse = tl.log(out_se) + max_lse + tl.store(output_lse + token_idx * num_heads + head_idx, out_lse) + + head_arange = tl.arange(0, PADDED_HEAD_SIZE) + head_mask = head_arange < HEAD_SIZE + p_out = tl.load( + prefix_output + + token_idx * num_heads * HEAD_SIZE + + head_idx * HEAD_SIZE + + head_arange, + mask=head_mask, + ) + s_out = tl.load( + suffix_output + + token_idx * num_heads * HEAD_SIZE + + head_idx * HEAD_SIZE + + head_arange, + mask=head_mask, + ) + + p_scale = tl.exp(p_lse) / out_se + s_scale = tl.exp(s_lse) / out_se + out = p_out * p_scale + s_out * s_scale + tl.store( + output + token_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE + head_arange, + out, + mask=head_mask, + ) + + +def merge_state_triton( + prefix_output: torch.Tensor, + prefix_lse: torch.Tensor, + suffix_output: torch.Tensor, + suffix_lse: torch.Tensor, + output: Optional[torch.Tensor] = None, + output_lse: Optional[torch.Tensor] = None, +) -> None: + num_tokens = output.shape[0] + num_query_heads = output.shape[1] + head_size = output.shape[2] + padded_head_size = triton.next_power_of_2(head_size) + # Avoid creating new tensors if they are already provided + if output is None: + output = torch.empty_like(prefix_output) + if output_lse is None: + output_lse = torch.empty_like(prefix_lse) + + merge_state_kernel[(num_tokens, num_query_heads)]( + output, + output_lse, + prefix_output, + prefix_lse, + suffix_output, + suffix_lse, + head_size, + padded_head_size, + output_lse is not None, + ) + return output, output_lse + + +# Naive PyTorch Implements of Merge Attn States +def merge_state_torch( + prefix_output: torch.Tensor, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] + prefix_lse: torch.Tensor, # [NUM_TOKENS, NUM_HEADS] + suffix_output: torch.Tensor, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] + suffix_lse: torch.Tensor, # [NUM_TOKENS, NUM_HEADS] + output: Optional[torch.Tensor] = None, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] + output_lse: Optional[torch.Tensor] = None, # [NUM_TOKENS, NUM_HEADS] +): + # Avoid creating new tensors if they are already provided + if output is None: + output = torch.empty_like(prefix_output) + if output_lse is None: + output_lse = torch.empty_like(prefix_lse) + p_lse = prefix_lse + s_lse = suffix_lse + # inf -> -inf + p_lse[p_lse == torch.inf] = -torch.inf + s_lse[s_lse == torch.inf] = -torch.inf + # max_lse [NUM_HEADS, NUM_TOKENS] + max_lse = torch.maximum(p_lse, s_lse) + p_lse = p_lse - max_lse + s_lse = s_lse - max_lse + p_lse_exp = torch.exp(p_lse) + s_lse_exp = torch.exp(s_lse) + out_se = p_lse_exp + s_lse_exp + if output_lse is not None: + output_lse = torch.log(out_se) + max_lse + p_scale = p_lse_exp / out_se + s_scale = s_lse_exp / out_se + p_scale = p_scale.unsqueeze(2) # [NUM_TOKENS, NUM_HEADS, 1] + s_scale = s_scale.unsqueeze(2) # [NUM_TOKENS, NUM_HEADS, 1] + output = prefix_output * p_scale + suffix_output * s_scale + return output, output_lse + + +NUM_BATCH_TOKENS = [256, 512, 613, 1024, 1536] +NUM_QUERY_HEADS = [8, 16, 32] +HEAD_SIZES = [32, 48, 64, 128, 256] +DTYPES = [torch.half, torch.bfloat16] + +all_case_info: list[tuple] = [] + + +def generate_markdown_table(): + global all_case_info + table_header = ( + "| tokens | heads | headsize | dtype " + "| device | torch | triton | v1 | v2 | speedup(vs triton) | speedup(vs v1)|" + ) + table_separator = ( + "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |" + ) + + def shortly_dtype(dtype: torch.dtype) -> str: + return str(dtype).removeprefix("torch.") + + def shortly_device(device: str) -> str: + return device.removeprefix("NVIDIA").strip() + + print(table_header) + print(table_separator) + for info in all_case_info: + ( + num_tokens, + num_heads, + head_size, + dtype, + device, + time_torch, + time_triton, + time_v1, + time_v2, + ) = info + dtype = shortly_dtype(dtype) + device = shortly_device(device) + improved_triton = time_triton / time_v2 + improved_v1 = time_v1 / time_v2 + print( + f"| {num_tokens} | {num_heads} | {head_size} " + f"| {dtype} | {device} | {time_torch:.4f}ms " + f"| {time_triton:.4f}ms " + f"| {time_v1:.4f}ms " + f"| {time_v2:.4f}ms " + f"| {improved_triton:.4f}x " + f"| {improved_v1:.4f}x |" + ) + + +@pytest.mark.parametrize("num_tokens", NUM_BATCH_TOKENS) +@pytest.mark.parametrize("num_query_heads", NUM_QUERY_HEADS) +@pytest.mark.parametrize("head_size", HEAD_SIZES) +@pytest.mark.parametrize("output_dtype", DTYPES) +@torch.inference_mode() +def test_merge_attn_states( + num_tokens: int, num_query_heads: int, head_size: int, output_dtype: torch.dtype +): + if not torch.cuda.is_available(): + pytest.skip( + "Currently only support compare triton merge_attn_states " + "with custom cuda merge_attn_states kernel" + ) + + NUM_TOKENS = num_tokens + NUM_HEADS = num_query_heads + HEAD_SIZE = head_size + + print( + f"\nNUM_TOKENS:{NUM_TOKENS}, NUM_HEADS:{NUM_HEADS}, " + f"HEAD_SIZE:{HEAD_SIZE}, DTYPE: {output_dtype}, " + f"Device: {torch.cuda.get_device_name()}" + ) + + # prefix_lse and suffix_lse contain inf and normal values + prefix_lse = torch.randn(NUM_TOKENS, NUM_HEADS, dtype=torch.float32, device="cuda") + suffix_lse = torch.randn(NUM_TOKENS, NUM_HEADS, dtype=torch.float32, device="cuda") + + # Generate boolean masks + mask_prefix = torch.rand(NUM_TOKENS, NUM_HEADS) < 0.1 + mask_suffix = torch.rand(NUM_TOKENS, NUM_HEADS) < 0.1 + # Ensure that the same position is not True at the same time + combined_mask = torch.logical_and(mask_prefix, mask_suffix) + mask_prefix = torch.logical_and(mask_prefix, ~combined_mask) + mask_suffix = torch.logical_and(mask_suffix, ~combined_mask) + + prefix_lse[mask_prefix] = float("inf") + suffix_lse[mask_suffix] = float("inf") + + # Other input tensors (need to be initialized but + # no actual calculation needed) + output = torch.zeros( + (NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=output_dtype, device="cuda" + ) + output_lse = torch.zeros( + (NUM_TOKENS, NUM_HEADS), dtype=torch.float32, device="cuda" + ) + prefix_output = torch.randn( + (NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=output_dtype, device="cuda" + ) + suffix_output = torch.randn( + (NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=output_dtype, device="cuda" + ) + + warmup_times = 2 + repeat_times = 20 + + def perf_kernel_fn( + output_fn: torch.Tensor, + output_lse_fn: torch.Tensor, + kernel_fn: callable, + fn_type: str = "torch", + ): + # Avoid inplace inf -> -inf, we have to use prefix_lse + # and suffix_lse for other kernel. + if fn_type == "torch": + prefix_lse_ = prefix_lse.clone() + suffix_lse_ = suffix_lse.clone() + else: + prefix_lse_ = prefix_lse + suffix_lse_ = suffix_lse + + if fn_type == "cuda_v1": + # merge_state v1 kernel not support float32 + if output_dtype not in (torch.half, torch.bfloat16): + return 0, output_fn, output_lse_fn + + total_time = 0 + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + + try: + for _ in range(warmup_times): + output_fn, output_lse_fn = kernel_fn( + prefix_output, + prefix_lse_, + suffix_output, + suffix_lse_, + output_fn, + output_lse_fn, + ) + torch.cuda.synchronize() + + for _ in range(repeat_times): + start.record() + output_fn, output_lse_fn = kernel_fn( + prefix_output, + prefix_lse_, + suffix_output, + suffix_lse_, + output_fn, + output_lse_fn, + ) + end.record() + torch.cuda.synchronize() + total_time += start.elapsed_time(end) + + avg_time = total_time / repeat_times + return avg_time, output_fn, output_lse_fn + except Exception as e: + return 0, output_fn, output_lse_fn + + # 0. Run the Torch kernel + output_torch = output.clone() + output_lse_torch = output_lse.clone() + time_torch, output_torch, output_lse_torch = perf_kernel_fn( + output_torch, output_lse_torch, merge_state_torch, fn_type="torch" + ) + + # 1. Run the Triton kernel + output_ref_triton = output.clone() + output_lse_ref_triton = output_lse.clone() + time_triton, output_ref_triton, output_lse_ref_triton = perf_kernel_fn( + output_ref_triton, + output_lse_ref_triton, + merge_state_triton, + fn_type="triton", + ) + + # 2. Run the merge_state V1 kernel + output_v1 = output.clone() + output_lse_v1 = output_lse.clone() + time_v1, output_v1, output_lse_v1 = perf_kernel_fn( + output_v1, output_lse_v1, merge_state, fn_type="cuda_v1" + ) + + # 3. Run the merge_state V2 kernel + output_v2 = output.clone() + output_lse_v2 = output_lse.clone() + time_v2, output_v2, output_lse_v2 = perf_kernel_fn( + output_v2, output_lse_v2, merge_state_v2, fn_type="cuda_v2" + ) + + # 4. Performance compare + improved = time_triton / time_v2 + print(f" Torch time: {time_torch:.6f}ms") + print(f" Triton time: {time_triton:.6f}ms") + print(f"CUDA v1 time: {time_v1:.6f}ms") + print(f"CUDA v2 time: {time_v2:.6f}ms, Performance: {improved:.5f}x") + print("-" * 100) + + # 5. Correctness compare + # Liger Kernel: Efficient Triton Kernels for LLM Training + # https://arxiv.org/pdf/2410.10989, 3.3 Correctness + # use rtol = 1e-2 for bfloat16. + rtol = 1e-2 if output_dtype == torch.bfloat16 else 1e-3 + + def diff(a: torch.Tensor, b: torch.Tensor): + max_diff = torch.max(torch.abs(a.float() - b.float())) + return max_diff + + # Use Triton output as reference because we want to replace + # the Triton kernel with custom CUDA kernel for merge attn + # states operation. + output_ref = output_ref_triton + output_lse_ref = output_lse_ref_triton + torch.testing.assert_close( + output_v2.float(), output_ref.float(), atol=1e-3, rtol=rtol + ) + print("Output all match, max abs diff:") + print(f"(Triton vs Torch) : {diff(output_torch, output_ref)}") + print(f"(CUDA v2 vs Torch) : {diff(output_torch, output_v2)}") + print(f"(CUDA v2 vs Triton): {diff(output_ref, output_v2)}") + print("-" * 100) + + torch.testing.assert_close( + output_lse_v2.float(), output_lse_ref.float(), atol=1e-3, rtol=rtol + ) + print("Output LSE all match, max abs diff:") + print(f"(Triton vs Torch) : {diff(output_lse_torch, output_lse_ref)}") + print(f"(CUDA v2 vs Torch) : {diff(output_lse_torch, output_lse_v2)}") + print(f"(CUDA v2 vs Triton): {diff(output_lse_ref, output_lse_v2)}") + print("-" * 100) + + print( + "All output values test passed! All inf values " + "are correctly replaced with -inf." + ) + print("-" * 100) + + device = torch.cuda.get_device_name() + all_case_info.append( + ( + NUM_TOKENS, + NUM_HEADS, + HEAD_SIZE, + output_dtype, + device, + time_torch, + time_triton, + time_v1, + time_v2, + ) + ) + if len(all_case_info) == ( + len(NUM_BATCH_TOKENS) * len(HEAD_SIZES) * len(NUM_QUERY_HEADS) * len(DTYPES) + ): + generate_markdown_table() + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_moe_align.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_moe_align.py new file mode 100644 index 00000000..6a27f6e7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_moe_align.py @@ -0,0 +1,274 @@ +import itertools + +import pytest +import torch +import triton +import triton.language as tl +from sgl_kernel import moe_align_block_size, moe_sum + + +def is_hip() -> bool: + return torch.version.hip is not None + + +_is_hip = is_hip() + + +def ceil_div(a, b): + return (a + b - 1) // b + + +@triton.jit +def moe_align_block_size_stage1( + topk_ids_ptr, + tokens_cnts_ptr, + num_experts: tl.constexpr, + numel: tl.constexpr, + tokens_per_thread: tl.constexpr, +): + pid = tl.program_id(0) + start_idx = pid * tokens_per_thread + off_c = (pid + 1) * num_experts + + for i in range(tokens_per_thread): + if start_idx + i < numel: + idx = tl.load(topk_ids_ptr + start_idx + i) + token_cnt = tl.load(tokens_cnts_ptr + off_c + idx) + tl.store(tokens_cnts_ptr + off_c + idx, token_cnt + 1) + + +@triton.jit +def moe_align_block_size_stage2( + tokens_cnts_ptr, + num_experts: tl.constexpr, +): + pid = tl.program_id(0) + last_cnt = 0 + for i in range(1, num_experts + 1): + token_cnt = tl.load(tokens_cnts_ptr + i * num_experts + pid) + last_cnt = last_cnt + token_cnt + tl.store(tokens_cnts_ptr + i * num_experts + pid, last_cnt) + + +@triton.jit +def moe_align_block_size_stage3( + total_tokens_post_pad_ptr, + tokens_cnts_ptr, + cumsum_ptr, + num_experts: tl.constexpr, + block_size: tl.constexpr, +): + last_cumsum = 0 + off_cnt = num_experts * num_experts + for i in range(1, num_experts + 1): + token_cnt = tl.load(tokens_cnts_ptr + off_cnt + i - 1) + last_cumsum = last_cumsum + tl.cdiv(token_cnt, block_size) * block_size + tl.store(cumsum_ptr + i, last_cumsum) + tl.store(total_tokens_post_pad_ptr, last_cumsum) + + +@triton.jit +def moe_align_block_size_stage4( + topk_ids_ptr, + sorted_token_ids_ptr, + expert_ids_ptr, + tokens_cnts_ptr, + cumsum_ptr, + num_experts: tl.constexpr, + block_size: tl.constexpr, + numel: tl.constexpr, + tokens_per_thread: tl.constexpr, +): + pid = tl.program_id(0) + start_idx = tl.load(cumsum_ptr + pid) + end_idx = tl.load(cumsum_ptr + pid + 1) + + for i in range(start_idx, end_idx, block_size): + tl.store(expert_ids_ptr + i // block_size, pid) + + start_idx = pid * tokens_per_thread + off_t = pid * num_experts + + for i in range(start_idx, tl.minimum(start_idx + tokens_per_thread, numel)): + expert_id = tl.load(topk_ids_ptr + i) + token_cnt = tl.load(tokens_cnts_ptr + off_t + expert_id) + rank_post_pad = token_cnt + tl.load(cumsum_ptr + expert_id) + tl.store(sorted_token_ids_ptr + rank_post_pad, i) + tl.store(tokens_cnts_ptr + off_t + expert_id, token_cnt + 1) + + +def moe_align_block_size_triton( + topk_ids: torch.Tensor, + num_experts: int, + block_size: int, + sorted_token_ids: torch.Tensor, + expert_ids: torch.Tensor, + num_tokens_post_pad: torch.Tensor, +) -> None: + numel = topk_ids.numel() + grid = (num_experts,) + tokens_cnts = torch.zeros( + (num_experts + 1, num_experts), dtype=torch.int32, device=topk_ids.device + ) + cumsum = torch.zeros((num_experts + 1,), dtype=torch.int32, device=topk_ids.device) + tokens_per_thread = ceil_div(numel, num_experts) + + moe_align_block_size_stage1[grid]( + topk_ids, + tokens_cnts, + num_experts, + numel, + tokens_per_thread, + ) + moe_align_block_size_stage2[grid]( + tokens_cnts, + num_experts, + ) + moe_align_block_size_stage3[(1,)]( + num_tokens_post_pad, + tokens_cnts, + cumsum, + num_experts, + block_size, + ) + moe_align_block_size_stage4[grid]( + topk_ids, + sorted_token_ids, + expert_ids, + tokens_cnts, + cumsum, + num_experts, + block_size, + numel, + tokens_per_thread, + ) + + +@pytest.mark.parametrize( + "block_size,num_tokens,topk,num_experts,pad_sorted_token_ids", + list( + itertools.product( + [32, 64, 128, 256], # block_size + [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096], # num_tokens + [1, 2, 4, 8, 16, 32, 64], # topk + [64, 160, 256, 257, 260, 264], # num_experts + [True, False], # pad_sorted_token_ids + ) + ), +) +def test_moe_align_block_size_compare_implementations( + block_size, num_tokens, topk, num_experts, pad_sorted_token_ids +): + + topk_ids = torch.argsort(torch.rand(num_tokens, num_experts, device="cuda"), dim=1)[ + :, :topk + ] + + max_num_tokens_padded = topk_ids.numel() + (num_experts + 1) * (block_size - 1) + if topk_ids.numel() < num_experts + 1: + max_num_tokens_padded = topk_ids.numel() * block_size + + sorted_ids_cuda = torch.empty( + (max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device + ) + if not pad_sorted_token_ids: + sorted_ids_cuda.fill_(topk_ids.numel()) + max_num_m_blocks = max_num_tokens_padded // block_size + expert_ids_cuda = torch.zeros( + (max_num_m_blocks,), dtype=torch.int32, device=topk_ids.device + ) + num_tokens_post_pad_cuda = torch.empty( + (1), dtype=torch.int32, device=topk_ids.device + ) + cumsum_buffer = torch.empty( + num_experts + 2, dtype=torch.int32, device=topk_ids.device + ) + + sorted_ids_triton = torch.empty_like(sorted_ids_cuda) + sorted_ids_triton.fill_(topk_ids.numel()) + expert_ids_triton = torch.zeros_like(expert_ids_cuda) + num_tokens_post_pad_triton = torch.empty_like(num_tokens_post_pad_cuda) + + moe_align_block_size( + topk_ids, + num_experts + 1, + block_size, + sorted_ids_cuda, + expert_ids_cuda, + num_tokens_post_pad_cuda, + cumsum_buffer, + pad_sorted_token_ids, + ) + + moe_align_block_size_triton( + topk_ids, + num_experts + 1, + block_size, + sorted_ids_triton, + expert_ids_triton, + num_tokens_post_pad_triton, + ) + + assert torch.allclose(expert_ids_cuda, expert_ids_triton, atol=0, rtol=0), ( + f"Expert IDs mismatch for block_size={block_size}, " + f"num_tokens={num_tokens}, topk={topk}\n" + f"CUDA expert_ids: {expert_ids_cuda}\n" + f"Triton expert_ids: {expert_ids_triton}" + ) + + assert torch.allclose( + num_tokens_post_pad_cuda, num_tokens_post_pad_triton, atol=0, rtol=0 + ), ( + f"Num tokens post pad mismatch for block_size={block_size}, " + f"num_tokens={num_tokens}, topk={topk}\n" + f"CUDA num_tokens_post_pad: {num_tokens_post_pad_cuda}\n" + f"Triton num_tokens_post_pad: {num_tokens_post_pad_triton}" + ) + + # Select an expert to check + expert_idx = expert_ids_cuda.max().item() + + # Get the first and last block id where expert_ids_cuda == expert_idx + matching_indices = torch.where(expert_ids_cuda == expert_idx)[0] + block_sorted_start = matching_indices[0].item() * block_size + block_sorted_end = min( + (matching_indices[-1].item() + 1) * block_size, num_tokens_post_pad_cuda.item() + ) + + selected_sorted_ids_cuda = sorted_ids_cuda[ + block_sorted_start:block_sorted_end + ].sort()[0] + selected_sorted_ids_triton = sorted_ids_triton[ + block_sorted_start:block_sorted_end + ].sort()[0] + + assert torch.allclose( + selected_sorted_ids_cuda, + selected_sorted_ids_triton, + atol=0, + rtol=0, + ), ( + f"Sorted IDs mismatch for block_size={block_size}, " + f"num_tokens={num_tokens}, topk={topk}\n" + f"CUDA sorted_ids: {selected_sorted_ids_cuda}\n" + f"Triton sorted_ids: {selected_sorted_ids_triton}" + ) + + +@pytest.mark.parametrize("m", [1, 33, 64, 222]) +@pytest.mark.parametrize("topk", [2, 6]) +@pytest.mark.parametrize("k", [128, 511, 1024]) +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +@pytest.mark.skipif(_is_hip, reason="Skip for AMD GPU") +def test_moe_sum(m: int, topk: int, k: int, dtype: torch.dtype): + input = torch.randn((m, topk, k), device="cuda", dtype=dtype) + actual = torch.empty((m, k), device="cuda", dtype=dtype) + + expected = input.sum(dim=1) + moe_sum(input, actual) + + torch.testing.assert_close(actual, expected, atol=2e-2, rtol=0) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_moe_fused_gate.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_moe_fused_gate.py new file mode 100644 index 00000000..98389575 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_moe_fused_gate.py @@ -0,0 +1,103 @@ +import pytest +import torch +from sgl_kernel import moe_fused_gate + +from sglang.srt.layers.moe.topk import biased_grouped_topk + + +@pytest.mark.parametrize( + "seq_length", + list(range(1, 10)) + + [16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536], +) +@pytest.mark.parametrize( + "params", + [ + (128, 4, 2, 4), + (256, 8, 4, 8), # deepseek v3 + (512, 16, 8, 16), + ], +) +@pytest.mark.parametrize("num_fused_shared_experts", [0, 1, 2]) +@pytest.mark.parametrize("apply_routed_scaling_factor_on_output", [False, True]) +def test_moe_fused_gate_combined( + seq_length, params, num_fused_shared_experts, apply_routed_scaling_factor_on_output +): + num_experts, num_expert_group, topk_group, topk = params + dtype = torch.float32 + + torch.manual_seed(seq_length) + tensor = torch.rand((seq_length, num_experts), dtype=dtype, device="cuda") + scores = tensor.clone() + bias = torch.rand(num_experts, dtype=dtype, device="cuda") + topk = topk + num_fused_shared_experts + + output, indices = moe_fused_gate( + tensor, + bias, + num_expert_group=num_expert_group, + topk_group=topk_group, + topk=topk, + num_fused_shared_experts=num_fused_shared_experts, + routed_scaling_factor=2.5, + apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output, + ) + ref_output, ref_indices = biased_grouped_topk( + scores, + scores, + bias, + topk=topk, + renormalize=True, + num_expert_group=num_expert_group, + topk_group=topk_group, + num_fused_shared_experts=num_fused_shared_experts, + routed_scaling_factor=2.5, + apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output, + ) + + # When num_fused_shared_experts > 0, ignore the comparison of the last topk dimension + if num_fused_shared_experts > 0: + original_indices = indices.clone() + original_ref_indices = ref_indices.clone() + + indices = indices[:, :-1] + ref_indices = ref_indices[:, :-1] + + valid_min = num_experts + valid_max = num_experts + num_fused_shared_experts + shared_indices = original_indices[:, -1] + shared_ref_indices = original_ref_indices[:, -1] + if shared_indices is not None: + assert torch.all( + (shared_indices >= valid_min) & (shared_indices < valid_max) + ), f"Shared expert indices out of range: found values outside [{valid_min}, {valid_max})" + if shared_ref_indices is not None: + assert torch.all( + (shared_ref_indices >= valid_min) & (shared_ref_indices < valid_max) + ), f"Shared expert reference indices out of range: found values outside [{valid_min}, {valid_max})" + + idx_check = torch.allclose( + ref_indices.sort()[0].to(torch.int32), + indices.sort()[0].to(torch.int32), + rtol=1e-04, + atol=1e-05, + ) + output_check = torch.allclose( + ref_output.sort()[0].to(torch.float32), + output.sort()[0].to(torch.float32), + rtol=1e-02, + atol=1e-03, + ) + + assert idx_check, ( + f"Indices mismatch at seq_length {seq_length}, dtype {dtype}, " + f"params {params}, num_fused_shared_experts {num_fused_shared_experts}" + ) + assert output_check, ( + f"Output mismatch at seq_length {seq_length}, dtype {dtype}, " + f"params {params}, num_fused_shared_experts {num_fused_shared_experts}" + ) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_moe_topk_sigmoid.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_moe_topk_sigmoid.py new file mode 100644 index 00000000..45b8222a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_moe_topk_sigmoid.py @@ -0,0 +1,183 @@ +import itertools + +import pytest +import torch +from sgl_kernel import topk_sigmoid + + +@pytest.mark.parametrize( + "num_tokens, num_experts, topk", + list( + itertools.product( + [1, 16, 128, 512, 1024, 2048], # num_tokens + [4, 8, 16, 32, 64, 128, 256], # num_experts + [1, 2, 4], # topk + ) + ), +) +def test_topk_sigmoid(num_tokens, num_experts, topk): + gating_output = torch.randn( + (num_tokens, num_experts), dtype=torch.float32, device="cuda" + ) + + topk_weights = torch.empty((num_tokens, topk), dtype=torch.float32, device="cuda") + topk_indices = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda") + + topk_sigmoid( + topk_weights, + topk_indices, + gating_output, + ) + + # Native torch implementation + sigmoid_output = torch.sigmoid(gating_output) + topk_weights_ref, topk_indices_ref = torch.topk(sigmoid_output, topk, dim=-1) + + # Verify the top-k weights and indices match the torch native ones + assert torch.allclose( + topk_weights_ref, topk_weights, atol=1e-3, rtol=1e-3 + ), f"Weights mismatch: torch={topk_weights_ref} vs SGLang={topk_weights}" + + assert torch.allclose( + topk_indices_ref.int(), topk_indices, atol=0, rtol=0 + ), f"Indices mismatch: torch={topk_indices_ref}, SGLang={topk_indices}" + + +@pytest.mark.parametrize( + "num_tokens, num_experts, topk, dtype", + list( + itertools.product( + [1, 16, 128, 512, 1024, 2048], # num_tokens + [4, 8, 16, 32, 64, 128, 256], # num_experts + [1, 2, 4], # topk + [torch.float16, torch.bfloat16, torch.float32], # dtype + ) + ), +) +def test_topk_sigmoid_dtype_regression(num_tokens, num_experts, topk, dtype): + gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda") + + topk_weights = torch.empty((num_tokens, topk), dtype=torch.float32, device="cuda") + topk_indices = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda") + + topk_sigmoid( + topk_weights, + topk_indices, + gating_output, + ) + + topk_weights_ref = torch.empty( + (num_tokens, topk), dtype=torch.float32, device="cuda" + ) + topk_indices_ref = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda") + + topk_sigmoid( + topk_weights_ref, + topk_indices_ref, + gating_output.float(), + ) + + assert torch.allclose( + topk_weights_ref, topk_weights, atol=1e-3, rtol=1e-3 + ), f"Weights mismatch: SGLang old interface={topk_weights_ref} vs SGLang new interface={topk_weights}" + + assert torch.allclose( + topk_indices_ref.int(), topk_indices, atol=0, rtol=0 + ), f"Indices mismatch: SGLang old interface={topk_indices_ref}, SGLang new interface={topk_indices}" + + +@pytest.mark.parametrize( + "num_tokens, num_experts, topk", + list( + itertools.product( + [1, 16, 128, 512, 1024, 2048], # num_tokens + [4, 8, 16, 32, 64, 128, 256], # num_experts + [1, 2, 4], # topk + ) + ), +) +def test_topk_sigmoid_renormalize(num_tokens, num_experts, topk): + gating_output = torch.randn( + (num_tokens, num_experts), dtype=torch.bfloat16, device="cuda" + ) + + topk_weights = torch.empty((num_tokens, topk), dtype=torch.float32, device="cuda") + topk_indices = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda") + + topk_sigmoid( + topk_weights, + topk_indices, + gating_output, + renormalize=True, + ) + + topk_weights_ref = torch.empty( + (num_tokens, topk), dtype=torch.float32, device="cuda" + ) + topk_indices_ref = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda") + token_expert_indices_ref = torch.empty( + (num_tokens, topk), dtype=torch.int32, device="cuda" + ) + + topk_sigmoid( + topk_weights_ref, + topk_indices_ref, + gating_output, + ) + topk_weights_ref = topk_weights_ref / topk_weights_ref.sum(dim=-1, keepdim=True) + + assert torch.allclose( + topk_weights_ref, topk_weights, atol=1e-3, rtol=1e-3 + ), f"Weights mismatch: SGLang w/o fused renormalize={topk_weights_ref} vs SGLang w/ fused renormalize={topk_weights}" + + assert torch.allclose( + topk_indices_ref.int(), topk_indices, atol=0, rtol=0 + ), f"Indices mismatch: SGLang w/o fused renormalize={topk_indices_ref}, SGLang w/ fused renormalize={topk_indices}" + + +@pytest.mark.parametrize( + "num_tokens, num_experts, topk", + list( + itertools.product( + [1, 16, 128, 512, 1024, 2048], # num_tokens + [4, 8, 16, 32, 48, 64, 128, 256], # num_experts + [1, 2, 4], # topk + ) + ), +) +def test_topk_sigmoid_renormalize_correction_bias(num_tokens, num_experts, topk): + gating_output = torch.randn( + (num_tokens, num_experts), dtype=torch.float32, device="cuda" + ) + correction_bias = torch.randn((num_experts), dtype=torch.float32, device="cuda") + + topk_weights = torch.empty((num_tokens, topk), dtype=torch.float32, device="cuda") + topk_indices = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda") + + topk_sigmoid( + topk_weights, + topk_indices, + gating_output, + renormalize=True, + correction_bias=correction_bias, + ) + + # Native torch implementation + sigmoid_output = torch.sigmoid(gating_output) + sigmoid_scores = sigmoid_output.view(-1, num_experts) + correction_bias.unsqueeze(0) + _, topk_indices_ref = torch.topk(sigmoid_scores, k=topk, dim=-1) + topk_weights_ref = sigmoid_output.gather(1, topk_indices_ref) + topk_weights_ref = topk_weights_ref / topk_weights_ref.sum(dim=-1, keepdim=True) + + # Verify the top-k weights and indices match the torch native ones + assert torch.allclose( + topk_weights_ref, topk_weights, atol=1e-3, rtol=1e-3 + ), f"Weights mismatch: torch={topk_weights_ref} vs SGLang={topk_weights}" + + assert torch.allclose( + topk_indices_ref.int(), topk_indices, atol=0, rtol=0 + ), f"Indices mismatch: torch={topk_indices_ref}, SGLang={topk_indices}" + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_moe_topk_softmax.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_moe_topk_softmax.py new file mode 100644 index 00000000..b9a802c5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_moe_topk_softmax.py @@ -0,0 +1,139 @@ +import itertools + +import pytest +import torch +from sgl_kernel import topk_softmax + + +@pytest.mark.parametrize( + "num_tokens, num_experts, topk", + list( + itertools.product( + [1, 16, 128, 512, 1024, 2048], # num_tokens + [4, 8, 16, 32, 64, 128, 256], # num_experts + [1, 2, 4], # topk + ) + ), +) +def test_topk_softmax(num_tokens, num_experts, topk): + gating_output = torch.randn( + (num_tokens, num_experts), dtype=torch.float32, device="cuda" + ) + + topk_weights = torch.empty((num_tokens, topk), dtype=torch.float32, device="cuda") + topk_indices = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda") + + topk_softmax( + topk_weights, + topk_indices, + gating_output, + ) + + # Native torch implementation + softmax_output = torch.softmax(gating_output, dim=-1) + topk_weights_ref, topk_indices_ref = torch.topk(softmax_output, topk, dim=-1) + + # Verify the top-k weights and indices match the torch native ones + assert torch.allclose( + topk_weights_ref, topk_weights, atol=1e-3, rtol=1e-3 + ), f"Weights mismatch: torch={topk_indices_ref} vs SGLang={topk_weights}" + + assert torch.allclose( + topk_indices_ref.int(), topk_indices, atol=0, rtol=0 + ), f"Indices mismatch: torch={topk_indices_ref}, SGLang={topk_indices}" + + +@pytest.mark.parametrize( + "num_tokens, num_experts, topk, dtype", + list( + itertools.product( + [1, 16, 128, 512, 1024, 2048], # num_tokens + [4, 8, 16, 32, 64, 128, 256], # num_experts + [1, 2, 4], # topk + [torch.float16, torch.bfloat16, torch.float32], # dtype + ) + ), +) +def test_topk_softmax_dtype_regression(num_tokens, num_experts, topk, dtype): + gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda") + + topk_weights = torch.empty((num_tokens, topk), dtype=torch.float32, device="cuda") + topk_indices = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda") + + topk_softmax( + topk_weights, + topk_indices, + gating_output, + ) + + topk_weights_ref = torch.empty( + (num_tokens, topk), dtype=torch.float32, device="cuda" + ) + topk_indices_ref = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda") + + topk_softmax( + topk_weights_ref, + topk_indices_ref, + gating_output.float(), + ) + + assert torch.allclose( + topk_weights_ref, topk_weights, atol=1e-3, rtol=1e-3 + ), f"Weights mismatch: SGLang old interface={topk_indices_ref} vs SGLang new interface={topk_weights}" + + assert torch.allclose( + topk_indices_ref.int(), topk_indices, atol=0, rtol=0 + ), f"Indices mismatch: SGLang old interface={topk_indices_ref}, SGLang new interface={topk_indices}" + + +@pytest.mark.parametrize( + "num_tokens, num_experts, topk", + list( + itertools.product( + [1, 16, 128, 512, 1024, 2048], # num_tokens + [4, 8, 16, 32, 64, 128, 256], # num_experts + [1, 2, 4], # topk + ) + ), +) +def test_topk_softmax_renormalize(num_tokens, num_experts, topk): + gating_output = torch.randn( + (num_tokens, num_experts), dtype=torch.bfloat16, device="cuda" + ) + + topk_weights = torch.empty((num_tokens, topk), dtype=torch.float32, device="cuda") + topk_indices = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda") + + topk_softmax( + topk_weights, + topk_indices, + gating_output, + renormalize=True, + ) + + topk_weights_ref = torch.empty( + (num_tokens, topk), dtype=torch.float32, device="cuda" + ) + topk_indices_ref = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda") + token_expert_indices_ref = torch.empty( + (num_tokens, topk), dtype=torch.int32, device="cuda" + ) + + topk_softmax( + topk_weights_ref, + topk_indices_ref, + gating_output, + ) + topk_weights_ref = topk_weights_ref / topk_weights_ref.sum(dim=-1, keepdim=True) + + assert torch.allclose( + topk_weights_ref, topk_weights, atol=1e-3, rtol=1e-3 + ), f"Weights mismatch: SGLang w/o fused renormalize={topk_indices_ref} vs SGLang w/ fused renormalize={topk_weights}" + + assert torch.allclose( + topk_indices_ref.int(), topk_indices, atol=0, rtol=0 + ), f"Indices mismatch: SGLang w/o fused renormalize={topk_indices_ref}, SGLang w/ fused renormalize={topk_indices}" + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_mscclpp.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_mscclpp.py new file mode 100644 index 00000000..0a4332bd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_mscclpp.py @@ -0,0 +1,146 @@ +import multiprocessing as mp +import os +import socket +import unittest +from enum import IntEnum +from typing import Any + +import sgl_kernel.allreduce as custom_ops +import torch +import torch.distributed as dist + + +class MscclContextSelection(IntEnum): + MSCCL1SHOT1NODELL = 1 + MSCCL1SHOT2NODELL = 2 + + +def _run_correctness_worker(world_size, rank, distributed_init_port, test_sizes): + device = torch.device(f"cuda:{rank % torch.cuda.device_count()}") + torch.cuda.set_device(device) + distributed_init_method = f"tcp://localhost:{distributed_init_port}" + dist.init_process_group( + backend="nccl", + init_method=distributed_init_method, + rank=rank, + world_size=world_size, + ) + group = dist.group.WORLD + cpu_group = torch.distributed.new_group(list(range(world_size)), backend="gloo") + if rank == 0: + unique_id = [custom_ops.mscclpp_generate_unique_id()] + else: + unique_id = [None] + dist.broadcast_object_list( + unique_id, src=0, device=torch.device("cpu"), group=cpu_group + ) + unique_id = unique_id[0] + rank_to_node, rank_to_ib = list(range(world_size)), list(range(world_size)) + for r in range(world_size): + rank_to_node[r] = r // 8 + rank_to_ib[r] = rank % 8 + MAX_BYTES = 2**20 + scratch = torch.empty( + MAX_BYTES * 8, dtype=torch.bfloat16, device=torch.cuda.current_device() + ) + put_buffer = torch.empty( + MAX_BYTES, dtype=torch.bfloat16, device=torch.cuda.current_device() + ) + print(f"[{rank}] start mscclpp_context init") + nranks_per_node = torch.cuda.device_count() + selection = int(MscclContextSelection.MSCCL1SHOT1NODELL) + mscclpp_context = custom_ops.mscclpp_init_context( + unique_id, + rank, + world_size, + scratch, + put_buffer, + nranks_per_node, + rank_to_node, + rank_to_ib, + selection, + ) + try: + test_loop = 10 + for sz in test_sizes: + for dtype in [torch.float32, torch.float16, torch.bfloat16]: + if sz * dtype.itemsize > MAX_BYTES: + continue + if rank == 0: + print(f"mscclpp allreduce test sz {sz}, dtype {dtype}") + for _ in range(test_loop): + inp1 = torch.randint(1, 16, (sz,), dtype=dtype, device=device) + inp1_ref = inp1.clone() + out1 = torch.empty_like(inp1) + custom_ops.mscclpp_allreduce( + mscclpp_context, inp1, out1, nthreads=512, nblocks=21 + ) + dist.all_reduce(inp1_ref, group=group) + torch.testing.assert_close(out1, inp1_ref) + finally: + dist.barrier(group=group) + dist.destroy_process_group(group=group) + + +def get_open_port() -> int: + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + except OSError: + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: + s.bind(("::1", 0)) + return s.getsockname()[1] + + +def multi_process_parallel( + world_size: int, test_target: Any, target_args: tuple = () +) -> None: + mp.set_start_method("spawn", force=True) + + procs = [] + distributed_init_port = get_open_port() + for i in range(world_size): + proc_args = (world_size, i, distributed_init_port) + target_args + proc = mp.Process(target=test_target, args=proc_args, name=f"Worker-{i}") + proc.start() + procs.append(proc) + + for i in range(world_size): + procs[i].join() + assert ( + procs[i].exitcode == 0 + ), f"Process {i} failed with exit code {procs[i].exitcode}" + + +class TestMSCCLAllReduce(unittest.TestCase): + test_sizes = [ + 512, + 2560, + 4096, + 5120, + 7680, + 32768, + 262144, + 524288, + ] + world_sizes = [8] + + def test_correctness(self): + for world_size in self.world_sizes: + available_gpus = torch.cuda.device_count() + if world_size > available_gpus: + print( + f"Skipping world_size={world_size}, found {available_gpus} and now ray is not supported here" + ) + continue + + print(f"Running test for world_size={world_size}") + multi_process_parallel( + world_size, _run_correctness_worker, target_args=(self.test_sizes,) + ) + print(f"custom allreduce tp = {world_size}: OK") + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_norm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_norm.py new file mode 100644 index 00000000..ed61663e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_norm.py @@ -0,0 +1,142 @@ +# Adapted from https://github.com/flashinfer-ai/flashinfer/blob/4e8eb1879f9c3ba6d75511e5893183bf8f289a62/tests/test_norm.py + +import pytest +import sgl_kernel +import torch +from sgl_kernel.utils import is_arch_support_pdl + + +def llama_rms_norm(x, w, eps=1e-6): + orig_dtype = x.dtype + x = x.float() + variance = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + eps) + x = x * w.float() + x = x.to(orig_dtype) + return x + + +def gemma_rms_norm(x, w, eps=1e-6): + orig_dtype = x.dtype + x = x.float() + variance = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + eps) + x = x * (1.0 + w.float()) + x = x.to(orig_dtype) + return x + + +def gemma_fused_add_rms_norm(x, residual, w, eps=1e-6): + orig_dtype = x.dtype + x = x + residual + residual = x + x = x.float() + variance = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + eps) + x = x * (1.0 + w.float()) + x = x.to(orig_dtype) + return x, residual + + +def fused_add_rms_norm(x, residual, weight, eps): + orig_dtype = x.dtype + x = x.to(torch.float32) + x = x + residual.to(torch.float32) + residual = x.to(orig_dtype) + + variance = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + eps) + x = (x * weight.float()).to(orig_dtype) + return x, residual + + +@pytest.mark.parametrize("batch_size", [1, 19, 99, 989]) +@pytest.mark.parametrize("hidden_size", [111, 500, 1024, 3072, 3584, 4096, 8192, 16384]) +@pytest.mark.parametrize("dtype", [torch.float16]) +@pytest.mark.parametrize("specify_out", [True, False]) +def test_norm(batch_size, hidden_size, dtype, specify_out): + x = torch.randn(batch_size, hidden_size).to(0).to(dtype) + w = torch.randn(hidden_size).to(0).to(dtype) + + y_ref = llama_rms_norm(x, w) + enable_pdl = is_arch_support_pdl() + if specify_out: + y = torch.empty_like(x) + sgl_kernel.rmsnorm(x, w, out=y, enable_pdl=enable_pdl) + else: + y = sgl_kernel.rmsnorm(x, w, enable_pdl=enable_pdl) + + torch.testing.assert_close(y_ref, y, rtol=1e-3, atol=1e-3) + + +@pytest.mark.parametrize("batch_size", [1, 19, 99, 989]) +@pytest.mark.parametrize("hidden_size", [111, 500, 1024, 3072, 3584, 4096, 8192, 16384]) +@pytest.mark.parametrize("dtype", [torch.float16, torch.float32]) +def test_fused_add_rmsnorm(batch_size, hidden_size, dtype): + eps = 1e-6 + + x = torch.randn(batch_size, hidden_size, dtype=dtype, device="cuda") + residual = torch.randn_like(x) + weight = torch.randn(hidden_size, dtype=dtype, device="cuda") + + x_native, residual_native = fused_add_rms_norm( + x.clone(), residual.clone(), weight, eps + ) + + x_fused = x.clone() + residual_fused = residual.clone() + enable_pdl = is_arch_support_pdl() + sgl_kernel.fused_add_rmsnorm( + x_fused, residual_fused, weight, eps, enable_pdl=enable_pdl + ) + + torch.testing.assert_close(x_fused, x_native, rtol=1e-3, atol=1e-3) + torch.testing.assert_close(residual_fused, residual_native, rtol=1e-3, atol=1e-3) + + +@pytest.mark.parametrize("batch_size", [1, 19, 99, 989]) +@pytest.mark.parametrize("hidden_size", [111, 500, 1024, 3072, 3584, 4096, 8192, 16384]) +@pytest.mark.parametrize("dtype", [torch.float16]) +@pytest.mark.parametrize("specify_out", [True, False]) +def test_gemma_norm(batch_size, hidden_size, dtype, specify_out): + x = torch.randn(batch_size, hidden_size).to(0).to(dtype) + w = torch.randn(hidden_size).to(0).to(dtype) + + y_ref = gemma_rms_norm(x, w) + enable_pdl = is_arch_support_pdl() + if specify_out: + y = torch.empty_like(x) + sgl_kernel.gemma_rmsnorm(x, w, out=y, enable_pdl=enable_pdl) + else: + y = sgl_kernel.gemma_rmsnorm(x, w, enable_pdl=enable_pdl) + + torch.testing.assert_close(y_ref, y, rtol=1e-3, atol=1e-3) + + +@pytest.mark.parametrize("batch_size", [1, 19, 99, 989]) +@pytest.mark.parametrize("hidden_size", [111, 500, 1024, 3072, 3584, 4096, 8192, 16384]) +@pytest.mark.parametrize("dtype", [torch.float16]) +def test_gemma_fused_add_rmsnorm(batch_size, hidden_size, dtype): + eps = 1e-6 + + x = torch.randn(batch_size, hidden_size, dtype=dtype, device="cuda") + residual = torch.randn_like(x) + weight = torch.randn(hidden_size, dtype=dtype, device="cuda") + + x_native, residual_native = gemma_fused_add_rms_norm( + x.clone(), residual.clone(), weight, eps + ) + + x_fused = x.clone() + residual_fused = residual.clone() + enable_pdl = is_arch_support_pdl() + sgl_kernel.gemma_fused_add_rmsnorm( + x_fused, residual_fused, weight, eps, enable_pdl=enable_pdl + ) + + torch.testing.assert_close(x_fused, x_native, rtol=1e-3, atol=1e-3) + torch.testing.assert_close(residual_fused, residual_native, rtol=1e-3, atol=1e-3) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_per_tensor_quant_fp8.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_per_tensor_quant_fp8.py new file mode 100644 index 00000000..0840f298 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_per_tensor_quant_fp8.py @@ -0,0 +1,67 @@ +import itertools +from typing import Optional, Tuple + +import pytest +import torch +from sgl_kernel import sgl_per_tensor_quant_fp8 + +from sglang.srt.utils import is_hip + +_is_hip = is_hip() +fp8_type_ = torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn + + +def sglang_scaled_fp8_quant( + input: torch.Tensor, + scale: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + fp8_type_: torch.dtype = torch.float8_e4m3fn + output = torch.empty_like(input, device=input.device, dtype=fp8_type_) + is_static = True + if scale is None: + scale = torch.zeros(1, device=input.device, dtype=torch.float32) + is_static = False + sgl_per_tensor_quant_fp8(input, output, scale, is_static) + + return output, scale + + +def torch_scaled_fp8_quant(tensor, inv_scale): + # The reference implementation that fully aligns to + # the kernel being tested. + finfo = torch.finfo(torch.float8_e4m3fn) + scale = inv_scale.reciprocal() + qweight = (tensor.to(torch.float32) * scale).clamp(min=finfo.min, max=finfo.max) + qweight = qweight.to(torch.float8_e4m3fn) + return qweight + + +@pytest.mark.parametrize( + "num_tokens,hidden_dim", + list(itertools.product([128, 256, 512], [512, 2048, 4096])), +) +def test_per_tensor_quant_compare_implementations( + num_tokens: int, + hidden_dim: int, +): + device = torch.device("cuda") + x = torch.rand((num_tokens, hidden_dim), dtype=torch.float16, device=device) + + sglang_out, sglang_scale = sglang_scaled_fp8_quant(x) + torch_out = torch_scaled_fp8_quant(x, sglang_scale) + + torch.testing.assert_close( + sglang_out.float(), torch_out.float(), rtol=1e-3, atol=1e-3 + ) + + scale = torch.rand(1, dtype=torch.float32, device=device) + sglang_out, sglang_scale = sglang_scaled_fp8_quant(x, scale) + torch_out = torch_scaled_fp8_quant(x, scale) + + torch.testing.assert_close( + sglang_out.float(), torch_out.float(), rtol=1e-3, atol=1e-3 + ) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_per_token_group_quant_8bit.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_per_token_group_quant_8bit.py new file mode 100644 index 00000000..1f19af3f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_per_token_group_quant_8bit.py @@ -0,0 +1,183 @@ +import itertools +import os +import time +from pathlib import Path + +import pytest +import torch +from sgl_kernel.test_utils import ( + assert_all_close_or_tiny_diff, + create_per_token_group_quant_test_data, +) + +from sglang.srt.layers.quantization.fp8_kernel import ( + per_token_group_quant_8bit as triton_per_token_group_quant_8bit, +) +from sglang.srt.layers.quantization.fp8_kernel import sglang_per_token_group_quant_8bit +from sglang.srt.utils import get_bool_env_var, is_hip + +_is_hip = is_hip() +fp8_type_ = torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn + +configs = list( + itertools.product( + [1, 4, 16, 64, 127, 128, 512, 1024, 4096, 8192], # num_tokens + [128, 256, 384, 512, 1024, 1536, 1664, 2048, 4096, 7168, 16384], # hidden_dim + [16, 32, 64, 128], # group_size + [None], # num_ranks + [fp8_type_, torch.int8], # dtype + [ + dict( + column_major_scales=False, + scale_tma_aligned=False, + scale_ue8m0=False, + fuse_silu_and_mul=False, + masked_layout_mode=None, + ), + dict( + column_major_scales=True, + scale_tma_aligned=False, + scale_ue8m0=False, + fuse_silu_and_mul=False, + masked_layout_mode=None, + ), + dict( + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=False, + fuse_silu_and_mul=False, + masked_layout_mode=None, + ), + dict( + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + fuse_silu_and_mul=False, + masked_layout_mode=None, + ), + ], + ) +) + list( + itertools.product( + [1, 4, 1 * 8, 4 * 8, 64 * 8, 256 * 8, 768 * 8], + # TODO support more + [2048], + [128], + [8, 16, 32, 48], + [fp8_type_], + [ + dict( + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + fuse_silu_and_mul=True, + masked_layout_mode=None, + ), + dict( + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + fuse_silu_and_mul=True, + masked_layout_mode="balanced", + ), + dict( + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + fuse_silu_and_mul=True, + masked_layout_mode="imbalanced", + ), + dict( + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + fuse_silu_and_mul=True, + masked_layout_mode="extreme", + ), + ], + ) +) + + +@pytest.mark.parametrize( + "num_tokens, hidden_dim, group_size, num_ranks, dst_dtype, flags", configs +) +def test_per_token_group_quant_with_column_major( + num_tokens, + hidden_dim, + group_size, + num_ranks, + dst_dtype, + flags, +): + print( + f"{num_tokens=} {hidden_dim=} {group_size=} {num_ranks=} {dst_dtype=} {flags=}" + ) + + arch_major, _ = torch.cuda.get_device_capability(torch.cuda.current_device()) + if flags["scale_ue8m0"] and (arch_major <= 9): + pytest.skip("Only Blackwell need ue8m0 fusion") + return + + if (flags["scale_ue8m0"] and (group_size != 128)) or ( + (dst_dtype == torch.int8) and flags["column_major_scales"] + ): + pytest.skip() + return + + x, masked_m = create_per_token_group_quant_test_data( + num_tokens=num_tokens, hidden_dim=hidden_dim, num_ranks=num_ranks, flags=flags + ) + + # print("hack data!!!") + # x = torch.full_like(x, fill_value=100) + + execute_kwargs = dict( + x=x, + masked_m=masked_m, + group_size=group_size, + eps=1e-10, + dst_dtype=dst_dtype, + **{k: v for k, v in flags.items() if k not in ["masked_layout_mode"]}, + ) + + def _postprocess(x_q, x_s): + if masked_m is not None: + print(f"Mask tokens after {masked_m} to be zero") + for i in range(len(masked_m)): + x_q[i, masked_m[i] :, :] = 0 + x_s[i, masked_m[i] :, :] = 0 + return x_q, x_s + + x_q_triton, x_s_triton = _postprocess( + *triton_per_token_group_quant_8bit(**execute_kwargs) + ) + x_q_sglang, x_s_sglang = _postprocess( + *sglang_per_token_group_quant_8bit(**execute_kwargs, enable_v2=True) + ) + + try: + assert_all_close_or_tiny_diff(x_q_triton, x_q_sglang) + torch.testing.assert_close( + x_s_triton.contiguous(), + x_s_sglang.contiguous(), + rtol=1e-3, + atol=1e-5, + msg=lambda message: message + f" {x_s_triton=} {x_s_sglang=}", + ) + except AssertionError: + print( + f"{x.shape=} {x_q_triton.shape=} {x_s_triton.shape=} {x_q_sglang.shape=} {x_s_sglang.shape=}" + ) + print(f"{x=}") + print(f"{masked_m=}") + print(f"{x_q_triton=}") + print(f"{x_s_triton=}") + print(f"{x_q_sglang=}") + print(f"{x_s_sglang=}") + + raise + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_per_token_quant_fp8.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_per_token_quant_fp8.py new file mode 100644 index 00000000..4e1f8a11 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_per_token_quant_fp8.py @@ -0,0 +1,57 @@ +import itertools +from typing import Optional, Tuple + +import pytest +import torch +from sgl_kernel import sgl_per_token_quant_fp8 + +from sglang.srt.utils import is_hip + +_is_hip = is_hip() +fp8_type_ = torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn + + +def torch_per_token_quant_fp8(tensor, inv_scale): + # The reference implementation that fully aligns to + # the kernel being tested. + finfo = torch.finfo(torch.float8_e4m3fn) + inv_scale = inv_scale.view(-1, 1) + scale = inv_scale.reciprocal() + qweight = (tensor.to(torch.float32) * scale).clamp(min=finfo.min, max=finfo.max) + qweight = qweight.to(torch.float8_e4m3fn) + return qweight + + +def sglang_per_token_quant_fp8( + input: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + scale = torch.zeros(input.size(0), device=input.device, dtype=torch.float32) + output = torch.empty_like(input, device=input.device, dtype=fp8_type_) + + sgl_per_token_quant_fp8(input, output, scale) + scale = scale.reshape(-1, 1) + + return output, scale + + +@pytest.mark.parametrize( + "num_tokens,hidden_dim", + list(itertools.product([128, 256, 512], [512, 1076, 1368, 2048, 4096])), +) +def test_per_token_quant_compare_implementations( + num_tokens: int, + hidden_dim: int, +): + device = torch.device("cuda") + x = torch.rand((num_tokens, hidden_dim), dtype=torch.float16, device=device) + + sglang_out, sglang_scale = sglang_per_token_quant_fp8(x) + torch_out = torch_per_token_quant_fp8(x, sglang_scale) + + torch.testing.assert_close( + sglang_out.float(), torch_out.float(), rtol=1e-3, atol=1e-3 + ) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_qserve_w4a8_per_chn_gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_qserve_w4a8_per_chn_gemm.py new file mode 100644 index 00000000..9410710d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_qserve_w4a8_per_chn_gemm.py @@ -0,0 +1,118 @@ +import pytest +import torch +from sgl_kernel import qserve_w4a8_per_chn_gemm + + +# Adapted from https://github.com/mit-han-lab/omniserve/blob/main/omniserve/modeling/layers/quantized_linear/w4a8_linear.py +def convert_to_qserve_format(qweight, scale, zero): + assert qweight.min() >= 0 and qweight.max() <= 15, "Quantized weight out of range" + in_features = qweight.shape[1] + out_features = qweight.shape[0] + assert in_features % 32 == 0, "Input features must be divisible by 32" + assert out_features % 32 == 0, "Output features must be divisible by 32" + + # ---- Repack the weight ---- # + # pack to M // 32, K // 32, (8, 4), ([2], 2, 2, 4) + qweight_unpack_reorder = ( + qweight.reshape( + out_features // 32, + 2, + 2, + 8, + in_features // 32, + 2, + 4, + 4, + ) + .permute(0, 4, 3, 6, 1, 5, 2, 7) + .contiguous() + ) + qweight_unpack_reorder = ( + qweight_unpack_reorder.permute(0, 1, 2, 3, 5, 6, 7, 4) + .contiguous() + .to(torch.int8) + ) + # B_fp16_reorder = B_fp16_reorder[:, :, :, :, :, :, [3, 2, 1, 0]].contiguous() + # [16, 0, 17, 1, ...] + qweight_unpack_repacked = ( + qweight_unpack_reorder[..., 1] << 4 + ) + qweight_unpack_reorder[..., 0] + qweight_unpack_repacked = qweight_unpack_repacked.reshape( + out_features // 32, in_features // 32, 32, 16 + ) + qweight_unpack_repacked = qweight_unpack_repacked.reshape( + out_features, in_features // 2 + ).contiguous() + + # ---- Pack the scales ---- # + scale = scale.reshape(out_features).to(torch.float16).contiguous() + szero = zero.reshape(out_features).to(torch.float16).contiguous() * scale + + return qweight_unpack_repacked, scale, szero + + +# INT4 Quantization +def asym_quantize_tensor(tensor): + tensor_min = tensor.min(dim=-1, keepdim=True)[0] + tensor_max = tensor.max(dim=-1, keepdim=True)[0] + q_min = 0 + q_max = 15 + tensor_scale = (tensor_max - tensor_min) / (q_max - q_min) + tensor_zero = q_min - torch.round(tensor_min / tensor_scale) + tensor_q = torch.clamp( + torch.round(tensor / tensor_scale) + tensor_zero, q_min, q_max + ).to(torch.int8) + return tensor_q, tensor_scale.to(torch.float16), tensor_zero.to(torch.int8) + + +# INT8 Quantization +def sym_quantize_tensor(tensor): + tensor_scale = tensor.abs().max(dim=-1, keepdim=True)[0] / 127 + tensor_q = torch.clamp(torch.round(tensor / tensor_scale), -128, 127).to(torch.int8) + return tensor_q, tensor_scale.to(torch.float16) + + +def torch_w4a8_per_chn_gemm(a, b, a_scale, b_scale, b_zero, out_dtype): + print(a.shape) + print(b.shape) + print(b_zero.shape) + o = torch.matmul( + a.to(torch.float16), (b.to(torch.float16) - b_zero.to(torch.float16)).t() + ) + o = o * a_scale.view(-1, 1) * b_scale.view(1, -1) + return o.to(out_dtype) + + +def _test_accuracy_once(M, N, K, out_dtype, device): + # to avoid overflow, multiply 0.01 + a = torch.randn((M, K), device=device, dtype=torch.float32) * 0.01 + b = torch.randn((N, K), device=device, dtype=torch.float32) * 0.01 + + # symmetric quantize a + a_q, a_scale = sym_quantize_tensor(a) + # asymmetric quantize b + b_q, b_scale, b_zero = asym_quantize_tensor(b) + # convert to qserve format + b_q_format, b_scale_format, b_szero_format = convert_to_qserve_format( + b_q, b_scale, b_zero + ) + + # cal sum of every row of a + a_sum = a.sum(dim=-1, keepdim=True).to(torch.float16) + out = qserve_w4a8_per_chn_gemm( + a_q, b_q_format, b_scale_format, a_scale, b_szero_format, a_sum + ) + ref_out = torch_w4a8_per_chn_gemm(a_q, b_q, a_scale, b_scale, b_zero, out_dtype) + torch.testing.assert_close(out, ref_out, rtol=1e-3, atol=1e-2) + + +@pytest.mark.parametrize("M", [1, 16, 32, 64, 128, 512, 1024, 4096, 8192]) +@pytest.mark.parametrize("N", [128, 512, 1024, 4096, 8192, 16384]) +@pytest.mark.parametrize("K", [512, 1024, 4096, 8192, 16384]) +@pytest.mark.parametrize("out_dtype", [torch.float16]) +def test_accuracy(M, N, K, out_dtype): + _test_accuracy_once(M, N, K, out_dtype, "cuda") + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_qserve_w4a8_per_group_gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_qserve_w4a8_per_group_gemm.py new file mode 100644 index 00000000..fc26a2e6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_qserve_w4a8_per_group_gemm.py @@ -0,0 +1,183 @@ +import pytest +import torch +from sgl_kernel import qserve_w4a8_per_group_gemm + + +# Adapted from https://github.com/mit-han-lab/omniserve/blob/main/omniserve/modeling/layers/quantized_linear/w4a8_linear.py +def convert_to_qserve_format(qweight, chn_scale, scale_i8, zero_i8, group_size): + assert qweight.min() >= 0 and qweight.max() <= 15, "Quantized weight out of range" + in_features = qweight.shape[1] + out_features = qweight.shape[0] + assert in_features % 32 == 0, "Input features must be divisible by 32" + assert out_features % 32 == 0, "Output features must be divisible by 32" + assert group_size == 128, "Group size must be 128" + assert ( + in_features % group_size == 0 + ), "Input features must be divisible by group_size" + + # ---- Repack the weight ---- # + # pack to M // 32, K // 32, (8, 4), ([2], 2, 2, 4) + qweight_unpack_reorder = ( + qweight.reshape( + out_features // 32, + 2, + 2, + 8, + in_features // 32, + 2, + 4, + 4, + ) + .permute(0, 4, 3, 6, 1, 5, 2, 7) + .contiguous() + ) + qweight_unpack_reorder = ( + qweight_unpack_reorder.permute(0, 1, 2, 3, 5, 6, 7, 4) + .contiguous() + .to(torch.int8) + ) + # B_fp16_reorder = B_fp16_reorder[:, :, :, :, :, :, [3, 2, 1, 0]].contiguous() + # [16, 0, 17, 1, ...] + qweigth_unpack_repacked = ( + qweight_unpack_reorder[..., 1] << 4 + ) + qweight_unpack_reorder[..., 0] + qweigth_unpack_repacked = qweigth_unpack_repacked.reshape( + out_features // 32, in_features // 32, 32, 16 + ) + qweigth_unpack_repacked = qweigth_unpack_repacked.reshape( + out_features, in_features // 2 + ) + + # ---- Pack the scales ---- # + chn_scale = chn_scale.reshape(out_features) + + scale_i8 = ( + scale_i8.reshape(out_features, in_features // group_size) + .transpose(0, 1) + .contiguous() + ) + scale_i8 = scale_i8.reshape(in_features // group_size, out_features // 32, 32) + scale_i8 = ( + scale_i8.reshape(in_features // group_size, out_features // 32, 4, 8) + .transpose(-2, -1) + .contiguous() + ) + scale_i8 = scale_i8.reshape(in_features // group_size, out_features).contiguous() + + # ---- Pack the zeros ---- # + zero_i8 = -zero_i8 + # zero_i8 = zero_i8.int() # convert to 2-complement + + zero_i8 = ( + zero_i8.reshape(out_features, in_features // group_size) + .transpose(0, 1) + .contiguous() + ) + zero_i8 = zero_i8.reshape(in_features // group_size, out_features // 32, 32) + # for the last dimension, organize as 0, 8, 16, 24, 1, 9, 17, 25, ... following the requirement of tensor core gemm + zero_i8 = ( + zero_i8.reshape(in_features // group_size, out_features // 32, 4, 8) + .transpose(-2, -1) + .contiguous() + ) + zero_i8 = ( + zero_i8.reshape(in_features // group_size, out_features).contiguous() * scale_i8 + ) + + return qweigth_unpack_repacked, chn_scale, scale_i8, zero_i8 + + +# Progressive Group INT4 Quantization +def progressive_group_quantize_tensor(tensor, group_size): + assert group_size == 128, "Group size must be 128" + assert ( + tensor.shape[-1] % group_size == 0 + ), "Input features must be divisible by group_size" + # Channel scale + # NOTE(HandH1998): use protective quantization range + chn_scale = tensor.abs().max(dim=-1, keepdim=True)[0] / 119 + tensor_i8 = torch.clamp(torch.round(tensor / chn_scale), -119, 119) + + # Group scale + tensor_i8 = tensor_i8.reshape(-1, group_size) + tensor_i8_min = tensor_i8.min(dim=-1, keepdim=True)[0] + tensor_i8_max = tensor_i8.max(dim=-1, keepdim=True)[0] + q_min = 0 + q_max = 15 + scale_i8 = torch.round((tensor_i8_max - tensor_i8_min) / (q_max - q_min)) + zero_i8 = q_min - torch.round(tensor_i8_min / scale_i8) + tensor_q = ( + torch.clamp(torch.round(tensor_i8 / scale_i8) + zero_i8, q_min, q_max) + .reshape(tensor.shape[0], -1) + .to(torch.int8) + ) + return ( + tensor_q, + chn_scale.to(torch.float16), + scale_i8.reshape(tensor.shape[0], -1).to(torch.int8), + zero_i8.reshape(tensor.shape[0], -1).to(torch.int8), + ) + + +# INT8 Quantization +def sym_quantize_tensor(tensor): + tensor_scale = tensor.abs().max(dim=-1, keepdim=True)[0] / 127 + tensor_q = torch.clamp(torch.round(tensor / tensor_scale), -128, 127).to(torch.int8) + return tensor_q, tensor_scale.to(torch.float16) + + +def torch_w4a8_per_group_gemm( + a, b, a_scale, b_chn_scale, b_scale_i8, b_zero_i8, group_size, out_dtype +): + assert group_size == 128, "Group size must be 128" + b_dq = ( + b.reshape(-1, group_size).to(torch.float32) + - b_zero_i8.reshape(-1, 1).to(torch.float32) + ) * b_scale_i8.reshape(-1, 1).to(torch.float32) + b_dq = b_dq.reshape(b.shape[0], b.shape[1]) + o = torch.matmul(a.to(torch.float32), b_dq.t()) + o = o * a_scale.view(-1, 1) * b_chn_scale.view(1, -1) + return o.to(out_dtype) + + +def _test_accuracy_once(M, N, K, group_size, out_dtype, device): + # to avoid overflow, multiply 0.01 + a = torch.randn((M, K), device=device, dtype=torch.float32) * 0.01 + b = torch.randn((N, K), device=device, dtype=torch.float32) * 0.01 + + # symmetric quantize a + a_q, a_scale = sym_quantize_tensor(a) + # asymmetric quantize b + b_q, b_chn_scale, b_scale_i8, b_zero_i8 = progressive_group_quantize_tensor( + b, group_size + ) + # convert to qserve format + b_q_format, b_chn_scale_format, b_scale_i8_format, b_zero_i8_format = ( + convert_to_qserve_format(b_q, b_chn_scale, b_scale_i8, b_zero_i8, group_size) + ) + + out = qserve_w4a8_per_group_gemm( + a_q, + b_q_format, + b_zero_i8_format, + b_scale_i8_format, + b_chn_scale_format, + a_scale, + ) + ref_out = torch_w4a8_per_group_gemm( + a_q, b_q, a_scale, b_chn_scale, b_scale_i8, b_zero_i8, group_size, out_dtype + ) + torch.testing.assert_close(out, ref_out, rtol=1e-3, atol=1e-5) + + +@pytest.mark.parametrize("M", [1, 16, 32, 64, 128, 512, 1024, 4096, 8192]) +@pytest.mark.parametrize("N", [128, 512, 1024, 4096, 8192, 16384]) +@pytest.mark.parametrize("K", [512, 1024, 4096, 8192, 16384]) +@pytest.mark.parametrize("group_size", [128]) +@pytest.mark.parametrize("out_dtype", [torch.float16]) +def test_accuracy(M, N, K, group_size, out_dtype): + _test_accuracy_once(M, N, K, group_size, out_dtype, "cuda") + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_rotary_embedding.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_rotary_embedding.py new file mode 100644 index 00000000..cc5374db --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_rotary_embedding.py @@ -0,0 +1,144 @@ +from typing import Any, Dict, List, Optional, Tuple, Union + +import pytest +import torch +from sgl_kernel import FusedSetKVBufferArg, apply_rope_with_cos_sin_cache_inplace +from sgl_kernel.testing.rotary_embedding import ( + FlashInferRotaryEmbedding, + MHATokenToKVPool, + RotaryEmbedding, + create_inputs, +) + + +@pytest.mark.parametrize( + "head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype, device, batch_size, seq_len, num_q_heads, num_kv_heads, save_kv_cache", + [ + # GPT-OSS cases + *[ + ( + 64, + 64, + 4096, + 8000, + True, + torch.bfloat16, + "cuda", + batch_size, + seq_len, + 64, + 8, + save_kv_cache, + ) + for batch_size, seq_len in ( + (1, 1), + (32, 1), + (128, 1), + (512, 1), + (2, 512), + (4, 4096), + ) + for save_kv_cache in (False, True) + ], + # Other cases + (64, 64, 32, 8000, True, torch.bfloat16, "cuda", 32, 32, 1, 1, False), + (256, 128, 4096, 10000, True, torch.bfloat16, "cuda", 2, 512, 4, 2, False), + (512, 128, 311, 10000, True, torch.bfloat16, "cuda", 3, 39, 4, 2, False), + (128, 128, 2048, 10000, False, torch.bfloat16, "cuda", 2, 512, 32, 8, False), + (128, 128, 2048, 10000, False, torch.bfloat16, "cuda", 2, 512, 16, 4, False), + (512, 128, 311, 10000, False, torch.bfloat16, "cuda", 3, 39, 4, 2, False), + (64, 64, 32, 8000, True, torch.float32, "cuda", 32, 32, 1, 1, False), + (256, 128, 4096, 10000, True, torch.float32, "cuda", 2, 512, 4, 2, False), + (512, 128, 311, 10000, True, torch.float32, "cuda", 3, 39, 4, 2, False), + (128, 128, 2048, 10000, False, torch.float32, "cuda", 2, 512, 32, 8, False), + (128, 128, 2048, 10000, False, torch.float32, "cuda", 2, 512, 16, 4, False), + (512, 128, 311, 10000, False, torch.float32, "cuda", 3, 39, 4, 2, False), + ], +) +def test_correctness( + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + dtype: torch.dtype, + device: str, + batch_size: int, + seq_len: int, + num_q_heads: int, + num_kv_heads: int, + save_kv_cache: bool, +): + config = dict( + head_size=head_size, + rotary_dim=rotary_dim, + max_position_embeddings=max_position_embeddings, + base=base, + is_neox_style=is_neox_style, + dtype=dtype, + ) + + rope_ref = RotaryEmbedding(**config).to(device) + rope_flashinfer = FlashInferRotaryEmbedding(**config).to(device) + + inputs = create_inputs( + head_size=head_size, + batch_size=batch_size, + seq_len=seq_len, + device=device, + dtype=dtype, + num_q_heads=num_q_heads, + num_kv_heads=num_kv_heads, + ) + + if save_kv_cache: + pool_ref = MHATokenToKVPool(head_num=num_kv_heads, head_dim=head_size) + pool_flashinfer = MHATokenToKVPool(head_num=num_kv_heads, head_dim=head_size) + + query_ref, key_ref = inputs["query"].clone(), inputs["key"].clone() + query_flashinfer, key_flashinfer = inputs["query"].clone(), inputs["key"].clone() + + query_ref_out, key_ref_out = rope_ref.forward_native( + inputs["pos_ids"], query_ref, key_ref + ) + if save_kv_cache: + pool_ref.set_kv_buffer( + loc=inputs["out_cache_loc"], + cache_k=key_ref_out.view(-1, num_kv_heads, head_size), + cache_v=inputs["value"].view(-1, num_kv_heads, head_size), + ) + + query_flashinfer_out, key_flashinfer_out = rope_flashinfer.forward_cuda( + inputs["pos_ids"], + query_flashinfer, + key_flashinfer, + fused_set_kv_buffer_arg=( + FusedSetKVBufferArg( + value=inputs["value"], + k_buffer=pool_flashinfer.k_buffer[0].view(-1, num_kv_heads * head_size), + v_buffer=pool_flashinfer.v_buffer[0].view(-1, num_kv_heads * head_size), + k_scale=None, + v_scale=None, + cache_loc=inputs["out_cache_loc"], + ) + if save_kv_cache + else None + ), + ) + + torch.testing.assert_close( + query_ref_out, query_flashinfer_out, atol=1e-2, rtol=1e-2 + ) + torch.testing.assert_close(key_ref_out, key_flashinfer_out, atol=1e-2, rtol=1e-2) + if save_kv_cache: + for field in ["k_buffer", "v_buffer"]: + x_ref = getattr(pool_ref, field)[0] + x_flashinfer = getattr(pool_flashinfer, field)[0] + torch.testing.assert_close(x_ref, x_flashinfer, atol=1e-2, rtol=1e-2) + nonzero_ref = x_ref != 0 + nonzero_flashinfer = x_ref != 0 + assert torch.all(nonzero_ref == nonzero_flashinfer) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_sampling.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_sampling.py new file mode 100644 index 00000000..dc5734cb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_sampling.py @@ -0,0 +1,185 @@ +# Adapted from https://github.com/flashinfer-ai/flashinfer/blob/93e1a2634e22355b0856246b032b285ad1d1da6b/tests/test_sampling.py + +import pytest +import sgl_kernel +import torch + + +@pytest.mark.parametrize("batch_size", [1, 99, 989]) +@pytest.mark.parametrize("vocab_size", [111, 32000, 128256]) +@pytest.mark.parametrize("k", [100]) +@pytest.mark.parametrize("p", [0.1, 0.5]) +def test_top_k_top_p_sampling_from_probs_logits_top_k_first_alignment( + batch_size, vocab_size, k, p +): + torch.manual_seed(42) + logits = torch.randn(batch_size, vocab_size, device="cuda:0") * 5 + generator_logits = torch.Generator("cuda:0") + generator_probs = generator_logits.clone_state() + samples = sgl_kernel.sampling.top_k_top_p_sampling_from_logits( + logits, k, p, filter_apply_order="top_k_first", generator=generator_logits + ) + samples_ref = sgl_kernel.sampling.top_k_top_p_sampling_from_probs( + torch.softmax(logits, dim=-1), + k, + p, + filter_apply_order="top_k_first", + generator=generator_probs, + ) + assert torch.all(samples == samples_ref) + + +@pytest.mark.parametrize("batch_size", [1, 99, 989]) +@pytest.mark.parametrize("vocab_size", [111, 32000, 128256]) +@pytest.mark.parametrize("k", [100]) +@pytest.mark.parametrize("p", [0.1, 0.5]) +def test_top_k_top_p_sampling_from_probs_logits_joint_alignment( + batch_size, vocab_size, k, p +): + torch.manual_seed(42) + logits = torch.randn(batch_size, vocab_size, device="cuda:0") * 5 + generator_logits = torch.Generator("cuda:0") + generator_probs = generator_logits.clone_state() + samples = sgl_kernel.sampling.top_k_top_p_sampling_from_logits( + logits, k, p, filter_apply_order="joint", generator=generator_logits + ) + samples_ref = sgl_kernel.sampling.top_k_top_p_sampling_from_probs( + torch.softmax(logits, dim=-1), + k, + p, + filter_apply_order="joint", + generator=generator_probs, + ) + assert torch.all(samples == samples_ref) + + +@pytest.mark.parametrize("batch_size", [1, 99, 989]) +@pytest.mark.parametrize("vocab_size", [111, 32000, 128256]) +@pytest.mark.parametrize("p", [0.1, 0.5]) +def test_top_k_top_p_joint_sampling_from_probs(batch_size, vocab_size, p): + torch.manual_seed(42) + if p == 0.1: + k = int(vocab_size * 0.5) + elif p == 0.5: + k = int(vocab_size * 0.1) + else: + raise ValueError("p not recognized") + eps = 1e-4 + pre_norm_prob = torch.rand(batch_size, vocab_size, device="cuda:0") + normalized_prob = pre_norm_prob / pre_norm_prob.sum(dim=-1, keepdim=True) + # top-p mask + sorted_prob, indices = torch.sort(normalized_prob, descending=False) + cdf = torch.cumsum(sorted_prob, dim=-1) + mask_top_p = torch.zeros(batch_size, vocab_size, dtype=torch.int32, device="cuda:0") + mask_top_p.scatter_add_(1, indices, (cdf > (1 - p) - eps).int()) + # top-k mask + sorted_prob, _ = torch.sort(normalized_prob, descending=True) + pivot = sorted_prob[:, k - 1] + mask_top_k = (normalized_prob >= pivot.unsqueeze(-1)).int() + # overall mask + mask = torch.minimum(mask_top_p, mask_top_k) + top_p_tensor = torch.full((batch_size,), p, device="cuda:0") + top_k_tensor = torch.full((batch_size,), k, device="cuda:0") + + num_trails = 1000 + for _ in range(num_trails): + samples = sgl_kernel.top_k_top_p_sampling_from_probs( + normalized_prob, + top_k_tensor, + top_p_tensor, + filter_apply_order="joint", + ) + assert torch.all(samples < vocab_size) and torch.all(samples >= 0) + assert torch.all(mask[torch.arange(batch_size), samples] == 1), normalized_prob[ + torch.arange(batch_size), samples + ] + + +@pytest.mark.parametrize("batch_size", [1, 99, 989]) +@pytest.mark.parametrize("vocab_size", [111, 32000, 128256]) +@pytest.mark.parametrize("p", [0.1, 0.5, 0.9]) +def test_top_p_renorm_probs(batch_size, vocab_size, p): + torch.manual_seed(42) + pre_norm_prob = torch.rand(batch_size, vocab_size, device="cuda:0") + normalized_prob = pre_norm_prob / pre_norm_prob.sum(dim=-1, keepdim=True) + sorted_prob, indices = torch.sort(normalized_prob, descending=False) + cdf = torch.cumsum(sorted_prob, dim=-1) + mask = torch.zeros(batch_size, vocab_size, dtype=torch.int32, device="cuda:0") + mask.scatter_add_(1, indices, (cdf >= (1 - p)).int()) + renorm_prob_ground_truth = normalized_prob.clone() + renorm_prob_ground_truth[mask == 0] = 0 + renorm_prob_ground_truth = renorm_prob_ground_truth / renorm_prob_ground_truth.sum( + dim=-1, keepdim=True + ) + + renorm_prob = sgl_kernel.top_p_renorm_prob(normalized_prob, p) + torch.testing.assert_close( + renorm_prob_ground_truth, + renorm_prob, + rtol=1e-3, + atol=1e-3, + ) + + +@pytest.mark.parametrize("batch_size", [1, 99, 989]) +@pytest.mark.parametrize("vocab_size", [111, 32000, 128256]) +@pytest.mark.parametrize("k", [10, 100, 500]) +def test_top_k_renorm_probs(batch_size, vocab_size, k): + if k > vocab_size: + pytest.skip("k should be less than vocab_size") + torch.manual_seed(42) + pre_norm_prob = torch.rand(batch_size, vocab_size, device="cuda:0") + normalized_prob = pre_norm_prob / pre_norm_prob.sum(dim=-1, keepdim=True) + sorted_prob, _ = torch.sort(normalized_prob, descending=True) + pivot = sorted_prob[:, k - 1] + mask = (normalized_prob >= pivot.unsqueeze(-1)).int() + renorm_prob_ground_truth = normalized_prob.clone() + renorm_prob_ground_truth[mask == 0] = 0 + renorm_prob_ground_truth = renorm_prob_ground_truth / renorm_prob_ground_truth.sum( + dim=-1, keepdim=True + ) + + renorm_prob = sgl_kernel.top_k_renorm_prob(normalized_prob, k) + for i in range(batch_size): + torch.testing.assert_close( + renorm_prob_ground_truth[i], + renorm_prob[i], + rtol=1e-3, + atol=1e-3, + ) + + +@pytest.mark.parametrize("batch_size", [1, 99, 989]) +@pytest.mark.parametrize("vocab_size", [111, 32000, 128256]) +@pytest.mark.parametrize("p", [0.05, 0.1, 0.2, 0.7, 1]) +def test_min_p_sampling(batch_size, vocab_size, p): + torch.manual_seed(42) + pre_norm_prob = torch.rand(batch_size, vocab_size, device="cuda:0") + normalized_prob = pre_norm_prob / pre_norm_prob.sum(dim=-1, keepdim=True) + sorted_prob, indices = torch.sort(normalized_prob, descending=False) + # scale min-p + top_probs = sorted_prob[:, -1].unsqueeze(-1) + scaled_p = p * top_probs + # min-p mask + mask = torch.zeros(batch_size, vocab_size, dtype=torch.int32, device="cuda:0") + mask.scatter_add_(1, indices, (sorted_prob >= scaled_p).int()) + min_p_tensor = torch.full((batch_size,), p, device="cuda:0") + + num_trails = 1000 + for _ in range(num_trails): + samples = sgl_kernel.min_p_sampling_from_probs( + normalized_prob, + min_p_tensor, + ) + + assert torch.all(mask[torch.arange(batch_size), samples] == 1), samples[ + torch.nonzero(mask[torch.arange(batch_size), samples] == 0) + ] + + assert torch.all(mask[torch.arange(batch_size), samples] == 1), samples[ + torch.nonzero(mask[torch.arange(batch_size), samples] == 0) + ] + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_sparse_flash_attn.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_sparse_flash_attn.py new file mode 100644 index 00000000..28c64cb6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_sparse_flash_attn.py @@ -0,0 +1,492 @@ +import math +from typing import List, Optional, Tuple + +import pytest +import torch +from einops import rearrange, repeat +from sgl_kernel.sparse_flash_attn import ( + convert_vertical_slash_indexes, + convert_vertical_slash_indexes_mergehead, + sparse_attn_func, +) +from test_flash_attention import construct_local_mask, is_fa3_supported + + +def ref_attn( + q, + k, + v, + query_padding_mask=None, + key_padding_mask=None, + attn_bias=None, + dropout_p=0.0, + dropout_mask=None, + causal=False, + window_size=(-1, -1), # -1 means infinite window size + softcap=0.0, + upcast=True, + reorder_ops=False, + key_leftpad=None, +): + """ + Arguments: + q: (batch_size, seqlen_q, nheads, head_dim) + k: (batch_size, seqlen_k, nheads_k, head_dim) + v: (batch_size, seqlen_k, nheads_k, head_dim) + query_padding_mask: (batch_size, seqlen_q) + key_padding_mask: (batch_size, seqlen_k) + attn_bias: broadcastable to (batch_size, nheads, seqlen_q, seqlen_k) + dropout_p: float + dropout_mask: (batch_size, nheads, seqlen_q, seqlen_k) + causal: whether to apply causal masking + window_size: (int, int), left and right window size + upcast: whether to cast all inputs to fp32, do all computation in fp32, then cast + output back to fp16/bf16. + reorder_ops: whether to change the order of operations (scaling k instead of scaling q, etc.) + without changing the math. This is to estimate the numerical error from operation + reordering. + Output: + output: (batch_size, seqlen_q, nheads, head_dim) + lse: (batch_size, nheads, seqlen_q) + """ + if causal: + window_size = (window_size[0], 0) + dtype_og = q.dtype + if upcast: + q, k, v = q.float(), k.float(), v.float() + seqlen_q, seqlen_k = q.shape[1], k.shape[1] + k = repeat(k, "b s h d -> b s (h g) d", g=q.shape[2] // k.shape[2]) + v = repeat(v, "b s h d -> b s (h g) d", g=q.shape[2] // v.shape[2]) + d = q.shape[-1] + if not reorder_ops: + scores = torch.einsum("bthd,bshd->bhts", q / math.sqrt(d), k) + else: + scores = torch.einsum("bthd,bshd->bhts", q, k / math.sqrt(d)) + + lse_ref = scores.logsumexp(dim=-1) + + if softcap > 0: + scores = scores / softcap + scores = scores.tanh() + scores = scores * softcap + if key_padding_mask is not None: + scores.masked_fill_( + rearrange(~key_padding_mask, "b s -> b 1 1 s"), float("-inf") + ) + if window_size[0] >= 0 or window_size[1] >= 0: + local_mask = construct_local_mask( + seqlen_q, + seqlen_k, + window_size, + query_padding_mask, + key_padding_mask, + q.device, + key_leftpad=key_leftpad, + ) + scores.masked_fill_(local_mask, float("-inf")) + if attn_bias is not None: + scores = scores + attn_bias + attention = torch.softmax(scores, dim=-1).to(v.dtype) + # Some rows might be completely masked out so we fill them with zero instead of NaN + if window_size[0] >= 0 or window_size[1] >= 0: + attention = attention.masked_fill( + torch.all(local_mask, dim=-1, keepdim=True), 0.0 + ) + # We want to mask here so that the attention matrix doesn't have any NaNs + # Otherwise we'll get NaN in dV + if query_padding_mask is not None: + attention = attention.masked_fill( + rearrange(~query_padding_mask, "b s -> b 1 s 1"), 0.0 + ) + dropout_scaling = 1.0 / (1 - dropout_p) + # attention_drop = attention.masked_fill(~dropout_mask, 0.0) * dropout_scaling + # output = torch.einsum('bhts,bshd->bthd', attention_drop , v) + if dropout_mask is not None: + attention_drop = attention.masked_fill(~dropout_mask, 0.0) + else: + attention_drop = attention + output = torch.einsum("bhts,bshd->bthd", attention_drop, v * dropout_scaling) + if query_padding_mask is not None: + output.masked_fill_(rearrange(~query_padding_mask, "b s -> b s 1 1"), 0.0) + + return output.to(dtype=dtype_og), lse_ref + + +def ref_paged_attn( + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + query_lens: List[int], + kv_lens: List[int], + block_tables: torch.Tensor, + scale: float, + sliding_window: Optional[int] = None, + soft_cap: Optional[float] = None, +) -> torch.Tensor: + num_seqs = len(query_lens) + block_tables = block_tables.cpu().numpy() + _, block_size, num_kv_heads, head_size = key_cache.shape + + outputs: List[torch.Tensor] = [] + start_idx = 0 + for i in range(num_seqs): + query_len = query_lens[i] + kv_len = kv_lens[i] + # clone to avoid clobbering the query tensor + q = query[start_idx : start_idx + query_len].clone() + q *= scale + + num_kv_blocks = (kv_len + block_size - 1) // block_size + block_indices = block_tables[i, :num_kv_blocks] + + k = key_cache[block_indices].view(-1, num_kv_heads, head_size) + k = k[:kv_len] + v = value_cache[block_indices].view(-1, num_kv_heads, head_size) + v = v[:kv_len] + + if q.shape[1] != k.shape[1]: + k = torch.repeat_interleave(k, q.shape[1] // k.shape[1], dim=1) + v = torch.repeat_interleave(v, q.shape[1] // v.shape[1], dim=1) + attn = torch.einsum("qhd,khd->hqk", q, k).float() + empty_mask = torch.ones(query_len, kv_len) + mask = torch.triu(empty_mask, diagonal=kv_len - query_len + 1).bool() + if sliding_window is not None: + sliding_window_mask = ( + torch.triu( + empty_mask, diagonal=kv_len - (query_len + sliding_window) + 1 + ) + .bool() + .logical_not() + ) + mask |= sliding_window_mask + if soft_cap is not None: + attn = soft_cap * torch.tanh(attn / soft_cap) + attn.masked_fill_(mask, float("-inf")) + attn = torch.softmax(attn, dim=-1).to(v.dtype) + out = torch.einsum("hqk,khd->qhd", attn, v) + + outputs.append(out) + start_idx += query_len + + return torch.cat(outputs, dim=0) + + +@pytest.mark.skipif( + not is_fa3_supported(), + reason="flash_attn at sgl-kernel is only supported on sm90 or sm80", +) +@pytest.mark.parametrize("batch_size", [1, 2]) +@pytest.mark.parametrize( + "seq_lens", + [ + (1, 1), + (1, 1024), + (1, 2048), + (1023, 2049), + (1023, 1023), + (32, 32), + (65, 65), + (129, 129), + ], +) +@pytest.mark.parametrize("num_heads", [1, 2, 4]) +@pytest.mark.parametrize("head_size", [128]) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("NNZ_S", [0, 1, 2, 3, 7, 15, 32]) +@torch.inference_mode() +def test_sparse_attention( + batch_size, + seq_lens, + num_heads, + head_size, + dtype, + NNZ_S, +) -> None: + torch.set_default_device("cuda") + torch.cuda.manual_seed_all(0) + block_size_M = 64 + block_size_N = 64 + seqlen_q, seqlen_k = seq_lens + q = torch.randn( + batch_size, seqlen_q, num_heads, head_size, dtype=dtype, requires_grad=False + ) + k = torch.randn( + batch_size, seqlen_k, num_heads, head_size, dtype=dtype, requires_grad=False + ) + v = torch.randn( + batch_size, seqlen_k, num_heads, head_size, dtype=dtype, requires_grad=False + ) + NUM_ROWS = (seqlen_q + block_size_M - 1) // block_size_M + if NNZ_S * block_size_N > seqlen_k: + return + NNZ_V = seqlen_k - NNZ_S * block_size_N + block_count = torch.tensor( + [NNZ_S] * batch_size * NUM_ROWS * num_heads, dtype=torch.int32 + ).reshape(batch_size, num_heads, NUM_ROWS) + column_count = torch.tensor( + [NNZ_V] * batch_size * NUM_ROWS * num_heads, dtype=torch.int32 + ).reshape(batch_size, num_heads, NUM_ROWS) + block_offset = torch.tensor( + [[i * block_size_N for i in range(NNZ_S)]] * batch_size * NUM_ROWS * num_heads, + dtype=torch.int32, + ).reshape(batch_size, num_heads, NUM_ROWS, NNZ_S) + column_index = torch.tensor( + [[NNZ_S * block_size_N + i for i in range(NNZ_V)]] + * batch_size + * NUM_ROWS + * num_heads, + dtype=torch.int32, + ).reshape(batch_size, num_heads, NUM_ROWS, NNZ_V) + out, lse = sparse_attn_func( + q, + k, + v, + block_count, + block_offset, + column_count, + column_index, + return_softmax_lse=True, + ) + + ref_out, ref_lse = ref_attn(q, k, v) + + torch.testing.assert_close( + out, ref_out, atol=2e-2, rtol=1e-2 + ), f"{torch.max(torch.abs(out - ref_out))}" + torch.testing.assert_close( + lse, ref_lse, atol=2e-2, rtol=1e-2 + ), f"{torch.max(torch.abs(lse - ref_lse))}" + + +# sparse attention utils +# origin +@pytest.mark.skipif( + not is_fa3_supported(), + reason="flash_attn at sgl-kernel is only supported on sm90 or sm80", +) +@pytest.mark.parametrize("causal", [True, False]) +def test_convert_vertical_slash_indexes(causal): + # Prepare small, hand-checkable inputs + q_seqlens = torch.tensor([4], dtype=torch.int32, device="cuda") # [BATCH] + kv_seqlens = torch.tensor([4], dtype=torch.int32, device="cuda") + vertical_indexes = torch.tensor( + [[[1, 3]]], dtype=torch.int32, device="cuda" + ) # [BATCH, N_HEADS, NNZ_V] + slash_indexes = torch.tensor( + [[[2]]], dtype=torch.int32, device="cuda" + ) # [BATCH, N_HEADS, NNZ_S] + context_size = 4 + block_size_M = 2 + block_size_N = 2 + + # Call your CUDA kernel wrapper + block_count, block_offset, column_count, column_index = ( + convert_vertical_slash_indexes( + q_seqlens, + kv_seqlens, + vertical_indexes, + slash_indexes, + context_size, + block_size_M, + block_size_N, + causal=causal, + ) + ) + + # Manually create expected outputs for this input + # There are 2 rows (blocks): row0 (tokens 0-1), row1 (tokens 2-3) + # Fill these expected tensors based on your CUDA kernel's logic + # For demonstration, we assume: + # - block_count: how many slash indices fall into each block + # - block_offset: the value of those indices + # - column_count: number of valid vertical indices per block + # - column_index: the actual vertical indices + + expected_column_index = torch.tensor( + [[[[0, 0], [0, 0]]]], dtype=torch.int32, device="cuda" + ) + + # If causal=False, update these tensors according to expected behavior + if not causal: + # Update these values if your kernel produces different output in non-causal mode + expected_column_index = torch.tensor( + [[[[1, 0], [1, 3]]]], dtype=torch.int32, device="cuda" + ) + + # Assert that outputs match expectations + assert torch.equal(column_index, expected_column_index) + + +# mergehead +@pytest.mark.skipif( + not is_fa3_supported(), + reason="flash_attn at sgl-kernel is only supported on sm90 or sm80", +) +@pytest.mark.parametrize("causal", [True, False]) +def test_convert_vertical_slash_indexes_mergehead(causal): + # Prepare small, hand-checkable inputs for mergehead version + q_seqlens = torch.tensor([4], dtype=torch.int32, device="cuda") + kv_seqlens = torch.tensor([4], dtype=torch.int32, device="cuda") + vertical_indexes = torch.tensor( + [ + [ + [1, 3], # head 0 + [2, 0], # head 1 + ] + ], + dtype=torch.int32, + device="cuda", + ) # [BATCH, N_HEADS, NNZ_V] + slash_indexes = torch.tensor( + [ + [ + [2, 0], # head 0 + [1, 3], # head 1 + ] + ], + dtype=torch.int32, + device="cuda", + ) # [BATCH, N_HEADS, NNZ_S] + vertical_indices_count = torch.tensor([2, 1], dtype=torch.int32, device="cuda") + slash_indices_count = torch.tensor([1, 2], dtype=torch.int32, device="cuda") + context_size = 4 + block_size_M = 2 + block_size_N = 2 + + # Call your CUDA kernel wrapper + block_count, block_offset, column_count, column_index = ( + convert_vertical_slash_indexes_mergehead( + q_seqlens, + kv_seqlens, + vertical_indexes, + slash_indexes, + vertical_indices_count, + slash_indices_count, + context_size, + block_size_M, + block_size_N, + causal=causal, + ) + ) + + # Manually create expected outputs for this input + # For demonstration, assume: + # - batch=1, head=2, num_rows=2, nnz_v=2, nnz_s=2 + # Fill these expected tensors according to your kernel's behavior + + expected_column_index = torch.tensor( + [[[[1, 0], [1, 3]], [[-1079459945, -1077788999], [-1080050043, -1104625879]]]], + dtype=torch.int32, + device="cuda", + ) + + if not causal: + # If non-causal mode output is different, update these values + expected_column_index = torch.tensor( + [[[[1, 0], [1, 3]], [[2, -1077788999], [2, -1104625879]]]], + dtype=torch.int32, + device="cuda", + ) + + # Assert that outputs match expectations + assert torch.equal(column_index, expected_column_index) + + +# skip cause use fa2 for test +# @pytest.mark.parametrize("seq_lens", [[(1024, 1328)], +# [(1024, 1328), (1, 2048)], +# [(1025, 1328), (2, 2048)], +# [(1025, 2049), (2, 1281)], +# ]) +# @pytest.mark.parametrize("head_size", [128]) +# @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +# @torch.inference_mode() +# def test_sparse_attention_varlen( +# seq_lens, +# head_size, +# dtype, +# ) -> None: +# torch.set_default_device("cuda") +# torch.cuda.manual_seed_all(0) +# block_size_M = 64 +# block_size_N = 64 +# num_seqs = len(seq_lens) +# query_lens = [x[0] for x in seq_lens] +# kv_lens = [x[1] for x in seq_lens] +# num_heads = 1 +# query = torch.randn(sum(query_lens), +# num_heads, +# head_size, +# dtype=dtype) +# key = torch.randn(sum(kv_lens), +# num_heads, +# head_size, +# dtype=dtype) +# value = torch.randn_like(key) +# cu_query_lens = torch.tensor([0] + query_lens, +# dtype=torch.int32).cumsum(dim=0, +# dtype=torch.int32) +# cu_kv_lens = torch.tensor([0] + kv_lens, +# dtype=torch.int32).cumsum(dim=0, +# dtype=torch.int32) +# max_query_len = max(query_lens) +# max_kv_len = max(kv_lens) + +# NUM_ROWS = (max_query_len + block_size_M - 1) // block_size_M +# NNZ_S = 20 +# NNZ_V = 2048 +# batch_size = len(query_lens) + +# block_counts = [] +# column_counts = [] +# block_offsets = [] +# column_indices = [] +# for b in range(batch_size): +# block_counts.append(torch.tensor([NNZ_S] * NUM_ROWS * num_heads, dtype=torch.int32).reshape(num_heads, NUM_ROWS)) +# columns = kv_lens[b] - NNZ_S * block_size_N +# column_counts.append(torch.tensor([columns] * NUM_ROWS * num_heads, dtype=torch.int32).reshape(num_heads, NUM_ROWS)) +# block_offsets.append(torch.tensor([[i * block_size_N for i in range(NNZ_S)]] * NUM_ROWS * num_heads, dtype=torch.int32).reshape(num_heads, NUM_ROWS, NNZ_S)) +# column_indices.append(torch.tensor([[NNZ_S * block_size_N + i for i in range(NNZ_V)]] * NUM_ROWS * num_heads, dtype=torch.int32).reshape(num_heads, NUM_ROWS, NNZ_V)) +# block_count = torch.concat(block_counts).reshape(batch_size, num_heads, NUM_ROWS) +# column_count = torch.concat(column_counts).reshape(batch_size, num_heads, NUM_ROWS) +# block_offset = torch.concat(block_offsets).reshape(batch_size, num_heads, NUM_ROWS, NNZ_S) +# column_index = torch.concat(column_indices).reshape(batch_size, num_heads, NUM_ROWS, NNZ_V) +# out, lse = sparse_attn_varlen_func( +# query, +# key, +# value, +# block_count, +# block_offset, +# column_count, +# column_index, +# cu_seqlens_q=cu_query_lens, +# cu_seqlens_k=cu_kv_lens, +# max_seqlen_q=max_query_len, +# max_seqlen_k=max_kv_len, +# return_softmax_lse=True, +# ) + +# max_num_blocks_per_seq = (max_kv_len + 2048 - 1) // 2048 +# block_tables = torch.randint(0, +# 2048, +# (len(query_lens), max_num_blocks_per_seq), +# dtype=torch.int32) +# scale = head_size**-0.5 + +# ref_out, ref_lse, _ = ref_paged_attn( +# query, +# key, +# value, +# query_lens=query_lens, +# kv_lens=kv_lens, +# block_tables=block_tables, +# scale=scale +# ) + +# torch.testing.assert_close(out, ref_out, atol=2e-2, rtol=1e-2), \ +# f"{torch.max(torch.abs(out - ref_out))}" +# torch.testing.assert_close(lse, ref_lse, atol=2e-2, rtol=1e-2), \ +# f"{torch.max(torch.abs(lse - ref_lse))}" + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_topk.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_topk.py new file mode 100644 index 00000000..dba02321 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_topk.py @@ -0,0 +1,252 @@ +from typing import Any, Optional + +import pytest +import torch +from sgl_kernel import ( + fast_topk_transform_fused, + fast_topk_transform_ragged_fused, + fast_topk_v2, +) + + +def _ref_torch_impl( + score: torch.Tensor, + seq_len: int, + topk: int, + row_starts: Optional[torch.Tensor] = None, +) -> torch.Tensor: + assert score.dim() == 2 + if row_starts is None: + return torch.topk(score[:, :seq_len], topk, dim=-1, sorted=False).indices + else: + ks = row_starts.cpu().tolist() + ke = (row_starts + seq_len).tolist() + scores = [] + for i, (start, end) in enumerate(zip(ks, ke)): + scores.append(score[i, start:end].unsqueeze(0)) + score = torch.cat(scores, dim=0) + return torch.topk(score, topk, dim=-1, sorted=False).indices + + +def _ref_torch_transform_decode_impl( + score: torch.Tensor, + seq_len: int, + src_page_table: torch.Tensor, + topk: int, + row_starts: Optional[torch.Tensor] = None, +) -> torch.Tensor: + batch_size, _ = score.shape + assert score.shape[0] == src_page_table.shape[0] + assert seq_len >= topk + indices = _ref_torch_impl(score, seq_len, topk, row_starts=row_starts) + topk_indices = torch.empty( + (batch_size, topk), dtype=torch.int32, device=score.device + ) + for i in range(batch_size): + topk_indices[i] = src_page_table[i, indices[i]] + return topk_indices + + +def _ref_torch_transform_ragged_impl( + score: torch.Tensor, + seq_len: int, + topk_indices_offset: torch.Tensor, + topk: int, + row_starts: torch.Tensor, +) -> torch.Tensor: + assert score.shape[0] == topk_indices_offset.shape[0] + assert seq_len >= topk + indices = _ref_torch_impl(score, seq_len, topk, row_starts=row_starts) + + mask = indices != -1 + topk_indices_offset = topk_indices_offset.unsqueeze(1) + return torch.where(mask, indices + topk_indices_offset, indices) + + +MAX_SEQ_LEN = 131072 + + +def assert_equal( + score: torch.Tensor, + indices_ref: torch.Tensor, + indices_our: torch.Tensor, + bs: int, + k: int, + seq_len: int, + topk_indices_offset: Optional[torch.Tensor] = None, + max_permit_error: int = 0, +): + indices_our_cpu = indices_our.cpu().tolist() + indices_ref_cpu = indices_ref.cpu().tolist() + + wrong_values = 0 + for i in range(bs): + indices_ref_set_i = set(indices_ref_cpu[i]) + indices_our_set_i = set(indices_our_cpu[i]) + more = indices_our_set_i - indices_ref_set_i + less = indices_ref_set_i - indices_our_set_i + offset = topk_indices_offset[i].item() if topk_indices_offset is not None else 0 + if len(more) > 0 or len(less) > 0: + # check whether more values are the same with less values + # if so, either one is acceptable, since their values are the same + more_values = sorted(score[i, idx - offset].item() for idx in more) + less_values = sorted(score[i, idx - offset].item() for idx in less) + if more_values != less_values: + wrong_values += len(more) + print( + f"{bs=}, {k=}, {seq_len=}, {i=}, {more=}, {less=} failed, with {more_values=}, {less_values=}" + ) + assert wrong_values <= max_permit_error, f"{wrong_values=}, {max_permit_error=}" + + +@pytest.mark.parametrize("bs", [1, 132, 256, 4096]) +@pytest.mark.parametrize("k", [2048]) # we only support 2048 now +@pytest.mark.parametrize("seq_len", [2048, 4096, 16384, 65536]) +@pytest.mark.parametrize("has_row_starts", [True, False]) +@torch.inference_mode() +def test_topk_kernel(bs: int, k: int, seq_len: int, has_row_starts: bool) -> None: + torch.manual_seed(42) + + stream = torch.cuda.Stream() + torch.cuda.set_stream(stream) + score = torch.randn(bs, MAX_SEQ_LEN, dtype=torch.float32, device="cuda") + lengths = torch.full((bs,), seq_len, dtype=torch.int32, device="cuda") + + if has_row_starts: + row_starts = torch.randint(0, 2048, (bs,), dtype=torch.int32, device="cuda") + else: + row_starts = None + + indices_ref = _ref_torch_impl(score, seq_len, k, row_starts=row_starts) + indices_our = fast_topk_v2(score, lengths, k, row_starts=row_starts) + + # sort and compare + indices_ref = torch.sort(indices_ref, dim=-1).values + indices_our = torch.sort(indices_our, dim=-1).values + + # Tests can pass with max_permit_error=3, set to 5 for safety + assert_equal(score, indices_ref, indices_our, bs, k, seq_len, max_permit_error=5) + + +@pytest.mark.parametrize("bs", [1, 132, 256, 4096]) +@pytest.mark.parametrize("k", [2048]) # we only support 2048 now +@pytest.mark.parametrize("seq_len", [2048, 4096, 16384, 65536]) +@pytest.mark.parametrize("mode", ["extend", "decode", "target_verify"]) +@torch.inference_mode() +def test_topk_transform_kernel(bs: int, k: int, seq_len: int, mode: str) -> None: + torch.manual_seed(42) + + stream = torch.cuda.Stream() + torch.cuda.set_stream(stream) + + # NOTE: for decode, cumulative seqlens_q is just 0..=bs + # NOTE: since page table is arange, they equal topk indices + if mode == "decode": + step = 1 + else: + step = 4 if bs % 4 == 0 else 1 + num_tokens = bs + bs = bs // step + + if mode == "extend": + row_starts = torch.randint(0, 2048, (bs,), dtype=torch.int32, device="cuda") + else: + row_starts = None + + score = torch.randn(bs, MAX_SEQ_LEN, dtype=torch.float32, device="cuda") + lengths = torch.full((bs,), seq_len, dtype=torch.int32, device="cuda") + cu_seqlens_q = torch.arange( + 0, num_tokens + 1, step=step, dtype=torch.int32, device="cuda" + ) + src_page_table = torch.arange(0, seq_len, dtype=torch.int32, device="cuda") + src_page_table = src_page_table.unsqueeze(0).expand(bs, -1) + + dst_page_table_ref = _ref_torch_transform_decode_impl( + score=score, + seq_len=seq_len, + src_page_table=src_page_table, + topk=k, + row_starts=row_starts, + ) + dst_page_table_our = fast_topk_transform_fused( + score=score, + lengths=lengths, + page_table_size_1=src_page_table, + cu_seqlens_q=cu_seqlens_q, + topk=k, + row_starts=row_starts, + ) + + # sort and compare + dst_page_table_our = torch.sort(dst_page_table_our, dim=-1).values + dst_page_table_ref = torch.sort(dst_page_table_ref, dim=-1).values + + assert_equal( + score, + dst_page_table_ref, + dst_page_table_our, + bs, + k, + seq_len, + max_permit_error=5, + ) + + +@pytest.mark.parametrize("bs", [1, 132, 256, 4096]) +@pytest.mark.parametrize("k", [2048]) # we only support 2048 now +@pytest.mark.parametrize("seq_len", [2048, 4096, 16384, 65536]) +@pytest.mark.parametrize("has_row_starts", [True, False]) +@torch.inference_mode() +def test_topk_transform_ragged_kernel( + bs: int, k: int, seq_len: int, has_row_starts: bool +) -> None: + # Used in prefill only + torch.manual_seed(42) + + stream = torch.cuda.Stream() + torch.cuda.set_stream(stream) + # bs: # of q tokens + score = torch.randn(bs, MAX_SEQ_LEN, dtype=torch.float32, device="cuda") + # kv_len + if has_row_starts: + row_starts = torch.randint(0, 2048, (bs,), dtype=torch.int32, device="cuda") + else: + row_starts = None + lengths = torch.full((bs,), seq_len, dtype=torch.int32, device="cuda") + topk_indices_offset = torch.randint( + 0, 1024, (bs,), dtype=torch.int32, device="cuda" + ) + + dst_page_table_ref = _ref_torch_transform_ragged_impl( + score=score, + seq_len=seq_len, + topk_indices_offset=topk_indices_offset, + topk=k, + row_starts=row_starts, + ) + dst_page_table_our = fast_topk_transform_ragged_fused( + score=score, + lengths=lengths, + topk_indices_offset=topk_indices_offset, + topk=k, + row_starts=row_starts, + ) + + # sort and compare + dst_page_table_our = torch.sort(dst_page_table_our, dim=-1).values + dst_page_table_ref = torch.sort(dst_page_table_ref, dim=-1).values + + assert_equal( + score, + dst_page_table_ref, + dst_page_table_our, + bs, + k, + seq_len, + topk_indices_offset, + max_permit_error=5, + ) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_torch_defaults_reset.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_torch_defaults_reset.py new file mode 100644 index 00000000..f6fae5d9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/test_torch_defaults_reset.py @@ -0,0 +1,16 @@ +import pytest +import torch + + +def test_change_torch_defaults(): + torch.set_default_device("cpu:0") + torch.set_default_dtype(torch.float16) + + +def test_check_torch_defaults(): + assert torch.get_default_device() == torch.device("cpu") + assert torch.get_default_dtype() == torch.float32 + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/utils.py new file mode 100644 index 00000000..8fa9a223 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-kernel/tests/utils.py @@ -0,0 +1,9 @@ +import torch + + +def is_sm10x(): + return torch.cuda.get_device_capability() >= (10, 0) + + +def is_hopper(): + return torch.cuda.get_device_capability() == (9, 0) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/.cargo/config.toml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/.cargo/config.toml new file mode 100644 index 00000000..305d57fa --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/.cargo/config.toml @@ -0,0 +1,15 @@ +[build] +rustflags = [] +incremental = true + +[target.aarch64-apple-darwin] +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] + +[target.x86_64-apple-darwin] +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/Cargo.toml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/Cargo.toml new file mode 100644 index 00000000..45db679e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/Cargo.toml @@ -0,0 +1,166 @@ +[package] +name = "sgl-model-gateway" +version = "0.2.3" +edition = "2021" + +[features] +default = [] +vendored-openssl = ["openssl/vendored"] + +[lints.rust] +unused_qualifications = "warn" + +[lib] +name = "sgl_model_gateway" +crate-type = ["rlib"] + +[[bin]] +name = "sgl-model-gateway" +path = "src/main.rs" + +[[bin]] +name = "smg" +path = "src/main.rs" + +[[bin]] +name = "amg" +path = "src/main.rs" + +[dependencies] +clap = { version = "4", features = ["derive", "env"] } +axum = { version = "0.8.4", features = ["macros", "ws", "tracing"] } +tower = { version = "0.5", features = ["full"] } +tower-http = { version = "0.6", features = ["trace", "compression-gzip", "cors", "timeout", "limit", "request-id", "util"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0", default-features = false, features = [ + "std", + "preserve_order", +] } +bytes = "1.8.0" +rand = "0.9.2" +reqwest = { version = "0.12.8", features = ["stream", "blocking", "json", "rustls-tls"], default-features = false } +futures-util = "0.3" +futures = "0.3" +dashmap = "6.1.0" +lru = "0.16.2" +blake3 = "1.5" +http = "1.1.0" +tokio = { version = "1.42.0", features = ["full"] } +async-trait = "0.1" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter", "json", "chrono"] } +tracing-log = "0.2" +tracing-appender = "0.2.3" +chrono = "0.4" +kube = { version = "1.1.0", features = ["runtime", "derive"] } +k8s-openapi = { version = "0.25.0", features = ["v1_33"] } +metrics = "0.24.2" +metrics-exporter-prometheus = "0.17.0" +uuid = { version = "1.10", features = ["v4", "serde"] } +ulid = "1.2.1" +parking_lot = "0.12.4" +rayon = "1.10" +thiserror = "2.0.12" +regex = "1.10" +url = "2.5.4" +validator = { version = "0.20.0", features = ["derive"] } +tokio-stream = { version = "0.1", features = ["sync"] } +anyhow = "1.0" +tokenizers = { version = "0.22.0" } +tiktoken-rs = { version = "0.7.0" } +minijinja = { version = "2.0", features = ["unstable_machinery", "json", "builtins"] } +minijinja-contrib = { version = "2.0", features = ["pycompat"] } +rustls = { version = "0.23", default-features = false, features = ["ring", "std"] } +openssl = "0.10.73" +hf-hub = { version = "0.4.3", features = ["tokio"] } +rmcp = { version = "0.8.3", features = ["client", "server", + "transport-child-process", + "transport-sse-client-reqwest", + "transport-streamable-http-client-reqwest", + "transport-streamable-http-server", + "transport-streamable-http-server-session", + "reqwest", + "auth"] } +serde_yaml = "0.9" +oracle = { version = "0.6.3", features = ["chrono"] } +subtle = "2.6" +rustpython-parser = "0.4.0" +num-traits = "0.2" +openai-harmony = { git = "https://github.com/openai/harmony", tag = "v0.0.4" } +openmetrics-parser = "0.4.4" + +# gRPC and Protobuf dependencies +tonic = { version = "0.14.2", features = ["gzip", "transport"] } +prost = "0.14.1" +prost-types = "0.14.1" +tonic-prost = "0.14.2" +deadpool = { version = "0.12", features = ["managed", "rt_tokio_1"] } +backoff = { version = "0.4", features = ["tokio"] } +strum = { version = "0.26", features = ["derive"] } +bitflags = "2.10.0" +once_cell = "1.21.3" +tokio-postgres = { version = "0.7.15", features = ["runtime","with-chrono-0_4","with-serde_json-1","array-impls"] } +deadpool-postgres = "0.14.1" + + +[build-dependencies] +tonic-prost-build = "0.14.2" +prost-build = "0.14.1" +chrono = { version = "0.4", features = ["clock"] } +toml = "0.9" + +[dev-dependencies] +criterion = { version = "0.5", features = ["html_reports"] } +tower = { version = "0.5", features = ["util"] } +http-body-util = "0.1" +portpicker = "0.1" +tempfile = "3.8" +lazy_static = "1.4" + +[[bench]] +name = "request_processing" +harness = false +path = "benches/request_processing.rs" + +[[bench]] +name = "tokenizer_benchmark" +harness = false +path = "benches/tokenizer_benchmark.rs" + +[[bench]] +name = "tool_parser_benchmark" +harness = false +path = "benches/tool_parser_benchmark.rs" + +[profile.release] +opt-level = "z" # Optimize for size +lto = "fat" # Full LTO for smaller binaries +codegen-units = 1 # Better optimization, slower compile +strip = true # Strip debug symbols + +[profile.ci] +inherits = "release" +opt-level = 2 # Lighter optimization (still fast runtime, much faster compile) +lto = "thin" # Thin LTO - good balance +codegen-units = 16 # More parallelization for faster builds +strip = true + +[profile.dev] +opt-level = 0 +debug = 1 +split-debuginfo = "unpacked" +incremental = true +codegen-units = 256 + +[profile.dev.package."*"] +opt-level = 2 +debug = false + + +[profile.dev.build-override] +opt-level = 3 +codegen-units = 1 + +[profile.dev-opt] +inherits = "dev" +opt-level = 1 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/LICENSE b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/LICENSE new file mode 120000 index 00000000..ea5b6064 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/LICENSE @@ -0,0 +1 @@ +../LICENSE \ No newline at end of file diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/Makefile b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/Makefile new file mode 100644 index 00000000..5e6cf3f2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/Makefile @@ -0,0 +1,159 @@ +# Model Gateway Makefile +# Provides convenient shortcuts for common development tasks + +# Python bindings directory +PYTHON_DIR := bindings/python + +# Auto-detect CPU cores and cap at reasonable limit to avoid thread exhaustion +# Can be overridden: make python-dev JOBS=4 +NPROC := $(shell nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 8) +JOBS ?= $(shell echo $$(($(NPROC) > 16 ? 16 : $(NPROC)))) + +# Check if sccache is available and set RUSTC_WRAPPER accordingly +SCCACHE := $(shell which sccache 2>/dev/null) +ifdef SCCACHE + export RUSTC_WRAPPER := $(SCCACHE) + $(info Using sccache for compilation caching) +else + $(info sccache not found. Install it for faster builds: cargo install sccache) +endif + +.PHONY: help build test clean docs check fmt dev-setup pre-commit setup-sccache sccache-stats sccache-clean sccache-stop \ + python-dev python-build python-build-release python-install python-clean python-test python-check \ + release-notes + +help: ## Show this help message + @echo "Model Gateway Development Commands" + @echo "==================================" + @echo "" + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-20s\033[0m %s\n", $$1, $$2}' + @echo "" + +build: ## Build the project in release mode + @echo "Building SGLang Model Gateway..." + @cargo build --release + +test: ## Run all tests + @echo "Running tests..." + @cargo test + +clean: ## Clean build artifacts + @echo "Cleaning build artifacts..." + @cargo clean + +docs: ## Generate and open documentation + @echo "Generating documentation..." + @cargo doc --open + +check: ## Run cargo check and clippy + @echo "Running cargo check..." + @cargo check + @echo "Running clippy..." + @cargo clippy --all-targets --all-features -- -D warnings + +fmt: ## Format code with rustfmt + @echo "Formatting code..." + @rustup run nightly cargo fmt + +# Development workflow shortcuts +dev-setup: build test ## Set up development environment + @echo "Development environment ready!" + +pre-commit: fmt check test ## Run pre-commit checks + @echo "Pre-commit checks passed!" + +# sccache management targets +setup-sccache: ## Install and configure sccache + @echo "Setting up sccache..." + @./scripts/setup-sccache.sh + +sccache-stats: ## Show sccache statistics + @if [ -n "$(SCCACHE)" ]; then \ + echo "sccache statistics:"; \ + sccache -s; \ + else \ + echo "sccache not installed. Run 'make setup-sccache' to install it."; \ + fi + +sccache-clean: ## Clear sccache cache + @if [ -n "$(SCCACHE)" ]; then \ + echo "Clearing sccache cache..."; \ + sccache -C; \ + echo "sccache cache cleared"; \ + else \ + echo "sccache not installed"; \ + fi + +sccache-stop: ## Stop the sccache server + @if [ -n "$(SCCACHE)" ]; then \ + echo "Stopping sccache server..."; \ + sccache --stop-server || true; \ + else \ + echo "sccache not installed"; \ + fi + +# Python bindings (maturin) targets +python-dev: ## Build Python bindings in development mode (fast, debug build) + @echo "Building Python bindings in development mode (using $(JOBS) parallel jobs with sccache)..." + @cd $(PYTHON_DIR) && CARGO_BUILD_JOBS=$(JOBS) maturin develop + +python-build: ## Build Python wheel (release mode with vendored OpenSSL) + @echo "Building Python wheel (release, vendored OpenSSL, using $(JOBS) parallel jobs with sccache)..." + @cd $(PYTHON_DIR) && CARGO_BUILD_JOBS=$(JOBS) maturin build --release --out dist --features vendored-openssl + +python-build-release: python-build ## Alias for python-build + +python-install: python-build ## Build and install Python wheel + @echo "Installing Python wheel..." + @pip install --force-reinstall $(PYTHON_DIR)/dist/*.whl + @echo "Python package installed!" + +python-clean: ## Clean Python build artifacts + @echo "Cleaning Python build artifacts..." + @rm -rf $(PYTHON_DIR)/dist/ + @rm -rf $(PYTHON_DIR)/target/ + @rm -rf $(PYTHON_DIR)/sglang_router.egg-info/ + @rm -rf $(PYTHON_DIR)/sglang_router/__pycache__/ + @find $(PYTHON_DIR) -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true + @find $(PYTHON_DIR) -name "*.pyc" -delete 2>/dev/null || true + @echo "Python build artifacts cleaned!" + +python-test: ## Run Python tests + @echo "Running Python tests..." + @pytest py_test/ -v + +python-check: ## Check Python package with twine + @echo "Checking Python package..." + @cd $(PYTHON_DIR) && CARGO_BUILD_JOBS=$(JOBS) maturin build --release --out dist --features vendored-openssl + @pip install twine 2>/dev/null || true + @twine check $(PYTHON_DIR)/dist/* + @echo "Python package check passed!" + +# Combined shortcuts +dev: python-dev ## Quick development setup (build Python bindings in dev mode) + +install: python-install ## Build and install everything + +# Release management +release-notes: ## Generate release notes for gateway (usage: make release-notes PREV=gateway-v0.2.2 CURR=gateway-v1.0.0) + @if [ -z "$(PREV)" ] || [ -z "$(CURR)" ]; then \ + echo "Usage: make release-notes PREV= CURR="; \ + echo "Example: make release-notes PREV=gateway-v0.2.2 CURR=gateway-v1.0.0"; \ + echo ""; \ + echo "Options:"; \ + echo " OUTPUT= Save to file (default: stdout)"; \ + echo " CREATE_RELEASE=1 Create GitHub draft release via gh CLI (default: draft)"; \ + echo " DRAFT=0 Publish release immediately (skip draft)"; \ + exit 1; \ + fi + @ARGS="$(PREV) $(CURR)"; \ + if [ -n "$(OUTPUT)" ]; then \ + ARGS="$$ARGS --output $(OUTPUT)"; \ + fi; \ + if [ "$(CREATE_RELEASE)" = "1" ]; then \ + ARGS="$$ARGS --create-release"; \ + if [ "$(DRAFT)" = "0" ]; then \ + ARGS="$$ARGS --no-draft"; \ + fi; \ + fi; \ + ./scripts/generate_gateway_release_notes.sh $$ARGS diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/README.md new file mode 100644 index 00000000..a7b09997 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/README.md @@ -0,0 +1,675 @@ +# SGLang Model Gateway + +High-performance model routing control and data plane for large-scale LLM deployments. The gateway orchestrates fleets of workers, balances traffic across HTTP and gRPC backends, and exposes OpenAI-compatible APIs with pluggable history storage and tool integrations—while remaining deeply optimized for the SGLang serving runtime. + +## Overview +- Unified control plane for registering, monitoring, and orchestrating prefill, decode, and regular workers across heterogeneous model fleets. +- Data plane that routes requests across HTTP, PD (prefill/decode), gRPC, and OpenAI-compatible backends with shared reliability features. +- Industry-first gRPC pipeline with native Rust tokenization, reasoning, and tool-call execution for high-throughput OpenAI-compatible serving. +- Multi-model inference gateway mode (`--enable-igw`) that runs several routers at once and applies per-model policies. +- Conversation, response, and chat-history connectors that centralize state at the router, enabling compliant sharing across models/MCP loops with in-memory, no-op, or Oracle ATP storage options. +- Built-in reliability primitives: retries with exponential backoff, circuit breakers, token-bucket rate limiting, and queuing. +- First-class observability with structured logging and Prometheus metrics. + +### Architecture at a Glance +**Control Plane** +- Worker Manager validates workers, discovers capabilities, and keeps the registry in sync. +- Job Queue serializes background operations (add/remove) and exposes status via `/workers/{url}`. +- Background health checker and load monitor keep circuit breakers and policies informed. +- Optional Kubernetes service discovery keeps the registry aligned with pods. + +**Data Plane** +- SGLang HTTP routers for regular and PD (prefill/decode) traffic with policy-aware selection. +- SGLang gRPC router and pipeline that stream tokenized requests through SRT gRPC workers with fully Rust tokenizer, reasoning parser, and tool parser implementations for maximal OpenAI API performance, supporting both single-stage and PD serving topologies. +- OpenAI router that proxies OpenAI-style requests, responses, and conversations to remote vendors (OpenAI, xAI, Gemini, and other OpenAI-compatible providers) while preserving streaming/SSE semantics. +- Router Manager coordinates multiple router implementations when IGW is enabled. +- Resilience layer delivers token-bucket rate limiting, request queuing, retry executor, and per-worker circuit breakers to keep traffic flowing through failures. +- Advanced load balancing with cache-aware request reuse, load-aware (power-of-two) selection, and per-model policy overrides. + +## Feature Highlights +- Multiple load balancing strategies (`random`, `round_robin`, `cache_aware`, `power_of_two`) with DP-aware scheduling. +- Multi-model HTTP serving and inference gateway routing with model-specific policies. +- Prefill/decode disaggregation, including bootstrap port handling and cache-aware merging. +- gRPC routing with fully Rust tokenizer loading, reasoning parser selection, and tool parser integration for OpenAI-compatible endpoints—supporting streaming and non-streaming modes across DeepSeek, Llama, Kimi K2, Qwen, GPT-OSS, Mistral, Step-3, GLM4, and other reasoning-capable models. +- OpenAI-compatible `/v1/chat/completions`, `/v1/responses`, `/v1/conversations`, `/v1/embeddings`, and `/v1/rerank` endpoints. +- Native MCP client integration supporting all MCP transport protocols (STDIO, HTTP, SSE, and Streamable) for tool execution loops. +- Pluggable history connectors: in-memory, disabled, or Oracle ATP (with pooling and credential support). +- Reliability controls: retry with jitter, worker-scoped circuit breakers, token bucket limiter with optional queue, and cache flush APIs. +- Service discovery for regular and PD workloads with independent selectors. +- Prometheus metrics and structured tracing for every stage of routing. + +## Documentation +- **User Guide**: [docs.sglang.io/advanced_features/router.html](https://docs.sglang.io/advanced_features/router.html) +- Additional guides, API references, and deployment patterns are continuously updated alongside SGLang releases. + +## Installation +### Prerequisites +- **Rust and Cargo** + ```bash + # Install rustup (Rust installer and version manager) + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + + # Reload shell environment + source "$HOME/.cargo/env" + + # Verify installation + rustc --version + cargo --version + ``` +- **Python** with `pip` and virtualenv tooling available. + +### Rust Binary +```bash +# Build release binary +cargo build --release +``` + +### Python Package +```bash +pip install maturin + +# Fast development mode (debug build, no wheel, instant) +# Uses system OpenSSL (requires libssl-dev/openssl-devel) +cd bindings/python +maturin develop + +# Production build (optimized, creates wheel) +# Uses vendored OpenSSL (cross-platform compatibility) +cd bindings/python +maturin build --release --out dist --features vendored-openssl +pip install --force-reinstall dist/*.whl + +# Development build with system OpenSSL (faster) +# Requires: apt install libssl-dev pkg-config (Ubuntu/Debian) +# or: yum install openssl-devel (RHEL/CentOS) +cd bindings/python +maturin build --release --out dist +pip install --force-reinstall dist/*.whl +``` +> **Note:** Python bindings are located in `bindings/python/` with their own Cargo.toml. Use `maturin develop` for fast iteration during development (builds in debug mode and installs directly). Use `maturin build --release --features vendored-openssl` for production wheels with full optimizations (opt-level="z", lto="fat") and cross-platform compatibility. The package uses abi3 support for Python 3.8+ compatibility. + +## Checking Version + +After installation, verify the installation and check version information: + +```bash +# Simple version (Rust binary) +./target/release/sgl-model-gateway --version +# or use aliases +./target/release/smg --version +./target/release/amg --version + +# Full version info with build details +./target/release/sgl-model-gateway --version-verbose + +# Python CLI +amg --version +amg --version-verbose +python3 -m sglang_router --version +``` + +The `--version` (or `-V`) flag displays the version string. Use `--version-verbose` for comprehensive build information including Git commit, build time, compiler versions, and platform details. + +## Quick Start +### Regular HTTP Routing +- **Rust binary** + ```bash + ./target/release/sgl-model-gateway \ + --worker-urls http://worker1:8000 http://worker2:8000 \ + --policy cache_aware + ``` + `cargo run --release -- …` provides the same behavior during development. +- **Python launcher** + ```bash + python3 -m sglang_router.launch_router \ + --worker-urls http://worker1:8000 http://worker2:8000 \ + --policy cache_aware + ``` + +### Prefill/Decode Disaggregation (PD) +- **Rust binary** + ```bash + ./target/release/sgl-model-gateway \ + --pd-disaggregation \ + --prefill http://prefill1:30001 9001 \ + --prefill http://prefill2:30002 \ + --decode http://decode1:30011 \ + --decode http://decode2:30012 \ + --policy cache_aware \ + --prefill-policy cache_aware \ + --decode-policy power_of_two + ``` +- **Python launcher** + ```bash + python3 -m sglang_router.launch_router \ + --pd-disaggregation \ + --prefill http://prefill1:30001 9001 \ + --prefill http://prefill2:30002 \ + --decode http://decode1:30011 \ + --decode http://decode2:30012 \ + --policy cache_aware + ``` +Prefill entries accept an optional bootstrap port. PD mode merges prefill metadata with decode outputs and streams results back to the client. + +### Multi-Model Inference Gateway +Enable IGW mode to route multiple models through a single router while applying per-model policies: +```bash +./target/release/sgl-model-gateway \ + --enable-igw \ + --policy cache_aware \ + --max-concurrent-requests 512 + +# Register workers dynamically +curl -X POST http://localhost:30000/workers \ + -H "Content-Type: application/json" \ + -d '{ + "url": "http://worker-a:8000", + "model_id": "mistral", + "priority": 10, + "labels": {"tier": "gold"} + }' + +# Add another worker with a different model/policy hint +curl -X POST http://localhost:30000/workers \ + -H "Content-Type: application/json" \ + -d '{ + "url": "http://worker-b:8000", + "model_id": "llama3", + "priority": 20, + "labels": {"policy": "power_of_two", "tier": "silver"} + }' + +# Inspect registered workers +curl http://localhost:30000/workers +``` +Sample response (http workers): +```json +{ + "workers": [ + {"id":"http://0.0.0.0:31378","url":"http://0.0.0.0:31378","model_id":"mistral","priority":50,"cost":1.0,"worker_type":"regular","is_healthy":true,"load":0,"connection_mode":"Http"}, + {"id":"http://0.0.0.0:34881","url":"http://0.0.0.0:34881","model_id":"llama3","priority":50,"cost":1.0,"worker_type":"regular","is_healthy":true,"load":0,"connection_mode":"Http"} + ], + "total": 2, + "stats": { + "prefill_count": 0, + "decode_count": 0, + "regular_count": 2 + } +} +``` +Add more workers with the same API; include optional `labels` (for per-model policies) or `tokenizer_path` / `reasoning_parser` / `tool_parser` fields as needed. `/workers/{url}` exposes queued job status while background jobs finalize registration. + +### gRPC Routing +- **Rust binary** + ```bash + ./target/release/sgl-model-gateway \ + --worker-urls grpc://worker-grpc-0:31001 grpc://worker-grpc-1:31002 \ + --tokenizer-path /path/to/tokenizer.json \ + --reasoning-parser deepseek-r1 \ + --tool-call-parser json + ``` +- **Python router** + ```bash + python3 -m sglang_router.launch_router \ + --worker-urls grpc://127.0.0.1:20000 \ + --model-path meta-llama/Llama-3.1-8B-Instruct \ + --host 0.0.0.0 \ + --port 8080 + ``` +The gRPC router tokenizes inputs locally, supports tool-call parsing, and streams responses. It supports both regular HTTP-equivalent serving and PD (prefill/decode) serving when the worker registry contains PD workers. Provide `--model-path` or `--tokenizer-path` (HuggingFace ID or local directory) whenever connection mode resolves to gRPC. +Use `--reasoning-parser` to select built-in reasoning pipelines (DeepSeek-R1, Qwen3, Step-3, GLM4, etc.) and `--tool-call-parser` for JSON/Pythonic/XML tool contracts in streaming or non-streaming modes. + +### OpenAI Backend Mode +Route requests to OpenAI or OpenAI-compatible endpoints: + +```bash +# Route to OpenAI API +python3 -m sglang_router.launch_router \ + --backend openai \ + --worker-urls https://api.openai.com \ + +# Route to custom OpenAI-compatible endpoint (Gemini, xAI, etc.) +python3 -m sglang_router.launch_router \ + --backend openai \ + --worker-urls http://my-openai-compatible-service:8000 \ +``` + +**Notes** +- OpenAI backend mode acts as a proxy to a single remote endpoint; load balancing is not applied. +- Provide exactly one `--worker-urls` entry per router instance. +- The Rust binary supports the same flags (`./target/release/sgl-model-gateway --backend openai ...`). + +### MCP Integration +The SGL Model Gateway provides native Model Context Protocol (MCP) client integration, enabling tool calling across STDIO, SSE, and Streamable transports. MCP servers are configured via a YAML configuration file and registered at startup through the workflow engine. + +#### Basic Usage +```bash +# Rust binary +./target/release/sgl-model-gateway \ + --mcp-config-path /path/to/mcp-config.yaml \ + --worker-urls http://worker1:8000 + +# Python launcher +python3 -m sglang_router.launch_router \ + --mcp-config-path /path/to/mcp-config.yaml \ + --worker-urls http://worker1:8000 +``` + +#### MCP Configuration File +Create an MCP configuration file to define servers, transports, and connection settings: + +```yaml +servers: + - name: "filesystem" + command: "npx" + args: ["-y", "@modelcontextprotocol/server-filesystem", "/tmp"] + required: false + + - name: "github" + url: "https://api.github.com/mcp" + token: "ghp_xxxxx" + protocol: "sse" + required: false + + - name: "custom-tools" + url: "https://tools.example.com/mcp" + protocol: "streamable" + required: true + +pool: + max_connections: 100 + idle_timeout: 300 # seconds + +proxy: + http: "http://proxy.internal:8080" + https: "https://proxy.internal:8443" + no_proxy: "localhost,127.0.0.1,*.internal" + +inventory: + enable_refresh: true + tool_ttl: 300 # seconds - how long tools are considered fresh + refresh_interval: 300 # seconds - background refresh interval +``` + +#### Configuration Options + +**Server Configuration** (`servers` array): +- `name`: Unique identifier for the MCP server +- `command` + `args`: For STDIO transport (local process execution) +- `url`: For SSE or Streamable transports (HTTP/HTTPS endpoints) +- `token`: Optional authentication token for HTTP-based transports +- `protocol`: Protocol type (`"sse"` or `"streamable"`; STDIO is inferred from `command`) +- `required`: If `true`, router fails to start if server is unreachable (default: `false`) +- `envs`: Environment variables for STDIO processes (optional) +- `proxy`: Per-server proxy override (set to `null` to bypass global proxy) + +**Connection Pool** (`pool`): +- `max_connections`: Maximum pooled connections for dynamic servers (default: 100) +- `idle_timeout`: Idle connection timeout in seconds before cleanup (default: 300) + +**Proxy Configuration** (`proxy`): +- `http`/`https`: Proxy URLs for MCP server connections (not LLM traffic) +- `no_proxy`: Comma-separated hosts to exclude from proxying (supports wildcards) +- **Note**: Proxy settings are currently ignored for `streamable` transport. Use STDIO or SSE transports if proxy support is required. + +**Inventory Settings** (`inventory`): +- `enable_refresh`: Enable automatic background refresh of tool inventory (default: true) +- `tool_ttl`: Tool cache TTL in seconds - how long tools are considered fresh (default: 300) +- `refresh_interval`: Background refresh interval in seconds - proactive inventory refresh (default: 300) + +#### Transport Types + +**STDIO** (Local Process): +```yaml +name: "local-tools" +command: "python" +args: ["-m", "my_mcp_server"] +envs: + API_KEY: "secret" + DEBUG: "true" +``` + +**SSE** (Server-Sent Events): +```yaml +name: "remote-sse" +url: "https://mcp.example.com/events" +token: "bearer-token" +protocol: "sse" +``` + +**Streamable** (Bidirectional Streaming): +```yaml +name: "streaming-tools" +url: "https://mcp.example.com/stream" +protocol: "streamable" +required: true +``` + +#### Server Lifecycle +- MCP servers are registered via the workflow engine with retry logic (100 attempts, 2-hour timeout for STDIO servers) +- Discovery phase identifies tools, prompts, and resources +- Tool inventory is cached with configurable TTL and periodic refresh +- Failed optional servers log warnings; required servers halt startup +- Static servers (from config) are permanent; dynamic servers (per-request) use connection pooling + +Check Prometheus metrics for MCP activity (`mcp_*` metrics) and workflow job status via the admin API. + +### Python Launcher (Router + Workers) +Launch router and SGLang worker processes together; `launch_server` spins up workers (HTTP or gRPC) and the router in one shot. +```bash +python3 -m sglang_router.launch_server --host 0.0.0.0 +``` +Add flags as needed for production deployments: +```bash +python3 -m sglang_router.launch_server \ + --host 0.0.0.0 \ + --port 8080 \ + --model meta-llama/Llama-3.1-8B-Instruct \ + --tp-size 1 \ + --dp-size 8 \ + --grpc-mode +``` +Omit `--grpc-mode` to start HTTP workers; the router automatically configures worker URLs and schedules them based on the provided DP size. + +### Mini Load Balancer (Debug) +```bash +python3 -m sglang_router.launch_router \ + --mini-lb \ + --pd-disaggregation \ + --prefill http://localhost:30001 \ + --decode http://localhost:30011 +``` +MiniLB forwards PD requests using simple random routing and is intended for local debugging only. + +### Running Worker Servers +Use upstream SGLang binaries to start dedicated worker processes. +- **Prefill worker server (gRPC mode)**: + ```bash + python3 -m sglang.launch_server \ + --model meta-llama/Llama-3.1-8B-Instruct \ + --port 20000 \ + --tp-size 1 \ + --grpc-mode + ``` + Remove `--grpc-mode` for HTTP workers. Combine with the router commands above to register the worker via CLI flags or the control-plane API. + +## Control Plane + +### Worker Lifecycle & Job Queue +- `JobQueue` handles asynchronous add/remove operations to avoid blocking clients. +- `WorkerManager` inspects worker metadata (`/get_server_info`, `/get_model_info`), tracks load, and exposes `flush_cache` and `get_loads`. +- Per-worker circuit breakers and health probes keep the registry healthy; load monitor feeds metrics to cache-aware and power-of-two policies. + +### Administrative & Worker APIs +| Method | Path | Description | +|----------|------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| +| `POST` | `/workers` | Queue worker registration (prefill/decode/regular). Body matches `WorkerConfigRequest`. Returns `202 Accepted` while the job queue processes the request. | +| `GET` | `/workers` | List workers with health, load, policy metadata, and queued job status. | +| `GET` | `/workers/{url}` | Inspect a specific worker or job queue entry. | +| `DELETE` | `/workers/{url}` | Queue worker removal. | +| `POST` | `/flush_cache` | Trigger cache flush across HTTP workers with success/failure breakdown. | +| `GET` | `/get_loads` | Sample current load reported by each worker. | + +All administrative routes inherit router API-key protection when `--api-key` is supplied. Job status includes `pending`, `processing`, and `failed` phases with timestamps. + +### Service Discovery +Enable Kubernetes discovery to reconcile workers automatically: +```bash +./target/release/sgl-model-gateway \ + --service-discovery \ + --selector app=sglang-worker role=inference \ + --service-discovery-namespace sglang-system \ + --service-discovery-port 8000 +``` +PD mode accepts dedicated selectors: +```bash +--pd-disaggregation \ +--prefill-selector app=sglang component=prefill \ +--decode-selector app=sglang component=decode \ +--service-discovery +``` +Prefill pods can expose bootstrap ports via the `sglang.ai/bootstrap-port` annotation. RBAC must allow `get`, `list`, and `watch` on pods. + +## Data Plane + +### Router Capabilities (HTTP & gRPC) +Both router stacks: +- Share load-balancing policies (random, round-robin, cache-aware, power-of-two) with DP-aware scheduling, retries, circuit breakers, and rate limiting. +- Record metrics per request, track running load, and integrate with the router-wide policy registry. + +The HTTP router exposes the full OpenAI-compatible surface area (`/generate`, `/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`, `/v1/responses`, `/v1/rerank`, etc.). The gRPC router delivers blazing-fast `/generate` and `/v1/chat/completions` today, with the remaining endpoints returning `501 Not Implemented` until their pipelines are finalised. + +#### HTTP Router specifics +- **Regular router** handles classic single-stage workers with per-model policy overrides. +- **Prefill/Decode router** coordinates disaggregated prefill and decode workers, merges metadata, and manages streaming fan-in. + +#### gRPC Router specifics +- Industry-first fully Rust implementation of an OpenAI-compatible gRPC inference gateway, including tokenizer, reasoning parser, and tool parser execution in-process for maximum throughput. +- Supports both single-stage and PD (prefill/decode) worker topologies; the router automatically selects the appropriate pipeline per model. +- Provides the same `/v1/*` APIs as the HTTP router while streaming tokenized requests/responses directly to SRT gRPC workers. +- Built-in reasoning parsers for DeepSeek, Qwen, Llama, Mistral, GPT-OSS, Step-3, GLM4, Kimi K2, and other structured-thought models. +- Tool-call parsers for JSON, Pythonic, XML, and custom schemas with streaming and non-streaming execution loops. +- Tokenizer factory supporting HuggingFace models, local tokenizer.json files, and chat template overrides (see `src/tokenizer`). +- Explore the code paths in `src/reasoning_parser`, `src/tool_parser`, and `src/tokenizer` for the end-to-end Rust implementations that power gRPC mode. + +### OpenAI Router +- Proxies OpenAI-compatible chat completions and responses APIs, preserving headers and SSE streams end-to-end. +- Supports `/v1/responses` background jobs with cancellation, deletion, and listing input items—enabling agentic, multi-turn orchestration without persisting data at remote vendor endpoints. +- Conversation APIs (`/v1/conversations` and `/v1/conversations/{id}/items`) interact with the configured conversation storage backend for compliant chat-history management. Conversation state lives at the router tier, so the same history can drive different models or MCP loops without leaking data to upstream vendors. +- Chat history, agentic multi-turn `/v1/responses`, and the native MCP client (STDIO/HTTP/SSE/Streamable transports) are designed to satisfy enterprise data-privacy requirements by keeping sensitive state within the router. + +### Request Endpoints +| Endpoint | Notes | +|----------------------------------------------------------------------------------|------------------------------------------------------------| +| `POST /generate` | SGLang generate API. | +| `POST /v1/chat/completions` | OpenAI-compatible chat. Supports streaming and tool calls. | +| `POST /v1/completions` | OpenAI-compatible text completions. | +| `POST /v1/responses` | Create background responses, returns response IDs. | +| `GET /v1/responses/{id}` | Retrieve stored responses. | +| Conversation endpoints (`/v1/conversations`, `/v1/conversations/{id}`, `/v1/conversations/{id}/items`) | Manage chat history. | +| `POST /v1/embeddings` | Forward embedding requests. | +| `POST /v1/rerank`, `POST /rerank` | Ranking APIs. | + +Public health endpoints (`/liveness`, `/readiness`, `/health`, `/health_generate`) reflect registry state; readiness ensures PD workers are paired and IGW has at least one healthy route. + +## Conversations, Responses, and Data Connectors +- `--history-backend memory` (default) stores responses and conversations in-process. +- `--history-backend none` disables persistence while keeping APIs. +- `--history-backend oracle` uses Oracle Autonomous Database; provide credentials via flags or environment variables. +- `--history-backend postgres` uses PostgreSQL Database. +- Conversation item storage mirrors the history backend (Oracle or memory). The same storage powers OpenAI `/responses` and conversation APIs. + +### History Backend (OpenAI Router Mode) +Store conversation and response data for tracking, debugging, or analytics. + +> **Note:** History backends are currently supported only when running with `--backend openai`. gRPC mode support for the `/v1/responses` API is planned. + +#### Available storage options +- **Memory** (default): In-memory storage, fast but ephemeral. +- **None**: No storage, minimal overhead. +- **Oracle**: Persistent storage backed by Oracle Autonomous Database. +- **Postgres**: Persistent storage backed by PostgreSQL Database. + +```bash +# Memory backend (default) +python3 -m sglang_router.launch_router \ + --backend openai \ + --worker-urls https://api.openai.com \ + --history-backend memory + +# No storage for maximum performance +python3 -m sglang_router.launch_router \ + --backend openai \ + --worker-urls https://api.openai.com \ + --history-backend none + +# Oracle ATP backend (see configuration below) +python3 -m sglang_router.launch_router \ + --backend openai \ + --worker-urls https://api.openai.com \ + --history-backend oracle + +# PostgreSQL backend +python3 -m sglang_router.launch_router \ + --backend openai \ + --worker-urls https://api.openai.com \ + --history-backend postgres +``` + +#### Oracle configuration +Install the Oracle Instant Client and set `LD_LIBRARY_PATH` accordingly. Choose **one** connection method: +```bash +# Option 1: Full connection descriptor +export ATP_DSN="(description=(address=(protocol=tcps)(port=1522)(host=adb.region.oraclecloud.com))(connect_data=(service_name=service_name)))" + +# Option 2: TNS alias (requires wallet) +export ATP_TNS_ALIAS="sglroutertestatp_high" +export ATP_WALLET_PATH="/path/to/wallet" +``` +Provide database credentials and optional pool sizing: +```bash +export ATP_USER="admin" +export ATP_PASSWORD="YourPassword123" +export ATP_POOL_MIN=4 +export ATP_POOL_MAX=32 +``` + +Router flags map to these values: +- `--oracle-dsn` (env: `ATP_DSN`) or `--oracle-tns-alias` with `--oracle-wallet-path`. +- `--oracle-user` / `--oracle-password` (`ATP_USER` / `ATP_PASSWORD`). +- `--oracle-wallet-path` (`ATP_WALLET_PATH`) when using TNS alias. +- `--oracle-pool-min`, `--oracle-pool-max`, `--oracle-pool-timeout-secs`. + +Only one of `--oracle-dsn` or `--oracle-tns-alias` should be supplied. + +## Reliability & Flow Control +- **Retries**: Default max retries = 5 with exponential backoff (`--retry-max-retries`, `--retry-initial-backoff-ms`, `--retry-max-backoff-ms`, `--retry-backoff-multiplier`, `--retry-jitter-factor`). Retries trigger on 408/429/500/502/503/504. +- **Circuit Breakers**: Per worker thresholds (`--cb-failure-threshold`, `--cb-success-threshold`, `--cb-timeout-duration-secs`, `--cb-window-duration-secs`). Disable via `--disable-circuit-breaker`. +- **Rate Limiting**: Token bucket driven by `--max-concurrent-requests`. Set `--rate-limit-tokens-per-second` to override refill rate. Configure request queue via `--queue-size` and `--queue-timeout-secs`; queued requests observe FIFO order and respect cancellation. +- **Health Checks**: Runtime probes via `--health-check-interval-secs`, `--health-check-timeout-secs`, failure/success thresholds, and `--health-check-endpoint`. +- **Cache Management**: `/flush_cache` ensures LRU eviction when redeploying PD workers. + +## Load Balancing Policies +- `random`: uniform random worker selection. +- `round_robin`: sequential rotation with atomic counters. +- `cache_aware`: maintains a prefix tree of prompts to route repeat traffic and evens load with configurable thresholds (`--cache-threshold`, `--balance-abs-threshold`, `--balance-rel-threshold`, `--eviction-interval`, `--max-tree-size`). +- `power_of_two`: chooses the lighter worker among two random candidates; integrates with `LoadMonitor`. + Per-model overrides are available in PD mode (`--prefill-policy`, `--decode-policy`) and IGW mode via the worker registry. + +## Observability +- **Logging**: Structured tracing through `tracing` with optional file sink (`--log-dir`) and `--log-level` (`debug`, `info`, `warn`, `error`). +- **Prometheus Metrics**: Enable with `--prometheus-host`/`--prometheus-port` (defaults to `0.0.0.0:29000`). Metrics cover request latency, retry behavior, circuit breaker states, worker health/load, queue depth, PD pipeline stats, tokenizer timings, and MCP activity. +- **Request IDs**: Configurable headers via `--request-id-headers`; responses include `x-request-id`. +- **CORS**: Set `--cors-allowed-origins` for browser access. + +## Security + +### Router and Worker API Keys +- **Router API key (`--api-key`)** protects client access to router endpoints; all protected routes expect `Authorization: Bearer `. +- Workers listed in `--worker-urls` inherit the router API key automatically. +- When adding workers dynamically, provide explicit API keys via payload or query string; they do **not** inherit automatically. + +```bash +# Router and initial workers share the same key +python3 -m sglang_router.launch_router \ + --api-key "shared-api-key" \ + --worker-urls http://worker1:8000 http://worker2:8000 + +# Adding a worker without key while router has one triggers a warning and leaves the worker unprotected +curl -X POST http://localhost:8080/add_worker?url=http://worker3:8000 + +# Add worker with explicit key +curl -X POST "http://localhost:8080/add_worker?url=http://worker3:8000&api_key=worker3-specific-key" +``` + +### Security Configurations +1. **No Authentication** (default): Router and workers accept requests without keys—use only in trusted environments. +2. **Router-only Authentication**: Provide `--api-key`; clients must present the key, router accesses workers without credentials. +3. **Worker-only Authentication**: Router open to clients; each worker requires its own key. Supply keys when calling `/workers` or `/add_worker`. +4. **Full Authentication**: Set router API key and provide per-worker keys. Example: + ```bash + python3 -m sglang_router.launch_router --api-key "router-key" + curl -H "Authorization: Bearer router-key" \ + -X POST http://localhost:8080/add_worker?url=http://worker:8000&api_key=worker-key + ``` + +### Important Notes +- Initial workers declared via CLI inherit the router key; dynamic workers must supply keys explicitly. +- Router logs a warning when a worker is registered without a key while the router expects authentication. +- When router and workers share the same key, still include the key when invoking dynamic registration APIs. + +## Development & Testing +```bash +# Build Rust components (debug mode, fast) +cargo build + +# Run Rust tests +cargo test + +# Fast Python development (rebuilds and installs in debug mode) +cd bindings/python && maturin develop + +# Run Python tests +cd ../.. # Back to sgl-router root +pytest py_test/ +``` +For production builds, use `maturin build --release --out dist` from the `bindings/python/` directory to create optimized wheels. During development, `maturin develop` rebuilds and installs instantly without creating wheel files. Use `python -m sglang_router.launch_server` to co-launch router and SGLang workers in small clusters for local validation. + +--- + +## Release Management + +### Creating Gateway Releases + +Create releases for the Gateway/Router component with filtered commits: + +```bash +# Using make +make release-notes PREV=gateway-v0.2.2 CURR=gateway-v1.0.0 + +# Save to file +make release-notes PREV=gateway-v0.2.2 CURR=gateway-v1.0.0 OUTPUT=RELEASE_NOTES.md + +# Create draft release (requires gh CLI, DEFAULT behavior) +make release-notes PREV=gateway-v0.2.2 CURR=gateway-v1.0.0 CREATE_RELEASE=1 + +# Publish release immediately (requires gh CLI) +make release-notes PREV=gateway-v0.2.2 CURR=gateway-v1.0.0 CREATE_RELEASE=1 DRAFT=0 +``` + +**Tag Naming**: Use `gateway-*` or `router-*` prefixes to avoid triggering unrelated CI workflows. + +### Release Workflow + +1. **Create and push tag**: + ```bash + git tag -a gateway-v1.0.0 -m "Gateway release v1.0.0" + git push origin gateway-v1.0.0 + ``` + +2. **Generate release notes** (automatically filters gateway-related commits): + ```bash + make release-notes PREV=gateway-v0.2.2 CURR=gateway-v1.0.0 + ``` + +3. **Create GitHub release**: + ```bash + # Create draft (DEFAULT - review before publishing) + make release-notes PREV=gateway-v0.2.2 CURR=gateway-v1.0.0 CREATE_RELEASE=1 + + # Or publish immediately (skip draft) + make release-notes PREV=gateway-v0.2.2 CURR=gateway-v1.0.0 CREATE_RELEASE=1 DRAFT=0 + ``` + +### Filtered Paths + +Release notes only include commits touching: +- `sgl-router/` - Router codebase +- `python/sglang/srt/grpc/` - gRPC protocol +- `python/sglang/srt/entrypoints/grpc_server.py` - gRPC server + +The script automatically extracts author attribution, PR links, and identifies new contributors. + +--- + +SGLang Model Gateway continues to evolve alongside the core SGLang runtime. Contributions should keep CLI flags, documentation, and Python bindings in sync with the Rust implementation. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/benches/request_processing.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/benches/request_processing.rs new file mode 100644 index 00000000..60d51faf --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/benches/request_processing.rs @@ -0,0 +1,670 @@ +use std::time::Instant; + +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use serde_json::{from_str, to_string, to_value, to_vec}; +use sgl_model_gateway::{ + core::{BasicWorker, BasicWorkerBuilder, Worker, WorkerType}, + protocols::{ + chat::{ChatCompletionRequest, ChatMessage, MessageContent}, + common::StringOrArray, + completion::CompletionRequest, + generate::GenerateRequest, + sampling_params::SamplingParams, + }, + routers::http::pd_types::{generate_room_id, RequestWithBootstrap}, +}; + +fn create_test_worker() -> BasicWorker { + BasicWorkerBuilder::new("http://test-server:8000") + .worker_type(WorkerType::Prefill { + bootstrap_port: Some(5678), + }) + .build() +} + +// Helper function to get bootstrap info from worker +fn get_bootstrap_info(worker: &BasicWorker) -> (String, Option) { + let hostname = worker.bootstrap_host().to_string(); + let bootstrap_port = worker.bootstrap_port(); + (hostname, bootstrap_port) +} + +/// Create a default GenerateRequest for benchmarks with minimal fields set +fn default_generate_request() -> GenerateRequest { + GenerateRequest { + text: None, + model: None, + input_ids: None, + input_embeds: None, + image_data: None, + video_data: None, + audio_data: None, + sampling_params: None, + return_logprob: None, + logprob_start_len: None, + top_logprobs_num: None, + token_ids_logprob: None, + return_text_in_logprobs: false, + stream: false, + log_metrics: true, + return_hidden_states: false, + modalities: None, + session_params: None, + lora_path: None, + lora_id: None, + custom_logit_processor: None, + bootstrap_host: None, + bootstrap_port: None, + bootstrap_room: None, + bootstrap_pair_key: None, + data_parallel_rank: None, + background: false, + conversation_id: None, + priority: None, + extra_key: None, + no_logs: false, + custom_labels: None, + return_bytes: false, + return_entropy: false, + rid: None, + } +} + +/// Create a default ChatCompletionRequest for benchmarks with minimal fields set +#[allow(deprecated)] +fn default_chat_completion_request() -> ChatCompletionRequest { + ChatCompletionRequest { + // Required fields in OpenAI order + messages: vec![], + model: String::new(), + + // Use default for all other fields + ..Default::default() + } +} + +/// Create a default CompletionRequest for benchmarks with minimal fields set +fn default_completion_request() -> CompletionRequest { + CompletionRequest { + model: String::new(), + prompt: StringOrArray::String(String::new()), + suffix: None, + max_tokens: None, + temperature: None, + top_p: None, + n: None, + stream: false, + stream_options: None, + logprobs: None, + echo: false, + stop: None, + presence_penalty: None, + frequency_penalty: None, + best_of: None, + logit_bias: None, + user: None, + seed: None, + // SGLang Extensions + top_k: None, + min_p: None, + min_tokens: None, + repetition_penalty: None, + regex: None, + ebnf: None, + json_schema: None, + stop_token_ids: None, + no_stop_trim: false, + ignore_eos: false, + skip_special_tokens: true, + // SGLang Extensions + lora_path: None, + session_params: None, + return_hidden_states: false, + sampling_seed: None, + other: serde_json::Map::new(), + } +} + +// Sample request data for benchmarks +fn create_sample_generate_request() -> GenerateRequest { + GenerateRequest { + text: Some("Write a story about artificial intelligence".to_string()), + sampling_params: Some(SamplingParams { + max_new_tokens: Some(100), + temperature: Some(0.8), + top_p: Some(0.9), + top_k: Some(50), + frequency_penalty: Some(0.0), + presence_penalty: Some(0.0), + repetition_penalty: Some(1.0), + ..Default::default() + }), + ..default_generate_request() + } +} + +#[allow(deprecated)] +fn create_sample_chat_completion_request() -> ChatCompletionRequest { + ChatCompletionRequest { + model: "gpt-3.5-turbo".to_string(), + messages: vec![ + ChatMessage::System { + content: MessageContent::Text("You are a helpful assistant".to_string()), + name: None, + }, + ChatMessage::User { + content: MessageContent::Text( + "Explain quantum computing in simple terms".to_string(), + ), + name: None, + }, + ], + max_tokens: Some(150), + max_completion_tokens: Some(150), + temperature: Some(0.7), + top_p: Some(1.0), + n: Some(1), + presence_penalty: Some(0.0), + frequency_penalty: Some(0.0), + parallel_tool_calls: Some(true), + ..default_chat_completion_request() + } +} + +fn create_sample_completion_request() -> CompletionRequest { + CompletionRequest { + model: "text-davinci-003".to_string(), + prompt: StringOrArray::String("Complete this sentence: The future of AI is".to_string()), + max_tokens: Some(50), + temperature: Some(0.8), + top_p: Some(1.0), + n: Some(1), + presence_penalty: Some(0.0), + frequency_penalty: Some(0.0), + best_of: Some(1), + ..default_completion_request() + } +} + +#[allow(deprecated)] +fn create_large_chat_completion_request() -> ChatCompletionRequest { + let mut messages = vec![ChatMessage::System { + content: MessageContent::Text( + "You are a helpful assistant with extensive knowledge.".to_string(), + ), + name: None, + }]; + + // Add many user/assistant pairs to simulate a long conversation + for i in 0..50 { + messages.push(ChatMessage::User { + content: MessageContent::Text(format!("Question {}: What do you think about topic number {} which involves complex reasoning about multiple interconnected systems and their relationships?", i, i)), + name: None, + }); + messages.push(ChatMessage::Assistant { + content: Some(MessageContent::Text(format!("Answer {}: This is a detailed response about topic {} that covers multiple aspects and provides comprehensive analysis of the interconnected systems you mentioned.", i, i))), + name: None, + tool_calls: None, + reasoning_content: None, + }); + } + + ChatCompletionRequest { + model: "gpt-4".to_string(), + messages, + max_tokens: Some(1000), + max_completion_tokens: Some(1000), + temperature: Some(0.7), + top_p: Some(0.95), + n: Some(1), + presence_penalty: Some(0.1), + frequency_penalty: Some(0.1), + top_logprobs: Some(5), + seed: Some(42), + parallel_tool_calls: Some(true), + ..default_chat_completion_request() + } +} + +// Benchmark JSON serialization +fn bench_json_serialization(c: &mut Criterion) { + let mut group = c.benchmark_group("json_serialization"); + + let generate_req = create_sample_generate_request(); + let chat_req = create_sample_chat_completion_request(); + let completion_req = create_sample_completion_request(); + let large_chat_req = create_large_chat_completion_request(); + + group.bench_function("generate_request", |b| { + b.iter(|| { + let json = to_string(black_box(&generate_req)).unwrap(); + black_box(json); + }); + }); + + group.bench_function("chat_completion_request", |b| { + b.iter(|| { + let json = to_string(black_box(&chat_req)).unwrap(); + black_box(json); + }); + }); + + group.bench_function("completion_request", |b| { + b.iter(|| { + let json = to_string(black_box(&completion_req)).unwrap(); + black_box(json); + }); + }); + + group.bench_function("large_chat_completion_request", |b| { + b.iter(|| { + let json = to_string(black_box(&large_chat_req)).unwrap(); + black_box(json); + }); + }); + + group.bench_function("generate_request_to_bytes", |b| { + b.iter(|| { + let bytes = to_vec(black_box(&generate_req)).unwrap(); + black_box(bytes); + }); + }); + + group.finish(); +} + +// Benchmark JSON deserialization +fn bench_json_deserialization(c: &mut Criterion) { + let mut group = c.benchmark_group("json_deserialization"); + + let generate_json = to_string(&create_sample_generate_request()).unwrap(); + let chat_json = to_string(&create_sample_chat_completion_request()).unwrap(); + let completion_json = to_string(&create_sample_completion_request()).unwrap(); + let large_chat_json = to_string(&create_large_chat_completion_request()).unwrap(); + + group.bench_function("generate_request", |b| { + b.iter(|| { + let req: GenerateRequest = from_str(black_box(&generate_json)).unwrap(); + black_box(req); + }); + }); + + group.bench_function("chat_completion_request", |b| { + b.iter(|| { + let req: ChatCompletionRequest = from_str(black_box(&chat_json)).unwrap(); + black_box(req); + }); + }); + + group.bench_function("completion_request", |b| { + b.iter(|| { + let req: CompletionRequest = from_str(black_box(&completion_json)).unwrap(); + black_box(req); + }); + }); + + group.bench_function("large_chat_completion_request", |b| { + b.iter(|| { + let req: ChatCompletionRequest = from_str(black_box(&large_chat_json)).unwrap(); + black_box(req); + }); + }); + + group.finish(); +} + +// Benchmark bootstrap injection (replaces request adaptation) +fn bench_bootstrap_injection(c: &mut Criterion) { + let mut group = c.benchmark_group("bootstrap_injection"); + + let generate_req = create_sample_generate_request(); + let chat_req = create_sample_chat_completion_request(); + let completion_req = create_sample_completion_request(); + let large_chat_req = create_large_chat_completion_request(); + let worker = create_test_worker(); + let (hostname, bootstrap_port) = get_bootstrap_info(&worker); + + group.bench_function("generate_bootstrap_injection", |b| { + b.iter(|| { + let request_with_bootstrap = RequestWithBootstrap { + original: &generate_req, + bootstrap_host: hostname.clone(), + bootstrap_port, + bootstrap_room: generate_room_id(), + }; + let json = to_value(black_box(&request_with_bootstrap)).unwrap(); + black_box(json); + }); + }); + + group.bench_function("chat_completion_bootstrap_injection", |b| { + b.iter(|| { + let request_with_bootstrap = RequestWithBootstrap { + original: &chat_req, + bootstrap_host: hostname.clone(), + bootstrap_port, + bootstrap_room: generate_room_id(), + }; + let json = to_value(black_box(&request_with_bootstrap)).unwrap(); + black_box(json); + }); + }); + + group.bench_function("completion_bootstrap_injection", |b| { + b.iter(|| { + let request_with_bootstrap = RequestWithBootstrap { + original: &completion_req, + bootstrap_host: hostname.clone(), + bootstrap_port, + bootstrap_room: generate_room_id(), + }; + let json = to_value(black_box(&request_with_bootstrap)).unwrap(); + black_box(json); + }); + }); + + group.bench_function("large_chat_completion_bootstrap_injection", |b| { + b.iter(|| { + let request_with_bootstrap = RequestWithBootstrap { + original: &large_chat_req, + bootstrap_host: hostname.clone(), + bootstrap_port, + bootstrap_room: generate_room_id(), + }; + let json = to_value(black_box(&request_with_bootstrap)).unwrap(); + black_box(json); + }); + }); + + group.finish(); +} + +// Benchmark direct JSON routing (replaces regular routing) +fn bench_direct_json_routing(c: &mut Criterion) { + let mut group = c.benchmark_group("direct_json_routing"); + + let generate_req = create_sample_generate_request(); + let chat_req = create_sample_chat_completion_request(); + let completion_req = create_sample_completion_request(); + + group.bench_function("generate_to_json", |b| { + b.iter(|| { + let json = to_value(black_box(&generate_req)).unwrap(); + black_box(json); + }); + }); + + group.bench_function("generate_to_json_string", |b| { + b.iter(|| { + let json = to_string(black_box(&generate_req)).unwrap(); + black_box(json); + }); + }); + + group.bench_function("generate_to_bytes", |b| { + b.iter(|| { + let bytes = to_vec(black_box(&generate_req)).unwrap(); + black_box(bytes); + }); + }); + + group.bench_function("chat_completion_to_json", |b| { + b.iter(|| { + let json = to_value(black_box(&chat_req)).unwrap(); + black_box(json); + }); + }); + + group.bench_function("chat_completion_to_json_string", |b| { + b.iter(|| { + let json = to_string(black_box(&chat_req)).unwrap(); + black_box(json); + }); + }); + + group.bench_function("completion_to_json", |b| { + b.iter(|| { + let json = to_value(black_box(&completion_req)).unwrap(); + black_box(json); + }); + }); + + group.finish(); +} + +// Benchmark throughput with different request sizes +fn bench_throughput_by_size(c: &mut Criterion) { + let mut group = c.benchmark_group("throughput_by_size"); + + // Create requests of different sizes + let small_generate = GenerateRequest { + text: Some("Hi".to_string()), + ..default_generate_request() + }; + + let medium_generate = GenerateRequest { + text: Some("Write a medium length story about AI".repeat(10)), + ..default_generate_request() + }; + + let large_generate = GenerateRequest { + text: Some("Write a very long and detailed story about artificial intelligence and its impact on society".repeat(100)), + ..default_generate_request() + }; + + let worker = create_test_worker(); + let (hostname, bootstrap_port) = get_bootstrap_info(&worker); + + for (name, req) in [ + ("small", &small_generate), + ("medium", &medium_generate), + ("large", &large_generate), + ] { + let json = to_string(req).unwrap(); + let size_bytes = json.len(); + let hostname_clone = hostname.clone(); + + group.throughput(Throughput::Bytes(size_bytes as u64)); + group.bench_with_input(BenchmarkId::new("serialize", name), &req, |b, req| { + b.iter(|| { + let json = to_string(black_box(req)).unwrap(); + black_box(json); + }); + }); + + group.bench_with_input( + BenchmarkId::new("deserialize", name), + &json, + |b, json_str| { + b.iter(|| { + let req: GenerateRequest = black_box(from_str(json_str)).unwrap(); + black_box(req); + }); + }, + ); + + group.bench_with_input( + BenchmarkId::new("bootstrap_inject", name), + &req, + move |b, req| { + let hostname = hostname_clone.clone(); + b.iter(|| { + let request_with_bootstrap = RequestWithBootstrap { + original: req, + bootstrap_host: hostname.clone(), + bootstrap_port, + bootstrap_room: generate_room_id(), + }; + let json = to_value(&request_with_bootstrap).unwrap(); + black_box(json); + }); + }, + ); + } + + group.finish(); +} + +// Benchmark full round-trip: deserialize -> inject bootstrap -> serialize +fn bench_full_round_trip(c: &mut Criterion) { + let mut group = c.benchmark_group("full_round_trip"); + + let generate_json = to_string(&create_sample_generate_request()).unwrap(); + let chat_json = to_string(&create_sample_chat_completion_request()).unwrap(); + let completion_json = to_string(&create_sample_completion_request()).unwrap(); + let worker = create_test_worker(); + let (hostname, bootstrap_port) = get_bootstrap_info(&worker); + + group.bench_function("generate_openai_to_pd_pipeline", |b| { + b.iter(|| { + // Deserialize OpenAI request + let req: GenerateRequest = from_str(black_box(&generate_json)).unwrap(); + // Create wrapper with bootstrap fields + let request_with_bootstrap = RequestWithBootstrap { + original: &req, + bootstrap_host: hostname.clone(), + bootstrap_port, + bootstrap_room: generate_room_id(), + }; + // Serialize final request + let pd_json = to_string(&request_with_bootstrap).unwrap(); + black_box(pd_json); + }); + }); + + group.bench_function("chat_completion_openai_to_pd_pipeline", |b| { + b.iter(|| { + let req: ChatCompletionRequest = from_str(black_box(&chat_json)).unwrap(); + let request_with_bootstrap = RequestWithBootstrap { + original: &req, + bootstrap_host: hostname.clone(), + bootstrap_port, + bootstrap_room: generate_room_id(), + }; + let pd_json = to_string(&request_with_bootstrap).unwrap(); + black_box(pd_json); + }); + }); + + group.bench_function("completion_openai_to_pd_pipeline", |b| { + b.iter(|| { + let req: CompletionRequest = from_str(black_box(&completion_json)).unwrap(); + let request_with_bootstrap = RequestWithBootstrap { + original: &req, + bootstrap_host: hostname.clone(), + bootstrap_port, + bootstrap_room: generate_room_id(), + }; + let pd_json = to_string(&request_with_bootstrap).unwrap(); + black_box(pd_json); + }); + }); + + group.bench_function("generate_direct_json_pipeline", |b| { + b.iter(|| { + // Deserialize OpenAI request + let req: GenerateRequest = from_str(black_box(&generate_json)).unwrap(); + // Convert to JSON for direct routing (no bootstrap injection) + let routing_json = to_value(&req).unwrap(); + let json_string = to_string(&routing_json).unwrap(); + black_box(json_string); + }); + }); + + group.finish(); +} + +fn benchmark_summary(c: &mut Criterion) { + let group = c.benchmark_group("benchmark_summary"); + + println!("\nSGLang Router Performance Benchmark Suite"); + println!("============================================="); + + // Quick performance overview + let generate_req = create_sample_generate_request(); + let worker = create_test_worker(); + + println!("\nQuick Performance Overview:"); + + // Measure serialization + let start = Instant::now(); + for _ in 0..1000 { + let _ = black_box(to_string(&generate_req).unwrap()); + } + let serialize_time = start.elapsed().as_nanos() / 1000; + println!(" * Serialization (avg): {:>8} ns/req", serialize_time); + + // Measure deserialization + let json = to_string(&generate_req).unwrap(); + let start = Instant::now(); + for _ in 0..1000 { + let _: GenerateRequest = black_box(from_str(&json).unwrap()); + } + let deserialize_time = start.elapsed().as_nanos() / 1000; + println!( + " * Deserialization (avg): {:>8} ns/req", + deserialize_time + ); + + // Measure bootstrap injection (replaces adaptation) + let (hostname, bootstrap_port) = get_bootstrap_info(&worker); + let start = Instant::now(); + for _ in 0..1000 { + let request_with_bootstrap = RequestWithBootstrap { + original: &generate_req, + bootstrap_host: hostname.clone(), + bootstrap_port, + bootstrap_room: generate_room_id(), + }; + let _ = black_box(to_value(&request_with_bootstrap).unwrap()); + } + let inject_time = start.elapsed().as_nanos() / 1000; + println!(" * Bootstrap Injection (avg): {:>6} ns/req", inject_time); + + // Calculate ratios + let total_pipeline = serialize_time + deserialize_time + inject_time; + println!(" * Total Pipeline (avg): {:>8} ns/req", total_pipeline); + + println!("\nPerformance Insights:"); + if deserialize_time > serialize_time * 2 { + println!(" • Deserialization is significantly faster than serialization"); + } + if inject_time < serialize_time / 10 { + println!( + " • Bootstrap injection overhead is negligible ({:.1}% of serialization)", + (inject_time as f64 / serialize_time as f64) * 100.0 + ); + } + if total_pipeline < 100_000 { + println!(" • Total pipeline latency is excellent (< 100μs)"); + } + + println!("\nSimplification Benefits:"); + println!(" • Eliminated complex type conversion layer"); + println!(" • Reduced memory allocations"); + println!(" • Automatic field preservation (no manual mapping)"); + println!(" • Direct JSON manipulation improves performance"); + + println!("\nRecommendations:"); + if serialize_time > deserialize_time { + println!(" • Focus optimization efforts on serialization rather than deserialization"); + } + println!(" • PD mode overhead is minimal - safe to use for latency-sensitive workloads"); + println!(" • Consider batching small requests to improve overall throughput"); + + println!("\n{}", "=".repeat(50)); + + group.finish(); +} + +criterion_group!( + benches, + benchmark_summary, + bench_json_serialization, + bench_json_deserialization, + bench_bootstrap_injection, + bench_direct_json_routing, + bench_throughput_by_size, + bench_full_round_trip +); +criterion_main!(benches); diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/benches/tokenizer_benchmark.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/benches/tokenizer_benchmark.rs new file mode 100644 index 00000000..c53ee300 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/benches/tokenizer_benchmark.rs @@ -0,0 +1,1890 @@ +//! Comprehensive tokenizer benchmark with clean summary output +//! Each test adds a row to the final summary table + +use std::{ + collections::BTreeMap, + path::PathBuf, + sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, Mutex, OnceLock, + }, + thread, + time::{Duration, Instant}, +}; + +use criterion::{black_box, criterion_group, BenchmarkId, Criterion, Throughput}; +use sgl_model_gateway::tokenizer::{ + cache::{CacheConfig, CachedTokenizer}, + huggingface::HuggingFaceTokenizer, + sequence::Sequence, + stop::*, + stream::DecodeStream, + traits::*, +}; + +// Cache the tokenizer path for the entire benchmark run +static TOKENIZER_PATH: OnceLock = OnceLock::new(); + +fn get_tokenizer_path() -> &'static PathBuf { + TOKENIZER_PATH.get_or_init(|| { + // Use Qwen3-4B-Instruct which has ChatML special tokens (<|im_start|>, <|im_end|>) + // with special: true, normalized: false - perfect for demonstrating L1 cache + let rt = tokio::runtime::Runtime::new().expect("Failed to create tokio runtime"); + let tokenizer_dir = rt.block_on(async { + sgl_model_gateway::tokenizer::hub::download_tokenizer_from_hf( + "Qwen/Qwen3-4B-Instruct-2507", + ) + .await + .expect("Failed to download Qwen3-4B-Instruct tokenizer from HuggingFace") + }); + + // The download_tokenizer_from_hf returns the directory containing tokenizer.json + // We need to construct the full path to tokenizer.json + tokenizer_dir.join("tokenizer.json") + }) +} + +// Production target: 100k tokens per second +const TARGET_TOKENS_PER_SECOND: u64 = 100_000; + +// Typical prompt sizes +const SHORT_PROMPT: &str = "What is the capital of France?"; +const MEDIUM_PROMPT: &str = "Write a detailed explanation of quantum computing, including its principles, current applications, and future potential. Be sure to cover both the theoretical foundations and practical implementations."; +const LONG_PROMPT: &str = "You are an expert software engineer. Review the following code and provide detailed feedback on performance optimizations, potential bugs, and architectural improvements. Consider scalability, maintainability, and best practices. The code implements a distributed caching system with the following requirements: 1) High availability across multiple regions, 2) Sub-millisecond latency for cache hits, 3) Automatic failover and recovery, 4) Support for both LRU and LFU eviction policies, 5) Real-time monitoring and alerting. Please analyze each component thoroughly and suggest concrete improvements with code examples where appropriate."; + +// System prompts can be quite large +fn generate_system_prompt(size: usize) -> String { + let base = "You are a helpful AI assistant with expertise in "; + let domains = vec![ + "mathematics", + "physics", + "chemistry", + "biology", + "computer science", + "engineering", + "medicine", + "law", + "economics", + "philosophy", + ]; + + let mut prompt = base.to_string(); + while prompt.len() < size { + for domain in &domains { + prompt.push_str(domain); + prompt.push_str(", "); + if prompt.len() >= size { + break; + } + } + } + prompt +} + +// Global results storage +lazy_static::lazy_static! { + static ref BENCHMARK_RESULTS: Mutex> = Mutex::new(BTreeMap::new()); +} + +fn add_result(category: &str, result: String) { + let mut results = BENCHMARK_RESULTS.lock().unwrap(); + let index = results.len(); + results.insert(format!("{:03}_{}", index, category), result); +} + +fn bench_encode_throughput(c: &mut Criterion) { + let tokenizer_path = get_tokenizer_path(); + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(tokenizer_path.to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + // Pre-generate system prompts + let system_1k = generate_system_prompt(1000); + let system_4k = generate_system_prompt(4000); + let system_16k = generate_system_prompt(16000); + + let test_cases = vec![ + ("short_30B", SHORT_PROMPT), + ("medium_230B", MEDIUM_PROMPT), + ("long_670B", LONG_PROMPT), + ("system_1KB", system_1k.as_str()), + ("system_4KB", system_4k.as_str()), + ("system_16KB", system_16k.as_str()), + ]; + + let mut group = c.benchmark_group("encode_throughput"); + + for (name, prompt) in test_cases { + let prompt_len = prompt.len(); + let tokenizer_clone = tokenizer.clone(); + + // Get token count once + let encoding = tokenizer.encode(prompt).unwrap(); + let token_count = encoding.token_ids().len(); + + // Track if metrics have been printed for this test case + let printed = Arc::new(AtomicBool::new(false)); + + group.throughput(Throughput::Bytes(prompt_len as u64)); + group.bench_function(name, |b| { + let printed_clone = printed.clone(); + let tokenizer = tokenizer_clone.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + black_box(tokenizer.encode(prompt).unwrap()); + } + let duration = start.elapsed(); + + // Store result only once per test case + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let chars_per_sec = (iters as f64 * prompt_len as f64) / duration.as_secs_f64(); + let tokens_per_sec = + (iters as f64 * token_count as f64) / duration.as_secs_f64(); + + let result = format!( + "{:<15} | {:>8} | {:>8} | {:>12.0} | {:>12.0} | {:>10.0} | {:>10}", + name, + prompt_len, + token_count, + chars_per_sec, + tokens_per_sec, + ops_per_sec, + 1 + ); + add_result("encode", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + } + + group.finish(); +} + +fn bench_batch_encode(c: &mut Criterion) { + let tokenizer_path = get_tokenizer_path(); + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(tokenizer_path.to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + let batch_sizes = vec![1, 8, 16, 32, 64, 128]; + let prompt = MEDIUM_PROMPT; + let prompt_len = prompt.len(); + let encoding = tokenizer.encode(prompt).unwrap(); + let token_count = encoding.token_ids().len(); + + let mut group = c.benchmark_group("batch_encode"); + + for batch_size in batch_sizes { + let prompts: Vec<&str> = vec![prompt; batch_size]; + let printed = Arc::new(AtomicBool::new(false)); + let tokenizer_clone = tokenizer.clone(); + + group.throughput(Throughput::Elements(batch_size as u64)); + group.bench_with_input( + BenchmarkId::from_parameter(batch_size), + &batch_size, + |b, &size| { + let printed_clone = printed.clone(); + let tokenizer = tokenizer_clone.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + black_box(tokenizer.encode_batch(&prompts).unwrap()); + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let prompts_per_sec = (iters as f64 * size as f64) / duration.as_secs_f64(); + let tokens_per_sec = prompts_per_sec * token_count as f64; + let chars_per_sec = prompts_per_sec * prompt_len as f64; + + let result = format!( + "{:<15} | {:>8} | {:>8} | {:>12.0} | {:>12.0} | {:>10.0} | {:>10}", + format!("batch_{}", size), + prompt_len * size, + token_count * size, + prompts_per_sec, + tokens_per_sec, + chars_per_sec, + 1 + ); + add_result("batch", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }, + ); + } + + group.finish(); +} + +fn bench_concurrent_encode(c: &mut Criterion) { + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(get_tokenizer_path().to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + let client_counts = vec![1, 4, 8, 16, 32]; + + let mut group = c.benchmark_group("concurrent_encode"); + group.measurement_time(Duration::from_secs(2)); + + for num_clients in client_counts { + let printed = Arc::new(AtomicBool::new(false)); + let tokenizer_clone = tokenizer.clone(); + + group.bench_with_input( + BenchmarkId::from_parameter(num_clients), + &num_clients, + |b, &clients| { + let printed_clone = printed.clone(); + + b.iter_custom(|_iters| { + let tokenizer = tokenizer_clone.clone(); + let total_operations = Arc::new(AtomicU64::new(0)); + let total_chars = Arc::new(AtomicU64::new(0)); + let start = Instant::now(); + + let handles: Vec<_> = (0..clients) + .map(|client_id| { + let tokenizer = tokenizer.clone(); + let total_ops = total_operations.clone(); + let total_ch = total_chars.clone(); + + thread::spawn(move || { + let prompts = [SHORT_PROMPT, MEDIUM_PROMPT, LONG_PROMPT]; + let prompt = prompts[client_id % prompts.len()]; + let mut local_ops = 0u64; + let mut local_chars = 0u64; + + while start.elapsed() < Duration::from_millis(500) { + let _ = tokenizer.encode(prompt).unwrap(); + local_ops += 1; + local_chars += prompt.len() as u64; + } + + total_ops.fetch_add(local_ops, Ordering::Relaxed); + total_ch.fetch_add(local_chars, Ordering::Relaxed); + }) + }) + .collect(); + + for handle in handles { + handle.join().unwrap(); + } + + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let total_ops = total_operations.load(Ordering::Relaxed); + let total_ch = total_chars.load(Ordering::Relaxed); + let ops_per_sec = total_ops as f64 / duration.as_secs_f64(); + let chars_per_sec = total_ch as f64 / duration.as_secs_f64(); + let per_client = ops_per_sec / clients as f64; + + let result = format!( + "{:<15} | {:>10} | {:>12.0} | {:>12.0} | {:>15.0}", + format!("{}_clients", clients), + total_ops, + ops_per_sec, + chars_per_sec, + per_client + ); + add_result("concurrent", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }, + ); + } + + group.finish(); +} + +fn bench_decode_performance(c: &mut Criterion) { + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(get_tokenizer_path().to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + let test_text = "The quick brown fox jumps over the lazy dog. ".repeat(10); + let encoding = tokenizer.encode(&test_text).unwrap(); + let tokens = encoding.token_ids(); + let num_tokens = tokens.len(); + + let mut group = c.benchmark_group("decode_performance"); + + // Test direct decode + let printed_direct = Arc::new(AtomicBool::new(false)); + group.bench_function("direct_decode", |b| { + let printed = printed_direct.clone(); + let tokenizer = tokenizer.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + black_box(tokenizer.decode(tokens, false).unwrap()); + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let tokens_per_sec = ops_per_sec * num_tokens as f64; + + let result = format!( + "{:<20} | {:>10} | {:>12.0} | {:>12.0} | {:>10}", + "Direct", num_tokens, tokens_per_sec, ops_per_sec, 1 + ); + add_result("decode", result); + + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + // Test DecodeStream + let printed_stream = Arc::new(AtomicBool::new(false)); + group.bench_function("decode_stream", |b| { + let printed = printed_stream.clone(); + let tokenizer = tokenizer.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let mut decoder = DecodeStream::new(tokenizer.clone(), &[], false); + let mut output = String::new(); + for token in tokens { + if let Some(text) = decoder.step(*token).unwrap() { + output.push_str(&text); + } + } + black_box(output); + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let tokens_per_sec = ops_per_sec * num_tokens as f64; + + let result = format!( + "{:<20} | {:>10} | {:>12.0} | {:>12.0} | {:>10}", + "DecodeStream", num_tokens, tokens_per_sec, ops_per_sec, 1 + ); + add_result("decode", result); + + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + // Test Sequence + let printed_seq = Arc::new(AtomicBool::new(false)); + group.bench_function("sequence_decode", |b| { + let printed = printed_seq.clone(); + let tokenizer = tokenizer.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let mut sequence = Sequence::new(tokenizer.clone()); + let mut output = String::new(); + for token in tokens { + let text = sequence.append_token(*token).unwrap(); + output.push_str(&text); + } + black_box(output); + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let tokens_per_sec = ops_per_sec * num_tokens as f64; + + let result = format!( + "{:<20} | {:>10} | {:>12.0} | {:>12.0} | {:>10}", + "Sequence", num_tokens, tokens_per_sec, ops_per_sec, 1 + ); + add_result("decode", result); + + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + group.finish(); +} + +fn bench_streaming_decode_100k(c: &mut Criterion) { + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(get_tokenizer_path().to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + let sample_text = "The quick brown fox jumps over the lazy dog. ".repeat(1000); + let encoding = tokenizer.encode(&sample_text).unwrap(); + let all_tokens = encoding.token_ids(); + + let mut group = c.benchmark_group("streaming_100k"); + group.measurement_time(Duration::from_secs(1)); + + // Test DecodeStream + let printed_stream = Arc::new(AtomicBool::new(false)); + group.bench_function("decode_stream_100k", |b| { + let printed = printed_stream.clone(); + let tokenizer = tokenizer.clone(); + + b.iter_custom(|_iters| { + let start = Instant::now(); + let mut decoder = DecodeStream::new(tokenizer.clone(), &[], false); + let mut output = String::new(); + let mut tokens_processed = 0u64; + + for token in all_tokens.iter().cycle() { + if start.elapsed() >= Duration::from_millis(500) { + break; + } + + if let Some(text) = decoder.step(*token).unwrap() { + output.push_str(&text); + } + tokens_processed += 1; + } + + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let tokens_per_sec = tokens_processed as f64 / duration.as_secs_f64(); + let status = if tokens_per_sec >= TARGET_TOKENS_PER_SECOND as f64 { + "PASS" + } else { + "BELOW" + }; + + let result = format!( + "{:<20} | {:>12} | {:>12.0} | {:>12} | {:>10} | {:>12}", + "DecodeStream", + tokens_processed, + tokens_per_sec, + TARGET_TOKENS_PER_SECOND, + 1, + status + ); + add_result("streaming_100k", result); + + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + // Test Sequence + let printed_seq = Arc::new(AtomicBool::new(false)); + group.bench_function("sequence_100k", |b| { + let printed = printed_seq.clone(); + let tokenizer = tokenizer.clone(); + + b.iter_custom(|_iters| { + let start = Instant::now(); + let mut sequence = Sequence::new(tokenizer.clone()); + let mut output = String::new(); + let mut tokens_processed = 0u64; + + for token in all_tokens.iter().cycle() { + if start.elapsed() >= Duration::from_millis(500) { + break; + } + + let text = sequence.append_token(*token).unwrap(); + output.push_str(&text); + tokens_processed += 1; + } + + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let tokens_per_sec = tokens_processed as f64 / duration.as_secs_f64(); + let status = if tokens_per_sec >= TARGET_TOKENS_PER_SECOND as f64 { + "PASS" + } else { + "BELOW" + }; + + let result = format!( + "{:<20} | {:>12} | {:>12.0} | {:>12} | {:>10} | {:>12}", + "Sequence", + tokens_processed, + tokens_per_sec, + TARGET_TOKENS_PER_SECOND, + 1, + status + ); + add_result("streaming_100k", result); + + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + group.finish(); +} + +fn bench_latency_distribution(c: &mut Criterion) { + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(get_tokenizer_path().to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + // Test latency for individual token processing + let sample_tokens = vec![1, 450, 6635, 3290, 491, 278, 3474, 29892]; + + let mut group = c.benchmark_group("latency"); + + // Encode latency + let system_4k = generate_system_prompt(4000); + let test_cases = vec![ + ("encode_short", SHORT_PROMPT), + ("encode_medium", MEDIUM_PROMPT), + ("encode_long", LONG_PROMPT), + ("encode_4KB", system_4k.as_str()), + ]; + + for (name, prompt) in test_cases { + let printed = Arc::new(AtomicBool::new(false)); + group.bench_function(name, |b| { + let printed_clone = printed.clone(); + let tokenizer = tokenizer.clone(); + + b.iter_custom(|iters| { + // Only collect detailed latency on first iteration + let total_duration = if !printed_clone.load(Ordering::Relaxed) { + let mut latencies = Vec::new(); + + // Warm up + for _ in 0..100 { + let _ = tokenizer.encode(prompt).unwrap(); + } + + // Measure for statistics + for _ in 0..1000 { + let start = Instant::now(); + let _ = tokenizer.encode(prompt).unwrap(); + let latency = start.elapsed(); + latencies.push(latency); + } + + latencies.sort(); + let p50 = latencies[latencies.len() / 2]; + let p95 = latencies[latencies.len() * 95 / 100]; + let p99 = latencies[latencies.len() * 99 / 100]; + let max = latencies.last().unwrap(); + + let result = format!( + "{:<20} | {:>10.1} | {:>10.1} | {:>10.1} | {:>10.1} | {:>10}", + name, + p50.as_micros() as f64, + p95.as_micros() as f64, + p99.as_micros() as f64, + max.as_micros() as f64, + 1000 + ); + add_result("latency", result); + + printed_clone.store(true, Ordering::Relaxed); + + // Return median for consistency + p50 * iters as u32 + } else { + // Regular benchmark iterations + let start = Instant::now(); + for _ in 0..iters { + black_box(tokenizer.encode(prompt).unwrap()); + } + start.elapsed() + }; + + total_duration + }); + }); + } + + // Decode token latency + let printed_decode = Arc::new(AtomicBool::new(false)); + group.bench_function("decode_token", |b| { + let printed_clone = printed_decode.clone(); + let tokenizer = tokenizer.clone(); + let tokens = sample_tokens.clone(); + + b.iter_custom(|iters| { + let total_duration = if !printed_clone.load(Ordering::Relaxed) { + let mut latencies = Vec::new(); + let mut decoder = DecodeStream::new(tokenizer.clone(), &[], false); + + for token in tokens.iter().cycle().take(1000) { + let start = Instant::now(); + let _ = decoder.step(*token).unwrap(); + let latency = start.elapsed(); + latencies.push(latency); + } + + latencies.sort(); + let p50 = latencies[latencies.len() / 2]; + let p95 = latencies[latencies.len() * 95 / 100]; + let p99 = latencies[latencies.len() * 99 / 100]; + let max = latencies.last().unwrap(); + + let result = format!( + "{:<20} | {:>10.1} | {:>10.1} | {:>10.1} | {:>10.1} | {:>10}", + "decode_token", + p50.as_micros() as f64, + p95.as_micros() as f64, + p99.as_micros() as f64, + max.as_micros() as f64, + 1000 + ); + add_result("latency", result); + + // Check target latency + let target_latency = Duration::from_micros(10); + if p50 > target_latency { + let warning = format!( + "WARNING: P50 latency exceeds target of {:?} for 100k tokens/sec", + target_latency + ); + add_result("latency_warning", warning); + } + + printed_clone.store(true, Ordering::Relaxed); + + // Return approximate time for consistency + p50 * iters as u32 + } else { + // Regular benchmark iterations + let start = Instant::now(); + for _ in 0..iters { + let mut decoder = DecodeStream::new(tokenizer.clone(), &[], false); + for token in tokens.iter().take(10) { + black_box(decoder.step(*token).unwrap()); + } + } + start.elapsed() + }; + + total_duration + }); + }); + + group.finish(); +} + +fn bench_concurrent_streaming(c: &mut Criterion) { + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(get_tokenizer_path().to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + let num_sequences = 16; + let tokens_per_sequence = 10_000; + + let sample_text = "The quick brown fox jumps over the lazy dog. ".repeat(100); + let encoding = tokenizer.encode(&sample_text).unwrap(); + let token_batch: Vec = encoding.token_ids().to_vec(); + + let mut group = c.benchmark_group("concurrent_streaming"); + group.measurement_time(Duration::from_secs(2)); + + let printed = Arc::new(AtomicBool::new(false)); + group.bench_function("concurrent_16_sequences", |b| { + let printed_clone = printed.clone(); + let tokenizer = tokenizer.clone(); + let tokens = token_batch.clone(); + + b.iter_custom(|_iters| { + let total_tokens = Arc::new(AtomicU64::new(0)); + let start = Instant::now(); + + let handles: Vec<_> = (0..num_sequences) + .map(|_seq_id| { + let tokenizer = tokenizer.clone(); + let tokens = tokens.clone(); + let total_tokens = total_tokens.clone(); + + thread::spawn(move || { + let mut decoder = DecodeStream::new(tokenizer, &[], false); + let mut output = String::new(); + let mut local_count = 0u64; + + for token in tokens.iter().cycle().take(tokens_per_sequence) { + if let Some(text) = decoder.step(*token).unwrap() { + output.push_str(&text); + } + local_count += 1; + } + + total_tokens.fetch_add(local_count, Ordering::Relaxed); + }) + }) + .collect(); + + for handle in handles { + handle.join().unwrap(); + } + + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let total = total_tokens.load(Ordering::Relaxed); + let throughput = total as f64 / duration.as_secs_f64(); + let per_seq = throughput / num_sequences as f64; + + let result = format!( + "{:<20} | {:>10} | {:>12.0} | {:>15.0} | {:>15}", + format!("{}_sequences", num_sequences), + total, + throughput, + per_seq, + num_sequences + ); + add_result("concurrent_streaming", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + group.finish(); +} + +fn bench_stop_sequences(c: &mut Criterion) { + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(get_tokenizer_path().to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + let config = StopSequenceConfig::default() + .with_stop_sequence("") + .with_stop_sequence("\n\n") + .with_stop_sequence("###") + .with_stop_token(2); + + let sample_text = "Hello world! This is a test. ### Stop here. Continue after.".repeat(100); + let encoding = tokenizer.encode(&sample_text).unwrap(); + let tokens = encoding.token_ids(); + + let mut group = c.benchmark_group("stop_sequences"); + + // No stops + let printed_no_stop = Arc::new(AtomicBool::new(false)); + group.bench_function("no_stops", |b| { + let printed_clone = printed_no_stop.clone(); + let tokenizer = tokenizer.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + let mut total_tokens = 0u64; + + for _ in 0..iters { + let mut decoder = StopSequenceDecoder::new( + tokenizer.clone(), + StopSequenceConfig::default(), + false, + ); + for token in tokens { + let _ = decoder.process_token(*token).unwrap(); + total_tokens += 1; + } + } + + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let tokens_per_sec = total_tokens as f64 / duration.as_secs_f64(); + let seq_per_sec = iters as f64 / duration.as_secs_f64(); + + let result = format!( + "{:<20} | {:>10} | {:>12} | {:>12.0} | {:>10.0}", + "No stops", iters, total_tokens, tokens_per_sec, seq_per_sec + ); + add_result("stop_sequences", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + // With stops + let printed_with_stops = Arc::new(AtomicBool::new(false)); + group.bench_function("with_stops", |b| { + let printed_clone = printed_with_stops.clone(); + let tokenizer = tokenizer.clone(); + let config = config.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + let mut total_tokens = 0u64; + let mut total_sequences = 0u64; + + for _ in 0..iters { + let mut decoder = + StopSequenceDecoder::new(tokenizer.clone(), config.clone(), false); + let mut sequence_tokens = 0u64; + + for token in tokens { + let result = decoder.process_token(*token).unwrap(); + sequence_tokens += 1; + + if matches!( + result, + SequenceDecoderOutput::Stopped | SequenceDecoderOutput::StoppedWithText(_) + ) { + break; + } + } + + total_tokens += sequence_tokens; + total_sequences += 1; + } + + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let tokens_per_sec = total_tokens as f64 / duration.as_secs_f64(); + let seq_per_sec = total_sequences as f64 / duration.as_secs_f64(); + + let result = format!( + "{:<20} | {:>10} | {:>12} | {:>12.0} | {:>10.0}", + "With stops", total_sequences, total_tokens, tokens_per_sec, seq_per_sec + ); + add_result("stop_sequences", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + group.finish(); +} + +fn bench_multithreaded_encode(c: &mut Criterion) { + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(get_tokenizer_path().to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + let thread_counts = vec![1, 2, 4, 8, 16]; + let operations_per_thread = 1000; + + // Test with medium-sized prompt for balanced workload + let test_prompt = MEDIUM_PROMPT; + + let mut group = c.benchmark_group("multithreaded_encode"); + group.measurement_time(Duration::from_secs(2)); + + let mut baseline_throughput = 0.0; + + for num_threads in thread_counts { + let printed = Arc::new(AtomicBool::new(false)); + let tokenizer_clone = tokenizer.clone(); + + group.bench_with_input( + BenchmarkId::from_parameter(num_threads), + &num_threads, + |b, &threads| { + let printed_clone = printed.clone(); + let tokenizer = tokenizer_clone.clone(); + + b.iter_custom(|_iters| { + let total_operations = Arc::new(AtomicU64::new(0)); + let total_tokens = Arc::new(AtomicU64::new(0)); + let start = Instant::now(); + + let handles: Vec<_> = (0..threads) + .map(|_| { + let tokenizer = tokenizer.clone(); + let total_ops = total_operations.clone(); + let total_tok = total_tokens.clone(); + + thread::spawn(move || { + for _ in 0..operations_per_thread { + let encoding = tokenizer.encode(test_prompt).unwrap(); + total_tok.fetch_add( + encoding.token_ids().len() as u64, + Ordering::Relaxed, + ); + } + total_ops + .fetch_add(operations_per_thread as u64, Ordering::Relaxed); + }) + }) + .collect(); + + for handle in handles { + handle.join().unwrap(); + } + + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let total_ops = total_operations.load(Ordering::Relaxed); + let total_tok = total_tokens.load(Ordering::Relaxed); + let ops_per_sec = total_ops as f64 / duration.as_secs_f64(); + let tokens_per_sec = total_tok as f64 / duration.as_secs_f64(); + + if threads == 1 { + baseline_throughput = tokens_per_sec; + } + + let efficiency = if threads == 1 { + 100.0 + } else { + (tokens_per_sec / (baseline_throughput * threads as f64)) * 100.0 + }; + + let result = format!( + "{:<20} | {:>10} | {:>12.0} | {:>12.0} | {:>10} | {:>11.1}%", + format!("encode_{}_threads", threads), + total_ops, + ops_per_sec, + tokens_per_sec, + threads, + efficiency + ); + add_result("mt_encode", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }, + ); + } + + group.finish(); +} + +fn bench_multithreaded_decode(c: &mut Criterion) { + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(get_tokenizer_path().to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + let thread_counts = vec![1, 2, 4, 8, 16]; + let tokens_per_thread = 5000; + + // Generate tokens for decoding + let test_text = "The quick brown fox jumps over the lazy dog. ".repeat(100); + let encoding = tokenizer.encode(&test_text).unwrap(); + let test_tokens: Vec = encoding.token_ids().to_vec(); + + let mut group = c.benchmark_group("multithreaded_decode"); + group.measurement_time(Duration::from_secs(2)); + + let mut baseline_throughput = 0.0; + + for num_threads in thread_counts { + let printed = Arc::new(AtomicBool::new(false)); + let tokenizer_clone = tokenizer.clone(); + let tokens = test_tokens.clone(); + + group.bench_with_input( + BenchmarkId::from_parameter(num_threads), + &num_threads, + |b, &threads| { + let printed_clone = printed.clone(); + let tokenizer = tokenizer_clone.clone(); + let tokens = tokens.clone(); + + b.iter_custom(|_iters| { + let total_tokens = Arc::new(AtomicU64::new(0)); + let start = Instant::now(); + + let handles: Vec<_> = (0..threads) + .map(|_| { + let tokenizer = tokenizer.clone(); + let tokens = tokens.clone(); + let total_tok = total_tokens.clone(); + + thread::spawn(move || { + let mut decoder = DecodeStream::new(tokenizer, &[], false); + let mut output = String::new(); + let mut local_tokens = 0u64; + + for token in tokens.iter().cycle().take(tokens_per_thread) { + if let Some(text) = decoder.step(*token).unwrap() { + output.push_str(&text); + } + local_tokens += 1; + } + + total_tok.fetch_add(local_tokens, Ordering::Relaxed); + }) + }) + .collect(); + + for handle in handles { + handle.join().unwrap(); + } + + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let total = total_tokens.load(Ordering::Relaxed); + let tokens_per_sec = total as f64 / duration.as_secs_f64(); + + if threads == 1 { + baseline_throughput = tokens_per_sec; + } + + let efficiency = if threads == 1 { + 100.0 + } else { + (tokens_per_sec / (baseline_throughput * threads as f64)) * 100.0 + }; + + let result = format!( + "{:<20} | {:>12} | {:>12.0} | {:>10} | {:>11.1}%", + format!("decode_{}_threads", threads), + total, + tokens_per_sec, + threads, + efficiency + ); + add_result("mt_decode", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }, + ); + } + + group.finish(); +} + +fn bench_memory_efficiency(c: &mut Criterion) { + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(get_tokenizer_path().to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + let large_text = "The quick brown fox jumps over the lazy dog. ".repeat(1000); + let encoding = tokenizer.encode(&large_text).unwrap(); + + let mut group = c.benchmark_group("memory"); + + // Track owned baseline time + let mut owned_time_ns = 0.0; + + // Owned + let printed_owned = Arc::new(AtomicBool::new(false)); + group.bench_function("token_ids_owned", |b| { + let printed_clone = printed_owned.clone(); + let encoding = encoding.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let _ = black_box(encoding.token_ids()); + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let time_per_call = duration.as_nanos() as f64 / iters as f64; + owned_time_ns = time_per_call; + + let result = format!( + "{:<20} | {:>12.0} | {:>11.0}ns | {:>12}", + "token_ids(owned)", ops_per_sec, time_per_call, "baseline" + ); + add_result("memory", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + // Reference + let printed_ref = Arc::new(AtomicBool::new(false)); + + group.bench_function("token_ids_ref", |b| { + let printed_clone = printed_ref.clone(); + let encoding = encoding.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let _ = black_box(encoding.token_ids()); + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let time_per_call = duration.as_nanos() as f64 / iters as f64; + + // Calculate improvement + let improvement = if owned_time_ns > 0.0 { + format!("{:.1}x faster", owned_time_ns / time_per_call) + } else { + "N/A".to_string() + }; + + let result = format!( + "{:<20} | {:>12.0} | {:>11.0}ns | {:>12}", + "token_ids_ref(ref)", ops_per_sec, time_per_call, improvement + ); + add_result("memory", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + group.finish(); +} + +fn bench_scaling_characteristics(c: &mut Criterion) { + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(get_tokenizer_path().to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + let thread_counts = vec![1, 2, 4, 8, 16]; + let tokens_per_thread = 10000; + + let mut group = c.benchmark_group("scaling"); + group.measurement_time(Duration::from_secs(2)); + + let mut baseline_throughput = 0.0; + + for num_threads in thread_counts { + let printed = Arc::new(AtomicBool::new(false)); + + group.bench_with_input( + BenchmarkId::from_parameter(num_threads), + &num_threads, + |b, &threads| { + let printed_clone = printed.clone(); + let tokenizer = tokenizer.clone(); + + b.iter_custom(|_iters| { + let total_tokens = Arc::new(AtomicU64::new(0)); + let start = Instant::now(); + + let handles: Vec<_> = (0..threads) + .map(|_| { + let tokenizer = tokenizer.clone(); + let total_tokens = total_tokens.clone(); + + thread::spawn(move || { + let mut decoder = DecodeStream::new(tokenizer, &[], false); + let sample_tokens = [1, 450, 6635, 3290, 491]; + + for token in sample_tokens.iter().cycle().take(tokens_per_thread) { + let _ = decoder.step(*token).unwrap(); + } + + total_tokens.fetch_add(tokens_per_thread as u64, Ordering::Relaxed); + }) + }) + .collect(); + + for handle in handles { + handle.join().unwrap(); + } + + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let total = total_tokens.load(Ordering::Relaxed); + let throughput = total as f64 / duration.as_secs_f64(); + + if threads == 1 { + baseline_throughput = throughput; + } + + let efficiency = if threads == 1 { + 100.0 + } else { + (throughput / (baseline_throughput * threads as f64)) * 100.0 + }; + + let result = format!( + "{:<15} | {:>12} | {:>12.0} | {:>11.1}%", + format!("{}_threads", threads), + total, + throughput, + efficiency + ); + add_result("scaling", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }, + ); + } + + group.finish(); +} + +fn bench_l1_cache_chat_template(c: &mut Criterion) { + let tokenizer_path = get_tokenizer_path(); + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(tokenizer_path.to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + let mut group = c.benchmark_group("l1_cache_chat"); + + // ============================================================================ + // SCENARIO 1: High Prefix Reuse (95%+ - Realistic Chat Application) + // ============================================================================ + // Most realistic: Same system prompt across 95%+ of requests, different user queries + // This is typical for chat applications where the same system context is reused + + let system_prompt = generate_system_prompt(8000); + + // Generate 100 different user queries of varying lengths (realistic distribution) + let user_queries: Vec = (0..100) + .map(|i| { + let base_queries = [ + "What is the capital of France?", + "Explain quantum mechanics in simple terms.", + "How do I sort an array in Python?", + "What are the benefits of exercise?", + "Can you help me write a resume?", + ]; + let query = base_queries[i % base_queries.len()]; + // Add variation to make each unique + format!("{} (Query #{})", query, i) + }) + .collect(); + + // Create prompts with ChatML format (same system prefix, different queries) + let realistic_prompts: Vec = user_queries + .iter() + .map(|query| { + format!( + "<|im_start|>system\n{}<|im_end|><|im_start|>user\n{}<|im_end|><|im_start|>assistant\n", + system_prompt, query + ) + }) + .collect(); + + // Baseline: No cache + let printed_baseline = Arc::new(AtomicBool::new(false)); + group.bench_function("realistic_chat_uncached", |b| { + let printed = printed_baseline.clone(); + let tokenizer = tokenizer.clone(); + let test_prompts = realistic_prompts.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + // Simulate 100 requests with different queries (realistic workload) + for prompt in &test_prompts { + black_box(tokenizer.encode(prompt).unwrap()); + } + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let total_ops = iters * test_prompts.len() as u64; + let ops_per_sec = total_ops as f64 / duration.as_secs_f64(); + let avg_time_us = duration.as_micros() as f64 / total_ops as f64; + + let result = format!( + "{:<30} | {:>8} | {:>12.0} | {:>12.1} | {:>20}", + "Uncached (baseline)", + test_prompts[0].len(), + ops_per_sec, + avg_time_us, + "N/A" + ); + add_result("l1_cache", result); + + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + // L0-only: Should have 0% hit rate (all queries are unique) + let l0_only_config = CacheConfig { + enable_l0: true, + l0_max_entries: 10_000, + enable_l1: false, + l1_max_memory: 0, + }; + let cached_l0_only = Arc::new(CachedTokenizer::new(tokenizer.clone(), l0_only_config)); + + let printed_l0 = Arc::new(AtomicBool::new(false)); + group.bench_function("realistic_chat_l0_only", |b| { + let printed = printed_l0.clone(); + let cached = cached_l0_only.clone(); + let test_prompts = realistic_prompts.clone(); + + b.iter_custom(|iters| { + cached.clear_cache(); // Start fresh each iteration + let start = Instant::now(); + + for _ in 0..iters { + for prompt in &test_prompts { + black_box(cached.encode(prompt).unwrap()); + } + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let total_ops = iters * test_prompts.len() as u64; + let ops_per_sec = total_ops as f64 / duration.as_secs_f64(); + let avg_time_us = duration.as_micros() as f64 / total_ops as f64; + let stats = cached.cache_stats().unwrap(); + + let result = format!( + "{:<30} | {:>8} | {:>12.0} | {:>12.1} | L0:{:>6.1}% L1:{:>6}", + "L0-only (no benefit)", + test_prompts[0].len(), + ops_per_sec, + avg_time_us, + stats.hit_rate * 100.0, + "N/A" + ); + add_result("l1_cache", result); + + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + // L0+L1: Should show significant speedup from prefix caching + let l0_l1_config = CacheConfig { + enable_l0: true, + l0_max_entries: 10_000, + enable_l1: true, + l1_max_memory: 50 * 1024 * 1024, + }; + let cached_l0_l1 = Arc::new(CachedTokenizer::new( + tokenizer.clone(), + l0_l1_config.clone(), + )); + + let printed_l0_l1 = Arc::new(AtomicBool::new(false)); + group.bench_function("realistic_chat_l0_l1", |b| { + let printed = printed_l0_l1.clone(); + let cached = cached_l0_l1.clone(); + let test_prompts = realistic_prompts.clone(); + + b.iter_custom(|iters| { + cached.clear_cache(); // Start fresh + + // Prime with first request to populate L1 with system prefix + cached.encode(&test_prompts[0]).unwrap(); + + let start = Instant::now(); + for _ in 0..iters { + // All subsequent requests benefit from L1 prefix cache + for prompt in &test_prompts { + black_box(cached.encode(prompt).unwrap()); + } + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let total_ops = iters * test_prompts.len() as u64; + let ops_per_sec = total_ops as f64 / duration.as_secs_f64(); + let avg_time_us = duration.as_micros() as f64 / total_ops as f64; + let stats = cached.cache_stats().unwrap(); + let l1_stats = cached.l1_cache_stats().unwrap(); + + let result = format!( + "{:<30} | {:>8} | {:>12.0} | {:>12.1} | L0:{:>6.1}% L1:{:>6.1}%", + "L0+L1 (95%+ prefix reuse)", + test_prompts[0].len(), + ops_per_sec, + avg_time_us, + stats.hit_rate * 100.0, + l1_stats.hit_rate * 100.0 + ); + add_result("l1_cache", result); + + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + // ============================================================================ + // SCENARIO 2: Customer Service Bot (100% prefix reuse) + // ============================================================================ + // Identical greeting/instructions with different customer queries + + let service_system = "You are a helpful customer service assistant for TechCorp. \ + Always be polite, professional, and helpful. Our business hours are 9 AM to 5 PM EST. \ + We offer a 30-day return policy on all products. For technical issues, escalate to technical support. \ + For billing issues, escalate to accounting department.".repeat(20); // ~2KB + + let customer_queries = [ + "I need to return my laptop", + "My order hasn't arrived yet", + "How do I reset my password?", + "What's your return policy?", + "I was charged twice for my order", + "Can I change my shipping address?", + "Is my product under warranty?", + "I need help installing the software", + ]; + + let service_prompts: Vec = customer_queries + .iter() + .map(|query| { + format!( + "<|im_start|>system\n{}<|im_end|><|im_start|>user\n{}<|im_end|><|im_start|>assistant\n", + service_system, query + ) + }) + .collect(); + + // Service bot with L1-only (to compare against L0+L1) + let l1_only_config = CacheConfig { + enable_l0: false, + l0_max_entries: 0, + enable_l1: true, + l1_max_memory: 50 * 1024 * 1024, + }; + let service_cached_l1 = Arc::new(CachedTokenizer::new(tokenizer.clone(), l1_only_config)); + + let printed_service_l1 = Arc::new(AtomicBool::new(false)); + group.bench_function("customer_service_l1_only", |b| { + let printed = printed_service_l1.clone(); + let cached = service_cached_l1.clone(); + let test_prompts = service_prompts.clone(); + + b.iter_custom(|iters| { + cached.clear_cache(); + cached.encode(&test_prompts[0]).unwrap(); // Prime cache + + let start = Instant::now(); + for _ in 0..iters { + for prompt in &test_prompts { + black_box(cached.encode(prompt).unwrap()); + } + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let total_ops = iters * test_prompts.len() as u64; + let ops_per_sec = total_ops as f64 / duration.as_secs_f64(); + let avg_time_us = duration.as_micros() as f64 / total_ops as f64; + let l1_stats = cached.l1_cache_stats().unwrap(); + + let result = format!( + "{:<30} | {:>8} | {:>12.0} | {:>12.1} | L0:{:>6} L1:{:>6.1}%", + "Customer Service (L1-only)", + test_prompts[0].len(), + ops_per_sec, + avg_time_us, + "N/A", + l1_stats.hit_rate * 100.0 + ); + add_result("l1_cache", result); + + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + // Service bot with L0+L1 + let service_cached = Arc::new(CachedTokenizer::new( + tokenizer.clone(), + l0_l1_config.clone(), + )); + + let printed_service = Arc::new(AtomicBool::new(false)); + group.bench_function("customer_service_l0_l1", |b| { + let printed = printed_service.clone(); + let cached = service_cached.clone(); + let test_prompts = service_prompts.clone(); + + b.iter_custom(|iters| { + cached.clear_cache(); + cached.encode(&test_prompts[0]).unwrap(); // Prime cache + + let start = Instant::now(); + for _ in 0..iters { + for prompt in &test_prompts { + black_box(cached.encode(prompt).unwrap()); + } + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let total_ops = iters * test_prompts.len() as u64; + let ops_per_sec = total_ops as f64 / duration.as_secs_f64(); + let avg_time_us = duration.as_micros() as f64 / total_ops as f64; + let stats = cached.cache_stats().unwrap(); + let l1_stats = cached.l1_cache_stats().unwrap(); + + let result = format!( + "{:<30} | {:>8} | {:>12.0} | {:>12.1} | L0:{:>6.1}% L1:{:>6.1}%", + "Customer Service (100% reuse)", + test_prompts[0].len(), + ops_per_sec, + avg_time_us, + stats.hit_rate * 100.0, + l1_stats.hit_rate * 100.0 + ); + add_result("l1_cache", result); + + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + // ============================================================================ + // SCENARIO 3: Multi-Turn Conversation (Progressive context building) + // ============================================================================ + // Each turn builds on previous context (common in chat applications) + + let conversation_system = + "You are a helpful coding assistant. Help users write better code.".repeat(10); + + // Simulate a 5-turn conversation where context grows + let conversation_turns = vec![ + format!("<|im_start|>system\n{}<|im_end|><|im_start|>user\nHow do I sort an array in Python?<|im_end|><|im_start|>assistant\n", conversation_system), + format!("<|im_start|>system\n{}<|im_end|><|im_start|>user\nHow do I sort an array in Python?<|im_end|><|im_start|>assistant\nYou can use the sorted() function or list.sort() method.<|im_end|><|im_start|>user\nWhat's the difference between them?<|im_end|><|im_start|>assistant\n", conversation_system), + format!("<|im_start|>system\n{}<|im_end|><|im_start|>user\nHow do I sort an array in Python?<|im_end|><|im_start|>assistant\nYou can use the sorted() function or list.sort() method.<|im_end|><|im_start|>user\nWhat's the difference between them?<|im_end|><|im_start|>assistant\nsorted() creates a new list, sort() modifies in place.<|im_end|><|im_start|>user\nCan I sort by a custom key?<|im_end|><|im_start|>assistant\n", conversation_system), + ]; + + let conv_cached = Arc::new(CachedTokenizer::new( + tokenizer.clone(), + l0_l1_config.clone(), + )); + + let printed_conv = Arc::new(AtomicBool::new(false)); + group.bench_function("multi_turn_conversation", |b| { + let printed = printed_conv.clone(); + let cached = conv_cached.clone(); + let test_turns = conversation_turns.clone(); + + b.iter_custom(|iters| { + cached.clear_cache(); + + let start = Instant::now(); + for _ in 0..iters { + // Simulate progressive conversation (each turn shares prefix with previous) + for turn in &test_turns { + black_box(cached.encode(turn).unwrap()); + } + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let total_ops = iters * test_turns.len() as u64; + let ops_per_sec = total_ops as f64 / duration.as_secs_f64(); + let avg_time_us = duration.as_micros() as f64 / total_ops as f64; + let stats = cached.cache_stats().unwrap(); + let l1_stats = cached.l1_cache_stats().unwrap(); + + let result = format!( + "{:<30} | {:>8} | {:>12.0} | {:>12.1} | L0:{:>6.1}% L1:{:>6.1}%", + "Multi-turn Conversation", + test_turns[0].len(), + ops_per_sec, + avg_time_us, + stats.hit_rate * 100.0, + l1_stats.hit_rate * 100.0 + ); + add_result("l1_cache", result); + + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + // ============================================================================ + // SCENARIO 4: Code Review Assistant (Same guidelines, different code snippets) + // ============================================================================ + + let review_system = "You are a code review assistant. Check for: \ + 1) Code quality and readability \ + 2) Performance issues \ + 3) Security vulnerabilities \ + 4) Best practices \ + 5) Documentation completeness" + .repeat(15); + + let code_snippets = [ + "function add(a, b) { return a + b; }", + "def factorial(n): return 1 if n <= 1 else n * factorial(n-1)", + "SELECT * FROM users WHERE id = $_GET['id']", // Security issue + "for (var i = 0; i < 10; i++) { setTimeout(() => console.log(i), 100); }", // Closure issue + ]; + + let review_prompts: Vec = code_snippets + .iter() + .map(|code| { + format!( + "<|im_start|>system\n{}<|im_end|><|im_start|>user\nReview this code:\n```\n{}\n```<|im_end|><|im_start|>assistant\n", + review_system, code + ) + }) + .collect(); + + let review_cached = Arc::new(CachedTokenizer::new(tokenizer.clone(), l0_l1_config)); + + let printed_review = Arc::new(AtomicBool::new(false)); + group.bench_function("code_review_assistant", |b| { + let printed = printed_review.clone(); + let cached = review_cached.clone(); + let test_prompts = review_prompts.clone(); + + b.iter_custom(|iters| { + cached.clear_cache(); + cached.encode(&test_prompts[0]).unwrap(); // Prime cache + + let start = Instant::now(); + for _ in 0..iters { + for prompt in &test_prompts { + black_box(cached.encode(prompt).unwrap()); + } + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let total_ops = iters * test_prompts.len() as u64; + let ops_per_sec = total_ops as f64 / duration.as_secs_f64(); + let avg_time_us = duration.as_micros() as f64 / total_ops as f64; + let stats = cached.cache_stats().unwrap(); + let l1_stats = cached.l1_cache_stats().unwrap(); + + let result = format!( + "{:<30} | {:>8} | {:>12.0} | {:>12.1} | L0:{:>6.1}% L1:{:>6.1}%", + "Code Review (high reuse)", + test_prompts[0].len(), + ops_per_sec, + avg_time_us, + stats.hit_rate * 100.0, + l1_stats.hit_rate * 100.0 + ); + add_result("l1_cache", result); + + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + group.finish(); +} + +// Print final summary table +fn print_summary() { + println!("\n{}", "=".repeat(120)); + println!("TOKENIZER BENCHMARK SUMMARY"); + println!("{}", "=".repeat(120)); + + let results = BENCHMARK_RESULTS.lock().unwrap(); + + let mut current_category = String::new(); + for (key, value) in results.iter() { + let category = key.split('_').skip(1).collect::>().join("_"); + + if category != current_category { + current_category = category.clone(); + + // Print section header based on category + println!("\n{}", "-".repeat(120)); + match category.as_str() { + "encode" => { + println!("ENCODING THROUGHPUT"); + println!( + "{:<15} | {:>8} | {:>8} | {:>12} | {:>12} | {:>10} | {:>10}", + "Test Case", + "Size(B)", + "Tokens", + "Chars/sec", + "Tokens/sec", + "Ops/sec", + "Thread" + ); + } + "batch" => { + println!("BATCH ENCODING"); + println!( + "{:<15} | {:>8} | {:>8} | {:>12} | {:>12} | {:>10} | {:>10}", + "Batch Size", + "Size(B)", + "Tokens", + "Prompts/sec", + "Tokens/sec", + "Chars/sec", + "Thread" + ); + } + "concurrent" => { + println!("CONCURRENT ENCODING"); + println!( + "{:<15} | {:>10} | {:>12} | {:>12} | {:>15}", + "Clients", "Total Ops", "Ops/sec", "Chars/sec", "Per-Client/sec" + ); + } + "mt_encode" => { + println!("MULTI-THREADED ENCODING"); + println!( + "{:<20} | {:>10} | {:>12} | {:>12} | {:>10} | {:>12}", + "Configuration", + "Total Ops", + "Ops/sec", + "Tokens/sec", + "Threads", + "Efficiency" + ); + } + "decode" => { + println!("DECODE PERFORMANCE"); + println!( + "{:<20} | {:>10} | {:>12} | {:>12} | {:>10}", + "Method", "Tokens", "Tokens/sec", "Ops/sec", "Thread" + ); + } + "mt_decode" => { + println!("MULTI-THREADED DECODING"); + println!( + "{:<20} | {:>12} | {:>12} | {:>10} | {:>12}", + "Configuration", "Total Tokens", "Tokens/sec", "Threads", "Efficiency" + ); + } + "streaming_100k" => { + println!("STREAMING DECODE (100K Target)"); + println!( + "{:<20} | {:>12} | {:>12} | {:>12} | {:>10} | {:>12}", + "Method", "Tokens", "Tokens/sec", "Target", "Thread", "Status" + ); + } + "concurrent_streaming" => { + println!("CONCURRENT STREAMING"); + println!( + "{:<20} | {:>10} | {:>12} | {:>15} | {:>15}", + "Sequences", "Total", "Aggregate/sec", "Per-Seq/sec", "Threads" + ); + } + "stop_sequences" => { + println!("STOP SEQUENCE PERFORMANCE"); + println!( + "{:<20} | {:>10} | {:>12} | {:>12} | {:>10}", + "Config", "Sequences", "Tokens", "Tokens/sec", "Seq/sec" + ); + } + "latency" => { + println!("LATENCY DISTRIBUTION"); + println!( + "{:<20} | {:>10} | {:>10} | {:>10} | {:>10} | {:>10}", + "Operation", "P50(µs)", "P95(µs)", "P99(µs)", "Max(µs)", "Samples" + ); + } + "scaling" => { + println!("SCALING CHARACTERISTICS"); + println!( + "{:<15} | {:>12} | {:>12} | {:>12}", + "Threads", "Total Tokens", "Tokens/sec", "Efficiency" + ); + } + "memory" => { + println!("MEMORY EFFICIENCY"); + println!( + "{:<20} | {:>12} | {:>12} | {:>12}", + "Operation", "Calls/sec", "Time/call", "Improvement" + ); + } + "l1_cache" => { + println!("L1 CACHE (PREFIX MATCHING) - REALISTIC WORKLOADS"); + println!( + "{:<30} | {:>8} | {:>12} | {:>12} | {:>20}", + "Scenario", "Size(B)", "Ops/sec", "Time(µs)", "Hit Rates" + ); + } + _ => {} + } + println!("{}", "-".repeat(120)); + } + + println!("{}", value); + } + + println!("\n{}", "=".repeat(120)); +} + +fn run_benchmarks(c: &mut Criterion) { + bench_encode_throughput(c); + bench_batch_encode(c); + bench_concurrent_encode(c); + bench_multithreaded_encode(c); + bench_decode_performance(c); + bench_multithreaded_decode(c); + bench_streaming_decode_100k(c); + bench_concurrent_streaming(c); + bench_stop_sequences(c); + bench_latency_distribution(c); + bench_scaling_characteristics(c); + bench_memory_efficiency(c); + bench_l1_cache_chat_template(c); + + // Print summary at the end + print_summary(); +} + +criterion_group!(benches, run_benchmarks); +criterion::criterion_main!(benches); diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/benches/tool_parser_benchmark.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/benches/tool_parser_benchmark.rs new file mode 100644 index 00000000..30fd76f1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/benches/tool_parser_benchmark.rs @@ -0,0 +1,887 @@ +//! Comprehensive tool parser benchmark for measuring performance under various scenarios +//! +//! This benchmark tests: +//! - Single parser parsing performance +//! - Registry creation overhead +//! - Concurrent parsing with shared parsers +//! - Streaming vs complete parsing +//! - Different model formats (JSON, Mistral, Qwen, Pythonic, etc.) + +use std::{ + collections::BTreeMap, + sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, Mutex, + }, + thread, + time::{Duration, Instant}, +}; + +use criterion::{black_box, criterion_group, BenchmarkId, Criterion, Throughput}; +use serde_json::json; +use sgl_model_gateway::{ + protocols::common::{Function, Tool}, + tool_parser::{JsonParser, ParserFactory as ToolParserFactory, ToolParser}, +}; +use tokio::runtime::Runtime; + +// Test data for different parser formats - realistic complex examples +const JSON_SIMPLE: &str = r#"{"name": "code_interpreter", "arguments": "{\"language\": \"python\", \"code\": \"import numpy as np\\nimport matplotlib.pyplot as plt\\n\\n# Generate sample data\\nx = np.linspace(0, 10, 100)\\ny = np.sin(x) * np.exp(-x/10)\\n\\n# Create the plot\\nplt.figure(figsize=(10, 6))\\nplt.plot(x, y, 'b-', linewidth=2)\\nplt.grid(True)\\nplt.xlabel('Time (s)')\\nplt.ylabel('Amplitude')\\nplt.title('Damped Oscillation')\\nplt.show()\"}"}"#; + +const JSON_ARRAY: &str = r#"[{"name": "web_search", "arguments": "{\"query\": \"latest developments in quantum computing 2024\", \"num_results\": 10, \"search_type\": \"news\", \"date_range\": \"2024-01-01:2024-12-31\", \"exclude_domains\": [\"reddit.com\", \"facebook.com\"], \"language\": \"en\"}"}, {"name": "analyze_sentiment", "arguments": "{\"text\": \"The breakthrough in quantum error correction represents a significant milestone. Researchers are optimistic about practical applications within the next decade.\", \"granularity\": \"sentence\", \"aspects\": [\"technology\", \"timeline\", \"impact\"], \"confidence_threshold\": 0.85}"}, {"name": "create_summary", "arguments": "{\"content_ids\": [\"doc_1234\", \"doc_5678\", \"doc_9012\"], \"max_length\": 500, \"style\": \"technical\", \"include_citations\": true}"}]"#; + +const JSON_WITH_PARAMS: &str = r#"{"name": "database_query", "parameters": {"connection_string": "postgresql://user:pass@localhost:5432/analytics", "query": "SELECT customer_id, COUNT(*) as order_count, SUM(total_amount) as lifetime_value, AVG(order_amount) as avg_order_value FROM orders WHERE created_at >= '2024-01-01' GROUP BY customer_id HAVING COUNT(*) > 5 ORDER BY lifetime_value DESC LIMIT 100", "timeout_ms": 30000, "read_consistency": "strong", "partition_key": "customer_id"}}"#; + +const MISTRAL_FORMAT: &str = r#"I'll help you analyze the sales data and create visualizations. Let me start by querying the database and then create some charts. + +[TOOL_CALLS] [{"name": "sql_query", "arguments": {"database": "sales_analytics", "query": "WITH monthly_sales AS (SELECT DATE_TRUNC('month', order_date) as month, SUM(total_amount) as revenue, COUNT(DISTINCT customer_id) as unique_customers, COUNT(*) as total_orders FROM orders WHERE order_date >= CURRENT_DATE - INTERVAL '12 months' GROUP BY DATE_TRUNC('month', order_date)) SELECT month, revenue, unique_customers, total_orders, LAG(revenue) OVER (ORDER BY month) as prev_month_revenue, (revenue - LAG(revenue) OVER (ORDER BY month)) / LAG(revenue) OVER (ORDER BY month) * 100 as growth_rate FROM monthly_sales ORDER BY month DESC", "format": "json", "timeout": 60000}}] + +Based on the query results, I can see interesting trends in your sales data."#; + +const MISTRAL_MULTI: &str = r#"Let me help you with a comprehensive analysis of your application's performance. + +[TOOL_CALLS] [{"name": "get_metrics", "arguments": {"service": "api-gateway", "metrics": ["latency_p50", "latency_p95", "latency_p99", "error_rate", "requests_per_second"], "start_time": "2024-01-01T00:00:00Z", "end_time": "2024-01-01T23:59:59Z", "aggregation": "5m", "filters": {"environment": "production", "region": "us-east-1"}}}, {"name": "analyze_logs", "arguments": {"log_group": "/aws/lambda/process-orders", "query": "fields @timestamp, @message, @requestId, duration | filter @message like /ERROR/ | stats count() by bin(@timestamp, 5m) as time_window", "start_time": "2024-01-01T00:00:00Z", "end_time": "2024-01-01T23:59:59Z", "limit": 1000}}, {"name": "get_traces", "arguments": {"service": "order-processing", "operation": "ProcessOrder", "min_duration_ms": 1000, "max_results": 100, "include_downstream": true}}] + +Now let me create a comprehensive report based on this data."#; + +const QWEN_FORMAT: &str = r#"Let me search for information about machine learning frameworks and their performance benchmarks. + + +{"name": "academic_search", "arguments": {"query": "transformer architecture optimization techniques GPU inference latency reduction", "databases": ["arxiv", "ieee", "acm"], "year_range": [2020, 2024], "citation_count_min": 10, "include_code": true, "page_size": 25, "sort_by": "relevance"}} + + +I found several interesting papers on optimization techniques."#; + +const QWEN_MULTI: &str = r#"I'll help you set up a complete data pipeline for your analytics system. + + +{"name": "create_data_pipeline", "arguments": {"name": "customer_analytics_etl", "source": {"type": "kafka", "config": {"bootstrap_servers": "kafka1:9092,kafka2:9092", "topic": "customer_events", "consumer_group": "analytics_consumer", "auto_offset_reset": "earliest"}}, "transformations": [{"type": "filter", "condition": "event_type IN ('purchase', 'signup', 'churn')"}, {"type": "aggregate", "window": "1h", "group_by": ["customer_id", "event_type"], "metrics": ["count", "sum(amount)"]}], "destination": {"type": "bigquery", "dataset": "analytics", "table": "customer_metrics", "write_mode": "append"}}} + + +{"name": "schedule_job", "arguments": {"job_id": "customer_analytics_etl", "schedule": "0 */4 * * *", "timezone": "UTC", "retry_policy": {"max_attempts": 3, "backoff_multiplier": 2, "max_backoff": 3600}, "notifications": {"on_failure": ["ops-team@company.com"], "on_success": null}, "monitoring": {"sla_minutes": 30, "alert_threshold": 0.95}}} + + +{"name": "create_dashboard", "arguments": {"title": "Customer Analytics Dashboard", "widgets": [{"type": "time_series", "title": "Customer Acquisition", "query": "SELECT DATE(timestamp) as date, COUNT(DISTINCT customer_id) as new_customers FROM analytics.customer_metrics WHERE event_type = 'signup' GROUP BY date ORDER BY date", "visualization": "line"}, {"type": "metric", "title": "Total Revenue", "query": "SELECT SUM(amount) as total FROM analytics.customer_metrics WHERE event_type = 'purchase' AND DATE(timestamp) = CURRENT_DATE()", "format": "currency"}, {"type": "table", "title": "Top Customers", "query": "SELECT customer_id, COUNT(*) as purchases, SUM(amount) as total_spent FROM analytics.customer_metrics WHERE event_type = 'purchase' GROUP BY customer_id ORDER BY total_spent DESC LIMIT 10"}], "refresh_interval": 300}} + + +The data pipeline has been configured and the dashboard is ready."#; + +const LLAMA_FORMAT: &str = r#"<|python_tag|>{"name": "execute_code", "arguments": "{\"code\": \"import pandas as pd\\nimport numpy as np\\nfrom sklearn.model_selection import train_test_split\\nfrom sklearn.ensemble import RandomForestClassifier\\nfrom sklearn.metrics import classification_report, confusion_matrix\\nimport joblib\\n\\n# Load and preprocess data\\ndf = pd.read_csv('/data/customer_churn.csv')\\nprint(f'Dataset shape: {df.shape}')\\nprint(f'Missing values: {df.isnull().sum().sum()}')\\n\\n# Feature engineering\\ndf['tenure_months'] = pd.to_datetime('today') - pd.to_datetime(df['signup_date'])\\ndf['tenure_months'] = df['tenure_months'].dt.days // 30\\ndf['avg_monthly_spend'] = df['total_spend'] / df['tenure_months'].clip(lower=1)\\n\\n# Prepare features and target\\nfeature_cols = ['tenure_months', 'avg_monthly_spend', 'support_tickets', 'product_usage_hours', 'feature_adoption_score']\\nX = df[feature_cols]\\ny = df['churned']\\n\\n# Split and train\\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)\\nrf_model = RandomForestClassifier(n_estimators=100, max_depth=10, min_samples_split=5, random_state=42)\\nrf_model.fit(X_train, y_train)\\n\\n# Evaluate\\ny_pred = rf_model.predict(X_test)\\nprint('Classification Report:')\\nprint(classification_report(y_test, y_pred))\\n\\n# Save model\\njoblib.dump(rf_model, '/models/churn_predictor_v1.pkl')\\nprint('Model saved successfully!')\"}"}"#; + +const PYTHONIC_FORMAT: &str = r#"[retrieve_context(query="How do transformer models handle long-range dependencies in natural language processing tasks?", index="ml_knowledge_base", top_k=5, similarity_threshold=0.75, rerank=True, include_metadata=True, filters={"category": "deep_learning", "year": {"$gte": 2020}})]"#; + +const PYTHONIC_MULTI: &str = r#"[fetch_api_data(endpoint="https://api.weather.com/v1/forecast", params={"lat": 37.7749, "lon": -122.4194, "units": "metric", "days": 7, "hourly": True}, headers={"API-Key": "${WEATHER_API_KEY}"}, timeout=30, retry_count=3), process_weather_data(data="${response}", extract_fields=["temperature", "humidity", "precipitation", "wind_speed", "uv_index"], aggregation="daily", calculate_trends=True), generate_report(data="${processed_data}", template="weather_forecast", format="html", include_charts=True, language="en")]"#; + +const DEEPSEEK_FORMAT: &str = r#"I'll analyze your codebase and identify potential security vulnerabilities. + +🤔[{"name": "scan_repository", "arguments": {"repo_path": "/src/application", "scan_types": ["security", "dependencies", "secrets", "code_quality"], "file_patterns": ["*.py", "*.js", "*.java", "*.go"], "exclude_dirs": ["node_modules", ".git", "vendor", "build"], "vulnerability_databases": ["cve", "nvd", "ghsa"], "min_severity": "medium", "check_dependencies": true, "deep_scan": true, "parallel_workers": 8}}] + +Let me examine the scan results and provide recommendations."#; + +const KIMIK2_FORMAT: &str = r#"⍼validate_and_deploy⍁{"deployment_config": {"application": "payment-service", "version": "2.3.1", "environment": "staging", "region": "us-west-2", "deployment_strategy": "blue_green", "health_check": {"endpoint": "/health", "interval": 30, "timeout": 5, "healthy_threshold": 2, "unhealthy_threshold": 3}, "rollback_on_failure": true, "canary_config": {"percentage": 10, "duration_minutes": 30, "metrics": ["error_rate", "latency_p99", "success_rate"], "thresholds": {"error_rate": 0.01, "latency_p99": 500, "success_rate": 0.99}}, "pre_deployment_hooks": ["run_tests", "security_scan", "backup_database"], "post_deployment_hooks": ["smoke_tests", "notify_team", "update_documentation"]}}"#; + +const GLM4_FORMAT: &str = r#" +analyze_customer_behavior +dataset_id=customer_interactions_2024 +analysis_type=cohort_retention +cohort_definition=signup_month +retention_periods=[1, 7, 14, 30, 60, 90, 180, 365] +segment_by=["acquisition_channel", "pricing_tier", "industry", "company_size"] +metrics=["active_users", "revenue", "feature_usage", "engagement_score"] +statistical_tests=["chi_square", "anova", "trend_analysis"] +visualization_types=["heatmap", "line_chart", "funnel", "sankey"] +export_format=dashboard +confidence_level=0.95 +"#; + +const STEP3_FORMAT: &str = r#" + +orchestrate_ml_pipeline + +fraud_detection_model_v3 +s3://ml-datasets/transactions/2024/ + + {"remove_duplicates": true, "handle_missing": "interpolate", "outlier_method": "isolation_forest"} + {"create_ratios": true, "time_features": ["hour", "day_of_week", "month"], "aggregations": ["mean", "std", "max"]} + {"method": "robust_scaler", "clip_outliers": true} + +{"algorithm": "xgboost", "hyperparameters": {"n_estimators": 500, "max_depth": 8, "learning_rate": 0.01, "subsample": 0.8}, "cross_validation": {"method": "stratified_kfold", "n_splits": 5}} +["auc_roc", "precision_recall", "f1", "confusion_matrix"] +sagemaker_endpoint +{"drift_detection": true, "performance_threshold": 0.92, "alert_emails": ["ml-team@company.com"]} + + +"#; + +const GPT_OSS_FORMAT: &str = r#"{"collection": "technical_documentation", "query_embedding": [0.0234, -0.1456, 0.0891, 0.2341, -0.0567, 0.1234, 0.0456, -0.0789, 0.1567, 0.0234, -0.1123, 0.0678, 0.2345, -0.0456, 0.0891, 0.1234, -0.0567, 0.0789, 0.1456, -0.0234, 0.0891, 0.1567, -0.0678, 0.0345, 0.1234, -0.0456, 0.0789, 0.1891, -0.0234, 0.0567, 0.1345, -0.0891], "top_k": 10, "similarity_metric": "cosine", "filters": {"language": "en", "last_updated": {"$gte": "2023-01-01"}, "categories": {"$in": ["api", "sdk", "integration"]}}, "include_metadata": true, "rerank_with_cross_encoder": true}"#; + +// Create test tools for parsers that need them +fn create_test_tools() -> Vec { + vec![ + Tool { + tool_type: "function".to_string(), + function: Function { + name: "search".to_string(), + description: Some("Search for information".to_string()), + parameters: json!({ + "type": "object", + "properties": { + "query": {"type": "string"}, + "limit": {"type": "number"} + } + }), + strict: None, + }, + }, + Tool { + tool_type: "function".to_string(), + function: Function { + name: "code_interpreter".to_string(), + description: Some("Execute code".to_string()), + parameters: json!({ + "type": "object", + "properties": { + "language": {"type": "string"}, + "code": {"type": "string"} + } + }), + strict: None, + }, + }, + ] +} + +// Large test data for stress testing +fn generate_large_json(num_tools: usize) -> String { + let mut tools = Vec::new(); + for i in 0..num_tools { + tools.push(format!( + r#"{{"name": "tool_{}", "arguments": {{"param1": "value{}", "param2": {}, "param3": true}}}}"#, + i, i, i + )); + } + format!("[{}]", tools.join(", ")) +} + +// Global results storage +lazy_static::lazy_static! { + static ref BENCHMARK_RESULTS: Mutex> = Mutex::new(BTreeMap::new()); +} + +fn add_result(category: &str, result: String) { + let mut results = BENCHMARK_RESULTS.lock().unwrap(); + let index = results.len(); + results.insert(format!("{:03}_{}", index, category), result); +} + +fn bench_registry_creation(c: &mut Criterion) { + let mut group = c.benchmark_group("registry_creation"); + + let printed = Arc::new(AtomicBool::new(false)); + group.bench_function("new_registry", |b| { + let printed_clone = printed.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let registry = black_box(ToolParserFactory::new()); + // Force evaluation to prevent optimization + black_box(registry.list_parsers()); + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let time_per_op = duration.as_micros() as f64 / iters as f64; + + let result = format!( + "{:<25} | {:>12.0} | {:>12.1}µs | {:>15}", + "Registry Creation", ops_per_sec, time_per_op, "N/A" + ); + add_result("registry", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + group.finish(); +} + +fn bench_parser_lookup(c: &mut Criterion) { + let registry = Arc::new(ToolParserFactory::new()); + let models = vec![ + "gpt-4", + "mistral-large", + "qwen-72b", + "llama-3.2", + "deepseek-v3", + "unknown-model", + ]; + + let mut group = c.benchmark_group("parser_lookup"); + + for model in models { + let printed = Arc::new(AtomicBool::new(false)); + let registry_clone = registry.clone(); + + group.bench_function(model, |b| { + let printed_clone = printed.clone(); + let registry = registry_clone.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let parser = black_box(registry.get_parser(model)); + // Force evaluation + black_box(parser.is_some()); + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let time_per_op = duration.as_nanos() as f64 / iters as f64; + + let result = format!( + "{:<25} | {:>12.0} | {:>12.1}ns | {:>15}", + format!("Lookup {}", model), + ops_per_sec, + time_per_op, + if registry.get_parser(model).is_some() { + "Found" + } else { + "Fallback" + } + ); + add_result("lookup", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + } + + group.finish(); +} + +fn bench_complete_parsing(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let registry = Arc::new(ToolParserFactory::new()); + + let test_cases = vec![ + ("json_simple", "json", JSON_SIMPLE), + ("json_array", "json", JSON_ARRAY), + ("json_params", "json", JSON_WITH_PARAMS), + ("mistral_single", "mistral", MISTRAL_FORMAT), + ("mistral_multi", "mistral", MISTRAL_MULTI), + ("qwen_single", "qwen", QWEN_FORMAT), + ("qwen_multi", "qwen", QWEN_MULTI), + ("llama", "llama", LLAMA_FORMAT), + ("pythonic_single", "pythonic", PYTHONIC_FORMAT), + ("pythonic_multi", "pythonic", PYTHONIC_MULTI), + ("deepseek", "deepseek", DEEPSEEK_FORMAT), + ("kimik2", "kimik2", KIMIK2_FORMAT), + ("glm4", "glm4_moe", GLM4_FORMAT), + ("step3", "step3", STEP3_FORMAT), + ("gpt_oss", "gpt_oss", GPT_OSS_FORMAT), + ]; + + let mut group = c.benchmark_group("complete_parsing"); + + for (name, parser_name, input) in test_cases { + let printed = Arc::new(AtomicBool::new(false)); + let registry_clone = registry.clone(); + let input_len = input.len(); + + group.throughput(Throughput::Bytes(input_len as u64)); + group.bench_function(name, |b| { + let printed_clone = printed.clone(); + let registry = registry_clone.clone(); + let rt = rt.handle().clone(); + + b.iter_custom(|iters| { + let parser = registry.get_parser(parser_name).expect("Parser not found"); + + let start = Instant::now(); + for _ in 0..iters { + let parser = parser.clone(); + let result = rt.block_on(async { parser.parse_complete(input).await }); + black_box(result.unwrap()); + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let bytes_per_sec = (iters as f64 * input_len as f64) / duration.as_secs_f64(); + let time_per_op = duration.as_micros() as f64 / iters as f64; + + let result = format!( + "{:<25} | {:>10} | {:>12.0} | {:>12.0} | {:>10.1}µs", + name, input_len, ops_per_sec, bytes_per_sec, time_per_op + ); + add_result("complete", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + } + + group.finish(); +} + +fn bench_streaming_parsing(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + + // Streaming test with chunked input + let chunks = vec![ + r#"{"na"#, + r#"me": "sear"#, + r#"ch", "argu"#, + r#"ments": {"qu"#, + r#"ery": "rust prog"#, + r#"ramming", "li"#, + r#"mit": 10, "off"#, + r#"set": 0}"#, + r#"}"#, + ]; + + let mut group = c.benchmark_group("streaming_parsing"); + + let printed = Arc::new(AtomicBool::new(false)); + group.bench_function("json_streaming", |b| { + let printed_clone = printed.clone(); + let rt = rt.handle().clone(); + + b.iter_custom(|iters| { + let tools = create_test_tools(); + + let start = Instant::now(); + for _ in 0..iters { + let mut parser = JsonParser::new(); + let mut complete_tools = Vec::new(); + + rt.block_on(async { + for chunk in &chunks { + let result = parser.parse_incremental(chunk, &tools).await.unwrap(); + if !result.calls.is_empty() { + complete_tools.extend(result.calls); + } + } + }); + + black_box(complete_tools); + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let time_per_op = duration.as_micros() as f64 / iters as f64; + let chunks_per_sec = (iters as f64 * chunks.len() as f64) / duration.as_secs_f64(); + + let result = format!( + "{:<25} | {:>10} | {:>12.0} | {:>12.0} | {:>10.1}µs", + "JSON Streaming", + chunks.len(), + ops_per_sec, + chunks_per_sec, + time_per_op + ); + add_result("streaming", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + group.finish(); +} + +fn bench_concurrent_parsing(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let registry = Arc::new(ToolParserFactory::new()); + let parser = registry.get_parser("json").expect("Parser not found"); + + let thread_counts = vec![1, 2, 4, 8, 16, 32]; + let operations_per_thread = 100; + + let mut group = c.benchmark_group("concurrent_parsing"); + group.measurement_time(Duration::from_secs(3)); + + for num_threads in thread_counts { + let printed = Arc::new(AtomicBool::new(false)); + let parser_clone = parser.clone(); + + group.bench_with_input( + BenchmarkId::from_parameter(num_threads), + &num_threads, + |b, &threads| { + let printed_clone = printed.clone(); + let parser = parser_clone.clone(); + let rt = rt.handle().clone(); + + b.iter_custom(|_iters| { + let total_operations = Arc::new(AtomicU64::new(0)); + let total_parsed = Arc::new(AtomicU64::new(0)); + let start = Instant::now(); + + let handles: Vec<_> = (0..threads) + .map(|_thread_id| { + let parser = parser.clone(); + let total_ops = total_operations.clone(); + let total_p = total_parsed.clone(); + let rt = rt.clone(); + + thread::spawn(move || { + let test_inputs = [JSON_SIMPLE, JSON_ARRAY, JSON_WITH_PARAMS]; + + for i in 0..operations_per_thread { + let input = test_inputs[i % test_inputs.len()]; + let result = + rt.block_on(async { parser.parse_complete(input).await }); + + if let Ok((_normal_text, tools)) = result { + total_p.fetch_add(tools.len() as u64, Ordering::Relaxed); + } + } + + total_ops + .fetch_add(operations_per_thread as u64, Ordering::Relaxed); + }) + }) + .collect(); + + for handle in handles { + handle.join().unwrap(); + } + + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let total_ops = total_operations.load(Ordering::Relaxed); + let total_p = total_parsed.load(Ordering::Relaxed); + let ops_per_sec = total_ops as f64 / duration.as_secs_f64(); + let tools_per_sec = total_p as f64 / duration.as_secs_f64(); + + let result = format!( + "{:<25} | {:>10} | {:>12.0} | {:>12.0} | {:>10}", + format!("{}_threads", threads), + total_ops, + ops_per_sec, + tools_per_sec, + threads + ); + add_result("concurrent", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }, + ); + } + + group.finish(); +} + +fn bench_large_payloads(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let registry = Arc::new(ToolParserFactory::new()); + let parser = registry.get_parser("json").expect("Parser not found"); + + let sizes = vec![1, 10, 50, 100, 500]; + + let mut group = c.benchmark_group("large_payloads"); + + for size in sizes { + let large_json = generate_large_json(size); + let input_len = large_json.len(); + let printed = Arc::new(AtomicBool::new(false)); + let parser_clone = parser.clone(); + + group.throughput(Throughput::Bytes(input_len as u64)); + group.bench_with_input(BenchmarkId::from_parameter(size), &size, |b, &num_tools| { + let printed_clone = printed.clone(); + let parser = parser_clone.clone(); + let rt = rt.handle().clone(); + let input = &large_json; + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let parser = parser.clone(); + let result = rt.block_on(async { parser.parse_complete(input).await }); + black_box(result.unwrap()); + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let bytes_per_sec = (iters as f64 * input_len as f64) / duration.as_secs_f64(); + let time_per_op = duration.as_millis() as f64 / iters as f64; + + let result = format!( + "{:<25} | {:>10} | {:>10} | {:>12.0} | {:>12.0} | {:>10.1}ms", + format!("{}_tools", num_tools), + num_tools, + input_len, + ops_per_sec, + bytes_per_sec, + time_per_op + ); + add_result("large", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + } + + group.finish(); +} + +fn bench_parser_reuse(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + + let mut group = c.benchmark_group("parser_reuse"); + + // Benchmark creating new registry each time + let printed_new = Arc::new(AtomicBool::new(false)); + group.bench_function("new_registry_each_time", |b| { + let printed_clone = printed_new.clone(); + let rt = rt.handle().clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let registry = ToolParserFactory::new(); + let parser = registry.get_parser("json").unwrap(); + let result = rt.block_on(async { parser.parse_complete(JSON_SIMPLE).await }); + black_box(result.unwrap()); + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let time_per_op = duration.as_micros() as f64 / iters as f64; + + let result = format!( + "{:<25} | {:>12.0} | {:>12.1}µs | {:>15}", + "New Registry Each Time", ops_per_sec, time_per_op, "Baseline" + ); + add_result("reuse", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + // Benchmark reusing registry + let printed_reuse = Arc::new(AtomicBool::new(false)); + let shared_registry = Arc::new(ToolParserFactory::new()); + + group.bench_function("reuse_registry", |b| { + let printed_clone = printed_reuse.clone(); + let registry = shared_registry.clone(); + let rt = rt.handle().clone(); + + b.iter_custom(|iters| { + let parser = registry.get_parser("json").unwrap(); + + let start = Instant::now(); + for _ in 0..iters { + let parser = parser.clone(); + let result = rt.block_on(async { parser.parse_complete(JSON_SIMPLE).await }); + black_box(result.unwrap()); + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let time_per_op = duration.as_micros() as f64 / iters as f64; + + let result = format!( + "{:<25} | {:>12.0} | {:>12.1}µs | {:>15}", + "Reuse Registry", ops_per_sec, time_per_op, "Optimized" + ); + add_result("reuse", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + // Benchmark reusing parser + let printed_parser = Arc::new(AtomicBool::new(false)); + let shared_parser = shared_registry.get_parser("json").unwrap(); + + group.bench_function("reuse_parser", |b| { + let printed_clone = printed_parser.clone(); + let parser = shared_parser.clone(); + let rt = rt.handle().clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let parser = parser.clone(); + let result = rt.block_on(async { parser.parse_complete(JSON_SIMPLE).await }); + black_box(result.unwrap()); + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let time_per_op = duration.as_micros() as f64 / iters as f64; + + let result = format!( + "{:<25} | {:>12.0} | {:>12.1}µs | {:>15}", + "Reuse Parser", ops_per_sec, time_per_op, "Best" + ); + add_result("reuse", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + group.finish(); +} + +fn bench_latency_distribution(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let registry = Arc::new(ToolParserFactory::new()); + + let test_cases = vec![ + ("json", JSON_SIMPLE), + ("mistral", MISTRAL_FORMAT), + ("qwen", QWEN_FORMAT), + ("pythonic", PYTHONIC_FORMAT), + ]; + + let mut group = c.benchmark_group("latency"); + + for (parser_name, input) in test_cases { + let printed = Arc::new(AtomicBool::new(false)); + let registry_clone = registry.clone(); + + group.bench_function(parser_name, |b| { + let printed_clone = printed.clone(); + let registry = registry_clone.clone(); + let rt = rt.handle().clone(); + + b.iter_custom(|iters| { + let parser = registry.get_parser(parser_name).expect("Parser not found"); + + let total_duration = if !printed_clone.load(Ordering::Relaxed) { + let mut latencies = Vec::new(); + + // Warm up + for _ in 0..100 { + let parser = parser.clone(); + rt.block_on(async { parser.parse_complete(input).await }) + .unwrap(); + } + + // Measure for statistics + for _ in 0..1000 { + let parser = parser.clone(); + let start = Instant::now(); + rt.block_on(async { parser.parse_complete(input).await }) + .unwrap(); + let latency = start.elapsed(); + latencies.push(latency); + } + + latencies.sort(); + let p50 = latencies[latencies.len() / 2]; + let p95 = latencies[latencies.len() * 95 / 100]; + let p99 = latencies[latencies.len() * 99 / 100]; + let max = latencies.last().unwrap(); + + let result = format!( + "{:<25} | {:>10.1} | {:>10.1} | {:>10.1} | {:>10.1} | {:>10}", + parser_name, + p50.as_micros() as f64, + p95.as_micros() as f64, + p99.as_micros() as f64, + max.as_micros() as f64, + 1000 + ); + add_result("latency", result); + + printed_clone.store(true, Ordering::Relaxed); + + // Return median for consistency + p50 * iters as u32 + } else { + // Regular benchmark iterations + let start = Instant::now(); + for _ in 0..iters { + let parser = parser.clone(); + rt.block_on(async { parser.parse_complete(input).await }) + .unwrap(); + } + start.elapsed() + }; + + total_duration + }); + }); + } + + group.finish(); +} + +// Print final summary table +fn print_summary() { + println!("\n{}", "=".repeat(120)); + println!("TOOL PARSER BENCHMARK SUMMARY"); + println!("{}", "=".repeat(120)); + + let results = BENCHMARK_RESULTS.lock().unwrap(); + + let mut current_category = String::new(); + for (key, value) in results.iter() { + let category = key.split('_').skip(1).collect::>().join("_"); + + if category != current_category { + current_category = category.clone(); + + // Print section header based on category + println!("\n{}", "-".repeat(120)); + match category.as_str() { + "registry" => { + println!("REGISTRY OPERATIONS"); + println!( + "{:<25} | {:>12} | {:>12} | {:>15}", + "Operation", "Ops/sec", "Time/op", "Notes" + ); + } + "lookup" => { + println!("PARSER LOOKUP PERFORMANCE"); + println!( + "{:<25} | {:>12} | {:>12} | {:>15}", + "Model", "Lookups/sec", "Time/lookup", "Result" + ); + } + "complete" => { + println!("COMPLETE PARSING PERFORMANCE"); + println!( + "{:<25} | {:>10} | {:>12} | {:>12} | {:>12}", + "Parser Format", "Size(B)", "Ops/sec", "Bytes/sec", "Time/op" + ); + } + "streaming" => { + println!("STREAMING PARSING PERFORMANCE"); + println!( + "{:<25} | {:>10} | {:>12} | {:>12} | {:>12}", + "Parser", "Chunks", "Ops/sec", "Chunks/sec", "Time/op" + ); + } + "concurrent" => { + println!("CONCURRENT PARSING"); + println!( + "{:<25} | {:>10} | {:>12} | {:>12} | {:>10}", + "Configuration", "Total Ops", "Ops/sec", "Tools/sec", "Threads" + ); + } + "large" => { + println!("LARGE PAYLOAD PARSING"); + println!( + "{:<25} | {:>10} | {:>10} | {:>12} | {:>12} | {:>12}", + "Payload", "Tools", "Size(B)", "Ops/sec", "Bytes/sec", "Time/op" + ); + } + "reuse" => { + println!("PARSER REUSE COMPARISON"); + println!( + "{:<25} | {:>12} | {:>12} | {:>15}", + "Strategy", "Ops/sec", "Time/op", "Performance" + ); + } + "latency" => { + println!("LATENCY DISTRIBUTION"); + println!( + "{:<25} | {:>10} | {:>10} | {:>10} | {:>10} | {:>10}", + "Parser", "P50(µs)", "P95(µs)", "P99(µs)", "Max(µs)", "Samples" + ); + } + _ => {} + } + println!("{}", "-".repeat(120)); + } + + println!("{}", value); + } + + println!("\n{}", "=".repeat(120)); + + // Print performance analysis + println!("\nPERFORMANCE ANALYSIS:"); + println!("{}", "-".repeat(120)); + + // Calculate and display key metrics + if let Some(new_registry) = results.get("007_reuse") { + if let Some(reuse_parser) = results.get("009_reuse") { + // Extract ops/sec values + let new_ops: f64 = new_registry + .split('|') + .nth(1) + .and_then(|s| s.trim().parse().ok()) + .unwrap_or(0.0); + let reuse_ops: f64 = reuse_parser + .split('|') + .nth(1) + .and_then(|s| s.trim().parse().ok()) + .unwrap_or(0.0); + + if new_ops > 0.0 && reuse_ops > 0.0 { + let improvement = (reuse_ops / new_ops - 1.0) * 100.0; + println!("Parser Reuse Improvement: {:.1}% faster", improvement); + + if improvement < 100.0 { + println!("⚠️ WARNING: Parser reuse improvement is lower than expected!"); + println!(" Expected: >100% improvement with singleton pattern"); + println!(" Actual: {:.1}% improvement", improvement); + println!(" Recommendation: Implement global singleton registry"); + } + } + } + } + + println!("{}", "=".repeat(120)); +} + +fn run_benchmarks(c: &mut Criterion) { + bench_registry_creation(c); + bench_parser_lookup(c); + bench_complete_parsing(c); + bench_streaming_parsing(c); + bench_concurrent_parsing(c); + bench_large_payloads(c); + bench_parser_reuse(c); + bench_latency_distribution(c); + + // Print summary at the end + print_summary(); +} + +criterion_group!(benches, run_benchmarks); +criterion::criterion_main!(benches); diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/.gitignore b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/.gitignore new file mode 100644 index 00000000..9e1dd242 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/.gitignore @@ -0,0 +1,24 @@ +# Build artifacts +target/ +lib/ + +# Compiled binaries +examples/simple/simple +examples/streaming/streaming + +# Go build artifacts +*.o +*.a +*.so +*.dylib + +# IDE and editor files +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Environment files +.env +.env.local diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/Cargo.toml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/Cargo.toml new file mode 100644 index 00000000..28f20905 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "sgl-model-gateway-golang" +version = "0.2.3" +edition = "2021" + +[lib] +name = "sgl_model_gateway_go" +crate-type = ["cdylib"] + +[dependencies] +tokio = { version = "1.42.0", features = ["full"] } +serde_json = { version = "1.0", default-features = false, features = [ + "std", + "preserve_order", +] } +uuid = { version = "1.10", features = ["v4", "serde"] } +once_cell = "1.21.3" +futures-util = "0.3" +tracing = "0.1" + +[dependencies.sgl-model-gateway] +path = "../.." +default-features = true + +[features] +default = [] +vendored-openssl = ["sgl-model-gateway/vendored-openssl"] + +[profile.release] +opt-level = "z" # Optimize for size +lto = "fat" # Full LTO for smaller binaries +codegen-units = 1 # Better optimization, slower compile +strip = true # Strip debug symbols + +[profile.ci] +inherits = "release" +opt-level = 2 # Lighter optimization (still fast runtime, much faster compile) +lto = "thin" # Thin LTO - good balance +codegen-units = 16 # More parallelization for faster builds +strip = true + +[profile.dev] +opt-level = 0 +debug = 1 +split-debuginfo = "unpacked" +incremental = true +codegen-units = 256 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/Makefile b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/Makefile new file mode 100644 index 00000000..a1b73404 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/Makefile @@ -0,0 +1,103 @@ +# Makefile for sglang-router golang bindings +# This builds the Rust FFI library and provides convenience targets for Go development + +# Configuration +CARGO_BUILD_DIR ?= $(shell pwd)/target +BUILD_MODE ?= release +LIB_NAME = libsglang_router_rs + +# Detect OS +UNAME_S := $(shell uname -s) +ifeq ($(UNAME_S),Linux) + LIB_EXT = .so + LD_LIBRARY_PATH_VAR = LD_LIBRARY_PATH +endif +ifeq ($(UNAME_S),Darwin) + LIB_EXT = .dylib + LD_LIBRARY_PATH_VAR = DYLD_LIBRARY_PATH +endif + +# Paths +ROOT_DIR := $(shell pwd) +RUST_SRC_DIR := $(ROOT_DIR)/src +LIB_BUILD_DIR := $(CARGO_BUILD_DIR)/$(BUILD_MODE) +LIB_BUILD_PATH := $(LIB_BUILD_DIR)/$(LIB_NAME)$(LIB_EXT) +LIB_EXPORT_DIR := $(ROOT_DIR)/lib +LIB_EXPORT_PATH := $(LIB_EXPORT_DIR)/$(LIB_NAME)$(LIB_EXT) + +# Python LDFLAGS (needed for Rust FFI that depends on Python) +PYTHON_LDFLAGS := $(shell python3-config --ldflags --embed 2>/dev/null || python3-config --ldflags 2>/dev/null || echo "") + +# CGO flags - use exported lib directory if available, otherwise build directory +LIB_DIR := $(if $(wildcard $(LIB_EXPORT_PATH)),$(LIB_EXPORT_DIR),$(LIB_BUILD_DIR)) +export CGO_LDFLAGS = -L$(LIB_DIR) -lsglang_router_rs $(PYTHON_LDFLAGS) -ldl +export $(LD_LIBRARY_PATH_VAR) := $(LIB_DIR):$($(LD_LIBRARY_PATH_VAR)) + +.PHONY: all build build-dev lib lib-clean clean test examples help run-simple run-streaming check-lib + +help: + @echo "Available targets:" + @echo " build - Build release version of Rust FFI library" + @echo " build-dev - Build debug version of Rust FFI library" + @echo " lib - Copy built library to ./lib directory" + @echo " lib-clean - Clean ./lib directory" + @echo " clean - Clean build artifacts" + @echo " test - Run Go tests" + @echo " examples - Build example programs" + @echo " run-simple - Run simple example" + @echo " run-streaming - Run streaming example" + +all: build + +build: + @echo "Building Rust FFI library (release mode)..." + @CARGO_TARGET_DIR=$(CARGO_BUILD_DIR) cargo build --release --manifest-path Cargo.toml + @echo "Library built at: $(LIB_BUILD_PATH)" + +build-dev: + @echo "Building Rust FFI library (debug mode)..." + @CARGO_TARGET_DIR=$(CARGO_BUILD_DIR) cargo build --manifest-path Cargo.toml + @echo "Library built at: $(LIB_BUILD_DIR)/debug/$(LIB_NAME)$(LIB_EXT)" + +lib: build + @echo "Copying library to ./lib directory..." + @mkdir -p $(LIB_EXPORT_DIR) + @cp $(LIB_BUILD_PATH) $(LIB_EXPORT_PATH) + @echo "Library exported at: $(LIB_EXPORT_PATH)" + +lib-clean: + @echo "Cleaning ./lib directory..." + @rm -rf $(LIB_EXPORT_DIR) + @echo "Lib directory cleaned" + +clean: lib-clean + @echo "Cleaning build artifacts..." + @CARGO_TARGET_DIR=$(CARGO_BUILD_DIR) cargo clean --manifest-path Cargo.toml + @echo "Clean complete" + +test: build + @echo "Running Go tests..." + @go test ./... + +examples: build + @echo "Building example programs..." + @cd examples/simple && go build -o simple main.go + @cd examples/streaming && go build -o streaming main.go + @echo "Examples built" + +run-simple: build + @echo "Running simple example..." + @cd examples/simple && bash run.sh + +run-streaming: build + @echo "Running streaming example..." + @cd examples/streaming && bash run.sh + +# Check if library exists (either in lib dir or build dir) +check-lib: + @if [ ! -f "$(LIB_EXPORT_PATH)" ] && [ ! -f "$(LIB_BUILD_PATH)" ]; then \ + echo "Error: Library not found at $(LIB_EXPORT_PATH) or $(LIB_BUILD_PATH)"; \ + echo "Run 'make build' or 'make lib' first"; \ + exit 1; \ + fi + @echo "Library found at: $(LIB_DIR)/$(LIB_NAME)$(LIB_EXT)" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/README.md new file mode 100644 index 00000000..f140440f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/README.md @@ -0,0 +1,552 @@ +# SGLang Go gRPC SDK + +A high-level Go SDK for interacting with SGLang gRPC API, designed with an OpenAI-style API for familiarity and ease of use. + +**Location**: `sgl-router/bindings/golang/` + +## Table of Contents + +- [Features](#features) +- [Installation](#installation) +- [Quick Start](#quick-start) + - [Basic Usage](#basic-usage) + - [Streaming Usage](#streaming-usage) +- [Examples](#examples) +- [Configuration](#configuration) +- [API Reference](#api-reference) +- [Testing](#testing) + - [Unit Tests](#unit-tests) + - [Integration Tests](#integration-tests) + - [Benchmarks](#benchmarks) +- [Documentation](#documentation) +- [Development](#development) +- [Troubleshooting](#troubleshooting) +- [License](#license) + +## Features + +- **OpenAI-style API**: Familiar interface similar to OpenAI Go SDK +- **Streaming Support**: Real-time streaming chat completions +- **Non-streaming Support**: Simple request/response API +- **Tool Calling**: Support for function calling and tool use +- **Type-safe**: Full Go type definitions for requests and responses +- **Comprehensive Testing**: 18+ unit and integration tests +- **Thread-safe**: All public methods are safe for concurrent use +- **Well-documented**: Full API documentation with examples + +## Installation + +```bash +go get github.com/sglang/sglang-go-grpc-sdk +``` + +### Build Requirements + +- Go 1.21 or later +- Rust toolchain (for building the FFI library) +- Python 3.x (for Python bindings in Rust FFI) +- Tokio runtime for async operations + +## Quick Start + +### Basic Usage (Non-streaming) + +```go +package main + +import ( + "context" + "fmt" + "log" + + "github.com/sglang/sglang-go-grpc-sdk" +) + +func main() { + // Create client + client, err := sglang.NewClient(sglang.ClientConfig{ + Endpoint: "grpc://localhost:20000", + TokenizerPath: "/path/to/tokenizer", + }) + if err != nil { + log.Fatal(err) + } + defer client.Close() + + // Create completion + resp, err := client.CreateChatCompletion(context.Background(), sglang.ChatCompletionRequest{ + Model: "default", + Messages: []sglang.ChatMessage{ + {Role: "user", Content: "Hello!"}, + }, + Stream: false, + }) + if err != nil { + log.Fatal(err) + } + + fmt.Println(resp.Choices[0].Message.Content) + fmt.Printf("Usage: Prompt=%d, Completion=%d, Total=%d\n", + resp.Usage.PromptTokens, + resp.Usage.CompletionTokens, + resp.Usage.TotalTokens) +} +``` + +### Streaming Usage + +```go +package main + +import ( + "context" + "fmt" + "io" + "log" + + "github.com/sglang/sglang-go-grpc-sdk" +) + +func main() { + // Create client + client, err := sglang.NewClient(sglang.ClientConfig{ + Endpoint: "grpc://localhost:20000", + TokenizerPath: "/path/to/tokenizer", + }) + if err != nil { + log.Fatal(err) + } + defer client.Close() + + // Create streaming completion + ctx := context.Background() + stream, err := client.CreateChatCompletionStream(ctx, sglang.ChatCompletionRequest{ + Model: "default", + Messages: []sglang.ChatMessage{ + {Role: "user", Content: "Tell me a story"}, + }, + Stream: true, + MaxCompletionTokens: intPtr(500), + }) + if err != nil { + log.Fatal(err) + } + defer stream.Close() + + // Read streaming response + for { + chunk, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + log.Fatal(err) + } + + for _, choice := range chunk.Choices { + if choice.Delta.Content != "" { + fmt.Print(choice.Delta.Content) + } + } + } + fmt.Println() // newline +} + +// Helper functions for optional pointer fields +func intPtr(i int) *int { + return &i +} + +func float32Ptr(f float32) *float32 { + return &f +} +``` + +## Examples + +The SDK includes several examples in the `examples/` directory: + +- **simple**: Basic non-streaming chat completion example +- **streaming**: Real-time streaming with performance metrics + +### Running Examples + +```bash +# Run simple example +cd bindings/golang/examples/simple +bash run.sh + +# Run streaming example +cd bindings/golang/examples/streaming +bash run.sh + +# Or use Makefile from bindings/golang directory +cd bindings/golang +make run-simple +make run-streaming +``` + +Examples automatically detect the server endpoint and tokenizer path via environment variables or defaults. + +## Configuration + +### Environment Variables + +- `SGL_GRPC_ENDPOINT`: gRPC server endpoint (default: `grpc://localhost:20000`) +- `SGL_TOKENIZER_PATH`: Path to tokenizer directory (required) +- `CARGO_BUILD_DIR`: Rust build output directory (auto-detected if not set) + +### ClientConfig + +```go +type ClientConfig struct { + // Endpoint is the gRPC endpoint URL (e.g., "grpc://localhost:20000") + // Required field. Must include the scheme (grpc://) and port number. + Endpoint string + + // TokenizerPath is the path to the tokenizer directory containing + // tokenizer configuration files (e.g., tokenizer.json, vocab.json) + // Required field. + TokenizerPath string +} +``` + +## API Reference + +### Client Methods + +```go +type Client struct { + // Thread-safe client for SGLang gRPC API +} + +// Creates a new client with the given configuration +func NewClient(config ClientConfig) (*Client, error) + +// Closes the client and releases all resources +func (c *Client) Close() error + +// Creates a non-streaming chat completion +func (c *Client) CreateChatCompletion(ctx context.Context, req ChatCompletionRequest) (*ChatCompletionResponse, error) + +// Creates a streaming chat completion +func (c *Client) CreateChatCompletionStream(ctx context.Context, req ChatCompletionRequest) (*ChatCompletionStream, error) +``` + +### Request Types + +- `ChatCompletionRequest`: Main request type for chat completions + - Model, Messages, Stream, Temperature, TopP, MaxCompletionTokens, Tools, etc. +- `ChatMessage`: Individual message in a conversation + - Role, Content +- `Tool`: Tool/function definition for function calling + - Type, Function (name, description, parameters) + +### Response Types + +- `ChatCompletionResponse`: Non-streaming response + - ID, Model, Created, Choices, Usage +- `ChatCompletionStreamResponse`: Streaming response chunk + - Same structure as above but for incremental updates +- `Message`: Complete message with content and tool calls +- `ToolCall`: Tool call information with function and arguments +- `Usage`: Token usage statistics + - PromptTokens, CompletionTokens, TotalTokens + +## Testing + +The SDK includes comprehensive testing infrastructure with both unit and integration tests. + +### Unit Tests + +Unit tests are located in `client_test.go` and test individual components without requiring a server. + +#### Running Unit Tests + +```bash +# Run all unit tests +go test ./... + +# Run with verbose output +go test -v ./... + +# Run specific test +go test -run TestClientConfig + +# Run tests with race detector (detects concurrency issues) +go test -race ./... + +# Run with coverage analysis +go test -cover ./... + +# Generate detailed coverage report +go test -coverprofile=coverage.out ./... +go tool cover -html=coverage.out -o coverage.html +``` + +#### Unit Test Coverage + +- **Configuration validation** (`TestClientConfig`) - Validates ClientConfig requirements +- **Type structures** - Verifying all struct types work correctly +- **Response handling** - Testing response parsing and validation +- **Concurrent operations** (`TestConcurrentClientOperations`) - Thread-safety verification +- **Benchmarks** (`BenchmarkChatCompletionRequest`) - Performance measurement + +**Test Files**: +- `client_test.go` - 10 unit tests covering core functionality +- Tests cover: config validation, message types, request validation, close operations, response types, streaming, tools, concurrency, and context cancellation + +### Integration Tests + +Integration tests require a running SGLang server and test the full client-server interaction. + +#### Prerequisites + +1. Start an SGLang server: + +```bash +# Using Python (requires sglang package installed) +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-hf + +# Or using pre-built Docker image +docker run -p 20000:20000 lmsys/sglang:latest + +# Or build your own +sglang launch_server --model-path +``` + +2. Set required environment variables: + +```bash +# Set the gRPC endpoint (default: grpc://localhost:20000) +export SGL_GRPC_ENDPOINT=grpc://localhost:20000 + +# Set the tokenizer path (required) +export SGL_TOKENIZER_PATH=/path/to/tokenizer +``` + +#### Running Integration Tests + +```bash +# Run all integration tests +go test -tags=integration ./... + +# Run specific integration test +go test -tags=integration -run TestIntegrationNonStreamingCompletion + +# Run with verbose output +go test -tags=integration -v ./... + +# Run with race detector +go test -tags=integration -race ./... +``` + +#### Integration Test Coverage + +**Test File**: `integration_test.go` - 4 integration tests + +- `TestIntegrationNonStreamingCompletion` - Basic non-streaming request/response +- `TestIntegrationStreamingCompletion` - Streaming response handling +- `TestIntegrationConcurrentRequests` - Multiple simultaneous requests +- `TestIntegrationContextCancellation` - Context timeout and cancellation + +### Benchmarks + +Measure performance of SDK operations: + +```bash +# Run all benchmarks +go test -bench=. -benchmem ./... + +# Run specific benchmark +go test -bench=BenchmarkChatCompletionRequest -benchmem + +# Run for longer duration +go test -bench=. -benchtime=10s ./... +``` + +Current benchmarks: +- `BenchmarkChatCompletionRequest` - Measures request creation performance + +### CI/CD Integration + +Add to your GitHub Actions workflow: + +```yaml +- name: Run Go tests + run: | + go test -race -cover ./... + +- name: Run integration tests (on main branch) + if: github.ref == 'refs/heads/main' + env: + SGL_GRPC_ENDPOINT: grpc://localhost:20000 + SGL_TOKENIZER_PATH: /path/to/tokenizer + run: go test -tags=integration ./... +``` + +## Documentation + +### Code Documentation + +All public types and functions include comprehensive documentation: + +1. **Package-level documentation** in `client.go` with usage examples +2. **Type documentation** for all structs with field descriptions +3. **Function documentation** with: + - Purpose and behavior description + - Parameter documentation with types and constraints + - Return value documentation + - Error cases and handling + - Safety notes (for FFI functions) + - Usage examples + +### Key Documented Components + +- `Client` - Main client with thread-safety notes +- `ClientConfig` - Configuration requirements and validation rules +- `ChatCompletionRequest` - Request structure with field descriptions +- `ChatCompletionResponse` - Response structure and usage +- `ChatCompletionStreamResponse` - Streaming response format +- `Usage` - Token usage information structure +- `Tool`, `Function`, `ToolCall` - Tool call structures + +### Viewing Documentation + +Generate and view HTML documentation: + +```bash +# Install godoc (if not already installed) +go install golang.org/x/tools/cmd/godoc@latest + +# Generate and serve documentation +godoc -http=:6060 + +# Visit: http://localhost:6060/pkg/github.com/sglang/sglang-go-grpc-sdk/ +``` + +## Development + +### Building + +```bash +cd bindings/golang + +# Build the Go bindings (compiles Rust FFI library) +make build + +# Clean build +make clean && make build +``` + +### Code Quality + +Ensure code quality before committing: + +```bash +# Run Go vet (check for potential bugs) +go vet ./... + +# Format code +go fmt ./... + +# Run all tests with race detection +go test -race ./... +``` + +### Project Structure + +``` +bindings/golang/ +├── client.go # Main client implementation +├── client_test.go # Unit tests +├── integration_test.go # Integration tests +├── README.md # This file +├── Makefile # Build automation +├── Cargo.toml # Rust FFI dependencies +├── examples/ # Example programs +│ ├── simple/ # Non-streaming example +│ └── streaming/ # Streaming example +├── src/ # Rust FFI source +│ ├── client.rs # Client FFI +│ ├── stream.rs # Stream handling +│ ├── grpc_converter.rs # Response conversion +│ └── ... +└── internal/ # Internal packages + └── ffi/ # FFI bindings +``` + +## Troubleshooting + +### Connection Errors + +**Error**: `connection refused` or `failed to dial` + +**Solution**: +1. Ensure SGLang server is running: `python -m sglang.launch_server` +2. Check endpoint: `echo $SGL_GRPC_ENDPOINT` +3. Verify port is not blocked: `nc -zv localhost 20000` + +### Tokenizer Not Found + +**Error**: `tokenizer path not found` or `tokenizer configuration missing` + +**Solution**: +1. Set `SGL_TOKENIZER_PATH` environment variable +2. Verify path contains required files: `ls $SGL_TOKENIZER_PATH` +3. Files should include: `tokenizer.json`, `vocab.json`, `config.json` + +### Build Failures + +**Error**: `library 'sglang_router_rs' not found` + +**Solution**: +1. Rebuild Rust library: `cd sgl-router/bindings/golang && make build` +2. Or manually with cargo: `cd sgl-router/bindings/golang && cargo build --release` +3. Set `CARGO_BUILD_DIR` if using non-standard build location +4. Ensure Rust toolchain is installed: `rustup toolchain list` + +### Tests Hanging + +**Error**: Tests seem to hang indefinitely + +**Solution**: +1. Use timeout for hanging tests: `timeout 30s go test ./...` +2. Run with verbose output to see which test hangs: `go test -v ./...` +3. Ensure server is responsive: `grpcurl -plaintext localhost:20000 list` + +### Memory Issues + +**Error**: Out of memory during tests + +**Solution**: +```bash +# Run with memory limit for long-running tests +GODEBUG=madvdontneed=1 go test -timeout 5m ./... + +# Monitor memory during tests +watch -n1 'ps aux | grep test' +``` + +## Contributing + +When adding new features: + +1. Add comprehensive documentation to public types/functions +2. Include usage examples for complex APIs +3. Add unit tests covering happy path and error cases +4. Add integration tests if server interaction required +5. Ensure code passes `go vet` and `go test -race` +6. Update this README if adding new features + +## License + +See LICENSE file for details. + +--- + +**Need Help?** +- Check examples in `examples/` directory +- Run tests to see working code: `go test -v ./...` +- Review function documentation: `godoc` or inline comments +- Check troubleshooting section above diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/client.go b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/client.go new file mode 100644 index 00000000..110135da --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/client.go @@ -0,0 +1,510 @@ +// Package sglang provides a Go SDK for SGLang gRPC API. +// +// SGLang is a fast language model serving framework. This package provides a Go client +// library for interacting with SGLang's gRPC API, following the style of OpenAI's Go SDK. +// +// Basic usage: +// +// client, err := sglang.NewClient(sglang.ClientConfig{ +// Endpoint: "grpc://localhost:20000", +// TokenizerPath: "/path/to/tokenizer", +// }) +// if err != nil { +// log.Fatal(err) +// } +// defer client.Close() +// +// resp, err := client.CreateChatCompletion(ctx, sglang.ChatCompletionRequest{ +// Model: "default", +// Messages: []sglang.ChatMessage{ +// {Role: "user", Content: "Hello"}, +// }, +// }) +// +// For streaming responses, use CreateChatCompletionStream instead. +package sglang + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "strings" + "sync" + + "github.com/sglang/sglang-go-grpc-sdk/internal/ffi" +) + +// Client is the main client for interacting with SGLang gRPC API. +// It manages the connection to the SGLang server and handles both streaming +// and non-streaming chat completions. +// +// Thread-safe: All public methods are safe for concurrent use. +type Client struct { + endpoint string + tokenizerPath string + clientHandle *ffi.SglangClientHandle + mu sync.RWMutex +} + +// ClientConfig holds configuration for creating a new client. +type ClientConfig struct { + // Endpoint is the gRPC endpoint URL (e.g., "grpc://localhost:20000"). + // Required field. Must include the scheme (grpc://) and port number. + Endpoint string + + // TokenizerPath is the path to the tokenizer directory containing + // tokenizer configuration files (e.g., tokenizer.json, vocab.json). + // Required field. + TokenizerPath string +} + +// NewClient creates a new SGLang client with the given configuration. +// +// The client maintains a long-lived connection to the SGLang server and should +// be reused for multiple requests. Call Close() to release resources. +// +// Returns an error if: +// - Endpoint is empty +// - TokenizerPath is empty +// - Connection to the server fails +func NewClient(config ClientConfig) (*Client, error) { + if config.Endpoint == "" { + return nil, errors.New("endpoint is required") + } + if config.TokenizerPath == "" { + return nil, errors.New("tokenizer path is required") + } + + clientHandle, err := ffi.NewClient(config.Endpoint, config.TokenizerPath) + if err != nil { + return nil, fmt.Errorf("failed to create client: %w", err) + } + + return &Client{ + endpoint: config.Endpoint, + tokenizerPath: config.TokenizerPath, + clientHandle: clientHandle, + }, nil +} + +// Close closes the client and releases all resources. +// +// After Close() is called, the client cannot be used for further requests. +// Calling Close() multiple times is safe and idempotent. +func (c *Client) Close() error { + c.mu.Lock() + defer c.mu.Unlock() + + if c.clientHandle != nil { + c.clientHandle.Free() + c.clientHandle = nil + } + return nil +} + +// ChatCompletionRequest represents a request for chat completion. +// It follows the OpenAI API style for familiar usage. +type ChatCompletionRequest struct { + // Model specifies the model to use for completion (e.g., "default") + Model string `json:"model"` + // Messages is the list of messages in the conversation + Messages []ChatMessage `json:"messages"` + Temperature *float32 `json:"temperature,omitempty"` + TopP *float32 `json:"top_p,omitempty"` + TopK *int `json:"top_k,omitempty"` + MaxCompletionTokens *int `json:"max_completion_tokens,omitempty"` + Stream bool `json:"stream"` + Tools []Tool `json:"tools,omitempty"` + ToolChoice interface{} `json:"tool_choice,omitempty"` + Stop interface{} `json:"stop,omitempty"` + StopTokenIDs []int `json:"stop_token_ids,omitempty"` + SkipSpecialTokens bool `json:"skip_special_tokens,omitempty"` + FrequencyPenalty *float32 `json:"frequency_penalty,omitempty"` + PresencePenalty *float32 `json:"presence_penalty,omitempty"` + ResponseFormat *ResponseFormat `json:"response_format,omitempty"` + Seed *int `json:"seed,omitempty"` + Logprobs bool `json:"logprobs,omitempty"` + TopLogprobs *int `json:"top_logprobs,omitempty"` + User string `json:"user,omitempty"` +} + +// ChatMessage represents a single message in a chat conversation +type ChatMessage struct { + Role string `json:"role"` + Content interface{} `json:"content"` + Name string `json:"name,omitempty"` +} + +// Tool represents a tool/function that can be called +type Tool struct { + Type string `json:"type"` + Function Function `json:"function"` +} + +// Function represents a function definition +type Function struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + Parameters map[string]interface{} `json:"parameters"` +} + +// ResponseFormat represents the response format +type ResponseFormat struct { + Type string `json:"type"` +} + +// ChatCompletionResponse represents a non-streaming chat completion response +type ChatCompletionResponse struct { + ID string `json:"id"` + Object string `json:"object"` + Created int64 `json:"created"` + Model string `json:"model"` + SystemFingerprint string `json:"system_fingerprint,omitempty"` + Choices []Choice `json:"choices"` + Usage Usage `json:"usage"` +} + +// Choice represents a choice in the completion response +type Choice struct { + Index int `json:"index"` + Message Message `json:"message"` + FinishReason string `json:"finish_reason"` +} + +// Message represents a message in the response +type Message struct { + Role string `json:"role"` + Content string `json:"content"` + ToolCalls []ToolCall `json:"tool_calls,omitempty"` +} + +// ToolCall represents a tool call in the response +type ToolCall struct { + ID string `json:"id"` + Type string `json:"type"` + Function FunctionCall `json:"function"` +} + +// FunctionCall represents a function call +type FunctionCall struct { + Name string `json:"name"` + Arguments string `json:"arguments"` +} + +// Usage represents token usage information +type Usage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` +} + +// ChatCompletionStreamResponse represents a streaming chat completion response +type ChatCompletionStreamResponse struct { + ID string `json:"id"` + Object string `json:"object"` + Created int64 `json:"created"` + Model string `json:"model"` + SystemFingerprint string `json:"system_fingerprint,omitempty"` + Choices []StreamChoice `json:"choices"` + Usage *Usage `json:"usage,omitempty"` +} + +// StreamChoice represents a choice in a streaming response +type StreamChoice struct { + Index int `json:"index"` + Delta MessageDelta `json:"delta"` + FinishReason string `json:"finish_reason,omitempty"` +} + +// MessageDelta represents incremental message updates +type MessageDelta struct { + Role string `json:"role,omitempty"` + Content string `json:"content,omitempty"` + ToolCalls []ToolCall `json:"tool_calls,omitempty"` +} + +// CreateChatCompletion creates a non-streaming chat completion with context support. +// +// Context Support: +// The ctx parameter is fully supported for cancellation and timeouts: +// - If ctx is cancelled, the request will be interrupted on the next stream.Recv() call +// - If ctx times out, the request will return context.DeadlineExceeded +// +// Example with timeout: +// +// ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) +// defer cancel() +// resp, err := client.CreateChatCompletion(ctx, req) +// +// Note: Internally, this creates a stream and collects all chunks, +// so context monitoring happens at the chunk level. +func (c *Client) CreateChatCompletion(ctx context.Context, req ChatCompletionRequest) (*ChatCompletionResponse, error) { + // For non-streaming, we'll collect all chunks and return the final response + req.Stream = true // We still use streaming internally, but collect all chunks + + // Prepare request: if Tools is empty, set to nil for proper JSON serialization + if len(req.Tools) == 0 { + req.Tools = nil + } + + stream, err := c.CreateChatCompletionStream(ctx, req) + if err != nil { + return nil, err + } + defer stream.Close() + + var fullContent strings.Builder + var fullToolCalls []ToolCall + var finishReason string + var usage Usage + var responseID string + var created int64 + var model string + var systemFingerprint string + + for { + chunk, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if chunk.ID != "" { + responseID = chunk.ID + } + if chunk.Created > 0 { + created = chunk.Created + } + if chunk.Model != "" { + model = chunk.Model + } + if chunk.SystemFingerprint != "" { + systemFingerprint = chunk.SystemFingerprint + } + + for _, choice := range chunk.Choices { + if choice.Delta.Content != "" { + fullContent.WriteString(choice.Delta.Content) + } + if len(choice.Delta.ToolCalls) > 0 { + fullToolCalls = append(fullToolCalls, choice.Delta.ToolCalls...) + } + // Always update finish_reason if present (even if empty string, but should not be empty) + // The last chunk (Complete message) should have finish_reason set + if choice.FinishReason != "" { + finishReason = choice.FinishReason + } + } + + // Extract usage from chunk if available (usually in the last chunk) + // Always update usage if present, as the last chunk should have the final usage + if chunk.Usage != nil { + usage = *chunk.Usage + } + } + + // Build final response + message := Message{ + Role: "assistant", + Content: fullContent.String(), + } + if len(fullToolCalls) > 0 { + message.ToolCalls = fullToolCalls + } + + // Ensure finish_reason is set (defensive check) + // If finish_reason is still empty, default to "stop" + if finishReason == "" { + finishReason = "stop" + } + + return &ChatCompletionResponse{ + ID: responseID, + Object: "chat.completion", + Created: created, + Model: model, + SystemFingerprint: systemFingerprint, + Choices: []Choice{ + { + Index: 0, + Message: message, + FinishReason: finishReason, + }, + }, + Usage: usage, + }, nil +} + +// ChatCompletionStream represents a streaming chat completion +type ChatCompletionStream struct { + stream *ffi.SglangStreamHandle + mu sync.Mutex + done bool // Track if stream has been marked as done + ctx context.Context // Context for cancellation support + cancel context.CancelFunc // Cancel function to stop monitoring goroutine + closed chan struct{} // Signal when stream is closed +} + +// Recv receives the next chunk from the stream. +// +// Supports context cancellation: if the context passed to CreateChatCompletionStream +// is cancelled, Recv will return context.Canceled error on the next call. +func (s *ChatCompletionStream) Recv() (*ChatCompletionStreamResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + + // Check if context was cancelled + select { + case <-s.ctx.Done(): + return nil, s.ctx.Err() // Returns context.Canceled or context.DeadlineExceeded + default: + } + + if s.stream == nil { + return nil, io.EOF + } + + // If stream was already marked as done, immediately return EOF + // This prevents calling ReadNext() again after isDone=1 + if s.done { + return nil, io.EOF + } + + // Loop to handle empty responses (Ok(None) from Rust) + // Keep reading until we get actual data or stream ends + for { + responseJSON, isDone, err := s.stream.ReadNext() + if err != nil { + return nil, err + } + + // Mark stream as done if ReadNext indicates completion + if isDone { + s.done = true + } + + // If we have a response, parse and return it + if responseJSON != "" { + var response ChatCompletionStreamResponse + if err := json.Unmarshal([]byte(responseJSON), &response); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + return &response, nil + } + + // If stream is done but no response, return EOF + if isDone { + return nil, io.EOF + } + + // Empty response and stream not done - loop to read next chunk + // This handles Ok(None) cases where Rust returns no data but stream continues + } +} + +// Close closes the stream and cancels any pending operations. +func (s *ChatCompletionStream) Close() error { + s.mu.Lock() + defer s.mu.Unlock() + + // Cancel the context to signal the monitoring goroutine to stop + if s.cancel != nil { + s.cancel() + } + + // Signal that stream is closed + select { + case <-s.closed: + // Already closed + default: + close(s.closed) + } + + // Free the stream to mark it as completed + // This prevents AbortOnDropStream from sending abort when dropped + if s.stream != nil { + s.stream.Free() + s.stream = nil + } + return nil +} + +// CreateChatCompletionStream creates a streaming chat completion with context cancellation support. +// +// Context Support: +// The ctx parameter is now fully supported for cancellation and timeouts: +// - If ctx is cancelled, stream.Recv() will return context.Canceled on the next call +// - If ctx times out (WithTimeout), stream.Recv() will return context.DeadlineExceeded +// - Calling stream.Close() also cancels the context +// +// Example with timeout: +// +// ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) +// defer cancel() +// stream, err := client.CreateChatCompletionStream(ctx, req) +// // Stream will auto-close if 30 seconds elapse +// +// Example with cancellation: +// +// ctx, cancel := context.WithCancel(context.Background()) +// stream, err := client.CreateChatCompletionStream(ctx, req) +// go func() { +// time.Sleep(5*time.Second) +// cancel() // Cancel after 5 seconds +// }() +func (c *Client) CreateChatCompletionStream(ctx context.Context, req ChatCompletionRequest) (*ChatCompletionStream, error) { + c.mu.RLock() + defer c.mu.RUnlock() + + if c.clientHandle == nil { + return nil, errors.New("client is closed") + } + + // Marshal request to JSON, then ensure tools field is always present. + // Due to omitempty tag, empty Tools slice will be omitted from JSON. + // We need to ensure tools field is always present as [] when empty (not omitted), + // matching the behavior of complete_sdk example. + reqJSON, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + + // Unmarshal into map and ensure tools field is present + var reqMap map[string]interface{} + if err := json.Unmarshal(reqJSON, &reqMap); err != nil { + return nil, fmt.Errorf("failed to unmarshal request to map: %w", err) + } + + // Add empty tools array if not present + if _, exists := reqMap["tools"]; !exists { + reqMap["tools"] = []interface{}{} + } + + // Marshal back to JSON + reqJSON, err = json.Marshal(reqMap) + if err != nil { + return nil, fmt.Errorf("failed to marshal request map to JSON: %w", err) + } + + // Create stream + streamHandle, err := c.clientHandle.ChatCompletionStream(string(reqJSON)) + if err != nil { + return nil, fmt.Errorf("failed to create stream: %w", err) + } + + // Create a child context from the provided context for cancellation support + streamCtx, cancel := context.WithCancel(ctx) + + stream := &ChatCompletionStream{ + stream: streamHandle, + ctx: streamCtx, + cancel: cancel, + closed: make(chan struct{}), + } + + return stream, nil +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/client_test.go b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/client_test.go new file mode 100644 index 00000000..9b773669 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/client_test.go @@ -0,0 +1,325 @@ +package sglang + +import ( + "context" + "testing" +) + +// TestClientConfig tests ClientConfig validation +func TestClientConfig(t *testing.T) { + tests := []struct { + name string + config ClientConfig + wantErr bool + }{ + { + name: "valid config", + config: ClientConfig{ + Endpoint: "grpc://localhost:20000", + TokenizerPath: "/path/to/tokenizer", + }, + wantErr: false, + }, + { + name: "missing endpoint", + config: ClientConfig{ + Endpoint: "", + TokenizerPath: "/path/to/tokenizer", + }, + wantErr: true, + }, + { + name: "missing tokenizer path", + config: ClientConfig{ + Endpoint: "grpc://localhost:20000", + TokenizerPath: "", + }, + wantErr: true, + }, + { + name: "both missing", + config: ClientConfig{ + Endpoint: "", + TokenizerPath: "", + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := NewClient(tt.config) + if (err != nil) != tt.wantErr { + t.Errorf("NewClient() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +// TestChatMessageTypes tests ChatMessage struct and its variants +func TestChatMessageTypes(t *testing.T) { + msg := ChatMessage{ + Role: "user", + Content: "Hello", + } + + if msg.Role != "user" { + t.Errorf("Expected role 'user', got '%s'", msg.Role) + } + if msg.Content != "Hello" { + t.Errorf("Expected content 'Hello', got '%s'", msg.Content) + } +} + +// TestChatCompletionRequestValidation tests ChatCompletionRequest validation +func TestChatCompletionRequestValidation(t *testing.T) { + // Test valid request + req := ChatCompletionRequest{ + Model: "default", + Messages: []ChatMessage{ + {Role: "user", Content: "test"}, + }, + Stream: false, + } + + if req.Model == "" { + t.Error("Expected model to be set") + } + + if len(req.Messages) == 0 { + t.Error("Expected messages to be non-empty") + } + + if req.Messages[0].Role != "user" { + t.Errorf("Expected first message role 'user', got '%s'", req.Messages[0].Role) + } +} + +// TestClientClose tests that Close can be called multiple times safely +func TestClientClose(t *testing.T) { + // Create a mock client (note: in real tests, you might want to skip this + // if it requires actual server connection) + config := ClientConfig{ + Endpoint: "grpc://localhost:20000", + TokenizerPath: "/path/to/tokenizer", + } + + // Skip if connection fails (expected in unit test environment) + client, err := NewClient(config) + if err != nil { + t.Skip("Skipping client close test: server not available") + } + + // First close should succeed + if err := client.Close(); err != nil { + t.Errorf("First Close() failed: %v", err) + } + + // Second close should also succeed (idempotent) + if err := client.Close(); err != nil { + t.Errorf("Second Close() failed: %v", err) + } +} + +// TestChatCompletionResponseTypes tests response type structures +func TestChatCompletionResponseTypes(t *testing.T) { + resp := ChatCompletionResponse{ + ID: "test-id", + Model: "default", + Created: 1234567890, + Choices: []Choice{ + { + Message: Message{ + Role: "assistant", + Content: "Hello", + }, + FinishReason: "stop", + }, + }, + Usage: Usage{ + PromptTokens: 10, + CompletionTokens: 20, + TotalTokens: 30, + }, + } + + if resp.ID != "test-id" { + t.Errorf("Expected ID 'test-id', got '%s'", resp.ID) + } + + if len(resp.Choices) != 1 { + t.Errorf("Expected 1 choice, got %d", len(resp.Choices)) + } + + if resp.Choices[0].Message.Content != "Hello" { + t.Errorf("Expected content 'Hello', got '%s'", resp.Choices[0].Message.Content) + } + + if resp.Usage.TotalTokens != 30 { + t.Errorf("Expected total tokens 30, got %d", resp.Usage.TotalTokens) + } +} + +// TestStreamingResponseTypes tests streaming response structures +func TestStreamingResponseTypes(t *testing.T) { + chunk := ChatCompletionStreamResponse{ + ID: "stream-id", + Created: 1234567890, + Choices: []StreamChoice{ + { + Index: 0, + Delta: MessageDelta{ + Content: "Hello", + }, + FinishReason: "", + }, + }, + } + + if chunk.ID != "stream-id" { + t.Errorf("Expected ID 'stream-id', got '%s'", chunk.ID) + } + + if len(chunk.Choices) == 0 { + t.Error("Expected at least one choice") + } + + if chunk.Choices[0].Delta.Content != "Hello" { + t.Errorf("Expected delta content 'Hello', got '%s'", chunk.Choices[0].Delta.Content) + } +} + +// TestToolCallStructure tests Tool and ToolCall structures +func TestToolCallStructure(t *testing.T) { + tool := Tool{ + Type: "function", + Function: Function{ + Name: "get_weather", + Description: "Get the weather", + Parameters: map[string]interface{}{ + "location": "string", + }, + }, + } + + if tool.Type != "function" { + t.Errorf("Expected tool type 'function', got '%s'", tool.Type) + } + + if tool.Function.Name != "get_weather" { + t.Errorf("Expected function name 'get_weather', got '%s'", tool.Function.Name) + } + + toolCall := ToolCall{ + ID: "call-123", + Type: "function", + Function: FunctionCall{ + Name: "get_weather", + Arguments: `{"location": "San Francisco"}`, + }, + } + + if toolCall.ID != "call-123" { + t.Errorf("Expected tool call ID 'call-123', got '%s'", toolCall.ID) + } +} + +// TestConcurrentClientOperations tests thread safety +// This is a basic test that just verifies concurrent calls don't panic +func TestConcurrentClientOperations(t *testing.T) { + config := ClientConfig{ + Endpoint: "grpc://localhost:20000", + TokenizerPath: "/path/to/tokenizer", + } + + client, err := NewClient(config) + if err != nil { + t.Skip("Skipping concurrent operations test: server not available") + } + defer client.Close() + + // Try concurrent Close calls (should not panic or race) + done := make(chan bool, 2) + + go func() { + client.Close() + done <- true + }() + + go func() { + client.Close() + done <- true + }() + + <-done + <-done +} + +// BenchmarkChatCompletionRequest benchmarks request creation +func BenchmarkChatCompletionRequest(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = ChatCompletionRequest{ + Model: "default", + Messages: []ChatMessage{ + {Role: "user", Content: "test message"}, + }, + Stream: false, + Temperature: floatPtr(0.7), + MaxCompletionTokens: intPtr(100), + } + } +} + +// Helper functions for benchmarks +func floatPtr(f float32) *float32 { + return &f +} + +func intPtr(i int) *int { + return &i +} + +// TestContextCancellation tests that cancelled context is handled gracefully. +// +// NOTE: Currently, the FFI layer is blocking and doesn't actively monitor context cancellation. +// This test verifies that the client at least returns an error rather than panicking or +// hanging indefinitely when a pre-cancelled context is passed. +// +// Future: When FFI supports context cancellation (via signals or async operations), +// this test should be updated to assert that the error is context.Canceled or wrapped +// context cancellation error. +func TestContextCancellation(t *testing.T) { + config := ClientConfig{ + Endpoint: "grpc://localhost:20000", + TokenizerPath: "/path/to/tokenizer", + } + + client, err := NewClient(config) + if err != nil { + t.Skip("Skipping context cancellation test: server not available") + } + defer client.Close() + + // Create a pre-cancelled context + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + req := ChatCompletionRequest{ + Model: "default", + Messages: []ChatMessage{ + {Role: "user", Content: "test"}, + }, + } + + // Attempt request with cancelled context + // Since FFI is blocking, we expect either: + // 1. An error from the server/network + // 2. The call to complete normally (FFI doesn't check context) + // What we DON'T expect is a panic or indefinite hang + _, err = client.CreateChatCompletion(ctx, req) + if err != nil { + t.Logf("Request with cancelled context returned error: %v", err) + } else { + t.Logf("Request with cancelled context completed (FFI may not support context cancellation)") + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/examples/simple/main.go b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/examples/simple/main.go new file mode 100644 index 00000000..9983362c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/examples/simple/main.go @@ -0,0 +1,85 @@ +// Simple example demonstrating basic usage of SGLang Go SDK +package main + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/sglang/sglang-go-grpc-sdk" +) + +func main() { + // Get configuration from environment or command line + endpoint := os.Getenv("SGL_GRPC_ENDPOINT") + if endpoint == "" { + endpoint = "grpc://localhost:20000" + } + + tokenizerPath := os.Getenv("SGL_TOKENIZER_PATH") + if tokenizerPath == "" { + tokenizerPath = "./examples/tokenizer" + } + + // Create client + client, err := sglang.NewClient(sglang.ClientConfig{ + Endpoint: endpoint, + TokenizerPath: tokenizerPath, + }) + if err != nil { + log.Fatalf("Failed to create client: %v", err) + } + defer client.Close() + + // Create chat completion request + req := sglang.ChatCompletionRequest{ + Model: "default", + Messages: []sglang.ChatMessage{ + { + Role: "system", + Content: "You are a helpful assistant.", + }, + { + Role: "user", + Content: "写一首歌关于夏天", + }, + }, + Stream: false, + Temperature: float32Ptr(0.7), + MaxCompletionTokens: intPtr(200), + SkipSpecialTokens: true, + Tools: nil, // Use nil instead of empty slice to avoid template errors + } + + // Create completion + ctx := context.Background() + resp, err := client.CreateChatCompletion(ctx, req) + if err != nil { + log.Fatalf("Failed to create completion: %v", err) + } + + // Print response + fmt.Println("=== Response ===") + fmt.Printf("ID: %s\n", resp.ID) + fmt.Printf("Model: %s\n", resp.Model) + fmt.Printf("Created: %d\n", resp.Created) + fmt.Println("\nContent:") + for _, choice := range resp.Choices { + fmt.Println(choice.Message.Content) + } + fmt.Printf("\nFinish Reason: %s\n", resp.Choices[0].FinishReason) + fmt.Printf("\nUsage: Prompt=%d, Completion=%d, Total=%d\n", + resp.Usage.PromptTokens, + resp.Usage.CompletionTokens, + resp.Usage.TotalTokens, + ) +} + +func float32Ptr(f float32) *float32 { + return &f +} + +func intPtr(i int) *int { + return &i +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/examples/simple/run.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/examples/simple/run.sh new file mode 100755 index 00000000..9153f2e2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/examples/simple/run.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Simple example runner +# Usage: ./run.sh [tokenizer_path] [endpoint] + +# Set library path for Rust FFI library +# The library should be in ./lib directory (created by 'make lib') +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LIB_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)/lib" + +# Check if lib directory exists +if [ ! -d "$LIB_DIR" ]; then + echo "Error: Library directory not found at $LIB_DIR" + echo "Please run 'make lib' first to build and export the library" + exit 1 +fi + +# Get Python LDFLAGS (needed for Rust FFI that depends on Python) +PYTHON_LDFLAGS=$(python3-config --ldflags --embed 2>/dev/null || python3-config --ldflags 2>/dev/null || echo "") + +# Set CGO_LDFLAGS to link with the Rust library +export CGO_LDFLAGS="-L${LIB_DIR} -lsglang_router_rs ${PYTHON_LDFLAGS} -ldl" + +# macOS uses DYLD_LIBRARY_PATH, Linux uses LD_LIBRARY_PATH +if [[ "$OSTYPE" == "darwin"* ]]; then + export DYLD_LIBRARY_PATH="${LIB_DIR}:${DYLD_LIBRARY_PATH}" +else + export LD_LIBRARY_PATH="${LIB_DIR}:${LD_LIBRARY_PATH}" +fi + +# Default configuration (can be overridden by environment variables or command line arguments) +# Tokenizer path: ../tokenizer (relative to this script) +DEFAULT_TOKENIZER_PATH="${SGL_TOKENIZER_PATH:-../tokenizer}" +DEFAULT_ENDPOINT="${SGL_GRPC_ENDPOINT:-grpc://localhost:20000}" + +TOKENIZER_PATH="${1:-${DEFAULT_TOKENIZER_PATH}}" +ENDPOINT="${2:-${DEFAULT_ENDPOINT}}" + +echo "Running simple example..." +echo "Library path: ${LIB_DIR}" +echo "Tokenizer: $TOKENIZER_PATH" +echo "Endpoint: $ENDPOINT" +echo "" + +cd "$(dirname "${BASH_SOURCE[0]}")" +SGL_TOKENIZER_PATH="$TOKENIZER_PATH" SGL_GRPC_ENDPOINT="$ENDPOINT" go run main.go diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/examples/streaming/main.go b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/examples/streaming/main.go new file mode 100644 index 00000000..fa27909c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/examples/streaming/main.go @@ -0,0 +1,125 @@ +// Streaming example demonstrating real-time streaming with SGLang Go SDK +package main + +import ( + "context" + "fmt" + "io" + "log" + "os" + "strings" + "time" + + "github.com/sglang/sglang-go-grpc-sdk" +) + +func main() { + // Get configuration from environment or command line + endpoint := os.Getenv("SGL_GRPC_ENDPOINT") + if endpoint == "" { + endpoint = "grpc://localhost:20000" + } + + tokenizerPath := os.Getenv("SGL_TOKENIZER_PATH") + if tokenizerPath == "" { + tokenizerPath = "./examples/tokenizer" + } + + // Create client + client, err := sglang.NewClient(sglang.ClientConfig{ + Endpoint: endpoint, + TokenizerPath: tokenizerPath, + }) + if err != nil { + log.Fatalf("Failed to create client: %v", err) + } + defer client.Close() + + // Create streaming chat completion request + req := sglang.ChatCompletionRequest{ + Model: "default", + Messages: []sglang.ChatMessage{ + { + Role: "system", + Content: "You are a helpful assistant.", + }, + { + Role: "user", + Content: "写一首春天的诗歌", + }, + }, + Stream: true, + Temperature: float32Ptr(0.7), + MaxCompletionTokens: intPtr(500), + SkipSpecialTokens: true, + Tools: nil, // Use nil instead of empty slice to avoid template errors + } + + // Create streaming completion + ctx := context.Background() + stream, err := client.CreateChatCompletionStream(ctx, req) + if err != nil { + log.Fatalf("Failed to create stream: %v", err) + } + defer stream.Close() + + fmt.Println("=== Streaming Response ===") + fmt.Println() + + var fullContent strings.Builder + chunkCount := 0 + startTime := time.Now() + var firstTokenTime time.Time + firstTokenReceived := false + + for { + chunk, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + log.Fatalf("Stream error: %v", err) + } + + chunkCount++ + + // Extract content from delta + for _, choice := range chunk.Choices { + if choice.Delta.Content != "" { + fmt.Print(choice.Delta.Content) + fullContent.WriteString(choice.Delta.Content) + + // Track first token time (TTFT) + if !firstTokenReceived { + firstTokenTime = time.Now() + firstTokenReceived = true + ttft := firstTokenTime.Sub(startTime) + fmt.Printf("\n[TTFT: %v]\n", ttft) + } + } + + if choice.FinishReason != "" { + fmt.Printf("\n\n[Finished: %s]\n", choice.FinishReason) + } + } + } + + // Calculate metrics + if firstTokenReceived { + elapsed := time.Since(startTime) + tokensPerSecond := float64(fullContent.Len()) / elapsed.Seconds() + fmt.Printf("\n=== Metrics ===\n") + fmt.Printf("Total chunks: %d\n", chunkCount) + fmt.Printf("Total content length: %d characters\n", fullContent.Len()) + fmt.Printf("Time elapsed: %v\n", elapsed) + fmt.Printf("Tokens per second: %.2f\n", tokensPerSecond) + } +} + +func float32Ptr(f float32) *float32 { + return &f +} + +func intPtr(i int) *int { + return &i +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/examples/streaming/run.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/examples/streaming/run.sh new file mode 100755 index 00000000..49911cc0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/examples/streaming/run.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Streaming example runner +# Usage: ./run.sh [tokenizer_path] [endpoint] + +# Set library path for Rust FFI library +# The library should be in ./lib directory (created by 'make lib') +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LIB_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)/lib" + +# Check if lib directory exists +if [ ! -d "$LIB_DIR" ]; then + echo "Error: Library directory not found at $LIB_DIR" + echo "Please run 'make lib' first to build and export the library" + exit 1 +fi + +# Get Python LDFLAGS (needed for Rust FFI that depends on Python) +PYTHON_LDFLAGS=$(python3-config --ldflags --embed 2>/dev/null || python3-config --ldflags 2>/dev/null || echo "") + +# Set CGO_LDFLAGS to link with the Rust library +export CGO_LDFLAGS="-L${LIB_DIR} -lsglang_router_rs ${PYTHON_LDFLAGS} -ldl" + +# macOS uses DYLD_LIBRARY_PATH, Linux uses LD_LIBRARY_PATH +if [[ "$OSTYPE" == "darwin"* ]]; then + export DYLD_LIBRARY_PATH="${LIB_DIR}:${DYLD_LIBRARY_PATH}" +else + export LD_LIBRARY_PATH="${LIB_DIR}:${LD_LIBRARY_PATH}" +fi + +# Default configuration (can be overridden by environment variables or command line arguments) +# Tokenizer path: ../tokenizer (relative to this script) +DEFAULT_TOKENIZER_PATH="${SGL_TOKENIZER_PATH:-../tokenizer}" +DEFAULT_ENDPOINT="${SGL_GRPC_ENDPOINT:-grpc://localhost:20000}" + +TOKENIZER_PATH="${1:-${DEFAULT_TOKENIZER_PATH}}" +ENDPOINT="${2:-${DEFAULT_ENDPOINT}}" + +echo "Running streaming example..." +echo "Library path: ${LIB_DIR}" +echo "Tokenizer: $TOKENIZER_PATH" +echo "Endpoint: $ENDPOINT" +echo "" + +cd "$(dirname "${BASH_SOURCE[0]}")" +SGL_TOKENIZER_PATH="$TOKENIZER_PATH" SGL_GRPC_ENDPOINT="$ENDPOINT" go run main.go diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/integration_test.go b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/integration_test.go new file mode 100644 index 00000000..5ed9713f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/integration_test.go @@ -0,0 +1,228 @@ +//go:build integration +// +build integration + +// integration_test.go contains integration tests that require a running SGLang server +// +// To run these tests: +// 1. Start an SGLang server: python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-hf +// 2. Run: go test -tags=integration -run TestIntegration + +package sglang + +import ( + "context" + "io" + "os" + "testing" + "time" +) + +// getTestConfig returns test configuration from environment or defaults +func getTestConfig(t *testing.T) ClientConfig { + endpoint := os.Getenv("SGL_GRPC_ENDPOINT") + if endpoint == "" { + endpoint = "grpc://localhost:20000" + } + + tokenizerPath := os.Getenv("SGL_TOKENIZER_PATH") + if tokenizerPath == "" { + t.Skip("SGL_TOKENIZER_PATH not set") + } + + return ClientConfig{ + Endpoint: endpoint, + TokenizerPath: tokenizerPath, + } +} + +// TestIntegrationNonStreamingCompletion tests non-streaming chat completion +func TestIntegrationNonStreamingCompletion(t *testing.T) { + config := getTestConfig(t) + + client, err := NewClient(config) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + defer client.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + req := ChatCompletionRequest{ + Model: "default", + Messages: []ChatMessage{ + {Role: "user", Content: "Say 'Hello, World!' only"}, + }, + Stream: false, + Temperature: float32Ptr(0.0), + MaxCompletionTokens: intPtr(50), + } + + resp, err := client.CreateChatCompletion(ctx, req) + if err != nil { + t.Fatalf("CreateChatCompletion failed: %v", err) + } + + if resp.ID == "" { + t.Error("Response ID is empty") + } + + if len(resp.Choices) == 0 { + t.Error("Response has no choices") + } + + if resp.Choices[0].Message.Content == "" { + t.Error("Response content is empty") + } + + if resp.Usage == nil || resp.Usage.TotalTokens == 0 { + t.Error("Usage information is missing or invalid") + } + + t.Logf("Response: %s", resp.Choices[0].Message.Content) + t.Logf("Usage: %+v", resp.Usage) +} + +// TestIntegrationStreamingCompletion tests streaming chat completion +func TestIntegrationStreamingCompletion(t *testing.T) { + config := getTestConfig(t) + + client, err := NewClient(config) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + defer client.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + req := ChatCompletionRequest{ + Model: "default", + Messages: []ChatMessage{ + {Role: "user", Content: "Count from 1 to 5"}, + }, + Stream: true, + Temperature: float32Ptr(0.0), + MaxCompletionTokens: intPtr(100), + } + + stream, err := client.CreateChatCompletionStream(ctx, req) + if err != nil { + t.Fatalf("CreateChatCompletionStream failed: %v", err) + } + defer stream.Close() + + chunkCount := 0 + totalContent := "" + + for { + chunk, err := stream.Recv() + if err == io.EOF { + // io.EOF is expected at end of stream + break + } + if err != nil { + t.Fatalf("Stream error: %v", err) + } + + chunkCount++ + + for _, choice := range chunk.Choices { + if choice.Delta.Content != "" { + totalContent += choice.Delta.Content + } + } + } + + if chunkCount == 0 { + t.Error("Received no chunks from stream") + } + + if totalContent == "" { + t.Error("Received no content from stream") + } + + t.Logf("Received %d chunks with content: %s", chunkCount, totalContent) +} + +// TestIntegrationConcurrentRequests tests multiple concurrent requests +func TestIntegrationConcurrentRequests(t *testing.T) { + config := getTestConfig(t) + + client, err := NewClient(config) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + defer client.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + numRequests := 3 + done := make(chan error, numRequests) + + for i := 0; i < numRequests; i++ { + go func(idx int) { + req := ChatCompletionRequest{ + Model: "default", + Messages: []ChatMessage{ + {Role: "user", Content: "Say 'test'"}, + }, + Stream: false, + MaxCompletionTokens: intPtr(50), + } + + _, err := client.CreateChatCompletion(ctx, req) + done <- err + }(i) + } + + // Collect results + for i := 0; i < numRequests; i++ { + if err := <-done; err != nil { + t.Errorf("Request %d failed: %v", i, err) + } + } + + t.Logf("All %d concurrent requests completed successfully", numRequests) +} + +// TestIntegrationContextCancellation tests that context cancellation is handled +func TestIntegrationContextCancellation(t *testing.T) { + config := getTestConfig(t) + + client, err := NewClient(config) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + defer client.Close() + + // Create a context that cancels immediately + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + req := ChatCompletionRequest{ + Model: "default", + Messages: []ChatMessage{ + {Role: "user", Content: "test"}, + }, + Stream: false, + } + + // Should handle cancelled context gracefully + _, err = client.CreateChatCompletion(ctx, req) + if err == nil { + t.Error("Expected error from cancelled context") + } + + t.Logf("Cancelled context handled: %v", err) +} + +// Helper functions +func float32Ptr(f float32) *float32 { + return &f +} + +func intPtr(i int) *int { + return &i +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/internal/ffi/client.go b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/internal/ffi/client.go new file mode 100644 index 00000000..13439e3c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/internal/ffi/client.go @@ -0,0 +1,228 @@ +// Package ffi provides Go bindings for SGLang's Rust FFI (Foreign Function Interface). +// +// This package wraps the Rust FFI layer of SGLang, providing low-level access to: +// - Client creation and connection management +// - Chat completion streaming +// - Stream reading and response conversion +// - Memory management for C strings +// +// Internal use only: This package is intended for internal use by the sglang package. +// End users should use the public sglang package instead. +package ffi + +/* +#cgo LDFLAGS: -lsglang_router_rs -ldl +#include +#include + +// Error codes +typedef enum { + SGL_ERROR_SUCCESS = 0, + SGL_ERROR_INVALID_ARGUMENT = 1, + SGL_ERROR_TOKENIZATION_ERROR = 2, + SGL_ERROR_PARSING_ERROR = 3, + SGL_ERROR_MEMORY_ERROR = 4, + SGL_ERROR_UNKNOWN = 99 +} SglErrorCode; + +// Opaque handles +typedef void* SglangClientHandle; +typedef void* SglangStreamHandle; + +// Client SDK functions +SglangClientHandle* sgl_client_create(const char* endpoint, const char* tokenizer_path, char** error_out); +void sgl_client_free(SglangClientHandle* handle); +SglErrorCode sgl_client_chat_completion_stream(SglangClientHandle* client_handle, const char* request_json, SglangStreamHandle** stream_handle_out, char** error_out); +SglErrorCode sgl_stream_read_next(SglangStreamHandle* stream_handle, char** response_json_out, int* is_done_out, char** error_out); +void sgl_stream_free(SglangStreamHandle* handle); +void sgl_free_string(char* s); +*/ +import "C" + +import ( + "fmt" + "unsafe" +) + +// ErrorCode represents FFI error codes returned by Rust functions. +// +// These codes indicate the result of FFI operations. Use Error() to get a human-readable +// error message. +type ErrorCode int + +const ( + // ErrorSuccess indicates the operation completed successfully + ErrorSuccess ErrorCode = 0 + // ErrorInvalidArgument indicates invalid arguments were passed to the FFI function + ErrorInvalidArgument ErrorCode = 1 + // ErrorTokenizationError indicates an error during tokenization + ErrorTokenizationError ErrorCode = 2 + // ErrorParsingError indicates an error parsing the response or request + ErrorParsingError ErrorCode = 3 + // ErrorMemoryError indicates a memory allocation error + ErrorMemoryError ErrorCode = 4 + // ErrorUnknown indicates an unclassified error + ErrorUnknown ErrorCode = 99 +) + +// Error implements the error interface for ErrorCode. +func (e ErrorCode) Error() string { + switch e { + case ErrorSuccess: + return "success" + case ErrorInvalidArgument: + return "invalid argument" + case ErrorTokenizationError: + return "tokenization error" + case ErrorParsingError: + return "parsing error" + case ErrorMemoryError: + return "memory error" + case ErrorUnknown: + return "unknown error" + default: + return fmt.Sprintf("unknown error code: %d", e) + } +} + +// SglangClientHandle wraps the Rust client SDK FFI handle. +// +// This struct maintains a connection to the SGLang gRPC server and is used +// to create streams and manage the underlying Rust client resources. +type SglangClientHandle struct { + handle *C.SglangClientHandle +} + +// NewClient creates a new SGLang client handle via FFI. +// +// This function initializes the Rust client with the given endpoint and tokenizer path. +// +// Parameters: +// - endpoint: gRPC endpoint URL (e.g., "grpc://localhost:20000") +// - tokenizerPath: Path to tokenizer directory +// +// Returns: +// - *SglangClientHandle: A new client handle +// - error: An error if client creation failed +func NewClient(endpoint, tokenizerPath string) (*SglangClientHandle, error) { + cEndpoint := C.CString(endpoint) + defer C.free(unsafe.Pointer(cEndpoint)) + + cTokenizerPath := C.CString(tokenizerPath) + defer C.free(unsafe.Pointer(cTokenizerPath)) + + var errorPtr *C.char + handle := C.sgl_client_create(cEndpoint, cTokenizerPath, &errorPtr) + + if handle == nil { + errorMsg := "" + if errorPtr != nil { + errorMsg = C.GoString(errorPtr) + C.sgl_free_string(errorPtr) + } + if errorMsg == "" { + errorMsg = "failed to create client" + } + return nil, fmt.Errorf("%s", errorMsg) + } + + return &SglangClientHandle{handle: handle}, nil +} + +// Free releases the client handle +func (h *SglangClientHandle) Free() { + if h.handle != nil { + C.sgl_client_free(h.handle) + h.handle = nil + } +} + +// ChatCompletionStream creates a streaming chat completion request +func (h *SglangClientHandle) ChatCompletionStream(requestJSON string) (*SglangStreamHandle, error) { + if h.handle == nil { + return nil, fmt.Errorf("client handle is nil") + } + + cRequestJSON := C.CString(requestJSON) + defer C.free(unsafe.Pointer(cRequestJSON)) + + var streamHandle *C.SglangStreamHandle + var errorPtr *C.char + + result := C.sgl_client_chat_completion_stream( + h.handle, + cRequestJSON, + &streamHandle, + &errorPtr, + ) + + if ErrorCode(result) != ErrorSuccess { + errorMsg := "" + if errorPtr != nil { + errorMsg = C.GoString(errorPtr) + C.sgl_free_string(errorPtr) + } + if errorMsg == "" { + errorMsg = fmt.Sprintf("error code %d", result) + } + return nil, fmt.Errorf("%s", errorMsg) + } + + if streamHandle == nil { + return nil, fmt.Errorf("stream handle is nil") + } + + return &SglangStreamHandle{handle: streamHandle}, nil +} + +// SglangStreamHandle wraps the Rust stream FFI handle +type SglangStreamHandle struct { + handle *C.SglangStreamHandle +} + +// ReadNext reads the next chunk from the stream +// Returns: (responseJSON, isDone, error) +func (h *SglangStreamHandle) ReadNext() (string, bool, error) { + if h.handle == nil { + return "", true, fmt.Errorf("stream handle is nil") + } + + var responseJSON *C.char + var isDone C.int + var errorPtr *C.char + + result := C.sgl_stream_read_next( + h.handle, + &responseJSON, + &isDone, + &errorPtr, + ) + + if ErrorCode(result) != ErrorSuccess { + errorMsg := "" + if errorPtr != nil { + errorMsg = C.GoString(errorPtr) + C.sgl_free_string(errorPtr) + } + if errorMsg == "" { + errorMsg = fmt.Sprintf("error code %d", result) + } + return "", isDone == 1, fmt.Errorf("%s", errorMsg) + } + + responseStr := "" + if responseJSON != nil { + responseStr = C.GoString(responseJSON) + C.sgl_free_string(responseJSON) + } + + return responseStr, isDone == 1, nil +} + +// Free releases the stream handle +func (h *SglangStreamHandle) Free() { + if h.handle != nil { + C.sgl_stream_free(h.handle) + h.handle = nil + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/client.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/client.rs new file mode 100644 index 00000000..b4359567 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/client.rs @@ -0,0 +1,279 @@ +//! Client SDK FFI functions + +use std::ffi::{CStr, CString}; +use std::os::raw::{c_char}; +use std::ptr; +use std::sync::Arc; +use tokio::runtime::Runtime; +use once_cell::sync::Lazy; +use uuid::Uuid; + +use sgl_model_gateway::tokenizer::create_tokenizer_from_file; +use sgl_model_gateway::tokenizer::traits::Tokenizer; +use sgl_model_gateway::grpc_client::sglang_scheduler::SglangSchedulerClient; +use sgl_model_gateway::protocols::chat::ChatCompletionRequest; +use sgl_model_gateway::routers::grpc::utils::{process_chat_messages, generate_tool_constraints}; + +use super::error::{SglErrorCode, set_error_message}; +use super::grpc_converter::sgl_grpc_response_converter_create; +use super::tokenizer::TokenizerHandle; +use super::stream::SglangStreamHandle; + +/// Global tokio runtime for async operations +static RUNTIME: Lazy = Lazy::new(|| { + Runtime::new().expect("Failed to create tokio runtime for client FFI") +}); + +/// Handle for complete client SDK (gRPC client + tokenizer) +/// This handle manages the connection to sglang and provides a complete SDK interface +pub struct SglangClientHandle { + pub(crate) client: Arc, + pub(crate) tokenizer: Arc, +} + +/// Handle for streaming request (includes prompt token count) +#[allow(dead_code)] +pub struct StreamRequestState { + pub(crate) prompt_tokens: i32, // Number of prompt tokens for this request +} + +/// Create a new SGLang client handle +/// +/// # Arguments +/// * `endpoint` - gRPC endpoint (e.g., "grpc://localhost:20000") +/// * `tokenizer_path` - Path to tokenizer directory +/// * `error_out` - Optional pointer to receive error message +/// +/// # Returns +/// * Pointer to SglangClientHandle on success, null on failure +#[no_mangle] +pub unsafe extern "C" fn sgl_client_create( + endpoint: *const c_char, + tokenizer_path: *const c_char, + error_out: *mut *mut c_char, +) -> *mut SglangClientHandle { + if endpoint.is_null() || tokenizer_path.is_null() { + set_error_message(error_out, "Invalid arguments: null pointer"); + return ptr::null_mut(); + } + + let endpoint_str = match CStr::from_ptr(endpoint).to_str() { + Ok(s) => s, + Err(_) => { + set_error_message(error_out, "Invalid UTF-8 in endpoint"); + return ptr::null_mut(); + } + }; + + let tokenizer_path_str = match CStr::from_ptr(tokenizer_path).to_str() { + Ok(s) => s, + Err(_) => { + set_error_message(error_out, "Invalid UTF-8 in tokenizer_path"); + return ptr::null_mut(); + } + }; + + // Create tokenizer + let tokenizer = match create_tokenizer_from_file(tokenizer_path_str) { + Ok(t) => t, + Err(e) => { + set_error_message(error_out, &format!("Failed to create tokenizer: {}", e)); + return ptr::null_mut(); + } + }; + + // Create gRPC client + let client = match RUNTIME.block_on(async { + SglangSchedulerClient::connect(endpoint_str).await + }) { + Ok(c) => Arc::new(c), + Err(e) => { + set_error_message(error_out, &format!("Failed to connect to endpoint: {}", e)); + return ptr::null_mut(); + } + }; + + Box::into_raw(Box::new(SglangClientHandle { + client, + tokenizer, + })) +} + +/// Free a client handle +#[no_mangle] +pub unsafe extern "C" fn sgl_client_free(handle: *mut SglangClientHandle) { + if !handle.is_null() { + let _ = Box::from_raw(handle); + } +} + +/// Send a chat completion request and start streaming +/// +/// # Arguments +/// * `client_handle` - Client handle +/// * `request_json` - OpenAI ChatCompletionRequest as JSON string +/// * `stream_handle_out` - Pointer to receive stream handle +/// * `error_out` - Optional pointer to receive error message +/// +/// # Returns +/// * SglErrorCode::Success on success, error code on failure +#[no_mangle] +pub unsafe extern "C" fn sgl_client_chat_completion_stream( + client_handle: *mut SglangClientHandle, + request_json: *const c_char, + stream_handle_out: *mut *mut SglangStreamHandle, + error_out: *mut *mut c_char, +) -> SglErrorCode { + if client_handle.is_null() || request_json.is_null() || stream_handle_out.is_null() { + set_error_message(error_out, "Invalid arguments: null pointer"); + return SglErrorCode::InvalidArgument; + } + + let request_str = match CStr::from_ptr(request_json).to_str() { + Ok(s) => s, + Err(_) => { + set_error_message(error_out, "Invalid UTF-8 in request_json"); + return SglErrorCode::InvalidArgument; + } + }; + + let client_ref = &*client_handle; + let client = Arc::clone(&client_ref.client); + let tokenizer = Arc::clone(&client_ref.tokenizer); + + // Parse OpenAI ChatCompletionRequest + let chat_request: ChatCompletionRequest = match serde_json::from_str(request_str) { + Ok(req) => req, + Err(e) => { + set_error_message(error_out, &format!("Failed to parse request JSON: {}", e)); + return SglErrorCode::ParsingError; + } + }; + + // Process messages and apply chat template + let processed_messages = match process_chat_messages(&chat_request, tokenizer.as_ref()) { + Ok(msgs) => msgs, + Err(e) => { + set_error_message(error_out, &format!("Failed to process messages: {}", e)); + return SglErrorCode::TokenizationError; + } + }; + + // Tokenize + let token_ids = match tokenizer.encode(&processed_messages.text) { + Ok(encoding) => encoding.token_ids().to_vec(), + Err(e) => { + set_error_message(error_out, &format!("Failed to tokenize: {}", e)); + return SglErrorCode::TokenizationError; + } + }; + let prompt_tokens = token_ids.len() as i32; // Save prompt token count + + // Generate tool constraints if needed + let tool_constraint = if let Some(tools) = chat_request.tools.as_ref() { + match generate_tool_constraints(tools, &chat_request.tool_choice, &chat_request.model) { + Ok(Some((constraint_type, constraint_value))) => Some((constraint_type, constraint_value)), + Ok(None) => None, + Err(e) => { + set_error_message(error_out, &format!("Failed to generate tool constraints: {}", e)); + return SglErrorCode::ParsingError; + } + } + } else { + None + }; + + // Build GenerateRequest + let request_id = format!("chatcmpl-{}", Uuid::new_v4()); + let proto_request = match client.build_generate_request_from_chat( + request_id.clone(), + &chat_request, + processed_messages.text, + token_ids, + processed_messages.multimodal_inputs, + tool_constraint, + ) { + Ok(req) => req, + Err(e) => { + set_error_message(error_out, &format!("Failed to build generate request: {}", e)); + return SglErrorCode::ParsingError; + } + }; + + // Send request and get stream + let stream = match RUNTIME.block_on(async { + client.generate(proto_request).await + }) { + Ok(s) => s, + Err(e) => { + set_error_message(error_out, &format!("Failed to send request: {}", e)); + return SglErrorCode::UnknownError; + } + }; + + // Create response converter + let tools_json = chat_request.tools.as_ref() + .and_then(|t| serde_json::to_string(t).ok()) + .map(|s| CString::new(s).unwrap().into_raw()); + let tool_choice_json = chat_request.tool_choice.as_ref() + .and_then(|tc| serde_json::to_string(tc).ok()) + .map(|s| CString::new(s).unwrap().into_raw()); + let stop_json = chat_request.stop.as_ref() + .and_then(|s| serde_json::to_string(s).ok()) + .map(|s| CString::new(s).unwrap().into_raw()); + let stop_token_ids_json = chat_request.stop_token_ids.as_ref() + .and_then(|ids| serde_json::to_string(ids).ok()) + .map(|s| CString::new(s).unwrap().into_raw()); + + // Create tokenizer handle for converter (we'll create a temporary one) + let tokenizer_handle = Box::into_raw(Box::new(TokenizerHandle { + tokenizer: Arc::clone(&tokenizer), + })); + + let converter = sgl_grpc_response_converter_create( + tokenizer_handle, + CString::new(chat_request.model.clone()).unwrap().as_ptr(), + CString::new(request_id.clone()).unwrap().as_ptr(), + tools_json.unwrap_or(ptr::null_mut()), + tool_choice_json.unwrap_or(ptr::null_mut()), + stop_json.unwrap_or(ptr::null_mut()), + stop_token_ids_json.unwrap_or(ptr::null_mut()), + if chat_request.skip_special_tokens { 1 } else { 0 }, + error_out, + ); + + // Free temporary tokenizer handle (converter now owns the tokenizer) + let _ = Box::from_raw(tokenizer_handle); + + if converter.is_null() { + return SglErrorCode::MemoryError; + } + + // Clean up temporary CStrings + if let Some(ptr) = tools_json { + let _ = CString::from_raw(ptr); + } + if let Some(ptr) = tool_choice_json { + let _ = CString::from_raw(ptr); + } + if let Some(ptr) = stop_json { + let _ = CString::from_raw(ptr); + } + if let Some(ptr) = stop_token_ids_json { + let _ = CString::from_raw(ptr); + } + + // Create converter handle and set initial_prompt_tokens immediately + let mut converter_handle = *Box::from_raw(converter); + converter_handle.initial_prompt_tokens = Some(prompt_tokens); + + // Create stream handle with prompt_tokens + *stream_handle_out = Box::into_raw(Box::new(SglangStreamHandle { + stream: Arc::new(tokio::sync::Mutex::new(stream)), + converter: Arc::new(tokio::sync::Mutex::new(converter_handle)), + client: Arc::clone(&client), + prompt_tokens, + })); + + SglErrorCode::Success +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/error.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/error.rs new file mode 100644 index 00000000..3e43945e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/error.rs @@ -0,0 +1,50 @@ +//! Error handling for FFI functions + +use std::ffi::CString; +use std::os::raw::c_char; +use std::ptr; + +/// Error codes returned by FFI functions +#[repr(C)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SglErrorCode { + Success = 0, + InvalidArgument = 1, + TokenizationError = 2, + ParsingError = 3, + MemoryError = 4, + UnknownError = 99, +} + +/// Helper to set error message in FFI output parameter +pub fn set_error_message(error_out: *mut *mut c_char, message: &str) { + unsafe { + if !error_out.is_null() { + if let Ok(cstr) = CString::new(message) { + *error_out = cstr.into_raw(); + } else { + *error_out = ptr::null_mut(); + } + } + } +} + +/// Helper to set error message from format string +pub fn set_error_message_fmt(error_out: *mut *mut c_char, fmt: std::fmt::Arguments) { + if !error_out.is_null() { + let msg = format!("{}", fmt); + set_error_message(error_out, &msg); + } +} + +/// Helper to clear error message +pub fn clear_error_message(error_out: *mut *mut c_char) { + unsafe { + if !error_out.is_null() { + *error_out = ptr::null_mut(); + } + } +} + +// Helper functions for error handling +// Note: Some helper functions are kept for potential future use diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/grpc_converter.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/grpc_converter.rs new file mode 100644 index 00000000..e808884b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/grpc_converter.rs @@ -0,0 +1,758 @@ +//! gRPC response converter FFI functions + +use std::ffi::{CStr, CString}; +use std::os::raw::{c_char, c_int}; +use std::ptr; +use std::sync::Arc; +use std::collections::HashMap; +use serde_json::Value; +use tokio::runtime::Runtime; +use once_cell::sync::Lazy; + +use sgl_model_gateway::tokenizer::traits::Tokenizer; +use sgl_model_gateway::tokenizer::stream::DecodeStream; +use sgl_model_gateway::tool_parser::ToolParser; +use sgl_model_gateway::protocols::common::{Tool, ToolChoice, ToolChoiceValue, ToolCallDelta, FunctionCallDelta, Usage, StringOrArray}; +use sgl_model_gateway::tokenizer::stop::StopSequenceDecoder; +use sgl_model_gateway::grpc_client::sglang_proto as proto; + +use super::error::{SglErrorCode, set_error_message, clear_error_message}; +use super::tokenizer::TokenizerHandle; +use super::utils::generate_tool_call_id; + +/// Global parser factory (initialized once) +// Use the re-exported ParserFactory from tool_parser module +static PARSER_FACTORY: Lazy = Lazy::new(|| { + // ParserFactory is re-exported from tool_parser::factory, so we can use it directly + sgl_model_gateway::tool_parser::ParserFactory::default() +}); + +/// Global tokio runtime for async operations +static RUNTIME: Lazy = Lazy::new(|| { + Runtime::new().expect("Failed to create tokio runtime for gRPC converter FFI") +}); + +/// Handle for gRPC response converter (maintains state for streaming) +#[repr(C)] +pub struct GrpcResponseConverterHandle { + pub(crate) tokenizer: Arc, + pub(crate) tool_parser: Option>>>, + pub(crate) stop_decoder: Option>>, + pub(crate) model: String, + pub(crate) request_id: String, + pub(crate) created: u64, + pub(crate) system_fingerprint: Option, + pub(crate) tools: Option>, + pub(crate) tool_choice: Option, + pub(crate) history_tool_calls_count: usize, + pub(crate) stream_buffers: HashMap, // Per-index text buffers + pub(crate) decode_streams: HashMap, // Per-index incremental decoders + pub(crate) has_tool_calls: HashMap, // Track if tool calls were emitted + pub(crate) is_first_chunk: HashMap, // Track first chunk per index + pub(crate) prompt_tokens: HashMap, // Track prompt tokens per index (from chunks) + pub(crate) completion_tokens: HashMap, // Track completion tokens per index (cumulative) + pub(crate) initial_prompt_tokens: Option, // Initial prompt tokens from request (if available) + pub(crate) skip_special_tokens: bool, // Whether to skip special tokens when decoding +} + +/// Create a gRPC response converter handle +/// +/// # Arguments +/// * `tokenizer_handle` - Tokenizer handle (must be valid) +/// * `model` - Model name +/// * `request_id` - Request ID +/// * `tools_json` - Optional JSON array of tools +/// * `tool_choice_json` - Optional JSON object for tool_choice +/// * `stop` - Optional stop sequences (JSON array) +/// * `stop_token_ids` - Optional stop token IDs (JSON array) +/// * `skip_special_tokens` - Whether to skip special tokens +/// * `error_out` - Optional pointer to receive error message +/// +/// # Returns +/// * Pointer to GrpcResponseConverterHandle on success, null on failure +#[no_mangle] +pub unsafe extern "C" fn sgl_grpc_response_converter_create( + tokenizer_handle: *mut TokenizerHandle, + model: *const c_char, + request_id: *const c_char, + tools_json: *const c_char, + tool_choice_json: *const c_char, + stop: *const c_char, + stop_token_ids: *const c_char, + skip_special_tokens: c_int, + error_out: *mut *mut c_char, +) -> *mut GrpcResponseConverterHandle { + if tokenizer_handle.is_null() || model.is_null() || request_id.is_null() { + set_error_message(error_out, "Invalid arguments: null pointer"); + return ptr::null_mut(); + } + + let model_str = match CStr::from_ptr(model).to_str() { + Ok(s) => s, + Err(_) => { + set_error_message(error_out, "Invalid UTF-8 in model"); + return ptr::null_mut(); + } + }; + + let request_id_str = match CStr::from_ptr(request_id).to_str() { + Ok(s) => s, + Err(_) => { + set_error_message(error_out, "Invalid UTF-8 in request_id"); + return ptr::null_mut(); + } + }; + + let handle_ref = &*tokenizer_handle; + let tokenizer = Arc::clone(&handle_ref.tokenizer); + + // Parse tools if provided + let tools: Option> = if !tools_json.is_null() { + match CStr::from_ptr(tools_json).to_str() { + Ok(s) => serde_json::from_str::>(s).ok(), + Err(_) => None, + } + } else { + None + }; + + // Parse tool_choice if provided + let tool_choice: Option = if !tool_choice_json.is_null() { + match CStr::from_ptr(tool_choice_json).to_str() { + Ok(s) => serde_json::from_str::(s).ok(), + Err(_) => None, + } + } else { + None + }; + + // Parse stop sequences + let stop: Option = if !stop.is_null() { + let stop_str = match CStr::from_ptr(stop).to_str() { + Ok(s) => s, + Err(_) => return ptr::null_mut(), + }; + serde_json::from_str::(stop_str).ok() + } else { + None + }; + + // Parse stop token IDs + let stop_token_ids: Option> = if !stop_token_ids.is_null() { + let ids_str = match CStr::from_ptr(stop_token_ids).to_str() { + Ok(s) => s, + Err(_) => return ptr::null_mut(), + }; + serde_json::from_str::>(ids_str).ok() + } else { + None + }; + + // Create stop decoder if needed + let stop_decoder = if stop.is_some() || stop_token_ids.is_some() { + Some(Arc::new(tokio::sync::Mutex::new( + sgl_model_gateway::routers::grpc::utils::create_stop_decoder( + &tokenizer, + stop.as_ref(), + stop_token_ids.as_ref(), + skip_special_tokens != 0, + false, // no_stop_trim + ), + ))) + } else { + None + }; + + // Create tool parser if tools are provided + let tool_parser = if tools.is_some() { + PARSER_FACTORY.registry().create_for_model(model_str) + .map(|p| Arc::new(tokio::sync::Mutex::new(p))) + } else { + None + }; + + // Get system fingerprint from model (simplified) + let system_fingerprint = Some("fp_placeholder".to_string()); // TODO: Get actual fingerprint + + Box::into_raw(Box::new(GrpcResponseConverterHandle { + tokenizer, + tool_parser, + stop_decoder, + model: model_str.to_string(), + request_id: request_id_str.to_string(), + created: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(), + system_fingerprint, + tools, + tool_choice, + history_tool_calls_count: 0, + stream_buffers: HashMap::new(), + decode_streams: HashMap::new(), + has_tool_calls: HashMap::new(), + is_first_chunk: HashMap::new(), + prompt_tokens: HashMap::new(), + completion_tokens: HashMap::new(), + initial_prompt_tokens: None, // Will be set from stream handle + skip_special_tokens: skip_special_tokens != 0, + })) +} + +/// Convert a gRPC GenerateResponse chunk to OpenAI format +/// +/// # Arguments +/// * `handle` - Converter handle +/// * `response_json` - JSON string of proto.GenerateResponse +/// * `result_json_out` - Pointer to receive OpenAI format JSON (must be freed with sgl_free_string) +/// * `error_out` - Optional pointer to receive error message +/// +/// # Returns +/// * SglErrorCode::Success on success, error code on failure +#[no_mangle] +pub unsafe extern "C" fn sgl_grpc_response_converter_convert_chunk( + handle: *mut GrpcResponseConverterHandle, + response_json: *const c_char, + result_json_out: *mut *mut c_char, + error_out: *mut *mut c_char, +) -> SglErrorCode { + if handle.is_null() || response_json.is_null() || result_json_out.is_null() { + set_error_message(error_out, "Invalid arguments: null pointer"); + return SglErrorCode::InvalidArgument; + } + + let response_str = match CStr::from_ptr(response_json).to_str() { + Ok(s) => s, + Err(_) => { + set_error_message(error_out, "Invalid UTF-8 in response_json"); + return SglErrorCode::InvalidArgument; + } + }; + + // Parse proto.GenerateResponse from JSON + let json_value: Value = match serde_json::from_str(response_str) { + Ok(v) => v, + Err(e) => { + set_error_message(error_out, &format!("Failed to parse response JSON: {}", e)); + return SglErrorCode::ParsingError; + } + }; + + // Build proto::GenerateResponse from JSON value + let mut proto_response = proto::GenerateResponse { + request_id: json_value.get("request_id") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(), + response: None, + }; + + // Parse the response oneof field + if let Some(chunk_json) = json_value.get("chunk") { + let chunk = proto::GenerateStreamChunk { + token_ids: chunk_json.get("token_ids") + .and_then(|v| v.as_array()) + .map(|arr| arr.iter().filter_map(|v| v.as_u64().map(|n| n as u32)).collect()) + .unwrap_or_default(), + prompt_tokens: chunk_json.get("prompt_tokens") + .and_then(|v| v.as_i64()) + .map(|n| n as i32) + .unwrap_or(0), + completion_tokens: chunk_json.get("completion_tokens") + .and_then(|v| v.as_i64()) + .map(|n| n as i32) + .unwrap_or(0), + cached_tokens: chunk_json.get("cached_tokens") + .and_then(|v| v.as_i64()) + .map(|n| n as i32) + .unwrap_or(0), + output_logprobs: None, + hidden_states: vec![], + input_logprobs: None, + index: 0, + }; + proto_response.response = Some(proto::generate_response::Response::Chunk(chunk)); + } else if let Some(complete_json) = json_value.get("complete") { + let complete = proto::GenerateComplete { + output_ids: complete_json.get("output_ids") + .and_then(|v| v.as_array()) + .map(|arr| arr.iter().filter_map(|v| v.as_u64().map(|n| n as u32)).collect()) + .unwrap_or_default(), + finish_reason: complete_json.get("finish_reason") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(), + prompt_tokens: complete_json.get("prompt_tokens") + .and_then(|v| v.as_i64()) + .map(|n| n as i32) + .unwrap_or(0), + completion_tokens: complete_json.get("completion_tokens") + .and_then(|v| v.as_i64()) + .map(|n| n as i32) + .unwrap_or(0), + cached_tokens: complete_json.get("cached_tokens") + .and_then(|v| v.as_i64()) + .map(|n| n as i32) + .unwrap_or(0), + output_logprobs: None, + all_hidden_states: vec![], + input_logprobs: None, + matched_stop: None, + index: 0, + }; + proto_response.response = Some(proto::generate_response::Response::Complete(complete)); + } else if let Some(error_json) = json_value.get("error") { + let error = proto::GenerateError { + message: error_json.get("message") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(), + http_status_code: error_json.get("http_status_code") + .and_then(|v| v.as_str()) + .unwrap_or("500") + .to_string(), + details: error_json.get("details") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(), + }; + proto_response.response = Some(proto::generate_response::Response::Error(error)); + } else { + set_error_message(error_out, "Response JSON must contain 'chunk', 'complete', or 'error' field"); + return SglErrorCode::ParsingError; + } + + let handle_ref = &mut *handle; + let tokenizer = Arc::clone(&handle_ref.tokenizer); + let model = handle_ref.model.clone(); + let request_id = handle_ref.request_id.clone(); + let created = handle_ref.created; + let system_fingerprint = handle_ref.system_fingerprint.clone(); + + // Use tokio runtime to run async code + let result = RUNTIME.block_on(async { + convert_proto_chunk_to_openai( + proto_response, + handle_ref, + &tokenizer, + &model, + &request_id, + created, + system_fingerprint.as_deref(), + ) + .await + }); + + match result { + Ok(Some(openai_response)) => { + // Serialize to JSON + let result_str = match serde_json::to_string(&openai_response) { + Ok(s) => s, + Err(e) => { + set_error_message(error_out, &format!("Failed to serialize response: {}", e)); + return SglErrorCode::ParsingError; + } + }; + + let result_cstr = match CString::new(result_str) { + Ok(s) => s, + Err(e) => { + set_error_message(error_out, &format!("Failed to create result string: {}", e)); + return SglErrorCode::MemoryError; + } + }; + + *result_json_out = result_cstr.into_raw(); + clear_error_message(error_out); + SglErrorCode::Success + } + Ok(None) => { + // No response to send (e.g., empty chunk) + let empty = CString::new("").unwrap(); + *result_json_out = empty.into_raw(); + clear_error_message(error_out); + SglErrorCode::Success + } + Err(e) => { + set_error_message(error_out, &format!("Conversion error: {}", e)); + SglErrorCode::ParsingError + } + } +} + +/// Helper function to convert proto chunk to OpenAI format +pub(crate) async fn convert_proto_chunk_to_openai( + proto_response: proto::GenerateResponse, + handle: &mut GrpcResponseConverterHandle, + tokenizer: &Arc, + model: &str, + request_id: &str, + created: u64, + system_fingerprint: Option<&str>, +) -> Result, String> { + use sgl_model_gateway::grpc_client::sglang_proto::generate_response::Response::*; + use sgl_model_gateway::protocols::chat::{ChatCompletionStreamResponse, ChatMessageDelta, ChatStreamChoice}; + + match proto_response.response { + Some(Chunk(chunk)) => { + let index = chunk.index; + + // Mark as not first chunk if we've seen this index before + let is_first = handle.is_first_chunk.entry(index).or_insert(true); + let first_chunk = *is_first; + *is_first = false; + + // Track token counts from chunks (cumulative values from proto) + // These are cumulative values, so we always use the latest value + // For prompt_tokens, if chunk value is 0, preserve existing value or use initial_prompt_tokens + // This prevents overwriting valid prompt_tokens with 0 + if chunk.prompt_tokens > 0 { + handle.prompt_tokens.insert(index, chunk.prompt_tokens); + } else { + // If chunk.prompt_tokens is 0, try to preserve existing value or use initial_prompt_tokens + if !handle.prompt_tokens.contains_key(&index) { + // No existing value, try to use initial_prompt_tokens + if let Some(initial_prompt) = handle.initial_prompt_tokens { + handle.prompt_tokens.insert(index, initial_prompt); + } + } + // If existing value exists, keep it (don't overwrite with 0) + } + // For completion_tokens, always update (even if 0) as it's cumulative + handle.completion_tokens.insert(index, chunk.completion_tokens); + + // Process tokens through stop decoder if available, otherwise use incremental decoder + let chunk_text = if let Some(ref stop_decoder) = handle.stop_decoder { + let mut decoder_guard = stop_decoder.lock().await; + let mut text = String::new(); + for &token_id in &chunk.token_ids { + match decoder_guard.process_token(token_id).unwrap_or_else(|_| { + sgl_model_gateway::tokenizer::stop::SequenceDecoderOutput::Held + }) { + sgl_model_gateway::tokenizer::stop::SequenceDecoderOutput::Text(t) => { + text.push_str(&t); + } + sgl_model_gateway::tokenizer::stop::SequenceDecoderOutput::StoppedWithText(t) => { + text.push_str(&t); + break; + } + sgl_model_gateway::tokenizer::stop::SequenceDecoderOutput::Stopped => { + break; + } + sgl_model_gateway::tokenizer::stop::SequenceDecoderOutput::Held => {} + } + } + text + } else { + // Use incremental decoder to handle multi-byte character boundaries + let decode_stream = handle.decode_streams.entry(index).or_insert_with(|| { + DecodeStream::new( + Arc::clone(&tokenizer), + &[], // No prompt tokens for completion + handle.skip_special_tokens, + ) + }); + + // Process tokens incrementally + let mut text_parts = Vec::new(); + for &token_id in &chunk.token_ids { + if let Ok(Some(text)) = decode_stream.step(token_id) { + text_parts.push(text); + } + } + text_parts.join("") + }; + + if chunk_text.is_empty() { + return Ok(None); + } + + // Send first chunk with role + if first_chunk { + let first_response = ChatCompletionStreamResponse { + id: request_id.to_string(), + object: "chat.completion.chunk".to_string(), + created, + model: model.to_string(), + system_fingerprint: system_fingerprint.map(|s| s.to_string()), + choices: vec![ChatStreamChoice { + index, + delta: ChatMessageDelta { + role: Some("assistant".to_string()), + content: None, + tool_calls: None, + reasoning_content: None, + }, + logprobs: None, + finish_reason: None, + matched_stop: None, + }], + usage: None, + }; + return Ok(Some(first_response)); + } + + // Update stream buffer + let stream_buffer = handle.stream_buffers.entry(index).or_default(); + stream_buffer.push_str(&chunk_text); + + // Handle tool calls if tools are provided + if let (Some(ref tools), Some(ref tool_parser)) = (handle.tools.as_ref(), handle.tool_parser.as_ref()) { + let tool_choice_enabled = !matches!( + handle.tool_choice, + Some(ToolChoice::Value(ToolChoiceValue::None)) + ); + + if tool_choice_enabled { + let mut parser_guard = tool_parser.lock().await; + match parser_guard.parse_incremental(&chunk_text, tools).await { + Ok(streaming_result) => { + if !streaming_result.calls.is_empty() { + handle.has_tool_calls.insert(index, true); + // Convert tool call items to OpenAI format + let tool_call_deltas: Vec<_> = streaming_result + .calls + .into_iter() + .map(|item| { + let id = if let Some(ref name) = item.name { + generate_tool_call_id( + model, + name, + item.tool_index, + handle.history_tool_calls_count, + ) + } else { + format!("call_{}", item.tool_index) + }; + + ToolCallDelta { + index: item.tool_index as u32, + id: Some(id), + tool_type: if item.name.is_some() { + Some("function".to_string()) + } else { + None + }, + function: Some(FunctionCallDelta { + name: item.name, + arguments: if !item.parameters.is_empty() { + Some(item.parameters) + } else { + None + }, + }), + } + }) + .collect(); + + let tool_response = ChatCompletionStreamResponse { + id: request_id.to_string(), + object: "chat.completion.chunk".to_string(), + created, + model: model.to_string(), + system_fingerprint: system_fingerprint.map(|s| s.to_string()), + choices: vec![ChatStreamChoice { + index, + delta: ChatMessageDelta { + role: Some("assistant".to_string()), + content: None, + tool_calls: Some(tool_call_deltas), + reasoning_content: None, + }, + logprobs: None, + finish_reason: None, + matched_stop: None, + }], + usage: None, + }; + return Ok(Some(tool_response)); + } + } + Err(e) => { + // Log error but continue with regular content + tracing::warn!("Tool parser error: {}", e); + } + } + } + } + + // Regular content emission + let content_response = ChatCompletionStreamResponse { + id: request_id.to_string(), + object: "chat.completion.chunk".to_string(), + created, + model: model.to_string(), + system_fingerprint: system_fingerprint.map(|s| s.to_string()), + choices: vec![ChatStreamChoice { + index, + delta: ChatMessageDelta { + role: Some("assistant".to_string()), + content: Some(chunk_text), + tool_calls: None, + reasoning_content: None, + }, + logprobs: None, + finish_reason: None, + matched_stop: None, + }], + usage: None, + }; + + Ok(Some(content_response)) + } + Some(Complete(complete)) => { + let index = complete.index; + + // Flush any remaining text + // Flush any remaining text from decode stream + let mut final_text = handle.stream_buffers.remove(&index).unwrap_or_default(); + if let Some(ref mut decode_stream) = handle.decode_streams.get_mut(&index) { + if let Ok(Some(remaining)) = decode_stream.flush() { + final_text.push_str(&remaining); + } + } + handle.decode_streams.remove(&index); + + // Determine finish reason - ensure it's never empty + // If finish_reason is empty, try to infer from other fields or use default + let finish_reason = if handle.has_tool_calls.get(&index).copied().unwrap_or(false) + && (complete.finish_reason == "stop" || complete.finish_reason.is_empty()) + { + "tool_calls".to_string() + } else if complete.finish_reason.is_empty() || complete.finish_reason.trim().is_empty() { + // If finish_reason is empty, try to infer from completion_tokens or use default + if complete.completion_tokens > 0 { + // If we have completion tokens, likely stopped normally + "stop".to_string() + } else if !complete.output_ids.is_empty() { + // If we have output_ids, likely stopped normally + "stop".to_string() + } else { + // Default fallback - always ensure we have a value + "stop".to_string() + } + } else { + complete.finish_reason.clone() + }; + + // Ensure finish_reason is never empty (defensive check) + let finish_reason = if finish_reason.is_empty() || finish_reason.trim().is_empty() { + "stop".to_string() + } else { + finish_reason + }; + + // Extract matched_stop + let matched_stop = match &complete.matched_stop { + Some(proto::generate_complete::MatchedStop::MatchedTokenId(token_id)) => { + Some(Value::Number(serde_json::Number::from(*token_id))) + } + Some(proto::generate_complete::MatchedStop::MatchedStopStr(stop_str)) => { + Some(Value::String(stop_str.clone())) + } + None => None, + }; + + // Build usage - prefer values from complete message, but fallback to accumulated values from chunks + // Complete message should have the final values, but sometimes they might be 0 or missing + // Always use the latest cumulative value from chunks if available, otherwise use complete message value + let mut prompt_tokens = handle.prompt_tokens.get(&index) + .copied() + .filter(|&v| v > 0) + .unwrap_or(complete.prompt_tokens); + let mut completion_tokens = handle.completion_tokens.get(&index) + .copied() + .filter(|&v| v > 0) + .unwrap_or(complete.completion_tokens); + + // Always try to use initial_prompt_tokens if prompt_tokens is 0 or missing + // This is the most reliable source for prompt tokens since we calculate it from the request + if prompt_tokens == 0 { + if let Some(initial_prompt) = handle.initial_prompt_tokens { + prompt_tokens = initial_prompt; + } + } + + // If completion_tokens is 0, try to infer from output_ids or accumulated chunks + if completion_tokens == 0 { + // Try to use completion_tokens from complete message even if 0 + // Or calculate from output_ids + if complete.completion_tokens > 0 { + completion_tokens = complete.completion_tokens; + } else if !complete.output_ids.is_empty() { + completion_tokens = complete.output_ids.len() as i32; + } else if let Some(&last_completion) = handle.completion_tokens.get(&index) { + completion_tokens = last_completion; + } + } + + // Final fallback: if both are still 0, try to use initial_prompt_tokens for prompt + // and calculate completion from output_ids + if prompt_tokens == 0 && completion_tokens == 0 { + // Try to infer from output_ids if available + let output_ids_len = complete.output_ids.len() as i32; + if output_ids_len > 0 { + completion_tokens = output_ids_len; + // Always try to use initial_prompt_tokens for prompt + if let Some(initial_prompt) = handle.initial_prompt_tokens { + prompt_tokens = initial_prompt; + } + } + } + + // Final defensive check: ensure prompt_tokens is set if we have initial_prompt_tokens + if prompt_tokens == 0 { + if let Some(initial_prompt) = handle.initial_prompt_tokens { + prompt_tokens = initial_prompt; + } + } + + // Always create usage, even if values are 0 (defensive) + let usage = Some(Usage { + prompt_tokens: prompt_tokens.max(0) as u32, + completion_tokens: completion_tokens.max(0) as u32, + total_tokens: (prompt_tokens.max(0) + completion_tokens.max(0)) as u32, + completion_tokens_details: None, + }); + + let finish_response = ChatCompletionStreamResponse { + id: request_id.to_string(), + object: "chat.completion.chunk".to_string(), + created, + model: model.to_string(), + system_fingerprint: system_fingerprint.map(|s| s.to_string()), + choices: vec![ChatStreamChoice { + index, + delta: ChatMessageDelta { + role: Some("assistant".to_string()), + content: if !final_text.is_empty() { + Some(final_text) + } else { + None + }, + tool_calls: None, + reasoning_content: None, + }, + logprobs: None, + finish_reason: Some(finish_reason), + matched_stop, + }], + usage, + }; + + Ok(Some(finish_response)) + } + Some(Error(error)) => { + Err(format!("Server error: {} (status: {})", error.message, error.http_status_code)) + } + None => Ok(None), + } +} + +/// Free a gRPC response converter handle +#[no_mangle] +pub unsafe extern "C" fn sgl_grpc_response_converter_free(handle: *mut GrpcResponseConverterHandle) { + if !handle.is_null() { + let _ = Box::from_raw(handle); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/lib.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/lib.rs new file mode 100644 index 00000000..5bbe027a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/lib.rs @@ -0,0 +1,88 @@ +//! FFI module for exposing sgl-router preprocessing and postprocessing functions +//! to C-compatible languages (e.g., Golang via cgo) +//! +//! This module provides C-compatible function signatures for: +//! - Tokenizer operations (encode, decode, chat template) +//! - Tool parser operations (parse tool calls) +//! - Tool constraint generation +//! - gRPC client SDK (complete request-response flow) +//! +//! # Safety +//! All functions marked with `#[no_mangle]` and `extern "C"` must be called +//! with valid pointers and follow the documented memory management rules. + +// Re-export error types +pub use error::{SglErrorCode, set_error_message, set_error_message_fmt, clear_error_message}; + +// Re-export memory management functions +pub use memory::{sgl_free_string, sgl_free_token_ids}; + +// Re-export tokenizer functions +pub use tokenizer::{ + TokenizerHandle, + sgl_tokenizer_create_from_file, + sgl_tokenizer_encode, + sgl_tokenizer_apply_chat_template, + sgl_tokenizer_apply_chat_template_with_tools, + sgl_tokenizer_decode, + sgl_tokenizer_free, +}; + +// Re-export tool parser functions +pub use tool_parser::{ + ToolParserHandle, + sgl_tool_parser_create, + sgl_tool_parser_parse_complete, + sgl_tool_parser_parse_incremental, + sgl_tool_parser_reset, + sgl_tool_parser_free, +}; + +// Re-export gRPC converter functions +pub use grpc_converter::{ + GrpcResponseConverterHandle, + sgl_grpc_response_converter_create, + sgl_grpc_response_converter_convert_chunk, + sgl_grpc_response_converter_free, +}; + +// Re-export client SDK functions +pub use client::{ + SglangClientHandle, + sgl_client_create, + sgl_client_free, +}; + +// Re-export stream functions +pub use stream::{ + SglangStreamHandle, + sgl_stream_read_next, + sgl_stream_free, +}; + +// Re-export client stream function (defined in client.rs but used by stream) +pub use client::sgl_client_chat_completion_stream; + +// Re-export utility functions +pub use utils::sgl_generate_tool_constraints; + +// Sub-modules +mod error; +mod memory; +mod tokenizer; +mod tool_parser; +mod grpc_converter; +mod client; +mod stream; +mod utils; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_error_codes() { + assert_eq!(SglErrorCode::Success as i32, 0); + assert_eq!(SglErrorCode::InvalidArgument as i32, 1); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/memory.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/memory.rs new file mode 100644 index 00000000..9aac9b33 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/memory.rs @@ -0,0 +1,28 @@ +//! Memory management for FFI functions + +use std::ffi::CString; +use std::os::raw::c_char; + +/// Free a C string allocated by Rust +/// +/// # Safety +/// This function must only be called with pointers returned by other FFI functions. +/// Calling with arbitrary pointers or multiple times on the same pointer is undefined behavior. +#[no_mangle] +pub unsafe extern "C" fn sgl_free_string(s: *mut c_char) { + if !s.is_null() { + let _ = CString::from_raw(s); + } +} + +/// Free token IDs array allocated by Rust +/// +/// # Safety +/// This function must only be called with pointers returned by `sgl_tokenizer_encode`. +/// The `count` parameter must match the length of the array. +#[no_mangle] +pub unsafe extern "C" fn sgl_free_token_ids(ptr: *mut u32, count: usize) { + if !ptr.is_null() && count > 0 { + let _ = Vec::from_raw_parts(ptr, count, count); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/stream.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/stream.rs new file mode 100644 index 00000000..65cd3247 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/stream.rs @@ -0,0 +1,288 @@ +//! Stream handling FFI functions +//! +//! This module provides FFI (Foreign Function Interface) functions for managing +//! streaming responses from the SGLang gRPC API. It handles: +//! +//! - Creating and managing stream handles +//! - Reading chunks from streams and converting them to OpenAI format +//! - Managing automatic abort on stream drop (via AbortOnDropStream) +//! - Thread-safe access to streams and response converters +//! +//! # Safety +//! +//! All FFI functions are marked `unsafe` as per Rust FFI conventions. Callers must: +//! - Pass valid pointers +//! - Ensure proper pointer lifetime management +//! - Call corresponding free functions for cleanup + +use std::ffi::CString; +use std::os::raw::{c_char, c_int}; +use std::ptr; +use std::sync::Arc; +use tokio::runtime::Runtime; +use once_cell::sync::Lazy; +use futures_util::StreamExt; + +use sgl_model_gateway::grpc_client::{sglang_proto as proto, sglang_scheduler::{SglangSchedulerClient, AbortOnDropStream}}; + +use super::error::{SglErrorCode, set_error_message}; +use super::grpc_converter::{GrpcResponseConverterHandle, convert_proto_chunk_to_openai}; + +/// Global tokio runtime for async operations +static RUNTIME: Lazy = Lazy::new(|| { + Runtime::new().expect("Failed to create tokio runtime for stream FFI") +}); + +/// Handle for an active streaming request. +/// +/// This struct manages the stream and response converter for a single request. +/// It is wrapped in Arc and Mutex for thread-safe concurrent access. +/// +/// # Fields +/// +/// * `stream` - The gRPC stream wrapped in AbortOnDropStream for automatic cleanup +/// * `converter` - Response converter that transforms proto messages to OpenAI format +/// * `client` - The underlying gRPC client connection +/// * `prompt_tokens` - Number of prompt tokens from the original request +pub struct SglangStreamHandle { + pub(crate) stream: Arc>, + pub(crate) converter: Arc>, + #[allow(dead_code)] + pub(crate) client: Arc, + #[allow(dead_code)] + pub(crate) prompt_tokens: i32, // Number of prompt tokens for this request +} + +/// Read next chunk from stream and convert to OpenAI format. +/// +/// This function reads the next chunk from the gRPC stream, converts it from the +/// internal protocol format to OpenAI-compatible JSON format, and returns it via +/// the output parameters. +/// +/// # Arguments +/// +/// * `stream_handle` - Mutable pointer to the stream handle +/// * `response_json_out` - Pointer to receive OpenAI format JSON string +/// - Caller must free this with `sgl_free_string` +/// - May be NULL if no data available +/// * `is_done_out` - Pointer to receive completion status +/// - 0 = stream has more data +/// - 1 = stream is complete +/// * `error_out` - Optional pointer to receive error message +/// - Only set if function returns an error code +/// - Must be freed with `sgl_free_string` if not NULL +/// +/// # Returns +/// +/// * `SglErrorCode::Success` - Successfully read a chunk or reached end of stream +/// * Other error codes - See `SglErrorCode` for details +/// +/// # Safety +/// +/// - All pointers must be valid and properly aligned +/// - `stream_handle` must point to a valid `SglangStreamHandle` +/// - Output pointers must be writable +/// +/// # Notes +/// +/// - Complete messages are identified by the presence of `proto::GenerateResponse::Complete` +/// - When is_done=1, this may be the last readable chunk or the stream may be ending +/// - Subsequent calls after is_done=1 will mark the stream as complete internally +#[no_mangle] +pub unsafe extern "C" fn sgl_stream_read_next( + stream_handle: *mut SglangStreamHandle, + response_json_out: *mut *mut c_char, + is_done_out: *mut c_int, + error_out: *mut *mut c_char, +) -> SglErrorCode { + if stream_handle.is_null() || response_json_out.is_null() || is_done_out.is_null() { + set_error_message(error_out, "Invalid arguments: null pointer"); + return SglErrorCode::InvalidArgument; + } + + let handle_ref = &*stream_handle; + let stream = Arc::clone(&handle_ref.stream); + let converter = Arc::clone(&handle_ref.converter); + + // Read next chunk from stream + let chunk_result = RUNTIME.block_on(async { + let mut stream_guard = stream.lock().await; + stream_guard.next().await + }); + + match chunk_result { + Some(Ok(proto_response)) => { + // Convert proto response to OpenAI format + // We need to get the converter lock first + let conversion_result = RUNTIME.block_on(async { + let mut converter_guard = converter.lock().await; + + // Clone necessary fields for conversion + let tokenizer = Arc::clone(&converter_guard.tokenizer); + let model = converter_guard.model.clone(); + let request_id = converter_guard.request_id.clone(); + let created = converter_guard.created; + let system_fingerprint = converter_guard.system_fingerprint.clone(); + + // Call the conversion function + convert_proto_chunk_to_openai( + proto_response.clone(), + &mut *converter_guard, + &tokenizer, + &model, + &request_id, + created, + system_fingerprint.as_deref(), + ) + .await + }); + + match conversion_result { + Ok(Some(openai_response)) => { + // Serialize to JSON + let result_str = match serde_json::to_string(&openai_response) { + Ok(s) => s, + Err(e) => { + set_error_message(error_out, &format!("Failed to serialize response: {}", e)); + return SglErrorCode::ParsingError; + } + }; + + let result_cstr = match CString::new(result_str) { + Ok(s) => s, + Err(e) => { + set_error_message(error_out, &format!("Failed to create result string: {}", e)); + return SglErrorCode::MemoryError; + } + }; + + // Check if this is a complete response (stream done) + let is_complete = matches!(proto_response.response, Some(proto::generate_response::Response::Complete(_)) | Some(proto::generate_response::Response::Error(_))); + + *response_json_out = result_cstr.into_raw(); + *is_done_out = if is_complete { 1 } else { 0 }; + + if is_complete { + // Mark stream as completed + // Ensure mark_completed() completes and is visible before returning + // Use yield_now to ensure Release ordering is fully propagated + RUNTIME.block_on(async { + let stream_guard = stream.lock().await; + stream_guard.mark_completed(); + // Keep the guard until mark_completed() is fully executed + drop(stream_guard); + // Yield to ensure Release ordering is propagated before returning + // This prevents race condition where Free() is called immediately + // and Drop might not see the mark_completed() write + tokio::task::yield_now().await; + }); + } + + SglErrorCode::Success + } + Ok(None) => { + // No response to send (e.g., empty chunk) + // Don't mark as completed - stream might continue + // Just return null and let caller read more + *response_json_out = ptr::null_mut(); + *is_done_out = 0; // Keep stream open, not done yet + SglErrorCode::Success + } + Err(e) => { + // Conversion error - don't mark as completed + // Let the stream end naturally or return error without stopping stream + set_error_message(error_out, &format!("Conversion error: {}", e)); + *response_json_out = ptr::null_mut(); + *is_done_out = 0; // Don't mark as done - let caller decide + SglErrorCode::ParsingError + } + } + } + Some(Err(e)) => { + // Stream error - mark as completed to prevent abort + RUNTIME.block_on(async { + let stream_guard = stream.lock().await; + stream_guard.mark_completed(); + drop(stream_guard); + // Yield to ensure Release ordering is propagated + tokio::task::yield_now().await; + }); + + set_error_message(error_out, &format!("Stream error: {}", e)); + *is_done_out = 1; + SglErrorCode::UnknownError + } + None => { + // Stream ended naturally (no more chunks) + // Mark stream as completed before returning to prevent abort + RUNTIME.block_on(async { + let stream_guard = stream.lock().await; + stream_guard.mark_completed(); + drop(stream_guard); + // Yield to ensure Release ordering is propagated + tokio::task::yield_now().await; + }); + + *response_json_out = ptr::null_mut(); + *is_done_out = 1; + SglErrorCode::Success + } + } +} + +/// Free a stream handle and release all associated resources. +/// +/// This function must be called exactly once for each stream handle returned by +/// `sgl_client_chat_completion_stream`. It marks the stream as completed internally +/// to prevent abort signals from being sent when resources are cleaned up. +/// +/// # Arguments +/// +/// * `handle` - Mutable pointer to the stream handle to free +/// - If NULL, this function does nothing +/// +/// # Safety +/// +/// - Must be called only once per handle +/// - Handle must not be used after calling this function +/// - After this call, the stream is no longer valid +/// +/// # Notes +/// +/// - This function internally calls `mark_completed()` before freeing to ensure +/// the stream cleanup doesn't trigger an abort RPC to the server +/// - Memory fences are used to ensure visibility across threads +#[no_mangle] +pub unsafe extern "C" fn sgl_stream_free(handle: *mut SglangStreamHandle) { + if !handle.is_null() { + let handle_ref = Box::from_raw(handle); + + // Mark stream as completed to prevent abort on drop + // By this point, the stream should already be completed by ReadNext() + // but we call it again to be safe + RUNTIME.block_on(async { + let stream_guard = handle_ref.stream.lock().await; + stream_guard.mark_completed(); + // Keep guard alive to ensure mark_completed() write completes + drop(stream_guard); + // Yield to ensure the atomic write is visible + tokio::task::yield_now().await; + }); + + // Use a strong memory fence to ensure mark_completed()'s Release write + // is visible before we drop the last Arc reference + std::sync::atomic::fence(std::sync::atomic::Ordering::SeqCst); + + // Now drop all references - if mark_completed() was called successfully, + // the drop won't send an abort + drop(handle_ref.stream); + + // Free converter + let converter = Arc::try_unwrap(handle_ref.converter) + .ok() + .map(|m| m.into_inner()); + if let Some(conv) = converter { + super::grpc_converter::sgl_grpc_response_converter_free(Box::into_raw(Box::new(conv))); + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/tokenizer.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/tokenizer.rs new file mode 100644 index 00000000..567abefc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/tokenizer.rs @@ -0,0 +1,379 @@ +//! Tokenizer FFI functions + +use std::ffi::{CStr, CString}; +use std::os::raw::{c_char, c_int}; +use std::ptr; +use std::sync::Arc; +use serde_json::Value; + +use sgl_model_gateway::tokenizer::{ + create_tokenizer_from_file, + traits::Tokenizer as TokenizerTrait, + chat_template::ChatTemplateParams, + huggingface::HuggingFaceTokenizer, +}; + +use super::error::{SglErrorCode, set_error_message, clear_error_message}; + +/// Opaque handle for a tokenizer instance +#[repr(C)] +pub struct TokenizerHandle { + pub(crate) tokenizer: Arc, +} + +/// Create a tokenizer from a file path +/// +/// # Arguments +/// * `path` - Path to tokenizer.json file (null-terminated C string) +/// * `error_out` - Optional pointer to receive error message (must be freed with sgl_free_string) +/// +/// # Returns +/// * Pointer to TokenizerHandle on success, null on failure +/// +/// # Safety +/// The returned handle must be freed with `sgl_tokenizer_free`. +#[no_mangle] +pub unsafe extern "C" fn sgl_tokenizer_create_from_file( + path: *const c_char, + error_out: *mut *mut c_char, +) -> *mut TokenizerHandle { + if path.is_null() { + set_error_message(error_out, "path cannot be null"); + return ptr::null_mut(); + } + + let path_str = match CStr::from_ptr(path).to_str() { + Ok(s) => s, + Err(e) => { + set_error_message(error_out, &format!("Invalid UTF-8 in path: {}", e)); + return ptr::null_mut(); + } + }; + + match create_tokenizer_from_file(path_str) { + Ok(tokenizer) => { + clear_error_message(error_out); + Box::into_raw(Box::new(TokenizerHandle { + tokenizer, + })) + } + Err(e) => { + set_error_message(error_out, &e.to_string()); + ptr::null_mut() + } + } +} + +/// Encode text to token IDs +/// +/// # Arguments +/// * `handle` - Tokenizer handle (must not be null) +/// * `text` - Input text (null-terminated C string) +/// * `token_ids_out` - Pointer to receive array of token IDs (must be freed with sgl_free_token_ids) +/// * `token_count_out` - Pointer to receive token count +/// * `error_out` - Optional pointer to receive error message +/// +/// # Returns +/// * SglErrorCode::Success on success, error code on failure +/// +/// # Safety +/// The token_ids_out array must be freed with sgl_free_token_ids() after use. +#[no_mangle] +pub unsafe extern "C" fn sgl_tokenizer_encode( + handle: *mut TokenizerHandle, + text: *const c_char, + token_ids_out: *mut *mut u32, + token_count_out: *mut usize, + error_out: *mut *mut c_char, +) -> SglErrorCode { + if handle.is_null() || text.is_null() || token_ids_out.is_null() || token_count_out.is_null() { + set_error_message(error_out, "Invalid arguments: null pointer"); + return SglErrorCode::InvalidArgument; + } + + let text_str = match CStr::from_ptr(text).to_str() { + Ok(s) => s, + Err(_) => { + set_error_message(error_out, "Invalid UTF-8 in text"); + return SglErrorCode::InvalidArgument; + } + }; + + let tokenizer = &(*handle).tokenizer; + match tokenizer.encode(text_str) { + Ok(encoding) => { + let token_ids = encoding.token_ids(); + let count = token_ids.len(); + + // Allocate memory for token IDs using Vec, then leak to give ownership to C + let vec = token_ids.to_vec(); + let ptr = vec.as_ptr() as *mut u32; + let _ = std::mem::ManuallyDrop::new(vec); + + *token_ids_out = ptr; + *token_count_out = count; + clear_error_message(error_out); + SglErrorCode::Success + } + Err(e) => { + set_error_message(error_out, &e.to_string()); + SglErrorCode::TokenizationError + } + } +} + +/// Apply chat template to messages with tools support +/// +/// # Arguments +/// * `handle` - Tokenizer handle +/// * `messages_json` - JSON string of messages array +/// * `tools_json` - Optional JSON string of tools array (null or empty string for no tools) +/// * `result_out` - Pointer to receive result string (must be freed with sgl_free_string) +/// * `error_out` - Optional pointer to receive error message +/// +/// # Returns +/// * SglErrorCode::Success on success, error code on failure +#[no_mangle] +pub unsafe extern "C" fn sgl_tokenizer_apply_chat_template_with_tools( + handle: *mut TokenizerHandle, + messages_json: *const c_char, + tools_json: *const c_char, + result_out: *mut *mut c_char, + error_out: *mut *mut c_char, +) -> SglErrorCode { + if handle.is_null() || messages_json.is_null() || result_out.is_null() { + set_error_message(error_out, "Invalid arguments: null pointer"); + return SglErrorCode::InvalidArgument; + } + + let messages_str = match CStr::from_ptr(messages_json).to_str() { + Ok(s) => s, + Err(_) => { + set_error_message(error_out, "Invalid UTF-8 in messages_json"); + return SglErrorCode::InvalidArgument; + } + }; + + // Parse JSON messages + let messages: Vec = match serde_json::from_str(messages_str) { + Ok(msgs) => msgs, + Err(e) => { + set_error_message(error_out, &format!("Failed to parse messages JSON: {}", e)); + return SglErrorCode::InvalidArgument; + } + }; + + // Parse tools JSON if provided + let tools: Option> = if tools_json.is_null() { + None + } else { + let tools_str = match CStr::from_ptr(tools_json).to_str() { + Ok(s) => { + if s.is_empty() { + None + } else { + match serde_json::from_str::>(s) { + Ok(t) => Some(t), + Err(e) => { + set_error_message(error_out, &format!("Failed to parse tools JSON: {}", e)); + return SglErrorCode::InvalidArgument; + } + } + } + } + Err(_) => { + set_error_message(error_out, "Invalid UTF-8 in tools_json"); + return SglErrorCode::InvalidArgument; + } + }; + tools_str + }; + + // Get the tokenizer from handle + let handle_ref = &*handle; + let tokenizer = &handle_ref.tokenizer; + + // Try to downcast to HuggingFaceTokenizer + if let Some(hf_tokenizer) = tokenizer.as_any().downcast_ref::() { + // Apply chat template with tools + let empty_docs: [Value; 0] = []; + let tools_slice = tools.as_ref().map(|t| t.as_slice()); + let params = ChatTemplateParams { + add_generation_prompt: true, + tools: tools_slice, + documents: Some(&empty_docs), + template_kwargs: None, + }; + + match hf_tokenizer.apply_chat_template(&messages, params) { + Ok(result) => { + let result_cstr = match CString::new(result) { + Ok(s) => s, + Err(e) => { + set_error_message(error_out, &format!("Failed to create result string: {}", e)); + return SglErrorCode::MemoryError; + } + }; + *result_out = result_cstr.into_raw(); + clear_error_message(error_out); + SglErrorCode::Success + } + Err(e) => { + set_error_message(error_out, &format!("Failed to apply chat template: {}", e)); + SglErrorCode::TokenizationError + } + } + } else { + set_error_message(error_out, "Chat template is only supported for HuggingFace tokenizers"); + SglErrorCode::TokenizationError + } +} + +/// Apply chat template to messages +/// +/// # Arguments +/// * `handle` - Tokenizer handle +/// * `messages_json` - JSON string of messages array +/// * `result_out` - Pointer to receive result string (must be freed with sgl_free_string) +/// * `error_out` - Optional pointer to receive error message +/// +/// # Returns +/// * SglErrorCode::Success on success, error code on failure +#[no_mangle] +pub unsafe extern "C" fn sgl_tokenizer_apply_chat_template( + handle: *mut TokenizerHandle, + messages_json: *const c_char, + result_out: *mut *mut c_char, + error_out: *mut *mut c_char, +) -> SglErrorCode { + if handle.is_null() || messages_json.is_null() || result_out.is_null() { + set_error_message(error_out, "Invalid arguments: null pointer"); + return SglErrorCode::InvalidArgument; + } + + let messages_str = match CStr::from_ptr(messages_json).to_str() { + Ok(s) => s, + Err(_) => { + set_error_message(error_out, "Invalid UTF-8 in messages_json"); + return SglErrorCode::InvalidArgument; + } + }; + + // Parse JSON messages + let messages: Vec = match serde_json::from_str(messages_str) { + Ok(msgs) => msgs, + Err(e) => { + set_error_message(error_out, &format!("Failed to parse messages JSON: {}", e)); + return SglErrorCode::InvalidArgument; + } + }; + + // Get the tokenizer from handle + let handle_ref = &*handle; + let tokenizer = &handle_ref.tokenizer; + + // Try to downcast to HuggingFaceTokenizer + if let Some(hf_tokenizer) = tokenizer.as_any().downcast_ref::() { + // Apply chat template with default parameters + // Use empty arrays instead of None to avoid template errors + // Set add_generation_prompt to true so the model knows to start generating + let empty_tools: [Value; 0] = []; + let empty_docs: [Value; 0] = []; + let params = ChatTemplateParams { + add_generation_prompt: true, // Important: tells the model to start generating + tools: Some(&empty_tools), + documents: Some(&empty_docs), + template_kwargs: None, + }; + + match hf_tokenizer.apply_chat_template(&messages, params) { + Ok(result) => { + let result_cstr = match CString::new(result) { + Ok(s) => s, + Err(e) => { + set_error_message(error_out, &format!("Failed to create result string: {}", e)); + return SglErrorCode::MemoryError; + } + }; + *result_out = result_cstr.into_raw(); + clear_error_message(error_out); + SglErrorCode::Success + } + Err(e) => { + set_error_message(error_out, &format!("Failed to apply chat template: {}", e)); + SglErrorCode::TokenizationError + } + } + } else { + set_error_message(error_out, "Chat template is only supported for HuggingFace tokenizers"); + SglErrorCode::TokenizationError + } +} + +/// Decode token IDs to text +/// +/// # Arguments +/// * `handle` - Tokenizer handle +/// * `token_ids` - Array of token IDs +/// * `token_count` - Number of tokens +/// * `skip_special_tokens` - Whether to skip special tokens +/// * `result_out` - Pointer to receive result string (must be freed with sgl_free_string) +/// * `error_out` - Optional pointer to receive error message +/// +/// # Returns +/// * SglErrorCode::Success on success, error code on failure +#[no_mangle] +pub unsafe extern "C" fn sgl_tokenizer_decode( + handle: *mut TokenizerHandle, + token_ids: *const u32, + token_count: usize, + skip_special_tokens: c_int, + result_out: *mut *mut c_char, + error_out: *mut *mut c_char, +) -> SglErrorCode { + if handle.is_null() || token_ids.is_null() || result_out.is_null() { + set_error_message(error_out, "Invalid arguments: null pointer"); + return SglErrorCode::InvalidArgument; + } + + if token_count == 0 { + let empty = CString::new("").unwrap(); + *result_out = empty.into_raw(); + clear_error_message(error_out); + return SglErrorCode::Success; + } + + // Convert C array to Rust slice + let token_slice = std::slice::from_raw_parts(token_ids, token_count); + + let tokenizer = &(*handle).tokenizer; + match tokenizer.decode(token_slice, skip_special_tokens != 0) { + Ok(text) => { + let result_cstr = match CString::new(text) { + Ok(s) => s, + Err(e) => { + set_error_message(error_out, &format!("Failed to create result string: {}", e)); + return SglErrorCode::MemoryError; + } + }; + *result_out = result_cstr.into_raw(); + clear_error_message(error_out); + SglErrorCode::Success + } + Err(e) => { + set_error_message(error_out, &e.to_string()); + SglErrorCode::TokenizationError + } + } +} + +/// Free a tokenizer handle +/// +/// # Safety +/// This function must only be called once per handle, and the handle must not be used after calling. +#[no_mangle] +pub unsafe extern "C" fn sgl_tokenizer_free(handle: *mut TokenizerHandle) { + if !handle.is_null() { + let _ = Box::from_raw(handle); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/tool_parser.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/tool_parser.rs new file mode 100644 index 00000000..49b2639a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/tool_parser.rs @@ -0,0 +1,329 @@ +//! Tool parser FFI functions + +use std::ffi::{CStr, CString}; +use std::os::raw::{c_char}; +use std::ptr; +use std::sync::Arc; +use std::collections::HashMap; +use serde_json::{json, Value}; +use tokio::runtime::Runtime; +use once_cell::sync::Lazy; + +use sgl_model_gateway::tool_parser::{ParserFactory, ToolParser}; +use sgl_model_gateway::protocols::common::Tool; + +use super::error::{SglErrorCode, set_error_message, clear_error_message}; +use super::utils::generate_tool_call_id; + +/// Global parser factory (initialized once) +static PARSER_FACTORY: Lazy = Lazy::new(|| ParserFactory::new()); + +/// Global tokio runtime for async operations +static RUNTIME: Lazy = Lazy::new(|| { + Runtime::new().expect("Failed to create tokio runtime for tool parser FFI") +}); + +/// Opaque handle for a tool parser instance +/// Note: For streaming, we need mutable access, so we use Arc> internally +/// Note: This is an opaque handle, C code doesn't access fields directly +pub struct ToolParserHandle { + parser: Arc>>, + model: String, // Store model name for ID generation + history_tool_calls_count: usize, // Track tool call count for ID generation + tool_index_to_id: HashMap, // Map tool_index to ID for incremental updates +} + +/// Create a tool parser +/// +/// # Arguments +/// * `parser_type` - Parser type name (e.g., "json", "llama", "mistral") or model name (e.g., "gpt-4") +/// * `error_out` - Optional pointer to receive error message +/// +/// # Returns +/// * Pointer to ToolParserHandle on success, null on failure +#[no_mangle] +pub unsafe extern "C" fn sgl_tool_parser_create( + parser_type: *const c_char, + error_out: *mut *mut c_char, +) -> *mut ToolParserHandle { + if parser_type.is_null() { + set_error_message(error_out, "parser_type cannot be null"); + return ptr::null_mut(); + } + + let type_str = match CStr::from_ptr(parser_type).to_str() { + Ok(s) => s, + Err(_) => { + set_error_message(error_out, "Invalid UTF-8 in parser_type"); + return ptr::null_mut(); + } + }; + + // Create parser using factory + // The factory will determine the parser type based on model name or use the provided type + let parser = if let Some(parser_box) = PARSER_FACTORY.registry().create_for_model(type_str) { + parser_box + } else if let Some(parser_box) = PARSER_FACTORY.registry().create_parser(type_str) { + parser_box + } else { + set_error_message(error_out, &format!("Unknown parser type: {}", type_str)); + return ptr::null_mut(); + }; + + Box::into_raw(Box::new(ToolParserHandle { + parser: Arc::new(tokio::sync::Mutex::new(parser)), + model: type_str.to_string(), + history_tool_calls_count: 0, + tool_index_to_id: HashMap::new(), + })) +} + +/// Parse complete tool calls from text +/// +/// # Arguments +/// * `handle` - Tool parser handle +/// * `text` - Input text to parse +/// * `result_json_out` - Pointer to receive JSON result (must be freed with sgl_free_string) +/// * `error_out` - Optional pointer to receive error message +/// +/// # Returns +/// * SglErrorCode::Success on success, error code on failure +#[no_mangle] +pub unsafe extern "C" fn sgl_tool_parser_parse_complete( + handle: *mut ToolParserHandle, + text: *const c_char, + result_json_out: *mut *mut c_char, + error_out: *mut *mut c_char, +) -> SglErrorCode { + if handle.is_null() || text.is_null() || result_json_out.is_null() { + set_error_message(error_out, "Invalid arguments: null pointer"); + return SglErrorCode::InvalidArgument; + } + + let text_str = match CStr::from_ptr(text).to_str() { + Ok(s) => s, + Err(_) => { + set_error_message(error_out, "Invalid UTF-8 in text"); + return SglErrorCode::InvalidArgument; + } + }; + + let handle_ref = &*handle; + let parser = Arc::clone(&handle_ref.parser); + let model = handle_ref.model.clone(); + let history_count = handle_ref.history_tool_calls_count; + + // Use tokio runtime to run async code + let result = RUNTIME.block_on(async { + let parser_guard = parser.lock().await; + parser_guard.parse_complete(text_str).await + }); + + match result { + Ok((normal_text, tool_calls)) => { + // Convert Rust ToolCall to OpenAI format + let openai_tool_calls: Vec = tool_calls + .into_iter() + .enumerate() + .map(|(index, tc)| { + // Generate ID for this tool call + let id = generate_tool_call_id(&model, &tc.function.name, index, history_count); + json!({ + "id": id, + "type": "function", + "function": { + "name": tc.function.name, + "arguments": tc.function.arguments + } + }) + }) + .collect(); + + // Build result JSON + let result_json = json!({ + "normal_text": normal_text, + "tool_calls": openai_tool_calls + }); + + let result_str = match serde_json::to_string(&result_json) { + Ok(s) => s, + Err(e) => { + set_error_message(error_out, &format!("Failed to serialize JSON: {}", e)); + return SglErrorCode::ParsingError; + } + }; + + let result_cstr = match CString::new(result_str) { + Ok(s) => s, + Err(e) => { + set_error_message(error_out, &format!("Failed to create result string: {}", e)); + return SglErrorCode::MemoryError; + } + }; + + *result_json_out = result_cstr.into_raw(); + clear_error_message(error_out); + SglErrorCode::Success + } + Err(e) => { + set_error_message(error_out, &format!("Parse error: {}", e)); + SglErrorCode::ParsingError + } + } +} + +/// Parse tool calls incrementally from streaming chunks +/// +/// # Arguments +/// * `handle` - Tool parser handle +/// * `chunk` - New text chunk from stream +/// * `tools_json` - JSON array of available tools (for validation, can be null/empty) +/// * `result_json_out` - Pointer to receive JSON result (must be freed with sgl_free_string) +/// * `error_out` - Optional pointer to receive error message +/// +/// # Returns +/// * SglErrorCode::Success on success, error code on failure +#[no_mangle] +pub unsafe extern "C" fn sgl_tool_parser_parse_incremental( + handle: *mut ToolParserHandle, + chunk: *const c_char, + tools_json: *const c_char, + result_json_out: *mut *mut c_char, + error_out: *mut *mut c_char, +) -> SglErrorCode { + if handle.is_null() || chunk.is_null() || result_json_out.is_null() { + set_error_message(error_out, "Invalid arguments: null pointer"); + return SglErrorCode::InvalidArgument; + } + + let chunk_str = match CStr::from_ptr(chunk).to_str() { + Ok(s) => s, + Err(_) => { + set_error_message(error_out, "Invalid UTF-8 in chunk"); + return SglErrorCode::InvalidArgument; + } + }; + + // Parse tools JSON if provided + let tools: Vec = if !tools_json.is_null() { + let tools_str = match CStr::from_ptr(tools_json).to_str() { + Ok(s) => s, + Err(_) => { + set_error_message(error_out, "Invalid UTF-8 in tools_json"); + return SglErrorCode::InvalidArgument; + } + }; + match serde_json::from_str::>(tools_str) { + Ok(t) => t, + Err(_) => vec![], // If parsing fails, use empty tools + } + } else { + vec![] + }; + + let handle_ref = &*handle; + let parser = Arc::clone(&handle_ref.parser); + let model = handle_ref.model.clone(); + let history_count = handle_ref.history_tool_calls_count; + + // Use tokio runtime to run async code + let result = RUNTIME.block_on(async { + let mut parser_guard = parser.lock().await; + parser_guard.parse_incremental(chunk_str, &tools).await + }); + + match result { + Ok(streaming_result) => { + // Convert StreamingParseResult to OpenAI format + let handle_mut = &mut *handle; + let openai_tool_calls: Vec = streaming_result + .calls + .into_iter() + .map(|item| { + // For incremental parsing, we may not have complete tool calls yet + // Generate or reuse ID based on tool_index + let id = if let Some(ref name) = item.name { + // New tool call with name - generate ID and store it + let id = generate_tool_call_id(&model, name, item.tool_index, history_count); + handle_mut.tool_index_to_id.insert(item.tool_index, id.clone()); + id + } else { + // Parameter update - reuse existing ID for this tool_index + handle_mut.tool_index_to_id + .get(&item.tool_index) + .cloned() + .unwrap_or_else(|| format!("call_{}", item.tool_index)) + }; + + json!({ + "id": id, + "type": "function", + "function": { + "name": item.name.unwrap_or_default(), + "arguments": item.parameters + } + }) + }) + .collect(); + + // Build result JSON + let result_json = json!({ + "normal_text": streaming_result.normal_text, + "tool_calls": openai_tool_calls + }); + + let result_str = match serde_json::to_string(&result_json) { + Ok(s) => s, + Err(e) => { + set_error_message(error_out, &format!("Failed to serialize JSON: {}", e)); + return SglErrorCode::ParsingError; + } + }; + + let result_cstr = match CString::new(result_str) { + Ok(s) => s, + Err(e) => { + set_error_message(error_out, &format!("Failed to create result string: {}", e)); + return SglErrorCode::MemoryError; + } + }; + + *result_json_out = result_cstr.into_raw(); + clear_error_message(error_out); + SglErrorCode::Success + } + Err(e) => { + set_error_message(error_out, &format!("Parse incremental error: {}", e)); + SglErrorCode::ParsingError + } + } +} + +/// Reset the parser state for reuse +#[no_mangle] +pub unsafe extern "C" fn sgl_tool_parser_reset(handle: *mut ToolParserHandle) { + if handle.is_null() { + return; + } + + let handle_ref = &mut *handle; + let parser = Arc::clone(&handle_ref.parser); + + // Reset parser state + RUNTIME.block_on(async { + let mut parser_guard = parser.lock().await; + parser_guard.reset(); + }); + + // Reset history count and tool index mapping + handle_ref.history_tool_calls_count = 0; + handle_ref.tool_index_to_id.clear(); +} + +/// Free a tool parser handle +#[no_mangle] +pub unsafe extern "C" fn sgl_tool_parser_free(handle: *mut ToolParserHandle) { + if !handle.is_null() { + let _ = Box::from_raw(handle); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/utils.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/utils.rs new file mode 100644 index 00000000..11bc82d9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/golang/src/utils.rs @@ -0,0 +1,44 @@ +//! Utility functions for FFI + +use uuid::Uuid; + +/// Helper function to generate tool call ID (matches router implementation) +pub fn generate_tool_call_id( + model: &str, + function_name: &str, + index: usize, + history_tool_calls_count: usize, +) -> String { + if model.to_lowercase().contains("kimi") { + // KimiK2 format: functions.{name}:{global_index} + format!("functions.{}:{}", function_name, history_tool_calls_count + index) + } else { + // Standard OpenAI format: call_{24-char-uuid} + format!("call_{}", &Uuid::new_v4().simple().to_string()[..24]) + } +} + +/// Generate tool constraints (placeholder implementation) +/// +/// # Arguments +/// * `tools_json` - JSON array of tools +/// * `tool_choice_json` - JSON object representing tool_choice +/// * `constraint_type_out` - Pointer to receive constraint type (e.g., "json_schema") +/// * `constraint_schema_out` - Pointer to receive constraint schema JSON +/// * `error_out` - Optional pointer to receive error message +/// +/// # Returns +/// * SglErrorCode::Success on success, error code on failure +#[no_mangle] +pub unsafe extern "C" fn sgl_generate_tool_constraints( + _tools_json: *const std::os::raw::c_char, + _tool_choice_json: *const std::os::raw::c_char, + _constraint_type_out: *mut *mut std::os::raw::c_char, + _constraint_schema_out: *mut *mut std::os::raw::c_char, + error_out: *mut *mut std::os::raw::c_char, +) -> super::error::SglErrorCode { + // Implementation would parse JSON and call generate_tool_constraints + // This is a placeholder + super::error::set_error_message(error_out, "Tool constraint generation not yet implemented in FFI"); + super::error::SglErrorCode::UnknownError +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/.coveragerc b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/.coveragerc new file mode 100644 index 00000000..cdadb9b5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/.coveragerc @@ -0,0 +1,9 @@ +[run] +source = sglang_router +omit = + */mini_lb.py + */cli.py + */__main__.py + +[report] +fail_under = 80 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/Cargo.toml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/Cargo.toml new file mode 100644 index 00000000..68f73fd1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "sgl-model-gateway-python" +version = "0.2.3" +edition = "2021" + +[lib] +name = "sglang_router_rs" +crate-type = ["cdylib"] + +[dependencies] +pyo3 = { version = "0.27.1", features = ["extension-module", "abi3-py38"] } +tokio = { version = "1.42.0", features = ["full"] } + +[dependencies.sgl-model-gateway] +path = "../.." +default-features = true + +[features] +default = ["pyo3/extension-module"] +vendored-openssl = ["sgl-model-gateway/vendored-openssl"] + +[profile.ci] +inherits = "release" +opt-level = 2 # Lighter optimization (still fast runtime, much faster compile) +lto = "thin" # Thin LTO - good balance +codegen-units = 16 # More parallelization for faster builds +strip = true diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/MANIFEST.in b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/MANIFEST.in new file mode 100644 index 00000000..6c65950f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/MANIFEST.in @@ -0,0 +1,9 @@ +# Must include: +include Cargo.toml # Python bindings Cargo configuration +include ../../Cargo.toml # Main Rust project configuration +include ../../build.rs # Build script for protobuf generation +include ../../LICENSE +recursive-include src *.rs # Python bindings wrapper +recursive-include ../../src *.rs # Main Rust source files +recursive-include ../../src/proto *.proto # Protobuf definitions +recursive-include sglang_router *.py # Python source files diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/README.md new file mode 100644 index 00000000..41848c45 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/README.md @@ -0,0 +1,71 @@ +# SGLang Model Gateway Python Bindings + +This directory contains the Python bindings for the SGLang Router, built using [maturin](https://github.com/PyO3/maturin) and [PyO3](https://github.com/PyO3/pyo3). + +## Directory Structure + +``` +bindings/python/ +├── src/ # Rust source code for Python bindings +│ └── lib.rs # PyO3 bindings implementation +├── sglang_router/ # Python source code +│ ├── __init__.py +│ ├── version.py +│ ├── launch_server.py +│ ├── launch_router.py +│ ├── router.py +│ ├── router_args.py +│ └── mini_lb.py +├── Cargo.toml # Rust package configuration for bindings +├── pyproject.toml # Python package configuration +├── setup.py # Setup configuration +├── MANIFEST.in # Package manifest +├── .coveragerc # Test coverage configuration +└── README.md # This file +``` + +## Building + +### Development Build + +```bash +# Install maturin +pip install maturin + +# Build and install in development mode +cd sgl-router/bindings/python +maturin develop --features vendored-openssl +``` + +### Production Build + +```bash +# Build wheel +cd sgl-router/bindings/python +maturin build --release --out dist --features vendored-openssl + +# Install the built wheel +pip install dist/sglang_router-*.whl +``` + +## Testing + +```bash +# Run Python tests +cd sgl-router +pytest py_test/ +``` + +## Configuration + +- **pyproject.toml**: Defines package metadata, dependencies, and build configuration +- **python-source**: Set to "." to indicate Python source is in the same directory as pyproject.toml +- **module-name**: `sglang_router.sglang_router_rs` - the Rust extension module name + +## Notes + +- The Rust bindings source code is located in `src/lib.rs` +- The bindings have their own `Cargo.toml` in this directory +- The main sglang-router library is located in `../../` and is used as a dependency +- The package includes both Python code and Rust extensions built with PyO3 +- PyO3 types are prefixed with `Py` in Rust but exposed to Python without the prefix using the `name` attribute diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/pyproject.toml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/pyproject.toml new file mode 100644 index 00000000..10b15249 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/pyproject.toml @@ -0,0 +1,54 @@ +[build-system] +requires = ["maturin>=1.0,<2.0"] +build-backend = "maturin" + +[project] +name = "sglang-router" +version = "0.2.3" +description = "High-performance Rust-based load balancer for SGLang with multiple routing algorithms and prefill-decode disaggregation support" +authors = [ + {name = "Simo Lin", email = "linsimo.mark@gmail.com"}, + {name = "Chang Su", email = "mckvtl@gmail.com"}, + {name = "Keyang Ru", email = "rukeyang@gmail.com"}, + {name = "Byron Hsu", email = "byronhsu1230@gmail.com"} +] +requires-python = ">=3.8" +readme = "../../README.md" +license = { text = "Apache-2.0" } +classifiers = [ + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Rust", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] + +dependencies = [ + "setproctitle", + "aiohttp", + "orjson", + "uvicorn", + "fastapi", +] + +[project.optional-dependencies] +dev = [ + "requests>=2.25.0", +] + +[project.scripts] +smg = "sglang_router.cli:main" +amg = "sglang_router.cli:main" +sglang-router = "sglang_router.cli:main" + + +[tool.maturin] +python-source = "." +module-name = "sglang_router.sglang_router_rs" +# Exclude bindings/python/README.md to use root README only +exclude = ["README.md"] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/setup.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/setup.py new file mode 100644 index 00000000..17d782a7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/setup.py @@ -0,0 +1,28 @@ +import os +import warnings + +from setuptools import setup + +with_rust = os.environ.get("SGLANG_ROUTER_BUILD_WITH_RUST", None) +with_rust = with_rust is None or (not with_rust.lower() in ["0", "false", "no"]) + +rust_extensions = [] +if with_rust: + from setuptools_rust import Binding, RustExtension + + rust_extensions.append( + RustExtension( + target="sglang_router_rs", + path="Cargo.toml", + binding=Binding.PyO3, + ) + ) +else: + warnings.warn( + "Building 'sglang-router' without Rust support. Performance may be degraded." + ) + +setup( + rust_extensions=rust_extensions, + zip_safe=False, +) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/__init__.py new file mode 100644 index 00000000..9c7fa208 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/__init__.py @@ -0,0 +1,3 @@ +from sglang_router.version import __version__ + +__all__ = ["__version__"] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/__main__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/__main__.py new file mode 100644 index 00000000..02211dc6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/__main__.py @@ -0,0 +1,8 @@ +""" +Allow running the CLI via: python -m sglang_router +""" + +from sglang_router.cli import main + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/cli.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/cli.py new file mode 100755 index 00000000..1447211b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/cli.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +""" +SGLang Model Gateway CLI + +Provides convenient command-line interface for launching the router and server. + +Usage: + smg launch [args] # Launch router only + smg server [args] # Launch router + server + smg --help # Show help +""" + +import argparse +import os +import sys +from typing import List, Optional + +from sglang_router.sglang_router_rs import ( + get_verbose_version_string, + get_version_string, +) + + +def create_parser() -> argparse.ArgumentParser: + """Create the main CLI parser with subcommands.""" + prog_name = os.path.basename(sys.argv[0]) if sys.argv else "smg" + parser = argparse.ArgumentParser( + prog=prog_name, + description="SGLang Model Gateway - High-performance inference router", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + subparsers = parser.add_subparsers(dest="command", help="Available commands") + + # Launch router subcommand + launch_parser = subparsers.add_parser( + "launch", + help="Launch router only (requires existing worker URLs)", + description="Launch the SGLang router with existing worker instances", + add_help=False, # Let router handle --help + ) + + # Launch server + router subcommand + server_parser = subparsers.add_parser( + "server", + help="Launch router and server processes together", + description="Launch both SGLang router and server processes", + add_help=False, # Let server handle --help + ) + + return parser + + +def main(argv: Optional[List[str]] = None) -> None: + """Main CLI entry point.""" + if argv is None: + argv = sys.argv[1:] + + # Handle version flags before parsing + if argv and argv[0] in ["--version", "-V", "--version-verbose"]: + if argv[0] == "--version-verbose": + print(get_verbose_version_string()) + else: + print(get_version_string()) + sys.exit(0) + + # Handle empty command - show help + if not argv or argv[0] not in ["launch", "server", "-h", "--help"]: + parser = create_parser() + parser.print_help() + sys.exit(1) + + parser = create_parser() + args, unknown = parser.parse_known_args(argv) + + if args.command == "launch": + # Import and call launch_router functions directly + from sglang_router.launch_router import launch_router, parse_router_args + + # All router args are in unknown + router_args = parse_router_args(unknown) + launch_router(router_args) + + elif args.command == "server": + # Import and call launch_server main with proper argv + # Note: launch_server.main() uses argparse internally which reads sys.argv + # We need to temporarily set sys.argv for compatibility + import sglang_router.launch_server as launch_server_module + + # Preserve original sys.argv + original_argv = sys.argv + try: + # All server args are in unknown + prog_name = os.path.basename(sys.argv[0]) if sys.argv else "smg" + sys.argv = [f"{prog_name} server"] + unknown + launch_server_module.main() + finally: + # Restore original sys.argv + sys.argv = original_argv + + else: + parser.print_help() + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/launch_router.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/launch_router.py new file mode 100644 index 00000000..8da5191c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/launch_router.py @@ -0,0 +1,113 @@ +import argparse +import logging +import sys +from typing import List, Optional + +import setproctitle +from sglang_router.mini_lb import MiniLoadBalancer +from sglang_router.router_args import RouterArgs + +logger = logging.getLogger("router") + +try: + from sglang_router.router import Router +except ImportError: + Router = None + logger.warning( + "Rust Router is not installed, only python MiniLB (debugging only) is available" + ) + + +def launch_router(args: argparse.Namespace) -> Optional[Router]: + """ + Launch the SGLang router with the configuration from parsed arguments. + + Args: + args: Namespace object containing router configuration + Can be either raw argparse.Namespace or converted RouterArgs + + Returns: + Router instance if successful, None if failed + """ + setproctitle.setproctitle("sglang::router") + try: + # Convert to RouterArgs if needed + if not isinstance(args, RouterArgs): + router_args = RouterArgs.from_cli_args(args) + else: + router_args = args + + if router_args.mini_lb: + mini_lb = MiniLoadBalancer(router_args) + mini_lb.start() + else: + # TODO: support tracing for router(Rust). + del router_args.enable_trace + del router_args.otlp_traces_endpoint + + if Router is None: + raise RuntimeError("Rust Router is not installed") + router_args._validate_router_args() + router = Router.from_args(router_args) + router.start() + + except Exception as e: + logger.error(f"Error starting router: {e}") + raise e + + +class CustomHelpFormatter( + argparse.RawDescriptionHelpFormatter, argparse.ArgumentDefaultsHelpFormatter +): + """Custom formatter that preserves both description formatting and shows defaults""" + + pass + + +def parse_router_args(args: List[str]) -> RouterArgs: + """Parse command line arguments and return RouterArgs instance.""" + parser = argparse.ArgumentParser( + description="""SGLang Router - High-performance request distribution across worker nodes + +Usage: +This launcher enables starting a router with individual worker instances. It is useful for +multi-node setups or when you want to start workers and router separately. + +Examples: + # Regular mode + python -m sglang_router.launch_router --worker-urls http://worker1:8000 http://worker2:8000 + + # PD disaggregated mode with same policy for both + python -m sglang_router.launch_router --pd-disaggregation \\ + --prefill http://prefill1:8000 9000 --prefill http://prefill2:8000 \\ + --decode http://decode1:8001 --decode http://decode2:8001 \\ + --policy cache_aware + + # PD mode with optional bootstrap ports + python -m sglang_router.launch_router --pd-disaggregation \\ + --prefill http://prefill1:8000 9000 \\ # With bootstrap port + --prefill http://prefill2:8000 none \\ # Explicitly no bootstrap port + --prefill http://prefill3:8000 \\ # Defaults to no bootstrap port + --decode http://decode1:8001 --decode http://decode2:8001 + + # PD mode with different policies for prefill and decode + python -m sglang_router.launch_router --pd-disaggregation \\ + --prefill http://prefill1:8000 --prefill http://prefill2:8000 \\ + --decode http://decode1:8001 --decode http://decode2:8001 \\ + --prefill-policy cache_aware --decode-policy power_of_two + + """, + formatter_class=CustomHelpFormatter, + ) + + RouterArgs.add_cli_args(parser, use_router_prefix=False) + return RouterArgs.from_cli_args(parser.parse_args(args), use_router_prefix=False) + + +def main() -> None: + router_args = parse_router_args(sys.argv[1:]) + launch_router(router_args) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/launch_server.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/launch_server.py new file mode 100644 index 00000000..fd72f950 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/launch_server.py @@ -0,0 +1,213 @@ +import argparse +import asyncio +import copy +import logging +import multiprocessing as mp +import os +import random +import signal +import sys +import time +from typing import List + +import requests +from setproctitle import setproctitle +from sglang_router.launch_router import RouterArgs, launch_router + +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import is_port_available + + +def setup_logger(): + logger = logging.getLogger("router") + logger.setLevel(logging.INFO) + + formatter = logging.Formatter( + "[Router (Python)] %(asctime)s - %(levelname)s - %(message)s - %(filename)s:%(lineno)d", + datefmt="%Y-%m-%d %H:%M:%S", + ) + + handler = logging.StreamHandler() + handler.setFormatter(formatter) + logger.addHandler(handler) + + return logger + + +logger = setup_logger() + + +# Create new process group +def run_server(server_args, dp_rank): + """ + Note: + + 1. Without os.setpgrp(), all processes share the same PGID. When you press Ctrl+C, the terminal sends SIGINT to all processes in the group simultaneously. + This can cause leaf processes to terminate first, which messes up the cleaning order and produces orphaned processes. + + Terminal (PGID=100) + └── Main Python Process (PGID=100) + └── Server Process 1 (PGID=100) + └── Scheduler 1 + └── Detokenizer 1 + └── Server Process 2 (PGID=100) + └── Scheduler 2 + └── Detokenizer 2 + + 2. With os.setpgrp(), the main Python process and its children are in a separate group. Now: + + Terminal (PGID=100) + └── Main Python Process (PGID=200) + └── Server Process 1 (PGID=300) + └── Scheduler 1 + └── Detokenizer 1 + └── Server Process 2 (PGID=400) + └── Scheduler 2 + └── Detokenizer 2 + """ + # create new process group + os.setpgrp() + + setproctitle("sglang::server") + # Set SGLANG_DP_RANK environment variable + os.environ["SGLANG_DP_RANK"] = str(dp_rank) + + # Launch server in appropriate mode (HTTP or gRPC) + if server_args.grpc_mode: + from sglang.srt.entrypoints.grpc_server import serve_grpc + + asyncio.run(serve_grpc(server_args)) + else: + from sglang.srt.entrypoints.http_server import launch_server + + launch_server(server_args) + + +def launch_server_process( + server_args: ServerArgs, worker_port: int, dp_id: int +) -> mp.Process: + """Launch a single server process with the given args and port.""" + server_args = copy.deepcopy(server_args) + server_args.port = worker_port + server_args.base_gpu_id = dp_id * server_args.tp_size + server_args.dp_size = 1 + + proc = mp.Process(target=run_server, args=(server_args, dp_id)) + proc.start() + return proc + + +def wait_for_server_health(host: str, port: int, timeout: int = 300) -> bool: + """Wait for server to be healthy by checking /health endpoint.""" + start_time = time.perf_counter() + url = f"http://{host}:{port}/health" + + while time.perf_counter() - start_time < timeout: + try: + response = requests.get(url, timeout=5) + if response.status_code == 200: + return True + except requests.exceptions.RequestException: + pass + time.sleep(1) + return False + + +def find_available_ports(base_port: int, count: int) -> List[int]: + """Find consecutive available ports starting from base_port.""" + available_ports = [] + current_port = base_port + + while len(available_ports) < count: + if is_port_available(current_port): + available_ports.append(current_port) + current_port += random.randint(100, 1000) + + return available_ports + + +def cleanup_processes(processes: List[mp.Process]): + for process in processes: + logger.info(f"Terminating process group {process.pid}") + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + # Process group may already be terminated + pass + + # Wait for processes to terminate + for process in processes: + process.join(timeout=5) + if process.is_alive(): + logger.warning( + f"Process {process.pid} did not terminate gracefully, forcing kill" + ) + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + logger.info("All process groups terminated") + + +def main(): + # CUDA runtime isn't fork-safe, which can lead to subtle bugs or crashes + mp.set_start_method("spawn") + + parser = argparse.ArgumentParser( + description="Launch SGLang router and server processes" + ) + + ServerArgs.add_cli_args(parser) + RouterArgs.add_cli_args(parser, use_router_prefix=True, exclude_host_port=True) + parser.add_argument( + "--router-dp-worker-base-port", + type=int, + default=31000, + help="Base port number for data parallel workers", + ) + # No extra retry/CB flags here; RouterArgs.add_cli_args already defines them with router- prefix + + args = parser.parse_args() + server_args = ServerArgs.from_cli_args(args) + router_args = RouterArgs.from_cli_args(args, use_router_prefix=True) + + # Find available ports for workers + worker_ports = find_available_ports( + args.router_dp_worker_base_port, server_args.dp_size + ) + + # Start server processes + server_processes = [] + + for i, worker_port in enumerate(worker_ports): + logger.info(f"Launching DP server process {i} on port {worker_port}") + proc = launch_server_process(server_args, worker_port, i) + server_processes.append(proc) + + signal.signal(signal.SIGINT, lambda sig, frame: cleanup_processes(server_processes)) + signal.signal( + signal.SIGTERM, lambda sig, frame: cleanup_processes(server_processes) + ) + signal.signal( + signal.SIGQUIT, lambda sig, frame: cleanup_processes(server_processes) + ) + + # Update router args with worker URLs + # Use grpc:// protocol if server is in gRPC mode, otherwise http:// + protocol = "grpc" if server_args.grpc_mode else "http" + router_args.worker_urls = [ + f"{protocol}://{server_args.host}:{port}" for port in worker_ports + ] + + # Start the router + try: + launch_router(router_args) + except Exception as e: + logger.error(f"Failed to start router: {e}") + cleanup_processes(server_processes) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/mini_lb.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/mini_lb.py new file mode 100644 index 00000000..7c3089b5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/mini_lb.py @@ -0,0 +1,488 @@ +""" +Minimal HTTP load balancer for prefill and decode servers for testing. +""" + +import asyncio +import ipaddress +import logging +import random +import urllib +from http import HTTPStatus +from itertools import chain +from typing import Optional + +import aiohttp +import orjson +import uvicorn +from fastapi import FastAPI, HTTPException +from fastapi.responses import ORJSONResponse, Response, StreamingResponse +from sglang_router.router_args import RouterArgs + +try: + from sglang.srt.tracing.trace import ( + process_tracing_init, + trace_get_remote_propagate_context, + trace_req_finish, + trace_req_start, + trace_set_thread_info, + trace_slice_end, + trace_slice_start, + ) + + trace_package_imported = True +except ImportError: + trace_package_imported = False + +logger = logging.getLogger(__name__) + +AIOHTTP_STREAM_READ_CHUNK_SIZE = ( + 1024 * 64 +) # 64KB, to prevent aiohttp's "Chunk too big" error + + +def maybe_wrap_ipv6_address(address: str) -> str: + try: + ipaddress.IPv6Address(address) + return f"[{address}]" + except ValueError: + return address + + +class MiniLoadBalancer: + def __init__( + self, + router_args: RouterArgs, + ): + self._validate_router_args(router_args) + + self.host = router_args.host + self.port = router_args.port + self.timeout = router_args.request_timeout_secs + self.prefill_urls = [url[0] for url in router_args.prefill_urls] + self.prefill_bootstrap_ports = [url[1] for url in router_args.prefill_urls] + self.decode_urls = router_args.decode_urls + self.otlp_traces_endpoint = router_args.otlp_traces_endpoint + self.enable_trace = router_args.enable_trace + if self.enable_trace and not trace_package_imported: + logger.warning( + "Tracing is not supported in this environment. Please install sglang." + ) + self.enable_trace = False + + def _validate_router_args(self, router_args: RouterArgs): + logger.warning( + "\x1b[33mMiniLB is only for debugging purposes, it only supports random policy!\033[0m" + ) + + # NOTE: too many arguments unsupported, just validate some important ones + if router_args.policy != "random": + logger.warning("[MiniLB] Overriding policy to random") + router_args.policy = "random" + + if not router_args.pd_disaggregation: + raise ValueError("MiniLB only supports PD disaggregation mode") + + if len(router_args.prefill_urls) == 0 or len(router_args.decode_urls) == 0: + raise ValueError( + "MiniLB requires at least one prefill and one decode server" + ) + + def start(self): + global lb + lb = self + if self.enable_trace: + process_tracing_init(self.otlp_traces_endpoint, "sglang") + trace_set_thread_info("Mini lb") + uvicorn.run(app, host=self.host, port=self.port) + + def select_pair(self): + assert len(self.prefill_urls) > 0, "No prefill servers available" + assert len(self.decode_urls) > 0, "No decode servers available" + pidx = random.randint(0, len(self.prefill_urls) - 1) + didx = random.randint(0, len(self.decode_urls) - 1) + return ( + self.prefill_urls[pidx], + self.prefill_bootstrap_ports[pidx], + self.decode_urls[didx], + ) + + async def generate( + self, modified_request, prefill_server, decode_server, endpoint + ) -> ORJSONResponse: + assert endpoint[0] != "/", f"Endpoint should not start with '/': {endpoint}" + + async with aiohttp.ClientSession( + timeout=aiohttp.ClientTimeout( + total=self.timeout + ) # Add timeout for request reliability + ) as session: + headers = {} + bootstrap_room_list = [] + if self.enable_trace: + bootstrap_room_list = ( + modified_request["bootstrap_room"] + if isinstance(modified_request["bootstrap_room"], list) + else [modified_request["bootstrap_room"]] + ) + trace_context = trace_get_remote_propagate_context(bootstrap_room_list) + headers = {"trace_context": trace_context} + + tasks = [ + session.post( + f"{prefill_server}/{endpoint}", + json=modified_request, + headers=headers, + ), + session.post( + f"{decode_server}/{endpoint}", + json=modified_request, + headers=headers, + ), + ] + + for bootstrap_room in bootstrap_room_list: + trace_slice_end("mini_lb_launch", bootstrap_room, auto_next_anon=True) + + # Wait for both responses to complete. Prefill should end first. + prefill_response, decode_response = await asyncio.gather(*tasks) + + if "return_logprob" in modified_request: + + prefill_json = await prefill_response.json() + ret_json = await decode_response.json() + + # merge `meta_info.input_token_logprobs` from prefill to decode + if "meta_info" in ret_json: + if "input_token_logprobs" in ret_json["meta_info"]: + ret_json["meta_info"]["input_token_logprobs"] = ( + prefill_json["meta_info"]["input_token_logprobs"] + + ret_json["meta_info"]["input_token_logprobs"] + ) + else: + ret_json = await decode_response.json() + + for bootstrap_room in bootstrap_room_list: + trace_slice_end( + "wait_PD_finish", + bootstrap_room, + thread_finish_flag=True, + ) + trace_req_finish(bootstrap_room) + + return ORJSONResponse( + content=ret_json, + status_code=decode_response.status, + ) + + async def generate_stream( + self, modified_request, prefill_server, decode_server, endpoint="generate" + ): + assert endpoint[0] != "/", f"Endpoint should not start with '/': {endpoint}" + + async def stream_results(): + async with aiohttp.ClientSession( + timeout=aiohttp.ClientTimeout( + total=self.timeout + ) # Add timeout for request reliability + ) as session: + # Create the tasks for both prefill and decode requests + headers = {} + bootstrap_room_list = [] + if self.enable_trace: + bootstrap_room_list = ( + modified_request["bootstrap_room"] + if isinstance(modified_request["bootstrap_room"], list) + else [modified_request["bootstrap_room"]] + ) + trace_context = trace_get_remote_propagate_context( + bootstrap_room_list + ) + headers = {"trace_context": trace_context} + + tasks = [ + session.post( + f"{prefill_server}/{endpoint}", + json=modified_request, + headers=headers, + ), + session.post( + f"{decode_server}/{endpoint}", + json=modified_request, + headers=headers, + ), + ] + + for bootstrap_room in bootstrap_room_list: + trace_slice_end( + "mini_lb_launch", bootstrap_room, auto_next_anon=True + ) + # Wait for both responses to complete. Since this is streaming, they return immediately. + prefill_response, decode_response = await asyncio.gather(*tasks) + + if modified_request.get("return_logprob", False): + prefill_chunks = [] + async for chunk in prefill_response.content: + prefill_chunks.append(chunk) + + first_prefill_chunk = ( + prefill_chunks[0].decode("utf-8")[5:].strip("\n") + ) + first_prefill_chunk_json = orjson.loads(first_prefill_chunk) + + async for chunk in decode_response.content: + # Note: This is inefficient + # merge prefill input_token_logprobs, output_token_logprobs to decode + decoded_chunk = chunk.decode("utf-8") + if ( + decoded_chunk + and decoded_chunk.startswith("data:") + and "[DONE]" not in decoded_chunk + ): + ret_json = orjson.loads(decoded_chunk[5:].strip("\n")) + ret_json["meta_info"]["input_token_logprobs"] = ( + first_prefill_chunk_json["meta_info"][ + "input_token_logprobs" + ] + + ret_json["meta_info"]["input_token_logprobs"] + ) + + yield b"data: " + orjson.dumps(ret_json) + b"\n\n" + else: + yield chunk + else: + async for chunk in decode_response.content.iter_chunked( + AIOHTTP_STREAM_READ_CHUNK_SIZE + ): + yield chunk + + for bootstrap_room in bootstrap_room_list: + trace_slice_end( + "wait_PD_finish", + bootstrap_room, + thread_finish_flag=True, + ) + trace_req_finish(bootstrap_room) + + return StreamingResponse( + stream_results(), + media_type="text/event-stream", + ) + + +app = FastAPI() +lb: Optional[MiniLoadBalancer] = None + + +@app.get("/health") +async def health_check(): + return Response(status_code=200) + + +@app.get("/health_generate") +async def health_generate(): + async with aiohttp.ClientSession() as session: + # Create the tasks + tasks = [] + for server in chain(lb.prefill_urls, lb.decode_urls): + tasks.append(session.get(f"{server}/health_generate")) + for i, response in enumerate(asyncio.as_completed(tasks)): + await response + return Response(status_code=200) + + +@app.post("/flush_cache") +async def flush_cache(): + async with aiohttp.ClientSession() as session: + # Create the tasks + tasks = [] + for server in chain(lb.prefill_urls, lb.decode_urls): + tasks.append(session.post(f"{server}/flush_cache")) + for i, response in enumerate(asyncio.as_completed(tasks)): + await response + return Response(status_code=200) + + +@app.get("/get_server_info") +async def get_server_info(): + prefill_infos = [] + decode_infos = [] + all_internal_states = [] + + async with aiohttp.ClientSession() as session: + for server in lb.prefill_urls: + server_info = await session.get(f"{server}/get_server_info") + prefill_infos.append(await server_info.json()) + for server in lb.decode_urls: + server_info = await session.get(f"{server}/get_server_info") + info_json = await server_info.json() + decode_infos.append(info_json) + # Extract internal_states from decode servers + if "internal_states" in info_json: + all_internal_states.extend(info_json["internal_states"]) + + # Return format expected by bench_one_batch_server.py + if all_internal_states: + return { + "internal_states": all_internal_states, + "prefill": prefill_infos, + "decode": decode_infos, + } + else: + # Fallback with dummy data if no internal states found + return { + "internal_states": [ + { + "last_gen_throughput": 0.0, + "avg_spec_accept_length": None, + } + ], + "prefill": prefill_infos, + "decode": decode_infos, + } + + +@app.get("/get_model_info") +async def get_model_info(): + if not lb or not lb.prefill_urls: + raise HTTPException( + status_code=HTTPStatus.SERVICE_UNAVAILABLE, + detail="There is no server registered", + ) + + target_server_url = lb.prefill_urls[0] + endpoint_url = f"{target_server_url}/get_model_info" + + async with aiohttp.ClientSession() as session: + try: + async with session.get(endpoint_url) as response: + if response.status != 200: + error_text = await response.text() + raise HTTPException( + status_code=HTTPStatus.BAD_GATEWAY, + detail=( + f"Failed to get model info from {target_server_url}" + f"Status: {response.status}, Response: {error_text}" + ), + ) + + model_info_json = await response.json() + return ORJSONResponse(content=model_info_json) + + except aiohttp.ClientError as e: + raise HTTPException( + status_code=HTTPStatus.SERVICE_UNAVAILABLE, + detail=f"Failed to get model info from backend", + ) + + +@app.post("/generate") +async def handle_generate_request(request_data: dict): + prefill_server, bootstrap_port, decode_server = lb.select_pair() + + # Parse and transform prefill_server for bootstrap data + parsed_url = urllib.parse.urlparse(prefill_server) + hostname = maybe_wrap_ipv6_address(parsed_url.hostname) + modified_request = request_data.copy() + + batch_size = _get_request_batch_size(modified_request) + if batch_size is not None: + modified_request.update( + { + "bootstrap_host": [hostname] * batch_size, + "bootstrap_port": [bootstrap_port] * batch_size, + "bootstrap_room": [ + _generate_bootstrap_room() for _ in range(batch_size) + ], + } + ) + else: + modified_request.update( + { + "bootstrap_host": hostname, + "bootstrap_port": bootstrap_port, + "bootstrap_room": _generate_bootstrap_room(), + } + ) + + if request_data.get("stream", False): + return await lb.generate_stream( + modified_request, prefill_server, decode_server, "generate" + ) + else: + return await lb.generate( + modified_request, prefill_server, decode_server, "generate" + ) + + +async def _forward_to_backend(request_data: dict, endpoint_name: str): + prefill_server, bootstrap_port, decode_server = lb.select_pair() + + # Parse and transform prefill_server for bootstrap data + parsed_url = urllib.parse.urlparse(prefill_server) + hostname = maybe_wrap_ipv6_address(parsed_url.hostname) + modified_request = request_data.copy() + modified_request.update( + { + "bootstrap_host": hostname, + "bootstrap_port": bootstrap_port, + "bootstrap_room": _generate_bootstrap_room(), + } + ) + + if request_data.get("stream", False): + return await lb.generate_stream( + modified_request, + prefill_server, + decode_server, + endpoint=endpoint_name, + ) + else: + return await lb.generate( + modified_request, + prefill_server, + decode_server, + endpoint=endpoint_name, + ) + + +@app.post("/v1/chat/completions") +async def handle_chat_completion_request(request_data: dict): + return await _forward_to_backend(request_data, "v1/chat/completions") + + +@app.post("/v1/completions") +async def handle_completion_request(request_data: dict): + return await _forward_to_backend(request_data, "v1/completions") + + +def _generate_bootstrap_room(): + bootstrap_room = random.randint(0, 2**63 - 1) + if lb.enable_trace: + trace_req_start(bootstrap_room, bootstrap_room, role="router") + trace_slice_start("mini_lb_launch", bootstrap_room) + return bootstrap_room + + +# We may utilize `GenerateReqInput`'s logic later +def _get_request_batch_size(request): + if (text := request.get("text")) is not None: + return None if isinstance(text, str) else len(text) + if (input_ids := request.get("input_ids")) is not None: + return None if isinstance(input_ids[0], int) else len(input_ids) + return None + + +@app.get("/v1/models") +async def get_models(): + prefill_server = lb.prefill_urls[0] # Get the first prefill server + async with aiohttp.ClientSession() as session: + try: + response = await session.get(f"{prefill_server}/v1/models") + if response.status != 200: + raise HTTPException( + status_code=response.status, + detail=f"Prefill server error: Status {response.status}", + ) + return ORJSONResponse(content=await response.json()) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/router.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/router.py new file mode 100644 index 00000000..05506e1c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/router.py @@ -0,0 +1,226 @@ +from typing import Optional + +from sglang_router.router_args import RouterArgs +from sglang_router.sglang_router_rs import ( + BackendType, + HistoryBackendType, + PolicyType, + PyOracleConfig, + PyPostgresConfig, +) +from sglang_router.sglang_router_rs import Router as _Router + + +def policy_from_str(policy_str: Optional[str]) -> PolicyType: + """Convert policy string to PolicyType enum.""" + if policy_str is None: + return None + policy_map = { + "random": PolicyType.Random, + "round_robin": PolicyType.RoundRobin, + "cache_aware": PolicyType.CacheAware, + "power_of_two": PolicyType.PowerOfTwo, + "bucket": PolicyType.Bucket, + } + return policy_map[policy_str] + + +def backend_from_str(backend_str: Optional[str]) -> BackendType: + """Convert backend string to BackendType enum.""" + if isinstance(backend_str, BackendType): + return backend_str + if backend_str is None: + return BackendType.Sglang + backend_map = {"sglang": BackendType.Sglang, "openai": BackendType.Openai} + backend_lower = backend_str.lower() + if backend_lower not in backend_map: + raise ValueError( + f"Unknown backend: {backend_str}. Valid options: {', '.join(backend_map.keys())}" + ) + return backend_map[backend_lower] + + +def history_backend_from_str(backend_str: Optional[str]) -> HistoryBackendType: + """Convert history backend string to HistoryBackendType enum.""" + if isinstance(backend_str, HistoryBackendType): + return backend_str + if backend_str is None: + return HistoryBackendType.Memory + backend_lower = backend_str.lower() + if backend_lower == "memory": + return HistoryBackendType.Memory + elif backend_lower == "none": + # Use getattr to access 'None' which is a Python keyword + return getattr(HistoryBackendType, "None") + elif backend_lower == "oracle": + return HistoryBackendType.Oracle + elif backend_lower == "postgres": + return HistoryBackendType.Postgres + else: + raise ValueError(f"Unknown history backend: {backend_str}") + + +class Router: + """ + A high-performance router for distributing requests across worker nodes. + + Args: + worker_urls: List of URLs for worker nodes that will handle requests. Each URL should include + the protocol, host, and port (e.g., ['http://worker1:8000', 'http://worker2:8000']) + policy: Load balancing policy to use. Options: + - PolicyType.Random: Randomly select workers + - PolicyType.RoundRobin: Distribute requests in round-robin fashion + - PolicyType.CacheAware: Distribute requests based on cache state and load balance + - PolicyType.PowerOfTwo: Select best of two random workers based on load (PD mode only) + host: Host address to bind the router server. Supports IPv4, IPv6 (e.g., ::, ::1), or 0.0.0.0 for all interfaces. Default: '0.0.0.0' + port: Port number to bind the router server. Default: 3001 + worker_startup_timeout_secs: Timeout in seconds for worker startup and registration. Large models can take significant time to load into GPU memory. Default: 1800 (30 minutes) + worker_startup_check_interval: Interval in seconds between checks for worker initialization. Default: 10 + cache_threshold: Cache threshold (0.0-1.0) for cache-aware routing. Routes to cached worker + if the match rate exceeds threshold, otherwise routes to the worker with the smallest + tree. Default: 0.5 + balance_abs_threshold: Load balancing is triggered when (max_load - min_load) > abs_threshold + AND max_load > min_load * rel_threshold. Otherwise, use cache aware. Default: 32 + balance_rel_threshold: Load balancing is triggered when (max_load - min_load) > abs_threshold + AND max_load > min_load * rel_threshold. Otherwise, use cache aware. Default: 1.0001 + eviction_interval_secs: Interval in seconds between cache eviction operations in cache-aware + routing. Default: 60 + max_payload_size: Maximum payload size in bytes. Default: 256MB + max_tree_size: Maximum size of the approximation tree for cache-aware routing. Default: 2^24 + dp_aware: Enable data parallelism aware schedule. Default: False + enable_igw: Enable IGW (Inference-Gateway) mode for multi-model support. When enabled, + the router can manage multiple models simultaneously with per-model load balancing + policies. Default: False + api_key: The api key used for the authorization with the worker. + Useful when the dp aware scheduling strategy is enabled. + Default: None + log_dir: Directory to store log files. If None, logs are only output to console. Default: None + log_level: Logging level. Options: 'debug', 'info', 'warn', 'error'. + service_discovery: Enable Kubernetes service discovery. When enabled, the router will + automatically discover worker pods based on the selector. Default: False + selector: Dictionary mapping of label keys to values for Kubernetes pod selection. + Example: {"app": "sglang-worker"}. Default: {} + service_discovery_port: Port to use for service discovery. The router will generate + worker URLs using this port. Default: 80 + service_discovery_namespace: Kubernetes namespace to watch for pods. If not provided, + watches pods across all namespaces (requires cluster-wide permissions). Default: None + prefill_selector: Dictionary mapping of label keys to values for Kubernetes pod selection + for prefill servers (PD mode only). Default: {} + decode_selector: Dictionary mapping of label keys to values for Kubernetes pod selection + for decode servers (PD mode only). Default: {} + prometheus_port: Port to expose Prometheus metrics. Default: None + prometheus_host: Host address to bind the Prometheus metrics server. Default: None + pd_disaggregation: Enable PD (Prefill-Decode) disaggregated mode. Default: False + prefill_urls: List of (url, bootstrap_port) tuples for prefill servers (PD mode only) + decode_urls: List of URLs for decode servers (PD mode only) + prefill_policy: Specific load balancing policy for prefill nodes (PD mode only). + If not specified, uses the main policy. Default: None + decode_policy: Specific load balancing policy for decode nodes (PD mode only). + If not specified, uses the main policy. Default: None + request_id_headers: List of HTTP headers to check for request IDs. If not specified, + uses common defaults: ['x-request-id', 'x-correlation-id', 'x-trace-id', 'request-id']. + Example: ['x-my-request-id', 'x-custom-trace-id']. Default: None + bootstrap_port_annotation: Kubernetes annotation name for bootstrap port (PD mode). + Default: 'sglang.ai/bootstrap-port' + request_timeout_secs: Request timeout in seconds. Default: 600 + max_concurrent_requests: Maximum number of concurrent requests allowed for rate limiting. Default: 256 + queue_size: Queue size for pending requests when max concurrent limit reached (0 = no queue, return 429 immediately). Default: 100 + queue_timeout_secs: Maximum time (in seconds) a request can wait in queue before timing out. Default: 60 + rate_limit_tokens_per_second: Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests. Default: None + cors_allowed_origins: List of allowed origins for CORS. Empty list allows all origins. Default: [] + health_failure_threshold: Number of consecutive health check failures before marking worker unhealthy. Default: 3 + health_success_threshold: Number of consecutive health check successes before marking worker healthy. Default: 2 + health_check_timeout_secs: Timeout in seconds for health check requests. Default: 5 + health_check_interval_secs: Interval in seconds between runtime health checks. Default: 60 + health_check_endpoint: Health check endpoint path. Default: '/health' + model_path: Model path for loading tokenizer (HuggingFace model ID or local path). Default: None + tokenizer_path: Explicit tokenizer path (overrides model_path tokenizer if provided). Default: None + """ + + def __init__(self, router: _Router): + self._router = router + + @staticmethod + def from_args(args: RouterArgs) -> "Router": + """Create a router from a RouterArgs instance.""" + + args_dict = vars(args) + # Convert RouterArgs to _Router parameters + args_dict["worker_urls"] = ( + [] + if args_dict["service_discovery"] or args_dict["pd_disaggregation"] + else args_dict["worker_urls"] + ) + args_dict["policy"] = policy_from_str(args_dict["policy"]) + args_dict["prefill_urls"] = ( + args_dict["prefill_urls"] if args_dict["pd_disaggregation"] else None + ) + args_dict["decode_urls"] = ( + args_dict["decode_urls"] if args_dict["pd_disaggregation"] else None + ) + args_dict["prefill_policy"] = policy_from_str(args_dict["prefill_policy"]) + args_dict["decode_policy"] = policy_from_str(args_dict["decode_policy"]) + + # Convert backend + args_dict["backend"] = backend_from_str(args_dict.get("backend")) + + # Convert history_backend to enum first + history_backend_raw = args_dict.get("history_backend", "memory") + history_backend = history_backend_from_str(history_backend_raw) + + # Convert Oracle config if needed + oracle_config = None + if history_backend == HistoryBackendType.Oracle: + # Prioritize TNS alias over connect descriptor + tns_alias = args_dict.get("oracle_tns_alias") + connect_descriptor = args_dict.get("oracle_connect_descriptor") + + # Use TNS alias if provided, otherwise use connect descriptor + final_descriptor = tns_alias if tns_alias else connect_descriptor + + oracle_config = PyOracleConfig( + password=args_dict.get("oracle_password"), + username=args_dict.get("oracle_username"), + connect_descriptor=final_descriptor, + wallet_path=args_dict.get("oracle_wallet_path"), + pool_min=args_dict.get("oracle_pool_min", 1), + pool_max=args_dict.get("oracle_pool_max", 16), + pool_timeout_secs=args_dict.get("oracle_pool_timeout_secs", 30), + ) + args_dict["oracle_config"] = oracle_config + args_dict["history_backend"] = history_backend + + # Convert Postgres config if needed + postgres_config = None + if history_backend == HistoryBackendType.Postgres: + postgres_config = PyPostgresConfig( + db_url=args_dict.get("postgres_db_url"), + pool_max=args_dict.get("postgres_pool_max", 16), + ) + args_dict["postgres_config"] = postgres_config + + # Remove fields that shouldn't be passed to Rust Router constructor + fields_to_remove = [ + "mini_lb", + "oracle_wallet_path", + "oracle_tns_alias", + "oracle_connect_descriptor", + "oracle_username", + "oracle_password", + "oracle_pool_min", + "oracle_pool_max", + "oracle_pool_timeout_secs", + "postgres_db_url", + "postgres_pool_max", + ] + for field in fields_to_remove: + args_dict.pop(field, None) + + return Router(_Router(**args_dict)) + + def start(self) -> None: + """Start the router server. + + This method blocks until the server is shut down. + """ + self._router.start() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/router_args.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/router_args.py new file mode 100644 index 00000000..f950fda2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/router_args.py @@ -0,0 +1,778 @@ +import argparse +import dataclasses +import logging +import os +from typing import Dict, List, Optional + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass +class RouterArgs: + # Worker configuration + worker_urls: List[str] = dataclasses.field(default_factory=list) + host: str = "0.0.0.0" + port: int = 30000 + + # PD-specific configuration + mini_lb: bool = False + pd_disaggregation: bool = False # Enable PD disaggregated mode + prefill_urls: List[tuple] = dataclasses.field( + default_factory=list + ) # List of (url, bootstrap_port) + decode_urls: List[str] = dataclasses.field(default_factory=list) + + # Routing policy + policy: str = "cache_aware" + prefill_policy: Optional[str] = None # Specific policy for prefill nodes in PD mode + decode_policy: Optional[str] = None # Specific policy for decode nodes in PD mode + worker_startup_timeout_secs: int = 1800 + worker_startup_check_interval: int = 30 + cache_threshold: float = 0.3 + balance_abs_threshold: int = 64 + balance_rel_threshold: float = 1.5 + eviction_interval_secs: int = 120 + max_tree_size: int = 2**26 + max_payload_size: int = 512 * 1024 * 1024 # 512MB default for large batches + bucket_adjust_interval_secs: int = 5 + dp_aware: bool = False + enable_igw: bool = False # Enable IGW (Inter-Gateway) mode for multi-model support + api_key: Optional[str] = None + log_dir: Optional[str] = None + log_level: Optional[str] = None + # Service discovery configuration + service_discovery: bool = False + selector: Dict[str, str] = dataclasses.field(default_factory=dict) + service_discovery_port: int = 80 + service_discovery_namespace: Optional[str] = None + # PD service discovery configuration + prefill_selector: Dict[str, str] = dataclasses.field(default_factory=dict) + decode_selector: Dict[str, str] = dataclasses.field(default_factory=dict) + bootstrap_port_annotation: str = "sglang.ai/bootstrap-port" + # Prometheus configuration + prometheus_port: Optional[int] = None + prometheus_host: Optional[str] = None + # Request ID headers configuration + request_id_headers: Optional[List[str]] = None + # Request timeout in seconds + request_timeout_secs: int = 1800 + # Max concurrent requests for rate limiting (-1 to disable) + max_concurrent_requests: int = -1 + # Queue size for pending requests when max concurrent limit reached + queue_size: int = 100 + # Maximum time (in seconds) a request can wait in queue before timing out + queue_timeout_secs: int = 60 + # Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests + rate_limit_tokens_per_second: Optional[int] = None + # CORS allowed origins + cors_allowed_origins: List[str] = dataclasses.field(default_factory=list) + # Retry configuration + retry_max_retries: int = 5 + retry_initial_backoff_ms: int = 50 + retry_max_backoff_ms: int = 30_000 + retry_backoff_multiplier: float = 1.5 + retry_jitter_factor: float = 0.2 + disable_retries: bool = False + # Health check configuration + health_failure_threshold: int = 3 + health_success_threshold: int = 2 + health_check_timeout_secs: int = 5 + health_check_interval_secs: int = 60 + health_check_endpoint: str = "/health" + # Circuit breaker configuration + cb_failure_threshold: int = 10 + cb_success_threshold: int = 3 + cb_timeout_duration_secs: int = 60 + cb_window_duration_secs: int = 120 + disable_circuit_breaker: bool = False + model_path: Optional[str] = None + tokenizer_path: Optional[str] = None + chat_template: Optional[str] = None + # Tokenizer cache configuration + tokenizer_cache_enable_l0: bool = False + tokenizer_cache_l0_max_entries: int = 10000 + tokenizer_cache_enable_l1: bool = False + tokenizer_cache_l1_max_memory: int = 50 * 1024 * 1024 # 50MB + reasoning_parser: Optional[str] = None + tool_call_parser: Optional[str] = None + # MCP server configuration + mcp_config_path: Optional[str] = None + # Backend selection + backend: str = "sglang" + # History backend configuration + history_backend: str = "memory" + oracle_wallet_path: Optional[str] = None + oracle_tns_alias: Optional[str] = None + oracle_connect_descriptor: Optional[str] = None + oracle_username: Optional[str] = None + oracle_password: Optional[str] = None + oracle_pool_min: int = 1 + oracle_pool_max: int = 16 + oracle_pool_timeout_secs: int = 30 + postgres_db_url: Optional[str] = None + postgres_pool_max: int = 16 + # mTLS configuration for worker communication + client_cert_path: Optional[str] = None + client_key_path: Optional[str] = None + ca_cert_paths: List[str] = dataclasses.field(default_factory=list) + # Trace + enable_trace: bool = False + otlp_traces_endpoint: str = "localhost:4317" + + @staticmethod + def add_cli_args( + parser: argparse.ArgumentParser, + use_router_prefix: bool = False, + exclude_host_port: bool = False, + ): + """ + Add router-specific arguments to an argument parser. + + Args: + parser: The argument parser to add arguments to + use_router_prefix: If True, prefix all arguments with 'router-' to avoid conflicts + exclude_host_port: If True, don't add host and port arguments (used when inheriting from server) + """ + prefix = "router-" if use_router_prefix else "" + + # Worker configuration + if not exclude_host_port: + parser.add_argument( + "--host", + type=str, + default=RouterArgs.host, + help="Host address to bind the router server. Supports IPv4, IPv6 (e.g., ::, ::1), or 0.0.0.0 for all interfaces", + ) + parser.add_argument( + "--port", + type=int, + default=RouterArgs.port, + help="Port number to bind the router server", + ) + + parser.add_argument( + "--worker-urls", + type=str, + nargs="*", + default=[], + help="List of worker URLs. Supports IPv4 and IPv6 addresses (use brackets for IPv6, e.g., http://[::1]:8000 http://192.168.1.1:8000)", + ) + + # Routing policy configuration + parser.add_argument( + f"--{prefix}policy", + type=str, + default=RouterArgs.policy, + choices=["random", "round_robin", "cache_aware", "power_of_two"], + help="Load balancing policy to use. In PD mode, this is used for both prefill and decode unless overridden", + ) + parser.add_argument( + f"--{prefix}prefill-policy", + type=str, + default=None, + choices=["random", "round_robin", "cache_aware", "power_of_two", "bucket"], + help="Specific policy for prefill nodes in PD mode. If not specified, uses the main policy", + ) + parser.add_argument( + f"--{prefix}decode-policy", + type=str, + default=None, + choices=["random", "round_robin", "cache_aware", "power_of_two"], + help="Specific policy for decode nodes in PD mode. If not specified, uses the main policy", + ) + + # PD-specific arguments + parser.add_argument( + f"--{prefix}mini-lb", + action="store_true", + help="Enable MiniLB", + ) + parser.add_argument( + f"--{prefix}pd-disaggregation", + action="store_true", + help="Enable PD (Prefill-Decode) disaggregated mode", + ) + parser.add_argument( + f"--{prefix}prefill", + nargs="+", + action="append", + help="Prefill server URL and optional bootstrap port. Can be specified multiple times. " + "Format: --prefill URL [BOOTSTRAP_PORT]. " + "BOOTSTRAP_PORT can be a port number, 'none', or omitted (defaults to none).", + ) + parser.add_argument( + f"--{prefix}decode", + nargs=1, + action="append", + metavar=("URL",), + help="Decode server URL. Can be specified multiple times.", + ) + parser.add_argument( + f"--{prefix}worker-startup-timeout-secs", + type=int, + default=RouterArgs.worker_startup_timeout_secs, + help="Timeout in seconds for worker startup and registration (default: 1800 / 30 minutes). Large models can take significant time to load into GPU memory.", + ) + parser.add_argument( + f"--{prefix}worker-startup-check-interval", + type=int, + default=RouterArgs.worker_startup_check_interval, + help="Interval in seconds between checks for worker startup", + ) + parser.add_argument( + f"--{prefix}cache-threshold", + type=float, + default=RouterArgs.cache_threshold, + help="Cache threshold (0.0-1.0) for cache-aware routing", + ) + parser.add_argument( + f"--{prefix}balance-abs-threshold", + type=int, + default=RouterArgs.balance_abs_threshold, + help="Load balancing is triggered when (max_load - min_load) > abs_threshold AND max_load > min_load * rel_threshold. Otherwise, use cache aware", + ) + parser.add_argument( + f"--{prefix}balance-rel-threshold", + type=float, + default=RouterArgs.balance_rel_threshold, + help="Load balancing is triggered when (max_load - min_load) > abs_threshold AND max_load > min_load * rel_threshold. Otherwise, use cache aware", + ) + parser.add_argument( + f"--{prefix}bucket-adjust-interval-secs", + type=int, + default=RouterArgs.bucket_adjust_interval_secs, + help="Interval in seconds between bucket boundary adjustment operations", + ) + parser.add_argument( + f"--{prefix}eviction-interval-secs", + type=int, + default=RouterArgs.eviction_interval_secs, + help="Interval in seconds between cache eviction operations", + ) + parser.add_argument( + f"--{prefix}max-tree-size", + type=int, + default=RouterArgs.max_tree_size, + help="Maximum size of the approximation tree for cache-aware routing", + ) + parser.add_argument( + f"--{prefix}max-payload-size", + type=int, + default=RouterArgs.max_payload_size, + help="Maximum payload size in bytes", + ) + parser.add_argument( + f"--{prefix}dp-aware", + action="store_true", + help="Enable data parallelism aware schedule", + ) + parser.add_argument( + f"--{prefix}enable-igw", + action="store_true", + help="Enable IGW (Inference-Gateway) mode for multi-model support", + ) + parser.add_argument( + f"--{prefix}api-key", + type=str, + default=None, + help="The api key used for the authorization with the worker. Useful when the dp aware scheduling strategy is enaled.", + ) + parser.add_argument( + f"--{prefix}log-dir", + type=str, + default=None, + help="Directory to store log files. If not specified, logs are only output to console.", + ) + parser.add_argument( + f"--{prefix}log-level", + type=str, + default="info", + choices=["debug", "info", "warn", "error"], + help="Set the logging level. If not specified, defaults to INFO.", + ) + parser.add_argument( + f"--{prefix}service-discovery", + action="store_true", + help="Enable Kubernetes service discovery", + ) + parser.add_argument( + f"--{prefix}selector", + type=str, + nargs="+", + default={}, + help="Label selector for Kubernetes service discovery (format: key1=value1 key2=value2)", + ) + parser.add_argument( + f"--{prefix}service-discovery-port", + type=int, + default=RouterArgs.service_discovery_port, + help="Port to use for discovered worker pods", + ) + parser.add_argument( + f"--{prefix}service-discovery-namespace", + type=str, + help="Kubernetes namespace to watch for pods. If not provided, watches all namespaces (requires cluster-wide permissions)", + ) + parser.add_argument( + f"--{prefix}prefill-selector", + type=str, + nargs="+", + default={}, + help="Label selector for prefill server pods in PD mode (format: key1=value1 key2=value2)", + ) + parser.add_argument( + f"--{prefix}decode-selector", + type=str, + nargs="+", + default={}, + help="Label selector for decode server pods in PD mode (format: key1=value1 key2=value2)", + ) + # Prometheus configuration + parser.add_argument( + f"--{prefix}prometheus-port", + type=int, + default=29000, + help="Port to expose Prometheus metrics. If not specified, Prometheus metrics are disabled", + ) + parser.add_argument( + f"--{prefix}prometheus-host", + type=str, + default="0.0.0.0", + help="Host address to bind the Prometheus metrics server. Supports IPv4, IPv6 (e.g., ::, ::1), or 0.0.0.0 for all interfaces", + ) + parser.add_argument( + f"--{prefix}request-id-headers", + type=str, + nargs="*", + help="Custom HTTP headers to check for request IDs (e.g., x-request-id x-trace-id). If not specified, uses common defaults.", + ) + parser.add_argument( + f"--{prefix}request-timeout-secs", + type=int, + default=RouterArgs.request_timeout_secs, + help="Request timeout in seconds", + ) + # Retry configuration + parser.add_argument( + f"--{prefix}retry-max-retries", + type=int, + default=RouterArgs.retry_max_retries, + ) + parser.add_argument( + f"--{prefix}retry-initial-backoff-ms", + type=int, + default=RouterArgs.retry_initial_backoff_ms, + ) + parser.add_argument( + f"--{prefix}retry-max-backoff-ms", + type=int, + default=RouterArgs.retry_max_backoff_ms, + ) + parser.add_argument( + f"--{prefix}retry-backoff-multiplier", + type=float, + default=RouterArgs.retry_backoff_multiplier, + ) + parser.add_argument( + f"--{prefix}retry-jitter-factor", + type=float, + default=RouterArgs.retry_jitter_factor, + ) + parser.add_argument( + f"--{prefix}disable-retries", + action="store_true", + help="Disable retries (equivalent to setting retry_max_retries=1)", + ) + # Circuit breaker configuration + parser.add_argument( + f"--{prefix}cb-failure-threshold", + type=int, + default=RouterArgs.cb_failure_threshold, + ) + parser.add_argument( + f"--{prefix}cb-success-threshold", + type=int, + default=RouterArgs.cb_success_threshold, + ) + parser.add_argument( + f"--{prefix}cb-timeout-duration-secs", + type=int, + default=RouterArgs.cb_timeout_duration_secs, + ) + parser.add_argument( + f"--{prefix}cb-window-duration-secs", + type=int, + default=RouterArgs.cb_window_duration_secs, + ) + parser.add_argument( + f"--{prefix}disable-circuit-breaker", + action="store_true", + help="Disable circuit breaker (equivalent to setting cb_failure_threshold to u32::MAX)", + ) + # Health check configuration + parser.add_argument( + f"--{prefix}health-failure-threshold", + type=int, + default=RouterArgs.health_failure_threshold, + help="Number of consecutive health check failures before marking worker unhealthy", + ) + parser.add_argument( + f"--{prefix}health-success-threshold", + type=int, + default=RouterArgs.health_success_threshold, + help="Number of consecutive health check successes before marking worker healthy", + ) + parser.add_argument( + f"--{prefix}health-check-timeout-secs", + type=int, + default=RouterArgs.health_check_timeout_secs, + help="Timeout in seconds for health check requests", + ) + parser.add_argument( + f"--{prefix}health-check-interval-secs", + type=int, + default=RouterArgs.health_check_interval_secs, + help="Interval in seconds between runtime health checks", + ) + parser.add_argument( + f"--{prefix}health-check-endpoint", + type=str, + default=RouterArgs.health_check_endpoint, + help="Health check endpoint path", + ) + parser.add_argument( + f"--{prefix}max-concurrent-requests", + type=int, + default=RouterArgs.max_concurrent_requests, + help="Maximum number of concurrent requests allowed (for rate limiting). Set to -1 to disable rate limiting.", + ) + parser.add_argument( + f"--{prefix}queue-size", + type=int, + default=RouterArgs.queue_size, + help="Queue size for pending requests when max concurrent limit reached (0 = no queue, return 429 immediately)", + ) + parser.add_argument( + f"--{prefix}queue-timeout-secs", + type=int, + default=RouterArgs.queue_timeout_secs, + help="Maximum time (in seconds) a request can wait in queue before timing out", + ) + parser.add_argument( + f"--{prefix}rate-limit-tokens-per-second", + type=int, + default=RouterArgs.rate_limit_tokens_per_second, + help="Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests", + ) + parser.add_argument( + f"--{prefix}cors-allowed-origins", + type=str, + nargs="*", + default=[], + help="CORS allowed origins (e.g., http://localhost:3000 https://example.com)", + ) + # Tokenizer configuration + parser.add_argument( + f"--{prefix}model-path", + type=str, + default=None, + help="Model path for loading tokenizer (HuggingFace model ID or local path)", + ) + parser.add_argument( + f"--{prefix}tokenizer-path", + type=str, + default=None, + help="Explicit tokenizer path (overrides model_path tokenizer if provided)", + ) + parser.add_argument( + f"--{prefix}chat-template", + type=str, + default=None, + help="Chat template path (optional)", + ) + parser.add_argument( + f"--{prefix}tokenizer-cache-enable-l0", + action="store_true", + default=RouterArgs.tokenizer_cache_enable_l0, + help="Enable L0 (whole-string exact match) tokenizer cache (default: False)", + ) + parser.add_argument( + f"--{prefix}tokenizer-cache-l0-max-entries", + type=int, + default=RouterArgs.tokenizer_cache_l0_max_entries, + help="Maximum number of entries in L0 tokenizer cache (default: 10000)", + ) + parser.add_argument( + f"--{prefix}tokenizer-cache-enable-l1", + action="store_true", + default=RouterArgs.tokenizer_cache_enable_l1, + help="Enable L1 (prefix matching) tokenizer cache (default: False)", + ) + parser.add_argument( + f"--{prefix}tokenizer-cache-l1-max-memory", + type=int, + default=RouterArgs.tokenizer_cache_l1_max_memory, + help="Maximum memory for L1 tokenizer cache in bytes (default: 50MB)", + ) + parser.add_argument( + f"--{prefix}reasoning-parser", + type=str, + default=None, + help="Specify the parser for reasoning models (e.g., deepseek-r1, qwen3)", + ) + parser.add_argument( + f"--{prefix}tool-call-parser", + type=str, + default=None, + help="Specify the parser for handling tool-call interactions", + ) + # MCP server configuration + parser.add_argument( + f"--{prefix}mcp-config-path", + type=str, + default=None, + help="Path to MCP (Model Context Protocol) server configuration file", + ) + # Backend selection + parser.add_argument( + f"--{prefix}backend", + type=str, + default=RouterArgs.backend, + choices=["sglang", "openai"], + help="Backend runtime to use (default: sglang)", + ) + # History backend configuration + parser.add_argument( + f"--{prefix}history-backend", + type=str, + default=RouterArgs.history_backend, + choices=["memory", "none", "oracle", "postgres"], + help="History storage backend for conversations and responses (default: memory)", + ) + # Oracle configuration + parser.add_argument( + f"--{prefix}oracle-wallet-path", + type=str, + default=os.getenv("ATP_WALLET_PATH"), + help="Path to Oracle ATP wallet directory (env: ATP_WALLET_PATH)", + ) + parser.add_argument( + f"--{prefix}oracle-tns-alias", + type=str, + default=os.getenv("ATP_TNS_ALIAS"), + help="Oracle TNS alias from tnsnames.ora (env: ATP_TNS_ALIAS).", + ) + parser.add_argument( + f"--{prefix}oracle-connect-descriptor", + type=str, + default=os.getenv("ATP_DSN"), + help="Oracle connection descriptor/DSN (full connection string) (env: ATP_DSN)", + ) + parser.add_argument( + f"--{prefix}oracle-username", + type=str, + default=os.getenv("ATP_USER"), + help="Oracle database username (env: ATP_USER)", + ) + parser.add_argument( + f"--{prefix}oracle-password", + type=str, + default=os.getenv("ATP_PASSWORD"), + help="Oracle database password (env: ATP_PASSWORD)", + ) + parser.add_argument( + f"--{prefix}oracle-pool-min", + type=int, + default=int(os.getenv("ATP_POOL_MIN", RouterArgs.oracle_pool_min)), + help="Minimum Oracle connection pool size (default: 1, env: ATP_POOL_MIN)", + ) + parser.add_argument( + f"--{prefix}oracle-pool-max", + type=int, + default=int(os.getenv("ATP_POOL_MAX", RouterArgs.oracle_pool_max)), + help="Maximum Oracle connection pool size (default: 16, env: ATP_POOL_MAX)", + ) + parser.add_argument( + f"--{prefix}oracle-pool-timeout-secs", + type=int, + default=int( + os.getenv("ATP_POOL_TIMEOUT_SECS", RouterArgs.oracle_pool_timeout_secs) + ), + help="Oracle connection pool timeout in seconds (default: 30, env: ATP_POOL_TIMEOUT_SECS)", + ) + # Postgres configuration + parser.add_argument( + f"--{prefix}postgres-db-url", + type=str, + default=os.getenv("POSTGRES_DB_URL"), + help="PostgreSQL database connection URL (env: POSTGRES_DB_URL)", + ) + parser.add_argument( + f"--{prefix}postgres-pool-max", + type=int, + default=int(os.getenv("POSTGRES_POOL_MAX", RouterArgs.postgres_pool_max)), + help="Maximum PostgreSQL connection pool size (default: 16, env: POSTGRES_POOL_MAX)", + ) + # mTLS configuration + parser.add_argument( + f"--{prefix}client-cert-path", + type=str, + default=None, + help="Path to client certificate for mTLS authentication with workers", + ) + parser.add_argument( + f"--{prefix}client-key-path", + type=str, + default=None, + help="Path to client private key for mTLS authentication with workers", + ) + parser.add_argument( + f"--{prefix}ca-cert-paths", + type=str, + nargs="*", + default=[], + help="Path(s) to CA certificate(s) for verifying worker TLS certificates. Can specify multiple CAs.", + ) + parser.add_argument( + f"--{prefix}enable-trace", + action="store_true", + help="Enable opentelemetry trace", + ) + parser.add_argument( + f"--{prefix}otlp-traces-endpoint", + type=str, + default="localhost:4317", + help="Config opentelemetry collector endpoint if --enable-trace is set. format: :", + ) + + @classmethod + def from_cli_args( + cls, args: argparse.Namespace, use_router_prefix: bool = False + ) -> "RouterArgs": + """ + Create RouterArgs instance from parsed command line arguments. + + Args: + args: Parsed command line arguments + use_router_prefix: If True, look for arguments with 'router-' prefix + """ + prefix = "router_" if use_router_prefix else "" + cli_args_dict = vars(args) + args_dict = {} + + for attr in dataclasses.fields(cls): + # Auto strip prefix from args + if f"{prefix}{attr.name}" in cli_args_dict: + args_dict[attr.name] = cli_args_dict[f"{prefix}{attr.name}"] + elif attr.name in cli_args_dict: + args_dict[attr.name] = cli_args_dict[attr.name] + + # parse special arguments and remove "--prefill" and "--decode" from cli_args_dict + args_dict["prefill_urls"] = cls._parse_prefill_urls( + cli_args_dict.get(f"{prefix}prefill", None) + ) + args_dict["decode_urls"] = cls._parse_decode_urls( + cli_args_dict.get(f"{prefix}decode", None) + ) + args_dict["selector"] = cls._parse_selector( + cli_args_dict.get(f"{prefix}selector", None) + ) + args_dict["prefill_selector"] = cls._parse_selector( + cli_args_dict.get(f"{prefix}prefill_selector", None) + ) + args_dict["decode_selector"] = cls._parse_selector( + cli_args_dict.get(f"{prefix}decode_selector", None) + ) + + # Mooncake-specific annotation + args_dict["bootstrap_port_annotation"] = "sglang.ai/bootstrap-port" + + return cls(**args_dict) + + def _validate_router_args(self): + # Validate configuration based on mode + if self.pd_disaggregation: + # Allow empty URLs even without service discovery to support dynamic worker addition + # URLs will be validated separately if provided + pass + + # Warn about policy usage in PD mode + if self.prefill_policy and self.decode_policy and self.policy: + logger.warning( + "Both --prefill-policy and --decode-policy are specified. " + "The main --policy flag will be ignored for PD mode." + ) + elif self.prefill_policy and not self.decode_policy and self.policy: + logger.info( + f"Using --prefill-policy '{self.prefill_policy}' for prefill nodes " + f"and --policy '{self.policy}' for decode nodes." + ) + elif self.decode_policy and not self.prefill_policy and self.policy: + logger.info( + f"Using --policy '{self.policy}' for prefill nodes " + f"and --decode-policy '{self.decode_policy}' for decode nodes." + ) + + @staticmethod + def _parse_selector(selector_list): + if not selector_list: + return {} + + selector = {} + for item in selector_list: + if "=" in item: + key, value = item.split("=", 1) + selector[key] = value + return selector + + @staticmethod + def _parse_prefill_urls(prefill_list): + """Parse prefill URLs from --prefill arguments. + + Format: --prefill URL [BOOTSTRAP_PORT] + Example: + --prefill http://prefill1:8080 9000 # With bootstrap port + --prefill http://prefill2:8080 none # Explicitly no bootstrap port + --prefill http://prefill3:8080 # Defaults to no bootstrap port + """ + if not prefill_list: + return [] + + prefill_urls = [] + for prefill_args in prefill_list: + + url = prefill_args[0] + + # Handle optional bootstrap port + if len(prefill_args) >= 2: + bootstrap_port_str = prefill_args[1] + # Handle 'none' as None + if bootstrap_port_str.lower() == "none": + bootstrap_port = None + else: + try: + bootstrap_port = int(bootstrap_port_str) + except ValueError: + raise ValueError( + f"Invalid bootstrap port: {bootstrap_port_str}. Must be a number or 'none'" + ) + else: + # No bootstrap port specified, default to None + bootstrap_port = None + + prefill_urls.append((url, bootstrap_port)) + + return prefill_urls + + @staticmethod + def _parse_decode_urls(decode_list): + """Parse decode URLs from --decode arguments. + + Format: --decode URL + Example: --decode http://decode1:8081 --decode http://decode2:8081 + """ + if not decode_list: + return [] + + # decode_list is a list of single-element lists due to nargs=1 + return [url[0] for url in decode_list] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/version.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/version.py new file mode 100644 index 00000000..d31c31ea --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/sglang_router/version.py @@ -0,0 +1 @@ +__version__ = "0.2.3" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/src/lib.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/src/lib.rs new file mode 100644 index 00000000..8a321cc9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/bindings/python/src/lib.rs @@ -0,0 +1,730 @@ +use pyo3::prelude::*; +use sgl_model_gateway::*; +use std::collections::HashMap; + +// Define the enums with PyO3 bindings +#[pyclass(eq)] +#[derive(Clone, PartialEq, Debug)] +pub enum PolicyType { + Random, + RoundRobin, + CacheAware, + PowerOfTwo, + Bucket, +} + +#[pyclass(eq)] +#[derive(Clone, PartialEq, Debug)] +pub enum BackendType { + Sglang, + Openai, +} + +#[pyclass(eq)] +#[derive(Clone, PartialEq, Debug)] +pub enum HistoryBackendType { + Memory, + None, + Oracle, + Postgres, +} + +#[pyclass] +#[derive(Clone, PartialEq)] +pub struct PyOracleConfig { + #[pyo3(get, set)] + pub wallet_path: Option, + #[pyo3(get, set)] + pub connect_descriptor: Option, + #[pyo3(get, set)] + pub username: Option, + #[pyo3(get, set)] + pub password: Option, + #[pyo3(get, set)] + pub pool_min: usize, + #[pyo3(get, set)] + pub pool_max: usize, + #[pyo3(get, set)] + pub pool_timeout_secs: u64, +} + +impl std::fmt::Debug for PyOracleConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PyOracleConfig") + .field("wallet_path", &self.wallet_path) + .field("connect_descriptor", &"") + .field("username", &self.username) + .field("password", &"") + .field("pool_min", &self.pool_min) + .field("pool_max", &self.pool_max) + .field("pool_timeout_secs", &self.pool_timeout_secs) + .finish() + } +} + +#[pymethods] +impl PyOracleConfig { + #[new] + #[pyo3(signature = ( + password = None, + username = None, + connect_descriptor = None, + wallet_path = None, + pool_min = 1, + pool_max = 16, + pool_timeout_secs = 30, + ))] + fn new( + password: Option, + username: Option, + connect_descriptor: Option, + wallet_path: Option, + pool_min: usize, + pool_max: usize, + pool_timeout_secs: u64, + ) -> PyResult { + if pool_min == 0 { + return Err(pyo3::exceptions::PyValueError::new_err( + "pool_min must be at least 1", + )); + } + if pool_max < pool_min { + return Err(pyo3::exceptions::PyValueError::new_err( + "pool_max must be >= pool_min", + )); + } + + Ok(PyOracleConfig { + wallet_path, + connect_descriptor, + username, + password, + pool_min, + pool_max, + pool_timeout_secs, + }) + } +} + +impl PyOracleConfig { + pub fn to_config_oracle(&self) -> config::OracleConfig { + config::OracleConfig { + wallet_path: self.wallet_path.clone(), + connect_descriptor: self.connect_descriptor.clone().unwrap_or_default(), + username: self.username.clone().unwrap_or_default(), + password: self.password.clone().unwrap_or_default(), + pool_min: self.pool_min, + pool_max: self.pool_max, + pool_timeout_secs: self.pool_timeout_secs, + } + } +} + +#[pyclass] +#[derive(Debug, Clone, PartialEq)] +pub struct PyPostgresConfig { + #[pyo3(get, set)] + pub db_url: Option, + + #[pyo3(get, set)] + pub pool_max: usize, +} + +#[pymethods] +impl PyPostgresConfig { + #[new] + #[pyo3(signature = (db_url = None,pool_max = 16,))] + fn new(db_url: Option, pool_max: usize) -> PyResult { + Ok(PyPostgresConfig { db_url, pool_max }) + } +} + +impl PyPostgresConfig { + pub fn to_config_postgres(&self) -> config::PostgresConfig { + config::PostgresConfig { + db_url: self.db_url.clone().unwrap_or_default(), + pool_max: self.pool_max, + } + } +} + +#[pyclass] +#[derive(Debug, Clone, PartialEq)] +struct Router { + host: String, + port: u16, + worker_urls: Vec, + policy: PolicyType, + worker_startup_timeout_secs: u64, + worker_startup_check_interval: u64, + cache_threshold: f32, + balance_abs_threshold: usize, + balance_rel_threshold: f32, + eviction_interval_secs: u64, + max_tree_size: usize, + max_payload_size: usize, + dp_aware: bool, + api_key: Option, + log_dir: Option, + log_level: Option, + service_discovery: bool, + selector: HashMap, + service_discovery_port: u16, + service_discovery_namespace: Option, + prefill_selector: HashMap, + decode_selector: HashMap, + bootstrap_port_annotation: String, + prometheus_port: Option, + prometheus_host: Option, + request_timeout_secs: u64, + request_id_headers: Option>, + pd_disaggregation: bool, + bucket_adjust_interval_secs: usize, + prefill_urls: Option)>>, + decode_urls: Option>, + prefill_policy: Option, + decode_policy: Option, + max_concurrent_requests: i32, + cors_allowed_origins: Vec, + retry_max_retries: u32, + retry_initial_backoff_ms: u64, + retry_max_backoff_ms: u64, + retry_backoff_multiplier: f32, + retry_jitter_factor: f32, + disable_retries: bool, + cb_failure_threshold: u32, + cb_success_threshold: u32, + cb_timeout_duration_secs: u64, + cb_window_duration_secs: u64, + disable_circuit_breaker: bool, + health_failure_threshold: u32, + health_success_threshold: u32, + health_check_timeout_secs: u64, + health_check_interval_secs: u64, + health_check_endpoint: String, + enable_igw: bool, + queue_size: usize, + queue_timeout_secs: u64, + rate_limit_tokens_per_second: Option, + connection_mode: core::ConnectionMode, + model_path: Option, + tokenizer_path: Option, + chat_template: Option, + tokenizer_cache_enable_l0: bool, + tokenizer_cache_l0_max_entries: usize, + tokenizer_cache_enable_l1: bool, + tokenizer_cache_l1_max_memory: usize, + reasoning_parser: Option, + tool_call_parser: Option, + mcp_config_path: Option, + backend: BackendType, + history_backend: HistoryBackendType, + oracle_config: Option, + postgres_config: Option, + client_cert_path: Option, + client_key_path: Option, + ca_cert_paths: Vec, +} + +impl Router { + fn determine_connection_mode(worker_urls: &[String]) -> core::ConnectionMode { + for url in worker_urls { + if url.starts_with("grpc://") || url.starts_with("grpcs://") { + return core::ConnectionMode::Grpc { port: None }; + } + } + core::ConnectionMode::Http + } + + pub fn to_router_config(&self) -> config::ConfigResult { + use config::{ + DiscoveryConfig, MetricsConfig, PolicyConfig as ConfigPolicyConfig, RoutingMode, + }; + + let convert_policy = |policy: &PolicyType| -> ConfigPolicyConfig { + match policy { + PolicyType::Random => ConfigPolicyConfig::Random, + PolicyType::RoundRobin => ConfigPolicyConfig::RoundRobin, + PolicyType::CacheAware => ConfigPolicyConfig::CacheAware { + cache_threshold: self.cache_threshold, + balance_abs_threshold: self.balance_abs_threshold, + balance_rel_threshold: self.balance_rel_threshold, + eviction_interval_secs: self.eviction_interval_secs, + max_tree_size: self.max_tree_size, + }, + PolicyType::PowerOfTwo => ConfigPolicyConfig::PowerOfTwo { + load_check_interval_secs: 5, + }, + PolicyType::Bucket => ConfigPolicyConfig::Bucket { + balance_abs_threshold: self.balance_abs_threshold, + balance_rel_threshold: self.balance_rel_threshold, + bucket_adjust_interval_secs: self.bucket_adjust_interval_secs, + }, + } + }; + + let mode = if self.enable_igw { + RoutingMode::Regular { + worker_urls: vec![], + } + } else if matches!(self.backend, BackendType::Openai) { + RoutingMode::OpenAI { + worker_urls: self.worker_urls.clone(), + } + } else if self.pd_disaggregation { + RoutingMode::PrefillDecode { + prefill_urls: self.prefill_urls.clone().unwrap_or_default(), + decode_urls: self.decode_urls.clone().unwrap_or_default(), + prefill_policy: self.prefill_policy.as_ref().map(convert_policy), + decode_policy: self.decode_policy.as_ref().map(convert_policy), + } + } else { + RoutingMode::Regular { + worker_urls: self.worker_urls.clone(), + } + }; + + let policy = convert_policy(&self.policy); + + let discovery = if self.service_discovery { + Some(DiscoveryConfig { + enabled: true, + namespace: self.service_discovery_namespace.clone(), + port: self.service_discovery_port, + check_interval_secs: 60, + selector: self.selector.clone(), + prefill_selector: self.prefill_selector.clone(), + decode_selector: self.decode_selector.clone(), + bootstrap_port_annotation: self.bootstrap_port_annotation.clone(), + }) + } else { + None + }; + + let metrics = match (self.prometheus_port, self.prometheus_host.as_ref()) { + (Some(port), Some(host)) => Some(MetricsConfig { + port, + host: host.clone(), + }), + _ => None, + }; + + let history_backend = match self.history_backend { + HistoryBackendType::Memory => config::HistoryBackend::Memory, + HistoryBackendType::None => config::HistoryBackend::None, + HistoryBackendType::Oracle => config::HistoryBackend::Oracle, + HistoryBackendType::Postgres => config::HistoryBackend::Postgres, + }; + + let oracle = if matches!(self.history_backend, HistoryBackendType::Oracle) { + self.oracle_config + .as_ref() + .map(|cfg| cfg.to_config_oracle()) + } else { + None + }; + + let postgres_config = if matches!(self.history_backend, HistoryBackendType::Postgres) { + self.postgres_config + .as_ref() + .map(|cfg| cfg.to_config_postgres()) + } else { + None + }; + + config::RouterConfig::builder() + .mode(mode) + .policy(policy) + .host(&self.host) + .port(self.port) + .connection_mode(self.connection_mode.clone()) + .max_payload_size(self.max_payload_size) + .request_timeout_secs(self.request_timeout_secs) + .worker_startup_timeout_secs(self.worker_startup_timeout_secs) + .worker_startup_check_interval_secs(self.worker_startup_check_interval) + .max_concurrent_requests(self.max_concurrent_requests) + .queue_size(self.queue_size) + .queue_timeout_secs(self.queue_timeout_secs) + .cors_allowed_origins(self.cors_allowed_origins.clone()) + .retry_config(config::RetryConfig { + max_retries: self.retry_max_retries, + initial_backoff_ms: self.retry_initial_backoff_ms, + max_backoff_ms: self.retry_max_backoff_ms, + backoff_multiplier: self.retry_backoff_multiplier, + jitter_factor: self.retry_jitter_factor, + }) + .circuit_breaker_config(config::CircuitBreakerConfig { + failure_threshold: self.cb_failure_threshold, + success_threshold: self.cb_success_threshold, + timeout_duration_secs: self.cb_timeout_duration_secs, + window_duration_secs: self.cb_window_duration_secs, + }) + .health_check_config(config::HealthCheckConfig { + failure_threshold: self.health_failure_threshold, + success_threshold: self.health_success_threshold, + timeout_secs: self.health_check_timeout_secs, + check_interval_secs: self.health_check_interval_secs, + endpoint: self.health_check_endpoint.clone(), + }) + .tokenizer_cache(config::TokenizerCacheConfig { + enable_l0: self.tokenizer_cache_enable_l0, + l0_max_entries: self.tokenizer_cache_l0_max_entries, + enable_l1: self.tokenizer_cache_enable_l1, + l1_max_memory: self.tokenizer_cache_l1_max_memory, + }) + .history_backend(history_backend) + .maybe_api_key(self.api_key.as_ref()) + .maybe_discovery(discovery) + .maybe_metrics(metrics) + .maybe_log_dir(self.log_dir.as_ref()) + .maybe_log_level(self.log_level.as_ref()) + .maybe_request_id_headers(self.request_id_headers.clone()) + .maybe_rate_limit_tokens_per_second(self.rate_limit_tokens_per_second) + .maybe_model_path(self.model_path.as_ref()) + .maybe_tokenizer_path(self.tokenizer_path.as_ref()) + .maybe_chat_template(self.chat_template.as_ref()) + .maybe_oracle(oracle) + .maybe_postgres(postgres_config) + .maybe_reasoning_parser(self.reasoning_parser.as_ref()) + .maybe_tool_call_parser(self.tool_call_parser.as_ref()) + .maybe_mcp_config_path(self.mcp_config_path.as_ref()) + .dp_aware(self.dp_aware) + .retries(!self.disable_retries) + .circuit_breaker(!self.disable_circuit_breaker) + .igw(self.enable_igw) + .maybe_client_cert_and_key( + self.client_cert_path.as_ref(), + self.client_key_path.as_ref(), + ) + .add_ca_certificates(self.ca_cert_paths.clone()) + .build() + } +} + +#[pymethods] +impl Router { + #[new] + #[pyo3(signature = ( + worker_urls, + policy = PolicyType::RoundRobin, + host = String::from("0.0.0.0"), + port = 3001, + worker_startup_timeout_secs = 600, + worker_startup_check_interval = 30, + cache_threshold = 0.3, + balance_abs_threshold = 64, + balance_rel_threshold = 1.5, + eviction_interval_secs = 120, + max_tree_size = 2usize.pow(26), + max_payload_size = 512 * 1024 * 1024, + dp_aware = false, + api_key = None, + log_dir = None, + log_level = None, + service_discovery = false, + selector = HashMap::new(), + service_discovery_port = 80, + service_discovery_namespace = None, + prefill_selector = HashMap::new(), + decode_selector = HashMap::new(), + bootstrap_port_annotation = String::from("sglang.ai/bootstrap-port"), + prometheus_port = None, + prometheus_host = None, + request_timeout_secs = 1800, + request_id_headers = None, + pd_disaggregation = false, + bucket_adjust_interval_secs = 5, + prefill_urls = None, + decode_urls = None, + prefill_policy = None, + decode_policy = None, + max_concurrent_requests = -1, + cors_allowed_origins = vec![], + retry_max_retries = 5, + retry_initial_backoff_ms = 50, + retry_max_backoff_ms = 30_000, + retry_backoff_multiplier = 1.5, + retry_jitter_factor = 0.2, + disable_retries = false, + cb_failure_threshold = 10, + cb_success_threshold = 3, + cb_timeout_duration_secs = 60, + cb_window_duration_secs = 120, + disable_circuit_breaker = false, + health_failure_threshold = 3, + health_success_threshold = 2, + health_check_timeout_secs = 5, + health_check_interval_secs = 60, + health_check_endpoint = String::from("/health"), + enable_igw = false, + queue_size = 100, + queue_timeout_secs = 60, + rate_limit_tokens_per_second = None, + model_path = None, + tokenizer_path = None, + chat_template = None, + tokenizer_cache_enable_l0 = false, + tokenizer_cache_l0_max_entries = 10000, + tokenizer_cache_enable_l1 = false, + tokenizer_cache_l1_max_memory = 52428800, + reasoning_parser = None, + tool_call_parser = None, + mcp_config_path = None, + backend = BackendType::Sglang, + history_backend = HistoryBackendType::Memory, + oracle_config = None, + postgres_config = None, + client_cert_path = None, + client_key_path = None, + ca_cert_paths = vec![], + ))] + #[allow(clippy::too_many_arguments)] + fn new( + worker_urls: Vec, + policy: PolicyType, + host: String, + port: u16, + worker_startup_timeout_secs: u64, + worker_startup_check_interval: u64, + cache_threshold: f32, + balance_abs_threshold: usize, + balance_rel_threshold: f32, + eviction_interval_secs: u64, + max_tree_size: usize, + max_payload_size: usize, + dp_aware: bool, + api_key: Option, + log_dir: Option, + log_level: Option, + service_discovery: bool, + selector: HashMap, + service_discovery_port: u16, + service_discovery_namespace: Option, + prefill_selector: HashMap, + decode_selector: HashMap, + bootstrap_port_annotation: String, + prometheus_port: Option, + prometheus_host: Option, + request_timeout_secs: u64, + request_id_headers: Option>, + pd_disaggregation: bool, + bucket_adjust_interval_secs: usize, + prefill_urls: Option)>>, + decode_urls: Option>, + prefill_policy: Option, + decode_policy: Option, + max_concurrent_requests: i32, + cors_allowed_origins: Vec, + retry_max_retries: u32, + retry_initial_backoff_ms: u64, + retry_max_backoff_ms: u64, + retry_backoff_multiplier: f32, + retry_jitter_factor: f32, + disable_retries: bool, + cb_failure_threshold: u32, + cb_success_threshold: u32, + cb_timeout_duration_secs: u64, + cb_window_duration_secs: u64, + disable_circuit_breaker: bool, + health_failure_threshold: u32, + health_success_threshold: u32, + health_check_timeout_secs: u64, + health_check_interval_secs: u64, + health_check_endpoint: String, + enable_igw: bool, + queue_size: usize, + queue_timeout_secs: u64, + rate_limit_tokens_per_second: Option, + model_path: Option, + tokenizer_path: Option, + chat_template: Option, + tokenizer_cache_enable_l0: bool, + tokenizer_cache_l0_max_entries: usize, + tokenizer_cache_enable_l1: bool, + tokenizer_cache_l1_max_memory: usize, + reasoning_parser: Option, + tool_call_parser: Option, + mcp_config_path: Option, + backend: BackendType, + history_backend: HistoryBackendType, + oracle_config: Option, + postgres_config: Option, + client_cert_path: Option, + client_key_path: Option, + ca_cert_paths: Vec, + ) -> PyResult { + let mut all_urls = worker_urls.clone(); + + if let Some(ref prefill_urls) = prefill_urls { + for (url, _) in prefill_urls { + all_urls.push(url.clone()); + } + } + + if let Some(ref decode_urls) = decode_urls { + all_urls.extend(decode_urls.clone()); + } + + let connection_mode = Self::determine_connection_mode(&all_urls); + + Ok(Router { + host, + port, + worker_urls, + policy, + worker_startup_timeout_secs, + worker_startup_check_interval, + cache_threshold, + balance_abs_threshold, + balance_rel_threshold, + eviction_interval_secs, + max_tree_size, + max_payload_size, + dp_aware, + api_key, + log_dir, + log_level, + service_discovery, + selector, + service_discovery_port, + service_discovery_namespace, + prefill_selector, + decode_selector, + bootstrap_port_annotation, + prometheus_port, + prometheus_host, + request_timeout_secs, + request_id_headers, + pd_disaggregation, + bucket_adjust_interval_secs, + prefill_urls, + decode_urls, + prefill_policy, + decode_policy, + max_concurrent_requests, + cors_allowed_origins, + retry_max_retries, + retry_initial_backoff_ms, + retry_max_backoff_ms, + retry_backoff_multiplier, + retry_jitter_factor, + disable_retries, + cb_failure_threshold, + cb_success_threshold, + cb_timeout_duration_secs, + cb_window_duration_secs, + disable_circuit_breaker, + health_failure_threshold, + health_success_threshold, + health_check_timeout_secs, + health_check_interval_secs, + health_check_endpoint, + enable_igw, + queue_size, + queue_timeout_secs, + rate_limit_tokens_per_second, + connection_mode, + model_path, + tokenizer_path, + chat_template, + tokenizer_cache_enable_l0, + tokenizer_cache_l0_max_entries, + tokenizer_cache_enable_l1, + tokenizer_cache_l1_max_memory, + reasoning_parser, + tool_call_parser, + mcp_config_path, + backend, + history_backend, + oracle_config, + postgres_config, + client_cert_path, + client_key_path, + ca_cert_paths, + }) + } + + fn start(&self) -> PyResult<()> { + use metrics::PrometheusConfig; + + let router_config = self.to_router_config().map_err(|e| { + pyo3::exceptions::PyValueError::new_err(format!("Configuration error: {}", e)) + })?; + + router_config.validate().map_err(|e| { + pyo3::exceptions::PyValueError::new_err(format!( + "Configuration validation failed: {}", + e + )) + })?; + + let service_discovery_config = if self.service_discovery { + Some(service_discovery::ServiceDiscoveryConfig { + enabled: true, + selector: self.selector.clone(), + check_interval: std::time::Duration::from_secs(60), + port: self.service_discovery_port, + namespace: self.service_discovery_namespace.clone(), + pd_mode: self.pd_disaggregation, + prefill_selector: self.prefill_selector.clone(), + decode_selector: self.decode_selector.clone(), + bootstrap_port_annotation: self.bootstrap_port_annotation.clone(), + }) + } else { + None + }; + + let prometheus_config = Some(PrometheusConfig { + port: self.prometheus_port.unwrap_or(29000), + host: self + .prometheus_host + .clone() + .unwrap_or_else(|| "127.0.0.1".to_string()), + }); + + let runtime = tokio::runtime::Runtime::new() + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))?; + + runtime.block_on(async move { + server::startup(server::ServerConfig { + host: self.host.clone(), + port: self.port, + router_config, + max_payload_size: self.max_payload_size, + log_dir: self.log_dir.clone(), + log_level: self.log_level.clone(), + service_discovery_config, + prometheus_config, + request_timeout_secs: self.request_timeout_secs, + request_id_headers: self.request_id_headers.clone(), + }) + .await + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string())) + }) + } +} + +/// Get simple version string (default for --version) +#[pyfunction] +fn get_version_string() -> String { + version::get_version_string() +} + +/// Get verbose version information string with full build details (for --version-verbose) +#[pyfunction] +fn get_verbose_version_string() -> String { + version::get_verbose_version_string() +} + +#[pymodule] +fn sglang_router_rs(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_function(wrap_pyfunction!(get_version_string, m)?)?; + m.add_function(wrap_pyfunction!(get_verbose_version_string, m)?)?; + Ok(()) +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/build.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/build.rs new file mode 100644 index 00000000..57a02656 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/build.rs @@ -0,0 +1,124 @@ +use std::process::Command; + +const DEFAULT_VERSION: &str = "0.0.0"; +const DEFAULT_PROJECT_NAME: &str = "sgl-model-gateway"; + +/// Set a compile-time environment variable with the SGL_MODEL_GATEWAY_ prefix +macro_rules! set_env { + ($name:expr, $value:expr) => { + println!("cargo:rustc-env=SGL_MODEL_GATEWAY_{}={}", $name, $value); + }; +} + +fn main() -> Result<(), Box> { + // Rebuild triggers + println!("cargo:rerun-if-changed=src/proto/sglang_scheduler.proto"); + println!("cargo:rerun-if-changed=src/proto/vllm_engine.proto"); + println!("cargo:rerun-if-changed=Cargo.toml"); + + // Compile protobuf files + tonic_prost_build::configure() + .build_server(true) + .build_client(true) + .type_attribute("GetModelInfoResponse", "#[derive(serde::Serialize)]") + .protoc_arg("--experimental_allow_proto3_optional") + .compile_protos( + &[ + "src/proto/sglang_scheduler.proto", + "src/proto/vllm_engine.proto", + ], + &["src/proto"], + )?; + + // Set version info environment variables + let version = read_cargo_version().unwrap_or_else(|_| DEFAULT_VERSION.to_string()); + let target = std::env::var("TARGET").unwrap_or_else(|_| get_rustc_host().unwrap_or_default()); + let profile = std::env::var("PROFILE").unwrap_or_default(); + + set_env!("PROJECT_NAME", DEFAULT_PROJECT_NAME); + set_env!("VERSION", version); + set_env!( + "BUILD_TIME", + chrono::Utc::now().format("%Y-%m-%d %H:%M:%S UTC") + ); + set_env!( + "BUILD_MODE", + if profile == "release" { + "release" + } else { + "debug" + } + ); + set_env!("TARGET_TRIPLE", target); + set_env!( + "GIT_BRANCH", + git_branch().unwrap_or_else(|| "unknown".into()) + ); + set_env!( + "GIT_COMMIT", + git_commit().unwrap_or_else(|| "unknown".into()) + ); + set_env!( + "GIT_STATUS", + git_status().unwrap_or_else(|| "unknown".into()) + ); + set_env!( + "RUSTC_VERSION", + rustc_version().unwrap_or_else(|| "unknown".into()) + ); + set_env!( + "CARGO_VERSION", + cargo_version().unwrap_or_else(|| "unknown".into()) + ); + + Ok(()) +} + +fn read_cargo_version() -> Result> { + let content = std::fs::read_to_string("Cargo.toml")?; + let toml: toml::Value = toml::from_str(&content)?; + toml.get("package") + .and_then(|p| p.get("version")) + .and_then(|v| v.as_str()) + .map(String::from) + .ok_or_else(|| "Missing version in Cargo.toml".into()) +} + +fn run_cmd(cmd: &str, args: &[&str]) -> Option { + Command::new(cmd) + .args(args) + .output() + .ok() + .filter(|o| o.status.success()) + .and_then(|o| String::from_utf8(o.stdout).ok()) + .map(|s| s.trim().to_string()) +} + +fn git_branch() -> Option { + run_cmd("git", &["rev-parse", "--abbrev-ref", "HEAD"]) +} + +fn git_commit() -> Option { + run_cmd("git", &["rev-parse", "--short", "HEAD"]) +} + +fn git_status() -> Option { + run_cmd("git", &["status", "--porcelain"]) + .map(|s| if s.is_empty() { "clean" } else { "dirty" }.into()) +} + +fn rustc_version() -> Option { + run_cmd("rustc", &["--version"]) +} + +fn cargo_version() -> Option { + run_cmd("cargo", &["--version"]) +} + +fn get_rustc_host() -> Option { + run_cmd("rustc", &["-vV"])? + .lines() + .find(|l| l.starts_with("host: ")) + .and_then(|l| l.strip_prefix("host: ")) + .map(|s| s.trim().to_string()) +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/__init__.py new file mode 100644 index 00000000..89309778 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/__init__.py @@ -0,0 +1 @@ +"""Test package root for router Python tests.""" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/conftest.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/conftest.py new file mode 100644 index 00000000..fe4fc699 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/conftest.py @@ -0,0 +1,15 @@ +import sys +from importlib.util import find_spec +from pathlib import Path + +# Only add bindings/python to path if the wheel is not installed (for local development) +# This ensures CI tests use the installed wheel which contains the Rust extension +_ROOT = Path(__file__).resolve().parents[1] +_SRC = _ROOT / "bindings" / "python" + +# Check if sglang_router is already installed with the Rust extension +_wheel_installed = find_spec("sglang_router.sglang_router_rs") is not None + +# Only add bindings/python if wheel is not installed (development mode) +if not _wheel_installed and str(_SRC) not in sys.path: + sys.path.insert(0, str(_SRC)) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/basic/test_openai_server.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/basic/test_openai_server.py new file mode 100644 index 00000000..a3c73b95 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/basic/test_openai_server.py @@ -0,0 +1,343 @@ +""" +gRPC Router E2E Test - OpenAI Server API Compatibility + +This test file is REUSED from test/srt/openai_server/basic/test_openai_server.py +with minimal changes: +- Swap popen_launch_server() → popen_launch_workers_and_router() +- Update teardown to cleanup router + workers +- All test logic and assertions remain identical + +Run with: + python3 -m pytest e2e_grpc/basic/test_openai_server.py -v + python3 -m unittest e2e_grpc.basic.test_openai_server.TestOpenAIServer.test_completion +""" + +import json +import sys +import unittest +from pathlib import Path + +import openai + +_TEST_DIR = Path(__file__).parent +sys.path.insert(0, str(_TEST_DIR.parent)) +from fixtures import popen_launch_workers_and_router +from util import ( + DEFAULT_GPT_OSS_MODEL_PATH, + DEFAULT_MODEL_PATH, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + get_tokenizer, + kill_process_tree, +) + + +class TestOpenAIServer(CustomTestCase): + """ + Test OpenAI API through gRPC router. + + REUSED from test/srt/openai_server/basic/test_openai_server.py + ONLY CHANGE: Server launch mechanism + - Launches SGLang workers with --enable-grpc + - Launches gRPC router pointing to those workers + """ + + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + cls.cluster = popen_launch_workers_and_router( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + num_workers=1, + tp_size=2, + policy="round_robin", + api_key=cls.api_key, + ) + + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + @classmethod + def tearDownClass(cls): + # Cleanup router and workers + kill_process_tree(cls.cluster["router"].pid) + for worker in cls.cluster.get("workers", []): + kill_process_tree(worker.pid) + + # ALL TEST METHODS BELOW ARE UNCHANGED FROM ORIGINAL + # They validate that the router maintains OpenAI API compatibility + def run_chat_completion(self, logprobs, parallel_sample_num): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + { + "role": "user", + "content": "What is the capital of France? Answer in a few words.", + }, + ], + temperature=0, + logprobs=logprobs is not None and logprobs > 0, + top_logprobs=logprobs, + n=parallel_sample_num, + ) + + if logprobs: + assert isinstance( + response.choices[0].logprobs.content[0].top_logprobs[0].token, str + ) + + ret_num_top_logprobs = len( + response.choices[0].logprobs.content[0].top_logprobs + ) + assert ( + ret_num_top_logprobs == logprobs + ), f"{ret_num_top_logprobs} vs {logprobs}" + + assert len(response.choices) == parallel_sample_num + assert response.choices[0].message.role == "assistant" + assert isinstance(response.choices[0].message.content, str) + assert response.id + assert response.created + assert response.usage.prompt_tokens > 0 + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def run_chat_completion_stream(self, logprobs, parallel_sample_num=1): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + generator = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "What is the capital of France?"}, + ], + temperature=0, + logprobs=logprobs is not None and logprobs > 0, + top_logprobs=logprobs, + stream=True, + stream_options={"include_usage": True}, + n=parallel_sample_num, + ) + + is_firsts = {} + is_finished = {} + finish_reason_counts = {} + for response in generator: + usage = response.usage + if usage is not None: + assert usage.prompt_tokens > 0, f"usage.prompt_tokens was zero" + assert usage.completion_tokens > 0, f"usage.completion_tokens was zero" + assert usage.total_tokens > 0, f"usage.total_tokens was zero" + continue + + index = response.choices[0].index + finish_reason = response.choices[0].finish_reason + if finish_reason is not None: + is_finished[index] = True + finish_reason_counts[index] = finish_reason_counts.get(index, 0) + 1 + + data = response.choices[0].delta + + if is_firsts.get(index, True): + assert ( + data.role == "assistant" + ), f"data.role was not 'assistant' for first chunk" + is_firsts[index] = False + continue + + if logprobs and not is_finished.get(index, False): + assert response.choices[0].logprobs, f"logprobs was not returned" + assert isinstance( + response.choices[0].logprobs.content[0].top_logprobs[0].token, str + ), f"top_logprobs token was not a string" + assert isinstance( + response.choices[0].logprobs.content[0].top_logprobs, list + ), f"top_logprobs was not a list" + ret_num_top_logprobs = len( + response.choices[0].logprobs.content[0].top_logprobs + ) + assert ( + ret_num_top_logprobs == logprobs + ), f"{ret_num_top_logprobs} vs {logprobs}" + + assert ( + isinstance(data.content, str) + or isinstance(data.reasoning_content, str) + or (isinstance(data.tool_calls, list) and len(data.tool_calls) > 0) + or response.choices[0].finish_reason + ) + assert response.id + assert response.created + + for index in [i for i in range(parallel_sample_num)]: + assert not is_firsts.get( + index, True + ), f"index {index} is not found in the response" + + for index in range(parallel_sample_num): + assert ( + index in finish_reason_counts + ), f"No finish_reason found for index {index}" + assert ( + finish_reason_counts[index] == 1 + ), f"Expected 1 finish_reason chunk for index {index}, got {finish_reason_counts[index]}" + + def test_chat_completion(self): + for logprobs in [None, 5]: + for parallel_sample_num in [1, 2]: + self.run_chat_completion(logprobs, parallel_sample_num) + + def test_chat_completion_stream(self): + for logprobs in [None, 5]: + for parallel_sample_num in [1, 2]: + self.run_chat_completion_stream(logprobs, parallel_sample_num) + + def test_regex(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + regex = ( + r"""\{\n""" + + r""" "name": "[\w]+",\n""" + + r""" "population": [\d]+\n""" + + r"""\}""" + ) + + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "Introduce the capital of France."}, + ], + temperature=0, + max_tokens=128, + extra_body={"regex": regex}, + ) + text = response.choices[0].message.content + + try: + js_obj = json.loads(text) + except (TypeError, json.decoder.JSONDecodeError): + raise + assert isinstance(js_obj["name"], str) + assert isinstance(js_obj["population"], int) + + def test_penalty(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "Introduce the capital of France."}, + ], + temperature=0, + max_tokens=32, + frequency_penalty=1.0, + ) + text = response.choices[0].message.content + assert isinstance(text, str) + + def test_response_prefill(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + { + "role": "user", + "content": """ +Extract the name, size, price, and color from this product description as a JSON object: + + +The SmartHome Mini is a compact smart home assistant available in black or white for only $49.99. At just 5 inches wide, it lets you control lights, thermostats, and other connected devices via voice or app—no matter where you place it in your home. This affordable little hub brings convenient hands-free control to your smart devices. + +""", + }, + { + "role": "assistant", + "content": "{\n", + }, + ], + temperature=0, + extra_body={"continue_final_message": True}, + ) + + assert ( + response.choices[0] + .message.content.strip() + .startswith('"name": "SmartHome Mini",') + ) + + def test_model_list(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + models = list(client.models.list().data) + assert len(models) == 1 + # assert isinstance(getattr(models[0], "max_model_len", None), int) + + @unittest.skip("Skipping retrieve model test as it is not supported by the router") + def test_retrieve_model(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + retrieved_model = client.models.retrieve(self.model) + self.assertEqual(retrieved_model.id, self.model) + self.assertEqual(retrieved_model.root, self.model) + + with self.assertRaises(openai.NotFoundError): + client.models.retrieve("non-existent-model") + + +class TestOpenAIServerGptOss(TestOpenAIServer): + """ + Test OpenAI API through gRPC router with openai/gpt-oss-20b model. + Extends TestOpenAIServer and only changes the model. + """ + + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_GPT_OSS_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + cls.cluster = popen_launch_workers_and_router( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + num_workers=1, + tp_size=2, + policy="round_robin", + api_key=cls.api_key, + ) + + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + def test_chat_completion(self): + for parallel_sample_num in [1, 2]: + self.run_chat_completion(None, parallel_sample_num) + + def test_chat_completion_stream(self): + for parallel_sample_num in [1, 2]: + self.run_chat_completion_stream(None, parallel_sample_num) + + @unittest.skip("Skipping for OSS models") + def test_regex(self): + super().test_regex() + + @unittest.skip("Skipping for OSS models") + def test_response_prefill(self): + super().test_response_prefill() + + @unittest.skip("Skipping for OSS models") + def test_penalty(self): + super().test_penalty() + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/conftest.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/conftest.py new file mode 100644 index 00000000..b39d0d2c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/conftest.py @@ -0,0 +1,29 @@ +""" +Pytest configuration for gRPC router e2e tests. + +This module provides shared fixtures that can be used across all gRPC router tests. +""" + +import sys +from pathlib import Path + +import pytest # noqa: F401 + +# Ensure router bindings/python is importable +_ROUTER_ROOT = Path(__file__).resolve().parents[2] +_ROUTER_SRC = _ROUTER_ROOT / "bindings" / "python" +if str(_ROUTER_SRC) not in sys.path: + sys.path.insert(0, str(_ROUTER_SRC)) + +# Ensure e2e_grpc test utilities are importable +_E2E_GRPC_DIR = Path(__file__).parent +if str(_E2E_GRPC_DIR) not in sys.path: + sys.path.insert(0, str(_E2E_GRPC_DIR)) + + +# Pytest markers for test organization +def pytest_configure(config): + config.addinivalue_line("markers", "e2e: end-to-end tests with real workers") + config.addinivalue_line("markers", "grpc: gRPC-specific tests") + config.addinivalue_line("markers", "slow: slow-running tests") + config.addinivalue_line("markers", "pd: prefill-decode disaggregation tests") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/features/test_enable_thinking.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/features/test_enable_thinking.py new file mode 100644 index 00000000..a5042c37 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/features/test_enable_thinking.py @@ -0,0 +1,194 @@ +""" +Usage: +python3 -m unittest openai_server.features.test_enable_thinking.TestEnableThinking.test_chat_completion_with_reasoning +python3 -m unittest openai_server.features.test_enable_thinking.TestEnableThinking.test_chat_completion_without_reasoning +python3 -m unittest openai_server.features.test_enable_thinking.TestEnableThinking.test_stream_chat_completion_with_reasoning +python3 -m unittest openai_server.features.test_enable_thinking.TestEnableThinking.test_stream_chat_completion_without_reasoning +""" + +import json +import sys +import unittest +from pathlib import Path + +import requests + +_TEST_DIR = Path(__file__).parent +sys.path.insert(0, str(_TEST_DIR.parent)) +from fixtures import popen_launch_workers_and_router +from util import ( + DEFAULT_ENABLE_THINKING_MODEL_PATH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + kill_process_tree, +) + + +class TestEnableThinking(CustomTestCase): + @classmethod + def setUpClass(cls): + # CHANGE: Launch gRPC router with integrated workers (single command) + cls.model = DEFAULT_ENABLE_THINKING_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-1234" + cls.cluster = popen_launch_workers_and_router( + cls.model, + cls.base_url, + timeout=120, + api_key=cls.api_key, + router_args=[ + "--reasoning-parser", + "qwen3", + ], + num_workers=1, + tp_size=4, + ) + cls.additional_chat_kwargs = {} + + @classmethod + def tearDownClass(cls): + # Cleanup router and workers + kill_process_tree(cls.cluster["router"].pid) + for worker in cls.cluster.get("workers", []): + kill_process_tree(worker.pid) + + def test_chat_completion_with_reasoning(self): + # Test non-streaming with "enable_thinking": True, reasoning_content should not be empty + client = requests.post( + f"{self.base_url}/v1/chat/completions", + headers={"Authorization": f"Bearer {self.api_key}"}, + json={ + "model": self.model, + "messages": [{"role": "user", "content": "Hello"}], + "temperature": 0, + "separate_reasoning": True, + "chat_template_kwargs": {"enable_thinking": True}, + **self.additional_chat_kwargs, + }, + ) + + self.assertEqual(client.status_code, 200, f"Failed with: {client.text}") + data = client.json() + + self.assertIn("choices", data) + self.assertTrue(len(data["choices"]) > 0) + self.assertIn("message", data["choices"][0]) + self.assertIn("reasoning_content", data["choices"][0]["message"]) + self.assertIsNotNone(data["choices"][0]["message"]["reasoning_content"]) + + def test_chat_completion_without_reasoning(self): + # Test non-streaming with "enable_thinking": False, reasoning_content should be empty + client = requests.post( + f"{self.base_url}/v1/chat/completions", + headers={"Authorization": f"Bearer {self.api_key}"}, + json={ + "model": self.model, + "messages": [{"role": "user", "content": "Hello"}], + "temperature": 0, + "separate_reasoning": True, + "chat_template_kwargs": {"enable_thinking": False}, + **self.additional_chat_kwargs, + }, + ) + + self.assertEqual(client.status_code, 200, f"Failed with: {client.text}") + data = client.json() + + self.assertIn("choices", data) + self.assertTrue(len(data["choices"]) > 0) + self.assertIn("message", data["choices"][0]) + + if "reasoning_content" in data["choices"][0]["message"]: + self.assertIsNone(data["choices"][0]["message"]["reasoning_content"]) + + def test_stream_chat_completion_with_reasoning(self): + # Test streaming with "enable_thinking": True, reasoning_content should not be empty + response = requests.post( + f"{self.base_url}/v1/chat/completions", + headers={"Authorization": f"Bearer {self.api_key}"}, + json={ + "model": self.model, + "messages": [{"role": "user", "content": "Hello"}], + "temperature": 0, + "separate_reasoning": True, + "stream": True, + "chat_template_kwargs": {"enable_thinking": True}, + **self.additional_chat_kwargs, + }, + stream=True, + ) + + self.assertEqual(response.status_code, 200, f"Failed with: {response.text}") + + has_reasoning = False + has_content = False + + for line in response.iter_lines(): + if line: + line = line.decode("utf-8") + if line.startswith("data:") and not line.startswith("data: [DONE]"): + data = json.loads(line[6:]) + if "choices" in data and len(data["choices"]) > 0: + delta = data["choices"][0].get("delta", {}) + + if "reasoning_content" in delta and delta["reasoning_content"]: + has_reasoning = True + + if "content" in delta and delta["content"]: + has_content = True + + self.assertTrue( + has_reasoning, + "The reasoning content is not included in the stream response", + ) + self.assertTrue( + has_content, "The stream response does not contain normal content" + ) + + def test_stream_chat_completion_without_reasoning(self): + # Test streaming with "enable_thinking": False, reasoning_content should be empty + response = requests.post( + f"{self.base_url}/v1/chat/completions", + headers={"Authorization": f"Bearer {self.api_key}"}, + json={ + "model": self.model, + "messages": [{"role": "user", "content": "Hello"}], + "temperature": 0, + "separate_reasoning": True, + "stream": True, + "chat_template_kwargs": {"enable_thinking": False}, + **self.additional_chat_kwargs, + }, + stream=True, + ) + + self.assertEqual(response.status_code, 200, f"Failed with: {response.text}") + + has_reasoning = False + has_content = False + + for line in response.iter_lines(): + if line: + line = line.decode("utf-8") + if line.startswith("data:") and not line.startswith("data: [DONE]"): + data = json.loads(line[6:]) + if "choices" in data and len(data["choices"]) > 0: + delta = data["choices"][0].get("delta", {}) + + if "reasoning_content" in delta and delta["reasoning_content"]: + has_reasoning = True + + if "content" in delta and delta["content"]: + has_content = True + + self.assertFalse( + has_reasoning, + "The reasoning content should not be included in the stream response", + ) + self.assertTrue( + has_content, "The stream response does not contain normal content" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/features/test_reasoning_content.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/features/test_reasoning_content.py new file mode 100644 index 00000000..28f7e01b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/features/test_reasoning_content.py @@ -0,0 +1,194 @@ +""" +Usage: +python3 -m unittest openai_server.features.test_reasoning_content.TestReasoningContentAPI.test_streaming_separate_reasoning_false +python3 -m unittest openai_server.features.test_reasoning_content.TestReasoningContentAPI.test_streaming_separate_reasoning_true +python3 -m unittest openai_server.features.test_reasoning_content.TestReasoningContentAPI.test_streaming_separate_reasoning_true_stream_reasoning_false +python3 -m unittest openai_server.features.test_reasoning_content.TestReasoningContentAPI.test_nonstreaming_separate_reasoning_false +python3 -m unittest openai_server.features.test_reasoning_content.TestReasoningContentAPI.test_nonstreaming_separate_reasoning_true +python3 -m unittest openai_server.features.test_reasoning_content.TestReasoningContentStartup.test_nonstreaming +python3 -m unittest openai_server.features.test_reasoning_content.TestReasoningContentStartup.test_streaming +""" + +import sys +import unittest +from pathlib import Path + +import openai + +_TEST_DIR = Path(__file__).parent +sys.path.insert(0, str(_TEST_DIR.parent)) +from fixtures import popen_launch_workers_and_router +from util import ( + DEFAULT_REASONING_MODEL_PATH, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + kill_process_tree, +) + + +class TestReasoningContentAPI(CustomTestCase): + @classmethod + def setUpClass(cls): + # CHANGE: Launch gRPC router with integrated workers (single command) + cls.model = DEFAULT_REASONING_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-1234" + cls.cluster = popen_launch_workers_and_router( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + router_args=[ + "--reasoning-parser", + "deepseek_r1", + ], + num_workers=1, + tp_size=2, + ) + cls.base_url += "/v1" + + @classmethod + def tearDownClass(cls): + # Cleanup router and workers + kill_process_tree(cls.cluster["router"].pid) + for worker in cls.cluster.get("workers", []): + kill_process_tree(worker.pid) + + def test_streaming_separate_reasoning_false(self): + # Test streaming with separate_reasoning=False, reasoning_content should be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "stream": True, + "extra_body": {"separate_reasoning": False}, + } + response = client.chat.completions.create(**payload) + + reasoning_content = "" + content = "" + for chunk in response: + if chunk.choices[0].delta.content: + content += chunk.choices[0].delta.content + elif chunk.choices[0].delta.reasoning_content: + reasoning_content += chunk.choices[0].delta.reasoning_content + + assert len(reasoning_content) == 0 + assert len(content) > 0 + + def test_streaming_separate_reasoning_true(self): + # Test streaming with separate_reasoning=True, reasoning_content should not be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "stream": True, + "extra_body": {"separate_reasoning": True}, + } + response = client.chat.completions.create(**payload) + + reasoning_content = "" + content = "" + for chunk in response: + if chunk.choices[0].delta.content: + content += chunk.choices[0].delta.content + elif chunk.choices[0].delta.reasoning_content: + reasoning_content += chunk.choices[0].delta.reasoning_content + + assert len(reasoning_content) > 0 + assert len(content) > 0 + + def test_streaming_separate_reasoning_true_stream_reasoning_false(self): + # Test streaming with separate_reasoning=True, reasoning_content should not be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "stream": True, + "extra_body": {"separate_reasoning": True, "stream_reasoning": False}, + } + response = client.chat.completions.create(**payload) + + reasoning_content = "" + content = "" + first_chunk = False + for chunk in response: + if chunk.choices[0].delta.reasoning_content: + reasoning_content = chunk.choices[0].delta.reasoning_content + first_chunk = True + if chunk.choices[0].delta.content: + content += chunk.choices[0].delta.content + if not first_chunk: + reasoning_content = chunk.choices[0].delta.reasoning_content + first_chunk = True + if not first_chunk: + assert ( + not chunk.choices[0].delta.reasoning_content + or len(chunk.choices[0].delta.reasoning_content) == 0 + ) + assert len(reasoning_content) > 0 + assert len(content) > 0 + + def test_nonstreaming_separate_reasoning_false(self): + # Test non-streaming with separate_reasoning=False, reasoning_content should be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "extra_body": {"separate_reasoning": False}, + } + response = client.chat.completions.create(**payload) + + assert ( + not response.choices[0].message.reasoning_content + or len(response.choices[0].message.reasoning_content) == 0 + ) + assert len(response.choices[0].message.content) > 0 + + def test_nonstreaming_separate_reasoning_true(self): + # Test non-streaming with separate_reasoning=True, reasoning_content should not be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "extra_body": {"separate_reasoning": True}, + } + response = client.chat.completions.create(**payload) + + assert len(response.choices[0].message.reasoning_content) > 0 + assert len(response.choices[0].message.content) > 0 + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/fixtures.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/fixtures.py new file mode 100644 index 00000000..e00fedd1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/fixtures.py @@ -0,0 +1,342 @@ +""" +Fixtures for launching gRPC router + workers for e2e testing. + +This module provides fixtures for launching SGLang workers and gRPC router separately: + 1. Launch N SGLang workers with gRPC enabled + 2. Launch router pointing to those workers + +This approach gives more control and matches production deployment patterns. +""" + +import logging +import socket +import subprocess +import time +from typing import Optional + +import requests + +logger = logging.getLogger(__name__) + + +def find_free_port() -> int: + """Find an available port on localhost.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def wait_for_workers_ready( + router_url: str, + expected_workers: int, + timeout: int = 300, + api_key: Optional[str] = None, +) -> None: + """ + Wait for router to have all workers connected. + + Polls the /workers endpoint until the 'total' field matches expected_workers. + + Example response from /workers endpoint: + {"workers":[],"total":0,"stats":{"prefill_count":0,"decode_count":0,"regular_count":0}} + + Args: + router_url: Base URL of router (e.g., "http://127.0.0.1:30000") + expected_workers: Number of workers expected to be connected + timeout: Max seconds to wait + api_key: Optional API key for authentication + """ + start_time = time.time() + last_error = None + attempt = 0 + + headers = {} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + + with requests.Session() as session: + while time.time() - start_time < timeout: + attempt += 1 + elapsed = int(time.time() - start_time) + + # Log progress every 10 seconds + if elapsed > 0 and elapsed % 10 == 0 and attempt % 10 == 0: + logger.info( + f" Still waiting for workers... ({elapsed}/{timeout}s elapsed)" + ) + + try: + response = session.get( + f"{router_url}/workers", headers=headers, timeout=5 + ) + if response.status_code == 200: + data = response.json() + total_workers = data.get("total", 0) + + if total_workers == expected_workers: + logger.info( + f" All {expected_workers} workers connected after {elapsed}s" + ) + return + else: + last_error = f"Workers: {total_workers}/{expected_workers}" + else: + last_error = f"HTTP {response.status_code}" + except requests.ConnectionError: + last_error = "Connection refused (router not ready yet)" + except requests.Timeout: + last_error = "Timeout" + except requests.RequestException as e: + last_error = str(e) + except (ValueError, KeyError) as e: + last_error = f"Invalid response: {e}" + + time.sleep(1) + + raise TimeoutError( + f"Router at {router_url} did not get {expected_workers} workers within {timeout}s.\n" + f"Last status: {last_error}\n" + f"Hint: Run with SHOW_ROUTER_LOGS=1 to see startup logs" + ) + + +def popen_launch_workers_and_router( + model: str, + base_url: str, + timeout: int = 300, + num_workers: int = 2, + policy: str = "round_robin", + api_key: Optional[str] = None, + worker_args: Optional[list] = None, + router_args: Optional[list] = None, + tp_size: int = 1, + env: Optional[dict] = None, + stdout=None, + stderr=None, +) -> dict: + """ + Launch SGLang workers and gRPC router separately. + + This approach: + 1. Starts N SGLang workers with --grpc-mode flag + 2. Waits for workers to initialize (process startup) + 3. Starts a gRPC router pointing to those workers + 4. Waits for router health check to pass (router validates worker connectivity) + + This matches production deployment patterns better than the integrated approach. + + Args: + model: Model path (e.g., /home/ubuntu/models/llama-3.1-8b-instruct) + base_url: Base URL for router (e.g., "http://127.0.0.1:8080") + timeout: Timeout for server startup (default: 300s) + num_workers: Number of workers to launch + policy: Routing policy (round_robin, random, power_of_two, cache_aware) + api_key: Optional API key for router + worker_args: Additional arguments for workers (e.g., ["--context-len", "8192"]) + router_args: Additional arguments for router (e.g., ["--max-total-token", "1536"]) + tp_size: Tensor parallelism size for workers (default: 1) + env: Optional environment variables for workers (e.g., {"SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION": "256"}) + stdout: Optional file handle for worker stdout (default: subprocess.PIPE) + stderr: Optional file handle for worker stderr (default: subprocess.PIPE) + + Returns: + dict with: + - workers: list of worker process objects + - worker_urls: list of gRPC worker URLs + - router: router process object + - base_url: router URL (HTTP endpoint) + + Example: + >>> cluster = popen_launch_workers_and_router(model, base_url, num_workers=2) + >>> # Use cluster['base_url'] for HTTP requests + >>> # Cleanup: + >>> for worker in cluster['workers']: + >>> kill_process_tree(worker.pid) + >>> kill_process_tree(cluster['router'].pid) + """ + import os + + show_output = os.environ.get("SHOW_ROUTER_LOGS", "0") == "1" + + # Note: timeout parameter is used for router health check below + + # Parse router port from base_url + if ":" in base_url.split("//")[-1]: + router_port = int(base_url.split(":")[-1]) + else: + router_port = find_free_port() + + logger.info(f"\n{'='*70}") + logger.info(f"Launching gRPC cluster (separate workers + router)") + logger.info(f"{'='*70}") + logger.info(f" Model: {model}") + logger.info(f" Router port: {router_port}") + logger.info(f" Workers: {num_workers}") + logger.info(f" TP size: {tp_size}") + logger.info(f" Policy: {policy}") + + # Step 1: Launch workers with gRPC enabled + workers = [] + worker_urls = [] + + for i in range(num_workers): + worker_port = find_free_port() + worker_url = f"grpc://127.0.0.1:{worker_port}" + worker_urls.append(worker_url) + + logger.info(f"\n[Worker {i+1}/{num_workers}]") + logger.info(f" Port: {worker_port}") + logger.info(f" URL: {worker_url}") + + # Build worker command + worker_cmd = [ + "python3", + "-m", + "sglang.launch_server", + "--model-path", + model, + "--host", + "127.0.0.1", + "--port", + str(worker_port), + "--grpc-mode", # Enable gRPC for this worker + "--mem-fraction-static", + "0.8", + ] + + # Add TP size + if tp_size > 1: + worker_cmd.extend(["--tp-size", str(tp_size)]) + + # Add worker-specific args + if worker_args: + worker_cmd.extend(worker_args) + + # Launch worker with optional environment variables + if show_output: + worker_proc = subprocess.Popen( + worker_cmd, + env=env, + stdout=stdout, + stderr=stderr, + ) + else: + worker_proc = subprocess.Popen( + worker_cmd, + stdout=stdout if stdout is not None else subprocess.PIPE, + stderr=stderr if stderr is not None else subprocess.PIPE, + env=env, + ) + + workers.append(worker_proc) + logger.info(f" PID: {worker_proc.pid}") + + # Give workers a moment to start binding to ports + # The router will check worker health when it starts + logger.info(f"\nWaiting for {num_workers} workers to initialize (20s)...") + time.sleep(20) + + # Quick check: make sure worker processes are still alive + for i, worker in enumerate(workers): + if worker.poll() is not None: + logger.error( + f" ✗ Worker {i+1} died during startup (exit code: {worker.poll()})" + ) + # Cleanup: kill all workers + for w in workers: + try: + w.kill() + except: + pass + raise RuntimeError(f"Worker {i+1} failed to start") + + logger.info( + f"✓ All {num_workers} workers started (router will verify connectivity)" + ) + + # Step 2: Launch router pointing to workers + logger.info(f"\n[Router]") + logger.info(f" Port: {router_port}") + logger.info(f" Worker URLs: {', '.join(worker_urls)}") + + # Build router command + router_cmd = [ + "python3", + "-m", + "sglang_router.launch_router", + "--host", + "127.0.0.1", + "--port", + str(router_port), + "--prometheus-port", + "9321", + "--policy", + policy, + "--model-path", + model, + "--log-level", + "warn", + ] + + # Add worker URLs + router_cmd.append("--worker-urls") + router_cmd.extend(worker_urls) + + # Add API key + if api_key: + router_cmd.extend(["--api-key", api_key]) + + # Add router-specific args + if router_args: + router_cmd.extend(router_args) + + if show_output: + logger.info(f" Command: {' '.join(router_cmd)}") + + # Launch router + if show_output: + router_proc = subprocess.Popen(router_cmd) + else: + router_proc = subprocess.Popen( + router_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + logger.info(f" PID: {router_proc.pid}") + + # Wait for router to be ready + router_url = f"http://127.0.0.1:{router_port}" + logger.info(f"\nWaiting for router to start at {router_url}...") + + try: + wait_for_workers_ready( + router_url, expected_workers=num_workers, timeout=180, api_key=api_key + ) + logger.info(f"✓ Router ready at {router_url}") + except TimeoutError: + logger.error(f"✗ Router failed to start") + # Cleanup: kill router and all workers + try: + router_proc.kill() + except: + pass + for worker in workers: + try: + worker.kill() + except: + pass + raise + + logger.info(f"\n{'='*70}") + logger.info(f"✓ gRPC cluster ready!") + logger.info(f" Router: {router_url}") + logger.info(f" Workers: {len(workers)}") + logger.info(f"{'='*70}\n") + + return { + "workers": workers, + "worker_urls": worker_urls, + "router": router_proc, + "base_url": router_url, + } diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/function_call/test_openai_function_calling.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/function_call/test_openai_function_calling.py new file mode 100644 index 00000000..096a7cbc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/function_call/test_openai_function_calling.py @@ -0,0 +1,950 @@ +""" +gRPC Router E2E Test - Test Openai Function Calling + +This test file is REUSED from test/srt/openai_server/function_call/test_openai_function_calling.py +with minimal changes: + num_workers=2, +- Swap popen_launch_server() → popen_launch_workers_and_router() +- Update teardown to cleanup router + workers +- All test logic and assertions remain identical + +Run with: + pytest py_test/e2e_grpc/e2e_grpc/function_call/test_openai_function_calling.py -v +""" + +import json +import sys +import unittest +from pathlib import Path + +import openai + +_TEST_DIR = Path(__file__).parent +sys.path.insert(0, str(_TEST_DIR.parent)) +from fixtures import popen_launch_workers_and_router +from util import ( + DEFAULT_MODEL_PATH, + DEFAULT_SMALL_MODEL_PATH, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + get_tokenizer, + kill_process_tree, +) + + +class TestOpenAIServerFunctionCalling(CustomTestCase): + # NOTE: this system_message is for Llama3.2 system prompt. Without this, + # sometimes Llama3.2 gives a different tool call format such as: + # '<|python_tag|>{"type": "function", "function": "add", "parameters": {"a": "3", "b": "5"}}' + SYSTEM_MESSAGE = ( + "You are a helpful assistant with tool calling capabilities. " + "Only reply with a tool call if the function exists in the library provided by the user. " + "If it doesn't exist, just reply directly in natural language. " + "When you receive a tool call response, use the output to format an answer to the original user question. " + "You have access to the following functions. " + "To call a function, please respond with JSON for a function call. " + 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. ' + "Do not use variables.\n\n" + ) + + @classmethod + def setUpClass(cls): + # CHANGE: Launch gRPC router with integrated workers (single command) + # Using small model for function calling tests + cls.model = DEFAULT_SMALL_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + # Start the local OpenAI Server. If necessary, you can add other parameters such as --enable-tools. + cls.cluster = popen_launch_workers_and_router( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + router_args=[ + # If your server needs extra parameters to test function calling, please add them here. + "--tool-call-parser", + "llama", + ], + num_workers=1, + tp_size=2, + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + @classmethod + def tearDownClass(cls): + # Cleanup router and workers + kill_process_tree(cls.cluster["router"].pid) + for worker in cls.cluster.get("workers", []): + kill_process_tree(worker.pid) + + def test_function_calling_format(self): + """ + Test: Whether the function call format returned by the AI is correct. + When returning a tool call, message.content should be None, and tool_calls should be a list. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "add", + "description": "Compute the sum of two numbers", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "integer", + "description": "A number", + }, + "b": { + "type": "integer", + "description": "A number", + }, + }, + "required": ["a", "b"], + }, + }, + } + ] + + messages = [ + {"role": "system", "content": self.SYSTEM_MESSAGE}, + {"role": "user", "content": "Compute (3+5)"}, + ] + response = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=False, + tools=tools, + ) + + tool_calls = response.choices[0].message.tool_calls + + assert ( + isinstance(tool_calls, list) and len(tool_calls) > 0 + ), "tool_calls should be a non-empty list" + + function_name = tool_calls[0].function.name + assert function_name == "add", "Function name should be 'add'" + + # This unit test is too difficult for default model. Mark it as optional unit tests so it won't trigger unless specified. + def _test_function_calling_multiturn(self): + """ + Test: Whether the function call format returned by the AI is correct. + When returning a tool call, message.content should be None, and tool_calls should be a list. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "add", + "description": "Compute the sum of two numbers", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "integer", + "description": "A number", + }, + "b": { + "type": "integer", + "description": "A number", + }, + }, + "required": ["a", "b"], + }, + }, + } + ] + + messages = [{"role": "user", "content": "Compute (3+5)"}] + + response = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=False, + tools=tools, + ) + + tool_call = response.choices[0].message.tool_calls[0] + function_name = tool_call.function.name + assert function_name == "add", "Function name should be 'add'" + function_arguments = tool_call.function.arguments + function_arguments = json.loads(tool_call.function.arguments) + assert function_arguments in [ + {"a": 3, "b": 5}, + {"a": "3", "b": "5"}, + ], f"Unexpected function arguments: {function_arguments}" + + messages.append(response.choices[0].message) + messages.append( + { + "role": "tool", + "tool_call_id": tool_call.id, + "content": "8", + "name": function_name, + } + ) + + final_response = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=False, + tools=tools, + ) + + assert ( + "8" in final_response.choices[0].message.content + ), "tool_call response should have the sum 8 in the content" + + def test_function_calling_streaming_simple(self): + """ + Test: Whether the function name can be correctly recognized in streaming mode. + - Expect a function call to be found, and the function name to be correct. + - Verify that streaming mode returns at least multiple chunks. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city to find the weather for", + }, + "unit": { + "type": "string", + "description": "Weather unit (celsius or fahrenheit)", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["city", "unit"], + }, + }, + } + ] + + messages = [ + {"role": "system", "content": self.SYSTEM_MESSAGE}, + { + "role": "user", + "content": "What is the temperature in Paris in celsius??", + }, + ] + + response_stream = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=True, + tools=tools, + ) + + chunks = list(response_stream) + self.assertTrue(len(chunks) > 0, "Streaming should return at least one chunk") + + found_function_name = False + for chunk in chunks: + choice = chunk.choices[0] + # Check whether the current chunk contains tool_calls + if choice.delta.tool_calls: + tool_call = choice.delta.tool_calls[0] + if tool_call.function.name: + self.assertEqual( + tool_call.function.name, + "get_current_weather", + "Function name should be 'get_current_weather'", + ) + found_function_name = True + break + + self.assertTrue( + found_function_name, + "Target function name 'get_current_weather' was not found in the streaming chunks", + ) + + finish_reason = chunks[-1].choices[0].finish_reason + self.assertEqual( + finish_reason, + "tool_calls", + "Final response of function calling should have finish_reason 'tool_calls'", + ) + + def test_function_calling_streaming_args_parsing(self): + """ + Test: Whether the function call arguments returned in streaming mode can be correctly concatenated into valid JSON. + - The user request requires multiple parameters. + - AI may return the arguments in chunks that need to be concatenated. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "add", + "description": "Compute the sum of two integers", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "integer", + "description": "First integer", + }, + "b": { + "type": "integer", + "description": "Second integer", + }, + }, + "required": ["a", "b"], + }, + "strict": True, # Llama-3.2-1B is flaky in tool call. It won't always respond with parameters unless we set strict. + }, + } + ] + + messages = [ + {"role": "system", "content": self.SYSTEM_MESSAGE}, + {"role": "user", "content": "Please sum 5 and 7, just call the function."}, + ] + + response_stream = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.9, + top_p=0.9, + stream=True, + tools=tools, + ) + + argument_fragments = [] + chunks = list(response_stream) + function_name = None + for chunk in chunks: + choice = chunk.choices[0] + if choice.delta.tool_calls: + tool_call = choice.delta.tool_calls[0] + # Record the function name on first occurrence + function_name = tool_call.function.name or function_name + # In case of multiple chunks, JSON fragments may need to be concatenated + if tool_call.function.arguments is not None: + argument_fragments.append(tool_call.function.arguments) + + self.assertEqual(function_name, "add", "Function name should be 'add'") + joined_args = "".join(argument_fragments) + self.assertTrue( + len(joined_args) > 0, + "No parameter fragments were returned in the function call", + ) + + finish_reason = chunks[-1].choices[0].finish_reason + self.assertEqual( + finish_reason, + "tool_calls", + "Final response of function calling should have finish_reason 'tool_calls'", + ) + + # Check whether the concatenated JSON is valid + try: + args_obj = json.loads(joined_args) + except json.JSONDecodeError: + self.fail( + "The concatenated tool call arguments are not valid JSON, parsing failed" + ) + + self.assertIn("a", args_obj, "Missing parameter 'a'") + self.assertIn("b", args_obj, "Missing parameter 'b'") + self.assertEqual(str(args_obj["a"]), "5", "Parameter a should be 5") + self.assertEqual(str(args_obj["b"]), "7", "Parameter b should be 7") + + @unittest.skip( + "Skipping function call strict test as it is not supported by the router" + ) + def test_function_call_strict(self): + """ + Test: Whether the strict mode of function calling works as expected. + - When strict mode is enabled, the AI should not return a function call if the function name is not recognized. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "sub", + "description": "Compute the difference of two integers", + "parameters": { + "type": "object", + "properties": { + "int_a": { + "type": "integer", + "description": "First integer", + }, + "int_b": { + "type": "integer", + "description": "Second integer", + }, + }, + "required": ["int_a", "int_b"], + }, + "strict": True, + }, + } + ] + + messages = [ + {"role": "user", "content": "Please compute 5 - 7, using your tool."} + ] + response = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=False, + tools=tools, + ) + + tool_calls = response.choices[0].message.tool_calls + function_name = tool_calls[0].function.name + arguments = tool_calls[0].function.arguments + args_obj = json.loads(arguments) + + self.assertEqual(function_name, "sub", "Function name should be 'sub'") + self.assertEqual(str(args_obj["int_a"]), "5", "Parameter int_a should be 5") + self.assertEqual(str(args_obj["int_b"]), "7", "Parameter int_b should be 7") + + def test_function_call_required(self): + """ + Test: Whether tool_choice: "required" works as expected + - When tool_choice == "required", the model should return one or more tool_calls. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "sub", + "description": "Compute the difference of two integers", + "parameters": { + "type": "object", + "properties": { + "int_a": { + "type": "integer", + "description": "First integer", + }, + "int_b": { + "type": "integer", + "description": "Second integer", + }, + }, + "required": ["int_a", "int_b"], + }, + "strict": True, + }, + }, + { + "type": "function", + "function": { + "name": "get_weather", + "description": "use this to get latest weather information for a city given its name", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "name of the city to get weather for", + } + }, + "required": ["city"], + }, + }, + }, + ] + + messages = [{"role": "user", "content": "What is the capital of France?"}] + response = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=False, + tools=tools, + tool_choice="required", + ) + + tool_calls = response.choices[0].message.tool_calls + self.assertIsNotNone(tool_calls, "No tool_calls in the response") + function_name = tool_calls[0].function.name + arguments = tool_calls[0].function.arguments + args_obj = json.loads(arguments) + + self.assertEqual( + function_name, + "get_weather", + f"Function name should be 'get_weather', got: {function_name}", + ) + self.assertIn( + "city", args_obj, f"Function arguments should have 'city', got: {args_obj}" + ) + + # Make the test more robust by checking type and accepting valid responses + city_value = args_obj["city"] + self.assertIsInstance( + city_value, + str, + f"Parameter city should be a string, got: {type(city_value)}", + ) + self.assertTrue( + "Paris" in city_value or "France" in city_value, + f"Parameter city should contain either 'Paris' or 'France', got: {city_value}", + ) + + def test_function_call_specific(self): + """ + Test: Whether tool_choice: ToolChoice works as expected + - When tool_choice is a specific ToolChoice, the model should return one or more tool_calls. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "sub", + "description": "Compute the difference of two integers", + "parameters": { + "type": "object", + "properties": { + "int_a": { + "type": "integer", + "description": "First integer", + }, + "int_b": { + "type": "integer", + "description": "Second integer", + }, + }, + "required": ["int_a", "int_b"], + }, + "strict": True, + }, + }, + { + "type": "function", + "function": { + "name": "get_weather", + "description": "use this to get latest weather information for a city given its name", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "name of the city to get weather for", + } + }, + "required": ["city"], + }, + }, + }, + ] + + messages = [{"role": "user", "content": "What is the capital of France?"}] + response = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=False, + tools=tools, + tool_choice={"type": "function", "function": {"name": "get_weather"}}, + ) + + tool_calls = response.choices[0].message.tool_calls + self.assertIsNotNone(tool_calls, "No tool_calls in the response") + function_name = tool_calls[0].function.name + arguments = tool_calls[0].function.arguments + args_obj = json.loads(arguments) + + self.assertEqual( + function_name, "get_weather", "Function name should be 'get_weather'" + ) + self.assertIn("city", args_obj, "Function arguments should have 'city'") + + def test_streaming_multiple_choices_finish_reason(self): + """ + Test: Verify that each choice gets its own finish_reason chunk in streaming mode with n > 1. + This tests the fix for the bug where only the last index got a finish_reason chunk. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["location"], + }, + }, + } + ] + + messages = [ + {"role": "user", "content": "What is the weather like in Los Angeles?"} + ] + + # Request with n=2 to get multiple choices + response_stream = client.chat.completions.create( + model=self.model, + messages=messages, + max_tokens=2048, + temperature=0.8, + stream=True, + tools=tools, + tool_choice="required", # Force tool calls + n=2, # Multiple choices + ) + + chunks = list(response_stream) + + # Track finish_reason chunks for each index + finish_reason_chunks = {} + for chunk in chunks: + if chunk.choices: + for choice in chunk.choices: + if choice.finish_reason is not None: + index = choice.index + if index not in finish_reason_chunks: + finish_reason_chunks[index] = [] + finish_reason_chunks[index].append(choice.finish_reason) + + # Verify we got finish_reason chunks for both indices + self.assertEqual( + len(finish_reason_chunks), + 2, + f"Expected finish_reason chunks for 2 indices, got {len(finish_reason_chunks)}", + ) + + # Verify both index 0 and 1 have finish_reason + self.assertIn( + 0, finish_reason_chunks, "Missing finish_reason chunk for index 0" + ) + self.assertIn( + 1, finish_reason_chunks, "Missing finish_reason chunk for index 1" + ) + + # Verify the finish_reason is "tool_calls" since we forced tool calls + for index, reasons in finish_reason_chunks.items(): + self.assertEqual( + reasons[-1], # Last finish_reason for this index + "tool_calls", + f"Expected finish_reason 'tool_calls' for index {index}, got {reasons[-1]}", + ) + + def test_function_calling_streaming_no_tool_call(self): + """ + Test: Whether the finish_reason is stop in streaming mode when no tool call is given. + - Expect no function call to be found. + - Verify that finish_reason is stop + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city to find the weather for", + }, + "unit": { + "type": "string", + "description": "Weather unit (celsius or fahrenheit)", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["city", "unit"], + }, + }, + } + ] + + messages = [{"role": "user", "content": "Who are you?"}] + + response_stream = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=True, + tools=tools, + tool_choice="none", + ) + + chunks = list(response_stream) + self.assertTrue(len(chunks) > 0, "Streaming should return at least one chunk") + + found_tool_call = False + for chunk in chunks: + choice = chunk.choices[0] + # Check whether the current chunk contains tool_calls + found_tool_call = choice.delta.tool_calls is not None + + self.assertFalse( + found_tool_call, + "Shouldn't have any tool_call in the streaming chunks", + ) + + finish_reason = chunks[-1].choices[0].finish_reason + self.assertEqual( + finish_reason, + "stop", + "Final response of no function calling should have finish_reason 'stop'", + ) + + def test_streaming_multiple_choices_without_tools(self): + """ + Test: Verify that each choice gets its own finish_reason chunk without tool calls. + This tests the fix for regular content streaming with multiple choices. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + messages = [{"role": "user", "content": "Say hello in one word."}] + + # Request with n=2 to get multiple choices, no tools + response_stream = client.chat.completions.create( + model=self.model, + messages=messages, + temperature=0.8, + stream=True, + max_tokens=10, # Keep it short + n=2, # Multiple choices + ) + + chunks = list(response_stream) + + # Track finish_reason chunks for each index + finish_reason_chunks = {} + for chunk in chunks: + if chunk.choices: + for choice in chunk.choices: + if choice.finish_reason is not None: + index = choice.index + if index not in finish_reason_chunks: + finish_reason_chunks[index] = [] + finish_reason_chunks[index].append(choice.finish_reason) + + # Verify we got finish_reason chunks for both indices + self.assertEqual( + len(finish_reason_chunks), + 2, + f"Expected finish_reason chunks for 2 indices, got {len(finish_reason_chunks)}", + ) + + # Verify both index 0 and 1 have finish_reason + self.assertIn( + 0, finish_reason_chunks, "Missing finish_reason chunk for index 0" + ) + self.assertIn( + 1, finish_reason_chunks, "Missing finish_reason chunk for index 1" + ) + + # Verify the finish_reason is "stop" (regular completion) + for index, reasons in finish_reason_chunks.items(): + self.assertIn( + reasons[-1], + ["stop", "length"], # Could be either depending on how model responds + f"Expected finish_reason 'stop' or 'length' for index {index}, got {reasons[-1]}", + ) + + +class TestOpenAIPythonicFunctionCalling(CustomTestCase): + PYTHONIC_TOOLS = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a given location.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The name of the city or location.", + } + }, + "required": ["location"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_tourist_attractions", + "description": "Get a list of top tourist attractions for a given city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The name of the city to find attractions for.", + } + }, + "required": ["city"], + }, + }, + }, + ] + + PYTHONIC_MESSAGES = [ + { + "role": "system", + "content": ( + "You are a travel assistant. " + "When asked to call functions, ALWAYS respond ONLY with a python list of function calls, " + "using this format: [func_name1(param1=value1, param2=value2), func_name2(param=value)]. " + "Do NOT use JSON, do NOT use variables, do NOT use any other format. " + "Here is an example:\n" + '[get_weather(location="Paris"), get_tourist_attractions(city="Paris")]' + ), + }, + { + "role": "user", + "content": ( + "I'm planning a trip to Tokyo next week. What's the weather like and what are some top tourist attractions? " + "Propose parallel tool calls at once, using the python list of function calls format as shown above." + ), + }, + ] + + @classmethod + def setUpClass(cls): + # CHANGE: Launch gRPC router with integrated workers (single command) + cls.model = DEFAULT_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.cluster = popen_launch_workers_and_router( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + router_args=[ + "--tool-call-parser", + "pythonic", + ], + num_workers=1, + tp_size=2, + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + @classmethod + def tearDownClass(cls): + # Cleanup router and workers + kill_process_tree(cls.cluster["router"].pid) + for worker in cls.cluster.get("workers", []): + kill_process_tree(worker.pid) + + def test_pythonic_tool_call_prompt(self): + """ + Test: Explicit prompt for pythonic tool call format without chat template. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.chat.completions.create( + model=self.model, + messages=self.PYTHONIC_MESSAGES, + tools=self.PYTHONIC_TOOLS, + temperature=0.1, + stream=False, + ) + tool_calls = response.choices[0].message.tool_calls + self.assertIsInstance(tool_calls, list, "No tool_calls found") + self.assertGreaterEqual(len(tool_calls), 1) + names = [tc.function.name for tc in tool_calls] + self.assertTrue( + "get_weather" in names or "get_tourist_attractions" in names, + f"Function name '{names}' should container either 'get_weather' or 'get_tourist_attractions'", + ) + + def test_pythonic_tool_call_streaming(self): + """ + Test: Streaming pythonic tool call format; assert tool_call index is present. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response_stream = client.chat.completions.create( + model=self.model, + messages=self.PYTHONIC_MESSAGES, + tools=self.PYTHONIC_TOOLS, + temperature=0.1, + stream=True, + ) + found_tool_calls = False + found_index = False + found_names = set() + for chunk in response_stream: + choice = chunk.choices[0] + if getattr(choice.delta, "tool_calls", None): + found_tool_calls = True + tool_call = choice.delta.tool_calls[0] + if hasattr(tool_call, "index") or ( + isinstance(tool_call, dict) and "index" in tool_call + ): + found_index = True + found_names.add(str(tool_call.function.name)) + + self.assertTrue(found_tool_calls, "No tool_calls found in streaming response") + self.assertTrue(found_index, "No index field found in any streamed tool_call") + self.assertTrue( + "get_weather" in found_names or "get_tourist_attractions" in found_names, + f"Function name '{found_names}' should container either 'get_weather' or 'get_tourist_attractions'", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/function_call/test_tool_choice.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/function_call/test_tool_choice.py new file mode 100644 index 00000000..4e166852 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/function_call/test_tool_choice.py @@ -0,0 +1,795 @@ +""" +Test script for tool_choice functionality in SGLang +Tests: required, auto, and specific function choices in both streaming and non-streaming modes + +# To run the tests, use the following command: +# +# python3 -m unittest openai_server.function_call.test_tool_choice +""" + +import json +import sys +import unittest +from pathlib import Path + +import openai + +_TEST_DIR = Path(__file__).parent +sys.path.insert(0, str(_TEST_DIR.parent)) +from fixtures import popen_launch_workers_and_router +from util import ( + DEFAULT_MISTRAL_FUNCTION_CALLING_MODEL_PATH, + DEFAULT_QWEN_FUNCTION_CALLING_MODEL_PATH, + DEFAULT_SMALL_MODEL_PATH, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + get_tokenizer, + kill_process_tree, +) + + +class TestToolChoiceLlama32(CustomTestCase): + + @classmethod + def setUpClass(cls): + # CHANGE: Launch gRPC router with integrated workers (single command) + # Mark flaky tests for this model + cls.flaky_tests = { + "test_multi_tool_scenario_auto", + "test_multi_tool_scenario_required", + } + + # Use a model that supports function calling + cls.model = DEFAULT_SMALL_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + # Start the local OpenAI Server with tool calling support + cls.cluster = popen_launch_workers_and_router( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + router_args=[ + "--tool-call-parser", + "llama", # Default parser for the test model + ], + num_workers=1, + tp_size=2, + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + @classmethod + def tearDownClass(cls): + # Cleanup router and workers + kill_process_tree(cls.cluster["router"].pid) + for worker in cls.cluster.get("workers", []): + kill_process_tree(worker.pid) + + def setUp(self): + self.client = openai.Client(base_url=self.base_url, api_key=self.api_key) + self.model_name = self.client.models.list().data[0].id + + def _is_flaky_test(self): + """Check if the current test is marked as flaky for this class""" + return ( + hasattr(self.__class__, "flaky_tests") + and self._testMethodName in self.__class__.flaky_tests + ) + + def get_test_tools(self): + """Get the test tools for function calling""" + return [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "use this to get latest weather information for a city given its name", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "name of the city to get weather for", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["city"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_pokemon_info", + "description": "get detailed information about a pokemon given its name", + "parameters": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "name of the pokemon to get info for", + } + }, + "required": ["name"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "make_next_step_decision", + "description": "You will be given a trace of thinking process in the following format.\n\nQuestion: the input question you must answer\nTOOL: think about what to do, and choose a tool to use ONLY IF there are defined tools. \n You should never call the same tool with the same input twice in a row.\n If the previous conversation history already contains the information that can be retrieved from the tool, you should not call the tool again.\nOBSERVATION: the result of the tool call, NEVER include this in your response, this information will be provided\n... (this TOOL/OBSERVATION can repeat N times)\nANSWER: If you know the answer to the original question, require for more information,\n or you don't know the answer and there are no defined tools or all available tools are not helpful, respond with the answer without mentioning anything else.\n If the previous conversation history already contains the answer, respond with the answer right away.\n\n If no tools are configured, naturally mention this limitation while still being helpful. Briefly note that adding tools in the agent configuration would expand capabilities.\n\nYour task is to respond with the next step to take, based on the traces, \nor answer the question if you have enough information.", + "parameters": { + "type": "object", + "properties": { + "decision": { + "type": "string", + "description": 'The next step to take, it must be either "TOOL" or "ANSWER". If the previous conversation history already contains the information that can be retrieved from the tool, you should not call the tool again. If there are no defined tools, you should not return "TOOL" in your response.', + }, + "content": { + "type": "string", + "description": 'The content of the next step. If the decision is "TOOL", this should be a short and concise reasoning of why you chose the tool, MUST include the tool name. If the decision is "ANSWER", this should be the answer to the question. If no tools are available, integrate this limitation conversationally without sounding scripted.', + }, + }, + "required": ["decision", "content"], + }, + }, + }, + ] + + def get_test_messages(self): + """Get test messages that should trigger tool usage""" + return [ + { + "role": "user", + "content": "Answer the following questions as best you can:\n\nYou will be given a trace of thinking process in the following format.\n\nQuestion: the input question you must answer\nTOOL: think about what to do, and choose a tool to use ONLY IF there are defined tools\nOBSERVATION: the result of the tool call or the observation of the current task, NEVER include this in your response, this information will be provided\n... (this TOOL/OBSERVATION can repeat N times)\nANSWER: If you know the answer to the original question, require for more information, \nif the previous conversation history already contains the answer, \nor you don't know the answer and there are no defined tools or all available tools are not helpful, respond with the answer without mentioning anything else.\nYou may use light Markdown formatting to improve clarity (e.g. lists, **bold**, *italics*), but keep it minimal and unobtrusive.\n\nYour task is to respond with the next step to take, based on the traces, \nor answer the question if you have enough information.\n\nQuestion: what is the weather in top 5 populated cities in the US in celsius?\n\nTraces:\n\n\nThese are some additional instructions that you should follow:", + } + ] + + def get_travel_tools(self): + """Get tools for travel assistant scenario that should trigger multiple tool calls""" + return [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a given location.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The name of the city or location.", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["location"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_tourist_attractions", + "description": "Get a list of top tourist attractions for a given city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The name of the city to find attractions for.", + } + }, + "required": ["city"], + }, + }, + }, + ] + + def get_travel_messages(self): + """Get travel assistant messages that should trigger multiple tool calls""" + return [ + { + "content": "You are a travel assistant providing real-time weather updates and top tourist attractions.", + "role": "system", + }, + { + "content": "I'm planning a trip to Tokyo next week. What's the weather like? What are the most amazing sights?", + "role": "user", + }, + ] + + def test_tool_choice_auto_non_streaming(self): + """Test tool_choice='auto' in non-streaming mode""" + tools = self.get_test_tools() + messages = self.get_test_messages() + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + tools=tools, + tool_choice="auto", + stream=False, + ) + + self.assertIsNotNone(response.choices[0].message) + # With auto, tool calls are optional + + def test_tool_choice_auto_streaming(self): + """Test tool_choice='auto' in streaming mode""" + tools = self.get_test_tools() + messages = self.get_test_messages() + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + tools=tools, + tool_choice="auto", + stream=True, + ) + + # Collect streaming response + content_chunks = [] + tool_call_chunks = [] + + for chunk in response: + if chunk.choices[0].delta.content: + content_chunks.append(chunk.choices[0].delta.content) + elif chunk.choices[0].delta.tool_calls: + tool_call_chunks.extend(chunk.choices[0].delta.tool_calls) + + # Should complete without errors + self.assertIsInstance(content_chunks, list) + self.assertIsInstance(tool_call_chunks, list) + + def test_tool_choice_required_non_streaming(self): + """Test tool_choice='required' in non-streaming mode""" + tools = self.get_test_tools() + messages = self.get_test_messages() + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + temperature=0.2, + tools=tools, + tool_choice="required", + stream=False, + ) + + # With required, we should get tool calls + tool_calls = response.choices[0].message.tool_calls + self.assertIsNotNone(tool_calls) + self.assertGreater(len(tool_calls), 0) + + def test_tool_choice_required_streaming(self): + """Test tool_choice='required' in streaming mode""" + tools = self.get_test_tools() + messages = self.get_test_messages() + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + tools=tools, + tool_choice="required", + stream=True, + ) + + # Collect streaming response + tool_call_chunks = [] + + for chunk in response: + if chunk.choices[0].delta.tool_calls: + tool_call_chunks.extend(chunk.choices[0].delta.tool_calls) + + # With required, we should get tool call chunks + self.assertGreater(len(tool_call_chunks), 0) + + def test_tool_choice_specific_function_non_streaming(self): + """Test tool_choice with specific function in non-streaming mode""" + tools = self.get_test_tools() + messages = self.get_test_messages() + + tool_choice = {"type": "function", "function": {"name": "get_weather"}} + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + tools=tools, + tool_choice=tool_choice, + stream=False, + ) + + # Should call the specific function + tool_calls = response.choices[0].message.tool_calls + self.assertIsNotNone(tool_calls) + # Our messages ask the top 5 populated cities in the US, so the model could get 5 tool calls + self.assertGreaterEqual(len(tool_calls), 1) + for tool_call in tool_calls: + self.assertEqual(tool_call.function.name, "get_weather") + + def test_tool_choice_specific_function_streaming(self): + """Test tool_choice with specific function in streaming mode""" + tools = self.get_test_tools() + messages = self.get_test_messages() + + tool_choice = {"type": "function", "function": {"name": "get_weather"}} + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + tools=tools, + tool_choice=tool_choice, + stream=True, + ) + + # Collect streaming response + tool_call_chunks = [] + + for chunk in response: + if chunk.choices[0].delta.tool_calls: + tool_call_chunks.extend(chunk.choices[0].delta.tool_calls) + + # Should get tool call chunks for the specific function + self.assertGreater(len(tool_call_chunks), 0) + + # Find function name in chunks + found_name = None + for chunk in tool_call_chunks: + if chunk.function and chunk.function.name: + found_name = chunk.function.name + break + + self.assertEqual(found_name, "get_weather") + + def test_required_streaming_arguments_chunks_json(self): + """In streaming required mode, complete tool call arguments should be valid JSON when all chunks are combined""" + tools = self.get_test_tools() + messages = self.get_test_messages() + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=1024, + temperature=0.1, + tools=tools, + tool_choice="required", + stream=True, + ) + + # Collect all tool call chunks and reconstruct complete tool calls + tool_calls_by_index = {} + for chunk in response: + if chunk.choices[0].delta.tool_calls: + for tool_call_delta in chunk.choices[0].delta.tool_calls: + tool_index = tool_call_delta.index + + # Initialize tool call if not seen before + if tool_index not in tool_calls_by_index: + tool_calls_by_index[tool_index] = { + "id": tool_call_delta.id, + "type": "function", + "function": {"name": "", "arguments": ""}, + } + + # Update function name if present (first chunk) + if tool_call_delta.function and tool_call_delta.function.name: + tool_calls_by_index[tool_index]["function"][ + "name" + ] = tool_call_delta.function.name + + # Accumulate arguments (all chunks) + if tool_call_delta.function and tool_call_delta.function.arguments: + tool_calls_by_index[tool_index]["function"][ + "arguments" + ] += tool_call_delta.function.arguments + + self.assertGreater(len(tool_calls_by_index), 0) + + # Validate that complete tool calls have valid JSON arguments + for tool_call in tool_calls_by_index.values(): + self.assertIsNotNone(tool_call["function"]["name"]) + self.assertIsNotNone(tool_call["function"]["arguments"]) + + # The complete arguments should be valid JSON + try: + args = json.loads(tool_call["function"]["arguments"]) + self.assertIsInstance(args, dict) + except json.JSONDecodeError: + self.fail( + f"Invalid JSON in complete tool call arguments: {tool_call['function']['arguments']}" + ) + + def test_complex_parameters_required_non_streaming(self): + """Validate complex nested parameter schemas in non-streaming required mode""" + complex_tools = [ + { + "type": "function", + "function": { + "name": "analyze_data", + "description": "Analyze complex data structures", + "parameters": { + "type": "object", + "properties": { + "data": { + "type": "object", + "properties": { + "metrics": { + "type": "array", + "items": {"type": "string"}, + }, + "config": { + "type": "object", + "properties": { + "threshold": {"type": "number"}, + "enabled": {"type": "boolean"}, + }, + }, + }, + "required": ["metrics"], + }, + "options": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "value": {"type": "string"}, + }, + }, + }, + }, + "required": ["data"], + }, + }, + } + ] + + messages = [ + { + "role": "user", + "content": "Analyze some data with metrics and configuration", + } + ] + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=1024, + temperature=0.1, + tools=complex_tools, + tool_choice="required", + stream=False, + ) + + tool_calls = response.choices[0].message.tool_calls + self.assertIsNotNone(tool_calls) + self.assertGreater(len(tool_calls), 0) + + for tool_call in tool_calls: + self.assertEqual(tool_call.function.name, "analyze_data") + try: + args = json.loads(tool_call.function.arguments) + self.assertIsInstance(args, dict) + self.assertIn("data", args) + self.assertIsInstance(args["data"], dict) + except json.JSONDecodeError: + self.fail( + f"Invalid JSON in complex tool call arguments: {tool_call.function.arguments}" + ) + + def test_multi_tool_scenario_auto(self): + """Test multi-tool scenario with tool_choice='auto'""" + tools = self.get_travel_tools() + messages = self.get_travel_messages() + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + temperature=0.2, + tools=tools, + tool_choice="auto", + stream=False, + ) + + # Should complete without errors + self.assertIsNotNone(response.choices[0].message) + + tool_calls = response.choices[0].message.tool_calls + expected_functions = {"get_weather", "get_tourist_attractions"} + + if self._is_flaky_test(): + # For flaky tests, just verify all called functions are available tools + if tool_calls: + available_names = [tool["function"]["name"] for tool in tools] + for call in tool_calls: + self.assertIn(call.function.name, available_names) + else: + # For non-flaky tests, enforce strict requirements + self.assertIsNotNone(tool_calls, "Expected tool calls but got none") + self.assertEqual( + len(tool_calls), 2, f"Expected 2 tool calls, got {len(tool_calls)}" + ) + + called_functions = {call.function.name for call in tool_calls} + self.assertEqual( + called_functions, + expected_functions, + f"Expected functions {expected_functions}, got {called_functions}", + ) + + def test_multi_tool_scenario_required(self): + """Test multi-tool scenario with tool_choice='required'""" + tools = self.get_travel_tools() + messages = self.get_travel_messages() + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + temperature=0.2, + tools=tools, + tool_choice="required", + stream=False, + ) + + # With required, we should get at least one tool call + tool_calls = response.choices[0].message.tool_calls + self.assertIsNotNone(tool_calls) + self.assertGreater(len(tool_calls), 0) + + # Verify all called functions are available tools + available_names = [tool["function"]["name"] for tool in tools] + expected_functions = {"get_weather", "get_tourist_attractions"} + + for tool_call in tool_calls: + self.assertIsNotNone(tool_call.function.name) + self.assertIsNotNone(tool_call.function.arguments) + + if self._is_flaky_test(): + # For flaky tests, just ensure basic functionality works + self.assertGreater( + len(tool_calls), + 0, + f"Expected at least 1 tool call, got {len(tool_calls)}", + ) + for call in tool_calls: + self.assertIn(call.function.name, available_names) + else: + # For non-flaky tests, enforce strict requirements + self.assertEqual( + len(tool_calls), 2, f"Expected 2 tool calls, got {len(tool_calls)}" + ) + + called_functions = {call.function.name for call in tool_calls} + self.assertEqual( + called_functions, + expected_functions, + f"Expected functions {expected_functions}, got {called_functions}", + ) + + def test_error_handling_invalid_tool_choice(self): + """Test error handling for invalid tool_choice""" + tools = self.get_test_tools() + messages = self.get_test_messages() + + # Test with invalid function name + tool_choice = {"type": "function", "function": {"name": "nonexistent_function"}} + + # Expect a 400 BadRequestError to be raised for invalid tool_choice + with self.assertRaises(openai.BadRequestError) as context: + self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + tools=tools, + tool_choice=tool_choice, + stream=False, + ) + + # Verify the error message contains the expected text + self.assertIn( + "function 'nonexistent_function' not found in", + str(context.exception), + ) + + def test_invalid_tool_missing_name(self): + """Test what happens when user doesn't provide a tool name in request""" + # Test with malformed JSON in tool parameters - missing required "name" field + invalid_tools = [ + { + "type": "function", + "function": { + # Missing required "name" field + "description": "Test function with invalid schema", + "parameters": { + "type": "object", + "properties": { + "test_field": { + "type": "string", + "description": "Test field", + } + }, + "required": ["test_field"], + }, + }, + } + ] + + messages = [ + { + "role": "user", + "content": "Test the function", + } + ] + + # Should raise BadRequestError due to missing required 'name' field + with self.assertRaises(openai.BadRequestError) as context: + self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=100, + temperature=0.1, + tools=invalid_tools, + tool_choice="required", + stream=False, + ) + + # Verify the error message indicates missing name field + error_msg = str(context.exception).lower() + self.assertIn("name", error_msg) + + def test_conflicting_defs_required_tool_choice(self): + """Test that conflicting $defs with required tool_choice returns 400 error""" + conflicting_tools = [ + { + "type": "function", + "function": { + "name": "tool1", + "description": "Tool 1 with conflicting $defs", + "parameters": { + "type": "object", + "properties": { + "data": {"$ref": "#/$defs/DataType"}, + }, + "required": ["data"], + "$defs": { + "DataType": { + "type": "object", + "properties": {"value": {"type": "string"}}, + "required": ["value"], + }, + }, + }, + }, + }, + { + "type": "function", + "function": { + "name": "tool2", + "description": "Tool 2 with conflicting $defs", + "parameters": { + "type": "object", + "properties": { + "data": {"$ref": "#/$defs/DataType"}, + }, + "required": ["data"], + "$defs": { + "DataType": { # Different definition for DataType + "type": "object", + "properties": {"value": {"type": "number"}}, + "required": ["value"], + }, + }, + }, + }, + }, + ] + + messages = [ + { + "role": "user", + "content": "Test the conflicting tools", + } + ] + + # Should raise BadRequestError due to conflicting $defs + with self.assertRaises(openai.BadRequestError) as context: + self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=100, + temperature=0.1, + tools=conflicting_tools, + tool_choice="required", + stream=False, + ) + + # Verify the error message indicates conflicting tool definitions + error_msg = str(context.exception).lower() + self.assertIn("invalid tool configuration", error_msg) + self.assertIn("not supported", error_msg) + + +class TestToolChoiceQwen25(TestToolChoiceLlama32): + """Test tool_choice functionality with Qwen2.5 model""" + + @classmethod + def setUpClass(cls): + # CHANGE: Launch gRPC router with integrated workers (single command) + cls.flaky_tests = {} + + cls.model = DEFAULT_QWEN_FUNCTION_CALLING_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + cls.cluster = popen_launch_workers_and_router( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + router_args=[ + "--tool-call-parser", + "qwen", + ], + num_workers=1, + tp_size=2, + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + +class TestToolChoiceMistral(TestToolChoiceLlama32): + """Test tool_choice functionality with Mistral model""" + + @classmethod + def setUpClass(cls): + # CHANGE: Launch gRPC router with integrated workers (single command) + # Mark flaky tests for this model + cls.flaky_tests = { + "test_multi_tool_scenario_auto", + "test_multi_tool_scenario_required", + } + + cls.model = DEFAULT_MISTRAL_FUNCTION_CALLING_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + cls.cluster = popen_launch_workers_and_router( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + router_args=[ + "--tool-call-parser", + "mistral", + ], + num_workers=1, + tp_size=2, + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + @unittest.skip("Fails due to whitespace issue with Mistral - skipping") + def test_complex_parameters_required_non_streaming(self): + """Validate complex nested parameter schemas in non-streaming required mode""" + super().test_complex_parameters_required_non_streaming() + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/pytest.ini b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/pytest.ini new file mode 100644 index 00000000..aa52e232 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/pytest.ini @@ -0,0 +1,13 @@ +[pytest] +# Show print statements and logs +log_cli = true +log_cli_level = INFO +log_cli_format = %(asctime)s [%(levelname)8s] %(message)s +log_cli_date_format = %Y-%m-%d %H:%M:%S + +# Show stdout/stderr +addopts = -v -s --tb=short + +# Capture settings +# -s means don't capture stdout (show print statements) +# --tb=short means short traceback format diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/util.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/util.py new file mode 100644 index 00000000..3f85cd67 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/util.py @@ -0,0 +1,260 @@ +""" +Standalone utilities for e2e_grpc tests. + +This module provides all necessary utilities without depending on sglang Python package. +Extracted and adapted from: +- sglang.srt.utils.kill_process_tree +- sglang.srt.utils.hf_transformers_utils.get_tokenizer +- sglang.test.test_utils (constants and CustomTestCase) +""" + +import logging +import os +import signal +import threading +import unittest +from pathlib import Path +from typing import Optional, Union + +import psutil + +logger = logging.getLogger(__name__) + +try: + from transformers import ( + AutoTokenizer, + PreTrainedTokenizer, + PreTrainedTokenizerBase, + PreTrainedTokenizerFast, + ) +except ImportError: + raise ImportError( + "transformers is required for tokenizer utilities. " + "Install with: pip install transformers" + ) + + +# ============================================================================ +# Constants +# ============================================================================ + +# Server and timeout constants +DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 600 +DEFAULT_PORT_FOR_SRT_TEST_RUNNER = 20000 +DEFAULT_URL_FOR_TEST = f"http://127.0.0.1:{DEFAULT_PORT_FOR_SRT_TEST_RUNNER + 1000}" + +# File name constants for test output +STDOUT_FILENAME = "/tmp/sglang_test_stdout.txt" +STDERR_FILENAME = "/tmp/sglang_test_stderr.txt" + +# Model base path - can be overridden via environment variable +# By default, use HuggingFace model identifiers (no local path prefix) +# Set ROUTER_LOCAL_MODEL_PATH to use local models (e.g., "/home/ubuntu/models") +ROUTER_LOCAL_MODEL_PATH = os.environ.get("ROUTER_LOCAL_MODEL_PATH", "") + + +# Helper function to build model paths +def _get_model_path(model_identifier: str) -> str: + """ + Build model path from base path and model identifier. + + If ROUTER_LOCAL_MODEL_PATH is set, prepend it to the identifier. + Otherwise, return the identifier as-is (for HuggingFace download). + """ + if ROUTER_LOCAL_MODEL_PATH: + return os.path.join(ROUTER_LOCAL_MODEL_PATH, model_identifier) + return model_identifier + + +# Model paths used in e2e_grpc tests +# These can be either HuggingFace identifiers or local paths (depending on ROUTER_LOCAL_MODEL_PATH) + +# Main test model - Llama 3.1 8B Instruct +DEFAULT_MODEL_PATH = _get_model_path("meta-llama/Llama-3.1-8B-Instruct") + +# Small models for function calling tests +DEFAULT_SMALL_MODEL_PATH = _get_model_path("meta-llama/Llama-3.2-1B-Instruct") + +# Reasoning models +DEFAULT_REASONING_MODEL_PATH = _get_model_path( + "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B" +) + +# Thinking-enabled models +DEFAULT_ENABLE_THINKING_MODEL_PATH = _get_model_path("Qwen/Qwen3-30B-A3B") + +# Function calling models +DEFAULT_QWEN_FUNCTION_CALLING_MODEL_PATH = _get_model_path("Qwen/Qwen2.5-7B-Instruct") +DEFAULT_MISTRAL_FUNCTION_CALLING_MODEL_PATH = _get_model_path( + "mistralai/Mistral-7B-Instruct-v0.3" +) + +# GPT-OSS models +DEFAULT_GPT_OSS_MODEL_PATH = _get_model_path("openai/gpt-oss-20b") + + +# ============================================================================ +# Process Management +# ============================================================================ + + +def kill_process_tree(parent_pid, include_parent: bool = True, skip_pid: int = None): + """ + Kill the process and all its child processes. + + Args: + parent_pid: PID of the parent process + include_parent: Whether to kill the parent process itself + skip_pid: Optional PID to skip during cleanup + """ + # Remove sigchld handler to avoid spammy logs + if threading.current_thread() is threading.main_thread(): + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + + if parent_pid is None: + parent_pid = os.getpid() + include_parent = False + + try: + itself = psutil.Process(parent_pid) + except psutil.NoSuchProcess: + return + + children = itself.children(recursive=True) + for child in children: + if child.pid == skip_pid: + continue + try: + child.kill() + except psutil.NoSuchProcess: + pass + + if include_parent: + try: + itself.kill() + except psutil.NoSuchProcess: + pass + + +# ============================================================================ +# Tokenizer Utilities +# ============================================================================ + + +def check_gguf_file(model_path: str) -> bool: + """Check if the model path points to a GGUF file.""" + if not isinstance(model_path, str): + return False + return model_path.endswith(".gguf") + + +def is_remote_url(path: str) -> bool: + """Check if the path is a remote URL.""" + if not isinstance(path, str): + return False + return path.startswith("http://") or path.startswith("https://") + + +def get_tokenizer( + tokenizer_name: str, + *args, + tokenizer_mode: str = "auto", + trust_remote_code: bool = False, + tokenizer_revision: Optional[str] = None, + **kwargs, +) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]: + """ + Gets a tokenizer for the given model name via Huggingface. + + Args: + tokenizer_name: Name or path of the tokenizer + tokenizer_mode: Mode for tokenizer loading ("auto", "slow") + trust_remote_code: Whether to trust remote code + tokenizer_revision: Specific revision to use + **kwargs: Additional arguments passed to AutoTokenizer.from_pretrained + + Returns: + Loaded tokenizer instance + """ + if tokenizer_mode == "slow": + if kwargs.get("use_fast", False): + raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.") + kwargs["use_fast"] = False + + # Handle special model name mapping + if tokenizer_name == "mistralai/Devstral-Small-2505": + tokenizer_name = "mistralai/Mistral-Small-3.1-24B-Instruct-2503" + + is_gguf = check_gguf_file(tokenizer_name) + if is_gguf: + kwargs["gguf_file"] = tokenizer_name + tokenizer_name = Path(tokenizer_name).parent + + # Note: Removed remote URL handling and local directory download + # as they depend on sglang-specific utilities + + try: + tokenizer = AutoTokenizer.from_pretrained( + tokenizer_name, + *args, + trust_remote_code=trust_remote_code, + tokenizer_revision=tokenizer_revision, + **kwargs, + ) + except TypeError as e: + # Handle specific errors + err_msg = ( + "Failed to load the tokenizer. If you are running a model with " + "a custom tokenizer, please set the --trust-remote-code flag." + ) + raise RuntimeError(err_msg) from e + + if not isinstance(tokenizer, PreTrainedTokenizerFast): + logger.warning( + f"Using a slow tokenizer. This might cause a performance " + f"degradation. Consider using a fast tokenizer instead." + ) + + return tokenizer + + +def get_tokenizer_from_processor(processor): + """Extract tokenizer from a processor object.""" + if isinstance(processor, PreTrainedTokenizerBase): + return processor + return processor.tokenizer + + +# ============================================================================ +# Test Utilities +# ============================================================================ + + +class CustomTestCase(unittest.TestCase): + """ + Custom test case base class with retry support. + + This provides automatic test retry functionality based on environment variables. + """ + + def _callTestMethod(self, method): + """Override to add retry logic.""" + max_retry = int(os.environ.get("SGLANG_TEST_MAX_RETRY", "0")) + + if max_retry == 0: + # No retry, just run once + return super(CustomTestCase, self)._callTestMethod(method) + + # Retry logic + for attempt in range(max_retry + 1): + try: + return super(CustomTestCase, self)._callTestMethod(method) + except Exception as e: + if attempt < max_retry: + logger.info( + f"Test failed on attempt {attempt + 1}/{max_retry + 1}, retrying..." + ) + continue + else: + # Last attempt, re-raise the exception + raise diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/validation/test_large_max_new_tokens.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/validation/test_large_max_new_tokens.py new file mode 100644 index 00000000..2ea48697 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/validation/test_large_max_new_tokens.py @@ -0,0 +1,115 @@ +""" +python3 -m unittest openai_server.validation.test_large_max_new_tokens.TestLargeMaxNewTokens.test_chat_completion +""" + +import os +import sys +import time +import unittest +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import openai + +_TEST_DIR = Path(__file__).parent +sys.path.insert(0, str(_TEST_DIR.parent)) +from fixtures import popen_launch_workers_and_router +from util import ( + DEFAULT_MODEL_PATH, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + STDERR_FILENAME, + STDOUT_FILENAME, + CustomTestCase, + get_tokenizer, + kill_process_tree, +) + + +class TestLargeMaxNewTokens(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + cls.stdout = open(STDOUT_FILENAME, "w") + cls.stderr = open(STDERR_FILENAME, "w") + + cls.cluster = popen_launch_workers_and_router( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + worker_args=( + "--max-total-token", + "1536", + "--context-len", + "8192", + "--decode-log-interval", + "2", + ), + num_workers=1, + tp_size=2, + env={"SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION": "256", **os.environ}, + stdout=cls.stdout, + stderr=cls.stderr, + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + @classmethod + def tearDownClass(cls): + # Cleanup router and workers + kill_process_tree(cls.cluster["router"].pid) + for worker in cls.cluster.get("workers", []): + kill_process_tree(worker.pid) + cls.stdout.close() + cls.stderr.close() + os.remove(STDOUT_FILENAME) + os.remove(STDERR_FILENAME) + + def run_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + { + "role": "user", + "content": "Please repeat the world 'hello' for 10000 times.", + }, + ], + temperature=0, + ) + return response + + def test_chat_completion(self): + num_requests = 4 + all_requests_running = False + + futures = [] + with ThreadPoolExecutor(num_requests) as executor: + # Send multiple requests + for i in range(num_requests): + futures.append(executor.submit(self.run_chat_completion)) + + # Ensure that they are running concurrently + pt = 0 + while pt >= 0: + time.sleep(5) + # Flush stderr to ensure logs are written + self.stderr.flush() + lines = open(STDERR_FILENAME).readlines() + for line in lines[pt:]: + if f"#running-req: {num_requests}" in line: + all_requests_running = True + pt = -1 + break + pt += 1 + + assert all_requests_running + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/validation/test_openai_server_ignore_eos.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/validation/test_openai_server_ignore_eos.py new file mode 100644 index 00000000..0b255de4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_grpc/validation/test_openai_server_ignore_eos.py @@ -0,0 +1,109 @@ +""" +gRPC Router E2E Test - Test Openai Server Ignore Eos + +This test file is REUSED from test/srt/openai_server/validation/test_openai_server_ignore_eos.py +with minimal changes: + num_workers=2, +- Swap popen_launch_server() → popen_launch_workers_and_router() +- Update teardown to cleanup router + workers +- All test logic and assertions remain identical + +Run with: + pytest py_test/e2e_grpc/e2e_grpc/validation/test_openai_server_ignore_eos.py -v +""" + +import sys +from pathlib import Path + +import openai + +_TEST_DIR = Path(__file__).parent +sys.path.insert(0, str(_TEST_DIR.parent)) +from fixtures import popen_launch_workers_and_router +from util import ( + DEFAULT_MODEL_PATH, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + get_tokenizer, + kill_process_tree, +) + + +class TestOpenAIServerIgnoreEOS(CustomTestCase): + @classmethod + def setUpClass(cls): + # CHANGE: Launch gRPC router with integrated workers (single command) + cls.model = DEFAULT_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.cluster = popen_launch_workers_and_router( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + num_workers=1, + tp_size=2, + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + @classmethod + def tearDownClass(cls): + # Cleanup router and workers + kill_process_tree(cls.cluster["router"].pid) + for worker in cls.cluster.get("workers", []): + kill_process_tree(worker.pid) + + def test_ignore_eos(self): + """ + Test that ignore_eos=True allows generation to continue beyond EOS token + and reach the max_tokens limit. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + max_tokens = 200 + + response_default = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Count from 1 to 20."}, + ], + temperature=0, + max_tokens=max_tokens, + extra_body={"ignore_eos": False}, + ) + + response_ignore_eos = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Count from 1 to 20."}, + ], + temperature=0, + max_tokens=max_tokens, + extra_body={"ignore_eos": True}, + ) + + default_tokens = len( + self.tokenizer.encode(response_default.choices[0].message.content) + ) + ignore_eos_tokens = len( + self.tokenizer.encode(response_ignore_eos.choices[0].message.content) + ) + + # Check if ignore_eos resulted in more tokens or exactly max_tokens + # The ignore_eos response should either: + # 1. Have more tokens than the default response (if default stopped at EOS before max_tokens) + # 2. Have exactly max_tokens (if it reached the max_tokens limit) + self.assertTrue( + ignore_eos_tokens > default_tokens or ignore_eos_tokens >= max_tokens, + f"ignore_eos did not generate more tokens: {ignore_eos_tokens} vs {default_tokens}", + ) + + self.assertEqual( + response_ignore_eos.choices[0].finish_reason, + "length", + f"Expected finish_reason='length' for ignore_eos=True, got {response_ignore_eos.choices[0].finish_reason}", + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_http/conftest.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_http/conftest.py new file mode 100644 index 00000000..4e0b241c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_http/conftest.py @@ -0,0 +1,807 @@ +import json +import logging +import os +import shutil +import signal +import socket +import subprocess +import time +from pathlib import Path +from types import SimpleNamespace +from typing import Callable, Optional +from urllib.parse import urlparse + +import pytest +import requests + +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, +) + +logger = logging.getLogger(__name__) + + +def _find_available_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _parse_url(base_url: str) -> tuple[str, str]: + """Parse a base URL and return (host, port) as strings. + + This is more robust than simple string splitting and supports different schemes + and URL shapes like trailing paths. + """ + parsed = urlparse(base_url) + return parsed.hostname or "127.0.0.1", ( + str(parsed.port) if parsed.port is not None else "" + ) + + +def _wait_router_health(base_url: str, timeout: float) -> None: + start = time.perf_counter() + with requests.Session() as session: + while time.perf_counter() - start < timeout: + try: + r = session.get(f"{base_url}/health", timeout=5) + if r.status_code == 200: + return + except requests.RequestException: + pass + time.sleep(2) + raise TimeoutError("Router failed to become healthy in time") + + +def _popen_launch_router( + model: str, + base_url: str, + dp_size: int, + timeout: float, + policy: str = "cache_aware", +) -> subprocess.Popen: + host, port = _parse_url(base_url) + + prom_port = _find_available_port() + + cmd = [ + "python3", + "-m", + "sglang_router.launch_server", + "--model-path", + model, + "--host", + host, + "--port", + port, + "--dp", + str(dp_size), + "--router-policy", + policy, + "--allow-auto-truncate", + "--router-prometheus-port", + str(prom_port), + "--router-prometheus-host", + "127.0.0.1", + "--router-log-level", + "warn", + ] + + proc = subprocess.Popen(cmd) + _wait_router_health(base_url, timeout) + return proc + + +def _popen_launch_worker( + model: str, + base_url: str, + *, + dp_size: int | None = None, + api_key: str | None = None, + base_gpu_id: int | None = 0, +) -> subprocess.Popen: + host, port = _parse_url(base_url) + + cmd = [ + "python3", + "-m", + "sglang.launch_server", + "--model-path", + model, + "--host", + host, + "--port", + port, + "--base-gpu-id", + str(base_gpu_id or 0), + "--log-level", + "warning", + ] + if dp_size is not None: + cmd += ["--dp-size", str(dp_size)] + if api_key is not None: + cmd += ["--api-key", api_key] + return subprocess.Popen(cmd) + + +def _popen_launch_router_only( + base_url: str, + policy: str = "round_robin", + timeout: float = 120.0, + *, + dp_aware: bool = False, + enable_igw: bool = False, + api_key: str | None = None, +) -> subprocess.Popen: + host, port = _parse_url(base_url) + + prom_port = _find_available_port() + cmd = [ + "python3", + "-m", + "sglang_router.launch_router", + "--host", + host, + "--port", + port, + "--policy", + policy, + ] + if dp_aware: + cmd += ["--dp-aware"] + if enable_igw: + cmd += ["--enable-igw"] + if api_key is not None: + cmd += ["--api-key", api_key] + cmd += [ + "--prometheus-port", + str(prom_port), + "--prometheus-host", + "127.0.0.1", + "--log-level", + "warn", + ] + proc = subprocess.Popen(cmd) + _wait_router_health(base_url, timeout) + return proc + + +def _terminate(proc: subprocess.Popen, timeout: float = 120) -> None: + if proc is None: + return + proc.terminate() + start = time.perf_counter() + while proc.poll() is None: + if time.perf_counter() - start > timeout: + proc.kill() + break + time.sleep(1) + + +def _which(cmd: str) -> Optional[str]: + try: + return shutil.which(cmd) + except Exception as e: + logger.warning("shutil.which(%r) failed: %s", cmd, e) + return None + + +def _graceful_stop_popen(p: subprocess.Popen) -> None: + if p is None: + return + try: + if p.poll() is None: + p.terminate() + for _ in range(5): + if p.poll() is not None: + break + time.sleep(1) + if p.poll() is None: + p.kill() + except Exception as e: + logger.warning("Exception during graceful stop of popen: %s", e) + + +def _pid_alive(pid: int) -> bool: + try: + os.kill(pid, 0) + return True + except Exception: + return False + + +def _graceful_stop_pid(pid: int) -> None: + try: + if _pid_alive(pid): + try: + os.kill(pid, signal.SIGTERM) + except Exception: + pass + for _ in range(5): + if not _pid_alive(pid): + break + time.sleep(1) + if _pid_alive(pid): + try: + os.kill(pid, signal.SIGKILL) + except Exception: + pass + except Exception: + pass + + +def _graceful_stop_any(obj) -> None: + try: + if isinstance(obj, subprocess.Popen): + _graceful_stop_popen(obj) + return + if isinstance(obj, int): + _graceful_stop_pid(obj) + return + proc_obj = getattr(obj, "proc", None) + if isinstance(proc_obj, subprocess.Popen): + _graceful_stop_popen(proc_obj) + except Exception: + pass + + +def _gpu_monitor_should_run(thresholds: Optional[dict]) -> bool: + """Decide whether to enable the GPU monitor. + + Runs if thresholds request GPU checks or if GPU_UTIL_LOG is truthy. + """ + want = False + try: + mean_th = None if thresholds is None else thresholds.get("gpu_util_mean_min") + p50_th = None if thresholds is None else thresholds.get("gpu_util_p50_min") + want = bool(mean_th is not None or p50_th is not None) + except Exception: + want = False + if not want: + env_flag = os.environ.get("GPU_UTIL_LOG", "").lower() in ("1", "true", "yes") + want = want or env_flag + return want + + +def _gpu_monitor_path(experiment_folder: str) -> str: + """Return the JSON path for storing GPU monitor results.""" + base = Path.cwd() / experiment_folder + return str(base / "gpu_utilization.json") + + +def _launch_gpu_monitor(bench_pid: int, experiment_folder: str, interval: float): + """Start the GPU monitor process. Returns (proc, path) or (None, None).""" + try: + from multiprocessing import Process + + out_path = _gpu_monitor_path(experiment_folder) + proc = Process( + target=_gpu_monitor_proc_entry, + args=(bench_pid, out_path, interval), + daemon=True, + ) + proc.start() + return proc, out_path + except Exception as e: + logger.warning("Failed to launch GPU monitor: %s", e) + return None, None + + +def _read_gpu_monitor_result(path: Optional[str]) -> Optional[dict]: + try: + if path and os.path.exists(path): + with open(path, "r") as f: + return json.load(f) + except Exception as e: + logger.warning("Failed to read GPU monitor result from %r: %s", path, e) + return None + + +def _log_and_assert_gpu_thresholds( + result: Optional[dict], thresholds: Optional[dict] +) -> None: + if not result or not isinstance(result, dict) or result.get("count", 0) <= 0: + logger.warning("GPU utilization monitor produced no samples.") + return + + overall = result.get("overall", {}) if isinstance(result, dict) else {} + count = int(result.get("count", 0)) + mean_th = None if thresholds is None else thresholds.get("gpu_util_mean_min") + p50_th = None if thresholds is None else thresholds.get("gpu_util_p50_min") + + mean_v = float(overall.get("mean", 0.0)) + p50_v = overall.get("p50") + + logger.info( + "GPU utilization overall: mean=%.2f%% p50=%s (samples=%d)", + mean_v, + (f"{float(p50_v):.2f}%" if p50_v is not None else "n/a"), + count, + ) + + if mean_th is not None: + assert mean_v >= float( + mean_th + ), f"GPU utilization mean below threshold: {mean_v:.2f}% < {mean_th}%" + if p50_th is not None and p50_v is not None: + p50_f = float(p50_v) + assert p50_f >= float( + p50_th + ), f"GPU utilization p50 below threshold: {p50_f:.2f}% < {p50_th}%" + + +def _gpu_monitor_proc_entry(bench_pid: int, out_file: str, interval: float) -> None: + """Low-impact GPU utilization monitor using NVML in a separate process. + + Writes JSON to out_file that includes overall and per-GPU raw samples and summary stats. + """ + try: + try: + os.nice(10) + except Exception: + pass + total = 0.0 + n = 0 + try: + import pynvml # type: ignore + + pynvml.nvmlInit() + except Exception: + with open(out_file, "w") as f: + os.makedirs(os.path.dirname(out_file), exist_ok=True) + json.dump( + { + "count": 0, + "overall": {"mean": 0.0}, + "per_gpu": {}, + "raw": {}, + }, + f, + ) + return + try: + import pynvml # type: ignore + + count = pynvml.nvmlDeviceGetCount() + handles = [pynvml.nvmlDeviceGetHandleByIndex(i) for i in range(count)] + except Exception: + with open(out_file, "w") as f: + os.makedirs(os.path.dirname(out_file), exist_ok=True) + json.dump( + { + "count": 0, + "overall": {"mean": 0.0}, + "per_gpu": {}, + "raw": {}, + }, + f, + ) + return + + # Prepare per-GPU and overall raw collectors + per_gpu_samples: dict[str, list[float]] = {} + overall_samples: list[float] = [] + + while True: + if not os.path.exists(f"/proc/{bench_pid}"): + break + try: + vals = [] + import pynvml # type: ignore + + for idx, h in enumerate(handles): + try: + util = pynvml.nvmlDeviceGetUtilizationRates(h).gpu + vals.append(float(util)) + key = str(idx) + per_gpu_samples.setdefault(key, []).append(float(util)) + except Exception: + continue + if vals: + avg = sum(vals) / len(vals) + overall_samples.append(avg) + total += avg + n += 1 + except Exception: + pass + time.sleep(interval) + finally: + try: + os.makedirs(os.path.dirname(out_file), exist_ok=True) + with open(out_file, "w") as f: + + def pct_from(samples: list[float], p: float) -> float: + if not samples: + return 0.0 + srt = sorted(samples) + i = max( + 0, min(len(srt) - 1, int(round((p / 100.0) * (len(srt) - 1)))) + ) + return float(srt[i]) + + overall_mean = (total / n) if n > 0 else 0.0 + + per_gpu_summary: dict[str, dict] = {} + for key, arr in per_gpu_samples.items(): + per_gpu_summary[key] = { + "mean": float(sum(arr) / len(arr)) if arr else 0.0, + "p5": pct_from(arr, 5), + "p10": pct_from(arr, 10), + "p25": pct_from(arr, 25), + "p50": pct_from(arr, 50), + "p75": pct_from(arr, 75), + "p90": pct_from(arr, 90), + "p95": pct_from(arr, 95), + "min": float(min(arr)) if arr else 0.0, + "max": float(max(arr)) if arr else 0.0, + "count": len(arr), + } + + out_payload = { + "bench_pid": bench_pid, + "interval_sec": interval, + "count": n, + "overall": { + "mean": float(overall_mean), + "p5": pct_from(overall_samples, 5), + "p10": pct_from(overall_samples, 10), + "p25": pct_from(overall_samples, 25), + "p50": pct_from(overall_samples, 50), + "p75": pct_from(overall_samples, 75), + "p90": pct_from(overall_samples, 90), + "p95": pct_from(overall_samples, 95), + "min": float(min(overall_samples)) if overall_samples else 0.0, + "max": float(max(overall_samples)) if overall_samples else 0.0, + }, + "per_gpu": per_gpu_summary, + "raw": { + "overall": overall_samples, + "per_gpu": per_gpu_samples, + }, + } + json.dump(out_payload, f) + except Exception: + pass + try: + import pynvml # type: ignore + + pynvml.nvmlShutdown() + except Exception: + pass + + +@pytest.fixture(scope="session") +def genai_bench_runner() -> Callable[..., None]: + """Provide a callable to run genai-bench and validate metrics. + + Usage in tests: + def test(..., genai_bench_runner): + genai_bench_runner(router_url=..., model_path=..., experiment_folder=...) + """ + + def _run( + *, + router_url: str, + model_path: str, + experiment_folder: str, + timeout_sec: int | None = None, + thresholds: dict | None = None, + extra_env: dict | None = None, + num_concurrency: int = 32, + traffic_scenario: str = "D(4000,100)", + max_requests_per_run: int | None = None, + clean_experiment: bool = True, + kill_procs: list | None = None, + drain_delay_sec: int = 6, + ) -> None: + cli = _which("genai-bench") + if not cli: + pytest.fail( + "genai-bench CLI not found; please install it to run benchmarks" + ) + + # Clean previous experiment folder under current working directory + if clean_experiment: + exp_dir = Path.cwd() / experiment_folder + if exp_dir.exists(): + shutil.rmtree(exp_dir, ignore_errors=True) + + # Default requests per run if not provided + mrr = ( + max_requests_per_run + if max_requests_per_run is not None + else num_concurrency * 5 + ) + + cmd = [ + cli, + "benchmark", + "--api-backend", + "openai", + "--api-base", + router_url, + "--api-key", + "dummy-token", + "--api-model-name", + model_path, + "--model-tokenizer", + model_path, + "--task", + "text-to-text", + "--num-concurrency", + str(num_concurrency), + "--traffic-scenario", + traffic_scenario, + "--max-requests-per-run", + str(mrr), + "--max-time-per-run", + "3", + "--experiment-folder-name", + experiment_folder, + "--experiment-base-dir", + str(Path.cwd()), + ] + + env = os.environ.copy() + if extra_env: + env.update(extra_env) + + to = timeout_sec or int(os.environ.get("GENAI_BENCH_TEST_TIMEOUT", "120")) + proc = subprocess.Popen( + cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + # Optional GPU utilization monitor in a low-priority child process (pynvml only) + # Enabled only when gpu_util_mean_min is provided in thresholds. + monitor_path = None + monitor_proc = None + gpu_util_result: dict | None = None + want_gpu_monitor = _gpu_monitor_should_run(thresholds) + if want_gpu_monitor: + interval = float(os.environ.get("GPU_UTIL_SAMPLE_INTERVAL", "2.0")) + monitor_proc, monitor_path = _launch_gpu_monitor( + bench_pid=proc.pid, + experiment_folder=experiment_folder, + interval=interval, + ) + stdout = stderr = "" + rc = None + try: + try: + stdout, stderr = proc.communicate(timeout=to) + except subprocess.TimeoutExpired: + # Simple: kill the CLI process if it doesn't exit in time + try: + proc.kill() + except Exception: + pass + stdout, stderr = proc.communicate() + rc = proc.returncode + + # Prefer exact path under cwd; fallback to rglob search + base = Path.cwd() + direct = base / experiment_folder + candidates = [direct] if direct.is_dir() else [] + if not candidates: + for p in base.rglob(experiment_folder): + if p.is_dir() and p.name == experiment_folder: + candidates = [p] + break + if not candidates: + raise AssertionError( + "Benchmark failed: experiment folder not found: " + f"{experiment_folder}\nExit code: {rc}\nSTDOUT (tail):\n{stdout[-1000:]}\nSTDERR (tail):\n{stderr[-1000:]}" + ) + actual_folder = candidates[0] + + json_files = [] + for _ in range(10): + json_files = [ + p + for p in actual_folder.rglob("*.json") + if "experiment_metadata" not in p.name + ] + if json_files: + break + time.sleep(1) + if not json_files: + raise AssertionError( + "Benchmark failed: no JSON results found\n" + f"Exit code: {rc}\nSTDOUT (tail):\n{stdout[-1000:]}\nSTDERR (tail):\n{stderr[-1000:]}" + ) + + th = thresholds # None means "log only", no validation + + for jf in json_files: + with jf.open("r") as f: + data = json.load(f) + stats = data.get("aggregated_metrics", {}).get("stats", {}) + ttft_mean = float(stats.get("ttft", {}).get("mean", float("inf"))) + e2e_latency_mean = float( + stats.get("e2e_latency", {}).get("mean", float("inf")) + ) + input_tp_mean = float( + stats.get("input_throughput", {}).get("mean", 0.0) + ) + output_tp_mean = float( + stats.get("output_throughput", {}).get("mean", 0.0) + ) + + logger.info( + "genai-bench[%s] %s ttft_mean=%.3fs e2e_latency_mean=%.3fs input_tp_mean=%.1f tok/s output_tp_mean=%.1f tok/s", + experiment_folder, + jf.name, + ttft_mean, + e2e_latency_mean, + input_tp_mean, + output_tp_mean, + ) + + if th is not None: + assert ( + ttft_mean <= th["ttft_mean_max"] + ), f"TTFT validation failed: {ttft_mean} > {th['ttft_mean_max']} (file={jf.name})" + assert ( + e2e_latency_mean <= th["e2e_latency_mean_max"] + ), f"E2E latency validation failed: {e2e_latency_mean} > {th['e2e_latency_mean_max']} (file={jf.name})" + assert ( + input_tp_mean >= th["input_throughput_mean_min"] + ), f"Input throughput validation failed: {input_tp_mean} < {th['input_throughput_mean_min']} (file={jf.name})" + assert ( + output_tp_mean >= th["output_throughput_mean_min"] + ), f"Output throughput validation failed: {output_tp_mean} < {th['output_throughput_mean_min']} (file={jf.name})" + + # Validate optional GPU utilization threshold if provided + if want_gpu_monitor: + try: + if monitor_proc is not None: + monitor_proc.join(timeout=5) + except Exception: + pass + gpu_util_result = _read_gpu_monitor_result(monitor_path) + _log_and_assert_gpu_thresholds(gpu_util_result, thresholds) + + finally: + # Always attempt to stop workers to avoid resource leakage + if kill_procs: + # Give router/workers a small grace period to finish any last drains + if drain_delay_sec > 0: + try: + time.sleep(drain_delay_sec) + except Exception: + pass + for p in kill_procs: + _graceful_stop_any(p) + try: + time.sleep(2) + except Exception: + pass + # Ensure GPU monitor process is cleaned up + if monitor_proc is not None and monitor_proc.is_alive(): + try: + monitor_proc.terminate() + except Exception: + pass + + return _run + + +def pytest_configure(config): + config.addinivalue_line("markers", "e2e: mark as end-to-end test") + + +@pytest.fixture(scope="session") +def e2e_model() -> str: + # Always use the default test model + return os.getenv("E2E_PRIMARY_MODEL", DEFAULT_MODEL_NAME_FOR_TEST) + + +@pytest.fixture +def e2e_router(e2e_model: str): + # Keep this available but tests below use router-only to avoid GPU contention + base_url = DEFAULT_URL_FOR_TEST + proc = _popen_launch_router( + e2e_model, base_url, dp_size=2, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + ) + try: + yield SimpleNamespace(proc=proc, url=base_url) + finally: + _terminate(proc) + + +@pytest.fixture +def e2e_router_only_rr(): + port = _find_available_port() + base_url = f"http://127.0.0.1:{port}" + proc = _popen_launch_router_only(base_url, policy="round_robin") + try: + yield SimpleNamespace(proc=proc, url=base_url) + finally: + _terminate(proc) + + +@pytest.fixture(scope="session") +def e2e_embedding_model() -> str: + """Embedding model to use for E2E tests. + + Defaults to an E5 Mistral model, can be overridden via E2E_EMBEDDING_MODEL env var. + """ + import os + + return os.getenv("E2E_EMBEDDING_MODEL", "intfloat/e5-mistral-7b-instruct") + + +@pytest.fixture +def e2e_primary_embedding_worker(e2e_embedding_model: str): + """Launch a single embedding worker using the specified model.""" + port = _find_available_port() + base_url = f"http://127.0.0.1:{port}" + proc = _popen_launch_worker(e2e_embedding_model, base_url) + try: + yield SimpleNamespace(proc=proc, url=base_url) + finally: + _terminate(proc) + + +@pytest.fixture(scope="session") +def e2e_primary_worker(e2e_model: str): + port = _find_available_port() + base_url = f"http://127.0.0.1:{port}" + proc = _popen_launch_worker(e2e_model, base_url) + # Router health gate will handle worker readiness + try: + yield SimpleNamespace(proc=proc, url=base_url) + finally: + _terminate(proc) + + +@pytest.fixture +def e2e_router_only_rr_dp_aware_api(): + """Router-only with dp-aware enabled and an API key.""" + port = _find_available_port() + base_url = f"http://127.0.0.1:{port}" + api_key = "secret" + proc = _popen_launch_router_only( + base_url, policy="round_robin", timeout=180.0, dp_aware=True, api_key=api_key + ) + try: + yield SimpleNamespace(proc=proc, url=base_url, api_key=api_key) + finally: + _terminate(proc) + + +@pytest.fixture +def e2e_worker_dp2_api(e2e_model: str, e2e_router_only_rr_dp_aware_api): + """Worker with dp-size=2 and the same API key as the dp-aware router.""" + port = _find_available_port() + base_url = f"http://127.0.0.1:{port}" + api_key = e2e_router_only_rr_dp_aware_api.api_key + proc = _popen_launch_worker(e2e_model, base_url, dp_size=2, api_key=api_key) + try: + yield SimpleNamespace(proc=proc, url=base_url) + finally: + _terminate(proc) + + +@pytest.fixture(scope="session") +def e2e_two_workers_dp2(e2e_model: str): + """Launch two workers, each with dp_size=2, mapped to GPUs [0,1] and [2,3].""" + workers = [] + try: + # Worker A on GPUs 0-1 + port_a = _find_available_port() + url_a = f"http://127.0.0.1:{port_a}" + proc_a = _popen_launch_worker(e2e_model, url_a, dp_size=2, base_gpu_id=0) + workers.append(SimpleNamespace(proc=proc_a, url=url_a)) + + # Worker B on GPUs 2-3 + port_b = _find_available_port() + url_b = f"http://127.0.0.1:{port_b}" + proc_b = _popen_launch_worker(e2e_model, url_b, dp_size=2, base_gpu_id=2) + workers.append(SimpleNamespace(proc=proc_b, url=url_b)) + + yield workers + finally: + for w in workers: + _terminate(w.proc) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_http/test_e2e_embeddings.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_http/test_e2e_embeddings.py new file mode 100644 index 00000000..31727e94 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_http/test_e2e_embeddings.py @@ -0,0 +1,62 @@ +import time + +import pytest +import requests + + +def _wait_for_workers( + base_url: str, expected_count: int, timeout: float = 60.0, headers: dict = None +) -> None: + """Poll /workers endpoint until expected number of workers are registered.""" + start = time.perf_counter() + with requests.Session() as session: + while time.perf_counter() - start < timeout: + try: + r = session.get(f"{base_url}/workers", headers=headers, timeout=5) + if r.status_code == 200: + workers = r.json().get("workers", []) + if len(workers) >= expected_count: + return + except requests.RequestException: + pass + time.sleep(0.5) + raise TimeoutError( + f"Expected {expected_count} workers at {base_url}, timed out after {timeout}s" + ) + + +@pytest.mark.e2e +def test_embeddings_basic( + e2e_router_only_rr, e2e_primary_embedding_worker, e2e_embedding_model +): + base = e2e_router_only_rr.url + worker_url = e2e_primary_embedding_worker.url + + # Attach embedding worker to router-only instance + r = requests.post(f"{base}/workers", json={"url": worker_url}, timeout=180) + assert r.status_code == 202, f"Expected 202 ACCEPTED, got {r.status_code}: {r.text}" + + # Wait for worker to be registered + _wait_for_workers(base, expected_count=1, timeout=60.0) + + # Simple embedding request with two inputs + payload = { + "model": e2e_embedding_model, + "input": [ + "the quick brown fox", + "jumps over the lazy dog", + ], + } + r = requests.post(f"{base}/v1/embeddings", json=payload, timeout=120) + + assert r.status_code == 200, f"unexpected status: {r.status_code} {r.text}" + + data = r.json() + assert "data" in data and isinstance(data["data"], list) + assert len(data["data"]) == 2 + + # Validate shape of embedding objects + for item in data["data"]: + assert "embedding" in item and isinstance(item["embedding"], list) + # Ensure non-empty vectors + assert len(item["embedding"]) > 0 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_http/test_pd_router.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_http/test_pd_router.py new file mode 100644 index 00000000..4827cbb4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_http/test_pd_router.py @@ -0,0 +1,264 @@ +import logging +import socket +import subprocess +import time +from types import SimpleNamespace +from typing import Optional + +import pytest +import requests + +from sglang.test.run_eval import run_eval + +logger = logging.getLogger(__name__) + + +def _find_available_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _wait_health(url: str, timeout: float = 180.0) -> None: + start = time.perf_counter() + with requests.Session() as session: + while time.perf_counter() - start < timeout: + try: + r = session.get(f"{url}/health", timeout=5) + if r.status_code == 200: + return + except requests.RequestException: + pass + time.sleep(1) + raise TimeoutError(f"Service at {url} failed to become healthy in time") + + +def _detect_ib_device() -> Optional[str]: + """Return first active IB device name (e.g., mlx5_0) or None if unavailable.""" + # Fast check that ibv_devinfo exists + try: + subprocess.run( + ["ibv_devinfo", "-l"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=1, + ) + except (FileNotFoundError, subprocess.TimeoutExpired): + return None + + for i in range(12): + dev = f"mlx5_{i}" + try: + res = subprocess.run( + ["ibv_devinfo", dev], + capture_output=True, + text=True, + timeout=2, + ) + if res.returncode == 0 and ("state:" in res.stdout): + for line in res.stdout.splitlines(): + if "state:" in line and "PORT_ACTIVE" in line: + return dev + except Exception: + pass + return None + + +def _popen_launch_prefill_worker( + model: str, + bootstrap_port: int, + ib_device: Optional[str] = None, + base_gpu_id: int = 0, +) -> SimpleNamespace: + port = _find_available_port() + url = f"http://127.0.0.1:{port}" + cmd = [ + "python3", + "-m", + "sglang.launch_server", + "--model-path", + model, + "--disaggregation-mode", + "prefill", + "--host", + "127.0.0.1", + "--port", + str(port), + "--disaggregation-bootstrap-port", + str(bootstrap_port), + "--base-gpu-id", + str(base_gpu_id), + ] + if ib_device: + cmd += ["--disaggregation-ib-device", ib_device] + proc = subprocess.Popen(cmd) + _wait_health(url, timeout=300.0) + return SimpleNamespace(proc=proc, url=url, bootstrap_port=bootstrap_port) + + +def _popen_launch_decode_worker( + model: str, ib_device: Optional[str] = None, base_gpu_id: int = 0 +) -> SimpleNamespace: + port = _find_available_port() + url = f"http://127.0.0.1:{port}" + cmd = [ + "python3", + "-m", + "sglang.launch_server", + "--model-path", + model, + "--disaggregation-mode", + "decode", + "--host", + "127.0.0.1", + "--port", + str(port), + "--base-gpu-id", + str(base_gpu_id), + ] + if ib_device: + cmd += ["--disaggregation-ib-device", ib_device] + proc = subprocess.Popen(cmd) + _wait_health(url, timeout=300.0) + return SimpleNamespace(proc=proc, url=url) + + +def _terminate(proc: subprocess.Popen, timeout: float = 120) -> None: + if proc is None: + return + proc.terminate() + start = time.perf_counter() + while proc.poll() is None: + if time.perf_counter() - start > timeout: + proc.kill() + break + time.sleep(1) + + +@pytest.fixture(scope="module") +def pd_cluster(e2e_model: str): + """Start 2 prefill + 2 decode workers and one PD router, once per module.""" + # Environment capability checks: require sgl_kernel and GPU backend + try: + import sgl_kernel # noqa: F401 + except Exception as e: # pragma: no cover - environment dependent + pytest.fail(f"PD e2e requires sgl_kernel but it is not available: {e}") + + try: + import torch # noqa: F401 + except Exception as e: # pragma: no cover - environment dependent + pytest.fail( + f"PD e2e requires torch but it is not available or misconfigured: {e}" + ) + + if not torch.cuda.is_available(): # pragma: no cover - environment dependent + pytest.fail("PD e2e requires CUDA backend, but CUDA is not available") + + workers: list[SimpleNamespace] = [] + router_proc = None + try: + ib_device = _detect_ib_device() + + # Launch 4 workers across 4 GPUs: prefill on 0,1 and decode on 2,3 + pf1 = _popen_launch_prefill_worker( + e2e_model, + bootstrap_port=_find_available_port(), + ib_device=ib_device, + base_gpu_id=0, + ) + pf2 = _popen_launch_prefill_worker( + e2e_model, + bootstrap_port=_find_available_port(), + ib_device=ib_device, + base_gpu_id=1, + ) + dc1 = _popen_launch_decode_worker(e2e_model, ib_device=ib_device, base_gpu_id=2) + dc2 = _popen_launch_decode_worker(e2e_model, ib_device=ib_device, base_gpu_id=3) + prefills = [pf1, pf2] + decodes = [dc1, dc2] + workers.extend(prefills + decodes) + + # PD router with two prefill and two decode endpoints + rport = _find_available_port() + router_url = f"http://127.0.0.1:{rport}" + pport = _find_available_port() + + prefill = [(pf.url, pf.bootstrap_port) for pf in prefills] + decode = [dc.url for dc in decodes] + + cmd = [ + "python3", + "-m", + "sglang_router.launch_router", + "--host", + "127.0.0.1", + "--port", + str(rport), + "--policy", + "round_robin", + "--pd-disaggregation", + "--log-level", + "warn", + ] + for url, bport in prefill: + cmd += ["--prefill", url, str(bport)] + for url in decode: + cmd += ["--decode", url] + cmd += [ + "--prometheus-port", + str(pport), + "--prometheus-host", + "127.0.0.1", + ] + + router_proc = subprocess.Popen(cmd) + _wait_health(router_url, timeout=180.0) + + yield SimpleNamespace( + router_url=router_url, workers=workers, router_proc=router_proc + ) + finally: + if router_proc is not None: + _terminate(router_proc) + for w in workers: + _terminate(w.proc) + + +@pytest.mark.e2e +def test_pd_mmlu(e2e_model: str, pd_cluster): + """ + Launch 4 workers, start a PD router (2 prefill + 2 decode), then run MMLU. + """ + args = SimpleNamespace( + base_url=pd_cluster.router_url, + model=e2e_model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + temperature=0.1, + ) + metrics = run_eval(args) + assert metrics["score"] >= 0.65 + + +@pytest.mark.e2e +def test_pd_genai_bench(e2e_model: str, pd_cluster, genai_bench_runner): + """ + Launch 4 workers, start a PD router (2 prefill + 2 decode), then run a + short genai-bench benchmark and validate aggregate metrics. + """ + # Run genai-bench against the shared router + policy_label = "benchmark_round_robin_pd" + genai_bench_runner( + router_url=pd_cluster.router_url, + model_path=e2e_model, + experiment_folder=policy_label, + thresholds={ + "ttft_mean_max": 13, + "e2e_latency_mean_max": 16, + "input_throughput_mean_min": 350, + "output_throughput_mean_min": 18, + "gpu_util_p50_min": 99, + }, + kill_procs=pd_cluster.workers, + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_http/test_regular_router.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_http/test_regular_router.py new file mode 100644 index 00000000..effb39ef --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_http/test_regular_router.py @@ -0,0 +1,228 @@ +import threading +import time +from types import SimpleNamespace + +import pytest +import requests + +from sglang.test.run_eval import run_eval + + +def _wait_for_workers( + base_url: str, expected_count: int, timeout: float = 60.0, headers: dict = None +) -> None: + """Poll /workers endpoint until expected number of workers are registered.""" + start = time.perf_counter() + with requests.Session() as session: + while time.perf_counter() - start < timeout: + try: + r = session.get(f"{base_url}/workers", headers=headers, timeout=5) + if r.status_code == 200: + workers = r.json().get("workers", []) + if len(workers) >= expected_count: + return + except requests.RequestException: + pass + time.sleep(0.5) + raise TimeoutError( + f"Expected {expected_count} workers at {base_url}, timed out after {timeout}s" + ) + + +@pytest.mark.e2e +def test_mmlu(e2e_router_only_rr, e2e_two_workers_dp2, e2e_model): + # Attach two dp=2 workers (total 4 GPUs) to a fresh router-only instance + base = e2e_router_only_rr.url + for w in e2e_two_workers_dp2: + r = requests.post(f"{base}/workers", json={"url": w.url}, timeout=180) + assert ( + r.status_code == 202 + ), f"Expected 202 ACCEPTED, got {r.status_code}: {r.text}" + + # Wait for workers to be registered + _wait_for_workers(base, expected_count=2, timeout=60.0) + + args = SimpleNamespace( + base_url=base, + model=e2e_model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + temperature=0.1, + ) + metrics = run_eval(args) + assert metrics["score"] >= 0.65 + + +@pytest.mark.e2e +def test_genai_bench( + e2e_router_only_rr, e2e_two_workers_dp2, e2e_model, genai_bench_runner +): + """Attach a worker to the regular router and run a short genai-bench.""" + base = e2e_router_only_rr.url + for w in e2e_two_workers_dp2: + r = requests.post(f"{base}/workers", json={"url": w.url}, timeout=180) + assert ( + r.status_code == 202 + ), f"Expected 202 ACCEPTED, got {r.status_code}: {r.text}" + + # Wait for workers to be registered + _wait_for_workers(base, expected_count=2, timeout=60.0) + + genai_bench_runner( + router_url=base, + model_path=e2e_model, + experiment_folder="benchmark_round_robin_regular", + thresholds={ + "ttft_mean_max": 6, + "e2e_latency_mean_max": 14, + "input_throughput_mean_min": 800, # temp relax from 1000 to 800 for now + "output_throughput_mean_min": 12, + # Enforce GPU utilization p50 >= 99% during the run. + "gpu_util_p50_min": 99, + }, + kill_procs=e2e_two_workers_dp2, + ) + + +@pytest.mark.e2e +def test_add_and_remove_worker_live(e2e_router_only_rr, e2e_primary_worker, e2e_model): + base = e2e_router_only_rr.url + worker_url = e2e_primary_worker.url + + r = requests.post(f"{base}/workers", json={"url": worker_url}, timeout=180) + assert r.status_code == 202, f"Expected 202 ACCEPTED, got {r.status_code}: {r.text}" + + # Wait for worker to be registered + _wait_for_workers(base, expected_count=1, timeout=60.0) + + with requests.Session() as s: + for i in range(8): + r = s.post( + f"{base}/v1/completions", + json={ + "model": e2e_model, + "prompt": f"x{i}", + "max_tokens": 1, + "stream": False, + }, + timeout=120, + ) + r.raise_for_status() + + # Remove the worker + from urllib.parse import quote + + encoded_url = quote(worker_url, safe="") + r = requests.delete(f"{base}/workers/{encoded_url}", timeout=60) + assert r.status_code == 202, f"Expected 202 ACCEPTED, got {r.status_code}: {r.text}" + + +@pytest.mark.e2e +def test_lazy_fault_tolerance_live(e2e_router_only_rr, e2e_primary_worker, e2e_model): + base = e2e_router_only_rr.url + worker = e2e_primary_worker + + r = requests.post(f"{base}/workers", json={"url": worker.url}, timeout=180) + assert r.status_code == 202, f"Expected 202 ACCEPTED, got {r.status_code}: {r.text}" + + # Wait for worker to be registered + _wait_for_workers(base, expected_count=1, timeout=60.0) + + def killer(): + time.sleep(10) + try: + worker.proc.terminate() + except Exception: + pass + + t = threading.Thread(target=killer, daemon=True) + t.start() + + args = SimpleNamespace( + base_url=base, + model=e2e_model, + eval_name="mmlu", + num_examples=32, + num_threads=16, + temperature=0.0, + ) + metrics = run_eval(args) + assert 0.0 <= metrics["score"] <= 1.0 + + +@pytest.mark.e2e +def test_dp_aware_worker_expansion_and_api_key( + e2e_model, + e2e_router_only_rr_dp_aware_api, + e2e_worker_dp2_api, +): + """ + Launch a router-only instance in dp_aware mode and a single worker with dp_size=2 + and API key protection. Verify expansion, auth enforcement, and basic eval. + """ + import os + + router_url = e2e_router_only_rr_dp_aware_api.url + worker_url = e2e_worker_dp2_api.url + api_key = e2e_router_only_rr_dp_aware_api.api_key + + # Attach worker; router should expand to dp_size logical workers + r = requests.post( + f"{router_url}/workers", + json={"url": worker_url, "api_key": api_key}, + headers={"Authorization": f"Bearer {api_key}"}, + timeout=180, + ) + assert r.status_code == 202, f"Expected 202 ACCEPTED, got {r.status_code}: {r.text}" + + # Wait for workers to be registered and expanded + _wait_for_workers( + router_url, + expected_count=2, + timeout=60.0, + headers={"Authorization": f"Bearer {api_key}"}, + ) + + # Verify the expanded workers have correct URLs + r = requests.get( + f"{router_url}/workers", + headers={"Authorization": f"Bearer {api_key}"}, + timeout=30, + ) + r.raise_for_status() + workers = r.json().get("workers", []) + urls = [w["url"] for w in workers] + assert len(urls) == 2 + assert set(urls) == {f"{worker_url}@0", f"{worker_url}@1"} + + # Verify API key enforcement + # 1) Without Authorization -> Should get 401 Unauthorized + r = requests.post( + f"{router_url}/v1/completions", + json={"model": e2e_model, "prompt": "hi", "max_tokens": 1}, + timeout=60, + ) + assert r.status_code == 401 + + # 2) With correct Authorization -> 200 + r = requests.post( + f"{router_url}/v1/completions", + json={"model": e2e_model, "prompt": "hi", "max_tokens": 1}, + headers={"Authorization": f"Bearer {api_key}"}, + timeout=60, + ) + assert r.status_code == 200 + + # Finally, run MMLU eval through the router with auth + os.environ["OPENAI_API_KEY"] = api_key + args = SimpleNamespace( + base_url=router_url, + model=e2e_model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + temperature=0.1, + ) + metrics = run_eval(args) + assert metrics["score"] >= 0.65 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/conftest.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/conftest.py new file mode 100644 index 00000000..786c4a91 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/conftest.py @@ -0,0 +1,132 @@ +""" +pytest configuration for e2e_response_api tests. + +This configures pytest to not collect base test classes that are meant to be inherited. +""" + +import os + +import openai +import pytest # noqa: F401 +from router_fixtures import ( + popen_launch_openai_xai_router, + popen_launch_workers_and_router, +) +from util import kill_process_tree + +# ------------------------------ +# Backend Configuration Map +# ------------------------------ +BACKENDS = { + "openai": { + "model": "gpt-5-nano", + "base_url_port": "http://127.0.0.1:30010", + "launcher": popen_launch_openai_xai_router, + "launcher_kwargs": { + "backend": "openai", + "history_backend": "memory", + }, + "api_key_env": "OPENAI_API_KEY", + "needs_workers": False, + }, + "xai": { + "model": "grok-4-fast", + "base_url_port": "http://127.0.0.1:30023", + "launcher": popen_launch_openai_xai_router, + "launcher_kwargs": { + "backend": "xai", + "history_backend": "memory", + }, + "api_key_env": "XAI_API_KEY", + "needs_workers": False, + }, + "grpc": { + "model": "/home/ubuntu/models/Qwen/Qwen2.5-14B-Instruct", + "base_url_port": "http://127.0.0.1:30030", + "launcher": popen_launch_workers_and_router, + "launcher_kwargs": { + "timeout": 90, + "num_workers": 1, + "tp_size": 2, + "policy": "round_robin", + "worker_args": ["--context-length=1000"], + "router_args": [ + "--history-backend", + "memory", + "--tool-call-parser", + "qwen", + ], + }, + "api_key_env": None, # grpc does not use API keys + "needs_workers": True, + }, + "grpc_harmony": { + "model": "/home/ubuntu/models/openai/gpt-oss-20b", + "base_url_port": "http://127.0.0.1:30030", + "launcher": popen_launch_workers_and_router, + "launcher_kwargs": { + "timeout": 90, + "num_workers": 1, + "tp_size": 2, + "policy": "round_robin", + "worker_args": ["--reasoning-parser=gpt-oss"], + "router_args": ["--history-backend", "memory"], + }, + "api_key_env": None, + "needs_workers": True, + }, + "oracle_store": { + "model": "gpt-5-nano", + "base_url_port": "http://127.0.0.1:30040", + "launcher": popen_launch_openai_xai_router, + "launcher_kwargs": { + "backend": "openai", + "history_backend": "oracle", + }, + "api_key_env": "OPENAI_API_KEY", + "needs_workers": False, + }, +} + + +@pytest.fixture(scope="class") +def setup_backend(request): + backend = request.param + if backend not in BACKENDS: + raise RuntimeError(f"Unknown backend {backend}") + + cfg = BACKENDS[backend] + + # Launch cluster + cluster = ( + cfg["launcher"]( + cfg["model"], + cfg["base_url_port"], + **cfg["launcher_kwargs"], + ) + if cfg["launcher"] is popen_launch_workers_and_router + else cfg["launcher"]( + backend=cfg["launcher_kwargs"]["backend"], + base_url=cfg["base_url_port"], + history_backend=cfg["launcher_kwargs"]["history_backend"], + ) + ) + + # Build client + api_key = os.environ.get(cfg["api_key_env"]) if cfg["api_key_env"] else None + client = openai.Client( + api_key=api_key, + base_url=cluster["base_url"] + "/v1", + ) + + # Yield data to test + try: + yield backend, cfg["model"], client + finally: + # Always kill router + kill_process_tree(cluster["router"].pid) + + # If workers exist, kill them as well + if cfg["needs_workers"]: + for w in cluster.get("workers", []): + kill_process_tree(w.pid) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/features/test_basic_crud.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/features/test_basic_crud.py new file mode 100644 index 00000000..0695feaa --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/features/test_basic_crud.py @@ -0,0 +1,241 @@ +""" +Base test class for Response API e2e tests. + +This module provides base test classes that can be reused across different backends +(OpenAI, XAI, gRPC) with common test logic. +""" + +import sys +import time +from pathlib import Path + +import openai +import pytest +from openai import OpenAI +from openai.types import responses + +# Add current directory for local imports +_TEST_DIR = Path(__file__).parent +sys.path.insert(0, str(_TEST_DIR)) + + +@pytest.mark.parametrize("setup_backend", ["openai", "oracle_store"], indirect=True) +class TestResponseCRUD: + """Base class for Response API CRUD tests.""" + + def test_create_and_get_response(self, setup_backend): + """Test creating response and retrieving it.""" + _, model, client = setup_backend + + # Create response + create_resp = client.responses.create(model=model, input="Hello, world!") + assert create_resp.id is not None + assert create_resp.error is None + assert create_resp.status == "completed" + assert len(create_resp.output_text) > 0 + response_id = create_resp.id + + # Get response + get_resp = client.responses.retrieve(response_id=response_id) + assert get_resp.error is None + assert get_resp.id == response_id + assert get_resp.status == "completed" + + input_resp = client.responses.input_items.list(response_id=get_resp.id) + assert input_resp.data is not None + assert len(input_resp.data) > 0 + + @pytest.mark.skip(reason="TODO: Add delete response feature") + def test_delete_response(self, setup_backend): + """Test deleting response.""" + _, model, client = setup_backend + + # Create response + create_resp = client.responses.create(model=model, input="Test deletion") + assert create_resp.id is not None + assert create_resp.error is None + assert create_resp.status == "completed" + assert len(create_resp.output_text) > 0 + + response_id = create_resp.id + + # Delete response + client.responses.delete(response_id=response_id) + + # Verify it's deleted (should return 404) + with pytest.raises(openai.NotFoundError): + client.responses.retrieve(response_id=response_id) + + @pytest.mark.skip(reason="TODO: Add background response feature") + def test_background_response(self, setup_backend): + """Test background response execution.""" + _, model, client = setup_backend + + # Create background response + create_resp = client.responses.create( + model=model, + input="Write a short story", + background=True, + max_output_tokens=100, + ) + assert create_resp.id is not None + assert create_resp.error is None + assert create_resp.status in ["in_progress", "queued"] + + response_id = create_resp.id + + # Wait for completion + final_data = wait_for_background_task(client, response_id, timeout=60) + assert final_data.status == "completed" + + +@pytest.mark.parametrize("setup_backend", ["openai", "oracle_store"], indirect=True) +class TestConversationCRUD: + """Base class for Conversation API CRUD tests.""" + + def test_create_and_get_conversation(self, setup_backend): + """Test creating and retrieving conversation.""" + _, model, client = setup_backend + + # Create conversation + create_resp = client.conversations.create(metadata={"user": "test_user"}) + assert create_resp.id is not None + assert create_resp.created_at is not None + + create_data = create_resp.metadata + assert create_data["user"] == "test_user" + conversation_id = create_resp.id + + # Get conversation + get_resp = client.conversations.retrieve(conversation_id=conversation_id) + assert get_resp.id is not None + assert get_resp.created_at is not None + + get_data = get_resp.metadata + assert get_resp.id == conversation_id + assert get_data["user"] == "test_user" + + def test_update_conversation(self, setup_backend): + """Test updating conversation metadata.""" + _, model, client = setup_backend + + # Create conversation + create_resp = client.conversations.create(metadata={"key1": "value1"}) + assert create_resp.id is not None + assert create_resp.created_at is not None + + create_data = create_resp.metadata + assert create_data["key1"] == "value1" + assert "key2" not in create_data + conversation_id = create_resp.id + + # Update conversation + update_resp = client.conversations.update( + conversation_id=conversation_id, + metadata={"key1": "value1", "key2": "value2"}, + ) + assert update_resp.id == conversation_id + update_data = update_resp.metadata + assert update_data["key1"] == "value1" + assert update_data["key2"] == "value2" + + # Verify update + get_resp = client.conversations.retrieve(conversation_id=conversation_id) + get_data = get_resp.metadata + assert update_data["key1"] == "value1" + assert update_data["key2"] == "value2" + + def test_delete_conversation(self, setup_backend): + """Test deleting conversation.""" + _, model, client = setup_backend + + # Create conversation + create_resp = client.conversations.create() + assert create_resp.id is not None + assert create_resp.created_at is not None + conversation_id = create_resp.id + + # Delete conversation + delete_resp = client.conversations.delete(conversation_id=conversation_id) + assert delete_resp.id is not None + assert delete_resp.deleted + + # Verify deletion + with pytest.raises(openai.NotFoundError): + client.conversations.retrieve(conversation_id=conversation_id) + + def test_list_conversation_items(self, setup_backend): + """Test listing conversation items.""" + _, model, client = setup_backend + + # Create conversation + conv_resp = client.conversations.create() + assert conv_resp.id is not None + conversation_id = conv_resp.id + + # Create response with conversation + resp1 = client.responses.create( + model=model, + input="First message", + conversation=conversation_id, + max_output_tokens=50, + ) + assert resp1.error is None + resp2 = client.responses.create( + model=model, + input="Second message", + conversation=conversation_id, + max_output_tokens=50, + ) + assert resp2.error is None + + # List items + list_resp = client.conversations.items.list(conversation_id=conversation_id) + assert list_resp is not None + assert list_resp.data is not None + + list_data = list_resp.data + # Should have at least 4 items (2 inputs + 2 outputs) + assert len(list_data) >= 4 + + +def wait_for_background_task( + client: OpenAI, response_id: str, timeout: int = 30, poll_interval: float = 0.5 +) -> responses.Response: + """ + Wait for background task to complete. + + Args: + client: openai client + response_id: Response ID to poll + timeout: Max seconds to wait + poll_interval: Seconds between polls + + Returns: + Final response data + + Raises: + TimeoutError: If task doesn't complete in time + AssertionError: If task fails + """ + start_time = time.time() + + while time.time() - start_time < timeout: + resp = client.responses.retrieve(response_id=response_id) + assert resp.error is None + assert resp.id == response_id + + status = resp.status + + if status == "completed": + return resp + elif status == "failed": + raise AssertionError(f"Background task failed: {resp.error}") + elif status == "cancelled": + raise AssertionError("Background task was cancelled") + + time.sleep(poll_interval) + + raise TimeoutError( + f"Background task {response_id} did not complete within {timeout}s" + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/features/test_state_management.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/features/test_state_management.py new file mode 100644 index 00000000..88f5346a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/features/test_state_management.py @@ -0,0 +1,161 @@ +""" +State management tests for Response API. + +Tests both previous_response_id and conversation-based state management. +These tests should work across all backends (OpenAI, XAI, gRPC). +""" + +import openai +import pytest + + +@pytest.mark.parametrize( + "setup_backend", ["openai", "xai", "grpc", "grpc_harmony"], indirect=True +) +class TestStateManagement: + """Tests for state management using previous_response_id and conversation.""" + + def test_basic_response_creation(self, setup_backend): + """Test basic response creation without state.""" + _, model, client = setup_backend + + resp = client.responses.create(model=model, input="What is 2+2?") + + assert resp.id is not None + assert resp.error is None + assert resp.status == "completed" + assert len(resp.output_text) > 0 + assert resp.usage is not None + + def test_streaming_response(self, setup_backend): + """Test streaming response.""" + _, model, client = setup_backend + + resp = client.responses.create( + model=model, input="Count to 5", stream=True, max_output_tokens=50 + ) + + # Check for response.created event + events = [event for event in resp] + created_events = [event for event in events if event.type == "response.created"] + assert len(created_events) > 0 + + # Check for final completed event or in_progress events + assert any( + event.type in ["response.completed", "response.in_progress"] + for event in events + ) + + def test_previous_response_id_chaining(self, setup_backend): + """Test chaining responses using previous_response_id.""" + _, model, client = setup_backend + # First response + resp1 = client.responses.create( + model=model, input="My name is Alice and my friend is Bob. Remember it." + ) + assert resp1.error is None + assert resp1.status == "completed" + response1_id = resp1.id + + # Second response referencing first + resp2 = client.responses.create( + model=model, input="What is my name", previous_response_id=response1_id + ) + assert resp2.error is None + assert resp2.status == "completed" + + # The model should remember the name from previous response + assert "Alice" in resp2.output_text + + # Third response referencing second + resp3 = client.responses.create( + model=model, + input="What is my friend name?", + previous_response_id=resp2.id, + ) + assert resp3.error is None + assert resp3.status == "completed" + assert "Bob" in resp3.output_text + + @pytest.mark.skip(reason="TODO: Add the invalid previous_response_id check") + def test_previous_response_id_invalid(self, setup_backend): + """Test using invalid previous_response_id.""" + _, model, client = setup_backend + with pytest.raises(openai.BadRequestError): + client.responses.create( + model=model, + input="Test", + previous_response_id="resp_invalid123", + max_output_tokens=50, + ) + + def test_conversation_with_multiple_turns(self, setup_backend): + """Test state management using conversation ID.""" + backend, model, client = setup_backend + + if backend in ["grpc", "grpc_harmony"]: + pytest.skip("TODO: 501 Not Implemented") + + # Create conversation + conv_resp = client.conversations.create(metadata={"topic": "math"}) + assert conv_resp.id is not None + assert conv_resp.created_at is not None + + conversation_id = conv_resp.id + + # First response in conversation + resp1 = client.responses.create( + model=model, input="I have 5 apples.", conversation=conversation_id + ) + assert resp1.error is None + assert resp1.status == "completed" + + # Second response in same conversation + resp2 = client.responses.create( + model=model, + input="How many apples do I have?", + conversation=conversation_id, + ) + assert resp2.error is None + assert resp2.status == "completed" + output_text = resp2.output_text + + # Should remember "5 apples" + assert "5" in output_text or "five" in output_text.lower() + + # Third response in same conversation + resp3 = client.responses.create( + model=model, + input="If I get 3 more, how many total?", + conversation=conversation_id, + ) + assert resp3.error is None + assert resp3.status == "completed" + output_text = resp3.output_text + + # Should calculate 5 + 3 = 8 + assert "8" in output_text or "eight" in output_text.lower() + list_resp = client.conversations.items.list(conversation_id) + assert list_resp.data is not None + items = list_resp.data + # Should have at least 6 items (3 inputs + 3 outputs) + assert len(items) >= 6 + + def test_mutually_exclusive_parameters(self, setup_backend): + """Test that previous_response_id and conversation are mutually exclusive.""" + _, model, client = setup_backend + + # TODO: Remove this once the conversation API is implemented for GRPC backend + conversation_id = "conv_123" + + resp1 = client.responses.create(model=model, input="Test") + response1_id = resp1.id + + # Try to use both parameters + with pytest.raises(openai.BadRequestError): + client.responses.create( + model=model, + input="This should fail", + previous_response_id=response1_id, + conversation=conversation_id, + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/features/test_streaming_events.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/features/test_streaming_events.py new file mode 100644 index 00000000..7b23d8b1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/features/test_streaming_events.py @@ -0,0 +1,152 @@ +""" +Streaming events tests for Response API. +Tests for streaming event validation including: +- Zero-based output_index for reasoning content +- OutputItemDone event emission and output array construction +""" + +import pytest + + +@pytest.mark.parametrize("setup_backend", ["grpc", "grpc_harmony"], indirect=True) +class TestStreamingEvents: + """Tests for streaming event validation.""" + + def test_output_item_event_emitted(self, setup_backend): + """ + Test that output_index is zero-based in streaming responses. + Verifies that the first output item has output_index: 0. + """ + _, model, client = setup_backend + + resp = client.responses.create( + model=model, + input="Count from 1 to 3", + stream=True, + max_output_tokens=50, + ) + + events = [event for event in resp] + assert len(events) > 0 + + # Find output_item.added events + output_item_added_events = [ + event for event in events if event.type == "response.output_item.added" + ] + assert len(output_item_added_events) > 0, "Should have output_item.added events" + + # Verify first output item has output_index: 0 + first_item_event = output_item_added_events[0] + assert first_item_event.item is not None + assert first_item_event.output_index is not None + assert ( + first_item_event.output_index == 0 + ), "First output item must have output_index: 0 (zero-based indexing)" + + # Verify subsequent items increment correctly + for i, event in enumerate(output_item_added_events): + assert ( + event.output_index == i + ), f"Output item {i} should have output_index: {i}" + + # Verify output_item.done event exists + output_item_done_events = [ + event for event in events if event.type == "response.output_item.done" + ] + assert len(output_item_done_events) > 0 + + # Verify output_item.done event structure + for event in output_item_done_events: + assert event.item is not None + assert event.output_index is not None + assert event.item.type is not None + + # Find response.completed event + completed_events = [ + event for event in events if event.type == "response.completed" + ] + assert len(completed_events) == 1, "Should have exactly one completed event" + + # Verify output array exists and contains items + completed_event = completed_events[0] + + assert completed_event.response.output is not None + output_array = completed_event.response.output + assert isinstance(output_array, list) + assert len(output_array) > 0, "Output array should contain at least one item" + + # Verify each item in output array has proper structure + for i, item in enumerate(output_array): + assert item.type is not None + + # Verify output_item.added events match items in final output array + output_item_added_events = [ + event for event in events if event.type == "response.output_item.added" + ] + + assert len(output_item_added_events) == len( + output_array + ), "Number of output_item.added events should match output array length" + + def test_reasoning_content(self, setup_backend): + """ + Test that reasoning content has correct zero-based output_index. + Specifically tests that reasoning item has output_index: 0 + and message item has output_index: 1. + """ + backend, model, client = setup_backend + if backend in ["grpc"]: + pytest.skip("skip test_reasoning_content for grpc") + + resp = client.responses.create( + model=model, + input="What is the capital of France? Think step by step.", + stream=True, + max_output_tokens=200, + ) + + events = [event for event in resp] + assert len(events) > 0 + + # Find output_item.added events + output_item_added_events = [ + event for event in events if event.type == "response.output_item.added" + ] + assert len(output_item_added_events) > 0 + + reasoning_items = [ + item for item in output_item_added_events if item.item.type == "reasoning" + ] + message_items = [ + item for item in output_item_added_events if item.item.type == "message" + ] + + # If reasoning is present, verify it has output_index: 0 + if reasoning_items: + reasoning_item = reasoning_items[0] + assert ( + reasoning_item.output_index == 0 + ), "Reasoning item should have output_index: 0" + + # If message is present after reasoning, verify it has output_index: 1 + if reasoning_items and message_items: + message_item = message_items[0] + assert ( + message_item.output_index == 1 + ), "Message item after reasoning should have output_index: 1" + + # Find response.completed event + completed_events = [ + event for event in events if event.type == "response.completed" + ] + assert len(completed_events) == 1 + + # Get output array from completed event + output_array = completed_events[0].response.output + assert len(output_array) > 0 + + # Check if reasoning items are in output array + reasoning_items_in_output = [ + item for item in output_array if item.type == "reasoning" + ] + assert len(reasoning_items_in_output) > 0 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/features/test_structured_output.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/features/test_structured_output.py new file mode 100644 index 00000000..adfce1e0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/features/test_structured_output.py @@ -0,0 +1,172 @@ +""" +Structured output tests for Response API. + +Tests for text.format field with json_object and json_schema formats. +""" + +import json +import sys +from pathlib import Path + +import pytest + +# Add current directory for local imports +_TEST_DIR = Path(__file__).parent +sys.path.insert(0, str(_TEST_DIR)) + + +@pytest.mark.parametrize("setup_backend", ["openai", "grpc_harmony"], indirect=True) +class TestStructuredOutput: + + def test_structured_output_json_schema(self, setup_backend): + """Test structured output with json_schema format.""" + _, model, client = setup_backend + + # Create response with structured output + params = { + "model": model, + "input": [ + { + "role": "system", + "content": "You are a helpful math tutor. Guide the user through the solution step by step.", + }, + {"role": "user", "content": "how can I solve 8x + 7 = -23"}, + ], + "text": { + "format": { + "type": "json_schema", + "name": "math_reasoning", + "schema": { + "type": "object", + "properties": { + "steps": { + "type": "array", + "items": { + "type": "object", + "properties": { + "explanation": {"type": "string"}, + "output": {"type": "string"}, + }, + "required": ["explanation", "output"], + "additionalProperties": False, + }, + }, + "final_answer": {"type": "string"}, + }, + "required": ["steps", "final_answer"], + "additionalProperties": False, + }, + "strict": True, + } + }, + } + + create_resp = client.responses.create(**params) + assert create_resp.error is None + assert create_resp.id is not None + assert create_resp.output is not None + assert create_resp.text is not None + + # Verify text format was echoed back correctly + assert create_resp.text.format is not None + assert create_resp.text.format.type == "json_schema" + assert create_resp.text.format.name == "math_reasoning" + assert create_resp.text.format.schema_ is not None + assert create_resp.text.format.strict + + # Find the message output (output[0] may be reasoning, output[1] is message) + output_text = next( + ( + content.text + for item in create_resp.output + if item.type == "message" + for content in item.content + if content.type == "output_text" + ), + None, + ) + + assert output_text is not None, "No output_text found in response" + assert output_text.strip(), "output_text is empty" + + # Parse JSON output + output_json = json.loads(output_text) + + # Verify schema structure + assert "steps" in output_json + assert "final_answer" in output_json + assert isinstance(output_json["steps"], list) + assert len(output_json["steps"]) > 0 + + # Verify each step has required fields + for step in output_json["steps"]: + assert "explanation" in step + assert "output" in step + + +@pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True) +class TestSimpleSchemaStructuredOutput: + + def test_structured_output_json_schema(self, setup_backend): + """Override with simpler schema for Llama model (complex schemas not well supported).""" + _, model, client = setup_backend + + params = { + "model": model, + "input": [ + { + "role": "system", + "content": "You are a math solver. Return ONLY a JSON object that matches the schema—no extra text.", + }, + { + "role": "user", + "content": "What is 1 + 1?", + }, + ], + "text": { + "format": { + "type": "json_schema", + "name": "math_answer", + "schema": { + "type": "object", + "properties": {"answer": {"type": "string"}}, + "required": ["answer"], + }, + } + }, + } + + create_resp = client.responses.create(**params) + assert create_resp.error is None + assert create_resp.id is not None + assert create_resp.output is not None + assert create_resp.text is not None + + # Verify text format was echoed back correctly + assert create_resp.text.format is not None + assert create_resp.text.format.type == "json_schema" + assert create_resp.text.format.name == "math_answer" + assert create_resp.text.format.schema_ is not None + + # Find the message output + output_text = next( + ( + content.text + for item in create_resp.output + if item.type == "message" + for content in item.content + if content.type == "output_text" + ), + None, + ) + + assert output_text is not None, "No output_text found in response" + assert output_text.strip(), "output_text is empty" + + # Parse JSON output + output_json = json.loads(output_text) + + # Verify simple schema structure (just answer field) + assert "answer" in output_json + assert isinstance(output_json["answer"], str) + assert output_json["answer"], "Answer is empty" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/features/test_tools_call.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/features/test_tools_call.py new file mode 100644 index 00000000..29d58f18 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/features/test_tools_call.py @@ -0,0 +1,765 @@ +""" +Test class for tool calling tests. + +This module provides test cases for function calling functionality, tool choices +and mcp calling functionality across different backends. +""" + +import json +import sys +import time +from pathlib import Path + +import pytest + +# Add current directory for local imports +_TEST_DIR = Path(__file__).parent +sys.path.insert(0, str(_TEST_DIR)) + + +@pytest.mark.parametrize( + "setup_backend", ["openai", "grpc", "grpc_harmony"], indirect=True +) +class TestToolCalling: + + # Shared function tool definitions + SYSTEM_DIAGNOSTICS_FUNCTION = { + "type": "function", + "name": "get_system_diagnostics", + "description": "Retrieve real-time diagnostics for a spacecraft system.", + "parameters": { + "type": "object", + "properties": { + "system_name": { + "type": "string", + "description": "Name of the spacecraft system to query. " + "Example: 'Astra-7 Core Reactor'.", + } + }, + "required": ["system_name"], + }, + } + + GET_WEATHER_FUNCTION = { + "type": "function", + "name": "get_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city name, e.g., San Francisco", + } + }, + "required": ["location"], + }, + } + + CALCULATE_FUNCTION = { + "type": "function", + "name": "calculate", + "description": "Perform a mathematical calculation", + "parameters": { + "type": "object", + "properties": { + "expression": { + "type": "string", + "description": "The mathematical expression to evaluate", + } + }, + "required": ["expression"], + }, + } + + SEARCH_WEB_FUNCTION = { + "type": "function", + "name": "search_web", + "description": "Search the web for information", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + } + + LOCAL_SEARCH_FUNCTION = { + "type": "function", + "name": "local_search", + "description": "Search local database", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + } + + # Shared constants for MCP tests + BRAVE_MCP_TOOL = { + "type": "mcp", + "server_label": "brave", + "server_description": "A Tool to do web search", + "server_url": "http://localhost:8001/sse", + "require_approval": "never", + } + + DEEPWIKI_MCP_TOOL = { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "require_approval": "never", + } + + MCP_TEST_PROMPT = ( + "show me some news about sglang router, use the tool to just search " + "one result and return one sentence response" + ) + + # Test cases for basic function calling functionality + + def test_basic_function_call(self, setup_backend): + """ + Test basic function calling workflow. + + This test follows the pattern from function_call_test.py: + 1. Define a function tool (get_horoscope) + 2. Send user message asking for horoscope + 3. Model should return function_call + 4. Execute function locally and provide output + 5. Model should generate final response using the function output + """ + backend, model, client = setup_backend + + if backend in ["grpc"]: + pytest.skip("skip for grpc") + + # 1. Define a list of callable tools for the model + tools = [ + { + "type": "function", + "name": "get_horoscope", + "description": "Get today's horoscope for an astrological sign.", + "parameters": { + "type": "object", + "properties": { + "sign": { + "type": "string", + "description": "An astrological sign like Taurus or Aquarius", + }, + }, + "required": ["sign"], + }, + }, + ] + system_prompt = ( + "You are a helpful assistant that can call functions. " + "When a user asks for horoscope information, call the function. " + "IMPORTANT: Don't reply directly to the user, only call the function. " + ) + + # Create a running input list we will add to over time + input_list = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": "What is my horoscope? I am an Aquarius."}, + ] + + # 2. Prompt the model with tools defined + resp = client.responses.create(model=model, input=input_list, tools=tools) + + # Should successfully make the request + assert resp.error is None + + # Basic response structure + assert resp.id is not None + assert resp.status == "completed" + assert resp.output is not None + + # Verify output array is not empty + output = resp.output + assert isinstance(output, list) + assert len(output) > 0 + + # Check for function_call in output + function_calls = [item for item in output if item.type == "function_call"] + assert ( + len(function_calls) > 0 + ), "Response should contain at least one function_call" + + # Verify function_call structure + function_call = function_calls[0] + assert function_call.call_id is not None + assert function_call.name is not None + assert function_call.name == "get_horoscope" + assert function_call.arguments is not None + + # Parse arguments + args = json.loads(function_call.arguments) + assert "sign" in args + assert args["sign"].lower() == "aquarius" + + # 3. Save function call outputs for subsequent requests + input_list.append(function_call) + + # 4. Execute the function logic for get_horoscope + horoscope = f"{args['sign']}: Next Tuesday you will befriend a baby otter." + + # 5. Provide function call results to the model + input_list.append( + { + "type": "function_call_output", + "call_id": function_call.call_id, + "output": json.dumps({"horoscope": horoscope}), + } + ) + + # 6. Make second request with function output + resp2 = client.responses.create( + model=model, + input=input_list, + instructions="Respond only with a horoscope generated by a tool.", + tools=tools, + ) + assert resp2.error is None + assert resp2.status == "completed" + + # The model should be able to give a response using the function output + output2 = resp2.output + assert len(output2) > 0 + + # Find message output + messages = [item for item in output2 if item.type == "message"] + assert len(messages) > 0, "Response should contain at least one message" + + # Verify message contains the horoscope + message = messages[0] + assert message.content is not None + content_parts = message.content + assert len(content_parts) > 0 + + # Get text from content + text_parts = [part.text for part in content_parts if part.type == "output_text"] + full_text = " ".join(text_parts).lower() + + # Should mention the horoscope or baby otter + assert ( + "baby otter" in full_text or "aquarius" in full_text + ), "Response should reference the horoscope content" + + # Test cases for tool_choice parameter support, these tests require --reasoning-parser + + def test_tool_choice_auto(self, setup_backend): + """ + Test tool_choice="auto" allows model to decide whether to use tools. + + The model should be able to choose to call a tool or not. + """ + backend, model, client = setup_backend + + if backend in ["openai"]: + pytest.skip("skip for openai") + + tools = [self.GET_WEATHER_FUNCTION] + + # Query that should trigger tool use + resp = client.responses.create( + model=model, + input="What is the weather in Seattle?", + tools=tools, + tool_choice="auto", + stream=False, + ) + + assert resp.id is not None + assert resp.error is None + + output = resp.output + assert len(output) > 0 + + # With auto, model should choose to call get_weather for this query + function_calls = [item for item in output if item.type == "function_call"] + assert ( + len(function_calls) > 0 + ), "Model should choose to call function with tool_choice='auto'" + + def test_tool_choice_required(self, setup_backend): + """ + Test tool_choice="required" forces the model to call at least one tool. + + The model must make at least one function call. + """ + backend, model, client = setup_backend + + if backend in ["openai"]: + pytest.skip("skip for openai") + + tools = [self.CALCULATE_FUNCTION] + + resp = client.responses.create( + model=model, + input="What is 15 * 23?", + tools=tools, + tool_choice="required", + stream=False, + ) + + assert resp.id is not None + assert resp.error is None + + output = resp.output + + # Must have at least one function call + function_calls = [item for item in output if item.type == "function_call"] + assert ( + len(function_calls) > 0 + ), "tool_choice='required' must force at least one function call" + + def test_tool_choice_specific_function(self, setup_backend): + """ + Test tool_choice with specific function name forces that function to be called. + + The model must call the specified function. + """ + backend, model, client = setup_backend + + if backend in ["openai"]: + pytest.skip("skip for openai") + + tools = [self.SEARCH_WEB_FUNCTION, self.GET_WEATHER_FUNCTION] + + # Force specific function call + resp = client.responses.create( + model=model, + input="What's happening in the news today?", + tools=tools, + tool_choice={"type": "function", "function": {"name": "search_web"}}, + stream=False, + ) + + assert resp.id is not None + assert resp.error is None + + output = resp.output + + # Must have function call + function_calls = [item for item in output if item.type == "function_call"] + assert len(function_calls) > 0, "Must call the specified function" + + # Must be the specified function + called_function = function_calls[0] + assert ( + called_function.name == "search_web" + ), "Must call the function specified in tool_choice" + + def test_tool_choice_streaming(self, setup_backend): + """ + Test tool_choice parameter works correctly with streaming. + + Verifies that tool_choice constraints are applied in streaming mode. + """ + backend, model, client = setup_backend + + if backend in ["openai", "grpc"]: + pytest.skip("skip for openai") + + tools = [self.CALCULATE_FUNCTION] + + resp = client.responses.create( + model=model, + input="Calculate 42 * 17", + tools=tools, + tool_choice="required", + stream=True, + ) + + events = [event for event in resp] + assert len(events) > 0 + + event_types = [e.type for e in events] + + # Should have function call events + assert ( + "response.function_call_arguments.delta" in event_types + ), "Should have function_call_arguments.delta events" + + # Verify completed event has function call + completed_events = [e for e in events if e.type == "response.completed"] + assert len(completed_events) == 1 + + output = completed_events[0].response.output + + function_calls = [item for item in output if item.type == "function_call"] + assert ( + len(function_calls) > 0 + ), "Streaming with tool_choice='required' must produce function call" + + def test_tool_choice_with_mcp_tools(self, setup_backend): + """ + Test tool_choice parameter works with MCP tools. + + Verifies that tool_choice can control MCP tool usage. + """ + backend, model, client = setup_backend + + if backend in ["openai"]: + pytest.skip("skip for openai") + + tools = [self.DEEPWIKI_MCP_TOOL] + + # With tool_choice="auto", should allow MCP tool calls + resp = client.responses.create( + model=model, + input="What transport protocols does the 2025-03-26 version of the MCP spec (modelcontextprotocol/modelcontextprotocol) support?", + tools=tools, + tool_choice="auto", + stream=False, + ) + + assert resp.id is not None + assert resp.error is None + + output = resp.output + + # Should have mcp_call with auto + mcp_calls = [item for item in output if item.type == "mcp_call"] + assert len(mcp_calls) > 0, "tool_choice='auto' should allow MCP tool calls" + + def test_tool_choice_mixed_function_and_mcp(self, setup_backend): + """ + Test tool_choice with mixed function and MCP tools. + + Verifies tool_choice can select specific tools when both function and MCP tools are available. + """ + backend, model, client = setup_backend + + if backend in ["openai"]: + pytest.skip("skip for openai") + + tools = [self.DEEPWIKI_MCP_TOOL, self.LOCAL_SEARCH_FUNCTION] + + # Force specific function call + resp = client.responses.create( + model=model, + input="Search for information about Python", + tools=tools, + tool_choice={"type": "function", "function": {"name": "local_search"}}, + stream=False, + ) + + assert resp.id is not None + assert resp.error is None + + output = resp.output + + # Must call local_search, not MCP + function_calls = [item for item in output if item.type == "function_call"] + assert len(function_calls) > 0 + assert function_calls[0].name == "local_search" + + # Should not have mcp_call + mcp_calls = [item for item in output if item.type == "mcp_call"] + assert len(mcp_calls) == 0, "Should only call specified function, not MCP tools" + + # Tests for MCP tool calling in both streaming and non-streaming modes. + + def test_mcp_basic_tool_call(self, setup_backend): + """ + Test basic MCP tool call (non-streaming). + Validation strictness is controlled by parameter `backend` from setup_backend fixture. + Set to "strict" if backend is http. + """ + backend, model, client = setup_backend + + # To avoid being rate-limited by brave search server + time.sleep(2) + + resp = client.responses.create( + model=model, + input=self.MCP_TEST_PROMPT, + tools=[self.BRAVE_MCP_TOOL], + stream=False, + reasoning={"effort": "low"}, + ) + + # Should successfully make the request + assert resp.error is None + + # Basic response structure + assert resp.id is not None + assert resp.status == "completed" + assert resp.model is not None + assert resp.output is not None + + # Verify output array is not empty + assert len(resp.output_text) > 0 + + # Check for MCP-specific output types + output_types = [item.type for item in resp.output] + + # Should have mcp_list_tools - tools are listed before calling + assert ( + "mcp_list_tools" in output_types + ), "Response should contain mcp_list_tools" + + # Should have at least one mcp_call + mcp_calls = [item for item in resp.output if item.type == "mcp_call"] + assert len(mcp_calls) > 0, "Response should contain at least one mcp_call" + + # Verify mcp_call structure + for mcp_call in mcp_calls: + assert mcp_call.id is not None + assert mcp_call.error is None + assert mcp_call.status == "completed" + assert mcp_call.server_label == "brave" + assert mcp_call.name is not None + assert mcp_call.arguments is not None + assert mcp_call.output is not None + + # Strict mode: additional validation for HTTP backends + if backend == "openai": + # Should have final message output + messages = [item for item in resp.output if item.type == "message"] + assert len(messages) > 0, "Response should contain at least one message" + # Verify message structure + for msg in messages: + assert msg.content is not None + assert isinstance(msg.content, list) + + # Check content has text + for content_item in msg.content: + if content_item.type == "output_text": + assert content_item.text is not None + assert isinstance(content_item.text, str) + assert len(content_item.text) > 0 + + def test_mcp_basic_tool_call_streaming(self, setup_backend): + """Test basic MCP tool call (streaming). + + Validation strictness is controlled by the class attribute `mcp_validation_mode`. + Set to "strict" in subclasses for additional HTTP-specific validation. + """ + backend, model, client = setup_backend + + # To avoid being rate-limited by brave search server + time.sleep(2) + + resp = client.responses.create( + model=model, + input=self.MCP_TEST_PROMPT, + tools=[self.BRAVE_MCP_TOOL], + stream=True, + reasoning={"effort": "low"}, + ) + + # Should successfully make the request + events = [event for event in resp] + assert len(events) > 0 + + event_types = [event.type for event in events] + # Check for lifecycle events + assert "response.created" in event_types, "Should have response.created event" + assert ( + "response.completed" in event_types + ), "Should have response.completed event" + + # Check for MCP list tools events + assert ( + "response.output_item.added" in event_types + ), "Should have output_item.added events" + assert ( + "response.mcp_list_tools.in_progress" in event_types + ), "Should have mcp_list_tools.in_progress event" + assert ( + "response.mcp_list_tools.completed" in event_types + ), "Should have mcp_list_tools.completed event" + + # Check for MCP call events + assert ( + "response.mcp_call.in_progress" in event_types + ), "Should have mcp_call.in_progress event" + assert ( + "response.mcp_call_arguments.delta" in event_types + ), "Should have mcp_call_arguments.delta event" + assert ( + "response.mcp_call_arguments.done" in event_types + ), "Should have mcp_call_arguments.done event" + assert ( + "response.mcp_call.completed" in event_types + ), "Should have mcp_call.completed event" + + # Verify final completed event has full response + completed_events = [e for e in events if e.type == "response.completed"] + assert len(completed_events) == 1 + + final_response = completed_events[0].response + assert final_response.id is not None + assert final_response.status == "completed" + assert final_response.output is not None + + # Verify final output contains expected items + final_output = final_response.output + final_output_types = [item.type for item in final_output] + + assert "mcp_list_tools" in final_output_types + assert "mcp_call" in final_output_types + + # Verify mcp_call items in final output + mcp_calls = [item for item in final_output if item.type == "mcp_call"] + assert len(mcp_calls) > 0 + + for mcp_call in mcp_calls: + assert mcp_call.error is None + assert mcp_call.status == "completed" + assert mcp_call.server_label == "brave" + assert mcp_call.name is not None + assert mcp_call.arguments is not None + assert mcp_call.output is not None + + # Strict mode: additional validation for HTTP backends + if backend == "openai": + # Check for text output events + assert ( + "response.content_part.added" in event_types + ), "Should have content_part.added event" + assert ( + "response.output_text.delta" in event_types + ), "Should have output_text.delta events" + assert ( + "response.output_text.done" in event_types + ), "Should have output_text.done event" + assert ( + "response.content_part.done" in event_types + ), "Should have content_part.done event" + + assert "message" in final_output_types + + # Verify text deltas combine to final message + text_deltas = [ + e.delta for e in events if e.type == "response.output_text.delta" + ] + assert len(text_deltas) > 0, "Should have text deltas" + + # Get final text from output_text.done event + text_done_events = [ + e for e in events if e.type == "response.output_text.done" + ] + assert len(text_done_events) > 0 + + final_text = text_done_events[0].text + assert len(final_text) > 0, "Final text should not be empty" + + def test_mixed_mcp_and_function_tools(self, setup_backend): + """Test mixed MCP and function tools (non-streaming).""" + backend, model, client = setup_backend + + if backend in ["openai"]: + pytest.skip( + "Requires external MCP server (deepwiki) - may not be accessible in CI" + ) + + resp = client.responses.create( + model=model, + input="Give me diagnostics for the Astra-7 Core Reactor.", + tools=[self.BRAVE_MCP_TOOL, self.SYSTEM_DIAGNOSTICS_FUNCTION], + stream=False, + tool_choice="auto", + ) + + # Should successfully make the request + assert resp.error is None + + # Basic response structure + assert resp.id is not None + assert resp.status is not None + assert resp.output is not None + + # Verify output array is not empty + output = resp.output + assert isinstance(output, list) + assert len(output) > 0 + + # Check for function_call (not mcp_call for get_system_diagnostics) + function_calls = [item for item in output if item.type == "function_call"] + assert ( + len(function_calls) > 0 + ), "Response should contain at least one function_call" + + # Verify function_call structure for get_system_diagnostics + system_diagnostics_call = function_calls[0] + assert system_diagnostics_call.name == "get_system_diagnostics" + assert system_diagnostics_call.call_id is not None + assert system_diagnostics_call.arguments is not None + assert system_diagnostics_call.status is not None + + # Parse and verify arguments + args = json.loads(system_diagnostics_call.arguments) + assert "system_name" in args + assert "astra-7" in args["system_name"].lower() + + def test_mixed_mcp_and_function_tools_streaming(self, setup_backend): + """Test mixed MCP and function tools (streaming).""" + backend, model, client = setup_backend + + if backend in ["openai"]: + pytest.skip( + "Requires external MCP server (deepwiki) - may not be accessible in CI" + ) + + resp = client.responses.create( + model=model, + input="Give me diagnostics for the Astra-7 Core Reactor.", + tools=[self.BRAVE_MCP_TOOL, self.SYSTEM_DIAGNOSTICS_FUNCTION], + stream=True, + tool_choice="auto", # Encourage tool usage + ) + + # Should successfully make the request + events = [event for event in resp] + assert len(events) > 0 + + event_types = [e.type for e in events] + + # Check for lifecycle events + assert "response.created" in event_types, "Should have response.created event" + + # Should have mcp_list_tools events + assert ( + "response.mcp_list_tools.completed" in event_types + ), "Should have mcp_list_tools.completed event" + + # Should have function_call_arguments events (not mcp_call_arguments) + assert ( + "response.function_call_arguments.delta" in event_types + ), "Should have function_call_arguments.delta event for function tools" + assert ( + "response.function_call_arguments.done" in event_types + ), "Should have function_call_arguments.done event for function tools" + + # Should NOT have mcp_call_arguments events for function tools + # (get_system_diagnostics should use function_call_arguments, not mcp_call_arguments) + mcp_call_arg_events = [ + e + for e in events + if e.type == "response.mcp_call_arguments.delta" + and "get_system_diagnostics" in str(e.delta) + ] + assert ( + len(mcp_call_arg_events) == 0 + ), "Should NOT emit mcp_call_arguments.delta for function tools (get_system_diagnostics)" + + # Verify function_call_arguments.delta event structure + func_arg_deltas = [ + e for e in events if e.type == "response.function_call_arguments.delta" + ] + assert ( + len(func_arg_deltas) > 0 + ), "Should have function_call_arguments.delta events" + + # Check that delta event contains system_name arguments + full_delta_event = "" + for event in func_arg_deltas: + full_delta_event += event.delta + + assert ( + "system_name" in full_delta_event.lower() + and "astra-7" in full_delta_event.lower() + ), "function_call_arguments.delta should contain system_name and astra-7" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/router_fixtures.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/router_fixtures.py new file mode 100644 index 00000000..74b6b1b0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/router_fixtures.py @@ -0,0 +1,565 @@ +""" +Fixtures for launching OpenAI/XAI router for response API e2e testing. + +This module provides fixtures for launching SGLang router with OpenAI or XAI backends: + 1. Launch router with --backend openai pointing to OpenAI or XAI API + 2. Configure history backend (memory or oracle) + +This supports testing the Response API against real cloud providers. +""" + +import logging +import os +import socket +import subprocess +import time +from typing import Optional + +import requests + +logger = logging.getLogger(__name__) + + +def wait_for_workers_ready( + router_url: str, + expected_workers: int, + timeout: int = 300, + api_key: Optional[str] = None, +) -> None: + """ + Wait for router to have all workers connected. + + Polls the /workers endpoint until the 'total' field matches expected_workers. + + Example response from /workers endpoint: + {"workers":[],"total":0,"stats":{"prefill_count":0,"decode_count":0,"regular_count":0}} + + Args: + router_url: Base URL of router (e.g., "http://127.0.0.1:30000") + expected_workers: Number of workers expected to be connected + timeout: Max seconds to wait + api_key: Optional API key for authentication + """ + start_time = time.time() + last_error = None + attempt = 0 + + headers = {} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + + with requests.Session() as session: + while time.time() - start_time < timeout: + attempt += 1 + elapsed = int(time.time() - start_time) + + # Log progress every 10 seconds + if elapsed > 0 and elapsed % 10 == 0 and attempt % 10 == 0: + logger.info( + f" Still waiting for workers... ({elapsed}/{timeout}s elapsed)" + ) + + try: + response = session.get( + f"{router_url}/workers", headers=headers, timeout=5 + ) + if response.status_code == 200: + data = response.json() + total_workers = data.get("total", 0) + + if total_workers == expected_workers: + logger.info( + f" All {expected_workers} workers connected after {elapsed}s" + ) + return + else: + last_error = f"Workers: {total_workers}/{expected_workers}" + else: + last_error = f"HTTP {response.status_code}" + except requests.ConnectionError: + last_error = "Connection refused (router not ready yet)" + except requests.Timeout: + last_error = "Timeout" + except requests.RequestException as e: + last_error = str(e) + except (ValueError, KeyError) as e: + last_error = f"Invalid response: {e}" + + time.sleep(1) + + raise TimeoutError( + f"Router at {router_url} did not get {expected_workers} workers within {timeout}s.\n" + f"Last status: {last_error}\n" + f"Hint: Run with SHOW_ROUTER_LOGS=1 to see startup logs" + ) + + +def find_free_port() -> int: + """Find an available port on localhost.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def wait_for_router_ready( + router_url: str, + timeout: int = 60, + api_key: Optional[str] = None, +) -> None: + """ + Wait for router to be ready. + + Polls the /health endpoint until it returns 200. + + Args: + router_url: Base URL of router (e.g., "http://127.0.0.1:30000") + timeout: Max seconds to wait + api_key: Optional API key for authentication + """ + start_time = time.time() + last_error = None + attempt = 0 + + headers = {} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + + with requests.Session() as session: + while time.time() - start_time < timeout: + attempt += 1 + elapsed = int(time.time() - start_time) + + # Log progress every 10 seconds + if elapsed > 0 and elapsed % 10 == 0 and attempt % 10 == 0: + logger.info( + f" Still waiting for router... ({elapsed}/{timeout}s elapsed)" + ) + + try: + response = session.get( + f"{router_url}/health", headers=headers, timeout=5 + ) + if response.status_code == 200: + logger.info(f" Router ready after {elapsed}s") + return + else: + last_error = f"HTTP {response.status_code}" + except requests.ConnectionError: + last_error = "Connection refused (router not ready yet)" + except requests.Timeout: + last_error = "Timeout" + except requests.RequestException as e: + last_error = str(e) + + time.sleep(1) + + raise TimeoutError( + f"Router at {router_url} did not become ready within {timeout}s.\n" + f"Last status: {last_error}\n" + f"Hint: Run with SHOW_ROUTER_LOGS=1 to see startup logs" + ) + + +def popen_launch_openai_xai_router( + backend: str, # "openai" or "xai" + base_url: str, + timeout: int = 60, + history_backend: str = "memory", + api_key: Optional[str] = None, + router_args: Optional[list] = None, + stdout=None, + stderr=None, + prometheus_port: Optional[int] = None, +) -> dict: + """ + Launch SGLang router with OpenAI or XAI backend. + + This approach: + 1. Starts router with --backend openai + 2. Points to OpenAI or XAI API via --worker-urls + 3. Configures history backend (memory or oracle) + 4. Waits for router health check to pass + + Args: + backend: "openai" or "xai" + base_url: Base URL for router (e.g., "http://127.0.0.1:30000") + timeout: Timeout for router startup (default: 60s) + history_backend: "memory" or "oracle" (default: memory) + api_key: Optional API key for router authentication + router_args: Additional arguments for router + stdout: Optional file handle for router stdout + stderr: Optional file handle for router stderr + + Returns: + dict with: + - router: router process object + - base_url: router URL (HTTP endpoint) + + Example: + >>> cluster = popen_launch_openai_xai_router( + ... "openai", "http://127.0.0.1:30000" + ... ) + >>> # Use cluster['base_url'] for HTTP requests + >>> # Cleanup: + >>> kill_process_tree(cluster['router'].pid) + """ + show_output = os.environ.get("SHOW_ROUTER_LOGS", "0") == "1" + + # Parse router port from base_url + if ":" in base_url.split("//")[-1]: + router_port = int(base_url.split(":")[-1]) + else: + router_port = find_free_port() + + logger.info(f"\n{'='*70}") + logger.info(f"Launching {backend.upper()} router") + logger.info(f"{'='*70}") + logger.info(f" Backend: {backend}") + logger.info(f" Router port: {router_port}") + logger.info(f" History backend: {history_backend}") + + # Determine worker URL based on backend + if backend == "openai": + worker_url = "https://api.openai.com" + # Get API key from environment + backend_api_key = os.environ.get("OPENAI_API_KEY") + if not backend_api_key: + raise ValueError( + "OPENAI_API_KEY environment variable must be set for OpenAI backend" + ) + elif backend == "xai": + worker_url = "https://api.x.ai" + # Get API key from environment + backend_api_key = os.environ.get("XAI_API_KEY") + if not backend_api_key: + raise ValueError( + "XAI_API_KEY environment variable must be set for XAI backend" + ) + else: + raise ValueError(f"Unsupported backend: {backend}") + + logger.info(f" Worker URL: {worker_url}") + + # Build router command + router_cmd = [ + "python3", + "-m", + "sglang_router.launch_router", + "--host", + "127.0.0.1", + "--port", + str(router_port), + "--backend", + "openai", + "--worker-urls", + worker_url, + "--history-backend", + history_backend, + "--log-level", + "warn", + ] + + # Note: Not adding --api-key to router command for local testing + # The router will not require authentication + + # Add Prometheus port to avoid conflicts (use unique port or disable) + if prometheus_port is None: + # Auto-assign a unique prometheus port based on router port + prometheus_port = router_port + 1000 + router_cmd.extend(["--prometheus-port", str(prometheus_port)]) + + # Add router-specific args + if router_args: + router_cmd.extend(router_args) + + if show_output: + logger.info(f" Command: {' '.join(router_cmd)}") + + # Set up environment with backend API key + env = os.environ.copy() + if backend == "openai": + env["OPENAI_API_KEY"] = backend_api_key + else: + env["XAI_API_KEY"] = backend_api_key + + # Launch router + if show_output: + router_proc = subprocess.Popen( + router_cmd, + env=env, + stdout=stdout, + stderr=stderr, + ) + else: + router_proc = subprocess.Popen( + router_cmd, + stdout=stdout if stdout is not None else subprocess.PIPE, + stderr=stderr if stderr is not None else subprocess.PIPE, + env=env, + ) + + print(f" PID: {router_proc.pid}") + + # Wait for router to be ready + router_url = f"http://127.0.0.1:{router_port}" + print(f"\nWaiting for router to start at {router_url}...") + + try: + wait_for_router_ready(router_url, timeout=timeout, api_key=None) + logger.info(f"✓ Router ready at {router_url}") + except TimeoutError: + logger.error(f"✗ Router failed to start") + # Cleanup: kill router + try: + router_proc.kill() + except: + pass + raise + + logger.info(f"\n{'='*70}") + logger.info(f"✓ {backend.upper()} router ready!") + logger.info(f" Router: {router_url}") + logger.info(f"{'='*70}\n") + + return { + "router": router_proc, + "base_url": router_url, + } + + +def popen_launch_workers_and_router( + model: str, + base_url: str, + timeout: int = 300, + num_workers: int = 2, + policy: str = "round_robin", + api_key: Optional[str] = None, + worker_args: Optional[list] = None, + router_args: Optional[list] = None, + tp_size: int = 1, + env: Optional[dict] = None, + stdout=None, + stderr=None, +) -> dict: + """ + Launch SGLang workers and gRPC router separately. + + This approach: + 1. Starts N SGLang workers with --grpc-mode flag + 2. Waits for workers to initialize (process startup) + 3. Starts a gRPC router pointing to those workers + 4. Waits for router health check to pass (router validates worker connectivity) + + This matches production deployment patterns better than the integrated approach. + + Args: + model: Model path (e.g., /home/ubuntu/models/llama-3.1-8b-instruct) + base_url: Base URL for router (e.g., "http://127.0.0.1:8080") + timeout: Timeout for server startup (default: 300s) + num_workers: Number of workers to launch + policy: Routing policy (round_robin, random, power_of_two, cache_aware) + api_key: Optional API key for router + worker_args: Additional arguments for workers (e.g., ["--context-len", "8192"]) + router_args: Additional arguments for router (e.g., ["--max-total-token", "1536"]) + tp_size: Tensor parallelism size for workers (default: 1) + env: Optional environment variables for workers (e.g., {"SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION": "256"}) + stdout: Optional file handle for worker stdout (default: subprocess.PIPE) + stderr: Optional file handle for worker stderr (default: subprocess.PIPE) + + Returns: + dict with: + - workers: list of worker process objects + - worker_urls: list of gRPC worker URLs + - router: router process object + - base_url: router URL (HTTP endpoint) + + Example: + >>> cluster = popen_launch_workers_and_router(model, base_url, num_workers=2) + >>> # Use cluster['base_url'] for HTTP requests + >>> # Cleanup: + >>> for worker in cluster['workers']: + >>> kill_process_tree(worker.pid) + >>> kill_process_tree(cluster['router'].pid) + """ + show_output = os.environ.get("SHOW_ROUTER_LOGS", "0") == "1" + + # Parse router port from base_url + if ":" in base_url.split("//")[-1]: + router_port = int(base_url.split(":")[-1]) + else: + router_port = find_free_port() + + logger.info(f"\n{'='*70}") + logger.info(f"Launching gRPC cluster (separate workers + router)") + logger.info(f"{'='*70}") + logger.info(f" Model: {model}") + logger.info(f" Router port: {router_port}") + logger.info(f" Workers: {num_workers}") + logger.info(f" TP size: {tp_size}") + logger.info(f" Policy: {policy}") + + # Step 1: Launch workers with gRPC enabled + workers = [] + worker_urls = [] + + for i in range(num_workers): + worker_port = find_free_port() + worker_url = f"grpc://127.0.0.1:{worker_port}" + worker_urls.append(worker_url) + + logger.info(f"\n[Worker {i+1}/{num_workers}]") + logger.info(f" Port: {worker_port}") + logger.info(f" URL: {worker_url}") + + # Build worker command + worker_cmd = [ + "python3", + "-m", + "sglang.launch_server", + "--model-path", + model, + "--host", + "127.0.0.1", + "--port", + str(worker_port), + "--grpc-mode", # Enable gRPC for this worker + "--mem-fraction-static", + "0.8", + ] + + # Add TP size + if tp_size > 1: + worker_cmd.extend(["--tp-size", str(tp_size)]) + + # Add worker-specific args + if worker_args: + worker_cmd.extend(worker_args) + + # Launch worker with optional environment variables + if show_output: + worker_proc = subprocess.Popen( + worker_cmd, + env=env, + stdout=stdout, + stderr=stderr, + ) + else: + worker_proc = subprocess.Popen( + worker_cmd, + stdout=stdout if stdout is not None else subprocess.PIPE, + stderr=stderr if stderr is not None else subprocess.PIPE, + env=env, + ) + + workers.append(worker_proc) + logger.info(f" PID: {worker_proc.pid}") + + # Give workers a moment to start binding to ports + # The router will check worker health when it starts + logger.info(f"\nWaiting for {num_workers} workers to initialize (20s)...") + time.sleep(20) + + # Quick check: make sure worker processes are still alive + for i, worker in enumerate(workers): + if worker.poll() is not None: + logger.error( + f" ✗ Worker {i+1} died during startup (exit code: {worker.poll()})" + ) + # Cleanup: kill all workers + for w in workers: + try: + w.kill() + except: + pass + raise RuntimeError(f"Worker {i+1} failed to start") + + logger.info( + f"✓ All {num_workers} workers started (router will verify connectivity)" + ) + + # Step 2: Launch router pointing to workers + logger.info(f"\n[Router]") + logger.info(f" Port: {router_port}") + logger.info(f" Worker URLs: {', '.join(worker_urls)}") + + # Build router command + router_cmd = [ + "python3", + "-m", + "sglang_router.launch_router", + "--host", + "127.0.0.1", + "--port", + str(router_port), + "--prometheus-port", + "9321", + "--policy", + policy, + "--model-path", + model, + "--log-level", + "warn", + ] + + # Add worker URLs + router_cmd.append("--worker-urls") + router_cmd.extend(worker_urls) + + # Add API key + if api_key: + router_cmd.extend(["--api-key", api_key]) + + # Add router-specific args + if router_args: + router_cmd.extend(router_args) + + if show_output: + logger.info(f" Command: {' '.join(router_cmd)}") + + # Launch router + if show_output: + router_proc = subprocess.Popen(router_cmd) + else: + router_proc = subprocess.Popen( + router_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + logger.info(f" PID: {router_proc.pid}") + + # Wait for router to be ready + router_url = f"http://127.0.0.1:{router_port}" + logger.info(f"\nWaiting for router to start at {router_url}...") + + try: + wait_for_workers_ready( + router_url, expected_workers=num_workers, timeout=180, api_key=api_key + ) + logger.info(f"✓ Router ready at {router_url}") + except TimeoutError: + logger.error(f"✗ Router failed to start") + # Cleanup: kill router and all workers + try: + router_proc.kill() + except: + pass + for worker in workers: + try: + worker.kill() + except: + pass + raise + + logger.info(f"\n{'='*70}") + logger.info(f"✓ gRPC cluster ready!") + logger.info(f" Router: {router_url}") + logger.info(f" Workers: {len(workers)}") + logger.info(f"{'='*70}\n") + + return { + "workers": workers, + "worker_urls": worker_urls, + "router": router_proc, + "base_url": router_url, + } diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/util.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/util.py new file mode 100644 index 00000000..222b648b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/e2e_response_api/util.py @@ -0,0 +1,81 @@ +""" +Utility functions for Response API e2e tests. +""" + +import logging +import os +import signal +import threading +import unittest + +import psutil + +logger = logging.getLogger(__name__) + + +def kill_process_tree(parent_pid, include_parent: bool = True, skip_pid: int = None): + """ + Kill the process and all its child processes. + + Args: + parent_pid: PID of the parent process + include_parent: Whether to kill the parent process itself + skip_pid: Optional PID to skip during cleanup + """ + # Remove sigchld handler to avoid spammy logs + if threading.current_thread() is threading.main_thread(): + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + + if parent_pid is None: + parent_pid = os.getpid() + include_parent = False + + try: + itself = psutil.Process(parent_pid) + except psutil.NoSuchProcess: + return + + children = itself.children(recursive=True) + for child in children: + if child.pid == skip_pid: + continue + try: + child.kill() + except psutil.NoSuchProcess: + pass + + if include_parent: + try: + itself.kill() + except psutil.NoSuchProcess: + pass + + +class CustomTestCase(unittest.TestCase): + """ + Custom test case base class with retry support. + + This provides automatic test retry functionality based on environment variables. + """ + + def _callTestMethod(self, method): + """Override to add retry logic.""" + max_retry = int(os.environ.get("SGLANG_TEST_MAX_RETRY", "0")) + + if max_retry == 0: + # No retry, just run once + return super(CustomTestCase, self)._callTestMethod(method) + + # Retry logic + for attempt in range(max_retry + 1): + try: + return super(CustomTestCase, self)._callTestMethod(method) + except Exception as e: + if attempt < max_retry: + logger.info( + f"Test failed on attempt {attempt + 1}/{max_retry + 1}, retrying..." + ) + continue + else: + # Last attempt, re-raise the exception + raise diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/fixtures/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/fixtures/__init__.py new file mode 100644 index 00000000..4ac754df --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/fixtures/__init__.py @@ -0,0 +1 @@ +"""Shared fixtures for router integration tests.""" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/fixtures/generate_test_certs.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/fixtures/generate_test_certs.py new file mode 100644 index 00000000..e24d0744 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/fixtures/generate_test_certs.py @@ -0,0 +1,236 @@ +""" +Generate self-signed certificates for mTLS integration testing. +Creates a Certificate Authority (CA), server certificates, and client certificates. +""" + +import datetime +import ipaddress +from pathlib import Path + +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.x509.oid import NameOID + + +def generate_private_key(): + """Generate an RSA private key.""" + return rsa.generate_private_key( + public_exponent=65537, + key_size=2048, + ) + + +def generate_ca_certificate(): + """Generate a self-signed CA certificate.""" + private_key = generate_private_key() + + subject = issuer = x509.Name( + [ + x509.NameAttribute(NameOID.COUNTRY_NAME, "US"), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "Test"), + x509.NameAttribute(NameOID.LOCALITY_NAME, "Test"), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, "SGLang Test"), + x509.NameAttribute(NameOID.ORGANIZATIONAL_UNIT_NAME, "Test"), + x509.NameAttribute(NameOID.COMMON_NAME, "Test CA"), + ] + ) + + cert = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(issuer) + .public_key(private_key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.datetime.utcnow()) + .not_valid_after(datetime.datetime.utcnow() + datetime.timedelta(days=3650)) + .add_extension( + x509.BasicConstraints(ca=True, path_length=None), + critical=True, + ) + .add_extension( + x509.KeyUsage( + digital_signature=True, + key_cert_sign=True, + crl_sign=True, + key_encipherment=False, + content_commitment=False, + data_encipherment=False, + key_agreement=False, + encipher_only=False, + decipher_only=False, + ), + critical=True, + ) + .sign(private_key, hashes.SHA256()) + ) + + return private_key, cert + + +def generate_server_certificate(ca_key, ca_cert): + """Generate a server certificate signed by the CA.""" + private_key = generate_private_key() + + subject = x509.Name( + [ + x509.NameAttribute(NameOID.COUNTRY_NAME, "US"), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "Test"), + x509.NameAttribute(NameOID.LOCALITY_NAME, "Test"), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, "SGLang Test"), + x509.NameAttribute(NameOID.ORGANIZATIONAL_UNIT_NAME, "Test"), + x509.NameAttribute(NameOID.COMMON_NAME, "localhost"), + ] + ) + + cert = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(ca_cert.subject) + .public_key(private_key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.datetime.utcnow()) + .not_valid_after(datetime.datetime.utcnow() + datetime.timedelta(days=365)) + .add_extension( + x509.SubjectAlternativeName( + [ + x509.DNSName("localhost"), + x509.IPAddress(ipaddress.IPv4Address("127.0.0.1")), + ] + ), + critical=False, + ) + .add_extension( + x509.KeyUsage( + digital_signature=True, + key_encipherment=True, + key_cert_sign=False, + crl_sign=False, + content_commitment=False, + data_encipherment=False, + key_agreement=False, + encipher_only=False, + decipher_only=False, + ), + critical=True, + ) + .add_extension( + x509.ExtendedKeyUsage( + [ + x509.oid.ExtendedKeyUsageOID.SERVER_AUTH, + ] + ), + critical=False, + ) + .sign(ca_key, hashes.SHA256()) + ) + + return private_key, cert + + +def generate_client_certificate(ca_key, ca_cert): + """Generate a client certificate signed by the CA.""" + private_key = generate_private_key() + + subject = x509.Name( + [ + x509.NameAttribute(NameOID.COUNTRY_NAME, "US"), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "Test"), + x509.NameAttribute(NameOID.LOCALITY_NAME, "Test"), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, "SGLang Test"), + x509.NameAttribute(NameOID.ORGANIZATIONAL_UNIT_NAME, "Test"), + x509.NameAttribute(NameOID.COMMON_NAME, "test-client"), + ] + ) + + cert = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(ca_cert.subject) + .public_key(private_key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.datetime.utcnow()) + .not_valid_after(datetime.datetime.utcnow() + datetime.timedelta(days=365)) + .add_extension( + x509.KeyUsage( + digital_signature=True, + key_encipherment=True, + key_cert_sign=False, + crl_sign=False, + content_commitment=False, + data_encipherment=False, + key_agreement=False, + encipher_only=False, + decipher_only=False, + ), + critical=True, + ) + .add_extension( + x509.ExtendedKeyUsage( + [ + x509.oid.ExtendedKeyUsageOID.CLIENT_AUTH, + ] + ), + critical=False, + ) + .sign(ca_key, hashes.SHA256()) + ) + + return private_key, cert + + +def save_key(key, path: Path): + """Save private key to PEM file.""" + with open(path, "wb") as f: + f.write( + key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + ) + + +def save_cert(cert, path: Path): + """Save certificate to PEM file.""" + with open(path, "wb") as f: + f.write(cert.public_bytes(serialization.Encoding.PEM)) + + +def generate_all_certificates(output_dir: Path): + """Generate all certificates and keys for mTLS testing.""" + output_dir.mkdir(parents=True, exist_ok=True) + + print("==> Generating CA certificate...") + ca_key, ca_cert = generate_ca_certificate() + save_key(ca_key, output_dir / "ca-key.pem") + save_cert(ca_cert, output_dir / "ca-cert.pem") + + print("==> Generating server certificate...") + server_key, server_cert = generate_server_certificate(ca_key, ca_cert) + save_key(server_key, output_dir / "server-key.pem") + save_cert(server_cert, output_dir / "server-cert.pem") + + print("==> Generating client certificate...") + client_key, client_cert = generate_client_certificate(ca_key, ca_cert) + save_key(client_key, output_dir / "client-key.pem") + save_cert(client_cert, output_dir / "client-cert.pem") + + print(f"==> Certificates generated successfully in {output_dir}") + print() + print("Files created:") + print(" - ca-cert.pem : CA certificate (for verifying server/client certs)") + print(" - ca-key.pem : CA private key") + print(" - server-cert.pem : Server certificate") + print(" - server-key.pem : Server private key") + print(" - client-cert.pem : Client certificate") + print(" - client-key.pem : Client private key") + print() + print("Test server can use: server-cert.pem + server-key.pem") + print("Test router can use: client-cert.pem + client-key.pem + ca-cert.pem") + + +if __name__ == "__main__": + script_dir = Path(__file__).parent + certs_dir = script_dir / "test_certs" + generate_all_certificates(certs_dir) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/fixtures/mock_worker.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/fixtures/mock_worker.py new file mode 100644 index 00000000..2f302251 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/fixtures/mock_worker.py @@ -0,0 +1,285 @@ +""" +Lightweight mock worker HTTP server for router integration tests. + +Implements minimal endpoints used by the router: +- GET /health, /health_generate +- POST /generate, /v1/completions, /v1/chat/completions +- POST /flush_cache +- GET /get_server_info, /get_model_info, /v1/models + +Behavior knobs are controlled via CLI flags to simulate failures, latency, and load. +""" + +import argparse +import asyncio +import json +import os +import random +import signal +import sys +import time +from contextlib import asynccontextmanager +from typing import Optional + +import uvicorn +from fastapi import FastAPI, HTTPException, Request +from fastapi.responses import JSONResponse, PlainTextResponse, StreamingResponse + +# Global state (per-process) +_inflight = 0 +_failures_seen = 0 + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser() + p.add_argument("--host", default="127.0.0.1") + p.add_argument("--port", type=int, required=True) + p.add_argument("--worker-id", default=None) + p.add_argument("--latency-ms", type=int, default=0) + p.add_argument("--timeout", action="store_true") + p.add_argument("--status-code", type=int, default=200) + p.add_argument("--fail-first-n", type=int, default=0) + p.add_argument("--random-fail-rate", type=float, default=0.0) + p.add_argument("--require-api-key", action="store_true") + p.add_argument("--api-key", default=None) + p.add_argument("--max-payload-bytes", type=int, default=10 * 1024 * 1024) + p.add_argument("--stream", action="store_true") + p.add_argument("--dp-size", type=int, default=1) + p.add_argument("--crash-on-request", action="store_true") + p.add_argument("--health-fail-after-ms", type=int, default=0) + # TLS/mTLS configuration + p.add_argument( + "--ssl-certfile", type=str, default=None, help="Path to SSL certificate file" + ) + p.add_argument("--ssl-keyfile", type=str, default=None, help="Path to SSL key file") + p.add_argument( + "--ssl-ca-certs", + type=str, + default=None, + help="Path to CA certificates for client verification", + ) + return p.parse_args() + + +def _extract_worker_id(args: argparse.Namespace) -> str: + if args.worker_id: + return str(args.worker_id) + # default to port (unique enough for tests) + return f"worker-{args.port}" + + +def create_app(args: argparse.Namespace) -> FastAPI: + app = FastAPI() + worker_id = _extract_worker_id(args) + start_ts = time.time() + crashed = {"done": False} + + async def maybe_delay(): + if args.latency_ms > 0: + await asyncio.sleep(args.latency_ms / 1000.0) + + def should_fail() -> Optional[int]: + global _failures_seen + # Fail first N requests (500) + if args.fail_first_n > 0 and _failures_seen < args.fail_first_n: + _failures_seen += 1 + return 500 + # Random failure probability (500) + if args.random_fail_rate > 0.0 and random.random() < args.random_fail_rate: + return 500 + # Forced status code override (non-200) for all responses + if args.status_code != 200: + return int(args.status_code) + return None + + def check_api_key(request: Request): + if not args.require_api_key: + return + auth = request.headers.get("Authorization") + if not auth or not auth.startswith("Bearer "): + raise HTTPException(status_code=401, detail="Unauthorized") + key = auth.split(" ", 1)[1] + if args.api_key and key != args.api_key: + raise HTTPException(status_code=401, detail="Unauthorized") + + @asynccontextmanager + async def track_inflight(): + global _inflight + _inflight += 1 + try: + yield + finally: + _inflight -= 1 + + @app.get("/health") + async def health(): + if ( + args.health_fail_after_ms + and (time.time() - start_ts) * 1000.0 >= args.health_fail_after_ms + ): + return PlainTextResponse("bad", status_code=500) + return PlainTextResponse("ok", status_code=200) + + @app.get("/health_generate") + async def health_generate(): + return PlainTextResponse("ok", status_code=200) + + @app.post("/flush_cache") + async def flush_cache(): + return PlainTextResponse("ok", status_code=200) + + @app.get("/get_model_info") + async def get_model_info(): + return JSONResponse({"model": "mock", "vocab_size": 32000}) + + @app.get("/v1/models") + async def list_models(): + return JSONResponse({"data": [{"id": "mock", "object": "model"}]}) + + @app.get("/get_server_info") + async def get_server_info(request: Request): + # Enforce API key on server info when required (used by dp_aware probing) + check_api_key(request) + return JSONResponse( + { + "worker_id": worker_id, + "load_in_flight": _inflight, + "cache": {"size": 0, "hit_rate": 0.0}, + "dp_size": int(args.dp_size), + } + ) + + @app.get("/get_load") + async def get_load(request: Request): + check_api_key(request) + # Return format matching real workers: array of load info per DP rank + return JSONResponse( + [ + { + "dp_rank": 0, + "num_reqs": _inflight, + "num_waiting_reqs": 0, + "num_tokens": _inflight, + } + ] + ) + + def make_json_response(obj: dict, status_code: int = 200) -> JSONResponse: + resp = JSONResponse(obj, status_code=status_code) + resp.headers["X-Worker-Id"] = worker_id + return resp + + async def handle_text_request(request: Request): + # Authorization + check_api_key(request) + + # Payload limit + body = await request.body() + if len(body) > args.max_payload_bytes: + return make_json_response({"error": "payload too large"}, status_code=413) + + # Simulate crash on first request + if args.crash_on_request and not crashed["done"]: + crashed["done"] = True + os._exit(1) + + # Optional timeout (simulate hang) + if args.timeout: + await asyncio.sleep(3600) + + # Optional latency + await maybe_delay() + + # Optional failures + fail_code = should_fail() + if fail_code is not None and fail_code != 200: + return make_json_response( + {"error": f"mock failure {fail_code}"}, status_code=fail_code + ) + + # Build response echoing minimal shape + try: + data = await request.json() + except (json.JSONDecodeError, ValueError): + data = {} + + now = time.time() + ret = { + "id": f"cmpl-{int(now*1000)}", + "object": "text_completion", + "created": int(now), + "model": "mock", + "choices": [ + { + "text": "ok", + "index": 0, + "finish_reason": "stop", + } + ], + "worker_id": worker_id, + "echo": data, + } + return make_json_response(ret, status_code=200) + + async def handle_stream_request(request: Request): + check_api_key(request) + + async def gen(): + # minimal 2-chunk stream then [DONE] + for i in range(2): + await asyncio.sleep(0.01) + chunk = { + "choices": [{"delta": {"content": "x"}}], + "worker_id": worker_id, + } + yield f"data: {json.dumps(chunk)}\n\n" + yield "data: [DONE]\n\n" + + headers = {"X-Worker-Id": worker_id} + return StreamingResponse(gen(), media_type="text/event-stream", headers=headers) + + @app.post("/generate") + async def generate(request: Request): + async with track_inflight(): + if args.stream: + return await handle_stream_request(request) + return await handle_text_request(request) + + @app.post("/v1/completions") + async def completions(request: Request): + async with track_inflight(): + if args.stream: + return await handle_stream_request(request) + return await handle_text_request(request) + + @app.post("/v1/chat/completions") + async def chat_completions(request: Request): + async with track_inflight(): + if args.stream: + return await handle_stream_request(request) + return await handle_text_request(request) + + return app + + +def main() -> None: + args = _parse_args() + app = create_app(args) + # Handle SIGTERM gracefully for fast test teardown + signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)) + + # Configure SSL if certificates are provided + ssl_config = {} + if args.ssl_certfile and args.ssl_keyfile: + ssl_config["ssl_certfile"] = args.ssl_certfile + ssl_config["ssl_keyfile"] = args.ssl_keyfile + # If CA certs provided, require client certificates (mTLS) + if args.ssl_ca_certs: + ssl_config["ssl_ca_certs"] = args.ssl_ca_certs + ssl_config["ssl_cert_reqs"] = 2 # ssl.CERT_REQUIRED + + uvicorn.run(app, host=args.host, port=args.port, log_level="warning", **ssl_config) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/fixtures/ports.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/fixtures/ports.py new file mode 100644 index 00000000..d616cffa --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/fixtures/ports.py @@ -0,0 +1,8 @@ +import socket + + +def find_free_port() -> int: + """Return an available TCP port on localhost.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/fixtures/router_manager.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/fixtures/router_manager.py new file mode 100644 index 00000000..576e00b2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/fixtures/router_manager.py @@ -0,0 +1,238 @@ +import subprocess +import time +from dataclasses import dataclass +from typing import Dict, List, Optional + +import requests + +from .ports import find_free_port + + +@dataclass +class ProcHandle: + process: subprocess.Popen + url: str + + +class RouterManager: + """Helper to spawn a router process and interact with admin endpoints.""" + + def __init__(self): + self._children: List[subprocess.Popen] = [] + + def start_router( + self, + worker_urls: Optional[List[str]] = None, + policy: str = "round_robin", + port: Optional[int] = None, + extra: Optional[Dict] = None, + # PD options + pd_disaggregation: bool = False, + prefill_urls: Optional[List[tuple]] = None, + decode_urls: Optional[List[str]] = None, + prefill_policy: Optional[str] = None, + decode_policy: Optional[str] = None, + ) -> ProcHandle: + worker_urls = worker_urls or [] + port = port or find_free_port() + cmd = [ + "python3", + "-m", + "sglang_router.launch_router", + "--host", + "127.0.0.1", + "--port", + str(port), + "--policy", + policy, + ] + # Avoid Prometheus port collisions by assigning a free port per router + prom_port = find_free_port() + cmd.extend( + ["--prometheus-port", str(prom_port), "--prometheus-host", "127.0.0.1"] + ) + if worker_urls: + cmd.extend(["--worker-urls", *worker_urls]) + + # PD routing configuration + if pd_disaggregation: + cmd.append("--pd-disaggregation") + if prefill_urls: + for url, bport in prefill_urls: + if bport is None: + cmd.extend(["--prefill", url, "none"]) + else: + cmd.extend(["--prefill", url, str(bport)]) + if decode_urls: + for url in decode_urls: + cmd.extend(["--decode", url]) + if prefill_policy: + cmd.extend(["--prefill-policy", prefill_policy]) + if decode_policy: + cmd.extend(["--decode-policy", decode_policy]) + + # Map supported extras to CLI flags (subset for integration) + if extra: + flag_map = { + "max_payload_size": "--max-payload-size", + "dp_aware": "--dp-aware", + "api_key": "--api-key", + # Health/monitoring + "worker_startup_check_interval": "--worker-startup-check-interval", + # Cache-aware tuning + "cache_threshold": "--cache-threshold", + "balance_abs_threshold": "--balance-abs-threshold", + "balance_rel_threshold": "--balance-rel-threshold", + # Retry + "retry_max_retries": "--retry-max-retries", + "retry_initial_backoff_ms": "--retry-initial-backoff-ms", + "retry_max_backoff_ms": "--retry-max-backoff-ms", + "retry_backoff_multiplier": "--retry-backoff-multiplier", + "retry_jitter_factor": "--retry-jitter-factor", + "disable_retries": "--disable-retries", + # Circuit breaker + "cb_failure_threshold": "--cb-failure-threshold", + "cb_success_threshold": "--cb-success-threshold", + "cb_timeout_duration_secs": "--cb-timeout-duration-secs", + "cb_window_duration_secs": "--cb-window-duration-secs", + "disable_circuit_breaker": "--disable-circuit-breaker", + # Rate limiting + "max_concurrent_requests": "--max-concurrent-requests", + "queue_size": "--queue-size", + "queue_timeout_secs": "--queue-timeout-secs", + "rate_limit_tokens_per_second": "--rate-limit-tokens-per-second", + # mTLS configuration + "client_cert_path": "--client-cert-path", + "client_key_path": "--client-key-path", + "ca_cert_paths": "--ca-cert-paths", + } + for k, v in extra.items(): + if v is None: + continue + flag = flag_map.get(k) + if not flag: + continue + if isinstance(v, bool): + if v: + cmd.append(flag) + elif isinstance(v, list): + # Handle list arguments (e.g., ca_cert_paths) + if v: # Only add if list is not empty + cmd.append(flag) + cmd.extend([str(item) for item in v]) + else: + cmd.extend([flag, str(v)]) + + proc = subprocess.Popen(cmd) + self._children.append(proc) + url = f"http://127.0.0.1:{port}" + self._wait_health(url) + return ProcHandle(process=proc, url=url) + + def _wait_health(self, base_url: str, timeout: float = 30.0): + start = time.time() + with requests.Session() as s: + while time.time() - start < timeout: + try: + r = s.get(f"{base_url}/health", timeout=2) + if r.status_code == 200: + return + except requests.RequestException: + pass + time.sleep(0.2) + raise TimeoutError(f"Router at {base_url} did not become healthy") + + def add_worker(self, base_url: str, worker_url: str, timeout: float = 30.0) -> None: + r = requests.post(f"{base_url}/workers", json={"url": worker_url}) + assert ( + r.status_code == 202 + ), f"add_worker failed: {r.status_code} {r.text}" # ACCEPTED status + + # Poll until worker is actually added and healthy + from urllib.parse import quote + + encoded_url = quote(worker_url, safe="") + start = time.time() + with requests.Session() as s: + while time.time() - start < timeout: + try: + r = s.get(f"{base_url}/workers/{encoded_url}", timeout=2) + if r.status_code == 200: + data = r.json() + # Check if registration job failed + job_status = data.get("job_status") + if job_status and job_status.get("state") == "failed": + raise RuntimeError( + f"Worker registration failed: {job_status.get('message', 'Unknown error')}" + ) + # Check if worker is healthy and registered (not just in job queue) + if data.get("is_healthy", False): + return + # Worker not ready yet, continue polling + except requests.RequestException: + pass + time.sleep(0.1) + raise TimeoutError( + f"Worker {worker_url} was not added and healthy after {timeout}s" + ) + + def remove_worker( + self, base_url: str, worker_url: str, timeout: float = 30.0 + ) -> None: + # URL encode the worker_url for path parameter + from urllib.parse import quote + + encoded_url = quote(worker_url, safe="") + r = requests.delete(f"{base_url}/workers/{encoded_url}") + assert ( + r.status_code == 202 + ), f"remove_worker failed: {r.status_code} {r.text}" # ACCEPTED status + + # Poll until worker is actually removed (GET returns 404) or timeout + start = time.time() + last_status = None + with requests.Session() as s: + while time.time() - start < timeout: + try: + r = s.get(f"{base_url}/workers/{encoded_url}", timeout=2) + if r.status_code == 404: + # Worker successfully removed + return + elif r.status_code == 200: + # Check if removal job failed + data = r.json() + job_status = data.get("job_status") + if job_status: + last_status = job_status + if job_status.get("state") == "failed": + raise RuntimeError( + f"Worker removal failed: {job_status.get('message', 'Unknown error')}" + ) + # Worker still being processed, continue polling + except requests.RequestException: + pass + time.sleep(0.1) + + # Provide detailed timeout error with last known status + error_msg = f"Worker {worker_url} was not removed after {timeout}s" + if last_status: + error_msg += f". Last job status: {last_status}" + raise TimeoutError(error_msg) + + def list_workers(self, base_url: str) -> list[str]: + r = requests.get(f"{base_url}/workers") + assert r.status_code == 200, f"list_workers failed: {r.status_code} {r.text}" + data = r.json() + # Extract URLs from WorkerInfo objects + workers = data.get("workers", []) + return [w["url"] for w in workers] + + def stop_all(self): + for p in self._children: + if p.poll() is None: + p.terminate() + try: + p.wait(timeout=5) + except subprocess.TimeoutExpired: + p.kill() + self._children.clear() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/__init__.py new file mode 100644 index 00000000..1e342eca --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/__init__.py @@ -0,0 +1 @@ +"""Integration test package for the router.""" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/conftest.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/conftest.py new file mode 100644 index 00000000..25bd7c2b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/conftest.py @@ -0,0 +1,128 @@ +import shutil +import subprocess +import time +from pathlib import Path +from typing import Iterable, List, Optional, Tuple + +import pytest +import requests + +from ..fixtures.generate_test_certs import generate_all_certificates +from ..fixtures.ports import find_free_port +from ..fixtures.router_manager import RouterManager + + +def pytest_configure(config): + config.addinivalue_line("markers", "integration: mark as router integration test") + + +@pytest.fixture +def router_manager() -> Iterable[RouterManager]: + mgr = RouterManager() + try: + yield mgr + finally: + mgr.stop_all() + + +def _spawn_mock_worker(args: List[str]) -> Tuple[subprocess.Popen, str, str]: + repo_root = Path(__file__).resolve().parents[2] + script = repo_root / "py_test" / "fixtures" / "mock_worker.py" + port = find_free_port() + worker_id = f"worker-{port}" + base_cmd = [ + "python3", + str(script), + "--port", + str(port), + "--worker-id", + worker_id, + ] + cmd = base_cmd + args + proc = subprocess.Popen(cmd) + url = f"http://127.0.0.1:{port}" + _wait_health(url) + return proc, url, worker_id + + +def _wait_health(url: str, timeout: float = 10.0): + start = time.time() + with requests.Session() as s: + while time.time() - start < timeout: + try: + r = s.get(f"{url}/health", timeout=1) + if r.status_code == 200: + return + except requests.RequestException: + pass + time.sleep(0.1) + raise TimeoutError(f"Mock worker at {url} did not become healthy") + + +@pytest.fixture +def mock_worker(): + """Start a single healthy mock worker; yields (process, url, worker_id).""" + proc, url, worker_id = _spawn_mock_worker([]) + try: + yield proc, url, worker_id + finally: + if proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=3) + except subprocess.TimeoutExpired: + proc.kill() + + +@pytest.fixture +def mock_workers(): + """Factory to start N workers with custom args. + + Usage: + procs, urls, ids = mock_workers(n=3, args=["--latency-ms", "5"]) # same args for all + ... + """ + + procs: List[subprocess.Popen] = [] + + def _start(n: int, args: Optional[List[str]] = None): + args = args or [] + new_procs: List[subprocess.Popen] = [] + urls: List[str] = [] + ids: List[str] = [] + for _ in range(n): + p, url, wid = _spawn_mock_worker(args) + procs.append(p) + new_procs.append(p) + urls.append(url) + ids.append(wid) + return new_procs, urls, ids + + try: + yield _start + finally: + for p in procs: + if p.poll() is None: + p.terminate() + try: + p.wait(timeout=3) + except subprocess.TimeoutExpired: + p.kill() + + +@pytest.fixture(scope="session") +def test_certificates(): + """Generate test certificates for mTLS tests, clean up after session.""" + # Get the test_certs directory path + fixtures_dir = Path(__file__).parent.parent / "fixtures" + certs_dir = fixtures_dir / "test_certs" + + # Generate certificates + generate_all_certificates(certs_dir) + + # Yield the path to the certificates directory + yield certs_dir + + # Cleanup: remove the generated certificates + if certs_dir.exists(): + shutil.rmtree(certs_dir) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/load_balancing/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/load_balancing/__init__.py new file mode 100644 index 00000000..77b8c246 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/load_balancing/__init__.py @@ -0,0 +1 @@ +"""Load balancing integration tests.""" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/load_balancing/test_cache_aware.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/load_balancing/test_cache_aware.py new file mode 100644 index 00000000..acbbd368 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/load_balancing/test_cache_aware.py @@ -0,0 +1,73 @@ +import collections +import concurrent.futures +import uuid + +import pytest +import requests + + +@pytest.mark.integration +def test_cache_aware_affinity(mock_workers, router_manager): + # Two workers; same prompt should stick to one due to cache tree + _, urls, ids = mock_workers(n=2) + rh = router_manager.start_router(worker_urls=urls, policy="cache_aware") + + counts = collections.Counter() + with requests.Session() as s: + for i in range(12): + r = s.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "repeated prompt for cache", + "max_tokens": 1, + "stream": False, + }, + ) + assert r.status_code == 200 + wid = r.headers.get("X-Worker-Id") or r.json().get("worker_id") + counts[wid] += 1 + + # Expect strong skew toward one worker (tree match); majority > 80% + top = max(counts.values()) + assert top >= 10, counts + + +@pytest.mark.integration +def test_cache_aware_diverse_prompts_balances(mock_workers, router_manager): + # Add latency so concurrent requests overlap and influence load-based selection + _, urls, ids = mock_workers(n=3, args=["--latency-ms", "30"]) + rh = router_manager.start_router( + worker_urls=urls, + policy="cache_aware", + extra={ + "cache_threshold": 0.99, + "balance_abs_threshold": 0, + "balance_rel_threshold": 1.0, + }, + ) + + counts = collections.Counter() + + def call(i): + # Use diverse, unrelated prompts to avoid prefix matches entirely + prompt = str(uuid.uuid4()) + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": prompt, + "max_tokens": 1, + "stream": False, + }, + timeout=5, + ) + assert r.status_code == 200 + return r.headers.get("X-Worker-Id") or r.json().get("worker_id") + + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as ex: + for wid in ex.map(call, range(40)): + counts[wid] += 1 + + # Expect participation of at least two workers + assert sum(1 for v in counts.values() if v > 0) >= 2, counts diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/load_balancing/test_power_of_two.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/load_balancing/test_power_of_two.py new file mode 100644 index 00000000..0a8d9eab --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/load_balancing/test_power_of_two.py @@ -0,0 +1,99 @@ +import collections +import concurrent.futures +import time + +import pytest +import requests + + +@pytest.mark.integration +def test_power_of_two_prefers_less_loaded(mock_workers, router_manager): + # Start two workers: one slow (higher inflight), one fast + # Router monitors /get_load and Power-of-Two uses cached loads to choose + # Start one slow and one fast worker using the fixture factory + procs_slow, urls_slow, ids_slow = mock_workers(n=1, args=["--latency-ms", "200"]) + procs_fast, urls_fast, ids_fast = mock_workers(n=1, args=["--latency-ms", "0"]) + procs = procs_slow + procs_fast + urls = urls_slow + urls_fast + ids = ids_slow + ids_fast + slow_id = ids_slow[0] + slow_url = urls_slow[0] + + rh = router_manager.start_router( + worker_urls=urls, + policy="power_of_two", + extra={"worker_startup_check_interval": 1}, + ) + + # Prime: fire a burst to create measurable load on slow worker, then wait for monitor tick + + def _prime_call(i): + try: + requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"warm-{i}", + "max_tokens": 1, + "stream": False, + }, + timeout=5, + ) + except Exception: + pass + + with concurrent.futures.ThreadPoolExecutor(max_workers=32) as ex: + list(ex.map(_prime_call, range(128))) + time.sleep(2) + + # Apply direct background load on the slow worker to amplify load diff + def _direct_load(i): + try: + requests.post( + f"{slow_url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"bg-{i}", + "max_tokens": 1, + "stream": False, + }, + timeout=5, + ) + except Exception: + pass + + # Start background load in a non-blocking way to keep slow worker busy + background_executor = concurrent.futures.ThreadPoolExecutor(max_workers=8) + background_futures = [] + for i in range(32): + future = background_executor.submit(_direct_load, i) + background_futures.append(future) + + # Wait longer for the load monitor to update (at least 2 monitor intervals) + time.sleep(3) + + def call(i): + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"p{i}", + "max_tokens": 1, + "stream": False, + }, + timeout=5, + ) + assert r.status_code == 200 + return r.headers.get("X-Worker-Id") or r.json().get("worker_id") + + counts = collections.Counter() + with concurrent.futures.ThreadPoolExecutor(max_workers=32) as ex: + for wid in ex.map(call, range(200)): + counts[wid] += 1 + + # Clean up background executor + background_executor.shutdown(wait=False) + + # Expect the slow worker (higher latency/inflight) to receive fewer requests + fast_worker_id = [i for i in ids if i != slow_id][0] + assert counts[slow_id] < counts[fast_worker_id], counts diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/load_balancing/test_random.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/load_balancing/test_random.py new file mode 100644 index 00000000..4662dbce --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/load_balancing/test_random.py @@ -0,0 +1,32 @@ +import collections + +import pytest +import requests + + +@pytest.mark.integration +def test_random_distribution(mock_workers, router_manager): + procs, urls, ids = mock_workers(n=4) + rh = router_manager.start_router(worker_urls=urls, policy="random") + + counts = collections.Counter() + N = 200 + with requests.Session() as s: + for i in range(N): + r = s.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"p{i}", + "max_tokens": 1, + "stream": False, + }, + ) + assert r.status_code == 200 + wid = r.headers.get("X-Worker-Id") or r.json().get("worker_id") + counts[wid] += 1 + + # simple statistical tolerance: each worker should be within ±50% of mean + mean = N / len(ids) + for wid in ids: + assert 0.5 * mean <= counts[wid] <= 1.5 * mean, counts diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/load_balancing/test_round_robin.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/load_balancing/test_round_robin.py new file mode 100644 index 00000000..13f14963 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/load_balancing/test_round_robin.py @@ -0,0 +1,33 @@ +import collections + +import pytest +import requests + + +@pytest.mark.integration +def test_round_robin_distribution(mock_workers, router_manager): + procs, urls, ids = mock_workers(n=3) + + rh = router_manager.start_router(worker_urls=urls, policy="round_robin") + + counts = collections.Counter() + with requests.Session() as s: + for i in range(30): + r = s.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"hello {i}", + "max_tokens": 1, + "stream": False, + }, + ) + assert r.status_code == 200 + wid = r.headers.get("X-Worker-Id") or r.json().get("worker_id") + assert wid in ids + counts[wid] += 1 + + # Expect near-even distribution across 3 workers + # 30 requests -> ideally 10 each; allow small tolerance ±3 + for wid in ids: + assert 7 <= counts[wid] <= 13, counts diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_api_auth.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_api_auth.py new file mode 100644 index 00000000..b8ba5c67 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_api_auth.py @@ -0,0 +1,38 @@ +import pytest +import requests + + +@pytest.mark.integration +def test_router_api_key_enforcement(router_manager, mock_workers): + # Start backend requiring API key; router should forward Authorization header transparently + _, urls, _ = mock_workers( + n=1, args=["--require-api-key", "--api-key", "correct_api_key"] + ) + rh = router_manager.start_router( + worker_urls=urls, + policy="round_robin", + extra={}, + ) + + # No auth -> 401 + r = requests.post( + f"{rh.url}/v1/completions", + json={"model": "test-model", "prompt": "x", "max_tokens": 1, "stream": False}, + ) + assert r.status_code == 401 + + # Invalid auth -> 401 + r = requests.post( + f"{rh.url}/v1/completions", + json={"model": "test-model", "prompt": "x", "max_tokens": 1, "stream": False}, + headers={"Authorization": "Bearer wrong"}, + ) + assert r.status_code == 401 + + # Correct auth -> 200 + r = requests.post( + f"{rh.url}/v1/completions", + json={"model": "test-model", "prompt": "x", "max_tokens": 1, "stream": False}, + headers={"Authorization": "Bearer correct_api_key"}, + ) + assert r.status_code == 200 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_circuit_breaker.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_circuit_breaker.py new file mode 100644 index 00000000..16c61992 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_circuit_breaker.py @@ -0,0 +1,228 @@ +import time + +import pytest +import requests + + +@pytest.mark.integration +def test_circuit_breaker_opens_and_recovers(router_manager, mock_workers): + # A single worker that fails first 3 requests, then succeeds + _, [wurl], _ = mock_workers(n=1, args=["--fail-first-n", "3"]) # fails first 3 + rh = router_manager.start_router( + worker_urls=[wurl], + policy="round_robin", + extra={ + "cb_failure_threshold": 3, + "cb_success_threshold": 2, + "cb_timeout_duration_secs": 3, + "cb_window_duration_secs": 10, + "disable_retries": True, + }, + ) + + def post_once(): + return requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "trigger", + "max_tokens": 1, + "stream": False, + }, + timeout=3, + ) + + # should see 500 when worker actually starts, before that should see 503 + saw_500 = False + for _ in range(8): + r = post_once() + if r.status_code == 500: + # Worker starts, continue to circuit breaker test + saw_500 = True + break + assert ( + r.status_code == 503 + ), "Should only see 503 when waiting for worker to start" + assert saw_500, "Worker didn't start after 8 requests" + + saw_503 = False + for _ in range(4): + r = post_once() + if r.status_code == 503: + saw_503 = True + break + assert saw_503, "circuit breaker did not open to return 503" + + time.sleep(4) + r1 = post_once() + r2 = post_once() + assert r1.status_code == 200 and r2.status_code == 200 + + +@pytest.mark.integration +def test_circuit_breaker_half_open_failure_reopens(router_manager, mock_workers): + _, [wurl], _ = mock_workers(n=1, args=["--status-code", "500"]) # always fail + rh = router_manager.start_router( + worker_urls=[wurl], + policy="round_robin", + extra={ + "cb_failure_threshold": 2, + "cb_success_threshold": 2, + "cb_timeout_duration_secs": 2, + "cb_window_duration_secs": 5, + "disable_retries": True, + }, + ) + + def post_once(): + return requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "x", + "max_tokens": 1, + "stream": False, + }, + timeout=3, + ) + + # should see 500 when worker actually starts, before that should see 503 + saw_500 = False + for _ in range(8): + r = post_once() + if r.status_code == 500: + # Worker starts, continue to circuit breaker test + saw_500 = True + break + assert ( + r.status_code == 503 + ), "Should only see 503 when waiting for worker to start" + assert saw_500, "Worker didn't start after 8 requests" + + opened = False + for _ in range(8): + r = post_once() + if r.status_code == 503: + opened = True + break + assert opened, "circuit breaker did not open" + + time.sleep(3) + r = post_once() + assert r.status_code == 500 + r2 = post_once() + assert r2.status_code == 503 + + +@pytest.mark.integration +def test_circuit_breaker_disable_flag(router_manager, mock_workers): + _, [wurl], _ = mock_workers(n=1, args=["--status-code", "500"]) # always fail + rh = router_manager.start_router( + worker_urls=[wurl], + policy="round_robin", + extra={ + "disable_circuit_breaker": True, + "disable_retries": True, + }, + ) + + saw_500 = False + for _ in range(8): + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "x", + "max_tokens": 1, + "stream": False, + }, + timeout=3, + ) + if r.status_code == 500: + # Worker starts, continue to check + saw_500 = True + break + assert ( + r.status_code == 503 + ), "Should only see 503 when waiting for worker to start" + + assert saw_500 + + +@pytest.mark.integration +def test_circuit_breaker_per_worker_isolation(router_manager, mock_workers): + _, [fail_url], _ = mock_workers(n=1, args=["--status-code", "500"]) # always fail + _, [ok_url], _ = mock_workers(n=1) + rh = router_manager.start_router( + worker_urls=[fail_url, ok_url], + policy="round_robin", + extra={ + "cb_failure_threshold": 2, + "cb_success_threshold": 1, + "cb_timeout_duration_secs": 2, + "cb_window_duration_secs": 10, + "disable_retries": True, + }, + ) + + def post_once(): + return requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "y", + "max_tokens": 1, + "stream": False, + }, + timeout=3, + ) + + failures = 0 + successes_after_open = 0 + opened = False + for _ in range(30): + r = post_once() + if not opened: + if r.status_code == 500: + failures += 1 + if failures >= 2: + _ = post_once() + _ = post_once() + opened = True + else: + if r.status_code == 200: + successes_after_open += 1 + else: + assert False, f"Unexpected non-200 after CB open: {r.status_code}" + assert opened and successes_after_open >= 5 + + +@pytest.mark.integration +def test_circuit_breaker_with_retries(router_manager, mock_workers): + _, [fail_url], _ = mock_workers(n=1, args=["--status-code", "500"]) # always fail + _, [ok_url], _ = mock_workers(n=1) + rh = router_manager.start_router( + worker_urls=[fail_url, ok_url], + policy="round_robin", + extra={ + "retry_max_retries": 3, + "retry_initial_backoff_ms": 10, + "retry_max_backoff_ms": 50, + "cb_failure_threshold": 2, + "cb_success_threshold": 1, + "cb_timeout_duration_secs": 2, + "cb_window_duration_secs": 10, + }, + ) + + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "z", + "max_tokens": 1, + "stream": False, + }, + timeout=5, + ) + assert r.status_code == 200 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_fault_tolerance.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_fault_tolerance.py new file mode 100644 index 00000000..6cadf1fa --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_fault_tolerance.py @@ -0,0 +1,32 @@ +import pytest +import requests + + +@pytest.mark.integration +def test_worker_crash_reroute_with_retries(router_manager, mock_workers): + # Start one healthy and one that will crash on first request + _, [ok_url], _ = mock_workers(n=1) + _, [crash_url], _ = mock_workers(n=1, args=["--crash-on-request"]) + rh = router_manager.start_router( + worker_urls=[crash_url, ok_url], + policy="round_robin", + extra={ + "retry_max_retries": 3, + "retry_initial_backoff_ms": 10, + "retry_max_backoff_ms": 50, + }, + ) + + # A single request should succeed via retry to the healthy worker + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "crash", + "max_tokens": 1, + "stream": False, + }, + timeout=5, + ) + assert r.status_code == 200 + # mock_workers fixture handles cleanup diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_mtls.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_mtls.py new file mode 100644 index 00000000..caa0db92 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_mtls.py @@ -0,0 +1,332 @@ +""" +Integration tests for mTLS (mutual TLS) authentication between router and workers. + +Tests verify that: +1. Router can successfully connect to TLS-enabled workers with proper certificates +2. Router fails to connect to mTLS-required workers without client certificates +3. Router with CA certs can connect to TLS-only workers (server auth only) +""" + +import subprocess +import time +from pathlib import Path +from typing import Tuple + +import pytest +import requests + +from ..fixtures.ports import find_free_port + + +def get_test_certs_dir() -> Path: + """Get the path to the test certificates directory.""" + return Path(__file__).parent.parent / "fixtures" / "test_certs" + + +def _spawn_tls_worker( + port: int, + worker_id: str, + ssl_certfile: str, + ssl_keyfile: str, + ssl_ca_certs: str = None, +) -> Tuple[subprocess.Popen, str]: + """Spawn a mock worker with TLS/mTLS enabled.""" + repo_root = Path(__file__).resolve().parents[2] + script = repo_root / "py_test" / "fixtures" / "mock_worker.py" + + cmd = [ + "python3", + str(script), + "--port", + str(port), + "--worker-id", + worker_id, + "--ssl-certfile", + ssl_certfile, + "--ssl-keyfile", + ssl_keyfile, + ] + + if ssl_ca_certs: + cmd.extend(["--ssl-ca-certs", ssl_ca_certs]) + + # Use DEVNULL for stdout to avoid blocking, but keep stderr for debugging + proc = subprocess.Popen( + cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True + ) + url = f"https://127.0.0.1:{port}" + + # Give worker a moment to start or fail + import time + + time.sleep(3) # Increased delay to ensure TLS server is fully initialized + + # Check if process died immediately + if proc.poll() is not None: + _, stderr = proc.communicate() + raise RuntimeError(f"Worker failed to start.\nStderr: {stderr}") + + # Wait for worker to be ready (with retries for SSL startup) + # For mTLS workers (with ssl_ca_certs), provide client cert for health check + certs_dir = get_test_certs_dir() + client_cert = certs_dir / "client-cert.pem" if ssl_ca_certs else None + client_key = certs_dir / "client-key.pem" if ssl_ca_certs else None + + try: + _wait_tls_health(url, certs_dir / "ca-cert.pem", client_cert, client_key) + except TimeoutError: + # If health check times out, capture stderr for debugging + if proc.poll() is not None: + _, stderr = proc.communicate() + raise RuntimeError(f"Worker died during health check.\nStderr: {stderr}") + raise + return proc, url + + +def _wait_tls_health( + url: str, + ca_cert_path: Path = None, + client_cert_path: Path = None, + client_key_path: Path = None, + timeout: float = 10.0, +): + """Wait for TLS-enabled worker to become healthy. + + Args: + url: HTTPS URL of the worker + ca_cert_path: Path to CA certificate for verifying server cert + client_cert_path: Path to client certificate for mTLS + client_key_path: Path to client private key for mTLS + timeout: Maximum time to wait in seconds + """ + start = time.time() + last_error = None + with requests.Session() as s: + while time.time() - start < timeout: + try: + # Verify server cert with CA if provided, otherwise skip verification + verify = str(ca_cert_path) if ca_cert_path else False + + # Provide client cert for mTLS if specified + cert = None + if client_cert_path and client_key_path: + cert = (str(client_cert_path), str(client_key_path)) + + r = s.get(f"{url}/health", timeout=1, verify=verify, cert=cert) + if r.status_code == 200: + return + except requests.RequestException as e: + # Save last error for debugging + last_error = e + time.sleep(0.2) + raise TimeoutError( + f"TLS worker at {url} did not become healthy. Last error: {last_error}" + ) + + +@pytest.mark.integration +def test_mtls_successful_communication(router_manager, test_certificates): + """Test that router can successfully communicate with mTLS-enabled worker.""" + certs_dir = test_certificates + + # Start worker with mTLS (requires client certificate) + port = find_free_port() + worker_id = f"tls-worker-{port}" + worker_proc, worker_url = _spawn_tls_worker( + port=port, + worker_id=worker_id, + ssl_certfile=str(certs_dir / "server-cert.pem"), + ssl_keyfile=str(certs_dir / "server-key.pem"), + ssl_ca_certs=str(certs_dir / "ca-cert.pem"), # Require client cert + ) + + try: + # Start router with mTLS configuration + rh = router_manager.start_router( + worker_urls=[worker_url], + policy="round_robin", + extra={ + "client_cert_path": str(certs_dir / "client-cert.pem"), + "client_key_path": str(certs_dir / "client-key.pem"), + "ca_cert_paths": [str(certs_dir / "ca-cert.pem")], + }, + ) + + # Make request through router - should succeed + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "hello", + "max_tokens": 1, + "stream": False, + }, + timeout=5, + ) + + assert r.status_code == 200, f"Request failed: {r.status_code} {r.text}" + data = r.json() + assert "choices" in data + assert data.get("worker_id") == worker_id + + finally: + if worker_proc.poll() is None: + worker_proc.terminate() + try: + worker_proc.wait(timeout=3) + except subprocess.TimeoutExpired: + worker_proc.kill() + + +@pytest.mark.integration +def test_mtls_failure_without_client_cert(router_manager, test_certificates): + """Test that router fails to connect to mTLS worker without client certificates.""" + certs_dir = test_certificates + + # Start worker with mTLS (requires client certificate) + port = find_free_port() + worker_id = f"tls-worker-{port}" + worker_proc, worker_url = _spawn_tls_worker( + port=port, + worker_id=worker_id, + ssl_certfile=str(certs_dir / "server-cert.pem"), + ssl_keyfile=str(certs_dir / "server-key.pem"), + ssl_ca_certs=str(certs_dir / "ca-cert.pem"), # Require client cert + ) + + try: + # Start router WITHOUT client certificates (but with CA to verify server) + rh = router_manager.start_router( + worker_urls=[worker_url], + policy="round_robin", + extra={ + "ca_cert_paths": [str(certs_dir / "ca-cert.pem")], + # Note: no client_cert_path or client_key_path + }, + ) + + # Make request through router - should fail because worker requires client cert + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "hello", + "max_tokens": 1, + "stream": False, + }, + timeout=5, + ) + + # Router should return 503 (service unavailable) or 500 because it can't connect to worker + assert r.status_code in [500, 503], f"Expected 500/503 but got {r.status_code}" + + finally: + if worker_proc.poll() is None: + worker_proc.terminate() + try: + worker_proc.wait(timeout=3) + except subprocess.TimeoutExpired: + worker_proc.kill() + + +@pytest.mark.integration +def test_tls_server_auth_only(router_manager, test_certificates): + """Test router can connect to TLS worker that doesn't require client certificates.""" + certs_dir = test_certificates + + # Start worker with TLS but WITHOUT requiring client certificates + port = find_free_port() + worker_id = f"tls-worker-{port}" + worker_proc, worker_url = _spawn_tls_worker( + port=port, + worker_id=worker_id, + ssl_certfile=str(certs_dir / "server-cert.pem"), + ssl_keyfile=str(certs_dir / "server-key.pem"), + ssl_ca_certs=None, # Don't require client cert + ) + + try: + # Start router with only CA cert (to verify server), no client cert + rh = router_manager.start_router( + worker_urls=[worker_url], + policy="round_robin", + extra={ + "ca_cert_paths": [str(certs_dir / "ca-cert.pem")], + # Note: no client_cert_path or client_key_path needed + }, + ) + + # Make request through router - should succeed with server-only TLS + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "hello", + "max_tokens": 1, + "stream": False, + }, + timeout=5, + ) + + assert r.status_code == 200, f"Request failed: {r.status_code} {r.text}" + data = r.json() + assert "choices" in data + assert data.get("worker_id") == worker_id + + finally: + if worker_proc.poll() is None: + worker_proc.terminate() + try: + worker_proc.wait(timeout=3) + except subprocess.TimeoutExpired: + worker_proc.kill() + + +@pytest.mark.integration +def test_tls_failure_without_ca_cert(router_manager, test_certificates): + """Test that router fails to connect to TLS worker without CA certificate.""" + certs_dir = test_certificates + + # Start worker with TLS + port = find_free_port() + worker_id = f"tls-worker-{port}" + worker_proc, worker_url = _spawn_tls_worker( + port=port, + worker_id=worker_id, + ssl_certfile=str(certs_dir / "server-cert.pem"), + ssl_keyfile=str(certs_dir / "server-key.pem"), + ssl_ca_certs=None, + ) + + try: + # Start router WITHOUT CA certificate (can't verify server cert) + rh = router_manager.start_router( + worker_urls=[worker_url], + policy="round_robin", + extra={ + # Note: no ca_cert_paths - router won't trust self-signed cert + }, + ) + + # Make request through router - should fail because router can't verify server cert + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "hello", + "max_tokens": 1, + "stream": False, + }, + timeout=5, + ) + + # Router should return 503 (service unavailable) or 500 because it can't verify worker cert + assert r.status_code in [500, 503], f"Expected 500/503 but got {r.status_code}" + + finally: + if worker_proc.poll() is None: + worker_proc.terminate() + try: + worker_proc.wait(timeout=3) + except subprocess.TimeoutExpired: + worker_proc.kill() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_payload_size.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_payload_size.py new file mode 100644 index 00000000..b3583ab2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_payload_size.py @@ -0,0 +1,33 @@ +import pytest +import requests + + +@pytest.mark.integration +def test_payload_size_limit(router_manager, mock_workers): + # Start one backend and a router with a 1MB payload limit + _, urls, _ = mock_workers(n=1) + rh = router_manager.start_router( + worker_urls=urls, + policy="round_robin", + extra={"max_payload_size": 1 * 1024 * 1024}, # 1MB + ) + + # Payload just under 1MB should succeed + payload_small = { + "model": "test-model", + "prompt": "x" * int(0.5 * 1024 * 1024), # ~0.5MB + "max_tokens": 1, + "stream": False, + } + r = requests.post(f"{rh.url}/v1/completions", json=payload_small) + assert r.status_code == 200 + + # Payload over 1MB should fail with 413 + payload_large = { + "model": "test-model", + "prompt": "x" * int(1.2 * 1024 * 1024), # ~1.2MB + "max_tokens": 1, + "stream": False, + } + r = requests.post(f"{rh.url}/v1/completions", json=payload_large) + assert r.status_code == 413 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_pd_routing.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_pd_routing.py new file mode 100644 index 00000000..00919868 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_pd_routing.py @@ -0,0 +1,126 @@ +import collections +import concurrent.futures +import time + +import pytest +import requests + + +@pytest.mark.integration +def test_pd_power_of_two_decode_attribution(router_manager, mock_workers): + # Start two prefill and three decode mock workers via fixture + _, prefill_urls_raw, prefill_ids = mock_workers(n=2) + _, decode_urls_raw, decode_ids_list = mock_workers(n=3) + prefill_urls = [(u, None) for u in prefill_urls_raw] + decode_urls = list(decode_urls_raw) + decode_ids = set(decode_ids_list) + + rh = router_manager.start_router( + policy="power_of_two", + pd_disaggregation=True, + prefill_urls=prefill_urls, + decode_urls=decode_urls, + extra={"worker_startup_check_interval": 1}, + ) + + counts = collections.Counter() + with requests.Session() as s: + for i in range(30): + r = s.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"p{i}", + "max_tokens": 1, + "stream": False, + }, + ) + assert r.status_code == 200 + wid = r.headers.get("X-Worker-Id") or r.json().get("worker_id") + assert wid in decode_ids + counts[wid] += 1 + + assert sum(1 for v in counts.values() if v > 0) >= 2 + + +@pytest.mark.integration +def test_pd_power_of_two_skews_to_faster_decode(router_manager, mock_workers): + # Start two prefill workers (fast) + _, prefill_urls_raw, _ = mock_workers(n=2) + + # Start two decode workers: one slow, one fast + _, [decode_slow_url], [slow_id] = mock_workers( + n=1, args=["--latency-ms", "300"] + ) # slower decode + _, [decode_fast_url], [fast_id] = mock_workers(n=1) + decode_urls_raw = [decode_slow_url, decode_fast_url] + + prefill_urls = [(u, None) for u in prefill_urls_raw] + decode_urls = list(decode_urls_raw) + + rh = router_manager.start_router( + policy="power_of_two", + pd_disaggregation=True, + prefill_urls=prefill_urls, + decode_urls=decode_urls, + extra={"worker_startup_check_interval": 1}, + ) + + def _prime_call(i): + try: + requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"warm-{i}", + "max_tokens": 1, + "stream": False, + }, + timeout=8, + ) + except Exception: + pass + + with concurrent.futures.ThreadPoolExecutor(max_workers=32) as ex: + list(ex.map(_prime_call, range(128))) + time.sleep(2) + + def _direct_decode_load(i): + try: + requests.post( + f"{decode_slow_url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"bg-{i}", + "max_tokens": 1, + "stream": False, + }, + timeout=8, + ) + except Exception: + pass + + with concurrent.futures.ThreadPoolExecutor(max_workers=32) as ex: + list(ex.map(_direct_decode_load, range(128))) + time.sleep(1) + + def call(i): + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"p{i}", + "max_tokens": 1, + "stream": False, + }, + timeout=8, + ) + assert r.status_code == 200 + return r.headers.get("X-Worker-Id") or r.json().get("worker_id") + + counts = collections.Counter() + with concurrent.futures.ThreadPoolExecutor(max_workers=32) as ex: + for wid in ex.map(call, range(200)): + counts[wid] += 1 + + assert counts[slow_id] < counts[fast_id], counts diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_rate_limiting.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_rate_limiting.py new file mode 100644 index 00000000..960c67a9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_rate_limiting.py @@ -0,0 +1,90 @@ +import concurrent.futures + +import pytest +import requests + + +@pytest.mark.integration +def test_rate_limit_and_queue(router_manager, mock_workers): + # One fast backend + _, urls, _ = mock_workers(n=1) + rh = router_manager.start_router( + worker_urls=urls, + policy="round_robin", + extra={ + "max_concurrent_requests": 2, + "queue_size": 0, # no queue -> immediate 429 when limit exceeded + }, + ) + + def call_once(i): + try: + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"p{i}", + "max_tokens": 1, + "stream": False, + }, + timeout=3, + ) + return r.status_code + except Exception: + return 599 + + # Fire a burst of concurrent requests + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as ex: + results = list(ex.map(call_once, range(16))) + + # Expect some to succeed and some to be rate limited (429) + assert any(code == 200 for code in results) + assert any(code == 429 for code in results) + + +@pytest.mark.integration +def test_rate_limit_queue_and_timeout(router_manager, mock_workers): + # Slow backend: ~2s per request ensures queue wait > timeout + _, urls, _ = mock_workers(n=1, args=["--latency-ms", "2000"]) # 2.0s per request + + # Allow 1 concurrent, queue up to 1, with 1s queue timeout + rh = router_manager.start_router( + worker_urls=urls, + policy="round_robin", + extra={ + "max_concurrent_requests": 1, + "queue_size": 1, + "queue_timeout_secs": 1, + }, + ) + + def call_once(i): + try: + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"q{i}", + "max_tokens": 1, + "stream": False, + }, + timeout=5, + ) + return r.status_code + except Exception: + return 599 + + # Fire 4 concurrent requests: 1 runs (~2s), 1 queued (times out at 1s -> 408), 2 overflow -> 429 + import concurrent.futures + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as ex: + results = list(ex.map(call_once, range(4))) + + # We expect: + # - Some 200s (processed) + # - At least one 408 (queued too long and timed out) + # - Remaining non-200s are either 429 (queue overflow) or additional 408s depending on scheduling + assert any(code == 200 for code in results) + assert any(code == 408 for code in results), results + non200 = [c for c in results if c != 200] + assert len(non200) >= 2 and all(c in (408, 429) for c in non200), results diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_retries.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_retries.py new file mode 100644 index 00000000..0c88ca7d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_retries.py @@ -0,0 +1,71 @@ +import pytest +import requests + + +@pytest.mark.integration +def test_retry_reroutes_to_healthy_worker(router_manager, mock_workers): + # Worker A always 500; Worker B healthy + # Worker A always 500; Worker B/C healthy + _, [url_a], [id_a] = mock_workers(n=1, args=["--status-code", "500"]) # fail + _, [url_b], [id_b] = mock_workers(n=1) + _, [url_c], [id_c] = mock_workers(n=1) + rh = router_manager.start_router( + worker_urls=[url_a, url_b, url_c], + policy="round_robin", + extra={ + "retry_max_retries": 3, + "retry_initial_backoff_ms": 10, + "retry_max_backoff_ms": 50, + }, + ) + + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "x", + "max_tokens": 1, + "stream": False, + }, + timeout=5, + ) + assert r.status_code == 200 + wid = r.headers.get("X-Worker-Id") or r.json().get("worker_id") + assert wid in [id_b, id_c] # should have retried onto a healthy worker (B or C) + # mock_workers fixture handles cleanup + + +@pytest.mark.integration +def test_disable_retries_surfaces_failure(router_manager, mock_workers): + # Single failing worker, retries disabled -> should return 500 + _, [url], [wid] = mock_workers(n=1, args=["--status-code", "500"]) # always fail + rh = router_manager.start_router( + worker_urls=[url], + policy="round_robin", + extra={ + "disable_retries": True, + }, + ) + + saw_500 = False + for _ in range(8): + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "x", + "max_tokens": 1, + "stream": False, + }, + timeout=5, + ) + if r.status_code == 500: + # Worker starts, continue to check + saw_500 = True + break + assert ( + r.status_code == 503 + ), "Should only see 503 when waiting for worker to start" + + assert saw_500 + # mock_workers fixture handles cleanup diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_service_discovery_shim.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_service_discovery_shim.py new file mode 100644 index 00000000..5cc1d673 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_service_discovery_shim.py @@ -0,0 +1,36 @@ +import pytest +import requests + + +@pytest.mark.integration +def test_discovery_shim_add_remove(router_manager, mock_workers): + # Start router without workers + rh = router_manager.start_router(worker_urls=[], policy="round_robin") + + # Initially empty + urls = router_manager.list_workers(rh.url) + assert urls == [] + + # Add a worker (simulate discovery event) + _, [wurl], [wid] = mock_workers(n=1) + router_manager.add_worker(rh.url, wurl) + urls = router_manager.list_workers(rh.url) + assert wurl in urls + + # Can serve a request + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "hi", + "max_tokens": 1, + "stream": False, + }, + ) + assert r.status_code == 200 + + # Remove worker (simulate pod deletion) + router_manager.remove_worker(rh.url, wurl) + urls = router_manager.list_workers(rh.url) + assert wurl not in urls + # mock_workers fixture handles cleanup diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_worker_management.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_worker_management.py new file mode 100644 index 00000000..4eace76e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/integration_mock/test_worker_management.py @@ -0,0 +1,57 @@ +import pytest +import requests + + +@pytest.mark.integration +def test_add_and_remove_worker(mock_worker, router_manager, mock_workers): + # Start with a single worker + proc1, url1, id1 = mock_worker + rh = router_manager.start_router(worker_urls=[url1], policy="round_robin") + + # Add a second worker + + procs2, urls2, ids2 = mock_workers(n=1) + url2 = urls2[0] + id2 = ids2[0] + router_manager.add_worker(rh.url, url2) + + # Send some requests and ensure both workers are seen + seen = set() + with requests.Session() as s: + for i in range(20): + r = s.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"x{i}", + "max_tokens": 1, + "stream": False, + }, + ) + assert r.status_code == 200 + wid = r.headers.get("X-Worker-Id") or r.json().get("worker_id") + seen.add(wid) + if len(seen) == 2: + break + + assert id1 in seen and id2 in seen + + # Now remove the second worker + router_manager.remove_worker(rh.url, url2) + + # After removal, subsequent requests should only come from first worker + with requests.Session() as s: + for i in range(10): + r = s.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"y{i}", + "max_tokens": 1, + "stream": False, + }, + ) + assert r.status_code == 200 + wid = r.headers.get("X-Worker-Id") or r.json().get("worker_id") + assert wid == id1 + # mock_workers fixture handles cleanup diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/unit/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/unit/__init__.py new file mode 100644 index 00000000..42cbd8be --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/unit/__init__.py @@ -0,0 +1,7 @@ +""" +Unit tests for sglang_router. + +This package contains fast, isolated unit tests for Python components +of the SGLang router. These tests focus on testing individual functions +and classes in isolation without starting actual router instances. +""" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/unit/test_arg_parser.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/unit/test_arg_parser.py new file mode 100644 index 00000000..f5e191e2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/unit/test_arg_parser.py @@ -0,0 +1,626 @@ +""" +Unit tests for argument parsing functionality in sglang_router. + +These tests focus on testing the argument parsing logic in isolation, +without starting actual router instances. +""" + +from types import SimpleNamespace + +import pytest +from sglang_router.launch_router import RouterArgs, parse_router_args +from sglang_router.router import policy_from_str + + +class TestRouterArgs: + """Test RouterArgs dataclass and its methods.""" + + def test_default_values(self): + """Test that RouterArgs has correct default values.""" + args = RouterArgs() + + # Test basic defaults + assert args.host == "0.0.0.0" + assert args.port == 30000 + assert args.policy == "cache_aware" + assert args.worker_urls == [] + assert args.pd_disaggregation is False + assert args.prefill_urls == [] + assert args.decode_urls == [] + + # Test PD-specific defaults + assert args.prefill_policy is None + assert args.decode_policy is None + + # Test service discovery defaults + assert args.service_discovery is False + assert args.selector == {} + assert args.service_discovery_port == 80 + assert args.service_discovery_namespace is None + + # Test retry and circuit breaker defaults + assert args.retry_max_retries == 5 + assert args.cb_failure_threshold == 10 + assert args.disable_retries is False + assert args.disable_circuit_breaker is False + + def test_parse_selector_valid(self): + """Test parsing valid selector arguments.""" + # Test single key-value pair + result = RouterArgs._parse_selector(["app=worker"]) + assert result == {"app": "worker"} + + # Test multiple key-value pairs + result = RouterArgs._parse_selector(["app=worker", "env=prod", "version=v1"]) + assert result == {"app": "worker", "env": "prod", "version": "v1"} + + # Test empty list + result = RouterArgs._parse_selector([]) + assert result == {} + + # Test None + result = RouterArgs._parse_selector(None) + assert result == {} + + def test_parse_selector_invalid(self): + """Test parsing invalid selector arguments.""" + # Test malformed selector (no equals sign) + result = RouterArgs._parse_selector(["app"]) + assert result == {} + + # Test multiple equals signs (should use first one) + result = RouterArgs._parse_selector(["app=worker=extra"]) + assert result == {"app": "worker=extra"} + + def test_parse_prefill_urls_valid(self): + """Test parsing valid prefill URL arguments.""" + # Test with bootstrap port + result = RouterArgs._parse_prefill_urls([["http://prefill1:8000", "9000"]]) + assert result == [("http://prefill1:8000", 9000)] + + # Test with 'none' bootstrap port + result = RouterArgs._parse_prefill_urls([["http://prefill1:8000", "none"]]) + assert result == [("http://prefill1:8000", None)] + + # Test without bootstrap port + result = RouterArgs._parse_prefill_urls([["http://prefill1:8000"]]) + assert result == [("http://prefill1:8000", None)] + + # Test multiple prefill URLs + result = RouterArgs._parse_prefill_urls( + [ + ["http://prefill1:8000", "9000"], + ["http://prefill2:8000", "none"], + ["http://prefill3:8000"], + ] + ) + expected = [ + ("http://prefill1:8000", 9000), + ("http://prefill2:8000", None), + ("http://prefill3:8000", None), + ] + assert result == expected + + # Test empty list + result = RouterArgs._parse_prefill_urls([]) + assert result == [] + + # Test None + result = RouterArgs._parse_prefill_urls(None) + assert result == [] + + def test_parse_prefill_urls_invalid(self): + """Test parsing invalid prefill URL arguments.""" + # Test invalid bootstrap port + with pytest.raises(ValueError, match="Invalid bootstrap port"): + RouterArgs._parse_prefill_urls([["http://prefill1:8000", "invalid"]]) + + def test_parse_decode_urls_valid(self): + """Test parsing valid decode URL arguments.""" + # Test single decode URL + result = RouterArgs._parse_decode_urls([["http://decode1:8001"]]) + assert result == ["http://decode1:8001"] + + # Test multiple decode URLs + result = RouterArgs._parse_decode_urls( + [["http://decode1:8001"], ["http://decode2:8001"]] + ) + assert result == ["http://decode1:8001", "http://decode2:8001"] + + # Test empty list + result = RouterArgs._parse_decode_urls([]) + assert result == [] + + # Test None + result = RouterArgs._parse_decode_urls(None) + assert result == [] + + def test_from_cli_args_basic(self): + """Test creating RouterArgs from basic CLI arguments.""" + args = SimpleNamespace( + host="0.0.0.0", + port=30001, + worker_urls=["http://worker1:8000", "http://worker2:8000"], + policy="round_robin", + prefill=None, + decode=None, + router_policy="round_robin", + router_pd_disaggregation=False, + router_prefill_policy=None, + router_decode_policy=None, + router_worker_startup_timeout_secs=300, + router_worker_startup_check_interval=15, + router_cache_threshold=0.7, + router_balance_abs_threshold=128, + router_balance_rel_threshold=2.0, + router_eviction_interval=180, + router_max_tree_size=2**28, + router_max_payload_size=1024 * 1024 * 1024, # 1GB + router_dp_aware=True, + router_api_key="test-key", + router_log_dir="/tmp/logs", + router_log_level="debug", + router_service_discovery=True, + router_selector=["app=worker", "env=test"], + router_service_discovery_port=8080, + router_service_discovery_namespace="default", + router_prefill_selector=["app=prefill"], + router_decode_selector=["app=decode"], + router_prometheus_port=29000, + router_prometheus_host="0.0.0.0", + router_request_id_headers=["x-request-id", "x-trace-id"], + router_request_timeout_secs=1200, + router_max_concurrent_requests=512, + router_queue_size=200, + router_queue_timeout_secs=120, + router_rate_limit_tokens_per_second=100, + router_cors_allowed_origins=["http://localhost:3000"], + router_retry_max_retries=3, + router_retry_initial_backoff_ms=100, + router_retry_max_backoff_ms=10000, + router_retry_backoff_multiplier=2.0, + router_retry_jitter_factor=0.1, + router_cb_failure_threshold=5, + router_cb_success_threshold=2, + router_cb_timeout_duration_secs=30, + router_cb_window_duration_secs=60, + router_disable_retries=False, + router_disable_circuit_breaker=False, + router_health_failure_threshold=2, + router_health_success_threshold=1, + router_health_check_timeout_secs=3, + router_health_check_interval_secs=30, + router_health_check_endpoint="/healthz", + ) + + router_args = RouterArgs.from_cli_args(args, use_router_prefix=True) + + # Test basic configuration + assert router_args.host == "0.0.0.0" + assert router_args.port == 30001 + assert router_args.worker_urls == ["http://worker1:8000", "http://worker2:8000"] + assert router_args.policy == "round_robin" + + # Test PD configuration + assert router_args.pd_disaggregation is False + assert router_args.prefill_urls == [] + assert router_args.decode_urls == [] + + # Test service discovery + assert router_args.service_discovery is True + assert router_args.selector == {"app": "worker", "env": "test"} + assert router_args.service_discovery_port == 8080 + assert router_args.service_discovery_namespace == "default" + assert router_args.prefill_selector == {"app": "prefill"} + assert router_args.decode_selector == {"app": "decode"} + + # Test other configurations + assert router_args.dp_aware is True + assert router_args.api_key == "test-key" + assert router_args.log_dir == "/tmp/logs" + assert router_args.log_level == "debug" + assert router_args.prometheus_port == 29000 + assert router_args.prometheus_host == "0.0.0.0" + assert router_args.request_id_headers == ["x-request-id", "x-trace-id"] + assert router_args.request_timeout_secs == 1200 + assert router_args.max_concurrent_requests == 512 + assert router_args.queue_size == 200 + assert router_args.queue_timeout_secs == 120 + assert router_args.rate_limit_tokens_per_second == 100 + assert router_args.cors_allowed_origins == ["http://localhost:3000"] + + # Test retry configuration + assert router_args.retry_max_retries == 3 + assert router_args.retry_initial_backoff_ms == 100 + assert router_args.retry_max_backoff_ms == 10000 + assert router_args.retry_backoff_multiplier == 2.0 + assert router_args.retry_jitter_factor == 0.1 + + # Test circuit breaker configuration + assert router_args.cb_failure_threshold == 5 + assert router_args.cb_success_threshold == 2 + assert router_args.cb_timeout_duration_secs == 30 + assert router_args.cb_window_duration_secs == 60 + assert router_args.disable_retries is False + assert router_args.disable_circuit_breaker is False + + # Test health check configuration + assert router_args.health_failure_threshold == 2 + assert router_args.health_success_threshold == 1 + assert router_args.health_check_timeout_secs == 3 + assert router_args.health_check_interval_secs == 30 + assert router_args.health_check_endpoint == "/healthz" + + # Note: model_path and tokenizer_path are not available in current RouterArgs + + def test_from_cli_args_pd_mode(self): + """Test creating RouterArgs from CLI arguments in PD mode.""" + args = SimpleNamespace( + host="127.0.0.1", + port=30000, + worker_urls=[], + policy="cache_aware", + prefill=[ + ["http://prefill1:8000", "9000"], + ["http://prefill2:8000", "none"], + ], + decode=[["http://decode1:8001"], ["http://decode2:8001"]], + router_prefill=[ + ["http://prefill1:8000", "9000"], + ["http://prefill2:8000", "none"], + ], + router_decode=[["http://decode1:8001"], ["http://decode2:8001"]], + router_policy="cache_aware", + router_pd_disaggregation=True, + router_prefill_policy="power_of_two", + router_decode_policy="round_robin", + # Include all required fields with defaults + router_worker_startup_timeout_secs=600, + router_worker_startup_check_interval=30, + router_cache_threshold=0.3, + router_balance_abs_threshold=64, + router_balance_rel_threshold=1.5, + router_eviction_interval=120, + router_max_tree_size=2**26, + router_max_payload_size=512 * 1024 * 1024, + router_dp_aware=False, + router_api_key=None, + router_log_dir=None, + router_log_level=None, + router_service_discovery=False, + router_selector=None, + router_service_discovery_port=80, + router_service_discovery_namespace=None, + router_prefill_selector=None, + router_decode_selector=None, + router_prometheus_port=None, + router_prometheus_host=None, + router_request_id_headers=None, + router_request_timeout_secs=1800, + router_max_concurrent_requests=256, + router_queue_size=100, + router_queue_timeout_secs=60, + router_rate_limit_tokens_per_second=None, + router_cors_allowed_origins=[], + router_retry_max_retries=5, + router_retry_initial_backoff_ms=50, + router_retry_max_backoff_ms=30000, + router_retry_backoff_multiplier=1.5, + router_retry_jitter_factor=0.2, + router_cb_failure_threshold=10, + router_cb_success_threshold=3, + router_cb_timeout_duration_secs=60, + router_cb_window_duration_secs=120, + router_disable_retries=False, + router_disable_circuit_breaker=False, + router_health_failure_threshold=3, + router_health_success_threshold=2, + router_health_check_timeout_secs=5, + router_health_check_interval_secs=60, + router_health_check_endpoint="/health", + ) + + router_args = RouterArgs.from_cli_args(args, use_router_prefix=True) + + # Test PD configuration + assert router_args.pd_disaggregation is True + assert router_args.prefill_urls == [ + ("http://prefill1:8000", 9000), + ("http://prefill2:8000", None), + ] + assert router_args.decode_urls == ["http://decode1:8001", "http://decode2:8001"] + assert router_args.prefill_policy == "power_of_two" + assert router_args.decode_policy == "round_robin" + assert router_args.policy == "cache_aware" # Main policy still set + + def test_from_cli_args_without_prefix(self): + """Test creating RouterArgs from CLI arguments without router prefix.""" + args = SimpleNamespace( + host="127.0.0.1", + port=30000, + worker_urls=["http://worker1:8000"], + policy="random", + prefill=None, + decode=None, + pd_disaggregation=False, + prefill_policy=None, + decode_policy=None, + worker_startup_timeout_secs=600, + worker_startup_check_interval=30, + cache_threshold=0.3, + balance_abs_threshold=64, + balance_rel_threshold=1.5, + eviction_interval=120, + max_tree_size=2**26, + max_payload_size=512 * 1024 * 1024, + dp_aware=False, + api_key=None, + log_dir=None, + log_level=None, + service_discovery=False, + selector=None, + service_discovery_port=80, + service_discovery_namespace=None, + prefill_selector=None, + decode_selector=None, + prometheus_port=None, + prometheus_host=None, + request_id_headers=None, + request_timeout_secs=1800, + max_concurrent_requests=256, + queue_size=100, + queue_timeout_secs=60, + rate_limit_tokens_per_second=None, + cors_allowed_origins=[], + retry_max_retries=5, + retry_initial_backoff_ms=50, + retry_max_backoff_ms=30000, + retry_backoff_multiplier=1.5, + retry_jitter_factor=0.2, + cb_failure_threshold=10, + cb_success_threshold=3, + cb_timeout_duration_secs=60, + cb_window_duration_secs=120, + disable_retries=False, + disable_circuit_breaker=False, + health_failure_threshold=3, + health_success_threshold=2, + health_check_timeout_secs=5, + health_check_interval_secs=60, + health_check_endpoint="/health", + model_path=None, + tokenizer_path=None, + ) + + router_args = RouterArgs.from_cli_args(args, use_router_prefix=False) + + assert router_args.host == "127.0.0.1" + assert router_args.port == 30000 + assert router_args.worker_urls == ["http://worker1:8000"] + assert router_args.policy == "random" + assert router_args.pd_disaggregation is False + + +class TestPolicyFromStr: + """Test policy string to enum conversion.""" + + def test_valid_policies(self): + """Test conversion of valid policy strings.""" + from sglang_router.sglang_router_rs import PolicyType + + assert policy_from_str("random") == PolicyType.Random + assert policy_from_str("round_robin") == PolicyType.RoundRobin + assert policy_from_str("cache_aware") == PolicyType.CacheAware + assert policy_from_str("power_of_two") == PolicyType.PowerOfTwo + + def test_invalid_policy(self): + """Test conversion of invalid policy string.""" + with pytest.raises(KeyError): + policy_from_str("invalid_policy") + + +class TestParseRouterArgs: + """Test the parse_router_args function.""" + + def test_parse_basic_args(self): + """Test parsing basic router arguments.""" + args = [ + "--host", + "0.0.0.0", + "--port", + "30001", + "--worker-urls", + "http://worker1:8000", + "http://worker2:8000", + "--policy", + "round_robin", + ] + + router_args = parse_router_args(args) + + assert router_args.host == "0.0.0.0" + assert router_args.port == 30001 + assert router_args.worker_urls == ["http://worker1:8000", "http://worker2:8000"] + assert router_args.policy == "round_robin" + + def test_parse_pd_args(self): + """Test parsing PD disaggregated mode arguments.""" + args = [ + "--pd-disaggregation", + "--prefill", + "http://prefill1:8000", + "9000", + "--prefill", + "http://prefill2:8000", + "none", + "--decode", + "http://decode1:8001", + "--decode", + "http://decode2:8001", + "--prefill-policy", + "power_of_two", + "--decode-policy", + "round_robin", + ] + + router_args = parse_router_args(args) + + assert router_args.pd_disaggregation is True + assert router_args.prefill_urls == [ + ("http://prefill1:8000", 9000), + ("http://prefill2:8000", None), + ] + assert router_args.decode_urls == ["http://decode1:8001", "http://decode2:8001"] + assert router_args.prefill_policy == "power_of_two" + assert router_args.decode_policy == "round_robin" + + def test_parse_service_discovery_args(self): + """Test parsing service discovery arguments.""" + args = [ + "--service-discovery", + "--selector", + "app=worker", + "env=prod", + "--service-discovery-port", + "8080", + "--service-discovery-namespace", + "default", + ] + + router_args = parse_router_args(args) + + assert router_args.service_discovery is True + assert router_args.selector == {"app": "worker", "env": "prod"} + assert router_args.service_discovery_port == 8080 + assert router_args.service_discovery_namespace == "default" + + def test_parse_retry_and_circuit_breaker_args(self): + """Test parsing retry and circuit breaker arguments.""" + args = [ + "--retry-max-retries", + "3", + "--retry-initial-backoff-ms", + "100", + "--retry-max-backoff-ms", + "10000", + "--retry-backoff-multiplier", + "2.0", + "--retry-jitter-factor", + "0.1", + "--disable-retries", + "--cb-failure-threshold", + "5", + "--cb-success-threshold", + "2", + "--cb-timeout-duration-secs", + "30", + "--cb-window-duration-secs", + "60", + "--disable-circuit-breaker", + ] + + router_args = parse_router_args(args) + + # Test retry configuration + assert router_args.retry_max_retries == 3 + assert router_args.retry_initial_backoff_ms == 100 + assert router_args.retry_max_backoff_ms == 10000 + assert router_args.retry_backoff_multiplier == 2.0 + assert router_args.retry_jitter_factor == 0.1 + assert router_args.disable_retries is True + + # Test circuit breaker configuration + assert router_args.cb_failure_threshold == 5 + assert router_args.cb_success_threshold == 2 + assert router_args.cb_timeout_duration_secs == 30 + assert router_args.cb_window_duration_secs == 60 + assert router_args.disable_circuit_breaker is True + + def test_parse_rate_limiting_args(self): + """Test parsing rate limiting arguments.""" + args = [ + "--max-concurrent-requests", + "512", + "--queue-size", + "200", + "--queue-timeout-secs", + "120", + "--rate-limit-tokens-per-second", + "100", + ] + + router_args = parse_router_args(args) + + assert router_args.max_concurrent_requests == 512 + assert router_args.queue_size == 200 + assert router_args.queue_timeout_secs == 120 + assert router_args.rate_limit_tokens_per_second == 100 + + def test_parse_health_check_args(self): + """Test parsing health check arguments.""" + args = [ + "--health-failure-threshold", + "2", + "--health-success-threshold", + "1", + "--health-check-timeout-secs", + "3", + "--health-check-interval-secs", + "30", + "--health-check-endpoint", + "/healthz", + ] + + router_args = parse_router_args(args) + + assert router_args.health_failure_threshold == 2 + assert router_args.health_success_threshold == 1 + assert router_args.health_check_timeout_secs == 3 + assert router_args.health_check_interval_secs == 30 + assert router_args.health_check_endpoint == "/healthz" + + def test_parse_cors_args(self): + """Test parsing CORS arguments.""" + args = [ + "--cors-allowed-origins", + "http://localhost:3000", + "https://example.com", + ] + + router_args = parse_router_args(args) + + assert router_args.cors_allowed_origins == [ + "http://localhost:3000", + "https://example.com", + ] + + def test_parse_tokenizer_args(self): + """Test parsing tokenizer arguments.""" + # Note: model-path and tokenizer-path arguments are not available in current implementation + # This test is skipped until those arguments are added + pytest.skip("Tokenizer arguments not available in current implementation") + + def test_parse_invalid_args(self): + """Test parsing invalid arguments.""" + # Test invalid policy + with pytest.raises(SystemExit): + parse_router_args(["--policy", "invalid_policy"]) + + # Test invalid bootstrap port + with pytest.raises(ValueError, match="Invalid bootstrap port"): + parse_router_args( + [ + "--pd-disaggregation", + "--prefill", + "http://prefill1:8000", + "invalid_port", + ] + ) + + def test_help_output(self): + """Test that help output is generated correctly.""" + with pytest.raises(SystemExit) as exc_info: + parse_router_args(["--help"]) + + # SystemExit with code 0 indicates help was displayed + assert exc_info.value.code == 0 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/unit/test_router_config.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/unit/test_router_config.py new file mode 100644 index 00000000..5ba91cb1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/unit/test_router_config.py @@ -0,0 +1,423 @@ +""" +Unit tests for router configuration validation and setup. + +These tests focus on testing the router configuration logic in isolation, +including validation of configuration parameters and their interactions. +""" + +from unittest.mock import MagicMock, patch + +import pytest +from sglang_router.launch_router import RouterArgs, launch_router +from sglang_router.router import policy_from_str +from sglang_router.sglang_router_rs import PolicyType + + +class TestRouterConfigValidation: + """Test router configuration validation logic.""" + + def test_valid_basic_config(self): + """Test that a valid basic configuration passes validation.""" + args = RouterArgs( + host="127.0.0.1", + port=30000, + worker_urls=["http://worker1:8000", "http://worker2:8000"], + policy="cache_aware", + ) + + # Should not raise any exceptions + assert args.host == "127.0.0.1" + assert args.port == 30000 + assert args.worker_urls == ["http://worker1:8000", "http://worker2:8000"] + assert args.policy == "cache_aware" + + def test_valid_pd_config(self): + """Test that a valid PD configuration passes validation.""" + args = RouterArgs( + host="127.0.0.1", + port=30000, + pd_disaggregation=True, + prefill_urls=[ + ("http://prefill1:8000", 9000), + ("http://prefill2:8000", None), + ], + decode_urls=["http://decode1:8001", "http://decode2:8001"], + policy="cache_aware", + ) + + assert args.pd_disaggregation is True + assert args.prefill_urls == [ + ("http://prefill1:8000", 9000), + ("http://prefill2:8000", None), + ] + assert args.decode_urls == ["http://decode1:8001", "http://decode2:8001"] + assert args.policy == "cache_aware" + + def test_pd_config_without_urls_allowed(self): + """Test that PD mode without URLs is now allowed (URLs are optional).""" + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[], + decode_urls=[], + service_discovery=False, + ) + + # Should not raise validation error - URLs are now optional + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + # This should succeed without raising an error + launch_router(args) + router_mod.from_args.assert_called_once() + + def test_pd_config_with_service_discovery_allows_empty_urls(self): + """Test that PD mode with service discovery allows empty URLs.""" + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[], + decode_urls=[], + service_discovery=True, + ) + + # Should not raise validation error when service discovery is enabled + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + launch_router(args) + + # Should create router instance via from_args + router_mod.from_args.assert_called_once() + + def test_regular_mode_without_workers_allows_empty_urls(self): + """Test that regular mode allows empty worker URLs.""" + args = RouterArgs(worker_urls=[], service_discovery=False) + + # Should not raise validation error + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + launch_router(args) + + # Should create router instance via from_args + router_mod.from_args.assert_called_once() + + def test_cache_threshold_validation(self): + """Test cache threshold validation.""" + # Valid cache threshold + args = RouterArgs(cache_threshold=0.5) + assert args.cache_threshold == 0.5 + + # Edge cases + args = RouterArgs(cache_threshold=0.0) + assert args.cache_threshold == 0.0 + + args = RouterArgs(cache_threshold=1.0) + assert args.cache_threshold == 1.0 + + def test_balance_threshold_validation(self): + """Test load balancing threshold validation.""" + # Valid thresholds + args = RouterArgs(balance_abs_threshold=64, balance_rel_threshold=1.5) + assert args.balance_abs_threshold == 64 + assert args.balance_rel_threshold == 1.5 + + # Edge cases + args = RouterArgs(balance_abs_threshold=0, balance_rel_threshold=1.0) + assert args.balance_abs_threshold == 0 + assert args.balance_rel_threshold == 1.0 + + def test_timeout_validation(self): + """Test timeout parameter validation.""" + # Valid timeouts + args = RouterArgs( + worker_startup_timeout_secs=600, + worker_startup_check_interval=30, + request_timeout_secs=1800, + queue_timeout_secs=60, + ) + assert args.worker_startup_timeout_secs == 600 + assert args.worker_startup_check_interval == 30 + assert args.request_timeout_secs == 1800 + assert args.queue_timeout_secs == 60 + + def test_retry_config_validation(self): + """Test retry configuration validation.""" + # Valid retry config + args = RouterArgs( + retry_max_retries=5, + retry_initial_backoff_ms=50, + retry_max_backoff_ms=30000, + retry_backoff_multiplier=1.5, + retry_jitter_factor=0.2, + disable_retries=False, + ) + assert args.retry_max_retries == 5 + assert args.retry_initial_backoff_ms == 50 + assert args.retry_max_backoff_ms == 30000 + assert args.retry_backoff_multiplier == 1.5 + assert args.retry_jitter_factor == 0.2 + assert args.disable_retries is False + + def test_circuit_breaker_config_validation(self): + """Test circuit breaker configuration validation.""" + # Valid circuit breaker config + args = RouterArgs( + cb_failure_threshold=10, + cb_success_threshold=3, + cb_timeout_duration_secs=60, + cb_window_duration_secs=120, + disable_circuit_breaker=False, + ) + assert args.cb_failure_threshold == 10 + assert args.cb_success_threshold == 3 + assert args.cb_timeout_duration_secs == 60 + assert args.cb_window_duration_secs == 120 + assert args.disable_circuit_breaker is False + + def test_health_check_config_validation(self): + """Test health check configuration validation.""" + # Valid health check config + args = RouterArgs( + health_failure_threshold=3, + health_success_threshold=2, + health_check_timeout_secs=5, + health_check_interval_secs=60, + health_check_endpoint="/health", + ) + assert args.health_failure_threshold == 3 + assert args.health_success_threshold == 2 + assert args.health_check_timeout_secs == 5 + assert args.health_check_interval_secs == 60 + assert args.health_check_endpoint == "/health" + + def test_rate_limiting_config_validation(self): + """Test rate limiting configuration validation.""" + # Valid rate limiting config + args = RouterArgs( + max_concurrent_requests=256, + queue_size=100, + queue_timeout_secs=60, + rate_limit_tokens_per_second=100, + ) + assert args.max_concurrent_requests == 256 + assert args.queue_size == 100 + assert args.queue_timeout_secs == 60 + assert args.rate_limit_tokens_per_second == 100 + + def test_service_discovery_config_validation(self): + """Test service discovery configuration validation.""" + # Valid service discovery config + args = RouterArgs( + service_discovery=True, + selector={"app": "worker", "env": "prod"}, + service_discovery_port=8080, + service_discovery_namespace="default", + ) + assert args.service_discovery is True + assert args.selector == {"app": "worker", "env": "prod"} + assert args.service_discovery_port == 8080 + assert args.service_discovery_namespace == "default" + + def test_pd_service_discovery_config_validation(self): + """Test PD service discovery configuration validation.""" + # Valid PD service discovery config + args = RouterArgs( + pd_disaggregation=True, + service_discovery=True, + prefill_selector={"app": "prefill"}, + decode_selector={"app": "decode"}, + bootstrap_port_annotation="sglang.ai/bootstrap-port", + ) + assert args.pd_disaggregation is True + assert args.service_discovery is True + assert args.prefill_selector == {"app": "prefill"} + assert args.decode_selector == {"app": "decode"} + assert args.bootstrap_port_annotation == "sglang.ai/bootstrap-port" + + def test_prometheus_config_validation(self): + """Test Prometheus configuration validation.""" + # Valid Prometheus config + args = RouterArgs(prometheus_port=29000, prometheus_host="127.0.0.1") + assert args.prometheus_port == 29000 + assert args.prometheus_host == "127.0.0.1" + + def test_cors_config_validation(self): + """Test CORS configuration validation.""" + # Valid CORS config + args = RouterArgs( + cors_allowed_origins=["http://localhost:3000", "https://example.com"] + ) + assert args.cors_allowed_origins == [ + "http://localhost:3000", + "https://example.com", + ] + + def test_tokenizer_config_validation(self): + """Test tokenizer configuration validation.""" + # Note: model_path and tokenizer_path are not available in current RouterArgs + pytest.skip("Tokenizer configuration not available in current implementation") + + def test_dp_aware_config_validation(self): + """Test data parallelism aware configuration validation.""" + # Valid DP aware config + args = RouterArgs(dp_aware=True, api_key="test-api-key") + assert args.dp_aware is True + assert args.api_key == "test-api-key" + + def test_request_id_headers_validation(self): + """Test request ID headers configuration validation.""" + # Valid request ID headers config + args = RouterArgs( + request_id_headers=["x-request-id", "x-trace-id", "x-correlation-id"] + ) + assert args.request_id_headers == [ + "x-request-id", + "x-trace-id", + "x-correlation-id", + ] + + def test_policy_consistency_validation(self): + """Test policy consistency validation in PD mode.""" + # Test with both prefill and decode policies specified + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[("http://prefill1:8000", None)], + decode_urls=["http://decode1:8001"], + policy="cache_aware", + prefill_policy="power_of_two", + decode_policy="round_robin", + ) + + # Should not raise validation error + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + launch_router(args) + + # Should create router instance via from_args + router_mod.from_args.assert_called_once() + + def test_policy_fallback_validation(self): + """Test policy fallback validation in PD mode.""" + # Test with only prefill policy specified + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[("http://prefill1:8000", None)], + decode_urls=["http://decode1:8001"], + policy="cache_aware", + prefill_policy="power_of_two", + decode_policy=None, + ) + + # Should not raise validation error + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + launch_router(args) + + # Should create router instance via from_args + router_mod.from_args.assert_called_once() + + def test_policy_enum_conversion(self): + """Test policy string to enum conversion.""" + # Test all valid policy conversions + assert policy_from_str("random") == PolicyType.Random + assert policy_from_str("round_robin") == PolicyType.RoundRobin + assert policy_from_str("cache_aware") == PolicyType.CacheAware + assert policy_from_str("power_of_two") == PolicyType.PowerOfTwo + + def test_invalid_policy_enum_conversion(self): + """Test invalid policy string to enum conversion.""" + with pytest.raises(KeyError): + policy_from_str("invalid_policy") + + def test_config_immutability(self): + """Test that configuration objects are properly immutable.""" + args = RouterArgs( + host="127.0.0.1", port=30000, worker_urls=["http://worker1:8000"] + ) + + # Test that we can't modify the configuration after creation + # (This is more of a design test - dataclasses are mutable by default) + original_host = args.host + args.host = "0.0.0.0" + assert args.host == "0.0.0.0" # Dataclasses are mutable + assert args.host != original_host + + def test_config_defaults_consistency(self): + """Test that configuration defaults are consistent.""" + args1 = RouterArgs() + args2 = RouterArgs() + + # Both instances should have the same defaults + assert args1.host == args2.host + assert args1.port == args2.port + assert args1.policy == args2.policy + assert args1.worker_urls == args2.worker_urls + assert args1.pd_disaggregation == args2.pd_disaggregation + + def test_config_serialization(self): + """Test that configuration can be serialized/deserialized.""" + args = RouterArgs( + host="127.0.0.1", + port=30000, + worker_urls=["http://worker1:8000"], + policy="cache_aware", + cache_threshold=0.5, + ) + + # Test that we can access all attributes + assert hasattr(args, "host") + assert hasattr(args, "port") + assert hasattr(args, "worker_urls") + assert hasattr(args, "policy") + assert hasattr(args, "cache_threshold") + + def test_config_with_none_values(self): + """Test configuration with None values.""" + args = RouterArgs( + api_key=None, + log_dir=None, + log_level=None, + prometheus_port=None, + prometheus_host=None, + request_id_headers=None, + rate_limit_tokens_per_second=None, + service_discovery_namespace=None, + ) + + # All None values should be preserved + assert args.api_key is None + assert args.log_dir is None + assert args.log_level is None + assert args.prometheus_port is None + assert args.prometheus_host is None + assert args.request_id_headers is None + assert args.rate_limit_tokens_per_second is None + assert args.service_discovery_namespace is None + + def test_config_with_empty_lists(self): + """Test configuration with empty lists.""" + args = RouterArgs( + worker_urls=[], prefill_urls=[], decode_urls=[], cors_allowed_origins=[] + ) + + # All empty lists should be preserved + assert args.worker_urls == [] + assert args.prefill_urls == [] + assert args.decode_urls == [] + assert args.cors_allowed_origins == [] + + def test_config_with_empty_dicts(self): + """Test configuration with empty dictionaries.""" + args = RouterArgs(selector={}, prefill_selector={}, decode_selector={}) + + # All empty dictionaries should be preserved + assert args.selector == {} + assert args.prefill_selector == {} + assert args.decode_selector == {} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/unit/test_startup_sequence.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/unit/test_startup_sequence.py new file mode 100644 index 00000000..6a40a67f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/unit/test_startup_sequence.py @@ -0,0 +1,1056 @@ +""" +Unit tests for startup sequence logic in sglang_router. + +These tests focus on testing the startup sequence logic in isolation, +including router initialization, configuration validation, and startup flow. +""" + +import logging +from unittest.mock import MagicMock, patch + +import pytest +from sglang_router.launch_router import RouterArgs, launch_router +from sglang_router.router import policy_from_str + + +# Local helper mirroring the router logger setup used in production +def setup_logger(): + logger = logging.getLogger("router") + logger.setLevel(logging.INFO) + if not logger.handlers: + formatter = logging.Formatter( + "[Router (Python)] %(asctime)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + handler = logging.StreamHandler() + handler.setFormatter(formatter) + logger.addHandler(handler) + return logger + + +from sglang_router.sglang_router_rs import PolicyType + + +class TestSetupLogger: + """Test logger setup functionality.""" + + def test_setup_logger_returns_logger(self): + """Test that setup_logger returns a logger instance.""" + logger = setup_logger() + + assert isinstance(logger, logging.Logger) + assert logger.name == "router" + assert logger.level == logging.INFO + + def test_setup_logger_has_handler(self): + """Test that setup_logger configures a handler.""" + logger = setup_logger() + + assert len(logger.handlers) > 0 + handler = logger.handlers[0] + assert isinstance(handler, logging.StreamHandler) + + def test_setup_logger_has_formatter(self): + """Test that setup_logger configures a formatter.""" + logger = setup_logger() + + handler = logger.handlers[0] + formatter = handler.formatter + + assert formatter is not None + assert "[Router (Python)]" in formatter._fmt + + def test_setup_logger_multiple_calls(self): + """Test that multiple calls to setup_logger work correctly.""" + logger1 = setup_logger() + logger2 = setup_logger() + + # Should return the same logger instance + assert logger1 is logger2 + + +class TestPolicyFromStr: + """Test policy string to enum conversion in startup context.""" + + def test_policy_conversion_in_startup(self): + """Test policy conversion during startup sequence.""" + # Test all valid policies + policies = ["random", "round_robin", "cache_aware", "power_of_two"] + expected_enums = [ + PolicyType.Random, + PolicyType.RoundRobin, + PolicyType.CacheAware, + PolicyType.PowerOfTwo, + ] + + for policy_str, expected_enum in zip(policies, expected_enums): + result = policy_from_str(policy_str) + assert result == expected_enum + + def test_invalid_policy_in_startup(self): + """Test handling of invalid policy during startup.""" + with pytest.raises(KeyError): + policy_from_str("invalid_policy") + + +class TestRouterInitialization: + """Test router initialization logic.""" + + def test_router_initialization_basic(self): + """Test basic router initialization.""" + args = RouterArgs( + host="127.0.0.1", + port=30000, + worker_urls=["http://worker1:8000"], + policy="cache_aware", + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + captured_args = {} + + mock_router_instance = MagicMock() + + def fake_from_args(router_args): + # capture needed fields from RouterArgs + captured_args.update( + dict( + host=router_args.host, + port=router_args.port, + worker_urls=router_args.worker_urls, + policy=policy_from_str(router_args.policy), + ) + ) + return mock_router_instance + + router_mod.from_args = MagicMock(side_effect=fake_from_args) + + result = launch_router(args) + + # Verify Router.from_args was called and captured fields match + router_mod.from_args.assert_called_once() + assert captured_args["host"] == "127.0.0.1" + assert captured_args["port"] == 30000 + assert captured_args["worker_urls"] == ["http://worker1:8000"] + assert captured_args["policy"] == PolicyType.CacheAware + + # Verify router.start() was called + mock_router_instance.start.assert_called_once() + + # Function returns None; ensure start was invoked + + def test_router_initialization_pd_mode(self): + """Test router initialization in PD mode.""" + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[("http://prefill1:8000", 9000)], + decode_urls=["http://decode1:8001"], + policy="power_of_two", + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + captured_args = {} + mock_router_instance = MagicMock() + + def fake_from_args(router_args): + captured_args.update( + dict( + pd_disaggregation=router_args.pd_disaggregation, + prefill_urls=router_args.prefill_urls, + decode_urls=router_args.decode_urls, + policy=policy_from_str(router_args.policy), + ) + ) + return mock_router_instance + + router_mod.from_args = MagicMock(side_effect=fake_from_args) + + result = launch_router(args) + + # Verify Router.from_args was called with PD parameters + router_mod.from_args.assert_called_once() + assert captured_args["pd_disaggregation"] is True + assert captured_args["prefill_urls"] == [("http://prefill1:8000", 9000)] + assert captured_args["decode_urls"] == ["http://decode1:8001"] + assert captured_args["policy"] == PolicyType.PowerOfTwo + + # Verify router.start() was called + mock_router_instance.start.assert_called_once() + + # Function returns None; ensure start was invoked + + def test_router_initialization_with_service_discovery(self): + """Test router initialization with service discovery.""" + args = RouterArgs( + service_discovery=True, + selector={"app": "worker", "env": "prod"}, + service_discovery_port=8080, + service_discovery_namespace="default", + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + captured_args = {} + mock_router_instance = MagicMock() + + def fake_from_args(router_args): + captured_args.update( + dict( + service_discovery=router_args.service_discovery, + selector=router_args.selector, + service_discovery_port=router_args.service_discovery_port, + service_discovery_namespace=router_args.service_discovery_namespace, + ) + ) + return mock_router_instance + + router_mod.from_args = MagicMock(side_effect=fake_from_args) + + result = launch_router(args) + + # Verify Router.from_args was called with service discovery parameters + router_mod.from_args.assert_called_once() + assert captured_args["service_discovery"] is True + assert captured_args["selector"] == {"app": "worker", "env": "prod"} + assert captured_args["service_discovery_port"] == 8080 + assert captured_args["service_discovery_namespace"] == "default" + + # Verify router.start() was called + mock_router_instance.start.assert_called_once() + + # Function returns None; ensure start was invoked + + def test_router_initialization_with_retry_config(self): + """Test router initialization with retry configuration.""" + args = RouterArgs( + retry_max_retries=3, + retry_initial_backoff_ms=100, + retry_max_backoff_ms=10000, + retry_backoff_multiplier=2.0, + retry_jitter_factor=0.1, + disable_retries=False, + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + captured_args = {} + mock_router_instance = MagicMock() + + def fake_from_args(router_args): + captured_args.update( + dict( + retry_max_retries=router_args.retry_max_retries, + retry_initial_backoff_ms=router_args.retry_initial_backoff_ms, + retry_max_backoff_ms=router_args.retry_max_backoff_ms, + retry_backoff_multiplier=router_args.retry_backoff_multiplier, + retry_jitter_factor=router_args.retry_jitter_factor, + disable_retries=router_args.disable_retries, + ) + ) + return mock_router_instance + + router_mod.from_args = MagicMock(side_effect=fake_from_args) + + result = launch_router(args) + + # Verify router was created with retry parameters + router_mod.from_args.assert_called_once() + assert captured_args["retry_max_retries"] == 3 + assert captured_args["retry_initial_backoff_ms"] == 100 + assert captured_args["retry_max_backoff_ms"] == 10000 + assert captured_args["retry_backoff_multiplier"] == 2.0 + assert captured_args["retry_jitter_factor"] == 0.1 + assert captured_args["disable_retries"] is False + + # Verify router.start() was called + mock_router_instance.start.assert_called_once() + + # Function returns None; ensure start was invoked + + def test_router_initialization_with_circuit_breaker_config(self): + """Test router initialization with circuit breaker configuration.""" + args = RouterArgs( + cb_failure_threshold=5, + cb_success_threshold=2, + cb_timeout_duration_secs=30, + cb_window_duration_secs=60, + disable_circuit_breaker=False, + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + captured_args = {} + mock_router_instance = MagicMock() + + def fake_from_args(router_args): + captured_args.update( + dict( + cb_failure_threshold=router_args.cb_failure_threshold, + cb_success_threshold=router_args.cb_success_threshold, + cb_timeout_duration_secs=router_args.cb_timeout_duration_secs, + cb_window_duration_secs=router_args.cb_window_duration_secs, + disable_circuit_breaker=router_args.disable_circuit_breaker, + ) + ) + return mock_router_instance + + router_mod.from_args = MagicMock(side_effect=fake_from_args) + + result = launch_router(args) + + # Verify router was created with circuit breaker parameters + router_mod.from_args.assert_called_once() + assert captured_args["cb_failure_threshold"] == 5 + assert captured_args["cb_success_threshold"] == 2 + assert captured_args["cb_timeout_duration_secs"] == 30 + assert captured_args["cb_window_duration_secs"] == 60 + assert captured_args["disable_circuit_breaker"] is False + + # Verify router.start() was called + mock_router_instance.start.assert_called_once() + + # Function returns None; ensure start was invoked + + def test_router_initialization_with_rate_limiting_config(self): + """Test router initialization with rate limiting configuration.""" + args = RouterArgs( + max_concurrent_requests=512, + queue_size=200, + queue_timeout_secs=120, + rate_limit_tokens_per_second=100, + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + captured_args = {} + mock_router_instance = MagicMock() + + def fake_from_args(router_args): + captured_args.update( + dict( + max_concurrent_requests=router_args.max_concurrent_requests, + queue_size=router_args.queue_size, + queue_timeout_secs=router_args.queue_timeout_secs, + rate_limit_tokens_per_second=router_args.rate_limit_tokens_per_second, + ) + ) + return mock_router_instance + + router_mod.from_args = MagicMock(side_effect=fake_from_args) + + result = launch_router(args) + + # Verify router was created with rate limiting parameters + router_mod.from_args.assert_called_once() + assert captured_args["max_concurrent_requests"] == 512 + assert captured_args["queue_size"] == 200 + assert captured_args["queue_timeout_secs"] == 120 + assert captured_args["rate_limit_tokens_per_second"] == 100 + + # Verify router.start() was called + mock_router_instance.start.assert_called_once() + + # Function returns None; ensure start was invoked + + def test_router_initialization_with_health_check_config(self): + """Test router initialization with health check configuration.""" + args = RouterArgs( + health_failure_threshold=2, + health_success_threshold=1, + health_check_timeout_secs=3, + health_check_interval_secs=30, + health_check_endpoint="/healthz", + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + captured_args = {} + mock_router_instance = MagicMock() + + def fake_from_args(router_args): + captured_args.update( + dict( + health_failure_threshold=router_args.health_failure_threshold, + health_success_threshold=router_args.health_success_threshold, + health_check_timeout_secs=router_args.health_check_timeout_secs, + health_check_interval_secs=router_args.health_check_interval_secs, + health_check_endpoint=router_args.health_check_endpoint, + ) + ) + return mock_router_instance + + router_mod.from_args = MagicMock(side_effect=fake_from_args) + + result = launch_router(args) + + # Verify router was created with health check parameters + router_mod.from_args.assert_called_once() + assert captured_args["health_failure_threshold"] == 2 + assert captured_args["health_success_threshold"] == 1 + assert captured_args["health_check_timeout_secs"] == 3 + assert captured_args["health_check_interval_secs"] == 30 + assert captured_args["health_check_endpoint"] == "/healthz" + + # Verify router.start() was called + mock_router_instance.start.assert_called_once() + + # Function returns None; ensure start was invoked + + def test_router_initialization_with_prometheus_config(self): + """Test router initialization with Prometheus configuration.""" + args = RouterArgs(prometheus_port=29000, prometheus_host="127.0.0.1") + + with patch("sglang_router.launch_router.Router") as router_mod: + captured_args = {} + mock_router_instance = MagicMock() + + def fake_from_args(router_args): + captured_args.update( + dict( + prometheus_port=router_args.prometheus_port, + prometheus_host=router_args.prometheus_host, + ) + ) + return mock_router_instance + + router_mod.from_args = MagicMock(side_effect=fake_from_args) + + result = launch_router(args) + + # Verify router was created with Prometheus parameters + router_mod.from_args.assert_called_once() + assert captured_args["prometheus_port"] == 29000 + assert captured_args["prometheus_host"] == "127.0.0.1" + + # Verify router.start() was called + mock_router_instance.start.assert_called_once() + + # Function returns None; ensure start was invoked + + def test_router_initialization_with_cors_config(self): + """Test router initialization with CORS configuration.""" + args = RouterArgs( + cors_allowed_origins=["http://localhost:3000", "https://example.com"] + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + captured_args = {} + mock_router_instance = MagicMock() + + def fake_from_args(router_args): + captured_args.update( + dict(cors_allowed_origins=router_args.cors_allowed_origins) + ) + return mock_router_instance + + router_mod.from_args = MagicMock(side_effect=fake_from_args) + + result = launch_router(args) + + # Verify router was created with CORS parameters + router_mod.from_args.assert_called_once() + assert captured_args["cors_allowed_origins"] == [ + "http://localhost:3000", + "https://example.com", + ] + + # Verify router.start() was called + mock_router_instance.start.assert_called_once() + + # Function returns None; ensure start was invoked + + def test_router_initialization_with_tokenizer_config(self): + """Test router initialization with tokenizer configuration.""" + # Note: model_path and tokenizer_path are not available in current RouterArgs + pytest.skip("Tokenizer configuration not available in current implementation") + + +class TestStartupValidation: + """Test startup validation logic.""" + + def test_pd_mode_validation_during_startup(self): + """Test PD mode validation during startup.""" + # PD mode without URLs is now allowed (URLs are optional) + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[], + decode_urls=[], + service_discovery=False, + ) + + # Should not raise validation error - URLs are now optional + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + # This should succeed without raising an error + launch_router(args) + router_mod.from_args.assert_called_once() + + def test_pd_mode_with_service_discovery_validation(self): + """Test PD mode with service discovery validation during startup.""" + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[], + decode_urls=[], + service_discovery=True, + ) + + # Should not raise validation error + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + result = launch_router(args) + + # Should create router instance + router_mod.from_args.assert_called_once() + + def test_policy_warning_during_startup(self): + """Test policy warning during startup in PD mode.""" + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[("http://prefill1:8000", None)], + decode_urls=["http://decode1:8001"], + policy="cache_aware", + prefill_policy="power_of_two", + decode_policy="round_robin", + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + # The policy messages are emitted by router_args logger + with patch("sglang_router.router_args.logger") as mock_logger: + result = launch_router(args) + + # Should log warning about policy usage + mock_logger.warning.assert_called_once() + warning_call = mock_logger.warning.call_args[0][0] + assert ( + "Both --prefill-policy and --decode-policy are specified" + in warning_call + ) + + # Should create router instance + router_mod.from_args.assert_called_once() + + def test_policy_info_during_startup(self): + """Test policy info logging during startup in PD mode.""" + # Test with only prefill policy specified + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[("http://prefill1:8000", None)], + decode_urls=["http://decode1:8001"], + policy="cache_aware", + prefill_policy="power_of_two", + decode_policy=None, + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + # The policy messages are emitted by router_args logger + with patch("sglang_router.router_args.logger") as mock_logger: + result = launch_router(args) + + # Should log info about policy usage + mock_logger.info.assert_called_once() + info_call = mock_logger.info.call_args[0][0] + assert "Using --prefill-policy 'power_of_two'" in info_call + assert "and --policy 'cache_aware'" in info_call + + # Should create router instance + router_mod.from_args.assert_called_once() + + def test_policy_info_decode_only_during_startup(self): + """Test policy info logging during startup with only decode policy specified.""" + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[("http://prefill1:8000", None)], + decode_urls=["http://decode1:8001"], + policy="cache_aware", + prefill_policy=None, + decode_policy="round_robin", + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + # The policy messages are emitted by router_args logger + with patch("sglang_router.router_args.logger") as mock_logger: + result = launch_router(args) + + # Should log info about policy usage + mock_logger.info.assert_called_once() + info_call = mock_logger.info.call_args[0][0] + assert "Using --policy 'cache_aware'" in info_call + assert "and --decode-policy 'round_robin'" in info_call + + # Should create router instance + router_mod.from_args.assert_called_once() + + +class TestStartupErrorHandling: + """Test startup error handling logic.""" + + def test_router_creation_error_handling(self): + """Test error handling when router creation fails.""" + args = RouterArgs( + host="127.0.0.1", port=30000, worker_urls=["http://worker1:8000"] + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + # Simulate router creation failure in from_args + router_mod.from_args = MagicMock( + side_effect=Exception("Router creation failed") + ) + + with patch("sglang_router.launch_router.logger") as mock_logger: + with pytest.raises(Exception, match="Router creation failed"): + launch_router(args) + + # Should log error + mock_logger.error.assert_called_once() + error_call = mock_logger.error.call_args[0][0] + assert "Error starting router: Router creation failed" in error_call + + def test_router_start_error_handling(self): + """Test error handling when router start fails.""" + args = RouterArgs( + host="127.0.0.1", port=30000, worker_urls=["http://worker1:8000"] + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + # Simulate router start failure + mock_router_instance.start.side_effect = Exception("Router start failed") + + with patch("sglang_router.launch_router.logger") as mock_logger: + with pytest.raises(Exception, match="Router start failed"): + launch_router(args) + + # Should log error + mock_logger.error.assert_called_once() + error_call = mock_logger.error.call_args[0][0] + assert "Error starting router: Router start failed" in error_call + + +# --- Added unit tests for Router wrapper and launch_server helpers --- + + +def _install_sglang_stubs(monkeypatch): + """Install lightweight stubs for sglang.srt to avoid heavy deps during unit tests.""" + import sys + import types + + sglang_mod = types.ModuleType("sglang") + srt_mod = types.ModuleType("sglang.srt") + entry_mod = types.ModuleType("sglang.srt.entrypoints") + http_server_mod = types.ModuleType("sglang.srt.entrypoints.http_server") + server_args_mod = types.ModuleType("sglang.srt.server_args") + utils_mod = types.ModuleType("sglang.srt.utils") + + def launch_server(_args): + return None + + class ServerArgs: + # Minimal fields used by launch_server_process + def __init__(self): + self.port = 0 + self.base_gpu_id = 0 + self.dp_size = 1 + self.tp_size = 1 + + @staticmethod + def add_cli_args(_parser): + return None + + @staticmethod + def from_cli_args(_args): + sa = ServerArgs() + if hasattr(_args, "dp_size"): + sa.dp_size = _args.dp_size + if hasattr(_args, "tp_size"): + sa.tp_size = _args.tp_size + if hasattr(_args, "host"): + sa.host = _args.host + else: + sa.host = "127.0.0.1" + return sa + + def is_port_available(_port: int) -> bool: + return True + + http_server_mod.launch_server = launch_server + server_args_mod.ServerArgs = ServerArgs + utils_mod.is_port_available = is_port_available + + # Also stub external deps imported at module top-level + def _dummy_get(*_a, **_k): + raise NotImplementedError + + requests_stub = types.SimpleNamespace( + exceptions=types.SimpleNamespace(RequestException=Exception), get=_dummy_get + ) + setproctitle_stub = types.SimpleNamespace(setproctitle=lambda *_a, **_k: None) + + monkeypatch.setitem(sys.modules, "requests", requests_stub) + monkeypatch.setitem(sys.modules, "setproctitle", setproctitle_stub) + + monkeypatch.setitem(sys.modules, "sglang", sglang_mod) + monkeypatch.setitem(sys.modules, "sglang.srt", srt_mod) + monkeypatch.setitem(sys.modules, "sglang.srt.entrypoints", entry_mod) + monkeypatch.setitem( + sys.modules, "sglang.srt.entrypoints.http_server", http_server_mod + ) + monkeypatch.setitem(sys.modules, "sglang.srt.server_args", server_args_mod) + monkeypatch.setitem(sys.modules, "sglang.srt.utils", utils_mod) + + +def test_router_defaults_and_start(monkeypatch): + """Router wrapper: defaults normalization and start() call. + + Mocks the Rust-backed _Router to avoid native deps. + """ + from sglang_router import router as router_mod + + captured = {} + + class FakeRouter: + def __init__(self, **kwargs): + captured.update(kwargs) + + def start(self): + captured["started"] = True + + monkeypatch.setattr(router_mod, "_Router", FakeRouter, raising=True) + + from sglang_router.router_args import RouterArgs as _RouterArgs + + Router = router_mod.Router + args = _RouterArgs( + worker_urls=["http://w1:8000"], + policy="round_robin", + selector=None, + prefill_selector=None, + decode_selector=None, + cors_allowed_origins=None, + ) + + r = Router.from_args(args) + + # Defaults preserved/normalized by Router.from_args + assert captured["selector"] is None + assert captured["prefill_selector"] is None + assert captured["decode_selector"] is None + assert captured["cors_allowed_origins"] is None + assert captured["worker_urls"] == ["http://w1:8000"] + from sglang_router.sglang_router_rs import PolicyType + + assert captured["policy"] == PolicyType.RoundRobin + + r.start() + assert captured.get("started") is True + + +def test_find_available_ports_and_wait_health(monkeypatch): + """launch_server helpers: port finding and health waiting with transient error.""" + _install_sglang_stubs(monkeypatch) + import importlib + + ls = importlib.import_module("sglang_router.launch_server") + + # Deterministic increments + monkeypatch.setattr(ls.random, "randint", lambda a, b: 100) + ports = ls.find_available_ports(30000, 3) + assert ports == [30000, 30100, 30200] + + calls = {"n": 0} + + class Ok: + status_code = 200 + + def fake_get(_url, timeout=5): + calls["n"] += 1 + if calls["n"] == 1: + raise ls.requests.exceptions.RequestException("boom") + return Ok() + + monkeypatch.setattr(ls.requests, "get", fake_get) + monkeypatch.setattr(ls.time, "sleep", lambda _s: None) + base = {"t": 0.0} + monkeypatch.setattr( + ls.time, + "perf_counter", + lambda: (base.__setitem__("t", base["t"] + 0.1) or base["t"]), + ) + + assert ls.wait_for_server_health("127.0.0.1", 12345, timeout=1) + + +def test_launch_server_process_and_cleanup(monkeypatch): + """launch_server: process creation args and cleanup SIGTERM/SIGKILL logic.""" + _install_sglang_stubs(monkeypatch) + import importlib + + ls = importlib.import_module("sglang_router.launch_server") + + created = {} + + class FakeProcess: + def __init__(self, target, args): + created["target"] = target + created["args"] = args + self.pid = 4242 + self._alive = True + + def start(self): + created["started"] = True + + def join(self, timeout=None): + return None + + def is_alive(self): + return self._alive + + monkeypatch.setattr(ls.mp, "Process", FakeProcess) + + import sys as _sys + + SA = _sys.modules["sglang.srt.server_args"].ServerArgs + sa = SA() + sa.tp_size = 2 + + proc = ls.launch_server_process(sa, worker_port=31001, dp_id=3) + assert created.get("started") is True + targ, targ_args = created["target"], created["args"] + assert targ is ls.run_server + passed_sa = targ_args[0] + assert passed_sa.port == 31001 + assert passed_sa.base_gpu_id == 3 * 2 + assert passed_sa.dp_size == 1 + + # cleanup_processes + p1 = FakeProcess(target=None, args=()) + p1._alive = False + p2 = FakeProcess(target=None, args=()) + p2._alive = True + + calls = [] + + def fake_killpg(pid, sig): + calls.append((pid, sig)) + + monkeypatch.setattr(ls.os, "killpg", fake_killpg) + + ls.cleanup_processes([p1, p2]) + + import signal as _sig + + assert (p1.pid, _sig.SIGTERM) in calls and (p2.pid, _sig.SIGTERM) in calls + assert (p2.pid, _sig.SIGKILL) in calls + + def test_validation_error_handling(self): + """Test error handling when validation fails.""" + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[], + decode_urls=[], + service_discovery=False, + ) + + with patch("sglang_router.launch_router.logger") as mock_logger: + + with pytest.raises( + ValueError, match="PD disaggregation mode requires --prefill" + ): + launch_router(args) + + # Should log error for validation failures + mock_logger.error.assert_called_once() + + +class TestStartupFlow: + """Test complete startup flow.""" + + def test_complete_startup_flow_basic(self): + """Test complete startup flow for basic configuration.""" + args = RouterArgs( + host="127.0.0.1", + port=30000, + worker_urls=["http://worker1:8000", "http://worker2:8000"], + policy="cache_aware", + cache_threshold=0.5, + balance_abs_threshold=32, + balance_rel_threshold=1.5, + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + result = launch_router(args) + + # Verify complete flow + router_mod.from_args.assert_called_once() + mock_router_instance.start.assert_called_once() + + def test_complete_startup_flow_pd_mode(self): + """Test complete startup flow for PD mode configuration.""" + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[ + ("http://prefill1:8000", 9000), + ("http://prefill2:8000", None), + ], + decode_urls=["http://decode1:8001", "http://decode2:8001"], + policy="power_of_two", + prefill_policy="cache_aware", + decode_policy="round_robin", + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + with patch("sglang_router.router_args.logger") as mock_logger: + result = launch_router(args) + + # Verify complete flow + router_mod.from_args.assert_called_once() + mock_router_instance.start.assert_called_once() + + # Verify policy warning was logged + mock_logger.warning.assert_called_once() + + def test_complete_startup_flow_with_all_features(self): + """Test complete startup flow with all features enabled.""" + args = RouterArgs( + host="0.0.0.0", + port=30001, + worker_urls=["http://worker1:8000"], + policy="round_robin", + service_discovery=True, + selector={"app": "worker"}, + service_discovery_port=8080, + service_discovery_namespace="default", + dp_aware=True, + api_key="test-key", + log_dir="/tmp/logs", + log_level="debug", + prometheus_port=29000, + prometheus_host="0.0.0.0", + request_id_headers=["x-request-id", "x-trace-id"], + request_timeout_secs=1200, + max_concurrent_requests=512, + queue_size=200, + queue_timeout_secs=120, + rate_limit_tokens_per_second=100, + cors_allowed_origins=["http://localhost:3000"], + retry_max_retries=3, + retry_initial_backoff_ms=100, + retry_max_backoff_ms=10000, + retry_backoff_multiplier=2.0, + retry_jitter_factor=0.1, + cb_failure_threshold=5, + cb_success_threshold=2, + cb_timeout_duration_secs=30, + cb_window_duration_secs=60, + health_failure_threshold=2, + health_success_threshold=1, + health_check_timeout_secs=3, + health_check_interval_secs=30, + health_check_endpoint="/healthz", + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + captured_args = {} + mock_router_instance = MagicMock() + + def fake_from_args(router_args): + captured_args.update( + dict( + host=router_args.host, + port=router_args.port, + worker_urls=router_args.worker_urls, + policy=policy_from_str(router_args.policy), + service_discovery=router_args.service_discovery, + selector=router_args.selector, + service_discovery_port=router_args.service_discovery_port, + service_discovery_namespace=router_args.service_discovery_namespace, + dp_aware=router_args.dp_aware, + api_key=router_args.api_key, + log_dir=router_args.log_dir, + log_level=router_args.log_level, + prometheus_port=router_args.prometheus_port, + prometheus_host=router_args.prometheus_host, + request_id_headers=router_args.request_id_headers, + request_timeout_secs=router_args.request_timeout_secs, + max_concurrent_requests=router_args.max_concurrent_requests, + queue_size=router_args.queue_size, + queue_timeout_secs=router_args.queue_timeout_secs, + rate_limit_tokens_per_second=router_args.rate_limit_tokens_per_second, + cors_allowed_origins=router_args.cors_allowed_origins, + retry_max_retries=router_args.retry_max_retries, + retry_initial_backoff_ms=router_args.retry_initial_backoff_ms, + retry_max_backoff_ms=router_args.retry_max_backoff_ms, + retry_backoff_multiplier=router_args.retry_backoff_multiplier, + retry_jitter_factor=router_args.retry_jitter_factor, + cb_failure_threshold=router_args.cb_failure_threshold, + cb_success_threshold=router_args.cb_success_threshold, + cb_timeout_duration_secs=router_args.cb_timeout_duration_secs, + cb_window_duration_secs=router_args.cb_window_duration_secs, + health_failure_threshold=router_args.health_failure_threshold, + health_success_threshold=router_args.health_success_threshold, + health_check_timeout_secs=router_args.health_check_timeout_secs, + health_check_interval_secs=router_args.health_check_interval_secs, + health_check_endpoint=router_args.health_check_endpoint, + ) + ) + return mock_router_instance + + router_mod.from_args = MagicMock(side_effect=fake_from_args) + + result = launch_router(args) + + # Verify complete flow + router_mod.from_args.assert_called_once() + mock_router_instance.start.assert_called_once() + + # Verify key parameters were propagated into RouterArgs + assert captured_args["host"] == "0.0.0.0" + assert captured_args["port"] == 30001 + assert captured_args["worker_urls"] == ["http://worker1:8000"] + assert captured_args["policy"] == PolicyType.RoundRobin + assert captured_args["service_discovery"] is True + assert captured_args["selector"] == {"app": "worker"} + assert captured_args["service_discovery_port"] == 8080 + assert captured_args["service_discovery_namespace"] == "default" + assert captured_args["dp_aware"] is True + assert captured_args["api_key"] == "test-key" + assert captured_args["log_dir"] == "/tmp/logs" + assert captured_args["log_level"] == "debug" + assert captured_args["prometheus_port"] == 29000 + assert captured_args["prometheus_host"] == "0.0.0.0" + assert captured_args["request_id_headers"] == ["x-request-id", "x-trace-id"] + assert captured_args["request_timeout_secs"] == 1200 + assert captured_args["max_concurrent_requests"] == 512 + assert captured_args["queue_size"] == 200 + assert captured_args["queue_timeout_secs"] == 120 + assert captured_args["rate_limit_tokens_per_second"] == 100 + assert captured_args["cors_allowed_origins"] == ["http://localhost:3000"] + assert captured_args["retry_max_retries"] == 3 + assert captured_args["retry_initial_backoff_ms"] == 100 + assert captured_args["retry_max_backoff_ms"] == 10000 + assert captured_args["retry_backoff_multiplier"] == 2.0 + assert captured_args["retry_jitter_factor"] == 0.1 + assert captured_args["cb_failure_threshold"] == 5 + assert captured_args["cb_success_threshold"] == 2 + assert captured_args["cb_timeout_duration_secs"] == 30 + assert captured_args["cb_window_duration_secs"] == 60 + assert captured_args["health_failure_threshold"] == 2 + assert captured_args["health_success_threshold"] == 1 + assert captured_args["health_check_timeout_secs"] == 3 + assert captured_args["health_check_interval_secs"] == 30 + assert captured_args["health_check_endpoint"] == "/healthz" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/unit/test_validation.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/unit/test_validation.py new file mode 100644 index 00000000..587cd950 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/py_test/unit/test_validation.py @@ -0,0 +1,509 @@ +""" +Unit tests for validation logic in sglang_router. + +These tests focus on testing the validation logic in isolation, +including parameter validation, URL validation, and configuration validation. +""" + +from unittest.mock import MagicMock, patch + +import pytest +from sglang_router.launch_router import RouterArgs, launch_router + + +class TestURLValidation: + """Test URL validation logic.""" + + def test_valid_worker_urls(self): + """Test validation of valid worker URLs.""" + valid_urls = [ + "http://worker1:8000", + "https://worker2:8000", + "http://localhost:8000", + "http://127.0.0.1:8000", + "http://192.168.1.100:8000", + "http://worker.example.com:8000", + ] + + for url in valid_urls: + args = RouterArgs(worker_urls=[url]) + # Should not raise any validation errors + assert url in args.worker_urls + + def test_valid_prefill_urls(self): + """Test validation of valid prefill URLs.""" + valid_prefill_urls = [ + ("http://prefill1:8000", 9000), + ("https://prefill2:8000", None), + ("http://localhost:8000", 9000), + ("http://127.0.0.1:8000", None), + ] + + for url, bootstrap_port in valid_prefill_urls: + args = RouterArgs(prefill_urls=[(url, bootstrap_port)]) + # Should not raise any validation errors + assert (url, bootstrap_port) in args.prefill_urls + + def test_valid_decode_urls(self): + """Test validation of valid decode URLs.""" + valid_decode_urls = [ + "http://decode1:8001", + "https://decode2:8001", + "http://localhost:8001", + "http://127.0.0.1:8001", + ] + + for url in valid_decode_urls: + args = RouterArgs(decode_urls=[url]) + # Should not raise any validation errors + assert url in args.decode_urls + + def test_malformed_urls(self): + """Test handling of malformed URLs.""" + # Note: The current implementation doesn't validate URL format + # This test documents the current behavior + malformed_urls = [ + "not-a-url", + "ftp://worker1:8000", # Wrong protocol + "http://", # Missing host + ":8000", # Missing protocol and host + "http://worker1", # Missing port + ] + + for url in malformed_urls: + args = RouterArgs(worker_urls=[url]) + # Currently, malformed URLs are accepted + # This might be something to improve in the future + assert url in args.worker_urls + + +class TestPortValidation: + """Test port validation logic.""" + + def test_valid_ports(self): + """Test validation of valid port numbers.""" + valid_ports = [1, 80, 8000, 30000, 65535] + + for port in valid_ports: + args = RouterArgs(port=port) + assert args.port == port + + def test_invalid_ports(self): + """Test handling of invalid port numbers.""" + # Note: The current implementation doesn't validate port ranges + # This test documents the current behavior + invalid_ports = [0, -1, 65536, 70000] + + for port in invalid_ports: + args = RouterArgs(port=port) + # Currently, invalid ports are accepted + # This might be something to improve in the future + assert args.port == port + + def test_bootstrap_port_validation(self): + """Test validation of bootstrap ports in PD mode.""" + valid_bootstrap_ports = [1, 80, 9000, 30000, 65535, None] + + for bootstrap_port in valid_bootstrap_ports: + args = RouterArgs(prefill_urls=[("http://prefill1:8000", bootstrap_port)]) + assert args.prefill_urls[0][1] == bootstrap_port + + +class TestParameterValidation: + """Test parameter validation logic.""" + + def test_cache_threshold_validation(self): + """Test cache threshold parameter validation.""" + # Valid cache thresholds + valid_thresholds = [0.0, 0.1, 0.5, 0.9, 1.0] + + for threshold in valid_thresholds: + args = RouterArgs(cache_threshold=threshold) + assert args.cache_threshold == threshold + + def test_balance_threshold_validation(self): + """Test load balancing threshold parameter validation.""" + # Valid absolute thresholds + valid_abs_thresholds = [0, 1, 32, 64, 128, 1000] + for threshold in valid_abs_thresholds: + args = RouterArgs(balance_abs_threshold=threshold) + assert args.balance_abs_threshold == threshold + + # Valid relative thresholds + valid_rel_thresholds = [1.0, 1.1, 1.5, 2.0, 10.0] + for threshold in valid_rel_thresholds: + args = RouterArgs(balance_rel_threshold=threshold) + assert args.balance_rel_threshold == threshold + + def test_timeout_validation(self): + """Test timeout parameter validation.""" + # Valid timeouts + valid_timeouts = [1, 30, 60, 300, 600, 1800, 3600] + + for timeout in valid_timeouts: + args = RouterArgs( + worker_startup_timeout_secs=timeout, + worker_startup_check_interval=timeout, + request_timeout_secs=timeout, + queue_timeout_secs=timeout, + ) + assert args.worker_startup_timeout_secs == timeout + assert args.worker_startup_check_interval == timeout + assert args.request_timeout_secs == timeout + assert args.queue_timeout_secs == timeout + + def test_retry_parameter_validation(self): + """Test retry parameter validation.""" + # Valid retry parameters + valid_retry_counts = [0, 1, 3, 5, 10] + for count in valid_retry_counts: + args = RouterArgs(retry_max_retries=count) + assert args.retry_max_retries == count + + # Valid backoff parameters + valid_backoff_ms = [1, 50, 100, 1000, 30000] + for backoff in valid_backoff_ms: + args = RouterArgs( + retry_initial_backoff_ms=backoff, retry_max_backoff_ms=backoff + ) + assert args.retry_initial_backoff_ms == backoff + assert args.retry_max_backoff_ms == backoff + + # Valid multiplier parameters + valid_multipliers = [1.0, 1.5, 2.0, 3.0] + for multiplier in valid_multipliers: + args = RouterArgs(retry_backoff_multiplier=multiplier) + assert args.retry_backoff_multiplier == multiplier + + # Valid jitter parameters + valid_jitter = [0.0, 0.1, 0.2, 0.5] + for jitter in valid_jitter: + args = RouterArgs(retry_jitter_factor=jitter) + assert args.retry_jitter_factor == jitter + + def test_circuit_breaker_parameter_validation(self): + """Test circuit breaker parameter validation.""" + # Valid failure thresholds + valid_failure_thresholds = [1, 3, 5, 10, 20] + for threshold in valid_failure_thresholds: + args = RouterArgs(cb_failure_threshold=threshold) + assert args.cb_failure_threshold == threshold + + # Valid success thresholds + valid_success_thresholds = [1, 2, 3, 5] + for threshold in valid_success_thresholds: + args = RouterArgs(cb_success_threshold=threshold) + assert args.cb_success_threshold == threshold + + # Valid timeout durations + valid_timeouts = [10, 30, 60, 120, 300] + for timeout in valid_timeouts: + args = RouterArgs( + cb_timeout_duration_secs=timeout, cb_window_duration_secs=timeout + ) + assert args.cb_timeout_duration_secs == timeout + assert args.cb_window_duration_secs == timeout + + def test_health_check_parameter_validation(self): + """Test health check parameter validation.""" + # Valid failure thresholds + valid_failure_thresholds = [1, 2, 3, 5, 10] + for threshold in valid_failure_thresholds: + args = RouterArgs(health_failure_threshold=threshold) + assert args.health_failure_threshold == threshold + + # Valid success thresholds + valid_success_thresholds = [1, 2, 3, 5] + for threshold in valid_success_thresholds: + args = RouterArgs(health_success_threshold=threshold) + assert args.health_success_threshold == threshold + + # Valid timeouts and intervals + valid_times = [1, 5, 10, 30, 60, 120] + for time_val in valid_times: + args = RouterArgs( + health_check_timeout_secs=time_val, health_check_interval_secs=time_val + ) + assert args.health_check_timeout_secs == time_val + assert args.health_check_interval_secs == time_val + + def test_rate_limiting_parameter_validation(self): + """Test rate limiting parameter validation.""" + # Valid concurrent request limits + valid_limits = [1, 10, 64, 256, 512, 1000] + for limit in valid_limits: + args = RouterArgs(max_concurrent_requests=limit) + assert args.max_concurrent_requests == limit + + # Valid queue sizes + valid_queue_sizes = [0, 10, 50, 100, 500, 1000] + for size in valid_queue_sizes: + args = RouterArgs(queue_size=size) + assert args.queue_size == size + + # Valid token rates + valid_rates = [1, 10, 50, 100, 500, 1000] + for rate in valid_rates: + args = RouterArgs(rate_limit_tokens_per_second=rate) + assert args.rate_limit_tokens_per_second == rate + + def test_tree_size_validation(self): + """Test tree size parameter validation.""" + # Valid tree sizes (powers of 2) + valid_sizes = [2**10, 2**20, 2**24, 2**26, 2**28, 2**30] + + for size in valid_sizes: + args = RouterArgs(max_tree_size=size) + assert args.max_tree_size == size + + def test_payload_size_validation(self): + """Test payload size parameter validation.""" + # Valid payload sizes + valid_sizes = [ + 1024, # 1KB + 1024 * 1024, # 1MB + 10 * 1024 * 1024, # 10MB + 100 * 1024 * 1024, # 100MB + 512 * 1024 * 1024, # 512MB + 1024 * 1024 * 1024, # 1GB + ] + + for size in valid_sizes: + args = RouterArgs(max_payload_size=size) + assert args.max_payload_size == size + + +class TestConfigurationValidation: + """Test configuration validation logic.""" + + def test_pd_mode_validation(self): + """Test PD mode configuration validation.""" + # Valid PD configuration + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[("http://prefill1:8000", 9000)], + decode_urls=["http://decode1:8001"], + ) + + assert args.pd_disaggregation is True + assert len(args.prefill_urls) > 0 + assert len(args.decode_urls) > 0 + + def test_service_discovery_validation(self): + """Test service discovery configuration validation.""" + # Valid service discovery configuration + args = RouterArgs( + service_discovery=True, + selector={"app": "worker", "env": "prod"}, + service_discovery_port=8080, + service_discovery_namespace="default", + ) + + assert args.service_discovery is True + assert args.selector == {"app": "worker", "env": "prod"} + assert args.service_discovery_port == 8080 + assert args.service_discovery_namespace == "default" + + def test_pd_service_discovery_validation(self): + """Test PD service discovery configuration validation.""" + # Valid PD service discovery configuration + args = RouterArgs( + pd_disaggregation=True, + service_discovery=True, + prefill_selector={"app": "prefill"}, + decode_selector={"app": "decode"}, + ) + + assert args.pd_disaggregation is True + assert args.service_discovery is True + assert args.prefill_selector == {"app": "prefill"} + assert args.decode_selector == {"app": "decode"} + + def test_policy_validation(self): + """Test policy configuration validation.""" + # Valid policies + valid_policies = ["random", "round_robin", "cache_aware", "power_of_two"] + + for policy in valid_policies: + args = RouterArgs(policy=policy) + assert args.policy == policy + + def test_pd_policy_validation(self): + """Test PD policy configuration validation.""" + # Valid PD policies + valid_policies = ["random", "round_robin", "cache_aware", "power_of_two"] + + for prefill_policy in valid_policies: + for decode_policy in valid_policies: + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[("http://prefill1:8000", None)], + decode_urls=["http://decode1:8001"], + prefill_policy=prefill_policy, + decode_policy=decode_policy, + ) + assert args.prefill_policy == prefill_policy + assert args.decode_policy == decode_policy + + def test_cors_validation(self): + """Test CORS configuration validation.""" + # Valid CORS origins + valid_origins = [ + [], + ["http://localhost:3000"], + ["https://example.com"], + ["http://localhost:3000", "https://example.com"], + ["*"], # Wildcard (if supported) + ] + + for origins in valid_origins: + args = RouterArgs(cors_allowed_origins=origins) + assert args.cors_allowed_origins == origins + + def test_logging_validation(self): + """Test logging configuration validation.""" + # Valid log levels + valid_log_levels = ["debug", "info", "warning", "error", "critical"] + + for level in valid_log_levels: + args = RouterArgs(log_level=level) + assert args.log_level == level + + def test_prometheus_validation(self): + """Test Prometheus configuration validation.""" + # Valid Prometheus configuration + args = RouterArgs(prometheus_port=29000, prometheus_host="127.0.0.1") + + assert args.prometheus_port == 29000 + assert args.prometheus_host == "127.0.0.1" + + def test_tokenizer_validation(self): + """Test tokenizer configuration validation.""" + # Note: model_path and tokenizer_path are not available in current RouterArgs + pytest.skip("Tokenizer configuration not available in current implementation") + + def test_request_id_headers_validation(self): + """Test request ID headers configuration validation.""" + # Valid request ID headers + valid_headers = [ + ["x-request-id"], + ["x-request-id", "x-trace-id"], + ["x-request-id", "x-trace-id", "x-correlation-id"], + ["custom-header"], + ] + + for headers in valid_headers: + args = RouterArgs(request_id_headers=headers) + assert args.request_id_headers == headers + + +class TestLaunchValidation: + """Test launch-time validation logic.""" + + def test_pd_mode_allows_empty_urls(self): + """Test that PD mode now allows empty URLs (URLs are optional).""" + # PD mode without URLs is now allowed + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[], + decode_urls=[], + service_discovery=False, + ) + + # Should not raise validation error - URLs are now optional + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + # This should succeed without raising an error + launch_router(args) + router_mod.from_args.assert_called_once() + + def test_pd_mode_with_service_discovery_allows_empty_urls(self): + """Test that PD mode with service discovery allows empty URLs.""" + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[], + decode_urls=[], + service_discovery=True, + ) + + # Should not raise validation error + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + launch_router(args) + + # Should create router instance via from_args + router_mod.from_args.assert_called_once() + + def test_regular_mode_allows_empty_worker_urls(self): + """Test that regular mode allows empty worker URLs.""" + args = RouterArgs(worker_urls=[], service_discovery=False) + + # Should not raise validation error + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + launch_router(args) + + # Should create router instance via from_args + router_mod.from_args.assert_called_once() + + def test_launch_with_valid_config(self): + """Test launching with valid configuration.""" + args = RouterArgs( + host="127.0.0.1", + port=30000, + worker_urls=["http://worker1:8000"], + policy="cache_aware", + ) + + # Should not raise validation error + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + launch_router(args) + + # Should create router instance via from_args + router_mod.from_args.assert_called_once() + + def test_launch_with_pd_config(self): + """Test launching with valid PD configuration.""" + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[("http://prefill1:8000", 9000)], + decode_urls=["http://decode1:8001"], + policy="cache_aware", + ) + + # Should not raise validation error + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + launch_router(args) + + # Should create router instance via from_args + router_mod.from_args.assert_called_once() + + def test_launch_with_service_discovery_config(self): + """Test launching with valid service discovery configuration.""" + args = RouterArgs( + service_discovery=True, + selector={"app": "worker"}, + service_discovery_port=8080, + ) + + # Should not raise validation error + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + launch_router(args) + + # Should create router instance via from_args + router_mod.from_args.assert_called_once() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/pytest.ini b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/pytest.ini new file mode 100644 index 00000000..c9f40075 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +testpaths = py_test +python_files = test_*.py +python_classes = Test* +python_functions = test_* diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/rustfmt.toml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/rustfmt.toml new file mode 100644 index 00000000..19e1ab31 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/rustfmt.toml @@ -0,0 +1,8 @@ +# Rust formatting configuration + +# Enforce grouped imports by crate +imports_granularity = "Crate" +# Group std, external crates, and local crate imports separately +group_imports = "StdExternalCrate" +reorder_imports = true +reorder_modules = true diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/scripts/generate_gateway_release_notes.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/scripts/generate_gateway_release_notes.sh new file mode 100755 index 00000000..bb7f6ba0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/scripts/generate_gateway_release_notes.sh @@ -0,0 +1,312 @@ +#!/bin/bash +# Generate release notes for SGLang Gateway/Router +# Only includes commits that affect gateway-related paths + +set -e + +# Configuration +GATEWAY_PATHS=( + "sgl-router" + "python/sglang/srt/grpc" + "python/sglang/srt/entrypoints/grpc_server.py" +) + +# Colors for output +GREEN='\033[0;32m' +BLUE='\033[0;34m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Function to display usage +usage() { + echo "Usage: $0 " + echo "" + echo "Example: $0 gateway-v0.2.2 gateway-v0.2.3" + echo "" + echo "Options:" + echo " -o, --output FILE Save output to file (default: stdout)" + echo " -f, --format FORMAT Output format: markdown|github|plain (default: markdown)" + echo " --create-release Create GitHub release using gh CLI (default: draft)" + echo " --draft Create as draft release (default when using --create-release)" + echo " --no-draft Publish release immediately (skip draft)" + echo " -h, --help Show this help message" + exit 1 +} + +# Parse arguments +OUTPUT_FILE="" +FORMAT="markdown" +CREATE_RELEASE=false +DRAFT_RELEASE="default" # Default to draft unless explicitly disabled +PREV_TAG="" +CURR_TAG="" + +while [[ $# -gt 0 ]]; do + case $1 in + -o|--output) + OUTPUT_FILE="$2" + shift 2 + ;; + -f|--format) + FORMAT="$2" + shift 2 + ;; + --create-release) + CREATE_RELEASE=true + shift + ;; + --draft) + DRAFT_RELEASE=true + shift + ;; + --no-draft) + DRAFT_RELEASE=false + shift + ;; + -h|--help) + usage + ;; + *) + if [[ -z "$PREV_TAG" ]]; then + PREV_TAG="$1" + elif [[ -z "$CURR_TAG" ]]; then + CURR_TAG="$1" + else + echo "Error: Too many arguments" + usage + fi + shift + ;; + esac +done + +# Validate arguments +if [[ -z "$PREV_TAG" ]] || [[ -z "$CURR_TAG" ]]; then + echo "Error: Both previous and current tags are required" + usage +fi + +# Navigate to repo root (main sglang repo, not sgl-router) +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +cd "$REPO_ROOT" + +echo -e "${BLUE}Generating Gateway/Router release notes...${NC}" >&2 +echo -e "${BLUE}Previous: $PREV_TAG${NC}" >&2 +echo -e "${BLUE}Current: $CURR_TAG${NC}" >&2 +echo "" >&2 + +# Verify tags exist +if ! git rev-parse "$PREV_TAG" >/dev/null 2>&1; then + echo -e "${YELLOW}Warning: Tag $PREV_TAG not found, using initial commit${NC}" >&2 + PREV_TAG=$(git rev-list --max-parents=0 HEAD) +fi + +if ! git rev-parse "$CURR_TAG" >/dev/null 2>&1; then + echo -e "${YELLOW}Warning: Tag $CURR_TAG not found, using HEAD${NC}" >&2 + CURR_TAG="HEAD" +fi + +# Build path filter arguments +PATH_ARGS=() +for path in "${GATEWAY_PATHS[@]}"; do + PATH_ARGS+=("--" "$path") +done + +# Get filtered commit list +COMMITS=$(git log "$PREV_TAG..$CURR_TAG" --oneline --no-merges "${PATH_ARGS[@]}") + +if [[ -z "$COMMITS" ]]; then + echo -e "${YELLOW}No commits found for gateway paths between $PREV_TAG and $CURR_TAG${NC}" >&2 + exit 0 +fi + +COMMIT_COUNT=$(echo "$COMMITS" | wc -l | tr -d ' ') +echo -e "${GREEN}Found $COMMIT_COUNT gateway-related commits${NC}" >&2 + +# Get contributors +echo -e "${BLUE}Analyzing contributors...${NC}" >&2 + +# Get all contributors in this release (with commit count) +CONTRIBUTORS=$(git log "$PREV_TAG..$CURR_TAG" --format='%aN <%aE>' --no-merges "${PATH_ARGS[@]}" | sort | uniq -c | sort -rn) + +# Get all contributors before this release (from initial commit up to PREV_TAG) +# Using $(git rev-list --max-parents=0 HEAD) to get initial commit ensures we check entire history +INITIAL_COMMIT=$(git rev-list --max-parents=0 HEAD | tail -1) +PREV_CONTRIBUTORS=$(git log "$INITIAL_COMMIT..$PREV_TAG" --format='%aN <%aE>' --no-merges "${PATH_ARGS[@]}" | sort | uniq) + +# Find new contributors +NEW_CONTRIBUTORS="" +while IFS= read -r line; do + contributor=$(echo "$line" | sed 's/^[[:space:]]*[0-9]*[[:space:]]*//') + if [[ -n "$contributor" ]] && ! echo "$PREV_CONTRIBUTORS" | grep -Fxq "$contributor"; then + NEW_CONTRIBUTORS="$NEW_CONTRIBUTORS$contributor"$'\n' + fi +done <<< "$CONTRIBUTORS" + +CONTRIBUTOR_COUNT=$(echo "$CONTRIBUTORS" | grep -c '^' || echo 0) +NEW_CONTRIBUTOR_COUNT=$(echo "$NEW_CONTRIBUTORS" | grep -c '^' || echo 0) + +echo -e "${GREEN}Found $CONTRIBUTOR_COUNT contributors ($NEW_CONTRIBUTOR_COUNT new)${NC}" >&2 +echo "" >&2 + +# Generate release notes based on format +generate_notes() { + case $FORMAT in + markdown|github) + echo "## What's Changed in Gateway" + echo "" + echo "### Gateway Changes ($COMMIT_COUNT commits)" + echo "" + + # Categorize commits with author attribution + echo "$COMMITS" | while IFS= read -r line; do + commit_hash=$(echo "$line" | awk '{print $1}') + commit_msg=$(echo "$line" | cut -d' ' -f2-) + + # Get PR number from commit message + pr_num=$(echo "$commit_msg" | grep -o '(#[0-9]*' | grep -o '[0-9]*' | head -1) + + # Try to get GitHub username from PR if gh CLI is available + gh_user="" + if [[ -n "$pr_num" ]] && command -v gh &> /dev/null; then + gh_user=$(gh pr view "$pr_num" --json author --jq '.author.login' 2>/dev/null || echo "") + fi + + # Fallback: try to extract from email (works for users.noreply.github.com emails) + if [[ -z "$gh_user" ]]; then + email=$(git show -s --format='%aE' "$commit_hash") + gh_user=$(echo "$email" | sed 's/@users\.noreply\.github\.com$//' | sed 's/^[0-9]*+//') + # If still contains @, it's not a GitHub username + if [[ "$gh_user" == *"@"* ]]; then + gh_user="" + fi + fi + + # Format author link + if [[ -n "$gh_user" ]]; then + author_link="by @$gh_user" + else + # Final fallback: use full name + author=$(git show -s --format='%aN' "$commit_hash") + author_link="by $author" + fi + + # Format PR link + if [[ -n "$pr_num" ]]; then + pr_link="in https://github.com/sgl-project/sglang/pull/$pr_num" + else + pr_link="" + fi + + echo "- $commit_msg $author_link $pr_link" + done + + # New Contributors section + if [[ -n "$NEW_CONTRIBUTORS" ]] && [[ "$NEW_CONTRIBUTOR_COUNT" -gt 0 ]]; then + echo "" + echo "### New Contributors" + echo "" + while IFS= read -r contributor; do + if [[ -n "$contributor" ]]; then + # Extract name and email + name=$(echo "$contributor" | sed 's/ <.*//') + email=$(echo "$contributor" | sed 's/.*<\(.*\)>/\1/') + + # Get their first commit + first_commit=$(git log "$PREV_TAG..$CURR_TAG" --author="$contributor" --format='%h' --reverse --no-merges "${PATH_ARGS[@]}" | head -1) + + # Try to get GitHub username from first commit's PR + gh_user="" + if command -v gh &> /dev/null; then + commit_msg=$(git log --format=%s -n 1 "$first_commit") + pr_num=$(echo "$commit_msg" | grep -o '(#[0-9]*' | grep -o '[0-9]*' | head -1) + if [[ -n "$pr_num" ]]; then + gh_user=$(gh pr view "$pr_num" --json author --jq '.author.login' 2>/dev/null || echo "") + fi + fi + + # Fallback: try to get GitHub username from email + if [[ -z "$gh_user" ]]; then + gh_user=$(echo "$email" | sed 's/@users\.noreply\.github\.com$//' | sed 's/^[0-9]*+//') + # If still contains @, it's not a GitHub username + if [[ "$gh_user" == *"@"* ]]; then + gh_user="" + fi + fi + + if [[ -n "$gh_user" ]]; then + echo "* @$gh_user made their first contribution in https://github.com/sgl-project/sglang/commit/$first_commit" + else + echo "* $name made their first contribution in https://github.com/sgl-project/sglang/commit/$first_commit" + fi + fi + done <<< "$NEW_CONTRIBUTORS" + fi + + echo "" + echo "### Paths Included" + echo "" + for path in "${GATEWAY_PATHS[@]}"; do + echo "- \`$path\`" + done + echo "" + echo "**Full Changelog**: https://github.com/sgl-project/sglang/compare/$PREV_TAG...$CURR_TAG" + ;; + plain) + echo "Gateway/Router Release Notes: $CURR_TAG" + echo "==========================================" + echo "" + echo "$COMMITS" + echo "" + echo "Contributors: $CONTRIBUTOR_COUNT ($NEW_CONTRIBUTOR_COUNT new)" + ;; + esac +} + +# Output release notes +if [[ -n "$OUTPUT_FILE" ]]; then + generate_notes > "$OUTPUT_FILE" + echo -e "${GREEN}Release notes saved to: $OUTPUT_FILE${NC}" >&2 +else + generate_notes +fi + +# Create GitHub release if requested +if [[ "$CREATE_RELEASE" == true ]]; then + if ! command -v gh &> /dev/null; then + echo -e "${YELLOW}Error: gh CLI not found. Install from https://cli.github.com/${NC}" >&2 + exit 1 + fi + + NOTES_FILE=$(mktemp) + generate_notes > "$NOTES_FILE" + + # Default to draft if not explicitly set to false + if [[ "$DRAFT_RELEASE" == "default" ]]; then + DRAFT_RELEASE=true + fi + + echo "" >&2 + if [[ "$DRAFT_RELEASE" == true ]]; then + echo -e "${BLUE}Creating GitHub DRAFT release...${NC}" >&2 + else + echo -e "${BLUE}Creating GitHub release (publishing immediately)...${NC}" >&2 + fi + + # Build gh command with optional --draft flag + GH_ARGS=("$CURR_TAG" --title "Gateway/Router $CURR_TAG" --notes-file "$NOTES_FILE" --repo sgl-project/sglang) + if [[ "$DRAFT_RELEASE" == true ]]; then + GH_ARGS+=(--draft) + fi + + gh release create "${GH_ARGS[@]}" + + rm -f "$NOTES_FILE" + if [[ "$DRAFT_RELEASE" == true ]]; then + echo -e "${GREEN}Draft release created successfully!${NC}" >&2 + echo -e "${YELLOW}Visit https://github.com/sgl-project/sglang/releases to review and publish${NC}" >&2 + else + echo -e "${GREEN}Release published successfully!${NC}" >&2 + fi +fi diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/scripts/run_benchmarks.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/scripts/run_benchmarks.py new file mode 100755 index 00000000..a7ece9d9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/scripts/run_benchmarks.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +""" +SGLang Router Benchmark Runner + +A Python script to run Rust benchmarks with various options and modes. +Replaces the shell script for better maintainability and integration. +""" + +import argparse +import os +import subprocess +import sys +import time +from pathlib import Path +from typing import Dict, List, Optional + + +class BenchmarkRunner: + """Handles running Rust benchmarks for the SGLang router.""" + + def __init__(self, project_root: str): + self.project_root = Path(project_root) + self.timestamp = time.strftime("%a %b %d %H:%M:%S UTC %Y", time.gmtime()) + + def run_command( + self, cmd: List[str], capture_output: bool = False + ) -> subprocess.CompletedProcess: + """Run a command and handle errors.""" + try: + if capture_output: + result = subprocess.run( + cmd, capture_output=True, text=True, cwd=self.project_root + ) + else: + result = subprocess.run(cmd, cwd=self.project_root) + return result + except subprocess.CalledProcessError as e: + print(f"Error running command: {' '.join(cmd)}") + print(f"Exit code: {e.returncode}") + sys.exit(1) + + def print_header(self): + """Print the benchmark runner header.""" + print("SGLang Router Benchmark Runner") + print("=" * 30) + print(f"Project: {self.project_root.absolute()}") + print(f"Timestamp: {self.timestamp}") + print() + + def build_release(self): + """Build the project in release mode.""" + print("Building in release mode...") + result = self.run_command(["cargo", "build", "--release", "--quiet"]) + if result.returncode != 0: + print("Failed to build in release mode") + sys.exit(1) + + def run_benchmarks( + self, + quick_mode: bool = False, + save_baseline: Optional[str] = None, + compare_baseline: Optional[str] = None, + ) -> str: + """Run benchmarks with specified options.""" + bench_args = ["cargo", "bench", "--bench", "request_processing"] + + if quick_mode: + bench_args.append("benchmark_summary") + print("Running quick benchmarks...") + else: + print("Running full benchmark suite...") + + # Note: Criterion baselines are handled via target directory structure + # For now, we'll implement baseline functionality via file copying + if save_baseline: + print(f"Will save results as baseline: {save_baseline}") + + if compare_baseline: + print(f"Will compare with baseline: {compare_baseline}") + + print(f"Executing: {' '.join(bench_args)}") + result = self.run_command(bench_args, capture_output=True) + + if result.returncode != 0: + print("Benchmark execution failed!") + print("STDOUT:", result.stdout) + print("STDERR:", result.stderr) + sys.exit(1) + + # Handle baseline saving after successful run + if save_baseline: + self._save_baseline(save_baseline, result.stdout) + + return result.stdout + + def _save_baseline(self, filename: str, output: str): + """Save benchmark results to a file as baseline.""" + filepath = self.project_root / filename + with open(filepath, "w") as f: + f.write(output) + print(f"Baseline saved to: {filepath}") + + def parse_benchmark_results(self, output: str) -> Dict[str, str]: + """Parse benchmark output to extract performance metrics.""" + results = {} + + # Look for performance overview section + lines = output.split("\n") + parsing_overview = False + + for line in lines: + line = line.strip() + + if "Quick Performance Overview:" in line: + parsing_overview = True + continue + + if parsing_overview and line.startswith("* "): + # Parse lines like "* Serialization (avg): 481 ns/req" + if "Serialization (avg):" in line: + results["serialization_time"] = self._extract_time(line) + elif "Deserialization (avg):" in line: + results["deserialization_time"] = self._extract_time(line) + elif "Bootstrap Injection (avg):" in line: + results["bootstrap_injection_time"] = self._extract_time(line) + elif "Total Pipeline (avg):" in line: + results["total_time"] = self._extract_time(line) + + # Stop parsing after the overview section + if parsing_overview and line.startswith("Performance Insights:"): + break + + return results + + def _extract_time(self, line: str) -> str: + """Extract time value from a benchmark line.""" + # Extract number followed by ns/req + import re + + match = re.search(r"(\d+)\s*ns/req", line) + return match.group(1) if match else "N/A" + + def validate_thresholds(self, results: Dict[str, str]) -> bool: + """Validate benchmark results against performance thresholds.""" + thresholds = { + "serialization_time": 2000, # 2μs max + "deserialization_time": 2000, # 2μs max + "bootstrap_injection_time": 5000, # 5μs max + "total_time": 10000, # 10μs max + } + + all_passed = True + print("\nPerformance Threshold Validation:") + print("=" * 35) + + for metric, threshold in thresholds.items(): + if metric in results and results[metric] != "N/A": + try: + value = int(results[metric]) + passed = value <= threshold + status = "✓ PASS" if passed else "✗ FAIL" + print(f"{metric:20}: {value:>6}ns <= {threshold:>6}ns {status}") + if not passed: + all_passed = False + except ValueError: + print(f"{metric:20}: Invalid value: {results[metric]}") + all_passed = False + else: + print(f"{metric:20}: No data available") + all_passed = False + + print() + if all_passed: + print("All performance thresholds passed!") + else: + print("Some performance thresholds failed!") + + return all_passed + + def save_results_to_file( + self, results: Dict[str, str], filename: str = "benchmark_results.env" + ): + """Save benchmark results to a file for CI consumption.""" + filepath = self.project_root / filename + with open(filepath, "w") as f: + for key, value in results.items(): + f.write(f"{key}={value}\n") + print(f"Results saved to: {filepath}") + + +def main(): + parser = argparse.ArgumentParser(description="Run SGLang router benchmarks") + parser.add_argument( + "--quick", action="store_true", help="Run quick benchmarks (summary only)" + ) + parser.add_argument( + "--save-baseline", type=str, help="Save benchmark results as baseline" + ) + parser.add_argument( + "--compare-baseline", type=str, help="Compare with saved baseline" + ) + parser.add_argument( + "--validate-thresholds", + action="store_true", + help="Validate results against performance thresholds", + ) + parser.add_argument( + "--save-results", action="store_true", help="Save results to file for CI" + ) + + args = parser.parse_args() + + # Determine project root (script is in scripts/ subdirectory) + script_dir = Path(__file__).parent + project_root = script_dir.parent + + runner = BenchmarkRunner(str(project_root)) + runner.print_header() + + # Build in release mode + runner.build_release() + + # Run benchmarks + output = runner.run_benchmarks( + quick_mode=args.quick, + save_baseline=args.save_baseline, + compare_baseline=args.compare_baseline, + ) + + # Print the raw output + print(output) + + # Parse and validate results if requested + if args.validate_thresholds or args.save_results: + results = runner.parse_benchmark_results(output) + + if args.save_results: + runner.save_results_to_file(results) + + if args.validate_thresholds: + passed = runner.validate_thresholds(results) + if not passed: + print("Validation failed - performance regression detected!") + sys.exit(1) + + print("\nBenchmark run completed successfully!") + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/scripts/setup-sccache.sh b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/scripts/setup-sccache.sh new file mode 100755 index 00000000..2cdfb1e5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/scripts/setup-sccache.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +IFS=$'\n\t' + +echo "Setting up sccache for faster Rust compilation..." + +has_cmd() { command -v "$1" >/dev/null 2>&1; } + +install_sccache() { + echo "sccache not found." + if [[ "${AUTO_INSTALL:-0}" != "1" ]]; then + read -r -p "Install sccache now? [y/N] " response + response=${response:-N} + if [[ ! "$response" =~ ^[Yy]$ ]]; then + echo "Skipping installation. Please install sccache manually:" + echo " cargo install sccache" + echo " or" + echo " brew install sccache (macOS)" + echo " or" + echo " sudo apt-get install -y sccache (Debian/Ubuntu)" + echo " or" + echo " sudo dnf install -y sccache (RHEL/Fedora)" + echo " or" + echo " sudo pacman -S sccache (Arch)" + exit 0 + fi + fi + + if has_cmd cargo; then + echo "Installing via cargo..." + cargo install sccache --locked + elif has_cmd brew; then + echo "Installing via Homebrew..." + brew install sccache + elif has_cmd apt-get; then + echo "Installing via apt-get..." + sudo apt-get update -y && sudo apt-get install -y sccache + elif has_cmd dnf; then + echo "Installing via dnf..." + sudo dnf install -y sccache + elif has_cmd pacman; then + echo "Installing via pacman..." + sudo pacman -S --noconfirm sccache + else + echo "No supported package manager detected. Install manually:" + echo " cargo install sccache" + exit 1 + fi +} + +if ! has_cmd sccache; then + install_sccache +fi + +echo "Configuring sccache..." + +export SCCACHE_CACHE_SIZE="${SCCACHE_CACHE_SIZE:-10G}" +export SCCACHE_STATS="${SCCACHE_STATS:-1}" + +# Set RUSTC_WRAPPER to sccache for this shell session. +SCCACHE_BIN="$(command -v sccache)" +if [[ -z "${SCCACHE_BIN}" ]]; then + echo "Unexpected: sccache still not on PATH after install. Check your environment." + exit 1 +fi +export RUSTC_WRAPPER="${SCCACHE_BIN}" + +echo "sccache version: $(sccache --version || echo 'unknown')" +echo "Current cache stats:" +sccache -s || true + +# If script not sourced, remind user about persistence. +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + echo + echo "Environment variables exported for this process only." + echo "To persist, add to your shell profile (e.g., ~/.bashrc or ~/.zshrc):" + echo ' export RUSTC_WRAPPER="$(command -v sccache 2>/dev/null || echo "")"' + echo ' export SCCACHE_CACHE_SIZE="10G"' + # echo ' export SCCACHE_DIR="$HOME/.cache/sccache"' + echo ' export SCCACHE_STATS="1"' +fi + +echo "sccache is configured." diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/app_context.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/app_context.rs new file mode 100644 index 00000000..31e3161c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/app_context.rs @@ -0,0 +1,514 @@ +use std::{ + sync::{Arc, OnceLock}, + time::Duration, +}; + +use reqwest::Client; +use tracing::info; + +use crate::{ + config::RouterConfig, + core::{workflow::WorkflowEngine, ConnectionMode, JobQueue, LoadMonitor, WorkerRegistry}, + data_connector::{ + create_storage, ConversationItemStorage, ConversationStorage, ResponseStorage, + }, + mcp::McpManager, + middleware::TokenBucket, + policies::PolicyRegistry, + reasoning_parser::ParserFactory as ReasoningParserFactory, + routers::router_manager::RouterManager, + tokenizer::{ + cache::{CacheConfig, CachedTokenizer}, + factory as tokenizer_factory, + traits::Tokenizer, + }, + tool_parser::ParserFactory as ToolParserFactory, +}; + +/// Error type for AppContext builder +#[derive(Debug)] +pub struct AppContextBuildError(&'static str); + +impl std::fmt::Display for AppContextBuildError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Missing required field: {}", self.0) + } +} + +impl std::error::Error for AppContextBuildError {} + +#[derive(Clone)] +pub struct AppContext { + pub client: Client, + pub router_config: RouterConfig, + pub rate_limiter: Option>, + pub tokenizer: Option>, + pub reasoning_parser_factory: Option, + pub tool_parser_factory: Option, + pub worker_registry: Arc, + pub policy_registry: Arc, + pub router_manager: Option>, + pub response_storage: Arc, + pub conversation_storage: Arc, + pub conversation_item_storage: Arc, + pub load_monitor: Option>, + pub configured_reasoning_parser: Option, + pub configured_tool_parser: Option, + pub worker_job_queue: Arc>>, + pub workflow_engine: Arc>>, + pub mcp_manager: Arc>>, +} + +pub struct AppContextBuilder { + client: Option, + router_config: Option, + rate_limiter: Option>, + tokenizer: Option>, + reasoning_parser_factory: Option, + tool_parser_factory: Option, + worker_registry: Option>, + policy_registry: Option>, + router_manager: Option>, + response_storage: Option>, + conversation_storage: Option>, + conversation_item_storage: Option>, + load_monitor: Option>, + worker_job_queue: Option>>>, + workflow_engine: Option>>>, + mcp_manager: Option>>>, +} + +impl AppContext { + pub fn builder() -> AppContextBuilder { + AppContextBuilder::new() + } + + /// Create AppContext from config with all components initialized + /// This is the main entry point that replaces ~194 lines of initialization in server.rs + pub async fn from_config( + router_config: RouterConfig, + request_timeout_secs: u64, + ) -> Result { + AppContextBuilder::from_config(router_config, request_timeout_secs) + .await? + .build() + .map_err(|e| e.to_string()) + } +} + +impl AppContextBuilder { + pub fn new() -> Self { + Self { + client: None, + router_config: None, + rate_limiter: None, + tokenizer: None, + reasoning_parser_factory: None, + tool_parser_factory: None, + worker_registry: None, + policy_registry: None, + router_manager: None, + response_storage: None, + conversation_storage: None, + conversation_item_storage: None, + load_monitor: None, + worker_job_queue: None, + workflow_engine: None, + mcp_manager: None, + } + } + + pub fn client(mut self, client: Client) -> Self { + self.client = Some(client); + self + } + + pub fn router_config(mut self, router_config: RouterConfig) -> Self { + self.router_config = Some(router_config); + self + } + + pub fn rate_limiter(mut self, rate_limiter: Option>) -> Self { + self.rate_limiter = rate_limiter; + self + } + + pub fn tokenizer(mut self, tokenizer: Option>) -> Self { + self.tokenizer = tokenizer; + self + } + + pub fn reasoning_parser_factory( + mut self, + reasoning_parser_factory: Option, + ) -> Self { + self.reasoning_parser_factory = reasoning_parser_factory; + self + } + + pub fn tool_parser_factory(mut self, tool_parser_factory: Option) -> Self { + self.tool_parser_factory = tool_parser_factory; + self + } + + pub fn worker_registry(mut self, worker_registry: Arc) -> Self { + self.worker_registry = Some(worker_registry); + self + } + + pub fn policy_registry(mut self, policy_registry: Arc) -> Self { + self.policy_registry = Some(policy_registry); + self + } + + pub fn router_manager(mut self, router_manager: Option>) -> Self { + self.router_manager = router_manager; + self + } + + pub fn response_storage(mut self, response_storage: Arc) -> Self { + self.response_storage = Some(response_storage); + self + } + + pub fn conversation_storage( + mut self, + conversation_storage: Arc, + ) -> Self { + self.conversation_storage = Some(conversation_storage); + self + } + + pub fn conversation_item_storage( + mut self, + conversation_item_storage: Arc, + ) -> Self { + self.conversation_item_storage = Some(conversation_item_storage); + self + } + + pub fn load_monitor(mut self, load_monitor: Option>) -> Self { + self.load_monitor = load_monitor; + self + } + + pub fn worker_job_queue(mut self, worker_job_queue: Arc>>) -> Self { + self.worker_job_queue = Some(worker_job_queue); + self + } + + pub fn workflow_engine(mut self, workflow_engine: Arc>>) -> Self { + self.workflow_engine = Some(workflow_engine); + self + } + + pub fn mcp_manager(mut self, mcp_manager: Arc>>) -> Self { + self.mcp_manager = Some(mcp_manager); + self + } + + pub fn build(self) -> Result { + let router_config = self + .router_config + .ok_or(AppContextBuildError("router_config"))?; + let configured_reasoning_parser = router_config.reasoning_parser.clone(); + let configured_tool_parser = router_config.tool_call_parser.clone(); + + Ok(AppContext { + client: self.client.ok_or(AppContextBuildError("client"))?, + router_config, + rate_limiter: self.rate_limiter, + tokenizer: self.tokenizer, + reasoning_parser_factory: self.reasoning_parser_factory, + tool_parser_factory: self.tool_parser_factory, + worker_registry: self + .worker_registry + .ok_or(AppContextBuildError("worker_registry"))?, + policy_registry: self + .policy_registry + .ok_or(AppContextBuildError("policy_registry"))?, + router_manager: self.router_manager, + response_storage: self + .response_storage + .ok_or(AppContextBuildError("response_storage"))?, + conversation_storage: self + .conversation_storage + .ok_or(AppContextBuildError("conversation_storage"))?, + conversation_item_storage: self + .conversation_item_storage + .ok_or(AppContextBuildError("conversation_item_storage"))?, + load_monitor: self.load_monitor, + configured_reasoning_parser, + configured_tool_parser, + worker_job_queue: self + .worker_job_queue + .ok_or(AppContextBuildError("worker_job_queue"))?, + workflow_engine: self + .workflow_engine + .ok_or(AppContextBuildError("workflow_engine"))?, + mcp_manager: self + .mcp_manager + .ok_or(AppContextBuildError("mcp_manager"))?, + }) + } + + /// Initialize AppContext from config - creates ALL components + /// This replaces ~194 lines of initialization logic from server.rs + pub async fn from_config( + router_config: RouterConfig, + request_timeout_secs: u64, + ) -> Result { + Ok(Self::new() + .with_client(&router_config, request_timeout_secs)? + .maybe_rate_limiter(&router_config) + .maybe_tokenizer(&router_config)? + .maybe_reasoning_parser_factory(&router_config) + .maybe_tool_parser_factory(&router_config) + .with_worker_registry() + .with_policy_registry(&router_config) + .with_storage(&router_config)? + .with_load_monitor(&router_config) + .with_worker_job_queue() + .with_workflow_engine() + .with_mcp_manager(&router_config) + .await? + .router_config(router_config)) + } + + /// Create HTTP client with TLS/mTLS configuration + fn with_client(mut self, config: &RouterConfig, timeout_secs: u64) -> Result { + // FIXME: Current implementation creates a single HTTP client for all workers. + // This works well for single security domain deployments where all workers share + // the same CA and can accept the same client certificate. + // + // For multi-domain deployments (e.g., different model families with different CAs), + // this architecture needs significant refactoring: + // 1. Move client creation into worker registration workflow (per-worker clients) + // 2. Store client per worker in WorkerRegistry + // 3. Update PDRouter and other routers to fetch client from worker + // 4. Add per-worker TLS spec in WorkerConfigRequest + // + // Current single-domain approach is sufficient for most deployments. + // + // Use rustls TLS backend when TLS/mTLS is configured (client cert or CA certs provided). + // This ensures proper PKCS#8 key format support. For plain HTTP workers, use default + // backend to avoid unnecessary TLS initialization overhead. + let has_tls_config = config.client_identity.is_some() || !config.ca_certificates.is_empty(); + + let mut client_builder = Client::builder() + .pool_idle_timeout(Some(Duration::from_secs(50))) + .pool_max_idle_per_host(500) + .timeout(Duration::from_secs(timeout_secs)) + .connect_timeout(Duration::from_secs(10)) + .tcp_nodelay(true) + .tcp_keepalive(Some(Duration::from_secs(30))); + + // Force rustls backend when TLS is configured + if has_tls_config { + client_builder = client_builder.use_rustls_tls(); + info!("Using rustls TLS backend for TLS/mTLS connections"); + } + + // Configure mTLS client identity if provided (certificates already loaded during config creation) + if let Some(identity_pem) = &config.client_identity { + let identity = reqwest::Identity::from_pem(identity_pem) + .map_err(|e| format!("Failed to create client identity: {}", e))?; + client_builder = client_builder.identity(identity); + info!("mTLS client authentication enabled"); + } + + // Add CA certificates for verifying worker TLS (certificates already loaded during config creation) + for ca_cert in &config.ca_certificates { + let cert = reqwest::Certificate::from_pem(ca_cert) + .map_err(|e| format!("Failed to add CA certificate: {}", e))?; + client_builder = client_builder.add_root_certificate(cert); + } + if !config.ca_certificates.is_empty() { + info!( + "Added {} CA certificate(s) for worker verification", + config.ca_certificates.len() + ); + } + + let client = client_builder + .build() + .map_err(|e| format!("Failed to create HTTP client: {}", e))?; + + self.client = Some(client); + Ok(self) + } + + /// Create rate limiter based on config + fn maybe_rate_limiter(mut self, config: &RouterConfig) -> Self { + self.rate_limiter = match config.max_concurrent_requests { + n if n <= 0 => None, + n => { + let rate_limit_tokens = config + .rate_limit_tokens_per_second + .filter(|&t| t > 0) + .unwrap_or(n); + Some(Arc::new(TokenBucket::new( + n as usize, + rate_limit_tokens as usize, + ))) + } + }; + self + } + + /// Create tokenizer for gRPC mode + fn maybe_tokenizer(mut self, config: &RouterConfig) -> Result { + if matches!(config.connection_mode, ConnectionMode::Grpc { .. }) { + let tokenizer_path = config + .tokenizer_path + .clone() + .or_else(|| config.model_path.clone()) + .ok_or_else(|| { + "gRPC mode requires either --tokenizer-path or --model-path to be specified" + .to_string() + })?; + + let base_tokenizer = tokenizer_factory::create_tokenizer_with_chat_template_blocking( + &tokenizer_path, + config.chat_template.as_deref(), + ) + .map_err(|e| { + format!( + "Failed to create tokenizer from '{}': {}. \ + Ensure the path is valid and points to a tokenizer file (tokenizer.json) \ + or a HuggingFace model ID. For directories, ensure they contain tokenizer files.", + tokenizer_path, e + ) + })?; + + // Conditionally wrap with caching layer if at least one cache is enabled + self.tokenizer = if config.tokenizer_cache.enable_l0 || config.tokenizer_cache.enable_l1 + { + let cache_config = CacheConfig { + enable_l0: config.tokenizer_cache.enable_l0, + l0_max_entries: config.tokenizer_cache.l0_max_entries, + enable_l1: config.tokenizer_cache.enable_l1, + l1_max_memory: config.tokenizer_cache.l1_max_memory, + }; + Some(Arc::new(CachedTokenizer::new(base_tokenizer, cache_config)) + as Arc) + } else { + // Use base tokenizer directly without caching + Some(base_tokenizer) + }; + } + + Ok(self) + } + + /// Create reasoning parser factory for gRPC mode + fn maybe_reasoning_parser_factory(mut self, config: &RouterConfig) -> Self { + if matches!(config.connection_mode, ConnectionMode::Grpc { .. }) { + self.reasoning_parser_factory = Some(ReasoningParserFactory::new()); + } + self + } + + /// Create tool parser factory for gRPC mode + fn maybe_tool_parser_factory(mut self, config: &RouterConfig) -> Self { + if matches!(config.connection_mode, ConnectionMode::Grpc { .. }) { + self.tool_parser_factory = Some(ToolParserFactory::new()); + } + self + } + + /// Create worker registry + fn with_worker_registry(mut self) -> Self { + self.worker_registry = Some(Arc::new(WorkerRegistry::new())); + self + } + + /// Create policy registry + fn with_policy_registry(mut self, config: &RouterConfig) -> Self { + self.policy_registry = Some(Arc::new(PolicyRegistry::new(config.policy.clone()))); + self + } + + /// Create all storage backends using the factory function + fn with_storage(mut self, config: &RouterConfig) -> Result { + let (response_storage, conversation_storage, conversation_item_storage) = + create_storage(config)?; + + self.response_storage = Some(response_storage); + self.conversation_storage = Some(conversation_storage); + self.conversation_item_storage = Some(conversation_item_storage); + + Ok(self) + } + + /// Create load monitor + fn with_load_monitor(mut self, config: &RouterConfig) -> Self { + let client = self + .client + .as_ref() + .expect("client must be set before load monitor"); + self.load_monitor = Some(Arc::new(LoadMonitor::new( + self.worker_registry + .as_ref() + .expect("worker_registry must be set") + .clone(), + self.policy_registry + .as_ref() + .expect("policy_registry must be set") + .clone(), + client.clone(), + config.worker_startup_check_interval_secs, + ))); + self + } + + /// Create worker job queue OnceLock container + fn with_worker_job_queue(mut self) -> Self { + self.worker_job_queue = Some(Arc::new(OnceLock::new())); + self + } + + /// Create workflow engine OnceLock container + fn with_workflow_engine(mut self) -> Self { + self.workflow_engine = Some(Arc::new(OnceLock::new())); + self + } + + /// Create and initialize MCP manager with empty config + /// + /// This initializes the MCP manager with an empty config and default settings. + /// MCP servers will be registered later via the InitializeMcpServers job. + async fn with_mcp_manager(mut self, _router_config: &RouterConfig) -> Result { + // Create OnceLock container + let mcp_manager_lock = Arc::new(OnceLock::new()); + + // Always create with empty config and defaults + info!("Initializing MCP manager with empty config and default settings (5 min TTL, 100 max connections)"); + + let empty_config = crate::mcp::McpConfig { + servers: Vec::new(), + pool: Default::default(), + proxy: None, + warmup: Vec::new(), + inventory: Default::default(), + }; + + let manager = McpManager::with_defaults(empty_config) + .await + .map_err(|e| format!("Failed to initialize MCP manager with defaults: {}", e))?; + + // Store the initialized manager in the OnceLock + mcp_manager_lock + .set(Arc::new(manager)) + .map_err(|_| "Failed to set MCP manager in OnceLock".to_string())?; + + self.mcp_manager = Some(mcp_manager_lock); + Ok(self) + } +} + +impl Default for AppContextBuilder { + fn default() -> Self { + Self::new() + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/config/builder.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/config/builder.rs new file mode 100644 index 00000000..f7e15aae --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/config/builder.rs @@ -0,0 +1,740 @@ +use super::{ + CircuitBreakerConfig, ConfigError, ConfigResult, DiscoveryConfig, HealthCheckConfig, + HistoryBackend, MetricsConfig, OracleConfig, PolicyConfig, PostgresConfig, RetryConfig, + RouterConfig, RoutingMode, TokenizerCacheConfig, +}; +use crate::{core::ConnectionMode, mcp::McpConfig}; + +/// Builder for RouterConfig that wraps the config itself +/// This eliminates field duplication and stays in sync automatically +#[derive(Debug, Clone, Default)] +pub struct RouterConfigBuilder { + config: RouterConfig, + // Temporary fields for certificate paths (read during build) + client_cert_path: Option, + client_key_path: Option, + ca_cert_paths: Vec, + mcp_config_path: Option, +} + +impl RouterConfigBuilder { + pub fn new() -> Self { + Self::default() + } + + /// Takes ownership + pub fn from_config(config: RouterConfig) -> Self { + Self { + config, + client_cert_path: None, + client_key_path: None, + ca_cert_paths: Vec::new(), + mcp_config_path: None, + } + } + + pub fn from_config_ref(config: &RouterConfig) -> Self { + Self::from_config(config.clone()) + } + + // ==================== Routing Mode ==================== + + pub fn regular_mode(mut self, worker_urls: Vec) -> Self { + self.config.mode = RoutingMode::Regular { worker_urls }; + self + } + + pub fn prefill_decode_mode( + mut self, + prefill_urls: Vec<(String, Option)>, + decode_urls: Vec, + ) -> Self { + self.config.mode = RoutingMode::PrefillDecode { + prefill_urls, + decode_urls, + prefill_policy: None, + decode_policy: None, + }; + self + } + + /// With separate policies + pub fn prefill_decode_mode_with_policies( + mut self, + prefill_urls: Vec<(String, Option)>, + decode_urls: Vec, + prefill_policy: Option, + decode_policy: Option, + ) -> Self { + self.config.mode = RoutingMode::PrefillDecode { + prefill_urls, + decode_urls, + prefill_policy, + decode_policy, + }; + self + } + + pub fn openai_mode(mut self, worker_urls: Vec) -> Self { + self.config.mode = RoutingMode::OpenAI { worker_urls }; + self + } + + pub fn mode(mut self, mode: RoutingMode) -> Self { + self.config.mode = mode; + self + } + + // ==================== Policy ==================== + + pub fn policy(mut self, policy: PolicyConfig) -> Self { + self.config.policy = policy; + self + } + + pub fn random_policy(mut self) -> Self { + self.config.policy = PolicyConfig::Random; + self + } + + pub fn round_robin_policy(mut self) -> Self { + self.config.policy = PolicyConfig::RoundRobin; + self + } + + pub fn cache_aware_policy( + mut self, + cache_threshold: f32, + balance_abs_threshold: usize, + balance_rel_threshold: f32, + eviction_interval_secs: u64, + max_tree_size: usize, + ) -> Self { + self.config.policy = PolicyConfig::CacheAware { + cache_threshold, + balance_abs_threshold, + balance_rel_threshold, + eviction_interval_secs, + max_tree_size, + }; + self + } + + pub fn power_of_two_policy(mut self, load_check_interval_secs: u64) -> Self { + self.config.policy = PolicyConfig::PowerOfTwo { + load_check_interval_secs, + }; + self + } + + // ==================== Connection ==================== + + pub fn connection_mode(mut self, mode: ConnectionMode) -> Self { + self.config.connection_mode = mode; + self + } + + pub fn http_connection(mut self) -> Self { + self.config.connection_mode = ConnectionMode::Http; + self + } + + pub fn grpc_connection(mut self, port: Option) -> Self { + self.config.connection_mode = ConnectionMode::Grpc { port }; + self + } + + pub fn grpc_connection_default(mut self) -> Self { + self.config.connection_mode = ConnectionMode::Grpc { port: None }; + self + } + + pub fn host>(mut self, host: S) -> Self { + self.config.host = host.into(); + self + } + + pub fn port(mut self, port: u16) -> Self { + self.config.port = port; + self + } + + // ==================== Request ==================== + + pub fn max_payload_size(mut self, size: usize) -> Self { + self.config.max_payload_size = size; + self + } + + pub fn request_timeout_secs(mut self, timeout: u64) -> Self { + self.config.request_timeout_secs = timeout; + self + } + + pub fn worker_startup_timeout_secs(mut self, timeout: u64) -> Self { + self.config.worker_startup_timeout_secs = timeout; + self + } + + pub fn worker_startup_check_interval_secs(mut self, interval: u64) -> Self { + self.config.worker_startup_check_interval_secs = interval; + self + } + + // ==================== Rate Limiting ==================== + + pub fn max_concurrent_requests(mut self, max: i32) -> Self { + self.config.max_concurrent_requests = max; + self + } + + pub fn disable_rate_limiting(mut self) -> Self { + self.config.max_concurrent_requests = -1; + self + } + + pub fn queue_size(mut self, size: usize) -> Self { + self.config.queue_size = size; + self + } + + pub fn queue_timeout_secs(mut self, timeout: u64) -> Self { + self.config.queue_timeout_secs = timeout; + self + } + + pub fn rate_limit_tokens_per_second(mut self, tokens: i32) -> Self { + self.config.rate_limit_tokens_per_second = Some(tokens); + self + } + + // ==================== Security & CORS ==================== + + pub fn api_key>(mut self, key: S) -> Self { + self.config.api_key = Some(key.into()); + self + } + + pub fn cors_allowed_origins(mut self, origins: Vec) -> Self { + self.config.cors_allowed_origins = origins; + self + } + + pub fn add_cors_origin>(mut self, origin: S) -> Self { + self.config.cors_allowed_origins.push(origin.into()); + self + } + + // ==================== Retry ==================== + + pub fn retry_config(mut self, retry: RetryConfig) -> Self { + self.config.retry = retry; + self + } + + pub fn disable_retries(mut self) -> Self { + self.config.disable_retries = true; + self + } + + pub fn enable_retries(mut self) -> Self { + self.config.disable_retries = false; + self + } + + // ==================== Circuit Breaker ==================== + + pub fn circuit_breaker_config(mut self, circuit_breaker: CircuitBreakerConfig) -> Self { + self.config.circuit_breaker = circuit_breaker; + self + } + + pub fn disable_circuit_breaker(mut self) -> Self { + self.config.disable_circuit_breaker = true; + self + } + + pub fn enable_circuit_breaker(mut self) -> Self { + self.config.disable_circuit_breaker = false; + self + } + + // ==================== Health Check ==================== + + pub fn health_check_config(mut self, health_check: HealthCheckConfig) -> Self { + self.config.health_check = health_check; + self + } + + // ==================== Discovery ==================== + + pub fn discovery_config(mut self, discovery: DiscoveryConfig) -> Self { + self.config.discovery = Some(discovery); + self + } + + /// With default settings + pub fn enable_discovery(mut self) -> Self { + self.config.discovery = Some(DiscoveryConfig { + enabled: true, + ..Default::default() + }); + self + } + + // ==================== Metrics ==================== + + pub fn metrics_config(mut self, metrics: MetricsConfig) -> Self { + self.config.metrics = Some(metrics); + self + } + + pub fn enable_metrics>(mut self, host: S, port: u16) -> Self { + self.config.metrics = Some(MetricsConfig { + host: host.into(), + port, + }); + self + } + + // ==================== Logging ==================== + + pub fn log_dir>(mut self, dir: S) -> Self { + self.config.log_dir = Some(dir.into()); + self + } + + pub fn log_level>(mut self, level: S) -> Self { + self.config.log_level = Some(level.into()); + self + } + + pub fn request_id_headers(mut self, headers: Vec) -> Self { + self.config.request_id_headers = Some(headers); + self + } + + // ==================== IGW Mode ==================== + + pub fn enable_igw(mut self) -> Self { + self.config.enable_igw = true; + self + } + + /// Use proxy mode + pub fn disable_igw(mut self) -> Self { + self.config.enable_igw = false; + self + } + + pub fn model_path>(mut self, path: S) -> Self { + self.config.model_path = Some(path.into()); + self + } + + /// Overrides model_path tokenizer + pub fn tokenizer_path>(mut self, path: S) -> Self { + self.config.tokenizer_path = Some(path.into()); + self + } + + pub fn chat_template>(mut self, path: S) -> Self { + self.config.chat_template = Some(path.into()); + self + } + + // ==================== History Backend ==================== + + pub fn history_backend(mut self, backend: HistoryBackend) -> Self { + self.config.history_backend = backend; + self + } + + pub fn memory_history(mut self) -> Self { + self.config.history_backend = HistoryBackend::Memory; + self + } + + pub fn no_history(mut self) -> Self { + self.config.history_backend = HistoryBackend::None; + self + } + + pub fn oracle_history(mut self, oracle_config: OracleConfig) -> Self { + self.config.history_backend = HistoryBackend::Oracle; + self.config.oracle = Some(oracle_config); + self + } + + // ==================== Parsers ==================== + + pub fn reasoning_parser>(mut self, parser: S) -> Self { + self.config.reasoning_parser = Some(parser.into()); + self + } + + pub fn tool_call_parser>(mut self, parser: S) -> Self { + self.config.tool_call_parser = Some(parser.into()); + self + } + + // ==================== Tokenizer Cache ==================== + + pub fn tokenizer_cache(mut self, cache: TokenizerCacheConfig) -> Self { + self.config.tokenizer_cache = cache; + self + } + + pub fn enable_l0_cache(mut self, max_entries: usize) -> Self { + self.config.tokenizer_cache.enable_l0 = true; + self.config.tokenizer_cache.l0_max_entries = max_entries; + self + } + + pub fn enable_l1_cache(mut self, max_memory: usize) -> Self { + self.config.tokenizer_cache.enable_l1 = true; + self.config.tokenizer_cache.l1_max_memory = max_memory; + self + } + + // ==================== Data Parallelism ==================== + + pub fn enable_dp_aware(mut self) -> Self { + self.config.dp_aware = true; + self + } + + pub fn disable_dp_aware(mut self) -> Self { + self.config.dp_aware = false; + self + } + + // ==================== Boolean Setters ==================== + // Accept bool parameters to conditionally set flags without if statements + + pub fn dp_aware(mut self, enable: bool) -> Self { + self.config.dp_aware = enable; + self + } + + /// Inverse of disable_retries field + pub fn retries(mut self, enable: bool) -> Self { + self.config.disable_retries = !enable; + self + } + + /// Inverse of disable_circuit_breaker field + pub fn circuit_breaker(mut self, enable: bool) -> Self { + self.config.disable_circuit_breaker = !enable; + self + } + + pub fn igw(mut self, enable: bool) -> Self { + self.config.enable_igw = enable; + self + } + + // ==================== Option Setters ==================== + // Accept Option and only set if Some + + pub fn maybe_api_key(mut self, key: Option>) -> Self { + if let Some(k) = key { + self.config.api_key = Some(k.into()); + } + self + } + + pub fn maybe_discovery(mut self, discovery: Option) -> Self { + self.config.discovery = discovery; + self + } + + pub fn maybe_metrics(mut self, metrics: Option) -> Self { + self.config.metrics = metrics; + self + } + + pub fn maybe_log_dir(mut self, dir: Option>) -> Self { + self.config.log_dir = dir.map(|d| d.into()); + self + } + + pub fn maybe_log_level(mut self, level: Option>) -> Self { + self.config.log_level = level.map(|l| l.into()); + self + } + + pub fn maybe_request_id_headers(mut self, headers: Option>) -> Self { + self.config.request_id_headers = headers; + self + } + + pub fn maybe_rate_limit_tokens_per_second(mut self, tokens: Option) -> Self { + self.config.rate_limit_tokens_per_second = tokens; + self + } + + pub fn maybe_model_path(mut self, path: Option>) -> Self { + self.config.model_path = path.map(|p| p.into()); + self + } + + pub fn maybe_tokenizer_path(mut self, path: Option>) -> Self { + self.config.tokenizer_path = path.map(|p| p.into()); + self + } + + pub fn maybe_chat_template(mut self, template: Option>) -> Self { + self.config.chat_template = template.map(|t| t.into()); + self + } + + pub fn maybe_oracle(mut self, oracle: Option) -> Self { + if let Some(cfg) = oracle { + self.config.history_backend = HistoryBackend::Oracle; + self.config.oracle = Some(cfg); + } + self + } + + pub fn maybe_postgres(mut self, postgres: Option) -> Self { + if let Some(cfg) = postgres { + self.config.history_backend = HistoryBackend::Postgres; + self.config.postgres = Some(cfg); + } + self + } + + pub fn maybe_reasoning_parser(mut self, parser: Option>) -> Self { + self.config.reasoning_parser = parser.map(|p| p.into()); + self + } + + pub fn maybe_tool_call_parser(mut self, parser: Option>) -> Self { + self.config.tool_call_parser = parser.map(|p| p.into()); + self + } + + // ==================== mTLS ==================== + + /// Both paths must be provided together. Files read during build() + pub fn client_cert_and_key, S2: Into>( + mut self, + cert_path: S1, + key_path: S2, + ) -> Self { + self.client_cert_path = Some(cert_path.into()); + self.client_key_path = Some(key_path.into()); + self + } + + /// Files read during build() + pub fn maybe_client_cert_and_key( + mut self, + cert_path: Option>, + key_path: Option>, + ) -> Self { + self.client_cert_path = cert_path.map(|p| p.into()); + self.client_key_path = key_path.map(|p| p.into()); + self + } + + /// File read during build() + pub fn add_ca_certificate>(mut self, ca_cert_path: S) -> Self { + self.ca_cert_paths.push(ca_cert_path.into()); + self + } + + /// Files read during build() + pub fn add_ca_certificates>(mut self, ca_cert_paths: Vec) -> Self { + self.ca_cert_paths + .extend(ca_cert_paths.into_iter().map(|p| p.into())); + self + } + + // ==================== MCP ==================== + + /// Config file loaded during build() + pub fn mcp_config_path>(mut self, path: S) -> Self { + self.mcp_config_path = Some(path.into()); + self + } + + /// Config file loaded during build() + pub fn maybe_mcp_config_path(mut self, path: Option>) -> Self { + self.mcp_config_path = path.map(|p| p.into()); + self + } + + // ==================== Build ==================== + + pub fn build(self) -> ConfigResult { + self.build_with_validation(true) + } + + pub fn build_unchecked(self) -> RouterConfig { + self.into() + } + + pub fn build_with_validation(mut self, validate: bool) -> ConfigResult { + // Read mTLS certificates from paths if provided + self = self.read_mtls_certificates()?; + + // Read MCP config from path if provided + self = self.read_mcp_config()?; + + let config: RouterConfig = self.into(); + if validate { + config.validate()?; + } + Ok(config) + } + + /// Internal method to read mTLS certificates from paths + fn read_mtls_certificates(mut self) -> ConfigResult { + // Read client certificate and key + match (&self.client_cert_path, &self.client_key_path) { + (Some(cert_path), Some(key_path)) => { + let cert = std::fs::read(cert_path).map_err(|e| ConfigError::ValidationFailed { + reason: format!( + "Failed to read client certificate from {}: {}", + cert_path, e + ), + })?; + let key = std::fs::read(key_path).map_err(|e| ConfigError::ValidationFailed { + reason: format!("Failed to read client key from {}: {}", key_path, e), + })?; + + // Combine cert and key into single PEM for reqwest::Identity + // When using rustls, certificate must come first, then key + // Ensure proper PEM formatting with newlines + let mut combined = cert; + if !combined.ends_with(b"\n") { + combined.push(b'\n'); + } + combined.extend_from_slice(&key); + if !combined.ends_with(b"\n") { + combined.push(b'\n'); + } + + self.config.client_identity = Some(combined); + } + (None, None) => { + // No client cert configured, that's fine + } + _ => { + return Err(ConfigError::ValidationFailed { + reason: + "Both --client-cert-path and --client-key-path must be specified together" + .to_string(), + }); + } + } + + // Read CA certificates + for path in &self.ca_cert_paths { + let cert = std::fs::read(path).map_err(|e| ConfigError::ValidationFailed { + reason: format!("Failed to read CA certificate from {}: {}", path, e), + })?; + self.config.ca_certificates.push(cert); + } + + Ok(self) + } + + /// Internal method to read MCP config from path + fn read_mcp_config(mut self) -> ConfigResult { + if let Some(mcp_config_path) = &self.mcp_config_path { + let contents = std::fs::read_to_string(mcp_config_path).map_err(|e| { + ConfigError::ValidationFailed { + reason: format!("Failed to read MCP config from {}: {}", mcp_config_path, e), + } + })?; + let mcp_config: McpConfig = + serde_yaml::from_str(&contents).map_err(|e| ConfigError::ValidationFailed { + reason: format!("Failed to parse MCP config from {}: {}", mcp_config_path, e), + })?; + self.config.mcp_config = Some(mcp_config); + } + + Ok(self) + } +} + +impl From for RouterConfig { + fn from(builder: RouterConfigBuilder) -> Self { + builder.config + } +} + +impl RouterConfig { + /// Create a builder for RouterConfig + pub fn builder() -> RouterConfigBuilder { + RouterConfigBuilder::new() + } + + /// Create a builder from this configuration + pub fn to_builder(&self) -> RouterConfigBuilder { + RouterConfigBuilder::from_config_ref(self) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Test that .to_builder() round-trip conversion works correctly + #[test] + fn test_builder_from_existing_config() { + let original = RouterConfigBuilder::new() + .regular_mode(vec!["http://worker1:8000".to_string()]) + .port(3000) + .build() + .unwrap(); + + let modified = original + .to_builder() + .port(4000) + .enable_metrics("0.0.0.0", 29000) + .build() + .unwrap(); + + assert_eq!(modified.port, 4000); + assert!(modified.metrics.is_some()); + } + + /// Test complex routing mode helper method + #[test] + fn test_builder_prefill_decode_mode() { + let config = RouterConfigBuilder::new() + .prefill_decode_mode( + vec![("http://prefill:8000".to_string(), Some(8001))], + vec!["http://decode:8000".to_string()], + ) + .power_of_two_policy(60) + .build() + .unwrap(); + + assert!(config.mode.is_pd_mode()); + assert_eq!(config.mode.worker_count(), 2); + } + + /// Test complex policy helper method with multiple parameters + #[test] + fn test_builder_cache_aware_policy() { + let config = RouterConfigBuilder::new() + .regular_mode(vec!["http://worker1:8000".to_string()]) + .cache_aware_policy(0.8, 10, 1.5, 300, 1000) + .build() + .unwrap(); + + match config.policy { + PolicyConfig::CacheAware { + cache_threshold, .. + } => { + assert!((cache_threshold - 0.8).abs() < 0.0001); + } + _ => panic!("Expected CacheAware policy"), + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/config/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/config/mod.rs new file mode 100644 index 00000000..018683f6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/config/mod.rs @@ -0,0 +1,28 @@ +pub mod builder; +pub mod types; +pub mod validation; + +pub use builder::*; +pub use types::*; +pub use validation::*; + +#[derive(Debug, thiserror::Error)] +pub enum ConfigError { + #[error("Validation failed: {reason}")] + ValidationFailed { reason: String }, + + #[error("Invalid value for field '{field}': {value} - {reason}")] + InvalidValue { + field: String, + value: String, + reason: String, + }, + + #[error("Incompatible configuration: {reason}")] + IncompatibleConfig { reason: String }, + + #[error("Missing required field: {field}")] + MissingRequired { field: String }, +} + +pub type ConfigResult = Result; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/config/types.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/config/types.rs new file mode 100644 index 00000000..6bfd53cf --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/config/types.rs @@ -0,0 +1,1247 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use url::Url; + +use super::ConfigResult; +use crate::core::ConnectionMode; + +/// Main router configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RouterConfig { + pub mode: RoutingMode, + #[serde(default)] + pub connection_mode: ConnectionMode, + pub policy: PolicyConfig, + pub host: String, + pub port: u16, + pub max_payload_size: usize, + pub request_timeout_secs: u64, + pub worker_startup_timeout_secs: u64, + pub worker_startup_check_interval_secs: u64, + pub dp_aware: bool, + pub api_key: Option, + pub discovery: Option, + pub metrics: Option, + pub log_dir: Option, + pub log_level: Option, + pub request_id_headers: Option>, + /// Set to -1 to disable rate limiting + pub max_concurrent_requests: i32, + pub queue_size: usize, + pub queue_timeout_secs: u64, + /// If not set, defaults to max_concurrent_requests + pub rate_limit_tokens_per_second: Option, + pub cors_allowed_origins: Vec, + pub retry: RetryConfig, + pub circuit_breaker: CircuitBreakerConfig, + /// When true, overrides retry.max_retries to 1 + #[serde(default)] + pub disable_retries: bool, + /// When true, overrides circuit_breaker.failure_threshold to u32::MAX + #[serde(default)] + pub disable_circuit_breaker: bool, + pub health_check: HealthCheckConfig, + #[serde(default)] + pub enable_igw: bool, + /// Can be a HuggingFace model ID or local path + pub model_path: Option, + /// Overrides model_path tokenizer if provided + pub tokenizer_path: Option, + pub chat_template: Option, + #[serde(default = "default_history_backend")] + pub history_backend: HistoryBackend, + /// Required when history_backend = "oracle" + #[serde(skip_serializing_if = "Option::is_none")] + pub oracle: Option, + /// Required when history_backend = "postgres" + #[serde(skip_serializing_if = "Option::is_none")] + pub postgres: Option, + /// For reasoning models (e.g., deepseek-r1, qwen3) + pub reasoning_parser: Option, + /// For tool-call interactions + pub tool_call_parser: Option, + #[serde(default)] + pub tokenizer_cache: TokenizerCacheConfig, + /// Combined certificate + key in PEM format, loaded from client_cert_path and client_key_path during config creation + #[serde(skip)] + pub client_identity: Option>, + /// PEM format, loaded from ca_cert_paths during config creation + #[serde(default)] + pub ca_certificates: Vec>, + /// Loaded from mcp_config_path during config creation + #[serde(skip)] + pub mcp_config: Option, +} + +/// Tokenizer cache configuration +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct TokenizerCacheConfig { + /// Whole-string exact match cache + #[serde(default = "default_enable_l0")] + pub enable_l0: bool, + #[serde(default = "default_l0_max_entries")] + pub l0_max_entries: usize, + /// Prefix matching at fixed boundaries + #[serde(default = "default_enable_l1")] + pub enable_l1: bool, + #[serde(default = "default_l1_max_memory")] + pub l1_max_memory: usize, +} + +fn default_enable_l0() -> bool { + false +} + +fn default_l0_max_entries() -> usize { + 10_000 +} + +fn default_enable_l1() -> bool { + false +} + +fn default_l1_max_memory() -> usize { + 50 * 1024 * 1024 // 50MB +} + +impl Default for TokenizerCacheConfig { + fn default() -> Self { + Self { + enable_l0: default_enable_l0(), + l0_max_entries: default_l0_max_entries(), + enable_l1: default_enable_l1(), + l1_max_memory: default_l1_max_memory(), + } + } +} + +fn default_history_backend() -> HistoryBackend { + HistoryBackend::Memory +} + +/// History backend configuration +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum HistoryBackend { + Memory, + None, + Oracle, + Postgres, +} + +/// Oracle history backend configuration +#[derive(Clone, Serialize, Deserialize, PartialEq)] +pub struct OracleConfig { + /// ATP wallet or TLS config files directory + #[serde(skip_serializing_if = "Option::is_none")] + pub wallet_path: Option, + /// DSN (e.g. `tcps://host:port/service`) + pub connect_descriptor: String, + pub username: String, + pub password: String, + #[serde(default = "default_pool_min")] + pub pool_min: usize, + #[serde(default = "default_pool_max")] + pub pool_max: usize, + #[serde(default = "default_pool_timeout_secs")] + pub pool_timeout_secs: u64, +} + +impl OracleConfig { + pub fn default_pool_min() -> usize { + default_pool_min() + } + + pub fn default_pool_max() -> usize { + default_pool_max() + } + + pub fn default_pool_timeout_secs() -> u64 { + default_pool_timeout_secs() + } +} + +fn default_pool_min() -> usize { + 1 +} + +fn default_pool_max() -> usize { + 16 +} + +fn default_pool_timeout_secs() -> u64 { + 30 +} + +impl std::fmt::Debug for OracleConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OracleConfig") + .field("wallet_path", &self.wallet_path) + .field("connect_descriptor", &self.connect_descriptor) + .field("username", &self.username) + .field("pool_min", &self.pool_min) + .field("pool_max", &self.pool_max) + .field("pool_timeout_secs", &self.pool_timeout_secs) + .finish() + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct PostgresConfig { + // Database connection URL, + // postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1&...] + pub db_url: String, + // Database pool max size + pub pool_max: usize, +} + +impl PostgresConfig { + pub fn default_pool_max() -> usize { + 16 + } + + pub fn validate(&self) -> Result<(), String> { + let s = self.db_url.trim(); + if s.is_empty() { + return Err("is it db-url should be not empty".to_string()); + } + + let url = Url::parse(s).map_err(|e| format!("invalid db_url: {}", e))?; + + let scheme = url.scheme(); + if scheme != "postgres" && scheme != "postgresql" { + return Err(format!("don't support URL scheme: {}", scheme)); + } + + if url.host().is_none() { + return Err("db_url must need host".to_string()); + } + + let path = url.path(); + let dbname = path + .strip_prefix('/') + .filter(|p| !p.is_empty()) + .map(|s| s.to_string()); + if dbname.is_none() { + return Err("db_url must need database name".to_string()); + } + + if self.pool_max == 0 { + return Err("pool_max must be greater 1, default is 16".to_string()); + } + + Ok(()) + } +} + +/// Routing mode configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum RoutingMode { + #[serde(rename = "regular")] + Regular { worker_urls: Vec }, + #[serde(rename = "prefill_decode")] + PrefillDecode { + /// With optional bootstrap ports + prefill_urls: Vec<(String, Option)>, + decode_urls: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + prefill_policy: Option, + #[serde(skip_serializing_if = "Option::is_none")] + decode_policy: Option, + }, + #[serde(rename = "openai")] + OpenAI { worker_urls: Vec }, +} + +impl RoutingMode { + pub fn is_pd_mode(&self) -> bool { + matches!(self, RoutingMode::PrefillDecode { .. }) + } + + pub fn worker_count(&self) -> usize { + match self { + RoutingMode::Regular { worker_urls } => worker_urls.len(), + RoutingMode::PrefillDecode { + prefill_urls, + decode_urls, + .. + } => prefill_urls.len() + decode_urls.len(), + RoutingMode::OpenAI { .. } => 1, + } + } + + /// Get the effective prefill policy for PD mode + /// Falls back to the main policy if no specific prefill policy is set + pub fn get_prefill_policy<'a>(&'a self, main_policy: &'a PolicyConfig) -> &'a PolicyConfig { + match self { + RoutingMode::PrefillDecode { prefill_policy, .. } => { + prefill_policy.as_ref().unwrap_or(main_policy) + } + _ => main_policy, + } + } + + /// Get the effective decode policy for PD mode + /// Falls back to the main policy if no specific decode policy is set + pub fn get_decode_policy<'a>(&'a self, main_policy: &'a PolicyConfig) -> &'a PolicyConfig { + match self { + RoutingMode::PrefillDecode { decode_policy, .. } => { + decode_policy.as_ref().unwrap_or(main_policy) + } + _ => main_policy, + } + } +} + +/// Policy configuration for routing +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum PolicyConfig { + #[serde(rename = "random")] + Random, + + #[serde(rename = "round_robin")] + RoundRobin, + + #[serde(rename = "cache_aware")] + CacheAware { + cache_threshold: f32, + balance_abs_threshold: usize, + balance_rel_threshold: f32, + eviction_interval_secs: u64, + max_tree_size: usize, + }, + + #[serde(rename = "power_of_two")] + PowerOfTwo { load_check_interval_secs: u64 }, + + #[serde(rename = "bucket")] + Bucket { + /// Absolute load difference threshold for load balancing + balance_abs_threshold: usize, + /// Relative load ratio threshold for load balancing + balance_rel_threshold: f32, + /// Interval between bucket boundary adjustment cycles (seconds) + bucket_adjust_interval_secs: usize, + }, +} + +impl PolicyConfig { + pub fn name(&self) -> &'static str { + match self { + PolicyConfig::Random => "random", + PolicyConfig::RoundRobin => "round_robin", + PolicyConfig::CacheAware { .. } => "cache_aware", + PolicyConfig::PowerOfTwo { .. } => "power_of_two", + PolicyConfig::Bucket { .. } => "bucket", + } + } +} + +/// Service discovery configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DiscoveryConfig { + pub enabled: bool, + /// None = all namespaces + pub namespace: Option, + pub port: u16, + pub check_interval_secs: u64, + /// Regular mode + pub selector: HashMap, + /// PD mode prefill + pub prefill_selector: HashMap, + /// PD mode decode + pub decode_selector: HashMap, + pub bootstrap_port_annotation: String, +} + +impl Default for DiscoveryConfig { + fn default() -> Self { + Self { + enabled: false, + namespace: None, + port: 8000, + check_interval_secs: 120, + selector: HashMap::new(), + prefill_selector: HashMap::new(), + decode_selector: HashMap::new(), + bootstrap_port_annotation: "sglang.ai/bootstrap-port".to_string(), + } + } +} + +/// Retry configuration for request handling +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RetryConfig { + pub max_retries: u32, + pub initial_backoff_ms: u64, + pub max_backoff_ms: u64, + pub backoff_multiplier: f32, + /// D' = D * (1 + U[-j, +j]) where j is jitter factor + #[serde(default = "default_retry_jitter_factor")] + pub jitter_factor: f32, +} + +impl Default for RetryConfig { + fn default() -> Self { + Self { + max_retries: 5, + initial_backoff_ms: 50, + max_backoff_ms: 30000, + backoff_multiplier: 1.5, + jitter_factor: 0.2, + } + } +} + +fn default_retry_jitter_factor() -> f32 { + 0.2 +} + +/// Health check configuration for worker monitoring +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HealthCheckConfig { + pub failure_threshold: u32, + pub success_threshold: u32, + pub timeout_secs: u64, + pub check_interval_secs: u64, + pub endpoint: String, +} + +impl Default for HealthCheckConfig { + fn default() -> Self { + Self { + failure_threshold: 3, + success_threshold: 2, + timeout_secs: 5, + check_interval_secs: 60, + endpoint: "/health".to_string(), + } + } +} + +/// Circuit breaker configuration for worker reliability +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CircuitBreakerConfig { + pub failure_threshold: u32, + pub success_threshold: u32, + pub timeout_duration_secs: u64, + pub window_duration_secs: u64, +} + +impl Default for CircuitBreakerConfig { + fn default() -> Self { + Self { + failure_threshold: 10, + success_threshold: 3, + timeout_duration_secs: 60, + window_duration_secs: 120, + } + } +} + +/// Metrics configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MetricsConfig { + pub port: u16, + pub host: String, +} + +impl Default for MetricsConfig { + fn default() -> Self { + Self { + port: 29000, + host: "0.0.0.0".to_string(), + } + } +} + +impl Default for RouterConfig { + fn default() -> Self { + Self { + mode: RoutingMode::Regular { + worker_urls: vec![], + }, + policy: PolicyConfig::Random, + host: "0.0.0.0".to_string(), + port: 3001, + max_payload_size: 536_870_912, // 512MB + request_timeout_secs: 1800, // 30 minutes + worker_startup_timeout_secs: 1800, // 30 minutes for large model loading + worker_startup_check_interval_secs: 30, + dp_aware: false, + api_key: None, + discovery: None, + metrics: None, + log_dir: None, + log_level: None, + request_id_headers: None, + max_concurrent_requests: -1, + queue_size: 100, + queue_timeout_secs: 60, + rate_limit_tokens_per_second: None, + cors_allowed_origins: vec![], + retry: RetryConfig::default(), + circuit_breaker: CircuitBreakerConfig::default(), + disable_retries: false, + disable_circuit_breaker: false, + health_check: HealthCheckConfig::default(), + enable_igw: false, + connection_mode: ConnectionMode::Http, + model_path: None, + tokenizer_path: None, + chat_template: None, + history_backend: default_history_backend(), + oracle: None, + postgres: None, + reasoning_parser: None, + tool_call_parser: None, + tokenizer_cache: TokenizerCacheConfig::default(), + client_identity: None, + ca_certificates: vec![], + mcp_config: None, + } + } +} + +impl RouterConfig { + /// Create a new configuration with mode and policy + pub fn new(mode: RoutingMode, policy: PolicyConfig) -> Self { + Self { + mode, + policy, + ..Default::default() + } + } + + /// Validate the configuration + pub fn validate(&self) -> ConfigResult<()> { + crate::config::validation::ConfigValidator::validate(self) + } + + /// Get the routing mode type as a string + pub fn mode_type(&self) -> &'static str { + match self.mode { + RoutingMode::Regular { .. } => "regular", + RoutingMode::PrefillDecode { .. } => "prefill_decode", + RoutingMode::OpenAI { .. } => "openai", + } + } + + /// Check if service discovery is enabled + pub fn has_service_discovery(&self) -> bool { + self.discovery.as_ref().is_some_and(|d| d.enabled) + } + + /// Check if metrics are enabled + pub fn has_metrics(&self) -> bool { + self.metrics.is_some() + } + + /// Compute the effective retry config considering disable flag + pub fn effective_retry_config(&self) -> RetryConfig { + let mut cfg = self.retry.clone(); + if self.disable_retries { + cfg.max_retries = 1; + } + cfg + } + + /// Compute the effective circuit breaker config considering disable flag + pub fn effective_circuit_breaker_config(&self) -> CircuitBreakerConfig { + let mut cfg = self.circuit_breaker.clone(); + if self.disable_circuit_breaker { + cfg.failure_threshold = u32::MAX; + } + cfg + } + + /// Check if running in IGW (Inference Gateway) mode + pub fn is_igw_mode(&self) -> bool { + self.enable_igw + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_router_config_default() { + let config = RouterConfig::default(); + + assert!( + matches!(config.mode, RoutingMode::Regular { worker_urls } if worker_urls.is_empty()) + ); + assert!(matches!(config.policy, PolicyConfig::Random)); + assert_eq!(config.host, "0.0.0.0"); + assert_eq!(config.port, 3001); + assert_eq!(config.max_payload_size, 536_870_912); + assert_eq!(config.request_timeout_secs, 1800); + assert_eq!(config.worker_startup_timeout_secs, 1800); + assert_eq!(config.worker_startup_check_interval_secs, 30); + assert!(config.discovery.is_none()); + assert!(config.metrics.is_none()); + assert!(config.log_dir.is_none()); + assert!(config.log_level.is_none()); + } + + #[test] + fn test_router_config_new() { + let mode = RoutingMode::Regular { + worker_urls: vec!["http://worker1".to_string(), "http://worker2".to_string()], + }; + let policy = PolicyConfig::RoundRobin; + + let config = RouterConfig::new(mode, policy); + + match config.mode { + RoutingMode::Regular { worker_urls } => { + assert_eq!(worker_urls.len(), 2); + assert_eq!(worker_urls[0], "http://worker1"); + assert_eq!(worker_urls[1], "http://worker2"); + } + _ => panic!("Expected Regular mode"), + } + + assert!(matches!(config.policy, PolicyConfig::RoundRobin)); + assert_eq!(config.host, "0.0.0.0"); + assert_eq!(config.port, 3001); + } + + #[test] + fn test_router_config_serialization() { + let config = RouterConfig::builder() + .regular_mode(vec!["http://worker1".to_string()]) + .random_policy() + .host("0.0.0.0") + .port(8080) + .log_dir("/var/log") + .log_level("debug") + .build_unchecked(); + + let json = serde_json::to_string(&config).unwrap(); + let deserialized: RouterConfig = serde_json::from_str(&json).unwrap(); + + assert_eq!(config.host, deserialized.host); + assert_eq!(config.port, deserialized.port); + assert_eq!(config.max_payload_size, deserialized.max_payload_size); + assert_eq!(config.log_dir, deserialized.log_dir); + assert_eq!(config.log_level, deserialized.log_level); + assert!(deserialized.discovery.is_none()); + assert!(deserialized.metrics.is_none()); + } + + #[test] + fn test_routing_mode_is_pd_mode() { + let regular = RoutingMode::Regular { + worker_urls: vec!["http://worker1".to_string()], + }; + assert!(!regular.is_pd_mode()); + + let pd = RoutingMode::PrefillDecode { + prefill_urls: vec![("http://prefill1".to_string(), Some(8001))], + decode_urls: vec!["http://decode1".to_string()], + prefill_policy: None, + decode_policy: None, + }; + assert!(pd.is_pd_mode()); + } + + #[test] + fn test_routing_mode_worker_count() { + let regular = RoutingMode::Regular { + worker_urls: vec![ + "http://worker1".to_string(), + "http://worker2".to_string(), + "http://worker3".to_string(), + ], + }; + assert_eq!(regular.worker_count(), 3); + + let pd = RoutingMode::PrefillDecode { + prefill_urls: vec![ + ("http://prefill1".to_string(), Some(8001)), + ("http://prefill2".to_string(), None), + ], + decode_urls: vec![ + "http://decode1".to_string(), + "http://decode2".to_string(), + "http://decode3".to_string(), + ], + prefill_policy: None, + decode_policy: None, + }; + assert_eq!(pd.worker_count(), 5); + + let empty_regular = RoutingMode::Regular { + worker_urls: vec![], + }; + assert_eq!(empty_regular.worker_count(), 0); + } + + #[test] + fn test_routing_mode_serialization() { + let regular = RoutingMode::Regular { + worker_urls: vec!["http://worker1".to_string()], + }; + let json = serde_json::to_string(®ular).unwrap(); + assert!(json.contains("\"type\":\"regular\"")); + assert!(json.contains("\"worker_urls\"")); + + let pd = RoutingMode::PrefillDecode { + prefill_urls: vec![("http://prefill1".to_string(), Some(8001))], + decode_urls: vec!["http://decode1".to_string()], + prefill_policy: None, + decode_policy: None, + }; + let json = serde_json::to_string(&pd).unwrap(); + assert!(json.contains("\"type\":\"prefill_decode\"")); + assert!(json.contains("\"prefill_urls\"")); + assert!(json.contains("\"decode_urls\"")); + } + + #[test] + fn test_policy_config_name() { + assert_eq!(PolicyConfig::Random.name(), "random"); + assert_eq!(PolicyConfig::RoundRobin.name(), "round_robin"); + + let cache_aware = PolicyConfig::CacheAware { + cache_threshold: 0.8, + balance_abs_threshold: 10, + balance_rel_threshold: 1.5, + eviction_interval_secs: 300, + max_tree_size: 1000, + }; + assert_eq!(cache_aware.name(), "cache_aware"); + + let power_of_two = PolicyConfig::PowerOfTwo { + load_check_interval_secs: 60, + }; + assert_eq!(power_of_two.name(), "power_of_two"); + } + + #[test] + fn test_policy_config_serialization() { + let random = PolicyConfig::Random; + let json = serde_json::to_string(&random).unwrap(); + assert_eq!(json, r#"{"type":"random"}"#); + + let cache_aware = PolicyConfig::CacheAware { + cache_threshold: 0.8, + balance_abs_threshold: 10, + balance_rel_threshold: 1.5, + eviction_interval_secs: 300, + max_tree_size: 1000, + }; + let json = serde_json::to_string(&cache_aware).unwrap(); + assert!(json.contains("\"type\":\"cache_aware\"")); + assert!(json.contains("\"cache_threshold\":0.8")); + assert!(json.contains("\"balance_abs_threshold\":10")); + + let power_of_two = PolicyConfig::PowerOfTwo { + load_check_interval_secs: 60, + }; + let json = serde_json::to_string(&power_of_two).unwrap(); + assert!(json.contains("\"type\":\"power_of_two\"")); + assert!(json.contains("\"load_check_interval_secs\":60")); + } + + #[test] + fn test_cache_aware_parameters() { + let cache_aware = PolicyConfig::CacheAware { + cache_threshold: 0.75, + balance_abs_threshold: 20, + balance_rel_threshold: 2.0, + eviction_interval_secs: 600, + max_tree_size: 5000, + }; + + match cache_aware { + PolicyConfig::CacheAware { + cache_threshold, + balance_abs_threshold, + balance_rel_threshold, + eviction_interval_secs, + max_tree_size, + } => { + assert!((cache_threshold - 0.75).abs() < 0.0001); + assert_eq!(balance_abs_threshold, 20); + assert!((balance_rel_threshold - 2.0).abs() < 0.0001); + assert_eq!(eviction_interval_secs, 600); + assert_eq!(max_tree_size, 5000); + } + _ => panic!("Expected CacheAware"), + } + } + + #[test] + fn test_power_of_two_parameters() { + let power_of_two = PolicyConfig::PowerOfTwo { + load_check_interval_secs: 120, + }; + + match power_of_two { + PolicyConfig::PowerOfTwo { + load_check_interval_secs, + } => { + assert_eq!(load_check_interval_secs, 120); + } + _ => panic!("Expected PowerOfTwo"), + } + } + + #[test] + fn test_bucket_parameters() { + let bucket = PolicyConfig::Bucket { + balance_abs_threshold: 20, + balance_rel_threshold: 2.0, + bucket_adjust_interval_secs: 5, + }; + + match bucket { + PolicyConfig::Bucket { + balance_abs_threshold, + balance_rel_threshold, + bucket_adjust_interval_secs, + } => { + assert_eq!(balance_abs_threshold, 20); + assert!((balance_rel_threshold - 2.0).abs() < 0.0001); + assert_eq!(bucket_adjust_interval_secs, 5); + } + _ => panic!("Expected Bucket"), + } + } + + #[test] + fn test_discovery_config_default() { + let config = DiscoveryConfig::default(); + + assert!(!config.enabled); + assert!(config.namespace.is_none()); + assert_eq!(config.port, 8000); + assert_eq!(config.check_interval_secs, 120); + assert!(config.selector.is_empty()); + assert!(config.prefill_selector.is_empty()); + assert!(config.decode_selector.is_empty()); + assert_eq!(config.bootstrap_port_annotation, "sglang.ai/bootstrap-port"); + } + + #[test] + fn test_discovery_config_with_selectors() { + let mut selector = HashMap::new(); + selector.insert("app".to_string(), "sglang".to_string()); + selector.insert("role".to_string(), "worker".to_string()); + + let config = DiscoveryConfig { + enabled: true, + namespace: Some("default".to_string()), + port: 9000, + check_interval_secs: 30, + selector: selector.clone(), + prefill_selector: selector.clone(), + decode_selector: selector.clone(), + bootstrap_port_annotation: "custom.io/port".to_string(), + }; + + assert!(config.enabled); + assert_eq!(config.namespace, Some("default".to_string())); + assert_eq!(config.port, 9000); + assert_eq!(config.selector.len(), 2); + assert_eq!(config.selector.get("app"), Some(&"sglang".to_string())); + } + + #[test] + fn test_discovery_config_namespace() { + let config = DiscoveryConfig { + namespace: None, + ..Default::default() + }; + assert!(config.namespace.is_none()); + + let config = DiscoveryConfig { + namespace: Some("production".to_string()), + ..Default::default() + }; + assert_eq!(config.namespace, Some("production".to_string())); + } + + #[test] + fn test_metrics_config_default() { + let config = MetricsConfig::default(); + + assert_eq!(config.port, 29000); + assert_eq!(config.host, "0.0.0.0"); + } + + #[test] + fn test_metrics_config_custom() { + let config = MetricsConfig { + port: 9090, + host: "0.0.0.0".to_string(), + }; + + assert_eq!(config.port, 9090); + assert_eq!(config.host, "0.0.0.0"); + } + + #[test] + fn test_mode_type() { + let config = RouterConfig::builder() + .regular_mode(vec![]) + .build_unchecked(); + assert_eq!(config.mode_type(), "regular"); + + let config = RouterConfig::builder() + .prefill_decode_mode(vec![], vec![]) + .build_unchecked(); + assert_eq!(config.mode_type(), "prefill_decode"); + } + + #[test] + fn test_has_service_discovery() { + let config = RouterConfig::default(); + assert!(!config.has_service_discovery()); + + let config = RouterConfig::builder() + .discovery_config(DiscoveryConfig { + enabled: false, + ..Default::default() + }) + .build_unchecked(); + assert!(!config.has_service_discovery()); + + let config = RouterConfig::builder().enable_discovery().build_unchecked(); + assert!(config.has_service_discovery()); + } + + #[test] + fn test_has_metrics() { + let config = RouterConfig::default(); + assert!(!config.has_metrics()); + + let config = RouterConfig::builder() + .metrics_config(MetricsConfig::default()) + .build_unchecked(); + assert!(config.has_metrics()); + } + + #[test] + fn test_large_worker_lists() { + let large_urls: Vec = (0..1000).map(|i| format!("http://worker{}", i)).collect(); + + let config = RouterConfig::builder() + .regular_mode(large_urls.clone()) + .build_unchecked(); + + assert_eq!(config.mode.worker_count(), 1000); + + let json = serde_json::to_string(&config).unwrap(); + let deserialized: RouterConfig = serde_json::from_str(&json).unwrap(); + + match deserialized.mode { + RoutingMode::Regular { worker_urls } => { + assert_eq!(worker_urls.len(), 1000); + } + _ => panic!("Expected Regular mode"), + } + } + + #[test] + fn test_unicode_in_config() { + let config = RouterConfig::builder() + .regular_mode(vec![ + "http://работник1".to_string(), + "http://工作者2".to_string(), + ]) + .log_dir("/日志/目录") + .build_unchecked(); + + let json = serde_json::to_string(&config).unwrap(); + let deserialized: RouterConfig = serde_json::from_str(&json).unwrap(); + + match deserialized.mode { + RoutingMode::Regular { worker_urls } => { + assert_eq!(worker_urls[0], "http://работник1"); + assert_eq!(worker_urls[1], "http://工作者2"); + } + _ => panic!("Expected Regular mode"), + } + + assert_eq!(deserialized.log_dir, Some("/日志/目录".to_string())); + } + + #[test] + fn test_empty_string_fields() { + let config = RouterConfig::builder() + .host("") + .log_dir("") + .log_level("") + .build_unchecked(); + + assert_eq!(config.host, ""); + assert_eq!(config.log_dir, Some("".to_string())); + assert_eq!(config.log_level, Some("".to_string())); + } + + #[test] + fn test_full_pd_mode_config() { + let config = RouterConfig::builder() + .prefill_decode_mode( + vec![ + ("http://prefill1:8000".to_string(), Some(8001)), + ("http://prefill2:8000".to_string(), None), + ], + vec![ + "http://decode1:8000".to_string(), + "http://decode2:8000".to_string(), + ], + ) + .power_of_two_policy(30) + .host("0.0.0.0") + .port(3000) + .max_payload_size(1048576) + .request_timeout_secs(120) + .worker_startup_timeout_secs(60) + .worker_startup_check_interval_secs(5) + .discovery_config(DiscoveryConfig { + enabled: true, + namespace: Some("sglang".to_string()), + ..Default::default() + }) + .enable_metrics("0.0.0.0", 9090) + .log_dir("/var/log/sglang") + .log_level("info") + .max_concurrent_requests(64) + .build_unchecked(); + + assert!(config.mode.is_pd_mode()); + assert_eq!(config.mode.worker_count(), 4); + assert_eq!(config.policy.name(), "power_of_two"); + assert!(config.has_service_discovery()); + assert!(config.has_metrics()); + } + + #[test] + fn test_full_regular_mode_config() { + let mut selector = HashMap::new(); + selector.insert("app".to_string(), "sglang".to_string()); + + let config = RouterConfig::builder() + .regular_mode(vec![ + "http://worker1:8000".to_string(), + "http://worker2:8000".to_string(), + "http://worker3:8000".to_string(), + ]) + .cache_aware_policy(0.9, 5, 1.2, 600, 10000) + .host("0.0.0.0") + .port(3001) + .max_payload_size(536870912) + .request_timeout_secs(300) + .worker_startup_timeout_secs(180) + .worker_startup_check_interval_secs(15) + .discovery_config(DiscoveryConfig { + enabled: true, + namespace: None, + port: 8080, + check_interval_secs: 45, + selector, + ..Default::default() + }) + .metrics_config(MetricsConfig::default()) + .log_level("debug") + .max_concurrent_requests(64) + .build_unchecked(); + + assert!(!config.mode.is_pd_mode()); + assert_eq!(config.mode.worker_count(), 3); + assert_eq!(config.policy.name(), "cache_aware"); + assert!(config.has_service_discovery()); + assert!(config.has_metrics()); + } + + #[test] + fn test_config_with_all_options() { + let mut selectors = HashMap::new(); + selectors.insert("env".to_string(), "prod".to_string()); + selectors.insert("version".to_string(), "v1".to_string()); + + let config = RouterConfig::builder() + .regular_mode(vec!["http://worker1".to_string()]) + .round_robin_policy() + .host("::1") // IPv6 + .port(8888) + .max_payload_size(1024 * 1024 * 512) // 512MB + .request_timeout_secs(900) + .worker_startup_timeout_secs(600) + .worker_startup_check_interval_secs(20) + .discovery_config(DiscoveryConfig { + enabled: true, + namespace: Some("production".to_string()), + port: 8443, + check_interval_secs: 120, + selector: selectors.clone(), + prefill_selector: selectors.clone(), + decode_selector: selectors, + bootstrap_port_annotation: "mycompany.io/bootstrap".to_string(), + }) + .enable_metrics("::", 9999) // IPv6 any + .log_dir("/opt/logs/sglang") + .log_level("trace") + .max_concurrent_requests(64) + .build_unchecked(); + + assert!(config.has_service_discovery()); + assert!(config.has_metrics()); + assert_eq!(config.mode_type(), "regular"); + + let json = serde_json::to_string_pretty(&config).unwrap(); + let deserialized: RouterConfig = serde_json::from_str(&json).unwrap(); + + assert_eq!(deserialized.host, "::1"); + assert_eq!(deserialized.port, 8888); + assert_eq!( + deserialized.discovery.unwrap().namespace, + Some("production".to_string()) + ); + } + + #[test] + fn test_pd_policy_fallback_both_specified() { + let pd = RoutingMode::PrefillDecode { + prefill_urls: vec![("http://prefill1".to_string(), None)], + decode_urls: vec!["http://decode1".to_string()], + prefill_policy: Some(PolicyConfig::CacheAware { + cache_threshold: 0.5, + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + eviction_interval_secs: 60, + max_tree_size: 1000, + }), + decode_policy: Some(PolicyConfig::PowerOfTwo { + load_check_interval_secs: 60, + }), + }; + + let main_policy = PolicyConfig::Random; + + match pd.get_prefill_policy(&main_policy) { + PolicyConfig::CacheAware { .. } => {} + _ => panic!("Expected CacheAware for prefill"), + } + + match pd.get_decode_policy(&main_policy) { + PolicyConfig::PowerOfTwo { .. } => {} + _ => panic!("Expected PowerOfTwo for decode"), + } + } + + #[test] + fn test_pd_policy_fallback_only_prefill() { + let pd = RoutingMode::PrefillDecode { + prefill_urls: vec![("http://prefill1".to_string(), None)], + decode_urls: vec!["http://decode1".to_string()], + prefill_policy: Some(PolicyConfig::CacheAware { + cache_threshold: 0.5, + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + eviction_interval_secs: 60, + max_tree_size: 1000, + }), + decode_policy: None, + }; + + let main_policy = PolicyConfig::RoundRobin; + + match pd.get_prefill_policy(&main_policy) { + PolicyConfig::CacheAware { .. } => {} + _ => panic!("Expected CacheAware for prefill"), + } + + match pd.get_decode_policy(&main_policy) { + PolicyConfig::RoundRobin => {} + _ => panic!("Expected RoundRobin for decode"), + } + } + + #[test] + fn test_pd_policy_fallback_only_decode() { + let pd = RoutingMode::PrefillDecode { + prefill_urls: vec![("http://prefill1".to_string(), None)], + decode_urls: vec!["http://decode1".to_string()], + prefill_policy: None, + decode_policy: Some(PolicyConfig::PowerOfTwo { + load_check_interval_secs: 60, + }), + }; + + let main_policy = PolicyConfig::Random; + + match pd.get_prefill_policy(&main_policy) { + PolicyConfig::Random => {} + _ => panic!("Expected Random for prefill"), + } + + match pd.get_decode_policy(&main_policy) { + PolicyConfig::PowerOfTwo { .. } => {} + _ => panic!("Expected PowerOfTwo for decode"), + } + } + + #[test] + fn test_pd_policy_fallback_none_specified() { + let pd = RoutingMode::PrefillDecode { + prefill_urls: vec![("http://prefill1".to_string(), None)], + decode_urls: vec!["http://decode1".to_string()], + prefill_policy: None, + decode_policy: None, + }; + + let main_policy = PolicyConfig::CacheAware { + cache_threshold: 0.7, + balance_abs_threshold: 20, + balance_rel_threshold: 1.5, + eviction_interval_secs: 300, + max_tree_size: 2000, + }; + + match pd.get_prefill_policy(&main_policy) { + PolicyConfig::CacheAware { + cache_threshold, .. + } => { + assert!((cache_threshold - 0.7).abs() < 0.0001); + } + _ => panic!("Expected CacheAware for prefill"), + } + + match pd.get_decode_policy(&main_policy) { + PolicyConfig::CacheAware { + cache_threshold, .. + } => { + assert!((cache_threshold - 0.7).abs() < 0.0001); + } + _ => panic!("Expected CacheAware for decode"), + } + } + + #[test] + fn test_regular_mode_policy_fallback() { + let regular = RoutingMode::Regular { + worker_urls: vec!["http://worker1".to_string()], + }; + + let main_policy = PolicyConfig::RoundRobin; + + match regular.get_prefill_policy(&main_policy) { + PolicyConfig::RoundRobin => {} + _ => panic!("Expected RoundRobin for regular mode"), + } + + match regular.get_decode_policy(&main_policy) { + PolicyConfig::RoundRobin => {} + _ => panic!("Expected RoundRobin for regular mode"), + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/config/validation.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/config/validation.rs new file mode 100644 index 00000000..ed906ed7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/config/validation.rs @@ -0,0 +1,969 @@ +use super::*; +use crate::core::ConnectionMode; + +/// Configuration validator +pub struct ConfigValidator; + +impl ConfigValidator { + pub fn validate(config: &RouterConfig) -> ConfigResult<()> { + Self::validate_mode(&config.mode)?; + Self::validate_policy(&config.policy)?; + Self::validate_server_settings(config)?; + + if let Some(discovery) = &config.discovery { + Self::validate_discovery(discovery, &config.mode)?; + } + + if let Some(metrics) = &config.metrics { + Self::validate_metrics(metrics)?; + } + + Self::validate_compatibility(config)?; + + let retry_cfg = config.effective_retry_config(); + let cb_cfg = config.effective_circuit_breaker_config(); + Self::validate_retry(&retry_cfg)?; + Self::validate_circuit_breaker(&cb_cfg)?; + + if config.history_backend == HistoryBackend::Oracle { + if config.oracle.is_none() { + return Err(ConfigError::MissingRequired { + field: "oracle".to_string(), + }); + } + if let Some(oracle) = &config.oracle { + Self::validate_oracle(oracle)?; + } + } + + Self::validate_tokenizer_cache(&config.tokenizer_cache)?; + + Ok(()) + } + + fn validate_oracle(oracle: &OracleConfig) -> ConfigResult<()> { + if oracle.username.is_empty() { + return Err(ConfigError::MissingRequired { + field: "oracle.username".to_string(), + }); + } + + if oracle.password.is_empty() { + return Err(ConfigError::MissingRequired { + field: "oracle.password".to_string(), + }); + } + + if oracle.connect_descriptor.is_empty() { + return Err(ConfigError::MissingRequired { + field: "oracle_dsn or oracle_tns_alias".to_string(), + }); + } + + if oracle.pool_min < 1 { + return Err(ConfigError::InvalidValue { + field: "oracle.pool_min".to_string(), + value: oracle.pool_min.to_string(), + reason: "Must be at least 1".to_string(), + }); + } + + if oracle.pool_max < oracle.pool_min { + return Err(ConfigError::InvalidValue { + field: "oracle.pool_max".to_string(), + value: oracle.pool_max.to_string(), + reason: "Must be >= oracle.pool_min".to_string(), + }); + } + + if oracle.pool_timeout_secs == 0 { + return Err(ConfigError::InvalidValue { + field: "oracle.pool_timeout_secs".to_string(), + value: oracle.pool_timeout_secs.to_string(), + reason: "Must be > 0".to_string(), + }); + } + + Ok(()) + } + + fn validate_mode(mode: &RoutingMode) -> ConfigResult<()> { + match mode { + RoutingMode::Regular { worker_urls } => { + if !worker_urls.is_empty() { + Self::validate_urls(worker_urls)?; + } + // Allow empty URLs without service discovery to match legacy behavior + } + RoutingMode::PrefillDecode { + prefill_urls, + decode_urls, + prefill_policy, + decode_policy, + } => { + // Allow empty URLs even without service discovery to support dynamic worker addition + // URLs will be validated if provided + if !prefill_urls.is_empty() { + let prefill_url_strings: Vec = + prefill_urls.iter().map(|(url, _)| url.clone()).collect(); + Self::validate_urls(&prefill_url_strings)?; + } + if !decode_urls.is_empty() { + Self::validate_urls(decode_urls)?; + } + + for (_url, port) in prefill_urls { + if let Some(port) = port { + if *port == 0 { + return Err(ConfigError::InvalidValue { + field: "bootstrap_port".to_string(), + value: port.to_string(), + reason: "Port must be between 1 and 65535".to_string(), + }); + } + } + } + + if let Some(p_policy) = prefill_policy { + Self::validate_policy(p_policy)?; + } + if let Some(d_policy) = decode_policy { + Self::validate_policy(d_policy)?; + } + } + RoutingMode::OpenAI { worker_urls } => { + // Allow empty URLs to support dynamic worker addition + // URLs will be validated if provided + if !worker_urls.is_empty() { + Self::validate_urls(worker_urls)?; + } + } + } + Ok(()) + } + + fn validate_policy(policy: &PolicyConfig) -> ConfigResult<()> { + match policy { + PolicyConfig::Random | PolicyConfig::RoundRobin => {} + PolicyConfig::CacheAware { + cache_threshold, + balance_abs_threshold: _, + balance_rel_threshold, + eviction_interval_secs, + max_tree_size, + } => { + if !(0.0..=1.0).contains(cache_threshold) { + return Err(ConfigError::InvalidValue { + field: "cache_threshold".to_string(), + value: cache_threshold.to_string(), + reason: "Must be between 0.0 and 1.0".to_string(), + }); + } + + if *balance_rel_threshold < 1.0 { + return Err(ConfigError::InvalidValue { + field: "balance_rel_threshold".to_string(), + value: balance_rel_threshold.to_string(), + reason: "Must be >= 1.0".to_string(), + }); + } + + if *eviction_interval_secs == 0 { + return Err(ConfigError::InvalidValue { + field: "eviction_interval_secs".to_string(), + value: eviction_interval_secs.to_string(), + reason: "Must be > 0".to_string(), + }); + } + + if *max_tree_size == 0 { + return Err(ConfigError::InvalidValue { + field: "max_tree_size".to_string(), + value: max_tree_size.to_string(), + reason: "Must be > 0".to_string(), + }); + } + } + PolicyConfig::PowerOfTwo { + load_check_interval_secs, + } => { + if *load_check_interval_secs == 0 { + return Err(ConfigError::InvalidValue { + field: "load_check_interval_secs".to_string(), + value: load_check_interval_secs.to_string(), + reason: "Must be > 0".to_string(), + }); + } + } + PolicyConfig::Bucket { + balance_abs_threshold: _, + balance_rel_threshold, + bucket_adjust_interval_secs, + } => { + if *balance_rel_threshold < 1.0 { + return Err(ConfigError::InvalidValue { + field: "balance_rel_threshold".to_string(), + value: balance_rel_threshold.to_string(), + reason: "Must be >= 1.0".to_string(), + }); + } + + if *bucket_adjust_interval_secs < 1 { + return Err(ConfigError::InvalidValue { + field: "bucket_adjust_interval_secs".to_string(), + value: bucket_adjust_interval_secs.to_string(), + reason: "Must be >= 1s".to_string(), + }); + } + if *bucket_adjust_interval_secs >= 4294967296 { + return Err(ConfigError::InvalidValue { + field: "bucket_adjust_interval_secs".to_string(), + value: bucket_adjust_interval_secs.to_string(), + reason: "Must be < 4294967296s".to_string(), + }); + } + } + } + Ok(()) + } + + fn validate_server_settings(config: &RouterConfig) -> ConfigResult<()> { + if config.port == 0 { + return Err(ConfigError::InvalidValue { + field: "port".to_string(), + value: config.port.to_string(), + reason: "Port must be > 0".to_string(), + }); + } + + if config.max_payload_size == 0 { + return Err(ConfigError::InvalidValue { + field: "max_payload_size".to_string(), + value: config.max_payload_size.to_string(), + reason: "Must be > 0".to_string(), + }); + } + + if config.request_timeout_secs == 0 { + return Err(ConfigError::InvalidValue { + field: "request_timeout_secs".to_string(), + value: config.request_timeout_secs.to_string(), + reason: "Must be > 0".to_string(), + }); + } + + if config.queue_size > 0 && config.queue_timeout_secs == 0 { + return Err(ConfigError::InvalidValue { + field: "queue_timeout_secs".to_string(), + value: config.queue_timeout_secs.to_string(), + reason: "Must be > 0 when queue_size > 0".to_string(), + }); + } + + if let Some(tokens_per_second) = config.rate_limit_tokens_per_second { + // Allow 0 for pure concurrency limiting (semaphore behavior) + if tokens_per_second < 0 { + return Err(ConfigError::InvalidValue { + field: "rate_limit_tokens_per_second".to_string(), + value: tokens_per_second.to_string(), + reason: "Must be >= 0 when specified".to_string(), + }); + } + } + + if config.worker_startup_timeout_secs == 0 { + return Err(ConfigError::InvalidValue { + field: "worker_startup_timeout_secs".to_string(), + value: config.worker_startup_timeout_secs.to_string(), + reason: "Must be > 0".to_string(), + }); + } + + if config.worker_startup_check_interval_secs == 0 { + return Err(ConfigError::InvalidValue { + field: "worker_startup_check_interval_secs".to_string(), + value: config.worker_startup_check_interval_secs.to_string(), + reason: "Must be > 0".to_string(), + }); + } + + Ok(()) + } + + fn validate_discovery(discovery: &DiscoveryConfig, mode: &RoutingMode) -> ConfigResult<()> { + if !discovery.enabled { + return Ok(()); + } + + if discovery.port == 0 { + return Err(ConfigError::InvalidValue { + field: "discovery.port".to_string(), + value: discovery.port.to_string(), + reason: "Port must be > 0".to_string(), + }); + } + + if discovery.check_interval_secs == 0 { + return Err(ConfigError::InvalidValue { + field: "discovery.check_interval_secs".to_string(), + value: discovery.check_interval_secs.to_string(), + reason: "Must be > 0".to_string(), + }); + } + + match mode { + RoutingMode::Regular { .. } => { + if discovery.selector.is_empty() { + return Err(ConfigError::ValidationFailed { + reason: "Regular mode with service discovery requires a non-empty selector" + .to_string(), + }); + } + } + RoutingMode::PrefillDecode { .. } => { + if discovery.prefill_selector.is_empty() && discovery.decode_selector.is_empty() { + return Err(ConfigError::ValidationFailed { + reason: "PD mode with service discovery requires at least one non-empty selector (prefill or decode)".to_string(), + }); + } + } + RoutingMode::OpenAI { .. } => { + return Err(ConfigError::ValidationFailed { + reason: "OpenAI mode does not support service discovery".to_string(), + }); + } + } + + Ok(()) + } + + fn validate_metrics(metrics: &MetricsConfig) -> ConfigResult<()> { + if metrics.port == 0 { + return Err(ConfigError::InvalidValue { + field: "metrics.port".to_string(), + value: metrics.port.to_string(), + reason: "Port must be > 0".to_string(), + }); + } + + if metrics.host.is_empty() { + return Err(ConfigError::InvalidValue { + field: "metrics.host".to_string(), + value: metrics.host.clone(), + reason: "Host cannot be empty".to_string(), + }); + } + + Ok(()) + } + + fn validate_retry(retry: &RetryConfig) -> ConfigResult<()> { + if retry.max_retries < 1 { + return Err(ConfigError::InvalidValue { + field: "retry.max_retries".to_string(), + value: retry.max_retries.to_string(), + reason: "Must be >= 1 (set to 1 to effectively disable retries)".to_string(), + }); + } + if retry.initial_backoff_ms == 0 { + return Err(ConfigError::InvalidValue { + field: "retry.initial_backoff_ms".to_string(), + value: retry.initial_backoff_ms.to_string(), + reason: "Must be > 0".to_string(), + }); + } + if retry.max_backoff_ms < retry.initial_backoff_ms { + return Err(ConfigError::InvalidValue { + field: "retry.max_backoff_ms".to_string(), + value: retry.max_backoff_ms.to_string(), + reason: "Must be >= initial_backoff_ms".to_string(), + }); + } + if retry.backoff_multiplier < 1.0 { + return Err(ConfigError::InvalidValue { + field: "retry.backoff_multiplier".to_string(), + value: retry.backoff_multiplier.to_string(), + reason: "Must be >= 1.0".to_string(), + }); + } + if !(0.0..=1.0).contains(&retry.jitter_factor) { + return Err(ConfigError::InvalidValue { + field: "retry.jitter_factor".to_string(), + value: retry.jitter_factor.to_string(), + reason: "Must be between 0.0 and 1.0".to_string(), + }); + } + Ok(()) + } + + fn validate_circuit_breaker(cb: &CircuitBreakerConfig) -> ConfigResult<()> { + if cb.failure_threshold < 1 { + return Err(ConfigError::InvalidValue { + field: "circuit_breaker.failure_threshold".to_string(), + value: cb.failure_threshold.to_string(), + reason: "Must be >= 1 (set to u32::MAX to effectively disable CB)".to_string(), + }); + } + if cb.success_threshold < 1 { + return Err(ConfigError::InvalidValue { + field: "circuit_breaker.success_threshold".to_string(), + value: cb.success_threshold.to_string(), + reason: "Must be >= 1".to_string(), + }); + } + if cb.timeout_duration_secs == 0 { + return Err(ConfigError::InvalidValue { + field: "circuit_breaker.timeout_duration_secs".to_string(), + value: cb.timeout_duration_secs.to_string(), + reason: "Must be > 0".to_string(), + }); + } + if cb.window_duration_secs == 0 { + return Err(ConfigError::InvalidValue { + field: "circuit_breaker.window_duration_secs".to_string(), + value: cb.window_duration_secs.to_string(), + reason: "Must be > 0".to_string(), + }); + } + Ok(()) + } + + fn validate_tokenizer_cache(cache: &TokenizerCacheConfig) -> ConfigResult<()> { + if cache.enable_l0 && cache.l0_max_entries == 0 { + return Err(ConfigError::InvalidValue { + field: "tokenizer_cache.l0_max_entries".to_string(), + value: cache.l0_max_entries.to_string(), + reason: "Must be > 0 when L0 cache is enabled".to_string(), + }); + } + + if cache.enable_l1 && cache.l1_max_memory == 0 { + return Err(ConfigError::InvalidValue { + field: "tokenizer_cache.l1_max_memory".to_string(), + value: cache.l1_max_memory.to_string(), + reason: "Must be > 0 when L1 cache is enabled".to_string(), + }); + } + + Ok(()) + } + + fn validate_mtls(config: &RouterConfig) -> ConfigResult<()> { + if let Some(identity) = &config.client_identity { + if identity.is_empty() { + return Err(ConfigError::ValidationFailed { + reason: "Client identity cannot be empty".to_string(), + }); + } + } + + for (idx, ca_cert) in config.ca_certificates.iter().enumerate() { + if ca_cert.is_empty() { + return Err(ConfigError::ValidationFailed { + reason: format!("CA certificate at index {} cannot be empty", idx), + }); + } + } + + Ok(()) + } + + fn validate_compatibility(config: &RouterConfig) -> ConfigResult<()> { + if config.enable_igw { + return Ok(()); + } + + if matches!(config.connection_mode, ConnectionMode::Grpc { .. }) + && config.tokenizer_path.is_none() + && config.model_path.is_none() + { + return Err(ConfigError::ValidationFailed { + reason: "gRPC connection mode requires either --tokenizer-path or --model-path to be specified".to_string(), + }); + } + + Self::validate_mtls(config)?; + + let has_service_discovery = config.discovery.as_ref().is_some_and(|d| d.enabled); + + if !has_service_discovery { + if let PolicyConfig::PowerOfTwo { .. } = &config.policy { + let worker_count = config.mode.worker_count(); + if worker_count < 2 { + return Err(ConfigError::IncompatibleConfig { + reason: "Power-of-two policy requires at least 2 workers".to_string(), + }); + } + } + + if let RoutingMode::PrefillDecode { + prefill_urls, + decode_urls, + prefill_policy, + decode_policy, + } = &config.mode + { + if let Some(PolicyConfig::PowerOfTwo { .. }) = prefill_policy { + if prefill_urls.len() < 2 { + return Err(ConfigError::IncompatibleConfig { + reason: "Power-of-two policy for prefill requires at least 2 prefill workers".to_string(), + }); + } + } + + if let Some(PolicyConfig::PowerOfTwo { .. }) = decode_policy { + if decode_urls.len() < 2 { + return Err(ConfigError::IncompatibleConfig { + reason: + "Power-of-two policy for decode requires at least 2 decode workers" + .to_string(), + }); + } + } + + // Check bucket for decode + if let Some(PolicyConfig::Bucket { .. }) = decode_policy { + return Err(ConfigError::IncompatibleConfig { + reason: "Decode policy should not be allowed to be bucket".to_string(), + }); + } + } + } + + if has_service_discovery && config.dp_aware { + return Err(ConfigError::IncompatibleConfig { + reason: "DP-aware routing is not compatible with service discovery".to_string(), + }); + } + + Ok(()) + } + + fn validate_urls(urls: &[String]) -> ConfigResult<()> { + for url in urls { + if url.is_empty() { + return Err(ConfigError::InvalidValue { + field: "worker_url".to_string(), + value: url.clone(), + reason: "URL cannot be empty".to_string(), + }); + } + + if !url.starts_with("http://") + && !url.starts_with("https://") + && !url.starts_with("grpc://") + { + return Err(ConfigError::InvalidValue { + field: "worker_url".to_string(), + value: url.clone(), + reason: "URL must start with http://, https://, or grpc://".to_string(), + }); + } + + match ::url::Url::parse(url) { + Ok(parsed) => { + if parsed.host_str().is_none() { + return Err(ConfigError::InvalidValue { + field: "worker_url".to_string(), + value: url.clone(), + reason: "URL must have a valid host".to_string(), + }); + } + } + Err(e) => { + return Err(ConfigError::InvalidValue { + field: "worker_url".to_string(), + value: url.clone(), + reason: format!("Invalid URL format: {}", e), + }); + } + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_validate_regular_mode() { + let config = RouterConfig::new( + RoutingMode::Regular { + worker_urls: vec!["http://worker:8000".to_string()], + }, + PolicyConfig::Random, + ); + + assert!(ConfigValidator::validate(&config).is_ok()); + } + + #[test] + fn test_validate_empty_worker_urls() { + let config = RouterConfig::new( + RoutingMode::Regular { + worker_urls: vec![], + }, + PolicyConfig::Random, + ); + + // Empty worker URLs are now allowed to match legacy behavior + assert!(ConfigValidator::validate(&config).is_ok()); + } + + #[test] + fn test_validate_empty_worker_urls_with_service_discovery() { + let mut config = RouterConfig::new( + RoutingMode::Regular { + worker_urls: vec![], + }, + PolicyConfig::Random, + ); + + // Enable service discovery + config.discovery = Some(DiscoveryConfig { + enabled: true, + selector: vec![("app".to_string(), "test".to_string())] + .into_iter() + .collect(), + ..Default::default() + }); + + // Should pass validation since service discovery is enabled + assert!(ConfigValidator::validate(&config).is_ok()); + } + + #[test] + fn test_validate_invalid_urls() { + let config = RouterConfig::new( + RoutingMode::Regular { + worker_urls: vec!["invalid-url".to_string()], + }, + PolicyConfig::Random, + ); + + assert!(ConfigValidator::validate(&config).is_err()); + } + + #[test] + fn test_validate_cache_aware_thresholds() { + let config = RouterConfig::new( + RoutingMode::Regular { + worker_urls: vec![ + "http://worker1:8000".to_string(), + "http://worker2:8000".to_string(), + ], + }, + PolicyConfig::CacheAware { + cache_threshold: 1.5, // Invalid: > 1.0 + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + eviction_interval_secs: 60, + max_tree_size: 1000, + }, + ); + + assert!(ConfigValidator::validate(&config).is_err()); + } + + #[test] + fn test_validate_cache_aware_single_worker() { + // Cache-aware with single worker should be allowed (even if not optimal) + let config = RouterConfig::new( + RoutingMode::Regular { + worker_urls: vec!["http://worker1:8000".to_string()], + }, + PolicyConfig::CacheAware { + cache_threshold: 0.5, + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + eviction_interval_secs: 60, + max_tree_size: 1000, + }, + ); + + assert!(ConfigValidator::validate(&config).is_ok()); + } + + #[test] + fn test_validate_pd_mode() { + let config = RouterConfig::new( + RoutingMode::PrefillDecode { + prefill_urls: vec![("http://prefill:8000".to_string(), Some(8081))], + decode_urls: vec!["http://decode:8000".to_string()], + prefill_policy: None, + decode_policy: None, + }, + PolicyConfig::Random, + ); + + assert!(ConfigValidator::validate(&config).is_ok()); + } + + #[test] + fn test_validate_roundrobin_with_pd_mode() { + // RoundRobin with PD mode is now supported + let config = RouterConfig::new( + RoutingMode::PrefillDecode { + prefill_urls: vec![("http://prefill:8000".to_string(), None)], + decode_urls: vec!["http://decode:8000".to_string()], + prefill_policy: None, + decode_policy: None, + }, + PolicyConfig::RoundRobin, + ); + + let result = ConfigValidator::validate(&config); + assert!(result.is_ok()); + } + + #[test] + fn test_validate_cache_aware_with_pd_mode() { + // CacheAware with PD mode is now supported + let config = RouterConfig::new( + RoutingMode::PrefillDecode { + prefill_urls: vec![("http://prefill:8000".to_string(), None)], + decode_urls: vec!["http://decode:8000".to_string()], + prefill_policy: None, + decode_policy: None, + }, + PolicyConfig::CacheAware { + cache_threshold: 0.5, + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + eviction_interval_secs: 60, + max_tree_size: 1000, + }, + ); + + let result = ConfigValidator::validate(&config); + assert!(result.is_ok()); + } + + #[test] + fn test_validate_power_of_two_with_regular_mode() { + // PowerOfTwo with Regular mode is now supported + let config = RouterConfig::new( + RoutingMode::Regular { + worker_urls: vec![ + "http://worker1:8000".to_string(), + "http://worker2:8000".to_string(), + ], + }, + PolicyConfig::PowerOfTwo { + load_check_interval_secs: 60, + }, + ); + + let result = ConfigValidator::validate(&config); + assert!(result.is_ok()); + } + + #[test] + fn test_validate_pd_mode_with_separate_policies() { + let config = RouterConfig::new( + RoutingMode::PrefillDecode { + prefill_urls: vec![ + ("http://prefill1:8000".to_string(), None), + ("http://prefill2:8000".to_string(), None), + ], + decode_urls: vec![ + "http://decode1:8000".to_string(), + "http://decode2:8000".to_string(), + ], + prefill_policy: Some(PolicyConfig::CacheAware { + cache_threshold: 0.5, + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + eviction_interval_secs: 60, + max_tree_size: 1000, + }), + decode_policy: Some(PolicyConfig::PowerOfTwo { + load_check_interval_secs: 60, + }), + }, + PolicyConfig::Random, // Main policy as fallback + ); + + let result = ConfigValidator::validate(&config); + assert!(result.is_ok()); + } + + #[test] + fn test_validate_pd_mode_power_of_two_insufficient_workers() { + let config = RouterConfig::new( + RoutingMode::PrefillDecode { + prefill_urls: vec![("http://prefill1:8000".to_string(), None)], // Only 1 prefill + decode_urls: vec![ + "http://decode1:8000".to_string(), + "http://decode2:8000".to_string(), + ], + prefill_policy: Some(PolicyConfig::PowerOfTwo { + load_check_interval_secs: 60, + }), // Requires 2+ workers + decode_policy: None, + }, + PolicyConfig::Random, + ); + + let result = ConfigValidator::validate(&config); + assert!(result.is_err()); + if let Err(e) = result { + assert!(e.to_string().contains("prefill requires at least 2")); + } + } + + #[test] + fn test_validate_pd_mode_bucket_policy_restrictions() { + let config = RouterConfig::new( + RoutingMode::PrefillDecode { + prefill_urls: vec![ + ("http://prefill1:8000".to_string(), None), + ("http://prefill2:8000".to_string(), None), + ], + decode_urls: vec![ + "http://decode1:8000".to_string(), + "http://decode2:8000".to_string(), + ], + prefill_policy: Some(PolicyConfig::Bucket { + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + bucket_adjust_interval_secs: 5, + }), + decode_policy: Some(PolicyConfig::PowerOfTwo { + load_check_interval_secs: 60, + }), + }, + PolicyConfig::Random, // Main policy as fallback + ); + + let result = ConfigValidator::validate(&config); + assert!( + result.is_ok(), + "Prefill policy should be allowed to be bucket" + ); + + let config = RouterConfig::new( + RoutingMode::PrefillDecode { + prefill_urls: vec![ + ("http://prefill1:8000".to_string(), None), + ("http://prefill2:8000".to_string(), None), + ], + decode_urls: vec![ + "http://decode1:8000".to_string(), + "http://decode2:8000".to_string(), + ], + prefill_policy: Some(PolicyConfig::Bucket { + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + bucket_adjust_interval_secs: 5, + }), + decode_policy: Some(PolicyConfig::Bucket { + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + bucket_adjust_interval_secs: 5, + }), + }, + PolicyConfig::Random, // Main policy as fallback + ); + + let result = ConfigValidator::validate(&config); + assert!( + result.is_err(), + "Decode policy should not be allowed to be bucket" + ); + } + + #[test] + fn test_validate_empty_urls_allowed_without_service_discovery() { + // Test that empty URLs are now allowed in PD mode + let config = RouterConfig::new( + RoutingMode::PrefillDecode { + prefill_urls: vec![], + decode_urls: vec![], + prefill_policy: None, + decode_policy: None, + }, + PolicyConfig::Random, + ); + + // Should pass validation even with empty URLs + assert!(ConfigValidator::validate(&config).is_ok()); + + // Test that empty URLs are allowed in Regular mode + let config = RouterConfig::new( + RoutingMode::Regular { + worker_urls: vec![], + }, + PolicyConfig::Random, + ); + + // Should pass validation even with empty URLs + assert!(ConfigValidator::validate(&config).is_ok()); + + // Test that empty URLs are allowed in OpenAI mode + let config = RouterConfig::new( + RoutingMode::OpenAI { + worker_urls: vec![], + }, + PolicyConfig::Random, + ); + + // Should pass validation even with empty URLs + assert!(ConfigValidator::validate(&config).is_ok()); + } + + #[test] + fn test_validate_grpc_requires_tokenizer() { + let mut config = RouterConfig::new( + RoutingMode::Regular { + worker_urls: vec!["grpc://worker:50051".to_string()], + }, + PolicyConfig::Random, + ); + + // Set connection mode to gRPC without tokenizer config + config.connection_mode = ConnectionMode::Grpc { port: None }; + config.tokenizer_path = None; + config.model_path = None; + + let result = ConfigValidator::validate(&config); + assert!(result.is_err()); + if let Err(e) = result { + assert!(e.to_string().contains("gRPC connection mode requires")); + } + } + + #[test] + fn test_validate_grpc_with_model_path() { + let mut config = RouterConfig::new( + RoutingMode::Regular { + worker_urls: vec!["grpc://worker:50051".to_string()], + }, + PolicyConfig::Random, + ); + + config.connection_mode = ConnectionMode::Grpc { port: None }; + config.model_path = Some("meta-llama/Llama-3-8B".to_string()); + + let result = ConfigValidator::validate(&config); + assert!(result.is_ok()); + } + + #[test] + fn test_validate_grpc_with_tokenizer_path() { + let mut config = RouterConfig::new( + RoutingMode::Regular { + worker_urls: vec!["grpc://worker:50051".to_string()], + }, + PolicyConfig::Random, + ); + + config.connection_mode = ConnectionMode::Grpc { port: None }; + config.tokenizer_path = Some("/path/to/tokenizer.json".to_string()); + + let result = ConfigValidator::validate(&config); + assert!(result.is_ok()); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/circuit_breaker.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/circuit_breaker.rs new file mode 100644 index 00000000..3a91f32f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/circuit_breaker.rs @@ -0,0 +1,523 @@ +use std::{ + sync::{ + atomic::{AtomicU32, AtomicU64, Ordering}, + Arc, RwLock, + }, + time::{Duration, Instant}, +}; + +use tracing::info; + +/// Circuit breaker configuration +#[derive(Debug, Clone)] +pub struct CircuitBreakerConfig { + /// Number of consecutive failures to open the circuit + pub failure_threshold: u32, + /// Success threshold to close circuit from half-open + pub success_threshold: u32, + /// Duration to wait before attempting half-open + pub timeout_duration: Duration, + /// Time window for failure counting + pub window_duration: Duration, +} + +impl Default for CircuitBreakerConfig { + fn default() -> Self { + Self { + failure_threshold: 5, + success_threshold: 2, + timeout_duration: Duration::from_secs(30), + window_duration: Duration::from_secs(60), + } + } +} + +/// Circuit breaker state +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CircuitState { + /// Normal operation - requests are allowed + Closed, + /// Circuit is open - requests are rejected + Open, + /// Testing if service has recovered - limited requests allowed + HalfOpen, +} + +impl std::fmt::Display for CircuitState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + CircuitState::Closed => write!(f, "Closed"), + CircuitState::Open => write!(f, "Open"), + CircuitState::HalfOpen => write!(f, "HalfOpen"), + } + } +} + +/// Circuit breaker implementation +#[derive(Debug)] +pub struct CircuitBreaker { + state: Arc>, + consecutive_failures: Arc, + consecutive_successes: Arc, + total_failures: Arc, + total_successes: Arc, + last_failure_time: Arc>>, + last_state_change: Arc>, + config: CircuitBreakerConfig, +} + +impl CircuitBreaker { + /// Create a new circuit breaker with default configuration + pub fn new() -> Self { + Self::with_config(CircuitBreakerConfig::default()) + } + + /// Create a new circuit breaker with custom configuration + pub fn with_config(config: CircuitBreakerConfig) -> Self { + Self { + state: Arc::new(RwLock::new(CircuitState::Closed)), + consecutive_failures: Arc::new(AtomicU32::new(0)), + consecutive_successes: Arc::new(AtomicU32::new(0)), + total_failures: Arc::new(AtomicU64::new(0)), + total_successes: Arc::new(AtomicU64::new(0)), + last_failure_time: Arc::new(RwLock::new(None)), + last_state_change: Arc::new(RwLock::new(Instant::now())), + config, + } + } + + /// Check if a request can be executed + pub fn can_execute(&self) -> bool { + self.check_and_update_state(); + + let state = *self.state.read().unwrap(); + match state { + CircuitState::Closed => true, + CircuitState::Open => false, + CircuitState::HalfOpen => true, + } + } + + /// Get the current state + pub fn state(&self) -> CircuitState { + self.check_and_update_state(); + *self.state.read().unwrap() + } + + /// Record the outcome of a request + pub fn record_outcome(&self, success: bool) { + if success { + self.record_success(); + } else { + self.record_failure(); + } + } + + /// Record a successful request + pub fn record_success(&self) { + self.total_successes.fetch_add(1, Ordering::Relaxed); + self.consecutive_failures.store(0, Ordering::Release); + let successes = self.consecutive_successes.fetch_add(1, Ordering::AcqRel) + 1; + + let current_state = *self.state.read().unwrap(); + + match current_state { + CircuitState::HalfOpen => { + if successes >= self.config.success_threshold { + self.transition_to(CircuitState::Closed); + } + } + CircuitState::Closed => {} + CircuitState::Open => { + tracing::warn!("Success recorded while circuit is open"); + } + } + } + + /// Record a failed request + pub fn record_failure(&self) { + self.total_failures.fetch_add(1, Ordering::Relaxed); + self.consecutive_successes.store(0, Ordering::Release); + let failures = self.consecutive_failures.fetch_add(1, Ordering::AcqRel) + 1; + + { + let mut last_failure = self.last_failure_time.write().unwrap(); + *last_failure = Some(Instant::now()); + } + + let current_state = *self.state.read().unwrap(); + + match current_state { + CircuitState::Closed => { + if failures >= self.config.failure_threshold { + self.transition_to(CircuitState::Open); + } + } + CircuitState::HalfOpen => { + self.transition_to(CircuitState::Open); + } + CircuitState::Open => {} + } + } + + /// Check and update state based on timeout + fn check_and_update_state(&self) { + let current_state = *self.state.read().unwrap(); + + if current_state == CircuitState::Open { + let last_change = *self.last_state_change.read().unwrap(); + if last_change.elapsed() >= self.config.timeout_duration { + self.transition_to(CircuitState::HalfOpen); + } + } + } + + /// Transition to a new state + fn transition_to(&self, new_state: CircuitState) { + let mut state = self.state.write().unwrap(); + let old_state = *state; + + if old_state != new_state { + *state = new_state; + + let mut last_change = self.last_state_change.write().unwrap(); + *last_change = Instant::now(); + + match new_state { + CircuitState::Closed => { + self.consecutive_failures.store(0, Ordering::Release); + self.consecutive_successes.store(0, Ordering::Release); + } + CircuitState::Open => { + self.consecutive_successes.store(0, Ordering::Release); + } + CircuitState::HalfOpen => { + self.consecutive_failures.store(0, Ordering::Release); + self.consecutive_successes.store(0, Ordering::Release); + } + } + + let from = match old_state { + CircuitState::Closed => "closed", + CircuitState::Open => "open", + CircuitState::HalfOpen => "half_open", + }; + let to = match new_state { + CircuitState::Closed => "closed", + CircuitState::Open => "open", + CircuitState::HalfOpen => "half_open", + }; + info!("Circuit breaker state transition: {} -> {}", from, to); + } + } + + /// Get the number of consecutive failures + pub fn failure_count(&self) -> u32 { + self.consecutive_failures.load(Ordering::Acquire) + } + + /// Get the number of consecutive successes + pub fn success_count(&self) -> u32 { + self.consecutive_successes.load(Ordering::Acquire) + } + + /// Get total failures + pub fn total_failures(&self) -> u64 { + self.total_failures.load(Ordering::Relaxed) + } + + /// Get total successes + pub fn total_successes(&self) -> u64 { + self.total_successes.load(Ordering::Relaxed) + } + + /// Get time since last failure + pub fn time_since_last_failure(&self) -> Option { + self.last_failure_time.read().unwrap().map(|t| t.elapsed()) + } + + /// Get time since last state change + pub fn time_since_last_state_change(&self) -> Duration { + self.last_state_change.read().unwrap().elapsed() + } + + /// Check if the circuit is in a half-open state + pub fn is_half_open(&self) -> bool { + self.state() == CircuitState::HalfOpen + } + + /// Record a test success (for health check probing) + pub fn record_test_success(&self) { + if self.is_half_open() { + self.record_success(); + } + } + + /// Record a test failure (for health check probing) + pub fn record_test_failure(&self) { + if self.is_half_open() { + self.record_failure(); + } + } + + /// Reset the circuit breaker to closed state + pub fn reset(&self) { + self.transition_to(CircuitState::Closed); + self.consecutive_failures.store(0, Ordering::Release); + self.consecutive_successes.store(0, Ordering::Release); + } + + /// Force the circuit to open (for manual intervention) + pub fn force_open(&self) { + self.transition_to(CircuitState::Open); + } + + /// Get circuit breaker statistics + pub fn stats(&self) -> CircuitBreakerStats { + CircuitBreakerStats { + state: self.state(), + consecutive_failures: self.failure_count(), + consecutive_successes: self.success_count(), + total_failures: self.total_failures(), + total_successes: self.total_successes(), + time_since_last_failure: self.time_since_last_failure(), + time_since_last_state_change: self.time_since_last_state_change(), + } + } +} + +impl Clone for CircuitBreaker { + fn clone(&self) -> Self { + Self { + state: Arc::clone(&self.state), + consecutive_failures: Arc::clone(&self.consecutive_failures), + consecutive_successes: Arc::clone(&self.consecutive_successes), + total_failures: Arc::clone(&self.total_failures), + total_successes: Arc::clone(&self.total_successes), + last_failure_time: Arc::clone(&self.last_failure_time), + last_state_change: Arc::clone(&self.last_state_change), + config: self.config.clone(), + } + } +} + +impl Default for CircuitBreaker { + fn default() -> Self { + Self::new() + } +} + +/// Circuit breaker statistics +#[derive(Debug, Clone)] +pub struct CircuitBreakerStats { + pub state: CircuitState, + pub consecutive_failures: u32, + pub consecutive_successes: u32, + pub total_failures: u64, + pub total_successes: u64, + pub time_since_last_failure: Option, + pub time_since_last_state_change: Duration, +} + +#[cfg(test)] +mod tests { + use std::thread; + + use super::*; + + #[test] + fn test_circuit_breaker_initial_state() { + let cb = CircuitBreaker::new(); + assert_eq!(cb.state(), CircuitState::Closed); + assert!(cb.can_execute()); + assert_eq!(cb.failure_count(), 0); + assert_eq!(cb.success_count(), 0); + } + + #[test] + fn test_circuit_opens_on_threshold() { + let config = CircuitBreakerConfig { + failure_threshold: 3, + ..Default::default() + }; + let cb = CircuitBreaker::with_config(config); + + assert_eq!(cb.state(), CircuitState::Closed); + cb.record_failure(); + assert_eq!(cb.state(), CircuitState::Closed); + cb.record_failure(); + assert_eq!(cb.state(), CircuitState::Closed); + cb.record_failure(); + + assert_eq!(cb.state(), CircuitState::Open); + assert!(!cb.can_execute()); + assert_eq!(cb.failure_count(), 3); + } + + #[test] + fn test_circuit_half_open_after_timeout() { + let config = CircuitBreakerConfig { + failure_threshold: 1, + timeout_duration: Duration::from_millis(100), + ..Default::default() + }; + let cb = CircuitBreaker::with_config(config); + + cb.record_failure(); + assert_eq!(cb.state(), CircuitState::Open); + + thread::sleep(Duration::from_millis(150)); + + assert_eq!(cb.state(), CircuitState::HalfOpen); + assert!(cb.can_execute()); + } + + #[test] + fn test_circuit_closes_on_success_threshold() { + let config = CircuitBreakerConfig { + failure_threshold: 1, + success_threshold: 2, + timeout_duration: Duration::from_millis(50), + ..Default::default() + }; + let cb = CircuitBreaker::with_config(config); + + cb.record_failure(); + assert_eq!(cb.state(), CircuitState::Open); + + thread::sleep(Duration::from_millis(100)); + assert_eq!(cb.state(), CircuitState::HalfOpen); + + cb.record_success(); + assert_eq!(cb.state(), CircuitState::HalfOpen); + cb.record_success(); + + assert_eq!(cb.state(), CircuitState::Closed); + assert!(cb.can_execute()); + } + + #[test] + fn test_circuit_reopens_on_half_open_failure() { + let config = CircuitBreakerConfig { + failure_threshold: 1, + timeout_duration: Duration::from_millis(50), + ..Default::default() + }; + let cb = CircuitBreaker::with_config(config); + + cb.record_failure(); + assert_eq!(cb.state(), CircuitState::Open); + + thread::sleep(Duration::from_millis(100)); + assert_eq!(cb.state(), CircuitState::HalfOpen); + + cb.record_failure(); + + assert_eq!(cb.state(), CircuitState::Open); + assert!(!cb.can_execute()); + } + + #[test] + fn test_success_resets_failure_count() { + let config = CircuitBreakerConfig { + failure_threshold: 3, + ..Default::default() + }; + let cb = CircuitBreaker::with_config(config); + + cb.record_failure(); + cb.record_failure(); + assert_eq!(cb.failure_count(), 2); + + cb.record_success(); + assert_eq!(cb.failure_count(), 0); + assert_eq!(cb.success_count(), 1); + + cb.record_failure(); + cb.record_failure(); + assert_eq!(cb.state(), CircuitState::Closed); + } + + #[test] + fn test_manual_reset() { + let config = CircuitBreakerConfig { + failure_threshold: 1, + ..Default::default() + }; + let cb = CircuitBreaker::with_config(config); + + cb.record_failure(); + assert_eq!(cb.state(), CircuitState::Open); + + cb.reset(); + assert_eq!(cb.state(), CircuitState::Closed); + assert_eq!(cb.failure_count(), 0); + assert_eq!(cb.success_count(), 0); + } + + #[test] + fn test_force_open() { + let cb = CircuitBreaker::new(); + assert_eq!(cb.state(), CircuitState::Closed); + + cb.force_open(); + assert_eq!(cb.state(), CircuitState::Open); + assert!(!cb.can_execute()); + } + + #[test] + fn test_stats() { + let config = CircuitBreakerConfig { + failure_threshold: 2, + ..Default::default() + }; + let cb = CircuitBreaker::with_config(config); + + cb.record_success(); + cb.record_failure(); + cb.record_failure(); + + let stats = cb.stats(); + assert_eq!(stats.state, CircuitState::Open); + assert_eq!(stats.consecutive_failures, 2); + assert_eq!(stats.consecutive_successes, 0); + assert_eq!(stats.total_failures, 2); + assert_eq!(stats.total_successes, 1); + } + + #[test] + fn test_clone() { + let cb1 = CircuitBreaker::new(); + cb1.record_failure(); + + let cb2 = cb1.clone(); + assert_eq!(cb2.failure_count(), 1); + + cb1.record_failure(); + assert_eq!(cb2.failure_count(), 2); + } + + #[test] + fn test_thread_safety() { + use std::sync::Arc; + + let cb = Arc::new(CircuitBreaker::new()); + let mut handles = vec![]; + + for _ in 0..10 { + let cb_clone = Arc::clone(&cb); + let handle = thread::spawn(move || { + for _ in 0..100 { + cb_clone.record_failure(); + } + }); + handles.push(handle); + } + + for handle in handles { + handle.join().unwrap(); + } + + assert_eq!(cb.total_failures(), 1000); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/error.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/error.rs new file mode 100644 index 00000000..740e9205 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/error.rs @@ -0,0 +1,236 @@ +//! Error types for the SGLang router core +//! +//! This module defines error types used throughout the router for worker operations. + +use std::fmt; + +/// Worker-related errors +#[derive(Debug)] +pub enum WorkerError { + /// Health check failed + HealthCheckFailed { url: String, reason: String }, + /// Worker not found + WorkerNotFound { url: String }, + /// Invalid worker configuration + InvalidConfiguration { message: String }, + /// Network error + NetworkError { url: String, error: String }, + /// Worker is at capacity + WorkerAtCapacity { url: String }, + /// Invalid URL format + InvalidUrl { url: String }, + /// Connection failed + ConnectionFailed { url: String, reason: String }, +} + +impl fmt::Display for WorkerError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + WorkerError::HealthCheckFailed { url, reason } => { + write!(f, "Health check failed for worker {}: {}", url, reason) + } + WorkerError::WorkerNotFound { url } => { + write!(f, "Worker not found: {}", url) + } + WorkerError::InvalidConfiguration { message } => { + write!(f, "Invalid worker configuration: {}", message) + } + WorkerError::NetworkError { url, error } => { + write!(f, "Network error for worker {}: {}", url, error) + } + WorkerError::WorkerAtCapacity { url } => { + write!(f, "Worker at capacity: {}", url) + } + WorkerError::InvalidUrl { url } => { + write!(f, "Invalid URL format: {}", url) + } + WorkerError::ConnectionFailed { url, reason } => { + write!(f, "Connection failed for worker {}: {}", url, reason) + } + } + } +} + +impl std::error::Error for WorkerError {} + +/// Result type for worker operations +pub type WorkerResult = Result; + +/// Convert from reqwest errors to worker errors +impl From for WorkerError { + fn from(err: reqwest::Error) -> Self { + WorkerError::NetworkError { + url: err.url().map(|u| u.to_string()).unwrap_or_default(), + error: err.to_string(), + } + } +} + +#[cfg(test)] +mod tests { + use std::error::Error; + + use super::*; + + #[test] + fn test_health_check_failed_display() { + let error = WorkerError::HealthCheckFailed { + url: "http://worker1:8080".to_string(), + reason: "Connection refused".to_string(), + }; + assert_eq!( + error.to_string(), + "Health check failed for worker http://worker1:8080: Connection refused" + ); + } + + #[test] + fn test_worker_not_found_display() { + let error = WorkerError::WorkerNotFound { + url: "http://worker2:8080".to_string(), + }; + assert_eq!(error.to_string(), "Worker not found: http://worker2:8080"); + } + + #[test] + fn test_invalid_configuration_display() { + let error = WorkerError::InvalidConfiguration { + message: "Missing port number".to_string(), + }; + assert_eq!( + error.to_string(), + "Invalid worker configuration: Missing port number" + ); + } + + #[test] + fn test_network_error_display() { + let error = WorkerError::NetworkError { + url: "http://worker3:8080".to_string(), + error: "Timeout after 30s".to_string(), + }; + assert_eq!( + error.to_string(), + "Network error for worker http://worker3:8080: Timeout after 30s" + ); + } + + #[test] + fn test_worker_at_capacity_display() { + let error = WorkerError::WorkerAtCapacity { + url: "http://worker4:8080".to_string(), + }; + assert_eq!(error.to_string(), "Worker at capacity: http://worker4:8080"); + } + + #[test] + fn test_worker_error_implements_std_error() { + let error = WorkerError::WorkerNotFound { + url: "http://test".to_string(), + }; + let _: &dyn Error = &error; + assert!(error.source().is_none()); + } + + #[test] + fn test_error_send_sync() { + fn assert_send_sync() {} + assert_send_sync::(); + } + + #[test] + fn test_worker_result_type_alias() { + let result: WorkerResult = Ok(42); + assert!(matches!(result, Ok(42))); + + let error = WorkerError::WorkerNotFound { + url: "test".to_string(), + }; + let result: WorkerResult = Err(error); + assert!(result.is_err()); + } + + #[test] + fn test_empty_url_handling() { + let error1 = WorkerError::HealthCheckFailed { + url: "".to_string(), + reason: "No connection".to_string(), + }; + assert_eq!( + error1.to_string(), + "Health check failed for worker : No connection" + ); + + let error2 = WorkerError::NetworkError { + url: "".to_string(), + error: "DNS failure".to_string(), + }; + assert_eq!(error2.to_string(), "Network error for worker : DNS failure"); + + let error3 = WorkerError::WorkerNotFound { + url: "".to_string(), + }; + assert_eq!(error3.to_string(), "Worker not found: "); + } + + #[test] + fn test_special_characters_in_messages() { + let error = WorkerError::InvalidConfiguration { + message: "Invalid JSON: {\"error\": \"test\"}".to_string(), + }; + assert_eq!( + error.to_string(), + "Invalid worker configuration: Invalid JSON: {\"error\": \"test\"}" + ); + + let error2 = WorkerError::HealthCheckFailed { + url: "http://测试:8080".to_string(), + reason: "连接被拒绝".to_string(), + }; + assert_eq!( + error2.to_string(), + "Health check failed for worker http://测试:8080: 连接被拒绝" + ); + } + + #[test] + fn test_very_long_error_messages() { + let long_message = "A".repeat(10000); + let error = WorkerError::InvalidConfiguration { + message: long_message.clone(), + }; + let display = error.to_string(); + assert!(display.contains(&long_message)); + assert_eq!( + display.len(), + "Invalid worker configuration: ".len() + long_message.len() + ); + } + + #[test] + fn test_reqwest_error_conversion() { + let network_error = WorkerError::NetworkError { + url: "http://example.com".to_string(), + error: "connection timeout".to_string(), + }; + + match network_error { + WorkerError::NetworkError { url, error } => { + assert_eq!(url, "http://example.com"); + assert_eq!(error, "connection timeout"); + } + _ => panic!("Expected NetworkError variant"), + } + } + + #[test] + fn test_error_equality() { + let error1 = WorkerError::WorkerNotFound { + url: "http://test".to_string(), + }; + let error2 = WorkerError::WorkerNotFound { + url: "http://test".to_string(), + }; + assert_eq!(error1.to_string(), error2.to_string()); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/job_queue.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/job_queue.rs new file mode 100644 index 00000000..b47fe391 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/job_queue.rs @@ -0,0 +1,734 @@ +//! Async job queue for control plane operations +//! +//! Provides non-blocking worker management by queuing operations and processing +//! them asynchronously in background worker tasks. + +use std::{ + collections::HashMap, + sync::{Arc, Weak}, + time::{Duration, SystemTime}, +}; + +use dashmap::DashMap; +use tokio::sync::{mpsc, Semaphore}; +use tracing::{debug, error, info, warn}; + +use crate::{ + app_context::AppContext, + config::{RouterConfig, RoutingMode}, + core::workflow::{ + steps::{McpServerConfigRequest, WorkerRemovalRequest}, + WorkflowContext, WorkflowEngine, WorkflowId, WorkflowInstanceId, WorkflowStatus, + }, + mcp::McpConfig, + metrics::RouterMetrics, + protocols::worker_spec::{JobStatus, WorkerConfigRequest}, +}; + +/// Job types for control plane operations +#[derive(Debug, Clone)] +pub enum Job { + AddWorker { config: Box }, + RemoveWorker { url: String }, + InitializeWorkersFromConfig { router_config: Box }, + InitializeMcpServers { mcp_config: Box }, + RegisterMcpServer { config: Box }, +} + +impl Job { + /// Get job type as string for logging + pub fn job_type(&self) -> &str { + match self { + Job::AddWorker { .. } => "AddWorker", + Job::RemoveWorker { .. } => "RemoveWorker", + Job::InitializeWorkersFromConfig { .. } => "InitializeWorkersFromConfig", + Job::InitializeMcpServers { .. } => "InitializeMcpServers", + Job::RegisterMcpServer { .. } => "RegisterMcpServer", + } + } + + /// Get worker URL or MCP server name for logging + pub fn worker_url(&self) -> &str { + match self { + Job::AddWorker { config } => &config.url, + Job::RemoveWorker { url } => url, + Job::InitializeWorkersFromConfig { .. } => "startup", + Job::InitializeMcpServers { .. } => "startup", + Job::RegisterMcpServer { config } => &config.name, + } + } +} + +impl JobStatus { + fn pending(job_type: &str, worker_url: &str) -> Self { + Self { + job_type: job_type.to_string(), + worker_url: worker_url.to_string(), + status: "pending".to_string(), + message: None, + timestamp: SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(), + } + } + + fn processing(job_type: &str, worker_url: &str) -> Self { + Self { + job_type: job_type.to_string(), + worker_url: worker_url.to_string(), + status: "processing".to_string(), + message: None, + timestamp: SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(), + } + } + + fn failed(job_type: &str, worker_url: &str, error: String) -> Self { + Self { + job_type: job_type.to_string(), + worker_url: worker_url.to_string(), + status: "failed".to_string(), + message: Some(error), + timestamp: SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(), + } + } +} + +/// Job queue configuration +#[derive(Clone, Debug)] +pub struct JobQueueConfig { + /// Maximum pending jobs in queue + pub queue_capacity: usize, + /// Maximum number of jobs executing concurrently + pub max_concurrent_jobs: usize, +} + +impl Default for JobQueueConfig { + fn default() -> Self { + Self { + queue_capacity: 1000, + max_concurrent_jobs: 10, + } + } +} + +/// Job queue manager for worker validation and removal operations +pub struct JobQueue { + /// Channel for submitting jobs + tx: mpsc::Sender, + /// Weak reference to AppContext to avoid circular dependencies + context: Weak, + /// Job status tracking by worker URL + status_map: Arc>, + /// Semaphore to limit concurrent job execution + concurrency_limit: Arc, +} + +impl std::fmt::Debug for JobQueue { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("JobQueue") + .field("status_count", &self.status_map.len()) + .finish() + } +} + +impl JobQueue { + /// Create a new job queue with semaphore-based concurrency control + /// + /// Takes a Weak reference to AppContext to avoid circular strong references. + /// Spawns a single dispatcher task that spawns individual job tasks with semaphore control. + pub fn new(config: JobQueueConfig, context: Weak) -> Arc { + let (tx, mut rx) = mpsc::channel(config.queue_capacity); + + debug!( + "Initializing job queue: capacity={}, max_concurrent={}", + config.queue_capacity, config.max_concurrent_jobs + ); + + let status_map = Arc::new(DashMap::new()); + let concurrency_limit = Arc::new(Semaphore::new(config.max_concurrent_jobs)); + + let queue = Arc::new(Self { + tx, + context: context.clone(), + status_map: status_map.clone(), + concurrency_limit: concurrency_limit.clone(), + }); + + // Single dispatcher task + let ctx = context.clone(); + let status = status_map.clone(); + let sem = concurrency_limit.clone(); + + tokio::spawn(async move { + while let Some(job) = rx.recv().await { + // Acquire permit (blocks if at concurrency limit) + let Ok(permit) = sem.clone().acquire_owned().await else { + error!("Semaphore closed, stopping dispatcher"); + break; + }; + + let ctx_clone = ctx.clone(); + let status_clone = status.clone(); + + tokio::spawn(async move { + Self::process_job(job, ctx_clone, status_clone, permit).await; + }); + } + + debug!("Job dispatcher stopped"); + }); + + // Spawn cleanup task for old job statuses (TTL 5 minutes) + let cleanup_status_map = status_map.clone(); + tokio::spawn(async move { + Self::cleanup_old_statuses(cleanup_status_map).await; + }); + + queue + } + + /// Get current queue and concurrency status + pub fn get_load_info(&self) -> (usize, usize) { + let queue_depth = self.tx.max_capacity() - self.tx.capacity(); + let available_permits = self.concurrency_limit.available_permits(); + (queue_depth, available_permits) + } + + /// Submit a job with detailed queue status + pub async fn submit(&self, job: Job) -> Result<(), String> { + // Check if context is still alive before accepting jobs + if self.context.upgrade().is_none() { + RouterMetrics::record_job_shutdown_rejected(); + return Err("Job queue shutting down: AppContext dropped".to_string()); + } + + // Extract values before moving job + let job_type = job.job_type().to_string(); + let worker_url = job.worker_url().to_string(); + + // Record pending status + self.status_map.insert( + worker_url.clone(), + JobStatus::pending(&job_type, &worker_url), + ); + + match self.tx.send(job).await { + Ok(_) => { + let (queue_depth, available_permits) = self.get_load_info(); + RouterMetrics::set_job_queue_depth(queue_depth); + debug!( + "Job submitted: type={}, worker={}, queue_depth={}, available_slots={}", + job_type, worker_url, queue_depth, available_permits + ); + Ok(()) + } + Err(_) => { + RouterMetrics::record_job_queue_full(); + self.status_map.remove(&worker_url); + let (queue_depth, _) = self.get_load_info(); + Err(format!( + "Job queue full: {} jobs pending (capacity: {})", + queue_depth, + self.tx.max_capacity() + )) + } + } + } + + /// Get job status by worker URL + pub fn get_status(&self, worker_url: &str) -> Option { + self.status_map.get(worker_url).map(|entry| entry.clone()) + } + + /// Remove job status (called when worker is deleted) + pub fn remove_status(&self, worker_url: &str) { + self.status_map.remove(worker_url); + } + + /// Process a single job with status tracking and error handling + async fn process_job( + job: Job, + context: Weak, + status_map: Arc>, + _permit: tokio::sync::OwnedSemaphorePermit, + ) { + let job_type = job.job_type().to_string(); + let worker_url = job.worker_url().to_string(); + let start = std::time::Instant::now(); + + // Update to processing + status_map.insert( + worker_url.clone(), + JobStatus::processing(&job_type, &worker_url), + ); + + debug!("Processing job: type={}, worker={}", job_type, worker_url); + + // Execute job + match context.upgrade() { + Some(ctx) => { + let result = Self::execute_job(&job, &ctx).await; + let duration = start.elapsed(); + Self::record_job_completion(&job_type, &worker_url, duration, &result, &status_map); + } + None => { + let error_msg = "AppContext dropped".to_string(); + status_map.insert( + worker_url.clone(), + JobStatus::failed(&job_type, &worker_url, error_msg), + ); + error!( + "AppContext dropped, cannot process job: type={}, worker={}", + job_type, worker_url + ); + } + } + + // Permit automatically released when dropped + } + + /// Execute a specific job + async fn execute_job(job: &Job, context: &Arc) -> Result { + match job { + Job::AddWorker { config } => { + let engine = context + .workflow_engine + .get() + .ok_or_else(|| "Workflow engine not initialized".to_string())?; + + let instance_id = Self::start_worker_workflow(engine, config, context).await?; + + debug!( + "Started worker registration workflow for {} (instance: {})", + config.url, instance_id + ); + + let timeout_duration = + Duration::from_secs(context.router_config.worker_startup_timeout_secs + 30); + + Self::wait_for_workflow_completion( + engine, + instance_id, + &config.url, + timeout_duration, + ) + .await + } + Job::RemoveWorker { url } => { + let engine = context + .workflow_engine + .get() + .ok_or_else(|| "Workflow engine not initialized".to_string())?; + + let instance_id = Self::start_worker_removal_workflow(engine, url, context).await?; + + debug!( + "Started worker removal workflow for {} (instance: {})", + url, instance_id + ); + + let timeout_duration = Duration::from_secs(30); + + let result = + Self::wait_for_workflow_completion(engine, instance_id, url, timeout_duration) + .await; + + // Clean up job status when removing worker + if let Some(queue) = context.worker_job_queue.get() { + queue.remove_status(url); + } + + result + } + Job::InitializeWorkersFromConfig { router_config } => { + let api_key = router_config.api_key.clone(); + let mut worker_count = 0; + + // Create iterator of (url, worker_type, bootstrap_port) tuples based on mode + let workers: Vec<(String, &str, Option)> = match &router_config.mode { + RoutingMode::Regular { worker_urls } => worker_urls + .iter() + .map(|url| (url.clone(), "regular", None)) + .collect(), + RoutingMode::PrefillDecode { + prefill_urls, + decode_urls, + .. + } => { + let prefill_workers = prefill_urls + .iter() + .map(|(url, port)| (url.clone(), "prefill", *port)); + + let decode_workers = + decode_urls.iter().map(|url| (url.clone(), "decode", None)); + + prefill_workers.chain(decode_workers).collect() + } + RoutingMode::OpenAI { worker_urls } => { + // OpenAI mode: submit AddWorker jobs with runtime: "external" + // The external_worker_registration workflow handles model discovery + let api_key = router_config.api_key.clone(); + let mut submitted_count = 0; + + for url in worker_urls { + let url_for_error = url.clone(); + let config = WorkerConfigRequest { + url: url.clone(), + api_key: api_key.clone(), + worker_type: Some("regular".to_string()), + labels: HashMap::new(), + model_id: None, + priority: None, + cost: None, + runtime: Some("external".to_string()), + tokenizer_path: None, + reasoning_parser: None, + tool_parser: None, + chat_template: None, + bootstrap_port: None, + health_check_timeout_secs: router_config.health_check.timeout_secs, + health_check_interval_secs: router_config + .health_check + .check_interval_secs, + health_success_threshold: router_config + .health_check + .success_threshold, + health_failure_threshold: router_config + .health_check + .failure_threshold, + max_connection_attempts: router_config + .health_check + .success_threshold + * 10, + dp_aware: false, + }; + + let job = Job::AddWorker { + config: Box::new(config), + }; + + if let Some(queue) = context.worker_job_queue.get() { + queue.submit(job).await.map_err(|e| { + format!( + "Failed to submit AddWorker job for external endpoint {}: {}", + url_for_error, e + ) + })?; + submitted_count += 1; + } else { + return Err("JobQueue not available".to_string()); + } + } + + if submitted_count == 0 { + info!("OpenAI mode: no worker URLs provided"); + return Ok("OpenAI mode: no worker URLs to initialize".to_string()); + } + + return Ok(format!( + "Submitted {} AddWorker jobs for external endpoints", + submitted_count + )); + } + }; + + debug!( + "Creating AddWorker jobs for {} workers from config", + workers.len() + ); + + // Process all workers with unified loop + for (url, worker_type, bootstrap_port) in workers { + let url_for_error = url.clone(); // Clone for error message + let config = WorkerConfigRequest { + url, + api_key: api_key.clone(), + worker_type: Some(worker_type.to_string()), + labels: HashMap::new(), + model_id: None, + priority: None, + cost: None, + runtime: None, + tokenizer_path: None, + reasoning_parser: None, + tool_parser: None, + chat_template: None, + bootstrap_port, + health_check_timeout_secs: router_config.health_check.timeout_secs, + health_check_interval_secs: router_config.health_check.check_interval_secs, + health_success_threshold: router_config.health_check.success_threshold, + health_failure_threshold: router_config.health_check.failure_threshold, + max_connection_attempts: router_config.health_check.success_threshold * 10, + dp_aware: router_config.dp_aware, + }; + + let job = Job::AddWorker { + config: Box::new(config), + }; + + if let Some(queue) = context.worker_job_queue.get() { + queue.submit(job).await.map_err(|e| { + format!( + "Failed to submit AddWorker job for {} worker {}: {}", + worker_type, url_for_error, e + ) + })?; + worker_count += 1; + } else { + return Err("JobQueue not available".to_string()); + } + } + + Ok(format!("Submitted {} AddWorker jobs", worker_count)) + } + Job::InitializeMcpServers { mcp_config } => { + let mut server_count = 0; + + debug!( + "Creating RegisterMcpServer jobs for {} MCP servers from config", + mcp_config.servers.len() + ); + + // Submit RegisterMcpServer jobs for each server in the config + for server_config in &mcp_config.servers { + let mcp_server_request = McpServerConfigRequest { + name: server_config.name.clone(), + config: server_config.clone(), + }; + + let job = Job::RegisterMcpServer { + config: Box::new(mcp_server_request), + }; + + if let Some(queue) = context.worker_job_queue.get() { + queue.submit(job).await.map_err(|e| { + format!( + "Failed to submit RegisterMcpServer job for '{}': {}", + server_config.name, e + ) + })?; + server_count += 1; + } else { + return Err("JobQueue not available".to_string()); + } + } + + Ok(format!("Submitted {} RegisterMcpServer jobs", server_count)) + } + Job::RegisterMcpServer { config } => { + let engine = context + .workflow_engine + .get() + .ok_or_else(|| "Workflow engine not initialized".to_string())?; + + let instance_id = + Self::start_mcp_registration_workflow(engine, config, context).await?; + + debug!( + "Started MCP registration workflow for {} (instance: {})", + config.name, instance_id + ); + + let timeout_duration = Duration::from_secs(7200 + 30); // 2hr + margin + + Self::wait_for_workflow_completion( + engine, + instance_id, + &config.name, + timeout_duration, + ) + .await + } + } + } + + /// Start a workflow and return its instance ID + async fn start_worker_workflow( + engine: &Arc, + config: &WorkerConfigRequest, + context: &Arc, + ) -> Result { + let mut workflow_context = WorkflowContext::new(WorkflowInstanceId::new()); + workflow_context.set("worker_config", config.clone()); + workflow_context.set_arc("app_context", Arc::clone(context)); + + // Select workflow based on runtime field + let workflow_id = match config.runtime.as_deref() { + Some("external") => WorkflowId::new("external_worker_registration"), + _ => WorkflowId::new("worker_registration"), + }; + + engine + .start_workflow(workflow_id, workflow_context) + .await + .map_err(|e| format!("Failed to start worker registration workflow: {:?}", e)) + } + + /// Start worker removal workflow + async fn start_worker_removal_workflow( + engine: &Arc, + url: &str, + context: &Arc, + ) -> Result { + let removal_request = WorkerRemovalRequest { + url: url.to_string(), + dp_aware: context.router_config.dp_aware, + }; + + let mut workflow_context = WorkflowContext::new(WorkflowInstanceId::new()); + workflow_context.set("removal_request", removal_request); + workflow_context.set_arc("app_context", Arc::clone(context)); + + engine + .start_workflow(WorkflowId::new("worker_removal"), workflow_context) + .await + .map_err(|e| format!("Failed to start worker removal workflow: {:?}", e)) + } + + /// Start MCP server registration workflow + async fn start_mcp_registration_workflow( + engine: &Arc, + config: &McpServerConfigRequest, + context: &Arc, + ) -> Result { + let mut workflow_context = WorkflowContext::new(WorkflowInstanceId::new()); + workflow_context.set("mcp_server_config", config.clone()); + workflow_context.set_arc("app_context", Arc::clone(context)); + + engine + .start_workflow(WorkflowId::new("mcp_registration"), workflow_context) + .await + .map_err(|e| format!("Failed to start MCP registration workflow: {:?}", e)) + } + + /// Wait for workflow completion with adaptive polling + async fn wait_for_workflow_completion( + engine: &Arc, + instance_id: WorkflowInstanceId, + worker_url: &str, + timeout_duration: Duration, + ) -> Result { + let start = std::time::Instant::now(); + let mut poll_interval = Duration::from_millis(100); + let max_poll_interval = Duration::from_millis(2000); + let poll_backoff = Duration::from_millis(200); + + loop { + // Check timeout + if start.elapsed() > timeout_duration { + return Err(format!( + "Workflow timeout after {}s for worker {}", + timeout_duration.as_secs(), + worker_url + )); + } + + // Get workflow status + let state = engine + .get_status(instance_id) + .map_err(|e| format!("Failed to get workflow status: {:?}", e))?; + + let result = match state.status { + WorkflowStatus::Completed => Ok(format!( + "Worker {} registered and activated successfully via workflow", + worker_url + )), + WorkflowStatus::Failed => { + let current_step = state.current_step.as_ref(); + let step_name = current_step + .map(|s| s.to_string()) + .unwrap_or_else(|| "unknown".to_string()); + let error_msg = current_step + .and_then(|step_id| state.step_states.get(step_id)) + .and_then(|s| s.last_error.as_deref()) + .unwrap_or("Unknown error"); + Err(format!( + "Workflow failed at step {}: {}", + step_name, error_msg + )) + } + WorkflowStatus::Cancelled => { + Err(format!("Workflow cancelled for worker {}", worker_url)) + } + WorkflowStatus::Pending | WorkflowStatus::Paused | WorkflowStatus::Running => { + tokio::time::sleep(poll_interval).await; + poll_interval = (poll_interval + poll_backoff).min(max_poll_interval); + continue; + } + }; + + // Clean up terminal workflow states + engine.state_store().cleanup_if_terminal(instance_id); + return result; + } + } + + /// Record job completion metrics and update status + fn record_job_completion( + job_type: &str, + worker_url: &str, + duration: Duration, + result: &Result, + status_map: &Arc>, + ) { + RouterMetrics::record_job_duration(job_type, duration); + + match result { + Ok(message) => { + RouterMetrics::record_job_success(job_type); + status_map.remove(worker_url); + debug!( + "Completed job: type={}, worker={}, duration={:.3}s, result={}", + job_type, + worker_url, + duration.as_secs_f64(), + message + ); + } + Err(error) => { + RouterMetrics::record_job_failure(job_type); + status_map.insert( + worker_url.to_string(), + JobStatus::failed(job_type, worker_url, error.clone()), + ); + warn!( + "Failed job: type={}, worker={}, duration={:.3}s, error={}", + job_type, + worker_url, + duration.as_secs_f64(), + error + ); + } + } + } + + /// Cleanup old job statuses (TTL 5 minutes) + async fn cleanup_old_statuses(status_map: Arc>) { + const CLEANUP_INTERVAL: Duration = Duration::from_secs(60); // Run every minute + const STATUS_TTL: u64 = 300; // 5 minutes in seconds + + loop { + tokio::time::sleep(CLEANUP_INTERVAL).await; + + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(); + + // Remove statuses older than TTL + status_map.retain(|_key, value| now - value.timestamp < STATUS_TTL); + + debug!( + "Cleaned up old job statuses, remaining: {}", + status_map.len() + ); + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/metrics_aggregator.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/metrics_aggregator.rs new file mode 100644 index 00000000..36073609 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/metrics_aggregator.rs @@ -0,0 +1,91 @@ +use anyhow::ensure; +use openmetrics_parser::{MetricFamily, MetricsExposition, PrometheusType, PrometheusValue}; +use tracing::warn; + +#[derive(Debug)] +pub struct MetricPack { + pub labels: Vec<(String, String)>, + pub metrics_text: String, +} + +type PrometheusExposition = MetricsExposition; +type PrometheusFamily = MetricFamily; + +/// Aggregate Prometheus metrics scraped from multiple sources into a unified one +pub fn aggregate_metrics(metric_packs: Vec) -> anyhow::Result { + let mut expositions = vec![]; + for metric_pack in metric_packs { + let metrics_text = &metric_pack.metrics_text; + // Hacky workaround since the parser do not understand `:`, should improve later + let metrics_text = metrics_text.replace(":", "_"); + + let exposition = match openmetrics_parser::prometheus::parse_prometheus(&metrics_text) { + Ok(x) => x, + Err(err) => { + warn!( + "aggregate_metrics error when parsing text: pack={:?} err={:?}", + metric_pack, err + ); + continue; + } + }; + let exposition = transform_metrics(exposition, &metric_pack.labels); + expositions.push(exposition); + } + + let text = try_reduce(expositions.into_iter(), merge_exposition)? + .map(|x| format!("{x}")) + .unwrap_or_default(); + Ok(text) +} + +fn transform_metrics( + mut exposition: PrometheusExposition, + extra_labels: &[(String, String)], +) -> PrometheusExposition { + for family in exposition.families.values_mut() { + *family = family.with_labels(extra_labels.iter().map(|(k, v)| (k.as_str(), v.as_str()))); + } + exposition +} + +fn merge_exposition( + a: PrometheusExposition, + b: PrometheusExposition, +) -> anyhow::Result { + let mut ans = a; + for (name, family_b) in b.families.into_iter() { + let family_merged = if let Some(family_a) = ans.families.remove(&name) { + merge_family(family_a, family_b)? + } else { + family_b + }; + ans.families.insert(name, family_merged); + } + Ok(ans) +} + +fn merge_family(a: PrometheusFamily, b: PrometheusFamily) -> anyhow::Result { + ensure!( + a.get_label_names() == b.get_label_names(), + "Label names should agree a={:?} b={:?}", + a.get_label_names(), + b.get_label_names() + ); + a.with_samples(b.into_iter_samples()) + .map_err(|e| anyhow::anyhow!("failed to merge samples: {e:?}")) +} + +pub fn try_reduce(iterable: I, f: F) -> Result, E> +where + I: IntoIterator, + F: FnMut(T, T) -> Result, +{ + let mut it = iterable.into_iter(); + let first = match it.next() { + None => return Ok(None), + Some(x) => x, + }; + + Ok(Some(it.try_fold(first, f)?)) +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/mod.rs new file mode 100644 index 00000000..ab7191d8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/mod.rs @@ -0,0 +1,40 @@ +//! Core abstractions for the SGLang router +//! +//! This module contains the fundamental types and traits used throughout the router: +//! - Worker trait and implementations +//! - Model types and endpoint definitions +//! - Error types +//! - Circuit breaker for reliability +//! - Token buckets for rate limiting +//! - Workflow engine for multi-step operations +//! - Common utilities + +pub mod circuit_breaker; +pub mod error; +pub mod job_queue; +pub mod metrics_aggregator; +pub mod model_card; +pub mod model_type; +pub mod retry; +pub mod token_bucket; +pub mod worker; +pub mod worker_builder; +pub mod worker_manager; +pub mod worker_registry; +pub mod workflow; + +pub use circuit_breaker::{ + CircuitBreaker, CircuitBreakerConfig, CircuitBreakerStats, CircuitState, +}; +pub use error::{WorkerError, WorkerResult}; +pub use job_queue::{Job, JobQueue, JobQueueConfig}; +pub use model_card::{ModelCard, ProviderType}; +pub use model_type::{Endpoint, ModelType}; +pub use retry::{is_retryable_status, BackoffCalculator, RetryError, RetryExecutor}; +pub use worker::{ + worker_to_info, BasicWorker, ConnectionMode, DPAwareWorker, HealthChecker, HealthConfig, + RuntimeType, Worker, WorkerFactory, WorkerLoadGuard, WorkerType, +}; +pub use worker_builder::{BasicWorkerBuilder, DPAwareWorkerBuilder}; +pub use worker_manager::{LoadMonitor, WorkerManager}; +pub use worker_registry::{WorkerId, WorkerRegistry, WorkerRegistryStats}; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/model_card.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/model_card.rs new file mode 100644 index 00000000..f1d31bcb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/model_card.rs @@ -0,0 +1,323 @@ +//! Model card definitions for worker model configuration. +//! +//! This module defines [`ModelCard`] which consolidates model-related configuration +//! that was previously scattered in `WorkerMetadata.labels` HashMap. +//! +//! Also defines [`ProviderType`] for vendor-specific API transformations. +//! +//! Inspired by Dynamo's ModelDeploymentCard but simplified for router needs. + +use serde::{Deserialize, Serialize}; + +use super::model_type::{Endpoint, ModelType}; + +/// Provider type for external API transformations. +/// +/// Different providers have different API formats and requirements. +/// This enum identifies which vendor's API format to use for transformations. +/// +/// Note: `None` (when used as `Option`) means native/passthrough - +/// no transformation needed. This is the case for local SGLang backends. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ProviderType { + /// OpenAI API - strip SGLang-specific fields + #[serde(alias = "openai")] + OpenAI, + /// xAI/Grok - special handling for input items + #[serde(alias = "xai", alias = "grok")] + XAI, + /// Anthropic Claude - different API format + #[serde(alias = "anthropic", alias = "claude")] + Anthropic, + /// Google Gemini - special logprobs handling + #[serde(alias = "gemini", alias = "google")] + Gemini, + /// Custom provider with string identifier + #[serde(untagged)] + Custom(String), +} + +impl ProviderType { + /// Get provider name as string + pub fn as_str(&self) -> &str { + match self { + Self::OpenAI => "openai", + Self::XAI => "xai", + Self::Anthropic => "anthropic", + Self::Gemini => "gemini", + Self::Custom(s) => s.as_str(), + } + } + + /// Detect provider from model name (heuristic fallback). + /// Returns `None` for models that don't match known external providers + /// (i.e., models served by local/native backends). + pub fn from_model_name(model: &str) -> Option { + let model_lower = model.to_lowercase(); + if model_lower.starts_with("grok") { + Some(Self::XAI) + } else if model_lower.starts_with("gemini") { + Some(Self::Gemini) + } else if model_lower.starts_with("claude") { + Some(Self::Anthropic) + } else if model_lower.starts_with("gpt") + || model_lower.starts_with("o1") + || model_lower.starts_with("o3") + { + Some(Self::OpenAI) + } else { + None // Native/local model, no provider transformation needed + } + } +} + +impl std::fmt::Display for ProviderType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.as_str()) + } +} + +/// Model card containing model configuration and capabilities. +/// +/// Consolidates fields previously scattered in `WorkerMetadata.labels`: +/// - `model_id` -> `id` +/// - `tokenizer_path` -> `tokenizer_path` +/// - `chat_template` -> `chat_template` +/// - `reasoning_parser` -> `reasoning_parser` +/// - `tool_parser` -> `tool_parser` +/// +/// # Example +/// +/// ``` +/// use sgl_model_gateway::core::{ModelCard, ModelType, ProviderType}; +/// +/// let card = ModelCard::new("meta-llama/Llama-3.1-8B-Instruct") +/// .with_display_name("Llama 3.1 8B Instruct") +/// .with_alias("llama-3.1-8b") +/// .with_model_type(ModelType::VISION_LLM) +/// .with_context_length(128_000) +/// .with_tokenizer_path("meta-llama/Llama-3.1-8B-Instruct"); +/// +/// assert!(card.matches("llama-3.1-8b")); +/// assert!(card.model_type.supports_vision()); +/// assert!(card.provider.is_none()); // Local model, no external provider +/// ``` +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ModelCard { + // === Identity === + /// Primary model ID (e.g., "meta-llama/Llama-3.1-8B-Instruct") + /// Previously: labels.get("model_id") + pub id: String, + + /// Optional display name (e.g., "Llama 3.1 8B Instruct") + #[serde(default, skip_serializing_if = "Option::is_none")] + pub display_name: Option, + + /// Alternative names/aliases for this model + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub aliases: Vec, + + // === Capabilities === + /// Supported endpoint types (bitflags) + #[serde(default = "default_model_type")] + pub model_type: ModelType, + + /// Provider hint for API transformations. + /// `None` means native/passthrough (no transformation needed). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider: Option, + + /// Maximum context length in tokens + #[serde(default, skip_serializing_if = "Option::is_none")] + pub context_length: Option, + + // === Tokenization & Parsing (previously in labels) === + /// Path to tokenizer (e.g., HuggingFace model ID or local path) + /// Previously: labels.get("tokenizer_path") + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tokenizer_path: Option, + + /// Chat template (Jinja2 template string or path) + /// Previously: labels.get("chat_template") + #[serde(default, skip_serializing_if = "Option::is_none")] + pub chat_template: Option, + + /// Reasoning parser type (e.g., "deepseek", "qwen") + /// Previously: labels.get("reasoning_parser") + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning_parser: Option, + + /// Tool/function calling parser type (e.g., "llama", "mistral") + /// Previously: labels.get("tool_parser") + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_parser: Option, + + /// User-defined metadata (for fields not covered above) + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metadata: Option, +} + +fn default_model_type() -> ModelType { + ModelType::LLM +} + +impl ModelCard { + /// Create a new model card with minimal configuration. + /// + /// Defaults to `ModelType::LLM` and no provider (native/passthrough). + pub fn new(id: impl Into) -> Self { + Self { + id: id.into(), + display_name: None, + aliases: Vec::new(), + model_type: ModelType::LLM, + provider: None, + context_length: None, + tokenizer_path: None, + chat_template: None, + reasoning_parser: None, + tool_parser: None, + metadata: None, + } + } + + // === Builder-style methods === + + /// Set the display name + pub fn with_display_name(mut self, name: impl Into) -> Self { + self.display_name = Some(name.into()); + self + } + + /// Add a single alias + pub fn with_alias(mut self, alias: impl Into) -> Self { + self.aliases.push(alias.into()); + self + } + + /// Add multiple aliases + pub fn with_aliases(mut self, aliases: impl IntoIterator>) -> Self { + self.aliases.extend(aliases.into_iter().map(|a| a.into())); + self + } + + /// Set the model type (capabilities) + pub fn with_model_type(mut self, model_type: ModelType) -> Self { + self.model_type = model_type; + self + } + + /// Set the provider type (for external API transformations) + pub fn with_provider(mut self, provider: ProviderType) -> Self { + self.provider = Some(provider); + self + } + + /// Set the context length + pub fn with_context_length(mut self, length: u32) -> Self { + self.context_length = Some(length); + self + } + + /// Set the tokenizer path + pub fn with_tokenizer_path(mut self, path: impl Into) -> Self { + self.tokenizer_path = Some(path.into()); + self + } + + /// Set the chat template + pub fn with_chat_template(mut self, template: impl Into) -> Self { + self.chat_template = Some(template.into()); + self + } + + /// Set the reasoning parser type + pub fn with_reasoning_parser(mut self, parser: impl Into) -> Self { + self.reasoning_parser = Some(parser.into()); + self + } + + /// Set the tool parser type + pub fn with_tool_parser(mut self, parser: impl Into) -> Self { + self.tool_parser = Some(parser.into()); + self + } + + /// Set custom metadata + pub fn with_metadata(mut self, metadata: serde_json::Value) -> Self { + self.metadata = Some(metadata); + self + } + + // === Query methods === + + /// Check if this model matches the given ID (including aliases) + pub fn matches(&self, model_id: &str) -> bool { + self.id == model_id || self.aliases.iter().any(|a| a == model_id) + } + + /// Check if this model supports a given endpoint + pub fn supports_endpoint(&self, endpoint: Endpoint) -> bool { + self.model_type.supports_endpoint(endpoint) + } + + /// Get the display name or fall back to ID + pub fn name(&self) -> &str { + self.display_name.as_deref().unwrap_or(&self.id) + } + + /// Check if this is a native/local model (no external provider) + #[inline] + pub fn is_native(&self) -> bool { + self.provider.is_none() + } + + /// Check if this model uses an external provider + #[inline] + pub fn has_external_provider(&self) -> bool { + self.provider.is_some() + } + + /// Check if this is an LLM (supports chat) + #[inline] + pub fn is_llm(&self) -> bool { + self.model_type.is_llm() + } + + /// Check if this is an embedding model + #[inline] + pub fn is_embedding_model(&self) -> bool { + self.model_type.is_embedding_model() + } + + /// Check if this model supports vision/multimodal + #[inline] + pub fn supports_vision(&self) -> bool { + self.model_type.supports_vision() + } + + /// Check if this model supports tools/function calling + #[inline] + pub fn supports_tools(&self) -> bool { + self.model_type.supports_tools() + } + + /// Check if this model supports reasoning + #[inline] + pub fn supports_reasoning(&self) -> bool { + self.model_type.supports_reasoning() + } +} + +impl Default for ModelCard { + fn default() -> Self { + Self::new("default") + } +} + +impl std::fmt::Display for ModelCard { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.name()) + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/model_type.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/model_type.rs new file mode 100644 index 00000000..9e938197 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/model_type.rs @@ -0,0 +1,332 @@ +//! Model type definitions using bitflags for endpoint support. +//! +//! This module defines [`ModelType`] using bitflags to represent which endpoints +//! a model can support. This allows combining capabilities like +//! `ModelType::CHAT | ModelType::COMPLETIONS`. +//! +//! Inspired by Dynamo's model_type.rs implementation. + +use bitflags::bitflags; +use serde::{Deserialize, Serialize}; + +bitflags! { + #[derive(Copy, Debug, Default, Clone, Eq, PartialEq, Hash)] + pub struct ModelType: u16 { + /// OpenAI Chat Completions API (/v1/chat/completions) + const CHAT = 1 << 0; + /// OpenAI Completions API - legacy (/v1/completions) + const COMPLETIONS = 1 << 1; + /// OpenAI Responses API (/v1/responses) + const RESPONSES = 1 << 2; + /// Embeddings API (/v1/embeddings) + const EMBEDDINGS = 1 << 3; + /// Rerank API (/v1/rerank) + const RERANK = 1 << 4; + /// SGLang Generate API (/generate) + const GENERATE = 1 << 5; + /// Vision/multimodal support (images in input) + const VISION = 1 << 6; + /// Tool/function calling support + const TOOLS = 1 << 7; + /// Reasoning/thinking support (e.g., o1, DeepSeek-R1) + const REASONING = 1 << 8; + /// Image generation (DALL-E, Sora, gpt-image) + const IMAGE_GEN = 1 << 9; + /// Audio models (TTS, Whisper, realtime, transcribe) + const AUDIO = 1 << 10; + /// Content moderation models + const MODERATION = 1 << 11; + + /// Standard LLM: chat + completions + responses + tools + const LLM = Self::CHAT.bits() | Self::COMPLETIONS.bits() + | Self::RESPONSES.bits() | Self::TOOLS.bits(); + + /// Vision-capable LLM: LLM + vision + const VISION_LLM = Self::LLM.bits() | Self::VISION.bits(); + + /// Reasoning LLM: LLM + reasoning (e.g., o1, o3, DeepSeek-R1) + const REASONING_LLM = Self::LLM.bits() | Self::REASONING.bits(); + + /// Full-featured LLM: all text generation capabilities + const FULL_LLM = Self::VISION_LLM.bits() | Self::REASONING.bits(); + + /// Embedding model only + const EMBED_MODEL = Self::EMBEDDINGS.bits(); + + /// Reranker model only + const RERANK_MODEL = Self::RERANK.bits(); + + /// Image generation model only (DALL-E, Sora, gpt-image) + const IMAGE_MODEL = Self::IMAGE_GEN.bits(); + + /// Audio model only (TTS, Whisper, realtime) + const AUDIO_MODEL = Self::AUDIO.bits(); + + /// Content moderation model only + const MODERATION_MODEL = Self::MODERATION.bits(); + } +} + +/// Mapping of individual capability flags to their names. +/// Used by `as_capability_names()` for a data-driven approach. +const CAPABILITY_NAMES: &[(ModelType, &str)] = &[ + (ModelType::CHAT, "chat"), + (ModelType::COMPLETIONS, "completions"), + (ModelType::RESPONSES, "responses"), + (ModelType::EMBEDDINGS, "embeddings"), + (ModelType::RERANK, "rerank"), + (ModelType::GENERATE, "generate"), + (ModelType::VISION, "vision"), + (ModelType::TOOLS, "tools"), + (ModelType::REASONING, "reasoning"), + (ModelType::IMAGE_GEN, "image_gen"), + (ModelType::AUDIO, "audio"), + (ModelType::MODERATION, "moderation"), +]; + +impl ModelType { + /// Check if this model type supports the chat completions endpoint + #[inline] + pub fn supports_chat(&self) -> bool { + self.contains(Self::CHAT) + } + + /// Check if this model type supports the legacy completions endpoint + #[inline] + pub fn supports_completions(&self) -> bool { + self.contains(Self::COMPLETIONS) + } + + /// Check if this model type supports the responses endpoint + #[inline] + pub fn supports_responses(&self) -> bool { + self.contains(Self::RESPONSES) + } + + /// Check if this model type supports the embeddings endpoint + #[inline] + pub fn supports_embeddings(&self) -> bool { + self.contains(Self::EMBEDDINGS) + } + + /// Check if this model type supports the rerank endpoint + #[inline] + pub fn supports_rerank(&self) -> bool { + self.contains(Self::RERANK) + } + + /// Check if this model type supports the generate endpoint + #[inline] + pub fn supports_generate(&self) -> bool { + self.contains(Self::GENERATE) + } + + /// Check if this model type supports vision/multimodal input + #[inline] + pub fn supports_vision(&self) -> bool { + self.contains(Self::VISION) + } + + /// Check if this model type supports tool/function calling + #[inline] + pub fn supports_tools(&self) -> bool { + self.contains(Self::TOOLS) + } + + /// Check if this model type supports reasoning/thinking + #[inline] + pub fn supports_reasoning(&self) -> bool { + self.contains(Self::REASONING) + } + + /// Check if this model type supports image generation + #[inline] + pub fn supports_image_gen(&self) -> bool { + self.contains(Self::IMAGE_GEN) + } + + /// Check if this model type supports audio (TTS, Whisper, etc.) + #[inline] + pub fn supports_audio(&self) -> bool { + self.contains(Self::AUDIO) + } + + /// Check if this model type supports content moderation + #[inline] + pub fn supports_moderation(&self) -> bool { + self.contains(Self::MODERATION) + } + + /// Check if this model type supports a given endpoint + pub fn supports_endpoint(&self, endpoint: Endpoint) -> bool { + match endpoint { + Endpoint::Chat => self.supports_chat(), + Endpoint::Completions => self.supports_completions(), + Endpoint::Responses => self.supports_responses(), + Endpoint::Embeddings => self.supports_embeddings(), + Endpoint::Rerank => self.supports_rerank(), + Endpoint::Generate => self.supports_generate(), + Endpoint::Models => true, // Models endpoint is always supported + } + } + + /// Convert to a list of supported capability names + pub fn as_capability_names(&self) -> Vec<&'static str> { + let mut result = Vec::with_capacity(CAPABILITY_NAMES.len()); + for &(flag, name) in CAPABILITY_NAMES { + if self.contains(flag) { + result.push(name); + } + } + result + } + + /// Check if this is an LLM (supports at least chat) + #[inline] + pub fn is_llm(&self) -> bool { + self.supports_chat() + } + + /// Check if this is an embedding model + #[inline] + pub fn is_embedding_model(&self) -> bool { + self.supports_embeddings() && !self.supports_chat() + } + + /// Check if this is a reranker model + #[inline] + pub fn is_reranker(&self) -> bool { + self.supports_rerank() && !self.supports_chat() + } + + /// Check if this is an image generation model + #[inline] + pub fn is_image_model(&self) -> bool { + self.supports_image_gen() && !self.supports_chat() + } + + /// Check if this is an audio model + #[inline] + pub fn is_audio_model(&self) -> bool { + self.supports_audio() && !self.supports_chat() + } + + /// Check if this is a moderation model + #[inline] + pub fn is_moderation_model(&self) -> bool { + self.supports_moderation() && !self.supports_chat() + } +} + +impl std::fmt::Display for ModelType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let names = self.as_capability_names(); + if names.is_empty() { + write!(f, "none") + } else { + write!(f, "{}", names.join(",")) + } + } +} + +// Custom Serialize/Deserialize for ModelType to handle bitflags properly +impl Serialize for ModelType { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + // Serialize as the underlying u16 bits + serializer.serialize_u16(self.bits()) + } +} + +impl<'de> Deserialize<'de> for ModelType { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let bits = u16::deserialize(deserializer)?; + ModelType::from_bits(bits) + .ok_or_else(|| serde::de::Error::custom(format!("invalid ModelType bits: {}", bits))) + } +} + +/// Endpoint types for routing decisions. +/// +/// This enum represents the different API endpoints that can be routed to workers. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Endpoint { + /// Chat completions endpoint (/v1/chat/completions) + Chat, + /// Legacy completions endpoint (/v1/completions) + Completions, + /// Responses endpoint (/v1/responses) + Responses, + /// Embeddings endpoint (/v1/embeddings) + Embeddings, + /// Rerank endpoint (/v1/rerank) + Rerank, + /// SGLang generate endpoint (/generate) + Generate, + /// Models listing endpoint (/v1/models) + Models, +} + +impl Endpoint { + /// Get the URL path for this endpoint + pub fn path(&self) -> &'static str { + match self { + Endpoint::Chat => "/v1/chat/completions", + Endpoint::Completions => "/v1/completions", + Endpoint::Responses => "/v1/responses", + Endpoint::Embeddings => "/v1/embeddings", + Endpoint::Rerank => "/v1/rerank", + Endpoint::Generate => "/generate", + Endpoint::Models => "/v1/models", + } + } + + /// Parse an endpoint from a URL path + pub fn from_path(path: &str) -> Option { + // Normalize: strip trailing slash and match + let path = path.trim_end_matches('/'); + match path { + "/v1/chat/completions" => Some(Endpoint::Chat), + "/v1/completions" => Some(Endpoint::Completions), + "/v1/responses" => Some(Endpoint::Responses), + "/v1/embeddings" => Some(Endpoint::Embeddings), + "/v1/rerank" => Some(Endpoint::Rerank), + "/generate" => Some(Endpoint::Generate), + "/v1/models" => Some(Endpoint::Models), + _ => None, + } + } + + /// Get the required ModelType flag for this endpoint + pub fn required_capability(&self) -> Option { + match self { + Endpoint::Chat => Some(ModelType::CHAT), + Endpoint::Completions => Some(ModelType::COMPLETIONS), + Endpoint::Responses => Some(ModelType::RESPONSES), + Endpoint::Embeddings => Some(ModelType::EMBEDDINGS), + Endpoint::Rerank => Some(ModelType::RERANK), + Endpoint::Generate => Some(ModelType::GENERATE), + Endpoint::Models => None, // No specific capability required + } + } +} + +impl std::fmt::Display for Endpoint { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Endpoint::Chat => write!(f, "chat"), + Endpoint::Completions => write!(f, "completions"), + Endpoint::Responses => write!(f, "responses"), + Endpoint::Embeddings => write!(f, "embeddings"), + Endpoint::Rerank => write!(f, "rerank"), + Endpoint::Generate => write!(f, "generate"), + Endpoint::Models => write!(f, "models"), + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/retry.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/retry.rs new file mode 100644 index 00000000..d2ced03a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/retry.rs @@ -0,0 +1,401 @@ +use std::time::Duration; + +use axum::{http::StatusCode, response::Response}; +use rand::Rng; +use tracing::debug; + +use crate::config::types::RetryConfig; + +/// Check if an HTTP status code indicates a retryable error +pub fn is_retryable_status(status: StatusCode) -> bool { + matches!( + status, + StatusCode::REQUEST_TIMEOUT + | StatusCode::TOO_MANY_REQUESTS + | StatusCode::INTERNAL_SERVER_ERROR + | StatusCode::BAD_GATEWAY + | StatusCode::SERVICE_UNAVAILABLE + | StatusCode::GATEWAY_TIMEOUT + ) +} + +/// Computes exponential backoff with optional jitter. +#[derive(Debug, Clone)] +pub struct BackoffCalculator; + +impl BackoffCalculator { + /// Calculate backoff delay for a given attempt index (0-based). + pub fn calculate_delay(config: &RetryConfig, attempt: u32) -> Duration { + let pow = config.backoff_multiplier.powi(attempt as i32); + let mut delay_ms = (config.initial_backoff_ms as f32 * pow) as u64; + if delay_ms > config.max_backoff_ms { + delay_ms = config.max_backoff_ms; + } + + let jitter = config.jitter_factor.clamp(0.0, 1.0); + if jitter > 0.0 { + let mut rng = rand::rng(); + let jitter_scale: f32 = rng.random_range(-jitter..=jitter); + let jitter_ms = (delay_ms as f32 * jitter_scale) + .round() + .max(-(delay_ms as f32)); + let adjusted = (delay_ms as i64 + jitter_ms as i64).max(0) as u64; + return Duration::from_millis(adjusted); + } + + Duration::from_millis(delay_ms) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum RetryError { + #[error("no available workers")] + NoAvailableWorkers, + #[error("maximum retry attempts exceeded")] + MaxRetriesExceeded, +} + +/// A thin async retry executor for generic operations. +#[derive(Debug, Clone, Default)] +pub struct RetryExecutor; + +impl RetryExecutor { + /// Execute an async operation with retries and backoff. + /// The `operation` closure is invoked each attempt with the attempt index. + pub async fn execute_with_retry( + config: &RetryConfig, + mut operation: F, + ) -> Result + where + F: FnMut(u32) -> Fut, + Fut: std::future::Future>, + { + let max = config.max_retries.max(1); + let mut attempt: u32 = 0; + loop { + match operation(attempt).await { + Ok(val) => return Ok(val), + Err(_) => { + let is_last = attempt + 1 >= max; + if is_last { + return Err(RetryError::MaxRetriesExceeded); + } + let delay = BackoffCalculator::calculate_delay(config, attempt); + attempt += 1; + tokio::time::sleep(delay).await; + } + } + } + } + + /// Execute an operation that returns an HTTP Response with retries and backoff. + /// + /// Usage pattern: + /// - `operation(attempt)`: perform one attempt (0-based). Construct and send the request, + /// then return the `Response`. Do any per-attempt bookkeeping (e.g., load tracking, + /// circuit-breaker outcome recording) inside this closure. + /// - `should_retry(&response, attempt)`: decide if the given response should be retried + /// (e.g., based on HTTP status). Returning false short-circuits and returns the response. + /// - `on_backoff(delay, next_attempt)`: called before sleeping between attempts. + /// Use this to record metrics. + /// - `on_exhausted()`: called when the executor has exhausted all retry attempts. + /// + /// Example: + /// ```ignore + /// let resp = RetryExecutor::execute_response_with_retry( + /// &retry_cfg, + /// |attempt| async move { + /// let worker = select_cb_aware_worker()?; + /// let resp = send_request(worker).await; + /// worker.record_outcome(resp.status().is_success()); + /// resp + /// }, + /// |res, _| matches!(res.status(), StatusCode::REQUEST_TIMEOUT | StatusCode::TOO_MANY_REQUESTS | StatusCode::INTERNAL_SERVER_ERROR | StatusCode::BAD_GATEWAY | StatusCode::SERVICE_UNAVAILABLE | StatusCode::GATEWAY_TIMEOUT), + /// |delay, attempt| RouterMetrics::record_retry_backoff_duration(delay, attempt), + /// || RouterMetrics::record_retries_exhausted("/route"), + /// ).await; + /// ``` + pub async fn execute_response_with_retry( + config: &RetryConfig, + mut operation: Op, + should_retry: ShouldRetry, + on_backoff: OnBackoff, + mut on_exhausted: OnExhausted, + ) -> Response + where + Op: FnMut(u32) -> Fut, + Fut: std::future::Future, + ShouldRetry: Fn(&Response, u32) -> bool, + OnBackoff: Fn(Duration, u32), + OnExhausted: FnMut(), + { + let max = config.max_retries.max(1); + + let mut attempt: u32 = 0; + loop { + let response = operation(attempt).await; + let is_last = attempt + 1 >= max; + + if !should_retry(&response, attempt) { + return response; + } + + if is_last { + on_exhausted(); + return response; + } + + let next_attempt = attempt + 1; + let delay = BackoffCalculator::calculate_delay(config, attempt); + debug!( + attempt = attempt, + next_attempt = next_attempt, + delay_ms = delay.as_millis() as u64, + "Retry backoff" + ); + on_backoff(delay, next_attempt); + tokio::time::sleep(delay).await; + + attempt = next_attempt; + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::{ + atomic::{AtomicU32, Ordering}, + Arc, + }; + + use axum::{http::StatusCode, response::IntoResponse}; + + use super::*; + + fn base_retry_config() -> RetryConfig { + RetryConfig { + max_retries: 3, + initial_backoff_ms: 1, + max_backoff_ms: 4, + backoff_multiplier: 2.0, + jitter_factor: 0.0, + } + } + + #[test] + fn test_backoff_no_jitter_progression_and_cap() { + let cfg = RetryConfig { + max_retries: 10, + initial_backoff_ms: 100, + max_backoff_ms: 250, + backoff_multiplier: 2.0, + jitter_factor: 0.0, + }; + assert_eq!( + BackoffCalculator::calculate_delay(&cfg, 0), + Duration::from_millis(100) + ); + assert_eq!( + BackoffCalculator::calculate_delay(&cfg, 1), + Duration::from_millis(200) + ); + assert_eq!( + BackoffCalculator::calculate_delay(&cfg, 2), + Duration::from_millis(250) + ); + assert_eq!( + BackoffCalculator::calculate_delay(&cfg, 10), + Duration::from_millis(250) + ); + } + + #[test] + fn test_backoff_with_jitter_within_bounds() { + let cfg = RetryConfig { + max_retries: 5, + initial_backoff_ms: 100, + max_backoff_ms: 10_000, + backoff_multiplier: 2.0, + jitter_factor: 0.5, + }; + let base = 400.0; + for _ in 0..50 { + let d = BackoffCalculator::calculate_delay(&cfg, 2).as_millis() as f32; + assert!(d >= base * 0.5 - 1.0 && d <= base * 1.5 + 1.0); + } + } + + #[tokio::test] + async fn test_execute_with_retry_success_after_failures() { + let cfg = base_retry_config(); + let remaining = Arc::new(AtomicU32::new(2)); + let calls = Arc::new(AtomicU32::new(0)); + + let res: Result = RetryExecutor::execute_with_retry(&cfg, { + let remaining = remaining.clone(); + let calls = calls.clone(); + move |_attempt| { + calls.fetch_add(1, Ordering::Relaxed); + let remaining = remaining.clone(); + async move { + if remaining + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |v| v.checked_sub(1)) + .is_ok() + { + Err(()) + } else { + Ok(42u32) + } + } + } + }) + .await; + + assert!(res.is_ok()); + assert_eq!(res.unwrap(), 42); + assert_eq!(calls.load(Ordering::Relaxed), 3); + } + + #[tokio::test] + async fn test_execute_with_retry_exhausted() { + let cfg = base_retry_config(); + let calls = Arc::new(AtomicU32::new(0)); + let res: Result = RetryExecutor::execute_with_retry(&cfg, { + let calls = calls.clone(); + move |_attempt| { + calls.fetch_add(1, Ordering::Relaxed); + async move { Err(()) } + } + }) + .await; + + assert!(matches!(res, Err(RetryError::MaxRetriesExceeded))); + assert_eq!(calls.load(Ordering::Relaxed), cfg.max_retries); + } + + #[tokio::test] + async fn test_execute_response_with_retry_success_path_and_hooks() { + let cfg = base_retry_config(); + let remaining = Arc::new(AtomicU32::new(2)); + let calls = Arc::new(AtomicU32::new(0)); + let backoffs = Arc::new(AtomicU32::new(0)); + let exhausted = Arc::new(AtomicU32::new(0)); + + let response = RetryExecutor::execute_response_with_retry( + &cfg, + { + let remaining = remaining.clone(); + let calls = calls.clone(); + move |_attempt| { + calls.fetch_add(1, Ordering::Relaxed); + let remaining = remaining.clone(); + async move { + if remaining + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |v| v.checked_sub(1)) + .is_ok() + { + (StatusCode::SERVICE_UNAVAILABLE, "fail").into_response() + } else { + (StatusCode::OK, "ok").into_response() + } + } + } + }, + |res, _attempt| !res.status().is_success(), + { + let backoffs = backoffs.clone(); + move |_delay, _next_attempt| { + backoffs.fetch_add(1, Ordering::Relaxed); + } + }, + { + let exhausted = exhausted.clone(); + move || { + exhausted.fetch_add(1, Ordering::Relaxed); + } + }, + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(calls.load(Ordering::Relaxed), 3); + assert_eq!(backoffs.load(Ordering::Relaxed), 2); + assert_eq!(exhausted.load(Ordering::Relaxed), 0); + } + + #[tokio::test] + async fn test_execute_response_with_retry_non_retryable_short_circuit() { + let cfg = base_retry_config(); + let calls = Arc::new(AtomicU32::new(0)); + let backoffs = Arc::new(AtomicU32::new(0)); + let exhausted = Arc::new(AtomicU32::new(0)); + + let response = RetryExecutor::execute_response_with_retry( + &cfg, + { + let calls = calls.clone(); + move |_attempt| { + calls.fetch_add(1, Ordering::Relaxed); + async move { (StatusCode::BAD_REQUEST, "bad").into_response() } + } + }, + |_res, _attempt| false, + { + let backoffs = backoffs.clone(); + move |_delay, _next_attempt| { + backoffs.fetch_add(1, Ordering::Relaxed); + } + }, + { + let exhausted = exhausted.clone(); + move || { + exhausted.fetch_add(1, Ordering::Relaxed); + } + }, + ) + .await; + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert_eq!(calls.load(Ordering::Relaxed), 1); + assert_eq!(backoffs.load(Ordering::Relaxed), 0); + assert_eq!(exhausted.load(Ordering::Relaxed), 0); + } + + #[tokio::test] + async fn test_execute_response_with_retry_exhausted_hooks() { + let cfg = base_retry_config(); + let calls = Arc::new(AtomicU32::new(0)); + let backoffs = Arc::new(AtomicU32::new(0)); + let exhausted = Arc::new(AtomicU32::new(0)); + + let response = RetryExecutor::execute_response_with_retry( + &cfg, + { + let calls = calls.clone(); + move |_attempt| { + calls.fetch_add(1, Ordering::Relaxed); + async move { (StatusCode::SERVICE_UNAVAILABLE, "fail").into_response() } + } + }, + |_res, _attempt| true, + { + let backoffs = backoffs.clone(); + move |_delay, _next_attempt| { + backoffs.fetch_add(1, Ordering::Relaxed); + } + }, + { + let exhausted = exhausted.clone(); + move || { + exhausted.fetch_add(1, Ordering::Relaxed); + } + }, + ) + .await; + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(calls.load(Ordering::Relaxed), cfg.max_retries); + assert_eq!(backoffs.load(Ordering::Relaxed), cfg.max_retries - 1); + assert_eq!(exhausted.load(Ordering::Relaxed), 1); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/token_bucket.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/token_bucket.rs new file mode 100644 index 00000000..a2bddeb6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/token_bucket.rs @@ -0,0 +1,243 @@ +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use tokio::sync::{Mutex, Notify}; +use tracing::{debug, trace}; + +/// Token bucket for rate limiting +/// +/// This implementation provides: +/// - Smooth rate limiting with configurable refill rate +/// - Burst capacity handling +/// - Fair queuing for waiting requests +#[derive(Clone)] +pub struct TokenBucket { + inner: Arc>, + notify: Arc, + capacity: f64, + refill_rate: f64, // tokens per second +} + +struct TokenBucketInner { + tokens: f64, + last_refill: Instant, +} + +impl TokenBucket { + /// Create a new token bucket + /// + /// # Arguments + /// * `capacity` - Maximum number of tokens (burst capacity) + /// * `refill_rate` - Tokens added per second + pub fn new(capacity: usize, refill_rate: usize) -> Self { + let capacity = capacity as f64; + // Allow refill_rate=0 for pure concurrency limiting (semaphore behavior) + // When refill_rate=0, tokens are only returned via return_tokens() + let refill_rate = refill_rate as f64; + + Self { + inner: Arc::new(Mutex::new(TokenBucketInner { + tokens: capacity, + last_refill: Instant::now(), + })), + notify: Arc::new(Notify::new()), + capacity, + refill_rate, + } + } + + /// Try to acquire tokens immediately + pub async fn try_acquire(&self, tokens: f64) -> Result<(), ()> { + let mut inner = self.inner.lock().await; + + let now = Instant::now(); + let elapsed = now.duration_since(inner.last_refill).as_secs_f64(); + let refill_amount = elapsed * self.refill_rate; + + inner.tokens = (inner.tokens + refill_amount).min(self.capacity); + inner.last_refill = now; + + trace!( + "Token bucket: {} tokens available, requesting {}", + inner.tokens, + tokens + ); + + if inner.tokens >= tokens { + inner.tokens -= tokens; + debug!( + "Token bucket: acquired {} tokens, {} remaining", + tokens, inner.tokens + ); + Ok(()) + } else { + Err(()) + } + } + + /// Acquire tokens, waiting if necessary + pub async fn acquire(&self, tokens: f64) -> Result<(), tokio::time::error::Elapsed> { + if self.try_acquire(tokens).await.is_ok() { + return Ok(()); + } + + // When refill_rate=0 (pure concurrency limiting), tokens only come back + // via return_tokens(), so we wait indefinitely on notify signal. + // The caller should use acquire_timeout() to set an appropriate timeout. + if self.refill_rate == 0.0 { + debug!( + "Token bucket: waiting indefinitely for {} tokens (refill_rate=0)", + tokens + ); + + loop { + if self.try_acquire(tokens).await.is_ok() { + return Ok(()); + } + + // Wait for notify signal from return_tokens() + self.notify.notified().await; + } + } + + let wait_time = { + let inner = self.inner.lock().await; + let tokens_needed = tokens - inner.tokens; + let wait_secs = tokens_needed / self.refill_rate; + Duration::from_secs_f64(wait_secs) + }; + + debug!( + "Token bucket: waiting {:?} for {} tokens", + wait_time, tokens + ); + + tokio::time::timeout(wait_time, async { + loop { + if self.try_acquire(tokens).await.is_ok() { + return; + } + + tokio::select! { + _ = self.notify.notified() => {}, + _ = tokio::time::sleep(Duration::from_millis(10)) => {}, + } + } + }) + .await?; + + Ok(()) + } + + /// Acquire tokens with custom timeout + pub async fn acquire_timeout( + &self, + tokens: f64, + timeout: Duration, + ) -> Result<(), tokio::time::error::Elapsed> { + tokio::time::timeout(timeout, self.acquire(tokens)).await? + } + + /// Return tokens to the bucket (for cancelled requests) + pub async fn return_tokens(&self, tokens: f64) { + let mut inner = self.inner.lock().await; + inner.tokens = (inner.tokens + tokens).min(self.capacity); + self.notify.notify_waiters(); + debug!( + "Token bucket: returned {} tokens, {} available", + tokens, inner.tokens + ); + } + + /// Get current available tokens (for monitoring) + pub async fn available_tokens(&self) -> f64 { + let mut inner = self.inner.lock().await; + + let now = Instant::now(); + let elapsed = now.duration_since(inner.last_refill).as_secs_f64(); + let refill_amount = elapsed * self.refill_rate; + + inner.tokens = (inner.tokens + refill_amount).min(self.capacity); + inner.last_refill = now; + + inner.tokens + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_token_bucket_basic() { + let bucket = TokenBucket::new(10, 5); + + assert!(bucket.try_acquire(5.0).await.is_ok()); + assert!(bucket.try_acquire(5.0).await.is_ok()); + + assert!(bucket.try_acquire(1.0).await.is_err()); + + tokio::time::sleep(Duration::from_millis(300)).await; + + assert!(bucket.try_acquire(1.0).await.is_ok()); + } + + #[tokio::test] + async fn test_token_bucket_refill() { + let bucket = TokenBucket::new(10, 10); + + assert!(bucket.try_acquire(10.0).await.is_ok()); + + tokio::time::sleep(Duration::from_millis(500)).await; + + let available = bucket.available_tokens().await; + assert!((4.0..=6.0).contains(&available)); + } + + #[tokio::test] + async fn test_token_bucket_zero_refill_rate() { + // With refill_rate=0, tokens should only come back via return_tokens() + let bucket = TokenBucket::new(2, 0); + + // Acquire both tokens + assert!(bucket.try_acquire(1.0).await.is_ok()); + assert!(bucket.try_acquire(1.0).await.is_ok()); + + // No more tokens available + assert!(bucket.try_acquire(1.0).await.is_err()); + + // Wait - should NOT refill automatically + tokio::time::sleep(Duration::from_millis(500)).await; + assert!(bucket.try_acquire(1.0).await.is_err()); + + // Return a token - now we should be able to acquire + bucket.return_tokens(1.0).await; + assert!(bucket.try_acquire(1.0).await.is_ok()); + + // No more tokens again + assert!(bucket.try_acquire(1.0).await.is_err()); + } + + #[tokio::test] + async fn test_token_bucket_zero_refill_with_notify() { + // Test that acquire wakes up when tokens are returned + let bucket = Arc::new(TokenBucket::new(1, 0)); + + // Acquire the only token + assert!(bucket.try_acquire(1.0).await.is_ok()); + + let bucket_clone = bucket.clone(); + + // Spawn a task that will return the token after a delay + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(100)).await; + bucket_clone.return_tokens(1.0).await; + }); + + // This should wait and then succeed when token is returned + let result = bucket.acquire_timeout(1.0, Duration::from_secs(1)).await; + assert!(result.is_ok()); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/worker.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/worker.rs new file mode 100644 index 00000000..c1457df5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/worker.rs @@ -0,0 +1,1945 @@ +use std::{ + fmt, + sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, LazyLock, + }, + time::{Duration, Instant}, +}; + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use serde_json; +use tokio::{sync::RwLock, time}; + +use super::{ + CircuitBreaker, Endpoint, ModelCard, ModelType, ProviderType, WorkerError, WorkerResult, +}; +use crate::{ + core::{BasicWorkerBuilder, CircuitState, DPAwareWorkerBuilder}, + metrics::RouterMetrics, + protocols::worker_spec::WorkerInfo, + routers::grpc::client::GrpcClient, +}; + +static WORKER_CLIENT: LazyLock = LazyLock::new(|| { + reqwest::Client::builder() + .timeout(Duration::from_secs(30)) + .build() + .expect("Failed to create worker HTTP client") +}); + +/// Core worker abstraction that represents a backend service +#[async_trait] +pub trait Worker: Send + Sync + fmt::Debug { + /// Get the worker's URL + fn url(&self) -> &str; + /// Get the worker's API key + fn api_key(&self) -> &Option; + /// Get the worker's type (Regular, Prefill, or Decode) + fn worker_type(&self) -> WorkerType; + + /// Get the worker's connection mode (HTTP or gRPC) + fn connection_mode(&self) -> ConnectionMode; + + /// Get the bootstrap hostname for PD mode + /// Returns cached hostname parsed from URL at construction time + fn bootstrap_host(&self) -> &str { + &self.metadata().bootstrap_host + } + + /// Get the bootstrap port for PD mode + /// Returns cached port from WorkerType::Prefill + fn bootstrap_port(&self) -> Option { + self.metadata().bootstrap_port + } + + /// Check if the worker is currently healthy + fn is_healthy(&self) -> bool; + + /// Set the worker's health status + fn set_healthy(&self, healthy: bool); + + /// Perform an async health check on the worker + async fn check_health_async(&self) -> WorkerResult<()>; + + /// Synchronous health check wrapper (for compatibility) + fn check_health(&self) -> WorkerResult<()> { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|e| WorkerError::HealthCheckFailed { + url: self.url().to_string(), + reason: format!("Failed to create runtime: {}", e), + })? + .block_on(self.check_health_async()) + } + + /// Get the current load (number of active requests) + fn load(&self) -> usize; + + /// Increment the load counter + fn increment_load(&self); + + /// Decrement the load counter + fn decrement_load(&self); + + /// Reset the load counter to 0 (for sync/recovery) + fn reset_load(&self) {} + + /// Get the number of processed requests + fn processed_requests(&self) -> usize; + + /// Increment the processed requests counter + fn increment_processed(&self); + + /// Get worker-specific metadata + fn metadata(&self) -> &WorkerMetadata; + + /// Get the circuit breaker for this worker + fn circuit_breaker(&self) -> &CircuitBreaker; + + /// Check if the worker is available (healthy + circuit closed/half-open) + fn is_available(&self) -> bool { + self.is_healthy() && self.circuit_breaker().can_execute() + } + + /// Record the outcome of a request to this worker + fn record_outcome(&self, success: bool) { + let outcome_str = if success { "success" } else { "failure" }; + RouterMetrics::record_cb_outcome(self.url(), outcome_str); + + let before = self.circuit_breaker().state(); + self.circuit_breaker().record_outcome(success); + let after = self.circuit_breaker().state(); + + if before != after { + let from = match before { + CircuitState::Closed => "closed", + CircuitState::Open => "open", + CircuitState::HalfOpen => "half_open", + }; + let to = match after { + CircuitState::Closed => "closed", + CircuitState::Open => "open", + CircuitState::HalfOpen => "half_open", + }; + RouterMetrics::record_cb_state_transition(self.url(), from, to); + } + + let state_code = match self.circuit_breaker().state() { + CircuitState::Closed => 0u8, + CircuitState::Open => 1u8, + CircuitState::HalfOpen => 2u8, + }; + RouterMetrics::set_cb_state(self.url(), state_code); + } + + /// Check if this worker is DP-aware + fn is_dp_aware(&self) -> bool { + false + } + + /// Get the base URL without any DP rank suffix + fn base_url(&self) -> &str { + self.url() + } + + /// Get DP rank if this is a DP-aware worker + fn dp_rank(&self) -> Option { + None + } + + /// Get DP size if this worker is part of a DP group + fn dp_size(&self) -> Option { + None + } + + /// Transform a request for DP-aware routing + async fn prepare_request(&self, req: serde_json::Value) -> WorkerResult { + Ok(req) + } + + /// Get the actual endpoint URL for requests + fn endpoint_url(&self, route: &str) -> String { + format!("{}{}", self.base_url(), route) + } + + /// Check if this worker can handle a specific request + fn can_handle(&self, _req: &serde_json::Value) -> bool { + true + } + + /// Get the model ID this worker serves + /// Checks ModelCards first, then falls back to labels + fn model_id(&self) -> &str { + // Check ModelCards first + self.metadata() + .models + .first() + .map(|m| m.id.as_str()) + .or_else(|| { + // Fall back to labels + self.metadata().labels.get("model_id").map(|s| s.as_str()) + }) + .unwrap_or("unknown") + } + + /// Get the priority of this worker (higher value = higher priority) + fn priority(&self) -> u32 { + self.metadata() + .labels + .get("priority") + .and_then(|s| s.parse().ok()) + .unwrap_or(50) // Default priority is 50 (mid-range) + } + + /// Get the cost factor of this worker (1.0 = baseline) + fn cost(&self) -> f32 { + self.metadata() + .labels + .get("cost") + .and_then(|s| s.parse().ok()) + .unwrap_or(1.0) + } + + /// Get tokenizer path for a specific model. + fn tokenizer_path(&self, model_id: &str) -> Option<&str> { + self.metadata() + .find_model(model_id) + .and_then(|m| m.tokenizer_path.as_deref()) + } + + /// Get reasoning parser for a specific model. + fn reasoning_parser(&self, model_id: &str) -> Option<&str> { + self.metadata() + .find_model(model_id) + .and_then(|m| m.reasoning_parser.as_deref()) + } + + /// Get tool parser for a specific model. + fn tool_parser(&self, model_id: &str) -> Option<&str> { + self.metadata() + .find_model(model_id) + .and_then(|m| m.tool_parser.as_deref()) + } + + /// Get chat template for a specific model. + fn chat_template(&self, model_id: &str) -> Option<&str> { + self.metadata() + .find_model(model_id) + .and_then(|m| m.chat_template.as_deref()) + } + + /// Get the default provider type for this worker. + /// `None` means native/passthrough. + fn default_provider(&self) -> Option<&ProviderType> { + self.metadata().default_provider.as_ref() + } + + /// Get provider for a specific model. + /// Priority: ModelCard.provider > worker.default_provider + fn provider_for_model(&self, model_id: &str) -> Option<&ProviderType> { + self.metadata().provider_for_model(model_id) + } + + /// Check if this worker supports a specific model. + /// If models list is empty, worker accepts any model. + fn supports_model(&self, model_id: &str) -> bool { + self.metadata().supports_model(model_id) + } + + /// Check if this worker supports an endpoint for a given model. + /// Falls back to default_model_type if model not found. + fn supports_endpoint(&self, model_id: &str, endpoint: Endpoint) -> bool { + self.metadata().supports_endpoint(model_id, endpoint) + } + + /// Get all models this worker can serve. + fn models(&self) -> &[ModelCard] { + &self.metadata().models + } + + /// Get or create a gRPC client for this worker + /// Returns None for HTTP workers, Some(client) for gRPC workers + async fn get_grpc_client(&self) -> WorkerResult>>; + + /// Reset the gRPC client connection (for reconnection scenarios) + /// No-op for HTTP workers + async fn reset_grpc_client(&self) -> WorkerResult<()> { + Ok(()) + } + async fn grpc_health_check(&self) -> WorkerResult; + async fn http_health_check(&self) -> WorkerResult; +} + +/// Connection mode for worker communication +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(tag = "type", rename_all = "lowercase")] +pub enum ConnectionMode { + /// HTTP/REST connection + #[default] + Http, + /// gRPC connection + Grpc { + /// Optional port for gRPC endpoint (if different from URL) + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + port: Option, + }, +} + +impl ConnectionMode { + /// Check if this connection mode matches another, with special handling for gRPC + /// This allows matching any gRPC connection regardless of port when comparing + /// Grpc { port: None } as a wildcard + pub fn matches(&self, filter: &ConnectionMode) -> bool { + match (self, filter) { + (ConnectionMode::Http, ConnectionMode::Http) => true, + (ConnectionMode::Grpc { .. }, ConnectionMode::Grpc { port: None }) => true, + (ConnectionMode::Grpc { port: p1 }, ConnectionMode::Grpc { port: p2 }) => p1 == p2, + _ => false, + } + } +} + +impl fmt::Display for ConnectionMode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ConnectionMode::Http => write!(f, "HTTP"), + ConnectionMode::Grpc { port } => match port { + Some(p) => write!(f, "gRPC(port:{})", p), + None => write!(f, "gRPC"), + }, + } + } +} + +/// Runtime implementation type for workers +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "lowercase")] +pub enum RuntimeType { + /// SGLang runtime (default) + #[default] + Sglang, + /// vLLM runtime + Vllm, + /// External OpenAI-compatible API (not local inference) + /// Used for routing to external providers like OpenAI, Azure OpenAI, xAI, etc. + External, +} + +impl fmt::Display for RuntimeType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + RuntimeType::Sglang => write!(f, "sglang"), + RuntimeType::Vllm => write!(f, "vllm"), + RuntimeType::External => write!(f, "external"), + } + } +} + +impl std::str::FromStr for RuntimeType { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_str() { + "sglang" => Ok(RuntimeType::Sglang), + "vllm" => Ok(RuntimeType::Vllm), + "external" => Ok(RuntimeType::External), + _ => Err(format!("Unknown runtime type: {}", s)), + } + } +} + +/// Worker type classification +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum WorkerType { + /// Regular worker for standard routing + Regular, + /// Prefill worker for PD disaggregated mode + Prefill { + /// Bootstrap port for communication with decode workers + bootstrap_port: Option, + }, + /// Decode worker for PD disaggregated mode + Decode, +} + +impl fmt::Display for WorkerType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + WorkerType::Regular => write!(f, "Regular"), + WorkerType::Prefill { bootstrap_port } => match bootstrap_port { + Some(port) => write!(f, "Prefill(bootstrap:{})", port), + None => write!(f, "Prefill"), + }, + WorkerType::Decode => write!(f, "Decode"), + } + } +} + +/// Health check configuration +#[derive(Debug, Clone)] +pub struct HealthConfig { + /// Timeout for health checks in seconds + pub timeout_secs: u64, + /// Interval between health checks in seconds + pub check_interval_secs: u64, + /// Health check endpoint path + pub endpoint: String, + /// Number of consecutive failures before marking unhealthy + pub failure_threshold: u32, + /// Number of consecutive successes before marking healthy + pub success_threshold: u32, +} + +impl Default for HealthConfig { + fn default() -> Self { + Self { + timeout_secs: 5, + check_interval_secs: 30, + endpoint: "/health".to_string(), + failure_threshold: 3, + success_threshold: 2, + } + } +} + +/// Metadata associated with a worker +#[derive(Debug, Clone)] +pub struct WorkerMetadata { + /// Worker URL + pub url: String, + /// Worker type + pub worker_type: WorkerType, + /// Connection mode + pub connection_mode: ConnectionMode, + /// Runtime type (for gRPC workers) + pub runtime_type: RuntimeType, + /// Additional labels/tags + pub labels: std::collections::HashMap, + /// Health check configuration + pub health_config: HealthConfig, + /// API key + pub api_key: Option, + /// Cached bootstrap hostname (parsed from URL at construction time) + pub bootstrap_host: String, + /// Cached bootstrap port (from WorkerType::Prefill) + pub bootstrap_port: Option, + /// Models this worker can serve. + /// If empty, worker accepts any model (backward compatible behavior). + pub models: Vec, + /// Default provider for this worker (used when model doesn't specify one). + /// `None` means native/passthrough. + pub default_provider: Option, + /// Default model type for unknown models (defaults to LLM capabilities). + pub default_model_type: ModelType, +} + +impl WorkerMetadata { + /// Find a model card by ID (including aliases) + pub fn find_model(&self, model_id: &str) -> Option<&ModelCard> { + self.models.iter().find(|m| m.matches(model_id)) + } + + /// Check if this worker can serve a given model. + /// If models list is empty, worker accepts any model (backward compatible). + pub fn supports_model(&self, model_id: &str) -> bool { + self.models.is_empty() || self.find_model(model_id).is_some() + } + + /// Check if this worker supports an endpoint for a given model. + /// Falls back to default_model_type if model not found. + pub fn supports_endpoint(&self, model_id: &str, endpoint: Endpoint) -> bool { + if let Some(model) = self.find_model(model_id) { + model.supports_endpoint(endpoint) + } else { + self.default_model_type.supports_endpoint(endpoint) + } + } + + /// Get the provider for a given model. + /// Returns the model's provider if found, otherwise the worker's default provider. + pub fn provider_for_model(&self, model_id: &str) -> Option<&ProviderType> { + self.find_model(model_id) + .and_then(|m| m.provider.as_ref()) + .or(self.default_provider.as_ref()) + } + + /// Get all model IDs this worker can serve + pub fn model_ids(&self) -> impl Iterator { + self.models.iter().map(|m| m.id.as_str()) + } +} + +/// Basic worker implementation +#[derive(Clone)] +pub struct BasicWorker { + pub metadata: WorkerMetadata, + pub load_counter: Arc, + pub processed_counter: Arc, + pub healthy: Arc, + pub consecutive_failures: Arc, + pub consecutive_successes: Arc, + pub circuit_breaker: CircuitBreaker, + /// Lazily initialized gRPC client for gRPC workers + pub grpc_client: Arc>>>, +} + +impl fmt::Debug for BasicWorker { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("BasicWorker") + .field("metadata", &self.metadata) + .field("healthy", &self.healthy.load(Ordering::Relaxed)) + .field("circuit_breaker", &self.circuit_breaker) + .field("grpc_client", &"") + .finish() + } +} + +impl BasicWorker { + pub fn normalised_url(&self) -> WorkerResult<&str> { + if self.url().contains("@") { + // Use rfind to split from the right, handling IPv6 addresses with brackets + // e.g., "http://[::1]:8080@0" -> "http://[::1]:8080" and "0" + if let Some(at_pos) = self.url().rfind('@') { + let base_url = &self.url()[..at_pos]; + let rank_str = &self.url()[at_pos + 1..]; + + // Validate that the rank part is actually a number + match rank_str.parse::() { + Ok(_) => Ok(base_url), + Err(_) => { + // The '@' is not a DP rank separator, return full URL + Ok(self.url()) + } + } + } else { + Ok(self.url()) + } + } else { + Ok(self.url()) + } + } +} + +#[async_trait] +impl Worker for BasicWorker { + fn url(&self) -> &str { + &self.metadata.url + } + + fn api_key(&self) -> &Option { + &self.metadata.api_key + } + + fn worker_type(&self) -> WorkerType { + self.metadata.worker_type.clone() + } + + fn connection_mode(&self) -> ConnectionMode { + self.metadata.connection_mode.clone() + } + + fn is_healthy(&self) -> bool { + self.healthy.load(Ordering::Acquire) + } + + fn set_healthy(&self, healthy: bool) { + self.healthy.store(healthy, Ordering::Release); + RouterMetrics::set_worker_health(self.url(), healthy); + } + + async fn check_health_async(&self) -> WorkerResult<()> { + let health_result = match &self.metadata.connection_mode { + ConnectionMode::Http => self.http_health_check().await?, + ConnectionMode::Grpc { .. } => self.grpc_health_check().await?, + }; + + if health_result { + self.consecutive_failures.store(0, Ordering::Release); + let successes = self.consecutive_successes.fetch_add(1, Ordering::AcqRel) + 1; + + if !self.is_healthy() + && successes >= self.metadata.health_config.success_threshold as usize + { + self.set_healthy(true); + self.consecutive_successes.store(0, Ordering::Release); + } + Ok(()) + } else { + self.consecutive_successes.store(0, Ordering::Release); + let failures = self.consecutive_failures.fetch_add(1, Ordering::AcqRel) + 1; + + if self.is_healthy() + && failures >= self.metadata.health_config.failure_threshold as usize + { + self.set_healthy(false); + self.consecutive_failures.store(0, Ordering::Release); + } + + Err(WorkerError::HealthCheckFailed { + url: self.metadata.url.clone(), + reason: format!("Health check failed (consecutive failures: {})", failures), + }) + } + } + + fn load(&self) -> usize { + self.load_counter.load(Ordering::Relaxed) + } + + fn increment_load(&self) { + self.load_counter.fetch_add(1, Ordering::Relaxed); + } + + fn decrement_load(&self) { + self.load_counter + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + current.checked_sub(1) + }) + .ok(); + } + + fn reset_load(&self) { + self.load_counter.store(0, Ordering::Relaxed); + } + + fn processed_requests(&self) -> usize { + self.processed_counter.load(Ordering::Relaxed) + } + + fn increment_processed(&self) { + self.processed_counter.fetch_add(1, Ordering::Relaxed); + } + + fn metadata(&self) -> &WorkerMetadata { + &self.metadata + } + + fn circuit_breaker(&self) -> &CircuitBreaker { + &self.circuit_breaker + } + + async fn get_grpc_client(&self) -> WorkerResult>> { + match self.metadata.connection_mode { + ConnectionMode::Http => Ok(None), + ConnectionMode::Grpc { .. } => { + { + let client_guard = self.grpc_client.read().await; + if let Some(ref client) = *client_guard { + return Ok(Some(client.clone())); + } + } + + let mut client_guard = self.grpc_client.write().await; + + if let Some(ref client) = *client_guard { + return Ok(Some(client.clone())); + } + + let runtime_str = self.metadata.runtime_type.to_string(); + tracing::info!( + "Lazily initializing gRPC client ({}) for worker: {}", + runtime_str, + self.metadata.url + ); + match GrpcClient::connect(&self.metadata.url, &runtime_str).await { + Ok(client) => { + let client_arc = Arc::new(client); + *client_guard = Some(client_arc.clone()); + tracing::info!( + "Successfully connected gRPC client ({}) for worker: {}", + runtime_str, + self.metadata.url + ); + Ok(Some(client_arc)) + } + Err(e) => { + tracing::error!( + "Failed to connect gRPC client for worker {}: {}", + self.metadata.url, + e + ); + Err(WorkerError::ConnectionFailed { + url: self.metadata.url.clone(), + reason: format!("Failed to connect to gRPC server: {}", e), + }) + } + } + } + } + } + + async fn reset_grpc_client(&self) -> WorkerResult<()> { + match self.metadata.connection_mode { + ConnectionMode::Http => Ok(()), + ConnectionMode::Grpc { .. } => { + let mut client_guard = self.grpc_client.write().await; + if client_guard.is_some() { + tracing::info!("Resetting gRPC client for worker: {}", self.metadata.url); + *client_guard = None; + } + Ok(()) + } + } + } + + async fn grpc_health_check(&self) -> WorkerResult { + let timeout = Duration::from_secs(self.metadata.health_config.timeout_secs); + let maybe = self.get_grpc_client().await?; + let Some(grpc_client) = maybe else { + tracing::error!( + "Worker {} is not a gRPC worker but connection mode is gRPC", + self.metadata.url + ); + return Ok(false); + }; + + match time::timeout(timeout, grpc_client.health_check()).await { + Ok(Ok(resp)) => { + tracing::debug!( + "gRPC health OK for {}: healthy={}", + self.metadata.url, + resp.healthy + ); + Ok(resp.healthy) + } + Ok(Err(err)) => { + tracing::warn!("gRPC health RPC error for {}: {err:?}", self.metadata.url); + Ok(false) + } + Err(_) => { + tracing::warn!("gRPC health timed out for {}", self.metadata.url); + Ok(false) + } + } + } + + async fn http_health_check(&self) -> WorkerResult { + let timeout = Duration::from_secs(self.metadata.health_config.timeout_secs); + + let url = self.normalised_url()?; + let health_url = format!("{}{}", url, self.metadata.health_config.endpoint); + + let mut req = WORKER_CLIENT.get(health_url).timeout(timeout); + if let Some(api_key) = &self.metadata.api_key { + req = req.bearer_auth(api_key); + } + + match req.send().await { + Ok(resp) => Ok(resp.status().is_success()), + Err(err) => { + tracing::warn!( + "HTTP health check failed for {}: {err:?}", + self.metadata.url + ); + Ok(false) + } + } + } +} + +/// A DP-aware worker that handles data-parallel routing +#[derive(Debug, Clone)] +pub struct DPAwareWorker { + /// The underlying basic worker + base_worker: BasicWorker, + /// DP rank for this worker + dp_rank: usize, + /// Total DP size + dp_size: usize, + /// Base URL without DP suffix + base_url: String, +} + +impl DPAwareWorker { + /// Create a new DP-aware worker with a pre-configured base worker + /// This is primarily used by the builder pattern + pub fn with_base_worker( + base_worker: BasicWorker, + base_url: String, + dp_rank: usize, + dp_size: usize, + ) -> Self { + Self { + base_worker, + dp_rank, + dp_size, + base_url, + } + } +} + +#[async_trait] +impl Worker for DPAwareWorker { + fn url(&self) -> &str { + self.base_worker.url() + } + + fn api_key(&self) -> &Option { + self.base_worker.api_key() + } + + fn worker_type(&self) -> WorkerType { + self.base_worker.worker_type() + } + + fn connection_mode(&self) -> ConnectionMode { + self.base_worker.connection_mode() + } + + fn is_healthy(&self) -> bool { + self.base_worker.is_healthy() + } + + fn set_healthy(&self, healthy: bool) { + self.base_worker.set_healthy(healthy); + } + + async fn check_health_async(&self) -> WorkerResult<()> { + self.base_worker.check_health_async().await + } + + fn load(&self) -> usize { + self.base_worker.load() + } + + fn increment_load(&self) { + self.base_worker.increment_load(); + } + + fn decrement_load(&self) { + self.base_worker.decrement_load(); + } + + fn reset_load(&self) { + self.base_worker.reset_load(); + } + + fn processed_requests(&self) -> usize { + self.base_worker.processed_requests() + } + + fn increment_processed(&self) { + self.base_worker.increment_processed(); + } + + fn metadata(&self) -> &WorkerMetadata { + self.base_worker.metadata() + } + + fn circuit_breaker(&self) -> &CircuitBreaker { + self.base_worker.circuit_breaker() + } + + fn is_dp_aware(&self) -> bool { + true + } + + fn base_url(&self) -> &str { + &self.base_url + } + + fn dp_rank(&self) -> Option { + Some(self.dp_rank) + } + + fn dp_size(&self) -> Option { + Some(self.dp_size) + } + + async fn prepare_request(&self, mut req: serde_json::Value) -> WorkerResult { + if let Some(map) = req.as_object_mut() { + map.insert( + "data_parallel_rank".to_string(), + serde_json::json!(self.dp_rank), + ); + Ok(req) + } else { + Err(WorkerError::InvalidConfiguration { + message: "Request must be a JSON object for DP-aware routing".to_string(), + }) + } + } + + fn endpoint_url(&self, route: &str) -> String { + format!("{}{}", self.base_url, route) + } + + async fn get_grpc_client(&self) -> WorkerResult>> { + self.base_worker.get_grpc_client().await + } + + async fn reset_grpc_client(&self) -> WorkerResult<()> { + self.base_worker.reset_grpc_client().await + } + + async fn grpc_health_check(&self) -> WorkerResult { + self.base_worker.grpc_health_check().await + } + + async fn http_health_check(&self) -> WorkerResult { + self.base_worker.http_health_check().await + } +} + +/// Worker factory for creating workers of different types +pub struct WorkerFactory; + +impl WorkerFactory { + /// Create a DP-aware worker of specified type + pub fn create_dp_aware( + base_url: String, + dp_rank: usize, + dp_size: usize, + worker_type: WorkerType, + api_key: Option, + ) -> Box { + let mut builder = + DPAwareWorkerBuilder::new(base_url, dp_rank, dp_size).worker_type(worker_type); + if let Some(api_key) = api_key { + builder = builder.api_key(api_key); + } + Box::new(builder.build()) + } + + /// Static health validation before creating a worker + /// This replaces wait_for_worker_health in handlers + pub async fn validate_health(url: &str, timeout_secs: u64) -> WorkerResult<()> { + let start_time = Instant::now(); + let timeout = Duration::from_secs(timeout_secs); + + loop { + if start_time.elapsed() > timeout { + return Err(WorkerError::HealthCheckFailed { + url: url.to_string(), + reason: format!( + "Timeout {}s waiting for worker to become healthy", + timeout_secs + ), + }); + } + + // Note: This static function doesn't have access to worker's API key + // API key authentication is handled in the worker instance's check_health_async method + match WORKER_CLIENT + .get(format!("{}/health", url)) + .timeout(Duration::from_secs(5)) + .send() + .await + { + Ok(res) if res.status().is_success() => { + tracing::info!("Worker {} is healthy", url); + return Ok(()); + } + Ok(res) => { + tracing::warn!( + "Worker {} health check failed with status: {}", + url, + res.status() + ); + } + Err(e) => { + tracing::warn!("Failed to contact worker {}: {}", url, e); + } + } + + time::sleep(Duration::from_secs(1)).await; + } + } +} + +/// Convert a list of worker URLs to worker trait objects +pub fn urls_to_workers(urls: Vec, api_key: Option) -> Vec> { + urls.into_iter() + .map(|url| { + let worker_builder = BasicWorkerBuilder::new(url).worker_type(WorkerType::Regular); + + let worker = if let Some(ref api_key) = api_key { + worker_builder.api_key(api_key.clone()).build() + } else { + worker_builder.build() + }; + + Box::new(worker) as Box + }) + .collect() +} + +/// Convert worker trait objects back to URLs +pub fn workers_to_urls(workers: &[Box]) -> Vec { + workers.iter().map(|w| w.url().to_string()).collect() +} + +/// RAII guard for worker load management +pub struct WorkerLoadGuard<'a> { + workers: Vec<&'a dyn Worker>, +} + +impl<'a> WorkerLoadGuard<'a> { + /// Create a new load guard for a single worker + pub fn new(worker: &'a dyn Worker) -> Self { + worker.increment_load(); + Self { + workers: vec![worker], + } + } + + /// Create a new load guard for multiple workers + pub fn new_multi(workers: Vec<&'a dyn Worker>) -> Self { + // Increment load counters for all workers + for worker in &workers { + worker.increment_load(); + } + Self { workers } + } +} + +impl<'a> Drop for WorkerLoadGuard<'a> { + fn drop(&mut self) { + // Decrement load counters for all workers + for worker in &self.workers { + worker.decrement_load(); + } + } +} + +/// Health checker handle with graceful shutdown +pub struct HealthChecker { + handle: tokio::task::JoinHandle<()>, + shutdown: Arc, +} + +impl fmt::Debug for HealthChecker { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("HealthChecker") + .field("shutdown", &self.shutdown.load(Ordering::Relaxed)) + .finish() + } +} + +impl HealthChecker { + /// Create a new HealthChecker + pub fn new(handle: tokio::task::JoinHandle<()>, shutdown: Arc) -> Self { + Self { handle, shutdown } + } + + /// Shutdown the health checker gracefully + pub async fn shutdown(self) { + self.shutdown.store(true, Ordering::Release); + let _ = self.handle.await; + } +} + +/// Helper to convert Worker trait object to WorkerInfo struct +pub fn worker_to_info(worker: &Arc) -> WorkerInfo { + let worker_type_str = match worker.worker_type() { + WorkerType::Regular => "regular", + WorkerType::Prefill { .. } => "prefill", + WorkerType::Decode => "decode", + }; + + let bootstrap_port = match worker.worker_type() { + WorkerType::Prefill { bootstrap_port } => bootstrap_port, + _ => None, + }; + + let runtime_type = match worker.connection_mode() { + ConnectionMode::Grpc { .. } => Some(worker.metadata().runtime_type.to_string()), + ConnectionMode::Http => None, + }; + + let model_id = worker.model_id(); + WorkerInfo { + id: worker.url().to_string(), + url: worker.url().to_string(), + model_id: model_id.to_string(), + priority: worker.priority(), + cost: worker.cost(), + worker_type: worker_type_str.to_string(), + is_healthy: worker.is_healthy(), + load: worker.load(), + connection_mode: format!("{:?}", worker.connection_mode()), + runtime_type, + tokenizer_path: worker.tokenizer_path(model_id).map(String::from), + reasoning_parser: worker.reasoning_parser(model_id).map(String::from), + tool_parser: worker.tool_parser(model_id).map(String::from), + chat_template: worker.chat_template(model_id).map(String::from), + bootstrap_port, + metadata: worker.metadata().labels.clone(), + job_status: None, + } +} + +#[cfg(test)] +mod tests { + use std::{thread, time::Duration}; + + use super::*; + use crate::core::CircuitBreakerConfig; + + #[test] + fn test_worker_type_display() { + assert_eq!(WorkerType::Regular.to_string(), "Regular"); + assert_eq!( + WorkerType::Prefill { + bootstrap_port: Some(8080) + } + .to_string(), + "Prefill(bootstrap:8080)" + ); + assert_eq!( + WorkerType::Prefill { + bootstrap_port: None + } + .to_string(), + "Prefill" + ); + assert_eq!(WorkerType::Decode.to_string(), "Decode"); + } + + #[test] + fn test_worker_type_equality() { + assert_eq!(WorkerType::Regular, WorkerType::Regular); + assert_ne!(WorkerType::Regular, WorkerType::Decode); + assert_eq!( + WorkerType::Prefill { + bootstrap_port: Some(8080) + }, + WorkerType::Prefill { + bootstrap_port: Some(8080) + } + ); + assert_ne!( + WorkerType::Prefill { + bootstrap_port: Some(8080) + }, + WorkerType::Prefill { + bootstrap_port: Some(8081) + } + ); + } + + #[test] + fn test_worker_type_clone() { + let original = WorkerType::Prefill { + bootstrap_port: Some(8080), + }; + let cloned = original.clone(); + assert_eq!(original, cloned); + } + + #[test] + fn test_health_config_default() { + let config = HealthConfig::default(); + assert_eq!(config.timeout_secs, 5); + assert_eq!(config.check_interval_secs, 30); + assert_eq!(config.endpoint, "/health"); + assert_eq!(config.failure_threshold, 3); + assert_eq!(config.success_threshold, 2); + } + + #[test] + fn test_health_config_custom() { + let config = HealthConfig { + timeout_secs: 10, + check_interval_secs: 60, + endpoint: "/healthz".to_string(), + failure_threshold: 5, + success_threshold: 3, + }; + assert_eq!(config.timeout_secs, 10); + assert_eq!(config.check_interval_secs, 60); + assert_eq!(config.endpoint, "/healthz"); + assert_eq!(config.failure_threshold, 5); + assert_eq!(config.success_threshold, 3); + } + + #[test] + fn test_basic_worker_creation() { + use crate::core::BasicWorkerBuilder; + let worker = BasicWorkerBuilder::new("http://test:8080") + .worker_type(WorkerType::Regular) + .build(); + assert_eq!(worker.url(), "http://test:8080"); + assert_eq!(worker.worker_type(), WorkerType::Regular); + assert!(worker.is_healthy()); + assert_eq!(worker.load(), 0); + assert_eq!(worker.processed_requests(), 0); + } + + #[test] + fn test_worker_with_labels() { + let mut labels = std::collections::HashMap::new(); + labels.insert("env".to_string(), "prod".to_string()); + labels.insert("zone".to_string(), "us-west".to_string()); + + use crate::core::BasicWorkerBuilder; + let worker = BasicWorkerBuilder::new("http://test:8080") + .worker_type(WorkerType::Regular) + .labels(labels.clone()) + .build(); + + assert_eq!(worker.metadata().labels, labels); + } + + #[test] + fn test_worker_with_health_config() { + let custom_config = HealthConfig { + timeout_secs: 15, + check_interval_secs: 45, + endpoint: "/custom-health".to_string(), + failure_threshold: 4, + success_threshold: 2, + }; + + use crate::core::BasicWorkerBuilder; + let worker = BasicWorkerBuilder::new("http://test:8080") + .worker_type(WorkerType::Regular) + .health_config(custom_config.clone()) + .build(); + + assert_eq!(worker.metadata().health_config.timeout_secs, 15); + assert_eq!(worker.metadata().health_config.check_interval_secs, 45); + assert_eq!(worker.metadata().health_config.endpoint, "/custom-health"); + } + + #[test] + fn test_worker_url() { + use crate::core::BasicWorkerBuilder; + let worker = BasicWorkerBuilder::new("http://worker1:8080") + .worker_type(WorkerType::Regular) + .build(); + assert_eq!(worker.url(), "http://worker1:8080"); + } + + #[test] + fn test_worker_type_getter() { + use crate::core::BasicWorkerBuilder; + let regular = BasicWorkerBuilder::new("http://test:8080") + .worker_type(WorkerType::Regular) + .build(); + assert_eq!(regular.worker_type(), WorkerType::Regular); + + let prefill = BasicWorkerBuilder::new("http://test:8080") + .worker_type(WorkerType::Prefill { + bootstrap_port: Some(9090), + }) + .build(); + assert_eq!( + prefill.worker_type(), + WorkerType::Prefill { + bootstrap_port: Some(9090) + } + ); + + let decode = BasicWorkerBuilder::new("http://test:8080") + .worker_type(WorkerType::Decode) + .build(); + assert_eq!(decode.worker_type(), WorkerType::Decode); + } + + #[test] + fn test_health_status() { + use crate::core::BasicWorkerBuilder; + let worker = BasicWorkerBuilder::new("http://test:8080") + .worker_type(WorkerType::Regular) + .build(); + + assert!(worker.is_healthy()); + + worker.set_healthy(false); + assert!(!worker.is_healthy()); + + worker.set_healthy(true); + assert!(worker.is_healthy()); + } + + #[test] + fn test_load_counter_operations() { + use crate::core::BasicWorkerBuilder; + let worker = BasicWorkerBuilder::new("http://test:8080") + .worker_type(WorkerType::Regular) + .build(); + + assert_eq!(worker.load(), 0); + + worker.increment_load(); + assert_eq!(worker.load(), 1); + + worker.increment_load(); + worker.increment_load(); + assert_eq!(worker.load(), 3); + + worker.decrement_load(); + assert_eq!(worker.load(), 2); + + worker.decrement_load(); + worker.decrement_load(); + assert_eq!(worker.load(), 0); + + worker.decrement_load(); + assert_eq!(worker.load(), 0); + } + + #[test] + fn test_processed_counter() { + use crate::core::BasicWorkerBuilder; + let worker = BasicWorkerBuilder::new("http://test:8080") + .worker_type(WorkerType::Regular) + .build(); + + assert_eq!(worker.processed_requests(), 0); + + for i in 1..=100 { + worker.increment_processed(); + assert_eq!(worker.processed_requests(), i); + } + } + + #[tokio::test] + async fn test_concurrent_load_increments() { + use crate::core::BasicWorkerBuilder; + let worker = Arc::new( + BasicWorkerBuilder::new("http://test:8080") + .worker_type(WorkerType::Regular) + .build(), + ); + + let mut handles = vec![]; + + for _ in 0..100 { + let worker_clone = Arc::clone(&worker); + let handle = tokio::spawn(async move { + worker_clone.increment_load(); + }); + handles.push(handle); + } + + for handle in handles { + handle.await.unwrap(); + } + + assert_eq!(worker.load(), 100); + } + + #[tokio::test] + async fn test_concurrent_load_decrements() { + use crate::core::BasicWorkerBuilder; + let worker = Arc::new( + BasicWorkerBuilder::new("http://test:8080") + .worker_type(WorkerType::Regular) + .build(), + ); + + for _ in 0..100 { + worker.increment_load(); + } + assert_eq!(worker.load(), 100); + + let mut handles = vec![]; + + for _ in 0..100 { + let worker_clone = Arc::clone(&worker); + let handle = tokio::spawn(async move { + worker_clone.decrement_load(); + }); + handles.push(handle); + } + + for handle in handles { + handle.await.unwrap(); + } + + assert_eq!(worker.load(), 0); + } + + #[tokio::test] + async fn test_concurrent_health_updates() { + use crate::core::BasicWorkerBuilder; + let worker = Arc::new( + BasicWorkerBuilder::new("http://test:8080") + .worker_type(WorkerType::Regular) + .build(), + ); + + let mut handles = vec![]; + + for i in 0..100 { + let worker_clone = Arc::clone(&worker); + let handle = tokio::spawn(async move { + worker_clone.set_healthy(i % 2 == 0); + time::sleep(Duration::from_micros(10)).await; + }); + handles.push(handle); + } + + for handle in handles { + handle.await.unwrap(); + } + } + + #[test] + fn test_create_regular_worker() { + let worker: Box = Box::new( + BasicWorkerBuilder::new("http://regular:8080") + .worker_type(WorkerType::Regular) + .build(), + ); + assert_eq!(worker.url(), "http://regular:8080"); + assert_eq!(worker.worker_type(), WorkerType::Regular); + } + + #[test] + fn test_create_prefill_worker() { + let worker1: Box = Box::new( + BasicWorkerBuilder::new("http://prefill:8080") + .worker_type(WorkerType::Prefill { + bootstrap_port: Some(9090), + }) + .build(), + ); + assert_eq!(worker1.url(), "http://prefill:8080"); + assert_eq!( + worker1.worker_type(), + WorkerType::Prefill { + bootstrap_port: Some(9090) + } + ); + + let worker2: Box = Box::new( + BasicWorkerBuilder::new("http://prefill:8080") + .worker_type(WorkerType::Prefill { + bootstrap_port: None, + }) + .build(), + ); + assert_eq!( + worker2.worker_type(), + WorkerType::Prefill { + bootstrap_port: None + } + ); + } + + #[test] + fn test_create_decode_worker() { + let worker: Box = Box::new( + BasicWorkerBuilder::new("http://decode:8080") + .worker_type(WorkerType::Decode) + .build(), + ); + assert_eq!(worker.url(), "http://decode:8080"); + assert_eq!(worker.worker_type(), WorkerType::Decode); + } + + #[test] + fn test_load_guard_single_worker() { + use crate::core::BasicWorkerBuilder; + let worker = BasicWorkerBuilder::new("http://test:8080") + .worker_type(WorkerType::Regular) + .build(); + assert_eq!(worker.load(), 0); + + { + let _guard = WorkerLoadGuard::new(&worker); + assert_eq!(worker.load(), 1); + } + + assert_eq!(worker.load(), 0); + } + + #[test] + fn test_load_guard_multiple_workers() { + let workers: Vec> = vec![ + Box::new( + BasicWorkerBuilder::new("http://w1:8080") + .worker_type(WorkerType::Regular) + .build(), + ), + Box::new( + BasicWorkerBuilder::new("http://w2:8080") + .worker_type(WorkerType::Regular) + .build(), + ), + Box::new( + BasicWorkerBuilder::new("http://w3:8080") + .worker_type(WorkerType::Regular) + .build(), + ), + ]; + + let worker_refs: Vec<&dyn Worker> = workers.iter().map(|w| w.as_ref()).collect(); + + { + let _guard = WorkerLoadGuard::new_multi(worker_refs); + assert_eq!(workers[0].load(), 1); + assert_eq!(workers[1].load(), 1); + assert_eq!(workers[2].load(), 1); + } + + assert_eq!(workers[0].load(), 0); + assert_eq!(workers[1].load(), 0); + assert_eq!(workers[2].load(), 0); + } + + #[test] + fn test_load_guard_panic_safety() { + use crate::core::BasicWorkerBuilder; + let worker = Arc::new( + BasicWorkerBuilder::new("http://test:8080") + .worker_type(WorkerType::Regular) + .build(), + ); + assert_eq!(worker.load(), 0); + + let worker_clone = Arc::clone(&worker); + + use std::panic::AssertUnwindSafe; + + let result = std::panic::catch_unwind(AssertUnwindSafe(|| { + let _guard = WorkerLoadGuard::new(worker_clone.as_ref()); + assert_eq!(worker_clone.load(), 1); + panic!("Test panic"); + })); + + assert!(result.is_err()); + + assert_eq!(worker.load(), 0); + } + + #[test] + fn test_urls_to_workers() { + let urls = vec!["http://w1:8080".to_string(), "http://w2:8080".to_string()]; + + let workers = urls_to_workers(urls, Some("test_api_key".to_string())); + assert_eq!(workers.len(), 2); + assert_eq!(workers[0].url(), "http://w1:8080"); + assert_eq!(workers[1].url(), "http://w2:8080"); + assert_eq!(workers[0].worker_type(), WorkerType::Regular); + } + + #[test] + fn test_workers_to_urls() { + let workers: Vec> = vec![ + Box::new( + BasicWorkerBuilder::new("http://w1:8080") + .worker_type(WorkerType::Regular) + .build(), + ), + Box::new( + BasicWorkerBuilder::new("http://w2:8080") + .worker_type(WorkerType::Regular) + .build(), + ), + ]; + + let urls = workers_to_urls(&workers); + assert_eq!(urls, vec!["http://w1:8080", "http://w2:8080"]); + } + + #[test] + fn test_check_health_sync_wrapper() { + use crate::core::BasicWorkerBuilder; + let worker = BasicWorkerBuilder::new("http://test:8080") + .worker_type(WorkerType::Regular) + .build(); + + let result = worker.check_health(); + assert!(result.is_err()); + } + + #[test] + fn test_load_counter_performance() { + use std::time::Instant; + + use crate::core::BasicWorkerBuilder; + + let worker = BasicWorkerBuilder::new("http://test:8080") + .worker_type(WorkerType::Regular) + .build(); + let iterations = 1_000_000; + + let start = Instant::now(); + for _ in 0..iterations { + worker.increment_load(); + } + let duration = start.elapsed(); + + let ops_per_sec = iterations as f64 / duration.as_secs_f64(); + println!("Load counter operations per second: {:.0}", ops_per_sec); + + assert!(ops_per_sec > 1_000_000.0); + } + + #[test] + fn test_dp_aware_worker_creation() { + let dp_worker = DPAwareWorkerBuilder::new("http://worker1:8080", 2, 4) + .worker_type(WorkerType::Regular) + .build(); + + assert_eq!(dp_worker.url(), "http://worker1:8080@2"); + assert_eq!(dp_worker.base_url(), "http://worker1:8080"); + assert!(dp_worker.is_dp_aware()); + assert_eq!(dp_worker.dp_rank(), Some(2)); + assert_eq!(dp_worker.dp_size(), Some(4)); + assert_eq!(dp_worker.worker_type(), WorkerType::Regular); + } + + #[test] + fn test_dp_aware_worker_creation_prefill() { + let dp_worker = DPAwareWorkerBuilder::new("http://worker1:8080", 1, 2) + .worker_type(WorkerType::Prefill { + bootstrap_port: Some(9090), + }) + .build(); + + assert_eq!(dp_worker.url(), "http://worker1:8080@1"); + assert!(dp_worker.is_dp_aware()); + assert_eq!( + dp_worker.worker_type(), + WorkerType::Prefill { + bootstrap_port: Some(9090) + } + ); + } + + #[test] + fn test_dp_aware_worker_creation_decode() { + let dp_worker = DPAwareWorkerBuilder::new("http://worker1:8080", 0, 4) + .worker_type(WorkerType::Decode) + .build(); + + assert_eq!(dp_worker.url(), "http://worker1:8080@0"); + assert!(dp_worker.is_dp_aware()); + assert_eq!(dp_worker.worker_type(), WorkerType::Decode); + } + + #[tokio::test] + async fn test_dp_aware_prepare_request() { + let dp_worker = DPAwareWorkerBuilder::new("http://worker1:8080", 3, 8) + .worker_type(WorkerType::Regular) + .build(); + + let original_req = serde_json::json!({ + "prompt": "Hello", + "max_tokens": 100 + }); + + let prepared_req = dp_worker.prepare_request(original_req).await.unwrap(); + + assert_eq!(prepared_req["prompt"], "Hello"); + assert_eq!(prepared_req["max_tokens"], 100); + assert_eq!(prepared_req["data_parallel_rank"], 3); + } + + #[tokio::test] + async fn test_dp_aware_prepare_request_invalid() { + let dp_worker = DPAwareWorkerBuilder::new("http://worker1:8080", 0, 4) + .worker_type(WorkerType::Regular) + .build(); + + // Non-object JSON should fail + let invalid_req = serde_json::json!("not an object"); + let result = dp_worker.prepare_request(invalid_req).await; + + assert!(result.is_err()); + match result.unwrap_err() { + WorkerError::InvalidConfiguration { message } => { + assert!(message.contains("JSON object")); + } + _ => panic!("Expected InvalidConfiguration error"), + } + } + + #[test] + fn test_dp_aware_endpoint_url() { + let dp_worker = DPAwareWorkerBuilder::new("http://worker1:8080", 1, 4) + .worker_type(WorkerType::Regular) + .build(); + + assert_eq!( + dp_worker.endpoint_url("/generate"), + "http://worker1:8080/generate" + ); + assert_eq!( + dp_worker.endpoint_url("/health"), + "http://worker1:8080/health" + ); + } + + #[test] + fn test_dp_aware_worker_delegated_methods() { + let dp_worker = DPAwareWorkerBuilder::new("http://worker1:8080", 0, 2) + .worker_type(WorkerType::Regular) + .build(); + + assert!(dp_worker.is_healthy()); + dp_worker.set_healthy(false); + assert!(!dp_worker.is_healthy()); + + assert_eq!(dp_worker.load(), 0); + dp_worker.increment_load(); + assert_eq!(dp_worker.load(), 1); + dp_worker.decrement_load(); + assert_eq!(dp_worker.load(), 0); + + assert_eq!(dp_worker.processed_requests(), 0); + dp_worker.increment_processed(); + assert_eq!(dp_worker.processed_requests(), 1); + } + + #[tokio::test] + async fn test_factory_create_dp_aware() { + let worker = WorkerFactory::create_dp_aware( + "http://worker1:8080".to_string(), + 1, + 4, + WorkerType::Regular, + Some("test_api_key".to_string()), + ); + + assert_eq!(worker.url(), "http://worker1:8080@1"); + assert!(worker.is_dp_aware()); + assert_eq!(worker.dp_rank(), Some(1)); + assert_eq!(worker.dp_size(), Some(4)); + assert_eq!(worker.worker_type(), WorkerType::Regular); + } + + #[tokio::test] + async fn test_factory_create_dp_aware_prefill() { + let worker = WorkerFactory::create_dp_aware( + "http://worker1:8080".to_string(), + 0, + 2, + WorkerType::Prefill { + bootstrap_port: Some(8090), + }, + Some("test_api_key".to_string()), + ); + + assert_eq!(worker.url(), "http://worker1:8080@0"); + assert!(worker.is_dp_aware()); + assert_eq!( + worker.worker_type(), + WorkerType::Prefill { + bootstrap_port: Some(8090) + } + ); + } + + #[test] + fn test_worker_circuit_breaker() { + use crate::core::BasicWorkerBuilder; + let worker = BasicWorkerBuilder::new("http://test:8080") + .worker_type(WorkerType::Regular) + .build(); + + assert!(worker.is_available()); + assert_eq!(worker.circuit_breaker().state(), CircuitState::Closed); + + worker.record_outcome(false); + worker.record_outcome(false); + + assert!(worker.is_available()); + + worker.record_outcome(false); + worker.record_outcome(false); + worker.record_outcome(false); + + assert!(!worker.is_available()); + assert!(worker.is_healthy()); + assert!(!worker.circuit_breaker().can_execute()); + } + + #[test] + fn test_worker_with_circuit_breaker_config() { + let config = CircuitBreakerConfig { + failure_threshold: 2, + success_threshold: 1, + timeout_duration: Duration::from_millis(100), + window_duration: Duration::from_secs(60), + }; + + use crate::core::BasicWorkerBuilder; + let worker = BasicWorkerBuilder::new("http://test:8080") + .worker_type(WorkerType::Regular) + .circuit_breaker_config(config) + .build(); + + worker.record_outcome(false); + assert!(worker.is_available()); + worker.record_outcome(false); + assert!(!worker.is_available()); + + thread::sleep(Duration::from_millis(150)); + + assert!(worker.is_available()); + assert_eq!(worker.circuit_breaker().state(), CircuitState::HalfOpen); + + worker.record_outcome(true); + assert_eq!(worker.circuit_breaker().state(), CircuitState::Closed); + } + + #[test] + fn test_dp_aware_worker_circuit_breaker() { + let dp_worker = DPAwareWorkerBuilder::new("http://worker:8080", 0, 2) + .worker_type(WorkerType::Regular) + .build(); + + assert!(dp_worker.is_available()); + + for _ in 0..5 { + dp_worker.record_outcome(false); + } + + assert!(!dp_worker.is_available()); + assert_eq!(dp_worker.circuit_breaker().state(), CircuitState::Open); + } + + #[tokio::test] + async fn test_mixed_worker_types() { + let regular: Box = Box::new( + BasicWorkerBuilder::new("http://regular:8080") + .worker_type(WorkerType::Regular) + .build(), + ); + let prefill: Box = Box::new( + BasicWorkerBuilder::new("http://prefill:8080") + .worker_type(WorkerType::Prefill { + bootstrap_port: Some(9090), + }) + .build(), + ); + let decode: Box = Box::new( + BasicWorkerBuilder::new("http://decode:8080") + .worker_type(WorkerType::Decode) + .build(), + ); + let dp_aware_regular = WorkerFactory::create_dp_aware( + "http://dp:8080".to_string(), + 0, + 2, + WorkerType::Regular, + Some("test_api_key".to_string()), + ); + let dp_aware_prefill = WorkerFactory::create_dp_aware( + "http://dp-prefill:8080".to_string(), + 1, + 2, + WorkerType::Prefill { + bootstrap_port: None, + }, + Some("test_api_key".to_string()), + ); + let dp_aware_decode = WorkerFactory::create_dp_aware( + "http://dp-decode:8080".to_string(), + 0, + 4, + WorkerType::Decode, + Some("test_api_key".to_string()), + ); + + let workers: Vec> = vec![ + regular, + prefill, + decode, + dp_aware_regular, + dp_aware_prefill, + dp_aware_decode, + ]; + + for worker in &workers { + assert!(worker.is_healthy()); + assert_eq!(worker.load(), 0); + assert_eq!(worker.processed_requests(), 0); + } + + assert!(!workers[0].is_dp_aware()); + assert!(!workers[1].is_dp_aware()); + assert!(!workers[2].is_dp_aware()); + assert!(workers[3].is_dp_aware()); + assert!(workers[4].is_dp_aware()); + assert!(workers[5].is_dp_aware()); + + assert_eq!(workers[0].worker_type(), WorkerType::Regular); + assert_eq!( + workers[1].worker_type(), + WorkerType::Prefill { + bootstrap_port: Some(9090) + } + ); + assert_eq!(workers[2].worker_type(), WorkerType::Decode); + assert_eq!(workers[3].worker_type(), WorkerType::Regular); + assert_eq!( + workers[4].worker_type(), + WorkerType::Prefill { + bootstrap_port: None + } + ); + assert_eq!(workers[5].worker_type(), WorkerType::Decode); + } + + // === Phase 1.3: WorkerMetadata model methods tests === + + #[test] + fn test_worker_metadata_empty_models_accepts_all() { + let metadata = WorkerMetadata { + url: "http://test:8080".to_string(), + worker_type: WorkerType::Regular, + connection_mode: ConnectionMode::Http, + runtime_type: RuntimeType::default(), + labels: std::collections::HashMap::new(), + health_config: HealthConfig::default(), + api_key: None, + bootstrap_host: "test".to_string(), + bootstrap_port: None, + models: Vec::new(), // Empty = accepts any model + default_provider: None, + default_model_type: ModelType::LLM, + }; + + // Empty models list should accept any model + assert!(metadata.supports_model("any-model")); + assert!(metadata.supports_model("gpt-4")); + assert!(metadata.supports_model("llama-3.1")); + } + + #[test] + fn test_worker_metadata_find_model() { + use super::ModelCard; + + let model1 = ModelCard::new("meta-llama/Llama-3.1-8B") + .with_alias("llama-3.1-8b") + .with_alias("llama3.1"); + let model2 = ModelCard::new("gpt-4o"); + + let metadata = WorkerMetadata { + url: "http://test:8080".to_string(), + worker_type: WorkerType::Regular, + connection_mode: ConnectionMode::Http, + runtime_type: RuntimeType::default(), + labels: std::collections::HashMap::new(), + health_config: HealthConfig::default(), + api_key: None, + bootstrap_host: "test".to_string(), + bootstrap_port: None, + models: vec![model1, model2], + default_provider: None, + default_model_type: ModelType::LLM, + }; + + // Find by primary ID + assert!(metadata.find_model("meta-llama/Llama-3.1-8B").is_some()); + assert!(metadata.find_model("gpt-4o").is_some()); + + // Find by alias + assert!(metadata.find_model("llama-3.1-8b").is_some()); + assert!(metadata.find_model("llama3.1").is_some()); + + // Not found + assert!(metadata.find_model("unknown-model").is_none()); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/worker_builder.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/worker_builder.rs new file mode 100644 index 00000000..de035495 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/worker_builder.rs @@ -0,0 +1,534 @@ +use std::collections::HashMap; + +use super::{ + circuit_breaker::{CircuitBreaker, CircuitBreakerConfig}, + model_card::ModelCard, + model_type::ModelType, + worker::{ + BasicWorker, ConnectionMode, DPAwareWorker, HealthConfig, RuntimeType, WorkerMetadata, + WorkerType, + }, +}; +use crate::routers::grpc::client::GrpcClient; + +/// Builder for creating BasicWorker instances with fluent API +pub struct BasicWorkerBuilder { + url: String, + api_key: Option, + worker_type: WorkerType, + connection_mode: ConnectionMode, + runtime_type: RuntimeType, + labels: HashMap, + models: Vec, + health_config: HealthConfig, + circuit_breaker_config: CircuitBreakerConfig, + grpc_client: Option, +} + +impl BasicWorkerBuilder { + /// Create a new builder with only the URL + pub fn new(url: impl Into) -> Self { + Self { + url: url.into(), + api_key: None, + worker_type: WorkerType::Regular, + connection_mode: ConnectionMode::Http, + runtime_type: RuntimeType::default(), + labels: HashMap::new(), + models: Vec::new(), + health_config: HealthConfig::default(), + circuit_breaker_config: CircuitBreakerConfig::default(), + grpc_client: None, + } + } + + /// Create a new builder with URL and worker type (for backwards compatibility) + pub fn new_with_type(url: impl Into, worker_type: WorkerType) -> Self { + Self { + url: url.into(), + api_key: None, + worker_type, + connection_mode: ConnectionMode::Http, + runtime_type: RuntimeType::default(), + labels: HashMap::new(), + models: Vec::new(), + health_config: HealthConfig::default(), + circuit_breaker_config: CircuitBreakerConfig::default(), + grpc_client: None, + } + } + + /// Set the API key + pub fn api_key(mut self, api_key: impl Into) -> Self { + self.api_key = Some(api_key.into()); + self + } + + /// Set the worker type (Regular, Prefill, or Decode) + pub fn worker_type(mut self, worker_type: WorkerType) -> Self { + self.worker_type = worker_type; + self + } + + /// Set the connection mode (HTTP or gRPC) + pub fn connection_mode(mut self, mode: ConnectionMode) -> Self { + self.connection_mode = mode; + self + } + + /// Set the runtime type (SGLang or vLLM) + pub fn runtime_type(mut self, runtime_type: RuntimeType) -> Self { + self.runtime_type = runtime_type; + self + } + + /// Set labels for worker identification + pub fn labels(mut self, labels: HashMap) -> Self { + self.labels = labels; + self + } + + /// Add a single label + pub fn label(mut self, key: impl Into, value: impl Into) -> Self { + self.labels.insert(key.into(), value.into()); + self + } + + /// Set health check configuration + pub fn health_config(mut self, config: HealthConfig) -> Self { + self.health_config = config; + self + } + + /// Set circuit breaker configuration + pub fn circuit_breaker_config(mut self, config: CircuitBreakerConfig) -> Self { + self.circuit_breaker_config = config; + self + } + + /// Set gRPC client for gRPC workers + pub fn grpc_client(mut self, client: GrpcClient) -> Self { + self.grpc_client = Some(client); + self + } + + /// Set models this worker can serve + pub fn models(mut self, models: Vec) -> Self { + self.models = models; + self + } + + /// Add a single model this worker can serve + pub fn model(mut self, model: ModelCard) -> Self { + self.models.push(model); + self + } + + /// Build the BasicWorker instance + pub fn build(self) -> BasicWorker { + use std::sync::{ + atomic::{AtomicBool, AtomicUsize}, + Arc, + }; + + use tokio::sync::RwLock; + + let bootstrap_host = match url::Url::parse(&self.url) { + Ok(parsed) => parsed.host_str().unwrap_or("localhost").to_string(), + Err(_) if !self.url.contains("://") => { + match url::Url::parse(&format!("http://{}", self.url)) { + Ok(parsed) => parsed.host_str().unwrap_or("localhost").to_string(), + Err(_) => { + tracing::warn!( + "Failed to parse URL '{}', defaulting to localhost", + self.url + ); + "localhost".to_string() + } + } + } + Err(_) => { + tracing::warn!( + "Failed to parse URL '{}', defaulting to localhost", + self.url + ); + "localhost".to_string() + } + }; + + let bootstrap_port = match self.worker_type { + WorkerType::Prefill { bootstrap_port } => bootstrap_port, + _ => None, + }; + + let metadata = WorkerMetadata { + url: self.url.clone(), + api_key: self.api_key, + worker_type: self.worker_type, + connection_mode: self.connection_mode, + runtime_type: self.runtime_type, + labels: self.labels, + health_config: self.health_config, + bootstrap_host, + bootstrap_port, + models: self.models, // Empty = accepts any model + default_provider: None, // Native/passthrough + default_model_type: ModelType::LLM, // Standard LLM capabilities + }; + + let grpc_client = Arc::new(RwLock::new(self.grpc_client.map(Arc::new))); + + BasicWorker { + metadata, + load_counter: Arc::new(AtomicUsize::new(0)), + processed_counter: Arc::new(AtomicUsize::new(0)), + healthy: Arc::new(AtomicBool::new(true)), + consecutive_failures: Arc::new(AtomicUsize::new(0)), + consecutive_successes: Arc::new(AtomicUsize::new(0)), + circuit_breaker: CircuitBreaker::with_config(self.circuit_breaker_config), + grpc_client, + } + } +} + +/// Builder for creating DPAwareWorker instances with fluent API +pub struct DPAwareWorkerBuilder { + base_url: String, + api_key: Option, + dp_rank: usize, + dp_size: usize, + worker_type: WorkerType, + connection_mode: ConnectionMode, + runtime_type: RuntimeType, + labels: HashMap, + models: Vec, + health_config: HealthConfig, + circuit_breaker_config: CircuitBreakerConfig, + grpc_client: Option, +} + +impl DPAwareWorkerBuilder { + /// Create a new DP-aware worker builder + pub fn new(base_url: impl Into, dp_rank: usize, dp_size: usize) -> Self { + Self { + base_url: base_url.into(), + api_key: None, + dp_rank, + dp_size, + worker_type: WorkerType::Regular, + connection_mode: ConnectionMode::Http, + runtime_type: RuntimeType::default(), + labels: HashMap::new(), + models: Vec::new(), + health_config: HealthConfig::default(), + circuit_breaker_config: CircuitBreakerConfig::default(), + grpc_client: None, + } + } + + /// Create a new DP-aware worker builder with worker type (for backwards compatibility) + pub fn new_with_type( + base_url: impl Into, + dp_rank: usize, + dp_size: usize, + worker_type: WorkerType, + ) -> Self { + Self { + base_url: base_url.into(), + api_key: None, + dp_rank, + dp_size, + worker_type, + connection_mode: ConnectionMode::Http, + runtime_type: RuntimeType::default(), + labels: HashMap::new(), + models: Vec::new(), + health_config: HealthConfig::default(), + circuit_breaker_config: CircuitBreakerConfig::default(), + grpc_client: None, + } + } + + /// Set the API key + pub fn api_key(mut self, api_key: impl Into) -> Self { + self.api_key = Some(api_key.into()); + self + } + + /// Set the worker type (Regular, Prefill, or Decode) + pub fn worker_type(mut self, worker_type: WorkerType) -> Self { + self.worker_type = worker_type; + self + } + + /// Set the connection mode (HTTP or gRPC) + pub fn connection_mode(mut self, mode: ConnectionMode) -> Self { + self.connection_mode = mode; + self + } + + /// Set the runtime type (SGLang or vLLM) + pub fn runtime_type(mut self, runtime_type: RuntimeType) -> Self { + self.runtime_type = runtime_type; + self + } + + /// Set labels for worker identification + pub fn labels(mut self, labels: HashMap) -> Self { + self.labels = labels; + self + } + + /// Add a single label + pub fn label(mut self, key: impl Into, value: impl Into) -> Self { + self.labels.insert(key.into(), value.into()); + self + } + + /// Set health check configuration + pub fn health_config(mut self, config: HealthConfig) -> Self { + self.health_config = config; + self + } + + /// Set circuit breaker configuration + pub fn circuit_breaker_config(mut self, config: CircuitBreakerConfig) -> Self { + self.circuit_breaker_config = config; + self + } + + /// Set gRPC client for gRPC workers + pub fn grpc_client(mut self, client: GrpcClient) -> Self { + self.grpc_client = Some(client); + self + } + + /// Set models this worker can serve + pub fn models(mut self, models: Vec) -> Self { + self.models = models; + self + } + + /// Add a single model this worker can serve + pub fn model(mut self, model: ModelCard) -> Self { + self.models.push(model); + self + } + + /// Build the DPAwareWorker instance + pub fn build(self) -> DPAwareWorker { + let worker_url = format!("{}@{}", self.base_url, self.dp_rank); + let mut builder = BasicWorkerBuilder::new(worker_url) + .models(self.models) + .worker_type(self.worker_type) + .connection_mode(self.connection_mode) + .runtime_type(self.runtime_type) + .labels(self.labels) + .health_config(self.health_config) + .circuit_breaker_config(self.circuit_breaker_config); + + if let Some(client) = self.grpc_client { + builder = builder.grpc_client(client); + } + if let Some(api_key) = self.api_key { + builder = builder.api_key(api_key); + } + + let base_worker = builder.build(); + DPAwareWorker::with_base_worker(base_worker, self.base_url, self.dp_rank, self.dp_size) + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + use crate::core::worker::Worker; + + #[test] + fn test_basic_worker_builder_minimal() { + let worker = BasicWorkerBuilder::new("http://localhost:8080").build(); + + assert_eq!(worker.url(), "http://localhost:8080"); + assert_eq!(worker.worker_type(), WorkerType::Regular); + assert_eq!(worker.connection_mode(), ConnectionMode::Http); + assert!(worker.is_healthy()); + } + + #[test] + fn test_basic_worker_builder_with_type() { + let worker = BasicWorkerBuilder::new("http://localhost:8080") + .worker_type(WorkerType::Decode) + .build(); + + assert_eq!(worker.url(), "http://localhost:8080"); + assert_eq!(worker.worker_type(), WorkerType::Decode); + assert_eq!(worker.connection_mode(), ConnectionMode::Http); + assert!(worker.is_healthy()); + } + + #[test] + fn test_basic_worker_builder_full() { + let mut labels = HashMap::new(); + labels.insert("env".to_string(), "prod".to_string()); + labels.insert("region".to_string(), "us-east".to_string()); + + let health_config = HealthConfig { + endpoint: "/health".to_string(), + timeout_secs: 30, + check_interval_secs: 60, + failure_threshold: 3, + success_threshold: 2, + }; + + let cb_config = CircuitBreakerConfig { + failure_threshold: 10, + success_threshold: 5, + timeout_duration: Duration::from_millis(2000), + window_duration: Duration::from_millis(30000), + }; + + let worker = BasicWorkerBuilder::new("http://localhost:8080") + .worker_type(WorkerType::Prefill { + bootstrap_port: None, + }) + .connection_mode(ConnectionMode::Grpc { port: Some(50051) }) + .labels(labels.clone()) + .health_config(health_config.clone()) + .circuit_breaker_config(cb_config) + .build(); + + assert_eq!(worker.url(), "http://localhost:8080"); + assert_eq!( + worker.worker_type(), + WorkerType::Prefill { + bootstrap_port: None + } + ); + assert_eq!( + worker.connection_mode(), + ConnectionMode::Grpc { port: Some(50051) } + ); + assert_eq!(worker.metadata().labels, labels); + assert_eq!( + worker.metadata().health_config.endpoint, + health_config.endpoint + ); + assert_eq!( + worker.metadata().health_config.timeout_secs, + health_config.timeout_secs + ); + assert_eq!( + worker.metadata().health_config.check_interval_secs, + health_config.check_interval_secs + ); + assert_eq!( + worker.metadata().health_config.failure_threshold, + health_config.failure_threshold + ); + assert_eq!( + worker.metadata().health_config.success_threshold, + health_config.success_threshold + ); + } + + #[test] + fn test_basic_worker_builder_with_single_label() { + let worker = BasicWorkerBuilder::new("http://localhost:8080") + .worker_type(WorkerType::Decode) + .label("env", "staging") + .label("version", "v1.2.3") + .build(); + + assert_eq!( + worker.metadata().labels.get("env"), + Some(&"staging".to_string()) + ); + assert_eq!( + worker.metadata().labels.get("version"), + Some(&"v1.2.3".to_string()) + ); + } + + #[test] + fn test_dp_aware_worker_builder_minimal() { + let worker = DPAwareWorkerBuilder::new("http://localhost:8080", 2, 8).build(); + + assert_eq!(worker.url(), "http://localhost:8080@2"); + assert_eq!(worker.dp_rank(), Some(2)); + assert_eq!(worker.dp_size(), Some(8)); + assert_eq!(worker.worker_type(), WorkerType::Regular); + } + + #[test] + fn test_dp_aware_worker_builder_full() { + let mut labels = HashMap::new(); + labels.insert("cluster".to_string(), "main".to_string()); + + let health_config = HealthConfig { + endpoint: "/status".to_string(), + timeout_secs: 20, + check_interval_secs: 45, + failure_threshold: 5, + success_threshold: 3, + }; + + let worker = DPAwareWorkerBuilder::new("http://localhost:8080", 3, 16) + .worker_type(WorkerType::Prefill { + bootstrap_port: Some(9090), + }) + .connection_mode(ConnectionMode::Http) + .labels(labels.clone()) + .health_config(health_config.clone()) + .api_key("test_api_key") + .build(); + + assert_eq!(worker.url(), "http://localhost:8080@3"); + assert_eq!(worker.dp_rank(), Some(3)); + assert_eq!(worker.dp_size(), Some(16)); + assert_eq!(worker.metadata().labels, labels); + assert_eq!( + worker.metadata().health_config.endpoint, + health_config.endpoint + ); + assert_eq!( + worker.metadata().health_config.timeout_secs, + health_config.timeout_secs + ); + assert_eq!( + worker.metadata().health_config.check_interval_secs, + health_config.check_interval_secs + ); + assert_eq!( + worker.metadata().health_config.failure_threshold, + health_config.failure_threshold + ); + assert_eq!( + worker.metadata().health_config.success_threshold, + health_config.success_threshold + ); + } + + #[test] + fn test_dp_aware_worker_with_grpc() { + let worker = DPAwareWorkerBuilder::new("grpc://cluster.local", 1, 4) + .worker_type(WorkerType::Decode) + .connection_mode(ConnectionMode::Grpc { port: Some(50051) }) + .label("transport", "grpc") + .build(); + + assert_eq!(worker.url(), "grpc://cluster.local@1"); + assert_eq!(worker.dp_rank(), Some(1)); + assert_eq!(worker.dp_size(), Some(4)); + assert_eq!(worker.worker_type(), WorkerType::Decode); + assert_eq!( + worker.connection_mode(), + ConnectionMode::Grpc { port: Some(50051) } + ); + assert_eq!( + worker.metadata().labels.get("transport"), + Some(&"grpc".to_string()) + ); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/worker_manager.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/worker_manager.rs new file mode 100644 index 00000000..5f94e55f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/worker_manager.rs @@ -0,0 +1,456 @@ +//! Unified Worker Management Module +//! +//! Handles all aspects of worker lifecycle including discovery, initialization, +//! runtime management, and health monitoring. + +use std::{collections::HashMap, sync::Arc, time::Duration}; + +use axum::response::{IntoResponse, Response}; +use futures::future; +use http::{Method, StatusCode}; +use serde_json::Value; +use tokio::{ + sync::{watch, Mutex}, + task::JoinHandle, +}; +use tracing::{debug, error, info, warn}; + +use crate::{ + core::{metrics_aggregator::MetricPack, ConnectionMode, WorkerRegistry, WorkerType}, + policies::PolicyRegistry, + protocols::worker_spec::{FlushCacheResult, WorkerLoadInfo, WorkerLoadsResult}, +}; + +/// Unified worker management +pub struct WorkerManager; + +impl WorkerManager { + pub fn get_worker_urls(registry: &Arc) -> Vec { + registry + .get_all() + .iter() + .map(|w| w.url().to_string()) + .collect() + } + + /// Flush cache on all workers + /// + /// Sends a POST request to /flush_cache endpoint on all HTTP workers. + /// Returns detailed results showing which workers succeeded and which failed. + pub async fn flush_cache_all( + worker_registry: &WorkerRegistry, + client: &reqwest::Client, + ) -> Result { + warn!("Flushing cache for ALL workers - this may impact performance temporarily"); + + let workers = worker_registry.get_all(); + + let http_workers: Vec<_> = workers + .iter() + .filter(|w| matches!(w.connection_mode(), ConnectionMode::Http)) + .collect(); + + if http_workers.is_empty() { + return Ok(FlushCacheResult { + successful: vec![], + failed: vec![], + total_workers: workers.len(), + http_workers: 0, + message: "No HTTP workers available for cache flush".to_string(), + }); + } + + info!( + "Flushing cache on {} HTTP workers (out of {} total workers)", + http_workers.len(), + workers.len() + ); + + let mut tasks = Vec::new(); + for worker in &http_workers { + let url = worker.url().to_string(); + let flush_url = format!("{}/flush_cache", url); + let mut request = client.post(&flush_url); + + if let Some(api_key) = worker.api_key() { + request = request.header("Authorization", format!("Bearer {}", api_key)); + } + + let worker_url = url.clone(); + tasks.push(async move { + let result = request.send().await; + (worker_url, result) + }); + } + + let results = future::join_all(tasks).await; + + let mut successful = Vec::new(); + let mut failed = Vec::new(); + + for (url, result) in results { + match result { + Ok(response) if response.status().is_success() => { + debug!("Successfully flushed cache on worker: {}", url); + successful.push(url); + } + Ok(response) => { + let error = format!("HTTP {}", response.status()); + warn!("Failed to flush cache on worker {}: {}", url, error); + failed.push((url, error)); + } + Err(e) => { + let error = e.to_string(); + error!("Failed to connect to worker {}: {}", url, error); + failed.push((url, error)); + } + } + } + + let message = if failed.is_empty() { + format!( + "Successfully flushed cache on all {} HTTP workers", + successful.len() + ) + } else { + format!( + "Cache flush completed: {} succeeded, {} failed (out of {} HTTP workers)", + successful.len(), + failed.len(), + http_workers.len() + ) + }; + + info!("{}", message); + + Ok(FlushCacheResult { + successful, + failed, + total_workers: workers.len(), + http_workers: http_workers.len(), + message, + }) + } + pub async fn get_worker_load( + url: &str, + api_key: Option<&str>, + client: &reqwest::Client, + ) -> Option { + let load_url = format!("{}/get_load", url); + let mut request = client.get(&load_url); + + if let Some(key) = api_key { + request = request.bearer_auth(key); + } + + match request.send().await { + Ok(response) if response.status().is_success() => { + match response.json::().await { + Ok(json) => { + // The /get_load endpoint returns an array of load info objects (one per DP rank) + // Each object has: {dp_rank, num_reqs, num_waiting_reqs, num_tokens} + if let Some(array) = json.as_array() { + let total_tokens: i64 = array + .iter() + .filter_map(|entry| { + entry.get("num_tokens").and_then(|v| v.as_i64()) + }) + .sum(); + debug!("Worker {} load (total tokens): {}", url, total_tokens); + Some(total_tokens as isize) + } else { + warn!( + "Invalid load response from {}: expected array, got {:?}", + url, json + ); + None + } + } + Err(e) => { + warn!("Failed to parse load response from {}: {}", url, e); + None + } + } + } + Ok(response) => { + warn!( + "Failed to get load from {}: HTTP {}", + url, + response.status() + ); + None + } + Err(e) => { + warn!("Failed to connect to {} for load check: {}", url, e); + None + } + } + } + + pub async fn get_all_worker_loads( + worker_registry: &WorkerRegistry, + client: &reqwest::Client, + ) -> WorkerLoadsResult { + let workers = worker_registry.get_all(); + let total_workers = workers.len(); + + // Prepare tasks for parallel execution + let mut tasks = Vec::new(); + for worker in &workers { + let url = worker.url().to_string(); + let api_key = worker.api_key().clone(); + let worker_type = match worker.worker_type() { + WorkerType::Regular => None, + WorkerType::Prefill { .. } => Some("prefill".to_string()), + WorkerType::Decode => Some("decode".to_string()), + }; + let is_http = matches!(worker.connection_mode(), ConnectionMode::Http); + let client = client.clone(); + + tasks.push(async move { + let load = if is_http { + Self::get_worker_load(&url, api_key.as_deref(), &client) + .await + .unwrap_or(-1) + } else { + -1 + }; + + WorkerLoadInfo { + worker: url, + worker_type, + load, + } + }); + } + + let loads = future::join_all(tasks).await; + + let successful = loads.iter().filter(|l| l.load >= 0).count(); + let failed = loads.iter().filter(|l| l.load < 0).count(); + + WorkerLoadsResult { + loads, + total_workers, + successful, + failed, + } + } + + pub async fn get_engine_metrics( + worker_registry: &WorkerRegistry, + client: &reqwest::Client, + ) -> Response { + let engine_responses = + match Self::fan_out_simple_request(worker_registry, client, "metrics", Method::GET) + .await + { + Ok(x) => x, + Err(e) => return e, + }; + let engine_responses = engine_responses + .into_iter() + .map(|(worker_base_url, metrics_text)| MetricPack { + labels: vec![("worker_addr".into(), worker_base_url)], + metrics_text, + }) + .collect(); + let text = match crate::core::metrics_aggregator::aggregate_metrics(engine_responses) { + Ok(x) => x, + Err(e) => { + let error_msg = format!("Failed to aggregate metrics: {}", e); + return (StatusCode::INTERNAL_SERVER_ERROR, error_msg).into_response(); + } + }; + (StatusCode::OK, text).into_response() + } + + async fn fan_out_simple_request( + worker_registry: &WorkerRegistry, + client: &reqwest::Client, + endpoint: &str, + method: Method, + ) -> Result, Response> { + let workers = worker_registry.get_all(); + if workers.is_empty() { + return Err((StatusCode::SERVICE_UNAVAILABLE, "No available workers").into_response()); + } + + let mut responses = vec![]; + // May do parallel requests later + for worker in workers { + let worker_url = worker.url().to_string(); + + let url = format!("{}/{}", worker_url, endpoint); + let mut request_builder = match method { + Method::GET => client.get(url), + Method::POST => client.post(url), + _ => { + return Err(( + StatusCode::METHOD_NOT_ALLOWED, + "Unsupported method for simple routing", + ) + .into_response()) + } + }; + + if let Some(api_key) = worker.api_key() { + request_builder = + request_builder.header("Authorization", format!("Bearer {}", api_key)); + } + + match request_builder.send().await { + Ok(res) => { + let status = StatusCode::from_u16(res.status().as_u16()) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + match res.text().await { + Ok(body_text) => { + if status.is_success() { + responses.push((worker_url, body_text)); + } + } + Err(e) => { + warn!("fan_out_simple_request failed when reading text: {}", e) + } + } + } + Err(e) => warn!("fan_out_simple_request failed when sending: {}", e), + } + } + + Ok(responses) + } +} + +/// Load monitoring service that periodically fetches worker loads +pub struct LoadMonitor { + worker_registry: Arc, + policy_registry: Arc, + client: reqwest::Client, + interval: Duration, + tx: watch::Sender>, + rx: watch::Receiver>, + monitor_handle: Arc>>>, +} + +impl LoadMonitor { + /// Create a new load monitor + pub fn new( + worker_registry: Arc, + policy_registry: Arc, + client: reqwest::Client, + interval_secs: u64, + ) -> Self { + let (tx, rx) = watch::channel(HashMap::new()); + + Self { + worker_registry, + policy_registry, + client, + interval: Duration::from_secs(interval_secs), + tx, + rx, + monitor_handle: Arc::new(Mutex::new(None)), + } + } + + /// Start monitoring worker loads + pub async fn start(&self) { + let mut handle_guard = self.monitor_handle.lock().await; + if handle_guard.is_some() { + debug!("Load monitoring already running"); + return; + } + + info!( + "Starting load monitoring with interval: {:?}", + self.interval + ); + + let worker_registry = Arc::clone(&self.worker_registry); + let policy_registry = Arc::clone(&self.policy_registry); + let client = self.client.clone(); + let interval = self.interval; + let tx = self.tx.clone(); + + let handle = tokio::spawn(async move { + Self::monitor_loop(worker_registry, policy_registry, client, interval, tx).await; + }); + + *handle_guard = Some(handle); + } + + /// Stop monitoring worker loads + pub async fn stop(&self) { + let mut handle_guard = self.monitor_handle.lock().await; + if let Some(handle) = handle_guard.take() { + info!("Stopping load monitoring"); + handle.abort(); + let _ = handle.await; // Wait for task to finish + } + } + + /// Get a receiver for load updates + pub fn subscribe(&self) -> watch::Receiver> { + self.rx.clone() + } + + /// The main monitoring loop + async fn monitor_loop( + worker_registry: Arc, + policy_registry: Arc, + client: reqwest::Client, + interval: Duration, + tx: watch::Sender>, + ) { + let mut interval_timer = tokio::time::interval(interval); + + loop { + interval_timer.tick().await; + + let power_of_two_policies = policy_registry.get_all_power_of_two_policies(); + + if power_of_two_policies.is_empty() { + debug!("No PowerOfTwo policies found, skipping load fetch"); + continue; + } + + let result = WorkerManager::get_all_worker_loads(&worker_registry, &client).await; + + let mut loads = HashMap::new(); + for load_info in result.loads { + loads.insert(load_info.worker, load_info.load); + } + + if !loads.is_empty() { + debug!( + "Fetched loads from {} workers, updating {} PowerOfTwo policies", + loads.len(), + power_of_two_policies.len() + ); + for policy in &power_of_two_policies { + policy.update_loads(&loads); + } + let _ = tx.send(loads); + } else { + warn!("No loads fetched from workers"); + } + } + } + + /// Check if monitoring is currently active + pub async fn is_running(&self) -> bool { + let handle_guard = self.monitor_handle.lock().await; + handle_guard.is_some() + } +} + +impl Drop for LoadMonitor { + fn drop(&mut self) { + if let Ok(mut handle_guard) = self.monitor_handle.try_lock() { + if let Some(handle) = handle_guard.take() { + handle.abort(); + } + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/worker_registry.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/worker_registry.rs new file mode 100644 index 00000000..95e09f87 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/worker_registry.rs @@ -0,0 +1,546 @@ +//! Worker Registry for multi-router support +//! +//! Provides centralized registry for workers with model-based indexing + +use std::sync::{Arc, RwLock}; + +use dashmap::DashMap; +use uuid::Uuid; + +use crate::core::{ConnectionMode, Worker, WorkerType}; + +/// Unique identifier for a worker +#[derive(Debug, Clone, Hash, Eq, PartialEq)] +pub struct WorkerId(String); + +impl WorkerId { + /// Create a new worker ID + pub fn new() -> Self { + Self(Uuid::new_v4().to_string()) + } + + /// Create a worker ID from a string + pub fn from_string(s: String) -> Self { + Self(s) + } + + /// Get the ID as a string + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl Default for WorkerId { + fn default() -> Self { + Self::new() + } +} + +type ModelIndex = Arc>>>>>; + +/// Worker registry with model-based indexing +#[derive(Debug)] +pub struct WorkerRegistry { + /// All workers indexed by ID + workers: Arc>>, + + /// Workers indexed by model ID (stores WorkerId for reference) + model_workers: Arc>>, + + /// Optimized model index for O(1) lookups (stores Arc directly) + model_index: ModelIndex, + + /// Workers indexed by worker type + type_workers: Arc>>, + + /// Workers indexed by connection mode + connection_workers: Arc>>, + /// URL to worker ID mapping + url_to_id: Arc>, +} + +impl WorkerRegistry { + /// Create a new worker registry + pub fn new() -> Self { + Self { + workers: Arc::new(DashMap::new()), + model_workers: Arc::new(DashMap::new()), + model_index: Arc::new(DashMap::new()), + type_workers: Arc::new(DashMap::new()), + connection_workers: Arc::new(DashMap::new()), + url_to_id: Arc::new(DashMap::new()), + } + } + + /// Register a new worker + pub fn register(&self, worker: Arc) -> WorkerId { + let worker_id = if let Some(existing_id) = self.url_to_id.get(worker.url()) { + // Worker with this URL already exists, update it + existing_id.clone() + } else { + WorkerId::new() + }; + + // Store worker + self.workers.insert(worker_id.clone(), worker.clone()); + + // Update URL mapping + self.url_to_id + .insert(worker.url().to_string(), worker_id.clone()); + + // Update model index (both ID-based and optimized) + let model_id = worker.model_id().to_string(); + self.model_workers + .entry(model_id.clone()) + .or_default() + .push(worker_id.clone()); + + // Update optimized model index for O(1) lookups + self.model_index + .entry(model_id) + .or_insert_with(|| Arc::new(RwLock::new(Vec::new()))) + .write() + .expect("RwLock for model_index is poisoned") + .push(worker.clone()); + + // Update type index + self.type_workers + .entry(worker.worker_type()) + .or_default() + .push(worker_id.clone()); + + // Update connection mode index + self.connection_workers + .entry(worker.connection_mode()) + .or_default() + .push(worker_id.clone()); + + worker_id + } + + /// Remove a worker by ID + pub fn remove(&self, worker_id: &WorkerId) -> Option> { + if let Some((_, worker)) = self.workers.remove(worker_id) { + // Remove from URL mapping + self.url_to_id.remove(worker.url()); + + // Remove from model index (both ID-based and optimized) + if let Some(mut model_workers) = self.model_workers.get_mut(worker.model_id()) { + model_workers.retain(|id| id != worker_id); + } + + // Remove from optimized model index + if let Some(model_index_entry) = self.model_index.get(worker.model_id()) { + let worker_url = worker.url(); + model_index_entry + .write() + .expect("RwLock for model_index is poisoned") + .retain(|w| w.url() != worker_url); + } + + // Remove from type index + if let Some(mut type_workers) = self.type_workers.get_mut(&worker.worker_type()) { + type_workers.retain(|id| id != worker_id); + } + + // Remove from connection mode index + if let Some(mut conn_workers) = + self.connection_workers.get_mut(&worker.connection_mode()) + { + conn_workers.retain(|id| id != worker_id); + } + + Some(worker) + } else { + None + } + } + + /// Remove a worker by URL + pub fn remove_by_url(&self, url: &str) -> Option> { + if let Some((_, worker_id)) = self.url_to_id.remove(url) { + self.remove(&worker_id) + } else { + None + } + } + + /// Get a worker by ID + pub fn get(&self, worker_id: &WorkerId) -> Option> { + self.workers.get(worker_id).map(|entry| entry.clone()) + } + + /// Get a worker by URL + pub fn get_by_url(&self, url: &str) -> Option> { + self.url_to_id.get(url).and_then(|id| self.get(&id)) + } + + /// Get all workers for a model + pub fn get_by_model(&self, model_id: &str) -> Vec> { + self.model_workers + .get(model_id) + .map(|ids| ids.iter().filter_map(|id| self.get(id)).collect()) + .unwrap_or_default() + } + + /// Get all workers for a model (O(1) optimized version) + /// This method uses the pre-indexed model_index for fast lookups + pub fn get_by_model_fast(&self, model_id: &str) -> Vec> { + self.model_index + .get(model_id) + .map(|workers| { + workers + .read() + .expect("RwLock for model_index is poisoned") + .clone() + }) + .unwrap_or_default() + } + + /// Get all workers by worker type + pub fn get_by_type(&self, worker_type: &WorkerType) -> Vec> { + self.type_workers + .get(worker_type) + .map(|ids| ids.iter().filter_map(|id| self.get(id)).collect()) + .unwrap_or_default() + } + + /// Get all prefill workers (regardless of bootstrap_port) + pub fn get_prefill_workers(&self) -> Vec> { + self.workers + .iter() + .filter_map(|entry| { + let worker = entry.value(); + match worker.worker_type() { + WorkerType::Prefill { .. } => Some(worker.clone()), + _ => None, + } + }) + .collect() + } + + /// Get all decode workers + pub fn get_decode_workers(&self) -> Vec> { + self.get_by_type(&WorkerType::Decode) + } + + /// Get all workers by connection mode + pub fn get_by_connection(&self, connection_mode: &ConnectionMode) -> Vec> { + self.connection_workers + .get(connection_mode) + .map(|ids| ids.iter().filter_map(|id| self.get(id)).collect()) + .unwrap_or_default() + } + + /// Get all workers + pub fn get_all(&self) -> Vec> { + self.workers + .iter() + .map(|entry| entry.value().clone()) + .collect() + } + + /// Get all workers with their IDs + pub fn get_all_with_ids(&self) -> Vec<(WorkerId, Arc)> { + self.workers + .iter() + .map(|entry| (entry.key().clone(), entry.value().clone())) + .collect() + } + + /// Get all worker URLs + pub fn get_all_urls(&self) -> Vec { + self.workers + .iter() + .map(|entry| entry.value().url().to_string()) + .collect() + } + + pub fn get_all_urls_with_api_key(&self) -> Vec<(String, Option)> { + self.workers + .iter() + .map(|entry| { + ( + entry.value().url().to_string(), + entry.value().api_key().clone(), + ) + }) + .collect() + } + + /// Get all model IDs with workers + pub fn get_models(&self) -> Vec { + self.model_workers + .iter() + .filter(|entry| !entry.value().is_empty()) + .map(|entry| entry.key().clone()) + .collect() + } + + /// Get workers filtered by multiple criteria + /// + /// This method allows flexible filtering of workers based on: + /// - model_id: Filter by specific model + /// - worker_type: Filter by worker type (Regular, Prefill, Decode) + /// - connection_mode: Filter by connection mode (Http, Grpc) + /// - healthy_only: Only return healthy workers + pub fn get_workers_filtered( + &self, + model_id: Option<&str>, + worker_type: Option, + connection_mode: Option, + healthy_only: bool, + ) -> Vec> { + // Start with the most efficient collection based on filters + // Use model index when possible as it's O(1) lookup + let workers = if let Some(model) = model_id { + self.get_by_model_fast(model) + } else { + self.get_all() + }; + + // Apply remaining filters + workers + .into_iter() + .filter(|w| { + // Check worker_type if specified + if let Some(ref wtype) = worker_type { + if w.worker_type() != *wtype { + return false; + } + } + + // Check connection_mode if specified (using matches for flexible gRPC matching) + if let Some(ref conn) = connection_mode { + if !w.connection_mode().matches(conn) { + return false; + } + } + + // Check health if required + if healthy_only && !w.is_healthy() { + return false; + } + + true + }) + .collect() + } + + /// Get worker statistics + pub fn stats(&self) -> WorkerRegistryStats { + let total_workers = self.workers.len(); + let total_models = self.get_models().len(); + + let mut healthy_count = 0; + let mut total_load = 0; + let mut regular_count = 0; + let mut prefill_count = 0; + let mut decode_count = 0; + + for worker in self.get_all() { + if worker.is_healthy() { + healthy_count += 1; + } + total_load += worker.load(); + + match worker.worker_type() { + WorkerType::Regular => regular_count += 1, + WorkerType::Prefill { .. } => prefill_count += 1, + WorkerType::Decode => decode_count += 1, + } + } + + WorkerRegistryStats { + total_workers, + total_models, + healthy_workers: healthy_count, + total_load, + regular_workers: regular_count, + prefill_workers: prefill_count, + decode_workers: decode_count, + } + } + + /// Start a health checker for all workers in the registry + /// This should be called once after the registry is populated with workers + pub fn start_health_checker(&self, check_interval_secs: u64) -> crate::core::HealthChecker { + use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }; + + let shutdown = Arc::new(AtomicBool::new(false)); + let shutdown_clone = shutdown.clone(); + let workers_ref = self.workers.clone(); + + let handle = tokio::spawn(async move { + let mut interval = + tokio::time::interval(tokio::time::Duration::from_secs(check_interval_secs)); + + // Counter for periodic load reset (every 10 health check cycles) + let mut check_count = 0u64; + const LOAD_RESET_INTERVAL: u64 = 10; + + loop { + interval.tick().await; + + // Check for shutdown signal + if shutdown_clone.load(Ordering::Acquire) { + tracing::debug!("Registry health checker shutting down"); + break; + } + + // Get all workers from registry + let workers: Vec> = workers_ref + .iter() + .map(|entry| entry.value().clone()) + .collect(); + + // Perform health checks + for worker in &workers { + let _ = worker.check_health_async().await; // Use async version directly + } + + // Reset loads periodically + check_count += 1; + if check_count.is_multiple_of(LOAD_RESET_INTERVAL) { + tracing::debug!("Resetting worker loads (cycle {})", check_count); + for worker in &workers { + worker.reset_load(); + } + } + } + }); + + crate::core::HealthChecker::new(handle, shutdown) + } +} + +impl Default for WorkerRegistry { + fn default() -> Self { + Self::new() + } +} + +/// Statistics for the worker registry +#[derive(Debug, Clone)] +pub struct WorkerRegistryStats { + pub total_workers: usize, + pub total_models: usize, + pub healthy_workers: usize, + pub total_load: usize, + pub regular_workers: usize, + pub prefill_workers: usize, + pub decode_workers: usize, +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + use crate::core::{BasicWorkerBuilder, CircuitBreakerConfig}; + + #[test] + fn test_worker_registry() { + let registry = WorkerRegistry::new(); + + // Create a worker with labels + let mut labels = HashMap::new(); + labels.insert("model_id".to_string(), "llama-3-8b".to_string()); + labels.insert("priority".to_string(), "50".to_string()); + labels.insert("cost".to_string(), "0.8".to_string()); + + let worker: Box = Box::new( + BasicWorkerBuilder::new("http://worker1:8080") + .worker_type(WorkerType::Regular) + .labels(labels) + .circuit_breaker_config(CircuitBreakerConfig::default()) + .api_key("test_api_key") + .build(), + ); + + // Register worker (WorkerFactory returns Box, convert to Arc) + let worker_id = registry.register(Arc::from(worker)); + + assert!(registry.get(&worker_id).is_some()); + assert!(registry.get_by_url("http://worker1:8080").is_some()); + assert_eq!(registry.get_by_model("llama-3-8b").len(), 1); + assert_eq!(registry.get_by_type(&WorkerType::Regular).len(), 1); + assert_eq!(registry.get_by_connection(&ConnectionMode::Http).len(), 1); + + let stats = registry.stats(); + assert_eq!(stats.total_workers, 1); + assert_eq!(stats.total_models, 1); + + // Remove worker + registry.remove(&worker_id); + assert!(registry.get(&worker_id).is_none()); + } + + #[test] + fn test_model_index_fast_lookup() { + let registry = WorkerRegistry::new(); + + // Create workers for different models + let mut labels1 = HashMap::new(); + labels1.insert("model_id".to_string(), "llama-3".to_string()); + let worker1: Box = Box::new( + BasicWorkerBuilder::new("http://worker1:8080") + .worker_type(WorkerType::Regular) + .labels(labels1) + .circuit_breaker_config(CircuitBreakerConfig::default()) + .api_key("test_api_key") + .build(), + ); + + let mut labels2 = HashMap::new(); + labels2.insert("model_id".to_string(), "llama-3".to_string()); + let worker2: Box = Box::new( + BasicWorkerBuilder::new("http://worker2:8080") + .worker_type(WorkerType::Regular) + .labels(labels2) + .circuit_breaker_config(CircuitBreakerConfig::default()) + .api_key("test_api_key") + .build(), + ); + + let mut labels3 = HashMap::new(); + labels3.insert("model_id".to_string(), "gpt-4".to_string()); + let worker3: Box = Box::new( + BasicWorkerBuilder::new("http://worker3:8080") + .worker_type(WorkerType::Regular) + .labels(labels3) + .circuit_breaker_config(CircuitBreakerConfig::default()) + .api_key("test_api_key") + .build(), + ); + + // Register workers + registry.register(Arc::from(worker1)); + registry.register(Arc::from(worker2)); + registry.register(Arc::from(worker3)); + + let llama_workers = registry.get_by_model_fast("llama-3"); + assert_eq!(llama_workers.len(), 2); + let urls: Vec = llama_workers.iter().map(|w| w.url().to_string()).collect(); + assert!(urls.contains(&"http://worker1:8080".to_string())); + assert!(urls.contains(&"http://worker2:8080".to_string())); + + let gpt_workers = registry.get_by_model_fast("gpt-4"); + assert_eq!(gpt_workers.len(), 1); + assert_eq!(gpt_workers[0].url(), "http://worker3:8080"); + + let unknown_workers = registry.get_by_model_fast("unknown-model"); + assert_eq!(unknown_workers.len(), 0); + + let llama_workers_slow = registry.get_by_model("llama-3"); + assert_eq!(llama_workers.len(), llama_workers_slow.len()); + + registry.remove_by_url("http://worker1:8080"); + let llama_workers_after = registry.get_by_model_fast("llama-3"); + assert_eq!(llama_workers_after.len(), 1); + assert_eq!(llama_workers_after[0].url(), "http://worker2:8080"); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/definition.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/definition.rs new file mode 100644 index 00000000..737d5735 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/definition.rs @@ -0,0 +1,98 @@ +//! Workflow definition types + +use std::{sync::Arc, time::Duration}; + +use super::{ + executor::StepExecutor, + types::{FailureAction, RetryPolicy, StepId, WorkflowId}, +}; + +/// Definition of a single step within a workflow +pub struct StepDefinition { + pub id: StepId, + pub name: String, + pub executor: Arc, + pub retry_policy: Option, + pub timeout: Option, + pub on_failure: FailureAction, +} + +impl StepDefinition { + pub fn new( + id: impl Into, + name: impl Into, + executor: Arc, + ) -> Self { + Self { + id: StepId::new(id.into()), + name: name.into(), + executor, + retry_policy: None, + timeout: None, + on_failure: FailureAction::FailWorkflow, + } + } + + pub fn with_retry(mut self, policy: RetryPolicy) -> Self { + self.retry_policy = Some(policy); + self + } + + pub fn with_timeout(mut self, timeout: Duration) -> Self { + self.timeout = Some(timeout); + self + } + + pub fn with_failure_action(mut self, action: FailureAction) -> Self { + self.on_failure = action; + self + } +} + +/// Complete workflow definition +pub struct WorkflowDefinition { + pub id: WorkflowId, + pub name: String, + pub steps: Vec, + pub default_retry_policy: RetryPolicy, + pub default_timeout: Duration, +} + +impl WorkflowDefinition { + pub fn new(id: impl Into, name: impl Into) -> Self { + Self { + id: WorkflowId::new(id.into()), + name: name.into(), + steps: Vec::new(), + default_retry_policy: RetryPolicy::default(), + default_timeout: Duration::from_secs(300), // 5 minutes + } + } + + pub fn add_step(mut self, step: StepDefinition) -> Self { + self.steps.push(step); + self + } + + pub fn with_default_retry(mut self, policy: RetryPolicy) -> Self { + self.default_retry_policy = policy; + self + } + + pub fn with_default_timeout(mut self, timeout: Duration) -> Self { + self.default_timeout = timeout; + self + } + + /// Get the retry policy for a step (step-specific or default) + pub fn get_retry_policy<'a>(&'a self, step: &'a StepDefinition) -> &'a RetryPolicy { + step.retry_policy + .as_ref() + .unwrap_or(&self.default_retry_policy) + } + + /// Get the timeout for a step (step-specific or default) + pub fn get_timeout(&self, step: &StepDefinition) -> Duration { + step.timeout.unwrap_or(self.default_timeout) + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/engine.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/engine.rs new file mode 100644 index 00000000..630b3630 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/engine.rs @@ -0,0 +1,484 @@ +//! Workflow execution engine + +use std::{collections::HashMap, sync::Arc, time::Duration}; + +use backoff::{backoff::Backoff, ExponentialBackoffBuilder}; +use chrono::Utc; +use parking_lot::RwLock; +use tokio::time::timeout; + +use super::{ + definition::{StepDefinition, WorkflowDefinition}, + event::{EventBus, WorkflowEvent}, + state::WorkflowStateStore, + types::*, +}; + +/// Linear backoff implementation that increases delay by a fixed amount each retry +struct LinearBackoff { + current: Duration, + increment: Duration, + max: Duration, +} + +impl LinearBackoff { + fn new(increment: Duration, max: Duration) -> Self { + Self { + current: increment, + increment, + max, + } + } +} + +impl Backoff for LinearBackoff { + fn next_backoff(&mut self) -> Option { + let next = self.current; + self.current = (self.current + self.increment).min(self.max); + Some(next) + } + + fn reset(&mut self) { + self.current = self.increment; + } +} + +/// Main workflow execution engine +pub struct WorkflowEngine { + definitions: Arc>>>, + state_store: WorkflowStateStore, + event_bus: Arc, +} + +impl WorkflowEngine { + pub fn new() -> Self { + Self { + definitions: Arc::new(RwLock::new(HashMap::new())), + state_store: WorkflowStateStore::new(), + event_bus: Arc::new(EventBus::new()), + } + } + + /// Start a background task to periodically clean up old workflow states + /// + /// This prevents unbounded memory growth by removing completed/failed workflows + /// that are older than the specified TTL. + /// + /// # Arguments + /// + /// * `ttl` - Time-to-live for terminal workflows (default: 1 hour) + /// * `interval` - How often to run cleanup (default: 5 minutes) + /// + /// # Returns + /// + /// A join handle for the cleanup task that can be used to stop it. + pub fn start_cleanup_task( + &self, + ttl: Option, + interval: Option, + ) -> tokio::task::JoinHandle<()> { + let state_store = self.state_store.clone(); + let ttl = ttl.unwrap_or(Duration::from_secs(3600)); // 1 hour default + let interval = interval.unwrap_or(Duration::from_secs(300)); // 5 minutes default + + tokio::spawn(async move { + let mut ticker = tokio::time::interval(interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + loop { + ticker.tick().await; + state_store.cleanup_old_workflows(ttl); + } + }) + } + + /// Register a workflow definition + pub fn register_workflow(&self, definition: WorkflowDefinition) { + let id = definition.id.clone(); + self.definitions.write().insert(id, Arc::new(definition)); + } + + /// Get the event bus for subscribing to workflow events + pub fn event_bus(&self) -> Arc { + Arc::clone(&self.event_bus) + } + + /// Get the state store + pub fn state_store(&self) -> &WorkflowStateStore { + &self.state_store + } + + /// Start a new workflow instance + pub async fn start_workflow( + &self, + definition_id: WorkflowId, + context: WorkflowContext, + ) -> WorkflowResult { + // Get workflow definition + let definition = { + let definitions = self.definitions.read(); + definitions + .get(&definition_id) + .cloned() + .ok_or_else(|| WorkflowError::DefinitionNotFound(definition_id.clone()))? + }; + + // Create new workflow instance + let instance_id = context.instance_id; + let mut state = WorkflowState::new(instance_id, definition_id.clone()); + state.status = WorkflowStatus::Running; + state.context = context; + + // Initialize step states + for step in &definition.steps { + state + .step_states + .insert(step.id.clone(), StepState::default()); + } + + // Save initial state + self.state_store.save(state)?; + + // Emit workflow started event + self.event_bus + .publish(WorkflowEvent::WorkflowStarted { + instance_id, + definition_id, + }) + .await; + + // Execute workflow in background + let engine = self.clone_for_execution(); + let def = Arc::clone(&definition); + tokio::spawn(async move { + if let Err(e) = engine.execute_workflow(instance_id, def).await { + tracing::error!(instance_id = %instance_id, error = ?e, "Workflow execution failed"); + } + }); + + Ok(instance_id) + } + + /// Execute a workflow (internal) + async fn execute_workflow( + &self, + instance_id: WorkflowInstanceId, + definition: Arc, + ) -> WorkflowResult<()> { + let start_time = std::time::Instant::now(); + + for step in &definition.steps { + // Check if workflow was cancelled + let state = self.state_store.load(instance_id)?; + if state.status == WorkflowStatus::Cancelled { + self.event_bus + .publish(WorkflowEvent::WorkflowCancelled { instance_id }) + .await; + return Ok(()); + } + + // Execute step with retry + match self + .execute_step_with_retry(instance_id, step, &definition) + .await + { + Ok(StepResult::Success) => { + // Continue to next step + } + Ok(StepResult::Skip) => { + // Step was skipped, continue to next + continue; + } + Ok(StepResult::Failure) | Err(_) => { + // Handle failure based on failure action + match step.on_failure { + FailureAction::FailWorkflow => { + let error_msg = format!("Step {} failed", step.id); + self.state_store.update(instance_id, |s| { + s.status = WorkflowStatus::Failed; + })?; + + self.event_bus + .publish(WorkflowEvent::WorkflowFailed { + instance_id, + failed_step: step.id.clone(), + error: error_msg, + }) + .await; + + return Ok(()); + } + FailureAction::ContinueNextStep => { + // Mark step as skipped and continue + self.state_store.update(instance_id, |s| { + if let Some(step_state) = s.step_states.get_mut(&step.id) { + step_state.status = StepStatus::Skipped; + } + })?; + continue; + } + FailureAction::RetryIndefinitely => { + // This should not happen as execute_step_with_retry handles it + unreachable!("RetryIndefinitely should be handled in retry logic"); + } + } + } + } + } + + // Workflow completed successfully + self.state_store.update(instance_id, |s| { + s.status = WorkflowStatus::Completed; + })?; + + let duration = start_time.elapsed(); + self.event_bus + .publish(WorkflowEvent::WorkflowCompleted { + instance_id, + duration, + }) + .await; + + Ok(()) + } + + /// Execute a step with retry logic + async fn execute_step_with_retry( + &self, + instance_id: WorkflowInstanceId, + step: &StepDefinition, + definition: &WorkflowDefinition, + ) -> WorkflowResult { + let retry_policy = definition.get_retry_policy(step); + let step_timeout = definition.get_timeout(step); + + let mut attempt = 1; + let max_attempts = if matches!(step.on_failure, FailureAction::RetryIndefinitely) { + u32::MAX + } else { + retry_policy.max_attempts + }; + + let mut backoff = Self::create_backoff(&retry_policy.backoff); + + loop { + // Check for cancellation before starting/retrying step + { + let state = self.state_store.load(instance_id)?; + if state.status == WorkflowStatus::Cancelled { + return Err(WorkflowError::Cancelled(instance_id)); + } + } + + // Update step state + self.state_store.update(instance_id, |s| { + s.current_step = Some(step.id.clone()); + if let Some(step_state) = s.step_states.get_mut(&step.id) { + step_state.status = if attempt == 1 { + StepStatus::Running + } else { + StepStatus::Retrying + }; + step_state.attempt = attempt; + step_state.started_at = Some(Utc::now()); + } + })?; + + // Emit step started event + self.event_bus + .publish(WorkflowEvent::StepStarted { + instance_id, + step_id: step.id.clone(), + attempt, + }) + .await; + + // Get current context + let mut context = self.state_store.load(instance_id)?.context; + + // Execute step with timeout + let step_start = std::time::Instant::now(); + let result = timeout(step_timeout, step.executor.execute(&mut context)).await; + + let step_duration = step_start.elapsed(); + + // Save updated context + self.state_store.update(instance_id, |s| { + s.context = context.clone(); + })?; + + match result { + Ok(Ok(StepResult::Success)) => { + // Step succeeded + self.state_store.update(instance_id, |s| { + if let Some(step_state) = s.step_states.get_mut(&step.id) { + step_state.status = StepStatus::Succeeded; + step_state.completed_at = Some(Utc::now()); + } + })?; + + self.event_bus + .publish(WorkflowEvent::StepSucceeded { + instance_id, + step_id: step.id.clone(), + duration: step_duration, + }) + .await; + + // Call on_success hook + if let Err(e) = step.executor.on_success(&context).await { + tracing::warn!(step_id = %step.id, error = ?e, "on_success hook failed"); + } + + return Ok(StepResult::Success); + } + Ok(Ok(StepResult::Skip)) => { + return Ok(StepResult::Skip); + } + Ok(Ok(StepResult::Failure)) | Ok(Err(_)) | Err(_) => { + let (error_msg, should_retry) = match result { + Ok(Err(e)) => { + let msg = format!("{}", e); + let retryable = step.executor.is_retryable(&e); + (msg, retryable) + } + Err(_) => ( + format!("Step timeout after {:?}", step_timeout), + true, // Timeouts are retryable + ), + _ => ("Step failed".to_string(), false), + }; + + let will_retry = should_retry && attempt < max_attempts; + + // Update step state + self.state_store.update(instance_id, |s| { + if let Some(step_state) = s.step_states.get_mut(&step.id) { + step_state.status = if will_retry { + StepStatus::Retrying + } else { + StepStatus::Failed + }; + step_state.last_error = Some(error_msg.clone()); + if !will_retry { + step_state.completed_at = Some(Utc::now()); + } + } + })?; + + // Emit step failed event + self.event_bus + .publish(WorkflowEvent::StepFailed { + instance_id, + step_id: step.id.clone(), + error: error_msg.clone(), + will_retry, + }) + .await; + + if will_retry { + // Calculate backoff delay + let delay = backoff + .next_backoff() + .unwrap_or_else(|| Duration::from_secs(1)); + + self.event_bus + .publish(WorkflowEvent::StepRetrying { + instance_id, + step_id: step.id.clone(), + attempt: attempt + 1, + delay, + }) + .await; + + tokio::time::sleep(delay).await; + attempt += 1; + } else { + // No more retries, call on_failure hook + // Create a generic error for the hook + let hook_error = WorkflowError::StepFailed { + step_id: step.id.clone(), + message: error_msg, + }; + if let Err(hook_err) = step.executor.on_failure(&context, &hook_error).await + { + tracing::warn!(step_id = %step.id, error = ?hook_err, "on_failure hook failed"); + } + + return Ok(StepResult::Failure); + } + } + } + } + } + + /// Create a backoff instance from strategy + fn create_backoff(strategy: &BackoffStrategy) -> Box { + match strategy { + BackoffStrategy::Fixed(duration) => { + // For fixed backoff, use exponential with multiplier 1.0 + let backoff = ExponentialBackoffBuilder::new() + .with_initial_interval(*duration) + .with_multiplier(1.0) + .with_max_interval(*duration) + .with_max_elapsed_time(None) + .build(); + Box::new(backoff) + } + BackoffStrategy::Exponential { base, max } => { + let backoff = ExponentialBackoffBuilder::new() + .with_initial_interval(*base) + .with_max_interval(*max) + .with_max_elapsed_time(None) + .build(); + Box::new(backoff) + } + BackoffStrategy::Linear { increment, max } => { + // Use proper linear backoff: increment, 2*increment, 3*increment, ... + Box::new(LinearBackoff::new(*increment, *max)) + } + } + } + + /// Cancel a running workflow + pub async fn cancel_workflow(&self, instance_id: WorkflowInstanceId) -> WorkflowResult<()> { + self.state_store.update(instance_id, |s| { + s.status = WorkflowStatus::Cancelled; + })?; + + self.event_bus + .publish(WorkflowEvent::WorkflowCancelled { instance_id }) + .await; + + Ok(()) + } + + /// Get workflow status + pub fn get_status(&self, instance_id: WorkflowInstanceId) -> WorkflowResult { + self.state_store.load(instance_id) + } + + /// Clone engine for async execution + fn clone_for_execution(&self) -> Self { + Self { + definitions: Arc::clone(&self.definitions), + state_store: self.state_store.clone(), + event_bus: Arc::clone(&self.event_bus), + } + } +} + +impl Default for WorkflowEngine { + fn default() -> Self { + Self::new() + } +} + +impl std::fmt::Debug for WorkflowEngine { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("WorkflowEngine") + .field("definitions_count", &self.definitions.read().len()) + .field("state_count", &self.state_store.count()) + .finish() + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/event.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/event.rs new file mode 100644 index 00000000..1fd5202e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/event.rs @@ -0,0 +1,188 @@ +//! Workflow event system for observability and monitoring + +use std::{sync::Arc, time::Duration}; + +use async_trait::async_trait; +use tokio::sync::RwLock; +use tracing::{error, info, warn}; + +use super::types::{StepId, WorkflowId, WorkflowInstanceId}; + +/// Events emitted by the workflow engine +#[derive(Debug, Clone)] +pub enum WorkflowEvent { + WorkflowStarted { + instance_id: WorkflowInstanceId, + definition_id: WorkflowId, + }, + StepStarted { + instance_id: WorkflowInstanceId, + step_id: StepId, + attempt: u32, + }, + StepSucceeded { + instance_id: WorkflowInstanceId, + step_id: StepId, + duration: Duration, + }, + StepFailed { + instance_id: WorkflowInstanceId, + step_id: StepId, + error: String, + will_retry: bool, + }, + StepRetrying { + instance_id: WorkflowInstanceId, + step_id: StepId, + attempt: u32, + delay: Duration, + }, + WorkflowCompleted { + instance_id: WorkflowInstanceId, + duration: Duration, + }, + WorkflowFailed { + instance_id: WorkflowInstanceId, + failed_step: StepId, + error: String, + }, + WorkflowCancelled { + instance_id: WorkflowInstanceId, + }, +} + +/// Trait for subscribing to workflow events +#[async_trait] +pub trait EventSubscriber: Send + Sync { + async fn on_event(&self, event: &WorkflowEvent); +} + +/// Event bus for publishing and subscribing to workflow events +pub struct EventBus { + subscribers: Arc>>>, +} + +impl EventBus { + pub fn new() -> Self { + Self { + subscribers: Arc::new(RwLock::new(Vec::new())), + } + } + + /// Subscribe to workflow events + pub async fn subscribe(&self, subscriber: Arc) { + self.subscribers.write().await.push(subscriber); + } + + /// Publish an event to all subscribers + pub async fn publish(&self, event: WorkflowEvent) { + let subscribers = self.subscribers.read().await; + for subscriber in subscribers.iter() { + subscriber.on_event(&event).await; + } + } +} + +impl Default for EventBus { + fn default() -> Self { + Self::new() + } +} + +/// Logging subscriber that logs events using tracing +pub struct LoggingSubscriber; + +#[async_trait] +impl EventSubscriber for LoggingSubscriber { + async fn on_event(&self, event: &WorkflowEvent) { + match event { + WorkflowEvent::WorkflowStarted { + instance_id, + definition_id, + } => { + info!( + instance_id = %instance_id, + definition_id = %definition_id, + "Workflow started" + ); + } + WorkflowEvent::StepStarted { + instance_id, + step_id, + attempt, + } => { + info!( + instance_id = %instance_id, + step_id = %step_id, + attempt = attempt, + "Step started" + ); + } + WorkflowEvent::StepSucceeded { + instance_id, + step_id, + duration, + } => { + info!( + instance_id = %instance_id, + step_id = %step_id, + duration_ms = duration.as_millis(), + "Step succeeded" + ); + } + WorkflowEvent::StepFailed { + instance_id, + step_id, + error, + will_retry, + } => { + warn!( + instance_id = %instance_id, + step_id = %step_id, + error = error, + will_retry = will_retry, + "Step failed" + ); + } + WorkflowEvent::StepRetrying { + instance_id, + step_id, + attempt, + delay, + } => { + info!( + instance_id = %instance_id, + step_id = %step_id, + attempt = attempt, + delay_ms = delay.as_millis(), + "Step retrying" + ); + } + WorkflowEvent::WorkflowCompleted { + instance_id, + duration, + } => { + info!( + instance_id = %instance_id, + duration_ms = duration.as_millis(), + "Workflow completed" + ); + } + WorkflowEvent::WorkflowFailed { + instance_id, + failed_step, + error, + } => { + error!( + instance_id = %instance_id, + failed_step = %failed_step, + error = error, + "Workflow failed" + ); + } + WorkflowEvent::WorkflowCancelled { instance_id } => { + info!(instance_id = %instance_id, "Workflow cancelled"); + } + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/executor.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/executor.rs new file mode 100644 index 00000000..1a27c192 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/executor.rs @@ -0,0 +1,129 @@ +//! Step executor trait and implementations + +use async_trait::async_trait; + +use super::types::{StepResult, WorkflowContext, WorkflowError, WorkflowResult}; + +/// Trait for executing individual workflow steps +#[async_trait] +pub trait StepExecutor: Send + Sync { + /// Execute the step with the given context + async fn execute(&self, context: &mut WorkflowContext) -> WorkflowResult; + + /// Check if an error is retry-able + /// + /// Override this method to customize which errors should trigger retries. + /// By default, all errors are considered retry-able. + fn is_retryable(&self, _error: &WorkflowError) -> bool { + true + } + + /// Called when the step succeeds + /// + /// This hook allows steps to perform cleanup or additional actions + /// after successful execution. + async fn on_success(&self, _context: &WorkflowContext) -> WorkflowResult<()> { + Ok(()) + } + + /// Called when the step fails after all retries + /// + /// This hook allows steps to perform cleanup or compensation logic + /// when the step cannot complete successfully. + async fn on_failure( + &self, + _context: &WorkflowContext, + _error: &WorkflowError, + ) -> WorkflowResult<()> { + Ok(()) + } +} + +/// Simple function-based step executor +pub struct FunctionStep +where + F: Fn( + &mut WorkflowContext, + ) -> std::pin::Pin< + Box> + Send + '_>, + > + Send + + Sync, +{ + func: F, +} + +impl FunctionStep +where + F: Fn( + &mut WorkflowContext, + ) -> std::pin::Pin< + Box> + Send + '_>, + > + Send + + Sync, +{ + pub fn new(func: F) -> Self { + Self { func } + } +} + +#[async_trait] +impl StepExecutor for FunctionStep +where + F: Fn( + &mut WorkflowContext, + ) -> std::pin::Pin< + Box> + Send + '_>, + > + Send + + Sync, +{ + async fn execute(&self, context: &mut WorkflowContext) -> WorkflowResult { + (self.func)(context).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::workflow::types::WorkflowInstanceId; + + struct TestStep { + should_succeed: bool, + } + + #[async_trait] + impl StepExecutor for TestStep { + async fn execute(&self, _context: &mut WorkflowContext) -> WorkflowResult { + if self.should_succeed { + Ok(StepResult::Success) + } else { + Err(WorkflowError::StepFailed { + step_id: crate::core::workflow::types::StepId::new("test"), + message: "test error".to_string(), + }) + } + } + } + + #[tokio::test] + async fn test_step_executor_success() { + let step = TestStep { + should_succeed: true, + }; + let mut context = WorkflowContext::new(WorkflowInstanceId::new()); + + let result = step.execute(&mut context).await; + assert!(result.is_ok()); + assert_eq!(result.unwrap(), StepResult::Success); + } + + #[tokio::test] + async fn test_step_executor_failure() { + let step = TestStep { + should_succeed: false, + }; + let mut context = WorkflowContext::new(WorkflowInstanceId::new()); + + let result = step.execute(&mut context).await; + assert!(result.is_err()); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/mod.rs new file mode 100644 index 00000000..341ebea3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/mod.rs @@ -0,0 +1,21 @@ +//! Workflow engine for managing multi-step operations + +mod definition; +mod engine; +mod event; +mod executor; +mod state; +pub mod steps; +pub mod types; + +// Re-export main types +pub use definition::{StepDefinition, WorkflowDefinition}; +pub use engine::WorkflowEngine; +pub use event::{EventBus, EventSubscriber, LoggingSubscriber, WorkflowEvent}; +pub use executor::{FunctionStep, StepExecutor}; +pub use state::WorkflowStateStore; +pub use steps::{ + create_external_worker_registration_workflow, create_mcp_registration_workflow, + create_worker_registration_workflow, create_worker_removal_workflow, +}; +pub use types::*; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/state.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/state.rs new file mode 100644 index 00000000..2811903b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/state.rs @@ -0,0 +1,175 @@ +//! Workflow state management + +use std::{collections::HashMap, sync::Arc}; + +use parking_lot::RwLock; + +use super::types::{ + WorkflowError, WorkflowInstanceId, WorkflowResult, WorkflowState, WorkflowStatus, +}; + +/// In-memory state storage for workflow instances +#[derive(Clone)] +pub struct WorkflowStateStore { + states: Arc>>, +} + +impl WorkflowStateStore { + pub fn new() -> Self { + Self { + states: Arc::new(RwLock::new(HashMap::new())), + } + } + + /// Save workflow state + /// + /// # Warning + /// + /// This emits a warning if the workflow context contains unserializable data, + /// which would be lost if state persistence is later implemented. + pub fn save(&self, state: WorkflowState) -> WorkflowResult<()> { + if state.context.has_unserializable_data() { + tracing::warn!( + instance_id = %state.instance_id, + data_count = state.context.data_len(), + "Saving workflow state with {} unserializable context entries. \ + This data cannot be persisted and will be lost on restart.", + state.context.data_len() + ); + } + self.states.write().insert(state.instance_id, state); + Ok(()) + } + + /// Load workflow state by instance ID + pub fn load(&self, instance_id: WorkflowInstanceId) -> WorkflowResult { + self.states + .read() + .get(&instance_id) + .cloned() + .ok_or(WorkflowError::NotFound(instance_id)) + } + + /// List all active workflows (Running or Pending) + pub fn list_active(&self) -> WorkflowResult> { + let states = self.states.read(); + Ok(states + .values() + .filter(|s| matches!(s.status, WorkflowStatus::Running | WorkflowStatus::Pending)) + .cloned() + .collect()) + } + + /// List all workflows + pub fn list_all(&self) -> WorkflowResult> { + let states = self.states.read(); + Ok(states.values().cloned().collect()) + } + + /// Delete workflow state + pub fn delete(&self, instance_id: WorkflowInstanceId) -> WorkflowResult<()> { + self.states.write().remove(&instance_id); + Ok(()) + } + + /// Update workflow state using a closure + pub fn update(&self, instance_id: WorkflowInstanceId, f: F) -> WorkflowResult<()> + where + F: FnOnce(&mut WorkflowState), + { + let mut states = self.states.write(); + let state = states + .get_mut(&instance_id) + .ok_or(WorkflowError::NotFound(instance_id))?; + f(state); + state.updated_at = chrono::Utc::now(); + Ok(()) + } + + /// Get count of workflows by status + pub fn count_by_status(&self, status: WorkflowStatus) -> usize { + self.states + .read() + .values() + .filter(|s| s.status == status) + .count() + } + + /// Get total count of all workflows + pub fn count(&self) -> usize { + self.states.read().len() + } + + /// Clean up old completed/failed/cancelled workflows beyond a time threshold + /// + /// This prevents unbounded memory growth by removing workflow states that + /// have been in a terminal state (Completed, Failed, Cancelled) for longer + /// than the specified TTL (time-to-live). + /// + /// Active workflows (Running, Pending, Paused) are never cleaned up. + /// + /// # Arguments + /// + /// * `ttl` - Time-to-live for terminal workflows. Workflows in terminal states + /// older than this will be removed. + /// + /// # Returns + /// + /// The number of workflow states removed. + pub fn cleanup_old_workflows(&self, ttl: std::time::Duration) -> usize { + let now = chrono::Utc::now(); + let mut states = self.states.write(); + let initial_count = states.len(); + + states.retain(|_, state| { + // Keep active workflows + if matches!( + state.status, + WorkflowStatus::Running | WorkflowStatus::Pending | WorkflowStatus::Paused + ) { + return true; + } + + // For terminal workflows, check age + let age = now + .signed_duration_since(state.updated_at) + .to_std() + .unwrap_or_default(); + age < ttl + }); + + let removed_count = initial_count - states.len(); + if removed_count > 0 { + tracing::info!( + removed = removed_count, + remaining = states.len(), + "Cleaned up old workflow states" + ); + } + removed_count + } + + /// Clean up a specific completed workflow immediately + /// + /// This is useful for cleaning up workflows right after they complete + /// when you know they won't be queried again. + pub fn cleanup_if_terminal(&self, instance_id: WorkflowInstanceId) -> bool { + let mut states = self.states.write(); + if let Some(state) = states.get(&instance_id) { + if matches!( + state.status, + WorkflowStatus::Completed | WorkflowStatus::Failed | WorkflowStatus::Cancelled + ) { + states.remove(&instance_id); + return true; + } + } + false + } +} + +impl Default for WorkflowStateStore { + fn default() -> Self { + Self::new() + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/steps/external_worker_registration.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/steps/external_worker_registration.rs new file mode 100644 index 00000000..aaefac8c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/steps/external_worker_registration.rs @@ -0,0 +1,586 @@ +//! External worker registration workflow steps +//! +//! This workflow handles registration of external API endpoints (OpenAI, xAI, Anthropic, etc.) +//! +//! Key features: +//! - Fetches models from /v1/models endpoint +//! - Groups dated model variants under base model names (e.g., gpt-4o, gpt-4o-2024-08-06) +//! - Infers ModelType from model ID patterns (LLM, embedding, image gen, audio, etc.) +//! +//! Workflow order: +//! 1. DiscoverModels - Fetch available models from /v1/models endpoint +//! 2. CreateExternalWorkers - Build worker objects for each discovered model +//! 3. RegisterWorkers - Register workers in registry +//! 4. UpdatePolicies - Update policy registry +//! 5. ActivateWorkers - Mark workers as healthy + +use std::{collections::HashMap, sync::Arc, time::Duration}; + +use async_trait::async_trait; +use once_cell::sync::Lazy; +use regex::Regex; +use reqwest::Client; +use serde::Deserialize; +use tracing::{debug, info}; + +use crate::{ + app_context::AppContext, + core::{ + model_card::{ModelCard, ProviderType}, + model_type::ModelType, + workflow::*, + BasicWorkerBuilder, CircuitBreakerConfig, ConnectionMode, HealthConfig, RuntimeType, + Worker, WorkerType, + }, + protocols::worker_spec::WorkerConfigRequest, +}; + +// HTTP client for API calls +static HTTP_CLIENT: Lazy = Lazy::new(|| { + Client::builder() + .timeout(Duration::from_secs(30)) + .build() + .expect("Failed to create HTTP client") +}); + +// Regex to strip date suffix: -YYYY-MM-DD or -YYYY-MM +static DATE_SUFFIX_PATTERN: Lazy = + Lazy::new(|| Regex::new(r"-\d{4}-\d{2}(-\d{2})?$").expect("Invalid date regex")); + +// ============================================================================ +// Model Discovery Types +// ============================================================================ + +/// OpenAI /v1/models response format +#[derive(Debug, Clone, Deserialize)] +pub struct ModelsResponse { + pub data: Vec, + #[serde(default)] + pub object: String, +} + +/// Individual model information from /v1/models +#[derive(Debug, Clone, Deserialize)] +pub struct ModelInfo { + pub id: String, + #[serde(default)] + pub object: String, + #[serde(default)] + pub created: Option, + #[serde(default)] + pub owned_by: Option, +} + +// ============================================================================ +// Model Discovery Functions +// ============================================================================ + +/// Group models by base name (stripping date suffixes) and create ModelCards with aliases. +/// +/// # Example +/// Input: `["gpt-4o", "gpt-4o-2024-05-13", "gpt-4o-2024-08-06", "gpt-4o-2024-11-20"]` +/// Output: `ModelCard { id: "gpt-4o", aliases: ["gpt-4o-2024-05-13", "gpt-4o-2024-08-06", "gpt-4o-2024-11-20"] }` +pub fn group_models_into_cards(models: Vec) -> Vec { + // Group model IDs by base name (with date stripped) + let mut groups: HashMap> = HashMap::new(); + for model in &models { + let base = DATE_SUFFIX_PATTERN.replace(&model.id, "").to_string(); + groups.entry(base).or_default().push(model.id.clone()); + } + + // Create ModelCard for each group + groups + .into_values() + .map(|mut variants| { + // Sort: shortest first (base name), then alphabetically + variants.sort_by(|a, b| a.len().cmp(&b.len()).then_with(|| a.cmp(b))); + + let primary_id = variants.remove(0); // shortest = primary ID + let aliases = variants; // rest = aliases + + let model_type = infer_model_type_from_id(&primary_id); + let provider = infer_provider_from_id(&primary_id); + + let mut card = ModelCard::new(&primary_id) + .with_aliases(aliases) + .with_model_type(model_type); + + if let Some(p) = provider { + card = card.with_provider(p); + } + + card + }) + .collect() +} + +/// Infer ModelType from model ID string. +/// +/// This function analyzes the model ID to determine what type of model it is +/// (LLM, embedding, image generation, audio, etc.). +pub fn infer_model_type_from_id(id: &str) -> ModelType { + let id_lower = id.to_lowercase(); + + // Embedding models + if id_lower.contains("embed") || id_lower.contains("ada-002") { + return ModelType::EMBED_MODEL; + } + + // Rerank models + if id_lower.contains("rerank") { + return ModelType::RERANK_MODEL; + } + + // Image generation models (DALL-E, Sora, gpt-image) + if id_lower.starts_with("dall-e") + || id_lower.starts_with("sora") + || (id_lower.contains("image") && !id_lower.contains("vision")) + { + return ModelType::IMAGE_MODEL; + } + + // Audio models (TTS, Whisper, realtime, audio) + if id_lower.starts_with("tts") + || id_lower.starts_with("whisper") + || id_lower.contains("audio") + || id_lower.contains("realtime") + || id_lower.contains("transcribe") + { + return ModelType::AUDIO_MODEL; + } + + // Moderation models + if id_lower.contains("moderation") { + return ModelType::MODERATION_MODEL; + } + + // Vision LLM (models with vision capability) + // gpt-4o, gpt-4-vision, etc. + if id_lower.contains("vision") || id_lower.contains("4o") { + return ModelType::VISION_LLM; + } + + // Reasoning models (o1, o3, etc.) + if id_lower.starts_with("o1") || id_lower.starts_with("o3") { + return ModelType::REASONING_LLM; + } + + // Default to standard LLM + ModelType::LLM +} + +/// Infer provider type from model ID string. +/// +/// Returns `None` for models that don't clearly indicate a provider. +fn infer_provider_from_id(id: &str) -> Option { + let id_lower = id.to_lowercase(); + + // OpenAI models + if id_lower.starts_with("gpt") + || id_lower.starts_with("o1") + || id_lower.starts_with("o3") + || id_lower.starts_with("dall-e") + || id_lower.starts_with("whisper") + || id_lower.starts_with("tts") + || id_lower.starts_with("text-embedding") + || id_lower.starts_with("babbage") + || id_lower.starts_with("davinci") + || id_lower.contains("omni") + { + return Some(ProviderType::OpenAI); + } + + // xAI/Grok models + if id_lower.starts_with("grok") { + return Some(ProviderType::XAI); + } + + // Anthropic Claude models + if id_lower.starts_with("claude") { + return Some(ProviderType::Anthropic); + } + + // Google Gemini models + if id_lower.starts_with("gemini") { + return Some(ProviderType::Gemini); + } + + None +} + +// ============================================================================ +// Workflow Steps +// ============================================================================ + +/// Step 1: Discover models from external /v1/models endpoint +pub struct DiscoverModelsStep; + +#[async_trait] +impl StepExecutor for DiscoverModelsStep { + async fn execute(&self, context: &mut WorkflowContext) -> WorkflowResult { + let config: Arc = context + .get("worker_config") + .ok_or_else(|| WorkflowError::ContextValueNotFound("worker_config".to_string()))?; + + debug!("Discovering models from external endpoint {}", config.url); + + let model_cards = fetch_models(&config.url, config.api_key.as_deref()) + .await + .map_err(|e| WorkflowError::StepFailed { + step_id: StepId::new("discover_models"), + message: format!("Failed to discover models from {}: {}", config.url, e), + })?; + + if model_cards.is_empty() { + return Err(WorkflowError::StepFailed { + step_id: StepId::new("discover_models"), + message: format!("No models discovered from {}", config.url), + }); + } + + info!( + "Discovered {} models from {}: {:?}", + model_cards.len(), + config.url, + model_cards.iter().map(|c| &c.id).collect::>() + ); + + context.set("model_cards", model_cards); + Ok(StepResult::Success) + } + + fn is_retryable(&self, _error: &WorkflowError) -> bool { + true // Network issues are retryable + } +} + +/// Fetch models from /v1/models endpoint +async fn fetch_models(url: &str, api_key: Option<&str>) -> Result, String> { + let base_url = url.trim_end_matches('/'); + let models_url = format!("{}/v1/models", base_url); + + let mut req = HTTP_CLIENT.get(&models_url); + if let Some(key) = api_key { + req = req.bearer_auth(key); + } + + let response = req + .send() + .await + .map_err(|e| format!("Failed to connect to {}: {}", models_url, e))?; + + if !response.status().is_success() { + return Err(format!( + "Server returned status {} from {}", + response.status(), + models_url + )); + } + + let models_response: ModelsResponse = response + .json() + .await + .map_err(|e| format!("Failed to parse models response: {}", e))?; + + debug!( + "Fetched {} raw models from {}", + models_response.data.len(), + url + ); + + // Group models into cards (e.g., gpt-4o-2024-08-06 → gpt-4o with aliases) + let model_cards = group_models_into_cards(models_response.data); + + debug!( + "Grouped into {} model cards with aliases", + model_cards.len() + ); + + Ok(model_cards) +} + +/// Step 2: Create worker objects for each discovered model +pub struct CreateExternalWorkersStep; + +#[async_trait] +impl StepExecutor for CreateExternalWorkersStep { + async fn execute(&self, context: &mut WorkflowContext) -> WorkflowResult { + let config: Arc = context + .get("worker_config") + .ok_or_else(|| WorkflowError::ContextValueNotFound("worker_config".to_string()))?; + let app_context: Arc = context + .get("app_context") + .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?; + let model_cards: Arc> = context + .get("model_cards") + .ok_or_else(|| WorkflowError::ContextValueNotFound("model_cards".to_string()))?; + + debug!( + "Creating {} external workers for {}", + model_cards.len(), + config.url + ); + + // Build configs from router settings + let circuit_breaker_config = { + let cfg = app_context.router_config.effective_circuit_breaker_config(); + CircuitBreakerConfig { + failure_threshold: cfg.failure_threshold, + success_threshold: cfg.success_threshold, + timeout_duration: Duration::from_secs(cfg.timeout_duration_secs), + window_duration: Duration::from_secs(cfg.window_duration_secs), + } + }; + + let health_config = { + let cfg = &app_context.router_config.health_check; + HealthConfig { + timeout_secs: cfg.timeout_secs, + check_interval_secs: cfg.check_interval_secs, + endpoint: cfg.endpoint.clone(), + failure_threshold: cfg.failure_threshold, + success_threshold: cfg.success_threshold, + } + }; + + // Build labels + let mut labels = config.labels.clone(); + if let Some(priority) = config.priority { + labels.insert("priority".to_string(), priority.to_string()); + } + if let Some(cost) = config.cost { + labels.insert("cost".to_string(), cost.to_string()); + } + + // Normalize URL (ensure https:// for external APIs) + let normalized_url = normalize_external_url(&config.url); + + // Create a worker for each model + let mut workers = Vec::new(); + for model_card in model_cards.iter() { + let mut builder = BasicWorkerBuilder::new(normalized_url.clone()) + .model(model_card.clone()) + .worker_type(WorkerType::Regular) + .connection_mode(ConnectionMode::Http) + .runtime_type(RuntimeType::External) + .circuit_breaker_config(circuit_breaker_config.clone()) + .health_config(health_config.clone()); + + if let Some(ref api_key) = config.api_key { + builder = builder.api_key(api_key.clone()); + } + + if !labels.is_empty() { + builder = builder.labels(labels.clone()); + } + + let worker = Arc::new(builder.build()) as Arc; + worker.set_healthy(false); + + debug!( + "Created external worker for model {} at {}", + model_card.id, normalized_url + ); + + workers.push(worker); + } + + info!( + "Created {} external workers from {}", + workers.len(), + config.url + ); + + context.set("workers", workers); + context.set("labels", labels); + Ok(StepResult::Success) + } + + fn is_retryable(&self, _error: &WorkflowError) -> bool { + false // Creation failures are config issues + } +} + +/// Normalize URL for external APIs (ensure https://) +fn normalize_external_url(url: &str) -> String { + if url.starts_with("http://") || url.starts_with("https://") { + url.to_string() + } else { + format!("https://{}", url) + } +} + +/// Step 3: Register workers in registry +pub struct RegisterExternalWorkersStep; + +#[async_trait] +impl StepExecutor for RegisterExternalWorkersStep { + async fn execute(&self, context: &mut WorkflowContext) -> WorkflowResult { + let config: Arc = context + .get("worker_config") + .ok_or_else(|| WorkflowError::ContextValueNotFound("worker_config".to_string()))?; + let app_context: Arc = context + .get("app_context") + .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?; + let workers: Arc>> = context + .get("workers") + .ok_or_else(|| WorkflowError::ContextValueNotFound("workers".to_string()))?; + + let mut worker_ids = Vec::new(); + for worker in workers.iter() { + let worker_id = app_context.worker_registry.register(Arc::clone(worker)); + worker_ids.push(worker_id.clone()); + debug!( + "Registered external worker {} (model: {}) with ID {:?}", + config.url, + worker.model_id(), + worker_id + ); + } + + context.set("worker_ids", worker_ids); + Ok(StepResult::Success) + } + + fn is_retryable(&self, _error: &WorkflowError) -> bool { + false + } +} + +/// Step 4: Update policy registry +pub struct UpdateExternalPoliciesStep; + +#[async_trait] +impl StepExecutor for UpdateExternalPoliciesStep { + async fn execute(&self, context: &mut WorkflowContext) -> WorkflowResult { + let config: Arc = context + .get("worker_config") + .ok_or_else(|| WorkflowError::ContextValueNotFound("worker_config".to_string()))?; + let labels: Arc> = context + .get("labels") + .ok_or_else(|| WorkflowError::ContextValueNotFound("labels".to_string()))?; + let app_context: Arc = context + .get("app_context") + .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?; + let workers: Arc>> = context + .get("workers") + .ok_or_else(|| WorkflowError::ContextValueNotFound("workers".to_string()))?; + + let policy_hint = labels.get("policy").map(|s| s.as_str()); + + // Each external worker has a different model_id + for worker in workers.iter() { + let model_id = worker.model_id().to_string(); + app_context + .policy_registry + .on_worker_added(&model_id, policy_hint); + } + + debug!( + "Updated policies for {} external workers from {}", + workers.len(), + config.url + ); + + Ok(StepResult::Success) + } + + fn is_retryable(&self, _error: &WorkflowError) -> bool { + false + } +} + +/// Step 5: Activate workers by marking them healthy +pub struct ActivateExternalWorkersStep; + +#[async_trait] +impl StepExecutor for ActivateExternalWorkersStep { + async fn execute(&self, context: &mut WorkflowContext) -> WorkflowResult { + let config: Arc = context + .get("worker_config") + .ok_or_else(|| WorkflowError::ContextValueNotFound("worker_config".to_string()))?; + let workers: Arc>> = context + .get("workers") + .ok_or_else(|| WorkflowError::ContextValueNotFound("workers".to_string()))?; + + for worker in workers.iter() { + worker.set_healthy(true); + } + + info!( + "Activated {} external workers from {} (marked as healthy)", + workers.len(), + config.url + ); + + Ok(StepResult::Success) + } + + fn is_retryable(&self, _error: &WorkflowError) -> bool { + false + } +} + +// ============================================================================ +// Workflow Definition +// ============================================================================ + +/// Create external worker registration workflow definition +pub fn create_external_worker_registration_workflow() -> WorkflowDefinition { + WorkflowDefinition::new( + "external_worker_registration", + "External Worker Registration", + ) + .add_step( + StepDefinition::new( + "discover_models", + "Discover Models", + Arc::new(DiscoverModelsStep), + ) + .with_retry(RetryPolicy { + max_attempts: 3, + backoff: BackoffStrategy::Exponential { + base: Duration::from_secs(1), + max: Duration::from_secs(10), + }, + }) + .with_timeout(Duration::from_secs(30)) + .with_failure_action(FailureAction::FailWorkflow), + ) + .add_step( + StepDefinition::new( + "create_workers", + "Create Workers", + Arc::new(CreateExternalWorkersStep), + ) + .with_timeout(Duration::from_secs(5)) + .with_failure_action(FailureAction::FailWorkflow), + ) + .add_step( + StepDefinition::new( + "register_workers", + "Register Workers", + Arc::new(RegisterExternalWorkersStep), + ) + .with_timeout(Duration::from_secs(5)) + .with_failure_action(FailureAction::FailWorkflow), + ) + .add_step( + StepDefinition::new( + "update_policies", + "Update Policies", + Arc::new(UpdateExternalPoliciesStep), + ) + .with_timeout(Duration::from_secs(5)) + .with_failure_action(FailureAction::ContinueNextStep), + ) + .add_step( + StepDefinition::new( + "activate_workers", + "Activate Workers", + Arc::new(ActivateExternalWorkersStep), + ) + .with_timeout(Duration::from_secs(5)) + .with_failure_action(FailureAction::FailWorkflow), + ) +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/steps/mcp_registration.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/steps/mcp_registration.rs new file mode 100644 index 00000000..94fbdd10 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/steps/mcp_registration.rs @@ -0,0 +1,304 @@ +//! MCP server registration workflow steps +//! +//! Each step is atomic and performs a single operation in the MCP server registration process. +//! Updated for flat manager architecture - single McpManager manages all clients directly. +//! +//! Workflow order: +//! 1. ConnectMcpServer - Establish connection to MCP server using McpManager::connect_server() +//! 2. DiscoverMcpInventory - Discover and cache inventory using McpManager::load_server_inventory() +//! 3. RegisterMcpServer - Register McpClient in McpManager's client map + +use std::{sync::Arc, time::Duration}; + +use async_trait::async_trait; +use rmcp::{service::RunningService, RoleClient}; +use tracing::{debug, error, info, warn}; + +use crate::{ + app_context::AppContext, + core::workflow::*, + mcp::{config::McpServerConfig, manager::McpManager}, +}; + +/// MCP server connection configuration +#[derive(Debug, Clone)] +pub struct McpServerConfigRequest { + /// Server name (unique identifier) + pub name: String, + /// Server configuration (transport, proxy, etc.) + pub config: McpServerConfig, +} + +impl McpServerConfigRequest { + /// Check if this server is required for router startup + pub fn is_required(&self) -> bool { + self.config.required + } +} + +/// Step 1: Connect to MCP server +/// +/// This step establishes a connection to the MCP server using the flat manager architecture. +/// The connection is retried aggressively (100 attempts) with a long timeout (2 hours) +/// to handle slow-starting servers or network issues. +pub struct ConnectMcpServerStep; + +#[async_trait] +impl StepExecutor for ConnectMcpServerStep { + async fn execute(&self, context: &mut WorkflowContext) -> WorkflowResult { + let config_request: Arc = context + .get("mcp_server_config") + .ok_or_else(|| WorkflowError::ContextValueNotFound("mcp_server_config".to_string()))?; + let app_context: Arc = context + .get("app_context") + .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?; + + debug!("Connecting to MCP server: {}", config_request.name); + + // Get proxy config from router_config if available, otherwise fall back to env + let proxy_config = app_context + .router_config + .mcp_config + .as_ref() + .and_then(|cfg| cfg.proxy.as_ref()); + + // Connect to MCP server + let client = McpManager::connect_server(&config_request.config, proxy_config) + .await + .map_err(|e| WorkflowError::StepFailed { + step_id: StepId::new("connect_mcp_server"), + message: format!( + "Failed to connect to MCP server {}: {}", + config_request.name, e + ), + })?; + + info!( + "Successfully connected to MCP server: {}", + config_request.name + ); + + // Store client in context (context.set() will wrap in Arc) + context.set("mcp_client", client); + + Ok(StepResult::Success) + } + + fn is_retryable(&self, _error: &WorkflowError) -> bool { + true // Connection failures are retryable + } +} + +/// Step 2: Discover MCP inventory (tools, prompts, resources) +/// +/// This step queries the MCP server for its capabilities using McpManager::load_server_inventory(). +/// - Tools: Available function calls +/// - Prompts: Reusable prompt templates +/// - Resources: Accessible files/data +pub struct DiscoverMcpInventoryStep; + +#[async_trait] +impl StepExecutor for DiscoverMcpInventoryStep { + async fn execute(&self, context: &mut WorkflowContext) -> WorkflowResult { + use rmcp::{service::RunningService, RoleClient}; + + let config_request: Arc = context + .get("mcp_server_config") + .ok_or_else(|| WorkflowError::ContextValueNotFound("mcp_server_config".to_string()))?; + let app_context: Arc = context + .get("app_context") + .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?; + let mcp_client: Arc> = context + .get("mcp_client") + .ok_or_else(|| WorkflowError::ContextValueNotFound("mcp_client".to_string()))?; + + debug!( + "Discovering inventory for MCP server: {}", + config_request.name + ); + + // Get shared ToolInventory from McpManager + let mcp_manager = + app_context + .mcp_manager + .get() + .ok_or_else(|| WorkflowError::StepFailed { + step_id: StepId::new("discover_mcp_inventory"), + message: "MCP manager not initialized".to_string(), + })?; + + let inventory = mcp_manager.inventory(); + + // Use the public load_server_inventory method + McpManager::load_server_inventory(&inventory, &config_request.name, &mcp_client).await; + + info!("Completed inventory discovery for {}", config_request.name); + + Ok(StepResult::Success) + } + + fn is_retryable(&self, _error: &WorkflowError) -> bool { + true // Discovery failures are retryable + } +} + +/// Step 3: Register MCP server in manager +/// +/// This step adds the MCP client to the McpManager's client map so it can be +/// used for tool calls and inventory management. +pub struct RegisterMcpServerStep; + +#[async_trait] +impl StepExecutor for RegisterMcpServerStep { + async fn execute(&self, context: &mut WorkflowContext) -> WorkflowResult { + use rmcp::{service::RunningService, RoleClient}; + + let config_request: Arc = context + .get("mcp_server_config") + .ok_or_else(|| WorkflowError::ContextValueNotFound("mcp_server_config".to_string()))?; + let app_context: Arc = context + .get("app_context") + .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?; + let mcp_client: Arc> = context + .get("mcp_client") + .ok_or_else(|| WorkflowError::ContextValueNotFound("mcp_client".to_string()))?; + + debug!("Registering MCP server: {}", config_request.name); + + // Get MCP manager from app context + let mcp_manager = + app_context + .mcp_manager + .get() + .ok_or_else(|| WorkflowError::StepFailed { + step_id: StepId::new("register_mcp_server"), + message: "MCP manager not initialized".to_string(), + })?; + + // Register the client in the manager's client map + mcp_manager.register_static_server(config_request.name.clone(), mcp_client); + + info!("Registered MCP server: {}", config_request.name); + + Ok(StepResult::Success) + } + + fn is_retryable(&self, _error: &WorkflowError) -> bool { + false // Registration is a simple operation, not retryable + } +} + +/// Step 4: Validate registration based on required flag +/// +/// This step checks if the server is marked as required. If the server is required +/// but wasn't successfully registered (client not in context), this step fails the workflow. +/// For optional servers, this step always succeeds, allowing the workflow to complete +/// even if earlier steps failed. +pub struct ValidateRegistrationStep; + +#[async_trait] +impl StepExecutor for ValidateRegistrationStep { + async fn execute(&self, context: &mut WorkflowContext) -> WorkflowResult { + let config_request: Arc = context + .get("mcp_server_config") + .ok_or_else(|| WorkflowError::ContextValueNotFound("mcp_server_config".to_string()))?; + + let client_registered = context + .get::>("mcp_client") + .is_some(); + + if client_registered { + info!( + "MCP server '{}' registered successfully", + config_request.name + ); + return Ok(StepResult::Success); + } + + if config_request.is_required() { + error!( + "Required MCP server '{}' failed to register", + config_request.name + ); + Err(WorkflowError::StepFailed { + step_id: StepId::new("validate_registration"), + message: format!( + "Required MCP server '{}' failed to register", + config_request.name + ), + }) + } else { + warn!( + "Optional MCP server '{}' failed to register, continuing workflow", + config_request.name + ); + Ok(StepResult::Success) + } + } + + fn is_retryable(&self, _error: &WorkflowError) -> bool { + false + } +} + +/// Create MCP server registration workflow +/// +/// This workflow adapts its failure behavior based on the `required` field in the server config: +/// - If `required == true`: Uses FailWorkflow - router startup fails if server cannot be reached +/// - If `required == false` (default): Uses ContinueNextStep - logs warning but continues +/// +/// Workflow configuration: +/// - ConnectMcpServer: 100 retries, 2hr timeout (aggressive retry for slow servers) +/// - DiscoverMcpInventory: 3 retries, 10s timeout (discovery + caching) +/// - RegisterMcpServer: No retry, 5s timeout (fast registration) +/// - ValidateRegistration: Final validation step +pub fn create_mcp_registration_workflow() -> WorkflowDefinition { + WorkflowDefinition::new("mcp_registration", "MCP Server Registration") + .add_step( + StepDefinition::new( + "connect_mcp_server", + "Connect to MCP Server", + Arc::new(ConnectMcpServerStep), + ) + .with_retry(RetryPolicy { + max_attempts: 100, + backoff: BackoffStrategy::Linear { + increment: Duration::from_secs(1), + max: Duration::from_secs(5), + }, + }) + .with_timeout(Duration::from_secs(7200)) // 2 hours + .with_failure_action(FailureAction::ContinueNextStep), + ) + .add_step( + StepDefinition::new( + "discover_mcp_inventory", + "Discover and Cache MCP Inventory", + Arc::new(DiscoverMcpInventoryStep), + ) + .with_retry(RetryPolicy { + max_attempts: 3, + backoff: BackoffStrategy::Fixed(Duration::from_secs(1)), + }) + .with_timeout(Duration::from_secs(10)) + .with_failure_action(FailureAction::ContinueNextStep), + ) + .add_step( + StepDefinition::new( + "register_mcp_server", + "Register MCP Server", + Arc::new(RegisterMcpServerStep), + ) + .with_timeout(Duration::from_secs(5)) + .with_failure_action(FailureAction::ContinueNextStep), + ) + .add_step( + StepDefinition::new( + "validate_registration", + "Validate MCP Registration", + Arc::new(ValidateRegistrationStep), + ) + .with_timeout(Duration::from_secs(1)) + .with_failure_action(FailureAction::FailWorkflow), + ) +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/steps/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/steps/mod.rs new file mode 100644 index 00000000..a3f67cc0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/steps/mod.rs @@ -0,0 +1,32 @@ +//! Workflow step implementations +//! +//! This module contains concrete step implementations for various workflows: +//! - Local worker registration (SGLang, vLLM - gRPC or HTTP modes) +//! - External worker registration (OpenAI, xAI, Anthropic, etc. - HTTPS only) +//! - Worker removal +//! - MCP server registration + +pub mod external_worker_registration; +pub mod mcp_registration; +pub mod worker_registration; +pub mod worker_removal; + +pub use external_worker_registration::{ + create_external_worker_registration_workflow, group_models_into_cards, + infer_model_type_from_id, ActivateExternalWorkersStep, CreateExternalWorkersStep, + DiscoverModelsStep, ModelInfo, ModelsResponse, RegisterExternalWorkersStep, + UpdateExternalPoliciesStep, +}; +pub use mcp_registration::{ + create_mcp_registration_workflow, ConnectMcpServerStep, DiscoverMcpInventoryStep, + McpServerConfigRequest, RegisterMcpServerStep, ValidateRegistrationStep, +}; +pub use worker_registration::{ + create_worker_registration_workflow, ActivateWorkerStep, CreateWorkerStep, + DetectConnectionModeStep, DiscoverDPInfoStep, DiscoverMetadataStep, RegisterWorkerStep, + UpdatePoliciesStep, +}; +pub use worker_removal::{ + create_worker_removal_workflow, FindWorkersToRemoveStep, RemoveFromPolicyRegistryStep, + RemoveFromWorkerRegistryStep, UpdateRemainingPoliciesStep, WorkerRemovalRequest, +}; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/steps/worker_registration.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/steps/worker_registration.rs new file mode 100644 index 00000000..c8b4a09c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/steps/worker_registration.rs @@ -0,0 +1,926 @@ +//! Local worker registration workflow steps +//! +//! This workflow handles registration of local inference workers (SGLang, vLLM). +//! For external API endpoints (OpenAI, xAI, etc.), see external_worker_registration.rs. +//! +//! Workflow order: +//! 1. DetectConnectionMode - Probe HTTP and gRPC to determine connection mode +//! 2. DiscoverMetadata - Fetch metadata from /get_server_info or gRPC +//! 3. DiscoverDPInfo - Fetch DP (Data Parallel) information (only for DP-aware workers) +//! 4. CreateWorker - Build worker object(s) with merged config + metadata +//! 5. RegisterWorker - Register worker(s) in registry +//! 6. UpdatePolicies - Update policy registry with worker information +//! 7. ActivateWorker - Mark worker(s) as healthy + +use std::{collections::HashMap, sync::Arc, time::Duration}; + +use async_trait::async_trait; +use once_cell::sync::Lazy; +use reqwest::Client; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tracing::{debug, info, warn}; + +use crate::{ + app_context::AppContext, + core::{ + workflow::*, BasicWorkerBuilder, CircuitBreakerConfig, ConnectionMode, + DPAwareWorkerBuilder, HealthConfig, ModelCard, RuntimeType, Worker, WorkerType, + }, + protocols::worker_spec::WorkerConfigRequest, + routers::grpc::client::GrpcClient, +}; + +// HTTP client for metadata fetching +static HTTP_CLIENT: Lazy = Lazy::new(|| { + Client::builder() + .timeout(Duration::from_secs(10)) + .build() + .expect("Failed to create HTTP client") +}); + +/// Server information returned from /get_server_info endpoint +#[derive(Debug, Clone, Deserialize, Serialize)] +struct ServerInfo { + #[serde(alias = "model")] + model_id: Option, + model_path: Option, + served_model_name: Option, + dp_size: Option, + version: Option, + max_batch_size: Option, + max_total_tokens: Option, + max_prefill_tokens: Option, + max_running_requests: Option, + max_num_reqs: Option, +} + +#[derive(Debug, Clone)] +pub struct DpInfo { + pub dp_size: usize, + pub model_id: String, +} + +/// Parse server info from JSON response +fn parse_server_info(json: Value) -> Result { + serde_json::from_value(json).map_err(|e| format!("Failed to parse server info: {}", e)) +} + +/// Get server info from /get_server_info endpoint +async fn get_server_info(url: &str, api_key: Option<&str>) -> Result { + let base_url = url.trim_end_matches('/'); + let server_info_url = format!("{}/get_server_info", base_url); + + let mut req = HTTP_CLIENT.get(&server_info_url); + if let Some(key) = api_key { + req = req.bearer_auth(key); + } + + let response = req + .send() + .await + .map_err(|e| format!("Failed to connect to {}: {}", server_info_url, e))?; + + if !response.status().is_success() { + return Err(format!( + "Server returned status {} from {}", + response.status(), + server_info_url + )); + } + + let json = response + .json::() + .await + .map_err(|e| format!("Failed to parse response from {}: {}", server_info_url, e))?; + + parse_server_info(json) +} + +/// Get DP info for a worker URL +async fn get_dp_info(url: &str, api_key: Option<&str>) -> Result { + let info = get_server_info(url, api_key).await?; + + let dp_size = info + .dp_size + .ok_or_else(|| format!("No dp_size in response from {}", url))?; + + let model_id = info + .model_id + .filter(|s| !s.is_empty()) + .or(info.served_model_name.filter(|s| !s.is_empty())) + .or_else(|| { + info.model_path + .and_then(|path| path.split('/').next_back().map(|s| s.to_string())) + }) + .unwrap_or_else(|| "unknown".to_string()); + + Ok(DpInfo { dp_size, model_id }) +} + +/// Strip protocol prefix from URL +fn strip_protocol(url: &str) -> String { + url.trim_start_matches("http://") + .trim_start_matches("https://") + .trim_start_matches("grpc://") + .to_string() +} + +/// Try HTTP health check +async fn try_http_health_check( + url: &str, + timeout_secs: u64, + client: &Client, +) -> Result<(), String> { + let is_https = url.starts_with("https://"); + let protocol = if is_https { "https" } else { "http" }; + let clean_url = strip_protocol(url); + let health_url = format!("{}://{}/health", protocol, clean_url); + + client + .get(&health_url) + .timeout(Duration::from_secs(timeout_secs)) + .send() + .await + .and_then(reqwest::Response::error_for_status) + .map_err(|e| format!("Health check failed: {}", e))?; + + Ok(()) +} + +/// Perform gRPC health check with runtime type +async fn do_grpc_health_check( + grpc_url: &str, + timeout_secs: u64, + runtime_type: &str, +) -> Result<(), String> { + let connect_future = GrpcClient::connect(grpc_url, runtime_type); + let client = tokio::time::timeout(Duration::from_secs(timeout_secs), connect_future) + .await + .map_err(|_| "gRPC connection timeout".to_string())? + .map_err(|e| format!("gRPC connection failed: {}", e))?; + + let health_future = client.health_check(); + tokio::time::timeout(Duration::from_secs(timeout_secs), health_future) + .await + .map_err(|_| "gRPC health check timeout".to_string())? + .map_err(|e| format!("gRPC health check failed: {}", e))?; + + Ok(()) +} + +/// Try gRPC health check (tries SGLang first, then vLLM if not specified) +async fn try_grpc_health_check( + url: &str, + timeout_secs: u64, + runtime_type: Option<&str>, +) -> Result<(), String> { + let grpc_url = if url.starts_with("grpc://") { + url.to_string() + } else { + format!("grpc://{}", strip_protocol(url)) + }; + + match runtime_type { + Some(runtime) => do_grpc_health_check(&grpc_url, timeout_secs, runtime).await, + None => { + // Try SGLang first, then vLLM as fallback + if let Ok(()) = do_grpc_health_check(&grpc_url, timeout_secs, "sglang").await { + return Ok(()); + } + do_grpc_health_check(&grpc_url, timeout_secs, "vllm") + .await + .map_err(|e| format!("gRPC failed (tried SGLang and vLLM): {}", e)) + } + } +} + +/// Fetch metadata from gRPC server +async fn do_fetch_grpc_metadata( + grpc_url: &str, + runtime_type: &str, +) -> Result, String> { + let client = GrpcClient::connect(grpc_url, runtime_type) + .await + .map_err(|e| format!("Failed to connect to gRPC: {}", e))?; + + let model_info = client + .get_model_info() + .await + .map_err(|e| format!("Failed to fetch gRPC metadata: {}", e))?; + + Ok(model_info.to_labels()) +} + +/// Fetch gRPC metadata (returns labels and detected runtime type) +async fn fetch_grpc_metadata( + url: &str, + runtime_type: Option<&str>, +) -> Result<(HashMap, String), String> { + let grpc_url = if url.starts_with("grpc://") { + url.to_string() + } else { + format!("grpc://{}", strip_protocol(url)) + }; + + match runtime_type { + Some(runtime) => { + let labels = do_fetch_grpc_metadata(&grpc_url, runtime).await?; + Ok((labels, runtime.to_string())) + } + None => { + // Try SGLang first, then vLLM as fallback + if let Ok(labels) = do_fetch_grpc_metadata(&grpc_url, "sglang").await { + return Ok((labels, "sglang".to_string())); + } + let labels = do_fetch_grpc_metadata(&grpc_url, "vllm") + .await + .map_err(|e| format!("gRPC metadata failed (tried SGLang and vLLM): {}", e))?; + Ok((labels, "vllm".to_string())) + } + } +} + +/// Step 1: Detect connection mode by probing HTTP and gRPC +pub struct DetectConnectionModeStep; + +#[async_trait] +impl StepExecutor for DetectConnectionModeStep { + async fn execute(&self, context: &mut WorkflowContext) -> WorkflowResult { + let config: Arc = context + .get("worker_config") + .ok_or_else(|| WorkflowError::ContextValueNotFound("worker_config".to_string()))?; + let app_context: Arc = context + .get("app_context") + .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?; + + debug!( + "Detecting connection mode for {} (timeout: {}s, max_attempts: {})", + config.url, config.health_check_timeout_secs, config.max_connection_attempts + ); + + // Try both protocols in parallel + let url = config.url.clone(); + let timeout = config.health_check_timeout_secs; + let client = &app_context.client; + let runtime_type = config.runtime.as_deref(); + + let (http_result, grpc_result) = tokio::join!( + try_http_health_check(&url, timeout, client), + try_grpc_health_check(&url, timeout, runtime_type) + ); + + let connection_mode = match (http_result, grpc_result) { + (Ok(_), _) => { + debug!("{} detected as HTTP", config.url); + ConnectionMode::Http + } + (_, Ok(_)) => { + debug!("{} detected as gRPC", config.url); + ConnectionMode::Grpc { port: None } + } + (Err(http_err), Err(grpc_err)) => { + return Err(WorkflowError::StepFailed { + step_id: StepId::new("detect_connection_mode"), + message: format!( + "Both HTTP and gRPC health checks failed for {}: HTTP: {}, gRPC: {}", + config.url, http_err, grpc_err + ), + }); + } + }; + + context.set("connection_mode", connection_mode); + Ok(StepResult::Success) + } + + fn is_retryable(&self, _error: &WorkflowError) -> bool { + true + } +} + +/// Step 2: Discover metadata from worker +pub struct DiscoverMetadataStep; + +#[async_trait] +impl StepExecutor for DiscoverMetadataStep { + async fn execute(&self, context: &mut WorkflowContext) -> WorkflowResult { + let config: Arc = context + .get("worker_config") + .ok_or_else(|| WorkflowError::ContextValueNotFound("worker_config".to_string()))?; + let connection_mode: Arc = context + .get("connection_mode") + .ok_or_else(|| WorkflowError::ContextValueNotFound("connection_mode".to_string()))?; + + debug!( + "Discovering metadata for {} ({:?})", + config.url, *connection_mode + ); + + let (discovered_labels, detected_runtime) = match connection_mode.as_ref() { + ConnectionMode::Http => { + match get_server_info(&config.url, config.api_key.as_deref()).await { + Ok(server_info) => { + let mut labels = HashMap::new(); + if let Some(model_path) = server_info.model_path.filter(|s| !s.is_empty()) { + labels.insert("model_path".to_string(), model_path); + } + if let Some(served_model_name) = + server_info.served_model_name.filter(|s| !s.is_empty()) + { + labels.insert("served_model_name".to_string(), served_model_name); + } + Ok((labels, None)) + } + Err(e) => Err(e), + } + } + ConnectionMode::Grpc { .. } => { + let runtime_type = config.runtime.as_deref(); + fetch_grpc_metadata(&config.url, runtime_type) + .await + .map(|(labels, runtime)| (labels, Some(runtime))) + } + } + .unwrap_or_else(|e| { + warn!("Failed to fetch metadata for {}: {}", config.url, e); + (HashMap::new(), None) + }); + + debug!( + "Discovered {} metadata labels for {}", + discovered_labels.len(), + config.url + ); + + context.set("discovered_labels", discovered_labels); + if let Some(runtime) = detected_runtime { + debug!("Detected runtime type: {}", runtime); + context.set("detected_runtime_type", runtime); + } + + Ok(StepResult::Success) + } + + fn is_retryable(&self, _error: &WorkflowError) -> bool { + true + } +} + +/// Step 2.5: Discover DP (Data Parallel) information (only for DP-aware workers) +pub struct DiscoverDPInfoStep; + +#[async_trait] +impl StepExecutor for DiscoverDPInfoStep { + async fn execute(&self, context: &mut WorkflowContext) -> WorkflowResult { + let config: Arc = context + .get("worker_config") + .ok_or_else(|| WorkflowError::ContextValueNotFound("worker_config".to_string()))?; + + if !config.dp_aware { + debug!( + "Worker {} is not DP-aware, skipping DP discovery", + config.url + ); + return Ok(StepResult::Success); + } + + debug!("Discovering DP info for {} (DP-aware)", config.url); + + let dp_info = get_dp_info(&config.url, config.api_key.as_deref()) + .await + .map_err(|e| WorkflowError::StepFailed { + step_id: StepId::new("discover_dp_info"), + message: format!("Failed to get DP info: {}", e), + })?; + + debug!( + "Discovered DP size {} for {} (model: {})", + dp_info.dp_size, config.url, dp_info.model_id + ); + + context.set("dp_info", dp_info); + Ok(StepResult::Success) + } + + fn is_retryable(&self, _error: &WorkflowError) -> bool { + true + } +} + +/// Step 3: Create worker object with merged configuration + metadata +pub struct CreateWorkerStep; + +#[async_trait] +impl StepExecutor for CreateWorkerStep { + async fn execute(&self, context: &mut WorkflowContext) -> WorkflowResult { + let config: Arc = context + .get("worker_config") + .ok_or_else(|| WorkflowError::ContextValueNotFound("worker_config".to_string()))?; + let app_context: Arc = context + .get("app_context") + .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?; + let connection_mode: Arc = context + .get("connection_mode") + .ok_or_else(|| WorkflowError::ContextValueNotFound("connection_mode".to_string()))?; + let discovered_labels: Arc> = context + .get("discovered_labels") + .ok_or_else(|| WorkflowError::ContextValueNotFound("discovered_labels".to_string()))?; + + // Check if worker already exists + if app_context + .worker_registry + .get_by_url(&config.url) + .is_some() + { + return Err(WorkflowError::StepFailed { + step_id: StepId::new("create_worker"), + message: format!("Worker {} already exists", config.url), + }); + } + + // Build labels from config + let mut config_labels = config.labels.clone(); + if let Some(priority) = config.priority { + config_labels.insert("priority".to_string(), priority.to_string()); + } + if let Some(cost) = config.cost { + config_labels.insert("cost".to_string(), cost.to_string()); + } + + // Merge: discovered labels first, then config labels (config takes precedence) + let mut final_labels = discovered_labels.as_ref().clone(); + for (key, value) in &config_labels { + final_labels.insert(key.clone(), value.clone()); + } + + // Determine model_id: config > served_model_name > model_path > "unknown" + let model_id = config + .model_id + .clone() + .or_else(|| final_labels.get("served_model_name").cloned()) + .or_else(|| final_labels.get("model_path").cloned()) + .unwrap_or_else(|| "unknown".to_string()); + + if model_id != "unknown" { + debug!("Using model_id: {}", model_id); + } + + // Create ModelCard + let model_card = { + let mut card = ModelCard::new(&model_id); + if let Some(ref tokenizer_path) = config.tokenizer_path { + card = card.with_tokenizer_path(tokenizer_path.clone()); + } + if let Some(ref reasoning_parser) = config.reasoning_parser { + card = card.with_reasoning_parser(reasoning_parser.clone()); + } + if let Some(ref tool_parser) = config.tool_parser { + card = card.with_tool_parser(tool_parser.clone()); + } + if let Some(ref chat_template) = config.chat_template { + card = card.with_chat_template(chat_template.clone()); + } + card + }; + + debug!( + "Creating worker {} with {} discovered + {} config = {} final labels", + config.url, + discovered_labels.len(), + config_labels.len(), + final_labels.len() + ); + + // Parse worker type + let worker_type = config + .worker_type + .as_ref() + .map(|t| match t.as_str() { + "prefill" => WorkerType::Prefill { + bootstrap_port: config.bootstrap_port, + }, + "decode" => WorkerType::Decode, + _ => WorkerType::Regular, + }) + .unwrap_or(WorkerType::Regular); + + // Get runtime type (for gRPC workers) + let runtime_type = if matches!(connection_mode.as_ref(), ConnectionMode::Grpc { .. }) { + if let Some(detected_runtime) = context.get::("detected_runtime_type") { + match detected_runtime.as_str() { + "vllm" => RuntimeType::Vllm, + _ => RuntimeType::Sglang, + } + } else if let Some(ref runtime) = config.runtime { + match runtime.as_str() { + "vllm" => RuntimeType::Vllm, + _ => RuntimeType::Sglang, + } + } else { + RuntimeType::Sglang + } + } else { + RuntimeType::Sglang + }; + + // Build circuit breaker config + let circuit_breaker_config = { + let cfg = app_context.router_config.effective_circuit_breaker_config(); + CircuitBreakerConfig { + failure_threshold: cfg.failure_threshold, + success_threshold: cfg.success_threshold, + timeout_duration: Duration::from_secs(cfg.timeout_duration_secs), + window_duration: Duration::from_secs(cfg.window_duration_secs), + } + }; + + // Build health config + let health_config = { + let cfg = &app_context.router_config.health_check; + HealthConfig { + timeout_secs: cfg.timeout_secs, + check_interval_secs: cfg.check_interval_secs, + endpoint: cfg.endpoint.clone(), + failure_threshold: cfg.failure_threshold, + success_threshold: cfg.success_threshold, + } + }; + + // Normalize URL: add protocol prefix only if missing + let normalized_url = if config.url.starts_with("http://") + || config.url.starts_with("https://") + || config.url.starts_with("grpc://") + { + config.url.clone() + } else { + match connection_mode.as_ref() { + ConnectionMode::Http => format!("http://{}", config.url), + ConnectionMode::Grpc { .. } => format!("grpc://{}", config.url), + } + }; + + if normalized_url != config.url { + debug!( + "Normalized worker URL: {} -> {} ({:?})", + config.url, + normalized_url, + connection_mode.as_ref() + ); + } + + // Handle DP-aware vs non-DP-aware workers + if config.dp_aware { + let dp_info: Arc = context + .get("dp_info") + .ok_or_else(|| WorkflowError::ContextValueNotFound("dp_info".to_string()))?; + + debug!( + "Creating {} DP-aware workers for {} (dp_size: {})", + dp_info.dp_size, config.url, dp_info.dp_size + ); + + let mut workers = Vec::new(); + for rank in 0..dp_info.dp_size { + let mut builder = + DPAwareWorkerBuilder::new(normalized_url.clone(), rank, dp_info.dp_size) + .model(model_card.clone()) + .worker_type(worker_type.clone()) + .connection_mode(connection_mode.as_ref().clone()) + .runtime_type(runtime_type.clone()) + .circuit_breaker_config(circuit_breaker_config.clone()) + .health_config(health_config.clone()); + + if let Some(ref api_key) = config.api_key { + builder = builder.api_key(api_key.clone()); + } + if !final_labels.is_empty() { + builder = builder.labels(final_labels.clone()); + } + + let worker = Arc::new(builder.build()) as Arc; + worker.set_healthy(false); + workers.push(worker); + + debug!( + "Created DP-aware worker {}@{}/{} ({:?})", + config.url, + rank, + dp_info.dp_size, + connection_mode.as_ref() + ); + } + + context.set("workers", workers); + context.set("labels", final_labels); + } else { + let mut builder = BasicWorkerBuilder::new(normalized_url.clone()) + .model(model_card) + .worker_type(worker_type) + .connection_mode(connection_mode.as_ref().clone()) + .runtime_type(runtime_type) + .circuit_breaker_config(circuit_breaker_config) + .health_config(health_config); + + if let Some(ref api_key) = config.api_key { + builder = builder.api_key(api_key.clone()); + } + if !final_labels.is_empty() { + builder = builder.labels(final_labels.clone()); + } + + let worker = Arc::new(builder.build()) as Arc; + worker.set_healthy(false); + + debug!( + "Created worker object for {} ({:?}) with {} labels", + config.url, + connection_mode.as_ref(), + final_labels.len() + ); + + context.set("worker", worker); + context.set("labels", final_labels); + } + + Ok(StepResult::Success) + } + + fn is_retryable(&self, _error: &WorkflowError) -> bool { + false + } +} + +/// Step 4: Register worker(s) in registry +pub struct RegisterWorkerStep; + +#[async_trait] +impl StepExecutor for RegisterWorkerStep { + async fn execute(&self, context: &mut WorkflowContext) -> WorkflowResult { + let config: Arc = context + .get("worker_config") + .ok_or_else(|| WorkflowError::ContextValueNotFound("worker_config".to_string()))?; + let app_context: Arc = context + .get("app_context") + .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?; + + if config.dp_aware { + let workers: Arc>> = context + .get("workers") + .ok_or_else(|| WorkflowError::ContextValueNotFound("workers".to_string()))?; + + let mut worker_ids = Vec::new(); + for worker in workers.iter() { + let worker_id = app_context.worker_registry.register(Arc::clone(worker)); + worker_ids.push(worker_id.clone()); + debug!( + "Registered DP-aware worker {} with ID {:?}", + config.url, worker_id + ); + } + + context.set("worker_ids", worker_ids); + } else { + let worker: Arc> = context + .get("worker") + .ok_or_else(|| WorkflowError::ContextValueNotFound("worker".to_string()))?; + + let worker_id = app_context + .worker_registry + .register(Arc::clone(worker.as_ref())); + debug!("Registered worker {} with ID {:?}", config.url, worker_id); + context.set("worker_id", worker_id); + } + + Ok(StepResult::Success) + } + + fn is_retryable(&self, _error: &WorkflowError) -> bool { + false + } +} + +/// Step 5: Update policy registry with worker information +pub struct UpdatePoliciesStep; + +#[async_trait] +impl StepExecutor for UpdatePoliciesStep { + async fn execute(&self, context: &mut WorkflowContext) -> WorkflowResult { + let config: Arc = context + .get("worker_config") + .ok_or_else(|| WorkflowError::ContextValueNotFound("worker_config".to_string()))?; + let labels: Arc> = context + .get("labels") + .ok_or_else(|| WorkflowError::ContextValueNotFound("labels".to_string()))?; + let app_context: Arc = context + .get("app_context") + .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?; + + let policy_hint = labels.get("policy").map(|s| s.as_str()); + + if config.dp_aware { + let workers: Arc>> = context + .get("workers") + .ok_or_else(|| WorkflowError::ContextValueNotFound("workers".to_string()))?; + + let model_id = workers[0].model_id().to_string(); + + for _ in 0..workers.len() { + app_context + .policy_registry + .on_worker_added(&model_id, policy_hint); + } + + // Initialize cache-aware policy if needed + let all_workers = app_context.worker_registry.get_by_model_fast(&model_id); + if let Some(policy) = app_context.policy_registry.get_policy(&model_id) { + if policy.name() == "cache_aware" { + app_context + .policy_registry + .init_cache_aware_policy(&model_id, &all_workers); + } + } + + debug!( + "Updated policies for {} DP-aware workers {} (model: {})", + workers.len(), + config.url, + model_id + ); + } else { + let worker: Arc> = context + .get("worker") + .ok_or_else(|| WorkflowError::ContextValueNotFound("worker".to_string()))?; + + let model_id = worker.model_id().to_string(); + + app_context + .policy_registry + .on_worker_added(&model_id, policy_hint); + + // Initialize cache-aware policy if needed + let all_workers = app_context.worker_registry.get_by_model_fast(&model_id); + if let Some(policy) = app_context.policy_registry.get_policy(&model_id) { + if policy.name() == "cache_aware" { + app_context + .policy_registry + .init_cache_aware_policy(&model_id, &all_workers); + } + } + + // Initialize bucket policies for prefill workers + let prefill_workers = app_context.worker_registry.get_prefill_workers(); + let policy = app_context.policy_registry.get_prefill_policy(); + if policy.name() == "bucket" { + app_context + .policy_registry + .init_pd_bucket_policies(&prefill_workers); + } + + debug!( + "Updated policies for worker {} (model: {})", + config.url, model_id + ); + } + + Ok(StepResult::Success) + } + + fn is_retryable(&self, _error: &WorkflowError) -> bool { + false + } +} + +/// Step 6: Activate worker(s) by marking them as healthy +pub struct ActivateWorkerStep; + +#[async_trait] +impl StepExecutor for ActivateWorkerStep { + async fn execute(&self, context: &mut WorkflowContext) -> WorkflowResult { + let config: Arc = context + .get("worker_config") + .ok_or_else(|| WorkflowError::ContextValueNotFound("worker_config".to_string()))?; + + if config.dp_aware { + let workers: Arc>> = context + .get("workers") + .ok_or_else(|| WorkflowError::ContextValueNotFound("workers".to_string()))?; + + for worker in workers.iter() { + worker.set_healthy(true); + } + + info!( + "Activated {} DP-aware workers from {} (marked as healthy)", + workers.len(), + config.url + ); + } else { + let worker: Arc> = context + .get("worker") + .ok_or_else(|| WorkflowError::ContextValueNotFound("worker".to_string()))?; + + worker.set_healthy(true); + + info!("Activated worker {} (marked as healthy)", config.url); + } + + Ok(StepResult::Success) + } + + fn is_retryable(&self, _error: &WorkflowError) -> bool { + false + } +} + +/// Create local worker registration workflow definition +pub fn create_worker_registration_workflow( + router_config: &crate::config::RouterConfig, +) -> WorkflowDefinition { + let detect_timeout = Duration::from_secs(router_config.worker_startup_timeout_secs); + + // Calculate max_attempts based on timeout + let timeout_secs = detect_timeout.as_secs() as f64; + let effective_timeout = timeout_secs * 0.9; + let max_attempts = if effective_timeout > 10.0 { + (5 + ((effective_timeout - 10.0) / 5.0).ceil() as u32).max(3) + } else { + 3 + }; + + WorkflowDefinition::new("worker_registration", "Worker Registration") + .add_step( + StepDefinition::new( + "detect_connection_mode", + "Detect Connection Mode", + Arc::new(DetectConnectionModeStep), + ) + .with_retry(RetryPolicy { + max_attempts, + backoff: BackoffStrategy::Linear { + increment: Duration::from_secs(1), + max: Duration::from_secs(5), + }, + }) + .with_timeout(detect_timeout) + .with_failure_action(FailureAction::FailWorkflow), + ) + .add_step( + StepDefinition::new( + "discover_metadata", + "Discover Metadata", + Arc::new(DiscoverMetadataStep), + ) + .with_retry(RetryPolicy { + max_attempts: 3, + backoff: BackoffStrategy::Fixed(Duration::from_secs(1)), + }) + .with_timeout(Duration::from_secs(10)) + .with_failure_action(FailureAction::ContinueNextStep), + ) + .add_step( + StepDefinition::new( + "discover_dp_info", + "Discover DP Info", + Arc::new(DiscoverDPInfoStep), + ) + .with_retry(RetryPolicy { + max_attempts: 3, + backoff: BackoffStrategy::Fixed(Duration::from_secs(1)), + }) + .with_timeout(Duration::from_secs(10)) + .with_failure_action(FailureAction::FailWorkflow), + ) + .add_step( + StepDefinition::new("create_worker", "Create Worker", Arc::new(CreateWorkerStep)) + .with_timeout(Duration::from_secs(5)) + .with_failure_action(FailureAction::FailWorkflow), + ) + .add_step( + StepDefinition::new( + "register_worker", + "Register Worker", + Arc::new(RegisterWorkerStep), + ) + .with_timeout(Duration::from_secs(5)) + .with_failure_action(FailureAction::FailWorkflow), + ) + .add_step( + StepDefinition::new( + "update_policies", + "Update Policies", + Arc::new(UpdatePoliciesStep), + ) + .with_timeout(Duration::from_secs(5)) + .with_failure_action(FailureAction::ContinueNextStep), + ) + .add_step( + StepDefinition::new( + "activate_worker", + "Activate Worker", + Arc::new(ActivateWorkerStep), + ) + .with_timeout(Duration::from_secs(5)) + .with_failure_action(FailureAction::FailWorkflow), + ) +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/steps/worker_removal.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/steps/worker_removal.rs new file mode 100644 index 00000000..a26ab85e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/steps/worker_removal.rs @@ -0,0 +1,310 @@ +//! Worker Removal Workflow Steps +//! +//! This module implements the workflow steps for removing workers from the router. +//! Handles both single worker removal and DP-aware worker removal with prefix matching. +//! +//! Steps: +//! 1. FindWorkersToRemove - Identify workers to remove based on URL (handles DP-aware prefix matching) +//! 2. RemoveFromPolicyRegistry - Remove workers from policy registry and cache-aware policies +//! 3. RemoveFromWorkerRegistry - Remove workers from worker registry +//! 4. UpdateRemainingPolicies - Update cache-aware policies for remaining workers + +use std::{collections::HashSet, sync::Arc}; + +use async_trait::async_trait; +use tracing::{debug, info}; + +use crate::{ + app_context::AppContext, + core::{workflow::*, Worker}, +}; + +/// Request structure for worker removal +#[derive(Debug, Clone)] +pub struct WorkerRemovalRequest { + pub url: String, + pub dp_aware: bool, +} + +/// Step 1: Find workers to remove based on URL +pub struct FindWorkersToRemoveStep; + +#[async_trait] +impl StepExecutor for FindWorkersToRemoveStep { + async fn execute(&self, context: &mut WorkflowContext) -> WorkflowResult { + let request: Arc = context + .get("removal_request") + .ok_or_else(|| WorkflowError::ContextValueNotFound("removal_request".to_string()))?; + let app_context: Arc = context + .get("app_context") + .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?; + + debug!( + "Finding workers to remove for {} (dp_aware: {})", + request.url, request.dp_aware + ); + + let workers_to_remove: Vec> = if request.dp_aware { + // DP-aware: Find all workers with matching prefix + let worker_url_prefix = format!("{}@", request.url); + let all_workers = app_context.worker_registry.get_all(); + + all_workers + .iter() + .filter(|worker| worker.url().starts_with(&worker_url_prefix)) + .cloned() + .collect() + } else { + // Non-DP-aware: Find single worker by exact URL + match app_context.worker_registry.get_by_url(&request.url) { + Some(worker) => vec![worker], + None => Vec::new(), + } + }; + + if workers_to_remove.is_empty() { + let error_msg = if request.dp_aware { + format!("No workers found with prefix {}@", request.url) + } else { + format!("Worker {} not found", request.url) + }; + return Err(WorkflowError::StepFailed { + step_id: StepId::new("find_workers_to_remove"), + message: error_msg, + }); + } + + debug!( + "Found {} worker(s) to remove for {}", + workers_to_remove.len(), + request.url + ); + + // Store workers and their model IDs for subsequent steps + let worker_urls: Vec = workers_to_remove + .iter() + .map(|w| w.url().to_string()) + .collect(); + + let affected_models: HashSet = workers_to_remove + .iter() + .map(|w| w.model_id().to_string()) + .collect(); + + context.set("workers_to_remove", workers_to_remove); + context.set("worker_urls", worker_urls); + context.set("affected_models", affected_models); + + Ok(StepResult::Success) + } + + fn is_retryable(&self, _error: &WorkflowError) -> bool { + false // Worker not found is not retryable + } +} + +/// Step 2: Remove workers from policy registry +pub struct RemoveFromPolicyRegistryStep; + +#[async_trait] +impl StepExecutor for RemoveFromPolicyRegistryStep { + async fn execute(&self, context: &mut WorkflowContext) -> WorkflowResult { + let app_context: Arc = context + .get("app_context") + .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?; + let workers_to_remove: Arc>> = context + .get("workers_to_remove") + .ok_or_else(|| WorkflowError::ContextValueNotFound("workers_to_remove".to_string()))?; + + debug!( + "Removing {} worker(s) from policy registry", + workers_to_remove.len() + ); + + for worker in workers_to_remove.iter() { + let model_id = worker.model_id().to_string(); + let worker_url = worker.url(); + + // Remove from cache-aware policy + app_context + .policy_registry + .remove_worker_from_cache_aware(&model_id, worker_url); + + // Notify policy registry + app_context.policy_registry.on_worker_removed(&model_id); + + debug!( + "Removed worker {} from policy registry (model: {})", + worker_url, model_id + ); + } + + Ok(StepResult::Success) + } + + fn is_retryable(&self, _error: &WorkflowError) -> bool { + false // Policy removal is not retryable + } +} + +/// Step 3: Remove workers from worker registry +pub struct RemoveFromWorkerRegistryStep; + +#[async_trait] +impl StepExecutor for RemoveFromWorkerRegistryStep { + async fn execute(&self, context: &mut WorkflowContext) -> WorkflowResult { + let app_context: Arc = context + .get("app_context") + .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?; + let worker_urls: Arc> = context + .get("worker_urls") + .ok_or_else(|| WorkflowError::ContextValueNotFound("worker_urls".to_string()))?; + + debug!( + "Removing {} worker(s) from worker registry", + worker_urls.len() + ); + + let mut removed_count = 0; + for worker_url in worker_urls.iter() { + if app_context + .worker_registry + .remove_by_url(worker_url) + .is_some() + { + removed_count += 1; + debug!("Removed worker {} from registry", worker_url); + } + } + + if removed_count != worker_urls.len() { + return Err(WorkflowError::StepFailed { + step_id: StepId::new("remove_from_worker_registry"), + message: format!( + "Expected to remove {} workers but only removed {}", + worker_urls.len(), + removed_count + ), + }); + } + + Ok(StepResult::Success) + } + + fn is_retryable(&self, _error: &WorkflowError) -> bool { + false // Worker removal is not retryable + } +} + +/// Step 4: Update cache-aware policies for remaining workers +pub struct UpdateRemainingPoliciesStep; + +#[async_trait] +impl StepExecutor for UpdateRemainingPoliciesStep { + async fn execute(&self, context: &mut WorkflowContext) -> WorkflowResult { + let app_context: Arc = context + .get("app_context") + .ok_or_else(|| WorkflowError::ContextValueNotFound("app_context".to_string()))?; + let affected_models: Arc> = context + .get("affected_models") + .ok_or_else(|| WorkflowError::ContextValueNotFound("affected_models".to_string()))?; + let worker_urls: Arc> = context + .get("worker_urls") + .ok_or_else(|| WorkflowError::ContextValueNotFound("worker_urls".to_string()))?; + + debug!( + "Updating cache-aware policies for {} affected model(s)", + affected_models.len() + ); + + for model_id in affected_models.iter() { + let remaining_workers = app_context.worker_registry.get_by_model_fast(model_id); + + if let Some(policy) = app_context.policy_registry.get_policy(model_id) { + if policy.name() == "cache_aware" && !remaining_workers.is_empty() { + app_context + .policy_registry + .init_cache_aware_policy(model_id, &remaining_workers); + + debug!( + "Updated cache-aware policy for model {} ({} remaining workers)", + model_id, + remaining_workers.len() + ); + } + } + } + + // Log final result at info level + if worker_urls.len() == 1 { + info!("Removed worker {}", worker_urls[0]); + } else { + info!( + "Removed {} DP-aware workers: {:?}", + worker_urls.len(), + worker_urls + ); + } + + Ok(StepResult::Success) + } + + fn is_retryable(&self, _error: &WorkflowError) -> bool { + false // Policy update is not retryable + } +} + +/// Create a worker removal workflow definition +pub fn create_worker_removal_workflow() -> WorkflowDefinition { + use std::time::Duration; + + WorkflowDefinition::new("worker_removal", "Remove worker from router") + .add_step( + StepDefinition::new( + "find_workers_to_remove", + "Find workers to remove", + Arc::new(FindWorkersToRemoveStep), + ) + .with_timeout(Duration::from_secs(10)) + .with_retry(RetryPolicy { + max_attempts: 1, + backoff: BackoffStrategy::Fixed(Duration::from_secs(0)), + }), + ) + .add_step( + StepDefinition::new( + "remove_from_policy_registry", + "Remove workers from policy registry", + Arc::new(RemoveFromPolicyRegistryStep), + ) + .with_timeout(Duration::from_secs(10)) + .with_retry(RetryPolicy { + max_attempts: 1, + backoff: BackoffStrategy::Fixed(Duration::from_secs(0)), + }), + ) + .add_step( + StepDefinition::new( + "remove_from_worker_registry", + "Remove workers from worker registry", + Arc::new(RemoveFromWorkerRegistryStep), + ) + .with_timeout(Duration::from_secs(10)) + .with_retry(RetryPolicy { + max_attempts: 1, + backoff: BackoffStrategy::Fixed(Duration::from_secs(0)), + }), + ) + .add_step( + StepDefinition::new( + "update_remaining_policies", + "Update cache-aware policies for remaining workers", + Arc::new(UpdateRemainingPoliciesStep), + ) + .with_timeout(Duration::from_secs(10)) + .with_retry(RetryPolicy { + max_attempts: 1, + backoff: BackoffStrategy::Fixed(Duration::from_secs(0)), + }), + ) +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/types.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/types.rs new file mode 100644 index 00000000..e7af5c65 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/core/workflow/types.rs @@ -0,0 +1,271 @@ +//! Core workflow types and definitions + +use std::{collections::HashMap, fmt, sync::Arc, time::Duration}; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +/// Unique identifier for a workflow definition +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct WorkflowId(String); + +impl WorkflowId { + pub fn new(id: impl Into) -> Self { + Self(id.into()) + } +} + +impl fmt::Display for WorkflowId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +/// Unique identifier for a workflow instance +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct WorkflowInstanceId(Uuid); + +impl WorkflowInstanceId { + pub fn new() -> Self { + Self(Uuid::new_v4()) + } +} + +impl Default for WorkflowInstanceId { + fn default() -> Self { + Self::new() + } +} + +impl fmt::Display for WorkflowInstanceId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +/// Unique identifier for a workflow step +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct StepId(String); + +impl StepId { + pub fn new(id: impl Into) -> Self { + Self(id.into()) + } +} + +impl fmt::Display for StepId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +/// Retry policy configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RetryPolicy { + pub max_attempts: u32, + pub backoff: BackoffStrategy, +} + +impl Default for RetryPolicy { + fn default() -> Self { + Self { + max_attempts: 3, + backoff: BackoffStrategy::Exponential { + base: Duration::from_secs(1), + max: Duration::from_secs(30), + }, + } + } +} + +/// Backoff strategy for retries +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum BackoffStrategy { + /// Fixed delay between retries + Fixed(Duration), + /// Exponential backoff with base and max duration + Exponential { base: Duration, max: Duration }, + /// Linear backoff with increment and max duration + Linear { increment: Duration, max: Duration }, +} + +/// Action to take when a step fails +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum FailureAction { + /// Stop the entire workflow + FailWorkflow, + /// Skip this step and continue to the next + ContinueNextStep, + /// Keep retrying indefinitely until manual intervention + RetryIndefinitely, +} + +/// Workflow execution status +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum WorkflowStatus { + Pending, + Running, + Paused, + Completed, + Failed, + Cancelled, +} + +/// Step execution status +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum StepStatus { + Pending, + Running, + Succeeded, + Failed, + Retrying, + Skipped, +} + +/// State of a workflow step +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StepState { + pub status: StepStatus, + pub attempt: u32, + pub last_error: Option, + pub started_at: Option>, + pub completed_at: Option>, +} + +impl Default for StepState { + fn default() -> Self { + Self { + status: StepStatus::Pending, + attempt: 0, + last_error: None, + started_at: None, + completed_at: None, + } + } +} + +/// Workflow instance state +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WorkflowState { + pub instance_id: WorkflowInstanceId, + pub definition_id: WorkflowId, + pub status: WorkflowStatus, + pub current_step: Option, + pub step_states: HashMap, + pub context: WorkflowContext, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +impl WorkflowState { + pub fn new(instance_id: WorkflowInstanceId, definition_id: WorkflowId) -> Self { + let now = Utc::now(); + Self { + instance_id, + definition_id, + status: WorkflowStatus::Pending, + current_step: None, + step_states: HashMap::new(), + context: WorkflowContext::new(instance_id), + created_at: now, + updated_at: now, + } + } +} + +/// Shared context passed between workflow steps +/// +/// # Serialization Warning +/// +/// The `data` field contains type-erased values that cannot be serialized. +/// This means workflow context is **not preserved** across: +/// - Process restarts +/// - State persistence to disk +/// - Network serialization +/// +/// The workflow engine only supports **in-memory execution**. If you need +/// durable workflows, consider implementing a custom serializable context type. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WorkflowContext { + pub instance_id: WorkflowInstanceId, + #[serde(skip)] + data: HashMap>, +} + +impl WorkflowContext { + pub fn new(instance_id: WorkflowInstanceId) -> Self { + Self { + instance_id, + data: HashMap::new(), + } + } + + /// Store a value in the context (will be wrapped in Arc) + pub fn set(&mut self, key: impl Into, value: T) { + self.data.insert(key.into(), Arc::new(value)); + } + + /// Store an Arc directly without double-wrapping + pub fn set_arc(&mut self, key: impl Into, value: Arc) { + self.data.insert(key.into(), value); + } + + /// Retrieve a value from the context + pub fn get(&self, key: &str) -> Option> { + self.data + .get(key) + .and_then(|v| v.clone().downcast::().ok()) + } + + /// Check if the context has any data that would be lost during serialization + pub fn has_unserializable_data(&self) -> bool { + !self.data.is_empty() + } + + /// Get the number of context entries (useful for debugging) + pub fn data_len(&self) -> usize { + self.data.len() + } +} + +/// Result returned by a step execution +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StepResult { + Success, + Failure, + Skip, +} + +/// Error kinds for workflow operations +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum WorkflowError { + #[error("Workflow not found: {0}")] + NotFound(WorkflowInstanceId), + + #[error("Workflow definition not found: {0}")] + DefinitionNotFound(WorkflowId), + + #[error("Step failed: {step_id} - {message}")] + StepFailed { step_id: StepId, message: String }, + + #[error("Step timeout: {step_id}")] + StepTimeout { step_id: StepId }, + + #[error("Workflow cancelled: {0}")] + Cancelled(WorkflowInstanceId), + + #[error("Invalid state transition: {from:?} -> {to:?}")] + InvalidStateTransition { + from: WorkflowStatus, + to: WorkflowStatus, + }, + + #[error("Context value not found: {0}")] + ContextValueNotFound(String), + + #[error("Context value type mismatch: {0}")] + ContextTypeMismatch(String), +} + +pub type WorkflowResult = Result; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/common.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/common.rs new file mode 100644 index 00000000..79b7b5e4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/common.rs @@ -0,0 +1,78 @@ +use std::collections::HashMap; + +use serde_json::Value; + +pub fn parse_tool_calls(raw: Option) -> Result, String> { + match raw { + Some(s) if !s.is_empty() => serde_json::from_str(&s).map_err(|e| e.to_string()), + _ => Ok(Vec::new()), + } +} + +pub fn parse_metadata(raw: Option) -> Result, String> { + match raw { + Some(s) if !s.is_empty() => serde_json::from_str(&s).map_err(|e| e.to_string()), + _ => Ok(HashMap::new()), + } +} + +pub fn parse_raw_response(raw: Option) -> Result { + match raw { + Some(s) if !s.is_empty() => serde_json::from_str(&s).map_err(|e| e.to_string()), + _ => Ok(Value::Null), + } +} + +pub fn parse_json_value(raw: Option) -> Result { + match raw { + Some(s) if !s.is_empty() => serde_json::from_str(&s).map_err(|e| e.to_string()), + _ => Ok(Value::Array(vec![])), + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn parse_tool_calls_handles_empty_input() { + assert!(parse_tool_calls(None).unwrap().is_empty()); + assert!(parse_tool_calls(Some(String::new())).unwrap().is_empty()); + } + + #[test] + fn parse_tool_calls_round_trips() { + let payload = json!([{ "type": "test", "value": 1 }]).to_string(); + let parsed = parse_tool_calls(Some(payload)).unwrap(); + assert_eq!(parsed.len(), 1); + assert_eq!(parsed[0]["type"], "test"); + assert_eq!(parsed[0]["value"], 1); + } + + #[test] + fn parse_metadata_defaults_to_empty_map() { + assert!(parse_metadata(None).unwrap().is_empty()); + } + + #[test] + fn parse_metadata_round_trips() { + let payload = json!({"key": "value", "nested": {"bool": true}}).to_string(); + let parsed = parse_metadata(Some(payload)).unwrap(); + assert_eq!(parsed.get("key").unwrap(), "value"); + assert_eq!(parsed["nested"]["bool"], true); + } + + #[test] + fn parse_raw_response_handles_null() { + assert_eq!(parse_raw_response(None).unwrap(), Value::Null); + } + + #[test] + fn parse_raw_response_round_trips() { + let payload = json!({"id": "abc"}).to_string(); + let parsed = parse_raw_response(Some(payload)).unwrap(); + assert_eq!(parsed["id"], "abc"); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/core.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/core.rs new file mode 100644 index 00000000..e61b4163 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/core.rs @@ -0,0 +1,483 @@ +// core.rs +// +// Core types for the data connector module. +// Contains all traits, data types, error types, and IDs for all storage backends. +// +// Structure: +// 1. Conversation types + trait +// 2. ConversationItem types + trait +// 3. Response types + trait + +use std::{ + collections::HashMap, + fmt::{Display, Formatter}, +}; + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use rand::RngCore; +use serde::{Deserialize, Serialize}; +use serde_json::{Map as JsonMap, Value}; + +// ============================================================================ +// PART 1: Conversation Storage +// ============================================================================ + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)] +pub struct ConversationId(pub String); + +impl ConversationId { + pub fn new() -> Self { + let mut rng = rand::rng(); + let mut bytes = [0u8; 25]; + rng.fill_bytes(&mut bytes); + let hex_string: String = bytes.iter().map(|b| format!("{:02x}", b)).collect(); + Self(format!("conv_{}", hex_string)) + } +} + +impl Default for ConversationId { + fn default() -> Self { + Self::new() + } +} + +impl From for ConversationId { + fn from(value: String) -> Self { + Self(value) + } +} + +impl From<&str> for ConversationId { + fn from(value: &str) -> Self { + Self(value.to_string()) + } +} + +impl Display for ConversationId { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +/// Metadata payload persisted with a conversation +pub type ConversationMetadata = JsonMap; + +/// Input payload for creating a conversation +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct NewConversation { + /// Optional conversation ID (if None, a random ID will be generated) + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metadata: Option, +} + +/// Stored conversation data structure +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct Conversation { + pub id: ConversationId, + pub created_at: DateTime, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metadata: Option, +} + +impl Conversation { + pub fn new(new_conversation: NewConversation) -> Self { + Self { + id: new_conversation.id.unwrap_or_default(), + created_at: Utc::now(), + metadata: new_conversation.metadata, + } + } + + pub fn with_parts( + id: ConversationId, + created_at: DateTime, + metadata: Option, + ) -> Self { + Self { + id, + created_at, + metadata, + } + } +} + +/// Result alias for conversation storage operations +pub type ConversationResult = Result; + +/// Error type for conversation storage operations +#[derive(Debug, thiserror::Error)] +pub enum ConversationStorageError { + #[error("Conversation not found: {0}")] + ConversationNotFound(String), + + #[error("Storage error: {0}")] + StorageError(String), + + #[error("Serialization error: {0}")] + SerializationError(#[from] serde_json::Error), +} + +/// Trait describing the CRUD interface for conversation storage backends +#[async_trait] +pub trait ConversationStorage: Send + Sync + 'static { + async fn create_conversation(&self, input: NewConversation) + -> ConversationResult; + + async fn get_conversation( + &self, + id: &ConversationId, + ) -> ConversationResult>; + + async fn update_conversation( + &self, + id: &ConversationId, + metadata: Option, + ) -> ConversationResult>; + + async fn delete_conversation(&self, id: &ConversationId) -> ConversationResult; +} + +// ============================================================================ +// PART 2: ConversationItem Storage +// ============================================================================ + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)] +pub struct ConversationItemId(pub String); + +impl Display for ConversationItemId { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl From for ConversationItemId { + fn from(value: String) -> Self { + Self(value) + } +} + +impl From<&str> for ConversationItemId { + fn from(value: &str) -> Self { + Self(value.to_string()) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ConversationItem { + pub id: ConversationItemId, + pub response_id: Option, + pub item_type: String, + pub role: Option, + pub content: Value, + pub status: Option, + pub created_at: DateTime, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NewConversationItem { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + pub response_id: Option, + pub item_type: String, + pub role: Option, + pub content: Value, + pub status: Option, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum SortOrder { + Asc, + Desc, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ListParams { + pub limit: usize, + pub order: SortOrder, + pub after: Option, // item_id cursor +} + +pub type ConversationItemResult = Result; + +#[derive(Debug, thiserror::Error)] +pub enum ConversationItemStorageError { + #[error("Not found: {0}")] + NotFound(String), + + #[error("Storage error: {0}")] + StorageError(String), + + #[error("Serialization error: {0}")] + SerializationError(#[from] serde_json::Error), +} + +#[async_trait] +pub trait ConversationItemStorage: Send + Sync + 'static { + async fn create_item( + &self, + item: NewConversationItem, + ) -> ConversationItemResult; + + async fn link_item( + &self, + conversation_id: &ConversationId, + item_id: &ConversationItemId, + added_at: DateTime, + ) -> ConversationItemResult<()>; + + async fn list_items( + &self, + conversation_id: &ConversationId, + params: ListParams, + ) -> ConversationItemResult>; + + /// Get a single item by ID + async fn get_item( + &self, + item_id: &ConversationItemId, + ) -> ConversationItemResult>; + + /// Check if an item is linked to a conversation + async fn is_item_linked( + &self, + conversation_id: &ConversationId, + item_id: &ConversationItemId, + ) -> ConversationItemResult; + + /// Delete an item link from a conversation (does not delete the item itself) + async fn delete_item( + &self, + conversation_id: &ConversationId, + item_id: &ConversationItemId, + ) -> ConversationItemResult<()>; +} + +/// Helper to build id prefix based on item_type +pub fn make_item_id(item_type: &str) -> ConversationItemId { + // Generate exactly 50 hex characters (25 bytes) for the part after the underscore + let mut rng = rand::rng(); + let mut bytes = [0u8; 25]; + rng.fill_bytes(&mut bytes); + let hex_string: String = bytes.iter().map(|b| format!("{:02x}", b)).collect(); + + let prefix: String = match item_type { + "message" => "msg".to_string(), + "reasoning" => "rs".to_string(), + "mcp_call" => "mcp".to_string(), + "mcp_list_tools" => "mcpl".to_string(), + "function_call" => "fc".to_string(), + other => { + // Fallback: first 3 letters of type or "itm" + let mut p = other.chars().take(3).collect::(); + if p.is_empty() { + p = "itm".to_string(); + } + p + } + }; + ConversationItemId(format!("{}_{}", prefix, hex_string)) +} + +// ============================================================================ +// PART 3: Response Storage +// ============================================================================ + +/// Response identifier +#[derive(Debug, Clone, Hash, Eq, PartialEq, Serialize, Deserialize)] +pub struct ResponseId(pub String); + +impl ResponseId { + pub fn new() -> Self { + Self(ulid::Ulid::new().to_string()) + } +} + +impl Default for ResponseId { + fn default() -> Self { + Self::new() + } +} + +impl From for ResponseId { + fn from(value: String) -> Self { + Self(value) + } +} + +impl From<&str> for ResponseId { + fn from(value: &str) -> Self { + Self(value.to_string()) + } +} + +/// Stored response data +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StoredResponse { + /// Unique response ID + pub id: ResponseId, + + /// ID of the previous response in the chain (if any) + pub previous_response_id: Option, + + /// Input items as JSON array + pub input: Value, + + /// System instructions used + pub instructions: Option, + + /// Output items as JSON array + pub output: Value, + + /// Tool calls made by the model (if any) + pub tool_calls: Vec, + + /// Custom metadata + pub metadata: HashMap, + + /// When this response was created + pub created_at: DateTime, + + /// Safety identifier for content moderation + pub safety_identifier: Option, + + /// Model used for generation + pub model: Option, + + /// Conversation id if associated with a conversation + #[serde(default)] + pub conversation_id: Option, + + /// Raw OpenAI response payload + #[serde(default)] + pub raw_response: Value, +} + +impl StoredResponse { + pub fn new(previous_response_id: Option) -> Self { + Self { + id: ResponseId::new(), + previous_response_id, + input: Value::Array(vec![]), + instructions: None, + output: Value::Array(vec![]), + tool_calls: Vec::new(), + metadata: HashMap::new(), + created_at: Utc::now(), + safety_identifier: None, + model: None, + conversation_id: None, + raw_response: Value::Null, + } + } +} + +/// Response chain - a sequence of related responses +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ResponseChain { + /// The responses in chronological order + pub responses: Vec, + + /// Metadata about the chain + pub metadata: HashMap, +} + +impl Default for ResponseChain { + fn default() -> Self { + Self::new() + } +} + +impl ResponseChain { + pub fn new() -> Self { + Self { + responses: Vec::new(), + metadata: HashMap::new(), + } + } + + /// Get the ID of the most recent response in the chain + pub fn latest_response_id(&self) -> Option<&ResponseId> { + self.responses.last().map(|r| &r.id) + } + + /// Add a response to the chain + pub fn add_response(&mut self, response: StoredResponse) { + self.responses.push(response); + } + + /// Build context from the chain for the next request + pub fn build_context(&self, max_responses: Option) -> Vec<(Value, Value)> { + let responses = if let Some(max) = max_responses { + let start = self.responses.len().saturating_sub(max); + &self.responses[start..] + } else { + &self.responses[..] + }; + + responses + .iter() + .map(|r| (r.input.clone(), r.output.clone())) + .collect() + } +} + +/// Error type for response storage operations +#[derive(Debug, thiserror::Error)] +pub enum ResponseStorageError { + #[error("Response not found: {0}")] + ResponseNotFound(String), + + #[error("Invalid chain: {0}")] + InvalidChain(String), + + #[error("Storage error: {0}")] + StorageError(String), + + #[error("Serialization error: {0}")] + SerializationError(#[from] serde_json::Error), +} + +pub type ResponseResult = Result; + +/// Trait for response storage +#[async_trait] +pub trait ResponseStorage: Send + Sync { + /// Store a new response + async fn store_response(&self, response: StoredResponse) -> ResponseResult; + + /// Get a response by ID + async fn get_response( + &self, + response_id: &ResponseId, + ) -> ResponseResult>; + + /// Delete a response + async fn delete_response(&self, response_id: &ResponseId) -> ResponseResult<()>; + + /// Get the chain of responses leading to a given response + /// Returns responses in chronological order (oldest first) + async fn get_response_chain( + &self, + response_id: &ResponseId, + max_depth: Option, + ) -> ResponseResult; + + /// List recent responses for a safety identifier + async fn list_identifier_responses( + &self, + identifier: &str, + limit: Option, + ) -> ResponseResult>; + + /// Delete all responses for a safety identifier + async fn delete_identifier_responses(&self, identifier: &str) -> ResponseResult; +} + +impl Default for StoredResponse { + fn default() -> Self { + Self::new(None) + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/factory.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/factory.rs new file mode 100644 index 00000000..5c62934d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/factory.rs @@ -0,0 +1,140 @@ +// factory.rs +// +// Factory function to create storage backends based on configuration. +// This centralizes storage initialization logic and fixes the bug where +// conversation_item_storage was missing/incorrect in server.rs. + +use std::sync::Arc; + +use tracing::info; +use url::Url; + +use super::{ + core::{ConversationItemStorage, ConversationStorage, ResponseStorage}, + memory::{MemoryConversationItemStorage, MemoryConversationStorage, MemoryResponseStorage}, + noop::{NoOpConversationItemStorage, NoOpConversationStorage, NoOpResponseStorage}, + oracle::{OracleConversationItemStorage, OracleConversationStorage, OracleResponseStorage}, +}; +use crate::{ + config::{HistoryBackend, OracleConfig, PostgresConfig, RouterConfig}, + data_connector::postgres::{ + PostgresConversationItemStorage, PostgresConversationStorage, PostgresResponseStorage, + PostgresStore, + }, +}; + +/// Type alias for the storage tuple returned by factory functions. +/// This avoids clippy::type_complexity warnings while keeping Arc explicit. +pub type StorageTuple = ( + Arc, + Arc, + Arc, +); + +/// Create all three storage backends based on router configuration. +/// +/// # Arguments +/// * `config` - Router configuration containing history_backend and oracle settings +/// +/// # Returns +/// Tuple of (response_storage, conversation_storage, conversation_item_storage) +/// +/// # Errors +/// Returns error string if Oracle configuration is missing or initialization fails +pub fn create_storage(config: &RouterConfig) -> Result { + match config.history_backend { + HistoryBackend::Memory => { + info!("Initializing data connector: Memory"); + Ok(( + Arc::new(MemoryResponseStorage::new()), + Arc::new(MemoryConversationStorage::new()), + Arc::new(MemoryConversationItemStorage::new()), + )) + } + HistoryBackend::None => { + info!("Initializing data connector: None (no persistence)"); + Ok(( + Arc::new(NoOpResponseStorage::new()), + Arc::new(NoOpConversationStorage::new()), + Arc::new(NoOpConversationItemStorage::new()), + )) + } + HistoryBackend::Oracle => { + let oracle_cfg = config + .oracle + .clone() + .ok_or("oracle configuration is required when history_backend=oracle")?; + + info!( + "Initializing data connector: Oracle ATP (pool: {}-{})", + oracle_cfg.pool_min, oracle_cfg.pool_max + ); + + let storages = create_oracle_storage(&oracle_cfg)?; + + info!("Data connector initialized successfully: Oracle ATP"); + Ok(storages) + } + HistoryBackend::Postgres => { + let postgres_cfg = config + .postgres + .clone() + .ok_or("Postgres configuration is required when history_backend=postgres")?; + + let log_db_url = match Url::parse(&postgres_cfg.db_url) { + Ok(mut url) => { + if url.password().is_some() { + let _ = url.set_password(Some("****")); + } + url.to_string() + } + Err(_) => "".to_string(), + }; + + info!( + "Initializing data connector: Postgres (db_url: {}, pool_max: {})", + log_db_url, postgres_cfg.pool_max + ); + + let storages = create_postgres_storage(&postgres_cfg)?; + + info!("Data connector initialized successfully: Postgres"); + + Ok(storages) + } + } +} + +/// Create Oracle storage backends +fn create_oracle_storage(oracle_cfg: &OracleConfig) -> Result { + let response_storage = OracleResponseStorage::new(oracle_cfg.clone()) + .map_err(|err| format!("failed to initialize Oracle response storage: {err}"))?; + + let conversation_storage = OracleConversationStorage::new(oracle_cfg.clone()) + .map_err(|err| format!("failed to initialize Oracle conversation storage: {err}"))?; + + let conversation_item_storage = OracleConversationItemStorage::new(oracle_cfg.clone()) + .map_err(|err| format!("failed to initialize Oracle conversation item storage: {err}"))?; + + Ok(( + Arc::new(response_storage), + Arc::new(conversation_storage), + Arc::new(conversation_item_storage), + )) +} + +fn create_postgres_storage(postgres_cfg: &PostgresConfig) -> Result { + let store = PostgresStore::new(postgres_cfg.clone())?; + let postgres_resp = PostgresResponseStorage::new(store.clone()) + .map_err(|err| format!("failed to initialize Postgres response storage: {err}"))?; + let postgres_conv = PostgresConversationStorage::new(store.clone()) + .map_err(|err| format!("failed to initialize Postgres conversation storage: {err}"))?; + let postgres_item = PostgresConversationItemStorage::new(store.clone()) + .map_err(|err| format!("failed to initialize Postgres conversation item storage: {err}"))?; + + Ok(( + Arc::new(postgres_resp), + Arc::new(postgres_conv), + Arc::new(postgres_item), + )) +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/memory.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/memory.rs new file mode 100644 index 00000000..74df6d98 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/memory.rs @@ -0,0 +1,730 @@ +//! In-memory storage implementations +//! +//! Used for development and testing - no persistence. +//! +//! Structure: +//! 1. MemoryConversationStorage +//! 2. MemoryConversationItemStorage +//! 3. MemoryResponseStorage + +use std::{ + collections::{BTreeMap, HashMap}, + sync::{Arc, RwLock}, +}; + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use parking_lot::RwLock as ParkingLotRwLock; + +use super::core::*; + +// ============================================================================ +// PART 1: MemoryConversationStorage +// ============================================================================ + +/// In-memory conversation storage used for development and tests +#[derive(Default, Clone)] +pub struct MemoryConversationStorage { + inner: Arc>>, +} + +impl MemoryConversationStorage { + pub fn new() -> Self { + Self { + inner: Arc::new(ParkingLotRwLock::new(HashMap::new())), + } + } +} + +#[async_trait] +impl ConversationStorage for MemoryConversationStorage { + async fn create_conversation( + &self, + input: NewConversation, + ) -> ConversationResult { + let conversation = Conversation::new(input); + self.inner + .write() + .insert(conversation.id.clone(), conversation.clone()); + Ok(conversation) + } + + async fn get_conversation( + &self, + id: &ConversationId, + ) -> ConversationResult> { + Ok(self.inner.read().get(id).cloned()) + } + + async fn update_conversation( + &self, + id: &ConversationId, + metadata: Option, + ) -> ConversationResult> { + let mut store = self.inner.write(); + if let Some(entry) = store.get_mut(id) { + entry.metadata = metadata; + return Ok(Some(entry.clone())); + } + + Ok(None) + } + + async fn delete_conversation(&self, id: &ConversationId) -> ConversationResult { + let removed = self.inner.write().remove(id).is_some(); + Ok(removed) + } +} + +// ============================================================================ +// PART 2: MemoryConversationItemStorage +// ============================================================================ + +#[derive(Default)] +pub struct MemoryConversationItemStorage { + items: RwLock>, // item_id -> item + #[allow(clippy::type_complexity)] + links: RwLock>>, + // Per-conversation reverse index for fast after cursor lookup: item_id_str -> (ts, item_id_str) + #[allow(clippy::type_complexity)] + rev_index: RwLock>>, +} + +impl MemoryConversationItemStorage { + pub fn new() -> Self { + Self::default() + } +} + +#[async_trait] +impl ConversationItemStorage for MemoryConversationItemStorage { + async fn create_item( + &self, + new_item: NewConversationItem, + ) -> ConversationItemResult { + let id = new_item + .id + .clone() + .unwrap_or_else(|| make_item_id(&new_item.item_type)); + let created_at = Utc::now(); + let item = ConversationItem { + id: id.clone(), + response_id: new_item.response_id, + item_type: new_item.item_type, + role: new_item.role, + content: new_item.content, + status: new_item.status, + created_at, + }; + let mut items = self.items.write().unwrap(); + items.insert(id.clone(), item.clone()); + Ok(item) + } + + async fn link_item( + &self, + conversation_id: &ConversationId, + item_id: &ConversationItemId, + added_at: DateTime, + ) -> ConversationItemResult<()> { + { + let mut links = self.links.write().unwrap(); + let entry = links.entry(conversation_id.clone()).or_default(); + entry.insert((added_at.timestamp(), item_id.0.clone()), item_id.clone()); + } + { + let mut rev = self.rev_index.write().unwrap(); + let entry = rev.entry(conversation_id.clone()).or_default(); + entry.insert(item_id.0.clone(), (added_at.timestamp(), item_id.0.clone())); + } + Ok(()) + } + + async fn list_items( + &self, + conversation_id: &ConversationId, + params: ListParams, + ) -> ConversationItemResult> { + let links_guard = self.links.read().unwrap(); + let map = match links_guard.get(conversation_id) { + Some(m) => m, + None => return Ok(Vec::new()), + }; + + let mut results: Vec = Vec::new(); + let after_key: Option<(i64, String)> = if let Some(after_id) = ¶ms.after { + // O(1) lookup via reverse index for this conversation + if let Some(conv_idx) = self.rev_index.read().unwrap().get(conversation_id) { + conv_idx.get(after_id).cloned() + } else { + None + } + } else { + None + }; + + let take = params.limit; + let items_guard = self.items.read().unwrap(); + + use std::ops::Bound::{Excluded, Unbounded}; + + // Helper to push item if it exists and stop when reaching the limit + let mut push_item = |key: &ConversationItemId| -> bool { + if let Some(it) = items_guard.get(key) { + results.push(it.clone()); + if results.len() == take { + return true; + } + } + false + }; + + match (params.order, after_key) { + (SortOrder::Desc, Some(k)) => { + for ((_ts, _id), item_key) in map.range(..k).rev() { + if push_item(item_key) { + break; + } + } + } + (SortOrder::Desc, None) => { + for ((_ts, _id), item_key) in map.iter().rev() { + if push_item(item_key) { + break; + } + } + } + (SortOrder::Asc, Some(k)) => { + for ((_ts, _id), item_key) in map.range((Excluded(k), Unbounded)) { + if push_item(item_key) { + break; + } + } + } + (SortOrder::Asc, None) => { + for ((_ts, _id), item_key) in map.iter() { + if push_item(item_key) { + break; + } + } + } + } + + Ok(results) + } + + async fn get_item( + &self, + item_id: &ConversationItemId, + ) -> ConversationItemResult> { + let items = self.items.read().unwrap(); + Ok(items.get(item_id).cloned()) + } + + async fn is_item_linked( + &self, + conversation_id: &ConversationId, + item_id: &ConversationItemId, + ) -> ConversationItemResult { + let rev = self.rev_index.read().unwrap(); + if let Some(conv_idx) = rev.get(conversation_id) { + Ok(conv_idx.contains_key(&item_id.0)) + } else { + Ok(false) + } + } + + async fn delete_item( + &self, + conversation_id: &ConversationId, + item_id: &ConversationItemId, + ) -> ConversationItemResult<()> { + // Get the key from rev_index and remove the entry at the same time + let key_to_remove = { + let mut rev = self.rev_index.write().unwrap(); + if let Some(conv_idx) = rev.get_mut(conversation_id) { + conv_idx.remove(&item_id.0) + } else { + None + } + }; + + // If the item was in rev_index, remove it from links as well + if let Some(key) = key_to_remove { + let mut links = self.links.write().unwrap(); + if let Some(conv_links) = links.get_mut(conversation_id) { + conv_links.remove(&key); + } + } + + Ok(()) + } +} + +// ============================================================================ +// PART 3: MemoryResponseStorage +// ============================================================================ + +/// Internal store structure holding both maps together +#[derive(Default)] +struct InnerStore { + /// All stored responses indexed by ID + responses: HashMap, + /// Index of response IDs by safety identifier + identifier_index: HashMap>, +} + +/// In-memory implementation of response storage +pub struct MemoryResponseStorage { + /// Single lock wrapping both maps to prevent deadlocks and ensure atomic updates + store: Arc>, +} + +impl MemoryResponseStorage { + pub fn new() -> Self { + Self { + store: Arc::new(ParkingLotRwLock::new(InnerStore::default())), + } + } + + /// Get statistics about the store + pub fn stats(&self) -> MemoryStoreStats { + let store = self.store.read(); + MemoryStoreStats { + response_count: store.responses.len(), + identifier_count: store.identifier_index.len(), + } + } + + /// Clear all data (useful for testing) + pub fn clear(&self) { + let mut store = self.store.write(); + store.responses.clear(); + store.identifier_index.clear(); + } +} + +impl Default for MemoryResponseStorage { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl ResponseStorage for MemoryResponseStorage { + async fn store_response(&self, mut response: StoredResponse) -> ResponseResult { + // Generate ID if not set + if response.id.0.is_empty() { + response.id = ResponseId::new(); + } + + let response_id = response.id.clone(); + + // Single lock acquisition for atomic update + let mut store = self.store.write(); + + // Update safety identifier index if specified + if let Some(ref safety_identifier) = response.safety_identifier { + store + .identifier_index + .entry(safety_identifier.clone()) + .or_default() + .push(response_id.clone()); + } + + // Store the response + store.responses.insert(response_id.clone(), response); + tracing::info!("memory_store_size" = store.responses.len()); + + Ok(response_id) + } + + async fn get_response( + &self, + response_id: &ResponseId, + ) -> ResponseResult> { + let store = self.store.read(); + let result = store.responses.get(response_id).cloned(); + tracing::info!("memory_get_response" = %response_id.0, found = result.is_some()); + Ok(result) + } + + async fn delete_response(&self, response_id: &ResponseId) -> ResponseResult<()> { + let mut store = self.store.write(); + + // Remove the response and update user index if needed + if let Some(response) = store.responses.remove(response_id) { + if let Some(ref safety_identifier) = response.safety_identifier { + if let Some(user_responses) = store.identifier_index.get_mut(safety_identifier) { + user_responses.retain(|id| id != response_id); + } + } + } + + Ok(()) + } + + async fn get_response_chain( + &self, + response_id: &ResponseId, + max_depth: Option, + ) -> ResponseResult { + let mut chain = ResponseChain::new(); + let max_depth = max_depth.unwrap_or(100); // Default max depth to prevent infinite loops + + // Collect all response IDs first + let mut response_ids = Vec::new(); + let mut current_id = Some(response_id.clone()); + let mut depth = 0; + + // Single lock acquisition to collect the chain + { + let store = self.store.read(); + while let Some(id) = current_id { + if depth >= max_depth { + break; + } + + if let Some(response) = store.responses.get(&id) { + response_ids.push(id); + current_id = response.previous_response_id.clone(); + depth += 1; + } else { + break; + } + } + } + + // Reverse to get chronological order (oldest first) + response_ids.reverse(); + + // Now collect the actual responses + let store = self.store.read(); + for id in response_ids { + if let Some(response) = store.responses.get(&id) { + chain.add_response(response.clone()); + } + } + + Ok(chain) + } + + async fn list_identifier_responses( + &self, + identifier: &str, + limit: Option, + ) -> ResponseResult> { + let store = self.store.read(); + + if let Some(user_response_ids) = store.identifier_index.get(identifier) { + // Collect responses with their timestamps for sorting + let mut responses_with_time: Vec<_> = user_response_ids + .iter() + .filter_map(|id| store.responses.get(id).map(|r| (r.created_at, id))) + .collect(); + + // Sort by creation time (newest first) + responses_with_time.sort_by(|a, b| b.0.cmp(&a.0)); + + // Apply limit and collect the actual responses + let limit = limit.unwrap_or(responses_with_time.len()); + let user_responses: Vec = responses_with_time + .into_iter() + .take(limit) + .filter_map(|(_, id)| store.responses.get(id).cloned()) + .collect(); + + Ok(user_responses) + } else { + Ok(Vec::new()) + } + } + + async fn delete_identifier_responses(&self, identifier: &str) -> ResponseResult { + let mut store = self.store.write(); + + if let Some(user_response_ids) = store.identifier_index.remove(identifier) { + let count = user_response_ids.len(); + for id in user_response_ids { + store.responses.remove(&id); + } + Ok(count) + } else { + Ok(0) + } + } +} + +/// Statistics for the memory store +#[derive(Debug, Clone)] +pub struct MemoryStoreStats { + pub response_count: usize, + pub identifier_count: usize, +} + +#[cfg(test)] +mod tests { + use chrono::{TimeZone, Utc}; + use serde_json::json; + + use super::*; + + // ======================================================================== + // ConversationItem Tests + // ======================================================================== + + fn make_item( + item_type: &str, + role: Option<&str>, + content: serde_json::Value, + ) -> NewConversationItem { + NewConversationItem { + id: None, + response_id: None, + item_type: item_type.to_string(), + role: role.map(|r| r.to_string()), + content, + status: Some("completed".to_string()), + } + } + + #[tokio::test] + async fn test_list_ordering_and_cursors() { + let store = MemoryConversationItemStorage::new(); + let conv: ConversationId = "conv_test".into(); + + // Create 3 items and link them at controlled timestamps + let i1 = store + .create_item(make_item("message", Some("user"), json!([]))) + .await + .unwrap(); + let i2 = store + .create_item(make_item("message", Some("assistant"), json!([]))) + .await + .unwrap(); + let i3 = store + .create_item(make_item("reasoning", None, json!([]))) + .await + .unwrap(); + + let t1 = Utc.timestamp_opt(1_700_000_001, 0).single().unwrap(); + let t2 = Utc.timestamp_opt(1_700_000_002, 0).single().unwrap(); + let t3 = Utc.timestamp_opt(1_700_000_003, 0).single().unwrap(); + + store.link_item(&conv, &i1.id, t1).await.unwrap(); + store.link_item(&conv, &i2.id, t2).await.unwrap(); + store.link_item(&conv, &i3.id, t3).await.unwrap(); + + // Desc order, no cursor + let desc = store + .list_items( + &conv, + ListParams { + limit: 2, + order: SortOrder::Desc, + after: None, + }, + ) + .await + .unwrap(); + assert!(desc.len() >= 2); + assert_eq!(desc[0].id, i3.id); + assert_eq!(desc[1].id, i2.id); + + // Desc with cursor = i2 -> expect i1 next + let desc_after = store + .list_items( + &conv, + ListParams { + limit: 2, + order: SortOrder::Desc, + after: Some(i2.id.0.clone()), + }, + ) + .await + .unwrap(); + assert!(!desc_after.is_empty()); + assert_eq!(desc_after[0].id, i1.id); + + // Asc order, no cursor + let asc = store + .list_items( + &conv, + ListParams { + limit: 2, + order: SortOrder::Asc, + after: None, + }, + ) + .await + .unwrap(); + assert!(asc.len() >= 2); + assert_eq!(asc[0].id, i1.id); + assert_eq!(asc[1].id, i2.id); + + // Asc with cursor = i2 -> expect i3 next + let asc_after = store + .list_items( + &conv, + ListParams { + limit: 2, + order: SortOrder::Asc, + after: Some(i2.id.0.clone()), + }, + ) + .await + .unwrap(); + assert!(!asc_after.is_empty()); + assert_eq!(asc_after[0].id, i3.id); + } + + // ======================================================================== + // Response Tests + // ======================================================================== + + #[tokio::test] + async fn test_store_with_custom_id() { + let store = MemoryResponseStorage::new(); + let mut response = StoredResponse::new(None); + response.id = ResponseId::from("resp_custom"); + response.input = json!("Input"); + response.output = json!("Output"); + store.store_response(response.clone()).await.unwrap(); + let retrieved = store + .get_response(&ResponseId::from("resp_custom")) + .await + .unwrap(); + assert!(retrieved.is_some()); + assert_eq!(retrieved.unwrap().output, json!("Output")); + } + + #[tokio::test] + async fn test_memory_store_basic() { + let store = MemoryResponseStorage::new(); + + // Store a response + let mut response = StoredResponse::new(None); + response.input = json!("Hello"); + response.output = json!("Hi there!"); + let response_id = store.store_response(response).await.unwrap(); + + // Retrieve it + let retrieved = store.get_response(&response_id).await.unwrap(); + assert!(retrieved.is_some()); + assert_eq!(retrieved.unwrap().input, json!("Hello")); + + // Delete it + store.delete_response(&response_id).await.unwrap(); + let deleted = store.get_response(&response_id).await.unwrap(); + assert!(deleted.is_none()); + } + + #[tokio::test] + async fn test_response_chain() { + let store = MemoryResponseStorage::new(); + + // Create a chain of responses + let mut response1 = StoredResponse::new(None); + response1.input = json!("First"); + response1.output = json!("First response"); + let id1 = store.store_response(response1).await.unwrap(); + + let mut response2 = StoredResponse::new(Some(id1.clone())); + response2.input = json!("Second"); + response2.output = json!("Second response"); + let id2 = store.store_response(response2).await.unwrap(); + + let mut response3 = StoredResponse::new(Some(id2.clone())); + response3.input = json!("Third"); + response3.output = json!("Third response"); + let id3 = store.store_response(response3).await.unwrap(); + + // Get the chain + let chain = store.get_response_chain(&id3, None).await.unwrap(); + assert_eq!(chain.responses.len(), 3); + assert_eq!(chain.responses[0].input, json!("First")); + assert_eq!(chain.responses[1].input, json!("Second")); + assert_eq!(chain.responses[2].input, json!("Third")); + + let limited_chain = store.get_response_chain(&id3, Some(2)).await.unwrap(); + assert_eq!(limited_chain.responses.len(), 2); + assert_eq!(limited_chain.responses[0].input, json!("Second")); + assert_eq!(limited_chain.responses[1].input, json!("Third")); + } + + #[tokio::test] + async fn test_user_responses() { + let store = MemoryResponseStorage::new(); + + // Store responses for different users + let mut response1 = StoredResponse::new(None); + response1.input = json!("User1 message"); + response1.output = json!("Response to user1"); + response1.safety_identifier = Some("user1".to_string()); + store.store_response(response1).await.unwrap(); + + let mut response2 = StoredResponse::new(None); + response2.input = json!("Another user1 message"); + response2.output = json!("Another response to user1"); + response2.safety_identifier = Some("user1".to_string()); + store.store_response(response2).await.unwrap(); + + let mut response3 = StoredResponse::new(None); + response3.input = json!("User2 message"); + response3.output = json!("Response to user2"); + response3.safety_identifier = Some("user2".to_string()); + store.store_response(response3).await.unwrap(); + + // List user1's responses + let user1_responses = store + .list_identifier_responses("user1", None) + .await + .unwrap(); + assert_eq!(user1_responses.len(), 2); + + // List user2's responses + let user2_responses = store + .list_identifier_responses("user2", None) + .await + .unwrap(); + assert_eq!(user2_responses.len(), 1); + + // Delete user1's responses + let deleted_count = store.delete_identifier_responses("user1").await.unwrap(); + assert_eq!(deleted_count, 2); + + let user1_responses_after = store + .list_identifier_responses("user1", None) + .await + .unwrap(); + assert_eq!(user1_responses_after.len(), 0); + + // User2's responses should still be there + let user2_responses_after = store + .list_identifier_responses("user2", None) + .await + .unwrap(); + assert_eq!(user2_responses_after.len(), 1); + } + + #[tokio::test] + async fn test_memory_store_stats() { + let store = MemoryResponseStorage::new(); + + let mut response1 = StoredResponse::new(None); + response1.input = json!("Test1"); + response1.output = json!("Reply1"); + response1.safety_identifier = Some("user1".to_string()); + store.store_response(response1).await.unwrap(); + + let mut response2 = StoredResponse::new(None); + response2.input = json!("Test2"); + response2.output = json!("Reply2"); + response2.safety_identifier = Some("user2".to_string()); + store.store_response(response2).await.unwrap(); + + let stats = store.stats(); + assert_eq!(stats.response_count, 2); + assert_eq!(stats.identifier_count, 2); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/mod.rs new file mode 100644 index 00000000..dc1e2dec --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/mod.rs @@ -0,0 +1,25 @@ +// Data connector module for response storage and conversation storage +// +// Simplified module structure: +// - core.rs: All traits, data types, and errors +// - memory.rs: All in-memory storage implementations +// - noop.rs: All no-op storage implementations +// - oracle.rs: All Oracle ATP storage implementations +// - factory.rs: Storage creation function + +mod common; +mod core; +mod factory; +mod memory; +mod noop; +mod oracle; +mod postgres; + +pub use core::{ + Conversation, ConversationId, ConversationItem, ConversationItemId, ConversationItemStorage, + ConversationStorage, ListParams, NewConversation, NewConversationItem, ResponseId, + ResponseStorage, SortOrder, StoredResponse, +}; + +pub use factory::create_storage; +pub use memory::{MemoryConversationItemStorage, MemoryConversationStorage, MemoryResponseStorage}; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/noop.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/noop.rs new file mode 100644 index 00000000..a130e156 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/noop.rs @@ -0,0 +1,189 @@ +//! NoOp storage implementations +//! +//! These implementations do nothing - useful for when persistence is disabled. +//! +//! Structure: +//! 1. NoOpConversationStorage +//! 2. NoOpConversationItemStorage +//! 3. NoOpResponseStorage + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; + +use super::core::*; + +// ============================================================================ +// PART 1: NoOpConversationStorage +// ============================================================================ + +/// No-op implementation that synthesizes conversation responses without persistence +#[derive(Default, Debug, Clone)] +pub struct NoOpConversationStorage; + +impl NoOpConversationStorage { + pub fn new() -> Self { + Self + } +} + +#[async_trait] +impl ConversationStorage for NoOpConversationStorage { + async fn create_conversation( + &self, + input: NewConversation, + ) -> ConversationResult { + Ok(Conversation::new(input)) + } + + async fn get_conversation( + &self, + _id: &ConversationId, + ) -> ConversationResult> { + Ok(None) + } + + async fn update_conversation( + &self, + _id: &ConversationId, + _metadata: Option, + ) -> ConversationResult> { + Ok(None) + } + + async fn delete_conversation(&self, _id: &ConversationId) -> ConversationResult { + Ok(false) + } +} + +// ============================================================================ +// PART 2: NoOpConversationItemStorage +// ============================================================================ + +/// No-op conversation item storage (does nothing) +#[derive(Clone, Copy, Default)] +pub struct NoOpConversationItemStorage; + +impl NoOpConversationItemStorage { + pub fn new() -> Self { + Self + } +} + +#[async_trait] +impl ConversationItemStorage for NoOpConversationItemStorage { + async fn create_item( + &self, + item: NewConversationItem, + ) -> ConversationItemResult { + let id = item + .id + .clone() + .unwrap_or_else(|| make_item_id(&item.item_type)); + Ok(ConversationItem { + id, + response_id: item.response_id, + item_type: item.item_type, + role: item.role, + content: item.content, + status: item.status, + created_at: Utc::now(), + }) + } + + async fn link_item( + &self, + _conversation_id: &ConversationId, + _item_id: &ConversationItemId, + _added_at: DateTime, + ) -> ConversationItemResult<()> { + Ok(()) + } + + async fn list_items( + &self, + _conversation_id: &ConversationId, + _params: ListParams, + ) -> ConversationItemResult> { + Ok(Vec::new()) + } + + async fn get_item( + &self, + _item_id: &ConversationItemId, + ) -> ConversationItemResult> { + Ok(None) + } + + async fn is_item_linked( + &self, + _conversation_id: &ConversationId, + _item_id: &ConversationItemId, + ) -> ConversationItemResult { + Ok(false) + } + + async fn delete_item( + &self, + _conversation_id: &ConversationId, + _item_id: &ConversationItemId, + ) -> ConversationItemResult<()> { + Ok(()) + } +} + +// ============================================================================ +// PART 3: NoOpResponseStorage +// ============================================================================ + +/// No-op implementation of response storage (does nothing) +pub struct NoOpResponseStorage; + +impl NoOpResponseStorage { + pub fn new() -> Self { + Self + } +} + +impl Default for NoOpResponseStorage { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl ResponseStorage for NoOpResponseStorage { + async fn store_response(&self, response: StoredResponse) -> ResponseResult { + Ok(response.id) + } + + async fn get_response( + &self, + _response_id: &ResponseId, + ) -> ResponseResult> { + Ok(None) + } + + async fn delete_response(&self, _response_id: &ResponseId) -> ResponseResult<()> { + Ok(()) + } + + async fn get_response_chain( + &self, + _response_id: &ResponseId, + _max_depth: Option, + ) -> ResponseResult { + Ok(ResponseChain::new()) + } + + async fn list_identifier_responses( + &self, + _identifier: &str, + _limit: Option, + ) -> ResponseResult> { + Ok(Vec::new()) + } + + async fn delete_identifier_responses(&self, _identifier: &str) -> ResponseResult { + Ok(0) + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/oracle.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/oracle.rs new file mode 100644 index 00000000..2ab4cb77 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/oracle.rs @@ -0,0 +1,1130 @@ +//! Oracle storage implementation using OracleStore helper. +//! +//! Structure: +//! 1. OracleStore helper and common utilities +//! 2. OracleConversationStorage +//! 3. OracleConversationItemStorage +//! 4. OracleResponseStorage + +use std::{path::Path, sync::Arc, time::Duration}; + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use deadpool::managed::{Manager, Metrics, Pool, RecycleError, RecycleResult}; +use oracle::{ + sql_type::{OracleType, ToSql}, + Connection, Row, +}; +use serde_json::Value; + +use super::core::{ + make_item_id, Conversation, ConversationId, ConversationItem, ConversationItemId, + ConversationItemStorage, ConversationItemStorageError, ConversationMetadata, + ConversationStorage, ConversationStorageError, ListParams, NewConversation, + NewConversationItem, ResponseChain, ResponseId, ResponseStorage, ResponseStorageError, + SortOrder, StoredResponse, +}; +use crate::{ + config::OracleConfig, + data_connector::common::{ + parse_json_value, parse_metadata, parse_raw_response, parse_tool_calls, + }, +}; +// ============================================================================ +// PART 1: OracleStore Helper + Common Utilities +// ============================================================================ + +/// Shared Oracle connection pool infrastructure +/// +/// This helper eliminates ~540 LOC of duplication across storage implementations. +/// It handles connection pooling, error mapping, and client configuration. +pub(crate) struct OracleStore { + pool: Pool, +} + +impl OracleStore { + /// Create pool with custom schema initialization + /// + /// The `init_schema` function receives a connection and should: + /// - Check if tables/indexes exist + /// - Create them if needed + /// - Return Ok(()) on success or Err(message) on failure + pub fn new( + config: &OracleConfig, + init_schema: impl FnOnce(&Connection) -> Result<(), String>, + ) -> Result { + // Configure Oracle client (wallet, etc.) + configure_oracle_client(config)?; + + // Initialize schema using the provided function + let conn = Connection::connect( + &config.username, + &config.password, + &config.connect_descriptor, + ) + .map_err(map_oracle_error)?; + + init_schema(&conn)?; + drop(conn); + + // Create connection pool + let config_arc = Arc::new(config.clone()); + let manager = OracleConnectionManager { + params: Arc::new(OracleConnectParams::from_config(&config_arc)), + }; + + let mut builder = Pool::builder(manager) + .max_size(config.pool_max) + .runtime(deadpool::Runtime::Tokio1); + + if config.pool_timeout_secs > 0 { + builder = builder.wait_timeout(Some(Duration::from_secs(config.pool_timeout_secs))); + } + + let pool = builder + .build() + .map_err(|e| format!("Failed to build Oracle pool: {e}"))?; + + Ok(Self { pool }) + } + + /// Execute function with a connection from the pool + pub async fn execute(&self, func: F) -> Result + where + F: FnOnce(&Connection) -> Result + Send + 'static, + T: Send + 'static, + { + let connection = self + .pool + .get() + .await + .map_err(|e| format!("Failed to get Oracle connection: {e}"))?; + + tokio::task::spawn_blocking(move || { + let result = func(&connection); + drop(connection); + result + }) + .await + .map_err(|e| format!("Task execution failed: {e}"))? + } +} + +impl Clone for OracleStore { + fn clone(&self) -> Self { + Self { + pool: self.pool.clone(), + } + } +} + +// Error mapping helper +pub(crate) fn map_oracle_error(err: oracle::Error) -> String { + if let Some(db_err) = err.db_error() { + format!( + "Oracle error (code {}): {}", + db_err.code(), + db_err.message() + ) + } else { + err.to_string() + } +} + +// Client configuration helper +fn configure_oracle_client(config: &OracleConfig) -> Result<(), String> { + if let Some(wallet_path) = &config.wallet_path { + let path = Path::new(wallet_path); + + if !path.is_dir() { + return Err(format!( + "Oracle wallet path '{}' is not a directory", + wallet_path + )); + } + + if !path.join("tnsnames.ora").exists() && !path.join("sqlnet.ora").exists() { + return Err(format!( + "Oracle wallet path '{}' is missing tnsnames.ora or sqlnet.ora", + wallet_path + )); + } + + std::env::set_var("TNS_ADMIN", wallet_path); + } + Ok(()) +} + +// Connection parameters +#[derive(Clone)] +pub(crate) struct OracleConnectParams { + pub username: String, + pub password: String, + pub connect_descriptor: String, +} + +impl OracleConnectParams { + pub fn from_config(config: &OracleConfig) -> Self { + Self { + username: config.username.clone(), + password: config.password.clone(), + connect_descriptor: config.connect_descriptor.clone(), + } + } +} + +impl std::fmt::Debug for OracleConnectParams { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OracleConnectParams") + .field("username", &self.username) + .field("connect_descriptor", &self.connect_descriptor) + .finish() + } +} + +// Connection manager (same for all stores) +#[derive(Clone)] +struct OracleConnectionManager { + params: Arc, +} + +impl std::fmt::Debug for OracleConnectionManager { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OracleConnectionManager") + .field("username", &self.params.username) + .field("connect_descriptor", &self.params.connect_descriptor) + .finish() + } +} + +#[async_trait] +impl Manager for OracleConnectionManager { + type Type = Connection; + type Error = oracle::Error; + + fn create( + &self, + ) -> impl std::future::Future> + Send { + let params = self.params.clone(); + async move { + let mut conn = Connection::connect( + ¶ms.username, + ¶ms.password, + ¶ms.connect_descriptor, + )?; + conn.set_autocommit(true); + Ok(conn) + } + } + + #[allow(clippy::manual_async_fn)] + fn recycle( + &self, + conn: &mut Connection, + _: &Metrics, + ) -> impl std::future::Future> + Send { + async move { conn.ping().map_err(RecycleError::Backend) } + } +} + +// ============================================================================ +// PART 2: OracleConversationStorage +// ============================================================================ + +#[derive(Clone)] +pub struct OracleConversationStorage { + store: OracleStore, +} + +impl OracleConversationStorage { + pub fn new(config: OracleConfig) -> Result { + let store = OracleStore::new(&config, |conn| { + // Check if table exists + let exists: i64 = conn + .query_row_as( + "SELECT COUNT(*) FROM user_tables WHERE table_name = 'CONVERSATIONS'", + &[], + ) + .map_err(map_oracle_error)?; + + // Create table if missing + if exists == 0 { + conn.execute( + "CREATE TABLE conversations ( + id VARCHAR2(64) PRIMARY KEY, + created_at TIMESTAMP WITH TIME ZONE, + metadata CLOB + )", + &[], + ) + .map_err(map_oracle_error)?; + } + + Ok(()) + }) + .map_err(ConversationStorageError::StorageError)?; + + Ok(Self { store }) + } + + fn parse_metadata( + raw: Option, + ) -> Result, ConversationStorageError> { + match raw { + Some(json) if !json.is_empty() => { + let value: Value = serde_json::from_str(&json)?; + match value { + Value::Object(map) => Ok(Some(map)), + Value::Null => Ok(None), + other => Err(ConversationStorageError::StorageError(format!( + "conversation metadata expected object, got {other}" + ))), + } + } + _ => Ok(None), + } + } +} + +#[async_trait] +impl ConversationStorage for OracleConversationStorage { + async fn create_conversation( + &self, + input: NewConversation, + ) -> Result { + let conversation = Conversation::new(input); + let id_str = conversation.id.0.clone(); + let created_at = conversation.created_at; + let metadata_json = conversation + .metadata + .as_ref() + .map(serde_json::to_string) + .transpose()?; + + self.store + .execute(move |conn| { + conn.execute( + "INSERT INTO conversations (id, created_at, metadata) VALUES (:1, :2, :3)", + &[&id_str, &created_at, &metadata_json], + ) + .map(|_| ()) + .map_err(map_oracle_error) + }) + .await + .map_err(ConversationStorageError::StorageError)?; + + Ok(conversation) + } + + async fn get_conversation( + &self, + id: &ConversationId, + ) -> Result, ConversationStorageError> { + let lookup = id.0.clone(); + self.store + .execute(move |conn| { + let mut stmt = conn + .statement("SELECT id, created_at, metadata FROM conversations WHERE id = :1") + .build() + .map_err(map_oracle_error)?; + let mut rows = stmt.query(&[&lookup]).map_err(map_oracle_error)?; + + if let Some(row_res) = rows.next() { + let row = row_res.map_err(map_oracle_error)?; + let id: String = row.get(0).map_err(map_oracle_error)?; + let created_at: DateTime = row.get(1).map_err(map_oracle_error)?; + let metadata_raw: Option = row.get(2).map_err(map_oracle_error)?; + let metadata = Self::parse_metadata(metadata_raw).map_err(|e| e.to_string())?; + Ok(Some(Conversation::with_parts( + ConversationId(id), + created_at, + metadata, + ))) + } else { + Ok(None) + } + }) + .await + .map_err(ConversationStorageError::StorageError) + } + + async fn update_conversation( + &self, + id: &ConversationId, + metadata: Option, + ) -> Result, ConversationStorageError> { + let id_str = id.0.clone(); + let metadata_json = metadata.as_ref().map(serde_json::to_string).transpose()?; + let conversation_id = id.clone(); + + self.store + .execute(move |conn| { + let mut stmt = conn + .statement( + "UPDATE conversations \ + SET metadata = :1 \ + WHERE id = :2 \ + RETURNING created_at INTO :3", + ) + .build() + .map_err(map_oracle_error)?; + + stmt.bind(3, &OracleType::TimestampTZ(6)) + .map_err(map_oracle_error)?; + stmt.execute(&[&metadata_json, &id_str]) + .map_err(map_oracle_error)?; + + if stmt.row_count().map_err(map_oracle_error)? == 0 { + return Ok(None); + } + + let mut created_at: Vec> = + stmt.returned_values(3).map_err(map_oracle_error)?; + let created_at = created_at + .pop() + .ok_or_else(|| "Oracle update did not return created_at".to_string())?; + + Ok(Some(Conversation::with_parts( + conversation_id, + created_at, + metadata, + ))) + }) + .await + .map_err(ConversationStorageError::StorageError) + } + + async fn delete_conversation( + &self, + id: &ConversationId, + ) -> Result { + let id_str = id.0.clone(); + let res = self + .store + .execute(move |conn| { + conn.execute("DELETE FROM conversations WHERE id = :1", &[&id_str]) + .map_err(map_oracle_error) + }) + .await + .map_err(ConversationStorageError::StorageError)?; + + Ok(res + .row_count() + .map_err(|e| ConversationStorageError::StorageError(map_oracle_error(e)))? + > 0) + } +} + +// ============================================================================ +// PART 3: OracleConversationItemStorage +// ============================================================================ + +#[derive(Clone)] +pub struct OracleConversationItemStorage { + store: OracleStore, +} + +impl OracleConversationItemStorage { + pub fn new(config: OracleConfig) -> Result { + let store = OracleStore::new(&config, |conn| { + // Create conversation_items table + let exists_items: i64 = conn + .query_row_as( + "SELECT COUNT(*) FROM user_tables WHERE table_name = 'CONVERSATION_ITEMS'", + &[], + ) + .map_err(map_oracle_error)?; + + if exists_items == 0 { + conn.execute( + "CREATE TABLE conversation_items ( + id VARCHAR2(64) PRIMARY KEY, + response_id VARCHAR2(64), + item_type VARCHAR2(32) NOT NULL, + role VARCHAR2(32), + content CLOB, + status VARCHAR2(32), + created_at TIMESTAMP WITH TIME ZONE + )", + &[], + ) + .map_err(map_oracle_error)?; + } + + // Create conversation_item_links table + let exists_links: i64 = conn + .query_row_as( + "SELECT COUNT(*) FROM user_tables WHERE table_name = 'CONVERSATION_ITEM_LINKS'", + &[], + ) + .map_err(map_oracle_error)?; + + if exists_links == 0 { + conn.execute( + "CREATE TABLE conversation_item_links ( + conversation_id VARCHAR2(64) NOT NULL, + item_id VARCHAR2(64) NOT NULL, + added_at TIMESTAMP WITH TIME ZONE, + CONSTRAINT pk_conv_item_link PRIMARY KEY (conversation_id, item_id) + )", + &[], + ) + .map_err(map_oracle_error)?; + + conn.execute( + "CREATE INDEX conv_item_links_conv_idx ON conversation_item_links (conversation_id, added_at)", + &[], + ) + .map_err(map_oracle_error)?; + } + + Ok(()) + }) + .map_err(ConversationItemStorageError::StorageError)?; + + Ok(Self { store }) + } +} + +#[async_trait] +impl ConversationItemStorage for OracleConversationItemStorage { + async fn create_item( + &self, + item: NewConversationItem, + ) -> Result { + let id = item + .id + .clone() + .unwrap_or_else(|| make_item_id(&item.item_type)); + let created_at = Utc::now(); + let content_json = serde_json::to_string(&item.content)?; + + let conversation_item = ConversationItem { + id: id.clone(), + response_id: item.response_id.clone(), + item_type: item.item_type.clone(), + role: item.role.clone(), + content: item.content, + status: item.status.clone(), + created_at, + }; + + let id_str = conversation_item.id.0.clone(); + let response_id = conversation_item.response_id.clone(); + let item_type = conversation_item.item_type.clone(); + let role = conversation_item.role.clone(); + let status = conversation_item.status.clone(); + + self.store + .execute(move |conn| { + conn.execute( + "INSERT INTO conversation_items (id, response_id, item_type, role, content, status, created_at) \ + VALUES (:1, :2, :3, :4, :5, :6, :7)", + &[&id_str, &response_id, &item_type, &role, &content_json, &status, &created_at], + ) + .map_err(map_oracle_error)?; + Ok(()) + }) + .await + .map_err(ConversationItemStorageError::StorageError)?; + + Ok(conversation_item) + } + + async fn link_item( + &self, + conversation_id: &ConversationId, + item_id: &ConversationItemId, + added_at: DateTime, + ) -> Result<(), ConversationItemStorageError> { + let cid = conversation_id.0.clone(); + let iid = item_id.0.clone(); + self.store + .execute(move |conn| { + conn.execute( + "INSERT INTO conversation_item_links (conversation_id, item_id, added_at) VALUES (:1, :2, :3)", + &[&cid, &iid, &added_at], + ) + .map_err(map_oracle_error)?; + Ok(()) + }) + .await + .map_err(ConversationItemStorageError::StorageError) + } + + async fn list_items( + &self, + conversation_id: &ConversationId, + params: ListParams, + ) -> Result, ConversationItemStorageError> { + let cid = conversation_id.0.clone(); + let limit: i64 = params.limit as i64; + let order_desc = matches!(params.order, SortOrder::Desc); + let after_id = params.after.clone(); + + // Resolve the added_at of the after cursor if provided + let after_key: Option<(DateTime, String)> = if let Some(ref aid) = after_id { + self.store + .execute({ + let cid = cid.clone(); + let aid = aid.clone(); + move |conn| { + let mut stmt = conn + .statement( + "SELECT added_at FROM conversation_item_links WHERE conversation_id = :1 AND item_id = :2", + ) + .build() + .map_err(map_oracle_error)?; + let mut rows = stmt.query(&[&cid, &aid]).map_err(map_oracle_error)?; + if let Some(row_res) = rows.next() { + let row = row_res.map_err(map_oracle_error)?; + let ts: DateTime = row.get(0).map_err(map_oracle_error)?; + Ok(Some((ts, aid))) + } else { + Ok(None) + } + } + }) + .await + .map_err(ConversationItemStorageError::StorageError)? + } else { + None + }; + + // Build the main list query + let rows: Vec<(String, Option, String, Option, Option, Option, DateTime)> = + self.store + .execute({ + let cid = cid.clone(); + move |conn| { + let mut sql = String::from( + "SELECT i.id, i.response_id, i.item_type, i.role, i.content, i.status, i.created_at \ + FROM conversation_item_links l \ + JOIN conversation_items i ON i.id = l.item_id \ + WHERE l.conversation_id = :cid", + ); + + // Cursor predicate + if let Some((_ts, _iid)) = &after_key { + if order_desc { + sql.push_str(" AND (l.added_at < :ats OR (l.added_at = :ats AND l.item_id < :iid))"); + } else { + sql.push_str(" AND (l.added_at > :ats OR (l.added_at = :ats AND l.item_id > :iid))"); + } + } + + // Order and limit + if order_desc { + sql.push_str(" ORDER BY l.added_at DESC, l.item_id DESC"); + } else { + sql.push_str(" ORDER BY l.added_at ASC, l.item_id ASC"); + } + sql.push_str(" FETCH NEXT :limit ROWS ONLY"); + + // Build params and perform a named SELECT query + let mut params_vec: Vec<(&str, &dyn ToSql)> = vec![("cid", &cid)]; + if let Some((ts, iid)) = &after_key { + params_vec.push(("ats", ts)); + params_vec.push(("iid", iid)); + } + params_vec.push(("limit", &limit)); + + let rows_iter = conn.query_named(&sql, ¶ms_vec).map_err(map_oracle_error)?; + + let mut out = Vec::new(); + for row_res in rows_iter { + let row = row_res.map_err(map_oracle_error)?; + let id: String = row.get(0).map_err(map_oracle_error)?; + let resp_id: Option = row.get(1).map_err(map_oracle_error)?; + let item_type: String = row.get(2).map_err(map_oracle_error)?; + let role: Option = row.get(3).map_err(map_oracle_error)?; + let content_raw: Option = row.get(4).map_err(map_oracle_error)?; + let status: Option = row.get(5).map_err(map_oracle_error)?; + let created_at: DateTime = row.get(6).map_err(map_oracle_error)?; + out.push((id, resp_id, item_type, role, content_raw, status, created_at)); + } + Ok(out) + } + }) + .await + .map_err(ConversationItemStorageError::StorageError)?; + + // Map rows to ConversationItem + rows.into_iter() + .map( + |(id, resp_id, item_type, role, content_raw, status, created_at)| { + let content = match content_raw { + Some(s) => { + serde_json::from_str(&s).map_err(ConversationItemStorageError::from)? + } + None => Value::Null, + }; + Ok(ConversationItem { + id: ConversationItemId(id), + response_id: resp_id, + item_type, + role, + content, + status, + created_at, + }) + }, + ) + .collect() + } + + async fn get_item( + &self, + item_id: &ConversationItemId, + ) -> Result, ConversationItemStorageError> { + let iid = item_id.0.clone(); + + self.store + .execute(move |conn| { + let mut stmt = conn + .statement( + "SELECT id, response_id, item_type, role, content, status, created_at \ + FROM conversation_items WHERE id = :1", + ) + .build() + .map_err(map_oracle_error)?; + + let mut rows = stmt.query(&[&iid]).map_err(map_oracle_error)?; + + if let Some(row_res) = rows.next() { + let row = row_res.map_err(map_oracle_error)?; + let id: String = row.get(0).map_err(map_oracle_error)?; + let response_id: Option = row.get(1).map_err(map_oracle_error)?; + let item_type: String = row.get(2).map_err(map_oracle_error)?; + let role: Option = row.get(3).map_err(map_oracle_error)?; + let content_raw: Option = row.get(4).map_err(map_oracle_error)?; + let status: Option = row.get(5).map_err(map_oracle_error)?; + let created_at: DateTime = row.get(6).map_err(map_oracle_error)?; + + let content = match content_raw { + Some(s) => serde_json::from_str(&s).map_err(|e| e.to_string())?, + None => Value::Null, + }; + + Ok(Some(ConversationItem { + id: ConversationItemId(id), + response_id, + item_type, + role, + content, + status, + created_at, + })) + } else { + Ok(None) + } + }) + .await + .map_err(ConversationItemStorageError::StorageError) + } + + async fn is_item_linked( + &self, + conversation_id: &ConversationId, + item_id: &ConversationItemId, + ) -> Result { + let cid = conversation_id.0.clone(); + let iid = item_id.0.clone(); + + self.store + .execute(move |conn| { + let count: i64 = conn + .query_row_as( + "SELECT COUNT(*) FROM conversation_item_links WHERE conversation_id = :1 AND item_id = :2", + &[&cid, &iid], + ) + .map_err(map_oracle_error)?; + Ok(count > 0) + }) + .await + .map_err(ConversationItemStorageError::StorageError) + } + + async fn delete_item( + &self, + conversation_id: &ConversationId, + item_id: &ConversationItemId, + ) -> Result<(), ConversationItemStorageError> { + let cid = conversation_id.0.clone(); + let iid = item_id.0.clone(); + + self.store + .execute(move |conn| { + conn.execute( + "DELETE FROM conversation_item_links WHERE conversation_id = :1 AND item_id = :2", + &[&cid, &iid], + ) + .map_err(map_oracle_error)?; + Ok(()) + }) + .await + .map_err(ConversationItemStorageError::StorageError) + } +} + +// ============================================================================ +// PART 4: OracleResponseStorage +// ============================================================================ + +const SELECT_BASE: &str = "SELECT id, previous_response_id, input, instructions, output, \ + tool_calls, metadata, created_at, safety_identifier, model, conversation_id, raw_response FROM responses"; + +#[derive(Clone)] +pub struct OracleResponseStorage { + store: OracleStore, +} + +impl OracleResponseStorage { + pub fn new(config: OracleConfig) -> Result { + let store = OracleStore::new(&config, |conn| { + // Create responses table + let exists: i64 = conn + .query_row_as( + "SELECT COUNT(*) FROM user_tables WHERE table_name = 'RESPONSES'", + &[], + ) + .map_err(map_oracle_error)?; + + if exists == 0 { + conn.execute( + "CREATE TABLE responses ( + id VARCHAR2(64) PRIMARY KEY, + conversation_id VARCHAR2(64), + previous_response_id VARCHAR2(64), + input CLOB, + instructions CLOB, + output CLOB, + tool_calls CLOB, + metadata CLOB, + created_at TIMESTAMP WITH TIME ZONE, + safety_identifier VARCHAR2(128), + model VARCHAR2(128), + raw_response CLOB + )", + &[], + ) + .map_err(map_oracle_error)?; + } else { + Self::alter_safety_identifier_column(conn)?; + Self::remove_user_id_column_if_exists(conn)?; + } + + // Create indexes + create_index_if_missing( + conn, + "RESPONSES_PREV_IDX", + "CREATE INDEX responses_prev_idx ON responses(previous_response_id)", + )?; + create_index_if_missing( + conn, + "RESPONSES_USER_IDX", + "CREATE INDEX responses_user_idx ON responses(safety_identifier)", + )?; + + Ok(()) + }) + .map_err(ResponseStorageError::StorageError)?; + + Ok(Self { store }) + } + + // Alter safety_identifier column if missing + fn alter_safety_identifier_column(conn: &Connection) -> Result<(), String> { + let present: i64 = conn + .query_row_as( + "SELECT COUNT(*) FROM user_tab_columns WHERE table_name = 'RESPONSES' AND column_name = 'SAFETY_IDENTIFIER'", + &[], + ) + .map_err(map_oracle_error)?; + + if present == 0 { + if let Err(err) = conn.execute( + "ALTER TABLE responses ADD (safety_identifier VARCHAR2(128))", + &[], + ) { + let present_after: i64 = conn + .query_row_as( + "SELECT COUNT(*) FROM user_tab_columns WHERE table_name = 'RESPONSES' AND column_name = 'SAFETY_IDENTIFIER'", + &[], + ) + .map_err(map_oracle_error)?; + if present_after == 0 { + return Err(map_oracle_error(err)); + } + } + } + + Ok(()) + } + + // Remove user_id column if exists + fn remove_user_id_column_if_exists(conn: &Connection) -> Result<(), String> { + let present: i64 = conn + .query_row_as( + "SELECT COUNT(*) FROM user_tab_columns WHERE table_name = 'RESPONSES' AND column_name = 'USER_ID'", + &[], + ) + .map_err(map_oracle_error)?; + + if present > 0 { + if let Err(err) = conn.execute("ALTER TABLE responses DROP COLUMN USER_ID", &[]) { + let present_after: i64 = conn + .query_row_as( + "SELECT COUNT(*) FROM user_tab_columns WHERE table_name = 'RESPONSES' AND column_name = 'USER_ID'", + &[], + ) + .map_err(map_oracle_error)?; + if present_after > 0 { + return Err(map_oracle_error(err)); + } + } + } + + Ok(()) + } + + fn build_response_from_row(row: &Row) -> Result { + let id: String = row.get(0).map_err(map_oracle_error)?; + let previous: Option = row.get(1).map_err(map_oracle_error)?; + let input_json: Option = row.get(2).map_err(map_oracle_error)?; + let instructions: Option = row.get(3).map_err(map_oracle_error)?; + let output_json: Option = row.get(4).map_err(map_oracle_error)?; + let tool_calls_json: Option = row.get(5).map_err(map_oracle_error)?; + let metadata_json: Option = row.get(6).map_err(map_oracle_error)?; + let created_at: DateTime = row.get(7).map_err(map_oracle_error)?; + let safety_identifier: Option = row.get(8).map_err(map_oracle_error)?; + let model: Option = row.get(9).map_err(map_oracle_error)?; + let conversation_id: Option = row.get(10).map_err(map_oracle_error)?; + let raw_response_json: Option = row.get(11).map_err(map_oracle_error)?; + + let previous_response_id = previous.map(ResponseId); + let tool_calls = parse_tool_calls(tool_calls_json)?; + let metadata = parse_metadata(metadata_json)?; + let raw_response = parse_raw_response(raw_response_json)?; + let input = parse_json_value(input_json)?; + let output = parse_json_value(output_json)?; + + Ok(StoredResponse { + id: ResponseId(id), + previous_response_id, + input, + instructions, + output, + tool_calls, + metadata, + created_at, + safety_identifier, + model, + conversation_id, + raw_response, + }) + } +} + +#[async_trait] +impl ResponseStorage for OracleResponseStorage { + async fn store_response( + &self, + response: StoredResponse, + ) -> Result { + let response_id = response.id.clone(); + let response_id_str = response_id.0.clone(); + let previous_id = response.previous_response_id.map(|r| r.0); + let json_input = serde_json::to_string(&response.input)?; + let json_output = serde_json::to_string(&response.output)?; + let json_tool_calls = serde_json::to_string(&response.tool_calls)?; + let json_metadata = serde_json::to_string(&response.metadata)?; + let json_raw_response = serde_json::to_string(&response.raw_response)?; + let instructions = response.instructions.clone(); + let created_at = response.created_at; + let safety_identifier = response.safety_identifier.clone(); + let model = response.model.clone(); + let conversation_id = response.conversation_id.clone(); + + self.store + .execute(move |conn| { + conn.execute( + "INSERT INTO responses (id, previous_response_id, input, instructions, output, \ + tool_calls, metadata, created_at, safety_identifier, model, conversation_id, raw_response) \ + VALUES (:1, :2, :3, :4, :5, :6, :7, :8, :9, :10, :11, :12)", + &[ + &response_id_str, + &previous_id, + &json_input, + &instructions, + &json_output, + &json_tool_calls, + &json_metadata, + &created_at, + &safety_identifier, + &model, + &conversation_id, + &json_raw_response, + ], + ) + .map(|_| ()) + .map_err(map_oracle_error) + }) + .await + .map_err(ResponseStorageError::StorageError)?; + + Ok(response_id) + } + + async fn get_response( + &self, + response_id: &ResponseId, + ) -> Result, ResponseStorageError> { + let id = response_id.0.clone(); + self.store + .execute(move |conn| { + let mut stmt = conn + .statement(&format!("{} WHERE id = :1", SELECT_BASE)) + .build() + .map_err(map_oracle_error)?; + let mut rows = stmt.query(&[&id]).map_err(map_oracle_error)?; + match rows.next() { + Some(row) => { + let row = row.map_err(map_oracle_error)?; + Self::build_response_from_row(&row).map(Some) + } + None => Ok(None), + } + }) + .await + .map_err(ResponseStorageError::StorageError) + } + + async fn delete_response(&self, response_id: &ResponseId) -> Result<(), ResponseStorageError> { + let id = response_id.0.clone(); + self.store + .execute(move |conn| { + conn.execute("DELETE FROM responses WHERE id = :1", &[&id]) + .map(|_| ()) + .map_err(map_oracle_error) + }) + .await + .map_err(ResponseStorageError::StorageError) + } + + async fn get_response_chain( + &self, + response_id: &ResponseId, + max_depth: Option, + ) -> Result { + let mut chain = ResponseChain::new(); + let mut current_id = Some(response_id.clone()); + let mut visited = 0usize; + + while let Some(ref lookup_id) = current_id { + if let Some(limit) = max_depth { + if visited >= limit { + break; + } + } + + let fetched = self.get_response(lookup_id).await?; + match fetched { + Some(response) => { + current_id = response.previous_response_id.clone(); + chain.responses.push(response); + visited += 1; + } + None => break, + } + } + + chain.responses.reverse(); + Ok(chain) + } + + async fn list_identifier_responses( + &self, + identifier: &str, + limit: Option, + ) -> Result, ResponseStorageError> { + let identifier = identifier.to_string(); + + self.store + .execute(move |conn| { + let sql = if let Some(limit) = limit { + format!( + "SELECT * FROM ({} WHERE safety_identifier = :1 ORDER BY created_at DESC) WHERE ROWNUM <= {}", + SELECT_BASE, limit + ) + } else { + format!("{} WHERE safety_identifier = :1 ORDER BY created_at DESC", SELECT_BASE) + }; + + let mut stmt = conn.statement(&sql).build().map_err(map_oracle_error)?; + let mut rows = stmt.query(&[&identifier]).map_err(map_oracle_error)?; + let mut results = Vec::new(); + + for row in &mut rows { + let row = row.map_err(map_oracle_error)?; + results.push(Self::build_response_from_row(&row)?); + } + + Ok(results) + }) + .await + .map_err(ResponseStorageError::StorageError) + } + + async fn delete_identifier_responses( + &self, + identifier: &str, + ) -> Result { + let identifier = identifier.to_string(); + let affected = self + .store + .execute(move |conn| { + conn.execute( + "DELETE FROM responses WHERE safety_identifier = :1", + &[&identifier], + ) + .map_err(map_oracle_error) + }) + .await + .map_err(ResponseStorageError::StorageError)?; + + let deleted = affected + .row_count() + .map_err(|e| ResponseStorageError::StorageError(map_oracle_error(e)))? + as usize; + Ok(deleted) + } +} + +// Helper functions for response parsing + +fn create_index_if_missing(conn: &Connection, index_name: &str, ddl: &str) -> Result<(), String> { + let count: i64 = conn + .query_row_as( + "SELECT COUNT(*) FROM user_indexes WHERE table_name = 'RESPONSES' AND index_name = :1", + &[&index_name], + ) + .map_err(map_oracle_error)?; + + if count == 0 { + if let Err(err) = conn.execute(ddl, &[]) { + if let Some(db_err) = err.db_error() { + // ORA-00955: name is already used by an existing object + // ORA-01408: such column list already indexed + // Both errors indicate the index already exists (race condition) + if db_err.code() != 955 && db_err.code() != 1408 { + return Err(map_oracle_error(err)); + } + } else { + return Err(map_oracle_error(err)); + } + } + } + + Ok(()) +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/postgres.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/postgres.rs new file mode 100644 index 00000000..96aabde1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/data_connector/postgres.rs @@ -0,0 +1,699 @@ +//! Postgres storage implementation using PostgresStore helper +//! +//! Structure: +//! 1. PostgresStore helper and common utilities +//! 2. PostgresConversationStorage +//! 3. PostgresConversationItemStorage +//! 4. PostgresResponseStorage + +use std::str::FromStr; + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use deadpool_postgres::{Manager, ManagerConfig, Pool, RecyclingMethod}; +use serde_json::Value; +use tokio_postgres::{NoTls, Row}; + +use crate::{ + config::PostgresConfig, + data_connector::{ + common::{parse_json_value, parse_metadata, parse_raw_response, parse_tool_calls}, + core::{ + make_item_id, ConversationItemResult, ConversationItemStorageError, + ConversationMetadata, ConversationResult, ConversationStorageError, ResponseChain, + ResponseResult, ResponseStorageError, + }, + Conversation, ConversationId, ConversationItem, ConversationItemId, + ConversationItemStorage, ConversationStorage, ListParams, NewConversation, + NewConversationItem, ResponseId, ResponseStorage, SortOrder, StoredResponse, + }, +}; + +pub(crate) struct PostgresStore { + pool: Pool, +} + +impl PostgresStore { + pub fn new(config: PostgresConfig) -> Result { + let pg_config = tokio_postgres::Config::from_str(config.db_url.as_str()).unwrap(); + let mgr_config = ManagerConfig { + recycling_method: RecyclingMethod::Fast, + }; + let mgr = Manager::from_config(pg_config, NoTls, mgr_config); + let pool = Pool::builder(mgr) + .max_size(config.pool_max) + .build() + .unwrap(); + + Ok(Self { pool }) + } +} + +impl Clone for PostgresStore { + fn clone(&self) -> Self { + Self { + pool: self.pool.clone(), + } + } +} + +pub struct PostgresConversationStorage { + store: PostgresStore, +} + +impl PostgresConversationStorage { + pub fn new(store: PostgresStore) -> Result { + futures::executor::block_on(Self::initialize_schema(store.clone())) + .expect("Failed to initialize conversations schema"); + Ok(Self { store }) + } + + async fn initialize_schema(store: PostgresStore) -> Result<(), ConversationStorageError> { + let client = store.pool.get().await.unwrap(); + client + .batch_execute( + " + CREATE TABLE IF NOT EXISTS conversations ( + id VARCHAR(64) PRIMARY KEY, + created_at TIMESTAMPTZ, + metadata JSON + );", + ) + .await + .unwrap(); + + Ok(()) + } + + fn parse_metadata( + metadata: Option, + ) -> Result, ConversationStorageError> { + match metadata { + None => Ok(None), + Some(s) => { + let s = s.trim(); + if s.is_empty() || s.eq_ignore_ascii_case("null") { + return Ok(None); + } + serde_json::from_str::(s) + .map(Some) + .map_err(|e| ConversationStorageError::StorageError(e.to_string())) + } + } + } +} + +#[async_trait] +impl ConversationStorage for PostgresConversationStorage { + async fn create_conversation( + &self, + input: NewConversation, + ) -> Result { + let conversation = Conversation::new(input); + let id_str = conversation.id.0.as_str(); + let created_at: DateTime = conversation.created_at; + let metadata_json = conversation + .metadata + .as_ref() + .map(serde_json::to_string) + .transpose()?; + let client = self.store.pool.get().await.unwrap(); + client + .execute( + "INSERT INTO conversations (id, created_at, metadata) VALUES ($1, $2, $3)", + &[&id_str, &created_at, &metadata_json], + ) + .await + .unwrap(); + Ok(conversation) + } + + async fn get_conversation( + &self, + id: &ConversationId, + ) -> Result, ConversationStorageError> { + let conversation_id = id.0.clone(); + let client = self.store.pool.get().await.unwrap(); + let rows = client + .query( + "SELECT id, created_at, metadata FROM conversations WHERE id = $1", + &[&conversation_id], + ) + .await + .unwrap(); + if rows.is_empty() { + return Ok(None); + } + let row = &rows[0]; + let id_str: String = row.get(0); + let created_at: DateTime = row.get(1); + let metadata_json: Option = row.get(2); + let metadata = Self::parse_metadata(metadata_json)?; + Ok(Some(Conversation::with_parts( + ConversationId(id_str), + created_at, + metadata, + ))) + } + + async fn update_conversation( + &self, + id: &ConversationId, + metadata: Option, + ) -> Result, ConversationStorageError> { + let conversation_id = id.0.clone(); + let metadata_json = metadata.as_ref().map(serde_json::to_string).transpose()?; + let client = self.store.pool.get().await.unwrap(); + let rows = client + .query( + "UPDATE conversations SET metadata = $1 WHERE id = $2 RETURNING created_at", + &[&metadata_json, &conversation_id], + ) + .await + .unwrap(); + if rows.is_empty() { + return Ok(None); + } + let row = &rows[0]; + let created_at: DateTime = row.get(0); + Ok(Some(Conversation::with_parts( + ConversationId(conversation_id), + created_at, + metadata, + ))) + } + + async fn delete_conversation(&self, id: &ConversationId) -> ConversationResult { + let conversation_id = id.0.clone(); + let client = self.store.pool.get().await.unwrap(); + let rows_deleted = client + .execute( + "DELETE FROM conversations WHERE id = $1", + &[&conversation_id], + ) + .await + .unwrap(); + Ok(rows_deleted > 0) + } +} + +pub struct PostgresConversationItemStorage { + store: PostgresStore, +} + +impl PostgresConversationItemStorage { + pub fn new(store: PostgresStore) -> Result { + futures::executor::block_on(Self::initialize_schema(store.clone())) + .expect("Failed to initialize conversation_items or conversation_item_links schema"); + Ok(Self { store }) + } + + async fn initialize_schema(store: PostgresStore) -> Result<(), ConversationItemStorageError> { + let client = store.pool.get().await.unwrap(); + client + .batch_execute( + " + CREATE TABLE IF NOT EXISTS conversation_items ( + id SERIAL PRIMARY KEY, + response_id VARCHAR(64), + item_type VARCHAR(32) NOT NULL, + role VARCHAR(32), + content JSON, + status VARCHAR(32), + created_at TIMESTAMPTZ + );", + ) + .await + .unwrap(); + + // Create conversation_item_links table + client + .batch_execute( + " + CREATE TABLE IF NOT EXISTS conversation_item_links ( + conversation_id VARCHAR(64), + item_id VARCHAR(64) NOT NULL, + added_at TIMESTAMPTZ, + CONSTRAINT pk_conv_item_link PRIMARY KEY (conversation_id, item_id) + ); + CREATE INDEX IF NOT EXISTS conv_item_links_conv_idx ON conversation_item_links (conversation_id, added_at);", + ) + .await + .unwrap(); + Ok(()) + } +} + +#[async_trait] +impl ConversationItemStorage for PostgresConversationItemStorage { + async fn create_item( + &self, + item: NewConversationItem, + ) -> Result { + let id = item + .id + .clone() + .unwrap_or_else(|| make_item_id(&item.item_type)); + let created_at = Utc::now(); + let content_json = serde_json::to_string(&item.content)?; + let conversation_item = ConversationItem { + id: id.clone(), + response_id: item.response_id.clone(), + item_type: item.item_type.clone(), + role: item.role.clone(), + content: item.content.clone(), + status: item.status.clone(), + created_at, + }; + let id_str = conversation_item.id.0.clone(); + let response_id = conversation_item.response_id.clone(); + let item_type = conversation_item.item_type.clone(); + let role = conversation_item.role.clone(); + let status = conversation_item.status.clone(); + + let client = self.store.pool.get().await.unwrap(); + client.execute("INSERT INTO conversation_items (id, response_id, item_type, role, content, status, created_at) VALUES ($1, $2, $3, $4, $5, $6, $7)", + &[&id_str, &response_id, &item_type, &role, &content_json, &status, &created_at]) + .await + .unwrap(); + Ok(conversation_item) + } + + async fn link_item( + &self, + conversation_id: &ConversationId, + item_id: &ConversationItemId, + added_at: DateTime, + ) -> ConversationItemResult<()> { + let cid = conversation_id.0.clone(); + let iid = item_id.0.clone(); + let client = self.store.pool.get().await.unwrap(); + client.execute("INSERT INTO conversation_item_links (conversation_id, item_id, added_at) VALUES ($1, $2, $3)", + &[&cid, &iid, &added_at]) + .await + .unwrap(); + Ok(()) + } + + async fn list_items( + &self, + conversation_id: &ConversationId, + params: ListParams, + ) -> ConversationItemResult> { + let cid = conversation_id.0.clone(); + let limit: i64 = params.limit as i64; + let order_desc = matches!(params.order, SortOrder::Desc); + let after_id = params.after.clone(); + + let after_key: Option<(DateTime, String)> = if let Some(ref aid) = after_id { + let cid = cid.clone(); + let aid = aid.clone(); + let client = self.store.pool.get().await.unwrap(); + let rows = client + .query( + "SELECT added_at FROM conversation_item_links WHERE conversation_id = $1 AND item_id = $2", + &[&cid, &aid], + ) + .await + .unwrap(); + if !rows.is_empty() { + let row = &rows[0]; + let ts: DateTime = row.get(0); + Some((ts, aid)) + } else { + None + } + } else { + None + }; + + let cid = cid.clone(); + let mut sql = String::from( + "SELECT i.id, i.response_id, i.item_type, i.role, i.content, i.status, i.created_at \ + FROM conversation_item_links l \ + JOIN conversation_items i ON i.id = l.item_id \ + WHERE l.conversation_id = $1", + ); + // If cursor provided, append predicate using $2/$3 + if let Some((_ts, _iid)) = &after_key { + if order_desc { + sql.push_str(" AND (l.added_at < $2 OR (l.added_at = $2 AND l.item_id < $3))"); + } else { + sql.push_str(" AND (l.added_at > $2 OR (l.added_at = $2 AND l.item_id > $3))"); + } + } + // Order and limit + if order_desc { + sql.push_str(" ORDER BY l.added_at DESC, l.item_id DESC"); + } else { + sql.push_str(" ORDER BY l.added_at ASC, l.item_id ASC"); + } + // PostgreSQL LIMIT + if after_key.is_some() { + sql.push_str(" LIMIT $4"); + } else { + sql.push_str(" LIMIT $2"); + } + let client = self.store.pool.get().await.unwrap(); + let rows = if let Some((ts, iid)) = &after_key { + client.query(&sql, &[&cid, ts, iid, &limit]).await.unwrap() + } else { + client.query(&sql, &[&cid, &limit]).await.unwrap() + }; + let mut out = Vec::new(); + for row in rows { + let id = row.get(0); + let resp_id: Option = row.get(1); + let item_type: String = row.get(2); + let role: Option = row.get(3); + let content_raw: Option = row.get(4); + let status: Option = row.get(5); + let created_at: DateTime = row.get(6); + out.push(( + id, + resp_id, + item_type, + role, + content_raw, + status, + created_at, + )); + } + out.into_iter() + .map( + |(id, resp_id, item_type, role, content_raw, status, created_at)| { + let content = match content_raw { + Some(s) => { + serde_json::from_str(&s).map_err(ConversationItemStorageError::from)? + } + None => Value::Null, + }; + Ok(ConversationItem { + id: ConversationItemId(id), + response_id: resp_id, + item_type, + role, + content, + status, + created_at, + }) + }, + ) + .collect() + } + + async fn get_item( + &self, + item_id: &ConversationItemId, + ) -> Result, ConversationItemStorageError> { + let iid = item_id.0.clone(); + let client = self.store.pool.get().await.unwrap(); + let row = client.query_one("SELECT id, response_id, item_type, role, content, status, created_at FROM converstation_items WHERE id = $1", &[&iid]).await.unwrap(); + if row.is_empty() { + Ok(None) + } else { + let id: String = row.get(0); + let response_id: Option = row.get(1); + let item_type: String = row.get(2); + let role: Option = row.get(3); + let content_raw: Option = row.get(4); + let status: Option = row.get(5); + let created_at: DateTime = row.get(6); + + let content = match content_raw { + Some(s) => serde_json::from_str(&s) + .map_err(ConversationItemStorageError::SerializationError)?, + None => Value::Null, + }; + + Ok(Some(ConversationItem { + id: ConversationItemId(id), + response_id, + item_type, + role, + content, + status, + created_at, + })) + } + } + + async fn is_item_linked( + &self, + conversation_id: &ConversationId, + item_id: &ConversationItemId, + ) -> ConversationItemResult { + let cid = conversation_id.0.clone(); + let iid = item_id.0.clone(); + let client = self.store.pool.get().await.unwrap(); + let row = client + .query_one( + "SELECT COUNT(*) FROM conversation_item_links WHERE conversation_id = $1 AND item_id = $2", + &[&cid, &iid], + ) + .await + .unwrap(); + let count: i64 = row.get(0); + Ok(count > 0) + } + + async fn delete_item( + &self, + conversation_id: &ConversationId, + item_id: &ConversationItemId, + ) -> ConversationItemResult<()> { + let cid = conversation_id.0.clone(); + let iid = item_id.0.clone(); + let client = self.store.pool.get().await.unwrap(); + client + .execute( + "DELETE FROM conversation_item_links WHERE conversation_id = $1 AND item_id = $2", + &[&cid, &iid], + ) + .await + .unwrap(); + Ok(()) + } +} + +pub struct PostgresResponseStorage { + store: PostgresStore, +} + +impl PostgresResponseStorage { + pub fn new(store: PostgresStore) -> Result { + futures::executor::block_on(Self::initialize_schema(store.clone())) + .expect("Failed to initialize responses schema"); + Ok(Self { store }) + } + + async fn initialize_schema(store: PostgresStore) -> Result<(), ResponseStorageError> { + let client = store.pool.get().await.unwrap(); + client + .batch_execute( + " + CREATE TABLE IF NOT EXISTS responses ( + id VARCHAR(64) PRIMARY KEY, + conversation_id VARCHAR(64), + previous_response_id VARCHAR(64), + input JSON, + instructions TEXT, + output JSON, + tool_calls JSON, + metadata JSON, + created_at TIMESTAMPTZ, + safety_identifier VARCHAR(128), + model VARCHAR(128), + raw_response JSON + );", + ) + .await + .unwrap(); + Ok(()) + } + + pub fn build_response_from_now(row: &Row) -> Result { + let id: String = row.get("id"); + let conversation_id: Option = row.get("conversation_id"); + let previous: Option = row.get("previous_response_id"); + let input_json: Option = row.get("input"); + let instructions: Option = row.get("instructions"); + let output_json: Option = row.get("output"); + let tool_calls_json: Option = row.get("tool_calls"); + let metadata_json: Option = row.get("metadata"); + let created_at: DateTime = row.get("created_at"); + let safety_identifier: Option = row.get("safety_identifier"); + let model: Option = row.get("model"); + let raw_response_json: Option = row.get("raw_response"); + + let previous_response_id = previous.map(ResponseId); + let tool_calls = parse_tool_calls(tool_calls_json)?; + let metadata = parse_metadata(metadata_json)?; + let raw_response = parse_raw_response(raw_response_json)?; + let input = parse_json_value(input_json)?; + let output = parse_json_value(output_json)?; + + Ok(StoredResponse { + id: ResponseId(id), + previous_response_id, + input, + instructions, + output, + tool_calls, + metadata, + created_at, + safety_identifier, + model, + conversation_id, + raw_response, + }) + } +} + +#[async_trait] +impl ResponseStorage for PostgresResponseStorage { + async fn store_response( + &self, + response: StoredResponse, + ) -> Result { + let response_id = response.id.clone(); + let response_id_str = response_id.0.clone(); + let previous_id = response.previous_response_id.map(|r| r.0); + let json_input = &response.input; + let json_output = &response.output; + let json_tool_calls = serde_json::to_string(&response.tool_calls)?; + let json_metadata = serde_json::to_string(&response.metadata)?; + let json_raw_response = &response.raw_response; + let instructions = response.instructions.clone(); + let created_at = response.created_at; + let safety_identifier = response.safety_identifier.clone(); + let model = response.model.clone(); + let conversation_id = response.conversation_id.clone(); + let client = self.store.pool.get().await.unwrap(); + let insert_count = client.execute( + "INSERT INTO responses (id, previous_response_id, input, instructions, output, \ + tool_calls, metadata, created_at, safety_identifier, model, conversation_id, raw_response) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)", + &[ + &response_id_str, + &previous_id, + &json_input, + &instructions, + &json_output, + &serde_json::json!(&json_tool_calls), + &serde_json::json!(&json_metadata), + &created_at, + &safety_identifier, + &model, + &conversation_id, + &json_raw_response, + ]).await.unwrap(); + println!("INSERT INTO responses VALUES {insert_count}"); + Ok(response_id) + } + + async fn get_response( + &self, + response_id: &ResponseId, + ) -> Result, ResponseStorageError> { + let id = response_id.0.clone(); + let client = self.store.pool.get().await.unwrap(); + let row = client + .query_one("SELECT * FROM responses WHERE id = $1", &[&id]) + .await + .unwrap(); + Self::build_response_from_now(&row) + .map(Some) + .map_err(|err| ResponseStorageError::StorageError(err.to_string())) + } + + async fn delete_response(&self, response_id: &ResponseId) -> ResponseResult<()> { + let id = response_id.0.clone(); + let client = self.store.pool.get().await.unwrap(); + client + .execute("DELETE FROM responses WHERE id = $1", &[&id]) + .await + .unwrap(); + Ok(()) + } + + async fn get_response_chain( + &self, + response_id: &ResponseId, + max_depth: Option, + ) -> ResponseResult { + let mut chain = ResponseChain::new(); + let mut current_id = Some(response_id.clone()); + let mut visited = 0usize; + + while let Some(ref lookup_id) = current_id { + if let Some(limit) = max_depth { + if visited >= limit { + break; + } + } + + let fetched = self.get_response(lookup_id).await?; + match fetched { + Some(response) => { + current_id = response.previous_response_id.clone(); + chain.responses.push(response); + visited += 1; + } + None => break, + } + } + + chain.responses.reverse(); + Ok(chain) + } + + async fn list_identifier_responses( + &self, + identifier: &str, + limit: Option, + ) -> ResponseResult> { + let identifier = identifier.to_string(); + let client = self.store.pool.get().await.unwrap(); + let rows = if let Some(l) = limit { + let l_i64: i64 = l as i64; + client + .query( + "SELECT * FROM responses WHERE safety_identifier = $1 ORDER BY created_at DESC LIMIT $2", + &[&identifier, &l_i64], + ) + .await + .unwrap() + } else { + client + .query( + "SELECT * FROM responses WHERE safety_identifier = $1 ORDER BY created_at DESC", + &[&identifier], + ) + .await + .unwrap() + }; + + let mut out = Vec::with_capacity(rows.len()); + for row in rows { + let resp = + Self::build_response_from_now(&row).map_err(ResponseStorageError::StorageError)?; + out.push(resp); + } + + Ok(out) + } + + async fn delete_identifier_responses(&self, identifier: &str) -> ResponseResult { + let identifier = identifier.to_string(); + let client = self.store.pool.get().await.unwrap(); + let rows_deleted = client + .execute( + "DELETE FROM responses WHERE safety_identifier = $1", + &[&identifier], + ) + .await + .unwrap(); + Ok(rows_deleted as usize) + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/grpc_client/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/grpc_client/mod.rs new file mode 100644 index 00000000..8edf7ce6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/grpc_client/mod.rs @@ -0,0 +1,7 @@ +pub mod sglang_scheduler; +pub mod vllm_engine; + +// Export both clients +// Re-export proto modules with explicit names +pub use sglang_scheduler::{proto as sglang_proto, SglangSchedulerClient}; +pub use vllm_engine::{proto as vllm_proto, VllmEngineClient}; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/grpc_client/sglang_scheduler.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/grpc_client/sglang_scheduler.rs new file mode 100644 index 00000000..579f094e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/grpc_client/sglang_scheduler.rs @@ -0,0 +1,786 @@ +use std::{ + convert::TryFrom, + pin::Pin, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + task::{Context, Poll}, + time::Duration, +}; + +use tonic::{transport::Channel, Request, Streaming}; +use tracing::{debug, warn}; + +use crate::protocols::{ + chat::ChatCompletionRequest, + common::{ResponseFormat, StringOrArray, ToolChoice, ToolChoiceValue}, + generate::GenerateRequest, + responses::ResponsesRequest, + sampling_params::SamplingParams as GenerateSamplingParams, +}; + +// Include the generated protobuf code +#[allow(clippy::all)] +pub mod proto { + #![allow(clippy::all, unused_qualifications)] + tonic::include_proto!("sglang.grpc.scheduler"); +} + +// The generated module structure depends on the package name in the .proto file +// package sglang.grpc.scheduler; generates a nested module structure + +/// A smart wrapper around Streaming that automatically +/// sends abort when dropped (e.g., due to client disconnection or early termination). +/// +/// This leverages Rust's RAII pattern to ensure cleanup happens automatically, +/// regardless of how the stream is dropped (panic, early return, client disconnect, etc.). +pub struct AbortOnDropStream { + inner: Streaming, + request_id: String, + client: SglangSchedulerClient, + aborted: Arc, +} + +impl AbortOnDropStream { + /// Create a new auto-aborting stream wrapper + pub fn new( + stream: Streaming, + request_id: String, + client: SglangSchedulerClient, + ) -> Self { + debug!("Created AbortOnDropStream for request {}", request_id); + Self { + inner: stream, + request_id, + client, + aborted: Arc::new(AtomicBool::new(false)), + } + } + + /// Manually mark the request as completed to prevent abort on drop. + /// Call this when the request completes successfully to avoid unnecessary abort RPC. + pub fn mark_completed(&self) { + // Use Release ordering to ensure that this write is visible to other threads + // that use Acquire on the same atomic variable + self.aborted.store(true, Ordering::Release); + debug!("Request {} marked as completed", self.request_id); + } +} + +impl Drop for AbortOnDropStream { + fn drop(&mut self) { + // Atomically check and set the aborted flag using compare_exchange. + // If compare_exchange fails, it means the flag was already true (from mark_completed), + // so we don't need to send abort. AcqRel is used for success to synchronize with + // mark_completed's Release, and Acquire for failure to see writes from mark_completed. + if self + .aborted + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return; + } + + let client = self.client.clone(); + let request_id = self.request_id.clone(); + + // Spawn a background task to send abort (since Drop is sync but abort_request is async) + tokio::spawn(async move { + debug!( + "Stream dropped without completion for request {}, sending abort", + request_id + ); + // Clone request_id for the error message since abort_request takes ownership + let request_id_for_log = request_id.clone(); + if let Err(e) = client + .abort_request(request_id, "Stream dropped".to_string()) + .await + { + warn!( + "Failed to send abort on drop for request {}: {}", + request_id_for_log, e + ); + } + }); + } +} + +// Implement Stream trait to make AbortOnDropStream work like the original Streaming +impl futures::Stream for AbortOnDropStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + // Delegate to the inner stream + Pin::new(&mut self.inner).poll_next(cx) + } +} + +/// gRPC client for SGLang scheduler +#[derive(Clone)] +pub struct SglangSchedulerClient { + client: proto::sglang_scheduler_client::SglangSchedulerClient, +} + +impl SglangSchedulerClient { + /// Create a new client and connect to the scheduler + pub async fn connect(endpoint: &str) -> Result> { + debug!("Connecting to SGLang scheduler at {}", endpoint); + + // Convert grpc:// to http:// for tonic + let http_endpoint = if let Some(addr) = endpoint.strip_prefix("grpc://") { + format!("http://{}", addr) + } else { + endpoint.to_string() + }; + + let channel = Channel::from_shared(http_endpoint)? + .http2_keep_alive_interval(Duration::from_secs(30)) + .keep_alive_timeout(Duration::from_secs(10)) + .keep_alive_while_idle(true) + .tcp_keepalive(Some(Duration::from_secs(60))) + .tcp_nodelay(true) + .http2_adaptive_window(true) + .initial_stream_window_size(Some(16 * 1024 * 1024)) // 16MB + .initial_connection_window_size(Some(32 * 1024 * 1024)) // 32MB + .connect() + .await?; + + let client = proto::sglang_scheduler_client::SglangSchedulerClient::new(channel); + + Ok(Self { client }) + } + + /// Submit a generation request (returns auto-aborting streaming response) + /// + /// The returned stream automatically sends an abort request when dropped, + /// ensuring proper cleanup even if the HTTP client disconnects or an error occurs. + /// Call `mark_completed()` on the stream after successful completion to prevent + /// unnecessary abort RPCs. + pub async fn generate( + &self, + req: proto::GenerateRequest, + ) -> Result> { + let request_id = req.request_id.clone(); + let mut client = self.client.clone(); + let request = Request::new(req); + let response = client.generate(request).await?; + + Ok(AbortOnDropStream::new( + response.into_inner(), + request_id, + self.clone(), + )) + } + + /// Perform health check + pub async fn health_check( + &self, + ) -> Result> { + debug!("Sending health check request"); + // HealthCheckRequest is now empty - server generates its own health check internally + let request = Request::new(proto::HealthCheckRequest {}); + + let mut client = self.client.clone(); + let response = client.health_check(request).await?; + debug!("Health check response received"); + Ok(response.into_inner()) + } + + /// Abort a request + pub async fn abort_request( + &self, + request_id: String, + reason: String, + ) -> Result<(), Box> { + debug!( + "Sending abort request for {} (reason: {})", + request_id, reason + ); + let request = Request::new(proto::AbortRequest { + request_id: request_id.clone(), + reason, + }); + + let mut client = self.client.clone(); + let response = client.abort(request).await?; + debug!( + "Abort response for {}: success={}, message={}", + request_id, + response.get_ref().success, + response.get_ref().message + ); + Ok(()) + } + + /// Get model information + pub async fn get_model_info( + &self, + ) -> Result> { + debug!("Requesting model info"); + let request = Request::new(proto::GetModelInfoRequest {}); + + let mut client = self.client.clone(); + let response = client.get_model_info(request).await?; + debug!("Model info response received"); + Ok(response.into_inner()) + } + + /// Get server information + pub async fn get_server_info( + &self, + ) -> Result> { + debug!("Requesting server info"); + let request = Request::new(proto::GetServerInfoRequest {}); + + let mut client = self.client.clone(); + let response = client.get_server_info(request).await?; + debug!("Server info response received"); + Ok(response.into_inner()) + } + + /// Build a single SGLang GenerateRequest from OpenAI ChatCompletionRequest + pub fn build_generate_request_from_chat( + &self, + request_id: String, + body: &ChatCompletionRequest, + processed_text: String, + token_ids: Vec, + multimodal_inputs: Option, + tool_call_constraint: Option<(String, String)>, // (constraint_type, constraint_value) + ) -> Result { + // Build sampling params + let sampling_params = + self.build_grpc_sampling_params_from_chat(body, tool_call_constraint)?; + + let grpc_request = proto::GenerateRequest { + request_id, + tokenized: Some(proto::TokenizedInput { + original_text: processed_text, + input_ids: token_ids, + }), + mm_inputs: multimodal_inputs, + sampling_params: Some(sampling_params), + return_logprob: body.logprobs, + logprob_start_len: -1, + top_logprobs_num: body.top_logprobs.unwrap_or(0) as i32, + return_hidden_states: body.return_hidden_states, + stream: body.stream, + ..Default::default() + }; + + Ok(grpc_request) + } + + /// Build a basic GenerateRequest from the SGLang spec GenerateRequest + pub fn build_plain_generate_request( + &self, + request_id: String, + body: &GenerateRequest, + original_text: Option, + token_ids: Vec, + ) -> Result { + let sampling_params = + Self::build_sampling_params_from_plain(body.sampling_params.as_ref())?; + + let grpc_request = proto::GenerateRequest { + request_id, + tokenized: Some(proto::TokenizedInput { + original_text: original_text.unwrap_or_default(), + input_ids: token_ids, + }), + sampling_params: Some(sampling_params), + return_logprob: body.return_logprob.unwrap_or(false), + logprob_start_len: body.logprob_start_len.unwrap_or(-1), + top_logprobs_num: body.top_logprobs_num.unwrap_or(0), + token_ids_logprob: body.token_ids_logprob.clone().unwrap_or_default(), + return_hidden_states: body.return_hidden_states, + stream: body.stream, + log_metrics: body.log_metrics, + ..Default::default() + }; + + Ok(grpc_request) + } + + /// Build a GenerateRequest from ResponsesRequest (OpenAI Responses API) + /// + /// NOTE: This is used by the Harmony router only. The Regular router uses + /// responses_to_chat() conversion and goes through the chat pipeline. + pub fn build_generate_request_from_responses( + &self, + request_id: String, + body: &ResponsesRequest, + processed_text: String, + token_ids: Vec, + harmony_stop_ids: Option>, + constraint: Option<(String, String)>, + ) -> Result { + // Build sampling params from ResponsesRequest + let mut sampling_params = + self.build_grpc_sampling_params_from_responses(body, constraint)?; + + // Inject Harmony stop token IDs if provided + if let Some(stop_ids) = harmony_stop_ids { + sampling_params.stop_token_ids = stop_ids; + } + + let grpc_request = proto::GenerateRequest { + request_id, + tokenized: Some(proto::TokenizedInput { + original_text: processed_text, + input_ids: token_ids, + }), + mm_inputs: None, // Responses API doesn't support multimodal yet + sampling_params: Some(sampling_params), + return_logprob: false, // Responses API uses top_logprobs field instead + logprob_start_len: -1, + top_logprobs_num: body.top_logprobs.unwrap_or(0) as i32, + return_hidden_states: false, + stream: body.stream.unwrap_or(false), + ..Default::default() + }; + + Ok(grpc_request) + } + + /// Build gRPC SamplingParams from ChatCompletionRequest + fn build_grpc_sampling_params_from_chat( + &self, + request: &ChatCompletionRequest, + tool_call_constraint: Option<(String, String)>, + ) -> Result { + let stop_sequences = self.extract_stop_strings(request); + + let max_new_tokens = request.max_completion_tokens.map(|v| v as i32); + + // Handle skip_special_tokens: set to false if tools are present and tool_choice is not "none" + let skip_special_tokens = if request.tools.is_some() { + match &request.tool_choice { + Some(ToolChoice::Value(ToolChoiceValue::None)) => request.skip_special_tokens, + Some(_) => false, // tool_choice is not "none" + None => false, // TODO: this assumes tool_choice defaults to "auto" when tools present + } + } else { + request.skip_special_tokens + }; + + Ok(proto::SamplingParams { + temperature: request.temperature.unwrap_or(1.0), + top_p: request.top_p.unwrap_or(1.0), + top_k: request.top_k.unwrap_or(-1), + min_p: request.min_p.unwrap_or(0.0), + frequency_penalty: request.frequency_penalty.unwrap_or(0.0), + presence_penalty: request.presence_penalty.unwrap_or(0.0), + repetition_penalty: request.repetition_penalty.unwrap_or(1.0), + max_new_tokens, + stop: stop_sequences, + stop_token_ids: request.stop_token_ids.clone().unwrap_or_default(), + skip_special_tokens, + spaces_between_special_tokens: true, // Default from Python SamplingParams + ignore_eos: request.ignore_eos, + no_stop_trim: request.no_stop_trim, + n: request.n.unwrap_or(1) as i32, + constraint: self.build_constraint_for_chat(request, tool_call_constraint)?, + ..Default::default() + }) + } + + /// Extract stop strings from request + fn extract_stop_strings(&self, request: &ChatCompletionRequest) -> Vec { + match &request.stop { + Some(StringOrArray::String(s)) => vec![s.clone()], + Some(StringOrArray::Array(arr)) => arr.clone(), + None => vec![], + } + } + + /// Build constraint for structured generation + fn build_constraint_for_chat( + &self, + request: &ChatCompletionRequest, + tool_call_constraint: Option<(String, String)>, + ) -> Result, String> { + let mut constraints = Vec::new(); + + // Handle response_format constraints + match &request.response_format { + Some(ResponseFormat::JsonObject) => { + // json_object mode - constrain to valid JSON object + let schema = serde_json::json!({"type": "object"}); + let schema_str = serde_json::to_string(&schema) + .map_err(|e| format!("Failed to serialize JSON schema: {}", e))?; + constraints.push(proto::sampling_params::Constraint::JsonSchema(schema_str)); + } + Some(ResponseFormat::JsonSchema { json_schema }) => { + let schema_str = serde_json::to_string(&json_schema.schema) + .map_err(|e| format!("Failed to serialize JSON schema: {}", e))?; + constraints.push(proto::sampling_params::Constraint::JsonSchema(schema_str)); + } + Some(ResponseFormat::Text) | None => { + // No constraint for text format + } + } + + if let Some(ebnf) = &request.ebnf { + constraints.push(proto::sampling_params::Constraint::EbnfGrammar( + ebnf.clone(), + )); + } + + if let Some(regex) = &request.regex { + constraints.push(proto::sampling_params::Constraint::Regex(regex.clone())); + } + + // Handle tool call constraint from preparation stage + if let Some((constraint_type, constraint_value)) = tool_call_constraint { + if !constraints.is_empty() { + return Err("Constrained decoding is not compatible with tool calls.".to_string()); + } + let tool_constraint = match constraint_type.as_str() { + "structural_tag" => { + proto::sampling_params::Constraint::StructuralTag(constraint_value) + } + "json_schema" => proto::sampling_params::Constraint::JsonSchema(constraint_value), + "ebnf" => proto::sampling_params::Constraint::EbnfGrammar(constraint_value), + "regex" => proto::sampling_params::Constraint::Regex(constraint_value), + _ => return Err(format!("Unknown constraint type: {}", constraint_type)), + }; + constraints.push(tool_constraint); + } + + match constraints.len() { + 0 => Ok(None), + 1 => Ok(constraints.pop()), + _ => Err("Multiple constraints are not allowed.".to_string()), + } + } + + /// Build gRPC SamplingParams from ResponsesRequest + fn build_grpc_sampling_params_from_responses( + &self, + request: &ResponsesRequest, + constraint: Option<(String, String)>, + ) -> Result { + // Used by Harmony models only. Regular models use Chat API path. + // Constraints come from Harmony preparation stage (structural_tag) or tool handling. + + let max_new_tokens = request.max_output_tokens.map(|v| v as i32); + + Ok(proto::SamplingParams { + temperature: request.temperature.unwrap_or(1.0), + top_p: request.top_p.unwrap_or(1.0), + top_k: -1, // ResponsesRequest doesn't expose top_k + min_p: 0.0, // ResponsesRequest doesn't expose min_p + frequency_penalty: 0.0, // ResponsesRequest doesn't expose frequency_penalty + presence_penalty: 0.0, // ResponsesRequest doesn't expose presence_penalty + repetition_penalty: 1.0, // ResponsesRequest doesn't expose repetition_penalty + max_new_tokens, + stop: vec![], // No stop sequences in Responses API + stop_token_ids: vec![], // Handled by Harmony stop tokens + skip_special_tokens: false, // Keep special tokens for Harmony + spaces_between_special_tokens: true, + ignore_eos: false, + no_stop_trim: false, + n: 1, // Responses API doesn't support n>1 + constraint: self.build_constraint_for_responses(constraint)?, + ..Default::default() + }) + } + + /// Build constraint for Responses API + /// + /// Handles constraints from Harmony preparation stage (structural_tag for Harmony models, + /// structured output via text field, or tool call constraints). + /// + /// Note: Regular gRPC models use Chat API path with response_format, not this function. + fn build_constraint_for_responses( + &self, + constraint: Option<(String, String)>, + ) -> Result, String> { + if let Some((constraint_type, constraint_value)) = constraint { + let parsed_constraint = match constraint_type.as_str() { + "structural_tag" => { + // Harmony models: structural tag from preparation stage + proto::sampling_params::Constraint::StructuralTag(constraint_value) + } + "json_schema" => proto::sampling_params::Constraint::JsonSchema(constraint_value), + "ebnf" => proto::sampling_params::Constraint::EbnfGrammar(constraint_value), + "regex" => proto::sampling_params::Constraint::Regex(constraint_value), + _ => return Err(format!("Unknown constraint type: {}", constraint_type)), + }; + Ok(Some(parsed_constraint)) + } else { + Ok(None) + } + } + + fn build_single_constraint_from_plain( + params: &GenerateSamplingParams, + ) -> Result, String> { + let mut constraints = Vec::new(); + if let Some(json_schema) = ¶ms.json_schema { + constraints.push(proto::sampling_params::Constraint::JsonSchema( + json_schema.clone(), + )); + } + if let Some(regex) = ¶ms.regex { + constraints.push(proto::sampling_params::Constraint::Regex(regex.clone())); + } + if let Some(ebnf) = ¶ms.ebnf { + constraints.push(proto::sampling_params::Constraint::EbnfGrammar( + ebnf.clone(), + )); + } + + match constraints.len() { + 0 => Ok(None), + 1 => Ok(constraints.pop()), + _ => Err("Multiple structured constraints are not allowed".to_string()), + } + } + + fn build_sampling_params_from_plain( + params: Option<&GenerateSamplingParams>, + ) -> Result { + let mut sampling = proto::SamplingParams { + temperature: 1.0, + top_p: 1.0, + top_k: -1, + repetition_penalty: 1.0, + n: 1, + skip_special_tokens: true, + spaces_between_special_tokens: true, + ..Default::default() + }; + + let Some(p) = params else { + return Ok(sampling); + }; + + // Simple field mappings using a macro + macro_rules! map_field { + ($field:ident) => { + if let Some(val) = p.$field { + sampling.$field = val; + } + }; + } + + map_field!(temperature); + map_field!(top_p); + map_field!(top_k); + map_field!(frequency_penalty); + map_field!(presence_penalty); + map_field!(repetition_penalty); + map_field!(min_p); + map_field!(ignore_eos); + map_field!(skip_special_tokens); + map_field!(no_stop_trim); + + // Handle stop sequences + if let Some(stop) = &p.stop { + match stop { + StringOrArray::String(s) => sampling.stop.push(s.clone()), + StringOrArray::Array(arr) => sampling.stop.extend(arr.clone()), + } + } + + // Handle stop token IDs + if let Some(stop_token_ids) = &p.stop_token_ids { + sampling.stop_token_ids = stop_token_ids.clone(); + } + + // Handle max_new_tokens with conversion + if let Some(max_new_tokens) = p.max_new_tokens { + sampling.max_new_tokens = + Some(i32::try_from(max_new_tokens).map_err(|_| { + "max_new_tokens must fit into a 32-bit signed integer".to_string() + })?); + } + + // Handle min_new_tokens with conversion + if let Some(min_new_tokens) = p.min_new_tokens { + sampling.min_new_tokens = i32::try_from(min_new_tokens) + .map_err(|_| "min_new_tokens must fit into a 32-bit signed integer".to_string())?; + } + + // Handle n with conversion + if let Some(n) = p.n { + sampling.n = i32::try_from(n) + .map_err(|_| "n must fit into a 32-bit signed integer".to_string())?; + } + + // Handle constraints (exactly one allowed) + sampling.constraint = Self::build_single_constraint_from_plain(p)?; + + Ok(sampling) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_proto_types_compilation() { + let _health_req = proto::HealthCheckRequest {}; + // HealthCheckRequest is now empty - no fields to test + } + + #[test] + fn test_generate_request_construction() { + let sampling_params = proto::SamplingParams { + temperature: 0.7, + max_new_tokens: Some(128), + top_p: 0.9, + top_k: 50, + stop: vec!["".to_string()], + ..Default::default() + }; + + let gen_req = proto::GenerateRequest { + request_id: "test-req-123".to_string(), + tokenized: Some(proto::TokenizedInput { + original_text: "Hello world".to_string(), + input_ids: vec![9906, 1917], // Mock token IDs for "Hello world" + }), + sampling_params: Some(sampling_params), + return_logprob: true, + logprob_start_len: 0, + top_logprobs_num: 5, + ..Default::default() + }; + + assert_eq!(gen_req.request_id, "test-req-123"); + if let Some(ref tokenized) = &gen_req.tokenized { + assert_eq!(tokenized.original_text, "Hello world"); + } + assert!(gen_req.return_logprob); + assert_eq!(gen_req.top_logprobs_num, 5); + + let params = gen_req.sampling_params.unwrap(); + assert_eq!(params.temperature, 0.7); + assert_eq!(params.max_new_tokens, Some(128)); + assert_eq!(params.stop, vec![""]); + } + + #[test] + fn test_health_check_request() { + let _health_req = proto::HealthCheckRequest {}; + // HealthCheckRequest is now empty - server generates its own test internally + } + + #[test] + fn test_abort_request_construction() { + let abort_req = proto::AbortRequest { + request_id: "req-456".to_string(), + reason: "User canceled".to_string(), + }; + assert_eq!(abort_req.request_id, "req-456"); + assert_eq!(abort_req.reason, "User canceled"); + } + + #[test] + fn test_sampling_params_defaults() { + let params = proto::SamplingParams::default(); + // Numeric fields have proto defaults (0) + assert_eq!(params.temperature, 0.0); + assert_eq!(params.top_p, 0.0); + assert_eq!(params.top_k, 0); + assert_eq!(params.repetition_penalty, 0.0); + assert_eq!(params.n, 0); + // Bool fields have proto defaults (false) + assert!(!params.skip_special_tokens); + assert!(!params.spaces_between_special_tokens); + assert!(!params.ignore_eos); + assert!(!params.no_stop_trim); + // Optional int fields should be None + assert_eq!(params.max_new_tokens, None); + assert_eq!(params.stream_interval, None); + // Other non-optional fields + assert_eq!(params.min_p, 0.0); + assert_eq!(params.frequency_penalty, 0.0); + assert_eq!(params.presence_penalty, 0.0); + assert!(params.stop.is_empty()); + } + + #[test] + fn test_multimodal_inputs() { + let mm_inputs = proto::MultimodalInputs { + image_urls: vec!["http://example.com/image.jpg".to_string()], + video_urls: vec![], + audio_urls: vec![], + image_data: vec![], + video_data: vec![], + audio_data: vec![], + modalities: vec!["image".to_string()], + ..Default::default() + }; + + assert_eq!(mm_inputs.image_urls.len(), 1); + assert_eq!(mm_inputs.image_urls[0], "http://example.com/image.jpg"); + assert_eq!(mm_inputs.modalities[0], "image"); + } + + // TODO: SessionParams not in current proto - skip test + + #[test] + fn test_embed_request() { + let embed_req = proto::EmbedRequest { + request_id: "embed-req-202".to_string(), + tokenized: Some(proto::TokenizedInput { + original_text: "This is a test sentence for embedding".to_string(), + input_ids: vec![2028, 374, 264, 1296, 11914, 369, 28537], // Mock token IDs + }), + log_metrics: true, + data_parallel_rank: 0, + ..Default::default() + }; + + assert_eq!(embed_req.request_id, "embed-req-202"); + if let Some(ref tokenized) = &embed_req.tokenized { + assert_eq!( + tokenized.original_text, + "This is a test sentence for embedding" + ); + } + assert!(embed_req.log_metrics); + assert_eq!(embed_req.data_parallel_rank, 0); + } + + #[tokio::test] + async fn test_client_connect_invalid_endpoint() { + let result = SglangSchedulerClient::connect("invalid://endpoint").await; + assert!(result.is_err()); + } + + #[test] + fn test_tokenized_input() { + let tokenized = proto::TokenizedInput { + original_text: "Hello world".to_string(), + input_ids: vec![1, 15043, 1917, 2], + }; + + assert_eq!(tokenized.original_text, "Hello world"); + assert_eq!(tokenized.input_ids, vec![1, 15043, 1917, 2]); + } + + #[test] + fn test_generate_stream_chunk() { + let chunk = proto::GenerateStreamChunk { + token_ids: vec![1234, 5678], + prompt_tokens: 5, + completion_tokens: 2, + cached_tokens: 3, + ..Default::default() + }; + + assert_eq!(chunk.token_ids, vec![1234, 5678]); + assert_eq!(chunk.prompt_tokens, 5); + assert_eq!(chunk.completion_tokens, 2); + assert_eq!(chunk.cached_tokens, 3); + } + + // TODO: ModelInfo not in current proto - skip test +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/grpc_client/vllm_engine.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/grpc_client/vllm_engine.rs new file mode 100644 index 00000000..5a95bc11 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/grpc_client/vllm_engine.rs @@ -0,0 +1,736 @@ +use std::{ + convert::TryFrom, + pin::Pin, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + task::{Context, Poll}, + time::Duration, +}; + +use tonic::{transport::Channel, Request, Streaming}; +use tracing::{debug, warn}; + +use crate::protocols::{ + chat::ChatCompletionRequest, + common::{ResponseFormat, StringOrArray, ToolChoice, ToolChoiceValue}, + generate::GenerateRequest, + responses::ResponsesRequest, + sampling_params::SamplingParams as GenerateSamplingParams, +}; + +// Include the generated protobuf code +#[allow(clippy::all)] +pub mod proto { + #![allow(clippy::all, unused_qualifications)] + tonic::include_proto!("vllm.grpc.engine"); +} + +// The generated module structure depends on the package name in the .proto file +// package vllm.grpc.engine; generates a nested module structure + +/// A smart wrapper around Streaming that automatically +/// sends abort when dropped (e.g., due to client disconnection or early termination). +/// +/// This leverages Rust's RAII pattern to ensure cleanup happens automatically, +/// regardless of how the stream is dropped (panic, early return, client disconnect, etc.). +pub struct AbortOnDropStream { + inner: Streaming, + request_id: String, + client: VllmEngineClient, + aborted: Arc, +} + +impl AbortOnDropStream { + /// Create a new auto-aborting stream wrapper + pub fn new( + stream: Streaming, + request_id: String, + client: VllmEngineClient, + ) -> Self { + debug!("Created AbortOnDropStream for request {}", request_id); + Self { + inner: stream, + request_id, + client, + aborted: Arc::new(AtomicBool::new(false)), + } + } + + /// Manually mark the request as completed to prevent abort on drop. + /// Call this when the request completes successfully to avoid unnecessary abort RPC. + pub fn mark_completed(&self) { + // Use Release ordering to ensure that this write is visible to other threads + // that use Acquire on the same atomic variable + self.aborted.store(true, Ordering::Release); + debug!("Request {} marked as completed", self.request_id); + } +} + +impl Drop for AbortOnDropStream { + fn drop(&mut self) { + // Atomically check and set the aborted flag using compare_exchange. + // If compare_exchange fails, it means the flag was already true (from mark_completed), + // so we don't need to send abort. AcqRel is used for success to synchronize with + // mark_completed's Release, and Acquire for failure to see writes from mark_completed. + if self + .aborted + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return; + } + + let client = self.client.clone(); + let request_id = self.request_id.clone(); + + // Spawn a background task to send abort (since Drop is sync but abort_request is async) + tokio::spawn(async move { + debug!( + "Stream dropped without completion for request {}, sending abort", + request_id + ); + // Clone request_id for the error message since abort_request takes ownership + let request_id_for_log = request_id.clone(); + if let Err(e) = client + .abort_request(request_id, "Stream dropped".to_string()) + .await + { + warn!( + "Failed to send abort on drop for request {}: {}", + request_id_for_log, e + ); + } + }); + } +} + +// Implement Stream trait to make AbortOnDropStream work like the original Streaming +impl futures::Stream for AbortOnDropStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + // Delegate to the inner stream + Pin::new(&mut self.inner).poll_next(cx) + } +} + +/// gRPC client for vLLM scheduler +#[derive(Clone)] +pub struct VllmEngineClient { + client: proto::vllm_engine_client::VllmEngineClient, +} + +impl VllmEngineClient { + /// Create a new client and connect to the vLLM server + pub async fn connect(endpoint: &str) -> Result> { + debug!("Connecting to vLLM gRPC server at {}", endpoint); + + // Convert grpc:// to http:// for tonic + let http_endpoint = if let Some(addr) = endpoint.strip_prefix("grpc://") { + format!("http://{}", addr) + } else { + endpoint.to_string() + }; + + let channel = Channel::from_shared(http_endpoint)? + .http2_keep_alive_interval(Duration::from_secs(30)) + .keep_alive_timeout(Duration::from_secs(10)) + .keep_alive_while_idle(true) + .tcp_keepalive(Some(Duration::from_secs(60))) + .tcp_nodelay(true) + .http2_adaptive_window(true) + .initial_stream_window_size(Some(16 * 1024 * 1024)) // 16MB + .initial_connection_window_size(Some(32 * 1024 * 1024)) // 32MB + .connect() + .await?; + + let client = proto::vllm_engine_client::VllmEngineClient::new(channel); + + Ok(Self { client }) + } + + /// Submit a generation request (returns auto-aborting streaming response) + /// + /// The returned stream automatically sends an abort request when dropped, + /// ensuring proper cleanup even if the HTTP client disconnects or an error occurs. + /// Call `mark_completed()` on the stream after successful completion to prevent + /// unnecessary abort RPCs. + pub async fn generate( + &self, + req: proto::GenerateRequest, + ) -> Result> { + let request_id = req.request_id.clone(); + let mut client = self.client.clone(); + let request = Request::new(req); + let response = client.generate(request).await?; + + Ok(AbortOnDropStream::new( + response.into_inner(), + request_id, + self.clone(), + )) + } + + /// Perform health check + pub async fn health_check( + &self, + ) -> Result> { + debug!("Sending health check request"); + // HealthCheckRequest is now empty - server generates its own health check internally + let request = Request::new(proto::HealthCheckRequest {}); + + let mut client = self.client.clone(); + let response = client.health_check(request).await?; + debug!("Health check response received"); + Ok(response.into_inner()) + } + + /// Abort a request + pub async fn abort_request( + &self, + request_id: String, + reason: String, + ) -> Result<(), Box> { + debug!( + "Sending abort request for {} (reason: {})", + request_id, reason + ); + let request = Request::new(proto::AbortRequest { + request_id: request_id.clone(), + reason, + }); + + let mut client = self.client.clone(); + let response = client.abort(request).await?; + debug!( + "Abort response for {}: success={}, message={}", + request_id, + response.get_ref().success, + response.get_ref().message + ); + Ok(()) + } + + /// Get model information + pub async fn get_model_info( + &self, + ) -> Result> { + debug!("Requesting model info"); + let request = Request::new(proto::GetModelInfoRequest {}); + + let mut client = self.client.clone(); + let response = client.get_model_info(request).await?; + debug!("Model info response received"); + Ok(response.into_inner()) + } + + /// Get server information + pub async fn get_server_info( + &self, + ) -> Result> { + debug!("Requesting server info"); + let request = Request::new(proto::GetServerInfoRequest {}); + + let mut client = self.client.clone(); + let response = client.get_server_info(request).await?; + debug!("Server info response received"); + Ok(response.into_inner()) + } + + /// Build a single vLLM GenerateRequest from OpenAI ChatCompletionRequest + pub fn build_generate_request_from_chat( + &self, + request_id: String, + body: &ChatCompletionRequest, + processed_text: String, + token_ids: Vec, + tool_call_constraint: Option<(String, String)>, // (constraint_type, constraint_value) + ) -> Result { + // Build sampling params + let sampling_params = + self.build_grpc_sampling_params_from_chat(body, tool_call_constraint)?; + + let grpc_request = proto::GenerateRequest { + request_id, + tokenized: Some(proto::TokenizedInput { + original_text: processed_text, + input_ids: token_ids, + }), + sampling_params: Some(sampling_params), + stream: body.stream, + }; + + Ok(grpc_request) + } + + /// Build a basic GenerateRequest from the vLLM spec GenerateRequest + pub fn build_plain_generate_request( + &self, + request_id: String, + body: &GenerateRequest, + original_text: Option, + token_ids: Vec, + ) -> Result { + let sampling_params = + Self::build_sampling_params_from_plain(body.sampling_params.as_ref())?; + + let grpc_request = proto::GenerateRequest { + request_id, + tokenized: Some(proto::TokenizedInput { + original_text: original_text.unwrap_or_default(), + input_ids: token_ids, + }), + sampling_params: Some(sampling_params), + stream: body.stream, + }; + + Ok(grpc_request) + } + + /// Build a GenerateRequest from ResponsesRequest (OpenAI Responses API) + /// + /// NOTE: This is used by the Harmony router only. The Regular router uses + /// responses_to_chat() conversion and goes through the chat pipeline. + pub fn build_generate_request_from_responses( + &self, + request_id: String, + body: &ResponsesRequest, + processed_text: String, + token_ids: Vec, + harmony_stop_ids: Option>, + constraint: Option<(String, String)>, + ) -> Result { + // Build sampling params from ResponsesRequest + let mut sampling_params = + self.build_grpc_sampling_params_from_responses(body, constraint)?; + + // Inject Harmony stop token IDs if provided + if let Some(stop_ids) = harmony_stop_ids { + sampling_params.stop_token_ids = stop_ids; + } + + let grpc_request = proto::GenerateRequest { + request_id, + tokenized: Some(proto::TokenizedInput { + original_text: processed_text, + input_ids: token_ids, + }), + sampling_params: Some(sampling_params), + stream: body.stream.unwrap_or(false), + }; + + Ok(grpc_request) + } + + /// Build gRPC SamplingParams from ChatCompletionRequest + fn build_grpc_sampling_params_from_chat( + &self, + request: &ChatCompletionRequest, + tool_call_constraint: Option<(String, String)>, + ) -> Result { + let stop_sequences = self.extract_stop_strings(request); + + let max_tokens = request.max_completion_tokens.map(|v| v as i32); + + // Handle skip_special_tokens: set to false if tools are present and tool_choice is not "none" + let skip_special_tokens = if request.tools.is_some() { + match &request.tool_choice { + Some(ToolChoice::Value(ToolChoiceValue::None)) => request.skip_special_tokens, + Some(_) => false, // tool_choice is not "none" + None => false, // TODO: this assumes tool_choice defaults to "auto" when tools present + } + } else { + request.skip_special_tokens + }; + + Ok(proto::SamplingParams { + temperature: request.temperature.unwrap_or(1.0), + top_p: request.top_p.unwrap_or(1.0), + top_k: request.top_k.unwrap_or(-1), + min_p: request.min_p.unwrap_or(0.0), + frequency_penalty: request.frequency_penalty.unwrap_or(0.0), + presence_penalty: request.presence_penalty.unwrap_or(0.0), + repetition_penalty: request.repetition_penalty.unwrap_or(1.0), + max_tokens, + stop: stop_sequences, + stop_token_ids: request.stop_token_ids.clone().unwrap_or_default(), + skip_special_tokens, + spaces_between_special_tokens: true, // Default from Python SamplingParams + ignore_eos: request.ignore_eos, + n: request.n.unwrap_or(1) as i32, + constraint: self.build_constraint_for_chat(request, tool_call_constraint)?, + ..Default::default() + }) + } + + /// Extract stop strings from request + fn extract_stop_strings(&self, request: &ChatCompletionRequest) -> Vec { + match &request.stop { + Some(StringOrArray::String(s)) => vec![s.clone()], + Some(StringOrArray::Array(arr)) => arr.clone(), + None => vec![], + } + } + + /// Build constraint for structured generation + fn build_constraint_for_chat( + &self, + request: &ChatCompletionRequest, + tool_call_constraint: Option<(String, String)>, + ) -> Result, String> { + let mut constraints = Vec::new(); + + // Handle response_format constraints + match &request.response_format { + Some(ResponseFormat::JsonObject) => { + // json_object mode - constrain to valid JSON object + let schema = serde_json::json!({"type": "object"}); + let schema_str = serde_json::to_string(&schema) + .map_err(|e| format!("Failed to serialize JSON schema: {}", e))?; + constraints.push(proto::sampling_params::Constraint::JsonSchema(schema_str)); + } + Some(ResponseFormat::JsonSchema { json_schema }) => { + let schema_str = serde_json::to_string(&json_schema.schema) + .map_err(|e| format!("Failed to serialize JSON schema: {}", e))?; + constraints.push(proto::sampling_params::Constraint::JsonSchema(schema_str)); + } + Some(ResponseFormat::Text) | None => { + // No constraint for text format + } + } + + // vLLM supports: json_schema, regex, grammar, structural_tag, json_object, choice + if let Some(ebnf) = &request.ebnf { + constraints.push(proto::sampling_params::Constraint::Grammar(ebnf.clone())); + } + + if let Some(regex) = &request.regex { + constraints.push(proto::sampling_params::Constraint::Regex(regex.clone())); + } + + // Handle tool call constraint from preparation stage + if let Some((constraint_type, constraint_value)) = tool_call_constraint { + if !constraints.is_empty() { + return Err("Constrained decoding is not compatible with tool calls.".to_string()); + } + let tool_constraint = match constraint_type.as_str() { + "structural_tag" => { + proto::sampling_params::Constraint::StructuralTag(constraint_value) + } + "json_schema" => proto::sampling_params::Constraint::JsonSchema(constraint_value), + "grammar" | "ebnf" => proto::sampling_params::Constraint::Grammar(constraint_value), + "regex" => proto::sampling_params::Constraint::Regex(constraint_value), + _ => return Err(format!("Unknown constraint type: {}", constraint_type)), + }; + constraints.push(tool_constraint); + } + + match constraints.len() { + 0 => Ok(None), + 1 => Ok(constraints.pop()), + _ => Err("Multiple constraints are not allowed.".to_string()), + } + } + + /// Build gRPC SamplingParams from ResponsesRequest + fn build_grpc_sampling_params_from_responses( + &self, + request: &ResponsesRequest, + constraint: Option<(String, String)>, + ) -> Result { + // Used by Harmony models only. Regular models use Chat API path. + // Constraints come from Harmony preparation stage (structural_tag) or tool handling. + + let max_tokens = request.max_output_tokens.map(|v| v as i32); + + Ok(proto::SamplingParams { + temperature: request.temperature.unwrap_or(1.0), + top_p: request.top_p.unwrap_or(1.0), + top_k: -1, // ResponsesRequest doesn't expose top_k + min_p: 0.0, // ResponsesRequest doesn't expose min_p + frequency_penalty: 0.0, // ResponsesRequest doesn't expose frequency_penalty + presence_penalty: 0.0, // ResponsesRequest doesn't expose presence_penalty + repetition_penalty: 1.0, // ResponsesRequest doesn't expose repetition_penalty + max_tokens, + stop: vec![], // No stop sequences in Responses API + stop_token_ids: vec![], // Handled by Harmony stop tokens + skip_special_tokens: false, // Keep special tokens for Harmony + spaces_between_special_tokens: true, + ignore_eos: false, + n: 1, // Responses API doesn't support n>1 + constraint: self.build_constraint_for_responses(constraint)?, + ..Default::default() + }) + } + + /// Build constraint for Responses API + /// + /// Handles constraints from Harmony preparation stage (structural_tag for Harmony models, + /// structured output via text field, or tool call constraints). + /// + /// Note: Regular gRPC models use Chat API path with response_format, not this function. + fn build_constraint_for_responses( + &self, + constraint: Option<(String, String)>, + ) -> Result, String> { + if let Some((constraint_type, constraint_value)) = constraint { + let parsed_constraint = match constraint_type.as_str() { + "structural_tag" => { + proto::sampling_params::Constraint::StructuralTag(constraint_value) + } + "json_schema" => proto::sampling_params::Constraint::JsonSchema(constraint_value), + "grammar" | "ebnf" => proto::sampling_params::Constraint::Grammar(constraint_value), + "regex" => proto::sampling_params::Constraint::Regex(constraint_value), + _ => return Err(format!("Unknown constraint type: {}", constraint_type)), + }; + Ok(Some(parsed_constraint)) + } else { + Ok(None) + } + } + + fn build_single_constraint_from_plain( + params: &GenerateSamplingParams, + ) -> Result, String> { + let mut constraints = Vec::new(); + if let Some(json_schema) = ¶ms.json_schema { + constraints.push(proto::sampling_params::Constraint::JsonSchema( + json_schema.clone(), + )); + } + if let Some(regex) = ¶ms.regex { + constraints.push(proto::sampling_params::Constraint::Regex(regex.clone())); + } + if let Some(ebnf) = ¶ms.ebnf { + constraints.push(proto::sampling_params::Constraint::Grammar(ebnf.clone())); + } + + match constraints.len() { + 0 => Ok(None), + 1 => Ok(constraints.pop()), + _ => Err("Multiple structured constraints are not allowed".to_string()), + } + } + + fn build_sampling_params_from_plain( + params: Option<&GenerateSamplingParams>, + ) -> Result { + let mut sampling = proto::SamplingParams { + temperature: 1.0, + top_p: 1.0, + top_k: -1, + repetition_penalty: 1.0, + n: 1, + skip_special_tokens: true, + spaces_between_special_tokens: true, + ..Default::default() + }; + + let Some(p) = params else { + return Ok(sampling); + }; + + // Simple field mappings using a macro + macro_rules! map_field { + ($field:ident) => { + if let Some(val) = p.$field { + sampling.$field = val; + } + }; + } + + map_field!(temperature); + map_field!(top_p); + map_field!(top_k); + map_field!(frequency_penalty); + map_field!(presence_penalty); + map_field!(repetition_penalty); + map_field!(min_p); + map_field!(ignore_eos); + map_field!(skip_special_tokens); + // Note: no_stop_trim not supported in vLLM + + // Handle stop sequences + if let Some(stop) = &p.stop { + match stop { + StringOrArray::String(s) => sampling.stop.push(s.clone()), + StringOrArray::Array(arr) => sampling.stop.extend(arr.clone()), + } + } + + // Handle stop token IDs + if let Some(stop_token_ids) = &p.stop_token_ids { + sampling.stop_token_ids = stop_token_ids.clone(); + } + + // Handle max_tokens with conversion (read from internal max_new_tokens) + if let Some(max_new_tokens) = p.max_new_tokens { + sampling.max_tokens = Some( + i32::try_from(max_new_tokens) + .map_err(|_| "max_tokens must fit into a 32-bit signed integer".to_string())?, + ); + } + + // Handle min_tokens with conversion (read from internal min_new_tokens) + if let Some(min_new_tokens) = p.min_new_tokens { + sampling.min_tokens = i32::try_from(min_new_tokens) + .map_err(|_| "min_tokens must fit into a 32-bit signed integer".to_string())?; + } + + // Handle n with conversion + if let Some(n) = p.n { + sampling.n = i32::try_from(n) + .map_err(|_| "n must fit into a 32-bit signed integer".to_string())?; + } + + // Handle constraints (exactly one allowed) + sampling.constraint = Self::build_single_constraint_from_plain(p)?; + + Ok(sampling) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_proto_types_compilation() { + let _health_req = proto::HealthCheckRequest {}; + // HealthCheckRequest is now empty - no fields to test + } + + #[test] + fn test_generate_request_construction() { + let sampling_params = proto::SamplingParams { + temperature: 0.7, + max_tokens: Some(128), + top_p: 0.9, + top_k: 50, + stop: vec!["".to_string()], + ..Default::default() + }; + + let gen_req = proto::GenerateRequest { + request_id: "test-req-123".to_string(), + tokenized: Some(proto::TokenizedInput { + original_text: "Hello world".to_string(), + input_ids: vec![9906, 1917], // Mock token IDs for "Hello world" + }), + sampling_params: Some(sampling_params), + stream: false, + }; + + assert_eq!(gen_req.request_id, "test-req-123"); + if let Some(ref tokenized) = &gen_req.tokenized { + assert_eq!(tokenized.original_text, "Hello world"); + } + // vLLM: logprobs are in SamplingParams, not GenerateRequest + + let params = gen_req.sampling_params.unwrap(); + assert_eq!(params.temperature, 0.7); + assert_eq!(params.max_tokens, Some(128)); + assert_eq!(params.stop, vec![""]); + } + + #[test] + fn test_health_check_request() { + let _health_req = proto::HealthCheckRequest {}; + // HealthCheckRequest is now empty - server generates its own test internally + } + + #[test] + fn test_abort_request_construction() { + let abort_req = proto::AbortRequest { + request_id: "req-456".to_string(), + reason: "User canceled".to_string(), + }; + assert_eq!(abort_req.request_id, "req-456"); + assert_eq!(abort_req.reason, "User canceled"); + } + + #[test] + fn test_sampling_params_defaults() { + let params = proto::SamplingParams::default(); + // Numeric fields have proto defaults (0) + assert_eq!(params.temperature, 0.0); + assert_eq!(params.top_p, 0.0); + assert_eq!(params.top_k, 0); + assert_eq!(params.repetition_penalty, 0.0); + assert_eq!(params.n, 0); + // Bool fields have proto defaults (false) + assert!(!params.skip_special_tokens); + assert!(!params.spaces_between_special_tokens); + assert!(!params.ignore_eos); + assert!(!params.include_stop_str_in_output); + // Optional int fields should be None + assert_eq!(params.max_tokens, None); + assert_eq!(params.logprobs, None); + // Other non-optional fields + assert_eq!(params.min_p, 0.0); + assert_eq!(params.frequency_penalty, 0.0); + assert_eq!(params.presence_penalty, 0.0); + assert!(params.stop.is_empty()); + } + + // TODO: MultimodalInputs not in vLLM proto - skip test + // vLLM handles multimodal inputs differently than SGLang + + // TODO: SessionParams not in current proto - skip test + + #[test] + fn test_embed_request() { + let embed_req = proto::EmbedRequest { + request_id: "embed-req-202".to_string(), + tokenized: Some(proto::TokenizedInput { + original_text: "This is a test sentence for embedding".to_string(), + input_ids: vec![2028, 374, 264, 1296, 11914, 369, 28537], // Mock token IDs + }), + }; + + assert_eq!(embed_req.request_id, "embed-req-202"); + if let Some(ref tokenized) = &embed_req.tokenized { + assert_eq!( + tokenized.original_text, + "This is a test sentence for embedding" + ); + } + // vLLM: no data_parallel_rank or log_metrics in EmbedRequest + } + + #[tokio::test] + async fn test_client_connect_invalid_endpoint() { + let result = VllmEngineClient::connect("invalid://endpoint").await; + assert!(result.is_err()); + } + + #[test] + fn test_tokenized_input() { + let tokenized = proto::TokenizedInput { + original_text: "Hello world".to_string(), + input_ids: vec![1, 15043, 1917, 2], + }; + + assert_eq!(tokenized.original_text, "Hello world"); + assert_eq!(tokenized.input_ids, vec![1, 15043, 1917, 2]); + } + + #[test] + fn test_generate_stream_chunk() { + let chunk = proto::GenerateStreamChunk { + token_ids: vec![1234, 5678], + prompt_tokens: 5, + completion_tokens: 2, + cached_tokens: 3, + }; + + assert_eq!(chunk.token_ids, vec![1234, 5678]); + assert_eq!(chunk.prompt_tokens, 5); + assert_eq!(chunk.completion_tokens, 2); + assert_eq!(chunk.cached_tokens, 3); + } + + // TODO: ModelInfo not in current proto - skip test +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/lib.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/lib.rs new file mode 100644 index 00000000..c2206458 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/lib.rs @@ -0,0 +1,19 @@ +pub mod app_context; +pub mod config; +pub mod logging; + +pub mod core; +pub mod data_connector; +pub mod grpc_client; +pub mod mcp; +pub mod metrics; +pub mod middleware; +pub mod policies; +pub mod protocols; +pub mod reasoning_parser; +pub mod routers; +pub mod server; +pub mod service_discovery; +pub mod tokenizer; +pub mod tool_parser; +pub mod version; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/logging.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/logging.rs new file mode 100644 index 00000000..67414927 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/logging.rs @@ -0,0 +1,132 @@ +use std::path::PathBuf; + +use tracing::Level; +use tracing_appender::{ + non_blocking::WorkerGuard, + rolling::{RollingFileAppender, Rotation}, +}; +use tracing_log::LogTracer; +use tracing_subscriber::{ + fmt::time::ChronoUtc, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer, +}; + +#[derive(Debug, Clone)] +pub struct LoggingConfig { + pub level: Level, + pub json_format: bool, + pub log_dir: Option, + pub colorize: bool, + pub log_file_name: String, + pub log_targets: Option>, +} + +impl Default for LoggingConfig { + fn default() -> Self { + Self { + level: Level::INFO, + json_format: false, + log_dir: None, + colorize: true, + log_file_name: "sgl-router".to_string(), + log_targets: Some(vec!["sgl_model_gateway".to_string()]), + } + } +} + +#[allow(dead_code)] +pub struct LogGuard { + _file_guard: Option, +} + +pub fn init_logging(config: LoggingConfig) -> LogGuard { + let _ = LogTracer::init(); + + let level_filter = match config.level { + Level::TRACE => "trace", + Level::DEBUG => "debug", + Level::INFO => "info", + Level::WARN => "warn", + Level::ERROR => "error", + }; + + let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| { + let filter_string = if let Some(targets) = &config.log_targets { + targets + .iter() + .enumerate() + .map(|(i, target)| { + if i > 0 { + format!(",{}={}", target, level_filter) + } else { + format!("{}={}", target, level_filter) + } + }) + .collect::() + } else { + format!("sgl_model_gateway={}", level_filter) + }; + + EnvFilter::new(filter_string) + }); + + let mut layers = Vec::new(); + + let time_format = "%Y-%m-%d %H:%M:%S".to_string(); + + let stdout_layer = tracing_subscriber::fmt::layer() + .with_ansi(config.colorize) + .with_file(true) + .with_line_number(true) + .with_timer(ChronoUtc::new(time_format.clone())); + + let stdout_layer = if config.json_format { + stdout_layer.json().flatten_event(true).boxed() + } else { + stdout_layer.boxed() + }; + + layers.push(stdout_layer); + + let mut file_guard = None; + + if let Some(log_dir) = &config.log_dir { + let file_name = config.log_file_name.clone(); + let log_dir = PathBuf::from(log_dir); + + if !log_dir.exists() { + if let Err(e) = std::fs::create_dir_all(&log_dir) { + eprintln!("Failed to create log directory: {}", e); + return LogGuard { _file_guard: None }; + } + } + + let file_appender = RollingFileAppender::new(Rotation::DAILY, log_dir, file_name); + + let (non_blocking, guard) = tracing_appender::non_blocking(file_appender); + file_guard = Some(guard); + + let file_layer = tracing_subscriber::fmt::layer() + .with_ansi(false) + .with_file(true) + .with_line_number(true) + .with_timer(ChronoUtc::new(time_format)) + .with_writer(non_blocking); + + let file_layer = if config.json_format { + file_layer.json().flatten_event(true).boxed() + } else { + file_layer.boxed() + }; + + layers.push(file_layer); + } + + let _ = tracing_subscriber::registry() + .with(env_filter) + .with(layers) + .try_init(); + + LogGuard { + _file_guard: file_guard, + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/main.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/main.rs new file mode 100644 index 00000000..87b39aa1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/main.rs @@ -0,0 +1,769 @@ +use std::collections::HashMap; + +use clap::{ArgAction, Parser, Subcommand, ValueEnum}; +use sgl_model_gateway::{ + config::{ + CircuitBreakerConfig, ConfigError, ConfigResult, DiscoveryConfig, HealthCheckConfig, + HistoryBackend, MetricsConfig, OracleConfig, PolicyConfig, PostgresConfig, RetryConfig, + RouterConfig, RoutingMode, TokenizerCacheConfig, + }, + core::ConnectionMode, + metrics::PrometheusConfig, + server::{self, ServerConfig}, + service_discovery::ServiceDiscoveryConfig, + version, +}; +fn parse_prefill_args() -> Vec<(String, Option)> { + let args: Vec = std::env::args().collect(); + let mut prefill_entries = Vec::new(); + let mut i = 0; + + while i < args.len() { + if args[i] == "--prefill" && i + 1 < args.len() { + let url = args[i + 1].clone(); + let bootstrap_port = if i + 2 < args.len() && !args[i + 2].starts_with("--") { + if let Ok(port) = args[i + 2].parse::() { + i += 1; + Some(port) + } else if args[i + 2].to_lowercase() == "none" { + i += 1; + None + } else { + None + } + } else { + None + }; + prefill_entries.push((url, bootstrap_port)); + i += 2; + } else { + i += 1; + } + } + + prefill_entries +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq, ValueEnum)] +pub enum Backend { + #[value(name = "sglang")] + Sglang, + #[value(name = "vllm")] + Vllm, + #[value(name = "trtllm")] + Trtllm, + #[value(name = "openai")] + Openai, + #[value(name = "anthropic")] + Anthropic, +} + +impl std::fmt::Display for Backend { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + Backend::Sglang => "sglang", + Backend::Vllm => "vllm", + Backend::Trtllm => "trtllm", + Backend::Openai => "openai", + Backend::Anthropic => "anthropic", + }; + write!(f, "{}", s) + } +} + +#[derive(Parser, Debug)] +#[command(name = "sglang-router", alias = "smg", alias = "amg")] +#[command(about = "SGLang Model Gateway - High-performance inference gateway")] +#[command(args_conflicts_with_subcommands = true)] +#[command(long_about = r#" +SGLang Model Gateway - Rust-based inference gateway + +Usage: + smg launch [OPTIONS] Launch router (short command) + amg launch [OPTIONS] Launch router (alternative) + sglang-router [OPTIONS] Launch router (full name) + +Examples: + # Regular mode + smg launch --worker-urls http://worker1:8000 http://worker2:8000 + + # PD disaggregated mode + smg launch --pd-disaggregation \ + --prefill http://127.0.0.1:30001 9001 \ + --prefill http://127.0.0.2:30002 9002 \ + --decode http://127.0.0.3:30003 \ + --decode http://127.0.0.4:30004 \ + --policy cache_aware + + # With different policies + smg launch --pd-disaggregation \ + --prefill http://127.0.0.1:30001 9001 \ + --prefill http://127.0.0.2:30002 \ + --decode http://127.0.0.3:30003 \ + --decode http://127.0.0.4:30004 \ + --prefill-policy cache_aware --decode-policy power_of_two + +"#)] +struct Cli { + #[command(subcommand)] + command: Option, + + #[command(flatten)] + router_args: CliArgs, +} + +#[derive(Subcommand, Debug)] +enum Commands { + /// Launch the router (same as running without subcommand) + #[command(visible_alias = "start")] + Launch { + #[command(flatten)] + args: CliArgs, + }, +} + +#[derive(Parser, Debug)] +struct CliArgs { + #[arg(long, default_value = "0.0.0.0")] + host: String, + + #[arg(long, default_value_t = 30000)] + port: u16, + + #[arg(long, num_args = 0..)] + worker_urls: Vec, + + #[arg(long, default_value = "cache_aware", value_parser = ["random", "round_robin", "cache_aware", "power_of_two"])] + policy: String, + + #[arg(long, default_value_t = false)] + pd_disaggregation: bool, + + #[arg(long, action = ArgAction::Append)] + decode: Vec, + + #[arg(long, value_parser = ["random", "round_robin", "cache_aware", "power_of_two"])] + prefill_policy: Option, + + #[arg(long, value_parser = ["random", "round_robin", "cache_aware", "power_of_two"])] + decode_policy: Option, + + #[arg(long, default_value_t = 1800)] + worker_startup_timeout_secs: u64, + + #[arg(long, default_value_t = 30)] + worker_startup_check_interval: u64, + + #[arg(long, default_value_t = 0.3)] + cache_threshold: f32, + + #[arg(long, default_value_t = 64)] + balance_abs_threshold: usize, + + #[arg(long, default_value_t = 1.5)] + balance_rel_threshold: f32, + + #[arg(long, default_value_t = 120)] + eviction_interval: u64, + + #[arg(long, default_value_t = 67108864)] + max_tree_size: usize, + + #[arg(long, default_value_t = 536870912)] + max_payload_size: usize, + + #[arg(long, default_value_t = false)] + dp_aware: bool, + + #[arg(long)] + api_key: Option, + + #[arg(long, value_enum, default_value_t = Backend::Sglang, alias = "runtime")] + backend: Backend, + + #[arg(long)] + log_dir: Option, + + #[arg(long, default_value = "info", value_parser = ["debug", "info", "warn", "error"])] + log_level: String, + + #[arg(long, default_value_t = false)] + service_discovery: bool, + + #[arg(long, num_args = 0..)] + selector: Vec, + + #[arg(long, default_value_t = 80)] + service_discovery_port: u16, + + #[arg(long)] + service_discovery_namespace: Option, + + #[arg(long, num_args = 0..)] + prefill_selector: Vec, + + #[arg(long, num_args = 0..)] + decode_selector: Vec, + + #[arg(long, default_value_t = 29000)] + prometheus_port: u16, + + #[arg(long, default_value = "0.0.0.0")] + prometheus_host: String, + + #[arg(long, num_args = 0..)] + request_id_headers: Vec, + + #[arg(long, default_value_t = 1800)] + request_timeout_secs: u64, + + #[arg(long, default_value_t = -1)] + max_concurrent_requests: i32, + + #[arg(long, default_value_t = 100)] + queue_size: usize, + + #[arg(long, default_value_t = 60)] + queue_timeout_secs: u64, + + #[arg(long)] + rate_limit_tokens_per_second: Option, + + #[arg(long, num_args = 0..)] + cors_allowed_origins: Vec, + + #[arg(long, default_value_t = 5)] + retry_max_retries: u32, + + #[arg(long, default_value_t = 50)] + retry_initial_backoff_ms: u64, + + #[arg(long, default_value_t = 30000)] + retry_max_backoff_ms: u64, + + #[arg(long, default_value_t = 1.5)] + retry_backoff_multiplier: f32, + + #[arg(long, default_value_t = 0.2)] + retry_jitter_factor: f32, + + #[arg(long, default_value_t = false)] + disable_retries: bool, + + #[arg(long, default_value_t = 10)] + cb_failure_threshold: u32, + + #[arg(long, default_value_t = 3)] + cb_success_threshold: u32, + + #[arg(long, default_value_t = 60)] + cb_timeout_duration_secs: u64, + + #[arg(long, default_value_t = 120)] + cb_window_duration_secs: u64, + + #[arg(long, default_value_t = false)] + disable_circuit_breaker: bool, + + #[arg(long, default_value_t = 3)] + health_failure_threshold: u32, + + #[arg(long, default_value_t = 2)] + health_success_threshold: u32, + + #[arg(long, default_value_t = 5)] + health_check_timeout_secs: u64, + + #[arg(long, default_value_t = 60)] + health_check_interval_secs: u64, + + #[arg(long, default_value = "/health")] + health_check_endpoint: String, + + #[arg(long, default_value_t = false)] + enable_igw: bool, + + #[arg(long)] + model_path: Option, + + #[arg(long)] + tokenizer_path: Option, + + #[arg(long)] + chat_template: Option, + + #[arg(long, default_value_t = false)] + tokenizer_cache_enable_l0: bool, + + #[arg(long, default_value_t = 10000)] + tokenizer_cache_l0_max_entries: usize, + + #[arg(long, default_value_t = false)] + tokenizer_cache_enable_l1: bool, + + #[arg(long, default_value_t = 52428800)] + tokenizer_cache_l1_max_memory: usize, + + #[arg(long, default_value = "memory", value_parser = ["memory", "none", "oracle","postgres"])] + history_backend: String, + + #[arg(long, env = "ATP_WALLET_PATH")] + oracle_wallet_path: Option, + + #[arg(long, env = "ATP_TNS_ALIAS")] + oracle_tns_alias: Option, + + #[arg(long, env = "ATP_DSN")] + oracle_dsn: Option, + + #[arg(long, env = "ATP_USER")] + oracle_user: Option, + + #[arg(long, env = "ATP_PASSWORD")] + oracle_password: Option, + + #[arg(long, env = "ATP_POOL_MIN")] + oracle_pool_min: Option, + + #[arg(long, env = "ATP_POOL_MAX")] + oracle_pool_max: Option, + + #[arg(long, env = "ATP_POOL_TIMEOUT_SECS")] + oracle_pool_timeout_secs: Option, + + #[arg(long)] + postgres_db_url: Option, + + #[arg(long)] + postgres_pool_max_size: Option, + + #[arg(long)] + reasoning_parser: Option, + + #[arg(long)] + tool_call_parser: Option, + + #[arg(long)] + mcp_config_path: Option, +} + +enum OracleConnectSource { + Dsn { descriptor: String }, + Wallet { path: String, alias: String }, +} + +impl CliArgs { + fn determine_connection_mode(worker_urls: &[String]) -> ConnectionMode { + for url in worker_urls { + if url.starts_with("grpc://") || url.starts_with("grpcs://") { + return ConnectionMode::Grpc { port: None }; + } + } + ConnectionMode::Http + } + + fn parse_selector(selector_list: &[String]) -> HashMap { + let mut map = HashMap::new(); + for item in selector_list { + if let Some(eq_pos) = item.find('=') { + let key = item[..eq_pos].to_string(); + let value = item[eq_pos + 1..].to_string(); + map.insert(key, value); + } + } + map + } + + fn parse_policy(&self, policy_str: &str) -> PolicyConfig { + match policy_str { + "random" => PolicyConfig::Random, + "round_robin" => PolicyConfig::RoundRobin, + "cache_aware" => PolicyConfig::CacheAware { + cache_threshold: self.cache_threshold, + balance_abs_threshold: self.balance_abs_threshold, + balance_rel_threshold: self.balance_rel_threshold, + eviction_interval_secs: self.eviction_interval, + max_tree_size: self.max_tree_size, + }, + "power_of_two" => PolicyConfig::PowerOfTwo { + load_check_interval_secs: 5, + }, + _ => PolicyConfig::RoundRobin, + } + } + + fn resolve_oracle_connect_details(&self) -> ConfigResult { + if let Some(dsn) = self.oracle_dsn.clone() { + return Ok(OracleConnectSource::Dsn { descriptor: dsn }); + } + + let wallet_path = self + .oracle_wallet_path + .clone() + .ok_or(ConfigError::MissingRequired { + field: "oracle_wallet_path or ATP_WALLET_PATH".to_string(), + })?; + + let tns_alias = self + .oracle_tns_alias + .clone() + .ok_or(ConfigError::MissingRequired { + field: "oracle_tns_alias or ATP_TNS_ALIAS".to_string(), + })?; + + Ok(OracleConnectSource::Wallet { + path: wallet_path, + alias: tns_alias, + }) + } + + fn build_oracle_config(&self) -> ConfigResult { + let (wallet_path, connect_descriptor) = match self.resolve_oracle_connect_details()? { + OracleConnectSource::Dsn { descriptor } => (None, descriptor), + OracleConnectSource::Wallet { path, alias } => (Some(path), alias), + }; + let username = self + .oracle_user + .clone() + .ok_or(ConfigError::MissingRequired { + field: "oracle_user or ATP_USER".to_string(), + })?; + let password = self + .oracle_password + .clone() + .ok_or(ConfigError::MissingRequired { + field: "oracle_password or ATP_PASSWORD".to_string(), + })?; + + let pool_min = self + .oracle_pool_min + .unwrap_or_else(OracleConfig::default_pool_min); + let pool_max = self + .oracle_pool_max + .unwrap_or_else(OracleConfig::default_pool_max); + + if pool_min == 0 { + return Err(ConfigError::InvalidValue { + field: "oracle_pool_min".to_string(), + value: pool_min.to_string(), + reason: "pool minimum must be at least 1".to_string(), + }); + } + + if pool_max < pool_min { + return Err(ConfigError::InvalidValue { + field: "oracle_pool_max".to_string(), + value: pool_max.to_string(), + reason: "pool maximum must be greater than or equal to minimum".to_string(), + }); + } + + let pool_timeout_secs = self + .oracle_pool_timeout_secs + .unwrap_or_else(OracleConfig::default_pool_timeout_secs); + + Ok(OracleConfig { + wallet_path, + connect_descriptor, + username, + password, + pool_min, + pool_max, + pool_timeout_secs, + }) + } + + fn build_postgres_config(&self) -> ConfigResult { + let db_url = self.postgres_db_url.clone().unwrap_or_default(); + let pool_max = self + .postgres_pool_max_size + .unwrap_or_else(PostgresConfig::default_pool_max); + let pcf = PostgresConfig { db_url, pool_max }; + pcf.validate().map_err(|e| ConfigError::ValidationFailed { + reason: e.to_string(), + })?; + Ok(pcf) + } + + fn to_router_config( + &self, + prefill_urls: Vec<(String, Option)>, + ) -> ConfigResult { + let mode = if self.enable_igw { + RoutingMode::Regular { + worker_urls: vec![], + } + } else if matches!(self.backend, Backend::Openai) { + RoutingMode::OpenAI { + worker_urls: self.worker_urls.clone(), + } + } else if self.pd_disaggregation { + let decode_urls = self.decode.clone(); + + // Allow empty URLs to support dynamic worker addition + RoutingMode::PrefillDecode { + prefill_urls, + decode_urls, + prefill_policy: self.prefill_policy.as_ref().map(|p| self.parse_policy(p)), + decode_policy: self.decode_policy.as_ref().map(|p| self.parse_policy(p)), + } + } else { + // Allow empty URLs to support dynamic worker addition + RoutingMode::Regular { + worker_urls: self.worker_urls.clone(), + } + }; + + let policy = self.parse_policy(&self.policy); + + let discovery = if self.service_discovery { + Some(DiscoveryConfig { + enabled: true, + namespace: self.service_discovery_namespace.clone(), + port: self.service_discovery_port, + check_interval_secs: 60, + selector: Self::parse_selector(&self.selector), + prefill_selector: Self::parse_selector(&self.prefill_selector), + decode_selector: Self::parse_selector(&self.decode_selector), + bootstrap_port_annotation: "sglang.ai/bootstrap-port".to_string(), + }) + } else { + None + }; + + let metrics = Some(MetricsConfig { + port: self.prometheus_port, + host: self.prometheus_host.clone(), + }); + + let mut all_urls = Vec::new(); + match &mode { + RoutingMode::Regular { worker_urls } => { + all_urls.extend(worker_urls.clone()); + } + RoutingMode::PrefillDecode { + prefill_urls, + decode_urls, + .. + } => { + for (url, _) in prefill_urls { + all_urls.push(url.clone()); + } + all_urls.extend(decode_urls.clone()); + } + RoutingMode::OpenAI { .. } => {} + } + let connection_mode = match &mode { + RoutingMode::OpenAI { .. } => ConnectionMode::Http, + _ => Self::determine_connection_mode(&all_urls), + }; + + let history_backend = match self.history_backend.as_str() { + "none" => HistoryBackend::None, + "oracle" => HistoryBackend::Oracle, + "postgres" => HistoryBackend::Postgres, + _ => HistoryBackend::Memory, + }; + + let oracle = if history_backend == HistoryBackend::Oracle { + Some(self.build_oracle_config()?) + } else { + None + }; + let postgres = if history_backend == HistoryBackend::Postgres { + Some(self.build_postgres_config()?) + } else { + None + }; + + let builder = RouterConfig::builder() + .mode(mode) + .policy(policy) + .connection_mode(connection_mode) + .host(&self.host) + .port(self.port) + .max_payload_size(self.max_payload_size) + .request_timeout_secs(self.request_timeout_secs) + .worker_startup_timeout_secs(self.worker_startup_timeout_secs) + .worker_startup_check_interval_secs(self.worker_startup_check_interval) + .max_concurrent_requests(self.max_concurrent_requests) + .queue_size(self.queue_size) + .queue_timeout_secs(self.queue_timeout_secs) + .cors_allowed_origins(self.cors_allowed_origins.clone()) + .retry_config(RetryConfig { + max_retries: self.retry_max_retries, + initial_backoff_ms: self.retry_initial_backoff_ms, + max_backoff_ms: self.retry_max_backoff_ms, + backoff_multiplier: self.retry_backoff_multiplier, + jitter_factor: self.retry_jitter_factor, + }) + .circuit_breaker_config(CircuitBreakerConfig { + failure_threshold: self.cb_failure_threshold, + success_threshold: self.cb_success_threshold, + timeout_duration_secs: self.cb_timeout_duration_secs, + window_duration_secs: self.cb_window_duration_secs, + }) + .health_check_config(HealthCheckConfig { + failure_threshold: self.health_failure_threshold, + success_threshold: self.health_success_threshold, + timeout_secs: self.health_check_timeout_secs, + check_interval_secs: self.health_check_interval_secs, + endpoint: self.health_check_endpoint.clone(), + }) + .tokenizer_cache(TokenizerCacheConfig { + enable_l0: self.tokenizer_cache_enable_l0, + l0_max_entries: self.tokenizer_cache_l0_max_entries, + enable_l1: self.tokenizer_cache_enable_l1, + l1_max_memory: self.tokenizer_cache_l1_max_memory, + }) + .history_backend(history_backend) + .log_level(&self.log_level) + .maybe_api_key(self.api_key.as_ref()) + .maybe_discovery(discovery) + .maybe_metrics(metrics) + .maybe_log_dir(self.log_dir.as_ref()) + .maybe_request_id_headers( + (!self.request_id_headers.is_empty()).then(|| self.request_id_headers.clone()), + ) + .maybe_rate_limit_tokens_per_second(self.rate_limit_tokens_per_second) + .maybe_model_path(self.model_path.as_ref()) + .maybe_tokenizer_path(self.tokenizer_path.as_ref()) + .maybe_chat_template(self.chat_template.as_ref()) + .maybe_oracle(oracle) + .maybe_postgres(postgres) + .maybe_reasoning_parser(self.reasoning_parser.as_ref()) + .maybe_tool_call_parser(self.tool_call_parser.as_ref()) + .maybe_mcp_config_path(self.mcp_config_path.as_ref()) + .dp_aware(self.dp_aware) + .retries(!self.disable_retries) + .circuit_breaker(!self.disable_circuit_breaker) + .igw(self.enable_igw); + + builder.build() + } + + fn to_server_config(&self, router_config: RouterConfig) -> ServerConfig { + let service_discovery_config = if self.service_discovery { + Some(ServiceDiscoveryConfig { + enabled: true, + selector: Self::parse_selector(&self.selector), + check_interval: std::time::Duration::from_secs(60), + port: self.service_discovery_port, + namespace: self.service_discovery_namespace.clone(), + pd_mode: self.pd_disaggregation, + prefill_selector: Self::parse_selector(&self.prefill_selector), + decode_selector: Self::parse_selector(&self.decode_selector), + bootstrap_port_annotation: "sglang.ai/bootstrap-port".to_string(), + }) + } else { + None + }; + + let prometheus_config = Some(PrometheusConfig { + port: self.prometheus_port, + host: self.prometheus_host.clone(), + }); + + ServerConfig { + host: self.host.clone(), + port: self.port, + router_config, + max_payload_size: self.max_payload_size, + log_dir: self.log_dir.clone(), + log_level: Some(self.log_level.clone()), + service_discovery_config, + prometheus_config, + request_timeout_secs: self.request_timeout_secs, + request_id_headers: if self.request_id_headers.is_empty() { + None + } else { + Some(self.request_id_headers.clone()) + }, + } + } +} + +fn main() -> Result<(), Box> { + // Check for version flags before parsing other args to avoid errors + let args: Vec = std::env::args().collect(); + for arg in &args { + if arg == "--version" || arg == "-V" { + println!("{}", version::get_version_string()); + return Ok(()); + } + if arg == "--version-verbose" { + println!("{}", version::get_verbose_version_string()); + return Ok(()); + } + } + + let prefill_urls = parse_prefill_args(); + + let mut filtered_args: Vec = Vec::new(); + let raw_args: Vec = std::env::args().collect(); + let mut i = 0; + + while i < raw_args.len() { + if raw_args[i] == "--prefill" && i + 1 < raw_args.len() { + i += 2; + if i < raw_args.len() + && !raw_args[i].starts_with("--") + && (raw_args[i].parse::().is_ok() || raw_args[i].to_lowercase() == "none") + { + i += 1; + } + } else { + filtered_args.push(raw_args[i].clone()); + i += 1; + } + } + + let cli = Cli::parse_from(filtered_args); + + // Handle subcommands or use direct args + let cli_args = match cli.command { + Some(Commands::Launch { args }) => args, + None => cli.router_args, + }; + + println!("SGLang Router starting..."); + println!("Host: {}:{}", cli_args.host, cli_args.port); + let mode_str = if cli_args.enable_igw { + "IGW (Inference Gateway)".to_string() + } else if matches!(cli_args.backend, Backend::Openai) { + "OpenAI Backend".to_string() + } else if cli_args.pd_disaggregation { + "PD Disaggregated".to_string() + } else { + format!("Regular ({})", cli_args.backend) + }; + println!("Mode: {}", mode_str); + + match cli_args.backend { + Backend::Vllm | Backend::Trtllm | Backend::Anthropic => { + println!( + "WARNING: runtime '{}' not implemented yet; falling back to regular routing. \ +Provide --worker-urls or PD flags as usual.", + cli_args.backend + ); + } + Backend::Sglang | Backend::Openai => {} + } + + if !cli_args.enable_igw { + println!("Policy: {}", cli_args.policy); + + if cli_args.pd_disaggregation && !prefill_urls.is_empty() { + println!("Prefill nodes: {:?}", prefill_urls); + println!("Decode nodes: {:?}", cli_args.decode); + } + } + + let router_config = cli_args.to_router_config(prefill_urls)?; + router_config.validate()?; + let server_config = cli_args.to_server_config(router_config); + let runtime = tokio::runtime::Runtime::new()?; + runtime.block_on(async move { server::startup(server_config).await })?; + + Ok(()) +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/config.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/config.rs new file mode 100644 index 00000000..e5d913aa --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/config.rs @@ -0,0 +1,528 @@ +//! MCP configuration types and utilities. +//! +//! Defines configuration structures for MCP servers, transports, proxies, and inventory. + +use std::{collections::HashMap, fmt}; + +pub use rmcp::model::{Prompt, RawResource, Tool}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct McpConfig { + /// Static MCP servers (loaded at startup) + pub servers: Vec, + + /// Connection pool settings + #[serde(default)] + pub pool: McpPoolConfig, + + /// Global MCP proxy configuration (default for all servers) + /// Can be overridden per-server + #[serde(default)] + pub proxy: Option, + + /// Pre-warm these connections at startup + #[serde(default)] + pub warmup: Vec, + + /// Tool inventory refresh settings + #[serde(default)] + pub inventory: InventoryConfig, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct McpServerConfig { + pub name: String, + #[serde(flatten)] + pub transport: McpTransport, + + /// Per-server proxy override (overrides global proxy) + /// Set to `null` in YAML to force direct connection (no proxy) + #[serde(default)] + pub proxy: Option, + + /// Whether this server is required for router startup + /// - true: Router startup fails if this server cannot be reached + /// - false: Log warning but continue (default) + #[serde(default)] + pub required: bool, +} + +#[derive(Clone, Deserialize, Serialize)] +#[serde(tag = "protocol", rename_all = "lowercase")] +pub enum McpTransport { + Stdio { + command: String, + #[serde(default)] + args: Vec, + #[serde(default)] + envs: HashMap, + }, + Sse { + url: String, + #[serde(skip_serializing_if = "Option::is_none")] + token: Option, + }, + Streamable { + url: String, + #[serde(skip_serializing_if = "Option::is_none")] + token: Option, + }, +} + +impl fmt::Debug for McpTransport { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + McpTransport::Stdio { + command, + args, + envs, + } => f + .debug_struct("Stdio") + .field("command", command) + .field("args", args) + .field("envs", envs) + .finish(), + McpTransport::Sse { url, token } => f + .debug_struct("Sse") + .field("url", url) + .field("token", &token.as_ref().map(|_| "****")) + .finish(), + McpTransport::Streamable { url, token } => f + .debug_struct("Streamable") + .field("url", url) + .field("token", &token.as_ref().map(|_| "****")) + .finish(), + } + } +} + +/// MCP-specific proxy configuration (does NOT affect LLM API traffic) +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct McpProxyConfig { + /// HTTP proxy URL (e.g., "http://proxy.internal:8080") + pub http: Option, + + /// HTTPS proxy URL + pub https: Option, + + /// Comma-separated hosts to exclude from proxying + /// Example: "localhost,127.0.0.1,*.internal,10.*" + pub no_proxy: Option, + + /// Custom proxy authentication (if needed) + #[serde(skip_serializing_if = "Option::is_none")] + pub username: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub password: Option, +} + +/// Connection pool configuration +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct McpPoolConfig { + /// Maximum cached connections per server URL + #[serde(default = "default_max_connections")] + pub max_connections: usize, + + /// Idle timeout before closing connection (seconds) + #[serde(default = "default_idle_timeout")] + pub idle_timeout: u64, +} + +/// Tool inventory refresh configuration +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct InventoryConfig { + /// Enable automatic tool inventory refresh + #[serde(default = "default_true")] + pub enable_refresh: bool, + + /// Tool cache TTL (seconds) - how long tools are considered fresh + #[serde(default = "default_tool_ttl")] + pub tool_ttl: u64, + + /// Background refresh interval (seconds) - proactive refresh + #[serde(default = "default_refresh_interval")] + pub refresh_interval: u64, + + /// Refresh on tool call failure (try refreshing if tool not found) + #[serde(default = "default_true")] + pub refresh_on_error: bool, +} + +/// Pre-warm server connections at startup +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct WarmupServer { + /// Server URL + pub url: String, + + /// Server label/name + pub label: String, + + /// Optional authentication token + #[serde(skip_serializing_if = "Option::is_none")] + pub token: Option, +} + +// Default value functions +fn default_max_connections() -> usize { + 100 +} + +fn default_idle_timeout() -> u64 { + 300 // 5 minutes +} + +fn default_true() -> bool { + true +} + +fn default_tool_ttl() -> u64 { + 300 // 5 minutes +} + +fn default_refresh_interval() -> u64 { + 60 // 1 minute +} + +// Default implementations +impl Default for McpPoolConfig { + fn default() -> Self { + Self { + max_connections: default_max_connections(), + idle_timeout: default_idle_timeout(), + } + } +} + +impl Default for InventoryConfig { + fn default() -> Self { + Self { + enable_refresh: true, + tool_ttl: default_tool_ttl(), + refresh_interval: default_refresh_interval(), + refresh_on_error: true, + } + } +} + +impl McpProxyConfig { + /// Load proxy config from standard environment variables + pub fn from_env() -> Option { + let http = std::env::var("MCP_HTTP_PROXY") + .ok() + .or_else(|| std::env::var("HTTP_PROXY").ok()); + + let https = std::env::var("MCP_HTTPS_PROXY") + .ok() + .or_else(|| std::env::var("HTTPS_PROXY").ok()); + + let no_proxy = std::env::var("MCP_NO_PROXY") + .ok() + .or_else(|| std::env::var("NO_PROXY").ok()); + + if http.is_some() || https.is_some() { + Some(Self { + http, + https, + no_proxy, + username: None, + password: None, + }) + } else { + None + } + } +} + +impl McpConfig { + /// Load configuration from a YAML file + pub async fn from_file(path: &str) -> Result> { + let content = tokio::fs::read_to_string(path).await?; + let config: Self = serde_yaml::from_str(&content)?; + Ok(config) + } + + /// Load configuration from environment variables (optional) + pub fn from_env() -> Option { + // This could be expanded to read from env vars + // For now, return None to indicate env config not implemented + None + } + + /// Merge with environment-based proxy config + pub fn with_env_proxy(mut self) -> Self { + if self.proxy.is_none() { + self.proxy = McpProxyConfig::from_env(); + } + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_default_pool_config() { + let config = McpPoolConfig::default(); + assert_eq!(config.max_connections, 100); + assert_eq!(config.idle_timeout, 300); + } + + #[test] + fn test_default_inventory_config() { + let config = InventoryConfig::default(); + assert!(config.enable_refresh); + assert_eq!(config.tool_ttl, 300); + assert_eq!(config.refresh_interval, 60); + assert!(config.refresh_on_error); + } + + #[test] + fn test_proxy_from_env_empty() { + // Ensure no proxy env vars are set for this test + std::env::remove_var("MCP_HTTP_PROXY"); + std::env::remove_var("MCP_HTTPS_PROXY"); + std::env::remove_var("HTTP_PROXY"); + std::env::remove_var("HTTPS_PROXY"); + + let proxy = McpProxyConfig::from_env(); + assert!(proxy.is_none(), "Should return None when no env vars set"); + } + + #[test] + fn test_proxy_from_env_with_vars() { + std::env::set_var("MCP_HTTP_PROXY", "http://test-proxy:8080"); + std::env::set_var("MCP_NO_PROXY", "localhost,127.0.0.1"); + + let proxy = McpProxyConfig::from_env(); + assert!(proxy.is_some(), "Should return Some when env vars set"); + + let proxy = proxy.unwrap(); + assert_eq!(proxy.http.as_ref().unwrap(), "http://test-proxy:8080"); + assert_eq!(proxy.no_proxy.as_ref().unwrap(), "localhost,127.0.0.1"); + + // Cleanup + std::env::remove_var("MCP_HTTP_PROXY"); + std::env::remove_var("MCP_NO_PROXY"); + } + + #[tokio::test] + async fn test_yaml_minimal_config() { + let yaml = r#" +servers: + - name: "test-server" + protocol: sse + url: "http://localhost:3000/sse" +"#; + + let config: McpConfig = serde_yaml::from_str(yaml).expect("Failed to parse YAML"); + assert_eq!(config.servers.len(), 1); + assert_eq!(config.servers[0].name, "test-server"); + assert!(!config.servers[0].required); // Should default to false + assert!(config.servers[0].proxy.is_none()); // Should default to None + assert_eq!(config.pool.max_connections, 100); // Should use default + assert_eq!(config.inventory.tool_ttl, 300); // Should use default + } + + #[tokio::test] + async fn test_yaml_full_config() { + let yaml = r#" +# Global proxy configuration +proxy: + http: "http://global-proxy:8080" + https: "http://global-proxy:8080" + no_proxy: "localhost,127.0.0.1,*.internal" + +# Connection pool settings +pool: + max_connections: 50 + idle_timeout: 600 + +# Tool inventory settings +inventory: + enable_refresh: true + tool_ttl: 600 + refresh_interval: 120 + refresh_on_error: true + +# Static servers +servers: + - name: "required-server" + protocol: sse + url: "https://api.example.com/sse" + token: "secret-token" + required: true + + - name: "optional-server" + protocol: stdio + command: "mcp-server" + args: ["--port", "3000"] + required: false + proxy: + http: "http://server-specific-proxy:9090" + +# Pre-warm connections +warmup: + - url: "http://localhost:3000/sse" + label: "local-dev" +"#; + + let config: McpConfig = serde_yaml::from_str(yaml).expect("Failed to parse YAML"); + + // Check global proxy + assert!(config.proxy.is_some()); + let global_proxy = config.proxy.as_ref().unwrap(); + assert_eq!( + global_proxy.http.as_ref().unwrap(), + "http://global-proxy:8080" + ); + + // Check pool config + assert_eq!(config.pool.max_connections, 50); + assert_eq!(config.pool.idle_timeout, 600); + + // Check inventory config + assert_eq!(config.inventory.tool_ttl, 600); + assert_eq!(config.inventory.refresh_interval, 120); + + // Check servers + assert_eq!(config.servers.len(), 2); + + // Required server + assert_eq!(config.servers[0].name, "required-server"); + assert!(config.servers[0].required); + assert!(config.servers[0].proxy.is_none()); // Inherits global proxy + + // Optional server with custom proxy + assert_eq!(config.servers[1].name, "optional-server"); + assert!(!config.servers[1].required); + assert!(config.servers[1].proxy.is_some()); + assert_eq!( + config.servers[1] + .proxy + .as_ref() + .unwrap() + .http + .as_ref() + .unwrap(), + "http://server-specific-proxy:9090" + ); + + // Check warmup + assert_eq!(config.warmup.len(), 1); + assert_eq!(config.warmup[0].label, "local-dev"); + } + + #[tokio::test] + async fn test_yaml_backward_compatibility() { + // Old config format should still work + let yaml = r#" +servers: + - name: "legacy-server" + protocol: sse + url: "http://localhost:3000/sse" +"#; + + let config: McpConfig = serde_yaml::from_str(yaml).expect("Failed to parse old format"); + assert_eq!(config.servers.len(), 1); + assert_eq!(config.servers[0].name, "legacy-server"); + assert!(!config.servers[0].required); // New field defaults to false + assert!(config.servers[0].proxy.is_none()); // New field defaults to None + assert!(config.proxy.is_none()); // New field defaults to None + assert!(config.warmup.is_empty()); // New field defaults to empty + } + + #[tokio::test] + async fn test_yaml_null_proxy_override() { + // Test that explicit null in YAML sets proxy to None + let yaml = r#" +proxy: + http: "http://global-proxy:8080" + +servers: + - name: "direct-connection" + protocol: sse + url: "http://localhost:3000/sse" + proxy: null +"#; + + let config: McpConfig = serde_yaml::from_str(yaml).expect("Failed to parse YAML"); + assert!(config.proxy.is_some()); // Global proxy set + assert_eq!(config.servers.len(), 1); + assert!(config.servers[0].proxy.is_none()); // Explicitly set to null + } + + #[test] + fn test_transport_stdio() { + let yaml = r#" +name: "test" +protocol: stdio +command: "mcp-server" +args: ["--port", "3000"] +envs: + VAR1: "value1" + VAR2: "value2" +"#; + + let config: McpServerConfig = serde_yaml::from_str(yaml).expect("Failed to parse stdio"); + assert_eq!(config.name, "test"); + + match config.transport { + McpTransport::Stdio { + command, + args, + envs, + } => { + assert_eq!(command, "mcp-server"); + assert_eq!(args.len(), 2); + assert_eq!(args[0], "--port"); + assert_eq!(envs.get("VAR1").unwrap(), "value1"); + } + _ => panic!("Expected Stdio transport"), + } + } + + #[test] + fn test_transport_sse() { + let yaml = r#" +name: "test" +protocol: sse +url: "http://localhost:3000/sse" +token: "secret" +"#; + + let config: McpServerConfig = serde_yaml::from_str(yaml).expect("Failed to parse sse"); + assert_eq!(config.name, "test"); + + match config.transport { + McpTransport::Sse { url, token } => { + assert_eq!(url, "http://localhost:3000/sse"); + assert_eq!(token.unwrap(), "secret"); + } + _ => panic!("Expected Sse transport"), + } + } + + #[test] + fn test_transport_streamable() { + let yaml = r#" +name: "test" +protocol: streamable +url: "http://localhost:3000" +"#; + + let config: McpServerConfig = + serde_yaml::from_str(yaml).expect("Failed to parse streamable"); + assert_eq!(config.name, "test"); + + match config.transport { + McpTransport::Streamable { url, token } => { + assert_eq!(url, "http://localhost:3000"); + assert!(token.is_none()); + } + _ => panic!("Expected Streamable transport"), + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/connection_pool.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/connection_pool.rs new file mode 100644 index 00000000..cd2066d9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/connection_pool.rs @@ -0,0 +1,318 @@ +/// MCP Connection Pool +/// +/// This module provides connection pooling for dynamic MCP servers (per-request). +use std::sync::Arc; + +use lru::LruCache; +use parking_lot::Mutex; +use rmcp::{service::RunningService, RoleClient}; + +use crate::mcp::{ + config::{McpProxyConfig, McpServerConfig}, + error::McpResult, +}; + +/// Type alias for MCP client +type McpClient = RunningService; + +/// Type alias for eviction callback +type EvictionCallback = Arc; + +/// Cached MCP connection with metadata +#[derive(Clone)] +pub struct CachedConnection { + /// The MCP client instance + pub client: Arc, + /// Server configuration used to create this connection + pub config: McpServerConfig, +} + +impl CachedConnection { + /// Create a new cached connection + pub fn new(client: Arc, config: McpServerConfig) -> Self { + Self { client, config } + } +} + +/// Connection pool for dynamic MCP servers +/// +/// Provides thread-safe connection pooling with LRU eviction. +/// Connections are keyed by server URL and reused across requests. +pub struct McpConnectionPool { + /// LRU cache of server_url -> cached connection + connections: Arc>>, + + /// Maximum number of cached connections (LRU capacity) + max_connections: usize, + + /// Global proxy configuration (applied to all dynamic servers) + /// Can be overridden per-server via McpServerConfig.proxy + global_proxy: Option, + + /// Optional eviction callback (called when LRU evicts a connection) + /// Used to clean up tools from inventory + eviction_callback: Option, +} + +impl McpConnectionPool { + /// Default max connections for pool + const DEFAULT_MAX_CONNECTIONS: usize = 200; + + /// Create a new connection pool with default settings + /// + /// Default settings: + /// - max_connections: 200 + /// - global_proxy: Loaded from environment variables (MCP_HTTP_PROXY, etc.) + pub fn new() -> Self { + Self { + connections: Arc::new(Mutex::new(LruCache::new( + std::num::NonZeroUsize::new(Self::DEFAULT_MAX_CONNECTIONS).unwrap(), + ))), + max_connections: Self::DEFAULT_MAX_CONNECTIONS, + global_proxy: McpProxyConfig::from_env(), + eviction_callback: None, + } + } + + /// Create a new connection pool with custom capacity + pub fn with_capacity(max_connections: usize) -> Self { + Self { + connections: Arc::new(Mutex::new(LruCache::new( + std::num::NonZeroUsize::new(max_connections).unwrap(), + ))), + max_connections, + global_proxy: McpProxyConfig::from_env(), + eviction_callback: None, + } + } + + /// Create a new connection pool with full custom configuration + pub fn with_full_config(max_connections: usize, global_proxy: Option) -> Self { + Self { + connections: Arc::new(Mutex::new(LruCache::new( + std::num::NonZeroUsize::new(max_connections).unwrap(), + ))), + max_connections, + global_proxy, + eviction_callback: None, + } + } + + /// Set the eviction callback (called when LRU evicts a connection) + pub fn set_eviction_callback(&mut self, callback: F) + where + F: Fn(&str) + Send + Sync + 'static, + { + self.eviction_callback = Some(Arc::new(callback)); + } + + /// Get an existing connection or create a new one + /// + /// This method: + /// 1. Checks if a connection exists for the given URL (fast path <1ms) + /// 2. If exists, promotes it in LRU and returns it + /// 3. If not exists, creates new connection (slow path 70-650ms) + /// + /// # Arguments + /// * `server_url` - The MCP server URL (used as cache key) + /// * `server_config` - Server configuration (used to create new connection if needed) + /// * `connect_fn` - Async function to create a new client connection + /// + /// # Returns + /// Arc to the MCP client, either from cache or newly created + pub async fn get_or_create( + &self, + server_url: &str, + server_config: McpServerConfig, + connect_fn: F, + ) -> McpResult> + where + F: FnOnce(McpServerConfig, Option) -> Fut, + Fut: std::future::Future>, + { + // Fast path: Check if connection exists in LRU cache + { + let mut connections = self.connections.lock(); + if let Some(cached) = connections.get(server_url) { + // LRU get() promotes the entry + return Ok(Arc::clone(&cached.client)); + } + } + + // Slow path: Create new connection + let client = connect_fn(server_config.clone(), self.global_proxy.clone()).await?; + let client_arc = Arc::new(client); + + // Cache the new connection (LRU will automatically evict oldest if at capacity) + let cached = CachedConnection::new(Arc::clone(&client_arc), server_config); + { + let mut connections = self.connections.lock(); + if let Some((evicted_key, _evicted_conn)) = + connections.push(server_url.to_string(), cached) + { + // Call eviction callback if set + if let Some(callback) = &self.eviction_callback { + callback(&evicted_key); + } + } + } + + Ok(client_arc) + } + + /// Get current number of cached connections + pub fn len(&self) -> usize { + self.connections.lock().len() + } + + /// Check if pool is empty + pub fn is_empty(&self) -> bool { + self.connections.lock().is_empty() + } + + /// Clear all connections + pub fn clear(&self) { + self.connections.lock().clear(); + } + + /// Get connection statistics + pub fn stats(&self) -> PoolStats { + let total = self.connections.lock().len(); + + PoolStats { + total_connections: total, + capacity: self.max_connections, + } + } + + /// List all server keys in the pool + pub fn list_server_keys(&self) -> Vec { + self.connections + .lock() + .iter() + .map(|(key, _)| key.clone()) + .collect() + } + + /// Get a connection by server key without creating it + /// Promotes the entry in LRU cache if found + pub fn get(&self, server_key: &str) -> Option> { + self.connections + .lock() + .get(server_key) + .map(|cached| Arc::clone(&cached.client)) + } +} + +impl Default for McpConnectionPool { + fn default() -> Self { + Self::new() + } +} + +/// Connection pool statistics +#[derive(Debug, Clone)] +pub struct PoolStats { + pub total_connections: usize, + pub capacity: usize, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mcp::McpTransport; + + // Helper to create test server config + fn create_test_config(url: &str) -> McpServerConfig { + McpServerConfig { + name: "test_server".to_string(), + transport: McpTransport::Streamable { + url: url.to_string(), + token: None, + }, + proxy: None, + required: false, + } + } + + #[tokio::test] + async fn test_pool_creation() { + let pool = McpConnectionPool::new(); + assert_eq!(pool.len(), 0); + assert!(pool.is_empty()); + } + + #[test] + fn test_pool_stats() { + let pool = McpConnectionPool::with_capacity(10); + + let stats = pool.stats(); + assert_eq!(stats.total_connections, 0); + assert_eq!(stats.capacity, 10); + } + + #[test] + #[allow(invalid_value)] + fn test_pool_clear() { + let pool = McpConnectionPool::new(); + + // Add a connection + let config = create_test_config("http://localhost:3000"); + let client: Arc = + Arc::new(unsafe { std::mem::MaybeUninit::zeroed().assume_init() }); + let cached = CachedConnection::new(client.clone(), config); + pool.connections + .lock() + .push("http://localhost:3000".to_string(), cached); + + assert_eq!(pool.len(), 1); + + pool.clear(); + assert_eq!(pool.len(), 0); + assert!(pool.is_empty()); + + // Prevent drop of invalid Arc (would segfault) + std::mem::forget(client); + } + + #[test] + fn test_pool_with_global_proxy() { + use crate::mcp::McpProxyConfig; + + // Create proxy config + let proxy = McpProxyConfig { + http: Some("http://proxy.example.com:8080".to_string()), + https: None, + no_proxy: Some("localhost,127.0.0.1".to_string()), + username: None, + password: None, + }; + + // Create pool with proxy + let pool = McpConnectionPool::with_full_config(100, Some(proxy.clone())); + + // Verify proxy is stored + assert!(pool.global_proxy.is_some()); + let stored_proxy = pool.global_proxy.as_ref().unwrap(); + assert_eq!( + stored_proxy.http.as_ref().unwrap(), + "http://proxy.example.com:8080" + ); + assert_eq!( + stored_proxy.no_proxy.as_ref().unwrap(), + "localhost,127.0.0.1" + ); + } + + #[test] + fn test_pool_proxy_from_env() { + // Note: This test depends on environment variables + // In production, proxy is loaded from MCP_HTTP_PROXY or HTTP_PROXY env vars + let pool = McpConnectionPool::new(); + + // Pool should either have proxy from env or None + // We can't assert specific value since it depends on test environment + // Just verify it doesn't panic + assert!(pool.global_proxy.is_some() || pool.global_proxy.is_none()); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/error.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/error.rs new file mode 100644 index 00000000..d6af2d3e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/error.rs @@ -0,0 +1,50 @@ +//! MCP error types. +//! +//! Defines error variants for MCP operations including connection, tool execution, +//! and configuration errors. + +use thiserror::Error; + +pub type McpResult = Result; + +#[derive(Debug, Error)] +pub enum McpError { + #[error("Server not found: {0}")] + ServerNotFound(String), + + #[error("Tool not found: {0}")] + ToolNotFound(String), + + #[error("Transport error: {0}")] + Transport(String), + + #[error("Tool execution failed: {0}")] + ToolExecution(String), + + #[error("Connection failed: {0}")] + ConnectionFailed(String), + + #[error("Configuration error: {0}")] + Config(String), + + #[error("Authentication error: {0}")] + Auth(String), + + #[error("Resource not found: {0}")] + ResourceNotFound(String), + + #[error("Prompt not found: {0}")] + PromptNotFound(String), + + #[error("Invalid arguments: {0}")] + InvalidArguments(String), + + #[error(transparent)] + Sdk(#[from] Box), + + #[error(transparent)] + Io(#[from] std::io::Error), + + #[error(transparent)] + Http(#[from] reqwest::Error), +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/inventory.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/inventory.rs new file mode 100644 index 00000000..e03e7551 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/inventory.rs @@ -0,0 +1,447 @@ +//! MCP tool, prompt, and resource inventory. +//! +//! Thread-safe cache for MCP capabilities across all connected servers. + +use dashmap::DashMap; + +use crate::mcp::config::{Prompt, RawResource, Tool}; + +/// Cached tool with metadata +#[derive(Clone)] +pub struct CachedTool { + pub server_name: String, + pub tool: Tool, +} + +/// Cached prompt with metadata +#[derive(Clone)] +pub struct CachedPrompt { + pub server_name: String, + pub prompt: Prompt, +} + +/// Cached resource with metadata +#[derive(Clone)] +pub struct CachedResource { + pub server_name: String, + pub resource: RawResource, +} + +/// Tool inventory with periodic refresh +/// +/// Provides thread-safe caching of MCP tools, prompts, and resources. +/// Entries are refreshed periodically by background tasks. +pub struct ToolInventory { + /// Map of tool_name -> cached tool + tools: DashMap, + + /// Map of prompt_name -> cached prompt + prompts: DashMap, + + /// Map of resource_uri -> cached resource + resources: DashMap, +} + +impl ToolInventory { + /// Create a new tool inventory + pub fn new() -> Self { + Self { + tools: DashMap::new(), + prompts: DashMap::new(), + resources: DashMap::new(), + } + } +} + +impl Default for ToolInventory { + fn default() -> Self { + Self::new() + } +} + +impl ToolInventory { + // ============================================================================ + // Tool Methods + // ============================================================================ + + /// Get a tool if it exists + pub fn get_tool(&self, tool_name: &str) -> Option<(String, Tool)> { + self.tools + .get(tool_name) + .map(|entry| (entry.server_name.clone(), entry.tool.clone())) + } + + /// Check if tool exists + pub fn has_tool(&self, tool_name: &str) -> bool { + self.tools.contains_key(tool_name) + } + + /// Insert or update a tool + pub fn insert_tool(&self, tool_name: String, server_name: String, tool: Tool) { + self.tools + .insert(tool_name, CachedTool { server_name, tool }); + } + + /// Get all tools + pub fn list_tools(&self) -> Vec<(String, String, Tool)> { + self.tools + .iter() + .map(|entry| { + let (name, cached) = entry.pair(); + ( + name.clone(), + cached.server_name.clone(), + cached.tool.clone(), + ) + }) + .collect() + } + + // ============================================================================ + // Prompt Methods + // ============================================================================ + + /// Get a prompt if it exists + pub fn get_prompt(&self, prompt_name: &str) -> Option<(String, Prompt)> { + self.prompts + .get(prompt_name) + .map(|entry| (entry.server_name.clone(), entry.prompt.clone())) + } + + /// Check if prompt exists + pub fn has_prompt(&self, prompt_name: &str) -> bool { + self.prompts.contains_key(prompt_name) + } + + /// Insert or update a prompt + pub fn insert_prompt(&self, prompt_name: String, server_name: String, prompt: Prompt) { + self.prompts.insert( + prompt_name, + CachedPrompt { + server_name, + prompt, + }, + ); + } + + /// Get all prompts + pub fn list_prompts(&self) -> Vec<(String, String, Prompt)> { + self.prompts + .iter() + .map(|entry| { + let (name, cached) = entry.pair(); + ( + name.clone(), + cached.server_name.clone(), + cached.prompt.clone(), + ) + }) + .collect() + } + + // ============================================================================ + // Resource Methods + // ============================================================================ + + /// Get a resource if it exists + pub fn get_resource(&self, resource_uri: &str) -> Option<(String, RawResource)> { + self.resources + .get(resource_uri) + .map(|entry| (entry.server_name.clone(), entry.resource.clone())) + } + + /// Check if resource exists + pub fn has_resource(&self, resource_uri: &str) -> bool { + self.resources.contains_key(resource_uri) + } + + /// Insert or update a resource + pub fn insert_resource( + &self, + resource_uri: String, + server_name: String, + resource: RawResource, + ) { + self.resources.insert( + resource_uri, + CachedResource { + server_name, + resource, + }, + ); + } + + /// Get all resources + pub fn list_resources(&self) -> Vec<(String, String, RawResource)> { + self.resources + .iter() + .map(|entry| { + let (uri, cached) = entry.pair(); + ( + uri.clone(), + cached.server_name.clone(), + cached.resource.clone(), + ) + }) + .collect() + } + + // ============================================================================ + // Server Management Methods + // ============================================================================ + + /// Clear all cached items for a specific server (called when LRU evicts client) + pub fn clear_server_tools(&self, server_name: &str) { + self.tools + .retain(|_, cached| cached.server_name != server_name); + self.prompts + .retain(|_, cached| cached.server_name != server_name); + self.resources + .retain(|_, cached| cached.server_name != server_name); + } + + /// Get count of cached items + pub fn counts(&self) -> (usize, usize, usize) { + (self.tools.len(), self.prompts.len(), self.resources.len()) + } + + /// Clear all cached items + pub fn clear_all(&self) { + self.tools.clear(); + self.prompts.clear(); + self.resources.clear(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mcp::config::{Prompt, RawResource, Tool}; + + // Helper to create a test tool + fn create_test_tool(name: &str) -> Tool { + use std::{borrow::Cow, sync::Arc}; + + let schema_obj = serde_json::json!({ + "type": "object", + "properties": {} + }); + + let schema_map = if let serde_json::Value::Object(m) = schema_obj { + m + } else { + serde_json::Map::new() + }; + + Tool { + name: Cow::Owned(name.to_string()), + title: None, + description: Some(Cow::Owned(format!("Test tool: {}", name))), + input_schema: Arc::new(schema_map), + output_schema: None, + annotations: None, + icons: None, + } + } + + // Helper to create a test prompt + fn create_test_prompt(name: &str) -> Prompt { + Prompt { + name: name.to_string(), + title: None, + description: Some(format!("Test prompt: {}", name)), + arguments: None, + icons: None, + } + } + + // Helper to create a test resource + fn create_test_resource(uri: &str) -> RawResource { + RawResource { + uri: uri.to_string(), + name: uri.to_string(), + title: None, + description: Some(format!("Test resource: {}", uri)), + mime_type: Some("text/plain".to_string()), + size: None, + icons: None, + } + } + + #[test] + fn test_tool_insert_and_get() { + let inventory = ToolInventory::new(); + let tool = create_test_tool("test_tool"); + + inventory.insert_tool("test_tool".to_string(), "server1".to_string(), tool.clone()); + + let result = inventory.get_tool("test_tool"); + assert!(result.is_some()); + + let (server_name, retrieved_tool) = result.unwrap(); + assert_eq!(server_name, "server1"); + assert_eq!(retrieved_tool.name, "test_tool"); + } + + #[test] + fn test_has_tool() { + let inventory = ToolInventory::new(); + let tool = create_test_tool("check_tool"); + + assert!(!inventory.has_tool("check_tool")); + + inventory.insert_tool("check_tool".to_string(), "server1".to_string(), tool); + + assert!(inventory.has_tool("check_tool")); + } + + #[test] + fn test_list_tools() { + let inventory = ToolInventory::new(); + + inventory.insert_tool( + "tool1".to_string(), + "server1".to_string(), + create_test_tool("tool1"), + ); + inventory.insert_tool( + "tool2".to_string(), + "server1".to_string(), + create_test_tool("tool2"), + ); + inventory.insert_tool( + "tool3".to_string(), + "server2".to_string(), + create_test_tool("tool3"), + ); + + let tools = inventory.list_tools(); + assert_eq!(tools.len(), 3); + } + + #[test] + fn test_clear_server_tools() { + let inventory = ToolInventory::new(); + + inventory.insert_tool( + "tool1".to_string(), + "server1".to_string(), + create_test_tool("tool1"), + ); + inventory.insert_tool( + "tool2".to_string(), + "server2".to_string(), + create_test_tool("tool2"), + ); + + assert_eq!(inventory.list_tools().len(), 2); + + inventory.clear_server_tools("server1"); + + let tools = inventory.list_tools(); + assert_eq!(tools.len(), 1); + assert_eq!(tools[0].0, "tool2"); + } + + #[test] + fn test_prompt_operations() { + let inventory = ToolInventory::new(); + let prompt = create_test_prompt("test_prompt"); + + inventory.insert_prompt( + "test_prompt".to_string(), + "server1".to_string(), + prompt.clone(), + ); + + assert!(inventory.has_prompt("test_prompt")); + + let result = inventory.get_prompt("test_prompt"); + assert!(result.is_some()); + + let (server_name, retrieved_prompt) = result.unwrap(); + assert_eq!(server_name, "server1"); + assert_eq!(retrieved_prompt.name, "test_prompt"); + } + + #[test] + fn test_resource_operations() { + let inventory = ToolInventory::new(); + let resource = create_test_resource("file:///test.txt"); + + inventory.insert_resource( + "file:///test.txt".to_string(), + "server1".to_string(), + resource.clone(), + ); + + assert!(inventory.has_resource("file:///test.txt")); + + let result = inventory.get_resource("file:///test.txt"); + assert!(result.is_some()); + + let (server_name, retrieved_resource) = result.unwrap(); + assert_eq!(server_name, "server1"); + assert_eq!(retrieved_resource.uri, "file:///test.txt"); + } + + #[tokio::test] + async fn test_concurrent_access() { + use std::sync::Arc; + + let inventory = Arc::new(ToolInventory::new()); + + // Spawn multiple tasks that insert tools concurrently + let mut handles = vec![]; + for i in 0..10 { + let inv = Arc::clone(&inventory); + let handle = tokio::spawn(async move { + let tool = create_test_tool(&format!("tool_{}", i)); + inv.insert_tool(format!("tool_{}", i), format!("server_{}", i % 3), tool); + }); + handles.push(handle); + } + + // Wait for all tasks to complete + for handle in handles { + handle.await.unwrap(); + } + + // Should have 10 tools + let (tools, _, _) = inventory.counts(); + assert_eq!(tools, 10); + } + + #[test] + fn test_clear_all() { + let inventory = ToolInventory::new(); + + inventory.insert_tool( + "tool1".to_string(), + "server1".to_string(), + create_test_tool("tool1"), + ); + inventory.insert_prompt( + "prompt1".to_string(), + "server1".to_string(), + create_test_prompt("prompt1"), + ); + inventory.insert_resource( + "res1".to_string(), + "server1".to_string(), + create_test_resource("res1"), + ); + + let (tools, prompts, resources) = inventory.counts(); + assert_eq!(tools, 1); + assert_eq!(prompts, 1); + assert_eq!(resources, 1); + + inventory.clear_all(); + + let (tools, prompts, resources) = inventory.counts(); + assert_eq!(tools, 0); + assert_eq!(prompts, 0); + assert_eq!(resources, 0); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/manager.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/manager.rs new file mode 100644 index 00000000..ffd114c7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/manager.rs @@ -0,0 +1,782 @@ +//! MCP client management and orchestration. +//! +//! Manages both static MCP servers (from config) and dynamic MCP servers (from requests). +//! Static clients are never evicted; dynamic clients use LRU eviction via connection pool. + +use std::{borrow::Cow, sync::Arc, time::Duration}; + +use backoff::ExponentialBackoffBuilder; +use dashmap::DashMap; +use rmcp::{ + model::{ + CallToolRequestParam, CallToolResult, GetPromptRequestParam, GetPromptResult, + ReadResourceRequestParam, ReadResourceResult, SubscribeRequestParam, + UnsubscribeRequestParam, + }, + service::RunningService, + transport::{ + sse_client::SseClientConfig, streamable_http_client::StreamableHttpClientTransportConfig, + ConfigureCommandExt, SseClientTransport, StreamableHttpClientTransport, TokioChildProcess, + }, + RoleClient, ServiceExt, +}; +use serde_json::Map; +use tracing::{debug, error, info, warn}; + +use crate::mcp::{ + config::{McpConfig, McpProxyConfig, McpServerConfig, McpTransport, Prompt, RawResource, Tool}, + connection_pool::McpConnectionPool, + error::{McpError, McpResult}, + inventory::ToolInventory, + tool_args::ToolArgs, +}; + +/// Type alias for MCP client +type McpClient = RunningService; + +pub struct McpManager { + static_clients: Arc>>, + inventory: Arc, + connection_pool: Arc, + _config: McpConfig, +} + +impl McpManager { + const MAX_DYNAMIC_CLIENTS: usize = 200; + + pub async fn new(config: McpConfig, pool_max_connections: usize) -> McpResult { + let inventory = Arc::new(ToolInventory::new()); + + let mut connection_pool = + McpConnectionPool::with_full_config(pool_max_connections, config.proxy.clone()); + + let inventory_clone = Arc::clone(&inventory); + connection_pool.set_eviction_callback(move |server_key: &str| { + debug!( + "LRU evicted dynamic server '{}' - clearing tools from inventory", + server_key + ); + inventory_clone.clear_server_tools(server_key); + }); + + let connection_pool = Arc::new(connection_pool); + + // Create storage for static clients + let static_clients = Arc::new(DashMap::new()); + + // Get global proxy config for all servers + let global_proxy = config.proxy.as_ref(); + + // Connect to all static servers from config + for server_config in &config.servers { + match Self::connect_server(server_config, global_proxy).await { + Ok(client) => { + let client_arc = Arc::new(client); + // Load inventory for this server + Self::load_server_inventory(&inventory, &server_config.name, &client_arc).await; + static_clients.insert(server_config.name.clone(), client_arc); + info!("Connected to static server '{}'", server_config.name); + } + Err(e) => { + error!( + "Failed to connect to static server '{}': {}", + server_config.name, e + ); + } + } + } + + if static_clients.is_empty() { + warn!("No static MCP servers connected"); + } + + Ok(Self { + static_clients, + inventory, + connection_pool, + _config: config, + }) + } + + pub async fn with_defaults(config: McpConfig) -> McpResult { + Self::new(config, Self::MAX_DYNAMIC_CLIENTS).await + } + + pub async fn get_client(&self, server_name: &str) -> Option> { + if let Some(client) = self.static_clients.get(server_name) { + return Some(Arc::clone(client.value())); + } + self.connection_pool.get(server_name) + } + + pub async fn get_or_create_client( + &self, + server_config: McpServerConfig, + ) -> McpResult> { + let server_name = server_config.name.clone(); + + if let Some(client) = self.static_clients.get(&server_name) { + return Ok(Arc::clone(client.value())); + } + + let server_key = Self::server_key(&server_config); + let client = self + .connection_pool + .get_or_create( + &server_key, + server_config, + |config, global_proxy| async move { + Self::connect_server(&config, global_proxy.as_ref()).await + }, + ) + .await?; + + self.inventory.clear_server_tools(&server_key); + Self::load_server_inventory(&self.inventory, &server_key, &client).await; + Ok(client) + } + + pub fn list_static_servers(&self) -> Vec { + self.static_clients + .iter() + .map(|e| e.key().clone()) + .collect() + } + + pub fn is_static_server(&self, server_name: &str) -> bool { + self.static_clients.contains_key(server_name) + } + + pub fn register_static_server(&self, name: String, client: Arc) { + self.static_clients.insert(name.clone(), client); + info!("Registered static MCP server: {}", name); + } + + /// List all available tools from all servers + pub fn list_tools(&self) -> Vec { + self.inventory + .list_tools() + .into_iter() + .map(|(_tool_name, _server_name, tool_info)| tool_info) + .collect() + } + + /// Call a tool by name with automatic type coercion + /// + /// Accepts either JSON string or parsed Map as arguments. + /// Automatically converts string numbers to actual numbers based on tool schema. + pub async fn call_tool( + &self, + tool_name: &str, + args: impl Into, + ) -> McpResult { + // Get tool info for schema and server + let (server_name, tool_info) = self + .inventory + .get_tool(tool_name) + .ok_or_else(|| McpError::ToolNotFound(tool_name.to_string()))?; + + // Convert args with type coercion based on schema + let tool_schema = Some(serde_json::Value::Object((*tool_info.input_schema).clone())); + let args_map = args + .into() + .into_map(tool_schema.as_ref()) + .map_err(McpError::InvalidArguments)?; + + // Get client for that server + let client = self + .get_client(&server_name) + .await + .ok_or_else(|| McpError::ServerNotFound(server_name.clone()))?; + + // Call the tool + let request = CallToolRequestParam { + name: Cow::Owned(tool_name.to_string()), + arguments: args_map, + }; + + client + .call_tool(request) + .await + .map_err(|e| McpError::ToolExecution(format!("Failed to call tool: {}", e))) + } + + /// Get a tool by name + pub fn get_tool(&self, tool_name: &str) -> Option { + self.inventory + .get_tool(tool_name) + .map(|(_server_name, tool_info)| tool_info) + } + + /// Get a prompt by name + pub async fn get_prompt( + &self, + prompt_name: &str, + args: Option>, + ) -> McpResult { + // Get server that owns this prompt + let (server_name, _prompt_info) = self + .inventory + .get_prompt(prompt_name) + .ok_or_else(|| McpError::PromptNotFound(prompt_name.to_string()))?; + + // Get client for that server + let client = self + .get_client(&server_name) + .await + .ok_or_else(|| McpError::ServerNotFound(server_name.clone()))?; + + // Get the prompt + let request = GetPromptRequestParam { + name: prompt_name.to_string(), + arguments: args, + }; + + client + .get_prompt(request) + .await + .map_err(|e| McpError::Transport(format!("Failed to get prompt: {}", e))) + } + + /// List all available prompts + pub fn list_prompts(&self) -> Vec { + self.inventory + .list_prompts() + .into_iter() + .map(|(_prompt_name, _server_name, prompt_info)| prompt_info) + .collect() + } + + /// Read a resource by URI + pub async fn read_resource(&self, uri: &str) -> McpResult { + // Get server that owns this resource + let (server_name, _resource_info) = self + .inventory + .get_resource(uri) + .ok_or_else(|| McpError::ResourceNotFound(uri.to_string()))?; + + // Get client for that server + let client = self + .get_client(&server_name) + .await + .ok_or_else(|| McpError::ServerNotFound(server_name.clone()))?; + + // Read the resource + let request = ReadResourceRequestParam { + uri: uri.to_string(), + }; + + client + .read_resource(request) + .await + .map_err(|e| McpError::Transport(format!("Failed to read resource: {}", e))) + } + + /// List all available resources + pub fn list_resources(&self) -> Vec { + self.inventory + .list_resources() + .into_iter() + .map(|(_resource_uri, _server_name, resource_info)| resource_info) + .collect() + } + + /// Refresh inventory for a specific server + pub async fn refresh_server_inventory(&self, server_name: &str) -> McpResult<()> { + let client = self + .get_client(server_name) + .await + .ok_or_else(|| McpError::ServerNotFound(server_name.to_string()))?; + + info!("Refreshing inventory for server: {}", server_name); + self.load_server_inventory_internal(server_name, &client) + .await; + Ok(()) + } + + /// Start background refresh for ALL servers (static + dynamic) + /// Refreshes every 10-15 minutes to keep tool inventory up-to-date + pub fn spawn_background_refresh_all( + self: Arc, + refresh_interval: Duration, + ) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let mut interval = tokio::time::interval(refresh_interval); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + loop { + interval.tick().await; + + // Get all static server keys + // Note: Dynamic clients in the connection pool are refreshed on-demand + // when they are accessed via get_or_create_client() + let server_keys: Vec = self + .static_clients + .iter() + .map(|e| e.key().clone()) + .collect(); + + if !server_keys.is_empty() { + debug!( + "Background refresh: Refreshing {} static server(s)", + server_keys.len() + ); + + for server_key in server_keys { + if let Err(e) = self.refresh_server_inventory(&server_key).await { + warn!("Background refresh failed for '{}': {}", server_key, e); + } + } + + debug!("Background refresh: Completed refresh cycle"); + } + } + }) + } + + /// Check if a tool exists + pub fn has_tool(&self, name: &str) -> bool { + self.inventory.has_tool(name) + } + + /// Get prompt info by name + pub fn get_prompt_info(&self, name: &str) -> Option { + self.inventory.get_prompt(name).map(|(_server, info)| info) + } + + /// Get resource info by URI + pub fn get_resource_info(&self, uri: &str) -> Option { + self.inventory.get_resource(uri).map(|(_server, info)| info) + } + + /// Subscribe to resource changes + pub async fn subscribe_resource(&self, uri: &str) -> McpResult<()> { + let (server_name, _resource_info) = self + .inventory + .get_resource(uri) + .ok_or_else(|| McpError::ResourceNotFound(uri.to_string()))?; + + let client = self + .get_client(&server_name) + .await + .ok_or_else(|| McpError::ServerNotFound(server_name.clone()))?; + + debug!("Subscribing to '{}' on '{}'", uri, server_name); + + client + .peer() + .subscribe(SubscribeRequestParam { + uri: uri.to_string(), + }) + .await + .map_err(|e| McpError::ToolExecution(format!("Failed to subscribe: {}", e))) + } + + /// Unsubscribe from resource changes + pub async fn unsubscribe_resource(&self, uri: &str) -> McpResult<()> { + let (server_name, _resource_info) = self + .inventory + .get_resource(uri) + .ok_or_else(|| McpError::ResourceNotFound(uri.to_string()))?; + + let client = self + .get_client(&server_name) + .await + .ok_or_else(|| McpError::ServerNotFound(server_name.clone()))?; + + debug!("Unsubscribing from '{}' on '{}'", uri, server_name); + + client + .peer() + .unsubscribe(UnsubscribeRequestParam { + uri: uri.to_string(), + }) + .await + .map_err(|e| McpError::ToolExecution(format!("Failed to unsubscribe: {}", e))) + } + + /// List all connected servers (static + dynamic) + pub fn list_servers(&self) -> Vec { + let mut servers = Vec::new(); + + // Add static servers + servers.extend(self.static_clients.iter().map(|e| e.key().clone())); + + // Add dynamic servers from connection pool + servers.extend(self.connection_pool.list_server_keys()); + + servers + } + + /// Disconnect from all servers (for cleanup) + pub async fn shutdown(&self) { + // Shutdown static servers + let static_keys: Vec = self + .static_clients + .iter() + .map(|e| e.key().clone()) + .collect(); + for name in static_keys { + if let Some((_key, client)) = self.static_clients.remove(&name) { + // Try to unwrap Arc to call cancel + match Arc::try_unwrap(client) { + Ok(client) => { + if let Err(e) = client.cancel().await { + warn!("Error disconnecting from static server '{}': {}", name, e); + } + } + Err(_) => { + warn!( + "Could not shutdown static server '{}': client still in use", + name + ); + } + } + } + } + + // Clear dynamic clients from connection pool + // The pool will handle cleanup on drop + self.connection_pool.clear(); + } + + /// Get statistics about the manager + pub fn stats(&self) -> McpManagerStats { + let (tools, prompts, resources) = self.inventory.counts(); + McpManagerStats { + static_server_count: self.static_clients.len(), + pool_stats: self.connection_pool.stats(), + tool_count: tools, + prompt_count: prompts, + resource_count: resources, + } + } + + /// Get the shared tool inventory + pub fn inventory(&self) -> Arc { + Arc::clone(&self.inventory) + } + + /// Get the connection pool + pub fn connection_pool(&self) -> Arc { + Arc::clone(&self.connection_pool) + } + + // ======================================================================== + // Internal Helper Methods + // ======================================================================== + + /// Static helper for loading inventory (for new()) + /// Discover and cache tools/prompts/resources for a connected server + /// + /// This method is public to allow workflow-based inventory loading. + /// It discovers all tools, prompts, and resources from the client and caches them in the inventory. + pub async fn load_server_inventory( + inventory: &Arc, + server_key: &str, + client: &Arc, + ) { + // Tools + match client.peer().list_all_tools().await { + Ok(ts) => { + info!("Discovered {} tools from '{}'", ts.len(), server_key); + for t in ts { + inventory.insert_tool(t.name.to_string(), server_key.to_string(), t); + } + } + Err(e) => warn!("Failed to list tools from '{}': {}", server_key, e), + } + + // Prompts + match client.peer().list_all_prompts().await { + Ok(ps) => { + info!("Discovered {} prompts from '{}'", ps.len(), server_key); + for p in ps { + inventory.insert_prompt(p.name.clone(), server_key.to_string(), p); + } + } + Err(e) => debug!("No prompts or failed to list on '{}': {}", server_key, e), + } + + // Resources + match client.peer().list_all_resources().await { + Ok(rs) => { + info!("Discovered {} resources from '{}'", rs.len(), server_key); + for r in rs { + inventory.insert_resource(r.uri.clone(), server_key.to_string(), r.raw); + } + } + Err(e) => debug!("No resources or failed to list on '{}': {}", server_key, e), + } + } + + /// Discover and cache tools/prompts/resources for a connected server (internal wrapper) + async fn load_server_inventory_internal(&self, server_name: &str, client: &McpClient) { + // Tools + match client.peer().list_all_tools().await { + Ok(ts) => { + info!("Discovered {} tools from '{}'", ts.len(), server_name); + for t in ts { + self.inventory + .insert_tool(t.name.to_string(), server_name.to_string(), t); + } + } + Err(e) => warn!("Failed to list tools from '{}': {}", server_name, e), + } + + // Prompts + match client.peer().list_all_prompts().await { + Ok(ps) => { + info!("Discovered {} prompts from '{}'", ps.len(), server_name); + for p in ps { + self.inventory + .insert_prompt(p.name.clone(), server_name.to_string(), p); + } + } + Err(e) => debug!("No prompts or failed to list on '{}': {}", server_name, e), + } + + // Resources + match client.peer().list_all_resources().await { + Ok(rs) => { + info!("Discovered {} resources from '{}'", rs.len(), server_name); + for r in rs { + self.inventory + .insert_resource(r.uri.clone(), server_name.to_string(), r.raw); + } + } + Err(e) => debug!("No resources or failed to list on '{}': {}", server_name, e), + } + } + + // ======================================================================== + // Connection Logic (from client_manager.rs) + // ======================================================================== + + /// Connect to an MCP server + /// + /// This method is public to allow workflow-based server registration at runtime. + /// It handles connection with automatic retry for network-based transports (SSE/Streamable). + pub async fn connect_server( + config: &McpServerConfig, + global_proxy: Option<&McpProxyConfig>, + ) -> McpResult { + let needs_retry = matches!( + &config.transport, + McpTransport::Sse { .. } | McpTransport::Streamable { .. } + ); + if needs_retry { + Self::connect_server_with_retry(config, global_proxy).await + } else { + Self::connect_server_impl(config, global_proxy).await + } + } + + /// Connect with exponential backoff retry for remote servers + async fn connect_server_with_retry( + config: &McpServerConfig, + global_proxy: Option<&McpProxyConfig>, + ) -> McpResult { + let backoff = ExponentialBackoffBuilder::new() + .with_initial_interval(Duration::from_secs(1)) + .with_max_interval(Duration::from_secs(30)) + .with_max_elapsed_time(Some(Duration::from_secs(30))) + .build(); + + backoff::future::retry(backoff, || async { + match Self::connect_server_impl(config, global_proxy).await { + Ok(client) => Ok(client), + Err(e) => { + if Self::is_permanent_error(&e) { + error!( + "Permanent error connecting to '{}': {} - not retrying", + config.name, e + ); + Err(backoff::Error::permanent(e)) + } else { + warn!("Failed to connect to '{}', retrying: {}", config.name, e); + Err(backoff::Error::transient(e)) + } + } + } + }) + .await + } + + /// Determine if an error is permanent (should not retry) or transient + fn is_permanent_error(error: &McpError) -> bool { + match error { + McpError::Config(_) => true, + McpError::Auth(_) => true, + McpError::ServerNotFound(_) => true, + McpError::Transport(_) => true, + McpError::ConnectionFailed(msg) => { + msg.contains("initialize") + || msg.contains("connection closed") + || msg.contains("connection refused") + || msg.contains("invalid URL") + || msg.contains("not found") + } + _ => false, + } + } + + /// Internal implementation of server connection (stdio/sse/streamable) + async fn connect_server_impl( + config: &McpServerConfig, + global_proxy: Option<&McpProxyConfig>, + ) -> McpResult { + info!( + "Connecting to MCP server '{}' via {:?}", + config.name, config.transport + ); + + match &config.transport { + McpTransport::Stdio { + command, + args, + envs, + } => { + let transport = TokioChildProcess::new( + tokio::process::Command::new(command).configure(|cmd| { + cmd.args(args) + .envs(envs.iter()) + .stderr(std::process::Stdio::inherit()); + }), + ) + .map_err(|e| McpError::Transport(format!("create stdio transport: {}", e)))?; + + let client = ().serve(transport).await.map_err(|e| { + McpError::ConnectionFailed(format!("initialize stdio client: {}", e)) + })?; + + info!("Connected to stdio server '{}'", config.name); + Ok(client) + } + + McpTransport::Sse { url, token } => { + // Resolve proxy configuration + let proxy_config = crate::mcp::proxy::resolve_proxy_config(config, global_proxy); + + // Create HTTP client with proxy support + let client = if token.is_some() { + let mut builder = reqwest::Client::builder() + .timeout(Duration::from_secs(30)) + .connect_timeout(Duration::from_secs(10)); + + // Apply proxy configuration using proxy.rs helper + if let Some(proxy_cfg) = proxy_config { + builder = crate::mcp::proxy::apply_proxy_to_builder(builder, proxy_cfg)?; + } + + // Add Authorization header + builder = builder.default_headers({ + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert( + reqwest::header::AUTHORIZATION, + format!("Bearer {}", token.as_ref().unwrap()) + .parse() + .map_err(|e| McpError::Transport(format!("auth token: {}", e)))?, + ); + headers + }); + + builder + .build() + .map_err(|e| McpError::Transport(format!("build HTTP client: {}", e)))? + } else { + crate::mcp::proxy::create_http_client(proxy_config)? + }; + + let cfg = SseClientConfig { + sse_endpoint: url.clone().into(), + ..Default::default() + }; + + let transport = SseClientTransport::start_with_client(client, cfg) + .await + .map_err(|e| McpError::Transport(format!("create SSE transport: {}", e)))?; + + let client = ().serve(transport).await.map_err(|e| { + McpError::ConnectionFailed(format!("initialize SSE client: {}", e)) + })?; + + info!("Connected to SSE server '{}' at {}", config.name, url); + Ok(client) + } + + McpTransport::Streamable { url, token } => { + // Note: Streamable transport doesn't support proxy yet + let _proxy_config = crate::mcp::proxy::resolve_proxy_config(config, global_proxy); + if _proxy_config.is_some() { + warn!( + "Proxy configuration detected but not supported for Streamable transport on server '{}'", + config.name + ); + } + + let transport = if let Some(tok) = token { + let mut cfg = StreamableHttpClientTransportConfig::with_uri(url.as_str()); + cfg.auth_header = Some(format!("Bearer {}", tok)); + StreamableHttpClientTransport::from_config(cfg) + } else { + StreamableHttpClientTransport::from_uri(url.as_str()) + }; + + let client = ().serve(transport).await.map_err(|e| { + McpError::ConnectionFailed(format!("initialize streamable client: {}", e)) + })?; + + info!( + "Connected to streamable HTTP server '{}' at {}", + config.name, url + ); + Ok(client) + } + } + } + + /// Generate a unique key for a server config + fn server_key(config: &McpServerConfig) -> String { + // Extract URL from transport or use name + match &config.transport { + McpTransport::Streamable { url, .. } => url.clone(), + McpTransport::Sse { url, .. } => url.clone(), + McpTransport::Stdio { command, .. } => command.clone(), + } + } +} + +/// Statistics about the MCP manager +#[derive(Debug, Clone)] +pub struct McpManagerStats { + /// Number of static servers registered + pub static_server_count: usize, + /// Connection pool statistics + pub pool_stats: crate::mcp::connection_pool::PoolStats, + /// Number of cached tools + pub tool_count: usize, + /// Number of cached prompts + pub prompt_count: usize, + /// Number of cached resources + pub resource_count: usize, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_manager_creation() { + let config = McpConfig { + servers: vec![], + pool: Default::default(), + proxy: None, + warmup: vec![], + inventory: Default::default(), + }; + + let manager = McpManager::new(config, 100).await.unwrap(); + assert_eq!(manager.list_static_servers().len(), 0); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/mod.rs new file mode 100644 index 00000000..3ba98e34 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/mod.rs @@ -0,0 +1,25 @@ +//! Model Context Protocol (MCP) client implementation. +//! +//! Provides MCP client functionality including tools, prompts, resources, and OAuth. +//! Supports stdio, SSE, and HTTP transports with connection pooling and caching. + +pub mod config; +pub mod connection_pool; +pub mod error; +pub mod inventory; +pub mod manager; +pub mod oauth; +pub mod proxy; +pub mod tool_args; + +// Re-export the main types for convenience +pub use config::{ + InventoryConfig, McpConfig, McpPoolConfig, McpProxyConfig, McpServerConfig, McpTransport, + Prompt, RawResource, Tool, WarmupServer, +}; +pub use connection_pool::{CachedConnection, McpConnectionPool, PoolStats}; +pub use error::{McpError, McpResult}; +pub use inventory::ToolInventory; +pub use manager::{McpManager, McpManagerStats}; +pub use proxy::{create_http_client, resolve_proxy_config}; +pub use tool_args::ToolArgs; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/oauth.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/oauth.rs new file mode 100644 index 00000000..d8eaac4c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/oauth.rs @@ -0,0 +1,194 @@ +//! OAuth authentication for MCP servers. +//! +//! Handles OAuth flow including callback server and token exchange. + +use std::{net::SocketAddr, sync::Arc}; + +use axum::{ + extract::{Query, State}, + response::Html, + routing::get, + Router, +}; +use rmcp::transport::auth::OAuthState; +use serde::Deserialize; +use tokio::sync::{oneshot, Mutex}; + +use crate::mcp::error::{McpError, McpResult}; + +/// OAuth callback parameters +#[derive(Debug, Deserialize)] +struct CallbackParams { + code: String, + #[allow(dead_code)] + state: Option, +} + +/// State for the callback server +#[derive(Clone)] +struct CallbackState { + code_receiver: Arc>>>, +} + +/// HTML page returned after successful OAuth callback +const CALLBACK_HTML: &str = r#" + + + + OAuth Success + + + +

+
✓
+

Authentication Successful!

+

You can now close this window and return to your application.

+
+ + +"#; + +/// OAuth authentication helper for MCP servers +pub struct OAuthHelper { + server_url: String, + redirect_uri: String, + callback_port: u16, +} + +impl OAuthHelper { + /// Create a new OAuth helper + pub fn new(server_url: String, redirect_uri: String, callback_port: u16) -> Self { + Self { + server_url, + redirect_uri, + callback_port, + } + } + + /// Perform OAuth authentication flow + pub async fn authenticate( + &self, + scopes: &[&str], + ) -> McpResult { + // Initialize OAuth state machine + let mut oauth_state = OAuthState::new(&self.server_url, None) + .await + .map_err(|e| McpError::Auth(format!("Failed to initialize OAuth: {}", e)))?; + + oauth_state + .start_authorization(scopes, &self.redirect_uri, None) + .await + .map_err(|e| McpError::Auth(format!("Failed to start authorization: {}", e)))?; + + // Get authorization URL + let auth_url = oauth_state + .get_authorization_url() + .await + .map_err(|e| McpError::Auth(format!("Failed to get authorization URL: {}", e)))?; + + tracing::info!("OAuth authorization URL: {}", auth_url); + + // Start callback server and wait for code + let auth_code = self.start_callback_server().await?; + + // Exchange code for token + oauth_state + .handle_callback(&auth_code, "") + .await + .map_err(|e| McpError::Auth(format!("Failed to handle OAuth callback: {}", e)))?; + + // Get authorization manager + oauth_state + .into_authorization_manager() + .ok_or_else(|| McpError::Auth("Failed to get authorization manager".to_string())) + } + + /// Start a local HTTP server to receive the OAuth callback + async fn start_callback_server(&self) -> McpResult { + let (code_sender, code_receiver) = oneshot::channel::(); + + let state = CallbackState { + code_receiver: Arc::new(Mutex::new(Some(code_sender))), + }; + + // Create router for callback + let app = Router::new() + .route("/callback", get(Self::callback_handler)) + .with_state(state); + + let addr = SocketAddr::from(([127, 0, 0, 1], self.callback_port)); + + // Start server in background + let listener = tokio::net::TcpListener::bind(addr).await.map_err(|e| { + McpError::Auth(format!( + "Failed to bind to callback port {}: {}", + self.callback_port, e + )) + })?; + + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + + tracing::info!( + "OAuth callback server started on port {}", + self.callback_port + ); + + // Wait for authorization code + code_receiver + .await + .map_err(|_| McpError::Auth("Failed to receive authorization code".to_string())) + } + + /// Handle OAuth callback + async fn callback_handler( + Query(params): Query, + State(state): State, + ) -> Html { + tracing::debug!("Received OAuth callback with code"); + + // Send code to waiting task + if let Some(sender) = state.code_receiver.lock().await.take() { + let _ = sender.send(params.code); + } + + Html(CALLBACK_HTML.to_string()) + } +} + +/// Create an OAuth-authenticated client +pub async fn create_oauth_client( + server_url: String, + _sse_url: String, + redirect_uri: String, + callback_port: u16, + scopes: &[&str], +) -> McpResult> { + let helper = OAuthHelper::new(server_url, redirect_uri, callback_port); + let auth_manager = helper.authenticate(scopes).await?; + + let client = rmcp::transport::auth::AuthClient::new(reqwest::Client::default(), auth_manager); + + Ok(client) +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/proxy.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/proxy.rs new file mode 100644 index 00000000..f80551a7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/proxy.rs @@ -0,0 +1,251 @@ +//! HTTP proxy configuration for MCP connections. +//! +//! Resolves proxy settings and creates HTTP clients for MCP server connections. + +use std::time::Duration; + +use crate::mcp::{McpError, McpProxyConfig, McpResult, McpServerConfig}; + +/// Resolve proxy configuration for a server +/// Priority: server.proxy > global.proxy > None +/// +/// # Arguments +/// * `server_config` - Server-specific configuration +/// * `global_proxy` - Global proxy configuration from McpConfig +/// +/// # Returns +/// The resolved proxy configuration, or None for direct connection +pub fn resolve_proxy_config<'a>( + server_config: &'a McpServerConfig, + global_proxy: Option<&'a McpProxyConfig>, +) -> Option<&'a McpProxyConfig> { + // Priority 1: Check if server has explicit proxy config + // Note: server.proxy = Some(config) uses that config + // server.proxy = None (set explicitly in YAML as null) forces direct connection + // server.proxy not set (field missing) falls back to global + if server_config.proxy.is_some() { + server_config.proxy.as_ref() + } else { + // Priority 2: Fall back to global proxy + global_proxy + } +} + +/// Apply proxy configuration to a ClientBuilder +/// +/// This is a reusable helper that applies proxy settings without building the client, +/// allowing additional configuration (like auth headers) to be added afterward. +/// +/// # Arguments +/// * `builder` - The reqwest::ClientBuilder to configure +/// * `proxy_config` - The proxy configuration to apply +/// +/// # Returns +/// The configured builder or error +pub fn apply_proxy_to_builder( + mut builder: reqwest::ClientBuilder, + proxy_cfg: &McpProxyConfig, +) -> McpResult { + // Configure HTTP proxy + if let Some(ref http_proxy) = proxy_cfg.http { + let mut proxy = reqwest::Proxy::http(http_proxy) + .map_err(|e| McpError::Config(format!("Invalid HTTP proxy: {}", e)))?; + + // Apply no_proxy exclusions + if let Some(ref no_proxy) = proxy_cfg.no_proxy { + proxy = proxy.no_proxy(reqwest::NoProxy::from_string(no_proxy)); + } + + // Apply authentication if configured + if let (Some(ref username), Some(ref password)) = (&proxy_cfg.username, &proxy_cfg.password) + { + proxy = proxy.basic_auth(username, password); + } + + builder = builder.proxy(proxy); + } + + // Configure HTTPS proxy + if let Some(ref https_proxy) = proxy_cfg.https { + let mut proxy = reqwest::Proxy::https(https_proxy) + .map_err(|e| McpError::Config(format!("Invalid HTTPS proxy: {}", e)))?; + + // Apply no_proxy exclusions + if let Some(ref no_proxy) = proxy_cfg.no_proxy { + proxy = proxy.no_proxy(reqwest::NoProxy::from_string(no_proxy)); + } + + // Apply authentication if configured + if let (Some(ref username), Some(ref password)) = (&proxy_cfg.username, &proxy_cfg.password) + { + proxy = proxy.basic_auth(username, password); + } + + builder = builder.proxy(proxy); + } + + Ok(builder) +} + +/// Create HTTP client with MCP-specific proxy configuration +/// +/// # Arguments +/// * `proxy_config` - Optional proxy configuration to apply +/// +/// # Returns +/// A configured reqwest::Client or error +pub fn create_http_client(proxy_config: Option<&McpProxyConfig>) -> McpResult { + let mut builder = reqwest::Client::builder() + .timeout(Duration::from_secs(30)) + .connect_timeout(Duration::from_secs(10)); + + // Apply MCP-specific proxy if configured + if let Some(proxy_cfg) = proxy_config { + builder = apply_proxy_to_builder(builder, proxy_cfg)?; + } + + builder + .build() + .map_err(|e| McpError::Transport(format!("Failed to build HTTP client: {}", e))) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mcp::McpTransport; + + #[test] + fn test_resolve_proxy_no_config() { + let server = McpServerConfig { + name: "test".to_string(), + transport: McpTransport::Sse { + url: "http://localhost:3000/sse".to_string(), + token: None, + }, + proxy: None, + required: false, + }; + + let result = resolve_proxy_config(&server, None); + assert!( + result.is_none(), + "Should return None when no proxy configured" + ); + } + + #[test] + fn test_resolve_proxy_global_only() { + let server = McpServerConfig { + name: "test".to_string(), + transport: McpTransport::Sse { + url: "http://localhost:3000/sse".to_string(), + token: None, + }, + proxy: None, + required: false, + }; + + let global = McpProxyConfig { + http: Some("http://global-proxy:8080".to_string()), + https: None, + no_proxy: None, + username: None, + password: None, + }; + + let result = resolve_proxy_config(&server, Some(&global)); + assert!(result.is_some(), "Should use global proxy"); + assert_eq!( + result.unwrap().http.as_ref().unwrap(), + "http://global-proxy:8080" + ); + } + + #[test] + fn test_resolve_proxy_server_override() { + let server_proxy = McpProxyConfig { + http: Some("http://server-proxy:9090".to_string()), + https: None, + no_proxy: None, + username: None, + password: None, + }; + + let server = McpServerConfig { + name: "test".to_string(), + transport: McpTransport::Sse { + url: "http://localhost:3000/sse".to_string(), + token: None, + }, + proxy: Some(server_proxy), + required: false, + }; + + let global = McpProxyConfig { + http: Some("http://global-proxy:8080".to_string()), + https: None, + no_proxy: None, + username: None, + password: None, + }; + + let result = resolve_proxy_config(&server, Some(&global)); + assert!(result.is_some(), "Should use server-specific proxy"); + assert_eq!( + result.unwrap().http.as_ref().unwrap(), + "http://server-proxy:9090", + "Server proxy should override global" + ); + } + + #[test] + fn test_create_http_client_no_proxy() { + let client = create_http_client(None); + assert!(client.is_ok(), "Should create client without proxy"); + } + + #[test] + fn test_create_http_client_with_proxy() { + let proxy = McpProxyConfig { + http: Some("http://proxy.example.com:8080".to_string()), + https: None, + no_proxy: Some("localhost,127.0.0.1".to_string()), + username: None, + password: None, + }; + + let client = create_http_client(Some(&proxy)); + assert!(client.is_ok(), "Should create client with proxy"); + } + + #[test] + fn test_create_http_client_with_auth() { + let proxy = McpProxyConfig { + http: Some("http://proxy.example.com:8080".to_string()), + https: None, + no_proxy: None, + username: Some("user".to_string()), + password: Some("pass".to_string()), + }; + + let client = create_http_client(Some(&proxy)); + assert!( + client.is_ok(), + "Should create client with proxy authentication" + ); + } + + #[test] + fn test_create_http_client_invalid_proxy() { + let proxy = McpProxyConfig { + http: Some("://invalid".to_string()), // Invalid URL format + https: None, + no_proxy: None, + username: None, + password: None, + }; + + let client = create_http_client(Some(&proxy)); + assert!(client.is_err(), "Should fail with invalid proxy URL"); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/tool_args.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/tool_args.rs new file mode 100644 index 00000000..3207b9ac --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/mcp/tool_args.rs @@ -0,0 +1,103 @@ +//! MCP tool argument handling. +//! +//! Supports both JSON strings and parsed Maps with automatic type coercion. + +use serde_json::Map; + +/// Tool arguments input - supports both JSON strings and parsed Maps +pub enum ToolArgs { + /// JSON string that needs parsing + JsonString(String), + /// Already parsed map + Map(Option>), +} + +impl ToolArgs { + /// Convert to Map with type coercion based on tool schema + pub(crate) fn into_map( + self, + tool_schema: Option<&serde_json::Value>, + ) -> Result>, String> { + match self { + ToolArgs::JsonString(s) => { + if s.is_empty() || s == "{}" { + return Ok(None); + } + let mut value: serde_json::Value = + serde_json::from_str(&s).map_err(|e| format!("parse tool args: {}", e))?; + Self::coerce_types(&mut value, tool_schema)?; + let result = match value { + serde_json::Value::Object(m) => Some(m), + _ => None, + }; + Ok(result) + } + ToolArgs::Map(map) => { + if let Some(m) = map { + let mut value = serde_json::Value::Object(m); + Self::coerce_types(&mut value, tool_schema)?; + let result = match value { + serde_json::Value::Object(m) => Some(m), + _ => None, + }; + Ok(result) + } else { + Ok(None) + } + } + } + } + + /// Coerce string numbers to actual numbers based on schema + /// LLMs often output numbers as strings, so we need to convert them + fn coerce_types( + value: &mut serde_json::Value, + tool_schema: Option<&serde_json::Value>, + ) -> Result<(), String> { + let Some(params) = tool_schema else { + return Ok(()); + }; + let Some(props) = params.get("properties").and_then(|p| p.as_object()) else { + return Ok(()); + }; + let Some(args) = value.as_object_mut() else { + return Ok(()); + }; + + for (key, val) in args.iter_mut() { + let should_be_number = props + .get(key) + .and_then(|s| s.get("type")) + .and_then(|t| t.as_str()) + .is_some_and(|t| matches!(t, "number" | "integer")); + + if should_be_number { + if let Some(s) = val.as_str() { + if let Ok(num) = s.parse::() { + *val = serde_json::json!(num); + } + } + } + } + Ok(()) + } +} + +// Implement From traits for convenient conversion +impl From for ToolArgs { + fn from(s: String) -> Self { + ToolArgs::JsonString(s) + } +} + +impl From<&str> for ToolArgs { + fn from(s: &str) -> Self { + ToolArgs::JsonString(s.to_string()) + } +} + +impl From>> for ToolArgs { + fn from(map: Option>) -> Self { + ToolArgs::Map(map) + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/metrics.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/metrics.rs new file mode 100644 index 00000000..3f1d4b09 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/metrics.rs @@ -0,0 +1,1046 @@ +use std::{ + net::{IpAddr, Ipv4Addr, SocketAddr}, + time::Duration, +}; + +use metrics::{counter, describe_counter, describe_gauge, describe_histogram, gauge, histogram}; +use metrics_exporter_prometheus::{Matcher, PrometheusBuilder}; + +#[derive(Debug, Clone)] +pub struct PrometheusConfig { + pub port: u16, + pub host: String, +} + +impl Default for PrometheusConfig { + fn default() -> Self { + Self { + port: 29000, + host: "0.0.0.0".to_string(), + } + } +} + +pub fn init_metrics() { + describe_counter!( + "sgl_router_requests_total", + "Total number of requests by route and method" + ); + describe_histogram!( + "sgl_router_request_duration_seconds", + "Request duration in seconds by route" + ); + describe_counter!( + "sgl_router_request_errors_total", + "Total number of request errors by route and error type" + ); + describe_counter!( + "sgl_router_retries_total", + "Total number of request retries by route" + ); + describe_histogram!( + "sgl_router_retry_backoff_duration_seconds", + "Backoff duration in seconds by attempt index" + ); + describe_counter!( + "sgl_router_retries_exhausted_total", + "Total number of requests that exhausted retries by route" + ); + + describe_gauge!( + "sgl_router_cb_state", + "Circuit breaker state per worker (0=closed, 1=open, 2=half_open)" + ); + describe_counter!( + "sgl_router_cb_state_transitions_total", + "Total number of circuit breaker state transitions by worker" + ); + describe_counter!( + "sgl_router_cb_outcomes_total", + "Total number of circuit breaker outcomes by worker and outcome type (success/failure)" + ); + + describe_gauge!( + "sgl_router_active_workers", + "Number of currently active workers" + ); + describe_gauge!( + "sgl_router_worker_health", + "Worker health status (1=healthy, 0=unhealthy)" + ); + describe_gauge!("sgl_router_worker_load", "Current load on each worker"); + describe_counter!( + "sgl_router_processed_requests_total", + "Total requests processed by each worker" + ); + + describe_gauge!( + "sgl_router_job_queue_depth", + "Current number of pending jobs in the queue" + ); + describe_histogram!( + "sgl_router_job_duration_seconds", + "Job processing duration in seconds by job type" + ); + describe_counter!( + "sgl_router_job_success_total", + "Total successful job completions by job type" + ); + describe_counter!( + "sgl_router_job_failure_total", + "Total failed job completions by job type" + ); + describe_counter!( + "sgl_router_job_queue_full_total", + "Total number of jobs rejected due to queue full" + ); + describe_counter!( + "sgl_router_job_shutdown_rejected_total", + "Total number of jobs rejected due to shutdown" + ); + + describe_counter!( + "sgl_router_policy_decisions_total", + "Total routing policy decisions by policy and worker" + ); + describe_counter!("sgl_router_cache_hits_total", "Total cache hits"); + describe_counter!("sgl_router_cache_misses_total", "Total cache misses"); + describe_gauge!( + "sgl_router_tree_size", + "Current tree size for cache-aware routing" + ); + describe_counter!( + "sgl_router_load_balancing_events_total", + "Total load balancing trigger events" + ); + describe_gauge!("sgl_router_max_load", "Maximum worker load"); + describe_gauge!("sgl_router_min_load", "Minimum worker load"); + + describe_counter!("sgl_router_pd_requests_total", "Total PD requests by route"); + describe_counter!( + "sgl_router_pd_prefill_requests_total", + "Total prefill requests per worker" + ); + describe_counter!( + "sgl_router_pd_decode_requests_total", + "Total decode requests per worker" + ); + describe_counter!( + "sgl_router_pd_errors_total", + "Total PD errors by error type" + ); + describe_counter!( + "sgl_router_pd_prefill_errors_total", + "Total prefill server errors" + ); + describe_counter!( + "sgl_router_pd_decode_errors_total", + "Total decode server errors" + ); + describe_counter!( + "sgl_router_pd_stream_errors_total", + "Total streaming errors per worker" + ); + describe_histogram!( + "sgl_router_pd_request_duration_seconds", + "PD request duration by route" + ); + + describe_counter!( + "sgl_router_discovery_updates_total", + "Total service discovery update events" + ); + describe_gauge!( + "sgl_router_discovery_workers_added", + "Number of workers added in last discovery update" + ); + describe_gauge!( + "sgl_router_discovery_workers_removed", + "Number of workers removed in last discovery update" + ); + + describe_histogram!( + "sgl_router_generate_duration_seconds", + "Generate request duration" + ); + + describe_counter!("sgl_router_embeddings_total", "Total embedding requests"); + describe_histogram!( + "sgl_router_embeddings_duration_seconds", + "Embedding request duration" + ); + describe_counter!( + "sgl_router_embeddings_errors_total", + "Embedding request errors" + ); + describe_gauge!("sgl_router_embeddings_queue_size", "Embedding queue size"); + + describe_gauge!( + "sgl_router_running_requests", + "Number of running requests per worker" + ); + + describe_histogram!( + "sgl_tokenizer_encode_duration_seconds", + "Time to encode text to tokens" + ); + describe_histogram!( + "sgl_tokenizer_decode_duration_seconds", + "Time to decode tokens to text" + ); + describe_histogram!( + "sgl_tokenizer_encode_batch_duration_seconds", + "Time to encode a batch of texts" + ); + describe_counter!( + "sgl_tokenizer_encode_requests_total", + "Total number of encode requests by tokenizer type" + ); + describe_counter!( + "sgl_tokenizer_decode_requests_total", + "Total number of decode requests by tokenizer type" + ); + describe_counter!( + "sgl_tokenizer_encode_errors_total", + "Total number of encode errors by error type" + ); + describe_counter!( + "sgl_tokenizer_decode_errors_total", + "Total number of decode errors by error type" + ); + describe_histogram!( + "sgl_tokenizer_tokens_per_encode", + "Number of tokens produced per encode operation" + ); + describe_histogram!( + "sgl_tokenizer_chars_per_encode", + "Number of characters in input text per encode" + ); + describe_histogram!( + "sgl_tokenizer_tokens_per_decode", + "Number of tokens decoded per operation" + ); + describe_gauge!( + "sgl_tokenizer_vocab_size", + "Vocabulary size of the loaded tokenizer" + ); + + describe_counter!( + "sgl_tokenizer_stop_sequences_detected_total", + "Total stop sequences detected by type" + ); + describe_counter!( + "sgl_tokenizer_partial_matches_total", + "Total partial stop sequence matches (jailed text)" + ); + describe_histogram!( + "sgl_tokenizer_stop_detection_duration_seconds", + "Time to check for stop sequences per token" + ); + + describe_counter!( + "sgl_tokenizer_stream_tokens_total", + "Total tokens processed in streaming decode" + ); + describe_counter!( + "sgl_tokenizer_stream_incomplete_utf8_total", + "Total incomplete UTF-8 sequences detected" + ); + describe_histogram!( + "sgl_tokenizer_stream_step_duration_seconds", + "Time per streaming decode step" + ); + + describe_counter!( + "sgl_tokenizer_factory_loads_total", + "Total tokenizer loads by file type" + ); + describe_counter!( + "sgl_tokenizer_factory_errors_total", + "Total tokenizer loading errors by type" + ); + describe_histogram!( + "sgl_tokenizer_factory_load_duration_seconds", + "Time to load and initialize tokenizer" + ); +} + +pub fn start_prometheus(config: PrometheusConfig) { + init_metrics(); + + let duration_matcher = Matcher::Suffix(String::from("duration_seconds")); + let duration_bucket = [ + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 15.0, 30.0, 45.0, + 60.0, 90.0, 120.0, 180.0, 240.0, + ]; + + let ip_addr: IpAddr = config + .host + .parse() + .unwrap_or(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0))); + let socket_addr = SocketAddr::new(ip_addr, config.port); + + PrometheusBuilder::new() + .with_http_listener(socket_addr) + .upkeep_timeout(Duration::from_secs(5 * 60)) + .set_buckets_for_metric(duration_matcher, &duration_bucket) + .expect("failed to set duration bucket") + .install() + .expect("failed to install Prometheus metrics exporter"); +} + +pub struct RouterMetrics; + +pub struct TokenizerMetrics; + +impl RouterMetrics { + pub fn record_request(route: &str) { + counter!("sgl_router_requests_total", + "route" => route.to_string() + ) + .increment(1); + } + + pub fn record_request_duration(route: &str, duration: Duration) { + histogram!("sgl_router_request_duration_seconds", + "route" => route.to_string() + ) + .record(duration.as_secs_f64()); + } + + pub fn record_request_error(route: &str, error_type: &str) { + counter!("sgl_router_request_errors_total", + "route" => route.to_string(), + "error_type" => error_type.to_string() + ) + .increment(1); + } + + pub fn record_retry(route: &str) { + counter!("sgl_router_retries_total", + "route" => route.to_string() + ) + .increment(1); + } + + pub fn record_retry_backoff_duration(duration: Duration, attempt: u32) { + histogram!("sgl_router_retry_backoff_duration_seconds", + "attempt" => attempt.to_string() + ) + .record(duration.as_secs_f64()); + } + + pub fn record_retries_exhausted(route: &str) { + counter!("sgl_router_retries_exhausted_total", + "route" => route.to_string() + ) + .increment(1); + } + + pub fn set_active_workers(count: usize) { + gauge!("sgl_router_active_workers").set(count as f64); + } + + pub fn set_worker_health(worker_url: &str, healthy: bool) { + gauge!("sgl_router_worker_health", + "worker" => worker_url.to_string() + ) + .set(if healthy { 1.0 } else { 0.0 }); + } + + pub fn set_worker_load(worker_url: &str, load: usize) { + gauge!("sgl_router_worker_load", + "worker" => worker_url.to_string() + ) + .set(load as f64); + } + + pub fn record_processed_request(worker_url: &str) { + counter!("sgl_router_processed_requests_total", + "worker" => worker_url.to_string() + ) + .increment(1); + } + + pub fn record_policy_decision(policy: &str, worker: &str) { + counter!("sgl_router_policy_decisions_total", + "policy" => policy.to_string(), + "worker" => worker.to_string() + ) + .increment(1); + } + + pub fn record_cache_hit() { + counter!("sgl_router_cache_hits_total").increment(1); + } + + pub fn record_cache_miss() { + counter!("sgl_router_cache_misses_total").increment(1); + } + + pub fn set_tree_size(worker: &str, size: usize) { + gauge!("sgl_router_tree_size", + "worker" => worker.to_string() + ) + .set(size as f64); + } + + pub fn record_load_balancing_event() { + counter!("sgl_router_load_balancing_events_total").increment(1); + } + + pub fn set_load_range(max_load: usize, min_load: usize) { + gauge!("sgl_router_max_load").set(max_load as f64); + gauge!("sgl_router_min_load").set(min_load as f64); + } + + pub fn record_pd_request(route: &str) { + counter!("sgl_router_pd_requests_total", + "route" => route.to_string() + ) + .increment(1); + } + + pub fn record_pd_request_duration(route: &str, duration: Duration) { + histogram!("sgl_router_pd_request_duration_seconds", + "route" => route.to_string() + ) + .record(duration.as_secs_f64()); + } + + pub fn record_pd_prefill_request(worker: &str) { + counter!("sgl_router_pd_prefill_requests_total", + "worker" => worker.to_string() + ) + .increment(1); + } + + pub fn record_pd_decode_request(worker: &str) { + counter!("sgl_router_pd_decode_requests_total", + "worker" => worker.to_string() + ) + .increment(1); + } + + pub fn record_pd_error(error_type: &str) { + counter!("sgl_router_pd_errors_total", + "error_type" => error_type.to_string() + ) + .increment(1); + } + + pub fn record_pd_prefill_error(worker: &str) { + counter!("sgl_router_pd_prefill_errors_total", + "worker" => worker.to_string() + ) + .increment(1); + } + + pub fn record_pd_decode_error(worker: &str) { + counter!("sgl_router_pd_decode_errors_total", + "worker" => worker.to_string() + ) + .increment(1); + } + + pub fn record_pd_stream_error(worker: &str) { + counter!("sgl_router_pd_stream_errors_total", + "worker" => worker.to_string() + ) + .increment(1); + } + + pub fn record_discovery_update(added: usize, removed: usize) { + counter!("sgl_router_discovery_updates_total").increment(1); + gauge!("sgl_router_discovery_workers_added").set(added as f64); + gauge!("sgl_router_discovery_workers_removed").set(removed as f64); + } + + pub fn record_generate_duration(duration: Duration) { + histogram!("sgl_router_generate_duration_seconds").record(duration.as_secs_f64()); + } + + pub fn record_embeddings_request() { + counter!("sgl_router_embeddings_total").increment(1); + } + + pub fn record_embeddings_duration(duration: Duration) { + histogram!("sgl_router_embeddings_duration_seconds").record(duration.as_secs_f64()); + } + + pub fn record_embeddings_error(error_type: &str) { + counter!( + "sgl_router_embeddings_errors_total", + "error_type" => error_type.to_string() + ) + .increment(1); + } + + pub fn set_embeddings_queue_size(size: usize) { + gauge!("sgl_router_embeddings_queue_size").set(size as f64); + } + + pub fn record_classify_request() { + counter!("sgl_router_classify_total").increment(1); + } + + pub fn record_classify_duration(duration: Duration) { + histogram!("sgl_router_classify_duration_seconds").record(duration.as_secs_f64()); + } + + pub fn record_classify_error(error_type: &str) { + counter!( + "sgl_router_classify_errors_total", + "error_type" => error_type.to_string() + ) + .increment(1); + } + + pub fn set_classify_queue_size(size: usize) { + gauge!("sgl_router_classify_queue_size").set(size as f64); + } + + pub fn set_running_requests(worker: &str, count: usize) { + gauge!("sgl_router_running_requests", + "worker" => worker.to_string() + ) + .set(count as f64); + } + + pub fn set_cb_state(worker: &str, state_code: u8) { + gauge!("sgl_router_cb_state", + "worker" => worker.to_string() + ) + .set(state_code as f64); + } + + pub fn record_cb_state_transition(worker: &str, from: &str, to: &str) { + counter!("sgl_router_cb_state_transitions_total", + "worker" => worker.to_string(), + "from" => from.to_string(), + "to" => to.to_string() + ) + .increment(1); + } + + pub fn record_cb_outcome(worker: &str, outcome: &str) { + counter!("sgl_router_cb_outcomes_total", + "worker" => worker.to_string(), + "outcome" => outcome.to_string() + ) + .increment(1); + } + + pub fn set_job_queue_depth(depth: usize) { + gauge!("sgl_router_job_queue_depth").set(depth as f64); + } + + pub fn record_job_duration(job_type: &str, duration: Duration) { + histogram!("sgl_router_job_duration_seconds", + "job_type" => job_type.to_string() + ) + .record(duration.as_secs_f64()); + } + + pub fn record_job_success(job_type: &str) { + counter!("sgl_router_job_success_total", + "job_type" => job_type.to_string() + ) + .increment(1); + } + + pub fn record_job_failure(job_type: &str) { + counter!("sgl_router_job_failure_total", + "job_type" => job_type.to_string() + ) + .increment(1); + } + + pub fn record_job_queue_full() { + counter!("sgl_router_job_queue_full_total").increment(1); + } + + pub fn record_job_shutdown_rejected() { + counter!("sgl_router_job_shutdown_rejected_total").increment(1); + } +} + +impl TokenizerMetrics { + pub fn record_encode_request(tokenizer_type: &str) { + counter!("sgl_tokenizer_encode_requests_total", + "tokenizer_type" => tokenizer_type.to_string() + ) + .increment(1); + } + + pub fn record_encode_duration(duration: Duration) { + histogram!("sgl_tokenizer_encode_duration_seconds").record(duration.as_secs_f64()); + } + + pub fn record_encode_error(error_type: &str) { + counter!("sgl_tokenizer_encode_errors_total", + "error_type" => error_type.to_string() + ) + .increment(1); + } + + pub fn record_tokens_per_encode(token_count: usize) { + histogram!("sgl_tokenizer_tokens_per_encode").record(token_count as f64); + } + + pub fn record_chars_per_encode(char_count: usize) { + histogram!("sgl_tokenizer_chars_per_encode").record(char_count as f64); + } + + pub fn record_decode_request(tokenizer_type: &str) { + counter!("sgl_tokenizer_decode_requests_total", + "tokenizer_type" => tokenizer_type.to_string() + ) + .increment(1); + } + + pub fn record_decode_duration(duration: Duration) { + histogram!("sgl_tokenizer_decode_duration_seconds").record(duration.as_secs_f64()); + } + + pub fn record_decode_error(error_type: &str) { + counter!("sgl_tokenizer_decode_errors_total", + "error_type" => error_type.to_string() + ) + .increment(1); + } + + pub fn record_tokens_per_decode(token_count: usize) { + histogram!("sgl_tokenizer_tokens_per_decode").record(token_count as f64); + } + + pub fn record_encode_batch_duration(duration: Duration, batch_size: usize) { + histogram!("sgl_tokenizer_encode_batch_duration_seconds", + "batch_size" => batch_size.to_string() + ) + .record(duration.as_secs_f64()); + } + + pub fn record_stop_sequence_detected(stop_type: &str) { + counter!("sgl_tokenizer_stop_sequences_detected_total", + "type" => stop_type.to_string() + ) + .increment(1); + } + + pub fn record_partial_match() { + counter!("sgl_tokenizer_partial_matches_total").increment(1); + } + + pub fn record_stop_detection_duration(duration: Duration) { + histogram!("sgl_tokenizer_stop_detection_duration_seconds").record(duration.as_secs_f64()); + } + + pub fn record_stream_token() { + counter!("sgl_tokenizer_stream_tokens_total").increment(1); + } + + pub fn record_incomplete_utf8() { + counter!("sgl_tokenizer_stream_incomplete_utf8_total").increment(1); + } + + pub fn record_stream_step_duration(duration: Duration) { + histogram!("sgl_tokenizer_stream_step_duration_seconds").record(duration.as_secs_f64()); + } + + pub fn record_factory_load(file_type: &str) { + counter!("sgl_tokenizer_factory_loads_total", + "file_type" => file_type.to_string() + ) + .increment(1); + } + + pub fn record_factory_error(error_type: &str) { + counter!("sgl_tokenizer_factory_errors_total", + "error_type" => error_type.to_string() + ) + .increment(1); + } + + pub fn record_factory_load_duration(duration: Duration) { + histogram!("sgl_tokenizer_factory_load_duration_seconds").record(duration.as_secs_f64()); + } + + pub fn set_vocab_size(tokenizer_type: &str, size: usize) { + gauge!("sgl_tokenizer_vocab_size", + "tokenizer_type" => tokenizer_type.to_string() + ) + .set(size as f64); + } +} + +#[cfg(test)] +mod tests { + use std::net::TcpListener; + + use super::*; + + #[test] + fn test_prometheus_config_default() { + let config = PrometheusConfig::default(); + assert_eq!(config.port, 29000); + assert_eq!(config.host, "0.0.0.0"); + } + + #[test] + fn test_prometheus_config_custom() { + let config = PrometheusConfig { + port: 8080, + host: "127.0.0.1".to_string(), + }; + assert_eq!(config.port, 8080); + assert_eq!(config.host, "127.0.0.1"); + } + + #[test] + fn test_prometheus_config_clone() { + let config = PrometheusConfig { + port: 9090, + host: "192.168.1.1".to_string(), + }; + let cloned = config.clone(); + assert_eq!(cloned.port, config.port); + assert_eq!(cloned.host, config.host); + } + + #[test] + fn test_valid_ipv4_parsing() { + let test_cases = vec!["127.0.0.1", "192.168.1.1", "0.0.0.0"]; + + for ip_str in test_cases { + let config = PrometheusConfig { + port: 29000, + host: ip_str.to_string(), + }; + + let ip_addr: IpAddr = config.host.parse().unwrap(); + assert!(matches!(ip_addr, IpAddr::V4(_))); + } + } + + #[test] + fn test_valid_ipv6_parsing() { + let test_cases = vec!["::1", "2001:db8::1", "::"]; + + for ip_str in test_cases { + let config = PrometheusConfig { + port: 29000, + host: ip_str.to_string(), + }; + + let ip_addr: IpAddr = config.host.parse().unwrap(); + assert!(matches!(ip_addr, IpAddr::V6(_))); + } + } + + #[test] + fn test_invalid_ip_parsing() { + let test_cases = vec!["invalid", "256.256.256.256", "hostname"]; + + for ip_str in test_cases { + let config = PrometheusConfig { + port: 29000, + host: ip_str.to_string(), + }; + + let ip_addr: IpAddr = config + .host + .parse() + .unwrap_or(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0))); + + assert_eq!(ip_addr, IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0))); + } + } + + #[test] + fn test_socket_addr_creation() { + let test_cases = vec![("127.0.0.1", 8080), ("0.0.0.0", 29000), ("::1", 9090)]; + + for (host, port) in test_cases { + let config = PrometheusConfig { + port, + host: host.to_string(), + }; + + let ip_addr: IpAddr = config.host.parse().unwrap(); + let socket_addr = SocketAddr::new(ip_addr, config.port); + + assert_eq!(socket_addr.port(), port); + assert_eq!(socket_addr.ip().to_string(), host); + } + } + + #[test] + fn test_socket_addr_with_different_ports() { + let ports = vec![0, 80, 8080, 65535]; + + for port in ports { + let config = PrometheusConfig { + port, + host: "127.0.0.1".to_string(), + }; + + let ip_addr: IpAddr = config.host.parse().unwrap(); + let socket_addr = SocketAddr::new(ip_addr, config.port); + + assert_eq!(socket_addr.port(), port); + } + } + + #[test] + fn test_duration_bucket_coverage() { + let test_cases: [(f64, &str); 7] = [ + (0.0005, "sub-millisecond"), + (0.005, "5ms"), + (0.05, "50ms"), + (1.0, "1s"), + (10.0, "10s"), + (60.0, "1m"), + (240.0, "4m"), + ]; + + let buckets: [f64; 20] = [ + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 15.0, 30.0, 45.0, + 60.0, 90.0, 120.0, 180.0, 240.0, + ]; + + for (duration, label) in test_cases { + let bucket_found = buckets + .iter() + .any(|&b| (b - duration).abs() < 0.0001 || b > duration); + assert!(bucket_found, "No bucket found for {} ({})", duration, label); + } + } + + #[test] + fn test_duration_suffix_matcher() { + let matcher = Matcher::Suffix(String::from("duration_seconds")); + + let _matching_metrics = [ + "request_duration_seconds", + "response_duration_seconds", + "sgl_router_request_duration_seconds", + ]; + + let _non_matching_metrics = ["duration_total", "duration_seconds_total", "other_metric"]; + + match matcher { + Matcher::Suffix(suffix) => assert_eq!(suffix, "duration_seconds"), + _ => panic!("Expected Suffix matcher"), + } + } + + #[test] + fn test_prometheus_builder_configuration() { + let _config = PrometheusConfig::default(); + + let duration_matcher = Matcher::Suffix(String::from("duration_seconds")); + let duration_bucket = [ + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 15.0, 30.0, 45.0, + 60.0, 90.0, 120.0, 180.0, 240.0, + ]; + + assert_eq!(duration_bucket.len(), 20); + + match duration_matcher { + Matcher::Suffix(s) => assert_eq!(s, "duration_seconds"), + _ => panic!("Expected Suffix matcher"), + } + } + + #[test] + fn test_upkeep_timeout_duration() { + let timeout = Duration::from_secs(5 * 60); + assert_eq!(timeout.as_secs(), 300); + } + + #[test] + fn test_custom_buckets_for_different_metrics() { + let request_buckets = [0.001, 0.01, 0.1, 1.0, 10.0]; + let generate_buckets = [0.1, 0.5, 1.0, 5.0, 30.0, 60.0]; + + assert_eq!(request_buckets.len(), 5); + assert_eq!(generate_buckets.len(), 6); + + for i in 1..request_buckets.len() { + assert!(request_buckets[i] > request_buckets[i - 1]); + } + + for i in 1..generate_buckets.len() { + assert!(generate_buckets[i] > generate_buckets[i - 1]); + } + } + + #[test] + fn test_metrics_static_methods() { + RouterMetrics::record_request("/generate"); + RouterMetrics::record_request_duration("/generate", Duration::from_millis(100)); + RouterMetrics::record_request_error("/generate", "timeout"); + RouterMetrics::record_retry("/generate"); + + RouterMetrics::set_active_workers(5); + RouterMetrics::set_worker_health("http://worker1", true); + RouterMetrics::set_worker_load("http://worker1", 10); + RouterMetrics::record_processed_request("http://worker1"); + + RouterMetrics::record_policy_decision("random", "http://worker1"); + RouterMetrics::record_cache_hit(); + RouterMetrics::record_cache_miss(); + RouterMetrics::set_tree_size("http://worker1", 1000); + RouterMetrics::record_load_balancing_event(); + RouterMetrics::set_load_range(20, 5); + + RouterMetrics::record_pd_request("/v1/chat/completions"); + RouterMetrics::record_pd_request_duration("/v1/chat/completions", Duration::from_secs(1)); + RouterMetrics::record_pd_prefill_request("http://prefill1"); + RouterMetrics::record_pd_decode_request("http://decode1"); + RouterMetrics::record_pd_error("invalid_request"); + RouterMetrics::record_pd_prefill_error("http://prefill1"); + RouterMetrics::record_pd_decode_error("http://decode1"); + RouterMetrics::record_pd_stream_error("http://decode1"); + + RouterMetrics::record_discovery_update(3, 1); + RouterMetrics::record_generate_duration(Duration::from_secs(2)); + RouterMetrics::set_running_requests("http://worker1", 15); + } + + #[test] + fn test_tokenizer_metrics_static_methods() { + TokenizerMetrics::record_encode_request("huggingface"); + TokenizerMetrics::record_encode_duration(Duration::from_millis(10)); + TokenizerMetrics::record_encode_error("invalid_input"); + TokenizerMetrics::record_tokens_per_encode(100); + TokenizerMetrics::record_chars_per_encode(500); + + TokenizerMetrics::record_decode_request("huggingface"); + TokenizerMetrics::record_decode_duration(Duration::from_millis(5)); + TokenizerMetrics::record_decode_error("invalid_tokens"); + TokenizerMetrics::record_tokens_per_decode(50); + + TokenizerMetrics::record_encode_batch_duration(Duration::from_millis(100), 10); + + TokenizerMetrics::record_stop_sequence_detected("token"); + TokenizerMetrics::record_stop_sequence_detected("string"); + TokenizerMetrics::record_partial_match(); + TokenizerMetrics::record_stop_detection_duration(Duration::from_micros(100)); + + TokenizerMetrics::record_stream_token(); + TokenizerMetrics::record_incomplete_utf8(); + TokenizerMetrics::record_stream_step_duration(Duration::from_micros(50)); + + TokenizerMetrics::record_factory_load("json"); + TokenizerMetrics::record_factory_error("unsupported_format"); + TokenizerMetrics::record_factory_load_duration(Duration::from_millis(200)); + + TokenizerMetrics::set_vocab_size("huggingface", 50000); + } + + #[test] + fn test_port_already_in_use() { + let port = 29123; + + if let Ok(_listener) = TcpListener::bind(("127.0.0.1", port)) { + let config = PrometheusConfig { + port, + host: "127.0.0.1".to_string(), + }; + + assert_eq!(config.port, port); + } + } + + #[test] + fn test_metrics_endpoint_accessibility() { + let config = PrometheusConfig { + port: 29000, + host: "127.0.0.1".to_string(), + }; + + let ip_addr: IpAddr = config.host.parse().unwrap(); + let socket_addr = SocketAddr::new(ip_addr, config.port); + + assert_eq!(socket_addr.to_string(), "127.0.0.1:29000"); + } + + #[test] + fn test_concurrent_metric_updates() { + use std::{ + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + thread, + }; + + let done = Arc::new(AtomicBool::new(false)); + let mut handles = vec![]; + + for i in 0..3 { + let done_clone = done.clone(); + let handle = thread::spawn(move || { + let worker = format!("http://worker{}", i); + while !done_clone.load(Ordering::Relaxed) { + RouterMetrics::set_worker_load(&worker, i * 10); + RouterMetrics::record_processed_request(&worker); + thread::sleep(Duration::from_millis(1)); + } + }); + handles.push(handle); + } + + thread::sleep(Duration::from_millis(10)); + done.store(true, Ordering::Relaxed); + + for handle in handles { + handle.join().unwrap(); + } + } + + #[test] + fn test_empty_string_metrics() { + RouterMetrics::record_request(""); + RouterMetrics::set_worker_health("", true); + RouterMetrics::record_policy_decision("", ""); + } + + #[test] + fn test_very_long_metric_labels() { + let long_label = "a".repeat(1000); + + RouterMetrics::record_request(&long_label); + RouterMetrics::set_worker_health(&long_label, false); + } + + #[test] + fn test_special_characters_in_labels() { + let special_labels = [ + "test/with/slashes", + "test-with-dashes", + "test_with_underscores", + "test.with.dots", + "test:with:colons", + ]; + + for label in special_labels { + RouterMetrics::record_request(label); + RouterMetrics::set_worker_health(label, true); + } + } + + #[test] + fn test_extreme_metric_values() { + RouterMetrics::set_active_workers(0); + RouterMetrics::set_active_workers(usize::MAX); + + RouterMetrics::set_worker_load("worker", 0); + RouterMetrics::set_worker_load("worker", usize::MAX); + + RouterMetrics::record_request_duration("route", Duration::from_nanos(1)); + RouterMetrics::record_request_duration("route", Duration::from_secs(86400)); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/middleware.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/middleware.rs new file mode 100644 index 00000000..e4811b1e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/middleware.rs @@ -0,0 +1,556 @@ +use std::{ + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, + time::{Duration, Instant}, +}; + +use axum::{ + body::Body, + extract::{Request, State}, + http::{header, HeaderValue, StatusCode}, + middleware::Next, + response::{IntoResponse, Response}, +}; +use rand::Rng; +use subtle::ConstantTimeEq; +use tokio::sync::{mpsc, oneshot}; +use tower::{Layer, Service}; +use tower_http::trace::{MakeSpan, OnRequest, OnResponse, TraceLayer}; +use tracing::{debug, error, field::Empty, info, info_span, warn, Span}; + +pub use crate::core::token_bucket::TokenBucket; +use crate::{metrics::RouterMetrics, server::AppState}; + +#[derive(Clone)] +pub struct AuthConfig { + pub api_key: Option, +} + +/// Middleware to validate Bearer token against configured API key +/// Only active when router has an API key configured +pub async fn auth_middleware( + State(auth_config): State, + request: Request, + next: Next, +) -> Result { + if let Some(expected_key) = &auth_config.api_key { + // Extract Authorization header + let auth_header = request + .headers() + .get(header::AUTHORIZATION) + .and_then(|h| h.to_str().ok()); + + match auth_header { + Some(header_value) if header_value.starts_with("Bearer ") => { + let token = &header_value[7..]; // Skip "Bearer " + // Use constant-time comparison to prevent timing attacks + let token_bytes = token.as_bytes(); + let expected_bytes = expected_key.as_bytes(); + + // Check if lengths match first (this is not constant-time but necessary) + if token_bytes.len() != expected_bytes.len() { + return Err(StatusCode::UNAUTHORIZED); + } + + // Constant-time comparison of the actual values + if token_bytes.ct_eq(expected_bytes).unwrap_u8() != 1 { + return Err(StatusCode::UNAUTHORIZED); + } + } + _ => return Err(StatusCode::UNAUTHORIZED), + } + } + + Ok(next.run(request).await) +} + +/// Generate OpenAI-compatible request ID based on endpoint +fn generate_request_id(path: &str) -> String { + let prefix = if path.contains("/chat/completions") { + "chatcmpl-" + } else if path.contains("/completions") { + "cmpl-" + } else if path.contains("/generate") { + "gnt-" + } else if path.contains("/responses") { + "resp-" + } else { + "req-" + }; + + // Generate a random string similar to OpenAI's format + let chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + let mut rng = rand::rng(); + let random_part: String = (0..24) + .map(|_| { + let idx = rng.random_range(0..chars.len()); + chars.chars().nth(idx).unwrap() + }) + .collect(); + + format!("{}{}", prefix, random_part) +} + +/// Extension type for storing request ID +#[derive(Clone, Debug)] +pub struct RequestId(pub String); + +/// Tower Layer for request ID middleware +#[derive(Clone)] +pub struct RequestIdLayer { + headers: Arc>, +} + +impl RequestIdLayer { + pub fn new(headers: Vec) -> Self { + Self { + headers: Arc::new(headers), + } + } +} + +impl Layer for RequestIdLayer { + type Service = RequestIdMiddleware; + + fn layer(&self, inner: S) -> Self::Service { + RequestIdMiddleware { + inner, + headers: self.headers.clone(), + } + } +} + +/// Tower Service for request ID middleware +#[derive(Clone)] +pub struct RequestIdMiddleware { + inner: S, + headers: Arc>, +} + +impl Service for RequestIdMiddleware +where + S: Service + Send + 'static, + S::Future: Send + 'static, +{ + type Response = S::Response; + type Error = S::Error; + type Future = std::pin::Pin< + Box> + Send>, + >; + + fn poll_ready( + &mut self, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, mut req: Request) -> Self::Future { + let headers = self.headers.clone(); + + // Extract request ID from headers or generate new one + let mut request_id = None; + + for header_name in headers.iter() { + if let Some(header_value) = req.headers().get(header_name) { + if let Ok(value) = header_value.to_str() { + request_id = Some(value.to_string()); + break; + } + } + } + + let request_id = request_id.unwrap_or_else(|| generate_request_id(req.uri().path())); + + // Insert request ID into request extensions for other middleware/handlers to use + req.extensions_mut().insert(RequestId(request_id.clone())); + + // Call the inner service + let future = self.inner.call(req); + + Box::pin(async move { + let mut response = future.await?; + + // Add request ID to response headers + response.headers_mut().insert( + "x-request-id", + HeaderValue::from_str(&request_id) + .unwrap_or_else(|_| HeaderValue::from_static("invalid-request-id")), + ); + + Ok(response) + }) + } +} + +/// Custom span maker that includes request ID +#[derive(Clone, Debug)] +pub struct RequestSpan; + +impl MakeSpan for RequestSpan { + fn make_span(&mut self, request: &Request) -> Span { + // Don't try to extract request ID here - it won't be available yet + // The RequestIdLayer runs after TraceLayer creates the span + info_span!( + "http_request", + method = %request.method(), + uri = %request.uri(), + version = ?request.version(), + request_id = Empty, // Will be set later + status_code = Empty, + latency = Empty, + error = Empty, + ) + } +} + +/// Custom on_request handler +#[derive(Clone, Debug)] +pub struct RequestLogger; + +impl OnRequest for RequestLogger { + fn on_request(&mut self, request: &Request, span: &Span) { + let _enter = span.enter(); + + // Try to get the request ID from extensions + // This will work if RequestIdLayer has already run + if let Some(request_id) = request.extensions().get::() { + span.record("request_id", request_id.0.as_str()); + } + + // Log the request start + info!( + target: "sgl_model_gateway::request", + "started processing request" + ); + } +} + +/// Custom on_response handler +#[derive(Clone, Debug)] +pub struct ResponseLogger { + _start_time: Instant, +} + +impl Default for ResponseLogger { + fn default() -> Self { + Self { + _start_time: Instant::now(), + } + } +} + +impl OnResponse for ResponseLogger { + fn on_response(self, response: &Response, latency: Duration, span: &Span) { + let status = response.status(); + + // Record these in the span for structured logging/observability tools + span.record("status_code", status.as_u16()); + span.record("latency", format!("{:?}", latency)); + + // Log the response completion + let _enter = span.enter(); + if status.is_server_error() { + error!( + target: "sgl_model_gateway::response", + "request failed with server error" + ); + } else if status.is_client_error() { + warn!( + target: "sgl_model_gateway::response", + "request failed with client error" + ); + } else { + info!( + target: "sgl_model_gateway::response", + "finished processing request" + ); + } + } +} + +/// Create a configured TraceLayer for HTTP logging +/// Note: Actual request/response logging with request IDs is done in RequestIdService +pub fn create_logging_layer() -> TraceLayer< + tower_http::classify::SharedClassifier, + RequestSpan, + RequestLogger, + ResponseLogger, +> { + TraceLayer::new_for_http() + .make_span_with(RequestSpan) + .on_request(RequestLogger) + .on_response(ResponseLogger::default()) +} + +/// Structured logging data for requests +#[derive(Debug, serde::Serialize)] +pub struct RequestLogEntry { + pub timestamp: String, + pub request_id: String, + pub method: String, + pub uri: String, + pub status: u16, + pub latency_ms: u64, + pub user_agent: Option, + pub remote_addr: Option, + pub error: Option, +} + +/// Log a request with structured data +pub fn log_request(entry: RequestLogEntry) { + if entry.status >= 500 { + tracing::error!( + target: "sgl_model_gateway::http", + request_id = %entry.request_id, + method = %entry.method, + uri = %entry.uri, + status = entry.status, + latency_ms = entry.latency_ms, + user_agent = ?entry.user_agent, + remote_addr = ?entry.remote_addr, + error = ?entry.error, + "HTTP request failed" + ); + } else if entry.status >= 400 { + tracing::warn!( + target: "sgl_model_gateway::http", + request_id = %entry.request_id, + method = %entry.method, + uri = %entry.uri, + status = entry.status, + latency_ms = entry.latency_ms, + user_agent = ?entry.user_agent, + remote_addr = ?entry.remote_addr, + "HTTP request client error" + ); + } else { + tracing::info!( + target: "sgl_model_gateway::http", + request_id = %entry.request_id, + method = %entry.method, + uri = %entry.uri, + status = entry.status, + latency_ms = entry.latency_ms, + user_agent = ?entry.user_agent, + remote_addr = ?entry.remote_addr, + "HTTP request completed" + ); + } +} + +/// Request queue entry +pub struct QueuedRequest { + /// Time when the request was queued + queued_at: Instant, + /// Channel to send the permit back when acquired + permit_tx: oneshot::Sender>, +} + +/// Queue metrics for monitoring +#[derive(Debug, Default)] +pub struct QueueMetrics { + pub total_queued: AtomicU64, + pub current_queued: AtomicU64, + pub total_timeout: AtomicU64, + pub total_rejected: AtomicU64, +} + +/// Queue processor that handles queued requests +pub struct QueueProcessor { + token_bucket: Arc, + queue_rx: mpsc::Receiver, + queue_timeout: Duration, +} + +impl QueueProcessor { + pub fn new( + token_bucket: Arc, + queue_rx: mpsc::Receiver, + queue_timeout: Duration, + ) -> Self { + Self { + token_bucket, + queue_rx, + queue_timeout, + } + } + + pub async fn run(mut self) { + info!("Starting concurrency queue processor"); + + // Process requests in a single task to reduce overhead + while let Some(queued) = self.queue_rx.recv().await { + // Check timeout immediately + let elapsed = queued.queued_at.elapsed(); + if elapsed >= self.queue_timeout { + warn!("Request already timed out in queue"); + let _ = queued.permit_tx.send(Err(StatusCode::REQUEST_TIMEOUT)); + continue; + } + + let remaining_timeout = self.queue_timeout - elapsed; + + // Try to acquire token for this request + if self.token_bucket.try_acquire(1.0).await.is_ok() { + // Got token immediately + debug!("Queue: acquired token immediately for queued request"); + let _ = queued.permit_tx.send(Ok(())); + } else { + // Need to wait for token + let token_bucket = self.token_bucket.clone(); + + // Spawn task only when we actually need to wait + tokio::spawn(async move { + if token_bucket + .acquire_timeout(1.0, remaining_timeout) + .await + .is_ok() + { + debug!("Queue: acquired token after waiting"); + let _ = queued.permit_tx.send(Ok(())); + } else { + warn!("Queue: request timed out waiting for token"); + let _ = queued.permit_tx.send(Err(StatusCode::REQUEST_TIMEOUT)); + } + }); + } + } + + warn!("Concurrency queue processor shutting down"); + } +} + +/// State for the concurrency limiter +pub struct ConcurrencyLimiter { + pub queue_tx: Option>, +} + +impl ConcurrencyLimiter { + /// Create new concurrency limiter with optional queue + pub fn new( + token_bucket: Option>, + queue_size: usize, + queue_timeout: Duration, + ) -> (Self, Option) { + match (token_bucket, queue_size) { + (None, _) => (Self { queue_tx: None }, None), + (Some(bucket), size) if size > 0 => { + let (queue_tx, queue_rx) = mpsc::channel(size); + let processor = QueueProcessor::new(bucket, queue_rx, queue_timeout); + ( + Self { + queue_tx: Some(queue_tx), + }, + Some(processor), + ) + } + (Some(_), _) => (Self { queue_tx: None }, None), + } + } +} + +/// Middleware function for concurrency limiting with optional queuing +pub async fn concurrency_limit_middleware( + State(app_state): State>, + request: Request, + next: Next, +) -> Response { + let token_bucket = match &app_state.context.rate_limiter { + Some(bucket) => bucket.clone(), + None => { + // Rate limiting disabled, pass through immediately + return next.run(request).await; + } + }; + + // Static counter for embeddings queue size + static EMBEDDINGS_QUEUE_SIZE: AtomicU64 = AtomicU64::new(0); + + // Identify if this is an embeddings request based on path + let is_embeddings = request.uri().path().contains("/v1/embeddings"); + + // Try to acquire token immediately + if token_bucket.try_acquire(1.0).await.is_ok() { + debug!("Acquired token immediately"); + let response = next.run(request).await; + + // Return the token to the bucket + token_bucket.return_tokens(1.0).await; + + response + } else { + // No tokens available, try to queue if enabled + if let Some(queue_tx) = &app_state.concurrency_queue_tx { + debug!("No tokens available, attempting to queue request"); + + // Create a channel for the token response + let (permit_tx, permit_rx) = oneshot::channel(); + + let queued = QueuedRequest { + queued_at: Instant::now(), + permit_tx, + }; + + // Try to send to queue + match queue_tx.try_send(queued) { + Ok(_) => { + // On successful enqueue, update embeddings queue gauge if applicable + if is_embeddings { + let new_val = EMBEDDINGS_QUEUE_SIZE.fetch_add(1, Ordering::Relaxed) + 1; + RouterMetrics::set_embeddings_queue_size(new_val as usize); + } + + // Wait for token from queue processor + match permit_rx.await { + Ok(Ok(())) => { + debug!("Acquired token from queue"); + // Dequeue for embeddings + if is_embeddings { + let new_val = + EMBEDDINGS_QUEUE_SIZE.fetch_sub(1, Ordering::Relaxed) - 1; + RouterMetrics::set_embeddings_queue_size(new_val as usize); + } + + let response = next.run(request).await; + + // Return the token to the bucket + token_bucket.return_tokens(1.0).await; + + response + } + Ok(Err(status)) => { + warn!("Queue returned error status: {}", status); + // Dequeue for embeddings on error + if is_embeddings { + let new_val = + EMBEDDINGS_QUEUE_SIZE.fetch_sub(1, Ordering::Relaxed) - 1; + RouterMetrics::set_embeddings_queue_size(new_val as usize); + } + status.into_response() + } + Err(_) => { + error!("Queue response channel closed"); + // Dequeue for embeddings on channel error + if is_embeddings { + let new_val = + EMBEDDINGS_QUEUE_SIZE.fetch_sub(1, Ordering::Relaxed) - 1; + RouterMetrics::set_embeddings_queue_size(new_val as usize); + } + StatusCode::INTERNAL_SERVER_ERROR.into_response() + } + } + } + Err(_) => { + warn!("Request queue is full, returning 429"); + StatusCode::TOO_MANY_REQUESTS.into_response() + } + } + } else { + warn!("No tokens available and queuing is disabled, returning 429"); + StatusCode::TOO_MANY_REQUESTS.into_response() + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/bucket.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/bucket.rs new file mode 100644 index 00000000..ea4a3a44 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/bucket.rs @@ -0,0 +1,1167 @@ +use std::{ + collections::{HashMap, HashSet, VecDeque}, + sync::{Arc, Mutex, RwLock}, + thread, + time::{Duration, SystemTime}, +}; + +use dashmap::DashMap; +use rand::Rng; +use tracing::{error, info, warn}; +use uuid::Uuid; + +use super::{get_healthy_worker_indices, BucketConfig, LoadBalancingPolicy}; +use crate::core::Worker; + +#[derive(Debug)] +pub struct BucketPolicy { + config: BucketConfig, + buckets: Arc>>>, + adjustment_handle: Option>, +} + +impl Default for BucketPolicy { + fn default() -> Self { + Self::new() + } +} + +impl Drop for BucketPolicy { + fn drop(&mut self) { + if let Some(handle) = self.adjustment_handle.take() { + drop(handle); + } + } +} + +impl BucketPolicy { + pub fn new() -> Self { + Self::with_config(BucketConfig::default()) + } + + pub fn with_config(config: BucketConfig) -> Self { + let buckets = Arc::new(DashMap::>>::new()); + + let adjustment_handle = { + let buckets_clone = Arc::clone(&buckets); + + let interval_secs = config.bucket_adjust_interval_secs; + + Some(thread::spawn(move || loop { + thread::sleep(Duration::from_secs(interval_secs as u64)); + + for bucket_ref in buckets_clone.iter() { + let model_id = bucket_ref.key(); + let bucket = bucket_ref.value(); + match bucket.write() { + Ok(mut bucket_guard) => { + bucket_guard.adjust_boundary(); + } + Err(e) => { + eprintln!( + "Failed to acquire write lock for bucket {}: {}", + model_id, e + ); + } + } + } + })) + }; + + Self { + config, + buckets, + adjustment_handle, + } + } + + pub fn init_prefill_worker_urls(&self, prefill_workers: &[Arc]) { + // Group workers by model + let mut model_workers: HashMap>> = HashMap::new(); + for worker in prefill_workers { + // Use "default" for unknown/empty model_ids for backward compatibility + let model_id = worker.model_id(); + let model_key = if model_id.is_empty() || model_id == "unknown" { + "default" + } else { + model_id + }; + model_workers + .entry(model_key.to_string()) + .or_default() + .push(worker); + } + // Initialize bucket for each model + for (model_key, model_workers) in model_workers { + let bucket = self + .buckets + .entry(model_key) + .or_insert_with(|| { + Arc::new(RwLock::new(Bucket::new( + self.config.bucket_adjust_interval_secs * 1000, + ))) + }) + .clone(); + + let worker_urls: Vec = model_workers + .iter() + .map(|worker| worker.url().to_string()) + .collect(); + + let lock_result = bucket.write(); + if let Ok(mut bucket_guard) = lock_result { + bucket_guard.init_prefill_worker_urls(worker_urls); + } else { + eprintln!("Failed to acquire write lock for bucket initialization"); + } + } + } + + pub fn add_prefill_url(&self, worker: &dyn Worker) { + let model_id = worker.model_id(); + let model_key = if model_id.is_empty() || model_id == "unknown" { + "default" + } else { + model_id + }; + let bucket = self + .buckets + .entry(model_key.to_string()) + .or_insert_with(|| { + Arc::new(RwLock::new(Bucket::new( + self.config.bucket_adjust_interval_secs * 1000, + ))) + }) + .clone(); + + let lock_result = bucket.write(); + if let Ok(mut bucket_guard) = lock_result { + let worker_url = worker.url().to_string(); + + let prefill_worker_urls_clone = { + let mut prefill_worker_urls = bucket_guard.prefill_worker_urls.lock().unwrap(); + if !prefill_worker_urls.contains(&worker_url) { + prefill_worker_urls.push(worker_url.clone()); + } + let cloned = prefill_worker_urls.clone(); + + let mut chars_per_url = bucket_guard.chars_per_url.lock().unwrap(); + chars_per_url.entry(worker_url.clone()).or_insert(0); + + cloned + }; + + bucket_guard.init_prefill_worker_urls(prefill_worker_urls_clone); + + info!( + "Added worker {} to bucket for model {}", + worker_url, model_key + ); + } else { + error!( + "Failed to acquire write lock for bucket of model {}", + model_key + ); + } + } + + pub fn remove_prefill_url(&self, worker: &dyn Worker) { + let model_id = worker.model_id(); + let model_key = if model_id.is_empty() || model_id == "unknown" { + "default" + } else { + model_id + }; + + if let Some(bucket_entry) = self.buckets.get(model_key) { + let bucket = bucket_entry.value(); + let worker_url = worker.url().to_string(); + + let lock_result = bucket.write(); + if let Ok(mut bucket_guard) = lock_result { + let (updated_len, updated_urls) = { + let mut prefill_worker_urls = bucket_guard.prefill_worker_urls.lock().unwrap(); + prefill_worker_urls.retain(|u| u != &worker_url); + let len = prefill_worker_urls.len(); + let urls_clone = prefill_worker_urls.clone(); + + let mut chars_per_url = bucket_guard.chars_per_url.lock().unwrap(); + chars_per_url.remove(&worker_url); + + (len, urls_clone) + }; + + bucket_guard.bucket_cnt = updated_len; + + if updated_len > 0 { + bucket_guard.init_prefill_worker_urls(updated_urls); + } + + info!( + "Removed worker {} from bucket for model {} (remaining workers: {})", + worker_url, model_key, bucket_guard.bucket_cnt + ); + } else { + error!( + "Failed to acquire write lock for bucket of model {}", + model_key + ); + } + } else { + warn!( + "No bucket found for model {} when trying to remove worker", + model_key + ); + } + } +} + +impl LoadBalancingPolicy for BucketPolicy { + fn select_worker( + &self, + workers: &[Arc], + request_text: Option<&str>, + ) -> Option { + let healthy_indices = get_healthy_worker_indices(workers); + + if healthy_indices.is_empty() { + return None; + } + + let char_count = match request_text { + None => 0, + Some(text) => text.chars().count(), + }; + + // Determine the model for this set of workers (router pre-filters by model) + // All workers should be from the same model + let first_model = workers[healthy_indices[0]].model_id(); + let model_key = if first_model.is_empty() || first_model == "unknown" { + "default" + } else { + first_model + }; + + let bucket = self + .buckets + .get(model_key) + .map(|entry| entry.value().clone()); + let prefill_url = if let Some(bucket) = bucket { + let (choiced_url, chars_per_url_snapshot) = { + let buc = bucket.read().unwrap(); + let chars_per_url_snapshot = buc.chars_per_url.lock().unwrap().clone(); + let choiced_url = buc.find_boundary(char_count); + (choiced_url, chars_per_url_snapshot) + }; + let max_load = chars_per_url_snapshot.values().copied().max().unwrap_or(0); + let min_load = chars_per_url_snapshot.values().copied().min().unwrap_or(0); + let abs_diff = max_load.saturating_sub(min_load); + let rel_threshold = self.config.balance_rel_threshold * min_load as f32; + let is_imbalanced = + abs_diff > self.config.balance_abs_threshold && max_load as f32 > rel_threshold; + info!( + "Current PD instance status | is_imbalanced={}", + is_imbalanced + ); + + let mut rng = rand::rng(); + let prefill_url = if is_imbalanced { + info!("select prefill instance by Load Balance policy"); + let min_url = chars_per_url_snapshot + .iter() + .min_by_key(|(_, &chars)| chars) + .map(|(url, _)| url.clone()) + .unwrap_or_else(|| { + let idx = rng.random_range(0..healthy_indices.len()); + let url = workers[healthy_indices[idx]].url(); + warn!("No URL found, randomly selecting: {}", url); + url.to_string() + }); + min_url + } else { + info!("select prefill instance by Bucket policy"); + match choiced_url { + Some(url) if !url.is_empty() => url, + _ => { + let idx = rng.random_range(0..healthy_indices.len()); + let selected_url = workers[healthy_indices[idx]].url(); + warn!("Boundary not found, randomly selection: {}", selected_url); + selected_url.to_string() + } + } + }; + + { + let mut buc = bucket.write().unwrap(); + buc.post_process_request(char_count, prefill_url.clone()); + } + + prefill_url + } else { + warn!( + "No bucket found for model {}, randomly selecting healthy worker", + model_key + ); + let mut rng = rand::rng(); + let idx = rng.random_range(0..healthy_indices.len()); + let selected_worker = &workers[healthy_indices[idx]]; + let prefill_url = selected_worker.url().to_string(); + prefill_url + }; + + workers.iter().position(|w| w.url() == prefill_url) + } + + fn select_worker_pair( + &self, + prefill_workers: &[Arc], + decode_workers: &[Arc], + request_text: Option<&str>, + ) -> Option<(usize, usize)> { + let prefill_idx = self.select_worker(prefill_workers, request_text)?; + + let healthy_decode = get_healthy_worker_indices(decode_workers); + if healthy_decode.is_empty() { + return None; + } + + let mut rng = rand::rng(); + let decode_idx = rng.random_range(0..healthy_decode.len()); + + Some((prefill_idx, decode_idx)) + } + + fn name(&self) -> &'static str { + "bucket" + } + + fn needs_request_text(&self) -> bool { + true // Bucket policy needs request text + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +#[derive(Debug, Clone)] +pub struct Bucket { + l_max: usize, + bucket_cnt: usize, + pub prefill_worker_urls: Arc>>, + load_total: usize, + pub period: usize, + bucket_load: usize, + boundary: Vec, + request_list: VecDeque, + t_req_loads: HashMap, + pub chars_per_url: Arc>>, +} + +#[derive(Debug, Clone)] +pub struct SequencerRequest { + pub id: String, + pub char_cnt: usize, + pub timestamp: SystemTime, + pub prefill_worker_url: String, +} + +#[derive(Debug, Clone)] +pub struct Boundary { + pub url: String, + pub range: [usize; 2], +} + +impl Boundary { + pub fn new(url: String, range: [usize; 2]) -> Self { + Boundary { url, range } + } +} + +impl Bucket { + pub fn new(period: usize) -> Self { + let l_max = 4096; + + let bucket_cnt = 0; + + let load_total = 0; + let bucket_load = 0; + + let t_req_loads = HashMap::new(); + let request_list = VecDeque::new(); + + let initial_map = HashMap::new(); + + let boundary = Vec::new(); + + let prefill_worker_urls = Arc::new(Mutex::new(Vec::new())); + + Bucket { + l_max, + bucket_cnt, + prefill_worker_urls, + load_total, + period, + bucket_load, + boundary, + request_list, + t_req_loads, + chars_per_url: Arc::new(Mutex::new(initial_map)), + } + } + + pub fn init_prefill_worker_urls(&mut self, prefill_worker_urls: Vec) { + let bucket_cnt = prefill_worker_urls.len(); + self.bucket_cnt = bucket_cnt; + let mut urls_lock = self.prefill_worker_urls.lock().unwrap(); + *urls_lock = prefill_worker_urls.clone(); + + let mut chars_lock = self.chars_per_url.lock().unwrap(); + chars_lock.clear(); + + for url in prefill_worker_urls.iter() { + chars_lock.insert(url.clone(), 0); + } + + let worker_cnt = bucket_cnt; + let boundary = if worker_cnt == 0 { + Vec::new() + } else { + let gap = self.l_max / worker_cnt; + self.l_max = usize::MAX; + prefill_worker_urls + .iter() + .enumerate() + .map(|(i, url)| { + let min = i * gap; + let max = if i == worker_cnt - 1 { + self.l_max + } else { + (i + 1) * gap - 1 + }; + Boundary::new(url.clone(), [min, max]) + }) + .collect() + }; + + self.boundary = boundary; + info!("Init boundary:{:?}", self.boundary); + } + + pub fn post_process_request(&mut self, char_cnt: usize, prefill_url: String) { + { + let mut map = self.chars_per_url.lock().unwrap(); + *map.entry(prefill_url.clone()).or_insert(0) += char_cnt; + } + + let now = SystemTime::now(); + let time_window_duration = Duration::from_millis(self.period as u64); + let mut removed_load = 0; + + while let Some(req) = self.request_list.front() { + let expired = match now.duration_since(req.timestamp) { + Ok(duration) => duration > time_window_duration, + Err(_) => true, + }; + + if !expired { + break; + } + + if let Some(removed_req) = self.request_list.pop_front() { + self.t_req_loads.remove(&removed_req.id); + removed_load += removed_req.char_cnt; + + let mut map = self.chars_per_url.lock().unwrap(); + if let Some(count) = map.get_mut(&removed_req.prefill_worker_url) { + *count = count.saturating_sub(removed_req.char_cnt); + } + } + } + + self.load_total = self.load_total.saturating_sub(removed_load); + + let id = Uuid::new_v4().to_string(); + + self.t_req_loads.insert(id.clone(), char_cnt); + + self.request_list.push_back(SequencerRequest { + id, + char_cnt, + timestamp: now, + prefill_worker_url: prefill_url, + }); + + self.load_total = self.load_total.saturating_add(char_cnt); + } + + pub fn find_boundary(&self, char_count: usize) -> Option { + let mut left = 0; + let mut right = self.boundary.len(); + let mut _steps = 0; + + while left < right { + _steps += 1; + let mid = left + (right - left) / 2; + let range = self.boundary[mid].range; + + if char_count < range[0] { + right = mid; + } else if char_count > range[1] { + left = mid + 1; + } else { + return Some(self.boundary[mid].url.clone()); + } + } + None + } + + pub fn get_total_load(&self) -> usize { + self.load_total + } + + fn update_workers_cnt(&mut self) { + let pwu = self.prefill_worker_urls.lock().unwrap(); + self.bucket_cnt = pwu.len(); + + let mut char_map = self.chars_per_url.lock().unwrap(); + let current_urls: HashSet<_> = char_map.keys().cloned().collect(); + let new_urls: HashSet<_> = pwu.iter().cloned().collect(); + + for url in new_urls.difference(¤t_urls) { + char_map.insert(url.clone(), 0); + } + + for url in current_urls.difference(&new_urls) { + if char_map.get(url) == Some(&0) { + char_map.remove(url); + } + } + } + + pub fn adjust_boundary(&mut self) { + if self.t_req_loads.is_empty() { + return; + } + + self.update_workers_cnt(); + let worker_cnt = self.bucket_cnt; + if worker_cnt == 0 { + return; + } + let new_single_bucket_load = self.get_total_load() / worker_cnt; + let old_single_bucket_load = self.bucket_load; + + if new_single_bucket_load <= 2 * old_single_bucket_load + && (old_single_bucket_load <= 2 * new_single_bucket_load && old_single_bucket_load != 0) + { + info!("No need to adjust the bucket boundaries."); + return; + } + + info!("Before adjusting boundary | {:?}", self.boundary); + self.bucket_load = new_single_bucket_load; + let mut new_boundary = Vec::new(); + let mut hist_load: Vec = self.t_req_loads.values().cloned().collect(); + hist_load.sort(); + let mut upper_bound: usize = 0; + let mut last_load_index: usize = 0; + let max_value = usize::MAX; + + let worker_url = { + let guard = self.prefill_worker_urls.lock().unwrap(); + (*guard).clone() + }; + + let mut iter = worker_url.iter().peekable(); + // let mut curr_worker_id = 0; + while let Some(url) = iter.next() { + if last_load_index >= hist_load.len() && iter.peek().is_none() { + new_boundary.push(Boundary::new(url.clone(), [upper_bound, max_value])); + break; + } + let mut load_accumulator = 0; + let mut break_flag = false; + for &load in hist_load[last_load_index..].iter() { + load_accumulator += load; + if load_accumulator >= new_single_bucket_load { + if iter.peek().is_none() { + new_boundary.push(Boundary::new(url.clone(), [upper_bound, max_value])); + break_flag = true; + break; + } + let real_load = upper_bound + new_single_bucket_load; + if load <= upper_bound { + new_boundary.push(Boundary::new(url.clone(), [upper_bound, real_load])); + upper_bound = real_load + 1; + } else { + new_boundary.push(Boundary::new(url.clone(), [upper_bound, load])); + upper_bound = load + 1; + } + last_load_index += 1; + break_flag = true; + break; + } else { + last_load_index += 1; + } + } + if !break_flag { + let mut right_bound_value = upper_bound + new_single_bucket_load; + if iter.peek().is_none() { + right_bound_value = max_value; + new_boundary.push(Boundary::new(url.clone(), [upper_bound, right_bound_value])); + break; + } + new_boundary.push(Boundary::new(url.clone(), [upper_bound, right_bound_value])); + upper_bound = right_bound_value + 1; + } + } + self.boundary = new_boundary; + info!("After adjusting boundary | {:?}", self.boundary); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::{BasicWorkerBuilder, WorkerType}; + + #[tokio::test] + async fn test_load_balancing_conditions() { + // Test 1: Basic load balancing trigger + let config = BucketConfig { + balance_abs_threshold: 32, + balance_rel_threshold: 1.0001, + bucket_adjust_interval_secs: 10, + }; + let policy = BucketPolicy::with_config(config); + let prefill_workers: Vec> = vec![ + Arc::new( + BasicWorkerBuilder::new("http://w1:8000") + .worker_type(WorkerType::Regular) + .api_key("test_api_key") + .build(), + ), + Arc::new( + BasicWorkerBuilder::new("http://w2:8000") + .worker_type(WorkerType::Regular) + .api_key("test_api_key") + .build(), + ), + Arc::new( + BasicWorkerBuilder::new("http://w3:8000") + .worker_type(WorkerType::Regular) + .api_key("test_api_key") + .build(), + ), + ]; + + // Initialize the policy with prefill_workers + policy.init_prefill_worker_urls(&prefill_workers); + + // === Phase S1: Construct bucket boundaries === + // Requests len =33 -> Bucket 1(expected range: 0-33) + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(33))) + .unwrap(); + // Two requests len =34 ->load balancing + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(34))) + .unwrap(); + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(34))) + .unwrap(); + + tokio::time::sleep(Duration::from_secs(11)).await; + { + let model_key = "default"; + + let bucket = policy + .buckets + .get(model_key) + .map(|entry| entry.value().clone()); + if let Some(bucket) = bucket { + let lock_result = bucket.write(); + if let Ok(bucket_guard) = lock_result { + // Expected Boundary: [0, 33] [34, 67] [68, MAX] + assert_eq!(bucket_guard.boundary[0].range[1], 33); + assert_eq!(bucket_guard.boundary[1].range[1], 67); + } else { + error!( + "Failed to acquire write lock for bucket of model {}", + model_key + ); + } + } + } + // === Phase S2: Validate load balancing === + // Three consecutive len=33 requests (Should route to different buckets) + let idx_1 = policy + .select_worker(&prefill_workers, Some(&*"a".repeat(33))) + .unwrap(); + let idx_2 = policy + .select_worker(&prefill_workers, Some(&*"a".repeat(33))) + .unwrap(); + let idx_3 = policy + .select_worker(&prefill_workers, Some(&*"a".repeat(33))) + .unwrap(); + assert_eq!(idx_1, 0, "Should not trigger load balancing"); + assert_ne!(idx_2, idx_3, "Should trigger load balancing"); + assert_ne!(idx_2, 0, "Should trigger load balancing"); + assert_ne!(idx_3, 0, "Should trigger load balancing"); + + // Test 2: Not triggering when absolute threshold not met + let config = BucketConfig { + balance_abs_threshold: 30, + balance_rel_threshold: 2.0, + ..Default::default() + }; + let policy = BucketPolicy::with_config(config); + policy.init_prefill_worker_urls(&prefill_workers); + + // Create load difference below absolute threshold(20 + 8 = 28 < 30) + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(20))) + .unwrap(); // worker1: 20 + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(8))) + .unwrap(); // worker1: 8 + + // Next request should not use bucket scheduling (no load balancing) + let idx = policy + .select_worker(&prefill_workers, Some("request")) + .unwrap(); + assert_eq!( + idx, 0, + "Should not trigger load balancing when relative threshold not met" + ); + + // Test 3: Not triggering when relative threshold not met + let config = BucketConfig { + balance_abs_threshold: 5, + balance_rel_threshold: 3.0, + ..Default::default() + }; + let policy = BucketPolicy::with_config(config); + policy.init_prefill_worker_urls(&prefill_workers); + + // Create load difference (but relative threshold not met) + // Max/Min ratio = 15/5 = 3.0 + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(15))) + .unwrap(); // worker1: 15 + policy + .select_worker(&prefill_workers, Some("short")) + .unwrap(); // worker2: 5 + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(10))) + .unwrap(); // worker3: 10 + + // Next request should use bucket scheduling (load balancing) + let idx = policy + .select_worker(&prefill_workers, Some("request")) + .unwrap(); + assert_eq!( + idx, 0, + "Should not trigger load balancing when relative threshold not met" + ); + } + + #[tokio::test] + async fn test_adjust_boundary_1() { + // Test configuration: Set high threshold to prevent load balancing policy. + let config = BucketConfig { + balance_abs_threshold: 300, + balance_rel_threshold: 1.0001, + bucket_adjust_interval_secs: 3, + }; + let policy = BucketPolicy::with_config(config); + let prefill_workers: Vec> = vec![ + Arc::new( + BasicWorkerBuilder::new("http://w1:8000") + .worker_type(WorkerType::Regular) + .api_key("test_api_key") + .build(), + ), + Arc::new( + BasicWorkerBuilder::new("http://w2:8000") + .worker_type(WorkerType::Regular) + .api_key("test_api_key") + .build(), + ), + Arc::new( + BasicWorkerBuilder::new("http://w3:8000") + .worker_type(WorkerType::Regular) + .api_key("test_api_key") + .build(), + ), + ]; + + // Initialize the policy with prefill_workers + policy.init_prefill_worker_urls(&prefill_workers); + + // Initial boundary + { + let model_key = "default"; + + let bucket = policy + .buckets + .get(model_key) + .map(|entry| entry.value().clone()); + if let Some(bucket) = bucket { + let lock_result = bucket.write(); + if let Ok(bucket_guard) = lock_result { + // Expected Boundary: [0, 33] [34, 67] [68, MAX] + assert_eq!(bucket_guard.boundary[0].range[1], 1364); + assert_eq!(bucket_guard.boundary[1].range[1], 2729); + } else { + error!( + "Failed to acquire write lock for bucket of model {}", + model_key + ); + } + } + } + + // ===Phase S1: Initial requests to trigger boundary adjustment === + // Send requests with lengths: [5, 10, 15, 20, 24, 26] (total = 100) + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(5))) + .unwrap(); + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(10))) + .unwrap(); + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(15))) + .unwrap(); + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(20))) + .unwrap(); + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(24))) + .unwrap(); + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(26))) + .unwrap(); + + tokio::time::sleep(Duration::from_secs(4)).await; + // Verify boundaries adjusted to: [0, 20], [21, 26], [27, MAX] + { + let model_key = "default"; + + let bucket = policy + .buckets + .get(model_key) + .map(|entry| entry.value().clone()); + if let Some(bucket) = bucket { + let lock_result = bucket.write(); + if let Ok(bucket_guard) = lock_result { + // Expected Boundary: [0, 33] [34, 67] [68, MAX] + assert_eq!(bucket_guard.boundary[0].range[1], 20); + assert_eq!(bucket_guard.boundary[1].range[1], 26); + } else { + error!( + "Failed to acquire write lock for bucket of model {}", + model_key + ); + } + } + } + + // ===Phase S2: Second set of requests to trigger boundary adjustment === + // Send requests with lengths: [10, 20, 30, 40, 45, 57] (total = 202) + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(10))) + .unwrap(); + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(20))) + .unwrap(); + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(30))) + .unwrap(); + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(40))) + .unwrap(); + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(45))) + .unwrap(); + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(57))) + .unwrap(); + + tokio::time::sleep(Duration::from_secs(4)).await; + // Verify boundaries adjusted to: [0, 40], [41, 57], [58, MAX] + { + let model_key = "default"; + + let bucket = policy + .buckets + .get(model_key) + .map(|entry| entry.value().clone()); + if let Some(bucket) = bucket { + let lock_result = bucket.write(); + if let Ok(bucket_guard) = lock_result { + // Expected Boundary: [0, 33] [34, 67] [68, MAX] + assert_eq!(bucket_guard.boundary[0].range[1], 40); + assert_eq!(bucket_guard.boundary[1].range[1], 57); + } else { + error!( + "Failed to acquire write lock for bucket of model {}", + model_key + ); + } + } + } + } + + #[tokio::test] + async fn test_adjust_boundary_2() { + let config = BucketConfig { + balance_abs_threshold: 300, + balance_rel_threshold: 1.0001, + bucket_adjust_interval_secs: 3, + }; + let policy = BucketPolicy::with_config(config); + let prefill_workers: Vec> = vec![ + Arc::new( + BasicWorkerBuilder::new("http://w1:8000") + .worker_type(WorkerType::Regular) + .api_key("test_api_key") + .build(), + ), + Arc::new( + BasicWorkerBuilder::new("http://w2:8000") + .worker_type(WorkerType::Regular) + .api_key("test_api_key") + .build(), + ), + Arc::new( + BasicWorkerBuilder::new("http://w3:8000") + .worker_type(WorkerType::Regular) + .api_key("test_api_key") + .build(), + ), + ]; + + // Initialize the policy with prefill_workers + policy.init_prefill_worker_urls(&prefill_workers); + + // Initial boundary + { + let model_key = "default"; + + let bucket = policy + .buckets + .get(model_key) + .map(|entry| entry.value().clone()); + if let Some(bucket) = bucket { + let lock_result = bucket.write(); + if let Ok(bucket_guard) = lock_result { + // Expected Boundary: [0, 33] [34, 67] [68, MAX] + assert_eq!(bucket_guard.boundary[0].range[1], 1364); + assert_eq!(bucket_guard.boundary[1].range[1], 2729); + } else { + error!( + "Failed to acquire write lock for bucket of model {}", + model_key + ); + } + } + } + + // Send requests with char_count 20 + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(20))) + .unwrap(); + + tokio::time::sleep(Duration::from_secs(4)).await; + { + let model_key = "default"; + + let bucket = policy + .buckets + .get(model_key) + .map(|entry| entry.value().clone()); + if let Some(bucket) = bucket { + let lock_result = bucket.write(); + if let Ok(bucket_guard) = lock_result { + // Expected Boundary: [0, 33] [34, 67] [68, MAX] + assert_eq!(bucket_guard.boundary[0].range[1], 20); + assert_eq!(bucket_guard.boundary[1].range[1], 27); + } else { + error!( + "Failed to acquire write lock for bucket of model {}", + model_key + ); + } + } + } + + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(7))) + .unwrap(); + + tokio::time::sleep(Duration::from_secs(4)).await; + { + let model_key = "default"; + + let bucket = policy + .buckets + .get(model_key) + .map(|entry| entry.value().clone()); + if let Some(bucket) = bucket { + let lock_result = bucket.write(); + if let Ok(bucket_guard) = lock_result { + // Expected Boundary: [0, 33] [34, 67] [68, MAX] + assert_eq!(bucket_guard.boundary[0].range[1], 7); + assert_eq!(bucket_guard.boundary[1].range[1], 10); + } else { + error!( + "Failed to acquire write lock for bucket of model {}", + model_key + ); + } + } + } + } + + #[tokio::test] + async fn test_not_adjust_boundary() { + let config = BucketConfig { + balance_abs_threshold: 300, + balance_rel_threshold: 1.0001, + bucket_adjust_interval_secs: 3, + }; + let policy = BucketPolicy::with_config(config); + let prefill_workers: Vec> = vec![ + Arc::new( + BasicWorkerBuilder::new("http://w1:8000") + .worker_type(WorkerType::Regular) + .api_key("test_api_key") + .build(), + ), + Arc::new( + BasicWorkerBuilder::new("http://w2:8000") + .worker_type(WorkerType::Regular) + .api_key("test_api_key") + .build(), + ), + Arc::new( + BasicWorkerBuilder::new("http://w3:8000") + .worker_type(WorkerType::Regular) + .api_key("test_api_key") + .build(), + ), + ]; + + // Initialize the policy with prefill_workers + policy.init_prefill_worker_urls(&prefill_workers); + + // Initial boundary + { + let model_key = "default"; + + let bucket = policy + .buckets + .get(model_key) + .map(|entry| entry.value().clone()); + if let Some(bucket) = bucket { + let lock_result = bucket.write(); + if let Ok(bucket_guard) = lock_result { + // Expected Boundary: [0, 33] [34, 67] [68, MAX] + assert_eq!(bucket_guard.boundary[0].range[1], 1364); + assert_eq!(bucket_guard.boundary[1].range[1], 2729); + } else { + error!( + "Failed to acquire write lock for bucket of model {}", + model_key + ); + } + } + } + + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(5))) + .unwrap(); + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(10))) + .unwrap(); + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(15))) + .unwrap(); + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(20))) + .unwrap(); + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(24))) + .unwrap(); + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(26))) + .unwrap(); + + tokio::time::sleep(Duration::from_secs(4)).await; + { + let model_key = "default"; + + let bucket = policy + .buckets + .get(model_key) + .map(|entry| entry.value().clone()); + if let Some(bucket) = bucket { + let lock_result = bucket.write(); + if let Ok(bucket_guard) = lock_result { + // Expected Boundary: [0, 33] [34, 67] [68, MAX] + assert_eq!(bucket_guard.boundary[0].range[1], 20); + assert_eq!(bucket_guard.boundary[1].range[1], 26); + } else { + error!( + "Failed to acquire write lock for bucket of model {}", + model_key + ); + } + } + } + + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(10))) + .unwrap(); + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(20))) + .unwrap(); + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(30))) + .unwrap(); + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(32))) + .unwrap(); + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(45))) + .unwrap(); + policy + .select_worker(&prefill_workers, Some(&*"a".repeat(55))) + .unwrap(); + + tokio::time::sleep(Duration::from_secs(4)).await; + { + let model_key = "default"; + + let bucket = policy + .buckets + .get(model_key) + .map(|entry| entry.value().clone()); + if let Some(bucket) = bucket { + let lock_result = bucket.write(); + if let Ok(bucket_guard) = lock_result { + // Expected Boundary: [0, 33] [34, 67] [68, MAX] + assert_eq!(bucket_guard.boundary[0].range[1], 20); + assert_eq!(bucket_guard.boundary[1].range[1], 26); + } else { + error!( + "Failed to acquire write lock for bucket of model {}", + model_key + ); + } + } + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/cache_aware.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/cache_aware.rs new file mode 100644 index 00000000..c43d34ed --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/cache_aware.rs @@ -0,0 +1,533 @@ +/* + Cache-Aware Load Balancing Router + + This router combines two strategies to optimize both cache utilization and request distribution: + + 1. Cache-Aware Routing (Approximate Tree) + 2. Load Balancing (Shortest Queue with Balance Thresholds) + + The router dynamically switches between these strategies based on load conditions: + - Uses load balancing when the system is imbalanced + - Uses cache-aware routing when the system is balanced + + A system is considered imbalanced if both conditions are met: + 1. (max - min) > abs_threshold + 2. max > rel_threshold * min + + Strategy Details: + + 1. Cache-Aware Routing (Approximate Tree) + ------------------------------------------- + This strategy maintains an approximate radix tree for each worker based on request history, + eliminating the need for direct cache state queries. The tree stores raw text characters + instead of token IDs to avoid tokenization overhead. + + Process: + a. For each request, find the worker with the highest prefix match + b. If match rate > cache_threshold: + Route to the worker with highest match (likely has relevant data cached) + c. If match rate ≤ cache_threshold: + Route to the worker with smallest tree size (most available cache capacity) + d. Background maintenance: + Periodically evict least recently used leaf nodes to prevent memory overflow + + 2. Load Balancing (Shortest Queue) + ------------------------------------------- + This strategy tracks pending request counts per worker and routes new requests + to the least busy worker when the system is detected to be imbalanced. + + Configuration Parameters: + ------------------------ + 1. cache_threshold: (float, 0.0 to 1.0) + Minimum prefix match ratio to use highest-match routing. + Below this threshold, routes to worker with most available cache space. + + 2. balance_abs_threshold: (integer) + Absolute difference threshold for load imbalance detection. + System is potentially imbalanced if (max_load - min_load) > abs_threshold + + 3. balance_rel_threshold: (float) + Relative ratio threshold for load imbalance detection. + System is potentially imbalanced if max_load > min_load * rel_threshold + Used in conjunction with abs_threshold to determine final imbalance state. + + 4. eviction_interval_secs: (integer) + Interval between LRU eviction cycles for the approximate trees. + + 5. max_tree_size: (integer) + Maximum nodes per tree. When exceeded, LRU leaf nodes are evicted + during the next eviction cycle. +*/ + +use std::{sync::Arc, thread, time::Duration}; + +use dashmap::DashMap; +use rand::Rng; +use tracing::debug; + +use super::{get_healthy_worker_indices, tree::Tree, CacheAwareConfig, LoadBalancingPolicy}; +use crate::{core::Worker, metrics::RouterMetrics}; + +/// Cache-aware routing policy +/// +/// Routes requests based on cache affinity when load is balanced, +/// switches to shortest-queue routing when load is imbalanced. +/// Maintains separate trees per model for multi-model support. +#[derive(Debug)] +pub struct CacheAwarePolicy { + config: CacheAwareConfig, + trees: Arc>>, + eviction_handle: Option>, +} + +impl CacheAwarePolicy { + pub fn new() -> Self { + Self::with_config(CacheAwareConfig::default()) + } + + pub fn with_config(config: CacheAwareConfig) -> Self { + let trees = Arc::new(DashMap::>::new()); + + // Start background eviction thread if configured + let eviction_handle = if config.eviction_interval_secs > 0 { + let trees_clone = Arc::clone(&trees); + let max_tree_size = config.max_tree_size; + let interval = config.eviction_interval_secs; + + Some(thread::spawn(move || loop { + thread::sleep(Duration::from_secs(interval)); + + // Evict for all model trees + for tree_ref in trees_clone.iter() { + let model_id = tree_ref.key(); + let tree = tree_ref.value(); + tree.evict_tenant_by_size(max_tree_size); + debug!( + "Cache eviction completed for model {}, max_size: {}", + model_id, max_tree_size + ); + } + })) + } else { + None + }; + + Self { + config, + trees, + eviction_handle, + } + } + + /// Initialize the tree with worker URLs (used only during initial setup) + pub fn init_workers(&self, workers: &[Arc]) { + // Group workers by model + let mut model_workers: std::collections::HashMap>> = + std::collections::HashMap::new(); + for worker in workers { + // Use "default" for unknown/empty model_ids for backward compatibility + let model_id = worker.model_id(); + let tree_key = if model_id.is_empty() || model_id == "unknown" { + "default" + } else { + model_id + }; + model_workers + .entry(tree_key.to_string()) + .or_default() + .push(worker); + } + + // Initialize tree for each model + for (tree_key, model_workers) in model_workers { + let tree = self + .trees + .entry(tree_key) + .or_insert_with(|| Arc::new(Tree::new())); + for worker in model_workers { + tree.insert("", worker.url()); + } + } + } + + /// Add a single worker to the tree (incremental update) + pub fn add_worker(&self, worker: &dyn Worker) { + // For backward compatibility: if model_id is "unknown" or empty, + // use a default tree. This preserves existing behavior for single-model routers. + let model_id = worker.model_id(); + let tree_key = if model_id.is_empty() || model_id == "unknown" { + "default" + } else { + model_id + }; + let tree = self + .trees + .entry(tree_key.to_string()) + .or_insert_with(|| Arc::new(Tree::new())); + tree.insert("", worker.url()); + } + + /// Add a worker by URL and model (for backward compatibility) + pub fn add_worker_by_url(&self, url: &str, model_id: &str) { + let tree = self + .trees + .entry(model_id.to_string()) + .or_insert_with(|| Arc::new(Tree::new())); + tree.insert("", url); + } + + /// Remove a worker from the tree + pub fn remove_worker(&self, worker: &dyn Worker) { + // Use same logic as add_worker for consistency + let model_id = worker.model_id(); + let tree_key = if model_id.is_empty() || model_id == "unknown" { + "default" + } else { + model_id + }; + if let Some(tree) = self.trees.get(tree_key) { + tree.remove_tenant(worker.url()); + } + } + + /// Remove a worker by URL (removes from all model trees for backward compatibility) + pub fn remove_worker_by_url(&self, url: &str) { + // Remove from all trees since we don't know which model it belongs to + for tree_ref in self.trees.iter() { + tree_ref.value().remove_tenant(url); + } + } + + /// Run cache eviction to prevent unbounded growth + pub fn evict_cache(&self, max_size: usize) { + for tree_ref in self.trees.iter() { + let model_id = tree_ref.key(); + let tree = tree_ref.value(); + tree.evict_tenant_by_size(max_size); + debug!( + "Cache eviction for model {}, max_size: {}", + model_id, max_size + ); + } + } +} + +impl LoadBalancingPolicy for CacheAwarePolicy { + fn select_worker( + &self, + workers: &[Arc], + request_text: Option<&str>, + ) -> Option { + let healthy_indices = get_healthy_worker_indices(workers); + + if healthy_indices.is_empty() { + return None; + } + + // Determine the model for this set of workers (router pre-filters by model) + // All workers should be from the same model + let first_model = workers[healthy_indices[0]].model_id(); + let model_id = if first_model.is_empty() || first_model == "unknown" { + "default" + } else { + first_model + }; + + // Get current load statistics + let loads: Vec = workers.iter().map(|w| w.load()).collect(); + let max_load = *loads.iter().max().unwrap_or(&0); + let min_load = *loads.iter().min().unwrap_or(&0); + + // Check if load is imbalanced + let is_imbalanced = max_load.saturating_sub(min_load) > self.config.balance_abs_threshold + && (max_load as f32) > (min_load as f32 * self.config.balance_rel_threshold); + + if is_imbalanced { + // Log load balancing trigger + let worker_loads: Vec<(String, usize)> = workers + .iter() + .map(|w| (w.url().to_string(), w.load())) + .collect(); + + debug!( + "Load balancing triggered | max: {} | min: {} | workers: {:?}", + max_load, min_load, worker_loads + ); + + RouterMetrics::record_load_balancing_event(); + RouterMetrics::set_load_range(max_load, min_load); + + // Use shortest queue when imbalanced + let min_load_idx = healthy_indices + .iter() + .min_by_key(|&&idx| workers[idx].load()) + .copied()?; + + // Even in imbalanced mode, update the tree to maintain cache state + if let Some(text) = request_text { + // Get the tree reference without locking the entire HashMap + // DashMap only locks the specific shard containing this key + let tree = self.trees.get(model_id).map(|entry| entry.value().clone()); + + if let Some(tree) = tree { + // Now we can work with the tree without holding the HashMap lock + tree.insert(text, workers[min_load_idx].url()); + } else { + debug!( + "Warning: No tree found for model '{}', skipping cache update", + model_id + ); + } + } + + // Increment processed counter + workers[min_load_idx].increment_processed(); + RouterMetrics::record_processed_request(workers[min_load_idx].url()); + RouterMetrics::record_policy_decision(self.name(), workers[min_load_idx].url()); + + return Some(min_load_idx); + } + + // Use cache-aware routing when balanced + let text = request_text.unwrap_or(""); + + // Get the tree reference without locking the entire HashMap + // DashMap only locks the specific shard containing this key + let tree = self.trees.get(model_id).map(|entry| entry.value().clone()); + + if let Some(tree) = tree { + // Now we work with the tree without holding the HashMap lock + let (matched_text, matched_worker) = tree.prefix_match(text); + let match_rate = if text.is_empty() { + 0.0 + } else { + matched_text.chars().count() as f32 / text.chars().count() as f32 + }; + + let selected_url = if match_rate > self.config.cache_threshold { + RouterMetrics::record_cache_hit(); + matched_worker.to_string() + } else { + RouterMetrics::record_cache_miss(); + tree.get_smallest_tenant() + }; + + // Find the index of the selected worker + if let Some(selected_idx) = workers.iter().position(|w| w.url() == selected_url) { + // Only proceed if the worker is healthy + if workers[selected_idx].is_healthy() { + // Update the tree with this request + tree.insert(text, &selected_url); + + // Increment processed counter + workers[selected_idx].increment_processed(); + RouterMetrics::record_processed_request(&selected_url); + + return Some(selected_idx); + } + } else { + // Selected worker no longer exists, remove it from tree + tree.remove_tenant(&selected_url); + debug!("Removed stale worker {} from cache tree", selected_url); + } + + // Fallback to first healthy worker + healthy_indices.first().copied() + } else { + // No tree for this model, log warning and use random selection + debug!( + "Warning: No tree found for model '{}', using random worker selection", + model_id + ); + // Return a random healthy worker + let mut rng = rand::rng(); + let random_idx = rng.random_range(0..healthy_indices.len()); + Some(healthy_indices[random_idx]) + } + } + + fn select_worker_pair( + &self, + prefill_workers: &[Arc], + decode_workers: &[Arc], + request_text: Option<&str>, + ) -> Option<(usize, usize)> { + // DEPRECATED: This method is no longer used when separate policies are configured. + // The PD router now uses separate policies for prefill and decode selection. + // This implementation remains for backward compatibility when a single policy is used. + + // In PD mode with single policy: + // - Prefill: Use cache-aware routing for better cache utilization + // - Decode: Use least-load routing for better load distribution + + // Select prefill worker using cache-aware logic + let prefill_idx = self.select_worker(prefill_workers, request_text)?; + + // Select decode worker using least-load logic + let healthy_decode = get_healthy_worker_indices(decode_workers); + if healthy_decode.is_empty() { + return None; + } + + let decode_idx = healthy_decode + .iter() + .min_by_key(|&&idx| decode_workers[idx].load()) + .copied()?; + + Some((prefill_idx, decode_idx)) + } + + fn on_request_complete(&self, worker_url: &str, success: bool) { + // Could track success rates per worker for more intelligent routing + if !success { + // Optionally reduce affinity for failed requests + tracing::debug!( + "Request to {} completed with success={}", + worker_url, + success + ); + } + } + + fn name(&self) -> &'static str { + "cache_aware" + } + + fn needs_request_text(&self) -> bool { + true // Cache-aware policy needs request text for cache affinity + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +impl Default for CacheAwarePolicy { + fn default() -> Self { + Self::new() + } +} + +impl Drop for CacheAwarePolicy { + fn drop(&mut self) { + // Note: We can't properly stop the eviction thread since it's in an infinite loop + // In a production system, we'd use a channel or atomic flag to signal shutdown + if let Some(handle) = self.eviction_handle.take() { + // The thread will continue running until the program exits + // This is acceptable for now since the router typically runs for the lifetime of the program + drop(handle); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::{BasicWorkerBuilder, WorkerType}; + + #[test] + fn test_cache_aware_with_balanced_load() { + // Create policy without eviction thread for testing + let config = CacheAwareConfig { + eviction_interval_secs: 0, // Disable eviction thread + ..Default::default() + }; + let policy = CacheAwarePolicy::with_config(config); + let workers: Vec> = vec![ + Arc::new( + BasicWorkerBuilder::new("http://w1:8000") + .worker_type(WorkerType::Regular) + .api_key("test_api_key") + .build(), + ), + Arc::new( + BasicWorkerBuilder::new("http://w2:8000") + .worker_type(WorkerType::Regular) + .api_key("test_api_key") + .build(), + ), + ]; + + // Initialize the policy with workers + policy.init_workers(&workers); + + // First request should be distributed + let idx1 = policy.select_worker(&workers, Some("hello world")).unwrap(); + + // Same request should go to same worker (cache hit) + let idx2 = policy.select_worker(&workers, Some("hello world")).unwrap(); + assert_eq!(idx1, idx2); + + // Similar request should also go to same worker + let idx3 = policy.select_worker(&workers, Some("hello")).unwrap(); + assert_eq!(idx1, idx3); + } + + #[test] + fn test_cache_aware_with_imbalanced_load() { + let policy = CacheAwarePolicy::with_config(CacheAwareConfig { + cache_threshold: 0.5, + balance_abs_threshold: 5, + balance_rel_threshold: 2.0, + eviction_interval_secs: 0, // Disable eviction thread + max_tree_size: 10000, + }); + + let worker1 = BasicWorkerBuilder::new("http://w1:8000") + .worker_type(WorkerType::Regular) + .build(); + let worker2 = BasicWorkerBuilder::new("http://w2:8000") + .worker_type(WorkerType::Regular) + .build(); + + // Create significant load imbalance + for _ in 0..20 { + worker1.increment_load(); + } + // worker2 has load 0 + + let workers: Vec> = vec![Arc::new(worker1), Arc::new(worker2)]; + policy.init_workers(&workers); + + // Should select worker2 (lower load) despite cache affinity + for _ in 0..5 { + let idx = policy.select_worker(&workers, Some("test")).unwrap(); + assert_eq!(idx, 1); // Should always pick worker2 + } + } + + #[test] + fn test_cache_aware_worker_removal() { + let config = CacheAwareConfig { + eviction_interval_secs: 0, // Disable eviction thread + ..Default::default() + }; + let policy = CacheAwarePolicy::with_config(config); + let workers: Vec> = vec![ + Arc::new( + BasicWorkerBuilder::new("http://w1:8000") + .worker_type(WorkerType::Regular) + .build(), + ), + Arc::new( + BasicWorkerBuilder::new("http://w2:8000") + .worker_type(WorkerType::Regular) + .build(), + ), + ]; + + policy.init_workers(&workers); + + // Route some requests + policy.select_worker(&workers, Some("test1")); + policy.select_worker(&workers, Some("test2")); + + // Remove a worker + policy.remove_worker_by_url("http://w1:8000"); + workers[0].set_healthy(false); + + // All requests should now go to worker2 + let idx = policy.select_worker(&workers, Some("test1")).unwrap(); + assert_eq!(idx, 1); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/factory.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/factory.rs new file mode 100644 index 00000000..a9db1842 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/factory.rs @@ -0,0 +1,113 @@ +//! Factory for creating load balancing policies + +use std::sync::Arc; + +use super::{ + BucketConfig, BucketPolicy, CacheAwareConfig, CacheAwarePolicy, LoadBalancingPolicy, + PowerOfTwoPolicy, RandomPolicy, RoundRobinPolicy, +}; +use crate::config::PolicyConfig; + +/// Factory for creating policy instances +pub struct PolicyFactory; + +impl PolicyFactory { + /// Create a policy from configuration + pub fn create_from_config(config: &PolicyConfig) -> Arc { + match config { + PolicyConfig::Random => Arc::new(RandomPolicy::new()), + PolicyConfig::RoundRobin => Arc::new(RoundRobinPolicy::new()), + PolicyConfig::PowerOfTwo { .. } => Arc::new(PowerOfTwoPolicy::new()), + PolicyConfig::CacheAware { + cache_threshold, + balance_abs_threshold, + balance_rel_threshold, + eviction_interval_secs, + max_tree_size, + } => { + let config = CacheAwareConfig { + cache_threshold: *cache_threshold, + balance_abs_threshold: *balance_abs_threshold, + balance_rel_threshold: *balance_rel_threshold, + eviction_interval_secs: *eviction_interval_secs, + max_tree_size: *max_tree_size, + }; + Arc::new(CacheAwarePolicy::with_config(config)) + } + PolicyConfig::Bucket { + balance_abs_threshold, + balance_rel_threshold, + bucket_adjust_interval_secs, + } => { + let config = BucketConfig { + balance_abs_threshold: *balance_abs_threshold, + balance_rel_threshold: *balance_rel_threshold, + bucket_adjust_interval_secs: *bucket_adjust_interval_secs, + }; + Arc::new(BucketPolicy::with_config(config)) + } + } + } + + /// Create a policy by name (for dynamic loading) + pub fn create_by_name(name: &str) -> Option> { + match name.to_lowercase().as_str() { + "random" => Some(Arc::new(RandomPolicy::new())), + "round_robin" | "roundrobin" => Some(Arc::new(RoundRobinPolicy::new())), + "power_of_two" | "poweroftwo" => Some(Arc::new(PowerOfTwoPolicy::new())), + "cache_aware" | "cacheaware" => Some(Arc::new(CacheAwarePolicy::new())), + "bucket" => Some(Arc::new(BucketPolicy::new())), + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_create_from_config() { + let policy = PolicyFactory::create_from_config(&PolicyConfig::Random); + assert_eq!(policy.name(), "random"); + + let policy = PolicyFactory::create_from_config(&PolicyConfig::RoundRobin); + assert_eq!(policy.name(), "round_robin"); + + let policy = PolicyFactory::create_from_config(&PolicyConfig::PowerOfTwo { + load_check_interval_secs: 60, + }); + assert_eq!(policy.name(), "power_of_two"); + + let policy = PolicyFactory::create_from_config(&PolicyConfig::CacheAware { + cache_threshold: 0.7, + balance_abs_threshold: 10, + balance_rel_threshold: 1.5, + eviction_interval_secs: 30, + max_tree_size: 1000, + }); + assert_eq!(policy.name(), "cache_aware"); + + let policy = PolicyFactory::create_from_config(&PolicyConfig::Bucket { + balance_abs_threshold: 10, + balance_rel_threshold: 1.5, + bucket_adjust_interval_secs: 5, + }); + assert_eq!(policy.name(), "bucket"); + } + + #[tokio::test] + async fn test_create_by_name() { + assert!(PolicyFactory::create_by_name("random").is_some()); + assert!(PolicyFactory::create_by_name("RANDOM").is_some()); + assert!(PolicyFactory::create_by_name("round_robin").is_some()); + assert!(PolicyFactory::create_by_name("RoundRobin").is_some()); + assert!(PolicyFactory::create_by_name("power_of_two").is_some()); + assert!(PolicyFactory::create_by_name("PowerOfTwo").is_some()); + assert!(PolicyFactory::create_by_name("cache_aware").is_some()); + assert!(PolicyFactory::create_by_name("CacheAware").is_some()); + assert!(PolicyFactory::create_by_name("bucket").is_some()); + assert!(PolicyFactory::create_by_name("Bucket").is_some()); + assert!(PolicyFactory::create_by_name("unknown").is_none()); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/mod.rs new file mode 100644 index 00000000..7eca8609 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/mod.rs @@ -0,0 +1,178 @@ +//! Load balancing policies for SGLang router +//! +//! This module provides a unified abstraction for routing policies that work +//! across both regular and prefill-decode (PD) routing modes. + +use std::{fmt::Debug, sync::Arc}; + +use crate::core::Worker; + +mod bucket; +mod cache_aware; +mod factory; +mod power_of_two; +mod random; +mod registry; +mod round_robin; +mod tree; + +pub use bucket::BucketPolicy; +pub use cache_aware::CacheAwarePolicy; +pub use factory::PolicyFactory; +pub use power_of_two::PowerOfTwoPolicy; +pub use random::RandomPolicy; +pub use registry::PolicyRegistry; +pub use round_robin::RoundRobinPolicy; + +/// Core trait for load balancing policies +/// +/// This trait provides a unified interface for implementing routing algorithms +/// that can work with both regular single-worker selection and PD dual-worker selection. +pub trait LoadBalancingPolicy: Send + Sync + Debug { + /// Select a single worker from the available workers + /// + /// This is used for regular routing mode where requests go to a single worker. + /// Now uses Arc for better performance and to avoid unnecessary cloning. + fn select_worker( + &self, + workers: &[Arc], + request_text: Option<&str>, + ) -> Option; + + /// Select a pair of workers (prefill and decode) for PD routing + /// + /// Returns indices of (prefill_worker, decode_worker) from their respective arrays. + /// Default implementation uses select_worker for each array independently. + fn select_worker_pair( + &self, + prefill_workers: &[Arc], + decode_workers: &[Arc], + request_text: Option<&str>, + ) -> Option<(usize, usize)> { + // Default implementation: independently select from each pool + let prefill_idx = self.select_worker(prefill_workers, request_text)?; + let decode_idx = self.select_worker(decode_workers, request_text)?; + Some((prefill_idx, decode_idx)) + } + + /// Update policy state after request completion + /// + /// This is called when a request completes (successfully or not) to allow + /// policies to update their internal state. + fn on_request_complete(&self, _worker_url: &str, _success: bool) { + // Default: no-op for stateless policies + } + + /// Get policy name for metrics and debugging + fn name(&self) -> &'static str; + + /// Check if this policy needs request text for routing decisions + fn needs_request_text(&self) -> bool { + false // Default: most policies don't need request text + } + + /// Update worker load information + /// + /// This is called periodically with current load information for load-aware policies. + fn update_loads(&self, _loads: &std::collections::HashMap) { + // Default: no-op for policies that don't use load information + } + + /// Reset any internal state + /// + /// This is useful for policies that maintain state (e.g., round-robin counters). + fn reset(&self) { + // Default: no-op for stateless policies + } + + /// Get as Any for downcasting + fn as_any(&self) -> &dyn std::any::Any; +} + +/// Configuration for cache-aware policy +#[derive(Debug, Clone)] +pub struct CacheAwareConfig { + pub cache_threshold: f32, + pub balance_abs_threshold: usize, + pub balance_rel_threshold: f32, + pub eviction_interval_secs: u64, + pub max_tree_size: usize, +} + +impl Default for CacheAwareConfig { + fn default() -> Self { + Self { + cache_threshold: 0.5, + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + eviction_interval_secs: 30, + max_tree_size: 10000, + } + } +} + +#[derive(Debug, Clone)] +pub struct BucketConfig { + pub balance_abs_threshold: usize, + pub balance_rel_threshold: f32, + pub bucket_adjust_interval_secs: usize, +} + +impl Default for BucketConfig { + fn default() -> Self { + Self { + balance_abs_threshold: 32, + balance_rel_threshold: 1.0001, + bucket_adjust_interval_secs: 5, + } + } +} + +/// Helper function to filter healthy workers and return their indices +pub(crate) fn get_healthy_worker_indices(workers: &[Arc]) -> Vec { + workers + .iter() + .enumerate() + .filter(|(_, w)| w.is_healthy() && w.circuit_breaker().can_execute()) + .map(|(idx, _)| idx) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::{BasicWorkerBuilder, WorkerType}; + + #[test] + fn test_get_healthy_worker_indices() { + let workers: Vec> = vec![ + Arc::new( + BasicWorkerBuilder::new("http://w1:8000") + .worker_type(WorkerType::Regular) + .api_key("test_api_key") + .build(), + ), + Arc::new( + BasicWorkerBuilder::new("http://w2:8000") + .worker_type(WorkerType::Regular) + .api_key("test_api_key2") + .build(), + ), + Arc::new( + BasicWorkerBuilder::new("http://w3:8000") + .worker_type(WorkerType::Regular) + .api_key("test_api_key") + .build(), + ), + ]; + + // All healthy initially + let indices = get_healthy_worker_indices(&workers); + assert_eq!(indices, vec![0, 1, 2]); + + // Mark one unhealthy + workers[1].set_healthy(false); + let indices = get_healthy_worker_indices(&workers); + assert_eq!(indices, vec![0, 2]); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/power_of_two.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/power_of_two.rs new file mode 100644 index 00000000..b7edef82 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/power_of_two.rs @@ -0,0 +1,213 @@ +//! Power-of-two choices load balancing policy + +use std::{ + collections::HashMap, + sync::{Arc, RwLock}, +}; + +use rand::Rng; +use tracing::info; + +use super::{get_healthy_worker_indices, LoadBalancingPolicy}; +use crate::{core::Worker, metrics::RouterMetrics}; + +/// Power-of-two choices policy +/// +/// Randomly selects two workers and routes to the one with lower load. +/// This provides good load distribution with minimal coordination overhead. +#[derive(Debug)] +pub struct PowerOfTwoPolicy { + /// Cached load information from external monitoring + cached_loads: RwLock>, +} + +impl PowerOfTwoPolicy { + pub fn new() -> Self { + Self { + cached_loads: RwLock::new(HashMap::new()), + } + } + + fn get_worker_load(&self, worker: &dyn Worker) -> isize { + // First check cached loads (from external monitoring) + if let Ok(loads) = self.cached_loads.read() { + if let Some(&load) = loads.get(worker.url()) { + return load; + } + } + + // Fall back to local load counter + worker.load() as isize + } +} + +impl LoadBalancingPolicy for PowerOfTwoPolicy { + fn select_worker( + &self, + workers: &[Arc], + _request_text: Option<&str>, + ) -> Option { + let healthy_indices = get_healthy_worker_indices(workers); + + if healthy_indices.is_empty() { + return None; + } + + if healthy_indices.len() == 1 { + return Some(healthy_indices[0]); + } + + // Select two random workers + let mut rng = rand::rng(); + let idx1 = rng.random_range(0..healthy_indices.len()); + let mut idx2 = rng.random_range(0..healthy_indices.len()); + + // Ensure we pick two different workers + while idx2 == idx1 { + idx2 = rng.random_range(0..healthy_indices.len()); + } + + let worker_idx1 = healthy_indices[idx1]; + let worker_idx2 = healthy_indices[idx2]; + + // Compare loads and select the less loaded one + let load1 = self.get_worker_load(workers[worker_idx1].as_ref()); + let load2 = self.get_worker_load(workers[worker_idx2].as_ref()); + + // Log selection for debugging + let selected_idx = if load1 <= load2 { + worker_idx1 + } else { + worker_idx2 + }; + + info!( + "Power-of-two selection: {}={} vs {}={} -> selected {}", + workers[worker_idx1].url(), + load1, + workers[worker_idx2].url(), + load2, + workers[selected_idx].url() + ); + + // Increment processed counter + workers[selected_idx].increment_processed(); + RouterMetrics::record_processed_request(workers[selected_idx].url()); + RouterMetrics::record_policy_decision(self.name(), workers[selected_idx].url()); + + Some(selected_idx) + } + + fn name(&self) -> &'static str { + "power_of_two" + } + + fn update_loads(&self, loads: &HashMap) { + if let Ok(mut cached) = self.cached_loads.write() { + *cached = loads.clone(); + } + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +impl Default for PowerOfTwoPolicy { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::{BasicWorkerBuilder, WorkerType}; + + #[test] + fn test_power_of_two_selection() { + let policy = PowerOfTwoPolicy::new(); + let worker1 = BasicWorkerBuilder::new("http://w1:8000") + .worker_type(WorkerType::Regular) + .build(); + let worker2 = BasicWorkerBuilder::new("http://w2:8000") + .worker_type(WorkerType::Regular) + .build(); + let worker3 = BasicWorkerBuilder::new("http://w3:8000") + .worker_type(WorkerType::Regular) + .build(); + + // Set different loads + for _ in 0..10 { + worker1.increment_load(); + } + for _ in 0..5 { + worker2.increment_load(); + } + // worker3 has load 0 + + let workers: Vec> = + vec![Arc::new(worker1), Arc::new(worker2), Arc::new(worker3)]; + + // Run multiple selections + let mut selected_counts = [0; 3]; + for _ in 0..100 { + if let Some(idx) = policy.select_worker(&workers, None) { + selected_counts[idx] += 1; + } + } + + // Worker with lowest load (worker3) should be selected most often + assert!(selected_counts[2] > selected_counts[1]); + assert!(selected_counts[1] > selected_counts[0]); + } + + #[test] + fn test_power_of_two_with_cached_loads() { + let policy = PowerOfTwoPolicy::new(); + let workers: Vec> = vec![ + Arc::new( + BasicWorkerBuilder::new("http://w1:8000") + .worker_type(WorkerType::Regular) + .build(), + ), + Arc::new( + BasicWorkerBuilder::new("http://w2:8000") + .worker_type(WorkerType::Regular) + .build(), + ), + ]; + + // Update cached loads + let mut loads = HashMap::new(); + loads.insert("http://w1:8000".to_string(), 100); + loads.insert("http://w2:8000".to_string(), 10); + policy.update_loads(&loads); + + // Should prefer worker2 with lower cached load + let mut w2_selected = 0; + for _ in 0..50 { + if let Some(idx) = policy.select_worker(&workers, None) { + if idx == 1 { + w2_selected += 1; + } + } + } + + // Worker2 should be selected significantly more often + assert!(w2_selected > 35); // Should win most of the time + } + + #[test] + fn test_power_of_two_single_worker() { + let policy = PowerOfTwoPolicy::new(); + let workers: Vec> = vec![Arc::new( + BasicWorkerBuilder::new("http://w1:8000") + .worker_type(WorkerType::Regular) + .build(), + )]; + + // With single worker, should always select it + assert_eq!(policy.select_worker(&workers, None), Some(0)); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/random.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/random.rs new file mode 100644 index 00000000..5b92b2d7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/random.rs @@ -0,0 +1,129 @@ +//! Random load balancing policy + +use std::sync::Arc; + +use rand::Rng; + +use super::{get_healthy_worker_indices, LoadBalancingPolicy}; +use crate::{core::Worker, metrics::RouterMetrics}; + +/// Random selection policy +/// +/// Selects workers randomly with uniform distribution among healthy workers. +#[derive(Debug, Default)] +pub struct RandomPolicy; + +impl RandomPolicy { + pub fn new() -> Self { + Self + } +} + +impl LoadBalancingPolicy for RandomPolicy { + fn select_worker( + &self, + workers: &[Arc], + _request_text: Option<&str>, + ) -> Option { + let healthy_indices = get_healthy_worker_indices(workers); + + if healthy_indices.is_empty() { + return None; + } + + let mut rng = rand::rng(); + let random_idx = rng.random_range(0..healthy_indices.len()); + let worker = workers[healthy_indices[random_idx]].url(); + + RouterMetrics::record_processed_request(worker); + RouterMetrics::record_policy_decision(self.name(), worker); + Some(healthy_indices[random_idx]) + } + + fn name(&self) -> &'static str { + "random" + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + use crate::core::{BasicWorkerBuilder, WorkerType}; + + #[test] + fn test_random_selection() { + let policy = RandomPolicy::new(); + let workers: Vec> = vec![ + Arc::new( + BasicWorkerBuilder::new("http://w1:8000") + .worker_type(WorkerType::Regular) + .build(), + ), + Arc::new( + BasicWorkerBuilder::new("http://w2:8000") + .worker_type(WorkerType::Regular) + .build(), + ), + Arc::new( + BasicWorkerBuilder::new("http://w3:8000") + .worker_type(WorkerType::Regular) + .build(), + ), + ]; + + let mut counts = HashMap::new(); + for _ in 0..100 { + if let Some(idx) = policy.select_worker(&workers, None) { + *counts.entry(idx).or_insert(0) += 1; + } + } + + // All workers should be selected at least once + assert_eq!(counts.len(), 3); + assert!(counts.values().all(|&count| count > 0)); + } + + #[test] + fn test_random_with_unhealthy_workers() { + let policy = RandomPolicy::new(); + let workers: Vec> = vec![ + Arc::new( + BasicWorkerBuilder::new("http://w1:8000") + .worker_type(WorkerType::Regular) + .build(), + ), + Arc::new( + BasicWorkerBuilder::new("http://w2:8000") + .worker_type(WorkerType::Regular) + .build(), + ), + ]; + + // Mark first worker as unhealthy + workers[0].set_healthy(false); + + // Should always select the healthy worker (index 1) + for _ in 0..10 { + assert_eq!(policy.select_worker(&workers, None), Some(1)); + } + } + + #[test] + fn test_random_no_healthy_workers() { + let policy = RandomPolicy::new(); + let workers: Vec> = vec![Arc::new( + BasicWorkerBuilder::new("http://w1:8000") + .worker_type(WorkerType::Regular) + .build(), + )]; + + workers[0].set_healthy(false); + assert_eq!(policy.select_worker(&workers, None), None); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/registry.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/registry.rs new file mode 100644 index 00000000..5fe5b24a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/registry.rs @@ -0,0 +1,483 @@ +use std::{ + collections::HashMap, + sync::{Arc, RwLock}, +}; + +use tracing::{debug, info, warn}; + +/// Policy Registry for managing model-to-policy mappings +/// +/// This registry manages the dynamic assignment of load balancing policies to models. +/// When the first worker of a new model is added, it determines the policy for that model. +/// All subsequent workers of the same model use the established policy. +/// When the last worker of a model is removed, the policy mapping is cleaned up. +use super::{ + BucketConfig, BucketPolicy, CacheAwareConfig, CacheAwarePolicy, LoadBalancingPolicy, + PowerOfTwoPolicy, RandomPolicy, RoundRobinPolicy, +}; +use crate::{config::types::PolicyConfig, core::Worker}; + +/// Registry for managing model-to-policy mappings +#[derive(Clone)] +pub struct PolicyRegistry { + /// Model ID -> Policy instance mapping + model_policies: Arc>>>, + + /// Model ID -> Worker count for cleanup tracking + model_worker_counts: Arc>>, + + /// Default policy instance (cached) + default_policy: Arc, + + /// Prefill policy for PD mode + prefill_policy: Arc>>>, + + /// Decode policy for PD mode + decode_policy: Arc>>>, +} + +impl PolicyRegistry { + /// Create a new PolicyRegistry with a default policy + pub fn new(default_policy_config: PolicyConfig) -> Self { + let default_policy = Self::create_policy_from_config(&default_policy_config); + + Self { + model_policies: Arc::new(RwLock::new(HashMap::new())), + model_worker_counts: Arc::new(RwLock::new(HashMap::new())), + default_policy, + prefill_policy: Arc::new(RwLock::new(None)), + decode_policy: Arc::new(RwLock::new(None)), + } + } + + /// Called when a worker is added + /// Returns the policy that should be used for this worker's model + pub fn on_worker_added( + &self, + model_id: &str, + policy_hint: Option<&str>, + ) -> Arc { + // Increment worker count + { + let mut counts = self.model_worker_counts.write().unwrap(); + *counts.entry(model_id.to_string()).or_insert(0) += 1; + debug!( + "Worker added for model {}, count: {}", + model_id, + counts.get(model_id).unwrap() + ); + } + + // Check if model already has a policy + { + let policies = self.model_policies.read().unwrap(); + if let Some(existing_policy) = policies.get(model_id) { + debug!( + "Model {} already has policy: {}", + model_id, + existing_policy.name() + ); + return Arc::clone(existing_policy); + } + } + + // New model - determine policy + let policy = self.determine_policy_for_model(model_id, policy_hint); + + info!( + "Assigning policy {} to new model {}", + policy.name(), + model_id + ); + + // Store policy for this model + { + let mut policies = self.model_policies.write().unwrap(); + policies.insert(model_id.to_string(), Arc::clone(&policy)); + } + + policy + } + + /// Called when a worker is removed + pub fn on_worker_removed(&self, model_id: &str) { + let should_cleanup = { + let mut counts = self.model_worker_counts.write().unwrap(); + if let Some(count) = counts.get_mut(model_id) { + *count = count.saturating_sub(1); + debug!("Worker removed for model {}, count: {}", model_id, *count); + if *count == 0 { + counts.remove(model_id); + true + } else { + false + } + } else { + warn!( + "Attempted to remove worker for model {} with no registered workers", + model_id + ); + false + } + }; + + // Clean up policy if this was the last worker + if should_cleanup { + let mut policies = self.model_policies.write().unwrap(); + if let Some(policy) = policies.remove(model_id) { + info!( + "Removed policy {} for model {} (last worker removed)", + policy.name(), + model_id + ); + // Policy will be dropped here, cleaning up any resources + drop(policy); + } + } + } + + /// Get the policy for a model + pub fn get_policy(&self, model_id: &str) -> Option> { + self.model_policies.read().unwrap().get(model_id).cloned() + } + + /// Get the default policy + pub fn get_default_policy(&self) -> Arc { + Arc::clone(&self.default_policy) + } + + /// Get policy for a model, or default if not found + pub fn get_policy_or_default(&self, model_id: &str) -> Arc { + self.get_policy(model_id) + .unwrap_or_else(|| self.get_default_policy()) + } + + /// Determine policy for a new model + fn determine_policy_for_model( + &self, + model_id: &str, + policy_hint: Option<&str>, + ) -> Arc { + // 1. Check policy hint from worker + if let Some(policy_type) = policy_hint { + debug!("Using policy hint '{}' for model {}", policy_type, model_id); + return self.create_policy_from_type(policy_type); + } + + // 2. Use default policy + debug!("Using default policy for model {}", model_id); + Arc::clone(&self.default_policy) + } + + /// Create a policy from a type string + fn create_policy_from_type(&self, policy_type: &str) -> Arc { + match policy_type { + "round_robin" => Arc::new(RoundRobinPolicy::new()), + "random" => Arc::new(RandomPolicy::new()), + "cache_aware" => Arc::new(CacheAwarePolicy::new()), + "power_of_two" => Arc::new(PowerOfTwoPolicy::new()), + "bucket" => Arc::new(BucketPolicy::new()), + _ => { + warn!("Unknown policy type '{}', using default", policy_type); + Arc::clone(&self.default_policy) + } + } + } + + /// Create a policy from a PolicyConfig + fn create_policy_from_config(config: &PolicyConfig) -> Arc { + match config { + PolicyConfig::RoundRobin => Arc::new(RoundRobinPolicy::new()), + PolicyConfig::Random => Arc::new(RandomPolicy::new()), + PolicyConfig::CacheAware { + cache_threshold, + balance_abs_threshold, + balance_rel_threshold, + eviction_interval_secs, + max_tree_size, + } => { + let cache_config = CacheAwareConfig { + cache_threshold: *cache_threshold, + balance_abs_threshold: *balance_abs_threshold, + balance_rel_threshold: *balance_rel_threshold, + eviction_interval_secs: *eviction_interval_secs, + max_tree_size: *max_tree_size, + }; + Arc::new(CacheAwarePolicy::with_config(cache_config)) + } + PolicyConfig::PowerOfTwo { .. } => Arc::new(PowerOfTwoPolicy::new()), + PolicyConfig::Bucket { + balance_abs_threshold, + balance_rel_threshold, + bucket_adjust_interval_secs, + } => { + let config = BucketConfig { + balance_abs_threshold: *balance_abs_threshold, + balance_rel_threshold: *balance_rel_threshold, + bucket_adjust_interval_secs: *bucket_adjust_interval_secs, + }; + Arc::new(BucketPolicy::with_config(config)) + } + } + } + + /// Get current model->policy mappings (for debugging/monitoring) + pub fn get_all_mappings(&self) -> HashMap { + let policies = self.model_policies.read().unwrap(); + policies + .iter() + .map(|(model, policy)| (model.clone(), policy.name().to_string())) + .collect() + } + + /// Get worker counts per model + pub fn get_worker_counts(&self) -> HashMap { + self.model_worker_counts.read().unwrap().clone() + } + + /// Clear all policies (useful for testing) + pub fn clear(&self) { + let mut policies = self.model_policies.write().unwrap(); + policies.clear(); + let mut counts = self.model_worker_counts.write().unwrap(); + counts.clear(); + } + + /// Set the prefill policy for PD mode + pub fn set_prefill_policy(&self, policy: Arc) { + let mut prefill_policy = self.prefill_policy.write().unwrap(); + *prefill_policy = Some(policy); + } + + /// Set the decode policy for PD mode + pub fn set_decode_policy(&self, policy: Arc) { + let mut decode_policy = self.decode_policy.write().unwrap(); + *decode_policy = Some(policy); + } + + /// Get the prefill policy for PD mode, or default if not set + pub fn get_prefill_policy(&self) -> Arc { + let prefill_policy = self.prefill_policy.read().unwrap(); + prefill_policy + .as_ref() + .map(Arc::clone) + .unwrap_or_else(|| self.get_default_policy()) + } + + /// Get the decode policy for PD mode, or default if not set + pub fn get_decode_policy(&self) -> Arc { + let decode_policy = self.decode_policy.read().unwrap(); + decode_policy + .as_ref() + .map(Arc::clone) + .unwrap_or_else(|| self.get_default_policy()) + } + + /// Get all PowerOfTwo policies that need load updates + pub fn get_all_power_of_two_policies(&self) -> Vec> { + let mut power_of_two_policies = Vec::new(); + + if self.default_policy.name() == "power_of_two" { + power_of_two_policies.push(Arc::clone(&self.default_policy)); + } + + if let Some(ref policy) = *self.prefill_policy.read().unwrap() { + if policy.name() == "power_of_two" && !Arc::ptr_eq(policy, &self.default_policy) { + power_of_two_policies.push(Arc::clone(policy)); + } + } + + if let Some(ref policy) = *self.decode_policy.read().unwrap() { + if policy.name() == "power_of_two" + && !Arc::ptr_eq(policy, &self.default_policy) + && !self + .prefill_policy + .read() + .unwrap() + .as_ref() + .is_some_and(|p| Arc::ptr_eq(p, policy)) + { + power_of_two_policies.push(Arc::clone(policy)); + } + } + + let model_policies = self.model_policies.read().unwrap(); + for policy in model_policies.values() { + if policy.name() == "power_of_two" { + let already_added = power_of_two_policies.iter().any(|p| Arc::ptr_eq(p, policy)); + if !already_added { + power_of_two_policies.push(Arc::clone(policy)); + } + } + } + + power_of_two_policies + } + + /// Initialize cache-aware policy with workers if applicable + /// This should be called after workers are registered for a model + pub fn init_cache_aware_policy(&self, model_id: &str, workers: &[Arc]) { + // Get the policy for this model + if let Some(policy) = self.get_policy(model_id) { + if policy.name() == "cache_aware" { + if let Some(cache_aware) = policy.as_any().downcast_ref::() { + debug!( + "Initializing cache-aware policy with {} workers for model {}", + workers.len(), + model_id + ); + cache_aware.init_workers(workers); + } + } + } + } + + /// Remove a worker from cache-aware policy if applicable + /// This should be called when a worker is being removed + pub fn remove_worker_from_cache_aware(&self, model_id: &str, worker_url: &str) { + // Get the policy for this model + if let Some(policy) = self.get_policy(model_id) { + if policy.name() == "cache_aware" { + if let Some(cache_aware) = policy.as_any().downcast_ref::() { + cache_aware.remove_worker_by_url(worker_url); + debug!( + "Removed worker {} from cache-aware policy for model {}", + worker_url, model_id + ); + } + } + } + } + + /// Initialize cache-aware policies for PD mode (prefill and decode) + pub fn init_pd_cache_aware_policies( + &self, + prefill_workers: &[Arc], + decode_workers: &[Arc], + ) { + // Initialize prefill policy if it's cache-aware + if let Some(prefill_policy) = self.prefill_policy.read().unwrap().as_ref() { + if prefill_policy.name() == "cache_aware" { + if let Some(cache_aware) = + prefill_policy.as_any().downcast_ref::() + { + if !prefill_workers.is_empty() { + debug!( + "Initializing prefill cache-aware policy with {} workers", + prefill_workers.len() + ); + cache_aware.init_workers(prefill_workers); + } + } + } + } + + // Initialize decode policy if it's cache-aware + if let Some(decode_policy) = self.decode_policy.read().unwrap().as_ref() { + if decode_policy.name() == "cache_aware" { + if let Some(cache_aware) = decode_policy.as_any().downcast_ref::() + { + if !decode_workers.is_empty() { + debug!( + "Initializing decode cache-aware policy with {} workers", + decode_workers.len() + ); + cache_aware.init_workers(decode_workers); + } + } + } + } + } + + pub fn init_pd_bucket_policies(&self, prefill_workers: &[Arc]) { + // Initialize prefill policy if it's bucket + if let Some(prefill_policy) = self.prefill_policy.read().unwrap().as_ref() { + if prefill_policy.name() == "bucket" { + if let Some(bucket) = prefill_policy.as_any().downcast_ref::() { + if !prefill_workers.is_empty() { + debug!( + "Initializing prefill bucket policy with {} workers", + prefill_workers.len() + ); + bucket.init_prefill_worker_urls(prefill_workers); + } + } + } + } + } +} + +impl std::fmt::Debug for PolicyRegistry { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PolicyRegistry") + .field("model_policies", &self.model_policies) + .field("model_worker_counts", &self.model_worker_counts) + .field("default_policy", &self.default_policy.name()) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_policy_registry_basic() { + let registry = PolicyRegistry::new(PolicyConfig::RoundRobin); + + // First worker of a model sets the policy + let policy1 = registry.on_worker_added("llama-3", Some("cache_aware")); + assert_eq!(policy1.name(), "cache_aware"); + + // Second worker of same model uses existing policy + let policy2 = registry.on_worker_added("llama-3", Some("round_robin")); + assert_eq!(policy2.name(), "cache_aware"); // Ignores hint, uses existing + + // Different model can have different policy + let policy3 = registry.on_worker_added("gpt-4", Some("random")); + assert_eq!(policy3.name(), "random"); + + // Check mappings + let mappings = registry.get_all_mappings(); + assert_eq!(mappings.get("llama-3").unwrap(), "cache_aware"); + assert_eq!(mappings.get("gpt-4").unwrap(), "random"); + + // Check worker counts + let counts = registry.get_worker_counts(); + assert_eq!(*counts.get("llama-3").unwrap(), 2); + assert_eq!(*counts.get("gpt-4").unwrap(), 1); + } + + #[test] + fn test_policy_registry_cleanup() { + let registry = PolicyRegistry::new(PolicyConfig::RoundRobin); + + // Add workers + registry.on_worker_added("llama-3", Some("cache_aware")); + registry.on_worker_added("llama-3", None); + assert_eq!(registry.get_worker_counts().get("llama-3"), Some(&2)); + + // Remove one worker - policy should remain + registry.on_worker_removed("llama-3"); + assert!(registry.get_policy("llama-3").is_some()); + assert_eq!(registry.get_worker_counts().get("llama-3"), Some(&1)); + + // Remove last worker - policy should be cleaned up + registry.on_worker_removed("llama-3"); + assert!(registry.get_policy("llama-3").is_none()); + assert_eq!(registry.get_worker_counts().get("llama-3"), None); + } + + #[test] + fn test_default_policy() { + let registry = PolicyRegistry::new(PolicyConfig::RoundRobin); + + // No hint, no template - uses default + let policy = registry.on_worker_added("unknown-model", None); + assert_eq!(policy.name(), "round_robin"); + + // Get default directly + let default = registry.get_default_policy(); + assert_eq!(default.name(), "round_robin"); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/round_robin.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/round_robin.rs new file mode 100644 index 00000000..5b077625 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/round_robin.rs @@ -0,0 +1,151 @@ +//! Round-robin load balancing policy + +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +use super::{get_healthy_worker_indices, LoadBalancingPolicy}; +use crate::{core::Worker, metrics::RouterMetrics}; + +/// Round-robin selection policy +/// +/// Selects workers in sequential order, cycling through all healthy workers. +#[derive(Debug, Default)] +pub struct RoundRobinPolicy { + counter: AtomicUsize, +} + +impl RoundRobinPolicy { + pub fn new() -> Self { + Self { + counter: AtomicUsize::new(0), + } + } +} + +impl LoadBalancingPolicy for RoundRobinPolicy { + fn select_worker( + &self, + workers: &[Arc], + _request_text: Option<&str>, + ) -> Option { + let healthy_indices = get_healthy_worker_indices(workers); + + if healthy_indices.is_empty() { + return None; + } + + // Get and increment counter atomically + let count = self.counter.fetch_add(1, Ordering::Relaxed); + let selected_idx = count % healthy_indices.len(); + let worker = workers[healthy_indices[selected_idx]].url(); + + RouterMetrics::record_processed_request(worker); + RouterMetrics::record_policy_decision(self.name(), worker); + Some(healthy_indices[selected_idx]) + } + + fn name(&self) -> &'static str { + "round_robin" + } + + fn reset(&self) { + self.counter.store(0, Ordering::Relaxed); + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::{BasicWorkerBuilder, WorkerType}; + + #[test] + fn test_round_robin_selection() { + let policy = RoundRobinPolicy::new(); + let workers: Vec> = vec![ + Arc::new( + BasicWorkerBuilder::new("http://w1:8000") + .worker_type(WorkerType::Regular) + .build(), + ), + Arc::new( + BasicWorkerBuilder::new("http://w2:8000") + .worker_type(WorkerType::Regular) + .build(), + ), + Arc::new( + BasicWorkerBuilder::new("http://w3:8000") + .worker_type(WorkerType::Regular) + .build(), + ), + ]; + + // Should select workers in order: 0, 1, 2, 0, 1, 2, ... + assert_eq!(policy.select_worker(&workers, None), Some(0)); + assert_eq!(policy.select_worker(&workers, None), Some(1)); + assert_eq!(policy.select_worker(&workers, None), Some(2)); + assert_eq!(policy.select_worker(&workers, None), Some(0)); + assert_eq!(policy.select_worker(&workers, None), Some(1)); + } + + #[test] + fn test_round_robin_with_unhealthy_workers() { + let policy = RoundRobinPolicy::new(); + let workers: Vec> = vec![ + Arc::new( + BasicWorkerBuilder::new("http://w1:8000") + .worker_type(WorkerType::Regular) + .build(), + ), + Arc::new( + BasicWorkerBuilder::new("http://w2:8000") + .worker_type(WorkerType::Regular) + .build(), + ), + Arc::new( + BasicWorkerBuilder::new("http://w3:8000") + .worker_type(WorkerType::Regular) + .build(), + ), + ]; + + // Mark middle worker as unhealthy + workers[1].set_healthy(false); + + // Should skip unhealthy worker: 0, 2, 0, 2, ... + assert_eq!(policy.select_worker(&workers, None), Some(0)); + assert_eq!(policy.select_worker(&workers, None), Some(2)); + assert_eq!(policy.select_worker(&workers, None), Some(0)); + assert_eq!(policy.select_worker(&workers, None), Some(2)); + } + + #[test] + fn test_round_robin_reset() { + let policy = RoundRobinPolicy::new(); + let workers: Vec> = vec![ + Arc::new( + BasicWorkerBuilder::new("http://w1:8000") + .worker_type(WorkerType::Regular) + .build(), + ), + Arc::new( + BasicWorkerBuilder::new("http://w2:8000") + .worker_type(WorkerType::Regular) + .build(), + ), + ]; + + // Advance the counter + assert_eq!(policy.select_worker(&workers, None), Some(0)); + assert_eq!(policy.select_worker(&workers, None), Some(1)); + + // Reset should start from beginning + policy.reset(); + assert_eq!(policy.select_worker(&workers, None), Some(0)); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/tree.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/tree.rs new file mode 100644 index 00000000..4ad10fff --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/policies/tree.rs @@ -0,0 +1,1450 @@ +use std::{ + cmp::Reverse, + collections::{BinaryHeap, HashMap, VecDeque}, + sync::{Arc, RwLock}, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use dashmap::{mapref::entry::Entry, DashMap}; +use tracing::info; + +type NodeRef = Arc; + +#[derive(Debug)] +struct Node { + children: DashMap, + text: RwLock, + tenant_last_access_time: DashMap, + parent: RwLock>, +} + +#[derive(Debug)] +pub struct Tree { + root: NodeRef, + pub tenant_char_count: DashMap, +} + +// For the heap + +struct EvictionEntry { + timestamp: u128, + tenant: String, + node: NodeRef, +} + +impl Eq for EvictionEntry {} + +#[allow(clippy::non_canonical_partial_ord_impl)] +impl PartialOrd for EvictionEntry { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.timestamp.cmp(&other.timestamp)) + } +} + +impl Ord for EvictionEntry { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.timestamp.cmp(&other.timestamp) + } +} + +impl PartialEq for EvictionEntry { + fn eq(&self, other: &Self) -> bool { + self.timestamp == other.timestamp + } +} + +// For char operations +// Note that in rust, `.len()` or slice is operated on the "byte" level. It causes issues for UTF-8 characters because one character might use multiple bytes. +// https://en.wikipedia.org/wiki/UTF-8 + +fn shared_prefix_count(a: &str, b: &str) -> usize { + let mut i = 0; + let mut a_iter = a.chars(); + let mut b_iter = b.chars(); + + loop { + match (a_iter.next(), b_iter.next()) { + (Some(a_char), Some(b_char)) if a_char == b_char => { + i += 1; + } + _ => break, + } + } + + i +} + +fn slice_by_chars(s: &str, start: usize, end: usize) -> String { + s.chars().skip(start).take(end - start).collect() +} + +impl Default for Tree { + fn default() -> Self { + Self::new() + } +} + +impl Tree { + /* + Thread-safe multi tenant radix tree + + 1. Storing data for multiple tenants (the overlap of multiple radix tree) + 2. Node-level lock to enable concurrent access on nodes + 3. Leaf LRU eviction based on tenant access time + */ + + pub fn new() -> Self { + Tree { + root: Arc::new(Node { + children: DashMap::new(), + text: RwLock::new("".to_string()), + tenant_last_access_time: DashMap::new(), + parent: RwLock::new(None), + }), + tenant_char_count: DashMap::new(), + } + } + + pub fn insert(&self, text: &str, tenant: &str) { + // Insert text into tree with given tenant + + let mut curr = Arc::clone(&self.root); + let mut curr_idx = 0; + + let timestamp_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis(); + + curr.tenant_last_access_time + .insert(tenant.to_string(), timestamp_ms); + + self.tenant_char_count + .entry(tenant.to_string()) + .or_insert(0); + + let mut prev = Arc::clone(&self.root); + + let text_count = text.chars().count(); + + while curr_idx < text_count { + let first_char = text.chars().nth(curr_idx).unwrap(); + + curr = prev; + + // dashmap.entry locks the entry until the op is done + // if using contains_key + insert, there will be an issue that + // 1. "apple" and "app" entered at the same time + // 2. and get inserted to the dashmap concurrently, so only one is inserted + + match curr.children.entry(first_char) { + Entry::Vacant(entry) => { + /* + no matched + [curr] + becomes + [curr] => [new node] + */ + + let curr_text = slice_by_chars(text, curr_idx, text_count); + let curr_text_count = curr_text.chars().count(); + let new_node = Arc::new(Node { + children: DashMap::new(), + text: RwLock::new(curr_text), + tenant_last_access_time: DashMap::new(), + parent: RwLock::new(Some(Arc::clone(&curr))), + }); + + // Attach tenant to the new node (map is empty here) and increment count once + self.tenant_char_count + .entry(tenant.to_string()) + .and_modify(|count| *count += curr_text_count) + .or_insert(curr_text_count); + new_node + .tenant_last_access_time + .insert(tenant.to_string(), timestamp_ms); + + entry.insert(Arc::clone(&new_node)); + + prev = Arc::clone(&new_node); + curr_idx = text_count; + } + + Entry::Occupied(mut entry) => { + // matched + let matched_node = entry.get().clone(); + + let matched_node_text = matched_node.text.read().unwrap().to_owned(); + let matched_node_text_count = matched_node_text.chars().count(); + + let curr_text = slice_by_chars(text, curr_idx, text_count); + let shared_count = shared_prefix_count(&matched_node_text, &curr_text); + + if shared_count < matched_node_text_count { + /* + split the matched node + [curr] -> [matched_node] => + becomes + [curr] -> [new_node] -> [contracted_matched_node] + */ + + let matched_text = slice_by_chars(&matched_node_text, 0, shared_count); + let contracted_text = slice_by_chars( + &matched_node_text, + shared_count, + matched_node_text_count, + ); + let matched_text_count = matched_text.chars().count(); + + let new_node = Arc::new(Node { + text: RwLock::new(matched_text), + children: DashMap::new(), + parent: RwLock::new(Some(Arc::clone(&curr))), + tenant_last_access_time: matched_node.tenant_last_access_time.clone(), + }); + + let first_new_char = contracted_text.chars().nth(0).unwrap(); + new_node + .children + .insert(first_new_char, Arc::clone(&matched_node)); + + entry.insert(Arc::clone(&new_node)); + + *matched_node.text.write().unwrap() = contracted_text; + *matched_node.parent.write().unwrap() = Some(Arc::clone(&new_node)); + + prev = Arc::clone(&new_node); + + // Atomically attach tenant to the new split node and increment count once + match prev.tenant_last_access_time.entry(tenant.to_string()) { + Entry::Vacant(v) => { + self.tenant_char_count + .entry(tenant.to_string()) + .and_modify(|count| *count += matched_text_count) + .or_insert(matched_text_count); + v.insert(timestamp_ms); + } + Entry::Occupied(mut o) => { + o.insert(timestamp_ms); + } + } + + curr_idx += shared_count; + } else { + // move to next node + prev = Arc::clone(&matched_node); + + // Atomically attach tenant to existing node and increment count once + match prev.tenant_last_access_time.entry(tenant.to_string()) { + Entry::Vacant(v) => { + self.tenant_char_count + .entry(tenant.to_string()) + .and_modify(|count| *count += matched_node_text_count) + .or_insert(matched_node_text_count); + v.insert(timestamp_ms); + } + Entry::Occupied(mut o) => { + o.insert(timestamp_ms); + } + } + curr_idx += shared_count; + } + } + } + } + } + + #[allow(unused_assignments)] + pub fn prefix_match(&self, text: &str) -> (String, String) { + let mut curr = Arc::clone(&self.root); + let mut curr_idx = 0; + + let mut prev = Arc::clone(&self.root); + let text_count = text.chars().count(); + + while curr_idx < text_count { + let first_char = text.chars().nth(curr_idx).unwrap(); + let curr_text = slice_by_chars(text, curr_idx, text_count); + + curr = prev.clone(); + + if let Some(entry) = curr.children.get(&first_char) { + let matched_node = entry.value().clone(); + let matched_text_guard = matched_node.text.read().unwrap(); + let shared_count = shared_prefix_count(&matched_text_guard, &curr_text); + let matched_node_text_count = matched_text_guard.chars().count(); + drop(matched_text_guard); + + if shared_count == matched_node_text_count { + // Full match with current node's text, continue to next node + curr_idx += shared_count; + prev = Arc::clone(&matched_node); + } else { + // Partial match, stop here + curr_idx += shared_count; + prev = Arc::clone(&matched_node); + break; + } + } else { + // No match found, stop here + break; + } + } + + curr = prev.clone(); + + // Select the first tenant (key in the map) + let tenant = curr + .tenant_last_access_time + .iter() + .next() + .map(|kv| kv.key().to_owned()) + .unwrap_or("empty".to_string()); + + // Traverse from the curr node to the root and update the timestamp + + let timestamp_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis(); + + if !tenant.eq("empty") { + let mut current_node = Some(curr); + while let Some(node) = current_node { + node.tenant_last_access_time + .insert(tenant.clone(), timestamp_ms); + current_node = node.parent.read().unwrap().clone(); + } + } + + let ret_text = slice_by_chars(text, 0, curr_idx); + (ret_text, tenant) + } + + #[allow(unused_assignments, dead_code)] + pub fn prefix_match_tenant(&self, text: &str, tenant: &str) -> String { + let mut curr = Arc::clone(&self.root); + let mut curr_idx = 0; + + let mut prev = Arc::clone(&self.root); + let text_count = text.chars().count(); + + while curr_idx < text_count { + let first_char = text.chars().nth(curr_idx).unwrap(); + let curr_text = slice_by_chars(text, curr_idx, text_count); + + curr = prev.clone(); + + if let Some(entry) = curr.children.get(&first_char) { + let matched_node = entry.value().clone(); + + // Only continue matching if this node belongs to the specified tenant + if !matched_node.tenant_last_access_time.contains_key(tenant) { + break; + } + + let matched_text_guard = matched_node.text.read().unwrap(); + let shared_count = shared_prefix_count(&matched_text_guard, &curr_text); + let matched_node_text_count = matched_text_guard.chars().count(); + drop(matched_text_guard); + + if shared_count == matched_node_text_count { + // Full match with current node's text, continue to next node + curr_idx += shared_count; + prev = Arc::clone(&matched_node); + } else { + // Partial match, stop here + curr_idx += shared_count; + prev = Arc::clone(&matched_node); + break; + } + } else { + // No match found, stop here + break; + } + } + + curr = prev.clone(); + + // Only update timestamp if we found a match for the specified tenant + if curr.tenant_last_access_time.contains_key(tenant) { + let timestamp_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis(); + + let mut current_node = Some(curr); + while let Some(node) = current_node { + node.tenant_last_access_time + .insert(tenant.to_string(), timestamp_ms); + current_node = node.parent.read().unwrap().clone(); + } + } + + slice_by_chars(text, 0, curr_idx) + } + + fn leaf_of(node: &NodeRef) -> Vec { + /* + Return the list of tenants if it's a leaf for the tenant + */ + let mut candidates: HashMap = node + .tenant_last_access_time + .iter() + .map(|entry| (entry.key().clone(), true)) + .collect(); + + for child in node.children.iter() { + for tenant in child.value().tenant_last_access_time.iter() { + candidates.insert(tenant.key().clone(), false); + } + } + + candidates + .into_iter() + .filter(|(_, is_leaf)| *is_leaf) + .map(|(tenant, _)| tenant) + .collect() + } + + pub fn evict_tenant_by_size(&self, max_size: usize) { + // Calculate used size and collect leaves + let mut stack = vec![Arc::clone(&self.root)]; + let mut pq = BinaryHeap::new(); + + while let Some(curr) = stack.pop() { + for child in curr.children.iter() { + stack.push(Arc::clone(child.value())); + } + + // Add leaves to priority queue + for tenant in Tree::leaf_of(&curr) { + if let Some(timestamp) = curr.tenant_last_access_time.get(&tenant) { + pq.push(Reverse(EvictionEntry { + timestamp: *timestamp, + tenant: tenant.clone(), + node: Arc::clone(&curr), + })); + } + } + } + + info!("Before eviction - Used size per tenant:"); + for entry in self.tenant_char_count.iter() { + info!("Tenant: {}, Size: {}", entry.key(), entry.value()); + } + + // Process eviction + while let Some(Reverse(entry)) = pq.pop() { + let EvictionEntry { tenant, node, .. } = entry; + + if let Some(used_size) = self.tenant_char_count.get(&tenant) { + if *used_size <= max_size { + continue; + } + } + + // Decrement when removing tenant from node + if node.tenant_last_access_time.contains_key(&tenant) { + let node_len = node.text.read().unwrap().chars().count(); + self.tenant_char_count + .entry(tenant.clone()) + .and_modify(|count| { + *count = count.saturating_sub(node_len); + }); + } + + // Remove tenant from node + node.tenant_last_access_time.remove(&tenant); + + // Remove empty nodes + if node.children.is_empty() && node.tenant_last_access_time.is_empty() { + if let Some(parent) = node.parent.read().unwrap().as_ref() { + let text_guard = node.text.read().unwrap(); + if let Some(first_char) = text_guard.chars().next() { + parent.children.remove(&first_char); + } + } + } + + // Add parent to queue if it becomes a leaf + if let Some(parent) = node.parent.read().unwrap().as_ref() { + if Tree::leaf_of(parent).contains(&tenant) { + if let Some(timestamp) = parent.tenant_last_access_time.get(&tenant) { + pq.push(Reverse(EvictionEntry { + timestamp: *timestamp, + tenant: tenant.clone(), + node: Arc::clone(parent), + })); + } + } + }; + } + + info!("After eviction - Used size per tenant:"); + for entry in self.tenant_char_count.iter() { + info!("Tenant: {}, Size: {}", entry.key(), entry.value()); + } + } + + pub fn remove_tenant(&self, tenant: &str) { + // 1. Find all the leaves for the tenant + let mut stack = vec![Arc::clone(&self.root)]; + let mut queue = VecDeque::new(); + + while let Some(curr) = stack.pop() { + for child in curr.children.iter() { + stack.push(Arc::clone(child.value())); + } + + if Tree::leaf_of(&curr).contains(&tenant.to_string()) { + queue.push_back(Arc::clone(&curr)); + } + } + + // 2. Start from the leaves and traverse up to the root, removing the tenant from each node + while let Some(curr) = queue.pop_front() { + // remove tenant from node + curr.tenant_last_access_time.remove(&tenant.to_string()); + + // remove empty nodes + if curr.children.is_empty() && curr.tenant_last_access_time.is_empty() { + if let Some(parent) = curr.parent.read().unwrap().as_ref() { + let text_guard = curr.text.read().unwrap(); + if let Some(first_char) = text_guard.chars().next() { + parent.children.remove(&first_char); + } + } + } + + // add parent to queue if it becomes a leaf + if let Some(parent) = curr.parent.read().unwrap().as_ref() { + if Tree::leaf_of(parent).contains(&tenant.to_string()) { + queue.push_back(Arc::clone(parent)); + } + } + } + + // 3. Remove the tenant from the tenant_char_count map + self.tenant_char_count.remove(&tenant.to_string()); + } + + #[allow(dead_code)] + pub fn get_tenant_char_count(&self) -> HashMap { + self.tenant_char_count + .iter() + .map(|entry| (entry.key().clone(), *entry.value())) + .collect() + } + + pub fn get_smallest_tenant(&self) -> String { + // Return a placeholder if there are no tenants + if self.tenant_char_count.is_empty() { + return "empty".to_string(); + } + + // Find the tenant with minimum char count + let mut min_tenant = None; + let mut min_count = usize::MAX; + + for entry in self.tenant_char_count.iter() { + let tenant = entry.key(); + let count = *entry.value(); + + if count < min_count { + min_count = count; + min_tenant = Some(tenant.clone()); + } + } + + // Return the found tenant or "empty" if somehow none was found + min_tenant.unwrap_or_else(|| "empty".to_string()) + } + + #[allow(dead_code)] + pub fn get_used_size_per_tenant(&self) -> HashMap { + // perform a DFS to traverse all nodes and calculate the total size used by each tenant + + let mut used_size_per_tenant: HashMap = HashMap::new(); + let mut stack = vec![Arc::clone(&self.root)]; + + while let Some(curr) = stack.pop() { + let text_count = curr.text.read().unwrap().chars().count(); + + for tenant in curr.tenant_last_access_time.iter() { + let size = used_size_per_tenant + .entry(tenant.key().clone()) + .or_insert(0); + *size += text_count; + } + + for child in curr.children.iter() { + stack.push(Arc::clone(child.value())); + } + } + + used_size_per_tenant + } + + #[allow(dead_code)] + fn node_to_string(node: &NodeRef, prefix: &str, is_last: bool) -> String { + let mut result = String::new(); + + // Add prefix and branch character + result.push_str(prefix); + result.push_str(if is_last { "└── " } else { "├── " }); + + // Add node text + let node_text = node.text.read().unwrap(); + result.push_str(&format!("'{}' [", node_text)); + + // Add tenant information with timestamps + let mut tenant_info = Vec::new(); + for entry in node.tenant_last_access_time.iter() { + let tenant_id = entry.key(); + let timestamp_ms = entry.value(); + + // Convert milliseconds to seconds and remaining milliseconds + let seconds = (timestamp_ms / 1000) as u64; + let millis = (timestamp_ms % 1000) as u32; + + // Create SystemTime from Unix timestamp + let system_time = UNIX_EPOCH + Duration::from_secs(seconds); + + // Format time as HH:MM:SS.mmm + let datetime = system_time.duration_since(UNIX_EPOCH).unwrap(); + let hours = (datetime.as_secs() % 86400) / 3600; + let minutes = (datetime.as_secs() % 3600) / 60; + let seconds = datetime.as_secs() % 60; + + tenant_info.push(format!( + "{} | {:02}:{:02}:{:02}.{:03}", + tenant_id, hours, minutes, seconds, millis + )); + } + + result.push_str(&tenant_info.join(", ")); + result.push_str("]\n"); + + // Process children + let children: Vec<_> = node.children.iter().collect(); + let child_count = children.len(); + + for (i, entry) in children.iter().enumerate() { + let is_last_child = i == child_count - 1; + let new_prefix = format!("{}{}", prefix, if is_last { " " } else { "│ " }); + + result.push_str(&Tree::node_to_string( + entry.value(), + &new_prefix, + is_last_child, + )); + } + + result + } + + #[allow(dead_code)] + pub fn pretty_print(&self) { + if self.root.children.is_empty() { + return; + } + + let mut result = String::new(); + let children: Vec<_> = self.root.children.iter().collect(); + let child_count = children.len(); + + for (i, entry) in children.iter().enumerate() { + let is_last = i == child_count - 1; + result.push_str(&Tree::node_to_string(entry.value(), "", is_last)); + } + + println!("{result}"); + } +} + +// Unit tests +#[cfg(test)] +mod tests { + use std::{thread, time::Instant}; + + use rand::{ + distr::{Alphanumeric, SampleString}, + rng as thread_rng, Rng, + }; + + use super::*; + + #[test] + fn test_get_smallest_tenant() { + let tree = Tree::new(); + + assert_eq!(tree.get_smallest_tenant(), "empty"); + + // Insert data for tenant1 - "ap" + "icot" = 6 chars + tree.insert("ap", "tenant1"); + tree.insert("icot", "tenant1"); + + // Insert data for tenant2 - "cat" = 3 chars + tree.insert("cat", "tenant2"); + + assert_eq!( + tree.get_smallest_tenant(), + "tenant2", + "Expected tenant2 to be smallest with 3 characters." + ); + + // Insert overlapping data for tenant3 and tenant4 to test equal counts + // tenant3: "do" = 2 chars + // tenant4: "hi" = 2 chars + tree.insert("do", "tenant3"); + tree.insert("hi", "tenant4"); + + let smallest = tree.get_smallest_tenant(); + assert!( + smallest == "tenant3" || smallest == "tenant4", + "Expected either tenant3 or tenant4 (both have 2 characters), got {}", + smallest + ); + + // Add more text to tenant4 to make it larger + tree.insert("hello", "tenant4"); // Now tenant4 has "hi" + "hello" = 6 chars + + // Now tenant3 should be smallest (2 chars vs 6 chars for tenant4) + assert_eq!( + tree.get_smallest_tenant(), + "tenant3", + "Expected tenant3 to be smallest with 2 characters" + ); + + tree.evict_tenant_by_size(3); // This should evict tenants with more than 3 chars + + let post_eviction_smallest = tree.get_smallest_tenant(); + println!("Smallest tenant after eviction: {}", post_eviction_smallest); + } + + #[test] + fn test_tenant_char_count() { + let tree = Tree::new(); + + tree.insert("apple", "tenant1"); + tree.insert("apricot", "tenant1"); + tree.insert("banana", "tenant1"); + tree.insert("amplify", "tenant2"); + tree.insert("application", "tenant2"); + + let computed_sizes = tree.get_used_size_per_tenant(); + let maintained_counts: HashMap = tree + .tenant_char_count + .iter() + .map(|entry| (entry.key().clone(), *entry.value())) + .collect(); + + println!("Phase 1 - Maintained vs Computed counts:"); + println!( + "Maintained: {:?}\nComputed: {:?}", + maintained_counts, computed_sizes + ); + assert_eq!( + maintained_counts, computed_sizes, + "Phase 1: Initial insertions" + ); + + tree.insert("apartment", "tenant1"); + tree.insert("appetite", "tenant2"); + tree.insert("ball", "tenant1"); + tree.insert("box", "tenant2"); + + let computed_sizes = tree.get_used_size_per_tenant(); + let maintained_counts: HashMap = tree + .tenant_char_count + .iter() + .map(|entry| (entry.key().clone(), *entry.value())) + .collect(); + + println!("Phase 2 - Maintained vs Computed counts:"); + println!( + "Maintained: {:?}\nComputed: {:?}", + maintained_counts, computed_sizes + ); + assert_eq!( + maintained_counts, computed_sizes, + "Phase 2: Additional insertions" + ); + + tree.insert("zebra", "tenant1"); + tree.insert("zebra", "tenant2"); + tree.insert("zero", "tenant1"); + tree.insert("zero", "tenant2"); + + let computed_sizes = tree.get_used_size_per_tenant(); + let maintained_counts: HashMap = tree + .tenant_char_count + .iter() + .map(|entry| (entry.key().clone(), *entry.value())) + .collect(); + + println!("Phase 3 - Maintained vs Computed counts:"); + println!( + "Maintained: {:?}\nComputed: {:?}", + maintained_counts, computed_sizes + ); + assert_eq!( + maintained_counts, computed_sizes, + "Phase 3: Overlapping insertions" + ); + + tree.evict_tenant_by_size(10); + + let computed_sizes = tree.get_used_size_per_tenant(); + let maintained_counts: HashMap = tree + .tenant_char_count + .iter() + .map(|entry| (entry.key().clone(), *entry.value())) + .collect(); + + println!("Phase 4 - Maintained vs Computed counts:"); + println!( + "Maintained: {:?}\nComputed: {:?}", + maintained_counts, computed_sizes + ); + assert_eq!(maintained_counts, computed_sizes, "Phase 4: After eviction"); + } + + fn random_string(len: usize) -> String { + Alphanumeric.sample_string(&mut thread_rng(), len) + } + + #[test] + fn test_cold_start() { + let tree = Tree::new(); + + let (matched_text, tenant) = tree.prefix_match("hello"); + + assert_eq!(matched_text, ""); + assert_eq!(tenant, "empty"); + } + + #[test] + fn test_exact_match_seq() { + let tree = Tree::new(); + tree.insert("hello", "tenant1"); + tree.pretty_print(); + tree.insert("apple", "tenant2"); + tree.pretty_print(); + tree.insert("banana", "tenant3"); + tree.pretty_print(); + + let (matched_text, tenant) = tree.prefix_match("hello"); + assert_eq!(matched_text, "hello"); + assert_eq!(tenant, "tenant1"); + + let (matched_text, tenant) = tree.prefix_match("apple"); + assert_eq!(matched_text, "apple"); + assert_eq!(tenant, "tenant2"); + + let (matched_text, tenant) = tree.prefix_match("banana"); + assert_eq!(matched_text, "banana"); + assert_eq!(tenant, "tenant3"); + } + + #[test] + fn test_exact_match_concurrent() { + let tree = Arc::new(Tree::new()); + + // spawn 3 threads for insert + let tree_clone = Arc::clone(&tree); + + let texts = ["hello", "apple", "banana"]; + let tenants = ["tenant1", "tenant2", "tenant3"]; + + let mut handles = vec![]; + + for i in 0..3 { + let tree_clone = Arc::clone(&tree_clone); + let text = texts[i]; + let tenant = tenants[i]; + + let handle = thread::spawn(move || { + tree_clone.insert(text, tenant); + }); + + handles.push(handle); + } + + // wait + for handle in handles { + handle.join().unwrap(); + } + + // spawn 3 threads for match + let mut handles = vec![]; + + let tree_clone = Arc::clone(&tree); + + for i in 0..3 { + let tree_clone = Arc::clone(&tree_clone); + let text = texts[i]; + let tenant = tenants[i]; + + let handle = thread::spawn(move || { + let (matched_text, matched_tenant) = tree_clone.prefix_match(text); + assert_eq!(matched_text, text); + assert_eq!(matched_tenant, tenant); + }); + + handles.push(handle); + } + + // wait + for handle in handles { + handle.join().unwrap(); + } + } + + #[test] + fn test_partial_match_concurrent() { + let tree = Arc::new(Tree::new()); + + // spawn 3 threads for insert + let tree_clone = Arc::clone(&tree); + + static TEXTS: [&str; 3] = ["apple", "apabc", "acbdeds"]; + + let mut handles = vec![]; + + for text in TEXTS.iter() { + let tree_clone = Arc::clone(&tree_clone); + let tenant = "tenant0"; + + let handle = thread::spawn(move || { + tree_clone.insert(text, tenant); + }); + + handles.push(handle); + } + + // wait + for handle in handles { + handle.join().unwrap(); + } + + // spawn 3 threads for match + let mut handles = vec![]; + + let tree_clone = Arc::clone(&tree); + + for text in TEXTS.iter() { + let tree_clone = Arc::clone(&tree_clone); + let tenant = "tenant0"; + + let handle = thread::spawn(move || { + let (matched_text, matched_tenant) = tree_clone.prefix_match(text); + assert_eq!(matched_text, *text); + assert_eq!(matched_tenant, tenant); + }); + + handles.push(handle); + } + + // wait + for handle in handles { + handle.join().unwrap(); + } + } + + #[test] + fn test_group_prefix_insert_match_concurrent() { + static PREFIXES: [&str; 4] = [ + "Clock strikes midnight, I'm still wide awake", + "Got dreams bigger than these city lights", + "Time waits for no one, gotta make my move", + "Started from the bottom, that's no metaphor", + ]; + let suffixes = [ + "Got too much to prove, ain't got time to lose", + "History in the making, yeah, you can't erase this", + ]; + let tree = Arc::new(Tree::new()); + + let mut handles = vec![]; + + for (i, prefix) in PREFIXES.iter().enumerate() { + for suffix in suffixes.iter() { + let tree_clone = Arc::clone(&tree); + let text = format!("{} {}", prefix, suffix); + let tenant = format!("tenant{}", i); + + let handle = thread::spawn(move || { + tree_clone.insert(&text, &tenant); + }); + + handles.push(handle); + } + } + + // wait + for handle in handles { + handle.join().unwrap(); + } + + tree.pretty_print(); + + // check matching using multi threads + let mut handles = vec![]; + + for (i, prefix) in PREFIXES.iter().enumerate() { + let tree_clone = Arc::clone(&tree); + + let handle = thread::spawn(move || { + let (matched_text, matched_tenant) = tree_clone.prefix_match(prefix); + let tenant = format!("tenant{}", i); + assert_eq!(matched_text, *prefix); + assert_eq!(matched_tenant, tenant); + }); + + handles.push(handle); + } + + // wait + for handle in handles { + handle.join().unwrap(); + } + } + + #[test] + fn test_mixed_concurrent_insert_match() { + // ensure it does not deadlock instead of doing correctness check + + static PREFIXES: [&str; 4] = [ + "Clock strikes midnight, I'm still wide awake", + "Got dreams bigger than these city lights", + "Time waits for no one, gotta make my move", + "Started from the bottom, that's no metaphor", + ]; + let suffixes = [ + "Got too much to prove, ain't got time to lose", + "History in the making, yeah, you can't erase this", + ]; + let tree = Arc::new(Tree::new()); + + let mut handles = vec![]; + + for (i, prefix) in PREFIXES.iter().enumerate() { + for suffix in suffixes.iter() { + let tree_clone = Arc::clone(&tree); + let text = format!("{} {}", prefix, suffix); + let tenant = format!("tenant{}", i); + + let handle = thread::spawn(move || { + tree_clone.insert(&text, &tenant); + }); + + handles.push(handle); + } + } + + // check matching using multi threads + for prefix in PREFIXES.iter() { + let tree_clone = Arc::clone(&tree); + + let handle = thread::spawn(move || { + let (_matched_text, _matched_tenant) = tree_clone.prefix_match(prefix); + }); + + handles.push(handle); + } + + // wait + for handle in handles { + handle.join().unwrap(); + } + } + + #[test] + fn test_utf8_split_seq() { + // The string should be indexed and split by a utf-8 value basis instead of byte basis + // use .chars() to get the iterator of the utf-8 value + let tree = Arc::new(Tree::new()); + + static TEST_PAIRS: [(&str, &str); 3] = [ + ("你好嗎", "tenant1"), + ("你好喔", "tenant2"), + ("你心情好嗎", "tenant3"), + ]; + + // Insert sequentially + for (text, tenant) in TEST_PAIRS.iter() { + tree.insert(text, tenant); + } + + tree.pretty_print(); + + for (text, tenant) in TEST_PAIRS.iter() { + let (matched_text, matched_tenant) = tree.prefix_match(text); + assert_eq!(matched_text, *text); + assert_eq!(matched_tenant, *tenant); + } + } + + #[test] + fn test_utf8_split_concurrent() { + let tree = Arc::new(Tree::new()); + + static TEST_PAIRS: [(&str, &str); 3] = [ + ("你好嗎", "tenant1"), + ("你好喔", "tenant2"), + ("你心情好嗎", "tenant3"), + ]; + + // Create multiple threads for insertion + let mut handles = vec![]; + + for (text, tenant) in TEST_PAIRS.iter() { + let tree_clone = Arc::clone(&tree); + + let handle = thread::spawn(move || { + tree_clone.insert(text, tenant); + }); + + handles.push(handle); + } + + // Wait for all insertions to complete + for handle in handles { + handle.join().unwrap(); + } + + tree.pretty_print(); + + // Create multiple threads for matching + let mut handles = vec![]; + + for (text, tenant) in TEST_PAIRS.iter() { + let tree_clone = Arc::clone(&tree); + + let handle = thread::spawn(move || { + let (matched_text, matched_tenant) = tree_clone.prefix_match(text); + assert_eq!(matched_text, *text); + assert_eq!(matched_tenant, *tenant); + }); + + handles.push(handle); + } + + // Wait for all matches to complete + for handle in handles { + handle.join().unwrap(); + } + } + + #[test] + fn test_simple_eviction() { + let tree = Tree::new(); + let max_size = 5; + + // Insert strings for both tenants + tree.insert("hello", "tenant1"); // size 5 + + tree.insert("hello", "tenant2"); // size 5 + thread::sleep(Duration::from_millis(10)); + tree.insert("world", "tenant2"); // size 5, total for tenant2 = 10 + + tree.pretty_print(); + + let sizes_before = tree.get_used_size_per_tenant(); + assert_eq!(sizes_before.get("tenant1").unwrap(), &5); // "hello" = 5 + assert_eq!(sizes_before.get("tenant2").unwrap(), &10); // "hello" + "world" = 10 + + // Evict - should remove "hello" from tenant2 as it's the oldest + tree.evict_tenant_by_size(max_size); + + tree.pretty_print(); + + let sizes_after = tree.get_used_size_per_tenant(); + assert_eq!(sizes_after.get("tenant1").unwrap(), &5); // Should be unchanged + assert_eq!(sizes_after.get("tenant2").unwrap(), &5); // Only "world" remains + + let (matched, tenant) = tree.prefix_match("world"); + assert_eq!(matched, "world"); + assert_eq!(tenant, "tenant2"); + } + + #[test] + fn test_advanced_eviction() { + let tree = Tree::new(); + + // Set limits for each tenant + let max_size: usize = 100; + + // Define prefixes + let prefixes = ["aqwefcisdf", "iajsdfkmade", "kjnzxcvewqe", "iejksduqasd"]; + + // Insert strings with shared prefixes + for _i in 0..100 { + for (j, prefix) in prefixes.iter().enumerate() { + let random_suffix = random_string(10); + let text = format!("{}{}", prefix, random_suffix); + let tenant = format!("tenant{}", j + 1); + tree.insert(&text, &tenant); + } + } + + // Perform eviction + tree.evict_tenant_by_size(max_size); + + // Check sizes after eviction + let sizes_after = tree.get_used_size_per_tenant(); + for (tenant, &size) in sizes_after.iter() { + assert!( + size <= max_size, + "Tenant {} exceeds size limit. Current size: {}, Limit: {}", + tenant, + size, + max_size + ); + } + } + + #[test] + fn test_concurrent_operations_with_eviction() { + // Ensure eviction works fine with concurrent insert and match operations for a given period + + let tree = Arc::new(Tree::new()); + let mut handles = vec![]; + let test_duration = Duration::from_secs(10); + let start_time = Instant::now(); + let max_size = 100; // Single max size for all tenants + + // Spawn eviction thread + { + let tree = Arc::clone(&tree); + let handle = thread::spawn(move || { + while start_time.elapsed() < test_duration { + // Run eviction + tree.evict_tenant_by_size(max_size); + + // Sleep for 5 seconds + thread::sleep(Duration::from_secs(5)); + } + }); + handles.push(handle); + } + + // Spawn 4 worker threads + for thread_id in 0..4 { + let tree = Arc::clone(&tree); + let handle = thread::spawn(move || { + let mut rng = rand::rng(); + let tenant = format!("tenant{}", thread_id + 1); + let prefix = format!("prefix{}", thread_id); + + while start_time.elapsed() < test_duration { + // Random decision: match or insert (70% match, 30% insert) + if rng.random_bool(0.7) { + // Perform match operation + let random_len = rng.random_range(3..10); + let search_str = format!("{}{}", prefix, random_string(random_len)); + let (_matched, _) = tree.prefix_match(&search_str); + } else { + // Perform insert operation + let random_len = rng.random_range(5..15); + let insert_str = format!("{}{}", prefix, random_string(random_len)); + tree.insert(&insert_str, &tenant); + // println!("Thread {} inserted: {}", thread_id, insert_str); + } + + // Small random sleep to vary timing + thread::sleep(Duration::from_millis(rng.random_range(10..100))); + } + }); + handles.push(handle); + } + + // Wait for all threads to complete + for handle in handles { + handle.join().unwrap(); + } + + // final eviction + tree.evict_tenant_by_size(max_size); + + // Final size check + let final_sizes = tree.get_used_size_per_tenant(); + println!("Final sizes after test completion: {:?}", final_sizes); + + for (_, &size) in final_sizes.iter() { + assert!( + size <= max_size, + "Tenant exceeds size limit. Final size: {}, Limit: {}", + size, + max_size + ); + } + } + + #[test] + fn test_leaf_of() { + let tree = Tree::new(); + + // Single node + tree.insert("hello", "tenant1"); + let leaves = Tree::leaf_of(&tree.root.children.get(&'h').unwrap()); + assert_eq!(leaves, vec!["tenant1"]); + + // Node with multiple tenants + tree.insert("hello", "tenant2"); + let leaves = Tree::leaf_of(&tree.root.children.get(&'h').unwrap()); + assert_eq!(leaves.len(), 2); + assert!(leaves.contains(&"tenant1".to_string())); + assert!(leaves.contains(&"tenant2".to_string())); + + // Non-leaf node + tree.insert("hi", "tenant1"); + let leaves = Tree::leaf_of(&tree.root.children.get(&'h').unwrap()); + assert!(leaves.is_empty()); + } + + #[test] + fn test_get_used_size_per_tenant() { + let tree = Tree::new(); + + // Single tenant + tree.insert("hello", "tenant1"); + tree.insert("world", "tenant1"); + let sizes = tree.get_used_size_per_tenant(); + + tree.pretty_print(); + println!("{:?}", sizes); + assert_eq!(sizes.get("tenant1").unwrap(), &10); // "hello" + "world" + + // Multiple tenants sharing nodes + tree.insert("hello", "tenant2"); + tree.insert("help", "tenant2"); + let sizes = tree.get_used_size_per_tenant(); + + tree.pretty_print(); + println!("{:?}", sizes); + assert_eq!(sizes.get("tenant1").unwrap(), &10); + assert_eq!(sizes.get("tenant2").unwrap(), &6); // "hello" + "p" + + // UTF-8 characters + tree.insert("你好", "tenant3"); + let sizes = tree.get_used_size_per_tenant(); + tree.pretty_print(); + println!("{:?}", sizes); + assert_eq!(sizes.get("tenant3").unwrap(), &2); // 2 Chinese characters + + tree.pretty_print(); + } + + #[test] + fn test_prefix_match_tenant() { + let tree = Tree::new(); + + // Insert overlapping prefixes for different tenants + tree.insert("hello", "tenant1"); // tenant1: hello + tree.insert("hello", "tenant2"); // tenant2: hello + tree.insert("hello world", "tenant2"); // tenant2: hello -> world + tree.insert("help", "tenant1"); // tenant1: hel -> p + tree.insert("helicopter", "tenant2"); // tenant2: hel -> icopter + + assert_eq!(tree.prefix_match_tenant("hello", "tenant1"), "hello"); // Full match for tenant1 + assert_eq!(tree.prefix_match_tenant("help", "tenant1"), "help"); // Exclusive to tenant1 + assert_eq!(tree.prefix_match_tenant("hel", "tenant1"), "hel"); // Shared prefix + assert_eq!(tree.prefix_match_tenant("hello world", "tenant1"), "hello"); // Should stop at tenant1's boundary + assert_eq!(tree.prefix_match_tenant("helicopter", "tenant1"), "hel"); // Should stop at tenant1's boundary + + assert_eq!(tree.prefix_match_tenant("hello", "tenant2"), "hello"); // Full match for tenant2 + assert_eq!( + tree.prefix_match_tenant("hello world", "tenant2"), + "hello world" + ); // Exclusive to tenant2 + assert_eq!( + tree.prefix_match_tenant("helicopter", "tenant2"), + "helicopter" + ); // Exclusive to tenant2 + assert_eq!(tree.prefix_match_tenant("hel", "tenant2"), "hel"); // Shared prefix + assert_eq!(tree.prefix_match_tenant("help", "tenant2"), "hel"); // Should stop at tenant2's boundary + + assert_eq!(tree.prefix_match_tenant("hello", "tenant3"), ""); // Non-existent tenant + assert_eq!(tree.prefix_match_tenant("help", "tenant3"), ""); // Non-existent tenant + } + + #[test] + fn test_simple_tenant_eviction() { + let tree = Tree::new(); + + // Insert data for multiple tenants + tree.insert("hello", "tenant1"); + tree.insert("world", "tenant1"); + tree.insert("hello", "tenant2"); + tree.insert("help", "tenant2"); + + let initial_sizes = tree.get_used_size_per_tenant(); + assert_eq!(initial_sizes.get("tenant1").unwrap(), &10); // "hello" + "world" + assert_eq!(initial_sizes.get("tenant2").unwrap(), &6); // "hello" + "p" + + // Evict tenant1 + tree.remove_tenant("tenant1"); + + let final_sizes = tree.get_used_size_per_tenant(); + assert!( + !final_sizes.contains_key("tenant1"), + "tenant1 should be completely removed" + ); + assert_eq!( + final_sizes.get("tenant2").unwrap(), + &6, + "tenant2 should be unaffected" + ); + + assert_eq!(tree.prefix_match_tenant("hello", "tenant1"), ""); + assert_eq!(tree.prefix_match_tenant("world", "tenant1"), ""); + + assert_eq!(tree.prefix_match_tenant("hello", "tenant2"), "hello"); + assert_eq!(tree.prefix_match_tenant("help", "tenant2"), "help"); + } + + #[test] + fn test_complex_tenant_eviction() { + let tree = Tree::new(); + + // Create a more complex tree structure with shared prefixes + tree.insert("apple", "tenant1"); + tree.insert("application", "tenant1"); + tree.insert("apple", "tenant2"); + tree.insert("appetite", "tenant2"); + tree.insert("banana", "tenant1"); + tree.insert("banana", "tenant2"); + tree.insert("ball", "tenant2"); + + let initial_sizes = tree.get_used_size_per_tenant(); + println!("Initial sizes: {:?}", initial_sizes); + tree.pretty_print(); + + // Evict tenant1 + tree.remove_tenant("tenant1"); + + let final_sizes = tree.get_used_size_per_tenant(); + println!("Final sizes: {:?}", final_sizes); + tree.pretty_print(); + + assert!( + !final_sizes.contains_key("tenant1"), + "tenant1 should be completely removed" + ); + + assert_eq!(tree.prefix_match_tenant("apple", "tenant1"), ""); + assert_eq!(tree.prefix_match_tenant("application", "tenant1"), ""); + assert_eq!(tree.prefix_match_tenant("banana", "tenant1"), ""); + + assert_eq!(tree.prefix_match_tenant("apple", "tenant2"), "apple"); + assert_eq!(tree.prefix_match_tenant("appetite", "tenant2"), "appetite"); + assert_eq!(tree.prefix_match_tenant("banana", "tenant2"), "banana"); + assert_eq!(tree.prefix_match_tenant("ball", "tenant2"), "ball"); + + let tenant2_size = final_sizes.get("tenant2").unwrap(); + assert_eq!(tenant2_size, &(5 + 5 + 6 + 2)); // "apple" + "etite" + "banana" + "ll" + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/proto/sglang_scheduler.proto b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/proto/sglang_scheduler.proto new file mode 100644 index 00000000..f1d330d4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/proto/sglang_scheduler.proto @@ -0,0 +1,459 @@ +syntax = "proto3"; + +package sglang.grpc.scheduler; + +import "google/protobuf/timestamp.proto"; +import "google/protobuf/struct.proto"; + +// Service definition for SGLang scheduler communication +// This protocol bridges the Rust router and Python scheduler +service SglangScheduler { + // Submit a generation request (supports streaming) + rpc Generate(GenerateRequest) returns (stream GenerateResponse); + + // Submit an embedding request + rpc Embed(EmbedRequest) returns (EmbedResponse); + + // Health check and metrics + rpc HealthCheck(HealthCheckRequest) returns (HealthCheckResponse); + + // Abort a running request + rpc Abort(AbortRequest) returns (AbortResponse); + + // Get model information + rpc GetModelInfo(GetModelInfoRequest) returns (GetModelInfoResponse); + + // Get server information + rpc GetServerInfo(GetServerInfoRequest) returns (GetServerInfoResponse); + +} + +// ===================== +// Common Types +// ===================== + +// Sampling parameters matching SGLang's SamplingParams +// +// IMPORTANT: Do not use SamplingParams::default() directly! +// The proto3 defaults (0 for numeric fields) do NOT match the semantic defaults +// (temperature=1.0, top_p=1.0, top_k=-1, etc.). Always construct with explicit values +// or use the conversion functions in sglang_scheduler.rs / grpc_server.py. +message SamplingParams { + float temperature = 1; + float top_p = 2; + int32 top_k = 3; + float min_p = 4; + float frequency_penalty = 5; + float presence_penalty = 6; + float repetition_penalty = 7; + + optional int32 max_new_tokens = 8; + repeated string stop = 9; + repeated uint32 stop_token_ids = 10; + bool skip_special_tokens = 11; + bool spaces_between_special_tokens = 12; + + // Structured generation + oneof constraint { + string regex = 13; + string json_schema = 14; + string ebnf_grammar = 15; + string structural_tag = 16; + } + + // Speculative decoding + int32 n = 17; // Number of samples + + // Additional parameters + int32 min_new_tokens = 18; + bool ignore_eos = 19; + bool no_stop_trim = 20; + optional int32 stream_interval = 21; + map logit_bias = 22; + + // Custom parameters for extensibility + google.protobuf.Struct custom_params = 23; +} + + +// Disaggregated serving parameters +message DisaggregatedParams { + string bootstrap_host = 1; + int32 bootstrap_port = 2; + int32 bootstrap_room = 3; +} + +// ===================== +// Generate Request +// ===================== + +message GenerateRequest { + string request_id = 1; + + // Input must be tokenized (no raw text) + TokenizedInput tokenized = 2; + + // Multimodal inputs + MultimodalInputs mm_inputs = 3; + + // Generation parameters + SamplingParams sampling_params = 4; + + // Return options + bool return_logprob = 5; + int32 logprob_start_len = 6; + int32 top_logprobs_num = 7; + repeated uint32 token_ids_logprob = 8; + bool return_hidden_states = 9; + + // For disaggregated serving + DisaggregatedParams disaggregated_params = 10; + + // Custom logit processor (serialized) + string custom_logit_processor = 11; + + // Request metadata + google.protobuf.Timestamp timestamp = 12; + bool log_metrics = 13; + + // Input embeddings (alternative to text/tokens) + repeated float input_embeds = 14; + + // LoRA adapter ID (if pre-loaded) + string lora_id = 15; + + // Data parallel routing + int32 data_parallel_rank = 16; + + // Whether client wants streaming response + bool stream = 17; +} + +message TokenizedInput { + string original_text = 1; // For reference + repeated uint32 input_ids = 2; +} + +message MultimodalInputs { + // Simplified multimodal handling - actual data processed by tokenizer + repeated string image_urls = 1; + repeated string video_urls = 2; + repeated string audio_urls = 3; + + // Pre-processed multimodal features (if available) + google.protobuf.Struct processed_features = 4; + + // Raw data for direct processing + repeated bytes image_data = 5; + repeated bytes video_data = 6; + repeated bytes audio_data = 7; + + // Modality metadata + repeated string modalities = 8; +} + +// ===================== +// Generate Response +// ===================== + +message GenerateResponse { + string request_id = 1; + + // Response type + oneof response { + GenerateStreamChunk chunk = 2; + GenerateComplete complete = 3; + GenerateError error = 4; + } +} + +message GenerateStreamChunk { + // Generated tokens (incremental chunk) + repeated uint32 token_ids = 1; + + // Cumulative counts + int32 prompt_tokens = 2; + int32 completion_tokens = 3; + int32 cached_tokens = 4; + + // Output logprobs (if requested) - incremental for streaming + OutputLogProbs output_logprobs = 5; + + // Hidden states (if requested) + repeated float hidden_states = 6; + + // Input logprobs (if requested) - only in first chunk + InputLogProbs input_logprobs = 7; + + // Index for ordering when n>1 (for parallel request multiplexing) + uint32 index = 8; +} + +message GenerateComplete { + // Final output + repeated uint32 output_ids = 1; + + // Finish reason as OpenAI-compatible string ("stop", "length", "abort") + string finish_reason = 2; + + // Token usage counts + int32 prompt_tokens = 3; + int32 completion_tokens = 4; + int32 cached_tokens = 5; + + // Output logprobs if requested (cumulative) + OutputLogProbs output_logprobs = 6; + + // All hidden states if requested + repeated HiddenStates all_hidden_states = 7; + + // Matched stop information (for stop sequences) + oneof matched_stop { + uint32 matched_token_id = 8; + string matched_stop_str = 9; + } + + // Input logprobs if requested (for prompt tokens) + InputLogProbs input_logprobs = 10; + + // Index for ordering when n>1 (for parallel request multiplexing) + uint32 index = 11; +} + +message GenerateError { + string message = 1; + string http_status_code = 2; + string details = 3; +} + +// Output logprobs - all values are present (no None) +message OutputLogProbs { + repeated float token_logprobs = 1; + repeated int32 token_ids = 2; + + // Top logprobs at each position + repeated TopLogProbs top_logprobs = 3; +} + +// Input logprobs - first token has no logprob (None) +message InputLogProbs { + repeated InputTokenLogProb token_logprobs = 1; + repeated int32 token_ids = 2; + + // Top logprobs at each position + repeated TopLogProbs top_logprobs = 3; +} + +// Wrapper to represent optional logprob (first input token has no logprob) +message InputTokenLogProb { + optional float value = 1; +} + +message TopLogProbs { + repeated float values = 1; + repeated int32 token_ids = 2; +} + +message HiddenStates { + repeated float values = 1; + int32 layer = 2; + int32 position = 3; +} + +// ===================== +// Embedding Request +// ===================== + +message EmbedRequest { + string request_id = 1; + + // Input must be tokenized (no raw text) + TokenizedInput tokenized = 2; + + // Multimodal inputs + MultimodalInputs mm_inputs = 4; + + // Dummy sampling params for compatibility + // EmbedRequest doesn't use sampling_params + SamplingParams sampling_params = 5; + + bool log_metrics = 6; + + // Token type IDs for models that require them + repeated int32 token_type_ids = 7; + + // Data parallel routing + int32 data_parallel_rank = 8; + + // For cross-encoder requests + bool is_cross_encoder = 9; + repeated string texts = 10; // For cross-encoder batch +} + +message EmbedResponse { + string request_id = 1; + + oneof response { + EmbedComplete complete = 2; + EmbedError error = 3; + } +} + +message EmbedComplete { + repeated float embedding = 1; + int32 prompt_tokens = 2; + int32 cached_tokens = 3; + + // Additional metadata + int32 embedding_dim = 4; + + // For batch embeddings + repeated Embedding batch_embeddings = 5; +} + +message Embedding { + repeated float values = 1; + int32 index = 2; +} + +message EmbedError { + string message = 1; + string code = 2; + string details = 3; +} + +// ===================== +// Management Operations +// ===================== + +message HealthCheckRequest {} + +message HealthCheckResponse { + bool healthy = 1; + string message = 2; +} + +message AbortRequest { + string request_id = 1; + string reason = 2; +} + +message AbortResponse { + bool success = 1; + string message = 2; +} + + +// ===================== +// Additional Operations (Future) +// ===================== + +// Load LoRA adapter +message LoadLoRARequest { + string adapter_id = 1; + string adapter_path = 2; + int32 rank = 3; +} + +message LoadLoRAResponse { + bool success = 1; + string adapter_id = 2; + string message = 3; +} + +// Unload LoRA adapter +message UnloadLoRARequest { + string adapter_id = 1; +} + +message UnloadLoRAResponse { + bool success = 1; + string message = 2; +} + +// Update weights +message UpdateWeightsRequest { + oneof source { + string disk_path = 1; + bytes tensor_data = 2; + string remote_url = 3; + } + string weight_name = 4; +} + +message UpdateWeightsResponse { + bool success = 1; + string message = 2; +} + +// Get internal state for debugging +message GetInternalStateRequest { + repeated string state_keys = 1; +} + +message GetInternalStateResponse { + google.protobuf.Struct state = 1; +} + +// Set internal state for testing +message SetInternalStateRequest { + google.protobuf.Struct state = 1; +} + +message SetInternalStateResponse { + bool success = 1; + string message = 2; +} + +// ===================== +// Model and Server Info +// ===================== + +// Get model information +message GetModelInfoRequest {} + +message GetModelInfoResponse { + string model_path = 1; + string tokenizer_path = 2; + bool is_generation = 3; + string preferred_sampling_params = 4; // JSON string or empty + string weight_version = 5; + string served_model_name = 6; + int32 max_context_length = 7; + int32 vocab_size = 8; + bool supports_vision = 9; + string model_type = 10; + repeated int32 eos_token_ids = 11; + int32 pad_token_id = 12; + int32 bos_token_id = 13; + int32 max_req_input_len = 14; +} + +// Get server information +message GetServerInfoRequest {} + +message GetServerInfoResponse { + // Server configuration (as structured data) + google.protobuf.Struct server_args = 1; + + // Scheduler metrics (from scheduler initialization) + google.protobuf.Struct scheduler_info = 2; + + // Runtime state + int32 active_requests = 3; + bool is_paused = 4; + double last_receive_timestamp = 5; + double uptime_seconds = 6; + + // Version info + string sglang_version = 7; + + // Server metadata + string server_type = 8; // "grpc" + google.protobuf.Timestamp start_time = 9; + + // Note: internal_states not provided in gRPC mode + // Scheduler-side metrics (memory usage, throughput) require + // bidirectional communicator infrastructure not available in gRPC. + // Use HTTP /get_server_info if scheduler internal state is needed. +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/proto/vllm_engine.proto b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/proto/vllm_engine.proto new file mode 100644 index 00000000..f4e7934b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/proto/vllm_engine.proto @@ -0,0 +1,218 @@ +syntax = "proto3"; + +package vllm.grpc.engine; + +// Service definition for vLLM engine communication +// This protocol is designed for efficient binary communication between +// the Rust router and vLLM Python engine (AsyncLLM). +service VllmEngine { + // Submit a generation request (supports streaming) + rpc Generate(GenerateRequest) returns (stream GenerateResponse); + + // Submit an embedding request + rpc Embed(EmbedRequest) returns (EmbedResponse); + + // Health check + rpc HealthCheck(HealthCheckRequest) returns (HealthCheckResponse); + + // Abort a running request + rpc Abort(AbortRequest) returns (AbortResponse); + + // Get model information + rpc GetModelInfo(GetModelInfoRequest) returns (GetModelInfoResponse); + + // Get server information + rpc GetServerInfo(GetServerInfoRequest) returns (GetServerInfoResponse); +} + +// ===================== +// Common Types +// ===================== + +// Sampling parameters for text generation +message SamplingParams { + float temperature = 1; + float top_p = 2; + int32 top_k = 3; + float min_p = 4; + float frequency_penalty = 5; + float presence_penalty = 6; + float repetition_penalty = 7; + + optional int32 max_tokens = 8; + int32 min_tokens = 9; + + repeated string stop = 10; + repeated uint32 stop_token_ids = 11; + + bool skip_special_tokens = 12; + bool spaces_between_special_tokens = 13; + bool ignore_eos = 14; + + int32 n = 15; // Number of parallel samples + + // Logprobs configuration + optional int32 logprobs = 22; // Number of log probabilities per output token (-1 for all) + optional int32 prompt_logprobs = 23; // Number of log probabilities per prompt token (-1 for all) + + // Additional vLLM fields + optional int32 seed = 24; // Random seed for reproducibility + bool include_stop_str_in_output = 25; // Whether to include stop strings in output + map logit_bias = 26; // Token ID to bias mapping (-100 to 100) + optional int32 truncate_prompt_tokens = 27; // Prompt truncation (-1 for model max) + + // Structured outputs (one of) - matches vLLM's StructuredOutputsParams + oneof constraint { + string json_schema = 16; // JSON schema for structured output + string regex = 17; // Regex pattern + string grammar = 18; // Grammar/EBNF for structured output + string structural_tag = 19; // Structural tag (e.g., Harmony models) + bool json_object = 20; // Force JSON object output + ChoiceConstraint choice = 21; // List of allowed choices + } +} + +// Choice constraint for structured outputs +message ChoiceConstraint { + repeated string choices = 1; +} + +// Pre-tokenized input from Rust router +message TokenizedInput { + string original_text = 1; // For reference/debugging + repeated uint32 input_ids = 2; // Actual token IDs to process +} + +// ===================== +// Generate Request +// ===================== + +message GenerateRequest { + string request_id = 1; + + // Pre-tokenized input (required) + TokenizedInput tokenized = 2; + + // Generation parameters (includes logprobs config) + SamplingParams sampling_params = 3; + + // Streaming + bool stream = 4; +} + +// ===================== +// Generate Response +// ===================== + +message GenerateResponse { + string request_id = 1; + + oneof response { + GenerateStreamChunk chunk = 2; // For streaming + GenerateComplete complete = 3; // For final/non-streaming + GenerateError error = 4; // For errors + } +} + +message GenerateStreamChunk { + repeated uint32 token_ids = 1; // Incremental tokens + int32 prompt_tokens = 2; + int32 completion_tokens = 3; + int32 cached_tokens = 4; + + // Logprobs support (TODO: implement in Phase 4) + // OutputLogProbs output_logprobs = 5; + // InputLogProbs input_logprobs = 6; // Only in first chunk +} + +message GenerateComplete { + repeated uint32 output_ids = 1; // All output tokens + string finish_reason = 2; // "stop", "length", "abort" + int32 prompt_tokens = 3; + int32 completion_tokens = 4; + int32 cached_tokens = 5; + + // Logprobs support (TODO: implement in Phase 4) + // OutputLogProbs output_logprobs = 6; + // InputLogProbs input_logprobs = 7; +} + +message GenerateError { + string message = 1; + string http_status_code = 2; + string details = 3; +} + +// ===================== +// Embedding Request +// ===================== + +message EmbedRequest { + string request_id = 1; + TokenizedInput tokenized = 2; +} + +message EmbedResponse { + string request_id = 1; + + oneof response { + EmbedComplete complete = 2; + EmbedError error = 3; + } +} + +message EmbedComplete { + repeated float embedding = 1; + int32 prompt_tokens = 2; + int32 embedding_dim = 3; +} + +message EmbedError { + string message = 1; + string code = 2; +} + +// ===================== +// Management Operations +// ===================== + +message HealthCheckRequest {} + +message HealthCheckResponse { + bool healthy = 1; + string message = 2; +} + +message AbortRequest { + string request_id = 1; + string reason = 2; +} + +message AbortResponse { + bool success = 1; + string message = 2; +} + +// ===================== +// Model and Server Info +// ===================== + +message GetModelInfoRequest {} + +message GetModelInfoResponse { + string model_path = 1; + bool is_generation = 2; + int32 max_context_length = 3; + int32 vocab_size = 4; + bool supports_vision = 5; +} + +message GetServerInfoRequest {} + +message GetServerInfoResponse { + int32 active_requests = 1; + bool is_paused = 2; + double last_receive_timestamp = 3; + double uptime_seconds = 4; + string server_type = 5; // "vllm-grpc" +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/chat/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/chat/mod.rs new file mode 100644 index 00000000..61386b50 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/chat/mod.rs @@ -0,0 +1,7 @@ +//! Builders for Chat Completion API response types + +pub mod response; +pub mod stream_response; + +pub use response::ChatCompletionResponseBuilder; +pub use stream_response::ChatCompletionStreamResponseBuilder; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/chat/response.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/chat/response.rs new file mode 100644 index 00000000..7267e2ee --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/chat/response.rs @@ -0,0 +1,218 @@ +//! Builder for ChatCompletionResponse +//! +//! Provides an ergonomic fluent API for constructing chat completion responses. + +use crate::protocols::{chat::*, common::Usage}; + +/// Builder for ChatCompletionResponse +/// +/// Provides a fluent interface for constructing chat completion responses with sensible defaults. +#[must_use = "Builder does nothing until .build() is called"] +#[derive(Clone, Debug)] +pub struct ChatCompletionResponseBuilder { + id: String, + object: String, + created: u64, + model: String, + choices: Vec, + usage: Option, + system_fingerprint: Option, +} + +impl ChatCompletionResponseBuilder { + /// Create a new builder with required fields + /// + /// # Arguments + /// - `id`: Completion ID (e.g., "chatcmpl_abc123") + /// - `model`: Model name used for generation + pub fn new(id: impl Into, model: impl Into) -> Self { + Self { + id: id.into(), + object: "chat.completion".to_string(), + created: chrono::Utc::now().timestamp() as u64, + model: model.into(), + choices: Vec::new(), + usage: None, + system_fingerprint: None, + } + } + + /// Copy common fields from a ChatCompletionRequest + /// + /// This populates the model field from the request. + pub fn copy_from_request(mut self, request: &ChatCompletionRequest) -> Self { + self.model = request.model.clone(); + self + } + + /// Set the object type (default: "chat.completion") + pub fn object(mut self, object: impl Into) -> Self { + self.object = object.into(); + self + } + + /// Set the creation timestamp (default: current time) + pub fn created(mut self, timestamp: u64) -> Self { + self.created = timestamp; + self + } + + /// Set the choices + pub fn choices(mut self, choices: Vec) -> Self { + self.choices = choices; + self + } + + /// Add a single choice + pub fn add_choice(mut self, choice: ChatChoice) -> Self { + self.choices.push(choice); + self + } + + /// Set usage information + pub fn usage(mut self, usage: Usage) -> Self { + self.usage = Some(usage); + self + } + + /// Set usage if provided (handles Option) + pub fn maybe_usage(mut self, usage: Option) -> Self { + if let Some(u) = usage { + self.usage = Some(u); + } + self + } + + /// Set system fingerprint if provided (handles Option) + pub fn maybe_system_fingerprint(mut self, fingerprint: Option>) -> Self { + if let Some(fp) = fingerprint { + self.system_fingerprint = Some(fp.into()); + } + self + } + + /// Build the ChatCompletionResponse + pub fn build(self) -> ChatCompletionResponse { + ChatCompletionResponse { + id: self.id, + object: self.object, + created: self.created, + model: self.model, + choices: self.choices, + usage: self.usage, + system_fingerprint: self.system_fingerprint, + } + } +} + +// ============================================================================ +// Tests +// ============================================================================ + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_build_minimal() { + let response = ChatCompletionResponse::builder("chatcmpl_123", "gpt-4").build(); + + assert_eq!(response.id, "chatcmpl_123"); + assert_eq!(response.model, "gpt-4"); + assert_eq!(response.object, "chat.completion"); + assert!(response.choices.is_empty()); + assert!(response.usage.is_none()); + assert!(response.system_fingerprint.is_none()); + } + + #[test] + fn test_build_complete() { + let choice = ChatChoice { + index: 0, + message: ChatCompletionMessage { + role: "assistant".to_string(), + content: Some("Hello!".to_string()), + tool_calls: None, + reasoning_content: None, + }, + logprobs: None, + finish_reason: Some("stop".to_string()), + matched_stop: None, + hidden_states: None, + }; + + let usage = Usage { + prompt_tokens: 10, + completion_tokens: 20, + total_tokens: 30, + completion_tokens_details: None, + }; + + let response = ChatCompletionResponse::builder("chatcmpl_456", "gpt-4") + .choices(vec![choice.clone()]) + .maybe_usage(Some(usage)) + .maybe_system_fingerprint(Some("fp_123abc")) + .build(); + + assert_eq!(response.id, "chatcmpl_456"); + assert_eq!(response.choices.len(), 1); + assert_eq!(response.choices[0].index, 0); + assert!(response.usage.is_some()); + assert_eq!(response.system_fingerprint.as_ref().unwrap(), "fp_123abc"); + } + + #[test] + fn test_add_multiple_choices() { + let choice1 = ChatChoice { + index: 0, + message: ChatCompletionMessage { + role: "assistant".to_string(), + content: Some("Option 1".to_string()), + tool_calls: None, + reasoning_content: None, + }, + logprobs: None, + finish_reason: Some("stop".to_string()), + matched_stop: None, + hidden_states: None, + }; + + let choice2 = ChatChoice { + index: 1, + message: ChatCompletionMessage { + role: "assistant".to_string(), + content: Some("Option 2".to_string()), + tool_calls: None, + reasoning_content: None, + }, + logprobs: None, + finish_reason: Some("stop".to_string()), + matched_stop: None, + hidden_states: None, + }; + + let response = ChatCompletionResponse::builder("chatcmpl_789", "gpt-4") + .add_choice(choice1) + .add_choice(choice2) + .build(); + + assert_eq!(response.choices.len(), 2); + assert_eq!(response.choices[0].index, 0); + assert_eq!(response.choices[1].index, 1); + } + + #[test] + fn test_copy_from_request() { + let request = ChatCompletionRequest { + messages: vec![], + model: "gpt-3.5-turbo".to_string(), + ..Default::default() + }; + + let response = ChatCompletionResponse::builder("chatcmpl_101", "gpt-4") + .copy_from_request(&request) + .build(); + + assert_eq!(response.model, "gpt-3.5-turbo"); // Copied from request + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/chat/stream_response.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/chat/stream_response.rs new file mode 100644 index 00000000..9afe486a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/chat/stream_response.rs @@ -0,0 +1,421 @@ +//! Builder for ChatCompletionStreamResponse +//! +//! Provides an ergonomic fluent API for constructing streaming chat completion responses. + +use std::borrow::Cow; + +use crate::protocols::{ + chat::*, + common::{FunctionCallDelta, ToolCallDelta, Usage}, +}; + +/// Builder for ChatCompletionStreamResponse +/// +/// Provides a fluent interface for constructing streaming chat completion chunks with sensible defaults. +#[must_use = "Builder does nothing until .build() is called"] +#[derive(Clone, Debug)] +pub struct ChatCompletionStreamResponseBuilder { + id: String, + object: String, + created: u64, + model: String, + choices: Vec, + usage: Option, + system_fingerprint: Option, +} + +impl ChatCompletionStreamResponseBuilder { + /// Create a new builder with required fields + /// + /// # Arguments + /// - `id`: Completion ID (e.g., "chatcmpl_abc123") + /// - `model`: Model name used for generation + pub fn new(id: impl Into, model: impl Into) -> Self { + Self { + id: id.into(), + object: "chat.completion.chunk".to_string(), + created: chrono::Utc::now().timestamp() as u64, + model: model.into(), + choices: Vec::new(), + usage: None, + system_fingerprint: None, + } + } + + /// Copy common fields from a ChatCompletionRequest + /// + /// This populates the model field from the request. + pub fn copy_from_request(mut self, request: &ChatCompletionRequest) -> Self { + self.model = request.model.clone(); + self + } + + /// Set the object type (default: "chat.completion.chunk") + pub fn object(mut self, object: impl Into) -> Self { + self.object = object.into(); + self + } + + /// Set the creation timestamp (default: current time) + pub fn created(mut self, timestamp: u64) -> Self { + self.created = timestamp; + self + } + + /// Set the choices + pub fn choices(mut self, choices: Vec) -> Self { + self.choices = choices; + self + } + + /// Add a single choice (delta) + pub fn add_choice(mut self, choice: ChatStreamChoice) -> Self { + self.choices.push(choice); + self + } + + /// Set usage information (typically sent in final chunk) + pub fn usage(mut self, usage: Usage) -> Self { + self.usage = Some(usage); + self + } + + /// Set system fingerprint if provided (handles Option) + pub fn maybe_system_fingerprint(mut self, fingerprint: Option>) -> Self { + if let Some(fp) = fingerprint { + self.system_fingerprint = Some(fp.into()); + } + self + } + + /// Set usage if provided (handles Option) + pub fn maybe_usage(mut self, usage: Option) -> Self { + if let Some(u) = usage { + self.usage = Some(u); + } + self + } + + /// Add a choice delta that sets `role` and `content` + pub fn add_choice_content( + mut self, + index: u32, + role: impl Into, + content: impl Into, + ) -> Self { + self.choices.push(ChatStreamChoice { + index, + delta: ChatMessageDelta { + role: Some(role.into()), + content: Some(content.into()), + tool_calls: None, + reasoning_content: None, + }, + logprobs: None, + finish_reason: None, + matched_stop: None, + }); + self + } + + /// Add a choice delta that sets `role`, `content`, and `logprobs` + pub fn add_choice_content_with_logprobs( + mut self, + index: u32, + role: impl Into, + content: impl Into, + logprobs: Option, + ) -> Self { + self.choices.push(ChatStreamChoice { + index, + delta: ChatMessageDelta { + role: Some(role.into()), + content: Some(content.into()), + tool_calls: None, + reasoning_content: None, + }, + logprobs, + finish_reason: None, + matched_stop: None, + }); + self + } + + /// Add a choice delta that only sets `role` + pub fn add_choice_role(mut self, index: u32, role: impl Into) -> Self { + self.choices.push(ChatStreamChoice { + index, + delta: ChatMessageDelta { + role: Some(role.into()), + content: None, + tool_calls: None, + reasoning_content: None, + }, + logprobs: None, + finish_reason: None, + matched_stop: None, + }); + self + } + + /// Add a choice delta that appends a tool-call *arguments delta* + /// Uses `Cow` so you can pass `&str` or `String` without extra clones + pub fn add_choice_tool_args( + mut self, + index: u32, + args_delta: impl Into>, + ) -> Self { + self.choices.push(ChatStreamChoice { + index, + delta: ChatMessageDelta { + role: Some("assistant".to_string()), + content: None, + tool_calls: Some(vec![ToolCallDelta { + index: 0, + id: None, + tool_type: None, + function: Some(FunctionCallDelta { + name: None, + arguments: Some(args_delta.into().into_owned()), + }), + }]), + reasoning_content: None, + }, + logprobs: None, + finish_reason: None, + matched_stop: None, + }); + self + } + + /// Add a choice delta that sets reasoning content (for models that stream reasoning) + pub fn add_choice_reasoning(mut self, index: u32, reasoning: impl Into) -> Self { + self.choices.push(ChatStreamChoice { + index, + delta: ChatMessageDelta { + role: Some("assistant".to_string()), + content: None, + tool_calls: None, + reasoning_content: Some(reasoning.into()), + }, + logprobs: None, + finish_reason: None, + matched_stop: None, + }); + self + } + + /// Add a choice delta for tool call with function name and ID + pub fn add_choice_tool_name( + mut self, + index: u32, + tool_call_id: impl Into, + function_name: impl Into, + ) -> Self { + self.choices.push(ChatStreamChoice { + index, + delta: ChatMessageDelta { + role: Some("assistant".to_string()), + content: None, + tool_calls: Some(vec![ToolCallDelta { + index: 0, + id: Some(tool_call_id.into()), + tool_type: Some("function".to_string()), + function: Some(FunctionCallDelta { + name: Some(function_name.into()), + arguments: None, + }), + }]), + reasoning_content: None, + }, + logprobs: None, + finish_reason: None, + matched_stop: None, + }); + self + } + + /// Add a choice delta with a pre-constructed ToolCallDelta + /// Useful when you already have a ToolCallDelta object to emit + pub fn add_choice_tool_call_delta( + mut self, + index: u32, + tool_call_delta: ToolCallDelta, + ) -> Self { + self.choices.push(ChatStreamChoice { + index, + delta: ChatMessageDelta { + role: Some("assistant".to_string()), + content: None, + tool_calls: Some(vec![tool_call_delta]), + reasoning_content: None, + }, + logprobs: None, + finish_reason: None, + matched_stop: None, + }); + self + } + + /// Add a choice with finish_reason (final chunk) + /// This is used for the last chunk in a stream to signal completion + pub fn add_choice_finish_reason( + mut self, + index: u32, + finish_reason: impl Into, + matched_stop: Option, + ) -> Self { + self.choices.push(ChatStreamChoice { + index, + delta: ChatMessageDelta { + role: None, + content: None, + tool_calls: None, + reasoning_content: None, + }, + logprobs: None, + finish_reason: Some(finish_reason.into()), + matched_stop, + }); + self + } + + /// Build the ChatCompletionStreamResponse + pub fn build(self) -> ChatCompletionStreamResponse { + ChatCompletionStreamResponse { + id: self.id, + object: self.object, + created: self.created, + model: self.model, + system_fingerprint: self.system_fingerprint, + choices: self.choices, + usage: self.usage, + } + } +} + +// ============================================================================ +// Tests +// ============================================================================ + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_build_minimal() { + let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_123", "gpt-4").build(); + + assert_eq!(chunk.id, "chatcmpl_123"); + assert_eq!(chunk.model, "gpt-4"); + assert_eq!(chunk.object, "chat.completion.chunk"); + assert!(chunk.choices.is_empty()); + assert!(chunk.usage.is_none()); + } + + #[test] + fn test_with_content_delta() { + let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_456", "gpt-4") + .add_choice_content(0, "assistant", "Hello") + .build(); + + assert_eq!(chunk.choices.len(), 1); + assert_eq!(chunk.choices[0].index, 0); + assert_eq!(chunk.choices[0].delta.content.as_ref().unwrap(), "Hello"); + assert_eq!(chunk.choices[0].delta.role.as_ref().unwrap(), "assistant"); + assert!(chunk.choices[0].finish_reason.is_none()); + } + + #[test] + fn test_with_role_delta() { + let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_789", "gpt-4") + .add_choice_role(0, "assistant") + .build(); + + assert_eq!(chunk.choices.len(), 1); + assert_eq!(chunk.choices[0].delta.role.as_ref().unwrap(), "assistant"); + assert!(chunk.choices[0].delta.content.is_none()); + } + + #[test] + fn test_with_finish_reason() { + let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_101", "gpt-4") + .add_choice_finish_reason(0, "stop", None) + .build(); + + assert_eq!(chunk.choices.len(), 1); + assert_eq!(chunk.choices[0].finish_reason.as_ref().unwrap(), "stop"); + assert!(chunk.choices[0].delta.content.is_none()); + assert!(chunk.choices[0].delta.role.is_none()); + } + + #[test] + fn test_multiple_deltas() { + let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_202", "gpt-4") + .add_choice_role(0, "assistant") + .add_choice_content(0, "assistant", "Hello") + .add_choice_content(0, "assistant", " world") + .add_choice_finish_reason(0, "stop", None) + .build(); + + assert_eq!(chunk.choices.len(), 4); // role + 2 content + finish + } + + #[test] + fn test_with_usage() { + let usage = Usage { + prompt_tokens: 10, + completion_tokens: 20, + total_tokens: 30, + completion_tokens_details: None, + }; + + let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_303", "gpt-4") + .add_choice_finish_reason(0, "stop", None) + .usage(usage) + .build(); + + assert!(chunk.usage.is_some()); + assert_eq!(chunk.usage.as_ref().unwrap().total_tokens, 30); + } + + #[test] + fn test_copy_from_request() { + let request = ChatCompletionRequest { + messages: vec![], + model: "gpt-3.5-turbo".to_string(), + ..Default::default() + }; + + let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_404", "gpt-4") + .copy_from_request(&request) + .add_choice_content(0, "assistant", "test") + .build(); + + assert_eq!(chunk.model, "gpt-3.5-turbo"); // Copied from request + } + + #[test] + fn test_add_choice_explicit() { + let choice = ChatStreamChoice { + index: 0, + delta: ChatMessageDelta { + role: Some("assistant".to_string()), + content: Some("Hello".to_string()), + tool_calls: None, + reasoning_content: None, + }, + logprobs: None, + finish_reason: None, + matched_stop: None, + }; + + let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_505", "gpt-4") + .add_choice(choice) + .build(); + + assert_eq!(chunk.choices.len(), 1); + assert_eq!(chunk.choices[0].delta.role.as_ref().unwrap(), "assistant"); + assert_eq!(chunk.choices[0].delta.content.as_ref().unwrap(), "Hello"); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/mod.rs new file mode 100644 index 00000000..92084412 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/mod.rs @@ -0,0 +1,27 @@ +//! Builder patterns for protocol response types +//! +//! This module provides ergonomic builders for response types with many optional fields. +//! Builders help avoid telescoping constructors and make construction intent clear. +//! +//! # Organization +//! +//! Builders are organized by API: +//! - `chat/` - Chat Completion API builders (response, stream_response) +//! - `responses/` - Responses API builder (response) +//! +//! # Optional Fields +//! +//! For optional fields, builders provide `maybe_*` methods that handle `Option` directly: +//! ```ignore +//! builder +//! .field(value) +//! .maybe_optional_field(optional_value) // Accepts Option +//! .build() +//! ``` + +pub mod chat; +pub mod responses; + +// Re-export all builders for convenient access +pub use chat::{ChatCompletionResponseBuilder, ChatCompletionStreamResponseBuilder}; +pub use responses::ResponsesResponseBuilder; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/responses/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/responses/mod.rs new file mode 100644 index 00000000..aca9d249 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/responses/mod.rs @@ -0,0 +1,5 @@ +//! Builders for Responses API response types + +pub mod response; + +pub use response::ResponsesResponseBuilder; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/responses/response.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/responses/response.rs new file mode 100644 index 00000000..50d3f1bc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/builders/responses/response.rs @@ -0,0 +1,411 @@ +//! Builder for ResponsesResponse +//! +//! Provides an ergonomic fluent API for constructing ResponsesResponse instances. + +use std::collections::HashMap; + +use serde_json::Value; + +use crate::protocols::responses::*; + +/// Builder for ResponsesResponse +/// +/// Provides a fluent interface for constructing responses with sensible defaults. +#[must_use = "Builder does nothing until .build() is called"] +#[derive(Clone, Debug)] +pub struct ResponsesResponseBuilder { + id: String, + object: String, + created_at: i64, + status: ResponseStatus, + error: Option, + incomplete_details: Option, + instructions: Option, + max_output_tokens: Option, + model: String, + output: Vec, + parallel_tool_calls: bool, + previous_response_id: Option, + reasoning: Option, + store: bool, + temperature: Option, + text: Option, + tool_choice: String, + tools: Vec, + top_p: Option, + truncation: Option, + usage: Option, + user: Option, + safety_identifier: Option, + metadata: HashMap, +} + +impl ResponsesResponseBuilder { + /// Create a new builder with required fields + /// + /// # Arguments + /// - `id`: Response ID (e.g., "resp_abc123") + /// - `model`: Model name used for generation + pub fn new(id: impl Into, model: impl Into) -> Self { + Self { + id: id.into(), + object: "response".to_string(), + created_at: chrono::Utc::now().timestamp(), + status: ResponseStatus::InProgress, + error: None, + incomplete_details: None, + instructions: None, + max_output_tokens: None, + model: model.into(), + output: Vec::new(), + parallel_tool_calls: true, + previous_response_id: None, + reasoning: None, + store: true, + temperature: None, + text: None, + tool_choice: "auto".to_string(), + tools: Vec::new(), + top_p: None, + truncation: None, + usage: None, + user: None, + safety_identifier: None, + metadata: HashMap::new(), + } + } + + /// Copy common fields from a ResponsesRequest + /// + /// This populates fields like instructions, max_output_tokens, temperature, etc. + /// from the original request, making it easy to construct a response that mirrors + /// the request parameters. + /// + /// Note: `safety_identifier` is intentionally NOT copied as it is for content moderation + /// and should be set independently from the request's `user` field (which is for billing/tracking). + pub fn copy_from_request(mut self, request: &ResponsesRequest) -> Self { + self.instructions = request.instructions.clone(); + self.max_output_tokens = request.max_output_tokens; + self.parallel_tool_calls = request.parallel_tool_calls.unwrap_or(true); + self.previous_response_id = request.previous_response_id.clone(); + self.store = request.store.unwrap_or(true); + self.temperature = request.temperature; + self.tool_choice = if let Some(ref tc) = request.tool_choice { + serde_json::to_string(tc).unwrap_or_else(|_| "auto".to_string()) + } else { + "auto".to_string() + }; + self.tools = request.tools.clone().unwrap_or_default(); + self.top_p = request.top_p; + self.user = request.user.clone(); + self.metadata = request.metadata.clone().unwrap_or_default(); + self + } + + /// Set the object type (default: "response") + pub fn object(mut self, object: impl Into) -> Self { + self.object = object.into(); + self + } + + /// Set the creation timestamp (default: current time) + pub fn created_at(mut self, timestamp: i64) -> Self { + self.created_at = timestamp; + self + } + + /// Set the response status + pub fn status(mut self, status: ResponseStatus) -> Self { + self.status = status; + self + } + + /// Set error information (if status is failed) + pub fn error(mut self, error: Value) -> Self { + self.error = Some(error); + self + } + + /// Set incomplete details (if response was truncated) + pub fn incomplete_details(mut self, details: Value) -> Self { + self.incomplete_details = Some(details); + self + } + + /// Set system instructions + pub fn instructions(mut self, instructions: impl Into) -> Self { + self.instructions = Some(instructions.into()); + self + } + + /// Set max output tokens + pub fn max_output_tokens(mut self, tokens: u32) -> Self { + self.max_output_tokens = Some(tokens); + self + } + + /// Set output items + pub fn output(mut self, output: Vec) -> Self { + self.output = output; + self + } + + /// Add a single output item + pub fn add_output(mut self, item: ResponseOutputItem) -> Self { + self.output.push(item); + self + } + + /// Set whether parallel tool calls are enabled + pub fn parallel_tool_calls(mut self, enabled: bool) -> Self { + self.parallel_tool_calls = enabled; + self + } + + /// Set previous response ID (if continuation) + pub fn previous_response_id(mut self, id: impl Into) -> Self { + self.previous_response_id = Some(id.into()); + self + } + + /// Set reasoning information + pub fn reasoning(mut self, reasoning: ReasoningInfo) -> Self { + self.reasoning = Some(reasoning); + self + } + + /// Set whether the response is stored + pub fn store(mut self, store: bool) -> Self { + self.store = store; + self + } + + /// Set temperature setting + pub fn temperature(mut self, temperature: f32) -> Self { + self.temperature = Some(temperature); + self + } + + /// Set text format settings if provided (handles Option) + pub fn maybe_text(mut self, text: Option) -> Self { + if let Some(t) = text { + self.text = Some(t); + } + self + } + + /// Set tool choice setting + pub fn tool_choice(mut self, tool_choice: impl Into) -> Self { + self.tool_choice = tool_choice.into(); + self + } + + /// Set available tools + pub fn tools(mut self, tools: Vec) -> Self { + self.tools = tools; + self + } + + /// Set top-p setting + pub fn top_p(mut self, top_p: f32) -> Self { + self.top_p = Some(top_p); + self + } + + /// Set truncation strategy + pub fn truncation(mut self, truncation: impl Into) -> Self { + self.truncation = Some(truncation.into()); + self + } + + /// Set usage information + pub fn usage(mut self, usage: ResponsesUsage) -> Self { + self.usage = Some(usage); + self + } + + /// Set usage if provided (handles Option) + pub fn maybe_usage(mut self, usage: Option) -> Self { + if let Some(u) = usage { + self.usage = Some(u); + } + self + } + + /// Copy from request if provided (handles Option) + pub fn maybe_copy_from_request(mut self, request: Option<&ResponsesRequest>) -> Self { + if let Some(req) = request { + self = self.copy_from_request(req); + } + self + } + + /// Set user identifier + pub fn user(mut self, user: impl Into) -> Self { + self.user = Some(user.into()); + self + } + + /// Set safety identifier + pub fn safety_identifier(mut self, identifier: impl Into) -> Self { + self.safety_identifier = Some(identifier.into()); + self + } + + /// Set metadata + pub fn metadata(mut self, metadata: HashMap) -> Self { + self.metadata = metadata; + self + } + + /// Add a single metadata entry + pub fn add_metadata(mut self, key: impl Into, value: Value) -> Self { + self.metadata.insert(key.into(), value); + self + } + + /// Build the ResponsesResponse + pub fn build(self) -> ResponsesResponse { + ResponsesResponse { + id: self.id, + object: self.object, + created_at: self.created_at, + status: self.status, + error: self.error, + incomplete_details: self.incomplete_details, + instructions: self.instructions, + max_output_tokens: self.max_output_tokens, + model: self.model, + output: self.output, + parallel_tool_calls: self.parallel_tool_calls, + previous_response_id: self.previous_response_id, + reasoning: self.reasoning, + store: self.store, + temperature: self.temperature, + text: self.text, + tool_choice: self.tool_choice, + tools: self.tools, + top_p: self.top_p, + truncation: self.truncation, + usage: self.usage, + user: self.user, + safety_identifier: self.safety_identifier, + metadata: self.metadata, + } + } +} + +// ============================================================================ +// Tests +// ============================================================================ + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_build_minimal() { + let response = ResponsesResponse::builder("resp_123", "gpt-4").build(); + + assert_eq!(response.id, "resp_123"); + assert_eq!(response.model, "gpt-4"); + assert_eq!(response.object, "response"); + assert_eq!(response.status, ResponseStatus::InProgress); + assert!(response.output.is_empty()); + assert!(response.parallel_tool_calls); + assert!(response.store); + } + + #[test] + fn test_build_complete() { + let response = ResponsesResponse::builder("resp_123", "gpt-4") + .status(ResponseStatus::Completed) + .instructions("You are a helpful assistant") + .max_output_tokens(1000) + .temperature(0.7) + .top_p(0.9) + .parallel_tool_calls(false) + .store(false) + .build(); + + assert_eq!(response.status, ResponseStatus::Completed); + assert_eq!( + response.instructions.as_ref().unwrap(), + "You are a helpful assistant" + ); + assert_eq!(response.max_output_tokens, Some(1000)); + assert_eq!(response.temperature, Some(0.7)); + assert_eq!(response.top_p, Some(0.9)); + assert!(!response.parallel_tool_calls); + assert!(!response.store); + } + + #[test] + fn test_copy_from_request() { + let request = ResponsesRequest { + model: "gpt-4".to_string(), + input: ResponseInput::Text("test".to_string()), + instructions: Some("Be helpful".to_string()), + max_output_tokens: Some(500), + temperature: Some(0.8), + top_p: Some(0.95), + parallel_tool_calls: Some(false), + store: Some(false), + user: Some("user_123".to_string()), + metadata: Some(HashMap::from([( + "key".to_string(), + serde_json::json!("value"), + )])), + ..Default::default() + }; + + let response = ResponsesResponse::builder("resp_456", "gpt-4") + .copy_from_request(&request) + .status(ResponseStatus::Completed) + .build(); + + assert_eq!(response.instructions.as_ref().unwrap(), "Be helpful"); + assert_eq!(response.max_output_tokens, Some(500)); + assert_eq!(response.temperature, Some(0.8)); + assert_eq!(response.top_p, Some(0.95)); + assert!(!response.parallel_tool_calls); + assert!(!response.store); + assert_eq!(response.user.as_ref().unwrap(), "user_123"); + assert_eq!( + response.metadata.get("key").unwrap(), + &serde_json::json!("value") + ); + } + + #[test] + fn test_add_output_items() { + let response = ResponsesResponse::builder("resp_789", "gpt-4") + .add_output(ResponseOutputItem::Message { + id: "msg_1".to_string(), + role: "assistant".to_string(), + content: vec![], + status: "completed".to_string(), + }) + .add_output(ResponseOutputItem::Message { + id: "msg_2".to_string(), + role: "assistant".to_string(), + content: vec![], + status: "completed".to_string(), + }) + .build(); + + assert_eq!(response.output.len(), 2); + } + + #[test] + fn test_add_metadata() { + let response = ResponsesResponse::builder("resp_101", "gpt-4") + .add_metadata("key1", serde_json::json!("value1")) + .add_metadata("key2", serde_json::json!(42)) + .build(); + + assert_eq!(response.metadata.len(), 2); + assert_eq!(response.metadata.get("key1").unwrap(), "value1"); + assert_eq!(response.metadata.get("key2").unwrap(), 42); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/chat.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/chat.rs new file mode 100644 index 00000000..6c3f2699 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/chat.rs @@ -0,0 +1,712 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use validator::Validate; + +use super::{ + common::{ + default_model, default_true, validate_stop, ChatLogProbs, ContentPart, Function, + FunctionCall, FunctionChoice, GenerationRequest, ResponseFormat, StreamOptions, + StringOrArray, Tool, ToolCall, ToolCallDelta, ToolChoice, ToolChoiceValue, ToolReference, + Usage, + }, + sampling_params::{validate_top_k_value, validate_top_p_value}, +}; +use crate::protocols::{ + builders::{ChatCompletionResponseBuilder, ChatCompletionStreamResponseBuilder}, + validated::Normalizable, +}; + +// ============================================================================ +// Chat Messages +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "role")] +pub enum ChatMessage { + #[serde(rename = "system")] + System { + content: MessageContent, + #[serde(skip_serializing_if = "Option::is_none")] + name: Option, + }, + #[serde(rename = "user")] + User { + content: MessageContent, + #[serde(skip_serializing_if = "Option::is_none")] + name: Option, + }, + #[serde(rename = "assistant")] + Assistant { + #[serde(skip_serializing_if = "Option::is_none")] + content: Option, + #[serde(skip_serializing_if = "Option::is_none")] + name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + tool_calls: Option>, + /// Reasoning content for O1-style models (SGLang extension) + #[serde(skip_serializing_if = "Option::is_none")] + reasoning_content: Option, + }, + #[serde(rename = "tool")] + Tool { + content: MessageContent, + tool_call_id: String, + }, + #[serde(rename = "function")] + Function { content: String, name: String }, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +#[serde(untagged)] +pub enum MessageContent { + Text(String), + Parts(Vec), +} + +impl MessageContent { + pub fn to_simple_string(&self) -> String { + match self { + MessageContent::Text(text) => text.clone(), + MessageContent::Parts(parts) => { + let texts: Vec = parts + .iter() + .filter_map(|part| match part { + ContentPart::Text { text } => Some(text.clone()), + _ => None, + }) + .collect(); + texts.join(" ") + } + } + } +} + +// ============================================================================ +// Chat Completion Request +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize, Default, Validate)] +#[validate(schema(function = "validate_chat_cross_parameters"))] +pub struct ChatCompletionRequest { + /// A list of messages comprising the conversation so far + #[validate(custom(function = "validate_messages"))] + pub messages: Vec, + + /// ID of the model to use + #[serde(default = "default_model")] + pub model: String, + + /// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = -2.0, max = 2.0))] + pub frequency_penalty: Option, + + /// Deprecated: Replaced by tool_choice + #[serde(skip_serializing_if = "Option::is_none")] + #[deprecated(note = "Use tool_choice instead")] + pub function_call: Option, + + /// Deprecated: Replaced by tools + #[serde(skip_serializing_if = "Option::is_none")] + #[deprecated(note = "Use tools instead")] + pub functions: Option>, + + /// Modify the likelihood of specified tokens appearing in the completion + #[serde(skip_serializing_if = "Option::is_none")] + pub logit_bias: Option>, + + /// Whether to return log probabilities of the output tokens + #[serde(default)] + pub logprobs: bool, + + /// Deprecated: Replaced by max_completion_tokens + #[serde(skip_serializing_if = "Option::is_none")] + #[deprecated(note = "Use max_completion_tokens instead")] + #[validate(range(min = 1))] + pub max_tokens: Option, + + /// An upper bound for the number of tokens that can be generated for a completion + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 1))] + pub max_completion_tokens: Option, + + /// Developer-defined tags and values used for filtering completions in the dashboard + #[serde(skip_serializing_if = "Option::is_none")] + pub metadata: Option>, + + /// Output types that you would like the model to generate for this request + #[serde(skip_serializing_if = "Option::is_none")] + pub modalities: Option>, + + /// How many chat completion choices to generate for each input message + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 1, max = 10))] + pub n: Option, + + /// Whether to enable parallel function calling during tool use + #[serde(skip_serializing_if = "Option::is_none")] + pub parallel_tool_calls: Option, + + /// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = -2.0, max = 2.0))] + pub presence_penalty: Option, + + /// Cache key for prompts (beta feature) + #[serde(skip_serializing_if = "Option::is_none")] + pub prompt_cache_key: Option, + + /// Effort level for reasoning models (low, medium, high) + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning_effort: Option, + + /// An object specifying the format that the model must output + #[serde(skip_serializing_if = "Option::is_none")] + pub response_format: Option, + + /// Safety identifier for content moderation + #[serde(skip_serializing_if = "Option::is_none")] + pub safety_identifier: Option, + + /// Deprecated: This feature is in Legacy mode + #[serde(skip_serializing_if = "Option::is_none")] + #[deprecated(note = "This feature is in Legacy mode")] + pub seed: Option, + + /// The service tier to use for this request + #[serde(skip_serializing_if = "Option::is_none")] + pub service_tier: Option, + + /// Up to 4 sequences where the API will stop generating further tokens + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(custom(function = "validate_stop"))] + pub stop: Option, + + /// If set, partial message deltas will be sent + #[serde(default)] + pub stream: bool, + + /// Options for streaming response + #[serde(skip_serializing_if = "Option::is_none")] + pub stream_options: Option, + + /// What sampling temperature to use, between 0 and 2 + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 0.0, max = 2.0))] + pub temperature: Option, + + /// Controls which (if any) tool is called by the model + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_choice: Option, + + /// A list of tools the model may call + #[serde(skip_serializing_if = "Option::is_none")] + pub tools: Option>, + + /// An integer between 0 and 20 specifying the number of most likely tokens to return + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 0, max = 20))] + pub top_logprobs: Option, + + /// An alternative to sampling with temperature + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(custom(function = "validate_top_p_value"))] + pub top_p: Option, + + /// Verbosity level for debugging + #[serde(skip_serializing_if = "Option::is_none")] + pub verbosity: Option, + + // ============================================================================= + // Engine-Specific Sampling Parameters + // ============================================================================= + // These parameters are extensions beyond the OpenAI API specification and + // control model generation behavior in engine-specific ways. + // ============================================================================= + /// Top-k sampling parameter (-1 to disable) + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(custom(function = "validate_top_k_value"))] + pub top_k: Option, + + /// Min-p nucleus sampling parameter + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 0.0, max = 1.0))] + pub min_p: Option, + + /// Minimum number of tokens to generate + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 1))] + pub min_tokens: Option, + + /// Repetition penalty for reducing repetitive text + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 0.0, max = 2.0))] + pub repetition_penalty: Option, + + /// Regex constraint for output generation + #[serde(skip_serializing_if = "Option::is_none")] + pub regex: Option, + + /// EBNF grammar constraint for structured output + #[serde(skip_serializing_if = "Option::is_none")] + pub ebnf: Option, + + /// Specific token IDs to use as stop conditions + #[serde(skip_serializing_if = "Option::is_none")] + pub stop_token_ids: Option>, + + /// Skip trimming stop tokens from output + #[serde(default)] + pub no_stop_trim: bool, + + /// Ignore end-of-sequence tokens during generation + #[serde(default)] + pub ignore_eos: bool, + + /// Continue generating from final assistant message + #[serde(default)] + pub continue_final_message: bool, + + /// Skip special tokens during detokenization + #[serde(default = "default_true")] + pub skip_special_tokens: bool, + + /// Path to LoRA adapter(s) for model customization + #[serde(skip_serializing_if = "Option::is_none")] + pub lora_path: Option, + + /// Session parameters for continual prompting + #[serde(skip_serializing_if = "Option::is_none")] + pub session_params: Option>, + + /// Separate reasoning content from final answer (O1-style models) + #[serde(default = "default_true")] + pub separate_reasoning: bool, + + /// Stream reasoning tokens during generation + #[serde(default = "default_true")] + pub stream_reasoning: bool, + + /// Chat template kwargs + #[serde(skip_serializing_if = "Option::is_none")] + pub chat_template_kwargs: Option>, + + /// Return model hidden states + #[serde(default)] + pub return_hidden_states: bool, + + /// Random seed for sampling for deterministic outputs + #[serde(skip_serializing_if = "Option::is_none")] + pub sampling_seed: Option, +} + +// ============================================================================ +// Validation Functions +// ============================================================================ + +/// Validates messages array is not empty and has valid content +fn validate_messages(messages: &[ChatMessage]) -> Result<(), validator::ValidationError> { + if messages.is_empty() { + return Err(validator::ValidationError::new("messages cannot be empty")); + } + + for msg in messages.iter() { + if let ChatMessage::User { content, .. } = msg { + match content { + MessageContent::Text(text) if text.is_empty() => { + return Err(validator::ValidationError::new( + "message content cannot be empty", + )); + } + MessageContent::Parts(parts) if parts.is_empty() => { + return Err(validator::ValidationError::new( + "message content parts cannot be empty", + )); + } + _ => {} + } + } + } + Ok(()) +} + +/// Schema-level validation for cross-field dependencies +fn validate_chat_cross_parameters( + req: &ChatCompletionRequest, +) -> Result<(), validator::ValidationError> { + // 1. Validate logprobs dependency + if req.top_logprobs.is_some() && !req.logprobs { + let mut e = validator::ValidationError::new("top_logprobs_requires_logprobs"); + e.message = Some("top_logprobs is only allowed when logprobs is enabled".into()); + return Err(e); + } + + // 2. Validate stream_options dependency + if req.stream_options.is_some() && !req.stream { + let mut e = validator::ValidationError::new("stream_options_requires_stream"); + e.message = + Some("The 'stream_options' parameter is only allowed when 'stream' is enabled".into()); + return Err(e); + } + + // 3. Validate token limits - min <= max + if let (Some(min), Some(max)) = (req.min_tokens, req.max_completion_tokens) { + if min > max { + let mut e = validator::ValidationError::new("min_tokens_exceeds_max"); + e.message = Some("min_tokens cannot exceed max_tokens/max_completion_tokens".into()); + return Err(e); + } + } + + // 4. Validate structured output conflicts + let has_json_format = matches!( + req.response_format, + Some(ResponseFormat::JsonObject | ResponseFormat::JsonSchema { .. }) + ); + + if has_json_format && req.regex.is_some() { + let mut e = validator::ValidationError::new("regex_conflicts_with_json"); + e.message = Some("cannot use regex constraint with JSON response format".into()); + return Err(e); + } + + if has_json_format && req.ebnf.is_some() { + let mut e = validator::ValidationError::new("ebnf_conflicts_with_json"); + e.message = Some("cannot use EBNF constraint with JSON response format".into()); + return Err(e); + } + + // 5. Validate mutually exclusive structured output constraints + let constraint_count = [ + req.regex.is_some(), + req.ebnf.is_some(), + matches!(req.response_format, Some(ResponseFormat::JsonSchema { .. })), + ] + .iter() + .filter(|&&x| x) + .count(); + + if constraint_count > 1 { + let mut e = validator::ValidationError::new("multiple_constraints"); + e.message = Some("only one structured output constraint (regex, ebnf, or json_schema) can be active at a time".into()); + return Err(e); + } + + // 6. Validate response format JSON schema name + if let Some(ResponseFormat::JsonSchema { json_schema }) = &req.response_format { + if json_schema.name.is_empty() { + let mut e = validator::ValidationError::new("json_schema_name_empty"); + e.message = Some("JSON schema name cannot be empty".into()); + return Err(e); + } + } + + // 7. Validate tool_choice requires tools (except for "none") + if let Some(ref tool_choice) = req.tool_choice { + let has_tools = req.tools.as_ref().is_some_and(|t| !t.is_empty()); + + // Check if tool_choice is anything other than "none" + let is_some_choice = !matches!(tool_choice, ToolChoice::Value(ToolChoiceValue::None)); + + if is_some_choice && !has_tools { + let mut e = validator::ValidationError::new("tool_choice_requires_tools"); + e.message = Some("Invalid value for 'tool_choice': 'tool_choice' is only allowed when 'tools' are specified.".into()); + return Err(e); + } + + // Additional validation when tools are present + if has_tools { + let tools = req.tools.as_ref().unwrap(); + + match tool_choice { + ToolChoice::Function { function, .. } => { + // Validate that the specified function name exists in tools + let function_exists = tools.iter().any(|tool| { + tool.tool_type == "function" && tool.function.name == function.name + }); + + if !function_exists { + let mut e = + validator::ValidationError::new("tool_choice_function_not_found"); + e.message = Some( + format!( + "Invalid value for 'tool_choice': function '{}' not found in 'tools'.", + function.name + ) + .into(), + ); + return Err(e); + } + } + ToolChoice::AllowedTools { + mode, + tools: allowed_tools, + .. + } => { + // Validate mode is "auto" or "required" + if mode != "auto" && mode != "required" { + let mut e = validator::ValidationError::new("tool_choice_invalid_mode"); + e.message = Some(format!( + "Invalid value for 'tool_choice.mode': must be 'auto' or 'required', got '{}'.", + mode + ).into()); + return Err(e); + } + + // Validate that all ToolReferences are Function type (Chat API only supports function tools) + for tool_ref in allowed_tools { + match tool_ref { + ToolReference::Function { name } => { + // Validate that the function exists in tools array + let tool_exists = tools.iter().any(|tool| { + tool.tool_type == "function" && tool.function.name == *name + }); + + if !tool_exists { + let mut e = validator::ValidationError::new( + "tool_choice_tool_not_found", + ); + e.message = Some( + format!( + "Invalid value for 'tool_choice.tools': tool '{}' not found in 'tools'.", + name + ) + .into(), + ); + return Err(e); + } + } + _ => { + // Chat Completion API only supports function tools in tool_choice + let mut e = validator::ValidationError::new( + "tool_choice_invalid_tool_type", + ); + e.message = Some( + format!( + "Invalid value for 'tool_choice.tools': Chat Completion API only supports function tools, got '{}'.", + tool_ref.identifier() + ) + .into(), + ); + return Err(e); + } + } + } + } + _ => {} + } + } + } + + Ok(()) +} + +// ============================================================================ +// Normalizable Implementation +// ============================================================================ + +impl Normalizable for ChatCompletionRequest { + /// Normalize the request by applying migrations and defaults: + /// 1. Migrate deprecated fields to their replacements + /// 2. Clear deprecated fields and log warnings + /// 3. Apply OpenAI defaults for tool_choice + fn normalize(&mut self) { + // Migrate deprecated max_tokens → max_completion_tokens + #[allow(deprecated)] + if self.max_completion_tokens.is_none() && self.max_tokens.is_some() { + self.max_completion_tokens = self.max_tokens; + self.max_tokens = None; // Clear deprecated field + } + + // Migrate deprecated functions → tools + #[allow(deprecated)] + if self.tools.is_none() && self.functions.is_some() { + tracing::warn!("functions is deprecated, use tools instead"); + self.tools = self.functions.as_ref().map(|functions| { + functions + .iter() + .map(|func| Tool { + tool_type: "function".to_string(), + function: func.clone(), + }) + .collect() + }); + self.functions = None; // Clear deprecated field + } + + // Migrate deprecated function_call → tool_choice + #[allow(deprecated)] + if self.tool_choice.is_none() && self.function_call.is_some() { + tracing::warn!("function_call is deprecated, use tool_choice instead"); + self.tool_choice = self.function_call.as_ref().map(|fc| match fc { + FunctionCall::None => ToolChoice::Value(ToolChoiceValue::None), + FunctionCall::Auto => ToolChoice::Value(ToolChoiceValue::Auto), + FunctionCall::Function { name } => ToolChoice::Function { + tool_type: "function".to_string(), + function: FunctionChoice { name: name.clone() }, + }, + }); + self.function_call = None; // Clear deprecated field + } + + // Apply tool_choice defaults + if self.tool_choice.is_none() { + if let Some(tools) = &self.tools { + let choice_value = if !tools.is_empty() { + ToolChoiceValue::Auto + } else { + ToolChoiceValue::None + }; + self.tool_choice = Some(ToolChoice::Value(choice_value)); + } + // If tools is None, leave tool_choice as None (don't set it) + } + } +} + +// ============================================================================ +// GenerationRequest Trait Implementation +// ============================================================================ + +impl GenerationRequest for ChatCompletionRequest { + fn is_stream(&self) -> bool { + self.stream + } + + fn get_model(&self) -> Option<&str> { + Some(&self.model) + } + + fn extract_text_for_routing(&self) -> String { + // Extract text from messages for routing decisions + self.messages + .iter() + .filter_map(|msg| match msg { + ChatMessage::System { content, .. } => Some(content.to_simple_string()), + ChatMessage::User { content, .. } => Some(content.to_simple_string()), + ChatMessage::Assistant { + content, + reasoning_content, + .. + } => { + // Combine content and reasoning content for routing decisions + let main_content = content + .as_ref() + .map(|c| c.to_simple_string()) + .unwrap_or_default(); + let reasoning = reasoning_content.clone().unwrap_or_default(); + if main_content.is_empty() && reasoning.is_empty() { + None + } else { + Some(format!("{} {}", main_content, reasoning).trim().to_string()) + } + } + ChatMessage::Tool { content, .. } => Some(content.to_simple_string()), + ChatMessage::Function { content, .. } => Some(content.clone()), + }) + .collect::>() + .join(" ") + } +} + +// ============================================================================ +// Response Types +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ChatCompletionResponse { + pub id: String, + pub object: String, // "chat.completion" + pub created: u64, + pub model: String, + pub choices: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub usage: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub system_fingerprint: Option, +} + +impl ChatCompletionResponse { + /// Create a new builder for ChatCompletionResponse + pub fn builder( + id: impl Into, + model: impl Into, + ) -> ChatCompletionResponseBuilder { + ChatCompletionResponseBuilder::new(id, model) + } +} + +/// Response message structure for ChatCompletionResponse (different from request ChatMessage) +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ChatCompletionMessage { + pub role: String, // Always "assistant" for responses + #[serde(skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_calls: Option>, + pub reasoning_content: Option, + // Note: function_call is deprecated and not included + // Note: refusal, annotations, audio are not added yet +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ChatChoice { + pub index: u32, + pub message: ChatCompletionMessage, + #[serde(skip_serializing_if = "Option::is_none")] + pub logprobs: Option, + pub finish_reason: Option, // "stop", "length", "tool_calls", "content_filter", "function_call" + /// Information about which stop condition was matched + #[serde(skip_serializing_if = "Option::is_none")] + pub matched_stop: Option, // Can be string or integer + /// Hidden states from the model (SGLang extension) + #[serde(skip_serializing_if = "Option::is_none")] + pub hidden_states: Option>, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ChatCompletionStreamResponse { + pub id: String, + pub object: String, // "chat.completion.chunk" + pub created: u64, + pub model: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub system_fingerprint: Option, + pub choices: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub usage: Option, +} + +impl ChatCompletionStreamResponse { + /// Create a new builder for ChatCompletionStreamResponse + pub fn builder( + id: impl Into, + model: impl Into, + ) -> ChatCompletionStreamResponseBuilder { + ChatCompletionStreamResponseBuilder::new(id, model) + } +} + +/// Delta structure for streaming chat completion responses +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ChatMessageDelta { + #[serde(skip_serializing_if = "Option::is_none")] + pub role: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_calls: Option>, + pub reasoning_content: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ChatStreamChoice { + pub index: u32, + pub delta: ChatMessageDelta, + pub logprobs: Option, + pub finish_reason: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub matched_stop: Option, +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/classify.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/classify.rs new file mode 100644 index 00000000..fc7e8b87 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/classify.rs @@ -0,0 +1,57 @@ +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use super::common::GenerationRequest; + +// ============================================================================ +// Embedding API +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ClassifyRequest { + /// ID of the model to use + pub model: String, + + /// Input can be a string, array of strings, tokens, or batch inputs + pub input: Value, + + /// Optional encoding format (e.g., "float", "base64") + #[serde(skip_serializing_if = "Option::is_none")] + pub encoding_format: Option, + + /// Optional user identifier + #[serde(skip_serializing_if = "Option::is_none")] + pub user: Option, + + /// Optional number of dimensions for the embedding + #[serde(skip_serializing_if = "Option::is_none")] + pub dimensions: Option, + + /// SGLang extension: request id for tracking + #[serde(skip_serializing_if = "Option::is_none")] + pub rid: Option, +} + +impl GenerationRequest for ClassifyRequest { + fn is_stream(&self) -> bool { + // Embeddings are non-streaming + false + } + + fn get_model(&self) -> Option<&str> { + Some(&self.model) + } + + fn extract_text_for_routing(&self) -> String { + // Best effort: extract text content for routing decisions + match &self.input { + Value::String(s) => s.clone(), + Value::Array(arr) => arr + .iter() + .filter_map(|v| v.as_str()) + .collect::>() + .join(" "), + _ => String::new(), + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/common.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/common.rs new file mode 100644 index 00000000..ac51e85e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/common.rs @@ -0,0 +1,457 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use validator; + +// ============================================================================ +// Default value helpers +// ============================================================================ + +/// Default model value when not specified +pub(crate) fn default_model() -> String { + "unknown".to_string() +} + +/// Helper function for serde default value (returns true) +pub fn default_true() -> bool { + true +} + +// ============================================================================ +// GenerationRequest Trait +// ============================================================================ + +/// Trait for unified access to generation request properties +/// Implemented by ChatCompletionRequest, CompletionRequest, GenerateRequest, +/// EmbeddingRequest, RerankRequest, and ResponsesRequest +pub trait GenerationRequest: Send + Sync { + /// Check if the request is for streaming + fn is_stream(&self) -> bool; + + /// Get the model name if specified + fn get_model(&self) -> Option<&str>; + + /// Extract text content for routing decisions + fn extract_text_for_routing(&self) -> String; +} + +// ============================================================================ +// String/Array Utilities +// ============================================================================ + +/// A type that can be either a single string or an array of strings +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +#[serde(untagged)] +pub enum StringOrArray { + String(String), + Array(Vec), +} + +impl StringOrArray { + /// Get the number of items in the StringOrArray + pub fn len(&self) -> usize { + match self { + StringOrArray::String(_) => 1, + StringOrArray::Array(arr) => arr.len(), + } + } + + /// Check if the StringOrArray is empty + pub fn is_empty(&self) -> bool { + match self { + StringOrArray::String(s) => s.is_empty(), + StringOrArray::Array(arr) => arr.is_empty(), + } + } + + /// Convert to a vector of strings + pub fn to_vec(&self) -> Vec { + match self { + StringOrArray::String(s) => vec![s.clone()], + StringOrArray::Array(arr) => arr.clone(), + } + } +} + +/// Validates stop sequences (max 4, non-empty strings) +/// Used by both ChatCompletionRequest and ResponsesRequest +pub fn validate_stop(stop: &StringOrArray) -> Result<(), validator::ValidationError> { + match stop { + StringOrArray::String(s) => { + if s.is_empty() { + return Err(validator::ValidationError::new( + "stop sequences cannot be empty", + )); + } + } + StringOrArray::Array(arr) => { + if arr.len() > 4 { + return Err(validator::ValidationError::new( + "maximum 4 stop sequences allowed", + )); + } + for s in arr { + if s.is_empty() { + return Err(validator::ValidationError::new( + "stop sequences cannot be empty", + )); + } + } + } + } + Ok(()) +} + +// ============================================================================ +// Content Parts (for multimodal messages) +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +#[serde(tag = "type")] +pub enum ContentPart { + #[serde(rename = "text")] + Text { text: String }, + #[serde(rename = "image_url")] + ImageUrl { image_url: ImageUrl }, + #[serde(rename = "video_url")] + VideoUrl { video_url: VideoUrl }, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +pub struct ImageUrl { + pub url: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub detail: Option, // "auto", "low", or "high" +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +pub struct VideoUrl { + pub url: String, +} + +// ============================================================================ +// Response Format (for structured outputs) +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type")] +pub enum ResponseFormat { + #[serde(rename = "text")] + Text, + #[serde(rename = "json_object")] + JsonObject, + #[serde(rename = "json_schema")] + JsonSchema { json_schema: JsonSchemaFormat }, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct JsonSchemaFormat { + pub name: String, + pub schema: Value, + #[serde(skip_serializing_if = "Option::is_none")] + pub strict: Option, +} + +// ============================================================================ +// Streaming +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct StreamOptions { + #[serde(skip_serializing_if = "Option::is_none")] + pub include_usage: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ToolCallDelta { + pub index: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(rename = "type")] + pub tool_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub function: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct FunctionCallDelta { + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub arguments: Option, +} + +// ============================================================================ +// Tools and Function Calling +// ============================================================================ + +/// Tool choice value for simple string options +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ToolChoiceValue { + Auto, + Required, + None, +} + +/// Tool choice for both Chat Completion and Responses APIs +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum ToolChoice { + Value(ToolChoiceValue), + Function { + #[serde(rename = "type")] + tool_type: String, // "function" + function: FunctionChoice, + }, + AllowedTools { + #[serde(rename = "type")] + tool_type: String, // "allowed_tools" + mode: String, // "auto" | "required" TODO: need validation + tools: Vec, + }, +} + +impl Default for ToolChoice { + fn default() -> Self { + Self::Value(ToolChoiceValue::Auto) + } +} + +impl ToolChoice { + /// Serialize tool_choice to string for ResponsesResponse + /// + /// Returns the JSON-serialized tool_choice or "auto" as default + pub fn serialize_to_string(tool_choice: &Option) -> String { + tool_choice + .as_ref() + .map(|tc| serde_json::to_string(tc).unwrap_or_else(|_| "auto".to_string())) + .unwrap_or_else(|| "auto".to_string()) + } +} + +/// Function choice specification for ToolChoice::Function +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct FunctionChoice { + pub name: String, +} + +/// Tool reference for ToolChoice::AllowedTools +/// +/// Represents a reference to a specific tool in the allowed_tools array. +/// Different tool types have different required fields. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type")] +#[serde(rename_all = "snake_case")] +pub enum ToolReference { + /// Reference to a function tool + #[serde(rename = "function")] + Function { name: String }, + + /// Reference to an MCP tool + #[serde(rename = "mcp")] + Mcp { + server_label: String, + #[serde(skip_serializing_if = "Option::is_none")] + name: Option, + }, + + /// File search hosted tool + #[serde(rename = "file_search")] + FileSearch, + + /// Web search preview hosted tool + #[serde(rename = "web_search_preview")] + WebSearchPreview, + + /// Computer use preview hosted tool + #[serde(rename = "computer_use_preview")] + ComputerUsePreview, + + /// Code interpreter hosted tool + #[serde(rename = "code_interpreter")] + CodeInterpreter, + + /// Image generation hosted tool + #[serde(rename = "image_generation")] + ImageGeneration, +} + +impl ToolReference { + /// Get a unique identifier for this tool reference + pub fn identifier(&self) -> String { + match self { + ToolReference::Function { name } => format!("function:{}", name), + ToolReference::Mcp { server_label, name } => { + if let Some(n) = name { + format!("mcp:{}:{}", server_label, n) + } else { + format!("mcp:{}", server_label) + } + } + ToolReference::FileSearch => "file_search".to_string(), + ToolReference::WebSearchPreview => "web_search_preview".to_string(), + ToolReference::ComputerUsePreview => "computer_use_preview".to_string(), + ToolReference::CodeInterpreter => "code_interpreter".to_string(), + ToolReference::ImageGeneration => "image_generation".to_string(), + } + } + + /// Get the tool name if this is a function tool + pub fn function_name(&self) -> Option<&str> { + match self { + ToolReference::Function { name } => Some(name.as_str()), + _ => None, + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct Tool { + #[serde(rename = "type")] + pub tool_type: String, // "function" + pub function: Function, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct Function { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + pub parameters: Value, // JSON Schema + /// Whether to enable strict schema adherence (OpenAI structured outputs) + #[serde(skip_serializing_if = "Option::is_none")] + pub strict: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ToolCall { + pub id: String, + #[serde(rename = "type")] + pub tool_type: String, // "function" + pub function: FunctionCallResponse, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum FunctionCall { + None, + Auto, + Function { name: String }, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct FunctionCallResponse { + pub name: String, + #[serde(default)] + pub arguments: Option, // JSON string +} + +// ============================================================================ +// Usage and Logging +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct Usage { + pub prompt_tokens: u32, + pub completion_tokens: u32, + pub total_tokens: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub completion_tokens_details: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct CompletionTokensDetails { + pub reasoning_tokens: Option, +} + +/// Usage information (used by rerank and other endpoints) +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct UsageInfo { + pub prompt_tokens: u32, + pub completion_tokens: u32, + pub total_tokens: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub prompt_tokens_details: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct PromptTokenUsageInfo { + pub cached_tokens: u32, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct LogProbs { + pub tokens: Vec, + pub token_logprobs: Vec>, + pub top_logprobs: Vec>>, + pub text_offset: Vec, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum ChatLogProbs { + Detailed { + #[serde(skip_serializing_if = "Option::is_none")] + content: Option>, + }, + Raw(Value), +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ChatLogProbsContent { + pub token: String, + pub logprob: f32, + pub bytes: Option>, + pub top_logprobs: Vec, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct TopLogProb { + pub token: String, + pub logprob: f32, + pub bytes: Option>, +} + +// ============================================================================ +// Error Types +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ErrorResponse { + pub error: ErrorDetail, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ErrorDetail { + pub message: String, + #[serde(rename = "type")] + pub error_type: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub param: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub code: Option, +} + +// ============================================================================ +// Input Types +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum InputIds { + Single(Vec), + Batch(Vec>), +} + +/// LoRA adapter path - can be single path or batch of paths (SGLang extension) +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum LoRAPath { + Single(Option), + Batch(Vec>), +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/completion.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/completion.rs new file mode 100644 index 00000000..c6a4f638 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/completion.rs @@ -0,0 +1,214 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +use super::common::*; + +// ============================================================================ +// Completions API (v1/completions) - DEPRECATED but still supported +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct CompletionRequest { + /// ID of the model to use (required for OpenAI, optional for some implementations, such as SGLang) + pub model: String, + + /// The prompt(s) to generate completions for + pub prompt: StringOrArray, + + /// The suffix that comes after a completion of inserted text + #[serde(skip_serializing_if = "Option::is_none")] + pub suffix: Option, + + /// The maximum number of tokens to generate + #[serde(skip_serializing_if = "Option::is_none")] + pub max_tokens: Option, + + /// What sampling temperature to use, between 0 and 2 + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + + /// An alternative to sampling with temperature (nucleus sampling) + #[serde(skip_serializing_if = "Option::is_none")] + pub top_p: Option, + + /// How many completions to generate for each prompt + #[serde(skip_serializing_if = "Option::is_none")] + pub n: Option, + + /// Whether to stream back partial progress + #[serde(default)] + pub stream: bool, + + /// Options for streaming response + #[serde(skip_serializing_if = "Option::is_none")] + pub stream_options: Option, + + /// Include the log probabilities on the logprobs most likely tokens + #[serde(skip_serializing_if = "Option::is_none")] + pub logprobs: Option, + + /// Echo back the prompt in addition to the completion + #[serde(default)] + pub echo: bool, + + /// Up to 4 sequences where the API will stop generating further tokens + #[serde(skip_serializing_if = "Option::is_none")] + pub stop: Option, + + /// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far + #[serde(skip_serializing_if = "Option::is_none")] + pub presence_penalty: Option, + + /// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far + #[serde(skip_serializing_if = "Option::is_none")] + pub frequency_penalty: Option, + + /// Generates best_of completions server-side and returns the "best" + #[serde(skip_serializing_if = "Option::is_none")] + pub best_of: Option, + + /// Modify the likelihood of specified tokens appearing in the completion + #[serde(skip_serializing_if = "Option::is_none")] + pub logit_bias: Option>, + + /// A unique identifier representing your end-user + #[serde(skip_serializing_if = "Option::is_none")] + pub user: Option, + + /// If specified, our system will make a best effort to sample deterministically + #[serde(skip_serializing_if = "Option::is_none")] + pub seed: Option, + + // -------- Engine Specific Sampling Parameters -------- + /// Top-k sampling parameter (-1 to disable) + #[serde(skip_serializing_if = "Option::is_none")] + pub top_k: Option, + + /// Min-p nucleus sampling parameter + #[serde(skip_serializing_if = "Option::is_none")] + pub min_p: Option, + + /// Minimum number of tokens to generate + #[serde(skip_serializing_if = "Option::is_none")] + pub min_tokens: Option, + + /// Repetition penalty for reducing repetitive text + #[serde(skip_serializing_if = "Option::is_none")] + pub repetition_penalty: Option, + + /// Regex constraint for output generation + #[serde(skip_serializing_if = "Option::is_none")] + pub regex: Option, + + /// EBNF grammar constraint for structured output + #[serde(skip_serializing_if = "Option::is_none")] + pub ebnf: Option, + + /// JSON schema constraint for structured output + #[serde(skip_serializing_if = "Option::is_none")] + pub json_schema: Option, + + /// Specific token IDs to use as stop conditions + #[serde(skip_serializing_if = "Option::is_none")] + pub stop_token_ids: Option>, + + /// Skip trimming stop tokens from output + #[serde(default)] + pub no_stop_trim: bool, + + /// Ignore end-of-sequence tokens during generation + #[serde(default)] + pub ignore_eos: bool, + + /// Skip special tokens during detokenization + #[serde(default = "default_true")] + pub skip_special_tokens: bool, + + /// Path to LoRA adapter(s) for model customization + #[serde(skip_serializing_if = "Option::is_none")] + pub lora_path: Option, + + /// Session parameters for continual prompting + #[serde(skip_serializing_if = "Option::is_none")] + pub session_params: Option>, + + /// Return model hidden states + #[serde(default)] + pub return_hidden_states: bool, + + /// Sampling seed for deterministic outputs + #[serde(skip_serializing_if = "Option::is_none")] + pub sampling_seed: Option, + + /// Additional fields including bootstrap info for PD routing + #[serde(flatten)] + pub other: Map, +} + +impl GenerationRequest for CompletionRequest { + fn is_stream(&self) -> bool { + self.stream + } + + fn get_model(&self) -> Option<&str> { + Some(&self.model) + } + + fn extract_text_for_routing(&self) -> String { + match &self.prompt { + StringOrArray::String(s) => s.clone(), + StringOrArray::Array(v) => v.join(" "), + } + } +} + +// ============================================================================ +// Response Types +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct CompletionResponse { + pub id: String, + pub object: String, // "text_completion" + pub created: u64, + pub model: String, + pub choices: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub usage: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub system_fingerprint: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct CompletionChoice { + pub text: String, + pub index: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub logprobs: Option, + pub finish_reason: Option, // "stop", "length", "content_filter", etc. + /// Information about which stop condition was matched + #[serde(skip_serializing_if = "Option::is_none")] + pub matched_stop: Option, // Can be string or integer +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct CompletionStreamResponse { + pub id: String, + pub object: String, // "text_completion" + pub created: u64, + pub choices: Vec, + pub model: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub system_fingerprint: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct CompletionStreamChoice { + pub text: String, + pub index: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub logprobs: Option, + pub finish_reason: Option, +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/embedding.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/embedding.rs new file mode 100644 index 00000000..a76c9b67 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/embedding.rs @@ -0,0 +1,57 @@ +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use super::common::GenerationRequest; + +// ============================================================================ +// Embedding API +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct EmbeddingRequest { + /// ID of the model to use + pub model: String, + + /// Input can be a string, array of strings, tokens, or batch inputs + pub input: Value, + + /// Optional encoding format (e.g., "float", "base64") + #[serde(skip_serializing_if = "Option::is_none")] + pub encoding_format: Option, + + /// Optional user identifier + #[serde(skip_serializing_if = "Option::is_none")] + pub user: Option, + + /// Optional number of dimensions for the embedding + #[serde(skip_serializing_if = "Option::is_none")] + pub dimensions: Option, + + /// SGLang extension: request id for tracking + #[serde(skip_serializing_if = "Option::is_none")] + pub rid: Option, +} + +impl GenerationRequest for EmbeddingRequest { + fn is_stream(&self) -> bool { + // Embeddings are non-streaming + false + } + + fn get_model(&self) -> Option<&str> { + Some(&self.model) + } + + fn extract_text_for_routing(&self) -> String { + // Best effort: extract text content for routing decisions + match &self.input { + Value::String(s) => s.clone(), + Value::Array(arr) => arr + .iter() + .filter_map(|v| v.as_str()) + .collect::>() + .join(" "), + _ => String::new(), + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/generate.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/generate.rs new file mode 100644 index 00000000..d5819095 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/generate.rs @@ -0,0 +1,297 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use validator::Validate; + +use super::{ + common::{default_true, GenerationRequest, InputIds}, + sampling_params::SamplingParams, +}; +use crate::protocols::validated::Normalizable; + +// ============================================================================ +// SGLang Generate API (native format) +// ============================================================================ + +#[derive(Clone, Debug, Serialize, Deserialize, Validate)] +#[validate(schema(function = "validate_generate_request"))] +pub struct GenerateRequest { + /// Text input - SGLang native format + #[serde(skip_serializing_if = "Option::is_none")] + pub text: Option, + + pub model: Option, + + /// Input IDs for tokenized input + #[serde(skip_serializing_if = "Option::is_none")] + pub input_ids: Option, + + /// Input embeddings for direct embedding input + /// Can be a 2D array (single request) or 3D array (batch of requests) + /// Placeholder for future use + #[serde(skip_serializing_if = "Option::is_none")] + pub input_embeds: Option, + + /// Image input data + /// Can be an image instance, file name, URL, or base64 encoded string + /// Supports single images, lists of images, or nested lists for batch processing + /// Placeholder for future use + #[serde(skip_serializing_if = "Option::is_none")] + pub image_data: Option, + + /// Video input data + /// Can be a file name, URL, or base64 encoded string + /// Supports single videos, lists of videos, or nested lists for batch processing + /// Placeholder for future use + #[serde(skip_serializing_if = "Option::is_none")] + pub video_data: Option, + + /// Audio input data + /// Can be a file name, URL, or base64 encoded string + /// Supports single audio files, lists of audio, or nested lists for batch processing + /// Placeholder for future use + #[serde(skip_serializing_if = "Option::is_none")] + pub audio_data: Option, + + /// Sampling parameters (sglang style) + #[serde(skip_serializing_if = "Option::is_none")] + pub sampling_params: Option, + + /// Whether to return logprobs + #[serde(skip_serializing_if = "Option::is_none")] + pub return_logprob: Option, + + /// If return logprobs, the start location in the prompt for returning logprobs. + #[serde(skip_serializing_if = "Option::is_none")] + pub logprob_start_len: Option, + + /// If return logprobs, the number of top logprobs to return at each position. + #[serde(skip_serializing_if = "Option::is_none")] + pub top_logprobs_num: Option, + + /// If return logprobs, the token ids to return logprob for. + #[serde(skip_serializing_if = "Option::is_none")] + pub token_ids_logprob: Option>, + + /// Whether to detokenize tokens in text in the returned logprobs. + #[serde(default)] + pub return_text_in_logprobs: bool, + + /// Whether to stream the response + #[serde(default)] + pub stream: bool, + + /// Whether to log metrics for this request (e.g. health_generate calls do not log metrics) + #[serde(default = "default_true")] + pub log_metrics: bool, + + /// Return model hidden states + #[serde(default)] + pub return_hidden_states: bool, + + /// The modalities of the image data [image, multi-images, video] + #[serde(skip_serializing_if = "Option::is_none")] + pub modalities: Option>, + + /// Session parameters for continual prompting + #[serde(skip_serializing_if = "Option::is_none")] + pub session_params: Option>, + + /// Path to LoRA adapter(s) for model customization + #[serde(skip_serializing_if = "Option::is_none")] + pub lora_path: Option, + + /// LoRA adapter ID (if pre-loaded) + #[serde(skip_serializing_if = "Option::is_none")] + pub lora_id: Option, + + /// Custom logit processor for advanced sampling control. Must be a serialized instance + /// of `CustomLogitProcessor` in python/sglang/srt/sampling/custom_logit_processor.py + /// Use the processor's `to_str()` method to generate the serialized string. + #[serde(skip_serializing_if = "Option::is_none")] + pub custom_logit_processor: Option, + + /// For disaggregated inference + #[serde(skip_serializing_if = "Option::is_none")] + pub bootstrap_host: Option, + + /// For disaggregated inference + #[serde(skip_serializing_if = "Option::is_none")] + pub bootstrap_port: Option, + + /// For disaggregated inference + #[serde(skip_serializing_if = "Option::is_none")] + pub bootstrap_room: Option, + + /// For disaggregated inference + #[serde(skip_serializing_if = "Option::is_none")] + pub bootstrap_pair_key: Option, + + /// Data parallel rank routing + #[serde(skip_serializing_if = "Option::is_none")] + pub data_parallel_rank: Option, + + /// Background response + #[serde(default)] + pub background: bool, + + /// Conversation ID for tracking + #[serde(skip_serializing_if = "Option::is_none")] + pub conversation_id: Option, + + /// Priority for the request + #[serde(skip_serializing_if = "Option::is_none")] + pub priority: Option, + + /// Extra key for classifying the request (e.g. cache_salt) + #[serde(skip_serializing_if = "Option::is_none")] + pub extra_key: Option, + + /// Whether to disallow logging for this request (e.g. due to ZDR) + #[serde(default)] + pub no_logs: bool, + + /// Custom metric labels + #[serde(skip_serializing_if = "Option::is_none")] + pub custom_labels: Option>, + + /// Whether to return bytes for image generation + #[serde(default)] + pub return_bytes: bool, + + /// Whether to return entropy + #[serde(default)] + pub return_entropy: bool, + + /// Request ID for tracking (inherited from BaseReq in Python) + #[serde(skip_serializing_if = "Option::is_none")] + pub rid: Option, +} + +impl Normalizable for GenerateRequest { + // Use default no-op implementation - no normalization needed for GenerateRequest +} + +/// Validation function for GenerateRequest - ensure exactly one input type is provided +fn validate_generate_request(req: &GenerateRequest) -> Result<(), validator::ValidationError> { + // Exactly one of text or input_ids must be provided + // Note: input_embeds not yet supported in Rust implementation + let has_text = req.text.is_some(); + let has_input_ids = req.input_ids.is_some(); + + let count = [has_text, has_input_ids].iter().filter(|&&x| x).count(); + + if count == 0 { + return Err(validator::ValidationError::new( + "Either text or input_ids should be provided.", + )); + } + + if count > 1 { + return Err(validator::ValidationError::new( + "Either text or input_ids should be provided.", + )); + } + + Ok(()) +} + +impl GenerationRequest for GenerateRequest { + fn is_stream(&self) -> bool { + self.stream + } + + fn get_model(&self) -> Option<&str> { + // Generate requests have an optional model field + if let Some(s) = &self.model { + Some(s.as_str()) + } else { + None + } + } + + fn extract_text_for_routing(&self) -> String { + // Check fields in priority order: text, input_ids + if let Some(ref text) = self.text { + return text.clone(); + } + + if let Some(ref input_ids) = self.input_ids { + return match input_ids { + InputIds::Single(ids) => ids + .iter() + .map(|&id| id.to_string()) + .collect::>() + .join(" "), + InputIds::Batch(batches) => batches + .iter() + .flat_map(|batch| batch.iter().map(|&id| id.to_string())) + .collect::>() + .join(" "), + }; + } + + // No text input found + String::new() + } +} + +// ============================================================================ +// SGLang Generate Response Types +// ============================================================================ + +/// SGLang generate response (single completion or array for n>1) +/// +/// Format for n=1: +/// ```json +/// { +/// "text": "...", +/// "output_ids": [...], +/// "meta_info": { ... } +/// } +/// ``` +/// +/// Format for n>1: +/// ```json +/// [ +/// {"text": "...", "output_ids": [...], "meta_info": {...}}, +/// {"text": "...", "output_ids": [...], "meta_info": {...}} +/// ] +/// ``` +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GenerateResponse { + pub text: String, + pub output_ids: Vec, + pub meta_info: GenerateMetaInfo, +} + +/// Metadata for a single generate completion +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GenerateMetaInfo { + pub id: String, + pub finish_reason: GenerateFinishReason, + pub prompt_tokens: u32, + pub weight_version: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub input_token_logprobs: Option>>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub output_token_logprobs: Option>>>, + pub completion_tokens: u32, + pub cached_tokens: u32, + pub e2e_latency: f64, + #[serde(skip_serializing_if = "Option::is_none")] + pub matched_stop: Option, +} + +/// Finish reason for generate endpoint +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "lowercase")] +pub enum GenerateFinishReason { + Length { + length: u32, + }, + Stop, + #[serde(untagged)] + Other(Value), +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/mod.rs new file mode 100644 index 00000000..813ff48d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/mod.rs @@ -0,0 +1,15 @@ +// Protocol definitions and validation for various LLM APIs +// This module provides a structured approach to handling different API protocols + +pub mod builders; +pub mod chat; +pub mod classify; +pub mod common; +pub mod completion; +pub mod embedding; +pub mod generate; +pub mod rerank; +pub mod responses; +pub mod sampling_params; +pub mod validated; +pub mod worker_spec; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/rerank.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/rerank.rs new file mode 100644 index 00000000..6775f5d8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/rerank.rs @@ -0,0 +1,212 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use validator::Validate; + +use super::common::{default_model, default_true, GenerationRequest, StringOrArray, UsageInfo}; + +fn default_rerank_object() -> String { + "rerank".to_string() +} + +/// TODO: Create timestamp should not be in protocol layer +fn current_timestamp() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_else(|_| std::time::Duration::from_secs(0)) + .as_secs() as i64 +} + +// ============================================================================ +// Rerank API +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize, Validate)] +#[validate(schema(function = "validate_rerank_request"))] +pub struct RerankRequest { + /// The query text to rank documents against + #[validate(custom(function = "validate_query"))] + pub query: String, + + /// List of documents to be ranked + #[validate(custom(function = "validate_documents"))] + pub documents: Vec, + + /// Model to use for reranking + #[serde(default = "default_model")] + pub model: String, + + /// Maximum number of documents to return (optional) + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 1))] + pub top_k: Option, + + /// Whether to return documents in addition to scores + #[serde(default = "default_true")] + pub return_documents: bool, + + // SGLang specific extensions + /// Request ID for tracking + pub rid: Option, + + /// User identifier + pub user: Option, +} + +impl GenerationRequest for RerankRequest { + fn get_model(&self) -> Option<&str> { + Some(&self.model) + } + + fn is_stream(&self) -> bool { + false // Reranking doesn't support streaming + } + + fn extract_text_for_routing(&self) -> String { + self.query.clone() + } +} + +impl super::validated::Normalizable for RerankRequest { + // Use default no-op normalization +} + +// ============================================================================ +// Validation Functions +// ============================================================================ + +/// Validates that the query is not empty +fn validate_query(query: &str) -> Result<(), validator::ValidationError> { + if query.trim().is_empty() { + return Err(validator::ValidationError::new("query cannot be empty")); + } + Ok(()) +} + +/// Validates that the documents list is not empty +fn validate_documents(documents: &[String]) -> Result<(), validator::ValidationError> { + if documents.is_empty() { + return Err(validator::ValidationError::new( + "documents list cannot be empty", + )); + } + Ok(()) +} + +/// Schema-level validation for cross-field dependencies +fn validate_rerank_request(req: &RerankRequest) -> Result<(), validator::ValidationError> { + // Validate top_k if specified + if let Some(k) = req.top_k { + if k > req.documents.len() { + // This is allowed but we log a warning + tracing::warn!( + "top_k ({}) is greater than number of documents ({})", + k, + req.documents.len() + ); + } + } + Ok(()) +} + +impl RerankRequest { + /// Get the effective top_k value + pub fn effective_top_k(&self) -> usize { + self.top_k.unwrap_or(self.documents.len()) + } +} + +/// Individual rerank result +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RerankResult { + /// Relevance score for the document + pub score: f32, + + /// The document text (if return_documents was true) + #[serde(skip_serializing_if = "Option::is_none")] + pub document: Option, + + /// Original index of the document in the request + pub index: usize, + + /// Additional metadata about the ranking + #[serde(skip_serializing_if = "Option::is_none")] + pub meta_info: Option>, +} + +/// Rerank response containing sorted results +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RerankResponse { + /// Ranked results sorted by score (highest first) + pub results: Vec, + + /// Model used for reranking + pub model: String, + + /// Usage information + pub usage: Option, + + /// Response object type + #[serde(default = "default_rerank_object")] + pub object: String, + + /// Response ID + pub id: Option, + + /// Creation timestamp + pub created: i64, +} + +impl RerankResponse { + /// Create a new RerankResponse with the given results and model + pub fn new( + results: Vec, + model: String, + request_id: Option, + ) -> Self { + RerankResponse { + results, + model, + usage: None, + object: default_rerank_object(), + id: request_id, + created: current_timestamp(), + } + } + + /// Apply top_k limit to results + pub fn apply_top_k(&mut self, k: usize) { + self.results.truncate(k); + } + + /// Drop documents from results (when return_documents is false) + pub fn drop_documents(&mut self) { + for result in &mut self.results { + result.document = None; + } + } +} + +/// V1 API compatibility format for rerank requests +/// Matches Python's V1RerankReqInput +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct V1RerankReqInput { + pub query: String, + pub documents: Vec, +} + +/// Convert V1RerankReqInput to RerankRequest +impl From for RerankRequest { + fn from(v1: V1RerankReqInput) -> Self { + RerankRequest { + query: v1.query, + documents: v1.documents, + model: default_model(), + top_k: None, + return_documents: true, + rid: None, + user: None, + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/responses.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/responses.rs new file mode 100644 index 00000000..f2f41d34 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/responses.rs @@ -0,0 +1,1298 @@ +// OpenAI Responses API types +// https://platform.openai.com/docs/api-reference/responses + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use validator::Validate; + +use super::{ + common::{ + default_model, default_true, validate_stop, ChatLogProbs, Function, GenerationRequest, + PromptTokenUsageInfo, StringOrArray, ToolChoice, ToolChoiceValue, ToolReference, UsageInfo, + }, + sampling_params::{validate_top_k_value, validate_top_p_value}, +}; +use crate::protocols::{builders::ResponsesResponseBuilder, validated::Normalizable}; + +// ============================================================================ +// Response Tools (MCP and others) +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ResponseTool { + #[serde(rename = "type")] + pub r#type: ResponseToolType, + // Function tool fields (used when type == "function") + // In Responses API, function fields are flattened at the top level + #[serde(flatten)] + #[serde(skip_serializing_if = "Option::is_none")] + pub function: Option, + // MCP-specific fields (used when type == "mcp") + #[serde(skip_serializing_if = "Option::is_none")] + pub server_url: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub authorization: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub server_label: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub server_description: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub require_approval: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub allowed_tools: Option>, +} + +impl Default for ResponseTool { + fn default() -> Self { + Self { + r#type: ResponseToolType::WebSearchPreview, + function: None, + server_url: None, + authorization: None, + server_label: None, + server_description: None, + require_approval: None, + allowed_tools: None, + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum ResponseToolType { + Function, + WebSearchPreview, + CodeInterpreter, + Mcp, +} + +// ============================================================================ +// Reasoning Parameters +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ResponseReasoningParam { + #[serde(default = "default_reasoning_effort")] + #[serde(skip_serializing_if = "Option::is_none")] + pub effort: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, +} + +fn default_reasoning_effort() -> Option { + Some(ReasoningEffort::Medium) +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ReasoningEffort { + Minimal, + Low, + Medium, + High, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ReasoningSummary { + Auto, + Concise, + Detailed, +} + +// ============================================================================ +// Input/Output Items +// ============================================================================ + +/// Content can be either a simple string or array of content parts (for SimpleInputMessage) +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum StringOrContentParts { + String(String), + Array(Vec), +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type")] +#[serde(rename_all = "snake_case")] +pub enum ResponseInputOutputItem { + #[serde(rename = "message")] + Message { + id: String, + role: String, + content: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + status: Option, + }, + #[serde(rename = "reasoning")] + Reasoning { + id: String, + summary: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + #[serde(default)] + content: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + status: Option, + }, + #[serde(rename = "function_call")] + FunctionToolCall { + id: String, + call_id: String, + name: String, + arguments: String, + #[serde(skip_serializing_if = "Option::is_none")] + output: Option, + #[serde(skip_serializing_if = "Option::is_none")] + status: Option, + }, + #[serde(rename = "function_call_output")] + FunctionCallOutput { + id: Option, + call_id: String, + output: String, + #[serde(skip_serializing_if = "Option::is_none")] + status: Option, + }, + #[serde(untagged)] + SimpleInputMessage { + content: StringOrContentParts, + role: String, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(rename = "type")] + r#type: Option, + }, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type")] +#[serde(rename_all = "snake_case")] +pub enum ResponseContentPart { + #[serde(rename = "output_text")] + OutputText { + text: String, + #[serde(default)] + #[serde(skip_serializing_if = "Vec::is_empty")] + annotations: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + logprobs: Option, + }, + #[serde(rename = "input_text")] + InputText { text: String }, + #[serde(other)] + Unknown, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type")] +#[serde(rename_all = "snake_case")] +pub enum ResponseReasoningContent { + #[serde(rename = "reasoning_text")] + ReasoningText { text: String }, +} + +/// MCP Tool information for the mcp_list_tools output item +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct McpToolInfo { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + pub input_schema: Value, + #[serde(skip_serializing_if = "Option::is_none")] + pub annotations: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type")] +#[serde(rename_all = "snake_case")] +pub enum ResponseOutputItem { + #[serde(rename = "message")] + Message { + id: String, + role: String, + content: Vec, + status: String, + }, + #[serde(rename = "reasoning")] + Reasoning { + id: String, + summary: Vec, + content: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + status: Option, + }, + #[serde(rename = "function_call")] + FunctionToolCall { + id: String, + call_id: String, + name: String, + arguments: String, + #[serde(skip_serializing_if = "Option::is_none")] + output: Option, + status: String, + }, + #[serde(rename = "mcp_list_tools")] + McpListTools { + id: String, + server_label: String, + tools: Vec, + }, + #[serde(rename = "mcp_call")] + McpCall { + id: String, + status: String, + #[serde(skip_serializing_if = "Option::is_none")] + approval_request_id: Option, + arguments: String, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, + name: String, + output: String, + server_label: String, + }, +} + +// ============================================================================ +// Configuration Enums +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum ServiceTier { + #[default] + Auto, + Default, + Flex, + Scale, + Priority, +} + +#[derive(Debug, Clone, Deserialize, Serialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum Truncation { + Auto, + #[default] + Disabled, +} + +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ResponseStatus { + Queued, + InProgress, + Completed, + Failed, + Cancelled, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ReasoningInfo { + #[serde(skip_serializing_if = "Option::is_none")] + pub effort: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, +} + +// ============================================================================ +// Text Format (structured outputs) +// ============================================================================ + +/// Text configuration for structured output requests +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct TextConfig { + #[serde(skip_serializing_if = "Option::is_none")] + pub format: Option, +} + +/// Text format: text (default), json_object (legacy), or json_schema (recommended) +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type")] +pub enum TextFormat { + #[serde(rename = "text")] + Text, + + #[serde(rename = "json_object")] + JsonObject, + + #[serde(rename = "json_schema")] + JsonSchema { + name: String, + schema: Value, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + #[serde(skip_serializing_if = "Option::is_none")] + strict: Option, + }, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum IncludeField { + #[serde(rename = "code_interpreter_call.outputs")] + CodeInterpreterCallOutputs, + #[serde(rename = "computer_call_output.output.image_url")] + ComputerCallOutputImageUrl, + #[serde(rename = "file_search_call.results")] + FileSearchCallResults, + #[serde(rename = "message.input_image.image_url")] + MessageInputImageUrl, + #[serde(rename = "message.output_text.logprobs")] + MessageOutputTextLogprobs, + #[serde(rename = "reasoning.encrypted_content")] + ReasoningEncryptedContent, +} + +// ============================================================================ +// Usage Types (Responses API format) +// ============================================================================ + +/// OpenAI Responses API usage format (different from standard UsageInfo) +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ResponseUsage { + pub input_tokens: u32, + pub output_tokens: u32, + pub total_tokens: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub input_tokens_details: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub output_tokens_details: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum ResponsesUsage { + Classic(UsageInfo), + Modern(ResponseUsage), +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct InputTokensDetails { + pub cached_tokens: u32, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct OutputTokensDetails { + pub reasoning_tokens: u32, +} + +impl UsageInfo { + /// Convert to OpenAI Responses API format + pub fn to_response_usage(&self) -> ResponseUsage { + ResponseUsage { + input_tokens: self.prompt_tokens, + output_tokens: self.completion_tokens, + total_tokens: self.total_tokens, + input_tokens_details: self.prompt_tokens_details.as_ref().map(|details| { + InputTokensDetails { + cached_tokens: details.cached_tokens, + } + }), + output_tokens_details: self.reasoning_tokens.map(|tokens| OutputTokensDetails { + reasoning_tokens: tokens, + }), + } + } +} + +impl From for ResponseUsage { + fn from(usage: UsageInfo) -> Self { + usage.to_response_usage() + } +} + +impl ResponseUsage { + /// Convert back to standard UsageInfo format + pub fn to_usage_info(&self) -> UsageInfo { + UsageInfo { + prompt_tokens: self.input_tokens, + completion_tokens: self.output_tokens, + total_tokens: self.total_tokens, + reasoning_tokens: self + .output_tokens_details + .as_ref() + .map(|details| details.reasoning_tokens), + prompt_tokens_details: self.input_tokens_details.as_ref().map(|details| { + PromptTokenUsageInfo { + cached_tokens: details.cached_tokens, + } + }), + } + } +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +pub struct ResponsesGetParams { + #[serde(default)] + pub include: Vec, + #[serde(default)] + pub include_obfuscation: Option, + #[serde(default)] + pub starting_after: Option, + #[serde(default)] + pub stream: Option, +} + +impl ResponsesUsage { + pub fn to_response_usage(&self) -> ResponseUsage { + match self { + ResponsesUsage::Classic(usage) => usage.to_response_usage(), + ResponsesUsage::Modern(usage) => usage.clone(), + } + } + + pub fn to_usage_info(&self) -> UsageInfo { + match self { + ResponsesUsage::Classic(usage) => usage.clone(), + ResponsesUsage::Modern(usage) => usage.to_usage_info(), + } + } +} + +// ============================================================================ +// Helper Functions for Defaults +// ============================================================================ + +fn default_top_k() -> i32 { + -1 +} + +fn default_repetition_penalty() -> f32 { + 1.0 +} + +fn default_temperature() -> Option { + Some(1.0) +} + +fn default_top_p() -> Option { + Some(1.0) +} + +// ============================================================================ +// Request/Response Types +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize, Validate)] +#[validate(schema(function = "validate_responses_cross_parameters"))] +pub struct ResponsesRequest { + /// Run the request in the background + #[serde(skip_serializing_if = "Option::is_none")] + pub background: Option, + + /// Fields to include in the response + #[serde(skip_serializing_if = "Option::is_none")] + pub include: Option>, + + /// Input content - can be string or structured items + #[validate(custom(function = "validate_response_input"))] + pub input: ResponseInput, + + /// System instructions for the model + #[serde(skip_serializing_if = "Option::is_none")] + pub instructions: Option, + + /// Maximum number of output tokens + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 1))] + pub max_output_tokens: Option, + + /// Maximum number of tool calls + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 1))] + pub max_tool_calls: Option, + + /// Additional metadata + #[serde(skip_serializing_if = "Option::is_none")] + pub metadata: Option>, + + /// Model to use + #[serde(default = "default_model")] + pub model: String, + + /// Optional conversation id to persist input/output as items + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(custom(function = "validate_conversation_id"))] + pub conversation: Option, + + /// Whether to enable parallel tool calls + #[serde(skip_serializing_if = "Option::is_none")] + pub parallel_tool_calls: Option, + + /// ID of previous response to continue from + #[serde(skip_serializing_if = "Option::is_none")] + pub previous_response_id: Option, + + /// Reasoning configuration + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning: Option, + + /// Service tier + #[serde(skip_serializing_if = "Option::is_none")] + pub service_tier: Option, + + /// Whether to store the response + #[serde(skip_serializing_if = "Option::is_none")] + pub store: Option, + + /// Whether to stream the response + #[serde(default)] + pub stream: Option, + + /// Temperature for sampling + #[serde( + default = "default_temperature", + skip_serializing_if = "Option::is_none" + )] + #[validate(range(min = 0.0, max = 2.0))] + pub temperature: Option, + + /// Tool choice behavior + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_choice: Option, + + /// Available tools + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(custom(function = "validate_response_tools"))] + pub tools: Option>, + + /// Number of top logprobs to return + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 0, max = 20))] + pub top_logprobs: Option, + + /// Top-p sampling parameter + #[serde(default = "default_top_p", skip_serializing_if = "Option::is_none")] + #[validate(custom(function = "validate_top_p_value"))] + pub top_p: Option, + + /// Truncation behavior + #[serde(skip_serializing_if = "Option::is_none")] + pub truncation: Option, + + /// Text format for structured outputs (text, json_object, json_schema) + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(custom(function = "validate_text_format"))] + pub text: Option, + + /// User identifier + #[serde(skip_serializing_if = "Option::is_none")] + pub user: Option, + + /// Request ID + #[serde(skip_serializing_if = "Option::is_none")] + pub request_id: Option, + + /// Request priority + #[serde(default)] + pub priority: i32, + + /// Frequency penalty + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = -2.0, max = 2.0))] + pub frequency_penalty: Option, + + /// Presence penalty + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = -2.0, max = 2.0))] + pub presence_penalty: Option, + + /// Stop sequences + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(custom(function = "validate_stop"))] + pub stop: Option, + + /// Top-k sampling parameter (SGLang extension) + #[serde(default = "default_top_k")] + #[validate(custom(function = "validate_top_k_value"))] + pub top_k: i32, + + /// Min-p sampling parameter (SGLang extension) + #[serde(default)] + #[validate(range(min = 0.0, max = 1.0))] + pub min_p: f32, + + /// Repetition penalty (SGLang extension) + #[serde(default = "default_repetition_penalty")] + #[validate(range(min = 0.0, max = 2.0))] + pub repetition_penalty: f32, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum ResponseInput { + Items(Vec), + Text(String), +} + +impl Default for ResponsesRequest { + fn default() -> Self { + Self { + background: None, + include: None, + input: ResponseInput::Text(String::new()), + instructions: None, + max_output_tokens: None, + max_tool_calls: None, + metadata: None, + model: default_model(), + conversation: None, + parallel_tool_calls: None, + previous_response_id: None, + reasoning: None, + service_tier: None, + store: None, + stream: None, + temperature: None, + tool_choice: None, + tools: None, + top_logprobs: None, + top_p: None, + truncation: None, + text: None, + user: None, + request_id: None, + priority: 0, + frequency_penalty: None, + presence_penalty: None, + stop: None, + top_k: default_top_k(), + min_p: 0.0, + repetition_penalty: default_repetition_penalty(), + } + } +} + +impl Normalizable for ResponsesRequest { + /// Normalize the request by applying defaults: + /// 1. Apply tool_choice defaults based on tools presence + /// 2. Apply parallel_tool_calls defaults + /// 3. Apply store field defaults + fn normalize(&mut self) { + // 1. Apply tool_choice defaults + if self.tool_choice.is_none() { + if let Some(tools) = &self.tools { + let choice_value = if !tools.is_empty() { + ToolChoiceValue::Auto + } else { + ToolChoiceValue::None + }; + self.tool_choice = Some(ToolChoice::Value(choice_value)); + } + // If tools is None, leave tool_choice as None (don't set it) + } + + // 2. Apply default for parallel_tool_calls if tools are present + if self.parallel_tool_calls.is_none() && self.tools.is_some() { + self.parallel_tool_calls = Some(true); + } + + // 3. Ensure store defaults to true if not specified + if self.store.is_none() { + self.store = Some(true); + } + } +} + +impl GenerationRequest for ResponsesRequest { + fn is_stream(&self) -> bool { + self.stream.unwrap_or(false) + } + + fn get_model(&self) -> Option<&str> { + Some(self.model.as_str()) + } + + fn extract_text_for_routing(&self) -> String { + match &self.input { + ResponseInput::Text(text) => text.clone(), + ResponseInput::Items(items) => items + .iter() + .filter_map(|item| match item { + ResponseInputOutputItem::Message { content, .. } => { + let texts: Vec = content + .iter() + .filter_map(|part| match part { + ResponseContentPart::OutputText { text, .. } => Some(text.clone()), + ResponseContentPart::InputText { text } => Some(text.clone()), + ResponseContentPart::Unknown => None, + }) + .collect(); + if texts.is_empty() { + None + } else { + Some(texts.join(" ")) + } + } + ResponseInputOutputItem::SimpleInputMessage { content, .. } => { + match content { + StringOrContentParts::String(s) => Some(s.clone()), + StringOrContentParts::Array(parts) => { + // SimpleInputMessage only supports InputText + let texts: Vec = parts + .iter() + .filter_map(|part| match part { + ResponseContentPart::InputText { text } => { + Some(text.clone()) + } + _ => None, + }) + .collect(); + if texts.is_empty() { + None + } else { + Some(texts.join(" ")) + } + } + } + } + ResponseInputOutputItem::Reasoning { content, .. } => { + let texts: Vec = content + .iter() + .map(|part| match part { + ResponseReasoningContent::ReasoningText { text } => text.clone(), + }) + .collect(); + if texts.is_empty() { + None + } else { + Some(texts.join(" ")) + } + } + ResponseInputOutputItem::FunctionToolCall { arguments, .. } => { + Some(arguments.clone()) + } + ResponseInputOutputItem::FunctionCallOutput { output, .. } => { + Some(output.clone()) + } + }) + .collect::>() + .join(" "), + } + } +} + +/// Validate conversation ID format +pub fn validate_conversation_id(conv_id: &str) -> Result<(), validator::ValidationError> { + if !conv_id.starts_with("conv_") { + let mut error = validator::ValidationError::new("invalid_conversation_id"); + error.message = Some(std::borrow::Cow::Owned(format!( + "Invalid 'conversation': '{}'. Expected an ID that begins with 'conv_'.", + conv_id + ))); + return Err(error); + } + + // Check if the conversation ID contains only valid characters + let is_valid = conv_id + .chars() + .all(|c| c.is_alphanumeric() || c == '_' || c == '-'); + + if !is_valid { + let mut error = validator::ValidationError::new("invalid_conversation_id"); + error.message = Some(std::borrow::Cow::Owned(format!( + "Invalid 'conversation': '{}'. Expected an ID that contains letters, numbers, underscores, or dashes, but this value contained additional characters.", + conv_id + ))); + return Err(error); + } + Ok(()) +} + +/// Validates tool_choice requires tools and references exist +fn validate_tool_choice_with_tools( + request: &ResponsesRequest, +) -> Result<(), validator::ValidationError> { + let Some(tool_choice) = &request.tool_choice else { + return Ok(()); + }; + + let has_tools = request.tools.as_ref().is_some_and(|t| !t.is_empty()); + let is_some_choice = !matches!(tool_choice, ToolChoice::Value(ToolChoiceValue::None)); + + // Check if tool_choice requires tools but none are provided + if is_some_choice && !has_tools { + let mut e = validator::ValidationError::new("tool_choice_requires_tools"); + e.message = Some("Invalid value for 'tool_choice': 'tool_choice' is only allowed when 'tools' are specified.".into()); + return Err(e); + } + + // Validate tool references exist when tools are present + if !has_tools { + return Ok(()); + } + + // Extract function tool names from ResponseTools + let tools = request.tools.as_ref().unwrap(); + let function_tool_names: Vec<&str> = tools + .iter() + .filter_map(|t| match t.r#type { + ResponseToolType::Function => t.function.as_ref().map(|f| f.name.as_str()), + _ => None, + }) + .collect(); + + // Validate tool references exist + match tool_choice { + ToolChoice::Function { function, .. } => { + if !function_tool_names.contains(&function.name.as_str()) { + let mut e = validator::ValidationError::new("tool_choice_function_not_found"); + e.message = Some( + format!( + "Invalid value for 'tool_choice': function '{}' not found in 'tools'.", + function.name + ) + .into(), + ); + return Err(e); + } + } + ToolChoice::AllowedTools { + mode, + tools: allowed_tools, + .. + } => { + // Validate mode is "auto" or "required" + if mode != "auto" && mode != "required" { + let mut e = validator::ValidationError::new("tool_choice_invalid_mode"); + e.message = Some( + format!( + "Invalid value for 'tool_choice.mode': must be 'auto' or 'required', got '{}'.", + mode + ) + .into(), + ); + return Err(e); + } + + // Validate that all function tool references exist + for tool_ref in allowed_tools { + if let ToolReference::Function { name } = tool_ref { + if !function_tool_names.contains(&name.as_str()) { + let mut e = validator::ValidationError::new("tool_choice_tool_not_found"); + e.message = Some( + format!( + "Invalid value for 'tool_choice.tools': tool '{}' not found in 'tools'.", + name + ) + .into(), + ); + return Err(e); + } + } + // Note: MCP and hosted tools don't need existence validation here + // as they are resolved dynamically at runtime + } + } + _ => {} + } + + Ok(()) +} + +/// Schema-level validation for cross-field dependencies +fn validate_responses_cross_parameters( + request: &ResponsesRequest, +) -> Result<(), validator::ValidationError> { + // 1. Validate tool_choice requires tools (enhanced) + validate_tool_choice_with_tools(request)?; + + // 2. Validate top_logprobs requires include field + if request.top_logprobs.is_some() { + let has_logprobs_include = request + .include + .as_ref() + .is_some_and(|inc| inc.contains(&IncludeField::MessageOutputTextLogprobs)); + + if !has_logprobs_include { + let mut e = validator::ValidationError::new("top_logprobs_requires_include"); + e.message = Some( + "top_logprobs requires include field with 'message.output_text.logprobs'".into(), + ); + return Err(e); + } + } + + // 3. Validate background/stream conflict + if request.background == Some(true) && request.stream == Some(true) { + let mut e = validator::ValidationError::new("background_conflicts_with_stream"); + e.message = Some("Cannot use background mode with streaming".into()); + return Err(e); + } + + // 4. Validate conversation and previous_response_id are mutually exclusive + if request.conversation.is_some() && request.previous_response_id.is_some() { + let mut e = validator::ValidationError::new("mutually_exclusive_parameters"); + e.message = Some("Mutually exclusive parameters. Ensure you are only providing one of: 'previous_response_id' or 'conversation'.".into()); + return Err(e); + } + + // 5. Validate input items structure + if let ResponseInput::Items(items) = &request.input { + // Check for at least one valid input message + let has_valid_input = items.iter().any(|item| { + matches!( + item, + ResponseInputOutputItem::Message { .. } + | ResponseInputOutputItem::SimpleInputMessage { .. } + ) + }); + + if !has_valid_input { + let mut e = validator::ValidationError::new("input_missing_user_message"); + e.message = Some("Input items must contain at least one message".into()); + return Err(e); + } + } + + // 6. Validate text format conflicts (for future structured output constraints) + // Currently, Responses API doesn't have regex/ebnf like Chat API, + // but this is here for completeness and future-proofing + + Ok(()) +} + +// ============================================================================ +// Field-Level Validation Functions +// ============================================================================ + +/// Validates response input is not empty and has valid content +fn validate_response_input(input: &ResponseInput) -> Result<(), validator::ValidationError> { + match input { + ResponseInput::Text(text) => { + if text.is_empty() { + let mut e = validator::ValidationError::new("input_text_empty"); + e.message = Some("Input text cannot be empty".into()); + return Err(e); + } + } + ResponseInput::Items(items) => { + if items.is_empty() { + let mut e = validator::ValidationError::new("input_items_empty"); + e.message = Some("Input items cannot be empty".into()); + return Err(e); + } + // Validate each item has valid content + for item in items { + validate_input_item(item)?; + } + } + } + Ok(()) +} + +/// Validates individual input items have valid content +fn validate_input_item(item: &ResponseInputOutputItem) -> Result<(), validator::ValidationError> { + match item { + ResponseInputOutputItem::Message { content, .. } => { + if content.is_empty() { + let mut e = validator::ValidationError::new("message_content_empty"); + e.message = Some("Message content cannot be empty".into()); + return Err(e); + } + } + ResponseInputOutputItem::SimpleInputMessage { content, .. } => match content { + StringOrContentParts::String(s) if s.is_empty() => { + let mut e = validator::ValidationError::new("message_content_empty"); + e.message = Some("Message content cannot be empty".into()); + return Err(e); + } + StringOrContentParts::Array(parts) if parts.is_empty() => { + let mut e = validator::ValidationError::new("message_content_empty"); + e.message = Some("Message content parts cannot be empty".into()); + return Err(e); + } + _ => {} + }, + ResponseInputOutputItem::Reasoning { .. } => { + // Reasoning content can be empty - no validation needed + } + ResponseInputOutputItem::FunctionCallOutput { output, .. } => { + if output.is_empty() { + let mut e = validator::ValidationError::new("function_output_empty"); + e.message = Some("Function call output cannot be empty".into()); + return Err(e); + } + } + _ => {} + } + Ok(()) +} + +/// Validates ResponseTool structure based on tool type +fn validate_response_tools(tools: &[ResponseTool]) -> Result<(), validator::ValidationError> { + for tool in tools { + match tool.r#type { + ResponseToolType::Function => { + if tool.function.is_none() { + let mut e = validator::ValidationError::new("function_tool_missing_function"); + e.message = Some("Function tool must have a function definition".into()); + return Err(e); + } + } + ResponseToolType::Mcp => { + if tool.server_url.is_none() { + let mut e = validator::ValidationError::new("mcp_tool_missing_server_url"); + e.message = Some("MCP tool must have a server_url".into()); + return Err(e); + } + } + _ => {} + } + } + Ok(()) +} + +/// Validates text format configuration (JSON schema name cannot be empty) +fn validate_text_format(text: &TextConfig) -> Result<(), validator::ValidationError> { + if let Some(TextFormat::JsonSchema { name, .. }) = &text.format { + if name.is_empty() { + let mut e = validator::ValidationError::new("json_schema_name_empty"); + e.message = Some("JSON schema name cannot be empty".into()); + return Err(e); + } + } + Ok(()) +} + +/// Normalize a SimpleInputMessage to a proper Message item +/// +/// This helper converts SimpleInputMessage (which can have flexible content) +/// into a fully-structured Message item with a generated ID, role, and content array. +/// +/// SimpleInputMessage items are converted to Message items with IDs generated using +/// the centralized ID generation pattern with "msg_" prefix for consistency. +/// +/// # Arguments +/// * `item` - The input item to normalize +/// +/// # Returns +/// A normalized ResponseInputOutputItem (either Message if converted, or original if not SimpleInputMessage) +pub fn normalize_input_item(item: &ResponseInputOutputItem) -> ResponseInputOutputItem { + match item { + ResponseInputOutputItem::SimpleInputMessage { content, role, .. } => { + let content_vec = match content { + StringOrContentParts::String(s) => { + vec![ResponseContentPart::InputText { text: s.clone() }] + } + StringOrContentParts::Array(parts) => parts.clone(), + }; + + ResponseInputOutputItem::Message { + id: generate_id("msg"), + role: role.clone(), + content: content_vec, + status: Some("completed".to_string()), + } + } + _ => item.clone(), + } +} + +pub fn generate_id(prefix: &str) -> String { + use rand::RngCore; + let mut rng = rand::rng(); + // Generate exactly 50 hex characters (25 bytes) for the part after the underscore + let mut bytes = [0u8; 25]; + rng.fill_bytes(&mut bytes); + let hex_string: String = bytes.iter().map(|b| format!("{:02x}", b)).collect(); + format!("{}_{}", prefix, hex_string) +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ResponsesResponse { + /// Response ID + pub id: String, + + /// Object type + #[serde(default = "default_object_type")] + pub object: String, + + /// Creation timestamp + pub created_at: i64, + + /// Response status + pub status: ResponseStatus, + + /// Error information if status is failed + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + + /// Incomplete details if response was truncated + #[serde(skip_serializing_if = "Option::is_none")] + pub incomplete_details: Option, + + /// System instructions used + #[serde(skip_serializing_if = "Option::is_none")] + pub instructions: Option, + + /// Max output tokens setting + #[serde(skip_serializing_if = "Option::is_none")] + pub max_output_tokens: Option, + + /// Model name + pub model: String, + + /// Output items + #[serde(default)] + pub output: Vec, + + /// Whether parallel tool calls are enabled + #[serde(default = "default_true")] + pub parallel_tool_calls: bool, + + /// Previous response ID if this is a continuation + #[serde(skip_serializing_if = "Option::is_none")] + pub previous_response_id: Option, + + /// Reasoning information + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning: Option, + + /// Whether the response is stored + #[serde(default = "default_true")] + pub store: bool, + + /// Temperature setting used + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + + /// Text format settings + #[serde(skip_serializing_if = "Option::is_none")] + pub text: Option, + + /// Tool choice setting + #[serde(default = "default_tool_choice")] + pub tool_choice: String, + + /// Available tools + #[serde(default)] + pub tools: Vec, + + /// Top-p setting used + #[serde(skip_serializing_if = "Option::is_none")] + pub top_p: Option, + + /// Truncation strategy used + #[serde(skip_serializing_if = "Option::is_none")] + pub truncation: Option, + + /// Usage information + #[serde(skip_serializing_if = "Option::is_none")] + pub usage: Option, + + /// User identifier + #[serde(skip_serializing_if = "Option::is_none")] + pub user: Option, + + /// Safety identifier for content moderation + #[serde(skip_serializing_if = "Option::is_none")] + pub safety_identifier: Option, + + /// Additional metadata + #[serde(default)] + pub metadata: HashMap, +} + +fn default_object_type() -> String { + "response".to_string() +} + +fn default_tool_choice() -> String { + "auto".to_string() +} + +impl ResponsesResponse { + /// Create a builder for constructing a ResponsesResponse + pub fn builder(id: impl Into, model: impl Into) -> ResponsesResponseBuilder { + ResponsesResponseBuilder::new(id, model) + } + + /// Check if the response is complete + pub fn is_complete(&self) -> bool { + matches!(self.status, ResponseStatus::Completed) + } + + /// Check if the response is in progress + pub fn is_in_progress(&self) -> bool { + matches!(self.status, ResponseStatus::InProgress) + } + + /// Check if the response failed + pub fn is_failed(&self) -> bool { + matches!(self.status, ResponseStatus::Failed) + } +} + +impl ResponseOutputItem { + /// Create a new message output item + pub fn new_message( + id: String, + role: String, + content: Vec, + status: String, + ) -> Self { + Self::Message { + id, + role, + content, + status, + } + } + + /// Create a new reasoning output item + pub fn new_reasoning( + id: String, + summary: Vec, + content: Vec, + status: Option, + ) -> Self { + Self::Reasoning { + id, + summary, + content, + status, + } + } + + /// Create a new function tool call output item + pub fn new_function_tool_call( + id: String, + call_id: String, + name: String, + arguments: String, + output: Option, + status: String, + ) -> Self { + Self::FunctionToolCall { + id, + call_id, + name, + arguments, + output, + status, + } + } +} + +impl ResponseContentPart { + /// Create a new text content part + pub fn new_text( + text: String, + annotations: Vec, + logprobs: Option, + ) -> Self { + Self::OutputText { + text, + annotations, + logprobs, + } + } +} + +impl ResponseReasoningContent { + /// Create a new reasoning text content + pub fn new_reasoning_text(text: String) -> Self { + Self::ReasoningText { text } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/sampling_params.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/sampling_params.rs new file mode 100644 index 00000000..9055a53d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/sampling_params.rs @@ -0,0 +1,119 @@ +use serde::{Deserialize, Serialize}; +use validator::Validate; + +use super::common::StringOrArray; + +/// Sampling parameters for text generation +#[derive(Debug, Clone, Deserialize, Serialize, Default, Validate)] +#[validate(schema(function = "validate_sampling_params"))] +pub struct SamplingParams { + /// Temperature for sampling (must be >= 0.0, no upper limit) + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 0.0))] + pub temperature: Option, + /// Maximum number of new tokens to generate (must be >= 0) + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 0))] + pub max_new_tokens: Option, + /// Top-p nucleus sampling (0.0 < top_p <= 1.0) + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(custom(function = "validate_top_p_value"))] + pub top_p: Option, + /// Top-k sampling (-1 to disable, or >= 1) + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(custom(function = "validate_top_k_value"))] + pub top_k: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = -2.0, max = 2.0))] + pub frequency_penalty: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = -2.0, max = 2.0))] + pub presence_penalty: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 0.0, max = 2.0))] + pub repetition_penalty: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub stop: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ignore_eos: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub skip_special_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub json_schema: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub regex: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ebnf: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 0.0, max = 1.0))] + pub min_p: Option, + /// Minimum number of new tokens (validated in schema function for cross-field check with max_new_tokens) + #[serde(skip_serializing_if = "Option::is_none")] + pub min_new_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub stop_token_ids: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub no_stop_trim: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub n: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub sampling_seed: Option, +} + +// ============================================================================ +// Shared Validation Functions +// ============================================================================ + +/// Validates top_p: 0.0 < top_p <= 1.0 (can't use range validator for open interval) +pub fn validate_top_p_value(top_p: f32) -> Result<(), validator::ValidationError> { + if !(top_p > 0.0 && top_p <= 1.0) { + return Err(validator::ValidationError::new( + "top_p must be in (0, 1] - greater than 0.0 and at most 1.0", + )); + } + Ok(()) +} + +/// Validates top_k: -1 (disabled) or >= 1 (special -1 case - can't use range validator) +pub fn validate_top_k_value(top_k: i32) -> Result<(), validator::ValidationError> { + if top_k != -1 && top_k < 1 { + return Err(validator::ValidationError::new( + "top_k must be -1 (disabled) or at least 1", + )); + } + Ok(()) +} + +// ============================================================================ +// SamplingParams-Specific Validation +// ============================================================================ + +/// Validation function for SamplingParams - cross-field validation only +fn validate_sampling_params(params: &SamplingParams) -> Result<(), validator::ValidationError> { + // 1. Cross-field validation: min_new_tokens <= max_new_tokens + if let (Some(min), Some(max)) = (params.min_new_tokens, params.max_new_tokens) { + if min > max { + return Err(validator::ValidationError::new( + "min_new_tokens cannot exceed max_new_tokens", + )); + } + } + + // 2. Validate mutually exclusive structured output constraints + let constraint_count = [ + params.regex.is_some(), + params.ebnf.is_some(), + params.json_schema.is_some(), + ] + .iter() + .filter(|&&x| x) + .count(); + + if constraint_count > 1 { + return Err(validator::ValidationError::new( + "only one of regex, ebnf, or json_schema can be set", + )); + } + + Ok(()) +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/validated.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/validated.rs new file mode 100644 index 00000000..7eb5a812 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/validated.rs @@ -0,0 +1,164 @@ +// Validated JSON extractor for automatic request validation +// +// This module provides a ValidatedJson extractor that automatically validates +// requests using the validator crate's Validate trait. + +use axum::{ + extract::{rejection::JsonRejection, FromRequest, Request}, + http::StatusCode, + response::{IntoResponse, Response}, + Json, +}; +use serde::de::DeserializeOwned; +use serde_json::json; +use validator::Validate; + +/// Trait for request types that need post-deserialization normalization +pub trait Normalizable { + /// Normalize the request by applying defaults and transformations + fn normalize(&mut self) { + // Default: no-op + } +} + +/// A JSON extractor that automatically validates and normalizes the request body +/// +/// This extractor deserializes the request body and automatically calls `.validate()` +/// on types that implement the `Validate` trait. If validation fails, it returns +/// a 400 Bad Request with detailed error information. +/// +/// # Example +/// +/// ```rust,ignore +/// async fn create_chat( +/// ValidatedJson(request): ValidatedJson, +/// ) -> Response { +/// // request is guaranteed to be valid here +/// process_request(request).await +/// } +/// ``` +pub struct ValidatedJson(pub T); + +impl FromRequest for ValidatedJson +where + T: DeserializeOwned + Validate + Normalizable + Send, + S: Send + Sync, +{ + type Rejection = Response; + + async fn from_request(req: Request, state: &S) -> Result { + // First, extract and deserialize the JSON + let Json(mut data) = + Json::::from_request(req, state) + .await + .map_err(|err: JsonRejection| { + let error_message = match err { + JsonRejection::JsonDataError(e) => { + format!("Invalid JSON data: {}", e) + } + JsonRejection::JsonSyntaxError(e) => { + format!("JSON syntax error: {}", e) + } + JsonRejection::MissingJsonContentType(_) => { + "Missing Content-Type: application/json header".to_string() + } + _ => format!("Failed to parse JSON: {}", err), + }; + + ( + StatusCode::BAD_REQUEST, + Json(json!({ + "error": { + "message": error_message, + "type": "invalid_request_error", + "code": "json_parse_error" + } + })), + ) + .into_response() + })?; + + // Normalize the request (apply defaults based on other fields) + data.normalize(); + + // Then, automatically validate the data + data.validate().map_err(|validation_errors| { + ( + StatusCode::BAD_REQUEST, + Json(json!({ + "error": { + "message": validation_errors.to_string(), + "type": "invalid_request_error", + "code": 400 + } + })), + ) + .into_response() + })?; + + Ok(ValidatedJson(data)) + } +} + +// Implement Deref to allow transparent access to the inner value +impl std::ops::Deref for ValidatedJson { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl std::ops::DerefMut for ValidatedJson { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +#[cfg(test)] +mod tests { + use serde::{Deserialize, Serialize}; + use validator::Validate; + + use super::*; + + #[derive(Debug, Deserialize, Serialize, Validate)] + struct TestRequest { + #[validate(range(min = 0.0, max = 1.0))] + value: f32, + #[validate(length(min = 1))] + name: String, + } + + impl Normalizable for TestRequest { + // Use default no-op implementation + } + + #[tokio::test] + async fn test_validated_json_valid() { + // This test is conceptual - actual testing would require Axum test harness + let request = TestRequest { + value: 0.5, + name: "test".to_string(), + }; + assert!(request.validate().is_ok()); + } + + #[tokio::test] + async fn test_validated_json_invalid_range() { + let request = TestRequest { + value: 1.5, // Out of range + name: "test".to_string(), + }; + assert!(request.validate().is_err()); + } + + #[tokio::test] + async fn test_validated_json_invalid_length() { + let request = TestRequest { + value: 0.5, + name: "".to_string(), // Empty name + }; + assert!(request.validate().is_err()); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/worker_spec.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/worker_spec.rs new file mode 100644 index 00000000..753b84db --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/protocols/worker_spec.rs @@ -0,0 +1,315 @@ +//! Worker management API specifications +//! +//! Defines the request/response structures for worker management endpoints + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +/// Worker configuration for API requests +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct WorkerConfigRequest { + /// Worker URL (required) + pub url: String, + + /// Worker API key (optional) + #[serde(skip_serializing_if = "Option::is_none")] + pub api_key: Option, + + /// Model ID (optional, will query from server if not provided) + #[serde(skip_serializing_if = "Option::is_none")] + pub model_id: Option, + + /// Worker priority (optional, default: 50, higher = preferred) + #[serde(skip_serializing_if = "Option::is_none")] + pub priority: Option, + + /// Worker cost factor (optional, default: 1.0) + #[serde(skip_serializing_if = "Option::is_none")] + pub cost: Option, + + /// Worker type (optional: "regular", "prefill", "decode") + #[serde(skip_serializing_if = "Option::is_none")] + pub worker_type: Option, + + /// Bootstrap port for prefill workers (optional) + #[serde(skip_serializing_if = "Option::is_none")] + pub bootstrap_port: Option, + + /// Runtime type (optional: "sglang", "vllm", default: "sglang") + /// Only relevant for gRPC workers + #[serde(skip_serializing_if = "Option::is_none")] + pub runtime: Option, + + // gRPC-specific configuration (optional, ignored in HTTP mode) + /// Tokenizer path for gRPC mode + #[serde(skip_serializing_if = "Option::is_none")] + pub tokenizer_path: Option, + + /// Reasoning parser type for gRPC mode + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning_parser: Option, + + /// Tool parser type for gRPC mode + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_parser: Option, + + /// Chat template for gRPC mode + #[serde(skip_serializing_if = "Option::is_none")] + pub chat_template: Option, + + /// Additional labels (optional) + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub labels: HashMap, + + /// Health check timeout in seconds (default: 30) + #[serde(default = "default_health_check_timeout")] + pub health_check_timeout_secs: u64, + + /// Health check interval in seconds (default: 60) + #[serde(default = "default_health_check_interval")] + pub health_check_interval_secs: u64, + + /// Number of successful health checks needed to mark worker as healthy (default: 2) + #[serde(default = "default_health_success_threshold")] + pub health_success_threshold: u32, + + /// Number of failed health checks before marking worker as unhealthy (default: 3) + #[serde(default = "default_health_failure_threshold")] + pub health_failure_threshold: u32, + + /// Maximum connection attempts during worker registration (default: 20) + #[serde(default = "default_max_connection_attempts")] + pub max_connection_attempts: u32, + + /// Enable data parallelism aware scheduling (default: false) + #[serde(default)] + pub dp_aware: bool, +} + +// Default value functions for serde +fn default_health_check_timeout() -> u64 { + 30 +} + +fn default_health_check_interval() -> u64 { + 60 +} + +fn default_health_success_threshold() -> u32 { + 2 +} + +fn default_health_failure_threshold() -> u32 { + 3 +} + +fn default_max_connection_attempts() -> u32 { + 20 +} + +/// Worker information for API responses +#[derive(Debug, Clone, Serialize)] +pub struct WorkerInfo { + /// Worker unique identifier + pub id: String, + + /// Worker URL + pub url: String, + + /// Model ID this worker serves + pub model_id: String, + + /// Worker priority + pub priority: u32, + + /// Worker cost factor + pub cost: f32, + + /// Worker type + pub worker_type: String, + + /// Whether the worker is healthy + pub is_healthy: bool, + + /// Current load on the worker + pub load: usize, + + /// Connection mode (http or grpc) + pub connection_mode: String, + + /// Runtime type (sglang or vllm, for gRPC workers) + #[serde(skip_serializing_if = "Option::is_none")] + pub runtime_type: Option, + + // gRPC-specific fields (None for HTTP workers) + #[serde(skip_serializing_if = "Option::is_none")] + pub tokenizer_path: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning_parser: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_parser: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub chat_template: Option, + + /// Bootstrap port for prefill workers + #[serde(skip_serializing_if = "Option::is_none")] + pub bootstrap_port: Option, + + /// Additional metadata + #[serde(skip_serializing_if = "HashMap::is_empty")] + pub metadata: HashMap, + + /// Job status for async operations (if available) + #[serde(skip_serializing_if = "Option::is_none")] + pub job_status: Option, +} + +/// Job status for async control plane operations +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct JobStatus { + pub job_type: String, + pub worker_url: String, + pub status: String, + pub message: Option, + pub timestamp: u64, +} + +/// Worker list response +#[derive(Debug, Clone, Serialize)] +pub struct WorkerListResponse { + /// List of workers + pub workers: Vec, + + /// Total count + pub total: usize, + + /// Statistics + pub stats: WorkerStats, +} + +/// Worker statistics +#[derive(Debug, Clone, Serialize)] +pub struct WorkerStats { + pub total_workers: usize, + pub healthy_workers: usize, + pub total_models: usize, + pub total_load: usize, + pub by_type: WorkerTypeStats, +} + +/// Worker statistics by type +#[derive(Debug, Clone, Serialize)] +pub struct WorkerTypeStats { + pub regular: usize, + pub prefill: usize, + pub decode: usize, +} + +/// Worker update request +#[derive(Debug, Clone, Deserialize)] +pub struct WorkerUpdateRequest { + /// Update priority + #[serde(skip_serializing_if = "Option::is_none")] + pub priority: Option, + + /// Update cost + #[serde(skip_serializing_if = "Option::is_none")] + pub cost: Option, + + /// Update labels + #[serde(skip_serializing_if = "Option::is_none")] + pub labels: Option>, +} + +/// Generic API response +#[derive(Debug, Clone, Serialize)] +pub struct WorkerApiResponse { + pub success: bool, + pub message: String, + + #[serde(skip_serializing_if = "Option::is_none")] + pub worker: Option, +} + +/// Error response +#[derive(Debug, Clone, Serialize)] +pub struct WorkerErrorResponse { + pub error: String, + pub code: String, +} + +/// Server info response from /get_server_info endpoint +#[derive(Debug, Clone, Deserialize)] +pub struct ServerInfo { + #[serde(skip_serializing_if = "Option::is_none")] + pub model_id: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub model_path: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub priority: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub cost: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub worker_type: Option, + + // gRPC-specific + #[serde(skip_serializing_if = "Option::is_none")] + pub tokenizer_path: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning_parser: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_parser: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub chat_template: Option, +} + +/// Result from flush cache operations across workers +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct FlushCacheResult { + /// URLs of workers where cache flush succeeded + pub successful: Vec, + /// URLs and error messages for workers where cache flush failed + pub failed: Vec<(String, String)>, + /// Total number of workers attempted + pub total_workers: usize, + /// Number of HTTP workers (gRPC workers don't support flush cache) + pub http_workers: usize, + /// Human-readable summary message + pub message: String, +} + +/// Result from getting worker loads +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct WorkerLoadsResult { + /// Worker URL and load pairs + pub loads: Vec, + /// Total number of workers + pub total_workers: usize, + /// Number of workers with successful load fetches + pub successful: usize, + /// Number of workers with failed load fetches + pub failed: usize, +} + +/// Individual worker load information +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct WorkerLoadInfo { + /// Worker URL + pub worker: String, + /// Worker type (regular, prefill, decode) + #[serde(skip_serializing_if = "Option::is_none")] + pub worker_type: Option, + /// Current load (-1 indicates failure to fetch) + pub load: isize, +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/README.md new file mode 100644 index 00000000..5db6f11f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/README.md @@ -0,0 +1,474 @@ +# Reasoning Parser Architecture + +## 1. Executive Summary + +### High-Level Overview + +The reasoning parser layer provides a unified interface for detecting and extracting reasoning content from Large Language Model (LLM) outputs, particularly from models that support Chain-of-Thought (CoT) reasoning with explicit thinking blocks. The architecture follows a trait-based design pattern enabling pluggable parser implementations while maintaining consistent APIs across different model families that use various reasoning token formats. + +**Key Components:** +- **Factory Pattern**: Registry-based creation and pooling of model-specific parsers +- **Trait System**: `ReasoningParser` trait for implementation flexibility +- **Parser Pooling**: Efficient reuse of parser instances across concurrent requests +- **Streaming Support**: Incremental parsing with partial token buffering +- **Model Detection**: Pattern-based matching for automatic parser selection +- **State Management**: Stateful parsing for streaming scenarios with buffer management +- **Thread Safety**: Arc based sharing for high-concurrency environments +- **Extensibility**: Easy addition of new model-specific parsers + +**Data Flow:** +1. Request → Factory (model detection) → Pooled Parser Retrieval +2. One-Shot: Text → Parser → ParserResult (normal + reasoning text) +3. Streaming: Chunks → Parser (stateful) → Incremental ParserResult +4. Buffer Management: Partial Tokens → Buffer → Complete Token Detection +5. Reset: Parser State → Clear Buffers → Ready for Reuse + +### Architecture Highlights + +- **Model-Specific Parsers**: DeepSeek-R1, Qwen3, Kimi, GLM45, Step3 variants +- **Parser Pooling**: Singleton instances per model type for memory efficiency +- **High Concurrency**: Mutex-protected parsers handle 1000+ req/sec +- **Buffer Overflow Protection**: Configurable max buffer size (default 64KB) +- **Partial Token Detection**: Intelligent buffering for incomplete delimiters +- **Passthrough Mode**: Graceful fallback for unknown models +- **Zero-Copy Where Possible**: Efficient string handling in hot paths + +## 2. Mermaid Diagrams + +### Component Flow Diagram + +```mermaid +graph TB + subgraph Input + R[Request] --> MID[Model ID] + end + + subgraph Factory Layer + MID --> PF[ReasoningParserFactory] + PF --> REG[ParserRegistry] + REG --> PM[Pattern Matching] + PM --> PP[Parser Pool] + end + + subgraph Parser Pool + PP --> DS[DeepSeek-R1] + PP --> QW[Qwen3] + PP --> QWT[Qwen3-Thinking] + PP --> KM[Kimi] + PP --> GL[GLM45] + PP --> S3[Step3] + PP --> PT[Passthrough] + end + + subgraph Parser Instance + DS --> BP[BaseReasoningParser] + QW --> BP + KM --> BP + GL --> BP + S3 --> BP + end + + subgraph Processing + BP --> DAP[detect_and_parse] + BP --> PSI[parse_streaming] + BP --> RST[reset] + end + + subgraph State Management + BP --> BUF[Buffer] + BP --> IR[in_reasoning flag] + BP --> STS[stripped_think_start] + end + + subgraph Output + DAP --> PR[ParserResult] + PSI --> PR + PR --> NT[normal_text] + PR --> RT[reasoning_text] + end +``` + +### Sequence Flow Diagram + +```mermaid +sequenceDiagram + participant C as Client + participant F as ReasoningParserFactory + participant R as Registry + participant P as Parser Pool + participant BP as BaseParser + participant PR as ParserResult + + C->>F: get_pooled("deepseek-r1-model") + F->>R: find_pooled_parser_for_model() + R->>R: pattern_match("deepseek-r1") + R->>P: get_pooled_parser("deepseek_r1") + + alt Parser exists in pool + P-->>F: Arc> + else Create new parser + P->>BP: new DeepSeekR1Parser() + P->>P: insert into pool + P-->>F: Arc> + end + + F-->>C: PooledParser + + C->>BP: lock().parse_reasoning_streaming_incremental() + loop streaming chunks + C->>BP: parse_reasoning_streaming_incremental(chunk) + BP->>BP: buffer.push_str(chunk) + BP->>BP: check partial tokens + + alt Complete token found + BP->>PR: create result + BP->>BP: clear buffer + BP-->>C: ParserResult + else Partial token + BP->>BP: keep buffering + BP-->>C: ParserResult::default() + end + end + + C->>BP: reset() + BP->>BP: clear buffers & flags + C->>BP: unlock() +``` + +### Class/Type Diagram + +```mermaid +classDiagram + class ReasoningParser { + <> + +detect_and_parse_reasoning(&mut self, text: &str) Result~ParserResult~ + +parse_reasoning_streaming_incremental(&mut self, text: &str) Result~ParserResult~ + +reset(&mut self) + +model_type(&self) &str + } + + class ParserResult { + +normal_text: String + +reasoning_text: String + +new(normal: String, reasoning: String) Self + +normal(text: String) Self + +reasoning(text: String) Self + +is_empty() bool + } + + class ParserConfig { + +think_start_token: String + +think_end_token: String + +stream_reasoning: bool + +max_buffer_size: usize + +initial_in_reasoning: bool + +default() Self + } + + class BaseReasoningParser { + -config: ParserConfig + -in_reasoning: bool + -buffer: String + -stripped_think_start: bool + -model_type: String + +new(config: ParserConfig) Self + +with_model_type(model: String) Self + -is_partial_token(&self, text: &str) bool + } + + class DeepSeekR1Parser { + -base: BaseReasoningParser + +new() Self + } + + class Qwen3Parser { + -base: BaseReasoningParser + +new() Self + } + + class QwenThinkingParser { + -base: BaseReasoningParser + +new() Self + } + + class KimiParser { + -base: BaseReasoningParser + +new() Self + } + + class Glm45Parser { + -base: BaseReasoningParser + +new() Self + } + + class Step3Parser { + -base: BaseReasoningParser + +new() Self + } + + class ReasoningParserFactory { + -registry: ParserRegistry + +new() Self + +get_pooled(model_id: &str) PooledParser + +create(model_id: &str) Result~Box~dyn ReasoningParser~~ + +clear_pool() + } + + class ParserRegistry { + -creators: Arc~RwLock~HashMap~~ + -pool: Arc~RwLock~HashMap~~ + -patterns: Arc~RwLock~Vec~~ + +register_parser(name: &str, creator: F) + +register_pattern(pattern: &str, parser_name: &str) + +get_pooled_parser(name: &str) Option~PooledParser~ + +find_pooled_parser_for_model(model: &str) Option~PooledParser~ + } + + ReasoningParser <|.. BaseReasoningParser + ReasoningParser <|.. DeepSeekR1Parser + ReasoningParser <|.. Qwen3Parser + ReasoningParser <|.. QwenThinkingParser + ReasoningParser <|.. KimiParser + ReasoningParser <|.. Glm45Parser + ReasoningParser <|.. Step3Parser + + DeepSeekR1Parser o-- BaseReasoningParser + Qwen3Parser o-- BaseReasoningParser + QwenThinkingParser o-- BaseReasoningParser + KimiParser o-- BaseReasoningParser + Glm45Parser o-- BaseReasoningParser + Step3Parser o-- BaseReasoningParser + + BaseReasoningParser o-- ParserConfig + ReasoningParserFactory o-- ParserRegistry + ParserRegistry o-- ReasoningParser +``` + +## 3. Module-by-Module Deep Dive + +### 3.1 mod.rs (Main Module) + +**Key Responsibilities:** +- Module organization and public API surface +- Re-exports for convenient access to core types +- Separation of concerns across submodules + +**Module Structure:** +- `factory`: Parser creation and pooling logic +- `parsers`: Concrete parser implementations +- `traits`: Core trait definitions and types + +### 3.2 traits.rs (Trait Definitions) + +**ParserResult Methods**: +- `new()`: Create with both normal and reasoning text +- `normal()`: Create with only normal text (convenience) +- `reasoning()`: Create with only reasoning text (convenience) +- `is_empty()`: Check if result contains any text + +**ReasoningParser Trait**: +- **`detect_and_parse_reasoning`**: One-shot parsing for complete text +- **`parse_reasoning_streaming_incremental`**: Stateful streaming parser +- **`reset`**: Clear state for parser reuse +- **`model_type`**: Identify parser variant for debugging + +**ParserConfig Defaults**: +- Default tokens: `` and `` +- Stream reasoning: true (immediate output) +- Max buffer: 65536 bytes (64KB) +- Initial state: false (explicit reasoning blocks) + +### 3.3 factory.rs (Parser Creation & Pooling) + +**ParserRegistry Methods**: + +1. **`register_parser`**: + - Register creator function for parser type + - Lazy instantiation when requested + - Thread-safe registration + +2. **`register_pattern`**: + - Map model ID patterns to parser names + - First-match-wins ordering + - Case-insensitive matching + +3. **`get_pooled_parser`**: + - Check pool for existing instance + - Create and pool if not present + - Return Arc for sharing + +4. **`find_pooled_parser_for_model`**: + - Pattern match against model ID + - Delegate to get_pooled_parser + - Case-insensitive comparison + +**ReasoningParserFactory Methods**: + +1. **`new()`**: + - Register all built-in parsers + - Setup model pattern mappings + - Initialize empty pool + +2. **`get_pooled`**: + - Primary API for getting parsers + - Automatic passthrough fallback + - Guaranteed non-null return + +3. **`create`**: + - Create fresh parser instance + - No pooling (for testing/isolation) + - Returns Result for error handling + +**Registered Parsers**: +- `base`: Generic configurable parser +- `deepseek_r1`: DeepSeek-R1 (initial_in_reasoning=true) +- `qwen3`: Qwen3 base model (initial_in_reasoning=false) +- `qwen3_thinking`: Qwen3 thinking variant (initial_in_reasoning=true) +- `kimi`: Kimi with Unicode tokens +- `glm45`: GLM-4.5 / GLM-4.6 parser +- `step3`: Step3 parser +- `passthrough`: No-op fallback parser + +**Model Pattern Mappings**: +``` +"deepseek-r1" → "deepseek_r1" +"qwen3-thinking" → "qwen3_thinking" +"qwen-thinking" → "qwen3_thinking" +"qwen3" → "qwen3" +"qwen" → "qwen3" +"glm45" → "glm45" +"kimi" → "kimi" +"step3" → "step3" +``` + +### 3.4 parsers/base.rs (Base Implementation) + +**Key Methods:** + +**`detect_and_parse_reasoning`**: +``` +Algorithm: +1. Check buffer overflow protection +2. Detect reasoning presence (in_reasoning OR contains start_token) +3. If no reasoning → return as normal text +4. Remove start token and trim +5. If no end token → assume truncated reasoning +6. Split on end token +7. Extract reasoning and normal portions +``` + +**`parse_reasoning_streaming_incremental`**: +``` +Algorithm: +1. Check buffer capacity +2. Append text to buffer +3. Check if buffer is partial token prefix +4. If partial → buffer and return empty +5. Strip start token if present +6. Find end token position +7. Handle based on state: + - In reasoning + end found → split and return both + - In reasoning + streaming → return accumulated reasoning + - Not in reasoning → return as normal text + - In reasoning + no end → continue buffering +``` + +**Critical Features:** + +1. **Partial Token Detection**: + - Prevents premature token matching + - Buffers incomplete delimiters + - Essential for streaming correctness + +2. **Buffer Management**: + - Overflow protection + - Accumulation for partial content + - Clear on complete token detection + +3. **State Tracking**: + - `in_reasoning`: Current parsing state + - `stripped_think_start`: Prevent double processing + - `buffer`: Accumulated partial content + + +## 4. Extensibility Guide + +### Adding a New Parser + +**Step 1: Create Parser Implementation** + +```rust +// src/reasoning_parser/parsers/mymodel.rs +use crate::reasoning_parser::parsers::BaseReasoningParser; +use crate::reasoning_parser::traits::{ParserConfig, ReasoningParser}; + +pub struct MyModelParser { + base: BaseReasoningParser, +} + +impl MyModelParser { + pub fn new() -> Self { + let config = ParserConfig { + think_start_token: "".to_string(), + think_end_token: "".to_string(), + stream_reasoning: true, + max_buffer_size: 65536, + initial_in_reasoning: false, // or true for implicit + }; + + Self { + base: BaseReasoningParser::new(config) + .with_model_type("mymodel".to_string()), + } + } +} + +impl ReasoningParser for MyModelParser { + // Delegate to base or implement custom logic + fn detect_and_parse_reasoning(&mut self, text: &str) + -> Result { + self.base.detect_and_parse_reasoning(text) + } + + // ... other trait methods +} +``` + +**Step 2: Register in Factory** + +```rust +// In factory.rs ReasoningParserFactory::new() +registry.register_parser("mymodel", || { + Box::new(MyModelParser::new()) +}); + +// Register patterns +registry.register_pattern("my-model", "mymodel"); +registry.register_pattern("mymodel", "mymodel"); +``` + +**Step 3: Export from Module** + +```rust +// In parsers/mod.rs +pub use self::mymodel::MyModelParser; + +// In reasoning_parser/mod.rs +pub use parsers::MyModelParser; +``` + +### Custom Parsing Logic + +For parsers requiring custom logic beyond configuration: + +```rust +impl ReasoningParser for CustomParser { + fn parse_reasoning_streaming_incremental(&mut self, text: &str) + -> Result { + // Custom state machine + // Custom token detection + // Custom buffering strategy + // Return appropriate ParserResult + } +} +``` diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/factory.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/factory.rs new file mode 100644 index 00000000..1f0786f0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/factory.rs @@ -0,0 +1,594 @@ +// Factory and registry for creating model-specific reasoning parsers. +// Now with parser pooling support for efficient reuse across requests. + +use std::{ + collections::HashMap, + sync::{Arc, RwLock}, +}; + +use tokio::sync::Mutex; + +use crate::reasoning_parser::{ + parsers::{ + BaseReasoningParser, DeepSeekR1Parser, Glm45Parser, KimiParser, MiniMaxParser, Qwen3Parser, + QwenThinkingParser, Step3Parser, + }, + traits::{ParseError, ParserConfig, ReasoningParser}, +}; + +/// Type alias for pooled parser instances. +/// Uses tokio::Mutex to avoid blocking the async executor. +pub type PooledParser = Arc>>; + +/// Type alias for parser creator functions. +type ParserCreator = Arc Box + Send + Sync>; + +/// Registry for model-specific parsers with pooling support. +#[derive(Clone)] +pub struct ParserRegistry { + /// Creator functions for parsers (used when pool is empty) + creators: Arc>>, + /// Pooled parser instances for reuse + pool: Arc>>, + /// Model pattern to parser name mappings + patterns: Arc>>, // (pattern, parser_name) +} + +impl ParserRegistry { + /// Create a new empty registry. + pub fn new() -> Self { + Self { + creators: Arc::new(RwLock::new(HashMap::new())), + pool: Arc::new(RwLock::new(HashMap::new())), + patterns: Arc::new(RwLock::new(Vec::new())), + } + } + + /// Register a parser creator for a given parser type. + pub fn register_parser(&self, name: &str, creator: F) + where + F: Fn() -> Box + Send + Sync + 'static, + { + let mut creators = self.creators.write().unwrap(); + creators.insert(name.to_string(), Arc::new(creator)); + } + + /// Register a model pattern to parser mapping. + /// Patterns are checked in order, first match wins. + pub fn register_pattern(&self, pattern: &str, parser_name: &str) { + let mut patterns = self.patterns.write().unwrap(); + patterns.push((pattern.to_string(), parser_name.to_string())); + } + + /// Get a pooled parser by exact name. + /// Returns a shared parser instance from the pool, creating one if needed. + pub fn get_pooled_parser(&self, name: &str) -> Option { + // First check if we have a pooled instance + { + let pool = self.pool.read().unwrap(); + if let Some(parser) = pool.get(name) { + return Some(Arc::clone(parser)); + } + } + + // If not in pool, create one and add to pool + let creators = self.creators.read().unwrap(); + if let Some(creator) = creators.get(name) { + let parser = Arc::new(Mutex::new(creator())); + + // Add to pool for future use + let mut pool = self.pool.write().unwrap(); + pool.insert(name.to_string(), Arc::clone(&parser)); + + Some(parser) + } else { + None + } + } + + /// Check if a parser with the given name is registered. + pub fn has_parser(&self, name: &str) -> bool { + let creators = self.creators.read().unwrap(); + creators.contains_key(name) + } + + /// Create a fresh parser instance by exact name (not pooled). + /// Returns a new parser instance for each call - useful for streaming where state isolation is needed. + pub fn create_parser(&self, name: &str) -> Option> { + let creators = self.creators.read().unwrap(); + creators.get(name).map(|creator| creator()) + } + + /// Find a pooled parser for a given model ID by pattern matching. + pub fn find_pooled_parser_for_model(&self, model_id: &str) -> Option { + let patterns = self.patterns.read().unwrap(); + let model_lower = model_id.to_lowercase(); + + for (pattern, parser_name) in patterns.iter() { + if model_lower.contains(&pattern.to_lowercase()) { + return self.get_pooled_parser(parser_name); + } + } + None + } + + /// Check if a parser can be created for a specific model without actually creating it. + /// Returns true if a parser is available (registered) for this model. + pub fn has_parser_for_model(&self, model_id: &str) -> bool { + let patterns = self.patterns.read().unwrap(); + let model_lower = model_id.to_lowercase(); + + for (pattern, parser_name) in patterns.iter() { + if model_lower.contains(&pattern.to_lowercase()) { + let creators = self.creators.read().unwrap(); + return creators.contains_key(parser_name); + } + } + false + } + + /// Create a fresh parser instance for a given model ID by pattern matching (not pooled). + /// Returns a new parser instance for each call - useful for streaming where state isolation is needed. + pub fn create_for_model(&self, model_id: &str) -> Option> { + let patterns = self.patterns.read().unwrap(); + let model_lower = model_id.to_lowercase(); + + for (pattern, parser_name) in patterns.iter() { + if model_lower.contains(&pattern.to_lowercase()) { + return self.create_parser(parser_name); + } + } + None + } + + /// Clear the parser pool, forcing new instances to be created. + /// Useful for testing or when parsers need to be reset globally. + pub fn clear_pool(&self) { + let mut pool = self.pool.write().unwrap(); + pool.clear(); + } +} + +impl Default for ParserRegistry { + fn default() -> Self { + Self::new() + } +} + +/// Factory for creating reasoning parsers based on model type. +#[derive(Clone)] +pub struct ParserFactory { + registry: ParserRegistry, +} + +impl ParserFactory { + /// Create a new factory with default parsers registered. + pub fn new() -> Self { + let registry = ParserRegistry::new(); + + // Register base parser + registry.register_parser("base", || { + Box::new(BaseReasoningParser::new(ParserConfig::default())) + }); + + // Register DeepSeek-R1 parser (starts with in_reasoning=true) + registry.register_parser("deepseek_r1", || Box::new(DeepSeekR1Parser::new())); + + // Register Qwen3 parser (starts with in_reasoning=false) + registry.register_parser("qwen3", || Box::new(Qwen3Parser::new())); + + // Register Qwen3-thinking parser (starts with in_reasoning=true) + registry.register_parser("qwen3_thinking", || Box::new(QwenThinkingParser::new())); + + // Register Kimi parser with Unicode tokens (starts with in_reasoning=false) + registry.register_parser("kimi", || Box::new(KimiParser::new())); + + // Register GLM45 parser (same format as Qwen3 but separate for debugging) + registry.register_parser("glm45", || Box::new(Glm45Parser::new())); + + // Register Step3 parser (same format as DeepSeek-R1 but separate for debugging) + registry.register_parser("step3", || Box::new(Step3Parser::new())); + + // Register MiniMax parser (appends token at the beginning) + registry.register_parser("minimax", || Box::new(MiniMaxParser::new())); + + // Register model patterns + registry.register_pattern("deepseek-r1", "deepseek_r1"); + registry.register_pattern("qwen3-thinking", "qwen3_thinking"); + registry.register_pattern("qwen-thinking", "qwen3_thinking"); + registry.register_pattern("qwen3", "qwen3"); + registry.register_pattern("qwen", "qwen3"); + registry.register_pattern("glm45", "glm45"); + registry.register_pattern("kimi", "kimi"); + registry.register_pattern("step3", "step3"); + registry.register_pattern("minimax", "minimax"); + registry.register_pattern("minimax-m2", "minimax"); + registry.register_pattern("mm-m2", "minimax"); + + Self { registry } + } + + /// Get a pooled parser for the given model ID. + /// Returns a shared instance that can be used concurrently. + /// Falls back to a passthrough parser if model is not recognized. + pub fn get_pooled(&self, model_id: &str) -> PooledParser { + // First try to find by pattern + if let Some(parser) = self.registry.find_pooled_parser_for_model(model_id) { + return parser; + } + + // Fall back to no-op parser (get or create passthrough in pool) + self.registry + .get_pooled_parser("passthrough") + .unwrap_or_else(|| { + // Register passthrough if not already registered + self.registry.register_parser("passthrough", || { + let config = ParserConfig { + think_start_token: "".to_string(), + think_end_token: "".to_string(), + stream_reasoning: true, + max_buffer_size: 65536, + initial_in_reasoning: false, + }; + Box::new( + BaseReasoningParser::new(config).with_model_type("passthrough".to_string()), + ) + }); + self.registry.get_pooled_parser("passthrough").unwrap() + }) + } + + /// Create a new parser instance for the given model ID. + /// Returns a fresh instance (not pooled). + /// Use this when you need an isolated parser instance. + pub fn create(&self, model_id: &str) -> Result, ParseError> { + // First try to find by pattern + if let Some(parser) = self.registry.create_for_model(model_id) { + return Ok(parser); + } + + // Fall back to no-op parser (base parser without reasoning detection) + let config = ParserConfig { + think_start_token: "".to_string(), + think_end_token: "".to_string(), + stream_reasoning: true, + max_buffer_size: 65536, + initial_in_reasoning: false, + }; + Ok(Box::new( + BaseReasoningParser::new(config).with_model_type("passthrough".to_string()), + )) + } + + /// Get the internal registry for custom registration. + pub fn registry(&self) -> &ParserRegistry { + &self.registry + } + + /// Clear the parser pool. + /// Useful for testing or when parsers need to be reset globally. + pub fn clear_pool(&self) { + self.registry.clear_pool(); + } +} + +impl Default for ParserFactory { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_factory_creates_deepseek_r1() { + let factory = ParserFactory::new(); + let parser = factory.create("deepseek-r1-distill").unwrap(); + assert_eq!(parser.model_type(), "deepseek_r1"); + } + + #[test] + fn test_factory_creates_qwen3() { + let factory = ParserFactory::new(); + let parser = factory.create("qwen3-7b").unwrap(); + assert_eq!(parser.model_type(), "qwen3"); + } + + #[test] + fn test_factory_creates_kimi() { + let factory = ParserFactory::new(); + let parser = factory.create("kimi-chat").unwrap(); + assert_eq!(parser.model_type(), "kimi"); + } + + #[test] + fn test_factory_fallback_to_passthrough() { + let factory = ParserFactory::new(); + let parser = factory.create("unknown-model").unwrap(); + assert_eq!(parser.model_type(), "passthrough"); + } + + #[test] + fn test_case_insensitive_matching() { + let factory = ParserFactory::new(); + let parser1 = factory.create("DeepSeek-R1").unwrap(); + let parser2 = factory.create("QWEN3").unwrap(); + let parser3 = factory.create("Kimi").unwrap(); + + assert_eq!(parser1.model_type(), "deepseek_r1"); + assert_eq!(parser2.model_type(), "qwen3"); + assert_eq!(parser3.model_type(), "kimi"); + } + + #[test] + fn test_step3_model() { + let factory = ParserFactory::new(); + let step3 = factory.create("step3-model").unwrap(); + assert_eq!(step3.model_type(), "step3"); + } + + #[test] + fn test_glm45_model() { + let factory = ParserFactory::new(); + let glm45 = factory.create("glm45-v2").unwrap(); + assert_eq!(glm45.model_type(), "glm45"); + } + + #[test] + fn test_minimax_model() { + let factory = ParserFactory::new(); + let minimax = factory.create("minimax-m2").unwrap(); + assert_eq!(minimax.model_type(), "minimax"); + + // Also test alternate patterns + let mm = factory.create("mm-m2-chat").unwrap(); + assert_eq!(mm.model_type(), "minimax"); + } + + #[tokio::test] + async fn test_pooled_parser_reuse() { + let factory = ParserFactory::new(); + + // Get the same parser twice - should be the same instance + let parser1 = factory.get_pooled("deepseek-r1"); + let parser2 = factory.get_pooled("deepseek-r1"); + + // Both should point to the same Arc + assert!(Arc::ptr_eq(&parser1, &parser2)); + + // Different models should get different parsers + let parser3 = factory.get_pooled("qwen3"); + assert!(!Arc::ptr_eq(&parser1, &parser3)); + } + + #[tokio::test] + async fn test_pooled_parser_concurrent_access() { + let factory = ParserFactory::new(); + let parser = factory.get_pooled("deepseek-r1"); + + // Spawn multiple async tasks that use the same parser + let mut handles = vec![]; + + for i in 0..3 { + let parser_clone = Arc::clone(&parser); + let handle = tokio::spawn(async move { + let mut parser = parser_clone.lock().await; + let input = format!("thread {} reasoninganswer", i); + let result = parser.detect_and_parse_reasoning(&input).unwrap(); + assert_eq!(result.normal_text, "answer"); + assert!(result.reasoning_text.contains("reasoning")); + }); + handles.push(handle); + } + + // Wait for all tasks to complete + for handle in handles { + handle.await.unwrap(); + } + } + + #[tokio::test] + async fn test_pool_clearing() { + let factory = ParserFactory::new(); + + // Get a pooled parser + let parser1 = factory.get_pooled("deepseek-r1"); + + // Clear the pool + factory.clear_pool(); + + // Get another parser - should be a new instance + let parser2 = factory.get_pooled("deepseek-r1"); + + // They should be different instances (different Arc pointers) + assert!(!Arc::ptr_eq(&parser1, &parser2)); + } + + #[tokio::test] + async fn test_passthrough_parser_pooling() { + let factory = ParserFactory::new(); + + // Unknown models should get passthrough parser + let parser1 = factory.get_pooled("unknown-model-1"); + let parser2 = factory.get_pooled("unknown-model-2"); + + // Both should use the same passthrough parser instance + assert!(Arc::ptr_eq(&parser1, &parser2)); + + let parser = parser1.lock().await; + assert_eq!(parser.model_type(), "passthrough"); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 8)] + async fn test_high_concurrency_parser_access() { + use std::{ + sync::atomic::{AtomicUsize, Ordering}, + time::Instant, + }; + + let factory = ParserFactory::new(); + let num_tasks = 100; + let requests_per_task = 50; + let models = vec!["deepseek-r1", "qwen3", "kimi", "qwen3-thinking"]; + + // Track successful operations + let success_count = Arc::new(AtomicUsize::new(0)); + let error_count = Arc::new(AtomicUsize::new(0)); + + let start = Instant::now(); + let mut handles = vec![]; + + for task_id in 0..num_tasks { + let factory = factory.clone(); + let models = models.clone(); + let success_count = Arc::clone(&success_count); + let error_count = Arc::clone(&error_count); + + let handle = tokio::spawn(async move { + for request_id in 0..requests_per_task { + // Rotate through different models + let model = &models[(task_id + request_id) % models.len()]; + let parser = factory.get_pooled(model); + + // Use async lock - tokio::Mutex doesn't poison + let mut p = parser.lock().await; + + // Simulate realistic parsing work with substantial text + // Typical reasoning can be 500-5000 tokens + let reasoning_text = format!( + "Task {} is processing request {}. Let me think through this step by step. \ + First, I need to understand the problem. The problem involves analyzing data \ + and making calculations. Let me break this down: \n\ + 1. Initial analysis shows that we have multiple variables to consider. \ + 2. The data suggests a pattern that needs further investigation. \ + 3. Computing the values: {} * {} = {}. \ + 4. Cross-referencing with previous results indicates consistency. \ + 5. The mathematical proof follows from the axioms... \ + 6. Considering edge cases and boundary conditions... \ + 7. Validating against known constraints... \ + 8. The conclusion follows logically from premises A, B, and C. \ + This reasoning chain demonstrates the validity of our approach.", + task_id, request_id, task_id, request_id, task_id * request_id + ); + + let answer_text = format!( + "Based on my analysis, the answer for task {} request {} is: \ + The solution involves multiple steps as outlined in the reasoning. \ + The final result is {} with confidence level high. \ + This conclusion is supported by rigorous mathematical analysis \ + and has been validated against multiple test cases. \ + The implementation should handle edge cases appropriately.", + task_id, + request_id, + task_id * request_id + ); + + let input = format!("{}{}", reasoning_text, answer_text); + + match p.detect_and_parse_reasoning(&input) { + Ok(result) => { + // Note: Some parsers with stream_reasoning=true won't accumulate reasoning text + assert!(result.normal_text.contains(&format!("task {}", task_id))); + + // For parsers that accumulate reasoning (stream_reasoning=false) + // the reasoning_text should be populated + if !result.reasoning_text.is_empty() { + assert!(result + .reasoning_text + .contains(&format!("Task {}", task_id))); + assert!(result.reasoning_text.len() > 500); // Ensure substantial reasoning + } + + // Normal text should always be present + assert!(result.normal_text.len() > 100); // Ensure substantial answer + success_count.fetch_add(1, Ordering::Relaxed); + } + Err(e) => { + eprintln!("Parse error: {:?}", e); + error_count.fetch_add(1, Ordering::Relaxed); + } + } + + // Explicitly drop the lock to release it quickly + drop(p); + } + }); + handles.push(handle); + } + + // Wait for all tasks + for handle in handles { + handle.await.unwrap(); + } + + let duration = start.elapsed(); + let total_requests = num_tasks * requests_per_task; + let successes = success_count.load(Ordering::Relaxed); + let errors = error_count.load(Ordering::Relaxed); + + // Print stats for debugging + println!( + "High concurrency test: {} tasks, {} requests each", + num_tasks, requests_per_task + ); + println!( + "Completed in {:?}, {} successes, {} errors", + duration, successes, errors + ); + println!( + "Throughput: {:.0} requests/sec", + (total_requests as f64) / duration.as_secs_f64() + ); + + // All requests should succeed + assert_eq!(successes, total_requests); + assert_eq!(errors, 0); + + // Performance check: should handle at least 1000 req/sec + let throughput = (total_requests as f64) / duration.as_secs_f64(); + assert!( + throughput > 1000.0, + "Throughput too low: {:.0} req/sec", + throughput + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn test_concurrent_pool_modifications() { + let factory = ParserFactory::new(); + let mut handles = vec![]; + + // Task 1: Continuously get parsers + let factory1 = factory.clone(); + handles.push(tokio::spawn(async move { + for _ in 0..100 { + let _parser = factory1.get_pooled("deepseek-r1"); + } + })); + + // Task 2: Continuously clear pool + let factory2 = factory.clone(); + handles.push(tokio::spawn(async move { + for _ in 0..10 { + factory2.clear_pool(); + tokio::time::sleep(tokio::time::Duration::from_micros(100)).await; + } + })); + + // Task 3: Get different parsers + let factory3 = factory.clone(); + handles.push(tokio::spawn(async move { + for i in 0..100 { + let models = ["qwen3", "kimi", "unknown"]; + let _parser = factory3.get_pooled(models[i % 3]); + } + })); + + // Wait for all tasks - should not deadlock or panic + for handle in handles { + handle.await.unwrap(); + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/mod.rs new file mode 100644 index 00000000..0f5aaa18 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/mod.rs @@ -0,0 +1,10 @@ +pub mod factory; +pub mod parsers; +pub mod traits; + +pub use factory::{ParserFactory, ParserRegistry, PooledParser}; +pub use parsers::{ + BaseReasoningParser, DeepSeekR1Parser, Glm45Parser, KimiParser, MiniMaxParser, Qwen3Parser, + QwenThinkingParser, Step3Parser, +}; +pub use traits::{ParseError, ParserConfig, ParserResult, ReasoningParser}; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/base.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/base.rs new file mode 100644 index 00000000..99e94c8c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/base.rs @@ -0,0 +1,362 @@ +// Base implementation of reasoning parser that handles common logic +// for detecting and extracting reasoning blocks from text. + +use crate::reasoning_parser::traits::{ParseError, ParserConfig, ParserResult, ReasoningParser}; + +/// Base reasoning parser implementation. +/// +/// This parser handles the common logic for detecting reasoning blocks +/// delimited by start and end tokens (e.g., and ). +#[derive(Debug, Clone)] +pub struct BaseReasoningParser { + config: ParserConfig, + in_reasoning: bool, + buffer: String, + stripped_think_start: bool, + model_type: String, +} + +impl BaseReasoningParser { + /// Create a new BaseReasoningParser with the given configuration. + pub fn new(config: ParserConfig) -> Self { + let in_reasoning = config.initial_in_reasoning; + Self { + config, + in_reasoning, + buffer: String::new(), + stripped_think_start: false, + model_type: "base".to_string(), + } + } + + /// Create with custom model type identifier. + pub fn with_model_type(mut self, model_type: String) -> Self { + self.model_type = model_type; + self + } + + /// Check if the current buffer is a prefix of one of the tokens. + fn is_partial_token(&self, text: &str) -> bool { + (self.config.think_start_token.starts_with(text) && self.config.think_start_token != text) + || (self.config.think_end_token.starts_with(text) + && self.config.think_end_token != text) + } +} + +impl ReasoningParser for BaseReasoningParser { + fn detect_and_parse_reasoning(&mut self, text: &str) -> Result { + // Check input size against buffer limit + if text.len() > self.config.max_buffer_size { + return Err(ParseError::BufferOverflow(text.len())); + } + + let in_reasoning = self.in_reasoning || text.contains(&self.config.think_start_token); + + if !in_reasoning { + return Ok(ParserResult::normal(text.to_string())); + } + + // The text is considered to be in a reasoning block. + let processed_text = text + .replace(&self.config.think_start_token, "") + .trim() + .to_string(); + + if !processed_text.contains(&self.config.think_end_token) { + // Assume reasoning was truncated before end token + return Ok(ParserResult::reasoning(processed_text)); + } + + // Extract reasoning content + let splits: Vec<&str> = processed_text + .splitn(2, &self.config.think_end_token) + .collect(); + let reasoning_text = splits.first().unwrap_or(&"").to_string(); + let normal_text = splits + .get(1) + .map(|s| s.trim().to_string()) + .unwrap_or_default(); + + Ok(ParserResult::new(normal_text, reasoning_text)) + } + + fn parse_reasoning_streaming_incremental( + &mut self, + text: &str, + ) -> Result { + // Check if adding this text would exceed buffer limit + if self.buffer.len() + text.len() > self.config.max_buffer_size { + return Err(ParseError::BufferOverflow(self.buffer.len() + text.len())); + } + + // Incrementally parse the streaming text + self.buffer.push_str(text); + let mut current_text = self.buffer.clone(); + + // If the current text is a prefix of a token, keep buffering + if self.is_partial_token(¤t_text) { + return Ok(ParserResult::default()); + } + + // Strip start token if present + if !self.stripped_think_start && current_text.contains(&self.config.think_start_token) { + current_text = current_text.replace(&self.config.think_start_token, ""); + self.buffer = current_text.clone(); + self.stripped_think_start = true; + self.in_reasoning = true; + } + + // Handle end of reasoning block + let think_end_idx = if self.in_reasoning { + current_text + .find(&self.config.think_end_token) + .unwrap_or(current_text.len()) + } else { + current_text.len() + }; + + if self.in_reasoning && think_end_idx < current_text.len() { + let reasoning_text = ¤t_text[..think_end_idx]; + self.buffer.clear(); + self.in_reasoning = false; + let start_idx = think_end_idx + self.config.think_end_token.len(); + let normal_text = if start_idx < current_text.len() { + ¤t_text[start_idx..] + } else { + "" + }; + return Ok(ParserResult::new( + normal_text.to_string(), + reasoning_text.trim().to_string(), + )); + } + + // Continue with reasoning content + if self.in_reasoning && self.config.stream_reasoning { + // Stream the content immediately + let reasoning_text = current_text; + self.buffer.clear(); + Ok(ParserResult::reasoning(reasoning_text)) + } else if !self.in_reasoning { + // If we're not in a reasoning block, return as normal text + // CRITICAL FIX: Return current_text (with buffer) not just text + // This prevents buffer loss when partial tokens are followed by normal text + let normal_text = current_text; + self.buffer.clear(); + Ok(ParserResult::normal(normal_text)) + } else { + // If we are in a reasoning block but no end token is found, buffer it + Ok(ParserResult::default()) + } + } + + fn reset(&mut self) { + self.in_reasoning = self.config.initial_in_reasoning; + self.buffer.clear(); + self.stripped_think_start = false; + } + + fn model_type(&self) -> &str { + &self.model_type + } + + fn is_in_reasoning(&self) -> bool { + self.in_reasoning + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn create_test_parser( + initial_in_reasoning: bool, + stream_reasoning: bool, + ) -> BaseReasoningParser { + let config = ParserConfig { + think_start_token: "".to_string(), + think_end_token: "".to_string(), + stream_reasoning, + max_buffer_size: 65536, + initial_in_reasoning, + }; + BaseReasoningParser::new(config) + } + + #[test] + fn test_detect_and_parse_reasoning() { + let mut parser = create_test_parser(false, true); + let result = parser + .detect_and_parse_reasoning("with reasoning and more text.") + .unwrap(); + assert_eq!(result.normal_text, "and more text."); + assert_eq!(result.reasoning_text, "with reasoning"); + } + + #[test] + fn test_detect_and_parse_no_reasoning() { + let mut parser = create_test_parser(false, true); + let result = parser + .detect_and_parse_reasoning("This is a test without reasoning.") + .unwrap(); + assert_eq!(result.normal_text, "This is a test without reasoning."); + assert_eq!(result.reasoning_text, ""); + } + + #[test] + fn test_detect_and_parse_truncated_reasoning() { + let mut parser = create_test_parser(false, true); + let result = parser + .detect_and_parse_reasoning("with truncated reasoning") + .unwrap(); + assert_eq!(result.normal_text, ""); + assert_eq!(result.reasoning_text, "with truncated reasoning"); + } + + #[test] + fn test_parse_streaming_partial_token() { + let mut parser = create_test_parser(false, true); + let result = parser + .parse_reasoning_streaming_incremental("with reasoning and more text.") + .unwrap(); + assert_eq!(result.normal_text, " and more text."); + assert_eq!(result.reasoning_text, "with reasoning"); + } + + #[test] + fn test_parse_streaming_no_end_token() { + let mut parser = create_test_parser(true, true); + let result = parser + .parse_reasoning_streaming_incremental("with reasoning") + .unwrap(); + assert_eq!(result.normal_text, ""); + assert_eq!(result.reasoning_text, "with reasoning"); + } + + #[test] + fn test_initial_in_reasoning_true() { + // Parser starts with in_reasoning=true (like DeepSeek-R1) + let mut parser = create_test_parser(true, true); + let result = parser + .detect_and_parse_reasoning("no think tags here") + .unwrap(); + assert_eq!(result.normal_text, ""); + assert_eq!(result.reasoning_text, "no think tags here"); + } + + #[test] + fn test_buffer_loss_bug_fix() { + // Critical test for buffer preservation + let mut parser = create_test_parser(false, true); + + // Step 1: Send partial end tag when not in reasoning mode + let result1 = parser.parse_reasoning_streaming_incremental("reasoning ") + .unwrap(); + assert_eq!(result1.normal_text, ""); + assert_eq!(result1.reasoning_text, "reasoning "); + + // Continue streaming reasoning + let result2 = parser + .parse_reasoning_streaming_incremental("content ") + .unwrap(); + assert_eq!(result2.normal_text, ""); + assert_eq!(result2.reasoning_text, "content "); + + // End reasoning block + let result3 = parser + .parse_reasoning_streaming_incremental("more normal") + .unwrap(); + assert_eq!(result3.normal_text, " normal"); + assert_eq!(result3.reasoning_text, "more"); + } + + #[test] + fn test_reset_state() { + let mut parser = create_test_parser(false, true); + + // Process some text + parser + .parse_reasoning_streaming_incremental("reasoning normal") + .unwrap(); + + // Reset and verify state + parser.reset(); + assert!(!parser.in_reasoning); + assert!(parser.buffer.is_empty()); + assert!(!parser.stripped_think_start); + } + + #[test] + fn test_buffer_overflow_detect_and_parse() { + let config = ParserConfig { + max_buffer_size: 10, // Set a very small buffer + ..Default::default() + }; + let mut parser = BaseReasoningParser::new(config); + + let large_text = "a".repeat(20); + let result = parser.detect_and_parse_reasoning(&large_text); + + assert!(result.is_err()); + match result { + Err(ParseError::BufferOverflow(size)) => { + assert_eq!(size, 20); + } + _ => panic!("Expected BufferOverflow error"), + } + } + + #[test] + fn test_buffer_overflow_streaming() { + let config = ParserConfig { + max_buffer_size: 10, // Set a very small buffer + ..Default::default() + }; + let mut parser = BaseReasoningParser::new(config); + + // Send a partial token that will be buffered + let result1 = parser.parse_reasoning_streaming_incremental(" { + assert_eq!(size, 21); // 4 + 17 + } + _ => panic!("Expected BufferOverflow error"), + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/deepseek_r1.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/deepseek_r1.rs new file mode 100644 index 00000000..4982cecc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/deepseek_r1.rs @@ -0,0 +1,118 @@ +// DeepSeek-R1 specific reasoning parser. +// This parser starts with in_reasoning=true, assuming all text is reasoning +// until an end token is encountered. + +use crate::reasoning_parser::{ + parsers::BaseReasoningParser, + traits::{ParseError, ParserConfig, ParserResult, ReasoningParser}, +}; + +/// DeepSeek-R1 reasoning parser. +/// +/// This parser assumes reasoning from the start of text (in_reasoning=true) +/// and uses and tokens. +pub struct DeepSeekR1Parser { + base: BaseReasoningParser, +} + +impl DeepSeekR1Parser { + /// Create a new DeepSeek-R1 parser. + pub fn new() -> Self { + let config = ParserConfig { + think_start_token: "".to_string(), + think_end_token: "".to_string(), + stream_reasoning: true, + max_buffer_size: 65536, + initial_in_reasoning: true, // Always starts with reasoning + }; + + Self { + base: BaseReasoningParser::new(config).with_model_type("deepseek_r1".to_string()), + } + } +} + +impl Default for DeepSeekR1Parser { + fn default() -> Self { + Self::new() + } +} + +impl ReasoningParser for DeepSeekR1Parser { + fn detect_and_parse_reasoning(&mut self, text: &str) -> Result { + self.base.detect_and_parse_reasoning(text) + } + + fn parse_reasoning_streaming_incremental( + &mut self, + text: &str, + ) -> Result { + self.base.parse_reasoning_streaming_incremental(text) + } + + fn reset(&mut self) { + self.base.reset() + } + + fn model_type(&self) -> &str { + self.base.model_type() + } + + fn is_in_reasoning(&self) -> bool { + self.base.is_in_reasoning() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_deepseek_r1_initial_state() { + let mut parser = DeepSeekR1Parser::new(); + + // Should treat text as reasoning even without start token + let result = parser + .detect_and_parse_reasoning("This is reasoning content") + .unwrap(); + assert_eq!(result.normal_text, ""); + assert_eq!(result.reasoning_text, "This is reasoning content"); + } + + #[test] + fn test_deepseek_r1_with_end_token() { + let mut parser = DeepSeekR1Parser::new(); + + // Should extract reasoning until end token + let result = parser + .detect_and_parse_reasoning("reasoning contentnormal content") + .unwrap(); + assert_eq!(result.normal_text, "normal content"); + assert_eq!(result.reasoning_text, "reasoning content"); + } + + #[test] + fn test_deepseek_r1_streaming() { + let mut parser = DeepSeekR1Parser::new(); + + // First chunk - all reasoning + let result1 = parser + .parse_reasoning_streaming_incremental("thinking about") + .unwrap(); + assert_eq!(result1.reasoning_text, "thinking about"); + assert_eq!(result1.normal_text, ""); + + // Second chunk - ends reasoning + let result2 = parser + .parse_reasoning_streaming_incremental(" the problemanswer") + .unwrap(); + assert_eq!(result2.reasoning_text, "the problem"); // Text is trimmed + assert_eq!(result2.normal_text, "answer"); + } + + #[test] + fn test_model_type() { + let parser = DeepSeekR1Parser::new(); + assert_eq!(parser.model_type(), "deepseek_r1"); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/glm45.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/glm45.rs new file mode 100644 index 00000000..f21124d1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/glm45.rs @@ -0,0 +1,124 @@ +// GLM45 specific reasoning parser. +// Uses the same format as Qwen3 but has its own implementation for debugging. + +use crate::reasoning_parser::{ + parsers::BaseReasoningParser, + traits::{ParseError, ParserConfig, ParserResult, ReasoningParser}, +}; + +/// GLM45 reasoning parser. +/// +/// This parser uses the same format as Qwen3 (...) but has +/// its own implementation for better debugging and potential future customization. +pub struct Glm45Parser { + base: BaseReasoningParser, +} + +impl Glm45Parser { + /// Create a new GLM45 parser. + pub fn new() -> Self { + let config = ParserConfig { + think_start_token: "".to_string(), + think_end_token: "".to_string(), + stream_reasoning: true, + max_buffer_size: 65536, + initial_in_reasoning: false, // Requires explicit start token like Qwen3 + }; + + Self { + base: BaseReasoningParser::new(config).with_model_type("glm45".to_string()), + } + } +} + +impl Default for Glm45Parser { + fn default() -> Self { + Self::new() + } +} + +impl ReasoningParser for Glm45Parser { + fn detect_and_parse_reasoning(&mut self, text: &str) -> Result { + self.base.detect_and_parse_reasoning(text) + } + + fn parse_reasoning_streaming_incremental( + &mut self, + text: &str, + ) -> Result { + self.base.parse_reasoning_streaming_incremental(text) + } + + fn reset(&mut self) { + self.base.reset() + } + + fn model_type(&self) -> &str { + self.base.model_type() + } + + fn is_in_reasoning(&self) -> bool { + self.base.is_in_reasoning() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_glm45_initial_state() { + let mut parser = Glm45Parser::new(); + + // Should NOT treat text as reasoning without start token + let result = parser + .detect_and_parse_reasoning("This is normal content") + .unwrap(); + assert_eq!(result.normal_text, "This is normal content"); + assert_eq!(result.reasoning_text, ""); + } + + #[test] + fn test_glm45_with_tokens() { + let mut parser = Glm45Parser::new(); + + // Should extract reasoning with proper tokens + let result = parser + .detect_and_parse_reasoning("reasoning contentanswer") + .unwrap(); + assert_eq!(result.normal_text, "answer"); + assert_eq!(result.reasoning_text, "reasoning content"); + } + + #[test] + fn test_glm45_streaming() { + let mut parser = Glm45Parser::new(); + + // First chunk - normal text + let result1 = parser + .parse_reasoning_streaming_incremental("normal text ") + .unwrap(); + assert_eq!(result1.normal_text, "normal text "); + assert_eq!(result1.reasoning_text, ""); + + // Second chunk - enters reasoning + let result2 = parser + .parse_reasoning_streaming_incremental("reasoning") + .unwrap(); + assert_eq!(result2.normal_text, ""); + assert_eq!(result2.reasoning_text, "reasoning"); + + // Third chunk - exits reasoning + let result3 = parser + .parse_reasoning_streaming_incremental("answer") + .unwrap(); + assert_eq!(result3.normal_text, "answer"); + assert_eq!(result3.reasoning_text, ""); + } + + #[test] + fn test_model_type() { + let parser = Glm45Parser::new(); + assert_eq!(parser.model_type(), "glm45"); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/kimi.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/kimi.rs new file mode 100644 index 00000000..99827317 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/kimi.rs @@ -0,0 +1,142 @@ +// Kimi specific reasoning parser. +// This parser uses Unicode tokens and starts with in_reasoning=false. + +use crate::reasoning_parser::{ + parsers::BaseReasoningParser, + traits::{ParseError, ParserConfig, ParserResult, ReasoningParser}, +}; + +/// Kimi reasoning parser. +/// +/// This parser uses Unicode tokens (◁think▷ and ◁/think▷) and requires +/// explicit start tokens to enter reasoning mode. +pub struct KimiParser { + base: BaseReasoningParser, +} + +impl KimiParser { + /// Create a new Kimi parser. + pub fn new() -> Self { + let config = ParserConfig { + think_start_token: "◁think▷".to_string(), + think_end_token: "◁/think▷".to_string(), + stream_reasoning: true, + max_buffer_size: 65536, + initial_in_reasoning: false, // Requires explicit start token + }; + + Self { + base: BaseReasoningParser::new(config).with_model_type("kimi".to_string()), + } + } +} + +impl Default for KimiParser { + fn default() -> Self { + Self::new() + } +} + +impl ReasoningParser for KimiParser { + fn detect_and_parse_reasoning(&mut self, text: &str) -> Result { + self.base.detect_and_parse_reasoning(text) + } + + fn parse_reasoning_streaming_incremental( + &mut self, + text: &str, + ) -> Result { + self.base.parse_reasoning_streaming_incremental(text) + } + + fn reset(&mut self) { + self.base.reset() + } + + fn model_type(&self) -> &str { + self.base.model_type() + } + + fn is_in_reasoning(&self) -> bool { + self.base.is_in_reasoning() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_kimi_initial_state() { + let mut parser = KimiParser::new(); + + // Should NOT treat text as reasoning without start token + let result = parser + .detect_and_parse_reasoning("This is normal content") + .unwrap(); + assert_eq!(result.normal_text, "This is normal content"); + assert_eq!(result.reasoning_text, ""); + } + + #[test] + fn test_kimi_with_unicode_tokens() { + let mut parser = KimiParser::new(); + + // Should extract reasoning with Unicode tokens + let result = parser + .detect_and_parse_reasoning("◁think▷reasoning content◁/think▷answer") + .unwrap(); + assert_eq!(result.normal_text, "answer"); + assert_eq!(result.reasoning_text, "reasoning content"); + } + + #[test] + fn test_kimi_partial_unicode() { + let mut parser = KimiParser::new(); + + let result1 = parser + .parse_reasoning_streaming_incremental("◁thi") + .unwrap(); + assert_eq!(result1.normal_text, ""); + assert_eq!(result1.reasoning_text, ""); + + // Complete the token + let result2 = parser + .parse_reasoning_streaming_incremental("nk▷reasoning") + .unwrap(); + assert_eq!(result2.normal_text, ""); + assert_eq!(result2.reasoning_text, "reasoning"); + } + + #[test] + fn test_kimi_streaming() { + let mut parser = KimiParser::new(); + + // Normal text first + let result1 = parser + .parse_reasoning_streaming_incremental("normal ") + .unwrap(); + assert_eq!(result1.normal_text, "normal "); + assert_eq!(result1.reasoning_text, ""); + + // Enter reasoning with Unicode token + let result2 = parser + .parse_reasoning_streaming_incremental("◁think▷thinking") + .unwrap(); + assert_eq!(result2.normal_text, ""); + assert_eq!(result2.reasoning_text, "thinking"); + + // Exit reasoning + let result3 = parser + .parse_reasoning_streaming_incremental("◁/think▷answer") + .unwrap(); + assert_eq!(result3.normal_text, "answer"); + assert_eq!(result3.reasoning_text, ""); // Already returned in stream mode + } + + #[test] + fn test_model_type() { + let parser = KimiParser::new(); + assert_eq!(parser.model_type(), "kimi"); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/minimax.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/minimax.rs new file mode 100644 index 00000000..c59ed910 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/minimax.rs @@ -0,0 +1,166 @@ +// MiniMax M2 specific reasoning parser. +// This parser automatically appends token at the beginning of text, +// similar to the Python MiniMaxAppendThinkDetector. + +use crate::reasoning_parser::{ + parsers::BaseReasoningParser, + traits::{ParseError, ParserConfig, ParserResult, ReasoningParser}, +}; + +/// MiniMax M2 reasoning parser. +/// +/// This parser automatically appends token at the beginning of the first chunk +/// and uses and tokens for reasoning blocks. +pub struct MiniMaxParser { + base: BaseReasoningParser, + is_first_chunk: bool, +} + +impl MiniMaxParser { + /// Create a new MiniMax M2 parser. + pub fn new() -> Self { + let config = ParserConfig { + think_start_token: "".to_string(), + think_end_token: "".to_string(), + stream_reasoning: true, + max_buffer_size: 65536, + initial_in_reasoning: false, // Start with false, we'll add manually + }; + + Self { + base: BaseReasoningParser::new(config).with_model_type("minimax".to_string()), + is_first_chunk: true, + } + } +} + +impl Default for MiniMaxParser { + fn default() -> Self { + Self::new() + } +} + +impl ReasoningParser for MiniMaxParser { + fn detect_and_parse_reasoning(&mut self, text: &str) -> Result { + // For one-shot parsing, prepend token to the text + let modified_text = format!("{}", text); + self.base.detect_and_parse_reasoning(&modified_text) + } + + fn parse_reasoning_streaming_incremental( + &mut self, + text: &str, + ) -> Result { + // For the first chunk, prepend token + let modified_text = if self.is_first_chunk { + self.is_first_chunk = false; + format!("{}", text) + } else { + text.to_string() + }; + + self.base + .parse_reasoning_streaming_incremental(&modified_text) + } + + fn reset(&mut self) { + self.base.reset(); + self.is_first_chunk = true; // Reset the first chunk flag + } + + fn model_type(&self) -> &str { + self.base.model_type() + } + + fn is_in_reasoning(&self) -> bool { + self.base.is_in_reasoning() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_minimax_append_think_oneshot() { + let mut parser = MiniMaxParser::new(); + + // Should automatically prepend and parse as reasoning + let result = parser + .detect_and_parse_reasoning("reasoning contentnormal content") + .unwrap(); + assert_eq!(result.normal_text, "normal content"); + assert_eq!(result.reasoning_text, "reasoning content"); + } + + #[test] + fn test_minimax_without_end_token() { + let mut parser = MiniMaxParser::new(); + + // Should treat all content as reasoning when no end token + let result = parser + .detect_and_parse_reasoning("all reasoning content") + .unwrap(); + assert_eq!(result.normal_text, ""); + assert_eq!(result.reasoning_text, "all reasoning content"); + } + + #[test] + fn test_minimax_streaming_first_chunk() { + let mut parser = MiniMaxParser::new(); + + // First chunk should have prepended + let result1 = parser + .parse_reasoning_streaming_incremental("thinking about") + .unwrap(); + assert_eq!(result1.reasoning_text, "thinking about"); + assert_eq!(result1.normal_text, ""); + + // Second chunk should not have prepended + let result2 = parser + .parse_reasoning_streaming_incremental(" the problemanswer") + .unwrap(); + assert_eq!(result2.reasoning_text, "the problem"); // Text is trimmed + assert_eq!(result2.normal_text, "answer"); + } + + #[test] + fn test_minimax_reset() { + let mut parser = MiniMaxParser::new(); + + // First use + let result1 = parser + .parse_reasoning_streaming_incremental("first") + .unwrap(); + assert_eq!(result1.reasoning_text, "first"); + + // Reset the parser + parser.reset(); + + // After reset, should be first chunk again + let result2 = parser + .parse_reasoning_streaming_incremental("second") + .unwrap(); + assert_eq!(result2.reasoning_text, "second"); + } + + #[test] + fn test_minimax_already_has_think() { + let mut parser = MiniMaxParser::new(); + + // Even if text already has , it will add another one + // This mimics the Python behavior + let result = parser + .detect_and_parse_reasoning("contentanswer") + .unwrap(); + // The double gets handled by the base parser which removes duplicates + assert_eq!(result.normal_text, "answer"); + assert_eq!(result.reasoning_text, "content"); + } + + #[test] + fn test_model_type() { + let parser = MiniMaxParser::new(); + assert_eq!(parser.model_type(), "minimax"); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/mod.rs new file mode 100644 index 00000000..3da78272 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/mod.rs @@ -0,0 +1,15 @@ +pub mod base; +pub mod deepseek_r1; +pub mod glm45; +pub mod kimi; +pub mod minimax; +pub mod qwen3; +pub mod step3; + +pub use base::BaseReasoningParser; +pub use deepseek_r1::DeepSeekR1Parser; +pub use glm45::Glm45Parser; +pub use kimi::KimiParser; +pub use minimax::MiniMaxParser; +pub use qwen3::{Qwen3Parser, QwenThinkingParser}; +pub use step3::Step3Parser; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/qwen3.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/qwen3.rs new file mode 100644 index 00000000..3233808f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/qwen3.rs @@ -0,0 +1,188 @@ +// Qwen3 specific reasoning parser. +// This parser starts with in_reasoning=false, requiring an explicit +// start token to enter reasoning mode. + +use crate::reasoning_parser::{ + parsers::BaseReasoningParser, + traits::{ParseError, ParserConfig, ParserResult, ReasoningParser}, +}; + +/// Qwen3 reasoning parser. +/// +/// This parser requires explicit tokens to enter reasoning mode +/// (in_reasoning=false initially). +pub struct Qwen3Parser { + base: BaseReasoningParser, +} + +impl Qwen3Parser { + /// Create a new Qwen3 parser. + pub fn new() -> Self { + let config = ParserConfig { + think_start_token: "".to_string(), + think_end_token: "".to_string(), + stream_reasoning: true, + max_buffer_size: 65536, + initial_in_reasoning: false, // Requires explicit start token + }; + + Self { + base: BaseReasoningParser::new(config).with_model_type("qwen3".to_string()), + } + } +} + +impl Default for Qwen3Parser { + fn default() -> Self { + Self::new() + } +} + +impl ReasoningParser for Qwen3Parser { + fn detect_and_parse_reasoning(&mut self, text: &str) -> Result { + self.base.detect_and_parse_reasoning(text) + } + + fn parse_reasoning_streaming_incremental( + &mut self, + text: &str, + ) -> Result { + self.base.parse_reasoning_streaming_incremental(text) + } + + fn reset(&mut self) { + self.base.reset() + } + + fn model_type(&self) -> &str { + self.base.model_type() + } + + fn is_in_reasoning(&self) -> bool { + self.base.is_in_reasoning() + } +} + +/// QwenThinking parser - variant that assumes reasoning from start. +/// +/// This is for qwen*thinking models that behave like DeepSeek-R1. +pub struct QwenThinkingParser { + base: BaseReasoningParser, +} + +impl QwenThinkingParser { + /// Create a new QwenThinking parser. + pub fn new() -> Self { + let config = ParserConfig { + think_start_token: "".to_string(), + think_end_token: "".to_string(), + stream_reasoning: true, + max_buffer_size: 65536, + initial_in_reasoning: true, // Assumes reasoning from start + }; + + Self { + base: BaseReasoningParser::new(config).with_model_type("qwen_thinking".to_string()), + } + } +} + +impl Default for QwenThinkingParser { + fn default() -> Self { + Self::new() + } +} + +impl ReasoningParser for QwenThinkingParser { + fn detect_and_parse_reasoning(&mut self, text: &str) -> Result { + self.base.detect_and_parse_reasoning(text) + } + + fn parse_reasoning_streaming_incremental( + &mut self, + text: &str, + ) -> Result { + self.base.parse_reasoning_streaming_incremental(text) + } + + fn reset(&mut self) { + self.base.reset() + } + + fn model_type(&self) -> &str { + self.base.model_type() + } + + fn is_in_reasoning(&self) -> bool { + self.base.is_in_reasoning() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_qwen3_initial_state() { + let mut parser = Qwen3Parser::new(); + + // Should NOT treat text as reasoning without start token + let result = parser + .detect_and_parse_reasoning("This is normal content") + .unwrap(); + assert_eq!(result.normal_text, "This is normal content"); + assert_eq!(result.reasoning_text, ""); + } + + #[test] + fn test_qwen3_with_tokens() { + let mut parser = Qwen3Parser::new(); + + // Should extract reasoning with proper tokens + let result = parser + .detect_and_parse_reasoning("reasoninganswer") + .unwrap(); + assert_eq!(result.normal_text, "answer"); + assert_eq!(result.reasoning_text, "reasoning"); + } + + #[test] + fn test_qwen_thinking_initial_state() { + let mut parser = QwenThinkingParser::new(); + + // Should treat text as reasoning even without start token + let result = parser + .detect_and_parse_reasoning("This is reasoning content") + .unwrap(); + assert_eq!(result.normal_text, ""); + assert_eq!(result.reasoning_text, "This is reasoning content"); + } + + #[test] + fn test_qwen3_streaming() { + let mut parser = Qwen3Parser::new(); + + // First chunk - normal text (no start token yet) + let result1 = parser + .parse_reasoning_streaming_incremental("normal text ") + .unwrap(); + assert_eq!(result1.normal_text, "normal text "); + assert_eq!(result1.reasoning_text, ""); + + // Second chunk - enters reasoning + let result2 = parser + .parse_reasoning_streaming_incremental("reasoning") + .unwrap(); + assert_eq!(result2.normal_text, ""); + assert_eq!(result2.reasoning_text, "reasoning"); + } + + #[test] + fn test_model_types() { + let qwen3 = Qwen3Parser::new(); + assert_eq!(qwen3.model_type(), "qwen3"); + + let qwen_thinking = QwenThinkingParser::new(); + assert_eq!(qwen_thinking.model_type(), "qwen_thinking"); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/step3.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/step3.rs new file mode 100644 index 00000000..de30c438 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/parsers/step3.rs @@ -0,0 +1,129 @@ +// Step3 specific reasoning parser. +// Uses the same format as DeepSeek-R1 but has its own implementation for debugging. + +use crate::reasoning_parser::{ + parsers::BaseReasoningParser, + traits::{ParseError, ParserConfig, ParserResult, ReasoningParser}, +}; + +/// Step3 reasoning parser. +/// +/// This parser uses the same format as DeepSeek-R1 (...) but has +/// its own implementation for better debugging and potential future customization. +pub struct Step3Parser { + base: BaseReasoningParser, +} + +impl Step3Parser { + /// Create a new Step3 parser. + pub fn new() -> Self { + let config = ParserConfig { + think_start_token: "".to_string(), + think_end_token: "".to_string(), + stream_reasoning: true, + max_buffer_size: 65536, + initial_in_reasoning: true, // Assumes reasoning from start like DeepSeek-R1 + }; + + Self { + base: BaseReasoningParser::new(config).with_model_type("step3".to_string()), + } + } +} + +impl Default for Step3Parser { + fn default() -> Self { + Self::new() + } +} + +impl ReasoningParser for Step3Parser { + fn detect_and_parse_reasoning(&mut self, text: &str) -> Result { + self.base.detect_and_parse_reasoning(text) + } + + fn parse_reasoning_streaming_incremental( + &mut self, + text: &str, + ) -> Result { + self.base.parse_reasoning_streaming_incremental(text) + } + + fn reset(&mut self) { + self.base.reset() + } + + fn model_type(&self) -> &str { + self.base.model_type() + } + + fn is_in_reasoning(&self) -> bool { + self.base.is_in_reasoning() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_step3_initial_state() { + let mut parser = Step3Parser::new(); + + // Should treat text as reasoning even without start token + let result = parser + .detect_and_parse_reasoning("This is reasoning content") + .unwrap(); + assert_eq!(result.normal_text, ""); + assert_eq!(result.reasoning_text, "This is reasoning content"); + } + + #[test] + fn test_step3_with_end_token() { + let mut parser = Step3Parser::new(); + + // Should handle text with end token + let result = parser + .detect_and_parse_reasoning("reasoning contentanswer") + .unwrap(); + assert_eq!(result.normal_text, "answer"); + assert_eq!(result.reasoning_text, "reasoning content"); + } + + #[test] + fn test_step3_with_both_tokens() { + let mut parser = Step3Parser::new(); + + // Should handle both start and end tokens + let result = parser + .detect_and_parse_reasoning("reasoning contentanswer") + .unwrap(); + assert_eq!(result.normal_text, "answer"); + assert_eq!(result.reasoning_text, "reasoning content"); + } + + #[test] + fn test_step3_streaming() { + let mut parser = Step3Parser::new(); + + // First chunk - treated as reasoning (initial_in_reasoning=true) + let result1 = parser + .parse_reasoning_streaming_incremental("reasoning text ") + .unwrap(); + assert_eq!(result1.normal_text, ""); + assert_eq!(result1.reasoning_text, "reasoning text "); + + // Second chunk - continues reasoning until end token + let result2 = parser + .parse_reasoning_streaming_incremental("more reasoninganswer") + .unwrap(); + assert_eq!(result2.normal_text, "answer"); + assert_eq!(result2.reasoning_text, "more reasoning"); + } + + #[test] + fn test_model_type() { + let parser = Step3Parser::new(); + assert_eq!(parser.model_type(), "step3"); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/traits.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/traits.rs new file mode 100644 index 00000000..c21e342f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/reasoning_parser/traits.rs @@ -0,0 +1,135 @@ +use std::fmt; + +/// Result of parsing text for reasoning content. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct ParserResult { + /// The normal text outside reasoning blocks. + pub normal_text: String, + + /// The extracted reasoning text from within reasoning blocks. + pub reasoning_text: String, +} + +impl ParserResult { + /// Create a new ParserResult with the given normal and reasoning text. + pub fn new(normal_text: String, reasoning_text: String) -> Self { + Self { + normal_text, + reasoning_text, + } + } + + /// Create a result with only normal text. + pub fn normal(text: String) -> Self { + Self { + normal_text: text, + reasoning_text: String::new(), + } + } + + /// Create a result with only reasoning text. + pub fn reasoning(text: String) -> Self { + Self { + normal_text: String::new(), + reasoning_text: text, + } + } + + /// Check if this result contains any text. + pub fn is_empty(&self) -> bool { + self.normal_text.is_empty() && self.reasoning_text.is_empty() + } +} + +/// Trait for parsing reasoning content from LLM outputs. +pub trait ReasoningParser: Send + Sync { + /// Detects and parses reasoning from the input text (one-time parsing). + /// + /// This method is used for non-streaming scenarios where the complete + /// text is available at once. + /// + /// Returns an error if the text exceeds buffer limits or contains invalid UTF-8. + fn detect_and_parse_reasoning(&mut self, text: &str) -> Result; + + /// Parses reasoning incrementally from streaming input. + /// + /// This method maintains internal state across calls to handle partial + /// tokens and chunk boundaries correctly. + /// + /// Returns an error if the buffer exceeds max_buffer_size. + fn parse_reasoning_streaming_incremental( + &mut self, + text: &str, + ) -> Result; + + /// Reset the parser state for reuse. + /// + /// This should clear any buffers and reset flags to initial state. + fn reset(&mut self); + + /// Get the model type this parser is designed for. + fn model_type(&self) -> &str; + + /// Check if the parser is currently in reasoning mode. + /// + /// Returns true if the parser is currently parsing reasoning content. + fn is_in_reasoning(&self) -> bool; +} + +/// Error types for reasoning parsing operations. +#[derive(Debug, thiserror::Error)] +pub enum ParseError { + #[error("Invalid UTF-8 in stream: {0}")] + Utf8Error(#[from] std::str::Utf8Error), + + #[error("Buffer overflow: {0} bytes exceeds maximum")] + BufferOverflow(usize), + + #[error("Unknown model type: {0}")] + UnknownModel(String), + + #[error("Parser configuration error: {0}")] + ConfigError(String), +} + +/// Configuration for parser behavior. +#[derive(Debug, Clone)] +pub struct ParserConfig { + /// The token that marks the start of reasoning content. + pub think_start_token: String, + + /// The token that marks the end of reasoning content. + pub think_end_token: String, + + /// Whether to stream reasoning content as it arrives. + pub stream_reasoning: bool, + + /// Maximum buffer size in bytes. + pub max_buffer_size: usize, + + /// Initial state for in_reasoning flag (fixed per parser type). + pub initial_in_reasoning: bool, +} + +impl Default for ParserConfig { + fn default() -> Self { + Self { + think_start_token: "".to_string(), + think_end_token: "".to_string(), + stream_reasoning: true, + max_buffer_size: 65536, // 64KB default + initial_in_reasoning: false, // Default to false (explicit reasoning) + } + } +} + +impl fmt::Display for ParserResult { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "ParserResult {{ normal: {} chars, reasoning: {} chars }}", + self.normal_text.len(), + self.reasoning_text.len() + ) + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/factory.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/factory.rs new file mode 100644 index 00000000..5f6c962f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/factory.rs @@ -0,0 +1,134 @@ +//! Factory for creating router instances + +use std::sync::Arc; + +use super::{ + grpc::{pd_router::GrpcPDRouter, router::GrpcRouter}, + http::{pd_router::PDRouter, router::Router}, + openai::OpenAIRouter, + RouterTrait, +}; +use crate::{ + app_context::AppContext, + config::{PolicyConfig, RoutingMode}, + core::ConnectionMode, + policies::PolicyFactory, +}; + +/// Factory for creating router instances based on configuration +pub struct RouterFactory; + +impl RouterFactory { + /// Create a router instance from application context + pub async fn create_router(ctx: &Arc) -> Result, String> { + match ctx.router_config.connection_mode { + ConnectionMode::Grpc { .. } => match &ctx.router_config.mode { + RoutingMode::Regular { .. } => Self::create_grpc_router(ctx).await, + RoutingMode::PrefillDecode { + prefill_policy, + decode_policy, + .. + } => { + Self::create_grpc_pd_router( + prefill_policy.as_ref(), + decode_policy.as_ref(), + &ctx.router_config.policy, + ctx, + ) + .await + } + RoutingMode::OpenAI { .. } => { + Err("OpenAI mode requires HTTP connection_mode".to_string()) + } + }, + ConnectionMode::Http => match &ctx.router_config.mode { + RoutingMode::Regular { .. } => Self::create_regular_router(ctx).await, + RoutingMode::PrefillDecode { + prefill_policy, + decode_policy, + .. + } => { + Self::create_pd_router( + prefill_policy.as_ref(), + decode_policy.as_ref(), + &ctx.router_config.policy, + ctx, + ) + .await + } + RoutingMode::OpenAI { worker_urls } => { + Self::create_openai_router(worker_urls.clone(), ctx).await + } + }, + } + } + + /// Create a regular router + pub async fn create_regular_router( + ctx: &Arc, + ) -> Result, String> { + let router = Router::new(ctx).await?; + + Ok(Box::new(router)) + } + + /// Create a PD router with injected policy + pub async fn create_pd_router( + prefill_policy_config: Option<&PolicyConfig>, + decode_policy_config: Option<&PolicyConfig>, + main_policy_config: &PolicyConfig, + ctx: &Arc, + ) -> Result, String> { + let prefill_policy = + PolicyFactory::create_from_config(prefill_policy_config.unwrap_or(main_policy_config)); + let decode_policy = + PolicyFactory::create_from_config(decode_policy_config.unwrap_or(main_policy_config)); + + ctx.policy_registry.set_prefill_policy(prefill_policy); + ctx.policy_registry.set_decode_policy(decode_policy); + + let router = PDRouter::new(ctx).await?; + + Ok(Box::new(router)) + } + + /// Create a gRPC router with injected policy + pub async fn create_grpc_router(ctx: &Arc) -> Result, String> { + let router = GrpcRouter::new(ctx).await?; + + Ok(Box::new(router)) + } + + /// Create a gRPC PD router with tokenizer and worker configuration + pub async fn create_grpc_pd_router( + prefill_policy_config: Option<&PolicyConfig>, + decode_policy_config: Option<&PolicyConfig>, + main_policy_config: &PolicyConfig, + ctx: &Arc, + ) -> Result, String> { + let prefill_policy = + PolicyFactory::create_from_config(prefill_policy_config.unwrap_or(main_policy_config)); + let decode_policy = + PolicyFactory::create_from_config(decode_policy_config.unwrap_or(main_policy_config)); + + ctx.policy_registry.set_prefill_policy(prefill_policy); + ctx.policy_registry.set_decode_policy(decode_policy); + let router = GrpcPDRouter::new(ctx).await?; + + Ok(Box::new(router)) + } + + /// Create an OpenAI router + async fn create_openai_router( + worker_urls: Vec, + ctx: &Arc, + ) -> Result, String> { + if worker_urls.is_empty() { + return Err("OpenAI mode requires at least one worker URL".to_string()); + } + + let router = OpenAIRouter::new(worker_urls, ctx).await?; + + Ok(Box::new(router)) + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/client.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/client.rs new file mode 100644 index 00000000..79f1e084 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/client.rs @@ -0,0 +1,177 @@ +//! Unified gRPC client wrapper for SGLang and vLLM backends + +use crate::{ + grpc_client::{SglangSchedulerClient, VllmEngineClient}, + routers::grpc::proto_wrapper::{ProtoGenerateRequest, ProtoStream}, +}; + +/// Health check response (common across backends) +#[derive(Debug, Clone)] +pub struct HealthCheckResponse { + pub healthy: bool, + pub message: String, +} + +/// Polymorphic gRPC client that wraps either SGLang or vLLM +#[derive(Clone)] +pub enum GrpcClient { + Sglang(SglangSchedulerClient), + Vllm(VllmEngineClient), +} + +impl GrpcClient { + /// Get reference to SGLang client (panics if vLLM) + pub fn as_sglang(&self) -> &SglangSchedulerClient { + match self { + Self::Sglang(client) => client, + Self::Vllm(_) => panic!("Expected SGLang client, got vLLM"), + } + } + + /// Get mutable reference to SGLang client (panics if vLLM) + pub fn as_sglang_mut(&mut self) -> &mut SglangSchedulerClient { + match self { + Self::Sglang(client) => client, + Self::Vllm(_) => panic!("Expected SGLang client, got vLLM"), + } + } + + /// Get reference to vLLM client (panics if SGLang) + pub fn as_vllm(&self) -> &VllmEngineClient { + match self { + Self::Vllm(client) => client, + Self::Sglang(_) => panic!("Expected vLLM client, got SGLang"), + } + } + + /// Get mutable reference to vLLM client (panics if SGLang) + pub fn as_vllm_mut(&mut self) -> &mut VllmEngineClient { + match self { + Self::Vllm(client) => client, + Self::Sglang(_) => panic!("Expected vLLM client, got SGLang"), + } + } + + /// Check if this is a SGLang client + pub fn is_sglang(&self) -> bool { + matches!(self, Self::Sglang(_)) + } + + /// Check if this is a vLLM client + pub fn is_vllm(&self) -> bool { + matches!(self, Self::Vllm(_)) + } + + /// Connect to gRPC server (runtime-aware) + pub async fn connect( + url: &str, + runtime_type: &str, + ) -> Result> { + match runtime_type { + "sglang" => Ok(Self::Sglang(SglangSchedulerClient::connect(url).await?)), + "vllm" => Ok(Self::Vllm(VllmEngineClient::connect(url).await?)), + _ => Err(format!("Unknown runtime type: {}", runtime_type).into()), + } + } + + /// Perform health check (dispatches to appropriate backend) + pub async fn health_check( + &self, + ) -> Result> { + match self { + Self::Sglang(client) => { + let resp = client.health_check().await?; + Ok(HealthCheckResponse { + healthy: resp.healthy, + message: resp.message, + }) + } + Self::Vllm(client) => { + let resp = client.health_check().await?; + Ok(HealthCheckResponse { + healthy: resp.healthy, + message: resp.message, + }) + } + } + } + + /// Get model info (returns enum wrapping backend-specific response) + pub async fn get_model_info( + &self, + ) -> Result> { + match self { + Self::Sglang(client) => { + let info = client.get_model_info().await?; + Ok(ModelInfo::Sglang(info)) + } + Self::Vllm(client) => { + let info = client.get_model_info().await?; + Ok(ModelInfo::Vllm(info)) + } + } + } + + /// Generate streaming response from request + /// + /// Dispatches to the appropriate backend client and wraps the result in ProtoStream + pub async fn generate( + &mut self, + req: ProtoGenerateRequest, + ) -> Result> { + match (self, req) { + (Self::Sglang(client), ProtoGenerateRequest::Sglang(boxed_req)) => { + let stream = client.generate(*boxed_req).await?; + Ok(ProtoStream::Sglang(stream)) + } + (Self::Vllm(client), ProtoGenerateRequest::Vllm(boxed_req)) => { + let stream = client.generate(*boxed_req).await?; + Ok(ProtoStream::Vllm(stream)) + } + _ => panic!("Mismatched client and request types"), + } + } +} + +/// Unified ModelInfo wrapper +pub enum ModelInfo { + Sglang(crate::grpc_client::sglang_proto::GetModelInfoResponse), + Vllm(crate::grpc_client::vllm_proto::GetModelInfoResponse), +} + +impl ModelInfo { + /// Convert model info to label map for worker metadata + pub fn to_labels(&self) -> std::collections::HashMap { + let mut labels = std::collections::HashMap::new(); + + // Serialize to JSON Value (like pydantic's model_dump) + let value = match self { + ModelInfo::Sglang(info) => serde_json::to_value(info).ok(), + ModelInfo::Vllm(info) => serde_json::to_value(info).ok(), + }; + + // Convert JSON object to HashMap, filtering out empty/zero/false values + if let Some(serde_json::Value::Object(obj)) = value { + for (key, val) in obj { + match val { + // Insert non-empty strings + serde_json::Value::String(s) if !s.is_empty() => { + labels.insert(key, s); + } + // Insert positive numbers + serde_json::Value::Number(n) if n.as_i64().unwrap_or(0) > 0 => { + labels.insert(key, n.to_string()); + } + // Insert true booleans + serde_json::Value::Bool(true) => { + labels.insert(key, "true".to_string()); + } + // Skip empty strings, zeros, false, nulls, arrays, objects + _ => {} + } + } + } + + labels + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/mod.rs new file mode 100644 index 00000000..c42ea3b0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/mod.rs @@ -0,0 +1,6 @@ +//! Shared code for both regular and harmony routers + +pub mod response_collection; +pub mod response_formatting; +pub mod responses; +pub mod stages; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/response_collection.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/response_collection.rs new file mode 100644 index 00000000..ff415871 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/response_collection.rs @@ -0,0 +1,85 @@ +//! Shared response collection logic +//! +//! This module contains common logic for collecting responses from execution results. +//! Both regular and harmony processors use these functions to avoid duplication. + +use axum::response::Response; + +use crate::routers::grpc::{ + context::ExecutionResult, error, proto_wrapper::ProtoGenerateComplete, utils, +}; + +/// Collect and merge responses from execution result +/// +/// Handles both Single and Dual (prefill-decode) execution modes. +/// For Dual mode, merges prefill input_logprobs into decode responses if requested. +/// +/// # Arguments +/// * `execution_result` - The execution result containing stream(s) +/// * `merge_logprobs` - Whether to merge prefill input_logprobs (for chat with logprobs=true) +/// +/// # Returns +/// Vector of GenerateComplete responses, one per index (n parameter) +pub async fn collect_responses( + execution_result: ExecutionResult, + merge_logprobs: bool, +) -> Result, Response> { + let all_responses = match execution_result { + ExecutionResult::Single { mut stream } => { + let responses = utils::collect_stream_responses(&mut stream, "Single").await?; + stream.mark_completed(); + responses + } + ExecutionResult::Dual { + mut prefill, + decode, + } => { + // Collect prefill for input_logprobs (don't mark completed yet) + let prefill_responses = + utils::collect_stream_responses(&mut prefill, "Prefill").await?; + + // Collect decode for actual output (don't mark completed yet) + let mut decode_stream = *decode; + let mut decode_responses = + utils::collect_stream_responses(&mut decode_stream, "Decode").await?; + + // Mark both streams as completed now that both succeeded + prefill.mark_completed(); + decode_stream.mark_completed(); + + // Merge prefill input_logprobs if requested + if merge_logprobs { + merge_prefill_logprobs(&prefill_responses, &mut decode_responses); + } + + decode_responses + } + }; + + if all_responses.is_empty() { + return Err(error::internal_error("No responses from server")); + } + + Ok(all_responses) +} + +/// Merge prefill input_logprobs into decode responses +/// +/// Takes input_logprobs from the first prefill response and copies them +/// into all decode responses. This is used in PD mode when logprobs are requested. +/// Only works with SGLang (vLLM doesn't support PD mode). +fn merge_prefill_logprobs( + prefill_responses: &[ProtoGenerateComplete], + decode_responses: &mut [ProtoGenerateComplete], +) { + // Only SGLang supports PD mode and has input_logprobs + if let Some(ProtoGenerateComplete::Sglang(prefill_first)) = prefill_responses.first() { + if let Some(prefill_input_logprobs) = prefill_first.input_logprobs.clone() { + for response in decode_responses.iter_mut() { + if let ProtoGenerateComplete::Sglang(decode_resp) = response { + decode_resp.input_logprobs = Some(prefill_input_logprobs.clone()); + } + } + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/response_formatting.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/response_formatting.rs new file mode 100644 index 00000000..7148461f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/response_formatting.rs @@ -0,0 +1,29 @@ +//! Shared response formatting logic +//! +//! This module contains common logic for formatting responses, including: +//! - Usage calculation from gRPC responses +//! - ChatCompletionResponse construction + +use crate::{protocols::common::Usage, routers::grpc::proto_wrapper::ProtoGenerateComplete}; + +/// Build usage information from collected gRPC responses +/// +/// Sums prompt_tokens and completion_tokens across all responses. +/// Typically used with n>1 parameter where multiple completions are generated. +/// +/// # Arguments +/// * `responses` - Vector of GenerateComplete responses from the backend +/// +/// # Returns +/// Usage object with aggregated token counts +pub fn build_usage(responses: &[ProtoGenerateComplete]) -> Usage { + let total_prompt_tokens: u32 = responses.iter().map(|r| r.prompt_tokens() as u32).sum(); + let total_completion_tokens: u32 = responses.iter().map(|r| r.completion_tokens() as u32).sum(); + + Usage { + prompt_tokens: total_prompt_tokens, + completion_tokens: total_completion_tokens, + total_tokens: total_prompt_tokens + total_completion_tokens, + completion_tokens_details: None, + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/responses/handlers.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/responses/handlers.rs new file mode 100644 index 00000000..4b7b8c09 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/responses/handlers.rs @@ -0,0 +1,196 @@ +//! Shared response handlers for both regular and harmony implementations +//! +//! These handlers are used by both pipelines for retrieving and cancelling responses. + +use axum::{ + http::StatusCode, + response::{IntoResponse, Response}, +}; +use serde_json::json; +use tracing::{debug, error, warn}; + +use crate::{ + data_connector::ResponseId, routers::grpc::regular::responses::context::ResponsesContext, +}; + +/// Implementation for GET /v1/responses/{response_id} +/// +/// Retrieves a stored response from the database. +/// Used by both regular and harmony implementations. +pub async fn get_response_impl(ctx: &ResponsesContext, response_id: &str) -> Response { + let resp_id = ResponseId::from(response_id); + + // Retrieve response from storage + match ctx.response_storage.get_response(&resp_id).await { + Ok(Some(stored_response)) => axum::Json(stored_response.raw_response).into_response(), + Ok(None) => ( + StatusCode::NOT_FOUND, + axum::Json(json!({ + "error": { + "message": format!("Response with id '{}' not found", response_id), + "type": "not_found_error", + "code": "response_not_found" + } + })), + ) + .into_response(), + Err(e) => ( + StatusCode::INTERNAL_SERVER_ERROR, + axum::Json(json!({ + "error": { + "message": format!("Failed to retrieve response: {}", e), + "type": "internal_error" + } + })), + ) + .into_response(), + } +} + +/// Implementation for POST /v1/responses/{response_id}/cancel +/// +/// Cancels a background response if it's still in progress. +pub async fn cancel_response_impl(ctx: &ResponsesContext, response_id: &str) -> Response { + let resp_id = ResponseId::from(response_id); + + // Retrieve response from storage to check if it exists and get current status + match ctx.response_storage.get_response(&resp_id).await { + Ok(Some(stored_response)) => { + // Check current status - only queued or in_progress responses can be cancelled + let current_status = stored_response + .raw_response + .get("status") + .and_then(|v| v.as_str()) + .unwrap_or("unknown"); + + match current_status { + "queued" | "in_progress" => { + // Attempt to abort the background task + let mut tasks = ctx.background_tasks.write().await; + if let Some(task_info) = tasks.remove(response_id) { + // Abort the Rust task immediately + task_info.handle.abort(); + + // Abort the Python/scheduler request via gRPC (if client is available) + let client_opt = task_info.client.read().await; + if let Some(ref client) = *client_opt { + if let Err(e) = client + .abort_request( + task_info.grpc_request_id.clone(), + "User cancelled via API".to_string(), + ) + .await + { + warn!( + "Failed to abort Python request {}: {}", + task_info.grpc_request_id, e + ); + } else { + debug!( + "Successfully aborted Python request: {}", + task_info.grpc_request_id + ); + } + } else { + debug!("Client not yet available for abort, request may not have started yet"); + } + + // Task was found and aborted + ( + StatusCode::OK, + axum::Json(json!({ + "id": response_id, + "status": "cancelled", + "message": "Background task has been cancelled" + })), + ) + .into_response() + } else { + // Task handle not found but status is queued/in_progress + // This can happen if: (1) task crashed, or (2) storage persistence failed + error!( + "Response {} has status '{}' but task handle is missing. Task may have crashed or storage update failed.", + response_id, current_status + ); + ( + StatusCode::INTERNAL_SERVER_ERROR, + axum::Json(json!({ + "error": { + "message": "Internal error: background task completed but failed to update status in storage", + "type": "internal_error", + "code": "status_update_failed" + } + })), + ) + .into_response() + } + } + "completed" => ( + StatusCode::BAD_REQUEST, + axum::Json(json!({ + "error": { + "message": "Cannot cancel completed response", + "type": "invalid_request_error", + "code": "response_already_completed" + } + })), + ) + .into_response(), + "failed" => ( + StatusCode::BAD_REQUEST, + axum::Json(json!({ + "error": { + "message": "Cannot cancel failed response", + "type": "invalid_request_error", + "code": "response_already_failed" + } + })), + ) + .into_response(), + "cancelled" => ( + StatusCode::OK, + axum::Json(json!({ + "id": response_id, + "status": "cancelled", + "message": "Response was already cancelled" + })), + ) + .into_response(), + _ => { + // Unknown status + ( + StatusCode::INTERNAL_SERVER_ERROR, + axum::Json(json!({ + "error": { + "message": format!("Unknown response status: {}", current_status), + "type": "internal_error" + } + })), + ) + .into_response() + } + } + } + Ok(None) => ( + StatusCode::NOT_FOUND, + axum::Json(json!({ + "error": { + "message": format!("Response with id '{}' not found", response_id), + "type": "not_found_error", + "code": "response_not_found" + } + })), + ) + .into_response(), + Err(e) => ( + StatusCode::INTERNAL_SERVER_ERROR, + axum::Json(json!({ + "error": { + "message": format!("Failed to retrieve response: {}", e), + "type": "internal_error" + } + })), + ) + .into_response(), + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/responses/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/responses/mod.rs new file mode 100644 index 00000000..5ebf6603 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/responses/mod.rs @@ -0,0 +1,9 @@ +//! Shared response functionality used by both regular and harmony implementations + +pub mod handlers; +pub mod streaming; +pub mod utils; + +pub use handlers::{cancel_response_impl, get_response_impl}; +pub use streaming::{build_sse_response, OutputItemType, ResponseStreamEventEmitter}; +pub use utils::{ensure_mcp_connection, persist_response_if_needed}; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/responses/streaming.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/responses/streaming.rs new file mode 100644 index 00000000..47bea0d9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/responses/streaming.rs @@ -0,0 +1,836 @@ +//! Streaming infrastructure for /v1/responses endpoint + +use std::collections::HashMap; + +use axum::{body::Body, http::StatusCode, response::Response}; +use bytes::Bytes; +use serde_json::json; +use tokio::sync::mpsc; +use tokio_stream::wrappers::UnboundedReceiverStream; +use uuid::Uuid; + +use crate::{ + mcp, + protocols::{ + chat::ChatCompletionStreamResponse, + common::{Usage, UsageInfo}, + responses::{ + ResponseOutputItem, ResponseStatus, ResponsesRequest, ResponsesResponse, ResponsesUsage, + }, + }, + routers::grpc::harmony::responses::ToolResult, +}; + +pub enum OutputItemType { + Message, + McpListTools, + McpCall, + FunctionCall, + Reasoning, +} + +/// Status of an output item +#[derive(Debug, Clone, PartialEq)] +enum ItemStatus { + InProgress, + Completed, +} + +/// State tracking for a single output item +#[derive(Debug, Clone)] +struct OutputItemState { + output_index: usize, + status: ItemStatus, + item_data: Option, +} + +/// OpenAI-compatible event emitter for /v1/responses streaming +/// +/// Manages state and sequence numbers to emit proper event types: +/// - response.created +/// - response.in_progress +/// - response.output_item.added +/// - response.content_part.added +/// - response.output_text.delta (multiple) +/// - response.output_text.done +/// - response.content_part.done +/// - response.output_item.done +/// - response.completed +/// - response.mcp_list_tools.in_progress +/// - response.mcp_list_tools.completed +/// - response.mcp_call.in_progress +/// - response.mcp_call_arguments.delta +/// - response.mcp_call_arguments.done +/// - response.mcp_call.completed +/// - response.mcp_call.failed +pub struct ResponseStreamEventEmitter { + sequence_number: u64, + pub response_id: String, + model: String, + created_at: u64, + message_id: String, + accumulated_text: String, + has_emitted_created: bool, + has_emitted_in_progress: bool, + has_emitted_output_item_added: bool, + has_emitted_content_part_added: bool, + // MCP call tracking + mcp_call_accumulated_args: HashMap, + pub(crate) mcp_server_label: Option, // Server label for MCP tools + // Output item tracking + output_items: Vec, + next_output_index: usize, + current_message_output_index: Option, // Tracks output_index of current message + current_item_id: Option, // Tracks item_id of current item + original_request: Option, +} + +impl ResponseStreamEventEmitter { + pub fn new(response_id: String, model: String, created_at: u64) -> Self { + let message_id = format!("msg_{}", Uuid::new_v4()); + + Self { + sequence_number: 0, + response_id, + model, + created_at, + message_id, + accumulated_text: String::new(), + has_emitted_created: false, + has_emitted_in_progress: false, + has_emitted_output_item_added: false, + has_emitted_content_part_added: false, + mcp_call_accumulated_args: HashMap::new(), + mcp_server_label: None, + output_items: Vec::new(), + next_output_index: 0, + current_message_output_index: None, + current_item_id: None, + original_request: None, + } + } + + /// Set the original request for including all fields in response.completed + pub fn set_original_request(&mut self, request: ResponsesRequest) { + self.original_request = Some(request); + } + + /// Set the MCP server label for MCP tool calls + pub fn set_mcp_server_label(&mut self, server_label: String) { + self.mcp_server_label = Some(server_label); + } + + /// Update mcp_call output items with tool execution results + /// + /// After MCP tools are executed, this updates the stored output items + /// to include the output field from the tool results. + pub(crate) fn update_mcp_call_outputs(&mut self, tool_results: &[ToolResult]) { + for tool_result in tool_results { + // Find the output item with matching call_id + for item_state in self.output_items.iter_mut() { + if let Some(ref mut item_data) = item_state.item_data { + // Check if this is an mcp_call item with matching call_id + if item_data.get("type").and_then(|t| t.as_str()) == Some("mcp_call") + && item_data.get("call_id").and_then(|c| c.as_str()) + == Some(&tool_result.call_id) + { + // Add output field + let output_str = serde_json::to_string(&tool_result.output) + .unwrap_or_else(|_| "{}".to_string()); + item_data["output"] = json!(output_str); + + // Update status based on success + if tool_result.is_error { + item_data["status"] = json!("failed"); + } + break; + } + } + } + } + } + + fn next_sequence(&mut self) -> u64 { + let seq = self.sequence_number; + self.sequence_number += 1; + seq + } + + pub fn emit_created(&mut self) -> serde_json::Value { + self.has_emitted_created = true; + json!({ + "type": "response.created", + "sequence_number": self.next_sequence(), + "response": { + "id": self.response_id, + "object": "response", + "created_at": self.created_at, + "status": "in_progress", + "model": self.model, + "output": [] + } + }) + } + + pub fn emit_in_progress(&mut self) -> serde_json::Value { + self.has_emitted_in_progress = true; + json!({ + "type": "response.in_progress", + "sequence_number": self.next_sequence(), + "response": { + "id": self.response_id, + "object": "response", + "status": "in_progress" + } + }) + } + + pub fn emit_content_part_added( + &mut self, + output_index: usize, + item_id: &str, + content_index: usize, + ) -> serde_json::Value { + self.has_emitted_content_part_added = true; + json!({ + "type": "response.content_part.added", + "sequence_number": self.next_sequence(), + "output_index": output_index, + "item_id": item_id, + "content_index": content_index, + "part": { + "type": "text", + "text": "" + } + }) + } + + pub fn emit_text_delta( + &mut self, + delta: &str, + output_index: usize, + item_id: &str, + content_index: usize, + ) -> serde_json::Value { + self.accumulated_text.push_str(delta); + json!({ + "type": "response.output_text.delta", + "sequence_number": self.next_sequence(), + "output_index": output_index, + "item_id": item_id, + "content_index": content_index, + "delta": delta + }) + } + + pub fn emit_text_done( + &mut self, + output_index: usize, + item_id: &str, + content_index: usize, + ) -> serde_json::Value { + json!({ + "type": "response.output_text.done", + "sequence_number": self.next_sequence(), + "output_index": output_index, + "item_id": item_id, + "content_index": content_index, + "text": self.accumulated_text.clone() + }) + } + + pub fn emit_content_part_done( + &mut self, + output_index: usize, + item_id: &str, + content_index: usize, + ) -> serde_json::Value { + json!({ + "type": "response.content_part.done", + "sequence_number": self.next_sequence(), + "output_index": output_index, + "item_id": item_id, + "content_index": content_index, + "part": { + "type": "text", + "text": self.accumulated_text.clone() + } + }) + } + + pub fn emit_completed(&mut self, usage: Option<&serde_json::Value>) -> serde_json::Value { + // Build output array from tracked items + let output: Vec = self + .output_items + .iter() + .filter_map(|item| { + if item.status == ItemStatus::Completed { + item.item_data.clone() + } else { + None + } + }) + .collect(); + + // If no items were tracked (legacy path), fall back to generic message + let output = if output.is_empty() { + vec![json!({ + "id": self.message_id.clone(), + "type": "message", + "role": "assistant", + "content": [{ + "type": "text", + "text": self.accumulated_text.clone() + }] + })] + } else { + output + }; + + // Build base response object + let mut response_obj = json!({ + "id": self.response_id, + "object": "response", + "created_at": self.created_at, + "status": "completed", + "model": self.model, + "output": output + }); + + // Add usage if provided + if let Some(usage_val) = usage { + response_obj["usage"] = usage_val.clone(); + } + + // Add all original request fields if available + if let Some(ref req) = self.original_request { + Self::add_optional_field(&mut response_obj, "instructions", &req.instructions); + Self::add_optional_field( + &mut response_obj, + "max_output_tokens", + &req.max_output_tokens, + ); + Self::add_optional_field(&mut response_obj, "max_tool_calls", &req.max_tool_calls); + Self::add_optional_field( + &mut response_obj, + "previous_response_id", + &req.previous_response_id, + ); + Self::add_optional_field(&mut response_obj, "reasoning", &req.reasoning); + Self::add_optional_field(&mut response_obj, "temperature", &req.temperature); + Self::add_optional_field(&mut response_obj, "top_p", &req.top_p); + Self::add_optional_field(&mut response_obj, "truncation", &req.truncation); + Self::add_optional_field(&mut response_obj, "user", &req.user); + + response_obj["parallel_tool_calls"] = json!(req.parallel_tool_calls.unwrap_or(true)); + response_obj["store"] = json!(req.store.unwrap_or(true)); + response_obj["tools"] = json!(req.tools.as_ref().unwrap_or(&vec![])); + response_obj["metadata"] = json!(req.metadata.as_ref().unwrap_or(&Default::default())); + + // tool_choice: serialize if present, otherwise use "auto" + if let Some(ref tc) = req.tool_choice { + response_obj["tool_choice"] = json!(tc); + } else { + response_obj["tool_choice"] = json!("auto"); + } + } + + json!({ + "type": "response.completed", + "sequence_number": self.next_sequence(), + "response": response_obj + }) + } + + /// Helper to add optional fields to JSON object + fn add_optional_field( + obj: &mut serde_json::Value, + key: &str, + value: &Option, + ) { + if let Some(val) = value { + obj[key] = json!(val); + } + } + + // ======================================================================== + // MCP Event Emission Methods + // ======================================================================== + + pub fn emit_mcp_list_tools_in_progress(&mut self, output_index: usize) -> serde_json::Value { + json!({ + "type": "response.mcp_list_tools.in_progress", + "sequence_number": self.next_sequence(), + "output_index": output_index + }) + } + + pub fn emit_mcp_list_tools_completed( + &mut self, + output_index: usize, + tools: &[mcp::Tool], + ) -> serde_json::Value { + let tool_items: Vec<_> = tools + .iter() + .map(|t| { + json!({ + "name": &t.name, + "description": &t.description, + "input_schema": t.input_schema.clone() + }) + }) + .collect(); + + json!({ + "type": "response.mcp_list_tools.completed", + "sequence_number": self.next_sequence(), + "output_index": output_index, + "tools": tool_items + }) + } + + pub fn emit_mcp_call_in_progress( + &mut self, + output_index: usize, + item_id: &str, + ) -> serde_json::Value { + json!({ + "type": "response.mcp_call.in_progress", + "sequence_number": self.next_sequence(), + "output_index": output_index, + "item_id": item_id + }) + } + + pub fn emit_mcp_call_arguments_delta( + &mut self, + output_index: usize, + item_id: &str, + delta: &str, + ) -> serde_json::Value { + // Accumulate arguments for this call + self.mcp_call_accumulated_args + .entry(item_id.to_string()) + .or_default() + .push_str(delta); + + json!({ + "type": "response.mcp_call_arguments.delta", + "sequence_number": self.next_sequence(), + "output_index": output_index, + "item_id": item_id, + "delta": delta + }) + } + + pub fn emit_mcp_call_arguments_done( + &mut self, + output_index: usize, + item_id: &str, + arguments: &str, + ) -> serde_json::Value { + json!({ + "type": "response.mcp_call_arguments.done", + "sequence_number": self.next_sequence(), + "output_index": output_index, + "item_id": item_id, + "arguments": arguments + }) + } + + pub fn emit_mcp_call_completed( + &mut self, + output_index: usize, + item_id: &str, + ) -> serde_json::Value { + json!({ + "type": "response.mcp_call.completed", + "sequence_number": self.next_sequence(), + "output_index": output_index, + "item_id": item_id + }) + } + + pub fn emit_mcp_call_failed( + &mut self, + output_index: usize, + item_id: &str, + error: &str, + ) -> serde_json::Value { + json!({ + "type": "response.mcp_call.failed", + "sequence_number": self.next_sequence(), + "output_index": output_index, + "item_id": item_id, + "error": error + }) + } + + // ======================================================================== + // Function Call Event Emission Methods + // ======================================================================== + + pub fn emit_function_call_arguments_delta( + &mut self, + output_index: usize, + item_id: &str, + delta: &str, + ) -> serde_json::Value { + json!({ + "type": "response.function_call_arguments.delta", + "sequence_number": self.next_sequence(), + "output_index": output_index, + "item_id": item_id, + "delta": delta + }) + } + + pub fn emit_function_call_arguments_done( + &mut self, + output_index: usize, + item_id: &str, + arguments: &str, + ) -> serde_json::Value { + json!({ + "type": "response.function_call_arguments.done", + "sequence_number": self.next_sequence(), + "output_index": output_index, + "item_id": item_id, + "arguments": arguments + }) + } + + // ======================================================================== + // Output Item Wrapper Events + // ======================================================================== + + /// Emit response.output_item.added event + pub fn emit_output_item_added( + &mut self, + output_index: usize, + item: &serde_json::Value, + ) -> serde_json::Value { + json!({ + "type": "response.output_item.added", + "sequence_number": self.next_sequence(), + "output_index": output_index, + "item": item + }) + } + + /// Emit response.output_item.done event + pub fn emit_output_item_done( + &mut self, + output_index: usize, + item: &serde_json::Value, + ) -> serde_json::Value { + // Store the item data for later use in emit_completed + self.store_output_item_data(output_index, item.clone()); + + json!({ + "type": "response.output_item.done", + "sequence_number": self.next_sequence(), + "output_index": output_index, + "item": item + }) + } + + /// Generate unique ID for item type + fn generate_item_id(prefix: &str) -> String { + format!("{}_{}", prefix, Uuid::new_v4().to_string().replace("-", "")) + } + + /// Allocate next output index and track item + pub fn allocate_output_index(&mut self, item_type: OutputItemType) -> (usize, String) { + let index = self.next_output_index; + self.next_output_index += 1; + + let id_prefix = match &item_type { + OutputItemType::McpListTools => "mcpl", + OutputItemType::McpCall => "mcp", + OutputItemType::FunctionCall => "fc", + OutputItemType::Message => "msg", + OutputItemType::Reasoning => "rs", + }; + + let id = Self::generate_item_id(id_prefix); + + self.output_items.push(OutputItemState { + output_index: index, + status: ItemStatus::InProgress, + item_data: None, + }); + + (index, id) + } + + /// Mark output item as completed and store its data + pub fn complete_output_item(&mut self, output_index: usize) { + if let Some(item) = self + .output_items + .iter_mut() + .find(|i| i.output_index == output_index) + { + item.status = ItemStatus::Completed; + } + } + + /// Store output item data when emitting output_item.done + pub fn store_output_item_data(&mut self, output_index: usize, item_data: serde_json::Value) { + if let Some(item) = self + .output_items + .iter_mut() + .find(|i| i.output_index == output_index) + { + item.item_data = Some(item_data); + } + } + + /// Finalize and return the complete ResponsesResponse + /// + /// This constructs the final ResponsesResponse from all accumulated output items + /// for persistence. Should be called after streaming is complete. + pub fn finalize(&self, usage: Option) -> ResponsesResponse { + // Build output array from tracked items + let output: Vec = self + .output_items + .iter() + .filter_map(|item| { + item.item_data + .as_ref() + .and_then(|data| serde_json::from_value(data.clone()).ok()) + }) + .collect(); + + // Convert Usage to ResponsesUsage + let responses_usage = usage.map(|u| { + let usage_info = UsageInfo { + prompt_tokens: u.prompt_tokens, + completion_tokens: u.completion_tokens, + total_tokens: u.total_tokens, + reasoning_tokens: u + .completion_tokens_details + .as_ref() + .and_then(|d| d.reasoning_tokens), + prompt_tokens_details: None, + }; + ResponsesUsage::Classic(usage_info) + }); + + // Build response using builder + ResponsesResponse::builder(&self.response_id, &self.model) + .created_at(self.created_at as i64) + .status(ResponseStatus::Completed) + .output(output) + .maybe_copy_from_request(self.original_request.as_ref()) + .maybe_usage(responses_usage) + .build() + } + + /// Emit reasoning item wrapper events (added + done) + /// + /// Reasoning items in OpenAI format are simple placeholders emitted between tool iterations. + /// They don't have streaming content - just wrapper events with empty/null content. + pub fn emit_reasoning_item( + &mut self, + tx: &mpsc::UnboundedSender>, + reasoning_content: Option, + ) -> Result<(), String> { + // Allocate output index and generate ID + let (output_index, item_id) = self.allocate_output_index(OutputItemType::Reasoning); + + // Build reasoning item structure + let item = json!({ + "id": item_id, + "type": "reasoning", + "summary": [], + "content": reasoning_content, + "encrypted_content": null, + "status": null + }); + + // Emit output_item.added + let added_event = self.emit_output_item_added(output_index, &item); + self.send_event(&added_event, tx)?; + + // Immediately emit output_item.done (no streaming for reasoning) + let done_event = self.emit_output_item_done(output_index, &item); + self.send_event(&done_event, tx)?; + + // Mark as completed + self.complete_output_item(output_index); + + Ok(()) + } + + /// Process a chunk and emit appropriate events + pub fn process_chunk( + &mut self, + chunk: &ChatCompletionStreamResponse, + tx: &mpsc::UnboundedSender>, + ) -> Result<(), String> { + // Process content if present + if let Some(choice) = chunk.choices.first() { + if let Some(content) = &choice.delta.content { + if !content.is_empty() { + // Allocate output_index and item_id for this message item (once per message) + if self.current_item_id.is_none() { + let (output_index, item_id) = + self.allocate_output_index(OutputItemType::Message); + + // Build message item structure + let item = json!({ + "id": item_id, + "type": "message", + "role": "assistant", + "content": [] + }); + + // Emit output_item.added + let event = self.emit_output_item_added(output_index, &item); + self.send_event(&event, tx)?; + self.has_emitted_output_item_added = true; + + // Store for subsequent events + self.current_item_id = Some(item_id); + self.current_message_output_index = Some(output_index); + } + + let output_index = self.current_message_output_index.unwrap(); + let item_id = self.current_item_id.clone().unwrap(); // Clone to avoid borrow checker issues + let content_index = 0; // Single content part for now + + // Emit content_part.added before first delta + if !self.has_emitted_content_part_added { + let event = + self.emit_content_part_added(output_index, &item_id, content_index); + self.send_event(&event, tx)?; + self.has_emitted_content_part_added = true; + } + + // Emit text delta + let event = + self.emit_text_delta(content, output_index, &item_id, content_index); + self.send_event(&event, tx)?; + } + } + + // Check for finish_reason to emit completion events + if let Some(reason) = &choice.finish_reason { + if reason == "stop" || reason == "length" { + let output_index = self.current_message_output_index.unwrap(); + let item_id = self.current_item_id.clone().unwrap(); // Clone to avoid borrow checker issues + let content_index = 0; + + // Emit closing events + if self.has_emitted_content_part_added { + let event = self.emit_text_done(output_index, &item_id, content_index); + self.send_event(&event, tx)?; + let event = + self.emit_content_part_done(output_index, &item_id, content_index); + self.send_event(&event, tx)?; + } + + if self.has_emitted_output_item_added { + // Build complete message item for output_item.done + let item = json!({ + "id": item_id, + "type": "message", + "role": "assistant", + "content": [{ + "type": "text", + "text": self.accumulated_text.clone() + }] + }); + let event = self.emit_output_item_done(output_index, &item); + self.send_event(&event, tx)?; + } + + // Mark item as completed + self.complete_output_item(output_index); + } + } + } + + Ok(()) + } + + pub fn send_event( + &self, + event: &serde_json::Value, + tx: &mpsc::UnboundedSender>, + ) -> Result<(), String> { + let event_json = serde_json::to_string(event) + .map_err(|e| format!("Failed to serialize event: {}", e))?; + + // Extract event type from the JSON for SSE event field + let event_type = event + .get("type") + .and_then(|v| v.as_str()) + .unwrap_or("message"); + + // Format as SSE with event: field + let sse_message = format!("event: {}\ndata: {}\n\n", event_type, event_json); + + if tx.send(Ok(Bytes::from(sse_message))).is_err() { + return Err("Client disconnected".to_string()); + } + + Ok(()) + } + + /// Send event and log any errors (typically client disconnect) + /// + /// This is a convenience method for streaming scenarios where client + /// disconnection is expected and should be logged but not fail the operation. + /// Returns true if sent successfully, false if client disconnected. + pub fn send_event_best_effort( + &self, + event: &serde_json::Value, + tx: &mpsc::UnboundedSender>, + ) -> bool { + match self.send_event(event, tx) { + Ok(()) => true, + Err(e) => { + tracing::debug!("Failed to send event (likely client disconnect): {}", e); + false + } + } + } + + /// Emit an error event + /// + /// Creates and sends an error event with the given error message. + /// Uses OpenAI's error event format. + /// Use this for terminal errors that should abort the streaming response. + pub fn emit_error( + &mut self, + error_msg: &str, + error_code: Option<&str>, + tx: &mpsc::UnboundedSender>, + ) { + let event = json!({ + "type": "error", + "code": error_code.unwrap_or("internal_error"), + "message": error_msg, + "param": null, + "sequence_number": self.next_sequence() + }); + let sse_data = format!("data: {}\n\n", serde_json::to_string(&event).unwrap()); + let _ = tx.send(Ok(Bytes::from(sse_data))); + } +} + +/// Build a Server-Sent Events (SSE) response +/// +/// Creates a Response with proper SSE headers and streaming body. +pub fn build_sse_response(rx: mpsc::UnboundedReceiver>) -> Response { + let stream = UnboundedReceiverStream::new(rx); + Response::builder() + .status(StatusCode::OK) + .header("Content-Type", "text/event-stream") + .header("Cache-Control", "no-cache") + .header("Connection", "keep-alive") + .body(Body::from_stream(stream)) + .unwrap() +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/responses/utils.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/responses/utils.rs new file mode 100644 index 00000000..3fbe20ef --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/responses/utils.rs @@ -0,0 +1,166 @@ +//! Utility functions for /v1/responses endpoint + +use std::sync::Arc; + +use axum::{ + http::StatusCode, + response::{IntoResponse, Response}, +}; +use serde_json::{json, to_value}; +use tracing::{debug, error, warn}; + +use crate::{ + core::WorkerRegistry, + data_connector::{ConversationItemStorage, ConversationStorage, ResponseStorage}, + mcp::McpManager, + protocols::{ + common::Tool, + responses::{ResponseTool, ResponseToolType, ResponsesRequest, ResponsesResponse}, + }, + routers::{ + grpc::error, + openai::{conversations::persist_conversation_items, mcp::ensure_request_mcp_client}, + }, +}; + +/// Ensure MCP connection succeeds if MCP tools are declared +/// +/// Checks if request declares MCP tools, and if so, validates that +/// the MCP client can be created and connected. +pub async fn ensure_mcp_connection( + mcp_manager: &Arc, + tools: Option<&[ResponseTool]>, +) -> Result { + let has_mcp_tools = tools + .map(|t| { + t.iter() + .any(|tool| matches!(tool.r#type, ResponseToolType::Mcp)) + }) + .unwrap_or(false); + + if has_mcp_tools { + if let Some(tools) = tools { + if ensure_request_mcp_client(mcp_manager, tools) + .await + .is_none() + { + error!( + function = "ensure_mcp_connection", + "Failed to connect to MCP server" + ); + return Err(error::failed_dependency( + "Failed to connect to MCP server. Check server_url and authorization.", + )); + } + } + } + + Ok(has_mcp_tools) +} + +/// Validate that workers are available for the requested model +pub fn validate_worker_availability( + worker_registry: &Arc, + model: &str, +) -> Option { + let available_models = worker_registry.get_models(); + + if !available_models.contains(&model.to_string()) { + return Some( + ( + StatusCode::SERVICE_UNAVAILABLE, + axum::Json(json!({ + "error": { + "message": format!( + "No workers available for model '{}'. Available models: {}", + model, + available_models.join(", ") + ), + "type": "service_unavailable", + "param": "model", + "code": "no_available_workers" + } + })), + ) + .into_response(), + ); + } + + None +} + +/// Extract function tools (and optionally MCP tools) from ResponseTools +/// +/// This utility consolidates the logic for extracting tools with schemas from ResponseTools. +/// It's used by both Harmony and Regular routers for different purposes: +/// +/// - **Harmony router**: Extracts both Function and MCP tools (with `include_mcp: true`) +/// because MCP schemas are populated by convert_mcp_tools_to_response_tools() before the +/// pipeline runs. These tools are used to generate structural constraints in the +/// Harmony preparation stage. +/// +/// - **Regular router**: Extracts only Function tools (with `include_mcp: false`) during +/// the initial conversion from ResponsesRequest to ChatCompletionRequest. MCP tools +/// are merged later by the tool loop before being sent to the chat pipeline, where +/// tool_choice constraints are generated for ALL tools (function + MCP combined). +pub fn extract_tools_from_response_tools( + response_tools: Option<&[ResponseTool]>, + include_mcp: bool, +) -> Vec { + let Some(tools) = response_tools else { + return Vec::new(); + }; + + tools + .iter() + .filter_map(|rt| { + match rt.r#type { + // Function tools: Schema in request + ResponseToolType::Function => rt.function.as_ref().map(|f| Tool { + tool_type: "function".to_string(), + function: f.clone(), + }), + // MCP tools: Schema populated by convert_mcp_tools_to_response_tools() + // Only include if requested (Harmony case) + ResponseToolType::Mcp if include_mcp => rt.function.as_ref().map(|f| Tool { + tool_type: "function".to_string(), + function: f.clone(), + }), + // Hosted tools: No schema available, skip + _ => None, + } + }) + .collect() +} + +/// Persist response to storage if store=true +/// +/// Common helper function to avoid duplication across sync and streaming paths +/// in both harmony and regular responses implementations. +pub async fn persist_response_if_needed( + conversation_storage: Arc, + conversation_item_storage: Arc, + response_storage: Arc, + response: &ResponsesResponse, + original_request: &ResponsesRequest, +) { + if !original_request.store.unwrap_or(true) { + return; + } + + if let Ok(response_json) = to_value(response) { + if let Err(e) = persist_conversation_items( + conversation_storage, + conversation_item_storage, + response_storage, + &response_json, + original_request, + ) + .await + { + warn!("Failed to persist response: {}", e); + } else { + debug!("Persisted response: {}", response.id); + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/client_acquisition.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/client_acquisition.rs new file mode 100644 index 00000000..38ac075a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/client_acquisition.rs @@ -0,0 +1,62 @@ +//! Client acquisition stage: Get gRPC clients from selected workers + +use async_trait::async_trait; +use axum::response::Response; +use tracing::error; + +use super::PipelineStage; +use crate::routers::grpc::{ + context::{ClientSelection, RequestContext, WorkerSelection}, + error, utils, +}; + +/// Client acquisition stage: Get gRPC clients from selected workers +pub struct ClientAcquisitionStage; + +#[async_trait] +impl PipelineStage for ClientAcquisitionStage { + async fn execute(&self, ctx: &mut RequestContext) -> Result, Response> { + let workers = ctx.state.workers.as_ref().ok_or_else(|| { + error!( + function = "ClientAcquisitionStage::execute", + "Worker selection stage not completed" + ); + error::internal_error("Worker selection not completed") + })?; + + let clients = match workers { + WorkerSelection::Single { worker } => { + let client = utils::get_grpc_client_from_worker(worker).await?; + ClientSelection::Single { client } + } + WorkerSelection::Dual { prefill, decode } => { + let prefill_client = utils::get_grpc_client_from_worker(prefill).await?; + let decode_client = utils::get_grpc_client_from_worker(decode).await?; + + // vLLM does not support dual (PD disaggregated) mode + if prefill_client.is_vllm() || decode_client.is_vllm() { + error!( + function = "ClientAcquisitionStage::execute", + "vLLM backend does not support dual (PD disaggregated) mode" + ); + return Err(error::bad_request( + "vLLM backend does not support prefill/decode disaggregated mode. \ + Please use runtime_type: sglang for PD mode, or use a regular (non-PD) worker configuration." + )); + } + + ClientSelection::Dual { + prefill: prefill_client, + decode: decode_client, + } + } + }; + + ctx.state.clients = Some(clients); + Ok(None) + } + + fn name(&self) -> &'static str { + "ClientAcquisition" + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/dispatch_metadata.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/dispatch_metadata.rs new file mode 100644 index 00000000..3f63782e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/dispatch_metadata.rs @@ -0,0 +1,73 @@ +//! Dispatch metadata stage: Prepare metadata for dispatch + +use std::time::{SystemTime, UNIX_EPOCH}; + +use async_trait::async_trait; +use axum::response::Response; +use tracing::error; + +use super::PipelineStage; +use crate::routers::grpc::{ + context::{DispatchMetadata, RequestContext, RequestType, WorkerSelection}, + error, +}; + +/// Dispatch metadata stage: Prepare metadata for dispatch +pub struct DispatchMetadataStage; + +#[async_trait] +impl PipelineStage for DispatchMetadataStage { + async fn execute(&self, ctx: &mut RequestContext) -> Result, Response> { + let proto_request = ctx.state.proto_request.as_ref().ok_or_else(|| { + error!( + function = "DispatchMetadataStage::execute", + "Proto request not built" + ); + error::internal_error("Proto request not built") + })?; + + let request_id = proto_request.request_id().to_string(); + let model = match &ctx.input.request_type { + RequestType::Chat(req) => req.model.clone(), + RequestType::Generate(_req) => { + // Generate requests don't have a model field + // Use model_id from input or default + ctx.input + .model_id + .clone() + .unwrap_or_else(|| "default".to_string()) + } + RequestType::Responses(req) => req.model.clone(), + }; + + let weight_version = ctx + .state + .workers + .as_ref() + .map(|w| match w { + WorkerSelection::Single { worker } => worker, + WorkerSelection::Dual { decode, .. } => decode, + }) + .and_then(|w| w.metadata().labels.get("weight_version").cloned()) + .unwrap_or_else(|| "default".to_string()); + + let created = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + + ctx.state.dispatch = Some(DispatchMetadata { + request_id, + model, + created, + weight_version: Some(weight_version), + is_streaming: ctx.is_streaming(), + }); + + Ok(None) + } + + fn name(&self) -> &'static str { + "DispatchMetadata" + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/helpers.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/helpers.rs new file mode 100644 index 00000000..eb2ae521 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/helpers.rs @@ -0,0 +1,43 @@ +//! Common helper functions shared across stages + +use std::sync::Arc; + +use rand::Rng; +use tracing::debug; + +use crate::{ + core::Worker, grpc_client::sglang_proto::DisaggregatedParams, + routers::grpc::proto_wrapper::ProtoGenerateRequest, +}; + +/// Inject PD bootstrap metadata into a gRPC request +/// +/// Used by both chat and generate request building stages when in PD mode. +/// Only SGLang supports PD (prefill/decode) disaggregated mode. +pub fn inject_bootstrap_metadata( + request: &mut ProtoGenerateRequest, + prefill_worker: &Arc, +) { + let hostname = prefill_worker.bootstrap_host(); + let bootstrap_port = prefill_worker.bootstrap_port().unwrap_or(8998); + + // Generate room ID for bootstrap + let room_id = rand::rng().random_range(0..i32::MAX); + + // Create DisaggregatedParams + let disagg_params = DisaggregatedParams { + bootstrap_host: hostname.to_string(), + bootstrap_port: bootstrap_port as i32, + bootstrap_room: room_id, + }; + + // Inject metadata directly into SGLang request + // (vLLM doesn't support PD mode, so this will panic if called with vLLM) + let sglang_request = request.as_sglang_mut(); + sglang_request.disaggregated_params = Some(disagg_params); + + debug!( + "Injected bootstrap metadata: host={}, port={}, room={}", + hostname, bootstrap_port, room_id + ); +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/mod.rs new file mode 100644 index 00000000..eafc2261 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/mod.rs @@ -0,0 +1,39 @@ +//! Common pipeline stages shared across all endpoints and model types +//! +//! These stages are endpoint-agnostic and model-agnostic: +//! - Worker selection +//! - Client acquisition +//! - Dispatch metadata generation +//! - Request execution + +use async_trait::async_trait; +use axum::response::Response; + +use crate::routers::grpc::context::RequestContext; + +/// Trait for pipeline stages that process requests +#[async_trait] +pub trait PipelineStage: Send + Sync { + /// Execute this stage, mutating the context + /// + /// Returns: + /// - `Ok(None)` - Continue to next stage + /// - `Ok(Some(response))` - Pipeline complete, return this response (e.g., streaming) + /// - `Err(response)` - Error occurred, return this error response + async fn execute(&self, ctx: &mut RequestContext) -> Result, Response>; + + /// Stage name for logging + fn name(&self) -> &'static str; +} + +mod client_acquisition; +mod dispatch_metadata; +pub mod helpers; +mod request_execution; +mod worker_selection; + +// Export stage implementations +pub use client_acquisition::ClientAcquisitionStage; +pub use dispatch_metadata::DispatchMetadataStage; +pub use request_execution::{ExecutionMode, RequestExecutionStage}; +pub use worker_selection::{WorkerSelectionMode, WorkerSelectionStage}; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/request_execution.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/request_execution.rs new file mode 100644 index 00000000..ed2b44a6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/request_execution.rs @@ -0,0 +1,154 @@ +//! Request execution stage: Execute gRPC requests (single or dual dispatch) + +use async_trait::async_trait; +use axum::response::Response; +use tracing::error; + +use super::PipelineStage; +use crate::routers::grpc::{ + context::{ClientSelection, ExecutionResult, RequestContext}, + error, + proto_wrapper::{ProtoGenerateRequest, ProtoStream}, +}; + +type StreamResult = Result>; + +/// Request execution stage: Execute gRPC requests (single or dual dispatch) +pub struct RequestExecutionStage { + mode: ExecutionMode, +} + +pub enum ExecutionMode { + /// Regular mode: single worker execution + Single, + /// PD mode: dual dispatch to prefill + decode workers + DualDispatch, +} + +impl RequestExecutionStage { + pub fn new(mode: ExecutionMode) -> Self { + Self { mode } + } +} + +#[async_trait] +impl PipelineStage for RequestExecutionStage { + async fn execute(&self, ctx: &mut RequestContext) -> Result, Response> { + let proto_request = ctx.state.proto_request.take().ok_or_else(|| { + error!( + function = "RequestExecutionStage::execute", + "Proto request not built" + ); + error::internal_error("Proto request not built") + })?; + + let clients = ctx.state.clients.as_mut().ok_or_else(|| { + error!( + function = "RequestExecutionStage::execute", + "Client acquisition not completed" + ); + error::internal_error("Client acquisition not completed") + })?; + + let result = match self.mode { + ExecutionMode::Single => self.execute_single(proto_request, clients).await?, + ExecutionMode::DualDispatch => { + self.execute_dual_dispatch(proto_request, clients).await? + } + }; + + // Store result in context for ResponseProcessingStage + ctx.state.response.execution_result = Some(result); + Ok(None) + } + + fn name(&self) -> &'static str { + "RequestExecution" + } +} + +impl RequestExecutionStage { + async fn execute_single( + &self, + proto_request: ProtoGenerateRequest, + clients: &mut ClientSelection, + ) -> Result { + let client = clients.single_mut().ok_or_else(|| { + error!( + function = "execute_single", + "Expected single client but got dual" + ); + error::internal_error("Expected single client but got dual") + })?; + + let stream = client.generate(proto_request).await.map_err(|e| { + error!( + function = "execute_single", + error = %e, + "Failed to start generation" + ); + error::internal_error(format!("Failed to start generation: {}", e)) + })?; + + Ok(ExecutionResult::Single { stream }) + } + + async fn execute_dual_dispatch( + &self, + proto_request: ProtoGenerateRequest, + clients: &mut ClientSelection, + ) -> Result { + let (prefill_client, decode_client) = clients.dual_mut().ok_or_else(|| { + error!( + function = "execute_dual_dispatch", + "Expected dual clients but got single" + ); + error::internal_error("Expected dual clients but got single") + })?; + + let prefill_request = proto_request.clone_inner(); + let decode_request = proto_request; + + let (prefill_result, decode_result): (StreamResult, StreamResult) = tokio::join!( + prefill_client.generate(prefill_request), + decode_client.generate(decode_request) + ); + + // Handle prefill result + let prefill_stream = match prefill_result { + Ok(s) => s, + Err(e) => { + error!( + function = "execute_dual_dispatch", + error = %e, + "Prefill worker failed to start" + ); + return Err(error::internal_error(format!( + "Prefill worker failed to start: {}", + e + ))); + } + }; + + // Handle decode result + let decode_stream = match decode_result { + Ok(s) => s, + Err(e) => { + error!( + function = "execute_dual_dispatch", + error = %e, + "Decode worker failed to start" + ); + return Err(error::internal_error(format!( + "Decode worker failed to start: {}", + e + ))); + } + }; + + Ok(ExecutionResult::Dual { + prefill: prefill_stream, + decode: Box::new(decode_stream), + }) + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/worker_selection.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/worker_selection.rs new file mode 100644 index 00000000..d5281937 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/common/stages/worker_selection.rs @@ -0,0 +1,197 @@ +//! Worker selection stage: Select appropriate worker(s) based on routing mode + +use std::sync::Arc; + +use async_trait::async_trait; +use axum::response::Response; +use tracing::{error, warn}; + +use super::PipelineStage; +use crate::{ + core::{ConnectionMode, Worker, WorkerRegistry, WorkerType}, + policies::PolicyRegistry, + routers::grpc::{ + context::{RequestContext, WorkerSelection}, + error, + }, +}; + +/// Worker selection stage: Select appropriate worker(s) based on routing mode +pub struct WorkerSelectionStage { + worker_registry: Arc, + policy_registry: Arc, + mode: WorkerSelectionMode, +} + +pub enum WorkerSelectionMode { + /// Regular mode: select single worker + Regular, + /// PD mode: select prefill + decode workers + PrefillDecode, +} + +impl WorkerSelectionStage { + pub fn new( + worker_registry: Arc, + policy_registry: Arc, + mode: WorkerSelectionMode, + ) -> Self { + Self { + worker_registry, + policy_registry, + mode, + } + } +} + +#[async_trait] +impl PipelineStage for WorkerSelectionStage { + async fn execute(&self, ctx: &mut RequestContext) -> Result, Response> { + let prep = ctx.state.preparation.as_ref().ok_or_else(|| { + error!( + function = "WorkerSelectionStage::execute", + "Preparation stage not completed" + ); + error::internal_error("Preparation stage not completed") + })?; + + // For Harmony, use selection_text produced during Harmony encoding + // Otherwise, use original_text from regular preparation + let text = if prep.harmony_mode { + prep.selection_text.as_deref() + } else { + prep.original_text.as_deref() + }; + + let workers = match self.mode { + WorkerSelectionMode::Regular => { + match self.select_single_worker(ctx.input.model_id.as_deref(), text) { + Some(w) => WorkerSelection::Single { worker: w }, + None => { + error!( + function = "WorkerSelectionStage::execute", + mode = "Regular", + model_id = ?ctx.input.model_id, + "No available workers for model" + ); + return Err(error::service_unavailable(format!( + "No available workers for model: {:?}", + ctx.input.model_id + ))); + } + } + } + WorkerSelectionMode::PrefillDecode => { + match self.select_pd_pair(ctx.input.model_id.as_deref(), text) { + Some((prefill, decode)) => WorkerSelection::Dual { prefill, decode }, + None => { + error!( + function = "WorkerSelectionStage::execute", + mode = "PrefillDecode", + model_id = ?ctx.input.model_id, + "No available PD worker pairs for model" + ); + return Err(error::service_unavailable(format!( + "No available PD worker pairs for model: {:?}", + ctx.input.model_id + ))); + } + } + } + }; + + ctx.state.workers = Some(workers); + Ok(None) + } + + fn name(&self) -> &'static str { + "WorkerSelection" + } +} + +impl WorkerSelectionStage { + fn select_single_worker( + &self, + model_id: Option<&str>, + text: Option<&str>, + ) -> Option> { + // Get workers for the specified model, filtered by connection mode + let workers = self.worker_registry.get_workers_filtered( + model_id, + Some(WorkerType::Regular), + Some(ConnectionMode::Grpc { port: None }), + false, // get all workers, we'll filter by is_available() next + ); + + let available: Vec> = workers + .iter() + .filter(|w| w.is_available()) + .cloned() + .collect(); + + if available.is_empty() { + return None; + } + + // Get the appropriate policy for this model + let policy = match model_id { + Some(model) => self.policy_registry.get_policy_or_default(model), + None => self.policy_registry.get_default_policy(), + }; + + // Select worker using the policy + let idx = policy.select_worker(&available, text)?; + Some(available[idx].clone()) + } + + fn select_pd_pair( + &self, + model_id: Option<&str>, + text: Option<&str>, + ) -> Option<(Arc, Arc)> { + let all_workers = self.worker_registry.get_workers_filtered( + model_id, + None, + Some(ConnectionMode::Grpc { port: None }), // Match any gRPC worker + false, + ); + + let (available_prefill, available_decode): (Vec<_>, Vec<_>) = + all_workers + .into_iter() + .fold((Vec::new(), Vec::new()), |mut acc, w| { + if w.is_available() { + match w.metadata().worker_type { + WorkerType::Prefill { .. } => acc.0.push(w), + WorkerType::Decode => acc.1.push(w), + _ => {} + } + } + acc + }); + + if available_prefill.is_empty() { + warn!("No available prefill workers"); + return None; + } + + if available_decode.is_empty() { + warn!("No available decode workers"); + return None; + } + + // Select using policies + let policy = match model_id { + Some(model) => self.policy_registry.get_policy_or_default(model), + None => self.policy_registry.get_default_policy(), + }; + + let prefill_idx = policy.select_worker(&available_prefill, text)?; + let decode_idx = policy.select_worker(&available_decode, text)?; + + Some(( + available_prefill[prefill_idx].clone(), + available_decode[decode_idx].clone(), + )) + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/context.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/context.rs new file mode 100644 index 00000000..f01eaa94 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/context.rs @@ -0,0 +1,426 @@ +//! Request context types for gRPC router pipeline +//! +//! This module provides the core context types that flow through the router pipeline, +//! eliminating deep parameter passing chains and providing a single source of truth +//! for request state. + +use std::{collections::HashMap, sync::Arc}; + +use axum::http::HeaderMap; +use serde_json::Value; + +use super::{ + client::GrpcClient, + proto_wrapper::{ProtoGenerateComplete, ProtoGenerateRequest, ProtoStream}, +}; +use crate::{ + core::Worker, + protocols::{ + chat::{ChatCompletionRequest, ChatCompletionResponse}, + generate::{GenerateRequest, GenerateResponse}, + responses::ResponsesRequest, + }, + reasoning_parser::ParserFactory as ReasoningParserFactory, + tokenizer::{stop::StopSequenceDecoder, traits::Tokenizer}, + tool_parser::ParserFactory as ToolParserFactory, +}; + +/// Main request processing context +/// +/// This is the single source of truth for all request state as it flows +/// through the pipeline stages. Uses Rust's type system to enforce proper +/// stage ordering at compile time. +pub struct RequestContext { + pub input: RequestInput, + pub components: Arc, + pub state: ProcessingState, +} + +/// Immutable request input +pub struct RequestInput { + pub request_type: RequestType, + pub headers: Option, + pub model_id: Option, +} + +/// Request type variants +/// Using Arc instead of Box to enable cheap cloning for background tasks +pub enum RequestType { + Chat(Arc), + Generate(Arc), + Responses(Arc), +} + +/// Shared components (injected once at creation) +pub struct SharedComponents { + pub tokenizer: Arc, + pub tool_parser_factory: ToolParserFactory, + pub reasoning_parser_factory: ReasoningParserFactory, +} + +/// Mutable processing state (evolves through pipeline stages) +#[derive(Default)] +pub struct ProcessingState { + // Stage 1: Preparation outputs + pub preparation: Option, + + // Stage 2: Worker selection outputs + pub workers: Option, + + // Stage 3: Client acquisition outputs + pub clients: Option, + + // Stage 4: Request building outputs + pub proto_request: Option, + + // Stage 5: Dispatch metadata + pub dispatch: Option, + + // Stage 6: Response processing state + pub response: ResponseState, +} + +/// Output from preparation stage (Step 1) +pub struct PreparationOutput { + /// Original text (for chat) or resolved text (for generate) + pub original_text: Option, + + /// Tokenized input + pub token_ids: Vec, + + /// Processed messages (chat only) + pub processed_messages: Option, + + /// Tool call constraints (if applicable) + pub tool_constraints: Option<(String, String)>, + + /// Filtered request (if tools were filtered) + pub filtered_request: Option, + + // Harmony-specific fields + /// Whether this is a Harmony request (default: false) + pub harmony_mode: bool, + + /// Selection text for worker routing (Harmony only) + pub selection_text: Option, + + /// Harmony messages for history tracking (Harmony only) + pub harmony_messages: Option>, + + /// Stop token IDs for Harmony models + pub harmony_stop_ids: Option>, +} + +/// Worker selection (Step 2) +pub enum WorkerSelection { + Single { + worker: Arc, + }, + Dual { + prefill: Arc, + decode: Arc, + }, +} + +/// Client selection (Step 3) +pub enum ClientSelection { + Single { + client: GrpcClient, + }, + Dual { + prefill: GrpcClient, + decode: GrpcClient, + }, +} + +/// Dispatch metadata (Step 5) +#[derive(Clone)] +pub struct DispatchMetadata { + pub request_id: String, + pub model: String, + pub created: u64, + pub weight_version: Option, + pub is_streaming: bool, +} + +/// Response processing state (Step 6) +#[derive(Default)] +pub struct ResponseState { + /// Stop sequence decoder + pub stop_decoder: Option, + + /// Per-index streaming state (for n>1 support) + pub streaming: StreamingState, + + /// Collected responses (non-streaming) + pub collected: Option>, + + /// Execution result (streams from workers) + pub execution_result: Option, + + /// Final processed response + pub final_response: Option, + + /// Responses API iteration result (Harmony only, for tool loop orchestration) + pub responses_iteration_result: Option, + + // Harmony-specific parser state + /// Harmony parser for non-streaming (single parser for all indices) + pub harmony_parser: Option, + + /// Harmony parsers for streaming (one per index for n>1 support) + pub harmony_parser_per_index: Option>, +} + +/// Streaming state (per-choice tracking) +#[derive(Default)] +pub struct StreamingState { + pub is_firsts: HashMap, + pub stream_buffers: HashMap, + pub finish_reasons: HashMap, + pub matched_stops: HashMap>, + pub prompt_tokens: HashMap, + pub completion_tokens: HashMap, + pub cached_tokens: HashMap, + + // Parser state (lazy initialization per index) + pub reasoning_parsers: + HashMap>>>, + pub tool_parsers: + HashMap>>>, + pub has_tool_calls: HashMap, +} + +impl RequestContext { + /// Create context for chat completion request + pub fn for_chat( + request: Arc, + headers: Option, + model_id: Option, + components: Arc, + ) -> Self { + Self { + input: RequestInput { + request_type: RequestType::Chat(request), + headers, + model_id, + }, + components, + state: ProcessingState::default(), + } + } + + /// Create context for generate request + pub fn for_generate( + request: Arc, + headers: Option, + model_id: Option, + components: Arc, + ) -> Self { + Self { + input: RequestInput { + request_type: RequestType::Generate(request), + headers, + model_id, + }, + components, + state: ProcessingState::default(), + } + } + + /// Create context for Responses API request + pub fn for_responses( + request: Arc, + headers: Option, + model_id: Option, + components: Arc, + ) -> Self { + Self { + input: RequestInput { + request_type: RequestType::Responses(request), + headers, + model_id, + }, + components, + state: ProcessingState::default(), + } + } + + /// Get reference to original request (type-safe) + pub fn request(&self) -> &RequestType { + &self.input.request_type + } + + /// Get chat request (panics if not chat) + pub fn chat_request(&self) -> &ChatCompletionRequest { + match &self.input.request_type { + RequestType::Chat(req) => req.as_ref(), + _ => panic!("Expected chat request"), + } + } + + /// Get Arc clone of chat request (panics if not chat) + pub fn chat_request_arc(&self) -> Arc { + match &self.input.request_type { + RequestType::Chat(req) => Arc::clone(req), + _ => panic!("Expected chat request"), + } + } + + /// Get generate request (panics if not generate) + pub fn generate_request(&self) -> &GenerateRequest { + match &self.input.request_type { + RequestType::Generate(req) => req.as_ref(), + _ => panic!("Expected generate request"), + } + } + + /// Get Arc clone of generate request (panics if not generate) + pub fn generate_request_arc(&self) -> Arc { + match &self.input.request_type { + RequestType::Generate(req) => Arc::clone(req), + _ => panic!("Expected generate request"), + } + } + + /// Get responses request (panics if not responses) + pub fn responses_request(&self) -> &ResponsesRequest { + match &self.input.request_type { + RequestType::Responses(req) => req.as_ref(), + _ => panic!("Expected responses request"), + } + } + + /// Get Arc clone of responses request (panics if not responses) + pub fn responses_request_arc(&self) -> Arc { + match &self.input.request_type { + RequestType::Responses(req) => Arc::clone(req), + _ => panic!("Expected responses request"), + } + } + + /// Check if request is streaming + pub fn is_streaming(&self) -> bool { + match &self.input.request_type { + RequestType::Chat(req) => req.stream, + RequestType::Generate(req) => req.stream, + RequestType::Responses(req) => req.stream.unwrap_or(false), + } + } +} + +impl WorkerSelection { + pub fn is_dual(&self) -> bool { + matches!(self, Self::Dual { .. }) + } + + pub fn single(&self) -> Option<&Arc> { + match self { + Self::Single { worker } => Some(worker), + _ => None, + } + } + + #[allow(clippy::type_complexity)] + pub fn dual(&self) -> Option<(&Arc, &Arc)> { + match self { + Self::Dual { prefill, decode } => Some((prefill, decode)), + _ => None, + } + } + + pub fn prefill_worker(&self) -> Option<&Arc> { + match self { + Self::Dual { prefill, .. } => Some(prefill), + _ => None, + } + } + + pub fn decode_worker(&self) -> Option<&Arc> { + match self { + Self::Dual { decode, .. } => Some(decode), + _ => None, + } + } +} + +impl ClientSelection { + pub fn is_dual(&self) -> bool { + matches!(self, Self::Dual { .. }) + } + + pub fn single(&self) -> Option<&GrpcClient> { + match self { + Self::Single { client } => Some(client), + _ => None, + } + } + + pub fn single_mut(&mut self) -> Option<&mut GrpcClient> { + match self { + Self::Single { client } => Some(client), + _ => None, + } + } + + pub fn dual(&self) -> Option<(&GrpcClient, &GrpcClient)> { + match self { + Self::Dual { prefill, decode } => Some((prefill, decode)), + _ => None, + } + } + + pub fn dual_mut(&mut self) -> Option<(&mut GrpcClient, &mut GrpcClient)> { + match self { + Self::Dual { prefill, decode } => Some((prefill, decode)), + _ => None, + } + } + + pub fn prefill_client(&self) -> Option<&GrpcClient> { + match self { + Self::Dual { prefill, .. } => Some(prefill), + _ => None, + } + } + + pub fn prefill_client_mut(&mut self) -> Option<&mut GrpcClient> { + match self { + Self::Dual { prefill, .. } => Some(prefill), + _ => None, + } + } + + pub fn decode_client(&self) -> Option<&GrpcClient> { + match self { + Self::Dual { decode, .. } => Some(decode), + _ => None, + } + } + + pub fn decode_client_mut(&mut self) -> Option<&mut GrpcClient> { + match self { + Self::Dual { decode, .. } => Some(decode), + _ => None, + } + } +} + +/// Result of request execution (streams from workers) +/// Uses ProtoStream to automatically abort on cancellation +pub enum ExecutionResult { + Single { + stream: ProtoStream, + }, + Dual { + prefill: ProtoStream, + decode: Box, + }, +} + +/// Final processed response +pub enum FinalResponse { + Chat(ChatCompletionResponse), + /// Generate response is a Vec of GenerateResponse (n=1 returns single item, n>1 returns multiple) + Generate(Vec), +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/error.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/error.rs new file mode 100644 index 00000000..22797c1a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/error.rs @@ -0,0 +1,184 @@ +//! Centralized error response handling for all routers +//! +//! This module provides consistent error responses across OpenAI and gRPC routers, +//! ensuring all errors follow OpenAI's API error format. + +use axum::{ + http::StatusCode, + response::{IntoResponse, Response}, + Json, +}; +use serde_json::json; + +/// Create a 500 Internal Server Error response +/// +/// Use this for unexpected server-side errors, database failures, etc. +/// +/// # Example +/// ```ignore +/// return Err(internal_error("Database connection failed")); +/// ``` +pub fn internal_error(message: impl Into) -> Response { + let msg = message.into(); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": { + "message": msg, + "type": "internal_error", + "code": 500 + } + })), + ) + .into_response() +} + +/// Create a 400 Bad Request response +/// +/// Use this for invalid request parameters, malformed JSON, validation errors, etc. +/// +/// # Example +/// ```ignore +/// return Err(bad_request("Invalid conversation ID format")); +/// ``` +pub fn bad_request(message: impl Into) -> Response { + let msg = message.into(); + ( + StatusCode::BAD_REQUEST, + Json(json!({ + "error": { + "message": msg, + "type": "invalid_request_error", + "code": 400 + } + })), + ) + .into_response() +} + +/// Create a 404 Not Found response +/// +/// Use this for resources that don't exist (conversations, responses, etc.) +/// +/// # Example +/// ```ignore +/// return Err(not_found(format!("Conversation '{}' not found", id))); +/// ``` +pub fn not_found(message: impl Into) -> Response { + let msg = message.into(); + ( + StatusCode::NOT_FOUND, + Json(json!({ + "error": { + "message": msg, + "type": "invalid_request_error", + "code": 404 + } + })), + ) + .into_response() +} + +/// Create a 503 Service Unavailable response +/// +/// Use this for temporary service issues like no workers available, rate limiting, etc. +/// +/// # Example +/// ```ignore +/// return Err(service_unavailable("No workers available for this model")); +/// ``` +pub fn service_unavailable(message: impl Into) -> Response { + let msg = message.into(); + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({ + "error": { + "message": msg, + "type": "service_unavailable", + "code": 503 + } + })), + ) + .into_response() +} + +/// Create a 424 Failed Dependency response +/// +/// Use this when an external dependency (like MCP server) fails. +/// +/// # Example +/// ```ignore +/// return Err(failed_dependency("Failed to connect to MCP server")); +/// ``` +pub fn failed_dependency(message: impl Into) -> Response { + let msg = message.into(); + ( + StatusCode::FAILED_DEPENDENCY, + Json(json!({ + "error": { + "message": msg, + "type": "external_connector_error", + "code": 424 + } + })), + ) + .into_response() +} + +/// Create a 501 Not Implemented response +/// +/// Use this for features that are not yet implemented or supported. +/// +/// # Example +/// ```ignore +/// return Err(not_implemented("vLLM backend integration is in progress")); +/// ``` +pub fn not_implemented(message: impl Into) -> Response { + let msg = message.into(); + ( + StatusCode::NOT_IMPLEMENTED, + Json(json!({ + "error": { + "message": msg, + "type": "not_implemented_error", + "code": 501 + } + })), + ) + .into_response() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_internal_error_string() { + let response = internal_error("Test error"); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + } + + #[test] + fn test_internal_error_format() { + let response = internal_error(format!("Error: {}", 42)); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + } + + #[test] + fn test_bad_request() { + let response = bad_request("Invalid input"); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } + + #[test] + fn test_not_found() { + let response = not_found("Resource not found"); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + } + + #[test] + fn test_service_unavailable() { + let response = service_unavailable("No workers"); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/builder.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/builder.rs new file mode 100644 index 00000000..d51da4ed --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/builder.rs @@ -0,0 +1,887 @@ +//! Harmony request builder +//! +//! Handles encoding of Chat/Responses requests into Harmony format using openai-harmony library. + +use std::sync::OnceLock; + +use chrono::Local; +use openai_harmony::{ + chat::{ + Author, ChannelConfig, Content, Conversation, DeveloperContent, Message as HarmonyMessage, + ReasoningEffort, Role, SystemContent, TextContent, ToolDescription, + }, + HarmonyEncoding, HarmonyEncodingName, +}; +use tracing::debug; + +use super::types::HarmonyBuildOutput; +use crate::protocols::{ + chat::{ChatCompletionRequest, ChatMessage, MessageContent}, + common::{ContentPart, Tool}, + responses::{ + ReasoningEffort as ResponsesReasoningEffort, ResponseContentPart, ResponseInput, + ResponseInputOutputItem, ResponseReasoningContent, ResponseTool, ResponseToolType, + ResponsesRequest, StringOrContentParts, + }, +}; + +/// Global Harmony encoding (lazy-initialized) +static HARMONY_ENCODING: OnceLock = OnceLock::new(); + +/// Get or initialize the Harmony encoding +/// +/// Uses HarmonyGptOss encoding which supports the gpt-oss model family. +pub(super) fn get_harmony_encoding() -> &'static HarmonyEncoding { + HARMONY_ENCODING.get_or_init(|| { + openai_harmony::load_harmony_encoding(HarmonyEncodingName::HarmonyGptOss) + .expect("Failed to load Harmony encoding") + }) +} + +/// Built-in tools that are added to the system message +const BUILTIN_TOOLS: &[&str] = &["web_search_preview", "code_interpreter", "container"]; + +/// Trait for tool-like objects that can be converted to Harmony ToolDescription +trait ToolLike { + /// Check if this is a built-in tool (should be skipped in developer message) + #[allow(dead_code)] + fn is_builtin(&self) -> bool; + + /// Check if this is a custom tool (function or MCP) + fn is_custom(&self) -> bool; + + /// Convert to ToolDescription + fn to_tool_description(&self) -> Option; +} + +/// Implement ToolLike for Chat Completion Tool +impl ToolLike for Tool { + fn is_builtin(&self) -> bool { + matches!( + self.tool_type.as_str(), + "web_search_preview" | "code_interpreter" | "container" + ) + } + + fn is_custom(&self) -> bool { + matches!(self.tool_type.as_str(), "mcp" | "function") + } + + fn to_tool_description(&self) -> Option { + Some(ToolDescription::new( + self.function.name.clone(), + self.function.description.clone().unwrap_or_default(), + Some(self.function.parameters.clone()), + )) + } +} + +/// Implement ToolLike for Responses API Tool +impl ToolLike for ResponseTool { + fn is_builtin(&self) -> bool { + matches!( + self.r#type, + ResponseToolType::WebSearchPreview | ResponseToolType::CodeInterpreter + ) + } + + fn is_custom(&self) -> bool { + matches!( + self.r#type, + ResponseToolType::Mcp | ResponseToolType::Function + ) + } + + fn to_tool_description(&self) -> Option { + self.function.as_ref().map(|func| { + ToolDescription::new( + func.name.clone(), + func.description.clone().unwrap_or_default(), + Some(func.parameters.clone()), + ) + }) + } +} + +fn has_custom_tools(tool_types: &[&str]) -> bool { + !tool_types.iter().all(|t| BUILTIN_TOOLS.contains(t)) +} + +/// Harmony request builder +/// +/// Converts OpenAI-format requests into Harmony-encoded format with input_ids, +/// stop tokens, and selection text for worker routing. +pub struct HarmonyBuilder { + encoding: &'static HarmonyEncoding, +} + +impl HarmonyBuilder { + /// Create a new Harmony builder + pub fn new() -> Self { + Self { + encoding: get_harmony_encoding(), + } + } + + /// Build Harmony request from Chat Completion request + /// + /// # Arguments + /// + /// * `request` - The ChatCompletionRequest to encode + /// + /// # Returns + /// + /// HarmonyBuildOutput containing input_ids, stop_token_ids, selection_text, and messages + pub fn build_from_chat( + &self, + request: &ChatCompletionRequest, + ) -> Result { + let mut all_messages = Vec::new(); + + let sys_msg = self.build_system_message_from_chat(request); + all_messages.push(sys_msg); + + let dev_msg = self.build_developer_message_from_chat(request.tools.as_ref()); + all_messages.push(dev_msg); + + let mut user_messages = self.convert_chat_messages(&request.messages)?; + all_messages.append(&mut user_messages); + + let conversation = Conversation::from_messages(all_messages.clone()); + let token_ids = self + .encoding + .render_conversation_for_completion(&conversation, Role::Assistant, None) + .map_err(|e| format!("Failed to encode Harmony conversation: {}", e))?; + + let selection_text = self.extract_selection_text(&all_messages); + + // Get stop tokens for Harmony assistant actions (<|return|> and <|call|>) + let stop_token_ids: Vec = self + .encoding + .stop_tokens_for_assistant_actions() + .into_iter() + .flat_map(|set| set.into_iter()) + .collect(); + + Ok(HarmonyBuildOutput { + input_ids: token_ids, + stop_token_ids, + selection_text, + harmony_messages: all_messages + .into_iter() + .map(super::types::HarmonyMessage::from_openai_harmony) + .collect(), + }) + } + + /// Build Harmony request from Responses request + /// + /// # Arguments + /// + /// * `request` - The ResponsesRequest to encode + /// + /// # Returns + /// + /// HarmonyBuildOutput containing input_ids, stop_token_ids, selection_text, and messages + pub fn build_from_responses( + &self, + request: &ResponsesRequest, + ) -> Result { + let all_messages = self.construct_input_messages_with_harmony(request)?; + + let conversation = Conversation::from_messages(all_messages.clone()); + let token_ids = self + .encoding + .render_conversation_for_completion(&conversation, Role::Assistant, None) + .map_err(|e| format!("Failed to encode Harmony conversation: {}", e))?; + + let selection_text = self.extract_selection_text(&all_messages); + + // Get stop tokens for Harmony assistant actions (<|return|> and <|call|>) + let stop_token_ids: Vec = self + .encoding + .stop_tokens_for_assistant_actions() + .into_iter() + .flat_map(|set| set.into_iter()) + .collect(); + + // Decode tokens to see what the model actually receives + let decoded_text = self + .encoding + .tokenizer() + .decode_utf8(&token_ids) + .unwrap_or_else(|_| "".to_string()); + debug!( + token_count = token_ids.len(), + token_preview = ?&token_ids[..token_ids.len().min(20)], + decoded_length = decoded_text.len(), + "Encoded conversation to tokens - decoded text follows:" + ); + debug!("DECODED_TEXT_START\n{}\nDECODED_TEXT_END", decoded_text); + + Ok(HarmonyBuildOutput { + input_ids: token_ids, + stop_token_ids, + selection_text, + harmony_messages: all_messages + .into_iter() + .map(super::types::HarmonyMessage::from_openai_harmony) + .collect(), + }) + } + + /// Build system message from ChatCompletionRequest + /// Build system message with common logic + /// + /// # Arguments + /// * `reasoning_effort` - Optional reasoning effort level + /// * `has_tools` - Whether custom tools are present + fn build_system_message( + &self, + reasoning_effort: Option, + has_tools: bool, + ) -> HarmonyMessage { + let mut sys_content = SystemContent::new(); + + // Add reasoning_effort if provided + if let Some(effort) = reasoning_effort { + sys_content = sys_content.with_reasoning_effort(effort); + } + + // Set conversation start date (always current date) + sys_content = + sys_content.with_conversation_start_date(Local::now().format("%Y-%m-%d").to_string()); + + // If no tools, remove "commentary" from valid channels + if !has_tools { + if let Some(channel_config) = &sys_content.channel_config { + let valid_channels: Vec = channel_config + .valid_channels + .iter() + .filter(|c| c.as_str() != "commentary") + .cloned() + .collect(); + sys_content = sys_content + .with_channel_config(ChannelConfig::require_channels(valid_channels)); + } + } + + HarmonyMessage::from_role_and_content(Role::System, sys_content) + } + + fn build_system_message_from_chat(&self, request: &ChatCompletionRequest) -> HarmonyMessage { + let reasoning_effort = request + .reasoning_effort + .as_deref() + .map(|effort| match effort { + "high" => ReasoningEffort::High, + "medium" => ReasoningEffort::Medium, + "low" => ReasoningEffort::Low, + // Harmony does not support minimal reasoning effort + "minimal" => ReasoningEffort::Low, + _ => ReasoningEffort::Medium, + }); + + let has_tools = request.tools.is_some(); + self.build_system_message(reasoning_effort, has_tools) + } + + /// Build system message from ResponsesRequest + /// + /// # Arguments + /// * `request` - The ResponsesRequest + /// * `with_custom_tools` - Whether custom tools (beyond built-ins) are present + fn build_system_message_from_responses( + &self, + request: &ResponsesRequest, + with_custom_tools: bool, + ) -> HarmonyMessage { + let reasoning_effort = request + .reasoning + .as_ref() + .and_then(|r| r.effort.as_ref()) + .map(|effort| match effort { + ResponsesReasoningEffort::High => ReasoningEffort::High, + ResponsesReasoningEffort::Medium => ReasoningEffort::Medium, + ResponsesReasoningEffort::Low => ReasoningEffort::Low, + ResponsesReasoningEffort::Minimal => ReasoningEffort::Low, + }); + + self.build_system_message(reasoning_effort, with_custom_tools) + } + + /// Build developer message with common logic + /// + /// Filters out built-in tools and converts custom tools to ToolDescription + /// + /// # Arguments + /// * `tools` - Optional list of tools + /// * `instructions` - Optional instructions (Responses API only) + fn build_developer_message( + &self, + tools: Option<&Vec>, + instructions: Option<&str>, + ) -> HarmonyMessage { + let mut dev_content = DeveloperContent::new(); + + // Add instructions if provided (Responses API only) + if let Some(instructions) = instructions { + dev_content = dev_content.with_instructions(instructions.to_string()); + } + + // Early return if no tools + let Some(tools) = tools else { + return HarmonyMessage::from_role_and_content(Role::Developer, dev_content); + }; + + // Filter to custom tools and convert to ToolDescription + let tool_descriptions: Vec = tools + .iter() + .filter(|t| t.is_custom()) + .filter_map(|t| t.to_tool_description()) + .collect(); + + // Add function tools to developer content + if !tool_descriptions.is_empty() { + dev_content = dev_content.with_function_tools(tool_descriptions); + } + + HarmonyMessage::from_role_and_content(Role::Developer, dev_content) + } + + fn build_developer_message_from_chat(&self, tools: Option<&Vec>) -> HarmonyMessage { + self.build_developer_message(tools, None) + } + + /// Build developer message from Responses request + /// + /// # Arguments + /// * `instructions` - Optional instructions (Responses API specific) + /// * `tools` - Optional list of tools + fn build_developer_message_from_responses( + &self, + instructions: Option<&str>, + tools: Option<&Vec>, + ) -> HarmonyMessage { + self.build_developer_message(tools, instructions) + } + + /// Construct input messages for Responses API with Harmony + /// + /// Handles both new conversations and continuations of previous responses. + /// + /// This handles: + /// - New conversation: system message, developer message, and user input + /// - Continuing conversation: loads previous messages, cleans up chain-of-thoughts + /// - MCP tool allowlisting for special tool types + /// - Complex response input parsing with function call tracking + /// + /// # Arguments + /// * `request` - The ResponsesRequest + /// * `prev_response` - Optional previous response to continue from + fn construct_input_messages_with_harmony( + &self, + request: &ResponsesRequest, + ) -> Result, String> { + let mut all_messages = Vec::new(); + + // Handle new vs continuing conversation + if request.previous_response_id.is_none() { + // New conversation + + let tool_types: Vec<&str> = request + .tools + .as_ref() + .map(|tools| { + tools + .iter() + .map(|tool| match tool.r#type { + ResponseToolType::Function => "function", + ResponseToolType::WebSearchPreview => "web_search_preview", + ResponseToolType::CodeInterpreter => "code_interpreter", + ResponseToolType::Mcp => "mcp", + }) + .collect() + }) + .unwrap_or_default(); + + let with_custom_tools = has_custom_tools(&tool_types); + + // Add system message + let sys_msg = self.build_system_message_from_responses(request, with_custom_tools); + all_messages.push(sys_msg); + + // Add developer message only if we have custom tools + if with_custom_tools { + let dev_msg = self.build_developer_message_from_responses( + request.instructions.as_deref(), + request.tools.as_ref(), + ); + all_messages.push(dev_msg); + } + } else { + // Continue the previous conversation + // NOTE: Previous messages are loaded by serve_harmony_responses() before calling this method. + // The request.input will already contain the conversation history when previous_response_id was set. + // We just proceed with parsing the input items as normal. + debug!("Continuing conversation (history already loaded in request.input)"); + } + + // Append the new input + // Responses API supports simple text inputs without chat format + match &request.input { + ResponseInput::Text(text) => { + let user_msg = HarmonyMessage { + author: Author { + role: Role::User, + name: None, + }, + recipient: None, + content: vec![Content::Text(TextContent { text: text.clone() })], + channel: None, + content_type: None, + }; + all_messages.push(user_msg); + } + ResponseInput::Items(items) => { + // Track function calls for looking up call_id → name mapping + let mut prev_outputs: Vec<&ResponseInputOutputItem> = Vec::new(); + + for item in items { + let msg = self.parse_response_item_to_harmony_message(item, &prev_outputs)?; + all_messages.push(msg); + + // Track function tool calls so that function_call_output can find the name + if matches!(item, ResponseInputOutputItem::FunctionToolCall { .. }) { + prev_outputs.push(item); + } + } + } + } + + debug!( + message_count = all_messages.len(), + "Constructed Harmony messages for Responses API" + ); + Ok(all_messages) + } + + /// Parse a ResponseInputOutputItem into a HarmonyMessage + /// + /// Handles conversion of various response item types (messages, function calls, reasoning, etc.) + /// to Harmony message format. + /// + /// # Arguments + /// * `item` - The ResponseInputOutputItem to parse + /// * `prev_outputs` - Previous items for looking up function call names (for function_call_output) + fn parse_response_item_to_harmony_message( + &self, + item: &ResponseInputOutputItem, + prev_outputs: &[&ResponseInputOutputItem], + ) -> Result { + match item { + // Regular message (user or assistant) + ResponseInputOutputItem::Message { role, content, .. } => { + let harmony_role = match role.as_str() { + "user" => Role::User, + "assistant" => Role::Assistant, + "system" => Role::System, + _ => Role::User, // Default to user for unknown roles + }; + + // Extract text from content parts + let text_parts: Vec = content + .iter() + .filter_map(|part| match part { + ResponseContentPart::OutputText { text, .. } => Some(text.clone()), + ResponseContentPart::InputText { text } => Some(text.clone()), + ResponseContentPart::Unknown => None, + }) + .collect(); + + let text = text_parts.join("\n"); + + Ok(HarmonyMessage { + author: Author { + role: harmony_role, + name: None, + }, + recipient: None, + content: vec![Content::Text(TextContent { text })], + channel: None, + content_type: None, + }) + } + + // Reasoning content (chain-of-thought) + ResponseInputOutputItem::Reasoning { content, .. } => { + // Extract reasoning text + let reasoning_texts: Vec = content + .iter() + .map(|rc| match rc { + ResponseReasoningContent::ReasoningText { text } => text.clone(), + }) + .collect(); + + let text = reasoning_texts.join("\n"); + + // Reasoning goes in the "analysis" channel for Harmony + Ok(HarmonyMessage { + author: Author { + role: Role::Assistant, + name: None, + }, + recipient: None, + content: vec![Content::Text(TextContent { text })], + channel: Some("analysis".to_string()), + content_type: None, + }) + } + + // Function tool call (with optional output) + ResponseInputOutputItem::FunctionToolCall { + name, + arguments, + output, + .. + } => { + // If there's an output, this represents the tool result + // Otherwise, it's the tool call itself + if let Some(output_str) = output { + // Tool result - use Tool role with "functions.{name}" as author name + // IMPORTANT: Must include recipient="assistant" for parser to recognize it. + // We keep channel=None to minimize what the model might copy. + let author_name = format!("functions.{}", name); + debug!( + tool_name = %name, + author_name = %author_name, + output_preview = %output_str.chars().take(100).collect::(), + "Building tool result message with Tool role (recipient=assistant, no channel)" + ); + Ok(HarmonyMessage { + author: Author { + role: Role::Tool, + name: Some(author_name), + }, + recipient: Some("assistant".to_string()), + content: vec![Content::Text(TextContent { + text: output_str.clone(), + })], + channel: None, + content_type: None, + }) + } else { + // Tool call - assistant message in commentary channel with recipient + // msg.with_channel("commentary").with_recipient(f"functions.{name}") + let recipient = format!("functions.{}", name); + debug!( + tool_name = %name, + recipient = %recipient, + "Building tool call message with recipient" + ); + Ok(HarmonyMessage { + author: Author { + role: Role::Assistant, + name: None, + }, + recipient: Some(recipient), + content: vec![Content::Text(TextContent { + text: arguments.clone(), + })], + channel: Some("commentary".to_string()), + content_type: Some("json".to_string()), + }) + } + } + + // Function call output (separate from call) - requires looking up the original call + ResponseInputOutputItem::FunctionCallOutput { + call_id, output, .. + } => { + // Search prev_outputs in reverse order to find the matching function call + let call = prev_outputs + .iter() + .rev() + .find_map(|item| match item { + ResponseInputOutputItem::FunctionToolCall { + call_id: item_call_id, + name, + .. + } if item_call_id == call_id => Some(name.clone()), + _ => None, + }) + .ok_or_else(|| format!("No function call found for call_id: {}", call_id))?; + + // Create Tool message with "functions.{name}" prefix + // IMPORTANT: Must include recipient="assistant" for parser to recognize it. + // We keep channel=None to minimize what the model might copy. + Ok(HarmonyMessage { + author: Author { + role: Role::Tool, + name: Some(format!("functions.{}", call)), + }, + recipient: Some("assistant".to_string()), + content: vec![Content::Text(TextContent { + text: output.clone(), + })], + channel: None, + content_type: None, + }) + } + + // Simple input message (usually user message) + ResponseInputOutputItem::SimpleInputMessage { content, role, .. } => { + let harmony_role = match role.as_str() { + "user" => Role::User, + "assistant" => Role::Assistant, + "system" => Role::System, + _ => Role::User, + }; + + let text = match content { + StringOrContentParts::String(s) => s.clone(), + StringOrContentParts::Array(parts) => { + // Extract text from content parts + parts + .iter() + .filter_map(|part| match part { + ResponseContentPart::OutputText { text, .. } => Some(text.clone()), + ResponseContentPart::InputText { text } => Some(text.clone()), + ResponseContentPart::Unknown => None, + }) + .collect::>() + .join("\n") + } + }; + + Ok(HarmonyMessage { + author: Author { + role: harmony_role, + name: None, + }, + recipient: None, + content: vec![Content::Text(TextContent { text })], + channel: None, + content_type: None, + }) + } + } + } + + /// Convert OpenAI ChatMessage format to Harmony messages + /// + /// - Assistant messages with tool_calls create multiple messages (one per tool call) + /// - Tool role messages use Role::Tool with proper author + /// - Tool-related messages use channel="commentary" + fn convert_chat_messages( + &self, + messages: &[ChatMessage], + ) -> Result, String> { + let mut harmony_messages = Vec::new(); + + // Build a map of tool_call_id -> function_name for tool responses + let mut tool_call_map = std::collections::HashMap::new(); + for msg in messages { + if let ChatMessage::Assistant { + tool_calls: Some(calls), + .. + } = msg + { + for call in calls { + tool_call_map.insert(call.id.clone(), call.function.name.clone()); + } + } + } + + for msg in messages { + match msg { + ChatMessage::System { content, name } => { + // System messages stay as-is + let harmony_msg = HarmonyMessage { + author: Author { + role: Role::System, + name: name.clone(), + }, + recipient: None, + content: vec![Content::Text(TextContent { + text: content.to_simple_string(), + })], + channel: None, + content_type: None, + }; + harmony_messages.push(harmony_msg); + } + + ChatMessage::User { content, name } => { + // Extract text from user content + let text = match content { + MessageContent::Text(text) => text.clone(), + MessageContent::Parts(parts) => { + // For multimodal content, extract text parts + parts + .iter() + .filter_map(|part| { + if let ContentPart::Text { text } = part { + Some(text.as_str()) + } else { + None + } + }) + .collect::>() + .join("\n") + } + }; + + let harmony_msg = HarmonyMessage { + author: Author { + role: Role::User, + name: name.clone(), + }, + recipient: None, + content: vec![Content::Text(TextContent { text })], + channel: None, + content_type: None, + }; + harmony_messages.push(harmony_msg); + } + + ChatMessage::Assistant { + content, + name, + tool_calls, + reasoning_content, + } => { + if let Some(calls) = tool_calls { + // Create one message per tool call with channel="commentary" + for call in calls { + let function_name = &call.function.name; + let arguments = call.function.arguments.clone().unwrap_or_default(); + + let tool_call_msg = HarmonyMessage { + author: Author { + role: Role::Assistant, + name: name.clone(), + }, + recipient: Some(format!("functions.{}", function_name)), + content: vec![Content::Text(TextContent { text: arguments })], + channel: Some("commentary".to_string()), + content_type: Some("json".to_string()), + }; + harmony_messages.push(tool_call_msg); + } + } else { + // Regular assistant message with content + // Combine content with reasoning if present + let mut text = content + .as_ref() + .map(|c| c.to_simple_string()) + .unwrap_or_default(); + + if let Some(reasoning) = reasoning_content { + if !text.is_empty() { + text.push('\n'); + } + text.push_str(reasoning); + } + + let harmony_msg = HarmonyMessage { + author: Author { + role: Role::Assistant, + name: name.clone(), + }, + recipient: None, + content: vec![Content::Text(TextContent { text })], + channel: Some("final".to_string()), + content_type: None, + }; + harmony_messages.push(harmony_msg); + } + } + + ChatMessage::Tool { + content, + tool_call_id, + } => { + // Look up the function name from the tool_call_id + let function_name = tool_call_map + .get(tool_call_id) + .cloned() + .unwrap_or_else(|| tool_call_id.clone()); + + // Tool result - Must include recipient="assistant" for parser to recognize it. + // We keep channel=None to minimize what the model might copy. + let harmony_msg = HarmonyMessage { + author: Author { + role: Role::Tool, + name: Some(format!("functions.{}", function_name)), + }, + recipient: Some("assistant".to_string()), + content: vec![Content::Text(TextContent { + text: content.to_simple_string(), + })], + channel: None, + content_type: None, + }; + harmony_messages.push(harmony_msg); + } + + ChatMessage::Function { content, name } => { + // Function messages also use Role::Tool + // Tool result - Must include recipient="assistant" for parser to recognize it. + // We keep channel=None to minimize what the model might copy. + let harmony_msg = HarmonyMessage { + author: Author { + role: Role::Tool, + name: Some(format!("functions.{}", name)), + }, + recipient: Some("assistant".to_string()), + content: vec![Content::Text(TextContent { + text: content.clone(), + })], + channel: None, + content_type: None, + }; + harmony_messages.push(harmony_msg); + } + } + } + + Ok(harmony_messages) + } + + /// Extract selection text for worker routing + /// + /// Uses the last user message for load balancing + fn extract_selection_text(&self, messages: &[HarmonyMessage]) -> String { + // Find the last user message + if let Some(last_user_msg) = messages.iter().rev().find(|m| m.author.role == Role::User) { + // Extract full text from content + return last_user_msg + .content + .iter() + .filter_map(|c| match c { + Content::Text(tc) => Some(tc.text.as_str()), + _ => None, + }) + .collect::>() + .join(""); + } + + // Fallback: concatenate all text + messages + .iter() + .flat_map(|m| &m.content) + .filter_map(|c| match c { + Content::Text(tc) => Some(tc.text.as_str()), + _ => None, + }) + .collect::>() + .join(" ") + } +} + +impl Default for HarmonyBuilder { + fn default() -> Self { + Self::new() + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/detector.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/detector.rs new file mode 100644 index 00000000..88ef6a6a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/detector.rs @@ -0,0 +1,17 @@ +//! Harmony model detection + +/// Harmony model detector +/// +/// Detects if a model name indicates support for Harmony encoding/parsing. +pub struct HarmonyDetector; + +impl HarmonyDetector { + pub fn is_harmony_model(model_name: &str) -> bool { + // Case-insensitive substring search without heap allocation + // More efficient than to_lowercase() which allocates a new String + model_name + .as_bytes() + .windows(7) // "gpt-oss".len() + .any(|window| window.eq_ignore_ascii_case(b"gpt-oss")) + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/mod.rs new file mode 100644 index 00000000..58f45756 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/mod.rs @@ -0,0 +1,56 @@ +//! Harmony pipeline implementation +//! +//! This module provides support for GPT-OSS models that use Harmony encoding/parsing. +//! The Harmony protocol uses a channel-based approach with three channels: +//! - **analysis**: Reasoning/thinking content (optional) +//! - **commentary**: Tool calls (optional) +//! - **final**: Final response text (required) +//! +//! ## Architecture +//! +//! The Harmony implementation is structured as follows: +//! +//! - **detector**: Model detection (is this a Harmony-capable model?) +//! - **builder**: Request encoding (convert Chat/Responses → input_ids) +//! - **parser**: Response parsing (output_ids → channels) +//! - **types**: Shared type definitions +//! +//! ## Usage +//! +//! ```ignore +//! use sgl_model_gateway::routers::grpc::harmony::{HarmonyDetector, HarmonyBuilder}; +//! +//! // Detect if model supports Harmony +//! if HarmonyDetector::is_harmony_model("gpt-4o") { +//! // Build Harmony request +//! let builder = HarmonyBuilder::new(); +//! let output = builder.build_from_chat(&request)?; +//! // ... use output.input_ids for gRPC request +//! } +//! ``` + +pub mod builder; +pub mod detector; +pub mod parser; +pub mod processor; +pub mod responses; +pub mod stages; +pub mod streaming; +pub mod types; + +// Re-export main types for convenience +pub use builder::HarmonyBuilder; +pub use detector::HarmonyDetector; +pub use parser::HarmonyParserAdapter; +pub use processor::{HarmonyResponseProcessor, ResponsesIterationResult}; +pub use responses::{ + serve_harmony_responses, serve_harmony_responses_stream, HarmonyResponsesContext, +}; +pub use stages::{ + HarmonyPreparationStage, HarmonyRequestBuildingStage, HarmonyResponseProcessingStage, +}; +pub use streaming::HarmonyStreamingProcessor; +pub use types::{ + FunctionDelta, HarmonyBuildOutput, HarmonyChannelDelta, HarmonyChannelOutput, HarmonyMessage, + ToolCallDelta, +}; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/parser.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/parser.rs new file mode 100644 index 00000000..bdc933a0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/parser.rs @@ -0,0 +1,535 @@ +//! Harmony response parser +//! +//! Adapter for openai_harmony::StreamableParser that handles channel-based parsing. + +use openai_harmony::{chat::Role, HarmonyEncoding, StreamableParser}; +use uuid::Uuid; + +use super::types::{HarmonyChannelDelta, HarmonyChannelOutput}; +use crate::protocols::common::{FunctionCallResponse, ToolCall}; + +/// Get the global Harmony encoding +/// +/// References the same encoding used by the builder for consistency +fn get_harmony_encoding() -> &'static HarmonyEncoding { + use super::builder::get_harmony_encoding; + get_harmony_encoding() +} + +/// Harmony parser adapter +/// +/// Wraps openai_harmony::StreamableParser and provides methods for parsing +/// complete responses and streaming chunks. +pub struct HarmonyParserAdapter { + parser: StreamableParser, + prev_recipient: Option, + reasoning_token_count: u32, +} + +impl HarmonyParserAdapter { + /// Create a new Harmony parser + pub fn new() -> Result { + let encoding = get_harmony_encoding(); + let parser = StreamableParser::new(encoding.clone(), Some(Role::Assistant)) + .map_err(|e| format!("Failed to create StreamableParser: {}", e))?; + + Ok(Self { + parser, + prev_recipient: None, + reasoning_token_count: 0, + }) + } + + /// Extract text from message content (private helper) + /// + /// Filters text content from a message's content array and joins them into a single string. + /// + /// # Arguments + /// + /// * `content` - The content array from a Harmony message + /// + /// # Returns + /// + /// Joined text string from all text content items + fn extract_text_from_content(content: &[openai_harmony::chat::Content]) -> String { + content + .iter() + .filter_map(|c| match c { + openai_harmony::chat::Content::Text(tc) => Some(tc.text.as_str()), + _ => None, + }) + .collect::>() + .join("") + } + + /// Handle incomplete content from parser state (private helper) + /// + /// Checks for any remaining incomplete content in the parser and appends it + /// to the appropriate channel (analysis or final_text). + /// + /// # Arguments + /// + /// * `parser` - Reference to the StreamableParser + /// * `analysis` - Mutable reference to analysis content + /// * `final_text` - Mutable reference to final text content + fn handle_incomplete_content( + parser: &StreamableParser, + analysis: &mut Option, + final_text: &mut String, + ) { + if let Ok(current_content) = parser.current_content() { + if !current_content.is_empty() { + let current_channel = parser.current_channel(); + match current_channel.as_deref() { + Some("analysis") => { + *analysis = Some(current_content); + } + Some("final") | None => { + final_text.push_str(¤t_content); + } + _ => {} + } + } + } + } + + /// Parse messages into channel outputs (private helper) + /// + /// Extracts analysis, commentary (tool calls), and final text from Harmony messages. + /// This is the core parsing logic shared by both parse_complete and finalize. + /// + /// # Arguments + /// + /// * `messages` - The messages to parse from the Harmony parser + /// + /// # Returns + /// + /// Tuple of (analysis, commentary, final_text) + pub fn parse_messages( + messages: &[openai_harmony::chat::Message], + ) -> (Option, Option>, String) { + let mut analysis: Option = None; + let mut commentary: Option> = None; + let mut final_text = String::new(); + + for msg in messages { + // Filter: Only process assistant messages + if msg.author.role != Role::Assistant { + continue; + } + + let channel = msg.channel.as_deref().unwrap_or(""); + let recipient = msg.recipient.as_deref(); + + // IMPORTANT: Check recipient FIRST before channel + // The model sometimes generates tool calls with channel="analysis" + recipient="functions.*" + // instead of channel="commentary" + recipient="functions.*" + // We should trust the recipient field to determine if this is a tool call + if let Some(recipient_str) = recipient { + if recipient_str.starts_with("functions.") { + // This is a tool call, regardless of channel + let function_name = recipient_str.strip_prefix("functions.").unwrap(); + + // Process each content item separately + for content in &msg.content { + if let openai_harmony::chat::Content::Text(tc) = content { + let call_id = format!("call_{}", Uuid::new_v4()); + let tool_call = ToolCall { + id: call_id, + tool_type: "function".to_string(), + function: FunctionCallResponse { + name: function_name.to_string(), + arguments: Some(tc.text.clone()), + }, + }; + + match commentary.as_mut() { + Some(calls) => calls.push(tool_call), + None => commentary = Some(vec![tool_call]), + } + } + } + // Skip further channel processing for this message + continue; + } else if recipient_str.starts_with("python") + || recipient_str.starts_with("browser") + || recipient_str.starts_with("container") + { + // Built-in tools → treat as reasoning + // For Chat API, we add to analysis content + let text = Self::extract_text_from_content(&msg.content); + + if !text.is_empty() { + // Append to analysis (built-in tools are reasoning) + match analysis.as_mut() { + Some(existing) => { + existing.push('\n'); + existing.push_str(&text); + } + None => analysis = Some(text), + } + } + // Skip further channel processing + continue; + } + } + + // Now process by channel (only if not already handled by recipient) + match channel { + "analysis" => { + // Process each content item + // For Chat API, we join them into a single reasoning_content + let text = Self::extract_text_from_content(&msg.content); + + if !text.is_empty() { + analysis = Some(text); + } + } + "commentary" => { + // If we reach here, recipient was not "functions.*" or built-in tools + // Commentary channel should always have a recipient + // This is likely a model bug - log warning and treat as reasoning + tracing::warn!( + channel = "commentary", + recipient = ?recipient, + "Commentary message without valid recipient, treating as reasoning" + ); + + let text = Self::extract_text_from_content(&msg.content); + + if !text.is_empty() { + match analysis.as_mut() { + Some(existing) => { + existing.push('\n'); + existing.push_str(&text); + } + None => analysis = Some(text), + } + } + } + "final" => { + // Process final channel content + let text = Self::extract_text_from_content(&msg.content); + final_text.push_str(&text); + } + _ => { + // Unknown channel, append to final text as fallback + let text = Self::extract_text_from_content(&msg.content); + final_text.push_str(&text); + } + } + } + + (analysis, commentary, final_text) + } + + /// Parse complete response + /// + /// Parses all output token IDs and returns the complete channel output + /// containing analysis, commentary (tool calls), and final text. + /// + /// # Arguments + /// + /// * `output_ids` - The complete output token IDs from the model + /// * `finish_reason` - The finish reason from GenerateComplete ("stop", "length", etc.) + /// * `matched_stop` - Optional matched stop token information from GenerateComplete + /// + /// # Returns + /// + /// Complete HarmonyChannelOutput with all three channels parsed + pub fn parse_complete( + &mut self, + output_ids: &[u32], + finish_reason: String, + matched_stop: Option, + ) -> Result { + let mut reasoning_token_count = 0u32; + + // Feed all tokens to the parser + for &token_id in output_ids { + self.parser + .process(token_id) + .map_err(|e| format!("Failed to process token {}: {}", token_id, e))?; + + // Count reasoning tokens (analysis + commentary channels) + if let Some(channel) = self.parser.current_channel() { + if channel == "analysis" || channel == "commentary" { + reasoning_token_count += 1; + } + } + } + + // Extract all completed messages from the parser + let messages = self.parser.messages(); + + // Parse messages into channel outputs using shared helper + let (mut analysis, commentary, mut final_text) = Self::parse_messages(messages); + + // Check for incomplete content in parser state + Self::handle_incomplete_content(&self.parser, &mut analysis, &mut final_text); + + // Determine finish reason: override to "tool_calls" if commentary has tool calls + let final_finish_reason = if commentary.is_some() { + "tool_calls".to_string() + } else { + finish_reason.clone() + }; + + Ok(HarmonyChannelOutput { + analysis, + commentary, + final_text, + finish_reason: final_finish_reason, + matched_stop, + reasoning_token_count, + }) + } + + /// Get all messages from the parser + /// + /// Returns the raw messages extracted by the Harmony parser. + /// Used for validation checks. + pub fn get_messages(&self) -> Vec { + self.parser.messages().to_vec() + } + + /// Extract incomplete commentary content from parser state + /// + /// When the stream ends, there may be incomplete commentary content in the parser + /// that hasn't been finalized into a completed message. This method extracts + /// such content and converts it to tool calls. + /// + /// # Returns + /// + /// Optional vector of ToolCall if incomplete commentary is found + pub fn extract_incomplete_commentary(&self) -> Option> { + // Check if current channel is commentary + let current_channel = self.parser.current_channel(); + if current_channel.as_deref() != Some("commentary") { + return None; + } + + // Get current recipient (should be "functions.{name}") + let recipient = self.parser.current_recipient()?; + if !recipient.starts_with("functions.") { + return None; + } + + // Get current incomplete content + let content = self.parser.current_content().ok()?; + if content.is_empty() { + return None; + } + + // Extract function name from recipient + let function_name = recipient.strip_prefix("functions.").unwrap(); + + // Create tool call from incomplete content + let call_id = format!("call_{}", Uuid::new_v4()); + let tool_call = ToolCall { + id: call_id, + tool_type: "function".to_string(), + function: FunctionCallResponse { + name: function_name.to_string(), + arguments: Some(content), + }, + }; + + Some(vec![tool_call]) + } + + /// Parse streaming chunk + /// + /// Parses incremental token IDs and returns a delta with any new content + /// from the analysis, commentary, or final channels. + /// + /// # Arguments + /// + /// * `chunk_ids` - New token IDs from the current chunk + /// + /// # Returns + /// + /// Optional HarmonyChannelDelta if there's new content to emit + pub fn parse_chunk( + &mut self, + chunk_ids: &[u32], + ) -> Result, String> { + let mut has_delta = false; + let mut analysis_delta = None; + let mut commentary_delta = None; + let mut final_delta = None; + + // Track message count before processing + let prev_message_count = self.parser.messages().len(); + + // Accumulate delta text for commentary channel + let mut accumulated_delta = String::new(); + + // Process each token + for &token_id in chunk_ids { + self.parser + .process(token_id) + .map_err(|e| format!("Failed to process token {}: {}", token_id, e))?; + + // Count reasoning tokens (analysis + commentary channels) + if let Some(channel) = self.parser.current_channel() { + if channel == "analysis" || channel == "commentary" { + self.reasoning_token_count += 1; + } + } + + // Check for content delta + if let Ok(Some(delta_text)) = self.parser.last_content_delta() { + has_delta = true; + + // Determine which channel this delta belongs to + let channel = self.parser.current_channel(); + match channel.as_deref() { + Some("analysis") => { + analysis_delta = Some(delta_text); + } + Some("final") | None => { + final_delta = Some(delta_text); + } + Some("commentary") => { + // Accumulate delta for commentary + accumulated_delta.push_str(&delta_text); + } + _ => {} + } + } + } + + // Handle commentary channel tool call deltas + if self.parser.current_channel().as_deref() == Some("commentary") { + if let Some(cur_recipient) = self.parser.current_recipient() { + if cur_recipient.starts_with("functions.") { + has_delta = true; + + // Count completed tool calls for index + let base_index = self + .parser + .messages() + .iter() + .filter(|msg| { + msg.channel.as_deref() == Some("commentary") + && msg + .recipient + .as_deref() + .is_some_and(|r| r.starts_with("functions.")) + }) + .count(); + + // Check if recipient changed (new tool call) + let recipient_changed = self.prev_recipient.as_deref() != Some(&cur_recipient); + + if recipient_changed { + // NEW tool call: emit name + id + let tool_name = cur_recipient.strip_prefix("functions.").unwrap(); + let call_id = format!("call_{}", Uuid::new_v4()); + + commentary_delta = Some(super::types::ToolCallDelta { + index: base_index, + id: Some(call_id), + function: Some(super::types::FunctionDelta { + name: Some(tool_name.to_string()), + arguments: Some(String::new()), + }), + }); + + // Update prev_recipient + self.prev_recipient = Some(cur_recipient); + } else if !accumulated_delta.is_empty() { + // CONTINUING tool call: emit arguments delta + commentary_delta = Some(super::types::ToolCallDelta { + index: base_index, + id: None, + function: Some(super::types::FunctionDelta { + name: None, + arguments: Some(accumulated_delta), + }), + }); + } + } + } + } + + // Check if new messages were completed + let current_message_count = self.parser.messages().len(); + let is_final = current_message_count > prev_message_count; + + if has_delta { + Ok(Some(HarmonyChannelDelta { + analysis_delta, + commentary_delta, + final_delta, + is_final, + })) + } else { + Ok(None) + } + } + + /// Finalize parsing + /// + /// Called at the end of streaming to get the final state and any + /// remaining content. + /// + /// # Arguments + /// + /// * `finish_reason` - The finish reason from GenerateComplete ("stop", "length", etc.) + /// * `matched_stop` - Optional matched stop token information from GenerateComplete + /// + /// # Returns + /// + /// Final HarmonyChannelOutput with complete parsed content + pub fn finalize( + &mut self, + finish_reason: String, + matched_stop: Option, + ) -> Result { + // Extract all completed messages + let messages = self.parser.messages(); + + // Parse messages into channel outputs using shared helper + let (mut analysis, commentary, mut final_text) = Self::parse_messages(messages); + + // Check for remaining incomplete content + Self::handle_incomplete_content(&self.parser, &mut analysis, &mut final_text); + + // Determine finish reason: override to "tool_calls" if commentary has tool calls + let final_finish_reason = if commentary.is_some() { + "tool_calls".to_string() + } else { + finish_reason + }; + + Ok(HarmonyChannelOutput { + analysis, + commentary, + final_text, + finish_reason: final_finish_reason, + matched_stop, + reasoning_token_count: self.reasoning_token_count, + }) + } + + /// Reset parser state + /// + /// Resets the parser to initial state for reuse + pub fn reset(&mut self) -> Result<(), String> { + // Create a new parser instance (StreamableParser doesn't have a reset method) + let encoding = get_harmony_encoding(); + self.parser = StreamableParser::new(encoding.clone(), Some(Role::Assistant)) + .map_err(|e| format!("Failed to reset parser: {}", e))?; + self.prev_recipient = None; + self.reasoning_token_count = 0; + Ok(()) + } +} + +impl Default for HarmonyParserAdapter { + fn default() -> Self { + Self::new().expect("Failed to create default parser") + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/processor.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/processor.rs new file mode 100644 index 00000000..deabab1a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/processor.rs @@ -0,0 +1,323 @@ +//! Harmony response processor for non-streaming responses + +use std::sync::Arc; + +use axum::response::Response; +use tracing::error; + +use super::HarmonyParserAdapter; +use crate::{ + grpc_client::sglang_proto::generate_complete::MatchedStop::{MatchedStopStr, MatchedTokenId}, + protocols::{ + chat::{ChatChoice, ChatCompletionMessage, ChatCompletionRequest, ChatCompletionResponse}, + common::{CompletionTokensDetails, ToolCall, Usage}, + responses::{ + OutputTokensDetails, ResponseContentPart, ResponseOutputItem, ResponseReasoningContent, + ResponseStatus, ResponseUsage, ResponsesRequest, ResponsesResponse, ResponsesUsage, + }, + }, + routers::grpc::{ + common::{response_collection, response_formatting}, + context::{DispatchMetadata, ExecutionResult}, + error, + }, +}; + +/// Processor for non-streaming Harmony responses +/// +/// Collects all output tokens from execution and parses them using +/// HarmonyParserAdapter to extract the complete response. +pub struct HarmonyResponseProcessor; + +impl HarmonyResponseProcessor { + /// Create a new Harmony response processor + pub fn new() -> Self { + Self + } + + /// Process a non-streaming Harmony chat response + pub async fn process_non_streaming_chat_response( + &self, + execution_result: ExecutionResult, + chat_request: Arc, + dispatch: DispatchMetadata, + ) -> Result { + // Collect all completed responses (one per choice) + let all_responses = response_collection::collect_responses(execution_result, false).await?; + if all_responses.is_empty() { + return Err(error::internal_error("No responses from server")); + } + + // Build choices by parsing output with HarmonyParserAdapter + let mut choices: Vec = Vec::new(); + let mut total_reasoning_tokens = 0u32; + + for (index, complete) in all_responses.iter().enumerate() { + // Convert matched_stop from proto to JSON + let matched_stop = complete.matched_stop().map(|m| match m { + MatchedTokenId(id) => { + serde_json::json!(id) + } + MatchedStopStr(s) => { + serde_json::json!(s) + } + }); + + // Parse Harmony channels with HarmonyParserAdapter + let mut parser = HarmonyParserAdapter::new().map_err(|e| { + error!( + function = "process_non_streaming_chat_response", + error = %e, + "Failed to create Harmony parser" + ); + error::internal_error(format!("Failed to create Harmony parser: {}", e)) + })?; + + // Parse Harmony channels with finish_reason and matched_stop + let parsed = parser + .parse_complete( + complete.output_ids(), + complete.finish_reason().to_string(), + matched_stop.clone(), + ) + .map_err(|e| { + error!( + function = "process_non_streaming_chat_response", + error = %e, + "Harmony parsing failed on complete response" + ); + error::internal_error(format!("Harmony parsing failed: {}", e)) + })?; + + // Build response message (assistant) + let message = ChatCompletionMessage { + role: "assistant".to_string(), + content: (!parsed.final_text.is_empty()).then_some(parsed.final_text), + tool_calls: parsed.commentary, + reasoning_content: parsed.analysis, + }; + + let finish_reason = parsed.finish_reason; + + // Accumulate reasoning tokens across all responses + total_reasoning_tokens += parsed.reasoning_token_count; + + choices.push(ChatChoice { + index: index as u32, + message, + logprobs: None, + finish_reason: Some(finish_reason), + matched_stop, + hidden_states: None, + }); + } + + // Build usage from proto fields + let mut usage = response_formatting::build_usage(&all_responses); + + // Add reasoning token count from parsed analysis/commentary channels + if total_reasoning_tokens > 0 { + usage.completion_tokens_details = Some(CompletionTokensDetails { + reasoning_tokens: Some(total_reasoning_tokens), + }); + } + + // Final ChatCompletionResponse + Ok( + ChatCompletionResponse::builder(&dispatch.request_id, &chat_request.model) + .created(dispatch.created) + .choices(choices) + .usage(usage) + .maybe_system_fingerprint(dispatch.weight_version.clone()) + .build(), + ) + } +} + +impl Default for HarmonyResponseProcessor { + fn default() -> Self { + Self::new() + } +} + +/// Result of processing a single Responses API iteration +/// +/// Used by the MCP tool loop to determine whether to continue +/// executing tools or return the final response. +pub enum ResponsesIterationResult { + /// Tool calls found in commentary channel - continue MCP loop + ToolCallsFound { + tool_calls: Vec, + analysis: Option, // For streaming emission or reasoning output + partial_text: String, // For streaming emission or message output + usage: Usage, // Token usage from this iteration + request_id: String, // Request ID from dispatch + }, + /// No tool calls - return final ResponsesResponse + Completed { + response: Box, + usage: Usage, + }, +} + +impl HarmonyResponseProcessor { + /// Process a single Responses API iteration + /// + /// Parses Harmony channels and determines if tool calls are present. + /// If tool calls found, returns ToolCallsFound for MCP loop to execute. + /// If no tool calls, builds final ResponsesResponse. + /// + /// # Arguments + /// + /// * `execution_result` - The execution result from the model + /// * `responses_request` - The original Responses API request + /// * `dispatch` - Dispatch metadata for request tracking + /// + /// # Returns + /// + /// ResponsesIterationResult indicating whether to continue loop or return + pub async fn process_responses_iteration( + &self, + execution_result: ExecutionResult, + responses_request: Arc, + dispatch: DispatchMetadata, + ) -> Result { + // Collect all completed responses + let all_responses = response_collection::collect_responses(execution_result, false).await?; + if all_responses.is_empty() { + return Err(error::internal_error("No responses from server")); + } + + // For Responses API, we only process the first response (n=1) + let complete = all_responses + .first() + .ok_or_else(|| error::internal_error("No complete response"))?; + + // Parse Harmony channels + let mut parser = HarmonyParserAdapter::new().map_err(|e| { + error!( + function = "process_responses_iteration", + error = %e, + "Failed to create Harmony parser" + ); + error::internal_error(format!("Failed to create Harmony parser: {}", e)) + })?; + + // Convert matched_stop from proto to JSON + let matched_stop = complete.matched_stop().map(|m| match m { + MatchedTokenId(id) => { + serde_json::json!(id) + } + MatchedStopStr(s) => { + serde_json::json!(s) + } + }); + + let parsed = parser + .parse_complete( + complete.output_ids(), + complete.finish_reason().to_string(), + matched_stop, + ) + .map_err(|e| { + error!( + function = "process_responses_iteration", + error = %e, + "Harmony parsing failed on complete response" + ); + error::internal_error(format!("Harmony parsing failed: {}", e)) + })?; + + // VALIDATION: Check if model incorrectly generated Tool role messages + // This happens when the model copies the format of tool result messages + // instead of continuing as assistant. This is a model hallucination bug. + let messages = parser.get_messages(); + let tool_messages_generated = messages.iter().any(|msg| { + msg.author.role == openai_harmony::chat::Role::Tool + && msg.recipient.as_deref() == Some("assistant") + }); + + if tool_messages_generated { + tracing::warn!( + "Model generated Tool->Assistant message instead of Assistant message. \ + This is a model hallucination bug where it copies tool result format." + ); + } + + // Build usage (needed for both ToolCallsFound and Completed) + let mut usage = response_formatting::build_usage(std::slice::from_ref(complete)); + + // Add reasoning token count from parsed analysis/commentary channels + if parsed.reasoning_token_count > 0 { + usage.completion_tokens_details = Some(CompletionTokensDetails { + reasoning_tokens: Some(parsed.reasoning_token_count), + }); + } + + // Check for tool calls in commentary channel + if let Some(tool_calls) = parsed.commentary { + // Tool calls found - return for MCP loop execution + return Ok(ResponsesIterationResult::ToolCallsFound { + tool_calls, + analysis: parsed.analysis, + partial_text: parsed.final_text, + usage, + request_id: dispatch.request_id.clone(), + }); + } + + // No tool calls - build final ResponsesResponse + let mut output: Vec = Vec::new(); + + // Map analysis channel → ResponseOutputItem::Reasoning + if let Some(analysis) = parsed.analysis { + let reasoning_item = ResponseOutputItem::Reasoning { + id: format!("reasoning_{}", dispatch.request_id), + summary: vec![], + content: vec![ResponseReasoningContent::ReasoningText { text: analysis }], + status: Some("completed".to_string()), + }; + output.push(reasoning_item); + } + + // Map final channel → ResponseOutputItem::Message + if !parsed.final_text.is_empty() { + let message_item = ResponseOutputItem::Message { + id: format!("msg_{}", dispatch.request_id), + role: "assistant".to_string(), + content: vec![ResponseContentPart::OutputText { + text: parsed.final_text, + annotations: vec![], + logprobs: None, + }], + status: "completed".to_string(), + }; + output.push(message_item); + } + + // Build ResponsesResponse with all required fields + let response = ResponsesResponse::builder(&dispatch.request_id, &responses_request.model) + .copy_from_request(&responses_request) + .created_at(dispatch.created as i64) + .status(ResponseStatus::Completed) + .output(output) + .maybe_text(responses_request.text.clone()) + .usage(ResponsesUsage::Modern(ResponseUsage { + input_tokens: usage.prompt_tokens, + output_tokens: usage.completion_tokens, + total_tokens: usage.total_tokens, + input_tokens_details: None, + output_tokens_details: usage.completion_tokens_details.as_ref().and_then(|d| { + d.reasoning_tokens.map(|tokens| OutputTokensDetails { + reasoning_tokens: tokens, + }) + }), + })) + .build(); + + Ok(ResponsesIterationResult::Completed { + response: Box::new(response), + usage, + }) + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/responses.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/responses.rs new file mode 100644 index 00000000..559ad5a2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/responses.rs @@ -0,0 +1,1610 @@ +//! Harmony Responses API implementation with multi-turn MCP tool support +//! +//! This module implements the Harmony Responses API orchestration logic, +//! coordinating full pipeline execution with MCP tool support for multi-turn conversations. +//! +//! ## Architecture +//! +//! Multi-turn pipeline orchestration (NOT just a tool loop): +//! - Serves Harmony Responses API requests end-to-end +//! - Each iteration executes FULL pipeline (worker selection + client acquisition + execution + parsing) +//! - Handles MCP tool execution and history building between iterations +//! - Clean separation: serving orchestration (this file) vs. pipeline stages (stages/) +//! +//! ## Flow +//! +//! ```text +//! loop { +//! // Execute through FULL pipeline +//! let result = pipeline.execute_harmony_responses(&request, &ctx).await?; +//! +//! match result { +//! ToolCallsFound { tool_calls, .. } => { +//! // Separate MCP tools from function tools +//! // Execute MCP tools, return if function tools found +//! // Continue loop with MCP results if only MCP tools +//! } +//! Completed { response, .. } => { +//! return Ok(response); +//! } +//! } +//! } +//! ``` +use std::{ + sync::Arc, + time::{SystemTime, UNIX_EPOCH}, +}; + +use axum::response::Response; +use bytes::Bytes; +use serde_json::{from_str, from_value, json, to_string, to_value, Value}; +use tokio::sync::mpsc; +use tracing::{debug, error, warn}; +use uuid::Uuid; + +use crate::{ + data_connector::{ConversationItemStorage, ConversationStorage, ResponseId, ResponseStorage}, + mcp::{self, McpManager}, + protocols::{ + common::{Function, ToolCall, ToolChoice, ToolChoiceValue, Usage}, + responses::{ + McpToolInfo, OutputTokensDetails, ResponseContentPart, ResponseInput, + ResponseInputOutputItem, ResponseOutputItem, ResponseReasoningContent, ResponseStatus, + ResponseTool, ResponseToolType, ResponseUsage, ResponsesRequest, ResponsesResponse, + ResponsesUsage, StringOrContentParts, + }, + }, + routers::grpc::{ + common::responses::{ + build_sse_response, ensure_mcp_connection, persist_response_if_needed, + streaming::{OutputItemType, ResponseStreamEventEmitter}, + }, + context::SharedComponents, + error, + harmony::{processor::ResponsesIterationResult, streaming::HarmonyStreamingProcessor}, + pipeline::RequestPipeline, + }, +}; + +/// Maximum number of tool execution iterations to prevent infinite loops +const MAX_TOOL_ITERATIONS: usize = 10; + +/// Record of a single MCP tool call execution +/// +/// Stores metadata needed to build mcp_call output items for Responses API format +#[derive(Debug, Clone)] +struct McpCallRecord { + /// Tool call ID (stored for potential future use, currently generate new IDs) + #[allow(dead_code)] + call_id: String, + /// Tool name + tool_name: String, + /// JSON-encoded arguments + arguments: String, + /// JSON-encoded output/result + output: String, + /// Whether execution succeeded + success: bool, + /// Error message if execution failed + error: Option, +} + +/// Tracking structure for MCP tool calls across iterations +/// +/// Accumulates all MCP tool call metadata during multi-turn conversation +/// so we can build proper mcp_list_tools and mcp_call output items. +#[derive(Debug, Clone)] +struct McpCallTracking { + /// MCP server label (e.g., "sglang-mcp") + server_label: String, + /// All tool call records across all iterations + tool_calls: Vec, +} + +impl McpCallTracking { + pub fn new(server_label: String) -> Self { + Self { + server_label, + tool_calls: Vec::new(), + } + } + + fn record_call( + &mut self, + call_id: String, + tool_name: String, + arguments: String, + output: String, + success: bool, + error: Option, + ) { + self.tool_calls.push(McpCallRecord { + call_id, + tool_name, + arguments, + output, + success, + error, + }); + } + + fn total_calls(&self) -> usize { + self.tool_calls.len() + } +} + +/// Context for Harmony Responses execution with MCP tool support +/// +/// Contains all dependencies needed for multi-turn Responses API execution. +/// Cheap to clone (all Arc references). +#[derive(Clone)] +pub struct HarmonyResponsesContext { + /// Pipeline for executing Harmony requests + pub pipeline: Arc, + + /// Shared components (tokenizer, parsers) + pub components: Arc, + + /// MCP manager for tool execution + pub mcp_manager: Arc, + + /// Response storage for loading conversation history + pub response_storage: Arc, + + /// Conversation storage for persisting conversations + pub conversation_storage: Arc, + + /// Conversation item storage for persisting conversation items + pub conversation_item_storage: Arc, + + /// Optional streaming sender (for future streaming support) + pub stream_tx: Option>>, +} + +impl HarmonyResponsesContext { + /// Create a new Harmony Responses context + pub fn new( + pipeline: Arc, + components: Arc, + mcp_manager: Arc, + response_storage: Arc, + conversation_storage: Arc, + conversation_item_storage: Arc, + ) -> Self { + Self { + pipeline, + components, + mcp_manager, + response_storage, + conversation_storage, + conversation_item_storage, + stream_tx: None, + } + } + + /// Create with streaming support + pub fn with_streaming( + pipeline: Arc, + components: Arc, + mcp_manager: Arc, + response_storage: Arc, + conversation_storage: Arc, + conversation_item_storage: Arc, + stream_tx: mpsc::UnboundedSender>, + ) -> Self { + Self { + pipeline, + components, + mcp_manager, + response_storage, + conversation_storage, + conversation_item_storage, + stream_tx: Some(stream_tx), + } + } +} + +/// Build a HashSet of MCP tool names for O(1) lookup +/// +/// Creates a HashSet containing the names of all MCP tools in the request, +/// allowing for efficient O(1) lookups when partitioning tool calls. +fn build_mcp_tool_names_set(request_tools: &[ResponseTool]) -> std::collections::HashSet<&str> { + request_tools + .iter() + .filter(|t| t.r#type == ResponseToolType::Mcp) + .filter_map(|t| t.function.as_ref().map(|f| f.name.as_str())) + .collect() +} + +/// Execute Harmony Responses API request with multi-turn MCP tool support +/// +/// This function orchestrates the multi-turn conversation flow: +/// 1. Execute request through full pipeline +/// 2. Check for tool calls in commentary channel +/// 3. If tool calls found: +/// - Execute MCP tools +/// - Build next request with tool results +/// - Repeat from step 1 (full pipeline re-execution) +/// 4. If no tool calls, return final response +/// +/// # Architecture +/// +/// Uses **external loop pattern**: wraps full pipeline execution rather than +/// implementing loop inside pipeline. Each iteration goes through: +/// - Worker selection (fresh selection based on current context) +/// - Client acquisition (new gRPC client if worker changed) +/// - Request building (Harmony prefill with complete history) +/// - Execution (model generation) +/// - Response processing (parse channels, detect tool calls) +/// +/// # Arguments +/// +/// * `ctx` - Harmony Responses context with pipeline, components, MCP manager +/// * `request` - Initial Responses API request +/// +/// # Returns +/// +/// Final ResponsesResponse after all tool iterations complete +/// +/// # Errors +/// +/// Returns error if: +/// - Max iterations exceeded (10 iterations) +/// - Pipeline execution fails +/// - MCP tool execution fails +/// - Response building fails +pub async fn serve_harmony_responses( + ctx: &HarmonyResponsesContext, + request: ResponsesRequest, +) -> Result { + // Clone request for persistence + let original_request = request.clone(); + + // Load previous conversation history if previous_response_id is set + let current_request = load_previous_messages(ctx, request).await?; + + // Check MCP connection and get whether MCP tools are present + let has_mcp_tools = + ensure_mcp_connection(&ctx.mcp_manager, current_request.tools.as_deref()).await?; + + let response = if has_mcp_tools { + execute_with_mcp_loop(ctx, current_request).await? + } else { + // No MCP tools - execute pipeline once (may have function tools or no tools) + execute_without_mcp_loop(ctx, current_request).await? + }; + + // Persist response to storage if store=true + persist_response_if_needed( + ctx.conversation_storage.clone(), + ctx.conversation_item_storage.clone(), + ctx.response_storage.clone(), + &response, + &original_request, + ) + .await; + + Ok(response) +} + +/// Execute Harmony Responses with MCP tool loop +/// +/// Automatically executes MCP tools in a loop until no more tool calls or max iterations +async fn execute_with_mcp_loop( + ctx: &HarmonyResponsesContext, + mut current_request: ResponsesRequest, +) -> Result { + let mut iteration_count = 0; + + // Extract server_label from request tools + let server_label = extract_mcp_server_label(current_request.tools.as_deref()); + let mut mcp_tracking = McpCallTracking::new(server_label.clone()); + + // Extract user's max_tool_calls limit (if set) + let max_tool_calls = current_request.max_tool_calls.map(|n| n as usize); + + // Add static MCP tools from inventory to the request + let mcp_tools = ctx.mcp_manager.list_tools(); + if !mcp_tools.is_empty() { + let mcp_response_tools = convert_mcp_tools_to_response_tools(&mcp_tools); + + let mut all_tools = current_request.tools.clone().unwrap_or_default(); + all_tools.extend(mcp_response_tools); + current_request.tools = Some(all_tools); + + debug!( + mcp_tool_count = mcp_tools.len(), + total_tool_count = current_request.tools.as_ref().map(|t| t.len()).unwrap_or(0), + "MCP client available - added static MCP tools to Harmony Responses request" + ); + } + + loop { + iteration_count += 1; + + // Safety check: prevent infinite loops + if iteration_count > MAX_TOOL_ITERATIONS { + error!( + function = "execute_with_mcp_loop", + iteration_count = iteration_count, + max_iterations = MAX_TOOL_ITERATIONS, + "Maximum tool iterations exceeded" + ); + return Err(error::internal_error(format!( + "Maximum tool iterations ({}) exceeded", + MAX_TOOL_ITERATIONS + ))); + } + + debug!( + iteration = iteration_count, + "Harmony Responses serving iteration" + ); + + // Execute through full pipeline + // This includes: + // - HarmonyPreparationStage (builder.rs: construct_input_messages_with_harmony) + // - WorkerSelectionStage (FRESH selection based on current context) + // - ClientAcquisitionStage (NEW gRPC client if needed) + // - HarmonyRequestBuildingStage (encode to token_ids) + // - RequestExecutionStage (model generation) + // - HarmonyResponseProcessingStage (processor.rs: process_responses_iteration) + let iteration_result = ctx + .pipeline + .execute_harmony_responses(¤t_request, ctx) + .await?; + + match iteration_result { + ResponsesIterationResult::ToolCallsFound { + tool_calls, + analysis, + partial_text, + usage, + request_id, + } => { + debug!( + tool_call_count = tool_calls.len(), + has_analysis = analysis.is_some(), + partial_text_len = partial_text.len(), + "Tool calls found - separating MCP and function tools" + ); + + // Separate MCP and function tool calls based on tool type + let request_tools = current_request.tools.as_deref().unwrap_or(&[]); + let mcp_tool_names = build_mcp_tool_names_set(request_tools); + let (mcp_tool_calls, function_tool_calls): (Vec<_>, Vec<_>) = tool_calls + .into_iter() + .partition(|tc| mcp_tool_names.contains(tc.function.name.as_str())); + + debug!( + mcp_calls = mcp_tool_calls.len(), + function_calls = function_tool_calls.len(), + "Tool calls separated by type" + ); + + // Check combined limit (user's max_tool_calls vs safety limit) + let effective_limit = match max_tool_calls { + Some(user_max) => user_max.min(MAX_TOOL_ITERATIONS), + None => MAX_TOOL_ITERATIONS, + }; + + // Check if we would exceed the limit with these new MCP tool calls + let total_calls_after = mcp_tracking.total_calls() + mcp_tool_calls.len(); + if total_calls_after > effective_limit { + warn!( + current_calls = mcp_tracking.total_calls(), + new_calls = mcp_tool_calls.len() + function_tool_calls.len(), + total_after = total_calls_after, + effective_limit = effective_limit, + user_max = ?max_tool_calls, + "Reached tool call limit - returning incomplete response" + ); + + // Combine back for response + let all_tool_calls: Vec<_> = mcp_tool_calls + .into_iter() + .chain(function_tool_calls) + .collect(); + + // Build response with incomplete status - no tools executed due to limit + let mut response = build_tool_response( + vec![], // No MCP tools executed + vec![], // No MCP results + all_tool_calls, // All tools returned as function calls (not executed) + analysis, + partial_text, + usage, + request_id, + Arc::new(current_request), + ); + + // Mark as completed with incomplete_details + response.status = ResponseStatus::Completed; + response.incomplete_details = Some(json!({ "reason": "max_tool_calls" })); + + // Inject MCP metadata if any calls were executed + if mcp_tracking.total_calls() > 0 { + inject_mcp_metadata(&mut response, &mcp_tracking, &ctx.mcp_manager); + } + + return Ok(response); + } + + // Execute MCP tools (if any) + let mcp_results = if !mcp_tool_calls.is_empty() { + execute_mcp_tools(&ctx.mcp_manager, &mcp_tool_calls, &mut mcp_tracking).await? + } else { + Vec::new() + }; + + // If there are function tools, exit MCP loop and return response + if !function_tool_calls.is_empty() { + debug!( + "Function tool calls present - exiting MCP loop and returning to caller" + ); + + // Build response that includes: + // 1. Reasoning/message from this iteration + // 2. MCP tools as completed (with output) - these were executed + // 3. Function tools as completed (without output) - need caller execution + let mut response = build_tool_response( + mcp_tool_calls, + mcp_results, + function_tool_calls, + analysis, + partial_text, + usage, + request_id, + Arc::new(current_request), + ); + + // Inject MCP metadata for all executed calls + if mcp_tracking.total_calls() > 0 { + inject_mcp_metadata(&mut response, &mcp_tracking, &ctx.mcp_manager); + } + + return Ok(response); + } + + // Only MCP tools - continue loop with their results + debug!("Only MCP tools - continuing loop with results"); + + // Build next request with appended history + current_request = build_next_request_with_tools( + current_request, + mcp_tool_calls, + mcp_results, + analysis, + partial_text, + ) + .map_err(|e| *e)?; + + // Continue loop - next iteration will select workers and execute + } + ResponsesIterationResult::Completed { + mut response, + usage, + } => { + debug!( + output_items = response.output.len(), + input_tokens = usage.prompt_tokens, + output_tokens = usage.completion_tokens, + "MCP loop completed - no more tool calls" + ); + + // Inject MCP metadata into final response + inject_mcp_metadata(&mut response, &mcp_tracking, &ctx.mcp_manager); + + debug!( + mcp_calls = mcp_tracking.total_calls(), + output_items_after = response.output.len(), + "Injected MCP metadata into final response" + ); + + // No tool calls - this is the final response + return Ok(*response); + } + } + } +} + +/// Execute Harmony Responses without MCP loop (single execution) +/// +/// For function tools or no tools - executes pipeline once and returns +async fn execute_without_mcp_loop( + ctx: &HarmonyResponsesContext, + current_request: ResponsesRequest, +) -> Result { + debug!("Executing Harmony Responses without MCP loop"); + + // Execute pipeline once + let iteration_result = ctx + .pipeline + .execute_harmony_responses(¤t_request, ctx) + .await?; + + match iteration_result { + ResponsesIterationResult::ToolCallsFound { + tool_calls, + analysis, + partial_text, + usage, + request_id, + } => { + // Function tool calls found - return to caller for execution + debug!( + tool_call_count = tool_calls.len(), + "Function tool calls found - returning to caller" + ); + + Ok(build_tool_response( + vec![], + vec![], + tool_calls, + analysis, + partial_text, + usage, + request_id, + Arc::new(current_request), + )) + } + ResponsesIterationResult::Completed { response, usage: _ } => { + // No tool calls - return completed response + debug!("No tool calls - returning completed response"); + Ok(*response) + } + } +} + +/// Serve Harmony Responses API with streaming (SSE) +/// +/// This is the streaming equivalent of `serve_harmony_responses()`. +/// Emits SSE events for lifecycle, MCP list_tools, and per-iteration streaming. +/// +/// # Architecture +/// +/// - Emits `response.created` and `response.in_progress` at start +/// - Emits `mcp_list_tools` events on first iteration (if MCP tools available) +/// - Loops through tool execution iterations (max 10) +/// - Calls `streaming::process_responses_iteration_stream()` for per-iteration events +/// - Emits `response.completed` at end +/// - Handles errors with `response.failed` +pub async fn serve_harmony_responses_stream( + ctx: &HarmonyResponsesContext, + request: ResponsesRequest, +) -> Response { + // Load previous conversation history if previous_response_id is set + let current_request = match load_previous_messages(ctx, request.clone()).await { + Ok(req) => req, + Err(err_response) => return err_response, + }; + + // Check MCP connection BEFORE starting stream and get whether MCP tools are present + let has_mcp_tools = + match ensure_mcp_connection(&ctx.mcp_manager, current_request.tools.as_deref()).await { + Ok(has_mcp) => has_mcp, + Err(response) => return response, + }; + + // Create SSE channel + let (tx, rx) = mpsc::unbounded_channel(); + + // Create response event emitter + let response_id = format!("resp_{}", Uuid::new_v4()); + let model = current_request.model.clone(); + let created_at = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + let mut emitter = ResponseStreamEventEmitter::new(response_id.clone(), model, created_at); + + // Set original request for complete response fields + emitter.set_original_request(current_request.clone()); + + // Clone context for spawned task + let ctx_clone = ctx.clone(); + + // Spawn async task to handle streaming + tokio::spawn(async move { + let ctx = &ctx_clone; + + // Emit initial response.created and response.in_progress events + let event = emitter.emit_created(); + if emitter.send_event(&event, &tx).is_err() { + return; + } + let event = emitter.emit_in_progress(); + if emitter.send_event(&event, &tx).is_err() { + return; + } + + if has_mcp_tools { + execute_mcp_tool_loop_streaming(ctx, current_request, &request, &mut emitter, &tx) + .await; + } else { + execute_without_mcp_streaming(ctx, ¤t_request, &request, &mut emitter, &tx).await; + } + }); + + // Return SSE stream response + build_sse_response(rx) +} + +// Execute MCP tool loop with streaming +/// +/// Handles the full MCP workflow: +/// - Adds static MCP tools to request +/// - Emits mcp_list_tools events +/// - Loops through tool execution iterations +/// - Emits final response.completed event +/// - Persists response internally +async fn execute_mcp_tool_loop_streaming( + ctx: &HarmonyResponsesContext, + mut current_request: ResponsesRequest, + original_request: &ResponsesRequest, + emitter: &mut ResponseStreamEventEmitter, + tx: &mpsc::UnboundedSender>, +) { + // Extract server_label from request tools + let server_label = extract_mcp_server_label(current_request.tools.as_deref()); + + // Set server label in emitter for MCP call items + emitter.set_mcp_server_label(server_label.clone()); + + // Initialize MCP call tracking + let mut mcp_tracking = McpCallTracking::new(server_label.clone()); + + // Extract user's max_tool_calls limit (if set) + let max_tool_calls = current_request.max_tool_calls.map(|n| n as usize); + + // Add static MCP tools from inventory + let mcp_tools = ctx.mcp_manager.list_tools(); + if !mcp_tools.is_empty() { + let mcp_response_tools = convert_mcp_tools_to_response_tools(&mcp_tools); + let mut all_tools = current_request.tools.clone().unwrap_or_default(); + all_tools.extend(mcp_response_tools); + current_request.tools = Some(all_tools); + + debug!( + mcp_tool_count = mcp_tools.len(), + total_tool_count = current_request.tools.as_ref().map(|t| t.len()).unwrap_or(0), + "MCP client available - added static MCP tools to Harmony Responses streaming request" + ); + } + + // Build HashSet of MCP tool names for O(1) lookup during streaming + // Clone tool names to owned strings to avoid borrowing current_request + let mcp_tool_names: std::collections::HashSet = current_request + .tools + .as_ref() + .map(|tools| { + tools + .iter() + .filter(|t| t.r#type == ResponseToolType::Mcp) + .filter_map(|t| t.function.as_ref().map(|f| f.name.clone())) + .collect() + }) + .unwrap_or_default(); + + // Emit mcp_list_tools on first iteration + let (output_index, item_id) = emitter.allocate_output_index(OutputItemType::McpListTools); + + // Build tools list for item structure + let tool_items: Vec<_> = mcp_tools + .iter() + .map(|t| { + json!({ + "name": t.name, + "description": t.description, + "input_schema": Value::Object((*t.input_schema).clone()) + }) + }) + .collect(); + + // Build final item with completed status and tools + let item_done = json!({ + "id": item_id, + "type": "mcp_list_tools", + "server_label": server_label, + "status": "completed", + "tools": tool_items + }); + + // Store the completed item data and mark as completed FIRST + // This ensures it appears in final response even if event sending fails + emitter.emit_output_item_done(output_index, &item_done); + emitter.complete_output_item(output_index); + + // Now emit all the events (failures won't affect the stored data) + // Emit output_item.added + let item = json!({ + "id": item_id, + "type": "mcp_list_tools", + "server_label": server_label, + "status": "in_progress", + "tools": [] + }); + let event = emitter.emit_output_item_added(output_index, &item); + if emitter.send_event(&event, tx).is_err() { + return; + } + + // Emit mcp_list_tools.in_progress + let event = emitter.emit_mcp_list_tools_in_progress(output_index); + if emitter.send_event(&event, tx).is_err() { + return; + } + + // Emit mcp_list_tools.completed + let event = emitter.emit_mcp_list_tools_completed(output_index, &mcp_tools); + if emitter.send_event(&event, tx).is_err() { + return; + } + + // Emit output_item.done + let event = emitter.emit_output_item_done(output_index, &item_done); + if emitter.send_event(&event, tx).is_err() { + return; + } + + debug!( + tool_count = mcp_tools.len(), + "Emitted mcp_list_tools on first iteration" + ); + + // MCP tool loop (max 10 iterations) + let mut iteration_count = 0; + loop { + iteration_count += 1; + + // Safety check: prevent infinite loops + if iteration_count > MAX_TOOL_ITERATIONS { + emitter.emit_error( + &format!("Maximum tool iterations ({}) exceeded", MAX_TOOL_ITERATIONS), + Some("max_iterations_exceeded"), + tx, + ); + return; + } + + debug!( + iteration = iteration_count, + "Harmony Responses streaming iteration" + ); + + // Execute pipeline and get stream + let execution_result = match ctx + .pipeline + .execute_harmony_responses_streaming(¤t_request, ctx) + .await + { + Ok(result) => result, + Err(err_response) => { + emitter.emit_error( + &format!("Pipeline execution failed: {:?}", err_response), + Some("pipeline_error"), + tx, + ); + return; + } + }; + + // Process stream with token-level streaming (mixed tools - emits correct events per tool type) + let iteration_result = match HarmonyStreamingProcessor::process_responses_iteration_stream( + execution_result, + emitter, + tx, + &mcp_tool_names, + ) + .await + { + Ok(result) => result, + Err(err_msg) => { + emitter.emit_error(&err_msg, Some("processing_error"), tx); + return; + } + }; + + // Handle iteration result (tool calls or completion) + match iteration_result { + ResponsesIterationResult::ToolCallsFound { + tool_calls, + analysis, + partial_text, + usage, + request_id: _, + } => { + debug!( + tool_call_count = tool_calls.len(), + has_analysis = analysis.is_some(), + partial_text_len = partial_text.len(), + "Tool calls found - separating MCP and function tools" + ); + + // Separate MCP and function tool calls based on tool type + let request_tools = current_request.tools.as_deref().unwrap_or(&[]); + let mcp_tool_names = build_mcp_tool_names_set(request_tools); + let (mcp_tool_calls, function_tool_calls): (Vec<_>, Vec<_>) = tool_calls + .into_iter() + .partition(|tc| mcp_tool_names.contains(tc.function.name.as_str())); + + debug!( + mcp_calls = mcp_tool_calls.len(), + function_calls = function_tool_calls.len(), + "Tool calls separated by type in streaming" + ); + + // Check combined limit (user's max_tool_calls vs safety limit) + let effective_limit = match max_tool_calls { + Some(user_max) => user_max.min(MAX_TOOL_ITERATIONS), + None => MAX_TOOL_ITERATIONS, + }; + + // Check if we would exceed the limit with these new MCP tool calls + let total_calls_after = mcp_tracking.total_calls() + mcp_tool_calls.len(); + if total_calls_after > effective_limit { + warn!( + current_calls = mcp_tracking.total_calls(), + new_calls = mcp_tool_calls.len() + function_tool_calls.len(), + total_after = total_calls_after, + effective_limit = effective_limit, + user_max = ?max_tool_calls, + "Reached tool call limit in streaming - emitting completion with incomplete_details" + ); + + // Emit response.completed with incomplete_details and usage + let incomplete_details = json!({ "reason": "max_tool_calls" }); + let usage_json = json!({ + "input_tokens": usage.prompt_tokens, + "output_tokens": usage.completion_tokens, + "total_tokens": usage.total_tokens, + "incomplete_details": incomplete_details, + }); + let event = emitter.emit_completed(Some(&usage_json)); + emitter.send_event_best_effort(&event, tx); + return; + } + + // Execute MCP tools (if any) + let mcp_results = if !mcp_tool_calls.is_empty() { + match execute_mcp_tools(&ctx.mcp_manager, &mcp_tool_calls, &mut mcp_tracking) + .await + { + Ok(results) => results, + Err(err_response) => { + emitter.emit_error( + &format!("MCP tool execution failed: {:?}", err_response), + Some("mcp_tool_error"), + tx, + ); + return; + } + } + } else { + Vec::new() + }; + + // Update mcp_call output items with execution results (if any MCP tools were executed) + if !mcp_results.is_empty() { + emitter.update_mcp_call_outputs(&mcp_results); + } + + // If there are function tools, exit MCP loop and emit completion + if !function_tool_calls.is_empty() { + debug!( + "Function tool calls present - exiting MCP loop and emitting completion" + ); + + // Function tool calls were already emitted during streaming processing + // Just emit response.completed with usage + let usage_json = json!({ + "input_tokens": usage.prompt_tokens, + "output_tokens": usage.completion_tokens, + "total_tokens": usage.total_tokens, + }); + let event = emitter.emit_completed(Some(&usage_json)); + emitter.send_event_best_effort(&event, tx); + return; + } + + // Only MCP tools - continue loop with their results + debug!("Only MCP tools - continuing loop with results"); + + // Build next request with appended history + current_request = match build_next_request_with_tools( + current_request, + mcp_tool_calls, + mcp_results, + analysis, + partial_text, + ) { + Ok(req) => req, + Err(e) => { + emitter.emit_error( + &format!("Failed to build next request: {:?}", e), + Some("request_building_error"), + tx, + ); + return; + } + }; + + // Continue loop + } + ResponsesIterationResult::Completed { response, usage } => { + debug!( + output_items = response.output.len(), + input_tokens = usage.prompt_tokens, + output_tokens = usage.completion_tokens, + "Harmony Responses streaming completed - no more tool calls" + ); + + // Finalize response from emitter's accumulated data + let final_response = emitter.finalize(Some(usage.clone())); + + // Persist response to storage if store=true + persist_response_if_needed( + ctx.conversation_storage.clone(), + ctx.conversation_item_storage.clone(), + ctx.response_storage.clone(), + &final_response, + original_request, + ) + .await; + + // Emit response.completed with usage + let usage_json = json!({ + "input_tokens": usage.prompt_tokens, + "output_tokens": usage.completion_tokens, + "total_tokens": usage.total_tokens, + }); + let event = emitter.emit_completed(Some(&usage_json)); + emitter.send_event_best_effort(&event, tx); + return; + } + } + } +} + +/// Execute without MCP tool loop (single execution with streaming) +/// +/// For function tools or no tools - executes pipeline once and emits completion. +/// The streaming processor handles all output items (reasoning, message, function tool calls). +async fn execute_without_mcp_streaming( + ctx: &HarmonyResponsesContext, + current_request: &ResponsesRequest, + original_request: &ResponsesRequest, + emitter: &mut ResponseStreamEventEmitter, + tx: &mpsc::UnboundedSender>, +) { + debug!("No MCP tools - executing single iteration"); + + // Execute pipeline and get stream + let execution_result = match ctx + .pipeline + .execute_harmony_responses_streaming(current_request, ctx) + .await + { + Ok(result) => result, + Err(err_response) => { + emitter.emit_error( + &format!("Pipeline execution failed: {:?}", err_response), + Some("pipeline_error"), + tx, + ); + return; + } + }; + + // Process stream (emits all output items during streaming - function tool path emits function_call_arguments.* events) + // Pass empty HashSet so all tools are treated as function tools (per-tool detection) + let empty_mcp_tools = std::collections::HashSet::new(); + let iteration_result = match HarmonyStreamingProcessor::process_responses_iteration_stream( + execution_result, + emitter, + tx, + &empty_mcp_tools, + ) + .await + { + Ok(result) => result, + Err(err_msg) => { + emitter.emit_error(&err_msg, Some("processing_error"), tx); + return; + } + }; + + // Extract usage from iteration result + let usage = match iteration_result { + ResponsesIterationResult::ToolCallsFound { usage, .. } => usage, + ResponsesIterationResult::Completed { usage, .. } => usage, + }; + + // Finalize response from emitter's accumulated data + let final_response = emitter.finalize(Some(usage.clone())); + + // Persist response to storage if store=true + persist_response_if_needed( + ctx.conversation_storage.clone(), + ctx.conversation_item_storage.clone(), + ctx.response_storage.clone(), + &final_response, + original_request, + ) + .await; + + // Emit response.completed with usage + let usage_json = json!({ + "input_tokens": usage.prompt_tokens, + "output_tokens": usage.completion_tokens, + "total_tokens": usage.total_tokens, + }); + let event = emitter.emit_completed(Some(&usage_json)); + emitter.send_event_best_effort(&event, tx); +} + +/// Build ResponsesResponse with tool calls (MCP and/or function tools) +/// +/// ResponsesResponse with tool calls +#[allow(clippy::too_many_arguments)] +fn build_tool_response( + mcp_tool_calls: Vec, + mcp_results: Vec, + function_tool_calls: Vec, + analysis: Option, // Analysis channel content (reasoning) + partial_text: String, // Final channel content (message) + usage: Usage, + request_id: String, + responses_request: Arc, +) -> ResponsesResponse { + let mut output: Vec = Vec::new(); + + // Add reasoning output item if analysis exists + if let Some(analysis_text) = analysis { + output.push(ResponseOutputItem::Reasoning { + id: format!("reasoning_{}", request_id), + summary: vec![], + content: vec![ResponseReasoningContent::ReasoningText { + text: analysis_text, + }], + status: Some("completed".to_string()), + }); + } + + // Add message output item if partial text exists + if !partial_text.is_empty() { + output.push(ResponseOutputItem::Message { + id: format!("msg_{}", request_id), + role: "assistant".to_string(), + content: vec![ResponseContentPart::OutputText { + text: partial_text, + annotations: vec![], + logprobs: None, + }], + status: "completed".to_string(), + }); + } + + // Add MCP tool calls WITH output (these were executed) + for (tool_call, result) in mcp_tool_calls.iter().zip(mcp_results.iter()) { + let output_str = to_string(&result.output).unwrap_or_else(|e| { + format!("{{\"error\": \"Failed to serialize tool output: {}\"}}", e) + }); + + output.push(ResponseOutputItem::FunctionToolCall { + id: tool_call.id.clone(), + call_id: tool_call.id.clone(), + name: tool_call.function.name.clone(), + arguments: tool_call.function.arguments.clone().unwrap_or_default(), + output: Some(output_str), + status: if result.is_error { + "failed" + } else { + "completed" + } + .to_string(), + }); + } + + // Add function tool calls WITHOUT output (need caller execution) + for tool_call in function_tool_calls { + output.push(ResponseOutputItem::FunctionToolCall { + id: tool_call.id.clone(), + call_id: tool_call.id.clone(), + name: tool_call.function.name.clone(), + arguments: tool_call.function.arguments.clone().unwrap_or_default(), + output: None, // No output = needs execution + status: "completed".to_string(), + }); + } + + // Build ResponsesResponse with Completed status + let created_at = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() as i64; + + ResponsesResponse::builder(&request_id, &responses_request.model) + .copy_from_request(&responses_request) + .created_at(created_at) + .status(ResponseStatus::Completed) + .output(output) + .usage(ResponsesUsage::Modern(ResponseUsage { + input_tokens: usage.prompt_tokens, + output_tokens: usage.completion_tokens, + total_tokens: usage.total_tokens, + input_tokens_details: None, + output_tokens_details: usage.completion_tokens_details.as_ref().and_then(|d| { + d.reasoning_tokens.map(|tokens| OutputTokensDetails { + reasoning_tokens: tokens, + }) + }), + })) + .build() +} + +/// Execute MCP tools and collect results +/// +/// Executes each tool call sequentially via the MCP manager. +/// Tool execution errors are returned as error results to the model +/// (allows model to handle gracefully). +/// +/// Vector of tool results (one per tool call) +async fn execute_mcp_tools( + mcp_manager: &Arc, + tool_calls: &[ToolCall], + tracking: &mut McpCallTracking, +) -> Result, Response> { + let mut results = Vec::new(); + + for tool_call in tool_calls { + debug!( + tool_name = %tool_call.function.name, + call_id = %tool_call.id, + "Executing MCP tool" + ); + + // Parse tool arguments from JSON string + let args_str = tool_call.function.arguments.as_deref().unwrap_or("{}"); + let args: Value = from_str(args_str).map_err(|e| { + error!( + function = "execute_mcp_tools", + tool_name = %tool_call.function.name, + call_id = %tool_call.id, + error = %e, + "Failed to parse tool arguments JSON" + ); + error::internal_error(format!( + "Invalid tool arguments JSON for tool '{}': {}", + tool_call.function.name, e + )) + })?; + + // Execute tool via MCP manager + let args_map = if let Value::Object(map) = args { + Some(map) + } else { + None + }; + + match mcp_manager + .call_tool(&tool_call.function.name, args_map) + .await + { + Ok(mcp_result) => { + debug!( + tool_name = %tool_call.function.name, + call_id = %tool_call.id, + "Tool execution succeeded" + ); + + // Extract content from MCP result + let output = if let Some(content) = mcp_result.content.first() { + // Serialize the entire content item + to_value(content) + .unwrap_or_else(|_| json!({"error": "Failed to serialize tool result"})) + } else { + json!({"result": "success"}) + }; + + let is_error = mcp_result.is_error.unwrap_or(false); + let output_str = to_string(&output) + .unwrap_or_else(|_| r#"{"error": "Failed to serialize output"}"#.to_string()); + + // Record this call in tracking + tracking.record_call( + tool_call.id.clone(), + tool_call.function.name.clone(), + args_str.to_string(), + output_str.clone(), + !is_error, + if is_error { + Some(output_str.clone()) + } else { + None + }, + ); + + results.push(ToolResult { + call_id: tool_call.id.clone(), + tool_name: tool_call.function.name.clone(), + output, + is_error, + }); + } + Err(e) => { + warn!( + tool_name = %tool_call.function.name, + call_id = %tool_call.id, + error = %e, + "Tool execution failed" + ); + + let error_msg = format!("Tool execution failed: {}", e); + let error_output = json!({ + "error": error_msg.clone() + }); + let error_output_str = to_string(&error_output) + .unwrap_or_else(|_| format!(r#"{{"error": "{}"}}"#, error_msg)); + + // Record failed call in tracking + tracking.record_call( + tool_call.id.clone(), + tool_call.function.name.clone(), + args_str.to_string(), + error_output_str.clone(), + false, + Some(error_msg), + ); + + // Return error result to model (let it handle gracefully) + results.push(ToolResult { + call_id: tool_call.id.clone(), + tool_name: tool_call.function.name.clone(), + output: error_output, + is_error: true, + }); + } + } + } + + Ok(results) +} + +/// Build next request with tool results appended to history +/// +/// Constructs a new ResponsesRequest with: +/// 1. Original input items (preserved) +/// 2. Assistant message with analysis (reasoning) + partial_text + tool_calls +/// 3. Tool result messages for each tool execution +fn build_next_request_with_tools( + mut request: ResponsesRequest, + tool_calls: Vec, + tool_results: Vec, + analysis: Option, // Analysis channel content (becomes reasoning content) + partial_text: String, // Final channel content (becomes message content) +) -> Result> { + // Get current input items (or empty vec if Text variant) + let mut items = match request.input { + ResponseInput::Items(items) => items, + ResponseInput::Text(text) => { + // Convert text to items format + vec![ResponseInputOutputItem::SimpleInputMessage { + content: StringOrContentParts::String(text), + role: "user".to_string(), + r#type: None, + }] + } + }; + + // Build assistant response item with reasoning + content + tool calls + // This represents what the model generated in this iteration + let assistant_id = format!("msg_{}", Uuid::new_v4()); + + // Add reasoning if present (from analysis channel) + if let Some(analysis_text) = analysis { + items.push(ResponseInputOutputItem::Reasoning { + id: format!("reasoning_{}", assistant_id), + summary: vec![], + content: vec![ResponseReasoningContent::ReasoningText { + text: analysis_text, + }], + status: Some("completed".to_string()), + }); + } + + // Add message content if present (from final channel) + if !partial_text.is_empty() { + items.push(ResponseInputOutputItem::Message { + id: assistant_id.clone(), + role: "assistant".to_string(), + content: vec![ResponseContentPart::OutputText { + text: partial_text, + annotations: vec![], + logprobs: None, + }], + status: Some("completed".to_string()), + }); + } + + // Add function tool calls (from commentary channel) + for tool_call in tool_calls { + items.push(ResponseInputOutputItem::FunctionToolCall { + id: tool_call.id.clone(), + call_id: tool_call.id.clone(), + name: tool_call.function.name.clone(), + arguments: tool_call + .function + .arguments + .unwrap_or_else(|| "{}".to_string()), + output: None, // Output will be added next + status: Some("in_progress".to_string()), + }); + } + + // Add tool results + for tool_result in tool_results { + // Serialize tool output to string + let output_str = to_string(&tool_result.output).unwrap_or_else(|e| { + format!("{{\"error\": \"Failed to serialize tool output: {}\"}}", e) + }); + + // Update the corresponding tool call with output and completed status + // Find and update the matching FunctionToolCall + if let Some(ResponseInputOutputItem::FunctionToolCall { + output, + status, + .. + }) = items + .iter_mut() + .find(|item| matches!(item, ResponseInputOutputItem::FunctionToolCall { call_id, .. } if call_id == &tool_result.call_id)) + { + *output = Some(output_str); + *status = if tool_result.is_error { + Some("failed".to_string()) + } else { + Some("completed".to_string()) + }; + } + } + + // Update request with new items + request.input = ResponseInput::Items(items); + + // Switch tool_choice to "auto" for subsequent iterations + // This prevents infinite loops when original tool_choice was "required" or specific function + // After receiving tool results, the model should be free to decide whether to call more tools or finish + request.tool_choice = Some(ToolChoice::Value(ToolChoiceValue::Auto)); + + Ok(request) +} + +/// Tool execution result +/// +/// Contains the result of executing a single MCP tool. +pub(crate) struct ToolResult { + /// Tool call ID (for matching with request) + pub(crate) call_id: String, + + /// Tool name + #[allow(dead_code)] // Kept for documentation and future use + pub(crate) tool_name: String, + + /// Tool output (JSON value) + pub(crate) output: Value, + + /// Whether this is an error result + pub(crate) is_error: bool, +} + +/// Convert MCP tools to Responses API tool format +/// +/// Converts MCP Tool entries (from rmcp SDK) to ResponseTool format so the model +/// knows about available MCP tools when making tool calls. +pub fn convert_mcp_tools_to_response_tools(mcp_tools: &[mcp::Tool]) -> Vec { + mcp_tools + .iter() + .map(|tool_info| ResponseTool { + r#type: ResponseToolType::Mcp, + function: Some(Function { + name: tool_info.name.to_string(), + description: tool_info.description.as_ref().map(|d| d.to_string()), + parameters: Value::Object((*tool_info.input_schema).clone()), + strict: None, + }), + server_url: None, // MCP tools from inventory don't have individual server URLs + authorization: None, + server_label: None, + server_description: tool_info.description.as_ref().map(|d| d.to_string()), + require_approval: None, + allowed_tools: None, + }) + .collect() +} + +/// Inject MCP metadata into final response +/// +/// Adds mcp_list_tools and mcp_call output items to the response output array. +/// Following non-Harmony pipeline pattern: +/// 1. Prepend mcp_list_tools at the beginning +/// 2. Append all mcp_call items at the end +/// +/// # Arguments +/// +/// * `response` - Final response to modify +/// * `tracking` - MCP call tracking data +/// * `mcp_manager` - MCP manager for listing tools +fn inject_mcp_metadata( + response: &mut ResponsesResponse, + tracking: &McpCallTracking, + mcp_manager: &Arc, +) { + // Build mcp_list_tools item + let tools = mcp_manager.list_tools(); + let tools_info: Vec = tools + .iter() + .map(|t| McpToolInfo { + name: t.name.to_string(), + description: t.description.as_ref().map(|d| d.to_string()), + input_schema: Value::Object((*t.input_schema).clone()), + annotations: Some(json!({ + "read_only": false + })), + }) + .collect(); + + let mcp_list_tools = ResponseOutputItem::McpListTools { + id: format!("mcpl_{}", Uuid::new_v4()), + server_label: tracking.server_label.clone(), + tools: tools_info, + }; + + // Build mcp_call items for each tracked call + let mcp_call_items: Vec = tracking + .tool_calls + .iter() + .map(|record| ResponseOutputItem::McpCall { + id: format!("mcp_{}", Uuid::new_v4()), + status: if record.success { + "completed" + } else { + "failed" + } + .to_string(), + approval_request_id: None, + arguments: record.arguments.clone(), + error: record.error.clone(), + name: record.tool_name.clone(), + output: record.output.clone(), + server_label: tracking.server_label.clone(), + }) + .collect(); + + // Inject into response output: + // 1. Prepend mcp_list_tools at the beginning + response.output.insert(0, mcp_list_tools); + + // 2. Append all mcp_call items at the end + response.output.extend(mcp_call_items); +} + +/// Extract MCP server label from request tools +/// +/// Searches for the first MCP tool in the tools array and returns its server_label. +/// Falls back to "sglang-mcp" if no MCP tool with server_label is found. +fn extract_mcp_server_label(tools: Option<&[ResponseTool]>) -> String { + tools + .and_then(|tools| { + tools.iter().find_map(|tool| { + if matches!(tool.r#type, ResponseToolType::Mcp) { + tool.server_label.clone() + } else { + None + } + }) + }) + .unwrap_or_else(|| "sglang-mcp".to_string()) +} + +/// Load previous conversation messages from storage +/// +/// If the request has `previous_response_id`, loads the response chain from storage +/// and prepends the conversation history to the request input items. +/// +/// # Arguments +/// +/// * `ctx` - Harmony Responses context with response_storage +/// * `request` - Current request (may have previous_response_id set) +/// +/// # Returns +/// +/// Modified request with conversation history prepended to input items +async fn load_previous_messages( + ctx: &HarmonyResponsesContext, + request: ResponsesRequest, +) -> Result { + let Some(ref prev_id_str) = request.previous_response_id else { + // No previous_response_id, return request as-is + return Ok(request); + }; + + let prev_id = ResponseId::from(prev_id_str.as_str()); + + // Load response chain from storage + let chain = ctx + .response_storage + .get_response_chain(&prev_id, None) + .await + .map_err(|e| { + error!( + function = "load_previous_messages", + prev_id = %prev_id_str, + error = %e, + "Failed to load previous response chain from storage" + ); + error::internal_error(format!( + "Failed to load previous response chain for {}: {}", + prev_id_str, e + )) + })?; + + // Build conversation history from stored responses + let mut history_items = Vec::new(); + + // Helper to deserialize and collect items from a JSON array + let deserialize_items = |arr: &Value, item_type: &str| -> Vec { + arr.as_array() + .into_iter() + .flat_map(|items| items.iter()) + .filter_map(|item| { + from_value::(item.clone()) + .map_err(|e| { + warn!( + "Failed to deserialize stored {} item: {}. Item: {}", + item_type, e, item + ); + }) + .ok() + }) + .collect() + }; + + for stored in chain.responses.iter() { + history_items.extend(deserialize_items(&stored.input, "input")); + history_items.extend(deserialize_items(&stored.output, "output")); + } + + debug!( + previous_response_id = %prev_id_str, + history_items_count = history_items.len(), + "Loaded conversation history from previous response" + ); + + // Build modified request with history prepended + let mut modified_request = request; + + // Convert current input to items format + let all_items = match modified_request.input { + ResponseInput::Items(items) => { + // Prepend history to existing items + let mut combined = history_items; + combined.extend(items); + combined + } + ResponseInput::Text(text) => { + // Convert text to item and prepend history + history_items.push(ResponseInputOutputItem::SimpleInputMessage { + content: StringOrContentParts::String(text), + role: "user".to_string(), + r#type: None, + }); + history_items + } + }; + + // Update request with combined items and clear previous_response_id + modified_request.input = ResponseInput::Items(all_items); + modified_request.previous_response_id = None; + + Ok(modified_request) +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/stages/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/stages/mod.rs new file mode 100644 index 00000000..8e130fff --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/stages/mod.rs @@ -0,0 +1,14 @@ +//! Harmony-specific pipeline stages +//! +//! These stages replace their regular counterparts in the Harmony pipeline: +//! - HarmonyPreparationStage: Harmony encoding instead of chat template + tokenization +//! - HarmonyRequestBuildingStage: Token-based request building +//! - HarmonyResponseProcessingStage: Harmony channel parsing + +pub mod preparation; +pub mod request_building; +pub mod response_processing; + +pub use preparation::HarmonyPreparationStage; +pub use request_building::HarmonyRequestBuildingStage; +pub use response_processing::HarmonyResponseProcessingStage; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/stages/preparation.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/stages/preparation.rs new file mode 100644 index 00000000..c370514c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/stages/preparation.rs @@ -0,0 +1,414 @@ +//! Harmony Preparation Stage: Harmony encoding for chat and generate requests + +use async_trait::async_trait; +use axum::response::Response; +use serde_json::json; +use tracing::error; + +use super::super::HarmonyBuilder; +use crate::{ + protocols::{ + chat::ChatCompletionRequest, + common::{Tool, ToolChoice, ToolChoiceValue}, + responses::ResponsesRequest, + }, + routers::grpc::{ + common::{responses::utils::extract_tools_from_response_tools, stages::PipelineStage}, + context::{PreparationOutput, RequestContext, RequestType}, + error, utils, + }, +}; + +/// Harmony Preparation stage: Encode requests using Harmony protocol +/// +/// Replaces the regular PreparationStage for Harmony models. +/// Converts chat/generate requests to Harmony-encoded token_ids and extraction_text. +pub struct HarmonyPreparationStage { + builder: HarmonyBuilder, +} + +impl HarmonyPreparationStage { + /// Create a new Harmony preparation stage + pub fn new() -> Self { + Self { + builder: HarmonyBuilder::new(), + } + } +} + +impl Default for HarmonyPreparationStage { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl PipelineStage for HarmonyPreparationStage { + async fn execute(&self, ctx: &mut RequestContext) -> Result, Response> { + // Clone Arc before match to avoid borrow checker issues + // Arc clone is cheap (8 bytes) - avoids full request clone (15KB-200KB) + let is_chat = matches!(&ctx.input.request_type, RequestType::Chat(_)); + let is_responses = matches!(&ctx.input.request_type, RequestType::Responses(_)); + + if is_chat { + let request_arc = ctx.chat_request_arc(); + self.prepare_chat(ctx, &request_arc).await?; + } else if is_responses { + let request_arc = ctx.responses_request_arc(); + self.prepare_responses(ctx, &request_arc).await?; + } else { + error!( + function = "HarmonyPreparationStage::execute", + "Unsupported request type for Harmony pipeline" + ); + return Err(error::bad_request( + "Only Chat and Responses requests supported in Harmony pipeline".to_string(), + )); + } + + Ok(None) + } + + fn name(&self) -> &'static str { + "HarmonyPreparation" + } +} + +impl HarmonyPreparationStage { + /// Prepare a chat completion request using Harmony encoding + async fn prepare_chat( + &self, + ctx: &mut RequestContext, + request: &ChatCompletionRequest, + ) -> Result, Response> { + // Validate - reject logprobs + if request.logprobs { + error!( + function = "prepare_chat", + "logprobs requested but not supported for Harmony models" + ); + return Err(error::bad_request( + "logprobs are not supported for Harmony models".to_string(), + )); + } + + // Step 1: Filter tools if needed + let body_ref = utils::filter_chat_request_by_tool_choice(request); + + // Step 2: Build tool constraints + let tool_constraints = if let Some(tools) = body_ref.tools.as_ref() { + Self::generate_tool_call_constraint(tools, &body_ref.tool_choice).map_err(|e| *e)? + } else { + None + }; + + // Step 3: Build via Harmony + let build_output = self.builder.build_from_chat(&body_ref).map_err(|e| { + error!( + function = "prepare_chat", + error = %e, + "Harmony build failed for chat request" + ); + error::bad_request(format!("Harmony build failed: {}", e)) + })?; + + // Step 4: Store results + ctx.state.preparation = Some(PreparationOutput { + original_text: None, + token_ids: build_output.input_ids, + processed_messages: None, + tool_constraints, + filtered_request: if matches!(body_ref, std::borrow::Cow::Owned(_)) { + Some(body_ref.into_owned()) + } else { + None + }, + harmony_mode: true, + selection_text: Some(build_output.selection_text), + harmony_messages: Some(build_output.harmony_messages), + harmony_stop_ids: Some(build_output.stop_token_ids), + }); + + Ok(None) + } + + /// Prepare a responses API request using Harmony encoding + /// + /// For responses API, we build from conversation history using the same Harmony + /// encoding that the builder provides. This handles the MCP loop integration. + pub async fn prepare_responses( + &self, + ctx: &mut RequestContext, + request: &ResponsesRequest, + ) -> Result, Response> { + // Step 1: Extract function and MCP tools with schemas from ResponseTools + let mut function_tools = extract_tools_from_response_tools(request.tools.as_deref(), true); + + // Step 2: Filter tools based on tool_choice (AllowedTools or Function) + // Note: Tool existence is already validated in ResponsesRequest::validate() + if let Some(filtered) = + utils::filter_tools_by_tool_choice(&function_tools, &request.tool_choice) + { + function_tools = filtered; + } + + // Step 3: Generate Harmony structural tags + let tool_constraint = if !function_tools.is_empty() { + Self::generate_tool_call_constraint(&function_tools, &request.tool_choice) + .map_err(|e| *e)? + } else { + None + }; + + let text_constraint = if let Some(text_config) = &request.text { + Self::generate_text_format_constraint(text_config).map_err(|e| *e)? + } else { + None + }; + + if tool_constraint.is_some() && text_constraint.is_some() { + error!( + function = "prepare_responses", + "Conflicting constraints: both tool_choice and text format specified" + ); + return Err(error::bad_request( + "Cannot use both tool_choice (required/function) and text format (json_object/json_schema) simultaneously".to_string(), + )); + } + + let constraint = tool_constraint.or(text_constraint); + + // Step 3: Build via Harmony from responses API request + let build_output = self.builder.build_from_responses(request).map_err(|e| { + error!( + function = "prepare_responses", + error = %e, + "Harmony build failed for responses request" + ); + error::bad_request(format!("Harmony build failed: {}", e)) + })?; + + // Step 4: Store results with constraint + ctx.state.preparation = Some(PreparationOutput { + original_text: None, + token_ids: build_output.input_ids, + processed_messages: None, + tool_constraints: constraint, + filtered_request: None, + harmony_mode: true, + selection_text: Some(build_output.selection_text), + harmony_messages: Some(build_output.harmony_messages), + harmony_stop_ids: Some(build_output.stop_token_ids), + }); + + Ok(None) + } + + /// Generate Harmony structural tag for structured output (text field) + /// + /// Converts text.format to structural tag that constrains the final channel. + /// Returns None if text.format is not specified or is "text". + fn generate_text_format_constraint( + text_config: &crate::protocols::responses::TextConfig, + ) -> Result, Box> { + use crate::protocols::responses::TextFormat; + + let Some(format) = &text_config.format else { + return Ok(None); + }; + + match format { + TextFormat::Text => Ok(None), + TextFormat::JsonObject => { + let tag = build_text_format_structural_tag(&serde_json::json!({"type": "object"})) + .map_err(|e| { + error!( + function = "generate_text_format_constraint", + error = %e, + "Failed to build text format structural tag for JsonObject" + ); + Box::new(error::internal_error(e)) + })?; + Ok(Some(("structural_tag".to_string(), tag))) + } + TextFormat::JsonSchema { schema, .. } => { + let tag = build_text_format_structural_tag(schema).map_err(|e| { + error!( + function = "generate_text_format_constraint", + error = %e, + "Failed to build text format structural tag for JsonSchema" + ); + Box::new(error::internal_error(e)) + })?; + Ok(Some(("structural_tag".to_string(), tag))) + } + } + } + + /// Generate Harmony structural tag for tool constraints + /// + /// Uses structural tags with `triggered_tags` format to force Harmony format output. + /// This ensures the model outputs in Harmony format (with channels) even when constrained. + fn generate_tool_call_constraint( + tools: &[Tool], + tool_choice: &Option, + ) -> Result, Box> { + let Some(choice) = tool_choice.as_ref() else { + return Ok(None); + }; + + match choice { + ToolChoice::Function { function, .. } => { + let tag = Self::build_tool_call_structural_tag(tools, Some(&function.name))?; + Ok(Some(("structural_tag".to_string(), tag))) + } + ToolChoice::Value(ToolChoiceValue::Required) => { + let tag = Self::build_tool_call_structural_tag(tools, None)?; + Ok(Some(("structural_tag".to_string(), tag))) + } + ToolChoice::AllowedTools { mode, .. } => { + if mode == "required" { + let tag = Self::build_tool_call_structural_tag(tools, None)?; + Ok(Some(("structural_tag".to_string(), tag))) + } else { + Ok(None) + } + } + _ => Ok(None), + } + } + + /// Build Harmony structural tag for tool calling constraints + /// + /// Supports both reasoning-enabled and reasoning-disabled modes: + /// - With reasoning: triggers on `<|start|>assistant<|channel|>commentary` (waits for analysis) + /// - Without reasoning: triggers on `<|channel|>commentary` (goes directly to commentary) + fn build_tool_call_structural_tag( + tools: &[Tool], + specific_function: Option<&str>, + ) -> Result> { + let mut tags = Vec::new(); + + // Filter tools if specific function requested + let tools_to_use: Vec<&Tool> = if let Some(func_name) = specific_function { + tools + .iter() + .filter(|t| t.function.name == func_name) + .collect() + } else { + tools.iter().collect() + }; + + // Validate specific function exists + match specific_function { + Some(tool_name) if tools_to_use.is_empty() => { + error!( + function = "generate_tool_call_constraint", + tool_name = %tool_name, + "Specified tool not found in tools list" + ); + return Err(Box::new(error::bad_request(format!( + "Tool '{}' not found in tools list", + tool_name + )))); + } + _ => {} + } + + // Build tags for each tool - need two patterns per tool for reasoning on/off + for tool in tools_to_use { + let tool_name = &tool.function.name; + let params_schema = &tool.function.parameters; + + // Pattern 1: For reasoning-enabled mode (with analysis channel before commentary) + tags.push(json!({ + "begin": format!("<|start|>assistant<|channel|>commentary to=functions.{}<|constrain|>json<|message|>", tool_name), + "content": { + "type": "json_schema", + "json_schema": params_schema + }, + "end": "" // `end` is empty because <|call|> comes naturally from Harmony stop tokens + })); + + // Pattern 2: For reasoning-disabled mode (goes directly to commentary channel) + tags.push(json!({ + "begin": format!("<|channel|>commentary to=functions.{}<|constrain|>json<|message|>", tool_name), + "content": { + "type": "json_schema", + "json_schema": params_schema + }, + "end": "" + })); + } + + let stop_after_first = specific_function.is_some(); + + let structural_tag = json!({ + "format": { + "type": "triggered_tags", + "triggers": ["<|start|>assistant<|channel|>commentary", "<|channel|>commentary"], + "tags": tags, + "at_least_one": true, + "stop_after_first": stop_after_first + } + }); + + serde_json::to_string(&structural_tag).map_err(|e| { + error!( + function = "generate_tool_call_constraint", + error = %e, + "Failed to serialize structural tag" + ); + Box::new(error::internal_error(format!( + "Failed to serialize structural tag: {}", + e + ))) + }) + } +} + +/// Build Harmony structural tag for structured output (JSON schema constraint) +/// +/// Creates a structural tag that applies JSON schema constraint to the final channel, +/// supporting both reasoning-enabled and reasoning-disabled modes: +/// - With reasoning: triggers on `<|start|>assistant<|channel|>final` (waits for analysis to complete) +/// - Without reasoning: triggers on `<|channel|>final` (goes directly to final channel) +/// +/// This is used for the Responses API text.format field (json_object or json_schema). +pub fn build_text_format_structural_tag(schema: &serde_json::Value) -> Result { + let structural_tag = json!({ + "format": { + "type": "triggered_tags", + "triggers": ["<|start|>assistant<|channel|>final", "<|channel|>final"], + "tags": [ + { + // Pattern 1: For reasoning-enabled mode (with analysis channel before final) + "begin": "<|start|>assistant<|channel|>final<|constrain|>json<|message|>", + "content": { + "type": "json_schema", + "json_schema": schema + }, + "end": "" + }, + { + // Pattern 2: For reasoning-disabled mode (goes directly to final channel) + "begin": "<|channel|>final<|constrain|>json<|message|>", + "content": { + "type": "json_schema", + "json_schema": schema + }, + "end": "" + } + ], + "at_least_one": true, + "stop_after_first": true + } + }); + + serde_json::to_string(&structural_tag).map_err(|e| { + format!( + "Failed to serialize structural tag for structured output: {}", + e + ) + }) +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/stages/request_building.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/stages/request_building.rs new file mode 100644 index 00000000..7d8fa095 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/stages/request_building.rs @@ -0,0 +1,156 @@ +//! Harmony Request Building Stage: Build gRPC request from Harmony-encoded tokens + +use async_trait::async_trait; +use axum::response::Response; +use tracing::{debug, error}; +use uuid::Uuid; + +use crate::routers::grpc::{ + common::stages::{helpers, PipelineStage}, + context::{ClientSelection, RequestContext, RequestType, WorkerSelection}, + error, + proto_wrapper::ProtoGenerateRequest, +}; + +/// Harmony Request Building stage: Convert Harmony tokens to gRPC request +/// +/// Takes the Harmony-encoded input_ids from preparation and builds a proto::GenerateRequest. +/// Unlike regular request building, this uses token_ids directly (Harmony encoding handles messages). +pub struct HarmonyRequestBuildingStage { + inject_pd_metadata: bool, +} + +impl HarmonyRequestBuildingStage { + /// Create a new Harmony request building stage + pub fn new(inject_pd_metadata: bool) -> Self { + Self { inject_pd_metadata } + } +} + +#[async_trait] +impl PipelineStage for HarmonyRequestBuildingStage { + async fn execute(&self, ctx: &mut RequestContext) -> Result, Response> { + // Get preparation output + let prep = ctx.state.preparation.as_ref().ok_or_else(|| { + error!( + function = "HarmonyRequestBuildingStage::execute", + "Preparation stage not completed" + ); + error::internal_error("Preparation not completed") + })?; + + // Get clients + let clients = ctx.state.clients.as_ref().ok_or_else(|| { + error!( + function = "HarmonyRequestBuildingStage::execute", + "Client acquisition stage not completed" + ); + error::internal_error("Client acquisition not completed") + })?; + let builder_client = match clients { + ClientSelection::Single { client } => client, + ClientSelection::Dual { prefill, .. } => prefill, + }; + + // Harmony model support not yet implemented for vLLM + if builder_client.is_vllm() { + return Err(error::not_implemented( + "Harmony model support is not yet implemented for vLLM backend. \ + Please use runtime_type: sglang for Harmony models.", + )); + } + + // Generate request_id based on request type + let request_id = match &ctx.input.request_type { + RequestType::Chat(_) => format!("chatcmpl-{}", Uuid::new_v4()), + RequestType::Responses(_) => format!("responses-{}", Uuid::new_v4()), + RequestType::Generate(_) => { + error!( + function = "HarmonyRequestBuildingStage::execute", + "Generate request type not supported for Harmony models" + ); + return Err(error::bad_request( + "Generate requests are not supported with Harmony models".to_string(), + )); + } + }; + + // Build gRPC request using token_ids directly (Harmony encoding already handled message rendering) + let placeholder_processed_text = "[harmony]".to_string(); + + // Harmony is SGLang-only, so we can safely unwrap as SGLang + let sglang_client = builder_client.as_sglang(); + let proto_request_inner = match &ctx.input.request_type { + RequestType::Chat(request) => { + // Use filtered request if present from preparation; otherwise original + let body = prep.filtered_request.as_ref().unwrap_or(request.as_ref()); + + sglang_client + .build_generate_request_from_chat( + request_id, + body, + placeholder_processed_text, + prep.token_ids.clone(), + None, + prep.tool_constraints.clone(), + ) + .map_err(|e| { + error!( + function = "HarmonyRequestBuildingStage::execute", + error = %e, + "Failed to build generate request from chat" + ); + error::bad_request(format!("Invalid request parameters: {}", e)) + })? + } + RequestType::Responses(request) => sglang_client + .build_generate_request_from_responses( + request_id, + request.as_ref(), + placeholder_processed_text, + prep.token_ids.clone(), + prep.harmony_stop_ids.clone(), + prep.tool_constraints.clone(), + ) + .map_err(|e| { + error!( + function = "HarmonyRequestBuildingStage::execute", + error = %e, + "Failed to build generate request from responses" + ); + error::bad_request(format!("Invalid request parameters: {}", e)) + })?, + _ => unreachable!(), + }; + + let mut proto_request = ProtoGenerateRequest::Sglang(Box::new(proto_request_inner)); + + // Inject Harmony stop token IDs into sampling params for ALL Harmony requests + // These stop tokens (<|return|> and <|call|>) prevent the model from generating + // malformed Harmony sequences + if let Some(harmony_stops) = &prep.harmony_stop_ids { + let sglang_req = proto_request.as_sglang_mut(); + if let Some(params) = sglang_req.sampling_params.as_mut() { + params.stop_token_ids.extend_from_slice(harmony_stops); + debug!( + stop_token_count = harmony_stops.len(), + "Injected Harmony stop tokens into sampling params" + ); + } + } + + // Inject PD metadata if needed + if self.inject_pd_metadata { + if let Some(WorkerSelection::Dual { prefill, .. }) = ctx.state.workers.as_ref() { + helpers::inject_bootstrap_metadata(&mut proto_request, prefill); + } + } + + ctx.state.proto_request = Some(proto_request); + Ok(None) + } + + fn name(&self) -> &'static str { + "HarmonyRequestBuilding" + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/stages/response_processing.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/stages/response_processing.rs new file mode 100644 index 00000000..924556e3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/stages/response_processing.rs @@ -0,0 +1,155 @@ +//! Harmony Response Processing Stage: Parse Harmony channels to ChatCompletionResponse + +use std::sync::Arc; + +use async_trait::async_trait; +use axum::response::Response; +use tracing::error; + +use super::super::{HarmonyResponseProcessor, HarmonyStreamingProcessor}; +use crate::routers::grpc::{ + common::stages::PipelineStage, + context::{FinalResponse, RequestContext, RequestType}, + error, +}; + +/// Harmony Response Processing stage: Parse and format Harmony responses +/// +/// Takes output tokens from execution and parses them using HarmonyParserAdapter +/// to extract analysis, tool calls, and final response text from Harmony channels. +pub struct HarmonyResponseProcessingStage { + processor: HarmonyResponseProcessor, + streaming_processor: Arc, +} + +impl HarmonyResponseProcessingStage { + /// Create a new Harmony response processing stage + pub fn new() -> Self { + Self { + processor: HarmonyResponseProcessor::new(), + streaming_processor: Arc::new(HarmonyStreamingProcessor::new()), + } + } +} + +impl Default for HarmonyResponseProcessingStage { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl PipelineStage for HarmonyResponseProcessingStage { + async fn execute(&self, ctx: &mut RequestContext) -> Result, Response> { + let is_streaming = ctx.is_streaming(); + + // Check request type to determine which processor method to call + match &ctx.input.request_type { + RequestType::Chat(_) => { + // Get execution result (output tokens from model) + let execution_result = + ctx.state.response.execution_result.take().ok_or_else(|| { + error!( + function = "HarmonyResponseProcessingStage::execute", + request_type = "Chat", + "No execution result available" + ); + error::internal_error("No execution result") + })?; + + let dispatch = ctx.state.dispatch.as_ref().cloned().ok_or_else(|| { + error!( + function = "HarmonyResponseProcessingStage::execute", + request_type = "Chat", + "Dispatch metadata not set" + ); + error::internal_error("Dispatch metadata not set") + })?; + + // For streaming, delegate to streaming processor and return SSE response + if is_streaming { + return Ok(Some( + self.streaming_processor + .clone() + .process_streaming_chat_response( + execution_result, + ctx.chat_request_arc(), + dispatch, + ), + )); + } + + // For non-streaming, delegate to Harmony response processor to build ChatCompletionResponse + let chat_request = ctx.chat_request_arc(); + let response = self + .processor + .process_non_streaming_chat_response(execution_result, chat_request, dispatch) + .await?; + + ctx.state.response.final_response = Some(FinalResponse::Chat(response)); + Ok(None) + } + RequestType::Responses(_) => { + // For streaming Responses API, leave execution_result in context + // for external streaming processor (serve_harmony_responses_stream) + if is_streaming { + // Don't take execution_result - let the caller handle it + return Ok(None); + } + + // For non-streaming, process normally + let execution_result = + ctx.state.response.execution_result.take().ok_or_else(|| { + error!( + function = "HarmonyResponseProcessingStage::execute", + request_type = "Responses", + "No execution result available" + ); + error::internal_error("No execution result") + })?; + + let dispatch = ctx.state.dispatch.as_ref().cloned().ok_or_else(|| { + error!( + function = "HarmonyResponseProcessingStage::execute", + request_type = "Responses", + "Dispatch metadata not set" + ); + error::internal_error("Dispatch metadata not set") + })?; + + let responses_request = ctx.responses_request_arc(); + let iteration_result = self + .processor + .process_responses_iteration(execution_result, responses_request, dispatch) + .await?; + + ctx.state.response.responses_iteration_result = Some(iteration_result); + Ok(None) + } + RequestType::Generate(_) => { + error!( + function = "HarmonyResponseProcessingStage::execute", + "Generate request type not supported in Harmony pipeline" + ); + Err(error::internal_error( + "Generate requests not supported in Harmony pipeline", + )) + } + } + } + + fn name(&self) -> &'static str { + "HarmonyResponseProcessing" + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_response_processing_stage_creation() { + let stage = HarmonyResponseProcessingStage::new(); + assert_eq!(stage.name(), "HarmonyResponseProcessing"); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/streaming.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/streaming.rs new file mode 100644 index 00000000..b7b3db36 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/streaming.rs @@ -0,0 +1,1151 @@ +//! Harmony streaming response processor + +use std::{ + collections::{hash_map::Entry::Vacant, HashMap}, + io, + sync::Arc, +}; + +use axum::{body::Body, http::StatusCode, response::Response}; +use bytes::Bytes; +use http::header::{HeaderValue, CONTENT_TYPE}; +use serde_json::json; +use tokio::sync::mpsc; +use tokio_stream::wrappers::UnboundedReceiverStream; +use tracing::{debug, error}; + +use super::{ + processor::ResponsesIterationResult, types::HarmonyChannelDelta, HarmonyParserAdapter, +}; +use crate::{ + grpc_client::sglang_proto::generate_complete::MatchedStop::{MatchedStopStr, MatchedTokenId}, + protocols::{ + chat::{ + ChatCompletionRequest, ChatCompletionStreamResponse, ChatMessageDelta, ChatStreamChoice, + }, + common::{CompletionTokensDetails, FunctionCallDelta, ToolCall, ToolCallDelta, Usage}, + responses::{ + OutputTokensDetails, ResponseStatus, ResponseUsage, ResponsesResponse, ResponsesUsage, + }, + }, + routers::grpc::{ + common::responses::streaming::{OutputItemType, ResponseStreamEventEmitter}, + context, + proto_wrapper::{ProtoResponseVariant, ProtoStream}, + }, +}; + +/// Mode for tool call event emission +#[derive(Debug, Clone, Copy)] +enum ToolCallMode { + /// MCP tool calls (emit .in_progress and .completed events) + Mcp, + /// Function tool calls (no status events, only arguments streaming) + Function, +} + +impl ToolCallMode { + /// Get the output item type for this mode + fn output_item_type(&self) -> OutputItemType { + match self { + Self::Mcp => OutputItemType::McpCall, + Self::Function => OutputItemType::FunctionCall, + } + } + + /// Get the type string for JSON output + fn type_str(&self) -> &'static str { + match self { + Self::Mcp => "mcp_call", + Self::Function => "function_call", + } + } + + /// Whether this mode emits status events (.in_progress, .completed) + fn emits_status_events(&self) -> bool { + matches!(self, Self::Mcp) + } + + /// Emit arguments delta event + fn emit_arguments_delta( + &self, + emitter: &mut ResponseStreamEventEmitter, + output_index: usize, + item_id: &str, + delta: &str, + ) -> serde_json::Value { + match self { + Self::Mcp => emitter.emit_mcp_call_arguments_delta(output_index, item_id, delta), + Self::Function => { + emitter.emit_function_call_arguments_delta(output_index, item_id, delta) + } + } + } + + /// Emit arguments done event + fn emit_arguments_done( + &self, + emitter: &mut ResponseStreamEventEmitter, + output_index: usize, + item_id: &str, + arguments: &str, + ) -> serde_json::Value { + match self { + Self::Mcp => emitter.emit_mcp_call_arguments_done(output_index, item_id, arguments), + Self::Function => { + emitter.emit_function_call_arguments_done(output_index, item_id, arguments) + } + } + } +} + +/// Processor for streaming Harmony responses +/// +/// Returns an SSE stream that parses Harmony tokens incrementally and +/// emits ChatCompletionChunk events for streaming responses. +pub struct HarmonyStreamingProcessor; + +impl HarmonyStreamingProcessor { + /// Create a new Harmony streaming processor + pub fn new() -> Self { + Self + } + + /// Process a streaming Harmony Chat Completion response + /// + /// Returns an SSE response with streaming token updates. + pub fn process_streaming_chat_response( + self: Arc, + execution_result: context::ExecutionResult, + chat_request: Arc, + dispatch: context::DispatchMetadata, + ) -> Response { + // Create SSE channel + let (tx, rx) = mpsc::unbounded_channel::>(); + + // Spawn background task based on execution mode + match execution_result { + context::ExecutionResult::Single { stream } => { + tokio::spawn(async move { + let result = + Self::process_single_stream(stream, dispatch, chat_request, &tx).await; + + if let Err(e) = result { + error!("Harmony streaming error: {}", e); + let error_chunk = format!( + "data: {}\n\n", + json!({ + "error": { + "message": e, + "type": "internal_error" + } + }) + ); + let _ = tx.send(Ok(Bytes::from(error_chunk))); + } + + let _ = tx.send(Ok(Bytes::from("data: [DONE]\n\n"))); + }); + } + context::ExecutionResult::Dual { prefill, decode } => { + tokio::spawn(async move { + let result = + Self::process_dual_stream(prefill, *decode, dispatch, chat_request, &tx) + .await; + + if let Err(e) = result { + error!("Harmony dual streaming error: {}", e); + let error_chunk = format!( + "data: {}\n\n", + json!({ + "error": { + "message": e, + "type": "internal_error" + } + }) + ); + let _ = tx.send(Ok(Bytes::from(error_chunk))); + } + + let _ = tx.send(Ok(Bytes::from("data: [DONE]\n\n"))); + }); + } + } + + // Return SSE response + Self::build_sse_response(rx) + } + + /// Process streaming chunks from a single stream + async fn process_single_stream( + mut grpc_stream: ProtoStream, + dispatch: context::DispatchMetadata, + original_request: Arc, + tx: &mpsc::UnboundedSender>, + ) -> Result<(), String> { + // Per-index state management (for n>1 support) + let mut parsers: HashMap = HashMap::new(); + let mut is_firsts: HashMap = HashMap::new(); + let mut finish_reasons: HashMap> = HashMap::new(); + let mut matched_stops: HashMap> = HashMap::new(); + let mut prompt_tokens: HashMap = HashMap::new(); + let mut completion_tokens: HashMap = HashMap::new(); + + let stream_options = &original_request.stream_options; + + // Process stream + while let Some(result) = grpc_stream.next().await { + let response = result.map_err(|e| format!("Stream error: {}", e))?; + + match response.into_response() { + ProtoResponseVariant::Chunk(chunk_wrapper) => { + let chunk = chunk_wrapper.as_sglang(); + let index = chunk.index; + + // Initialize parser for this index if needed + if let Vacant(e) = parsers.entry(index) { + e.insert( + HarmonyParserAdapter::new() + .map_err(|e| format!("Failed to create parser: {}", e))?, + ); + is_firsts.insert(index, true); + } + + // Track token counts + *completion_tokens.entry(index).or_insert(0) += 1; + + // Parse chunk via Harmony parser + let parser = parsers + .get_mut(&index) + .ok_or("Parser not found for index")?; + + let delta_result = parser + .parse_chunk(&chunk.token_ids) + .map_err(|e| format!("Parse error: {}", e))?; + + // Emit SSE event if there's a delta + if let Some(delta) = delta_result { + let is_first = is_firsts.get(&index).copied().unwrap_or(false); + Self::emit_chunk_delta( + &delta, + index, + is_first, + &dispatch, + &original_request, + tx, + )?; + + if is_first { + is_firsts.insert(index, false); + } + } + } + ProtoResponseVariant::Complete(complete_wrapper) => { + let complete = complete_wrapper.as_sglang(); + let index = complete.index; + + // Store final metadata + finish_reasons.insert(index, Some(complete.finish_reason.clone())); + matched_stops.insert( + index, + complete.matched_stop.as_ref().map(|m| match m { + MatchedTokenId(id) => { + json!(id) + } + MatchedStopStr(s) => { + json!(s) + } + }), + ); + prompt_tokens.insert(index, complete.prompt_tokens as u32); + *completion_tokens.entry(index).or_insert(0) = + complete.completion_tokens as u32; + + // Finalize parser and emit final chunk + if let Some(parser) = parsers.get_mut(&index) { + let matched_stop = matched_stops.get(&index).and_then(|m| m.clone()); + let final_output = parser + .finalize(complete.finish_reason.clone(), matched_stop.clone()) + .map_err(|e| format!("Finalize error: {}", e))?; + + Self::emit_final_chunk( + index, + &final_output.finish_reason, + final_output.matched_stop.as_ref(), + &dispatch, + &original_request, + tx, + )?; + } + } + ProtoResponseVariant::Error(error_wrapper) => { + return Err(format!("Server error: {}", error_wrapper.message())); + } + ProtoResponseVariant::None => {} + } + } + + // Emit final usage if requested + if let Some(true) = stream_options.as_ref().and_then(|so| so.include_usage) { + let total_prompt: u32 = prompt_tokens.values().sum(); + let total_completion: u32 = completion_tokens.values().sum(); + + Self::emit_usage_chunk( + total_prompt, + total_completion, + &dispatch, + &original_request, + tx, + )?; + } + + // Mark stream as completed successfully to prevent abort on drop + grpc_stream.mark_completed(); + + Ok(()) + } + + /// Process streaming chunks from dual streams (prefill + decode) + async fn process_dual_stream( + mut prefill_stream: ProtoStream, + mut decode_stream: ProtoStream, + dispatch: context::DispatchMetadata, + original_request: Arc, + tx: &mpsc::UnboundedSender>, + ) -> Result<(), String> { + // Phase 1: Process prefill stream (collect metadata) + let mut prompt_tokens: HashMap = HashMap::new(); + + while let Some(result) = prefill_stream.next().await { + let response = result.map_err(|e| format!("Prefill stream error: {}", e))?; + + if let ProtoResponseVariant::Complete(complete_wrapper) = response.into_response() { + let complete = complete_wrapper.as_sglang(); + prompt_tokens.insert(complete.index, complete.prompt_tokens as u32); + } + } + + // Phase 2: Process decode stream (same as single stream) + let mut parsers: HashMap = HashMap::new(); + let mut is_firsts: HashMap = HashMap::new(); + let mut finish_reasons: HashMap> = HashMap::new(); + let mut matched_stops: HashMap> = HashMap::new(); + let mut completion_tokens: HashMap = HashMap::new(); + + let stream_options = &original_request.stream_options; + + while let Some(result) = decode_stream.next().await { + let response = result.map_err(|e| format!("Decode stream error: {}", e))?; + + match response.into_response() { + ProtoResponseVariant::Chunk(chunk_wrapper) => { + let chunk = chunk_wrapper.as_sglang(); + let index = chunk.index; + + // Initialize parser for this index if needed + if let Vacant(e) = parsers.entry(index) { + e.insert( + HarmonyParserAdapter::new() + .map_err(|e| format!("Failed to create parser: {}", e))?, + ); + is_firsts.insert(index, true); + } + + *completion_tokens.entry(index).or_insert(0) += 1; + + let parser = parsers + .get_mut(&index) + .ok_or("Parser not found for index")?; + + let delta_result = parser + .parse_chunk(&chunk.token_ids) + .map_err(|e| format!("Parse error: {}", e))?; + + if let Some(delta) = delta_result { + let is_first = is_firsts.get(&index).copied().unwrap_or(false); + Self::emit_chunk_delta( + &delta, + index, + is_first, + &dispatch, + &original_request, + tx, + )?; + + if is_first { + is_firsts.insert(index, false); + } + } + } + ProtoResponseVariant::Complete(complete_wrapper) => { + let complete = complete_wrapper.as_sglang(); + let index = complete.index; + + finish_reasons.insert(index, Some(complete.finish_reason.clone())); + matched_stops.insert( + index, + complete.matched_stop.as_ref().map(|m| match m { + MatchedTokenId(id) => { + json!(id) + } + MatchedStopStr(s) => { + json!(s) + } + }), + ); + *completion_tokens.entry(index).or_insert(0) = + complete.completion_tokens as u32; + + if let Some(parser) = parsers.get_mut(&index) { + let matched_stop = matched_stops.get(&index).and_then(|m| m.clone()); + let final_output = parser + .finalize(complete.finish_reason.clone(), matched_stop.clone()) + .map_err(|e| format!("Finalize error: {}", e))?; + + Self::emit_final_chunk( + index, + &final_output.finish_reason, + final_output.matched_stop.as_ref(), + &dispatch, + &original_request, + tx, + )?; + } + } + ProtoResponseVariant::Error(error_wrapper) => { + return Err(format!("Server error: {}", error_wrapper.message())); + } + ProtoResponseVariant::None => {} + } + } + + decode_stream.mark_completed(); + + // Mark prefill stream as completed AFTER decode completes successfully + // This ensures that if client disconnects during decode, BOTH streams send abort + prefill_stream.mark_completed(); + + // Emit final usage if requested + if let Some(true) = stream_options.as_ref().and_then(|so| so.include_usage) { + let total_prompt: u32 = prompt_tokens.values().sum(); + let total_completion: u32 = completion_tokens.values().sum(); + + Self::emit_usage_chunk( + total_prompt, + total_completion, + &dispatch, + &original_request, + tx, + )?; + } + + Ok(()) + } + + /// Emit a chunk delta from Harmony channels + fn emit_chunk_delta( + delta: &HarmonyChannelDelta, + index: u32, + is_first: bool, + dispatch: &context::DispatchMetadata, + original_request: &ChatCompletionRequest, + tx: &mpsc::UnboundedSender>, + ) -> Result<(), String> { + // On first chunk, emit role announcement separately + if is_first { + let role_chunk = ChatCompletionStreamResponse::builder( + &dispatch.request_id, + &original_request.model, + ) + .created(dispatch.created) + .add_choice_role(index, "assistant") + .maybe_system_fingerprint(dispatch.weight_version.clone()) + .build(); + + let chunk_json = serde_json::to_string(&role_chunk) + .map_err(|e| format!("JSON serialization error: {}", e))?; + let sse_data = format!("data: {}\n\n", chunk_json); + + tx.send(Ok(Bytes::from(sse_data))) + .map_err(|_| "Failed to send role chunk".to_string())?; + } + + // Emit content delta (role is always None for content chunks) + let chat_delta = ChatMessageDelta { + role: None, + content: delta.final_delta.clone(), + tool_calls: delta.commentary_delta.as_ref().map(|tc_delta| { + vec![ToolCallDelta { + index: tc_delta.index as u32, + id: tc_delta.id.clone(), + tool_type: tc_delta.id.as_ref().map(|_| "function".to_string()), + function: tc_delta.function.as_ref().map(|f| FunctionCallDelta { + name: f.name.clone(), + arguments: f.arguments.clone(), + }), + }] + }), + reasoning_content: delta.analysis_delta.clone(), + }; + + // Build and emit chunk + let chunk = + ChatCompletionStreamResponse::builder(&dispatch.request_id, &original_request.model) + .created(dispatch.created) + .add_choice(ChatStreamChoice { + index, + delta: chat_delta, + logprobs: None, + finish_reason: None, + matched_stop: None, + }) + .maybe_system_fingerprint(dispatch.weight_version.clone()) + .build(); + + let chunk_json = serde_json::to_string(&chunk) + .map_err(|e| format!("JSON serialization error: {}", e))?; + let sse_data = format!("data: {}\n\n", chunk_json); + + tx.send(Ok(Bytes::from(sse_data))) + .map_err(|_| "Failed to send chunk".to_string())?; + + Ok(()) + } + + /// Emit final chunk with finish_reason + fn emit_final_chunk( + index: u32, + finish_reason: &str, + matched_stop: Option<&serde_json::Value>, + dispatch: &context::DispatchMetadata, + original_request: &ChatCompletionRequest, + tx: &mpsc::UnboundedSender>, + ) -> Result<(), String> { + let chunk = + ChatCompletionStreamResponse::builder(&dispatch.request_id, &original_request.model) + .created(dispatch.created) + .add_choice_finish_reason(index, finish_reason, matched_stop.cloned()) + .maybe_system_fingerprint(dispatch.weight_version.clone()) + .build(); + + let chunk_json = serde_json::to_string(&chunk) + .map_err(|e| format!("JSON serialization error: {}", e))?; + let sse_data = format!("data: {}\n\n", chunk_json); + + tx.send(Ok(Bytes::from(sse_data))) + .map_err(|_| "Failed to send final chunk".to_string())?; + + Ok(()) + } + + /// Emit usage chunk at the end + fn emit_usage_chunk( + prompt_tokens: u32, + completion_tokens: u32, + dispatch: &context::DispatchMetadata, + original_request: &ChatCompletionRequest, + tx: &mpsc::UnboundedSender>, + ) -> Result<(), String> { + let usage_chunk = + ChatCompletionStreamResponse::builder(&dispatch.request_id, &original_request.model) + .created(dispatch.created) + .usage(Usage { + prompt_tokens, + completion_tokens, + total_tokens: prompt_tokens + completion_tokens, + completion_tokens_details: None, + }) + .maybe_system_fingerprint(dispatch.weight_version.clone()) + .build(); + + let chunk_json = serde_json::to_string(&usage_chunk) + .map_err(|e| format!("JSON serialization error: {}", e))?; + let sse_data = format!("data: {}\n\n", chunk_json); + + tx.send(Ok(Bytes::from(sse_data))) + .map_err(|_| "Failed to send usage chunk".to_string())?; + + Ok(()) + } + + /// Process streaming chunks for Responses API iteration + /// + /// Emits correct event types based on tool names: mcp_call.* for MCP tools, function_call.* for function tools. + /// Pass empty HashSet for function-only tools, full MCP tool names for MCP-only, or subset for mixed. + pub async fn process_responses_iteration_stream( + execution_result: context::ExecutionResult, + emitter: &mut ResponseStreamEventEmitter, + tx: &mpsc::UnboundedSender>, + mcp_tool_names: &std::collections::HashSet, + ) -> Result { + match execution_result { + context::ExecutionResult::Single { stream } => { + debug!("Processing Responses API single stream mode"); + Self::process_responses_single_stream_mixed(stream, emitter, tx, mcp_tool_names) + .await + } + context::ExecutionResult::Dual { prefill, decode } => { + debug!("Processing Responses API dual stream mode"); + Self::process_responses_dual_stream_mixed( + prefill, + *decode, + emitter, + tx, + mcp_tool_names, + ) + .await + } + } + } + + /// Process streaming chunks from a single stream + async fn process_responses_single_stream_mixed( + grpc_stream: ProtoStream, + emitter: &mut ResponseStreamEventEmitter, + tx: &mpsc::UnboundedSender>, + mcp_tool_names: &std::collections::HashSet, + ) -> Result { + Self::process_decode_stream_with_tool_lookup(grpc_stream, emitter, tx, Some(mcp_tool_names)) + .await + } + + /// Process streaming chunks from dual streams + async fn process_responses_dual_stream_mixed( + mut prefill_stream: ProtoStream, + decode_stream: ProtoStream, + emitter: &mut ResponseStreamEventEmitter, + tx: &mpsc::UnboundedSender>, + mcp_tool_names: &std::collections::HashSet, + ) -> Result { + // Phase 1: Process prefill stream (collect metadata, no output) + while let Some(result) = prefill_stream.next().await { + let _response = result.map_err(|e| format!("Prefill stream error: {}", e))?; + } + + // Phase 2: Process decode stream with per-tool mode detection + let result = Self::process_decode_stream_with_tool_lookup( + decode_stream, + emitter, + tx, + Some(mcp_tool_names), + ) + .await; + + // Mark prefill stream as completed AFTER decode completes successfully + // This ensures that if client disconnects during decode, BOTH streams send abort + prefill_stream.mark_completed(); + result + } + + /// Decode stream processing with optional per-tool mode lookup + /// + /// If mcp_tool_names is Some, determines mode per-tool by checking tool name. + /// If mcp_tool_names is None, uses default MCP mode for all tools. + async fn process_decode_stream_with_tool_lookup( + mut decode_stream: ProtoStream, + emitter: &mut ResponseStreamEventEmitter, + tx: &mpsc::UnboundedSender>, + mcp_tool_names: Option<&std::collections::HashSet>, + ) -> Result { + // Initialize Harmony parser for this iteration + let mut parser = + HarmonyParserAdapter::new().map_err(|e| format!("Failed to create parser: {}", e))?; + + // State tracking for channels + let mut has_analysis = false; + let mut accumulated_final_text = String::new(); + let mut accumulated_tool_calls: Option> = None; + + // Track which items we've started + let mut has_emitted_reasoning = false; + let mut message_output_index: Option = None; + let mut message_item_id: Option = None; + let mut has_emitted_content_part_added = false; + + // Tool call tracking (call_index -> (output_index, item_id, mode)) + // Mode is determined per-tool when mcp_tool_names is provided + let mut tool_call_tracking: HashMap = HashMap::new(); + + // Metadata from Complete message + let mut finish_reason = String::from("stop"); + let mut matched_stop: Option = None; + let mut prompt_tokens: u32 = 0; + let mut completion_tokens: u32 = 0; + let mut reasoning_token_count: u32 = 0; + + // Process stream + let mut chunk_count = 0; + while let Some(result) = decode_stream.next().await { + chunk_count += 1; + let response = result.map_err(|e| format!("Decode stream error: {}", e))?; + + match response.into_response() { + ProtoResponseVariant::Chunk(chunk_wrapper) => { + let chunk = chunk_wrapper.as_sglang(); + // Parse chunk via Harmony parser + let delta_result = parser + .parse_chunk(&chunk.token_ids) + .map_err(|e| format!("Parse error: {}", e))?; + + // Emit SSE events if there's a delta + if let Some(delta) = delta_result { + // Analysis channel → Reasoning item (wrapper events only, emitted once) + if let Some(_analysis_text) = &delta.analysis_delta { + if !has_emitted_reasoning { + // Emit reasoning item (added + done in one call) + // Note: reasoning_content will be provided at finalize + emitter + .emit_reasoning_item(tx, None) + .map_err(|e| format!("Failed to emit reasoning item: {}", e))?; + + has_emitted_reasoning = true; + has_analysis = true; + } + } + + // Final channel → Message item (WITH text streaming) + if let Some(final_delta) = &delta.final_delta { + if !final_delta.is_empty() { + // Allocate message item if needed + if message_output_index.is_none() { + let (output_index, item_id) = + emitter.allocate_output_index(OutputItemType::Message); + message_output_index = Some(output_index); + message_item_id = Some(item_id.clone()); + + // Build message item structure + let item = json!({ + "id": item_id, + "type": "message", + "role": "assistant", + "content": [] + }); + + // Emit output_item.added + let event = emitter.emit_output_item_added(output_index, &item); + emitter.send_event_best_effort(&event, tx); + } + + let output_index = message_output_index.unwrap(); + let item_id = message_item_id.as_ref().unwrap(); + let content_index = 0; // Single content part + + // Emit content_part.added before first delta + if !has_emitted_content_part_added { + let event = emitter.emit_content_part_added( + output_index, + item_id, + content_index, + ); + emitter.send_event_best_effort(&event, tx); + has_emitted_content_part_added = true; + } + + // Emit text delta + let event = emitter.emit_text_delta( + final_delta, + output_index, + item_id, + content_index, + ); + emitter.send_event_best_effort(&event, tx); + + accumulated_final_text.push_str(final_delta); + } + } + + // Commentary channel → Tool call streaming + if let Some(tc_delta) = &delta.commentary_delta { + let call_index = tc_delta.index; + + // Check if this is a new tool call (has id and name) + if tc_delta.id.is_some() { + // Get tool name first to determine mode + let tool_name = tc_delta + .function + .as_ref() + .and_then(|f| f.name.as_ref()) + .map(|n| n.as_str()) + .unwrap_or(""); + + // Determine mode for this specific tool call + let tool_mode = if let Some(mcp_names) = mcp_tool_names { + // Mixed mode: check if tool is MCP + if mcp_names.contains(tool_name) { + ToolCallMode::Mcp + } else { + ToolCallMode::Function + } + } else { + // Single mode: use MCP (legacy behavior) + ToolCallMode::Mcp + }; + + // NEW TOOL CALL: Allocate output item + let (output_index, item_id) = + emitter.allocate_output_index(tool_mode.output_item_type()); + + // Store tracking info with mode + tool_call_tracking + .insert(call_index, (output_index, item_id.clone(), tool_mode)); + + // Emit output_item.added wrapper event + let call_id = tc_delta.id.as_ref().unwrap(); + let mut item = json!({ + "id": item_id, + "type": tool_mode.type_str(), + "name": tool_name, + "call_id": call_id, + "arguments": "", + "status": "in_progress" + }); + + // Add server_label for MCP calls + if tool_mode.emits_status_events() { + if let Some(ref server_label) = emitter.mcp_server_label { + item["server_label"] = json!(server_label); + } + } + + let event = emitter.emit_output_item_added(output_index, &item); + emitter.send_event_best_effort(&event, tx); + + // Emit status event if mode supports it (MCP only) + if tool_mode.emits_status_events() { + let event = + emitter.emit_mcp_call_in_progress(output_index, &item_id); + emitter.send_event_best_effort(&event, tx); + } + + // If we have function name, emit initial arguments delta + if let Some(func) = &tc_delta.function { + if func.name.is_some() { + let event = tool_mode.emit_arguments_delta( + emitter, + output_index, + &item_id, + "", + ); + emitter.send_event_best_effort(&event, tx); + } + } + } else { + // CONTINUING TOOL CALL: Emit arguments delta + if let Some((output_index, item_id, tool_mode)) = + tool_call_tracking.get(&call_index) + { + if let Some(args) = tc_delta + .function + .as_ref() + .and_then(|f| f.arguments.as_ref()) + .filter(|a| !a.is_empty()) + { + let event = tool_mode.emit_arguments_delta( + emitter, + *output_index, + item_id, + args, + ); + emitter.send_event_best_effort(&event, tx); + } + } + } + } + } + } + ProtoResponseVariant::Complete(complete_wrapper) => { + let complete = complete_wrapper.as_sglang(); + // Store final metadata + finish_reason = complete.finish_reason.clone(); + matched_stop = complete.matched_stop.as_ref().map(|m| match m { + MatchedTokenId(id) => { + json!(id) + } + MatchedStopStr(s) => { + json!(s) + } + }); + prompt_tokens = complete.prompt_tokens as u32; + completion_tokens = complete.completion_tokens as u32; + + // Finalize parser and get complete output + let final_output = parser + .finalize(finish_reason.clone(), matched_stop.clone()) + .map_err(|e| format!("Finalize error: {}", e))?; + + // Store finalized tool calls and reasoning token count + accumulated_tool_calls = final_output.commentary.clone(); + reasoning_token_count = final_output.reasoning_token_count; + + // Complete all tool calls if we have commentary + if let Some(ref tool_calls) = accumulated_tool_calls { + for (call_idx, tool_call) in tool_calls.iter().enumerate() { + if let Some((output_index, item_id, tool_mode)) = + tool_call_tracking.get(&call_idx) + { + let tool_name = &tool_call.function.name; + + // Emit arguments done with final arguments + let args_str = + tool_call.function.arguments.as_deref().unwrap_or(""); + + let event = tool_mode.emit_arguments_done( + emitter, + *output_index, + item_id, + args_str, + ); + emitter.send_event_best_effort(&event, tx); + + // Emit status event if mode supports it (MCP only) + if tool_mode.emits_status_events() { + let event = + emitter.emit_mcp_call_completed(*output_index, item_id); + emitter.send_event_best_effort(&event, tx); + } + + // Emit output_item.done wrapper event + let mut item = json!({ + "id": item_id, + "type": tool_mode.type_str(), + "name": tool_name, + "call_id": &tool_call.id, + "arguments": args_str, + "status": "completed" + }); + + // Add server_label for MCP calls + if tool_mode.emits_status_events() { + // MCP mode - include server_label + if let Some(ref server_label) = emitter.mcp_server_label { + item["server_label"] = json!(server_label); + } + } + + let event = emitter.emit_output_item_done(*output_index, &item); + + // Mark output item as completed before sending + emitter.complete_output_item(*output_index); + + emitter.send_event_best_effort(&event, tx); + } + } + } + + // Close message item if we opened one + if let Some(output_index) = message_output_index { + let item_id = message_item_id.as_ref().unwrap(); + let content_index = 0; + + // Emit text_done + let event = emitter.emit_text_done(output_index, item_id, content_index); + emitter.send_event_best_effort(&event, tx); + + // Emit content_part.done + let event = + emitter.emit_content_part_done(output_index, item_id, content_index); + emitter.send_event_best_effort(&event, tx); + + // Emit output_item.done + let item = json!({ + "id": item_id, + "type": "message", + "role": "assistant", + "content": [{ + "type": "text", + "text": accumulated_final_text.clone() + }] + }); + let event = emitter.emit_output_item_done(output_index, &item); + + // Mark as completed before sending (so it's included in final output even if send fails) + emitter.complete_output_item(output_index); + + emitter.send_event_best_effort(&event, tx); + } + } + ProtoResponseVariant::Error(error_wrapper) => { + return Err(format!("Server error: {}", error_wrapper.message())); + } + ProtoResponseVariant::None => {} + } + } + + debug!( + "Stream loop ended. Total chunks received: {}, has_analysis: {}, tool_calls: {}, final_text_len: {}", + chunk_count, + has_analysis, + accumulated_tool_calls.as_ref().map(|tc| tc.len()).unwrap_or(0), + accumulated_final_text.len() + ); + + // Extract tool calls from completed messages or incomplete commentary + if chunk_count > 0 && accumulated_tool_calls.is_none() { + let messages = parser.get_messages(); + + // Try extracting from completed messages first + let (analysis_opt, commentary_opt, final_text_extracted) = + HarmonyParserAdapter::parse_messages(&messages); + accumulated_tool_calls = commentary_opt.clone(); + + // If no tool calls found, check for incomplete commentary in parser state + if accumulated_tool_calls.is_none() { + accumulated_tool_calls = parser.extract_incomplete_commentary(); + } + + debug!( + "Tool call extraction: completed_msgs={}, tool_calls={}, has_analysis={}, final_text_len={}", + messages.len(), + accumulated_tool_calls.as_ref().map(|tc| tc.len()).unwrap_or(0), + analysis_opt.is_some(), + final_text_extracted.len() + ); + + // Complete any pending tool calls with data from completed messages + if let Some(ref tool_calls) = accumulated_tool_calls { + for (call_idx, tool_call) in tool_calls.iter().enumerate() { + if let Some((output_index, item_id, tool_mode)) = + tool_call_tracking.get(&call_idx) + { + let tool_name = &tool_call.function.name; + + // Emit arguments done with final arguments + let args_str = tool_call.function.arguments.as_deref().unwrap_or(""); + let event = tool_mode.emit_arguments_done( + emitter, + *output_index, + item_id, + args_str, + ); + emitter.send_event_best_effort(&event, tx); + + // Emit status event if mode supports it (MCP only) + if tool_mode.emits_status_events() { + let event = emitter.emit_mcp_call_completed(*output_index, item_id); + emitter.send_event_best_effort(&event, tx); + } + + // Emit output_item.done wrapper event + let mut item = json!({ + "id": item_id, + "type": tool_mode.type_str(), + "name": tool_name, + "call_id": &tool_call.id, + "arguments": args_str, + "status": "completed" + }); + + // Add server_label for MCP calls + if tool_mode.emits_status_events() { + if let Some(ref server_label) = emitter.mcp_server_label { + item["server_label"] = json!(server_label); + } + } + + let event = emitter.emit_output_item_done(*output_index, &item); + + // Mark output item as completed before sending + emitter.complete_output_item(*output_index); + + emitter.send_event_best_effort(&event, tx); + } + } + } + } + + // Mark stream as completed successfully to prevent abort on drop + decode_stream.mark_completed(); + + // Return result based on whether tool calls were found + if let Some(tool_calls) = accumulated_tool_calls { + if !tool_calls.is_empty() { + let analysis_content = if has_analysis { + // Get analysis from finalized parser output by calling finalize again + // This is safe because finalize can be called multiple times + let output = parser.finalize(finish_reason.clone(), matched_stop.clone())?; + output.analysis + } else { + None + }; + + return Ok(ResponsesIterationResult::ToolCallsFound { + tool_calls, + analysis: analysis_content, + partial_text: accumulated_final_text, + usage: Usage { + prompt_tokens, + completion_tokens, + total_tokens: prompt_tokens + completion_tokens, + completion_tokens_details: if reasoning_token_count > 0 { + Some(CompletionTokensDetails { + reasoning_tokens: Some(reasoning_token_count), + }) + } else { + None + }, + }, + request_id: emitter.response_id.clone(), + }); + } + } + + // For streaming, we don't build the full ResponsesResponse here + // The caller will build it from the SSE events + // Return a placeholder Completed result (caller ignores these fields in streaming mode) + Ok(ResponsesIterationResult::Completed { + response: Box::new( + ResponsesResponse::builder(&emitter.response_id, "") + .status(ResponseStatus::Completed) + .usage(ResponsesUsage::Modern(ResponseUsage { + input_tokens: prompt_tokens, + output_tokens: completion_tokens, + total_tokens: prompt_tokens + completion_tokens, + input_tokens_details: None, + output_tokens_details: if reasoning_token_count > 0 { + Some(OutputTokensDetails { + reasoning_tokens: reasoning_token_count, + }) + } else { + None + }, + })) + .build(), + ), + usage: Usage { + prompt_tokens, + completion_tokens, + total_tokens: prompt_tokens + completion_tokens, + completion_tokens_details: if reasoning_token_count > 0 { + Some(CompletionTokensDetails { + reasoning_tokens: Some(reasoning_token_count), + }) + } else { + None + }, + }, + }) + } + + /// Build SSE response from receiver + fn build_sse_response(rx: mpsc::UnboundedReceiver>) -> Response { + let stream = UnboundedReceiverStream::new(rx); + let body = Body::from_stream(stream); + + Response::builder() + .status(StatusCode::OK) + .header( + CONTENT_TYPE, + HeaderValue::from_static("text/event-stream; charset=utf-8"), + ) + .header("Cache-Control", HeaderValue::from_static("no-cache")) + .header("Connection", HeaderValue::from_static("keep-alive")) + .body(body) + .unwrap() + } +} + +impl Default for HarmonyStreamingProcessor { + fn default() -> Self { + Self::new() + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/types.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/types.rs new file mode 100644 index 00000000..abb1150c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/harmony/types.rs @@ -0,0 +1,139 @@ +//! Shared types for Harmony pipeline + +use openai_harmony::chat::Content; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::protocols::common::ToolCall; + +/// Harmony message format +/// +/// Represents messages in the Harmony encoding format with role and content. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HarmonyMessage { + pub role: String, + pub content: String, +} + +impl HarmonyMessage { + pub fn new(role: impl Into, content: impl Into) -> Self { + Self { + role: role.into(), + content: content.into(), + } + } + + pub fn user(content: impl Into) -> Self { + Self::new("user", content) + } + + pub fn assistant(content: impl Into) -> Self { + Self::new("assistant", content) + } + + pub fn system(content: impl Into) -> Self { + Self::new("system", content) + } + + /// Convert from openai_harmony::chat::Message to our simplified HarmonyMessage + pub fn from_openai_harmony(msg: openai_harmony::chat::Message) -> Self { + // Extract role as string + let role = match msg.author.role { + openai_harmony::chat::Role::User => "user", + openai_harmony::chat::Role::Assistant => "assistant", + openai_harmony::chat::Role::System => "system", + openai_harmony::chat::Role::Developer => "developer", + openai_harmony::chat::Role::Tool => "tool", + } + .to_string(); + + // Extract text content from all Content::Text parts + let content = msg + .content + .iter() + .filter_map(|c| match c { + Content::Text(tc) => Some(tc.text.as_str()), + _ => None, + }) + .collect::>() + .join(""); + + Self { role, content } + } +} + +/// Output from Harmony encoding process +/// +/// Contains the encoded input_ids, stop tokens, selection text for worker routing, +/// and the Harmony message history. +#[derive(Debug, Clone)] +pub struct HarmonyBuildOutput { + /// Encoded token IDs to send to the model + pub input_ids: Vec, + + /// Stop token IDs for this model (injected into sampling params) + pub stop_token_ids: Vec, + + /// Selection text for worker routing (concise snippet from last user message) + pub selection_text: String, + + /// Harmony messages for this conversation (used for history tracking) + pub harmony_messages: Vec, +} + +/// Parsed output from all three Harmony channels +/// +/// Represents the complete response after parsing analysis, commentary, and final channels. +#[derive(Debug, Clone)] +pub struct HarmonyChannelOutput { + /// Analysis/reasoning content (from analysis channel) + pub analysis: Option, + + /// Tool calls (from commentary channel) + pub commentary: Option>, + + /// Final text content (from final channel) + pub final_text: String, + + /// Finish reason + pub finish_reason: String, + + /// Matched stop token (if any) + pub matched_stop: Option, + + /// Number of reasoning tokens (from analysis and commentary channels) + pub reasoning_token_count: u32, +} + +/// Streaming delta for SSE responses +/// +/// Represents incremental updates as tokens are parsed from the stream. +#[derive(Debug, Clone)] +pub struct HarmonyChannelDelta { + /// Delta for analysis/reasoning content + pub analysis_delta: Option, + + /// Delta for tool calls + pub commentary_delta: Option, + + /// Delta for final text content + pub final_delta: Option, + + /// Whether this is the final delta + pub is_final: bool, +} + +/// Tool call delta for streaming +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ToolCallDelta { + pub index: usize, + pub id: Option, + pub function: Option, +} + +/// Function call delta for streaming +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FunctionDelta { + pub name: Option, + pub arguments: Option, +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/mod.rs new file mode 100644 index 00000000..58b017bf --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/mod.rs @@ -0,0 +1,23 @@ +//! gRPC router implementations + +use crate::{grpc_client::sglang_proto::MultimodalInputs, protocols::common::StringOrArray}; + +pub mod client; +pub mod common; +pub mod context; +pub mod error; +pub mod harmony; +pub mod pd_router; +pub mod pipeline; +pub mod proto_wrapper; +pub mod regular; +pub mod router; +pub mod utils; + +/// Processed chat messages ready for gRPC generation +#[derive(Debug)] +pub struct ProcessedMessages { + pub text: String, + pub multimodal_inputs: Option, + pub stop_sequences: Option, +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/pd_router.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/pd_router.rs new file mode 100644 index 00000000..953804c0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/pd_router.rs @@ -0,0 +1,260 @@ +use std::sync::Arc; + +use async_trait::async_trait; +use axum::{ + body::Body, + extract::Request, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, +}; +use tracing::debug; + +use super::{context::SharedComponents, pipeline::RequestPipeline}; +use crate::{ + app_context::AppContext, + core::{ConnectionMode, WorkerRegistry, WorkerType}, + protocols::{ + chat::ChatCompletionRequest, + classify::ClassifyRequest, + completion::CompletionRequest, + embedding::EmbeddingRequest, + generate::GenerateRequest, + rerank::RerankRequest, + responses::{ResponsesGetParams, ResponsesRequest}, + }, + routers::RouterTrait, +}; + +/// gRPC PD (Prefill-Decode) router implementation for SGLang +#[derive(Clone)] +pub struct GrpcPDRouter { + worker_registry: Arc, + pipeline: RequestPipeline, + shared_components: Arc, +} + +impl GrpcPDRouter { + /// Create a new gRPC PD router + pub async fn new(ctx: &Arc) -> Result { + // Get registries from context + let worker_registry = ctx.worker_registry.clone(); + let policy_registry = ctx.policy_registry.clone(); + + // Extract necessary components from context + let tokenizer = ctx + .tokenizer + .as_ref() + .ok_or_else(|| "gRPC PD router requires tokenizer".to_string())? + .clone(); + let reasoning_parser_factory = ctx + .reasoning_parser_factory + .as_ref() + .ok_or_else(|| "gRPC PD router requires reasoning parser factory".to_string())? + .clone(); + let tool_parser_factory = ctx + .tool_parser_factory + .as_ref() + .ok_or_else(|| "gRPC PD router requires tool parser factory".to_string())? + .clone(); + + // Create shared components for pipeline + let shared_components = Arc::new(SharedComponents { + tokenizer: tokenizer.clone(), + tool_parser_factory: tool_parser_factory.clone(), + reasoning_parser_factory: reasoning_parser_factory.clone(), + }); + + // Create PD pipeline + let pipeline = RequestPipeline::new_pd( + worker_registry.clone(), + policy_registry.clone(), + tokenizer.clone(), + tool_parser_factory.clone(), + reasoning_parser_factory.clone(), + ctx.configured_tool_parser.clone(), + ctx.configured_reasoning_parser.clone(), + ); + + Ok(GrpcPDRouter { + worker_registry, + pipeline, + shared_components, + }) + } + + /// Main route_generate implementation with PD dual dispatch + async fn route_generate_impl( + &self, + headers: Option<&HeaderMap>, + body: &GenerateRequest, + model_id: Option<&str>, + ) -> Response { + debug!( + "Processing generate request for model: {:?} (PD mode)", + model_id + ); + + // Use pipeline for ALL requests (streaming and non-streaming) + self.pipeline + .execute_generate( + Arc::new(body.clone()), + headers.cloned(), + model_id.map(|s| s.to_string()), + self.shared_components.clone(), + ) + .await + } + + /// Main route_chat implementation with PD dual dispatch + async fn route_chat_impl( + &self, + headers: Option<&HeaderMap>, + body: &ChatCompletionRequest, + model_id: Option<&str>, + ) -> Response { + debug!( + "Processing chat completion request for model: {:?} (PD mode)", + model_id + ); + + // Use pipeline for ALL requests (streaming and non-streaming) + self.pipeline + .execute_chat( + Arc::new(body.clone()), + headers.cloned(), + model_id.map(|s| s.to_string()), + self.shared_components.clone(), + ) + .await + } +} + +impl std::fmt::Debug for GrpcPDRouter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let prefill_workers = self.worker_registry.get_workers_filtered( + None, + Some(WorkerType::Prefill { + bootstrap_port: None, + }), + Some(ConnectionMode::Grpc { port: None }), + false, + ); + let decode_workers = self.worker_registry.get_workers_filtered( + None, + Some(WorkerType::Decode), + Some(ConnectionMode::Grpc { port: None }), + false, + ); + f.debug_struct("GrpcPDRouter") + .field("prefill_workers_count", &prefill_workers.len()) + .field("decode_workers_count", &decode_workers.len()) + .finish() + } +} + +#[async_trait] +impl RouterTrait for GrpcPDRouter { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + async fn health_generate(&self, _req: Request) -> Response { + ( + StatusCode::NOT_IMPLEMENTED, + "Health generate not yet implemented for gRPC PD", + ) + .into_response() + } + + async fn get_server_info(&self, _req: Request) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn get_models(&self, _req: Request) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn get_model_info(&self, _req: Request) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn route_generate( + &self, + headers: Option<&HeaderMap>, + body: &GenerateRequest, + model_id: Option<&str>, + ) -> Response { + self.route_generate_impl(headers, body, model_id).await + } + + async fn route_chat( + &self, + headers: Option<&HeaderMap>, + body: &ChatCompletionRequest, + model_id: Option<&str>, + ) -> Response { + self.route_chat_impl(headers, body, model_id).await + } + + async fn route_completion( + &self, + _headers: Option<&HeaderMap>, + _body: &CompletionRequest, + _model_id: Option<&str>, + ) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn route_responses( + &self, + _headers: Option<&HeaderMap>, + _body: &ResponsesRequest, + _model_id: Option<&str>, + ) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn get_response( + &self, + _headers: Option<&HeaderMap>, + _response_id: &str, + _params: &ResponsesGetParams, + ) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn cancel_response(&self, _headers: Option<&HeaderMap>, _response_id: &str) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn route_embeddings( + &self, + _headers: Option<&HeaderMap>, + _body: &EmbeddingRequest, + _model_id: Option<&str>, + ) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn route_classify( + &self, + _headers: Option<&HeaderMap>, + _body: &ClassifyRequest, + _model_id: Option<&str>, + ) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn route_rerank( + &self, + _headers: Option<&HeaderMap>, + _body: &RerankRequest, + _model_id: Option<&str>, + ) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + fn router_type(&self) -> &'static str { + "grpc_pd" + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/pipeline.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/pipeline.rs new file mode 100644 index 00000000..cb09f2a5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/pipeline.rs @@ -0,0 +1,470 @@ +//! Pipeline orchestrator for gRPC router request processing +//! +//! This module defines the RequestPipeline orchestrator that coordinates +//! the execution of pipeline stages from request preparation to response delivery. + +use std::sync::Arc; + +use axum::response::{IntoResponse, Response}; +use tracing::error; + +use super::{ + common::stages::*, + context::*, + error, harmony, + regular::{processor, stages::*, streaming}, +}; +use crate::{ + core::WorkerRegistry, + policies::PolicyRegistry, + protocols::{ + chat::{ChatCompletionRequest, ChatCompletionResponse}, + generate::GenerateRequest, + }, + reasoning_parser::ParserFactory as ReasoningParserFactory, + tokenizer::traits::Tokenizer, + tool_parser::ParserFactory as ToolParserFactory, +}; + +/// Generic request pipeline for all request types +/// +/// Orchestrates all stages from request preparation to response delivery. +/// Configured differently for regular vs PD mode. +#[derive(Clone)] +pub struct RequestPipeline { + stages: Arc>>, +} + +impl RequestPipeline { + /// Create a regular (single-worker) pipeline + pub fn new_regular( + worker_registry: Arc, + policy_registry: Arc, + tokenizer: Arc, + tool_parser_factory: ToolParserFactory, + reasoning_parser_factory: ReasoningParserFactory, + configured_tool_parser: Option, + configured_reasoning_parser: Option, + ) -> Self { + let processor = processor::ResponseProcessor::new( + tokenizer.clone(), + tool_parser_factory.clone(), + reasoning_parser_factory.clone(), + configured_tool_parser.clone(), + configured_reasoning_parser.clone(), + ); + + let streaming_processor = Arc::new(streaming::StreamingProcessor::new( + tokenizer, + tool_parser_factory, + reasoning_parser_factory, + configured_tool_parser, + configured_reasoning_parser, + )); + + let stages: Vec> = vec![ + Box::new(PreparationStage::new()), + Box::new(WorkerSelectionStage::new( + worker_registry, + policy_registry, + WorkerSelectionMode::Regular, + )), + Box::new(ClientAcquisitionStage), + Box::new(RequestBuildingStage::new(false)), // No PD metadata + Box::new(DispatchMetadataStage), + Box::new(RequestExecutionStage::new(ExecutionMode::Single)), + Box::new(ResponseProcessingStage::new(processor, streaming_processor)), + ]; + + Self { + stages: Arc::new(stages), + } + } + + /// Create a Harmony (single-worker) pipeline for Harmony-capable models + pub fn new_harmony( + worker_registry: Arc, + policy_registry: Arc, + _tokenizer: Arc, + _tool_parser_factory: ToolParserFactory, + _reasoning_parser_factory: ReasoningParserFactory, + _configured_tool_parser: Option, + _configured_reasoning_parser: Option, + ) -> Self { + let stages: Vec> = vec![ + Box::new(harmony::stages::HarmonyPreparationStage::new()), + Box::new(WorkerSelectionStage::new( + worker_registry, + policy_registry, + WorkerSelectionMode::Regular, + )), + Box::new(ClientAcquisitionStage), + Box::new(harmony::stages::HarmonyRequestBuildingStage::new(false)), + Box::new(DispatchMetadataStage), + Box::new(RequestExecutionStage::new(ExecutionMode::Single)), + Box::new(harmony::stages::HarmonyResponseProcessingStage::new()), + ]; + + Self { + stages: Arc::new(stages), + } + } + + /// Create a Harmony PD (prefill-decode) pipeline + pub fn new_harmony_pd( + worker_registry: Arc, + policy_registry: Arc, + _tokenizer: Arc, + _tool_parser_factory: ToolParserFactory, + _reasoning_parser_factory: ReasoningParserFactory, + _configured_tool_parser: Option, + _configured_reasoning_parser: Option, + ) -> Self { + let stages: Vec> = vec![ + Box::new(harmony::stages::HarmonyPreparationStage::new()), + Box::new(WorkerSelectionStage::new( + worker_registry, + policy_registry, + WorkerSelectionMode::PrefillDecode, + )), + Box::new(ClientAcquisitionStage), + Box::new(harmony::stages::HarmonyRequestBuildingStage::new(true)), + Box::new(DispatchMetadataStage), + Box::new(RequestExecutionStage::new(ExecutionMode::DualDispatch)), + Box::new(harmony::stages::HarmonyResponseProcessingStage::new()), + ]; + + Self { + stages: Arc::new(stages), + } + } + + /// Create a PD (prefill-decode) pipeline + pub fn new_pd( + worker_registry: Arc, + policy_registry: Arc, + tokenizer: Arc, + tool_parser_factory: ToolParserFactory, + reasoning_parser_factory: ReasoningParserFactory, + configured_tool_parser: Option, + configured_reasoning_parser: Option, + ) -> Self { + let processor = processor::ResponseProcessor::new( + tokenizer.clone(), + tool_parser_factory.clone(), + reasoning_parser_factory.clone(), + configured_tool_parser.clone(), + configured_reasoning_parser.clone(), + ); + + let streaming_processor = Arc::new(streaming::StreamingProcessor::new( + tokenizer, + tool_parser_factory, + reasoning_parser_factory, + configured_tool_parser, + configured_reasoning_parser, + )); + + let stages: Vec> = vec![ + Box::new(PreparationStage::new()), + Box::new(WorkerSelectionStage::new( + worker_registry, + policy_registry, + WorkerSelectionMode::PrefillDecode, + )), + Box::new(ClientAcquisitionStage), + Box::new(RequestBuildingStage::new(true)), // Inject PD metadata + Box::new(DispatchMetadataStage), + Box::new(RequestExecutionStage::new(ExecutionMode::DualDispatch)), + Box::new(ResponseProcessingStage::new(processor, streaming_processor)), + ]; + + Self { + stages: Arc::new(stages), + } + } + + /// Execute the complete pipeline for a chat request + pub async fn execute_chat( + &self, + request: Arc, + headers: Option, + model_id: Option, + components: Arc, + ) -> Response { + let mut ctx = RequestContext::for_chat(request, headers, model_id, components); + + for (idx, stage) in self.stages.iter().enumerate() { + match stage.execute(&mut ctx).await { + Ok(Some(response)) => { + // Stage completed successfully with a response (e.g., streaming) + return response; + } + Ok(None) => { + continue; + } + Err(response) => { + // Error occurred + error!( + "Stage {} ({}) failed with status {}", + idx + 1, + stage.name(), + response.status() + ); + return response; + } + } + } + + match ctx.state.response.final_response { + Some(FinalResponse::Chat(response)) => axum::Json(response).into_response(), + Some(FinalResponse::Generate(_)) => { + error!( + function = "execute_chat", + "Wrong response type: expected Chat, got Generate" + ); + error::internal_error("Internal error: wrong response type") + } + None => { + error!( + function = "execute_chat", + "No response produced by pipeline" + ); + error::internal_error("No response produced") + } + } + } + + /// Execute the complete pipeline for a generate request + pub async fn execute_generate( + &self, + request: Arc, + headers: Option, + model_id: Option, + components: Arc, + ) -> Response { + let mut ctx = RequestContext::for_generate(request, headers, model_id, components); + + for (idx, stage) in self.stages.iter().enumerate() { + match stage.execute(&mut ctx).await { + Ok(Some(response)) => { + // Stage completed successfully with a response (e.g., streaming) + return response; + } + Ok(None) => { + continue; + } + Err(response) => { + // Error occurred + error!( + "Stage {} ({}) failed with status {}", + idx + 1, + stage.name(), + response.status() + ); + return response; + } + } + } + + match ctx.state.response.final_response { + Some(FinalResponse::Generate(response)) => axum::Json(response).into_response(), + Some(FinalResponse::Chat(_)) => { + error!( + function = "execute_generate", + "Wrong response type: expected Generate, got Chat" + ); + error::internal_error("Internal error: wrong response type") + } + None => { + error!( + function = "execute_generate", + "No response produced by pipeline" + ); + error::internal_error("No response produced") + } + } + } + + /// Execute chat pipeline for responses endpoint + /// + /// Used by ALL non-streaming /v1/responses requests. + /// Uses the same 7 pipeline stages as execute_chat(), with two differences: + /// 1. Returns Result for tool_loop composition + /// 2. Disallows streaming (responses endpoint uses different SSE format) + pub async fn execute_chat_for_responses( + &self, + request: Arc, + headers: Option, + model_id: Option, + components: Arc, + ) -> Result { + let mut ctx = RequestContext::for_chat(request, headers, model_id, components); + + for (idx, stage) in self.stages.iter().enumerate() { + match stage.execute(&mut ctx).await { + Ok(Some(_response)) => { + // Streaming not supported for responses sync mode + error!( + function = "execute_chat_for_responses", + "Streaming attempted in responses context" + ); + return Err(error::bad_request( + "Streaming is not supported in this context".to_string(), + )); + } + Ok(None) => { + continue; + } + Err(response) => { + // Error occurred - return the response as-is to preserve HTTP status codes + error!( + "Stage {} ({}) failed with status {}", + idx + 1, + stage.name(), + response.status() + ); + return Err(response); + } + } + } + + match ctx.state.response.final_response { + Some(FinalResponse::Chat(response)) => Ok(response), + Some(FinalResponse::Generate(_)) => { + error!( + function = "execute_chat_for_responses", + "Wrong response type: expected Chat, got Generate" + ); + Err(error::internal_error("Internal error: wrong response type")) + } + None => { + error!( + function = "execute_chat_for_responses", + "No response produced by pipeline" + ); + Err(error::internal_error("No response produced")) + } + } + } + + /// Execute Harmony Responses API request through all pipeline stages + /// + /// This method runs a single iteration of the Responses API request, + /// returning either ToolCallsFound (continue serving) or Completed (final response). + /// + /// Called by harmony::responses::serve_harmony_responses() for each iteration. + /// + /// # Arguments + /// + /// * `request` - Responses API request + /// * `ctx` - Harmony Responses context with MCP manager and components + /// + /// # Returns + /// + /// ResponsesIterationResult indicating whether to continue iteration or return + pub async fn execute_harmony_responses( + &self, + request: &crate::protocols::responses::ResponsesRequest, + harmony_ctx: &harmony::responses::HarmonyResponsesContext, + ) -> Result { + // Create RequestContext for this Responses request + let mut ctx = RequestContext::for_responses( + Arc::new(request.clone()), + None, // No headers needed for internal pipeline execution + None, // Model ID already set in request + harmony_ctx.components.clone(), + ); + + for (idx, stage) in self.stages.iter().enumerate() { + match stage.execute(&mut ctx).await { + Ok(Some(response)) => { + // Stage returned early response (e.g., streaming) - not expected for Responses iteration + error!( + "Stage {} ({}) returned unexpected response during Responses iteration", + idx + 1, + stage.name() + ); + return Err(response); + } + Ok(None) => { + continue; + } + Err(response) => { + // Stage failed + error!( + "Stage {} ({}) failed with status {}", + idx + 1, + stage.name(), + response.status() + ); + return Err(response); + } + } + } + + // Extract ResponsesIterationResult from context + // This should have been set by HarmonyResponseProcessingStage + ctx.state + .response + .responses_iteration_result + .take() + .ok_or_else(|| { + error!( + function = "execute_harmony_responses", + "No ResponsesIterationResult produced by pipeline" + ); + error::internal_error("No ResponsesIterationResult produced by pipeline") + }) + } + + /// Execute Harmony Responses pipeline iteration with streaming support + /// + /// This version executes the pipeline up to the dispatch stage and returns + /// the raw ExecutionResult (with stream) for token-level streaming processing. + pub async fn execute_harmony_responses_streaming( + &self, + request: &crate::protocols::responses::ResponsesRequest, + harmony_ctx: &harmony::responses::HarmonyResponsesContext, + ) -> Result { + // Create RequestContext for this Responses request + let mut ctx = RequestContext::for_responses( + Arc::new(request.clone()), + None, + None, + harmony_ctx.components.clone(), + ); + + for (idx, stage) in self.stages.iter().enumerate() { + match stage.execute(&mut ctx).await { + Ok(Some(response)) => { + error!( + "Stage {} ({}) returned unexpected response during streaming Responses", + idx + 1, + stage.name() + ); + return Err(response); + } + Ok(None) => continue, + Err(response) => { + error!( + "Stage {} ({}) failed with status {}", + idx + 1, + stage.name(), + response.status() + ); + return Err(response); + } + } + } + + // Extract execution_result (the raw stream from workers) + ctx.state.response.execution_result.take().ok_or_else(|| { + error!( + function = "execute_harmony_responses_streaming", + "No ExecutionResult produced by pipeline" + ); + error::internal_error("No ExecutionResult produced by pipeline") + }) + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/proto_wrapper.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/proto_wrapper.rs new file mode 100644 index 00000000..493e3fec --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/proto_wrapper.rs @@ -0,0 +1,381 @@ +//! Protocol buffer type wrappers for SGLang and vLLM backends +//! +//! This module provides unified enums that wrap proto types from both SGLang and vLLM, +//! allowing the router to work with either backend transparently. + +use futures_util::StreamExt; + +use crate::grpc_client::{ + sglang_proto::{self as sglang, generate_complete::MatchedStop}, + sglang_scheduler::AbortOnDropStream as SglangStream, + vllm_engine::AbortOnDropStream as VllmStream, + vllm_proto as vllm, +}; + +/// Unified GenerateRequest that works with both backends +#[derive(Clone)] +pub enum ProtoGenerateRequest { + Sglang(Box), + Vllm(Box), +} + +impl ProtoGenerateRequest { + /// Get SGLang variant (panics if vLLM) + pub fn as_sglang(&self) -> &sglang::GenerateRequest { + match self { + Self::Sglang(req) => req, + Self::Vllm(_) => panic!("Expected SGLang GenerateRequest, got vLLM"), + } + } + + /// Get mutable SGLang variant (panics if vLLM) + pub fn as_sglang_mut(&mut self) -> &mut sglang::GenerateRequest { + match self { + Self::Sglang(req) => req, + Self::Vllm(_) => panic!("Expected SGLang GenerateRequest, got vLLM"), + } + } + + /// Get vLLM variant (panics if SGLang) + pub fn as_vllm(&self) -> &vllm::GenerateRequest { + match self { + Self::Vllm(req) => req, + Self::Sglang(_) => panic!("Expected vLLM GenerateRequest, got SGLang"), + } + } + + /// Get mutable vLLM variant (panics if SGLang) + pub fn as_vllm_mut(&mut self) -> &mut vllm::GenerateRequest { + match self { + Self::Vllm(req) => req, + Self::Sglang(_) => panic!("Expected vLLM GenerateRequest, got SGLang"), + } + } + + /// Check if this is SGLang + pub fn is_sglang(&self) -> bool { + matches!(self, Self::Sglang(_)) + } + + /// Check if this is vLLM + pub fn is_vllm(&self) -> bool { + matches!(self, Self::Vllm(_)) + } + + /// Clone the inner request (for passing to generate()) + pub fn clone_inner(&self) -> Self { + self.clone() + } + + /// Get request ID + pub fn request_id(&self) -> &str { + match self { + Self::Sglang(req) => &req.request_id, + Self::Vllm(req) => &req.request_id, + } + } +} + +/// Unified GenerateResponse from stream +pub enum ProtoGenerateResponse { + Sglang(sglang::GenerateResponse), + Vllm(vllm::GenerateResponse), +} + +impl ProtoGenerateResponse { + /// Get the response variant (chunk, complete, or error) + /// + /// Consumes self to avoid cloning large proto messages in hot streaming path + pub fn into_response(self) -> ProtoResponseVariant { + match self { + Self::Sglang(resp) => match resp.response { + Some(sglang::generate_response::Response::Chunk(chunk)) => { + ProtoResponseVariant::Chunk(ProtoGenerateStreamChunk::Sglang(chunk)) + } + Some(sglang::generate_response::Response::Complete(complete)) => { + ProtoResponseVariant::Complete(ProtoGenerateComplete::Sglang(complete)) + } + Some(sglang::generate_response::Response::Error(error)) => { + ProtoResponseVariant::Error(ProtoGenerateError::Sglang(error)) + } + None => ProtoResponseVariant::None, + }, + Self::Vllm(resp) => match resp.response { + Some(vllm::generate_response::Response::Chunk(chunk)) => { + ProtoResponseVariant::Chunk(ProtoGenerateStreamChunk::Vllm(chunk)) + } + Some(vllm::generate_response::Response::Complete(complete)) => { + ProtoResponseVariant::Complete(ProtoGenerateComplete::Vllm(complete)) + } + Some(vllm::generate_response::Response::Error(error)) => { + ProtoResponseVariant::Error(ProtoGenerateError::Vllm(error)) + } + None => ProtoResponseVariant::None, + }, + } + } +} + +/// Response variant extracted from GenerateResponse +pub enum ProtoResponseVariant { + Chunk(ProtoGenerateStreamChunk), + Complete(ProtoGenerateComplete), + Error(ProtoGenerateError), + None, +} + +/// Unified GenerateStreamChunk +#[derive(Clone)] +pub enum ProtoGenerateStreamChunk { + Sglang(sglang::GenerateStreamChunk), + Vllm(vllm::GenerateStreamChunk), +} + +impl ProtoGenerateStreamChunk { + /// Get SGLang variant (panics if vLLM) + pub fn as_sglang(&self) -> &sglang::GenerateStreamChunk { + match self { + Self::Sglang(chunk) => chunk, + Self::Vllm(_) => panic!("Expected SGLang GenerateStreamChunk, got vLLM"), + } + } + + /// Get vLLM variant (panics if SGLang) + pub fn as_vllm(&self) -> &vllm::GenerateStreamChunk { + match self { + Self::Vllm(chunk) => chunk, + Self::Sglang(_) => panic!("Expected vLLM GenerateStreamChunk, got SGLang"), + } + } + + /// Check if this is SGLang + pub fn is_sglang(&self) -> bool { + matches!(self, Self::Sglang(_)) + } + + /// Check if this is vLLM + pub fn is_vllm(&self) -> bool { + matches!(self, Self::Vllm(_)) + } + + /// Get token IDs from chunk (common field) + pub fn token_ids(&self) -> &[u32] { + match self { + Self::Sglang(c) => &c.token_ids, + Self::Vllm(c) => &c.token_ids, + } + } + + /// Get index (for n>1 support) + /// vLLM doesn't support n>1, so always returns 0 + pub fn index(&self) -> u32 { + match self { + Self::Sglang(c) => c.index, + Self::Vllm(_) => 0, // vLLM doesn't support n>1 + } + } + + /// Get output logprobs (SGLang only, returns None for vLLM) + pub fn output_logprobs(&self) -> Option<&sglang::OutputLogProbs> { + match self { + Self::Sglang(c) => c.output_logprobs.as_ref(), + Self::Vllm(_) => None, // TODO: vLLM logprobs mapping + } + } + + /// Get prompt tokens (cumulative) + pub fn prompt_tokens(&self) -> i32 { + match self { + Self::Sglang(c) => c.prompt_tokens, + Self::Vllm(c) => c.prompt_tokens, + } + } + + /// Get completion tokens (cumulative) + pub fn completion_tokens(&self) -> i32 { + match self { + Self::Sglang(c) => c.completion_tokens, + Self::Vllm(c) => c.completion_tokens, + } + } + + /// Get cached tokens (cumulative) + pub fn cached_tokens(&self) -> i32 { + match self { + Self::Sglang(c) => c.cached_tokens, + Self::Vllm(c) => c.cached_tokens, + } + } +} + +/// Unified GenerateComplete response +#[derive(Clone)] +pub enum ProtoGenerateComplete { + Sglang(sglang::GenerateComplete), + Vllm(vllm::GenerateComplete), +} + +impl ProtoGenerateComplete { + /// Get SGLang variant (panics if vLLM) + pub fn as_sglang(&self) -> &sglang::GenerateComplete { + match self { + Self::Sglang(complete) => complete, + Self::Vllm(_) => panic!("Expected SGLang GenerateComplete, got vLLM"), + } + } + + /// Get mutable SGLang variant (panics if vLLM) + pub fn as_sglang_mut(&mut self) -> &mut sglang::GenerateComplete { + match self { + Self::Sglang(complete) => complete, + Self::Vllm(_) => panic!("Expected SGLang GenerateComplete, got vLLM"), + } + } + + /// Get vLLM variant (panics if SGLang) + pub fn as_vllm(&self) -> &vllm::GenerateComplete { + match self { + Self::Vllm(complete) => complete, + Self::Sglang(_) => panic!("Expected vLLM GenerateComplete, got SGLang"), + } + } + + /// Check if this is SGLang + pub fn is_sglang(&self) -> bool { + matches!(self, Self::Sglang(_)) + } + + /// Check if this is vLLM + pub fn is_vllm(&self) -> bool { + matches!(self, Self::Vllm(_)) + } + + /// Get token IDs from either backend (output_ids in proto) + pub fn token_ids(&self) -> &[u32] { + match self { + Self::Sglang(c) => &c.output_ids, + Self::Vllm(c) => &c.output_ids, + } + } + + /// Get prompt tokens + pub fn prompt_tokens(&self) -> i32 { + match self { + Self::Sglang(c) => c.prompt_tokens, + Self::Vllm(c) => c.prompt_tokens, + } + } + + /// Get completion tokens + pub fn completion_tokens(&self) -> i32 { + match self { + Self::Sglang(c) => c.completion_tokens, + Self::Vllm(c) => c.completion_tokens, + } + } + + /// Get finish reason + pub fn finish_reason(&self) -> &str { + match self { + Self::Sglang(c) => &c.finish_reason, + Self::Vllm(c) => &c.finish_reason, + } + } + + /// Get index (for n>1 support) + /// vLLM doesn't support n>1, so always returns 0 + pub fn index(&self) -> u32 { + match self { + Self::Sglang(c) => c.index, + Self::Vllm(_) => 0, // vLLM doesn't have index field (n>1 not supported) + } + } + + /// Get matched stop (SGLang only, returns oneof) + /// vLLM doesn't have matched_stop, returns None + pub fn matched_stop(&self) -> Option<&MatchedStop> { + match self { + Self::Sglang(c) => c.matched_stop.as_ref(), + Self::Vllm(_) => None, // vLLM doesn't have matched_stop + } + } + + /// Get output IDs (decode tokens only) + pub fn output_ids(&self) -> &[u32] { + match self { + Self::Sglang(c) => &c.output_ids, + Self::Vllm(c) => &c.output_ids, + } + } + + /// Get cached tokens + pub fn cached_tokens(&self) -> i32 { + match self { + Self::Sglang(c) => c.cached_tokens, + Self::Vllm(_) => 0, // vLLM doesn't have cached_tokens field + } + } + + /// Get input logprobs (SGLang only) + pub fn input_logprobs(&self) -> Option<&sglang::InputLogProbs> { + match self { + Self::Sglang(c) => c.input_logprobs.as_ref(), + Self::Vllm(_) => None, // vLLM doesn't have input_logprobs + } + } + + /// Get output logprobs + pub fn output_logprobs(&self) -> Option<&sglang::OutputLogProbs> { + match self { + Self::Sglang(c) => c.output_logprobs.as_ref(), + Self::Vllm(_) => None, // TODO: vLLM logprobs mapping + } + } +} + +/// Unified GenerateError +#[derive(Clone)] +pub enum ProtoGenerateError { + Sglang(sglang::GenerateError), + Vllm(vllm::GenerateError), +} + +impl ProtoGenerateError { + /// Get error message + pub fn message(&self) -> &str { + match self { + Self::Sglang(e) => &e.message, + Self::Vllm(e) => &e.message, + } + } +} + +/// Unified stream wrapper +pub enum ProtoStream { + Sglang(SglangStream), + Vllm(VllmStream), +} + +impl ProtoStream { + /// Get next item from stream + pub async fn next(&mut self) -> Option> { + match self { + Self::Sglang(stream) => stream + .next() + .await + .map(|result| result.map(ProtoGenerateResponse::Sglang)), + Self::Vllm(stream) => stream + .next() + .await + .map(|result| result.map(ProtoGenerateResponse::Vllm)), + } + } + + /// Mark stream as completed (no abort needed) + pub fn mark_completed(&mut self) { + match self { + Self::Sglang(stream) => stream.mark_completed(), + Self::Vllm(stream) => stream.mark_completed(), + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/mod.rs new file mode 100644 index 00000000..2303a234 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/mod.rs @@ -0,0 +1,9 @@ +//! Regular (non-harmony) model processing +//! +//! This module contains all code specific to regular tokenizer-based models, +//! including pipeline stages, response processing, and streaming. + +pub mod processor; +pub mod responses; +pub mod stages; +pub mod streaming; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/processor.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/processor.rs new file mode 100644 index 00000000..8a7fd0f4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/processor.rs @@ -0,0 +1,463 @@ +//! Shared response processing logic for gRPC routers +//! +//! This module contains response processing functions that are shared between +//! the regular router and PD router. + +use std::{sync::Arc, time::Instant}; + +use serde_json::Value; +use tracing::error; + +use crate::{ + grpc_client::sglang_proto::generate_complete::MatchedStop, + protocols::{ + chat::{ChatChoice, ChatCompletionMessage, ChatCompletionRequest, ChatCompletionResponse}, + common::{FunctionCallResponse, ToolCall, ToolChoice, ToolChoiceValue}, + generate::{GenerateMetaInfo, GenerateRequest, GenerateResponse}, + }, + reasoning_parser::ParserFactory as ReasoningParserFactory, + routers::grpc::{ + common::{response_collection, response_formatting}, + context::{DispatchMetadata, ExecutionResult}, + error, + proto_wrapper::ProtoGenerateComplete, + utils, + }, + tokenizer::{ + stop::{SequenceDecoderOutput, StopSequenceDecoder}, + traits::Tokenizer, + }, + tool_parser::ParserFactory as ToolParserFactory, +}; + +/// Unified response processor for both routers +#[derive(Clone)] +pub struct ResponseProcessor { + pub tokenizer: Arc, + pub tool_parser_factory: ToolParserFactory, + pub reasoning_parser_factory: ReasoningParserFactory, + pub configured_tool_parser: Option, + pub configured_reasoning_parser: Option, +} + +impl ResponseProcessor { + pub fn new( + tokenizer: Arc, + tool_parser_factory: ToolParserFactory, + reasoning_parser_factory: ReasoningParserFactory, + configured_tool_parser: Option, + configured_reasoning_parser: Option, + ) -> Self { + Self { + tokenizer, + tool_parser_factory, + reasoning_parser_factory, + configured_tool_parser, + configured_reasoning_parser, + } + } + + /// Process a single choice from GenerateComplete response + #[allow(clippy::too_many_arguments)] + pub async fn process_single_choice( + &self, + complete: &ProtoGenerateComplete, + index: usize, + original_request: &ChatCompletionRequest, + stop_decoder: &mut StopSequenceDecoder, + history_tool_calls_count: usize, + reasoning_parser_available: bool, + tool_parser_available: bool, + ) -> Result { + stop_decoder.reset(); + // Decode tokens + let outputs = stop_decoder + .process_tokens(complete.output_ids()) + .map_err(|e| format!("Failed to process tokens: {}", e))?; + + // Accumulate text with early breaks + let mut final_text = String::new(); + for output in outputs { + match output { + SequenceDecoderOutput::Text(t) => final_text.push_str(&t), + SequenceDecoderOutput::StoppedWithText(t) => { + final_text.push_str(&t); + break; + } + SequenceDecoderOutput::Stopped => break, + SequenceDecoderOutput::Held => {} + } + } + + // Flush remaining text + if let SequenceDecoderOutput::Text(t) = stop_decoder.flush() { + final_text.push_str(&t); + } + + // Step 1: Handle reasoning content parsing + let mut reasoning_text: Option = None; + let mut processed_text = final_text; + + if original_request.separate_reasoning && reasoning_parser_available { + let pooled_parser = utils::get_reasoning_parser( + &self.reasoning_parser_factory, + self.configured_reasoning_parser.as_ref(), + &original_request.model, + ); + + let mut parser = pooled_parser.lock().await; + match parser.detect_and_parse_reasoning(&processed_text) { + Ok(result) => { + if !result.reasoning_text.is_empty() { + reasoning_text = Some(result.reasoning_text); + } + processed_text = result.normal_text; + } + Err(e) => { + return Err(format!("Reasoning parsing error: {}", e)); + } + } + } + + // Step 2: Handle tool call parsing + let mut tool_calls: Option> = None; + let tool_choice_enabled = !matches!( + &original_request.tool_choice, + Some(ToolChoice::Value(ToolChoiceValue::None)) + ); + + if tool_choice_enabled && original_request.tools.is_some() { + // Check if JSON schema constraint was used (specific function or required mode) + let used_json_schema = match &original_request.tool_choice { + Some(ToolChoice::Function { .. }) => true, + Some(ToolChoice::Value(ToolChoiceValue::Required)) => true, + Some(ToolChoice::AllowedTools { mode, .. }) => mode == "required", + _ => false, + }; + + if used_json_schema { + (tool_calls, processed_text) = utils::parse_json_schema_response( + &processed_text, + &original_request.tool_choice, + &original_request.model, + history_tool_calls_count, + ); + } else if tool_parser_available { + (tool_calls, processed_text) = self + .parse_tool_calls( + &processed_text, + &original_request.model, + history_tool_calls_count, + ) + .await; + } + } + + // Step 3: Use finish reason directly from proto (already OpenAI-compatible string) + let finish_reason_str = complete.finish_reason(); + + // Override finish reason if we have tool calls + let final_finish_reason_str = if tool_calls.is_some() { + "tool_calls" + } else { + finish_reason_str + }; + + // Extract matched_stop information from proto + let matched_stop = match complete.matched_stop() { + Some(MatchedStop::MatchedTokenId(token_id)) => { + Some(Value::Number(serde_json::Number::from(*token_id))) + } + Some(MatchedStop::MatchedStopStr(stop_str)) => Some(Value::String(stop_str.clone())), + None => None, + }; + + // Step 4: Convert output logprobs if present + let logprobs = if let Some(proto_logprobs) = complete.output_logprobs() { + match utils::convert_proto_to_openai_logprobs(proto_logprobs, &self.tokenizer) { + Ok(logprobs) => Some(logprobs), + Err(e) => { + error!("Failed to convert logprobs: {}", e); + None + } + } + } else { + None + }; + + // Step 5: Build ChatCompletionMessage (proper response message type) + let chat_message = ChatCompletionMessage { + role: "assistant".to_string(), + content: if processed_text.is_empty() { + None + } else { + Some(processed_text) + }, + tool_calls, + reasoning_content: reasoning_text, + }; + + // Step 6: Build ChatChoice + Ok(ChatChoice { + index: index as u32, + message: chat_message, + logprobs, + finish_reason: Some(final_finish_reason_str.to_string()), + matched_stop, + hidden_states: None, + }) + } + + /// Process non-streaming chat response (collects all responses and builds final response) + pub async fn process_non_streaming_chat_response( + &self, + execution_result: ExecutionResult, + chat_request: Arc, + dispatch: DispatchMetadata, + stop_decoder: &mut StopSequenceDecoder, + request_logprobs: bool, + ) -> Result { + // Collect all responses from the execution result + let all_responses = + response_collection::collect_responses(execution_result, request_logprobs).await?; + + let history_tool_calls_count = utils::get_history_tool_calls_count(&chat_request); + + // Check parser availability once upfront (not per choice) + let reasoning_parser_available = chat_request.separate_reasoning + && utils::check_reasoning_parser_availability( + &self.reasoning_parser_factory, + self.configured_reasoning_parser.as_ref(), + &chat_request.model, + ); + + let tool_choice_enabled = !matches!( + &chat_request.tool_choice, + Some(ToolChoice::Value(ToolChoiceValue::None)) + ); + + let tool_parser_available = tool_choice_enabled + && chat_request.tools.is_some() + && utils::check_tool_parser_availability( + &self.tool_parser_factory, + self.configured_tool_parser.as_ref(), + &chat_request.model, + ); + + // Log once per request (not per choice) + if chat_request.separate_reasoning && !reasoning_parser_available { + tracing::debug!( + "No reasoning parser found for model '{}', skipping reasoning parsing", + chat_request.model + ); + } + + if chat_request.tools.is_some() && tool_choice_enabled && !tool_parser_available { + tracing::debug!( + "No tool parser found for model '{}', skipping tool call parsing", + chat_request.model + ); + } + + // Process all choices + let mut choices = Vec::new(); + for (index, complete) in all_responses.iter().enumerate() { + match self + .process_single_choice( + complete, + index, + &chat_request, + stop_decoder, + history_tool_calls_count, + reasoning_parser_available, + tool_parser_available, + ) + .await + { + Ok(choice) => choices.push(choice), + Err(e) => { + return Err(error::internal_error(format!( + "Failed to process choice {}: {}", + index, e + ))); + } + } + } + + // Build usage + let usage = response_formatting::build_usage(&all_responses); + + // Build final ChatCompletionResponse + Ok( + ChatCompletionResponse::builder(&dispatch.request_id, &dispatch.model) + .created(dispatch.created) + .choices(choices) + .usage(usage) + .maybe_system_fingerprint(dispatch.weight_version.clone()) + .build(), + ) + } + + /// Parse tool calls using model-specific parser + pub async fn parse_tool_calls( + &self, + processed_text: &str, + model: &str, + history_tool_calls_count: usize, + ) -> (Option>, String) { + // Get pooled parser for this model + let pooled_parser = utils::get_tool_parser( + &self.tool_parser_factory, + self.configured_tool_parser.as_ref(), + model, + ); + + // Try parsing directly (parser will handle detection internally) + let result = { + let parser = pooled_parser.lock().await; + parser.parse_complete(processed_text).await + // Lock is dropped here + }; + + match result { + Ok((normal_text, parsed_tool_calls)) => { + if parsed_tool_calls.is_empty() { + return (None, normal_text); + } + + let spec_tool_calls = parsed_tool_calls + .into_iter() + .enumerate() + .map(|(index, tc)| { + // Generate ID for this tool call + let id = utils::generate_tool_call_id( + model, + &tc.function.name, + index, + history_tool_calls_count, + ); + ToolCall { + id, + tool_type: "function".to_string(), + function: FunctionCallResponse { + name: tc.function.name, + arguments: Some(tc.function.arguments), + }, + } + }) + .collect(); + (Some(spec_tool_calls), normal_text) + } + Err(e) => { + error!("Tool call parsing error: {}", e); + (None, processed_text.to_string()) + } + } + } + + /// Process non-streaming generate response (collects all responses and builds final response array) + pub async fn process_non_streaming_generate_response( + &self, + execution_result: ExecutionResult, + _generate_request: Arc, + dispatch: DispatchMetadata, + stop_decoder: &mut StopSequenceDecoder, + request_logprobs: bool, + start_time: Instant, + ) -> Result, axum::response::Response> { + // Collect all responses from the execution result + let all_responses = + response_collection::collect_responses(execution_result, request_logprobs).await?; + + // Process each completion + let mut result_array = Vec::new(); + for complete in all_responses { + stop_decoder.reset(); + + // Process tokens through stop decoder + let outputs = match stop_decoder.process_tokens(complete.output_ids()) { + Ok(outputs) => outputs, + Err(e) => { + return Err(error::internal_error(format!( + "Failed to process tokens: {}", + e + ))) + } + }; + + // Accumulate text with early breaks + let mut decoded_text = String::new(); + for output in outputs { + match output { + SequenceDecoderOutput::Text(t) => decoded_text.push_str(&t), + SequenceDecoderOutput::StoppedWithText(t) => { + decoded_text.push_str(&t); + break; + } + SequenceDecoderOutput::Stopped => break, + SequenceDecoderOutput::Held => {} + } + } + + // Flush remaining text + if let SequenceDecoderOutput::Text(t) = stop_decoder.flush() { + decoded_text.push_str(&t); + } + + let output_ids = complete.output_ids().to_vec(); + let finish_reason_str = complete.finish_reason(); + + // Parse finish_reason from string to proper type + let finish_reason = + utils::parse_finish_reason(finish_reason_str, complete.completion_tokens()); + + // Handle matched_stop if present + let matched_stop = complete.matched_stop().map(|matched| match matched { + MatchedStop::MatchedTokenId(id) => serde_json::json!(id), + MatchedStop::MatchedStopStr(s) => serde_json::json!(s), + }); + + // Extract logprobs if requested (convert proto types to Generate format) + let input_token_logprobs = if request_logprobs { + complete + .input_logprobs() + .map(utils::convert_generate_input_logprobs) + } else { + None + }; + + let output_token_logprobs = if request_logprobs { + complete + .output_logprobs() + .map(utils::convert_generate_output_logprobs) + } else { + None + }; + + // Build GenerateResponse struct + let meta_info = GenerateMetaInfo { + id: dispatch.request_id.clone(), + finish_reason, + prompt_tokens: complete.prompt_tokens() as u32, + weight_version: dispatch + .weight_version + .clone() + .unwrap_or_else(|| "default".to_string()), + input_token_logprobs, + output_token_logprobs, + completion_tokens: complete.completion_tokens() as u32, + cached_tokens: complete.cached_tokens() as u32, + e2e_latency: start_time.elapsed().as_secs_f64(), + matched_stop, + }; + + result_array.push(GenerateResponse { + text: decoded_text, + output_ids, + meta_info, + }); + } + + Ok(result_array) + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/context.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/context.rs new file mode 100644 index 00000000..80c0b98a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/context.rs @@ -0,0 +1,70 @@ +//! Context for /v1/responses endpoint handlers +//! +//! Bundles all dependencies needed by responses handlers to avoid passing +//! 10+ parameters to every function. + +use std::{collections::HashMap, sync::Arc}; + +use tokio::sync::RwLock; + +use super::types::BackgroundTaskInfo; +use crate::{ + core::WorkerRegistry, + data_connector::{ConversationItemStorage, ConversationStorage, ResponseStorage}, + mcp::McpManager, + routers::grpc::{context::SharedComponents, pipeline::RequestPipeline}, +}; + +/// Context for /v1/responses endpoint +/// +/// All fields are Arc/shared references, so cloning this context is cheap. +#[derive(Clone)] +pub struct ResponsesContext { + /// Chat pipeline for executing requests + pub pipeline: Arc, + + /// Shared components (tokenizer, parsers, worker_registry) + pub components: Arc, + + /// Worker registry for validation + pub worker_registry: Arc, + + /// Response storage backend + pub response_storage: Arc, + + /// Conversation storage backend + pub conversation_storage: Arc, + + /// Conversation item storage backend + pub conversation_item_storage: Arc, + + /// MCP manager for tool support + pub mcp_manager: Arc, + + /// Background task handles for cancellation support + pub background_tasks: Arc>>, +} + +impl ResponsesContext { + /// Create a new responses context + pub fn new( + pipeline: Arc, + components: Arc, + worker_registry: Arc, + response_storage: Arc, + conversation_storage: Arc, + conversation_item_storage: Arc, + mcp_manager: Arc, + ) -> Self { + Self { + pipeline, + components, + worker_registry, + response_storage, + conversation_storage, + conversation_item_storage, + mcp_manager, + background_tasks: Arc::new(RwLock::new(HashMap::new())), + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/conversions.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/conversions.rs new file mode 100644 index 00000000..e89d47f6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/conversions.rs @@ -0,0 +1,427 @@ +//! Conversion utilities for translating between /v1/responses and /v1/chat/completions formats +//! +//! This module implements the conversion approach where: +//! 1. ResponsesRequest → ChatCompletionRequest (for backend processing) +//! 2. ChatCompletionResponse → ResponsesResponse (for client response) +//! +//! This allows the gRPC router to reuse the existing chat pipeline infrastructure +//! without requiring Python backend changes. + +use crate::{ + protocols::{ + chat::{ChatCompletionRequest, ChatCompletionResponse, ChatMessage, MessageContent}, + common::{ + FunctionCallResponse, JsonSchemaFormat, ResponseFormat, StreamOptions, ToolCall, + UsageInfo, + }, + responses::{ + ResponseContentPart, ResponseInput, ResponseInputOutputItem, ResponseOutputItem, + ResponseReasoningContent::ReasoningText, ResponseStatus, ResponsesRequest, + ResponsesResponse, ResponsesUsage, StringOrContentParts, TextConfig, TextFormat, + }, + }, + routers::grpc::common::responses::utils::extract_tools_from_response_tools, +}; + +/// Convert a ResponsesRequest to ChatCompletionRequest for processing through the chat pipeline +/// +/// # Conversion Logic +/// - `input` (text/items) → `messages` (chat messages) +/// - `instructions` → system message (prepended) +/// - `max_output_tokens` → `max_completion_tokens` +/// - `tools` → function tools extracted from ResponseTools +/// - `tool_choice` → passed through from request +/// - Response-specific fields (previous_response_id, conversation) are handled by router +pub fn responses_to_chat(req: &ResponsesRequest) -> Result { + let mut messages = Vec::new(); + + // 1. Add system message if instructions provided + if let Some(instructions) = &req.instructions { + messages.push(ChatMessage::System { + content: MessageContent::Text(instructions.clone()), + name: None, + }); + } + + // 2. Convert input to chat messages + match &req.input { + ResponseInput::Text(text) => { + // Simple text input → user message + messages.push(ChatMessage::User { + content: MessageContent::Text(text.clone()), + name: None, + }); + } + ResponseInput::Items(items) => { + // Structured items → convert each to appropriate chat message + for item in items { + match item { + ResponseInputOutputItem::SimpleInputMessage { content, role, .. } => { + // Convert SimpleInputMessage to chat message + let text = match content { + StringOrContentParts::String(s) => s.clone(), + StringOrContentParts::Array(parts) => { + // Extract text from content parts (only InputText supported) + parts + .iter() + .filter_map(|part| match part { + ResponseContentPart::InputText { text } => { + Some(text.as_str()) + } + _ => None, + }) + .collect::>() + .join(" ") + } + }; + + messages.push(role_to_chat_message(role.as_str(), text)); + } + ResponseInputOutputItem::Message { role, content, .. } => { + // Extract text from content parts + let text = extract_text_from_content(content); + + messages.push(role_to_chat_message(role.as_str(), text)); + } + ResponseInputOutputItem::FunctionToolCall { + id, + name, + arguments, + output, + .. + } => { + // Tool call from history - add as assistant message with tool call + // followed by tool response if output exists + + // Add assistant message with tool_calls (the LLM's decision) + messages.push(ChatMessage::Assistant { + content: None, + name: None, + tool_calls: Some(vec![ToolCall { + id: id.clone(), + tool_type: "function".to_string(), + function: FunctionCallResponse { + name: name.clone(), + arguments: Some(arguments.clone()), + }, + }]), + reasoning_content: None, + }); + + // Add tool result message if output exists + if let Some(output_text) = output { + messages.push(ChatMessage::Tool { + content: MessageContent::Text(output_text.clone()), + tool_call_id: id.clone(), + }); + } + } + ResponseInputOutputItem::Reasoning { content, .. } => { + // Reasoning content - add as assistant message with reasoning_content + let reasoning_text = content + .iter() + .map(|c| match c { + ReasoningText { text } => text.as_str(), + }) + .collect::>() + .join("\n"); + + messages.push(ChatMessage::Assistant { + content: None, + name: None, + tool_calls: None, + reasoning_content: Some(reasoning_text), + }); + } + ResponseInputOutputItem::FunctionCallOutput { + call_id, output, .. + } => { + // Function call output - add as tool message + // Note: The function name is looked up from prev_outputs in Harmony path + // For Chat path, we just use the call_id + messages.push(ChatMessage::Tool { + content: MessageContent::Text(output.clone()), + tool_call_id: call_id.clone(), + }); + } + } + } + } + } + + // Ensure we have at least one message + if messages.is_empty() { + return Err("Request must contain at least one message".to_string()); + } + + // 3. Extract function tools from ResponseTools + // Only function tools are extracted here (include_mcp: false). + // MCP tools are merged later by the tool loop (see tool_loop.rs:prepare_chat_tools_and_choice) + // before the chat pipeline, where tool_choice constraints are applied to ALL tools combined. + let function_tools = extract_tools_from_response_tools(req.tools.as_deref(), false); + let tools = if function_tools.is_empty() { + None + } else { + Some(function_tools) + }; + + // 4. Build ChatCompletionRequest + let is_streaming = req.stream.unwrap_or(false); + + Ok(ChatCompletionRequest { + messages, + model: if req.model.is_empty() { + "default".to_string() + } else { + req.model.clone() + }, + temperature: req.temperature, + max_completion_tokens: req.max_output_tokens, + stream: is_streaming, + stream_options: if is_streaming { + Some(StreamOptions { + include_usage: Some(true), + }) + } else { + None + }, + parallel_tool_calls: req.parallel_tool_calls, + top_logprobs: req.top_logprobs, + top_p: req.top_p, + skip_special_tokens: true, + tools, + tool_choice: req.tool_choice.clone(), + response_format: map_text_to_response_format(&req.text), + ..Default::default() + }) +} + +/// Extract text content from ResponseContentPart array +fn extract_text_from_content(content: &[ResponseContentPart]) -> String { + content + .iter() + .filter_map(|part| match part { + ResponseContentPart::InputText { text } => Some(text.as_str()), + ResponseContentPart::OutputText { text, .. } => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("") +} + +/// Convert role and text to ChatMessage +fn role_to_chat_message(role: &str, text: String) -> ChatMessage { + match role { + "user" => ChatMessage::User { + content: MessageContent::Text(text), + name: None, + }, + "assistant" => ChatMessage::Assistant { + content: Some(MessageContent::Text(text)), + name: None, + tool_calls: None, + reasoning_content: None, + }, + "system" => ChatMessage::System { + content: MessageContent::Text(text), + name: None, + }, + _ => { + // Unknown role, treat as user message + ChatMessage::User { + content: MessageContent::Text(text), + name: None, + } + } + } +} + +/// Map TextConfig from Responses API to ResponseFormat for Chat API +/// +/// Converts the structured output configuration from the Responses API format +/// to the Chat API format for non-Harmony models. +fn map_text_to_response_format(text: &Option) -> Option { + let text_config = text.as_ref()?; + let format = text_config.format.as_ref()?; + + match format { + TextFormat::Text => Some(ResponseFormat::Text), + TextFormat::JsonObject => Some(ResponseFormat::JsonObject), + TextFormat::JsonSchema { + name, + schema, + description: _, + strict, + } => Some(ResponseFormat::JsonSchema { + json_schema: JsonSchemaFormat { + name: name.clone(), + schema: schema.clone(), + strict: *strict, + }, + }), + } +} + +/// Convert a ChatCompletionResponse to ResponsesResponse +/// +/// # Conversion Logic +/// - `id` → `response_id_override` if provided, otherwise `chat_resp.id` +/// - `model` → `model` (pass through) +/// - `choices[0].message` → `output` array (convert to ResponseOutputItem::Message) +/// - `choices[0].finish_reason` → determines `status` (stop/length → Completed) +/// - `created` timestamp → `created_at` +pub fn chat_to_responses( + chat_resp: &ChatCompletionResponse, + original_req: &ResponsesRequest, + response_id_override: Option, +) -> Result { + // Extract the first choice (responses API doesn't support n>1) + let choice = chat_resp + .choices + .first() + .ok_or_else(|| "Chat response contains no choices".to_string())?; + + // Convert assistant message to output items + let mut output: Vec = Vec::new(); + + // Convert message content to output item + if let Some(content) = &choice.message.content { + if !content.is_empty() { + output.push(ResponseOutputItem::Message { + id: format!("msg_{}", chat_resp.id), + role: "assistant".to_string(), + content: vec![ResponseContentPart::OutputText { + text: content.clone(), + annotations: vec![], + logprobs: choice.logprobs.clone(), + }], + status: "completed".to_string(), + }); + } + } + + // Convert reasoning content if present (O1-style models) + if let Some(reasoning) = &choice.message.reasoning_content { + if !reasoning.is_empty() { + output.push(ResponseOutputItem::Reasoning { + id: format!("reasoning_{}", chat_resp.id), + summary: vec![], + content: vec![ReasoningText { + text: reasoning.clone(), + }], + status: Some("completed".to_string()), + }); + } + } + + // Convert tool calls if present + if let Some(tool_calls) = &choice.message.tool_calls { + for tool_call in tool_calls { + output.push(ResponseOutputItem::FunctionToolCall { + id: tool_call.id.clone(), + call_id: tool_call.id.clone(), + name: tool_call.function.name.clone(), + arguments: tool_call.function.arguments.clone().unwrap_or_default(), + output: None, // Tool hasn't been executed yet + status: "in_progress".to_string(), + }); + } + } + + // Determine response status based on finish_reason + let status = match choice.finish_reason.as_deref() { + Some("stop") | Some("length") => ResponseStatus::Completed, + Some("tool_calls") => ResponseStatus::InProgress, // Waiting for tool execution + Some("failed") | Some("error") => ResponseStatus::Failed, + _ => ResponseStatus::Completed, // Default to completed + }; + + // Convert usage from Usage to UsageInfo, then wrap in ResponsesUsage + let usage = chat_resp.usage.as_ref().map(|u| { + let usage_info = UsageInfo { + prompt_tokens: u.prompt_tokens, + completion_tokens: u.completion_tokens, + total_tokens: u.total_tokens, + reasoning_tokens: u + .completion_tokens_details + .as_ref() + .and_then(|d| d.reasoning_tokens), + prompt_tokens_details: None, // Chat response doesn't have this + }; + ResponsesUsage::Classic(usage_info) + }); + + // Generate response + let response_id = response_id_override.unwrap_or_else(|| chat_resp.id.clone()); + Ok(ResponsesResponse::builder(&response_id, &chat_resp.model) + .copy_from_request(original_req) + .created_at(chat_resp.created as i64) + .status(status) + .output(output) + .maybe_text(original_req.text.clone()) + .maybe_usage(usage) + .build()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_text_input_conversion() { + let req = ResponsesRequest { + input: ResponseInput::Text("Hello, world!".to_string()), + instructions: Some("You are a helpful assistant.".to_string()), + model: "gpt-4".to_string(), + temperature: Some(0.7), + ..Default::default() + }; + + let chat_req = responses_to_chat(&req).unwrap(); + assert_eq!(chat_req.messages.len(), 2); // system + user + assert_eq!(chat_req.model, "gpt-4"); + assert_eq!(chat_req.temperature, Some(0.7)); + } + + #[test] + fn test_items_input_conversion() { + let req = ResponsesRequest { + input: ResponseInput::Items(vec![ + ResponseInputOutputItem::Message { + id: "msg_1".to_string(), + role: "user".to_string(), + content: vec![ResponseContentPart::InputText { + text: "Hello!".to_string(), + }], + status: None, + }, + ResponseInputOutputItem::Message { + id: "msg_2".to_string(), + role: "assistant".to_string(), + content: vec![ResponseContentPart::OutputText { + text: "Hi there!".to_string(), + annotations: vec![], + logprobs: None, + }], + status: None, + }, + ]), + ..Default::default() + }; + + let chat_req = responses_to_chat(&req).unwrap(); + assert_eq!(chat_req.messages.len(), 2); // user + assistant + } + + #[test] + fn test_empty_input_error() { + let req = ResponsesRequest { + input: ResponseInput::Text("".to_string()), + ..Default::default() + }; + + // Empty text should still create a user message, so this should succeed + let result = responses_to_chat(&req); + assert!(result.is_ok()); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/handlers.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/handlers.rs new file mode 100644 index 00000000..967d06c6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/handlers.rs @@ -0,0 +1,814 @@ +//! Handler functions for /v1/responses endpoints +//! +//! # Public API +//! +//! - `route_responses()` - POST /v1/responses (main entry point) +//! - `get_response_impl()` - GET /v1/responses/{response_id} +//! - `cancel_response_impl()` - POST /v1/responses/{response_id}/cancel +//! +//! # Architecture +//! +//! This module orchestrates all request handling for the /v1/responses endpoint. +//! It supports two execution modes: +//! +//! 1. **Synchronous** - Returns complete response immediately +//! 2. **Streaming** - Returns SSE stream with real-time events +//! +//! Note: Background mode is no longer supported. Requests with background=true +//! will be rejected with a 400 error. +//! +//! # Request Flow +//! +//! ```text +//! route_responses() +//! ├─► route_responses_sync() → route_responses_internal() +//! └─► route_responses_streaming() → convert_chat_stream_to_responses_stream() +//! +//! route_responses_internal() +//! ├─► load_conversation_history() +//! ├─► execute_tool_loop() (if MCP tools) +//! │ └─► pipeline.execute_chat_for_responses() [loop] +//! └─► execute_without_mcp() (if no MCP tools) +//! └─► pipeline.execute_chat_for_responses() +//! ``` + +use std::sync::Arc; + +use axum::{ + body::Body, + http::{self, StatusCode}, + response::{IntoResponse, Response}, +}; +use bytes::Bytes; +use futures_util::StreamExt; +use serde_json::json; +use tokio::sync::mpsc; +use tracing::{debug, error, warn}; +use uuid::Uuid; + +use super::{ + conversions, + tool_loop::{execute_tool_loop, execute_tool_loop_streaming}, +}; +use crate::{ + data_connector::{ + self, ConversationId, ConversationItemStorage, ConversationStorage, ResponseId, + ResponseStorage, + }, + protocols::{ + chat::{self, ChatCompletionStreamResponse}, + common::{self}, + responses::{ + self, ResponseContentPart, ResponseInput, ResponseInputOutputItem, ResponseOutputItem, + ResponseReasoningContent, ResponseStatus, ResponsesRequest, ResponsesResponse, + ResponsesUsage, + }, + }, + routers::grpc::{ + common::responses::{ + build_sse_response, ensure_mcp_connection, persist_response_if_needed, + streaming::ResponseStreamEventEmitter, + }, + error, + }, +}; + +/// Main handler for POST /v1/responses +/// +/// Validates request, determines execution mode (sync/async/streaming), and delegates +pub async fn route_responses( + ctx: &super::context::ResponsesContext, + request: Arc, + headers: Option, + model_id: Option, +) -> Response { + // 1. Reject background mode (no longer supported) + let is_background = request.background.unwrap_or(false); + if is_background { + return ( + StatusCode::BAD_REQUEST, + axum::Json(json!({ + "error": { + "message": "Background mode is not supported. Please set 'background' to false or omit it.", + "type": "invalid_request_error", + "param": "background", + "code": "unsupported_parameter" + } + })), + ) + .into_response(); + } + + // 2. Route based on execution mode + let is_streaming = request.stream.unwrap_or(false); + if is_streaming { + route_responses_streaming(ctx, request, headers, model_id).await + } else { + // Generate response ID for synchronous execution + let response_id = Some(format!("resp_{}", Uuid::new_v4())); + route_responses_sync(ctx, request, headers, model_id, response_id).await + } +} + +// ============================================================================ +// Synchronous Execution +// ============================================================================ + +/// Execute synchronous responses request +/// +/// This is the core execution path that: +/// 1. Loads conversation history / response chain +/// 2. Converts to ChatCompletionRequest +/// 3. Executes chat pipeline +/// 4. Converts back to ResponsesResponse +/// 5. Persists to storage +async fn route_responses_sync( + ctx: &super::context::ResponsesContext, + request: Arc, + headers: Option, + model_id: Option, + response_id: Option, +) -> Response { + match route_responses_internal(ctx, request, headers, model_id, response_id).await { + Ok(responses_response) => axum::Json(responses_response).into_response(), + Err(response) => response, // Already a Response with proper status code + } +} + +/// Internal implementation that returns Result for background task compatibility +async fn route_responses_internal( + ctx: &super::context::ResponsesContext, + request: Arc, + headers: Option, + model_id: Option, + response_id: Option, +) -> Result { + // 1. Load conversation history and build modified request + let modified_request = load_conversation_history(ctx, &request).await?; + + // 2. Check MCP connection and get whether MCP tools are present + let has_mcp_tools = ensure_mcp_connection(&ctx.mcp_manager, request.tools.as_deref()).await?; + + let responses_response = if has_mcp_tools { + debug!("MCP tools detected, using tool loop"); + + // Execute with MCP tool loop + execute_tool_loop( + ctx, + modified_request, + &request, + headers, + model_id, + response_id.clone(), + ) + .await? + } else { + // No MCP tools - execute without MCP (may have function tools or no tools) + execute_without_mcp( + ctx, + &modified_request, + &request, + headers, + model_id, + response_id.clone(), + ) + .await? + }; + + // 5. Persist response to storage if store=true + persist_response_if_needed( + ctx.conversation_storage.clone(), + ctx.conversation_item_storage.clone(), + ctx.response_storage.clone(), + &responses_response, + &request, + ) + .await; + + Ok(responses_response) +} + +/// Execute streaming responses request +async fn route_responses_streaming( + ctx: &super::context::ResponsesContext, + request: Arc, + headers: Option, + model_id: Option, +) -> Response { + // 1. Load conversation history + let modified_request = match load_conversation_history(ctx, &request).await { + Ok(req) => req, + Err(response) => return response, // Already a Response with proper status code + }; + + // 2. Check MCP connection and get whether MCP tools are present + let has_mcp_tools = + match ensure_mcp_connection(&ctx.mcp_manager, request.tools.as_deref()).await { + Ok(has_mcp) => has_mcp, + Err(response) => return response, + }; + + if has_mcp_tools { + debug!("MCP tools detected in streaming mode, using streaming tool loop"); + + return execute_tool_loop_streaming(ctx, modified_request, &request, headers, model_id) + .await; + } + + // 3. Convert ResponsesRequest → ChatCompletionRequest + let chat_request = match conversions::responses_to_chat(&modified_request) { + Ok(req) => Arc::new(req), + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + axum::Json(json!({ + "error": { + "message": format!("Failed to convert request: {}", e), + "type": "invalid_request_error" + } + })), + ) + .into_response(); + } + }; + + // 4. Execute chat pipeline and convert streaming format (no MCP tools) + convert_chat_stream_to_responses_stream(ctx, chat_request, headers, model_id, &request).await +} + +/// Convert chat streaming response to responses streaming format +/// +/// This function: +/// 1. Gets chat SSE stream from pipeline +/// 2. Intercepts and parses each SSE event +/// 3. Converts ChatCompletionStreamResponse → ResponsesResponse delta +/// 4. Accumulates response state for final persistence +/// 5. Emits transformed SSE events in responses format +async fn convert_chat_stream_to_responses_stream( + ctx: &super::context::ResponsesContext, + chat_request: Arc, + headers: Option, + model_id: Option, + original_request: &ResponsesRequest, +) -> Response { + debug!("Converting chat SSE stream to responses SSE format"); + + // Get chat streaming response + let chat_response = ctx + .pipeline + .execute_chat( + chat_request.clone(), + headers, + model_id, + ctx.components.clone(), + ) + .await; + + // Extract body from chat response + let (_parts, body) = chat_response.into_parts(); + + // Create channel for transformed SSE events + let (tx, rx) = mpsc::unbounded_channel::>(); + + // Spawn background task to transform stream + let original_request_clone = original_request.clone(); + let chat_request_clone = chat_request.clone(); + let response_storage = ctx.response_storage.clone(); + let conversation_storage = ctx.conversation_storage.clone(); + let conversation_item_storage = ctx.conversation_item_storage.clone(); + + tokio::spawn(async move { + if let Err(e) = process_and_transform_sse_stream( + body, + original_request_clone, + chat_request_clone, + response_storage, + conversation_storage, + conversation_item_storage, + tx.clone(), + ) + .await + { + warn!("Error transforming SSE stream: {}", e); + let error_event = json!({ + "error": { + "message": e, + "type": "stream_error" + } + }); + let _ = tx.send(Ok(Bytes::from(format!("data: {}\n\n", error_event)))); + } + + // Send final [DONE] event + let _ = tx.send(Ok(Bytes::from("data: [DONE]\n\n"))); + }); + + // Build SSE response with transformed stream + build_sse_response(rx) +} + +/// Process chat SSE stream and transform to responses format +async fn process_and_transform_sse_stream( + body: Body, + original_request: ResponsesRequest, + _chat_request: Arc, + response_storage: Arc, + conversation_storage: Arc, + conversation_item_storage: Arc, + tx: mpsc::UnboundedSender>, +) -> Result<(), String> { + // Create accumulator for final response + let mut accumulator = StreamingResponseAccumulator::new(&original_request); + + // Create event emitter for OpenAI-compatible streaming + let response_id = format!("resp_{}", Uuid::new_v4()); + let model = original_request.model.clone(); + let created_at = chrono::Utc::now().timestamp() as u64; + let mut event_emitter = ResponseStreamEventEmitter::new(response_id, model, created_at); + event_emitter.set_original_request(original_request.clone()); + + // Emit initial response.created and response.in_progress events + let event = event_emitter.emit_created(); + event_emitter + .send_event(&event, &tx) + .map_err(|_| "Failed to send response.created event".to_string())?; + + let event = event_emitter.emit_in_progress(); + event_emitter + .send_event(&event, &tx) + .map_err(|_| "Failed to send response.in_progress event".to_string())?; + + // Convert body to data stream + let mut stream = body.into_data_stream(); + + // Process stream chunks (each chunk is a complete SSE event) + while let Some(chunk_result) = stream.next().await { + let chunk = chunk_result.map_err(|e| format!("Stream read error: {}", e))?; + + // Convert chunk to string + let event_str = String::from_utf8_lossy(&chunk); + let event = event_str.trim(); + + // Check for end of stream + if event == "data: [DONE]" { + break; + } + + // Parse SSE event (format: "data: {...}\n\n" or "data: {...}") + if let Some(json_str) = event.strip_prefix("data: ") { + let json_str = json_str.trim(); + + // Try to parse as ChatCompletionStreamResponse + match serde_json::from_str::(json_str) { + Ok(chat_chunk) => { + // Update accumulator + accumulator.process_chunk(&chat_chunk); + + // Process chunk through event emitter (emits proper OpenAI events) + event_emitter.process_chunk(&chat_chunk, &tx)?; + } + Err(_) => { + // Not a valid chat chunk - might be error event, pass through + debug!("Non-chunk SSE event, passing through: {}", event); + if tx.send(Ok(Bytes::from(format!("{}\n\n", event)))).is_err() { + return Err("Client disconnected".to_string()); + } + } + } + } + } + + // Emit final response.completed event with accumulated usage + let usage_json = accumulator.usage.as_ref().map(|u| { + let mut usage_obj = json!({ + "input_tokens": u.prompt_tokens, + "output_tokens": u.completion_tokens, + "total_tokens": u.total_tokens + }); + + // Include reasoning_tokens if present + if let Some(details) = &u.completion_tokens_details { + if let Some(reasoning_tokens) = details.reasoning_tokens { + usage_obj["output_tokens_details"] = + json!({ "reasoning_tokens": reasoning_tokens }); + } + } + + usage_obj + }); + + let completed_event = event_emitter.emit_completed(usage_json.as_ref()); + event_emitter.send_event(&completed_event, &tx)?; + + // Finalize and persist accumulated response + let final_response = accumulator.finalize(); + persist_response_if_needed( + conversation_storage, + conversation_item_storage, + response_storage, + &final_response, + &original_request, + ) + .await; + + Ok(()) +} + +/// Response accumulator for streaming responses +struct StreamingResponseAccumulator { + // Response metadata + response_id: String, + model: String, + created_at: i64, + + // Accumulated content + content_buffer: String, + reasoning_buffer: String, + tool_calls: Vec, + + // Completion state + finish_reason: Option, + usage: Option, + + // Original request for final response construction + original_request: ResponsesRequest, +} + +impl StreamingResponseAccumulator { + fn new(original_request: &ResponsesRequest) -> Self { + Self { + response_id: String::new(), + model: String::new(), + created_at: 0, + content_buffer: String::new(), + reasoning_buffer: String::new(), + tool_calls: Vec::new(), + finish_reason: None, + usage: None, + original_request: original_request.clone(), + } + } + + fn process_chunk(&mut self, chunk: &ChatCompletionStreamResponse) { + // Initialize metadata on first chunk + if self.response_id.is_empty() { + self.response_id = chunk.id.clone(); + self.model = chunk.model.clone(); + self.created_at = chunk.created as i64; + } + + // Process first choice (responses API doesn't support n>1) + if let Some(choice) = chunk.choices.first() { + // Accumulate content + if let Some(content) = &choice.delta.content { + self.content_buffer.push_str(content); + } + + // Accumulate reasoning + if let Some(reasoning) = &choice.delta.reasoning_content { + self.reasoning_buffer.push_str(reasoning); + } + + // Process tool call deltas + if let Some(tool_call_deltas) = &choice.delta.tool_calls { + for delta in tool_call_deltas { + // Use index directly (it's a u32, not Option) + let index = delta.index as usize; + + // Ensure we have enough tool calls + while self.tool_calls.len() <= index { + self.tool_calls.push(ResponseOutputItem::FunctionToolCall { + id: String::new(), + call_id: String::new(), + name: String::new(), + arguments: String::new(), + output: None, + status: "in_progress".to_string(), + }); + } + + // Update the tool call at this index + if let ResponseOutputItem::FunctionToolCall { + id, + name, + arguments, + .. + } = &mut self.tool_calls[index] + { + if let Some(delta_id) = &delta.id { + id.push_str(delta_id); + } + if let Some(function) = &delta.function { + if let Some(delta_name) = &function.name { + name.push_str(delta_name); + } + if let Some(delta_args) = &function.arguments { + arguments.push_str(delta_args); + } + } + } + } + } + + // Update finish reason + if let Some(reason) = &choice.finish_reason { + self.finish_reason = Some(reason.clone()); + } + } + + // Update usage + if let Some(usage) = &chunk.usage { + self.usage = Some(usage.clone()); + } + } + + fn finalize(self) -> ResponsesResponse { + let mut output: Vec = Vec::new(); + + // Add message content if present + if !self.content_buffer.is_empty() { + output.push(ResponseOutputItem::Message { + id: format!("msg_{}", self.response_id), + role: "assistant".to_string(), + content: vec![ResponseContentPart::OutputText { + text: self.content_buffer, + annotations: vec![], + logprobs: None, + }], + status: "completed".to_string(), + }); + } + + // Add reasoning if present + if !self.reasoning_buffer.is_empty() { + output.push(ResponseOutputItem::Reasoning { + id: format!("reasoning_{}", self.response_id), + summary: vec![], + content: vec![ResponseReasoningContent::ReasoningText { + text: self.reasoning_buffer, + }], + status: Some("completed".to_string()), + }); + } + + // Add tool calls + output.extend(self.tool_calls); + + // Determine final status + let status = match self.finish_reason.as_deref() { + Some("stop") | Some("length") => ResponseStatus::Completed, + Some("tool_calls") => ResponseStatus::InProgress, + Some("failed") | Some("error") => ResponseStatus::Failed, + _ => ResponseStatus::Completed, + }; + + // Convert usage + let usage = self.usage.as_ref().map(|u| { + let usage_info = common::UsageInfo { + prompt_tokens: u.prompt_tokens, + completion_tokens: u.completion_tokens, + total_tokens: u.total_tokens, + reasoning_tokens: u + .completion_tokens_details + .as_ref() + .and_then(|d| d.reasoning_tokens), + prompt_tokens_details: None, + }; + ResponsesUsage::Classic(usage_info) + }); + + ResponsesResponse::builder(&self.response_id, &self.model) + .copy_from_request(&self.original_request) + .created_at(self.created_at) + .status(status) + .output(output) + .maybe_usage(usage) + .build() + } +} + +// ============================================================================ +// Helper Functions +// ============================================================================ + +/// Execute request without MCP tool loop (simple pipeline execution) +async fn execute_without_mcp( + ctx: &super::context::ResponsesContext, + modified_request: &ResponsesRequest, + original_request: &ResponsesRequest, + headers: Option, + model_id: Option, + response_id: Option, +) -> Result { + // Convert ResponsesRequest → ChatCompletionRequest + let chat_request = conversions::responses_to_chat(modified_request).map_err(|e| { + error!( + function = "execute_without_mcp", + error = %e, + "Failed to convert ResponsesRequest to ChatCompletionRequest" + ); + error::bad_request(format!("Failed to convert request: {}", e)) + })?; + + // Execute chat pipeline (errors already have proper HTTP status codes) + let chat_response = ctx + .pipeline + .execute_chat_for_responses( + Arc::new(chat_request), + headers, + model_id, + ctx.components.clone(), + ) + .await?; // Preserve the Response error as-is + + // Convert ChatCompletionResponse → ResponsesResponse + conversions::chat_to_responses(&chat_response, original_request, response_id).map_err(|e| { + error!( + function = "execute_without_mcp", + error = %e, + "Failed to convert ChatCompletionResponse to ResponsesResponse" + ); + error::internal_error(format!("Failed to convert to responses format: {}", e)) + }) +} + +/// Load conversation history and response chains, returning modified request +async fn load_conversation_history( + ctx: &super::context::ResponsesContext, + request: &ResponsesRequest, +) -> Result { + let mut modified_request = request.clone(); + let mut conversation_items: Option> = None; + + // Handle previous_response_id by loading response chain + if let Some(ref prev_id_str) = modified_request.previous_response_id { + let prev_id = ResponseId::from(prev_id_str.as_str()); + match ctx + .response_storage + .get_response_chain(&prev_id, None) + .await + { + Ok(chain) => { + let mut items = Vec::new(); + for stored in chain.responses.iter() { + // Convert input items from stored input (which is now a JSON array) + if let Some(input_arr) = stored.input.as_array() { + for item in input_arr { + match serde_json::from_value::(item.clone()) { + Ok(input_item) => { + items.push(input_item); + } + Err(e) => { + warn!( + "Failed to deserialize stored input item: {}. Item: {}", + e, item + ); + } + } + } + } + + // Convert output items from stored output (which is now a JSON array) + if let Some(output_arr) = stored.output.as_array() { + for item in output_arr { + match serde_json::from_value::(item.clone()) { + Ok(output_item) => { + items.push(output_item); + } + Err(e) => { + warn!( + "Failed to deserialize stored output item: {}. Item: {}", + e, item + ); + } + } + } + } + } + conversation_items = Some(items); + modified_request.previous_response_id = None; + } + Err(e) => { + warn!( + "Failed to load previous response chain for {}: {}", + prev_id_str, e + ); + } + } + } + + // Handle conversation by loading conversation history + if let Some(ref conv_id_str) = request.conversation { + let conv_id = ConversationId::from(conv_id_str.as_str()); + + // Check if conversation exists - return error if not found + let conversation = ctx + .conversation_storage + .get_conversation(&conv_id) + .await + .map_err(|e| { + error!( + function = "load_conversation_history", + conversation_id = %conv_id_str, + error = %e, + "Failed to check conversation existence in storage" + ); + error::internal_error(format!("Failed to check conversation: {}", e)) + })?; + + if conversation.is_none() { + return Err(error::not_found(format!( + "Conversation '{}' not found. Please create the conversation first using the conversations API.", + conv_id_str + ))); + } + + // Load conversation history + const MAX_CONVERSATION_HISTORY_ITEMS: usize = 100; + let params = data_connector::ListParams { + limit: MAX_CONVERSATION_HISTORY_ITEMS, + order: data_connector::SortOrder::Asc, + after: None, + }; + + match ctx + .conversation_item_storage + .list_items(&conv_id, params) + .await + { + Ok(stored_items) => { + let mut items: Vec = Vec::new(); + for item in stored_items.into_iter() { + if item.item_type == "message" { + if let Ok(content_parts) = + serde_json::from_value::>(item.content.clone()) + { + items.push(ResponseInputOutputItem::Message { + id: item.id.0.clone(), + role: item.role.clone().unwrap_or_else(|| "user".to_string()), + content: content_parts, + status: item.status.clone(), + }); + } + } + } + + // Append current request + match &modified_request.input { + ResponseInput::Text(text) => { + items.push(ResponseInputOutputItem::Message { + id: format!("msg_u_{}", conv_id.0), + role: "user".to_string(), + content: vec![ResponseContentPart::InputText { text: text.clone() }], + status: Some("completed".to_string()), + }); + } + ResponseInput::Items(current_items) => { + // Process all item types, converting SimpleInputMessage to Message + for item in current_items.iter() { + let normalized = responses::normalize_input_item(item); + items.push(normalized); + } + } + } + + modified_request.input = ResponseInput::Items(items); + } + Err(e) => { + warn!("Failed to load conversation history: {}", e); + } + } + } + + // If we have conversation_items from previous_response_id, merge them + if let Some(mut items) = conversation_items { + // Append current request + match &modified_request.input { + ResponseInput::Text(text) => { + items.push(ResponseInputOutputItem::Message { + id: format!( + "msg_u_{}", + request + .previous_response_id + .as_ref() + .unwrap_or(&"new".to_string()) + ), + role: "user".to_string(), + content: vec![ResponseContentPart::InputText { text: text.clone() }], + status: Some("completed".to_string()), + }); + } + ResponseInput::Items(current_items) => { + // Process all item types, converting SimpleInputMessage to Message + for item in current_items.iter() { + let normalized = responses::normalize_input_item(item); + items.push(normalized); + } + } + } + + modified_request.input = ResponseInput::Items(items); + } + + Ok(modified_request) +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/mod.rs new file mode 100644 index 00000000..ce67c019 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/mod.rs @@ -0,0 +1,20 @@ +//! Regular gRPC Router `/v1/responses` endpoint implementation +//! +//! This module handles all responses-specific logic for the regular (non-Harmony) pipeline including: +//! - Request validation +//! - Conversation history and response chain loading +//! - Streaming support +//! - MCP tool loop wrapper +//! - Response persistence + +// Module declarations +pub mod context; +mod conversions; +mod handlers; +pub mod tool_loop; +pub mod types; + +// Public exports +pub use context::ResponsesContext; +pub use handlers::route_responses; +pub use types::BackgroundTaskInfo; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/tool_loop.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/tool_loop.rs new file mode 100644 index 00000000..29e274e8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/tool_loop.rs @@ -0,0 +1,1203 @@ +//! MCP tool loop execution for /v1/responses endpoint + +use std::{ + collections::HashMap, + sync::Arc, + time::{SystemTime, UNIX_EPOCH}, +}; + +use axum::{ + body::Body, + http::{header, StatusCode}, + response::Response, +}; +use bytes::Bytes; +use futures_util::StreamExt; +use serde_json::{json, Value}; +use tokio::sync::mpsc; +use tokio_stream::wrappers::UnboundedReceiverStream; +use tracing::{debug, error, warn}; +use uuid::Uuid; + +use super::conversions; +use crate::{ + mcp::{self, McpManager}, + protocols::{ + chat::{ + ChatChoice, ChatCompletionMessage, ChatCompletionRequest, ChatCompletionResponse, + ChatCompletionStreamResponse, + }, + common::{Function, FunctionCallResponse, Tool, ToolCall, ToolChoice, ToolChoiceValue}, + responses::{ + self, McpToolInfo, ResponseContentPart, ResponseInput, ResponseInputOutputItem, + ResponseOutputItem, ResponseStatus, ResponseToolType, ResponsesRequest, + ResponsesResponse, + }, + }, + routers::grpc::{ + common::responses::streaming::{OutputItemType, ResponseStreamEventEmitter}, + error, + }, +}; + +/// Merge function tools from request with MCP tools and set tool_choice based on iteration +fn prepare_chat_tools_and_choice( + chat_request: &mut ChatCompletionRequest, + mcp_chat_tools: &[Tool], + iteration: usize, +) { + // Merge function tools from request with MCP tools + let mut all_tools = chat_request.tools.clone().unwrap_or_default(); + all_tools.extend(mcp_chat_tools.iter().cloned()); + chat_request.tools = Some(all_tools); + + // Set tool_choice based on iteration + // - Iteration 0: Use user's tool_choice or default to auto + // - Iteration 1+: Always use auto to avoid infinite loops + chat_request.tool_choice = if iteration == 0 { + chat_request + .tool_choice + .clone() + .or(Some(ToolChoice::Value(ToolChoiceValue::Auto))) + } else { + Some(ToolChoice::Value(ToolChoiceValue::Auto)) + }; +} + +/// Extract all tool calls from chat response (for parallel tool call support) +fn extract_all_tool_calls_from_chat( + response: &ChatCompletionResponse, +) -> Vec<(String, String, String)> { + // Check if response has choices with tool calls + let Some(choice) = response.choices.first() else { + return Vec::new(); + }; + let message = &choice.message; + + // Look for tool_calls in the message + if let Some(tool_calls) = &message.tool_calls { + tool_calls + .iter() + .map(|tool_call| { + ( + tool_call.id.clone(), + tool_call.function.name.clone(), + tool_call + .function + .arguments + .clone() + .unwrap_or_else(|| "{}".to_string()), + ) + }) + .collect() + } else { + Vec::new() + } +} + +/// State for tracking multi-turn tool calling loop +struct ToolLoopState { + iteration: usize, + total_calls: usize, + conversation_history: Vec, + original_input: ResponseInput, + mcp_call_items: Vec, + server_label: String, +} + +impl ToolLoopState { + fn new(original_input: ResponseInput, server_label: String) -> Self { + Self { + iteration: 0, + total_calls: 0, + conversation_history: Vec::new(), + original_input, + mcp_call_items: Vec::new(), + server_label, + } + } + + fn record_call( + &mut self, + call_id: String, + tool_name: String, + args_json_str: String, + output_str: String, + success: bool, + error: Option, + ) { + // Add function_tool_call item with both arguments and output + self.conversation_history + .push(ResponseInputOutputItem::FunctionToolCall { + id: call_id.clone(), + call_id: call_id.clone(), + name: tool_name.clone(), + arguments: args_json_str.clone(), + output: Some(output_str.clone()), + status: Some("completed".to_string()), + }); + + // Add mcp_call output item for metadata + let mcp_call = build_mcp_call_item( + &tool_name, + &args_json_str, + &output_str, + &self.server_label, + success, + error.as_deref(), + ); + self.mcp_call_items.push(mcp_call); + } +} + +// ============================================================================ +// MCP Metadata Builders +// ============================================================================ + +/// Generate unique ID for MCP items +fn generate_mcp_id(prefix: &str) -> String { + format!("{}_{}", prefix, Uuid::new_v4()) +} + +/// Build mcp_list_tools output item +fn build_mcp_list_tools_item(mcp: &Arc, server_label: &str) -> ResponseOutputItem { + let tools = mcp.list_tools(); + let tools_info: Vec = tools + .iter() + .map(|t| McpToolInfo { + name: t.name.to_string(), + description: t.description.as_ref().map(|d| d.to_string()), + input_schema: Value::Object((*t.input_schema).clone()), + annotations: Some(json!({ + "read_only": false + })), + }) + .collect(); + + ResponseOutputItem::McpListTools { + id: generate_mcp_id("mcpl"), + server_label: server_label.to_string(), + tools: tools_info, + } +} + +/// Build mcp_call output item +fn build_mcp_call_item( + tool_name: &str, + arguments: &str, + output: &str, + server_label: &str, + success: bool, + error: Option<&str>, +) -> ResponseOutputItem { + ResponseOutputItem::McpCall { + id: generate_mcp_id("mcp"), + status: if success { "completed" } else { "failed" }.to_string(), + approval_request_id: None, + arguments: arguments.to_string(), + error: error.map(|e| e.to_string()), + name: tool_name.to_string(), + output: output.to_string(), + server_label: server_label.to_string(), + } +} + +/// Execute the MCP tool calling loop +/// +/// This wraps pipeline.execute_chat_for_responses() in a loop that: +/// 1. Executes the chat pipeline +/// 2. Checks if response has tool calls +/// 3. If yes, executes MCP tools and builds resume request +/// 4. Repeats until no more tool calls or limit reached +pub(super) async fn execute_tool_loop( + ctx: &super::context::ResponsesContext, + mut current_request: ResponsesRequest, + original_request: &ResponsesRequest, + headers: Option, + model_id: Option, + response_id: Option, +) -> Result { + // Get server label from original request tools + let server_label = original_request + .tools + .as_ref() + .and_then(|tools| { + tools + .iter() + .find(|t| matches!(t.r#type, ResponseToolType::Mcp)) + .and_then(|t| t.server_label.clone()) + }) + .unwrap_or_else(|| "request-mcp".to_string()); + + let mut state = ToolLoopState::new(original_request.input.clone(), server_label.clone()); + + // Configuration: max iterations as safety limit + const MAX_ITERATIONS: usize = 10; + let max_tool_calls = original_request.max_tool_calls.map(|n| n as usize); + + debug!( + "Starting MCP tool loop: server_label={}, max_tool_calls={:?}, max_iterations={}", + server_label, max_tool_calls, MAX_ITERATIONS + ); + + // Get MCP tools and convert to chat format (do this once before loop) + let mcp_tools = ctx.mcp_manager.list_tools(); + let mcp_chat_tools = convert_mcp_tools_to_chat_tools(&mcp_tools); + debug!( + "Converted {} MCP tools to chat format", + mcp_chat_tools.len() + ); + + loop { + // Convert to chat request + let mut chat_request = conversions::responses_to_chat(¤t_request).map_err(|e| { + error!( + function = "tool_loop", + iteration = state.iteration, + error = %e, + "Failed to convert ResponsesRequest to ChatCompletionRequest in tool loop" + ); + error::bad_request(format!("Failed to convert request: {}", e)) + })?; + + // Prepare tools and tool_choice for this iteration + prepare_chat_tools_and_choice(&mut chat_request, &mcp_chat_tools, state.iteration); + + // Execute chat pipeline (errors already have proper HTTP status codes) + let chat_response = ctx + .pipeline + .execute_chat_for_responses( + Arc::new(chat_request), + headers.clone(), + model_id.clone(), + ctx.components.clone(), + ) + .await?; + + // Check for function calls (extract all for parallel execution) + let tool_calls = extract_all_tool_calls_from_chat(&chat_response); + + if !tool_calls.is_empty() { + state.iteration += 1; + + debug!( + "Tool loop iteration {}: found {} tool call(s)", + state.iteration, + tool_calls.len() + ); + + // Separate MCP and function tool calls + let mcp_tool_names: std::collections::HashSet<&str> = + mcp_tools.iter().map(|t| t.name.as_ref()).collect(); + let (mcp_tool_calls, function_tool_calls): (Vec<_>, Vec<_>) = tool_calls + .into_iter() + .partition(|(_, tool_name, _)| mcp_tool_names.contains(tool_name.as_str())); + + debug!( + "Separated tool calls: {} MCP, {} function", + mcp_tool_calls.len(), + function_tool_calls.len() + ); + + // If ANY tool call is a function tool, return to caller immediately + if !function_tool_calls.is_empty() { + // Convert chat response to responses format (includes all tool calls) + let responses_response = conversions::chat_to_responses( + &chat_response, + original_request, + response_id.clone(), + ) + .map_err(|e| { + error!( + function = "tool_loop", + iteration = state.iteration, + error = %e, + context = "function_tool_calls", + "Failed to convert ChatCompletionResponse to ResponsesResponse" + ); + error::internal_error(format!("Failed to convert to responses format: {}", e)) + })?; + + // Return response with function tool calls to caller + return Ok(responses_response); + } + + // All MCP tools - check combined limit BEFORE executing + let effective_limit = match max_tool_calls { + Some(user_max) => user_max.min(MAX_ITERATIONS), + None => MAX_ITERATIONS, + }; + + if state.total_calls + mcp_tool_calls.len() > effective_limit { + warn!( + "Reached tool call limit: {} + {} > {} (max_tool_calls={:?}, safety_limit={})", + state.total_calls, + mcp_tool_calls.len(), + effective_limit, + max_tool_calls, + MAX_ITERATIONS + ); + + // Convert chat response to responses format and mark as incomplete + let mut responses_response = conversions::chat_to_responses( + &chat_response, + original_request, + response_id.clone(), + ) + .map_err(|e| { + error!( + function = "tool_loop", + iteration = state.iteration, + error = %e, + context = "max_tool_calls_limit", + "Failed to convert ChatCompletionResponse to ResponsesResponse" + ); + error::internal_error(format!("Failed to convert to responses format: {}", e)) + })?; + + // Mark as completed but with incomplete details + responses_response.status = ResponseStatus::Completed; + responses_response.incomplete_details = Some(json!({ "reason": "max_tool_calls" })); + + return Ok(responses_response); + } + + // Execute all MCP tools + for (call_id, tool_name, args_json_str) in mcp_tool_calls { + debug!( + "Calling MCP tool '{}' (call_id: {}) with args: {}", + tool_name, call_id, args_json_str + ); + + let (output_str, success, error) = match ctx + .mcp_manager + .call_tool(tool_name.as_str(), args_json_str.as_str()) + .await + { + Ok(result) => match serde_json::to_string(&result) { + Ok(output) => (output, true, None), + Err(e) => { + let err = format!("Failed to serialize tool result: {}", e); + warn!("{}", err); + let error_json = json!({ "error": &err }).to_string(); + (error_json, false, Some(err)) + } + }, + Err(err) => { + let err_str = format!("tool call failed: {}", err); + warn!("Tool execution failed: {}", err_str); + // Return error as output, let model decide how to proceed + let error_json = json!({ "error": &err_str }).to_string(); + (error_json, false, Some(err_str)) + } + }; + + // Record the call in state + state.record_call( + call_id, + tool_name, + args_json_str, + output_str, + success, + error, + ); + + // Increment total calls counter + state.total_calls += 1; + } + + // Build resume request with conversation history + // Start with original input + let mut input_items = match &state.original_input { + ResponseInput::Text(text) => vec![ResponseInputOutputItem::Message { + id: format!("msg_u_{}", state.iteration), + role: "user".to_string(), + content: vec![ResponseContentPart::InputText { text: text.clone() }], + status: Some("completed".to_string()), + }], + ResponseInput::Items(items) => { + items.iter().map(responses::normalize_input_item).collect() + } + }; + + // Append all conversation history (function calls and outputs) + input_items.extend_from_slice(&state.conversation_history); + + // Build new request for next iteration + current_request = ResponsesRequest { + input: ResponseInput::Items(input_items), + model: current_request.model.clone(), + instructions: current_request.instructions.clone(), + tools: current_request.tools.clone(), + max_output_tokens: current_request.max_output_tokens, + temperature: current_request.temperature, + top_p: current_request.top_p, + stream: Some(false), // Always non-streaming in tool loop + store: Some(false), // Don't store intermediate responses + background: Some(false), + max_tool_calls: current_request.max_tool_calls, + tool_choice: current_request.tool_choice.clone(), + parallel_tool_calls: current_request.parallel_tool_calls, + previous_response_id: None, + conversation: None, + user: current_request.user.clone(), + metadata: current_request.metadata.clone(), + // Additional fields from ResponsesRequest + include: current_request.include.clone(), + reasoning: current_request.reasoning.clone(), + service_tier: current_request.service_tier.clone(), + top_logprobs: current_request.top_logprobs, + truncation: current_request.truncation.clone(), + text: current_request.text.clone(), + request_id: None, + priority: current_request.priority, + frequency_penalty: current_request.frequency_penalty, + presence_penalty: current_request.presence_penalty, + stop: current_request.stop.clone(), + top_k: current_request.top_k, + min_p: current_request.min_p, + repetition_penalty: current_request.repetition_penalty, + }; + + // Continue to next iteration + } else { + // No more tool calls, we're done + debug!( + "Tool loop completed: {} iterations, {} total calls", + state.iteration, state.total_calls + ); + + // Convert final chat response to responses format + let mut responses_response = conversions::chat_to_responses( + &chat_response, + original_request, + response_id.clone(), + ) + .map_err(|e| { + error!( + function = "tool_loop", + iteration = state.iteration, + error = %e, + context = "final_response", + "Failed to convert ChatCompletionResponse to ResponsesResponse" + ); + error::internal_error(format!("Failed to convert to responses format: {}", e)) + })?; + + // Inject MCP metadata into output + if state.total_calls > 0 { + // Prepend mcp_list_tools item + let mcp_list_tools = build_mcp_list_tools_item(&ctx.mcp_manager, &server_label); + responses_response.output.insert(0, mcp_list_tools); + + // Append all mcp_call items at the end + responses_response.output.extend(state.mcp_call_items); + + debug!( + "Injected MCP metadata: 1 mcp_list_tools + {} mcp_call items", + state.total_calls + ); + } + + return Ok(responses_response); + } + } +} + +/// Execute MCP tool loop with streaming support +/// +/// This streams each iteration's response to the client while accumulating +/// to check for tool calls. If tool calls are found, executes them and +/// continues with the next streaming iteration. +pub(super) async fn execute_tool_loop_streaming( + ctx: &super::context::ResponsesContext, + current_request: ResponsesRequest, + original_request: &ResponsesRequest, + headers: Option, + model_id: Option, +) -> Response { + // Create SSE channel for client + let (tx, rx) = mpsc::unbounded_channel::>(); + + // Clone data for background task + let ctx_clone = ctx.clone(); + let original_request_clone = original_request.clone(); + + // Spawn background task for tool loop + tokio::spawn(async move { + let result = execute_tool_loop_streaming_internal( + &ctx_clone, + current_request, + &original_request_clone, + headers, + model_id, + tx.clone(), + ) + .await; + + if let Err(e) = result { + warn!("Streaming tool loop error: {}", e); + let error_event = json!({ + "error": { + "message": e, + "type": "tool_loop_error" + } + }); + let _ = tx.send(Ok(Bytes::from(format!("data: {}\n\n", error_event)))); + } + + // Send [DONE] + let _ = tx.send(Ok(Bytes::from("data: [DONE]\n\n"))); + }); + + // Build SSE response + let stream = UnboundedReceiverStream::new(rx); + let body = Body::from_stream(stream); + + let mut response = Response::builder() + .status(StatusCode::OK) + .body(body) + .unwrap(); + + response.headers_mut().insert( + header::CONTENT_TYPE, + header::HeaderValue::from_static("text/event-stream"), + ); + response.headers_mut().insert( + header::CACHE_CONTROL, + header::HeaderValue::from_static("no-cache"), + ); + response.headers_mut().insert( + header::CONNECTION, + header::HeaderValue::from_static("keep-alive"), + ); + + response +} + +/// Internal streaming tool loop implementation +async fn execute_tool_loop_streaming_internal( + ctx: &super::context::ResponsesContext, + mut current_request: ResponsesRequest, + original_request: &ResponsesRequest, + headers: Option, + model_id: Option, + tx: mpsc::UnboundedSender>, +) -> Result<(), String> { + // Extract server label from original request tools + let server_label = original_request + .tools + .as_ref() + .and_then(|tools| { + tools + .iter() + .find(|t| matches!(t.r#type, ResponseToolType::Mcp)) + .and_then(|t| t.server_label.clone()) + }) + .unwrap_or_else(|| "request-mcp".to_string()); + + const MAX_ITERATIONS: usize = 10; + let mut state = ToolLoopState::new(original_request.input.clone(), server_label.clone()); + let max_tool_calls = original_request.max_tool_calls.map(|n| n as usize); + + // Create response event emitter + let response_id = format!("resp_{}", Uuid::new_v4()); + let model = current_request.model.clone(); + let created_at = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + let mut emitter = ResponseStreamEventEmitter::new(response_id, model, created_at); + emitter.set_original_request(original_request.clone()); + + // Emit initial response.created and response.in_progress events + let event = emitter.emit_created(); + emitter.send_event(&event, &tx)?; + let event = emitter.emit_in_progress(); + emitter.send_event(&event, &tx)?; + + // Get MCP tools and convert to chat format (do this once before loop) + let mcp_tools = ctx.mcp_manager.list_tools(); + let mcp_chat_tools = convert_mcp_tools_to_chat_tools(&mcp_tools); + debug!( + "Streaming: Converted {} MCP tools to chat format", + mcp_chat_tools.len() + ); + + // Flag to track if mcp_list_tools has been emitted + let mut mcp_list_tools_emitted = false; + + loop { + state.iteration += 1; + if state.iteration > MAX_ITERATIONS { + return Err(format!( + "Tool loop exceeded maximum iterations ({})", + MAX_ITERATIONS + )); + } + + debug!("Streaming MCP tool loop iteration {}", state.iteration); + + // Emit mcp_list_tools as first output item (only once, on first iteration) + if !mcp_list_tools_emitted { + let (output_index, item_id) = + emitter.allocate_output_index(OutputItemType::McpListTools); + + // Build tools list for item structure + let tool_items: Vec<_> = mcp_tools + .iter() + .map(|t| { + json!({ + "name": t.name, + "description": t.description, + "input_schema": Value::Object((*t.input_schema).clone()) + }) + }) + .collect(); + + // Build mcp_list_tools item + let item = json!({ + "id": item_id, + "type": "mcp_list_tools", + "server_label": state.server_label, + "status": "in_progress", + "tools": [] + }); + + // Emit output_item.added + let event = emitter.emit_output_item_added(output_index, &item); + emitter.send_event(&event, &tx)?; + + // Emit mcp_list_tools.in_progress + let event = emitter.emit_mcp_list_tools_in_progress(output_index); + emitter.send_event(&event, &tx)?; + + // Emit mcp_list_tools.completed + let event = emitter.emit_mcp_list_tools_completed(output_index, &mcp_tools); + emitter.send_event(&event, &tx)?; + + // Build complete item with tools + let item_done = json!({ + "id": item_id, + "type": "mcp_list_tools", + "server_label": state.server_label, + "status": "completed", + "tools": tool_items + }); + + // Emit output_item.done + let event = emitter.emit_output_item_done(output_index, &item_done); + emitter.send_event(&event, &tx)?; + + emitter.complete_output_item(output_index); + mcp_list_tools_emitted = true; + } + + // Convert to chat request + let mut chat_request = conversions::responses_to_chat(¤t_request) + .map_err(|e| format!("Failed to convert request: {}", e))?; + + // Prepare tools and tool_choice for this iteration (same logic as non-streaming) + prepare_chat_tools_and_choice(&mut chat_request, &mcp_chat_tools, state.iteration); + + // Execute chat streaming + let response = ctx + .pipeline + .execute_chat( + Arc::new(chat_request), + headers.clone(), + model_id.clone(), + ctx.components.clone(), + ) + .await; + + // Convert chat stream to Responses API events while accumulating for tool call detection + // Stream text naturally - it only appears on final iteration (tool iterations have empty content) + let accumulated_response = + convert_and_accumulate_stream(response.into_body(), &mut emitter, &tx).await?; + + // Check for tool calls (extract all of them for parallel execution) + let tool_calls = extract_all_tool_calls_from_chat(&accumulated_response); + + if !tool_calls.is_empty() { + debug!( + "Tool loop iteration {}: found {} tool call(s)", + state.iteration, + tool_calls.len() + ); + + // Separate MCP and function tool calls + let mcp_tool_names: std::collections::HashSet<&str> = + mcp_tools.iter().map(|t| t.name.as_ref()).collect(); + let (mcp_tool_calls, function_tool_calls): (Vec<_>, Vec<_>) = tool_calls + .into_iter() + .partition(|(_, tool_name, _)| mcp_tool_names.contains(tool_name.as_str())); + + debug!( + "Separated tool calls: {} MCP, {} function", + mcp_tool_calls.len(), + function_tool_calls.len() + ); + + // Check combined limit (only count MCP tools since function tools will be returned) + let effective_limit = match max_tool_calls { + Some(user_max) => user_max.min(MAX_ITERATIONS), + None => MAX_ITERATIONS, + }; + + if state.total_calls + mcp_tool_calls.len() > effective_limit { + warn!( + "Reached tool call limit: {} + {} > {} (max_tool_calls={:?}, safety_limit={})", + state.total_calls, + mcp_tool_calls.len(), + effective_limit, + max_tool_calls, + MAX_ITERATIONS + ); + break; + } + + // Process each MCP tool call + for (call_id, tool_name, args_json_str) in mcp_tool_calls { + state.total_calls += 1; + + debug!( + "Executing tool call {}/{}: {} (call_id: {})", + state.total_calls, state.total_calls, tool_name, call_id + ); + + // Allocate output_index for this mcp_call item + let (output_index, item_id) = + emitter.allocate_output_index(OutputItemType::McpCall); + + // Build initial mcp_call item + let item = json!({ + "id": item_id, + "type": "mcp_call", + "name": tool_name, + "server_label": state.server_label, + "status": "in_progress", + "arguments": "" + }); + + // Emit output_item.added + let event = emitter.emit_output_item_added(output_index, &item); + emitter.send_event(&event, &tx)?; + + // Emit mcp_call.in_progress + let event = emitter.emit_mcp_call_in_progress(output_index, &item_id); + emitter.send_event(&event, &tx)?; + + // Emit mcp_call_arguments.delta (simulate streaming by sending full arguments) + let event = + emitter.emit_mcp_call_arguments_delta(output_index, &item_id, &args_json_str); + emitter.send_event(&event, &tx)?; + + // Emit mcp_call_arguments.done + let event = + emitter.emit_mcp_call_arguments_done(output_index, &item_id, &args_json_str); + emitter.send_event(&event, &tx)?; + + // Execute the MCP tool - manager handles parsing and type coercion + debug!( + "Calling MCP tool '{}' with args: {}", + tool_name, args_json_str + ); + let (output_str, success, error) = match ctx + .mcp_manager + .call_tool(tool_name.as_str(), args_json_str.as_str()) + .await + { + Ok(result) => match serde_json::to_string(&result) { + Ok(output) => { + // Emit mcp_call.completed + let event = emitter.emit_mcp_call_completed(output_index, &item_id); + emitter.send_event(&event, &tx)?; + + // Build complete item with output + let item_done = json!({ + "id": item_id, + "type": "mcp_call", + "name": tool_name, + "server_label": state.server_label, + "status": "completed", + "arguments": args_json_str, + "output": output + }); + + // Emit output_item.done + let event = emitter.emit_output_item_done(output_index, &item_done); + emitter.send_event(&event, &tx)?; + + emitter.complete_output_item(output_index); + (output, true, None) + } + Err(e) => { + let err = format!("Failed to serialize tool result: {}", e); + warn!("{}", err); + // Emit mcp_call.failed + let event = emitter.emit_mcp_call_failed(output_index, &item_id, &err); + emitter.send_event(&event, &tx)?; + + // Build failed item + let item_done = json!({ + "id": item_id, + "type": "mcp_call", + "name": tool_name, + "server_label": state.server_label, + "status": "failed", + "arguments": args_json_str, + "error": &err + }); + + // Emit output_item.done + let event = emitter.emit_output_item_done(output_index, &item_done); + emitter.send_event(&event, &tx)?; + + emitter.complete_output_item(output_index); + let error_json = json!({ "error": &err }).to_string(); + (error_json, false, Some(err)) + } + }, + Err(err) => { + let err_str = format!("tool call failed: {}", err); + warn!("Tool execution failed: {}", err_str); + // Emit mcp_call.failed + let event = emitter.emit_mcp_call_failed(output_index, &item_id, &err_str); + emitter.send_event(&event, &tx)?; + + // Build failed item + let item_done = json!({ + "id": item_id, + "type": "mcp_call", + "name": tool_name, + "server_label": state.server_label, + "status": "failed", + "arguments": args_json_str, + "error": &err_str + }); + + // Emit output_item.done + let event = emitter.emit_output_item_done(output_index, &item_done); + emitter.send_event(&event, &tx)?; + + emitter.complete_output_item(output_index); + let error_json = json!({ "error": &err_str }).to_string(); + (error_json, false, Some(err_str)) + } + }; + + // Record the call in state + state.record_call( + call_id, + tool_name, + args_json_str, + output_str, + success, + error, + ); + } + + // If there are function tool calls, emit events and exit MCP loop + if !function_tool_calls.is_empty() { + debug!( + "Found {} function tool call(s) - emitting events and exiting MCP loop", + function_tool_calls.len() + ); + + // Emit function_tool_call events for each function tool + for (call_id, tool_name, args_json_str) in function_tool_calls { + // Allocate output_index for this function_tool_call item + let (output_index, item_id) = + emitter.allocate_output_index(OutputItemType::FunctionCall); + + // Build initial function_tool_call item + let item = json!({ + "id": item_id, + "type": "function_tool_call", + "call_id": call_id, + "name": tool_name, + "status": "in_progress", + "arguments": "" + }); + + // Emit output_item.added + let event = emitter.emit_output_item_added(output_index, &item); + emitter.send_event(&event, &tx)?; + + // Emit function_call_arguments.delta + let event = emitter.emit_function_call_arguments_delta( + output_index, + &item_id, + &args_json_str, + ); + emitter.send_event(&event, &tx)?; + + // Emit function_call_arguments.done + let event = emitter.emit_function_call_arguments_done( + output_index, + &item_id, + &args_json_str, + ); + emitter.send_event(&event, &tx)?; + + // Build complete item + let item_complete = json!({ + "id": item_id, + "type": "function_tool_call", + "call_id": call_id, + "name": tool_name, + "status": "completed", + "arguments": args_json_str + }); + + // Emit output_item.done + let event = emitter.emit_output_item_done(output_index, &item_complete); + emitter.send_event(&event, &tx)?; + + emitter.complete_output_item(output_index); + } + + // Break loop to return response to caller + break; + } + + // Build next request with conversation history + let mut input_items = match &state.original_input { + ResponseInput::Text(text) => vec![ResponseInputOutputItem::Message { + id: format!("msg_u_{}", state.iteration), + role: "user".to_string(), + content: vec![ResponseContentPart::InputText { text: text.clone() }], + status: Some("completed".to_string()), + }], + ResponseInput::Items(items) => { + items.iter().map(responses::normalize_input_item).collect() + } + }; + + input_items.extend_from_slice(&state.conversation_history); + + current_request = ResponsesRequest { + input: ResponseInput::Items(input_items), + model: current_request.model.clone(), + instructions: current_request.instructions.clone(), + tools: current_request.tools.clone(), + max_output_tokens: current_request.max_output_tokens, + temperature: current_request.temperature, + top_p: current_request.top_p, + stream: Some(true), + store: Some(false), + background: Some(false), + max_tool_calls: current_request.max_tool_calls, + tool_choice: current_request.tool_choice.clone(), + parallel_tool_calls: current_request.parallel_tool_calls, + previous_response_id: None, + conversation: None, + user: current_request.user.clone(), + metadata: current_request.metadata.clone(), + include: current_request.include.clone(), + reasoning: current_request.reasoning.clone(), + service_tier: current_request.service_tier.clone(), + top_logprobs: current_request.top_logprobs, + truncation: current_request.truncation.clone(), + text: current_request.text.clone(), + request_id: None, + priority: current_request.priority, + frequency_penalty: current_request.frequency_penalty, + presence_penalty: current_request.presence_penalty, + stop: current_request.stop.clone(), + top_k: current_request.top_k, + min_p: current_request.min_p, + repetition_penalty: current_request.repetition_penalty, + }; + + continue; + } + + // No tool calls, this is the final response + debug!("No tool calls found, ending streaming MCP loop"); + + // Check for reasoning content + let reasoning_content = accumulated_response + .choices + .first() + .and_then(|c| c.message.reasoning_content.clone()); + + // Emit reasoning item if present + if let Some(reasoning) = reasoning_content { + if !reasoning.is_empty() { + emitter.emit_reasoning_item(&tx, Some(reasoning))?; + } + } + + // Text message events already emitted naturally by process_chunk during stream processing + // (OpenAI router approach - text only appears on final iteration when no tool calls) + + // Emit final response.completed event + let usage_json = accumulated_response.usage.as_ref().map(|u| { + json!({ + "input_tokens": u.prompt_tokens, + "output_tokens": u.completion_tokens, + "total_tokens": u.total_tokens + }) + }); + let event = emitter.emit_completed(usage_json.as_ref()); + emitter.send_event(&event, &tx)?; + + break; + } + + Ok(()) +} + +/// Convert MCP tools to Chat API tool format +fn convert_mcp_tools_to_chat_tools(mcp_tools: &[mcp::Tool]) -> Vec { + mcp_tools + .iter() + .map(|tool_info| Tool { + tool_type: "function".to_string(), + function: Function { + name: tool_info.name.to_string(), + description: tool_info.description.as_ref().map(|d| d.to_string()), + parameters: Value::Object((*tool_info.input_schema).clone()), + strict: None, + }, + }) + .collect() +} + +/// Convert chat stream to Responses API events while accumulating for tool call detection +async fn convert_and_accumulate_stream( + body: Body, + emitter: &mut ResponseStreamEventEmitter, + tx: &mpsc::UnboundedSender>, +) -> Result { + let mut accumulator = ChatResponseAccumulator::new(); + let mut stream = body.into_data_stream(); + + while let Some(chunk_result) = stream.next().await { + let chunk = chunk_result.map_err(|e| format!("Stream read error: {}", e))?; + + // Parse chunk + let event_str = String::from_utf8_lossy(&chunk); + let event = event_str.trim(); + + if event == "data: [DONE]" { + break; + } + + if let Some(json_str) = event.strip_prefix("data: ") { + let json_str = json_str.trim(); + if let Ok(chat_chunk) = serde_json::from_str::(json_str) { + // Convert chat chunk to Responses API events and emit + emitter.process_chunk(&chat_chunk, tx)?; + + // Accumulate for tool call detection + accumulator.process_chunk(&chat_chunk); + } + } + } + + Ok(accumulator.finalize()) +} + +/// Accumulates chat streaming chunks into complete ChatCompletionResponse +struct ChatResponseAccumulator { + id: String, + model: String, + content: String, + tool_calls: HashMap, + finish_reason: Option, +} + +impl ChatResponseAccumulator { + fn new() -> Self { + Self { + id: String::new(), + model: String::new(), + content: String::new(), + tool_calls: HashMap::new(), + finish_reason: None, + } + } + + fn process_chunk(&mut self, chunk: &ChatCompletionStreamResponse) { + if !chunk.id.is_empty() { + self.id = chunk.id.clone(); + } + if !chunk.model.is_empty() { + self.model = chunk.model.clone(); + } + + if let Some(choice) = chunk.choices.first() { + // Accumulate content + if let Some(content) = &choice.delta.content { + self.content.push_str(content); + } + + // Accumulate tool calls + if let Some(tool_call_deltas) = &choice.delta.tool_calls { + for delta in tool_call_deltas { + let index = delta.index as usize; + let entry = self.tool_calls.entry(index).or_insert_with(|| ToolCall { + id: String::new(), + tool_type: "function".to_string(), + function: FunctionCallResponse { + name: String::new(), + arguments: Some(String::new()), + }, + }); + + if let Some(id) = &delta.id { + entry.id = id.clone(); + } + if let Some(function) = &delta.function { + if let Some(name) = &function.name { + entry.function.name = name.clone(); + } + if let Some(args) = &function.arguments { + if let Some(ref mut existing_args) = entry.function.arguments { + existing_args.push_str(args); + } + } + } + } + } + + // Capture finish reason + if let Some(reason) = &choice.finish_reason { + self.finish_reason = Some(reason.clone()); + } + } + } + + fn finalize(self) -> ChatCompletionResponse { + let mut tool_calls_vec: Vec<_> = self.tool_calls.into_iter().collect(); + tool_calls_vec.sort_by_key(|(index, _)| *index); + let tool_calls: Vec<_> = tool_calls_vec.into_iter().map(|(_, call)| call).collect(); + + ChatCompletionResponse::builder(&self.id, &self.model) + .choices(vec![ChatChoice { + index: 0, + message: ChatCompletionMessage { + role: "assistant".to_string(), + content: if self.content.is_empty() { + None + } else { + Some(self.content) + }, + tool_calls: if tool_calls.is_empty() { + None + } else { + Some(tool_calls) + }, + reasoning_content: None, + }, + finish_reason: self.finish_reason, + logprobs: None, + matched_stop: None, + hidden_states: None, + }]) + .build() + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/types.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/types.rs new file mode 100644 index 00000000..fb6921b2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/responses/types.rs @@ -0,0 +1,18 @@ +//! Type definitions for /v1/responses endpoint + +use std::sync::Arc; + +use tokio::{sync::RwLock, task::JoinHandle}; + +/// Information stored for background tasks to enable end-to-end cancellation +/// +/// This struct enables cancelling both the Rust task AND the Python scheduler processing. +/// The client field is lazily initialized during pipeline execution. +pub struct BackgroundTaskInfo { + /// Tokio task handle for aborting the Rust task + pub handle: JoinHandle<()>, + /// gRPC request_id sent to Python scheduler (chatcmpl-* prefix) + pub grpc_request_id: String, + /// gRPC client for sending abort requests to Python (set after client acquisition) + pub client: Arc>>, +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/chat/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/chat/mod.rs new file mode 100644 index 00000000..606849c7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/chat/mod.rs @@ -0,0 +1,12 @@ +//! Chat endpoint pipeline stages +//! +//! These stages handle chat-specific preprocessing, request building, and response processing. +//! They work with any model type by using injected model adapters. + +mod preparation; +mod request_building; +mod response_processing; + +pub use preparation::ChatPreparationStage; +pub use request_building::ChatRequestBuildingStage; +pub use response_processing::ChatResponseProcessingStage; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/chat/preparation.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/chat/preparation.rs new file mode 100644 index 00000000..09be7f1c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/chat/preparation.rs @@ -0,0 +1,112 @@ +//! Chat preparation stage: Filter tools, process messages, tokenize, build constraints + +use std::borrow::Cow; + +use async_trait::async_trait; +use axum::response::Response; +use tracing::error; + +use crate::{ + protocols::chat::ChatCompletionRequest, + routers::grpc::{ + common::stages::PipelineStage, + context::{PreparationOutput, RequestContext}, + error, utils, + }, +}; + +/// Chat preparation stage +/// +/// Extracts chat-specific preparation logic from the old unified PreparationStage. +/// This is a direct extraction without architectural changes. +pub struct ChatPreparationStage; + +#[async_trait] +impl PipelineStage for ChatPreparationStage { + async fn execute(&self, ctx: &mut RequestContext) -> Result, Response> { + let request = ctx.chat_request_arc(); + self.prepare_chat(ctx, &request).await?; + Ok(None) + } + + fn name(&self) -> &'static str { + "ChatPreparation" + } +} + +impl ChatPreparationStage { + async fn prepare_chat( + &self, + ctx: &mut RequestContext, + request: &ChatCompletionRequest, + ) -> Result<(), Response> { + // Step 1: Filter tools if needed + let body_ref = utils::filter_chat_request_by_tool_choice(request); + + // Step 2: Process messages and apply chat template + let processed_messages = match utils::process_chat_messages( + &body_ref, + &*ctx.components.tokenizer, + ) { + Ok(msgs) => msgs, + Err(e) => { + error!(function = "ChatPreparationStage::execute", error = %e, "Failed to process chat messages"); + return Err(error::bad_request(e)); + } + }; + + // Step 3: Tokenize the processed text + let encoding = match ctx.components.tokenizer.encode(&processed_messages.text) { + Ok(encoding) => encoding, + Err(e) => { + error!(function = "ChatPreparationStage::execute", error = %e, "Tokenization failed"); + return Err(error::internal_error(format!("Tokenization failed: {}", e))); + } + }; + + let token_ids = encoding.token_ids().to_vec(); + + // Step 4: Build tool constraints if needed + let tool_call_constraint = if let Some(tools) = body_ref.tools.as_ref() { + utils::generate_tool_constraints(tools, &request.tool_choice, &request.model) + .map_err(|e| { + error!(function = "ChatPreparationStage::execute", error = %e, "Invalid tool configuration"); + error::bad_request(format!("Invalid tool configuration: {}", e)) + })? + } else { + None + }; + + // Step 5: Create stop sequence decoder (build once, reuse in non-stream) + let stop_decoder = utils::create_stop_decoder( + &ctx.components.tokenizer, + request.stop.as_ref(), + request.stop_token_ids.as_ref(), + request.skip_special_tokens, + request.no_stop_trim, + ); + + // Store results in context + ctx.state.preparation = Some(PreparationOutput { + original_text: Some(processed_messages.text.clone()), + token_ids, + processed_messages: Some(processed_messages), + tool_constraints: tool_call_constraint, + filtered_request: if matches!(body_ref, Cow::Owned(_)) { + Some(body_ref.into_owned()) + } else { + None + }, + // Harmony fields (not used for regular preparation) + harmony_mode: false, + selection_text: None, + harmony_messages: None, + harmony_stop_ids: None, + }); + + // Store stop decoder for reuse in response processing + ctx.state.response.stop_decoder = Some(stop_decoder); + + Ok(()) + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/chat/request_building.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/chat/request_building.rs new file mode 100644 index 00000000..40cbd96d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/chat/request_building.rs @@ -0,0 +1,113 @@ +//! Chat request building stage: Build proto GenerateRequest for chat requests + +use async_trait::async_trait; +use axum::response::Response; +use tracing::error; +use uuid::Uuid; + +use crate::routers::grpc::{ + client::GrpcClient, + common::stages::{helpers, PipelineStage}, + context::{ClientSelection, RequestContext, WorkerSelection}, + error, + proto_wrapper::ProtoGenerateRequest, +}; + +/// Chat request building stage +/// +/// Extracts chat-specific request building logic from the old unified RequestBuildingStage. +pub struct ChatRequestBuildingStage { + inject_pd_metadata: bool, +} + +impl ChatRequestBuildingStage { + pub fn new(inject_pd_metadata: bool) -> Self { + Self { inject_pd_metadata } + } +} + +#[async_trait] +impl PipelineStage for ChatRequestBuildingStage { + async fn execute(&self, ctx: &mut RequestContext) -> Result, Response> { + let prep = ctx.state.preparation.as_ref().ok_or_else(|| { + error!( + function = "ChatRequestBuildingStage::execute", + "Preparation not completed" + ); + error::internal_error("Preparation not completed") + })?; + + let clients = ctx.state.clients.as_ref().ok_or_else(|| { + error!( + function = "ChatRequestBuildingStage::execute", + "Client acquisition not completed" + ); + error::internal_error("Client acquisition not completed") + })?; + + let chat_request = ctx.chat_request_arc(); + + // Get client for building request (use prefill client if PD mode) + let builder_client = match clients { + ClientSelection::Single { client } => client, + ClientSelection::Dual { prefill, .. } => prefill, + }; + + // Build chat request + let request_id = format!("chatcmpl-{}", Uuid::new_v4()); + let body_ref = prep.filtered_request.as_ref().unwrap_or(&chat_request); + + // Dispatch to the appropriate client based on backend type + let mut proto_request = match builder_client { + GrpcClient::Sglang(sglang_client) => { + let req = sglang_client + .build_generate_request_from_chat( + request_id, + body_ref, + prep.processed_messages.as_ref().unwrap().text.clone(), + prep.token_ids.clone(), + prep.processed_messages + .as_ref() + .unwrap() + .multimodal_inputs + .clone(), + prep.tool_constraints.clone(), + ) + .map_err(|e| { + error!(function = "ChatRequestBuildingStage::execute", error = %e, "Failed to build SGLang generate request"); + error::bad_request(format!("Invalid request parameters: {}", e)) + })?; + ProtoGenerateRequest::Sglang(Box::new(req)) + } + GrpcClient::Vllm(vllm_client) => { + let req = vllm_client + .build_generate_request_from_chat( + request_id, + body_ref, + prep.processed_messages.as_ref().unwrap().text.clone(), + prep.token_ids.clone(), + prep.tool_constraints.clone(), + ) + .map_err(|e| { + error!(function = "ChatRequestBuildingStage::execute", error = %e, "Failed to build vLLM generate request"); + error::bad_request(format!("Invalid request parameters: {}", e)) + })?; + ProtoGenerateRequest::Vllm(Box::new(req)) + } + }; + + // Inject PD metadata if needed + if self.inject_pd_metadata { + if let WorkerSelection::Dual { prefill, .. } = ctx.state.workers.as_ref().unwrap() { + helpers::inject_bootstrap_metadata(&mut proto_request, prefill); + } + } + + ctx.state.proto_request = Some(proto_request); + Ok(None) + } + + fn name(&self) -> &'static str { + "ChatRequestBuilding" + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/chat/response_processing.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/chat/response_processing.rs new file mode 100644 index 00000000..410ee817 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/chat/response_processing.rs @@ -0,0 +1,120 @@ +//! Chat response processing stage: Handles both streaming and non-streaming responses +//! +//! - For streaming: Spawns background task and returns SSE response (early exit) +//! - For non-streaming: Collects all responses and builds final ChatCompletionResponse + +use std::sync::Arc; + +use async_trait::async_trait; +use axum::response::Response; +use tracing::error; + +use crate::routers::grpc::{ + common::stages::PipelineStage, + context::{FinalResponse, RequestContext}, + error, + regular::{processor, streaming}, +}; + +/// Chat response processing stage +/// +/// Extracts chat-specific response processing logic from the old unified ResponseProcessingStage. +pub struct ChatResponseProcessingStage { + processor: processor::ResponseProcessor, + streaming_processor: Arc, +} + +impl ChatResponseProcessingStage { + pub fn new( + processor: processor::ResponseProcessor, + streaming_processor: Arc, + ) -> Self { + Self { + processor, + streaming_processor, + } + } +} + +#[async_trait] +impl PipelineStage for ChatResponseProcessingStage { + async fn execute(&self, ctx: &mut RequestContext) -> Result, Response> { + self.process_chat_response(ctx).await + } + + fn name(&self) -> &'static str { + "ChatResponseProcessing" + } +} + +impl ChatResponseProcessingStage { + async fn process_chat_response( + &self, + ctx: &mut RequestContext, + ) -> Result, Response> { + let is_streaming = ctx.is_streaming(); + + // Extract execution result + let execution_result = ctx.state.response.execution_result.take().ok_or_else(|| { + error!( + function = "ChatResponseProcessingStage::execute", + "No execution result" + ); + error::internal_error("No execution result") + })?; + + // Get dispatch metadata (needed by both streaming and non-streaming) + let dispatch = ctx + .state + .dispatch + .as_ref() + .ok_or_else(|| { + error!( + function = "ChatResponseProcessingStage::execute", + "Dispatch metadata not set" + ); + error::internal_error("Dispatch metadata not set") + })? + .clone(); + + if is_streaming { + // Streaming: Use StreamingProcessor and return SSE response (done) + return Ok(Some( + self.streaming_processor.clone().process_streaming_response( + execution_result, + ctx.chat_request_arc(), // Cheap Arc clone (8 bytes) + dispatch, + ), + )); + } + + // Non-streaming: Delegate to ResponseProcessor + let request_logprobs = ctx.chat_request().logprobs; + + let chat_request = ctx.chat_request_arc(); + + let stop_decoder = ctx.state.response.stop_decoder.as_mut().ok_or_else(|| { + error!( + function = "ChatResponseProcessingStage::execute", + "Stop decoder not initialized" + ); + error::internal_error("Stop decoder not initialized") + })?; + + let response = self + .processor + .process_non_streaming_chat_response( + execution_result, + chat_request, + dispatch, + stop_decoder, + request_logprobs, + ) + .await?; + + // Store the final response + ctx.state.response.final_response = Some(FinalResponse::Chat(response)); + + Ok(None) + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/generate/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/generate/mod.rs new file mode 100644 index 00000000..0d3d33e9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/generate/mod.rs @@ -0,0 +1,12 @@ +//! Generate endpoint pipeline stages +//! +//! These stages handle generate-specific preprocessing, request building, and response processing. +//! They work with any model type by using injected model adapters. + +mod preparation; +mod request_building; +mod response_processing; + +pub use preparation::GeneratePreparationStage; +pub use request_building::GenerateRequestBuildingStage; +pub use response_processing::GenerateResponseProcessingStage; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/generate/preparation.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/generate/preparation.rs new file mode 100644 index 00000000..ebd4fdf1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/generate/preparation.rs @@ -0,0 +1,121 @@ +//! Generate preparation stage: Resolve input, tokenize, create stop decoder + +use std::sync::Arc; + +use async_trait::async_trait; +use axum::response::Response; +use tracing::error; + +use crate::{ + protocols::{common::InputIds, generate::GenerateRequest}, + routers::grpc::{ + common::stages::PipelineStage, + context::{PreparationOutput, RequestContext}, + error, utils, + }, + tokenizer::traits::Tokenizer, +}; + +/// Generate preparation stage +/// +/// Extracts generate-specific preparation logic from the old unified PreparationStage. +/// This is a direct extraction without architectural changes. +pub struct GeneratePreparationStage; + +#[async_trait] +impl PipelineStage for GeneratePreparationStage { + async fn execute(&self, ctx: &mut RequestContext) -> Result, Response> { + let request = ctx.generate_request_arc(); + self.prepare_generate(ctx, &request).await?; + Ok(None) + } + + fn name(&self) -> &'static str { + "GeneratePreparation" + } +} + +impl GeneratePreparationStage { + async fn prepare_generate( + &self, + ctx: &mut RequestContext, + request: &GenerateRequest, + ) -> Result<(), Response> { + // Resolve input (text, prompt, or input_ids) + let (original_text, token_ids) = match self.resolve_generate_input(ctx, request) { + Ok(res) => res, + Err(msg) => { + error!(function = "GeneratePreparationStage::execute", error = %msg, "Failed to resolve generate input"); + return Err(error::bad_request(msg)); + } + }; + + // Create stop sequence decoder for generate requests + let params = request.sampling_params.as_ref(); + let stop_decoder = utils::create_stop_decoder( + &ctx.components.tokenizer, + params.and_then(|p| p.stop.as_ref()), + params.and_then(|p| p.stop_token_ids.as_ref()), + params.and_then(|p| p.skip_special_tokens).unwrap_or(true), + params.and_then(|p| p.no_stop_trim).unwrap_or(false), + ); + + ctx.state.preparation = Some(PreparationOutput { + original_text, + token_ids, + processed_messages: None, + tool_constraints: None, + filtered_request: None, + // Harmony fields (not used for generate requests) + harmony_mode: false, + selection_text: None, + harmony_messages: None, + harmony_stop_ids: None, + }); + + // Store stop decoder + ctx.state.response.stop_decoder = Some(stop_decoder); + + Ok(()) + } + + fn resolve_generate_input( + &self, + ctx: &RequestContext, + request: &GenerateRequest, + ) -> Result<(Option, Vec), String> { + if let Some(text) = &request.text { + return self + .tokenize_single_text(&ctx.components.tokenizer, text) + .map(|(original, ids)| (Some(original), ids)); + } + + // Handle input_ids - validate and convert + if let Some(input_ids) = &request.input_ids { + return match input_ids { + InputIds::Single(ids) => ids + .iter() + .map(|&id| u32::try_from(id)) + .collect::, _>>() + .map(|converted| (None, converted)) + .map_err(|_| "input_ids must be non-negative".to_string()), + InputIds::Batch(_) => { + Err("Batch input_ids are not supported over gRPC generate yet".to_string()) + } + }; + } + + Err("Either `text` or `input_ids` must be provided".to_string()) + } + + fn tokenize_single_text( + &self, + tokenizer: &Arc, + text: &str, + ) -> Result<(String, Vec), String> { + let encoding = tokenizer + .encode(text) + .map_err(|e| format!("Tokenization failed: {}", e))?; + Ok((text.to_string(), encoding.token_ids().to_vec())) + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/generate/request_building.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/generate/request_building.rs new file mode 100644 index 00000000..3ffb212f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/generate/request_building.rs @@ -0,0 +1,108 @@ +//! Generate request building stage: Build proto GenerateRequest for generate requests + +use async_trait::async_trait; +use axum::response::Response; +use tracing::error; +use uuid::Uuid; + +use crate::routers::grpc::{ + client::GrpcClient, + common::stages::{helpers, PipelineStage}, + context::{ClientSelection, RequestContext, WorkerSelection}, + error, + proto_wrapper::ProtoGenerateRequest, +}; + +/// Generate request building stage +/// +/// Extracts generate-specific request building logic from the old unified RequestBuildingStage. +pub struct GenerateRequestBuildingStage { + inject_pd_metadata: bool, +} + +impl GenerateRequestBuildingStage { + pub fn new(inject_pd_metadata: bool) -> Self { + Self { inject_pd_metadata } + } +} + +#[async_trait] +impl PipelineStage for GenerateRequestBuildingStage { + async fn execute(&self, ctx: &mut RequestContext) -> Result, Response> { + let prep = ctx.state.preparation.as_ref().ok_or_else(|| { + error!( + function = "GenerateRequestBuildingStage::execute", + "Preparation not completed" + ); + error::internal_error("Preparation not completed") + })?; + + let clients = ctx.state.clients.as_ref().ok_or_else(|| { + error!( + function = "GenerateRequestBuildingStage::execute", + "Client acquisition not completed" + ); + error::internal_error("Client acquisition not completed") + })?; + + let generate_request = ctx.generate_request_arc(); + + // Get client for building request (use prefill client if PD mode) + let builder_client = match clients { + ClientSelection::Single { client } => client, + ClientSelection::Dual { prefill, .. } => prefill, + }; + + // Build generate request + let request_id = generate_request + .rid + .clone() + .unwrap_or_else(|| format!("gen-{}", Uuid::new_v4())); + + // Dispatch to the appropriate client based on backend type + let mut proto_request = match builder_client { + GrpcClient::Sglang(sglang_client) => { + let req = sglang_client + .build_plain_generate_request( + request_id, + &generate_request, + prep.original_text.clone(), + prep.token_ids.clone(), + ) + .map_err(|e| { + error!(function = "GenerateRequestBuildingStage::execute", error = %e, "Failed to build SGLang generate request"); + error::bad_request(e) + })?; + ProtoGenerateRequest::Sglang(Box::new(req)) + } + GrpcClient::Vllm(vllm_client) => { + let req = vllm_client + .build_plain_generate_request( + request_id, + &generate_request, + prep.original_text.clone(), + prep.token_ids.clone(), + ) + .map_err(|e| { + error!(function = "GenerateRequestBuildingStage::execute", error = %e, "Failed to build vLLM generate request"); + error::bad_request(e) + })?; + ProtoGenerateRequest::Vllm(Box::new(req)) + } + }; + + // Inject PD metadata if needed + if self.inject_pd_metadata { + if let WorkerSelection::Dual { prefill, .. } = ctx.state.workers.as_ref().unwrap() { + helpers::inject_bootstrap_metadata(&mut proto_request, prefill); + } + } + + ctx.state.proto_request = Some(proto_request); + Ok(None) + } + + fn name(&self) -> &'static str { + "GenerateRequestBuilding" + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/generate/response_processing.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/generate/response_processing.rs new file mode 100644 index 00000000..3f325ac0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/generate/response_processing.rs @@ -0,0 +1,118 @@ +//! Generate response processing stage: Handles both streaming and non-streaming responses + +use std::{sync::Arc, time::Instant}; + +use async_trait::async_trait; +use axum::response::Response; +use tracing::error; + +use crate::routers::grpc::{ + common::stages::PipelineStage, + context::{FinalResponse, RequestContext}, + error, + regular::{processor, streaming}, +}; + +/// Generate response processing stage +/// +/// Extracts generate-specific response processing logic from the old unified ResponseProcessingStage. +pub struct GenerateResponseProcessingStage { + processor: processor::ResponseProcessor, + streaming_processor: Arc, +} + +impl GenerateResponseProcessingStage { + pub fn new( + processor: processor::ResponseProcessor, + streaming_processor: Arc, + ) -> Self { + Self { + processor, + streaming_processor, + } + } +} + +#[async_trait] +impl PipelineStage for GenerateResponseProcessingStage { + async fn execute(&self, ctx: &mut RequestContext) -> Result, Response> { + self.process_generate_response(ctx).await + } + + fn name(&self) -> &'static str { + "GenerateResponseProcessing" + } +} + +impl GenerateResponseProcessingStage { + async fn process_generate_response( + &self, + ctx: &mut RequestContext, + ) -> Result, Response> { + let start_time = Instant::now(); + let is_streaming = ctx.is_streaming(); + + // Extract execution result + let execution_result = ctx.state.response.execution_result.take().ok_or_else(|| { + error!( + function = "GenerateResponseProcessingStage::execute", + "No execution result" + ); + error::internal_error("No execution result") + })?; + + // Get dispatch metadata (needed by both streaming and non-streaming) + let dispatch = ctx + .state + .dispatch + .as_ref() + .ok_or_else(|| { + error!( + function = "GenerateResponseProcessingStage::execute", + "Dispatch metadata not set" + ); + error::internal_error("Dispatch metadata not set") + })? + .clone(); + + if is_streaming { + // Streaming: Use StreamingProcessor and return SSE response (done) + return Ok(Some( + self.streaming_processor.clone().process_streaming_generate( + execution_result, + ctx.generate_request_arc(), // Cheap Arc clone (8 bytes) + dispatch, + ), + )); + } + + // Non-streaming: Delegate to ResponseProcessor + let request_logprobs = ctx.generate_request().return_logprob.unwrap_or(false); + let generate_request = ctx.generate_request_arc(); + + let stop_decoder = ctx.state.response.stop_decoder.as_mut().ok_or_else(|| { + error!( + function = "GenerateResponseProcessingStage::execute", + "Stop decoder not initialized" + ); + error::internal_error("Stop decoder not initialized") + })?; + + let result_array = self + .processor + .process_non_streaming_generate_response( + execution_result, + generate_request, + dispatch, + stop_decoder, + request_logprobs, + start_time, + ) + .await?; + + // Store the final response + ctx.state.response.final_response = Some(FinalResponse::Generate(result_array)); + + Ok(None) + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/mod.rs new file mode 100644 index 00000000..a448a82e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/mod.rs @@ -0,0 +1,17 @@ +//! Pipeline stages for regular (non-harmony) model processing +//! +//! This module defines stages specific to regular tokenizer-based models. + +pub mod chat; +pub mod generate; +mod preparation; +mod request_building; +mod response_processing; + +pub use chat::{ChatPreparationStage, ChatRequestBuildingStage, ChatResponseProcessingStage}; +pub use generate::{ + GeneratePreparationStage, GenerateRequestBuildingStage, GenerateResponseProcessingStage, +}; +pub use preparation::PreparationStage; +pub use request_building::RequestBuildingStage; +pub use response_processing::ResponseProcessingStage; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/preparation.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/preparation.rs new file mode 100644 index 00000000..b0050867 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/preparation.rs @@ -0,0 +1,59 @@ +//! Preparation stage that delegates to endpoint-specific implementations +//! +//! This stage checks RequestType at runtime and delegates to the appropriate +//! endpoint-specific stage (ChatPreparationStage or GeneratePreparationStage). + +use async_trait::async_trait; +use axum::response::Response; +use tracing::error; + +use super::{chat::ChatPreparationStage, generate::GeneratePreparationStage}; +use crate::routers::grpc::{ + common::stages::PipelineStage, + context::{RequestContext, RequestType}, + error as grpc_error, +}; + +/// Preparation stage (delegates to endpoint-specific implementations) +pub struct PreparationStage { + chat_stage: ChatPreparationStage, + generate_stage: GeneratePreparationStage, +} + +impl PreparationStage { + pub fn new() -> Self { + Self { + chat_stage: ChatPreparationStage, + generate_stage: GeneratePreparationStage, + } + } +} + +impl Default for PreparationStage { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl PipelineStage for PreparationStage { + async fn execute(&self, ctx: &mut RequestContext) -> Result, Response> { + match &ctx.input.request_type { + RequestType::Chat(_) => self.chat_stage.execute(ctx).await, + RequestType::Generate(_) => self.generate_stage.execute(ctx).await, + RequestType::Responses(_) => { + error!( + function = "PreparationStage::execute", + "RequestType::Responses reached regular preparation stage" + ); + Err(grpc_error::internal_error( + "RequestType::Responses reached regular preparation stage", + )) + } + } + } + + fn name(&self) -> &'static str { + "Preparation" + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/request_building.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/request_building.rs new file mode 100644 index 00000000..4c763073 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/request_building.rs @@ -0,0 +1,50 @@ +//! Request building stage that delegates to endpoint-specific implementations + +use async_trait::async_trait; +use axum::response::Response; +use tracing::error; + +use super::{chat::ChatRequestBuildingStage, generate::GenerateRequestBuildingStage}; +use crate::routers::grpc::{ + common::stages::PipelineStage, + context::{RequestContext, RequestType}, + error as grpc_error, +}; + +/// Request building stage (delegates to endpoint-specific implementations) +pub struct RequestBuildingStage { + chat_stage: ChatRequestBuildingStage, + generate_stage: GenerateRequestBuildingStage, +} + +impl RequestBuildingStage { + pub fn new(inject_pd_metadata: bool) -> Self { + Self { + chat_stage: ChatRequestBuildingStage::new(inject_pd_metadata), + generate_stage: GenerateRequestBuildingStage::new(inject_pd_metadata), + } + } +} + +#[async_trait] +impl PipelineStage for RequestBuildingStage { + async fn execute(&self, ctx: &mut RequestContext) -> Result, Response> { + match &ctx.input.request_type { + RequestType::Chat(_) => self.chat_stage.execute(ctx).await, + RequestType::Generate(_) => self.generate_stage.execute(ctx).await, + RequestType::Responses(_request) => { + error!( + function = "RequestBuildingStage::execute", + "RequestType::Responses reached regular request building stage" + ); + Err(grpc_error::internal_error( + "RequestType::Responses reached regular request building stage", + )) + } + } + } + + fn name(&self) -> &'static str { + "RequestBuilding" + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/response_processing.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/response_processing.rs new file mode 100644 index 00000000..a5cb5a13 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/stages/response_processing.rs @@ -0,0 +1,59 @@ +//! Response processing stage that delegates to endpoint-specific implementations + +use std::sync::Arc; + +use async_trait::async_trait; +use axum::response::Response; +use tracing::error; + +use super::{chat::ChatResponseProcessingStage, generate::GenerateResponseProcessingStage}; +use crate::routers::grpc::{ + common::stages::PipelineStage, + context::{RequestContext, RequestType}, + error, + regular::{processor, streaming}, +}; + +/// Response processing stage (delegates to endpoint-specific implementations) +pub struct ResponseProcessingStage { + chat_stage: ChatResponseProcessingStage, + generate_stage: GenerateResponseProcessingStage, +} + +impl ResponseProcessingStage { + pub fn new( + processor: processor::ResponseProcessor, + streaming_processor: Arc, + ) -> Self { + Self { + chat_stage: ChatResponseProcessingStage::new( + processor.clone(), + streaming_processor.clone(), + ), + generate_stage: GenerateResponseProcessingStage::new(processor, streaming_processor), + } + } +} + +#[async_trait] +impl PipelineStage for ResponseProcessingStage { + async fn execute(&self, ctx: &mut RequestContext) -> Result, Response> { + match &ctx.input.request_type { + RequestType::Chat(_) => self.chat_stage.execute(ctx).await, + RequestType::Generate(_) => self.generate_stage.execute(ctx).await, + RequestType::Responses(_) => { + error!( + function = "ResponseProcessingStage::execute", + "RequestType::Responses reached regular response processing stage" + ); + Err(error::internal_error( + "RequestType::Responses reached regular response processing stage", + )) + } + } + } + + fn name(&self) -> &'static str { + "ResponseProcessing" + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/streaming.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/streaming.rs new file mode 100644 index 00000000..5ab0c0be --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/regular/streaming.rs @@ -0,0 +1,1260 @@ +//! Streaming response processor for gRPC routers +//! +//! This module contains shared streaming logic for both Regular and PD router. + +use std::{collections::HashMap, io, sync::Arc, time::Instant}; + +use axum::{body::Body, http::StatusCode, response::Response}; +use bytes::Bytes; +use http::header::{HeaderValue, CONTENT_TYPE}; +use serde_json::{json, Value}; +use tokio::sync::{mpsc, mpsc::UnboundedSender}; +use tokio_stream::wrappers::UnboundedReceiverStream; +use tracing::{debug, error, warn}; + +use crate::{ + grpc_client::sglang_proto::generate_complete::MatchedStop::{MatchedStopStr, MatchedTokenId}, + protocols::{ + chat::{ChatCompletionRequest, ChatCompletionStreamResponse}, + common::{ + FunctionCallDelta, StringOrArray, Tool, ToolCallDelta, ToolChoice, ToolChoiceValue, + Usage, + }, + generate::GenerateRequest, + }, + reasoning_parser::{ParserFactory as ReasoningParserFactory, ParserResult, ReasoningParser}, + routers::grpc::{ + context, + proto_wrapper::{ProtoResponseVariant, ProtoStream}, + utils, + }, + tokenizer::{ + stop::{SequenceDecoderOutput, StopSequenceDecoder}, + traits::Tokenizer, + }, + tool_parser::{ParserFactory as ToolParserFactory, StreamingParseResult, ToolParser}, +}; + +/// Shared streaming processor for both single and dual dispatch modes +#[derive(Clone)] +pub struct StreamingProcessor { + tokenizer: Arc, + tool_parser_factory: ToolParserFactory, + reasoning_parser_factory: ReasoningParserFactory, + configured_tool_parser: Option, + configured_reasoning_parser: Option, +} + +impl StreamingProcessor { + pub fn new( + tokenizer: Arc, + tool_parser_factory: ToolParserFactory, + reasoning_parser_factory: ReasoningParserFactory, + configured_tool_parser: Option, + configured_reasoning_parser: Option, + ) -> Self { + Self { + tokenizer, + tool_parser_factory, + reasoning_parser_factory, + configured_tool_parser, + configured_reasoning_parser, + } + } + + /// Process streaming chat response and return SSE response + /// + /// This is the high-level entry point for streaming responses, handling: + /// - Channel creation + /// - Background task spawning + /// - SSE response building + pub fn process_streaming_response( + self: Arc, + execution_result: context::ExecutionResult, + chat_request: Arc, + dispatch: context::DispatchMetadata, + ) -> Response { + use bytes::Bytes; + use tokio::sync::mpsc; + + let stop_params = ( + chat_request.stop.clone(), + chat_request.stop_token_ids.clone(), + chat_request.skip_special_tokens, + chat_request.no_stop_trim, + ); + + // Create SSE channel + let (tx, rx) = mpsc::unbounded_channel::>(); + + // Spawn background task based on execution mode + match execution_result { + context::ExecutionResult::Single { stream } => { + let processor = self.clone(); + let dispatch_clone = dispatch.clone(); + tokio::spawn(async move { + let result = processor + .process_streaming_chunks( + stream, + dispatch_clone, + stop_params, + chat_request, + &tx, + ) + .await; + + if let Err(e) = result { + let error_chunk = format!( + "data: {}\n\n", + json!({ + "error": { + "message": e, + "type": "internal_error" + } + }) + ); + let _ = tx.send(Ok(Bytes::from(error_chunk))); + } + + let _ = tx.send(Ok(Bytes::from("data: [DONE]\n\n"))); + }); + } + context::ExecutionResult::Dual { prefill, decode } => { + let processor = self.clone(); + tokio::spawn(async move { + let result = processor + .process_dual_streaming_chunks( + prefill, + *decode, + dispatch, + stop_params, + chat_request, + &tx, + ) + .await; + + if let Err(e) = result { + let error_chunk = format!( + "data: {}\n\n", + json!({ + "error": { + "message": e, + "type": "internal_error" + } + }) + ); + let _ = tx.send(Ok(Bytes::from(error_chunk))); + } + + let _ = tx.send(Ok(Bytes::from("data: [DONE]\n\n"))); + }); + } + } + + // Return SSE response + build_sse_response(rx) + } + + /// Process streaming chunks from a single stream (Regular mode) + pub async fn process_streaming_chunks( + &self, + mut grpc_stream: ProtoStream, + dispatch: context::DispatchMetadata, + stop_params: (Option, Option>, bool, bool), + original_request: Arc, + tx: &UnboundedSender>, + ) -> Result<(), String> { + // Extract request parameters + let separate_reasoning = original_request.separate_reasoning; + let tool_choice = &original_request.tool_choice; + let tools = &original_request.tools; + let history_tool_calls_count = utils::get_history_tool_calls_count(&original_request); + let stream_options = &original_request.stream_options; + + // Phase 1: Initialize state tracking (per-index for n>1 support) + let mut is_firsts: HashMap = HashMap::new(); + let mut stream_buffers: HashMap = HashMap::new(); + let mut finish_reasons: HashMap = HashMap::new(); + let mut matched_stops: HashMap> = HashMap::new(); + let mut prompt_tokens: HashMap = HashMap::new(); + let mut completion_tokens: HashMap = HashMap::new(); + let mut cached_tokens: HashMap = HashMap::new(); + + // Parser state (lazy initialization per index) + type PooledReasoningParser = Arc>>; + let mut reasoning_parsers: HashMap = HashMap::new(); + + type PooledToolParser = Arc>>; + let mut tool_parsers: HashMap = HashMap::new(); + let mut has_tool_calls: HashMap = HashMap::new(); + + // Per-index stop decoders (each index needs its own state for n>1 support) + let mut stop_decoders: HashMap = HashMap::new(); + + // Reusable SSE formatting buffer to avoid allocations per chunk + let mut sse_buffer = Vec::with_capacity(512); + + // Use dispatch metadata for consistent response fields + let request_id = &dispatch.request_id; + let model = &dispatch.model; + let created = dispatch.created; + let system_fingerprint = dispatch.weight_version.as_deref(); + + // Check parser availability once upfront (log warning only once per request) + let reasoning_parser_available = separate_reasoning + && utils::check_reasoning_parser_availability( + &self.reasoning_parser_factory, + self.configured_reasoning_parser.as_ref(), + model, + ); + + // Check if JSON schema constraint was used (specific function or required mode) + let used_json_schema = match tool_choice { + Some(ToolChoice::Function { .. }) => true, + Some(ToolChoice::Value(ToolChoiceValue::Required)) => true, + Some(ToolChoice::AllowedTools { mode, .. }) => mode == "required", + _ => false, + }; + + // Check if this is the specific function case (LLM generates parameters only, no name field) + let is_specific_function = matches!(tool_choice, Some(ToolChoice::Function { .. })); + + let tool_parser_available = tools.is_some() + && utils::check_tool_parser_availability( + &self.tool_parser_factory, + self.configured_tool_parser.as_ref(), + model, + ); + + if separate_reasoning && !reasoning_parser_available { + debug!( + "No reasoning parser found for model '{}', skipping reasoning parsing", + model + ); + } + + if tools.is_some() && !tool_parser_available { + debug!( + "No tool parser found for model '{}', skipping tool call parsing", + model + ); + } + + // Phase 2: Main streaming loop + while let Some(response) = grpc_stream.next().await { + let gen_response = response.map_err(|e| format!("Stream error: {}", e))?; + + match gen_response.into_response() { + ProtoResponseVariant::Chunk(chunk) => { + let index = chunk.index(); + + // For vLLM, accumulate completion tokens (vLLM sends deltas) + // For SGLang, skip (SGLang sends cumulative values) + if chunk.is_vllm() { + let tokens_count = completion_tokens.entry(index).or_insert(0); + *tokens_count += chunk.token_ids().len() as u32; + } + + // Get or create stop decoder for this index + let stop_decoder = stop_decoders.entry(index).or_insert_with(|| { + let (ref stop, ref stop_token_ids, skip_special_tokens, no_stop_trim) = + stop_params; + utils::create_stop_decoder( + &self.tokenizer, + stop.as_ref(), + stop_token_ids.as_ref(), + skip_special_tokens, + no_stop_trim, + ) + }); + + // Process tokens through stop decoder + let (chunk_text, _should_stop) = + Self::process_chunk_tokens(stop_decoder, chunk.token_ids()); + + if chunk_text.is_empty() { + continue; + } + + // Process logprobs if present + let choice_logprobs = if let Some(proto_logprobs) = chunk.output_logprobs() { + match utils::convert_proto_to_openai_logprobs( + proto_logprobs, + &self.tokenizer, + ) { + Ok(logprobs) => Some(logprobs), + Err(e) => { + warn!("Failed to process logprobs: {}", e); + None + } + } + } else { + None + }; + + // Initialize stream buffer if first time + let stream_buffer = stream_buffers.entry(index).or_default(); + + // Send first chunk with role + if is_firsts.get(&index).copied().unwrap_or(true) { + let first_chunk = ChatCompletionStreamResponse::builder(request_id, model) + .created(created) + .add_choice_role(index, "assistant") + .maybe_system_fingerprint(system_fingerprint.map(|s| s.to_string())) + .build(); + Self::format_sse_chunk_into(&mut sse_buffer, &first_chunk); + tx.send(Ok(Bytes::from(sse_buffer.clone()))) + .map_err(|_| "Failed to send first chunk".to_string())?; + is_firsts.insert(index, false); + } + + // Calculate delta + let mut delta = chunk_text; + stream_buffer.push_str(&delta); + + // Reasoning content handling + let in_reasoning = if separate_reasoning && reasoning_parser_available { + let (normal_text, reasoning_chunk, in_reasoning) = self + .process_reasoning_stream( + &delta, + index, + &mut reasoning_parsers, + request_id, + model, + created, + system_fingerprint, + ) + .await; + if let Some(chunk) = reasoning_chunk { + Self::format_sse_chunk_into(&mut sse_buffer, &chunk); + tx.send(Ok(Bytes::from(sse_buffer.clone()))) + .map_err(|_| "Failed to send reasoning chunk".to_string())?; + } + delta = normal_text; + in_reasoning + } else { + false + }; + + // Tool call handling + let tool_choice_enabled = + !matches!(tool_choice, Some(ToolChoice::Value(ToolChoiceValue::None))); + + if !in_reasoning + && tool_choice_enabled + && tools.is_some() + && (tool_parser_available || used_json_schema) + { + let tool_chunks = if is_specific_function { + // Handle specific function case - emit tool call deltas with arguments + Self::process_specific_function_stream( + &delta, + index, + &mut has_tool_calls, + tool_choice, + request_id, + model, + created, + system_fingerprint, + history_tool_calls_count, + ) + } else { + // Use incremental parser for regular/required modes + self.process_tool_calls_stream( + &delta, + index, + &mut tool_parsers, + &mut has_tool_calls, + tools.as_ref().unwrap(), + request_id, + model, + created, + system_fingerprint, + history_tool_calls_count, + used_json_schema, + ) + .await + }; + + for chunk in tool_chunks { + Self::format_sse_chunk_into(&mut sse_buffer, &chunk); + tx.send(Ok(Bytes::from(sse_buffer.clone()))) + .map_err(|_| "Failed to send tool call chunk".to_string())?; + } + + // Always skip regular content when tool parsing is active + // Parser either emitted chunks or buffered content + continue; + } + + // Regular content emission + if !delta.is_empty() { + let content_chunk = + ChatCompletionStreamResponse::builder(request_id, model) + .created(created) + .add_choice_content_with_logprobs( + index, + "assistant", + delta, + choice_logprobs, + ) + .maybe_system_fingerprint(system_fingerprint) + .build(); + Self::format_sse_chunk_into(&mut sse_buffer, &content_chunk); + tx.send(Ok(Bytes::from(sse_buffer.clone()))) + .map_err(|_| "Failed to send content chunk".to_string())?; + } + } + ProtoResponseVariant::Complete(complete) => { + let index = complete.index(); + + // Flush any remaining text for this index's stop_decoder + if let Some(decoder) = stop_decoders.get_mut(&index) { + if let SequenceDecoderOutput::Text(text) = decoder.flush() { + if !text.is_empty() { + let stream_buffer = stream_buffers.entry(index).or_default(); + stream_buffer.push_str(&text); + + let content_chunk = + ChatCompletionStreamResponse::builder(request_id, model) + .created(created) + .add_choice_content(index, "assistant", text) + .maybe_system_fingerprint( + system_fingerprint.map(|s| s.to_string()), + ) + .build(); + + let sse_chunk = + serde_json::to_string(&content_chunk).map_err(|e| { + format!("Failed to serialize content chunk: {}", e) + })?; + tx.send(Ok(Bytes::from(format!("data: {}\n\n", sse_chunk)))) + .map_err(|_| "Failed to send flushed content".to_string())?; + } + } + } + + // Store metadata + prompt_tokens.insert(index, complete.prompt_tokens() as u32); + + // For vLLM, use accumulated count (we tracked deltas) + // For SGLang, use complete value (already cumulative) + if complete.is_vllm() { + completion_tokens.entry(index).or_insert(0); + } else { + completion_tokens.insert(index, complete.completion_tokens() as u32); + } + + cached_tokens.insert(index, complete.cached_tokens() as u32); + finish_reasons.insert(index, complete.finish_reason().to_string()); + + // Extract matched_stop + let matched_stop_value = match complete.matched_stop() { + Some(MatchedTokenId(token_id)) => { + Some(Value::Number(serde_json::Number::from(*token_id))) + } + Some(MatchedStopStr(stop_str)) => Some(Value::String(stop_str.clone())), + None => None, + }; + matched_stops.insert(index, matched_stop_value); + + // Don't break - continue reading all Complete messages for n>1 + } + ProtoResponseVariant::Error(error) => { + return Err(error.message().to_string()); + } + ProtoResponseVariant::None => continue, + } + } + + // Phase 3: Check unstreamed tool args + for (index, parser) in &tool_parsers { + let parser_guard = parser.lock().await; + if let Some(unstreamed_items) = parser_guard.get_unstreamed_tool_args() { + for tool_call_item in unstreamed_items { + let tool_call_delta = ToolCallDelta { + index: tool_call_item.tool_index as u32, + id: None, + tool_type: None, + function: Some(FunctionCallDelta { + name: None, + arguments: if !tool_call_item.parameters.is_empty() { + Some(tool_call_item.parameters) + } else { + None + }, + }), + }; + + let tool_chunk = ChatCompletionStreamResponse::builder(request_id, model) + .created(created) + .add_choice_tool_call_delta(*index, tool_call_delta) + .maybe_system_fingerprint(system_fingerprint.map(|s| s.to_string())) + .build(); + + let sse_chunk = serde_json::to_string(&tool_chunk) + .map_err(|e| format!("Failed to serialize tool chunk: {}", e))?; + tx.send(Ok(Bytes::from(format!("data: {}\n\n", sse_chunk)))) + .map_err(|_| "Failed to send unstreamed tool args".to_string())?; + } + } + } + + // Phase 4: Finish reason chunks + for (index, finish_reason) in finish_reasons.iter() { + let final_finish_reason = + if has_tool_calls.get(index).copied().unwrap_or(false) && finish_reason == "stop" { + "tool_calls".to_string() + } else { + finish_reason.clone() + }; + + let matched_stop_value = matched_stops.get(index).and_then(|v| v.clone()); + + let finish_chunk = ChatCompletionStreamResponse::builder(request_id, model) + .created(created) + .add_choice_finish_reason(*index, final_finish_reason, matched_stop_value) + .maybe_system_fingerprint(system_fingerprint.map(|s| s.to_string())) + .build(); + + let sse_chunk = serde_json::to_string(&finish_chunk) + .map_err(|e| format!("Failed to serialize finish chunk: {}", e))?; + tx.send(Ok(Bytes::from(format!("data: {}\n\n", sse_chunk)))) + .map_err(|_| "Failed to send finish chunk".to_string())?; + } + + // Phase 5: Usage chunk + if let Some(stream_opts) = stream_options { + if stream_opts.include_usage.unwrap_or(false) { + let total_prompt: u32 = prompt_tokens.values().sum(); + let total_completion: u32 = completion_tokens.values().sum(); + + let usage_chunk = ChatCompletionStreamResponse::builder(request_id, model) + .created(created) + .usage(Usage { + prompt_tokens: total_prompt, + completion_tokens: total_completion, + total_tokens: total_prompt + total_completion, + completion_tokens_details: None, + }) + .maybe_system_fingerprint(system_fingerprint.map(|s| s.to_string())) + .build(); + + let sse_chunk = serde_json::to_string(&usage_chunk) + .map_err(|e| format!("Failed to serialize usage chunk: {}", e))?; + tx.send(Ok(Bytes::from(format!("data: {}\n\n", sse_chunk)))) + .map_err(|_| "Failed to send usage chunk".to_string())?; + } + } + + // Mark stream as completed successfully to prevent abort on drop + grpc_stream.mark_completed(); + + Ok(()) + } + + /// Process dual streaming chunks (prefill + decode) - PD mode + pub async fn process_dual_streaming_chunks( + &self, + mut prefill_stream: ProtoStream, + decode_stream: ProtoStream, + dispatch: context::DispatchMetadata, + stop_params: (Option, Option>, bool, bool), + original_request: Arc, + tx: &UnboundedSender>, + ) -> Result<(), String> { + // Phase 1.5: Collect input_logprobs from prefill stream if requested + if original_request.logprobs { + while let Some(response) = prefill_stream.next().await { + let gen_response = response.map_err(|e| format!("Prefill stream error: {}", e))?; + match gen_response.into_response() { + ProtoResponseVariant::Complete(_complete) => { + // Input logprobs collected but not yet used in streaming + // (OpenAI spec doesn't require prompt logprobs in streaming responses) + break; + } + ProtoResponseVariant::Error(error) => { + return Err(format!("Prefill error: {}", error.message())); + } + _ => continue, + } + } + } + + // Phase 2-5: Process decode stream (same as single mode) + // Note: decode_stream will be marked completed inside process_streaming_chunks + let result = self + .process_streaming_chunks(decode_stream, dispatch, stop_params, original_request, tx) + .await; + + // Mark prefill stream as completed AFTER decode completes successfully + // This ensures that if client disconnects during decode, BOTH streams send abort + if result.is_ok() { + prefill_stream.mark_completed(); + } + + result + } + + /// Process streaming generate response and return SSE response + /// + /// Simpler than chat - no tool/reasoning parsing, just text accumulation + pub fn process_streaming_generate( + self: Arc, + execution_result: context::ExecutionResult, + generate_request: Arc, + dispatch: context::DispatchMetadata, + ) -> Response { + let return_logprob = generate_request.return_logprob.unwrap_or(false); + + // Create SSE channel + let (tx, rx) = mpsc::unbounded_channel::>(); + + // Spawn background task based on execution mode + match execution_result { + context::ExecutionResult::Single { stream } => { + let tokenizer = self.tokenizer.clone(); + let request_id = dispatch.request_id.clone(); + let weight_version = dispatch + .weight_version + .clone() + .unwrap_or_else(|| "default".to_string()); + tokio::spawn(async move { + let result = Self::process_generate_streaming( + tokenizer, + stream, + request_id, + weight_version, + return_logprob, + &tx, + ) + .await; + + if let Err(e) = result { + let error_chunk = format!("data: {{\"error\": \"{}\"}}\n\n", e); + let _ = tx.send(Ok(Bytes::from(error_chunk))); + } + + let _ = tx.send(Ok(Bytes::from("data: [DONE]\n\n"))); + }); + } + context::ExecutionResult::Dual { prefill, decode } => { + // For PD mode, need to handle prefill stream for input_logprobs + let tokenizer = self.tokenizer.clone(); + let request_id = dispatch.request_id.clone(); + let weight_version = dispatch + .weight_version + .clone() + .unwrap_or_else(|| "default".to_string()); + tokio::spawn(async move { + let result = Self::process_generate_streaming_dual( + tokenizer, + prefill, + *decode, + request_id, + weight_version, + return_logprob, + &tx, + ) + .await; + + if let Err(e) = result { + let error_chunk = format!("data: {{\"error\": \"{}\"}}\n\n", e); + let _ = tx.send(Ok(Bytes::from(error_chunk))); + } + + let _ = tx.send(Ok(Bytes::from("data: [DONE]\n\n"))); + }); + } + } + + // Return SSE response + build_sse_response(rx) + } + + //TODO add streaming logprob support + /// Process streaming chunks for generate endpoint (no tool/reasoning parsing) + async fn process_generate_streaming( + tokenizer: Arc, + mut stream: ProtoStream, + request_id: String, + weight_version: String, + _include_logprobs: bool, + tx: &UnboundedSender>, + ) -> Result<(), String> { + let start_time = Instant::now(); + + // Track state per index for n>1 case + let mut accumulated_texts: HashMap = HashMap::new(); + let mut completion_tokens_map: HashMap = HashMap::new(); + + while let Some(response) = stream.next().await { + let gen_response = response.map_err(|e| format!("Stream error: {}", e))?; + + match gen_response.into_response() { + ProtoResponseVariant::Chunk(chunk) => { + let index = chunk.index(); + + // Both backends send delta token_ids, so accumulate for both + let completion_tokens = completion_tokens_map.entry(index).or_insert(0); + *completion_tokens += chunk.token_ids().len() as u32; + let current_completion_tokens = *completion_tokens; + + // Decode tokens to text (skip_special_tokens=true to handle newlines correctly) + let chunk_text = tokenizer + .decode(chunk.token_ids(), true) + .unwrap_or_default(); + + // Accumulate text for this index + let accumulated_text = accumulated_texts.entry(index).or_default(); + accumulated_text.push_str(&chunk_text); + + // Generate unique ID per index + let index_id = format!("{}-{}", request_id, index); + + // Build streaming response chunk (SGLang format) + let chunk_response = serde_json::json!({ + "text": accumulated_text.clone(), + "output_ids": chunk.token_ids(), + "meta_info": { + "id": index_id, + "finish_reason": null, + "prompt_tokens": chunk.prompt_tokens(), + "weight_version": &weight_version, + "completion_tokens": current_completion_tokens, + "cached_tokens": chunk.cached_tokens() + }, + "index": index + }); + + let sse_chunk = format!( + "data: {}\n\n", + serde_json::to_string(&chunk_response).unwrap() + ); + tx.send(Ok(Bytes::from(sse_chunk))) + .map_err(|_| "Failed to send chunk".to_string())?; + } + ProtoResponseVariant::Complete(complete) => { + let index = complete.index(); + let accumulated_text = + accumulated_texts.get(&index).cloned().unwrap_or_default(); + let completion_tokens = *completion_tokens_map.get(&index).unwrap_or(&0); + let index_id = format!("{}-{}", request_id, index); + let e2e_latency = start_time.elapsed().as_secs_f64(); + + // Send final chunk with finish_reason + let finish_response = serde_json::json!({ + "text": accumulated_text, + "output_ids": complete.output_ids()[complete.output_ids().len().saturating_sub(1)..].to_vec(), + "meta_info": { + "id": index_id, + "finish_reason": complete.finish_reason(), + "prompt_tokens": complete.prompt_tokens(), + "weight_version": &weight_version, + "completion_tokens": completion_tokens, + "cached_tokens": complete.cached_tokens(), + "e2e_latency": e2e_latency + }, + "index": index + }); + + let sse_chunk = format!( + "data: {}\n\n", + serde_json::to_string(&finish_response).unwrap() + ); + tx.send(Ok(Bytes::from(sse_chunk))) + .map_err(|_| "Failed to send finish chunk".to_string())?; + + // Continue to process all completions if n>1 + } + ProtoResponseVariant::Error(error) => { + return Err(error.message().to_string()); + } + ProtoResponseVariant::None => continue, + } + } + + // Mark stream as completed successfully to prevent abort on drop + stream.mark_completed(); + + Ok(()) + } + + /// Process dual streaming for generate endpoint (PD mode with logprobs support) + async fn process_generate_streaming_dual( + tokenizer: Arc, + mut prefill_stream: ProtoStream, + decode_stream: ProtoStream, + request_id: String, + weight_version: String, + return_logprob: bool, + tx: &UnboundedSender>, + ) -> Result<(), String> { + // Collect input_logprobs from prefill stream if requested + let input_token_logprobs = if return_logprob { + let mut input_logprobs = None; + while let Some(response) = prefill_stream.next().await { + let gen_response = response.map_err(|e| format!("Prefill stream error: {}", e))?; + match gen_response.into_response() { + ProtoResponseVariant::Complete(complete) => { + // Extract input_logprobs from prefill Complete message (convert proto to SGLang format) + input_logprobs = complete + .input_logprobs() + .map(utils::convert_generate_input_logprobs); + break; + } + ProtoResponseVariant::Error(error) => { + return Err(format!("Prefill error: {}", error.message())); + } + _ => continue, + } + } + input_logprobs + } else { + None + }; + + // Process decode stream with input_logprobs prepended + // Note: decode_stream will be marked completed inside the function + let result = Self::process_generate_streaming_with_input_logprobs( + tokenizer, + decode_stream, + request_id, + weight_version, + return_logprob, + input_token_logprobs, + tx, + ) + .await; + + // Mark prefill stream as completed AFTER decode completes successfully + // This ensures that if client disconnects during decode, BOTH streams send abort + if result.is_ok() { + prefill_stream.mark_completed(); + } + + result + } + + /// Process generate streaming with optional input_logprobs + async fn process_generate_streaming_with_input_logprobs( + tokenizer: Arc, + mut stream: ProtoStream, + request_id: String, + weight_version: String, + _include_logprobs: bool, + input_token_logprobs: Option>>>, + tx: &UnboundedSender>, + ) -> Result<(), String> { + let start_time = Instant::now(); + + // Track state per index for n>1 case + let mut accumulated_texts: HashMap = HashMap::new(); + let mut accumulated_output_logprobs: HashMap>>>> = + HashMap::new(); + let mut completion_tokens_map: HashMap = HashMap::new(); + + while let Some(response) = stream.next().await { + let gen_response = response.map_err(|e| format!("Stream error: {}", e))?; + + match gen_response.into_response() { + ProtoResponseVariant::Chunk(chunk) => { + let index = chunk.index(); + + // Both backends send delta token_ids, so accumulate for both + let completion_tokens = completion_tokens_map.entry(index).or_insert(0); + *completion_tokens += chunk.token_ids().len() as u32; + let current_completion_tokens = *completion_tokens; + + // Decode tokens to text + let chunk_text = tokenizer + .decode(chunk.token_ids(), true) + .unwrap_or_default(); + + // Accumulate text for this index + let accumulated_text = accumulated_texts.entry(index).or_default(); + accumulated_text.push_str(&chunk_text); + + // Store latest output logprobs (cumulative from proto, convert to SGLang format) + if let Some(output_logprobs) = chunk.output_logprobs() { + let converted = utils::convert_generate_output_logprobs(output_logprobs); + accumulated_output_logprobs.insert(index, Some(converted)); + } + + // Generate unique ID per index + let index_id = format!("{}-{}", request_id, index); + + // Build streaming response chunk with cumulative logprobs + let current_output_logprobs = accumulated_output_logprobs + .get(&index) + .and_then(|o| o.as_ref()); + + let chunk_response = json!({ + "text": accumulated_text.clone(), + "output_ids": chunk.token_ids(), + "meta_info": { + "id": index_id, + "finish_reason": null, + "prompt_tokens": chunk.prompt_tokens(), + "weight_version": &weight_version, + "input_token_logprobs": input_token_logprobs.as_ref(), + "output_token_logprobs": current_output_logprobs, + "completion_tokens": current_completion_tokens, + "cached_tokens": chunk.cached_tokens() + }, + "index": index + }); + + let sse_chunk = format!( + "data: {}\n\n", + serde_json::to_string(&chunk_response).unwrap() + ); + tx.send(Ok(Bytes::from(sse_chunk))) + .map_err(|_| "Failed to send chunk".to_string())?; + } + ProtoResponseVariant::Complete(complete) => { + let index = complete.index(); + let accumulated_text = + accumulated_texts.get(&index).cloned().unwrap_or_default(); + + // Use accumulated count (we tracked deltas from both backends) + let completion_tokens = *completion_tokens_map.get(&index).unwrap_or(&0); + + let final_output_logprobs = accumulated_output_logprobs + .get(&index) + .and_then(|o| o.as_ref()); + let index_id = format!("{}-{}", request_id, index); + let e2e_latency = start_time.elapsed().as_secs_f64(); + + // Parse finish_reason + let finish_reason = utils::parse_finish_reason( + complete.finish_reason(), + complete.completion_tokens(), + ); + + // Send final chunk with finish_reason + let finish_response = json!({ + "text": accumulated_text, + "output_ids": complete.output_ids()[complete.output_ids().len().saturating_sub(1)..].to_vec(), + "meta_info": { + "id": index_id, + "finish_reason": finish_reason, + "prompt_tokens": complete.prompt_tokens(), + "weight_version": &weight_version, + "input_token_logprobs": input_token_logprobs.as_ref(), + "output_token_logprobs": final_output_logprobs, + "completion_tokens": completion_tokens, + "cached_tokens": complete.cached_tokens(), + "e2e_latency": e2e_latency + }, + "index": index + }); + + let sse_chunk = format!( + "data: {}\n\n", + serde_json::to_string(&finish_response).unwrap() + ); + tx.send(Ok(Bytes::from(sse_chunk))) + .map_err(|_| "Failed to send finish chunk".to_string())?; + + // Continue to process all completions if n>1 + } + ProtoResponseVariant::Error(error) => { + return Err(error.message().to_string()); + } + ProtoResponseVariant::None => continue, + } + } + + // Mark stream as completed successfully to prevent abort on drop + stream.mark_completed(); + + Ok(()) + } + + // ======================================================================== + // Helper Methods + // ======================================================================== + + /// Process a chunk of tokens through the stop decoder + fn process_chunk_tokens( + stop_decoder: &mut StopSequenceDecoder, + token_ids: &[u32], + ) -> (String, bool) { + let mut chunk_text = String::new(); + + for &token_id in token_ids { + match stop_decoder.process_token(token_id).unwrap_or_else(|e| { + debug!( + "Error processing token {}: {}. Treating as Held.", + token_id, e + ); + SequenceDecoderOutput::Held + }) { + SequenceDecoderOutput::Text(text) => { + chunk_text.push_str(&text); + } + SequenceDecoderOutput::StoppedWithText(text) => { + chunk_text.push_str(&text); + return (chunk_text, true); + } + SequenceDecoderOutput::Stopped => { + return (chunk_text, true); + } + SequenceDecoderOutput::Held => {} + } + } + (chunk_text, false) + } + + /// Helper: Process reasoning content in streaming mode + #[allow(clippy::too_many_arguments)] + async fn process_reasoning_stream( + &self, + delta: &str, + index: u32, + reasoning_parsers: &mut HashMap>>>, + request_id: &str, + model: &str, + created: u64, + system_fingerprint: Option<&str>, + ) -> (String, Option, bool) { + // Create fresh parser for this index (not pooled, to avoid state pollution) + reasoning_parsers.entry(index).or_insert_with(|| { + let parser = utils::create_reasoning_parser( + &self.reasoning_parser_factory, + self.configured_reasoning_parser.as_ref(), + model, + ) + .expect("Parser should be available - checked upfront"); + Arc::new(tokio::sync::Mutex::new(parser)) + }); + + if let Some(pooled_parser) = reasoning_parsers.get(&index) { + let (parse_result, in_reasoning) = { + let mut parser = pooled_parser.lock().await; + let result = parser.parse_reasoning_streaming_incremental(delta); + let in_reasoning = parser.is_in_reasoning(); + (result, in_reasoning) + }; + + match parse_result { + Ok(ParserResult { + reasoning_text, + normal_text, + }) => { + let chunk = if !reasoning_text.is_empty() { + Some( + ChatCompletionStreamResponse::builder(request_id, model) + .created(created) + .add_choice_reasoning(index, reasoning_text) + .maybe_system_fingerprint(system_fingerprint.map(|s| s.to_string())) + .build(), + ) + } else { + None + }; + return (normal_text, chunk, in_reasoning); + } + Err(e) => { + warn!("Reasoning parsing error: {}", e); + } + } + } + + (delta.to_string(), None, false) + } + + /// Helper: Process specific function case - emit tool call deltas with arguments + #[allow(clippy::too_many_arguments)] + fn process_specific_function_stream( + delta: &str, + index: u32, + has_tool_calls: &mut HashMap, + tool_choice: &Option, + request_id: &str, + model: &str, + created: u64, + system_fingerprint: Option<&str>, + history_tool_calls_count: usize, + ) -> Vec { + let mut chunks = Vec::new(); + + if let Some(ToolChoice::Function { function, .. }) = tool_choice { + let is_first_call = !has_tool_calls.contains_key(&index); + + if is_first_call { + // First chunk: send name and id + has_tool_calls.insert(index, true); + + let tool_call_id = utils::generate_tool_call_id( + model, + &function.name, + 0, + history_tool_calls_count, + ); + + chunks.push( + ChatCompletionStreamResponse::builder(request_id, model) + .created(created) + .add_choice_tool_name(index, tool_call_id, function.name.clone()) + .maybe_system_fingerprint(system_fingerprint.map(|s| s.to_string())) + .build(), + ); + } + + // Emit arguments delta + if !delta.is_empty() { + chunks.push( + ChatCompletionStreamResponse::builder(request_id, model) + .created(created) + .add_choice_tool_args(index, delta.to_string()) + .maybe_system_fingerprint(system_fingerprint.map(|s| s.to_string())) + .build(), + ); + } + } + + chunks + } + + /// Helper: Process tool calls in streaming mode + #[allow(clippy::too_many_arguments)] + async fn process_tool_calls_stream( + &self, + delta: &str, + index: u32, + tool_parsers: &mut HashMap>>>, + has_tool_calls: &mut HashMap, + tools: &[Tool], + request_id: &str, + model: &str, + created: u64, + system_fingerprint: Option<&str>, + history_tool_calls_count: usize, + use_json_parser: bool, + ) -> Vec { + let mut chunks = Vec::new(); + + // Create fresh parser for this index (not pooled, to avoid state pollution) + tool_parsers.entry(index).or_insert_with(|| { + let parser = if use_json_parser { + utils::create_tool_parser( + &self.tool_parser_factory, + Some(&"json".to_string()), + model, + ) + .expect("JSON parser should be available") + } else { + utils::create_tool_parser( + &self.tool_parser_factory, + self.configured_tool_parser.as_ref(), + model, + ) + .expect("Parser should be available - checked upfront") + }; + Arc::new(tokio::sync::Mutex::new(parser)) + }); + + if let Some(pooled_parser) = tool_parsers.get(&index) { + let mut parser = pooled_parser.lock().await; + + match parser.parse_incremental(delta, tools).await { + Ok(StreamingParseResult { normal_text, calls }) => { + // Emit normal text if present + if !normal_text.is_empty() { + chunks.push( + ChatCompletionStreamResponse::builder(request_id, model) + .created(created) + .add_choice_content(index, "assistant", normal_text) + .maybe_system_fingerprint(system_fingerprint.map(|s| s.to_string())) + .build(), + ); + } + + // Emit tool call chunks + for tool_call_item in calls { + has_tool_calls.insert(index, true); + + let tool_call_id = if let Some(ref name) = tool_call_item.name { + Some(utils::generate_tool_call_id( + model, + name, + tool_call_item.tool_index, + history_tool_calls_count, + )) + } else { + None + }; + + let tool_call_delta = ToolCallDelta { + index: tool_call_item.tool_index as u32, + id: tool_call_id, + tool_type: if tool_call_item.name.is_some() { + Some("function".to_string()) + } else { + None + }, + function: Some(FunctionCallDelta { + name: tool_call_item.name, + arguments: if !tool_call_item.parameters.is_empty() { + Some(tool_call_item.parameters) + } else { + None + }, + }), + }; + + chunks.push( + ChatCompletionStreamResponse::builder(request_id, model) + .created(created) + .add_choice_tool_call_delta(index, tool_call_delta) + .maybe_system_fingerprint(system_fingerprint.map(|s| s.to_string())) + .build(), + ); + } + + return chunks; + } + Err(e) => { + error!("Tool call parsing error: {}", e); + } + } + } + + chunks + } + + /// Format a response as SSE chunk into a reusable buffer + /// This avoids allocations by reusing the same buffer across multiple chunks + #[inline] + fn format_sse_chunk_into(buffer: &mut Vec, chunk: &ChatCompletionStreamResponse) { + buffer.clear(); + buffer.extend_from_slice(b"data: "); + if let Err(e) = serde_json::to_writer(&mut *buffer, chunk) { + error!("Failed to serialize SSE chunk: {}", e); + buffer.clear(); + buffer.extend_from_slice(b"data: "); + let error_msg = json!({"error": "serialization_failed"}).to_string(); + buffer.extend_from_slice(error_msg.as_bytes()); + } + buffer.extend_from_slice(b"\n\n"); + } +} + +/// Build SSE response with proper headers +pub fn build_sse_response(rx: mpsc::UnboundedReceiver>) -> Response { + let stream = UnboundedReceiverStream::new(rx); + let mut response = Response::new(Body::from_stream(stream)); + *response.status_mut() = StatusCode::OK; + response + .headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static("text/event-stream")); + response + .headers_mut() + .insert("Cache-Control", HeaderValue::from_static("no-cache")); + response + .headers_mut() + .insert("Connection", HeaderValue::from_static("keep-alive")); + response +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/router.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/router.rs new file mode 100644 index 00000000..da3b89ae --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/router.rs @@ -0,0 +1,359 @@ +use std::sync::Arc; + +use async_trait::async_trait; +use axum::{ + body::Body, + extract::Request, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, +}; +use tracing::debug; + +use super::{ + common::responses::{ + handlers::{cancel_response_impl, get_response_impl}, + utils::validate_worker_availability, + }, + context::SharedComponents, + harmony::{ + serve_harmony_responses, serve_harmony_responses_stream, HarmonyDetector, + HarmonyResponsesContext, + }, + pipeline::RequestPipeline, + regular::responses, +}; +use crate::{ + app_context::AppContext, + core::WorkerRegistry, + protocols::{ + chat::ChatCompletionRequest, + classify::ClassifyRequest, + completion::CompletionRequest, + embedding::EmbeddingRequest, + generate::GenerateRequest, + rerank::RerankRequest, + responses::{ResponsesGetParams, ResponsesRequest}, + }, + routers::RouterTrait, +}; + +/// gRPC router implementation for SGLang +#[derive(Clone)] +#[allow(dead_code)] +pub struct GrpcRouter { + worker_registry: Arc, + pipeline: RequestPipeline, + harmony_pipeline: RequestPipeline, + shared_components: Arc, + responses_context: responses::ResponsesContext, + harmony_responses_context: responses::ResponsesContext, +} + +impl GrpcRouter { + /// Create a new gRPC router + pub async fn new(ctx: &Arc) -> Result { + // Extract necessary components from context + let tokenizer = ctx + .tokenizer + .as_ref() + .ok_or_else(|| "gRPC router requires tokenizer".to_string())? + .clone(); + let reasoning_parser_factory = ctx + .reasoning_parser_factory + .as_ref() + .ok_or_else(|| "gRPC router requires reasoning parser factory".to_string())? + .clone(); + let tool_parser_factory = ctx + .tool_parser_factory + .as_ref() + .ok_or_else(|| "gRPC router requires tool parser factory".to_string())? + .clone(); + + let worker_registry = ctx.worker_registry.clone(); + let _policy_registry = ctx.policy_registry.clone(); + + // Create shared components for pipeline + let shared_components = Arc::new(SharedComponents { + tokenizer: tokenizer.clone(), + tool_parser_factory: tool_parser_factory.clone(), + reasoning_parser_factory: reasoning_parser_factory.clone(), + }); + + // Create regular pipeline + let pipeline = RequestPipeline::new_regular( + worker_registry.clone(), + _policy_registry.clone(), + tokenizer.clone(), + tool_parser_factory.clone(), + reasoning_parser_factory.clone(), + ctx.configured_tool_parser.clone(), + ctx.configured_reasoning_parser.clone(), + ); + + // Create Harmony pipelines + let harmony_pipeline = RequestPipeline::new_harmony( + worker_registry.clone(), + _policy_registry.clone(), + tokenizer.clone(), + tool_parser_factory.clone(), + reasoning_parser_factory.clone(), + ctx.configured_tool_parser.clone(), + ctx.configured_reasoning_parser.clone(), + ); + + // Extract shared dependencies for responses contexts + let mcp_manager = ctx + .mcp_manager + .get() + .ok_or_else(|| "gRPC router requires MCP manager".to_string())? + .clone(); + + // Helper closure to create responses context with a given pipeline + let create_responses_context = |pipeline: &RequestPipeline| { + responses::ResponsesContext::new( + Arc::new(pipeline.clone()), + shared_components.clone(), + worker_registry.clone(), + ctx.response_storage.clone(), + ctx.conversation_storage.clone(), + ctx.conversation_item_storage.clone(), + mcp_manager.clone(), + ) + }; + + // Create responses contexts for both pipelines + let responses_context = create_responses_context(&pipeline); + let harmony_responses_context = create_responses_context(&harmony_pipeline); + + Ok(GrpcRouter { + worker_registry, + pipeline, + harmony_pipeline, + shared_components, + responses_context, + harmony_responses_context, + }) + } + + /// Main route_chat implementation + async fn route_chat_impl( + &self, + headers: Option<&HeaderMap>, + body: &ChatCompletionRequest, + model_id: Option<&str>, + ) -> Response { + // Choose Harmony pipeline if model indicates Harmony + let is_harmony = HarmonyDetector::is_harmony_model(&body.model); + + debug!( + "Processing chat completion request for model: {:?}, using_harmony={}", + model_id, is_harmony + ); + + let pipeline = if is_harmony { + &self.harmony_pipeline + } else { + &self.pipeline + }; + + pipeline + .execute_chat( + Arc::new(body.clone()), + headers.cloned(), + model_id.map(|s| s.to_string()), + self.shared_components.clone(), + ) + .await + } + + /// Main route_generate implementation + async fn route_generate_impl( + &self, + headers: Option<&HeaderMap>, + body: &GenerateRequest, + model_id: Option<&str>, + ) -> Response { + debug!("Processing generate request for model: {:?}", model_id); + + self.pipeline + .execute_generate( + Arc::new(body.clone()), + headers.cloned(), + model_id.map(|s| s.to_string()), + self.shared_components.clone(), + ) + .await + } + + /// Main route_responses implementation + /// + /// Routes to either Harmony or regular responses implementation based on model detection + async fn route_responses_impl( + &self, + headers: Option<&HeaderMap>, + body: &ResponsesRequest, + model_id: Option<&str>, + ) -> Response { + // 0. Fast worker validation (fail-fast before expensive operations) + let requested_model: Option<&str> = model_id.or(Some(body.model.as_str())); + + if let Some(error_response) = requested_model + .and_then(|model| validate_worker_availability(&self.worker_registry, model)) + { + return error_response; + } + + // Choose implementation based on Harmony model detection + let is_harmony = HarmonyDetector::is_harmony_model(&body.model); + + if is_harmony { + debug!( + "Processing Harmony responses request for model: {:?}, streaming: {:?}", + model_id, body.stream + ); + let harmony_ctx = HarmonyResponsesContext::new( + Arc::new(self.harmony_pipeline.clone()), + self.shared_components.clone(), + self.harmony_responses_context.mcp_manager.clone(), + self.harmony_responses_context.response_storage.clone(), + self.harmony_responses_context.conversation_storage.clone(), + self.harmony_responses_context + .conversation_item_storage + .clone(), + ); + + if body.stream.unwrap_or(false) { + serve_harmony_responses_stream(&harmony_ctx, body.clone()).await + } else { + match serve_harmony_responses(&harmony_ctx, body.clone()).await { + Ok(response) => axum::Json(response).into_response(), + Err(error_response) => error_response, + } + } + } else { + responses::route_responses( + &self.responses_context, + Arc::new(body.clone()), + headers.cloned(), + model_id.map(|s| s.to_string()), + ) + .await + } + } +} + +impl std::fmt::Debug for GrpcRouter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let stats = self.worker_registry.stats(); + f.debug_struct("GrpcRouter") + .field("workers_count", &stats.total_workers) + .finish() + } +} + +#[async_trait] +impl RouterTrait for GrpcRouter { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + async fn health_generate(&self, _req: Request) -> Response { + ( + StatusCode::NOT_IMPLEMENTED, + "Health generate not yet implemented for gRPC", + ) + .into_response() + } + + async fn get_server_info(&self, _req: Request) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn get_models(&self, _req: Request) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn get_model_info(&self, _req: Request) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn route_generate( + &self, + headers: Option<&HeaderMap>, + body: &GenerateRequest, + model_id: Option<&str>, + ) -> Response { + self.route_generate_impl(headers, body, model_id).await + } + + async fn route_chat( + &self, + headers: Option<&HeaderMap>, + body: &ChatCompletionRequest, + model_id: Option<&str>, + ) -> Response { + self.route_chat_impl(headers, body, model_id).await + } + + async fn route_completion( + &self, + _headers: Option<&HeaderMap>, + _body: &CompletionRequest, + _model_id: Option<&str>, + ) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn route_responses( + &self, + headers: Option<&HeaderMap>, + body: &ResponsesRequest, + model_id: Option<&str>, + ) -> Response { + self.route_responses_impl(headers, body, model_id).await + } + + async fn get_response( + &self, + _headers: Option<&HeaderMap>, + response_id: &str, + _params: &ResponsesGetParams, + ) -> Response { + get_response_impl(&self.responses_context, response_id).await + } + + async fn cancel_response(&self, _headers: Option<&HeaderMap>, response_id: &str) -> Response { + cancel_response_impl(&self.responses_context, response_id).await + } + + async fn route_embeddings( + &self, + _headers: Option<&HeaderMap>, + _body: &EmbeddingRequest, + _model_id: Option<&str>, + ) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn route_classify( + &self, + _headers: Option<&HeaderMap>, + _body: &ClassifyRequest, + _model_id: Option<&str>, + ) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn route_rerank( + &self, + _headers: Option<&HeaderMap>, + _body: &RerankRequest, + _model_id: Option<&str>, + ) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + fn router_type(&self) -> &'static str { + "grpc" + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/utils.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/utils.rs new file mode 100644 index 00000000..64bde929 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/grpc/utils.rs @@ -0,0 +1,1147 @@ +//! Shared utilities for gRPC routers + +use std::{collections::HashMap, sync::Arc}; + +use axum::response::Response; +use serde_json::{json, Map, Value}; +use tracing::{error, warn}; +use uuid::Uuid; + +use super::{ + client::GrpcClient, + error, + proto_wrapper::{ProtoGenerateComplete, ProtoStream}, + ProcessedMessages, +}; +use crate::{ + core::Worker, + grpc_client::sglang_proto::{InputLogProbs, OutputLogProbs}, + protocols::{ + chat::{ChatCompletionRequest, ChatMessage}, + common::{ + ChatLogProbs, ChatLogProbsContent, FunctionCallResponse, StringOrArray, Tool, ToolCall, + ToolChoice, ToolChoiceValue, TopLogProb, + }, + generate::GenerateFinishReason, + }, + reasoning_parser::{ + ParserFactory as ReasoningParserFactory, PooledParser as ReasoningPooledParser, + ReasoningParser, + }, + routers::grpc::proto_wrapper::ProtoResponseVariant, + tokenizer::{ + cache::CachedTokenizer, + chat_template::{ChatTemplateContentFormat, ChatTemplateParams}, + stop::StopSequenceDecoderBuilder, + traits::Tokenizer, + HuggingFaceTokenizer, StopSequenceDecoder, + }, + tool_parser::{ + ParserFactory as ToolParserFactory, PooledParser as ToolPooledParser, ToolParser, + }, +}; + +/// Get gRPC client from worker, returning appropriate error response on failure +pub async fn get_grpc_client_from_worker(worker: &Arc) -> Result { + // Get cached client from worker (or create one if not cached yet) + let client_arc = worker + .get_grpc_client() + .await + .map_err(|e| { + error!( + function = "get_grpc_client_from_worker", + error = %e, + "Failed to get gRPC client from worker" + ); + error::internal_error(format!("Failed to get gRPC client: {}", e)) + })? + .ok_or_else(|| { + error!( + function = "get_grpc_client_from_worker", + "Selected worker not configured for gRPC" + ); + error::internal_error("Selected worker is not configured for gRPC") + })?; + + Ok((*client_arc).clone()) +} + +/// Process tool call arguments in messages +/// Per Transformers docs, tool call arguments in assistant messages should be dicts +fn process_tool_call_arguments(messages: &mut [Value]) -> Result<(), String> { + for msg in messages { + let role = msg.get("role").and_then(|v| v.as_str()); + if role != Some("assistant") { + continue; + } + + let Some(tool_calls) = msg.get_mut("tool_calls").and_then(|tc| tc.as_array_mut()) else { + continue; + }; + + for call in tool_calls { + let Some(function) = call.get_mut("function") else { + continue; + }; + let Some(args) = function.get_mut("arguments") else { + continue; + }; + let Some(args_str) = args.as_str() else { + continue; + }; + + // Parse JSON string to object (like Python json.loads) + match serde_json::from_str::(args_str) { + Ok(parsed) => *args = parsed, + Err(e) => { + return Err(format!( + "Failed to parse tool call arguments as JSON: '{}'. Error: {}", + args_str, e + )) + } + } + } + } + Ok(()) +} + +/// Process messages based on content format for ANY message type +pub fn process_content_format( + messages: &[ChatMessage], + content_format: ChatTemplateContentFormat, +) -> Result, String> { + messages + .iter() + .map(|message| { + let mut message_json = serde_json::to_value(message) + .map_err(|e| format!("Failed to serialize message: {}", e))?; + + if let Some(obj) = message_json.as_object_mut() { + if let Some(content_value) = obj.get_mut("content") { + transform_content_field(content_value, content_format); + } + } + + Ok(message_json) + }) + .collect() +} + +/// Transform a single content field based on content format +fn transform_content_field(content_value: &mut Value, content_format: ChatTemplateContentFormat) { + let Some(content_array) = content_value.as_array() else { + return; // Not multimodal, keep as-is + }; + + match content_format { + ChatTemplateContentFormat::String => { + // Extract and join text parts only + let text_parts: Vec = content_array + .iter() + .filter_map(|part| { + part.as_object()? + .get("type")? + .as_str() + .filter(|&t| t == "text") + .and_then(|_| part.as_object()?.get("text")?.as_str()) + .map(String::from) + }) + .collect(); + + if !text_parts.is_empty() { + *content_value = Value::String(text_parts.join(" ")); + } + } + ChatTemplateContentFormat::OpenAI => { + // Replace media URLs with simple type placeholders + let processed_parts: Vec = content_array + .iter() + .map(|part| { + part.as_object() + .and_then(|obj| obj.get("type")?.as_str()) + .and_then(|type_str| match type_str { + "image_url" => Some(json!({"type": "image"})), + "video_url" => Some(json!({"type": "video"})), + "audio_url" => Some(json!({"type": "audio"})), + _ => None, + }) + .unwrap_or_else(|| part.clone()) + }) + .collect(); + + *content_value = Value::Array(processed_parts); + } + } +} + +/// Generate tool constraints for structured generation +/// Note: tools should already be filtered if needed (by allowed_tools or specific function) +pub fn generate_tool_constraints( + tools: &[Tool], + tool_choice: &Option, + _model: &str, +) -> Result, String> { + let Some(choice) = tool_choice.as_ref() else { + return Ok(None); + }; + + match choice { + // Specific function: Return parameters schema directly + // tools should already be filtered to contain only the specific function + ToolChoice::Function { .. } => { + if tools.is_empty() { + return Ok(None); + } + let tool = &tools[0]; + + // Return the tool's parameters schema directly (not wrapped in array) + let params_schema = serde_json::to_string(&tool.function.parameters) + .map_err(|e| format!("Failed to serialize tool parameters: {}", e))?; + Ok(Some(("json_schema".to_string(), params_schema))) + } + + // Required: Array of tool calls with minItems: 1 + ToolChoice::Value(ToolChoiceValue::Required) => { + let schema = build_required_array_schema(tools)?; + Ok(Some(("json_schema".to_string(), schema))) + } + + // AllowedTools with required mode: tools are already filtered + ToolChoice::AllowedTools { mode, .. } => { + if mode == "required" { + if tools.is_empty() { + return Ok(None); + } + let schema = build_required_array_schema(tools)?; + Ok(Some(("json_schema".to_string(), schema))) + } else { + // "auto" mode - no constraint needed + Ok(None) + } + } + + // "auto" or "none" - no constraint + _ => Ok(None), + } +} + +/// Build JSON schema for required tool calls (array with minItems: 1) +/// Includes $defs consolidation from all tools (matching Python's behavior) +fn build_required_array_schema(tools: &[Tool]) -> Result { + // Build anyOf schemas for each tool + let mut any_of_schemas = Vec::new(); + for tool in tools { + let tool_schema = json!({ + "properties": { + "name": { + "type": "string", + "enum": [tool.function.name] + }, + "parameters": tool.function.parameters + }, + "required": ["name", "parameters"] + }); + any_of_schemas.push(tool_schema); + } + + // Consolidate $defs from all tools (matching Python's _get_tool_schema_defs) + let mut all_defs: HashMap = HashMap::new(); + for tool in tools { + if let Value::Object(params) = &tool.function.parameters { + if let Some(Value::Object(defs)) = params.get("$defs") { + for (def_name, def_schema) in defs { + if let Some(existing) = all_defs.get(def_name) { + // Check for conflicts + if existing != def_schema { + let error_msg = format!( + "Tool definition '{}' has multiple conflicting schemas, which is not supported", + def_name + ); + error!("{}", error_msg); + return Err(error_msg); + } + } else { + all_defs.insert(def_name.clone(), def_schema.clone()); + } + } + } + } + } + + // Build the full array schema + let mut array_schema = json!({ + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "anyOf": any_of_schemas + } + }); + + // Add $defs if any were found (matching Python's behavior) + if !all_defs.is_empty() { + if let Value::Object(ref mut schema_obj) = array_schema { + let defs_value = Value::Object(all_defs.into_iter().collect::>()); + schema_obj.insert("$defs".to_string(), defs_value); + } + } + + serde_json::to_string(&array_schema) + .map_err(|e| format!("Failed to serialize tool schema: {}", e)) +} + +/// Filter tools based on tool_choice (generic helper) +/// +/// Returns filtered tools if filtering is needed, otherwise returns None. +/// Used by both Chat API and Responses API (Harmony) for constraint generation. +pub fn filter_tools_by_tool_choice( + tools: &[Tool], + tool_choice: &Option, +) -> Option> { + match tool_choice { + Some(ToolChoice::AllowedTools { tools: allowed, .. }) => { + let allowed_names: std::collections::HashSet<&str> = + allowed.iter().filter_map(|t| t.function_name()).collect(); + let filtered: Vec = tools + .iter() + .filter(|t| allowed_names.contains(t.function.name.as_str())) + .cloned() + .collect(); + Some(filtered) + } + Some(ToolChoice::Function { function, .. }) => { + let filtered: Vec = tools + .iter() + .filter(|t| t.function.name == function.name) + .cloned() + .collect(); + Some(filtered) + } + _ => None, // No filtering needed + } +} + +/// Filter ChatCompletionRequest by tool_choice +/// +/// Returns a reference to the original request if no filtering needed, +/// otherwise returns a cloned request with filtered tools. +/// +/// Note: Tool existence is validated earlier in ChatCompletionRequest::validate(), +/// so this function assumes tool_choice references valid tools. +pub fn filter_chat_request_by_tool_choice( + body: &ChatCompletionRequest, +) -> std::borrow::Cow<'_, ChatCompletionRequest> { + if let Some(tools) = &body.tools { + if let Some(filtered_tools) = filter_tools_by_tool_choice(tools, &body.tool_choice) { + let mut filtered_body = body.clone(); + filtered_body.tools = Some(filtered_tools); + return std::borrow::Cow::Owned(filtered_body); + } + } + + // No filtering needed - return original request + std::borrow::Cow::Borrowed(body) +} + +/// Process chat messages and apply template (shared by both routers) +/// Requires HuggingFace tokenizer with chat template support +pub fn process_chat_messages( + request: &ChatCompletionRequest, + tokenizer: &dyn Tokenizer, +) -> Result { + // Use the tokenizer's chat template - we require HuggingFace tokenizer for gRPC + // First try direct downcast, then try via CachedTokenizer wrapper + let hf_tokenizer = tokenizer + .as_any() + .downcast_ref::() + .or_else(|| { + // If direct downcast fails, try to get inner tokenizer from CachedTokenizer + tokenizer + .as_any() + .downcast_ref::() + .and_then(|cached| { + cached + .inner() + .as_any() + .downcast_ref::() + }) + }); + + let formatted_text = if let Some(hf_tokenizer) = hf_tokenizer { + // Get content format and transform messages accordingly + let content_format = hf_tokenizer.chat_template_content_format(); + let mut transformed_messages = process_content_format(&request.messages, content_format)?; + + // Process tool call arguments in assistant messages + process_tool_call_arguments(&mut transformed_messages)?; + + // Convert tools to JSON values for template processing + let tools_json: Option> = request + .tools + .as_ref() + .map(|tools| { + tools + .iter() + .map(serde_json::to_value) + .collect::, _>>() + }) + .transpose() + .map_err(|e| format!("Failed to serialize tools: {}", e))?; + + // Build template kwargs, merging reasoning_effort if present + let mut combined_template_kwargs = HashMap::new(); + + // Add reasoning_effort if present (like Python does) + if let Some(reasoning_effort) = &request.reasoning_effort { + combined_template_kwargs.insert( + "reasoning_effort".to_string(), + Value::String(reasoning_effort.clone()), + ); + } + + // Add any additional template kwargs from request + if let Some(template_kwargs) = &request.chat_template_kwargs { + for (key, value) in template_kwargs { + combined_template_kwargs.insert(key.clone(), value.clone()); + } + } + + let final_template_kwargs = if combined_template_kwargs.is_empty() { + None + } else { + Some(&combined_template_kwargs) + }; + + let params = ChatTemplateParams { + add_generation_prompt: true, + tools: tools_json.as_deref(), + template_kwargs: final_template_kwargs, + ..Default::default() + }; + + // Handle assistant prefix for continue_final_message + let assistant_prefix = if request.continue_final_message + && !transformed_messages.is_empty() + && transformed_messages + .last() + .and_then(|msg| msg.get("role")) + .and_then(|v| v.as_str()) + == Some("assistant") + { + // Pop the last message to handle it separately + let last_msg = transformed_messages.pop().unwrap(); + last_msg + .get("content") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + } else { + None + }; + + // Apply chat template with the (now possibly shorter) list of messages + let rendered = hf_tokenizer + .apply_chat_template(&transformed_messages, params) + .map_err(|e| format!("Failed to apply chat template: {}", e))?; + + // Append assistant prefix if we have one + if let Some(prefix) = assistant_prefix { + format!("{}{}", rendered, prefix) + } else { + rendered + } + } else { + return Err( + "gRPC router requires HuggingFace tokenizer with chat template support".to_string(), + ); + }; + + // Placeholder for multimodal inputs + let multimodal_inputs = None; + + Ok(ProcessedMessages { + text: formatted_text, + multimodal_inputs, + stop_sequences: request.stop.clone(), + }) +} + +/// Create a StopSequenceDecoder from stop parameters +pub fn create_stop_decoder( + tokenizer: &Arc, + stop: Option<&StringOrArray>, + stop_token_ids: Option<&Vec>, + skip_special_tokens: bool, + no_stop_trim: bool, +) -> StopSequenceDecoder { + // Extract stop sequences + let stop_sequences: Vec = match stop { + Some(StringOrArray::String(s)) => vec![s.clone()], + Some(StringOrArray::Array(arr)) => arr.clone(), + None => vec![], + }; + + // Build stop sequence decoder + let mut builder = + StopSequenceDecoderBuilder::new(tokenizer.clone()).skip_special_tokens(skip_special_tokens); + + // Add stop sequences (visible if no_stop_trim is true, hidden otherwise) + for seq in stop_sequences { + builder = if no_stop_trim { + builder.visible_stop_sequence(seq) + } else { + builder.stop_sequence(seq) + }; + } + + // Add stop token IDs (visible if no_stop_trim is true, hidden otherwise) + if let Some(token_ids) = stop_token_ids { + for &token_id in token_ids { + builder = if no_stop_trim { + builder.visible_stop_token(token_id) + } else { + builder.stop_token(token_id) + }; + } + } + + builder.build() +} + +/// Parse tool calls from JSON schema constrained response +pub fn parse_json_schema_response( + processed_text: &str, + tool_choice: &Option, + model: &str, + history_tool_calls_count: usize, +) -> (Option>, String) { + match tool_choice { + Some(ToolChoice::Function { function, .. }) => { + // Specific function: Parse parameters directly + match serde_json::from_str::(processed_text) { + Ok(params) => { + let tool_call = ToolCall { + id: generate_tool_call_id( + model, + &function.name, + 0, + history_tool_calls_count, + ), + tool_type: "function".to_string(), + function: FunctionCallResponse { + name: function.name.clone(), + arguments: Some( + serde_json::to_string(¶ms).unwrap_or_else(|_| "{}".to_string()), + ), + }, + }; + (Some(vec![tool_call]), String::new()) + } + Err(e) => { + error!("Failed to parse specific function parameters: {}", e); + (None, processed_text.to_string()) + } + } + } + Some(ToolChoice::Value(ToolChoiceValue::Required)) + | Some(ToolChoice::AllowedTools { .. }) => { + // Required mode: Parse array of tool calls + match serde_json::from_str::>(processed_text) { + Ok(parsed_array) => { + let spec_tool_calls: Vec = parsed_array + .into_iter() + .enumerate() + .filter_map(|(i, item)| { + let obj = item.as_object()?; + let name = obj.get("name")?.as_str()?.to_string(); + let parameters = obj.get("parameters")?; + + Some(ToolCall { + id: generate_tool_call_id( + model, + &name, + i, + history_tool_calls_count, + ), + tool_type: "function".to_string(), + function: FunctionCallResponse { + name, + arguments: Some( + serde_json::to_string(parameters) + .unwrap_or_else(|_| "{}".to_string()), + ), + }, + }) + }) + .collect(); + (Some(spec_tool_calls), String::new()) + } + Err(e) => { + error!("Failed to parse required tool call array: {}", e); + (None, processed_text.to_string()) + } + } + } + _ => (None, processed_text.to_string()), + } +} + +/// Collect responses from a gRPC stream +/// +/// This helper processes a gRPC GenerateResponse stream and collects all Complete responses. +/// Used by both regular and PD routers for non-streaming requests. +/// +/// # Arguments +/// * `stream` - The gRPC response stream to consume +/// * `worker_name` - Name for logging (e.g., "Prefill", "Decode", "Worker") +/// +/// # Returns +/// * `Ok(Vec)` - All complete responses collected from the stream +/// * `Err(Response)` - Error response if the stream fails or returns an error +pub async fn collect_stream_responses( + stream: &mut ProtoStream, + worker_name: &str, +) -> Result, Response> { + let mut all_responses = Vec::new(); + + while let Some(response) = stream.next().await { + match response { + Ok(gen_response) => { + match gen_response.into_response() { + ProtoResponseVariant::Complete(complete) => { + all_responses.push(complete); + } + ProtoResponseVariant::Error(err) => { + error!(function = "collect_stream_responses", worker = %worker_name, error = %err.message(), "Worker generation error"); + // Don't mark as completed - let Drop send abort for error cases + return Err(error::internal_error(format!( + "{} generation failed: {}", + worker_name, + err.message() + ))); + } + ProtoResponseVariant::Chunk(_chunk) => { + // Streaming chunk - no action needed + } + ProtoResponseVariant::None => { + // Empty response - no action needed + } + } + } + Err(e) => { + error!(function = "collect_stream_responses", worker = %worker_name, error = ?e, "Worker stream error"); + // Don't mark as completed - let Drop send abort for error cases + return Err(error::internal_error(format!( + "{} stream failed: {}", + worker_name, e + ))); + } + } + } + + Ok(all_responses) +} + +/// Count the number of tool calls in the request message history +/// This is used for KimiK2 format which needs globally unique indices +pub fn get_history_tool_calls_count(request: &ChatCompletionRequest) -> usize { + request + .messages + .iter() + .filter_map(|msg| { + if let ChatMessage::Assistant { tool_calls, .. } = msg { + tool_calls.as_ref().map(|calls| calls.len()) + } else { + None + } + }) + .sum() +} + +/// Generate a tool call ID based on model format +/// +/// # Arguments +/// * `model` - Model name to determine ID format +/// * `tool_name` - Name of the tool being called +/// * `tool_index` - Index of this tool call within the current message +/// * `history_count` - Number of tool calls in previous messages +/// +/// # Returns +/// A unique ID string. KimiK2 uses `functions.{name}:{global_index}`, others use `call_{uuid}` +pub fn generate_tool_call_id( + model: &str, + tool_name: &str, + tool_index: usize, + history_count: usize, +) -> String { + if model.to_lowercase().contains("kimi") { + // KimiK2 format: functions.{name}:{global_index} + format!("functions.{}:{}", tool_name, history_count + tool_index) + } else { + // Standard OpenAI format: call_{24-char-uuid} + format!("call_{}", &Uuid::new_v4().simple().to_string()[..24]) + } +} + +/// Check if a reasoning parser is available for the given model +pub fn check_reasoning_parser_availability( + reasoning_parser_factory: &ReasoningParserFactory, + configured_parser: Option<&String>, + model: &str, +) -> bool { + if let Some(parser_name) = configured_parser { + reasoning_parser_factory.registry().has_parser(parser_name) + } else { + reasoning_parser_factory + .registry() + .has_parser_for_model(model) + } +} + +/// Check if a tool parser is available for the given model +pub fn check_tool_parser_availability( + tool_parser_factory: &ToolParserFactory, + configured_parser: Option<&String>, + model: &str, +) -> bool { + if let Some(parser_name) = configured_parser { + tool_parser_factory.registry().has_parser(parser_name) + } else { + tool_parser_factory.registry().has_parser_for_model(model) + } +} + +/// Get the appropriate reasoning parser for a model +/// +/// If a parser name is explicitly configured, use that parser. +/// Otherwise, auto-detect based on the model name. +/// Get a pooled reasoning parser (for non-streaming where state doesn't matter) +pub fn get_reasoning_parser( + reasoning_parser_factory: &ReasoningParserFactory, + configured_parser: Option<&String>, + model: &str, +) -> ReasoningPooledParser { + if let Some(parser_name) = configured_parser { + // Use configured parser if specified + reasoning_parser_factory + .registry() + .get_pooled_parser(parser_name) + .unwrap_or_else(|| { + warn!( + "Configured reasoning parser '{}' not found, falling back to model-based selection", + parser_name + ); + reasoning_parser_factory.get_pooled(model) + }) + } else { + // Auto-detect based on model + reasoning_parser_factory.get_pooled(model) + } +} + +/// Create a fresh reasoning parser instance (for streaming where state isolation is needed) +pub fn create_reasoning_parser( + reasoning_parser_factory: &ReasoningParserFactory, + configured_parser: Option<&String>, + model: &str, +) -> Option> { + if let Some(parser_name) = configured_parser { + // Use configured parser if specified + reasoning_parser_factory + .registry() + .create_parser(parser_name) + .or_else(|| { + warn!( + "Configured reasoning parser '{}' not found, falling back to model-based selection", + parser_name + ); + reasoning_parser_factory.registry().create_for_model(model) + }) + } else { + // Auto-detect based on model + reasoning_parser_factory.registry().create_for_model(model) + } +} + +/// Get the appropriate tool parser for a model +/// +/// If a parser name is explicitly configured, use that parser. +/// Otherwise, auto-detect based on the model name. +/// Get a pooled tool parser (for non-streaming where state doesn't matter) +pub fn get_tool_parser( + tool_parser_factory: &ToolParserFactory, + configured_parser: Option<&String>, + model: &str, +) -> ToolPooledParser { + if let Some(parser_name) = configured_parser { + // Use configured parser if specified + tool_parser_factory + .registry() + .get_pooled_parser(parser_name) + .unwrap_or_else(|| { + warn!( + "Configured tool parser '{}' not found, falling back to model-based selection", + parser_name + ); + tool_parser_factory.get_pooled(model) + }) + } else { + // Auto-detect based on model + tool_parser_factory.get_pooled(model) + } +} + +/// Create a fresh tool parser instance (for streaming where state isolation is needed) +pub fn create_tool_parser( + tool_parser_factory: &ToolParserFactory, + configured_parser: Option<&String>, + model: &str, +) -> Option> { + if let Some(parser_name) = configured_parser { + // Use configured parser if specified + tool_parser_factory + .registry() + .create_parser(parser_name) + .or_else(|| { + warn!( + "Configured tool parser '{}' not found, falling back to model-based selection", + parser_name + ); + tool_parser_factory.registry().create_for_model(model) + }) + } else { + // Auto-detect based on model + tool_parser_factory.registry().create_for_model(model) + } +} + +/// Convert OutputLogProbs to OpenAI ChatLogProbs format +/// +/// This function decodes token IDs using the tokenizer and builds the logprobs structure +/// expected by the OpenAI API format. +pub fn convert_proto_to_openai_logprobs( + proto_logprobs: &OutputLogProbs, + tokenizer: &Arc, +) -> Result { + let mut content_items = Vec::new(); + + // Decode token IDs to text (always with skip_special_tokens=false for logprobs) + let token_texts: Vec = proto_logprobs + .token_ids + .iter() + .map(|&token_id| { + tokenizer + .decode(&[token_id as u32], false) + .unwrap_or_else(|_| format!("", token_id)) + }) + .collect(); + + // Build ChatLogProbsContent for each token (consume iterator to avoid clones) + for (i, (&logprob, token_text)) in proto_logprobs + .token_logprobs + .iter() + .zip(token_texts.into_iter()) + .enumerate() + { + let bytes = Some(token_text.as_bytes().to_vec()); + + // Build top_logprobs for this position + let mut top_logprobs = Vec::new(); + if let Some(top_logprobs_entry) = proto_logprobs.top_logprobs.get(i) { + // Decode top token IDs (always with skip_special_tokens=false) + let top_token_texts: Vec = top_logprobs_entry + .token_ids + .iter() + .map(|&tid| { + tokenizer + .decode(&[tid as u32], false) + .unwrap_or_else(|_| format!("", tid)) + }) + .collect(); + + for (j, (&top_logprob, &_top_token_id)) in top_logprobs_entry + .values + .iter() + .zip(top_logprobs_entry.token_ids.iter()) + .enumerate() + { + if let Some(top_token_text) = top_token_texts.get(j) { + top_logprobs.push(TopLogProb { + token: top_token_text.clone(), + logprob: top_logprob, + bytes: Some(top_token_text.as_bytes().to_vec()), + }); + } + } + } + + content_items.push(ChatLogProbsContent { + token: token_text, + logprob, + bytes, + top_logprobs, + }); + } + + Ok(ChatLogProbs::Detailed { + content: (!content_items.is_empty()).then_some(content_items), + }) +} + +/// Convert OutputLogProbs to Generate format Vec>> +/// +/// Generate format: [[logprob, token_id, ...], [logprob, token_id, ...], ...] +/// Each inner vec contains [logprob (f64), token_id (i32), ...] +pub fn convert_generate_output_logprobs(proto_logprobs: &OutputLogProbs) -> Vec>> { + proto_logprobs + .token_logprobs + .iter() + .zip(proto_logprobs.token_ids.iter()) + .map(|(&logprob, &token_id)| vec![Some(logprob as f64), Some(token_id as f64)]) + .collect() +} + +/// Convert InputLogProbs to Generate format Vec>> +/// +/// Generate format: [[logprob, token_id, ...], [logprob, token_id, ...], ...] +/// First token has null logprob: [[null, token_id], [logprob, token_id], ...] +pub fn convert_generate_input_logprobs(proto_logprobs: &InputLogProbs) -> Vec>> { + proto_logprobs + .token_logprobs + .iter() + .zip(proto_logprobs.token_ids.iter()) + .map(|(token_logprob, &token_id)| { + // InputTokenLogProb has optional value field + let logprob_value = token_logprob.value.map(|v| v as f64); + vec![logprob_value, Some(token_id as f64)] + }) + .collect() +} + +/// Parse finish_reason string into GenerateFinishReason enum +/// +/// Uses serde to deserialize the finish_reason, which handles all tagged variants automatically. +/// The GenerateFinishReason enum is tagged with `#[serde(tag = "type", rename_all = "lowercase")]`, +/// so it expects JSON objects like: +/// - `{"type":"stop"}` -> Stop +/// - `{"type":"length","length":100}` -> Length { length: 100 } +/// - Any other JSON -> Other(...) +/// +/// For backward compatibility, also handles simple string "stop" -> Stop +pub fn parse_finish_reason(reason_str: &str, completion_tokens: i32) -> GenerateFinishReason { + if reason_str == "stop" { + return GenerateFinishReason::Stop; + } + + if reason_str == "length" { + return GenerateFinishReason::Length { + length: completion_tokens.max(0) as u32, + }; + } + + match serde_json::from_str::(reason_str) { + Ok(finish_reason) => finish_reason, + Err(_) => match serde_json::from_str::(reason_str) { + Ok(json_value) => GenerateFinishReason::Other(json_value), + Err(_) => GenerateFinishReason::Other(Value::String(reason_str.to_string())), + }, + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + use crate::{ + protocols::{ + chat::{ChatMessage, MessageContent}, + common::{ContentPart, ImageUrl}, + }, + tokenizer::chat_template::ChatTemplateContentFormat, + }; + + #[test] + fn test_transform_messages_string_format() { + let messages = vec![ChatMessage::User { + content: MessageContent::Parts(vec![ + ContentPart::Text { + text: "Hello".to_string(), + }, + ContentPart::ImageUrl { + image_url: ImageUrl { + url: "https://example.com/image.jpg".to_string(), + detail: None, + }, + }, + ContentPart::Text { + text: "World".to_string(), + }, + ]), + name: None, + }]; + + let result = process_content_format(&messages, ChatTemplateContentFormat::String).unwrap(); + + assert_eq!(result.len(), 1); + let transformed_message = &result[0]; + + // Should flatten multimodal content to text only + assert_eq!( + transformed_message["content"].as_str().unwrap(), + "Hello World" + ); + assert_eq!(transformed_message["role"].as_str().unwrap(), "user"); + } + + #[test] + fn test_transform_messages_openai_format() { + let messages = vec![ChatMessage::User { + content: MessageContent::Parts(vec![ + ContentPart::Text { + text: "Describe this image:".to_string(), + }, + ContentPart::ImageUrl { + image_url: ImageUrl { + url: "https://example.com/image.jpg".to_string(), + detail: Some("high".to_string()), + }, + }, + ]), + name: None, + }]; + + let result = process_content_format(&messages, ChatTemplateContentFormat::OpenAI).unwrap(); + + assert_eq!(result.len(), 1); + let transformed_message = &result[0]; + + // Should replace media URLs with simple type placeholders + let content_array = transformed_message["content"].as_array().unwrap(); + assert_eq!(content_array.len(), 2); + + // Text part should remain unchanged + assert_eq!(content_array[0]["type"], "text"); + assert_eq!(content_array[0]["text"], "Describe this image:"); + + // Image part should be replaced with simple type placeholder + assert_eq!(content_array[1], json!({"type": "image"})); + } + + #[test] + fn test_transform_messages_simple_string_content() { + let messages = vec![ChatMessage::User { + content: MessageContent::Text("Simple text message".to_string()), + name: None, + }]; + + let result = process_content_format(&messages, ChatTemplateContentFormat::String).unwrap(); + + assert_eq!(result.len(), 1); + let transformed_message = &result[0]; + + // Simple string content should remain unchanged + assert_eq!( + transformed_message["content"].as_str().unwrap(), + "Simple text message" + ); + } + + #[test] + fn test_transform_messages_multiple_messages() { + let messages = vec![ + ChatMessage::System { + content: MessageContent::Text("System prompt".to_string()), + name: None, + }, + ChatMessage::User { + content: MessageContent::Parts(vec![ + ContentPart::Text { + text: "User message".to_string(), + }, + ContentPart::ImageUrl { + image_url: ImageUrl { + url: "https://example.com/image.jpg".to_string(), + detail: None, + }, + }, + ]), + name: None, + }, + ]; + + let result = process_content_format(&messages, ChatTemplateContentFormat::String).unwrap(); + + assert_eq!(result.len(), 2); + + // System message should remain unchanged + assert_eq!(result[0]["role"].as_str().unwrap(), "system"); + assert_eq!(result[0]["content"].as_str().unwrap(), "System prompt"); + + // User message should be flattened to text only + assert_eq!(result[1]["role"].as_str().unwrap(), "user"); + assert_eq!(result[1]["content"].as_str().unwrap(), "User message"); + } + + #[test] + fn test_transform_messages_empty_text_parts() { + let messages = vec![ChatMessage::User { + content: MessageContent::Parts(vec![ContentPart::ImageUrl { + image_url: ImageUrl { + url: "https://example.com/image.jpg".to_string(), + detail: None, + }, + }]), + name: None, + }]; + + let result = process_content_format(&messages, ChatTemplateContentFormat::String).unwrap(); + + assert_eq!(result.len(), 1); + let transformed_message = &result[0]; + + // Should keep original multimodal content when no text parts exist + assert!(transformed_message["content"].is_array()); + } + + #[test] + fn test_transform_messages_mixed_content_types() { + let messages = vec![ + ChatMessage::User { + content: MessageContent::Text("Plain text".to_string()), + name: None, + }, + ChatMessage::User { + content: MessageContent::Parts(vec![ + ContentPart::Text { + text: "With image".to_string(), + }, + ContentPart::ImageUrl { + image_url: ImageUrl { + url: "https://example.com/image.jpg".to_string(), + detail: Some("low".to_string()), + }, + }, + ]), + name: None, + }, + ]; + + let result_string = + process_content_format(&messages, ChatTemplateContentFormat::String).unwrap(); + + assert_eq!(result_string.len(), 2); + assert_eq!(result_string[0]["content"].as_str().unwrap(), "Plain text"); + assert_eq!(result_string[1]["content"].as_str().unwrap(), "With image"); + + let result_openai = + process_content_format(&messages, ChatTemplateContentFormat::OpenAI).unwrap(); + + assert_eq!(result_openai.len(), 2); + assert_eq!(result_openai[0]["content"].as_str().unwrap(), "Plain text"); + + let content_array = result_openai[1]["content"].as_array().unwrap(); + assert_eq!(content_array.len(), 2); + assert_eq!(content_array[0]["type"], "text"); + assert_eq!(content_array[1], json!({"type": "image"})); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/header_utils.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/header_utils.rs new file mode 100644 index 00000000..369d097e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/header_utils.rs @@ -0,0 +1,94 @@ +use axum::{body::Body, extract::Request, http::HeaderMap}; + +/// Copy request headers to a Vec of name-value string pairs +/// Used for forwarding headers to backend workers +pub fn copy_request_headers(req: &Request) -> Vec<(String, String)> { + req.headers() + .iter() + .filter_map(|(name, value)| { + // Convert header value to string, skipping non-UTF8 headers + value + .to_str() + .ok() + .map(|v| (name.to_string(), v.to_string())) + }) + .collect() +} + +/// Convert headers from reqwest Response to axum HeaderMap +/// Filters out hop-by-hop headers that shouldn't be forwarded +pub fn preserve_response_headers(reqwest_headers: &HeaderMap) -> HeaderMap { + let mut headers = HeaderMap::new(); + + for (name, value) in reqwest_headers.iter() { + // Skip hop-by-hop headers that shouldn't be forwarded + let name_str = name.as_str().to_lowercase(); + if should_forward_header(&name_str) { + // The original name and value are already valid, so we can just clone them + headers.insert(name.clone(), value.clone()); + } + } + + headers +} + +/// Determine if a header should be forwarded from backend to client +fn should_forward_header(name: &str) -> bool { + // List of headers that should NOT be forwarded (hop-by-hop headers) + !matches!( + name, + "connection" | + "keep-alive" | + "proxy-authenticate" | + "proxy-authorization" | + "te" | + "trailers" | + "transfer-encoding" | + "upgrade" | + "content-encoding" | // Let axum/hyper handle encoding + "host" // Should not forward the backend's host header + ) +} + +/// Apply headers to a reqwest request builder, filtering out headers that shouldn't be forwarded +/// or that will be set automatically by reqwest +pub fn apply_request_headers( + headers: &HeaderMap, + mut request_builder: reqwest::RequestBuilder, + skip_content_headers: bool, +) -> reqwest::RequestBuilder { + // Always forward Authorization header first if present + if let Some(auth) = headers + .get("authorization") + .or_else(|| headers.get("Authorization")) + { + request_builder = request_builder.header("Authorization", auth.clone()); + } + + // Forward other headers, filtering out problematic ones + for (key, value) in headers.iter() { + let key_str = key.as_str().to_lowercase(); + + // Skip headers that: + // - Are set automatically by reqwest (content-type, content-length for POST/PUT) + // - We already handled (authorization) + // - Are hop-by-hop headers (connection, transfer-encoding) + // - Should not be forwarded (host) + let should_skip = key_str == "authorization" || // Already handled above + key_str == "host" || + key_str == "connection" || + key_str == "transfer-encoding" || + key_str == "keep-alive" || + key_str == "te" || + key_str == "trailers" || + key_str == "accept-encoding" || + key_str == "upgrade" || + (skip_content_headers && (key_str == "content-type" || key_str == "content-length")); + + if !should_skip { + request_builder = request_builder.header(key.clone(), value.clone()); + } + } + + request_builder +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/http/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/http/mod.rs new file mode 100644 index 00000000..3f31b6f8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/http/mod.rs @@ -0,0 +1,5 @@ +//! HTTP router implementations + +pub mod pd_router; +pub mod pd_types; +pub mod router; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/http/pd_router.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/http/pd_router.rs new file mode 100644 index 00000000..eb4dc817 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/http/pd_router.rs @@ -0,0 +1,1406 @@ +use std::{sync::Arc, time::Instant}; + +use async_trait::async_trait; +use axum::{ + body::Body, + extract::Request, + http::{header::CONTENT_TYPE, HeaderMap, HeaderValue, StatusCode}, + response::{IntoResponse, Response}, +}; +use futures_util::StreamExt; +use reqwest::Client; +use serde::Serialize; +use serde_json::{json, Value}; +use tokio_stream::wrappers::UnboundedReceiverStream; +use tracing::{debug, error, warn}; + +use super::pd_types::api_path; +use crate::{ + config::types::RetryConfig, + core::{ + is_retryable_status, RetryExecutor, Worker, WorkerLoadGuard, WorkerRegistry, WorkerType, + }, + metrics::RouterMetrics, + policies::{LoadBalancingPolicy, PolicyRegistry}, + protocols::{ + chat::{ChatCompletionRequest, ChatMessage, MessageContent}, + classify::ClassifyRequest, + common::{InputIds, StringOrArray}, + completion::CompletionRequest, + embedding::EmbeddingRequest, + generate::GenerateRequest, + rerank::RerankRequest, + responses::{ResponsesGetParams, ResponsesRequest}, + }, + routers::{header_utils, RouterTrait}, +}; + +#[derive(Debug)] +pub struct PDRouter { + pub worker_registry: Arc, + pub policy_registry: Arc, + pub client: Client, + pub retry_config: RetryConfig, + pub api_key: Option, + pub enable_igw: bool, +} + +#[derive(Clone)] +struct PDRequestContext<'a> { + route: &'static str, + batch_size: Option, + is_stream: bool, + return_logprob: bool, + request_text: Option, + model_id: Option<&'a str>, +} + +impl PDRouter { + async fn proxy_to_first_prefill_worker( + &self, + endpoint: &str, + headers: Option>, + ) -> Response { + let workers = self.worker_registry.get_prefill_workers(); + let first_worker_url = workers.first().map(|w| w.url().to_string()); + + if let Some(worker_url) = first_worker_url { + self.proxy_to_worker(worker_url, endpoint, headers).await + } else { + ( + StatusCode::SERVICE_UNAVAILABLE, + "No prefill servers available".to_string(), + ) + .into_response() + } + } + + async fn proxy_to_worker( + &self, + worker_url: String, + endpoint: &str, + headers: Option>, + ) -> Response { + let url = format!("{}/{}", worker_url, endpoint); + let mut request_builder = self.client.get(&url); + + if let Some(headers) = headers { + for (name, value) in headers { + request_builder = request_builder.header(name, value); + } + } + + match request_builder.send().await { + Ok(res) if res.status().is_success() => { + let response_headers = header_utils::preserve_response_headers(res.headers()); + + match res.bytes().await { + Ok(body) => { + let mut response = Response::new(Body::from(body)); + *response.status_mut() = StatusCode::OK; + *response.headers_mut() = response_headers; + response + } + Err(e) => { + error!("Failed to read response body: {}", e); + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to read response body: {}", e), + ) + .into_response() + } + } + } + Ok(res) => { + let status = StatusCode::from_u16(res.status().as_u16()) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + (status, format!("{} server returned status: ", res.status())).into_response() + } + Err(e) => { + error!("Failed to proxy request server: {}", e); + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to proxy request: {}", e), + ) + .into_response() + } + } + } + + pub async fn new(ctx: &Arc) -> Result { + Ok(PDRouter { + worker_registry: Arc::clone(&ctx.worker_registry), + policy_registry: Arc::clone(&ctx.policy_registry), + client: ctx.client.clone(), + retry_config: ctx.router_config.effective_retry_config(), + api_key: ctx.router_config.api_key.clone(), + enable_igw: ctx.router_config.enable_igw, + }) + } + + fn handle_server_selection_error(error: String) -> Response { + error!("Failed to select PD pair error={}", error); + RouterMetrics::record_pd_error("server_selection"); + ( + StatusCode::SERVICE_UNAVAILABLE, + format!("No available servers: {}", error), + ) + .into_response() + } + + fn handle_serialization_error(error: impl std::fmt::Display) -> Response { + error!("Failed to serialize request error={}", error); + ( + StatusCode::INTERNAL_SERVER_ERROR, + "Failed to serialize request", + ) + .into_response() + } + + fn get_generate_batch_size(req: &GenerateRequest) -> Option { + // GenerateRequest doesn't support batch via arrays, only via input_ids + if let Some(InputIds::Batch(batches)) = &req.input_ids { + if !batches.is_empty() { + return Some(batches.len()); + } + } + None + } + + fn get_chat_batch_size(req: &ChatCompletionRequest) -> Option { + if let Some(n) = req.n { + if n > 1 { + return Some(n as usize); + } + } + None + } + + fn get_completion_batch_size(req: &CompletionRequest) -> Option { + if let StringOrArray::Array(arr) = &req.prompt { + if !arr.is_empty() { + return Some(arr.len()); + } + } + None + } + + fn inject_bootstrap_into_value( + mut original: Value, + prefill_worker: &dyn Worker, + batch_size: Option, + ) -> Result { + let obj = original + .as_object_mut() + .ok_or_else(|| "Request must be a JSON object".to_string())?; + + if let Some(n) = batch_size { + let mut hosts = Vec::with_capacity(n); + let mut ports = Vec::with_capacity(n); + let mut rooms = Vec::with_capacity(n); + for _ in 0..n { + hosts.push(prefill_worker.bootstrap_host()); + ports.push(prefill_worker.bootstrap_port()); + rooms.push(super::pd_types::generate_room_id()); + } + obj.insert( + "bootstrap_host".to_string(), + Value::Array(hosts.into_iter().map(Value::from).collect()), + ); + obj.insert( + "bootstrap_port".to_string(), + Value::Array( + ports + .into_iter() + .map(|p| match p { + Some(v) => Value::from(v), + None => Value::Null, + }) + .collect(), + ), + ); + obj.insert( + "bootstrap_room".to_string(), + Value::Array(rooms.into_iter().map(Value::from).collect()), + ); + } else { + obj.insert( + "bootstrap_host".to_string(), + Value::from(prefill_worker.bootstrap_host()), + ); + obj.insert( + "bootstrap_port".to_string(), + match prefill_worker.bootstrap_port() { + Some(v) => Value::from(v), + None => Value::Null, + }, + ); + obj.insert( + "bootstrap_room".to_string(), + Value::from(super::pd_types::generate_room_id()), + ); + } + Ok(original) + } + + async fn execute_dual_dispatch( + &self, + headers: Option<&HeaderMap>, + original_request: &T, + context: PDRequestContext<'_>, + ) -> Response { + let start_time = Instant::now(); + + let route = context.route; + RetryExecutor::execute_response_with_retry( + &self.retry_config, + { + let original_request = original_request.clone(); + move |attempt: u32| { + let original_request = original_request.clone(); + let context = context.clone(); + async move { + let (prefill, decode) = match self + .select_pd_pair(context.request_text.as_deref(), context.model_id) + .await + { + Ok(pair) => pair, + Err(e) => { + RouterMetrics::record_pd_error("server_selection"); + return Self::handle_server_selection_error(e); + } + }; + + debug!( + "PD retry attempt {} using prefill={} decode={}", + attempt, + prefill.url(), + decode.url() + ); + + let mut json_request = match serde_json::to_value(&original_request) { + Ok(v) => v, + Err(e) => return Self::handle_serialization_error(e), + }; + + json_request = match Self::inject_bootstrap_into_value( + json_request, + prefill.as_ref(), + context.batch_size, + ) { + Ok(v) => v, + Err(e) => return Self::handle_serialization_error(e), + }; + + let response = self + .execute_dual_dispatch_internal( + headers, + json_request, + context, + prefill.as_ref(), + decode.as_ref(), + start_time, + ) + .await; + + let _status = response.status(); + let not_error = _status.is_success() || _status.is_client_error(); + prefill.record_outcome(not_error); + decode.record_outcome(not_error); + + response + } + } + }, + |res, _attempt| is_retryable_status(res.status()), + |delay, attempt| { + RouterMetrics::record_retry(route); + RouterMetrics::record_retry_backoff_duration(delay, attempt); + }, + || RouterMetrics::record_retries_exhausted(route), + ) + .await + } + + async fn handle_decode_error_response( + &self, + res: reqwest::Response, + context: &PDRequestContext<'_>, + prefill: &dyn Worker, + decode: &dyn Worker, + ) -> Response { + let status = res.status(); + + if context.is_stream { + // Handle streaming error response + let response_headers = header_utils::preserve_response_headers(res.headers()); + let error_payload = match res.bytes().await { + Ok(error_body) => { + if let Ok(error_json) = serde_json::from_slice::(&error_body) { + json!({ "message": error_json, "status": status.as_u16() }) + } else { + json!({ "message": String::from_utf8_lossy(&error_body).to_string(), "status": status.as_u16() }) + } + } + Err(e) => { + json!({ "message": format!("Decode server error: {}", e), "status": status.as_u16() }) + } + }; + + let sse_data = format!( + "data: {{'error': {}}}", + serde_json::to_string(&error_payload).unwrap_or_default() + ); + let error_stream = tokio_stream::once(Ok(axum::body::Bytes::from(sse_data))); + + let decode_url = decode.url().to_string(); + self.create_streaming_response( + error_stream, + status, + None, + context.return_logprob, + Some(decode_url), + Some(response_headers), + prefill, + decode, + ) + } else { + // Handle non-streaming error response + match res.bytes().await { + Ok(error_body) => (status, error_body).into_response(), + Err(e) => (status, format!("Decode server error: {}", e)).into_response(), + } + } + } + + // Internal method that performs the actual dual dispatch (without retry logic) + async fn execute_dual_dispatch_internal( + &self, + headers: Option<&HeaderMap>, + json_request: Value, + context: PDRequestContext<'_>, + prefill: &dyn Worker, + decode: &dyn Worker, + start_time: Instant, + ) -> Response { + // For non-streaming: use guard for automatic load management + // For streaming: load will be managed in create_streaming_response + let _guard = if !context.is_stream { + Some(WorkerLoadGuard::new_multi(vec![prefill, decode])) + } else { + None + }; + + // Build both requests + let prefill_request = self.build_post_with_headers( + &self.client, + prefill.url(), + context.route, + &json_request, + headers, + false, + ); + let decode_request = self.build_post_with_headers( + &self.client, + decode.url(), + context.route, + &json_request, + headers, + false, + ); + + // Send both requests concurrently and wait for both + debug!( + "Sending concurrent requests to prefill={} decode={}", + prefill.url(), + decode.url() + ); + + let (prefill_result, decode_result) = + tokio::join!(prefill_request.send(), decode_request.send()); + debug!("Received responses from both servers"); + + let duration = start_time.elapsed(); + RouterMetrics::record_pd_request_duration(context.route, duration); + RouterMetrics::record_pd_request(context.route); + RouterMetrics::record_pd_prefill_request(prefill.url()); + RouterMetrics::record_pd_decode_request(decode.url()); + + // Process decode response + match decode_result { + Ok(res) => { + let status = StatusCode::from_u16(res.status().as_u16()) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + debug!("Decode response status: {}", status); + + if !status.is_success() { + RouterMetrics::record_pd_decode_error(decode.url()); + error!( + "Decode server returned error status decode_url={} status={}", + decode.url(), + status + ); + + return self + .handle_decode_error_response(res, &context, prefill, decode) + .await; + } + + // Process prefill response + let prefill_body = if context.return_logprob { + match self + .process_prefill_response( + prefill_result, + prefill.url(), + context.return_logprob, + ) + .await + { + Ok((_, body)) => body, + Err(error_response) => return error_response, + } + } else { + // Even if we don't need logprobs, we should check prefill status + match self + .process_prefill_response(prefill_result, prefill.url(), false) + .await + { + Ok((_, body)) => body, + Err(error_response) => return error_response, + } + }; + + if context.is_stream { + // Streaming response + let prefill_logprobs = if context.return_logprob { + prefill_body + .as_ref() + .and_then(|body| serde_json::from_slice::(body).ok()) + .and_then(|json| { + json.pointer("/meta_info/input_token_logprobs").cloned() + }) + } else { + None + }; + + let response_headers = header_utils::preserve_response_headers(res.headers()); + + self.create_streaming_response( + res.bytes_stream(), + status, + prefill_logprobs, + context.return_logprob, + None, + Some(response_headers), + prefill, + decode, + ) + } else { + // Non-streaming response + if context.return_logprob { + self.process_non_streaming_response( + res, + status, + context.return_logprob, + prefill_body, + ) + .await + } else { + // Direct passthrough when no logprobs needed + let response_headers = + header_utils::preserve_response_headers(res.headers()); + + match res.bytes().await { + Ok(decode_body) => { + let mut response = Response::new(Body::from(decode_body)); + *response.status_mut() = status; + *response.headers_mut() = response_headers; + response + } + Err(e) => { + error!("Failed to read decode response: {}", e); + (StatusCode::INTERNAL_SERVER_ERROR, "Failed to read response") + .into_response() + } + } + } + } + } + Err(e) => { + error!( + decode_url = %decode.url(), + error = %e, + "Decode request failed" + ); + RouterMetrics::record_pd_decode_error(decode.url()); + ( + StatusCode::BAD_GATEWAY, + format!("Decode server error: {}", e), + ) + .into_response() + } + } + } + + fn policies_need_request_text(&self) -> bool { + let prefill_policy = self.policy_registry.get_prefill_policy(); + let decode_policy = self.policy_registry.get_decode_policy(); + prefill_policy.needs_request_text() || decode_policy.needs_request_text() + } + + async fn select_pd_pair( + &self, + request_text: Option<&str>, + model_id: Option<&str>, + ) -> Result<(Arc, Arc), String> { + let effective_model_id = if !self.enable_igw { None } else { model_id }; + + debug!( + "Selecting PD pair: enable_igw={}, model_id={:?}, effective_model_id={:?}", + self.enable_igw, model_id, effective_model_id + ); + + let prefill_workers = if let Some(model) = effective_model_id { + self.worker_registry + .get_by_model_fast(model) + .into_iter() + .filter(|w| matches!(w.worker_type(), WorkerType::Prefill { .. })) + .collect() + } else { + self.worker_registry.get_prefill_workers() + }; + + let decode_workers = if let Some(model) = effective_model_id { + self.worker_registry + .get_by_model_fast(model) + .into_iter() + .filter(|w| matches!(w.worker_type(), WorkerType::Decode)) + .collect() + } else { + self.worker_registry.get_decode_workers() + }; + + let prefill_policy = self.policy_registry.get_prefill_policy(); + let decode_policy = self.policy_registry.get_decode_policy(); + + let prefill = Self::pick_worker_by_policy_arc( + &prefill_workers, + &*prefill_policy, + request_text, + "prefill", + )?; + + let decode = Self::pick_worker_by_policy_arc( + &decode_workers, + &*decode_policy, + request_text, + "decode", + )?; + + Ok((prefill, decode)) + } + + fn pick_worker_by_policy_arc( + workers: &[Arc], + policy: &dyn LoadBalancingPolicy, + request_text: Option<&str>, + worker_type: &str, + ) -> Result, String> { + if workers.is_empty() { + return Err(format!( + "No {} workers available. Please check if {} servers are configured and healthy.", + worker_type, worker_type + )); + } + + let available_workers: Vec> = workers + .iter() + .filter(|w| w.is_available()) + .cloned() + .collect(); + + if available_workers.is_empty() { + return Err(format!( + "No available {} workers (all circuits open or unhealthy)", + worker_type + )); + } + + let selected_idx = policy + .select_worker(&available_workers, request_text) + .ok_or_else(|| { + format!( + "Policy {} failed to select a {} worker", + policy.name(), + worker_type + ) + })?; + + Ok(available_workers[selected_idx].clone()) + } + + #[allow(clippy::too_many_arguments)] + fn create_streaming_response( + &self, + stream: impl futures_util::Stream> + Send + 'static, + status: StatusCode, + prefill_logprobs: Option, + return_logprob: bool, + decode_url: Option, + headers: Option, + prefill: &dyn Worker, + decode: &dyn Worker, + ) -> Response { + prefill.increment_load(); + decode.increment_load(); + + let prefill_url = prefill.url().to_string(); + let decode_url_str = decode.url().to_string(); + + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + + let registry = self.worker_registry.clone(); + + tokio::spawn(async move { + let mut stream_completed = false; + + futures_util::pin_mut!(stream); + while let Some(chunk_result) = stream.next().await { + match chunk_result { + Ok(chunk) => { + let is_done = chunk + .as_ref() + .windows(12) + .any(|window| window == b"data: [DONE]"); + + let result = if return_logprob && prefill_logprobs.is_some() { + Self::merge_streaming_logprobs(prefill_logprobs.clone(), &chunk) + .unwrap_or(chunk) + } else { + chunk + }; + + if tx.send(Ok(result)).is_err() { + break; + } + + if is_done { + stream_completed = true; + break; + } + } + Err(e) => { + if let Some(ref url) = decode_url { + error!("Stream error from decode server {}: {}", url, e); + RouterMetrics::record_pd_stream_error(url); + } + let _ = tx.send(Err(format!("Stream error: {}", e))); + break; + } + } + } + + if let Some(worker) = registry.get_by_url(&prefill_url) { + worker.decrement_load(); + debug!( + "Decremented load for prefill worker: {} (stream_completed: {})", + prefill_url, stream_completed + ); + } + + if let Some(worker) = registry.get_by_url(&decode_url_str) { + worker.decrement_load(); + debug!( + "Decremented load for decode worker: {} (stream_completed: {})", + decode_url_str, stream_completed + ); + } + }); + + let stream = UnboundedReceiverStream::new(rx); + let body = Body::from_stream(stream); + + let mut response = Response::new(body); + *response.status_mut() = status; + + let mut headers = headers.unwrap_or_default(); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("text/event-stream")); + *response.headers_mut() = headers; + + response + } + + // Helper to process non-streaming decode response with logprob merging + async fn process_non_streaming_response( + &self, + res: reqwest::Response, + status: StatusCode, + return_logprob: bool, + prefill_body: Option, + ) -> Response { + let response = res.bytes().await; + let decode_body = match response { + Ok(decode_body) => decode_body, + Err(e) => { + error!("Failed to read decode response: {}", e); + return (StatusCode::INTERNAL_SERVER_ERROR, "Failed to read response") + .into_response(); + } + }; + + if !return_logprob { + return (status, decode_body).into_response(); + } + + let Some(prefill_body) = prefill_body else { + return (status, decode_body).into_response(); + }; + + // Merge logprobs from prefill and decode + let (Ok(prefill_json), Ok(mut decode_json)) = ( + serde_json::from_slice::(&prefill_body), + serde_json::from_slice::(&decode_body), + ) else { + warn!("Failed to parse responses for logprob merging"); + return (status, decode_body).into_response(); + }; + + Self::merge_logprobs_in_json(&prefill_json, &mut decode_json); + + // Return merged response + match serde_json::to_vec(&decode_json) { + Ok(body) => (status, body).into_response(), + Err(e) => { + error!("Failed to serialize merged response: {}", e); + (status, decode_body).into_response() + } + } + } + + // Helper to process prefill response and extract body if needed for logprobs + async fn process_prefill_response( + &self, + prefill_result: Result, + prefill_url: &str, + return_logprob: bool, + ) -> Result<(StatusCode, Option), Response> { + // Check prefill result first - it's critical for disaggregated mode + let prefill_response = match prefill_result { + Ok(response) => response, + Err(e) => { + RouterMetrics::record_pd_prefill_error(prefill_url); + error!( + "Prefill server failed (CRITICAL) prefill_url={} error={}. Decode will timeout without prefill KV cache.", + prefill_url, + e + ); + + // Return error immediately - don't wait for decode to timeout + return Err(( + StatusCode::BAD_GATEWAY, + format!( + "Prefill server error: {}. This will cause decode timeout.", + e + ), + ) + .into_response()); + } + }; + + let prefill_status = StatusCode::from_u16(prefill_response.status().as_u16()) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + + // Check if prefill succeeded + if !prefill_status.is_success() { + RouterMetrics::record_pd_prefill_error(prefill_url); + + // Get error body from prefill + let error_msg = prefill_response + .text() + .await + .unwrap_or_else(|_| "Unknown prefill error".to_string()); + + error!( + "Prefill server returned error status prefill_url={} status={} body={}", + prefill_url, prefill_status, error_msg + ); + + return Err(( + prefill_status, + format!("Prefill server error ({}): {}", prefill_status, error_msg), + ) + .into_response()); + } + + // Read prefill body if needed for logprob merging + let prefill_body = if return_logprob { + match prefill_response.bytes().await { + Ok(body) => Some(body), + Err(e) => { + warn!("Failed to read prefill response body for logprobs: {}", e); + None + } + } + } else { + // For non-logprob requests, just consume the response without storing + debug!("Consuming prefill response body (non-logprob request)"); + match prefill_response.bytes().await { + Ok(_) => debug!("Prefill response consumed successfully"), + Err(e) => warn!("Error consuming prefill response: {}", e), + } + None + }; + + Ok((prefill_status, prefill_body)) + } + + fn build_post_with_headers( + &self, + client: &Client, + url: &str, + route: &str, + json_request: &Value, + headers: Option<&HeaderMap>, + connection_close: bool, + ) -> reqwest::RequestBuilder { + let mut request = client.post(api_path(url, route)).json(json_request); + if connection_close { + request = request.header("Connection", "close"); + } + if let Some(headers) = headers { + for (name, value) in headers.iter() { + let name_lc = name.as_str().to_ascii_lowercase(); + // Whitelist important end-to-end headers, skip hop-by-hop + let forward = matches!( + name_lc.as_str(), + "authorization" | "x-request-id" | "x-correlation-id" + ) || name_lc.starts_with("x-request-id-"); + if forward { + if let Ok(val) = value.to_str() { + request = request.header(name, val); + } + } + } + } + request + } + + // Helper to merge logprobs from prefill and decode responses + fn merge_logprobs_in_json(prefill_json: &Value, decode_json: &mut Value) -> bool { + if let (Some(prefill_meta), Some(decode_meta)) = ( + prefill_json.get("meta_info"), + decode_json.get_mut("meta_info"), + ) { + if let (Some(prefill_logprobs), Some(decode_logprobs)) = ( + prefill_meta.get("input_token_logprobs"), + decode_meta.get_mut("input_token_logprobs"), + ) { + if let (Some(prefill_arr), Some(decode_arr)) = + (prefill_logprobs.as_array(), decode_logprobs.as_array_mut()) + { + let mut merged = prefill_arr.clone(); + merged.extend(decode_arr.clone()); + decode_meta["input_token_logprobs"] = Value::Array(merged); + return true; + } + } + } + false + } + + // Simple helper to merge logprobs in streaming responses + fn merge_streaming_logprobs( + prefill_logprobs: Option, + decode_chunk: &[u8], + ) -> Result { + // Skip non-data chunks + let chunk_str = std::str::from_utf8(decode_chunk).map_err(|_| ())?; + if !chunk_str.starts_with("data: ") || chunk_str.contains("[DONE]") { + return Err(()); + } + + // Parse JSON from chunk + let json_str = chunk_str.trim_start_matches("data: ").trim(); + let mut decode_json: Value = serde_json::from_str(json_str).map_err(|_| ())?; + + // Merge prefill logprobs if available + if let Some(ref p_logprobs) = prefill_logprobs { + if let Some(meta) = decode_json.get_mut("meta_info") { + if let Some(d_logprobs) = meta.get_mut("input_token_logprobs") { + if let (Some(p_arr), Some(d_arr)) = + (p_logprobs.as_array(), d_logprobs.as_array()) + { + let mut merged = p_arr.clone(); + merged.extend(d_arr.clone()); + *d_logprobs = Value::Array(merged); + } + } + } + } + + // Re-serialize + let merged_str = format!( + "data: {}\n\n", + serde_json::to_string(&decode_json).unwrap_or_default() + ); + Ok(bytes::Bytes::from(merged_str)) + } +} + +#[async_trait] +impl RouterTrait for PDRouter { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + async fn health_generate(&self, _req: Request) -> Response { + // Note: This endpoint actually causes the model to generate tokens, so we only test one pair + + // Select a random worker pair using the policy + let (prefill, decode) = match self.select_pd_pair(None, None).await { + Ok(pair) => pair, + Err(e) => { + return ( + StatusCode::SERVICE_UNAVAILABLE, + format!("No healthy worker pair available: {}", e), + ) + .into_response(); + } + }; + + let prefill_url = format!("{}/health_generate", prefill.url()); + let (prefill_result, decode_result) = tokio::join!( + self.client.get(&prefill_url).send(), + self.client + .get(format!("{}/health_generate", decode.url())) + .send() + ); + + // Check results + let mut errors = Vec::new(); + + match prefill_result { + Ok(res) if res.status().is_success() => { + debug!( + "Health generate passed for prefill server: {}", + prefill.url() + ); + } + Ok(res) => { + errors.push(format!( + "Prefill {} returned status {}", + prefill.url(), + res.status() + )); + } + Err(e) => { + errors.push(format!("Prefill {} error: {}", prefill.url(), e)); + } + } + + match decode_result { + Ok(res) if res.status().is_success() => { + debug!("Health generate passed for decode server: {}", decode.url()); + } + Ok(res) => { + errors.push(format!( + "Decode {} returned status {}", + decode.url(), + res.status() + )); + } + Err(e) => { + errors.push(format!("Decode {} error: {}", decode.url(), e)); + } + } + + if errors.is_empty() { + ( + StatusCode::OK, + format!( + "Health generate passed on selected pair: prefill={}, decode={}", + prefill.url(), + decode.url() + ), + ) + .into_response() + } else { + ( + StatusCode::SERVICE_UNAVAILABLE, + format!("Health generate failed: {:?}", errors), + ) + .into_response() + } + } + + async fn get_server_info(&self, _req: Request) -> Response { + // Get info from the first decode server to match sglang's server info format + // Note: We use decode workers for server info to match expected format + self.proxy_to_first_prefill_worker("get_server_info", None) + .await + } + + async fn get_models(&self, req: Request) -> Response { + // Extract headers first to avoid Send issues + let headers = header_utils::copy_request_headers(&req); + + // Proxy to first prefill worker + self.proxy_to_first_prefill_worker("v1/models", Some(headers)) + .await + } + + async fn get_model_info(&self, req: Request) -> Response { + // Extract headers first to avoid Send issues + let headers = header_utils::copy_request_headers(&req); + + // Proxy to first prefill worker + self.proxy_to_first_prefill_worker("get_model_info", Some(headers)) + .await + } + + async fn route_generate( + &self, + headers: Option<&HeaderMap>, + body: &GenerateRequest, + model_id: Option<&str>, + ) -> Response { + let is_stream = body.stream; + let return_logprob = body.return_logprob.unwrap_or(false); + + let request_text = if self.policies_need_request_text() { + body.text.as_deref().map(|s| s.to_string()) + } else { + None + }; + + let batch_size = Self::get_generate_batch_size(body); + + let context = PDRequestContext { + route: "/generate", + batch_size, + is_stream, + return_logprob, + request_text, + model_id, + }; + + self.execute_dual_dispatch(headers, body, context).await + } + + async fn route_chat( + &self, + headers: Option<&HeaderMap>, + body: &ChatCompletionRequest, + model_id: Option<&str>, + ) -> Response { + let is_stream = body.stream; + let return_logprob = body.logprobs; + + let request_text = if self.policies_need_request_text() { + body.messages.first().and_then(|msg| match msg { + ChatMessage::User { content, .. } => match content { + MessageContent::Text(text) => Some(text.clone()), + MessageContent::Parts(_) => None, + }, + ChatMessage::System { content, .. } => Some(content.to_simple_string()), + _ => None, + }) + } else { + None + }; + + // Calculate batch size + let batch_size = Self::get_chat_batch_size(body); + + let context = PDRequestContext { + route: "/v1/chat/completions", + batch_size, + is_stream, + return_logprob, + request_text, + model_id, + }; + + self.execute_dual_dispatch(headers, body, context).await + } + + async fn route_completion( + &self, + headers: Option<&HeaderMap>, + body: &CompletionRequest, + model_id: Option<&str>, + ) -> Response { + let is_stream = body.stream; + let return_logprob = body.logprobs.is_some(); + + let request_text = if self.policies_need_request_text() { + match &body.prompt { + StringOrArray::String(s) => Some(s.clone()), + StringOrArray::Array(v) => v.first().map(|s| s.to_string()), + } + } else { + None + }; + + // Calculate batch size + let batch_size = Self::get_completion_batch_size(body); + + let context = PDRequestContext { + route: "/v1/completions", + batch_size, + is_stream, + return_logprob, + request_text, + model_id, + }; + + self.execute_dual_dispatch(headers, body, context).await + } + + async fn route_responses( + &self, + _headers: Option<&HeaderMap>, + _body: &ResponsesRequest, + _model_id: Option<&str>, + ) -> Response { + ( + StatusCode::NOT_IMPLEMENTED, + "Responses endpoint not implemented for PD router", + ) + .into_response() + } + + async fn get_response( + &self, + _headers: Option<&HeaderMap>, + _response_id: &str, + _params: &ResponsesGetParams, + ) -> Response { + ( + StatusCode::NOT_IMPLEMENTED, + "Responses retrieve endpoint not implemented for PD router", + ) + .into_response() + } + + async fn cancel_response(&self, _headers: Option<&HeaderMap>, _response_id: &str) -> Response { + ( + StatusCode::NOT_IMPLEMENTED, + "Responses cancel endpoint not implemented for PD router", + ) + .into_response() + } + + async fn route_classify( + &self, + _headers: Option<&HeaderMap>, + _body: &ClassifyRequest, + _model_id: Option<&str>, + ) -> Response { + ( + StatusCode::NOT_IMPLEMENTED, + "Classify endpoint not implemented for PD router", + ) + .into_response() + } + + async fn route_embeddings( + &self, + _headers: Option<&HeaderMap>, + _body: &EmbeddingRequest, + _model_id: Option<&str>, + ) -> Response { + ( + StatusCode::NOT_IMPLEMENTED, + "Embeddings endpoint not implemented for PD router", + ) + .into_response() + } + + async fn route_rerank( + &self, + headers: Option<&HeaderMap>, + body: &RerankRequest, + model_id: Option<&str>, + ) -> Response { + // Extract text for cache-aware routing + let req_text = if self.policies_need_request_text() { + Some(body.query.clone()) + } else { + None + }; + + let context = PDRequestContext { + route: "/v1/rerank", + batch_size: None, + is_stream: false, + return_logprob: false, + request_text: req_text, + model_id, + }; + + self.execute_dual_dispatch(headers, body, context).await + } + + fn router_type(&self) -> &'static str { + "pd" + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::{BasicWorkerBuilder, WorkerType}; + + fn create_test_pd_router() -> PDRouter { + let worker_registry = Arc::new(WorkerRegistry::new()); + let policy_registry = + Arc::new(PolicyRegistry::new(crate::config::PolicyConfig::RoundRobin)); + + PDRouter { + worker_registry, + policy_registry, + client: Client::new(), + retry_config: RetryConfig::default(), + api_key: Some("test_api_key".to_string()), + enable_igw: false, + } + } + + fn create_test_worker(url: String, worker_type: WorkerType, healthy: bool) -> Box { + let worker = BasicWorkerBuilder::new(url) + .worker_type(worker_type) + .build(); + worker.set_healthy(healthy); + Box::new(worker) + } + + #[tokio::test] + async fn test_select_healthy_prefill_worker() { + let router = create_test_pd_router(); + + let healthy_worker = create_test_worker( + "http://healthy".to_string(), + WorkerType::Prefill { + bootstrap_port: None, + }, + true, + ); + let unhealthy_worker = create_test_worker( + "http://unhealthy".to_string(), + WorkerType::Prefill { + bootstrap_port: None, + }, + false, + ); + let decode_worker = + create_test_worker("http://decode".to_string(), WorkerType::Decode, true); + + router.worker_registry.register(Arc::from(unhealthy_worker)); + router.worker_registry.register(Arc::from(healthy_worker)); + router.worker_registry.register(Arc::from(decode_worker)); + + let result = router.select_pd_pair(None, None).await; + + assert!(result.is_ok()); + let (prefill, _decode) = result.unwrap(); + + assert_eq!(prefill.url(), "http://healthy"); + assert!(prefill.is_healthy()); + } + + #[tokio::test] + async fn test_empty_worker_lists() { + let router = create_test_pd_router(); + + let result = router.select_pd_pair(None, None).await; + + assert!(result.is_err()); + assert!(result.unwrap_err().contains("No prefill workers available")); + } + + #[test] + fn test_worker_load_metrics() { + let prefill_worker = create_test_worker( + "http://prefill".to_string(), + WorkerType::Prefill { + bootstrap_port: None, + }, + true, + ); + let decode_worker = + create_test_worker("http://decode".to_string(), WorkerType::Decode, true); + + let _guard = + WorkerLoadGuard::new_multi(vec![prefill_worker.as_ref(), decode_worker.as_ref()]); + + assert_eq!(prefill_worker.load(), 1); + assert_eq!(decode_worker.load(), 1); + + drop(_guard); + + assert_eq!(prefill_worker.load(), 0); + assert_eq!(decode_worker.load(), 0); + } + + #[tokio::test] + async fn test_streaming_load_tracking() { + use futures_util::StreamExt; + use tokio::time::{sleep, Duration}; + + let router = create_test_pd_router(); + + let prefill_worker = create_test_worker( + "http://prefill".to_string(), + WorkerType::Prefill { + bootstrap_port: None, + }, + true, + ); + let decode_worker = + create_test_worker("http://decode".to_string(), WorkerType::Decode, true); + + router.worker_registry.register(Arc::from(prefill_worker)); + router.worker_registry.register(Arc::from(decode_worker)); + + let prefill_workers = router.worker_registry.get_prefill_workers(); + let decode_workers = router.worker_registry.get_decode_workers(); + + let prefill_ref = prefill_workers[0].clone(); + let decode_ref = decode_workers[0].clone(); + + assert_eq!(prefill_ref.load(), 0); + assert_eq!(decode_ref.load(), 0); + + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let stream = UnboundedReceiverStream::new(rx); + + let _response = router.create_streaming_response( + stream.map(Ok), + StatusCode::OK, + None, + false, + None, + None, + prefill_ref.as_ref(), + decode_ref.as_ref(), + ); + + assert_eq!(prefill_ref.load(), 1); + assert_eq!(decode_ref.load(), 1); + + tx.send(bytes::Bytes::from("test data")).unwrap(); + + sleep(Duration::from_millis(10)).await; + + assert_eq!(prefill_ref.load(), 1); + assert_eq!(decode_ref.load(), 1); + + drop(tx); + + sleep(Duration::from_millis(100)).await; + + assert_eq!(prefill_ref.load(), 0); + assert_eq!(decode_ref.load(), 0); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/http/pd_types.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/http/pd_types.rs new file mode 100644 index 00000000..9cb6ad9e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/http/pd_types.rs @@ -0,0 +1,78 @@ +// Custom error type for PD router operations +#[derive(Debug, thiserror::Error)] +pub enum PDRouterError { + #[error("Worker already exists: {url}")] + WorkerAlreadyExists { url: String }, + + #[error("Worker not found: {url}")] + WorkerNotFound { url: String }, + + #[error("Lock acquisition failed: {operation}")] + LockError { operation: String }, + + #[error("Health check failed for worker: {url}")] + HealthCheckFailed { url: String }, + + #[error("Invalid worker configuration: {reason}")] + InvalidConfiguration { reason: String }, + + #[error("Network error: {message}")] + NetworkError { message: String }, + + #[error("Timeout waiting for worker: {url}")] + Timeout { url: String }, +} + +// Helper functions for workers +pub fn api_path(url: &str, api_path: &str) -> String { + if api_path.starts_with("/") { + format!("{}{}", url, api_path) + } else { + format!("{}/{}", url, api_path) + } +} + +use serde::Serialize; + +// Optimized bootstrap wrapper for single requests +#[derive(Serialize)] +pub struct RequestWithBootstrap<'a, T: Serialize> { + #[serde(flatten)] + pub original: &'a T, + pub bootstrap_host: String, + pub bootstrap_port: Option, + pub bootstrap_room: u64, +} + +// Optimized bootstrap wrapper for batch requests +#[derive(Serialize)] +pub struct BatchRequestWithBootstrap<'a, T: Serialize> { + #[serde(flatten)] + pub original: &'a T, + pub bootstrap_host: Vec, + pub bootstrap_port: Vec>, + pub bootstrap_room: Vec, +} + +// Helper to generate bootstrap room ID +pub fn generate_room_id() -> u64 { + // Generate a value in the range [0, 2^63 - 1] to match Python's random.randint(0, 2**63 - 1) + rand::random::() & (i64::MAX as u64) +} + +// PD-specific routing policies +#[derive(Debug, Clone, PartialEq)] +pub enum PDSelectionPolicy { + Random, + PowerOfTwo, + CacheAware { + cache_threshold: f32, + balance_abs_threshold: usize, + balance_rel_threshold: f32, + }, + Bucket { + balance_abs_threshold: usize, + balance_rel_threshold: f32, + bucket_adjust_interval_secs: usize, + }, +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/http/router.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/http/router.rs new file mode 100644 index 00000000..e9f4dba4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/http/router.rs @@ -0,0 +1,880 @@ +use std::{sync::Arc, time::Instant}; + +use axum::{ + body::{to_bytes, Body}, + extract::Request, + http::{ + header::{CONTENT_LENGTH, CONTENT_TYPE}, + HeaderMap, HeaderValue, Method, StatusCode, + }, + response::{IntoResponse, Response}, + Json, +}; +use futures_util::StreamExt; +use reqwest::Client; +use tokio_stream::wrappers::UnboundedReceiverStream; +use tracing::{debug, error}; + +use crate::{ + config::types::RetryConfig, + core::{ + is_retryable_status, ConnectionMode, RetryExecutor, Worker, WorkerRegistry, WorkerType, + }, + metrics::RouterMetrics, + policies::PolicyRegistry, + protocols::{ + chat::ChatCompletionRequest, + classify::ClassifyRequest, + common::GenerationRequest, + completion::CompletionRequest, + embedding::EmbeddingRequest, + generate::GenerateRequest, + rerank::{RerankRequest, RerankResponse, RerankResult}, + responses::{ResponsesGetParams, ResponsesRequest}, + }, + routers::{header_utils, RouterTrait}, +}; + +/// Regular router that uses injected load balancing policies +#[derive(Debug)] +pub struct Router { + worker_registry: Arc, + policy_registry: Arc, + client: Client, + dp_aware: bool, + enable_igw: bool, + retry_config: RetryConfig, +} + +impl Router { + /// Create a new router with injected policy and client + pub async fn new(ctx: &Arc) -> Result { + let workers = ctx.worker_registry.get_workers_filtered( + None, // any model + Some(WorkerType::Regular), + Some(ConnectionMode::Http), + false, // include all workers + ); + + RouterMetrics::set_active_workers(workers.len()); + + Ok(Router { + worker_registry: ctx.worker_registry.clone(), + policy_registry: ctx.policy_registry.clone(), + client: ctx.client.clone(), + dp_aware: ctx.router_config.dp_aware, + enable_igw: ctx.router_config.enable_igw, + retry_config: ctx.router_config.effective_retry_config(), + }) + } + + fn select_first_worker(&self) -> Result { + let workers = self.worker_registry.get_all(); + let healthy_workers: Vec<_> = workers.iter().filter(|w| w.is_healthy()).collect(); + if healthy_workers.is_empty() { + Err("No workers are available".to_string()) + } else { + Ok(healthy_workers[0].url().to_string()) + } + } + + // Helper method to proxy GET requests to the first available worker + async fn proxy_get_request(&self, req: Request, endpoint: &str) -> Response { + let headers = header_utils::copy_request_headers(&req); + + match self.select_first_worker() { + Ok(worker_url) => { + let mut request_builder = self.client.get(format!("{}/{}", worker_url, endpoint)); + for (name, value) in headers { + let name_lc = name.to_lowercase(); + if name_lc != "content-type" && name_lc != "content-length" { + request_builder = request_builder.header(name, value); + } + } + + match request_builder.send().await { + Ok(res) => { + let status = StatusCode::from_u16(res.status().as_u16()) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + + // Preserve headers from backend + let response_headers = + header_utils::preserve_response_headers(res.headers()); + + match res.bytes().await { + Ok(body) => { + let mut response = Response::new(Body::from(body)); + *response.status_mut() = status; + *response.headers_mut() = response_headers; + response + } + Err(e) => ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to read response: {}", e), + ) + .into_response(), + } + } + Err(e) => ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Request failed: {}", e), + ) + .into_response(), + } + } + Err(e) => (StatusCode::SERVICE_UNAVAILABLE, e).into_response(), + } + } + + /// Select worker for a specific model considering circuit breaker state + fn select_worker_for_model( + &self, + model_id: Option<&str>, + text: Option<&str>, + ) -> Option> { + let effective_model_id = if !self.enable_igw { None } else { model_id }; + + // Get workers for the specified model O(1), filtered by connection mode + let workers = self.worker_registry.get_workers_filtered( + effective_model_id, + Some(WorkerType::Regular), + Some(ConnectionMode::Http), + false, // get all workers, we'll filter by is_available() next + ); + + let available: Vec> = workers + .iter() + .filter(|w| w.is_available()) + .cloned() + .collect(); + if available.is_empty() { + return None; + } + + // Get the appropriate policy for this model + let policy = match model_id { + Some(model) => self.policy_registry.get_policy_or_default(model), + None => self.policy_registry.get_default_policy(), + }; + + let idx = policy.select_worker(&available, text)?; + Some(available[idx].clone()) + } + + pub async fn route_typed_request( + &self, + headers: Option<&HeaderMap>, + typed_req: &T, + route: &str, + model_id: Option<&str>, + ) -> Response { + let start = Instant::now(); + let is_stream = typed_req.is_stream(); + let text = typed_req.extract_text_for_routing(); + + let response = RetryExecutor::execute_response_with_retry( + &self.retry_config, + // operation per attempt + |_: u32| async { + let worker = match self.select_worker_for_model(model_id, Some(&text)) { + Some(w) => w, + None => { + RouterMetrics::record_request_error(route, "no_available_workers"); + return ( + StatusCode::SERVICE_UNAVAILABLE, + "No available workers (all circuits open or unhealthy)", + ) + .into_response(); + } + }; + + // Optional load tracking for cache-aware policy + // Get the policy for this model to check if it's cache-aware + let policy = match model_id { + Some(model) => self.policy_registry.get_policy_or_default(model), + None => self.policy_registry.get_default_policy(), + }; + + let load_incremented = if policy.name() == "cache_aware" { + worker.increment_load(); + RouterMetrics::set_running_requests(worker.url(), worker.load()); + true + } else { + false + }; + + // Keep a clone for potential cleanup on retry + let worker_for_cleanup = if load_incremented { + Some(worker.clone()) + } else { + None + }; + + let response = self + .send_typed_request( + headers, + typed_req, + route, + worker.url(), + is_stream, + load_incremented, + ) + .await; + + worker.record_outcome(response.status().is_success()); + + // For retryable failures, we need to decrement load since send_typed_request + // won't have done it (it only decrements on success or non-retryable failures) + if is_retryable_status(response.status()) && load_incremented { + if let Some(cleanup_worker) = worker_for_cleanup { + cleanup_worker.decrement_load(); + RouterMetrics::set_running_requests( + cleanup_worker.url(), + cleanup_worker.load(), + ); + } + } + + response + }, + // should_retry predicate + |res, _attempt| is_retryable_status(res.status()), + // on_backoff hook + |delay, attempt| { + RouterMetrics::record_retry(route); + RouterMetrics::record_retry_backoff_duration(delay, attempt); + }, + // on_exhausted hook + || RouterMetrics::record_retries_exhausted(route), + ) + .await; + + if response.status().is_success() { + let duration = start.elapsed(); + RouterMetrics::record_request(route); + RouterMetrics::record_generate_duration(duration); + } else if !is_retryable_status(response.status()) { + RouterMetrics::record_request_error(route, "non_retryable_error"); + } + + response + } + + // Helper: return base worker URL (strips DP suffix when enabled) + fn worker_base_url(&self, worker_url: &str) -> String { + if self.dp_aware { + if let Ok((prefix, _)) = Self::extract_dp_rank(worker_url) { + return prefix.to_string(); + } + } + worker_url.to_string() + } + + // Generic simple routing for GET/POST without JSON body + async fn route_simple_request( + &self, + headers: Option<&HeaderMap>, + endpoint: &str, + method: Method, + ) -> Response { + // TODO: currently the sglang worker is using in-memory state management, so this implementation has to fan out to all workers. + // Eventually, we need to have router to manage the chat history with a proper database, will update this implementation accordingly. + let workers = self.worker_registry.get_all(); + if workers.is_empty() { + return (StatusCode::SERVICE_UNAVAILABLE, "No available workers").into_response(); + } + + let mut last_response: Option = None; + for worker in workers { + let worker_url = worker.url(); + let base = self.worker_base_url(worker_url); + + let url = format!("{}/{}", base, endpoint); + let mut request_builder = match method { + Method::GET => self.client.get(url), + Method::POST => self.client.post(url), + _ => { + return ( + StatusCode::METHOD_NOT_ALLOWED, + "Unsupported method for simple routing", + ) + .into_response() + } + }; + + if let Some(api_key) = worker.api_key() { + request_builder = + request_builder.header("Authorization", format!("Bearer {}", api_key)); + } + + if let Some(hdrs) = headers { + for (name, value) in hdrs { + let name_lc = name.as_str().to_lowercase(); + if name_lc != "content-type" && name_lc != "content-length" { + request_builder = request_builder.header(name, value); + } + } + } + + match request_builder.send().await { + Ok(res) => { + let status = StatusCode::from_u16(res.status().as_u16()) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + let response_headers = header_utils::preserve_response_headers(res.headers()); + match res.bytes().await { + Ok(body) => { + let mut response = Response::new(Body::from(body)); + *response.status_mut() = status; + *response.headers_mut() = response_headers; + if status.is_success() { + return response; + } + last_response = Some(response); + } + Err(e) => { + last_response = Some( + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to read response: {}", e), + ) + .into_response(), + ); + } + } + } + Err(e) => { + last_response = Some( + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Request failed: {}", e), + ) + .into_response(), + ); + } + } + } + + last_response + .unwrap_or_else(|| (StatusCode::BAD_GATEWAY, "No worker response").into_response()) + } + + // Route a GET request with provided headers to a specific endpoint + async fn route_get_request(&self, headers: Option<&HeaderMap>, endpoint: &str) -> Response { + self.route_simple_request(headers, endpoint, Method::GET) + .await + } + + // Route a POST request with empty body to a specific endpoint + async fn route_post_empty_request( + &self, + headers: Option<&HeaderMap>, + endpoint: &str, + ) -> Response { + self.route_simple_request(headers, endpoint, Method::POST) + .await + } + + // TODO (rui): Better accommodate to the Worker abstraction + fn extract_dp_rank(worker_url: &str) -> Result<(&str, usize), String> { + let parts: Vec<&str> = worker_url.split('@').collect(); + if parts.len() != 2 { + return Err(format!("invalid worker_url format: {}", worker_url)); + } + + // Parse the second part (dp_rank) into an integer + match parts[1].parse::() { + Ok(dp_rank) => Ok((parts[0], dp_rank)), + Err(_) => Err(format!( + "failed to parse dp_rank from worker_url: {}", + worker_url + )), + } + } + + // Send typed request directly without conversion + async fn send_typed_request( + &self, + headers: Option<&HeaderMap>, + typed_req: &T, + route: &str, + worker_url: &str, + is_stream: bool, + load_incremented: bool, // Whether load was incremented for this request + ) -> Response { + // Get the worker's API key if available + let api_key = self + .worker_registry + .get_by_url(worker_url) + .and_then(|w| w.api_key().clone()); + + let mut request_builder = if self.dp_aware { + let (worker_url_prefix, dp_rank) = match Self::extract_dp_rank(worker_url) { + Ok(tup) => tup, + Err(e) => { + error!("Failed to extract dp_rank: {}", e); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to extract dp_rank: {}", e), + ) + .into_response(); + } + }; + + let mut json_val = match serde_json::to_value(typed_req) { + Ok(j) => j, + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + format!("Convert into serde_json::Value failed: {}", e), + ) + .into_response(); + } + }; + + if let Some(map) = json_val.as_object_mut() { + map.insert( + String::from("data_parallel_rank"), + serde_json::json!(dp_rank), + ); + debug!( + "Modified request body: {}", + serde_json::to_string(&json_val).unwrap_or(String::from("ERR")) + ); + } else { + return ( + StatusCode::BAD_REQUEST, + "Failed to insert the data_parallel_rank field into the request body", + ) + .into_response(); + } + + self.client + .post(format!("{}{}", worker_url_prefix, route)) + .json(&json_val) + } else { + self.client + .post(format!("{}{}", worker_url, route)) + .json(typed_req) // Use json() directly with typed request + }; + + if let Some(key) = api_key { + request_builder = request_builder.header("Authorization", format!("Bearer {}", key)); + } + + // Copy all headers from original request if provided + if let Some(headers) = headers { + for (name, value) in headers { + // Skip Content-Type and Content-Length as .json() sets them + if *name != CONTENT_TYPE && *name != CONTENT_LENGTH { + request_builder = request_builder.header(name, value); + } + } + } + + let res = match request_builder.send().await { + Ok(res) => res, + Err(e) => { + error!( + "Failed to send typed request worker_url={} route={} error={}", + worker_url, route, e + ); + + // Decrement load on error if it was incremented + if load_incremented { + if let Some(worker) = self.worker_registry.get_by_url(worker_url) { + worker.decrement_load(); + RouterMetrics::set_running_requests(worker_url, worker.load()); + } + } + + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Request failed: {}", e), + ) + .into_response(); + } + }; + + let status = StatusCode::from_u16(res.status().as_u16()) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + + if !is_stream { + // For non-streaming requests, preserve headers + let response_headers = header_utils::preserve_response_headers(res.headers()); + + let response = match res.bytes().await { + Ok(body) => { + let mut response = Response::new(Body::from(body)); + *response.status_mut() = status; + *response.headers_mut() = response_headers; + response + } + Err(e) => { + // IMPORTANT: Decrement load on error before returning + if load_incremented { + if let Some(worker) = self.worker_registry.get_by_url(worker_url) { + worker.decrement_load(); + RouterMetrics::set_running_requests(worker_url, worker.load()); + } + } + + let error_msg = format!("Failed to get response body: {}", e); + (StatusCode::INTERNAL_SERVER_ERROR, error_msg).into_response() + } + }; + + // Decrement load counter for non-streaming requests if it was incremented + if load_incremented { + if let Some(worker) = self.worker_registry.get_by_url(worker_url) { + worker.decrement_load(); + RouterMetrics::set_running_requests(worker_url, worker.load()); + } + } + + response + } else if load_incremented { + // For streaming with load tracking, we need to manually decrement when done + let registry = Arc::clone(&self.worker_registry); + let worker_url = worker_url.to_string(); + + // Preserve headers for streaming response + let mut response_headers = header_utils::preserve_response_headers(res.headers()); + // Ensure we set the correct content-type for SSE + response_headers.insert(CONTENT_TYPE, HeaderValue::from_static("text/event-stream")); + + let stream = res.bytes_stream(); + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + + // Spawn task to forward stream and detect completion + tokio::spawn(async move { + let mut stream = stream; + let mut decremented = false; + while let Some(chunk) = stream.next().await { + match chunk { + Ok(bytes) => { + // Check for stream end marker + if bytes + .as_ref() + .windows(12) + .any(|window| window == b"data: [DONE]") + { + if let Some(worker) = registry.get_by_url(&worker_url) { + worker.decrement_load(); + RouterMetrics::set_running_requests(&worker_url, worker.load()); + decremented = true; + } + } + if tx.send(Ok(bytes)).is_err() { + break; + } + } + Err(e) => { + let _ = tx.send(Err(format!("Stream error: {}", e))); + break; + } + } + } + if !decremented { + if let Some(worker) = registry.get_by_url(&worker_url) { + worker.decrement_load(); + RouterMetrics::set_running_requests(&worker_url, worker.load()); + } + } + }); + + let stream = UnboundedReceiverStream::new(rx); + let body = Body::from_stream(stream); + + let mut response = Response::new(body); + *response.status_mut() = status; + *response.headers_mut() = response_headers; + response + } else { + // For requests without load tracking, just stream + // Preserve headers for streaming response + let mut response_headers = header_utils::preserve_response_headers(res.headers()); + // Ensure we set the correct content-type for SSE + response_headers.insert(CONTENT_TYPE, HeaderValue::from_static("text/event-stream")); + + let stream = res.bytes_stream(); + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + + // Spawn task to forward stream + tokio::spawn(async move { + let mut stream = stream; + while let Some(chunk) = stream.next().await { + match chunk { + Ok(bytes) => { + if tx.send(Ok(bytes)).is_err() { + break; + } + } + Err(e) => { + let _ = tx.send(Err(format!("Stream error: {}", e))); + break; + } + } + } + }); + + let stream = UnboundedReceiverStream::new(rx); + let body = Body::from_stream(stream); + + let mut response = Response::new(body); + *response.status_mut() = status; + *response.headers_mut() = response_headers; + response + } + } + + async fn build_rerank_response( + req: &RerankRequest, + response: Response, + ) -> anyhow::Result { + let (_, response_body) = response.into_parts(); + let body_bytes = to_bytes(response_body, usize::MAX).await?; + let rerank_results = serde_json::from_slice::>(&body_bytes)?; + let mut rerank_response = + RerankResponse::new(rerank_results, req.model.clone(), req.rid.clone()); + // Sorting is handled by Python worker (serving_rerank.py) + if let Some(top_k) = req.top_k { + rerank_response.apply_top_k(top_k); + } + if !req.return_documents { + rerank_response.drop_documents(); + } + Ok(Json(rerank_response).into_response()) + } +} + +use async_trait::async_trait; + +#[async_trait] +impl RouterTrait for Router { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + async fn health_generate(&self, req: Request) -> Response { + self.proxy_get_request(req, "health_generate").await + } + + async fn get_server_info(&self, req: Request) -> Response { + self.proxy_get_request(req, "get_server_info").await + } + + async fn get_models(&self, req: Request) -> Response { + self.proxy_get_request(req, "v1/models").await + } + + async fn get_model_info(&self, req: Request) -> Response { + self.proxy_get_request(req, "get_model_info").await + } + + async fn route_generate( + &self, + headers: Option<&HeaderMap>, + body: &GenerateRequest, + model_id: Option<&str>, + ) -> Response { + self.route_typed_request(headers, body, "/generate", model_id) + .await + } + + async fn route_chat( + &self, + headers: Option<&HeaderMap>, + body: &ChatCompletionRequest, + model_id: Option<&str>, + ) -> Response { + self.route_typed_request(headers, body, "/v1/chat/completions", model_id) + .await + } + + async fn route_completion( + &self, + headers: Option<&HeaderMap>, + body: &CompletionRequest, + model_id: Option<&str>, + ) -> Response { + self.route_typed_request(headers, body, "/v1/completions", model_id) + .await + } + + async fn route_responses( + &self, + headers: Option<&HeaderMap>, + body: &ResponsesRequest, + model_id: Option<&str>, + ) -> Response { + self.route_typed_request(headers, body, "/v1/responses", model_id) + .await + } + + async fn get_response( + &self, + headers: Option<&HeaderMap>, + response_id: &str, + _params: &ResponsesGetParams, + ) -> Response { + let endpoint = format!("v1/responses/{}", response_id); + self.route_get_request(headers, &endpoint).await + } + + async fn cancel_response(&self, headers: Option<&HeaderMap>, response_id: &str) -> Response { + let endpoint = format!("v1/responses/{}/cancel", response_id); + self.route_post_empty_request(headers, &endpoint).await + } + + async fn route_embeddings( + &self, + headers: Option<&HeaderMap>, + body: &EmbeddingRequest, + model_id: Option<&str>, + ) -> Response { + // Record embeddings-specific metrics in addition to general request metrics + let start = Instant::now(); + let res = self + .route_typed_request(headers, body, "/v1/embeddings", model_id) + .await; + + // Embedding specific metrics + if res.status().is_success() { + RouterMetrics::record_embeddings_request(); + RouterMetrics::record_embeddings_duration(start.elapsed()); + } else { + let error_type = format!("http_{}", res.status().as_u16()); + RouterMetrics::record_embeddings_error(&error_type); + } + + res + } + + async fn route_classify( + &self, + headers: Option<&HeaderMap>, + body: &ClassifyRequest, + model_id: Option<&str>, + ) -> Response { + // Record classification-specific metrics in addition to general request metrics + let start = Instant::now(); + let res = self + .route_typed_request(headers, body, "/v1/classify", model_id) + .await; + + // Classification specific metrics + if res.status().is_success() { + RouterMetrics::record_classify_request(); + RouterMetrics::record_classify_duration(start.elapsed()); + } else { + let error_type = format!("http_{}", res.status().as_u16()); + RouterMetrics::record_classify_error(&error_type); + } + + res + } + + async fn route_rerank( + &self, + headers: Option<&HeaderMap>, + body: &RerankRequest, + model_id: Option<&str>, + ) -> Response { + let response = self + .route_typed_request(headers, body, "/v1/rerank", model_id) + .await; + if response.status().is_success() { + match Self::build_rerank_response(body, response).await { + Ok(rerank_response) => rerank_response, + Err(e) => { + error!("Failed to build rerank response: {}", e); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + "Failed to build rerank response".to_string(), + ) + .into_response(); + } + } + } else { + response + } + } + + fn router_type(&self) -> &'static str { + "regular" + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::BasicWorkerBuilder; + + fn create_test_regular_router() -> Router { + // Create registries + let worker_registry = Arc::new(WorkerRegistry::new()); + let policy_registry = Arc::new(PolicyRegistry::new( + crate::config::types::PolicyConfig::RoundRobin, + )); + + // Register test workers + let worker1 = BasicWorkerBuilder::new("http://worker1:8080") + .worker_type(WorkerType::Regular) + .build(); + let worker2 = BasicWorkerBuilder::new("http://worker2:8080") + .worker_type(WorkerType::Regular) + .build(); + worker_registry.register(Arc::new(worker1)); + worker_registry.register(Arc::new(worker2)); + + Router { + worker_registry, + policy_registry, + dp_aware: false, + client: Client::new(), + retry_config: RetryConfig::default(), + enable_igw: false, + } + } + + fn create_test_unhealthy_router() -> Router { + let router = create_test_regular_router(); + let workers = router.worker_registry.get_all(); + workers[0].set_healthy(false); + router + } + + #[test] + fn test_router_get_worker_urls_regular() { + let router = create_test_regular_router(); + let workers = router.worker_registry.get_all(); + let urls: Vec = workers.iter().map(|w| w.url().to_string()).collect(); + + assert_eq!(urls.len(), 2); + assert!(urls.contains(&"http://worker1:8080".to_string())); + assert!(urls.contains(&"http://worker2:8080".to_string())); + } + + #[test] + fn test_select_first_worker_regular() { + let router = create_test_regular_router(); + let result = router.select_first_worker(); + + assert!(result.is_ok()); + let url = result.unwrap(); + // DashMap doesn't guarantee order, so just check we get one of the workers + assert!(url == "http://worker1:8080" || url == "http://worker2:8080"); + } + + #[test] + fn test_select_first_worker_with_unhealthy_worker() { + let router = create_test_unhealthy_router(); + let result = router.select_first_worker(); + + assert!(result.is_ok()); + let url = result.unwrap(); + + let worker = router.worker_registry.get_by_url(&url).unwrap(); + assert!(worker.is_healthy()); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/mod.rs new file mode 100644 index 00000000..aaa96346 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/mod.rs @@ -0,0 +1,257 @@ +//! Router implementations + +use std::fmt::Debug; + +use async_trait::async_trait; +use axum::{ + body::Body, + extract::Request, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, +}; +use serde_json::Value; + +use crate::protocols::{ + chat::ChatCompletionRequest, + classify::ClassifyRequest, + completion::CompletionRequest, + embedding::EmbeddingRequest, + generate::GenerateRequest, + rerank::RerankRequest, + responses::{ResponsesGetParams, ResponsesRequest}, +}; + +pub mod factory; +pub mod grpc; +pub mod header_utils; +pub mod http; +pub mod openai; // New refactored OpenAI router module +pub mod router_manager; + +pub use factory::RouterFactory; +// Re-export HTTP routers for convenience +pub use http::{pd_router, pd_types, router}; + +/// Core trait for all router implementations +/// +/// This trait provides a unified interface for routing requests, +/// regardless of whether it's a regular router or PD router. +#[async_trait] +pub trait RouterTrait: Send + Sync + Debug { + /// Get a reference to self as Any for downcasting + fn as_any(&self) -> &dyn std::any::Any; + + /// Route a health generate request + async fn health_generate(&self, req: Request) -> Response; + + /// Get server information + async fn get_server_info(&self, req: Request) -> Response; + + /// Get available models + async fn get_models(&self, req: Request) -> Response; + + /// Get model information + async fn get_model_info(&self, req: Request) -> Response; + + /// Route a generate request + async fn route_generate( + &self, + headers: Option<&HeaderMap>, + body: &GenerateRequest, + model_id: Option<&str>, + ) -> Response; + + /// Route a chat completion request + async fn route_chat( + &self, + headers: Option<&HeaderMap>, + body: &ChatCompletionRequest, + model_id: Option<&str>, + ) -> Response; + + /// Route a completion request + async fn route_completion( + &self, + headers: Option<&HeaderMap>, + body: &CompletionRequest, + model_id: Option<&str>, + ) -> Response; + + /// Route a responses request + async fn route_responses( + &self, + headers: Option<&HeaderMap>, + body: &ResponsesRequest, + model_id: Option<&str>, + ) -> Response; + + /// Retrieve a stored/background response by id + async fn get_response( + &self, + headers: Option<&HeaderMap>, + response_id: &str, + params: &ResponsesGetParams, + ) -> Response; + + /// Cancel a background response by id + async fn cancel_response(&self, headers: Option<&HeaderMap>, response_id: &str) -> Response; + + /// Delete a response by id + async fn delete_response(&self, _headers: Option<&HeaderMap>, _response_id: &str) -> Response { + ( + StatusCode::NOT_IMPLEMENTED, + "Responses delete endpoint not implemented", + ) + .into_response() + } + + /// List input items of a response by id + async fn list_response_input_items( + &self, + _headers: Option<&HeaderMap>, + _response_id: &str, + ) -> Response { + ( + StatusCode::NOT_IMPLEMENTED, + "Responses list input items endpoint not implemented", + ) + .into_response() + } + + /// Route embedding requests (OpenAI-compatible /v1/embeddings) + async fn route_embeddings( + &self, + headers: Option<&HeaderMap>, + body: &EmbeddingRequest, + model_id: Option<&str>, + ) -> Response; + + /// Route classification requests (OpenAI-compatible /v1/classify) + async fn route_classify( + &self, + headers: Option<&HeaderMap>, + body: &ClassifyRequest, + model_id: Option<&str>, + ) -> Response; + + async fn route_rerank( + &self, + headers: Option<&HeaderMap>, + body: &RerankRequest, + model_id: Option<&str>, + ) -> Response; + + // Conversations API + async fn create_conversation(&self, _headers: Option<&HeaderMap>, _body: &Value) -> Response { + ( + StatusCode::NOT_IMPLEMENTED, + "Conversations create endpoint not implemented", + ) + .into_response() + } + + async fn get_conversation( + &self, + _headers: Option<&HeaderMap>, + _conversation_id: &str, + ) -> Response { + ( + StatusCode::NOT_IMPLEMENTED, + "Conversations get endpoint not implemented", + ) + .into_response() + } + + async fn update_conversation( + &self, + _headers: Option<&HeaderMap>, + _conversation_id: &str, + _body: &Value, + ) -> Response { + ( + StatusCode::NOT_IMPLEMENTED, + "Conversations update endpoint not implemented", + ) + .into_response() + } + + async fn delete_conversation( + &self, + _headers: Option<&HeaderMap>, + _conversation_id: &str, + ) -> Response { + ( + StatusCode::NOT_IMPLEMENTED, + "Conversations delete endpoint not implemented", + ) + .into_response() + } + + /// List items for a conversation + async fn list_conversation_items( + &self, + _headers: Option<&HeaderMap>, + _conversation_id: &str, + _limit: Option, + _order: Option, + _after: Option, + ) -> Response { + ( + StatusCode::NOT_IMPLEMENTED, + "Conversation items list endpoint not implemented", + ) + .into_response() + } + + /// Create items in a conversation + async fn create_conversation_items( + &self, + _headers: Option<&HeaderMap>, + _conversation_id: &str, + _body: &Value, + ) -> Response { + ( + StatusCode::NOT_IMPLEMENTED, + "Conversation items create endpoint not implemented", + ) + .into_response() + } + + /// Get a single conversation item + /// The `include` parameter is accepted but not yet implemented + async fn get_conversation_item( + &self, + _headers: Option<&HeaderMap>, + _conversation_id: &str, + _item_id: &str, + _include: Option>, + ) -> Response { + ( + StatusCode::NOT_IMPLEMENTED, + "Conversation item get endpoint not implemented", + ) + .into_response() + } + + /// Delete a conversation item + async fn delete_conversation_item( + &self, + _headers: Option<&HeaderMap>, + _conversation_id: &str, + _item_id: &str, + ) -> Response { + ( + StatusCode::NOT_IMPLEMENTED, + "Conversation item delete endpoint not implemented", + ) + .into_response() + } + + /// Get router type name + fn router_type(&self) -> &'static str; + + /// Check if this is a PD router + fn is_pd_mode(&self) -> bool { + self.router_type() == "pd" + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/conversations.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/conversations.rs new file mode 100644 index 00000000..44332f22 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/conversations.rs @@ -0,0 +1,1338 @@ +//! Conversation CRUD operations and persistence + +use std::{collections::HashMap, sync::Arc}; + +use axum::{ + http::StatusCode, + response::{IntoResponse, Response}, + Json, +}; +use chrono::Utc; +use serde_json::{json, Value}; +use tracing::{debug, info, warn}; + +use super::responses::build_stored_response; +use crate::{ + data_connector::{ + Conversation, ConversationId, ConversationItemId, ConversationItemStorage, + ConversationStorage, ListParams, NewConversation, NewConversationItem, ResponseId, + ResponseStorage, SortOrder, + }, + protocols::responses::{generate_id, ResponseInput, ResponsesRequest}, +}; + +/// Maximum number of properties allowed in conversation metadata +pub(crate) const MAX_METADATA_PROPERTIES: usize = 16; + +// ============================================================================ +// Conversation CRUD Operations +// ============================================================================ + +/// Create a new conversation +pub(super) async fn create_conversation( + conversation_storage: &Arc, + body: Value, +) -> Response { + // TODO: The validation should be done in the right place + let metadata = match body.get("metadata") { + Some(Value::Object(map)) => { + if map.len() > MAX_METADATA_PROPERTIES { + return ( + StatusCode::BAD_REQUEST, + Json(json!({ + "error": + format!( + "metadata cannot have more than {} properties", + MAX_METADATA_PROPERTIES + ) + })), + ) + .into_response(); + } + Some(map.clone()) + } + Some(_) => { + return ( + StatusCode::BAD_REQUEST, + Json(json!({"error": "metadata must be an object"})), + ) + .into_response(); + } + None => None, + }; + + let new_conv = NewConversation { + id: None, // Generate random ID (OpenAI behavior for POST /v1/conversations) + metadata, + }; + + match conversation_storage.create_conversation(new_conv).await { + Ok(conversation) => { + info!(conversation_id = %conversation.id.0, "Created conversation"); + (StatusCode::OK, Json(conversation_to_json(&conversation))).into_response() + } + Err(e) => ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": format!("Failed to create conversation: {}", e) + })), + ) + .into_response(), + } +} + +/// Get a conversation by ID +pub(super) async fn get_conversation( + conversation_storage: &Arc, + conv_id: &str, +) -> Response { + let conversation_id = ConversationId::from(conv_id); + + match conversation_storage + .get_conversation(&conversation_id) + .await + { + Ok(Some(conversation)) => { + (StatusCode::OK, Json(conversation_to_json(&conversation))).into_response() + } + Ok(None) => ( + StatusCode::NOT_FOUND, + Json(json!({"error": "Conversation not found"})), + ) + .into_response(), + Err(e) => ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": format!("Failed to get conversation: {}", e) + })), + ) + .into_response(), + } +} + +/// Update a conversation's metadata +pub(super) async fn update_conversation( + conversation_storage: &Arc, + conv_id: &str, + body: Value, +) -> Response { + let conversation_id = ConversationId::from(conv_id); + + let current_meta = match conversation_storage + .get_conversation(&conversation_id) + .await + { + Ok(Some(meta)) => meta, + Ok(None) => { + return ( + StatusCode::NOT_FOUND, + Json(json!({"error": "Conversation not found"})), + ) + .into_response(); + } + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": format!("Failed to get conversation: {}", e) + })), + ) + .into_response(); + } + }; + + #[derive(Debug)] + enum Patch { + Set(String, Value), + Delete(String), + } + + let mut patches: Vec = Vec::new(); + + if let Some(metadata_val) = body.get("metadata") { + if let Some(map) = metadata_val.as_object() { + for (k, v) in map { + if v.is_null() { + patches.push(Patch::Delete(k.clone())); + } else { + patches.push(Patch::Set(k.clone(), v.clone())); + } + } + } else { + return ( + StatusCode::BAD_REQUEST, + Json(json!({"error": "metadata must be an object"})), + ) + .into_response(); + } + } + + let mut new_metadata = current_meta.metadata.clone().unwrap_or_default(); + for patch in patches { + match patch { + Patch::Set(k, v) => { + new_metadata.insert(k, v); + } + Patch::Delete(k) => { + new_metadata.remove(&k); + } + } + } + + if new_metadata.len() > MAX_METADATA_PROPERTIES { + return ( + StatusCode::BAD_REQUEST, + Json(json!({ + "error": + format!( + "metadata cannot have more than {} properties", + MAX_METADATA_PROPERTIES + ) + })), + ) + .into_response(); + } + + let final_metadata = if new_metadata.is_empty() { + None + } else { + Some(new_metadata) + }; + + match conversation_storage + .update_conversation(&conversation_id, final_metadata) + .await + { + Ok(Some(conversation)) => { + info!(conversation_id = %conversation_id.0, "Updated conversation"); + (StatusCode::OK, Json(conversation_to_json(&conversation))).into_response() + } + Ok(None) => ( + StatusCode::NOT_FOUND, + Json(json!({"error": "Conversation not found"})), + ) + .into_response(), + Err(e) => ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": format!("Failed to update conversation: {}", e) + })), + ) + .into_response(), + } +} + +/// Delete a conversation +pub(super) async fn delete_conversation( + conversation_storage: &Arc, + conv_id: &str, +) -> Response { + let conversation_id = ConversationId::from(conv_id); + + match conversation_storage + .get_conversation(&conversation_id) + .await + { + Ok(Some(_)) => {} + Ok(None) => { + return ( + StatusCode::NOT_FOUND, + Json(json!({"error": "Conversation not found"})), + ) + .into_response(); + } + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": format!("Failed to get conversation: {}", e) + })), + ) + .into_response(); + } + } + + match conversation_storage + .delete_conversation(&conversation_id) + .await + { + Ok(_) => { + info!(conversation_id = %conversation_id.0, "Deleted conversation"); + ( + StatusCode::OK, + Json(json!({ + "id": conversation_id.0, + "object": "conversation.deleted", + "deleted": true + })), + ) + .into_response() + } + Err(e) => ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": format!("Failed to delete conversation: {}", e) + })), + ) + .into_response(), + } +} + +/// List items in a conversation with pagination +pub(super) async fn list_conversation_items( + conversation_storage: &Arc, + item_storage: &Arc, + conv_id: &str, + query_params: HashMap, +) -> Response { + let conversation_id = ConversationId::from(conv_id); + + match conversation_storage + .get_conversation(&conversation_id) + .await + { + Ok(Some(_)) => {} + Ok(None) => { + return ( + StatusCode::NOT_FOUND, + Json(json!({"error": "Conversation not found"})), + ) + .into_response(); + } + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": format!("Failed to get conversation: {}", e) + })), + ) + .into_response(); + } + } + + let limit: usize = query_params + .get("limit") + .and_then(|s| s.parse().ok()) + .unwrap_or(100); + + let after = query_params.get("after").map(|s| s.to_string()); + + // Default to descending order (most recent first) + let order = query_params + .get("order") + .and_then(|s| match s.as_str() { + "asc" => Some(SortOrder::Asc), + "desc" => Some(SortOrder::Desc), + _ => None, + }) + .unwrap_or(SortOrder::Desc); + + let params = ListParams { + limit, + order, + after, + }; + + match item_storage.list_items(&conversation_id, params).await { + Ok(items) => { + let item_values: Vec = items + .iter() + .map(|item| { + let mut item_json = item_to_json(item); + // Add created_at field for list view + if let Some(obj) = item_json.as_object_mut() { + obj.insert("created_at".to_string(), json!(item.created_at)); + } + item_json + }) + .collect(); + + let has_more = items.len() == limit; + let last_id = items.last().map(|item| item.id.0.clone()); + + ( + StatusCode::OK, + Json(json!({ + "object": "list", + "data": item_values, + "has_more": has_more, + "first_id": items.first().map(|item| &item.id.0), + "last_id": last_id, + })), + ) + .into_response() + } + Err(e) => ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ "error": format!("Failed to list items: {}", e) })), + ) + .into_response(), + } +} + +// ============================================================================ +// Conversation Item Operations +// ============================================================================ + +/// Supported item types for creation +/// Types marked as "implemented" are fully supported +/// Types marked as "accepted" are stored but return not-implemented warnings +const SUPPORTED_ITEM_TYPES: &[&str] = &[ + // Fully implemented types + "message", + "reasoning", + "mcp_list_tools", + "mcp_call", + "item_reference", + "function_call", + "function_call_output", + // Accepted but not yet implemented (stored, warning returned) + "file_search_call", + "computer_call", + "computer_call_output", + "web_search_call", + "image_generation_call", + "code_interpreter_call", + "local_shell_call", + "local_shell_call_output", + "mcp_approval_request", + "mcp_approval_response", + "custom_tool_call", + "custom_tool_call_output", +]; + +/// Item types that are fully implemented with business logic +const IMPLEMENTED_ITEM_TYPES: &[&str] = &[ + "message", + "reasoning", + "mcp_list_tools", + "mcp_call", + "item_reference", +]; + +/// Create items in a conversation (bulk operation) +pub(super) async fn create_conversation_items( + conversation_storage: &Arc, + item_storage: &Arc, + conv_id: &str, + body: Value, +) -> Response { + let conversation_id = ConversationId::from(conv_id); + + // Verify conversation exists + match conversation_storage + .get_conversation(&conversation_id) + .await + { + Ok(Some(_)) => {} + Ok(None) => { + return ( + StatusCode::NOT_FOUND, + Json(json!({"error": "Conversation not found"})), + ) + .into_response(); + } + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": format!("Failed to get conversation: {}", e) + })), + ) + .into_response(); + } + } + + // Parse items array from request + let items_array = match body.get("items").and_then(|v| v.as_array()) { + Some(arr) => arr, + None => { + return ( + StatusCode::BAD_REQUEST, + Json(json!({"error": "Missing or invalid 'items' field"})), + ) + .into_response(); + } + }; + + // Validate limit (max 20 items per OpenAI spec) + if items_array.len() > 20 { + return ( + StatusCode::BAD_REQUEST, + Json(json!({"error": "Cannot add more than 20 items at a time"})), + ) + .into_response(); + } + + // Convert and create items + let mut created_items = Vec::new(); + let mut warnings = Vec::new(); + let added_at = Utc::now(); + + for item_val in items_array { + let item_type = item_val + .get("type") + .and_then(|v| v.as_str()) + .unwrap_or("message"); + + // Handle item_reference specially - link existing item instead of creating new + if item_type == "item_reference" { + let ref_id = match item_val.get("id").and_then(|v| v.as_str()) { + Some(id) => id, + None => { + return ( + StatusCode::BAD_REQUEST, + Json(json!({"error": "item_reference requires 'id' field"})), + ) + .into_response(); + } + }; + + let existing_item_id = ConversationItemId::from(ref_id); + + // Retrieve the existing item + let existing_item = match item_storage.get_item(&existing_item_id).await { + Ok(Some(item)) => item, + Ok(None) => { + return ( + StatusCode::NOT_FOUND, + Json(json!({ + "error": format!("Referenced item '{}' not found", ref_id) + })), + ) + .into_response(); + } + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": format!("Failed to get referenced item: {}", e) + })), + ) + .into_response(); + } + }; + + // Link existing item to this conversation + if let Err(e) = item_storage + .link_item(&conversation_id, &existing_item.id, added_at) + .await + { + warn!("Failed to link item {}: {}", existing_item.id.0, e); + } + + created_items.push(item_to_json(&existing_item)); + continue; + } + + // Check if user provided an ID + let user_provided_id = item_val.get("id").and_then(|v| v.as_str()); + + let item = if let Some(id_str) = user_provided_id { + // User provided an ID - check if it already exists in DB + let item_id = ConversationItemId::from(id_str); + + // First check if this item is already linked to this conversation + let is_already_linked = match item_storage + .is_item_linked(&conversation_id, &item_id) + .await + { + Ok(linked) => linked, + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": format!("Failed to check item link: {}", e) + })), + ) + .into_response(); + } + }; + + if is_already_linked { + // Item already linked to this conversation - return error + return ( + StatusCode::BAD_REQUEST, + Json(json!({ + "error": { + "message": "Item already in conversation", + "type": "invalid_request_error", + "param": "items", + "code": "item_already_in_conversation" + } + })), + ) + .into_response(); + } + + // Check if item exists in DB + let existing_item = match item_storage.get_item(&item_id).await { + Ok(Some(item)) => item, + Ok(None) => { + // Item doesn't exist in DB, create new one with user-provided content + let (new_item, warning) = match parse_item_from_value(item_val) { + Ok((mut item, warn)) => { + // Use the user-provided ID + item.id = Some(item_id.clone()); + (item, warn) + } + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + Json(json!({ "error": format!("Invalid item: {}", e) })), + ) + .into_response(); + } + }; + + // Collect warnings for not-implemented types + if let Some(w) = warning { + warnings.push(w); + } + + // Create item with provided ID + match item_storage.create_item(new_item).await { + Ok(item) => item, + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ "error": format!("Failed to create item: {}", e) })), + ) + .into_response(); + } + } + } + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": format!("Failed to check item existence: {}", e) + })), + ) + .into_response(); + } + }; + + existing_item + } else { + // No ID provided - parse and create new item normally + let (new_item, warning) = match parse_item_from_value(item_val) { + Ok((item, warn)) => (item, warn), + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + Json(json!({ "error": format!("Invalid item: {}", e) })), + ) + .into_response(); + } + }; + + // Collect warnings for not-implemented types + if let Some(w) = warning { + warnings.push(w); + } + + // Create item + match item_storage.create_item(new_item).await { + Ok(item) => item, + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ "error": format!("Failed to create item: {}", e) })), + ) + .into_response(); + } + } + }; + + // Link to conversation + if let Err(e) = item_storage + .link_item(&conversation_id, &item.id, added_at) + .await + { + warn!("Failed to link item {}: {}", item.id.0, e); + } + + created_items.push(item_to_json(&item)); + } + + // Build response matching OpenAI format + let first_id = created_items.first().and_then(|v| v.get("id")); + let last_id = created_items.last().and_then(|v| v.get("id")); + + let mut response = json!({ + "object": "list", + "data": created_items, + "first_id": first_id, + "last_id": last_id, + "has_more": false + }); + + // Add warnings if any not-implemented types were used + if !warnings.is_empty() { + if let Some(obj) = response.as_object_mut() { + obj.insert("warnings".to_string(), json!(warnings)); + } + } + + (StatusCode::OK, Json(response)).into_response() +} + +/// Get a single conversation item +/// Note: `include` query parameter is accepted but not yet implemented +pub(super) async fn get_conversation_item( + conversation_storage: &Arc, + item_storage: &Arc, + conv_id: &str, + item_id: &str, + _include: Option>, // Reserved for future use +) -> Response { + let conversation_id = ConversationId::from(conv_id); + let item_id = ConversationItemId::from(item_id); + + // Verify conversation exists + match conversation_storage + .get_conversation(&conversation_id) + .await + { + Ok(Some(_)) => {} + Ok(None) => { + return ( + StatusCode::NOT_FOUND, + Json(json!({"error": "Conversation not found"})), + ) + .into_response(); + } + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": format!("Failed to get conversation: {}", e) + })), + ) + .into_response(); + } + } + + // First check if the item is linked to this conversation + let is_linked = match item_storage + .is_item_linked(&conversation_id, &item_id) + .await + { + Ok(linked) => linked, + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": format!("Failed to check item link: {}", e) + })), + ) + .into_response(); + } + }; + + if !is_linked { + return ( + StatusCode::NOT_FOUND, + Json(json!({"error": "Item not found in this conversation"})), + ) + .into_response(); + } + + // Get the item + match item_storage.get_item(&item_id).await { + Ok(Some(item)) => { + // TODO: Process `include` parameter when implemented + // Example: include=["metadata", "timestamps"] + (StatusCode::OK, Json(item_to_json(&item))).into_response() + } + Ok(None) => ( + StatusCode::NOT_FOUND, + Json(json!({"error": "Item not found"})), + ) + .into_response(), + Err(e) => ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ "error": format!("Failed to get item: {}", e) })), + ) + .into_response(), + } +} + +/// Delete a conversation item +pub(super) async fn delete_conversation_item( + conversation_storage: &Arc, + item_storage: &Arc, + conv_id: &str, + item_id: &str, +) -> Response { + let conversation_id = ConversationId::from(conv_id); + let item_id = ConversationItemId::from(item_id); + + // Verify conversation exists and get it for response + let conversation = match conversation_storage + .get_conversation(&conversation_id) + .await + { + Ok(Some(conv)) => conv, + Ok(None) => { + return ( + StatusCode::NOT_FOUND, + Json(json!({"error": "Conversation not found"})), + ) + .into_response(); + } + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": format!("Failed to get conversation: {}", e) + })), + ) + .into_response(); + } + }; + + // Delete the item + match item_storage.delete_item(&conversation_id, &item_id).await { + Ok(_) => { + info!( + conversation_id = %conversation_id.0, + item_id = %item_id.0, + "Deleted conversation item" + ); + + // Return updated conversation object (per OpenAI spec) + (StatusCode::OK, Json(conversation_to_json(&conversation))).into_response() + } + Err(e) => ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ "error": format!("Failed to delete item: {}", e) })), + ) + .into_response(), + } +} + +/// Parse NewConversationItem from Value +/// Returns (NewConversationItem, Option) +/// Supports three top-level structures: +/// 1. Input message: {"type": "message", "role": "...", "content": [...]} +/// 2. Item: {"type": "message|function_tool_call|...", ...} +/// 3. Item reference: {"type": "item_reference", "id": "..."} +fn parse_item_from_value( + item_val: &Value, +) -> Result<(NewConversationItem, Option), String> { + // Detect structure type + let item_type = item_val + .get("type") + .and_then(|v| v.as_str()) + .unwrap_or("message"); + + // Validate item type is supported + if !SUPPORTED_ITEM_TYPES.contains(&item_type) { + return Err(format!( + "Unsupported item type '{}'. Supported types: {}", + item_type, + SUPPORTED_ITEM_TYPES.join(", ") + )); + } + + // Check if type is implemented or just accepted + let warning = if !IMPLEMENTED_ITEM_TYPES.contains(&item_type) { + Some(format!( + "Item type '{}' is accepted but not yet implemented. \ + The item will be stored but may not function as expected.", + item_type + )) + } else { + None + }; + + // Parse common fields + let role = item_val + .get("role") + .and_then(|v| v.as_str()) + .map(String::from); + let status = item_val + .get("status") + .and_then(|v| v.as_str()) + .map(String::from) + .or_else(|| Some("completed".to_string())); // Default status + + // Validate message types have role + if item_type == "message" && role.is_none() { + return Err("Message items require 'role' field".to_string()); + } + + // For special types (mcp_call, function_tool_call, etc.), store the entire item_val as content + // For message types, use the content field directly + let content = if item_type == "message" || item_type == "reasoning" { + item_val.get("content").cloned().unwrap_or(json!([])) + } else { + // Store entire item for extraction later + item_val.clone() + }; + + Ok(( + NewConversationItem { + id: None, + response_id: None, + item_type: item_type.to_string(), + role, + content, + status, + }, + warning, + )) +} + +/// Convert ConversationItem to JSON response format +/// Extracts fields from content for special types (mcp_call, mcp_list_tools, etc.) +fn item_to_json(item: &crate::data_connector::ConversationItem) -> Value { + let mut obj = serde_json::Map::new(); + obj.insert("id".to_string(), json!(item.id.0)); + obj.insert("type".to_string(), json!(item.item_type)); + + if let Some(role) = &item.role { + obj.insert("role".to_string(), json!(role)); + } + + // Handle special item types that need field extraction from content + match item.item_type.as_str() { + "mcp_call" => { + // Extract mcp_call fields: name, arguments, output, server_label, approval_request_id, error + if let Some(content_obj) = item.content.as_object() { + if let Some(name) = content_obj.get("name") { + obj.insert("name".to_string(), name.clone()); + } + if let Some(arguments) = content_obj.get("arguments") { + obj.insert("arguments".to_string(), arguments.clone()); + } + if let Some(output) = content_obj.get("output") { + obj.insert("output".to_string(), output.clone()); + } + if let Some(server_label) = content_obj.get("server_label") { + obj.insert("server_label".to_string(), server_label.clone()); + } + if let Some(approval_request_id) = content_obj.get("approval_request_id") { + obj.insert( + "approval_request_id".to_string(), + approval_request_id.clone(), + ); + } + if let Some(error) = content_obj.get("error") { + obj.insert("error".to_string(), error.clone()); + } + } + } + "mcp_list_tools" => { + // Extract mcp_list_tools fields: tools, server_label + if let Some(content_obj) = item.content.as_object() { + if let Some(tools) = content_obj.get("tools") { + obj.insert("tools".to_string(), tools.clone()); + } + if let Some(server_label) = content_obj.get("server_label") { + obj.insert("server_label".to_string(), server_label.clone()); + } + } + } + "function_call" => { + // Extract function_call fields: call_id, name, arguments, output + if let Some(content_obj) = item.content.as_object() { + for field in ["call_id", "name", "arguments", "output"] { + if let Some(value) = content_obj.get(field) { + obj.insert(field.to_string(), value.clone()); + } + } + } + } + "function_call_output" => { + // Extract function_call_output fields: call_id, output + if let Some(content_obj) = item.content.as_object() { + for field in ["call_id", "output"] { + if let Some(value) = content_obj.get(field) { + obj.insert(field.to_string(), value.clone()); + } + } + } + } + _ => { + // For all other types (message, reasoning, etc.), keep content as-is + obj.insert("content".to_string(), item.content.clone()); + } + } + + if let Some(status) = &item.status { + obj.insert("status".to_string(), json!(status)); + } + + Value::Object(obj) +} + +// ============================================================================ +// Persistence Operations +// ============================================================================ + +/// Persist conversation items (delegates to persist_items_with_storages) +pub async fn persist_conversation_items( + conversation_storage: Arc, + item_storage: Arc, + response_storage: Arc, + response_json: &Value, + original_body: &ResponsesRequest, +) -> Result<(), String> { + persist_items_with_storages( + conversation_storage, + item_storage, + response_storage, + response_json, + original_body, + ) + .await +} + +/// Helper function to create and optionally link a conversation item +/// If conv_id is None, only creates the item without linking +async fn create_and_link_item( + item_storage: &Arc, + conv_id_opt: Option<&ConversationId>, + mut new_item: NewConversationItem, +) -> Result<(), String> { + // Set default status if not provided + if new_item.status.is_none() { + new_item.status = Some("completed".to_string()); + } + + // Step 1: Create the item + let created = item_storage + .create_item(new_item) + .await + .map_err(|e| format!("Failed to create item: {}", e))?; + + // Step 2: Link it to the conversation (if provided) + if let Some(conv_id) = conv_id_opt { + item_storage + .link_item(conv_id, &created.id, Utc::now()) + .await + .map_err(|e| format!("Failed to link item: {}", e))?; + + debug!( + conversation_id = %conv_id.0, + item_id = %created.id.0, + item_type = %created.item_type, + "Persisted conversation item and link" + ); + } else { + debug!( + item_id = %created.id.0, + item_type = %created.item_type, + "Persisted conversation item (no conversation link)" + ); + } + + Ok(()) +} + +/// Persist conversation items with all storages +/// +/// This function: +/// 1. Extracts and normalizes input items from the request +/// 2. Extracts output items from the response +/// 3. Stores ALL items in response storage (always) +/// 4. If conversation provided, also links items to conversation +async fn persist_items_with_storages( + conversation_storage: Arc, + item_storage: Arc, + response_storage: Arc, + response_json: &Value, + original_body: &ResponsesRequest, +) -> Result<(), String> { + // Step 1: Extract response ID + let response_id_str = response_json + .get("id") + .and_then(|v| v.as_str()) + .ok_or_else(|| "Response missing id field".to_string())?; + let response_id = ResponseId::from(response_id_str); + + // Step 2: Parse and normalize input items from request + let input_items = extract_input_items(&original_body.input)?; + + // Step 3: Parse output items from response + let output_items = extract_output_items(response_json)?; + + // Step 4: Build StoredResponse with input and output as JSON arrays + let mut stored_response = build_stored_response(response_json, original_body); + stored_response.id = response_id.clone(); + stored_response.input = Value::Array(input_items.clone()); + stored_response.output = Value::Array(output_items.clone()); + + // Step 5: Store response (ALWAYS, regardless of conversation) + response_storage + .store_response(stored_response) + .await + .map_err(|e| format!("Failed to store response: {}", e))?; + + // Step 6: Check if conversation is provided and validate it + let conv_id_opt = match &original_body.conversation { + Some(id) => { + let conv_id = ConversationId::from(id.as_str()); + // Verify conversation exists + if conversation_storage + .get_conversation(&conv_id) + .await + .map_err(|e| format!("Failed to get conversation: {}", e))? + .is_none() + { + warn!(conversation_id = %conv_id.0, "Conversation not found, skipping item linking"); + None // Conversation doesn't exist, items already stored in response + } else { + Some(conv_id) + } + } + None => None, // No conversation provided, items already stored in response + }; + + // Step 7: If conversation exists, link items to it + if let Some(conv_id) = conv_id_opt { + link_items_to_conversation( + &item_storage, + &conv_id, + &input_items, + &output_items, + response_id_str, + ) + .await?; + + info!( + conversation_id = %conv_id.0, + response_id = %response_id.0, + input_count = input_items.len(), + output_count = output_items.len(), + "Persisted response and linked items to conversation" + ); + } else { + info!( + response_id = %response_id.0, + input_count = input_items.len(), + output_count = output_items.len(), + "Persisted response without conversation linking" + ); + } + + Ok(()) +} + +/// Extract and normalize input items from ResponseInput +fn extract_input_items(input: &ResponseInput) -> Result, String> { + use crate::protocols::responses::{ResponseInputOutputItem, StringOrContentParts}; + + let items = match input { + ResponseInput::Text(text) => { + // Convert simple text to message item + vec![json!({ + "id": generate_id("msg"), + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": text}], + "status": "completed" + })] + } + ResponseInput::Items(items) => { + // Process all item types and ensure IDs + items + .iter() + .map(|item| { + match item { + ResponseInputOutputItem::SimpleInputMessage { content, role, .. } => { + // Convert SimpleInputMessage to standard message format with ID + let content_json = match content { + StringOrContentParts::String(s) => { + json!([{"type": "input_text", "text": s}]) + } + StringOrContentParts::Array(parts) => serde_json::to_value(parts) + .map_err(|e| { + format!("Failed to serialize content: {}", e) + })?, + }; + + Ok(json!({ + "id": generate_id("msg"), + "type": "message", + "role": role, + "content": content_json, + "status": "completed" + })) + } + _ => { + // For other item types (Message, Reasoning, FunctionToolCall, FunctionCallOutput), serialize and ensure ID + let mut value = serde_json::to_value(item) + .map_err(|e| format!("Failed to serialize item: {}", e))?; + + // Ensure ID exists - generate if missing + if let Some(obj) = value.as_object_mut() { + if !obj.contains_key("id") + || obj + .get("id") + .and_then(|v| v.as_str()) + .map(|s| s.is_empty()) + .unwrap_or(true) + { + // Generate ID with appropriate prefix based on type + let item_type = + obj.get("type").and_then(|v| v.as_str()).unwrap_or("item"); + let prefix = match item_type { + "function_call" | "function_call_output" => "fc", + "message" => "msg", + _ => "item", + }; + obj.insert("id".to_string(), json!(generate_id(prefix))); + } + } + + Ok(value) + } + } + }) + .collect::, String>>()? + } + }; + + Ok(items) +} + +/// Extract ALL output items from response JSON +fn extract_output_items(response_json: &Value) -> Result, String> { + response_json + .get("output") + .and_then(|v| v.as_array()) + .cloned() + .ok_or_else(|| "No output array in response".to_string()) +} + +/// Link ALL input and output items to a conversation +async fn link_items_to_conversation( + item_storage: &Arc, + conv_id: &ConversationId, + input_items: &[Value], + output_items: &[Value], + response_id: &str, +) -> Result<(), String> { + let response_id_opt = Some(response_id.to_string()); + + // Link ALL input items (no filtering by type) + for input_item_value in input_items { + let item_type = input_item_value + .get("type") + .and_then(|v| v.as_str()) + .unwrap_or("message"); + let role = input_item_value + .get("role") + .and_then(|v| v.as_str()) + .map(String::from); + + // For function_call and function_call_output, store the entire item as content + // For message types, extract just the content field + let content = if item_type == "function_call" || item_type == "function_call_output" { + input_item_value.clone() + } else { + input_item_value + .get("content") + .cloned() + .unwrap_or(json!([])) + }; + + let status = input_item_value + .get("status") + .and_then(|v| v.as_str()) + .map(String::from); + + // Extract the original item ID from input if present + let item_id = input_item_value + .get("id") + .and_then(|v| v.as_str()) + .map(ConversationItemId::from); + + let new_item = NewConversationItem { + id: item_id, // Preserve ID if present + response_id: response_id_opt.clone(), + item_type: item_type.to_string(), + role, + content, + status, + }; + + create_and_link_item(item_storage, Some(conv_id), new_item).await?; + } + + // Link ALL output items (no filtering by type) + // Store reasoning, function_tool_call, mcp_call, and any other types + for output_item_value in output_items { + let item_type = output_item_value + .get("type") + .and_then(|v| v.as_str()) + .unwrap_or("message"); + let role = output_item_value + .get("role") + .and_then(|v| v.as_str()) + .map(String::from); + let status = output_item_value + .get("status") + .and_then(|v| v.as_str()) + .map(String::from); + + // Extract the original item ID from the response + let item_id = output_item_value + .get("id") + .and_then(|v| v.as_str()) + .map(ConversationItemId::from); + + // For non-message types, store the entire item as content + // For message types, extract just the content field + let content = if item_type == "message" { + output_item_value + .get("content") + .cloned() + .unwrap_or(json!([])) + } else { + // For other types (reasoning, function_call, function_call_output, mcp_call, etc.) + // store the entire item structure + output_item_value.clone() + }; + + let new_item = NewConversationItem { + id: item_id, // Preserve ID if present + response_id: response_id_opt.clone(), + item_type: item_type.to_string(), + role, + content, + status, + }; + + create_and_link_item(item_storage, Some(conv_id), new_item).await?; + } + + Ok(()) +} + +// ============================================================================ +// Helper Functions +// ============================================================================ + +/// Convert conversation to JSON response +pub(crate) fn conversation_to_json(conversation: &Conversation) -> Value { + let mut response = json!({ + "id": conversation.id.0, + "object": "conversation", + "created_at": conversation.created_at.timestamp() + }); + + if let Some(metadata) = &conversation.metadata { + if !metadata.is_empty() { + if let Some(obj) = response.as_object_mut() { + obj.insert("metadata".to_string(), Value::Object(metadata.clone())); + } + } + } + + response +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/mcp.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/mcp.rs new file mode 100644 index 00000000..8087bffa --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/mcp.rs @@ -0,0 +1,979 @@ +//! MCP (Model Context Protocol) Integration Module +//! +//! This module contains all MCP-related functionality for the OpenAI router: +//! - Tool loop state management for multi-turn tool calling +//! - MCP tool execution and result handling +//! - Output item builders for MCP-specific response formats +//! - SSE event generation for streaming MCP operations +//! - Payload transformation for MCP tool interception +//! - Metadata injection for MCP operations + +use std::{io, sync::Arc}; + +use axum::http::HeaderMap; +use bytes::Bytes; +use serde_json::{json, to_value, Value}; +use tokio::sync::mpsc; +use tracing::{debug, info, warn}; + +use super::utils::event_types; +use crate::{ + mcp, + protocols::responses::{ + generate_id, ResponseInput, ResponseTool, ResponseToolType, ResponsesRequest, + }, + routers::header_utils::apply_request_headers, +}; + +// ============================================================================ +// Configuration and State Types +// ============================================================================ + +/// Configuration for MCP tool calling loops +#[allow(dead_code)] +#[derive(Debug, Clone)] +pub(crate) struct McpLoopConfig { + /// Maximum iterations as safety limit (internal only, default: 10) + /// Prevents infinite loops when max_tool_calls is not set + pub max_iterations: usize, +} + +impl Default for McpLoopConfig { + fn default() -> Self { + Self { max_iterations: 10 } + } +} + +/// State for tracking multi-turn tool calling loop +pub(crate) struct ToolLoopState { + /// Current iteration number (starts at 0, increments with each tool call) + pub iteration: usize, + /// Total number of tool calls executed + pub total_calls: usize, + /// Conversation history (function_call and function_call_output items) + pub conversation_history: Vec, + /// Original user input (preserved for building resume payloads) + pub original_input: ResponseInput, +} + +impl ToolLoopState { + pub fn new(original_input: ResponseInput) -> Self { + Self { + iteration: 0, + total_calls: 0, + conversation_history: Vec::new(), + original_input, + } + } + + /// Record a tool call in the loop state + pub fn record_call( + &mut self, + call_id: String, + tool_name: String, + args_json_str: String, + output_str: String, + ) { + // Add function_call item to history + let func_item = json!({ + "type": event_types::ITEM_TYPE_FUNCTION_CALL, + "call_id": call_id, + "name": tool_name, + "arguments": args_json_str + }); + self.conversation_history.push(func_item); + + // Add function_call_output item to history + let output_item = json!({ + "type": "function_call_output", + "call_id": call_id, + "output": output_str + }); + self.conversation_history.push(output_item); + } +} + +/// Represents a function call being accumulated across delta events +#[derive(Debug, Clone)] +pub(crate) struct FunctionCallInProgress { + pub call_id: String, + pub name: String, + pub arguments_buffer: String, + pub output_index: usize, + pub last_obfuscation: Option, + pub assigned_output_index: Option, +} + +impl FunctionCallInProgress { + pub fn new(call_id: String, output_index: usize) -> Self { + Self { + call_id, + name: String::new(), + arguments_buffer: String::new(), + output_index, + last_obfuscation: None, + assigned_output_index: None, + } + } + + pub fn is_complete(&self) -> bool { + // A tool call is complete if it has a name + !self.name.is_empty() + } + + pub fn effective_output_index(&self) -> usize { + self.assigned_output_index.unwrap_or(self.output_index) + } +} + +// ============================================================================ +// MCP Manager Integration +// ============================================================================ + +/// Ensure a dynamic MCP client exists for request-scoped tools. +/// +/// This function parses request tools to extract MCP server configuration, +/// then ensures a dynamic client exists in the McpManager via `get_or_create_client()`. +/// The McpManager itself is returned (cloned Arc) for convenience, though the main +/// purpose is the side effect of registering the dynamic client. +/// +/// Returns Some(manager) if a dynamic MCP tool was found and client was created/retrieved, +/// None if no MCP tools were found or connection failed. +pub async fn ensure_request_mcp_client( + mcp_manager: &Arc, + tools: &[ResponseTool], +) -> Option> { + let tool = tools + .iter() + .find(|t| matches!(t.r#type, ResponseToolType::Mcp) && t.server_url.is_some())?; + let server_url = tool.server_url.as_ref()?.trim().to_string(); + if !(server_url.starts_with("http://") || server_url.starts_with("https://")) { + warn!( + "Ignoring MCP server_url with unsupported scheme: {}", + server_url + ); + return None; + } + let name = tool + .server_label + .clone() + .unwrap_or_else(|| "request-mcp".to_string()); + let token = tool.authorization.clone(); + let transport = if server_url.contains("/sse") { + mcp::McpTransport::Sse { + url: server_url.clone(), + token, + } + } else { + mcp::McpTransport::Streamable { + url: server_url.clone(), + token, + } + }; + + // Create server config + let server_config = mcp::McpServerConfig { + name, + transport, + proxy: None, + required: false, + }; + + // Use McpManager to get or create dynamic client + match mcp_manager.get_or_create_client(server_config).await { + Ok(_client) => Some(mcp_manager.clone()), + Err(err) => { + warn!("Failed to get/create MCP connection: {}", err); + None + } + } +} + +// ============================================================================ +// Tool Execution +// ============================================================================ + +/// Execute detected tool calls and send completion events to client +/// Returns false if client disconnected during execution +pub(super) async fn execute_streaming_tool_calls( + pending_calls: Vec, + active_mcp: &Arc, + tx: &mpsc::UnboundedSender>, + state: &mut ToolLoopState, + server_label: &str, + sequence_number: &mut u64, +) -> bool { + // Execute all pending tool calls (sequential, as PR3 is skipped) + for call in pending_calls { + // Skip if name is empty (invalid call) + if call.name.is_empty() { + warn!( + "Skipping incomplete tool call: name is empty, args_len={}", + call.arguments_buffer.len() + ); + continue; + } + + info!( + "Executing tool call during streaming: {} ({})", + call.name, call.call_id + ); + + // Use empty JSON object if arguments_buffer is empty + let args_str = if call.arguments_buffer.is_empty() { + "{}" + } else { + &call.arguments_buffer + }; + + // Call tool directly - manager handles parsing and type coercion + debug!("Calling MCP tool '{}' with args: {}", call.name, args_str); + let call_result = active_mcp.call_tool(&call.name, args_str).await; + let (output_str, success, error_msg) = match call_result { + Ok(result) => match serde_json::to_string(&result) { + Ok(output) => (output, true, None), + Err(e) => { + let err = format!("Failed to serialize tool result: {}", e); + warn!("{}", err); + (json!({ "error": &err }).to_string(), false, Some(err)) + } + }, + Err(err) => { + let err_str = format!("tool call failed: {}", err); + warn!("Tool execution failed during streaming: {}", err_str); + ( + json!({ "error": &err_str }).to_string(), + false, + Some(err_str), + ) + } + }; + + // Send mcp_call completion event to client + if !send_mcp_call_completion_events_with_error( + tx, + &call, + &output_str, + server_label, + success, + error_msg.as_deref(), + sequence_number, + ) { + // Client disconnected, no point continuing tool execution + return false; + } + + // Record the call + state.record_call(call.call_id, call.name, call.arguments_buffer, output_str); + } + true +} + +// ============================================================================ +// Payload Transformation +// ============================================================================ + +/// Transform payload to replace MCP tools with function tools for streaming +pub(super) fn prepare_mcp_payload_for_streaming( + payload: &mut Value, + active_mcp: &Arc, +) { + if let Some(obj) = payload.as_object_mut() { + // Remove any non-function tools from outgoing payload + if let Some(v) = obj.get_mut("tools") { + if let Some(arr) = v.as_array_mut() { + arr.retain(|item| { + item.get("type") + .and_then(|v| v.as_str()) + .map(|s| s == event_types::ITEM_TYPE_FUNCTION) + .unwrap_or(false) + }); + } + } + + // Build function tools for all discovered MCP tools + let mut tools_json = Vec::new(); + let tools = active_mcp.list_tools(); + for t in tools { + let parameters = Value::Object((*t.input_schema).clone()); + let tool = serde_json::json!({ + "type": event_types::ITEM_TYPE_FUNCTION, + "name": t.name, + "description": t.description, + "parameters": parameters + }); + tools_json.push(tool); + } + if !tools_json.is_empty() { + obj.insert("tools".to_string(), Value::Array(tools_json)); + obj.insert("tool_choice".to_string(), Value::String("auto".to_string())); + } + } +} + +/// Build a resume payload with conversation history +pub(super) fn build_resume_payload( + base_payload: &Value, + conversation_history: &[Value], + original_input: &ResponseInput, + tools_json: &Value, + is_streaming: bool, +) -> Result { + // Clone the base payload which already has cleaned fields + let mut payload = base_payload.clone(); + + let obj = payload + .as_object_mut() + .ok_or_else(|| "payload not an object".to_string())?; + + // Build input array: start with original user input + let mut input_array = Vec::new(); + + // Add original user message + // For structured input, serialize the original input items + match original_input { + ResponseInput::Text(text) => { + let user_item = json!({ + "type": "message", + "role": "user", + "content": [{ "type": "input_text", "text": text }] + }); + input_array.push(user_item); + } + ResponseInput::Items(items) => { + // Items are ResponseInputOutputItem (including SimpleInputMessage), convert to JSON + if let Ok(items_value) = to_value(items) { + if let Some(items_arr) = items_value.as_array() { + input_array.extend_from_slice(items_arr); + } + } + } + } + + // Add all conversation history (function calls and outputs) + input_array.extend_from_slice(conversation_history); + + obj.insert("input".to_string(), Value::Array(input_array)); + + // Use the transformed tools (function tools, not MCP tools) + if let Some(tools_arr) = tools_json.as_array() { + if !tools_arr.is_empty() { + obj.insert("tools".to_string(), tools_json.clone()); + } + } + + // Set streaming mode based on caller's context + obj.insert("stream".to_string(), Value::Bool(is_streaming)); + obj.insert("store".to_string(), Value::Bool(false)); + + // Note: SGLang-specific fields were already removed from base_payload + // before it was passed to execute_tool_loop (see route_responses lines 1935-1946) + + Ok(payload) +} + +// ============================================================================ +// SSE Event Senders +// ============================================================================ + +/// Send mcp_list_tools events to client at the start of streaming +/// Returns false if client disconnected +pub(super) fn send_mcp_list_tools_events( + tx: &mpsc::UnboundedSender>, + mcp: &Arc, + server_label: &str, + output_index: usize, + sequence_number: &mut u64, +) -> bool { + let tools_item_full = build_mcp_list_tools_item(mcp, server_label); + let item_id = tools_item_full + .get("id") + .and_then(|v| v.as_str()) + .unwrap_or(""); + + // Create empty tools version for the initial added event + let mut tools_item_empty = tools_item_full.clone(); + if let Some(obj) = tools_item_empty.as_object_mut() { + obj.insert("tools".to_string(), json!([])); + } + + // Event 1: response.output_item.added with empty tools + let event1_payload = json!({ + "type": event_types::OUTPUT_ITEM_ADDED, + "sequence_number": *sequence_number, + "output_index": output_index, + "item": tools_item_empty + }); + *sequence_number += 1; + let event1 = format!( + "event: {}\ndata: {}\n\n", + event_types::OUTPUT_ITEM_ADDED, + event1_payload + ); + if tx.send(Ok(Bytes::from(event1))).is_err() { + return false; // Client disconnected + } + + // Event 2: response.mcp_list_tools.in_progress + let event2_payload = json!({ + "type": event_types::MCP_LIST_TOOLS_IN_PROGRESS, + "sequence_number": *sequence_number, + "output_index": output_index, + "item_id": item_id + }); + *sequence_number += 1; + let event2 = format!( + "event: {}\ndata: {}\n\n", + event_types::MCP_LIST_TOOLS_IN_PROGRESS, + event2_payload + ); + if tx.send(Ok(Bytes::from(event2))).is_err() { + return false; + } + + // Event 3: response.mcp_list_tools.completed + let event3_payload = json!({ + "type": event_types::MCP_LIST_TOOLS_COMPLETED, + "sequence_number": *sequence_number, + "output_index": output_index, + "item_id": item_id + }); + *sequence_number += 1; + let event3 = format!( + "event: {}\ndata: {}\n\n", + event_types::MCP_LIST_TOOLS_COMPLETED, + event3_payload + ); + if tx.send(Ok(Bytes::from(event3))).is_err() { + return false; + } + + // Event 4: response.output_item.done with full tools list + let event4_payload = json!({ + "type": event_types::OUTPUT_ITEM_DONE, + "sequence_number": *sequence_number, + "output_index": output_index, + "item": tools_item_full + }); + *sequence_number += 1; + let event4 = format!( + "event: {}\ndata: {}\n\n", + event_types::OUTPUT_ITEM_DONE, + event4_payload + ); + tx.send(Ok(Bytes::from(event4))).is_ok() +} + +/// Send mcp_call completion events after tool execution +/// Returns false if client disconnected +pub(super) fn send_mcp_call_completion_events_with_error( + tx: &mpsc::UnboundedSender>, + call: &FunctionCallInProgress, + output: &str, + server_label: &str, + success: bool, + error_msg: Option<&str>, + sequence_number: &mut u64, +) -> bool { + let effective_output_index = call.effective_output_index(); + + // Build mcp_call item (reuse existing function) + let mcp_call_item = build_mcp_call_item( + &call.name, + &call.arguments_buffer, + output, + server_label, + success, + error_msg, + ); + + // Get the mcp_call item_id + let item_id = mcp_call_item + .get("id") + .and_then(|v| v.as_str()) + .unwrap_or(""); + + // Event 1: response.mcp_call.completed + let completed_payload = json!({ + "type": event_types::MCP_CALL_COMPLETED, + "sequence_number": *sequence_number, + "output_index": effective_output_index, + "item_id": item_id + }); + *sequence_number += 1; + + let completed_event = format!( + "event: {}\ndata: {}\n\n", + event_types::MCP_CALL_COMPLETED, + completed_payload + ); + if tx.send(Ok(Bytes::from(completed_event))).is_err() { + return false; + } + + // Event 2: response.output_item.done (with completed mcp_call) + let done_payload = json!({ + "type": event_types::OUTPUT_ITEM_DONE, + "sequence_number": *sequence_number, + "output_index": effective_output_index, + "item": mcp_call_item + }); + *sequence_number += 1; + + let done_event = format!( + "event: {}\ndata: {}\n\n", + event_types::OUTPUT_ITEM_DONE, + done_payload + ); + tx.send(Ok(Bytes::from(done_event))).is_ok() +} + +// ============================================================================ +// Metadata Injection +// ============================================================================ + +/// Inject MCP metadata into a streaming response +pub(super) fn inject_mcp_metadata_streaming( + response: &mut Value, + state: &ToolLoopState, + mcp: &Arc, + server_label: &str, +) { + if let Some(output_array) = response.get_mut("output").and_then(|v| v.as_array_mut()) { + output_array.retain(|item| { + item.get("type").and_then(|t| t.as_str()) != Some(event_types::ITEM_TYPE_MCP_LIST_TOOLS) + }); + + let list_tools_item = build_mcp_list_tools_item(mcp, server_label); + output_array.insert(0, list_tools_item); + + let mcp_call_items = + build_executed_mcp_call_items(&state.conversation_history, server_label); + let mut insert_pos = 1; + for item in mcp_call_items { + output_array.insert(insert_pos, item); + insert_pos += 1; + } + } else if let Some(obj) = response.as_object_mut() { + let mut output_items = Vec::new(); + output_items.push(build_mcp_list_tools_item(mcp, server_label)); + output_items.extend(build_executed_mcp_call_items( + &state.conversation_history, + server_label, + )); + obj.insert("output".to_string(), Value::Array(output_items)); + } +} + +// ============================================================================ +// Tool Loop Execution +// ============================================================================ + +/// Execute the tool calling loop +pub(super) async fn execute_tool_loop( + client: &reqwest::Client, + url: &str, + headers: Option<&HeaderMap>, + initial_payload: Value, + original_body: &ResponsesRequest, + active_mcp: &Arc, + config: &McpLoopConfig, +) -> Result { + let mut state = ToolLoopState::new(original_body.input.clone()); + + // Get max_tool_calls from request (None means no user-specified limit) + let max_tool_calls = original_body.max_tool_calls.map(|n| n as usize); + + // Keep initial_payload as base template (already has fields cleaned) + let base_payload = initial_payload.clone(); + let tools_json = base_payload.get("tools").cloned().unwrap_or(json!([])); + let mut current_payload = initial_payload; + + info!( + "Starting tool loop: max_tool_calls={:?}, max_iterations={}", + max_tool_calls, config.max_iterations + ); + + loop { + // Make request to upstream + let request_builder = client.post(url).json(¤t_payload); + let request_builder = if let Some(headers) = headers { + apply_request_headers(headers, request_builder, true) + } else { + request_builder + }; + + let response = request_builder + .send() + .await + .map_err(|e| format!("upstream request failed: {}", e))?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(format!("upstream error {}: {}", status, body)); + } + + let mut response_json = response + .json::() + .await + .map_err(|e| format!("parse response: {}", e))?; + + // Check for function call + if let Some((call_id, tool_name, args_json_str)) = extract_function_call(&response_json) { + state.iteration += 1; + state.total_calls += 1; + + info!( + "Tool loop iteration {}: calling {} (call_id: {})", + state.iteration, tool_name, call_id + ); + + // Check combined limit: use minimum of user's max_tool_calls (if set) and safety max_iterations + let effective_limit = match max_tool_calls { + Some(user_max) => user_max.min(config.max_iterations), + None => config.max_iterations, + }; + + if state.total_calls > effective_limit { + if let Some(user_max) = max_tool_calls { + if state.total_calls > user_max { + warn!("Reached user-specified max_tool_calls limit: {}", user_max); + } else { + warn!( + "Reached safety max_iterations limit: {}", + config.max_iterations + ); + } + } else { + warn!( + "Reached safety max_iterations limit: {}", + config.max_iterations + ); + } + + return build_incomplete_response( + response_json, + state, + "max_tool_calls", + active_mcp, + original_body, + ); + } + + // Execute tool - manager handles parsing and type coercion + debug!( + "Calling MCP tool '{}' with args: {}", + tool_name, args_json_str + ); + let call_result = active_mcp + .call_tool(&tool_name, args_json_str.as_str()) + .await; + + let output_str = match call_result { + Ok(result) => match serde_json::to_string(&result) { + Ok(output) => output, + Err(e) => { + warn!("Failed to serialize tool result: {}", e); + json!({ "error": format!("Serialization error: {}", e) }).to_string() + } + }, + Err(err) => { + warn!("Tool execution failed: {}", err); + // Return error as output, let model decide how to proceed + json!({ "error": format!("tool call failed: {}", err) }).to_string() + } + }; + + // Record the call + state.record_call(call_id, tool_name, args_json_str, output_str); + + // Build resume payload + current_payload = build_resume_payload( + &base_payload, + &state.conversation_history, + &state.original_input, + &tools_json, + false, // is_streaming = false (non-streaming tool loop) + )?; + } else { + // No more tool calls, we're done + info!( + "Tool loop completed: {} iterations, {} total calls", + state.iteration, state.total_calls + ); + + // Inject MCP output items if we executed any tools + if state.total_calls > 0 { + let server_label = original_body + .tools + .as_ref() + .and_then(|tools| { + tools + .iter() + .find(|t| matches!(t.r#type, ResponseToolType::Mcp)) + .and_then(|t| t.server_label.as_deref()) + }) + .unwrap_or("mcp"); + + // Build mcp_list_tools item + let list_tools_item = build_mcp_list_tools_item(active_mcp, server_label); + + // Insert at beginning of output array + if let Some(output_array) = response_json + .get_mut("output") + .and_then(|v| v.as_array_mut()) + { + output_array.insert(0, list_tools_item); + + // Build mcp_call items using helper function + let mcp_call_items = + build_executed_mcp_call_items(&state.conversation_history, server_label); + + // Insert mcp_call items after mcp_list_tools using mutable position + let mut insert_pos = 1; + for item in mcp_call_items { + output_array.insert(insert_pos, item); + insert_pos += 1; + } + } + } + + return Ok(response_json); + } + } +} + +/// Build an incomplete response when limits are exceeded +pub(super) fn build_incomplete_response( + mut response: Value, + state: ToolLoopState, + reason: &str, + active_mcp: &Arc, + original_body: &ResponsesRequest, +) -> Result { + let obj = response + .as_object_mut() + .ok_or_else(|| "response not an object".to_string())?; + + // Set status to completed (not failed - partial success) + obj.insert("status".to_string(), Value::String("completed".to_string())); + + // Set incomplete_details + obj.insert( + "incomplete_details".to_string(), + json!({ "reason": reason }), + ); + + // Convert any function_call in output to mcp_call format + if let Some(output_array) = obj.get_mut("output").and_then(|v| v.as_array_mut()) { + let server_label = original_body + .tools + .as_ref() + .and_then(|tools| { + tools + .iter() + .find(|t| matches!(t.r#type, ResponseToolType::Mcp)) + .and_then(|t| t.server_label.as_deref()) + }) + .unwrap_or("mcp"); + + // Find any function_call items and convert them to mcp_call (incomplete) + let mut mcp_call_items = Vec::new(); + for item in output_array.iter() { + let item_type = item.get("type").and_then(|t| t.as_str()); + if item_type == Some(event_types::ITEM_TYPE_FUNCTION_TOOL_CALL) + || item_type == Some(event_types::ITEM_TYPE_FUNCTION_CALL) + { + let tool_name = item.get("name").and_then(|v| v.as_str()).unwrap_or(""); + let args = item + .get("arguments") + .and_then(|v| v.as_str()) + .unwrap_or("{}"); + + // Mark as incomplete - not executed + let mcp_call_item = build_mcp_call_item( + tool_name, + args, + "", // No output - wasn't executed + server_label, + false, // Not successful + Some("Not executed - response stopped due to limit"), + ); + mcp_call_items.push(mcp_call_item); + } + } + + // Add mcp_list_tools and executed mcp_call items at the beginning + if state.total_calls > 0 || !mcp_call_items.is_empty() { + let list_tools_item = build_mcp_list_tools_item(active_mcp, server_label); + output_array.insert(0, list_tools_item); + + // Add mcp_call items for executed calls using helper + let executed_items = + build_executed_mcp_call_items(&state.conversation_history, server_label); + + let mut insert_pos = 1; + for item in executed_items { + output_array.insert(insert_pos, item); + insert_pos += 1; + } + + // Add incomplete mcp_call items + for item in mcp_call_items { + output_array.insert(insert_pos, item); + insert_pos += 1; + } + } + } + + // Add warning to metadata + if let Some(metadata_val) = obj.get_mut("metadata") { + if let Some(metadata_obj) = metadata_val.as_object_mut() { + if let Some(mcp_val) = metadata_obj.get_mut("mcp") { + if let Some(mcp_obj) = mcp_val.as_object_mut() { + mcp_obj.insert( + "truncation_warning".to_string(), + Value::String(format!( + "Loop terminated at {} iterations, {} total calls (reason: {})", + state.iteration, state.total_calls, reason + )), + ); + } + } + } + } + + Ok(response) +} + +// ============================================================================ +// Output Item Builders +// ============================================================================ + +/// Build an mcp_list_tools output item +pub(super) fn build_mcp_list_tools_item(mcp: &Arc, server_label: &str) -> Value { + let tools = mcp.list_tools(); + let tools_json: Vec = tools + .iter() + .map(|t| { + json!({ + "name": t.name, + "description": t.description, + "input_schema": Value::Object((*t.input_schema).clone()), + "annotations": { + "read_only": false + } + }) + }) + .collect(); + + json!({ + "id": generate_id("mcpl"), + "type": event_types::ITEM_TYPE_MCP_LIST_TOOLS, + "server_label": server_label, + "tools": tools_json + }) +} + +/// Build an mcp_call output item +pub(super) fn build_mcp_call_item( + tool_name: &str, + arguments: &str, + output: &str, + server_label: &str, + success: bool, + error: Option<&str>, +) -> Value { + json!({ + "id": generate_id("mcp"), + "type": event_types::ITEM_TYPE_MCP_CALL, + "status": if success { "completed" } else { "failed" }, + "approval_request_id": Value::Null, + "arguments": arguments, + "error": error, + "name": tool_name, + "output": output, + "server_label": server_label + }) +} + +/// Helper function to build mcp_call items from executed tool calls in conversation history +pub(super) fn build_executed_mcp_call_items( + conversation_history: &[Value], + server_label: &str, +) -> Vec { + let mut mcp_call_items = Vec::new(); + + for item in conversation_history { + if item.get("type").and_then(|t| t.as_str()) == Some(event_types::ITEM_TYPE_FUNCTION_CALL) { + let call_id = item.get("call_id").and_then(|v| v.as_str()).unwrap_or(""); + let tool_name = item.get("name").and_then(|v| v.as_str()).unwrap_or(""); + let args = item + .get("arguments") + .and_then(|v| v.as_str()) + .unwrap_or("{}"); + + // Find corresponding output + let output_item = conversation_history.iter().find(|o| { + o.get("type").and_then(|t| t.as_str()) == Some("function_call_output") + && o.get("call_id").and_then(|c| c.as_str()) == Some(call_id) + }); + + let output_str = output_item + .and_then(|o| o.get("output").and_then(|v| v.as_str())) + .unwrap_or("{}"); + + // Check if output contains error by parsing JSON + let is_error = serde_json::from_str::(output_str) + .map(|v| v.get("error").is_some()) + .unwrap_or(false); + + let mcp_call_item = build_mcp_call_item( + tool_name, + args, + output_str, + server_label, + !is_error, + if is_error { + Some("Tool execution failed") + } else { + None + }, + ); + mcp_call_items.push(mcp_call_item); + } + } + + mcp_call_items +} + +// ============================================================================ +// Helper Functions +// ============================================================================ + +/// Extract function call from a response +pub(super) fn extract_function_call(resp: &Value) -> Option<(String, String, String)> { + let output = resp.get("output")?.as_array()?; + for item in output { + let obj = item.as_object()?; + let t = obj.get("type")?.as_str()?; + if t == event_types::ITEM_TYPE_FUNCTION_TOOL_CALL + || t == event_types::ITEM_TYPE_FUNCTION_CALL + { + let call_id = obj + .get("call_id") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + .or_else(|| { + obj.get("id") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + })?; + let name = obj.get("name")?.as_str()?.to_string(); + let arguments = obj.get("arguments")?.as_str()?.to_string(); + return Some((call_id, name, arguments)); + } + } + None +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/mod.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/mod.rs new file mode 100644 index 00000000..e01af396 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/mod.rs @@ -0,0 +1,18 @@ +//! OpenAI-compatible router implementation +//! +//! This module provides OpenAI-compatible API routing with support for: +//! - Streaming and non-streaming responses +//! - MCP (Model Context Protocol) tool calling +//! - Response storage and conversation management +//! - Multi-turn tool execution loops +//! - SSE (Server-Sent Events) streaming + +pub mod conversations; +pub mod mcp; +mod responses; +mod router; +mod streaming; +mod utils; + +// Re-export the main router type for external use +pub use router::OpenAIRouter; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/responses.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/responses.rs new file mode 100644 index 00000000..b9d6a925 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/responses.rs @@ -0,0 +1,317 @@ +//! Response storage, patching, and extraction utilities + +use std::collections::HashMap; + +use serde_json::{json, Value}; +use tracing::warn; + +use super::utils::event_types; +use crate::{ + data_connector::{ResponseId, StoredResponse}, + protocols::responses::{ResponseToolType, ResponsesRequest}, +}; + +// ============================================================================ +// Response Storage Operations +// ============================================================================ + +/// Build a StoredResponse from response JSON and original request +pub(super) fn build_stored_response( + response_json: &Value, + original_body: &ResponsesRequest, +) -> StoredResponse { + let mut stored_response = StoredResponse::new(None); + + // Initialize empty arrays - will be populated by persist_items_with_storages + stored_response.input = Value::Array(vec![]); + stored_response.output = Value::Array(vec![]); + + stored_response.instructions = response_json + .get("instructions") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + .or_else(|| original_body.instructions.clone()); + + stored_response.model = response_json + .get("model") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + .or_else(|| Some(original_body.model.clone())); + + if let Some(safety_identifier) = original_body.user.clone() { + stored_response.safety_identifier = Some(safety_identifier); + } + + // Set conversation id from request if provided + if let Some(conv_id) = original_body.conversation.clone() { + stored_response.conversation_id = Some(conv_id); + } + + stored_response.metadata = response_json + .get("metadata") + .and_then(|v| v.as_object()) + .map(|m| { + m.iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect::>() + }) + .unwrap_or_else(|| original_body.metadata.clone().unwrap_or_default()); + + stored_response.previous_response_id = response_json + .get("previous_response_id") + .and_then(|v| v.as_str()) + .map(ResponseId::from) + .or_else(|| { + original_body + .previous_response_id + .as_ref() + .map(|id| ResponseId::from(id.as_str())) + }); + + if let Some(id_str) = response_json.get("id").and_then(|v| v.as_str()) { + stored_response.id = ResponseId::from(id_str); + } + + stored_response.raw_response = response_json.clone(); + + stored_response +} + +// ============================================================================ +// Response JSON Patching +// ============================================================================ + +/// Patch streaming response JSON with metadata from original request +pub(super) fn patch_streaming_response_json( + response_json: &mut Value, + original_body: &ResponsesRequest, + original_previous_response_id: Option<&str>, +) { + if let Some(obj) = response_json.as_object_mut() { + if let Some(prev_id) = original_previous_response_id { + let should_insert = obj + .get("previous_response_id") + .map(|v| v.is_null() || v.as_str().map(|s| s.is_empty()).unwrap_or(false)) + .unwrap_or(true); + if should_insert { + obj.insert( + "previous_response_id".to_string(), + Value::String(prev_id.to_string()), + ); + } + } + + if !obj.contains_key("instructions") + || obj + .get("instructions") + .map(|v| v.is_null()) + .unwrap_or(false) + { + if let Some(instructions) = &original_body.instructions { + obj.insert( + "instructions".to_string(), + Value::String(instructions.clone()), + ); + } + } + + if !obj.contains_key("metadata") + || obj.get("metadata").map(|v| v.is_null()).unwrap_or(false) + { + if let Some(metadata) = &original_body.metadata { + let metadata_map: serde_json::Map = metadata + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + obj.insert("metadata".to_string(), Value::Object(metadata_map)); + } + } + + obj.insert( + "store".to_string(), + Value::Bool(original_body.store.unwrap_or(false)), + ); + + if obj + .get("model") + .and_then(|v| v.as_str()) + .map(|s| s.is_empty()) + .unwrap_or(true) + { + obj.insert( + "model".to_string(), + Value::String(original_body.model.clone()), + ); + } + + if obj + .get("safety_identifier") + .map(|v| v.is_null()) + .unwrap_or(false) + { + if let Some(safety_identifier) = &original_body.user { + obj.insert( + "safety_identifier".to_string(), + Value::String(safety_identifier.clone()), + ); + } + } + + // Attach conversation id for client response if present (final aggregated JSON) + if let Some(conv_id) = original_body.conversation.clone() { + obj.insert("conversation".to_string(), json!({ "id": conv_id })); + } + } +} + +/// Rewrite streaming SSE block to include metadata from original request +pub(super) fn rewrite_streaming_block( + block: &str, + original_body: &ResponsesRequest, + original_previous_response_id: Option<&str>, +) -> Option { + let trimmed = block.trim(); + if trimmed.is_empty() { + return None; + } + + let mut data_lines: Vec = Vec::new(); + + for line in trimmed.lines() { + if line.starts_with("data:") { + data_lines.push(line.trim_start_matches("data:").trim_start().to_string()); + } + } + + if data_lines.is_empty() { + return None; + } + + let payload = data_lines.join("\n"); + let mut parsed: Value = match serde_json::from_str(&payload) { + Ok(value) => value, + Err(err) => { + warn!("Failed to parse streaming JSON payload: {}", err); + return None; + } + }; + + let event_type = parsed + .get("type") + .and_then(|v| v.as_str()) + .unwrap_or_default(); + + let should_patch = matches!( + event_type, + event_types::RESPONSE_CREATED + | event_types::RESPONSE_IN_PROGRESS + | event_types::RESPONSE_COMPLETED + ); + + if !should_patch { + return None; + } + + let mut changed = false; + if let Some(response_obj) = parsed.get_mut("response").and_then(|v| v.as_object_mut()) { + let desired_store = Value::Bool(original_body.store.unwrap_or(false)); + if response_obj.get("store") != Some(&desired_store) { + response_obj.insert("store".to_string(), desired_store); + changed = true; + } + + if let Some(prev_id) = original_previous_response_id { + let needs_previous = response_obj + .get("previous_response_id") + .map(|v| v.is_null() || v.as_str().map(|s| s.is_empty()).unwrap_or(false)) + .unwrap_or(true); + + if needs_previous { + response_obj.insert( + "previous_response_id".to_string(), + Value::String(prev_id.to_string()), + ); + changed = true; + } + } + + // Attach conversation id into streaming event response content with ordering + if let Some(conv_id) = original_body.conversation.clone() { + response_obj.insert("conversation".to_string(), json!({ "id": conv_id })); + changed = true; + } + } + + if !changed { + return None; + } + + let new_payload = match serde_json::to_string(&parsed) { + Ok(json) => json, + Err(err) => { + warn!("Failed to serialize modified streaming payload: {}", err); + return None; + } + }; + + let mut rebuilt_lines = Vec::new(); + let mut data_written = false; + for line in trimmed.lines() { + if line.starts_with("data:") { + if !data_written { + rebuilt_lines.push(format!("data: {}", new_payload)); + data_written = true; + } + } else { + rebuilt_lines.push(line.to_string()); + } + } + + if !data_written { + rebuilt_lines.push(format!("data: {}", new_payload)); + } + + Some(rebuilt_lines.join("\n")) +} + +/// Mask function tools as MCP tools in response for client +pub(super) fn mask_tools_as_mcp(resp: &mut Value, original_body: &ResponsesRequest) { + let mcp_tool = original_body.tools.as_ref().and_then(|tools| { + tools + .iter() + .find(|t| matches!(t.r#type, ResponseToolType::Mcp) && t.server_url.is_some()) + }); + let Some(t) = mcp_tool else { + return; + }; + + let mut m = serde_json::Map::new(); + m.insert("type".to_string(), Value::String("mcp".to_string())); + if let Some(label) = &t.server_label { + m.insert("server_label".to_string(), Value::String(label.clone())); + } + if let Some(url) = &t.server_url { + m.insert("server_url".to_string(), Value::String(url.clone())); + } + if let Some(desc) = &t.server_description { + m.insert( + "server_description".to_string(), + Value::String(desc.clone()), + ); + } + if let Some(req) = &t.require_approval { + m.insert("require_approval".to_string(), Value::String(req.clone())); + } + if let Some(allowed) = &t.allowed_tools { + m.insert( + "allowed_tools".to_string(), + Value::Array(allowed.iter().map(|s| Value::String(s.clone())).collect()), + ); + } + + if let Some(obj) = resp.as_object_mut() { + obj.insert("tools".to_string(), Value::Array(vec![Value::Object(m)])); + obj.entry("tool_choice") + .or_insert(Value::String("auto".to_string())); + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/router.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/router.rs new file mode 100644 index 00000000..01ec0b62 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/router.rs @@ -0,0 +1,1261 @@ +//! OpenAI router - main coordinator that delegates to specialized modules + +use std::{ + any::Any, + collections::HashSet, + sync::{atomic::AtomicBool, Arc}, + time::{Duration, Instant}, +}; + +use axum::{ + body::Body, + extract::Request, + http::{header::CONTENT_TYPE, HeaderMap, HeaderValue, StatusCode}, + response::{IntoResponse, Response}, + Json, +}; +use dashmap::DashMap; +use futures_util::StreamExt; +use once_cell::sync::Lazy; +use serde_json::{json, to_value, Value}; +use tokio::sync::mpsc; +use tokio_stream::wrappers::UnboundedReceiverStream; +use tracing::warn; + +// Import from sibling modules +use super::conversations::{ + create_conversation, create_conversation_items, delete_conversation, delete_conversation_item, + get_conversation, get_conversation_item, list_conversation_items, persist_conversation_items, + update_conversation, +}; +use super::{ + mcp::{ + ensure_request_mcp_client, execute_tool_loop, prepare_mcp_payload_for_streaming, + McpLoopConfig, + }, + responses::{mask_tools_as_mcp, patch_streaming_response_json}, + streaming::handle_streaming_response, + utils::{apply_provider_headers, extract_auth_header, probe_endpoint_for_model}, +}; +use crate::{ + core::{CircuitBreaker, CircuitBreakerConfig as CoreCircuitBreakerConfig}, + data_connector::{ + ConversationId, ConversationItemStorage, ConversationStorage, ListParams, ResponseId, + ResponseStorage, SortOrder, + }, + mcp::McpManager, + protocols::{ + chat::ChatCompletionRequest, + classify::ClassifyRequest, + completion::CompletionRequest, + embedding::EmbeddingRequest, + generate::GenerateRequest, + rerank::RerankRequest, + responses::{ + generate_id, ResponseContentPart, ResponseInput, ResponseInputOutputItem, + ResponsesGetParams, ResponsesRequest, + }, + }, + routers::header_utils::apply_request_headers, +}; + +// ============================================================================ +// OpenAIRouter Struct +// ============================================================================ + +/// Fields specific to SGLang that should be stripped when forwarding to OpenAI-compatible endpoints +static SGLANG_FIELDS: Lazy> = Lazy::new(|| { + HashSet::from([ + "request_id", + "priority", + "top_k", + "min_p", + "min_tokens", + "regex", + "ebnf", + "stop_token_ids", + "no_stop_trim", + "ignore_eos", + "continue_final_message", + "skip_special_tokens", + "lora_path", + "session_params", + "separate_reasoning", + "stream_reasoning", + "chat_template_kwargs", + "return_hidden_states", + "repetition_penalty", + "sampling_seed", + ]) +}); + +/// Cached endpoint information +#[derive(Clone, Debug)] +struct CachedEndpoint { + url: String, + cached_at: Instant, +} + +/// Router for OpenAI backend +pub struct OpenAIRouter { + /// HTTP client for upstream OpenAI-compatible API + client: reqwest::Client, + /// Multiple OpenAI-compatible API endpoints (OpenAI, xAI, etc.) + worker_urls: Vec, + /// Model cache: model_id -> endpoint URL + model_cache: Arc>, + /// Circuit breaker + circuit_breaker: CircuitBreaker, + /// Health status + healthy: AtomicBool, + /// Response storage for managing conversation history + response_storage: Arc, + /// Conversation storage backend + conversation_storage: Arc, + /// Conversation item storage backend + conversation_item_storage: Arc, + /// MCP manager (handles both static and dynamic servers) + mcp_manager: Arc, +} + +impl std::fmt::Debug for OpenAIRouter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OpenAIRouter") + .field("worker_urls", &self.worker_urls) + .field("healthy", &self.healthy) + .finish() + } +} + +impl OpenAIRouter { + /// Maximum number of conversation items to attach as input when a conversation is provided + const MAX_CONVERSATION_HISTORY_ITEMS: usize = 100; + + /// Model discovery cache TTL (1 hour) + const MODEL_CACHE_TTL_SECS: u64 = 3600; + + /// Create a new OpenAI router + pub async fn new( + worker_urls: Vec, + ctx: &Arc, + ) -> Result { + // Use HTTP client from AppContext + let client = ctx.client.clone(); + + // Normalize URLs (remove trailing slashes) + let worker_urls: Vec = worker_urls + .into_iter() + .map(|url| url.trim_end_matches('/').to_string()) + .collect(); + + // Convert circuit breaker config from AppContext + let cb = &ctx.router_config.circuit_breaker; + let core_cb_config = CoreCircuitBreakerConfig { + failure_threshold: cb.failure_threshold, + success_threshold: cb.success_threshold, + timeout_duration: Duration::from_secs(cb.timeout_duration_secs), + window_duration: Duration::from_secs(cb.window_duration_secs), + }; + + let circuit_breaker = CircuitBreaker::with_config(core_cb_config); + + // Get MCP manager from AppContext (must be initialized) + let mcp_manager = ctx + .mcp_manager + .get() + .ok_or_else(|| "MCP manager not initialized in AppContext".to_string())? + .clone(); + + Ok(Self { + client, + worker_urls, + model_cache: Arc::new(DashMap::new()), + circuit_breaker, + healthy: AtomicBool::new(true), + response_storage: ctx.response_storage.clone(), + conversation_storage: ctx.conversation_storage.clone(), + conversation_item_storage: ctx.conversation_item_storage.clone(), + mcp_manager, + }) + } + + /// Discover which endpoint has the model + async fn find_endpoint_for_model( + &self, + model_id: &str, + auth_header: Option<&str>, + ) -> Result { + // Single endpoint - fast path + if self.worker_urls.len() == 1 { + return Ok(self.worker_urls[0].clone()); + } + + // Check cache + if let Some(entry) = self.model_cache.get(model_id) { + if entry.cached_at.elapsed() < Duration::from_secs(Self::MODEL_CACHE_TTL_SECS) { + return Ok(entry.url.clone()); + } + } + + // Probe all endpoints in parallel + let mut handles = vec![]; + let model = model_id.to_string(); + let auth = auth_header.map(|s| s.to_string()); + + for url in &self.worker_urls { + let handle = tokio::spawn(probe_endpoint_for_model( + self.client.clone(), + url.clone(), + model.clone(), + auth.clone(), + )); + handles.push(handle); + } + + // Return first successful endpoint + for handle in handles { + if let Ok(Ok(url)) = handle.await { + // Cache it + self.model_cache.insert( + model_id.to_string(), + CachedEndpoint { + url: url.clone(), + cached_at: Instant::now(), + }, + ); + return Ok(url); + } + } + + // Model not found on any endpoint + Err(( + StatusCode::NOT_FOUND, + Json(json!({ + "error": { + "message": format!("Model '{}' not found on any endpoint", model_id), + "type": "model_not_found", + } + })), + ) + .into_response()) + } + + /// Handle non-streaming response with optional MCP tool loop + async fn handle_non_streaming_response( + &self, + url: String, + headers: Option<&HeaderMap>, + mut payload: Value, + original_body: &ResponsesRequest, + original_previous_response_id: Option, + ) -> Response { + // Check if MCP is active for this request + // Ensure dynamic client is created if needed + if let Some(ref tools) = original_body.tools { + ensure_request_mcp_client(&self.mcp_manager, tools.as_slice()).await; + } + + // Use the tool loop if the manager has any tools available (static or dynamic). + let active_mcp = if self.mcp_manager.list_tools().is_empty() { + None + } else { + Some(&self.mcp_manager) + }; + + let mut response_json: Value; + + // If MCP is active, execute tool loop + if let Some(mcp) = active_mcp { + let config = McpLoopConfig::default(); + + // Transform MCP tools to function tools + prepare_mcp_payload_for_streaming(&mut payload, mcp); + + match execute_tool_loop( + &self.client, + &url, + headers, + payload, + original_body, + mcp, + &config, + ) + .await + { + Ok(resp) => response_json = resp, + Err(err) => { + self.circuit_breaker.record_failure(); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"error": {"message": err}})), + ) + .into_response(); + } + } + } else { + // No MCP - simple request + + let mut request_builder = self.client.post(&url).json(&payload); + if let Some(h) = headers { + request_builder = apply_request_headers(h, request_builder, true); + } + + let response = match request_builder.send().await { + Ok(r) => r, + Err(e) => { + self.circuit_breaker.record_failure(); + tracing::error!( + url = %url, + error = %e, + "Failed to forward request to OpenAI" + ); + return ( + StatusCode::BAD_GATEWAY, + format!("Failed to forward request to OpenAI: {}", e), + ) + .into_response(); + } + }; + + if !response.status().is_success() { + self.circuit_breaker.record_failure(); + let status = StatusCode::from_u16(response.status().as_u16()) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + let body = response.text().await.unwrap_or_default(); + return (status, body).into_response(); + } + + response_json = match response.json::().await { + Ok(r) => r, + Err(e) => { + self.circuit_breaker.record_failure(); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to parse upstream response: {}", e), + ) + .into_response(); + } + }; + + self.circuit_breaker.record_success(); + } + + // Patch response with metadata + mask_tools_as_mcp(&mut response_json, original_body); + patch_streaming_response_json( + &mut response_json, + original_body, + original_previous_response_id.as_deref(), + ); + + // Always persist conversation items and response (even without conversation) + if let Err(err) = persist_conversation_items( + self.conversation_storage.clone(), + self.conversation_item_storage.clone(), + self.response_storage.clone(), + &response_json, + original_body, + ) + .await + { + warn!("Failed to persist conversation items: {}", err); + } + + (StatusCode::OK, Json(response_json)).into_response() + } +} + +// ============================================================================ +// RouterTrait Implementation +// ============================================================================ + +#[async_trait::async_trait] +impl crate::routers::RouterTrait for OpenAIRouter { + fn as_any(&self) -> &dyn Any { + self + } + + async fn health_generate(&self, _req: Request) -> Response { + // Check all endpoints in parallel - only healthy if ALL are healthy + if self.worker_urls.is_empty() { + return (StatusCode::SERVICE_UNAVAILABLE, "No endpoints configured").into_response(); + } + + let mut handles = vec![]; + for url in &self.worker_urls { + let url = url.clone(); + let client = self.client.clone(); + + let handle = tokio::spawn(async move { + let probe_url = format!("{}/v1/models", url); + match client + .get(&probe_url) + .timeout(Duration::from_secs(2)) + .send() + .await + { + Ok(resp) => { + let code = resp.status(); + // Treat success and auth-required as healthy (endpoint reachable) + if code.is_success() || code.as_u16() == 401 || code.as_u16() == 403 { + Ok(()) + } else { + Err(format!("Endpoint {} returned status {}", url, code)) + } + } + Err(e) => Err(format!("Endpoint {} error: {}", url, e)), + } + }); + + handles.push(handle); + } + + // Collect all results + let mut errors = Vec::new(); + for handle in handles { + match handle.await { + Ok(Ok(())) => (), + Ok(Err(e)) => errors.push(e), + Err(e) => errors.push(format!("Task join error: {}", e)), + } + } + + if errors.is_empty() { + (StatusCode::OK, "OK").into_response() + } else { + ( + StatusCode::SERVICE_UNAVAILABLE, + format!("Some endpoints unhealthy: {}", errors.join(", ")), + ) + .into_response() + } + } + + async fn get_server_info(&self, _req: Request) -> Response { + let info = json!({ + "router_type": "openai", + "workers": self.worker_urls.len(), + "worker_urls": &self.worker_urls + }); + (StatusCode::OK, info.to_string()).into_response() + } + + async fn get_models(&self, req: Request) -> Response { + // Aggregate models from all endpoints + if self.worker_urls.is_empty() { + return (StatusCode::SERVICE_UNAVAILABLE, "No endpoints configured").into_response(); + } + + let headers = req.headers(); + let auth = headers + .get("authorization") + .or_else(|| headers.get("Authorization")); + + // Query all endpoints in parallel + let mut handles = vec![]; + for url in &self.worker_urls { + let url = url.clone(); + let client = self.client.clone(); + let auth = auth.cloned(); + + let handle = tokio::spawn(async move { + let models_url = format!("{}/v1/models", url); + let req = client.get(&models_url); + + // Apply provider-specific headers (handles Anthropic, xAI, OpenAI, etc.) + let req = apply_provider_headers(req, &url, auth.as_ref()); + + match req.send().await { + Ok(res) => { + if res.status().is_success() { + match res.json::().await { + Ok(json) => Ok(json), + Err(e) => { + tracing::warn!( + "Failed to parse models response from '{}': {}", + url, + e + ); + Err(()) + } + } + } else { + tracing::warn!( + "Getting models from '{}' failed with status: {}", + url, + res.status() + ); + Err(()) + } + } + Err(e) => { + tracing::warn!("Request to get models from '{}' failed: {}", url, e); + Err(()) + } + } + }); + + handles.push(handle); + } + + // Collect all model lists + let mut all_models = Vec::new(); + for handle in handles { + if let Ok(Ok(json)) = handle.await { + if let Some(data) = json.get("data").and_then(|v| v.as_array()) { + all_models.extend_from_slice(data); + } + } + } + + // Return aggregated models + let response_json = json!({ + "object": "list", + "data": all_models + }); + + (StatusCode::OK, Json(response_json)).into_response() + } + + async fn get_model_info(&self, _req: Request) -> Response { + // Not directly supported without model param; return 501 + ( + StatusCode::NOT_IMPLEMENTED, + "get_model_info not implemented for OpenAI router", + ) + .into_response() + } + + async fn route_generate( + &self, + _headers: Option<&HeaderMap>, + _body: &GenerateRequest, + _model_id: Option<&str>, + ) -> Response { + // Generate endpoint is SGLang-specific, not supported for OpenAI backend + ( + StatusCode::NOT_IMPLEMENTED, + "Generate endpoint not supported for OpenAI backend", + ) + .into_response() + } + + async fn route_chat( + &self, + headers: Option<&HeaderMap>, + body: &ChatCompletionRequest, + _model_id: Option<&str>, + ) -> Response { + if !self.circuit_breaker.can_execute() { + return (StatusCode::SERVICE_UNAVAILABLE, "Circuit breaker open").into_response(); + } + + // Extract auth header + let auth = extract_auth_header(headers); + + // Find endpoint for model + let base_url = match self + .find_endpoint_for_model(body.model.as_str(), auth) + .await + { + Ok(url) => url, + Err(response) => return response, + }; + + // Serialize request body, removing SGLang-only fields + let mut payload = match to_value(body) { + Ok(v) => v, + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + format!("Failed to serialize request: {}", e), + ) + .into_response(); + } + }; + if let Some(obj) = payload.as_object_mut() { + // Always remove SGLang-specific fields (unsupported by OpenAI) + obj.retain(|k, _| !SGLANG_FIELDS.contains(&k.as_str())); + // Remove logprobs if false (Gemini don't accept it) + if obj.get("logprobs").and_then(|v| v.as_bool()) == Some(false) { + obj.remove("logprobs"); + } + } + + let url = format!("{}/v1/chat/completions", base_url); + let mut req = self.client.post(&url).json(&payload); + + // Forward Authorization header if provided + if let Some(h) = headers { + if let Some(auth) = h.get("authorization").or_else(|| h.get("Authorization")) { + req = req.header("Authorization", auth); + } + } + + // Accept SSE when stream=true + if body.stream { + req = req.header("Accept", "text/event-stream"); + } + + let resp = match req.send().await { + Ok(r) => r, + Err(e) => { + self.circuit_breaker.record_failure(); + return ( + StatusCode::SERVICE_UNAVAILABLE, + format!("Failed to contact upstream: {}", e), + ) + .into_response(); + } + }; + + let status = StatusCode::from_u16(resp.status().as_u16()) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + + if !body.stream { + // Capture Content-Type before consuming response body + let content_type = resp.headers().get(CONTENT_TYPE).cloned(); + match resp.bytes().await { + Ok(body) => { + self.circuit_breaker.record_success(); + let mut response = Response::new(Body::from(body)); + *response.status_mut() = status; + if let Some(ct) = content_type { + response.headers_mut().insert(CONTENT_TYPE, ct); + } + response + } + Err(e) => { + self.circuit_breaker.record_failure(); + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to read response: {}", e), + ) + .into_response() + } + } + } else { + // Stream SSE bytes to client + let stream = resp.bytes_stream(); + let (tx, rx) = mpsc::unbounded_channel(); + tokio::spawn(async move { + let mut s = stream; + while let Some(chunk) = s.next().await { + match chunk { + Ok(bytes) => { + if tx.send(Ok(bytes)).is_err() { + break; + } + } + Err(e) => { + let _ = tx.send(Err(format!("Stream error: {}", e))); + break; + } + } + } + }); + let mut response = Response::new(Body::from_stream(UnboundedReceiverStream::new(rx))); + *response.status_mut() = status; + response + .headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static("text/event-stream")); + response + } + } + + async fn route_completion( + &self, + _headers: Option<&HeaderMap>, + _body: &CompletionRequest, + _model_id: Option<&str>, + ) -> Response { + // Completion endpoint not implemented for OpenAI backend + ( + StatusCode::NOT_IMPLEMENTED, + "Completion endpoint not implemented for OpenAI backend", + ) + .into_response() + } + + async fn route_responses( + &self, + headers: Option<&HeaderMap>, + body: &ResponsesRequest, + model_id: Option<&str>, + ) -> Response { + // Extract auth header + let auth = extract_auth_header(headers); + + // Find endpoint for model (use model_id if provided, otherwise use body.model) + let model = model_id.unwrap_or(body.model.as_str()); + let base_url = match self.find_endpoint_for_model(model, auth).await { + Ok(url) => url, + Err(response) => return response, + }; + + let url = format!("{}/v1/responses", base_url); + + // Clone the body for validation and logic, but we'll build payload differently + let mut request_body = body.clone(); + if let Some(model) = model_id { + request_body.model = model.to_string(); + } + // Do not forward conversation field upstream; retain for local persistence only + request_body.conversation = None; + + // Store the original previous_response_id for the response + let original_previous_response_id = request_body.previous_response_id.clone(); + + // Handle previous_response_id by loading prior context + let mut conversation_items: Option> = None; + if let Some(prev_id_str) = request_body.previous_response_id.clone() { + let prev_id = ResponseId::from(prev_id_str.as_str()); + match self + .response_storage + .get_response_chain(&prev_id, None) + .await + { + Ok(chain) => { + let mut items = Vec::new(); + for stored in chain.responses.iter() { + // Convert input items from stored input (which is now a JSON array) + if let Some(input_arr) = stored.input.as_array() { + for item in input_arr { + match serde_json::from_value::( + item.clone(), + ) { + Ok(input_item) => { + items.push(input_item); + } + Err(e) => { + warn!( + "Failed to deserialize stored input item: {}. Item: {}", + e, item + ); + } + } + } + } + + // Convert output items from stored output (which is now a JSON array) + if let Some(output_arr) = stored.output.as_array() { + for item in output_arr { + match serde_json::from_value::( + item.clone(), + ) { + Ok(output_item) => { + items.push(output_item); + } + Err(e) => { + warn!("Failed to deserialize stored output item: {}. Item: {}", e, item); + } + } + } + } + } + conversation_items = Some(items); + request_body.previous_response_id = None; + } + Err(e) => { + warn!( + "Failed to load previous response chain for {}: {}", + prev_id_str, e + ); + } + } + } + + // Handle conversation by loading history + if let Some(conv_id_str) = body.conversation.clone() { + let conv_id = ConversationId::from(conv_id_str.as_str()); + + // Verify conversation exists + if let Ok(None) = self.conversation_storage.get_conversation(&conv_id).await { + return ( + StatusCode::NOT_FOUND, + Json(json!({"error": "Conversation not found"})), + ) + .into_response(); + } + + // Load conversation history (ascending order for chronological context) + let params = ListParams { + limit: Self::MAX_CONVERSATION_HISTORY_ITEMS, + order: SortOrder::Asc, + after: None, + }; + + match self + .conversation_item_storage + .list_items(&conv_id, params) + .await + { + Ok(stored_items) => { + let mut items: Vec = Vec::new(); + for item in stored_items.into_iter() { + // Include messages, function calls, and function call outputs + // Skip reasoning items as they're internal processing details + match item.item_type.as_str() { + "message" => { + match serde_json::from_value::>( + item.content.clone(), + ) { + Ok(content_parts) => { + items.push(ResponseInputOutputItem::Message { + id: item.id.0.clone(), + role: item + .role + .clone() + .unwrap_or_else(|| "user".to_string()), + content: content_parts, + status: item.status.clone(), + }); + } + Err(e) => { + tracing::error!( + "Failed to deserialize message content: {}", + e + ); + } + } + } + "function_call" => { + // The entire function_call item is stored in content field + match serde_json::from_value::( + item.content.clone(), + ) { + Ok(func_call) => items.push(func_call), + Err(e) => { + tracing::error!( + "Failed to deserialize function_call: {}", + e + ); + } + } + } + "function_call_output" => { + // The entire function_call_output item is stored in content field + tracing::debug!( + "Loading function_call_output from DB - content: {}", + serde_json::to_string_pretty(&item.content) + .unwrap_or_else(|_| "failed to serialize".to_string()) + ); + match serde_json::from_value::( + item.content.clone(), + ) { + Ok(func_output) => { + tracing::debug!( + "Successfully deserialized function_call_output" + ); + items.push(func_output); + } + Err(e) => { + tracing::error!( + "Failed to deserialize function_call_output: {}", + e + ); + } + } + } + "reasoning" => { + // Skip reasoning items - they're internal processing details + } + _ => { + // Skip unknown item types + warn!("Unknown item type in conversation: {}", item.item_type); + } + } + } + + // Append current request + match &request_body.input { + ResponseInput::Text(text) => { + items.push(ResponseInputOutputItem::Message { + id: format!("msg_u_{}", conv_id.0), + role: "user".to_string(), + content: vec![ResponseContentPart::InputText { + text: text.clone(), + }], + status: Some("completed".to_string()), + }); + } + ResponseInput::Items(current_items) => { + // Process all item types, converting SimpleInputMessage to Message + for item in current_items.iter() { + let normalized = + crate::protocols::responses::normalize_input_item(item); + items.push(normalized); + } + } + } + + request_body.input = ResponseInput::Items(items); + } + Err(e) => { + warn!("Failed to load conversation history: {}", e); + } + } + } + + // If we have conversation_items from previous_response_id, use them + if let Some(mut items) = conversation_items { + // Append current request + match &request_body.input { + ResponseInput::Text(text) => { + items.push(ResponseInputOutputItem::Message { + id: format!( + "msg_u_{}", + original_previous_response_id + .as_ref() + .unwrap_or(&"new".to_string()) + ), + role: "user".to_string(), + content: vec![ResponseContentPart::InputText { text: text.clone() }], + status: Some("completed".to_string()), + }); + } + ResponseInput::Items(current_items) => { + // Process all item types, converting SimpleInputMessage to Message + for item in current_items.iter() { + let normalized = crate::protocols::responses::normalize_input_item(item); + items.push(normalized); + } + } + } + + request_body.input = ResponseInput::Items(items); + } + + // Always set store=false for upstream (we store internally) + request_body.store = Some(false); + // Filter out reasoning items from input - they're internal processing details + if let ResponseInput::Items(ref mut items) = request_body.input { + items.retain(|item| !matches!(item, ResponseInputOutputItem::Reasoning { .. })); + } + + // Convert to JSON and strip SGLang-specific fields + let mut payload = match to_value(&request_body) { + Ok(v) => v, + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + format!("Failed to serialize request: {}", e), + ) + .into_response(); + } + }; + + // Remove SGLang-specific fields only + if let Some(obj) = payload.as_object_mut() { + // Remove SGLang-specific fields (not part of OpenAI API) + obj.retain(|k, _| !SGLANG_FIELDS.contains(&k.as_str())); + // XAI (Grok models) requires special handling of input items + // Check if model is a Grok model + let is_grok_model = obj + .get("model") + .and_then(|v| v.as_str()) + .map(|m| m.starts_with("grok")) + .unwrap_or(false); + + if is_grok_model { + // XAI doesn't support the OPENAI item type input: https://platform.openai.com/docs/api-reference/responses/create#responses-create-input-input-item-list-item + // To Achieve XAI compatibility, strip extra fields from input messages (id, status) + // XAI doesn't support output_text as type for content with role of assistant + // so normalize content types: output_text -> input_text + if let Some(input_arr) = obj.get_mut("input").and_then(Value::as_array_mut) { + for item_obj in input_arr.iter_mut().filter_map(Value::as_object_mut) { + // Remove fields not universally supported + item_obj.remove("id"); + item_obj.remove("status"); + + // Normalize content types to input_text (xAI compatibility) + if let Some(content_arr) = + item_obj.get_mut("content").and_then(Value::as_array_mut) + { + for content_obj in + content_arr.iter_mut().filter_map(Value::as_object_mut) + { + // Change output_text to input_text + if content_obj.get("type").and_then(Value::as_str) + == Some("output_text") + { + content_obj.insert( + "type".to_string(), + Value::String("input_text".to_string()), + ); + } + } + } + } + } + } + } + + // Delegate to streaming or non-streaming handler + if body.stream.unwrap_or(false) { + handle_streaming_response( + &self.client, + &self.circuit_breaker, + Some(&self.mcp_manager), + self.response_storage.clone(), + self.conversation_storage.clone(), + self.conversation_item_storage.clone(), + url, + headers, + payload, + body, + original_previous_response_id, + ) + .await + } else { + self.handle_non_streaming_response( + url, + headers, + payload, + body, + original_previous_response_id, + ) + .await + } + } + + async fn get_response( + &self, + _headers: Option<&HeaderMap>, + response_id: &str, + _params: &ResponsesGetParams, + ) -> Response { + let id = ResponseId::from(response_id); + match self.response_storage.get_response(&id).await { + Ok(Some(stored)) => { + let mut response_json = stored.raw_response; + if let Some(obj) = response_json.as_object_mut() { + obj.insert("id".to_string(), json!(id.0)); + } + (StatusCode::OK, Json(response_json)).into_response() + } + Ok(None) => ( + StatusCode::NOT_FOUND, + Json(json!({"error": "Response not found"})), + ) + .into_response(), + Err(e) => ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ "error": format!("Failed to get response: {}", e) })), + ) + .into_response(), + } + } + + async fn cancel_response(&self, _headers: Option<&HeaderMap>, _response_id: &str) -> Response { + ( + StatusCode::NOT_IMPLEMENTED, + "Cancel response not implemented for OpenAI router", + ) + .into_response() + } + + async fn list_response_input_items( + &self, + _headers: Option<&HeaderMap>, + response_id: &str, + ) -> Response { + let resp_id = ResponseId::from(response_id); + + match self.response_storage.get_response(&resp_id).await { + Ok(Some(stored)) => { + // Extract items from input field (which is a JSON array) + let items = match &stored.input { + Value::Array(arr) => arr.clone(), + _ => vec![], + }; + + // Generate IDs for items if they don't have them + let items_with_ids: Vec = items + .into_iter() + .map(|mut item| { + if item.get("id").is_none() { + // Generate ID if not present using centralized utility + if let Some(obj) = item.as_object_mut() { + obj.insert("id".to_string(), json!(generate_id("msg"))); + } + } + item + }) + .collect(); + + let response_body = json!({ + "object": "list", + "data": items_with_ids, + "first_id": items_with_ids.first().and_then(|v| v.get("id").and_then(|i| i.as_str())), + "last_id": items_with_ids.last().and_then(|v| v.get("id").and_then(|i| i.as_str())), + "has_more": false + }); + + (StatusCode::OK, Json(response_body)).into_response() + } + Ok(None) => ( + StatusCode::NOT_FOUND, + Json(json!({ + "error": { + "message": format!("No response found with id '{}'", response_id), + "type": "invalid_request_error", + "param": Value::Null, + "code": "not_found" + } + })), + ) + .into_response(), + Err(e) => { + warn!("Failed to retrieve input items for {}: {}", response_id, e); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": { + "message": format!("Failed to retrieve input items: {}", e), + "type": "internal_error", + "param": Value::Null, + "code": "storage_error" + } + })), + ) + .into_response() + } + } + } + + async fn route_embeddings( + &self, + _headers: Option<&HeaderMap>, + _body: &EmbeddingRequest, + _model_id: Option<&str>, + ) -> Response { + (StatusCode::NOT_IMPLEMENTED, "Embeddings not supported").into_response() + } + + async fn route_classify( + &self, + _headers: Option<&HeaderMap>, + _body: &ClassifyRequest, + _model_id: Option<&str>, + ) -> Response { + (StatusCode::NOT_IMPLEMENTED, "Classify not supported").into_response() + } + + async fn route_rerank( + &self, + _headers: Option<&HeaderMap>, + _body: &RerankRequest, + _model_id: Option<&str>, + ) -> Response { + (StatusCode::NOT_IMPLEMENTED, "Rerank not supported").into_response() + } + + async fn create_conversation(&self, _headers: Option<&HeaderMap>, body: &Value) -> Response { + create_conversation(&self.conversation_storage, body.clone()).await + } + + async fn get_conversation( + &self, + _headers: Option<&HeaderMap>, + conversation_id: &str, + ) -> Response { + get_conversation(&self.conversation_storage, conversation_id).await + } + + async fn update_conversation( + &self, + _headers: Option<&HeaderMap>, + conversation_id: &str, + body: &Value, + ) -> Response { + update_conversation(&self.conversation_storage, conversation_id, body.clone()).await + } + + async fn delete_conversation( + &self, + _headers: Option<&HeaderMap>, + conversation_id: &str, + ) -> Response { + delete_conversation(&self.conversation_storage, conversation_id).await + } + + async fn list_conversation_items( + &self, + _headers: Option<&HeaderMap>, + conversation_id: &str, + limit: Option, + order: Option, + after: Option, + ) -> Response { + let mut query_params = std::collections::HashMap::new(); + query_params.insert("limit".to_string(), limit.unwrap_or(100).to_string()); + if let Some(after_val) = after { + if !after_val.is_empty() { + query_params.insert("after".to_string(), after_val); + } + } + if let Some(order_val) = order { + query_params.insert("order".to_string(), order_val); + } + + list_conversation_items( + &self.conversation_storage, + &self.conversation_item_storage, + conversation_id, + query_params, + ) + .await + } + + async fn create_conversation_items( + &self, + _headers: Option<&HeaderMap>, + conversation_id: &str, + body: &Value, + ) -> Response { + create_conversation_items( + &self.conversation_storage, + &self.conversation_item_storage, + conversation_id, + body.clone(), + ) + .await + } + + async fn get_conversation_item( + &self, + _headers: Option<&HeaderMap>, + conversation_id: &str, + item_id: &str, + include: Option>, + ) -> Response { + get_conversation_item( + &self.conversation_storage, + &self.conversation_item_storage, + conversation_id, + item_id, + include, + ) + .await + } + + async fn delete_conversation_item( + &self, + _headers: Option<&HeaderMap>, + conversation_id: &str, + item_id: &str, + ) -> Response { + delete_conversation_item( + &self.conversation_storage, + &self.conversation_item_storage, + conversation_id, + item_id, + ) + .await + } + + fn router_type(&self) -> &'static str { + "openai" + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/streaming.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/streaming.rs new file mode 100644 index 00000000..bebda8eb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/streaming.rs @@ -0,0 +1,1550 @@ +//! Streaming response handling for OpenAI-compatible responses +//! +//! This module handles all streaming-related functionality including: +//! - SSE (Server-Sent Events) parsing and forwarding +//! - Streaming response accumulation for persistence +//! - Tool call detection and interception during streaming +//! - MCP tool execution loops within streaming responses +//! - Event transformation and output index remapping + +use std::{borrow::Cow, io, sync::Arc}; + +use axum::{ + body::Body, + http::{header::CONTENT_TYPE, HeaderMap, HeaderValue, StatusCode}, + response::{IntoResponse, Response}, +}; +use bytes::Bytes; +use futures_util::StreamExt; +use serde_json::{json, Value}; +use tokio::sync::mpsc; +use tokio_stream::wrappers::UnboundedReceiverStream; +use tracing::warn; + +// Import from sibling modules +use super::conversations::persist_conversation_items; +use super::{ + mcp::{ + build_resume_payload, ensure_request_mcp_client, execute_streaming_tool_calls, + inject_mcp_metadata_streaming, prepare_mcp_payload_for_streaming, + send_mcp_list_tools_events, McpLoopConfig, ToolLoopState, + }, + responses::{mask_tools_as_mcp, patch_streaming_response_json, rewrite_streaming_block}, + utils::{event_types, FunctionCallInProgress, OutputIndexMapper, StreamAction}, +}; +use crate::{ + data_connector::{ConversationItemStorage, ConversationStorage, ResponseStorage}, + protocols::responses::{ResponseToolType, ResponsesRequest}, + routers::header_utils::{apply_request_headers, preserve_response_headers}, +}; + +// ============================================================================ +// Streaming Response Accumulator +// ============================================================================ + +/// Helper that parses SSE frames from the OpenAI responses stream and +/// accumulates enough information to persist the final response locally. +pub(super) struct StreamingResponseAccumulator { + /// The initial `response.created` payload (if emitted). + initial_response: Option, + /// The final `response.completed` payload (if emitted). + completed_response: Option, + /// Collected output items keyed by the upstream output index, used when + /// a final response payload is absent and we need to synthesize one. + output_items: Vec<(usize, Value)>, + /// Captured error payload (if the upstream stream fails midway). + encountered_error: Option, +} + +impl StreamingResponseAccumulator { + pub fn new() -> Self { + Self { + initial_response: None, + completed_response: None, + output_items: Vec::new(), + encountered_error: None, + } + } + + /// Feed the accumulator with the next SSE chunk. + pub fn ingest_block(&mut self, block: &str) { + if block.trim().is_empty() { + return; + } + self.process_block(block); + } + + /// Consume the accumulator and produce the best-effort final response value. + pub fn into_final_response(mut self) -> Option { + if self.completed_response.is_some() { + return self.completed_response; + } + + self.build_fallback_response() + } + + pub fn encountered_error(&self) -> Option<&Value> { + self.encountered_error.as_ref() + } + + pub fn original_response_id(&self) -> Option<&str> { + self.initial_response + .as_ref() + .and_then(|response| response.get("id")) + .and_then(|id| id.as_str()) + } + + pub fn snapshot_final_response(&self) -> Option { + if let Some(resp) = &self.completed_response { + return Some(resp.clone()); + } + self.build_fallback_response_snapshot() + } + + fn build_fallback_response_snapshot(&self) -> Option { + let mut response = self.initial_response.clone()?; + + if let Some(obj) = response.as_object_mut() { + obj.insert("status".to_string(), Value::String("completed".to_string())); + + let mut output_items = self.output_items.clone(); + output_items.sort_by_key(|(index, _)| *index); + let outputs: Vec = output_items.into_iter().map(|(_, item)| item).collect(); + obj.insert("output".to_string(), Value::Array(outputs)); + } + + Some(response) + } + + fn process_block(&mut self, block: &str) { + let trimmed = block.trim(); + if trimmed.is_empty() { + return; + } + + let mut event_name: Option = None; + let mut data_lines: Vec = Vec::new(); + + for line in trimmed.lines() { + if let Some(rest) = line.strip_prefix("event:") { + event_name = Some(rest.trim().to_string()); + } else if let Some(rest) = line.strip_prefix("data:") { + data_lines.push(rest.trim_start().to_string()); + } + } + + let data_payload = data_lines.join("\n"); + if data_payload.is_empty() { + return; + } + + self.handle_event(event_name.as_deref(), &data_payload); + } + + fn handle_event(&mut self, event_name: Option<&str>, data_payload: &str) { + let parsed: Value = match serde_json::from_str(data_payload) { + Ok(value) => value, + Err(err) => { + warn!("Failed to parse streaming event JSON: {}", err); + return; + } + }; + + let event_type = event_name + .map(|s| s.to_string()) + .or_else(|| { + parsed + .get("type") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + }) + .unwrap_or_default(); + + match event_type.as_str() { + event_types::RESPONSE_CREATED => { + if self.initial_response.is_none() { + if let Some(response) = parsed.get("response") { + self.initial_response = Some(response.clone()); + } + } + } + event_types::RESPONSE_COMPLETED => { + if let Some(response) = parsed.get("response") { + self.completed_response = Some(response.clone()); + } + } + event_types::OUTPUT_ITEM_DONE => { + if let (Some(index), Some(item)) = ( + parsed + .get("output_index") + .and_then(|v| v.as_u64()) + .map(|v| v as usize), + parsed.get("item"), + ) { + self.output_items.push((index, item.clone())); + } + } + "response.error" => { + self.encountered_error = Some(parsed); + } + _ => {} + } + } + + fn build_fallback_response(&mut self) -> Option { + let mut response = self.initial_response.clone()?; + + if let Some(obj) = response.as_object_mut() { + obj.insert("status".to_string(), Value::String("completed".to_string())); + + self.output_items.sort_by_key(|(index, _)| *index); + let outputs: Vec = self + .output_items + .iter() + .map(|(_, item)| item.clone()) + .collect(); + obj.insert("output".to_string(), Value::Array(outputs)); + } + + Some(response) + } +} + +// ============================================================================ +// Streaming Tool Handler +// ============================================================================ + +/// Handles streaming responses with MCP tool call interception +pub(super) struct StreamingToolHandler { + /// Accumulator for response persistence + pub accumulator: StreamingResponseAccumulator, + /// Function calls being built from deltas + pub pending_calls: Vec, + /// Track if we're currently in a function call + in_function_call: bool, + /// Manage output_index remapping so they increment per item + output_index_mapper: OutputIndexMapper, + /// Original response id captured from the first response.created event + pub original_response_id: Option, +} + +impl StreamingToolHandler { + pub fn with_starting_index(start: usize) -> Self { + Self { + accumulator: StreamingResponseAccumulator::new(), + pending_calls: Vec::new(), + in_function_call: false, + output_index_mapper: OutputIndexMapper::with_start(start), + original_response_id: None, + } + } + + pub fn ensure_output_index(&mut self, upstream_index: usize) -> usize { + self.output_index_mapper.ensure_mapping(upstream_index) + } + + pub fn mapped_output_index(&self, upstream_index: usize) -> Option { + self.output_index_mapper.lookup(upstream_index) + } + + pub fn allocate_synthetic_output_index(&mut self) -> usize { + self.output_index_mapper.allocate_synthetic() + } + + pub fn next_output_index(&self) -> usize { + self.output_index_mapper.next_index() + } + + pub fn original_response_id(&self) -> Option<&str> { + self.original_response_id + .as_deref() + .or_else(|| self.accumulator.original_response_id()) + } + + pub fn snapshot_final_response(&self) -> Option { + self.accumulator.snapshot_final_response() + } + + /// Process an SSE event and determine what action to take + pub fn process_event(&mut self, event_name: Option<&str>, data: &str) -> StreamAction { + // Always feed to accumulator for storage + self.accumulator.ingest_block(&format!( + "{}data: {}", + event_name + .map(|n| format!("event: {}\n", n)) + .unwrap_or_default(), + data + )); + + let parsed: Value = match serde_json::from_str(data) { + Ok(v) => v, + Err(_) => return StreamAction::Forward, + }; + + let event_type = event_name + .map(|s| s.to_string()) + .or_else(|| { + parsed + .get("type") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + }) + .unwrap_or_default(); + + match event_type.as_str() { + event_types::RESPONSE_CREATED => { + if self.original_response_id.is_none() { + if let Some(response_obj) = parsed.get("response").and_then(|v| v.as_object()) { + if let Some(id) = response_obj.get("id").and_then(|v| v.as_str()) { + self.original_response_id = Some(id.to_string()); + } + } + } + StreamAction::Forward + } + event_types::RESPONSE_COMPLETED => StreamAction::Forward, + event_types::OUTPUT_ITEM_ADDED => { + if let Some(idx) = parsed.get("output_index").and_then(|v| v.as_u64()) { + self.ensure_output_index(idx as usize); + } + + // Check if this is a function_call item being added + if let Some(item) = parsed.get("item") { + if let Some(item_type) = item.get("type").and_then(|v| v.as_str()) { + if item_type == event_types::ITEM_TYPE_FUNCTION_CALL + || item_type == event_types::ITEM_TYPE_FUNCTION_TOOL_CALL + { + match parsed.get("output_index").and_then(|v| v.as_u64()) { + Some(idx) => { + let output_index = idx as usize; + let assigned_index = self.ensure_output_index(output_index); + let call_id = + item.get("call_id").and_then(|v| v.as_str()).unwrap_or(""); + let name = + item.get("name").and_then(|v| v.as_str()).unwrap_or(""); + + // Create or update the function call + let call = self.get_or_create_call(output_index, item); + call.call_id = call_id.to_string(); + call.name = name.to_string(); + call.assigned_output_index = Some(assigned_index); + + self.in_function_call = true; + } + None => { + warn!( + "Missing output_index in function_call added event, \ + forwarding without processing for tool execution" + ); + } + } + } + } + } + StreamAction::Forward + } + event_types::FUNCTION_CALL_ARGUMENTS_DELTA => { + // Accumulate arguments for the function call + if let Some(output_index) = parsed + .get("output_index") + .and_then(|v| v.as_u64()) + .map(|v| v as usize) + { + let assigned_index = self.ensure_output_index(output_index); + if let Some(delta) = parsed.get("delta").and_then(|v| v.as_str()) { + if let Some(call) = self + .pending_calls + .iter_mut() + .find(|c| c.output_index == output_index) + { + call.arguments_buffer.push_str(delta); + if let Some(obfuscation) = + parsed.get("obfuscation").and_then(|v| v.as_str()) + { + call.last_obfuscation = Some(obfuscation.to_string()); + } + if call.assigned_output_index.is_none() { + call.assigned_output_index = Some(assigned_index); + } + } + } + } + StreamAction::Forward + } + event_types::FUNCTION_CALL_ARGUMENTS_DONE => { + // Function call arguments complete - check if ready to execute + if let Some(output_index) = parsed + .get("output_index") + .and_then(|v| v.as_u64()) + .map(|v| v as usize) + { + let assigned_index = self.ensure_output_index(output_index); + if let Some(call) = self + .pending_calls + .iter_mut() + .find(|c| c.output_index == output_index) + { + if call.assigned_output_index.is_none() { + call.assigned_output_index = Some(assigned_index); + } + } + } + + if self.has_complete_calls() { + StreamAction::ExecuteTools + } else { + StreamAction::Forward + } + } + event_types::OUTPUT_ITEM_DELTA => self.process_output_delta(&parsed), + event_types::OUTPUT_ITEM_DONE => { + // Check if we have complete function calls ready to execute + if let Some(output_index) = parsed + .get("output_index") + .and_then(|v| v.as_u64()) + .map(|v| v as usize) + { + self.ensure_output_index(output_index); + } + + if self.has_complete_calls() { + StreamAction::ExecuteTools + } else { + StreamAction::Forward + } + } + _ => StreamAction::Forward, + } + } + + /// Process output delta events to detect and accumulate function calls + fn process_output_delta(&mut self, event: &Value) -> StreamAction { + let output_index = event + .get("output_index") + .and_then(|v| v.as_u64()) + .map(|v| v as usize) + .unwrap_or(0); + + let assigned_index = self.ensure_output_index(output_index); + + let delta = match event.get("delta") { + Some(d) => d, + None => return StreamAction::Forward, + }; + + // Check if this is a function call delta + let item_type = delta.get("type").and_then(|v| v.as_str()); + + if item_type == Some(event_types::ITEM_TYPE_FUNCTION_TOOL_CALL) + || item_type == Some(event_types::ITEM_TYPE_FUNCTION_CALL) + { + self.in_function_call = true; + + // Get or create function call for this output index + let call = self.get_or_create_call(output_index, delta); + call.assigned_output_index = Some(assigned_index); + + // Accumulate call_id if present + if let Some(call_id) = delta.get("call_id").and_then(|v| v.as_str()) { + call.call_id = call_id.to_string(); + } + + // Accumulate name if present + if let Some(name) = delta.get("name").and_then(|v| v.as_str()) { + call.name.push_str(name); + } + + // Accumulate arguments if present + if let Some(args) = delta.get("arguments").and_then(|v| v.as_str()) { + call.arguments_buffer.push_str(args); + } + + if let Some(obfuscation) = delta.get("obfuscation").and_then(|v| v.as_str()) { + call.last_obfuscation = Some(obfuscation.to_string()); + } + + // Buffer this event, don't forward to client + return StreamAction::Buffer; + } + + // Forward non-function-call events + StreamAction::Forward + } + + fn get_or_create_call( + &mut self, + output_index: usize, + delta: &Value, + ) -> &mut FunctionCallInProgress { + // Find existing call for this output index + if let Some(pos) = self + .pending_calls + .iter() + .position(|c| c.output_index == output_index) + { + return &mut self.pending_calls[pos]; + } + + // Create new call + let call_id = delta + .get("call_id") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + + let mut call = FunctionCallInProgress::new(call_id, output_index); + if let Some(obfuscation) = delta.get("obfuscation").and_then(|v| v.as_str()) { + call.last_obfuscation = Some(obfuscation.to_string()); + } + + self.pending_calls.push(call); + self.pending_calls + .last_mut() + .expect("Just pushed to pending_calls, must have at least one element") + } + + fn has_complete_calls(&self) -> bool { + !self.pending_calls.is_empty() && self.pending_calls.iter().all(|c| c.is_complete()) + } + + pub fn take_pending_calls(&mut self) -> Vec { + std::mem::take(&mut self.pending_calls) + } +} + +// ============================================================================ +// SSE Parsing +// ============================================================================ + +/// Parse an SSE block into event name and data +/// +/// Returns borrowed strings when possible to avoid allocations in hot paths. +/// Only allocates when multiple data lines need to be joined. +pub(super) fn parse_sse_block(block: &str) -> (Option<&str>, Cow<'_, str>) { + let mut event_name: Option<&str> = None; + let mut data_lines: Vec<&str> = Vec::new(); + + for line in block.lines() { + if let Some(rest) = line.strip_prefix("event:") { + event_name = Some(rest.trim()); + } else if let Some(rest) = line.strip_prefix("data:") { + data_lines.push(rest.trim_start()); + } + } + + let data = if data_lines.len() == 1 { + Cow::Borrowed(data_lines[0]) + } else { + Cow::Owned(data_lines.join("\n")) + }; + + (event_name, data) +} + +// ============================================================================ +// Event Transformation and Forwarding +// ============================================================================ + +/// Apply all transformations to event data in-place (rewrite + transform) +/// Optimized to parse JSON only once instead of multiple times +/// Returns true if any changes were made +pub(super) fn apply_event_transformations_inplace( + parsed_data: &mut Value, + server_label: &str, + original_request: &ResponsesRequest, + previous_response_id: Option<&str>, +) -> bool { + let mut changed = false; + + // 1. Apply rewrite_streaming_block logic (store, previous_response_id, tools masking) + let event_type = parsed_data + .get("type") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + .unwrap_or_default(); + + let should_patch = matches!( + event_type.as_str(), + event_types::RESPONSE_CREATED + | event_types::RESPONSE_IN_PROGRESS + | event_types::RESPONSE_COMPLETED + ); + + if should_patch { + if let Some(response_obj) = parsed_data + .get_mut("response") + .and_then(|v| v.as_object_mut()) + { + let desired_store = Value::Bool(original_request.store.unwrap_or(false)); + if response_obj.get("store") != Some(&desired_store) { + response_obj.insert("store".to_string(), desired_store); + changed = true; + } + + if let Some(prev_id) = previous_response_id { + let needs_previous = response_obj + .get("previous_response_id") + .map(|v| v.is_null() || v.as_str().map(|s| s.is_empty()).unwrap_or(false)) + .unwrap_or(true); + + if needs_previous { + response_obj.insert( + "previous_response_id".to_string(), + Value::String(prev_id.to_string()), + ); + changed = true; + } + } + + // Mask tools from function to MCP format (optimized without cloning) + if response_obj.get("tools").is_some() { + let requested_mcp = original_request + .tools + .as_ref() + .map(|tools| { + tools + .iter() + .any(|t| matches!(t.r#type, ResponseToolType::Mcp)) + }) + .unwrap_or(false); + + if requested_mcp { + if let Some(mcp_tools) = build_mcp_tools_value(original_request) { + response_obj.insert("tools".to_string(), mcp_tools); + response_obj + .entry("tool_choice".to_string()) + .or_insert(Value::String("auto".to_string())); + changed = true; + } + } + } + } + } + + // 2. Apply transform_streaming_event logic (function_call → mcp_call) + match event_type.as_str() { + event_types::OUTPUT_ITEM_ADDED | event_types::OUTPUT_ITEM_DONE => { + if let Some(item) = parsed_data.get_mut("item") { + if let Some(item_type) = item.get("type").and_then(|v| v.as_str()) { + if item_type == event_types::ITEM_TYPE_FUNCTION_CALL + || item_type == event_types::ITEM_TYPE_FUNCTION_TOOL_CALL + { + item["type"] = json!(event_types::ITEM_TYPE_MCP_CALL); + item["server_label"] = json!(server_label); + + // Transform ID from fc_* to mcp_* + if let Some(id) = item.get("id").and_then(|v| v.as_str()) { + if let Some(stripped) = id.strip_prefix("fc_") { + let new_id = format!("mcp_{}", stripped); + item["id"] = json!(new_id); + } + } + + changed = true; + } + } + } + } + event_types::FUNCTION_CALL_ARGUMENTS_DONE => { + parsed_data["type"] = json!(event_types::MCP_CALL_ARGUMENTS_DONE); + + // Transform item_id from fc_* to mcp_* + if let Some(item_id) = parsed_data.get("item_id").and_then(|v| v.as_str()) { + if let Some(stripped) = item_id.strip_prefix("fc_") { + let new_id = format!("mcp_{}", stripped); + parsed_data["item_id"] = json!(new_id); + } + } + + changed = true; + } + _ => {} + } + + changed +} + +/// Helper to build MCP tools value +fn build_mcp_tools_value(original_body: &ResponsesRequest) -> Option { + let tools = original_body.tools.as_ref()?; + let mcp_tool = tools + .iter() + .find(|t| matches!(t.r#type, ResponseToolType::Mcp) && t.server_url.is_some())?; + + let tools_array = vec![json!({ + "type": "mcp", + "server_label": mcp_tool.server_label, + "server_url": mcp_tool.server_url + })]; + + Some(Value::Array(tools_array)) +} + +/// Forward and transform a streaming event to the client +/// Returns false if client disconnected +#[allow(clippy::too_many_arguments)] +pub(super) fn forward_streaming_event( + raw_block: &str, + event_name: Option<&str>, + data: &str, + handler: &mut StreamingToolHandler, + tx: &mpsc::UnboundedSender>, + server_label: &str, + original_request: &ResponsesRequest, + previous_response_id: Option<&str>, + sequence_number: &mut u64, +) -> bool { + // Skip individual function_call_arguments.delta events - we'll send them as one + if event_name == Some(event_types::FUNCTION_CALL_ARGUMENTS_DELTA) { + return true; + } + + // Parse JSON data once (optimized!) + let mut parsed_data: Value = match serde_json::from_str(data) { + Ok(v) => v, + Err(_) => { + // If parsing fails, forward raw block as-is + let chunk_to_send = format!("{}\n\n", raw_block); + return tx.send(Ok(Bytes::from(chunk_to_send))).is_ok(); + } + }; + + let event_type = event_name + .or_else(|| parsed_data.get("type").and_then(|v| v.as_str())) + .unwrap_or(""); + + if event_type == event_types::RESPONSE_COMPLETED { + return true; + } + + // Check if this is function_call_arguments.done - need to send buffered args first + let mut mapped_output_index: Option = None; + + if event_name == Some(event_types::FUNCTION_CALL_ARGUMENTS_DONE) { + if let Some(output_index) = parsed_data + .get("output_index") + .and_then(|v| v.as_u64()) + .map(|v| v as usize) + { + let assigned_index = handler + .mapped_output_index(output_index) + .unwrap_or(output_index); + mapped_output_index = Some(assigned_index); + + if let Some(call) = handler + .pending_calls + .iter() + .find(|c| c.output_index == output_index) + { + let arguments_value = if call.arguments_buffer.is_empty() { + "{}".to_string() + } else { + call.arguments_buffer.clone() + }; + + // Make sure the done event carries full arguments + parsed_data["arguments"] = Value::String(arguments_value.clone()); + + // Get item_id and transform it + let item_id = parsed_data + .get("item_id") + .and_then(|v| v.as_str()) + .unwrap_or(""); + let mcp_item_id = if let Some(stripped) = item_id.strip_prefix("fc_") { + format!("mcp_{}", stripped) + } else { + item_id.to_string() + }; + + // Emit a synthetic MCP arguments delta event before the done event + let mut delta_event = json!({ + "type": event_types::MCP_CALL_ARGUMENTS_DELTA, + "sequence_number": *sequence_number, + "output_index": assigned_index, + "item_id": mcp_item_id, + "delta": arguments_value, + }); + + if let Some(obfuscation) = call.last_obfuscation.as_ref() { + if let Some(obj) = delta_event.as_object_mut() { + obj.insert( + "obfuscation".to_string(), + Value::String(obfuscation.clone()), + ); + } + } else if let Some(obfuscation) = parsed_data.get("obfuscation").cloned() { + if let Some(obj) = delta_event.as_object_mut() { + obj.insert("obfuscation".to_string(), obfuscation); + } + } + + let delta_block = format!( + "event: {}\ndata: {}\n\n", + event_types::MCP_CALL_ARGUMENTS_DELTA, + delta_event + ); + if tx.send(Ok(Bytes::from(delta_block))).is_err() { + return false; + } + + *sequence_number += 1; + } + } + } + + // Remap output_index (if present) so downstream sees sequential indices + if mapped_output_index.is_none() { + if let Some(output_index) = parsed_data + .get("output_index") + .and_then(|v| v.as_u64()) + .map(|v| v as usize) + { + mapped_output_index = handler.mapped_output_index(output_index); + } + } + + if let Some(mapped) = mapped_output_index { + parsed_data["output_index"] = json!(mapped); + } + + // Apply all transformations in-place (single parse/serialize!) + apply_event_transformations_inplace( + &mut parsed_data, + server_label, + original_request, + previous_response_id, + ); + + if let Some(response_obj) = parsed_data + .get_mut("response") + .and_then(|v| v.as_object_mut()) + { + if let Some(original_id) = handler.original_response_id() { + response_obj.insert("id".to_string(), Value::String(original_id.to_string())); + } + } + + // Update sequence number if present in the event + if parsed_data.get("sequence_number").is_some() { + parsed_data["sequence_number"] = json!(*sequence_number); + *sequence_number += 1; + } + + // Serialize once + let final_data = match serde_json::to_string(&parsed_data) { + Ok(s) => s, + Err(_) => { + // Serialization failed, forward original + let chunk_to_send = format!("{}\n\n", raw_block); + return tx.send(Ok(Bytes::from(chunk_to_send))).is_ok(); + } + }; + + // Rebuild SSE block with potentially transformed event name + let mut final_block = String::new(); + if let Some(evt) = event_name { + // Update event name for function_call_arguments events + if evt == event_types::FUNCTION_CALL_ARGUMENTS_DELTA { + final_block.push_str(&format!( + "event: {}\n", + event_types::MCP_CALL_ARGUMENTS_DELTA + )); + } else if evt == event_types::FUNCTION_CALL_ARGUMENTS_DONE { + final_block.push_str(&format!( + "event: {}\n", + event_types::MCP_CALL_ARGUMENTS_DONE + )); + } else { + final_block.push_str(&format!("event: {}\n", evt)); + } + } + final_block.push_str(&format!("data: {}", final_data)); + + let chunk_to_send = format!("{}\n\n", final_block); + if tx.send(Ok(Bytes::from(chunk_to_send))).is_err() { + return false; + } + + // After sending output_item.added for mcp_call, inject mcp_call.in_progress event + if event_name == Some(event_types::OUTPUT_ITEM_ADDED) { + if let Some(item) = parsed_data.get("item") { + if item.get("type").and_then(|v| v.as_str()) == Some(event_types::ITEM_TYPE_MCP_CALL) { + // Already transformed to mcp_call + if let (Some(item_id), Some(output_index)) = ( + item.get("id").and_then(|v| v.as_str()), + parsed_data.get("output_index").and_then(|v| v.as_u64()), + ) { + let in_progress_event = json!({ + "type": event_types::MCP_CALL_IN_PROGRESS, + "sequence_number": *sequence_number, + "output_index": output_index, + "item_id": item_id + }); + *sequence_number += 1; + let in_progress_block = format!( + "event: {}\ndata: {}\n\n", + event_types::MCP_CALL_IN_PROGRESS, + in_progress_event + ); + if tx.send(Ok(Bytes::from(in_progress_block))).is_err() { + return false; + } + } + } + } + } + + true +} + +/// Send final response.completed event to client +/// Returns false if client disconnected +#[allow(clippy::too_many_arguments)] +pub(super) fn send_final_response_event( + handler: &StreamingToolHandler, + tx: &mpsc::UnboundedSender>, + sequence_number: &mut u64, + state: &ToolLoopState, + active_mcp: Option<&Arc>, + original_request: &ResponsesRequest, + previous_response_id: Option<&str>, + server_label: &str, +) -> bool { + let mut final_response = match handler.snapshot_final_response() { + Some(resp) => resp, + None => { + warn!("Final response snapshot unavailable; skipping synthetic completion event"); + return true; + } + }; + + if let Some(original_id) = handler.original_response_id() { + if let Some(obj) = final_response.as_object_mut() { + obj.insert("id".to_string(), Value::String(original_id.to_string())); + } + } + + if let Some(mcp) = active_mcp { + inject_mcp_metadata_streaming(&mut final_response, state, mcp, server_label); + } + + mask_tools_as_mcp(&mut final_response, original_request); + patch_streaming_response_json(&mut final_response, original_request, previous_response_id); + + if let Some(obj) = final_response.as_object_mut() { + obj.insert("status".to_string(), Value::String("completed".to_string())); + } + + let completed_payload = json!({ + "type": event_types::RESPONSE_COMPLETED, + "sequence_number": *sequence_number, + "response": final_response + }); + *sequence_number += 1; + + let completed_event = format!( + "event: {}\ndata: {}\n\n", + event_types::RESPONSE_COMPLETED, + completed_payload + ); + tx.send(Ok(Bytes::from(completed_event))).is_ok() +} + +// ============================================================================ +// Main Streaming Handlers +// ============================================================================ + +/// Simple pass-through streaming without MCP interception +#[allow(clippy::too_many_arguments)] +pub(super) async fn handle_simple_streaming_passthrough( + client: &reqwest::Client, + circuit_breaker: &crate::core::CircuitBreaker, + response_storage: Arc, + conversation_storage: Arc, + conversation_item_storage: Arc, + url: String, + headers: Option<&HeaderMap>, + payload: Value, + original_body: &ResponsesRequest, + original_previous_response_id: Option, +) -> Response { + let mut request_builder = client.post(&url).json(&payload); + + if let Some(headers) = headers { + request_builder = apply_request_headers(headers, request_builder, true); + } + + request_builder = request_builder.header("Accept", "text/event-stream"); + + let response = match request_builder.send().await { + Ok(resp) => resp, + Err(err) => { + circuit_breaker.record_failure(); + return ( + StatusCode::BAD_GATEWAY, + format!("Failed to forward request to OpenAI: {}", err), + ) + .into_response(); + } + }; + + let status = response.status(); + let status_code = + StatusCode::from_u16(status.as_u16()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + + if !status.is_success() { + circuit_breaker.record_failure(); + let error_body = response + .text() + .await + .unwrap_or_else(|err| format!("Failed to read upstream error body: {}", err)); + return (status_code, error_body).into_response(); + } + + circuit_breaker.record_success(); + + let preserved_headers = preserve_response_headers(response.headers()); + let mut upstream_stream = response.bytes_stream(); + + let (tx, rx) = mpsc::unbounded_channel::>(); + + let should_store = original_body.store.unwrap_or(false); + let original_request = original_body.clone(); + let persist_needed = original_request.conversation.is_some(); + let previous_response_id = original_previous_response_id.clone(); + + tokio::spawn(async move { + let mut accumulator = StreamingResponseAccumulator::new(); + let mut upstream_failed = false; + let mut receiver_connected = true; + let mut pending = String::new(); + + while let Some(chunk_result) = upstream_stream.next().await { + match chunk_result { + Ok(chunk) => { + let chunk_text = match std::str::from_utf8(&chunk) { + Ok(text) => Cow::Borrowed(text), + Err(_) => Cow::Owned(String::from_utf8_lossy(&chunk).to_string()), + }; + + pending.push_str(&chunk_text.replace("\r\n", "\n")); + + while let Some(pos) = pending.find("\n\n") { + let raw_block = pending[..pos].to_string(); + pending.drain(..pos + 2); + + if raw_block.trim().is_empty() { + continue; + } + + let block_cow = if let Some(modified) = rewrite_streaming_block( + raw_block.as_str(), + &original_request, + previous_response_id.as_deref(), + ) { + Cow::Owned(modified) + } else { + Cow::Borrowed(raw_block.as_str()) + }; + + if should_store || persist_needed { + accumulator.ingest_block(block_cow.as_ref()); + } + + if receiver_connected { + let chunk_to_send = format!("{}\n\n", block_cow); + if tx.send(Ok(Bytes::from(chunk_to_send))).is_err() { + receiver_connected = false; + } + } + + if !receiver_connected && !should_store { + break; + } + } + + if !receiver_connected && !should_store { + break; + } + } + Err(err) => { + upstream_failed = true; + let io_err = io::Error::other(err); + let _ = tx.send(Err(io_err)); + break; + } + } + } + + if (should_store || persist_needed) && !upstream_failed { + if !pending.trim().is_empty() { + accumulator.ingest_block(&pending); + } + let encountered_error = accumulator.encountered_error().cloned(); + if let Some(mut response_json) = accumulator.into_final_response() { + patch_streaming_response_json( + &mut response_json, + &original_request, + previous_response_id.as_deref(), + ); + + // Always persist conversation items and response (even without conversation) + if let Err(err) = persist_conversation_items( + conversation_storage.clone(), + conversation_item_storage.clone(), + response_storage.clone(), + &response_json, + &original_request, + ) + .await + { + warn!("Failed to persist conversation items (stream): {}", err); + } + } else if let Some(error_payload) = encountered_error { + warn!("Upstream streaming error payload: {}", error_payload); + } else { + warn!("Streaming completed without a final response payload"); + } + } + }); + + let body_stream = UnboundedReceiverStream::new(rx); + let mut response = Response::new(Body::from_stream(body_stream)); + *response.status_mut() = status_code; + + let headers_mut = response.headers_mut(); + for (name, value) in preserved_headers.iter() { + headers_mut.insert(name, value.clone()); + } + + if !headers_mut.contains_key(CONTENT_TYPE) { + headers_mut.insert(CONTENT_TYPE, HeaderValue::from_static("text/event-stream")); + } + + response +} + +/// Handle streaming WITH MCP tool call interception and execution +#[allow(clippy::too_many_arguments)] +pub(super) async fn handle_streaming_with_tool_interception( + client: &reqwest::Client, + response_storage: Arc, + conversation_storage: Arc, + conversation_item_storage: Arc, + url: String, + headers: Option<&HeaderMap>, + mut payload: Value, + original_body: &ResponsesRequest, + original_previous_response_id: Option, + active_mcp: &Arc, +) -> Response { + // Transform MCP tools to function tools in payload + prepare_mcp_payload_for_streaming(&mut payload, active_mcp); + + let (tx, rx) = mpsc::unbounded_channel::>(); + let should_store = original_body.store.unwrap_or(false); + let original_request = original_body.clone(); + let persist_needed = original_request.conversation.is_some(); + let previous_response_id = original_previous_response_id.clone(); + + let client_clone = client.clone(); + let url_clone = url.clone(); + let headers_opt = headers.cloned(); + let payload_clone = payload.clone(); + let active_mcp_clone = Arc::clone(active_mcp); + + // Spawn the streaming loop task + tokio::spawn(async move { + let mut state = ToolLoopState::new(original_request.input.clone()); + let loop_config = McpLoopConfig::default(); + let max_tool_calls = original_request.max_tool_calls.map(|n| n as usize); + let tools_json = payload_clone.get("tools").cloned().unwrap_or(json!([])); + let base_payload = payload_clone.clone(); + let mut current_payload = payload_clone; + let mut mcp_list_tools_sent = false; + let mut is_first_iteration = true; + let mut sequence_number: u64 = 0; + let mut next_output_index: usize = 0; + let mut preserved_response_id: Option = None; + + let server_label = original_request + .tools + .as_ref() + .and_then(|tools| { + tools + .iter() + .find(|t| matches!(t.r#type, ResponseToolType::Mcp)) + .and_then(|t| t.server_label.as_deref()) + }) + .unwrap_or("mcp"); + + loop { + // Make streaming request + let mut request_builder = client_clone.post(&url_clone).json(¤t_payload); + if let Some(ref h) = headers_opt { + request_builder = apply_request_headers(h, request_builder, true); + } + request_builder = request_builder.header("Accept", "text/event-stream"); + + let response = match request_builder.send().await { + Ok(r) => r, + Err(e) => { + let error_event = format!( + "event: error\ndata: {{\"error\": {{\"message\": \"{}\"}}}}\n\n", + e + ); + let _ = tx.send(Ok(Bytes::from(error_event))); + return; + } + }; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + let error_event = format!("event: error\ndata: {{\"error\": {{\"message\": \"Upstream error {}: {}\"}}}}\n\n", status, body); + let _ = tx.send(Ok(Bytes::from(error_event))); + return; + } + + // Stream events and check for tool calls + let mut upstream_stream = response.bytes_stream(); + let mut handler = StreamingToolHandler::with_starting_index(next_output_index); + if let Some(ref id) = preserved_response_id { + handler.original_response_id = Some(id.clone()); + } + let mut pending = String::new(); + let mut tool_calls_detected = false; + let mut seen_in_progress = false; + + while let Some(chunk_result) = upstream_stream.next().await { + match chunk_result { + Ok(chunk) => { + let chunk_text = match std::str::from_utf8(&chunk) { + Ok(text) => Cow::Borrowed(text), + Err(_) => Cow::Owned(String::from_utf8_lossy(&chunk).to_string()), + }; + + pending.push_str(&chunk_text.replace("\r\n", "\n")); + + while let Some(pos) = pending.find("\n\n") { + let raw_block = pending[..pos].to_string(); + pending.drain(..pos + 2); + + if raw_block.trim().is_empty() { + continue; + } + + // Parse event + let (event_name, data) = parse_sse_block(&raw_block); + + if data.is_empty() { + continue; + } + + // Process through handler + let action = handler.process_event(event_name, data.as_ref()); + + match action { + StreamAction::Forward => { + // Skip response.created and response.in_progress on subsequent iterations + let should_skip = if !is_first_iteration { + if let Ok(parsed) = + serde_json::from_str::(data.as_ref()) + { + matches!( + parsed.get("type").and_then(|v| v.as_str()), + Some(event_types::RESPONSE_CREATED) + | Some(event_types::RESPONSE_IN_PROGRESS) + ) + } else { + false + } + } else { + false + }; + + if !should_skip { + // Forward the event + if !forward_streaming_event( + &raw_block, + event_name, + data.as_ref(), + &mut handler, + &tx, + server_label, + &original_request, + previous_response_id.as_deref(), + &mut sequence_number, + ) { + // Client disconnected + return; + } + } + + // After forwarding response.in_progress, send mcp_list_tools events (once) + if !seen_in_progress { + if let Ok(parsed) = + serde_json::from_str::(data.as_ref()) + { + if parsed.get("type").and_then(|v| v.as_str()) + == Some(event_types::RESPONSE_IN_PROGRESS) + { + seen_in_progress = true; + if !mcp_list_tools_sent { + let list_tools_index = + handler.allocate_synthetic_output_index(); + if !send_mcp_list_tools_events( + &tx, + &active_mcp_clone, + server_label, + list_tools_index, + &mut sequence_number, + ) { + // Client disconnected + return; + } + mcp_list_tools_sent = true; + } + } + } + } + } + StreamAction::Buffer => { + // Don't forward, just buffer + } + StreamAction::ExecuteTools => { + if !forward_streaming_event( + &raw_block, + event_name, + data.as_ref(), + &mut handler, + &tx, + server_label, + &original_request, + previous_response_id.as_deref(), + &mut sequence_number, + ) { + // Client disconnected + return; + } + tool_calls_detected = true; + break; // Exit stream processing to execute tools + } + } + } + + if tool_calls_detected { + break; + } + } + Err(e) => { + let error_event = format!("event: error\ndata: {{\"error\": {{\"message\": \"Stream error: {}\"}}}}\n\n", e); + let _ = tx.send(Ok(Bytes::from(error_event))); + return; + } + } + } + + next_output_index = handler.next_output_index(); + if let Some(id) = handler.original_response_id().map(|s| s.to_string()) { + preserved_response_id = Some(id); + } + + // If no tool calls, we're done - stream is complete + if !tool_calls_detected { + if !send_final_response_event( + &handler, + &tx, + &mut sequence_number, + &state, + Some(&active_mcp_clone), + &original_request, + previous_response_id.as_deref(), + server_label, + ) { + return; + } + + let final_response_json = if should_store || persist_needed { + handler.accumulator.into_final_response() + } else { + None + }; + + if let Some(mut response_json) = final_response_json { + if let Some(ref id) = preserved_response_id { + if let Some(obj) = response_json.as_object_mut() { + obj.insert("id".to_string(), Value::String(id.clone())); + } + } + inject_mcp_metadata_streaming( + &mut response_json, + &state, + &active_mcp_clone, + server_label, + ); + + mask_tools_as_mcp(&mut response_json, &original_request); + patch_streaming_response_json( + &mut response_json, + &original_request, + previous_response_id.as_deref(), + ); + + // Always persist conversation items and response (even without conversation) + if let Err(err) = persist_conversation_items( + conversation_storage.clone(), + conversation_item_storage.clone(), + response_storage.clone(), + &response_json, + &original_request, + ) + .await + { + warn!( + "Failed to persist conversation items (stream + MCP): {}", + err + ); + } + } + + let _ = tx.send(Ok(Bytes::from("data: [DONE]\n\n"))); + return; + } + + // Execute tools + let pending_calls = handler.take_pending_calls(); + + // Check iteration limit + state.iteration += 1; + state.total_calls += pending_calls.len(); + + let effective_limit = match max_tool_calls { + Some(user_max) => user_max.min(loop_config.max_iterations), + None => loop_config.max_iterations, + }; + + if state.total_calls > effective_limit { + warn!( + "Reached tool call limit during streaming: {}", + effective_limit + ); + let error_event = "event: error\ndata: {\"error\": {\"message\": \"Exceeded max_tool_calls limit\"}}\n\n".to_string(); + let _ = tx.send(Ok(Bytes::from(error_event))); + let _ = tx.send(Ok(Bytes::from("data: [DONE]\n\n"))); + return; + } + + // Execute all pending tool calls + if !execute_streaming_tool_calls( + pending_calls, + &active_mcp_clone, + &tx, + &mut state, + server_label, + &mut sequence_number, + ) + .await + { + // Client disconnected during tool execution + return; + } + + // Build resume payload + match build_resume_payload( + &base_payload, + &state.conversation_history, + &state.original_input, + &tools_json, + true, // is_streaming = true + ) { + Ok(resume_payload) => { + current_payload = resume_payload; + // Mark that we're no longer on the first iteration + is_first_iteration = false; + // Continue loop to make next streaming request + } + Err(e) => { + let error_event = format!("event: error\ndata: {{\"error\": {{\"message\": \"Failed to build resume payload: {}\"}}}}\n\n", e); + let _ = tx.send(Ok(Bytes::from(error_event))); + let _ = tx.send(Ok(Bytes::from("data: [DONE]\n\n"))); + return; + } + } + } + }); + + let body_stream = UnboundedReceiverStream::new(rx); + let mut response = Response::new(Body::from_stream(body_stream)); + *response.status_mut() = StatusCode::OK; + response + .headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static("text/event-stream")); + response +} + +/// Main entry point for handling streaming responses +/// Delegates to simple passthrough or MCP tool interception based on configuration +#[allow(clippy::too_many_arguments)] +pub(super) async fn handle_streaming_response( + client: &reqwest::Client, + circuit_breaker: &crate::core::CircuitBreaker, + mcp_manager: Option<&Arc>, + response_storage: Arc, + conversation_storage: Arc, + conversation_item_storage: Arc, + url: String, + headers: Option<&HeaderMap>, + payload: Value, + original_body: &ResponsesRequest, + original_previous_response_id: Option, +) -> Response { + // Check if MCP is active for this request + // Ensure dynamic client is created if needed + if let (Some(manager), Some(ref tools)) = (mcp_manager, &original_body.tools) { + ensure_request_mcp_client(manager, tools.as_slice()).await; + } + + // Use the tool loop if the manager has any tools available (static or dynamic). + let active_mcp = mcp_manager.and_then(|mgr| { + if mgr.list_tools().is_empty() { + None + } else { + Some(mgr) + } + }); + + // If no MCP is active, use simple pass-through streaming + if active_mcp.is_none() { + return handle_simple_streaming_passthrough( + client, + circuit_breaker, + response_storage, + conversation_storage, + conversation_item_storage, + url, + headers, + payload, + original_body, + original_previous_response_id, + ) + .await; + } + + let active_mcp = active_mcp.unwrap(); + + // MCP is active - transform tools and set up interception + handle_streaming_with_tool_interception( + client, + response_storage, + conversation_storage, + conversation_item_storage, + url, + headers, + payload, + original_body, + original_previous_response_id, + active_mcp, + ) + .await +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/utils.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/utils.rs new file mode 100644 index 00000000..aa1a80b2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/openai/utils.rs @@ -0,0 +1,226 @@ +//! Utility types and constants for OpenAI router + +use std::collections::HashMap; + +use axum::http::{HeaderMap, HeaderValue}; + +// ============================================================================ +// SSE Event Type Constants +// ============================================================================ + +/// SSE event type constants - single source of truth for event type strings +pub(crate) mod event_types { + // Response lifecycle events + pub const RESPONSE_CREATED: &str = "response.created"; + pub const RESPONSE_IN_PROGRESS: &str = "response.in_progress"; + pub const RESPONSE_COMPLETED: &str = "response.completed"; + + // Output item events + pub const OUTPUT_ITEM_ADDED: &str = "response.output_item.added"; + pub const OUTPUT_ITEM_DONE: &str = "response.output_item.done"; + pub const OUTPUT_ITEM_DELTA: &str = "response.output_item.delta"; + + // Function call events + pub const FUNCTION_CALL_ARGUMENTS_DELTA: &str = "response.function_call_arguments.delta"; + pub const FUNCTION_CALL_ARGUMENTS_DONE: &str = "response.function_call_arguments.done"; + + // MCP call events + pub const MCP_CALL_ARGUMENTS_DELTA: &str = "response.mcp_call_arguments.delta"; + pub const MCP_CALL_ARGUMENTS_DONE: &str = "response.mcp_call_arguments.done"; + pub const MCP_CALL_IN_PROGRESS: &str = "response.mcp_call.in_progress"; + pub const MCP_CALL_COMPLETED: &str = "response.mcp_call.completed"; + pub const MCP_LIST_TOOLS_IN_PROGRESS: &str = "response.mcp_list_tools.in_progress"; + pub const MCP_LIST_TOOLS_COMPLETED: &str = "response.mcp_list_tools.completed"; + + // Item types + pub const ITEM_TYPE_FUNCTION_CALL: &str = "function_call"; + pub const ITEM_TYPE_FUNCTION_TOOL_CALL: &str = "function_tool_call"; + pub const ITEM_TYPE_MCP_CALL: &str = "mcp_call"; + pub const ITEM_TYPE_FUNCTION: &str = "function"; + pub const ITEM_TYPE_MCP_LIST_TOOLS: &str = "mcp_list_tools"; +} + +// ============================================================================ +// Stream Action Enum +// ============================================================================ + +/// Action to take based on streaming event processing +#[derive(Debug)] +pub(crate) enum StreamAction { + Forward, // Pass event to client + Buffer, // Accumulate for tool execution + ExecuteTools, // Function call complete, execute now +} + +// ============================================================================ +// Output Index Mapper +// ============================================================================ + +/// Maps upstream output indices to sequential downstream indices +#[derive(Debug, Default)] +pub(crate) struct OutputIndexMapper { + next_index: usize, + // Map upstream output_index -> remapped output_index + assigned: HashMap, +} + +impl OutputIndexMapper { + pub fn with_start(next_index: usize) -> Self { + Self { + next_index, + assigned: HashMap::new(), + } + } + + pub fn ensure_mapping(&mut self, upstream_index: usize) -> usize { + *self.assigned.entry(upstream_index).or_insert_with(|| { + let assigned = self.next_index; + self.next_index += 1; + assigned + }) + } + + pub fn lookup(&self, upstream_index: usize) -> Option { + self.assigned.get(&upstream_index).copied() + } + + pub fn allocate_synthetic(&mut self) -> usize { + let assigned = self.next_index; + self.next_index += 1; + assigned + } + + pub fn next_index(&self) -> usize { + self.next_index + } +} + +// ============================================================================ +// Provider Detection and Header Handling +// ============================================================================ + +/// Extract authorization header from request headers +/// Checks both "authorization" and "Authorization" (case variations) +pub fn extract_auth_header(headers: Option<&HeaderMap>) -> Option<&str> { + headers.and_then(|h| { + h.get("authorization") + .or_else(|| h.get("Authorization")) + .and_then(|v| v.to_str().ok()) + }) +} + +/// API provider types +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ApiProvider { + Anthropic, + Xai, + OpenAi, + Gemini, + Generic, +} + +impl ApiProvider { + /// Detect provider type from URL + pub fn from_url(url: &str) -> Self { + if url.contains("anthropic") { + ApiProvider::Anthropic + } else if url.contains("x.ai") { + ApiProvider::Xai + } else if url.contains("openai.com") { + ApiProvider::OpenAi + } else if url.contains("googleapis.com") { + ApiProvider::Gemini + } else { + ApiProvider::Generic + } + } +} + +/// Apply provider-specific headers to request +pub fn apply_provider_headers( + mut req: reqwest::RequestBuilder, + url: &str, + auth_header: Option<&HeaderValue>, +) -> reqwest::RequestBuilder { + let provider = ApiProvider::from_url(url); + + match provider { + ApiProvider::Anthropic => { + // Anthropic requires x-api-key instead of Authorization + // Extract Bearer token and use as x-api-key + if let Some(auth) = auth_header { + if let Ok(auth_str) = auth.to_str() { + let api_key = auth_str.strip_prefix("Bearer ").unwrap_or(auth_str); + req = req + .header("x-api-key", api_key) + .header("anthropic-version", "2023-06-01"); + } + } + } + ApiProvider::Gemini | ApiProvider::Xai | ApiProvider::OpenAi | ApiProvider::Generic => { + // Standard OpenAI-compatible: use Authorization header as-is + if let Some(auth) = auth_header { + req = req.header("Authorization", auth); + } + } + } + + req +} + +/// Probe a single endpoint to check if it has the model +/// Returns Ok(url) if model found, Err(()) otherwise +pub async fn probe_endpoint_for_model( + client: reqwest::Client, + url: String, + model: String, + auth: Option, +) -> Result { + use tracing::debug; + + let probe_url = format!("{}/v1/models/{}", url, model); + let req = client + .get(&probe_url) + .timeout(std::time::Duration::from_secs(5)); + + // Apply provider-specific headers (handles Anthropic, xAI, OpenAI, etc.) + let auth_header_value = auth.as_ref().and_then(|a| HeaderValue::from_str(a).ok()); + let req = apply_provider_headers(req, &url, auth_header_value.as_ref()); + + match req.send().await { + Ok(resp) => { + let status = resp.status(); + if status.is_success() { + debug!( + url = %url, + model = %model, + status = %status, + "Model found on endpoint" + ); + Ok(url) + } else { + debug!( + url = %url, + model = %model, + status = %status, + "Model not found on endpoint (unsuccessful status)" + ); + Err(()) + } + } + Err(e) => { + debug!( + url = %url, + model = %model, + error = %e, + "Probe request to endpoint failed" + ); + Err(()) + } + } +} + +// ============================================================================ +// Re-export FunctionCallInProgress from mcp module +// ============================================================================ +pub(crate) use super::mcp::FunctionCallInProgress; diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/router_manager.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/router_manager.rs new file mode 100644 index 00000000..68fcfb3b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/routers/router_manager.rs @@ -0,0 +1,752 @@ +//! Router Manager for coordinating multiple routers and workers +//! +//! Provides centralized management based on enable_igw flag: +//! - Single Router Mode (enable_igw=false): Router owns workers directly +//! - Multi-Router Mode (enable_igw=true): RouterManager coordinates everything + +use std::sync::Arc; + +use async_trait::async_trait; +use axum::{ + body::Body, + extract::Request, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, +}; +use dashmap::DashMap; +use serde_json::Value; +use tracing::{debug, info, warn}; + +use crate::{ + app_context::AppContext, + config::RoutingMode, + core::{ConnectionMode, WorkerRegistry, WorkerType}, + protocols::{ + chat::ChatCompletionRequest, + classify::ClassifyRequest, + completion::CompletionRequest, + embedding::EmbeddingRequest, + generate::GenerateRequest, + rerank::RerankRequest, + responses::{ResponsesGetParams, ResponsesRequest}, + }, + routers::RouterTrait, + server::ServerConfig, +}; + +#[derive(Debug, Clone, Hash, Eq, PartialEq)] +pub struct RouterId(String); + +impl RouterId { + pub fn new(id: String) -> Self { + Self(id) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +pub struct RouterManager { + worker_registry: Arc, + routers: Arc>>, + default_router: Arc>>, + enable_igw: bool, +} + +impl RouterManager { + pub fn new(worker_registry: Arc) -> Self { + Self { + worker_registry, + routers: Arc::new(DashMap::new()), + default_router: Arc::new(std::sync::RwLock::new(None)), + enable_igw: false, // Will be set properly in from_config + } + } + + pub async fn from_config( + config: &ServerConfig, + app_context: &Arc, + ) -> Result, String> { + use crate::routers::RouterFactory; + + let mut manager = Self::new(app_context.worker_registry.clone()); + manager.enable_igw = config.router_config.enable_igw; + let manager = Arc::new(manager); + + if config.router_config.enable_igw { + info!("Initializing RouterManager in multi-router mode (IGW)"); + + match RouterFactory::create_regular_router(app_context).await { + Ok(http_regular) => { + info!("Created HTTP Regular router"); + manager.register_router( + RouterId::new("http-regular".to_string()), + Arc::from(http_regular), + ); + } + Err(e) => { + warn!("Failed to create HTTP Regular router: {e}"); + } + } + + match RouterFactory::create_pd_router( + None, + None, + &config.router_config.policy, + app_context, + ) + .await + { + Ok(http_pd) => { + info!("Created HTTP PD router"); + manager + .register_router(RouterId::new("http-pd".to_string()), Arc::from(http_pd)); + } + Err(e) => { + warn!("Failed to create HTTP PD router: {e}"); + } + } + + // TODO: Add gRPC routers once we have dynamic tokenizer loading + + info!( + "RouterManager initialized with {} routers for multi-router mode", + manager.router_count() + ); + } else { + info!("Initializing RouterManager in single-router mode"); + + let single_router = Arc::from(RouterFactory::create_router(app_context).await?); + let router_id = Self::determine_router_id( + &config.router_config.mode, + &config.router_config.connection_mode, + ); + + info!("Created single router with ID: {}", router_id.as_str()); + manager.register_router(router_id.clone(), single_router); + manager.set_default_router(router_id); + } + + if manager.router_count() == 0 { + return Err("No routers could be initialized".to_string()); + } + + Ok(manager) + } + + pub fn determine_router_id( + routing_mode: &RoutingMode, + connection_mode: &ConnectionMode, + ) -> RouterId { + match (connection_mode, routing_mode) { + (ConnectionMode::Http, RoutingMode::Regular { .. }) => { + RouterId::new("http-regular".to_string()) + } + (ConnectionMode::Http, RoutingMode::PrefillDecode { .. }) => { + RouterId::new("http-pd".to_string()) + } + (ConnectionMode::Http, RoutingMode::OpenAI { .. }) => { + RouterId::new("http-openai".to_string()) + } + (ConnectionMode::Grpc { .. }, RoutingMode::Regular { .. }) => { + RouterId::new("grpc-regular".to_string()) + } + (ConnectionMode::Grpc { .. }, RoutingMode::PrefillDecode { .. }) => { + RouterId::new("grpc-pd".to_string()) + } + (ConnectionMode::Grpc { .. }, RoutingMode::OpenAI { .. }) => { + RouterId::new("grpc-regular".to_string()) + } + } + } + + pub fn register_router(&self, id: RouterId, router: Arc) { + self.routers.insert(id.clone(), router); + + let mut default_router = self.default_router.write().unwrap(); + if default_router.is_none() { + *default_router = Some(id.clone()); + info!("Set default router to {}", id.as_str()); + } + } + + pub fn set_default_router(&self, id: RouterId) { + let mut default_router = self.default_router.write().unwrap(); + *default_router = Some(id); + } + + pub fn router_count(&self) -> usize { + self.routers.len() + } + + pub fn get_router_for_model(&self, model_id: &str) -> Option> { + let workers = self.worker_registry.get_by_model(model_id); + + if !workers.is_empty() { + let has_pd_workers = workers.iter().any(|w| { + matches!( + w.worker_type(), + WorkerType::Prefill { .. } | WorkerType::Decode + ) + }); + + let router_id = if has_pd_workers { + RouterId::new("http-pd".to_string()) + } else { + RouterId::new("http-regular".to_string()) + }; + + if let Some(router) = self.routers.get(&router_id) { + return Some(router.clone()); + } + } + + let default_router = self.default_router.read().unwrap(); + if let Some(ref default_id) = *default_router { + self.routers.get(default_id).map(|r| r.clone()) + } else { + None + } + } + + pub fn select_router_for_request( + &self, + headers: Option<&HeaderMap>, + model_id: Option<&str>, + ) -> Option> { + // In single-router mode (enable_igw=false), always use the default router + if !self.enable_igw { + let default_router = self.default_router.read().unwrap(); + if let Some(ref default_id) = *default_router { + debug!( + "Single-router mode: using default router {} for model {:?}", + default_id.as_str(), + model_id + ); + return self.routers.get(default_id).map(|r| r.clone()); + } + } + + // Multi-router mode logic follows + let _priority_threshold = headers.and_then(|h| { + h.get("x-worker-priority") + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.parse::().ok()) + }); + + let _max_cost = headers.and_then(|h| { + h.get("x-max-cost") + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.parse::().ok()) + }); + + let prefer_pd = headers + .and_then(|h| { + h.get("x-prefer-pd") + .and_then(|v| v.to_str().ok()) + .map(|s| s == "true" || s == "1") + }) + .unwrap_or(false); + + let candidate_routers = if let Some(model) = model_id { + if let Some(router) = self.get_router_for_model(model) { + vec![router] + } else { + Vec::new() + } + } else { + self.routers + .iter() + .map(|entry| entry.value().clone()) + .collect::>() + }; + + if candidate_routers.is_empty() { + return None; + } + + let mut best_router = None; + let mut best_score = 0.0; + + let num_regular_workers = self + .worker_registry + .get_all() + .iter() + .filter(|w| matches!(w.worker_type(), WorkerType::Regular)) + .count(); + let num_pd_workers = self.worker_registry.get_all().len() - num_regular_workers; + + for router in candidate_routers { + let mut score = 1.0; + + let is_pd = router.is_pd_mode(); + if prefer_pd && is_pd { + score += 2.0; + } else if !prefer_pd && !is_pd { + score += 1.0; + } + + // TODO: Once routers expose worker stats, we can evaluate: + // - Average worker priority vs priority_threshold + // - Average worker cost vs max_cost + // - Current load and health status + + let valid_router = (router.is_pd_mode() && num_pd_workers > 0) + || (!router.is_pd_mode() && num_regular_workers > 0); + if score > best_score && valid_router { + best_score = score; + best_router = Some(router); + } + } + + best_router + } +} + +#[async_trait] +impl RouterTrait for RouterManager { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + async fn health_generate(&self, _req: Request) -> Response { + // TODO: Should check if any router has healthy workers + ( + StatusCode::SERVICE_UNAVAILABLE, + "No routers with healthy workers available", + ) + .into_response() + } + + async fn get_server_info(&self, _req: Request) -> Response { + // TODO: Aggregate info from all routers with healthy workers + ( + StatusCode::OK, + serde_json::json!({ + "router_manager": true, + "routers_count": self.routers.len(), + "workers_count": self.worker_registry.get_all().len() + }) + .to_string(), + ) + .into_response() + } + + async fn get_models(&self, _req: Request) -> Response { + let model_names = self.worker_registry.get_models(); + + if model_names.is_empty() { + (StatusCode::SERVICE_UNAVAILABLE, "No models available").into_response() + } else { + // Convert model names to OpenAI-compatible model objects + let models: Vec = model_names + .iter() + .map(|name| { + serde_json::json!({ + "id": name, + "object": "model", + "owned_by": "local" + }) + }) + .collect(); + + ( + StatusCode::OK, + serde_json::json!({ + "object": "list", + "data": models + }) + .to_string(), + ) + .into_response() + } + } + + async fn get_model_info(&self, _req: Request) -> Response { + // TODO: Extract model from request and route to appropriate router + ( + StatusCode::NOT_IMPLEMENTED, + "Model info endpoint not yet implemented in RouterManager", + ) + .into_response() + } + + async fn route_generate( + &self, + headers: Option<&HeaderMap>, + body: &GenerateRequest, + model_id: Option<&str>, + ) -> Response { + let router = self.select_router_for_request(headers, model_id); + + if let Some(router) = router { + router.route_generate(headers, body, model_id).await + } else { + ( + StatusCode::NOT_FOUND, + "No router available for this request", + ) + .into_response() + } + } + + async fn route_chat( + &self, + headers: Option<&HeaderMap>, + body: &ChatCompletionRequest, + model_id: Option<&str>, + ) -> Response { + let router = self.select_router_for_request(headers, model_id); + + if let Some(router) = router { + router.route_chat(headers, body, model_id).await + } else { + ( + StatusCode::NOT_FOUND, + format!("Model '{}' not found or no router available", body.model), + ) + .into_response() + } + } + + async fn route_completion( + &self, + headers: Option<&HeaderMap>, + body: &CompletionRequest, + model_id: Option<&str>, + ) -> Response { + let router = self.select_router_for_request(headers, model_id); + + if let Some(router) = router { + router.route_completion(headers, body, model_id).await + } else { + ( + StatusCode::NOT_FOUND, + format!("Model '{}' not found or no router available", body.model), + ) + .into_response() + } + } + + async fn route_responses( + &self, + headers: Option<&HeaderMap>, + body: &ResponsesRequest, + model_id: Option<&str>, + ) -> Response { + let selected_model = model_id.or(Some(body.model.as_str())); + let router = self.select_router_for_request(headers, selected_model); + + if let Some(router) = router { + router.route_responses(headers, body, selected_model).await + } else { + ( + StatusCode::NOT_FOUND, + "No router available to handle responses request", + ) + .into_response() + } + } + + async fn get_response( + &self, + headers: Option<&HeaderMap>, + response_id: &str, + params: &ResponsesGetParams, + ) -> Response { + let router = self.select_router_for_request(headers, None); + if let Some(router) = router { + router.get_response(headers, response_id, params).await + } else { + ( + StatusCode::NOT_FOUND, + format!("No router available to get response '{}'", response_id), + ) + .into_response() + } + } + + async fn cancel_response(&self, headers: Option<&HeaderMap>, response_id: &str) -> Response { + let router = self.select_router_for_request(headers, None); + if let Some(router) = router { + router.cancel_response(headers, response_id).await + } else { + ( + StatusCode::NOT_FOUND, + format!("No router available to cancel response '{}'", response_id), + ) + .into_response() + } + } + + async fn delete_response(&self, _headers: Option<&HeaderMap>, _response_id: &str) -> Response { + ( + StatusCode::NOT_IMPLEMENTED, + "responses api not yet implemented in inference gateway mode", + ) + .into_response() + } + + async fn list_response_input_items( + &self, + headers: Option<&HeaderMap>, + response_id: &str, + ) -> Response { + // Delegate to the default router (typically http-regular) + // Response storage is shared across all routers via AppContext + let router = self.select_router_for_request(headers, None); + if let Some(router) = router { + router.list_response_input_items(headers, response_id).await + } else { + ( + StatusCode::NOT_FOUND, + "No router available to list response input items", + ) + .into_response() + } + } + + async fn route_embeddings( + &self, + headers: Option<&HeaderMap>, + body: &EmbeddingRequest, + model_id: Option<&str>, + ) -> Response { + let router = self.select_router_for_request(headers, model_id); + + if let Some(router) = router { + router.route_embeddings(headers, body, model_id).await + } else { + ( + StatusCode::NOT_FOUND, + format!("Model '{}' not found or no router available", body.model), + ) + .into_response() + } + } + + async fn route_classify( + &self, + headers: Option<&HeaderMap>, + body: &ClassifyRequest, + model_id: Option<&str>, + ) -> Response { + let router = self.select_router_for_request(headers, model_id); + + if let Some(router) = router { + router.route_classify(headers, body, model_id).await + } else { + ( + StatusCode::NOT_FOUND, + format!("Model '{}' not found or no router available", body.model), + ) + .into_response() + } + } + + async fn route_rerank( + &self, + headers: Option<&HeaderMap>, + body: &RerankRequest, + model_id: Option<&str>, + ) -> Response { + let router = self.select_router_for_request(headers, model_id); + + if let Some(router) = router { + router.route_rerank(headers, body, model_id).await + } else { + ( + StatusCode::NOT_FOUND, + "No router available for rerank request", + ) + .into_response() + } + } + + // Conversations API delegates + async fn create_conversation(&self, headers: Option<&HeaderMap>, body: &Value) -> Response { + let router = self.select_router_for_request(headers, None); + if let Some(router) = router { + router.create_conversation(headers, body).await + } else { + ( + StatusCode::NOT_FOUND, + "No router available to create conversation", + ) + .into_response() + } + } + + async fn get_conversation( + &self, + headers: Option<&HeaderMap>, + conversation_id: &str, + ) -> Response { + let router = self.select_router_for_request(headers, None); + if let Some(router) = router { + router.get_conversation(headers, conversation_id).await + } else { + ( + StatusCode::NOT_FOUND, + format!( + "No router available to get conversation '{}'", + conversation_id + ), + ) + .into_response() + } + } + + async fn update_conversation( + &self, + headers: Option<&HeaderMap>, + conversation_id: &str, + body: &Value, + ) -> Response { + let router = self.select_router_for_request(headers, None); + if let Some(router) = router { + router + .update_conversation(headers, conversation_id, body) + .await + } else { + ( + StatusCode::NOT_FOUND, + format!( + "No router available to update conversation '{}'", + conversation_id + ), + ) + .into_response() + } + } + + async fn delete_conversation( + &self, + headers: Option<&HeaderMap>, + conversation_id: &str, + ) -> Response { + let router = self.select_router_for_request(headers, None); + if let Some(router) = router { + router.delete_conversation(headers, conversation_id).await + } else { + ( + StatusCode::NOT_FOUND, + format!( + "No router available to delete conversation '{}'", + conversation_id + ), + ) + .into_response() + } + } + + async fn list_conversation_items( + &self, + headers: Option<&HeaderMap>, + conversation_id: &str, + limit: Option, + order: Option, + after: Option, + ) -> Response { + let router = self.select_router_for_request(headers, None); + if let Some(router) = router { + router + .list_conversation_items(headers, conversation_id, limit, order, after) + .await + } else { + ( + StatusCode::NOT_FOUND, + format!( + "No router available to list conversation items for '{}'", + conversation_id + ), + ) + .into_response() + } + } + + async fn create_conversation_items( + &self, + headers: Option<&HeaderMap>, + conversation_id: &str, + body: &Value, + ) -> Response { + let router = self.select_router_for_request(headers, None); + if let Some(router) = router { + router + .create_conversation_items(headers, conversation_id, body) + .await + } else { + ( + StatusCode::NOT_FOUND, + format!( + "No router available to create conversation items for '{}'", + conversation_id + ), + ) + .into_response() + } + } + + async fn get_conversation_item( + &self, + headers: Option<&HeaderMap>, + conversation_id: &str, + item_id: &str, + include: Option>, + ) -> Response { + let router = self.select_router_for_request(headers, None); + if let Some(router) = router { + router + .get_conversation_item(headers, conversation_id, item_id, include) + .await + } else { + ( + StatusCode::NOT_FOUND, + format!( + "No router available to get conversation item '{}' in '{}'", + item_id, conversation_id + ), + ) + .into_response() + } + } + + async fn delete_conversation_item( + &self, + headers: Option<&HeaderMap>, + conversation_id: &str, + item_id: &str, + ) -> Response { + let router = self.select_router_for_request(headers, None); + if let Some(router) = router { + router + .delete_conversation_item(headers, conversation_id, item_id) + .await + } else { + ( + StatusCode::NOT_FOUND, + format!( + "No router available to delete conversation item '{}' in '{}'", + item_id, conversation_id + ), + ) + .into_response() + } + } + + fn router_type(&self) -> &'static str { + "manager" + } +} + +impl std::fmt::Debug for RouterManager { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RouterManager") + .field("routers_count", &self.routers.len()) + .field("workers_count", &self.worker_registry.get_all().len()) + .field("default_router", &*self.default_router.read().unwrap()) + .finish() + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/server.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/server.rs new file mode 100644 index 00000000..79e6c87f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/server.rs @@ -0,0 +1,974 @@ +use std::{ + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::Duration, +}; + +use axum::{ + extract::{Path, Query, Request, State}, + http::StatusCode, + response::{IntoResponse, Response}, + routing::{delete, get, post}, + serve, Json, Router, +}; +use serde::Deserialize; +use serde_json::{json, Value}; +use tokio::{net::TcpListener, signal, spawn}; +use tracing::{error, info, warn, Level}; + +use crate::{ + app_context::AppContext, + config::{RouterConfig, RoutingMode}, + core::{ + worker_to_info, + workflow::{ + create_external_worker_registration_workflow, create_mcp_registration_workflow, + create_worker_registration_workflow, create_worker_removal_workflow, LoggingSubscriber, + WorkflowEngine, + }, + Job, JobQueue, JobQueueConfig, WorkerManager, WorkerType, + }, + logging::{self, LoggingConfig}, + metrics::{self, PrometheusConfig}, + middleware::{self, AuthConfig, QueuedRequest}, + protocols::{ + chat::ChatCompletionRequest, + classify::ClassifyRequest, + completion::CompletionRequest, + embedding::EmbeddingRequest, + generate::GenerateRequest, + rerank::{RerankRequest, V1RerankReqInput}, + responses::{ResponsesGetParams, ResponsesRequest}, + validated::ValidatedJson, + worker_spec::{WorkerConfigRequest, WorkerErrorResponse, WorkerInfo}, + }, + routers::{router_manager::RouterManager, RouterTrait}, + service_discovery::{start_service_discovery, ServiceDiscoveryConfig}, +}; + +#[derive(Clone)] +pub struct AppState { + pub router: Arc, + pub context: Arc, + pub concurrency_queue_tx: Option>, + pub router_manager: Option>, +} + +async fn sink_handler() -> Response { + StatusCode::NOT_FOUND.into_response() +} + +async fn liveness() -> Response { + (StatusCode::OK, "OK").into_response() +} + +async fn readiness(State(state): State>) -> Response { + let workers = state.context.worker_registry.get_all(); + let healthy_workers: Vec<_> = workers.iter().filter(|w| w.is_healthy()).collect(); + + let is_ready = if state.context.router_config.enable_igw { + !healthy_workers.is_empty() + } else { + match &state.context.router_config.mode { + RoutingMode::PrefillDecode { .. } => { + let has_prefill = healthy_workers + .iter() + .any(|w| matches!(w.worker_type(), WorkerType::Prefill { .. })); + let has_decode = healthy_workers + .iter() + .any(|w| matches!(w.worker_type(), WorkerType::Decode)); + has_prefill && has_decode + } + RoutingMode::Regular { .. } => !healthy_workers.is_empty(), + RoutingMode::OpenAI { .. } => !healthy_workers.is_empty(), + } + }; + + if is_ready { + ( + StatusCode::OK, + Json(json!({ + "status": "ready", + "healthy_workers": healthy_workers.len(), + "total_workers": workers.len() + })), + ) + .into_response() + } else { + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({ + "status": "not ready", + "reason": "insufficient healthy workers" + })), + ) + .into_response() + } +} + +async fn health(_state: State>) -> Response { + liveness().await +} + +async fn health_generate(State(state): State>, req: Request) -> Response { + state.router.health_generate(req).await +} + +async fn engine_metrics(State(state): State>) -> Response { + WorkerManager::get_engine_metrics(&state.context.worker_registry, &state.context.client).await +} + +async fn get_server_info(State(state): State>, req: Request) -> Response { + state.router.get_server_info(req).await +} + +async fn v1_models(State(state): State>, req: Request) -> Response { + state.router.get_models(req).await +} + +async fn get_model_info(State(state): State>, req: Request) -> Response { + state.router.get_model_info(req).await +} + +async fn generate( + State(state): State>, + headers: http::HeaderMap, + Json(body): Json, +) -> Response { + let model_id = body.model.as_deref(); + state + .router + .route_generate(Some(&headers), &body, model_id) + .await +} + +async fn v1_chat_completions( + State(state): State>, + headers: http::HeaderMap, + ValidatedJson(body): ValidatedJson, +) -> Response { + state + .router + .route_chat(Some(&headers), &body, Some(&body.model)) + .await +} + +async fn v1_completions( + State(state): State>, + headers: http::HeaderMap, + Json(body): Json, +) -> Response { + state + .router + .route_completion(Some(&headers), &body, Some(&body.model)) + .await +} + +async fn rerank( + State(state): State>, + headers: http::HeaderMap, + ValidatedJson(body): ValidatedJson, +) -> Response { + state + .router + .route_rerank(Some(&headers), &body, Some(&body.model)) + .await +} + +async fn v1_rerank( + State(state): State>, + headers: http::HeaderMap, + Json(body): Json, +) -> Response { + let rerank_body = &body.into(); + state + .router + .route_rerank(Some(&headers), rerank_body, Some(&rerank_body.model)) + .await +} + +async fn v1_responses( + State(state): State>, + headers: http::HeaderMap, + ValidatedJson(body): ValidatedJson, +) -> Response { + state + .router + .route_responses(Some(&headers), &body, Some(&body.model)) + .await +} + +async fn v1_embeddings( + State(state): State>, + headers: http::HeaderMap, + Json(body): Json, +) -> Response { + state + .router + .route_embeddings(Some(&headers), &body, Some(&body.model)) + .await +} + +async fn v1_classify( + State(state): State>, + headers: http::HeaderMap, + Json(body): Json, +) -> Response { + state + .router + .route_classify(Some(&headers), &body, Some(&body.model)) + .await +} + +async fn v1_responses_get( + State(state): State>, + Path(response_id): Path, + headers: http::HeaderMap, + Query(params): Query, +) -> Response { + state + .router + .get_response(Some(&headers), &response_id, ¶ms) + .await +} + +async fn v1_responses_cancel( + State(state): State>, + Path(response_id): Path, + headers: http::HeaderMap, +) -> Response { + state + .router + .cancel_response(Some(&headers), &response_id) + .await +} + +async fn v1_responses_delete( + State(state): State>, + Path(response_id): Path, + headers: http::HeaderMap, +) -> Response { + state + .router + .delete_response(Some(&headers), &response_id) + .await +} + +async fn v1_responses_list_input_items( + State(state): State>, + Path(response_id): Path, + headers: http::HeaderMap, +) -> Response { + state + .router + .list_response_input_items(Some(&headers), &response_id) + .await +} + +async fn v1_conversations_create( + State(state): State>, + headers: http::HeaderMap, + Json(body): Json, +) -> Response { + state + .router + .create_conversation(Some(&headers), &body) + .await +} + +async fn v1_conversations_get( + State(state): State>, + Path(conversation_id): Path, + headers: http::HeaderMap, +) -> Response { + state + .router + .get_conversation(Some(&headers), &conversation_id) + .await +} + +async fn v1_conversations_update( + State(state): State>, + Path(conversation_id): Path, + headers: http::HeaderMap, + Json(body): Json, +) -> Response { + state + .router + .update_conversation(Some(&headers), &conversation_id, &body) + .await +} + +async fn v1_conversations_delete( + State(state): State>, + Path(conversation_id): Path, + headers: http::HeaderMap, +) -> Response { + state + .router + .delete_conversation(Some(&headers), &conversation_id) + .await +} + +#[derive(Deserialize, Default)] +struct ListItemsQuery { + limit: Option, + order: Option, + after: Option, +} + +async fn v1_conversations_list_items( + State(state): State>, + Path(conversation_id): Path, + Query(ListItemsQuery { + limit, + order, + after, + }): Query, + headers: http::HeaderMap, +) -> Response { + state + .router + .list_conversation_items(Some(&headers), &conversation_id, limit, order, after) + .await +} + +#[derive(Deserialize, Default)] +struct GetItemQuery { + /// Additional fields to include in response (not yet implemented) + include: Option>, +} + +async fn v1_conversations_create_items( + State(state): State>, + Path(conversation_id): Path, + headers: http::HeaderMap, + Json(body): Json, +) -> Response { + state + .router + .create_conversation_items(Some(&headers), &conversation_id, &body) + .await +} + +async fn v1_conversations_get_item( + State(state): State>, + Path((conversation_id, item_id)): Path<(String, String)>, + Query(query): Query, + headers: http::HeaderMap, +) -> Response { + state + .router + .get_conversation_item(Some(&headers), &conversation_id, &item_id, query.include) + .await +} + +async fn v1_conversations_delete_item( + State(state): State>, + Path((conversation_id, item_id)): Path<(String, String)>, + headers: http::HeaderMap, +) -> Response { + state + .router + .delete_conversation_item(Some(&headers), &conversation_id, &item_id) + .await +} + +async fn flush_cache(State(state): State>, _req: Request) -> Response { + match WorkerManager::flush_cache_all(&state.context.worker_registry, &state.context.client) + .await + { + Ok(result) => { + if result.failed.is_empty() { + ( + StatusCode::OK, + Json(json!({ + "status": "success", + "message": result.message, + "workers_flushed": result.successful.len(), + "total_http_workers": result.http_workers, + "total_workers": result.total_workers + })), + ) + .into_response() + } else { + ( + StatusCode::PARTIAL_CONTENT, + Json(json!({ + "status": "partial_success", + "message": result.message, + "successful": result.successful, + "failed": result.failed.into_iter().map(|(url, err)| json!({ + "worker": url, + "error": err + })).collect::>(), + "total_http_workers": result.http_workers, + "total_workers": result.total_workers + })), + ) + .into_response() + } + } + Err(e) => { + error!("Failed to flush cache: {}", e); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "status": "error", + "message": format!("Failed to flush cache: {}", e) + })), + ) + .into_response() + } + } +} + +async fn get_loads(State(state): State>, _req: Request) -> Response { + let result = + WorkerManager::get_all_worker_loads(&state.context.worker_registry, &state.context.client) + .await; + + let loads: Vec = result + .loads + .iter() + .map(|info| { + json!({ + "worker": &info.worker, + "load": info.load + }) + }) + .collect(); + + (StatusCode::OK, Json(json!({ "workers": loads }))).into_response() +} + +async fn create_worker( + State(state): State>, + Json(config): Json, +) -> Response { + // Warn if router has API key but worker is being added without one + if state.context.router_config.api_key.is_some() && config.api_key.is_none() { + warn!( + "Adding worker {} without API key while router has API key configured. \ + Worker will be accessible without authentication. \ + If the worker requires the same API key as the router, please specify it explicitly.", + config.url + ); + } + + // Populate dp_aware from router's configuration + let config = WorkerConfigRequest { + dp_aware: state.context.router_config.dp_aware, + ..config + }; + + // Submit job for async processing + let worker_url = config.url.clone(); + let job = Job::AddWorker { + config: Box::new(config), + }; + + let job_queue = state + .context + .worker_job_queue + .get() + .expect("JobQueue not initialized"); + match job_queue.submit(job).await { + Ok(_) => { + let response = json!({ + "status": "accepted", + "worker_id": worker_url, + "message": "Worker addition queued for background processing" + }); + (StatusCode::ACCEPTED, Json(response)).into_response() + } + Err(error) => { + let error_response = WorkerErrorResponse { + error, + code: "INTERNAL_SERVER_ERROR".to_string(), + }; + (StatusCode::INTERNAL_SERVER_ERROR, Json(error_response)).into_response() + } + } +} + +async fn list_workers_rest(State(state): State>) -> Response { + let workers = state.context.worker_registry.get_all(); + let worker_infos: Vec = workers.iter().map(worker_to_info).collect(); + + let response = json!({ + "workers": worker_infos, + "total": workers.len(), + "stats": { + "prefill_count": state.context.worker_registry.get_prefill_workers().len(), + "decode_count": state.context.worker_registry.get_decode_workers().len(), + "regular_count": state.context.worker_registry.get_by_type(&WorkerType::Regular).len(), + } + }); + Json(response).into_response() +} + +async fn get_worker(State(state): State>, Path(url): Path) -> Response { + let job_queue = state + .context + .worker_job_queue + .get() + .expect("JobQueue not initialized"); + + if let Some(worker) = state.context.worker_registry.get_by_url(&url) { + // Worker exists in registry, get its full info and attach job status if any + let mut worker_info = worker_to_info(&worker); + if let Some(status) = job_queue.get_status(&url) { + worker_info.job_status = Some(status); + } + return Json(worker_info).into_response(); + } + + // Worker not in registry, check job queue for its status + if let Some(status) = job_queue.get_status(&url) { + // Create a partial WorkerInfo to report the job status + let worker_info = WorkerInfo { + id: url.clone(), + url: url.clone(), + model_id: "unknown".to_string(), + priority: 0, + cost: 1.0, + worker_type: "unknown".to_string(), + is_healthy: false, + load: 0, + connection_mode: "unknown".to_string(), + runtime_type: None, + tokenizer_path: None, + reasoning_parser: None, + tool_parser: None, + chat_template: None, + bootstrap_port: None, + metadata: std::collections::HashMap::new(), + job_status: Some(status), + }; + return Json(worker_info).into_response(); + } + + // Worker not found in registry or job queue + let error = WorkerErrorResponse { + error: format!("Worker {url} not found"), + code: "WORKER_NOT_FOUND".to_string(), + }; + (StatusCode::NOT_FOUND, Json(error)).into_response() +} + +async fn delete_worker(State(state): State>, Path(url): Path) -> Response { + let worker_id = url.clone(); + let job = Job::RemoveWorker { url }; + + let job_queue = state + .context + .worker_job_queue + .get() + .expect("JobQueue not initialized"); + match job_queue.submit(job).await { + Ok(_) => { + let response = json!({ + "status": "accepted", + "worker_id": worker_id, + "message": "Worker removal queued for background processing" + }); + (StatusCode::ACCEPTED, Json(response)).into_response() + } + Err(error) => { + let error_response = WorkerErrorResponse { + error, + code: "INTERNAL_SERVER_ERROR".to_string(), + }; + (StatusCode::INTERNAL_SERVER_ERROR, Json(error_response)).into_response() + } + } +} + +pub struct ServerConfig { + pub host: String, + pub port: u16, + pub router_config: RouterConfig, + pub max_payload_size: usize, + pub log_dir: Option, + pub log_level: Option, + pub service_discovery_config: Option, + pub prometheus_config: Option, + pub request_timeout_secs: u64, + pub request_id_headers: Option>, +} + +pub fn build_app( + app_state: Arc, + auth_config: AuthConfig, + max_payload_size: usize, + request_id_headers: Vec, + cors_allowed_origins: Vec, +) -> Router { + let protected_routes = Router::new() + .route("/generate", post(generate)) + .route("/v1/chat/completions", post(v1_chat_completions)) + .route("/v1/completions", post(v1_completions)) + .route("/rerank", post(rerank)) + .route("/v1/rerank", post(v1_rerank)) + .route("/v1/responses", post(v1_responses)) + .route("/v1/embeddings", post(v1_embeddings)) + .route("/v1/classify", post(v1_classify)) + .route("/v1/responses/{response_id}", get(v1_responses_get)) + .route( + "/v1/responses/{response_id}/cancel", + post(v1_responses_cancel), + ) + .route("/v1/responses/{response_id}", delete(v1_responses_delete)) + .route( + "/v1/responses/{response_id}/input_items", + get(v1_responses_list_input_items), + ) + .route("/v1/conversations", post(v1_conversations_create)) + .route( + "/v1/conversations/{conversation_id}", + get(v1_conversations_get) + .post(v1_conversations_update) + .delete(v1_conversations_delete), + ) + .route( + "/v1/conversations/{conversation_id}/items", + get(v1_conversations_list_items).post(v1_conversations_create_items), + ) + .route( + "/v1/conversations/{conversation_id}/items/{item_id}", + get(v1_conversations_get_item).delete(v1_conversations_delete_item), + ) + .route_layer(axum::middleware::from_fn_with_state( + app_state.clone(), + middleware::concurrency_limit_middleware, + )) + .route_layer(axum::middleware::from_fn_with_state( + auth_config.clone(), + middleware::auth_middleware, + )); + + let public_routes = Router::new() + .route("/liveness", get(liveness)) + .route("/readiness", get(readiness)) + .route("/health", get(health)) + .route("/health_generate", get(health_generate)) + .route("/engine_metrics", get(engine_metrics)) + .route("/v1/models", get(v1_models)) + .route("/get_model_info", get(get_model_info)) + .route("/get_server_info", get(get_server_info)); + + let admin_routes = Router::new() + .route("/flush_cache", post(flush_cache)) + .route("/get_loads", get(get_loads)) + .route_layer(axum::middleware::from_fn_with_state( + auth_config.clone(), + middleware::auth_middleware, + )); + + let worker_routes = Router::new() + .route("/workers", post(create_worker)) + .route("/workers", get(list_workers_rest)) + .route("/workers/{url}", get(get_worker)) + .route("/workers/{url}", delete(delete_worker)) + .route_layer(axum::middleware::from_fn_with_state( + auth_config.clone(), + middleware::auth_middleware, + )); + + Router::new() + .merge(protected_routes) + .merge(public_routes) + .merge(admin_routes) + .merge(worker_routes) + .layer(axum::extract::DefaultBodyLimit::max(max_payload_size)) + .layer(tower_http::limit::RequestBodyLimitLayer::new( + max_payload_size, + )) + .layer(middleware::create_logging_layer()) + .layer(middleware::RequestIdLayer::new(request_id_headers)) + .layer(create_cors_layer(cors_allowed_origins)) + .fallback(sink_handler) + .with_state(app_state) +} + +pub async fn startup(config: ServerConfig) -> Result<(), Box> { + static LOGGING_INITIALIZED: AtomicBool = AtomicBool::new(false); + + let _log_guard = if !LOGGING_INITIALIZED.swap(true, Ordering::SeqCst) { + Some(logging::init_logging(LoggingConfig { + level: config + .log_level + .as_deref() + .and_then(|s| match s.to_uppercase().parse::() { + Ok(l) => Some(l), + Err(_) => { + warn!("Invalid log level string: '{s}'. Defaulting to INFO."); + None + } + }) + .unwrap_or(Level::INFO), + json_format: false, + log_dir: config.log_dir.clone(), + colorize: true, + log_file_name: "sgl-router".to_string(), + log_targets: None, + })) + } else { + None + }; + + if let Some(prometheus_config) = &config.prometheus_config { + metrics::start_prometheus(prometheus_config.clone()); + } + + info!( + "Starting router on {}:{} | mode: {:?} | policy: {:?} | max_payload: {}MB", + config.host, + config.port, + config.router_config.mode, + config.router_config.policy, + config.max_payload_size / (1024 * 1024) + ); + + let app_context = Arc::new( + AppContext::from_config(config.router_config.clone(), config.request_timeout_secs).await?, + ); + + let weak_context = Arc::downgrade(&app_context); + let worker_job_queue = JobQueue::new(JobQueueConfig::default(), weak_context); + app_context + .worker_job_queue + .set(worker_job_queue) + .expect("JobQueue should only be initialized once"); + + // Initialize workflow engine and register workflows + let engine = Arc::new(WorkflowEngine::new()); + + engine + .event_bus() + .subscribe(Arc::new(LoggingSubscriber)) + .await; + + engine.register_workflow(create_worker_registration_workflow(&config.router_config)); + engine.register_workflow(create_external_worker_registration_workflow()); + engine.register_workflow(create_worker_removal_workflow()); + engine.register_workflow(create_mcp_registration_workflow()); + app_context + .workflow_engine + .set(engine) + .expect("WorkflowEngine should only be initialized once"); + info!( + "Workflow engine initialized with worker and MCP registration workflows (health check timeout: {}s)", + config.router_config.health_check.timeout_secs + ); + + info!( + "Initializing workers for routing mode: {:?}", + config.router_config.mode + ); + + // Submit worker initialization job to queue + let job_queue = app_context + .worker_job_queue + .get() + .expect("JobQueue should be initialized"); + let job = Job::InitializeWorkersFromConfig { + router_config: Box::new(config.router_config.clone()), + }; + job_queue + .submit(job) + .await + .map_err(|e| format!("Failed to submit worker initialization job: {}", e))?; + + info!("Worker initialization job submitted (will complete in background)"); + + if let Some(mcp_config) = &config.router_config.mcp_config { + info!("Found {} MCP server(s) in config", mcp_config.servers.len()); + let mcp_job = Job::InitializeMcpServers { + mcp_config: Box::new(mcp_config.clone()), + }; + job_queue + .submit(mcp_job) + .await + .map_err(|e| format!("Failed to submit MCP initialization job: {}", e))?; + } else { + info!("No MCP config provided, skipping MCP server initialization"); + } + + // Start background refresh for ALL MCP servers (static + dynamic in LRU cache) + if let Some(mcp_manager) = app_context.mcp_manager.get() { + let refresh_interval = Duration::from_secs(600); // 10 minutes + let _refresh_handle = + Arc::clone(mcp_manager).spawn_background_refresh_all(refresh_interval); + info!("Started background refresh for all MCP servers (every 10 minutes)"); + } + + let worker_stats = app_context.worker_registry.stats(); + info!( + "Workers initialized: {} total, {} healthy", + worker_stats.total_workers, worker_stats.healthy_workers + ); + + let router_manager = RouterManager::from_config(&config, &app_context).await?; + let router: Arc = router_manager.clone(); + + let _health_checker = app_context + .worker_registry + .start_health_checker(config.router_config.health_check.check_interval_secs); + info!( + "Started health checker for workers with {}s interval", + config.router_config.health_check.check_interval_secs + ); + + if let Some(ref load_monitor) = app_context.load_monitor { + load_monitor.start().await; + info!("Started LoadMonitor for PowerOfTwo policies"); + } + + let (limiter, processor) = middleware::ConcurrencyLimiter::new( + app_context.rate_limiter.clone(), + config.router_config.queue_size, + Duration::from_secs(config.router_config.queue_timeout_secs), + ); + + if app_context.rate_limiter.is_none() { + info!("Rate limiting is disabled (max_concurrent_requests = -1)"); + } + + match processor { + Some(proc) => { + spawn(proc.run()); + info!( + "Started request queue (size: {}, timeout: {}s)", + config.router_config.queue_size, config.router_config.queue_timeout_secs + ); + } + None => { + info!( + "Rate limiting enabled (max_concurrent_requests = {}, queue disabled)", + config.router_config.max_concurrent_requests + ); + } + } + + let app_state = Arc::new(AppState { + router, + context: app_context.clone(), + concurrency_queue_tx: limiter.queue_tx.clone(), + router_manager: Some(router_manager), + }); + if let Some(service_discovery_config) = config.service_discovery_config { + if service_discovery_config.enabled { + let app_context_arc = Arc::clone(&app_state.context); + match start_service_discovery(service_discovery_config, app_context_arc).await { + Ok(handle) => { + info!("Service discovery started"); + spawn(async move { + if let Err(e) = handle.await { + error!("Service discovery task failed: {:?}", e); + } + }); + } + Err(e) => { + error!("Failed to start service discovery: {e}"); + warn!("Continuing without service discovery"); + } + } + } + } + + info!( + "Router ready | workers: {:?}", + WorkerManager::get_worker_urls(&app_state.context.worker_registry) + ); + + let request_id_headers = config.request_id_headers.clone().unwrap_or_else(|| { + vec![ + "x-request-id".to_string(), + "x-correlation-id".to_string(), + "x-trace-id".to_string(), + "request-id".to_string(), + ] + }); + + let auth_config = AuthConfig { + api_key: config.router_config.api_key.clone(), + }; + + let app = build_app( + app_state, + auth_config, + config.max_payload_size, + request_id_headers, + config.router_config.cors_allowed_origins.clone(), + ); + + // TcpListener::bind accepts &str and handles IPv4/IPv6 via ToSocketAddrs + let bind_addr = format!("{}:{}", config.host, config.port); + info!("Starting server on {}", bind_addr); + let listener = TcpListener::bind(&bind_addr) + .await + .map_err(|e| format!("Failed to bind to {}: {}", bind_addr, e))?; + serve(listener, app) + .with_graceful_shutdown(shutdown_signal()) + .await + .map_err(|e| Box::new(e) as Box)?; + + Ok(()) +} + +async fn shutdown_signal() { + let ctrl_c = async { + signal::ctrl_c() + .await + .expect("failed to install Ctrl+C handler"); + }; + + #[cfg(unix)] + let terminate = async { + signal::unix::signal(signal::unix::SignalKind::terminate()) + .expect("failed to install signal handler") + .recv() + .await; + }; + + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + + tokio::select! { + _ = ctrl_c => { + info!("Received Ctrl+C, starting graceful shutdown"); + }, + _ = terminate => { + info!("Received terminate signal, starting graceful shutdown"); + }, + } +} + +fn create_cors_layer(allowed_origins: Vec) -> tower_http::cors::CorsLayer { + use tower_http::cors::Any; + + let cors = if allowed_origins.is_empty() { + tower_http::cors::CorsLayer::new() + .allow_origin(Any) + .allow_methods(Any) + .allow_headers(Any) + .expose_headers(Any) + } else { + let origins: Vec = allowed_origins + .into_iter() + .filter_map(|origin| origin.parse().ok()) + .collect(); + + tower_http::cors::CorsLayer::new() + .allow_origin(origins) + .allow_methods([http::Method::GET, http::Method::POST, http::Method::OPTIONS]) + .allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION]) + .expose_headers([http::header::HeaderName::from_static("x-request-id")]) + }; + + cors.max_age(Duration::from_secs(3600)) +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/service_discovery.rs b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/service_discovery.rs new file mode 100644 index 00000000..a211423c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/sgl-router/src/service_discovery.rs @@ -0,0 +1,1147 @@ +use std::{ + collections::{HashMap, HashSet}, + sync::{Arc, Mutex}, + time::Duration, +}; + +use futures::{StreamExt, TryStreamExt}; +use k8s_openapi::api::core::v1::Pod; +use kube::{ + api::Api, + runtime::{ + watcher::{watcher, Config}, + WatchStreamExt, + }, + Client, +}; +use rustls; +use tokio::{task, time}; +use tracing::{debug, error, info, warn}; + +use crate::{app_context::AppContext, core::Job, protocols::worker_spec::WorkerConfigRequest}; + +#[derive(Debug, Clone)] +pub struct ServiceDiscoveryConfig { + pub enabled: bool, + pub selector: HashMap, + pub check_interval: Duration, + pub port: u16, + pub namespace: Option, + // PD mode specific configuration + pub pd_mode: bool, + pub prefill_selector: HashMap, + pub decode_selector: HashMap, + // Bootstrap port annotation specific to mooncake implementation + pub bootstrap_port_annotation: String, +} + +impl Default for ServiceDiscoveryConfig { + fn default() -> Self { + ServiceDiscoveryConfig { + enabled: false, + selector: HashMap::new(), + check_interval: Duration::from_secs(60), + port: 8000, + namespace: None, + pd_mode: false, + prefill_selector: HashMap::new(), + decode_selector: HashMap::new(), + bootstrap_port_annotation: "sglang.ai/bootstrap-port".to_string(), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum PodType { + Prefill, + Decode, + Regular, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct PodInfo { + pub name: String, + pub ip: String, + pub status: String, + pub is_ready: bool, + pub pod_type: Option, + pub bootstrap_port: Option, +} + +impl PodInfo { + fn matches_selector(pod: &Pod, selector: &HashMap) -> bool { + if selector.is_empty() { + return false; + } + + pod.metadata + .labels + .as_ref() + .is_some_and(|labels| selector.iter().all(|(k, v)| labels.get(k) == Some(v))) + } + + pub fn should_include(pod: &Pod, config: &ServiceDiscoveryConfig) -> bool { + if config.pd_mode { + if config.prefill_selector.is_empty() && config.decode_selector.is_empty() { + warn!("PD mode enabled but both prefill_selector and decode_selector are empty"); + return false; + } + Self::matches_selector(pod, &config.prefill_selector) + || Self::matches_selector(pod, &config.decode_selector) + } else { + if config.selector.is_empty() { + warn!("Regular mode enabled but selector is empty"); + return false; + } + Self::matches_selector(pod, &config.selector) + } + } + + pub fn from_pod(pod: &Pod, config: Option<&ServiceDiscoveryConfig>) -> Option { + let name = pod.metadata.name.clone()?; + let status = pod.status.clone()?; + let pod_ip = status.pod_ip?; + + let is_ready = if let Some(conditions) = &status.conditions { + conditions + .iter() + .any(|condition| condition.type_ == "Ready" && condition.status == "True") + } else { + false + }; + + let pod_status = status.phase.unwrap_or_else(|| "Unknown".to_string()); + + let pod_type = if let Some(config) = config { + if config.pd_mode { + if Self::matches_selector(pod, &config.prefill_selector) { + Some(PodType::Prefill) + } else if Self::matches_selector(pod, &config.decode_selector) { + Some(PodType::Decode) + } else { + Some(PodType::Regular) + } + } else { + Some(PodType::Regular) + } + } else { + None + }; + + let bootstrap_port = if matches!(pod_type, Some(PodType::Prefill)) { + if let Some(config) = config { + pod.metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(&config.bootstrap_port_annotation)) + .and_then(|port_str| port_str.parse::().ok()) + } else { + None + } + } else { + None + }; + + Some(PodInfo { + name, + ip: pod_ip, + status: pod_status, + is_ready, + pod_type, + bootstrap_port, + }) + } + + pub fn is_healthy(&self) -> bool { + self.is_ready && self.status == "Running" + } + + pub fn worker_url(&self, port: u16) -> String { + // Default to http:// prefix; workflow will detect actual protocol (HTTP vs gRPC) + format!("http://{}:{}", self.ip, port) + } +} + +pub async fn start_service_discovery( + config: ServiceDiscoveryConfig, + app_context: Arc, +) -> Result, kube::Error> { + if !config.enabled { + return Err(kube::Error::Api(kube::error::ErrorResponse { + status: "Disabled".to_string(), + message: "Service discovery is disabled".to_string(), + reason: "ConfigurationError".to_string(), + code: 400, + })); + } + + let _ = rustls::crypto::ring::default_provider().install_default(); + + let client = Client::try_default().await?; + + // Log the appropriate selectors based on mode + if config.pd_mode { + let prefill_selector = config + .prefill_selector + .iter() + .map(|(k, v)| format!("{}={}", k, v)) + .collect::>() + .join(","); + + let decode_selector = config + .decode_selector + .iter() + .map(|(k, v)| format!("{}={}", k, v)) + .collect::>() + .join(","); + + info!( + "Starting K8s service discovery | PD mode | prefill: '{}' | decode: '{}'", + prefill_selector, decode_selector + ); + } else { + let label_selector = config + .selector + .iter() + .map(|(k, v)| format!("{}={}", k, v)) + .collect::>() + .join(","); + + info!( + "Starting K8s service discovery | selector: '{}'", + label_selector + ); + } + + let handle = task::spawn(async move { + let tracked_pods = Arc::new(Mutex::new(HashSet::new())); + + let pods: Api = if let Some(namespace) = &config.namespace { + Api::namespaced(client, namespace) + } else { + Api::all(client) + }; + + debug!("K8s service discovery initialized"); + + let config_arc = Arc::new(config.clone()); + let port = config.port; + + let mut retry_delay = Duration::from_secs(1); + const MAX_RETRY_DELAY: Duration = Duration::from_secs(300); + + loop { + let watcher_config = Config::default(); + let watcher_stream = watcher(pods.clone(), watcher_config).applied_objects(); + + let config_clone = Arc::clone(&config_arc); + let tracked_pods_clone = Arc::clone(&tracked_pods); + + let filtered_stream = watcher_stream.filter_map(move |obj_res| { + let config_inner = Arc::clone(&config_clone); + + async move { + match obj_res { + Ok(pod) => { + if PodInfo::should_include(&pod, &config_inner) { + Some(Ok(pod)) + } else { + None + } + } + Err(e) => Some(Err(e)), + } + } + }); + + let tracked_pods_clone2 = Arc::clone(&tracked_pods_clone); + let app_context_clone = Arc::clone(&app_context); + let config_clone2 = Arc::clone(&config_arc); + + match filtered_stream + .try_for_each(move |pod| { + let tracked_pods_inner = Arc::clone(&tracked_pods_clone2); + let app_context_inner = Arc::clone(&app_context_clone); + let config_inner = Arc::clone(&config_clone2); + + async move { + let pod_info = PodInfo::from_pod(&pod, Some(&config_inner)); + + if let Some(pod_info) = pod_info { + if pod.metadata.deletion_timestamp.is_some() { + handle_pod_deletion( + &pod_info, + tracked_pods_inner, + app_context_inner, + port, + ) + .await; + } else { + handle_pod_event( + &pod_info, + tracked_pods_inner, + app_context_inner, + port, + config_inner.pd_mode, + ) + .await; + } + } + Ok(()) + } + }) + .await + { + Ok(_) => { + retry_delay = Duration::from_secs(1); + } + Err(err) => { + error!("Error in Kubernetes watcher: {}", err); + warn!( + "Retrying in {} seconds with exponential backoff", + retry_delay.as_secs() + ); + time::sleep(retry_delay).await; + + retry_delay = std::cmp::min(retry_delay * 2, MAX_RETRY_DELAY); + } + } + + warn!( + "Kubernetes watcher exited, restarting in {} seconds", + config_arc.check_interval.as_secs() + ); + time::sleep(config_arc.check_interval).await; + } + }); + + Ok(handle) +} + +async fn handle_pod_event( + pod_info: &PodInfo, + tracked_pods: Arc>>, + app_context: Arc, + port: u16, + pd_mode: bool, +) { + let worker_url = pod_info.worker_url(port); + + if pod_info.is_healthy() { + let should_add = { + let mut tracker = match tracked_pods.lock() { + Ok(tracker) => tracker, + Err(e) => { + error!("Failed to acquire tracked_pods lock: {}", e); + return; + } + }; + + if tracker.contains(pod_info) { + false + } else { + tracker.insert(pod_info.clone()); + true + } + }; + + if should_add { + info!( + "Adding pod: {} | type: {:?} | url: {}", + pod_info.name, pod_info.pod_type, worker_url + ); + + let worker_type = if pd_mode { + match &pod_info.pod_type { + Some(PodType::Prefill) => Some("prefill".to_string()), + Some(PodType::Decode) => Some("decode".to_string()), + Some(PodType::Regular) | None => None, + } + } else { + None + }; + + let bootstrap_port = if pd_mode { + match &pod_info.pod_type { + Some(PodType::Prefill) => pod_info.bootstrap_port, + _ => None, + } + } else { + None + }; + + let config = WorkerConfigRequest { + url: worker_url.clone(), + model_id: None, + worker_type, + priority: None, + cost: None, + runtime: None, + labels: HashMap::new(), + bootstrap_port, + tokenizer_path: None, + reasoning_parser: None, + tool_parser: None, + chat_template: None, + api_key: app_context.router_config.api_key.clone(), + health_check_timeout_secs: app_context.router_config.health_check.timeout_secs, + health_check_interval_secs: app_context + .router_config + .health_check + .check_interval_secs, + health_success_threshold: app_context.router_config.health_check.success_threshold, + health_failure_threshold: app_context.router_config.health_check.failure_threshold, + max_connection_attempts: app_context.router_config.health_check.success_threshold + * 20, + dp_aware: false, + }; + + let job = Job::AddWorker { + config: Box::new(config.clone()), + }; + + if let Some(job_queue) = app_context.worker_job_queue.get() { + match job_queue.submit(job).await { + Ok(_) => { + debug!("Worker addition job submitted for: {}", worker_url); + } + Err(e) => { + error!( + "Failed to submit worker addition job for {}: {}", + worker_url, e + ); + if let Ok(mut tracker) = tracked_pods.lock() { + tracker.remove(pod_info); + } + } + } + } else { + debug!( + "JobQueue not initialized, skipping async worker addition for: {}", + worker_url + ); + } + } + } +} + +async fn handle_pod_deletion( + pod_info: &PodInfo, + tracked_pods: Arc>>, + app_context: Arc, + port: u16, +) { + let worker_url = pod_info.worker_url(port); + + let was_tracked = { + let mut tracked = match tracked_pods.lock() { + Ok(tracked) => tracked, + Err(e) => { + error!("Failed to acquire tracked_pods lock during deletion: {}", e); + return; + } + }; + tracked.remove(pod_info) + }; + + if was_tracked { + info!( + "Removing pod: {} | type: {:?} | url: {}", + pod_info.name, pod_info.pod_type, worker_url + ); + + let job = Job::RemoveWorker { + url: worker_url.clone(), + }; + + if let Some(job_queue) = app_context.worker_job_queue.get() { + if let Err(e) = job_queue.submit(job).await { + error!( + "Failed to submit worker removal job for {}: {}", + worker_url, e + ); + } else { + debug!("Submitted worker removal job for {}", worker_url); + } + } else { + error!( + "JobQueue not initialized, cannot remove worker {}", + worker_url + ); + } + } else { + debug!( + "Pod deletion event for untracked/already removed pod: {} (type: {:?}). Worker URL: {}", + pod_info.name, pod_info.pod_type, worker_url + ); + } +} + +#[cfg(test)] +mod tests { + use k8s_openapi::{ + api::core::v1::{Pod, PodCondition, PodSpec, PodStatus}, + apimachinery::pkg::apis::meta::v1::{ObjectMeta, Time}, + }; + + use super::*; + + fn create_k8s_pod( + name: Option<&str>, + ip: Option<&str>, + phase: Option<&str>, + ready_status: Option<&str>, + deletion_timestamp: Option +""" + + result = self.detector.detect_and_parse(model_output, tools=self.tools) + + self.assertEqual(result.normal_text, "") + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "get_current_weather") + + params = json.loads(result.calls[0].parameters) + self.assertEqual(params["city"], "Dallas") + self.assertEqual(params["state"], "TX") + self.assertEqual(params["unit"], "fahrenheit") + + def test_detect_and_parse_with_content(self): + """Test parsing tool call with surrounding text.""" + model_output = """Sure! Let me check the weather for you. + + +Dallas + + +TX + + +fahrenheit + + +""" + + result = self.detector.detect_and_parse(model_output, tools=self.tools) + + self.assertEqual(result.normal_text, "Sure! Let me check the weather for you.") + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "get_current_weather") + + def test_detect_and_parse_multiline_param(self): + """Test parsing tool call with multiline parameter values.""" + model_output = """ + + +rectangle + + +{"width": 10, + "height": 20} + + +2 + + +""" + + result = self.detector.detect_and_parse(model_output, tools=self.tools) + + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "calculate_area") + + params = json.loads(result.calls[0].parameters) + self.assertEqual(params["shape"], "rectangle") + self.assertEqual(params["dimensions"], {"width": 10, "height": 20}) + self.assertEqual(params["precision"], 2) + + def test_detect_and_parse_parallel_tools(self): + """Test parsing multiple tool calls.""" + model_output = """ + + +Dallas + + +TX + + +fahrenheit + + + + + + +Orlando + + +FL + + +fahrenheit + + +""" + + result = self.detector.detect_and_parse(model_output, tools=self.tools) + + self.assertEqual(result.normal_text, "\n") + self.assertEqual(len(result.calls), 2) + + # First call + self.assertEqual(result.calls[0].name, "get_current_weather") + params1 = json.loads(result.calls[0].parameters) + self.assertEqual(params1["city"], "Dallas") + self.assertEqual(params1["state"], "TX") + + # Second call + self.assertEqual(result.calls[1].name, "get_current_weather") + params2 = json.loads(result.calls[1].parameters) + self.assertEqual(params2["city"], "Orlando") + self.assertEqual(params2["state"], "FL") + + def test_parse_streaming_simple(self): + """Test basic streaming parsing.""" + chunks = [ + "Sure! ", + "Let me check ", + "the weather.", + "", + "\n", + "\n", + "\nDallas", + "\n", + "\n", + "\nTX", + "\n", + "\n", + "\n", + ] + + accumulated_text = "" + accumulated_calls = [] + tool_calls_by_index = {} + + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, tools=self.tools) + accumulated_text += result.normal_text + + # Track calls by tool_index to handle streaming properly + for call in result.calls: + if call.tool_index is not None: + if call.tool_index not in tool_calls_by_index: + tool_calls_by_index[call.tool_index] = { + "name": "", + "parameters": "", + } + + if call.name: + tool_calls_by_index[call.tool_index]["name"] = call.name + if call.parameters: + tool_calls_by_index[call.tool_index][ + "parameters" + ] += call.parameters + + self.assertEqual(accumulated_text, "Sure! Let me check the weather.") + self.assertEqual(len(tool_calls_by_index), 1) + + # Get the complete tool call + tool_call = tool_calls_by_index[0] + self.assertEqual(tool_call["name"], "get_current_weather") + + # Parse the accumulated parameters + params = json.loads(tool_call["parameters"]) + self.assertEqual(params["city"], "Dallas") + self.assertEqual(params["state"], "TX") + + def test_parse_streaming_incomplete(self): + """Test streaming with incomplete tool call.""" + # Send incomplete tool call + chunks = [ + "", + "\n", + "\n", + "\nDallas", + "\n", + "\n", + "\nTX", + # Missing , , + ] + + tool_calls_by_index = {} + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, tools=self.tools) + + # Track calls by tool_index to handle streaming properly + for call in result.calls: + if call.tool_index is not None: + if call.tool_index not in tool_calls_by_index: + tool_calls_by_index[call.tool_index] = { + "name": "", + "parameters": "", + } + + if call.name: + tool_calls_by_index[call.tool_index]["name"] = call.name + if call.parameters: + tool_calls_by_index[call.tool_index][ + "parameters" + ] += call.parameters + + # Should have partial tool call with name but incomplete parameters + self.assertGreater(len(tool_calls_by_index), 0) + self.assertEqual(tool_calls_by_index[0]["name"], "get_current_weather") + + # Parameters should be incomplete (no closing brace) + params_str = tool_calls_by_index[0]["parameters"] + self.assertTrue(params_str.startswith('{"city": "Dallas"')) + self.assertFalse(params_str.endswith("}")) + + # Now complete it + result = self.detector.parse_streaming_increment( + "\n\n\n", tools=self.tools + ) + + # Update the accumulated parameters + for call in result.calls: + if call.tool_index is not None and call.parameters: + tool_calls_by_index[call.tool_index]["parameters"] += call.parameters + + # Now should have complete parameters + final_params = json.loads(tool_calls_by_index[0]["parameters"]) + self.assertEqual(final_params["city"], "Dallas") + self.assertEqual(final_params["state"], "TX") + + def test_edge_case_no_parameters(self): + """Test tool call without parameters.""" + model_output = """ + + +""" + + result = self.detector.detect_and_parse(model_output, tools=self.tools) + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "get_current_weather") + self.assertEqual(json.loads(result.calls[0].parameters), {}) + + def test_edge_case_special_chars_in_value(self): + """Test parameter with special characters in value.""" + model_output = """ + + +Dallas->TX + + +""" + + result = self.detector.detect_and_parse(model_output, tools=self.tools) + self.assertEqual(len(result.calls), 1) + + params = json.loads(result.calls[0].parameters) + self.assertEqual(params["city"], "Dallas->TX") + + def test_extract_tool_calls_fallback_no_tags(self): + """Test fallback parsing when XML tags are missing (just function without tool_call wrapper).""" + model_output = """ + +Dallas + + +TX + +""" + + result = self.detector.detect_and_parse(model_output, tools=self.tools) + + self.assertIsNotNone(result) + + def test_extract_tool_calls_type_conversion(self): + """Test parameter type conversion based on tool schema.""" + test_tool = Tool( + type="function", + function=Function( + name="test_types", + parameters={ + "type": "object", + "properties": { + "int_param": {"type": "integer"}, + "float_param": {"type": "float"}, + "bool_param": {"type": "boolean"}, + "str_param": {"type": "string"}, + "obj_param": {"type": "object"}, + }, + }, + ), + ) + + model_output = """ + + +42 + + +3.14 + + +true + + +hello world + + +{"key": "value"} + + +""" + + result = self.detector.detect_and_parse(model_output, tools=[test_tool]) + + self.assertEqual(len(result.calls), 1) + params = json.loads(result.calls[0].parameters) + self.assertEqual(params["int_param"], 42) + self.assertEqual(params["float_param"], 3.14) + self.assertEqual(params["bool_param"], True) + self.assertEqual(params["str_param"], "hello world") + self.assertEqual(params["obj_param"], {"key": "value"}) + + def test_parse_streaming_incremental(self): + """Test that streaming is truly incremental with very small chunks.""" + model_output = """I'll check the weather. + + + Dallas + + + TX + + + """ + + # Simulate more realistic token-based chunks where is a single token + chunks = [ + "I'll check the weather.", + "", + "\n\n", + "\n", + "Dallas\n", + "\n", + "\n", + "TX\n", + "\n", + "\n", + "", + ] + + accumulated_text = "" + tool_calls = [] + chunks_count = 0 + + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + accumulated_text += result.normal_text + chunks_count += 1 + for tool_call_chunk in result.calls: + if ( + hasattr(tool_call_chunk, "tool_index") + and tool_call_chunk.tool_index is not None + ): + while len(tool_calls) <= tool_call_chunk.tool_index: + tool_calls.append({"name": "", "parameters": ""}) + tc = tool_calls[tool_call_chunk.tool_index] + if tool_call_chunk.name: + tc["name"] = tool_call_chunk.name + if tool_call_chunk.parameters: + tc["parameters"] += tool_call_chunk.parameters + + self.assertGreater(chunks_count, 3) + + # Verify the accumulated results + self.assertIn("I'll check the weather.", accumulated_text) + self.assertEqual(len(tool_calls), 1) + self.assertEqual(tool_calls[0]["name"], "get_current_weather") + + params = json.loads(tool_calls[0]["parameters"]) + self.assertEqual(params, {"city": "Dallas", "state": "TX"}) + + def test_parse_streaming_multiple_tools(self): + """Test streaming with multiple tool calls.""" + model_output = """ + + + Dallas + + + TX + + + + Some text in between. + + + + circle + + + {"radius": 5} + + + """ + + # Simulate streaming by chunks + chunk_size = 20 + chunks = [ + model_output[i : i + chunk_size] + for i in range(0, len(model_output), chunk_size) + ] + + accumulated_text = "" + tool_calls = [] + chunks_count = 0 + + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + accumulated_text += result.normal_text + chunks_count += 1 + for tool_call_chunk in result.calls: + if ( + hasattr(tool_call_chunk, "tool_index") + and tool_call_chunk.tool_index is not None + ): + while len(tool_calls) <= tool_call_chunk.tool_index: + tool_calls.append({"name": "", "parameters": ""}) + tc = tool_calls[tool_call_chunk.tool_index] + if tool_call_chunk.name: + tc["name"] = tool_call_chunk.name + if tool_call_chunk.parameters: + tc["parameters"] += tool_call_chunk.parameters + + self.assertIn("Some text in between.", accumulated_text) + self.assertEqual(len(tool_calls), 2) + self.assertEqual(tool_calls[0]["name"], "get_current_weather") + self.assertEqual(tool_calls[1]["name"], "calculate_area") + + # Verify parameters + params1 = json.loads(tool_calls[0]["parameters"]) + self.assertEqual(params1, {"city": "Dallas", "state": "TX"}) + + params2 = json.loads(tool_calls[1]["parameters"]) + self.assertEqual(params2, {"shape": "circle", "dimensions": {"radius": 5}}) + + +class TestGlm4MoeDetector(unittest.TestCase): + def setUp(self): + self.tools = [ + Tool( + type="function", + function=Function( + name="get_weather", + description="Get weather information", + parameters={ + "type": "object", + "properties": { + "city": {"type": "string", "description": "City name"}, + "date": {"type": "string", "description": "Date"}, + }, + "required": ["city", "date"], + }, + ), + ), + ] + self.detector = Glm4MoeDetector() + + def test_single_tool_call(self): + text = ( + "get_weather\n" + "city\nBeijing\n" + "date\n2024-06-27\n" + "" + ) + result = self.detector.detect_and_parse(text, self.tools) + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "get_weather") + self.assertEqual( + result.calls[0].parameters, '{"city": "Beijing", "date": "2024-06-27"}' + ) + self.assertEqual(result.normal_text, "") + + def test_multiple_tool_calls(self): + text = ( + "get_weather\n" + "city\nBeijing\n" + "date\n2024-06-27\n" + "" + "get_weather\n" + "city\nShanghai\n" + "date\n2024-06-28\n" + "" + ) + result = self.detector.detect_and_parse(text, self.tools) + self.assertEqual(len(result.calls), 2) + self.assertEqual(result.calls[0].name, "get_weather") + self.assertEqual( + result.calls[0].parameters, '{"city": "Beijing", "date": "2024-06-27"}' + ) + self.assertEqual(result.calls[1].name, "get_weather") + self.assertEqual( + result.calls[1].parameters, '{"city": "Shanghai", "date": "2024-06-28"}' + ) + self.assertEqual(result.normal_text, "") + + def test_streaming_tool_call(self): + """Test streaming incremental parsing of a tool call.""" + chunks = [ + "get_weather\n", + "city\nBeijing\n", + "date\n2024-06-27\n", + "", + ] + tool_calls = [] + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + for tool_call_chunk in result.calls: + if ( + hasattr(tool_call_chunk, "tool_index") + and tool_call_chunk.tool_index is not None + ): + while len(tool_calls) <= tool_call_chunk.tool_index: + tool_calls.append({"name": "", "parameters": {}}) + tc = tool_calls[tool_call_chunk.tool_index] + if tool_call_chunk.name: + tc["name"] = tool_call_chunk.name + if tool_call_chunk.parameters: + tc["parameters"] = tool_call_chunk.parameters + self.assertEqual(len(tool_calls), 1) + self.assertEqual(tool_calls[0]["name"], "get_weather") + self.assertEqual( + tool_calls[0]["parameters"], '{"city": "Beijing", "date": "2024-06-27"}' + ) + + def test_streaming_multiple_tool_calls(self): + """Test streaming incremental parsing of multiple tool calls.""" + chunks = [ + "get_weather\n", + "city\nBeijing\n", + "date\n2024-06-27\n", + "get_weather\n", + "city\nShanghai\n", + "date\n2024-06-28\n", + "", + ] + tool_calls = [] + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + for tool_call_chunk in result.calls: + if ( + hasattr(tool_call_chunk, "tool_index") + and tool_call_chunk.tool_index is not None + ): + while len(tool_calls) <= tool_call_chunk.tool_index: + tool_calls.append({"name": "", "parameters": {}}) + tc = tool_calls[tool_call_chunk.tool_index] + if tool_call_chunk.name: + tc["name"] = tool_call_chunk.name + if tool_call_chunk.parameters: + tc["parameters"] = tool_call_chunk.parameters + self.assertEqual(len(tool_calls), 2) + self.assertEqual(tool_calls[0]["name"], "get_weather") + self.assertEqual( + tool_calls[0]["parameters"], '{"city": "Beijing", "date": "2024-06-27"}' + ) + self.assertEqual(tool_calls[1]["name"], "get_weather") + self.assertEqual( + tool_calls[1]["parameters"], '{"city": "Shanghai", "date": "2024-06-28"}' + ) + + def test_tool_call_id(self): + """Test that the buffer and state are reset after a tool call is completed.""" + chunks = [ + "get_weather\n", + "city\nBeijing\n", + "date\n2024-06-27\n", + "", + ] + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + self.assertEqual(self.detector.current_tool_id, 1) + + def test_invalid_tool_call(self): + """Test that invalid tool calls are handled correctly.""" + text = "invalid_func\ncity\nBeijing\n" + result = self.detector.detect_and_parse(text, self.tools) + self.assertEqual(len(result.calls), 0) + + def test_partial_tool_call(self): + """Test parsing a partial tool call that spans multiple chunks.""" + text1 = "get_weather\ncity\n" + result1 = self.detector.parse_streaming_increment(text1, self.tools) + self.assertEqual(result1.normal_text, "") + self.assertEqual(result1.calls, []) + self.assertEqual(self.detector._buffer, text1) + text2 = "Beijing\ndate\n2024-06-27\n" + result2 = self.detector.parse_streaming_increment(text2, self.tools) + self.assertEqual(len(result2.calls), 1) + self.assertEqual(result2.calls[0].name, "get_weather") + self.assertEqual( + result2.calls[0].parameters, '{"city": "Beijing", "date": "2024-06-27"}' + ) + self.assertEqual(self.detector._buffer, "") + + def test_array_argument_with_escaped_json(self): + """Test that array arguments with escaped JSON are properly handled without double-escaping.""" + # Add a tool with array parameter + tools_with_array = [ + Tool( + type="function", + function=Function( + name="todo_write", + description="Write todos", + parameters={ + "type": "object", + "properties": { + "todos": { + "type": "array", + "description": "The updated todo list", + } + }, + "required": ["todos"], + }, + ), + ), + ] + + def check_params(result): + self.assertEqual(1, len(result.calls)) + self.assertEqual("todo_write", result.calls[0].name) + params = json.loads(result.calls[0].parameters) + self.assertIsInstance(params["todos"], list) + self.assertEqual(4, len(params["todos"])) + self.assertEqual("1", params["todos"][0]["id"]) + self.assertEqual( + "Check for hard-coded issues in the backend code", + params["todos"][0]["task"], + ) + self.assertEqual("in_progress", params["todos"][0]["status"]) + self.assertEqual("2", params["todos"][1]["id"]) + self.assertEqual( + "Check for hard-coded issues in the frontend code", + params["todos"][1]["task"], + ) + self.assertEqual("pending", params["todos"][1]["status"]) + self.assertEqual("3", params["todos"][2]["id"]) + self.assertEqual( + "Check for code violating the Single Responsibility Principle", + params["todos"][2]["task"], + ) + self.assertEqual("pending", params["todos"][2]["status"]) + self.assertEqual("4", params["todos"][3]["id"]) + self.assertEqual( + "Generate a rectification proposal report", params["todos"][3]["task"] + ) + self.assertEqual("pending", params["todos"][3]["status"]) + + # Simulate the raw response from GLM-4.6 model with normal and escaped JSON in XML + result = self.detector.detect_and_parse( + """todo_write\ntodos\n[{\"id\": \"1\", \"task\": \"Check for hard-coded issues in the backend code\", \"status\": \"in_progress\"}, {\"id\": \"2\", \"task\": \"Check for hard-coded issues in the frontend code\", \"status\": \"pending\"}, {\"id\": \"3\", \"task\": \"Check for code violating the Single Responsibility Principle\", \"status\": \"pending\"}, {\"id\": \"4\", \"task\": \"Generate a rectification proposal report\", \"status\": \"pending\"}] +""", + tools_with_array, + ) + check_params(result) + result = self.detector.detect_and_parse( + r"""todo_write\ntodos\n[{\"id\": \"1\", \"task\": \"Check for hard-coded issues in the backend code\", \"status\": \"in_progress\"}, {\"id\": \"2\", \"task\": \"Check for hard-coded issues in the frontend code\", \"status\": \"pending\"}, {\"id\": \"3\", \"task\": \"Check for code violating the Single Responsibility Principle\", \"status\": \"pending\"}, {\"id\": \"4\", \"task\": \"Generate a rectification proposal report\", \"status\": \"pending\"}] +""", + tools_with_array, + ) + check_params(result) + + def check_single_todos(tool_result, expected): + self.assertEqual(1, len(tool_result.calls)) + self.assertEqual("todo_write", tool_result.calls[0].name) + params = json.loads(tool_result.calls[0].parameters) + self.assertIsInstance(params["todos"], list) + self.assertEqual(1, len(params["todos"])) + self.assertEqual("1", params["todos"][0]["id"]) + self.assertEqual(expected, params["todos"][0]["task"]) + self.assertEqual("pending", params["todos"][0]["status"]) + + # Test with escaped JSON containing backslashes in content (e.g., Windows paths) + expected_path = r"Check file at C:\Users\test.txt" + result = self.detector.detect_and_parse( + """todo_write\ntodos\n[{\"id\": \"1\", \"task\": \"Check file at C:\\\\Users\\\\test.txt\", \"status\": \"pending\"}]""", + tools_with_array, + ) + check_single_todos(result, expected_path) + result = self.detector.detect_and_parse( + r"""todo_write\ntodos\n[{\"id\": \"1\", \"task\": \"Check file at C:\\\\Users\\\\test.txt\", \"status\": \"pending\"}]""", + tools_with_array, + ) + check_single_todos(result, expected_path) + + # Should contain literal \n, not actual newline + expected_output = r"Print \n to see newline" + result = self.detector.detect_and_parse( + """todo_write\ntodos\n[{\"id\": \"1\", \"task\": \"Print \\\\n to see newline\",\"status\": \"pending\"}]""", + tools_with_array, + ) + check_single_todos(result, expected_output) + result = self.detector.detect_and_parse( + r"""todo_write\ntodos\n[{\"id\": \"1\", \"task\": \"Print \\\\n to see newline\",\"status\": \"pending\"}]""", + tools_with_array, + ) + check_single_todos(result, expected_output) + + +class TestJsonArrayParser(unittest.TestCase): + def setUp(self): + # Create sample tools for testing + self.tools = [ + Tool( + type="function", + function=Function( + name="get_weather", + description="Get weather information", + parameters={ + "properties": { + "location": { + "type": "string", + "description": "Location to get weather for", + }, + "unit": { + "type": "string", + "description": "Temperature unit", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["location"], + }, + ), + ), + Tool( + type="function", + function=Function( + name="search", + description="Search for information", + parameters={ + "properties": { + "query": { + "type": "string", + "description": "Search query", + }, + }, + "required": ["query"], + }, + ), + ), + ] + self.detector = JsonArrayParser() + + def test_json_detector_has_no_ebnf(self): + """JsonArrayParser no longer exposes EBNF generation helpers.""" + self.assertFalse( + hasattr(self.detector, "build_ebnf"), + "JsonArrayParser should not expose EBNF helpers after cleanup", + ) + + def test_parse_streaming_increment_malformed_json(self): + """Test parsing with malformed JSON""" + # Test with malformed JSON + text = '[{"name": "get_weather", "parameters": {"location": "Tokyo"' + result = self.detector.parse_streaming_increment(text, self.tools) + + # Should not crash and return a valid result + self.assertIsInstance(result, StreamingParseResult) + + text = "[{}}}]" + result = self.detector.parse_streaming_increment(text, self.tools) + + self.assertIsInstance(result, StreamingParseResult) + + def test_parse_streaming_increment_empty_input(self): + """Test parsing with empty input""" + result = self.detector.parse_streaming_increment("", self.tools) + self.assertEqual(len(result.calls), 0) + self.assertEqual(result.normal_text, "") + + def test_parse_streaming_increment_whitespace_handling(self): + """Test parsing with various whitespace scenarios""" + # Test with leading/trailing whitespace split across chunks + chunk1 = ' [{"name": "get_weather", "parameters": ' + result1 = self.detector.parse_streaming_increment(chunk1, self.tools) + self.assertIsInstance(result1, StreamingParseResult) + chunk2 = '{"location": "Tokyo"}}] ' + result2 = self.detector.parse_streaming_increment(chunk2, self.tools) + + # The base class should handle this + self.assertIsInstance(result2, StreamingParseResult) + + def test_parse_streaming_increment_nested_objects(self): + """Test parsing with nested JSON objects""" + chunk1 = '[{"name": "get_weather", "parameters": {"location": "Tokyo", ' + result1 = self.detector.parse_streaming_increment(chunk1, self.tools) + self.assertIsInstance(result1, StreamingParseResult) + chunk2 = '"nested": {"key": "value"}}}]' + result2 = self.detector.parse_streaming_increment(chunk2, self.tools) + + # The base class should handle this + self.assertIsInstance(result2, StreamingParseResult) + + def test_json_parsing_with_commas(self): + """Test that JSON parsing works correctly with comma separators""" + # Stream two complete objects, at least 2 chunks per tool call + chunk1 = '[{"name": "get_weather", "parameters": {"location": "Tok' + result1 = self.detector.parse_streaming_increment(chunk1, self.tools) + self.assertIsInstance(result1, StreamingParseResult) + chunk2 = 'yo"}},' + result2 = self.detector.parse_streaming_increment(chunk2, self.tools) + self.assertIsInstance(result2, StreamingParseResult) + + chunk3 = '{"name": "get_weather", "parameters": {"location": "Par' + result3 = self.detector.parse_streaming_increment(chunk3, self.tools) + self.assertIsInstance(result3, StreamingParseResult) + chunk4 = 'is"}}]' + result4 = self.detector.parse_streaming_increment(chunk4, self.tools) + self.assertIsInstance(result4, StreamingParseResult) + self.assertGreater( + len(result4.calls), 0, "Should parse tool calls from text with separators" + ) + + def test_braces_in_strings(self): + """Test that JSON with } characters inside strings works correctly""" + # Test case: JSON array with } inside string values - streamed across chunks + chunk1 = '[{"name": "get_weather", "parameters": {"location": "has } inside"' + result1 = self.detector.parse_streaming_increment(chunk1, self.tools) + self.assertIsInstance(result1, StreamingParseResult) + chunk2 = "}}" + result2 = self.detector.parse_streaming_increment(chunk2, self.tools) + self.assertIsInstance(result2, StreamingParseResult) + self.assertGreater( + len(result2.calls), 0, "Should parse tool call with } in string" + ) + + # Test with separator (streaming in progress) + chunk3 = '[{"name": "get_weather", "parameters": {"location": "has } inside"}' + result3 = self.detector.parse_streaming_increment(chunk3, self.tools) + self.assertIsInstance(result3, StreamingParseResult) + chunk4 = "}," + result4 = self.detector.parse_streaming_increment(chunk4, self.tools) + self.assertIsInstance(result4, StreamingParseResult) + chunk5 = '{"name": "get_weather"' + result5 = self.detector.parse_streaming_increment(chunk5, self.tools) + self.assertIsInstance(result5, StreamingParseResult) + self.assertGreater( + len(result5.calls), + 0, + "Should parse tool calls with separator and } in string", + ) + + def test_separator_in_same_chunk(self): + """Test that separator already present in chunk works correctly""" + # Test case: separator already in the chunk (streaming in progress) with 2+ chunks per tool call + chunk1 = '[{"name": "get_weather", "parameters": {"location": "Tokyo"' + result1 = self.detector.parse_streaming_increment(chunk1, self.tools) + self.assertIsInstance(result1, StreamingParseResult) + chunk2 = '}},{"name": "get_weather"' + result2 = self.detector.parse_streaming_increment(chunk2, self.tools) + self.assertIsInstance(result2, StreamingParseResult) + self.assertGreater( + len(result2.calls), + 0, + "Should parse tool calls with separator in same chunk", + ) + + def test_separator_in_separate_chunk(self): + """Test that separator in separate chunk works correctly""" + # Test case: separator in separate chunk - this tests streaming behavior + chunk1 = '[{"name": "get_weather", "parameters": {"location": "Tokyo"}}' + chunk2 = "," + chunk3 = '{"name": "get_weather", "parameters": {"location": "Paris"}}' + + # Process first chunk + result1 = self.detector.parse_streaming_increment(chunk1, self.tools) + self.assertIsInstance(result1, StreamingParseResult) + + # Process separator chunk + result2 = self.detector.parse_streaming_increment(chunk2, self.tools) + self.assertIsInstance(result2, StreamingParseResult) + + # Process second chunk (streaming in progress) + result3 = self.detector.parse_streaming_increment(chunk3, self.tools) + self.assertIsInstance(result3, StreamingParseResult) + + def test_incomplete_json_across_chunks(self): + """Test that incomplete JSON across chunks works correctly""" + # Test case: incomplete JSON across chunks - this tests streaming behavior + chunk1 = '[{"name": "get_weather", "parameters": {"location": "Tokyo"' + chunk2 = '}},{"name": "get_weather"' + + # Process first chunk (incomplete) + result1 = self.detector.parse_streaming_increment(chunk1, self.tools) + self.assertIsInstance(result1, StreamingParseResult) + + # Process second chunk (completes first object and starts second, streaming in progress) + result2 = self.detector.parse_streaming_increment(chunk2, self.tools) + self.assertIsInstance(result2, StreamingParseResult) + + def test_malformed_json_recovery(self): + """Test that malformed JSON recovers gracefully""" + # Test with malformed JSON - should handle gracefully + malformed_text = ( + '[{"name": "get_weather", "parameters": {"location": "unclosed string' + ) + + result1 = self.detector.parse_streaming_increment(malformed_text, self.tools) + self.assertIsInstance(result1, StreamingParseResult) + + # Test valid JSON after malformed - streamed across 2 chunks (streaming in progress) + valid_chunk1 = '[{"name": "get_weather", "parameters": {"location": "Tok' + result2 = self.detector.parse_streaming_increment(valid_chunk1, self.tools) + self.assertIsInstance(result2, StreamingParseResult) + valid_chunk2 = 'yo"}}' + result3 = self.detector.parse_streaming_increment(valid_chunk2, self.tools) + self.assertIsInstance(result3, StreamingParseResult) + + def test_nested_objects_with_commas(self): + """Test that nested objects with commas inside work correctly""" + # Test with nested objects that have commas - should work with json.loads() + chunk1 = '[{"name": "get_weather", "parameters": {"location": "Tok' + result1 = self.detector.parse_streaming_increment(chunk1, self.tools) + self.assertIsInstance(result1, StreamingParseResult) + chunk2 = 'yo", "unit": "celsius"}}' + result2 = self.detector.parse_streaming_increment(chunk2, self.tools) + self.assertIsInstance(result2, StreamingParseResult) + self.assertGreater( + len(result2.calls), 0, "Should parse tool call with nested objects" + ) + + def test_empty_objects(self): + """Test that empty objects work correctly""" + # Test with empty objects - should work with json.loads() + chunk1 = '[{"name": "get_weather", "parameters": ' + result1 = self.detector.parse_streaming_increment(chunk1, self.tools) + self.assertIsInstance(result1, StreamingParseResult) + chunk2 = "{}}" + result2 = self.detector.parse_streaming_increment(chunk2, self.tools) + self.assertIsInstance(result2, StreamingParseResult) + + def test_whitespace_handling(self): + """Test that various whitespace scenarios work correctly""" + # Test with various whitespace patterns - should work with json.loads() + chunk1 = ' \n\n [{"name": "get_weather", "parameters": ' + result1 = self.detector.parse_streaming_increment(chunk1, self.tools) + self.assertIsInstance(result1, StreamingParseResult) + chunk2 = '{"location": "Tokyo"}}' + result2 = self.detector.parse_streaming_increment(chunk2, self.tools) + self.assertIsInstance(result2, StreamingParseResult) + + def test_multiple_commas_in_chunk(self): + """Test that multiple commas in a single chunk work correctly""" + # Stream multiple tool calls ensuring at least 2 chunks per complete tool call + chunk1 = '[{"name": "get_weather", "parameters": {"location": "To' + result1 = self.detector.parse_streaming_increment(chunk1, self.tools) + self.assertIsInstance(result1, StreamingParseResult) + chunk2 = 'kyo"}},' + result2 = self.detector.parse_streaming_increment(chunk2, self.tools) + self.assertIsInstance(result2, StreamingParseResult) + + chunk3 = '{"name": "get_weather", "parameters": {"location": "Pa' + result3 = self.detector.parse_streaming_increment(chunk3, self.tools) + self.assertIsInstance(result3, StreamingParseResult) + chunk4 = 'ris"}},' + result4 = self.detector.parse_streaming_increment(chunk4, self.tools) + self.assertIsInstance(result4, StreamingParseResult) + + chunk5 = '{"name": "get_weather"' + result5 = self.detector.parse_streaming_increment(chunk5, self.tools) + self.assertIsInstance(result5, StreamingParseResult) + self.assertGreater( + len(result5.calls), 0, "Should parse tool calls with multiple commas" + ) + + def test_complete_tool_call_with_trailing_comma(self): + """Test that complete tool call with trailing comma parses correctly""" + # Test case: complete tool call followed by comma at end of chunk (split across 2 chunks) + chunk1 = '[{"name": "get_weather", "parameters": {"location": "Tokyo"}' + result1 = self.detector.parse_streaming_increment(chunk1, self.tools) + self.assertIsInstance(result1, StreamingParseResult) + chunk2 = "}, " + result2 = self.detector.parse_streaming_increment(chunk2, self.tools) + self.assertIsInstance(result2, StreamingParseResult) + self.assertGreater(len(result2.calls), 0, "Should parse complete tool call") + + # Test that next chunk with opening brace gets the separator prepended + next_chunk = '{"name": "get_weather", "parameters": {"location": "Paris"}}' + result_next = self.detector.parse_streaming_increment(next_chunk, self.tools) + self.assertIsInstance(result_next, StreamingParseResult) + self.assertGreater( + len(result_next.calls), 0, "Should parse subsequent tool call" + ) + + def test_three_tool_calls_separate_chunks_with_commas(self): + """Test parsing 3 tool calls in separate chunks with commas at the end""" + # First tool call: 2 chunks + chunk1_1 = '[{"name": "get_weather", "parameters": ' + result1_1 = self.detector.parse_streaming_increment(chunk1_1, self.tools) + chunk1_2 = '{"location": "Tokyo"}},' + result1_2 = self.detector.parse_streaming_increment(chunk1_2, self.tools) + self.assertIsInstance(result1_2, StreamingParseResult) + self.assertGreater(len(result1_2.calls), 0, "Should parse first tool call") + + # Second tool call: 2 chunks + chunk2_1 = '{"name": "search", "parameters": ' + result2_1 = self.detector.parse_streaming_increment(chunk2_1, self.tools) + chunk2_2 = '{"query": "restaurants"}},' + result2_2 = self.detector.parse_streaming_increment(chunk2_2, self.tools) + self.assertIsInstance(result2_2, StreamingParseResult) + self.assertGreater(len(result2_2.calls), 0, "Should parse second tool call") + + # Third tool call: 2 chunks + chunk3_1 = '{"name": "get_weather", "parameters": ' + result3_1 = self.detector.parse_streaming_increment(chunk3_1, self.tools) + chunk3_2 = '{"location": "Paris"}}]' + result3_2 = self.detector.parse_streaming_increment(chunk3_2, self.tools) + self.assertIsInstance(result3_2, StreamingParseResult) + self.assertGreater(len(result3_2.calls), 0, "Should parse third tool call") + # Verify all tool calls were parsed correctly + total_calls = len(result1_2.calls) + len(result2_2.calls) + len(result3_2.calls) + self.assertEqual(total_calls, 3, "Should have parsed exactly 3 tool calls") + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/function_call/test_json_schema_constraint.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/function_call/test_json_schema_constraint.py new file mode 100644 index 00000000..6a7131ca --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/function_call/test_json_schema_constraint.py @@ -0,0 +1,567 @@ +""" +Tests for JSON schema constraint functionality used by JsonArrayParser +""" + +import unittest + +import jsonschema + +from sglang.srt.entrypoints.openai.protocol import ( + Function, + Tool, + ToolChoice, + ToolChoiceFuncName, +) +from sglang.srt.function_call.utils import ( + _get_tool_schema_defs, + get_json_schema_constraint, +) +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(1.0, "default") + + +class TestJsonSchemaConstraint(unittest.TestCase): + """Test JSON schema constraint generation for tool choices""" + + def setUp(self): + """Set up test tools""" + self.tools = [ + Tool( + type="function", + function=Function( + name="get_weather", + description="Get weather information", + parameters={ + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "Location to get weather for", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "Temperature unit", + }, + }, + "required": ["location"], + }, + ), + ), + Tool( + type="function", + function=Function( + name="search", + description="Search for information", + parameters={ + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Search query", + }, + }, + "required": ["query"], + }, + ), + ), + ] + + def test_required_tool_choice_schema(self): + """Test schema generation for tool_choice='required'""" + schema = get_json_schema_constraint(self.tools, "required") + + self.assertIsNotNone(schema) + jsonschema.Draft202012Validator.check_schema(schema) + + self.assertEqual(schema["type"], "array") + self.assertEqual(schema["minItems"], 1) + self.assertIn("items", schema) + self.assertIn("anyOf", schema["items"]) + + # Should have schemas for both tools + self.assertEqual(len(schema["items"]["anyOf"]), 2) + + # Check that each tool schema is present + tool_names = [ + item["properties"]["name"]["enum"][0] for item in schema["items"]["anyOf"] + ] + self.assertIn("get_weather", tool_names) + self.assertIn("search", tool_names) + + def test_specific_tool_choice_schema(self): + """Test schema generation for specific tool choice""" + tool_choice = ToolChoice( + type="function", function=ToolChoiceFuncName(name="get_weather") + ) + schema = get_json_schema_constraint(self.tools, tool_choice) + + self.assertIsNotNone(schema) + jsonschema.Draft202012Validator.check_schema(schema) + + self.assertEqual(schema["type"], "array") + self.assertEqual(schema["minItems"], 1) + self.assertEqual(schema["maxItems"], 1) + + # Should only have schema for the specific tool + item_schema = schema["items"] + self.assertEqual(item_schema["properties"]["name"]["enum"], ["get_weather"]) + self.assertIn("parameters", item_schema["properties"]) + + def test_specific_tool_choice_dict_schema(self): + """Test schema generation for specific tool choice as ToolChoice object""" + tool_choice = ToolChoice( + type="function", function=ToolChoiceFuncName(name="search") + ) + schema = get_json_schema_constraint(self.tools, tool_choice) + + self.assertIsNotNone(schema) + jsonschema.Draft202012Validator.check_schema(schema) + + self.assertEqual(schema["type"], "array") + self.assertEqual(schema["minItems"], 1) + self.assertEqual(schema["maxItems"], 1) + + # Should only have schema for the specific tool + item_schema = schema["items"] + self.assertEqual(item_schema["properties"]["name"]["enum"], ["search"]) + self.assertIn("parameters", item_schema["properties"]) + + def test_nonexistent_tool_choice(self): + """Test schema generation for nonexistent tool""" + tool_choice = ToolChoice( + type="function", function=ToolChoiceFuncName(name="nonexistent") + ) + schema = get_json_schema_constraint(self.tools, tool_choice) + + self.assertIsNone(schema) + + def test_nonexistent_tool_choice_dict(self): + """Test schema generation for nonexistent tool as dict""" + tool_choice = {"type": "function", "function": {"name": "nonexistent"}} + schema = get_json_schema_constraint(self.tools, tool_choice) + + self.assertIsNone(schema) + + def test_auto_tool_choice_schema(self): + """Test schema generation for tool_choice='auto'""" + schema = get_json_schema_constraint(self.tools, "auto") + + self.assertIsNone(schema) + + def test_none_tool_choice_schema(self): + """Test schema generation for tool_choice=None""" + schema = get_json_schema_constraint(self.tools, None) + + self.assertIsNone(schema) + + def test_tools_with_defs(self): + """Test schema generation with tools that have $defs""" + tools_with_defs = [ + Tool( + type="function", + function=Function( + name="complex_tool", + description="Tool with complex schema", + parameters={ + "type": "object", + "properties": { + "data": { + "type": "object", + "properties": { + "nested": {"$ref": "#/$defs/NestedType"}, + }, + }, + }, + "$defs": { + "NestedType": { + "type": "object", + "properties": { + "value": {"type": "string"}, + }, + }, + }, + }, + ), + ), + ] + + try: + _get_tool_schema_defs(tools_with_defs) + except ValueError as e: + self.fail(f"Should not raise ValueError, but got: {e}") + + schema = get_json_schema_constraint(tools_with_defs, "required") + + self.assertIsNotNone(schema) + jsonschema.Draft202012Validator.check_schema(schema) + + self.assertIn("$defs", schema) + self.assertIn("NestedType", schema["$defs"]) + + def test_tools_without_parameters(self): + """Test schema generation with tools that have no parameters""" + tools_without_params = [ + Tool( + type="function", + function=Function( + name="simple_tool", + description="Tool without parameters", + parameters=None, + ), + ), + ] + + schema = get_json_schema_constraint(tools_without_params, "required") + + self.assertIsNotNone(schema) + jsonschema.Draft202012Validator.check_schema(schema) + + item_schema = schema["items"]["anyOf"][0] + self.assertEqual( + item_schema["properties"]["parameters"], + {"type": "object", "properties": {}}, + ) + + def test_conflicting_defs_raises_valueerror(self): + """Test that conflicting tool definitions raise ValueError with proper message""" + tools_with_conflicting_defs = [ + Tool( + type="function", + function=Function( + name="tool1", + description="Tool 1", + parameters={ + "type": "object", + "properties": {}, + "$defs": { + "ConflictingType": { + "type": "object", + "properties": {"value": {"type": "string"}}, + }, + }, + }, + ), + ), + Tool( + type="function", + function=Function( + name="tool2", + description="Tool 2", + parameters={ + "type": "object", + "properties": {}, + "$defs": { + "ConflictingType": { + "type": "object", + "properties": {"value": {"type": "number"}}, + }, + }, + }, + ), + ), + ] + + with self.assertRaises(ValueError) as context: + _get_tool_schema_defs(tools_with_conflicting_defs) + + self.assertIn( + "Tool definition 'ConflictingType' has multiple schemas", + str(context.exception), + ) + self.assertIn("which is not supported", str(context.exception)) + + def test_tools_with_empty_defs(self): + """Test tools with empty $defs objects""" + tools_with_empty_defs = [ + Tool( + type="function", + function=Function( + name="empty_defs_tool", + description="Tool with empty $defs", + parameters={ + "type": "object", + "properties": { + "data": {"type": "string"}, + }, + "required": ["data"], + "$defs": {}, + }, + ), + ), + ] + + try: + _get_tool_schema_defs(tools_with_empty_defs) + except ValueError as e: + self.fail(f"Should not raise ValueError, but got: {e}") + + schema = get_json_schema_constraint(tools_with_empty_defs, "required") + self.assertIsNotNone(schema) + jsonschema.Draft202012Validator.check_schema(schema) + + # Should not have $defs section when empty + self.assertNotIn("$defs", schema) + + def test_tools_with_identical_defs(self): + """Test different tools with same $defs names but identical schemas (should not raise exception)""" + tools_with_identical_defs = [ + Tool( + type="function", + function=Function( + name="weather_tool", + description="Get weather information", + parameters={ + "type": "object", + "properties": { + "location": {"$ref": "#/$defs/Location"}, + }, + "required": ["location"], + "$defs": { + "Location": { + "type": "object", + "properties": { + "lat": {"type": "number"}, + "lon": {"type": "number"}, + }, + "required": ["lat", "lon"], + }, + }, + }, + ), + ), + Tool( + type="function", + function=Function( + name="address_tool", + description="Get address information", + parameters={ + "type": "object", + "properties": { + "address": {"$ref": "#/$defs/Location"}, + }, + "required": ["address"], + "$defs": { + "Location": { + "type": "object", + "properties": { + "lat": {"type": "number"}, + "lon": {"type": "number"}, + }, + "required": ["lat", "lon"], + }, + }, + }, + ), + ), + ] + + try: + _get_tool_schema_defs(tools_with_identical_defs) + except ValueError as e: + self.fail( + f"Should not raise ValueError for identical schemas, but got: {e}" + ) + + # Also test that schema generation works + schema = get_json_schema_constraint(tools_with_identical_defs, "required") + self.assertIsNotNone(schema) + jsonschema.Draft202012Validator.check_schema(schema) + + # Verify both tools are present + tool_names = [ + item["properties"]["name"]["enum"][0] for item in schema["items"]["anyOf"] + ] + self.assertIn("weather_tool", tool_names) + self.assertIn("address_tool", tool_names) + + # Should have $defs with Location + self.assertIn("$defs", schema) + self.assertIn("Location", schema["$defs"]) + + def test_tools_with_nested_defs(self): + """Test tools with nested $defs""" + tools_with_nested_defs = [ + Tool( + type="function", + function=Function( + name="complex_tool", + description="Tool with nested $defs", + parameters={ + "type": "object", + "properties": { + "user": {"$ref": "#/$defs/User"}, + "settings": {"$ref": "#/$defs/Settings"}, + }, + "required": ["user"], + "$defs": { + "User": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "profile": {"$ref": "#/$defs/Profile"}, + }, + "required": ["id"], + }, + "Profile": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "email": {"type": "string", "format": "email"}, + }, + "required": ["name"], + }, + "Settings": { + "type": "object", + "properties": { + "theme": { + "type": "string", + "enum": ["light", "dark"], + }, + "notifications": {"type": "boolean"}, + }, + }, + }, + }, + ), + ), + ] + + try: + _get_tool_schema_defs(tools_with_nested_defs) + except ValueError as e: + self.fail(f"Should not raise ValueError, but got: {e}") + + schema = get_json_schema_constraint(tools_with_nested_defs, "required") + self.assertIsNotNone(schema) + jsonschema.Draft202012Validator.check_schema(schema) + + # Verify all $defs are properly included + self.assertIn("$defs", schema) + self.assertIn("User", schema["$defs"]) + self.assertIn("Profile", schema["$defs"]) + self.assertIn("Settings", schema["$defs"]) + + def test_mixed_tools_with_and_without_defs(self): + """Test mixed tools with and without $defs""" + mixed_tools = [ + Tool( + type="function", + function=Function( + name="simple_tool", + description="Simple tool without $defs", + parameters={ + "type": "object", + "properties": { + "query": {"type": "string"}, + }, + "required": ["query"], + }, + ), + ), + Tool( + type="function", + function=Function( + name="complex_tool", + description="Complex tool with $defs", + parameters={ + "type": "object", + "properties": { + "data": {"$ref": "#/$defs/DataType"}, + }, + "required": ["data"], + "$defs": { + "DataType": { + "type": "object", + "properties": { + "value": {"type": "string"}, + "metadata": {"type": "object"}, + }, + "required": ["value"], + }, + }, + }, + ), + ), + Tool( + type="function", + function=Function( + name="another_simple_tool", + description="Another simple tool", + parameters={ + "type": "object", + "properties": { + "id": {"type": "integer"}, + }, + "required": ["id"], + }, + ), + ), + ] + + try: + _get_tool_schema_defs(mixed_tools) + except ValueError as e: + self.fail(f"Should not raise ValueError, but got: {e}") + + schema = get_json_schema_constraint(mixed_tools, "required") + self.assertIsNotNone(schema) + jsonschema.Draft202012Validator.check_schema(schema) + + # Should have $defs from the complex tool + self.assertIn("$defs", schema) + self.assertIn("DataType", schema["$defs"]) + + # Should have all three tools + tool_names = [ + item["properties"]["name"]["enum"][0] for item in schema["items"]["anyOf"] + ] + self.assertEqual(len(tool_names), 3) + self.assertIn("simple_tool", tool_names) + self.assertIn("complex_tool", tool_names) + self.assertIn("another_simple_tool", tool_names) + + def test_tools_with_defs_but_no_refs(self): + """Test tools with $defs but no $ref usage""" + tools_with_unused_defs = [ + Tool( + type="function", + function=Function( + name="unused_defs_tool", + description="Tool with $defs but no $ref usage", + parameters={ + "type": "object", + "properties": { + "data": {"type": "string"}, + }, + "required": ["data"], + "$defs": { + "UnusedType": { + "type": "object", + "properties": { + "value": {"type": "string"}, + }, + }, + }, + }, + ), + ), + ] + + try: + _get_tool_schema_defs(tools_with_unused_defs) + except ValueError as e: + self.fail(f"Should not raise ValueError, but got: {e}") + + schema = get_json_schema_constraint(tools_with_unused_defs, "required") + self.assertIsNotNone(schema) + jsonschema.Draft202012Validator.check_schema(schema) + + # Should still include $defs even if not referenced + self.assertIn("$defs", schema) + self.assertIn("UnusedType", schema["$defs"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/function_call/test_unknown_tool_name.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/function_call/test_unknown_tool_name.py new file mode 100644 index 00000000..e8baf55b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/function_call/test_unknown_tool_name.py @@ -0,0 +1,81 @@ +import json +import logging + +import pytest + +from sglang.srt.entrypoints.openai.protocol import Function, Tool +from sglang.srt.environ import envs +from sglang.srt.function_call.base_format_detector import BaseFormatDetector +from sglang.srt.function_call.core_types import StreamingParseResult +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(1.0, "default") + + +class DummyDetector(BaseFormatDetector): + def has_tool_call(self, text: str) -> bool: + return True + + def detect_and_parse(self, text: str, tools): + action = json.loads(text) + return StreamingParseResult( + normal_text="", calls=self.parse_base_json(action, tools) + ) + + def structure_info(self): + pass + + +def test_unknown_tool_name_dropped_default(caplog): + """Test that unknown tools are dropped by default (legacy behavior).""" + with envs.SGLANG_FORWARD_UNKNOWN_TOOLS.override(False): + tools = [ + Tool( + function=Function( + name="get_weather", parameters={"type": "object", "properties": {}} + ) + ) + ] + detector = DummyDetector() + with caplog.at_level( + logging.WARNING, logger="sglang.srt.function_call.base_format_detector" + ): + result = detector.detect_and_parse( + '{"name":"unknown_tool","parameters":{"city":"Paris"}}', tools + ) + assert any( + "Model attempted to call undefined function: unknown_tool" in m + for m in caplog.messages + ) + assert len(result.calls) == 0 # dropped in default mode + + +def test_unknown_tool_name_forwarded(caplog): + """Test that unknown tools are forwarded when env var is True.""" + with envs.SGLANG_FORWARD_UNKNOWN_TOOLS.override(True): + tools = [ + Tool( + function=Function( + name="get_weather", parameters={"type": "object", "properties": {}} + ) + ) + ] + detector = DummyDetector() + with caplog.at_level( + logging.WARNING, logger="sglang.srt.function_call.base_format_detector" + ): + result = detector.detect_and_parse( + '{"name":"unknown_tool","parameters":{"city":"Paris"}}', tools + ) + assert any( + "Model attempted to call undefined function: unknown_tool" in m + for m in caplog.messages + ) + assert len(result.calls) == 1 + assert result.calls[0].name == "unknown_tool" + assert result.calls[0].tool_index == -1 + assert json.loads(result.calls[0].parameters)["city"] == "Paris" + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/stress/test_stress_deepseek_v3.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/stress/test_stress_deepseek_v3.py new file mode 100644 index 00000000..fd54c35e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/stress/test_stress_deepseek_v3.py @@ -0,0 +1,51 @@ +"""Stress test for DeepSeek-V3 model.""" + +import os +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_stress_utils import StressTestRunner +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST + +MODEL_PATH = "deepseek-ai/DeepSeek-V3" +RANDOM_INPUT_LEN = 16384 +RANDOM_OUTPUT_LEN = 1024 +OUTPUT_FILE = "stress_test_deepseek_v3.jsonl" + +# Register for CI - estimated 45 minutes +register_cuda_ci(est_time=2700, suite="stress") + + +class TestStressDeepSeekV3(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model = MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.num_prompts = int(os.environ.get("NUM_PROMPTS", "50000")) + cls.duration_minutes = int(os.environ.get("DURATION_MINUTES", "45")) + + cls.runner = StressTestRunner( + test_name="DeepSeek-V3 Stress Test", + base_url=cls.base_url, + num_prompts=cls.num_prompts, + duration_minutes=cls.duration_minutes, + ) + + def test_stress_deepseek_v3(self): + try: + success = self.runner.run_stress_test_for_model( + model_path=self.model, + random_input_len=RANDOM_INPUT_LEN, + random_output_len=RANDOM_OUTPUT_LEN, + output_file=OUTPUT_FILE, + server_args=["--tp", "8", "--trust-remote-code"], + ) + + self.assertTrue(success, f"Stress test failed for {self.model}") + + finally: + self.runner.write_final_report() + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/stress/test_stress_glm_4_6.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/stress/test_stress_glm_4_6.py new file mode 100644 index 00000000..65e1861e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/stress/test_stress_glm_4_6.py @@ -0,0 +1,51 @@ +"""Stress test for GLM-4.6 model.""" + +import os +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_stress_utils import StressTestRunner +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST + +MODEL_PATH = "zai-org/GLM-4.6" +RANDOM_INPUT_LEN = 4096 +RANDOM_OUTPUT_LEN = 512 +OUTPUT_FILE = "stress_test_glm_4_6.jsonl" + +# Register for CI - estimated 45 minutes +register_cuda_ci(est_time=2700, suite="stress") + + +class TestStressGLM46(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model = MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.num_prompts = int(os.environ.get("NUM_PROMPTS", "50000")) + cls.duration_minutes = int(os.environ.get("DURATION_MINUTES", "45")) + + cls.runner = StressTestRunner( + test_name="GLM-4.6 Stress Test", + base_url=cls.base_url, + num_prompts=cls.num_prompts, + duration_minutes=cls.duration_minutes, + ) + + def test_stress_glm_4_6(self): + try: + success = self.runner.run_stress_test_for_model( + model_path=self.model, + random_input_len=RANDOM_INPUT_LEN, + random_output_len=RANDOM_OUTPUT_LEN, + output_file=OUTPUT_FILE, + server_args=["--tp", "8", "--trust-remote-code"], + ) + + self.assertTrue(success, f"Stress test failed for {self.model}") + + finally: + self.runner.write_final_report() + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/stress/test_stress_kimi_k2.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/stress/test_stress_kimi_k2.py new file mode 100644 index 00000000..3d921550 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/stress/test_stress_kimi_k2.py @@ -0,0 +1,59 @@ +"""Stress test for Kimi-K2-Thinking model.""" + +import os +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_stress_utils import StressTestRunner +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST + +MODEL_PATH = "moonshotai/Kimi-K2-Thinking" +RANDOM_INPUT_LEN = 4096 +RANDOM_OUTPUT_LEN = 512 +OUTPUT_FILE = "stress_test_kimi_k2.jsonl" + +# Register for CI - estimated 45 minutes +register_cuda_ci(est_time=2700, suite="stress") + + +class TestStressKimiK2(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model = MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.num_prompts = int(os.environ.get("NUM_PROMPTS", "50000")) + cls.duration_minutes = int(os.environ.get("DURATION_MINUTES", "45")) + + cls.runner = StressTestRunner( + test_name="Kimi-K2-Thinking Stress Test", + base_url=cls.base_url, + num_prompts=cls.num_prompts, + duration_minutes=cls.duration_minutes, + ) + + def test_stress_kimi_k2(self): + try: + success = self.runner.run_stress_test_for_model( + model_path=self.model, + random_input_len=RANDOM_INPUT_LEN, + random_output_len=RANDOM_OUTPUT_LEN, + output_file=OUTPUT_FILE, + server_args=[ + "--tp", + "8", + "--trust-remote-code", + "--tool-call-parser", + "kimi_k2", + "--reasoning-parser", + "kimi_k2", + ], + ) + + self.assertTrue(success, f"Stress test failed for {self.model}") + + finally: + self.runner.write_final_report() + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/stress/test_stress_qwen3_235b.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/stress/test_stress_qwen3_235b.py new file mode 100644 index 00000000..89a743f7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/stress/test_stress_qwen3_235b.py @@ -0,0 +1,51 @@ +"""Stress test for Qwen3-235B model.""" + +import os +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_stress_utils import StressTestRunner +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST + +MODEL_PATH = "Qwen/Qwen3-235B-A22B-Instruct-2507" +RANDOM_INPUT_LEN = 4096 +RANDOM_OUTPUT_LEN = 512 +OUTPUT_FILE = "stress_test_qwen3_235b.jsonl" + +# Register for CI - estimated 45 minutes +register_cuda_ci(est_time=2700, suite="stress") + + +class TestStressQwen3235B(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model = MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.num_prompts = int(os.environ.get("NUM_PROMPTS", "50000")) + cls.duration_minutes = int(os.environ.get("DURATION_MINUTES", "45")) + + cls.runner = StressTestRunner( + test_name="Qwen3-235B Stress Test", + base_url=cls.base_url, + num_prompts=cls.num_prompts, + duration_minutes=cls.duration_minutes, + ) + + def test_stress_qwen3_235b(self): + try: + success = self.runner.run_stress_test_for_model( + model_path=self.model, + random_input_len=RANDOM_INPUT_LEN, + random_output_len=RANDOM_OUTPUT_LEN, + output_file=OUTPUT_FILE, + server_args=["--tp", "8", "--trust-remote-code"], + ) + + self.assertTrue(success, f"Stress test failed for {self.model}") + + finally: + self.runner.write_final_report() + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/test_srt_backend.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/test_srt_backend.py new file mode 100644 index 00000000..f3ad67cf --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/registered/test_srt_backend.py @@ -0,0 +1,84 @@ +import unittest + +import sglang as sgl +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.test_programs import ( + test_decode_int, + test_decode_json_regex, + test_dtype_gen, + test_expert_answer, + test_few_shot_qa, + test_gen_min_new_tokens, + test_hellaswag_select, + test_mt_bench, + test_parallel_decoding, + test_regex, + test_select, + test_stream, + test_tool_use, +) +from sglang.test.test_utils import DEFAULT_MODEL_NAME_FOR_TEST, CustomTestCase + +register_cuda_ci(est_time=80, suite="stage-a-test-1") +register_amd_ci(est_time=120, suite="stage-a-test-1") + + +class TestSRTBackend(CustomTestCase): + backend = None + + @classmethod + def setUpClass(cls): + cls.backend = sgl.Runtime( + model_path=DEFAULT_MODEL_NAME_FOR_TEST, cuda_graph_max_bs=4 + ) + sgl.set_default_backend(cls.backend) + + @classmethod + def tearDownClass(cls): + cls.backend.shutdown() + + def test_few_shot_qa(self): + test_few_shot_qa() + + def test_mt_bench(self): + test_mt_bench() + + def test_select(self): + test_select(check_answer=False) + + def test_decode_int(self): + test_decode_int() + + def test_decode_json_regex(self): + test_decode_json_regex() + + def test_expert_answer(self): + test_expert_answer() + + def test_tool_use(self): + test_tool_use() + + def test_parallel_decoding(self): + test_parallel_decoding() + + def test_stream(self): + test_stream() + + def test_regex(self): + test_regex() + + def test_dtype_gen(self): + test_dtype_gen() + + def test_hellaswag_select(self): + # Run twice to capture more bugs + for _ in range(2): + accuracy, latency = test_hellaswag_select() + self.assertGreater(accuracy, 0.60) + + def test_gen_min_new_tokens(self): + test_gen_min_new_tokens() + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/run_suite.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/run_suite.py new file mode 100644 index 00000000..c903f1ab --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/run_suite.py @@ -0,0 +1,185 @@ +import argparse +import glob +import sys +from typing import List + +from sglang.test.ci.ci_register import CIRegistry, HWBackend, collect_tests +from sglang.test.ci.ci_utils import TestFile, run_unittest_files + +HW_MAPPING = { + "cpu": HWBackend.CPU, + "cuda": HWBackend.CUDA, + "amd": HWBackend.AMD, +} + +# Per-commit test suites (run on every PR) +PER_COMMIT_SUITES = { + HWBackend.CPU: ["default"], + HWBackend.AMD: ["stage-a-test-1"], + HWBackend.CUDA: ["stage-a-test-1"], +} + +# Nightly test suites (run nightly, organized by GPU configuration) +NIGHTLY_SUITES = { + HWBackend.CUDA: [ + "nightly-1-gpu", + "nightly-2-gpu", + "nightly-4-gpu", + "nightly-4-gpu-b200", + "nightly-8-gpu", + "nightly-8-gpu-h200", + "nightly-8-gpu-h20", + "nightly-8-gpu-b200", + ], + HWBackend.AMD: ["nightly-amd"], + HWBackend.CPU: [], +} + + +def filter_tests( + ci_tests: List[CIRegistry], hw: HWBackend, suite: str, nightly: bool = False +) -> List[CIRegistry]: + ci_tests = [ + t + for t in ci_tests + if t.backend == hw and t.suite == suite and t.nightly == nightly + ] + + ret = [] + valid_suites = ( + NIGHTLY_SUITES.get(hw, []) if nightly else PER_COMMIT_SUITES.get(hw, []) + ) + + if suite not in valid_suites: + print( + f"Warning: Unknown suite {suite} for backend {hw.name}, nightly={nightly}" + ) + + for t in ci_tests: + if t.disabled is None: + ret.append(t) + print(f"Including test {t.filename}") + else: + print(f"Skipping disabled test {t.filename} due to: {t.disabled}") + + return ret + + +def auto_partition(files, rank, size): + """ + Partition files into size sublists with approximately equal sums of estimated times + using a greedy algorithm (LPT heuristic), and return the partition for the specified rank. + """ + if not files or size <= 0: + return [] + + # Sort files by estimated_time in descending order (LPT heuristic) + sorted_files = sorted(files, key=lambda f: f.estimated_time, reverse=True) + + partitions = [[] for _ in range(size)] + partition_sums = [0.0] * size + + # Greedily assign each file to the partition with the smallest current total time + for file in sorted_files: + min_sum_idx = min(range(size), key=partition_sums.__getitem__) + partitions[min_sum_idx].append(file) + partition_sums[min_sum_idx] += file.estimated_time + + if rank < size: + return partitions[rank] + return [] + + +def run_a_suite(args): + hw = HW_MAPPING[args.hw] + suite = args.suite + nightly = args.nightly + auto_partition_id = args.auto_partition_id + auto_partition_size = args.auto_partition_size + + files = glob.glob("**/*.py", recursive=True) + ci_tests = filter_tests( + collect_tests(files, sanity_check=False), hw, suite, nightly + ) + test_files = [TestFile(t.filename, t.est_time) for t in ci_tests] + + if not test_files: + print(f"No tests found for hw={hw.name}, suite={suite}, nightly={nightly}") + print("This is expected during incremental migration. Skipping.") + return 0 + + if auto_partition_size: + test_files = auto_partition(test_files, auto_partition_id, auto_partition_size) + + print( + f"Running {len(test_files)} test(s) for hw={hw.name}, suite={suite}, nightly={nightly}" + ) + + return run_unittest_files( + test_files, + timeout_per_file=args.timeout_per_file, + continue_on_error=args.continue_on_error, + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Run CI test suites from test/registered/" + ) + parser.add_argument( + "--hw", + type=str, + choices=HW_MAPPING.keys(), + required=True, + help="Hardware backend to run tests on.", + ) + parser.add_argument("--suite", type=str, required=True, help="Test suite to run.") + parser.add_argument( + "--nightly", + action="store_true", + help="Run nightly tests instead of per-commit tests.", + ) + parser.add_argument( + "--timeout-per-file", + type=int, + default=1200, + help="The time limit for running one file in seconds (default: 1200).", + ) + parser.add_argument( + "--continue-on-error", + action="store_true", + default=False, + help="Continue running remaining tests even if one fails (default: False, useful for nightly tests).", + ) + parser.add_argument( + "--auto-partition-id", + type=int, + help="Use auto load balancing. The part id.", + ) + parser.add_argument( + "--auto-partition-size", + type=int, + help="Use auto load balancing. The number of parts.", + ) + args = parser.parse_args() + + # Validate auto-partition arguments + if (args.auto_partition_id is not None) != (args.auto_partition_size is not None): + parser.error( + "--auto-partition-id and --auto-partition-size must be specified together." + ) + if args.auto_partition_size is not None: + if args.auto_partition_size <= 0: + parser.error("--auto-partition-size must be positive.") + if not 0 <= args.auto_partition_id < args.auto_partition_size: + parser.error( + f"--auto-partition-id must be in range [0, {args.auto_partition_size}), " + f"but got {args.auto_partition_id}" + ) + + exit_code = run_a_suite(args) + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/run_suite_nightly.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/run_suite_nightly.py new file mode 100644 index 00000000..370ec082 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/run_suite_nightly.py @@ -0,0 +1,94 @@ +import argparse +import os +import sys +from pathlib import Path + +from sglang.test.ci.ci_utils import TestFile, run_unittest_files + +# Nightly test suites +suites = { + "nightly-1-gpu": [ + TestFile("test_nsa_indexer.py", 2), + TestFile("test_lora_qwen3.py", 97), + TestFile("test_lora_radix_cache.py", 200), + TestFile("test_lora_eviction_policy.py", 200), + TestFile("test_lora_openai_api.py", 30), + TestFile("test_lora_openai_compatible.py", 150), + TestFile("test_batch_invariant_ops.py", 10), + TestFile("test_cpp_radix_cache.py", 60), + TestFile("test_deepseek_v3_deterministic.py", 240), + ], + "nightly-4-gpu-b200": [ + TestFile("test_flashinfer_trtllm_gen_moe_backend.py", 300), + TestFile("test_gpt_oss_4gpu_perf.py", 600), + TestFile("test_flashinfer_trtllm_gen_attn_backend.py", 300), + TestFile("test_deepseek_v3_fp4_cutlass_moe.py", 900), + TestFile("test_fp4_moe.py", 300), + TestFile("test_qwen3_fp4_trtllm_gen_moe.py", 300), + ], + "nightly-8-gpu-b200": [ + TestFile("test_deepseek_r1_fp8_trtllm_backend.py", 3600), + ], + "nightly-4-gpu": [ + TestFile("test_encoder_dp.py", 500), + TestFile("test_qwen3_next_deterministic.py", 200), + ], + "nightly-8-gpu": [], + "nightly-8-gpu-h200": [ + TestFile("test_deepseek_v32_nsabackend.py", 600), + ], + "nightly-8-gpu-h20": [], +} + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--suite", + type=str, + required=True, + help="Test suite to run (e.g., nightly-1-gpu, nightly-4-gpu, etc.).", + ) + parser.add_argument( + "--timeout-per-file", + type=int, + default=1200, + help="The time limit for running one file in seconds (default: 1200).", + ) + parser.add_argument( + "--continue-on-error", + action="store_true", + default=False, + help="Continue running remaining tests even if one fails (default: False, useful for nightly tests).", + ) + args = parser.parse_args() + + if args.suite not in suites: + print(f"Error: Suite '{args.suite}' not found in available suites") + print(f"Available suites: {list(suites.keys())}") + exit(1) + + files = suites[args.suite] + + # Change directory to test/nightly where the test files are located + nightly_dir = Path(__file__).parent / "nightly" + os.chdir(nightly_dir) + + # Add test/ to PYTHONPATH so tests can import shared utils + test_dir = str(Path(__file__).parent) + pythonpath = os.environ.get("PYTHONPATH", "") + os.environ["PYTHONPATH"] = f"{test_dir}:{pythonpath}" if pythonpath else test_dir + + print(f"Running {len(files)} tests from suite: {args.suite}") + print(f"Test files: {[f.name for f in files]}") + + exit_code = run_unittest_files( + files, + timeout_per_file=args.timeout_per_file, + continue_on_error=args.continue_on_error, + ) + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_deepep.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_deepep.py new file mode 100644 index 00000000..d6a94890 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_deepep.py @@ -0,0 +1,118 @@ +import os +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_offline_throughput, +) + +TEST_MODEL_MATRIX = { + "/root/.cache/modelscope/hub/models/vllm-ascend/DeepSeek-R1-0528-W8A8": { + "accuracy": 0.95, + "latency": 1000, + "output_throughput": 6, + }, +} + + +class TestAscendDeepEP(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.models = TEST_MODEL_MATRIX.keys() + cls.base_url = DEFAULT_URL_FOR_TEST + cls.url = urlparse(DEFAULT_URL_FOR_TEST) + + cls.common_args = [ + "--trust-remote-code", + "--attention-backend", + "ascend", + "--quantization", + "w8a8_int8", + "--mem-fraction-static", + 0.8, + "--disable-radix-cache", + "--chunked-prefill-size", + 32768, + "--tp-size", + 16, + "--dp-size", + 1, + "--ep-size", + 16, + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "auto", + ] + + cls.extra_envs = { + "HCCL_BUFFSIZE": "1000", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "32", + "SGLANG_NPU_USE_MLAPO": "1", + } + os.environ.update(cls.extra_envs) + + def test_a_gsm8k(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing accuracy: {model} ===##") + + process = popen_launch_server( + model, + self.base_url, + timeout=1500, + other_args=[ + *self.common_args, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) + finally: + kill_process_tree(process.pid) + + def test_b_throughput(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing throughput: {model} ===##") + + output_throughput = run_bench_offline_throughput( + model, + [ + *self.common_args, + ], + ) + + print(f"##=== {model} throughput: {output_throughput} ===##") + + if is_in_ci(): + self.assertGreater( + output_throughput, + TEST_MODEL_MATRIX[model]["output_throughput"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_deepseek_mtp.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_deepseek_mtp.py new file mode 100644 index 00000000..d8f33dcd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_deepseek_mtp.py @@ -0,0 +1,99 @@ +import os +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +TEST_MODEL_MATRIX = { + "/root/.cache/modelscope/hub/models/vllm-ascend/DeepSeek-R1-0528-W8A8": { + "accuracy": 0.95, + "latency": 1000, + "output_throughput": 6, + }, +} + + +class TestAscendDeepSeekMTP(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.models = TEST_MODEL_MATRIX.keys() + cls.base_url = DEFAULT_URL_FOR_TEST + cls.url = urlparse(DEFAULT_URL_FOR_TEST) + + cls.common_args = [ + "--trust-remote-code", + "--attention-backend", + "ascend", + "--quantization", + "w8a8_int8", + "--mem-fraction-static", + 0.8, + "--disable-radix-cache", + "--chunked-prefill-size", + 32768, + "--tp-size", + 16, + "--dp-size", + 2, + "--enable-dp-attention", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + 1, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 2, + ] + + cls.extra_envs = { + "SGLANG_NPU_USE_MLAPO": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + } + os.environ.update(cls.extra_envs) + + def test_a_gsm8k(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing accuracy: {model} ===##") + + process = popen_launch_server( + model, + self.base_url, + timeout=1500, + other_args=[ + *self.common_args, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_graph_tp1_bf16.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_graph_tp1_bf16.py new file mode 100644 index 00000000..95c6b7bc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_graph_tp1_bf16.py @@ -0,0 +1,95 @@ +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_offline_throughput, +) + +TEST_MODEL_MATRIX = { + "Qwen/Qwen2.5-7B-Instruct": { + "accuracy": 0.85, + "latency": 150, + "output_throughput": 30, + }, +} + + +class TestAscendGraphTp1Bf16(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.models = TEST_MODEL_MATRIX.keys() + cls.base_url = DEFAULT_URL_FOR_TEST + cls.url = urlparse(DEFAULT_URL_FOR_TEST) + cls.common_args = [ + "--trust-remote-code", + "--mem-fraction-static", + 0.8, + "--attention-backend", + "ascend", + ] + + def test_a_gsm8k(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing accuracy: {model} ===##") + + process = popen_launch_server( + model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + *self.common_args, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) + finally: + kill_process_tree(process.pid) + + def test_b_throughput(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing throughput: {model} ===##") + + output_throughput = run_bench_offline_throughput( + model, + [ + *self.common_args, + ], + ) + + print(f"##=== {model} throughput: {output_throughput} ===##") + + if is_in_ci(): + self.assertGreater( + output_throughput, + TEST_MODEL_MATRIX[model]["output_throughput"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_graph_tp2_bf16.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_graph_tp2_bf16.py new file mode 100644 index 00000000..f7c3c653 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_graph_tp2_bf16.py @@ -0,0 +1,97 @@ +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_offline_throughput, +) + +TEST_MODEL_MATRIX = { + "Qwen/Qwen2.5-7B-Instruct": { + "accuracy": 0.85, + "latency": 180, + "output_throughput": 20, + }, +} + + +class TestAscendGraphTp2Bf16(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.models = TEST_MODEL_MATRIX.keys() + cls.base_url = DEFAULT_URL_FOR_TEST + cls.url = urlparse(DEFAULT_URL_FOR_TEST) + cls.common_args = [ + "--trust-remote-code", + "--mem-fraction-static", + 0.8, + "--attention-backend", + "ascend", + "--tp-size", + 2, + ] + + def test_a_gsm8k(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing accuracy: {model} ===##") + + process = popen_launch_server( + model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + *self.common_args, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) + finally: + kill_process_tree(process.pid) + + def test_b_throughput(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing throughput: {model} ===##") + + output_throughput = run_bench_offline_throughput( + model, + [ + *self.common_args, + ], + ) + + print(f"##=== {model} throughput: {output_throughput} ===##") + + if is_in_ci(): + self.assertGreater( + output_throughput, + TEST_MODEL_MATRIX[model]["output_throughput"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_hicache_mha.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_hicache_mha.py new file mode 100644 index 00000000..d6f1aa9c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_hicache_mha.py @@ -0,0 +1,98 @@ +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_offline_throughput, +) + +TEST_MODEL_MATRIX = { + "Qwen/Qwen2.5-7B-Instruct": { + "accuracy": 0.85, + "latency": 150, + "output_throughput": 30, + }, +} + + +class TestAscendMhaHicache(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.models = TEST_MODEL_MATRIX.keys() + cls.base_url = DEFAULT_URL_FOR_TEST + cls.url = urlparse(DEFAULT_URL_FOR_TEST) + cls.common_args = [ + "--trust-remote-code", + "--mem-fraction-static", + 0.8, + "--attention-backend", + "ascend", + "--enable-hierarchical-cache", + "--hicache-ratio", + 1.2, + ] + + def test_a_gsm8k(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing accuracy: {model} ===##") + + process = popen_launch_server( + model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + *self.common_args, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) + finally: + kill_process_tree(process.pid) + + def test_b_throughput(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing throughput: {model} ===##") + + output_throughput = run_bench_offline_throughput( + model, + [ + *self.common_args, + ], + ) + + print(f"##=== {model} throughput: {output_throughput} ===##") + + if is_in_ci(): + self.assertGreater( + output_throughput, + TEST_MODEL_MATRIX[model]["output_throughput"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_mla_fia_w8a8int8.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_mla_fia_w8a8int8.py new file mode 100644 index 00000000..6de97b04 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_mla_fia_w8a8int8.py @@ -0,0 +1,103 @@ +import os +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_offline_throughput, +) + +TEST_MODEL_MATRIX = { + "/root/.cache/modelscope/hub/models/vllm-ascend/DeepSeek-V2-Lite-W8A8": { + "accuracy": 0.34, + "latency": 1000, + "output_throughput": 6, + }, +} + + +class TestAscendMlaW8A8Int8(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.models = TEST_MODEL_MATRIX.keys() + cls.base_url = DEFAULT_URL_FOR_TEST + cls.url = urlparse(DEFAULT_URL_FOR_TEST) + cls.common_args = [ + "--trust-remote-code", + "--disable-cuda-graph", + "--mem-fraction-static", + 0.8, + "--attention-backend", + "ascend", + "--quantization", + "w8a8_int8", + "--tp-size", + 2, + "--disable-radix-cache", + ] + + def test_a_gsm8k(self): + os.environ["ASCEND_USE_FIA"] = "true" + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing accuracy: {model} ===##") + + process = popen_launch_server( + model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + *self.common_args, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) + finally: + kill_process_tree(process.pid) + + def test_b_throughput(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing throughput: {model} ===##") + + output_throughput = run_bench_offline_throughput( + model, + [ + *self.common_args, + ], + ) + + print(f"##=== {model} throughput: {output_throughput} ===##") + + if is_in_ci(): + self.assertGreater( + output_throughput, + TEST_MODEL_MATRIX[model]["output_throughput"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_mla_w8a8int8.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_mla_w8a8int8.py new file mode 100644 index 00000000..70f7edab --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_mla_w8a8int8.py @@ -0,0 +1,101 @@ +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_offline_throughput, +) + +TEST_MODEL_MATRIX = { + "/root/.cache/modelscope/hub/models/vllm-ascend/DeepSeek-V2-Lite-W8A8": { + "accuracy": 0.34, + "latency": 1000, + "output_throughput": 6, + }, +} + + +class TestAscendMlaW8A8Int8(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.models = TEST_MODEL_MATRIX.keys() + cls.base_url = DEFAULT_URL_FOR_TEST + cls.url = urlparse(DEFAULT_URL_FOR_TEST) + cls.common_args = [ + "--trust-remote-code", + "--disable-cuda-graph", + "--mem-fraction-static", + 0.8, + "--attention-backend", + "ascend", + "--quantization", + "w8a8_int8", + "--tp-size", + 4, + "--disable-radix-cache", + ] + + def test_a_gsm8k(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing accuracy: {model} ===##") + + process = popen_launch_server( + model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + *self.common_args, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) + finally: + kill_process_tree(process.pid) + + def test_b_throughput(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing throughput: {model} ===##") + + output_throughput = run_bench_offline_throughput( + model, + [ + *self.common_args, + ], + ) + + print(f"##=== {model} throughput: {output_throughput} ===##") + + if is_in_ci(): + self.assertGreater( + output_throughput, + TEST_MODEL_MATRIX[model]["output_throughput"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_sampling_backend.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_sampling_backend.py new file mode 100644 index 00000000..7b630791 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_sampling_backend.py @@ -0,0 +1,96 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestAscendSamplingBackend(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "Qwen/Qwen2.5-7B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--sampling-backend", + "ascend", + "--disable-radix-cache", + "--disable-cuda-graph", + "--mem-fraction-static", + 0.85, + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + temperature=0.1, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + def test_greedy(self): + + first_text = None + + # ensure the answer is identical across single response + for _ in range(5): + response_single = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of Germany is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, + ).json() + text = response_single["text"] + if first_text is None: + first_text = text + + self.assertEqual(text, first_text) + + first_text = None + + response_batch = requests.post( + self.base_url + "/generate", + json={ + "text": ["The capital of Germany is"] * 10, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, + ).json() + + # ensure the answer is identical among the batch + for i in range(10): + text = response_batch[i]["text"] + if first_text is None: + first_text = text + self.assertEqual(text, first_text) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_tp1_bf16.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_tp1_bf16.py new file mode 100644 index 00000000..fd0f96e7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_tp1_bf16.py @@ -0,0 +1,74 @@ +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +TEST_MODEL_MATRIX = { + "Qwen/Qwen2.5-7B-Instruct": { + "accuracy": 0.84, + "latency": 150, + "output_throughput": 30, + }, +} + + +class TestAscendTp1Bf16(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.models = TEST_MODEL_MATRIX.keys() + cls.base_url = DEFAULT_URL_FOR_TEST + cls.url = urlparse(DEFAULT_URL_FOR_TEST) + cls.common_args = [ + "--trust-remote-code", + "--disable-cuda-graph", + "--mem-fraction-static", + 0.8, + "--attention-backend", + "ascend", + ] + + def test_a_gsm8k(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing accuracy: {model} ===##") + + process = popen_launch_server( + model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + *self.common_args, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_tp2_bf16.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_tp2_bf16.py new file mode 100644 index 00000000..d5e141c9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_tp2_bf16.py @@ -0,0 +1,98 @@ +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_offline_throughput, +) + +TEST_MODEL_MATRIX = { + "Qwen/Qwen2.5-7B-Instruct": { + "accuracy": 0.85, + "latency": 180, + "output_throughput": 20, + }, +} + + +class TestAscendTp2Bf16(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.models = TEST_MODEL_MATRIX.keys() + cls.base_url = DEFAULT_URL_FOR_TEST + cls.url = urlparse(DEFAULT_URL_FOR_TEST) + cls.common_args = [ + "--trust-remote-code", + "--disable-cuda-graph", + "--mem-fraction-static", + 0.8, + "--attention-backend", + "ascend", + "--tp-size", + 2, + ] + + def test_a_gsm8k(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing accuracy: {model} ===##") + + process = popen_launch_server( + model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + *self.common_args, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) + finally: + kill_process_tree(process.pid) + + def test_b_throughput(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing throughput: {model} ===##") + + output_throughput = run_bench_offline_throughput( + model, + [ + *self.common_args, + ], + ) + + print(f"##=== {model} throughput: {output_throughput} ===##") + + if is_in_ci(): + self.assertGreater( + output_throughput, + TEST_MODEL_MATRIX[model]["output_throughput"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_tp2_fia_bf16.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_tp2_fia_bf16.py new file mode 100644 index 00000000..bdd1c573 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_tp2_fia_bf16.py @@ -0,0 +1,101 @@ +import os +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_offline_throughput, +) + +TEST_MODEL_MATRIX = { + "Qwen/Qwen2.5-7B-Instruct": { + "accuracy": 0.85, + "latency": 180, + "output_throughput": 20, + }, +} + + +class TestAscendTp2Bf16(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.models = TEST_MODEL_MATRIX.keys() + cls.base_url = DEFAULT_URL_FOR_TEST + cls.url = urlparse(DEFAULT_URL_FOR_TEST) + cls.common_args = [ + "--trust-remote-code", + "--disable-cuda-graph", + "--mem-fraction-static", + 0.8, + "--attention-backend", + "ascend", + "--tp-size", + 2, + "--disable-radix-cache", + ] + + def test_a_gsm8k(self): + os.environ["ASCEND_USE_FIA"] = "true" + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing accuracy: {model} ===##") + + process = popen_launch_server( + model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + *self.common_args, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) + finally: + kill_process_tree(process.pid) + + def test_b_throughput(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing throughput: {model} ===##") + + output_throughput = run_bench_offline_throughput( + model, + [ + *self.common_args, + ], + ) + + print(f"##=== {model} throughput: {output_throughput} ===##") + + if is_in_ci(): + self.assertGreater( + output_throughput, + TEST_MODEL_MATRIX[model]["output_throughput"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_tp4_bf16.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_tp4_bf16.py new file mode 100644 index 00000000..e1feac41 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ascend/test_ascend_tp4_bf16.py @@ -0,0 +1,100 @@ +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_offline_throughput, +) + +TEST_MODEL_MATRIX = { + "Qwen/Qwen3-30B-A3B-Instruct-2507": { + "accuracy": 0.90, + "latency": 180, + "output_throughput": 20, + }, +} + + +class TestAscendTp4Bf16(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.models = TEST_MODEL_MATRIX.keys() + cls.base_url = DEFAULT_URL_FOR_TEST + cls.url = urlparse(DEFAULT_URL_FOR_TEST) + cls.common_args = [ + "--trust-remote-code", + "--mem-fraction-static", + 0.7, + "--max-running-requests", + 32, + "--attention-backend", + "ascend", + "--cuda-graph-max-bs", + 32, + "--tp-size", + 4, + ] + + def test_a_gsm8k(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing accuracy: {model} ===##") + + process = popen_launch_server( + model, + self.base_url, + timeout=1800, + other_args=[ + *self.common_args, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) + finally: + kill_process_tree(process.pid) + + def test_b_throughput(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing throughput: {model} ===##") + + output_throughput = run_bench_offline_throughput( + model, + [ + *self.common_args, + ], + ) + + print(f"##=== {model} throughput: {output_throughput} ===##") + + if is_in_ci(): + self.assertGreater( + output_throughput, + TEST_MODEL_MATRIX[model]["output_throughput"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/deepseek_v3.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/deepseek_v3.yaml new file mode 100644 index 00000000..82d059cb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/deepseek_v3.yaml @@ -0,0 +1,28 @@ +tasks: + - name: sglang-8192-1024-concurrency1 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 1 --num-prompts 5 --output-file deepseek_v3_results.jsonl + + - name: sglang-8192-1024-concurrency2 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 2 --num-prompts 10 --output-file deepseek_v3_results.jsonl + + - name: sglang-8192-1024-concurrency4 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 4 --num-prompts 20 --output-file deepseek_v3_results.jsonl + + - name: sglang-8192-1024-concurrency8 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 8 --num-prompts 32 --output-file deepseek_v3_results.jsonl + + - name: sglang-8192-1024-concurrency16 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 16 --num-prompts 48 --output-file deepseek_v3_results.jsonl + + - name: sglang-8192-1024-concurrency24 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 24 --num-prompts 72 --output-file deepseek_v3_results.jsonl + + - name: sglang-8192-1024-concurrency32 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 32 --num-prompts 96 --output-file deepseek_v3_results.jsonl diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/deepseek_v3_long_context.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/deepseek_v3_long_context.yaml new file mode 100644 index 00000000..df416a42 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/deepseek_v3_long_context.yaml @@ -0,0 +1,28 @@ +tasks: + - name: sglang-32000-100-concurrency1 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 32000 --random-output-len 100 --max-concurrency 1 --num-prompts 5 --output-file deepseek_v3_long_context_results.jsonl + + - name: sglang-32000-100-concurrency2 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 32000 --random-output-len 100 --max-concurrency 2 --num-prompts 10 --output-file deepseek_v3_long_context_results.jsonl + + - name: sglang-32000-100-concurrency4 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 32000 --random-output-len 100 --max-concurrency 4 --num-prompts 20 --output-file deepseek_v3_long_context_results.jsonl + + - name: sglang-32000-100-concurrency8 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 32000 --random-output-len 100 --max-concurrency 8 --num-prompts 32 --output-file deepseek_v3_long_context_results.jsonl + + - name: sglang-32000-100-concurrency16 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 32000 --random-output-len 100 --max-concurrency 16 --num-prompts 48 --output-file deepseek_v3_long_context_results.jsonl + + - name: sglang-32000-100-concurrency24 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 32000 --random-output-len 100 --max-concurrency 24 --num-prompts 72 --output-file deepseek_v3_long_context_results.jsonl + + - name: sglang-32000-100-concurrency32 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 32000 --random-output-len 100 --max-concurrency 32 --num-prompts 96 --output-file deepseek_v3_long_context_results.jsonl diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/llama_405b.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/llama_405b.yaml new file mode 100644 index 00000000..db0c816f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/llama_405b.yaml @@ -0,0 +1,28 @@ +tasks: + - name: sglang-8192-1024-concurrency1 + server_cmd: python3 -m sglang.launch_server --model nvidia/Llama-3.1-405B-Instruct-FP8 --tp 8 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 1 --num-prompts 5 --output-file llama_405b_results.jsonl + + - name: sglang-8192-1024-concurrency2 + server_cmd: python3 -m sglang.launch_server --model nvidia/Llama-3.1-405B-Instruct-FP8 --tp 8 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 2 --num-prompts 10 --output-file llama_405b_results.jsonl + + - name: sglang-8192-1024-concurrency4 + server_cmd: python3 -m sglang.launch_server --model nvidia/Llama-3.1-405B-Instruct-FP8 --tp 8 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 4 --num-prompts 20 --output-file llama_405b_results.jsonl + + - name: sglang-8192-1024-concurrency8 + server_cmd: python3 -m sglang.launch_server --model nvidia/Llama-3.1-405B-Instruct-FP8 --tp 8 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 8 --num-prompts 32 --output-file llama_405b_results.jsonl + + - name: sglang-8192-1024-concurrency16 + server_cmd: python3 -m sglang.launch_server --model nvidia/Llama-3.1-405B-Instruct-FP8 --tp 8 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 16 --num-prompts 48 --output-file llama_405b_results.jsonl + + - name: sglang-8192-1024-concurrency24 + server_cmd: python3 -m sglang.launch_server --model nvidia/Llama-3.1-405B-Instruct-FP8 --tp 8 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 24 --num-prompts 72 --output-file llama_405b_results.jsonl + + - name: sglang-8192-1024-concurrency32 + server_cmd: python3 -m sglang.launch_server --model nvidia/Llama-3.1-405B-Instruct-FP8 --tp 8 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 32 --num-prompts 96 --output-file llama_405b_results.jsonl diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/random_config.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/random_config.yaml new file mode 100644 index 00000000..eae8c27f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/random_config.yaml @@ -0,0 +1,25 @@ +tasks: + - name: sglang-128-4 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 128 --random-output 4 --request-rate 24 --num-prompt 1440 + - name: vllm-128-4 + server_cmd: python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-8B-Instruct --disable-log-requests + client_cmd: python3 -m sglang.bench_serving --backend vllm --dataset-name random --random-input 128 --random-output 4 --request-rate 24 --num-prompt 1440 + - name: sglang-2000-100 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 2000 --random-output 100 --request-rate 2 --num-prompt 120 + - name: vllm-2000-100 + server_cmd: python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-8B-Instruct --disable-log-requests + client_cmd: python3 -m sglang.bench_serving --backend vllm --dataset-name random --random-input 2000 --random-output 100 --request-rate 2 --num-prompt 120 + - name: sglang-4000-200 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 4000 --random-output 200 --request-rate 8 --num-prompt 480 + - name: vllm-4000-200 + server_cmd: python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-8B-Instruct --disable-log-requests + client_cmd: python3 -m sglang.bench_serving --backend vllm --dataset-name random --random-input 4000 --random-output 200 --request-rate 8 --num-prompt 480 + - name: sglang-32000-100 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 32000 --random-output 100 --request-rate 1 --num-prompt 60 + - name: vllm-32000-100 + server_cmd: python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-8B-Instruct --disable-log-requests + client_cmd: python3 -m sglang.bench_serving --backend vllm --dataset-name random --random-input 32000 --random-output 100 --request-rate 1 --num-prompt 60 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/random_flashinfer_vs_triton_config.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/random_flashinfer_vs_triton_config.yaml new file mode 100644 index 00000000..7f4a386d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/random_flashinfer_vs_triton_config.yaml @@ -0,0 +1,25 @@ +tasks: + - name: sglang-128-4 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 128 --random-output 4 --request-rate 24 --num-prompt 1440 + - name: sglang-triton-128-4 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache --attention-backend triton + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 128 --random-output 4 --request-rate 24 --num-prompt 1440 + - name: sglang-2000-100 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 2000 --random-output 100 --request-rate 2 --num-prompt 120 + - name: sglang-triton-2000-100 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache --attention-backend triton + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 2000 --random-output 100 --request-rate 2 --num-prompt 120 + - name: sglang-4000-200 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 4000 --random-output 200 --request-rate 8 --num-prompt 480 + - name: sglang-triton-4000-200 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache --attention-backend triton + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 4000 --random-output 200 --request-rate 8 --num-prompt 480 + - name: sglang-32000-100 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 32000 --random-output 100 --request-rate 1 --num-prompt 60 + - name: sglang-triton-32000-100 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache --attention-backend triton + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 32000 --random-output 100 --request-rate 1 --num-prompt 60 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/sharegpt_config.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/sharegpt_config.yaml new file mode 100644 index 00000000..a80b96c8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/configs/sharegpt_config.yaml @@ -0,0 +1,7 @@ +tasks: + - name: sglang-benchmark + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name sharegpt --request-rate 16 + - name: vllm-benchmark + server_cmd: python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-8B-Instruct --disable-log-requests + client_cmd: python3 -m sglang.bench_serving --backend vllm --dataset-name sharegpt --request-rate 16 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_activation.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_activation.py new file mode 100644 index 00000000..ffb068fc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_activation.py @@ -0,0 +1,56 @@ +import itertools +import unittest + +import torch +from utils import GeluAndMul, SiluAndMul, precision + +from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler +from sglang.test.test_utils import CustomTestCase + +torch.manual_seed(1234) + + +class TestActivation(CustomTestCase): + M = [128, 129, 257] + N = [22016, 22018] + dtype = [torch.float16, torch.bfloat16] + + def _silu_and_mul_test(self, m, n, dtype): + set_global_server_args_for_scheduler(ServerArgs(model_path="dummy")) + + x = torch.randn([m, n], dtype=dtype) + + out = torch.ops.sgl_kernel.silu_and_mul_cpu(x) + ref_out = SiluAndMul(x) + + atol = rtol = precision[ref_out.dtype] + torch.testing.assert_close(ref_out, out, atol=atol, rtol=rtol) + + def _gelu_and_mul_test(self, m, n, dtype): + x = torch.randn([m, n], dtype=dtype) + + out = torch.ops.sgl_kernel.gelu_and_mul_cpu(x) + ref_out = GeluAndMul(x, approximate="none") + + atol = rtol = precision[ref_out.dtype] + torch.testing.assert_close(ref_out, out, atol=atol, rtol=rtol) + + def _gelu_tanh_and_mul_test(self, m, n, dtype): + x = torch.randn([m, n], dtype=dtype) + + out = torch.ops.sgl_kernel.gelu_tanh_and_mul_cpu(x) + ref_out = GeluAndMul(x, approximate="tanh") + + atol = rtol = precision[ref_out.dtype] + torch.testing.assert_close(ref_out, out, atol=atol, rtol=rtol) + + def test_activation(self): + for params in itertools.product(self.M, self.N, self.dtype): + with self.subTest(m=params[0], n=params[1], dtype=params[2]): + self._silu_and_mul_test(*params) + self._gelu_and_mul_test(*params) + self._gelu_tanh_and_mul_test(*params) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_binding.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_binding.py new file mode 100644 index 00000000..94dd5b11 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_binding.py @@ -0,0 +1,27 @@ +import re +import unittest + +import torch + +kernel = torch.ops.sgl_kernel + +from sglang.test.test_utils import CustomTestCase + + +class TestGemm(CustomTestCase): + def test_binding(self): + start_id = 1 + n_cpu = 6 + + expected_cores = list(map(str, range(start_id, start_id + n_cpu))) + cpu_ids = ",".join(expected_cores) + output = kernel.init_cpu_threads_env(cpu_ids) + + bindings = re.findall(r"OMP tid: \d+, core (\d+)", output) + self.assertEqual(len(bindings), n_cpu) + + self.assertEqual(bindings, expected_cores) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_cpu_graph.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_cpu_graph.py new file mode 100644 index 00000000..1adc0e89 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_cpu_graph.py @@ -0,0 +1,86 @@ +""" +Usage: +python3 -m unittest test_cpu_graph.TestCPUGraph.test_mmlu_torch_compile_cpu +""" + +import copy +import os +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import get_cpu_ids_by_node, kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + intel_amx_benchmark, + is_in_ci, + popen_launch_server, +) + + +class TestCPUGraph(CustomTestCase): + + @intel_amx_benchmark( + extra_args=[ + "--batch-size", + "1", + "--mem-fraction-static", + "0.05", + "--enable-torch-compile", + "--torch-compile-max-bs", + "1", + ], + min_throughput=10, + ) + def test_latency_torch_compile_cpu(self): + return DEFAULT_MLA_MODEL_NAME_FOR_TEST + + def test_mmlu_torch_compile_cpu(self): + model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + base_url = DEFAULT_URL_FOR_TEST + cpu_ids_by_node = get_cpu_ids_by_node() + n_numa_node = len(cpu_ids_by_node) + env = copy.deepcopy(os.environ) + env["SGLANG_CPU_OMP_THREADS_BIND"] = "all" + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--attention-backend", + "intel_amx", + "--mem-fraction-static", + "0.05", + "--disable-radix", + "--trust-remote-code", + "--disable-overlap-schedule", + "--enable-torch-compile", + "--torch-compile-max-bs", + "1", + "--tp", + f"{n_numa_node}", + ], + env=env, + ) + + try: + args = SimpleNamespace( + base_url=base_url, + model=model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + if is_in_ci(): + self.assertGreater(metrics["score"], 0.45) + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_decode.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_decode.py new file mode 100644 index 00000000..aaf5f5d4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_decode.py @@ -0,0 +1,169 @@ +import unittest + +import torch +from torch.nn.functional import scaled_dot_product_attention + +from sglang.test.test_utils import CustomTestCase + +torch.manual_seed(1234) + + +class TestDecodeAttention(CustomTestCase): + def _run_sdpa_forward_decode( + self, + query: torch.Tensor, + output: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + req_to_token: torch.Tensor, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + scaling=None, + enable_gqa=False, + causal=False, + ): + # [num_tokens, num_heads, head_size] -> [num_heads, num_tokens, head_size] + query = query.movedim(0, query.dim() - 2) + + start_q, start_kv = 0, 0 + for seq_idx in range(seq_lens.shape[0]): + seq_len_q = 1 + seq_len_kv = seq_lens[seq_idx] + end_q = start_q + seq_len_q + end_kv = start_kv + seq_len_kv + + per_req_query = query[:, start_q:end_q, :] + + # get key and value from cache. per_req_tokens contains the kv cache + # index for each token in the sequence. + req_pool_idx = req_pool_indices[seq_idx] + per_req_tokens = req_to_token[req_pool_idx, :seq_len_kv] + per_req_key = k_cache[per_req_tokens].movedim(0, query.dim() - 2) + per_req_value = v_cache[per_req_tokens].movedim(0, query.dim() - 2) + + per_req_out = ( + scaled_dot_product_attention( + per_req_query.unsqueeze(0), + per_req_key.unsqueeze(0), + per_req_value.unsqueeze(0), + enable_gqa=enable_gqa, + scale=scaling, + is_causal=causal, + ) + .squeeze(0) + .movedim(query.dim() - 2, 0) + ) + output[start_q:end_q, :, :] = per_req_out + start_q, start_kv = end_q, end_kv + + return output + + def _test_grouped_decode_attention_once(self, B, H_Q, H_KV, D, D_V, dtype, device): + # This represents the number of tokens already in the sequence + seq_len = 1024 + total_tokens = B * seq_len + sm_scale = 1.0 / (D**0.5) + logit_cap = 0.0 + num_kv_splits = 8 + enable_gqa = H_Q != H_KV + + # q represents the new token being generated, one per batch + q = torch.randn(B, H_Q, D, dtype=dtype, device=device) + + # k_buffer and v_buffer represent all previous tokens + k_buffer = torch.randn(total_tokens, H_KV, D, dtype=dtype, device=device) + v_buffer = torch.randn(total_tokens, H_KV, D_V, dtype=dtype, device=device) + + key = torch.randn(B, H_KV, D, dtype=dtype) + value = torch.randn(B, H_KV, D_V, dtype=dtype) + loc = torch.randint(0, 10, (B,)).to(torch.int64) + + # set kv cache + k_buffer[loc] = key + v_buffer[loc] = value + + # o will have the same shape as q + o = torch.zeros(B, H_Q, D_V, dtype=dtype, device=device) + o_grouped = torch.zeros(B, H_Q, D_V, dtype=dtype, device=device) + + req_to_token = ( + torch.arange(total_tokens, device=device) + .reshape(B, seq_len) + .to(torch.int32) + ) + b_req_idx = torch.arange(B, device=device).to(torch.int64) + b_seq_len = torch.full((B,), seq_len, device=device).to(torch.int64) + + attn_logits = torch.empty( + (B, H_Q, num_kv_splits, D_V + 1), + dtype=torch.float32, + device=device, + ) + + # k_buffer, v_buffer, query, key and value supports non-contiguous tensors + k_buffer = k_buffer.transpose(0, 1).contiguous().transpose(0, 1) + v_buffer = v_buffer.transpose(0, 1).contiguous().transpose(0, 1) + q = q.transpose(0, 1).contiguous().transpose(0, 1) + key = key.transpose(0, 1).contiguous().transpose(0, 1) + value = value.transpose(0, 1).contiguous().transpose(0, 1) + torch.ops.sgl_kernel.decode_attention_cpu( + q, + k_buffer, + v_buffer, + o, + key, + value, + loc, + attn_logits, + req_to_token, + b_req_idx, + b_seq_len, + sm_scale, + logit_cap, + ) + + self._run_sdpa_forward_decode( + q, + o_grouped, + k_buffer, + v_buffer, + req_to_token, + b_req_idx, + b_seq_len, + scaling=sm_scale, + enable_gqa=enable_gqa, + ) + + cos_sim = torch.nn.functional.cosine_similarity( + o.flatten(), o_grouped.flatten(), dim=0 + ) + self.assertGreater(cos_sim.item(), 0.99) + torch.testing.assert_close(o, o_grouped, atol=3e-2, rtol=1e-6) + + def _test_grouped_decode_attention(self, device="cuda"): + configs = [ + (2, 16, 16, 64, 64), + (2, 16, 1, 16, 16), + (2, 32, 8, 33, 55), + (2, 16, 1, 64, 64), + (2, 64, 1, 13, 13), + (2, 128, 1, 80, 80), + (2, 128, 2, 512, 512), + (1, 16, 1, 576, 512), + (1, 16, 16, 576, 512), + (1, 22, 1, 576, 512), + (1, 40, 8, 128, 128), + ] + + for B, H_Q, H_KV, D, D_V in configs: + for dtype in [torch.bfloat16, torch.float16]: + self._test_grouped_decode_attention_once( + B, H_Q, H_KV, D, D_V, dtype=dtype, device=device + ) + + def test_grouped_decode_attention(self): + self._test_grouped_decode_attention("cpu") + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_extend.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_extend.py new file mode 100644 index 00000000..7277050c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_extend.py @@ -0,0 +1,189 @@ +import unittest + +import torch +from torch.nn.functional import scaled_dot_product_attention + +from sglang.test.test_utils import CustomTestCase + +torch.manual_seed(1234) + + +class TestExtendAttention(CustomTestCase): + + def _run_sdpa_forward_extend( + self, + query: torch.Tensor, + output: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + req_to_token: torch.Tensor, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + extend_prefix_lens: torch.Tensor, + extend_seq_lens: torch.Tensor, + scaling=None, + enable_gqa=False, + causal=False, + ): + + assert seq_lens.shape[0] == extend_prefix_lens.shape[0] + assert seq_lens.shape[0] == extend_seq_lens.shape[0] + + # [num_tokens, num_heads, head_size] -> [num_heads, num_tokens, head_size] + query = query.movedim(0, query.dim() - 2) + + start_q, start_kv = 0, 0 + for seq_idx in range(seq_lens.shape[0]): + + extend_seq_len_q = extend_seq_lens[seq_idx] + prefill_seq_len_q = extend_prefix_lens[seq_idx] + + seq_len_kv = seq_lens[seq_idx] + end_q = start_q + extend_seq_len_q + end_kv = start_kv + seq_len_kv + + per_req_query = query[:, start_q:end_q, :] + per_req_query_redudant = torch.empty( + (per_req_query.shape[0], seq_len_kv, per_req_query.shape[2]), + dtype=per_req_query.dtype, + device=per_req_query.device, + ) + + per_req_query_redudant[:, prefill_seq_len_q:, :] = per_req_query + + # get key and value from cache. per_req_tokens contains the kv cache + # index for each token in the sequence. + req_pool_idx = req_pool_indices[seq_idx] + per_req_tokens = req_to_token[req_pool_idx, :seq_len_kv] + per_req_key = k_cache[per_req_tokens].movedim(0, query.dim() - 2) + per_req_value = v_cache[per_req_tokens].movedim(0, query.dim() - 2) + + per_req_out_redudant = ( + scaled_dot_product_attention( + per_req_query_redudant.unsqueeze(0), + per_req_key.unsqueeze(0), + per_req_value.unsqueeze(0), + enable_gqa=enable_gqa, + scale=scaling, + is_causal=causal, + ) + .squeeze(0) + .movedim(query.dim() - 2, 0) + ) + output[start_q:end_q, :, :] = per_req_out_redudant[prefill_seq_len_q:, :, :] + start_q, start_kv = end_q, end_kv + return output + + def _test_extend_attention_once(self, B, N_CTX, H_Q, H_KV, D, DV, mla=False): + dtype = torch.bfloat16 + + b_seq_len_prefix = torch.randint(1, N_CTX // 2, (B,), dtype=torch.int32) + if mla: + b_seq_len_prefix.zero_() + b_seq_len_extend = torch.randint(1, N_CTX // 2, (B,), dtype=torch.int32) + b_seq_len = b_seq_len_prefix + b_seq_len_extend + max_len_in_batch = torch.max(b_seq_len, 0)[0].item() + + b_req_idx = torch.arange(B, dtype=torch.int32) + req_to_tokens = torch.empty((B, max_len_in_batch), dtype=torch.int32) + b_start_loc = torch.zeros((B,), dtype=torch.int32) + b_start_loc[1:] = torch.cumsum(b_seq_len[:-1], 0) + b_start_loc_extend = torch.zeros((B,), dtype=torch.int32) + b_start_loc_extend[1:] = torch.cumsum(b_seq_len_extend[:-1], 0) + + for i in range(B): + req_to_tokens[i, : b_seq_len[i]] = torch.arange( + b_start_loc[i], b_start_loc[i] + b_seq_len[i] + ) + + total_token_num = torch.sum(b_seq_len).item() + extend_token_num = torch.sum(b_seq_len_extend).item() + + H_BUF = 1 if mla else H_KV + k_buffer = torch.randn((total_token_num, H_BUF, D), dtype=dtype) + v_buffer = torch.randn((total_token_num, H_BUF, DV), dtype=dtype) + + k_extend = torch.empty((extend_token_num, H_KV, D), dtype=dtype) + v_extend = torch.empty((extend_token_num, H_KV, DV), dtype=dtype) + q_extend = torch.empty((extend_token_num, H_Q, D), dtype=dtype) + + for i in range(B): + extend_start_in_buffer = b_start_loc[i] + b_seq_len_prefix[i] + extend_end_in_buffer = b_start_loc[i] + b_seq_len[i] + extend_start = b_start_loc_extend[i] + extend_end = b_start_loc_extend[i] + b_seq_len_extend[i] + k_extend[extend_start:extend_end] = k_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + v_extend[extend_start:extend_end] = v_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + q_extend[extend_start:extend_end] = torch.randn( + (b_seq_len_extend[i], H_Q, D), dtype=dtype + ) + + # q_extend, k_extend, v_extend, k_buffer and v_buffer supports non-contiguous tensors + q_extend = q_extend.transpose(0, 1).contiguous().transpose(0, 1) + k_extend = k_extend.transpose(0, 1).contiguous().transpose(0, 1) + v_extend = v_extend.transpose(0, 1).contiguous().transpose(0, 1) + k_buffer = k_buffer.transpose(0, 1).contiguous().transpose(0, 1) + v_buffer = v_buffer.transpose(0, 1).contiguous().transpose(0, 1) + + b_seq_len_extend = b_seq_len - b_seq_len_prefix + b_start_loc_extend = torch.zeros_like(b_seq_len) + b_start_loc_extend[1:] = torch.cumsum(b_seq_len_extend[:-1], 0) + max_len_extend = torch.max(b_seq_len_extend, 0)[0].item() + + sm_scale = 1.0 / (D**0.5) + logit_cap = 0.0 + + # handle index type + b_req_idx = b_req_idx.to(torch.int64) + b_seq_len = b_seq_len.to(torch.int64) + + enable_gqa = H_Q != H_KV + o_ref = torch.empty((extend_token_num, H_Q, DV), dtype=dtype) + self._run_sdpa_forward_extend( + q_extend, + o_ref, + k_buffer, + v_buffer, + req_to_tokens, + b_req_idx, + b_seq_len, + b_seq_len_prefix, + b_seq_len_extend, + scaling=sm_scale, + enable_gqa=enable_gqa, + causal=True, + ) + + o_extend = torch.empty((extend_token_num, H_Q, DV), dtype=dtype) + torch.ops.sgl_kernel.extend_attention_cpu( + q_extend, + k_extend, + v_extend, + o_extend, + k_buffer, + v_buffer, + req_to_tokens, + b_req_idx, + b_seq_len, + b_seq_len_extend, + b_start_loc_extend, + max_len_extend, + sm_scale, + logit_cap, + ) + + torch.testing.assert_close(o_ref, o_extend, atol=1e-2, rtol=1e-2) + + def test_extend_attention(self): + for is_mla in [True, False]: + self._test_extend_attention_once(1, 123, 1, 1, 128, 96, is_mla) + self._test_extend_attention_once(1, 123, 16, 1, 128, 96, is_mla) + self._test_extend_attention_once(4, 1230, 16, 4, 128, 96, is_mla) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_gemm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_gemm.py new file mode 100644 index 00000000..b9ec1e7b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_gemm.py @@ -0,0 +1,188 @@ +import itertools +import unittest + +# TODO: use interface in cpu.py +import torch +import torch.nn as nn +from utils import ( + convert_weight, + native_w8a8_per_token_matmul, + per_token_quant_int8, + precision, +) + +from sglang.test.test_utils import CustomTestCase + +torch.manual_seed(1234) + + +class Mod(nn.Module): + def __init__(self, input_channel, output_channel, has_bias): + super(Mod, self).__init__() + self.linear = torch.nn.Linear(input_channel, output_channel, has_bias) + + def forward(self, x): + return self.linear(x) + + +class TestGemm(CustomTestCase): + M = [1, 101] + N = [16, 32 * 13] + K = [32 * 16] + has_bias = [False, True] + + M_int8 = [2, 128] + N_int8 = [32 * 12] + K_int8 = [32 * 17] + + M_fp8 = [1, 11] + N_fp8 = [128, 224] + K_fp8 = [512, 576] + + def _bf16_gemm(self, M, N, K, has_bias): + + mat1 = torch.randn(M, K, dtype=torch.bfloat16) + mat2 = torch.randn(N, K, dtype=torch.bfloat16) + + ref = torch.matmul(mat1.float(), mat2.float().t()) + if has_bias: + bias = torch.randn(N, dtype=torch.float32) + ref.add_(bias.bfloat16()) + + ref = ref.bfloat16() + + out = torch.ops.sgl_kernel.weight_packed_linear( + mat1, mat2, bias if has_bias else None, False + ) + + packed_mat2 = torch.ops.sgl_kernel.convert_weight_packed(mat2) + out2 = torch.ops.sgl_kernel.weight_packed_linear( + mat1, packed_mat2, bias if has_bias else None, True + ) + + atol = rtol = precision[ref.dtype] + torch.testing.assert_close(ref, out, atol=atol, rtol=rtol) + torch.testing.assert_close(ref, out2, atol=atol, rtol=rtol) + + def test_bf16_gemm(self): + for params in itertools.product( + self.M, + self.N, + self.K, + self.has_bias, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + has_bias=params[3], + ): + self._bf16_gemm(*params) + + def _int8_gemm(self, M, N, K, has_bias): + dtype = torch.bfloat16 + A = torch.randn((M, K), dtype=dtype) / 10 + Aq, As = per_token_quant_int8(A) + + factor_for_scale = 1e-2 + int8_max = 127 + int8_min = -128 + + B = (torch.rand((N, K), dtype=torch.float32) - 0.5) * 2 + Bq = (B * int8_max).clamp(min=int8_min, max=int8_max).to(torch.int8) + Bs = torch.rand(N) * factor_for_scale + + bias = torch.randn(N) if has_bias else None + ref_out = native_w8a8_per_token_matmul(Aq, Bq, As, Bs, bias, dtype) + + atol = rtol = precision[ref_out.dtype] + + Aq2, As2 = torch.ops.sgl_kernel.per_token_quant_int8_cpu(A) + out = torch.ops.sgl_kernel.int8_scaled_mm_cpu( + Aq2, Bq, As2, Bs, bias if has_bias else None, torch.bfloat16, False + ) + torch.testing.assert_close(ref_out, out, atol=atol, rtol=rtol) + + # test the fused version + fused_out = torch.ops.sgl_kernel.int8_scaled_mm_with_quant( + A, Bq, Bs, bias if has_bias else None, torch.bfloat16, False + ) + torch.testing.assert_close(ref_out, fused_out, atol=atol, rtol=rtol) + + def test_int8_gemm(self): + for params in itertools.product( + self.M_int8, + self.N_int8, + self.K_int8, + self.has_bias, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + has_bias=params[3], + ): + self._int8_gemm(*params) + + def _fp8_gemm(self, M, N, K, has_bias): + prepack = True + chunk = False + scale_block_size_N = 64 + scale_block_size_K = 128 + assert scale_block_size_N <= N + assert scale_block_size_K <= K + A_dtype = torch.bfloat16 + + model = Mod(K, N, has_bias).eval() + if chunk: + data = torch.randn(M, K + 6, dtype=A_dtype).narrow(1, 0, K) + else: + data = torch.randn(M, K, dtype=A_dtype) + + weight = model.linear.weight # (N, K) + + if has_bias: + bias = model.linear.bias + + fp8_weight, scales, dq_weight = convert_weight( + weight, [scale_block_size_N, scale_block_size_K], A_dtype + ) + + if has_bias: + ref = torch.matmul(data.to(A_dtype), dq_weight.T) + bias.to(A_dtype) + else: + ref = torch.matmul(data.to(A_dtype), dq_weight.T) + + if prepack: + fp8_weight = torch.ops.sgl_kernel.convert_weight_packed(fp8_weight) + + opt = torch.ops.sgl_kernel.fp8_scaled_mm_cpu( + data, + fp8_weight, + scales, + [scale_block_size_N, scale_block_size_K], + bias if has_bias else None, + data.dtype, + prepack, + ) + atol = rtol = precision[ref.dtype] + torch.testing.assert_close(ref, opt, atol=atol, rtol=rtol) + + def test_fp8_gemm(self): + for params in itertools.product( + self.M_fp8, + self.N_fp8, + self.K_fp8, + self.has_bias, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + has_bias=params[3], + ): + self._fp8_gemm(*params) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_intel_amx_attention_backend_a.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_intel_amx_attention_backend_a.py new file mode 100644 index 00000000..d7cfb22f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_intel_amx_attention_backend_a.py @@ -0,0 +1,73 @@ +""" +Usage: +python3 -m unittest test_intel_amx_attention_backend.TestIntelAMXAttnBackend.test_latency_default_model +""" + +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + intel_amx_benchmark, + is_in_ci, + popen_launch_server, +) + + +class TestIntelAMXAttnBackend(CustomTestCase): + + @intel_amx_benchmark( + extra_args=["--batch-size", "4", "--mem-fraction-static", "0.3"], + min_throughput=10, + ) + def test_latency_mla_model(self): + return DEFAULT_MLA_MODEL_NAME_FOR_TEST + + @intel_amx_benchmark( + extra_args=["--batch-size", "4", "--mem-fraction-static", "0.1"], + min_throughput=40, + ) + def test_latency_default_model(self): + return DEFAULT_MODEL_NAME_FOR_TEST + + def test_mmlu(self): + model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + base_url = DEFAULT_URL_FOR_TEST + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--attention-backend", + "intel_amx", + "--mem-fraction-static", + "0.3", + "--disable-radix", + "--trust-remote-code", + "--disable-overlap-schedule", + ], + ) + + try: + args = SimpleNamespace( + base_url=base_url, + model=model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + metrics = run_eval(args) + if is_in_ci(): + self.assertGreater(metrics["score"], 0.45) + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_intel_amx_attention_backend_b.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_intel_amx_attention_backend_b.py new file mode 100644 index 00000000..58328e34 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_intel_amx_attention_backend_b.py @@ -0,0 +1,35 @@ +""" +For intel_amx attention backend FP8 tests +Usage: +python3 -m unittest test_intel_amx_attention_backend_1.TestIntelAMXAttnBackendQuant.test_latency_fp8_qwen +""" + +import unittest + +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST_FP8_WITH_MOE, + DEFAULT_MODEL_NAME_FOR_TEST_QWEN_FP8, + CustomTestCase, + intel_amx_benchmark, +) + + +class TestIntelAMXAttnBackendQuant(CustomTestCase): + + @intel_amx_benchmark( + extra_args=["--batch-size", "4", "--mem-fraction-static", "0.1"], + min_throughput=150, + ) + def test_latency_fp8_qwen(self): + return DEFAULT_MODEL_NAME_FOR_TEST_QWEN_FP8 + + @intel_amx_benchmark( + extra_args=["--batch-size", "4", "--mem-fraction-static", "0.1"], + min_throughput=50, + ) + def test_latency_fp8_moe_model(self): + return DEFAULT_MODEL_NAME_FOR_TEST_FP8_WITH_MOE + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_intel_amx_attention_backend_c.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_intel_amx_attention_backend_c.py new file mode 100644 index 00000000..42b1fd21 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_intel_amx_attention_backend_c.py @@ -0,0 +1,53 @@ +""" +For intel_amx attention backend w8a8 tests +Usage: +python3 -m unittest test_intel_amx_attention_backend_2.TestIntelAMXAttnBackendQuant.test_latency_w8a8_default_model +""" + +import unittest + +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST_W8A8, + DEFAULT_MODEL_NAME_FOR_TEST_W8A8_WITH_MOE, + CustomTestCase, + intel_amx_benchmark, +) + + +class TestIntelAMXAttnBackendQuant(CustomTestCase): + + @intel_amx_benchmark( + extra_args=[ + "--batch-size", + "4", + "--quantization", + "w8a8_int8", + "--mem-fraction-static", + "0.1", + ], + min_throughput=100, + ) + def test_latency_w8a8_default_model(self): + return DEFAULT_MODEL_NAME_FOR_TEST_W8A8 + + @intel_amx_benchmark( + extra_args=[ + "--batch-size", + "4", + "--quantization", + "w8a8_int8", + "--mem-fraction-static", + "0.9", + "--max-total-tokens", + "65536", + "--tp", + "6", + ], + min_throughput=100, + ) + def test_latency_w8a8_moe_model(self): + return DEFAULT_MODEL_NAME_FOR_TEST_W8A8_WITH_MOE + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_mla.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_mla.py new file mode 100644 index 00000000..3caa109d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_mla.py @@ -0,0 +1,155 @@ +import unittest + +import torch +from torch.nn.functional import scaled_dot_product_attention +from utils import precision + +from sglang.test.test_utils import CustomTestCase + +torch.manual_seed(1234) + + +class TestMLA(CustomTestCase): + def _run_sdpa_forward_decode( + self, + query: torch.Tensor, + output: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + key: torch.Tensor, + loc: torch.Tensor, + req_to_token: torch.Tensor, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + scaling=None, + enable_gqa=False, + causal=False, + ): + # set kv cache + k_cache[loc] = key + + # [num_tokens, num_heads, head_size] -> [num_heads, num_tokens, head_size] + query = query.movedim(0, query.dim() - 2) + + start_q, start_kv = 0, 0 + for seq_idx in range(seq_lens.shape[0]): + seq_len_q = 1 + seq_len_kv = seq_lens[seq_idx] + end_q = start_q + seq_len_q + end_kv = start_kv + seq_len_kv + + per_req_query = query[:, start_q:end_q, :] + + # get key and value from cache. per_req_tokens contains the kv cache + # index for each token in the sequence. + req_pool_idx = req_pool_indices[seq_idx] + per_req_tokens = req_to_token[req_pool_idx, :seq_len_kv] + per_req_key = k_cache[per_req_tokens].movedim(0, query.dim() - 2) + per_req_value = v_cache[per_req_tokens].movedim(0, query.dim() - 2) + + per_req_out = ( + scaled_dot_product_attention( + per_req_query.unsqueeze(0), + per_req_key.unsqueeze(0), + per_req_value.unsqueeze(0), + enable_gqa=enable_gqa, + scale=scaling, + is_causal=causal, + ) + .squeeze(0) + .movedim(query.dim() - 2, 0) + ) + output[start_q:end_q, :, :] = per_req_out + start_q, start_kv = end_q, end_kv + + return output + + def _test_grouped_decode_attention_once(self, B, H_Q, H_KV, D, D_V, seq_len): + dtype = torch.bfloat16 + + total_tokens = B * seq_len + sm_scale = 1.0 / (D**0.5) + logit_cap = 0.0 + num_kv_splits = 8 + enable_gqa = H_Q != H_KV + + # q represents the new token being generated, one per batch + q = torch.randn(B, H_Q, D, dtype=dtype) + + # k_buffer and v_buffer represent all previous tokens + k_buffer = torch.randn(total_tokens, H_KV, D, dtype=dtype) + v_buffer = k_buffer.narrow(2, 0, D_V) + + key = torch.randn(B, H_KV, D, dtype=dtype) + value = key.narrow(2, 0, D_V) + # make sure no duplicates in loc + loc = torch.randperm(total_tokens)[:B].to(torch.int64) + + k_buffer2 = k_buffer.clone() + v_buffer2 = k_buffer2.narrow(2, 0, D_V) + + # o will have the same shape as q + o = torch.zeros(B, H_Q, D_V, dtype=dtype) + o_grouped = torch.zeros(B, H_Q, D_V, dtype=dtype) + + req_to_token = torch.arange(total_tokens).reshape(B, seq_len).to(torch.int32) + b_req_idx = torch.arange(B).to(torch.int64) + b_seq_len = torch.full((B,), seq_len).to(torch.int64) + + attn_logits = torch.empty( + (B, H_Q, num_kv_splits, D_V + 1), + dtype=torch.float32, + ) + + torch.ops.sgl_kernel.decode_attention_cpu( + q, + k_buffer2, + v_buffer2, + o, + key, + value, + loc, + attn_logits, + req_to_token, + b_req_idx, + b_seq_len, + sm_scale, + logit_cap, + ) + + self._run_sdpa_forward_decode( + q, + o_grouped, + k_buffer, + v_buffer, + key, + loc, + req_to_token, + b_req_idx, + b_seq_len, + scaling=sm_scale, + enable_gqa=enable_gqa, + ) + + cos_sim = torch.nn.functional.cosine_similarity( + o.flatten(), o_grouped.flatten(), dim=0 + ) + atol = rtol = precision[q.dtype] + self.assertGreater(cos_sim.item(), 0.99) + torch.testing.assert_close(o, o_grouped, atol=atol, rtol=rtol) + torch.testing.assert_close(k_buffer, k_buffer2, atol=atol, rtol=rtol) + torch.testing.assert_close(v_buffer, v_buffer2, atol=atol, rtol=rtol) + + def test_grouped_decode_attention(self): + configs = [ + (1, 22, 1, 576, 512, 8 * 111), + (4, 22, 1, 576, 512, 8 * 128), + (40, 22, 1, 576, 512, 8 * 133), + ] + + for B, H_Q, H_KV, D, D_V, seqlen in configs: + self._test_grouped_decode_attention_once(B, H_Q, H_KV, D, D_V, seqlen) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_moe.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_moe.py new file mode 100644 index 00000000..7babd516 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_moe.py @@ -0,0 +1,264 @@ +import itertools +import math +import unittest + +# TODO: use interface in cpu.py +import torch + +kernel = torch.ops.sgl_kernel + +torch.manual_seed(1234) + +from utils import ( + BLOCK_K, + BLOCK_N, + factor_for_scale, + fp8_max, + fp8_min, + native_fp8_fused_moe, + precision, + scaled_weight, + torch_naive_fused_moe, + torch_w8a8_per_column_fused_moe, +) + +from sglang.test.test_utils import CustomTestCase + + +def fused_moe(a, w1, w2, score, topk, renormalize, prepack): + + G = 1 + topk_group = 1 + + B, D = a.shape + topk_weights = torch.empty(B, topk, dtype=torch.float32) + topk_ids = torch.empty(B, topk, dtype=torch.int32) + topk_weights, topk_ids = kernel.grouped_topk_cpu( + a, score, topk, renormalize, G, topk_group, 0, None, None + ) + + packed_w1 = kernel.convert_weight_packed(w1) if prepack else w1 + packed_w2 = kernel.convert_weight_packed(w2) if prepack else w2 + + inplace = True + return kernel.fused_experts_cpu( + a, + packed_w1, + packed_w2, + topk_weights, + topk_ids, + inplace, + False, + False, + None, + None, + None, + None, + None, + prepack, + ) + + +class TestFusedExperts(CustomTestCase): + M = [2, 114] + N = [32] + K = [32] + E = [4] + topk = [2] + renormalize = [False, True] + + M_int8 = [1, 39] + N_int8 = [128] + K_int8 = [256] + E_int8 = [8] + topk_int8 = [3] + + M_fp8 = [2, 121] + N_fp8 = [352, 512] + K_fp8 = [256, 320] + E_fp8 = [8] + topk_fp8 = [4] + + def _bf16_moe(self, m, n, k, e, topk, renormalize): + dtype = torch.bfloat16 + prepack = True + + a = torch.randn((m, k), device="cpu", dtype=dtype) / 10 + w1 = torch.randn((e, 2 * n, k), device="cpu", dtype=dtype) / 10 + w2 = torch.randn((e, k, n), device="cpu", dtype=dtype) / 10 + score = torch.randn((m, e), device="cpu", dtype=dtype) + + torch_output = torch_naive_fused_moe(a, w1, w2, score, topk, renormalize) + fused_output = fused_moe(a, w1, w2, score, topk, renormalize, prepack) + + atol = rtol = precision[torch_output.dtype] + torch.testing.assert_close(torch_output, fused_output, atol=atol, rtol=rtol) + + def test_bf16_moe(self): + for params in itertools.product( + self.M, + self.N, + self.K, + self.E, + self.topk, + self.renormalize, + ): + with self.subTest( + m=params[0], + n=params[1], + k=params[2], + e=params[3], + topk=params[4], + renormalize=params[5], + ): + self._bf16_moe(*params) + + def _int8_moe(self, M, N, K, E, topk): + dtype = torch.bfloat16 + prepack = True + + # Initialize int8 quantization parameters + int8_factor_for_scale = 1e-2 + int8_max = 127 + int8_min = -128 + + # Input tensor + # M * K + a = torch.randn((M, K), dtype=dtype) / math.sqrt(K) + + # Generate int8 weights + w1_fp32 = (torch.rand((E, 2 * N, K), dtype=torch.float32) - 0.5) * 2 + w1 = (w1_fp32 * int8_max).clamp(min=int8_min, max=int8_max).to(torch.int8) + + w2_fp32 = (torch.rand((E, K, N), dtype=torch.float32) - 0.5) * 2 + w2 = (w2_fp32 * int8_max).clamp(min=int8_min, max=int8_max).to(torch.int8) + + # Generate scale for each column (per-column quantization) + w1_s = torch.rand(E, 2 * N, device=w1_fp32.device) * int8_factor_for_scale + w2_s = torch.rand(E, K, device=w2_fp32.device) * int8_factor_for_scale + + # Calculate routing + score = torch.randn((M, E), dtype=dtype) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + + ref_out = torch_w8a8_per_column_fused_moe( + a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, topk + ) + + inplace = True + packed_w1 = kernel.convert_weight_packed(w1) if prepack else w1 + packed_w2 = kernel.convert_weight_packed(w2) if prepack else w2 + out = kernel.fused_experts_cpu( + a, + packed_w1, + packed_w2, + topk_weight, + topk_ids.to(torch.int32), + inplace, + True, + False, + w1_s, + w2_s, + None, + None, + None, + prepack, + ) + + atol = rtol = precision[ref_out.dtype] + # Increase the tolerance for large input shapes + if M > 35: + atol = rtol = 0.02 + torch.testing.assert_close(ref_out, out, atol=atol, rtol=rtol) + + def test_int8_moe(self): + for params in itertools.product( + self.M_int8, + self.N_int8, + self.K_int8, + self.E_int8, + self.topk_int8, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + E=params[3], + topk=params[4], + ): + self._int8_moe(*params) + + def _fp8_moe(self, M, N, K, E, topk): + dtype = torch.bfloat16 + + a = torch.randn(M, K, dtype=dtype) / math.sqrt(K) + + w1_fp32 = torch.randn(E, 2 * N, K) + w1 = (w1_fp32 * fp8_max).clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + w2_fp32 = torch.randn(E, K, N) + w2 = (w2_fp32 * fp8_max).clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + w1s = ( + torch.randn(E, math.ceil(2 * N / BLOCK_N), math.ceil(K / BLOCK_K)) + * factor_for_scale + ) + w2s = ( + torch.randn(E, math.ceil(K / BLOCK_N), math.ceil(N / BLOCK_K)) + * factor_for_scale + ) + + w1_scaled = scaled_weight(w1, w1s) + w2_scaled = scaled_weight(w2, w2s) + + score = torch.randn((M, E), dtype=dtype) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + + w1 = kernel.convert_weight_packed(w1) + w2 = kernel.convert_weight_packed(w2) + + ref_out = native_fp8_fused_moe( + a, w1_scaled, w2_scaled, topk_weight, topk_ids, topk + ) + out = kernel.fused_experts_cpu( + a, + w1, + w2, + topk_weight, + topk_ids.to(torch.int32), + False, + False, + True, + w1s, + w2s, + [BLOCK_N, BLOCK_K], + None, + None, + True, + ) + + atol = rtol = precision[dtype] + torch.testing.assert_close(ref_out.bfloat16(), out, atol=atol, rtol=rtol) + + def test_fp8_moe(self): + for params in itertools.product( + self.M_fp8, + self.N_fp8, + self.K_fp8, + self.E_fp8, + self.topk_fp8, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + E=params[3], + topk=params[4], + ): + self._fp8_moe(*params) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_norm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_norm.py new file mode 100644 index 00000000..8bb2eba0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_norm.py @@ -0,0 +1,135 @@ +import itertools +import unittest +from typing import Optional, Tuple, Union + +import torch +from utils import make_non_contiguous, precision + +from sglang.test.test_utils import CustomTestCase + +torch.manual_seed(1234) + + +class TestNorm(CustomTestCase): + M = [4096, 1024] + N = [4096, 4096 + 13] + dtype = [torch.float16, torch.bfloat16] + + def _forward_native( + self, + x: torch.Tensor, + weight: torch.Tensor, + variance_epsilon: float = 1e-6, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + orig_dtype = x.dtype + x = x.to(torch.float32) + if residual is not None: + x = x + residual.to(torch.float32) + residual = x.to(orig_dtype) + + variance = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + variance_epsilon) + x = x.to(orig_dtype) * weight + if residual is None: + return x + else: + return x, residual + + def _norm_test(self, m, n, dtype): + + x = torch.randn([m, n], dtype=dtype) + x = make_non_contiguous(x) + hidden_size = x.size(-1) + weight = torch.randn(hidden_size, dtype=dtype) + variance_epsilon = 1e-6 + + out = torch.ops.sgl_kernel.rmsnorm_cpu(x, weight, variance_epsilon) + ref_out = self._forward_native(x, weight, variance_epsilon) + + atol = rtol = precision[ref_out.dtype] + torch.testing.assert_close(ref_out, out, atol=atol, rtol=rtol) + + ref_x = x.clone() + residual = torch.randn([m, hidden_size], dtype=dtype) + ref_residual = residual.clone() + + torch.ops.sgl_kernel.fused_add_rmsnorm_cpu( + x, residual, weight, variance_epsilon + ) + + ref_x, ref_residual = self._forward_native( + ref_x, weight, variance_epsilon, ref_residual + ) + + torch.testing.assert_close(x, ref_x, atol=atol, rtol=rtol) + torch.testing.assert_close(residual, ref_residual, atol=atol, rtol=rtol) + + def _l2norm_test(self, m, n, dtype): + + x = torch.randn([m, n], dtype=dtype) + hidden_size = x.size(-1) + fake_ones_weight = torch.ones(hidden_size, dtype=dtype) + variance_epsilon = 1e-6 + + out = torch.ops.sgl_kernel.l2norm_cpu(x, variance_epsilon) + ref_out = self._forward_native(x, fake_ones_weight, variance_epsilon) + + atol = rtol = precision[ref_out.dtype] + torch.testing.assert_close(ref_out, out, atol=atol, rtol=rtol) + + def test_norm(self): + for params in itertools.product(self.M, self.N, self.dtype): + with self.subTest(m=params[0], n=params[1], dtype=params[2]): + self._norm_test(*params) + self._l2norm_test(*params) + + +class TestFusedRMSNormGated(CustomTestCase): + M = [4096, 1024] + N = [4096, 4096 + 13] + dtype = [torch.float16, torch.bfloat16] + + def _forward_native( + self, + hidden_states: torch.Tensor, + weight: torch.Tensor, + variance_epsilon: float = 1e-6, + gate: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + # Norm before gate + hidden_states = hidden_states * torch.rsqrt(variance + variance_epsilon) + hidden_states = weight * hidden_states.to(input_dtype) + hidden_states = hidden_states * torch.nn.functional.silu(gate.to(torch.float32)) + + return hidden_states.to(input_dtype) + + def _norm_test(self, m, n, dtype): + + x = torch.randn([m, n], dtype=dtype) + x = make_non_contiguous(x) + batch_size = x.size(0) + hidden_size = x.size(-1) + weight = torch.randn(hidden_size, dtype=dtype) + variance_epsilon = 1e-6 + gate = torch.randn([batch_size, hidden_size], dtype=dtype) + + out = torch.ops.sgl_kernel.fused_rmsnorm_gated_cpu( + x, weight, gate, variance_epsilon + ) + ref_out = self._forward_native(x, weight, variance_epsilon, gate) + + atol = rtol = precision[ref_out.dtype] * 2 + torch.testing.assert_close(ref_out, out, atol=atol, rtol=rtol) + + def test_norm(self): + for params in itertools.product(self.M, self.N, self.dtype): + with self.subTest(m=params[0], n=params[1], dtype=params[2]): + self._norm_test(*params) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_qkv_proj_with_rope.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_qkv_proj_with_rope.py new file mode 100644 index 00000000..b0b22d3b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_qkv_proj_with_rope.py @@ -0,0 +1,431 @@ +import unittest + +import torch +from utils import ( + convert_weight, + native_w8a8_per_token_matmul, + per_token_quant_int8, + precision, +) + +from sglang.srt.layers.rotary_embedding import _apply_rotary_emb +from sglang.test.test_utils import CustomTestCase + +convert_weight_packed = torch.ops.sgl_kernel.convert_weight_packed +qkv_proj_with_rope = torch.ops.sgl_kernel.qkv_proj_with_rope +qkv_proj_with_rope_fused_weight = torch.ops.sgl_kernel.qkv_proj_with_rope_fused_weight +torch.manual_seed(1234) +# constants +kv_lora_rank = 512 +qk_head_dim = 192 +qk_nope_head_dim = 128 +qk_rope_head_dim = 64 +rotary_dim = qk_rope_head_dim +num_heads = 22 +q_lora_rank = 1536 +hidden_size = 7168 +B = 1 +eps = 1e-6 + + +def layernorm(x, weight, variance_epsilon=1e-6, residual=None): + orig_dtype = x.dtype + x = x.to(torch.float32) + variance = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + variance_epsilon) + return (x * weight).to(orig_dtype) + + +def rotary_emb(q_pe, k_pe, pos, cos_sin_cache): + orig_dtype = q_pe.dtype + q_pe = q_pe.float() + k_pe = k_pe.float() + cos_sin_cache = cos_sin_cache.float() + + query_rot = q_pe[..., :rotary_dim] + key_rot = k_pe[..., :rotary_dim] + cos_sin = cos_sin_cache[pos] + cos, sin = cos_sin.chunk(2, dim=-1) + query_rot = _apply_rotary_emb(query_rot, cos, sin, False) + key_rot = _apply_rotary_emb(key_rot, cos, sin, False) + return query_rot.to(orig_dtype), key_rot.to(orig_dtype) + + +def native_torch( + q_input, + hidden_states, + q_a_proj_weight, + norm_weight1, + q_b_proj_weight, + w_kc, + kv_a_proj_weight, + norm_weight2, + pos, + cos_sin_cache, +): + + q = torch.matmul(hidden_states, q_a_proj_weight.t()) + q = layernorm(q, norm_weight1) + q = torch.matmul(q, q_b_proj_weight.t()).view(-1, num_heads, qk_head_dim) + + q_nope, q_pe = q.split([qk_nope_head_dim, qk_rope_head_dim], dim=-1) + q_nope_out = torch.bmm(q_nope.transpose(0, 1), w_kc) + + q_input[..., :kv_lora_rank] = q_nope_out.transpose(0, 1) + latent_cache = torch.matmul(hidden_states, kv_a_proj_weight.t()) + v_input = latent_cache[..., :kv_lora_rank] + v_input = layernorm(v_input.contiguous(), norm_weight2).unsqueeze(1) + k_input = latent_cache.unsqueeze(1) + k_input[..., :kv_lora_rank] = v_input + k_pe = k_input[..., kv_lora_rank:] + + q_pe, k_pe = rotary_emb(q_pe, k_pe, pos, cos_sin_cache) + q_input[..., kv_lora_rank:] = q_pe + k_input[..., kv_lora_rank:] = k_pe + + return q_input, k_input, v_input + + +def native_torch_int8( + q_input, + hidden_states, + w1_q, + w1_s, + norm_weight1, + w2_q, + w2_s, + w_kc, + w3_q, + w3_s, + norm_weight2, + pos, + cos_sin_cache, +): + + a_q, a_s = per_token_quant_int8(hidden_states) + q = native_w8a8_per_token_matmul(a_q, w1_q, a_s, w1_s, None, torch.bfloat16) + q = layernorm(q, norm_weight1) + + a_q, a_s = per_token_quant_int8(q) + q = native_w8a8_per_token_matmul(a_q, w2_q, a_s, w2_s, None, torch.bfloat16).view( + -1, num_heads, qk_head_dim + ) + + q_nope, q_pe = q.split([qk_nope_head_dim, qk_rope_head_dim], dim=-1) + q_nope_out = torch.bmm(q_nope.transpose(0, 1), w_kc) + + q_input[..., :kv_lora_rank] = q_nope_out.transpose(0, 1) + a_q, a_s = per_token_quant_int8(hidden_states) + latent_cache = native_w8a8_per_token_matmul( + a_q, w3_q, a_s, w3_s, None, torch.bfloat16 + ) + v_input = latent_cache[..., :kv_lora_rank] + v_input = layernorm(v_input.contiguous(), norm_weight2).unsqueeze(1) + k_input = latent_cache.unsqueeze(1) + k_input[..., :kv_lora_rank] = v_input + k_pe = k_input[..., kv_lora_rank:] + + q_pe, k_pe = rotary_emb(q_pe, k_pe, pos, cos_sin_cache) + q_input[..., kv_lora_rank:] = q_pe + k_input[..., kv_lora_rank:] = k_pe + + return q_input, k_input, v_input + + +class TestQKVProjWithROPE(CustomTestCase): + def test_bf16_qkv_proj_with_rope(self): + dtype = torch.bfloat16 + hidden_states = torch.randn(B, hidden_size, dtype=dtype) / hidden_size + q_input = torch.empty( + B, num_heads, kv_lora_rank + qk_rope_head_dim, dtype=dtype + ) + q_a_proj_weight = torch.randn(q_lora_rank, hidden_size, dtype=dtype) * 0.1 + norm_weight1 = torch.randn(q_lora_rank, dtype=dtype) + q_b_proj_weight = ( + torch.randn(num_heads * qk_head_dim, q_lora_rank, dtype=dtype) * 0.1 + ) + w_kc = torch.randn(num_heads, kv_lora_rank, qk_nope_head_dim, dtype=dtype) * 0.1 + kv_a_proj_weight = ( + torch.randn(kv_lora_rank + qk_rope_head_dim, hidden_size, dtype=dtype) * 0.1 + ) + fused_weight = torch.cat([q_a_proj_weight, kv_a_proj_weight], dim=0) + norm_weight2 = torch.randn(kv_lora_rank, dtype=dtype) + pos = torch.randint(10, 100, (B,)) + cos_sin_cache = torch.randn(100, rotary_dim, dtype=dtype) + q_ref, k_ref, v_ref = native_torch( + q_input, + hidden_states, + q_a_proj_weight, + norm_weight1, + q_b_proj_weight, + w_kc.transpose(1, 2), + kv_a_proj_weight, + norm_weight2, + pos, + cos_sin_cache, + ) + qa_packed = convert_weight_packed(q_a_proj_weight) + qb_packed = convert_weight_packed(q_b_proj_weight) + kva_packed = convert_weight_packed(kv_a_proj_weight) + wkc_packed = convert_weight_packed(w_kc) + fused_weight_packed = convert_weight_packed(fused_weight) + + q_out, k_out, v_out = qkv_proj_with_rope( + hidden_states, + qa_packed, + qb_packed, + kva_packed, + wkc_packed, + norm_weight1, + norm_weight2, + pos, + cos_sin_cache, + eps, + False, + False, + None, + None, + None, + True, + None, + ) + fused_q_out, fused_k_out, fused_v_out = qkv_proj_with_rope_fused_weight( + hidden_states, + fused_weight_packed, + qb_packed, + wkc_packed, + norm_weight1, + norm_weight2, + pos, + cos_sin_cache, + eps, + False, + False, + None, + None, + True, + None, + q_lora_rank, + kv_lora_rank, + qk_rope_head_dim, + ) + atol = rtol = precision[q_ref.dtype] + torch.testing.assert_close(q_ref, q_out, atol=atol, rtol=rtol) + torch.testing.assert_close(k_ref, k_out, atol=atol, rtol=rtol) + torch.testing.assert_close(v_ref, v_out, atol=atol, rtol=rtol) + torch.testing.assert_close(fused_q_out, q_out) + torch.testing.assert_close(fused_k_out, k_out) + torch.testing.assert_close(fused_v_out, v_out) + + def test_int8_qkv_proj_with_rope(self): + dtype = torch.bfloat16 + hidden_states = torch.randn(B, hidden_size, dtype=dtype) / hidden_size + q_input = torch.empty( + B, num_heads, kv_lora_rank + qk_rope_head_dim, dtype=dtype + ) + q_a_proj_weight = torch.randn(q_lora_rank, hidden_size, dtype=dtype) * 0.1 + norm_weight1 = torch.randn(q_lora_rank, dtype=dtype) + q_b_proj_weight = ( + torch.randn(num_heads * qk_head_dim, q_lora_rank, dtype=dtype) * 0.1 + ) + w_kc = torch.randn(num_heads, kv_lora_rank, qk_nope_head_dim, dtype=dtype) * 0.1 + kv_a_proj_weight = ( + torch.randn(kv_lora_rank + qk_rope_head_dim, hidden_size, dtype=dtype) * 0.1 + ) + norm_weight2 = torch.randn(kv_lora_rank, dtype=dtype) + pos = torch.randint(10, 100, (B,)) + cos_sin_cache = torch.randn(100, rotary_dim, dtype=dtype) + + w1_q, w1_s = per_token_quant_int8(q_a_proj_weight) + w2_q, w2_s = per_token_quant_int8(q_b_proj_weight) + w3_q, w3_s = per_token_quant_int8(kv_a_proj_weight) + q_ref, k_ref, v_ref = native_torch_int8( + q_input, + hidden_states, + w1_q, + w1_s, + norm_weight1, + w2_q, + w2_s, + w_kc.transpose(1, 2), + w3_q, + w3_s, + norm_weight2, + pos, + cos_sin_cache, + ) + w1_q_packed = convert_weight_packed(w1_q) + w2_q_packed = convert_weight_packed(w2_q) + w3_q_packed = convert_weight_packed(w3_q) + wkc_packed = convert_weight_packed(w_kc) + q_out, k_out, v_out = qkv_proj_with_rope( + hidden_states, + w1_q_packed, + w2_q_packed, + w3_q_packed, + wkc_packed, + norm_weight1, + norm_weight2, + pos, + cos_sin_cache, + eps, + True, + False, + w1_s, + w2_s, + w3_s, + True, + None, + ) + fused_weight = torch.cat([w1_q, w3_q], dim=0) + fused_weight_s = torch.cat([w1_s, w3_s], dim=0) + w_fused_q_packed = convert_weight_packed(fused_weight) + fused_q_out, fused_k_out, fused_v_out = qkv_proj_with_rope_fused_weight( + hidden_states, + w_fused_q_packed, + w2_q_packed, + wkc_packed, + norm_weight1, + norm_weight2, + pos, + cos_sin_cache, + eps, + True, + False, + fused_weight_s, + w2_s, + True, + None, + q_lora_rank, + kv_lora_rank, + qk_rope_head_dim, + ) + atol = rtol = precision[q_ref.dtype] + torch.testing.assert_close(q_ref, q_out, atol=atol, rtol=rtol) + torch.testing.assert_close(k_ref, k_out, atol=atol, rtol=rtol) + torch.testing.assert_close(v_ref, v_out, atol=atol, rtol=rtol) + torch.testing.assert_close(fused_q_out, q_out) + torch.testing.assert_close(fused_k_out, k_out) + torch.testing.assert_close(fused_v_out, v_out) + + def test_fp8_qkv_proj_with_rope(self): + dtype = torch.bfloat16 + hidden_states = torch.randn(B, hidden_size, dtype=dtype) / hidden_size + q_input = torch.empty( + B, num_heads, kv_lora_rank + qk_rope_head_dim, dtype=dtype + ) + q_a_proj_weight = torch.randn(q_lora_rank, hidden_size, dtype=dtype) * 0.1 + norm_weight1 = torch.randn(q_lora_rank, dtype=dtype) + q_b_proj_weight = ( + torch.randn(num_heads * qk_head_dim, q_lora_rank, dtype=dtype) * 0.1 + ) + w_kc = torch.randn(num_heads, kv_lora_rank, qk_nope_head_dim, dtype=dtype) * 0.1 + kv_a_proj_weight = ( + torch.randn(kv_lora_rank + qk_rope_head_dim, hidden_size, dtype=dtype) * 0.1 + ) + norm_weight2 = torch.randn(kv_lora_rank, dtype=dtype) + pos = torch.randint(10, 100, (B,)) + cos_sin_cache = torch.randn(100, rotary_dim, dtype=dtype) + + scale_block_size_N = 128 + scale_block_size_K = 128 + fp8_q_a_proj_weight, q_a_proj_weight_scale_inv, q_a_proj_weight_dq = ( + convert_weight( + q_a_proj_weight, + [scale_block_size_N, scale_block_size_K], + torch.bfloat16, + ) + ) + fp8_q_b_proj_weight, q_b_proj_weight_scale_inv, q_b_proj_weight_dq = ( + convert_weight( + q_b_proj_weight, + [scale_block_size_N, scale_block_size_K], + torch.bfloat16, + ) + ) + ( + fp8_kv_a_proj_with_mqa_weight, + kv_a_proj_with_mqa_weight_scale_inv, + kv_a_proj_with_mqa_weight_dq, + ) = convert_weight( + kv_a_proj_weight, [scale_block_size_N, scale_block_size_K], torch.bfloat16 + ) + q_ref, k_ref, v_ref = native_torch( + q_input, + hidden_states, + q_a_proj_weight_dq, + norm_weight1, + q_b_proj_weight_dq, + w_kc.transpose(1, 2), + kv_a_proj_with_mqa_weight_dq, + norm_weight2, + pos, + cos_sin_cache, + ) + fp8_q_a_proj_weight_packed = convert_weight_packed(fp8_q_a_proj_weight) + fp8_q_b_proj_weight_packed = convert_weight_packed(fp8_q_b_proj_weight) + fp8_kv_a_proj_with_mqa_weight_packed = convert_weight_packed( + fp8_kv_a_proj_with_mqa_weight + ) + w_kc = convert_weight_packed(w_kc) + q_out, k_out, v_out = qkv_proj_with_rope( + hidden_states, + fp8_q_a_proj_weight_packed, + fp8_q_b_proj_weight_packed, + fp8_kv_a_proj_with_mqa_weight_packed, + w_kc, + norm_weight1, + norm_weight2, + pos, + cos_sin_cache, + eps, + False, + True, + q_a_proj_weight_scale_inv.float(), + q_b_proj_weight_scale_inv.float(), + kv_a_proj_with_mqa_weight_scale_inv.float(), + True, + [scale_block_size_N, scale_block_size_K], + ) + + fused_weight = torch.cat( + [fp8_q_a_proj_weight, fp8_kv_a_proj_with_mqa_weight], dim=0 + ) + fused_weight_s = torch.cat( + [q_a_proj_weight_scale_inv, kv_a_proj_with_mqa_weight_scale_inv], dim=0 + ) + fused_weight_packed = convert_weight_packed(fused_weight) + fused_q_out, fused_k_out, fused_v_out = qkv_proj_with_rope_fused_weight( + hidden_states, + fused_weight_packed, + fp8_q_b_proj_weight_packed, + w_kc, + norm_weight1, + norm_weight2, + pos, + cos_sin_cache, + eps, + False, + True, + fused_weight_s.float(), + q_b_proj_weight_scale_inv.float(), + True, + [scale_block_size_N, scale_block_size_K], + q_lora_rank, + kv_lora_rank, + qk_rope_head_dim, + ) + atol = rtol = precision[q_ref.dtype] + # Due to the change in multiplication order, the error is amplified. + # In the model, with fewer layers, this doesn't cause issues, but in + # tests with more layers, we need to enlarge the tolerance to pass the tests. + torch.testing.assert_close(q_ref, q_out, atol=1e-1, rtol=1e-1) + torch.testing.assert_close(k_ref, k_out, atol=atol, rtol=rtol) + torch.testing.assert_close(v_ref, v_out, atol=atol, rtol=rtol) + torch.testing.assert_close(fused_q_out, q_out) + torch.testing.assert_close(fused_k_out, k_out) + torch.testing.assert_close(fused_v_out, v_out) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_rope.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_rope.py new file mode 100644 index 00000000..8c1dfe9a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_rope.py @@ -0,0 +1,180 @@ +import unittest + +import torch +from utils import precision + +from sglang.srt.layers.rotary_embedding import ( + DeepseekScalingRotaryEmbedding, + RotaryEmbedding, +) +from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler +from sglang.test.test_utils import CustomTestCase + +torch.manual_seed(1234) + + +class TestROPE(CustomTestCase): + def test_deepseek_v2_rope(self): + num_head = 16 + seq_len = 1024 + q_head_dim = 192 + qk_nope_head_dim = 128 + qk_rope_head_dim = 64 + max_pos = 256 + k_dim = 576 + rotary_dim = 64 + is_neox_style = False + set_global_server_args_for_scheduler(ServerArgs(model_path="dummy")) + + # Create cos_sin_cache + freqs = torch.rand(max_pos, qk_rope_head_dim // 2) + cos = freqs.cos() * 0.7 + sin = freqs.sin() * 0.7 + cos_sin_cache = torch.cat((cos, sin), dim=-1).to(torch.bfloat16) + positions = torch.randint(0, max_pos, (seq_len,)) + + rope = DeepseekScalingRotaryEmbedding( + qk_rope_head_dim, + rotary_dim, + max_pos, + 16, # not used since cos_sin_cache is provided + is_neox_style, + 1.0, + torch.bfloat16, + device="cpu", + ) + rope.register_buffer("cos_sin_cache", cos_sin_cache) + + for dtype in [torch.bfloat16]: + enable_autocast = True + + with torch.no_grad(), torch.amp.autocast("cpu", enabled=enable_autocast): + q = torch.randn(seq_len, num_head, q_head_dim, dtype=dtype) + q_clone = q.clone() + k = torch.randn(seq_len, 1, k_dim, dtype=dtype) + k_clone = k.clone() + _, q_pe = q.split([qk_nope_head_dim, qk_rope_head_dim], dim=-1) + _, q_pe_clone = q_clone.split( + [qk_nope_head_dim, qk_rope_head_dim], dim=-1 + ) + k_pe = k[:, :, k_dim - qk_rope_head_dim :] + k_pe_clone = k_clone[:, :, k_dim - qk_rope_head_dim :] + + # ref kernel + q_pe, k_pe = rope.forward_native( + query=q_pe, + key=k_pe, + positions=positions, + ) + + # fused rope kernel + q_pe_clone, k_pe_clone = torch.ops.sgl_kernel.rotary_embedding_cpu( + positions, + q_pe_clone, + k_pe_clone, + rope.head_size, + cos_sin_cache, + False, + ) + + atol = rtol = precision[q_pe.dtype] + torch.testing.assert_close(q_pe, q_pe_clone, atol=atol, rtol=rtol) + torch.testing.assert_close(k_pe, k_pe_clone, atol=atol, rtol=rtol) + torch.testing.assert_close(k_pe, k_pe_clone) + + def test_origin_rope(self): + def single_test( + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + dtype: torch.dtype, + device: str, + batch_size: int, + seq_len: int, + num_q_heads: int, + num_kv_heads: int, + ): + set_global_server_args_for_scheduler(ServerArgs(model_path="dummy")) + torch.manual_seed(100) + rope_ref = RotaryEmbedding( + head_size, + rotary_dim, + max_position_embeddings, + base, + is_neox_style, + dtype, + ).to(device) + pos_ids = torch.arange(seq_len, device=device).repeat(batch_size) + query = torch.randn( + batch_size * seq_len, + num_q_heads * head_size, + dtype=dtype, + device=device, + ) + key = torch.randn( + batch_size * seq_len, + num_kv_heads * head_size, + dtype=dtype, + device=device, + ) + + query_ref, key_ref = query.clone(), key.clone() + query_cpu, key_cpu = query.clone(), key.clone() + + query_ref_out, key_ref_out = rope_ref.forward_native( + pos_ids, query_ref, key_ref + ) + query_cpu_out, key_cpu_out = torch.ops.sgl_kernel.rotary_embedding_cpu( + pos_ids, + query_cpu, + key_cpu, + rope_ref.head_size, + rope_ref.cos_sin_cache.to(query.dtype), + rope_ref.is_neox_style, + ) + torch.testing.assert_close( + query_ref_out, query_cpu_out, atol=1e-2, rtol=1e-2 + ) + torch.testing.assert_close(key_ref_out, key_cpu_out, atol=1e-2, rtol=1e-2) + + test_config = [ + (64, 64, 32, 8000, True, torch.bfloat16, "cpu", 32, 32, 1, 1), + (256, 128, 4096, 10000, True, torch.bfloat16, "cpu", 2, 512, 32, 8), + (512, 128, 311, 10000, True, torch.bfloat16, "cpu", 3, 39, 4, 2), + (128, 128, 2048, 10000, False, torch.bfloat16, "cpu", 2, 512, 32, 8), + (128, 128, 2048, 10000, False, torch.bfloat16, "cpu", 2, 512, 16, 4), + (512, 128, 311, 10000, False, torch.bfloat16, "cpu", 3, 39, 4, 2), + ] + + for ( + head_size, + rotary_dim, + max_position_embeddings, + base, + is_neox_style, + dtype, + device, + batch_size, + seq_len, + num_q_heads, + num_kv_heads, + ) in test_config: + single_test( + head_size, + rotary_dim, + max_position_embeddings, + base, + is_neox_style, + dtype, + device, + batch_size, + seq_len, + num_q_heads, + num_kv_heads, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_shared_expert.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_shared_expert.py new file mode 100644 index 00000000..358709a6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_shared_expert.py @@ -0,0 +1,224 @@ +import itertools +import math +import unittest + +# TODO: use interface in cpu.py +import torch +from utils import ( + BLOCK_K, + BLOCK_N, + SiluAndMul, + factor_for_scale, + fp8_max, + fp8_min, + per_token_quant_int8, + precision, + scaled_weight, + torch_naive_moe, + torch_w8a8_per_column_moe, +) + +from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler +from sglang.test.test_utils import CustomTestCase + +torch.manual_seed(1234) + + +class TestSharedExpert(CustomTestCase): + M = [2, 121] + N = [32, 32 * 4] + K = [32, 32 * 2] + routed_scaling_factor = [16] + + M_fp8 = [2, 12] + N_fp8 = [512] + K_fp8 = [256] + + def _bf16_shared_expert(self, m, n, k, routed_scaling_factor): + dtype = torch.bfloat16 + prepack = True + + hidden_states = torch.randn(m, k, dtype=dtype) / k + w1 = torch.randn(2 * n, k, dtype=dtype) + w2 = torch.randn(k, n, dtype=dtype) + fused_output = torch.randn(m, k, dtype=dtype) / k + + # fused moe mutates content in hs + hidden_states2 = hidden_states.clone() + + # bfloat16 + ref = torch_naive_moe( + hidden_states.float(), + w1.float(), + w2.float(), + fused_output.float(), + routed_scaling_factor, + ).to(dtype=dtype) + res = torch.ops.sgl_kernel.shared_expert_cpu( + hidden_states, + w1, + w2, + fused_output, + routed_scaling_factor, + True, + False, + False, + None, + None, + None, + None, + None, + False, + ) + + atol = rtol = precision[ref.dtype] + torch.testing.assert_close(ref, res, atol=atol, rtol=rtol) + + def test_bf16_shared_expert(self): + for params in itertools.product( + self.M, + self.N, + self.K, + self.routed_scaling_factor, + ): + with self.subTest( + m=params[0], + n=params[1], + k=params[2], + routed_scaling_factor=params[3], + ): + self._bf16_shared_expert(*params) + + def _int8_shared_expert(self, m, n, k, routed_scaling_factor): + dtype = torch.bfloat16 + prepack = True + + hidden_states = torch.randn(m, k, dtype=dtype) / k + w1 = torch.randn(2 * n, k, dtype=dtype) + w2 = torch.randn(k, n, dtype=dtype) + fused_output = torch.randn(m, k, dtype=dtype) / k + + # fused moe mutates content in hs + hidden_states2 = hidden_states.clone() + + w1_q, w1_s = per_token_quant_int8(w1) + w2_q, w2_s = per_token_quant_int8(w2) + ref2 = torch_w8a8_per_column_moe( + hidden_states2.float(), + w1_q, + w2_q, + w1_s, + w2_s, + fused_output.float(), + routed_scaling_factor, + ).to(dtype=dtype) + res2 = torch.ops.sgl_kernel.shared_expert_cpu( + hidden_states2, + w1_q, + w2_q, + fused_output, + routed_scaling_factor, + True, + True, + False, + w1_s, + w2_s, + None, + None, + None, + False, + ) + + atol = rtol = precision[ref2.dtype] + torch.testing.assert_close(ref2, res2, atol=atol, rtol=rtol) + + def test_int8_shared_expert(self): + for params in itertools.product( + self.M, + self.N, + self.K, + self.routed_scaling_factor, + ): + with self.subTest( + m=params[0], + n=params[1], + k=params[2], + routed_scaling_factor=params[3], + ): + self._int8_shared_expert(*params) + + def _fp8_shared_expert(self, M, N, K, routed_scaling_factor): + set_global_server_args_for_scheduler(ServerArgs(model_path="dummy")) + + dtype = torch.bfloat16 + prepack = True + + a = torch.randn(M, K, dtype=dtype) / math.sqrt(K) + + w1_fp32 = torch.randn(1, 2 * N, K) + w1 = (w1_fp32 * fp8_max).clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + w2_fp32 = torch.randn(1, K, N) + w2 = (w2_fp32 * fp8_max).clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + w1s = torch.randn(1, 2 * N // BLOCK_N, K // BLOCK_K) * factor_for_scale + w2s = torch.randn(1, K // BLOCK_N, N // BLOCK_K) * factor_for_scale + + w1_scaled = scaled_weight(w1, w1s).view(2 * N, K) + w2_scaled = scaled_weight(w2, w2s).view(K, N) + + # change back to 2D + w1, w2 = w1.squeeze(0), w2.squeeze(0) + w1s, w2s = w1s.squeeze(0), w2s.squeeze(0) + w1_scaled, w2_scaled = w1_scaled.squeeze(0), w2_scaled.squeeze(0) + + fused_out = torch.randn(M, K, dtype=dtype) / math.sqrt(K) + a2 = a.clone() + + # ref + ic0 = torch.matmul(a.float(), w1_scaled.transpose(0, 1)) + ic1 = SiluAndMul(ic0) + shared_out = torch.matmul(ic1, w2_scaled.transpose(0, 1)) + ref_out = shared_out + fused_out.float() * routed_scaling_factor + ref_out = ref_out.to(dtype=dtype) + + w1 = torch.ops.sgl_kernel.convert_weight_packed(w1) # [2N, K] + w2 = torch.ops.sgl_kernel.convert_weight_packed(w2) # [K, N] + out = torch.ops.sgl_kernel.shared_expert_cpu( + a2, + w1, + w2, + fused_out, + routed_scaling_factor, + True, + False, + True, + w1s, + w2s, + [BLOCK_N, BLOCK_K], + None, + None, + True, + ) + + atol = rtol = precision[ref_out.dtype] + torch.testing.assert_close(ref_out, out, atol=atol, rtol=rtol) + + def test_fp8_shared_expert(self): + for params in itertools.product( + self.M_fp8, + self.N_fp8, + self.K_fp8, + self.routed_scaling_factor, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + routed_scaling_factor=params[3], + ): + self._fp8_shared_expert(*params) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_topk.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_topk.py new file mode 100644 index 00000000..9f3dfc1b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/test_topk.py @@ -0,0 +1,196 @@ +import unittest + +import torch + +from sglang.srt.layers.moe.topk import ( + biased_grouped_topk_impl as native_biased_grouped_topk, +) +from sglang.srt.layers.moe.topk import fused_topk_torch_native as native_fused_topk +from sglang.srt.layers.moe.topk import grouped_topk_gpu as native_grouped_topk +from sglang.srt.models.llama4 import Llama4MoE +from sglang.test.test_utils import CustomTestCase + +torch.manual_seed(1234) + + +# This is used by the Deepseek-V2 model +class TestGroupedTopK(CustomTestCase): + def _run_single_test(self, M, E, G, topk, topk_group, renormalize, dtype): + torch.manual_seed(1234) + + # expand gating_output by M, otherwise bfloat16 fall into same value aftering truncating + hidden_states = torch.randn(M, 100, dtype=dtype) + gating_output = torch.randn(M, E, dtype=dtype) * 2 * M + + ref_topk_weights, ref_topk_ids = native_grouped_topk( + hidden_states.float(), + gating_output.float(), + topk, + renormalize, + G, + topk_group, + ) + + # fused version + topk_weights, topk_ids = torch.ops.sgl_kernel.grouped_topk_cpu( + hidden_states, + gating_output, + topk, + renormalize, + G, + topk_group, + 0, + None, + None, + ) + + res = torch.zeros(M, E, dtype=torch.float) + ref = torch.zeros(M, E, dtype=torch.float) + res.scatter_(1, topk_ids.long(), topk_weights) + ref.scatter_(1, ref_topk_ids.long(), ref_topk_weights) + torch.testing.assert_close(res, ref) + + def test_grouped_topk(self): + for renormalize in [True, False]: + self._run_single_test(123, 8, 2, 2, 1, renormalize, torch.bfloat16) + self._run_single_test(123, 16, 4, 3, 2, renormalize, torch.bfloat16) + self._run_single_test(123, 32, 4, 3, 2, renormalize, torch.bfloat16) + self._run_single_test(1123, 32, 4, 3, 2, renormalize, torch.bfloat16) + self._run_single_test(123, 64, 1, 6, 1, renormalize, torch.bfloat16) + self._run_single_test(123, 256, 8, 4, 8, renormalize, torch.bfloat16) + self._run_single_test(123, 160, 8, 6, 2, renormalize, torch.bfloat16) + + +# DeepSeek V2/V3/R1 uses biased_grouped_top +class TestBiasedGroupedTopK(CustomTestCase): + def _run_single_test( + self, M, E, G, topk, topk_group, renormalize, dtype, bias_dtype + ): + torch.manual_seed(1234) + + # expand gating_output by M, otherwise bfloat16 fall into same value aftering truncating + hidden_states = torch.randn(M, 100, dtype=dtype) + gating_output = torch.randn(M, E, dtype=dtype) * 2 * M + correction_bias = torch.randn(E, dtype=bias_dtype) + + ref_topk_weights, ref_topk_ids = native_biased_grouped_topk( + hidden_states.float(), + gating_output.float(), + correction_bias.float(), + topk, + renormalize, + G, + topk_group, + ) + + # fused version + topk_weights, topk_ids = torch.ops.sgl_kernel.biased_grouped_topk_cpu( + hidden_states, + gating_output, + correction_bias, + topk, + renormalize, + G, + topk_group, + 0, + None, + None, + ) + + res = torch.zeros(M, E, dtype=torch.float) + ref = torch.zeros(M, E, dtype=torch.float) + res.scatter_(1, topk_ids.long(), topk_weights) + ref.scatter_(1, ref_topk_ids.long(), ref_topk_weights) + torch.testing.assert_close(res, ref) + + def test_biased_grouped_topk(self): + for renormalize in [True, False]: + for bias_dtype in [torch.float32, torch.bfloat16]: + self._run_single_test( + 122, 256, 8, 8, 2, renormalize, torch.bfloat16, bias_dtype + ) + + +class TestTopK(CustomTestCase): + def _run_single_test(self, M, E, topk, renormalize, dtype): + torch.manual_seed(1998) + + # expand gating_output by M, otherwise bfloat16 fall into same value aftering truncating + hidden_states = torch.randn(M, 100, dtype=dtype) + gating_output = torch.randn(M, E, dtype=dtype) * 2 * M + + ref_topk_weights, ref_topk_ids = native_fused_topk( + hidden_states.float(), + gating_output.float(), + topk, + renormalize, + ) + + # fused version + topk_weights, topk_ids = torch.ops.sgl_kernel.topk_softmax_cpu( + hidden_states, gating_output, topk, renormalize + ) + + res = torch.zeros(M, E, dtype=torch.float) + ref = torch.zeros(M, E, dtype=torch.float) + res.scatter_(1, topk_ids.long(), topk_weights) + ref.scatter_(1, ref_topk_ids.long(), ref_topk_weights) + torch.testing.assert_close(res, ref) + + def test_topk(self): + for renormalize in [True, False]: + self._run_single_test(123, 8, 2, renormalize, torch.bfloat16) + self._run_single_test(123, 16, 3, renormalize, torch.bfloat16) + self._run_single_test(123, 32, 3, renormalize, torch.bfloat16) + self._run_single_test(123, 32, 3, renormalize, torch.bfloat16) + self._run_single_test(123, 64, 6, renormalize, torch.bfloat16) + self._run_single_test(123, 256, 4, renormalize, torch.bfloat16) + self._run_single_test(123, 160, 6, renormalize, torch.bfloat16) + + +class TestCustomTopK(CustomTestCase): + def _run_single_test( + self, M, E, topk, renormalize, dtype, native_custom_f, fused_custom_f + ): + torch.manual_seed(16) + + # expand gating_output by M, otherwise bfloat16 fall into same value aftering truncating + hidden_states = torch.randn(M, 100, dtype=dtype) + gating_output = torch.randn(M, E, dtype=dtype) * 2 * M + + ref_topk_weights, ref_topk_ids = native_custom_f( + hidden_states.float(), + gating_output.float(), + topk, + renormalize, + ) + + # fused version + topk_weights, topk_ids = fused_custom_f( + hidden_states, gating_output, topk, renormalize + ) + + res = torch.zeros(M, E, dtype=torch.float) + ref = torch.zeros(M, E, dtype=torch.float) + res.scatter_(1, topk_ids.long(), topk_weights) + ref.scatter_(1, ref_topk_ids.long(), ref_topk_weights) + torch.testing.assert_close(res, ref) + + def test_custom_topk(self): + test_custom_functions = [ + (Llama4MoE.custom_routing_function, torch.ops.sgl_kernel.topk_sigmoid_cpu) + ] + for native_custom_f, fused_custom_f in test_custom_functions: + self._run_single_test( + 123, 8, 1, False, torch.bfloat16, native_custom_f, fused_custom_f + ) + self._run_single_test( + 123, 16, 1, False, torch.bfloat16, native_custom_f, fused_custom_f + ) + self._run_single_test( + 123, 32, 1, False, torch.bfloat16, native_custom_f, fused_custom_f + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/utils.py new file mode 100644 index 00000000..6435dad7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/cpu/utils.py @@ -0,0 +1,274 @@ +import math + +import torch +import torch.nn.functional as F + +precision = { + torch.bfloat16: 1e-2, + torch.float16: 1e-3, + torch.float32: 1e-5, +} + + +BLOCK_N, BLOCK_K = 64, 128 +factor_for_scale = 1e-3 +fp8_max, fp8_min = 400, -400 + + +def SiluAndMul(x: torch.Tensor) -> torch.Tensor: + d = x.shape[-1] // 2 + return F.silu(x[..., :d]) * x[..., d:] + + +def GeluAndMul(x: torch.Tensor, approximate="tanh") -> torch.Tensor: + d = x.shape[-1] // 2 + return F.gelu(x[..., :d], approximate=approximate) * x[..., d:] + + +def per_token_quant_int8(x): + x = x.float() + absmax = x.abs().max(dim=-1).values + absmax = absmax.clamp_min(1e-10).unsqueeze(-1) + scale_x = absmax / 127 + x_q = x.mul(127 / absmax) + x_q = torch.round(x_q).to(torch.int8) + + return x_q, scale_x + + +def convert_weight(weight, scale_block_size, A_dtype): + N, K = weight.size() + fp8_max = 448.0 + scale_block_size_N, scale_block_size_K = scale_block_size # (128, 128) + + pad_N = (scale_block_size_N - (N % scale_block_size_N)) % scale_block_size_N + pad_K = (scale_block_size_K - (K % scale_block_size_K)) % scale_block_size_K + + if pad_N > 0 or pad_K > 0: + weight = torch.nn.functional.pad(weight, (0, pad_K, 0, pad_N)) + + weight_blocks = weight.view( + math.ceil(N / scale_block_size_N), + scale_block_size_N, + math.ceil(K / scale_block_size_K), + scale_block_size_K, + ) # (8, 128, 8, 128) + weight_blocks = weight_blocks.permute(0, 2, 1, 3).contiguous() # (8, 8, 128, 128) + + # Step 2: compute per-block max abs values → scale + abs_max = weight_blocks.abs().amax(dim=(-2, -1), keepdim=True) # (8, 8, 1, 1) + scales = abs_max / fp8_max + scales = torch.where( + scales == 0, torch.ones_like(scales), scales + ) # avoid division by zero + + q_fp8 = (weight_blocks / scales).to(torch.float8_e4m3fn) + q_fp8_reshape = q_fp8.permute(0, 2, 1, 3).contiguous() + + if pad_N > 0 or pad_K > 0: + q_fp8_reshape = q_fp8_reshape.view(N + pad_N, K + pad_K) + q_fp8_reshape = q_fp8_reshape[:N, :K].contiguous() + else: + q_fp8_reshape = q_fp8_reshape.view(N, K) + + dq_weight = q_fp8.float() * scales + dq_weight = dq_weight.permute(0, 2, 1, 3).contiguous() # (8, 128, 8, 128) + + if pad_N > 0 or pad_K > 0: + w_dq = dq_weight.view(N + pad_N, K + pad_K).to(A_dtype) + w_dq = w_dq[:N, :K].contiguous() + else: + w_dq = dq_weight.view(N, K).to(A_dtype) + + scales = scales.view( + math.ceil(N / scale_block_size_N), math.ceil(K / scale_block_size_K) + ) + + return q_fp8_reshape, scales, w_dq + + +def native_w8a8_per_token_matmul(A, B, As, Bs, bias, output_dtype=torch.bfloat16): + """Matrix multiplication function that supports per-token input quantization and per-column weight quantization""" + A = A.to(torch.float32) + B = B.to(torch.float32) + + assert A.shape[-1] == B.shape[-1], "Dimension mismatch" + assert B.ndim == 2 and B.is_contiguous(), "B must be a 2D contiguous tensor" + + # Reshape input + M = A.numel() // A.shape[-1] + B = B.t() # Transpose weight matrix + N, K = B.shape + origin_C_shape = A.shape[:-1] + (K,) + A = A.reshape(M, N) + + # As is per-token [M, 1], Bs is per-column [1, K] + C = torch.matmul(A, B) # [M, K] + C = As * C * Bs.view(1, -1) # Broadcast per-column scale + + if bias is not None: + C.add_(bias.view(1, -1)) + + return C.reshape(origin_C_shape).to(output_dtype) + + +def torch_naive_moe(a, w1, w2, b, routed_scaling_factor): + + ic1 = torch.matmul(a, w1.transpose(0, 1)) + ic2 = SiluAndMul(ic1) + ic3 = torch.matmul(ic2, w2.transpose(0, 1)) + + return ic3 + b * routed_scaling_factor + + +def torch_w8a8_per_column_moe(a, w1_q, w2_q, w1_s, w2_s, b, routed_scaling_factor): + + # Perform per-token quantization + a_q, a_s = per_token_quant_int8(a) + + ic1 = native_w8a8_per_token_matmul( + a_q, w1_q, a_s, w1_s, bias=None, output_dtype=torch.float32 + ) + ic2 = SiluAndMul(ic1) + + a1_q, a1_s = per_token_quant_int8(ic2) + ic3 = native_w8a8_per_token_matmul( + a1_q, w2_q, a1_s, w2_s, bias=None, output_dtype=torch.float32 + ) + + return ic3 + b * routed_scaling_factor + + +def scaled_weight(weight, scales): + E, N, K = weight.shape + pad_N = (BLOCK_N - (N % BLOCK_N)) % BLOCK_N + pad_K = (BLOCK_K - (K % BLOCK_K)) % BLOCK_K + + if pad_N > 0 or pad_K > 0: + weight = torch.nn.functional.pad(weight, (0, pad_K, 0, pad_N)) + + weight_block = ( + weight.view(E, math.ceil(N / BLOCK_N), BLOCK_N, math.ceil(K / BLOCK_K), BLOCK_K) + .permute(0, 1, 3, 2, 4) + .float() + .contiguous() + ) + + weight_scaled = ( + ( + weight_block + * scales.view(E, math.ceil(N / BLOCK_N), math.ceil(K / BLOCK_K), 1, 1) + ) + .permute(0, 1, 3, 2, 4) + .contiguous() + ) + if pad_N > 0 or pad_K > 0: + weight_scaled = weight_scaled.view(E, N + pad_N, K + pad_K) + weight_scaled = weight_scaled[..., :N, :K].contiguous() + else: + weight_scaled = weight_scaled.view(E, N, K) + return weight_scaled + + +def torch_naive_fused_moe(a, w1, w2, score, topk, renormalize): + B, D = a.shape + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + + if renormalize: + topk_weight = topk_weight / topk_weight.sum(dim=-1, keepdim=True) + + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + for i in range(w1.shape[0]): + mask = topk_ids == i + if mask.sum(): + out[mask] = SiluAndMul(a[mask] @ w1[i].transpose(0, 1)) @ w2[i].transpose( + 0, 1 + ) + return ( + out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype) + ).sum(dim=1) + + +def torch_w8a8_per_column_fused_moe(a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, topk): + """This function performs fused moe with per-column int8 quantization using native torch.""" + + B, D = a.shape + # Perform per-token quantization + a_q, a_s = per_token_quant_int8(a) + # Repeat tokens to match topk + a_q = a_q.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + # Also repeat the scale + a_s = a_s.view(B, -1, 1).repeat(1, topk, 1).reshape(-1, 1) # [B*topk, 1] + + out = torch.zeros(B * topk, w2.shape[1], dtype=torch.float32, device=a.device) + + # Calculate routing + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + # Process each expert + for i in range(w1.shape[0]): + mask = topk_ids == i + if mask.sum(): + # First MLP layer: note that a_s is now per-token + inter_out = native_w8a8_per_token_matmul( + a_q[mask], + w1[i], + a_s[mask], + w1_s[i], + bias=None, + output_dtype=torch.float32, + ) + # Activation function + act_out = SiluAndMul(inter_out) + # Quantize activation output with per-token + act_out_q, act_out_s = per_token_quant_int8(act_out) + # Second MLP layer + out[mask] = native_w8a8_per_token_matmul( + act_out_q, + w2[i], + act_out_s, + w2_s[i], + bias=None, + output_dtype=torch.float32, + ) + # Apply routing weights and sum + return ( + (out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype)) + .sum(dim=1) + .to(a.dtype) + ) + + +def native_fp8_fused_moe(a, w1, w2, topk_weight, topk_ids, topk): + B, D = a.shape + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D).float() + out = torch.zeros(B * topk, w2.shape[1], dtype=torch.float32, device=a.device) + + # Calculate routing + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + + for i in range(w1.shape[0]): + mask = topk_ids == i + if mask.sum(): + ic0 = torch.matmul(a[mask], w1[i].transpose(0, 1)) + ic1 = SiluAndMul(ic0) + out[mask] = torch.matmul(ic1, w2[i].transpose(0, 1)) + + return ( + (out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype)) + .sum(dim=1) + .to(a.dtype) + ) + + +def make_non_contiguous(x: torch.Tensor) -> torch.Tensor: + """ + Make a tensor non-contiguous by slicing it via last dimension. + """ + last_dim = x.shape[-1] + return x[..., : last_dim // 2] if x.is_contiguous() else x diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/debug_utils/test_tensor_dump_forward_hook.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/debug_utils/test_tensor_dump_forward_hook.py new file mode 100644 index 00000000..c691f0f1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/debug_utils/test_tensor_dump_forward_hook.py @@ -0,0 +1,95 @@ +import unittest + +import torch +from torch import nn + +from sglang.srt.debug_utils.tensor_dump_forward_hook import ( + register_forward_hook_for_model, +) +from sglang.srt.distributed.parallel_state import ( + init_distributed_environment, + initialize_model_parallel, +) +from sglang.srt.layers.layernorm import RMSNorm +from sglang.srt.layers.linear import LinearBase +from sglang.srt.models.qwen2 import Qwen2MLP +from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler +from sglang.srt.utils import add_prefix + +TEST_HIDDEN_SIZE = 32 + + +class SimpleModel(nn.Module): + + def __init__(self) -> None: + super().__init__() + self.hidden_size = TEST_HIDDEN_SIZE + self.rms_norm_eps = 1e-5 + self.mlp = Qwen2MLP( + hidden_size=self.hidden_size, + intermediate_size=self.hidden_size, + hidden_act="silu", + quant_config=None, + prefix=add_prefix("mlp", ""), + ) + self.layernorm = RMSNorm(self.hidden_size, eps=self.rms_norm_eps) + + @torch.no_grad() + def forward( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + hidden_states = self.layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + return hidden_states + + +class MockCausalLM(nn.Module): + def __init__(self) -> None: + super().__init__() + self.model = SimpleModel() + + @torch.no_grad() + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + return self.model(hidden_states) + + +def init_weights(module): + if isinstance(module, LinearBase): + torch.nn.init.uniform_(module.weight) + if module.bias is not None: + torch.nn.init.zeros_(module.bias) + elif isinstance(module, RMSNorm): + torch.nn.init.ones_(module.weight) + + +def test_model_forward_dump(tmp_path): + set_global_server_args_for_scheduler(ServerArgs(model_path="dummy")) + init_distributed_environment( + backend="nccl", + world_size=1, + rank=0, + local_rank=0, + distributed_init_method="tcp://127.0.0.1:2646", + ) + initialize_model_parallel() + model = MockCausalLM() + model.apply(init_weights) + model = model.cuda().bfloat16() + dumper = register_forward_hook_for_model( + model, tmp_path / "sglang_dump", [0], 0, 0, 0 + ) + + dir_path = dumper.get_dump_dir() + inp = torch.randn(4, TEST_HIDDEN_SIZE, dtype=torch.bfloat16) * 0.01 + result = model(inp.cuda()) + data = torch.load(f"{dir_path}/Pass00000.pt") + assert "model.layernorm" in data + assert "model.mlp.down_proj" in data + assert torch.allclose( + data["model.mlp.down_proj"], result.cpu(), rtol=1e-5, atol=1e-5 + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/double-sparsity-config-Llama-3.1-8B-Instruct.json b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/double-sparsity-config-Llama-3.1-8B-Instruct.json new file mode 100644 index 00000000..f1652ec9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/double-sparsity-config-Llama-3.1-8B-Instruct.json @@ -0,0 +1 @@ +{"model.layers.0.self_attn.q_proj": [[39, 106, 104, 102, 33, 95, 13, 44, 11, 29, 12, 10, 53, 126, 27, 114, 121, 8, 124, 113, 112, 15, 23, 89, 69, 111, 54, 80, 4, 1, 20, 24, 83, 63, 115, 122, 66, 42, 22, 110, 3, 73, 21, 61, 97, 19, 25, 88, 117, 119, 116, 85, 70, 5, 56, 118, 68, 123, 2, 86, 71, 127, 93, 49, 109, 50, 67, 52, 91, 40, 17, 108, 60, 55, 78, 62, 65, 47, 6, 87, 51, 84, 58, 82, 94, 79, 57, 103, 76, 48, 0, 18, 81, 96, 9, 31, 16, 92, 26, 43, 30, 37, 74, 100, 46, 38, 32, 14, 90, 36, 101, 75, 35, 125, 45, 7, 72, 41, 77, 105, 28, 59, 34, 99, 98, 107, 120, 64], [39, 104, 102, 106, 33, 44, 95, 53, 126, 111, 114, 110, 29, 14, 113, 54, 121, 112, 109, 80, 119, 12, 63, 25, 78, 123, 42, 10, 24, 127, 108, 115, 122, 60, 116, 97, 9, 93, 61, 17, 117, 56, 82, 91, 21, 124, 62, 0, 43, 19, 31, 26, 49, 52, 57, 59, 1, 50, 41, 15, 46, 86, 88, 81, 118, 125, 2, 3, 55, 105, 84, 75, 120, 70, 47, 85, 32, 51, 37, 16, 22, 23, 89, 87, 67, 30, 68, 69, 77, 48, 66, 27, 36, 107, 20, 76, 94, 79, 96, 83, 99, 38, 100, 18, 8, 35, 28, 101, 13, 72, 92, 7, 4, 98, 90, 103, 6, 34, 74, 11, 65, 40, 71, 45, 58, 64, 73, 5], [106, 104, 39, 102, 2, 44, 95, 53, 121, 3, 4, 33, 1, 0, 122, 111, 126, 115, 63, 71, 119, 112, 113, 70, 109, 29, 61, 52, 114, 54, 127, 21, 10, 72, 5, 110, 116, 49, 123, 11, 12, 55, 24, 45, 15, 19, 117, 73, 80, 50, 6, 23, 64, 124, 75, 69, 25, 83, 14, 77, 97, 42, 78, 13, 79, 85, 27, 56, 87, 86, 57, 17, 118, 91, 81, 74, 65, 76, 9, 41, 88, 18, 51, 22, 62, 93, 89, 67, 105, 84, 82, 38, 48, 120, 60, 47, 26, 20, 43, 68, 28, 66, 108, 59, 58, 8, 125, 98, 46, 31, 90, 36, 94, 92, 34, 107, 35, 7, 99, 100, 30, 32, 37, 101, 16, 96, 40, 103], [39, 104, 106, 102, 33, 44, 53, 111, 113, 95, 54, 109, 41, 116, 124, 63, 121, 122, 61, 56, 114, 110, 60, 108, 127, 112, 125, 43, 52, 126, 12, 80, 119, 59, 62, 82, 123, 97, 117, 25, 37, 0, 88, 29, 57, 120, 26, 91, 115, 78, 24, 21, 118, 49, 93, 1, 42, 14, 58, 107, 10, 84, 67, 7, 9, 86, 87, 3, 50, 51, 64, 98, 18, 20, 72, 66, 100, 17, 32, 81, 69, 79, 2, 101, 105, 85, 15, 77, 31, 22, 75, 70, 68, 90, 55, 19, 89, 11, 74, 76, 8, 38, 30, 4, 65, 5, 45, 92, 23, 46, 73, 13, 94, 47, 35, 36, 6, 48, 34, 28, 99, 27, 96, 103, 71, 83, 16, 40], [41, 103, 38, 33, 42, 93, 109, 95, 88, 44, 107, 17, 25, 20, 110, 21, 47, 40, 78, 116, 115, 126, 59, 61, 64, 57, 48, 83, 119, 12, 45, 125, 11, 18, 124, 118, 53, 56, 89, 70, 22, 77, 86, 50, 92, 4, 87, 24, 97, 65, 49, 73, 26, 62, 3, 16, 108, 66, 72, 104, 67, 80, 58, 71, 79, 23, 91, 122, 19, 43, 113, 39, 14, 102, 111, 27, 9, 121, 10, 117, 29, 60, 69, 90, 76, 120, 0, 84, 15, 114, 51, 74, 68, 98, 6, 81, 85, 2, 1, 82, 100, 112, 99, 13, 52, 123, 94, 8, 35, 28, 55, 96, 63, 54, 7, 105, 5, 36, 127, 37, 32, 75, 34, 106, 31, 30, 101, 46], [38, 103, 44, 41, 107, 33, 42, 88, 95, 40, 93, 87, 25, 78, 110, 47, 48, 26, 20, 18, 21, 17, 64, 59, 45, 22, 109, 89, 118, 62, 91, 115, 116, 53, 49, 61, 126, 90, 16, 11, 119, 92, 111, 65, 125, 124, 43, 85, 83, 60, 24, 4, 77, 73, 104, 117, 57, 120, 86, 3, 76, 97, 29, 23, 102, 39, 51, 67, 70, 12, 122, 71, 9, 15, 30, 113, 56, 66, 0, 106, 7, 50, 108, 100, 114, 28, 46, 121, 80, 94, 72, 10, 6, 79, 69, 55, 19, 58, 74, 68, 123, 81, 27, 13, 2, 82, 98, 35, 8, 34, 36, 1, 101, 5, 105, 54, 63, 96, 37, 84, 127, 52, 32, 31, 14, 75, 112, 99], [103, 41, 107, 33, 38, 93, 110, 116, 64, 42, 48, 20, 123, 61, 124, 17, 126, 115, 88, 49, 25, 53, 47, 78, 57, 122, 21, 59, 95, 18, 44, 89, 77, 97, 16, 11, 125, 66, 86, 120, 119, 109, 4, 65, 12, 118, 71, 62, 22, 24, 80, 14, 50, 74, 73, 51, 70, 3, 7, 121, 67, 83, 27, 111, 72, 5, 69, 76, 23, 45, 84, 29, 60, 9, 31, 10, 81, 26, 8, 87, 19, 13, 0, 40, 2, 56, 113, 85, 82, 127, 91, 43, 34, 108, 117, 114, 6, 15, 96, 106, 79, 75, 94, 102, 100, 101, 68, 98, 63, 58, 99, 105, 55, 30, 90, 36, 1, 35, 32, 46, 112, 28, 104, 39, 37, 92, 52, 54], [103, 41, 33, 38, 42, 93, 95, 107, 110, 25, 59, 17, 48, 116, 88, 109, 44, 57, 11, 83, 20, 125, 78, 21, 62, 86, 115, 70, 97, 77, 4, 118, 18, 47, 43, 64, 12, 24, 50, 126, 22, 124, 45, 3, 61, 72, 65, 39, 89, 84, 113, 53, 73, 29, 49, 120, 66, 80, 79, 27, 71, 122, 10, 117, 40, 9, 92, 16, 67, 81, 19, 23, 119, 123, 51, 105, 36, 8, 15, 55, 108, 111, 60, 90, 14, 26, 87, 104, 74, 56, 91, 75, 0, 28, 69, 68, 100, 2, 102, 99, 58, 52, 85, 6, 63, 34, 54, 94, 127, 96, 76, 32, 121, 30, 1, 31, 7, 114, 82, 46, 37, 13, 112, 35, 5, 101, 98, 106], [102, 41, 112, 101, 108, 103, 33, 104, 29, 27, 115, 30, 42, 38, 16, 45, 114, 20, 32, 100, 121, 89, 98, 111, 87, 62, 59, 126, 13, 63, 107, 11, 43, 88, 117, 75, 118, 25, 84, 34, 31, 50, 51, 55, 23, 95, 56, 58, 53, 78, 94, 97, 82, 92, 8, 18, 127, 113, 110, 19, 40, 21, 35, 39, 85, 119, 37, 125, 96, 54, 6, 72, 90, 91, 14, 26, 124, 105, 9, 44, 28, 17, 116, 120, 60, 48, 80, 57, 1, 123, 79, 77, 86, 24, 69, 73, 15, 122, 68, 99, 81, 74, 4, 52, 71, 67, 7, 65, 76, 66, 61, 109, 106, 46, 93, 2, 64, 12, 10, 49, 3, 47, 83, 70, 36, 22, 0, 5], [107, 104, 102, 33, 108, 45, 95, 92, 29, 42, 103, 21, 99, 43, 85, 101, 23, 50, 27, 28, 113, 98, 19, 15, 120, 83, 26, 46, 74, 97, 34, 12, 86, 123, 52, 127, 90, 79, 39, 7, 17, 41, 112, 49, 126, 77, 48, 38, 125, 89, 44, 61, 111, 18, 100, 115, 32, 25, 35, 119, 93, 62, 30, 60, 84, 96, 31, 80, 122, 87, 114, 53, 55, 47, 56, 110, 13, 57, 63, 37, 51, 54, 116, 117, 68, 124, 105, 78, 40, 81, 118, 88, 9, 91, 11, 109, 76, 24, 59, 71, 121, 82, 6, 20, 1, 72, 106, 16, 58, 73, 5, 94, 67, 10, 69, 14, 4, 22, 75, 2, 36, 70, 8, 66, 3, 65, 64, 0], [102, 103, 33, 29, 41, 42, 104, 45, 64, 1, 107, 108, 66, 27, 16, 112, 88, 115, 3, 50, 101, 20, 127, 67, 118, 75, 58, 116, 53, 72, 13, 84, 68, 19, 86, 25, 89, 70, 11, 30, 21, 77, 117, 4, 6, 14, 56, 59, 69, 82, 80, 100, 126, 95, 32, 92, 97, 39, 85, 124, 79, 90, 121, 52, 54, 26, 5, 113, 71, 7, 123, 73, 51, 8, 0, 63, 111, 38, 23, 57, 15, 119, 10, 43, 60, 78, 22, 83, 87, 9, 74, 37, 99, 98, 76, 65, 61, 55, 17, 81, 18, 31, 24, 94, 28, 2, 47, 44, 40, 110, 114, 96, 105, 46, 109, 120, 34, 48, 12, 35, 62, 93, 106, 49, 91, 122, 125, 36], [103, 45, 102, 33, 26, 42, 95, 87, 79, 123, 104, 19, 113, 112, 41, 29, 21, 76, 108, 120, 30, 71, 10, 97, 81, 27, 101, 74, 44, 32, 115, 122, 89, 17, 12, 23, 7, 92, 25, 53, 15, 100, 90, 117, 13, 39, 105, 99, 83, 51, 85, 84, 50, 4, 20, 109, 61, 88, 35, 16, 98, 54, 94, 110, 125, 55, 66, 127, 70, 47, 65, 124, 116, 46, 80, 67, 114, 49, 57, 56, 78, 38, 86, 18, 60, 82, 11, 5, 121, 119, 126, 62, 48, 58, 59, 28, 111, 93, 37, 63, 2, 73, 52, 6, 8, 24, 3, 96, 106, 43, 40, 107, 1, 34, 118, 22, 0, 68, 69, 77, 91, 64, 72, 36, 75, 31, 14, 9], [104, 103, 33, 43, 44, 111, 102, 90, 49, 89, 8, 41, 93, 37, 116, 113, 95, 122, 123, 23, 74, 42, 14, 17, 83, 68, 85, 106, 91, 92, 13, 78, 63, 127, 115, 81, 112, 87, 121, 3, 70, 48, 72, 120, 28, 124, 53, 71, 30, 19, 47, 86, 59, 51, 96, 56, 94, 114, 108, 118, 76, 79, 55, 38, 29, 110, 5, 82, 15, 21, 11, 20, 26, 66, 6, 84, 22, 50, 9, 27, 100, 65, 61, 64, 57, 25, 35, 7, 16, 101, 80, 45, 52, 36, 46, 109, 107, 24, 126, 2, 4, 12, 67, 125, 75, 10, 32, 117, 97, 1, 69, 60, 77, 88, 98, 99, 73, 105, 58, 18, 62, 54, 34, 31, 0, 119, 40, 39], [103, 104, 102, 33, 43, 44, 111, 90, 115, 106, 113, 89, 42, 74, 56, 93, 48, 95, 85, 127, 41, 72, 92, 37, 107, 71, 17, 87, 123, 112, 122, 50, 83, 22, 55, 57, 116, 14, 53, 124, 49, 114, 26, 47, 110, 23, 63, 70, 98, 100, 51, 108, 121, 59, 13, 35, 28, 81, 94, 79, 32, 120, 29, 76, 101, 105, 126, 38, 7, 82, 9, 6, 118, 24, 125, 91, 20, 88, 15, 99, 36, 109, 18, 62, 34, 46, 84, 19, 3, 61, 2, 30, 11, 27, 86, 52, 58, 75, 25, 68, 12, 31, 117, 69, 54, 96, 119, 65, 78, 80, 8, 45, 16, 77, 40, 1, 21, 73, 10, 4, 66, 64, 60, 67, 0, 39, 97, 5], [104, 44, 103, 43, 33, 49, 111, 102, 95, 41, 37, 123, 27, 91, 9, 84, 42, 89, 61, 122, 24, 76, 124, 116, 30, 85, 79, 96, 51, 48, 118, 28, 87, 92, 98, 127, 59, 23, 109, 121, 20, 15, 88, 29, 100, 106, 53, 34, 35, 101, 126, 16, 120, 54, 108, 115, 47, 93, 83, 17, 56, 18, 90, 113, 55, 38, 22, 63, 8, 81, 13, 94, 72, 117, 60, 50, 114, 36, 12, 71, 45, 57, 105, 32, 110, 119, 62, 74, 58, 97, 31, 112, 125, 52, 19, 46, 86, 5, 80, 68, 73, 3, 78, 99, 107, 6, 70, 75, 40, 14, 65, 21, 39, 7, 25, 77, 64, 2, 11, 82, 26, 10, 69, 4, 0, 66, 1, 67], [102, 33, 44, 104, 43, 103, 79, 28, 27, 87, 95, 9, 12, 124, 49, 21, 23, 99, 76, 90, 85, 35, 32, 31, 29, 80, 94, 61, 98, 114, 84, 91, 18, 15, 73, 89, 81, 106, 112, 62, 51, 41, 54, 56, 58, 59, 117, 45, 119, 88, 26, 110, 105, 52, 115, 42, 111, 24, 30, 92, 109, 38, 123, 37, 108, 113, 57, 118, 60, 125, 82, 25, 86, 75, 96, 19, 16, 14, 40, 46, 77, 20, 101, 93, 122, 126, 36, 53, 100, 55, 47, 71, 39, 11, 72, 34, 50, 63, 116, 48, 107, 70, 121, 13, 120, 17, 127, 78, 83, 3, 22, 7, 6, 5, 8, 66, 74, 97, 69, 10, 68, 67, 64, 4, 65, 0, 1, 2], [39, 41, 101, 97, 29, 110, 24, 43, 32, 49, 121, 84, 120, 63, 3, 56, 44, 8, 59, 21, 38, 98, 40, 6, 60, 127, 53, 71, 2, 25, 52, 116, 99, 13, 62, 4, 125, 72, 46, 18, 23, 7, 31, 68, 85, 15, 92, 42, 79, 76, 115, 123, 87, 75, 109, 57, 108, 95, 102, 12, 54, 0, 36, 66, 61, 19, 50, 124, 94, 65, 111, 82, 69, 126, 77, 80, 83, 106, 113, 51, 33, 100, 114, 91, 14, 20, 78, 10, 47, 27, 117, 86, 48, 118, 67, 89, 11, 112, 58, 16, 35, 88, 30, 45, 28, 5, 17, 70, 93, 73, 90, 34, 105, 22, 26, 55, 64, 119, 107, 9, 122, 104, 103, 96, 81, 1, 74, 37], [39, 41, 97, 32, 101, 43, 29, 26, 116, 31, 54, 80, 99, 55, 125, 18, 21, 59, 24, 78, 73, 111, 23, 76, 71, 16, 87, 82, 119, 3, 25, 28, 11, 121, 112, 12, 123, 63, 98, 120, 85, 83, 47, 91, 94, 56, 13, 103, 7, 33, 17, 88, 51, 127, 4, 95, 104, 19, 38, 108, 60, 86, 100, 79, 124, 77, 30, 27, 109, 81, 52, 50, 84, 61, 67, 37, 93, 49, 92, 5, 10, 118, 46, 58, 69, 74, 110, 14, 53, 45, 122, 126, 57, 113, 114, 15, 42, 107, 68, 22, 72, 9, 62, 90, 89, 70, 115, 48, 102, 8, 75, 36, 40, 6, 117, 105, 2, 0, 96, 20, 106, 44, 35, 65, 66, 1, 34, 64], [6, 39, 41, 70, 101, 72, 8, 43, 74, 60, 108, 53, 92, 110, 98, 113, 50, 32, 44, 47, 61, 87, 62, 95, 28, 111, 26, 38, 58, 57, 67, 124, 18, 52, 56, 51, 99, 97, 112, 123, 118, 125, 116, 59, 109, 33, 40, 89, 120, 106, 19, 121, 30, 46, 14, 31, 68, 126, 88, 11, 4, 104, 24, 45, 2, 63, 127, 10, 119, 105, 80, 90, 107, 83, 34, 114, 36, 78, 3, 27, 69, 42, 55, 9, 20, 73, 122, 5, 15, 86, 25, 49, 48, 29, 21, 94, 66, 100, 22, 0, 54, 65, 75, 84, 17, 96, 93, 91, 35, 77, 12, 71, 115, 102, 103, 79, 7, 23, 81, 117, 37, 85, 16, 13, 64, 76, 1, 82], [41, 39, 70, 101, 10, 32, 19, 43, 87, 97, 11, 83, 46, 14, 60, 4, 123, 15, 125, 38, 44, 108, 56, 86, 16, 9, 118, 53, 18, 5, 109, 2, 92, 76, 78, 62, 61, 23, 120, 127, 24, 68, 49, 59, 42, 66, 84, 75, 21, 110, 8, 27, 106, 72, 63, 111, 47, 52, 121, 50, 73, 126, 28, 40, 17, 99, 67, 107, 79, 122, 89, 82, 33, 124, 105, 77, 58, 26, 1, 85, 25, 114, 119, 51, 22, 57, 98, 48, 31, 71, 94, 103, 115, 116, 13, 7, 102, 74, 113, 3, 20, 96, 6, 0, 81, 90, 69, 45, 104, 29, 34, 100, 12, 36, 91, 88, 80, 64, 54, 95, 30, 55, 35, 65, 93, 37, 112, 117], [40, 102, 31, 107, 22, 39, 78, 98, 9, 6, 73, 88, 12, 81, 17, 67, 76, 106, 25, 70, 48, 85, 86, 110, 91, 92, 114, 74, 82, 2, 58, 15, 83, 59, 14, 27, 108, 117, 127, 66, 79, 111, 47, 30, 51, 50, 5, 21, 57, 52, 109, 36, 89, 23, 35, 49, 101, 63, 72, 90, 122, 54, 100, 26, 84, 56, 124, 33, 18, 55, 62, 126, 29, 123, 11, 95, 119, 20, 96, 28, 77, 19, 24, 4, 87, 32, 115, 37, 99, 41, 53, 97, 125, 112, 8, 104, 71, 43, 93, 3, 116, 65, 16, 10, 64, 75, 94, 113, 120, 105, 68, 60, 46, 103, 7, 61, 69, 34, 13, 44, 0, 118, 80, 45, 121, 1, 42, 38], [40, 73, 70, 14, 39, 17, 110, 102, 114, 85, 82, 91, 51, 106, 99, 49, 48, 41, 98, 83, 36, 75, 115, 46, 88, 119, 92, 30, 69, 57, 28, 32, 15, 109, 126, 63, 86, 31, 123, 84, 6, 37, 72, 107, 90, 76, 77, 53, 71, 60, 22, 23, 124, 55, 93, 100, 21, 33, 58, 111, 81, 8, 24, 18, 125, 19, 44, 112, 79, 113, 59, 2, 74, 65, 62, 97, 35, 0, 61, 45, 78, 89, 101, 5, 4, 66, 67, 50, 42, 122, 11, 117, 26, 16, 108, 56, 3, 10, 9, 121, 25, 116, 94, 1, 105, 118, 12, 47, 120, 20, 96, 43, 29, 87, 64, 52, 13, 104, 80, 127, 7, 38, 54, 95, 27, 34, 68, 103], [40, 99, 41, 107, 9, 66, 96, 37, 91, 36, 73, 24, 58, 111, 94, 31, 47, 86, 114, 50, 92, 89, 10, 52, 29, 59, 34, 6, 39, 18, 127, 44, 100, 22, 76, 21, 56, 122, 121, 57, 102, 79, 70, 12, 46, 84, 81, 85, 63, 116, 72, 88, 43, 110, 5, 83, 87, 45, 69, 27, 3, 2, 8, 78, 51, 115, 106, 67, 74, 7, 54, 118, 19, 42, 104, 124, 11, 97, 4, 75, 105, 35, 33, 62, 55, 28, 30, 0, 82, 77, 95, 93, 120, 98, 20, 25, 117, 32, 53, 17, 90, 26, 71, 108, 64, 14, 23, 101, 48, 49, 65, 15, 119, 125, 103, 80, 68, 112, 1, 126, 13, 16, 113, 123, 61, 109, 38, 60], [40, 41, 102, 37, 92, 32, 90, 88, 96, 107, 23, 31, 74, 39, 84, 97, 25, 75, 82, 35, 79, 10, 91, 15, 7, 4, 18, 116, 20, 98, 87, 99, 69, 33, 86, 111, 100, 122, 121, 127, 65, 58, 13, 8, 56, 47, 95, 9, 50, 114, 81, 110, 59, 118, 72, 48, 77, 17, 29, 24, 21, 52, 104, 22, 54, 28, 94, 83, 85, 51, 57, 14, 73, 36, 16, 19, 66, 76, 71, 11, 93, 78, 120, 27, 6, 108, 5, 117, 43, 49, 63, 3, 67, 2, 61, 26, 64, 70, 68, 0, 101, 105, 124, 62, 55, 30, 80, 89, 103, 34, 44, 113, 45, 60, 1, 38, 106, 42, 115, 109, 12, 125, 119, 126, 53, 112, 123, 46], [103, 105, 96, 47, 59, 45, 42, 30, 19, 93, 14, 112, 113, 32, 86, 0, 127, 104, 38, 21, 84, 57, 48, 89, 72, 124, 87, 81, 51, 80, 55, 44, 25, 63, 126, 94, 118, 123, 34, 52, 68, 26, 40, 7, 115, 27, 3, 33, 88, 11, 120, 8, 125, 1, 23, 66, 70, 110, 77, 5, 107, 39, 50, 73, 58, 46, 13, 119, 6, 91, 9, 69, 15, 17, 76, 74, 43, 54, 24, 79, 95, 75, 22, 82, 78, 90, 12, 121, 92, 16, 71, 31, 114, 4, 117, 116, 18, 20, 67, 61, 53, 97, 37, 28, 41, 49, 35, 60, 36, 122, 99, 10, 85, 2, 109, 100, 101, 108, 83, 102, 56, 111, 65, 62, 98, 29, 64, 106], [104, 42, 103, 96, 45, 34, 47, 93, 32, 89, 27, 30, 84, 94, 26, 21, 81, 112, 116, 87, 105, 19, 51, 102, 86, 25, 38, 88, 80, 15, 41, 23, 119, 33, 20, 118, 52, 111, 114, 113, 44, 110, 85, 57, 98, 125, 124, 13, 48, 54, 11, 107, 43, 73, 55, 37, 106, 12, 100, 59, 10, 76, 24, 127, 77, 82, 115, 60, 14, 72, 101, 68, 90, 121, 29, 91, 0, 63, 97, 58, 75, 9, 61, 46, 126, 108, 4, 39, 56, 18, 22, 28, 70, 117, 120, 53, 109, 66, 6, 123, 3, 1, 95, 92, 49, 31, 50, 122, 99, 40, 69, 5, 62, 67, 83, 79, 16, 36, 64, 78, 2, 8, 74, 7, 35, 65, 71, 17], [103, 104, 96, 42, 45, 105, 30, 19, 93, 59, 47, 89, 113, 14, 86, 112, 84, 21, 32, 124, 80, 87, 57, 81, 44, 0, 72, 127, 88, 115, 25, 26, 48, 63, 119, 123, 51, 68, 118, 34, 52, 15, 55, 98, 126, 61, 43, 73, 24, 110, 102, 46, 107, 125, 121, 23, 116, 114, 41, 56, 54, 66, 27, 8, 3, 70, 91, 76, 6, 11, 39, 13, 1, 33, 97, 74, 4, 94, 22, 77, 58, 7, 37, 95, 82, 50, 36, 17, 117, 120, 53, 122, 75, 16, 90, 79, 67, 111, 40, 5, 64, 9, 2, 78, 18, 38, 31, 71, 60, 65, 12, 100, 108, 92, 85, 69, 29, 49, 99, 101, 62, 20, 109, 28, 35, 10, 106, 83], [42, 104, 103, 96, 45, 30, 89, 112, 34, 27, 113, 32, 102, 47, 93, 26, 105, 84, 94, 21, 81, 86, 88, 87, 124, 25, 59, 80, 44, 15, 19, 38, 114, 20, 57, 85, 52, 51, 23, 118, 127, 125, 119, 43, 100, 29, 33, 41, 46, 116, 82, 13, 126, 48, 11, 107, 77, 73, 98, 72, 122, 106, 54, 37, 95, 55, 24, 76, 12, 115, 10, 14, 97, 39, 110, 68, 101, 123, 56, 63, 92, 4, 0, 6, 91, 53, 90, 121, 50, 40, 61, 111, 22, 75, 9, 18, 16, 70, 117, 31, 60, 3, 58, 99, 66, 49, 74, 67, 1, 108, 109, 79, 2, 120, 35, 62, 64, 36, 28, 7, 83, 8, 69, 65, 5, 78, 71, 17], [100, 39, 31, 50, 91, 25, 47, 97, 96, 42, 116, 40, 90, 60, 41, 46, 19, 70, 44, 89, 22, 109, 48, 33, 95, 77, 27, 107, 2, 43, 101, 0, 18, 17, 108, 80, 49, 13, 53, 7, 10, 103, 64, 45, 21, 94, 12, 74, 79, 76, 118, 115, 124, 5, 73, 120, 35, 56, 99, 28, 55, 81, 54, 110, 14, 114, 126, 6, 68, 125, 127, 123, 93, 112, 113, 38, 58, 57, 61, 122, 32, 121, 75, 62, 84, 88, 119, 111, 59, 106, 3, 63, 72, 52, 30, 36, 26, 51, 86, 34, 98, 102, 29, 117, 16, 9, 8, 67, 83, 105, 11, 71, 65, 15, 4, 69, 1, 24, 104, 87, 66, 20, 82, 37, 92, 78, 23, 85], [40, 25, 64, 101, 42, 41, 45, 110, 72, 113, 56, 112, 108, 68, 107, 111, 114, 39, 2, 61, 125, 55, 119, 51, 97, 63, 31, 121, 22, 57, 54, 126, 59, 74, 123, 80, 87, 96, 52, 62, 18, 120, 7, 48, 0, 84, 58, 28, 91, 70, 115, 93, 11, 5, 14, 49, 60, 17, 30, 76, 66, 44, 19, 24, 88, 77, 3, 100, 13, 50, 1, 79, 109, 69, 90, 95, 99, 102, 104, 33, 53, 71, 21, 73, 8, 26, 106, 65, 75, 116, 117, 46, 98, 105, 89, 92, 67, 47, 35, 34, 81, 12, 4, 6, 122, 36, 78, 85, 10, 94, 43, 82, 83, 9, 118, 86, 103, 15, 127, 37, 16, 38, 27, 124, 23, 20, 32, 29], [0, 39, 22, 42, 96, 41, 91, 18, 10, 107, 79, 40, 14, 21, 19, 50, 77, 71, 80, 88, 70, 84, 48, 5, 45, 85, 44, 17, 11, 81, 100, 46, 28, 60, 76, 83, 72, 110, 66, 63, 57, 36, 47, 109, 2, 108, 56, 125, 120, 35, 111, 93, 61, 73, 114, 24, 112, 49, 55, 113, 87, 90, 121, 30, 64, 4, 99, 126, 116, 15, 51, 103, 52, 54, 6, 82, 97, 43, 115, 86, 119, 123, 58, 7, 101, 117, 69, 53, 13, 33, 59, 9, 62, 32, 68, 75, 31, 8, 3, 1, 12, 92, 29, 20, 78, 65, 104, 37, 27, 106, 122, 74, 16, 23, 34, 95, 25, 118, 26, 89, 94, 67, 98, 38, 105, 102, 127, 124], [108, 54, 112, 119, 59, 62, 113, 110, 123, 111, 26, 51, 61, 107, 55, 125, 89, 115, 114, 45, 121, 98, 58, 57, 63, 48, 52, 49, 34, 109, 122, 41, 126, 43, 127, 38, 32, 95, 56, 94, 100, 33, 120, 53, 50, 24, 124, 29, 40, 101, 44, 90, 25, 85, 8, 102, 60, 97, 88, 86, 83, 27, 42, 118, 46, 99, 66, 105, 4, 12, 19, 10, 47, 116, 96, 13, 82, 22, 16, 92, 30, 64, 31, 37, 117, 72, 28, 91, 87, 106, 69, 2, 103, 17, 9, 36, 70, 5, 84, 65, 76, 104, 75, 11, 39, 0, 78, 73, 20, 77, 23, 3, 6, 81, 68, 15, 74, 79, 80, 21, 35, 7, 93, 18, 71, 14, 67, 1]], "model.layers.0.self_attn.k_proj": [[0, 97, 42, 47, 103, 86, 93, 1, 25, 38, 88, 49, 40, 108, 65, 91, 117, 48, 57, 66, 78, 46, 68, 9, 64, 50, 52, 19, 45, 31, 118, 2, 55, 51, 70, 67, 85, 61, 77, 127, 82, 17, 63, 120, 80, 126, 87, 6, 75, 124, 72, 123, 3, 54, 62, 122, 14, 121, 7, 69, 53, 5, 56, 12, 10, 84, 41, 119, 115, 81, 34, 4, 76, 101, 107, 105, 60, 26, 74, 23, 79, 89, 15, 73, 24, 83, 22, 21, 18, 58, 43, 28, 33, 116, 29, 35, 27, 11, 112, 32, 30, 92, 16, 37, 104, 98, 59, 99, 94, 39, 90, 114, 13, 96, 36, 113, 100, 109, 95, 125, 111, 20, 106, 71, 8, 110, 44, 102], [39, 97, 105, 102, 112, 46, 65, 43, 29, 113, 64, 51, 77, 68, 111, 25, 86, 125, 18, 67, 106, 70, 9, 80, 17, 4, 59, 11, 116, 31, 124, 24, 126, 12, 88, 3, 108, 78, 52, 71, 73, 16, 45, 47, 90, 66, 42, 22, 53, 57, 21, 61, 122, 118, 20, 1, 6, 121, 114, 93, 109, 54, 89, 62, 76, 120, 74, 48, 69, 123, 33, 87, 50, 13, 55, 7, 10, 127, 84, 95, 83, 115, 41, 23, 26, 119, 2, 117, 60, 103, 56, 5, 30, 37, 72, 81, 40, 100, 101, 91, 14, 104, 94, 15, 92, 19, 38, 85, 8, 0, 49, 79, 75, 35, 36, 28, 63, 98, 82, 32, 34, 99, 110, 44, 27, 58, 96, 107], [38, 39, 97, 109, 105, 89, 25, 49, 106, 78, 44, 40, 20, 48, 36, 126, 33, 64, 62, 31, 104, 108, 82, 27, 122, 29, 120, 94, 34, 96, 113, 117, 51, 42, 107, 103, 11, 8, 115, 16, 18, 84, 123, 14, 30, 37, 124, 91, 125, 50, 88, 56, 61, 41, 17, 87, 114, 66, 21, 47, 12, 53, 54, 19, 60, 26, 52, 73, 28, 24, 23, 118, 90, 32, 81, 83, 59, 102, 85, 15, 1, 74, 93, 57, 121, 43, 75, 58, 3, 112, 70, 92, 9, 69, 13, 127, 111, 79, 5, 6, 80, 119, 22, 63, 55, 76, 99, 7, 116, 35, 110, 77, 68, 4, 101, 72, 86, 46, 67, 10, 71, 45, 0, 2, 95, 98, 65, 100], [107, 39, 108, 64, 40, 47, 105, 97, 113, 29, 65, 30, 3, 66, 36, 83, 48, 89, 116, 122, 101, 90, 2, 95, 17, 13, 16, 33, 74, 38, 85, 23, 19, 63, 87, 27, 109, 5, 71, 114, 24, 81, 26, 84, 104, 53, 50, 92, 82, 93, 4, 34, 22, 121, 106, 76, 1, 14, 78, 68, 35, 79, 115, 103, 120, 123, 9, 42, 55, 41, 69, 7, 126, 118, 88, 18, 127, 28, 112, 51, 59, 80, 54, 124, 111, 117, 62, 91, 15, 43, 77, 96, 125, 110, 98, 6, 45, 49, 119, 20, 100, 94, 60, 25, 75, 58, 86, 56, 52, 44, 0, 57, 21, 61, 12, 10, 46, 31, 102, 11, 32, 70, 8, 72, 99, 67, 37, 73], [103, 105, 107, 33, 0, 37, 65, 83, 24, 28, 95, 87, 78, 19, 84, 23, 25, 77, 34, 96, 125, 54, 94, 14, 92, 35, 21, 89, 79, 5, 116, 123, 22, 59, 13, 18, 29, 55, 73, 81, 36, 12, 106, 52, 127, 111, 102, 56, 121, 15, 97, 42, 63, 46, 48, 120, 11, 45, 91, 51, 71, 76, 126, 93, 85, 66, 119, 122, 90, 8, 68, 60, 26, 98, 2, 118, 40, 39, 109, 16, 113, 57, 32, 58, 3, 27, 112, 62, 61, 44, 30, 7, 80, 115, 114, 124, 41, 74, 117, 50, 17, 110, 49, 31, 69, 53, 47, 108, 4, 64, 82, 9, 104, 67, 100, 75, 43, 1, 88, 72, 38, 101, 10, 20, 70, 86, 99, 6], [104, 38, 43, 103, 95, 42, 45, 46, 105, 112, 84, 113, 53, 50, 59, 125, 96, 90, 115, 88, 60, 23, 13, 62, 34, 61, 91, 82, 58, 124, 0, 57, 92, 63, 111, 21, 123, 120, 119, 20, 79, 19, 55, 54, 17, 15, 126, 22, 81, 87, 27, 127, 44, 72, 49, 85, 51, 86, 56, 52, 117, 80, 48, 75, 114, 31, 89, 18, 94, 116, 29, 121, 122, 1, 33, 12, 16, 118, 32, 7, 35, 101, 64, 40, 78, 47, 24, 109, 65, 69, 71, 14, 76, 73, 100, 11, 97, 110, 41, 8, 26, 4, 108, 67, 39, 93, 25, 36, 70, 28, 10, 66, 74, 6, 77, 37, 107, 99, 5, 68, 83, 9, 98, 3, 102, 106, 30, 2], [39, 111, 32, 1, 0, 109, 2, 6, 48, 4, 40, 41, 29, 108, 67, 84, 66, 15, 107, 11, 65, 81, 115, 70, 89, 73, 52, 21, 30, 57, 104, 119, 80, 42, 118, 87, 124, 123, 13, 106, 72, 3, 49, 86, 58, 98, 63, 27, 114, 110, 127, 113, 126, 76, 53, 26, 25, 14, 83, 59, 96, 55, 77, 88, 19, 94, 121, 93, 23, 125, 7, 78, 51, 120, 46, 82, 116, 71, 68, 105, 85, 24, 97, 8, 61, 74, 103, 10, 54, 47, 117, 122, 34, 91, 18, 92, 20, 62, 100, 12, 31, 75, 56, 5, 90, 9, 79, 28, 33, 101, 69, 99, 22, 36, 95, 35, 43, 50, 60, 16, 112, 37, 44, 102, 17, 45, 64, 38], [64, 66, 4, 106, 6, 67, 1, 71, 10, 69, 105, 46, 43, 104, 37, 44, 92, 47, 109, 103, 94, 15, 8, 83, 85, 7, 56, 53, 73, 86, 126, 116, 32, 78, 22, 82, 75, 24, 77, 49, 9, 120, 76, 79, 50, 91, 114, 52, 127, 48, 17, 16, 35, 19, 124, 18, 102, 12, 20, 68, 58, 57, 63, 60, 115, 13, 113, 121, 123, 14, 81, 117, 45, 62, 59, 55, 51, 111, 122, 125, 23, 61, 119, 118, 110, 21, 3, 100, 112, 33, 54, 38, 72, 34, 2, 11, 80, 65, 29, 70, 88, 27, 0, 36, 98, 84, 89, 97, 74, 93, 30, 5, 107, 90, 96, 26, 87, 95, 28, 39, 108, 41, 42, 99, 101, 31, 25, 40]], "model.layers.0.self_attn.qk_proj": [[104, 39, 33, 42, 103, 41, 97, 64, 0, 93, 108, 25, 107, 89, 105, 113, 126, 29, 102, 121, 111, 48, 57, 95, 59, 115, 47, 123, 70, 88, 53, 63, 96, 30, 24, 20, 43, 87, 116, 23, 38, 109, 78, 124, 49, 19, 61, 112, 52, 51, 50, 120, 27, 83, 119, 22, 32, 114, 45, 21, 106, 125, 122, 40, 44, 127, 86, 54, 17, 118, 56, 85, 55, 14, 84, 46, 65, 81, 68, 1, 91, 3, 62, 4, 66, 18, 82, 77, 9, 79, 117, 2, 90, 92, 110, 8, 80, 67, 13, 76, 15, 75, 60, 12, 26, 16, 31, 11, 73, 10, 58, 71, 74, 7, 5, 69, 28, 101, 6, 34, 98, 94, 72, 37, 100, 35, 36, 99], [39, 104, 33, 42, 103, 41, 64, 97, 108, 0, 93, 107, 105, 25, 113, 126, 29, 89, 95, 102, 111, 59, 47, 38, 123, 121, 124, 88, 115, 50, 53, 106, 116, 48, 85, 119, 30, 70, 43, 63, 40, 87, 96, 24, 21, 57, 49, 23, 122, 27, 20, 22, 44, 114, 109, 52, 19, 120, 54, 61, 45, 67, 127, 17, 78, 32, 81, 83, 84, 62, 56, 125, 86, 51, 46, 14, 1, 118, 92, 112, 55, 79, 26, 18, 12, 2, 82, 15, 76, 65, 90, 3, 77, 91, 66, 117, 110, 73, 74, 4, 31, 9, 16, 13, 68, 75, 10, 80, 60, 8, 7, 58, 11, 71, 28, 98, 94, 6, 5, 72, 69, 34, 101, 100, 35, 36, 37, 99], [104, 39, 33, 42, 103, 41, 97, 108, 93, 0, 64, 25, 89, 29, 105, 107, 113, 95, 126, 102, 59, 123, 111, 115, 121, 47, 57, 88, 63, 24, 48, 124, 96, 38, 53, 116, 30, 87, 19, 52, 119, 43, 23, 20, 21, 40, 61, 127, 27, 109, 51, 84, 50, 22, 112, 106, 49, 32, 78, 86, 125, 14, 85, 54, 56, 17, 45, 120, 114, 70, 122, 44, 3, 118, 81, 46, 83, 65, 62, 82, 1, 91, 92, 15, 55, 76, 2, 117, 110, 12, 90, 79, 80, 26, 67, 16, 18, 77, 68, 11, 66, 13, 73, 9, 4, 31, 75, 6, 58, 74, 60, 72, 71, 10, 8, 28, 69, 7, 34, 98, 5, 94, 101, 100, 37, 35, 36, 99], [39, 104, 42, 33, 103, 41, 97, 108, 64, 0, 105, 93, 107, 113, 126, 89, 25, 29, 59, 124, 123, 111, 119, 63, 102, 115, 121, 95, 53, 47, 48, 50, 57, 87, 106, 96, 116, 43, 88, 38, 49, 30, 61, 23, 24, 22, 54, 20, 51, 6, 122, 44, 114, 27, 40, 109, 52, 78, 85, 125, 21, 83, 45, 17, 19, 118, 86, 84, 112, 62, 65, 81, 56, 55, 14, 120, 79, 127, 32, 68, 66, 67, 1, 46, 3, 9, 91, 110, 18, 4, 12, 73, 82, 26, 2, 72, 70, 92, 77, 15, 117, 76, 13, 16, 90, 58, 80, 75, 10, 31, 60, 74, 11, 7, 71, 69, 28, 5, 98, 94, 101, 34, 8, 100, 35, 37, 36, 99], [39, 104, 33, 42, 103, 41, 97, 64, 108, 0, 93, 105, 25, 89, 107, 113, 126, 115, 102, 59, 29, 123, 47, 95, 119, 121, 63, 53, 111, 124, 57, 48, 116, 43, 6, 20, 30, 23, 88, 96, 24, 106, 109, 38, 87, 50, 52, 22, 21, 49, 27, 17, 40, 61, 19, 112, 122, 78, 3, 54, 14, 44, 45, 114, 51, 91, 84, 86, 46, 65, 85, 125, 1, 127, 118, 120, 32, 81, 56, 62, 82, 83, 26, 73, 72, 55, 15, 9, 2, 110, 4, 79, 66, 76, 92, 117, 12, 67, 18, 16, 77, 11, 13, 68, 31, 80, 90, 75, 60, 58, 10, 74, 71, 70, 7, 98, 34, 28, 69, 5, 94, 101, 8, 100, 37, 36, 35, 99], [39, 104, 42, 33, 103, 41, 97, 108, 0, 64, 25, 93, 89, 105, 107, 126, 95, 113, 29, 102, 115, 123, 59, 48, 121, 53, 124, 6, 30, 88, 111, 47, 87, 63, 96, 106, 116, 23, 119, 38, 27, 24, 43, 57, 20, 49, 40, 22, 61, 44, 50, 54, 52, 122, 21, 109, 1, 51, 86, 81, 85, 19, 78, 114, 3, 84, 45, 125, 112, 32, 127, 14, 83, 118, 120, 17, 2, 46, 56, 55, 76, 62, 82, 91, 92, 18, 72, 77, 65, 79, 9, 117, 15, 12, 66, 90, 110, 67, 68, 4, 26, 16, 31, 13, 80, 73, 75, 10, 74, 60, 7, 11, 71, 58, 98, 5, 94, 28, 34, 70, 69, 101, 8, 100, 35, 37, 99, 36], [39, 104, 33, 42, 103, 97, 41, 0, 64, 93, 108, 113, 105, 107, 25, 89, 126, 47, 29, 95, 102, 30, 121, 59, 123, 115, 48, 111, 53, 6, 124, 88, 24, 50, 119, 49, 116, 27, 38, 106, 63, 43, 57, 96, 87, 23, 21, 32, 20, 125, 52, 61, 22, 109, 112, 44, 85, 45, 51, 19, 83, 122, 114, 40, 14, 54, 55, 78, 17, 120, 2, 84, 86, 62, 1, 118, 127, 66, 91, 3, 56, 72, 110, 4, 81, 18, 82, 77, 46, 68, 26, 79, 65, 16, 13, 9, 15, 76, 67, 12, 80, 92, 75, 117, 31, 90, 73, 60, 74, 58, 10, 11, 71, 7, 28, 70, 5, 69, 98, 34, 94, 101, 8, 100, 37, 35, 36, 99], [39, 33, 104, 42, 103, 97, 41, 93, 64, 108, 0, 25, 105, 113, 89, 107, 126, 29, 95, 47, 59, 102, 48, 88, 53, 115, 116, 123, 124, 111, 24, 57, 30, 38, 96, 63, 87, 52, 121, 19, 43, 23, 27, 20, 50, 119, 51, 85, 32, 6, 40, 21, 106, 49, 109, 61, 127, 78, 125, 86, 122, 84, 22, 54, 81, 120, 45, 112, 14, 114, 91, 17, 83, 56, 46, 44, 118, 2, 67, 16, 90, 110, 66, 62, 1, 117, 4, 79, 65, 76, 92, 72, 68, 82, 15, 18, 75, 13, 26, 55, 12, 9, 3, 80, 31, 77, 11, 73, 10, 58, 74, 60, 70, 28, 7, 98, 71, 34, 94, 101, 69, 5, 100, 37, 8, 35, 36, 99], [39, 104, 33, 42, 103, 41, 97, 64, 0, 108, 93, 105, 107, 113, 25, 95, 89, 29, 126, 48, 59, 124, 102, 88, 47, 38, 115, 123, 111, 53, 50, 106, 21, 27, 44, 30, 96, 43, 87, 116, 121, 109, 52, 23, 24, 119, 40, 63, 49, 57, 85, 19, 22, 114, 86, 122, 20, 127, 78, 54, 83, 56, 51, 17, 32, 84, 61, 14, 81, 90, 125, 120, 112, 45, 46, 118, 66, 1, 70, 82, 15, 62, 12, 79, 91, 55, 2, 76, 26, 65, 67, 18, 92, 6, 110, 9, 117, 73, 3, 72, 77, 16, 31, 68, 10, 13, 80, 74, 75, 4, 60, 11, 28, 58, 7, 98, 94, 34, 71, 5, 69, 8, 101, 100, 37, 35, 36, 99], [33, 104, 39, 42, 103, 41, 97, 89, 25, 93, 108, 29, 64, 95, 0, 113, 102, 107, 105, 126, 38, 87, 59, 88, 96, 48, 27, 24, 53, 47, 32, 116, 30, 115, 124, 85, 123, 19, 21, 40, 106, 23, 57, 43, 109, 121, 111, 22, 44, 20, 83, 63, 122, 78, 52, 50, 84, 86, 81, 49, 127, 2, 56, 45, 17, 26, 119, 66, 14, 114, 54, 61, 51, 125, 70, 91, 82, 4, 120, 112, 76, 12, 46, 118, 92, 18, 1, 67, 15, 55, 117, 68, 79, 65, 62, 80, 16, 90, 13, 77, 73, 74, 72, 31, 75, 9, 11, 10, 3, 110, 71, 7, 58, 60, 6, 28, 8, 34, 94, 98, 101, 69, 5, 100, 37, 36, 35, 99], [39, 104, 33, 42, 103, 0, 41, 64, 97, 108, 25, 93, 113, 105, 107, 126, 89, 59, 29, 115, 111, 47, 57, 121, 102, 63, 123, 48, 95, 124, 70, 52, 88, 96, 116, 53, 119, 43, 87, 61, 49, 23, 24, 30, 38, 50, 19, 125, 106, 20, 51, 21, 32, 109, 56, 22, 54, 14, 78, 1, 112, 120, 83, 67, 45, 40, 114, 44, 122, 118, 84, 127, 27, 46, 86, 81, 68, 65, 3, 17, 62, 85, 55, 91, 110, 2, 79, 18, 66, 77, 82, 4, 76, 9, 13, 117, 92, 15, 90, 26, 12, 73, 60, 11, 16, 31, 74, 58, 75, 80, 10, 8, 71, 7, 72, 28, 5, 69, 6, 98, 101, 94, 34, 100, 37, 35, 99, 36], [39, 104, 33, 42, 103, 0, 41, 64, 108, 97, 107, 105, 113, 93, 126, 25, 89, 59, 121, 63, 111, 115, 70, 123, 47, 29, 95, 102, 119, 48, 124, 96, 53, 116, 49, 57, 51, 88, 52, 87, 50, 61, 27, 30, 65, 109, 43, 106, 22, 120, 24, 23, 21, 114, 38, 56, 19, 122, 40, 54, 14, 45, 125, 118, 83, 84, 112, 20, 46, 44, 127, 55, 32, 86, 81, 78, 62, 17, 67, 82, 85, 3, 91, 2, 90, 117, 79, 66, 77, 92, 68, 4, 110, 76, 15, 13, 60, 73, 26, 31, 1, 9, 12, 75, 18, 80, 8, 58, 74, 11, 16, 71, 10, 28, 7, 69, 5, 98, 72, 34, 94, 6, 101, 100, 37, 36, 35, 99], [39, 33, 104, 42, 103, 97, 41, 108, 89, 93, 95, 105, 0, 113, 25, 64, 29, 126, 107, 30, 102, 27, 124, 48, 38, 47, 115, 53, 106, 88, 24, 59, 23, 116, 123, 21, 96, 40, 32, 87, 86, 111, 50, 121, 44, 85, 43, 22, 122, 20, 109, 63, 49, 70, 57, 119, 52, 83, 81, 19, 51, 14, 91, 17, 114, 118, 78, 54, 127, 125, 45, 112, 61, 26, 92, 84, 2, 56, 76, 66, 18, 15, 82, 12, 65, 62, 79, 46, 1, 120, 117, 80, 55, 73, 31, 68, 90, 110, 16, 67, 77, 3, 8, 13, 75, 74, 4, 9, 6, 10, 11, 60, 34, 58, 71, 28, 98, 94, 7, 72, 69, 101, 5, 100, 36, 37, 35, 99], [33, 104, 39, 42, 103, 41, 97, 93, 108, 89, 25, 0, 64, 95, 105, 113, 29, 107, 126, 102, 121, 115, 53, 30, 88, 38, 48, 47, 96, 59, 124, 123, 57, 43, 87, 21, 111, 116, 22, 23, 27, 63, 32, 40, 51, 52, 50, 106, 20, 109, 86, 44, 83, 122, 24, 14, 49, 78, 119, 127, 19, 112, 84, 81, 61, 54, 17, 114, 85, 55, 120, 91, 45, 46, 70, 18, 82, 62, 79, 65, 12, 2, 56, 92, 3, 66, 118, 4, 125, 90, 73, 26, 117, 13, 1, 68, 8, 80, 15, 76, 6, 31, 67, 77, 110, 16, 11, 9, 75, 10, 60, 74, 28, 7, 71, 5, 58, 98, 34, 101, 94, 69, 100, 72, 37, 35, 36, 99], [39, 104, 33, 42, 103, 0, 97, 41, 108, 93, 64, 113, 25, 107, 126, 47, 105, 89, 59, 29, 115, 124, 48, 63, 121, 123, 53, 95, 57, 111, 116, 102, 52, 30, 96, 24, 49, 88, 119, 51, 87, 23, 38, 20, 106, 78, 32, 61, 83, 44, 43, 27, 21, 22, 50, 127, 109, 122, 86, 54, 85, 19, 114, 6, 17, 125, 14, 3, 40, 112, 4, 81, 118, 46, 45, 56, 65, 62, 84, 2, 55, 120, 82, 66, 8, 91, 18, 76, 68, 79, 26, 117, 1, 12, 9, 11, 67, 73, 90, 70, 13, 15, 16, 110, 92, 77, 31, 60, 75, 80, 71, 10, 7, 58, 74, 28, 5, 98, 34, 69, 94, 72, 101, 37, 100, 35, 36, 99], [39, 104, 33, 42, 103, 41, 64, 0, 97, 108, 93, 113, 25, 105, 107, 89, 126, 115, 29, 102, 59, 123, 95, 111, 47, 6, 48, 121, 53, 88, 63, 57, 119, 124, 43, 49, 116, 30, 52, 109, 24, 38, 106, 87, 50, 96, 61, 20, 23, 21, 19, 44, 27, 114, 14, 122, 125, 51, 54, 85, 22, 112, 56, 40, 120, 84, 83, 118, 127, 17, 65, 62, 78, 55, 45, 46, 32, 66, 81, 91, 3, 86, 1, 67, 8, 76, 18, 79, 117, 68, 2, 73, 110, 26, 90, 82, 77, 9, 4, 13, 92, 12, 16, 80, 74, 15, 60, 31, 11, 75, 10, 7, 58, 28, 71, 70, 94, 69, 5, 98, 34, 101, 72, 37, 100, 35, 99, 36], [39, 33, 104, 42, 103, 41, 97, 108, 0, 93, 64, 105, 107, 25, 89, 95, 113, 126, 29, 102, 59, 30, 111, 121, 47, 123, 88, 38, 124, 115, 87, 6, 24, 43, 106, 50, 53, 48, 116, 119, 40, 27, 21, 23, 96, 86, 109, 85, 63, 20, 44, 83, 32, 122, 22, 84, 49, 52, 57, 17, 14, 91, 114, 81, 51, 19, 61, 54, 127, 45, 56, 65, 120, 92, 125, 90, 79, 66, 2, 76, 78, 112, 82, 46, 118, 26, 15, 18, 62, 55, 80, 3, 13, 67, 8, 68, 9, 31, 12, 117, 1, 74, 73, 16, 110, 4, 77, 11, 75, 10, 60, 58, 71, 7, 28, 34, 94, 98, 5, 72, 69, 100, 70, 101, 35, 36, 37, 99], [39, 104, 33, 42, 103, 41, 97, 108, 93, 64, 0, 89, 29, 105, 113, 25, 126, 107, 95, 115, 47, 124, 111, 116, 88, 59, 48, 121, 123, 53, 57, 87, 102, 24, 38, 63, 30, 20, 23, 96, 43, 21, 6, 40, 127, 22, 106, 119, 52, 50, 27, 51, 109, 49, 61, 32, 54, 17, 78, 84, 44, 112, 114, 19, 83, 118, 85, 82, 56, 120, 45, 122, 14, 125, 81, 1, 46, 117, 86, 65, 91, 67, 62, 68, 26, 2, 92, 90, 55, 4, 79, 12, 77, 66, 15, 18, 13, 76, 31, 3, 9, 110, 16, 73, 75, 80, 11, 74, 10, 60, 58, 8, 7, 98, 71, 70, 5, 72, 69, 28, 34, 94, 101, 100, 37, 35, 99, 36], [39, 104, 33, 42, 103, 64, 97, 41, 0, 108, 93, 107, 113, 89, 105, 126, 25, 29, 59, 121, 115, 102, 95, 47, 111, 123, 124, 48, 63, 53, 57, 88, 119, 43, 30, 38, 24, 50, 116, 96, 49, 61, 51, 106, 87, 20, 52, 54, 19, 23, 21, 44, 27, 22, 81, 6, 14, 125, 122, 85, 32, 112, 114, 109, 127, 120, 55, 78, 62, 40, 84, 83, 56, 2, 45, 17, 86, 118, 46, 1, 67, 82, 91, 65, 3, 110, 68, 79, 4, 66, 12, 26, 77, 18, 73, 15, 90, 117, 92, 76, 9, 70, 13, 80, 11, 16, 10, 75, 31, 74, 58, 60, 7, 71, 69, 72, 8, 28, 5, 98, 94, 34, 101, 37, 100, 35, 99, 36], [39, 33, 104, 42, 103, 41, 108, 97, 0, 64, 113, 25, 93, 89, 107, 105, 29, 95, 126, 102, 115, 59, 48, 124, 123, 121, 111, 38, 88, 53, 63, 21, 24, 47, 106, 116, 30, 87, 23, 49, 22, 27, 96, 119, 85, 43, 20, 57, 122, 40, 52, 109, 44, 65, 32, 14, 83, 17, 81, 51, 19, 61, 50, 54, 114, 125, 78, 91, 127, 3, 84, 112, 70, 79, 56, 86, 67, 120, 46, 45, 12, 62, 18, 2, 118, 26, 76, 66, 73, 55, 82, 92, 4, 16, 13, 6, 90, 1, 15, 9, 117, 110, 31, 77, 80, 74, 10, 68, 75, 72, 7, 71, 11, 28, 58, 60, 94, 98, 34, 5, 8, 69, 101, 100, 35, 37, 36, 99], [39, 104, 33, 42, 103, 41, 64, 97, 108, 0, 93, 105, 113, 89, 107, 59, 25, 29, 126, 95, 123, 47, 121, 48, 111, 115, 102, 63, 124, 70, 57, 87, 119, 53, 116, 88, 30, 109, 49, 43, 50, 24, 56, 96, 114, 61, 38, 52, 65, 51, 106, 40, 127, 20, 44, 83, 85, 120, 118, 54, 22, 86, 23, 46, 21, 19, 122, 78, 112, 27, 125, 81, 45, 14, 32, 62, 17, 84, 67, 91, 82, 90, 117, 15, 1, 55, 18, 3, 110, 76, 12, 13, 26, 66, 68, 77, 60, 72, 31, 2, 4, 9, 16, 75, 79, 92, 58, 73, 11, 10, 80, 6, 74, 7, 98, 28, 71, 69, 5, 34, 94, 100, 8, 101, 37, 35, 36, 99], [39, 104, 33, 42, 103, 41, 97, 108, 107, 0, 93, 113, 64, 105, 126, 89, 25, 95, 29, 30, 47, 59, 123, 102, 88, 124, 121, 111, 70, 115, 38, 43, 116, 24, 48, 53, 119, 106, 50, 19, 96, 63, 87, 57, 44, 21, 49, 23, 109, 27, 85, 52, 22, 14, 40, 51, 122, 20, 32, 56, 54, 114, 61, 17, 84, 127, 125, 2, 120, 62, 4, 83, 81, 67, 86, 78, 55, 79, 66, 112, 82, 118, 46, 26, 91, 68, 65, 1, 45, 15, 76, 16, 72, 18, 92, 117, 73, 12, 3, 9, 90, 77, 11, 110, 74, 13, 31, 80, 10, 71, 60, 58, 75, 7, 28, 94, 98, 69, 8, 34, 5, 101, 100, 37, 6, 35, 36, 99], [39, 104, 33, 42, 103, 41, 0, 97, 108, 105, 93, 64, 25, 89, 113, 107, 95, 29, 126, 102, 115, 59, 111, 123, 70, 124, 48, 53, 87, 24, 88, 121, 106, 47, 38, 27, 63, 21, 116, 119, 43, 50, 96, 22, 30, 40, 20, 23, 19, 57, 52, 49, 122, 114, 109, 85, 44, 54, 17, 51, 84, 78, 81, 65, 61, 3, 45, 112, 86, 120, 125, 56, 127, 32, 14, 91, 62, 82, 118, 46, 15, 92, 12, 79, 1, 18, 110, 26, 9, 76, 72, 83, 73, 67, 55, 117, 2, 80, 16, 66, 31, 77, 13, 10, 75, 4, 68, 90, 11, 60, 58, 7, 74, 6, 28, 34, 98, 5, 69, 71, 94, 101, 100, 35, 8, 36, 37, 99], [39, 104, 33, 42, 103, 41, 97, 108, 0, 64, 25, 93, 107, 105, 89, 29, 113, 95, 126, 102, 115, 123, 48, 124, 88, 53, 59, 121, 30, 87, 47, 38, 96, 22, 63, 111, 106, 23, 43, 27, 49, 24, 65, 40, 116, 20, 119, 50, 44, 109, 122, 52, 114, 57, 51, 21, 85, 14, 84, 54, 70, 83, 81, 127, 61, 17, 86, 82, 19, 32, 79, 56, 112, 46, 78, 118, 45, 120, 62, 91, 55, 2, 117, 125, 3, 12, 18, 9, 72, 92, 13, 76, 26, 16, 66, 31, 1, 67, 90, 68, 110, 11, 77, 10, 73, 15, 80, 6, 4, 60, 75, 71, 74, 58, 98, 28, 34, 7, 94, 5, 69, 101, 8, 100, 35, 37, 36, 99], [39, 104, 33, 42, 103, 97, 41, 93, 64, 0, 108, 25, 107, 113, 89, 105, 102, 126, 95, 29, 47, 30, 123, 59, 111, 88, 115, 24, 48, 121, 38, 124, 53, 119, 63, 49, 43, 57, 96, 50, 27, 52, 23, 116, 21, 87, 19, 106, 51, 20, 61, 32, 84, 44, 22, 109, 40, 78, 85, 114, 127, 14, 118, 83, 125, 17, 86, 112, 66, 122, 45, 120, 2, 54, 81, 62, 56, 1, 3, 68, 65, 4, 70, 46, 26, 79, 55, 12, 82, 6, 91, 15, 92, 72, 18, 80, 76, 117, 13, 9, 77, 90, 16, 67, 110, 31, 11, 10, 73, 60, 58, 75, 74, 71, 7, 28, 5, 98, 69, 34, 94, 8, 101, 100, 35, 37, 99, 36], [39, 104, 33, 42, 103, 41, 97, 108, 64, 93, 105, 95, 29, 107, 0, 89, 25, 126, 88, 113, 102, 123, 124, 47, 121, 48, 53, 30, 115, 111, 59, 38, 43, 87, 106, 116, 40, 96, 22, 50, 109, 6, 24, 63, 23, 49, 119, 27, 21, 54, 44, 32, 52, 61, 83, 114, 122, 57, 20, 86, 56, 85, 81, 112, 45, 127, 19, 84, 51, 14, 125, 120, 1, 78, 118, 46, 62, 82, 17, 91, 15, 65, 18, 3, 117, 90, 67, 79, 26, 31, 92, 12, 66, 2, 77, 68, 55, 76, 16, 110, 13, 73, 58, 4, 80, 10, 9, 75, 74, 11, 72, 98, 60, 70, 7, 71, 34, 28, 94, 69, 8, 5, 101, 100, 36, 35, 37, 99], [39, 104, 33, 42, 103, 41, 97, 0, 64, 108, 93, 25, 113, 29, 89, 105, 59, 126, 107, 95, 115, 111, 102, 48, 123, 57, 47, 87, 124, 53, 121, 6, 63, 116, 38, 52, 88, 24, 96, 109, 30, 43, 119, 61, 106, 21, 49, 127, 44, 50, 54, 19, 23, 40, 51, 84, 20, 125, 27, 114, 56, 112, 14, 86, 85, 22, 83, 45, 122, 17, 65, 32, 120, 46, 81, 66, 78, 55, 1, 91, 3, 18, 79, 82, 62, 118, 26, 67, 12, 110, 90, 2, 76, 13, 77, 92, 31, 117, 10, 4, 15, 16, 9, 80, 75, 60, 73, 58, 68, 11, 74, 8, 71, 7, 72, 28, 70, 69, 98, 5, 101, 34, 94, 100, 35, 37, 36, 99], [39, 33, 104, 42, 103, 41, 97, 108, 0, 93, 25, 89, 107, 64, 113, 105, 126, 29, 95, 102, 59, 111, 123, 48, 124, 47, 30, 38, 53, 24, 43, 121, 6, 57, 88, 27, 115, 63, 96, 23, 44, 106, 116, 87, 21, 83, 49, 119, 109, 85, 52, 20, 32, 22, 50, 40, 14, 122, 19, 51, 17, 86, 78, 54, 114, 127, 81, 61, 45, 46, 84, 65, 62, 91, 56, 125, 112, 66, 120, 18, 2, 12, 15, 68, 67, 26, 82, 90, 3, 55, 118, 4, 79, 92, 13, 31, 1, 117, 76, 9, 8, 73, 80, 11, 77, 10, 16, 110, 75, 60, 74, 7, 71, 28, 58, 72, 34, 94, 98, 69, 5, 70, 101, 100, 37, 35, 36, 99], [104, 39, 33, 42, 103, 64, 0, 41, 108, 97, 93, 113, 107, 126, 25, 89, 59, 29, 105, 121, 111, 63, 115, 47, 102, 123, 95, 57, 88, 48, 124, 6, 53, 119, 21, 24, 61, 87, 116, 43, 96, 30, 106, 49, 51, 20, 83, 50, 114, 52, 67, 38, 109, 27, 65, 54, 125, 56, 122, 1, 112, 14, 4, 85, 44, 22, 120, 127, 23, 78, 19, 32, 81, 45, 40, 55, 86, 46, 118, 84, 17, 62, 2, 3, 12, 15, 82, 66, 79, 68, 26, 76, 18, 91, 9, 117, 110, 73, 8, 77, 92, 13, 16, 90, 7, 75, 10, 11, 80, 74, 71, 60, 31, 58, 70, 69, 28, 5, 98, 101, 94, 72, 34, 100, 37, 35, 99, 36], [39, 104, 33, 42, 103, 41, 108, 97, 93, 0, 105, 64, 25, 95, 29, 89, 115, 113, 126, 107, 102, 59, 124, 48, 123, 111, 53, 43, 88, 106, 121, 30, 47, 38, 119, 24, 87, 50, 96, 27, 22, 63, 21, 20, 57, 17, 49, 23, 52, 86, 116, 44, 40, 54, 122, 114, 6, 109, 51, 84, 65, 78, 19, 112, 127, 61, 125, 14, 83, 32, 85, 3, 81, 45, 18, 56, 46, 70, 79, 120, 1, 15, 62, 91, 82, 55, 76, 118, 9, 67, 2, 12, 8, 92, 90, 73, 110, 4, 31, 13, 68, 117, 16, 26, 10, 66, 80, 77, 75, 11, 74, 58, 60, 71, 28, 69, 7, 98, 94, 34, 5, 101, 100, 72, 36, 35, 37, 99], [39, 33, 104, 42, 103, 97, 41, 93, 0, 108, 25, 95, 107, 29, 113, 89, 64, 105, 102, 126, 115, 59, 88, 124, 47, 27, 38, 30, 24, 53, 121, 87, 48, 23, 123, 116, 96, 21, 106, 111, 32, 50, 44, 43, 85, 19, 109, 40, 63, 52, 83, 22, 49, 119, 54, 122, 56, 84, 81, 14, 86, 125, 20, 57, 66, 127, 114, 112, 17, 91, 61, 78, 2, 1, 70, 51, 45, 18, 12, 118, 82, 65, 15, 120, 46, 55, 90, 76, 77, 16, 92, 4, 3, 26, 8, 62, 67, 79, 117, 73, 80, 75, 13, 31, 68, 74, 9, 110, 10, 6, 71, 11, 28, 60, 58, 7, 34, 98, 94, 5, 69, 72, 100, 101, 35, 37, 36, 99], [39, 104, 33, 42, 103, 97, 0, 41, 93, 108, 64, 25, 105, 89, 113, 29, 95, 107, 126, 102, 59, 115, 48, 47, 123, 124, 53, 70, 88, 111, 116, 30, 43, 52, 24, 57, 23, 38, 121, 96, 87, 63, 22, 119, 106, 78, 51, 50, 40, 32, 20, 19, 44, 109, 49, 61, 54, 122, 127, 91, 83, 21, 112, 27, 125, 81, 45, 85, 84, 118, 1, 86, 56, 17, 62, 46, 79, 114, 55, 3, 14, 66, 120, 65, 2, 18, 68, 82, 110, 12, 8, 26, 117, 77, 90, 67, 75, 16, 92, 4, 31, 13, 76, 15, 80, 9, 73, 10, 28, 11, 74, 58, 60, 7, 71, 34, 98, 5, 94, 69, 6, 101, 72, 100, 37, 35, 36, 99]], "model.layers.1.self_attn.q_proj": [[103, 12, 107, 104, 46, 33, 42, 105, 49, 123, 50, 117, 62, 48, 57, 122, 19, 6, 59, 31, 29, 52, 54, 56, 5, 44, 115, 23, 92, 63, 8, 76, 70, 80, 66, 79, 72, 20, 116, 85, 120, 111, 84, 65, 78, 83, 25, 3, 22, 18, 127, 55, 90, 9, 27, 7, 126, 11, 87, 74, 37, 58, 82, 16, 77, 124, 4, 67, 2, 35, 13, 109, 15, 125, 88, 102, 71, 75, 17, 34, 94, 21, 100, 86, 14, 61, 24, 81, 114, 69, 45, 1, 98, 112, 73, 0, 51, 10, 39, 113, 101, 38, 32, 30, 36, 26, 121, 89, 41, 43, 97, 108, 106, 47, 40, 28, 93, 96, 64, 60, 91, 99, 118, 53, 119, 68, 110, 95], [103, 107, 104, 42, 105, 117, 46, 123, 50, 62, 24, 48, 33, 3, 67, 57, 122, 111, 59, 70, 56, 7, 127, 25, 65, 29, 49, 63, 115, 31, 79, 58, 13, 124, 16, 74, 44, 6, 66, 0, 4, 11, 116, 22, 9, 92, 82, 54, 5, 77, 101, 23, 125, 45, 84, 52, 1, 90, 78, 12, 98, 20, 94, 27, 126, 71, 2, 83, 120, 17, 73, 109, 38, 18, 87, 37, 19, 86, 32, 36, 15, 75, 64, 80, 55, 35, 81, 10, 14, 102, 100, 8, 113, 99, 76, 61, 121, 69, 26, 85, 53, 72, 97, 21, 30, 112, 89, 68, 118, 39, 43, 88, 41, 106, 114, 34, 110, 91, 95, 93, 60, 47, 108, 40, 51, 28, 96, 119], [100, 44, 102, 113, 37, 38, 59, 115, 35, 57, 122, 116, 34, 104, 114, 103, 109, 123, 46, 107, 50, 48, 62, 120, 117, 54, 42, 105, 52, 49, 125, 63, 112, 111, 98, 61, 56, 60, 53, 126, 121, 101, 33, 25, 99, 110, 127, 32, 84, 86, 58, 47, 82, 124, 88, 108, 21, 94, 40, 119, 41, 45, 36, 43, 89, 29, 118, 7, 51, 78, 77, 96, 17, 13, 30, 72, 106, 81, 95, 70, 92, 97, 93, 27, 11, 4, 79, 14, 9, 16, 55, 71, 74, 26, 83, 20, 67, 65, 75, 5, 0, 87, 66, 19, 12, 76, 85, 80, 91, 39, 3, 90, 24, 31, 28, 18, 6, 2, 22, 1, 64, 8, 15, 23, 68, 69, 73, 10], [103, 107, 104, 42, 33, 105, 46, 123, 117, 50, 62, 57, 122, 48, 49, 9, 29, 125, 111, 63, 31, 92, 90, 115, 10, 56, 4, 18, 54, 83, 69, 35, 7, 84, 44, 20, 23, 71, 22, 59, 100, 79, 75, 14, 5, 58, 127, 25, 113, 77, 8, 74, 86, 87, 27, 82, 70, 66, 65, 99, 120, 24, 13, 2, 73, 6, 12, 52, 78, 85, 55, 124, 114, 38, 1, 47, 108, 116, 72, 121, 11, 45, 88, 89, 81, 126, 112, 32, 3, 80, 93, 21, 15, 37, 16, 30, 97, 26, 94, 19, 53, 28, 101, 60, 118, 102, 17, 109, 91, 36, 39, 119, 98, 61, 110, 0, 34, 76, 95, 41, 51, 43, 40, 106, 96, 67, 68, 64], [104, 38, 97, 103, 42, 43, 48, 56, 90, 45, 87, 110, 47, 125, 50, 120, 53, 121, 49, 124, 61, 126, 58, 123, 30, 6, 31, 122, 57, 62, 68, 12, 119, 84, 52, 67, 81, 29, 16, 4, 105, 70, 112, 75, 117, 77, 78, 28, 63, 127, 7, 51, 82, 59, 13, 10, 41, 76, 20, 66, 85, 1, 18, 9, 8, 83, 15, 55, 64, 100, 60, 89, 113, 108, 23, 115, 116, 69, 72, 88, 98, 24, 73, 3, 34, 2, 91, 118, 54, 17, 79, 65, 5, 14, 109, 35, 11, 21, 80, 86, 107, 101, 37, 32, 19, 71, 74, 106, 99, 44, 25, 0, 92, 93, 36, 22, 26, 96, 94, 27, 111, 114, 33, 46, 40, 102, 95, 39], [104, 38, 103, 97, 42, 43, 87, 47, 45, 110, 49, 53, 50, 123, 126, 61, 124, 6, 29, 57, 90, 120, 121, 48, 12, 66, 31, 13, 117, 64, 28, 59, 81, 56, 10, 68, 58, 122, 62, 78, 7, 44, 75, 105, 52, 67, 20, 125, 51, 16, 1, 72, 119, 54, 15, 2, 9, 55, 18, 69, 115, 113, 82, 127, 100, 80, 11, 109, 91, 99, 4, 35, 96, 8, 85, 71, 63, 60, 5, 116, 70, 17, 118, 22, 101, 3, 36, 84, 37, 89, 107, 14, 23, 46, 73, 79, 106, 112, 74, 27, 77, 30, 25, 76, 65, 21, 41, 98, 108, 24, 19, 88, 0, 114, 26, 32, 34, 111, 86, 83, 94, 33, 92, 93, 40, 95, 39, 102], [104, 103, 97, 38, 42, 43, 13, 49, 110, 47, 53, 50, 121, 56, 126, 59, 45, 87, 123, 124, 61, 6, 122, 48, 31, 117, 120, 29, 37, 78, 81, 90, 127, 28, 118, 119, 67, 51, 12, 68, 16, 55, 105, 70, 113, 58, 52, 82, 10, 35, 19, 72, 125, 8, 69, 24, 71, 66, 79, 75, 1, 23, 115, 112, 57, 54, 11, 62, 91, 73, 116, 18, 7, 85, 60, 25, 80, 44, 101, 20, 98, 30, 114, 41, 84, 17, 15, 76, 108, 88, 32, 63, 89, 34, 77, 46, 96, 27, 4, 64, 74, 14, 36, 100, 22, 21, 5, 107, 94, 9, 86, 92, 99, 2, 3, 65, 106, 83, 111, 26, 93, 109, 33, 0, 95, 102, 40, 39], [120, 49, 97, 124, 57, 103, 61, 104, 59, 87, 38, 18, 105, 121, 42, 122, 43, 54, 53, 50, 23, 31, 47, 35, 45, 48, 110, 101, 92, 126, 82, 84, 123, 117, 113, 56, 118, 115, 127, 51, 62, 41, 55, 109, 90, 12, 27, 89, 28, 114, 94, 91, 108, 81, 19, 26, 102, 44, 58, 16, 77, 78, 60, 88, 37, 15, 116, 95, 80, 119, 100, 107, 75, 6, 10, 29, 52, 20, 36, 111, 72, 46, 13, 73, 79, 74, 25, 22, 85, 33, 63, 106, 9, 86, 68, 30, 32, 11, 17, 98, 4, 93, 67, 71, 7, 125, 66, 1, 24, 76, 21, 96, 112, 3, 99, 69, 83, 34, 14, 8, 64, 70, 5, 39, 65, 2, 40, 0], [107, 42, 41, 40, 103, 33, 46, 118, 95, 24, 126, 109, 116, 125, 18, 14, 82, 115, 114, 63, 53, 56, 93, 113, 119, 111, 112, 58, 84, 54, 108, 117, 85, 59, 55, 49, 83, 60, 121, 20, 79, 80, 61, 87, 13, 27, 75, 76, 25, 50, 21, 99, 110, 48, 51, 124, 90, 92, 26, 7, 38, 89, 86, 77, 71, 94, 123, 8, 81, 74, 29, 23, 72, 120, 10, 15, 127, 22, 37, 32, 91, 96, 98, 73, 30, 52, 16, 34, 12, 122, 28, 11, 106, 47, 19, 31, 17, 35, 45, 102, 78, 97, 43, 9, 101, 88, 70, 36, 69, 67, 44, 66, 57, 3, 4, 65, 100, 62, 68, 104, 5, 105, 39, 0, 6, 1, 2, 64], [101, 98, 120, 108, 112, 127, 114, 47, 113, 34, 61, 49, 115, 62, 59, 122, 32, 111, 30, 118, 58, 36, 121, 119, 46, 48, 60, 123, 42, 50, 55, 53, 63, 94, 124, 41, 40, 125, 116, 37, 35, 126, 107, 56, 109, 44, 117, 54, 100, 88, 103, 38, 89, 22, 91, 45, 21, 68, 84, 102, 33, 57, 39, 85, 51, 25, 1, 93, 92, 6, 11, 64, 4, 52, 77, 69, 67, 8, 0, 71, 86, 99, 12, 24, 83, 95, 96, 97, 10, 81, 78, 80, 28, 9, 18, 110, 16, 7, 26, 2, 66, 104, 13, 79, 72, 31, 15, 82, 20, 3, 5, 19, 14, 65, 23, 73, 27, 70, 75, 76, 90, 43, 87, 29, 74, 17, 106, 105], [40, 42, 107, 41, 33, 112, 95, 46, 103, 116, 93, 20, 115, 114, 18, 121, 109, 27, 122, 15, 118, 78, 60, 16, 19, 72, 58, 54, 92, 25, 126, 125, 53, 17, 81, 26, 23, 119, 108, 12, 111, 13, 62, 83, 22, 56, 10, 49, 117, 48, 77, 55, 124, 11, 68, 120, 79, 14, 34, 87, 71, 113, 50, 29, 86, 59, 21, 74, 3, 57, 75, 127, 67, 5, 76, 80, 8, 104, 70, 89, 66, 84, 6, 61, 36, 82, 91, 9, 51, 98, 73, 38, 37, 90, 106, 85, 97, 69, 99, 28, 24, 52, 105, 1, 39, 31, 110, 63, 32, 2, 45, 4, 123, 94, 101, 88, 96, 43, 7, 47, 44, 30, 102, 65, 100, 35, 0, 64], [107, 41, 42, 103, 40, 109, 33, 85, 112, 108, 46, 95, 126, 54, 24, 66, 49, 68, 111, 117, 113, 38, 4, 114, 62, 122, 119, 125, 56, 22, 27, 1, 93, 61, 7, 118, 32, 36, 53, 18, 0, 17, 121, 10, 71, 6, 23, 8, 94, 78, 67, 2, 120, 98, 115, 116, 127, 59, 25, 81, 92, 60, 11, 21, 110, 37, 9, 15, 55, 80, 30, 57, 69, 77, 48, 64, 75, 12, 100, 3, 20, 19, 88, 63, 82, 123, 13, 16, 73, 34, 52, 5, 96, 124, 87, 70, 51, 101, 26, 35, 45, 99, 14, 79, 74, 43, 65, 58, 102, 29, 50, 83, 76, 86, 72, 84, 47, 28, 90, 91, 106, 44, 89, 97, 31, 105, 39, 104], [52, 55, 102, 57, 117, 111, 56, 95, 109, 54, 51, 101, 114, 63, 97, 53, 50, 127, 59, 104, 48, 122, 24, 86, 61, 98, 103, 90, 100, 112, 88, 26, 93, 33, 115, 41, 80, 110, 29, 31, 42, 116, 96, 125, 108, 16, 113, 85, 120, 27, 124, 126, 121, 60, 46, 49, 44, 22, 123, 74, 99, 37, 119, 30, 39, 17, 62, 81, 118, 107, 47, 58, 38, 43, 32, 40, 94, 10, 25, 20, 28, 21, 36, 45, 23, 91, 35, 34, 105, 84, 19, 92, 12, 76, 87, 106, 14, 89, 11, 18, 82, 15, 69, 78, 7, 13, 3, 75, 77, 72, 9, 8, 83, 5, 67, 79, 73, 6, 71, 4, 66, 2, 70, 68, 65, 0, 64, 1], [103, 104, 42, 41, 0, 33, 90, 44, 114, 109, 111, 63, 56, 59, 122, 102, 29, 61, 86, 127, 53, 101, 125, 121, 54, 51, 3, 95, 66, 16, 52, 10, 4, 112, 107, 48, 46, 83, 65, 50, 120, 84, 77, 49, 7, 76, 15, 69, 55, 71, 19, 115, 74, 12, 57, 25, 28, 78, 17, 72, 6, 85, 14, 91, 5, 9, 100, 123, 118, 80, 1, 126, 64, 124, 47, 21, 70, 13, 18, 2, 22, 75, 117, 23, 81, 68, 82, 11, 106, 87, 73, 113, 88, 67, 94, 96, 79, 105, 116, 108, 35, 110, 8, 20, 34, 39, 40, 98, 26, 24, 38, 30, 60, 36, 62, 99, 58, 89, 27, 37, 43, 45, 119, 32, 93, 92, 31, 97], [46, 44, 33, 52, 24, 111, 104, 103, 109, 124, 41, 102, 123, 57, 125, 107, 120, 108, 51, 114, 122, 61, 127, 59, 54, 90, 56, 85, 48, 29, 42, 53, 63, 95, 86, 55, 116, 76, 101, 50, 27, 80, 12, 22, 58, 60, 37, 110, 113, 26, 88, 119, 47, 21, 97, 81, 74, 93, 126, 96, 112, 34, 19, 117, 83, 7, 23, 3, 20, 15, 25, 10, 49, 77, 18, 43, 31, 89, 4, 84, 100, 0, 115, 17, 69, 98, 99, 78, 121, 87, 16, 9, 38, 28, 66, 14, 75, 92, 30, 11, 32, 91, 6, 62, 94, 105, 65, 118, 45, 106, 13, 8, 36, 35, 70, 79, 72, 67, 82, 71, 2, 68, 64, 73, 5, 1, 40, 39], [103, 57, 44, 101, 104, 41, 42, 33, 52, 109, 90, 55, 114, 46, 12, 111, 29, 54, 63, 56, 59, 127, 122, 61, 53, 51, 125, 95, 85, 72, 112, 81, 76, 48, 17, 3, 50, 11, 4, 100, 7, 121, 73, 115, 15, 120, 47, 69, 82, 19, 66, 80, 65, 23, 86, 78, 28, 67, 123, 102, 107, 0, 20, 22, 13, 16, 84, 88, 70, 10, 91, 30, 9, 126, 94, 5, 18, 14, 71, 83, 110, 25, 34, 58, 21, 32, 49, 6, 89, 2, 26, 8, 124, 77, 93, 64, 68, 75, 62, 45, 79, 117, 113, 87, 96, 43, 92, 24, 60, 98, 38, 106, 99, 27, 105, 1, 74, 36, 119, 108, 118, 31, 116, 35, 37, 97, 40, 39], [104, 101, 105, 45, 110, 97, 44, 106, 43, 64, 114, 95, 89, 113, 56, 116, 57, 93, 48, 119, 47, 58, 50, 68, 1, 15, 6, 70, 9, 84, 2, 76, 81, 16, 13, 74, 3, 14, 69, 11, 80, 20, 55, 0, 65, 60, 28, 72, 78, 7, 63, 17, 115, 71, 36, 46, 66, 8, 19, 121, 5, 109, 52, 73, 4, 118, 25, 40, 75, 34, 108, 41, 126, 82, 112, 59, 107, 111, 122, 61, 51, 42, 124, 123, 35, 125, 99, 62, 98, 67, 79, 88, 54, 85, 117, 49, 103, 27, 102, 38, 53, 32, 96, 91, 100, 90, 10, 21, 24, 12, 30, 22, 26, 127, 29, 86, 87, 83, 120, 77, 94, 23, 39, 18, 37, 92, 33, 31], [114, 104, 101, 97, 105, 45, 69, 44, 89, 110, 106, 43, 63, 95, 72, 75, 3, 93, 113, 56, 4, 57, 14, 55, 76, 119, 116, 58, 50, 48, 84, 47, 74, 20, 70, 67, 1, 9, 81, 80, 13, 60, 28, 118, 123, 16, 115, 18, 73, 64, 17, 11, 15, 22, 126, 78, 12, 7, 121, 46, 68, 77, 112, 62, 109, 65, 85, 61, 2, 83, 24, 23, 52, 127, 125, 108, 122, 6, 40, 51, 41, 79, 107, 10, 54, 26, 96, 42, 66, 21, 49, 19, 86, 25, 103, 38, 120, 37, 36, 124, 111, 90, 8, 102, 35, 87, 92, 27, 88, 91, 5, 82, 34, 100, 39, 117, 98, 94, 0, 71, 53, 32, 59, 29, 30, 99, 31, 33], [104, 64, 45, 110, 101, 105, 44, 106, 97, 43, 89, 113, 56, 57, 1, 119, 116, 0, 48, 58, 50, 47, 95, 93, 114, 68, 3, 84, 70, 123, 13, 2, 5, 63, 76, 28, 55, 60, 127, 59, 118, 14, 74, 122, 9, 81, 80, 15, 20, 46, 66, 8, 51, 16, 126, 36, 109, 112, 26, 40, 52, 4, 67, 41, 75, 108, 12, 69, 7, 11, 65, 107, 42, 77, 17, 53, 18, 78, 117, 61, 88, 6, 39, 35, 115, 72, 98, 111, 124, 32, 82, 71, 121, 91, 99, 86, 22, 85, 38, 62, 34, 10, 125, 73, 90, 49, 120, 19, 87, 30, 25, 21, 102, 83, 96, 27, 23, 100, 94, 24, 79, 92, 54, 103, 29, 31, 37, 33], [55, 63, 104, 97, 101, 72, 105, 45, 44, 110, 106, 43, 114, 52, 89, 93, 95, 77, 113, 58, 57, 56, 119, 116, 78, 15, 48, 60, 47, 83, 13, 62, 70, 50, 84, 115, 126, 80, 81, 118, 75, 8, 90, 17, 79, 38, 28, 91, 73, 20, 18, 12, 121, 112, 22, 125, 6, 69, 21, 16, 25, 86, 85, 46, 19, 37, 124, 29, 23, 9, 109, 76, 74, 102, 14, 61, 3, 10, 4, 59, 127, 111, 123, 108, 68, 71, 53, 92, 94, 36, 120, 24, 103, 35, 82, 54, 99, 27, 107, 51, 100, 11, 122, 117, 5, 88, 26, 41, 34, 96, 42, 87, 40, 32, 33, 67, 31, 30, 1, 49, 39, 98, 2, 7, 64, 65, 66, 0], [104, 0, 45, 106, 43, 103, 112, 1, 97, 102, 127, 2, 64, 116, 60, 93, 117, 67, 113, 4, 88, 115, 65, 50, 70, 71, 66, 57, 77, 78, 49, 63, 59, 69, 16, 76, 52, 62, 20, 48, 15, 73, 122, 74, 109, 22, 51, 58, 114, 123, 11, 81, 82, 8, 3, 124, 125, 107, 53, 87, 120, 17, 56, 83, 42, 19, 61, 84, 118, 18, 7, 24, 55, 40, 72, 39, 121, 86, 119, 6, 14, 54, 13, 12, 75, 10, 5, 38, 85, 9, 26, 21, 79, 92, 80, 89, 126, 111, 110, 44, 23, 68, 47, 32, 27, 25, 105, 98, 95, 108, 90, 28, 41, 91, 30, 100, 29, 37, 36, 96, 31, 34, 46, 94, 101, 33, 99, 35], [104, 103, 106, 45, 43, 97, 112, 102, 5, 0, 116, 3, 93, 68, 69, 127, 88, 20, 66, 4, 67, 60, 114, 117, 11, 71, 57, 70, 113, 49, 73, 115, 83, 2, 76, 122, 16, 82, 77, 78, 63, 51, 59, 48, 124, 8, 6, 22, 1, 15, 74, 81, 50, 65, 19, 58, 62, 52, 118, 123, 84, 85, 56, 120, 87, 75, 125, 61, 121, 109, 53, 55, 7, 64, 24, 17, 86, 54, 79, 107, 110, 80, 18, 72, 13, 42, 119, 10, 12, 14, 23, 47, 9, 111, 92, 108, 126, 29, 40, 27, 89, 44, 38, 21, 90, 46, 39, 28, 95, 25, 33, 26, 91, 31, 36, 41, 100, 30, 105, 94, 99, 37, 32, 96, 34, 35, 98, 101], [103, 114, 104, 116, 122, 59, 51, 97, 106, 48, 49, 43, 20, 124, 58, 18, 62, 88, 45, 57, 118, 63, 19, 55, 22, 102, 86, 108, 92, 123, 61, 79, 52, 112, 47, 125, 14, 121, 80, 126, 110, 120, 29, 33, 113, 17, 56, 75, 127, 84, 53, 82, 117, 24, 13, 115, 12, 50, 90, 46, 23, 93, 81, 26, 60, 119, 83, 91, 44, 10, 87, 27, 72, 54, 9, 32, 89, 15, 41, 111, 95, 94, 25, 28, 30, 36, 39, 38, 98, 77, 31, 85, 7, 37, 100, 6, 16, 99, 96, 105, 101, 5, 109, 35, 76, 78, 74, 21, 34, 73, 40, 11, 68, 8, 3, 71, 70, 66, 69, 4, 0, 1, 65, 67, 107, 2, 42, 64], [114, 118, 51, 103, 104, 49, 55, 53, 110, 125, 20, 123, 106, 122, 120, 121, 102, 56, 61, 43, 63, 124, 108, 126, 111, 88, 45, 19, 62, 119, 47, 54, 17, 58, 97, 18, 92, 22, 112, 79, 14, 80, 57, 41, 36, 99, 101, 100, 86, 116, 35, 90, 30, 98, 37, 34, 75, 59, 31, 44, 95, 32, 105, 96, 12, 13, 117, 127, 89, 91, 10, 50, 81, 33, 60, 24, 93, 72, 83, 94, 38, 9, 115, 46, 27, 52, 26, 7, 82, 113, 23, 84, 6, 87, 48, 29, 15, 5, 85, 109, 21, 28, 68, 25, 16, 11, 107, 3, 78, 39, 8, 77, 69, 66, 76, 42, 70, 65, 71, 74, 73, 4, 67, 40, 2, 0, 1, 64], [104, 102, 97, 44, 4, 103, 24, 107, 106, 29, 50, 126, 120, 55, 21, 62, 116, 59, 125, 95, 48, 3, 71, 121, 78, 67, 119, 45, 76, 0, 112, 82, 15, 65, 91, 123, 8, 92, 9, 49, 13, 1, 66, 10, 11, 68, 52, 122, 53, 80, 117, 51, 18, 110, 83, 6, 7, 58, 73, 77, 127, 75, 57, 113, 47, 101, 85, 108, 54, 81, 5, 20, 70, 16, 12, 111, 19, 43, 115, 74, 37, 32, 2, 69, 26, 42, 30, 56, 124, 64, 114, 46, 61, 72, 88, 96, 17, 118, 99, 22, 87, 89, 14, 98, 34, 63, 40, 41, 84, 27, 25, 94, 109, 79, 100, 60, 86, 23, 105, 36, 90, 35, 28, 93, 39, 33, 31, 38], [97, 102, 104, 44, 103, 105, 107, 106, 29, 24, 53, 50, 48, 126, 120, 116, 55, 119, 95, 57, 21, 121, 117, 125, 45, 82, 16, 123, 61, 78, 63, 15, 51, 62, 111, 11, 127, 92, 52, 13, 56, 9, 115, 5, 91, 72, 68, 12, 7, 59, 3, 2, 83, 6, 76, 118, 23, 108, 0, 49, 18, 54, 65, 30, 60, 75, 17, 80, 79, 89, 90, 77, 43, 98, 10, 32, 112, 74, 42, 19, 86, 81, 70, 14, 34, 84, 26, 25, 94, 35, 100, 27, 22, 85, 96, 36, 66, 124, 99, 37, 40, 28, 87, 20, 4, 8, 122, 113, 110, 109, 101, 47, 58, 71, 39, 31, 69, 67, 33, 73, 46, 64, 114, 41, 88, 38, 1, 93], [104, 102, 97, 44, 103, 29, 107, 106, 24, 50, 21, 126, 55, 120, 68, 82, 53, 52, 95, 11, 117, 78, 5, 13, 0, 125, 9, 59, 12, 62, 2, 6, 48, 92, 65, 15, 7, 45, 58, 51, 72, 3, 70, 116, 105, 18, 14, 123, 91, 46, 8, 121, 75, 57, 119, 109, 113, 122, 66, 76, 54, 26, 49, 110, 85, 88, 73, 17, 67, 71, 112, 30, 108, 114, 47, 60, 74, 80, 63, 23, 101, 1, 64, 79, 61, 43, 10, 22, 77, 41, 4, 42, 19, 111, 56, 37, 35, 124, 16, 90, 118, 84, 87, 34, 20, 69, 127, 100, 96, 115, 89, 36, 40, 32, 33, 94, 98, 83, 99, 25, 86, 81, 27, 28, 93, 38, 31, 39], [103, 102, 104, 105, 48, 97, 59, 44, 106, 107, 24, 116, 29, 117, 78, 21, 11, 120, 50, 126, 52, 16, 62, 55, 82, 53, 57, 74, 13, 51, 9, 18, 123, 95, 110, 119, 121, 75, 2, 15, 54, 85, 68, 125, 19, 122, 7, 88, 6, 70, 14, 112, 72, 92, 8, 80, 3, 45, 113, 30, 109, 66, 0, 17, 26, 76, 71, 65, 49, 58, 5, 1, 115, 114, 91, 47, 60, 22, 4, 77, 108, 67, 64, 10, 111, 32, 63, 124, 12, 87, 25, 43, 93, 56, 84, 86, 79, 83, 96, 42, 23, 41, 118, 94, 20, 61, 27, 69, 100, 127, 73, 46, 101, 98, 37, 40, 34, 81, 89, 90, 38, 36, 35, 99, 39, 28, 31, 33], [114, 103, 98, 44, 43, 49, 57, 48, 97, 105, 122, 95, 40, 14, 127, 63, 121, 107, 124, 83, 19, 24, 21, 120, 42, 78, 119, 94, 54, 32, 90, 58, 117, 110, 111, 74, 51, 92, 53, 84, 25, 26, 62, 29, 36, 18, 86, 109, 38, 115, 50, 46, 56, 112, 116, 15, 59, 123, 113, 85, 81, 33, 87, 99, 125, 88, 10, 17, 22, 31, 60, 100, 104, 118, 126, 82, 70, 93, 91, 52, 61, 28, 37, 55, 101, 108, 30, 35, 79, 47, 45, 6, 8, 102, 23, 16, 39, 13, 76, 11, 89, 77, 96, 12, 20, 34, 80, 27, 7, 9, 72, 4, 106, 75, 3, 68, 66, 73, 2, 41, 71, 65, 5, 67, 1, 69, 64, 0], [44, 63, 98, 62, 60, 111, 51, 49, 103, 124, 43, 59, 48, 127, 42, 97, 86, 17, 105, 22, 24, 120, 95, 55, 114, 92, 29, 50, 40, 99, 113, 110, 45, 90, 102, 25, 18, 57, 37, 36, 54, 77, 61, 47, 85, 28, 88, 94, 33, 74, 21, 87, 84, 101, 93, 100, 31, 26, 27, 108, 123, 125, 14, 91, 118, 35, 116, 115, 38, 117, 10, 52, 112, 121, 20, 56, 15, 109, 30, 81, 89, 13, 82, 78, 122, 104, 70, 83, 39, 126, 119, 96, 58, 32, 8, 19, 53, 71, 66, 7, 16, 4, 79, 34, 23, 107, 2, 46, 72, 67, 11, 68, 6, 41, 106, 80, 12, 0, 75, 1, 69, 64, 3, 73, 65, 9, 5, 76], [103, 105, 97, 42, 44, 111, 40, 121, 89, 95, 9, 119, 22, 58, 49, 57, 4, 48, 120, 124, 94, 62, 122, 65, 50, 60, 67, 11, 109, 51, 13, 59, 56, 92, 43, 99, 18, 79, 53, 127, 55, 16, 0, 70, 84, 63, 54, 5, 1, 73, 36, 81, 24, 26, 114, 46, 20, 71, 6, 98, 2, 82, 90, 112, 23, 66, 80, 86, 3, 77, 74, 85, 113, 100, 126, 12, 72, 76, 10, 7, 32, 38, 21, 115, 91, 78, 17, 118, 83, 125, 123, 27, 19, 29, 69, 68, 25, 47, 45, 102, 88, 64, 75, 8, 61, 14, 93, 116, 28, 52, 110, 34, 31, 37, 117, 87, 30, 15, 35, 101, 33, 96, 41, 107, 39, 108, 104, 106], [40, 109, 43, 44, 49, 97, 42, 103, 48, 53, 124, 105, 37, 118, 98, 120, 127, 62, 57, 56, 36, 95, 29, 51, 60, 59, 74, 92, 114, 22, 90, 84, 121, 55, 78, 81, 63, 10, 89, 54, 122, 16, 77, 58, 108, 24, 15, 115, 70, 38, 126, 119, 85, 86, 111, 125, 13, 47, 61, 82, 21, 25, 123, 8, 101, 93, 94, 83, 87, 45, 11, 102, 76, 19, 79, 18, 99, 20, 28, 88, 46, 116, 68, 80, 117, 32, 52, 100, 106, 9, 14, 17, 23, 67, 1, 4, 75, 96, 110, 26, 41, 2, 112, 33, 91, 113, 27, 30, 35, 50, 71, 107, 34, 72, 6, 12, 0, 3, 7, 64, 73, 69, 66, 39, 5, 31, 104, 65]], "model.layers.1.self_attn.k_proj": [[0, 39, 40, 43, 110, 4, 97, 11, 106, 41, 65, 95, 78, 84, 79, 62, 117, 74, 123, 9, 66, 50, 72, 122, 93, 5, 57, 87, 91, 73, 26, 47, 13, 69, 28, 68, 86, 16, 70, 7, 2, 59, 77, 81, 53, 112, 56, 67, 127, 17, 21, 46, 48, 10, 82, 121, 3, 64, 116, 76, 55, 63, 118, 124, 115, 58, 6, 114, 30, 89, 71, 113, 18, 51, 111, 14, 8, 108, 96, 54, 12, 120, 85, 60, 61, 109, 126, 75, 119, 15, 80, 125, 23, 1, 88, 52, 29, 49, 103, 38, 83, 94, 19, 34, 32, 45, 99, 33, 90, 36, 104, 27, 101, 22, 20, 24, 92, 98, 44, 105, 35, 100, 31, 107, 102, 37, 25, 42], [64, 40, 106, 107, 39, 46, 69, 111, 1, 33, 10, 66, 93, 7, 65, 2, 5, 113, 109, 95, 67, 75, 72, 117, 126, 12, 3, 123, 16, 112, 114, 81, 9, 61, 50, 4, 78, 53, 6, 102, 60, 124, 85, 15, 23, 47, 116, 21, 26, 125, 0, 92, 63, 27, 74, 71, 110, 108, 86, 22, 58, 68, 62, 11, 56, 70, 121, 83, 49, 80, 24, 19, 82, 94, 118, 55, 77, 17, 8, 98, 127, 57, 59, 52, 34, 51, 73, 25, 122, 119, 96, 14, 32, 99, 101, 87, 18, 37, 36, 79, 88, 76, 115, 84, 41, 20, 48, 91, 44, 35, 54, 89, 100, 28, 30, 97, 43, 105, 90, 120, 29, 45, 13, 42, 38, 31, 104, 103], [39, 97, 43, 105, 104, 106, 110, 31, 90, 29, 87, 45, 48, 54, 117, 83, 91, 81, 12, 15, 44, 78, 51, 28, 77, 10, 20, 0, 11, 89, 6, 122, 69, 7, 50, 2, 16, 126, 17, 79, 56, 125, 86, 8, 18, 52, 116, 65, 84, 60, 62, 1, 121, 13, 3, 55, 9, 73, 57, 19, 124, 53, 4, 119, 35, 58, 115, 64, 47, 75, 118, 59, 67, 49, 74, 5, 102, 114, 21, 61, 127, 24, 112, 103, 120, 80, 111, 34, 113, 46, 98, 23, 72, 96, 40, 101, 41, 68, 71, 93, 27, 32, 70, 107, 100, 30, 99, 42, 85, 36, 63, 82, 66, 37, 123, 88, 38, 94, 76, 109, 22, 108, 14, 26, 92, 33, 25, 95], [45, 0, 106, 105, 40, 39, 1, 65, 97, 108, 50, 47, 92, 31, 9, 61, 15, 127, 77, 63, 7, 122, 3, 6, 19, 110, 53, 59, 48, 4, 51, 118, 2, 120, 54, 68, 56, 23, 64, 49, 66, 43, 27, 114, 35, 87, 93, 126, 69, 70, 5, 36, 89, 79, 11, 71, 82, 8, 20, 119, 94, 75, 18, 121, 98, 62, 125, 76, 81, 37, 12, 78, 38, 32, 83, 22, 74, 58, 14, 111, 34, 13, 67, 115, 57, 84, 60, 109, 112, 99, 26, 21, 30, 123, 17, 116, 46, 73, 96, 113, 85, 107, 103, 80, 117, 124, 25, 33, 10, 24, 86, 91, 29, 104, 42, 88, 41, 72, 55, 100, 90, 44, 102, 101, 28, 52, 16, 95], [64, 40, 109, 65, 46, 108, 41, 107, 42, 66, 50, 0, 37, 3, 119, 57, 56, 33, 113, 116, 67, 48, 1, 74, 5, 58, 31, 68, 6, 76, 4, 14, 9, 13, 25, 47, 29, 71, 49, 20, 111, 11, 81, 112, 8, 7, 92, 15, 16, 2, 80, 61, 51, 94, 110, 85, 10, 60, 24, 84, 21, 123, 70, 59, 118, 126, 63, 100, 96, 54, 82, 83, 23, 102, 30, 79, 45, 39, 121, 99, 86, 19, 104, 73, 98, 27, 77, 12, 103, 32, 75, 91, 88, 87, 117, 90, 69, 38, 105, 120, 22, 44, 35, 18, 34, 106, 125, 17, 62, 78, 53, 43, 115, 122, 127, 52, 114, 124, 26, 55, 72, 89, 36, 28, 93, 101, 97, 95], [40, 0, 109, 42, 107, 39, 33, 48, 65, 64, 66, 113, 50, 52, 3, 115, 29, 117, 127, 60, 71, 70, 4, 16, 24, 73, 69, 67, 77, 68, 116, 57, 74, 76, 8, 11, 58, 82, 28, 83, 78, 63, 59, 62, 15, 86, 2, 124, 112, 84, 23, 54, 121, 122, 46, 81, 119, 125, 123, 120, 61, 26, 44, 5, 6, 1, 7, 41, 101, 37, 100, 111, 94, 99, 56, 27, 36, 35, 38, 96, 31, 98, 32, 105, 34, 55, 126, 118, 47, 87, 25, 9, 95, 45, 108, 110, 51, 114, 49, 91, 53, 30, 20, 88, 85, 90, 21, 72, 10, 19, 89, 17, 102, 14, 43, 12, 92, 106, 18, 79, 104, 22, 13, 75, 103, 93, 80, 97], [0, 108, 40, 43, 42, 39, 65, 55, 120, 126, 114, 3, 33, 50, 72, 117, 6, 2, 52, 7, 9, 41, 31, 109, 68, 5, 48, 70, 15, 125, 49, 13, 121, 62, 18, 12, 112, 78, 74, 115, 46, 85, 11, 110, 76, 69, 27, 93, 59, 116, 123, 28, 67, 38, 53, 88, 1, 4, 16, 82, 17, 20, 118, 84, 86, 90, 80, 63, 61, 60, 19, 47, 37, 83, 111, 96, 81, 57, 35, 87, 54, 21, 99, 75, 89, 36, 22, 51, 10, 73, 77, 98, 100, 25, 24, 66, 94, 23, 79, 30, 113, 34, 32, 14, 127, 124, 8, 119, 26, 71, 58, 101, 122, 44, 102, 45, 29, 64, 56, 97, 92, 105, 95, 91, 107, 106, 104, 103], [39, 41, 106, 33, 108, 104, 31, 0, 69, 34, 57, 113, 120, 25, 16, 112, 30, 47, 45, 1, 67, 84, 119, 46, 87, 2, 122, 124, 107, 5, 8, 51, 11, 121, 58, 71, 62, 68, 53, 9, 56, 35, 59, 15, 18, 60, 90, 92, 13, 125, 96, 86, 70, 76, 12, 7, 61, 24, 36, 81, 49, 4, 27, 126, 55, 123, 50, 66, 63, 127, 73, 101, 52, 75, 117, 77, 89, 54, 38, 85, 21, 110, 48, 102, 100, 28, 29, 72, 93, 22, 116, 118, 80, 97, 115, 3, 82, 43, 64, 95, 74, 23, 65, 19, 111, 20, 88, 17, 91, 98, 26, 37, 83, 99, 109, 32, 114, 79, 103, 6, 14, 44, 10, 94, 78, 40, 42, 105]], "model.layers.1.self_attn.qk_proj": [[50, 48, 0, 57, 64, 117, 56, 116, 59, 126, 113, 120, 122, 62, 123, 53, 55, 49, 65, 127, 46, 110, 1, 63, 97, 61, 124, 47, 58, 109, 67, 119, 121, 29, 3, 112, 108, 54, 51, 40, 60, 107, 4, 45, 125, 114, 43, 104, 70, 68, 2, 76, 33, 5, 66, 16, 42, 106, 95, 7, 77, 111, 93, 14, 6, 52, 41, 9, 13, 20, 11, 115, 84, 12, 71, 82, 75, 74, 81, 44, 73, 79, 69, 103, 78, 39, 10, 17, 15, 18, 72, 80, 87, 105, 24, 118, 31, 23, 8, 22, 83, 19, 90, 86, 85, 92, 89, 21, 25, 88, 26, 91, 102, 28, 36, 38, 27, 37, 98, 94, 101, 35, 100, 96, 34, 99, 32, 30], [50, 48, 0, 64, 57, 117, 56, 120, 59, 126, 116, 62, 113, 122, 46, 53, 55, 123, 127, 63, 65, 1, 110, 58, 49, 97, 61, 109, 47, 108, 124, 29, 40, 112, 121, 51, 68, 119, 114, 60, 43, 107, 54, 4, 45, 125, 66, 104, 2, 42, 67, 6, 70, 52, 106, 93, 33, 7, 95, 69, 3, 115, 77, 14, 20, 12, 41, 5, 76, 13, 111, 9, 71, 16, 44, 103, 11, 39, 84, 82, 17, 73, 105, 10, 18, 118, 78, 74, 79, 72, 75, 31, 80, 15, 81, 24, 87, 23, 8, 89, 88, 22, 83, 85, 102, 26, 19, 25, 90, 92, 21, 28, 86, 27, 98, 91, 38, 101, 37, 35, 30, 94, 36, 32, 100, 34, 99, 96], [50, 48, 57, 64, 0, 117, 56, 116, 120, 59, 126, 113, 62, 122, 53, 46, 123, 63, 127, 65, 110, 55, 58, 108, 97, 1, 47, 109, 51, 49, 29, 112, 121, 61, 124, 119, 67, 40, 3, 54, 114, 107, 60, 43, 6, 45, 4, 104, 125, 70, 2, 42, 66, 52, 106, 95, 68, 13, 71, 93, 115, 33, 5, 14, 76, 84, 111, 69, 11, 41, 16, 77, 12, 20, 9, 7, 73, 75, 15, 10, 39, 79, 78, 82, 18, 17, 103, 44, 81, 74, 72, 80, 118, 105, 31, 24, 87, 8, 23, 83, 22, 19, 89, 86, 88, 21, 85, 28, 25, 92, 102, 90, 26, 91, 98, 27, 37, 101, 38, 30, 94, 32, 35, 100, 36, 99, 96, 34], [50, 48, 0, 64, 57, 117, 56, 116, 59, 120, 126, 113, 122, 62, 123, 46, 53, 65, 127, 55, 58, 110, 63, 124, 49, 1, 97, 3, 61, 47, 108, 121, 109, 51, 112, 29, 114, 67, 54, 40, 119, 60, 6, 45, 2, 107, 104, 43, 125, 66, 52, 42, 5, 95, 106, 68, 115, 70, 111, 71, 93, 69, 20, 4, 76, 33, 41, 11, 12, 13, 77, 9, 14, 7, 103, 16, 78, 10, 84, 44, 79, 81, 80, 39, 82, 18, 73, 17, 31, 74, 75, 105, 72, 24, 118, 15, 23, 22, 8, 87, 88, 83, 86, 21, 89, 19, 92, 27, 90, 25, 85, 102, 28, 26, 91, 37, 38, 101, 98, 100, 94, 34, 35, 36, 30, 96, 32, 99], [50, 48, 64, 0, 57, 117, 56, 120, 116, 126, 62, 59, 113, 122, 46, 123, 63, 53, 58, 110, 127, 55, 108, 49, 1, 109, 61, 124, 47, 40, 65, 29, 97, 51, 54, 107, 112, 43, 121, 45, 68, 2, 66, 119, 4, 6, 104, 60, 114, 106, 42, 52, 125, 67, 3, 5, 70, 93, 115, 41, 111, 33, 71, 95, 77, 13, 103, 16, 69, 7, 44, 39, 20, 80, 12, 75, 84, 9, 14, 74, 17, 11, 76, 78, 73, 10, 105, 81, 79, 15, 72, 82, 31, 18, 8, 118, 87, 23, 24, 22, 83, 19, 86, 102, 89, 25, 88, 27, 21, 92, 28, 90, 91, 85, 26, 38, 101, 98, 37, 94, 36, 34, 100, 99, 32, 30, 35, 96], [50, 48, 57, 0, 64, 117, 116, 56, 126, 59, 120, 122, 113, 62, 46, 123, 55, 53, 110, 58, 109, 63, 49, 127, 97, 124, 29, 1, 61, 40, 108, 47, 65, 3, 43, 112, 67, 114, 51, 107, 54, 119, 121, 4, 2, 125, 104, 68, 45, 60, 6, 42, 52, 106, 66, 33, 93, 20, 41, 95, 70, 77, 13, 80, 111, 12, 115, 11, 69, 71, 10, 5, 84, 7, 16, 39, 74, 103, 9, 82, 14, 17, 76, 73, 79, 44, 78, 15, 75, 72, 81, 8, 105, 31, 118, 23, 18, 87, 24, 88, 22, 102, 83, 85, 21, 19, 89, 92, 26, 25, 86, 90, 91, 27, 28, 38, 101, 37, 36, 98, 100, 35, 34, 94, 99, 96, 30, 32], [50, 48, 57, 117, 0, 64, 120, 56, 59, 116, 122, 126, 62, 113, 123, 55, 46, 53, 110, 127, 29, 58, 108, 63, 49, 97, 1, 124, 47, 61, 109, 65, 51, 112, 114, 40, 121, 67, 119, 54, 3, 43, 107, 125, 52, 60, 45, 104, 70, 2, 93, 6, 42, 66, 115, 106, 33, 13, 20, 16, 77, 80, 95, 69, 71, 12, 68, 11, 84, 111, 4, 75, 78, 41, 103, 79, 9, 15, 7, 76, 82, 18, 14, 81, 8, 5, 73, 39, 10, 74, 44, 17, 72, 105, 118, 23, 87, 31, 24, 85, 22, 19, 26, 92, 83, 25, 89, 86, 21, 90, 88, 102, 27, 28, 91, 37, 98, 36, 101, 94, 38, 35, 34, 30, 99, 96, 100, 32], [50, 48, 57, 0, 117, 64, 56, 116, 122, 59, 120, 113, 62, 126, 123, 53, 55, 46, 127, 110, 63, 97, 1, 49, 108, 65, 58, 112, 29, 109, 54, 61, 119, 47, 124, 114, 40, 121, 51, 125, 60, 107, 45, 43, 70, 104, 4, 52, 77, 106, 42, 95, 115, 33, 93, 6, 68, 66, 71, 13, 11, 2, 3, 15, 20, 14, 12, 75, 80, 67, 84, 41, 7, 16, 9, 78, 82, 44, 69, 111, 5, 39, 76, 103, 73, 17, 81, 79, 74, 18, 10, 8, 105, 31, 24, 118, 87, 23, 21, 19, 72, 25, 86, 22, 83, 85, 89, 90, 92, 26, 88, 27, 102, 28, 98, 91, 37, 101, 36, 94, 38, 30, 34, 32, 100, 35, 96, 99], [50, 48, 57, 64, 0, 117, 56, 120, 59, 116, 122, 126, 62, 113, 123, 53, 63, 46, 127, 1, 97, 110, 109, 61, 55, 108, 47, 124, 65, 40, 49, 112, 58, 29, 119, 54, 121, 3, 67, 51, 107, 43, 60, 125, 114, 68, 45, 4, 42, 104, 70, 106, 66, 95, 52, 2, 20, 115, 33, 11, 77, 13, 93, 6, 41, 71, 76, 15, 7, 14, 39, 16, 5, 84, 82, 78, 111, 80, 12, 103, 73, 9, 74, 81, 75, 69, 44, 18, 17, 10, 31, 79, 24, 105, 87, 8, 118, 23, 72, 85, 19, 83, 25, 90, 88, 89, 22, 86, 21, 92, 102, 28, 26, 27, 91, 98, 38, 101, 36, 37, 34, 35, 94, 96, 32, 100, 30, 99], [50, 48, 57, 64, 0, 117, 56, 59, 116, 126, 62, 122, 120, 113, 123, 46, 53, 97, 108, 127, 55, 65, 110, 63, 47, 1, 109, 49, 112, 61, 58, 124, 29, 119, 40, 54, 114, 51, 121, 107, 43, 67, 60, 45, 3, 104, 125, 70, 52, 42, 68, 66, 115, 106, 95, 33, 4, 15, 20, 77, 6, 2, 13, 93, 12, 14, 75, 79, 41, 69, 78, 5, 84, 7, 76, 80, 11, 82, 71, 39, 73, 103, 44, 18, 16, 81, 9, 10, 111, 87, 17, 8, 74, 118, 31, 24, 105, 23, 21, 22, 19, 25, 85, 88, 92, 90, 89, 83, 26, 72, 86, 28, 102, 27, 98, 37, 38, 91, 101, 34, 35, 96, 100, 30, 94, 32, 36, 99], [50, 48, 64, 57, 0, 117, 56, 116, 59, 126, 120, 113, 122, 123, 62, 65, 53, 46, 55, 1, 63, 58, 110, 127, 49, 97, 61, 124, 108, 109, 47, 54, 121, 51, 29, 40, 45, 112, 119, 114, 125, 66, 60, 43, 107, 70, 2, 67, 104, 3, 42, 52, 106, 33, 95, 115, 41, 71, 68, 20, 111, 13, 4, 93, 7, 5, 12, 77, 6, 10, 76, 84, 78, 14, 39, 69, 17, 81, 103, 16, 75, 73, 74, 82, 44, 9, 11, 80, 15, 31, 105, 79, 118, 18, 24, 8, 72, 87, 23, 89, 22, 19, 83, 25, 88, 92, 86, 28, 85, 21, 90, 102, 26, 101, 38, 91, 37, 27, 98, 35, 36, 94, 100, 34, 32, 96, 30, 99], [50, 48, 64, 0, 57, 117, 56, 126, 116, 59, 120, 62, 122, 113, 46, 123, 55, 53, 110, 58, 63, 127, 49, 65, 1, 67, 108, 3, 109, 61, 124, 40, 97, 51, 47, 29, 119, 121, 43, 112, 4, 60, 107, 54, 114, 45, 125, 68, 104, 2, 66, 42, 106, 70, 52, 71, 6, 111, 41, 33, 77, 95, 93, 115, 20, 69, 16, 13, 39, 5, 103, 10, 80, 7, 75, 11, 84, 73, 15, 44, 12, 74, 78, 76, 17, 9, 14, 82, 105, 81, 8, 18, 31, 79, 23, 118, 87, 24, 72, 19, 21, 88, 22, 83, 25, 90, 85, 89, 102, 26, 86, 92, 28, 91, 27, 101, 38, 98, 36, 37, 94, 35, 32, 30, 34, 99, 100, 96], [50, 48, 57, 64, 117, 0, 116, 120, 56, 122, 62, 126, 113, 59, 123, 46, 110, 53, 127, 55, 97, 109, 63, 108, 40, 49, 112, 61, 47, 1, 58, 124, 65, 29, 119, 51, 114, 54, 107, 121, 43, 3, 4, 60, 6, 42, 52, 104, 67, 125, 106, 45, 68, 2, 66, 115, 33, 95, 70, 15, 79, 16, 20, 93, 41, 13, 84, 11, 12, 77, 39, 71, 5, 44, 76, 14, 75, 73, 103, 78, 82, 7, 18, 9, 69, 10, 105, 80, 17, 74, 31, 111, 81, 87, 118, 24, 72, 8, 23, 88, 22, 26, 19, 25, 21, 92, 85, 86, 83, 90, 89, 28, 27, 102, 91, 98, 101, 38, 37, 34, 30, 36, 35, 94, 32, 99, 100, 96], [50, 48, 57, 64, 0, 117, 56, 116, 59, 113, 126, 122, 62, 120, 123, 53, 46, 97, 63, 127, 110, 55, 49, 1, 61, 109, 58, 124, 65, 108, 112, 47, 29, 121, 54, 40, 51, 114, 119, 60, 43, 107, 125, 6, 104, 52, 2, 67, 45, 42, 66, 33, 95, 106, 70, 3, 68, 4, 41, 115, 77, 76, 93, 5, 7, 16, 13, 78, 12, 20, 71, 84, 9, 39, 69, 79, 11, 18, 44, 10, 15, 80, 75, 103, 81, 74, 111, 14, 82, 73, 17, 72, 31, 24, 8, 105, 118, 87, 23, 19, 85, 88, 21, 86, 92, 25, 89, 22, 26, 83, 102, 90, 28, 27, 91, 38, 36, 98, 37, 101, 35, 94, 34, 100, 32, 99, 30, 96], [50, 48, 0, 57, 117, 64, 56, 120, 116, 59, 126, 122, 62, 113, 123, 53, 46, 108, 63, 55, 110, 127, 1, 49, 47, 61, 97, 65, 29, 58, 124, 3, 109, 67, 112, 119, 121, 40, 114, 51, 54, 60, 107, 6, 52, 43, 125, 2, 4, 45, 104, 66, 68, 106, 42, 33, 5, 95, 70, 93, 111, 71, 13, 77, 115, 41, 75, 7, 12, 79, 9, 39, 69, 20, 78, 84, 76, 73, 103, 82, 18, 14, 44, 80, 16, 10, 11, 74, 17, 81, 87, 72, 15, 105, 118, 24, 31, 8, 23, 90, 83, 22, 85, 88, 25, 19, 26, 86, 102, 92, 89, 21, 28, 27, 91, 101, 38, 37, 35, 98, 94, 96, 30, 36, 32, 100, 34, 99], [50, 48, 57, 0, 64, 117, 56, 116, 120, 126, 59, 122, 113, 62, 53, 63, 55, 46, 123, 58, 127, 110, 108, 65, 47, 61, 109, 49, 1, 40, 124, 97, 29, 121, 51, 119, 112, 114, 43, 54, 4, 60, 67, 107, 3, 68, 6, 45, 104, 125, 42, 2, 106, 66, 33, 52, 93, 115, 7, 41, 95, 20, 13, 111, 77, 70, 39, 9, 78, 71, 103, 75, 44, 84, 74, 12, 5, 73, 69, 80, 72, 76, 11, 79, 18, 81, 82, 10, 17, 16, 31, 14, 15, 105, 118, 8, 23, 87, 24, 83, 90, 102, 85, 88, 89, 25, 26, 22, 91, 19, 28, 98, 86, 21, 27, 92, 37, 38, 101, 94, 36, 35, 96, 30, 99, 100, 32, 34], [50, 48, 57, 117, 64, 0, 56, 116, 59, 62, 120, 126, 122, 113, 53, 46, 55, 123, 110, 127, 49, 58, 63, 109, 47, 97, 29, 108, 124, 54, 61, 65, 112, 114, 121, 40, 1, 51, 119, 60, 43, 107, 104, 66, 125, 52, 68, 2, 42, 45, 4, 67, 6, 33, 3, 70, 106, 115, 93, 95, 79, 13, 7, 5, 20, 16, 12, 11, 75, 78, 41, 77, 80, 9, 84, 82, 15, 14, 10, 103, 72, 44, 76, 69, 81, 73, 39, 71, 18, 111, 17, 118, 87, 105, 74, 31, 24, 23, 83, 8, 85, 21, 88, 26, 89, 86, 22, 92, 90, 25, 19, 102, 28, 27, 98, 91, 38, 101, 37, 94, 35, 30, 36, 34, 32, 100, 99, 96], [50, 48, 57, 117, 0, 64, 56, 59, 116, 126, 120, 122, 62, 113, 123, 53, 46, 127, 63, 110, 97, 55, 49, 58, 1, 61, 29, 124, 109, 47, 112, 108, 121, 67, 65, 51, 54, 119, 3, 114, 40, 107, 60, 66, 45, 125, 43, 2, 104, 70, 52, 6, 95, 13, 93, 106, 42, 4, 33, 5, 12, 115, 16, 68, 9, 77, 20, 14, 76, 84, 79, 7, 111, 75, 78, 71, 69, 82, 18, 15, 103, 41, 17, 39, 44, 81, 73, 11, 10, 80, 72, 31, 118, 74, 105, 24, 22, 87, 23, 85, 83, 25, 88, 21, 89, 86, 8, 19, 90, 92, 26, 28, 27, 102, 98, 91, 94, 38, 37, 101, 36, 30, 35, 96, 34, 100, 32, 99], [50, 48, 64, 57, 0, 117, 56, 59, 116, 120, 126, 113, 62, 122, 123, 55, 53, 46, 63, 127, 1, 58, 110, 97, 65, 61, 49, 121, 108, 109, 47, 119, 124, 29, 112, 40, 51, 114, 54, 60, 2, 107, 3, 125, 4, 68, 43, 70, 104, 45, 66, 67, 42, 52, 6, 106, 115, 95, 41, 33, 93, 5, 77, 13, 16, 20, 7, 79, 82, 12, 76, 71, 69, 9, 84, 111, 81, 103, 75, 73, 78, 11, 14, 10, 39, 44, 80, 17, 72, 118, 31, 18, 74, 15, 87, 23, 105, 8, 24, 22, 88, 19, 21, 83, 89, 85, 25, 102, 90, 86, 92, 28, 26, 27, 91, 98, 38, 94, 101, 37, 100, 36, 32, 35, 30, 96, 34, 99], [50, 48, 57, 0, 64, 117, 120, 56, 126, 116, 59, 62, 122, 113, 53, 123, 46, 63, 55, 127, 110, 58, 108, 1, 109, 61, 97, 65, 40, 49, 47, 112, 124, 29, 121, 51, 107, 68, 119, 114, 43, 60, 70, 4, 54, 2, 104, 42, 125, 66, 106, 52, 45, 3, 67, 33, 93, 95, 6, 115, 5, 69, 41, 71, 77, 79, 75, 84, 39, 13, 78, 12, 20, 44, 9, 103, 111, 7, 73, 82, 17, 76, 16, 10, 14, 18, 80, 105, 74, 72, 11, 31, 8, 15, 81, 118, 24, 23, 87, 83, 88, 22, 85, 89, 102, 19, 27, 92, 25, 21, 86, 91, 90, 26, 28, 38, 101, 98, 37, 30, 96, 94, 35, 34, 36, 100, 32, 99], [50, 48, 57, 64, 0, 117, 116, 120, 56, 126, 122, 59, 113, 62, 53, 123, 46, 63, 127, 55, 49, 108, 58, 110, 97, 124, 29, 67, 47, 109, 61, 112, 3, 51, 1, 121, 114, 40, 119, 54, 65, 107, 43, 60, 70, 33, 125, 104, 45, 2, 42, 106, 52, 95, 93, 66, 68, 71, 4, 13, 84, 77, 69, 6, 20, 111, 115, 17, 75, 76, 82, 41, 9, 80, 39, 7, 16, 103, 79, 5, 78, 74, 14, 44, 11, 12, 73, 18, 10, 15, 31, 81, 105, 87, 24, 118, 23, 72, 83, 8, 22, 88, 86, 19, 89, 90, 28, 25, 85, 21, 102, 92, 26, 91, 98, 27, 37, 36, 38, 101, 94, 32, 30, 99, 35, 100, 96, 34], [50, 48, 57, 64, 0, 117, 56, 120, 116, 59, 126, 62, 113, 122, 123, 55, 53, 1, 46, 63, 110, 58, 127, 65, 49, 61, 121, 97, 108, 112, 47, 29, 119, 124, 109, 51, 54, 114, 40, 60, 2, 66, 3, 125, 45, 107, 70, 104, 52, 43, 4, 6, 5, 115, 67, 95, 42, 93, 106, 68, 71, 69, 33, 77, 13, 76, 75, 9, 12, 78, 20, 80, 15, 111, 41, 18, 82, 7, 84, 103, 11, 73, 10, 39, 14, 17, 79, 8, 44, 118, 16, 81, 74, 31, 87, 105, 23, 72, 24, 25, 22, 21, 102, 19, 88, 28, 85, 89, 86, 83, 90, 92, 26, 27, 91, 98, 37, 38, 101, 30, 94, 96, 35, 34, 32, 100, 36, 99], [50, 48, 57, 0, 117, 64, 56, 116, 120, 126, 122, 59, 113, 62, 123, 46, 63, 53, 110, 127, 55, 58, 124, 97, 108, 49, 109, 47, 61, 40, 65, 112, 114, 29, 51, 119, 54, 107, 121, 1, 60, 4, 43, 125, 104, 68, 52, 42, 6, 3, 106, 33, 45, 70, 2, 95, 66, 93, 115, 67, 20, 77, 9, 5, 7, 71, 41, 13, 11, 16, 111, 103, 80, 14, 84, 15, 82, 12, 39, 69, 75, 10, 79, 17, 78, 44, 76, 73, 8, 74, 18, 105, 81, 118, 31, 24, 87, 72, 19, 23, 25, 22, 21, 27, 88, 26, 28, 83, 91, 86, 85, 92, 89, 90, 102, 38, 37, 98, 101, 30, 94, 34, 35, 96, 32, 36, 100, 99], [50, 48, 57, 0, 64, 117, 56, 120, 126, 116, 59, 122, 113, 62, 123, 46, 53, 110, 63, 55, 58, 49, 109, 127, 124, 97, 65, 112, 3, 108, 61, 47, 40, 1, 29, 67, 114, 43, 119, 121, 54, 51, 107, 60, 125, 104, 66, 42, 6, 68, 2, 33, 106, 4, 45, 52, 93, 95, 70, 13, 69, 41, 115, 12, 71, 17, 39, 44, 20, 79, 84, 9, 77, 16, 103, 76, 14, 8, 82, 74, 7, 5, 111, 80, 75, 78, 73, 11, 18, 15, 105, 10, 118, 31, 81, 87, 24, 23, 72, 88, 22, 19, 92, 26, 102, 89, 25, 28, 83, 21, 85, 86, 90, 27, 91, 38, 98, 101, 37, 36, 35, 94, 30, 100, 96, 99, 34, 32], [50, 48, 57, 64, 117, 0, 116, 56, 120, 126, 59, 122, 113, 62, 123, 55, 53, 110, 46, 63, 108, 58, 127, 1, 29, 109, 97, 49, 124, 114, 61, 40, 119, 51, 121, 47, 112, 54, 65, 125, 43, 107, 52, 60, 6, 45, 104, 2, 66, 67, 42, 106, 93, 70, 33, 3, 115, 95, 4, 69, 68, 80, 13, 11, 76, 41, 84, 71, 7, 77, 5, 74, 75, 103, 111, 14, 8, 79, 9, 20, 16, 17, 39, 73, 15, 82, 12, 78, 18, 44, 10, 31, 105, 118, 23, 87, 81, 72, 85, 24, 83, 25, 92, 22, 88, 86, 19, 89, 26, 102, 21, 28, 90, 91, 101, 38, 37, 27, 98, 94, 35, 30, 36, 34, 99, 96, 100, 32], [50, 48, 64, 57, 0, 117, 116, 120, 56, 126, 59, 113, 62, 122, 46, 123, 53, 63, 55, 110, 127, 58, 97, 49, 108, 109, 47, 1, 65, 112, 124, 61, 121, 29, 40, 114, 119, 51, 54, 60, 43, 107, 67, 125, 66, 3, 45, 6, 104, 42, 33, 2, 106, 52, 95, 68, 93, 115, 20, 4, 70, 77, 13, 41, 16, 71, 5, 7, 79, 111, 84, 69, 44, 11, 12, 39, 17, 76, 73, 75, 80, 82, 18, 14, 9, 103, 15, 10, 78, 31, 105, 74, 81, 24, 118, 8, 23, 72, 87, 19, 86, 92, 85, 88, 89, 25, 83, 28, 26, 22, 102, 21, 90, 37, 27, 98, 91, 101, 38, 30, 94, 35, 99, 34, 36, 32, 100, 96], [50, 48, 57, 0, 117, 64, 116, 56, 126, 113, 120, 59, 62, 122, 123, 53, 46, 63, 65, 127, 110, 49, 47, 97, 108, 61, 109, 124, 55, 112, 58, 29, 67, 1, 3, 51, 40, 54, 114, 119, 121, 43, 107, 68, 4, 45, 104, 60, 125, 6, 66, 2, 42, 33, 106, 95, 52, 93, 115, 20, 111, 77, 44, 11, 12, 13, 76, 84, 39, 7, 70, 71, 69, 41, 5, 73, 17, 78, 82, 103, 79, 9, 18, 10, 80, 16, 75, 15, 14, 74, 105, 81, 31, 24, 118, 8, 87, 23, 72, 83, 22, 86, 25, 88, 21, 92, 28, 85, 19, 26, 90, 102, 89, 27, 98, 37, 91, 38, 101, 100, 35, 94, 36, 32, 30, 99, 34, 96], [50, 48, 0, 57, 64, 117, 56, 120, 116, 59, 62, 113, 126, 122, 123, 46, 53, 63, 55, 110, 1, 58, 65, 108, 127, 109, 61, 97, 47, 49, 40, 121, 29, 124, 112, 51, 107, 119, 66, 114, 43, 54, 125, 45, 104, 60, 52, 42, 70, 106, 4, 2, 6, 68, 95, 3, 33, 67, 69, 93, 115, 77, 5, 41, 20, 13, 76, 12, 71, 7, 111, 11, 73, 103, 78, 80, 39, 9, 10, 79, 16, 14, 75, 84, 74, 18, 8, 44, 15, 82, 81, 31, 17, 105, 118, 87, 23, 72, 24, 19, 22, 86, 21, 88, 92, 102, 83, 85, 89, 25, 26, 90, 27, 28, 91, 38, 98, 37, 101, 94, 35, 34, 32, 36, 99, 30, 100, 96], [50, 48, 57, 0, 64, 117, 116, 120, 56, 59, 126, 113, 122, 62, 123, 55, 46, 58, 53, 63, 127, 49, 1, 108, 110, 109, 61, 97, 65, 47, 124, 112, 29, 119, 121, 40, 3, 51, 67, 54, 114, 107, 43, 104, 60, 2, 45, 70, 125, 42, 66, 106, 33, 13, 6, 95, 52, 4, 41, 93, 7, 68, 69, 111, 115, 71, 20, 5, 76, 78, 77, 74, 73, 11, 103, 14, 9, 12, 79, 84, 16, 39, 80, 75, 18, 81, 82, 44, 105, 17, 72, 8, 10, 15, 31, 118, 87, 24, 23, 25, 88, 22, 19, 83, 102, 86, 21, 89, 26, 85, 90, 28, 92, 91, 101, 27, 37, 38, 98, 94, 36, 35, 100, 30, 34, 96, 99, 32], [50, 48, 0, 57, 117, 64, 120, 116, 56, 59, 126, 113, 122, 62, 46, 53, 123, 127, 58, 110, 1, 55, 63, 109, 29, 97, 124, 49, 108, 65, 40, 3, 61, 47, 67, 51, 114, 112, 54, 121, 43, 60, 107, 119, 68, 70, 104, 2, 125, 42, 4, 45, 66, 33, 106, 52, 115, 95, 93, 6, 71, 77, 20, 41, 73, 5, 69, 13, 44, 111, 80, 16, 78, 7, 76, 12, 103, 74, 11, 9, 79, 39, 84, 81, 75, 10, 18, 14, 105, 72, 15, 31, 17, 8, 82, 118, 24, 87, 23, 22, 88, 86, 83, 19, 25, 26, 21, 92, 89, 85, 102, 28, 90, 91, 27, 38, 101, 98, 34, 37, 36, 94, 32, 100, 30, 99, 96, 35], [50, 48, 57, 117, 64, 0, 116, 59, 126, 56, 122, 120, 62, 113, 123, 46, 53, 49, 55, 127, 58, 110, 97, 108, 63, 109, 47, 124, 29, 61, 54, 112, 119, 114, 121, 40, 65, 1, 51, 107, 104, 70, 43, 125, 4, 60, 52, 42, 45, 33, 68, 93, 95, 66, 11, 13, 106, 20, 2, 6, 15, 79, 76, 77, 115, 9, 5, 18, 84, 74, 80, 44, 14, 78, 16, 103, 81, 67, 41, 75, 69, 72, 82, 12, 3, 73, 71, 7, 39, 10, 105, 111, 17, 118, 87, 31, 23, 24, 8, 85, 22, 88, 25, 19, 83, 26, 92, 21, 89, 86, 102, 90, 27, 28, 91, 38, 98, 37, 101, 94, 36, 34, 99, 30, 100, 35, 96, 32], [50, 48, 57, 64, 117, 0, 116, 120, 56, 126, 113, 62, 122, 59, 123, 53, 46, 55, 63, 127, 110, 97, 109, 108, 58, 124, 61, 49, 112, 47, 65, 29, 40, 114, 125, 119, 1, 121, 43, 54, 51, 60, 3, 107, 67, 45, 66, 70, 104, 33, 52, 42, 95, 106, 115, 2, 6, 93, 13, 5, 16, 20, 77, 14, 7, 76, 41, 84, 9, 11, 12, 44, 4, 71, 111, 81, 79, 103, 68, 18, 80, 39, 17, 75, 15, 82, 74, 10, 105, 69, 78, 73, 72, 23, 87, 31, 118, 24, 8, 19, 85, 22, 86, 88, 83, 92, 89, 25, 28, 26, 102, 90, 21, 27, 91, 38, 98, 37, 101, 94, 36, 35, 30, 34, 100, 99, 96, 32]], "model.layers.2.self_attn.q_proj": [[104, 127, 46, 34, 49, 88, 113, 82, 21, 79, 75, 76, 24, 14, 71, 7, 5, 9, 92, 40, 107, 3, 15, 19, 52, 125, 62, 11, 119, 27, 121, 12, 78, 120, 20, 116, 66, 8, 93, 53, 50, 77, 109, 57, 47, 10, 56, 69, 2, 96, 18, 65, 84, 86, 85, 73, 60, 30, 91, 115, 72, 63, 41, 83, 54, 1, 45, 95, 17, 51, 74, 99, 29, 81, 25, 59, 28, 118, 16, 108, 55, 103, 31, 101, 64, 106, 48, 32, 97, 102, 13, 90, 87, 61, 80, 35, 112, 124, 94, 39, 43, 122, 42, 23, 67, 26, 38, 117, 4, 58, 100, 33, 111, 126, 6, 105, 22, 44, 37, 70, 36, 110, 123, 114, 89, 98, 68, 0], [104, 49, 46, 34, 127, 113, 21, 108, 24, 82, 79, 105, 40, 57, 9, 88, 75, 107, 76, 121, 56, 80, 5, 27, 120, 51, 90, 14, 30, 71, 92, 62, 26, 87, 48, 16, 22, 83, 35, 28, 52, 43, 17, 50, 125, 63, 84, 106, 94, 4, 117, 23, 111, 109, 61, 89, 59, 29, 3, 100, 45, 114, 119, 126, 42, 118, 19, 36, 47, 54, 67, 38, 110, 70, 116, 115, 10, 123, 96, 72, 98, 68, 33, 122, 11, 41, 44, 124, 93, 99, 20, 112, 102, 39, 103, 55, 95, 101, 77, 53, 37, 60, 7, 31, 18, 66, 32, 81, 58, 85, 86, 91, 97, 69, 78, 73, 65, 15, 25, 1, 13, 6, 8, 64, 12, 74, 0, 2], [104, 34, 49, 127, 46, 21, 82, 71, 88, 79, 113, 76, 9, 75, 3, 5, 40, 14, 26, 121, 7, 53, 64, 1, 125, 66, 23, 119, 8, 52, 67, 10, 107, 24, 11, 81, 73, 4, 15, 17, 57, 120, 69, 77, 50, 19, 2, 20, 85, 22, 18, 32, 94, 70, 84, 6, 74, 72, 62, 92, 33, 12, 65, 16, 93, 89, 90, 63, 87, 56, 96, 45, 30, 126, 38, 31, 27, 51, 35, 13, 78, 42, 80, 108, 44, 43, 118, 103, 47, 54, 124, 48, 100, 86, 58, 95, 115, 91, 60, 28, 36, 83, 25, 37, 29, 117, 123, 102, 116, 39, 114, 122, 101, 111, 97, 112, 99, 109, 105, 41, 55, 68, 0, 59, 61, 106, 110, 98], [104, 46, 49, 34, 127, 113, 82, 21, 107, 88, 75, 125, 14, 52, 90, 53, 9, 22, 28, 79, 26, 24, 92, 5, 38, 121, 40, 19, 105, 94, 77, 76, 120, 119, 84, 27, 48, 51, 103, 50, 111, 39, 93, 29, 62, 16, 41, 60, 30, 98, 17, 124, 71, 91, 36, 56, 57, 126, 99, 118, 3, 81, 47, 106, 33, 123, 110, 109, 117, 115, 54, 44, 112, 63, 86, 37, 80, 59, 116, 100, 42, 35, 108, 20, 87, 55, 31, 122, 102, 83, 73, 58, 101, 70, 114, 96, 32, 97, 11, 15, 78, 23, 95, 43, 69, 72, 45, 61, 10, 89, 66, 6, 1, 7, 13, 18, 85, 65, 64, 67, 8, 25, 4, 68, 0, 74, 12, 2], [104, 107, 91, 35, 23, 84, 17, 27, 15, 13, 81, 47, 20, 94, 112, 113, 82, 114, 61, 43, 55, 99, 75, 77, 40, 56, 89, 53, 86, 111, 10, 96, 79, 92, 25, 117, 119, 123, 11, 28, 125, 51, 21, 115, 97, 118, 30, 124, 9, 73, 80, 93, 12, 16, 57, 120, 29, 48, 87, 122, 46, 76, 42, 70, 6, 102, 24, 26, 72, 18, 88, 116, 7, 110, 49, 108, 85, 38, 83, 14, 98, 105, 74, 22, 50, 68, 67, 60, 101, 31, 90, 41, 59, 71, 106, 8, 37, 32, 33, 78, 39, 44, 19, 63, 58, 127, 126, 103, 34, 45, 54, 69, 121, 36, 100, 95, 4, 62, 52, 109, 2, 3, 0, 65, 66, 5, 1, 64], [104, 107, 35, 91, 27, 23, 99, 94, 84, 114, 47, 43, 113, 89, 112, 61, 9, 55, 119, 123, 53, 15, 40, 73, 111, 92, 17, 12, 82, 81, 120, 28, 98, 77, 56, 46, 105, 117, 125, 97, 25, 67, 86, 116, 70, 48, 115, 122, 7, 124, 42, 13, 36, 102, 118, 51, 32, 14, 90, 29, 57, 22, 79, 11, 49, 37, 10, 85, 63, 16, 41, 109, 68, 126, 6, 103, 44, 110, 54, 121, 72, 76, 106, 58, 75, 108, 34, 93, 45, 96, 101, 52, 59, 69, 39, 38, 24, 87, 30, 95, 100, 80, 50, 18, 60, 31, 127, 74, 26, 33, 71, 62, 88, 21, 19, 83, 65, 20, 2, 78, 4, 0, 3, 8, 66, 64, 5, 1], [107, 104, 47, 35, 99, 124, 94, 89, 96, 27, 55, 91, 43, 61, 119, 112, 113, 46, 25, 123, 85, 63, 105, 48, 114, 120, 53, 116, 125, 111, 23, 54, 40, 84, 102, 52, 28, 56, 117, 97, 37, 51, 115, 82, 121, 126, 92, 42, 109, 62, 22, 41, 32, 106, 60, 29, 118, 44, 110, 95, 49, 57, 86, 122, 98, 38, 108, 100, 58, 127, 36, 34, 50, 12, 101, 33, 103, 17, 59, 67, 83, 26, 45, 31, 93, 39, 14, 78, 90, 30, 21, 15, 73, 81, 16, 24, 77, 6, 88, 18, 7, 19, 80, 79, 9, 10, 20, 68, 66, 11, 71, 70, 3, 76, 75, 13, 87, 2, 74, 72, 65, 4, 0, 8, 64, 5, 1, 69], [104, 107, 35, 15, 75, 8, 13, 17, 91, 5, 84, 4, 23, 99, 73, 40, 43, 3, 1, 71, 72, 87, 112, 114, 69, 61, 68, 6, 113, 55, 67, 20, 2, 47, 53, 123, 7, 66, 0, 56, 65, 11, 120, 70, 115, 111, 10, 119, 89, 9, 25, 110, 77, 30, 64, 125, 57, 21, 16, 46, 124, 12, 94, 18, 51, 117, 86, 79, 74, 122, 108, 81, 76, 58, 14, 92, 49, 78, 85, 83, 82, 96, 127, 116, 121, 19, 90, 28, 88, 26, 102, 118, 31, 29, 80, 97, 105, 59, 60, 48, 32, 95, 93, 24, 22, 39, 37, 62, 63, 34, 50, 106, 33, 44, 42, 38, 101, 36, 27, 98, 100, 103, 54, 41, 109, 52, 45, 126], [110, 38, 97, 112, 46, 16, 89, 78, 10, 19, 8, 4, 12, 70, 68, 48, 25, 2, 69, 7, 6, 86, 82, 56, 1, 31, 20, 64, 50, 5, 102, 71, 22, 75, 125, 119, 11, 113, 3, 111, 13, 83, 67, 0, 127, 77, 61, 66, 47, 109, 51, 80, 14, 85, 73, 52, 118, 74, 72, 84, 94, 9, 81, 116, 117, 41, 93, 40, 120, 123, 18, 76, 87, 124, 107, 63, 15, 23, 21, 115, 65, 30, 104, 79, 36, 37, 43, 28, 34, 55, 88, 53, 91, 62, 24, 59, 17, 98, 108, 27, 60, 126, 99, 96, 45, 103, 32, 44, 57, 100, 101, 121, 58, 42, 122, 106, 29, 39, 92, 54, 35, 49, 95, 114, 26, 90, 105, 33], [110, 38, 112, 97, 46, 19, 78, 16, 12, 10, 8, 89, 4, 68, 69, 48, 71, 25, 5, 9, 70, 6, 102, 13, 24, 50, 2, 1, 88, 51, 62, 111, 66, 77, 65, 56, 109, 31, 118, 64, 113, 21, 15, 75, 61, 22, 127, 85, 73, 72, 3, 7, 67, 55, 14, 30, 74, 17, 27, 0, 34, 76, 83, 79, 124, 86, 115, 125, 20, 11, 100, 43, 94, 52, 122, 104, 107, 116, 87, 93, 99, 23, 105, 80, 84, 117, 95, 119, 47, 126, 121, 35, 82, 18, 91, 98, 44, 58, 120, 81, 123, 26, 40, 54, 42, 49, 45, 37, 101, 32, 92, 53, 39, 59, 36, 41, 106, 96, 108, 60, 29, 114, 103, 90, 28, 63, 57, 33], [38, 97, 110, 94, 112, 46, 25, 19, 83, 89, 78, 48, 93, 118, 85, 16, 50, 11, 96, 62, 39, 56, 21, 111, 127, 36, 77, 61, 27, 109, 90, 106, 80, 101, 17, 10, 88, 30, 12, 125, 60, 87, 95, 44, 86, 40, 105, 35, 121, 34, 31, 24, 104, 59, 116, 108, 115, 120, 57, 43, 54, 26, 123, 58, 81, 103, 126, 117, 20, 99, 23, 119, 13, 45, 14, 52, 124, 51, 37, 122, 29, 73, 100, 55, 42, 91, 92, 47, 107, 63, 22, 75, 32, 41, 114, 98, 53, 113, 9, 28, 33, 49, 15, 84, 6, 8, 82, 7, 18, 79, 69, 71, 76, 74, 70, 72, 5, 68, 4, 102, 2, 66, 3, 67, 65, 1, 0, 64], [110, 38, 97, 112, 46, 89, 78, 16, 10, 77, 25, 17, 19, 12, 8, 48, 6, 69, 11, 71, 102, 94, 51, 15, 50, 4, 75, 68, 2, 86, 21, 83, 62, 76, 31, 70, 52, 5, 72, 111, 81, 9, 127, 1, 22, 108, 99, 29, 121, 118, 109, 84, 124, 14, 7, 116, 82, 92, 36, 45, 37, 117, 74, 123, 90, 23, 79, 88, 80, 104, 35, 64, 105, 95, 61, 56, 66, 34, 58, 65, 103, 73, 93, 42, 44, 33, 107, 122, 28, 126, 20, 18, 55, 106, 87, 27, 113, 57, 47, 125, 53, 98, 85, 115, 41, 24, 54, 114, 26, 120, 30, 39, 101, 59, 40, 13, 96, 119, 67, 60, 100, 91, 43, 32, 49, 63, 3, 0], [40, 98, 121, 125, 53, 51, 17, 20, 82, 80, 18, 115, 27, 19, 21, 88, 104, 77, 22, 42, 119, 12, 89, 26, 95, 86, 30, 75, 92, 28, 84, 78, 25, 117, 96, 94, 29, 32, 120, 23, 93, 90, 57, 102, 116, 52, 79, 38, 127, 41, 24, 97, 50, 87, 123, 105, 56, 122, 10, 109, 113, 16, 31, 63, 101, 110, 43, 44, 48, 107, 33, 61, 49, 111, 36, 58, 62, 76, 124, 100, 91, 39, 114, 59, 45, 81, 108, 83, 46, 37, 35, 9, 103, 99, 60, 112, 47, 55, 85, 8, 126, 118, 54, 74, 14, 34, 71, 73, 15, 106, 7, 13, 11, 6, 72, 5, 70, 2, 69, 65, 68, 3, 64, 4, 67, 66, 1, 0], [40, 125, 121, 53, 51, 98, 27, 20, 82, 16, 84, 81, 12, 21, 42, 19, 115, 10, 119, 88, 8, 78, 74, 29, 50, 85, 86, 35, 72, 91, 36, 95, 5, 6, 117, 14, 94, 69, 104, 17, 18, 34, 93, 24, 76, 22, 65, 67, 97, 26, 79, 68, 75, 120, 38, 116, 15, 23, 30, 122, 44, 57, 63, 66, 107, 89, 58, 80, 71, 83, 109, 87, 7, 106, 9, 99, 108, 96, 111, 41, 33, 118, 13, 105, 103, 101, 48, 127, 114, 56, 100, 37, 124, 52, 28, 45, 90, 3, 113, 49, 92, 43, 77, 39, 32, 123, 59, 102, 110, 46, 31, 60, 126, 54, 70, 25, 61, 112, 62, 64, 47, 0, 55, 11, 4, 73, 1, 2], [40, 121, 42, 53, 125, 51, 98, 91, 123, 50, 110, 52, 111, 113, 58, 63, 105, 126, 47, 33, 115, 27, 54, 60, 118, 93, 117, 107, 97, 59, 122, 127, 48, 108, 61, 56, 15, 44, 124, 109, 57, 112, 116, 43, 45, 114, 62, 46, 120, 106, 38, 119, 49, 55, 41, 95, 23, 103, 39, 36, 101, 31, 100, 24, 86, 37, 99, 32, 35, 102, 30, 21, 12, 29, 96, 18, 85, 34, 89, 104, 94, 77, 92, 17, 81, 84, 83, 78, 79, 20, 74, 87, 9, 28, 69, 72, 13, 26, 88, 19, 75, 90, 25, 82, 22, 7, 10, 80, 2, 65, 70, 4, 76, 8, 11, 64, 3, 16, 6, 68, 1, 73, 67, 14, 5, 0, 71, 66], [53, 125, 51, 121, 42, 40, 115, 98, 119, 52, 108, 57, 123, 116, 91, 107, 50, 114, 117, 122, 127, 124, 120, 58, 111, 59, 118, 113, 63, 15, 55, 41, 60, 110, 48, 49, 9, 109, 56, 61, 100, 62, 44, 43, 46, 7, 105, 99, 112, 45, 54, 77, 126, 47, 102, 72, 39, 38, 33, 74, 12, 103, 69, 106, 93, 18, 101, 35, 30, 17, 95, 75, 78, 2, 70, 31, 11, 37, 27, 32, 80, 36, 84, 3, 24, 104, 34, 23, 96, 97, 4, 86, 68, 94, 83, 79, 64, 8, 29, 92, 10, 89, 85, 6, 65, 13, 0, 73, 76, 71, 26, 20, 87, 28, 21, 19, 88, 25, 90, 5, 1, 66, 16, 81, 67, 14, 22, 82], [102, 49, 98, 56, 116, 121, 113, 89, 20, 11, 81, 9, 77, 22, 15, 124, 52, 25, 2, 6, 73, 71, 18, 96, 75, 86, 110, 72, 88, 79, 70, 101, 19, 51, 10, 74, 100, 4, 3, 29, 67, 91, 1, 31, 83, 87, 92, 78, 17, 64, 13, 60, 76, 16, 120, 24, 84, 59, 61, 8, 30, 41, 80, 14, 21, 7, 58, 93, 47, 37, 12, 26, 28, 117, 62, 115, 90, 44, 46, 82, 32, 94, 27, 112, 63, 68, 103, 40, 105, 109, 33, 39, 42, 85, 95, 114, 97, 111, 35, 122, 48, 107, 23, 118, 53, 55, 54, 45, 127, 43, 123, 50, 104, 106, 119, 34, 126, 57, 99, 36, 108, 0, 125, 5, 69, 66, 65, 38], [102, 49, 116, 56, 98, 121, 113, 89, 81, 124, 20, 11, 15, 77, 68, 9, 71, 52, 70, 25, 29, 2, 64, 91, 19, 85, 4, 0, 105, 88, 1, 65, 66, 8, 3, 73, 101, 41, 46, 75, 12, 127, 110, 62, 60, 51, 72, 80, 93, 21, 67, 7, 27, 74, 14, 86, 96, 76, 47, 117, 13, 126, 18, 84, 44, 87, 6, 111, 115, 120, 79, 17, 22, 82, 90, 112, 10, 95, 107, 5, 61, 39, 24, 23, 16, 103, 30, 122, 92, 57, 83, 94, 32, 97, 118, 45, 78, 33, 26, 59, 36, 69, 43, 123, 109, 53, 37, 40, 100, 54, 114, 28, 50, 125, 99, 42, 108, 106, 48, 55, 35, 104, 58, 31, 63, 119, 34, 38], [102, 49, 116, 56, 98, 113, 121, 89, 77, 11, 81, 20, 9, 71, 15, 52, 68, 124, 110, 29, 19, 91, 86, 10, 101, 25, 6, 70, 2, 41, 66, 74, 47, 7, 4, 14, 46, 59, 67, 117, 82, 75, 72, 80, 64, 18, 96, 92, 73, 0, 31, 3, 95, 79, 115, 126, 12, 8, 21, 16, 93, 1, 22, 13, 87, 17, 60, 39, 88, 36, 107, 127, 50, 97, 32, 35, 120, 33, 122, 69, 53, 105, 85, 30, 61, 65, 84, 111, 62, 104, 5, 26, 42, 58, 118, 76, 94, 114, 23, 83, 112, 51, 24, 109, 54, 27, 44, 28, 40, 90, 125, 119, 100, 99, 78, 43, 57, 55, 48, 106, 123, 108, 103, 63, 45, 37, 34, 38], [102, 121, 110, 49, 98, 56, 116, 113, 101, 89, 25, 62, 93, 29, 31, 52, 91, 41, 51, 46, 107, 27, 45, 126, 34, 40, 100, 38, 115, 58, 106, 60, 112, 20, 105, 63, 96, 85, 39, 48, 103, 22, 57, 19, 61, 119, 86, 50, 47, 108, 53, 18, 123, 109, 114, 55, 127, 54, 120, 84, 104, 28, 111, 78, 94, 44, 118, 43, 83, 122, 81, 21, 42, 125, 124, 59, 92, 80, 30, 24, 16, 117, 35, 95, 37, 33, 26, 32, 88, 99, 36, 15, 97, 23, 77, 76, 14, 87, 90, 82, 10, 12, 5, 17, 71, 79, 2, 11, 69, 66, 8, 13, 9, 74, 7, 72, 70, 68, 75, 67, 4, 6, 64, 73, 65, 0, 3, 1], [126, 61, 56, 60, 54, 51, 123, 63, 124, 119, 100, 58, 117, 32, 127, 108, 112, 49, 39, 97, 114, 47, 104, 42, 125, 44, 106, 46, 40, 41, 48, 118, 99, 62, 110, 53, 120, 34, 52, 105, 121, 57, 107, 115, 116, 102, 55, 103, 43, 122, 50, 45, 113, 59, 111, 109, 38, 36, 33, 101, 96, 37, 94, 98, 30, 85, 31, 35, 8, 28, 25, 67, 70, 7, 5, 91, 29, 73, 2, 74, 27, 68, 92, 75, 77, 95, 93, 83, 10, 22, 4, 21, 65, 11, 76, 26, 71, 13, 89, 64, 12, 0, 9, 90, 19, 1, 78, 86, 23, 72, 66, 80, 88, 87, 15, 24, 6, 79, 14, 20, 69, 82, 81, 84, 3, 16, 17, 18], [55, 119, 123, 127, 115, 124, 60, 58, 56, 54, 63, 51, 61, 117, 62, 113, 126, 59, 96, 57, 52, 109, 48, 53, 121, 116, 120, 107, 114, 45, 108, 43, 50, 122, 110, 112, 42, 47, 125, 49, 118, 111, 44, 94, 46, 106, 105, 41, 32, 93, 104, 40, 103, 95, 39, 30, 23, 102, 98, 91, 34, 38, 90, 24, 92, 37, 17, 16, 101, 88, 22, 18, 100, 79, 7, 31, 35, 10, 78, 99, 75, 89, 8, 29, 84, 13, 26, 70, 36, 73, 5, 76, 19, 81, 15, 21, 20, 97, 80, 33, 9, 4, 27, 65, 0, 2, 87, 67, 85, 68, 66, 77, 25, 83, 28, 72, 14, 3, 69, 82, 12, 86, 1, 71, 74, 6, 11, 64], [119, 123, 55, 100, 124, 127, 56, 60, 58, 54, 115, 51, 63, 61, 36, 117, 126, 62, 32, 98, 59, 113, 52, 57, 121, 48, 45, 109, 120, 116, 53, 43, 107, 50, 30, 122, 114, 47, 106, 108, 110, 111, 112, 125, 118, 42, 49, 44, 46, 34, 41, 105, 39, 38, 103, 101, 104, 102, 40, 35, 37, 31, 96, 33, 99, 93, 97, 95, 28, 22, 88, 91, 94, 29, 19, 24, 68, 77, 64, 85, 90, 92, 27, 12, 26, 86, 2, 5, 83, 14, 74, 1, 13, 6, 25, 11, 72, 71, 67, 7, 8, 21, 69, 3, 73, 79, 78, 10, 9, 16, 81, 76, 20, 70, 89, 66, 80, 75, 87, 15, 65, 82, 18, 0, 4, 23, 17, 84], [126, 61, 100, 26, 54, 60, 28, 20, 63, 119, 51, 58, 123, 56, 124, 87, 55, 96, 18, 93, 24, 127, 25, 115, 86, 117, 85, 92, 30, 27, 94, 17, 97, 90, 33, 91, 62, 31, 32, 49, 112, 108, 106, 125, 114, 34, 104, 42, 39, 47, 120, 46, 44, 48, 53, 110, 121, 57, 41, 118, 40, 52, 105, 59, 43, 116, 107, 103, 50, 113, 122, 45, 109, 36, 101, 80, 102, 111, 98, 23, 81, 38, 15, 16, 89, 21, 99, 37, 83, 88, 95, 29, 35, 82, 19, 22, 14, 79, 78, 84, 9, 13, 75, 10, 71, 74, 12, 76, 8, 7, 67, 5, 70, 66, 72, 4, 77, 69, 65, 73, 11, 2, 0, 64, 1, 3, 6, 68], [33, 40, 108, 117, 57, 54, 22, 19, 118, 122, 82, 24, 16, 121, 77, 89, 127, 25, 76, 14, 52, 7, 125, 49, 44, 12, 123, 92, 109, 93, 106, 107, 3, 13, 69, 73, 11, 115, 46, 29, 43, 84, 62, 21, 53, 104, 120, 32, 119, 39, 28, 20, 41, 94, 124, 98, 61, 67, 87, 86, 88, 30, 111, 83, 34, 112, 59, 2, 15, 74, 64, 70, 65, 95, 48, 85, 71, 47, 60, 91, 105, 63, 96, 6, 17, 38, 68, 27, 81, 58, 42, 113, 116, 126, 110, 8, 23, 45, 72, 4, 114, 97, 79, 50, 90, 78, 5, 18, 10, 37, 26, 31, 1, 51, 101, 35, 103, 99, 102, 100, 55, 56, 80, 36, 66, 0, 9, 75], [108, 40, 33, 57, 54, 86, 25, 44, 118, 18, 117, 82, 80, 52, 91, 29, 104, 62, 121, 94, 115, 90, 92, 122, 89, 31, 56, 30, 126, 27, 95, 93, 48, 28, 88, 63, 34, 51, 110, 99, 124, 35, 59, 101, 112, 123, 26, 98, 113, 36, 60, 37, 43, 116, 103, 109, 41, 32, 39, 47, 58, 46, 42, 105, 119, 102, 100, 96, 45, 111, 106, 120, 49, 125, 38, 50, 107, 14, 53, 127, 55, 61, 23, 84, 87, 16, 114, 24, 85, 20, 83, 77, 21, 22, 76, 19, 78, 97, 81, 79, 17, 75, 11, 13, 66, 70, 12, 15, 74, 5, 4, 6, 7, 3, 67, 0, 9, 68, 8, 72, 65, 71, 2, 1, 10, 64, 69, 73], [33, 40, 108, 57, 54, 16, 14, 89, 118, 44, 117, 78, 82, 22, 13, 104, 52, 24, 121, 27, 71, 19, 122, 12, 75, 25, 90, 87, 28, 62, 109, 124, 93, 99, 94, 48, 115, 30, 31, 29, 123, 110, 101, 26, 126, 56, 91, 127, 112, 95, 53, 63, 119, 96, 46, 59, 37, 73, 21, 120, 51, 86, 36, 98, 92, 77, 43, 116, 102, 105, 85, 41, 103, 42, 38, 34, 106, 35, 60, 32, 125, 88, 111, 100, 47, 113, 58, 23, 39, 45, 76, 49, 9, 50, 61, 107, 114, 55, 5, 17, 20, 83, 6, 18, 81, 84, 79, 68, 15, 80, 10, 74, 7, 3, 11, 8, 67, 72, 66, 70, 2, 97, 4, 1, 69, 65, 0, 64], [40, 108, 33, 57, 54, 11, 12, 89, 82, 117, 118, 76, 44, 16, 14, 22, 19, 73, 7, 9, 24, 104, 86, 77, 10, 71, 78, 75, 121, 27, 13, 127, 52, 122, 93, 70, 85, 74, 123, 48, 109, 23, 87, 66, 124, 43, 126, 106, 90, 21, 68, 101, 115, 53, 110, 28, 112, 30, 81, 26, 80, 31, 15, 119, 79, 99, 4, 63, 2, 64, 84, 25, 46, 94, 83, 62, 51, 3, 56, 96, 114, 95, 29, 98, 111, 120, 116, 36, 37, 59, 35, 32, 91, 92, 42, 102, 1, 41, 100, 47, 113, 60, 38, 125, 65, 88, 49, 20, 34, 103, 39, 17, 105, 6, 45, 50, 58, 107, 55, 69, 61, 72, 18, 67, 0, 5, 8, 97], [110, 103, 33, 31, 46, 21, 78, 76, 80, 10, 8, 19, 114, 120, 79, 13, 112, 70, 4, 82, 17, 116, 49, 12, 6, 123, 16, 61, 54, 57, 89, 124, 58, 121, 25, 73, 125, 75, 47, 39, 22, 107, 18, 67, 117, 1, 118, 94, 27, 83, 56, 43, 85, 108, 111, 115, 69, 77, 48, 2, 14, 101, 63, 24, 32, 50, 51, 29, 106, 86, 93, 44, 74, 113, 23, 62, 41, 15, 28, 119, 71, 40, 91, 68, 36, 104, 127, 55, 30, 84, 11, 20, 59, 9, 92, 38, 42, 102, 72, 88, 35, 122, 37, 34, 99, 60, 52, 81, 26, 90, 45, 87, 7, 96, 98, 105, 100, 53, 126, 109, 97, 5, 66, 0, 95, 3, 65, 64], [103, 110, 33, 31, 46, 21, 19, 114, 49, 78, 120, 116, 23, 76, 12, 24, 57, 112, 79, 80, 72, 58, 16, 61, 118, 14, 4, 8, 6, 123, 48, 54, 50, 18, 67, 111, 47, 83, 98, 10, 85, 95, 45, 89, 17, 15, 108, 117, 121, 127, 73, 91, 109, 41, 63, 20, 1, 43, 90, 104, 37, 88, 81, 55, 102, 13, 39, 32, 119, 28, 113, 94, 2, 115, 70, 84, 44, 60, 51, 126, 42, 38, 96, 53, 105, 125, 122, 35, 34, 124, 106, 59, 97, 27, 29, 100, 74, 82, 9, 36, 107, 99, 93, 75, 56, 52, 40, 101, 71, 69, 86, 92, 30, 65, 3, 87, 22, 26, 62, 5, 25, 68, 11, 77, 0, 66, 7, 64], [110, 103, 33, 31, 46, 80, 21, 10, 8, 76, 78, 4, 114, 70, 68, 57, 120, 5, 49, 123, 58, 116, 112, 16, 17, 69, 3, 61, 1, 39, 65, 66, 12, 117, 121, 67, 74, 118, 54, 124, 9, 73, 7, 71, 127, 72, 14, 50, 32, 15, 11, 75, 107, 47, 125, 0, 51, 84, 111, 64, 45, 48, 79, 25, 77, 2, 85, 6, 91, 89, 87, 13, 93, 82, 60, 24, 63, 19, 109, 23, 81, 104, 18, 88, 122, 20, 83, 56, 92, 29, 38, 37, 115, 105, 86, 55, 119, 113, 35, 102, 36, 106, 53, 28, 44, 41, 62, 94, 42, 22, 30, 108, 95, 52, 34, 26, 96, 99, 43, 40, 101, 59, 100, 98, 90, 97, 27, 126], [110, 103, 33, 31, 46, 21, 10, 80, 78, 79, 69, 8, 76, 120, 114, 70, 82, 2, 6, 67, 49, 57, 112, 73, 4, 123, 58, 12, 61, 39, 5, 66, 1, 116, 68, 23, 71, 3, 16, 13, 0, 19, 17, 127, 56, 117, 111, 72, 74, 118, 121, 7, 107, 125, 14, 51, 50, 64, 54, 87, 11, 9, 45, 75, 124, 65, 85, 24, 48, 88, 81, 43, 20, 77, 15, 89, 84, 126, 119, 94, 59, 42, 18, 32, 47, 92, 26, 37, 115, 63, 86, 83, 25, 109, 36, 62, 28, 104, 53, 44, 106, 91, 122, 60, 55, 40, 34, 27, 41, 108, 99, 30, 113, 105, 102, 22, 52, 97, 90, 35, 98, 96, 29, 100, 95, 38, 93, 101]], "model.layers.2.self_attn.k_proj": [[40, 49, 110, 127, 46, 98, 76, 9, 21, 82, 24, 79, 71, 14, 5, 75, 67, 66, 0, 64, 65, 70, 125, 88, 53, 2, 92, 81, 50, 69, 26, 93, 121, 4, 27, 116, 77, 30, 112, 52, 105, 74, 80, 119, 118, 97, 62, 13, 51, 56, 72, 57, 25, 68, 31, 109, 23, 106, 1, 126, 54, 120, 113, 83, 28, 84, 63, 48, 107, 29, 124, 44, 43, 59, 89, 20, 99, 45, 73, 47, 86, 60, 42, 108, 87, 61, 16, 95, 8, 115, 90, 122, 117, 32, 39, 17, 19, 36, 96, 22, 114, 100, 41, 33, 103, 91, 123, 35, 58, 55, 37, 111, 10, 94, 102, 38, 101, 6, 104, 15, 3, 78, 7, 12, 85, 11, 34, 18], [43, 40, 99, 56, 48, 50, 119, 23, 117, 61, 84, 13, 113, 15, 17, 75, 123, 5, 51, 47, 0, 8, 111, 4, 57, 91, 55, 53, 46, 3, 104, 94, 65, 49, 6, 124, 73, 114, 44, 110, 125, 2, 89, 80, 10, 71, 60, 120, 64, 107, 122, 108, 121, 1, 12, 58, 118, 14, 86, 42, 85, 66, 25, 82, 70, 59, 52, 62, 28, 112, 76, 109, 127, 106, 27, 92, 126, 63, 83, 88, 97, 54, 22, 41, 19, 69, 33, 18, 39, 29, 115, 101, 38, 116, 21, 74, 78, 26, 87, 45, 105, 34, 37, 103, 7, 32, 98, 36, 90, 96, 31, 93, 102, 67, 16, 30, 24, 68, 95, 100, 72, 79, 35, 20, 81, 9, 11, 77], [46, 112, 102, 33, 110, 0, 12, 48, 16, 8, 10, 78, 25, 70, 69, 65, 66, 19, 2, 4, 64, 5, 77, 89, 50, 85, 17, 67, 47, 1, 7, 68, 22, 83, 30, 20, 95, 93, 88, 82, 125, 62, 56, 124, 116, 54, 87, 51, 9, 43, 27, 127, 55, 49, 118, 23, 24, 73, 3, 11, 114, 15, 13, 84, 106, 40, 92, 79, 32, 121, 91, 103, 41, 36, 61, 99, 126, 98, 101, 113, 109, 81, 29, 31, 107, 123, 105, 45, 57, 44, 108, 94, 104, 39, 59, 52, 96, 28, 120, 100, 119, 26, 63, 60, 35, 21, 18, 71, 37, 115, 58, 6, 90, 111, 53, 42, 86, 122, 117, 34, 74, 76, 75, 14, 80, 72, 97, 38], [104, 121, 125, 53, 51, 20, 16, 34, 78, 81, 82, 119, 19, 75, 42, 43, 10, 122, 88, 50, 61, 63, 127, 93, 70, 21, 13, 109, 118, 106, 116, 44, 113, 30, 7, 120, 8, 48, 57, 124, 114, 111, 62, 47, 60, 56, 58, 107, 54, 123, 115, 31, 76, 126, 9, 49, 23, 55, 38, 105, 110, 97, 22, 112, 39, 46, 41, 100, 45, 59, 91, 52, 102, 108, 37, 117, 36, 4, 3, 99, 86, 26, 89, 35, 95, 101, 32, 92, 28, 27, 33, 103, 94, 96, 24, 25, 2, 69, 90, 29, 14, 18, 87, 73, 17, 85, 15, 65, 5, 68, 83, 64, 80, 1, 84, 98, 0, 12, 72, 79, 74, 71, 11, 77, 40, 66, 6, 67], [113, 38, 116, 56, 34, 121, 49, 77, 11, 9, 81, 15, 71, 25, 0, 93, 68, 70, 20, 46, 65, 86, 67, 18, 66, 91, 16, 30, 124, 112, 102, 115, 31, 83, 61, 22, 19, 87, 72, 5, 118, 88, 76, 27, 85, 29, 1, 45, 78, 8, 82, 89, 23, 7, 6, 111, 48, 117, 36, 44, 74, 41, 10, 58, 105, 59, 99, 47, 40, 108, 51, 106, 119, 42, 127, 101, 62, 43, 92, 50, 123, 104, 114, 103, 120, 57, 84, 37, 90, 95, 97, 125, 122, 24, 32, 55, 63, 69, 35, 109, 96, 53, 126, 60, 54, 3, 28, 33, 94, 39, 12, 107, 26, 100, 79, 13, 80, 52, 4, 73, 14, 110, 21, 75, 17, 2, 64, 98], [126, 36, 124, 60, 61, 54, 63, 58, 119, 56, 115, 51, 123, 55, 127, 32, 117, 125, 118, 49, 122, 110, 112, 114, 46, 30, 50, 53, 52, 120, 121, 116, 48, 59, 111, 62, 113, 47, 57, 34, 108, 44, 45, 109, 106, 95, 42, 93, 40, 41, 107, 104, 105, 43, 98, 20, 24, 23, 39, 103, 96, 33, 102, 97, 90, 94, 38, 82, 81, 92, 101, 26, 99, 27, 91, 89, 37, 35, 100, 22, 29, 14, 80, 87, 15, 31, 85, 16, 25, 86, 28, 79, 12, 9, 19, 13, 11, 83, 88, 7, 4, 74, 66, 72, 5, 70, 75, 10, 69, 0, 3, 21, 18, 1, 65, 6, 71, 76, 8, 77, 17, 67, 78, 64, 84, 73, 68, 2], [44, 104, 54, 57, 97, 117, 82, 16, 22, 14, 77, 73, 25, 11, 76, 9, 127, 53, 108, 89, 7, 29, 122, 40, 92, 123, 12, 27, 19, 4, 24, 52, 91, 112, 93, 95, 90, 26, 31, 30, 94, 32, 88, 115, 110, 120, 119, 124, 21, 87, 113, 96, 28, 118, 45, 62, 46, 69, 121, 126, 47, 10, 125, 116, 98, 39, 34, 58, 63, 67, 37, 111, 107, 49, 41, 43, 99, 35, 51, 36, 60, 42, 103, 105, 102, 106, 59, 70, 101, 5, 100, 23, 50, 1, 38, 109, 55, 61, 2, 20, 56, 85, 48, 17, 8, 15, 114, 75, 78, 0, 84, 72, 81, 80, 74, 71, 66, 79, 83, 18, 64, 6, 86, 13, 3, 33, 68, 65], [46, 39, 110, 97, 95, 64, 78, 76, 80, 10, 8, 21, 70, 68, 79, 66, 73, 3, 65, 19, 120, 58, 50, 57, 123, 9, 5, 48, 1, 2, 82, 7, 61, 23, 69, 113, 116, 118, 114, 67, 125, 51, 0, 124, 24, 121, 127, 77, 117, 112, 53, 49, 81, 47, 84, 52, 59, 56, 109, 72, 6, 17, 60, 87, 119, 4, 91, 75, 122, 107, 20, 22, 106, 45, 89, 96, 54, 40, 90, 29, 11, 126, 25, 63, 43, 42, 62, 92, 86, 30, 101, 98, 105, 44, 14, 115, 37, 104, 26, 27, 102, 108, 93, 36, 38, 55, 111, 12, 32, 41, 99, 100, 15, 18, 34, 83, 35, 85, 88, 28, 94, 74, 13, 71, 16, 33, 31, 103]], "model.layers.2.self_attn.qk_proj": [[46, 110, 49, 121, 112, 54, 40, 51, 127, 104, 57, 56, 125, 113, 53, 116, 43, 44, 126, 61, 124, 102, 60, 119, 123, 117, 108, 48, 89, 39, 33, 25, 16, 14, 80, 115, 78, 63, 99, 107, 12, 76, 55, 18, 50, 85, 47, 98, 120, 21, 114, 58, 75, 97, 88, 86, 74, 8, 118, 82, 13, 11, 81, 95, 83, 19, 34, 20, 27, 84, 79, 15, 91, 77, 122, 17, 9, 87, 42, 62, 10, 94, 73, 29, 24, 31, 22, 38, 7, 52, 23, 36, 45, 41, 59, 92, 71, 6, 72, 111, 4, 69, 5, 30, 68, 105, 101, 35, 93, 32, 109, 70, 96, 28, 66, 2, 0, 64, 103, 90, 106, 1, 100, 67, 3, 65, 37, 26], [46, 110, 49, 121, 112, 54, 40, 56, 104, 51, 57, 53, 125, 127, 113, 116, 43, 44, 126, 123, 61, 119, 124, 102, 60, 117, 48, 39, 108, 89, 25, 16, 107, 58, 120, 80, 33, 14, 115, 63, 12, 99, 47, 78, 76, 8, 118, 55, 98, 50, 85, 82, 27, 21, 114, 18, 20, 95, 75, 62, 83, 91, 15, 97, 19, 122, 77, 81, 74, 34, 13, 79, 17, 11, 94, 86, 84, 10, 9, 87, 42, 24, 31, 73, 70, 88, 23, 36, 111, 22, 4, 29, 38, 5, 7, 71, 68, 109, 52, 69, 0, 96, 28, 106, 64, 45, 103, 32, 1, 92, 105, 93, 66, 59, 30, 35, 67, 2, 72, 6, 101, 3, 65, 90, 26, 100, 41, 37], [46, 110, 49, 121, 112, 54, 56, 40, 104, 53, 51, 57, 127, 113, 125, 116, 43, 44, 61, 126, 124, 102, 119, 123, 117, 60, 108, 63, 33, 107, 55, 48, 89, 16, 25, 39, 120, 47, 58, 80, 76, 14, 8, 115, 12, 99, 118, 78, 97, 85, 50, 21, 70, 18, 98, 19, 82, 27, 114, 74, 11, 20, 77, 84, 15, 79, 94, 9, 17, 75, 42, 95, 91, 34, 13, 86, 52, 87, 10, 73, 81, 5, 38, 23, 24, 36, 7, 71, 68, 69, 122, 83, 4, 29, 62, 59, 88, 31, 111, 32, 109, 22, 66, 30, 106, 45, 64, 96, 105, 2, 1, 28, 92, 93, 100, 0, 103, 67, 41, 72, 3, 26, 65, 35, 101, 90, 6, 37], [46, 110, 49, 121, 112, 54, 127, 51, 57, 40, 56, 104, 113, 125, 53, 116, 43, 126, 44, 61, 124, 123, 102, 117, 60, 58, 119, 48, 33, 25, 108, 63, 16, 39, 89, 80, 55, 14, 8, 107, 99, 76, 118, 12, 78, 98, 21, 19, 120, 47, 18, 50, 91, 114, 115, 70, 97, 84, 11, 82, 27, 95, 52, 85, 77, 10, 74, 34, 23, 79, 20, 75, 42, 15, 86, 94, 83, 87, 62, 81, 31, 13, 17, 5, 9, 71, 73, 88, 36, 4, 69, 38, 24, 122, 68, 109, 106, 29, 30, 22, 7, 92, 41, 66, 111, 67, 59, 28, 93, 32, 2, 103, 65, 101, 0, 45, 1, 96, 105, 3, 64, 100, 26, 90, 35, 72, 6, 37], [46, 110, 49, 121, 112, 54, 40, 127, 51, 57, 56, 125, 104, 113, 53, 116, 43, 61, 44, 126, 119, 102, 124, 123, 117, 58, 108, 25, 63, 33, 55, 16, 76, 48, 120, 8, 107, 12, 99, 118, 14, 60, 89, 80, 78, 39, 47, 115, 21, 82, 114, 98, 50, 18, 91, 95, 84, 97, 11, 20, 70, 9, 79, 94, 27, 74, 85, 17, 52, 75, 73, 15, 19, 69, 13, 77, 31, 24, 10, 88, 7, 87, 81, 68, 86, 42, 23, 62, 36, 5, 34, 38, 83, 71, 59, 4, 111, 22, 2, 67, 3, 122, 66, 28, 64, 65, 41, 0, 92, 29, 109, 30, 32, 35, 96, 103, 100, 101, 106, 93, 1, 72, 6, 105, 45, 90, 26, 37], [46, 110, 49, 121, 112, 51, 54, 56, 40, 104, 57, 127, 113, 125, 53, 116, 43, 44, 61, 126, 102, 124, 123, 119, 48, 60, 58, 25, 115, 108, 117, 33, 89, 39, 16, 107, 55, 80, 14, 12, 120, 8, 78, 76, 47, 99, 98, 18, 21, 50, 114, 82, 63, 85, 75, 52, 20, 19, 95, 27, 9, 31, 15, 11, 118, 111, 74, 84, 79, 73, 97, 34, 91, 122, 70, 88, 17, 42, 86, 13, 62, 94, 87, 24, 77, 23, 10, 83, 7, 4, 22, 36, 81, 68, 69, 38, 5, 71, 28, 32, 106, 6, 29, 93, 103, 72, 67, 96, 105, 30, 66, 41, 92, 35, 1, 59, 0, 65, 45, 64, 109, 101, 2, 3, 100, 90, 26, 37], [46, 110, 49, 121, 112, 54, 40, 56, 104, 127, 57, 51, 113, 125, 116, 53, 43, 44, 61, 126, 119, 124, 123, 102, 89, 60, 48, 117, 108, 115, 16, 107, 39, 14, 58, 33, 25, 12, 47, 55, 76, 80, 78, 99, 8, 120, 63, 85, 11, 18, 50, 9, 21, 83, 82, 118, 79, 52, 84, 75, 98, 15, 42, 17, 20, 95, 77, 91, 19, 111, 27, 13, 114, 97, 73, 74, 81, 24, 94, 88, 31, 86, 36, 122, 10, 34, 38, 87, 71, 23, 109, 22, 4, 6, 7, 29, 68, 62, 69, 5, 32, 106, 45, 72, 105, 70, 96, 30, 35, 93, 59, 103, 92, 101, 41, 28, 66, 3, 0, 1, 64, 2, 26, 67, 65, 100, 37, 90], [46, 110, 49, 121, 112, 54, 40, 104, 56, 57, 51, 127, 113, 125, 53, 116, 43, 44, 61, 119, 126, 102, 124, 123, 60, 89, 108, 117, 25, 33, 63, 55, 48, 39, 78, 107, 80, 14, 16, 12, 85, 99, 76, 115, 47, 118, 58, 18, 21, 82, 120, 50, 11, 15, 97, 74, 9, 83, 91, 31, 84, 98, 52, 20, 75, 79, 27, 19, 24, 94, 114, 17, 77, 81, 6, 10, 86, 29, 13, 73, 87, 8, 38, 111, 95, 34, 122, 42, 62, 23, 88, 72, 36, 7, 68, 69, 22, 59, 71, 30, 4, 5, 32, 103, 105, 106, 45, 28, 96, 41, 0, 3, 2, 35, 93, 64, 109, 67, 26, 66, 92, 65, 70, 101, 100, 1, 90, 37], [46, 110, 49, 121, 112, 40, 51, 56, 54, 104, 57, 127, 53, 125, 113, 116, 43, 44, 126, 61, 102, 119, 117, 124, 60, 123, 25, 39, 108, 89, 48, 16, 115, 58, 63, 55, 21, 14, 80, 78, 33, 107, 99, 12, 76, 120, 85, 62, 98, 82, 47, 18, 15, 19, 31, 97, 27, 50, 42, 72, 20, 84, 95, 6, 11, 91, 75, 114, 94, 81, 23, 9, 79, 83, 77, 74, 86, 17, 29, 10, 52, 34, 13, 118, 111, 24, 88, 122, 73, 87, 36, 38, 45, 22, 68, 41, 4, 7, 93, 105, 71, 30, 96, 106, 69, 8, 5, 28, 66, 109, 32, 59, 100, 64, 90, 0, 103, 3, 92, 35, 67, 65, 26, 2, 1, 101, 70, 37], [46, 110, 49, 121, 112, 54, 40, 56, 104, 57, 127, 125, 53, 51, 113, 116, 43, 44, 126, 61, 119, 102, 124, 60, 117, 123, 25, 108, 16, 89, 107, 39, 115, 48, 58, 80, 78, 14, 120, 33, 63, 12, 99, 76, 21, 98, 82, 27, 55, 97, 19, 47, 42, 84, 18, 72, 91, 11, 94, 6, 85, 15, 20, 83, 9, 75, 13, 17, 77, 79, 52, 24, 114, 95, 62, 10, 81, 50, 34, 118, 74, 87, 31, 88, 86, 23, 73, 111, 22, 36, 7, 71, 109, 69, 4, 68, 29, 105, 38, 5, 122, 106, 59, 96, 30, 41, 92, 8, 103, 32, 66, 28, 35, 45, 93, 2, 90, 26, 3, 70, 100, 101, 0, 67, 64, 1, 65, 37], [46, 110, 49, 121, 112, 40, 51, 54, 57, 56, 127, 104, 53, 113, 125, 116, 43, 126, 44, 61, 124, 102, 117, 63, 60, 119, 123, 108, 16, 89, 25, 33, 48, 39, 58, 80, 72, 78, 55, 12, 14, 98, 99, 107, 120, 115, 76, 50, 114, 95, 85, 82, 118, 42, 91, 21, 97, 18, 27, 47, 20, 15, 11, 19, 84, 79, 87, 94, 77, 74, 75, 10, 24, 17, 6, 73, 66, 86, 13, 83, 52, 62, 31, 69, 5, 9, 23, 111, 38, 88, 81, 4, 36, 68, 34, 29, 22, 106, 7, 71, 122, 2, 30, 64, 32, 0, 92, 70, 1, 59, 67, 3, 105, 65, 93, 96, 103, 109, 41, 28, 35, 45, 100, 90, 101, 8, 26, 37], [46, 110, 49, 121, 112, 54, 40, 127, 51, 57, 56, 104, 125, 53, 113, 116, 43, 44, 61, 126, 123, 119, 124, 102, 117, 63, 108, 33, 48, 89, 16, 72, 107, 115, 99, 58, 25, 60, 98, 39, 14, 55, 47, 114, 12, 78, 80, 76, 50, 21, 18, 82, 120, 27, 85, 118, 97, 91, 15, 84, 20, 95, 10, 111, 19, 11, 52, 87, 73, 75, 24, 42, 86, 36, 94, 31, 17, 34, 79, 83, 23, 74, 88, 81, 62, 77, 69, 9, 4, 70, 59, 5, 7, 13, 68, 22, 38, 71, 32, 30, 122, 29, 1, 3, 0, 6, 66, 105, 96, 92, 93, 106, 28, 65, 2, 109, 41, 45, 67, 35, 101, 64, 100, 103, 26, 90, 8, 37], [46, 110, 49, 121, 112, 54, 40, 104, 56, 57, 51, 125, 127, 53, 113, 116, 43, 44, 119, 61, 126, 123, 102, 124, 117, 89, 25, 108, 107, 48, 58, 63, 60, 39, 33, 16, 115, 99, 78, 80, 14, 12, 47, 21, 72, 76, 120, 18, 82, 77, 20, 85, 114, 79, 97, 55, 94, 27, 15, 111, 75, 11, 52, 118, 91, 19, 42, 84, 81, 17, 31, 73, 34, 10, 50, 83, 86, 87, 98, 24, 88, 95, 9, 70, 74, 13, 23, 38, 36, 29, 62, 7, 68, 22, 30, 122, 96, 5, 59, 41, 93, 105, 109, 69, 4, 71, 92, 28, 32, 106, 35, 45, 101, 100, 67, 66, 6, 26, 3, 8, 103, 37, 64, 1, 2, 90, 0, 65], [46, 110, 49, 121, 112, 56, 54, 51, 127, 40, 57, 104, 125, 113, 53, 116, 43, 44, 126, 61, 124, 102, 119, 60, 48, 123, 25, 108, 89, 117, 16, 39, 47, 33, 58, 21, 120, 99, 80, 107, 55, 12, 72, 115, 78, 14, 63, 76, 18, 97, 19, 82, 27, 85, 50, 70, 15, 42, 114, 11, 20, 84, 98, 75, 95, 118, 79, 77, 34, 13, 91, 86, 122, 17, 94, 10, 74, 73, 81, 83, 31, 59, 9, 62, 24, 88, 87, 52, 22, 4, 38, 5, 41, 23, 111, 29, 69, 71, 36, 68, 7, 105, 45, 92, 93, 64, 30, 35, 8, 28, 96, 101, 1, 100, 103, 32, 66, 2, 109, 3, 106, 67, 65, 26, 0, 6, 37, 90], [46, 110, 49, 112, 121, 54, 40, 104, 56, 57, 51, 127, 113, 125, 53, 116, 43, 44, 61, 126, 124, 123, 102, 119, 117, 89, 33, 60, 48, 25, 39, 63, 16, 108, 120, 78, 80, 47, 118, 99, 55, 107, 12, 14, 21, 76, 70, 58, 72, 115, 82, 18, 114, 98, 95, 85, 50, 42, 97, 91, 11, 27, 94, 20, 73, 79, 52, 74, 75, 31, 17, 15, 84, 88, 19, 86, 13, 83, 34, 81, 24, 62, 77, 23, 10, 22, 9, 71, 5, 68, 38, 4, 69, 122, 36, 8, 87, 29, 7, 32, 109, 105, 111, 93, 30, 59, 41, 67, 2, 92, 96, 66, 65, 45, 106, 1, 103, 0, 28, 101, 3, 100, 90, 64, 35, 26, 37, 6], [46, 110, 49, 112, 121, 54, 51, 40, 104, 127, 125, 56, 57, 113, 53, 116, 43, 44, 126, 61, 124, 117, 102, 123, 119, 63, 108, 107, 33, 60, 25, 48, 55, 115, 16, 39, 89, 120, 80, 58, 12, 78, 114, 47, 99, 118, 21, 52, 14, 50, 76, 18, 98, 97, 27, 42, 82, 95, 84, 85, 24, 72, 74, 11, 91, 20, 19, 88, 94, 70, 75, 36, 87, 73, 15, 8, 79, 23, 17, 86, 38, 31, 10, 111, 34, 13, 22, 4, 9, 81, 83, 77, 5, 29, 68, 122, 71, 69, 62, 41, 105, 7, 109, 28, 92, 65, 30, 59, 2, 32, 106, 96, 0, 3, 103, 67, 45, 93, 66, 64, 6, 35, 1, 26, 101, 90, 100, 37], [46, 110, 49, 121, 112, 54, 40, 104, 56, 51, 57, 125, 127, 113, 53, 116, 43, 44, 61, 126, 119, 102, 124, 107, 123, 117, 60, 25, 108, 89, 48, 58, 33, 63, 16, 115, 80, 76, 39, 78, 12, 14, 118, 99, 47, 55, 21, 120, 18, 98, 82, 15, 75, 84, 11, 9, 85, 19, 79, 91, 42, 52, 114, 20, 27, 74, 94, 77, 8, 31, 97, 73, 81, 86, 13, 17, 24, 50, 88, 10, 87, 83, 111, 70, 95, 36, 38, 72, 62, 23, 4, 34, 29, 71, 22, 68, 7, 122, 5, 69, 41, 109, 45, 28, 106, 6, 35, 59, 32, 92, 105, 93, 30, 96, 101, 103, 67, 100, 64, 2, 66, 3, 90, 26, 1, 0, 65, 37], [46, 110, 49, 121, 112, 54, 40, 56, 104, 57, 51, 53, 127, 113, 125, 116, 43, 44, 126, 61, 124, 102, 60, 119, 123, 117, 25, 108, 39, 16, 33, 89, 107, 48, 58, 118, 55, 21, 80, 78, 76, 14, 12, 115, 8, 99, 63, 18, 97, 82, 120, 91, 85, 98, 79, 11, 47, 19, 81, 20, 84, 50, 86, 77, 114, 15, 42, 95, 10, 122, 52, 75, 27, 88, 17, 73, 13, 34, 94, 62, 6, 74, 9, 83, 22, 31, 87, 24, 111, 5, 23, 68, 38, 45, 36, 71, 29, 41, 93, 7, 59, 4, 105, 69, 32, 30, 28, 106, 103, 72, 70, 66, 92, 96, 35, 0, 2, 64, 100, 109, 65, 101, 1, 67, 90, 26, 3, 37], [46, 110, 49, 121, 112, 127, 54, 40, 56, 104, 51, 57, 125, 113, 116, 53, 43, 126, 44, 61, 102, 124, 119, 117, 123, 60, 48, 108, 33, 25, 80, 39, 63, 58, 89, 16, 14, 8, 21, 12, 107, 78, 76, 120, 55, 115, 118, 99, 18, 19, 91, 97, 82, 47, 114, 50, 85, 20, 6, 62, 98, 42, 27, 11, 84, 75, 86, 79, 10, 81, 95, 77, 15, 13, 88, 9, 34, 23, 52, 24, 74, 73, 94, 17, 87, 122, 83, 5, 31, 22, 36, 4, 109, 29, 105, 71, 68, 111, 38, 45, 69, 7, 32, 0, 59, 92, 30, 106, 28, 2, 41, 67, 93, 66, 65, 64, 96, 3, 103, 101, 35, 1, 90, 100, 72, 70, 26, 37], [46, 110, 49, 121, 112, 54, 40, 104, 51, 56, 127, 57, 113, 53, 116, 125, 43, 44, 126, 61, 124, 123, 119, 102, 25, 108, 63, 117, 60, 89, 48, 8, 80, 16, 39, 14, 115, 107, 78, 33, 55, 120, 12, 76, 58, 47, 6, 99, 98, 82, 18, 50, 21, 95, 20, 75, 97, 114, 85, 11, 9, 91, 13, 27, 84, 10, 15, 73, 19, 79, 118, 17, 74, 42, 86, 24, 62, 31, 94, 52, 5, 111, 81, 77, 68, 59, 87, 23, 36, 7, 83, 4, 88, 34, 69, 45, 71, 22, 106, 122, 38, 29, 96, 105, 92, 35, 30, 66, 3, 32, 100, 67, 109, 93, 28, 103, 41, 64, 26, 1, 101, 65, 2, 0, 90, 72, 70, 37], [46, 110, 49, 121, 112, 54, 40, 51, 56, 104, 57, 127, 113, 53, 116, 125, 43, 44, 124, 61, 126, 117, 102, 119, 48, 123, 33, 25, 58, 107, 108, 47, 16, 39, 60, 63, 80, 89, 55, 8, 12, 99, 14, 98, 78, 76, 115, 114, 97, 82, 120, 18, 21, 6, 50, 91, 95, 75, 79, 27, 85, 10, 20, 42, 94, 84, 74, 81, 11, 17, 83, 15, 19, 9, 31, 87, 52, 36, 73, 38, 34, 86, 88, 77, 4, 13, 71, 24, 22, 118, 111, 62, 23, 29, 5, 122, 69, 7, 105, 68, 106, 59, 30, 32, 28, 93, 66, 45, 96, 103, 92, 109, 64, 0, 100, 41, 2, 101, 35, 72, 3, 67, 90, 70, 1, 26, 65, 37], [46, 110, 49, 121, 112, 54, 56, 40, 57, 104, 127, 51, 113, 53, 116, 125, 43, 61, 44, 126, 123, 102, 119, 117, 124, 60, 39, 48, 25, 107, 108, 63, 33, 89, 80, 115, 114, 78, 16, 58, 55, 14, 99, 47, 12, 98, 8, 76, 118, 21, 50, 120, 18, 82, 97, 84, 27, 95, 85, 83, 75, 11, 79, 52, 9, 20, 34, 19, 74, 15, 81, 91, 13, 87, 6, 42, 73, 10, 36, 77, 94, 17, 69, 86, 31, 38, 29, 71, 4, 88, 111, 5, 24, 23, 59, 22, 68, 7, 122, 28, 64, 109, 30, 45, 41, 62, 66, 96, 106, 70, 35, 93, 100, 103, 72, 92, 67, 1, 105, 32, 2, 3, 0, 101, 65, 26, 90, 37], [46, 110, 49, 121, 112, 51, 54, 40, 104, 57, 127, 113, 56, 125, 53, 116, 43, 44, 126, 61, 119, 117, 124, 102, 123, 25, 33, 58, 55, 89, 63, 107, 16, 108, 47, 48, 39, 60, 115, 14, 80, 78, 12, 76, 99, 21, 85, 82, 8, 18, 97, 114, 98, 120, 75, 27, 20, 84, 17, 91, 50, 118, 13, 19, 79, 10, 94, 95, 15, 74, 31, 87, 11, 77, 81, 9, 24, 34, 88, 23, 83, 73, 86, 68, 22, 29, 62, 52, 72, 4, 38, 36, 70, 42, 71, 111, 5, 109, 69, 45, 59, 30, 7, 28, 122, 41, 106, 32, 96, 6, 93, 92, 105, 35, 67, 100, 2, 1, 103, 3, 64, 0, 101, 90, 66, 26, 65, 37], [46, 110, 49, 121, 112, 51, 40, 57, 56, 54, 104, 127, 113, 125, 53, 116, 43, 44, 126, 61, 124, 102, 119, 123, 60, 117, 25, 108, 58, 107, 89, 48, 33, 80, 39, 16, 63, 78, 115, 47, 12, 76, 99, 55, 14, 98, 18, 120, 27, 19, 85, 97, 94, 114, 79, 21, 20, 83, 95, 82, 42, 91, 15, 75, 8, 52, 11, 84, 17, 74, 70, 50, 118, 24, 77, 73, 13, 86, 10, 72, 87, 22, 9, 36, 81, 34, 88, 31, 23, 38, 111, 4, 62, 122, 69, 68, 29, 7, 30, 71, 5, 103, 96, 45, 28, 41, 92, 106, 93, 67, 2, 109, 32, 105, 66, 101, 35, 59, 100, 0, 64, 1, 6, 26, 65, 90, 3, 37], [46, 110, 49, 121, 112, 40, 54, 104, 56, 57, 127, 51, 125, 113, 53, 116, 43, 44, 61, 126, 119, 123, 124, 117, 102, 33, 108, 39, 48, 63, 60, 89, 107, 25, 58, 16, 78, 115, 55, 99, 80, 14, 47, 76, 12, 18, 70, 120, 97, 118, 114, 85, 98, 21, 27, 20, 75, 84, 72, 11, 52, 73, 9, 82, 17, 15, 91, 42, 79, 50, 94, 87, 13, 62, 81, 74, 10, 19, 95, 24, 34, 83, 88, 77, 86, 5, 111, 31, 36, 7, 8, 122, 71, 4, 68, 22, 69, 23, 38, 109, 30, 29, 106, 32, 105, 45, 35, 2, 103, 92, 67, 96, 28, 101, 3, 93, 64, 66, 59, 41, 100, 65, 26, 1, 0, 90, 6, 37], [46, 110, 49, 121, 112, 54, 40, 51, 127, 56, 104, 57, 125, 113, 53, 116, 43, 44, 126, 61, 124, 102, 123, 119, 117, 89, 39, 107, 60, 48, 25, 55, 33, 108, 58, 47, 16, 78, 99, 80, 115, 114, 76, 63, 12, 14, 118, 85, 120, 18, 72, 27, 50, 70, 97, 98, 79, 91, 21, 19, 20, 82, 95, 11, 84, 83, 75, 15, 87, 74, 13, 52, 34, 9, 17, 122, 77, 81, 94, 10, 31, 42, 73, 22, 38, 24, 62, 86, 111, 36, 7, 5, 69, 88, 29, 23, 68, 30, 64, 4, 71, 32, 103, 106, 35, 41, 45, 28, 105, 109, 59, 92, 8, 67, 93, 1, 96, 66, 2, 26, 100, 0, 101, 3, 65, 90, 37, 6], [46, 110, 49, 121, 112, 40, 57, 104, 54, 56, 51, 127, 125, 53, 113, 116, 43, 44, 126, 61, 124, 117, 102, 119, 123, 60, 39, 25, 89, 108, 48, 63, 107, 33, 80, 16, 99, 115, 58, 78, 12, 55, 14, 76, 21, 47, 72, 50, 97, 82, 91, 20, 85, 98, 27, 18, 94, 83, 114, 79, 120, 118, 38, 24, 73, 19, 11, 95, 75, 77, 34, 86, 13, 74, 52, 15, 42, 88, 17, 81, 84, 62, 70, 87, 31, 10, 36, 45, 9, 111, 23, 5, 122, 4, 22, 71, 29, 68, 7, 69, 92, 30, 106, 93, 41, 32, 96, 103, 28, 59, 2, 109, 6, 101, 35, 105, 8, 90, 3, 1, 26, 67, 66, 0, 100, 65, 64, 37], [46, 110, 49, 121, 112, 40, 54, 56, 104, 51, 57, 113, 53, 125, 127, 116, 43, 44, 61, 126, 119, 102, 60, 124, 39, 123, 117, 108, 89, 72, 58, 25, 16, 33, 48, 80, 107, 78, 63, 76, 120, 12, 99, 14, 98, 55, 115, 47, 50, 118, 11, 18, 79, 85, 82, 73, 42, 114, 95, 21, 97, 91, 9, 75, 84, 10, 27, 20, 77, 17, 19, 13, 15, 83, 34, 38, 74, 52, 122, 94, 69, 24, 4, 31, 81, 23, 68, 22, 70, 88, 7, 71, 62, 36, 29, 86, 87, 6, 5, 111, 45, 109, 32, 3, 28, 59, 2, 30, 0, 1, 90, 103, 105, 67, 65, 96, 66, 35, 93, 64, 92, 101, 106, 100, 41, 26, 8, 37], [46, 110, 49, 121, 112, 54, 40, 57, 56, 104, 127, 51, 113, 125, 53, 116, 43, 126, 44, 61, 123, 124, 102, 119, 60, 117, 108, 58, 115, 33, 89, 48, 25, 72, 16, 107, 39, 63, 80, 76, 78, 99, 14, 120, 47, 55, 118, 12, 98, 50, 114, 82, 27, 21, 18, 52, 85, 97, 20, 9, 83, 11, 91, 84, 74, 75, 79, 15, 10, 95, 86, 87, 19, 122, 42, 13, 34, 81, 94, 24, 17, 73, 6, 77, 36, 5, 23, 88, 38, 69, 7, 111, 22, 45, 31, 68, 4, 71, 109, 30, 62, 29, 59, 28, 106, 92, 105, 3, 103, 32, 2, 70, 96, 65, 1, 64, 100, 0, 66, 90, 41, 93, 67, 8, 35, 101, 37, 26], [46, 110, 49, 121, 112, 54, 40, 51, 56, 57, 104, 127, 113, 125, 116, 53, 43, 44, 61, 126, 124, 102, 123, 119, 117, 108, 39, 107, 60, 25, 58, 48, 16, 89, 72, 33, 63, 80, 12, 76, 55, 99, 78, 14, 115, 98, 47, 50, 18, 82, 21, 6, 85, 95, 9, 114, 120, 11, 27, 97, 20, 79, 91, 118, 75, 84, 83, 73, 10, 42, 15, 34, 36, 86, 13, 77, 19, 24, 74, 88, 31, 87, 62, 94, 52, 17, 7, 4, 81, 68, 5, 111, 38, 71, 69, 103, 45, 23, 29, 122, 22, 93, 106, 32, 105, 109, 59, 30, 41, 35, 8, 28, 3, 92, 1, 64, 2, 67, 66, 70, 0, 65, 96, 100, 90, 101, 26, 37], [46, 110, 49, 121, 112, 54, 40, 56, 104, 51, 127, 57, 125, 113, 116, 53, 43, 44, 61, 126, 119, 102, 124, 123, 117, 60, 108, 25, 89, 39, 107, 48, 80, 115, 16, 12, 58, 99, 76, 78, 14, 33, 47, 18, 55, 72, 19, 21, 27, 6, 63, 75, 120, 94, 20, 82, 9, 50, 98, 83, 91, 73, 11, 15, 34, 42, 84, 13, 88, 85, 79, 81, 52, 97, 77, 74, 10, 95, 86, 114, 118, 17, 68, 87, 24, 31, 111, 36, 71, 106, 22, 62, 23, 5, 7, 38, 29, 109, 93, 4, 30, 8, 103, 122, 35, 69, 45, 92, 41, 59, 105, 32, 66, 67, 100, 28, 64, 0, 65, 3, 96, 2, 101, 90, 26, 70, 1, 37], [46, 110, 49, 112, 121, 54, 51, 40, 57, 56, 104, 127, 125, 113, 53, 116, 43, 44, 126, 61, 124, 102, 60, 123, 119, 117, 89, 58, 108, 48, 55, 39, 63, 33, 25, 80, 107, 99, 50, 16, 14, 47, 78, 76, 12, 18, 114, 115, 120, 42, 21, 85, 118, 83, 19, 98, 95, 27, 34, 97, 52, 82, 79, 91, 20, 84, 75, 6, 77, 94, 15, 11, 73, 17, 10, 13, 31, 81, 38, 72, 9, 88, 86, 62, 87, 74, 24, 8, 22, 59, 106, 36, 23, 122, 29, 5, 68, 7, 71, 4, 111, 93, 41, 30, 92, 103, 69, 109, 45, 32, 96, 100, 66, 35, 65, 90, 1, 64, 0, 28, 105, 101, 3, 67, 26, 37, 70, 2]], "model.layers.3.self_attn.q_proj": [[40, 60, 98, 56, 52, 57, 119, 62, 32, 58, 49, 37, 113, 118, 53, 59, 95, 123, 48, 105, 89, 50, 24, 45, 122, 63, 120, 25, 101, 26, 30, 93, 39, 86, 51, 61, 85, 42, 34, 17, 38, 109, 111, 102, 35, 107, 36, 121, 114, 43, 116, 21, 126, 103, 54, 83, 84, 125, 29, 112, 108, 55, 46, 97, 96, 115, 92, 127, 110, 124, 31, 99, 47, 44, 33, 100, 41, 91, 88, 94, 15, 106, 20, 18, 117, 90, 27, 28, 82, 87, 23, 22, 81, 104, 80, 11, 78, 75, 76, 79, 74, 77, 10, 19, 8, 72, 14, 12, 68, 13, 65, 71, 5, 73, 16, 4, 2, 67, 69, 6, 9, 7, 70, 3, 1, 66, 0, 64], [119, 62, 40, 98, 121, 52, 123, 58, 53, 48, 50, 118, 38, 25, 49, 30, 29, 54, 39, 63, 60, 57, 84, 56, 122, 59, 42, 115, 86, 102, 112, 33, 27, 37, 93, 113, 116, 28, 32, 111, 107, 89, 83, 15, 46, 108, 114, 120, 103, 110, 125, 101, 80, 18, 88, 36, 45, 41, 34, 47, 21, 43, 127, 24, 55, 51, 117, 44, 23, 109, 14, 124, 61, 100, 105, 97, 92, 106, 126, 73, 35, 26, 104, 12, 72, 19, 99, 31, 94, 17, 22, 95, 85, 90, 91, 96, 74, 20, 67, 70, 76, 79, 87, 81, 11, 10, 75, 71, 2, 13, 9, 5, 4, 65, 82, 68, 78, 7, 8, 0, 77, 3, 1, 66, 6, 64, 16, 69], [40, 98, 56, 60, 62, 119, 86, 29, 52, 58, 83, 14, 80, 57, 12, 54, 59, 10, 48, 93, 17, 15, 50, 63, 24, 25, 125, 4, 123, 118, 121, 101, 9, 120, 74, 113, 72, 26, 5, 85, 114, 51, 106, 33, 99, 53, 70, 45, 122, 95, 73, 71, 78, 76, 11, 16, 30, 81, 19, 22, 89, 104, 32, 103, 108, 21, 20, 88, 66, 67, 90, 84, 7, 2, 41, 49, 18, 111, 13, 1, 39, 23, 38, 87, 110, 75, 43, 36, 27, 6, 0, 82, 77, 79, 69, 37, 47, 8, 3, 97, 100, 102, 94, 92, 126, 96, 127, 34, 91, 42, 61, 28, 109, 115, 117, 35, 31, 116, 64, 105, 112, 68, 46, 55, 107, 65, 44, 124], [58, 40, 98, 119, 62, 60, 52, 29, 57, 86, 56, 48, 12, 80, 83, 59, 14, 24, 50, 51, 10, 9, 54, 72, 95, 73, 4, 19, 11, 71, 114, 116, 93, 66, 100, 76, 69, 63, 123, 32, 2, 49, 70, 106, 0, 125, 113, 16, 118, 68, 91, 5, 26, 7, 67, 75, 82, 53, 39, 90, 127, 79, 28, 120, 99, 25, 23, 103, 109, 27, 94, 22, 55, 111, 81, 13, 77, 108, 89, 18, 61, 117, 47, 46, 85, 87, 8, 121, 78, 88, 38, 36, 21, 31, 107, 3, 17, 37, 20, 30, 42, 15, 35, 122, 124, 96, 112, 84, 92, 45, 102, 43, 105, 101, 97, 33, 44, 64, 104, 126, 74, 6, 41, 115, 1, 110, 34, 65], [38, 97, 111, 120, 123, 122, 47, 84, 81, 10, 14, 71, 12, 8, 2, 67, 69, 58, 1, 0, 87, 88, 76, 19, 65, 83, 93, 106, 17, 86, 57, 34, 53, 11, 42, 32, 25, 118, 26, 20, 85, 99, 96, 56, 80, 49, 66, 31, 15, 78, 3, 105, 95, 89, 112, 109, 94, 52, 124, 101, 98, 110, 30, 126, 90, 92, 37, 22, 64, 55, 115, 82, 91, 103, 79, 44, 48, 74, 73, 119, 113, 60, 27, 45, 36, 4, 39, 29, 54, 62, 41, 61, 100, 21, 121, 108, 35, 63, 28, 51, 72, 46, 7, 18, 114, 127, 107, 125, 75, 77, 117, 16, 59, 23, 50, 43, 116, 104, 40, 13, 70, 5, 9, 24, 6, 68, 33, 102], [38, 97, 111, 122, 123, 120, 47, 81, 23, 84, 14, 71, 12, 10, 8, 69, 67, 75, 19, 73, 25, 7, 3, 1, 56, 76, 21, 105, 13, 78, 58, 5, 65, 2, 49, 79, 64, 4, 94, 70, 66, 30, 113, 0, 31, 98, 6, 112, 72, 16, 126, 86, 61, 114, 87, 52, 15, 88, 43, 35, 74, 42, 124, 62, 119, 63, 68, 37, 11, 104, 121, 45, 118, 17, 99, 20, 110, 57, 93, 82, 24, 26, 90, 77, 80, 32, 28, 22, 103, 85, 117, 46, 36, 9, 51, 106, 125, 18, 108, 92, 59, 83, 55, 115, 89, 60, 53, 41, 29, 44, 107, 33, 100, 95, 39, 127, 48, 54, 27, 101, 116, 50, 40, 91, 109, 96, 102, 34], [38, 111, 97, 123, 120, 122, 47, 81, 14, 84, 23, 12, 10, 8, 69, 71, 59, 75, 58, 3, 1, 86, 67, 16, 11, 40, 56, 127, 88, 25, 24, 105, 50, 31, 32, 87, 51, 9, 90, 45, 116, 76, 43, 73, 110, 74, 5, 125, 22, 21, 106, 4, 2, 62, 82, 19, 91, 78, 13, 98, 72, 113, 26, 93, 27, 28, 48, 100, 20, 83, 99, 66, 94, 15, 53, 117, 39, 17, 95, 42, 80, 49, 104, 108, 63, 65, 0, 118, 96, 126, 55, 60, 33, 92, 61, 77, 37, 85, 114, 30, 107, 35, 52, 119, 46, 44, 79, 103, 6, 41, 34, 109, 7, 29, 57, 18, 36, 115, 112, 64, 54, 101, 121, 89, 70, 102, 124, 68], [38, 97, 111, 123, 122, 120, 47, 84, 81, 14, 23, 12, 10, 71, 24, 8, 69, 67, 3, 1, 55, 105, 93, 76, 44, 59, 78, 86, 7, 56, 58, 70, 73, 16, 61, 107, 25, 30, 2, 21, 11, 57, 60, 126, 6, 51, 68, 4, 89, 80, 88, 79, 17, 104, 72, 32, 35, 20, 19, 74, 66, 95, 46, 41, 42, 26, 96, 43, 94, 75, 0, 98, 116, 106, 40, 15, 22, 48, 5, 50, 54, 9, 29, 63, 87, 124, 109, 114, 118, 110, 82, 37, 53, 112, 113, 27, 125, 45, 92, 121, 91, 85, 77, 34, 117, 62, 39, 49, 101, 65, 103, 119, 31, 90, 83, 28, 100, 115, 99, 36, 64, 52, 127, 18, 13, 108, 33, 102], [41, 34, 18, 20, 52, 76, 89, 79, 22, 16, 14, 120, 114, 62, 8, 10, 105, 46, 47, 63, 81, 126, 116, 69, 13, 125, 117, 7, 9, 127, 3, 59, 123, 72, 71, 5, 80, 23, 28, 73, 53, 70, 112, 12, 74, 50, 119, 24, 78, 19, 113, 6, 49, 66, 58, 15, 2, 110, 25, 75, 82, 54, 17, 109, 107, 95, 85, 86, 67, 11, 83, 94, 111, 118, 104, 60, 29, 37, 55, 61, 32, 93, 122, 27, 88, 68, 1, 39, 56, 115, 84, 64, 38, 42, 31, 43, 33, 101, 102, 87, 21, 26, 44, 103, 100, 0, 97, 35, 96, 121, 57, 48, 91, 92, 77, 45, 108, 106, 124, 4, 99, 51, 30, 90, 40, 65, 36, 98], [41, 34, 89, 52, 20, 22, 120, 18, 47, 46, 63, 114, 126, 80, 116, 76, 62, 105, 72, 14, 28, 78, 125, 117, 11, 79, 123, 82, 127, 13, 85, 7, 59, 74, 77, 10, 12, 19, 84, 58, 21, 15, 49, 9, 107, 119, 53, 54, 90, 24, 86, 81, 113, 5, 109, 73, 25, 95, 112, 29, 8, 23, 92, 103, 27, 104, 30, 37, 50, 61, 102, 39, 16, 88, 69, 96, 35, 32, 94, 3, 31, 71, 26, 110, 111, 101, 55, 33, 60, 124, 51, 93, 36, 83, 17, 38, 66, 57, 118, 43, 87, 42, 115, 98, 91, 75, 40, 4, 56, 99, 1, 108, 44, 122, 97, 100, 121, 6, 67, 68, 65, 45, 48, 106, 70, 2, 64, 0], [41, 34, 52, 81, 22, 18, 114, 20, 126, 76, 79, 120, 14, 10, 62, 69, 47, 116, 8, 3, 105, 63, 5, 46, 117, 125, 127, 72, 7, 9, 89, 54, 66, 123, 24, 1, 70, 6, 59, 74, 53, 71, 16, 113, 49, 28, 2, 64, 58, 77, 86, 110, 32, 73, 0, 119, 27, 68, 85, 50, 107, 112, 78, 12, 67, 17, 87, 83, 15, 13, 80, 84, 26, 104, 109, 61, 25, 92, 90, 60, 96, 118, 56, 38, 65, 57, 75, 95, 19, 45, 43, 82, 11, 111, 91, 99, 88, 35, 93, 33, 101, 31, 97, 122, 21, 29, 30, 37, 102, 39, 40, 100, 44, 42, 23, 106, 94, 55, 36, 124, 121, 115, 48, 103, 51, 4, 108, 98], [41, 34, 20, 18, 22, 52, 10, 76, 14, 6, 67, 62, 46, 120, 79, 126, 125, 114, 3, 116, 47, 105, 7, 8, 63, 9, 69, 127, 54, 2, 28, 117, 59, 58, 25, 0, 13, 123, 70, 110, 72, 80, 77, 112, 12, 49, 24, 90, 65, 75, 44, 113, 11, 119, 53, 71, 74, 23, 73, 91, 104, 35, 32, 78, 85, 15, 64, 87, 86, 89, 84, 88, 45, 66, 61, 21, 26, 5, 17, 68, 95, 60, 1, 42, 121, 83, 51, 122, 92, 100, 107, 56, 103, 109, 36, 106, 111, 94, 81, 48, 82, 38, 50, 55, 43, 16, 37, 19, 31, 102, 27, 29, 108, 124, 57, 97, 39, 93, 101, 99, 40, 115, 33, 96, 118, 30, 4, 98], [103, 117, 125, 109, 46, 56, 122, 97, 58, 52, 63, 57, 47, 28, 126, 59, 115, 89, 39, 21, 124, 119, 51, 49, 116, 62, 110, 60, 113, 55, 114, 111, 54, 86, 50, 87, 112, 123, 127, 61, 121, 53, 81, 33, 82, 48, 118, 108, 45, 120, 24, 107, 20, 44, 41, 31, 42, 19, 105, 106, 79, 43, 13, 96, 37, 30, 40, 100, 102, 104, 101, 99, 38, 36, 94, 34, 35, 95, 98, 32, 73, 80, 92, 93, 90, 17, 22, 91, 29, 76, 12, 26, 78, 88, 25, 71, 70, 27, 16, 83, 7, 9, 10, 85, 77, 4, 23, 68, 18, 84, 74, 5, 6, 8, 15, 3, 14, 67, 66, 0, 11, 65, 75, 69, 1, 64, 72, 2], [103, 117, 52, 109, 125, 56, 97, 122, 46, 119, 57, 28, 63, 58, 89, 21, 82, 115, 9, 47, 12, 78, 87, 126, 80, 20, 124, 33, 49, 116, 13, 62, 19, 110, 31, 83, 114, 59, 86, 54, 17, 111, 92, 61, 123, 127, 50, 60, 51, 94, 30, 55, 25, 6, 112, 74, 107, 81, 79, 71, 118, 77, 90, 11, 108, 121, 7, 24, 113, 53, 39, 48, 16, 101, 15, 18, 44, 32, 88, 95, 73, 4, 67, 22, 36, 45, 29, 84, 76, 69, 43, 41, 8, 68, 105, 37, 23, 26, 91, 85, 70, 5, 104, 93, 99, 14, 34, 75, 10, 100, 96, 120, 42, 106, 35, 3, 65, 38, 102, 66, 0, 27, 40, 98, 1, 2, 72, 64], [103, 117, 52, 125, 109, 97, 119, 122, 21, 89, 56, 28, 46, 59, 60, 20, 78, 82, 79, 87, 58, 47, 80, 55, 9, 124, 113, 50, 126, 121, 57, 12, 114, 123, 111, 62, 7, 74, 11, 63, 25, 118, 51, 53, 61, 92, 15, 54, 4, 22, 90, 116, 31, 112, 19, 110, 30, 115, 127, 17, 107, 33, 83, 70, 49, 95, 8, 13, 91, 98, 93, 6, 69, 24, 81, 44, 94, 77, 32, 71, 48, 67, 88, 73, 45, 38, 23, 99, 43, 42, 86, 29, 26, 66, 104, 84, 37, 5, 76, 101, 96, 102, 120, 39, 105, 41, 106, 34, 108, 100, 40, 1, 27, 16, 36, 10, 85, 68, 35, 18, 14, 0, 75, 2, 65, 64, 72, 3], [103, 117, 52, 46, 97, 125, 109, 119, 56, 122, 21, 63, 82, 78, 80, 87, 58, 9, 83, 89, 20, 12, 28, 92, 57, 8, 11, 6, 126, 15, 59, 75, 25, 79, 67, 124, 54, 74, 16, 13, 81, 114, 49, 62, 76, 115, 71, 123, 31, 53, 110, 68, 69, 60, 85, 90, 18, 19, 77, 95, 30, 14, 23, 127, 66, 111, 50, 72, 48, 33, 24, 84, 113, 10, 64, 70, 73, 51, 5, 3, 47, 112, 61, 4, 26, 22, 86, 55, 17, 93, 1, 2, 88, 27, 29, 0, 94, 118, 107, 91, 32, 96, 44, 99, 34, 65, 121, 108, 116, 101, 7, 41, 98, 100, 36, 35, 102, 106, 104, 120, 37, 43, 42, 38, 40, 105, 45, 39], [104, 33, 108, 20, 18, 23, 16, 13, 10, 44, 72, 68, 70, 48, 56, 107, 63, 55, 40, 117, 54, 84, 116, 57, 90, 21, 50, 12, 66, 14, 110, 31, 115, 51, 11, 9, 58, 73, 78, 61, 109, 114, 87, 111, 96, 121, 123, 25, 52, 120, 79, 1, 5, 82, 26, 100, 86, 43, 74, 125, 127, 59, 94, 119, 83, 118, 3, 76, 15, 126, 2, 38, 92, 45, 77, 34, 6, 8, 22, 102, 41, 62, 24, 88, 95, 71, 112, 36, 7, 60, 105, 106, 93, 80, 89, 42, 99, 85, 113, 47, 67, 101, 27, 39, 17, 98, 81, 46, 29, 103, 69, 53, 28, 30, 19, 35, 4, 91, 49, 124, 75, 122, 37, 32, 64, 65, 97, 0], [104, 108, 33, 65, 10, 13, 68, 18, 70, 1, 16, 64, 7, 44, 23, 67, 66, 87, 40, 0, 72, 20, 4, 56, 110, 69, 111, 116, 120, 3, 55, 5, 50, 12, 115, 6, 57, 61, 11, 123, 71, 54, 58, 84, 113, 77, 112, 48, 8, 2, 19, 62, 125, 114, 117, 51, 100, 43, 124, 52, 118, 78, 38, 14, 26, 9, 99, 80, 97, 63, 31, 96, 45, 24, 83, 49, 79, 21, 59, 107, 127, 29, 119, 74, 30, 60, 88, 109, 39, 122, 121, 89, 15, 85, 92, 36, 91, 41, 86, 32, 101, 25, 106, 94, 37, 93, 126, 34, 42, 90, 75, 82, 105, 53, 17, 73, 98, 103, 76, 47, 22, 27, 46, 28, 35, 81, 102, 95], [104, 108, 33, 64, 0, 44, 70, 13, 68, 10, 7, 66, 67, 16, 20, 18, 87, 1, 40, 23, 65, 72, 111, 56, 50, 110, 57, 55, 5, 120, 48, 116, 4, 84, 113, 115, 11, 123, 12, 61, 71, 51, 114, 100, 3, 117, 58, 21, 6, 118, 43, 125, 2, 80, 62, 45, 77, 52, 96, 75, 54, 69, 109, 86, 8, 126, 41, 38, 124, 121, 39, 47, 76, 101, 74, 27, 59, 89, 25, 99, 31, 35, 28, 79, 24, 81, 107, 83, 82, 34, 78, 92, 90, 60, 95, 37, 19, 94, 14, 9, 98, 127, 49, 17, 91, 29, 53, 30, 88, 106, 63, 85, 26, 112, 42, 36, 102, 32, 105, 46, 122, 103, 22, 93, 15, 119, 73, 97], [104, 108, 33, 23, 16, 18, 10, 44, 20, 13, 7, 72, 12, 116, 55, 87, 40, 110, 61, 67, 68, 107, 115, 117, 56, 70, 48, 114, 69, 50, 57, 125, 109, 1, 66, 118, 123, 120, 60, 54, 124, 111, 21, 81, 83, 25, 58, 126, 78, 64, 11, 59, 29, 63, 9, 88, 28, 90, 43, 32, 52, 46, 42, 62, 2, 8, 85, 47, 101, 73, 27, 89, 15, 113, 84, 76, 49, 103, 91, 99, 31, 24, 14, 3, 119, 86, 22, 98, 105, 122, 121, 36, 75, 79, 41, 106, 45, 30, 94, 127, 92, 80, 82, 93, 19, 96, 112, 77, 53, 17, 38, 6, 26, 95, 102, 39, 34, 100, 37, 4, 51, 71, 35, 5, 74, 97, 65, 0], [41, 46, 53, 127, 40, 97, 60, 105, 91, 31, 119, 39, 122, 47, 27, 112, 38, 24, 116, 99, 36, 115, 118, 42, 93, 57, 117, 33, 43, 125, 51, 126, 35, 110, 85, 55, 18, 114, 90, 87, 102, 88, 92, 96, 111, 100, 29, 124, 20, 109, 59, 49, 52, 95, 123, 63, 50, 56, 81, 44, 23, 30, 106, 48, 104, 79, 120, 108, 98, 25, 80, 107, 62, 121, 34, 37, 22, 94, 77, 82, 19, 113, 14, 26, 9, 58, 61, 28, 45, 54, 101, 75, 21, 83, 103, 13, 16, 86, 32, 84, 89, 74, 11, 76, 17, 5, 72, 15, 6, 66, 4, 73, 71, 2, 69, 10, 67, 70, 1, 68, 78, 64, 12, 0, 7, 8, 65, 3], [40, 97, 53, 127, 46, 24, 85, 31, 110, 14, 81, 75, 88, 60, 19, 90, 93, 41, 76, 47, 115, 71, 80, 74, 9, 4, 122, 82, 83, 119, 21, 91, 43, 38, 6, 104, 96, 55, 105, 42, 112, 27, 3, 72, 73, 33, 68, 101, 100, 37, 22, 26, 63, 124, 108, 13, 29, 54, 62, 126, 69, 30, 25, 125, 98, 114, 84, 15, 66, 117, 23, 45, 10, 77, 70, 118, 51, 16, 35, 106, 79, 87, 113, 17, 92, 34, 116, 1, 102, 89, 52, 44, 103, 12, 28, 48, 99, 56, 2, 20, 64, 111, 78, 36, 18, 123, 7, 61, 11, 94, 39, 65, 32, 50, 95, 86, 49, 8, 5, 59, 109, 57, 67, 121, 0, 107, 58, 120], [41, 110, 40, 53, 97, 127, 60, 43, 46, 38, 31, 47, 112, 91, 92, 35, 116, 52, 19, 122, 113, 33, 124, 119, 117, 26, 93, 59, 39, 27, 24, 125, 108, 63, 86, 56, 123, 95, 118, 57, 115, 49, 29, 126, 48, 45, 58, 88, 28, 76, 100, 96, 55, 42, 114, 18, 80, 85, 121, 99, 13, 87, 103, 37, 36, 22, 106, 51, 50, 111, 61, 90, 98, 109, 94, 12, 62, 105, 54, 14, 25, 107, 21, 20, 30, 44, 83, 82, 77, 101, 104, 70, 89, 6, 34, 102, 120, 84, 32, 81, 79, 23, 74, 16, 75, 15, 72, 9, 67, 3, 10, 78, 5, 71, 2, 0, 66, 73, 4, 65, 11, 8, 1, 69, 7, 68, 17, 64], [40, 46, 127, 97, 53, 43, 31, 38, 67, 41, 60, 75, 5, 14, 85, 71, 81, 65, 110, 24, 0, 9, 39, 90, 1, 19, 104, 47, 115, 112, 35, 91, 77, 21, 27, 88, 26, 17, 72, 7, 59, 102, 121, 124, 80, 93, 23, 79, 10, 76, 125, 48, 44, 2, 70, 8, 105, 100, 118, 54, 3, 56, 52, 103, 92, 64, 69, 116, 4, 12, 82, 57, 96, 87, 78, 49, 99, 36, 119, 42, 68, 74, 98, 11, 15, 117, 114, 18, 25, 86, 126, 37, 63, 28, 106, 122, 66, 34, 108, 120, 83, 84, 32, 62, 50, 94, 73, 107, 95, 22, 111, 30, 51, 113, 55, 16, 58, 61, 123, 109, 101, 20, 13, 45, 33, 89, 6, 29], [39, 124, 34, 117, 119, 47, 24, 62, 109, 20, 15, 13, 48, 26, 17, 11, 53, 70, 9, 72, 55, 19, 1, 65, 73, 28, 10, 78, 69, 123, 5, 21, 118, 68, 86, 83, 106, 81, 71, 49, 95, 112, 122, 25, 87, 31, 79, 116, 12, 92, 85, 18, 88, 27, 35, 115, 30, 90, 107, 14, 75, 8, 89, 7, 77, 66, 74, 22, 23, 94, 84, 45, 80, 91, 33, 3, 61, 6, 67, 76, 110, 52, 16, 82, 121, 2, 64, 4, 50, 93, 54, 29, 32, 43, 0, 97, 99, 41, 51, 111, 100, 63, 46, 37, 101, 36, 38, 56, 96, 59, 44, 127, 60, 40, 120, 114, 102, 42, 98, 113, 57, 105, 58, 108, 126, 104, 103, 125], [109, 119, 39, 62, 117, 34, 48, 124, 45, 61, 20, 30, 26, 94, 86, 110, 92, 24, 115, 17, 112, 116, 28, 122, 103, 19, 118, 114, 98, 15, 57, 123, 78, 51, 58, 125, 42, 63, 27, 53, 91, 55, 22, 47, 49, 126, 87, 95, 83, 54, 43, 127, 11, 59, 9, 44, 113, 111, 33, 121, 14, 38, 60, 50, 56, 52, 46, 108, 16, 68, 76, 41, 29, 32, 99, 89, 40, 13, 107, 105, 71, 106, 104, 102, 73, 90, 25, 21, 96, 37, 23, 100, 120, 72, 93, 35, 36, 101, 97, 31, 82, 80, 10, 85, 18, 84, 4, 70, 88, 69, 81, 7, 67, 75, 64, 0, 77, 74, 79, 2, 12, 8, 1, 3, 66, 5, 65, 6], [48, 39, 62, 34, 119, 109, 124, 117, 24, 47, 20, 11, 15, 13, 17, 94, 70, 73, 85, 10, 9, 68, 7, 86, 53, 80, 115, 19, 61, 118, 123, 45, 28, 22, 72, 121, 25, 55, 1, 3, 110, 93, 54, 92, 29, 89, 106, 30, 31, 69, 75, 112, 87, 95, 88, 66, 50, 12, 79, 26, 27, 81, 77, 16, 71, 82, 14, 60, 91, 64, 4, 43, 122, 84, 21, 40, 114, 90, 8, 23, 96, 0, 67, 18, 116, 76, 107, 57, 46, 83, 74, 97, 2, 32, 33, 102, 6, 100, 52, 111, 108, 37, 99, 105, 63, 78, 35, 44, 56, 49, 41, 36, 38, 65, 126, 127, 101, 42, 120, 51, 104, 125, 113, 58, 59, 5, 98, 103], [48, 39, 62, 124, 119, 109, 117, 34, 47, 53, 26, 30, 49, 86, 116, 45, 42, 112, 123, 20, 115, 63, 61, 55, 50, 125, 24, 111, 92, 110, 58, 33, 43, 122, 107, 118, 80, 54, 29, 114, 44, 41, 94, 46, 93, 96, 57, 40, 106, 52, 25, 120, 56, 60, 37, 113, 127, 73, 121, 51, 108, 36, 126, 104, 35, 22, 28, 98, 100, 105, 87, 82, 17, 83, 32, 102, 38, 99, 31, 59, 101, 91, 97, 95, 19, 23, 15, 27, 103, 89, 90, 85, 16, 76, 13, 21, 71, 78, 84, 74, 81, 9, 6, 14, 79, 18, 75, 72, 88, 11, 12, 10, 5, 77, 67, 68, 4, 8, 7, 70, 3, 2, 1, 69, 0, 65, 66, 64], [40, 97, 119, 31, 81, 23, 19, 52, 76, 85, 79, 111, 112, 93, 53, 110, 7, 73, 59, 61, 57, 104, 113, 117, 5, 126, 60, 58, 27, 3, 44, 54, 107, 48, 86, 13, 43, 24, 91, 114, 1, 10, 55, 67, 28, 115, 127, 78, 123, 50, 30, 90, 71, 15, 116, 56, 109, 82, 121, 21, 83, 94, 88, 108, 101, 12, 80, 87, 74, 103, 75, 29, 72, 77, 96, 47, 42, 17, 35, 16, 39, 84, 25, 11, 37, 100, 45, 26, 33, 122, 6, 22, 118, 124, 0, 34, 92, 63, 46, 8, 70, 65, 125, 36, 68, 106, 120, 18, 98, 69, 9, 99, 4, 89, 62, 32, 102, 105, 51, 38, 64, 20, 49, 14, 41, 95, 66, 2], [40, 97, 119, 31, 23, 81, 52, 13, 111, 53, 19, 79, 58, 112, 85, 25, 110, 60, 61, 117, 55, 123, 54, 24, 76, 57, 51, 126, 84, 20, 93, 113, 59, 73, 114, 104, 121, 88, 27, 91, 33, 108, 29, 47, 82, 18, 43, 26, 7, 28, 41, 86, 90, 72, 99, 103, 44, 109, 63, 107, 45, 77, 94, 122, 75, 30, 87, 70, 34, 105, 5, 21, 100, 42, 78, 74, 22, 127, 120, 50, 35, 101, 116, 32, 69, 96, 83, 56, 48, 14, 1, 115, 8, 15, 89, 38, 46, 124, 125, 17, 80, 37, 68, 49, 98, 12, 92, 16, 11, 36, 6, 118, 39, 9, 62, 102, 106, 2, 67, 95, 71, 0, 3, 66, 10, 4, 64, 65], [40, 97, 119, 31, 81, 85, 52, 23, 19, 111, 24, 79, 76, 73, 126, 57, 112, 54, 5, 93, 58, 53, 44, 104, 110, 117, 13, 48, 7, 59, 108, 107, 60, 15, 70, 123, 61, 113, 120, 34, 55, 114, 43, 25, 29, 3, 89, 1, 50, 109, 12, 30, 28, 6, 88, 82, 83, 116, 22, 39, 78, 91, 42, 56, 87, 2, 80, 121, 74, 69, 75, 127, 26, 51, 27, 115, 21, 68, 37, 103, 32, 46, 101, 94, 45, 47, 118, 99, 66, 62, 38, 122, 100, 124, 35, 20, 102, 77, 86, 64, 14, 41, 11, 96, 90, 92, 17, 33, 106, 105, 98, 16, 9, 63, 36, 10, 65, 8, 84, 125, 0, 49, 4, 71, 72, 18, 95, 67], [40, 97, 119, 31, 26, 81, 52, 19, 85, 79, 23, 112, 126, 111, 73, 110, 13, 76, 53, 57, 25, 58, 93, 55, 104, 114, 54, 78, 5, 117, 96, 51, 60, 123, 59, 7, 113, 99, 61, 42, 20, 27, 88, 84, 82, 45, 77, 43, 1, 125, 91, 87, 48, 44, 116, 24, 107, 109, 3, 70, 80, 102, 16, 94, 15, 69, 106, 21, 2, 9, 90, 56, 127, 121, 115, 83, 75, 12, 89, 33, 47, 122, 28, 101, 120, 92, 36, 34, 22, 17, 29, 6, 74, 41, 103, 108, 49, 32, 30, 63, 62, 14, 11, 86, 100, 105, 64, 0, 10, 67, 46, 35, 68, 118, 50, 39, 98, 37, 124, 66, 72, 38, 71, 4, 65, 18, 8, 95]], "model.layers.3.self_attn.k_proj": [[104, 34, 93, 56, 62, 119, 60, 58, 80, 83, 24, 14, 86, 12, 10, 0, 26, 70, 5, 2, 67, 125, 72, 54, 116, 114, 77, 25, 85, 73, 22, 9, 6, 71, 118, 115, 18, 13, 42, 112, 96, 32, 37, 68, 95, 117, 8, 92, 29, 49, 41, 51, 65, 84, 59, 7, 113, 61, 111, 100, 43, 105, 17, 103, 45, 127, 35, 121, 108, 55, 99, 110, 57, 109, 124, 30, 89, 75, 106, 122, 48, 107, 33, 63, 88, 50, 91, 28, 21, 120, 31, 76, 87, 101, 46, 102, 44, 47, 38, 82, 1, 15, 20, 78, 53, 27, 126, 97, 79, 4, 64, 123, 39, 36, 23, 52, 69, 94, 11, 90, 74, 81, 66, 3, 16, 19, 98, 40], [47, 122, 102, 123, 120, 64, 65, 69, 33, 8, 10, 12, 71, 81, 84, 67, 14, 23, 68, 66, 0, 2, 1, 75, 5, 3, 70, 24, 86, 113, 25, 13, 6, 19, 73, 43, 72, 88, 124, 87, 9, 42, 21, 40, 118, 59, 119, 111, 41, 38, 61, 103, 105, 39, 83, 79, 117, 49, 100, 114, 27, 15, 18, 16, 116, 60, 104, 82, 125, 121, 45, 50, 110, 53, 11, 108, 52, 37, 85, 55, 4, 109, 106, 127, 48, 115, 92, 96, 98, 58, 63, 28, 30, 89, 26, 107, 90, 31, 54, 80, 36, 77, 46, 95, 99, 56, 62, 51, 57, 112, 91, 29, 34, 35, 126, 94, 44, 22, 101, 20, 93, 74, 7, 17, 78, 32, 76, 97], [105, 98, 52, 79, 18, 9, 0, 20, 22, 76, 7, 111, 69, 110, 14, 120, 10, 2, 62, 50, 63, 67, 1, 4, 126, 53, 6, 127, 41, 59, 114, 125, 24, 113, 65, 43, 11, 8, 46, 48, 89, 123, 19, 77, 117, 119, 54, 58, 81, 90, 70, 17, 47, 60, 68, 95, 28, 64, 3, 40, 122, 112, 72, 91, 71, 108, 75, 85, 61, 44, 83, 51, 121, 124, 73, 115, 32, 93, 116, 66, 30, 74, 97, 29, 45, 87, 57, 33, 94, 38, 16, 55, 99, 39, 23, 21, 100, 31, 26, 88, 109, 118, 96, 56, 27, 101, 42, 103, 49, 104, 102, 78, 106, 35, 92, 37, 107, 36, 80, 25, 86, 82, 12, 15, 5, 13, 84, 34], [39, 52, 33, 45, 125, 110, 28, 74, 119, 87, 117, 46, 78, 89, 4, 21, 12, 80, 0, 66, 11, 86, 8, 17, 71, 82, 20, 1, 7, 58, 62, 65, 79, 24, 6, 53, 67, 57, 95, 3, 90, 94, 64, 72, 127, 5, 56, 50, 54, 9, 116, 126, 63, 48, 108, 55, 123, 19, 75, 43, 109, 61, 92, 69, 68, 15, 118, 49, 73, 59, 13, 60, 106, 122, 51, 77, 88, 41, 120, 113, 124, 115, 101, 112, 107, 44, 84, 32, 42, 105, 37, 121, 96, 114, 83, 102, 104, 30, 14, 111, 10, 16, 100, 40, 38, 29, 99, 35, 34, 47, 36, 2, 98, 31, 91, 27, 81, 76, 93, 26, 18, 103, 22, 70, 97, 23, 85, 25], [44, 40, 64, 97, 108, 13, 1, 112, 16, 10, 56, 23, 46, 50, 67, 18, 70, 66, 47, 7, 68, 69, 123, 2, 104, 61, 72, 20, 52, 55, 57, 120, 51, 12, 3, 125, 116, 118, 58, 119, 117, 54, 49, 109, 4, 124, 14, 73, 11, 39, 53, 6, 43, 115, 114, 86, 75, 21, 5, 85, 45, 60, 126, 62, 78, 22, 65, 107, 41, 90, 83, 42, 81, 36, 15, 76, 27, 122, 38, 19, 127, 99, 71, 95, 25, 106, 91, 89, 111, 110, 88, 79, 121, 103, 92, 17, 63, 29, 98, 31, 24, 96, 105, 26, 93, 30, 28, 59, 113, 94, 32, 84, 102, 101, 37, 77, 35, 87, 48, 9, 34, 100, 74, 82, 0, 80, 8, 33], [104, 127, 53, 110, 95, 33, 90, 85, 122, 35, 83, 64, 88, 111, 81, 19, 115, 91, 14, 87, 102, 106, 75, 70, 76, 9, 29, 71, 66, 80, 93, 24, 18, 61, 99, 48, 45, 63, 20, 92, 126, 54, 107, 103, 125, 13, 4, 97, 105, 1, 60, 108, 72, 121, 77, 2, 59, 57, 123, 6, 3, 62, 117, 49, 114, 56, 67, 5, 44, 42, 36, 50, 43, 51, 109, 28, 58, 47, 113, 116, 37, 74, 98, 55, 112, 101, 27, 124, 89, 118, 34, 46, 120, 119, 79, 32, 52, 25, 94, 69, 78, 86, 96, 30, 10, 15, 39, 82, 100, 16, 23, 84, 0, 8, 22, 41, 26, 68, 65, 21, 73, 31, 38, 11, 17, 12, 40, 7], [103, 98, 124, 119, 62, 48, 11, 24, 17, 117, 20, 15, 13, 72, 112, 26, 0, 70, 68, 1, 30, 69, 53, 2, 3, 45, 9, 47, 111, 109, 78, 7, 54, 43, 10, 67, 19, 22, 76, 52, 86, 115, 64, 87, 49, 125, 57, 46, 126, 51, 56, 122, 66, 61, 118, 63, 127, 60, 71, 44, 120, 113, 80, 107, 55, 92, 121, 101, 106, 40, 105, 110, 38, 74, 108, 95, 73, 100, 42, 35, 58, 114, 32, 29, 104, 41, 50, 28, 116, 21, 37, 33, 123, 102, 4, 31, 93, 94, 14, 25, 82, 36, 97, 59, 96, 27, 23, 83, 85, 99, 39, 5, 8, 84, 89, 91, 34, 90, 18, 6, 16, 75, 88, 12, 77, 65, 81, 79], [104, 33, 119, 23, 19, 79, 95, 47, 85, 76, 112, 81, 55, 52, 53, 73, 113, 7, 65, 75, 59, 58, 126, 5, 46, 60, 57, 29, 61, 108, 13, 64, 49, 3, 123, 107, 54, 71, 14, 44, 117, 127, 10, 69, 80, 109, 43, 68, 114, 67, 37, 86, 25, 48, 77, 24, 28, 91, 2, 56, 11, 27, 18, 111, 4, 118, 124, 72, 30, 9, 115, 66, 40, 121, 62, 42, 16, 106, 103, 22, 38, 125, 70, 63, 74, 120, 93, 50, 39, 20, 6, 105, 35, 98, 90, 17, 101, 34, 32, 94, 96, 88, 92, 26, 110, 100, 99, 102, 89, 116, 87, 45, 51, 82, 41, 122, 36, 78, 21, 84, 0, 8, 1, 12, 15, 97, 83, 31]], "model.layers.3.self_attn.qk_proj": [[119, 104, 47, 62, 120, 52, 44, 123, 122, 53, 117, 127, 105, 40, 108, 125, 97, 110, 124, 48, 46, 56, 84, 20, 58, 60, 12, 87, 41, 109, 23, 74, 17, 76, 81, 78, 111, 112, 82, 10, 7, 14, 126, 64, 39, 15, 0, 79, 71, 103, 77, 57, 18, 13, 16, 33, 54, 114, 63, 24, 72, 88, 38, 83, 22, 86, 70, 55, 3, 19, 80, 21, 67, 9, 85, 29, 8, 45, 50, 59, 65, 5, 73, 69, 34, 89, 11, 92, 66, 90, 4, 116, 1, 25, 61, 2, 95, 113, 93, 68, 98, 75, 31, 43, 118, 26, 49, 6, 27, 115, 51, 121, 94, 28, 102, 35, 42, 30, 107, 32, 37, 106, 99, 91, 36, 96, 101, 100], [119, 104, 47, 52, 62, 120, 123, 44, 122, 53, 117, 127, 105, 40, 125, 108, 97, 110, 56, 124, 48, 46, 60, 58, 20, 84, 41, 76, 87, 109, 23, 17, 12, 74, 81, 10, 0, 14, 111, 112, 78, 64, 82, 15, 7, 103, 71, 70, 18, 86, 33, 39, 114, 63, 22, 19, 24, 80, 72, 77, 79, 57, 38, 83, 88, 13, 54, 3, 1, 65, 59, 16, 50, 8, 5, 21, 126, 69, 85, 29, 55, 92, 73, 9, 67, 2, 90, 66, 4, 68, 31, 45, 34, 11, 113, 98, 25, 116, 61, 95, 43, 89, 93, 75, 26, 51, 27, 118, 49, 6, 115, 102, 28, 94, 107, 121, 35, 30, 37, 42, 32, 96, 91, 99, 106, 101, 36, 100], [119, 104, 47, 62, 52, 120, 44, 123, 122, 53, 117, 127, 40, 105, 125, 108, 97, 56, 110, 124, 48, 58, 46, 20, 60, 84, 87, 23, 41, 12, 76, 109, 17, 74, 81, 64, 111, 82, 78, 10, 0, 14, 7, 112, 39, 18, 33, 71, 15, 38, 79, 86, 70, 67, 13, 3, 22, 24, 103, 57, 77, 88, 83, 5, 80, 16, 21, 19, 54, 126, 72, 69, 8, 1, 59, 65, 9, 55, 73, 114, 66, 4, 63, 50, 34, 29, 85, 2, 68, 92, 11, 45, 113, 93, 25, 89, 98, 90, 31, 116, 95, 61, 43, 75, 6, 26, 28, 49, 27, 51, 94, 115, 102, 118, 107, 121, 35, 42, 32, 30, 37, 91, 99, 106, 36, 96, 100, 101], [119, 104, 47, 62, 52, 120, 123, 122, 53, 44, 127, 117, 105, 40, 125, 108, 97, 48, 110, 124, 56, 46, 58, 60, 84, 20, 87, 41, 12, 23, 74, 112, 17, 109, 76, 81, 111, 82, 0, 78, 10, 14, 18, 64, 79, 86, 7, 38, 71, 57, 39, 67, 24, 15, 103, 33, 19, 54, 13, 5, 8, 22, 65, 80, 114, 70, 55, 63, 88, 3, 83, 69, 126, 72, 1, 16, 9, 29, 50, 77, 85, 59, 45, 4, 92, 66, 31, 73, 2, 93, 68, 116, 90, 21, 113, 98, 89, 25, 6, 95, 34, 61, 75, 11, 43, 26, 27, 115, 118, 51, 102, 49, 28, 94, 107, 30, 121, 42, 32, 106, 91, 37, 35, 99, 96, 101, 100, 36], [119, 104, 47, 52, 62, 120, 123, 53, 44, 122, 127, 117, 40, 105, 125, 108, 97, 48, 124, 110, 56, 60, 46, 58, 84, 12, 20, 87, 17, 76, 41, 112, 10, 23, 81, 78, 74, 109, 111, 0, 7, 18, 64, 71, 79, 14, 82, 24, 86, 33, 8, 77, 5, 19, 63, 22, 114, 39, 103, 57, 126, 67, 69, 72, 15, 38, 13, 9, 54, 59, 16, 88, 55, 80, 68, 3, 1, 65, 83, 6, 50, 29, 113, 21, 85, 73, 4, 45, 66, 92, 70, 90, 2, 93, 98, 89, 11, 95, 116, 75, 31, 61, 43, 34, 25, 26, 94, 115, 27, 51, 118, 49, 121, 28, 102, 107, 42, 32, 106, 30, 35, 91, 37, 96, 36, 100, 99, 101], [119, 104, 47, 52, 62, 120, 123, 44, 53, 122, 127, 117, 105, 40, 125, 108, 97, 124, 110, 56, 46, 48, 84, 87, 58, 20, 60, 12, 76, 41, 17, 74, 23, 78, 109, 81, 112, 10, 111, 7, 0, 64, 14, 71, 18, 79, 82, 24, 86, 19, 39, 69, 15, 13, 54, 8, 114, 77, 67, 3, 9, 57, 63, 83, 103, 88, 5, 6, 33, 126, 55, 16, 38, 72, 22, 80, 85, 59, 65, 29, 21, 50, 73, 2, 66, 113, 68, 1, 116, 45, 11, 4, 92, 95, 90, 61, 31, 75, 89, 98, 34, 93, 25, 70, 43, 115, 26, 118, 27, 51, 49, 42, 94, 102, 121, 28, 107, 30, 37, 91, 106, 32, 99, 35, 101, 96, 36, 100], [119, 104, 47, 52, 120, 62, 123, 122, 44, 53, 105, 117, 40, 127, 108, 125, 97, 110, 56, 124, 58, 87, 84, 46, 41, 20, 60, 48, 76, 12, 74, 23, 17, 81, 112, 109, 82, 71, 14, 10, 78, 103, 18, 79, 39, 7, 64, 0, 111, 15, 6, 22, 13, 16, 24, 54, 19, 77, 88, 86, 126, 8, 69, 57, 85, 59, 83, 33, 5, 63, 9, 3, 72, 38, 21, 80, 67, 29, 61, 113, 114, 1, 50, 73, 55, 66, 34, 11, 68, 65, 116, 75, 2, 25, 93, 95, 90, 4, 92, 89, 45, 98, 31, 43, 115, 27, 118, 26, 94, 51, 49, 70, 121, 42, 102, 30, 107, 35, 28, 99, 32, 37, 91, 106, 101, 36, 96, 100], [119, 104, 52, 47, 120, 62, 123, 44, 122, 53, 117, 105, 127, 40, 125, 108, 97, 110, 56, 124, 58, 48, 87, 84, 20, 46, 41, 76, 12, 60, 17, 23, 81, 78, 74, 10, 109, 14, 79, 18, 0, 6, 111, 112, 7, 82, 64, 13, 15, 19, 86, 71, 39, 16, 103, 22, 77, 80, 88, 33, 9, 24, 8, 38, 54, 83, 63, 5, 57, 85, 21, 126, 73, 72, 59, 65, 3, 69, 55, 114, 1, 2, 29, 66, 50, 61, 34, 45, 93, 75, 68, 67, 90, 89, 95, 4, 11, 92, 98, 113, 25, 116, 31, 26, 118, 49, 70, 51, 115, 28, 27, 43, 42, 121, 94, 35, 102, 107, 32, 37, 30, 101, 91, 106, 36, 99, 96, 100], [119, 104, 52, 47, 62, 120, 123, 44, 122, 53, 127, 117, 105, 40, 125, 108, 97, 124, 110, 56, 58, 48, 46, 20, 60, 84, 87, 41, 76, 12, 81, 23, 17, 109, 78, 111, 10, 74, 14, 112, 79, 64, 7, 39, 82, 18, 0, 71, 63, 86, 13, 16, 15, 103, 19, 33, 6, 8, 24, 80, 67, 114, 22, 83, 38, 77, 88, 21, 3, 5, 50, 126, 1, 57, 59, 54, 72, 55, 29, 9, 85, 69, 65, 45, 2, 4, 89, 66, 116, 31, 73, 98, 34, 90, 92, 68, 25, 11, 75, 113, 93, 95, 61, 26, 70, 43, 49, 28, 27, 115, 121, 51, 118, 94, 102, 107, 35, 42, 30, 37, 101, 99, 32, 91, 106, 96, 100, 36], [119, 104, 47, 52, 62, 120, 123, 44, 122, 53, 105, 117, 40, 127, 108, 125, 97, 124, 56, 110, 58, 46, 48, 84, 87, 20, 12, 41, 23, 60, 81, 17, 109, 74, 76, 18, 10, 14, 78, 82, 15, 33, 111, 112, 7, 79, 86, 64, 22, 0, 103, 39, 80, 19, 83, 24, 71, 38, 13, 16, 77, 88, 57, 21, 63, 59, 55, 85, 8, 54, 29, 5, 114, 45, 69, 6, 9, 72, 3, 73, 92, 126, 67, 50, 2, 113, 66, 98, 34, 93, 89, 116, 1, 65, 61, 90, 68, 43, 95, 4, 75, 25, 26, 31, 70, 11, 49, 27, 115, 118, 28, 30, 102, 51, 94, 35, 42, 121, 107, 32, 37, 99, 106, 96, 91, 101, 36, 100], [119, 104, 47, 62, 120, 52, 123, 122, 44, 53, 127, 40, 117, 105, 125, 108, 97, 48, 124, 56, 110, 46, 60, 20, 84, 41, 58, 12, 87, 76, 17, 111, 109, 10, 23, 112, 81, 0, 14, 18, 74, 78, 64, 7, 71, 39, 79, 15, 103, 82, 63, 33, 1, 8, 24, 19, 38, 86, 80, 69, 77, 13, 57, 55, 114, 50, 85, 22, 83, 3, 59, 88, 72, 16, 54, 5, 73, 29, 65, 126, 4, 21, 45, 67, 70, 116, 66, 31, 2, 9, 95, 90, 93, 68, 75, 61, 26, 92, 98, 113, 6, 34, 25, 89, 11, 118, 49, 94, 43, 115, 51, 27, 102, 28, 121, 107, 42, 91, 106, 30, 35, 37, 32, 99, 100, 101, 96, 36], [119, 104, 47, 52, 62, 120, 123, 53, 44, 122, 127, 40, 117, 125, 105, 108, 97, 124, 56, 110, 48, 46, 20, 60, 58, 84, 87, 41, 76, 112, 12, 81, 17, 14, 23, 10, 109, 111, 74, 7, 0, 3, 39, 78, 82, 64, 71, 24, 67, 103, 18, 79, 86, 114, 15, 88, 70, 33, 5, 57, 69, 8, 38, 72, 54, 22, 19, 63, 77, 80, 1, 13, 126, 55, 83, 16, 65, 85, 59, 21, 50, 29, 73, 9, 61, 68, 66, 2, 90, 92, 4, 116, 75, 95, 34, 45, 113, 11, 25, 98, 89, 93, 31, 26, 6, 27, 118, 51, 94, 115, 43, 49, 102, 28, 121, 42, 30, 107, 32, 35, 37, 106, 99, 91, 100, 36, 101, 96], [119, 104, 52, 47, 62, 120, 123, 44, 122, 117, 53, 105, 40, 127, 108, 125, 97, 56, 110, 124, 58, 20, 84, 87, 48, 41, 46, 76, 23, 60, 12, 17, 81, 10, 14, 74, 109, 78, 71, 82, 18, 15, 79, 112, 7, 80, 86, 33, 83, 70, 39, 103, 19, 24, 22, 64, 13, 77, 111, 38, 0, 88, 63, 5, 21, 16, 57, 69, 72, 85, 8, 9, 54, 55, 29, 67, 126, 92, 116, 93, 50, 59, 113, 3, 1, 2, 114, 25, 61, 73, 66, 75, 34, 89, 4, 65, 11, 45, 90, 95, 68, 98, 31, 27, 118, 26, 43, 49, 94, 28, 115, 51, 30, 107, 102, 6, 35, 42, 32, 121, 96, 37, 91, 99, 106, 101, 100, 36], [119, 104, 47, 52, 62, 120, 123, 44, 122, 53, 127, 117, 105, 40, 125, 108, 97, 110, 124, 56, 20, 48, 58, 46, 84, 60, 23, 41, 76, 17, 12, 109, 87, 74, 111, 81, 10, 14, 71, 112, 64, 82, 15, 18, 78, 103, 7, 70, 0, 39, 79, 86, 22, 13, 33, 19, 69, 24, 57, 83, 16, 38, 80, 114, 72, 54, 59, 8, 63, 77, 126, 88, 1, 73, 5, 85, 50, 55, 66, 29, 9, 21, 67, 2, 3, 45, 25, 65, 68, 90, 92, 113, 4, 34, 116, 11, 61, 93, 95, 98, 75, 31, 89, 43, 49, 26, 115, 51, 118, 6, 27, 28, 94, 121, 102, 30, 107, 42, 32, 99, 91, 37, 35, 106, 36, 101, 100, 96], [119, 104, 47, 62, 120, 52, 123, 44, 122, 53, 117, 127, 105, 40, 125, 108, 97, 110, 124, 58, 56, 48, 20, 84, 60, 87, 41, 46, 23, 12, 76, 81, 111, 14, 74, 17, 71, 10, 112, 0, 109, 64, 7, 79, 39, 18, 67, 72, 103, 78, 15, 69, 3, 86, 82, 126, 1, 33, 24, 13, 55, 5, 38, 70, 22, 80, 54, 19, 16, 57, 83, 114, 88, 9, 73, 85, 77, 68, 113, 8, 63, 65, 59, 2, 21, 29, 50, 66, 90, 31, 98, 116, 6, 4, 75, 92, 61, 93, 25, 45, 89, 34, 11, 95, 26, 49, 118, 51, 27, 115, 28, 102, 43, 42, 94, 121, 106, 35, 30, 99, 107, 32, 91, 101, 37, 96, 36, 100], [119, 104, 47, 62, 52, 120, 123, 122, 44, 53, 127, 117, 105, 40, 125, 108, 97, 48, 110, 56, 124, 46, 20, 58, 84, 41, 60, 87, 76, 12, 17, 81, 23, 111, 112, 109, 71, 10, 74, 14, 78, 39, 7, 15, 0, 82, 72, 18, 103, 86, 64, 63, 24, 33, 57, 13, 55, 38, 79, 69, 114, 67, 83, 126, 22, 88, 3, 21, 80, 16, 59, 50, 29, 54, 19, 77, 9, 5, 85, 65, 73, 8, 1, 116, 92, 90, 113, 70, 68, 4, 45, 6, 98, 95, 93, 2, 31, 61, 66, 26, 11, 34, 43, 75, 25, 89, 118, 51, 49, 115, 94, 102, 27, 121, 28, 42, 30, 106, 107, 35, 91, 32, 99, 37, 100, 101, 96, 36], [119, 104, 52, 47, 120, 62, 123, 44, 122, 53, 117, 105, 127, 40, 108, 125, 97, 110, 124, 56, 20, 48, 46, 58, 84, 87, 23, 41, 76, 12, 60, 10, 17, 81, 74, 78, 18, 109, 14, 7, 111, 71, 15, 112, 82, 79, 22, 80, 13, 86, 64, 69, 19, 83, 0, 57, 33, 39, 24, 77, 9, 38, 103, 88, 6, 5, 21, 72, 29, 63, 85, 54, 55, 16, 73, 8, 126, 50, 3, 114, 75, 92, 67, 59, 68, 65, 66, 2, 89, 1, 113, 116, 95, 4, 34, 93, 45, 11, 25, 90, 98, 61, 115, 31, 70, 26, 49, 27, 43, 51, 28, 118, 102, 94, 30, 42, 121, 107, 37, 91, 35, 32, 99, 106, 100, 96, 101, 36], [119, 104, 52, 47, 62, 120, 123, 44, 122, 53, 117, 105, 127, 40, 108, 125, 97, 56, 124, 110, 20, 58, 84, 48, 41, 23, 76, 60, 87, 46, 17, 12, 10, 109, 74, 81, 82, 14, 78, 112, 6, 64, 15, 0, 71, 7, 111, 18, 22, 79, 69, 39, 16, 57, 86, 33, 19, 24, 103, 3, 13, 83, 77, 80, 38, 72, 67, 1, 114, 59, 63, 54, 21, 55, 88, 8, 73, 85, 29, 9, 2, 126, 93, 66, 113, 45, 4, 116, 75, 65, 50, 5, 92, 95, 98, 90, 61, 25, 31, 68, 34, 89, 11, 26, 70, 28, 43, 51, 49, 27, 118, 115, 102, 30, 42, 94, 37, 121, 107, 32, 91, 35, 106, 99, 101, 96, 36, 100], [119, 104, 47, 52, 62, 120, 123, 122, 53, 44, 127, 117, 105, 40, 125, 97, 108, 110, 124, 56, 48, 58, 60, 46, 20, 84, 76, 87, 23, 41, 12, 81, 17, 10, 74, 14, 112, 78, 111, 109, 64, 82, 71, 7, 15, 79, 39, 18, 0, 6, 86, 33, 103, 57, 126, 38, 16, 114, 69, 24, 1, 77, 19, 8, 67, 22, 13, 83, 80, 88, 54, 85, 63, 72, 55, 73, 59, 3, 29, 5, 9, 116, 50, 2, 75, 93, 4, 21, 66, 61, 92, 65, 98, 34, 90, 45, 68, 31, 95, 113, 89, 25, 11, 26, 70, 49, 43, 51, 118, 115, 27, 94, 28, 102, 42, 107, 121, 32, 30, 35, 37, 91, 99, 106, 101, 36, 96, 100], [119, 104, 47, 62, 52, 120, 122, 123, 44, 53, 117, 127, 105, 40, 125, 108, 97, 110, 56, 124, 58, 60, 46, 48, 84, 87, 20, 41, 12, 76, 17, 74, 81, 10, 109, 111, 23, 14, 112, 71, 7, 78, 79, 18, 82, 86, 69, 64, 15, 19, 33, 0, 24, 57, 39, 6, 13, 83, 22, 103, 63, 77, 38, 73, 80, 114, 72, 1, 88, 8, 55, 16, 67, 9, 5, 126, 54, 29, 21, 2, 3, 50, 85, 116, 59, 113, 45, 92, 68, 75, 4, 90, 93, 98, 31, 34, 65, 95, 66, 61, 89, 43, 26, 70, 11, 25, 49, 51, 102, 28, 115, 118, 27, 94, 42, 30, 107, 121, 35, 37, 32, 91, 101, 99, 96, 106, 100, 36], [119, 104, 62, 52, 47, 120, 123, 53, 122, 44, 127, 117, 40, 105, 125, 108, 97, 110, 48, 124, 56, 46, 20, 84, 60, 58, 87, 41, 109, 17, 12, 23, 112, 76, 111, 81, 74, 10, 82, 78, 14, 71, 39, 7, 0, 79, 57, 18, 38, 64, 24, 3, 22, 33, 15, 16, 126, 67, 54, 19, 55, 80, 114, 77, 86, 63, 59, 72, 9, 88, 69, 103, 83, 13, 8, 29, 5, 50, 1, 85, 21, 116, 4, 95, 68, 89, 92, 113, 90, 6, 45, 73, 70, 65, 98, 31, 66, 93, 75, 2, 11, 61, 34, 25, 26, 51, 115, 49, 94, 118, 42, 43, 121, 28, 27, 107, 30, 102, 37, 35, 91, 99, 106, 100, 32, 101, 36, 96], [119, 104, 47, 52, 120, 62, 123, 122, 53, 44, 117, 127, 105, 40, 125, 108, 97, 110, 58, 124, 56, 84, 46, 87, 48, 20, 60, 41, 12, 17, 23, 76, 112, 111, 81, 109, 10, 0, 64, 74, 7, 14, 18, 39, 71, 57, 79, 33, 82, 15, 78, 86, 24, 54, 70, 22, 126, 38, 77, 103, 69, 63, 1, 80, 8, 16, 114, 5, 55, 67, 2, 19, 9, 72, 83, 29, 50, 68, 3, 88, 13, 66, 73, 85, 21, 116, 4, 34, 65, 59, 31, 113, 93, 45, 61, 95, 89, 90, 92, 98, 11, 26, 75, 6, 43, 25, 118, 115, 49, 27, 51, 102, 28, 42, 94, 107, 121, 30, 35, 32, 91, 99, 106, 101, 37, 96, 36, 100], [119, 104, 52, 47, 62, 120, 123, 122, 44, 53, 117, 127, 40, 105, 125, 97, 108, 110, 48, 124, 56, 46, 84, 58, 60, 20, 87, 23, 41, 12, 76, 17, 74, 81, 109, 14, 10, 111, 112, 18, 7, 78, 71, 82, 79, 22, 15, 33, 86, 70, 0, 64, 24, 39, 19, 8, 38, 83, 57, 16, 103, 29, 88, 13, 5, 63, 80, 85, 77, 9, 59, 67, 54, 69, 114, 72, 55, 21, 65, 126, 50, 68, 116, 73, 4, 1, 113, 61, 3, 45, 92, 89, 98, 2, 93, 34, 31, 90, 75, 11, 66, 95, 25, 26, 118, 6, 43, 27, 94, 28, 49, 51, 102, 115, 42, 107, 30, 121, 32, 91, 35, 106, 96, 37, 99, 101, 36, 100], [119, 104, 47, 52, 62, 120, 123, 53, 44, 122, 127, 117, 105, 40, 125, 97, 108, 110, 48, 124, 56, 46, 20, 84, 58, 60, 87, 41, 12, 76, 74, 23, 17, 112, 81, 111, 78, 109, 10, 82, 18, 14, 15, 7, 86, 39, 71, 103, 57, 0, 24, 8, 70, 16, 19, 126, 64, 38, 67, 33, 13, 22, 83, 79, 63, 77, 114, 88, 55, 80, 50, 3, 59, 9, 5, 92, 54, 116, 29, 85, 1, 73, 113, 69, 21, 34, 68, 72, 89, 45, 98, 61, 95, 4, 65, 2, 11, 90, 31, 75, 66, 93, 43, 25, 26, 51, 115, 118, 49, 27, 94, 6, 28, 102, 42, 107, 30, 121, 37, 106, 91, 32, 35, 99, 101, 36, 100, 96], [119, 104, 47, 52, 62, 120, 123, 122, 44, 53, 117, 127, 105, 40, 125, 108, 97, 110, 56, 124, 58, 48, 60, 20, 84, 46, 87, 41, 12, 76, 23, 17, 109, 81, 10, 78, 74, 112, 7, 111, 82, 39, 15, 14, 57, 18, 79, 126, 71, 64, 0, 103, 33, 86, 24, 88, 16, 54, 83, 22, 5, 8, 13, 77, 85, 80, 38, 69, 19, 70, 114, 59, 21, 72, 29, 63, 3, 1, 67, 61, 9, 55, 92, 50, 73, 95, 31, 93, 90, 4, 2, 66, 65, 25, 113, 34, 45, 75, 11, 89, 116, 98, 68, 118, 26, 6, 51, 49, 43, 27, 94, 115, 42, 102, 28, 30, 35, 121, 32, 107, 37, 106, 99, 91, 96, 36, 101, 100], [119, 104, 52, 47, 62, 120, 123, 44, 122, 53, 127, 117, 40, 105, 125, 108, 97, 56, 110, 48, 124, 20, 46, 58, 87, 60, 84, 41, 12, 76, 23, 81, 10, 112, 17, 74, 109, 111, 7, 39, 14, 18, 0, 78, 82, 64, 15, 71, 103, 79, 57, 33, 19, 22, 83, 16, 13, 86, 126, 77, 38, 69, 80, 8, 88, 3, 24, 5, 50, 67, 54, 116, 85, 70, 72, 9, 63, 1, 114, 21, 55, 59, 92, 73, 2, 29, 34, 65, 25, 68, 61, 113, 89, 45, 4, 66, 90, 95, 31, 6, 11, 98, 118, 93, 51, 75, 28, 26, 115, 27, 49, 43, 30, 107, 94, 42, 102, 121, 106, 37, 35, 91, 32, 36, 101, 100, 99, 96], [119, 104, 47, 52, 62, 120, 123, 44, 122, 53, 117, 127, 105, 40, 108, 125, 97, 56, 110, 124, 20, 58, 48, 84, 41, 46, 60, 23, 12, 87, 109, 76, 74, 17, 81, 111, 10, 112, 18, 78, 14, 7, 33, 71, 0, 15, 39, 64, 83, 82, 57, 79, 77, 86, 38, 22, 103, 88, 16, 19, 24, 3, 80, 126, 67, 59, 8, 72, 54, 13, 55, 21, 63, 65, 85, 50, 69, 45, 5, 29, 92, 68, 6, 73, 1, 25, 89, 9, 114, 34, 2, 95, 113, 4, 93, 98, 11, 90, 66, 116, 75, 49, 70, 31, 61, 26, 27, 121, 43, 115, 118, 102, 51, 28, 94, 35, 42, 107, 30, 32, 91, 37, 106, 100, 99, 96, 36, 101], [119, 104, 47, 52, 120, 62, 123, 44, 122, 53, 117, 105, 127, 40, 125, 108, 97, 110, 124, 56, 48, 58, 84, 46, 76, 20, 87, 60, 41, 12, 74, 81, 23, 10, 17, 112, 0, 7, 71, 109, 14, 64, 78, 111, 18, 15, 57, 82, 79, 77, 126, 6, 39, 72, 69, 103, 86, 33, 8, 114, 59, 88, 1, 38, 80, 5, 16, 83, 13, 73, 22, 24, 63, 54, 67, 19, 3, 55, 9, 2, 85, 50, 21, 29, 65, 68, 66, 113, 93, 92, 34, 45, 75, 4, 98, 116, 31, 90, 61, 11, 95, 49, 25, 26, 89, 115, 27, 43, 118, 51, 70, 102, 94, 42, 28, 37, 121, 30, 107, 35, 99, 91, 32, 100, 106, 96, 36, 101], [119, 104, 47, 120, 62, 52, 123, 53, 122, 44, 127, 117, 40, 105, 125, 108, 97, 110, 124, 56, 20, 48, 60, 58, 46, 87, 84, 41, 76, 12, 74, 17, 23, 112, 111, 109, 10, 81, 78, 14, 7, 39, 71, 18, 82, 0, 79, 33, 86, 64, 114, 15, 103, 24, 77, 6, 83, 38, 8, 1, 72, 19, 22, 13, 57, 3, 126, 16, 54, 88, 55, 59, 5, 9, 63, 80, 50, 73, 67, 21, 69, 29, 65, 92, 116, 85, 113, 31, 95, 93, 68, 98, 4, 90, 66, 2, 45, 11, 34, 61, 75, 43, 25, 89, 118, 26, 115, 49, 70, 102, 27, 51, 121, 94, 28, 107, 30, 42, 37, 32, 35, 91, 106, 99, 100, 36, 101, 96], [119, 104, 47, 62, 52, 120, 123, 44, 122, 53, 127, 40, 105, 117, 125, 97, 108, 124, 48, 56, 46, 110, 20, 84, 60, 12, 58, 41, 87, 76, 23, 17, 74, 111, 112, 81, 109, 7, 10, 14, 78, 0, 39, 82, 71, 18, 15, 64, 79, 67, 6, 77, 103, 83, 72, 5, 22, 57, 33, 54, 59, 38, 126, 24, 86, 88, 13, 69, 16, 3, 63, 19, 8, 114, 29, 80, 9, 73, 55, 85, 65, 50, 68, 21, 1, 116, 2, 92, 11, 113, 4, 45, 89, 95, 31, 66, 61, 90, 34, 98, 43, 93, 25, 70, 118, 27, 75, 26, 115, 121, 51, 49, 102, 30, 94, 42, 91, 107, 28, 106, 37, 32, 35, 99, 101, 96, 100, 36], [119, 104, 47, 52, 120, 62, 123, 44, 122, 53, 117, 105, 40, 127, 108, 125, 97, 124, 56, 110, 58, 20, 87, 84, 46, 41, 76, 60, 23, 48, 12, 74, 17, 10, 81, 111, 0, 78, 14, 109, 18, 64, 82, 79, 71, 86, 7, 112, 103, 15, 33, 39, 83, 5, 22, 57, 72, 77, 38, 19, 13, 59, 24, 54, 16, 80, 6, 88, 9, 55, 126, 21, 69, 8, 3, 63, 73, 65, 29, 2, 85, 34, 4, 50, 113, 67, 114, 92, 66, 1, 89, 45, 75, 70, 95, 25, 68, 11, 116, 61, 98, 93, 31, 90, 43, 26, 115, 118, 27, 49, 102, 28, 51, 30, 94, 121, 35, 32, 42, 99, 37, 107, 91, 106, 96, 101, 36, 100], [119, 104, 47, 52, 120, 62, 123, 44, 122, 53, 127, 40, 105, 117, 125, 108, 97, 56, 46, 110, 124, 48, 60, 20, 87, 41, 84, 58, 76, 12, 17, 109, 23, 81, 10, 74, 111, 112, 14, 78, 82, 103, 7, 71, 64, 15, 18, 39, 79, 72, 0, 13, 22, 80, 33, 63, 86, 83, 5, 38, 24, 126, 3, 57, 88, 114, 16, 55, 77, 67, 19, 59, 85, 116, 69, 9, 8, 54, 29, 65, 21, 2, 50, 73, 1, 34, 70, 113, 66, 4, 89, 92, 98, 6, 61, 25, 45, 95, 11, 31, 90, 75, 68, 93, 118, 115, 43, 26, 51, 27, 49, 28, 102, 121, 42, 30, 94, 107, 32, 37, 35, 106, 101, 36, 99, 91, 100, 96]], "model.layers.4.self_attn.q_proj": [[42, 122, 117, 56, 53, 123, 101, 124, 102, 121, 33, 61, 27, 107, 104, 119, 59, 48, 110, 39, 116, 83, 55, 109, 127, 115, 37, 114, 113, 126, 92, 57, 60, 43, 58, 17, 50, 26, 38, 120, 47, 125, 44, 45, 51, 54, 89, 62, 105, 111, 94, 21, 52, 46, 87, 41, 36, 108, 32, 106, 91, 49, 63, 112, 86, 118, 40, 23, 31, 30, 76, 79, 99, 22, 34, 103, 84, 96, 100, 72, 97, 98, 28, 78, 82, 25, 24, 95, 93, 16, 75, 0, 29, 35, 15, 13, 81, 10, 66, 6, 5, 73, 2, 11, 14, 3, 85, 19, 64, 8, 12, 74, 4, 68, 90, 69, 20, 65, 70, 88, 67, 7, 80, 71, 77, 9, 1, 18], [42, 101, 117, 114, 123, 56, 122, 102, 33, 53, 48, 124, 107, 59, 111, 39, 89, 94, 91, 27, 86, 121, 54, 57, 21, 37, 104, 28, 125, 112, 49, 79, 116, 40, 109, 36, 113, 43, 44, 41, 60, 118, 99, 26, 47, 38, 84, 82, 58, 46, 87, 25, 32, 17, 115, 52, 110, 61, 95, 92, 108, 62, 119, 24, 127, 45, 50, 120, 55, 51, 85, 29, 63, 90, 126, 30, 15, 93, 105, 106, 81, 34, 83, 22, 12, 96, 76, 16, 31, 98, 78, 75, 19, 35, 23, 74, 97, 13, 88, 8, 73, 20, 14, 100, 103, 77, 68, 72, 71, 6, 7, 69, 5, 11, 9, 66, 10, 4, 1, 65, 70, 3, 64, 67, 2, 0, 18, 80], [101, 42, 117, 33, 123, 122, 56, 53, 114, 87, 84, 82, 124, 91, 94, 39, 61, 79, 78, 75, 16, 73, 13, 6, 48, 86, 27, 72, 46, 111, 107, 2, 57, 89, 32, 95, 30, 21, 68, 22, 104, 20, 19, 74, 54, 71, 58, 102, 23, 109, 67, 44, 14, 85, 25, 8, 11, 88, 63, 55, 90, 10, 24, 106, 65, 41, 110, 36, 98, 64, 120, 35, 121, 92, 105, 7, 81, 99, 17, 49, 113, 28, 15, 60, 3, 77, 118, 108, 40, 116, 93, 76, 80, 18, 29, 52, 12, 38, 83, 26, 96, 4, 69, 119, 34, 70, 43, 31, 1, 9, 62, 47, 103, 115, 125, 51, 112, 127, 126, 59, 50, 97, 37, 5, 100, 0, 45, 66], [101, 117, 56, 122, 123, 33, 53, 42, 87, 82, 91, 84, 124, 54, 73, 15, 16, 103, 76, 78, 75, 24, 71, 27, 121, 13, 88, 68, 6, 37, 72, 114, 107, 86, 85, 104, 58, 59, 67, 66, 10, 3, 48, 65, 1, 126, 116, 25, 105, 95, 12, 109, 61, 118, 44, 89, 64, 45, 14, 92, 7, 120, 79, 115, 9, 127, 18, 29, 80, 43, 26, 98, 90, 77, 4, 94, 102, 70, 49, 74, 93, 0, 110, 11, 28, 23, 55, 19, 8, 81, 34, 99, 35, 20, 36, 100, 41, 125, 111, 52, 22, 32, 31, 17, 113, 46, 47, 83, 39, 51, 96, 21, 69, 5, 2, 112, 30, 62, 50, 63, 40, 60, 57, 38, 106, 119, 108, 97], [39, 52, 118, 33, 31, 79, 13, 87, 8, 11, 53, 70, 20, 117, 54, 82, 81, 9, 68, 63, 50, 66, 113, 26, 120, 121, 111, 125, 90, 10, 61, 114, 49, 119, 46, 23, 74, 3, 51, 112, 25, 47, 116, 84, 55, 21, 6, 29, 95, 38, 67, 69, 12, 73, 18, 5, 19, 60, 77, 17, 107, 0, 28, 93, 85, 97, 57, 44, 106, 62, 15, 59, 22, 100, 72, 2, 24, 83, 92, 91, 7, 94, 86, 80, 98, 36, 14, 115, 71, 102, 78, 75, 105, 34, 1, 16, 123, 65, 58, 110, 30, 42, 76, 89, 43, 32, 4, 27, 45, 124, 104, 40, 88, 48, 96, 56, 99, 41, 109, 127, 37, 101, 64, 108, 35, 122, 126, 103], [39, 118, 52, 33, 31, 102, 20, 87, 63, 25, 116, 11, 54, 47, 82, 81, 90, 117, 13, 85, 79, 49, 80, 123, 111, 121, 56, 86, 76, 104, 26, 50, 28, 53, 23, 91, 9, 7, 105, 120, 41, 112, 51, 71, 61, 44, 70, 83, 55, 114, 27, 57, 8, 60, 125, 107, 46, 110, 115, 106, 101, 124, 40, 97, 42, 10, 14, 59, 127, 29, 43, 78, 126, 113, 35, 45, 16, 69, 122, 74, 62, 119, 65, 22, 98, 108, 58, 36, 48, 3, 93, 17, 84, 37, 30, 12, 68, 99, 109, 88, 92, 64, 18, 67, 38, 100, 34, 66, 96, 19, 89, 21, 24, 32, 75, 72, 77, 94, 5, 95, 15, 73, 1, 6, 4, 103, 0, 2], [53, 39, 33, 52, 118, 31, 90, 50, 102, 81, 87, 113, 59, 63, 54, 125, 20, 114, 98, 61, 55, 97, 60, 117, 108, 47, 107, 26, 124, 121, 92, 49, 24, 44, 86, 112, 42, 79, 126, 110, 28, 116, 103, 57, 111, 82, 14, 43, 83, 120, 45, 119, 10, 127, 46, 29, 122, 85, 106, 19, 12, 51, 56, 41, 37, 32, 115, 96, 48, 62, 109, 22, 100, 105, 34, 88, 13, 40, 89, 58, 123, 101, 11, 94, 91, 25, 104, 93, 36, 5, 78, 27, 38, 99, 35, 17, 71, 21, 80, 23, 76, 74, 84, 30, 16, 95, 77, 75, 8, 18, 15, 72, 67, 6, 9, 7, 73, 69, 3, 4, 65, 1, 66, 70, 0, 68, 64, 2], [39, 118, 52, 33, 53, 31, 8, 20, 82, 87, 13, 79, 11, 54, 70, 63, 117, 102, 81, 74, 9, 50, 125, 68, 121, 46, 90, 86, 23, 61, 16, 49, 66, 10, 55, 59, 60, 15, 47, 42, 51, 69, 115, 67, 95, 12, 26, 111, 119, 97, 22, 72, 112, 114, 77, 120, 17, 80, 124, 21, 18, 94, 89, 116, 3, 25, 2, 45, 56, 92, 75, 84, 78, 85, 28, 0, 19, 14, 38, 29, 57, 27, 83, 30, 71, 24, 113, 96, 107, 76, 40, 7, 123, 99, 62, 122, 88, 44, 48, 36, 6, 100, 41, 5, 73, 98, 106, 43, 34, 127, 91, 64, 1, 93, 32, 105, 37, 126, 4, 108, 110, 101, 104, 35, 58, 109, 65, 103], [41, 39, 99, 116, 124, 118, 52, 95, 54, 28, 86, 92, 62, 75, 105, 81, 26, 31, 24, 21, 122, 121, 58, 17, 83, 55, 87, 127, 22, 61, 79, 56, 123, 117, 93, 50, 12, 53, 14, 115, 35, 88, 110, 106, 13, 82, 112, 113, 94, 47, 18, 34, 125, 57, 42, 120, 101, 108, 30, 48, 20, 126, 98, 32, 111, 46, 89, 43, 51, 78, 114, 96, 49, 104, 90, 15, 72, 40, 25, 109, 9, 84, 59, 38, 37, 107, 100, 63, 60, 44, 45, 33, 23, 16, 11, 119, 29, 85, 102, 36, 97, 27, 91, 74, 19, 8, 65, 68, 1, 69, 73, 76, 5, 4, 70, 77, 71, 6, 3, 10, 80, 7, 66, 64, 2, 0, 67, 103], [39, 99, 28, 116, 124, 118, 16, 52, 41, 83, 74, 13, 86, 95, 72, 54, 70, 49, 75, 71, 79, 92, 4, 26, 22, 3, 21, 35, 24, 12, 81, 1, 62, 2, 122, 23, 61, 31, 18, 55, 32, 85, 123, 87, 30, 50, 11, 73, 77, 127, 58, 89, 7, 112, 19, 48, 27, 117, 78, 126, 64, 111, 25, 88, 80, 42, 47, 66, 82, 53, 14, 121, 8, 59, 67, 100, 10, 101, 51, 5, 17, 6, 106, 94, 110, 113, 76, 0, 15, 90, 105, 34, 65, 107, 91, 104, 60, 63, 119, 69, 20, 46, 93, 120, 102, 125, 84, 43, 33, 97, 37, 96, 29, 36, 115, 40, 108, 68, 98, 103, 56, 45, 9, 57, 109, 38, 44, 114], [39, 116, 99, 124, 118, 52, 41, 95, 92, 86, 26, 31, 28, 54, 49, 83, 98, 87, 81, 112, 24, 62, 79, 75, 105, 22, 30, 35, 61, 84, 16, 34, 93, 13, 117, 88, 47, 121, 120, 58, 106, 21, 123, 114, 44, 110, 46, 104, 45, 60, 122, 14, 42, 108, 53, 96, 89, 125, 113, 25, 33, 27, 50, 127, 90, 48, 40, 109, 59, 20, 38, 111, 63, 37, 97, 126, 17, 55, 51, 91, 43, 107, 32, 119, 102, 101, 36, 100, 12, 57, 94, 72, 29, 56, 115, 74, 23, 78, 85, 103, 82, 11, 18, 15, 19, 4, 70, 1, 68, 76, 71, 73, 9, 5, 69, 8, 6, 65, 2, 64, 80, 77, 66, 7, 0, 10, 3, 67], [39, 99, 118, 124, 52, 116, 28, 95, 72, 13, 83, 4, 79, 86, 16, 22, 70, 75, 74, 66, 54, 105, 2, 68, 65, 64, 81, 35, 84, 87, 92, 61, 1, 21, 0, 26, 24, 3, 17, 115, 106, 77, 98, 55, 11, 89, 31, 30, 8, 14, 19, 71, 67, 88, 62, 110, 7, 123, 23, 73, 58, 82, 6, 94, 42, 33, 15, 117, 10, 41, 109, 12, 5, 103, 49, 122, 121, 76, 80, 127, 25, 57, 125, 56, 51, 40, 93, 48, 53, 34, 113, 78, 85, 18, 96, 69, 47, 59, 114, 46, 63, 27, 20, 90, 45, 44, 100, 29, 32, 60, 120, 111, 126, 43, 101, 91, 36, 119, 50, 37, 97, 108, 38, 112, 9, 102, 107, 104], [45, 102, 0, 97, 64, 31, 109, 25, 4, 66, 1, 65, 6, 76, 10, 124, 79, 69, 9, 68, 22, 17, 2, 11, 48, 38, 105, 126, 95, 125, 81, 51, 82, 59, 20, 56, 52, 5, 78, 104, 90, 18, 127, 84, 67, 118, 61, 7, 41, 63, 60, 122, 53, 71, 113, 74, 70, 73, 57, 75, 100, 15, 50, 120, 42, 14, 77, 13, 72, 28, 110, 12, 47, 114, 24, 49, 87, 107, 93, 19, 88, 121, 23, 33, 26, 16, 83, 36, 3, 46, 80, 112, 55, 39, 40, 85, 92, 29, 58, 115, 8, 30, 43, 101, 86, 96, 108, 106, 34, 111, 37, 91, 35, 99, 89, 21, 98, 32, 117, 119, 27, 123, 94, 103, 54, 62, 116, 44], [102, 45, 97, 31, 124, 79, 10, 17, 11, 5, 109, 25, 76, 89, 22, 70, 9, 4, 69, 6, 84, 48, 52, 51, 66, 82, 1, 78, 115, 75, 56, 65, 59, 104, 126, 53, 19, 90, 61, 72, 2, 105, 14, 87, 125, 20, 41, 77, 81, 42, 57, 71, 122, 38, 114, 15, 86, 28, 16, 120, 127, 118, 3, 60, 73, 63, 23, 74, 7, 83, 67, 0, 18, 13, 50, 123, 92, 12, 121, 95, 99, 24, 35, 68, 111, 88, 43, 39, 80, 113, 21, 34, 106, 30, 27, 29, 46, 93, 107, 117, 85, 101, 8, 110, 100, 94, 108, 26, 47, 96, 116, 98, 91, 36, 119, 58, 37, 62, 33, 112, 49, 32, 40, 44, 54, 55, 103, 64], [45, 102, 97, 31, 65, 25, 124, 109, 6, 11, 4, 1, 10, 76, 9, 79, 17, 0, 67, 69, 3, 48, 81, 66, 84, 51, 20, 105, 5, 52, 71, 126, 61, 90, 56, 15, 122, 125, 70, 72, 78, 118, 22, 95, 59, 120, 41, 23, 53, 57, 63, 38, 8, 7, 16, 82, 60, 113, 18, 73, 12, 64, 24, 30, 74, 127, 2, 14, 75, 47, 83, 80, 85, 68, 111, 86, 28, 19, 39, 21, 46, 87, 50, 49, 13, 29, 77, 104, 121, 26, 100, 88, 91, 93, 42, 27, 110, 58, 107, 117, 94, 101, 119, 114, 40, 108, 115, 96, 36, 103, 89, 116, 123, 37, 62, 92, 44, 34, 43, 55, 106, 112, 35, 98, 99, 54, 32, 33], [124, 102, 97, 45, 31, 90, 22, 52, 86, 120, 84, 109, 89, 78, 111, 104, 41, 53, 121, 13, 122, 18, 36, 33, 38, 115, 51, 114, 59, 113, 107, 63, 25, 34, 117, 57, 79, 127, 106, 46, 42, 26, 48, 44, 62, 17, 123, 126, 61, 100, 39, 54, 96, 56, 116, 21, 60, 49, 108, 103, 58, 105, 29, 43, 112, 110, 28, 99, 32, 101, 27, 24, 118, 35, 119, 50, 37, 91, 55, 88, 23, 76, 98, 47, 30, 40, 94, 125, 93, 82, 20, 11, 92, 14, 19, 83, 85, 77, 80, 16, 87, 10, 72, 9, 8, 12, 81, 15, 73, 74, 95, 5, 75, 7, 71, 70, 69, 67, 68, 1, 3, 6, 2, 65, 4, 66, 64, 0], [63, 59, 52, 104, 119, 125, 56, 122, 60, 113, 62, 123, 114, 124, 50, 61, 127, 53, 54, 57, 58, 46, 109, 118, 111, 49, 48, 126, 116, 51, 47, 55, 117, 82, 120, 44, 115, 108, 45, 121, 112, 43, 110, 79, 88, 107, 84, 106, 42, 39, 41, 80, 87, 102, 28, 90, 105, 91, 76, 85, 86, 96, 103, 19, 30, 93, 9, 38, 100, 12, 81, 14, 101, 16, 13, 37, 36, 7, 89, 73, 4, 5, 33, 40, 74, 32, 99, 83, 35, 21, 97, 8, 94, 11, 25, 10, 15, 27, 65, 77, 72, 70, 92, 69, 34, 22, 2, 78, 23, 75, 17, 98, 31, 95, 1, 67, 24, 18, 29, 71, 0, 26, 20, 3, 6, 66, 68, 64], [104, 34, 52, 63, 59, 77, 17, 90, 10, 7, 78, 87, 68, 75, 27, 20, 31, 21, 13, 118, 4, 119, 66, 79, 89, 93, 124, 71, 46, 70, 125, 14, 100, 26, 23, 82, 6, 83, 73, 84, 22, 19, 122, 86, 95, 64, 74, 85, 11, 1, 76, 113, 8, 81, 88, 15, 18, 80, 25, 111, 29, 56, 28, 60, 2, 65, 94, 67, 108, 123, 50, 72, 120, 36, 0, 69, 12, 5, 3, 91, 40, 102, 41, 30, 32, 61, 116, 39, 92, 127, 48, 9, 98, 16, 115, 62, 24, 126, 47, 99, 121, 38, 42, 110, 45, 109, 37, 49, 57, 101, 96, 97, 117, 33, 54, 103, 106, 105, 58, 35, 55, 51, 44, 43, 112, 107, 53, 114], [104, 34, 52, 63, 59, 7, 92, 71, 87, 17, 78, 119, 28, 68, 10, 91, 31, 75, 83, 125, 90, 77, 85, 94, 60, 1, 22, 3, 122, 100, 18, 4, 16, 8, 124, 26, 65, 113, 118, 23, 84, 46, 74, 56, 89, 93, 72, 123, 14, 25, 5, 13, 76, 79, 9, 81, 19, 12, 21, 6, 82, 62, 30, 116, 11, 88, 86, 70, 27, 66, 98, 102, 80, 24, 67, 120, 36, 111, 69, 40, 108, 20, 2, 64, 61, 48, 39, 73, 15, 96, 95, 117, 43, 101, 0, 38, 29, 37, 33, 50, 121, 32, 55, 35, 97, 109, 47, 103, 45, 99, 105, 106, 127, 54, 110, 57, 44, 41, 42, 49, 115, 51, 107, 112, 58, 126, 114, 53], [104, 52, 63, 59, 119, 125, 124, 60, 46, 62, 113, 118, 27, 123, 48, 39, 116, 50, 56, 40, 122, 88, 34, 32, 108, 57, 111, 109, 106, 61, 115, 47, 110, 30, 28, 51, 54, 120, 55, 43, 35, 117, 49, 114, 107, 121, 42, 41, 98, 58, 103, 105, 36, 102, 100, 79, 33, 45, 53, 37, 97, 76, 44, 38, 126, 80, 86, 101, 24, 82, 112, 127, 85, 26, 99, 94, 95, 9, 96, 84, 19, 73, 20, 91, 22, 31, 7, 25, 93, 81, 14, 13, 5, 74, 92, 4, 87, 90, 8, 89, 11, 16, 1, 12, 70, 17, 69, 23, 18, 72, 10, 2, 29, 21, 83, 65, 77, 15, 78, 75, 71, 67, 3, 0, 64, 6, 66, 68], [105, 112, 39, 34, 101, 48, 50, 111, 118, 85, 124, 53, 23, 59, 21, 125, 90, 31, 82, 119, 30, 108, 56, 54, 117, 25, 93, 87, 60, 18, 96, 63, 127, 41, 19, 22, 52, 61, 122, 28, 123, 92, 88, 51, 103, 94, 89, 47, 95, 26, 37, 86, 16, 80, 57, 110, 121, 81, 38, 102, 84, 55, 29, 49, 91, 17, 97, 32, 45, 33, 11, 106, 44, 40, 107, 27, 99, 109, 62, 58, 114, 20, 78, 42, 120, 79, 14, 100, 104, 13, 116, 126, 46, 15, 24, 115, 36, 83, 12, 76, 35, 43, 75, 113, 71, 77, 9, 6, 5, 74, 98, 10, 73, 72, 8, 69, 7, 70, 2, 66, 68, 3, 65, 67, 0, 1, 64, 4], [50, 48, 105, 124, 34, 111, 38, 112, 117, 108, 119, 90, 57, 54, 39, 106, 116, 94, 114, 120, 122, 125, 118, 109, 127, 42, 63, 110, 103, 126, 52, 51, 93, 49, 92, 45, 85, 35, 82, 23, 59, 36, 31, 60, 107, 113, 55, 46, 58, 29, 104, 87, 9, 101, 47, 61, 83, 62, 40, 121, 89, 56, 53, 99, 97, 123, 44, 80, 96, 43, 115, 28, 91, 76, 37, 100, 24, 19, 88, 102, 78, 71, 16, 14, 33, 32, 86, 77, 22, 30, 95, 26, 41, 27, 84, 25, 74, 98, 20, 79, 72, 17, 13, 21, 15, 18, 7, 68, 73, 75, 11, 6, 10, 2, 67, 8, 12, 4, 81, 69, 70, 66, 64, 3, 5, 0, 1, 65], [105, 101, 34, 112, 48, 50, 89, 31, 39, 85, 83, 124, 23, 111, 87, 80, 54, 14, 82, 76, 118, 123, 59, 25, 41, 93, 21, 119, 16, 29, 63, 125, 108, 53, 19, 95, 56, 9, 117, 103, 94, 49, 116, 84, 90, 22, 20, 60, 15, 75, 52, 122, 47, 71, 26, 24, 109, 46, 74, 127, 17, 28, 32, 35, 11, 33, 57, 12, 38, 96, 62, 102, 42, 120, 92, 91, 113, 30, 114, 77, 13, 79, 107, 10, 88, 73, 104, 70, 121, 43, 78, 97, 100, 61, 81, 18, 99, 44, 27, 72, 51, 6, 106, 58, 8, 110, 55, 5, 115, 126, 7, 86, 40, 36, 69, 98, 45, 37, 68, 67, 3, 66, 4, 2, 65, 1, 0, 64], [50, 101, 38, 105, 34, 112, 118, 59, 119, 83, 31, 85, 89, 37, 23, 111, 90, 78, 49, 61, 103, 124, 21, 20, 74, 71, 125, 52, 88, 113, 53, 57, 19, 122, 104, 92, 11, 44, 106, 82, 54, 14, 109, 48, 63, 29, 22, 123, 40, 51, 28, 96, 76, 79, 108, 8, 41, 17, 9, 87, 60, 18, 68, 102, 16, 127, 56, 107, 126, 25, 114, 42, 72, 121, 24, 84, 30, 110, 93, 13, 27, 10, 117, 100, 62, 46, 39, 116, 4, 47, 35, 15, 81, 58, 91, 2, 115, 67, 45, 97, 120, 32, 94, 69, 33, 43, 7, 86, 26, 99, 55, 80, 77, 6, 1, 64, 36, 95, 70, 66, 5, 65, 0, 3, 75, 98, 12, 73], [110, 39, 33, 31, 46, 85, 70, 16, 53, 10, 14, 72, 94, 82, 12, 75, 68, 3, 0, 2, 1, 9, 65, 66, 6, 50, 67, 8, 104, 21, 86, 13, 120, 124, 17, 22, 4, 79, 18, 83, 57, 54, 118, 63, 90, 11, 69, 58, 123, 73, 119, 51, 126, 103, 78, 7, 127, 41, 24, 20, 99, 38, 115, 87, 80, 88, 29, 121, 56, 49, 101, 71, 102, 114, 35, 76, 98, 28, 74, 81, 36, 23, 84, 108, 125, 89, 47, 19, 96, 77, 113, 107, 45, 111, 100, 64, 5, 40, 61, 48, 117, 95, 34, 92, 26, 25, 91, 42, 52, 55, 37, 122, 93, 116, 112, 27, 105, 15, 43, 62, 106, 60, 59, 30, 109, 44, 32, 97], [110, 39, 33, 31, 46, 12, 94, 10, 6, 16, 82, 4, 72, 14, 85, 50, 74, 2, 79, 53, 70, 17, 86, 67, 115, 124, 51, 69, 21, 23, 28, 24, 76, 80, 120, 22, 57, 78, 8, 63, 117, 75, 18, 123, 66, 73, 127, 65, 126, 104, 49, 84, 118, 103, 88, 56, 58, 96, 64, 48, 77, 90, 13, 54, 20, 32, 9, 68, 121, 0, 38, 122, 108, 41, 62, 7, 11, 101, 81, 83, 59, 113, 45, 43, 93, 87, 44, 55, 47, 15, 89, 112, 27, 109, 92, 119, 114, 107, 25, 116, 5, 98, 125, 19, 95, 105, 99, 60, 91, 40, 34, 36, 26, 61, 37, 1, 111, 29, 52, 71, 35, 100, 102, 106, 42, 3, 30, 97], [39, 33, 110, 53, 31, 23, 85, 46, 82, 79, 94, 16, 12, 14, 88, 19, 104, 10, 126, 57, 17, 112, 50, 70, 75, 51, 5, 63, 72, 124, 66, 115, 68, 120, 106, 64, 65, 71, 86, 81, 60, 48, 84, 55, 8, 123, 9, 125, 47, 11, 108, 56, 21, 40, 42, 3, 121, 41, 59, 24, 116, 95, 25, 49, 91, 119, 58, 38, 101, 96, 54, 22, 111, 117, 99, 122, 87, 113, 35, 18, 26, 105, 32, 34, 73, 118, 90, 127, 78, 29, 37, 100, 107, 114, 83, 27, 15, 44, 13, 45, 61, 102, 28, 109, 52, 89, 7, 80, 98, 67, 20, 93, 43, 36, 62, 74, 103, 92, 4, 2, 6, 76, 30, 0, 69, 77, 1, 97], [110, 33, 39, 31, 85, 46, 94, 24, 74, 82, 16, 14, 72, 10, 12, 86, 88, 83, 22, 53, 79, 23, 4, 13, 124, 51, 73, 90, 50, 80, 126, 5, 115, 123, 11, 120, 57, 71, 6, 78, 18, 63, 2, 70, 8, 48, 17, 75, 9, 103, 21, 67, 58, 119, 65, 118, 91, 106, 68, 56, 30, 104, 15, 69, 84, 92, 121, 87, 49, 19, 54, 40, 28, 41, 117, 77, 61, 59, 81, 125, 76, 36, 114, 20, 26, 27, 111, 99, 127, 7, 25, 62, 3, 32, 55, 60, 35, 42, 96, 112, 101, 89, 45, 43, 47, 113, 44, 93, 34, 38, 100, 102, 52, 105, 98, 108, 109, 66, 37, 116, 122, 1, 64, 95, 97, 29, 107, 0], [112, 40, 34, 53, 23, 27, 122, 29, 19, 81, 76, 21, 25, 87, 63, 31, 56, 79, 118, 119, 77, 59, 15, 93, 62, 57, 30, 16, 72, 90, 47, 9, 82, 18, 26, 11, 92, 97, 109, 125, 33, 95, 78, 101, 123, 32, 44, 96, 49, 83, 35, 105, 73, 120, 24, 51, 86, 80, 106, 121, 103, 88, 98, 55, 36, 14, 22, 52, 38, 10, 71, 46, 67, 12, 28, 89, 91, 110, 127, 111, 61, 6, 54, 37, 107, 13, 94, 60, 42, 85, 84, 17, 102, 58, 5, 41, 99, 43, 113, 126, 74, 20, 115, 124, 108, 48, 45, 100, 8, 69, 116, 7, 39, 114, 50, 70, 4, 104, 117, 68, 2, 3, 1, 75, 65, 66, 0, 64], [112, 53, 40, 38, 34, 56, 47, 119, 44, 43, 52, 63, 62, 49, 122, 117, 114, 125, 61, 116, 120, 111, 109, 45, 123, 55, 31, 23, 25, 95, 59, 60, 126, 51, 118, 29, 106, 42, 127, 57, 107, 50, 54, 121, 27, 92, 41, 58, 24, 115, 46, 16, 113, 105, 48, 30, 108, 39, 110, 124, 84, 21, 93, 82, 96, 103, 88, 101, 19, 35, 22, 100, 90, 89, 102, 37, 36, 33, 32, 18, 87, 26, 17, 97, 99, 81, 86, 28, 20, 94, 85, 80, 77, 11, 14, 98, 78, 83, 13, 15, 12, 91, 76, 3, 74, 68, 8, 9, 10, 65, 7, 6, 79, 71, 4, 72, 75, 66, 73, 70, 2, 104, 5, 0, 69, 1, 67, 64], [53, 40, 112, 34, 19, 23, 81, 27, 56, 125, 21, 77, 93, 47, 82, 57, 122, 88, 120, 58, 25, 76, 59, 11, 9, 28, 14, 118, 61, 95, 62, 67, 79, 70, 104, 30, 91, 114, 92, 31, 26, 36, 44, 80, 16, 73, 119, 107, 52, 15, 7, 72, 90, 121, 20, 29, 63, 83, 126, 6, 85, 99, 12, 116, 105, 100, 18, 86, 106, 48, 60, 38, 22, 43, 78, 110, 115, 127, 69, 24, 50, 97, 87, 46, 103, 1, 84, 35, 123, 13, 41, 17, 4, 5, 89, 94, 32, 55, 10, 54, 102, 37, 98, 96, 74, 109, 45, 42, 108, 49, 111, 124, 75, 33, 51, 39, 117, 68, 113, 101, 3, 65, 71, 8, 0, 2, 64, 66], [40, 53, 34, 112, 91, 19, 21, 29, 81, 122, 79, 73, 93, 76, 6, 56, 16, 11, 47, 125, 52, 8, 63, 70, 67, 59, 62, 65, 95, 72, 17, 119, 9, 118, 27, 126, 57, 1, 12, 38, 104, 10, 120, 74, 77, 71, 61, 7, 121, 58, 86, 85, 24, 44, 30, 75, 23, 88, 84, 4, 3, 31, 68, 15, 28, 13, 78, 32, 83, 114, 14, 80, 46, 89, 20, 82, 25, 90, 123, 92, 87, 0, 2, 94, 117, 69, 26, 66, 36, 96, 18, 22, 5, 100, 35, 99, 60, 45, 37, 54, 51, 124, 108, 97, 33, 55, 101, 105, 102, 42, 103, 41, 107, 43, 49, 127, 64, 39, 106, 109, 111, 115, 110, 50, 113, 116, 48, 98]], "model.layers.4.self_attn.k_proj": [[37, 53, 56, 123, 122, 97, 117, 106, 30, 84, 16, 27, 87, 82, 13, 26, 75, 100, 50, 6, 40, 77, 73, 83, 78, 31, 42, 72, 17, 28, 64, 86, 103, 65, 38, 108, 21, 94, 35, 120, 5, 68, 118, 60, 45, 25, 47, 3, 74, 61, 127, 101, 2, 0, 89, 126, 24, 88, 43, 29, 58, 79, 52, 96, 113, 110, 55, 51, 4, 46, 49, 125, 119, 54, 66, 33, 109, 62, 105, 104, 57, 59, 121, 115, 63, 116, 41, 91, 111, 8, 10, 12, 112, 85, 44, 107, 23, 98, 93, 71, 34, 114, 22, 48, 124, 67, 32, 99, 39, 70, 92, 76, 14, 1, 11, 7, 90, 36, 102, 18, 95, 69, 19, 81, 80, 15, 20, 9], [103, 118, 52, 87, 97, 95, 20, 0, 13, 82, 81, 79, 70, 11, 9, 10, 68, 90, 3, 8, 66, 1, 47, 28, 110, 42, 53, 14, 63, 5, 71, 121, 86, 43, 22, 127, 55, 26, 65, 113, 92, 123, 2, 120, 69, 58, 46, 74, 122, 44, 88, 109, 48, 102, 39, 56, 61, 107, 125, 115, 16, 36, 89, 67, 57, 117, 126, 54, 50, 62, 73, 64, 45, 59, 29, 80, 93, 124, 114, 106, 116, 7, 38, 78, 6, 104, 100, 25, 96, 40, 27, 60, 108, 51, 83, 77, 37, 15, 119, 112, 101, 19, 99, 85, 105, 32, 21, 91, 30, 12, 76, 72, 98, 111, 4, 94, 41, 34, 35, 18, 49, 33, 24, 23, 17, 84, 75, 31], [103, 52, 118, 124, 35, 86, 92, 83, 79, 13, 1, 74, 72, 75, 31, 0, 16, 4, 70, 81, 105, 71, 26, 3, 21, 69, 2, 117, 116, 24, 53, 28, 14, 94, 5, 39, 12, 61, 112, 123, 41, 46, 95, 121, 82, 22, 56, 113, 54, 47, 55, 106, 84, 58, 59, 99, 48, 122, 9, 89, 98, 49, 126, 115, 50, 30, 40, 96, 34, 114, 63, 45, 23, 119, 111, 60, 51, 108, 100, 125, 17, 101, 7, 42, 107, 127, 87, 36, 93, 104, 67, 62, 43, 88, 20, 102, 85, 120, 33, 109, 29, 57, 91, 37, 44, 76, 32, 97, 38, 110, 90, 27, 25, 18, 73, 77, 80, 66, 19, 78, 68, 6, 64, 15, 65, 10, 11, 8], [109, 38, 0, 95, 33, 45, 89, 124, 66, 65, 17, 6, 79, 4, 10, 76, 2, 112, 64, 9, 69, 11, 126, 60, 56, 84, 59, 115, 61, 3, 47, 50, 18, 125, 52, 7, 105, 63, 118, 68, 78, 39, 116, 111, 46, 67, 40, 113, 110, 122, 121, 57, 23, 8, 127, 58, 51, 22, 44, 86, 53, 83, 88, 13, 70, 16, 108, 104, 117, 41, 92, 5, 37, 43, 119, 99, 48, 103, 101, 42, 90, 96, 80, 54, 29, 30, 55, 71, 77, 91, 123, 120, 35, 85, 94, 32, 93, 98, 62, 100, 114, 107, 26, 75, 102, 27, 36, 106, 87, 24, 28, 73, 19, 49, 34, 82, 1, 21, 74, 72, 97, 12, 15, 14, 81, 20, 25, 31], [40, 52, 59, 63, 98, 75, 26, 95, 17, 119, 78, 116, 22, 10, 77, 20, 0, 2, 6, 4, 83, 7, 118, 29, 113, 72, 123, 124, 110, 67, 18, 60, 125, 70, 111, 3, 23, 28, 50, 122, 15, 115, 55, 56, 54, 46, 121, 57, 48, 53, 47, 49, 120, 87, 62, 51, 117, 126, 114, 58, 127, 16, 61, 89, 106, 24, 9, 109, 112, 12, 43, 42, 45, 44, 21, 5, 103, 91, 108, 65, 107, 105, 8, 64, 41, 36, 39, 11, 102, 37, 27, 92, 99, 101, 93, 84, 38, 104, 35, 33, 31, 94, 32, 14, 90, 30, 96, 86, 97, 100, 69, 85, 66, 1, 25, 19, 74, 88, 80, 82, 71, 13, 79, 34, 81, 73, 76, 68], [41, 48, 98, 50, 37, 95, 23, 89, 82, 93, 90, 118, 114, 76, 80, 85, 74, 65, 44, 83, 64, 14, 59, 6, 77, 47, 53, 19, 68, 124, 86, 16, 27, 81, 20, 8, 111, 102, 9, 78, 66, 121, 125, 17, 71, 7, 112, 67, 119, 84, 106, 39, 94, 12, 32, 54, 63, 116, 117, 88, 72, 113, 96, 26, 62, 42, 69, 110, 115, 127, 3, 87, 22, 43, 107, 104, 4, 56, 51, 122, 60, 99, 58, 52, 123, 30, 15, 49, 92, 46, 108, 11, 38, 36, 61, 79, 35, 97, 33, 120, 45, 126, 91, 18, 28, 13, 109, 103, 70, 29, 40, 5, 75, 21, 2, 100, 57, 55, 10, 24, 0, 31, 25, 73, 1, 105, 101, 34], [46, 103, 110, 97, 53, 14, 0, 85, 95, 82, 16, 75, 12, 72, 17, 10, 66, 70, 71, 68, 79, 3, 51, 1, 118, 5, 124, 63, 30, 126, 8, 114, 11, 23, 40, 9, 50, 123, 90, 120, 56, 65, 86, 117, 77, 119, 57, 81, 113, 125, 122, 58, 112, 88, 67, 18, 121, 83, 84, 19, 116, 87, 62, 47, 32, 20, 127, 29, 78, 91, 44, 89, 28, 25, 93, 6, 24, 54, 2, 69, 92, 107, 27, 37, 42, 43, 76, 105, 100, 101, 34, 15, 22, 35, 59, 45, 7, 109, 99, 102, 36, 60, 55, 98, 108, 61, 115, 26, 96, 48, 111, 49, 80, 106, 41, 33, 38, 73, 52, 13, 94, 104, 74, 64, 4, 21, 31, 39], [104, 53, 112, 98, 48, 21, 11, 81, 19, 16, 0, 27, 93, 5, 73, 77, 117, 76, 2, 79, 8, 6, 65, 59, 122, 111, 25, 62, 102, 68, 110, 69, 57, 67, 72, 30, 50, 71, 56, 3, 84, 116, 15, 52, 94, 92, 74, 125, 91, 45, 121, 119, 31, 39, 58, 63, 13, 108, 70, 44, 55, 54, 7, 46, 61, 107, 90, 51, 32, 105, 118, 126, 28, 88, 109, 66, 41, 22, 96, 10, 114, 82, 113, 106, 86, 100, 60, 115, 120, 26, 124, 99, 24, 64, 127, 97, 23, 33, 29, 38, 103, 36, 14, 78, 42, 1, 95, 47, 43, 49, 35, 9, 18, 37, 101, 123, 87, 89, 75, 12, 85, 20, 83, 17, 4, 80, 40, 34]], "model.layers.4.self_attn.qk_proj": [[52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 50, 56, 48, 122, 117, 123, 39, 95, 103, 33, 41, 104, 31, 87, 17, 23, 11, 75, 81, 15, 38, 79, 85, 21, 28, 116, 102, 119, 40, 74, 18, 82, 42, 13, 80, 22, 10, 72, 97, 25, 64, 77, 20, 16, 6, 19, 83, 0, 12, 84, 70, 89, 91, 26, 86, 90, 78, 14, 76, 68, 125, 4, 37, 99, 61, 8, 94, 73, 9, 101, 126, 47, 65, 1, 35, 2, 98, 111, 54, 113, 66, 60, 114, 105, 51, 120, 29, 121, 92, 71, 7, 57, 108, 62, 67, 115, 127, 30, 44, 93, 5, 88, 27, 69, 3, 34, 106, 49, 24, 55, 107, 58, 43, 96, 32, 36, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 59, 63, 50, 48, 56, 117, 122, 123, 39, 95, 103, 33, 41, 31, 87, 104, 11, 23, 17, 81, 15, 38, 116, 28, 0, 79, 40, 85, 75, 64, 21, 74, 18, 72, 10, 97, 102, 25, 89, 42, 22, 6, 70, 19, 82, 26, 20, 83, 91, 68, 77, 16, 80, 12, 13, 125, 78, 119, 1, 90, 14, 84, 37, 4, 76, 86, 98, 101, 54, 105, 94, 65, 66, 113, 2, 47, 9, 111, 121, 73, 126, 35, 8, 114, 61, 120, 108, 99, 57, 51, 62, 3, 115, 60, 29, 69, 71, 5, 67, 92, 106, 7, 43, 58, 34, 24, 93, 127, 27, 30, 49, 44, 55, 88, 107, 96, 32, 100, 36], [52, 118, 53, 124, 46, 109, 112, 45, 110, 63, 59, 50, 56, 122, 117, 48, 123, 39, 103, 95, 33, 41, 87, 104, 31, 23, 11, 15, 81, 0, 38, 72, 70, 75, 85, 17, 28, 64, 18, 116, 74, 82, 10, 79, 21, 13, 80, 40, 16, 22, 6, 97, 4, 83, 25, 42, 102, 19, 77, 20, 14, 12, 91, 26, 89, 84, 65, 68, 78, 76, 1, 90, 119, 125, 54, 86, 37, 66, 105, 2, 98, 94, 99, 121, 9, 8, 101, 126, 47, 114, 73, 3, 61, 120, 35, 111, 67, 5, 113, 7, 71, 69, 93, 57, 60, 108, 92, 51, 24, 29, 62, 30, 115, 44, 34, 88, 127, 58, 55, 27, 106, 49, 107, 43, 96, 32, 36, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 59, 63, 50, 56, 122, 48, 117, 39, 123, 95, 103, 33, 41, 31, 104, 87, 79, 23, 17, 11, 64, 81, 75, 72, 18, 38, 0, 70, 28, 85, 15, 21, 102, 74, 80, 97, 25, 116, 13, 40, 19, 91, 10, 83, 82, 16, 42, 6, 89, 4, 14, 22, 84, 77, 12, 86, 20, 76, 119, 78, 26, 94, 90, 98, 68, 2, 121, 65, 125, 66, 37, 101, 1, 47, 99, 105, 54, 9, 8, 114, 67, 113, 73, 3, 69, 60, 111, 120, 115, 35, 126, 61, 71, 58, 34, 29, 92, 93, 5, 88, 57, 108, 30, 44, 7, 62, 51, 24, 127, 43, 27, 106, 49, 107, 55, 96, 32, 36, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 56, 117, 50, 122, 48, 123, 95, 103, 39, 41, 33, 104, 87, 79, 64, 11, 75, 81, 17, 31, 70, 72, 23, 0, 38, 40, 85, 6, 15, 116, 28, 18, 21, 80, 82, 10, 74, 13, 97, 16, 22, 83, 84, 4, 42, 77, 12, 19, 25, 78, 76, 102, 26, 20, 91, 2, 89, 68, 86, 8, 65, 119, 14, 66, 120, 1, 90, 98, 9, 94, 61, 125, 113, 73, 37, 126, 47, 111, 101, 99, 121, 35, 114, 7, 67, 115, 105, 51, 57, 69, 29, 5, 60, 71, 3, 93, 92, 54, 30, 34, 108, 62, 44, 58, 106, 127, 88, 27, 24, 49, 107, 55, 43, 36, 96, 32, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 59, 63, 50, 48, 56, 122, 117, 123, 95, 39, 103, 33, 41, 104, 81, 31, 87, 79, 11, 17, 23, 75, 0, 38, 85, 64, 15, 28, 82, 21, 40, 70, 10, 6, 72, 4, 102, 80, 18, 116, 13, 74, 97, 25, 12, 16, 77, 78, 76, 83, 20, 22, 8, 91, 84, 86, 68, 19, 89, 42, 119, 14, 90, 26, 66, 125, 47, 126, 2, 1, 73, 101, 61, 94, 65, 98, 37, 54, 99, 105, 9, 113, 35, 5, 111, 29, 121, 114, 3, 71, 120, 7, 51, 115, 34, 49, 62, 60, 67, 57, 92, 108, 69, 58, 127, 93, 107, 24, 106, 27, 30, 44, 88, 43, 55, 96, 36, 32, 100], [52, 118, 53, 124, 109, 46, 45, 112, 110, 59, 63, 50, 56, 48, 122, 117, 123, 39, 103, 95, 33, 41, 104, 17, 31, 87, 79, 23, 81, 38, 11, 40, 85, 75, 28, 25, 82, 116, 18, 15, 21, 6, 74, 0, 42, 102, 13, 22, 84, 97, 76, 26, 78, 10, 20, 64, 14, 83, 16, 80, 70, 72, 19, 12, 89, 86, 119, 8, 68, 91, 4, 77, 90, 47, 61, 37, 101, 126, 1, 94, 125, 120, 9, 66, 2, 73, 98, 65, 121, 54, 113, 35, 29, 99, 111, 114, 57, 60, 71, 105, 7, 34, 93, 127, 51, 67, 62, 3, 58, 92, 108, 69, 115, 27, 5, 44, 30, 88, 24, 106, 55, 49, 43, 107, 96, 36, 100, 32], [52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 50, 56, 122, 48, 117, 123, 39, 95, 103, 33, 41, 104, 81, 87, 23, 17, 79, 75, 31, 85, 28, 38, 6, 15, 11, 21, 40, 64, 82, 116, 74, 10, 25, 18, 97, 76, 0, 42, 16, 77, 70, 22, 19, 84, 80, 78, 8, 119, 13, 83, 89, 20, 26, 91, 102, 14, 12, 86, 68, 4, 101, 37, 125, 66, 72, 1, 90, 65, 9, 54, 111, 98, 94, 60, 35, 126, 44, 105, 73, 120, 113, 57, 2, 93, 47, 121, 61, 7, 99, 29, 51, 24, 69, 58, 92, 67, 114, 3, 27, 71, 115, 108, 34, 5, 30, 127, 62, 106, 43, 88, 49, 55, 107, 96, 36, 32, 100], [52, 118, 53, 124, 46, 109, 112, 45, 110, 59, 63, 50, 56, 48, 122, 117, 123, 39, 95, 103, 33, 41, 87, 104, 23, 81, 31, 17, 11, 79, 85, 21, 75, 38, 82, 6, 15, 40, 97, 10, 64, 74, 83, 0, 28, 18, 80, 25, 8, 116, 22, 20, 16, 13, 42, 102, 70, 91, 76, 89, 78, 19, 84, 86, 77, 90, 12, 68, 54, 14, 119, 26, 125, 105, 4, 37, 98, 101, 66, 47, 72, 94, 111, 65, 1, 2, 61, 121, 99, 9, 126, 120, 73, 58, 113, 35, 34, 92, 93, 29, 57, 114, 51, 115, 7, 67, 108, 69, 62, 60, 3, 30, 71, 24, 44, 27, 106, 5, 43, 55, 88, 127, 49, 32, 107, 96, 36, 100], [52, 118, 53, 124, 109, 46, 45, 112, 110, 59, 63, 50, 48, 56, 122, 117, 123, 39, 103, 95, 33, 87, 41, 81, 23, 31, 104, 17, 75, 15, 21, 38, 11, 28, 85, 116, 79, 42, 18, 25, 82, 74, 40, 97, 83, 102, 0, 78, 8, 84, 16, 13, 19, 10, 90, 6, 22, 64, 12, 80, 89, 86, 26, 91, 20, 77, 70, 68, 14, 76, 119, 37, 4, 125, 101, 105, 94, 99, 54, 1, 111, 65, 47, 121, 98, 120, 72, 60, 73, 114, 126, 9, 92, 35, 66, 29, 2, 71, 61, 24, 34, 69, 7, 3, 115, 27, 113, 67, 51, 106, 57, 93, 44, 108, 5, 127, 49, 88, 58, 30, 62, 55, 107, 43, 96, 36, 32, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 56, 48, 50, 122, 117, 39, 123, 95, 103, 33, 41, 104, 31, 87, 75, 11, 64, 17, 79, 8, 81, 15, 23, 6, 82, 38, 28, 10, 40, 0, 21, 116, 85, 42, 70, 80, 18, 102, 97, 13, 74, 16, 83, 68, 22, 19, 77, 12, 119, 25, 20, 84, 91, 78, 26, 4, 14, 76, 86, 89, 47, 1, 66, 2, 65, 72, 101, 99, 90, 94, 9, 73, 98, 125, 37, 126, 121, 114, 105, 35, 67, 115, 127, 61, 111, 5, 60, 120, 69, 54, 113, 71, 92, 7, 3, 57, 108, 106, 29, 58, 51, 93, 30, 62, 24, 34, 107, 43, 49, 44, 27, 88, 55, 96, 36, 32, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 59, 63, 56, 50, 122, 48, 117, 123, 95, 39, 33, 103, 41, 104, 31, 15, 87, 23, 81, 75, 8, 17, 116, 64, 28, 70, 11, 40, 38, 85, 102, 18, 10, 6, 21, 0, 82, 16, 97, 79, 83, 42, 25, 77, 68, 13, 91, 19, 12, 20, 74, 22, 84, 80, 119, 89, 26, 125, 4, 14, 66, 76, 86, 78, 113, 2, 90, 101, 9, 61, 72, 65, 94, 47, 121, 51, 98, 73, 126, 111, 1, 37, 120, 105, 99, 54, 67, 35, 57, 7, 34, 69, 115, 60, 5, 93, 114, 92, 3, 62, 108, 55, 29, 71, 44, 30, 106, 127, 24, 58, 27, 88, 43, 49, 107, 32, 96, 36, 100], [52, 118, 53, 124, 109, 46, 45, 112, 110, 50, 59, 63, 48, 56, 122, 117, 123, 39, 103, 95, 33, 41, 104, 87, 17, 81, 31, 23, 38, 40, 28, 85, 75, 15, 11, 79, 25, 116, 21, 82, 83, 97, 16, 10, 22, 0, 8, 70, 18, 102, 42, 86, 26, 74, 6, 20, 89, 12, 84, 14, 80, 13, 76, 91, 19, 119, 90, 77, 78, 64, 37, 4, 101, 94, 68, 9, 72, 125, 98, 113, 1, 65, 29, 35, 105, 111, 61, 126, 2, 47, 99, 54, 60, 66, 51, 93, 121, 120, 7, 114, 73, 34, 57, 3, 27, 92, 58, 24, 71, 44, 108, 55, 5, 115, 88, 62, 49, 69, 127, 106, 67, 30, 43, 107, 36, 96, 32, 100], [52, 118, 53, 124, 109, 46, 45, 112, 110, 59, 63, 50, 56, 48, 117, 122, 123, 95, 39, 33, 103, 41, 87, 31, 104, 81, 17, 11, 23, 38, 75, 28, 79, 70, 15, 0, 85, 40, 64, 18, 10, 74, 8, 25, 97, 16, 22, 42, 102, 21, 82, 83, 116, 80, 86, 6, 84, 77, 78, 20, 12, 13, 14, 19, 91, 76, 90, 68, 72, 89, 119, 4, 9, 26, 1, 37, 125, 98, 47, 65, 2, 66, 101, 99, 105, 94, 51, 7, 61, 121, 54, 111, 35, 126, 73, 29, 120, 92, 113, 5, 60, 57, 108, 114, 67, 34, 115, 44, 127, 27, 71, 88, 3, 62, 93, 69, 58, 55, 24, 49, 30, 106, 43, 107, 36, 96, 32, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 59, 63, 50, 56, 48, 122, 117, 123, 39, 95, 33, 103, 41, 104, 87, 31, 75, 11, 23, 17, 38, 116, 70, 15, 74, 0, 81, 79, 21, 85, 42, 64, 28, 40, 119, 82, 6, 18, 16, 97, 102, 72, 19, 8, 25, 10, 13, 83, 22, 20, 80, 89, 84, 14, 4, 68, 78, 91, 77, 12, 76, 86, 26, 2, 90, 125, 1, 94, 9, 98, 37, 66, 65, 101, 120, 47, 73, 121, 126, 54, 3, 67, 7, 111, 105, 44, 99, 35, 92, 113, 57, 5, 115, 51, 61, 60, 114, 93, 27, 69, 29, 34, 71, 30, 49, 127, 108, 62, 88, 24, 55, 58, 106, 43, 107, 36, 96, 100, 32], [52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 50, 56, 48, 122, 117, 123, 95, 39, 103, 41, 33, 104, 87, 31, 11, 75, 23, 81, 38, 79, 17, 74, 28, 15, 70, 85, 64, 40, 18, 20, 116, 97, 102, 21, 16, 0, 72, 25, 6, 13, 10, 82, 91, 19, 80, 83, 84, 89, 22, 42, 86, 12, 76, 119, 125, 77, 4, 90, 68, 78, 8, 14, 94, 101, 66, 120, 26, 54, 73, 2, 98, 37, 111, 9, 1, 35, 126, 47, 105, 65, 34, 113, 99, 61, 51, 108, 29, 57, 121, 114, 3, 115, 60, 71, 7, 92, 67, 62, 93, 69, 5, 106, 30, 127, 58, 107, 44, 24, 43, 88, 49, 55, 27, 96, 100, 36, 32], [52, 118, 53, 124, 46, 109, 45, 112, 110, 63, 59, 50, 122, 56, 48, 117, 123, 39, 103, 95, 33, 41, 87, 23, 17, 31, 81, 15, 38, 11, 85, 79, 28, 75, 104, 72, 82, 18, 21, 74, 16, 25, 40, 14, 20, 22, 97, 6, 86, 89, 19, 10, 0, 102, 70, 84, 76, 77, 116, 91, 83, 64, 68, 13, 42, 12, 78, 80, 90, 4, 26, 125, 8, 47, 1, 9, 101, 94, 119, 105, 61, 37, 65, 98, 73, 99, 126, 113, 29, 2, 121, 66, 120, 114, 35, 111, 7, 71, 60, 34, 51, 54, 5, 58, 92, 3, 57, 24, 69, 93, 67, 88, 127, 108, 27, 115, 55, 62, 44, 30, 49, 106, 43, 96, 107, 36, 32, 100], [52, 118, 53, 124, 46, 109, 45, 112, 110, 63, 59, 50, 56, 122, 48, 39, 123, 117, 103, 95, 33, 41, 87, 31, 17, 104, 15, 75, 81, 21, 85, 23, 72, 82, 11, 28, 38, 79, 6, 18, 16, 64, 42, 10, 40, 74, 116, 89, 83, 0, 97, 70, 20, 25, 22, 12, 19, 14, 13, 102, 125, 84, 78, 80, 76, 77, 91, 86, 68, 90, 119, 26, 4, 105, 9, 126, 37, 99, 1, 98, 94, 66, 65, 8, 101, 73, 121, 2, 51, 111, 113, 3, 54, 47, 60, 7, 120, 67, 29, 127, 114, 61, 35, 71, 93, 5, 92, 57, 115, 34, 69, 44, 49, 108, 24, 62, 27, 88, 30, 43, 58, 55, 106, 107, 32, 96, 36, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 50, 56, 122, 48, 117, 123, 39, 95, 103, 33, 41, 31, 104, 17, 87, 75, 38, 23, 15, 0, 72, 6, 81, 79, 21, 11, 85, 116, 82, 10, 40, 97, 16, 28, 64, 89, 18, 70, 83, 19, 42, 20, 74, 102, 25, 13, 76, 14, 84, 80, 119, 22, 77, 90, 78, 68, 91, 26, 12, 4, 65, 37, 86, 101, 66, 98, 1, 8, 111, 9, 105, 73, 125, 94, 114, 2, 120, 3, 99, 35, 47, 67, 126, 121, 29, 60, 113, 51, 54, 61, 62, 34, 69, 93, 7, 30, 57, 92, 5, 71, 44, 115, 108, 88, 127, 24, 49, 27, 43, 58, 55, 106, 107, 96, 32, 36, 100], [52, 118, 53, 124, 46, 109, 45, 112, 110, 63, 59, 50, 56, 48, 122, 117, 123, 39, 95, 33, 103, 41, 17, 31, 87, 81, 23, 75, 11, 116, 104, 38, 15, 79, 28, 72, 21, 85, 74, 10, 0, 6, 40, 64, 70, 20, 18, 97, 16, 83, 19, 77, 82, 102, 90, 78, 84, 68, 25, 89, 13, 91, 76, 22, 4, 80, 42, 119, 125, 86, 12, 26, 14, 94, 65, 8, 37, 1, 98, 9, 101, 73, 66, 54, 105, 99, 120, 113, 47, 2, 121, 111, 61, 35, 71, 29, 3, 126, 7, 58, 62, 5, 69, 60, 92, 34, 51, 115, 114, 57, 93, 67, 106, 108, 24, 43, 127, 88, 44, 49, 27, 55, 30, 96, 36, 32, 107, 100], [52, 118, 53, 124, 46, 109, 112, 45, 110, 63, 59, 56, 50, 122, 117, 48, 39, 123, 95, 33, 103, 41, 31, 87, 104, 11, 81, 79, 23, 75, 17, 38, 18, 15, 85, 6, 72, 21, 28, 97, 40, 102, 10, 83, 0, 74, 64, 119, 82, 16, 116, 91, 25, 70, 77, 89, 76, 22, 80, 4, 19, 84, 42, 13, 68, 20, 12, 121, 26, 86, 78, 90, 14, 8, 47, 99, 105, 125, 94, 65, 66, 73, 120, 9, 37, 98, 114, 61, 1, 2, 101, 35, 34, 60, 126, 113, 54, 111, 29, 51, 5, 3, 69, 57, 24, 71, 108, 115, 92, 62, 7, 106, 30, 67, 93, 127, 58, 49, 43, 44, 88, 27, 55, 107, 96, 32, 36, 100], [52, 118, 53, 124, 46, 109, 112, 45, 110, 63, 59, 50, 56, 48, 122, 117, 123, 39, 95, 103, 33, 31, 41, 104, 87, 17, 38, 79, 75, 81, 85, 23, 15, 64, 116, 28, 11, 21, 0, 70, 18, 102, 83, 97, 89, 72, 40, 19, 82, 42, 26, 10, 80, 74, 91, 16, 6, 25, 77, 14, 84, 13, 20, 78, 4, 8, 22, 76, 119, 12, 68, 90, 1, 86, 37, 65, 66, 98, 47, 125, 73, 105, 94, 101, 9, 99, 60, 113, 3, 2, 126, 54, 111, 115, 57, 114, 61, 121, 35, 92, 7, 24, 69, 29, 120, 5, 67, 62, 51, 34, 49, 27, 108, 127, 93, 43, 71, 30, 58, 44, 106, 107, 88, 55, 32, 96, 100, 36], [52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 50, 56, 122, 48, 123, 117, 39, 95, 103, 33, 41, 87, 104, 31, 81, 11, 38, 23, 15, 17, 21, 28, 40, 85, 79, 0, 116, 75, 70, 74, 10, 16, 83, 97, 18, 22, 19, 64, 89, 82, 8, 84, 25, 77, 80, 6, 20, 42, 86, 119, 12, 91, 26, 72, 76, 13, 102, 4, 90, 78, 68, 37, 14, 66, 94, 65, 98, 1, 73, 125, 105, 101, 2, 47, 9, 35, 121, 111, 61, 120, 29, 113, 99, 114, 3, 54, 126, 51, 92, 60, 69, 34, 7, 24, 93, 30, 71, 58, 115, 27, 67, 57, 106, 5, 127, 62, 49, 108, 44, 43, 88, 107, 55, 96, 32, 100, 36], [52, 118, 53, 124, 109, 46, 45, 112, 110, 59, 63, 50, 48, 56, 122, 117, 123, 39, 95, 33, 103, 41, 31, 87, 104, 11, 81, 23, 17, 79, 38, 85, 75, 74, 40, 15, 116, 64, 28, 97, 18, 70, 8, 83, 25, 22, 21, 102, 42, 91, 80, 16, 6, 82, 90, 119, 68, 76, 13, 77, 89, 10, 19, 12, 86, 84, 26, 20, 78, 72, 14, 4, 0, 98, 37, 101, 9, 111, 65, 66, 105, 94, 54, 126, 125, 120, 99, 47, 2, 1, 73, 61, 113, 35, 121, 51, 115, 127, 29, 60, 71, 34, 57, 55, 93, 114, 69, 3, 7, 62, 92, 5, 67, 108, 106, 24, 30, 27, 88, 58, 49, 44, 107, 43, 96, 100, 36, 32], [52, 118, 53, 124, 109, 46, 45, 112, 110, 59, 63, 50, 56, 122, 48, 117, 123, 39, 103, 95, 33, 41, 104, 81, 87, 31, 17, 11, 23, 116, 15, 38, 75, 70, 85, 21, 18, 40, 79, 28, 82, 74, 83, 22, 84, 8, 77, 119, 42, 16, 19, 25, 20, 97, 80, 0, 6, 89, 10, 26, 76, 64, 12, 91, 102, 14, 78, 90, 86, 68, 47, 13, 4, 72, 65, 1, 125, 101, 37, 94, 113, 126, 73, 98, 54, 111, 9, 35, 2, 61, 114, 66, 71, 29, 60, 57, 120, 127, 121, 105, 93, 99, 34, 108, 51, 3, 92, 7, 24, 62, 69, 67, 55, 30, 106, 115, 88, 5, 44, 58, 27, 49, 43, 107, 96, 32, 100, 36], [52, 118, 53, 124, 46, 109, 45, 112, 110, 63, 50, 59, 56, 48, 123, 117, 122, 39, 95, 103, 33, 41, 87, 31, 81, 23, 104, 75, 15, 85, 11, 79, 17, 38, 28, 8, 40, 70, 74, 116, 82, 21, 97, 25, 22, 64, 18, 83, 42, 10, 89, 16, 119, 0, 102, 19, 20, 6, 14, 91, 13, 77, 80, 84, 76, 12, 90, 78, 26, 4, 68, 101, 86, 37, 111, 125, 1, 9, 98, 61, 66, 120, 72, 73, 105, 47, 65, 2, 94, 60, 57, 35, 113, 121, 51, 99, 126, 58, 92, 54, 7, 29, 106, 34, 115, 93, 62, 127, 67, 108, 114, 24, 5, 3, 49, 69, 27, 43, 55, 44, 30, 88, 71, 107, 96, 32, 36, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 59, 63, 50, 56, 48, 117, 122, 123, 39, 103, 95, 33, 41, 87, 104, 31, 11, 81, 85, 23, 15, 97, 0, 17, 28, 75, 79, 21, 8, 40, 38, 82, 102, 70, 18, 25, 119, 74, 22, 116, 16, 83, 19, 20, 80, 13, 77, 91, 10, 6, 64, 12, 14, 42, 89, 4, 26, 84, 76, 68, 90, 78, 47, 86, 99, 94, 125, 101, 37, 105, 66, 65, 72, 1, 126, 98, 73, 60, 9, 57, 35, 113, 2, 111, 61, 114, 29, 54, 121, 92, 120, 3, 67, 7, 51, 127, 69, 93, 30, 24, 71, 5, 106, 34, 27, 62, 115, 49, 44, 108, 58, 88, 107, 43, 55, 96, 36, 32, 100], [52, 118, 53, 124, 46, 109, 45, 112, 110, 63, 59, 50, 56, 48, 122, 117, 123, 39, 95, 103, 33, 41, 104, 31, 11, 17, 87, 75, 81, 0, 8, 15, 23, 79, 64, 116, 74, 40, 85, 28, 6, 38, 42, 21, 18, 20, 10, 82, 83, 97, 70, 4, 80, 76, 77, 22, 25, 102, 16, 91, 13, 14, 78, 86, 19, 12, 89, 68, 1, 119, 26, 66, 84, 113, 125, 65, 90, 47, 98, 72, 120, 94, 9, 2, 101, 37, 105, 60, 111, 73, 54, 99, 121, 35, 126, 61, 51, 67, 114, 7, 71, 34, 57, 3, 29, 69, 5, 115, 92, 127, 44, 62, 93, 58, 88, 108, 49, 55, 30, 24, 27, 106, 43, 107, 36, 96, 32, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 50, 56, 48, 122, 117, 123, 39, 95, 33, 103, 41, 11, 31, 104, 87, 75, 81, 116, 38, 79, 8, 23, 28, 17, 6, 15, 21, 74, 0, 85, 40, 82, 80, 64, 97, 25, 10, 42, 83, 102, 18, 19, 70, 89, 76, 91, 22, 77, 14, 119, 68, 13, 84, 16, 20, 4, 86, 12, 78, 72, 26, 90, 101, 121, 2, 125, 1, 37, 73, 9, 105, 65, 47, 66, 114, 94, 113, 98, 120, 111, 35, 99, 51, 108, 54, 34, 115, 61, 71, 3, 60, 67, 7, 5, 92, 29, 69, 126, 57, 62, 93, 58, 24, 30, 127, 106, 55, 49, 88, 44, 27, 43, 107, 96, 32, 36, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 59, 63, 50, 122, 48, 56, 117, 123, 39, 95, 33, 103, 41, 104, 31, 15, 87, 11, 75, 23, 17, 81, 6, 79, 38, 0, 28, 8, 85, 10, 18, 64, 22, 21, 82, 40, 97, 116, 16, 83, 13, 74, 25, 102, 77, 80, 70, 19, 86, 42, 76, 4, 91, 68, 78, 20, 12, 84, 90, 14, 89, 72, 26, 47, 66, 60, 1, 119, 125, 101, 73, 94, 37, 105, 2, 65, 98, 9, 126, 35, 113, 114, 61, 111, 120, 99, 121, 69, 7, 51, 62, 3, 54, 57, 29, 71, 108, 67, 5, 34, 115, 58, 93, 92, 106, 127, 30, 49, 24, 27, 44, 107, 88, 43, 55, 96, 32, 36, 100], [52, 118, 53, 124, 109, 46, 45, 112, 110, 63, 50, 59, 56, 122, 48, 117, 123, 103, 39, 95, 33, 41, 87, 31, 38, 81, 17, 23, 104, 85, 75, 11, 15, 6, 28, 0, 18, 79, 40, 42, 97, 89, 21, 10, 82, 25, 64, 70, 80, 102, 86, 74, 22, 8, 14, 20, 19, 26, 90, 68, 76, 77, 83, 116, 78, 72, 16, 12, 4, 13, 91, 37, 101, 119, 65, 84, 9, 1, 125, 66, 98, 111, 113, 126, 105, 73, 94, 61, 35, 47, 120, 2, 54, 99, 121, 69, 67, 51, 7, 29, 34, 114, 115, 60, 71, 62, 57, 93, 49, 58, 92, 27, 5, 44, 3, 127, 108, 24, 30, 106, 88, 55, 43, 107, 96, 36, 32, 100], [52, 118, 53, 124, 109, 46, 45, 112, 110, 59, 63, 50, 56, 48, 122, 123, 117, 39, 95, 103, 41, 33, 104, 31, 81, 87, 23, 11, 75, 15, 17, 85, 40, 79, 38, 6, 10, 42, 82, 74, 97, 25, 72, 0, 28, 20, 89, 64, 21, 102, 70, 116, 77, 18, 16, 19, 91, 80, 119, 76, 14, 125, 101, 83, 8, 13, 78, 26, 68, 22, 12, 84, 86, 47, 4, 90, 37, 111, 98, 54, 105, 126, 9, 66, 2, 121, 1, 94, 61, 65, 73, 120, 114, 99, 60, 35, 115, 29, 71, 113, 57, 62, 51, 108, 58, 67, 3, 93, 5, 7, 34, 92, 24, 69, 44, 106, 127, 88, 30, 27, 49, 43, 96, 55, 107, 32, 36, 100]], "model.layers.5.self_attn.q_proj": [[39, 106, 98, 50, 20, 80, 42, 23, 127, 116, 14, 49, 18, 10, 124, 51, 27, 47, 74, 61, 120, 103, 54, 12, 24, 112, 87, 57, 84, 71, 25, 115, 68, 48, 63, 29, 111, 123, 16, 96, 15, 26, 76, 113, 85, 82, 83, 9, 79, 88, 125, 52, 118, 92, 114, 28, 121, 91, 122, 21, 78, 93, 41, 62, 94, 99, 8, 59, 75, 86, 58, 6, 89, 81, 126, 30, 44, 22, 90, 97, 35, 11, 95, 60, 31, 2, 45, 104, 109, 110, 119, 36, 108, 70, 117, 102, 107, 19, 33, 40, 105, 56, 38, 32, 46, 13, 53, 3, 17, 77, 37, 43, 100, 101, 55, 7, 69, 34, 73, 4, 72, 5, 1, 65, 0, 66, 67, 64], [39, 98, 106, 49, 47, 51, 127, 57, 54, 23, 48, 27, 120, 116, 44, 63, 124, 50, 111, 92, 59, 123, 126, 60, 28, 46, 121, 105, 61, 41, 42, 52, 55, 112, 115, 43, 107, 118, 58, 125, 108, 53, 36, 119, 22, 110, 62, 117, 18, 102, 45, 113, 32, 40, 82, 104, 109, 96, 100, 21, 38, 35, 114, 56, 101, 122, 99, 30, 37, 25, 80, 97, 93, 33, 95, 31, 85, 89, 29, 87, 83, 78, 94, 91, 90, 14, 26, 34, 10, 79, 84, 20, 24, 19, 88, 16, 17, 103, 86, 12, 76, 81, 4, 74, 75, 77, 73, 15, 7, 11, 67, 13, 64, 70, 69, 72, 8, 71, 9, 3, 66, 68, 5, 6, 0, 65, 1, 2], [39, 106, 98, 50, 18, 74, 20, 88, 80, 14, 12, 23, 42, 68, 103, 116, 49, 6, 47, 51, 10, 9, 54, 4, 2, 7, 91, 21, 44, 83, 61, 121, 75, 64, 124, 127, 57, 70, 120, 71, 115, 87, 72, 92, 125, 16, 73, 13, 15, 11, 48, 28, 3, 24, 76, 17, 52, 22, 78, 67, 84, 89, 111, 63, 85, 123, 82, 112, 77, 59, 126, 86, 5, 8, 58, 60, 30, 81, 79, 66, 27, 113, 25, 19, 0, 1, 26, 118, 69, 110, 33, 114, 38, 94, 105, 122, 65, 95, 32, 31, 29, 41, 97, 55, 36, 102, 109, 93, 96, 101, 119, 46, 90, 53, 99, 117, 100, 108, 43, 45, 35, 34, 37, 40, 104, 62, 107, 56], [106, 39, 50, 98, 18, 12, 80, 20, 6, 23, 14, 9, 2, 74, 42, 47, 49, 116, 103, 64, 70, 28, 127, 87, 54, 51, 57, 91, 7, 71, 124, 68, 120, 61, 75, 115, 60, 3, 10, 4, 76, 24, 78, 67, 121, 27, 48, 26, 69, 13, 1, 11, 118, 126, 63, 89, 125, 84, 19, 17, 44, 123, 82, 16, 21, 66, 111, 15, 59, 83, 112, 119, 73, 65, 22, 8, 55, 52, 5, 77, 58, 41, 0, 62, 72, 105, 46, 81, 88, 85, 110, 108, 53, 30, 107, 33, 113, 109, 40, 114, 79, 104, 101, 29, 38, 43, 35, 99, 45, 117, 96, 122, 25, 36, 100, 102, 97, 95, 93, 56, 31, 86, 94, 90, 92, 32, 37, 34], [38, 110, 126, 125, 115, 48, 72, 112, 7, 23, 5, 4, 74, 6, 76, 70, 67, 29, 20, 11, 16, 13, 46, 51, 90, 81, 75, 12, 93, 9, 78, 26, 83, 18, 73, 66, 14, 80, 68, 65, 10, 84, 2, 69, 62, 59, 117, 17, 21, 19, 79, 122, 123, 15, 121, 56, 113, 116, 120, 55, 50, 8, 105, 91, 31, 64, 63, 124, 85, 114, 95, 87, 118, 49, 61, 53, 47, 111, 32, 58, 104, 89, 109, 27, 54, 88, 119, 71, 40, 127, 41, 44, 77, 82, 96, 45, 52, 60, 35, 108, 106, 107, 43, 42, 37, 57, 94, 28, 101, 103, 24, 1, 100, 36, 22, 86, 98, 39, 34, 3, 99, 97, 33, 25, 92, 30, 102, 0], [110, 38, 126, 112, 48, 46, 115, 125, 51, 62, 17, 116, 122, 59, 123, 56, 16, 105, 50, 121, 117, 14, 124, 84, 120, 104, 85, 32, 113, 49, 26, 55, 114, 61, 13, 119, 63, 53, 102, 118, 47, 31, 111, 41, 58, 88, 109, 27, 107, 45, 29, 127, 54, 44, 60, 40, 15, 108, 93, 52, 11, 42, 74, 106, 100, 24, 43, 57, 37, 91, 96, 103, 87, 19, 36, 90, 99, 101, 39, 35, 76, 33, 98, 34, 82, 97, 28, 95, 81, 94, 9, 89, 22, 30, 21, 92, 20, 25, 23, 72, 18, 8, 86, 71, 83, 75, 80, 6, 77, 10, 79, 78, 7, 0, 68, 5, 69, 66, 3, 12, 1, 65, 73, 70, 64, 2, 4, 67], [38, 110, 48, 112, 125, 115, 126, 46, 93, 51, 78, 83, 26, 21, 11, 72, 88, 122, 32, 123, 10, 31, 7, 124, 62, 116, 79, 77, 81, 50, 95, 15, 74, 56, 5, 29, 41, 85, 75, 61, 84, 59, 121, 18, 55, 19, 86, 90, 73, 117, 17, 106, 53, 113, 27, 120, 63, 91, 40, 80, 70, 111, 127, 47, 58, 14, 6, 92, 96, 49, 109, 104, 60, 105, 114, 4, 25, 44, 23, 45, 119, 24, 118, 97, 9, 30, 107, 28, 99, 89, 37, 108, 13, 67, 35, 43, 94, 22, 39, 68, 52, 34, 100, 101, 20, 54, 87, 103, 33, 66, 98, 16, 36, 42, 76, 57, 65, 12, 82, 8, 102, 64, 71, 3, 69, 2, 0, 1], [38, 110, 112, 115, 48, 46, 125, 51, 26, 31, 27, 122, 62, 88, 50, 102, 93, 21, 116, 117, 59, 56, 123, 23, 124, 126, 120, 104, 105, 106, 32, 41, 83, 53, 40, 61, 58, 86, 101, 111, 17, 29, 100, 113, 121, 47, 60, 49, 63, 95, 78, 33, 45, 119, 35, 109, 114, 16, 22, 96, 91, 74, 54, 99, 55, 84, 36, 37, 85, 30, 52, 28, 89, 94, 43, 107, 11, 42, 108, 118, 87, 44, 34, 14, 103, 18, 72, 98, 90, 80, 127, 57, 39, 97, 92, 13, 20, 7, 6, 79, 24, 25, 9, 19, 5, 76, 75, 15, 66, 68, 10, 82, 65, 77, 12, 81, 3, 73, 1, 0, 71, 67, 70, 64, 8, 4, 2, 69], [113, 121, 86, 61, 125, 49, 57, 59, 85, 116, 52, 60, 56, 122, 22, 62, 55, 117, 53, 124, 114, 63, 127, 54, 115, 119, 51, 123, 18, 58, 118, 50, 112, 120, 110, 111, 126, 109, 48, 46, 45, 47, 44, 108, 107, 43, 106, 23, 42, 105, 41, 40, 90, 104, 39, 35, 36, 38, 94, 95, 102, 103, 87, 34, 37, 15, 96, 100, 82, 97, 98, 64, 99, 1, 30, 31, 101, 21, 91, 66, 33, 16, 28, 3, 84, 25, 93, 65, 88, 12, 92, 80, 0, 77, 14, 69, 26, 78, 13, 89, 74, 4, 75, 24, 73, 71, 29, 2, 67, 20, 83, 8, 6, 19, 17, 68, 27, 79, 9, 10, 32, 11, 5, 7, 70, 76, 81, 72], [121, 113, 61, 125, 49, 22, 57, 59, 116, 52, 56, 62, 122, 55, 60, 117, 63, 114, 53, 127, 124, 119, 115, 51, 58, 54, 123, 118, 50, 112, 120, 110, 111, 126, 25, 48, 109, 46, 90, 47, 45, 44, 64, 19, 108, 107, 106, 30, 43, 66, 3, 69, 65, 96, 105, 42, 95, 38, 41, 26, 92, 40, 39, 104, 34, 27, 4, 94, 102, 1, 100, 36, 103, 71, 35, 97, 23, 98, 9, 67, 29, 87, 99, 33, 70, 86, 32, 24, 6, 15, 2, 31, 37, 0, 101, 84, 28, 18, 93, 85, 68, 13, 8, 88, 10, 80, 91, 83, 5, 77, 76, 12, 89, 82, 11, 78, 17, 16, 79, 75, 81, 7, 20, 14, 72, 21, 73, 74], [113, 121, 61, 125, 49, 57, 86, 22, 59, 116, 52, 60, 56, 122, 117, 62, 55, 53, 63, 114, 124, 54, 119, 51, 127, 123, 58, 115, 118, 120, 21, 50, 110, 112, 111, 126, 109, 46, 48, 45, 47, 44, 83, 108, 107, 106, 87, 43, 92, 42, 105, 30, 25, 41, 23, 89, 96, 40, 39, 104, 38, 90, 20, 34, 88, 94, 36, 103, 102, 64, 35, 100, 99, 18, 24, 97, 66, 26, 19, 1, 27, 93, 95, 65, 3, 98, 69, 33, 31, 37, 28, 0, 101, 77, 4, 10, 32, 11, 29, 85, 71, 76, 67, 13, 9, 2, 81, 8, 12, 6, 68, 91, 14, 82, 80, 84, 72, 15, 70, 78, 79, 5, 75, 16, 17, 73, 74, 7], [37, 61, 125, 49, 121, 57, 113, 59, 22, 96, 62, 55, 122, 32, 52, 116, 20, 60, 127, 56, 91, 115, 88, 124, 83, 117, 30, 77, 28, 54, 119, 53, 123, 110, 51, 63, 29, 114, 58, 112, 45, 92, 94, 50, 118, 9, 46, 120, 44, 111, 47, 109, 48, 79, 126, 69, 11, 71, 89, 76, 18, 26, 108, 93, 105, 43, 66, 87, 107, 33, 41, 36, 106, 81, 64, 39, 8, 3, 82, 42, 6, 104, 34, 99, 40, 10, 65, 23, 90, 4, 27, 17, 78, 74, 24, 98, 97, 103, 102, 19, 101, 100, 84, 31, 1, 25, 38, 73, 14, 80, 35, 75, 95, 12, 68, 13, 16, 7, 15, 85, 70, 67, 21, 2, 72, 86, 0, 5], [38, 44, 115, 90, 51, 22, 84, 108, 17, 72, 79, 81, 74, 18, 29, 126, 8, 12, 127, 16, 10, 78, 33, 82, 28, 19, 89, 112, 123, 120, 36, 4, 116, 5, 124, 122, 13, 125, 71, 60, 76, 98, 103, 35, 39, 110, 49, 107, 50, 91, 62, 47, 52, 15, 67, 97, 111, 80, 42, 45, 121, 73, 118, 113, 119, 114, 105, 20, 25, 109, 63, 55, 59, 53, 96, 86, 58, 23, 41, 30, 54, 87, 61, 106, 6, 46, 40, 37, 43, 101, 117, 99, 48, 26, 31, 34, 70, 14, 100, 94, 57, 95, 32, 24, 104, 68, 21, 92, 9, 77, 65, 56, 85, 27, 88, 93, 2, 75, 11, 0, 102, 83, 7, 1, 3, 69, 66, 64], [38, 44, 115, 69, 71, 51, 79, 12, 17, 22, 18, 108, 84, 74, 81, 8, 78, 90, 2, 64, 3, 33, 5, 66, 102, 7, 68, 65, 120, 126, 1, 10, 49, 15, 67, 14, 57, 112, 70, 76, 95, 59, 125, 82, 20, 52, 23, 88, 9, 73, 19, 75, 26, 83, 56, 4, 16, 0, 13, 11, 87, 127, 119, 86, 92, 29, 98, 89, 116, 114, 124, 80, 72, 37, 6, 93, 58, 25, 61, 28, 91, 36, 110, 77, 104, 99, 41, 45, 27, 117, 35, 122, 106, 63, 21, 34, 96, 30, 85, 118, 121, 31, 62, 123, 100, 94, 24, 111, 48, 50, 60, 101, 32, 42, 39, 105, 97, 46, 40, 103, 109, 47, 55, 54, 53, 43, 113, 107], [38, 44, 115, 74, 18, 84, 79, 12, 51, 108, 69, 22, 71, 3, 8, 78, 2, 64, 70, 102, 33, 1, 17, 67, 90, 68, 76, 81, 6, 7, 15, 66, 120, 16, 28, 4, 112, 23, 82, 10, 52, 59, 126, 91, 65, 86, 88, 49, 98, 26, 73, 127, 125, 72, 92, 95, 5, 57, 56, 89, 41, 85, 83, 9, 24, 118, 114, 117, 35, 0, 13, 20, 14, 58, 46, 29, 93, 19, 25, 116, 45, 75, 123, 80, 105, 50, 21, 111, 62, 11, 31, 110, 37, 124, 40, 63, 30, 32, 42, 60, 107, 119, 104, 113, 36, 47, 94, 109, 101, 77, 48, 39, 99, 122, 61, 87, 103, 53, 121, 34, 106, 55, 100, 43, 96, 97, 54, 27], [38, 44, 115, 84, 51, 22, 18, 108, 12, 17, 8, 79, 78, 74, 16, 72, 33, 19, 70, 89, 67, 5, 68, 10, 4, 81, 24, 126, 120, 90, 71, 65, 127, 2, 92, 112, 76, 49, 82, 6, 73, 69, 125, 28, 7, 75, 13, 60, 14, 3, 80, 29, 116, 23, 25, 114, 86, 98, 35, 56, 9, 50, 57, 117, 64, 107, 20, 123, 0, 58, 52, 32, 36, 124, 103, 41, 87, 93, 88, 91, 21, 26, 15, 1, 102, 39, 96, 37, 118, 43, 11, 95, 53, 83, 30, 63, 111, 100, 110, 31, 61, 62, 101, 106, 85, 40, 104, 47, 94, 113, 55, 77, 27, 109, 45, 97, 121, 59, 46, 34, 119, 99, 48, 122, 105, 54, 42, 66], [38, 44, 34, 119, 118, 53, 56, 81, 30, 78, 23, 84, 126, 47, 11, 25, 71, 122, 5, 8, 12, 52, 79, 9, 82, 87, 93, 54, 27, 29, 61, 116, 20, 50, 19, 94, 90, 125, 86, 13, 14, 66, 55, 17, 67, 7, 75, 15, 18, 127, 28, 10, 106, 16, 51, 70, 89, 101, 4, 22, 62, 46, 77, 88, 21, 121, 26, 68, 85, 69, 102, 80, 48, 73, 3, 74, 2, 96, 49, 97, 63, 105, 1, 58, 98, 42, 0, 91, 114, 24, 83, 110, 35, 59, 72, 41, 92, 113, 76, 64, 32, 117, 33, 40, 65, 99, 36, 95, 39, 43, 123, 109, 37, 31, 103, 6, 100, 108, 120, 124, 104, 115, 60, 107, 57, 112, 111, 45], [44, 53, 38, 118, 47, 34, 27, 101, 119, 126, 116, 22, 122, 54, 58, 94, 102, 30, 79, 56, 82, 46, 114, 84, 108, 59, 41, 125, 23, 106, 57, 62, 55, 93, 103, 52, 110, 63, 26, 113, 90, 50, 109, 29, 123, 61, 115, 60, 42, 124, 51, 48, 117, 31, 40, 105, 49, 121, 43, 86, 104, 127, 112, 33, 45, 9, 120, 107, 100, 80, 98, 39, 18, 37, 24, 99, 36, 92, 111, 25, 91, 73, 96, 88, 83, 28, 3, 13, 32, 35, 76, 21, 95, 85, 97, 69, 81, 72, 71, 89, 11, 20, 14, 74, 16, 75, 19, 15, 70, 7, 12, 10, 87, 6, 77, 68, 67, 0, 65, 17, 8, 2, 4, 78, 66, 64, 5, 1], [44, 38, 119, 56, 118, 53, 34, 58, 27, 84, 30, 94, 42, 79, 101, 25, 122, 125, 21, 116, 114, 82, 41, 23, 26, 50, 9, 127, 18, 62, 117, 47, 24, 54, 93, 113, 59, 52, 92, 51, 46, 105, 99, 57, 108, 55, 86, 96, 126, 48, 61, 60, 81, 33, 29, 28, 123, 103, 39, 90, 124, 31, 110, 22, 89, 120, 88, 112, 87, 12, 106, 49, 15, 91, 102, 121, 63, 109, 13, 71, 45, 104, 40, 11, 35, 115, 83, 97, 19, 85, 100, 95, 77, 75, 36, 37, 20, 16, 111, 107, 43, 32, 80, 78, 73, 98, 3, 17, 7, 70, 10, 67, 74, 76, 0, 5, 6, 14, 2, 66, 4, 64, 69, 8, 68, 72, 65, 1], [38, 53, 118, 119, 56, 34, 12, 44, 88, 23, 25, 84, 30, 47, 27, 81, 108, 6, 8, 126, 94, 1, 18, 89, 9, 86, 87, 79, 19, 33, 67, 21, 112, 54, 15, 58, 20, 52, 105, 65, 82, 41, 42, 55, 125, 46, 16, 117, 114, 101, 102, 77, 123, 116, 11, 48, 92, 122, 31, 26, 72, 62, 80, 57, 120, 90, 39, 70, 76, 115, 83, 110, 63, 61, 50, 32, 78, 97, 37, 106, 75, 85, 29, 124, 113, 45, 95, 59, 51, 5, 103, 100, 35, 2, 104, 0, 127, 17, 43, 74, 14, 22, 28, 107, 121, 64, 60, 7, 96, 109, 40, 13, 69, 111, 49, 93, 99, 36, 68, 10, 4, 91, 24, 73, 71, 66, 3, 98], [39, 51, 50, 114, 97, 115, 113, 54, 117, 85, 124, 87, 120, 121, 58, 29, 63, 27, 24, 55, 61, 122, 126, 91, 111, 75, 92, 108, 25, 32, 112, 83, 57, 33, 26, 93, 53, 22, 56, 20, 59, 116, 79, 82, 73, 18, 17, 60, 88, 23, 49, 69, 21, 81, 14, 99, 48, 119, 77, 95, 34, 90, 80, 109, 16, 38, 40, 72, 41, 28, 30, 127, 107, 84, 7, 94, 78, 71, 52, 123, 15, 43, 76, 89, 100, 118, 110, 45, 125, 47, 44, 42, 86, 96, 104, 36, 98, 37, 106, 46, 102, 101, 31, 35, 105, 9, 12, 62, 11, 13, 74, 3, 5, 67, 70, 19, 8, 6, 66, 2, 4, 1, 0, 68, 10, 103, 64, 65], [51, 39, 114, 113, 50, 121, 97, 116, 61, 53, 59, 124, 60, 54, 122, 29, 120, 118, 87, 52, 105, 45, 125, 24, 56, 115, 112, 86, 63, 123, 103, 58, 26, 88, 27, 119, 109, 83, 25, 110, 107, 95, 62, 46, 85, 117, 40, 111, 92, 55, 33, 57, 44, 18, 82, 43, 127, 94, 126, 49, 98, 108, 47, 93, 106, 20, 42, 90, 41, 34, 35, 75, 23, 102, 101, 96, 38, 48, 15, 100, 89, 36, 14, 17, 37, 99, 104, 91, 19, 80, 73, 31, 32, 77, 28, 84, 12, 21, 22, 81, 30, 16, 72, 74, 70, 69, 78, 11, 0, 13, 68, 67, 64, 66, 79, 76, 1, 3, 71, 9, 6, 2, 5, 65, 8, 7, 10, 4], [39, 114, 51, 50, 97, 121, 122, 11, 9, 21, 87, 29, 36, 116, 83, 34, 54, 12, 30, 98, 14, 92, 6, 48, 19, 109, 110, 8, 119, 10, 62, 94, 53, 5, 113, 126, 35, 63, 15, 42, 13, 28, 55, 71, 47, 59, 67, 127, 57, 76, 111, 108, 58, 117, 70, 101, 43, 2, 16, 91, 79, 118, 93, 115, 68, 40, 125, 100, 4, 120, 52, 32, 60, 45, 106, 73, 102, 74, 112, 123, 56, 23, 105, 41, 46, 49, 95, 38, 89, 72, 96, 81, 44, 65, 90, 104, 99, 37, 22, 61, 107, 82, 0, 84, 18, 124, 86, 26, 85, 20, 75, 27, 78, 31, 17, 25, 88, 103, 33, 24, 80, 77, 1, 7, 69, 3, 64, 66], [39, 51, 114, 50, 97, 113, 54, 122, 121, 85, 24, 61, 87, 63, 92, 29, 48, 116, 124, 83, 95, 25, 117, 57, 27, 34, 26, 80, 126, 40, 82, 28, 111, 120, 20, 119, 33, 53, 110, 58, 91, 60, 77, 14, 17, 75, 108, 32, 15, 55, 47, 56, 73, 42, 21, 115, 89, 59, 23, 62, 88, 125, 107, 127, 19, 36, 98, 86, 74, 104, 49, 94, 81, 90, 52, 118, 18, 41, 38, 78, 101, 79, 105, 106, 22, 84, 72, 30, 43, 123, 35, 112, 13, 12, 44, 37, 76, 11, 102, 109, 93, 100, 70, 45, 99, 69, 16, 96, 31, 46, 67, 7, 68, 71, 9, 0, 10, 8, 1, 6, 2, 65, 3, 4, 5, 66, 64, 103], [108, 124, 102, 122, 62, 34, 107, 23, 30, 60, 50, 35, 119, 104, 25, 120, 45, 31, 123, 114, 121, 49, 117, 61, 103, 87, 40, 106, 47, 29, 55, 44, 110, 22, 20, 28, 32, 86, 118, 52, 38, 58, 59, 41, 33, 63, 53, 92, 115, 113, 48, 88, 43, 54, 126, 96, 105, 89, 46, 99, 56, 26, 57, 111, 51, 42, 91, 116, 112, 125, 127, 109, 101, 97, 94, 37, 79, 100, 77, 21, 39, 93, 84, 95, 27, 36, 90, 24, 11, 16, 17, 13, 83, 69, 81, 18, 72, 19, 75, 8, 15, 98, 78, 5, 10, 14, 74, 3, 65, 7, 64, 0, 67, 66, 68, 85, 1, 71, 4, 12, 80, 82, 2, 73, 9, 6, 76, 70], [102, 124, 34, 108, 62, 122, 107, 23, 17, 93, 25, 21, 11, 79, 104, 77, 18, 16, 120, 30, 69, 15, 114, 96, 29, 22, 32, 10, 83, 19, 85, 126, 89, 54, 5, 20, 117, 26, 49, 40, 31, 50, 92, 87, 127, 123, 110, 73, 2, 28, 90, 86, 119, 60, 1, 81, 51, 118, 103, 24, 91, 43, 109, 35, 41, 27, 47, 44, 84, 45, 38, 36, 116, 8, 57, 74, 78, 37, 125, 100, 13, 82, 112, 63, 94, 71, 68, 53, 67, 115, 97, 106, 0, 88, 75, 14, 61, 99, 55, 101, 105, 64, 111, 58, 42, 95, 33, 39, 46, 113, 121, 56, 72, 59, 52, 65, 4, 3, 7, 80, 48, 66, 12, 6, 9, 76, 98, 70], [102, 62, 122, 34, 124, 21, 93, 18, 108, 16, 40, 23, 12, 29, 83, 90, 77, 25, 73, 85, 11, 92, 61, 76, 126, 96, 100, 8, 24, 6, 26, 44, 27, 28, 116, 49, 82, 70, 107, 22, 119, 20, 32, 120, 84, 112, 78, 5, 42, 19, 57, 53, 58, 7, 46, 71, 17, 48, 36, 79, 106, 69, 80, 95, 123, 127, 67, 88, 4, 59, 114, 75, 109, 65, 103, 2, 43, 33, 63, 50, 0, 104, 13, 99, 9, 68, 86, 110, 14, 15, 45, 97, 66, 47, 41, 113, 117, 87, 105, 10, 89, 3, 55, 72, 37, 54, 98, 51, 91, 52, 111, 1, 64, 74, 30, 94, 81, 56, 31, 121, 115, 101, 118, 35, 60, 38, 39, 125], [62, 102, 122, 108, 34, 124, 26, 120, 23, 51, 35, 123, 30, 90, 40, 61, 21, 95, 24, 29, 96, 75, 110, 48, 93, 60, 16, 54, 5, 127, 37, 45, 114, 22, 20, 39, 50, 72, 31, 18, 94, 58, 79, 107, 49, 104, 8, 36, 32, 89, 118, 69, 44, 77, 117, 86, 59, 112, 125, 25, 103, 116, 11, 57, 106, 121, 111, 87, 56, 92, 52, 97, 55, 126, 53, 83, 76, 27, 119, 101, 91, 47, 46, 28, 41, 84, 113, 63, 109, 33, 100, 88, 74, 115, 82, 105, 3, 17, 15, 42, 98, 85, 13, 81, 73, 99, 10, 43, 38, 70, 14, 12, 71, 19, 80, 9, 0, 78, 2, 1, 68, 7, 4, 66, 67, 65, 64, 6], [38, 97, 53, 117, 21, 81, 80, 75, 14, 71, 4, 87, 76, 9, 6, 66, 13, 1, 83, 24, 123, 70, 74, 68, 118, 2, 85, 19, 12, 16, 90, 72, 0, 65, 10, 73, 94, 113, 11, 46, 18, 17, 88, 3, 92, 112, 79, 106, 64, 103, 124, 119, 86, 77, 84, 82, 25, 122, 78, 69, 7, 55, 93, 27, 111, 23, 5, 115, 42, 96, 91, 89, 15, 51, 28, 41, 62, 29, 35, 30, 120, 39, 109, 22, 56, 57, 98, 26, 54, 61, 99, 121, 50, 47, 67, 48, 36, 100, 31, 59, 52, 107, 45, 125, 20, 108, 40, 58, 104, 60, 110, 101, 44, 95, 116, 127, 114, 49, 105, 43, 32, 34, 37, 63, 126, 8, 33, 102], [38, 53, 97, 117, 87, 80, 21, 75, 76, 81, 6, 14, 70, 74, 4, 71, 9, 83, 68, 123, 24, 13, 118, 2, 1, 113, 19, 94, 66, 46, 72, 73, 55, 11, 103, 92, 119, 106, 17, 85, 16, 89, 93, 23, 12, 45, 0, 88, 96, 122, 79, 77, 112, 10, 78, 29, 20, 42, 51, 91, 111, 82, 69, 64, 84, 62, 52, 41, 86, 124, 35, 15, 28, 33, 8, 65, 7, 25, 40, 60, 18, 26, 3, 36, 105, 104, 50, 90, 120, 27, 48, 121, 107, 101, 109, 30, 116, 47, 56, 5, 108, 54, 110, 63, 99, 95, 57, 125, 100, 115, 39, 59, 98, 127, 58, 67, 114, 61, 34, 31, 22, 49, 44, 37, 43, 32, 126, 102], [38, 53, 97, 117, 21, 87, 81, 76, 75, 80, 123, 14, 4, 66, 70, 6, 118, 9, 74, 24, 19, 90, 12, 7, 69, 16, 46, 92, 71, 51, 65, 82, 112, 78, 64, 103, 56, 84, 30, 106, 23, 122, 25, 110, 15, 11, 98, 119, 121, 94, 17, 113, 47, 3, 85, 83, 1, 26, 31, 125, 35, 41, 39, 111, 88, 57, 91, 115, 73, 60, 49, 42, 93, 86, 52, 37, 18, 54, 27, 50, 29, 107, 34, 124, 58, 101, 22, 44, 59, 13, 10, 20, 126, 77, 61, 96, 36, 109, 120, 105, 79, 89, 40, 116, 63, 55, 99, 95, 28, 43, 127, 108, 104, 62, 48, 2, 45, 114, 100, 32, 67, 0, 5, 72, 8, 102, 68, 33], [38, 53, 97, 117, 87, 80, 6, 76, 21, 81, 14, 75, 74, 9, 1, 19, 3, 113, 68, 8, 71, 123, 7, 5, 82, 118, 2, 72, 85, 65, 46, 18, 70, 24, 23, 122, 91, 55, 41, 83, 35, 119, 78, 79, 10, 88, 67, 105, 12, 40, 112, 33, 106, 94, 111, 28, 15, 16, 66, 69, 84, 109, 27, 64, 45, 107, 36, 50, 86, 124, 61, 13, 90, 121, 103, 93, 26, 56, 98, 120, 77, 47, 30, 60, 115, 114, 116, 96, 25, 92, 59, 95, 42, 51, 73, 20, 104, 48, 58, 89, 0, 126, 29, 52, 17, 108, 57, 44, 110, 49, 125, 37, 22, 99, 34, 43, 62, 31, 54, 63, 127, 11, 101, 32, 39, 100, 102, 4]], "model.layers.5.self_attn.k_proj": [[42, 103, 50, 34, 12, 9, 80, 23, 20, 14, 18, 114, 7, 51, 66, 111, 6, 4, 74, 52, 64, 113, 79, 49, 126, 54, 8, 61, 127, 57, 67, 112, 116, 92, 27, 65, 108, 73, 0, 88, 121, 58, 1, 21, 124, 5, 77, 120, 59, 28, 125, 26, 86, 123, 69, 46, 62, 98, 106, 38, 81, 60, 94, 105, 91, 99, 55, 47, 44, 85, 93, 22, 45, 71, 24, 70, 41, 102, 122, 109, 37, 48, 117, 119, 90, 40, 63, 118, 11, 97, 53, 43, 25, 17, 95, 115, 104, 89, 56, 32, 3, 96, 110, 30, 101, 19, 100, 35, 107, 75, 16, 72, 29, 33, 83, 13, 31, 36, 10, 84, 15, 76, 82, 87, 78, 2, 68, 39], [46, 102, 115, 112, 125, 93, 126, 23, 26, 83, 9, 79, 80, 21, 76, 81, 20, 6, 68, 24, 3, 12, 122, 15, 13, 71, 124, 50, 77, 8, 41, 66, 116, 5, 106, 91, 123, 117, 113, 31, 16, 56, 55, 62, 58, 82, 69, 53, 78, 105, 63, 104, 95, 127, 74, 108, 111, 44, 45, 109, 118, 0, 47, 51, 96, 121, 107, 52, 40, 29, 59, 48, 60, 119, 114, 75, 33, 1, 120, 72, 49, 61, 43, 37, 19, 11, 54, 42, 36, 17, 10, 22, 18, 85, 39, 35, 110, 101, 34, 97, 99, 57, 32, 100, 88, 103, 98, 65, 84, 30, 94, 14, 25, 86, 27, 89, 28, 7, 92, 73, 87, 90, 2, 70, 38, 64, 67, 4], [61, 121, 125, 101, 57, 49, 113, 22, 32, 127, 62, 115, 55, 56, 122, 117, 17, 123, 52, 84, 51, 60, 58, 99, 59, 124, 63, 119, 54, 120, 116, 93, 114, 126, 50, 112, 88, 118, 53, 30, 48, 91, 110, 109, 47, 111, 46, 45, 43, 44, 92, 24, 95, 14, 12, 102, 35, 108, 79, 42, 82, 18, 107, 39, 10, 106, 83, 40, 41, 77, 100, 38, 37, 94, 21, 20, 97, 105, 16, 87, 104, 103, 11, 85, 36, 78, 72, 89, 80, 25, 98, 9, 33, 96, 29, 34, 31, 75, 86, 15, 90, 71, 13, 28, 74, 70, 19, 4, 23, 8, 26, 81, 66, 64, 27, 69, 3, 65, 73, 5, 6, 7, 1, 0, 76, 67, 68, 2], [108, 115, 102, 12, 22, 78, 84, 74, 18, 17, 79, 44, 69, 64, 71, 8, 68, 66, 90, 3, 75, 97, 120, 73, 2, 65, 48, 116, 1, 6, 16, 126, 24, 49, 70, 29, 67, 125, 113, 51, 93, 57, 59, 114, 19, 89, 88, 9, 21, 117, 11, 85, 124, 14, 23, 92, 58, 7, 91, 83, 0, 80, 25, 81, 62, 107, 39, 118, 30, 63, 77, 27, 55, 87, 95, 53, 10, 36, 127, 56, 99, 26, 32, 34, 31, 4, 119, 112, 41, 105, 28, 103, 43, 94, 60, 45, 52, 61, 13, 121, 104, 106, 40, 37, 100, 42, 54, 96, 123, 122, 82, 98, 110, 47, 35, 46, 101, 111, 109, 50, 33, 76, 86, 72, 38, 20, 15, 5], [102, 119, 118, 56, 53, 98, 94, 27, 108, 23, 84, 81, 25, 79, 64, 12, 9, 66, 117, 58, 5, 78, 57, 54, 37, 109, 125, 42, 67, 113, 8, 82, 114, 104, 55, 52, 105, 11, 39, 126, 24, 115, 85, 71, 62, 124, 123, 46, 110, 45, 50, 61, 19, 122, 48, 63, 121, 38, 1, 107, 6, 13, 22, 127, 112, 120, 41, 80, 4, 51, 43, 49, 60, 35, 59, 47, 74, 33, 111, 40, 18, 69, 65, 99, 90, 103, 31, 10, 116, 3, 29, 36, 97, 83, 21, 44, 68, 77, 100, 106, 96, 92, 70, 86, 32, 75, 93, 0, 30, 91, 7, 95, 88, 87, 16, 28, 14, 26, 89, 76, 72, 2, 101, 17, 20, 15, 34, 73], [103, 114, 51, 33, 31, 87, 85, 79, 49, 115, 83, 77, 27, 46, 17, 24, 7, 122, 76, 78, 25, 18, 108, 29, 4, 10, 84, 98, 121, 75, 57, 6, 3, 8, 74, 86, 30, 62, 82, 55, 48, 42, 65, 113, 69, 54, 119, 111, 5, 110, 99, 109, 127, 64, 101, 45, 89, 104, 63, 9, 112, 126, 56, 91, 123, 2, 38, 93, 58, 47, 92, 100, 52, 44, 13, 66, 125, 43, 118, 72, 94, 73, 41, 102, 40, 61, 35, 106, 23, 50, 96, 105, 116, 117, 26, 59, 12, 32, 81, 120, 36, 53, 67, 15, 22, 97, 60, 37, 107, 21, 95, 34, 88, 124, 28, 16, 80, 19, 14, 20, 90, 71, 70, 11, 39, 1, 0, 68], [38, 122, 62, 29, 98, 124, 44, 16, 18, 104, 78, 21, 22, 20, 25, 79, 73, 12, 85, 19, 11, 71, 23, 24, 6, 32, 108, 90, 77, 66, 14, 81, 82, 43, 64, 92, 17, 126, 26, 46, 68, 58, 30, 48, 40, 8, 27, 9, 117, 116, 49, 65, 57, 109, 114, 119, 102, 76, 34, 61, 60, 7, 74, 51, 103, 113, 41, 110, 91, 121, 55, 93, 101, 56, 39, 63, 10, 36, 33, 50, 59, 97, 28, 4, 105, 112, 45, 15, 99, 106, 13, 100, 125, 95, 87, 84, 75, 42, 52, 54, 96, 31, 86, 3, 53, 83, 118, 88, 120, 111, 5, 72, 115, 107, 94, 47, 123, 37, 127, 80, 35, 0, 67, 89, 1, 69, 2, 70], [117, 53, 102, 33, 14, 21, 80, 87, 81, 74, 76, 0, 75, 9, 68, 6, 8, 71, 2, 83, 65, 3, 5, 55, 24, 78, 7, 82, 1, 10, 73, 67, 42, 77, 38, 118, 123, 19, 113, 27, 84, 111, 66, 72, 110, 11, 69, 62, 91, 70, 105, 86, 94, 93, 90, 35, 26, 25, 89, 103, 112, 88, 79, 92, 15, 115, 46, 124, 57, 20, 119, 122, 48, 120, 54, 31, 59, 41, 106, 56, 17, 44, 60, 4, 30, 29, 116, 61, 37, 18, 43, 22, 114, 85, 49, 98, 127, 104, 126, 40, 100, 47, 125, 45, 51, 32, 109, 12, 28, 95, 52, 96, 58, 34, 99, 107, 36, 121, 16, 63, 108, 50, 13, 101, 39, 23, 64, 97]], "model.layers.5.self_attn.qk_proj": [[53, 115, 117, 108, 51, 125, 122, 124, 114, 62, 38, 61, 119, 112, 118, 44, 102, 50, 121, 42, 46, 56, 113, 126, 49, 23, 87, 57, 103, 76, 84, 85, 78, 82, 21, 17, 12, 81, 20, 18, 86, 80, 93, 14, 16, 110, 55, 91, 116, 10, 29, 22, 127, 74, 98, 97, 120, 15, 111, 79, 54, 26, 48, 106, 59, 63, 30, 52, 60, 9, 7, 73, 11, 47, 39, 34, 123, 8, 90, 71, 104, 75, 89, 45, 83, 24, 72, 27, 19, 5, 33, 28, 88, 109, 6, 58, 107, 70, 25, 13, 69, 77, 105, 68, 94, 0, 4, 32, 40, 35, 3, 43, 67, 37, 66, 31, 41, 64, 101, 2, 95, 1, 99, 92, 36, 65, 96, 100], [53, 115, 117, 108, 51, 38, 122, 125, 124, 62, 61, 114, 112, 118, 44, 102, 119, 46, 121, 50, 42, 113, 49, 56, 126, 23, 87, 103, 57, 82, 76, 48, 84, 86, 21, 85, 93, 81, 20, 12, 116, 78, 29, 18, 80, 17, 97, 127, 14, 91, 120, 16, 22, 55, 60, 15, 98, 110, 59, 106, 10, 30, 54, 47, 26, 63, 74, 34, 39, 58, 111, 79, 90, 123, 9, 8, 73, 6, 52, 75, 33, 11, 89, 7, 107, 71, 27, 109, 24, 28, 45, 83, 104, 5, 19, 88, 105, 25, 40, 64, 94, 72, 4, 3, 68, 69, 0, 37, 13, 35, 77, 95, 41, 2, 66, 32, 96, 70, 67, 1, 92, 43, 65, 99, 31, 101, 100, 36], [53, 115, 117, 108, 51, 124, 122, 125, 62, 38, 114, 61, 44, 112, 121, 102, 118, 46, 119, 42, 50, 56, 113, 49, 23, 103, 87, 126, 82, 57, 76, 21, 84, 12, 85, 93, 20, 86, 78, 81, 14, 116, 29, 18, 17, 16, 80, 22, 97, 98, 48, 8, 91, 120, 60, 15, 10, 110, 79, 74, 106, 26, 30, 39, 127, 55, 54, 6, 90, 9, 63, 71, 11, 7, 34, 47, 59, 75, 52, 109, 73, 123, 27, 89, 88, 19, 33, 28, 5, 58, 24, 107, 111, 104, 0, 83, 77, 64, 69, 40, 68, 25, 45, 67, 4, 13, 41, 94, 66, 105, 2, 3, 37, 35, 72, 32, 1, 43, 92, 95, 70, 65, 101, 31, 99, 96, 36, 100], [53, 115, 117, 108, 51, 114, 38, 122, 124, 125, 62, 61, 118, 102, 44, 46, 121, 112, 50, 119, 42, 113, 56, 49, 126, 103, 23, 87, 21, 85, 84, 86, 82, 17, 76, 57, 14, 20, 18, 93, 12, 80, 81, 78, 16, 22, 29, 98, 79, 91, 116, 97, 127, 55, 8, 30, 26, 54, 39, 48, 34, 106, 58, 63, 10, 59, 15, 110, 120, 90, 52, 123, 74, 47, 9, 6, 60, 75, 89, 71, 83, 73, 11, 7, 64, 24, 69, 28, 33, 88, 27, 104, 111, 19, 0, 77, 109, 25, 13, 107, 45, 5, 67, 4, 72, 3, 105, 94, 40, 66, 65, 37, 68, 1, 92, 2, 41, 96, 70, 32, 95, 31, 101, 35, 43, 99, 100, 36], [53, 115, 117, 108, 51, 122, 114, 124, 62, 125, 61, 38, 118, 121, 46, 102, 44, 112, 119, 50, 42, 113, 56, 49, 23, 103, 126, 87, 57, 82, 76, 12, 78, 85, 21, 20, 17, 86, 16, 84, 93, 81, 14, 18, 120, 98, 8, 80, 29, 22, 34, 52, 10, 110, 79, 106, 116, 15, 58, 39, 91, 123, 74, 63, 127, 6, 55, 48, 59, 7, 9, 71, 30, 54, 11, 75, 90, 111, 97, 64, 47, 109, 73, 33, 60, 26, 0, 5, 89, 83, 69, 104, 19, 27, 68, 88, 4, 3, 45, 77, 13, 67, 28, 24, 1, 40, 2, 107, 72, 25, 94, 41, 70, 66, 37, 35, 65, 32, 43, 31, 105, 99, 92, 95, 101, 36, 100, 96], [53, 115, 117, 51, 122, 124, 108, 125, 114, 61, 62, 38, 118, 119, 112, 121, 50, 102, 44, 46, 42, 56, 113, 49, 126, 87, 103, 57, 23, 76, 12, 18, 21, 78, 20, 84, 86, 85, 82, 17, 81, 93, 16, 14, 80, 55, 48, 110, 22, 120, 98, 29, 91, 74, 10, 15, 34, 47, 8, 54, 30, 59, 52, 127, 79, 123, 58, 97, 106, 116, 60, 63, 7, 9, 39, 11, 73, 71, 90, 75, 26, 111, 104, 6, 83, 33, 89, 5, 88, 68, 109, 24, 4, 45, 27, 19, 69, 72, 28, 94, 2, 70, 13, 25, 0, 64, 77, 40, 107, 3, 67, 105, 43, 41, 31, 1, 95, 32, 35, 37, 66, 101, 65, 96, 92, 100, 99, 36], [53, 115, 117, 51, 124, 125, 108, 122, 114, 38, 62, 119, 118, 61, 44, 112, 102, 121, 46, 50, 49, 42, 56, 113, 126, 87, 21, 23, 57, 103, 85, 76, 17, 82, 12, 78, 14, 110, 18, 20, 84, 81, 16, 93, 86, 80, 106, 54, 120, 48, 22, 29, 55, 60, 98, 79, 97, 127, 59, 11, 10, 111, 91, 63, 74, 39, 15, 30, 58, 34, 47, 116, 45, 8, 52, 9, 90, 123, 7, 26, 73, 88, 71, 75, 33, 19, 83, 89, 104, 27, 24, 70, 109, 40, 28, 94, 25, 41, 69, 43, 68, 6, 32, 5, 107, 72, 77, 4, 13, 3, 37, 35, 67, 105, 2, 95, 64, 31, 92, 0, 66, 101, 96, 65, 1, 36, 99, 100], [53, 115, 117, 51, 108, 125, 122, 114, 124, 62, 38, 61, 118, 119, 112, 102, 44, 46, 121, 42, 50, 49, 56, 113, 23, 87, 21, 126, 57, 103, 12, 20, 84, 85, 80, 86, 76, 17, 82, 14, 78, 18, 81, 93, 16, 54, 29, 127, 52, 79, 97, 106, 91, 15, 111, 120, 22, 110, 58, 116, 98, 55, 10, 48, 74, 39, 60, 70, 30, 34, 47, 9, 11, 63, 88, 26, 104, 45, 27, 33, 90, 75, 19, 7, 123, 8, 71, 73, 83, 24, 109, 89, 59, 28, 43, 25, 107, 5, 37, 41, 40, 72, 69, 77, 32, 13, 94, 66, 0, 35, 4, 67, 64, 3, 105, 68, 2, 92, 31, 65, 101, 1, 6, 99, 95, 36, 100, 96], [53, 115, 117, 108, 51, 122, 38, 125, 124, 62, 114, 61, 44, 118, 112, 102, 50, 119, 121, 46, 42, 56, 126, 113, 49, 23, 87, 103, 21, 57, 20, 17, 84, 85, 14, 18, 81, 76, 12, 86, 82, 93, 80, 16, 29, 22, 127, 78, 54, 120, 110, 98, 116, 55, 97, 106, 91, 48, 10, 34, 30, 111, 79, 15, 26, 70, 52, 47, 74, 39, 90, 123, 58, 59, 33, 27, 11, 75, 24, 104, 25, 88, 71, 7, 89, 72, 9, 83, 73, 8, 19, 60, 5, 45, 94, 109, 105, 63, 32, 28, 35, 13, 0, 69, 4, 64, 41, 40, 66, 77, 67, 68, 37, 107, 43, 95, 3, 31, 92, 101, 99, 2, 1, 96, 6, 65, 36, 100], [53, 115, 117, 108, 51, 122, 38, 114, 125, 62, 124, 61, 44, 118, 112, 119, 121, 102, 46, 42, 50, 113, 56, 49, 87, 23, 103, 85, 82, 17, 76, 12, 126, 21, 57, 86, 93, 18, 84, 20, 81, 14, 78, 80, 16, 29, 127, 22, 48, 15, 97, 55, 98, 79, 91, 110, 59, 52, 74, 54, 26, 30, 10, 106, 39, 60, 90, 120, 34, 47, 111, 109, 70, 9, 58, 11, 72, 89, 75, 71, 33, 116, 63, 7, 73, 83, 27, 88, 24, 104, 123, 19, 77, 94, 69, 45, 25, 105, 4, 68, 28, 40, 13, 8, 5, 43, 37, 3, 66, 0, 107, 32, 2, 35, 6, 92, 95, 67, 31, 96, 36, 64, 41, 1, 101, 100, 99, 65], [53, 115, 117, 108, 122, 125, 51, 114, 62, 38, 124, 61, 102, 118, 44, 121, 50, 112, 119, 42, 46, 56, 113, 49, 126, 23, 87, 103, 17, 18, 12, 76, 85, 78, 57, 93, 20, 21, 81, 84, 82, 14, 86, 74, 80, 110, 16, 98, 29, 22, 79, 15, 72, 91, 48, 97, 10, 54, 116, 58, 52, 30, 120, 34, 111, 47, 55, 106, 39, 127, 90, 109, 9, 11, 73, 26, 7, 75, 70, 33, 60, 63, 27, 71, 83, 59, 69, 88, 89, 5, 107, 104, 64, 4, 6, 68, 77, 0, 3, 13, 19, 123, 8, 28, 2, 65, 45, 24, 31, 94, 66, 105, 37, 25, 67, 1, 40, 95, 41, 32, 101, 43, 35, 96, 92, 100, 99, 36], [53, 115, 117, 51, 108, 122, 125, 114, 62, 124, 38, 61, 121, 102, 118, 50, 44, 112, 46, 119, 42, 56, 113, 49, 126, 87, 23, 103, 57, 76, 84, 21, 17, 85, 18, 20, 12, 14, 78, 86, 82, 93, 58, 48, 80, 81, 72, 16, 29, 98, 22, 91, 74, 34, 116, 79, 54, 30, 10, 55, 52, 120, 106, 63, 59, 15, 127, 97, 9, 111, 110, 11, 39, 109, 47, 90, 7, 89, 75, 104, 6, 71, 107, 26, 123, 68, 27, 73, 45, 70, 5, 0, 33, 19, 28, 88, 69, 60, 24, 83, 8, 64, 67, 4, 37, 77, 3, 13, 25, 66, 94, 105, 40, 1, 2, 43, 65, 35, 41, 32, 31, 95, 92, 101, 99, 100, 36, 96], [53, 115, 117, 108, 51, 122, 62, 125, 114, 38, 124, 61, 112, 44, 119, 118, 102, 46, 50, 121, 42, 49, 113, 56, 126, 87, 23, 103, 57, 84, 76, 21, 20, 85, 81, 18, 17, 86, 93, 82, 12, 16, 78, 14, 80, 58, 48, 29, 22, 59, 15, 52, 127, 54, 91, 63, 97, 106, 110, 98, 120, 111, 26, 34, 10, 39, 9, 55, 116, 79, 74, 30, 72, 47, 90, 7, 109, 6, 19, 11, 33, 75, 27, 71, 24, 89, 73, 88, 83, 104, 123, 28, 107, 25, 60, 40, 68, 5, 45, 94, 77, 4, 43, 37, 13, 32, 41, 69, 3, 35, 105, 67, 101, 66, 95, 70, 2, 0, 31, 64, 92, 36, 8, 65, 99, 100, 96, 1], [53, 115, 117, 108, 51, 122, 125, 38, 114, 62, 124, 61, 112, 118, 44, 119, 102, 121, 46, 42, 50, 49, 56, 113, 126, 23, 103, 87, 21, 76, 78, 20, 18, 12, 57, 85, 84, 17, 48, 82, 81, 80, 86, 55, 93, 110, 14, 22, 16, 54, 6, 127, 72, 79, 29, 91, 10, 15, 58, 26, 98, 74, 30, 75, 116, 9, 39, 106, 34, 123, 120, 7, 97, 90, 71, 59, 111, 52, 11, 89, 88, 63, 73, 19, 47, 60, 33, 24, 104, 83, 68, 27, 40, 28, 5, 25, 77, 69, 0, 4, 94, 105, 2, 64, 37, 66, 8, 43, 109, 45, 100, 107, 13, 32, 3, 92, 41, 101, 35, 67, 95, 36, 65, 1, 31, 96, 99, 70], [53, 115, 117, 108, 51, 122, 125, 62, 38, 124, 61, 114, 102, 118, 44, 121, 112, 119, 46, 42, 50, 56, 126, 49, 87, 23, 113, 103, 84, 57, 81, 48, 85, 18, 76, 20, 21, 12, 78, 93, 82, 17, 110, 80, 14, 22, 98, 16, 86, 10, 29, 91, 79, 6, 72, 34, 63, 15, 74, 120, 39, 55, 97, 106, 54, 30, 116, 90, 127, 71, 104, 26, 58, 7, 52, 60, 89, 27, 59, 109, 11, 75, 9, 73, 33, 123, 19, 64, 88, 83, 24, 47, 111, 37, 5, 8, 69, 68, 107, 28, 94, 25, 40, 13, 66, 77, 4, 45, 2, 0, 105, 3, 32, 43, 67, 1, 92, 41, 70, 65, 35, 96, 101, 100, 31, 95, 99, 36], [53, 115, 117, 108, 51, 125, 122, 62, 124, 114, 38, 61, 44, 118, 112, 102, 121, 119, 50, 46, 42, 113, 56, 126, 49, 87, 23, 103, 57, 20, 14, 82, 85, 21, 12, 84, 76, 17, 18, 81, 86, 63, 80, 48, 78, 93, 22, 91, 16, 98, 110, 58, 34, 15, 74, 10, 29, 106, 60, 116, 127, 79, 54, 120, 6, 55, 59, 52, 39, 30, 72, 97, 11, 104, 90, 8, 7, 9, 89, 33, 47, 73, 26, 19, 111, 109, 107, 37, 75, 71, 69, 5, 83, 28, 123, 24, 43, 88, 25, 27, 64, 67, 13, 0, 68, 4, 70, 3, 77, 40, 94, 35, 45, 105, 92, 41, 65, 31, 100, 32, 95, 2, 101, 1, 66, 36, 96, 99], [53, 115, 117, 108, 51, 125, 124, 114, 122, 38, 62, 61, 112, 119, 118, 44, 121, 46, 102, 42, 50, 49, 113, 56, 126, 23, 87, 103, 84, 85, 81, 20, 14, 21, 12, 76, 18, 82, 80, 93, 57, 86, 78, 17, 15, 22, 16, 97, 10, 116, 52, 110, 29, 98, 48, 59, 54, 55, 60, 127, 91, 106, 58, 79, 30, 9, 120, 74, 7, 90, 8, 63, 26, 33, 73, 11, 71, 34, 39, 19, 24, 111, 75, 83, 89, 109, 72, 27, 28, 88, 104, 107, 47, 105, 6, 68, 123, 5, 25, 13, 40, 77, 70, 69, 4, 37, 32, 94, 43, 45, 92, 0, 64, 2, 31, 66, 35, 101, 41, 3, 67, 95, 65, 96, 100, 99, 36, 1], [53, 115, 117, 108, 51, 122, 62, 125, 38, 124, 61, 114, 44, 118, 112, 121, 102, 119, 50, 42, 46, 23, 49, 56, 113, 126, 87, 103, 84, 21, 12, 20, 82, 81, 57, 76, 14, 78, 85, 93, 18, 86, 17, 80, 16, 48, 110, 29, 97, 22, 91, 15, 106, 98, 10, 26, 8, 79, 116, 120, 74, 59, 55, 39, 30, 90, 58, 127, 54, 34, 9, 60, 47, 71, 11, 75, 52, 70, 33, 89, 7, 27, 73, 19, 28, 123, 111, 88, 5, 63, 24, 105, 83, 25, 43, 45, 104, 77, 72, 94, 69, 107, 13, 4, 40, 64, 109, 68, 32, 6, 67, 92, 66, 37, 0, 2, 95, 96, 41, 3, 101, 31, 99, 65, 35, 36, 100, 1], [53, 115, 117, 108, 51, 125, 38, 122, 62, 114, 124, 61, 118, 44, 102, 112, 121, 119, 50, 42, 46, 113, 56, 49, 23, 126, 103, 87, 20, 21, 12, 84, 14, 76, 85, 93, 82, 18, 16, 78, 110, 80, 17, 81, 22, 57, 48, 86, 29, 98, 39, 8, 55, 58, 52, 74, 34, 79, 106, 10, 70, 63, 97, 54, 91, 15, 30, 26, 90, 127, 116, 120, 109, 9, 89, 59, 60, 11, 75, 47, 71, 104, 7, 73, 33, 111, 27, 28, 19, 83, 24, 69, 0, 88, 64, 5, 25, 68, 123, 40, 94, 67, 66, 3, 65, 32, 45, 77, 13, 37, 43, 4, 41, 31, 2, 92, 107, 105, 6, 72, 95, 100, 1, 36, 35, 99, 96, 101], [53, 115, 117, 51, 108, 125, 122, 114, 38, 124, 62, 61, 121, 118, 44, 119, 112, 102, 46, 50, 113, 42, 49, 56, 87, 23, 126, 21, 103, 84, 57, 12, 76, 18, 82, 16, 20, 81, 86, 14, 78, 17, 85, 80, 93, 22, 48, 8, 29, 91, 10, 79, 98, 74, 15, 70, 47, 39, 55, 54, 109, 110, 63, 97, 9, 11, 59, 30, 116, 58, 127, 106, 34, 90, 60, 7, 73, 71, 111, 120, 89, 75, 52, 26, 83, 107, 33, 27, 104, 24, 4, 5, 19, 69, 77, 28, 68, 88, 123, 25, 94, 45, 43, 40, 13, 0, 105, 64, 72, 2, 37, 32, 67, 66, 3, 35, 99, 41, 92, 31, 65, 1, 95, 6, 36, 100, 96, 101], [53, 115, 117, 51, 108, 114, 125, 62, 122, 38, 61, 124, 44, 102, 119, 121, 118, 50, 112, 46, 42, 113, 126, 56, 49, 87, 103, 23, 82, 84, 21, 76, 78, 12, 86, 57, 93, 85, 20, 17, 18, 14, 81, 16, 110, 80, 8, 91, 97, 120, 22, 29, 54, 116, 15, 98, 30, 79, 63, 60, 48, 74, 10, 106, 34, 39, 58, 55, 70, 90, 127, 47, 52, 9, 11, 71, 26, 73, 107, 7, 111, 28, 75, 33, 19, 24, 59, 89, 104, 69, 27, 88, 83, 123, 109, 25, 37, 5, 77, 105, 68, 43, 40, 94, 4, 13, 64, 3, 72, 0, 35, 45, 31, 32, 66, 65, 67, 92, 95, 101, 6, 41, 2, 99, 96, 36, 1, 100], [53, 115, 117, 51, 108, 114, 125, 62, 122, 38, 124, 61, 44, 118, 121, 102, 112, 119, 46, 42, 50, 113, 49, 126, 87, 56, 103, 57, 21, 23, 76, 82, 84, 93, 20, 12, 86, 80, 14, 78, 85, 18, 81, 17, 120, 110, 16, 22, 29, 60, 55, 79, 48, 98, 97, 91, 63, 8, 54, 106, 15, 127, 39, 59, 116, 30, 74, 34, 10, 71, 26, 90, 52, 33, 47, 11, 9, 58, 75, 104, 19, 7, 27, 111, 89, 123, 83, 24, 45, 25, 70, 107, 73, 69, 5, 88, 13, 77, 109, 28, 94, 64, 40, 0, 68, 37, 105, 72, 6, 3, 66, 43, 67, 4, 101, 92, 65, 1, 31, 2, 32, 41, 35, 99, 96, 95, 100, 36], [53, 115, 117, 108, 51, 114, 125, 62, 38, 124, 122, 61, 112, 121, 44, 118, 46, 102, 119, 42, 50, 113, 56, 49, 87, 126, 103, 23, 57, 21, 93, 76, 20, 18, 84, 82, 86, 14, 85, 81, 17, 12, 22, 16, 80, 78, 39, 106, 98, 97, 127, 29, 15, 120, 34, 79, 91, 48, 116, 10, 30, 55, 54, 111, 74, 110, 8, 52, 58, 59, 26, 9, 11, 71, 7, 90, 33, 60, 109, 104, 63, 6, 27, 73, 25, 75, 47, 83, 89, 19, 28, 5, 24, 88, 69, 77, 107, 13, 70, 45, 72, 123, 68, 94, 64, 37, 40, 0, 32, 43, 67, 4, 35, 3, 31, 65, 101, 1, 66, 99, 2, 105, 92, 41, 95, 36, 100, 96], [53, 115, 117, 51, 108, 122, 125, 62, 38, 114, 124, 61, 102, 118, 112, 44, 119, 50, 121, 46, 42, 56, 113, 87, 49, 126, 103, 23, 21, 76, 86, 82, 18, 85, 84, 20, 93, 17, 57, 12, 14, 59, 81, 78, 16, 48, 80, 29, 91, 22, 116, 97, 110, 98, 127, 15, 30, 10, 34, 26, 106, 111, 74, 55, 6, 60, 11, 39, 120, 52, 90, 54, 79, 63, 47, 75, 58, 8, 73, 72, 7, 89, 109, 9, 24, 33, 27, 19, 71, 83, 123, 104, 25, 68, 28, 88, 40, 13, 107, 69, 5, 94, 3, 37, 43, 64, 77, 105, 4, 101, 41, 95, 45, 0, 31, 70, 35, 67, 96, 32, 100, 66, 92, 36, 2, 99, 65, 1], [53, 115, 117, 51, 108, 122, 125, 114, 62, 124, 38, 118, 61, 44, 119, 112, 102, 121, 42, 46, 50, 113, 49, 56, 126, 87, 23, 103, 21, 57, 76, 20, 85, 18, 14, 12, 17, 86, 81, 84, 82, 78, 80, 93, 59, 54, 16, 110, 22, 116, 98, 91, 120, 6, 29, 15, 63, 10, 48, 97, 106, 39, 55, 60, 79, 74, 30, 75, 127, 111, 52, 71, 58, 9, 11, 90, 26, 34, 33, 19, 72, 73, 47, 109, 27, 7, 104, 89, 88, 28, 83, 8, 24, 37, 107, 69, 25, 68, 123, 5, 43, 13, 94, 105, 40, 77, 45, 32, 67, 41, 4, 70, 2, 3, 64, 0, 35, 31, 66, 101, 95, 65, 92, 1, 100, 99, 96, 36], [53, 115, 117, 51, 108, 122, 125, 38, 114, 124, 62, 61, 118, 44, 121, 119, 102, 112, 46, 42, 50, 113, 49, 126, 56, 87, 57, 103, 23, 21, 76, 20, 82, 81, 78, 84, 18, 85, 12, 86, 14, 80, 93, 17, 54, 16, 120, 97, 29, 48, 91, 59, 98, 22, 15, 116, 10, 110, 52, 79, 127, 106, 90, 26, 74, 30, 47, 72, 60, 11, 6, 39, 55, 34, 9, 63, 58, 75, 73, 33, 111, 104, 19, 24, 7, 27, 71, 88, 89, 5, 28, 37, 123, 13, 69, 25, 107, 83, 94, 41, 40, 45, 105, 0, 77, 109, 43, 4, 2, 8, 101, 70, 32, 68, 64, 31, 3, 35, 66, 96, 65, 67, 1, 95, 92, 99, 100, 36], [53, 115, 117, 108, 51, 122, 38, 125, 62, 124, 61, 114, 44, 118, 112, 102, 50, 121, 119, 42, 46, 126, 56, 113, 23, 49, 103, 87, 21, 57, 17, 85, 20, 18, 84, 12, 82, 93, 76, 81, 86, 80, 78, 14, 22, 110, 29, 97, 55, 48, 16, 98, 106, 10, 91, 72, 26, 127, 47, 79, 30, 116, 39, 15, 59, 34, 74, 58, 90, 120, 111, 54, 123, 73, 33, 60, 52, 7, 89, 75, 9, 11, 19, 63, 104, 6, 27, 25, 69, 83, 24, 88, 71, 94, 28, 109, 5, 45, 77, 4, 105, 64, 32, 68, 40, 35, 70, 13, 67, 8, 107, 31, 37, 0, 3, 66, 41, 101, 96, 95, 92, 2, 65, 43, 99, 1, 36, 100], [53, 115, 117, 108, 51, 122, 125, 124, 62, 38, 114, 61, 118, 121, 44, 119, 112, 102, 46, 42, 50, 126, 49, 56, 113, 87, 23, 103, 12, 21, 81, 14, 76, 18, 110, 84, 78, 57, 82, 17, 20, 86, 85, 93, 16, 22, 80, 72, 10, 98, 127, 48, 74, 58, 106, 29, 120, 116, 63, 91, 7, 15, 47, 9, 26, 30, 79, 75, 71, 55, 97, 34, 11, 39, 59, 73, 90, 54, 89, 52, 109, 19, 33, 70, 123, 45, 111, 24, 6, 69, 5, 60, 28, 27, 83, 68, 77, 4, 0, 64, 104, 40, 88, 3, 67, 13, 107, 66, 94, 2, 8, 25, 65, 37, 105, 35, 32, 43, 41, 1, 31, 92, 96, 95, 101, 99, 100, 36], [53, 115, 117, 51, 108, 122, 125, 114, 124, 62, 38, 61, 118, 44, 121, 119, 102, 112, 50, 42, 46, 56, 113, 126, 49, 87, 23, 103, 12, 57, 21, 18, 84, 78, 76, 20, 86, 82, 14, 110, 85, 93, 81, 48, 16, 80, 17, 22, 72, 58, 98, 60, 106, 91, 116, 54, 10, 29, 79, 55, 127, 74, 120, 97, 34, 30, 39, 15, 63, 59, 70, 52, 33, 47, 90, 73, 75, 26, 9, 11, 104, 111, 71, 7, 89, 109, 123, 5, 19, 37, 88, 69, 24, 28, 45, 27, 35, 68, 13, 83, 40, 4, 25, 0, 64, 105, 8, 6, 2, 77, 43, 94, 66, 107, 3, 67, 1, 95, 65, 32, 31, 100, 96, 41, 99, 36, 101, 92], [53, 115, 117, 51, 108, 125, 122, 114, 124, 62, 38, 61, 119, 118, 44, 112, 50, 121, 102, 46, 126, 42, 113, 49, 56, 87, 23, 57, 103, 85, 12, 21, 76, 20, 82, 18, 78, 14, 17, 86, 93, 84, 16, 59, 80, 81, 116, 54, 98, 22, 29, 55, 58, 47, 72, 70, 91, 120, 15, 110, 10, 74, 63, 79, 48, 106, 127, 34, 9, 7, 30, 97, 52, 39, 75, 11, 60, 71, 111, 123, 73, 90, 26, 33, 69, 89, 24, 28, 5, 109, 19, 4, 40, 67, 27, 83, 64, 88, 37, 104, 68, 45, 25, 94, 0, 3, 8, 105, 77, 13, 107, 66, 2, 43, 1, 32, 65, 35, 41, 6, 31, 100, 95, 101, 96, 92, 99, 36], [53, 115, 117, 108, 51, 125, 122, 124, 38, 62, 114, 119, 61, 44, 118, 112, 121, 46, 102, 50, 42, 49, 113, 126, 56, 87, 103, 23, 21, 57, 84, 85, 12, 18, 82, 76, 20, 93, 81, 78, 17, 14, 59, 86, 80, 22, 16, 98, 116, 106, 74, 79, 47, 54, 55, 127, 29, 110, 48, 120, 39, 15, 70, 63, 72, 60, 30, 26, 97, 11, 91, 10, 73, 9, 7, 34, 75, 52, 58, 19, 111, 71, 90, 33, 45, 69, 4, 109, 68, 123, 104, 27, 89, 28, 83, 24, 37, 8, 40, 13, 88, 25, 5, 66, 0, 107, 105, 67, 3, 64, 94, 77, 6, 2, 43, 41, 32, 31, 1, 35, 65, 95, 92, 101, 96, 36, 100, 99], [53, 115, 117, 108, 51, 122, 125, 124, 38, 114, 62, 61, 119, 44, 118, 50, 112, 121, 102, 46, 42, 49, 113, 56, 126, 87, 23, 103, 21, 76, 57, 54, 14, 85, 20, 81, 78, 18, 84, 82, 127, 86, 93, 12, 17, 80, 55, 60, 59, 16, 22, 34, 63, 116, 74, 110, 29, 10, 26, 120, 48, 91, 98, 70, 97, 79, 30, 15, 123, 73, 47, 11, 111, 39, 9, 75, 33, 52, 72, 106, 109, 7, 71, 24, 19, 89, 90, 88, 8, 105, 5, 104, 58, 27, 43, 69, 83, 28, 45, 37, 4, 25, 68, 77, 64, 6, 94, 40, 107, 13, 0, 32, 3, 66, 41, 101, 1, 92, 67, 2, 95, 31, 96, 100, 35, 65, 36, 99]], "model.layers.6.self_attn.q_proj": [[108, 55, 36, 96, 91, 23, 44, 84, 77, 81, 15, 86, 10, 16, 7, 114, 57, 75, 78, 32, 30, 28, 111, 89, 6, 25, 87, 29, 66, 70, 11, 69, 68, 22, 101, 13, 74, 72, 85, 19, 95, 17, 92, 26, 90, 51, 88, 79, 121, 14, 40, 18, 3, 9, 20, 61, 27, 76, 126, 4, 1, 54, 123, 80, 71, 107, 97, 103, 12, 64, 94, 118, 99, 102, 83, 33, 21, 39, 93, 73, 49, 58, 60, 24, 82, 125, 41, 31, 34, 98, 112, 35, 110, 48, 37, 104, 56, 5, 115, 109, 53, 38, 120, 50, 124, 2, 106, 59, 113, 47, 105, 127, 8, 46, 52, 122, 63, 42, 65, 43, 119, 116, 117, 62, 45, 67, 100, 0], [108, 55, 36, 23, 44, 84, 15, 75, 91, 81, 77, 10, 8, 96, 29, 114, 3, 72, 70, 74, 6, 64, 111, 11, 67, 86, 69, 30, 57, 2, 79, 89, 83, 27, 4, 26, 25, 65, 94, 28, 73, 18, 103, 0, 14, 5, 82, 17, 9, 87, 13, 31, 68, 85, 12, 93, 21, 1, 88, 20, 16, 80, 112, 78, 51, 7, 22, 95, 39, 32, 24, 19, 66, 97, 76, 90, 71, 33, 118, 121, 56, 123, 35, 92, 54, 98, 34, 107, 102, 48, 40, 52, 127, 41, 99, 37, 61, 59, 119, 120, 110, 49, 115, 125, 101, 104, 122, 60, 58, 100, 38, 126, 46, 109, 47, 43, 105, 124, 63, 53, 62, 106, 45, 113, 116, 42, 50, 117], [55, 108, 112, 39, 127, 36, 111, 61, 44, 54, 51, 124, 113, 118, 60, 121, 50, 59, 122, 63, 56, 58, 119, 115, 101, 116, 123, 120, 97, 62, 53, 42, 92, 26, 117, 19, 47, 43, 48, 40, 49, 94, 98, 52, 32, 90, 46, 109, 106, 110, 105, 96, 83, 45, 114, 104, 107, 41, 38, 35, 95, 88, 125, 102, 99, 33, 86, 30, 34, 23, 37, 103, 126, 29, 91, 100, 57, 78, 25, 89, 16, 93, 27, 28, 24, 31, 73, 21, 85, 81, 22, 18, 7, 14, 82, 15, 84, 75, 87, 65, 2, 77, 9, 12, 66, 11, 17, 10, 6, 4, 68, 67, 80, 76, 0, 64, 71, 5, 70, 8, 72, 79, 13, 1, 20, 69, 74, 3], [108, 55, 39, 44, 36, 94, 103, 126, 111, 90, 102, 114, 96, 57, 34, 121, 127, 29, 32, 83, 51, 54, 123, 91, 86, 60, 58, 63, 0, 19, 61, 59, 2, 110, 115, 49, 23, 125, 112, 52, 113, 98, 104, 56, 50, 38, 37, 107, 31, 42, 124, 47, 101, 106, 40, 78, 82, 117, 97, 45, 120, 109, 16, 89, 122, 92, 105, 119, 116, 33, 48, 53, 15, 10, 77, 118, 62, 46, 35, 84, 28, 43, 81, 70, 7, 41, 67, 88, 4, 68, 99, 65, 26, 95, 93, 100, 30, 5, 71, 64, 85, 25, 76, 21, 11, 75, 24, 72, 8, 12, 18, 66, 9, 73, 27, 80, 1, 22, 14, 6, 87, 69, 17, 79, 13, 3, 74, 20], [42, 102, 46, 32, 103, 89, 56, 20, 106, 76, 22, 17, 14, 19, 71, 127, 10, 110, 28, 93, 8, 120, 26, 51, 80, 60, 116, 29, 66, 50, 11, 27, 96, 38, 82, 25, 47, 123, 69, 36, 109, 63, 3, 59, 34, 118, 13, 113, 105, 79, 6, 88, 54, 33, 21, 41, 67, 52, 83, 30, 75, 81, 119, 15, 35, 77, 12, 7, 91, 86, 2, 114, 58, 112, 78, 61, 94, 84, 90, 5, 23, 16, 125, 85, 98, 101, 18, 62, 87, 70, 95, 97, 92, 74, 122, 49, 111, 4, 126, 24, 31, 9, 64, 99, 73, 72, 55, 43, 115, 57, 107, 108, 121, 44, 45, 37, 40, 124, 68, 0, 117, 53, 100, 104, 48, 39, 65, 1], [42, 41, 46, 103, 23, 32, 106, 102, 80, 92, 56, 8, 20, 26, 24, 89, 123, 29, 17, 110, 105, 96, 127, 19, 22, 116, 120, 25, 93, 28, 51, 52, 14, 109, 121, 60, 76, 27, 83, 54, 10, 69, 107, 47, 44, 49, 119, 86, 100, 61, 111, 35, 34, 58, 97, 5, 85, 91, 112, 122, 50, 63, 18, 104, 31, 39, 43, 59, 4, 9, 95, 66, 65, 117, 115, 15, 72, 124, 90, 68, 3, 126, 11, 62, 118, 48, 55, 101, 30, 98, 33, 21, 73, 108, 87, 114, 57, 53, 36, 45, 38, 94, 125, 99, 79, 37, 2, 40, 1, 77, 113, 82, 88, 71, 16, 78, 75, 81, 0, 70, 84, 6, 64, 13, 67, 12, 74, 7], [102, 46, 42, 56, 51, 116, 127, 110, 26, 123, 41, 47, 60, 50, 32, 59, 122, 120, 103, 63, 22, 111, 61, 90, 52, 19, 109, 33, 112, 27, 121, 115, 55, 49, 96, 34, 54, 118, 93, 119, 117, 43, 113, 53, 89, 57, 62, 48, 124, 16, 58, 39, 107, 126, 125, 114, 44, 29, 105, 31, 108, 101, 97, 45, 85, 79, 104, 36, 35, 98, 99, 38, 40, 24, 11, 25, 100, 37, 95, 94, 92, 106, 87, 28, 88, 20, 23, 91, 30, 83, 13, 80, 72, 86, 18, 15, 84, 17, 21, 14, 75, 73, 12, 8, 82, 69, 77, 9, 66, 5, 76, 78, 2, 81, 6, 70, 4, 74, 1, 68, 71, 10, 7, 65, 3, 0, 64, 67], [42, 46, 102, 32, 20, 17, 89, 76, 103, 22, 106, 10, 14, 71, 96, 29, 110, 69, 127, 56, 26, 51, 25, 66, 38, 11, 120, 93, 19, 5, 60, 24, 116, 79, 41, 61, 63, 119, 81, 78, 6, 109, 8, 118, 87, 82, 12, 58, 13, 75, 83, 2, 80, 23, 3, 86, 55, 15, 52, 30, 97, 54, 27, 113, 88, 59, 21, 16, 47, 28, 84, 77, 68, 85, 91, 90, 122, 35, 125, 0, 18, 72, 105, 111, 115, 50, 95, 74, 101, 70, 114, 7, 65, 112, 92, 73, 126, 99, 94, 67, 36, 43, 34, 49, 33, 98, 31, 107, 4, 37, 64, 1, 9, 121, 48, 100, 45, 123, 108, 104, 62, 44, 40, 117, 124, 39, 53, 57], [109, 103, 34, 45, 4, 20, 74, 71, 1, 18, 9, 14, 12, 0, 80, 88, 3, 66, 112, 70, 68, 79, 72, 39, 75, 67, 85, 10, 21, 7, 19, 5, 69, 13, 124, 77, 6, 16, 65, 23, 84, 11, 64, 76, 60, 8, 24, 73, 15, 2, 98, 87, 51, 22, 48, 82, 78, 83, 52, 81, 25, 86, 17, 108, 59, 57, 27, 63, 49, 55, 113, 89, 91, 121, 126, 123, 96, 110, 120, 92, 61, 114, 111, 30, 62, 90, 94, 119, 97, 28, 127, 54, 29, 41, 115, 95, 125, 93, 116, 44, 31, 58, 33, 38, 42, 26, 105, 106, 46, 107, 32, 40, 99, 122, 36, 35, 43, 56, 104, 50, 101, 100, 102, 117, 37, 118, 47, 53], [103, 109, 34, 113, 45, 88, 84, 24, 16, 76, 92, 11, 124, 8, 15, 82, 83, 73, 10, 60, 22, 127, 112, 7, 78, 18, 119, 63, 48, 57, 51, 52, 49, 62, 19, 123, 91, 114, 89, 56, 6, 115, 108, 53, 90, 120, 104, 94, 28, 9, 95, 110, 86, 26, 29, 111, 93, 97, 35, 126, 59, 122, 30, 54, 121, 117, 27, 50, 58, 96, 125, 118, 100, 31, 61, 101, 116, 32, 55, 20, 105, 33, 44, 46, 43, 42, 37, 38, 40, 99, 36, 41, 106, 25, 12, 107, 102, 47, 87, 77, 21, 13, 23, 69, 79, 75, 81, 17, 74, 72, 14, 85, 68, 98, 80, 71, 70, 66, 4, 5, 39, 67, 2, 65, 3, 1, 0, 64], [109, 103, 34, 45, 20, 74, 22, 18, 19, 4, 12, 14, 39, 67, 71, 79, 10, 72, 80, 68, 73, 1, 5, 3, 9, 13, 112, 88, 7, 66, 2, 77, 75, 0, 23, 85, 64, 70, 76, 21, 16, 15, 27, 69, 60, 92, 82, 8, 78, 65, 84, 11, 83, 81, 48, 6, 120, 123, 24, 124, 52, 63, 58, 91, 113, 51, 86, 55, 87, 17, 114, 41, 57, 25, 98, 89, 119, 90, 31, 49, 59, 126, 97, 127, 108, 94, 42, 29, 62, 28, 30, 56, 115, 96, 36, 61, 53, 105, 33, 106, 95, 32, 40, 46, 125, 93, 111, 122, 110, 47, 99, 118, 26, 107, 121, 102, 38, 100, 35, 104, 101, 54, 37, 50, 117, 44, 43, 116], [109, 103, 34, 45, 22, 80, 14, 20, 4, 18, 24, 74, 70, 75, 88, 19, 72, 112, 13, 12, 124, 89, 68, 10, 9, 60, 71, 63, 51, 73, 15, 66, 39, 76, 2, 8, 48, 52, 82, 78, 113, 77, 16, 83, 84, 49, 87, 6, 62, 127, 23, 57, 81, 11, 121, 5, 59, 79, 1, 17, 91, 0, 64, 25, 85, 92, 86, 94, 106, 7, 97, 21, 123, 27, 31, 90, 38, 28, 61, 110, 67, 69, 119, 36, 104, 126, 114, 99, 96, 3, 101, 111, 108, 95, 32, 41, 116, 40, 102, 55, 26, 120, 118, 98, 56, 50, 54, 100, 53, 30, 58, 35, 47, 93, 29, 115, 33, 125, 44, 107, 37, 117, 43, 46, 122, 65, 105, 42], [38, 124, 49, 30, 56, 86, 113, 15, 82, 84, 81, 26, 76, 77, 8, 5, 74, 71, 66, 122, 94, 67, 24, 11, 25, 1, 60, 22, 102, 72, 32, 12, 39, 97, 2, 120, 75, 21, 9, 17, 64, 10, 79, 14, 16, 89, 78, 91, 20, 6, 23, 80, 83, 87, 29, 4, 51, 73, 13, 68, 90, 27, 70, 112, 69, 33, 92, 93, 59, 85, 18, 96, 99, 117, 125, 31, 7, 114, 0, 19, 3, 107, 28, 105, 88, 118, 111, 110, 109, 44, 63, 104, 50, 108, 123, 95, 106, 52, 55, 36, 126, 101, 34, 65, 58, 116, 119, 47, 103, 62, 40, 45, 100, 42, 37, 121, 57, 54, 61, 48, 35, 41, 127, 115, 98, 43, 46, 53], [38, 49, 56, 124, 113, 30, 7, 84, 86, 82, 3, 81, 77, 15, 10, 13, 65, 89, 5, 76, 1, 67, 69, 8, 2, 64, 68, 94, 74, 73, 26, 72, 91, 122, 120, 66, 0, 27, 22, 97, 20, 79, 6, 18, 12, 9, 11, 14, 25, 87, 17, 23, 32, 24, 29, 51, 90, 75, 80, 60, 16, 99, 92, 78, 102, 83, 71, 59, 4, 88, 19, 39, 70, 33, 21, 31, 28, 103, 43, 114, 45, 125, 85, 112, 98, 93, 46, 111, 107, 118, 40, 95, 58, 37, 101, 126, 109, 104, 108, 44, 55, 105, 63, 41, 57, 61, 127, 96, 42, 115, 53, 116, 35, 54, 48, 100, 36, 47, 121, 62, 50, 117, 106, 119, 110, 34, 52, 123], [38, 49, 56, 124, 30, 86, 94, 89, 113, 81, 84, 91, 102, 122, 39, 15, 25, 82, 18, 51, 22, 26, 23, 76, 59, 97, 32, 120, 17, 33, 87, 105, 125, 29, 114, 99, 126, 72, 60, 103, 90, 62, 24, 31, 116, 77, 58, 20, 109, 118, 110, 52, 79, 45, 127, 112, 121, 63, 41, 117, 104, 96, 13, 55, 57, 50, 43, 44, 119, 111, 12, 115, 42, 48, 93, 95, 54, 101, 78, 40, 123, 75, 108, 37, 27, 53, 47, 46, 35, 100, 92, 61, 106, 36, 34, 107, 83, 28, 98, 21, 10, 19, 88, 9, 85, 80, 69, 14, 16, 73, 8, 7, 74, 11, 6, 68, 70, 3, 5, 71, 66, 4, 2, 65, 67, 1, 64, 0], [38, 49, 30, 124, 56, 113, 84, 86, 15, 77, 82, 74, 81, 5, 73, 76, 89, 8, 67, 71, 1, 20, 22, 72, 10, 122, 78, 66, 14, 64, 17, 9, 75, 79, 26, 94, 120, 16, 87, 11, 2, 7, 12, 6, 65, 99, 13, 91, 80, 69, 25, 27, 4, 3, 24, 39, 90, 97, 18, 32, 70, 93, 51, 85, 23, 92, 68, 29, 19, 21, 118, 31, 83, 60, 28, 88, 0, 114, 107, 33, 45, 46, 42, 43, 95, 103, 44, 117, 98, 125, 101, 100, 109, 63, 104, 62, 126, 54, 58, 111, 112, 40, 115, 57, 37, 59, 116, 50, 96, 55, 61, 52, 53, 34, 105, 41, 48, 35, 36, 119, 106, 121, 47, 110, 127, 123, 108, 102], [40, 98, 116, 32, 20, 78, 9, 81, 11, 89, 86, 7, 68, 69, 16, 47, 104, 4, 3, 53, 2, 49, 73, 51, 13, 117, 111, 126, 58, 121, 64, 8, 46, 56, 0, 67, 60, 94, 52, 88, 10, 127, 34, 42, 90, 71, 62, 72, 80, 74, 84, 113, 92, 50, 45, 79, 70, 63, 77, 75, 5, 27, 120, 6, 57, 44, 65, 1, 87, 14, 19, 106, 54, 17, 41, 101, 24, 125, 100, 21, 114, 99, 29, 122, 85, 105, 108, 33, 66, 15, 43, 124, 28, 119, 23, 115, 30, 12, 91, 18, 110, 25, 38, 39, 26, 22, 107, 118, 96, 61, 59, 103, 36, 83, 35, 97, 112, 123, 48, 37, 109, 102, 93, 95, 82, 31, 55, 76], [40, 98, 116, 32, 86, 20, 94, 88, 89, 28, 81, 62, 50, 121, 58, 78, 117, 52, 11, 47, 115, 45, 21, 104, 46, 126, 53, 111, 49, 87, 13, 30, 101, 41, 51, 96, 42, 16, 22, 92, 119, 90, 12, 122, 113, 24, 31, 91, 63, 26, 18, 29, 44, 37, 23, 109, 83, 93, 95, 107, 85, 38, 36, 125, 120, 110, 99, 103, 76, 79, 43, 114, 105, 102, 108, 8, 59, 25, 84, 124, 19, 9, 15, 7, 112, 54, 48, 60, 123, 82, 100, 127, 56, 27, 97, 106, 118, 35, 55, 80, 61, 57, 33, 17, 39, 14, 74, 34, 77, 75, 69, 10, 72, 2, 71, 3, 6, 68, 70, 73, 5, 67, 0, 66, 64, 4, 65, 1], [40, 98, 32, 86, 104, 116, 81, 20, 13, 47, 2, 66, 7, 8, 78, 49, 58, 51, 11, 9, 121, 60, 94, 5, 111, 45, 53, 52, 62, 68, 73, 46, 4, 16, 122, 92, 126, 70, 88, 89, 50, 0, 69, 56, 115, 34, 113, 10, 87, 64, 117, 125, 79, 3, 103, 82, 18, 28, 119, 77, 127, 63, 37, 57, 1, 108, 100, 27, 72, 25, 43, 114, 106, 44, 75, 120, 71, 22, 84, 65, 17, 30, 23, 42, 93, 14, 41, 19, 59, 54, 35, 48, 21, 31, 101, 76, 110, 12, 118, 29, 99, 90, 112, 80, 6, 85, 109, 15, 61, 124, 83, 36, 26, 91, 102, 55, 107, 24, 97, 39, 123, 95, 33, 38, 105, 74, 67, 96], [40, 98, 116, 86, 81, 32, 89, 78, 20, 88, 11, 7, 104, 47, 117, 111, 13, 49, 16, 28, 53, 74, 10, 51, 9, 121, 94, 58, 52, 126, 62, 46, 21, 45, 75, 60, 69, 25, 92, 96, 17, 8, 115, 79, 30, 50, 63, 77, 113, 90, 73, 71, 80, 56, 14, 67, 127, 95, 38, 2, 23, 42, 120, 122, 87, 72, 84, 99, 18, 44, 41, 29, 101, 105, 114, 125, 3, 22, 61, 19, 83, 91, 100, 6, 5, 54, 43, 97, 24, 27, 37, 119, 112, 108, 118, 15, 68, 85, 70, 102, 57, 107, 59, 34, 124, 55, 33, 12, 48, 110, 76, 66, 106, 26, 123, 103, 109, 31, 39, 93, 35, 65, 36, 82, 0, 1, 4, 64], [119, 38, 118, 42, 33, 25, 84, 31, 88, 52, 94, 103, 114, 99, 57, 55, 102, 28, 83, 78, 96, 54, 50, 62, 22, 106, 60, 45, 87, 93, 112, 29, 23, 53, 97, 27, 71, 61, 34, 74, 32, 85, 109, 111, 17, 24, 120, 46, 122, 110, 41, 80, 104, 21, 40, 18, 116, 125, 35, 121, 39, 105, 10, 123, 79, 56, 36, 92, 98, 47, 113, 107, 89, 127, 115, 126, 26, 20, 101, 16, 30, 43, 12, 49, 48, 81, 58, 91, 117, 37, 14, 13, 90, 63, 82, 100, 73, 11, 4, 7, 75, 44, 59, 51, 68, 124, 15, 108, 77, 95, 19, 72, 86, 1, 76, 8, 5, 6, 3, 0, 70, 69, 9, 65, 2, 67, 66, 64], [119, 118, 38, 33, 103, 42, 25, 94, 90, 54, 88, 109, 84, 31, 112, 120, 52, 102, 62, 111, 22, 45, 93, 53, 50, 57, 55, 83, 125, 47, 114, 87, 110, 106, 122, 74, 35, 18, 60, 101, 48, 58, 116, 85, 127, 61, 23, 78, 28, 59, 91, 100, 124, 121, 46, 63, 99, 49, 79, 126, 56, 123, 51, 113, 108, 30, 115, 105, 107, 117, 29, 43, 98, 26, 41, 39, 96, 0, 44, 24, 71, 34, 97, 40, 104, 27, 72, 89, 8, 11, 21, 75, 16, 68, 82, 12, 5, 37, 13, 69, 36, 64, 1, 4, 32, 95, 80, 67, 7, 2, 92, 65, 17, 14, 66, 73, 9, 3, 15, 70, 77, 10, 20, 81, 76, 6, 86, 19], [118, 38, 119, 42, 33, 25, 31, 109, 83, 55, 22, 94, 114, 84, 78, 90, 88, 106, 99, 52, 53, 54, 12, 74, 103, 120, 48, 111, 121, 102, 125, 17, 110, 50, 93, 62, 87, 107, 127, 34, 112, 60, 43, 97, 89, 115, 113, 122, 61, 57, 105, 58, 47, 91, 116, 73, 104, 35, 63, 46, 123, 30, 96, 28, 29, 72, 41, 59, 124, 126, 23, 44, 70, 37, 56, 117, 49, 101, 100, 27, 40, 85, 51, 79, 71, 95, 7, 45, 108, 80, 92, 68, 19, 39, 98, 81, 86, 26, 24, 75, 10, 36, 32, 67, 15, 76, 16, 21, 2, 18, 14, 11, 77, 82, 3, 66, 8, 1, 6, 20, 13, 0, 69, 4, 64, 5, 65, 9], [38, 118, 119, 33, 25, 83, 31, 79, 84, 89, 94, 12, 81, 78, 90, 88, 28, 85, 73, 22, 29, 54, 42, 24, 19, 112, 45, 26, 87, 93, 70, 17, 6, 102, 97, 48, 9, 55, 27, 116, 106, 35, 59, 127, 121, 114, 74, 110, 32, 125, 2, 103, 82, 20, 124, 52, 98, 96, 34, 21, 71, 50, 15, 80, 67, 18, 40, 16, 95, 109, 92, 0, 75, 91, 76, 62, 117, 72, 8, 39, 120, 69, 77, 65, 43, 30, 108, 36, 100, 99, 5, 58, 115, 57, 101, 14, 13, 23, 51, 60, 3, 61, 64, 107, 37, 86, 113, 104, 4, 66, 11, 126, 68, 41, 47, 122, 46, 111, 53, 1, 10, 123, 105, 49, 63, 7, 56, 44], [39, 57, 58, 52, 56, 116, 117, 55, 118, 51, 37, 59, 120, 86, 97, 31, 50, 126, 28, 98, 110, 99, 53, 54, 63, 112, 40, 122, 41, 20, 29, 47, 43, 92, 111, 109, 127, 124, 38, 125, 114, 95, 115, 108, 49, 123, 119, 105, 88, 89, 19, 61, 106, 62, 30, 60, 107, 113, 45, 34, 44, 104, 100, 121, 48, 36, 42, 46, 102, 33, 25, 32, 23, 6, 81, 101, 35, 80, 93, 10, 91, 24, 73, 96, 18, 87, 15, 78, 69, 27, 13, 14, 94, 22, 26, 90, 74, 66, 84, 67, 76, 71, 16, 12, 7, 68, 103, 64, 8, 21, 1, 2, 17, 72, 75, 5, 85, 83, 82, 11, 77, 0, 9, 3, 65, 79, 4, 70], [56, 39, 58, 52, 19, 59, 120, 34, 28, 90, 26, 31, 95, 96, 55, 37, 79, 53, 12, 21, 97, 88, 98, 86, 50, 116, 127, 118, 57, 24, 92, 8, 99, 25, 122, 49, 123, 111, 89, 14, 121, 18, 83, 27, 108, 29, 74, 63, 62, 51, 110, 44, 43, 87, 105, 109, 115, 91, 104, 38, 70, 126, 114, 1, 42, 10, 80, 41, 119, 68, 46, 33, 102, 72, 35, 60, 32, 45, 124, 125, 15, 93, 100, 106, 4, 48, 107, 85, 103, 112, 94, 47, 6, 78, 76, 23, 113, 81, 61, 75, 40, 101, 54, 11, 117, 0, 84, 16, 82, 36, 9, 2, 30, 13, 17, 22, 67, 71, 20, 64, 3, 69, 65, 73, 66, 77, 5, 7], [56, 39, 52, 117, 58, 57, 116, 55, 37, 118, 51, 53, 86, 54, 120, 10, 31, 73, 127, 126, 110, 111, 63, 97, 123, 122, 6, 119, 38, 76, 99, 121, 16, 29, 49, 43, 109, 13, 105, 112, 80, 42, 124, 125, 44, 78, 83, 106, 61, 7, 108, 93, 89, 28, 115, 50, 72, 48, 40, 62, 107, 45, 60, 46, 114, 77, 47, 113, 68, 88, 81, 11, 98, 41, 66, 104, 87, 82, 15, 67, 102, 69, 92, 36, 79, 30, 101, 34, 27, 59, 19, 96, 75, 22, 100, 33, 95, 20, 35, 1, 103, 71, 17, 32, 5, 84, 24, 94, 64, 26, 23, 85, 14, 4, 91, 8, 25, 0, 65, 9, 2, 12, 74, 21, 90, 70, 18, 3], [39, 52, 56, 34, 117, 57, 58, 51, 88, 116, 120, 50, 118, 95, 18, 98, 80, 20, 86, 55, 122, 21, 92, 30, 22, 90, 28, 53, 114, 97, 13, 74, 115, 19, 29, 32, 109, 60, 126, 62, 77, 61, 125, 99, 26, 89, 27, 25, 121, 37, 68, 11, 78, 49, 82, 14, 8, 54, 23, 48, 79, 24, 31, 71, 96, 112, 40, 105, 123, 127, 35, 46, 36, 94, 33, 124, 63, 12, 108, 119, 100, 110, 45, 10, 43, 44, 111, 47, 9, 107, 93, 85, 42, 16, 41, 38, 106, 15, 59, 113, 104, 102, 6, 101, 75, 91, 70, 81, 83, 2, 69, 84, 17, 1, 87, 76, 73, 0, 4, 3, 67, 5, 66, 72, 103, 7, 65, 64], [57, 48, 50, 38, 58, 54, 118, 56, 61, 100, 53, 63, 124, 115, 41, 116, 117, 119, 52, 102, 59, 84, 120, 46, 51, 121, 47, 126, 122, 125, 62, 113, 114, 111, 127, 97, 20, 44, 123, 49, 109, 60, 55, 45, 110, 112, 108, 43, 103, 106, 40, 107, 74, 42, 89, 105, 30, 27, 34, 31, 7, 33, 13, 101, 39, 24, 95, 78, 104, 32, 36, 98, 35, 21, 37, 88, 25, 99, 92, 80, 77, 87, 91, 96, 29, 23, 85, 93, 16, 71, 9, 69, 94, 28, 10, 66, 12, 90, 18, 73, 4, 26, 2, 5, 6, 65, 64, 3, 14, 19, 68, 72, 79, 83, 11, 70, 17, 0, 75, 82, 86, 1, 76, 15, 67, 22, 81, 8], [50, 38, 57, 48, 97, 112, 114, 7, 54, 115, 20, 88, 111, 77, 124, 90, 81, 100, 29, 95, 18, 79, 11, 28, 4, 27, 58, 24, 53, 68, 35, 23, 52, 59, 107, 94, 86, 119, 1, 101, 25, 63, 10, 56, 120, 99, 61, 42, 72, 104, 41, 121, 16, 103, 40, 125, 26, 43, 118, 117, 39, 108, 32, 123, 109, 110, 51, 37, 8, 46, 102, 116, 47, 105, 87, 106, 14, 93, 44, 6, 45, 85, 71, 49, 62, 126, 113, 55, 3, 76, 83, 65, 96, 75, 60, 34, 127, 89, 91, 30, 21, 122, 36, 31, 98, 78, 15, 84, 73, 12, 82, 17, 66, 5, 19, 80, 2, 74, 33, 0, 64, 92, 22, 9, 69, 13, 67, 70], [38, 57, 50, 48, 97, 23, 79, 72, 81, 76, 100, 18, 114, 28, 112, 6, 10, 3, 68, 14, 21, 65, 8, 70, 86, 77, 11, 82, 80, 87, 0, 67, 17, 54, 12, 75, 90, 88, 24, 56, 15, 2, 20, 95, 59, 73, 78, 5, 83, 9, 13, 85, 53, 84, 16, 29, 69, 58, 93, 102, 19, 92, 115, 91, 26, 25, 74, 27, 52, 66, 22, 117, 71, 94, 30, 121, 111, 61, 89, 32, 107, 7, 64, 124, 101, 36, 1, 55, 31, 35, 96, 63, 99, 4, 98, 119, 46, 34, 41, 60, 118, 125, 43, 110, 120, 106, 37, 126, 62, 123, 104, 40, 47, 42, 39, 103, 44, 127, 51, 116, 49, 105, 33, 108, 122, 109, 45, 113], [48, 50, 57, 38, 122, 115, 126, 51, 46, 54, 118, 56, 47, 62, 58, 109, 114, 124, 116, 49, 125, 55, 52, 112, 63, 113, 119, 61, 127, 60, 110, 59, 53, 45, 106, 43, 117, 44, 123, 121, 120, 108, 41, 107, 105, 40, 42, 103, 34, 102, 111, 101, 39, 89, 104, 20, 97, 37, 95, 32, 29, 90, 86, 100, 36, 31, 30, 35, 93, 99, 88, 77, 92, 96, 94, 98, 16, 22, 83, 33, 81, 27, 25, 91, 23, 80, 84, 14, 28, 24, 19, 21, 73, 26, 17, 82, 13, 18, 79, 78, 87, 85, 76, 11, 66, 10, 6, 12, 74, 9, 15, 0, 75, 2, 71, 69, 7, 1, 72, 8, 64, 68, 3, 4, 70, 5, 65, 67]], "model.layers.6.self_attn.k_proj": [[44, 55, 100, 86, 32, 84, 81, 57, 15, 77, 91, 108, 23, 12, 10, 64, 5, 75, 67, 8, 7, 65, 94, 114, 90, 16, 111, 29, 71, 76, 50, 82, 89, 103, 68, 72, 2, 83, 25, 39, 123, 78, 51, 66, 93, 121, 63, 120, 6, 58, 46, 48, 118, 125, 19, 124, 31, 126, 106, 61, 127, 104, 21, 70, 40, 1, 45, 18, 95, 88, 47, 0, 122, 69, 113, 53, 119, 34, 117, 62, 92, 54, 56, 42, 37, 109, 43, 73, 35, 41, 33, 11, 38, 116, 80, 30, 97, 49, 52, 98, 85, 28, 74, 107, 112, 59, 115, 110, 102, 4, 24, 60, 87, 99, 105, 101, 14, 27, 22, 26, 20, 79, 96, 13, 9, 17, 3, 36], [106, 110, 46, 96, 22, 93, 17, 89, 39, 10, 14, 19, 76, 20, 56, 71, 38, 120, 0, 11, 24, 45, 111, 42, 1, 123, 3, 52, 69, 66, 26, 28, 13, 103, 119, 58, 16, 5, 8, 49, 79, 37, 92, 18, 121, 116, 48, 107, 54, 43, 99, 114, 70, 105, 57, 127, 68, 122, 61, 85, 113, 60, 50, 90, 47, 73, 63, 55, 40, 36, 126, 117, 62, 109, 35, 53, 125, 124, 100, 112, 118, 30, 65, 102, 27, 115, 108, 4, 44, 104, 67, 23, 80, 82, 6, 101, 31, 51, 98, 34, 59, 97, 29, 9, 84, 64, 91, 33, 41, 94, 86, 7, 72, 21, 95, 74, 88, 2, 87, 77, 78, 12, 75, 15, 83, 25, 32, 81], [45, 39, 98, 109, 0, 20, 80, 18, 72, 12, 14, 24, 79, 75, 71, 66, 74, 5, 4, 70, 65, 9, 3, 112, 1, 67, 77, 69, 22, 92, 19, 64, 113, 123, 60, 120, 85, 91, 59, 86, 89, 23, 58, 48, 88, 17, 73, 6, 34, 78, 114, 25, 10, 90, 21, 27, 55, 63, 108, 13, 30, 122, 28, 47, 96, 126, 29, 119, 104, 33, 61, 105, 93, 83, 95, 35, 115, 31, 57, 11, 100, 51, 8, 54, 50, 38, 2, 121, 94, 111, 110, 49, 97, 26, 125, 81, 32, 87, 124, 7, 40, 46, 56, 52, 68, 101, 127, 42, 44, 117, 106, 116, 107, 118, 36, 43, 37, 41, 62, 102, 99, 53, 16, 76, 82, 84, 15, 103], [113, 124, 102, 56, 49, 94, 84, 82, 15, 76, 86, 81, 74, 77, 64, 5, 71, 8, 89, 73, 1, 66, 69, 39, 67, 122, 26, 91, 4, 75, 2, 51, 68, 70, 25, 65, 3, 72, 12, 24, 13, 38, 30, 6, 22, 23, 20, 59, 33, 79, 14, 83, 78, 90, 32, 80, 87, 7, 27, 9, 29, 88, 96, 10, 60, 99, 19, 16, 28, 21, 85, 18, 0, 11, 105, 97, 93, 17, 31, 95, 92, 103, 114, 35, 58, 125, 54, 47, 117, 43, 46, 104, 45, 40, 107, 118, 112, 36, 108, 109, 116, 115, 110, 126, 42, 62, 48, 111, 63, 98, 101, 55, 50, 53, 57, 44, 61, 127, 52, 106, 41, 34, 37, 121, 119, 120, 123, 100], [104, 116, 34, 111, 64, 20, 11, 53, 81, 86, 78, 51, 8, 16, 9, 7, 13, 121, 96, 3, 113, 2, 58, 62, 68, 69, 109, 60, 110, 122, 42, 50, 117, 56, 41, 89, 46, 127, 30, 49, 1, 126, 90, 108, 21, 98, 67, 39, 6, 105, 40, 63, 35, 37, 88, 83, 82, 32, 27, 124, 125, 65, 92, 44, 15, 57, 76, 107, 118, 61, 120, 54, 36, 74, 29, 0, 87, 70, 72, 28, 24, 31, 23, 25, 99, 93, 59, 106, 119, 47, 85, 91, 5, 43, 103, 77, 94, 33, 112, 97, 18, 55, 26, 45, 22, 38, 95, 115, 114, 19, 101, 123, 12, 52, 84, 100, 48, 71, 102, 66, 17, 79, 10, 75, 14, 4, 73, 80], [102, 119, 118, 22, 97, 25, 28, 83, 84, 78, 95, 12, 64, 74, 94, 71, 16, 18, 52, 79, 88, 30, 68, 125, 106, 111, 73, 38, 124, 46, 13, 91, 54, 45, 127, 66, 109, 6, 116, 17, 48, 104, 62, 43, 51, 58, 40, 53, 61, 57, 126, 90, 59, 123, 117, 67, 65, 1, 120, 121, 113, 44, 93, 3, 60, 50, 11, 56, 49, 69, 103, 108, 21, 47, 37, 112, 81, 19, 99, 42, 115, 122, 23, 63, 35, 82, 114, 41, 75, 2, 105, 20, 39, 26, 100, 70, 110, 55, 36, 87, 34, 32, 72, 15, 107, 101, 4, 80, 0, 98, 7, 5, 31, 77, 85, 96, 8, 27, 29, 92, 76, 86, 9, 89, 10, 24, 14, 33], [103, 57, 52, 56, 31, 58, 21, 88, 18, 90, 117, 59, 79, 98, 28, 14, 111, 72, 89, 75, 4, 120, 118, 25, 60, 12, 116, 110, 126, 74, 64, 85, 115, 55, 50, 62, 63, 124, 121, 66, 70, 46, 32, 112, 125, 49, 123, 65, 109, 39, 108, 105, 51, 53, 114, 48, 43, 73, 54, 34, 44, 92, 127, 27, 35, 113, 119, 81, 106, 19, 107, 102, 45, 47, 86, 84, 101, 20, 61, 122, 96, 7, 97, 104, 87, 77, 76, 91, 29, 41, 100, 13, 40, 82, 42, 71, 11, 94, 26, 16, 33, 6, 38, 95, 67, 3, 99, 36, 69, 37, 24, 80, 78, 30, 93, 1, 9, 68, 5, 8, 17, 15, 10, 23, 22, 83, 0, 2], [102, 50, 57, 86, 48, 33, 92, 18, 79, 76, 81, 23, 72, 6, 88, 83, 112, 14, 90, 11, 0, 93, 100, 30, 3, 10, 111, 31, 53, 58, 77, 124, 68, 73, 119, 61, 94, 117, 114, 2, 16, 63, 21, 118, 52, 36, 29, 54, 125, 38, 35, 99, 56, 115, 120, 9, 59, 65, 121, 108, 123, 105, 41, 98, 47, 113, 126, 127, 69, 55, 25, 45, 62, 51, 46, 24, 109, 60, 116, 20, 107, 110, 49, 85, 43, 32, 44, 66, 5, 96, 34, 122, 89, 106, 75, 42, 104, 1, 103, 91, 27, 39, 40, 101, 26, 37, 64, 4, 71, 95, 82, 84, 12, 78, 67, 80, 22, 15, 87, 19, 7, 8, 74, 28, 17, 13, 70, 97]], "model.layers.6.self_attn.qk_proj": [[57, 56, 118, 119, 45, 50, 55, 124, 113, 109, 48, 49, 38, 46, 44, 106, 22, 104, 52, 116, 102, 39, 108, 86, 20, 84, 58, 17, 110, 42, 30, 81, 25, 111, 34, 79, 15, 78, 76, 82, 98, 18, 88, 94, 89, 24, 10, 14, 32, 12, 112, 7, 13, 74, 0, 103, 40, 77, 117, 114, 51, 29, 72, 120, 8, 53, 75, 64, 16, 71, 11, 96, 90, 19, 28, 80, 67, 62, 31, 60, 87, 26, 121, 33, 3, 92, 63, 69, 70, 123, 127, 23, 83, 73, 97, 5, 66, 2, 9, 59, 126, 122, 68, 27, 61, 41, 47, 4, 93, 100, 21, 43, 95, 36, 115, 125, 54, 107, 1, 91, 65, 99, 85, 105, 35, 6, 101, 37], [57, 56, 119, 118, 45, 50, 113, 55, 124, 109, 49, 48, 38, 46, 44, 106, 104, 22, 116, 52, 102, 39, 84, 86, 108, 20, 58, 110, 17, 42, 111, 81, 30, 25, 103, 34, 89, 94, 79, 82, 78, 98, 32, 15, 18, 76, 112, 14, 117, 24, 88, 12, 51, 10, 40, 74, 29, 7, 53, 75, 77, 120, 16, 28, 71, 72, 60, 11, 8, 64, 13, 114, 90, 80, 0, 96, 19, 121, 26, 127, 59, 97, 83, 87, 66, 31, 69, 2, 23, 62, 92, 4, 5, 33, 9, 122, 73, 123, 126, 27, 68, 67, 63, 1, 70, 100, 54, 61, 93, 125, 95, 3, 47, 43, 65, 21, 41, 91, 107, 6, 115, 99, 85, 36, 105, 35, 37, 101], [57, 56, 119, 118, 45, 113, 50, 55, 109, 48, 124, 49, 38, 46, 44, 102, 22, 106, 104, 39, 52, 116, 86, 84, 108, 20, 58, 110, 42, 17, 30, 25, 81, 89, 111, 98, 79, 94, 15, 34, 18, 82, 88, 14, 103, 32, 78, 76, 74, 0, 120, 13, 77, 75, 40, 112, 10, 24, 51, 71, 117, 12, 8, 7, 53, 60, 64, 114, 96, 11, 28, 72, 16, 90, 5, 29, 67, 31, 80, 26, 19, 83, 92, 69, 66, 2, 62, 3, 123, 97, 23, 68, 87, 73, 121, 9, 59, 127, 4, 122, 27, 93, 100, 63, 33, 54, 126, 6, 61, 1, 115, 70, 95, 43, 107, 21, 85, 41, 65, 91, 47, 125, 99, 105, 35, 36, 37, 101], [57, 56, 119, 118, 45, 50, 113, 55, 124, 48, 109, 49, 38, 46, 44, 106, 104, 102, 52, 116, 39, 22, 86, 108, 84, 20, 58, 42, 110, 17, 30, 81, 89, 18, 79, 34, 103, 25, 98, 94, 15, 111, 14, 78, 32, 82, 64, 112, 13, 117, 88, 76, 10, 51, 40, 24, 12, 8, 74, 77, 0, 3, 71, 120, 11, 60, 7, 28, 114, 53, 67, 96, 127, 90, 66, 123, 75, 121, 80, 16, 29, 31, 5, 62, 83, 72, 26, 19, 2, 69, 92, 73, 97, 33, 122, 23, 63, 59, 87, 4, 6, 27, 126, 100, 9, 54, 68, 93, 115, 1, 61, 95, 65, 21, 41, 107, 43, 70, 47, 91, 85, 125, 37, 105, 36, 99, 35, 101], [57, 56, 118, 119, 45, 50, 55, 113, 48, 124, 109, 38, 49, 46, 44, 106, 104, 52, 116, 102, 39, 22, 108, 86, 20, 84, 42, 58, 17, 110, 81, 34, 78, 30, 89, 111, 79, 15, 14, 94, 76, 18, 32, 12, 98, 112, 74, 10, 25, 8, 71, 82, 103, 88, 64, 51, 7, 11, 40, 114, 77, 13, 120, 0, 72, 24, 96, 53, 16, 28, 75, 29, 62, 80, 69, 117, 83, 66, 19, 90, 3, 92, 127, 5, 4, 60, 2, 6, 123, 26, 63, 73, 9, 67, 68, 31, 121, 59, 100, 65, 33, 23, 97, 87, 122, 126, 93, 115, 27, 41, 54, 95, 1, 61, 105, 43, 91, 47, 36, 107, 125, 85, 37, 70, 35, 99, 21, 101], [57, 56, 119, 118, 45, 50, 55, 113, 124, 109, 48, 49, 46, 38, 44, 106, 104, 52, 116, 102, 39, 22, 86, 84, 108, 20, 58, 110, 17, 42, 81, 34, 79, 82, 111, 30, 25, 14, 98, 12, 78, 15, 76, 8, 74, 89, 10, 94, 103, 18, 13, 112, 88, 11, 7, 40, 51, 16, 32, 77, 117, 24, 71, 114, 0, 72, 64, 3, 53, 80, 120, 75, 29, 123, 60, 28, 127, 69, 5, 90, 96, 19, 26, 83, 97, 23, 31, 4, 68, 6, 73, 67, 66, 122, 59, 92, 62, 121, 9, 100, 87, 126, 63, 2, 33, 93, 27, 47, 41, 54, 107, 115, 95, 1, 61, 65, 43, 36, 21, 125, 85, 105, 99, 91, 70, 37, 101, 35], [57, 56, 119, 118, 45, 50, 113, 55, 124, 109, 48, 49, 38, 46, 44, 104, 106, 52, 102, 22, 116, 39, 86, 20, 84, 108, 17, 58, 81, 30, 110, 42, 82, 79, 89, 25, 40, 10, 78, 15, 18, 13, 12, 77, 98, 76, 34, 74, 51, 14, 94, 111, 8, 103, 16, 88, 11, 32, 112, 24, 72, 7, 96, 117, 75, 80, 120, 19, 90, 114, 71, 53, 29, 60, 59, 28, 83, 26, 127, 92, 73, 61, 63, 97, 62, 64, 123, 122, 121, 0, 33, 100, 9, 87, 31, 69, 23, 67, 5, 27, 126, 2, 4, 68, 93, 6, 3, 66, 47, 125, 107, 21, 41, 54, 91, 43, 85, 105, 95, 1, 99, 115, 65, 36, 37, 101, 70, 35], [57, 56, 119, 118, 45, 50, 124, 113, 55, 109, 48, 49, 46, 38, 106, 44, 104, 52, 39, 116, 102, 22, 86, 84, 20, 108, 17, 58, 81, 89, 42, 110, 25, 78, 30, 34, 82, 14, 79, 15, 12, 18, 111, 74, 98, 77, 88, 103, 76, 10, 40, 13, 51, 8, 11, 16, 80, 94, 75, 117, 32, 24, 71, 112, 120, 19, 72, 114, 7, 28, 83, 53, 64, 96, 92, 121, 59, 73, 69, 31, 29, 5, 0, 87, 62, 90, 60, 97, 66, 9, 26, 100, 126, 27, 23, 122, 2, 123, 33, 63, 68, 4, 47, 3, 67, 127, 54, 21, 6, 91, 61, 93, 85, 125, 43, 95, 65, 70, 107, 41, 35, 1, 115, 105, 36, 99, 37, 101], [57, 56, 119, 118, 45, 50, 55, 124, 113, 109, 49, 48, 38, 46, 44, 106, 104, 52, 116, 102, 39, 22, 86, 84, 20, 108, 58, 81, 17, 42, 25, 110, 89, 78, 34, 82, 30, 14, 111, 79, 18, 32, 103, 98, 15, 94, 40, 10, 12, 88, 77, 80, 13, 74, 76, 24, 8, 112, 117, 29, 11, 75, 83, 53, 19, 114, 28, 51, 96, 72, 120, 90, 7, 71, 121, 16, 62, 92, 31, 87, 64, 26, 59, 127, 23, 97, 100, 66, 9, 67, 122, 60, 5, 123, 61, 33, 27, 73, 63, 3, 4, 68, 0, 126, 69, 2, 54, 93, 21, 125, 91, 95, 6, 47, 85, 65, 70, 41, 99, 35, 1, 43, 37, 115, 105, 107, 36, 101], [57, 56, 118, 119, 50, 45, 55, 113, 109, 124, 48, 49, 38, 46, 44, 116, 106, 104, 102, 52, 22, 39, 84, 86, 20, 108, 58, 81, 42, 17, 110, 30, 25, 82, 79, 111, 14, 15, 78, 76, 18, 89, 98, 34, 94, 103, 51, 77, 40, 32, 88, 10, 127, 12, 29, 24, 7, 74, 11, 112, 80, 117, 28, 8, 90, 13, 96, 72, 75, 53, 87, 16, 83, 60, 71, 19, 114, 120, 26, 31, 23, 0, 5, 123, 66, 92, 59, 64, 97, 62, 69, 73, 67, 100, 121, 68, 9, 33, 61, 27, 4, 126, 122, 63, 2, 3, 70, 54, 93, 47, 41, 125, 43, 91, 21, 85, 95, 107, 1, 105, 65, 115, 6, 35, 99, 37, 101, 36], [57, 56, 119, 118, 45, 50, 113, 55, 124, 109, 48, 38, 49, 44, 106, 46, 52, 104, 102, 116, 39, 22, 86, 108, 84, 20, 58, 42, 110, 17, 81, 34, 111, 30, 15, 14, 78, 25, 76, 98, 79, 103, 82, 18, 89, 94, 10, 0, 32, 88, 12, 64, 112, 7, 72, 71, 11, 13, 24, 74, 40, 114, 51, 117, 96, 53, 77, 8, 19, 28, 2, 70, 75, 5, 80, 66, 69, 31, 120, 127, 16, 62, 26, 123, 3, 92, 63, 90, 29, 83, 67, 60, 73, 68, 23, 9, 97, 121, 41, 4, 87, 122, 59, 93, 100, 61, 65, 47, 54, 33, 126, 27, 1, 95, 115, 107, 85, 125, 43, 21, 91, 105, 6, 36, 35, 99, 37, 101], [57, 56, 119, 118, 45, 50, 113, 55, 124, 109, 48, 38, 49, 44, 46, 106, 104, 52, 39, 116, 102, 22, 84, 86, 108, 20, 58, 42, 110, 17, 81, 111, 98, 34, 82, 30, 79, 25, 78, 15, 14, 89, 94, 32, 18, 103, 13, 88, 12, 76, 51, 40, 74, 72, 10, 7, 71, 24, 112, 80, 77, 53, 67, 75, 64, 0, 123, 3, 11, 121, 8, 5, 16, 29, 19, 117, 62, 127, 28, 92, 68, 120, 96, 114, 69, 83, 9, 31, 90, 122, 4, 59, 66, 60, 2, 70, 26, 87, 63, 97, 73, 100, 41, 27, 33, 23, 1, 65, 95, 93, 54, 47, 126, 61, 91, 107, 125, 115, 21, 85, 36, 105, 6, 99, 37, 43, 35, 101], [57, 56, 118, 119, 50, 45, 55, 124, 113, 48, 109, 49, 46, 38, 44, 104, 106, 116, 52, 102, 22, 39, 84, 86, 20, 108, 17, 58, 81, 110, 30, 42, 25, 89, 82, 111, 15, 79, 18, 78, 34, 14, 94, 12, 74, 10, 98, 32, 76, 103, 13, 72, 40, 88, 77, 24, 11, 29, 16, 7, 51, 80, 117, 8, 83, 75, 112, 71, 114, 53, 96, 120, 92, 122, 28, 97, 121, 5, 100, 90, 59, 60, 19, 127, 26, 31, 9, 87, 123, 27, 126, 69, 73, 23, 66, 4, 41, 68, 62, 33, 2, 64, 3, 0, 93, 61, 54, 63, 47, 91, 125, 115, 43, 107, 85, 67, 21, 70, 95, 65, 6, 35, 37, 99, 1, 36, 105, 101], [57, 56, 119, 118, 45, 55, 50, 124, 113, 109, 48, 49, 46, 38, 44, 106, 52, 104, 116, 22, 102, 39, 86, 84, 108, 20, 58, 17, 110, 81, 42, 34, 82, 78, 25, 18, 12, 30, 72, 79, 15, 103, 111, 74, 98, 14, 10, 0, 32, 89, 76, 88, 24, 112, 71, 77, 7, 51, 8, 94, 64, 40, 13, 16, 75, 117, 11, 123, 80, 5, 29, 83, 53, 19, 114, 62, 69, 60, 26, 96, 127, 28, 31, 92, 9, 126, 90, 59, 120, 122, 2, 66, 73, 23, 121, 33, 54, 63, 68, 4, 87, 97, 3, 61, 67, 100, 70, 27, 21, 6, 47, 93, 115, 43, 95, 91, 41, 125, 1, 65, 85, 105, 107, 36, 35, 99, 37, 101], [57, 56, 118, 119, 45, 113, 50, 124, 55, 109, 48, 49, 38, 46, 44, 106, 104, 52, 116, 39, 102, 86, 22, 84, 108, 20, 42, 58, 17, 81, 110, 30, 18, 82, 79, 89, 25, 78, 34, 14, 98, 15, 94, 111, 12, 103, 72, 32, 10, 74, 76, 71, 40, 77, 112, 88, 7, 24, 11, 51, 16, 8, 96, 117, 26, 5, 28, 29, 62, 13, 3, 75, 53, 83, 64, 31, 114, 0, 90, 19, 69, 120, 80, 127, 73, 66, 60, 121, 59, 92, 67, 2, 123, 100, 9, 97, 23, 68, 126, 87, 6, 63, 93, 54, 4, 122, 27, 33, 41, 115, 43, 65, 21, 95, 47, 1, 85, 61, 36, 35, 107, 70, 91, 125, 37, 105, 99, 101], [57, 56, 119, 118, 45, 50, 55, 124, 113, 48, 109, 38, 49, 44, 46, 106, 52, 104, 116, 102, 39, 22, 86, 108, 84, 20, 58, 42, 17, 110, 81, 34, 30, 111, 78, 25, 98, 15, 94, 14, 79, 82, 10, 74, 89, 76, 18, 12, 103, 117, 24, 77, 72, 88, 32, 112, 40, 7, 71, 11, 16, 8, 120, 26, 96, 51, 28, 75, 80, 29, 13, 64, 83, 121, 19, 114, 0, 92, 127, 69, 59, 53, 90, 3, 123, 31, 97, 47, 67, 62, 60, 73, 5, 122, 93, 126, 4, 87, 23, 100, 9, 68, 2, 66, 6, 41, 33, 63, 27, 54, 115, 43, 21, 95, 61, 125, 65, 1, 107, 35, 99, 105, 91, 70, 85, 36, 37, 101], [57, 56, 118, 119, 50, 45, 113, 124, 55, 109, 48, 38, 46, 49, 44, 116, 22, 106, 104, 52, 102, 39, 86, 84, 108, 20, 17, 42, 58, 110, 81, 30, 25, 18, 89, 15, 111, 77, 98, 78, 79, 34, 74, 82, 14, 10, 12, 94, 76, 11, 88, 40, 24, 7, 13, 16, 72, 51, 120, 117, 32, 103, 8, 75, 80, 53, 28, 71, 114, 112, 122, 123, 19, 26, 96, 29, 60, 92, 31, 0, 87, 127, 83, 5, 69, 126, 97, 23, 90, 62, 73, 66, 63, 59, 121, 9, 4, 33, 61, 64, 100, 68, 21, 27, 6, 47, 43, 54, 115, 107, 3, 2, 67, 93, 95, 41, 105, 125, 99, 35, 91, 1, 65, 36, 85, 37, 70, 101], [57, 56, 118, 119, 45, 50, 113, 124, 55, 109, 48, 46, 49, 38, 44, 106, 52, 104, 116, 22, 39, 102, 86, 84, 20, 108, 58, 42, 110, 17, 81, 89, 30, 82, 25, 18, 14, 78, 15, 98, 34, 74, 12, 10, 79, 111, 94, 72, 88, 40, 117, 8, 32, 13, 75, 76, 103, 24, 77, 7, 53, 28, 51, 112, 80, 71, 11, 16, 60, 31, 83, 64, 96, 19, 121, 5, 90, 114, 120, 126, 87, 122, 69, 26, 62, 66, 29, 92, 9, 3, 73, 23, 97, 123, 0, 127, 100, 59, 63, 67, 68, 33, 2, 61, 6, 27, 4, 47, 115, 54, 43, 21, 93, 91, 95, 41, 1, 85, 107, 65, 125, 70, 105, 99, 35, 37, 36, 101], [57, 56, 119, 118, 45, 50, 113, 55, 124, 109, 48, 38, 49, 46, 44, 106, 102, 52, 104, 116, 39, 22, 86, 84, 108, 20, 58, 42, 17, 110, 30, 81, 89, 94, 15, 34, 18, 82, 98, 14, 111, 12, 76, 88, 78, 79, 25, 103, 40, 10, 74, 8, 0, 51, 24, 112, 71, 7, 77, 72, 11, 32, 64, 96, 117, 31, 13, 75, 28, 26, 100, 60, 90, 5, 16, 80, 114, 29, 69, 121, 122, 66, 120, 3, 92, 53, 83, 19, 2, 127, 97, 23, 62, 87, 123, 33, 9, 73, 59, 4, 68, 126, 27, 67, 63, 41, 61, 6, 93, 95, 105, 54, 115, 70, 65, 1, 47, 91, 21, 37, 85, 99, 35, 43, 107, 36, 125, 101], [57, 56, 118, 119, 50, 45, 55, 124, 113, 48, 109, 49, 38, 46, 44, 106, 52, 116, 102, 104, 22, 39, 86, 84, 108, 20, 58, 17, 110, 42, 30, 81, 14, 34, 89, 98, 82, 76, 78, 18, 10, 74, 25, 94, 15, 8, 117, 88, 12, 111, 103, 32, 79, 112, 51, 71, 40, 11, 75, 13, 24, 72, 120, 77, 19, 123, 7, 29, 26, 96, 16, 126, 80, 114, 83, 28, 90, 127, 0, 60, 53, 5, 69, 121, 66, 73, 31, 4, 64, 122, 61, 97, 92, 68, 59, 100, 3, 62, 87, 2, 23, 33, 9, 63, 47, 70, 27, 93, 67, 43, 95, 21, 54, 41, 65, 1, 115, 6, 107, 85, 105, 91, 35, 125, 36, 99, 37, 101], [57, 56, 118, 119, 45, 50, 124, 55, 113, 48, 109, 49, 44, 38, 46, 106, 52, 104, 116, 22, 102, 39, 86, 108, 84, 20, 58, 17, 42, 110, 81, 30, 14, 98, 89, 18, 111, 79, 25, 88, 34, 78, 8, 15, 32, 82, 12, 94, 76, 10, 74, 103, 77, 112, 24, 40, 71, 117, 13, 120, 67, 16, 75, 114, 64, 51, 80, 11, 7, 28, 62, 3, 96, 72, 123, 60, 19, 92, 90, 31, 83, 29, 87, 26, 53, 127, 126, 5, 0, 121, 69, 97, 73, 66, 61, 63, 4, 47, 70, 115, 122, 33, 41, 9, 59, 93, 100, 27, 68, 107, 2, 23, 95, 54, 91, 125, 43, 21, 36, 85, 1, 6, 105, 65, 99, 37, 35, 101], [57, 56, 118, 119, 45, 50, 124, 55, 113, 109, 48, 49, 38, 44, 46, 52, 106, 104, 102, 116, 39, 22, 86, 108, 84, 20, 58, 42, 17, 110, 81, 30, 34, 15, 98, 25, 94, 79, 18, 89, 14, 111, 78, 82, 12, 103, 88, 74, 40, 24, 8, 112, 10, 117, 76, 77, 51, 32, 120, 71, 7, 16, 75, 64, 13, 11, 96, 0, 28, 60, 80, 26, 66, 90, 114, 72, 69, 53, 59, 5, 83, 19, 29, 92, 3, 126, 87, 97, 121, 62, 2, 31, 122, 123, 127, 9, 23, 63, 27, 73, 33, 100, 70, 67, 4, 43, 68, 61, 47, 93, 41, 95, 115, 107, 105, 1, 54, 91, 125, 21, 85, 36, 37, 65, 6, 99, 35, 101], [57, 56, 118, 119, 45, 50, 55, 124, 113, 48, 109, 49, 38, 46, 44, 52, 106, 116, 104, 102, 22, 39, 20, 108, 86, 84, 58, 42, 17, 110, 30, 81, 89, 25, 34, 18, 88, 79, 32, 78, 94, 14, 15, 12, 82, 98, 74, 111, 8, 10, 112, 13, 51, 103, 76, 117, 77, 24, 71, 11, 114, 7, 120, 75, 80, 40, 16, 19, 72, 28, 87, 83, 59, 29, 26, 0, 126, 62, 96, 92, 97, 69, 90, 127, 31, 121, 5, 64, 60, 3, 122, 53, 9, 73, 66, 123, 33, 4, 68, 100, 23, 2, 27, 70, 93, 67, 61, 63, 47, 43, 41, 1, 115, 95, 54, 85, 91, 125, 35, 37, 107, 105, 21, 6, 36, 65, 99, 101], [57, 56, 118, 119, 45, 50, 124, 55, 113, 109, 48, 38, 49, 46, 44, 106, 52, 116, 104, 102, 39, 22, 84, 108, 86, 20, 110, 58, 42, 17, 30, 81, 111, 98, 89, 78, 34, 25, 94, 10, 79, 88, 103, 8, 18, 74, 76, 117, 14, 12, 82, 32, 15, 51, 75, 112, 77, 24, 40, 71, 28, 7, 114, 120, 72, 26, 80, 13, 29, 96, 16, 127, 11, 3, 123, 19, 53, 83, 97, 90, 60, 87, 92, 69, 126, 121, 31, 64, 122, 23, 9, 62, 68, 5, 73, 67, 100, 33, 4, 59, 63, 27, 0, 66, 41, 47, 115, 54, 2, 70, 93, 61, 107, 105, 125, 36, 65, 95, 43, 21, 85, 91, 35, 1, 6, 99, 37, 101], [57, 56, 118, 119, 45, 50, 124, 113, 55, 109, 48, 49, 38, 46, 44, 52, 104, 102, 106, 116, 22, 39, 108, 20, 84, 86, 58, 17, 42, 81, 110, 30, 10, 89, 34, 79, 76, 18, 111, 25, 15, 98, 14, 78, 94, 82, 11, 88, 12, 74, 40, 51, 7, 103, 13, 77, 117, 112, 8, 71, 24, 32, 120, 16, 96, 72, 114, 75, 29, 80, 60, 26, 28, 90, 31, 19, 83, 53, 122, 5, 69, 92, 73, 123, 64, 9, 0, 62, 126, 100, 97, 121, 23, 59, 127, 63, 33, 87, 68, 2, 4, 67, 3, 41, 47, 66, 93, 115, 27, 61, 6, 125, 95, 43, 107, 70, 105, 21, 65, 1, 85, 36, 91, 54, 99, 35, 37, 101], [57, 56, 118, 119, 50, 45, 113, 124, 55, 109, 48, 49, 38, 46, 44, 106, 104, 52, 116, 102, 22, 39, 84, 86, 20, 108, 58, 17, 81, 42, 110, 111, 25, 98, 30, 89, 34, 14, 82, 76, 74, 79, 10, 15, 78, 117, 94, 18, 103, 88, 24, 72, 40, 12, 71, 120, 8, 13, 7, 77, 32, 80, 11, 75, 28, 51, 16, 112, 92, 126, 114, 64, 0, 26, 19, 96, 83, 53, 5, 29, 123, 9, 62, 23, 60, 121, 66, 122, 4, 73, 127, 90, 67, 69, 97, 100, 87, 31, 93, 59, 33, 2, 47, 6, 63, 61, 3, 91, 27, 115, 125, 95, 41, 54, 68, 21, 85, 1, 107, 105, 65, 43, 36, 70, 99, 35, 101, 37], [57, 56, 119, 118, 45, 50, 124, 113, 55, 109, 48, 49, 38, 46, 44, 104, 106, 52, 116, 22, 102, 39, 86, 84, 20, 108, 58, 17, 30, 81, 25, 42, 110, 82, 34, 98, 89, 79, 32, 15, 111, 103, 88, 76, 94, 14, 78, 24, 12, 74, 10, 40, 18, 77, 51, 75, 72, 112, 13, 120, 7, 29, 19, 90, 71, 16, 92, 114, 117, 96, 80, 28, 64, 8, 0, 83, 11, 67, 3, 62, 97, 59, 53, 121, 87, 63, 73, 60, 26, 31, 126, 5, 122, 33, 66, 4, 6, 23, 2, 69, 47, 61, 68, 100, 123, 127, 41, 9, 93, 125, 27, 65, 91, 43, 95, 107, 85, 21, 54, 1, 105, 99, 115, 36, 70, 35, 37, 101], [57, 56, 119, 118, 45, 50, 124, 55, 113, 109, 48, 49, 46, 38, 44, 52, 106, 116, 104, 39, 22, 102, 84, 108, 20, 86, 58, 17, 42, 110, 81, 34, 30, 15, 111, 79, 89, 10, 25, 98, 82, 78, 12, 18, 14, 72, 76, 74, 103, 112, 8, 7, 51, 24, 94, 88, 13, 75, 120, 40, 117, 64, 71, 77, 11, 0, 80, 32, 16, 53, 60, 114, 5, 62, 2, 19, 28, 29, 90, 69, 123, 83, 73, 67, 121, 4, 26, 97, 68, 126, 92, 66, 96, 23, 122, 9, 31, 63, 127, 6, 33, 61, 47, 54, 3, 59, 93, 65, 87, 100, 1, 27, 95, 43, 41, 115, 125, 107, 91, 105, 85, 21, 70, 36, 37, 99, 101, 35], [57, 56, 119, 118, 45, 50, 113, 55, 124, 48, 109, 38, 49, 46, 44, 106, 104, 102, 52, 116, 22, 39, 108, 86, 20, 84, 42, 58, 110, 111, 17, 30, 81, 34, 94, 14, 98, 15, 112, 25, 72, 103, 79, 7, 18, 76, 89, 88, 10, 12, 32, 120, 78, 24, 74, 82, 114, 40, 51, 117, 123, 28, 71, 80, 11, 96, 75, 8, 77, 0, 64, 13, 26, 5, 53, 29, 16, 60, 69, 83, 2, 31, 67, 92, 90, 62, 126, 122, 121, 127, 9, 97, 47, 66, 19, 87, 33, 4, 73, 3, 63, 100, 59, 93, 41, 6, 23, 27, 54, 68, 107, 65, 115, 61, 36, 95, 105, 1, 21, 91, 70, 99, 125, 35, 85, 43, 101, 37], [57, 56, 119, 118, 45, 55, 124, 50, 113, 48, 109, 49, 38, 46, 44, 106, 104, 52, 116, 102, 39, 22, 108, 86, 84, 20, 110, 58, 42, 17, 81, 30, 111, 14, 72, 34, 98, 25, 76, 79, 15, 18, 10, 89, 94, 103, 112, 114, 78, 74, 88, 82, 12, 13, 7, 75, 32, 117, 24, 71, 80, 77, 40, 16, 3, 8, 53, 120, 11, 51, 96, 64, 0, 28, 63, 19, 29, 83, 5, 31, 123, 60, 4, 126, 69, 67, 73, 62, 26, 127, 66, 92, 122, 97, 90, 115, 9, 121, 23, 2, 59, 33, 107, 54, 47, 87, 27, 41, 68, 93, 100, 6, 70, 1, 95, 125, 65, 61, 105, 21, 85, 43, 99, 91, 36, 35, 37, 101], [57, 56, 119, 118, 45, 50, 113, 55, 124, 109, 48, 49, 38, 46, 44, 106, 104, 116, 52, 39, 102, 22, 84, 108, 86, 20, 110, 58, 81, 17, 42, 34, 30, 25, 15, 98, 111, 18, 82, 103, 72, 14, 79, 94, 78, 12, 76, 74, 51, 89, 88, 13, 71, 112, 10, 7, 24, 120, 40, 32, 11, 77, 80, 60, 0, 117, 114, 64, 75, 66, 16, 83, 123, 5, 8, 69, 96, 26, 68, 53, 63, 62, 29, 28, 87, 19, 90, 31, 127, 122, 9, 2, 4, 92, 97, 73, 67, 59, 47, 23, 33, 100, 61, 115, 70, 121, 3, 27, 41, 126, 93, 21, 107, 54, 6, 65, 125, 95, 91, 36, 1, 85, 43, 105, 37, 99, 35, 101], [57, 56, 118, 119, 45, 50, 113, 55, 124, 109, 49, 48, 38, 46, 44, 106, 104, 52, 116, 39, 102, 22, 20, 86, 108, 84, 58, 81, 17, 42, 110, 30, 25, 34, 111, 82, 14, 18, 79, 98, 12, 72, 76, 94, 10, 15, 89, 88, 117, 78, 51, 74, 32, 40, 103, 112, 75, 7, 8, 71, 80, 16, 24, 13, 114, 77, 60, 11, 120, 83, 28, 19, 53, 31, 96, 90, 69, 9, 0, 26, 5, 87, 62, 59, 73, 29, 97, 92, 23, 123, 66, 126, 121, 64, 3, 127, 70, 122, 68, 4, 27, 33, 2, 115, 47, 67, 61, 63, 100, 41, 93, 54, 65, 43, 125, 21, 91, 95, 1, 105, 6, 85, 107, 36, 99, 37, 101, 35]], "model.layers.7.self_attn.q_proj": [[38, 109, 43, 89, 45, 18, 93, 60, 107, 78, 12, 22, 10, 8, 17, 5, 19, 77, 80, 79, 56, 24, 91, 71, 68, 72, 3, 92, 26, 115, 25, 51, 20, 23, 57, 69, 15, 50, 81, 4, 58, 65, 102, 14, 127, 86, 97, 125, 27, 70, 34, 126, 28, 90, 88, 32, 67, 108, 30, 76, 9, 122, 29, 111, 123, 1, 6, 13, 124, 116, 74, 110, 82, 16, 63, 11, 96, 33, 75, 117, 55, 113, 52, 83, 87, 62, 35, 112, 94, 95, 7, 59, 100, 73, 119, 106, 40, 85, 54, 21, 0, 48, 44, 61, 114, 39, 105, 47, 31, 49, 120, 103, 53, 99, 36, 118, 84, 121, 41, 37, 66, 2, 98, 46, 101, 64, 104, 42], [38, 109, 43, 60, 10, 12, 19, 77, 5, 80, 93, 45, 18, 68, 78, 71, 107, 25, 22, 89, 8, 1, 73, 69, 79, 17, 3, 70, 29, 125, 65, 51, 88, 72, 87, 23, 0, 16, 14, 66, 122, 20, 4, 56, 74, 94, 26, 91, 84, 15, 32, 28, 86, 85, 7, 115, 21, 34, 24, 11, 76, 82, 6, 30, 90, 83, 92, 81, 58, 75, 116, 102, 97, 2, 13, 120, 123, 27, 95, 64, 126, 9, 100, 35, 33, 67, 98, 31, 96, 52, 46, 36, 111, 108, 47, 39, 127, 105, 99, 118, 62, 117, 112, 110, 59, 49, 55, 37, 106, 50, 54, 63, 104, 61, 103, 101, 57, 40, 114, 113, 119, 41, 48, 124, 53, 42, 44, 121], [38, 109, 43, 10, 67, 19, 93, 77, 45, 107, 80, 78, 12, 65, 6, 79, 23, 4, 5, 89, 7, 71, 3, 72, 0, 69, 25, 68, 8, 74, 1, 22, 60, 20, 14, 17, 18, 76, 16, 21, 51, 28, 75, 13, 115, 64, 88, 9, 11, 70, 73, 90, 84, 102, 29, 57, 126, 2, 82, 15, 86, 83, 87, 125, 24, 66, 34, 30, 91, 26, 35, 85, 120, 92, 94, 32, 81, 56, 55, 122, 58, 116, 98, 95, 52, 62, 31, 111, 97, 106, 50, 27, 124, 96, 118, 123, 48, 110, 40, 47, 36, 127, 99, 33, 108, 112, 113, 49, 54, 37, 117, 39, 63, 104, 46, 100, 105, 42, 103, 59, 121, 44, 53, 101, 61, 114, 41, 119], [109, 43, 60, 45, 107, 123, 34, 50, 58, 62, 115, 56, 35, 113, 110, 57, 122, 48, 63, 108, 98, 126, 47, 127, 112, 124, 119, 55, 111, 117, 49, 121, 59, 53, 114, 52, 118, 54, 39, 125, 61, 116, 44, 51, 37, 120, 106, 46, 92, 40, 42, 36, 105, 41, 103, 104, 33, 94, 38, 101, 100, 31, 97, 99, 86, 82, 95, 29, 87, 96, 16, 22, 30, 81, 78, 89, 32, 91, 84, 28, 27, 10, 93, 76, 77, 20, 85, 13, 8, 83, 24, 11, 80, 88, 23, 19, 21, 102, 25, 90, 79, 14, 17, 71, 74, 26, 18, 5, 68, 15, 9, 12, 70, 73, 66, 1, 2, 3, 72, 7, 4, 69, 6, 75, 0, 67, 65, 64], [105, 97, 118, 23, 90, 84, 79, 82, 117, 13, 93, 33, 47, 9, 54, 111, 29, 41, 10, 11, 71, 85, 20, 22, 25, 15, 17, 26, 18, 74, 56, 83, 89, 77, 75, 87, 121, 80, 21, 91, 19, 81, 86, 116, 7, 6, 92, 76, 32, 38, 30, 95, 88, 27, 3, 78, 5, 49, 48, 53, 16, 12, 69, 52, 94, 24, 31, 96, 112, 8, 108, 73, 61, 57, 46, 43, 102, 124, 4, 126, 58, 35, 100, 113, 99, 14, 28, 68, 42, 110, 107, 101, 123, 115, 55, 63, 37, 106, 119, 39, 62, 51, 103, 98, 109, 34, 67, 114, 120, 127, 60, 72, 40, 70, 125, 59, 122, 44, 45, 104, 50, 36, 1, 65, 66, 2, 0, 64], [105, 118, 64, 0, 2, 97, 1, 79, 65, 13, 66, 67, 41, 71, 4, 84, 29, 11, 26, 10, 9, 82, 69, 68, 23, 117, 70, 111, 6, 54, 87, 101, 7, 3, 76, 74, 78, 20, 8, 5, 73, 18, 12, 90, 121, 89, 75, 86, 77, 16, 93, 15, 14, 25, 38, 115, 47, 55, 17, 45, 80, 83, 37, 72, 58, 59, 48, 92, 22, 51, 39, 114, 85, 56, 49, 53, 34, 21, 81, 91, 88, 27, 120, 19, 24, 62, 46, 98, 100, 110, 124, 44, 28, 116, 96, 119, 106, 42, 94, 57, 102, 40, 61, 60, 126, 107, 127, 52, 31, 30, 103, 109, 113, 108, 122, 125, 63, 36, 43, 104, 95, 112, 123, 99, 35, 50, 32, 33], [105, 47, 118, 93, 117, 111, 90, 89, 97, 101, 116, 56, 85, 54, 23, 41, 25, 80, 33, 52, 121, 32, 96, 61, 37, 55, 24, 17, 49, 84, 31, 29, 53, 123, 21, 99, 91, 58, 98, 88, 95, 75, 35, 92, 60, 113, 34, 77, 126, 100, 12, 18, 127, 48, 30, 36, 45, 115, 108, 39, 73, 59, 112, 102, 15, 28, 43, 107, 26, 38, 27, 110, 81, 46, 114, 82, 16, 40, 103, 106, 87, 63, 94, 120, 104, 20, 62, 8, 44, 42, 122, 69, 109, 124, 7, 119, 74, 125, 51, 57, 50, 78, 11, 86, 19, 79, 83, 70, 22, 14, 67, 76, 72, 68, 13, 9, 10, 71, 5, 4, 6, 3, 66, 65, 2, 1, 0, 64], [105, 47, 118, 93, 117, 101, 90, 97, 116, 111, 54, 56, 52, 123, 89, 24, 85, 33, 55, 61, 23, 53, 58, 21, 49, 60, 121, 17, 37, 59, 75, 96, 31, 113, 115, 25, 112, 120, 106, 126, 12, 80, 114, 57, 127, 107, 48, 29, 41, 43, 77, 46, 110, 108, 45, 63, 62, 27, 124, 99, 73, 109, 40, 44, 51, 119, 42, 98, 38, 39, 87, 103, 102, 84, 50, 122, 125, 104, 32, 91, 34, 81, 26, 36, 20, 95, 16, 35, 18, 100, 30, 15, 69, 8, 7, 92, 86, 94, 28, 22, 88, 78, 74, 72, 82, 14, 70, 83, 79, 11, 10, 71, 67, 76, 19, 13, 9, 4, 68, 5, 6, 3, 66, 2, 65, 1, 64, 0], [103, 42, 34, 30, 23, 85, 106, 83, 81, 127, 70, 78, 50, 16, 39, 94, 53, 10, 41, 82, 12, 15, 4, 112, 13, 74, 20, 54, 25, 5, 8, 66, 38, 48, 19, 117, 63, 76, 93, 121, 72, 90, 88, 123, 111, 124, 24, 32, 107, 14, 118, 40, 17, 73, 55, 1, 87, 115, 75, 105, 80, 125, 21, 26, 67, 60, 29, 47, 52, 22, 101, 113, 58, 95, 36, 84, 126, 120, 11, 97, 6, 49, 100, 104, 62, 79, 59, 86, 46, 43, 45, 122, 7, 69, 102, 44, 109, 89, 33, 3, 99, 57, 77, 37, 108, 98, 27, 114, 28, 110, 18, 68, 96, 71, 119, 56, 116, 51, 35, 31, 61, 91, 65, 9, 64, 92, 0, 2], [103, 42, 34, 30, 85, 106, 23, 81, 83, 53, 50, 94, 78, 41, 127, 16, 39, 62, 15, 121, 54, 38, 44, 112, 125, 25, 12, 124, 74, 93, 63, 22, 47, 87, 13, 48, 82, 118, 111, 60, 11, 24, 5, 33, 113, 59, 109, 40, 117, 8, 55, 80, 70, 72, 107, 122, 115, 114, 123, 51, 49, 95, 32, 56, 4, 76, 110, 46, 21, 101, 45, 119, 28, 36, 58, 116, 90, 10, 57, 120, 126, 79, 29, 61, 104, 91, 108, 99, 26, 97, 92, 19, 86, 37, 20, 96, 84, 52, 102, 89, 100, 105, 43, 17, 1, 35, 75, 3, 31, 77, 14, 27, 88, 7, 98, 66, 18, 67, 6, 71, 73, 9, 64, 68, 69, 65, 0, 2], [103, 42, 34, 30, 106, 50, 85, 81, 54, 41, 23, 16, 83, 53, 112, 48, 121, 38, 44, 95, 4, 70, 94, 124, 13, 127, 20, 26, 63, 123, 125, 113, 93, 52, 60, 10, 62, 109, 59, 29, 115, 47, 78, 15, 122, 19, 76, 111, 31, 87, 8, 46, 28, 25, 45, 118, 40, 92, 82, 32, 90, 66, 58, 61, 55, 97, 110, 88, 114, 43, 105, 36, 117, 107, 74, 51, 100, 101, 119, 22, 49, 116, 37, 126, 33, 9, 80, 89, 86, 104, 99, 120, 73, 56, 57, 5, 24, 84, 21, 27, 35, 17, 96, 102, 69, 108, 67, 91, 12, 1, 79, 14, 11, 65, 75, 39, 68, 72, 18, 7, 77, 98, 71, 6, 2, 0, 3, 64], [103, 42, 34, 30, 83, 81, 85, 106, 12, 23, 50, 74, 72, 15, 78, 127, 16, 54, 69, 5, 13, 4, 113, 48, 60, 124, 8, 39, 76, 41, 1, 2, 24, 53, 80, 87, 11, 55, 17, 65, 21, 121, 94, 6, 63, 68, 3, 45, 19, 25, 62, 32, 38, 75, 126, 70, 52, 82, 93, 9, 84, 20, 125, 67, 123, 90, 14, 115, 26, 79, 44, 22, 10, 112, 104, 77, 66, 96, 95, 120, 91, 97, 117, 86, 100, 28, 98, 0, 73, 88, 71, 36, 59, 111, 89, 108, 64, 18, 29, 114, 105, 61, 7, 47, 101, 27, 99, 43, 107, 122, 31, 35, 92, 118, 51, 119, 33, 102, 58, 116, 49, 46, 109, 110, 40, 37, 57, 56], [104, 121, 100, 33, 17, 13, 21, 23, 79, 83, 90, 122, 72, 11, 8, 26, 28, 67, 112, 37, 30, 60, 59, 6, 61, 126, 45, 80, 53, 22, 115, 125, 81, 48, 109, 20, 4, 10, 51, 42, 18, 14, 46, 86, 19, 15, 40, 77, 96, 78, 34, 97, 75, 69, 99, 85, 27, 5, 1, 41, 29, 74, 39, 0, 16, 87, 94, 50, 25, 118, 111, 24, 68, 82, 12, 92, 84, 101, 73, 52, 103, 32, 117, 98, 66, 93, 95, 110, 113, 123, 47, 62, 31, 7, 107, 71, 76, 9, 70, 89, 2, 58, 108, 102, 124, 106, 55, 54, 63, 91, 88, 116, 3, 105, 64, 127, 120, 35, 49, 44, 119, 43, 57, 38, 65, 56, 114, 36], [104, 100, 121, 33, 28, 90, 21, 23, 17, 13, 109, 112, 42, 83, 115, 92, 48, 122, 30, 79, 26, 116, 18, 86, 118, 37, 61, 123, 45, 95, 117, 51, 44, 108, 39, 101, 58, 11, 120, 52, 34, 35, 107, 60, 110, 29, 124, 96, 41, 98, 25, 22, 32, 31, 82, 126, 80, 53, 16, 24, 62, 59, 27, 54, 103, 125, 43, 111, 57, 87, 119, 46, 6, 114, 36, 56, 105, 84, 106, 63, 102, 50, 85, 49, 19, 97, 55, 127, 20, 113, 99, 91, 14, 88, 93, 81, 38, 94, 10, 47, 77, 75, 72, 89, 74, 78, 12, 76, 70, 40, 67, 3, 8, 15, 69, 7, 9, 68, 5, 73, 71, 65, 2, 1, 66, 64, 4, 0], [104, 121, 33, 100, 93, 37, 79, 23, 83, 17, 21, 109, 112, 13, 8, 11, 90, 6, 67, 26, 35, 122, 59, 86, 40, 46, 3, 1, 125, 126, 71, 30, 82, 61, 45, 81, 48, 28, 42, 51, 85, 115, 60, 29, 9, 18, 4, 92, 43, 15, 52, 77, 74, 101, 25, 80, 22, 34, 65, 105, 75, 107, 19, 111, 50, 108, 10, 94, 69, 20, 78, 58, 16, 120, 39, 62, 47, 7, 119, 123, 88, 70, 96, 73, 99, 98, 31, 87, 57, 14, 32, 66, 106, 49, 102, 5, 103, 116, 110, 53, 38, 114, 76, 68, 63, 55, 124, 64, 56, 27, 113, 127, 12, 118, 84, 41, 117, 24, 91, 54, 95, 89, 44, 2, 72, 36, 97, 0], [121, 104, 100, 33, 28, 112, 122, 90, 21, 23, 83, 37, 93, 13, 116, 126, 30, 45, 92, 91, 125, 101, 79, 52, 17, 86, 115, 61, 60, 59, 46, 26, 51, 109, 35, 82, 85, 57, 48, 18, 123, 42, 118, 105, 19, 15, 97, 43, 39, 11, 54, 29, 107, 111, 103, 22, 32, 84, 31, 53, 14, 117, 95, 47, 96, 38, 120, 58, 119, 41, 108, 49, 106, 113, 110, 87, 27, 127, 99, 25, 70, 55, 34, 44, 124, 94, 6, 114, 89, 62, 36, 102, 24, 63, 75, 9, 50, 77, 98, 81, 72, 78, 20, 74, 56, 88, 10, 12, 16, 3, 80, 67, 71, 7, 76, 73, 2, 68, 8, 40, 69, 5, 66, 64, 1, 65, 4, 0], [37, 40, 105, 93, 55, 30, 124, 14, 18, 87, 63, 20, 94, 126, 48, 29, 88, 49, 104, 41, 60, 16, 117, 43, 58, 76, 89, 50, 62, 51, 57, 116, 53, 52, 39, 61, 111, 72, 119, 102, 38, 109, 100, 42, 44, 120, 59, 112, 56, 46, 122, 121, 110, 54, 114, 45, 113, 127, 123, 103, 115, 125, 27, 47, 108, 106, 118, 96, 107, 95, 86, 21, 98, 82, 26, 99, 36, 32, 33, 31, 35, 34, 23, 97, 24, 101, 69, 77, 81, 90, 25, 85, 10, 83, 92, 91, 78, 15, 28, 12, 84, 13, 74, 22, 17, 9, 80, 5, 6, 8, 79, 11, 75, 19, 1, 73, 3, 65, 71, 70, 66, 64, 2, 7, 67, 68, 4, 0], [105, 37, 40, 93, 87, 41, 18, 16, 76, 14, 20, 124, 29, 55, 69, 63, 72, 10, 57, 94, 88, 60, 49, 1, 61, 126, 15, 48, 23, 77, 117, 90, 114, 58, 27, 96, 74, 116, 89, 52, 78, 25, 12, 62, 122, 24, 81, 26, 13, 8, 30, 79, 82, 50, 102, 6, 84, 3, 68, 119, 112, 95, 86, 51, 100, 80, 104, 43, 83, 46, 17, 53, 21, 44, 91, 56, 59, 120, 71, 22, 39, 92, 19, 107, 97, 85, 45, 28, 11, 123, 4, 70, 101, 111, 38, 32, 75, 54, 5, 67, 108, 31, 121, 9, 34, 36, 65, 103, 33, 73, 35, 0, 2, 127, 7, 110, 113, 125, 98, 109, 42, 64, 115, 118, 99, 47, 66, 106], [105, 37, 40, 93, 104, 29, 55, 18, 87, 14, 16, 20, 63, 124, 76, 72, 49, 30, 60, 126, 27, 57, 94, 48, 117, 61, 88, 10, 114, 69, 77, 116, 89, 62, 50, 56, 100, 24, 90, 51, 96, 102, 119, 112, 86, 52, 15, 54, 58, 23, 127, 122, 12, 95, 8, 84, 53, 38, 79, 43, 21, 26, 92, 111, 82, 34, 108, 107, 113, 13, 32, 120, 80, 91, 25, 81, 98, 110, 28, 22, 103, 121, 44, 123, 19, 31, 39, 59, 45, 1, 115, 33, 101, 125, 47, 36, 35, 106, 46, 85, 99, 118, 42, 78, 17, 97, 109, 83, 41, 74, 6, 71, 70, 11, 3, 75, 68, 9, 5, 2, 4, 65, 66, 73, 7, 0, 64, 67], [40, 37, 105, 63, 93, 14, 16, 87, 10, 20, 18, 76, 55, 69, 29, 72, 57, 104, 30, 124, 3, 60, 41, 94, 74, 58, 61, 1, 26, 114, 88, 49, 15, 13, 23, 84, 24, 12, 126, 90, 17, 67, 95, 112, 70, 11, 71, 82, 78, 62, 116, 8, 117, 48, 52, 122, 81, 6, 22, 68, 0, 102, 7, 75, 83, 50, 77, 79, 46, 56, 98, 123, 80, 96, 4, 53, 27, 65, 89, 43, 21, 103, 86, 5, 39, 85, 119, 101, 100, 113, 51, 44, 47, 45, 25, 38, 28, 59, 92, 32, 19, 127, 111, 107, 91, 9, 120, 73, 110, 34, 64, 36, 115, 54, 108, 31, 109, 99, 35, 2, 121, 97, 118, 42, 66, 125, 33, 106], [105, 99, 89, 96, 48, 54, 83, 92, 84, 16, 14, 25, 11, 112, 22, 19, 75, 78, 72, 27, 77, 18, 80, 115, 41, 69, 81, 32, 24, 10, 36, 108, 30, 5, 44, 85, 93, 26, 13, 88, 104, 63, 119, 117, 29, 82, 15, 17, 73, 118, 20, 124, 94, 6, 9, 66, 62, 67, 49, 79, 12, 28, 91, 70, 126, 33, 23, 56, 90, 59, 21, 120, 58, 87, 76, 31, 71, 7, 74, 8, 65, 37, 4, 0, 1, 38, 98, 97, 43, 127, 46, 95, 2, 68, 60, 64, 86, 3, 51, 116, 107, 52, 47, 101, 110, 123, 50, 35, 100, 102, 109, 40, 53, 34, 122, 103, 125, 114, 121, 106, 61, 39, 42, 113, 45, 111, 57, 55], [44, 36, 104, 54, 92, 100, 95, 48, 105, 108, 28, 47, 96, 53, 32, 98, 37, 26, 27, 123, 40, 24, 97, 60, 127, 51, 112, 91, 58, 55, 85, 88, 52, 78, 93, 31, 33, 38, 21, 90, 113, 107, 49, 99, 124, 10, 110, 120, 115, 18, 89, 63, 30, 45, 34, 83, 62, 82, 56, 111, 94, 29, 122, 102, 84, 106, 118, 74, 103, 116, 117, 50, 46, 43, 22, 81, 101, 121, 119, 61, 114, 17, 39, 23, 57, 109, 59, 16, 86, 79, 126, 14, 87, 35, 70, 42, 41, 80, 5, 125, 9, 15, 75, 2, 20, 3, 66, 64, 12, 7, 73, 1, 77, 19, 76, 6, 4, 68, 0, 13, 25, 11, 67, 69, 72, 71, 8, 65], [48, 54, 37, 105, 108, 104, 36, 107, 47, 115, 63, 119, 95, 50, 44, 56, 112, 114, 118, 94, 127, 30, 117, 62, 110, 52, 55, 123, 116, 121, 53, 38, 58, 51, 124, 40, 59, 111, 113, 92, 34, 101, 120, 60, 125, 32, 46, 122, 126, 109, 57, 61, 100, 49, 24, 102, 33, 45, 98, 106, 43, 103, 28, 21, 99, 85, 42, 39, 96, 31, 41, 15, 97, 25, 29, 87, 19, 18, 81, 11, 23, 88, 12, 90, 73, 77, 17, 84, 26, 75, 91, 8, 93, 79, 22, 78, 7, 20, 5, 82, 80, 4, 35, 2, 0, 1, 64, 76, 6, 3, 14, 86, 74, 72, 70, 83, 71, 9, 89, 69, 27, 10, 16, 65, 68, 66, 67, 13], [54, 105, 48, 36, 123, 41, 25, 44, 127, 98, 99, 112, 34, 96, 115, 50, 47, 52, 60, 121, 118, 57, 117, 122, 64, 107, 63, 114, 108, 74, 119, 45, 120, 56, 22, 30, 113, 55, 8, 126, 111, 51, 58, 53, 62, 106, 110, 116, 125, 1, 124, 104, 59, 37, 61, 94, 80, 100, 68, 43, 75, 97, 109, 84, 67, 49, 81, 14, 38, 77, 6, 46, 42, 73, 103, 40, 5, 39, 79, 2, 24, 18, 4, 16, 3, 92, 89, 71, 21, 69, 12, 102, 28, 29, 33, 15, 20, 101, 88, 26, 85, 86, 7, 17, 31, 27, 95, 82, 91, 35, 32, 87, 13, 93, 76, 70, 19, 23, 90, 9, 0, 83, 11, 65, 78, 10, 66, 72], [37, 114, 49, 50, 54, 113, 94, 91, 101, 28, 118, 84, 61, 119, 25, 62, 45, 58, 11, 32, 112, 56, 87, 60, 29, 22, 124, 15, 31, 14, 90, 79, 116, 96, 81, 48, 33, 97, 82, 55, 24, 47, 110, 83, 36, 35, 117, 46, 95, 93, 109, 53, 98, 51, 99, 27, 18, 126, 127, 34, 30, 39, 115, 107, 102, 38, 40, 43, 63, 100, 52, 104, 44, 21, 105, 57, 125, 122, 123, 103, 92, 121, 41, 59, 120, 42, 8, 16, 108, 89, 106, 20, 88, 85, 26, 111, 76, 23, 86, 9, 75, 78, 5, 13, 19, 12, 6, 17, 80, 4, 10, 73, 7, 77, 2, 74, 3, 1, 71, 66, 72, 0, 70, 68, 69, 64, 65, 67], [114, 54, 49, 113, 50, 124, 119, 118, 109, 62, 123, 61, 56, 48, 47, 55, 112, 53, 58, 36, 127, 46, 37, 120, 63, 45, 126, 52, 60, 107, 125, 44, 115, 51, 111, 121, 117, 110, 122, 116, 59, 43, 57, 108, 17, 42, 105, 39, 106, 41, 101, 104, 20, 102, 38, 40, 82, 80, 30, 96, 19, 75, 79, 103, 86, 77, 78, 76, 95, 85, 24, 93, 100, 27, 97, 98, 99, 35, 34, 33, 74, 18, 32, 89, 72, 28, 71, 9, 23, 29, 14, 90, 15, 31, 92, 94, 87, 12, 88, 69, 70, 66, 13, 64, 65, 91, 84, 21, 81, 67, 26, 73, 83, 11, 25, 68, 22, 16, 6, 10, 4, 5, 1, 3, 8, 7, 0, 2], [37, 49, 114, 50, 54, 113, 101, 94, 62, 25, 91, 11, 84, 83, 119, 87, 81, 118, 32, 79, 8, 56, 5, 15, 76, 9, 28, 22, 21, 45, 6, 60, 4, 58, 61, 7, 124, 13, 110, 16, 33, 75, 112, 85, 14, 10, 18, 30, 116, 3, 82, 90, 46, 24, 1, 73, 53, 23, 96, 77, 89, 109, 78, 117, 2, 63, 74, 125, 47, 12, 80, 52, 31, 126, 20, 17, 71, 48, 93, 57, 127, 66, 19, 115, 98, 105, 55, 27, 36, 44, 86, 29, 88, 40, 35, 95, 26, 43, 97, 122, 92, 72, 111, 68, 59, 38, 123, 41, 0, 51, 107, 34, 103, 39, 121, 42, 120, 102, 99, 106, 108, 100, 104, 70, 69, 65, 64, 67], [49, 37, 114, 54, 50, 113, 5, 9, 76, 6, 8, 7, 94, 4, 77, 79, 75, 12, 87, 16, 3, 25, 30, 22, 73, 11, 74, 83, 78, 80, 82, 17, 124, 2, 81, 71, 72, 91, 10, 45, 19, 56, 60, 62, 119, 61, 112, 1, 32, 118, 13, 15, 58, 84, 85, 63, 53, 26, 101, 21, 20, 117, 110, 109, 55, 23, 90, 93, 92, 66, 14, 89, 116, 31, 88, 46, 29, 48, 18, 107, 28, 24, 27, 125, 102, 127, 115, 57, 0, 126, 36, 52, 51, 47, 86, 123, 44, 70, 96, 121, 68, 43, 33, 42, 59, 39, 122, 104, 111, 120, 98, 67, 40, 69, 41, 100, 105, 35, 34, 38, 108, 97, 95, 103, 106, 65, 99, 64], [41, 101, 25, 23, 85, 110, 18, 105, 15, 34, 13, 20, 11, 91, 124, 30, 113, 9, 14, 83, 43, 8, 109, 24, 126, 94, 82, 68, 6, 51, 84, 48, 107, 118, 92, 29, 27, 59, 112, 111, 57, 4, 45, 77, 114, 44, 87, 88, 122, 89, 36, 53, 62, 17, 26, 98, 79, 31, 32, 16, 76, 80, 58, 117, 78, 103, 90, 96, 115, 66, 50, 75, 72, 21, 67, 38, 56, 2, 99, 55, 10, 47, 95, 37, 121, 0, 61, 52, 69, 5, 102, 120, 86, 108, 19, 28, 63, 60, 46, 22, 116, 106, 39, 123, 35, 127, 54, 65, 104, 71, 70, 1, 7, 97, 73, 119, 93, 12, 3, 33, 74, 40, 49, 42, 81, 100, 125, 64], [41, 101, 25, 85, 18, 23, 20, 105, 15, 11, 34, 13, 91, 124, 14, 9, 6, 51, 113, 2, 118, 8, 82, 112, 114, 29, 27, 53, 83, 98, 84, 68, 81, 109, 75, 48, 78, 32, 43, 107, 89, 30, 66, 87, 92, 111, 94, 126, 70, 36, 90, 45, 79, 110, 17, 44, 88, 122, 62, 57, 59, 117, 21, 7, 99, 103, 60, 127, 47, 76, 19, 38, 26, 42, 31, 24, 120, 52, 58, 16, 46, 69, 108, 77, 0, 35, 119, 72, 61, 115, 56, 80, 37, 22, 123, 102, 55, 63, 121, 10, 86, 95, 28, 67, 96, 97, 125, 100, 106, 39, 71, 54, 116, 50, 73, 93, 5, 74, 12, 40, 33, 65, 3, 49, 4, 1, 104, 64], [41, 101, 109, 110, 91, 25, 20, 18, 34, 85, 105, 23, 14, 124, 11, 15, 9, 13, 43, 53, 24, 113, 82, 84, 29, 8, 107, 45, 27, 126, 83, 6, 111, 90, 127, 59, 68, 98, 112, 51, 28, 44, 120, 78, 56, 47, 121, 118, 117, 114, 116, 62, 50, 106, 122, 55, 57, 92, 58, 95, 93, 94, 81, 36, 52, 97, 16, 87, 32, 30, 60, 102, 108, 42, 99, 38, 66, 75, 26, 40, 96, 119, 88, 63, 67, 33, 31, 89, 35, 37, 48, 80, 123, 115, 70, 61, 49, 46, 39, 104, 71, 125, 2, 22, 103, 76, 77, 86, 12, 54, 17, 79, 0, 4, 19, 21, 72, 7, 10, 5, 69, 100, 73, 65, 74, 1, 3, 64], [41, 101, 25, 85, 91, 105, 23, 15, 18, 20, 34, 13, 98, 113, 124, 81, 110, 29, 30, 11, 48, 95, 126, 43, 92, 114, 27, 9, 53, 14, 109, 84, 36, 44, 94, 24, 118, 45, 35, 26, 83, 100, 111, 89, 51, 107, 46, 57, 82, 120, 76, 123, 112, 61, 119, 38, 40, 103, 122, 121, 31, 56, 87, 116, 8, 62, 60, 50, 28, 106, 99, 47, 127, 125, 39, 80, 97, 117, 78, 42, 33, 6, 32, 58, 54, 59, 86, 102, 16, 88, 73, 21, 55, 63, 77, 22, 17, 96, 115, 52, 49, 79, 108, 4, 93, 12, 90, 104, 19, 68, 37, 66, 75, 10, 70, 74, 72, 5, 67, 7, 71, 1, 2, 69, 64, 3, 0, 65]], "model.layers.7.self_attn.k_proj": [[45, 107, 102, 0, 109, 60, 1, 12, 3, 77, 70, 19, 71, 43, 10, 8, 78, 115, 80, 29, 68, 22, 25, 5, 18, 17, 11, 125, 2, 73, 57, 126, 56, 48, 122, 111, 124, 79, 55, 110, 113, 63, 20, 66, 58, 67, 52, 47, 127, 62, 117, 49, 59, 114, 112, 96, 121, 51, 44, 54, 23, 118, 6, 119, 61, 92, 123, 53, 27, 95, 46, 116, 98, 50, 120, 106, 69, 89, 100, 64, 65, 21, 4, 108, 75, 9, 7, 41, 42, 105, 40, 39, 87, 30, 35, 24, 104, 94, 103, 36, 97, 31, 26, 101, 84, 37, 32, 33, 99, 90, 28, 85, 88, 81, 72, 93, 34, 15, 74, 76, 91, 82, 14, 16, 83, 13, 86, 38], [41, 118, 64, 111, 117, 33, 79, 82, 84, 1, 13, 23, 90, 93, 56, 71, 2, 11, 9, 4, 89, 85, 10, 105, 17, 65, 5, 37, 47, 121, 6, 30, 76, 67, 14, 116, 32, 80, 66, 72, 101, 22, 24, 115, 0, 48, 19, 28, 59, 29, 127, 44, 126, 49, 95, 110, 3, 26, 55, 112, 88, 83, 16, 38, 43, 35, 94, 52, 98, 27, 18, 46, 81, 40, 120, 107, 12, 100, 21, 102, 62, 124, 119, 61, 106, 113, 99, 69, 34, 53, 91, 123, 97, 103, 96, 42, 63, 57, 39, 51, 109, 114, 60, 36, 31, 58, 73, 104, 108, 54, 8, 122, 125, 68, 50, 25, 45, 78, 75, 92, 20, 77, 87, 86, 15, 74, 7, 70], [106, 39, 94, 98, 85, 23, 83, 127, 74, 12, 81, 78, 72, 16, 15, 50, 112, 65, 68, 42, 6, 41, 53, 69, 118, 2, 114, 49, 13, 0, 124, 88, 77, 18, 52, 111, 55, 8, 79, 38, 126, 125, 115, 34, 29, 36, 102, 109, 108, 120, 54, 7, 64, 107, 31, 40, 63, 110, 9, 47, 26, 101, 75, 60, 48, 3, 71, 61, 35, 122, 67, 117, 104, 25, 22, 58, 100, 43, 119, 62, 80, 113, 66, 56, 5, 11, 121, 14, 73, 105, 90, 57, 33, 123, 44, 92, 103, 46, 89, 24, 86, 116, 93, 37, 82, 27, 59, 99, 96, 19, 91, 45, 32, 95, 84, 28, 20, 97, 30, 76, 10, 17, 4, 51, 1, 87, 70, 21], [40, 121, 97, 90, 23, 21, 83, 36, 79, 48, 17, 82, 11, 13, 86, 8, 122, 73, 0, 30, 78, 50, 51, 107, 6, 109, 116, 69, 60, 59, 10, 65, 115, 58, 110, 101, 45, 29, 9, 4, 16, 68, 53, 123, 28, 84, 93, 66, 67, 39, 100, 126, 61, 37, 27, 42, 12, 89, 55, 124, 112, 3, 56, 96, 32, 47, 57, 70, 120, 125, 117, 103, 119, 2, 114, 95, 94, 35, 98, 14, 74, 54, 92, 106, 46, 43, 75, 88, 99, 41, 111, 105, 62, 52, 118, 49, 108, 34, 63, 38, 127, 91, 113, 44, 5, 31, 102, 15, 19, 76, 1, 71, 7, 18, 22, 24, 64, 25, 20, 72, 80, 77, 85, 26, 81, 87, 104, 33], [104, 41, 101, 29, 87, 55, 20, 63, 18, 16, 76, 10, 14, 60, 37, 57, 7, 0, 65, 72, 77, 67, 52, 69, 112, 58, 124, 113, 114, 71, 61, 66, 5, 86, 30, 17, 9, 126, 24, 15, 27, 75, 8, 93, 123, 4, 12, 102, 89, 62, 43, 64, 94, 21, 53, 2, 90, 70, 83, 13, 85, 39, 79, 19, 96, 88, 56, 100, 117, 3, 107, 116, 108, 45, 127, 82, 6, 91, 46, 111, 119, 47, 11, 121, 81, 23, 51, 54, 25, 49, 120, 73, 95, 32, 68, 42, 59, 26, 78, 34, 36, 28, 122, 80, 22, 31, 98, 92, 110, 84, 109, 103, 97, 106, 118, 44, 35, 99, 115, 74, 38, 48, 50, 125, 33, 40, 1, 105], [41, 54, 48, 35, 32, 22, 93, 16, 44, 127, 40, 10, 112, 30, 90, 89, 78, 84, 124, 83, 3, 70, 77, 27, 115, 123, 113, 51, 18, 117, 25, 52, 55, 101, 120, 111, 24, 7, 59, 121, 76, 119, 110, 63, 5, 122, 60, 50, 62, 126, 114, 58, 57, 43, 11, 118, 108, 31, 116, 81, 125, 1, 61, 56, 53, 109, 47, 19, 45, 100, 72, 79, 91, 46, 103, 102, 42, 39, 106, 9, 49, 2, 23, 107, 4, 80, 64, 73, 0, 13, 74, 66, 20, 12, 8, 33, 28, 21, 86, 104, 65, 38, 14, 26, 71, 87, 98, 34, 92, 15, 17, 85, 95, 68, 97, 29, 88, 36, 105, 96, 37, 94, 82, 75, 6, 99, 69, 67], [50, 101, 113, 49, 54, 94, 114, 22, 91, 87, 25, 21, 83, 81, 74, 84, 13, 18, 16, 86, 14, 72, 56, 71, 80, 110, 69, 77, 15, 32, 112, 58, 70, 60, 67, 66, 53, 117, 124, 26, 88, 92, 12, 45, 119, 68, 52, 55, 78, 62, 63, 125, 109, 9, 127, 73, 61, 64, 57, 4, 37, 51, 43, 126, 107, 11, 118, 115, 48, 19, 100, 123, 98, 116, 31, 47, 65, 121, 59, 75, 111, 120, 42, 29, 104, 102, 95, 6, 105, 44, 41, 103, 89, 76, 46, 40, 108, 122, 39, 1, 23, 97, 17, 38, 106, 3, 27, 10, 34, 35, 36, 99, 33, 85, 24, 30, 28, 96, 93, 79, 90, 20, 5, 8, 82, 7, 2, 0], [105, 37, 23, 18, 25, 15, 20, 11, 14, 85, 101, 13, 124, 91, 113, 34, 9, 46, 49, 8, 48, 126, 6, 1, 41, 118, 43, 68, 45, 115, 2, 111, 74, 110, 94, 16, 0, 63, 119, 114, 51, 83, 24, 62, 98, 64, 70, 59, 35, 53, 109, 92, 117, 58, 81, 102, 12, 65, 90, 55, 31, 10, 30, 57, 67, 3, 44, 29, 36, 61, 120, 7, 4, 32, 112, 103, 122, 108, 27, 116, 99, 96, 77, 50, 26, 86, 72, 107, 106, 121, 19, 95, 104, 93, 88, 56, 75, 71, 42, 125, 40, 54, 22, 38, 100, 123, 52, 97, 39, 5, 17, 79, 33, 69, 127, 47, 80, 28, 73, 60, 66, 21, 76, 78, 89, 84, 87, 82]], "model.layers.7.self_attn.qk_proj": [[121, 41, 54, 118, 105, 50, 37, 45, 106, 48, 101, 49, 109, 114, 107, 104, 113, 93, 43, 23, 87, 60, 30, 42, 89, 77, 40, 85, 25, 82, 83, 13, 14, 18, 21, 26, 78, 117, 19, 16, 112, 10, 111, 124, 79, 15, 20, 34, 63, 55, 81, 84, 74, 80, 103, 72, 17, 56, 119, 115, 94, 76, 29, 12, 39, 100, 86, 33, 57, 110, 75, 122, 126, 27, 47, 127, 52, 44, 125, 0, 11, 22, 53, 102, 90, 8, 58, 123, 32, 97, 64, 61, 7, 67, 69, 5, 62, 9, 59, 71, 51, 65, 73, 70, 98, 3, 4, 91, 120, 116, 38, 96, 6, 35, 24, 36, 46, 1, 88, 68, 31, 92, 108, 28, 2, 95, 99, 66], [121, 41, 118, 105, 54, 50, 37, 45, 106, 48, 49, 101, 109, 107, 114, 104, 43, 113, 93, 23, 30, 87, 40, 60, 42, 25, 21, 18, 85, 77, 19, 13, 14, 82, 89, 83, 26, 112, 111, 84, 103, 79, 78, 16, 10, 63, 80, 117, 15, 34, 55, 124, 94, 17, 20, 74, 12, 81, 72, 115, 119, 100, 56, 86, 39, 76, 122, 53, 29, 11, 125, 33, 47, 110, 127, 27, 61, 57, 22, 102, 75, 0, 52, 90, 126, 8, 97, 62, 64, 123, 32, 51, 44, 58, 98, 69, 7, 9, 4, 5, 6, 1, 65, 71, 67, 36, 91, 120, 35, 116, 73, 59, 68, 108, 70, 38, 3, 24, 96, 88, 2, 99, 28, 66, 46, 92, 31, 95], [121, 41, 118, 105, 54, 50, 37, 45, 106, 101, 48, 114, 109, 49, 107, 104, 113, 93, 23, 43, 30, 87, 42, 60, 40, 25, 89, 85, 21, 77, 26, 18, 19, 82, 13, 117, 55, 103, 14, 94, 83, 112, 84, 34, 111, 78, 81, 124, 20, 79, 100, 16, 10, 74, 15, 29, 80, 33, 39, 86, 63, 17, 110, 76, 97, 47, 12, 122, 90, 56, 53, 115, 72, 57, 8, 27, 119, 22, 127, 102, 52, 32, 75, 0, 64, 11, 58, 62, 69, 126, 125, 6, 98, 7, 5, 1, 44, 65, 61, 68, 123, 51, 71, 91, 3, 108, 9, 96, 4, 59, 36, 35, 67, 46, 116, 73, 24, 99, 38, 31, 88, 120, 70, 2, 66, 28, 95, 92], [121, 118, 41, 105, 54, 50, 45, 37, 106, 49, 48, 101, 114, 109, 107, 104, 93, 113, 43, 87, 23, 30, 89, 40, 42, 60, 25, 82, 26, 85, 117, 77, 112, 21, 18, 103, 55, 78, 127, 83, 13, 19, 79, 119, 20, 94, 84, 34, 124, 15, 63, 14, 111, 53, 100, 81, 10, 16, 47, 80, 17, 74, 29, 39, 27, 86, 12, 33, 102, 110, 64, 125, 11, 44, 115, 76, 126, 32, 52, 56, 58, 90, 97, 122, 22, 8, 72, 0, 51, 57, 69, 71, 62, 75, 5, 6, 67, 65, 116, 3, 73, 61, 91, 59, 7, 98, 120, 68, 1, 36, 35, 9, 123, 96, 4, 2, 46, 31, 38, 70, 88, 28, 66, 108, 92, 24, 95, 99], [121, 41, 118, 105, 54, 50, 45, 37, 106, 48, 49, 101, 114, 109, 107, 104, 113, 43, 93, 87, 23, 42, 40, 89, 30, 60, 77, 26, 18, 85, 82, 21, 117, 25, 55, 13, 34, 111, 112, 14, 103, 83, 78, 10, 84, 94, 16, 63, 47, 74, 15, 19, 20, 79, 124, 76, 81, 8, 80, 100, 39, 29, 17, 12, 119, 102, 115, 11, 33, 57, 56, 86, 64, 53, 0, 126, 110, 75, 32, 65, 22, 127, 90, 97, 122, 68, 69, 72, 62, 27, 58, 61, 44, 52, 5, 98, 4, 125, 7, 73, 3, 6, 9, 51, 67, 38, 91, 71, 1, 116, 59, 66, 24, 70, 123, 108, 35, 2, 36, 88, 120, 96, 46, 92, 28, 99, 31, 95], [121, 41, 118, 54, 105, 50, 45, 37, 106, 101, 48, 49, 114, 107, 109, 104, 43, 113, 93, 87, 23, 60, 42, 77, 85, 82, 89, 40, 30, 25, 18, 21, 26, 14, 78, 13, 19, 112, 79, 83, 111, 16, 103, 84, 117, 74, 20, 15, 63, 34, 17, 10, 11, 8, 81, 80, 124, 12, 76, 100, 115, 94, 47, 55, 86, 126, 53, 39, 125, 29, 57, 127, 110, 72, 61, 52, 75, 0, 119, 33, 102, 58, 62, 27, 22, 44, 64, 122, 32, 56, 7, 9, 120, 97, 3, 90, 91, 123, 68, 6, 73, 5, 69, 51, 4, 71, 70, 65, 59, 1, 67, 98, 24, 35, 66, 116, 36, 38, 96, 108, 2, 92, 88, 28, 46, 31, 99, 95], [121, 41, 54, 105, 118, 50, 37, 45, 101, 106, 49, 48, 114, 109, 107, 23, 104, 113, 93, 87, 43, 30, 82, 42, 60, 77, 18, 21, 25, 14, 89, 85, 13, 40, 78, 26, 16, 19, 15, 84, 117, 111, 83, 79, 20, 103, 63, 17, 34, 74, 8, 10, 81, 112, 80, 12, 76, 47, 94, 124, 29, 33, 110, 27, 11, 115, 86, 39, 57, 55, 52, 100, 22, 125, 126, 75, 119, 122, 127, 56, 58, 44, 90, 61, 91, 32, 102, 72, 53, 0, 59, 62, 97, 73, 69, 123, 98, 7, 96, 5, 64, 9, 71, 51, 38, 68, 6, 3, 116, 92, 35, 36, 24, 1, 88, 67, 70, 4, 120, 65, 46, 2, 31, 28, 108, 66, 95, 99], [121, 41, 105, 118, 54, 50, 45, 106, 37, 49, 101, 48, 109, 114, 107, 104, 113, 93, 43, 23, 42, 87, 82, 40, 18, 21, 30, 13, 85, 19, 77, 60, 89, 26, 14, 78, 25, 79, 16, 84, 83, 15, 103, 80, 17, 20, 74, 12, 81, 8, 76, 34, 117, 10, 124, 94, 111, 29, 86, 11, 63, 112, 55, 39, 119, 27, 75, 100, 57, 127, 33, 115, 22, 61, 44, 102, 0, 47, 64, 90, 1, 125, 52, 5, 91, 110, 72, 69, 58, 32, 123, 122, 70, 56, 7, 24, 53, 97, 65, 62, 126, 59, 71, 98, 73, 4, 88, 67, 9, 96, 38, 51, 36, 116, 68, 92, 3, 2, 6, 46, 120, 31, 35, 95, 66, 108, 28, 99], [121, 41, 118, 54, 105, 50, 45, 37, 106, 48, 101, 49, 109, 107, 114, 104, 43, 93, 113, 23, 87, 40, 42, 30, 89, 21, 19, 18, 82, 25, 85, 60, 26, 84, 13, 14, 111, 77, 16, 83, 94, 63, 20, 79, 103, 117, 15, 17, 78, 112, 55, 34, 80, 12, 74, 124, 81, 10, 57, 8, 29, 53, 76, 33, 127, 47, 110, 52, 27, 39, 86, 119, 11, 126, 90, 125, 58, 100, 22, 102, 97, 44, 51, 32, 62, 91, 115, 75, 56, 0, 64, 98, 7, 61, 123, 70, 122, 4, 72, 5, 36, 65, 71, 88, 116, 59, 69, 73, 68, 67, 38, 9, 24, 1, 96, 120, 35, 92, 3, 6, 108, 28, 66, 31, 95, 99, 46, 2], [121, 41, 118, 105, 54, 50, 37, 45, 106, 101, 49, 48, 109, 107, 114, 104, 113, 93, 43, 87, 23, 30, 60, 42, 25, 82, 21, 85, 40, 89, 13, 18, 111, 19, 16, 14, 77, 117, 83, 26, 84, 15, 20, 103, 79, 78, 34, 112, 17, 55, 80, 10, 81, 94, 74, 76, 8, 12, 115, 124, 102, 119, 11, 29, 126, 63, 27, 39, 33, 100, 86, 125, 47, 127, 57, 44, 56, 53, 110, 90, 122, 61, 51, 32, 64, 58, 91, 22, 0, 97, 75, 69, 72, 70, 62, 52, 71, 123, 73, 3, 98, 5, 65, 9, 116, 36, 67, 1, 7, 4, 92, 88, 68, 96, 59, 6, 120, 28, 24, 35, 66, 38, 46, 31, 99, 108, 95, 2], [121, 41, 118, 54, 105, 50, 37, 45, 106, 48, 49, 101, 114, 109, 107, 104, 113, 43, 93, 23, 42, 87, 30, 40, 13, 82, 25, 85, 89, 60, 77, 21, 18, 111, 55, 83, 117, 15, 16, 26, 79, 14, 63, 20, 78, 34, 119, 112, 19, 10, 103, 84, 74, 115, 47, 94, 80, 53, 127, 81, 17, 12, 52, 8, 39, 29, 110, 76, 86, 11, 33, 100, 72, 64, 102, 126, 122, 57, 124, 56, 44, 69, 62, 61, 51, 75, 125, 0, 97, 32, 1, 7, 58, 90, 5, 22, 123, 27, 70, 71, 9, 65, 73, 91, 67, 6, 98, 4, 68, 3, 38, 35, 36, 120, 96, 59, 116, 2, 66, 28, 92, 24, 31, 46, 108, 88, 95, 99], [121, 41, 118, 54, 105, 50, 45, 37, 106, 48, 49, 114, 101, 107, 109, 104, 113, 93, 87, 43, 23, 42, 30, 82, 85, 25, 40, 21, 18, 89, 13, 77, 60, 83, 26, 14, 15, 79, 111, 19, 84, 103, 16, 34, 20, 112, 78, 117, 81, 80, 10, 74, 94, 63, 124, 55, 76, 17, 86, 47, 39, 127, 12, 8, 53, 119, 29, 72, 33, 115, 100, 52, 27, 11, 110, 64, 56, 69, 90, 102, 22, 126, 75, 62, 44, 123, 91, 73, 32, 122, 67, 71, 57, 97, 125, 5, 61, 0, 4, 70, 68, 3, 9, 51, 7, 58, 1, 6, 98, 120, 65, 96, 38, 36, 66, 24, 116, 2, 59, 108, 88, 46, 35, 92, 28, 31, 99, 95], [121, 41, 118, 54, 105, 50, 45, 37, 101, 106, 48, 49, 109, 114, 107, 104, 113, 23, 93, 43, 87, 60, 30, 18, 42, 40, 25, 89, 21, 82, 13, 85, 19, 16, 26, 14, 83, 77, 78, 124, 84, 79, 103, 117, 111, 94, 15, 34, 81, 80, 20, 72, 74, 17, 112, 63, 10, 12, 115, 76, 29, 55, 119, 86, 100, 33, 27, 47, 90, 11, 126, 125, 39, 127, 110, 53, 75, 44, 102, 8, 56, 57, 52, 62, 97, 91, 22, 32, 61, 58, 122, 59, 64, 6, 4, 123, 73, 69, 0, 71, 68, 7, 65, 9, 3, 24, 36, 5, 51, 96, 98, 116, 70, 67, 38, 46, 92, 88, 120, 35, 31, 108, 1, 2, 66, 28, 95, 99], [121, 41, 118, 105, 54, 50, 37, 45, 106, 101, 49, 48, 109, 114, 107, 104, 113, 93, 43, 23, 60, 13, 40, 42, 87, 30, 18, 85, 25, 82, 89, 103, 21, 83, 26, 72, 78, 16, 14, 17, 117, 79, 19, 20, 15, 34, 77, 63, 74, 84, 81, 111, 124, 55, 10, 112, 80, 76, 12, 119, 47, 126, 29, 11, 75, 86, 0, 64, 27, 94, 115, 100, 127, 8, 125, 44, 39, 90, 61, 102, 33, 110, 22, 53, 6, 57, 52, 56, 69, 5, 58, 7, 65, 51, 71, 1, 9, 32, 68, 62, 97, 73, 116, 122, 91, 4, 3, 59, 96, 67, 70, 98, 36, 123, 88, 35, 24, 38, 66, 92, 31, 120, 46, 2, 99, 28, 95, 108], [121, 41, 118, 105, 54, 50, 45, 37, 49, 106, 101, 48, 109, 114, 107, 104, 43, 113, 93, 23, 87, 30, 42, 60, 25, 40, 89, 18, 21, 82, 85, 26, 13, 83, 111, 15, 16, 78, 72, 79, 19, 20, 112, 117, 77, 84, 63, 14, 124, 74, 103, 17, 34, 10, 81, 80, 55, 29, 47, 86, 94, 115, 12, 119, 76, 39, 127, 27, 57, 90, 33, 11, 110, 100, 53, 75, 126, 52, 61, 32, 8, 102, 62, 22, 69, 58, 6, 56, 0, 125, 44, 91, 64, 71, 73, 5, 9, 7, 122, 123, 51, 65, 4, 3, 1, 67, 36, 88, 97, 70, 68, 96, 24, 38, 120, 46, 116, 59, 92, 28, 98, 35, 2, 31, 66, 108, 99, 95], [121, 41, 118, 54, 105, 50, 45, 37, 106, 49, 48, 101, 114, 107, 109, 104, 113, 93, 43, 23, 30, 60, 42, 25, 87, 85, 40, 89, 82, 18, 13, 26, 83, 21, 112, 77, 14, 111, 117, 16, 94, 78, 19, 15, 79, 34, 20, 84, 63, 74, 72, 29, 124, 55, 80, 115, 10, 17, 53, 57, 126, 47, 86, 76, 39, 103, 81, 33, 100, 119, 110, 62, 125, 127, 12, 56, 11, 90, 22, 27, 52, 6, 64, 75, 102, 61, 32, 8, 122, 44, 97, 91, 7, 51, 123, 58, 9, 73, 69, 0, 38, 68, 4, 71, 5, 3, 36, 59, 67, 46, 1, 120, 116, 98, 96, 88, 24, 70, 35, 65, 92, 108, 66, 28, 31, 99, 2, 95], [121, 41, 118, 54, 105, 50, 37, 45, 106, 101, 49, 114, 109, 48, 107, 104, 113, 23, 93, 87, 60, 43, 42, 30, 25, 89, 18, 82, 85, 13, 83, 40, 21, 26, 16, 117, 19, 78, 14, 77, 84, 81, 79, 34, 15, 74, 112, 72, 20, 124, 80, 10, 111, 94, 17, 100, 12, 115, 39, 55, 76, 119, 103, 63, 110, 33, 11, 29, 90, 57, 86, 47, 127, 123, 27, 53, 102, 58, 22, 52, 56, 8, 125, 44, 32, 75, 122, 0, 91, 126, 9, 7, 71, 97, 62, 61, 5, 69, 68, 51, 6, 98, 64, 120, 1, 70, 36, 59, 4, 108, 38, 116, 73, 35, 65, 88, 24, 46, 96, 92, 67, 2, 31, 3, 99, 95, 28, 66], [121, 41, 118, 54, 105, 50, 37, 45, 106, 101, 49, 109, 48, 114, 107, 104, 113, 23, 93, 43, 42, 87, 30, 60, 85, 83, 40, 25, 13, 89, 82, 18, 21, 26, 19, 111, 78, 77, 16, 117, 84, 14, 79, 15, 20, 103, 80, 72, 34, 124, 10, 112, 17, 81, 29, 74, 94, 55, 76, 119, 12, 33, 63, 57, 75, 90, 8, 127, 100, 47, 110, 126, 11, 39, 86, 22, 115, 61, 52, 32, 27, 53, 62, 102, 125, 56, 64, 58, 7, 51, 0, 91, 122, 44, 71, 5, 69, 70, 97, 9, 59, 1, 67, 116, 73, 98, 6, 123, 36, 38, 3, 4, 46, 65, 120, 96, 24, 68, 99, 28, 92, 31, 108, 35, 88, 95, 2, 66], [121, 41, 118, 105, 54, 50, 37, 45, 106, 49, 101, 109, 48, 114, 107, 104, 43, 113, 23, 30, 87, 93, 89, 42, 60, 40, 25, 18, 117, 85, 13, 63, 82, 21, 83, 112, 26, 94, 16, 111, 77, 78, 119, 103, 20, 34, 55, 15, 84, 19, 80, 79, 124, 14, 33, 57, 74, 81, 29, 102, 10, 17, 52, 39, 72, 127, 90, 8, 12, 115, 53, 100, 97, 47, 76, 86, 110, 126, 32, 27, 44, 11, 64, 58, 51, 0, 62, 91, 56, 75, 22, 71, 69, 61, 7, 122, 70, 98, 5, 1, 67, 4, 3, 68, 35, 9, 36, 65, 125, 123, 59, 120, 73, 92, 24, 95, 38, 31, 6, 116, 99, 46, 28, 108, 88, 96, 2, 66], [121, 41, 118, 54, 105, 50, 37, 45, 106, 101, 49, 48, 109, 107, 114, 113, 104, 93, 23, 43, 30, 25, 87, 60, 42, 18, 77, 40, 26, 21, 89, 13, 85, 83, 19, 82, 117, 80, 111, 112, 15, 84, 14, 124, 10, 78, 34, 79, 16, 103, 74, 63, 76, 94, 55, 100, 72, 81, 17, 8, 33, 115, 39, 29, 12, 20, 126, 86, 27, 102, 11, 110, 62, 53, 75, 57, 56, 44, 97, 61, 127, 22, 70, 119, 47, 52, 71, 51, 58, 90, 69, 0, 122, 64, 5, 125, 123, 91, 32, 73, 7, 4, 9, 65, 1, 36, 98, 35, 120, 68, 59, 24, 46, 96, 38, 3, 116, 6, 67, 31, 88, 92, 108, 99, 2, 28, 95, 66], [121, 41, 118, 54, 105, 50, 45, 37, 106, 49, 101, 48, 109, 114, 107, 104, 113, 93, 43, 42, 23, 30, 60, 87, 89, 40, 25, 18, 13, 77, 85, 82, 21, 26, 103, 117, 14, 111, 74, 83, 19, 63, 79, 112, 119, 16, 34, 15, 81, 78, 84, 80, 94, 29, 33, 55, 124, 20, 10, 57, 17, 8, 76, 62, 110, 86, 127, 39, 53, 47, 12, 11, 72, 75, 126, 100, 22, 90, 102, 58, 115, 56, 9, 71, 123, 70, 27, 44, 52, 69, 5, 97, 125, 32, 51, 64, 120, 91, 7, 122, 3, 0, 61, 4, 59, 38, 73, 67, 68, 98, 46, 96, 36, 1, 6, 88, 65, 99, 35, 31, 116, 108, 28, 2, 92, 66, 24, 95], [121, 41, 118, 105, 54, 50, 37, 45, 106, 49, 48, 101, 109, 114, 107, 104, 113, 87, 93, 23, 43, 42, 30, 60, 89, 40, 25, 82, 85, 26, 13, 18, 21, 77, 19, 117, 83, 14, 79, 103, 124, 34, 16, 112, 78, 111, 63, 15, 74, 84, 119, 29, 94, 33, 55, 20, 17, 110, 80, 81, 10, 76, 86, 8, 127, 12, 100, 39, 90, 115, 57, 75, 102, 47, 126, 52, 32, 53, 91, 62, 27, 56, 22, 44, 58, 11, 64, 97, 72, 61, 51, 123, 0, 71, 5, 59, 125, 69, 9, 70, 1, 98, 122, 116, 73, 38, 7, 88, 120, 46, 96, 67, 36, 4, 3, 35, 65, 68, 28, 92, 31, 99, 6, 24, 95, 2, 108, 66], [121, 41, 118, 54, 105, 50, 37, 45, 101, 49, 48, 106, 114, 109, 107, 113, 104, 43, 93, 23, 42, 30, 87, 60, 25, 40, 89, 18, 82, 13, 26, 85, 117, 77, 21, 78, 34, 94, 19, 124, 103, 83, 14, 55, 84, 15, 79, 16, 74, 111, 80, 20, 17, 8, 81, 29, 10, 112, 119, 76, 47, 115, 86, 90, 39, 33, 11, 12, 56, 63, 126, 100, 102, 75, 44, 127, 22, 110, 53, 57, 0, 32, 91, 61, 27, 58, 97, 52, 72, 64, 73, 4, 62, 51, 69, 5, 1, 122, 125, 9, 123, 68, 38, 71, 6, 59, 7, 35, 65, 24, 96, 98, 46, 70, 88, 36, 3, 67, 116, 31, 92, 66, 28, 120, 95, 99, 2, 108], [121, 41, 118, 105, 54, 50, 37, 45, 106, 101, 48, 49, 109, 114, 107, 104, 113, 43, 93, 60, 23, 42, 30, 25, 87, 40, 85, 13, 117, 21, 89, 82, 18, 26, 19, 77, 78, 103, 111, 74, 124, 55, 94, 16, 79, 8, 14, 63, 84, 83, 112, 34, 20, 127, 17, 10, 15, 119, 80, 29, 33, 126, 47, 12, 56, 53, 81, 100, 86, 39, 75, 115, 110, 76, 62, 102, 11, 51, 90, 27, 97, 125, 61, 22, 44, 58, 32, 72, 7, 57, 64, 52, 73, 6, 122, 69, 9, 4, 123, 5, 91, 68, 71, 0, 96, 3, 98, 35, 116, 38, 120, 36, 67, 59, 65, 46, 108, 1, 88, 70, 28, 31, 66, 99, 24, 95, 92, 2], [121, 41, 118, 105, 54, 50, 37, 45, 106, 49, 101, 48, 109, 114, 104, 107, 113, 43, 93, 23, 60, 30, 42, 87, 25, 13, 40, 82, 89, 21, 85, 18, 78, 19, 117, 26, 34, 77, 74, 15, 79, 124, 111, 83, 112, 14, 16, 20, 80, 84, 103, 10, 63, 17, 8, 39, 94, 115, 33, 100, 55, 29, 12, 76, 81, 110, 11, 86, 57, 56, 126, 22, 44, 90, 127, 53, 75, 119, 27, 52, 47, 58, 72, 102, 62, 123, 122, 32, 64, 97, 125, 73, 6, 71, 38, 5, 91, 69, 61, 0, 36, 9, 59, 120, 7, 51, 4, 67, 31, 1, 98, 68, 3, 88, 116, 70, 65, 46, 96, 35, 28, 108, 99, 92, 24, 2, 95, 66], [121, 41, 118, 105, 54, 50, 45, 37, 106, 101, 49, 48, 109, 114, 107, 104, 113, 43, 93, 23, 60, 42, 87, 40, 25, 82, 13, 85, 30, 89, 18, 21, 26, 77, 117, 14, 19, 74, 83, 34, 15, 78, 79, 63, 112, 80, 84, 17, 103, 8, 111, 94, 10, 12, 20, 16, 124, 76, 29, 110, 55, 39, 100, 81, 33, 86, 119, 127, 115, 52, 75, 47, 53, 56, 27, 57, 11, 126, 102, 22, 72, 90, 97, 125, 58, 0, 122, 6, 44, 64, 7, 123, 32, 71, 51, 62, 69, 91, 59, 73, 9, 5, 61, 98, 1, 120, 65, 96, 116, 88, 68, 4, 3, 38, 24, 92, 36, 67, 46, 35, 108, 28, 31, 70, 2, 99, 95, 66], [121, 41, 118, 105, 54, 50, 45, 37, 106, 101, 48, 49, 114, 109, 107, 104, 93, 113, 43, 23, 42, 30, 87, 60, 25, 40, 85, 82, 26, 89, 21, 77, 13, 83, 18, 103, 14, 19, 111, 20, 34, 17, 117, 55, 84, 124, 16, 78, 79, 94, 112, 29, 15, 63, 127, 86, 80, 76, 126, 119, 10, 74, 110, 33, 39, 53, 58, 123, 56, 27, 90, 8, 122, 12, 115, 81, 47, 57, 11, 52, 22, 102, 44, 72, 97, 0, 100, 32, 125, 62, 38, 61, 75, 9, 51, 64, 59, 7, 4, 116, 98, 5, 69, 6, 36, 68, 67, 3, 96, 65, 91, 24, 71, 1, 120, 73, 70, 92, 35, 88, 31, 28, 46, 95, 66, 99, 2, 108], [121, 41, 118, 105, 54, 50, 45, 37, 106, 48, 49, 101, 109, 107, 114, 104, 113, 43, 93, 42, 87, 23, 60, 30, 40, 13, 82, 77, 85, 21, 25, 18, 19, 89, 78, 14, 34, 15, 117, 83, 74, 26, 103, 111, 112, 10, 20, 72, 124, 16, 79, 80, 17, 8, 12, 81, 84, 76, 47, 110, 64, 75, 56, 127, 115, 100, 55, 86, 29, 63, 94, 119, 11, 126, 39, 0, 53, 22, 122, 52, 102, 7, 61, 5, 57, 33, 58, 90, 68, 69, 9, 1, 62, 27, 97, 44, 65, 32, 73, 123, 59, 125, 98, 3, 71, 91, 4, 6, 67, 70, 51, 116, 66, 38, 36, 120, 88, 96, 35, 2, 24, 46, 108, 31, 92, 28, 99, 95], [41, 121, 118, 105, 54, 50, 45, 37, 106, 49, 48, 101, 109, 107, 114, 104, 113, 43, 93, 42, 23, 30, 87, 60, 40, 89, 25, 85, 55, 19, 77, 13, 112, 18, 21, 34, 117, 26, 103, 82, 78, 83, 111, 124, 94, 63, 15, 20, 10, 84, 16, 14, 74, 115, 79, 80, 29, 56, 100, 72, 39, 76, 33, 17, 81, 127, 47, 110, 53, 8, 126, 12, 102, 52, 86, 75, 119, 0, 11, 90, 123, 22, 57, 7, 27, 32, 44, 58, 97, 5, 125, 122, 62, 51, 70, 64, 61, 9, 91, 71, 69, 68, 38, 120, 65, 1, 67, 4, 96, 73, 92, 6, 59, 88, 98, 36, 116, 35, 3, 28, 108, 24, 66, 2, 46, 31, 95, 99], [121, 41, 118, 54, 105, 50, 45, 37, 106, 48, 49, 101, 114, 107, 109, 104, 113, 43, 93, 23, 42, 60, 87, 40, 25, 30, 89, 18, 77, 85, 21, 13, 82, 14, 78, 19, 83, 63, 26, 111, 117, 15, 79, 34, 10, 112, 115, 74, 103, 20, 119, 80, 72, 124, 16, 84, 17, 81, 11, 39, 94, 55, 12, 86, 76, 56, 100, 47, 126, 33, 110, 29, 102, 127, 44, 8, 0, 57, 75, 27, 122, 52, 7, 62, 64, 22, 90, 123, 53, 61, 73, 125, 70, 3, 68, 5, 67, 97, 69, 65, 51, 71, 9, 58, 32, 1, 91, 120, 4, 59, 98, 96, 2, 6, 36, 108, 35, 38, 88, 46, 92, 28, 99, 24, 116, 66, 95, 31], [41, 121, 118, 105, 54, 50, 45, 106, 37, 101, 48, 49, 114, 109, 107, 104, 113, 43, 60, 23, 87, 93, 42, 40, 30, 89, 25, 18, 13, 85, 21, 77, 82, 78, 26, 34, 19, 83, 14, 10, 80, 111, 119, 63, 79, 72, 15, 112, 103, 117, 55, 74, 20, 124, 16, 100, 115, 17, 84, 81, 29, 94, 12, 47, 76, 11, 52, 39, 102, 110, 57, 53, 75, 33, 22, 27, 62, 44, 86, 126, 0, 123, 5, 127, 73, 90, 122, 8, 7, 125, 59, 97, 64, 70, 4, 32, 61, 38, 56, 116, 58, 51, 69, 68, 65, 3, 91, 9, 71, 1, 88, 67, 98, 96, 120, 35, 6, 36, 2, 92, 99, 46, 28, 31, 108, 95, 66, 24], [121, 41, 118, 105, 54, 50, 45, 106, 37, 49, 48, 101, 109, 107, 104, 114, 113, 43, 93, 23, 60, 42, 87, 40, 13, 30, 25, 89, 85, 18, 19, 77, 82, 21, 14, 26, 111, 78, 72, 80, 10, 83, 34, 79, 103, 15, 63, 16, 117, 20, 74, 119, 124, 112, 84, 81, 12, 47, 11, 17, 76, 55, 127, 94, 39, 27, 75, 100, 126, 29, 52, 115, 110, 33, 86, 44, 53, 56, 22, 102, 58, 61, 125, 90, 70, 57, 51, 8, 62, 123, 32, 7, 122, 0, 69, 9, 73, 64, 59, 5, 71, 91, 4, 1, 97, 67, 3, 65, 68, 120, 98, 24, 6, 96, 116, 38, 35, 92, 36, 88, 46, 2, 31, 99, 66, 28, 95, 108]], "model.layers.8.self_attn.q_proj": [[60, 102, 116, 51, 123, 117, 91, 34, 52, 33, 118, 124, 115, 62, 119, 18, 126, 63, 48, 97, 58, 57, 50, 79, 54, 55, 53, 56, 61, 85, 49, 121, 59, 125, 94, 111, 122, 35, 101, 127, 45, 112, 120, 77, 44, 105, 43, 113, 89, 114, 73, 47, 26, 110, 46, 42, 13, 90, 21, 19, 109, 24, 28, 108, 107, 22, 12, 82, 14, 83, 96, 70, 37, 20, 87, 106, 39, 16, 40, 81, 72, 74, 84, 41, 103, 80, 7, 99, 30, 27, 32, 88, 31, 11, 15, 25, 36, 17, 95, 29, 104, 93, 23, 86, 92, 10, 100, 98, 76, 1, 4, 5, 3, 9, 75, 78, 38, 69, 64, 71, 67, 68, 66, 8, 6, 2, 65, 0], [117, 60, 51, 123, 116, 52, 122, 63, 126, 53, 55, 49, 48, 125, 57, 121, 120, 56, 110, 113, 111, 61, 47, 112, 114, 127, 54, 118, 119, 45, 46, 124, 62, 115, 109, 50, 58, 59, 44, 108, 43, 105, 107, 36, 42, 106, 41, 104, 40, 101, 39, 103, 102, 37, 35, 100, 24, 86, 98, 38, 95, 34, 99, 33, 87, 22, 96, 90, 66, 32, 0, 94, 31, 81, 5, 30, 97, 8, 84, 65, 19, 27, 93, 23, 85, 69, 29, 82, 75, 72, 91, 28, 21, 17, 15, 20, 89, 67, 88, 78, 92, 83, 2, 76, 13, 14, 64, 18, 25, 79, 80, 6, 70, 9, 26, 73, 68, 12, 77, 11, 1, 4, 74, 3, 16, 7, 71, 10], [102, 116, 60, 51, 33, 123, 117, 91, 52, 115, 85, 84, 15, 124, 70, 118, 79, 18, 7, 62, 34, 73, 12, 38, 119, 27, 26, 74, 89, 88, 11, 87, 59, 24, 94, 90, 58, 50, 20, 21, 93, 69, 30, 22, 3, 57, 54, 72, 126, 77, 4, 14, 82, 121, 35, 9, 98, 81, 61, 42, 37, 1, 13, 63, 56, 48, 92, 53, 10, 32, 23, 76, 25, 2, 28, 16, 64, 68, 55, 80, 31, 41, 105, 5, 17, 44, 95, 36, 101, 75, 49, 39, 67, 71, 103, 108, 66, 43, 29, 99, 120, 106, 96, 19, 127, 107, 46, 45, 125, 83, 78, 114, 97, 113, 40, 111, 100, 47, 104, 109, 6, 110, 8, 112, 86, 122, 65, 0], [116, 102, 60, 51, 123, 117, 115, 52, 62, 124, 119, 118, 34, 38, 39, 19, 58, 79, 57, 126, 85, 54, 50, 37, 43, 61, 48, 89, 53, 56, 121, 55, 94, 44, 104, 46, 42, 31, 107, 73, 108, 110, 125, 41, 40, 70, 72, 33, 111, 95, 36, 106, 109, 87, 63, 45, 49, 101, 59, 127, 103, 23, 105, 113, 120, 47, 32, 35, 28, 90, 114, 91, 13, 122, 96, 21, 112, 88, 100, 25, 11, 99, 74, 30, 27, 5, 93, 98, 92, 77, 29, 26, 1, 12, 17, 7, 24, 20, 14, 66, 80, 64, 10, 4, 3, 15, 76, 82, 16, 67, 78, 97, 18, 68, 22, 75, 71, 83, 69, 81, 84, 65, 2, 9, 86, 8, 0, 6], [38, 34, 119, 60, 18, 92, 78, 77, 9, 24, 88, 72, 85, 8, 75, 124, 68, 3, 21, 55, 59, 70, 80, 123, 69, 115, 28, 48, 71, 19, 22, 29, 37, 44, 47, 26, 61, 58, 117, 76, 4, 35, 73, 10, 95, 83, 125, 64, 0, 118, 14, 16, 79, 66, 12, 2, 62, 13, 31, 87, 93, 82, 46, 65, 57, 104, 49, 89, 42, 98, 103, 27, 15, 81, 84, 56, 63, 107, 67, 111, 30, 94, 110, 36, 6, 100, 7, 50, 25, 116, 17, 106, 105, 32, 101, 39, 53, 23, 120, 74, 86, 97, 113, 112, 43, 40, 20, 99, 41, 96, 126, 90, 1, 52, 11, 33, 121, 54, 108, 127, 114, 91, 5, 122, 45, 109, 51, 102], [38, 34, 59, 60, 119, 29, 18, 92, 88, 24, 115, 47, 77, 79, 9, 78, 68, 85, 1, 80, 55, 71, 124, 75, 22, 83, 70, 61, 65, 110, 44, 72, 64, 117, 35, 37, 69, 19, 123, 76, 125, 98, 3, 12, 58, 5, 16, 8, 73, 52, 21, 28, 95, 13, 7, 10, 100, 103, 27, 67, 0, 6, 48, 66, 93, 86, 106, 118, 26, 17, 4, 14, 112, 30, 20, 90, 46, 81, 63, 91, 32, 82, 107, 15, 2, 104, 102, 23, 94, 42, 74, 36, 11, 25, 99, 89, 39, 45, 54, 62, 49, 116, 120, 97, 50, 41, 96, 126, 31, 87, 113, 33, 43, 56, 127, 114, 122, 53, 105, 40, 84, 121, 101, 111, 51, 109, 108, 57], [38, 34, 60, 119, 29, 18, 85, 77, 88, 59, 78, 24, 37, 80, 9, 92, 124, 75, 71, 55, 69, 44, 21, 47, 8, 125, 3, 61, 93, 72, 16, 68, 19, 117, 35, 118, 79, 123, 22, 81, 95, 106, 11, 76, 115, 13, 32, 48, 58, 31, 10, 1, 83, 120, 65, 27, 63, 98, 122, 7, 116, 52, 84, 97, 89, 14, 26, 82, 99, 112, 107, 36, 94, 66, 64, 50, 42, 25, 30, 53, 28, 33, 100, 70, 40, 103, 87, 126, 110, 62, 51, 96, 104, 12, 41, 101, 39, 20, 54, 91, 86, 43, 49, 109, 45, 127, 23, 46, 114, 111, 57, 56, 108, 74, 113, 6, 17, 90, 15, 67, 121, 4, 0, 105, 73, 5, 2, 102], [38, 60, 119, 34, 29, 28, 88, 115, 85, 102, 93, 59, 19, 125, 80, 95, 27, 78, 92, 47, 87, 52, 98, 55, 18, 33, 44, 37, 35, 123, 118, 117, 24, 26, 83, 110, 22, 62, 58, 54, 46, 12, 89, 25, 61, 42, 48, 104, 103, 84, 30, 113, 40, 45, 97, 100, 101, 106, 91, 109, 31, 112, 116, 10, 81, 94, 114, 63, 21, 99, 107, 51, 49, 50, 105, 126, 96, 16, 111, 108, 127, 121, 124, 32, 57, 122, 43, 36, 56, 17, 120, 53, 41, 20, 86, 79, 39, 76, 23, 90, 8, 14, 74, 77, 71, 15, 75, 72, 11, 13, 82, 9, 7, 66, 2, 67, 70, 5, 69, 3, 4, 6, 73, 0, 68, 1, 64, 65], [59, 60, 116, 80, 55, 50, 73, 72, 117, 12, 62, 77, 10, 84, 120, 26, 54, 125, 103, 69, 19, 13, 24, 127, 68, 78, 15, 75, 93, 7, 21, 82, 17, 51, 90, 124, 6, 57, 49, 113, 3, 98, 56, 122, 121, 1, 66, 114, 16, 18, 112, 46, 58, 76, 70, 119, 36, 126, 123, 52, 28, 34, 118, 53, 88, 110, 61, 63, 0, 47, 29, 71, 109, 43, 27, 115, 96, 32, 104, 44, 107, 45, 108, 100, 92, 48, 102, 38, 42, 97, 106, 94, 105, 79, 111, 41, 101, 87, 89, 40, 37, 31, 33, 99, 95, 11, 91, 85, 35, 30, 86, 74, 9, 20, 81, 65, 83, 39, 23, 14, 4, 25, 8, 2, 22, 64, 67, 5], [59, 116, 60, 103, 55, 73, 117, 15, 87, 62, 80, 120, 125, 82, 54, 84, 10, 90, 127, 72, 13, 19, 69, 68, 76, 7, 6, 3, 78, 77, 51, 50, 124, 57, 66, 0, 75, 17, 113, 1, 49, 12, 21, 37, 56, 70, 121, 122, 11, 114, 119, 46, 123, 52, 36, 94, 74, 93, 32, 38, 88, 110, 58, 61, 118, 39, 43, 31, 112, 126, 28, 96, 47, 26, 104, 99, 53, 71, 42, 89, 65, 106, 101, 107, 115, 40, 100, 109, 111, 86, 83, 97, 102, 63, 108, 44, 105, 41, 81, 24, 33, 35, 30, 45, 95, 48, 8, 22, 92, 4, 98, 27, 20, 5, 16, 18, 79, 25, 91, 14, 34, 2, 29, 64, 23, 9, 85, 67], [103, 59, 116, 60, 20, 26, 16, 96, 84, 98, 24, 29, 89, 114, 93, 14, 10, 55, 125, 94, 91, 99, 88, 62, 19, 117, 70, 1, 87, 33, 18, 50, 31, 28, 80, 90, 68, 17, 57, 92, 52, 127, 69, 12, 95, 3, 25, 54, 27, 49, 23, 124, 72, 97, 120, 123, 66, 30, 74, 110, 71, 35, 22, 100, 36, 38, 32, 34, 113, 21, 0, 67, 64, 104, 102, 53, 82, 39, 78, 37, 76, 101, 40, 42, 51, 6, 119, 7, 108, 4, 122, 105, 83, 43, 81, 58, 75, 41, 118, 106, 44, 109, 48, 107, 45, 121, 47, 46, 9, 112, 56, 63, 73, 61, 111, 126, 115, 15, 5, 79, 85, 11, 8, 13, 65, 86, 77, 2], [59, 50, 60, 116, 19, 21, 15, 82, 80, 89, 76, 84, 13, 73, 88, 78, 87, 120, 62, 117, 17, 125, 54, 55, 7, 51, 90, 72, 127, 77, 6, 10, 3, 124, 61, 56, 121, 122, 58, 0, 63, 123, 112, 126, 118, 119, 11, 69, 57, 113, 53, 115, 47, 68, 75, 49, 66, 48, 111, 74, 46, 92, 86, 52, 110, 45, 109, 114, 108, 44, 1, 107, 93, 43, 106, 42, 105, 8, 41, 23, 104, 40, 27, 29, 65, 12, 100, 81, 103, 4, 28, 30, 38, 37, 102, 36, 39, 5, 70, 99, 20, 101, 14, 2, 83, 32, 35, 98, 97, 96, 22, 33, 95, 31, 67, 64, 26, 94, 71, 34, 24, 25, 85, 91, 16, 79, 9, 18], [104, 118, 34, 95, 50, 25, 70, 73, 51, 56, 45, 23, 55, 114, 49, 62, 5, 84, 3, 7, 40, 109, 117, 18, 121, 52, 74, 63, 54, 110, 124, 82, 58, 68, 14, 86, 126, 76, 15, 60, 8, 119, 112, 65, 17, 31, 123, 53, 80, 108, 97, 59, 127, 16, 27, 71, 125, 20, 35, 115, 46, 107, 89, 47, 32, 93, 120, 77, 44, 39, 111, 42, 29, 101, 61, 48, 90, 38, 41, 2, 113, 122, 100, 88, 30, 106, 92, 12, 102, 22, 37, 33, 103, 36, 105, 19, 13, 57, 94, 43, 116, 96, 28, 9, 91, 78, 85, 87, 81, 75, 99, 79, 21, 24, 83, 98, 10, 11, 26, 67, 0, 72, 6, 69, 4, 1, 66, 64], [104, 118, 34, 95, 84, 25, 73, 12, 40, 23, 18, 51, 16, 9, 49, 5, 65, 68, 70, 56, 1, 69, 66, 54, 8, 89, 80, 67, 63, 6, 13, 117, 15, 64, 62, 87, 60, 78, 45, 11, 50, 55, 3, 27, 58, 74, 14, 7, 20, 124, 72, 79, 2, 110, 24, 10, 71, 109, 76, 77, 82, 81, 127, 0, 28, 52, 4, 123, 22, 114, 31, 93, 83, 125, 97, 42, 91, 75, 126, 85, 112, 94, 86, 90, 21, 35, 46, 88, 33, 47, 19, 59, 121, 100, 38, 48, 39, 61, 119, 122, 17, 41, 26, 92, 29, 44, 101, 96, 108, 107, 111, 53, 113, 106, 36, 103, 32, 98, 120, 37, 99, 30, 102, 105, 43, 116, 115, 57], [118, 104, 34, 95, 50, 25, 23, 45, 52, 62, 84, 56, 117, 58, 49, 51, 18, 54, 16, 63, 114, 109, 123, 60, 55, 124, 106, 112, 12, 59, 27, 125, 57, 91, 121, 15, 126, 61, 110, 42, 32, 41, 31, 127, 119, 48, 43, 53, 33, 22, 39, 47, 115, 108, 111, 37, 113, 122, 120, 35, 28, 46, 116, 88, 107, 44, 99, 105, 93, 73, 103, 36, 97, 102, 38, 101, 92, 30, 82, 100, 86, 5, 87, 79, 96, 90, 13, 94, 85, 24, 89, 17, 78, 29, 9, 8, 70, 83, 26, 14, 74, 98, 21, 76, 20, 81, 80, 19, 7, 75, 3, 10, 77, 40, 72, 11, 69, 68, 65, 2, 71, 67, 6, 64, 4, 66, 1, 0], [104, 118, 34, 95, 25, 16, 12, 56, 23, 84, 18, 45, 15, 55, 69, 51, 78, 50, 117, 72, 58, 9, 40, 63, 8, 5, 49, 109, 66, 114, 67, 124, 54, 79, 71, 6, 60, 76, 4, 13, 110, 10, 80, 87, 86, 73, 31, 81, 119, 27, 62, 91, 108, 89, 75, 14, 1, 20, 70, 85, 52, 53, 112, 64, 29, 123, 11, 7, 19, 82, 42, 44, 59, 93, 121, 83, 61, 127, 39, 126, 90, 96, 94, 97, 32, 125, 74, 77, 103, 2, 21, 98, 107, 24, 41, 88, 111, 17, 22, 35, 106, 26, 57, 47, 115, 101, 92, 48, 43, 33, 30, 37, 68, 100, 122, 46, 28, 3, 36, 116, 105, 38, 102, 113, 120, 0, 99, 65], [41, 54, 59, 99, 79, 88, 101, 33, 13, 22, 90, 91, 31, 27, 73, 82, 39, 11, 123, 21, 84, 26, 71, 0, 66, 16, 40, 109, 94, 6, 74, 108, 2, 83, 28, 12, 38, 107, 68, 29, 126, 105, 51, 62, 7, 18, 95, 60, 76, 87, 34, 52, 20, 1, 98, 23, 9, 15, 32, 80, 78, 122, 67, 96, 125, 8, 110, 42, 55, 65, 4, 75, 57, 70, 36, 30, 106, 85, 49, 89, 45, 50, 46, 3, 124, 103, 56, 64, 77, 19, 127, 43, 17, 44, 58, 92, 100, 47, 104, 121, 97, 113, 118, 86, 112, 14, 115, 93, 114, 119, 10, 25, 120, 63, 102, 61, 5, 24, 81, 53, 72, 48, 69, 116, 111, 117, 37, 35], [41, 54, 99, 52, 91, 33, 126, 123, 27, 105, 113, 88, 108, 59, 61, 43, 36, 40, 101, 13, 125, 109, 22, 84, 107, 49, 16, 21, 57, 44, 118, 56, 45, 34, 115, 51, 63, 62, 17, 106, 50, 98, 31, 112, 127, 42, 25, 47, 120, 90, 39, 55, 100, 46, 124, 38, 121, 110, 114, 116, 58, 103, 29, 111, 96, 122, 94, 60, 95, 48, 53, 77, 71, 119, 117, 23, 104, 32, 102, 28, 93, 37, 82, 19, 4, 92, 85, 30, 81, 75, 20, 87, 89, 35, 26, 83, 97, 76, 68, 74, 10, 80, 73, 70, 18, 11, 24, 14, 72, 69, 12, 79, 78, 67, 6, 0, 2, 7, 8, 86, 15, 64, 65, 9, 5, 66, 1, 3], [41, 54, 99, 126, 33, 52, 91, 105, 27, 40, 101, 88, 21, 61, 89, 56, 125, 13, 45, 110, 123, 44, 34, 113, 82, 115, 36, 109, 85, 38, 59, 43, 107, 84, 22, 31, 42, 112, 81, 124, 94, 108, 114, 111, 120, 39, 116, 127, 57, 103, 16, 96, 58, 50, 63, 18, 118, 62, 60, 74, 90, 53, 100, 122, 17, 55, 37, 106, 23, 98, 104, 48, 87, 32, 49, 46, 121, 93, 28, 77, 51, 117, 19, 47, 119, 102, 25, 26, 97, 80, 29, 95, 68, 30, 83, 92, 35, 71, 24, 79, 75, 20, 11, 76, 4, 10, 78, 14, 12, 72, 86, 15, 70, 73, 7, 6, 1, 8, 9, 65, 64, 69, 2, 67, 5, 66, 0, 3], [41, 99, 59, 54, 88, 82, 31, 123, 27, 22, 91, 105, 79, 84, 13, 52, 21, 108, 87, 101, 12, 46, 33, 8, 18, 74, 83, 26, 39, 19, 81, 9, 94, 95, 118, 126, 10, 89, 77, 38, 17, 6, 92, 127, 20, 25, 55, 28, 15, 56, 40, 14, 69, 90, 70, 115, 44, 85, 68, 24, 16, 30, 97, 5, 86, 100, 23, 61, 98, 43, 32, 109, 110, 96, 104, 120, 113, 124, 76, 111, 29, 34, 122, 42, 75, 125, 80, 93, 107, 119, 57, 102, 58, 106, 36, 103, 47, 1, 63, 112, 114, 45, 72, 121, 4, 78, 50, 60, 117, 11, 3, 116, 53, 65, 49, 48, 71, 73, 37, 51, 62, 35, 67, 66, 0, 7, 64, 2], [103, 33, 59, 31, 82, 20, 23, 120, 81, 90, 114, 11, 76, 13, 14, 87, 22, 78, 7, 18, 71, 10, 21, 106, 83, 112, 16, 12, 15, 73, 124, 19, 119, 52, 77, 67, 96, 123, 38, 91, 25, 125, 41, 48, 40, 43, 107, 37, 94, 104, 98, 97, 88, 118, 27, 75, 86, 42, 51, 121, 84, 53, 117, 54, 55, 35, 58, 32, 3, 24, 30, 72, 80, 69, 111, 28, 34, 109, 126, 101, 47, 56, 100, 46, 8, 44, 85, 29, 93, 105, 115, 26, 79, 92, 108, 9, 1, 99, 62, 60, 36, 122, 127, 45, 61, 68, 113, 89, 4, 17, 57, 2, 39, 74, 63, 49, 5, 65, 102, 66, 110, 6, 95, 116, 50, 70, 64, 0], [103, 33, 59, 20, 31, 23, 120, 90, 83, 114, 82, 29, 16, 78, 19, 13, 84, 27, 88, 37, 94, 24, 32, 118, 81, 11, 72, 30, 17, 97, 7, 126, 48, 38, 92, 127, 96, 117, 21, 8, 77, 124, 56, 36, 87, 106, 25, 76, 22, 40, 9, 51, 12, 91, 119, 62, 105, 57, 41, 115, 10, 43, 45, 113, 52, 26, 110, 123, 14, 102, 79, 107, 15, 98, 54, 28, 93, 55, 34, 89, 61, 109, 18, 66, 44, 74, 80, 100, 95, 42, 99, 125, 85, 101, 47, 121, 63, 49, 35, 60, 73, 104, 68, 111, 58, 39, 108, 71, 4, 122, 53, 86, 75, 46, 112, 65, 116, 70, 50, 67, 2, 3, 6, 69, 5, 64, 0, 1], [103, 33, 59, 120, 31, 25, 13, 20, 11, 82, 78, 52, 16, 87, 10, 73, 23, 81, 7, 3, 65, 2, 66, 70, 90, 0, 5, 68, 114, 76, 69, 6, 19, 83, 18, 80, 106, 107, 67, 71, 27, 8, 75, 119, 9, 72, 96, 51, 77, 117, 48, 39, 28, 74, 105, 1, 43, 126, 4, 118, 22, 24, 37, 41, 123, 100, 46, 17, 53, 36, 127, 94, 56, 109, 115, 101, 44, 85, 40, 26, 92, 30, 54, 32, 104, 21, 84, 113, 62, 49, 47, 15, 12, 34, 14, 45, 121, 60, 93, 98, 91, 42, 88, 79, 50, 124, 29, 57, 111, 58, 110, 122, 125, 61, 116, 63, 99, 108, 35, 86, 102, 38, 89, 55, 112, 95, 64, 97], [103, 33, 114, 59, 31, 81, 120, 25, 76, 20, 78, 64, 15, 23, 82, 7, 21, 90, 16, 9, 3, 1, 83, 11, 5, 48, 106, 13, 65, 24, 66, 67, 27, 75, 115, 37, 47, 126, 77, 19, 74, 52, 44, 84, 55, 4, 10, 28, 53, 107, 93, 87, 22, 97, 71, 92, 29, 98, 39, 12, 43, 0, 109, 40, 96, 124, 46, 127, 51, 119, 17, 121, 88, 32, 105, 38, 2, 26, 100, 118, 112, 99, 111, 69, 8, 79, 104, 41, 86, 116, 58, 113, 73, 62, 45, 42, 110, 91, 18, 56, 94, 70, 14, 125, 6, 89, 54, 35, 57, 80, 108, 72, 63, 101, 123, 68, 122, 30, 50, 60, 61, 49, 34, 117, 36, 95, 102, 85], [44, 102, 50, 63, 98, 93, 108, 40, 123, 88, 80, 107, 43, 13, 31, 120, 119, 90, 7, 53, 115, 125, 57, 22, 29, 10, 114, 126, 19, 51, 84, 26, 20, 116, 92, 112, 109, 39, 55, 28, 12, 124, 127, 49, 121, 21, 45, 48, 36, 17, 18, 8, 101, 61, 52, 14, 46, 56, 85, 54, 47, 78, 118, 122, 99, 16, 113, 94, 59, 110, 74, 83, 60, 6, 58, 117, 70, 3, 97, 111, 68, 105, 81, 62, 25, 23, 4, 104, 42, 87, 32, 41, 91, 103, 30, 76, 79, 35, 106, 33, 15, 37, 82, 11, 27, 9, 96, 24, 89, 75, 66, 0, 95, 71, 73, 100, 86, 72, 67, 77, 65, 1, 2, 64, 69, 5, 38, 34], [44, 63, 102, 50, 98, 93, 40, 123, 107, 26, 108, 80, 31, 57, 115, 120, 43, 84, 90, 10, 127, 125, 53, 13, 22, 88, 116, 21, 126, 45, 19, 109, 121, 23, 51, 119, 112, 61, 124, 114, 104, 29, 18, 55, 47, 68, 56, 101, 54, 49, 36, 14, 62, 46, 52, 48, 122, 92, 17, 99, 58, 7, 60, 72, 105, 118, 12, 28, 8, 39, 97, 30, 78, 91, 59, 87, 110, 111, 117, 20, 42, 113, 77, 15, 103, 27, 81, 41, 106, 32, 95, 25, 85, 89, 94, 79, 83, 16, 24, 9, 66, 96, 2, 35, 37, 4, 76, 100, 75, 82, 11, 33, 5, 3, 73, 70, 1, 0, 74, 86, 6, 69, 65, 38, 34, 71, 67, 64], [44, 63, 102, 50, 108, 98, 120, 107, 40, 57, 53, 119, 51, 123, 88, 93, 26, 109, 125, 112, 58, 31, 127, 114, 43, 126, 61, 121, 116, 101, 118, 45, 48, 49, 54, 115, 36, 85, 47, 59, 56, 124, 21, 111, 122, 113, 55, 23, 95, 117, 80, 60, 39, 84, 52, 25, 62, 78, 104, 8, 46, 87, 41, 42, 100, 14, 19, 13, 18, 105, 90, 29, 89, 106, 110, 92, 35, 28, 32, 94, 99, 22, 103, 97, 37, 12, 91, 30, 17, 96, 33, 34, 81, 38, 66, 72, 82, 68, 24, 71, 83, 27, 20, 77, 7, 10, 74, 79, 2, 65, 16, 75, 11, 70, 15, 3, 73, 76, 9, 86, 6, 67, 69, 0, 4, 64, 5, 1], [44, 63, 102, 50, 108, 98, 40, 43, 115, 123, 120, 88, 125, 116, 31, 57, 26, 107, 112, 93, 114, 58, 53, 28, 127, 121, 84, 51, 19, 119, 101, 95, 52, 126, 56, 55, 49, 54, 36, 47, 109, 61, 39, 45, 59, 21, 104, 124, 46, 97, 62, 111, 8, 118, 48, 29, 90, 122, 23, 78, 14, 60, 113, 30, 117, 42, 110, 80, 22, 103, 12, 87, 92, 105, 99, 20, 37, 94, 33, 100, 106, 18, 32, 13, 17, 41, 27, 35, 38, 91, 81, 89, 72, 96, 24, 83, 15, 2, 77, 85, 66, 79, 25, 76, 82, 34, 7, 71, 73, 10, 68, 75, 9, 65, 16, 11, 6, 4, 70, 67, 0, 74, 69, 86, 3, 5, 1, 64], [54, 102, 122, 127, 120, 33, 28, 20, 125, 87, 92, 25, 116, 15, 12, 30, 97, 114, 47, 89, 63, 17, 96, 21, 22, 38, 32, 82, 19, 49, 94, 99, 7, 52, 48, 93, 123, 50, 16, 80, 13, 62, 115, 27, 91, 95, 121, 83, 74, 75, 72, 14, 88, 84, 31, 60, 85, 81, 55, 26, 51, 29, 10, 11, 59, 108, 35, 34, 113, 100, 111, 77, 90, 36, 98, 78, 23, 45, 118, 79, 37, 103, 24, 126, 53, 101, 57, 109, 58, 110, 112, 39, 117, 18, 105, 124, 56, 107, 41, 104, 43, 69, 42, 119, 40, 44, 46, 106, 76, 9, 2, 73, 61, 8, 71, 4, 86, 67, 6, 0, 70, 5, 68, 3, 1, 65, 66, 64], [54, 102, 122, 125, 127, 116, 25, 20, 114, 38, 33, 63, 30, 79, 47, 87, 120, 62, 123, 50, 52, 74, 48, 49, 53, 60, 55, 118, 126, 115, 28, 22, 58, 15, 121, 59, 7, 31, 57, 46, 51, 17, 94, 117, 92, 124, 113, 84, 76, 111, 70, 41, 44, 103, 89, 110, 35, 104, 32, 43, 12, 36, 107, 112, 39, 93, 34, 61, 119, 98, 96, 109, 99, 67, 40, 37, 56, 88, 105, 108, 95, 45, 2, 91, 106, 29, 77, 101, 83, 42, 97, 13, 69, 27, 100, 81, 19, 80, 72, 11, 90, 86, 21, 85, 82, 26, 73, 78, 23, 14, 75, 24, 71, 68, 18, 4, 0, 8, 16, 65, 10, 1, 9, 6, 5, 66, 64, 3], [122, 102, 54, 127, 125, 116, 120, 114, 87, 25, 33, 38, 92, 30, 12, 63, 47, 28, 20, 48, 22, 123, 62, 53, 17, 50, 59, 55, 118, 52, 60, 80, 49, 72, 82, 126, 57, 89, 24, 124, 51, 115, 76, 15, 58, 121, 91, 29, 81, 67, 94, 113, 97, 70, 109, 7, 0, 111, 44, 46, 21, 112, 2, 77, 103, 61, 56, 74, 119, 117, 69, 88, 110, 93, 108, 43, 79, 39, 65, 84, 107, 8, 36, 14, 1, 45, 73, 78, 83, 68, 32, 23, 90, 31, 27, 105, 40, 95, 96, 104, 6, 9, 37, 106, 16, 42, 3, 86, 85, 18, 100, 26, 10, 11, 41, 35, 34, 98, 101, 71, 19, 75, 13, 64, 4, 66, 5, 99], [122, 54, 102, 127, 116, 125, 114, 120, 38, 70, 25, 55, 63, 74, 53, 47, 59, 48, 62, 123, 118, 49, 103, 50, 60, 20, 67, 44, 52, 51, 124, 115, 126, 39, 57, 56, 58, 121, 113, 2, 45, 43, 33, 117, 76, 111, 7, 112, 110, 13, 46, 87, 119, 107, 109, 61, 93, 42, 108, 106, 36, 0, 105, 40, 101, 28, 104, 79, 92, 22, 41, 89, 37, 30, 35, 100, 19, 94, 26, 96, 90, 81, 88, 32, 34, 31, 29, 97, 71, 78, 23, 77, 99, 80, 69, 98, 27, 95, 84, 72, 10, 65, 4, 24, 1, 8, 12, 91, 68, 15, 73, 86, 17, 83, 11, 85, 9, 6, 14, 18, 82, 64, 21, 16, 5, 66, 75, 3]], "model.layers.8.self_attn.k_proj": [[116, 60, 38, 22, 51, 97, 94, 123, 117, 28, 89, 90, 84, 124, 18, 62, 16, 119, 81, 91, 36, 77, 52, 118, 87, 58, 15, 74, 35, 7, 54, 86, 19, 59, 50, 121, 61, 57, 21, 80, 42, 56, 99, 78, 126, 53, 100, 10, 73, 13, 48, 70, 41, 46, 75, 108, 125, 44, 127, 76, 40, 14, 114, 107, 103, 115, 104, 47, 98, 37, 63, 55, 106, 109, 105, 110, 24, 111, 26, 120, 43, 101, 45, 112, 49, 113, 39, 31, 122, 9, 95, 68, 67, 32, 85, 96, 82, 29, 1, 88, 30, 34, 64, 17, 102, 4, 66, 5, 23, 6, 92, 27, 20, 93, 71, 12, 83, 72, 3, 33, 65, 25, 11, 0, 2, 79, 8, 69], [60, 119, 98, 102, 88, 18, 85, 9, 80, 28, 78, 77, 64, 75, 71, 65, 59, 99, 3, 19, 93, 108, 68, 12, 69, 23, 115, 125, 2, 70, 61, 95, 111, 43, 51, 104, 29, 11, 38, 91, 5, 124, 123, 22, 117, 47, 37, 52, 20, 58, 46, 42, 24, 30, 10, 31, 49, 33, 66, 103, 36, 118, 53, 67, 40, 90, 113, 4, 15, 17, 122, 86, 81, 7, 114, 109, 6, 107, 83, 41, 48, 8, 92, 25, 63, 62, 54, 127, 32, 96, 13, 110, 94, 34, 0, 89, 84, 44, 106, 105, 100, 112, 120, 126, 26, 79, 97, 76, 121, 57, 55, 39, 82, 35, 116, 87, 16, 50, 45, 73, 101, 72, 56, 27, 74, 21, 1, 14], [59, 22, 60, 116, 39, 34, 114, 120, 54, 86, 124, 127, 51, 50, 35, 62, 55, 117, 113, 94, 57, 121, 56, 125, 122, 123, 49, 119, 52, 46, 29, 99, 61, 118, 43, 126, 58, 53, 110, 115, 112, 63, 103, 111, 47, 106, 108, 109, 76, 37, 18, 48, 28, 45, 44, 107, 42, 40, 81, 104, 96, 41, 14, 102, 38, 101, 105, 11, 85, 91, 95, 98, 15, 97, 100, 13, 74, 27, 26, 36, 73, 20, 25, 16, 82, 7, 32, 24, 30, 23, 78, 33, 92, 72, 6, 19, 8, 79, 80, 31, 89, 69, 88, 93, 90, 83, 66, 3, 68, 87, 17, 75, 0, 4, 1, 9, 5, 10, 67, 21, 12, 65, 84, 77, 2, 70, 71, 64], [40, 118, 98, 31, 23, 18, 25, 84, 16, 12, 56, 51, 78, 72, 10, 15, 9, 117, 55, 69, 71, 49, 63, 0, 54, 67, 62, 6, 1, 109, 4, 110, 60, 58, 45, 50, 127, 27, 124, 13, 2, 64, 66, 113, 74, 46, 8, 17, 47, 106, 42, 83, 123, 112, 44, 34, 19, 53, 91, 75, 126, 26, 24, 125, 52, 114, 61, 122, 29, 97, 68, 77, 30, 48, 11, 94, 108, 111, 59, 38, 119, 93, 107, 99, 120, 86, 103, 88, 39, 28, 121, 7, 21, 101, 33, 90, 100, 43, 105, 115, 41, 32, 96, 35, 92, 36, 102, 14, 37, 116, 57, 65, 89, 82, 85, 20, 70, 87, 22, 76, 79, 3, 80, 95, 5, 73, 81, 104], [105, 54, 59, 35, 95, 22, 91, 88, 21, 64, 110, 123, 68, 18, 97, 13, 16, 92, 74, 1, 6, 79, 37, 45, 71, 62, 52, 98, 44, 82, 41, 55, 107, 66, 83, 108, 126, 77, 57, 11, 50, 99, 48, 120, 127, 104, 58, 112, 53, 8, 90, 117, 56, 94, 122, 5, 40, 125, 60, 63, 51, 81, 121, 67, 103, 61, 106, 124, 114, 34, 12, 111, 115, 65, 3, 26, 113, 102, 116, 46, 42, 119, 30, 96, 47, 93, 73, 100, 15, 25, 109, 118, 36, 78, 17, 87, 43, 20, 14, 39, 2, 75, 38, 32, 0, 101, 76, 89, 84, 49, 29, 23, 19, 69, 72, 9, 33, 80, 31, 28, 24, 70, 7, 27, 10, 86, 4, 85], [39, 59, 120, 97, 23, 95, 83, 16, 11, 114, 20, 0, 78, 13, 82, 7, 76, 8, 73, 2, 42, 65, 48, 15, 50, 126, 69, 68, 3, 104, 107, 52, 25, 32, 27, 43, 117, 105, 119, 45, 62, 51, 4, 115, 53, 112, 118, 40, 9, 89, 81, 90, 88, 86, 58, 44, 98, 55, 1, 85, 109, 21, 10, 113, 79, 127, 66, 5, 54, 28, 30, 37, 35, 63, 122, 75, 47, 124, 46, 24, 102, 29, 61, 34, 106, 38, 94, 108, 121, 67, 91, 70, 57, 125, 49, 26, 110, 101, 22, 36, 96, 123, 72, 56, 60, 14, 18, 93, 99, 111, 100, 41, 116, 6, 84, 92, 19, 64, 17, 74, 12, 87, 77, 31, 80, 71, 103, 33], [108, 38, 22, 63, 34, 95, 50, 29, 23, 26, 18, 88, 44, 92, 104, 17, 19, 120, 20, 66, 115, 12, 114, 57, 125, 80, 13, 49, 107, 37, 123, 69, 53, 98, 112, 111, 8, 127, 45, 47, 121, 126, 51, 78, 113, 10, 55, 61, 35, 56, 52, 14, 54, 62, 91, 59, 119, 117, 65, 79, 85, 116, 109, 122, 7, 60, 64, 103, 118, 83, 124, 89, 46, 110, 105, 27, 70, 93, 58, 15, 9, 39, 100, 106, 48, 11, 40, 5, 97, 72, 42, 3, 28, 33, 94, 25, 41, 99, 96, 36, 77, 32, 30, 87, 84, 67, 43, 82, 24, 75, 76, 0, 21, 81, 31, 73, 101, 6, 4, 86, 1, 16, 90, 71, 2, 74, 68, 102], [122, 54, 38, 22, 97, 127, 63, 30, 114, 120, 125, 53, 47, 92, 50, 62, 126, 116, 82, 123, 51, 118, 59, 52, 124, 25, 57, 115, 46, 87, 55, 48, 58, 113, 44, 49, 111, 61, 56, 60, 121, 119, 117, 109, 12, 108, 98, 64, 43, 110, 112, 71, 15, 100, 17, 45, 40, 107, 20, 103, 106, 80, 32, 104, 105, 101, 41, 75, 39, 78, 42, 66, 36, 19, 67, 10, 35, 37, 102, 34, 94, 68, 5, 91, 99, 31, 77, 14, 3, 83, 8, 96, 18, 81, 73, 93, 24, 21, 26, 65, 95, 9, 11, 79, 88, 69, 29, 90, 33, 85, 70, 4, 16, 23, 27, 28, 89, 13, 6, 86, 1, 74, 72, 0, 2, 7, 84, 76]], "model.layers.8.self_attn.qk_proj": [[59, 60, 54, 118, 116, 119, 122, 120, 51, 63, 114, 108, 50, 38, 44, 125, 105, 117, 123, 40, 41, 102, 82, 86, 127, 47, 95, 34, 53, 87, 98, 18, 55, 62, 58, 39, 57, 52, 49, 124, 24, 103, 126, 88, 23, 115, 80, 97, 16, 78, 61, 84, 27, 20, 22, 89, 29, 13, 14, 104, 110, 83, 77, 19, 107, 113, 12, 92, 85, 31, 99, 76, 93, 46, 21, 11, 43, 56, 48, 15, 75, 111, 112, 109, 25, 33, 42, 9, 90, 121, 73, 79, 28, 71, 7, 106, 8, 26, 35, 45, 30, 81, 94, 10, 74, 64, 17, 36, 91, 5, 37, 3, 4, 0, 70, 72, 32, 67, 68, 69, 101, 100, 66, 2, 6, 65, 96, 1], [59, 60, 118, 54, 116, 122, 119, 120, 51, 63, 114, 108, 50, 38, 117, 44, 40, 105, 125, 123, 53, 102, 86, 41, 18, 103, 98, 127, 62, 82, 115, 87, 52, 126, 39, 88, 95, 24, 124, 34, 57, 84, 58, 97, 80, 23, 104, 55, 47, 20, 16, 22, 107, 78, 27, 77, 89, 49, 56, 14, 92, 19, 25, 85, 31, 13, 61, 83, 93, 12, 76, 21, 79, 42, 29, 99, 110, 48, 113, 11, 43, 75, 28, 9, 106, 26, 109, 8, 46, 73, 71, 45, 112, 15, 35, 37, 90, 121, 33, 111, 7, 94, 30, 36, 91, 17, 64, 10, 5, 81, 74, 0, 65, 4, 32, 67, 2, 1, 3, 70, 66, 101, 68, 69, 6, 100, 96, 72], [59, 60, 118, 54, 116, 119, 122, 120, 51, 63, 108, 38, 50, 114, 123, 44, 40, 117, 105, 53, 125, 102, 86, 34, 98, 39, 41, 95, 18, 24, 97, 87, 62, 82, 57, 127, 88, 23, 52, 115, 55, 84, 124, 22, 103, 20, 126, 49, 27, 104, 58, 89, 31, 47, 113, 16, 80, 93, 78, 29, 92, 77, 25, 56, 61, 48, 99, 33, 109, 85, 13, 75, 26, 45, 19, 21, 112, 76, 107, 43, 110, 14, 90, 35, 11, 94, 79, 12, 37, 83, 28, 46, 91, 111, 30, 42, 106, 15, 9, 7, 121, 8, 71, 81, 17, 36, 73, 32, 74, 10, 69, 100, 101, 64, 65, 0, 3, 72, 4, 66, 2, 68, 5, 6, 1, 67, 70, 96], [59, 60, 118, 54, 116, 119, 122, 120, 63, 51, 114, 108, 50, 38, 123, 44, 40, 117, 125, 105, 127, 86, 95, 41, 98, 53, 87, 102, 24, 115, 18, 62, 39, 34, 126, 82, 49, 103, 124, 88, 97, 23, 58, 47, 57, 84, 113, 52, 20, 22, 93, 55, 29, 80, 31, 16, 27, 14, 89, 25, 104, 83, 99, 77, 92, 110, 19, 48, 13, 78, 43, 109, 107, 61, 45, 11, 56, 85, 33, 21, 28, 46, 42, 15, 76, 37, 90, 26, 12, 75, 35, 73, 30, 94, 7, 36, 106, 79, 121, 112, 8, 111, 64, 9, 32, 71, 17, 91, 101, 81, 74, 10, 67, 3, 69, 4, 1, 2, 66, 0, 5, 65, 72, 96, 68, 70, 100, 6], [59, 60, 118, 54, 119, 122, 116, 120, 51, 63, 114, 108, 38, 50, 44, 117, 123, 40, 105, 125, 102, 98, 127, 57, 53, 95, 39, 24, 18, 82, 62, 41, 124, 87, 103, 34, 86, 23, 49, 113, 126, 115, 55, 80, 104, 88, 27, 97, 78, 84, 20, 58, 47, 52, 22, 77, 29, 16, 89, 93, 19, 46, 31, 110, 13, 14, 112, 45, 11, 12, 76, 92, 56, 107, 85, 28, 99, 83, 61, 48, 79, 109, 43, 21, 25, 73, 90, 35, 42, 37, 71, 0, 33, 9, 75, 111, 15, 26, 121, 7, 64, 36, 30, 106, 94, 17, 74, 72, 1, 3, 5, 32, 8, 4, 68, 66, 67, 10, 2, 81, 69, 91, 65, 101, 6, 96, 70, 100], [59, 60, 54, 118, 116, 119, 122, 120, 63, 51, 114, 108, 50, 38, 44, 123, 125, 117, 40, 105, 86, 102, 53, 127, 41, 95, 98, 82, 18, 24, 62, 124, 39, 87, 88, 80, 103, 16, 115, 34, 84, 57, 55, 52, 20, 78, 22, 47, 58, 23, 14, 126, 77, 49, 31, 83, 97, 104, 29, 27, 13, 107, 113, 76, 19, 89, 85, 28, 11, 12, 110, 92, 93, 99, 48, 42, 56, 109, 79, 75, 15, 25, 43, 61, 45, 46, 21, 30, 73, 112, 33, 9, 71, 90, 35, 37, 7, 72, 111, 121, 26, 106, 17, 81, 5, 94, 8, 36, 74, 10, 32, 64, 2, 0, 68, 91, 3, 69, 6, 67, 4, 65, 66, 101, 100, 1, 96, 70], [59, 60, 118, 54, 116, 119, 122, 120, 63, 51, 108, 114, 50, 38, 123, 117, 44, 40, 125, 105, 127, 86, 53, 95, 18, 87, 98, 82, 41, 102, 115, 24, 52, 88, 62, 84, 34, 126, 124, 22, 47, 16, 58, 20, 23, 39, 80, 97, 55, 57, 103, 104, 78, 29, 83, 13, 14, 113, 89, 27, 112, 49, 92, 107, 110, 19, 77, 76, 99, 48, 56, 31, 93, 11, 111, 21, 85, 12, 75, 61, 15, 28, 109, 9, 73, 79, 121, 43, 90, 33, 42, 94, 25, 81, 30, 46, 17, 7, 35, 26, 72, 71, 45, 106, 37, 74, 10, 91, 101, 32, 5, 8, 36, 68, 6, 2, 64, 100, 3, 69, 67, 4, 70, 65, 66, 96, 0, 1], [59, 60, 118, 54, 116, 119, 122, 63, 120, 108, 51, 114, 38, 50, 44, 117, 40, 105, 123, 86, 125, 87, 82, 18, 102, 127, 115, 95, 98, 88, 124, 41, 24, 34, 39, 22, 52, 58, 16, 20, 53, 80, 84, 107, 57, 23, 14, 47, 103, 104, 13, 27, 55, 56, 97, 76, 78, 62, 12, 126, 89, 83, 48, 77, 92, 85, 121, 29, 49, 93, 21, 31, 19, 75, 110, 113, 79, 99, 28, 11, 106, 15, 112, 73, 46, 9, 111, 42, 25, 30, 94, 90, 61, 109, 37, 43, 91, 71, 35, 45, 72, 33, 81, 26, 7, 10, 74, 17, 0, 64, 66, 32, 69, 36, 101, 5, 4, 65, 68, 6, 2, 67, 100, 8, 3, 1, 70, 96], [59, 60, 118, 54, 116, 119, 122, 120, 51, 63, 108, 50, 38, 114, 117, 123, 44, 105, 40, 125, 102, 127, 18, 98, 95, 62, 82, 24, 87, 115, 34, 39, 41, 86, 84, 88, 52, 16, 20, 103, 57, 22, 107, 126, 58, 89, 124, 80, 55, 97, 104, 27, 47, 23, 48, 13, 29, 78, 14, 92, 53, 46, 49, 77, 28, 21, 93, 19, 31, 110, 99, 25, 76, 83, 43, 56, 11, 113, 37, 85, 111, 94, 9, 109, 75, 12, 42, 15, 79, 112, 45, 33, 91, 61, 106, 35, 72, 30, 71, 73, 26, 90, 121, 7, 17, 101, 74, 81, 32, 64, 67, 36, 10, 1, 0, 5, 4, 68, 3, 69, 2, 66, 70, 65, 6, 8, 96, 100], [59, 60, 118, 54, 116, 119, 122, 120, 63, 51, 108, 114, 50, 117, 38, 123, 125, 44, 105, 40, 18, 102, 53, 41, 95, 82, 98, 62, 86, 87, 115, 103, 24, 88, 127, 39, 16, 49, 124, 52, 34, 80, 23, 126, 22, 84, 20, 47, 78, 58, 55, 13, 31, 97, 92, 19, 27, 89, 29, 77, 113, 107, 104, 14, 21, 75, 76, 42, 83, 57, 93, 61, 25, 12, 85, 110, 43, 99, 46, 33, 109, 26, 28, 79, 56, 112, 48, 111, 9, 45, 11, 7, 71, 72, 73, 121, 94, 15, 90, 37, 30, 35, 106, 91, 17, 36, 74, 81, 32, 10, 69, 3, 101, 0, 1, 64, 67, 68, 66, 100, 70, 5, 4, 6, 2, 65, 96, 8], [59, 60, 54, 118, 116, 119, 122, 120, 51, 63, 108, 114, 50, 38, 44, 123, 125, 40, 117, 105, 102, 95, 127, 58, 55, 41, 98, 18, 47, 87, 39, 86, 82, 24, 57, 53, 52, 126, 124, 16, 103, 88, 115, 34, 49, 80, 23, 78, 22, 13, 20, 62, 84, 27, 14, 104, 107, 97, 113, 19, 61, 29, 77, 83, 31, 89, 93, 109, 28, 110, 33, 121, 75, 46, 42, 43, 11, 12, 25, 85, 99, 76, 56, 21, 92, 71, 48, 7, 15, 79, 111, 45, 73, 26, 9, 112, 35, 72, 30, 106, 37, 94, 90, 10, 0, 64, 5, 81, 101, 67, 74, 36, 17, 91, 4, 66, 32, 68, 65, 2, 69, 1, 70, 3, 96, 8, 100, 6], [59, 60, 54, 118, 116, 119, 122, 120, 51, 63, 108, 114, 50, 38, 117, 125, 44, 40, 123, 105, 127, 18, 55, 102, 86, 98, 87, 82, 95, 24, 53, 41, 39, 22, 16, 88, 78, 47, 62, 126, 124, 57, 80, 34, 84, 23, 52, 20, 29, 49, 103, 27, 58, 113, 13, 97, 115, 104, 46, 31, 83, 14, 121, 89, 110, 77, 28, 93, 43, 76, 107, 11, 75, 19, 12, 112, 92, 61, 21, 73, 99, 48, 15, 25, 90, 79, 85, 9, 109, 45, 42, 106, 33, 72, 7, 71, 56, 81, 111, 35, 30, 0, 26, 37, 36, 64, 69, 91, 10, 17, 67, 74, 3, 4, 94, 65, 8, 68, 5, 70, 66, 2, 1, 6, 101, 32, 100, 96], [59, 60, 118, 54, 116, 119, 122, 120, 63, 51, 114, 50, 108, 38, 117, 44, 125, 123, 105, 40, 18, 102, 41, 127, 98, 52, 87, 34, 82, 86, 95, 88, 115, 22, 57, 23, 55, 124, 62, 24, 126, 39, 80, 84, 20, 16, 97, 58, 53, 78, 103, 110, 104, 47, 27, 89, 107, 61, 92, 49, 113, 29, 43, 13, 31, 14, 19, 28, 93, 85, 83, 77, 99, 21, 76, 25, 75, 11, 12, 56, 15, 73, 112, 109, 42, 45, 121, 46, 33, 111, 90, 9, 48, 79, 35, 26, 37, 106, 91, 7, 94, 30, 71, 17, 36, 8, 72, 81, 74, 10, 32, 69, 101, 100, 66, 68, 3, 70, 5, 4, 0, 67, 65, 64, 1, 6, 2, 96], [59, 60, 54, 118, 116, 119, 122, 120, 63, 51, 114, 50, 108, 125, 38, 44, 117, 123, 105, 40, 53, 102, 41, 55, 18, 86, 95, 82, 34, 62, 127, 87, 98, 52, 124, 16, 88, 80, 103, 22, 115, 23, 24, 47, 84, 20, 58, 57, 39, 13, 104, 14, 78, 61, 49, 85, 107, 43, 110, 12, 27, 92, 31, 89, 97, 113, 19, 93, 126, 83, 21, 45, 76, 11, 77, 28, 75, 79, 29, 99, 121, 15, 112, 48, 56, 25, 42, 9, 73, 46, 7, 71, 90, 33, 8, 106, 30, 109, 26, 94, 0, 64, 111, 35, 10, 72, 91, 17, 81, 37, 74, 69, 65, 66, 4, 67, 32, 5, 68, 70, 2, 36, 101, 1, 6, 3, 100, 96], [59, 60, 54, 118, 116, 122, 119, 120, 63, 51, 108, 50, 114, 38, 40, 44, 117, 123, 105, 102, 125, 86, 95, 98, 18, 82, 53, 124, 41, 127, 87, 55, 84, 34, 88, 39, 62, 24, 52, 104, 57, 23, 58, 47, 49, 103, 97, 20, 16, 80, 115, 126, 14, 22, 110, 31, 78, 43, 56, 46, 113, 27, 61, 13, 29, 99, 89, 12, 107, 45, 106, 85, 77, 93, 19, 48, 83, 28, 92, 109, 75, 15, 112, 76, 25, 37, 79, 33, 21, 7, 94, 11, 73, 90, 35, 42, 71, 111, 8, 121, 91, 9, 30, 36, 26, 17, 81, 10, 0, 101, 72, 32, 67, 3, 69, 64, 68, 74, 5, 65, 100, 2, 4, 6, 66, 1, 70, 96], [59, 60, 54, 118, 116, 119, 122, 120, 51, 63, 108, 114, 50, 38, 125, 44, 40, 123, 105, 102, 117, 86, 52, 124, 18, 127, 98, 115, 82, 57, 41, 88, 95, 103, 39, 87, 78, 62, 24, 34, 61, 107, 16, 20, 126, 23, 47, 53, 110, 97, 104, 22, 55, 80, 84, 49, 29, 31, 27, 89, 109, 19, 83, 113, 45, 85, 99, 58, 46, 93, 48, 14, 13, 21, 25, 43, 77, 28, 76, 92, 121, 11, 56, 37, 12, 42, 112, 75, 8, 106, 9, 79, 33, 30, 15, 7, 35, 73, 36, 90, 26, 71, 111, 17, 91, 94, 32, 67, 10, 81, 74, 68, 5, 64, 0, 65, 4, 2, 3, 101, 69, 1, 70, 96, 72, 66, 100, 6], [59, 60, 54, 118, 116, 122, 119, 63, 120, 51, 108, 114, 50, 38, 117, 44, 40, 123, 105, 125, 86, 18, 102, 82, 41, 98, 52, 87, 95, 22, 34, 124, 88, 24, 115, 39, 127, 16, 23, 62, 20, 80, 55, 104, 49, 103, 84, 126, 110, 58, 48, 56, 97, 78, 57, 27, 53, 93, 89, 83, 92, 29, 14, 47, 19, 45, 31, 107, 13, 77, 61, 85, 28, 113, 21, 76, 11, 12, 33, 42, 9, 99, 112, 75, 25, 46, 79, 43, 121, 8, 15, 90, 37, 73, 35, 71, 7, 30, 109, 106, 94, 26, 17, 10, 111, 91, 5, 81, 74, 36, 101, 32, 4, 64, 6, 0, 72, 68, 100, 66, 69, 70, 65, 2, 96, 1, 67, 3], [59, 60, 54, 118, 116, 119, 122, 120, 63, 51, 108, 50, 38, 114, 44, 40, 105, 117, 86, 125, 123, 102, 127, 41, 82, 98, 87, 18, 124, 34, 22, 24, 95, 39, 88, 62, 97, 115, 23, 52, 20, 16, 55, 103, 80, 84, 126, 110, 27, 56, 49, 57, 93, 58, 104, 19, 29, 31, 78, 48, 89, 83, 92, 47, 13, 113, 85, 14, 99, 25, 28, 12, 107, 76, 53, 61, 21, 77, 112, 26, 75, 42, 45, 46, 15, 111, 7, 90, 11, 33, 109, 79, 43, 30, 9, 35, 94, 8, 73, 37, 106, 71, 121, 17, 91, 32, 36, 5, 81, 101, 6, 74, 10, 100, 0, 64, 67, 72, 4, 68, 96, 66, 3, 70, 2, 69, 65, 1], [59, 60, 54, 118, 116, 119, 122, 120, 51, 63, 108, 114, 50, 38, 117, 40, 44, 123, 105, 125, 98, 86, 102, 87, 62, 34, 95, 82, 97, 39, 127, 124, 57, 41, 18, 52, 115, 24, 49, 88, 22, 55, 103, 23, 58, 53, 80, 126, 104, 29, 84, 16, 110, 56, 45, 20, 89, 61, 99, 25, 27, 31, 14, 26, 48, 46, 47, 83, 113, 93, 109, 92, 19, 33, 107, 28, 43, 42, 13, 78, 112, 12, 85, 77, 37, 35, 75, 15, 21, 73, 121, 76, 8, 9, 106, 36, 30, 94, 90, 11, 7, 79, 32, 91, 17, 71, 111, 101, 69, 3, 81, 74, 64, 10, 67, 0, 65, 4, 96, 5, 72, 6, 68, 66, 2, 1, 100, 70], [59, 60, 54, 118, 116, 122, 119, 63, 120, 51, 50, 108, 114, 38, 117, 44, 125, 40, 105, 102, 123, 86, 127, 98, 95, 41, 52, 82, 34, 18, 103, 39, 87, 115, 24, 57, 88, 55, 124, 62, 22, 23, 104, 80, 20, 16, 107, 126, 97, 27, 89, 110, 84, 56, 61, 58, 14, 83, 47, 53, 49, 12, 77, 31, 113, 25, 78, 93, 19, 99, 13, 28, 109, 48, 92, 29, 21, 46, 45, 30, 75, 76, 85, 112, 33, 43, 106, 42, 11, 73, 37, 26, 71, 35, 79, 121, 15, 9, 90, 7, 36, 17, 8, 91, 94, 111, 101, 74, 32, 10, 72, 69, 64, 81, 67, 65, 0, 2, 68, 96, 4, 5, 6, 100, 70, 66, 3, 1], [59, 60, 118, 54, 116, 119, 122, 63, 120, 51, 108, 50, 114, 38, 105, 44, 40, 86, 117, 125, 123, 102, 127, 18, 41, 124, 87, 126, 39, 52, 82, 88, 98, 55, 95, 22, 103, 49, 24, 115, 57, 34, 16, 23, 80, 20, 78, 62, 84, 53, 47, 56, 83, 104, 27, 97, 93, 13, 77, 29, 89, 14, 19, 110, 111, 61, 107, 113, 85, 75, 58, 31, 12, 48, 99, 76, 28, 21, 25, 109, 9, 71, 45, 92, 106, 79, 43, 11, 90, 37, 33, 26, 46, 42, 30, 121, 112, 73, 35, 7, 15, 36, 81, 94, 8, 32, 72, 67, 74, 17, 91, 64, 10, 68, 69, 3, 2, 0, 5, 101, 6, 1, 70, 65, 4, 66, 96, 100], [59, 60, 118, 54, 119, 122, 116, 120, 63, 51, 108, 50, 38, 114, 117, 44, 123, 105, 40, 86, 125, 87, 18, 102, 41, 98, 82, 55, 62, 124, 115, 39, 52, 34, 24, 95, 22, 57, 88, 103, 80, 127, 126, 23, 97, 47, 16, 84, 29, 48, 20, 78, 53, 89, 110, 27, 49, 61, 93, 56, 43, 92, 19, 58, 31, 42, 25, 83, 107, 104, 77, 113, 14, 75, 13, 99, 76, 85, 28, 112, 121, 12, 21, 46, 90, 106, 109, 79, 26, 11, 33, 30, 9, 15, 35, 73, 94, 111, 37, 45, 36, 71, 72, 8, 17, 7, 91, 81, 74, 5, 0, 101, 32, 64, 10, 3, 100, 68, 2, 67, 65, 69, 70, 96, 4, 1, 66, 6], [59, 60, 118, 54, 119, 116, 122, 63, 120, 51, 108, 114, 38, 50, 117, 44, 123, 105, 125, 40, 124, 41, 86, 98, 34, 102, 18, 39, 87, 127, 95, 62, 115, 52, 82, 24, 22, 55, 23, 57, 103, 88, 47, 16, 84, 53, 49, 97, 126, 80, 61, 20, 48, 58, 27, 104, 78, 56, 89, 13, 45, 19, 93, 113, 14, 92, 77, 31, 76, 110, 29, 83, 99, 107, 11, 85, 42, 12, 112, 28, 25, 75, 21, 26, 37, 121, 73, 43, 9, 46, 106, 111, 109, 33, 90, 79, 30, 15, 94, 7, 35, 72, 71, 36, 91, 17, 101, 32, 74, 0, 81, 64, 69, 100, 10, 65, 3, 8, 5, 68, 2, 96, 66, 67, 6, 4, 70, 1], [59, 60, 118, 54, 119, 116, 122, 63, 120, 51, 108, 114, 50, 38, 117, 125, 44, 40, 105, 123, 86, 102, 98, 127, 62, 95, 124, 87, 52, 41, 88, 82, 39, 18, 103, 115, 23, 34, 24, 57, 53, 16, 22, 126, 49, 84, 92, 110, 55, 58, 80, 97, 27, 47, 25, 20, 107, 56, 89, 31, 78, 83, 99, 42, 14, 104, 19, 121, 93, 77, 29, 48, 21, 13, 113, 12, 43, 45, 76, 28, 75, 85, 46, 73, 11, 72, 61, 15, 33, 109, 106, 30, 35, 112, 90, 37, 79, 26, 7, 111, 36, 91, 71, 9, 32, 94, 68, 17, 81, 0, 74, 69, 65, 70, 64, 100, 5, 67, 101, 3, 2, 10, 1, 66, 4, 6, 8, 96], [59, 60, 54, 118, 116, 119, 122, 63, 120, 51, 108, 50, 114, 38, 44, 117, 40, 125, 123, 105, 86, 102, 127, 98, 52, 124, 18, 41, 82, 39, 95, 87, 55, 57, 24, 34, 62, 88, 126, 22, 16, 97, 115, 53, 84, 49, 110, 47, 103, 58, 23, 80, 20, 83, 14, 29, 99, 104, 46, 48, 43, 56, 77, 89, 13, 61, 113, 78, 27, 93, 31, 25, 28, 73, 45, 76, 92, 85, 21, 107, 111, 19, 106, 121, 12, 75, 11, 112, 109, 72, 33, 15, 9, 42, 90, 35, 37, 79, 26, 71, 36, 94, 7, 81, 91, 74, 32, 5, 17, 30, 101, 10, 0, 64, 100, 68, 4, 70, 67, 2, 3, 69, 1, 96, 6, 8, 65, 66], [59, 60, 118, 54, 116, 119, 122, 63, 120, 51, 108, 38, 50, 114, 44, 117, 40, 125, 105, 123, 102, 86, 52, 98, 18, 127, 124, 82, 87, 95, 22, 39, 41, 88, 126, 115, 34, 55, 62, 24, 103, 97, 16, 23, 57, 84, 80, 20, 110, 47, 83, 58, 89, 27, 14, 53, 45, 56, 78, 49, 12, 19, 92, 29, 93, 61, 13, 107, 77, 104, 46, 21, 42, 43, 121, 99, 85, 28, 31, 76, 75, 9, 48, 15, 11, 79, 26, 112, 35, 73, 25, 106, 71, 90, 33, 72, 109, 113, 30, 37, 17, 36, 74, 94, 7, 81, 91, 32, 111, 64, 0, 101, 10, 100, 69, 4, 5, 66, 65, 8, 70, 3, 96, 6, 67, 2, 1, 68], [59, 60, 118, 54, 116, 119, 122, 120, 63, 51, 50, 108, 114, 38, 40, 123, 105, 44, 117, 125, 62, 102, 124, 98, 87, 41, 86, 55, 57, 95, 82, 126, 34, 22, 52, 47, 127, 24, 39, 18, 88, 103, 97, 115, 84, 27, 89, 104, 23, 20, 53, 16, 49, 29, 93, 80, 78, 99, 48, 31, 61, 58, 83, 92, 42, 121, 21, 45, 110, 13, 107, 25, 77, 14, 85, 46, 19, 111, 12, 43, 90, 33, 11, 75, 26, 112, 28, 37, 9, 56, 15, 94, 30, 109, 76, 113, 35, 91, 7, 73, 106, 72, 79, 71, 17, 32, 101, 100, 67, 74, 69, 36, 0, 10, 4, 2, 68, 1, 66, 64, 65, 5, 81, 3, 70, 96, 6, 8], [59, 60, 118, 54, 116, 119, 122, 63, 120, 51, 108, 114, 50, 38, 44, 40, 105, 123, 117, 125, 86, 82, 87, 41, 127, 124, 102, 95, 98, 52, 39, 126, 18, 24, 34, 62, 88, 57, 22, 55, 23, 47, 16, 80, 115, 58, 20, 97, 84, 78, 83, 103, 104, 27, 56, 14, 93, 89, 29, 12, 92, 110, 19, 13, 107, 77, 76, 75, 49, 53, 21, 121, 45, 48, 61, 85, 11, 99, 46, 31, 43, 42, 109, 79, 15, 28, 9, 7, 73, 71, 111, 112, 113, 106, 25, 30, 0, 26, 94, 64, 72, 37, 33, 90, 35, 69, 17, 65, 74, 3, 8, 6, 4, 68, 66, 91, 1, 67, 10, 101, 2, 81, 36, 32, 5, 70, 96, 100], [59, 60, 118, 54, 116, 119, 122, 120, 51, 63, 114, 108, 50, 38, 40, 117, 105, 44, 123, 102, 98, 127, 87, 125, 57, 41, 124, 18, 82, 95, 39, 52, 126, 86, 34, 62, 88, 24, 22, 103, 55, 97, 115, 58, 23, 84, 49, 20, 27, 16, 80, 104, 53, 31, 56, 89, 99, 47, 93, 78, 113, 29, 107, 83, 13, 121, 14, 110, 12, 77, 85, 25, 48, 109, 61, 43, 92, 45, 19, 28, 26, 106, 46, 21, 33, 90, 11, 94, 37, 75, 30, 76, 9, 111, 42, 35, 71, 15, 73, 112, 79, 36, 72, 91, 7, 8, 32, 64, 69, 74, 10, 17, 0, 66, 67, 1, 68, 6, 65, 101, 81, 3, 4, 5, 2, 96, 70, 100], [59, 60, 118, 54, 122, 116, 119, 120, 51, 63, 114, 108, 50, 38, 44, 125, 40, 105, 123, 117, 102, 86, 98, 52, 87, 41, 18, 127, 34, 82, 95, 124, 126, 88, 53, 57, 39, 115, 58, 55, 24, 80, 20, 47, 16, 103, 78, 97, 84, 22, 14, 23, 19, 31, 56, 62, 93, 13, 113, 49, 77, 27, 76, 42, 48, 83, 75, 43, 121, 89, 110, 85, 21, 12, 107, 29, 99, 92, 61, 104, 45, 25, 11, 46, 73, 109, 35, 15, 79, 9, 28, 106, 26, 30, 8, 90, 71, 7, 33, 112, 37, 94, 111, 17, 64, 72, 0, 10, 74, 66, 69, 65, 4, 81, 100, 3, 91, 36, 5, 1, 6, 68, 32, 67, 70, 101, 2, 96], [59, 60, 118, 54, 116, 122, 119, 120, 51, 63, 114, 108, 50, 38, 44, 117, 40, 123, 125, 105, 55, 41, 102, 98, 52, 86, 95, 127, 126, 34, 124, 18, 53, 82, 115, 49, 87, 58, 39, 57, 84, 24, 23, 88, 80, 103, 62, 97, 47, 22, 20, 16, 113, 14, 31, 104, 89, 27, 110, 48, 99, 45, 19, 29, 78, 61, 107, 77, 85, 92, 12, 56, 121, 93, 21, 13, 76, 83, 11, 43, 109, 112, 42, 106, 73, 28, 75, 25, 33, 15, 8, 46, 9, 111, 26, 71, 79, 90, 35, 94, 37, 30, 7, 64, 69, 17, 74, 10, 81, 68, 32, 5, 1, 0, 2, 72, 4, 36, 91, 67, 101, 6, 70, 66, 3, 65, 100, 96], [59, 60, 54, 118, 116, 122, 119, 120, 63, 51, 108, 114, 50, 38, 123, 44, 125, 117, 40, 105, 86, 41, 52, 127, 98, 102, 55, 87, 82, 34, 95, 58, 115, 18, 126, 62, 22, 24, 53, 124, 23, 88, 80, 47, 39, 103, 16, 20, 57, 84, 97, 49, 27, 14, 12, 77, 104, 19, 92, 93, 89, 21, 13, 15, 48, 83, 110, 76, 31, 78, 61, 45, 85, 11, 107, 113, 43, 42, 29, 73, 33, 9, 99, 121, 28, 75, 109, 25, 56, 46, 26, 8, 79, 71, 90, 35, 7, 37, 106, 30, 94, 112, 111, 81, 74, 64, 17, 91, 5, 10, 0, 32, 68, 36, 1, 65, 67, 69, 72, 4, 6, 101, 70, 2, 100, 66, 3, 96]], "model.layers.9.self_attn.q_proj": [[110, 101, 46, 124, 63, 59, 28, 32, 113, 25, 89, 61, 19, 115, 27, 87, 16, 60, 57, 49, 122, 83, 85, 26, 78, 22, 47, 114, 55, 58, 56, 53, 96, 99, 37, 92, 108, 41, 23, 54, 48, 125, 30, 79, 51, 11, 119, 67, 84, 1, 24, 118, 69, 35, 62, 91, 120, 123, 116, 70, 43, 88, 97, 117, 121, 68, 106, 112, 90, 126, 109, 29, 40, 2, 77, 127, 45, 111, 8, 17, 86, 0, 74, 50, 82, 107, 15, 5, 42, 9, 66, 21, 18, 10, 52, 14, 6, 104, 105, 94, 36, 93, 100, 81, 65, 102, 44, 34, 33, 76, 20, 80, 31, 103, 3, 64, 7, 38, 4, 71, 95, 75, 73, 12, 13, 39, 98, 72], [110, 124, 101, 63, 46, 113, 25, 59, 80, 28, 24, 61, 89, 115, 57, 122, 60, 56, 83, 114, 32, 53, 58, 47, 125, 27, 22, 51, 41, 54, 119, 49, 78, 55, 40, 11, 112, 118, 62, 48, 123, 43, 96, 120, 106, 69, 23, 121, 37, 108, 52, 117, 127, 50, 42, 109, 19, 104, 45, 100, 111, 116, 35, 126, 105, 16, 107, 85, 91, 87, 102, 30, 38, 39, 31, 44, 33, 92, 34, 103, 84, 36, 86, 20, 17, 94, 88, 26, 97, 73, 98, 29, 99, 93, 8, 90, 95, 77, 72, 14, 18, 1, 21, 12, 67, 81, 5, 79, 76, 15, 82, 70, 4, 75, 2, 0, 10, 9, 7, 13, 6, 74, 66, 3, 68, 71, 64, 65], [110, 124, 46, 101, 63, 59, 89, 61, 11, 37, 113, 57, 25, 122, 60, 115, 69, 56, 53, 19, 54, 58, 47, 55, 32, 114, 120, 48, 16, 51, 49, 41, 117, 118, 116, 28, 112, 50, 62, 27, 109, 24, 119, 22, 121, 111, 125, 92, 127, 99, 126, 123, 43, 87, 45, 52, 82, 30, 107, 108, 39, 100, 72, 35, 103, 42, 105, 106, 1, 96, 104, 83, 40, 0, 4, 44, 23, 78, 2, 86, 77, 80, 26, 102, 15, 38, 29, 34, 85, 97, 90, 84, 67, 73, 36, 31, 33, 14, 88, 94, 5, 6, 98, 95, 18, 17, 9, 93, 91, 7, 21, 8, 20, 68, 79, 12, 81, 75, 66, 10, 13, 76, 71, 70, 74, 64, 3, 65], [124, 110, 101, 59, 46, 89, 63, 61, 19, 28, 115, 25, 83, 60, 113, 56, 27, 57, 23, 114, 122, 24, 118, 54, 78, 53, 55, 22, 125, 119, 47, 49, 58, 120, 48, 51, 37, 26, 107, 93, 32, 50, 81, 96, 117, 121, 99, 112, 11, 106, 43, 62, 123, 92, 127, 41, 35, 109, 108, 17, 116, 111, 103, 45, 42, 36, 104, 52, 44, 82, 126, 40, 100, 105, 30, 39, 38, 85, 88, 95, 14, 12, 34, 86, 87, 8, 98, 97, 31, 33, 102, 29, 16, 91, 94, 77, 69, 84, 18, 21, 73, 20, 15, 90, 80, 75, 76, 67, 13, 5, 79, 72, 9, 6, 74, 7, 1, 10, 70, 4, 3, 71, 2, 0, 65, 68, 66, 64], [117, 41, 59, 61, 97, 126, 101, 39, 31, 87, 55, 43, 42, 44, 28, 100, 124, 112, 110, 116, 35, 92, 102, 113, 96, 122, 57, 86, 36, 121, 127, 89, 54, 58, 114, 63, 38, 49, 52, 115, 93, 125, 47, 94, 51, 60, 53, 105, 99, 120, 16, 80, 48, 111, 32, 45, 50, 84, 46, 98, 29, 104, 19, 56, 118, 109, 123, 108, 40, 62, 27, 78, 33, 24, 107, 23, 74, 18, 26, 17, 25, 83, 37, 85, 75, 30, 91, 22, 20, 119, 21, 34, 14, 106, 12, 88, 9, 68, 81, 13, 103, 15, 77, 10, 3, 7, 76, 73, 95, 11, 71, 79, 4, 82, 66, 8, 90, 2, 70, 5, 6, 0, 72, 69, 64, 67, 65, 1], [41, 64, 107, 101, 117, 31, 67, 63, 1, 56, 69, 55, 26, 97, 42, 126, 112, 87, 105, 2, 72, 10, 39, 18, 84, 79, 76, 98, 71, 124, 65, 43, 106, 28, 78, 4, 35, 9, 111, 6, 44, 100, 53, 70, 29, 93, 23, 114, 80, 12, 121, 3, 74, 58, 86, 66, 20, 92, 94, 57, 5, 81, 11, 89, 59, 127, 33, 110, 102, 99, 83, 38, 75, 7, 48, 125, 109, 8, 61, 19, 15, 17, 77, 27, 21, 0, 46, 113, 88, 62, 118, 68, 50, 85, 45, 116, 32, 54, 104, 47, 24, 25, 120, 52, 122, 96, 40, 13, 51, 115, 103, 60, 90, 30, 34, 14, 73, 119, 123, 49, 82, 36, 16, 108, 91, 22, 95, 37], [41, 117, 26, 87, 31, 42, 101, 97, 56, 98, 18, 124, 35, 84, 105, 79, 15, 23, 12, 76, 55, 39, 57, 78, 43, 20, 80, 7, 59, 112, 93, 106, 118, 44, 74, 90, 61, 102, 10, 89, 100, 63, 28, 83, 114, 126, 107, 82, 58, 9, 53, 25, 104, 92, 69, 81, 14, 48, 8, 96, 36, 21, 127, 95, 86, 70, 24, 49, 121, 110, 99, 11, 29, 60, 111, 22, 46, 27, 17, 62, 85, 51, 122, 109, 16, 4, 52, 120, 125, 32, 5, 19, 77, 33, 13, 123, 40, 54, 45, 91, 2, 30, 67, 94, 71, 47, 3, 73, 115, 38, 113, 116, 72, 88, 64, 108, 50, 37, 103, 66, 65, 75, 68, 34, 0, 119, 6, 1], [117, 56, 41, 61, 107, 43, 97, 39, 31, 126, 35, 44, 118, 100, 42, 57, 110, 89, 121, 102, 36, 105, 46, 87, 63, 48, 101, 94, 52, 124, 92, 28, 59, 53, 113, 50, 86, 55, 38, 98, 54, 122, 49, 51, 29, 112, 111, 25, 45, 115, 108, 83, 62, 123, 125, 127, 80, 40, 47, 96, 104, 19, 99, 114, 27, 58, 116, 120, 60, 33, 109, 34, 88, 26, 18, 32, 93, 85, 9, 23, 106, 22, 64, 21, 24, 16, 119, 84, 30, 17, 103, 13, 91, 37, 77, 14, 90, 1, 2, 81, 69, 78, 67, 76, 95, 11, 79, 4, 82, 66, 73, 71, 68, 7, 70, 10, 75, 72, 20, 5, 12, 74, 6, 8, 65, 15, 3, 0], [105, 98, 86, 18, 84, 53, 79, 81, 88, 13, 111, 41, 76, 119, 74, 51, 72, 55, 29, 10, 117, 52, 7, 61, 2, 63, 123, 114, 68, 110, 107, 71, 6, 126, 31, 59, 70, 26, 125, 116, 5, 15, 4, 17, 48, 28, 120, 50, 118, 8, 127, 108, 77, 75, 90, 12, 27, 1, 3, 82, 24, 85, 92, 11, 109, 21, 122, 42, 124, 20, 38, 106, 9, 62, 40, 57, 93, 78, 25, 113, 33, 49, 22, 66, 80, 56, 16, 67, 58, 39, 30, 32, 60, 19, 23, 102, 91, 37, 54, 46, 87, 89, 64, 100, 96, 43, 115, 36, 83, 101, 14, 112, 47, 94, 95, 121, 103, 44, 35, 97, 99, 104, 0, 45, 34, 69, 73, 65], [105, 98, 84, 79, 88, 53, 18, 72, 41, 86, 111, 5, 81, 51, 2, 13, 63, 15, 76, 10, 29, 55, 52, 6, 68, 114, 119, 126, 74, 116, 110, 12, 123, 48, 92, 7, 117, 61, 28, 26, 107, 71, 9, 59, 27, 11, 118, 4, 87, 90, 31, 3, 93, 8, 85, 127, 125, 24, 122, 70, 109, 78, 1, 58, 20, 108, 50, 120, 124, 77, 91, 38, 66, 42, 49, 17, 22, 40, 39, 60, 30, 102, 69, 95, 67, 57, 54, 34, 106, 83, 23, 37, 62, 89, 32, 101, 16, 82, 21, 75, 96, 36, 19, 56, 33, 35, 121, 43, 112, 103, 113, 80, 46, 73, 94, 25, 44, 99, 14, 115, 104, 100, 0, 97, 45, 64, 47, 65], [105, 98, 84, 41, 72, 86, 53, 6, 18, 2, 79, 12, 4, 1, 76, 68, 71, 111, 74, 51, 13, 0, 3, 67, 81, 88, 66, 5, 114, 126, 63, 61, 123, 119, 8, 117, 26, 52, 116, 55, 48, 118, 125, 107, 127, 29, 65, 110, 59, 27, 10, 92, 93, 7, 85, 90, 20, 17, 109, 34, 120, 24, 9, 87, 50, 28, 122, 124, 22, 58, 62, 78, 70, 80, 49, 42, 32, 57, 108, 31, 30, 106, 83, 101, 35, 25, 56, 54, 21, 102, 77, 91, 14, 82, 113, 95, 64, 89, 15, 112, 38, 75, 97, 39, 36, 43, 115, 60, 40, 33, 121, 23, 19, 103, 37, 94, 11, 100, 46, 104, 45, 99, 69, 16, 96, 47, 44, 73], [105, 98, 74, 3, 70, 0, 86, 53, 111, 4, 1, 81, 18, 79, 76, 61, 5, 41, 67, 66, 119, 13, 117, 71, 8, 84, 2, 51, 110, 123, 68, 114, 55, 12, 52, 72, 126, 50, 125, 65, 120, 116, 107, 63, 17, 6, 9, 20, 59, 57, 64, 118, 88, 29, 109, 48, 10, 127, 49, 56, 38, 62, 73, 124, 34, 26, 7, 22, 40, 106, 78, 87, 23, 30, 15, 16, 28, 11, 24, 69, 93, 92, 19, 77, 89, 80, 83, 14, 60, 31, 94, 113, 75, 42, 39, 32, 82, 85, 108, 122, 54, 25, 43, 58, 121, 27, 90, 21, 96, 102, 115, 44, 101, 99, 112, 91, 36, 46, 95, 35, 33, 103, 104, 45, 37, 100, 47, 97], [125, 62, 104, 48, 63, 119, 122, 120, 59, 108, 55, 57, 118, 54, 121, 116, 58, 60, 27, 124, 113, 114, 115, 117, 84, 123, 111, 61, 97, 56, 53, 50, 43, 51, 47, 30, 44, 87, 52, 49, 126, 127, 107, 45, 109, 25, 110, 89, 12, 46, 21, 105, 28, 22, 72, 99, 112, 103, 41, 36, 38, 94, 106, 34, 5, 98, 90, 39, 102, 101, 20, 100, 1, 37, 92, 14, 9, 66, 96, 68, 42, 19, 16, 83, 35, 32, 79, 17, 29, 33, 95, 2, 85, 69, 31, 81, 0, 23, 40, 3, 91, 77, 86, 88, 18, 4, 93, 6, 15, 64, 82, 71, 76, 13, 11, 26, 24, 10, 73, 80, 78, 75, 74, 7, 8, 70, 65, 67], [62, 125, 48, 104, 63, 119, 122, 120, 59, 108, 97, 115, 55, 58, 54, 57, 60, 117, 114, 116, 121, 52, 124, 113, 50, 118, 47, 123, 61, 56, 111, 84, 36, 53, 127, 51, 89, 49, 109, 126, 44, 87, 5, 43, 72, 45, 14, 96, 69, 103, 46, 112, 107, 12, 30, 41, 110, 1, 106, 102, 105, 68, 2, 27, 32, 101, 66, 37, 28, 39, 38, 42, 24, 88, 98, 21, 25, 20, 22, 19, 35, 95, 99, 92, 100, 31, 64, 34, 23, 40, 83, 3, 93, 4, 15, 90, 91, 71, 76, 29, 85, 75, 79, 0, 6, 18, 94, 78, 9, 8, 17, 73, 77, 81, 13, 16, 7, 26, 10, 80, 33, 86, 11, 70, 74, 67, 82, 65], [104, 125, 62, 48, 63, 97, 119, 30, 21, 57, 122, 60, 59, 28, 127, 115, 118, 120, 116, 108, 55, 85, 84, 27, 54, 121, 113, 126, 123, 53, 114, 49, 56, 25, 111, 58, 117, 61, 22, 89, 124, 87, 52, 47, 9, 112, 109, 90, 44, 107, 51, 50, 45, 43, 40, 35, 5, 79, 46, 36, 105, 103, 94, 66, 1, 18, 68, 24, 41, 78, 92, 16, 12, 34, 15, 2, 98, 99, 102, 38, 95, 110, 72, 39, 19, 32, 3, 14, 96, 69, 11, 106, 101, 64, 31, 42, 37, 0, 6, 93, 91, 73, 100, 88, 83, 17, 23, 75, 20, 8, 29, 81, 4, 77, 26, 33, 13, 70, 71, 74, 7, 10, 80, 86, 76, 67, 65, 82], [104, 62, 125, 90, 30, 97, 63, 122, 84, 18, 19, 115, 23, 60, 80, 22, 16, 119, 27, 74, 79, 38, 57, 49, 26, 36, 78, 55, 20, 54, 59, 39, 82, 13, 124, 99, 120, 108, 94, 116, 123, 112, 45, 87, 21, 121, 111, 118, 114, 28, 83, 24, 113, 61, 126, 33, 88, 50, 58, 44, 127, 56, 17, 75, 48, 107, 10, 53, 102, 92, 34, 81, 100, 91, 109, 117, 96, 43, 51, 101, 47, 77, 103, 52, 14, 46, 25, 93, 15, 105, 37, 41, 31, 106, 6, 42, 110, 73, 98, 85, 70, 40, 5, 89, 12, 35, 32, 76, 95, 86, 68, 8, 29, 4, 7, 11, 71, 3, 9, 64, 1, 66, 65, 72, 67, 69, 0, 2], [103, 124, 55, 56, 21, 33, 15, 57, 29, 113, 81, 83, 91, 88, 27, 122, 123, 125, 93, 41, 10, 25, 35, 49, 98, 114, 24, 12, 78, 61, 105, 60, 48, 30, 109, 86, 115, 22, 52, 120, 112, 85, 53, 51, 110, 58, 31, 116, 43, 62, 45, 17, 94, 108, 44, 84, 100, 42, 107, 102, 70, 119, 111, 90, 126, 23, 18, 76, 118, 121, 117, 47, 36, 46, 4, 106, 127, 26, 89, 54, 63, 32, 50, 59, 20, 19, 11, 99, 8, 28, 40, 79, 101, 96, 13, 104, 37, 80, 38, 82, 95, 92, 16, 77, 34, 87, 72, 6, 65, 14, 2, 75, 97, 9, 5, 68, 39, 73, 74, 71, 7, 69, 0, 66, 67, 1, 64, 3], [103, 124, 55, 57, 88, 33, 56, 21, 105, 81, 113, 93, 123, 29, 15, 83, 23, 41, 122, 91, 125, 19, 49, 70, 78, 27, 114, 10, 12, 44, 86, 24, 126, 117, 45, 90, 98, 109, 2, 22, 58, 76, 52, 74, 4, 108, 120, 30, 60, 118, 107, 115, 48, 111, 106, 84, 110, 43, 65, 5, 85, 51, 35, 53, 119, 32, 18, 38, 39, 102, 17, 112, 99, 95, 34, 100, 104, 79, 47, 68, 54, 116, 92, 101, 42, 121, 20, 97, 61, 62, 31, 26, 46, 28, 96, 127, 87, 36, 11, 6, 50, 82, 94, 73, 37, 14, 89, 75, 59, 77, 8, 25, 16, 67, 40, 7, 63, 80, 0, 13, 9, 69, 72, 71, 1, 66, 64, 3], [103, 56, 124, 55, 57, 33, 88, 29, 113, 21, 15, 93, 105, 23, 86, 10, 77, 81, 114, 123, 91, 122, 64, 67, 125, 120, 78, 109, 0, 45, 85, 83, 60, 49, 61, 51, 65, 41, 52, 115, 53, 58, 119, 13, 110, 116, 112, 118, 27, 34, 2, 108, 48, 73, 24, 84, 71, 62, 42, 63, 89, 30, 50, 90, 14, 8, 6, 46, 126, 117, 107, 43, 70, 35, 26, 11, 17, 44, 121, 47, 111, 80, 3, 127, 20, 54, 59, 102, 7, 104, 100, 87, 37, 38, 82, 79, 106, 40, 22, 31, 9, 16, 97, 99, 36, 1, 18, 28, 101, 98, 66, 76, 12, 32, 25, 94, 68, 96, 4, 75, 92, 95, 5, 74, 19, 72, 69, 39], [124, 103, 55, 57, 41, 123, 88, 56, 125, 70, 114, 74, 122, 33, 19, 113, 21, 93, 110, 115, 23, 45, 49, 109, 120, 118, 52, 58, 116, 61, 15, 53, 108, 2, 112, 48, 83, 51, 60, 43, 35, 117, 81, 62, 29, 119, 126, 105, 73, 44, 50, 91, 121, 84, 127, 65, 111, 47, 46, 12, 107, 86, 42, 54, 63, 98, 30, 68, 106, 39, 14, 22, 59, 78, 100, 79, 27, 40, 102, 101, 10, 90, 5, 4, 38, 104, 67, 96, 99, 92, 24, 13, 36, 37, 34, 75, 26, 69, 20, 32, 8, 97, 85, 87, 17, 7, 28, 80, 25, 82, 18, 31, 76, 11, 94, 0, 71, 95, 16, 6, 77, 89, 9, 72, 3, 64, 1, 66], [57, 126, 39, 34, 61, 45, 95, 111, 120, 125, 60, 21, 51, 63, 58, 87, 52, 90, 54, 113, 114, 119, 117, 123, 109, 55, 127, 48, 50, 62, 118, 112, 122, 19, 106, 53, 56, 49, 46, 72, 59, 43, 115, 124, 110, 5, 82, 116, 108, 100, 41, 22, 36, 121, 47, 105, 17, 104, 28, 44, 26, 92, 107, 102, 88, 35, 73, 9, 42, 79, 14, 24, 16, 40, 38, 37, 77, 76, 78, 89, 99, 101, 27, 18, 30, 97, 75, 10, 32, 71, 93, 83, 98, 1, 33, 94, 13, 81, 66, 96, 29, 86, 25, 91, 103, 85, 8, 67, 11, 4, 2, 0, 20, 12, 74, 31, 70, 23, 15, 84, 69, 65, 3, 6, 7, 80, 68, 64], [39, 126, 57, 34, 95, 90, 19, 87, 60, 120, 21, 26, 61, 17, 92, 11, 111, 48, 79, 28, 31, 5, 71, 72, 22, 10, 45, 63, 117, 84, 27, 9, 58, 70, 102, 42, 4, 67, 100, 125, 127, 105, 40, 36, 108, 51, 83, 113, 99, 16, 101, 13, 38, 73, 41, 54, 96, 75, 46, 35, 53, 77, 124, 122, 52, 106, 66, 50, 44, 37, 49, 81, 59, 47, 112, 118, 62, 23, 104, 109, 119, 33, 43, 116, 85, 115, 114, 121, 110, 30, 32, 55, 123, 29, 97, 107, 25, 56, 0, 24, 20, 89, 65, 93, 15, 7, 80, 94, 18, 98, 91, 74, 1, 88, 76, 78, 6, 68, 12, 14, 82, 103, 69, 86, 8, 2, 3, 64], [39, 57, 126, 34, 95, 87, 60, 21, 19, 90, 61, 16, 120, 73, 28, 26, 22, 79, 75, 17, 77, 48, 111, 125, 5, 58, 51, 83, 13, 63, 9, 92, 45, 104, 74, 33, 82, 113, 118, 27, 112, 122, 109, 127, 18, 123, 7, 71, 78, 20, 10, 89, 117, 62, 23, 119, 31, 30, 49, 50, 36, 115, 53, 56, 81, 94, 102, 54, 24, 84, 76, 52, 70, 25, 67, 47, 96, 68, 55, 110, 114, 91, 44, 88, 85, 116, 108, 41, 40, 8, 15, 6, 2, 46, 38, 121, 99, 80, 1, 105, 100, 37, 106, 14, 29, 97, 124, 93, 42, 98, 35, 12, 32, 72, 101, 107, 59, 0, 3, 43, 86, 11, 69, 103, 64, 4, 65, 66], [39, 57, 126, 34, 120, 87, 95, 90, 19, 60, 26, 92, 11, 61, 21, 111, 36, 17, 72, 22, 5, 48, 117, 63, 125, 71, 9, 4, 127, 67, 58, 83, 16, 79, 75, 70, 66, 73, 52, 51, 27, 113, 13, 122, 91, 119, 53, 50, 115, 45, 46, 31, 10, 28, 124, 65, 123, 23, 55, 118, 109, 54, 56, 114, 40, 62, 20, 121, 15, 18, 32, 49, 0, 77, 7, 102, 81, 44, 47, 82, 37, 85, 112, 101, 38, 106, 43, 110, 100, 35, 24, 116, 80, 107, 105, 78, 99, 33, 76, 41, 74, 94, 59, 1, 14, 84, 108, 97, 30, 104, 86, 42, 88, 96, 68, 93, 89, 6, 25, 98, 8, 12, 3, 2, 69, 29, 64, 103], [59, 61, 53, 101, 27, 88, 76, 78, 18, 16, 25, 20, 22, 85, 29, 69, 7, 96, 28, 75, 9, 30, 119, 62, 100, 56, 87, 66, 39, 14, 126, 83, 82, 117, 90, 37, 2, 71, 43, 67, 3, 12, 91, 72, 21, 79, 80, 74, 77, 73, 84, 0, 33, 24, 26, 93, 81, 65, 15, 23, 19, 6, 17, 1, 31, 54, 94, 8, 92, 36, 50, 10, 89, 5, 97, 105, 111, 41, 95, 13, 35, 106, 108, 116, 107, 42, 70, 103, 68, 113, 86, 32, 124, 48, 49, 4, 11, 99, 44, 60, 98, 64, 34, 55, 102, 63, 115, 120, 46, 52, 125, 104, 121, 123, 122, 110, 45, 57, 58, 127, 40, 47, 109, 118, 38, 114, 112, 51], [53, 59, 61, 100, 119, 56, 107, 50, 124, 52, 126, 113, 123, 116, 60, 54, 62, 115, 117, 120, 122, 58, 23, 125, 63, 110, 121, 57, 55, 111, 77, 45, 108, 39, 49, 114, 37, 43, 127, 48, 109, 112, 47, 118, 90, 51, 42, 46, 103, 104, 44, 38, 92, 41, 40, 101, 106, 32, 98, 105, 36, 84, 33, 99, 28, 31, 102, 35, 34, 17, 73, 26, 97, 95, 3, 94, 15, 64, 2, 11, 30, 12, 5, 74, 14, 96, 82, 70, 13, 29, 93, 83, 24, 85, 27, 7, 1, 25, 69, 19, 68, 87, 72, 66, 21, 71, 91, 79, 80, 76, 9, 8, 4, 20, 75, 16, 81, 67, 89, 18, 78, 6, 88, 22, 86, 65, 0, 10], [59, 53, 61, 101, 88, 71, 75, 18, 78, 27, 16, 76, 2, 22, 69, 25, 20, 9, 68, 117, 56, 0, 119, 79, 5, 74, 81, 66, 85, 91, 93, 14, 7, 96, 72, 11, 83, 10, 87, 24, 82, 90, 28, 30, 15, 6, 13, 80, 39, 3, 4, 89, 43, 29, 21, 32, 84, 12, 116, 23, 111, 33, 26, 70, 19, 35, 64, 41, 92, 94, 1, 8, 31, 100, 77, 49, 73, 67, 62, 17, 54, 37, 65, 99, 95, 103, 55, 126, 86, 124, 108, 38, 98, 50, 113, 34, 36, 97, 115, 60, 122, 105, 42, 44, 104, 123, 110, 47, 107, 102, 48, 121, 63, 52, 120, 106, 46, 58, 40, 45, 57, 125, 109, 127, 51, 112, 118, 114], [61, 53, 119, 124, 113, 63, 54, 47, 52, 57, 115, 117, 114, 120, 56, 50, 58, 111, 55, 118, 127, 125, 59, 62, 123, 121, 51, 110, 45, 49, 122, 116, 48, 112, 109, 126, 44, 43, 60, 108, 46, 42, 106, 107, 105, 32, 103, 36, 40, 101, 41, 100, 37, 33, 39, 104, 99, 86, 22, 102, 35, 23, 38, 34, 29, 95, 92, 98, 28, 97, 31, 96, 90, 83, 17, 94, 26, 19, 81, 20, 93, 30, 27, 25, 82, 16, 88, 87, 21, 18, 24, 80, 89, 14, 84, 85, 15, 91, 13, 78, 77, 79, 76, 67, 72, 74, 75, 70, 12, 10, 9, 71, 11, 1, 68, 0, 73, 69, 8, 6, 3, 2, 4, 5, 66, 64, 7, 65], [127, 124, 102, 33, 62, 51, 94, 56, 22, 18, 24, 121, 60, 116, 59, 37, 119, 39, 113, 122, 125, 114, 27, 91, 53, 93, 99, 30, 110, 38, 55, 126, 32, 25, 61, 58, 57, 49, 104, 118, 95, 112, 54, 28, 123, 48, 117, 47, 63, 50, 97, 46, 2, 111, 115, 108, 7, 120, 84, 29, 52, 45, 109, 31, 44, 21, 101, 71, 75, 107, 34, 106, 89, 3, 0, 90, 82, 85, 70, 35, 42, 98, 92, 103, 40, 78, 6, 105, 36, 41, 67, 65, 73, 83, 15, 43, 23, 20, 26, 14, 16, 87, 69, 88, 80, 68, 72, 96, 19, 100, 1, 76, 4, 64, 12, 77, 8, 66, 17, 79, 11, 10, 81, 74, 13, 5, 86, 9], [124, 102, 127, 47, 24, 33, 62, 94, 18, 51, 91, 84, 22, 75, 25, 36, 28, 121, 114, 56, 15, 37, 26, 60, 40, 110, 112, 116, 32, 53, 59, 122, 126, 119, 38, 27, 58, 79, 89, 20, 109, 107, 13, 125, 113, 11, 57, 87, 30, 123, 111, 34, 63, 55, 103, 35, 117, 48, 100, 49, 93, 46, 29, 54, 41, 45, 108, 9, 105, 61, 7, 42, 118, 31, 39, 85, 101, 88, 115, 106, 50, 80, 98, 96, 92, 99, 52, 120, 82, 90, 23, 104, 19, 3, 67, 97, 95, 44, 83, 17, 69, 21, 0, 16, 10, 76, 5, 71, 12, 74, 81, 2, 43, 14, 78, 86, 72, 68, 77, 73, 8, 4, 65, 66, 64, 6, 1, 70], [102, 127, 124, 94, 24, 33, 84, 13, 30, 93, 19, 22, 9, 25, 62, 18, 28, 17, 15, 92, 82, 88, 97, 16, 83, 73, 91, 38, 11, 23, 56, 51, 20, 79, 42, 85, 47, 112, 36, 89, 77, 69, 27, 46, 81, 31, 87, 95, 90, 106, 29, 67, 110, 119, 34, 2, 32, 80, 35, 86, 96, 8, 5, 0, 76, 26, 14, 12, 39, 122, 7, 74, 21, 78, 37, 10, 72, 108, 99, 71, 120, 3, 60, 59, 100, 114, 70, 123, 65, 107, 75, 63, 98, 116, 40, 68, 6, 103, 104, 105, 126, 101, 111, 4, 52, 55, 1, 57, 58, 54, 41, 66, 109, 45, 118, 50, 125, 64, 48, 117, 121, 53, 113, 115, 44, 43, 49, 61], [124, 127, 102, 47, 51, 62, 121, 56, 85, 59, 116, 119, 114, 60, 110, 113, 125, 58, 126, 53, 91, 122, 55, 123, 61, 112, 54, 48, 49, 63, 57, 50, 117, 118, 18, 109, 115, 108, 26, 41, 120, 111, 107, 101, 46, 105, 40, 45, 52, 21, 33, 38, 44, 25, 104, 42, 43, 36, 106, 39, 86, 22, 73, 99, 35, 37, 82, 76, 24, 34, 100, 30, 103, 79, 97, 94, 92, 3, 15, 98, 32, 29, 95, 96, 31, 27, 77, 90, 28, 89, 23, 75, 93, 65, 69, 7, 68, 84, 88, 17, 83, 12, 20, 81, 87, 0, 1, 72, 10, 66, 67, 19, 16, 80, 11, 64, 70, 6, 9, 5, 78, 71, 13, 14, 4, 74, 2, 8]], "model.layers.9.self_attn.k_proj": [[46, 37, 110, 96, 22, 63, 124, 99, 59, 113, 28, 49, 56, 122, 55, 114, 115, 54, 58, 57, 51, 61, 60, 119, 123, 53, 120, 48, 62, 25, 47, 85, 50, 121, 125, 112, 118, 127, 19, 117, 126, 116, 78, 43, 52, 8, 42, 108, 111, 109, 105, 107, 27, 40, 45, 65, 44, 30, 41, 106, 77, 26, 104, 17, 16, 103, 23, 39, 94, 102, 36, 7, 38, 4, 5, 3, 97, 95, 31, 29, 90, 81, 33, 35, 32, 82, 67, 74, 100, 93, 2, 21, 64, 18, 98, 101, 34, 76, 84, 24, 20, 14, 1, 87, 9, 73, 11, 10, 92, 91, 66, 13, 88, 12, 79, 15, 0, 80, 69, 75, 72, 6, 68, 70, 86, 71, 83, 89], [105, 117, 95, 56, 37, 58, 43, 106, 112, 34, 92, 23, 33, 108, 63, 114, 55, 29, 110, 103, 38, 111, 99, 22, 89, 26, 9, 78, 124, 46, 107, 119, 84, 50, 122, 87, 18, 82, 127, 2, 126, 0, 60, 51, 83, 4, 20, 14, 121, 98, 54, 65, 88, 12, 57, 120, 59, 69, 67, 16, 90, 115, 52, 40, 44, 109, 118, 62, 31, 36, 10, 42, 48, 70, 79, 24, 47, 76, 80, 41, 8, 30, 25, 100, 113, 96, 32, 27, 53, 101, 6, 125, 45, 17, 123, 64, 11, 91, 102, 15, 21, 86, 49, 116, 85, 13, 7, 104, 94, 61, 3, 97, 81, 72, 71, 1, 75, 5, 74, 66, 93, 35, 77, 73, 68, 19, 28, 39], [41, 0, 34, 47, 18, 74, 13, 81, 86, 117, 76, 8, 70, 53, 65, 55, 66, 79, 105, 71, 116, 84, 3, 4, 50, 63, 51, 112, 126, 59, 119, 111, 123, 43, 61, 127, 69, 44, 88, 106, 114, 46, 125, 93, 118, 67, 49, 87, 48, 98, 92, 85, 45, 115, 26, 90, 68, 122, 104, 52, 2, 27, 54, 42, 6, 124, 7, 73, 121, 120, 60, 30, 58, 29, 1, 31, 38, 83, 57, 110, 11, 95, 24, 37, 62, 10, 25, 77, 64, 75, 32, 12, 103, 5, 14, 19, 107, 9, 36, 72, 35, 80, 102, 39, 15, 91, 16, 28, 33, 109, 40, 17, 101, 97, 23, 89, 100, 56, 78, 20, 94, 113, 108, 96, 99, 82, 21, 22], [40, 125, 62, 22, 33, 94, 48, 63, 100, 122, 90, 18, 119, 117, 92, 57, 127, 16, 114, 60, 29, 52, 49, 47, 86, 19, 54, 78, 59, 87, 45, 58, 115, 109, 110, 120, 10, 84, 56, 123, 46, 106, 113, 108, 112, 89, 70, 61, 43, 99, 24, 116, 80, 126, 53, 124, 50, 51, 96, 107, 111, 118, 28, 77, 4, 121, 39, 2, 21, 38, 79, 55, 8, 44, 105, 42, 35, 41, 0, 74, 37, 23, 101, 88, 17, 102, 32, 91, 103, 31, 98, 36, 97, 81, 82, 34, 27, 75, 93, 95, 1, 30, 7, 11, 26, 3, 15, 73, 20, 13, 83, 76, 69, 25, 12, 65, 14, 72, 71, 67, 6, 104, 85, 64, 9, 5, 68, 66], [124, 39, 55, 56, 97, 93, 113, 88, 21, 83, 57, 91, 15, 122, 81, 114, 123, 125, 78, 105, 44, 117, 6, 11, 52, 23, 18, 10, 66, 4, 45, 64, 9, 43, 109, 69, 126, 76, 106, 22, 49, 12, 110, 47, 116, 53, 112, 120, 51, 80, 107, 61, 46, 58, 118, 62, 42, 32, 54, 1, 14, 41, 82, 119, 3, 50, 121, 127, 115, 27, 60, 111, 34, 89, 108, 33, 48, 101, 100, 38, 20, 63, 72, 102, 36, 84, 99, 59, 90, 25, 98, 35, 13, 37, 26, 96, 92, 19, 104, 94, 30, 95, 86, 40, 16, 73, 28, 87, 77, 7, 29, 85, 31, 5, 8, 71, 67, 75, 24, 103, 65, 68, 79, 0, 17, 2, 70, 74], [126, 57, 103, 31, 98, 61, 87, 90, 22, 60, 16, 92, 21, 120, 19, 17, 111, 48, 77, 79, 75, 125, 51, 7, 63, 58, 113, 74, 118, 84, 122, 53, 3, 127, 73, 115, 117, 6, 45, 55, 123, 65, 64, 100, 56, 119, 82, 62, 52, 116, 66, 49, 112, 50, 114, 99, 110, 46, 69, 44, 121, 88, 54, 38, 47, 76, 105, 124, 40, 109, 107, 68, 4, 28, 59, 15, 13, 36, 32, 91, 104, 41, 97, 43, 106, 101, 35, 20, 29, 30, 1, 102, 18, 42, 86, 10, 27, 108, 33, 5, 93, 80, 14, 25, 24, 37, 96, 94, 85, 23, 70, 34, 67, 89, 12, 81, 78, 83, 8, 2, 71, 11, 72, 9, 26, 39, 95, 0], [59, 61, 53, 37, 22, 16, 27, 18, 78, 32, 20, 119, 75, 88, 29, 25, 56, 76, 71, 85, 69, 79, 60, 116, 10, 8, 34, 2, 9, 81, 126, 98, 50, 105, 19, 113, 117, 15, 1, 108, 36, 96, 123, 122, 107, 103, 124, 87, 26, 115, 74, 39, 62, 120, 3, 0, 55, 52, 43, 13, 90, 111, 63, 70, 28, 54, 35, 6, 21, 38, 125, 109, 46, 106, 112, 30, 121, 49, 4, 47, 57, 65, 41, 17, 94, 58, 23, 104, 33, 110, 127, 77, 45, 68, 118, 48, 51, 114, 83, 97, 44, 99, 42, 93, 102, 5, 86, 72, 92, 95, 40, 89, 31, 80, 100, 73, 24, 67, 12, 64, 14, 82, 84, 7, 66, 91, 11, 101], [127, 124, 38, 22, 97, 51, 62, 30, 121, 24, 15, 114, 59, 111, 110, 18, 56, 60, 116, 112, 9, 119, 63, 91, 13, 122, 125, 93, 64, 48, 113, 17, 84, 11, 53, 5, 57, 126, 47, 106, 55, 54, 58, 102, 118, 94, 49, 117, 61, 123, 50, 66, 44, 16, 100, 19, 45, 115, 43, 101, 109, 120, 76, 25, 99, 88, 104, 108, 46, 28, 29, 10, 52, 23, 42, 103, 40, 105, 98, 41, 107, 37, 68, 26, 83, 1, 35, 14, 89, 95, 6, 87, 8, 79, 36, 71, 21, 81, 7, 96, 65, 34, 78, 74, 31, 33, 12, 39, 80, 85, 90, 86, 72, 32, 20, 3, 67, 75, 77, 92, 4, 27, 82, 69, 73, 2, 70, 0]], "model.layers.9.self_attn.qk_proj": [[124, 61, 127, 53, 125, 126, 62, 41, 57, 59, 110, 105, 117, 46, 56, 55, 51, 119, 114, 86, 63, 22, 113, 48, 98, 102, 82, 111, 112, 122, 116, 60, 52, 18, 123, 49, 58, 118, 26, 84, 50, 43, 39, 121, 28, 20, 17, 88, 47, 42, 81, 24, 101, 15, 38, 23, 29, 31, 79, 76, 103, 37, 13, 87, 97, 77, 34, 93, 33, 115, 45, 120, 74, 21, 10, 40, 0, 94, 64, 12, 54, 27, 66, 108, 107, 109, 89, 106, 16, 2, 7, 71, 44, 67, 91, 30, 19, 70, 90, 83, 78, 3, 95, 80, 92, 25, 32, 104, 85, 96, 72, 35, 8, 68, 6, 14, 99, 69, 75, 11, 9, 1, 36, 4, 100, 73, 5, 65], [124, 61, 127, 53, 126, 62, 125, 41, 59, 57, 110, 105, 117, 46, 56, 55, 63, 51, 86, 119, 114, 22, 48, 113, 111, 60, 112, 122, 98, 116, 49, 43, 82, 88, 118, 28, 24, 18, 102, 84, 123, 50, 37, 103, 26, 47, 52, 58, 39, 42, 15, 101, 87, 23, 77, 17, 20, 76, 115, 34, 45, 81, 31, 38, 79, 10, 0, 107, 13, 121, 33, 40, 97, 29, 120, 27, 54, 21, 64, 74, 94, 93, 30, 44, 12, 109, 83, 70, 108, 7, 89, 16, 90, 91, 25, 68, 106, 92, 96, 104, 100, 71, 80, 78, 85, 35, 95, 36, 66, 19, 8, 32, 4, 14, 11, 65, 99, 2, 5, 9, 75, 72, 69, 1, 3, 6, 73, 67], [124, 61, 127, 53, 126, 62, 41, 125, 59, 57, 105, 117, 110, 46, 56, 55, 63, 51, 119, 114, 86, 22, 113, 111, 48, 102, 98, 116, 60, 103, 112, 37, 122, 123, 47, 88, 39, 43, 115, 82, 18, 84, 28, 121, 118, 52, 38, 26, 34, 24, 49, 50, 20, 45, 120, 101, 87, 27, 23, 40, 17, 97, 31, 76, 42, 93, 109, 15, 81, 58, 94, 108, 54, 33, 79, 29, 106, 64, 70, 91, 77, 32, 74, 12, 44, 95, 0, 89, 10, 90, 30, 66, 107, 21, 100, 13, 83, 16, 4, 96, 7, 78, 99, 71, 8, 80, 85, 25, 92, 36, 104, 2, 35, 19, 11, 3, 68, 6, 9, 72, 67, 1, 14, 5, 75, 69, 65, 73], [124, 127, 53, 61, 125, 126, 62, 41, 57, 59, 110, 105, 46, 117, 56, 55, 63, 51, 114, 48, 86, 119, 22, 122, 111, 112, 118, 98, 60, 113, 116, 50, 49, 121, 102, 45, 18, 82, 47, 52, 43, 28, 88, 123, 38, 37, 58, 84, 115, 39, 103, 26, 17, 34, 87, 101, 42, 24, 31, 20, 54, 29, 120, 81, 93, 107, 33, 97, 23, 40, 27, 15, 94, 79, 108, 76, 0, 70, 44, 77, 92, 10, 106, 89, 13, 109, 96, 64, 12, 71, 74, 25, 66, 30, 80, 91, 35, 32, 95, 85, 21, 7, 8, 90, 100, 14, 104, 2, 83, 16, 19, 67, 69, 99, 36, 3, 78, 68, 11, 65, 72, 73, 6, 1, 4, 75, 9, 5], [124, 127, 53, 61, 125, 126, 62, 41, 59, 57, 110, 105, 117, 56, 46, 55, 51, 63, 119, 114, 116, 86, 22, 48, 98, 113, 82, 60, 18, 112, 39, 45, 102, 111, 123, 50, 122, 49, 84, 42, 24, 47, 118, 26, 38, 28, 121, 101, 52, 37, 88, 120, 34, 20, 103, 17, 115, 43, 58, 33, 87, 81, 64, 31, 40, 77, 29, 76, 23, 15, 97, 54, 93, 107, 2, 79, 44, 10, 108, 0, 12, 89, 7, 66, 13, 74, 27, 92, 109, 21, 104, 91, 96, 90, 80, 70, 94, 71, 95, 30, 106, 83, 35, 8, 78, 4, 68, 16, 85, 19, 6, 25, 65, 14, 9, 5, 75, 99, 72, 32, 11, 67, 1, 69, 100, 3, 73, 36], [124, 127, 61, 53, 126, 125, 62, 41, 57, 59, 110, 105, 117, 56, 46, 55, 51, 63, 119, 86, 114, 48, 22, 111, 113, 82, 98, 116, 58, 112, 49, 18, 122, 60, 26, 52, 102, 123, 88, 118, 28, 84, 103, 50, 47, 24, 39, 77, 45, 101, 13, 42, 43, 120, 15, 17, 20, 38, 87, 81, 79, 33, 31, 121, 34, 23, 76, 37, 54, 10, 29, 40, 97, 74, 12, 44, 7, 115, 107, 0, 83, 104, 21, 27, 2, 94, 89, 80, 106, 108, 96, 8, 71, 109, 30, 92, 64, 85, 19, 14, 16, 91, 93, 25, 95, 6, 4, 11, 78, 90, 66, 3, 75, 35, 70, 67, 9, 68, 32, 72, 69, 99, 100, 5, 65, 36, 73, 1], [124, 127, 61, 53, 126, 125, 62, 41, 59, 57, 110, 105, 117, 56, 46, 55, 51, 86, 22, 63, 119, 114, 48, 112, 116, 82, 111, 18, 52, 26, 49, 122, 118, 60, 84, 98, 20, 102, 120, 123, 28, 58, 88, 50, 39, 24, 101, 79, 81, 29, 15, 76, 17, 103, 121, 43, 113, 38, 13, 47, 31, 42, 37, 12, 34, 45, 106, 77, 10, 87, 21, 33, 27, 115, 8, 107, 23, 44, 40, 54, 94, 19, 97, 80, 93, 83, 74, 89, 85, 91, 25, 14, 109, 108, 92, 96, 7, 3, 30, 90, 0, 67, 71, 95, 6, 72, 16, 99, 78, 64, 11, 66, 36, 104, 32, 35, 2, 69, 9, 100, 5, 68, 75, 70, 1, 4, 65, 73], [124, 127, 61, 53, 126, 125, 41, 62, 59, 57, 110, 105, 117, 56, 46, 55, 51, 63, 22, 86, 119, 114, 48, 98, 112, 18, 82, 88, 122, 116, 84, 28, 26, 24, 111, 20, 60, 113, 118, 79, 15, 103, 102, 58, 50, 123, 38, 52, 43, 49, 34, 101, 17, 37, 87, 31, 42, 76, 29, 45, 13, 83, 77, 81, 39, 10, 120, 12, 21, 121, 47, 23, 97, 40, 94, 27, 74, 33, 0, 104, 64, 19, 54, 107, 115, 108, 7, 30, 80, 93, 25, 106, 6, 96, 89, 85, 16, 14, 66, 90, 92, 44, 8, 91, 109, 78, 100, 11, 2, 99, 35, 95, 68, 75, 71, 65, 72, 32, 36, 70, 4, 1, 69, 9, 5, 3, 73, 67], [124, 127, 53, 61, 126, 125, 41, 62, 59, 57, 110, 117, 105, 56, 46, 51, 55, 63, 114, 119, 22, 86, 122, 48, 98, 111, 88, 18, 60, 112, 49, 26, 84, 118, 82, 113, 116, 50, 20, 24, 39, 28, 47, 37, 38, 58, 103, 43, 102, 123, 52, 29, 94, 15, 121, 17, 79, 87, 101, 34, 27, 77, 31, 33, 120, 81, 13, 107, 97, 45, 12, 76, 23, 42, 0, 115, 40, 85, 30, 21, 74, 54, 10, 6, 19, 96, 83, 93, 25, 89, 78, 64, 92, 108, 44, 90, 16, 104, 80, 91, 68, 7, 106, 67, 109, 2, 32, 66, 95, 3, 100, 71, 4, 36, 1, 72, 70, 99, 65, 14, 35, 11, 69, 75, 8, 5, 9, 73], [124, 127, 53, 61, 126, 41, 125, 59, 62, 57, 105, 110, 117, 56, 46, 55, 63, 51, 114, 22, 86, 119, 48, 112, 116, 122, 111, 18, 98, 118, 113, 88, 60, 102, 50, 58, 82, 84, 28, 52, 79, 24, 26, 49, 38, 103, 81, 47, 15, 87, 20, 39, 34, 17, 123, 43, 29, 37, 23, 120, 107, 121, 97, 74, 76, 101, 45, 42, 31, 33, 13, 77, 27, 12, 54, 115, 21, 93, 44, 94, 0, 10, 40, 78, 64, 6, 90, 92, 83, 104, 66, 96, 7, 19, 80, 85, 25, 109, 108, 30, 91, 106, 16, 68, 71, 89, 70, 2, 72, 4, 95, 67, 32, 14, 99, 100, 5, 35, 69, 8, 3, 65, 36, 11, 73, 75, 1, 9], [124, 53, 127, 61, 125, 126, 41, 57, 62, 59, 105, 110, 117, 46, 56, 51, 55, 63, 119, 114, 22, 86, 48, 113, 111, 116, 122, 112, 98, 102, 18, 60, 82, 118, 49, 52, 50, 39, 47, 58, 20, 123, 88, 42, 31, 26, 84, 87, 15, 24, 37, 28, 103, 79, 121, 64, 81, 17, 107, 43, 34, 120, 101, 54, 77, 38, 93, 29, 12, 76, 97, 13, 66, 33, 45, 23, 27, 7, 115, 10, 40, 0, 74, 71, 2, 78, 16, 44, 21, 94, 90, 25, 6, 92, 70, 30, 72, 95, 104, 109, 85, 108, 19, 106, 35, 96, 83, 5, 68, 89, 91, 32, 75, 69, 80, 65, 4, 9, 14, 11, 3, 67, 8, 100, 99, 73, 1, 36], [124, 127, 53, 61, 126, 125, 62, 41, 59, 57, 110, 105, 117, 56, 46, 51, 55, 63, 119, 114, 86, 22, 48, 122, 111, 82, 60, 18, 116, 52, 113, 98, 39, 112, 102, 49, 118, 26, 58, 50, 88, 47, 123, 20, 15, 84, 28, 24, 120, 101, 17, 13, 77, 81, 42, 121, 29, 37, 33, 12, 43, 79, 40, 87, 31, 103, 107, 54, 38, 16, 27, 76, 0, 64, 21, 44, 34, 74, 23, 19, 10, 45, 97, 78, 93, 115, 71, 94, 66, 30, 72, 89, 108, 7, 104, 92, 2, 5, 70, 96, 90, 3, 83, 25, 106, 91, 67, 85, 109, 68, 1, 95, 35, 32, 80, 69, 14, 8, 11, 4, 100, 6, 75, 9, 73, 99, 65, 36], [124, 127, 61, 53, 126, 125, 62, 41, 57, 59, 110, 105, 117, 56, 46, 55, 51, 63, 119, 22, 86, 114, 60, 112, 116, 18, 48, 113, 122, 88, 111, 49, 118, 98, 82, 24, 102, 50, 28, 52, 123, 20, 43, 39, 26, 84, 101, 58, 103, 15, 47, 38, 37, 29, 81, 13, 87, 120, 17, 33, 121, 97, 45, 79, 34, 107, 27, 31, 12, 42, 115, 21, 77, 74, 104, 93, 23, 40, 16, 76, 94, 10, 54, 91, 19, 44, 83, 89, 30, 25, 96, 90, 85, 108, 64, 70, 78, 72, 92, 100, 106, 32, 95, 71, 80, 109, 0, 75, 14, 36, 35, 7, 68, 66, 2, 99, 69, 4, 3, 67, 11, 5, 8, 6, 65, 9, 1, 73], [124, 127, 53, 61, 126, 125, 62, 41, 57, 59, 110, 117, 105, 56, 46, 55, 63, 51, 86, 119, 22, 114, 48, 112, 111, 98, 113, 60, 18, 58, 116, 122, 82, 88, 49, 28, 15, 118, 26, 47, 123, 13, 120, 24, 43, 102, 50, 84, 52, 103, 79, 20, 34, 64, 101, 87, 81, 39, 21, 42, 45, 29, 23, 94, 17, 38, 40, 121, 74, 77, 10, 83, 104, 70, 12, 0, 33, 31, 37, 93, 107, 54, 27, 97, 89, 19, 7, 71, 76, 16, 25, 14, 90, 108, 44, 2, 92, 80, 91, 72, 115, 78, 85, 95, 30, 106, 66, 75, 4, 73, 11, 32, 96, 5, 68, 35, 6, 69, 9, 8, 65, 36, 109, 1, 100, 99, 3, 67], [124, 127, 53, 61, 126, 125, 62, 41, 59, 57, 110, 105, 117, 56, 46, 55, 63, 51, 119, 86, 114, 48, 22, 60, 111, 112, 118, 113, 52, 123, 49, 82, 98, 47, 18, 101, 116, 102, 120, 122, 88, 39, 58, 26, 28, 24, 50, 103, 13, 20, 38, 84, 37, 34, 45, 43, 33, 79, 15, 87, 23, 42, 17, 121, 97, 31, 81, 40, 107, 27, 77, 29, 108, 7, 12, 21, 0, 54, 16, 19, 44, 74, 10, 64, 94, 70, 93, 104, 92, 76, 89, 106, 25, 71, 91, 72, 83, 90, 66, 115, 2, 80, 95, 14, 85, 78, 96, 32, 30, 3, 67, 5, 4, 9, 100, 99, 35, 109, 1, 68, 6, 11, 36, 75, 69, 73, 65, 8], [124, 127, 53, 61, 126, 125, 41, 57, 62, 59, 110, 105, 117, 56, 46, 55, 63, 51, 86, 119, 22, 114, 113, 48, 112, 60, 111, 18, 102, 98, 52, 122, 118, 49, 39, 116, 43, 82, 123, 84, 101, 28, 50, 47, 58, 120, 88, 103, 26, 20, 42, 24, 29, 33, 13, 17, 15, 81, 97, 38, 45, 34, 27, 37, 79, 12, 121, 31, 40, 23, 107, 87, 77, 94, 16, 108, 104, 10, 54, 64, 30, 21, 0, 91, 25, 7, 76, 115, 74, 90, 83, 89, 44, 35, 93, 92, 71, 70, 69, 96, 85, 19, 32, 66, 106, 67, 2, 78, 72, 95, 80, 100, 109, 99, 6, 68, 3, 75, 14, 4, 8, 5, 1, 36, 9, 11, 65, 73], [124, 127, 126, 53, 61, 125, 41, 57, 62, 59, 110, 105, 56, 46, 117, 55, 51, 63, 22, 86, 114, 48, 119, 112, 52, 18, 111, 116, 102, 123, 60, 113, 98, 28, 84, 49, 122, 82, 88, 118, 50, 101, 103, 20, 26, 39, 79, 24, 15, 47, 58, 17, 120, 37, 13, 38, 81, 107, 104, 31, 45, 33, 27, 121, 29, 43, 34, 10, 40, 21, 12, 74, 23, 94, 42, 19, 91, 87, 77, 16, 97, 32, 90, 115, 54, 25, 108, 76, 106, 93, 78, 35, 71, 83, 44, 64, 89, 85, 80, 96, 99, 0, 30, 92, 14, 6, 95, 8, 7, 68, 72, 109, 69, 4, 66, 70, 100, 75, 9, 5, 2, 65, 36, 1, 73, 67, 11, 3], [124, 127, 53, 126, 61, 125, 41, 59, 62, 57, 110, 105, 117, 56, 46, 55, 51, 22, 86, 63, 119, 114, 111, 48, 112, 122, 123, 82, 98, 52, 28, 18, 116, 49, 101, 102, 24, 84, 88, 50, 47, 26, 58, 38, 60, 20, 113, 118, 17, 103, 37, 81, 42, 15, 39, 79, 34, 31, 29, 43, 33, 21, 23, 13, 87, 97, 120, 40, 107, 45, 12, 121, 27, 74, 10, 89, 93, 77, 19, 90, 94, 83, 16, 104, 0, 25, 7, 91, 115, 54, 6, 78, 80, 92, 108, 76, 30, 44, 96, 106, 64, 85, 99, 2, 95, 71, 66, 14, 35, 75, 32, 8, 100, 67, 9, 3, 109, 36, 68, 70, 4, 73, 1, 72, 11, 69, 5, 65], [124, 127, 53, 61, 126, 41, 125, 59, 62, 57, 105, 110, 117, 46, 56, 55, 51, 86, 114, 63, 119, 22, 48, 118, 52, 111, 98, 122, 116, 60, 39, 112, 37, 123, 18, 47, 102, 49, 113, 103, 101, 82, 43, 58, 88, 84, 38, 115, 24, 34, 26, 81, 28, 31, 42, 50, 120, 107, 17, 121, 33, 20, 79, 13, 87, 40, 91, 15, 45, 54, 23, 106, 29, 108, 27, 97, 94, 0, 99, 64, 12, 93, 32, 89, 44, 10, 6, 21, 76, 19, 83, 95, 92, 85, 90, 74, 16, 100, 2, 35, 109, 96, 77, 30, 80, 25, 14, 66, 7, 104, 68, 36, 8, 4, 71, 67, 70, 1, 3, 75, 78, 9, 69, 5, 65, 73, 72, 11], [124, 127, 53, 61, 126, 125, 57, 41, 62, 59, 110, 105, 117, 46, 56, 55, 51, 86, 63, 119, 22, 114, 48, 112, 111, 113, 116, 60, 123, 49, 118, 98, 102, 103, 18, 39, 122, 88, 52, 47, 101, 58, 28, 26, 82, 50, 33, 84, 43, 24, 37, 121, 31, 38, 79, 15, 34, 87, 20, 97, 13, 42, 29, 17, 81, 21, 120, 107, 27, 45, 23, 115, 54, 74, 12, 94, 25, 10, 6, 93, 104, 40, 89, 19, 90, 92, 30, 77, 32, 91, 64, 76, 0, 108, 35, 83, 80, 7, 44, 95, 85, 96, 78, 106, 16, 8, 4, 109, 71, 68, 14, 2, 75, 66, 9, 70, 99, 100, 36, 5, 11, 72, 69, 1, 3, 65, 67, 73], [124, 61, 53, 127, 126, 125, 41, 57, 62, 59, 110, 105, 117, 56, 46, 55, 51, 63, 86, 119, 22, 114, 48, 112, 111, 116, 98, 18, 60, 102, 84, 52, 123, 122, 58, 118, 50, 113, 88, 82, 49, 101, 17, 42, 28, 26, 43, 13, 47, 121, 39, 20, 15, 33, 27, 81, 38, 79, 24, 29, 103, 120, 10, 87, 107, 31, 45, 77, 40, 23, 37, 115, 12, 54, 94, 34, 74, 95, 6, 30, 90, 25, 7, 93, 16, 64, 21, 104, 71, 66, 76, 78, 0, 2, 83, 89, 97, 91, 44, 5, 80, 85, 19, 92, 108, 106, 8, 3, 96, 75, 4, 9, 14, 69, 67, 70, 100, 109, 35, 32, 72, 36, 99, 68, 11, 73, 65, 1], [124, 127, 53, 61, 125, 126, 59, 41, 62, 57, 110, 117, 105, 56, 46, 55, 51, 63, 119, 86, 22, 114, 48, 112, 98, 111, 118, 58, 52, 18, 116, 102, 49, 50, 37, 82, 88, 101, 42, 122, 20, 60, 113, 115, 123, 38, 39, 84, 24, 28, 81, 47, 103, 29, 26, 43, 15, 17, 34, 54, 33, 120, 45, 87, 12, 121, 13, 31, 79, 64, 23, 40, 27, 93, 107, 106, 94, 97, 21, 44, 92, 10, 108, 77, 89, 16, 74, 30, 90, 76, 91, 83, 95, 25, 71, 109, 7, 19, 0, 96, 35, 78, 85, 8, 100, 66, 104, 80, 99, 70, 4, 2, 32, 14, 75, 6, 5, 36, 3, 11, 67, 69, 1, 72, 68, 9, 73, 65], [124, 127, 61, 53, 126, 125, 41, 62, 57, 59, 110, 105, 117, 56, 46, 63, 55, 51, 119, 86, 22, 114, 48, 98, 112, 111, 18, 37, 118, 38, 60, 24, 116, 84, 102, 82, 52, 58, 113, 20, 39, 123, 101, 88, 50, 115, 120, 28, 103, 45, 15, 81, 49, 26, 43, 42, 23, 31, 47, 122, 87, 29, 27, 17, 34, 33, 94, 40, 97, 12, 13, 10, 0, 54, 91, 107, 79, 77, 121, 44, 109, 93, 21, 104, 108, 74, 64, 83, 90, 76, 106, 95, 70, 19, 80, 71, 25, 96, 66, 30, 7, 92, 16, 78, 2, 85, 89, 32, 14, 8, 100, 68, 35, 99, 4, 75, 72, 73, 5, 69, 36, 3, 1, 11, 65, 9, 6, 67], [124, 127, 53, 61, 126, 125, 62, 41, 57, 59, 110, 105, 56, 117, 46, 63, 55, 51, 86, 119, 114, 22, 48, 58, 111, 98, 18, 116, 112, 60, 113, 118, 52, 82, 49, 88, 45, 50, 103, 123, 39, 84, 38, 102, 122, 37, 43, 42, 47, 28, 26, 20, 101, 24, 81, 27, 54, 79, 15, 87, 13, 120, 23, 31, 10, 40, 77, 33, 17, 107, 94, 29, 121, 34, 115, 97, 83, 85, 30, 76, 64, 12, 70, 71, 104, 91, 21, 44, 93, 108, 95, 90, 25, 74, 89, 7, 19, 92, 16, 80, 3, 66, 0, 106, 14, 35, 78, 2, 96, 68, 69, 8, 4, 32, 109, 9, 75, 100, 11, 67, 5, 99, 73, 36, 72, 1, 6, 65], [124, 127, 61, 53, 126, 125, 62, 57, 41, 59, 110, 117, 105, 46, 56, 55, 51, 63, 86, 119, 22, 114, 48, 60, 111, 113, 118, 98, 52, 82, 122, 47, 123, 49, 116, 18, 102, 112, 39, 58, 88, 101, 84, 26, 37, 50, 20, 28, 43, 17, 24, 45, 38, 121, 81, 31, 103, 27, 42, 23, 34, 79, 29, 97, 15, 33, 54, 87, 76, 115, 120, 12, 108, 13, 21, 40, 94, 77, 70, 91, 10, 107, 106, 74, 93, 0, 44, 30, 83, 25, 89, 2, 7, 96, 16, 19, 32, 85, 71, 92, 64, 90, 95, 80, 14, 109, 100, 73, 78, 8, 35, 99, 66, 72, 104, 4, 3, 75, 69, 36, 5, 67, 6, 11, 65, 1, 68, 9], [124, 61, 127, 53, 126, 125, 59, 57, 62, 41, 110, 105, 117, 56, 46, 55, 63, 86, 51, 119, 22, 114, 48, 60, 113, 112, 58, 49, 18, 111, 98, 122, 88, 116, 118, 52, 82, 50, 102, 101, 84, 26, 123, 39, 24, 121, 81, 38, 15, 37, 20, 34, 45, 103, 28, 17, 79, 115, 76, 42, 120, 29, 33, 47, 13, 31, 43, 107, 27, 87, 40, 77, 74, 21, 23, 10, 25, 12, 97, 93, 94, 54, 90, 30, 83, 89, 64, 91, 80, 44, 7, 70, 85, 96, 92, 19, 78, 104, 108, 100, 0, 99, 71, 106, 35, 109, 16, 95, 36, 32, 72, 8, 73, 14, 75, 6, 68, 66, 69, 11, 2, 4, 5, 9, 1, 67, 3, 65], [124, 127, 61, 53, 126, 125, 41, 59, 62, 57, 110, 105, 117, 56, 46, 55, 63, 51, 119, 86, 22, 114, 122, 48, 58, 60, 112, 98, 88, 82, 116, 52, 111, 49, 120, 26, 43, 34, 18, 50, 102, 113, 121, 24, 101, 45, 28, 27, 123, 37, 40, 84, 42, 118, 38, 103, 31, 20, 29, 15, 47, 115, 39, 17, 94, 81, 23, 87, 21, 33, 79, 97, 91, 77, 107, 12, 109, 13, 93, 96, 76, 74, 0, 54, 104, 95, 108, 44, 80, 89, 25, 10, 90, 106, 64, 85, 30, 19, 3, 99, 78, 100, 70, 83, 7, 92, 68, 71, 2, 36, 72, 66, 67, 16, 35, 5, 6, 32, 4, 8, 73, 14, 65, 69, 1, 11, 9, 75], [124, 127, 53, 61, 126, 125, 59, 57, 41, 62, 110, 117, 105, 56, 46, 55, 63, 51, 119, 86, 22, 114, 111, 48, 98, 58, 82, 112, 113, 116, 49, 60, 18, 122, 102, 26, 84, 52, 118, 28, 79, 88, 120, 50, 121, 103, 123, 47, 20, 43, 101, 34, 45, 33, 29, 24, 39, 76, 81, 0, 42, 15, 17, 38, 37, 77, 87, 31, 94, 64, 54, 27, 10, 40, 23, 74, 12, 2, 97, 93, 7, 13, 107, 83, 21, 71, 115, 66, 6, 19, 80, 85, 44, 4, 109, 72, 106, 95, 89, 104, 96, 91, 30, 1, 92, 78, 16, 99, 25, 5, 108, 90, 14, 35, 32, 67, 9, 75, 68, 11, 100, 36, 73, 65, 70, 69, 3, 8], [124, 127, 61, 53, 125, 126, 41, 57, 62, 59, 105, 110, 117, 46, 56, 63, 55, 51, 86, 119, 22, 114, 113, 48, 111, 98, 112, 52, 122, 102, 116, 18, 47, 37, 49, 82, 123, 60, 118, 58, 50, 20, 26, 39, 101, 103, 24, 88, 34, 81, 42, 87, 38, 45, 33, 31, 120, 29, 28, 43, 15, 0, 76, 40, 121, 84, 97, 23, 17, 27, 21, 77, 79, 93, 107, 94, 7, 44, 74, 64, 2, 10, 92, 13, 106, 6, 115, 19, 89, 91, 12, 96, 90, 54, 95, 109, 80, 30, 108, 100, 16, 71, 104, 72, 66, 25, 32, 5, 78, 14, 69, 35, 85, 36, 83, 11, 9, 99, 4, 67, 75, 1, 3, 65, 68, 8, 73, 70], [124, 127, 61, 53, 126, 125, 41, 62, 57, 59, 110, 105, 56, 117, 46, 55, 51, 63, 86, 119, 114, 113, 22, 111, 48, 52, 58, 112, 60, 116, 98, 18, 102, 47, 49, 82, 50, 103, 26, 84, 88, 123, 37, 24, 28, 15, 122, 118, 38, 20, 39, 87, 101, 31, 81, 43, 42, 34, 121, 77, 97, 33, 17, 45, 40, 13, 29, 76, 10, 27, 79, 0, 23, 54, 74, 115, 108, 94, 12, 107, 6, 21, 89, 64, 99, 83, 2, 104, 72, 80, 90, 30, 120, 93, 44, 25, 71, 16, 109, 67, 96, 7, 91, 66, 19, 3, 106, 14, 92, 85, 95, 4, 78, 35, 75, 32, 36, 11, 5, 65, 100, 1, 73, 69, 8, 68, 70, 9], [124, 127, 61, 53, 126, 125, 41, 59, 62, 57, 110, 105, 117, 46, 56, 55, 63, 51, 119, 114, 22, 86, 60, 113, 48, 98, 111, 112, 52, 116, 18, 58, 122, 102, 82, 84, 118, 24, 47, 50, 88, 28, 103, 123, 43, 15, 26, 121, 49, 42, 39, 101, 20, 38, 45, 31, 17, 87, 97, 120, 81, 76, 23, 34, 29, 77, 37, 13, 40, 54, 33, 0, 27, 21, 79, 7, 44, 115, 12, 94, 74, 10, 89, 107, 106, 96, 93, 6, 83, 108, 72, 25, 2, 30, 64, 95, 71, 80, 68, 92, 104, 16, 35, 91, 4, 14, 85, 90, 19, 66, 75, 36, 100, 11, 78, 99, 32, 109, 9, 73, 70, 5, 69, 65, 1, 8, 67, 3], [124, 127, 61, 126, 53, 125, 62, 57, 59, 41, 110, 105, 56, 117, 46, 55, 51, 63, 119, 114, 86, 22, 48, 113, 60, 98, 18, 111, 116, 112, 24, 49, 82, 103, 123, 84, 88, 118, 122, 52, 58, 28, 26, 102, 50, 15, 20, 17, 76, 81, 121, 79, 13, 45, 39, 38, 43, 34, 87, 101, 23, 77, 31, 94, 21, 115, 47, 97, 29, 74, 37, 12, 10, 33, 42, 120, 40, 27, 54, 107, 44, 7, 83, 30, 0, 93, 89, 64, 71, 25, 91, 78, 11, 92, 6, 16, 90, 104, 96, 108, 72, 80, 66, 19, 85, 14, 5, 95, 70, 2, 75, 67, 4, 32, 3, 69, 36, 8, 106, 109, 99, 35, 100, 65, 68, 73, 9, 1]], "model.layers.10.self_attn.q_proj": [[121, 122, 26, 65, 36, 83, 50, 79, 21, 24, 81, 63, 89, 29, 94, 9, 0, 64, 68, 16, 6, 19, 99, 1, 91, 78, 8, 17, 23, 2, 58, 77, 75, 111, 69, 124, 86, 12, 114, 71, 95, 82, 85, 44, 52, 14, 120, 76, 30, 37, 67, 51, 96, 5, 118, 57, 31, 54, 70, 11, 123, 74, 66, 93, 20, 84, 47, 27, 60, 127, 61, 40, 4, 13, 10, 3, 18, 125, 73, 80, 102, 87, 53, 49, 117, 115, 119, 62, 113, 116, 100, 72, 112, 55, 45, 7, 15, 126, 59, 90, 48, 105, 104, 110, 92, 41, 88, 33, 56, 46, 109, 35, 42, 25, 103, 22, 39, 107, 97, 38, 106, 108, 43, 34, 32, 28, 101, 98], [121, 122, 63, 87, 50, 36, 75, 52, 118, 28, 12, 124, 17, 120, 94, 123, 60, 61, 54, 125, 51, 33, 73, 29, 57, 62, 127, 119, 40, 117, 34, 44, 115, 53, 113, 102, 15, 49, 47, 58, 111, 19, 112, 55, 23, 70, 126, 110, 114, 59, 14, 109, 46, 105, 24, 56, 97, 116, 45, 104, 99, 38, 31, 32, 21, 71, 16, 86, 13, 39, 107, 48, 76, 106, 80, 37, 35, 85, 103, 25, 101, 20, 88, 91, 100, 92, 96, 98, 108, 93, 41, 5, 18, 84, 66, 95, 67, 26, 22, 89, 90, 42, 64, 27, 83, 30, 43, 79, 11, 3, 74, 82, 4, 1, 8, 77, 72, 81, 7, 65, 10, 9, 0, 6, 2, 69, 78, 68], [50, 121, 122, 114, 63, 56, 62, 115, 120, 60, 58, 51, 49, 116, 84, 87, 59, 119, 53, 11, 113, 52, 15, 127, 44, 17, 117, 109, 45, 55, 124, 61, 126, 54, 47, 57, 123, 48, 107, 14, 111, 46, 72, 118, 108, 125, 110, 43, 112, 7, 24, 2, 27, 78, 106, 25, 99, 101, 75, 41, 4, 85, 42, 16, 77, 80, 19, 82, 10, 105, 1, 81, 3, 74, 104, 18, 13, 12, 9, 28, 83, 94, 40, 37, 97, 6, 71, 90, 73, 102, 103, 21, 39, 22, 29, 38, 23, 76, 30, 89, 93, 33, 5, 36, 0, 35, 32, 96, 34, 98, 70, 91, 95, 69, 31, 26, 92, 100, 86, 8, 65, 88, 79, 20, 66, 68, 67, 64], [121, 122, 50, 63, 14, 118, 52, 84, 124, 11, 123, 87, 120, 61, 97, 125, 15, 114, 54, 57, 62, 101, 60, 82, 47, 16, 17, 53, 119, 85, 28, 111, 115, 51, 112, 127, 117, 102, 58, 110, 40, 104, 55, 44, 109, 45, 75, 32, 59, 105, 46, 77, 113, 107, 49, 27, 106, 39, 37, 103, 126, 25, 33, 38, 81, 48, 116, 12, 34, 19, 41, 74, 80, 90, 108, 71, 72, 83, 24, 9, 13, 99, 91, 4, 2, 7, 56, 89, 21, 42, 78, 35, 6, 94, 36, 10, 96, 3, 76, 0, 73, 43, 92, 79, 93, 5, 30, 100, 23, 31, 98, 1, 22, 95, 29, 86, 26, 70, 88, 20, 66, 69, 18, 8, 67, 68, 65, 64], [103, 124, 113, 24, 82, 49, 21, 91, 31, 15, 32, 11, 62, 13, 83, 19, 16, 28, 50, 80, 72, 99, 27, 81, 63, 73, 93, 4, 76, 17, 88, 7, 75, 9, 14, 18, 74, 8, 85, 79, 12, 78, 26, 98, 77, 87, 86, 84, 64, 89, 100, 68, 1, 20, 25, 29, 45, 23, 35, 53, 92, 46, 70, 66, 90, 22, 71, 94, 10, 69, 33, 30, 34, 104, 95, 40, 6, 123, 101, 5, 67, 0, 60, 61, 122, 3, 110, 96, 59, 39, 109, 51, 111, 126, 2, 65, 117, 97, 112, 127, 116, 43, 55, 58, 54, 120, 125, 115, 105, 102, 44, 36, 37, 57, 48, 38, 56, 108, 41, 114, 121, 106, 107, 118, 52, 42, 47, 119], [62, 103, 124, 113, 49, 122, 61, 115, 57, 45, 40, 87, 26, 50, 119, 112, 53, 32, 48, 58, 104, 109, 29, 54, 121, 120, 126, 63, 59, 117, 118, 60, 90, 95, 108, 110, 93, 55, 127, 46, 116, 52, 125, 51, 94, 123, 111, 56, 114, 99, 47, 96, 44, 105, 42, 107, 31, 106, 41, 27, 43, 14, 33, 19, 17, 91, 38, 97, 28, 23, 102, 98, 21, 30, 20, 101, 74, 100, 81, 35, 36, 37, 84, 34, 24, 39, 92, 86, 78, 25, 22, 89, 70, 6, 13, 66, 88, 68, 15, 69, 83, 7, 18, 75, 85, 67, 9, 16, 79, 64, 77, 10, 5, 8, 72, 3, 12, 4, 82, 11, 71, 80, 65, 2, 0, 1, 73, 76], [113, 124, 103, 49, 62, 50, 53, 46, 26, 122, 45, 61, 32, 40, 120, 51, 59, 123, 63, 87, 126, 112, 109, 5, 58, 117, 57, 30, 47, 100, 35, 127, 115, 111, 90, 8, 108, 17, 54, 104, 121, 105, 56, 114, 68, 125, 33, 95, 48, 60, 1, 99, 52, 116, 83, 118, 97, 106, 7, 94, 31, 107, 101, 41, 25, 64, 110, 119, 43, 34, 80, 91, 75, 79, 6, 29, 93, 38, 44, 3, 36, 9, 42, 77, 92, 55, 102, 12, 98, 28, 37, 27, 0, 66, 23, 14, 96, 76, 78, 84, 20, 21, 2, 82, 86, 88, 24, 15, 19, 39, 74, 73, 89, 10, 85, 81, 67, 11, 22, 70, 71, 16, 65, 69, 4, 72, 13, 18], [113, 124, 62, 40, 103, 50, 49, 46, 122, 83, 59, 112, 126, 87, 5, 53, 8, 127, 109, 45, 77, 120, 61, 117, 1, 78, 26, 123, 6, 68, 80, 55, 63, 54, 75, 35, 58, 125, 42, 51, 57, 7, 114, 9, 115, 3, 60, 17, 111, 118, 76, 52, 56, 47, 15, 116, 119, 108, 48, 110, 121, 44, 41, 82, 30, 43, 107, 104, 32, 106, 0, 73, 105, 12, 101, 64, 33, 100, 38, 66, 102, 36, 37, 79, 11, 89, 85, 34, 2, 23, 71, 90, 84, 31, 99, 94, 24, 92, 95, 67, 29, 81, 98, 96, 97, 20, 21, 28, 25, 93, 14, 10, 19, 39, 74, 86, 65, 88, 91, 27, 22, 70, 69, 13, 4, 16, 18, 72], [102, 56, 49, 113, 23, 89, 55, 91, 19, 58, 85, 47, 37, 31, 20, 52, 18, 33, 14, 127, 95, 92, 122, 80, 124, 115, 117, 29, 125, 59, 120, 42, 112, 107, 123, 75, 61, 57, 46, 108, 28, 116, 114, 82, 50, 45, 121, 73, 48, 84, 109, 44, 77, 88, 41, 63, 51, 99, 39, 40, 53, 38, 103, 126, 34, 43, 97, 118, 32, 16, 13, 106, 11, 87, 100, 54, 104, 98, 94, 15, 86, 27, 105, 83, 60, 119, 111, 26, 35, 110, 78, 79, 62, 8, 93, 21, 30, 101, 5, 22, 25, 36, 90, 96, 74, 72, 6, 81, 10, 24, 71, 76, 17, 4, 7, 9, 12, 67, 66, 69, 65, 70, 0, 3, 1, 68, 2, 64], [102, 56, 113, 49, 85, 23, 91, 19, 80, 77, 9, 31, 11, 89, 58, 4, 13, 55, 6, 70, 68, 72, 47, 2, 73, 95, 75, 16, 21, 83, 52, 65, 8, 82, 64, 87, 39, 48, 27, 42, 7, 53, 15, 78, 37, 25, 109, 110, 124, 22, 26, 69, 125, 90, 112, 46, 5, 33, 81, 84, 79, 71, 50, 99, 88, 32, 120, 63, 12, 106, 17, 119, 35, 44, 127, 10, 61, 92, 30, 96, 107, 123, 29, 14, 0, 101, 76, 24, 28, 66, 118, 54, 40, 116, 57, 94, 115, 59, 114, 98, 45, 126, 60, 122, 41, 86, 67, 121, 18, 100, 20, 105, 36, 104, 43, 117, 111, 34, 93, 74, 97, 51, 62, 103, 108, 1, 3, 38], [102, 47, 113, 56, 49, 89, 91, 23, 85, 19, 31, 55, 58, 9, 80, 82, 77, 11, 14, 0, 120, 46, 68, 121, 72, 59, 51, 33, 66, 95, 83, 127, 37, 124, 114, 70, 52, 53, 88, 110, 78, 87, 84, 3, 48, 99, 10, 15, 112, 81, 57, 126, 60, 73, 62, 16, 35, 42, 1, 21, 109, 34, 20, 125, 32, 116, 108, 27, 79, 118, 5, 61, 12, 100, 96, 29, 7, 97, 30, 2, 93, 25, 44, 106, 40, 92, 71, 36, 104, 17, 43, 24, 39, 122, 123, 63, 22, 119, 67, 103, 69, 18, 26, 38, 13, 115, 4, 65, 117, 90, 50, 8, 54, 111, 98, 45, 105, 94, 64, 86, 101, 41, 74, 76, 28, 107, 75, 6], [102, 113, 56, 49, 23, 80, 19, 89, 9, 85, 77, 11, 70, 68, 31, 2, 55, 64, 83, 14, 1, 16, 72, 21, 91, 37, 87, 6, 58, 75, 25, 67, 82, 3, 66, 125, 13, 120, 47, 33, 27, 48, 52, 127, 7, 73, 42, 112, 78, 12, 110, 65, 8, 29, 53, 59, 5, 17, 57, 46, 24, 32, 44, 88, 22, 10, 81, 76, 116, 51, 50, 92, 124, 30, 114, 121, 40, 122, 61, 95, 109, 28, 39, 4, 84, 74, 93, 123, 90, 119, 99, 26, 43, 35, 86, 115, 18, 63, 79, 15, 62, 45, 106, 100, 118, 69, 126, 105, 103, 20, 107, 41, 104, 60, 71, 117, 94, 98, 101, 97, 54, 36, 108, 34, 96, 111, 0, 38], [52, 62, 122, 58, 102, 125, 61, 53, 60, 127, 123, 112, 121, 119, 63, 115, 114, 50, 57, 48, 117, 126, 124, 118, 56, 49, 46, 54, 55, 113, 120, 47, 116, 44, 111, 36, 110, 51, 107, 45, 59, 109, 108, 35, 42, 30, 105, 43, 106, 104, 89, 25, 40, 41, 101, 103, 100, 23, 39, 33, 38, 37, 96, 32, 92, 99, 98, 34, 95, 97, 94, 31, 87, 28, 27, 91, 75, 29, 26, 15, 93, 24, 90, 17, 84, 21, 9, 20, 82, 85, 88, 70, 2, 6, 11, 19, 81, 80, 77, 12, 0, 79, 16, 14, 74, 86, 7, 65, 10, 67, 73, 69, 3, 66, 13, 5, 4, 64, 83, 8, 18, 76, 71, 1, 72, 68, 22, 78], [58, 62, 52, 122, 60, 121, 63, 118, 125, 55, 112, 127, 115, 51, 114, 117, 126, 123, 50, 110, 46, 59, 61, 113, 119, 111, 43, 120, 56, 42, 48, 124, 47, 116, 49, 57, 45, 109, 104, 96, 53, 40, 41, 105, 54, 108, 106, 107, 37, 101, 39, 102, 32, 103, 44, 38, 100, 29, 25, 23, 35, 98, 36, 34, 99, 84, 33, 27, 95, 30, 19, 97, 86, 24, 94, 20, 90, 22, 31, 92, 28, 89, 93, 81, 91, 80, 17, 16, 88, 83, 13, 79, 77, 26, 87, 85, 15, 21, 73, 78, 76, 11, 18, 82, 14, 70, 75, 9, 72, 12, 6, 10, 0, 74, 66, 68, 7, 2, 71, 8, 3, 67, 69, 1, 65, 5, 4, 64], [62, 58, 52, 101, 121, 122, 60, 127, 125, 112, 55, 115, 118, 117, 123, 63, 51, 53, 57, 119, 114, 50, 113, 61, 126, 102, 111, 120, 124, 48, 46, 59, 110, 36, 116, 56, 42, 54, 109, 47, 45, 44, 106, 49, 108, 93, 105, 43, 107, 40, 41, 100, 96, 104, 90, 30, 39, 103, 38, 99, 94, 35, 34, 23, 29, 25, 28, 27, 97, 80, 95, 86, 98, 33, 19, 16, 85, 92, 89, 31, 91, 21, 32, 73, 75, 24, 87, 37, 83, 26, 9, 88, 81, 11, 3, 77, 13, 22, 20, 82, 68, 70, 84, 15, 71, 5, 69, 10, 74, 1, 78, 14, 17, 7, 18, 76, 8, 79, 6, 67, 64, 66, 4, 12, 2, 65, 72, 0], [58, 52, 62, 101, 24, 90, 34, 82, 93, 14, 12, 85, 19, 80, 121, 32, 60, 21, 86, 30, 37, 15, 20, 76, 78, 96, 9, 122, 18, 79, 69, 83, 16, 125, 67, 74, 10, 88, 94, 72, 55, 5, 8, 127, 100, 84, 29, 77, 87, 7, 17, 71, 1, 81, 112, 102, 26, 13, 25, 23, 118, 75, 89, 91, 113, 27, 68, 117, 0, 2, 114, 64, 98, 70, 120, 124, 11, 4, 61, 66, 115, 126, 73, 104, 119, 110, 92, 51, 63, 42, 28, 107, 123, 111, 31, 3, 116, 46, 50, 59, 45, 65, 57, 56, 53, 48, 95, 6, 47, 106, 97, 41, 33, 43, 99, 103, 109, 38, 40, 35, 105, 22, 108, 39, 36, 54, 49, 44], [118, 127, 26, 101, 17, 10, 86, 15, 77, 70, 98, 88, 82, 20, 74, 93, 6, 29, 68, 89, 13, 104, 30, 22, 72, 87, 2, 76, 85, 79, 81, 27, 92, 84, 90, 54, 24, 67, 37, 28, 34, 71, 18, 11, 7, 52, 19, 78, 3, 9, 80, 1, 12, 40, 91, 109, 59, 65, 126, 75, 95, 73, 97, 0, 66, 16, 42, 63, 64, 23, 5, 21, 83, 4, 94, 105, 62, 31, 14, 47, 113, 69, 121, 32, 33, 96, 25, 39, 43, 35, 36, 8, 38, 107, 50, 110, 99, 44, 41, 53, 103, 108, 100, 48, 56, 115, 122, 112, 46, 116, 124, 58, 102, 117, 45, 111, 55, 114, 49, 120, 61, 119, 106, 123, 125, 60, 51, 57], [118, 127, 101, 98, 37, 126, 88, 86, 62, 39, 30, 29, 50, 20, 63, 91, 104, 52, 26, 113, 72, 48, 121, 54, 112, 56, 119, 93, 82, 38, 97, 116, 43, 125, 59, 122, 44, 110, 60, 92, 66, 83, 117, 0, 35, 51, 25, 40, 109, 58, 77, 123, 53, 46, 102, 45, 55, 15, 69, 115, 41, 120, 108, 28, 31, 57, 61, 49, 1, 47, 90, 114, 99, 24, 11, 84, 111, 89, 106, 75, 36, 124, 94, 33, 103, 8, 6, 4, 79, 23, 68, 76, 80, 96, 18, 107, 73, 105, 42, 74, 3, 64, 13, 17, 100, 22, 32, 81, 78, 2, 95, 85, 16, 87, 19, 9, 27, 12, 14, 34, 21, 5, 71, 7, 65, 70, 10, 67], [127, 118, 62, 101, 126, 56, 104, 113, 63, 54, 119, 48, 112, 52, 82, 40, 60, 121, 44, 50, 122, 116, 98, 43, 51, 123, 59, 117, 88, 58, 97, 125, 120, 91, 37, 106, 110, 55, 57, 47, 109, 46, 53, 45, 114, 115, 61, 39, 86, 42, 111, 29, 49, 107, 23, 41, 8, 124, 108, 105, 100, 35, 102, 93, 36, 38, 103, 83, 96, 72, 31, 92, 26, 99, 33, 66, 30, 87, 28, 4, 95, 89, 32, 34, 80, 90, 25, 94, 22, 27, 16, 18, 68, 24, 20, 15, 19, 0, 77, 85, 12, 84, 2, 69, 78, 13, 14, 73, 21, 75, 11, 74, 71, 5, 76, 79, 81, 7, 17, 1, 64, 6, 9, 65, 3, 70, 10, 67], [127, 118, 101, 126, 56, 97, 88, 82, 62, 29, 63, 54, 52, 40, 113, 48, 44, 28, 23, 119, 109, 60, 43, 98, 112, 36, 122, 117, 50, 41, 121, 106, 45, 116, 59, 8, 86, 51, 37, 123, 115, 104, 47, 125, 91, 32, 26, 114, 58, 110, 120, 105, 89, 46, 39, 61, 102, 111, 55, 107, 93, 30, 57, 53, 108, 72, 42, 100, 38, 16, 20, 4, 92, 49, 12, 99, 124, 103, 35, 18, 34, 31, 33, 94, 77, 83, 13, 27, 15, 71, 79, 96, 95, 87, 81, 22, 14, 25, 80, 24, 75, 84, 21, 11, 85, 76, 69, 74, 68, 0, 6, 66, 90, 73, 1, 17, 19, 7, 3, 78, 9, 5, 2, 70, 10, 64, 65, 67], [42, 96, 106, 120, 45, 125, 26, 23, 20, 118, 62, 109, 78, 52, 98, 80, 67, 63, 124, 90, 122, 32, 82, 71, 126, 7, 34, 101, 94, 58, 27, 56, 87, 14, 73, 55, 77, 116, 12, 86, 97, 123, 13, 81, 48, 85, 91, 115, 35, 57, 103, 104, 114, 43, 50, 16, 49, 18, 99, 110, 111, 39, 76, 33, 22, 64, 65, 117, 24, 119, 37, 10, 1, 53, 92, 59, 31, 47, 60, 28, 112, 84, 17, 19, 46, 121, 83, 44, 127, 21, 29, 38, 89, 113, 54, 30, 74, 5, 107, 70, 61, 93, 41, 51, 95, 105, 8, 4, 102, 88, 15, 108, 100, 40, 25, 36, 3, 6, 68, 79, 9, 75, 66, 69, 72, 11, 2, 0], [42, 96, 106, 125, 45, 23, 26, 62, 20, 109, 52, 98, 80, 124, 118, 126, 120, 34, 63, 78, 90, 82, 55, 27, 94, 122, 101, 77, 56, 43, 116, 46, 111, 58, 103, 81, 44, 35, 119, 114, 97, 104, 53, 7, 123, 87, 48, 127, 115, 50, 99, 49, 107, 60, 39, 38, 41, 91, 113, 117, 110, 59, 47, 30, 100, 54, 121, 102, 40, 85, 37, 57, 67, 61, 108, 105, 36, 112, 31, 95, 18, 93, 16, 51, 28, 32, 33, 29, 73, 13, 92, 21, 14, 86, 84, 71, 24, 25, 17, 19, 12, 5, 22, 89, 10, 88, 83, 74, 9, 11, 1, 65, 69, 79, 75, 15, 76, 64, 3, 8, 72, 4, 6, 68, 70, 66, 0, 2], [42, 96, 125, 106, 52, 45, 23, 20, 26, 82, 80, 78, 120, 98, 118, 109, 5, 62, 73, 7, 77, 12, 124, 90, 101, 86, 126, 103, 67, 63, 122, 32, 94, 35, 27, 87, 9, 56, 16, 1, 55, 58, 21, 69, 85, 91, 83, 64, 84, 76, 114, 123, 75, 3, 50, 34, 13, 18, 116, 115, 22, 11, 29, 70, 17, 40, 19, 43, 15, 48, 10, 39, 97, 81, 14, 74, 30, 110, 4, 79, 37, 31, 89, 44, 108, 104, 24, 49, 25, 71, 47, 66, 59, 92, 99, 60, 113, 28, 95, 112, 107, 88, 51, 8, 111, 100, 117, 72, 121, 57, 93, 6, 119, 105, 38, 41, 33, 36, 65, 54, 127, 68, 61, 0, 2, 102, 46, 53], [42, 96, 45, 106, 125, 23, 124, 26, 20, 52, 109, 98, 120, 62, 118, 126, 80, 122, 82, 115, 94, 56, 103, 116, 55, 78, 34, 90, 63, 58, 43, 48, 87, 114, 97, 101, 50, 60, 77, 110, 39, 41, 35, 104, 13, 123, 31, 47, 108, 53, 107, 99, 119, 113, 30, 19, 121, 27, 117, 33, 57, 49, 51, 40, 7, 61, 38, 102, 54, 92, 44, 85, 100, 67, 46, 32, 111, 93, 95, 36, 59, 37, 105, 81, 91, 112, 18, 28, 16, 29, 84, 127, 71, 86, 24, 14, 25, 73, 21, 89, 88, 79, 76, 12, 22, 74, 72, 69, 83, 17, 11, 15, 10, 9, 5, 68, 75, 65, 1, 8, 4, 66, 6, 64, 70, 0, 3, 2], [102, 62, 126, 98, 87, 26, 110, 85, 51, 41, 30, 19, 38, 45, 50, 22, 15, 79, 103, 94, 90, 84, 12, 93, 31, 17, 16, 113, 21, 109, 69, 105, 112, 20, 125, 127, 23, 8, 53, 65, 83, 25, 32, 117, 119, 14, 88, 63, 6, 55, 99, 58, 1, 116, 122, 75, 46, 57, 60, 96, 100, 108, 34, 121, 124, 52, 80, 120, 115, 74, 59, 106, 24, 89, 114, 49, 97, 111, 10, 76, 104, 118, 0, 2, 95, 64, 27, 86, 47, 40, 73, 123, 48, 81, 36, 92, 70, 54, 67, 82, 28, 42, 66, 5, 35, 29, 91, 78, 18, 77, 9, 13, 56, 44, 72, 39, 11, 101, 71, 68, 107, 61, 33, 37, 4, 43, 7, 3], [102, 126, 62, 98, 87, 17, 85, 110, 90, 26, 22, 19, 50, 8, 14, 75, 45, 103, 41, 15, 94, 21, 93, 89, 6, 51, 24, 113, 109, 2, 30, 84, 88, 83, 68, 92, 12, 38, 27, 58, 20, 31, 72, 79, 78, 112, 122, 23, 25, 74, 80, 32, 4, 71, 16, 117, 119, 53, 49, 127, 77, 29, 47, 7, 35, 81, 76, 60, 105, 10, 111, 28, 55, 120, 116, 52, 82, 0, 124, 46, 48, 5, 91, 13, 99, 125, 34, 67, 39, 36, 33, 121, 96, 97, 57, 95, 86, 63, 37, 118, 65, 115, 18, 44, 100, 66, 104, 11, 73, 108, 59, 42, 114, 43, 101, 9, 61, 69, 123, 54, 56, 3, 106, 64, 107, 70, 1, 40], [102, 126, 98, 62, 26, 41, 45, 87, 85, 19, 17, 93, 38, 103, 22, 15, 94, 51, 21, 50, 12, 113, 92, 110, 32, 75, 14, 30, 8, 100, 90, 31, 79, 83, 84, 88, 89, 20, 86, 35, 36, 54, 55, 25, 117, 127, 125, 119, 76, 13, 77, 81, 123, 46, 82, 23, 6, 2, 16, 53, 109, 39, 18, 27, 121, 59, 116, 70, 74, 97, 69, 80, 114, 47, 111, 52, 57, 105, 29, 118, 78, 58, 24, 49, 11, 63, 95, 99, 108, 122, 96, 115, 112, 10, 48, 91, 104, 124, 42, 66, 33, 72, 120, 40, 60, 101, 34, 37, 7, 43, 73, 65, 28, 1, 4, 107, 67, 5, 71, 44, 106, 61, 68, 9, 56, 3, 64, 0], [62, 102, 126, 98, 87, 41, 110, 26, 85, 38, 50, 45, 22, 105, 103, 109, 30, 113, 94, 63, 112, 93, 58, 125, 84, 121, 53, 127, 119, 15, 122, 31, 51, 116, 59, 57, 19, 52, 114, 55, 47, 97, 117, 118, 90, 44, 46, 43, 60, 49, 111, 108, 56, 42, 106, 54, 120, 124, 48, 115, 101, 36, 107, 32, 40, 123, 104, 100, 99, 61, 35, 37, 25, 39, 20, 96, 12, 17, 83, 95, 79, 92, 88, 27, 28, 33, 91, 29, 16, 8, 89, 23, 74, 21, 73, 34, 6, 75, 24, 10, 18, 82, 14, 69, 78, 86, 80, 71, 2, 77, 76, 5, 13, 81, 9, 7, 1, 65, 68, 11, 72, 70, 67, 0, 64, 66, 3, 4], [120, 122, 54, 37, 53, 127, 19, 101, 87, 17, 113, 35, 91, 58, 93, 60, 123, 124, 13, 115, 50, 117, 55, 56, 89, 63, 51, 118, 116, 32, 75, 45, 74, 114, 15, 119, 57, 14, 125, 112, 80, 49, 33, 36, 62, 52, 48, 100, 98, 12, 61, 97, 103, 47, 34, 110, 96, 90, 44, 73, 106, 59, 43, 126, 111, 41, 7, 26, 38, 18, 39, 121, 102, 85, 46, 108, 99, 104, 109, 107, 105, 29, 95, 42, 31, 27, 21, 20, 83, 81, 40, 72, 23, 94, 28, 92, 25, 30, 88, 24, 86, 5, 70, 68, 82, 67, 65, 2, 22, 0, 78, 77, 79, 84, 11, 9, 76, 16, 71, 8, 3, 10, 4, 69, 6, 1, 64, 66], [120, 122, 37, 13, 19, 17, 75, 14, 87, 15, 80, 74, 32, 85, 20, 53, 12, 18, 54, 73, 113, 127, 93, 58, 123, 26, 88, 115, 7, 30, 60, 50, 35, 124, 51, 55, 91, 45, 117, 72, 101, 116, 56, 63, 118, 94, 119, 57, 114, 90, 98, 62, 125, 112, 52, 25, 92, 49, 48, 86, 24, 61, 23, 36, 126, 33, 31, 96, 59, 47, 46, 34, 44, 82, 111, 110, 43, 70, 79, 41, 121, 21, 109, 99, 84, 29, 81, 108, 106, 100, 5, 42, 103, 27, 105, 104, 107, 16, 39, 97, 40, 95, 38, 102, 78, 77, 28, 22, 89, 68, 11, 83, 76, 67, 9, 65, 10, 2, 8, 71, 3, 0, 4, 6, 69, 64, 1, 66], [122, 120, 37, 84, 7, 16, 67, 74, 66, 69, 19, 75, 1, 68, 72, 24, 5, 70, 73, 12, 94, 78, 71, 11, 8, 53, 6, 64, 23, 13, 4, 3, 82, 88, 15, 54, 14, 80, 20, 90, 32, 30, 58, 92, 87, 93, 18, 113, 26, 115, 79, 77, 10, 123, 127, 50, 51, 60, 76, 124, 17, 117, 55, 56, 85, 21, 116, 96, 83, 63, 45, 118, 35, 9, 25, 65, 98, 0, 114, 119, 57, 28, 112, 101, 62, 49, 52, 29, 99, 48, 86, 125, 31, 61, 126, 59, 36, 81, 46, 91, 41, 44, 111, 47, 43, 22, 121, 110, 27, 95, 100, 109, 108, 2, 97, 33, 34, 106, 89, 105, 42, 102, 104, 39, 107, 103, 40, 38], [120, 122, 54, 87, 19, 53, 127, 123, 113, 60, 58, 124, 125, 115, 117, 55, 51, 116, 50, 17, 62, 86, 56, 27, 57, 119, 118, 45, 94, 49, 25, 52, 85, 114, 61, 96, 48, 28, 63, 23, 126, 112, 59, 44, 47, 111, 121, 24, 46, 109, 81, 43, 110, 90, 41, 106, 108, 84, 42, 107, 39, 83, 105, 82, 103, 35, 104, 40, 21, 37, 98, 38, 88, 36, 102, 89, 91, 79, 20, 65, 100, 101, 77, 16, 80, 97, 34, 99, 22, 78, 0, 33, 13, 10, 30, 29, 26, 76, 95, 31, 64, 93, 11, 18, 2, 9, 14, 15, 32, 3, 92, 71, 68, 4, 8, 7, 12, 75, 5, 6, 67, 69, 74, 1, 73, 70, 72, 66]], "model.layers.10.self_attn.k_proj": [[122, 121, 100, 118, 86, 63, 114, 123, 54, 61, 124, 119, 62, 52, 125, 115, 57, 108, 111, 117, 120, 60, 112, 31, 53, 126, 51, 127, 116, 59, 47, 58, 110, 55, 102, 46, 48, 83, 0, 45, 109, 49, 113, 56, 98, 29, 105, 104, 77, 9, 40, 44, 76, 103, 39, 38, 91, 106, 80, 81, 24, 6, 107, 35, 101, 34, 41, 37, 43, 21, 42, 79, 97, 26, 23, 1, 30, 22, 28, 36, 99, 32, 89, 88, 84, 3, 69, 50, 10, 95, 96, 92, 78, 19, 93, 33, 4, 20, 90, 82, 11, 85, 87, 7, 27, 25, 94, 14, 2, 18, 72, 8, 74, 5, 16, 71, 13, 66, 15, 67, 73, 65, 68, 70, 12, 75, 64, 17], [113, 124, 39, 50, 53, 96, 93, 21, 123, 120, 49, 15, 62, 126, 125, 63, 51, 112, 11, 87, 24, 82, 122, 58, 55, 114, 48, 117, 13, 91, 104, 116, 46, 61, 73, 30, 54, 115, 57, 71, 52, 95, 84, 26, 81, 118, 76, 121, 59, 127, 47, 80, 60, 111, 108, 19, 110, 86, 8, 99, 109, 56, 40, 45, 119, 100, 92, 89, 3, 10, 44, 35, 14, 1, 2, 98, 43, 68, 107, 79, 6, 106, 41, 27, 88, 0, 105, 42, 18, 38, 5, 102, 101, 36, 37, 33, 28, 23, 97, 77, 16, 17, 32, 94, 69, 34, 78, 66, 70, 7, 25, 31, 22, 72, 64, 4, 74, 29, 9, 83, 65, 90, 85, 20, 67, 12, 75, 103], [113, 56, 38, 80, 19, 77, 85, 23, 89, 11, 91, 9, 95, 58, 70, 72, 68, 55, 64, 14, 76, 1, 66, 20, 46, 123, 78, 111, 127, 44, 59, 106, 116, 112, 47, 17, 57, 53, 121, 69, 114, 117, 115, 71, 125, 51, 101, 109, 86, 118, 126, 42, 79, 124, 97, 52, 74, 48, 108, 45, 18, 96, 120, 24, 28, 82, 15, 37, 50, 33, 98, 39, 60, 122, 29, 119, 31, 93, 92, 32, 43, 3, 62, 103, 90, 61, 99, 34, 67, 5, 63, 12, 40, 49, 26, 88, 110, 35, 7, 30, 94, 100, 54, 104, 36, 81, 41, 105, 22, 8, 107, 10, 75, 25, 27, 87, 6, 84, 21, 0, 13, 83, 2, 73, 102, 16, 65, 4], [58, 62, 86, 37, 52, 96, 121, 29, 90, 14, 82, 17, 19, 12, 80, 60, 127, 77, 10, 72, 118, 125, 117, 24, 119, 55, 51, 20, 115, 63, 114, 126, 57, 120, 123, 79, 7, 100, 124, 50, 56, 44, 53, 48, 116, 98, 34, 59, 113, 46, 111, 110, 43, 45, 61, 25, 54, 93, 76, 112, 99, 109, 106, 70, 47, 9, 40, 49, 36, 15, 41, 122, 108, 105, 107, 11, 69, 104, 38, 42, 39, 102, 2, 23, 8, 97, 3, 85, 91, 21, 27, 35, 4, 95, 103, 31, 84, 83, 101, 6, 28, 33, 18, 74, 94, 30, 81, 65, 0, 92, 13, 22, 1, 16, 71, 87, 88, 68, 64, 32, 78, 5, 26, 89, 75, 73, 67, 66], [118, 127, 37, 126, 86, 63, 62, 10, 52, 34, 17, 48, 113, 30, 26, 119, 56, 121, 122, 15, 54, 70, 77, 112, 60, 43, 29, 50, 104, 59, 88, 51, 20, 120, 117, 116, 41, 110, 96, 115, 61, 123, 0, 3, 109, 57, 58, 55, 125, 47, 45, 53, 114, 108, 111, 49, 124, 106, 76, 103, 107, 46, 44, 82, 42, 71, 100, 2, 83, 38, 9, 65, 40, 68, 1, 72, 99, 32, 105, 92, 85, 39, 75, 35, 33, 95, 97, 36, 11, 91, 102, 14, 23, 31, 4, 80, 94, 67, 7, 64, 5, 73, 78, 69, 19, 25, 21, 27, 101, 93, 98, 81, 24, 90, 28, 89, 16, 12, 66, 79, 84, 87, 22, 8, 18, 74, 13, 6], [106, 32, 52, 26, 23, 20, 82, 125, 120, 42, 118, 45, 80, 78, 109, 126, 73, 58, 124, 77, 62, 112, 34, 12, 123, 114, 115, 50, 56, 55, 59, 54, 7, 5, 63, 85, 116, 47, 81, 122, 3, 51, 37, 119, 61, 39, 43, 103, 30, 104, 6, 1, 110, 127, 117, 101, 0, 107, 108, 11, 27, 48, 86, 65, 91, 46, 66, 13, 44, 57, 75, 74, 49, 35, 70, 111, 40, 60, 2, 22, 76, 64, 105, 98, 53, 79, 121, 102, 113, 33, 94, 29, 72, 71, 92, 36, 41, 83, 68, 8, 19, 25, 88, 14, 24, 100, 21, 4, 69, 38, 31, 97, 89, 10, 99, 95, 15, 28, 93, 96, 84, 90, 9, 17, 18, 87, 16, 67], [126, 62, 38, 34, 26, 94, 45, 85, 19, 41, 87, 22, 17, 110, 51, 84, 15, 50, 113, 12, 103, 75, 8, 14, 1, 127, 53, 93, 57, 112, 119, 117, 55, 116, 120, 39, 124, 125, 111, 122, 109, 59, 114, 118, 69, 63, 74, 52, 121, 102, 6, 29, 60, 115, 66, 28, 106, 5, 95, 73, 24, 108, 49, 32, 48, 58, 77, 123, 97, 46, 43, 33, 68, 105, 104, 54, 56, 16, 36, 47, 27, 35, 31, 40, 86, 61, 100, 44, 91, 64, 107, 4, 71, 67, 25, 78, 18, 42, 80, 9, 101, 99, 72, 23, 7, 37, 90, 3, 98, 82, 89, 92, 96, 70, 88, 0, 30, 21, 76, 13, 81, 20, 10, 83, 79, 11, 2, 65], [122, 120, 101, 24, 30, 82, 54, 127, 32, 79, 28, 84, 117, 26, 21, 113, 115, 124, 123, 55, 16, 27, 53, 56, 78, 60, 116, 50, 11, 77, 62, 114, 118, 9, 76, 51, 89, 63, 119, 57, 112, 52, 49, 125, 48, 93, 45, 72, 58, 126, 47, 59, 46, 44, 2, 70, 23, 61, 108, 81, 109, 10, 111, 110, 99, 121, 12, 65, 43, 5, 41, 106, 107, 29, 22, 35, 42, 39, 71, 104, 8, 105, 3, 103, 40, 14, 96, 97, 100, 4, 102, 38, 95, 98, 75, 83, 36, 80, 18, 69, 34, 68, 73, 33, 86, 6, 37, 31, 20, 92, 15, 7, 64, 87, 0, 91, 13, 90, 94, 25, 67, 88, 74, 17, 19, 85, 1, 66]], "model.layers.10.self_attn.qk_proj": [[122, 113, 62, 120, 118, 127, 126, 56, 121, 58, 52, 124, 106, 42, 54, 50, 90, 45, 110, 102, 55, 63, 117, 37, 125, 51, 116, 123, 53, 49, 60, 114, 87, 115, 85, 61, 38, 93, 57, 46, 23, 101, 59, 80, 83, 47, 32, 21, 22, 41, 112, 96, 34, 119, 103, 19, 16, 26, 25, 44, 13, 109, 20, 48, 104, 14, 77, 82, 86, 39, 43, 88, 27, 18, 84, 24, 100, 12, 11, 94, 98, 75, 81, 78, 30, 89, 108, 15, 29, 17, 111, 9, 91, 79, 73, 76, 31, 40, 107, 36, 35, 95, 97, 72, 6, 74, 105, 92, 10, 70, 99, 28, 71, 8, 68, 33, 64, 69, 65, 2, 1, 7, 3, 67, 5, 4, 0, 66], [122, 113, 62, 120, 118, 56, 121, 127, 126, 58, 52, 124, 106, 42, 54, 50, 117, 90, 116, 45, 37, 55, 110, 102, 60, 87, 114, 63, 123, 125, 85, 49, 51, 115, 23, 61, 38, 53, 101, 41, 93, 46, 103, 21, 80, 83, 96, 22, 34, 112, 104, 32, 19, 109, 59, 20, 26, 111, 27, 48, 43, 13, 119, 57, 25, 47, 98, 24, 84, 16, 86, 77, 88, 18, 14, 89, 39, 100, 30, 81, 108, 79, 12, 82, 31, 94, 78, 11, 75, 17, 29, 44, 9, 40, 91, 73, 95, 76, 15, 70, 92, 72, 97, 33, 74, 36, 105, 10, 35, 99, 6, 107, 69, 4, 8, 0, 68, 28, 65, 71, 64, 7, 5, 3, 67, 2, 66, 1], [122, 113, 62, 120, 118, 127, 121, 56, 126, 58, 52, 106, 124, 42, 54, 50, 116, 102, 55, 110, 37, 125, 51, 45, 90, 38, 60, 53, 114, 49, 63, 123, 87, 101, 117, 23, 93, 57, 115, 34, 96, 32, 119, 26, 85, 27, 43, 41, 103, 83, 22, 104, 19, 109, 21, 48, 112, 24, 25, 46, 61, 111, 16, 80, 59, 30, 94, 20, 84, 47, 86, 100, 88, 13, 77, 95, 18, 89, 39, 31, 98, 91, 29, 81, 14, 82, 35, 108, 78, 75, 12, 79, 92, 44, 11, 99, 15, 17, 40, 9, 70, 36, 73, 33, 76, 72, 107, 105, 28, 74, 97, 71, 4, 10, 1, 6, 7, 68, 64, 69, 8, 5, 65, 0, 66, 67, 3, 2], [122, 113, 62, 120, 118, 121, 127, 126, 56, 58, 52, 106, 124, 42, 54, 55, 117, 50, 90, 102, 45, 125, 60, 51, 114, 123, 37, 110, 116, 87, 63, 49, 53, 112, 23, 93, 115, 46, 119, 101, 38, 85, 96, 19, 57, 27, 47, 41, 61, 43, 109, 34, 22, 16, 104, 25, 83, 32, 59, 21, 80, 24, 44, 100, 26, 77, 13, 48, 111, 39, 88, 103, 30, 81, 18, 86, 94, 82, 84, 89, 98, 78, 20, 108, 17, 91, 12, 14, 40, 75, 95, 92, 11, 31, 35, 15, 105, 79, 9, 29, 107, 72, 70, 36, 73, 33, 74, 76, 28, 99, 97, 10, 71, 4, 6, 8, 0, 69, 7, 65, 68, 1, 5, 64, 67, 66, 3, 2], [122, 113, 62, 118, 120, 121, 126, 56, 127, 58, 52, 124, 106, 42, 55, 110, 54, 50, 90, 123, 45, 125, 102, 51, 37, 60, 116, 53, 49, 114, 117, 63, 87, 38, 115, 47, 61, 93, 46, 57, 119, 109, 96, 112, 23, 85, 59, 26, 101, 34, 103, 83, 21, 111, 80, 19, 16, 48, 22, 41, 77, 32, 13, 43, 100, 104, 18, 25, 39, 82, 14, 89, 81, 86, 24, 30, 27, 44, 94, 108, 20, 84, 88, 78, 11, 98, 36, 91, 17, 29, 9, 79, 75, 15, 40, 35, 73, 31, 12, 76, 95, 72, 70, 74, 105, 107, 97, 33, 4, 92, 10, 68, 8, 7, 28, 6, 71, 99, 65, 64, 0, 1, 67, 69, 2, 5, 66, 3], [122, 113, 62, 118, 120, 127, 121, 56, 126, 58, 52, 124, 106, 42, 54, 55, 90, 45, 110, 125, 116, 114, 60, 50, 117, 87, 63, 102, 123, 37, 53, 51, 46, 85, 93, 61, 49, 38, 96, 119, 23, 19, 115, 80, 109, 22, 103, 21, 41, 57, 34, 83, 77, 112, 101, 47, 16, 13, 111, 48, 20, 26, 32, 59, 25, 18, 104, 14, 100, 82, 98, 27, 88, 12, 15, 86, 11, 78, 39, 108, 24, 17, 81, 30, 75, 79, 89, 84, 43, 9, 31, 76, 105, 94, 73, 35, 29, 44, 91, 40, 107, 36, 10, 72, 92, 74, 70, 95, 33, 6, 97, 8, 28, 99, 68, 4, 69, 5, 71, 67, 64, 0, 1, 66, 7, 2, 65, 3], [122, 113, 62, 120, 118, 56, 127, 126, 121, 58, 52, 106, 124, 42, 90, 45, 125, 102, 55, 116, 37, 110, 54, 114, 87, 60, 50, 53, 85, 101, 51, 63, 23, 115, 61, 38, 46, 123, 49, 119, 93, 117, 22, 112, 80, 59, 19, 83, 16, 103, 21, 20, 48, 32, 96, 77, 34, 109, 41, 47, 57, 26, 111, 13, 100, 18, 25, 86, 82, 24, 27, 81, 78, 104, 84, 44, 88, 11, 75, 15, 14, 39, 12, 30, 98, 89, 94, 43, 9, 105, 108, 17, 31, 91, 76, 79, 29, 73, 40, 35, 36, 95, 107, 92, 74, 72, 99, 10, 6, 33, 70, 97, 8, 71, 28, 68, 7, 69, 66, 4, 5, 1, 65, 0, 3, 64, 2, 67], [113, 122, 62, 120, 118, 126, 56, 127, 121, 58, 106, 52, 124, 42, 90, 54, 45, 50, 37, 125, 117, 87, 102, 55, 23, 49, 46, 110, 60, 85, 123, 116, 83, 53, 61, 38, 119, 93, 114, 112, 21, 101, 22, 51, 115, 19, 80, 16, 77, 47, 63, 109, 111, 96, 86, 26, 32, 20, 25, 57, 82, 103, 41, 59, 13, 18, 84, 24, 78, 34, 27, 81, 98, 48, 39, 30, 100, 44, 17, 108, 15, 14, 79, 88, 75, 11, 104, 12, 9, 76, 89, 91, 105, 73, 107, 31, 43, 29, 40, 94, 92, 95, 36, 35, 6, 97, 72, 74, 99, 33, 10, 71, 70, 8, 68, 7, 28, 69, 65, 5, 0, 4, 64, 3, 1, 2, 66, 67], [122, 113, 62, 118, 120, 127, 56, 121, 126, 58, 52, 106, 124, 42, 90, 54, 45, 50, 37, 110, 123, 102, 55, 125, 87, 117, 116, 23, 60, 49, 38, 101, 53, 21, 93, 85, 83, 115, 114, 46, 47, 112, 63, 109, 57, 51, 59, 32, 16, 26, 22, 61, 19, 34, 96, 41, 111, 20, 25, 98, 119, 80, 77, 39, 27, 104, 24, 84, 86, 13, 82, 100, 78, 30, 108, 88, 43, 44, 18, 81, 48, 94, 91, 40, 17, 15, 29, 103, 11, 31, 75, 79, 105, 92, 89, 6, 14, 76, 95, 36, 9, 12, 73, 107, 97, 33, 35, 74, 99, 10, 28, 8, 72, 71, 68, 70, 1, 64, 4, 65, 0, 7, 69, 66, 2, 5, 67, 3], [122, 113, 62, 120, 118, 126, 127, 56, 121, 58, 52, 124, 106, 42, 117, 90, 116, 125, 123, 50, 55, 102, 45, 37, 87, 60, 23, 51, 63, 54, 110, 101, 115, 114, 38, 83, 53, 112, 85, 49, 109, 21, 93, 59, 47, 61, 80, 46, 111, 96, 16, 26, 77, 22, 82, 57, 119, 41, 19, 32, 103, 20, 13, 24, 34, 86, 27, 100, 108, 30, 25, 98, 104, 84, 14, 48, 18, 78, 81, 91, 39, 43, 88, 17, 79, 94, 12, 75, 44, 89, 73, 29, 11, 76, 15, 92, 107, 9, 40, 6, 36, 31, 97, 105, 95, 8, 33, 10, 99, 74, 35, 72, 68, 70, 28, 71, 69, 4, 7, 67, 0, 5, 65, 3, 1, 2, 64, 66], [122, 113, 62, 118, 126, 120, 121, 56, 127, 58, 52, 124, 106, 42, 123, 54, 125, 55, 110, 45, 102, 51, 90, 60, 50, 116, 117, 37, 63, 49, 87, 114, 53, 112, 119, 23, 115, 101, 85, 96, 93, 109, 57, 59, 38, 80, 111, 21, 83, 61, 19, 46, 47, 22, 77, 34, 16, 25, 41, 26, 20, 86, 13, 44, 18, 32, 82, 98, 48, 103, 100, 39, 30, 14, 81, 108, 43, 91, 24, 27, 15, 29, 78, 17, 11, 84, 12, 94, 89, 104, 73, 9, 8, 36, 79, 75, 76, 40, 10, 88, 6, 31, 74, 92, 107, 70, 105, 97, 95, 35, 68, 33, 28, 71, 99, 4, 65, 72, 1, 69, 0, 7, 64, 3, 66, 5, 2, 67], [122, 113, 62, 118, 120, 126, 127, 56, 121, 58, 52, 124, 106, 42, 123, 90, 51, 54, 55, 102, 110, 49, 45, 116, 125, 117, 50, 60, 87, 37, 114, 63, 112, 23, 53, 93, 38, 85, 101, 80, 109, 115, 83, 111, 59, 57, 96, 19, 46, 22, 47, 34, 20, 61, 21, 48, 25, 16, 119, 103, 77, 13, 41, 26, 32, 18, 39, 86, 14, 43, 98, 81, 82, 100, 11, 108, 75, 94, 24, 78, 27, 84, 17, 79, 15, 88, 76, 44, 9, 104, 30, 40, 12, 29, 36, 89, 73, 91, 10, 8, 107, 70, 74, 31, 6, 95, 35, 72, 97, 4, 92, 68, 7, 33, 99, 0, 69, 71, 1, 28, 65, 66, 105, 64, 5, 2, 3, 67], [122, 113, 62, 120, 118, 126, 127, 56, 121, 58, 106, 52, 124, 42, 54, 50, 90, 45, 102, 37, 110, 117, 55, 51, 116, 87, 23, 125, 85, 123, 38, 60, 57, 53, 101, 59, 63, 19, 46, 21, 119, 115, 93, 80, 26, 83, 32, 114, 22, 96, 47, 49, 109, 77, 86, 16, 103, 41, 20, 111, 112, 39, 98, 48, 61, 34, 25, 27, 24, 18, 13, 81, 82, 100, 91, 78, 17, 84, 15, 14, 89, 30, 88, 12, 43, 94, 76, 79, 11, 75, 104, 44, 29, 95, 9, 40, 108, 31, 36, 107, 8, 73, 92, 35, 33, 70, 97, 105, 74, 99, 72, 10, 71, 4, 28, 6, 7, 69, 68, 0, 65, 5, 3, 1, 66, 67, 64, 2], [122, 113, 62, 118, 126, 120, 127, 56, 121, 58, 52, 106, 124, 42, 54, 117, 125, 55, 90, 45, 50, 123, 87, 102, 110, 60, 116, 37, 63, 46, 114, 49, 51, 23, 85, 119, 57, 93, 38, 115, 83, 77, 19, 101, 112, 47, 53, 61, 21, 59, 109, 16, 96, 80, 22, 111, 86, 13, 20, 25, 34, 81, 26, 18, 41, 82, 27, 48, 103, 78, 14, 98, 39, 32, 76, 24, 100, 30, 17, 43, 75, 12, 15, 84, 11, 9, 73, 44, 79, 8, 104, 70, 91, 108, 29, 94, 40, 88, 31, 89, 72, 74, 92, 97, 35, 107, 36, 6, 95, 10, 7, 64, 71, 69, 33, 105, 4, 0, 65, 68, 99, 5, 66, 1, 2, 28, 67, 3], [122, 113, 62, 120, 118, 127, 126, 56, 58, 121, 52, 124, 106, 42, 54, 110, 55, 102, 125, 123, 50, 117, 45, 37, 90, 60, 51, 116, 87, 49, 46, 114, 61, 38, 23, 57, 63, 101, 112, 96, 109, 85, 115, 93, 59, 19, 41, 32, 83, 21, 26, 111, 22, 119, 47, 80, 53, 77, 34, 103, 20, 48, 25, 27, 98, 16, 86, 24, 100, 43, 39, 82, 13, 18, 44, 78, 81, 17, 94, 84, 15, 79, 89, 30, 107, 14, 40, 108, 29, 70, 75, 91, 104, 73, 9, 76, 11, 12, 88, 36, 31, 8, 72, 95, 35, 74, 105, 92, 97, 10, 33, 6, 28, 71, 99, 64, 1, 4, 69, 68, 0, 7, 65, 5, 67, 2, 66, 3], [122, 113, 62, 118, 120, 127, 126, 56, 121, 58, 52, 106, 124, 42, 55, 54, 45, 110, 90, 102, 125, 123, 117, 60, 37, 114, 51, 50, 87, 63, 46, 109, 61, 116, 85, 93, 101, 115, 49, 103, 53, 23, 38, 112, 47, 96, 57, 22, 83, 80, 59, 19, 111, 21, 26, 20, 32, 86, 77, 16, 41, 82, 34, 98, 13, 25, 100, 119, 39, 27, 24, 44, 78, 81, 18, 43, 48, 14, 30, 84, 89, 104, 75, 17, 15, 11, 76, 79, 29, 73, 12, 40, 108, 91, 94, 70, 9, 31, 88, 105, 74, 33, 10, 72, 36, 92, 107, 35, 95, 97, 99, 8, 71, 6, 68, 28, 4, 7, 0, 1, 66, 69, 64, 67, 5, 3, 2, 65], [122, 113, 62, 120, 118, 126, 127, 56, 121, 58, 52, 106, 124, 42, 117, 37, 102, 54, 90, 125, 87, 45, 116, 110, 50, 55, 123, 60, 23, 85, 46, 63, 112, 51, 114, 101, 93, 80, 38, 115, 49, 96, 59, 19, 47, 83, 21, 53, 103, 32, 111, 109, 27, 57, 119, 22, 26, 86, 16, 20, 77, 61, 24, 34, 25, 98, 81, 82, 48, 100, 18, 39, 41, 78, 44, 14, 13, 11, 79, 30, 17, 91, 84, 15, 88, 43, 9, 94, 40, 89, 12, 75, 104, 108, 29, 76, 31, 95, 73, 72, 36, 10, 35, 74, 92, 70, 28, 33, 107, 8, 6, 99, 7, 105, 97, 71, 68, 4, 69, 66, 5, 0, 64, 67, 1, 3, 65, 2], [122, 113, 62, 118, 120, 126, 56, 127, 121, 58, 106, 52, 124, 42, 117, 102, 37, 110, 90, 45, 87, 54, 125, 50, 60, 55, 112, 116, 85, 51, 23, 38, 46, 49, 123, 114, 101, 63, 53, 93, 115, 21, 47, 109, 26, 19, 83, 96, 80, 103, 59, 32, 61, 111, 57, 119, 16, 27, 22, 18, 41, 44, 24, 39, 34, 25, 86, 98, 77, 20, 84, 82, 100, 108, 17, 15, 43, 81, 88, 78, 30, 13, 91, 11, 79, 14, 104, 75, 40, 76, 89, 48, 31, 94, 9, 95, 72, 29, 73, 36, 107, 92, 12, 10, 35, 33, 6, 99, 74, 28, 97, 8, 70, 105, 7, 71, 5, 4, 68, 64, 0, 1, 65, 69, 67, 2, 3, 66], [122, 113, 62, 118, 120, 126, 127, 121, 56, 58, 52, 124, 106, 42, 117, 55, 102, 37, 125, 45, 60, 116, 110, 54, 90, 51, 123, 50, 87, 46, 38, 101, 114, 49, 115, 23, 53, 63, 32, 112, 93, 47, 96, 26, 59, 119, 85, 21, 22, 109, 44, 57, 61, 34, 19, 83, 103, 43, 41, 16, 100, 111, 88, 24, 27, 80, 20, 91, 39, 98, 86, 25, 94, 18, 108, 82, 77, 89, 78, 107, 84, 48, 104, 81, 30, 40, 14, 17, 13, 95, 29, 79, 31, 15, 11, 75, 6, 9, 36, 35, 99, 72, 28, 33, 76, 92, 12, 73, 105, 74, 97, 10, 70, 7, 69, 68, 71, 8, 0, 1, 3, 4, 5, 64, 65, 67, 66, 2], [122, 113, 62, 120, 118, 127, 126, 56, 121, 58, 52, 106, 124, 42, 117, 54, 60, 37, 55, 102, 45, 90, 51, 125, 87, 110, 50, 116, 123, 114, 101, 46, 61, 85, 63, 112, 23, 96, 93, 59, 38, 49, 22, 103, 47, 109, 111, 119, 53, 32, 34, 115, 41, 19, 21, 26, 27, 16, 83, 57, 20, 24, 77, 18, 25, 44, 86, 80, 39, 48, 98, 100, 30, 88, 43, 13, 89, 82, 14, 108, 78, 75, 81, 84, 31, 79, 94, 104, 29, 11, 6, 17, 73, 15, 95, 9, 35, 72, 105, 92, 76, 33, 91, 40, 12, 36, 107, 74, 99, 28, 8, 97, 10, 68, 7, 5, 71, 70, 4, 64, 66, 1, 65, 69, 0, 67, 3, 2], [122, 113, 62, 118, 120, 126, 127, 56, 121, 58, 106, 52, 124, 42, 117, 55, 60, 125, 110, 54, 90, 51, 102, 114, 50, 45, 123, 112, 87, 37, 115, 53, 85, 116, 93, 23, 119, 61, 109, 46, 63, 38, 22, 47, 49, 103, 101, 59, 83, 111, 19, 21, 96, 80, 57, 77, 26, 34, 86, 44, 27, 98, 16, 39, 32, 13, 82, 24, 14, 25, 20, 48, 100, 81, 18, 108, 17, 41, 84, 30, 88, 11, 9, 89, 75, 72, 79, 91, 43, 6, 78, 104, 29, 94, 15, 76, 73, 36, 40, 31, 107, 12, 92, 74, 10, 35, 8, 33, 99, 95, 70, 68, 28, 97, 105, 4, 5, 7, 71, 66, 0, 1, 64, 67, 69, 65, 3, 2], [122, 113, 62, 118, 120, 126, 127, 121, 56, 58, 52, 106, 124, 42, 117, 55, 110, 45, 102, 54, 90, 51, 125, 37, 46, 87, 50, 114, 115, 112, 60, 63, 61, 123, 38, 116, 53, 101, 49, 85, 23, 119, 47, 109, 22, 93, 83, 57, 26, 21, 59, 19, 96, 41, 111, 80, 44, 103, 34, 18, 32, 16, 24, 27, 77, 39, 43, 48, 100, 13, 25, 88, 20, 84, 17, 94, 98, 82, 30, 91, 86, 78, 89, 11, 14, 76, 104, 75, 29, 79, 81, 36, 107, 31, 15, 9, 95, 73, 99, 35, 40, 92, 108, 12, 6, 33, 72, 74, 10, 97, 28, 105, 68, 70, 8, 7, 64, 71, 66, 0, 67, 65, 4, 5, 69, 2, 1, 3], [122, 113, 62, 118, 126, 120, 56, 127, 121, 58, 52, 106, 124, 42, 54, 125, 117, 110, 37, 55, 50, 90, 102, 53, 60, 45, 123, 114, 51, 115, 87, 63, 38, 61, 112, 101, 49, 23, 109, 116, 57, 85, 59, 119, 111, 46, 26, 21, 96, 19, 93, 22, 34, 47, 83, 41, 48, 32, 103, 80, 16, 13, 98, 44, 27, 25, 77, 91, 24, 84, 86, 82, 30, 94, 29, 18, 20, 40, 39, 100, 107, 17, 81, 108, 78, 43, 31, 88, 11, 76, 14, 89, 75, 9, 79, 36, 35, 104, 95, 15, 73, 99, 12, 92, 10, 70, 8, 28, 97, 72, 74, 68, 33, 105, 7, 6, 64, 0, 71, 1, 65, 4, 5, 67, 69, 2, 66, 3], [122, 113, 62, 118, 120, 126, 56, 127, 121, 58, 106, 52, 124, 42, 117, 60, 54, 125, 90, 45, 87, 50, 37, 55, 110, 102, 51, 123, 114, 49, 61, 46, 63, 116, 38, 119, 93, 23, 115, 85, 22, 101, 96, 21, 59, 83, 109, 53, 112, 103, 26, 111, 80, 34, 57, 16, 25, 47, 19, 27, 32, 41, 13, 98, 77, 24, 18, 30, 20, 48, 108, 43, 44, 39, 104, 82, 17, 86, 88, 75, 100, 81, 14, 84, 29, 78, 31, 91, 11, 94, 12, 73, 15, 9, 107, 40, 70, 92, 79, 76, 89, 36, 35, 33, 95, 99, 74, 8, 72, 10, 105, 97, 68, 6, 28, 4, 71, 69, 7, 0, 5, 64, 1, 67, 65, 66, 2, 3], [122, 113, 62, 120, 118, 127, 126, 56, 121, 58, 52, 106, 124, 42, 55, 125, 117, 123, 102, 50, 116, 37, 60, 90, 45, 110, 87, 54, 51, 61, 63, 46, 53, 38, 115, 101, 23, 114, 112, 93, 57, 85, 119, 22, 96, 83, 49, 47, 26, 109, 103, 48, 21, 80, 59, 111, 32, 27, 19, 16, 25, 20, 44, 24, 18, 34, 88, 41, 86, 43, 13, 98, 82, 77, 91, 104, 100, 94, 17, 84, 39, 30, 11, 14, 81, 40, 9, 89, 75, 79, 15, 78, 70, 29, 12, 107, 76, 73, 35, 33, 31, 36, 95, 99, 74, 92, 97, 108, 10, 8, 72, 71, 28, 7, 4, 6, 5, 69, 68, 0, 105, 65, 64, 66, 3, 67, 1, 2], [122, 113, 62, 120, 118, 126, 127, 56, 121, 58, 106, 52, 124, 42, 117, 90, 50, 125, 102, 45, 60, 54, 55, 123, 87, 37, 23, 49, 63, 110, 46, 112, 116, 51, 47, 114, 38, 85, 109, 93, 101, 83, 119, 96, 115, 103, 59, 86, 22, 57, 16, 41, 32, 26, 111, 53, 21, 27, 20, 19, 61, 80, 34, 98, 13, 48, 82, 77, 100, 24, 25, 44, 14, 30, 75, 17, 18, 84, 11, 79, 9, 81, 43, 88, 12, 78, 39, 91, 94, 40, 89, 73, 31, 108, 15, 104, 70, 92, 8, 76, 36, 29, 10, 33, 107, 95, 35, 97, 74, 72, 99, 105, 6, 71, 7, 28, 68, 4, 1, 64, 5, 69, 2, 0, 65, 67, 66, 3], [122, 113, 62, 120, 118, 126, 127, 121, 56, 58, 52, 106, 124, 42, 125, 50, 54, 123, 90, 55, 60, 45, 110, 102, 37, 117, 114, 112, 87, 116, 53, 115, 38, 49, 23, 51, 101, 109, 57, 63, 93, 119, 19, 96, 85, 32, 47, 21, 59, 46, 41, 61, 22, 34, 83, 80, 26, 27, 94, 86, 111, 44, 103, 25, 24, 16, 43, 20, 91, 77, 39, 84, 13, 108, 18, 88, 40, 100, 30, 98, 31, 17, 82, 48, 104, 14, 29, 89, 81, 11, 95, 107, 15, 12, 36, 79, 73, 75, 78, 35, 9, 8, 76, 92, 99, 70, 10, 33, 28, 105, 97, 74, 7, 4, 6, 72, 71, 68, 65, 1, 0, 69, 5, 64, 67, 2, 66, 3], [122, 113, 62, 118, 120, 126, 127, 56, 121, 58, 52, 106, 124, 42, 54, 90, 60, 123, 117, 45, 37, 50, 125, 102, 55, 49, 114, 87, 110, 116, 112, 115, 93, 57, 23, 38, 53, 51, 109, 85, 83, 63, 119, 101, 46, 61, 21, 96, 19, 59, 80, 41, 16, 22, 20, 47, 98, 27, 34, 77, 82, 13, 32, 25, 26, 100, 103, 111, 44, 30, 24, 86, 81, 39, 18, 14, 48, 108, 43, 78, 73, 11, 94, 84, 75, 9, 17, 79, 91, 104, 12, 29, 15, 89, 88, 76, 8, 31, 36, 40, 92, 10, 74, 107, 70, 6, 71, 33, 35, 7, 0, 4, 97, 72, 68, 65, 64, 28, 105, 1, 99, 95, 2, 5, 69, 67, 3, 66], [122, 113, 62, 118, 120, 127, 126, 121, 56, 58, 52, 106, 124, 42, 55, 54, 102, 51, 123, 90, 37, 50, 125, 110, 45, 53, 87, 114, 117, 116, 63, 60, 38, 49, 61, 57, 115, 101, 85, 23, 112, 93, 103, 83, 96, 32, 47, 46, 109, 22, 111, 41, 34, 59, 26, 19, 24, 27, 77, 21, 80, 16, 20, 82, 98, 17, 94, 86, 39, 43, 88, 30, 48, 13, 44, 100, 18, 14, 119, 29, 25, 84, 91, 78, 89, 31, 75, 79, 11, 108, 9, 73, 104, 35, 81, 6, 12, 36, 40, 107, 95, 15, 92, 8, 76, 33, 97, 10, 99, 74, 70, 105, 71, 4, 68, 72, 28, 7, 1, 64, 2, 5, 69, 65, 0, 66, 67, 3], [122, 113, 62, 120, 118, 127, 56, 126, 121, 58, 52, 124, 106, 42, 54, 90, 102, 50, 37, 125, 55, 60, 117, 45, 110, 87, 51, 116, 123, 46, 114, 53, 49, 101, 59, 57, 38, 61, 85, 93, 112, 63, 23, 109, 83, 22, 96, 115, 47, 32, 103, 21, 80, 119, 26, 19, 77, 34, 13, 41, 44, 16, 25, 20, 14, 18, 98, 81, 27, 100, 24, 11, 30, 78, 48, 75, 86, 82, 111, 17, 108, 39, 43, 9, 79, 12, 91, 94, 84, 76, 6, 73, 8, 89, 15, 40, 104, 31, 29, 88, 36, 107, 10, 72, 33, 92, 99, 74, 35, 97, 95, 4, 71, 70, 28, 105, 68, 64, 7, 2, 65, 5, 1, 0, 3, 69, 66, 67], [122, 113, 62, 118, 120, 126, 127, 56, 121, 58, 52, 106, 124, 42, 54, 55, 90, 102, 45, 110, 37, 125, 117, 123, 87, 50, 51, 116, 57, 60, 63, 46, 53, 101, 49, 114, 23, 85, 38, 61, 93, 115, 83, 59, 112, 103, 109, 47, 32, 21, 22, 16, 96, 19, 20, 13, 41, 80, 24, 77, 34, 48, 25, 100, 82, 27, 111, 26, 86, 104, 88, 14, 17, 44, 43, 94, 119, 78, 18, 39, 11, 12, 30, 81, 98, 73, 84, 31, 108, 15, 89, 76, 75, 29, 79, 35, 6, 91, 40, 9, 92, 107, 36, 95, 8, 10, 99, 97, 68, 72, 74, 33, 71, 105, 7, 4, 5, 70, 0, 1, 64, 28, 65, 67, 66, 69, 3, 2], [122, 113, 62, 120, 118, 126, 56, 121, 127, 58, 52, 106, 124, 42, 54, 117, 45, 90, 50, 55, 125, 102, 63, 87, 60, 37, 116, 123, 51, 110, 114, 115, 85, 38, 53, 101, 93, 49, 23, 57, 61, 46, 83, 21, 59, 47, 109, 80, 119, 20, 13, 96, 22, 34, 77, 103, 111, 48, 19, 112, 25, 41, 82, 32, 26, 86, 27, 24, 16, 44, 43, 14, 30, 75, 100, 78, 89, 18, 11, 104, 98, 84, 88, 81, 17, 12, 9, 76, 73, 15, 39, 79, 94, 29, 6, 31, 40, 8, 91, 108, 105, 92, 107, 10, 72, 35, 71, 99, 95, 36, 74, 97, 70, 4, 33, 68, 69, 28, 7, 5, 64, 0, 1, 65, 66, 3, 2, 67]], "model.layers.11.self_attn.q_proj": [[45, 36, 96, 109, 92, 23, 86, 62, 48, 28, 14, 20, 10, 16, 61, 81, 71, 54, 56, 25, 89, 73, 18, 111, 22, 32, 70, 66, 4, 58, 47, 67, 125, 76, 17, 83, 88, 94, 42, 37, 38, 11, 12, 84, 29, 107, 30, 82, 78, 27, 80, 87, 95, 74, 21, 24, 19, 75, 13, 97, 64, 6, 1, 50, 9, 5, 102, 91, 15, 46, 77, 55, 85, 40, 116, 53, 41, 79, 43, 90, 49, 113, 115, 39, 122, 60, 51, 100, 3, 34, 98, 101, 69, 126, 112, 26, 123, 59, 93, 33, 114, 7, 108, 120, 105, 72, 119, 31, 63, 118, 103, 106, 99, 124, 35, 110, 44, 52, 8, 57, 127, 121, 2, 104, 117, 65, 0, 68], [45, 36, 109, 96, 62, 92, 28, 25, 54, 86, 48, 4, 89, 81, 7, 3, 47, 102, 23, 100, 40, 20, 32, 43, 14, 22, 1, 111, 8, 38, 58, 10, 101, 66, 46, 12, 116, 41, 50, 125, 107, 55, 64, 18, 114, 56, 126, 122, 51, 49, 74, 120, 42, 5, 57, 97, 112, 59, 118, 127, 123, 19, 39, 108, 113, 37, 115, 105, 99, 53, 63, 31, 104, 0, 52, 121, 35, 29, 61, 119, 124, 34, 106, 44, 110, 60, 33, 117, 30, 11, 98, 103, 95, 17, 9, 68, 91, 93, 90, 70, 84, 65, 75, 27, 24, 80, 94, 26, 15, 85, 72, 88, 78, 21, 82, 2, 6, 73, 16, 69, 79, 76, 87, 83, 71, 77, 67, 13], [45, 36, 109, 48, 100, 25, 86, 96, 92, 125, 111, 58, 54, 56, 22, 50, 107, 46, 43, 113, 55, 47, 41, 122, 23, 116, 115, 114, 120, 101, 32, 119, 112, 53, 118, 57, 28, 62, 60, 117, 38, 127, 121, 40, 59, 97, 49, 51, 20, 63, 42, 39, 110, 126, 102, 52, 124, 44, 123, 108, 10, 81, 103, 105, 14, 106, 4, 7, 61, 88, 104, 89, 15, 37, 24, 83, 31, 99, 98, 33, 35, 34, 91, 78, 30, 12, 19, 93, 29, 3, 95, 74, 84, 85, 21, 94, 90, 66, 17, 80, 18, 16, 87, 1, 8, 9, 27, 75, 26, 82, 11, 76, 70, 79, 65, 64, 0, 73, 68, 72, 5, 13, 77, 71, 6, 69, 2, 67], [45, 36, 96, 109, 25, 28, 62, 92, 86, 48, 23, 20, 54, 89, 14, 81, 111, 61, 56, 58, 10, 18, 97, 107, 47, 32, 41, 22, 46, 125, 12, 102, 80, 100, 43, 30, 16, 4, 50, 42, 113, 53, 40, 38, 39, 9, 115, 101, 37, 66, 1, 55, 64, 71, 31, 122, 3, 17, 44, 114, 126, 105, 91, 116, 21, 108, 5, 87, 70, 51, 93, 112, 94, 8, 34, 29, 60, 49, 57, 110, 90, 83, 95, 7, 85, 121, 19, 118, 98, 120, 84, 78, 127, 24, 52, 123, 99, 88, 106, 35, 15, 76, 124, 26, 27, 104, 119, 33, 59, 79, 103, 11, 117, 63, 82, 75, 72, 74, 73, 0, 77, 68, 65, 69, 6, 13, 2, 67], [58, 123, 63, 37, 117, 26, 87, 33, 88, 95, 86, 76, 17, 20, 78, 80, 83, 122, 81, 91, 126, 118, 30, 92, 0, 85, 12, 124, 67, 5, 9, 97, 82, 69, 79, 24, 90, 93, 1, 60, 70, 22, 18, 29, 14, 2, 23, 3, 25, 96, 27, 21, 61, 71, 16, 45, 120, 84, 50, 73, 75, 125, 77, 94, 72, 15, 32, 28, 109, 31, 19, 115, 89, 62, 11, 52, 56, 13, 54, 8, 113, 55, 127, 10, 59, 49, 112, 7, 57, 74, 47, 66, 100, 46, 68, 116, 111, 121, 6, 48, 103, 53, 64, 34, 106, 114, 38, 110, 40, 35, 98, 101, 65, 36, 119, 107, 105, 42, 102, 39, 51, 43, 104, 108, 41, 44, 99, 4], [123, 63, 58, 37, 117, 122, 126, 118, 60, 62, 33, 112, 115, 124, 61, 50, 52, 125, 54, 59, 30, 56, 45, 49, 101, 48, 47, 114, 119, 116, 57, 113, 55, 127, 39, 111, 99, 53, 110, 109, 40, 105, 121, 100, 106, 108, 43, 51, 46, 44, 104, 42, 120, 41, 10, 103, 102, 16, 88, 107, 34, 38, 86, 35, 21, 96, 36, 92, 32, 98, 97, 84, 95, 26, 18, 77, 93, 19, 28, 8, 29, 91, 94, 83, 14, 24, 22, 31, 65, 13, 68, 89, 15, 7, 27, 12, 17, 25, 23, 87, 82, 64, 70, 67, 90, 85, 75, 80, 79, 5, 4, 73, 78, 66, 71, 20, 72, 9, 69, 11, 81, 1, 76, 74, 3, 2, 0, 6], [63, 123, 58, 117, 60, 126, 118, 124, 122, 10, 37, 50, 62, 115, 54, 61, 68, 125, 52, 59, 65, 45, 112, 116, 56, 113, 8, 49, 127, 13, 16, 121, 48, 114, 55, 47, 7, 64, 57, 111, 53, 46, 40, 67, 21, 15, 14, 119, 17, 51, 120, 84, 12, 109, 75, 78, 110, 43, 99, 77, 100, 70, 73, 108, 5, 104, 44, 18, 105, 42, 106, 33, 39, 107, 102, 4, 41, 92, 103, 19, 101, 38, 1, 30, 66, 96, 24, 86, 9, 93, 95, 36, 88, 0, 28, 97, 34, 35, 87, 22, 80, 98, 83, 29, 79, 32, 31, 69, 91, 26, 25, 71, 76, 81, 27, 20, 23, 89, 72, 94, 74, 85, 90, 11, 82, 3, 2, 6], [58, 123, 63, 37, 122, 117, 126, 118, 60, 88, 33, 10, 112, 115, 61, 54, 62, 124, 50, 125, 52, 101, 30, 49, 59, 45, 16, 56, 55, 116, 48, 39, 111, 40, 113, 127, 114, 57, 93, 47, 8, 106, 53, 110, 100, 119, 120, 28, 92, 18, 51, 77, 19, 84, 86, 109, 108, 121, 46, 65, 21, 96, 26, 14, 68, 43, 104, 12, 44, 102, 105, 99, 42, 13, 91, 15, 41, 103, 24, 38, 95, 35, 32, 107, 97, 34, 83, 29, 36, 7, 5, 17, 98, 70, 31, 64, 75, 67, 25, 73, 22, 94, 89, 78, 66, 27, 82, 87, 90, 20, 80, 79, 71, 9, 4, 76, 23, 85, 1, 81, 72, 2, 11, 74, 0, 69, 3, 6], [102, 46, 110, 19, 91, 88, 125, 86, 114, 113, 95, 43, 111, 122, 27, 12, 10, 50, 63, 16, 18, 60, 52, 31, 107, 56, 17, 42, 54, 45, 123, 93, 71, 126, 119, 55, 69, 120, 118, 76, 83, 99, 44, 116, 41, 48, 121, 59, 51, 108, 58, 112, 47, 124, 105, 61, 57, 109, 36, 70, 53, 100, 39, 24, 104, 49, 34, 117, 4, 103, 127, 115, 67, 30, 106, 40, 62, 97, 23, 94, 101, 33, 22, 35, 9, 32, 80, 72, 82, 96, 37, 29, 89, 79, 98, 28, 14, 81, 20, 25, 87, 26, 90, 21, 11, 92, 74, 85, 65, 84, 78, 15, 38, 66, 13, 75, 77, 2, 73, 8, 0, 6, 5, 68, 7, 3, 1, 64], [102, 46, 110, 91, 86, 88, 125, 19, 16, 95, 113, 27, 69, 76, 122, 111, 10, 12, 71, 70, 72, 67, 43, 4, 85, 18, 93, 82, 114, 66, 9, 31, 24, 58, 90, 13, 17, 14, 117, 83, 23, 100, 22, 49, 121, 79, 65, 116, 74, 99, 63, 28, 124, 45, 29, 120, 80, 30, 41, 42, 54, 44, 81, 123, 56, 39, 78, 57, 60, 35, 126, 8, 52, 92, 2, 11, 112, 25, 53, 94, 103, 75, 47, 40, 59, 119, 61, 38, 36, 104, 107, 51, 127, 21, 55, 84, 115, 118, 32, 98, 0, 96, 101, 62, 108, 97, 87, 89, 33, 15, 26, 48, 50, 105, 37, 34, 106, 20, 109, 5, 6, 73, 77, 7, 68, 3, 64, 1], [102, 46, 110, 19, 27, 86, 88, 122, 95, 1, 78, 3, 7, 74, 76, 67, 91, 71, 24, 64, 65, 16, 5, 93, 4, 73, 125, 22, 18, 114, 9, 14, 68, 69, 111, 0, 82, 12, 11, 8, 31, 72, 113, 2, 70, 79, 99, 10, 66, 83, 43, 6, 75, 77, 45, 80, 17, 124, 121, 112, 40, 21, 20, 58, 116, 120, 42, 56, 63, 119, 50, 13, 123, 25, 57, 23, 47, 89, 15, 33, 81, 44, 104, 54, 90, 32, 85, 107, 103, 87, 34, 29, 118, 28, 105, 51, 36, 39, 97, 96, 38, 100, 101, 126, 94, 49, 84, 26, 92, 52, 117, 106, 62, 59, 108, 41, 30, 37, 35, 61, 115, 109, 60, 48, 98, 53, 127, 55], [102, 46, 110, 91, 88, 86, 19, 16, 95, 27, 18, 125, 122, 76, 10, 93, 31, 113, 114, 24, 111, 74, 71, 12, 69, 78, 70, 22, 72, 14, 83, 43, 4, 17, 82, 30, 67, 79, 80, 94, 60, 99, 29, 15, 58, 9, 75, 120, 23, 100, 39, 21, 66, 6, 13, 89, 90, 49, 38, 11, 116, 33, 42, 52, 126, 32, 81, 123, 36, 85, 107, 97, 65, 28, 59, 61, 41, 103, 92, 45, 73, 112, 54, 84, 50, 63, 87, 25, 53, 51, 56, 35, 44, 26, 101, 119, 48, 77, 37, 0, 121, 55, 2, 34, 106, 124, 127, 117, 109, 98, 57, 104, 115, 20, 40, 105, 62, 96, 47, 108, 118, 5, 8, 7, 68, 3, 64, 1], [63, 59, 106, 36, 14, 42, 114, 121, 60, 15, 126, 117, 47, 12, 97, 48, 52, 124, 123, 100, 120, 11, 82, 125, 122, 109, 113, 84, 115, 49, 56, 116, 58, 16, 53, 119, 62, 110, 33, 54, 44, 111, 51, 85, 61, 24, 57, 45, 55, 46, 118, 83, 127, 108, 112, 107, 43, 104, 50, 105, 72, 0, 41, 92, 17, 103, 40, 39, 65, 102, 37, 69, 38, 99, 81, 10, 89, 101, 67, 91, 34, 98, 6, 9, 7, 21, 35, 18, 73, 88, 93, 22, 68, 31, 86, 13, 29, 2, 96, 32, 94, 64, 95, 19, 27, 66, 25, 30, 28, 23, 1, 26, 4, 90, 8, 20, 75, 87, 76, 70, 77, 80, 78, 79, 5, 3, 71, 74], [59, 63, 100, 97, 42, 36, 60, 121, 93, 87, 79, 88, 73, 27, 76, 20, 117, 33, 47, 52, 82, 15, 106, 114, 92, 78, 75, 12, 124, 68, 86, 6, 116, 125, 14, 25, 85, 8, 67, 105, 126, 81, 18, 83, 104, 80, 11, 28, 48, 95, 29, 69, 56, 26, 31, 110, 7, 19, 65, 2, 90, 72, 91, 22, 10, 84, 24, 17, 123, 45, 120, 34, 115, 38, 98, 96, 89, 113, 37, 99, 16, 109, 111, 23, 53, 107, 103, 49, 102, 35, 46, 74, 21, 127, 30, 94, 62, 122, 108, 77, 39, 13, 32, 50, 101, 112, 119, 41, 58, 54, 44, 70, 40, 43, 64, 71, 51, 57, 9, 55, 5, 61, 118, 0, 3, 66, 4, 1], [59, 63, 100, 97, 14, 68, 36, 6, 76, 42, 60, 73, 7, 12, 10, 67, 69, 72, 8, 11, 87, 88, 121, 2, 93, 82, 27, 65, 25, 52, 33, 20, 13, 17, 117, 81, 16, 15, 47, 78, 106, 79, 70, 74, 29, 83, 21, 80, 86, 22, 95, 77, 116, 126, 85, 48, 71, 105, 114, 124, 123, 91, 89, 26, 9, 64, 5, 84, 125, 23, 115, 110, 19, 90, 56, 75, 104, 31, 92, 24, 49, 3, 113, 122, 120, 109, 18, 127, 111, 119, 45, 46, 107, 98, 53, 28, 58, 38, 35, 54, 37, 44, 51, 55, 62, 94, 103, 41, 50, 96, 34, 61, 39, 40, 57, 43, 108, 0, 112, 118, 102, 4, 32, 99, 30, 101, 66, 1], [59, 63, 100, 97, 36, 42, 121, 106, 60, 114, 93, 27, 73, 20, 117, 33, 78, 76, 52, 12, 125, 123, 88, 126, 86, 48, 124, 116, 120, 87, 6, 47, 113, 110, 122, 99, 67, 82, 58, 68, 111, 51, 115, 79, 25, 8, 109, 49, 53, 54, 62, 37, 119, 118, 55, 104, 56, 112, 46, 45, 69, 81, 98, 14, 92, 18, 57, 105, 44, 29, 61, 7, 127, 108, 10, 102, 2, 50, 15, 31, 107, 80, 22, 65, 43, 84, 35, 85, 39, 90, 41, 34, 83, 91, 75, 17, 103, 38, 95, 96, 94, 32, 28, 101, 24, 26, 40, 30, 89, 23, 11, 19, 5, 72, 74, 13, 9, 16, 64, 21, 0, 71, 77, 70, 66, 3, 4, 1], [101, 121, 116, 94, 57, 17, 71, 19, 2, 52, 78, 21, 3, 5, 75, 24, 64, 88, 9, 66, 76, 69, 124, 30, 1, 7, 80, 6, 118, 18, 115, 127, 117, 0, 67, 53, 11, 63, 70, 37, 120, 50, 125, 14, 33, 87, 72, 73, 29, 60, 83, 105, 97, 12, 82, 104, 54, 61, 79, 91, 59, 81, 16, 74, 122, 20, 4, 96, 43, 93, 46, 51, 113, 22, 106, 112, 77, 110, 65, 27, 40, 86, 15, 85, 111, 45, 13, 58, 126, 25, 26, 23, 55, 41, 84, 39, 48, 42, 89, 107, 102, 28, 31, 68, 10, 90, 62, 8, 92, 103, 123, 114, 49, 108, 99, 44, 56, 35, 100, 109, 34, 47, 38, 95, 32, 98, 119, 36], [101, 121, 116, 94, 57, 24, 21, 52, 19, 17, 78, 76, 80, 30, 88, 9, 75, 5, 37, 7, 15, 71, 11, 18, 69, 97, 29, 67, 33, 124, 63, 118, 66, 82, 117, 53, 127, 3, 73, 16, 115, 105, 22, 0, 120, 50, 60, 46, 86, 6, 81, 14, 2, 72, 1, 28, 26, 54, 83, 106, 20, 45, 51, 59, 43, 79, 122, 12, 85, 89, 91, 74, 77, 90, 39, 58, 104, 87, 125, 42, 13, 114, 41, 70, 110, 113, 112, 23, 61, 25, 96, 10, 109, 92, 93, 56, 95, 108, 31, 84, 49, 123, 32, 40, 102, 107, 62, 48, 126, 98, 4, 8, 99, 27, 103, 65, 111, 100, 35, 47, 38, 34, 55, 64, 119, 44, 36, 68], [101, 116, 121, 94, 52, 37, 30, 21, 57, 24, 19, 85, 80, 33, 117, 115, 105, 16, 50, 127, 124, 114, 62, 113, 26, 104, 59, 53, 125, 58, 118, 110, 60, 43, 51, 39, 49, 95, 63, 97, 107, 123, 54, 120, 55, 41, 45, 61, 91, 76, 17, 102, 46, 48, 42, 112, 56, 111, 28, 126, 122, 109, 38, 108, 103, 89, 119, 78, 36, 32, 73, 40, 22, 93, 99, 47, 44, 100, 106, 20, 35, 23, 27, 96, 15, 12, 18, 98, 87, 34, 92, 31, 86, 29, 88, 83, 79, 90, 25, 81, 82, 7, 9, 84, 14, 74, 5, 11, 66, 70, 72, 8, 75, 10, 13, 67, 77, 4, 6, 69, 2, 71, 3, 64, 68, 65, 0, 1], [101, 116, 121, 94, 57, 21, 24, 19, 76, 52, 17, 80, 78, 30, 6, 75, 9, 3, 33, 88, 71, 37, 124, 2, 127, 67, 29, 117, 97, 60, 115, 69, 83, 118, 46, 7, 18, 53, 64, 63, 50, 1, 23, 0, 16, 54, 90, 22, 106, 73, 70, 82, 14, 68, 12, 77, 26, 81, 122, 72, 112, 85, 61, 43, 13, 93, 58, 59, 104, 86, 98, 27, 79, 120, 42, 105, 20, 11, 74, 96, 84, 28, 39, 35, 125, 40, 65, 91, 25, 110, 123, 15, 107, 95, 41, 10, 92, 111, 32, 51, 99, 89, 87, 109, 103, 31, 48, 45, 62, 36, 126, 34, 113, 38, 44, 4, 55, 114, 49, 47, 56, 8, 100, 119, 102, 108, 5, 66], [40, 126, 97, 25, 89, 95, 87, 86, 28, 92, 121, 84, 15, 83, 115, 63, 82, 122, 55, 17, 49, 44, 53, 37, 119, 62, 29, 50, 38, 104, 45, 111, 91, 107, 61, 101, 46, 43, 41, 35, 9, 114, 123, 120, 102, 118, 23, 52, 117, 30, 54, 58, 109, 60, 12, 124, 48, 75, 57, 13, 42, 47, 116, 36, 103, 32, 98, 20, 34, 56, 105, 18, 85, 39, 51, 127, 110, 125, 90, 106, 16, 67, 112, 21, 59, 99, 1, 31, 96, 108, 113, 5, 80, 27, 74, 93, 79, 22, 100, 94, 71, 88, 11, 24, 0, 19, 26, 68, 77, 33, 73, 81, 6, 72, 70, 64, 14, 4, 66, 78, 7, 2, 69, 76, 65, 10, 8, 3], [40, 126, 97, 95, 89, 92, 86, 25, 120, 61, 15, 122, 28, 87, 82, 52, 83, 62, 17, 53, 84, 9, 60, 29, 121, 45, 63, 59, 12, 58, 118, 16, 107, 54, 21, 123, 114, 116, 85, 49, 47, 13, 77, 23, 117, 30, 8, 55, 33, 20, 115, 46, 127, 125, 50, 18, 79, 119, 36, 27, 56, 22, 57, 111, 80, 112, 90, 124, 44, 102, 110, 113, 106, 43, 93, 101, 91, 31, 41, 108, 109, 48, 38, 72, 24, 51, 35, 81, 68, 104, 4, 70, 19, 103, 71, 75, 100, 88, 96, 105, 37, 76, 6, 42, 2, 98, 39, 11, 78, 34, 26, 73, 14, 32, 99, 69, 66, 10, 94, 74, 1, 5, 0, 67, 64, 7, 3, 65], [126, 40, 97, 95, 86, 121, 101, 28, 82, 122, 62, 58, 120, 25, 53, 115, 45, 61, 102, 111, 84, 52, 54, 44, 114, 127, 60, 49, 92, 59, 118, 56, 63, 125, 50, 83, 46, 37, 112, 47, 113, 87, 89, 15, 43, 123, 57, 117, 51, 119, 55, 100, 29, 18, 12, 9, 116, 33, 108, 68, 30, 21, 109, 35, 48, 107, 110, 27, 73, 71, 91, 66, 38, 69, 80, 36, 93, 78, 124, 106, 19, 72, 10, 103, 99, 41, 42, 76, 14, 79, 65, 24, 90, 77, 23, 85, 16, 17, 31, 2, 13, 20, 105, 104, 5, 7, 34, 26, 3, 67, 64, 39, 98, 11, 1, 32, 94, 22, 75, 96, 88, 70, 74, 0, 8, 4, 81, 6], [126, 40, 97, 121, 61, 122, 71, 67, 49, 62, 89, 1, 54, 101, 60, 25, 53, 52, 59, 5, 45, 43, 95, 127, 63, 58, 114, 125, 120, 92, 28, 47, 56, 68, 50, 12, 46, 0, 86, 118, 84, 102, 111, 55, 57, 112, 82, 115, 117, 116, 123, 113, 66, 88, 108, 64, 81, 15, 73, 44, 51, 11, 109, 107, 77, 105, 119, 110, 106, 100, 30, 87, 24, 48, 42, 38, 19, 9, 41, 76, 14, 124, 17, 69, 2, 34, 36, 18, 27, 6, 4, 20, 78, 94, 35, 13, 72, 32, 39, 23, 103, 37, 99, 98, 16, 91, 85, 80, 3, 7, 74, 29, 83, 70, 104, 75, 21, 22, 93, 65, 10, 96, 31, 90, 26, 33, 79, 8], [104, 111, 30, 119, 26, 19, 127, 24, 94, 21, 79, 62, 9, 113, 61, 123, 12, 40, 81, 32, 86, 122, 47, 87, 118, 63, 78, 114, 54, 83, 68, 36, 49, 110, 89, 88, 126, 107, 90, 85, 100, 105, 115, 74, 56, 96, 71, 43, 4, 18, 70, 15, 57, 46, 17, 76, 125, 55, 121, 51, 120, 106, 42, 23, 37, 80, 117, 73, 3, 14, 97, 20, 112, 1, 103, 53, 92, 69, 45, 50, 5, 95, 101, 59, 66, 48, 38, 98, 39, 41, 22, 124, 102, 93, 28, 91, 108, 116, 10, 75, 52, 82, 58, 33, 99, 67, 77, 64, 35, 84, 8, 34, 31, 16, 60, 29, 44, 13, 2, 109, 27, 72, 11, 7, 25, 0, 65, 6], [104, 111, 119, 30, 62, 127, 24, 19, 21, 26, 94, 79, 12, 113, 47, 9, 123, 81, 122, 40, 32, 54, 70, 1, 61, 96, 86, 87, 49, 90, 118, 4, 91, 107, 67, 110, 15, 63, 20, 57, 120, 82, 105, 23, 64, 68, 43, 126, 85, 124, 103, 22, 56, 114, 45, 121, 36, 53, 46, 42, 55, 78, 58, 52, 112, 100, 109, 117, 18, 88, 89, 108, 71, 59, 98, 101, 2, 17, 115, 28, 83, 106, 74, 13, 41, 37, 34, 80, 27, 48, 66, 5, 50, 116, 125, 51, 75, 95, 60, 38, 84, 76, 72, 93, 8, 39, 99, 44, 73, 35, 102, 33, 3, 29, 16, 6, 10, 7, 0, 14, 31, 25, 97, 92, 11, 77, 69, 65], [104, 111, 119, 30, 127, 24, 62, 26, 94, 19, 21, 79, 113, 9, 61, 32, 122, 107, 12, 81, 86, 87, 54, 123, 40, 63, 47, 36, 105, 49, 70, 103, 117, 43, 114, 110, 56, 118, 100, 57, 126, 41, 83, 108, 85, 89, 96, 120, 90, 46, 37, 53, 28, 45, 95, 68, 88, 48, 42, 74, 115, 51, 112, 121, 39, 101, 38, 55, 1, 18, 125, 15, 4, 20, 78, 58, 76, 23, 17, 35, 91, 99, 59, 13, 98, 116, 52, 106, 44, 124, 93, 109, 50, 34, 102, 29, 97, 71, 67, 73, 64, 60, 82, 31, 33, 92, 72, 22, 14, 77, 80, 27, 66, 25, 3, 75, 84, 10, 5, 16, 2, 11, 8, 69, 0, 6, 7, 65], [104, 111, 30, 119, 24, 26, 19, 94, 79, 127, 81, 12, 61, 9, 21, 62, 32, 113, 78, 40, 118, 87, 90, 47, 96, 91, 54, 22, 123, 88, 100, 36, 4, 23, 85, 86, 18, 124, 15, 89, 107, 74, 110, 80, 63, 95, 72, 17, 53, 43, 122, 57, 83, 126, 98, 70, 120, 114, 45, 50, 49, 56, 20, 42, 37, 58, 125, 101, 34, 102, 105, 46, 112, 55, 14, 103, 35, 48, 16, 84, 7, 29, 76, 115, 97, 93, 25, 31, 99, 5, 82, 27, 51, 75, 41, 117, 109, 39, 59, 121, 38, 28, 44, 33, 92, 71, 2, 106, 52, 116, 108, 11, 60, 10, 6, 13, 68, 67, 1, 8, 77, 73, 64, 66, 69, 3, 0, 65], [59, 119, 46, 104, 51, 113, 61, 48, 56, 115, 98, 120, 116, 109, 57, 52, 49, 55, 53, 124, 60, 126, 63, 42, 112, 117, 122, 110, 50, 54, 118, 58, 111, 45, 47, 107, 121, 62, 106, 114, 123, 37, 125, 105, 41, 43, 127, 44, 39, 108, 36, 101, 38, 30, 24, 103, 33, 93, 86, 96, 89, 40, 35, 92, 102, 26, 88, 32, 95, 90, 97, 99, 100, 29, 27, 34, 80, 28, 91, 25, 20, 94, 31, 22, 87, 84, 16, 17, 18, 82, 85, 75, 13, 77, 23, 19, 21, 11, 68, 5, 83, 8, 76, 3, 14, 78, 9, 7, 81, 6, 79, 73, 2, 74, 0, 4, 1, 10, 66, 64, 15, 72, 12, 69, 65, 67, 70, 71], [46, 59, 104, 51, 53, 98, 117, 93, 91, 113, 48, 119, 49, 36, 37, 26, 114, 121, 55, 89, 61, 127, 110, 41, 30, 33, 120, 23, 57, 112, 109, 97, 54, 90, 115, 38, 107, 50, 63, 44, 83, 122, 56, 111, 94, 78, 84, 95, 106, 82, 52, 125, 118, 108, 45, 60, 99, 123, 116, 39, 102, 62, 43, 58, 27, 101, 100, 103, 105, 47, 32, 34, 35, 124, 25, 126, 96, 92, 21, 28, 18, 87, 42, 88, 80, 24, 75, 85, 22, 31, 40, 86, 81, 29, 15, 19, 13, 17, 20, 16, 77, 11, 14, 72, 79, 66, 9, 76, 74, 68, 8, 4, 73, 2, 64, 0, 70, 6, 3, 12, 5, 65, 1, 69, 71, 10, 67, 7], [46, 59, 48, 104, 51, 119, 61, 117, 113, 123, 98, 53, 55, 57, 56, 109, 126, 50, 114, 120, 37, 116, 63, 52, 93, 115, 60, 112, 49, 124, 122, 127, 42, 54, 110, 121, 118, 106, 44, 58, 45, 47, 41, 105, 111, 125, 43, 38, 62, 32, 29, 85, 89, 108, 36, 103, 107, 35, 31, 92, 39, 90, 95, 33, 84, 102, 91, 86, 97, 101, 96, 80, 26, 100, 99, 21, 28, 82, 30, 40, 25, 88, 34, 23, 20, 27, 24, 78, 68, 18, 22, 94, 75, 87, 16, 17, 11, 3, 77, 2, 83, 8, 14, 13, 0, 72, 5, 4, 65, 6, 9, 19, 73, 81, 74, 1, 79, 66, 70, 15, 10, 7, 76, 69, 67, 64, 71, 12], [104, 59, 98, 46, 83, 79, 23, 21, 81, 37, 76, 74, 48, 91, 8, 5, 69, 12, 7, 72, 78, 119, 71, 2, 3, 64, 66, 110, 67, 15, 84, 40, 19, 26, 25, 62, 10, 85, 14, 16, 27, 17, 18, 89, 87, 93, 13, 75, 31, 90, 20, 1, 51, 30, 94, 0, 73, 88, 77, 22, 86, 11, 29, 24, 70, 65, 92, 82, 9, 80, 4, 53, 6, 28, 32, 68, 96, 33, 95, 52, 61, 100, 108, 34, 99, 116, 115, 112, 39, 35, 102, 106, 56, 120, 54, 97, 36, 55, 101, 103, 113, 45, 111, 117, 50, 44, 38, 118, 124, 127, 122, 123, 109, 126, 58, 49, 121, 105, 60, 41, 42, 43, 47, 114, 57, 63, 107, 125]], "model.layers.11.self_attn.k_proj": [[109, 45, 32, 100, 54, 48, 92, 25, 56, 20, 23, 81, 47, 14, 62, 58, 107, 22, 16, 61, 112, 76, 85, 18, 69, 0, 68, 65, 120, 125, 86, 30, 55, 10, 73, 31, 122, 42, 59, 12, 7, 50, 71, 123, 72, 111, 118, 51, 2, 57, 43, 46, 126, 116, 38, 19, 3, 53, 113, 63, 40, 37, 13, 114, 102, 41, 124, 60, 108, 95, 119, 115, 93, 104, 127, 117, 44, 110, 39, 121, 49, 80, 52, 94, 11, 33, 101, 106, 97, 75, 99, 90, 91, 87, 96, 98, 17, 35, 105, 15, 28, 74, 77, 6, 103, 34, 29, 67, 27, 26, 24, 84, 5, 66, 21, 83, 88, 79, 36, 9, 82, 8, 64, 89, 78, 1, 70, 4], [58, 123, 63, 101, 126, 118, 117, 31, 61, 60, 50, 124, 86, 125, 52, 91, 62, 115, 20, 17, 121, 97, 55, 47, 56, 113, 49, 120, 111, 127, 112, 122, 59, 57, 48, 54, 53, 116, 45, 26, 119, 114, 78, 46, 76, 109, 51, 110, 108, 88, 73, 43, 75, 107, 44, 106, 32, 40, 103, 66, 42, 69, 37, 104, 41, 80, 105, 82, 39, 102, 99, 79, 36, 38, 89, 100, 70, 98, 93, 87, 96, 30, 34, 35, 33, 92, 19, 72, 27, 94, 25, 21, 74, 83, 29, 67, 13, 28, 64, 65, 5, 81, 95, 6, 90, 23, 71, 24, 11, 7, 16, 4, 9, 18, 85, 12, 8, 15, 10, 22, 84, 77, 1, 68, 0, 3, 14, 2], [110, 38, 46, 86, 91, 78, 88, 125, 18, 122, 19, 16, 31, 76, 111, 7, 114, 73, 3, 74, 1, 43, 5, 113, 64, 45, 121, 58, 49, 47, 120, 93, 56, 124, 6, 9, 42, 68, 17, 40, 15, 116, 119, 57, 63, 72, 62, 50, 13, 44, 123, 112, 99, 54, 103, 67, 35, 27, 118, 61, 59, 75, 11, 105, 41, 36, 2, 126, 108, 101, 107, 96, 90, 30, 70, 51, 55, 20, 106, 115, 82, 104, 8, 127, 21, 48, 60, 109, 100, 52, 117, 81, 87, 29, 34, 26, 94, 97, 37, 89, 79, 98, 53, 32, 66, 33, 24, 85, 39, 92, 95, 10, 28, 22, 80, 65, 25, 23, 84, 69, 14, 12, 71, 83, 0, 77, 4, 102], [59, 63, 36, 33, 121, 87, 117, 124, 93, 19, 106, 25, 60, 126, 95, 114, 81, 27, 26, 56, 125, 20, 116, 113, 47, 48, 42, 21, 53, 119, 52, 110, 86, 49, 80, 54, 123, 46, 16, 79, 58, 77, 111, 10, 98, 104, 100, 115, 122, 18, 45, 120, 55, 88, 78, 39, 51, 50, 7, 101, 40, 13, 118, 76, 41, 112, 108, 127, 105, 61, 35, 62, 109, 22, 102, 44, 34, 9, 43, 57, 38, 4, 107, 103, 69, 72, 28, 99, 37, 70, 14, 32, 74, 85, 92, 75, 3, 11, 0, 94, 96, 91, 30, 31, 73, 15, 97, 66, 17, 67, 89, 82, 71, 90, 24, 83, 6, 29, 65, 1, 23, 68, 12, 84, 2, 8, 64, 5], [121, 116, 37, 21, 24, 80, 78, 9, 75, 19, 17, 30, 57, 76, 64, 71, 69, 6, 2, 3, 97, 46, 53, 94, 124, 66, 127, 118, 65, 60, 74, 1, 59, 120, 115, 5, 63, 70, 43, 4, 122, 125, 79, 41, 93, 82, 73, 40, 50, 13, 91, 117, 42, 8, 104, 113, 88, 61, 7, 101, 0, 90, 67, 89, 33, 112, 15, 58, 92, 12, 28, 110, 107, 45, 87, 114, 34, 27, 86, 32, 108, 111, 22, 96, 99, 26, 109, 126, 55, 54, 23, 39, 31, 68, 51, 11, 10, 48, 105, 49, 84, 47, 123, 18, 72, 62, 106, 35, 29, 38, 25, 100, 20, 95, 102, 119, 44, 56, 98, 103, 14, 77, 36, 52, 16, 83, 85, 81], [126, 104, 33, 86, 28, 122, 121, 31, 62, 54, 118, 25, 60, 58, 17, 61, 53, 115, 87, 52, 127, 120, 50, 15, 63, 56, 114, 47, 49, 125, 108, 84, 113, 112, 3, 59, 100, 57, 64, 45, 46, 65, 37, 9, 82, 111, 42, 12, 109, 110, 117, 43, 75, 92, 123, 55, 116, 7, 70, 69, 107, 48, 51, 99, 106, 13, 29, 41, 83, 16, 124, 85, 105, 44, 0, 38, 102, 74, 119, 1, 39, 91, 94, 36, 8, 103, 66, 68, 19, 78, 89, 2, 4, 26, 34, 98, 101, 88, 32, 27, 96, 93, 11, 35, 81, 71, 95, 21, 90, 10, 24, 30, 6, 80, 14, 18, 22, 97, 72, 77, 23, 67, 79, 76, 20, 73, 5, 40], [40, 111, 47, 119, 94, 24, 127, 113, 26, 21, 62, 19, 110, 81, 122, 78, 114, 49, 123, 79, 118, 56, 63, 32, 12, 55, 120, 121, 105, 57, 61, 58, 126, 115, 43, 108, 54, 59, 42, 107, 87, 106, 9, 46, 18, 45, 125, 124, 104, 112, 48, 69, 116, 117, 51, 53, 37, 7, 39, 11, 41, 100, 60, 52, 109, 50, 38, 16, 36, 80, 44, 86, 73, 96, 28, 103, 99, 95, 102, 3, 101, 33, 91, 10, 30, 93, 89, 0, 97, 65, 71, 4, 35, 90, 13, 75, 76, 98, 23, 20, 27, 34, 82, 92, 31, 29, 15, 68, 22, 88, 25, 8, 84, 17, 74, 2, 14, 77, 66, 83, 85, 6, 72, 64, 70, 5, 1, 67], [40, 59, 46, 34, 110, 74, 76, 79, 7, 83, 81, 6, 21, 8, 5, 78, 2, 23, 93, 3, 0, 48, 1, 119, 9, 62, 27, 91, 117, 52, 115, 116, 77, 30, 45, 4, 89, 57, 49, 90, 64, 84, 56, 113, 126, 58, 122, 55, 120, 51, 47, 123, 118, 65, 108, 54, 109, 60, 42, 121, 124, 127, 112, 53, 61, 92, 63, 43, 125, 50, 44, 68, 37, 101, 38, 80, 111, 107, 15, 97, 72, 70, 88, 75, 114, 82, 87, 73, 105, 31, 41, 36, 102, 22, 18, 33, 95, 66, 86, 28, 103, 99, 106, 12, 39, 96, 104, 85, 11, 13, 94, 35, 67, 98, 32, 25, 29, 100, 24, 71, 20, 17, 19, 16, 26, 14, 10, 69]], "model.layers.11.self_attn.qk_proj": [[59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 45, 109, 54, 117, 104, 60, 57, 40, 94, 56, 37, 125, 122, 62, 61, 101, 113, 114, 36, 88, 48, 24, 118, 127, 119, 27, 47, 22, 124, 83, 21, 49, 30, 19, 78, 53, 115, 76, 17, 14, 55, 120, 100, 51, 89, 81, 12, 85, 33, 52, 102, 95, 80, 50, 86, 32, 43, 16, 28, 97, 108, 87, 91, 9, 90, 23, 112, 42, 7, 92, 71, 38, 29, 31, 44, 93, 25, 41, 73, 107, 15, 10, 82, 96, 75, 69, 74, 34, 20, 79, 3, 11, 106, 5, 18, 67, 2, 84, 105, 0, 26, 6, 66, 98, 64, 99, 39, 70, 68, 72, 103, 13, 77, 4, 1, 65, 35, 8], [59, 126, 46, 63, 121, 111, 116, 110, 58, 45, 123, 109, 54, 117, 60, 104, 57, 94, 40, 56, 113, 122, 101, 127, 37, 36, 119, 114, 88, 118, 125, 24, 61, 62, 22, 49, 27, 53, 30, 48, 76, 52, 21, 83, 115, 124, 47, 55, 78, 120, 100, 19, 81, 51, 95, 43, 14, 12, 42, 86, 108, 17, 85, 50, 33, 97, 80, 91, 90, 16, 89, 28, 9, 23, 112, 102, 92, 44, 38, 25, 107, 7, 31, 20, 79, 32, 87, 93, 71, 34, 73, 29, 105, 15, 96, 10, 106, 18, 69, 41, 74, 6, 82, 26, 5, 75, 84, 11, 0, 98, 2, 64, 66, 99, 67, 72, 39, 103, 3, 1, 70, 65, 68, 77, 4, 35, 8, 13], [59, 126, 121, 46, 63, 111, 110, 116, 58, 45, 123, 109, 54, 117, 104, 40, 94, 122, 61, 37, 101, 36, 56, 125, 57, 60, 113, 114, 24, 88, 62, 119, 118, 48, 127, 124, 27, 53, 30, 47, 21, 100, 52, 51, 22, 83, 19, 55, 115, 50, 49, 33, 102, 95, 43, 42, 76, 28, 78, 120, 108, 17, 86, 91, 89, 85, 38, 112, 106, 81, 32, 12, 23, 97, 14, 92, 93, 31, 90, 80, 87, 44, 107, 16, 29, 96, 25, 9, 73, 105, 34, 41, 71, 20, 7, 79, 84, 98, 10, 99, 26, 69, 11, 82, 18, 74, 15, 75, 6, 5, 66, 0, 103, 64, 2, 3, 67, 39, 70, 1, 65, 68, 35, 72, 4, 8, 13, 77], [59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 45, 109, 54, 117, 60, 104, 40, 57, 94, 36, 37, 122, 61, 56, 62, 114, 127, 101, 113, 24, 49, 119, 88, 118, 124, 48, 30, 27, 52, 47, 115, 125, 120, 95, 19, 22, 55, 21, 53, 50, 100, 83, 42, 28, 89, 14, 76, 108, 78, 102, 43, 81, 90, 17, 51, 12, 97, 92, 33, 85, 91, 32, 16, 31, 86, 80, 44, 38, 93, 87, 105, 107, 23, 64, 25, 73, 71, 96, 75, 29, 106, 112, 26, 9, 41, 7, 69, 2, 67, 74, 20, 79, 34, 0, 66, 11, 18, 98, 99, 82, 10, 3, 39, 5, 84, 6, 15, 103, 70, 68, 72, 35, 65, 4, 1, 8, 13, 77], [59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 45, 109, 117, 54, 104, 40, 122, 61, 57, 60, 56, 94, 127, 36, 37, 118, 62, 113, 101, 119, 24, 125, 114, 88, 48, 49, 52, 124, 83, 21, 30, 120, 19, 76, 22, 115, 47, 55, 27, 78, 53, 12, 95, 85, 80, 28, 100, 42, 43, 50, 17, 51, 16, 102, 81, 89, 108, 33, 14, 112, 23, 97, 91, 92, 90, 32, 107, 25, 86, 7, 31, 71, 105, 93, 29, 44, 73, 9, 87, 41, 96, 0, 38, 15, 75, 106, 64, 82, 79, 20, 74, 66, 2, 10, 98, 67, 18, 84, 69, 26, 99, 34, 70, 11, 65, 3, 5, 103, 6, 39, 8, 1, 72, 68, 13, 77, 4, 35], [59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 45, 109, 54, 117, 104, 60, 40, 57, 127, 122, 56, 113, 94, 114, 61, 36, 118, 37, 48, 101, 62, 88, 24, 119, 49, 22, 125, 83, 30, 52, 21, 27, 78, 19, 124, 12, 120, 76, 53, 81, 85, 95, 47, 14, 97, 17, 80, 115, 55, 43, 42, 16, 89, 28, 50, 51, 108, 100, 92, 102, 33, 86, 91, 25, 90, 107, 32, 73, 23, 9, 31, 96, 87, 79, 71, 7, 105, 44, 29, 20, 15, 93, 74, 82, 10, 112, 75, 69, 38, 99, 41, 26, 67, 18, 34, 11, 106, 70, 103, 84, 66, 5, 98, 3, 0, 64, 68, 6, 8, 2, 65, 39, 4, 1, 35, 77, 13, 72], [59, 126, 46, 63, 121, 116, 111, 110, 58, 123, 45, 109, 117, 54, 60, 104, 94, 40, 57, 36, 37, 113, 101, 24, 114, 127, 122, 88, 48, 21, 61, 118, 119, 56, 83, 19, 125, 27, 22, 62, 78, 30, 53, 49, 14, 17, 52, 95, 85, 76, 81, 12, 120, 124, 89, 86, 16, 100, 80, 55, 50, 28, 32, 115, 42, 43, 47, 112, 33, 92, 91, 87, 9, 102, 23, 97, 107, 51, 108, 38, 90, 71, 20, 96, 93, 31, 7, 11, 82, 75, 73, 25, 44, 29, 5, 41, 74, 18, 15, 34, 99, 10, 79, 105, 26, 67, 106, 98, 84, 69, 3, 39, 70, 64, 103, 2, 66, 8, 1, 6, 13, 0, 68, 4, 35, 72, 65, 77], [59, 126, 46, 121, 63, 116, 111, 110, 58, 45, 123, 109, 104, 40, 54, 94, 117, 60, 113, 24, 101, 37, 122, 36, 27, 88, 83, 57, 114, 127, 61, 22, 21, 47, 30, 52, 19, 118, 76, 56, 119, 125, 12, 62, 53, 48, 124, 78, 86, 81, 85, 49, 95, 17, 100, 14, 89, 115, 80, 91, 33, 55, 16, 28, 120, 50, 102, 92, 23, 42, 87, 43, 51, 7, 90, 32, 44, 9, 112, 108, 20, 38, 25, 82, 11, 97, 93, 75, 31, 29, 71, 15, 79, 73, 107, 106, 18, 10, 26, 5, 84, 64, 34, 41, 69, 74, 2, 70, 96, 0, 105, 3, 98, 8, 66, 99, 103, 67, 39, 1, 65, 4, 72, 6, 13, 68, 77, 35], [59, 126, 121, 46, 63, 111, 116, 110, 58, 45, 123, 109, 117, 104, 40, 94, 54, 60, 56, 36, 57, 122, 88, 101, 62, 37, 24, 114, 61, 119, 27, 49, 125, 118, 83, 113, 22, 19, 30, 21, 52, 124, 48, 100, 47, 127, 78, 55, 51, 89, 120, 85, 115, 17, 14, 76, 81, 12, 43, 86, 53, 80, 33, 95, 102, 38, 91, 28, 50, 42, 90, 108, 16, 23, 32, 107, 112, 92, 106, 93, 97, 44, 87, 31, 73, 9, 34, 7, 71, 15, 25, 18, 29, 41, 20, 10, 96, 11, 79, 105, 5, 82, 69, 98, 75, 26, 103, 84, 2, 99, 64, 0, 74, 66, 70, 67, 65, 3, 8, 6, 1, 4, 39, 72, 68, 77, 13, 35], [59, 126, 46, 121, 63, 111, 116, 110, 58, 123, 45, 109, 104, 54, 56, 60, 117, 40, 122, 57, 113, 94, 101, 37, 61, 62, 88, 36, 24, 114, 125, 127, 27, 119, 48, 83, 118, 19, 49, 47, 115, 21, 52, 22, 30, 120, 124, 76, 78, 85, 100, 51, 43, 86, 33, 12, 17, 14, 89, 80, 53, 55, 42, 81, 95, 108, 91, 92, 32, 16, 28, 97, 23, 50, 73, 31, 87, 38, 9, 90, 7, 102, 93, 15, 25, 105, 71, 112, 96, 79, 44, 107, 41, 69, 10, 18, 29, 67, 75, 84, 74, 11, 34, 3, 82, 20, 6, 106, 103, 26, 66, 64, 98, 0, 2, 5, 70, 8, 99, 1, 13, 68, 4, 39, 77, 72, 65, 35], [59, 126, 63, 121, 46, 111, 110, 116, 58, 123, 45, 109, 104, 117, 122, 54, 40, 113, 56, 61, 60, 125, 94, 62, 57, 37, 48, 114, 101, 24, 127, 88, 118, 36, 124, 119, 49, 21, 115, 47, 19, 52, 78, 83, 120, 30, 33, 22, 27, 76, 95, 12, 100, 50, 80, 28, 17, 85, 53, 14, 102, 43, 42, 16, 51, 81, 97, 86, 89, 73, 55, 91, 23, 25, 92, 108, 41, 32, 90, 7, 106, 38, 74, 9, 71, 29, 107, 66, 96, 79, 15, 10, 18, 6, 31, 87, 11, 34, 69, 93, 82, 20, 2, 0, 5, 75, 105, 64, 98, 26, 112, 84, 70, 67, 8, 3, 44, 13, 99, 4, 68, 1, 65, 72, 103, 77, 39, 35], [59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 109, 45, 117, 104, 54, 40, 122, 60, 118, 56, 61, 113, 88, 57, 62, 114, 94, 48, 36, 101, 127, 37, 124, 24, 49, 19, 125, 21, 78, 83, 119, 30, 12, 22, 52, 115, 27, 76, 80, 17, 14, 47, 81, 16, 95, 120, 85, 86, 28, 100, 91, 33, 89, 50, 43, 102, 55, 108, 97, 51, 71, 53, 73, 32, 23, 90, 42, 7, 92, 10, 9, 11, 15, 38, 25, 41, 107, 82, 112, 105, 20, 87, 31, 74, 29, 18, 79, 96, 106, 69, 75, 5, 6, 0, 93, 84, 34, 3, 99, 67, 64, 1, 44, 98, 2, 26, 66, 65, 8, 72, 4, 70, 103, 13, 77, 39, 68, 35], [59, 126, 46, 63, 121, 111, 116, 110, 58, 45, 123, 109, 54, 117, 104, 40, 60, 122, 94, 24, 57, 88, 113, 48, 56, 49, 36, 101, 37, 114, 62, 61, 27, 124, 119, 125, 127, 19, 118, 83, 22, 78, 21, 12, 30, 55, 17, 115, 14, 80, 47, 50, 81, 76, 85, 52, 86, 120, 100, 108, 89, 95, 28, 53, 43, 91, 16, 33, 51, 42, 92, 97, 23, 25, 31, 38, 79, 32, 90, 44, 105, 15, 73, 112, 82, 9, 102, 29, 71, 87, 93, 74, 41, 7, 10, 96, 84, 69, 18, 34, 11, 107, 20, 75, 106, 26, 103, 3, 6, 99, 98, 67, 39, 5, 72, 0, 8, 2, 66, 65, 1, 70, 64, 13, 77, 35, 68, 4], [59, 126, 46, 63, 121, 111, 116, 110, 58, 123, 45, 109, 104, 54, 60, 40, 117, 114, 94, 122, 57, 48, 127, 37, 36, 61, 113, 49, 47, 56, 62, 101, 50, 21, 24, 88, 12, 27, 119, 125, 124, 22, 118, 78, 19, 14, 76, 83, 30, 52, 80, 97, 53, 95, 17, 28, 81, 55, 16, 120, 86, 115, 89, 100, 51, 73, 85, 9, 108, 112, 92, 43, 7, 25, 90, 71, 33, 91, 102, 10, 15, 87, 44, 79, 23, 107, 32, 31, 69, 29, 82, 42, 38, 18, 20, 64, 96, 6, 74, 11, 0, 106, 34, 93, 84, 75, 26, 41, 5, 66, 99, 67, 105, 2, 70, 1, 3, 98, 72, 39, 103, 65, 68, 4, 35, 13, 8, 77], [59, 126, 46, 121, 63, 111, 116, 110, 58, 123, 45, 109, 117, 104, 54, 40, 60, 114, 113, 94, 122, 61, 56, 57, 36, 49, 37, 88, 24, 101, 48, 125, 127, 118, 62, 119, 27, 19, 21, 124, 30, 78, 22, 12, 83, 17, 47, 55, 50, 52, 14, 16, 115, 76, 89, 80, 100, 51, 95, 85, 86, 53, 81, 97, 91, 28, 120, 108, 102, 33, 92, 43, 23, 107, 31, 9, 38, 42, 25, 32, 7, 90, 44, 41, 71, 112, 73, 93, 29, 15, 11, 87, 105, 74, 82, 10, 106, 18, 20, 79, 84, 34, 96, 69, 6, 98, 5, 99, 75, 0, 3, 26, 72, 70, 66, 39, 67, 64, 103, 2, 35, 65, 68, 4, 8, 1, 13, 77], [59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 45, 109, 117, 54, 60, 104, 40, 127, 56, 57, 61, 122, 94, 48, 37, 36, 114, 113, 118, 88, 124, 101, 49, 24, 62, 83, 30, 27, 19, 78, 52, 115, 119, 21, 12, 47, 125, 22, 81, 55, 100, 120, 17, 16, 95, 97, 50, 85, 80, 89, 14, 86, 43, 53, 76, 51, 102, 108, 91, 33, 107, 112, 90, 42, 28, 41, 32, 73, 23, 7, 9, 71, 31, 25, 10, 74, 29, 92, 96, 105, 34, 15, 44, 11, 106, 87, 18, 75, 20, 82, 98, 79, 69, 99, 93, 38, 67, 0, 84, 64, 3, 26, 5, 6, 70, 103, 66, 72, 2, 68, 35, 4, 65, 1, 8, 13, 39, 77], [59, 126, 46, 63, 121, 116, 111, 110, 58, 45, 123, 109, 104, 117, 60, 40, 54, 122, 49, 56, 24, 101, 57, 114, 94, 37, 88, 61, 27, 125, 48, 36, 127, 83, 30, 19, 113, 22, 62, 47, 85, 78, 21, 119, 118, 120, 12, 55, 52, 81, 16, 100, 14, 124, 86, 43, 17, 95, 76, 80, 89, 115, 97, 50, 33, 42, 9, 108, 51, 91, 28, 53, 112, 90, 31, 32, 23, 92, 73, 25, 41, 71, 44, 15, 107, 87, 74, 102, 106, 82, 20, 79, 38, 75, 7, 29, 93, 105, 96, 10, 34, 98, 18, 11, 84, 26, 72, 69, 99, 70, 5, 0, 3, 103, 64, 35, 67, 2, 4, 68, 39, 65, 66, 1, 77, 13, 6, 8], [59, 126, 46, 63, 121, 111, 116, 110, 58, 45, 123, 109, 104, 117, 60, 54, 40, 94, 56, 57, 127, 114, 101, 24, 122, 113, 36, 37, 118, 88, 119, 49, 27, 19, 48, 62, 125, 83, 21, 22, 61, 78, 12, 30, 55, 124, 47, 52, 86, 14, 95, 50, 120, 100, 115, 17, 81, 80, 85, 16, 97, 42, 76, 89, 33, 28, 91, 43, 44, 23, 53, 112, 31, 102, 51, 108, 9, 90, 32, 38, 25, 92, 20, 41, 107, 73, 82, 71, 106, 87, 18, 93, 105, 7, 79, 15, 84, 29, 74, 11, 96, 34, 26, 10, 98, 64, 69, 5, 67, 75, 103, 70, 2, 0, 66, 72, 99, 3, 35, 77, 1, 39, 6, 8, 65, 4, 68, 13], [59, 126, 46, 63, 121, 111, 116, 110, 58, 45, 123, 109, 117, 49, 104, 56, 36, 122, 57, 94, 118, 61, 54, 119, 114, 40, 60, 101, 127, 88, 125, 113, 37, 48, 24, 83, 27, 52, 62, 124, 30, 120, 19, 85, 115, 43, 55, 47, 50, 22, 17, 21, 14, 102, 51, 53, 81, 95, 108, 12, 100, 78, 33, 76, 38, 80, 112, 16, 86, 42, 97, 89, 32, 41, 91, 28, 107, 31, 105, 93, 25, 92, 106, 23, 90, 87, 82, 29, 73, 7, 96, 71, 99, 18, 44, 69, 20, 34, 0, 84, 9, 79, 11, 26, 10, 103, 70, 67, 74, 66, 98, 5, 64, 3, 35, 15, 75, 2, 39, 1, 68, 6, 8, 65, 77, 72, 4, 13], [59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 45, 109, 117, 104, 54, 40, 60, 114, 56, 94, 57, 49, 127, 101, 37, 36, 122, 24, 118, 113, 61, 88, 27, 22, 48, 125, 119, 124, 115, 30, 19, 83, 62, 52, 21, 14, 95, 47, 120, 81, 100, 53, 89, 12, 51, 17, 97, 50, 78, 55, 108, 33, 76, 86, 28, 85, 102, 16, 43, 91, 112, 80, 90, 25, 42, 31, 73, 92, 23, 93, 107, 106, 105, 41, 38, 44, 32, 9, 87, 96, 29, 20, 7, 82, 18, 34, 79, 10, 11, 5, 71, 98, 69, 15, 74, 84, 26, 70, 99, 103, 75, 64, 2, 0, 66, 72, 67, 6, 35, 3, 39, 8, 77, 65, 68, 4, 1, 13], [59, 126, 46, 63, 121, 111, 116, 110, 58, 123, 45, 109, 117, 104, 60, 54, 40, 56, 113, 114, 48, 94, 122, 118, 57, 49, 125, 36, 127, 124, 24, 88, 37, 61, 101, 62, 19, 52, 22, 27, 83, 119, 47, 21, 30, 78, 55, 12, 95, 81, 115, 76, 16, 14, 120, 100, 17, 89, 86, 108, 80, 97, 85, 50, 33, 51, 28, 42, 91, 53, 102, 9, 112, 90, 43, 23, 105, 41, 29, 32, 20, 7, 107, 25, 73, 92, 31, 106, 71, 38, 93, 74, 87, 18, 79, 10, 5, 44, 96, 82, 69, 75, 11, 98, 70, 64, 3, 34, 84, 66, 15, 26, 6, 67, 0, 2, 68, 99, 103, 39, 72, 77, 8, 4, 13, 65, 1, 35], [59, 126, 46, 63, 121, 111, 116, 110, 58, 123, 45, 109, 117, 104, 113, 94, 60, 40, 54, 114, 56, 118, 48, 125, 122, 57, 88, 49, 101, 37, 36, 24, 119, 61, 52, 124, 127, 62, 21, 27, 83, 30, 19, 22, 100, 33, 47, 43, 115, 55, 78, 50, 81, 17, 12, 95, 108, 85, 80, 120, 89, 53, 14, 102, 16, 97, 86, 91, 76, 28, 106, 90, 32, 51, 42, 112, 38, 31, 92, 107, 87, 23, 105, 9, 93, 7, 71, 96, 25, 20, 74, 11, 82, 29, 18, 73, 79, 34, 10, 41, 15, 99, 75, 26, 67, 6, 103, 0, 2, 84, 98, 69, 64, 44, 3, 5, 66, 35, 8, 72, 39, 70, 1, 77, 65, 68, 13, 4], [59, 126, 46, 63, 121, 116, 110, 111, 58, 45, 123, 109, 104, 40, 54, 117, 60, 56, 94, 113, 125, 122, 61, 37, 118, 57, 48, 24, 101, 119, 36, 88, 62, 49, 114, 22, 30, 127, 124, 83, 19, 100, 52, 27, 85, 78, 21, 33, 47, 115, 12, 43, 17, 91, 95, 120, 108, 50, 16, 97, 14, 76, 80, 28, 89, 53, 55, 81, 102, 86, 51, 32, 42, 106, 112, 38, 92, 23, 31, 105, 87, 107, 9, 96, 93, 90, 41, 73, 25, 71, 7, 79, 20, 34, 26, 74, 18, 29, 44, 10, 82, 99, 84, 15, 11, 75, 6, 5, 64, 0, 103, 69, 98, 67, 2, 1, 39, 66, 8, 72, 4, 3, 70, 13, 65, 77, 35, 68], [59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 45, 109, 104, 117, 56, 54, 60, 40, 113, 94, 36, 49, 57, 48, 122, 118, 125, 24, 114, 62, 37, 127, 101, 119, 88, 61, 22, 83, 124, 76, 47, 30, 27, 120, 52, 19, 97, 21, 95, 78, 100, 33, 85, 115, 80, 28, 14, 81, 55, 91, 16, 17, 50, 53, 86, 43, 12, 102, 108, 89, 73, 42, 23, 51, 107, 92, 31, 25, 90, 7, 32, 9, 112, 44, 93, 105, 41, 71, 79, 87, 38, 10, 29, 96, 82, 20, 74, 18, 26, 5, 6, 34, 84, 99, 15, 69, 67, 75, 103, 106, 11, 3, 66, 0, 98, 8, 2, 64, 70, 39, 68, 4, 65, 72, 1, 13, 35, 77], [59, 126, 46, 63, 121, 111, 116, 110, 58, 123, 45, 109, 104, 54, 117, 40, 94, 56, 62, 122, 57, 114, 60, 125, 101, 36, 48, 118, 24, 113, 37, 61, 88, 49, 127, 119, 21, 124, 83, 27, 19, 22, 86, 30, 52, 100, 120, 81, 115, 47, 78, 14, 17, 76, 85, 50, 91, 33, 55, 42, 32, 89, 102, 16, 53, 43, 95, 12, 51, 80, 97, 108, 41, 107, 92, 23, 28, 112, 90, 87, 25, 31, 73, 7, 38, 71, 29, 93, 20, 82, 74, 9, 106, 96, 105, 44, 75, 26, 18, 99, 34, 79, 10, 84, 15, 5, 69, 3, 98, 6, 11, 8, 64, 39, 70, 103, 66, 67, 0, 2, 35, 68, 1, 13, 65, 77, 4, 72], [59, 126, 46, 63, 121, 111, 116, 110, 58, 45, 123, 109, 104, 117, 40, 54, 94, 56, 57, 125, 60, 122, 113, 36, 24, 101, 62, 37, 88, 114, 49, 48, 119, 19, 118, 127, 22, 27, 83, 21, 86, 47, 100, 30, 124, 61, 95, 52, 115, 33, 17, 43, 78, 76, 55, 120, 85, 14, 81, 80, 12, 91, 97, 16, 108, 89, 42, 53, 102, 28, 51, 92, 41, 50, 90, 23, 20, 31, 9, 25, 32, 38, 87, 107, 7, 29, 105, 112, 82, 71, 11, 106, 79, 44, 73, 93, 96, 74, 18, 10, 69, 15, 84, 98, 5, 34, 75, 26, 2, 99, 0, 70, 64, 103, 6, 39, 67, 8, 66, 13, 3, 35, 72, 77, 68, 1, 65, 4], [59, 126, 46, 121, 63, 111, 116, 110, 58, 45, 123, 109, 104, 54, 40, 125, 117, 113, 122, 94, 56, 57, 62, 101, 37, 119, 60, 36, 88, 24, 61, 114, 118, 49, 48, 22, 127, 19, 21, 47, 30, 27, 124, 83, 52, 55, 115, 43, 17, 33, 50, 100, 86, 78, 12, 108, 95, 28, 76, 14, 80, 51, 85, 97, 89, 53, 91, 81, 120, 42, 16, 112, 102, 90, 23, 38, 31, 106, 92, 87, 32, 71, 93, 9, 41, 29, 7, 73, 79, 105, 25, 44, 74, 84, 18, 75, 96, 34, 20, 82, 5, 26, 15, 67, 10, 64, 2, 0, 66, 69, 107, 11, 103, 98, 3, 70, 99, 39, 6, 8, 35, 68, 4, 1, 65, 72, 13, 77], [59, 126, 46, 63, 121, 111, 116, 110, 58, 123, 45, 109, 104, 117, 54, 40, 57, 94, 122, 36, 118, 37, 113, 56, 60, 114, 61, 125, 62, 48, 101, 88, 24, 49, 47, 119, 124, 83, 30, 19, 27, 22, 50, 52, 21, 76, 17, 127, 78, 95, 120, 85, 12, 28, 55, 100, 81, 115, 89, 14, 43, 33, 86, 80, 53, 97, 16, 108, 9, 91, 51, 102, 73, 32, 23, 7, 71, 92, 90, 25, 87, 42, 75, 107, 41, 106, 93, 29, 64, 38, 5, 18, 74, 112, 96, 10, 15, 31, 70, 2, 105, 82, 66, 44, 11, 3, 79, 20, 0, 84, 69, 34, 67, 26, 99, 65, 6, 1, 39, 8, 98, 103, 72, 68, 77, 4, 13, 35], [59, 126, 46, 63, 121, 111, 116, 110, 58, 45, 123, 109, 117, 54, 104, 40, 61, 60, 57, 56, 113, 118, 122, 101, 37, 48, 36, 94, 62, 125, 114, 127, 88, 24, 47, 119, 21, 49, 83, 124, 22, 19, 30, 120, 27, 52, 76, 89, 53, 95, 78, 100, 115, 51, 33, 43, 55, 12, 50, 17, 102, 28, 81, 80, 108, 42, 86, 85, 31, 90, 97, 14, 32, 16, 23, 91, 87, 38, 93, 73, 112, 7, 92, 96, 41, 9, 71, 106, 25, 44, 20, 15, 107, 105, 79, 5, 26, 34, 18, 70, 99, 10, 29, 75, 84, 82, 0, 69, 3, 74, 64, 11, 66, 67, 2, 98, 1, 65, 103, 8, 39, 68, 6, 35, 72, 4, 77, 13], [59, 126, 46, 63, 121, 116, 111, 110, 58, 123, 45, 109, 117, 104, 40, 54, 60, 36, 56, 57, 113, 118, 61, 88, 94, 37, 101, 24, 122, 47, 62, 83, 125, 49, 114, 127, 27, 22, 48, 119, 21, 124, 30, 76, 19, 78, 95, 17, 52, 80, 43, 12, 85, 14, 115, 53, 108, 50, 55, 100, 16, 81, 33, 28, 120, 42, 89, 73, 51, 23, 32, 31, 9, 102, 86, 87, 90, 91, 97, 44, 71, 38, 92, 112, 25, 7, 106, 74, 93, 107, 34, 79, 29, 96, 41, 10, 67, 3, 64, 20, 5, 70, 18, 15, 11, 69, 75, 82, 84, 26, 103, 105, 0, 66, 65, 6, 99, 98, 72, 2, 4, 1, 39, 8, 13, 68, 77, 35], [59, 126, 46, 63, 121, 111, 116, 110, 58, 123, 45, 109, 117, 104, 54, 40, 60, 122, 57, 118, 113, 127, 94, 61, 56, 37, 101, 48, 36, 88, 125, 114, 24, 27, 119, 62, 124, 83, 21, 47, 49, 19, 22, 50, 76, 89, 30, 55, 14, 100, 78, 86, 120, 12, 81, 85, 42, 52, 115, 95, 53, 80, 33, 17, 16, 108, 51, 87, 112, 32, 102, 28, 97, 90, 43, 31, 23, 9, 107, 91, 92, 38, 71, 73, 41, 7, 20, 79, 18, 105, 15, 82, 74, 44, 25, 96, 11, 75, 10, 26, 34, 5, 29, 93, 99, 64, 106, 3, 98, 70, 69, 103, 84, 67, 2, 0, 66, 6, 8, 1, 72, 65, 39, 4, 13, 68, 35, 77], [59, 126, 46, 63, 121, 111, 116, 110, 58, 45, 123, 109, 117, 104, 54, 60, 40, 94, 122, 113, 127, 57, 88, 56, 36, 61, 37, 125, 48, 27, 24, 118, 101, 76, 62, 114, 22, 120, 83, 124, 119, 19, 49, 80, 30, 47, 85, 115, 14, 12, 50, 86, 55, 21, 52, 81, 17, 100, 78, 89, 95, 16, 28, 33, 108, 97, 91, 51, 42, 9, 73, 53, 90, 32, 43, 102, 87, 23, 112, 7, 71, 31, 92, 79, 107, 82, 74, 38, 44, 15, 41, 29, 10, 25, 20, 93, 69, 75, 18, 84, 3, 26, 11, 96, 105, 106, 5, 99, 34, 6, 70, 98, 0, 39, 67, 72, 64, 2, 66, 65, 1, 4, 103, 13, 8, 35, 68, 77]], "model.layers.12.self_attn.q_proj": [[102, 127, 49, 95, 21, 24, 16, 83, 80, 82, 88, 77, 28, 6, 51, 99, 44, 37, 38, 72, 2, 31, 74, 27, 106, 22, 75, 86, 13, 53, 42, 108, 26, 98, 112, 10, 93, 96, 18, 43, 62, 20, 94, 103, 91, 79, 4, 117, 90, 61, 85, 78, 29, 101, 104, 14, 40, 19, 25, 81, 11, 32, 92, 64, 89, 60, 17, 30, 36, 56, 12, 109, 125, 9, 113, 48, 111, 52, 116, 70, 87, 41, 115, 105, 84, 71, 76, 33, 15, 55, 114, 65, 8, 73, 118, 100, 45, 97, 54, 107, 34, 68, 39, 69, 0, 123, 35, 5, 66, 7, 110, 67, 58, 122, 126, 1, 47, 121, 59, 23, 120, 50, 57, 46, 119, 63, 124, 3], [127, 102, 49, 95, 51, 108, 24, 31, 26, 38, 98, 19, 83, 21, 96, 61, 106, 44, 42, 116, 97, 111, 46, 43, 109, 28, 125, 105, 29, 55, 115, 114, 93, 62, 112, 53, 40, 37, 52, 41, 126, 101, 22, 27, 54, 60, 121, 58, 45, 103, 107, 110, 117, 82, 123, 120, 104, 56, 113, 36, 124, 48, 119, 57, 50, 59, 32, 118, 94, 122, 20, 100, 88, 47, 92, 91, 13, 33, 99, 39, 84, 34, 86, 90, 85, 63, 15, 25, 77, 17, 30, 35, 79, 18, 80, 87, 89, 23, 81, 16, 14, 6, 78, 72, 12, 70, 76, 74, 64, 8, 4, 75, 11, 10, 73, 71, 9, 65, 2, 68, 66, 67, 1, 7, 0, 69, 3, 5], [102, 49, 127, 95, 21, 108, 82, 88, 77, 83, 24, 51, 96, 44, 86, 98, 99, 80, 113, 37, 28, 27, 6, 16, 31, 74, 117, 26, 42, 90, 72, 2, 62, 19, 46, 41, 53, 105, 116, 79, 8, 94, 22, 119, 25, 18, 121, 112, 85, 20, 61, 14, 38, 101, 109, 65, 29, 93, 68, 92, 12, 104, 36, 76, 43, 81, 115, 64, 10, 55, 125, 111, 56, 23, 87, 45, 91, 40, 126, 58, 107, 120, 13, 100, 59, 50, 52, 60, 97, 122, 39, 30, 4, 32, 110, 54, 71, 103, 78, 123, 34, 118, 17, 57, 33, 114, 106, 124, 11, 47, 35, 84, 63, 69, 89, 3, 48, 70, 15, 75, 7, 5, 73, 9, 0, 67, 1, 66], [102, 49, 127, 95, 21, 24, 83, 28, 82, 80, 77, 11, 99, 38, 88, 31, 96, 20, 74, 90, 42, 53, 51, 26, 113, 35, 86, 60, 105, 14, 70, 6, 68, 108, 44, 15, 22, 18, 16, 75, 98, 101, 97, 56, 79, 58, 27, 37, 62, 2, 114, 29, 85, 25, 103, 81, 19, 8, 43, 106, 110, 112, 87, 92, 9, 116, 61, 117, 5, 12, 67, 89, 10, 17, 34, 109, 30, 120, 111, 55, 13, 78, 73, 76, 36, 3, 94, 48, 46, 0, 121, 124, 119, 23, 115, 93, 39, 91, 107, 50, 122, 71, 100, 41, 123, 72, 33, 126, 65, 59, 1, 57, 125, 47, 52, 69, 104, 45, 118, 54, 32, 40, 7, 84, 64, 4, 63, 66], [45, 101, 109, 26, 28, 21, 24, 83, 88, 62, 31, 78, 95, 97, 72, 82, 124, 75, 87, 113, 4, 122, 30, 49, 44, 121, 53, 93, 94, 27, 98, 51, 16, 116, 85, 80, 110, 77, 66, 57, 111, 69, 102, 0, 18, 25, 86, 105, 106, 14, 63, 92, 68, 36, 115, 119, 100, 107, 118, 19, 114, 23, 76, 35, 34, 79, 61, 81, 15, 59, 41, 33, 47, 103, 9, 52, 91, 60, 112, 46, 120, 1, 8, 117, 39, 50, 13, 67, 104, 125, 48, 127, 10, 89, 40, 58, 32, 70, 71, 22, 56, 90, 20, 38, 11, 123, 54, 55, 42, 17, 29, 43, 108, 12, 37, 126, 99, 84, 96, 74, 6, 2, 73, 5, 7, 65, 3, 64], [45, 101, 109, 28, 62, 21, 26, 24, 31, 88, 83, 87, 97, 78, 122, 124, 75, 95, 16, 107, 53, 72, 113, 82, 98, 49, 116, 94, 17, 25, 91, 85, 47, 93, 121, 76, 102, 30, 77, 86, 34, 57, 106, 51, 59, 6, 103, 112, 22, 108, 44, 117, 36, 84, 29, 33, 10, 61, 52, 74, 63, 39, 118, 60, 27, 43, 12, 13, 18, 71, 123, 96, 48, 114, 23, 110, 105, 79, 41, 67, 92, 115, 42, 81, 111, 56, 66, 80, 69, 8, 58, 55, 50, 127, 14, 125, 0, 126, 120, 100, 35, 20, 119, 40, 11, 104, 54, 89, 68, 4, 46, 19, 32, 99, 38, 70, 37, 5, 7, 90, 73, 9, 1, 15, 2, 65, 3, 64], [45, 101, 109, 28, 26, 122, 21, 24, 83, 87, 88, 31, 97, 95, 78, 121, 62, 124, 113, 30, 75, 51, 53, 82, 116, 93, 86, 16, 110, 102, 98, 25, 36, 63, 106, 72, 59, 34, 79, 49, 112, 57, 35, 107, 47, 111, 38, 44, 52, 58, 105, 29, 39, 100, 27, 118, 61, 125, 123, 114, 115, 55, 41, 40, 48, 37, 119, 15, 92, 94, 50, 85, 6, 13, 46, 60, 103, 108, 127, 33, 99, 126, 74, 42, 12, 32, 104, 54, 20, 0, 84, 22, 14, 117, 56, 43, 120, 80, 18, 19, 90, 91, 23, 96, 8, 11, 68, 17, 89, 81, 69, 77, 76, 10, 67, 66, 9, 5, 70, 71, 73, 3, 1, 7, 4, 65, 2, 64], [45, 101, 109, 26, 28, 24, 88, 83, 21, 82, 93, 87, 121, 62, 95, 75, 78, 31, 124, 122, 16, 97, 98, 116, 30, 113, 34, 51, 53, 57, 77, 110, 49, 63, 107, 61, 125, 115, 18, 112, 41, 105, 22, 106, 94, 103, 71, 36, 111, 92, 86, 11, 118, 29, 37, 85, 40, 59, 9, 27, 67, 47, 50, 4, 127, 33, 100, 108, 44, 19, 117, 123, 32, 35, 126, 1, 60, 0, 48, 80, 69, 46, 23, 42, 84, 120, 55, 6, 79, 52, 91, 20, 81, 14, 72, 58, 68, 54, 5, 56, 8, 114, 13, 43, 66, 74, 102, 39, 90, 99, 15, 12, 119, 104, 76, 96, 17, 65, 25, 38, 89, 2, 73, 10, 70, 7, 3, 64], [43, 97, 107, 24, 91, 26, 31, 22, 83, 81, 100, 85, 103, 75, 33, 87, 30, 20, 50, 15, 88, 110, 13, 7, 114, 90, 49, 127, 95, 66, 93, 86, 84, 12, 94, 19, 102, 92, 98, 61, 113, 48, 70, 18, 35, 0, 115, 40, 118, 25, 38, 54, 1, 63, 36, 16, 80, 28, 117, 56, 58, 27, 101, 77, 42, 51, 99, 82, 32, 34, 120, 29, 55, 125, 71, 96, 4, 89, 21, 79, 17, 6, 39, 47, 112, 41, 111, 116, 8, 119, 121, 123, 23, 106, 45, 62, 52, 3, 122, 53, 46, 11, 57, 37, 105, 73, 67, 10, 9, 109, 74, 126, 108, 78, 44, 104, 14, 76, 60, 59, 124, 72, 69, 68, 5, 2, 64, 65], [43, 31, 97, 107, 22, 13, 24, 26, 15, 91, 83, 20, 85, 33, 9, 115, 100, 110, 75, 73, 18, 81, 86, 84, 114, 50, 90, 70, 68, 54, 36, 95, 72, 25, 17, 79, 27, 77, 88, 63, 121, 19, 71, 42, 111, 61, 12, 127, 76, 23, 82, 93, 74, 87, 16, 66, 49, 65, 8, 38, 122, 46, 48, 10, 7, 103, 112, 14, 101, 47, 78, 117, 69, 34, 113, 67, 55, 118, 80, 29, 21, 116, 126, 123, 92, 124, 56, 89, 28, 96, 106, 6, 1, 108, 120, 40, 102, 94, 30, 98, 11, 105, 45, 35, 125, 51, 39, 32, 5, 62, 58, 41, 4, 37, 57, 60, 64, 59, 52, 99, 44, 109, 119, 104, 0, 53, 3, 2], [43, 107, 31, 97, 26, 106, 42, 20, 113, 61, 115, 125, 50, 33, 24, 48, 127, 117, 11, 38, 114, 121, 36, 100, 49, 126, 95, 54, 63, 116, 110, 111, 56, 57, 118, 55, 123, 46, 47, 60, 58, 91, 103, 77, 108, 39, 119, 109, 101, 6, 51, 124, 45, 44, 59, 122, 7, 120, 62, 35, 85, 112, 40, 53, 41, 52, 30, 73, 90, 92, 105, 22, 37, 17, 81, 104, 16, 10, 99, 96, 3, 5, 84, 75, 1, 32, 21, 98, 79, 34, 19, 102, 93, 29, 87, 94, 4, 72, 28, 23, 25, 88, 89, 27, 0, 83, 82, 86, 18, 78, 80, 2, 71, 15, 9, 14, 12, 76, 13, 66, 8, 64, 70, 68, 65, 67, 69, 74], [43, 97, 31, 107, 24, 26, 91, 22, 54, 81, 42, 20, 115, 83, 75, 33, 13, 50, 100, 15, 18, 85, 48, 61, 84, 120, 66, 114, 110, 93, 90, 6, 57, 70, 127, 30, 113, 12, 49, 63, 103, 118, 95, 80, 19, 125, 121, 88, 86, 56, 9, 116, 77, 47, 0, 46, 1, 117, 71, 60, 36, 28, 41, 111, 45, 82, 112, 55, 123, 108, 106, 124, 89, 126, 92, 58, 14, 8, 38, 51, 29, 40, 76, 11, 109, 87, 23, 119, 122, 94, 34, 17, 73, 27, 79, 64, 25, 101, 99, 67, 44, 3, 39, 35, 68, 21, 32, 98, 52, 37, 96, 105, 53, 65, 7, 16, 69, 62, 78, 104, 74, 102, 59, 10, 72, 5, 4, 2], [38, 64, 113, 97, 93, 49, 57, 1, 6, 78, 67, 16, 70, 3, 82, 73, 65, 11, 4, 77, 23, 12, 90, 2, 31, 14, 72, 19, 8, 94, 121, 126, 66, 75, 89, 62, 71, 59, 20, 13, 84, 9, 122, 86, 18, 103, 22, 87, 37, 83, 125, 53, 69, 7, 21, 80, 118, 27, 42, 39, 54, 10, 101, 44, 76, 33, 5, 91, 58, 88, 74, 26, 17, 29, 40, 0, 30, 68, 32, 119, 79, 25, 50, 63, 96, 85, 45, 35, 15, 60, 99, 36, 95, 108, 123, 92, 81, 124, 48, 109, 100, 116, 120, 61, 111, 114, 51, 24, 115, 106, 46, 56, 117, 127, 52, 47, 102, 34, 104, 98, 28, 43, 55, 41, 105, 110, 112, 107], [38, 57, 113, 97, 49, 93, 23, 82, 78, 16, 73, 88, 11, 21, 12, 20, 25, 102, 6, 29, 5, 39, 94, 77, 72, 9, 26, 83, 37, 31, 32, 68, 27, 13, 125, 28, 90, 30, 92, 63, 71, 121, 106, 85, 19, 34, 101, 44, 22, 91, 62, 8, 65, 15, 80, 4, 74, 75, 81, 84, 59, 79, 87, 14, 17, 76, 18, 24, 122, 66, 7, 89, 33, 105, 10, 2, 95, 109, 119, 3, 108, 60, 36, 96, 42, 86, 45, 35, 98, 41, 70, 54, 100, 126, 52, 40, 99, 55, 50, 47, 51, 58, 56, 69, 53, 103, 118, 120, 61, 107, 115, 123, 112, 43, 46, 116, 104, 114, 48, 110, 127, 64, 111, 124, 67, 1, 117, 0], [113, 101, 57, 38, 62, 49, 97, 123, 126, 121, 119, 53, 52, 56, 102, 59, 60, 114, 127, 118, 125, 26, 116, 117, 51, 106, 58, 46, 100, 103, 124, 63, 44, 61, 31, 48, 111, 45, 50, 112, 120, 47, 90, 115, 109, 54, 122, 55, 110, 107, 108, 105, 40, 22, 41, 39, 32, 21, 43, 84, 104, 42, 28, 93, 94, 37, 99, 25, 36, 81, 95, 34, 30, 89, 86, 35, 85, 17, 98, 23, 19, 27, 88, 83, 96, 91, 29, 20, 76, 24, 33, 92, 79, 87, 4, 74, 75, 8, 14, 82, 9, 5, 72, 80, 13, 70, 15, 12, 18, 10, 71, 2, 73, 77, 66, 7, 16, 6, 67, 69, 65, 64, 68, 1, 11, 78, 3, 0], [38, 57, 97, 23, 93, 49, 113, 82, 29, 78, 16, 25, 11, 102, 59, 31, 62, 90, 85, 33, 87, 26, 21, 39, 83, 126, 77, 19, 27, 71, 88, 80, 18, 94, 13, 125, 101, 95, 89, 28, 15, 20, 22, 121, 79, 96, 30, 100, 75, 32, 92, 91, 119, 61, 37, 84, 68, 14, 86, 53, 98, 127, 81, 24, 35, 58, 47, 17, 44, 36, 8, 108, 34, 76, 63, 10, 99, 50, 43, 115, 40, 118, 48, 110, 12, 103, 74, 55, 106, 7, 72, 111, 114, 60, 109, 122, 123, 51, 46, 54, 42, 4, 41, 116, 56, 104, 66, 112, 117, 52, 105, 5, 2, 120, 124, 70, 69, 6, 45, 107, 67, 73, 9, 1, 0, 3, 65, 64], [39, 48, 62, 112, 29, 21, 14, 80, 61, 81, 76, 11, 67, 87, 73, 7, 56, 84, 5, 93, 24, 27, 69, 71, 75, 1, 109, 12, 0, 25, 41, 2, 96, 118, 6, 89, 116, 17, 9, 79, 117, 85, 66, 52, 4, 13, 16, 86, 54, 94, 32, 82, 101, 18, 78, 50, 95, 113, 119, 15, 19, 120, 34, 126, 83, 57, 23, 77, 26, 49, 106, 31, 45, 127, 55, 42, 107, 122, 36, 111, 28, 88, 44, 3, 22, 104, 40, 63, 97, 115, 121, 102, 35, 59, 33, 72, 108, 30, 114, 125, 60, 92, 124, 105, 8, 123, 58, 99, 110, 74, 53, 43, 51, 37, 100, 90, 38, 46, 98, 20, 47, 65, 91, 70, 10, 68, 103, 64], [39, 62, 48, 112, 87, 61, 29, 21, 81, 14, 73, 80, 11, 76, 69, 56, 24, 93, 41, 71, 77, 79, 118, 3, 86, 27, 111, 18, 9, 96, 32, 94, 109, 127, 125, 50, 101, 126, 2, 92, 37, 70, 95, 33, 6, 54, 117, 82, 85, 121, 5, 36, 28, 1, 66, 113, 22, 78, 0, 123, 74, 20, 17, 97, 91, 49, 116, 59, 120, 31, 122, 89, 99, 107, 72, 23, 119, 52, 16, 45, 84, 115, 108, 51, 43, 55, 38, 53, 40, 106, 60, 124, 12, 57, 13, 104, 19, 114, 83, 34, 102, 30, 110, 75, 35, 15, 88, 42, 58, 44, 67, 10, 100, 105, 46, 63, 26, 98, 25, 47, 90, 8, 65, 7, 68, 103, 64, 4], [39, 62, 48, 112, 87, 29, 61, 11, 25, 80, 81, 21, 14, 73, 69, 76, 5, 93, 66, 2, 71, 0, 77, 18, 67, 118, 27, 3, 1, 24, 94, 33, 117, 72, 109, 75, 125, 111, 86, 56, 92, 32, 96, 31, 41, 82, 74, 106, 50, 7, 116, 95, 37, 79, 113, 85, 28, 30, 19, 97, 70, 89, 83, 36, 122, 63, 78, 17, 123, 121, 126, 34, 120, 43, 88, 16, 38, 35, 54, 6, 12, 40, 59, 65, 26, 45, 91, 10, 9, 90, 23, 99, 127, 84, 51, 104, 52, 46, 114, 58, 98, 119, 49, 102, 20, 103, 100, 44, 124, 110, 108, 57, 101, 13, 115, 8, 4, 68, 55, 53, 15, 42, 105, 107, 60, 47, 22, 64], [39, 62, 48, 112, 71, 29, 14, 80, 73, 81, 21, 76, 11, 3, 69, 2, 61, 65, 1, 67, 56, 109, 118, 0, 72, 64, 7, 4, 13, 24, 12, 85, 8, 37, 79, 117, 17, 96, 111, 121, 87, 77, 66, 78, 93, 95, 15, 84, 5, 16, 54, 126, 75, 68, 50, 19, 83, 99, 33, 120, 53, 123, 94, 10, 116, 124, 88, 9, 103, 18, 101, 127, 106, 41, 92, 86, 28, 89, 82, 74, 38, 35, 40, 31, 57, 27, 30, 51, 20, 63, 26, 52, 70, 45, 115, 43, 36, 22, 97, 91, 107, 119, 32, 125, 34, 102, 6, 104, 105, 23, 25, 60, 110, 113, 46, 122, 100, 90, 44, 59, 98, 49, 42, 55, 108, 114, 47, 58], [105, 58, 34, 109, 26, 41, 86, 104, 88, 18, 84, 126, 24, 30, 55, 78, 92, 16, 73, 77, 36, 69, 19, 32, 49, 87, 71, 95, 91, 75, 119, 67, 121, 79, 111, 28, 45, 48, 1, 44, 2, 120, 51, 5, 94, 15, 29, 98, 114, 31, 12, 25, 40, 50, 57, 82, 112, 60, 33, 113, 64, 124, 107, 14, 125, 74, 10, 8, 123, 63, 108, 13, 37, 127, 101, 66, 62, 0, 22, 115, 81, 6, 39, 38, 118, 122, 65, 61, 72, 100, 3, 59, 47, 93, 7, 52, 90, 4, 23, 102, 46, 11, 56, 97, 42, 106, 21, 43, 76, 103, 70, 17, 99, 83, 68, 116, 9, 110, 20, 80, 54, 117, 27, 35, 53, 85, 89, 96], [105, 34, 26, 41, 58, 86, 36, 121, 84, 32, 104, 16, 126, 92, 62, 18, 51, 88, 118, 24, 78, 47, 57, 25, 52, 50, 49, 119, 116, 48, 109, 28, 102, 30, 90, 100, 114, 107, 95, 120, 23, 39, 46, 122, 55, 21, 61, 75, 113, 31, 38, 103, 127, 42, 37, 43, 83, 63, 11, 60, 80, 20, 76, 97, 112, 117, 14, 77, 108, 87, 125, 56, 45, 111, 7, 101, 22, 93, 96, 94, 19, 82, 123, 106, 115, 9, 59, 40, 33, 29, 91, 8, 124, 110, 98, 27, 44, 74, 72, 35, 85, 99, 79, 53, 81, 6, 89, 12, 54, 73, 71, 17, 15, 68, 13, 5, 10, 66, 4, 3, 70, 65, 67, 1, 64, 69, 2, 0], [105, 34, 58, 84, 26, 41, 86, 126, 77, 109, 16, 104, 73, 121, 32, 88, 18, 78, 75, 71, 30, 36, 49, 70, 108, 98, 48, 3, 29, 4, 51, 81, 24, 55, 120, 118, 11, 114, 50, 62, 111, 122, 100, 39, 2, 60, 94, 45, 102, 82, 80, 85, 52, 69, 115, 101, 37, 79, 116, 57, 65, 119, 44, 28, 125, 90, 0, 23, 19, 113, 92, 43, 38, 20, 112, 61, 83, 63, 13, 21, 124, 14, 22, 12, 95, 17, 68, 127, 117, 6, 123, 87, 46, 31, 103, 9, 99, 59, 47, 96, 76, 15, 27, 40, 67, 74, 106, 97, 25, 72, 42, 1, 35, 56, 93, 110, 53, 33, 54, 10, 89, 107, 91, 5, 8, 7, 64, 66], [105, 34, 92, 26, 41, 84, 18, 86, 109, 104, 16, 24, 126, 121, 78, 75, 77, 57, 71, 101, 58, 60, 73, 51, 36, 52, 120, 90, 118, 122, 107, 55, 62, 48, 49, 66, 45, 32, 95, 44, 30, 119, 108, 116, 14, 111, 102, 28, 29, 37, 63, 88, 50, 125, 46, 47, 7, 124, 114, 113, 106, 20, 127, 112, 40, 94, 5, 100, 87, 25, 123, 11, 31, 4, 42, 98, 82, 56, 38, 43, 17, 39, 19, 115, 103, 54, 21, 59, 91, 33, 8, 110, 35, 6, 97, 85, 83, 61, 80, 81, 117, 76, 27, 22, 12, 93, 53, 23, 96, 15, 79, 89, 99, 72, 2, 65, 64, 0, 68, 13, 69, 9, 3, 74, 1, 10, 70, 67], [104, 34, 20, 22, 124, 89, 92, 79, 95, 49, 18, 75, 119, 25, 77, 62, 48, 17, 114, 40, 120, 107, 6, 118, 46, 121, 117, 58, 54, 111, 70, 96, 93, 50, 45, 61, 53, 47, 9, 100, 84, 59, 43, 109, 68, 94, 15, 41, 110, 12, 82, 86, 19, 11, 90, 72, 55, 81, 66, 122, 98, 52, 1, 60, 123, 33, 99, 23, 126, 8, 76, 4, 56, 102, 64, 103, 3, 2, 112, 88, 37, 16, 113, 97, 73, 27, 80, 38, 78, 21, 32, 13, 24, 30, 28, 87, 91, 108, 29, 106, 105, 101, 44, 26, 35, 74, 0, 83, 85, 31, 51, 67, 10, 63, 57, 127, 42, 36, 39, 125, 115, 116, 7, 14, 5, 71, 69, 65], [104, 34, 89, 22, 49, 20, 124, 95, 18, 79, 48, 62, 119, 77, 25, 40, 118, 92, 75, 107, 31, 121, 46, 120, 114, 54, 9, 41, 6, 17, 111, 16, 82, 117, 58, 12, 15, 97, 84, 60, 38, 56, 93, 45, 115, 81, 53, 112, 101, 28, 76, 2, 86, 94, 68, 103, 110, 63, 98, 36, 55, 47, 70, 4, 96, 72, 61, 90, 85, 52, 88, 100, 127, 102, 73, 21, 113, 1, 123, 78, 50, 109, 59, 126, 24, 10, 0, 64, 91, 26, 23, 13, 87, 11, 19, 32, 99, 105, 5, 125, 108, 3, 66, 80, 30, 106, 57, 37, 67, 35, 33, 39, 43, 29, 69, 51, 83, 122, 8, 7, 27, 116, 74, 42, 14, 44, 71, 65], [104, 34, 22, 124, 89, 18, 92, 20, 48, 79, 62, 49, 119, 77, 95, 118, 25, 75, 40, 114, 17, 96, 46, 12, 9, 82, 121, 50, 111, 54, 58, 61, 117, 47, 84, 41, 120, 100, 4, 56, 107, 6, 110, 13, 59, 45, 98, 38, 15, 86, 81, 94, 52, 37, 55, 53, 108, 43, 35, 68, 63, 1, 122, 78, 112, 90, 101, 97, 126, 109, 33, 93, 87, 71, 70, 76, 19, 28, 60, 7, 72, 88, 123, 125, 39, 51, 23, 73, 31, 115, 127, 74, 26, 113, 105, 103, 2, 8, 21, 80, 91, 99, 14, 11, 3, 30, 85, 42, 116, 106, 16, 24, 10, 29, 102, 36, 44, 27, 32, 66, 57, 65, 67, 83, 64, 0, 5, 69], [104, 34, 119, 89, 22, 92, 18, 49, 20, 124, 95, 62, 48, 79, 77, 46, 25, 40, 114, 75, 121, 118, 120, 109, 41, 17, 54, 97, 12, 50, 111, 93, 9, 58, 16, 98, 82, 31, 112, 84, 4, 59, 47, 100, 117, 107, 80, 45, 61, 33, 53, 103, 108, 56, 38, 94, 21, 60, 32, 43, 24, 86, 90, 123, 55, 52, 127, 81, 126, 122, 113, 76, 96, 51, 110, 19, 35, 15, 63, 99, 105, 115, 28, 102, 85, 13, 87, 39, 44, 88, 125, 91, 37, 83, 27, 6, 101, 26, 106, 29, 78, 42, 23, 30, 1, 0, 2, 73, 72, 116, 36, 68, 69, 74, 57, 10, 70, 5, 7, 14, 71, 11, 8, 67, 65, 3, 66, 64], [41, 34, 53, 88, 93, 105, 20, 17, 79, 22, 35, 77, 72, 29, 75, 48, 25, 84, 115, 6, 66, 91, 18, 27, 82, 125, 24, 51, 55, 58, 62, 127, 126, 50, 8, 89, 63, 61, 108, 68, 2, 81, 15, 124, 13, 38, 0, 4, 16, 117, 80, 43, 114, 70, 52, 11, 19, 57, 103, 44, 60, 78, 100, 47, 90, 59, 76, 73, 98, 42, 109, 46, 26, 49, 83, 23, 86, 85, 40, 31, 112, 92, 33, 96, 28, 12, 10, 113, 14, 116, 87, 45, 21, 107, 39, 118, 101, 37, 122, 121, 64, 102, 97, 9, 36, 94, 104, 106, 119, 30, 123, 110, 7, 32, 74, 95, 99, 56, 69, 120, 71, 54, 5, 1, 111, 3, 65, 67], [41, 34, 53, 22, 105, 20, 88, 93, 48, 17, 29, 79, 50, 77, 44, 126, 62, 127, 27, 108, 35, 115, 58, 84, 80, 60, 96, 117, 43, 38, 125, 72, 112, 114, 75, 31, 49, 57, 124, 86, 24, 101, 21, 15, 118, 55, 61, 54, 123, 52, 63, 99, 26, 121, 90, 91, 8, 51, 37, 82, 103, 30, 39, 56, 28, 106, 122, 87, 6, 92, 18, 25, 47, 119, 81, 113, 40, 36, 120, 111, 70, 32, 102, 83, 110, 46, 16, 95, 33, 107, 85, 45, 23, 100, 89, 94, 97, 66, 104, 42, 71, 116, 12, 19, 73, 59, 109, 13, 78, 0, 14, 98, 10, 11, 7, 76, 9, 74, 67, 5, 68, 4, 2, 3, 69, 64, 65, 1], [41, 34, 53, 105, 22, 93, 88, 20, 27, 66, 91, 0, 79, 77, 17, 68, 127, 75, 19, 82, 72, 58, 112, 2, 67, 43, 124, 115, 48, 126, 44, 64, 35, 62, 70, 18, 65, 61, 29, 1, 60, 59, 6, 49, 38, 71, 51, 57, 84, 80, 50, 125, 3, 63, 52, 9, 96, 16, 69, 108, 109, 45, 55, 118, 8, 7, 31, 47, 15, 114, 110, 23, 5, 119, 101, 120, 99, 113, 117, 76, 46, 24, 73, 40, 12, 11, 37, 103, 4, 122, 25, 123, 74, 107, 10, 100, 116, 39, 32, 106, 87, 83, 26, 78, 13, 90, 85, 42, 56, 33, 102, 86, 81, 121, 54, 14, 97, 36, 28, 98, 104, 21, 111, 92, 89, 95, 94, 30], [41, 34, 53, 22, 88, 105, 17, 20, 58, 91, 93, 27, 29, 127, 115, 77, 126, 43, 75, 79, 61, 51, 48, 63, 35, 124, 108, 62, 112, 114, 55, 38, 57, 31, 81, 84, 44, 50, 113, 70, 123, 49, 82, 15, 25, 122, 59, 66, 52, 45, 40, 86, 39, 60, 118, 32, 106, 100, 119, 107, 116, 72, 80, 109, 101, 24, 0, 110, 125, 42, 117, 47, 37, 18, 96, 103, 46, 120, 11, 56, 19, 26, 102, 16, 54, 104, 121, 13, 99, 111, 23, 33, 83, 90, 98, 36, 92, 6, 78, 28, 21, 67, 68, 73, 95, 30, 97, 94, 87, 10, 76, 89, 12, 14, 85, 9, 71, 74, 4, 5, 8, 1, 69, 3, 2, 7, 65, 64]], "model.layers.12.self_attn.k_proj": [[38, 49, 127, 31, 113, 24, 21, 83, 80, 28, 82, 77, 35, 72, 108, 74, 62, 106, 105, 66, 96, 64, 1, 90, 112, 30, 40, 25, 102, 16, 107, 6, 51, 37, 34, 57, 116, 75, 86, 60, 125, 48, 45, 109, 76, 58, 91, 55, 44, 27, 117, 126, 29, 120, 119, 5, 46, 124, 59, 42, 67, 4, 87, 111, 85, 122, 47, 94, 23, 114, 26, 73, 84, 56, 43, 17, 63, 118, 115, 14, 79, 0, 104, 71, 98, 54, 41, 61, 69, 50, 78, 39, 12, 52, 110, 36, 92, 100, 53, 32, 123, 70, 103, 11, 121, 20, 15, 93, 97, 68, 89, 18, 101, 88, 99, 22, 81, 33, 19, 65, 10, 9, 7, 3, 8, 13, 2, 95], [109, 37, 45, 28, 24, 21, 31, 26, 82, 83, 78, 16, 62, 115, 75, 116, 124, 117, 49, 113, 121, 53, 110, 57, 30, 63, 8, 41, 105, 23, 64, 61, 127, 51, 60, 55, 126, 2, 103, 120, 33, 40, 90, 15, 93, 112, 42, 100, 118, 125, 107, 44, 111, 77, 108, 54, 34, 59, 17, 50, 18, 46, 43, 39, 56, 47, 22, 58, 5, 114, 85, 73, 65, 86, 48, 87, 38, 123, 97, 99, 95, 52, 27, 19, 102, 81, 104, 9, 106, 94, 68, 92, 13, 119, 96, 29, 98, 3, 80, 4, 20, 76, 32, 36, 12, 25, 122, 71, 35, 79, 89, 91, 84, 72, 10, 70, 66, 11, 74, 1, 7, 69, 14, 6, 67, 88, 0, 101], [107, 33, 43, 95, 22, 91, 24, 15, 26, 50, 115, 83, 13, 85, 118, 81, 20, 48, 61, 18, 49, 75, 63, 100, 117, 47, 126, 110, 9, 36, 46, 70, 54, 42, 112, 127, 124, 64, 39, 93, 116, 58, 108, 2, 121, 51, 101, 113, 111, 114, 68, 72, 123, 71, 57, 119, 120, 60, 12, 102, 125, 55, 65, 56, 99, 38, 44, 59, 40, 109, 19, 122, 45, 104, 52, 62, 41, 98, 103, 96, 53, 106, 14, 37, 105, 87, 11, 82, 92, 94, 3, 32, 30, 34, 29, 88, 74, 27, 35, 23, 76, 28, 21, 5, 78, 80, 25, 89, 16, 77, 79, 90, 31, 10, 97, 8, 86, 4, 1, 84, 17, 6, 67, 73, 7, 66, 69, 0], [113, 57, 102, 33, 82, 29, 23, 16, 11, 0, 3, 78, 65, 6, 77, 22, 26, 38, 59, 2, 71, 125, 108, 126, 73, 95, 83, 64, 63, 53, 84, 62, 93, 58, 72, 85, 12, 118, 114, 28, 79, 49, 89, 40, 56, 5, 60, 45, 61, 36, 124, 39, 68, 88, 123, 15, 96, 30, 119, 127, 117, 112, 106, 74, 116, 115, 52, 81, 94, 122, 111, 120, 27, 86, 121, 47, 100, 44, 31, 46, 4, 32, 34, 55, 109, 43, 91, 50, 103, 24, 51, 13, 37, 54, 35, 99, 41, 92, 19, 76, 48, 110, 105, 97, 98, 21, 17, 80, 20, 9, 42, 104, 107, 87, 1, 8, 69, 10, 25, 67, 14, 101, 75, 70, 90, 18, 7, 66], [103, 62, 48, 21, 81, 93, 14, 76, 11, 80, 61, 73, 71, 69, 3, 87, 0, 2, 65, 56, 45, 24, 8, 41, 70, 27, 64, 7, 66, 112, 6, 18, 117, 125, 32, 116, 54, 95, 119, 10, 94, 121, 13, 37, 33, 25, 47, 63, 74, 118, 1, 35, 51, 5, 72, 83, 4, 108, 97, 111, 99, 55, 67, 122, 50, 9, 105, 19, 75, 115, 68, 59, 15, 86, 120, 114, 31, 12, 113, 44, 43, 100, 17, 42, 107, 57, 126, 30, 28, 78, 104, 101, 22, 88, 92, 96, 84, 38, 26, 109, 16, 49, 106, 102, 60, 77, 20, 52, 79, 98, 85, 29, 123, 58, 53, 110, 127, 36, 89, 124, 23, 46, 40, 90, 34, 91, 82, 39], [41, 98, 26, 88, 18, 45, 84, 58, 16, 86, 112, 44, 111, 119, 109, 78, 40, 60, 75, 105, 77, 63, 50, 55, 96, 71, 121, 92, 73, 0, 104, 30, 2, 124, 122, 43, 4, 57, 56, 36, 49, 65, 39, 115, 125, 123, 29, 107, 101, 53, 31, 126, 110, 51, 62, 120, 85, 34, 113, 127, 69, 17, 74, 76, 47, 48, 68, 52, 42, 114, 79, 102, 100, 70, 61, 87, 38, 118, 59, 19, 89, 91, 28, 106, 94, 37, 6, 15, 103, 27, 117, 54, 9, 93, 35, 83, 108, 97, 32, 46, 11, 72, 20, 95, 116, 67, 33, 22, 23, 99, 25, 8, 81, 13, 90, 80, 10, 21, 5, 24, 12, 14, 64, 82, 66, 1, 3, 7], [40, 98, 89, 18, 22, 43, 119, 20, 79, 124, 48, 92, 49, 77, 112, 50, 17, 75, 31, 62, 61, 110, 9, 12, 6, 109, 60, 72, 117, 53, 113, 118, 54, 45, 68, 111, 107, 59, 41, 46, 84, 34, 58, 121, 64, 120, 14, 114, 47, 63, 35, 115, 56, 127, 65, 74, 44, 96, 83, 1, 100, 97, 66, 13, 70, 123, 28, 94, 104, 93, 55, 85, 33, 16, 36, 26, 2, 38, 3, 23, 67, 42, 102, 8, 103, 105, 11, 126, 29, 88, 24, 90, 81, 52, 39, 91, 122, 125, 101, 30, 51, 32, 57, 76, 116, 106, 7, 108, 37, 10, 27, 86, 99, 80, 69, 78, 73, 71, 15, 21, 0, 5, 4, 25, 19, 95, 87, 82], [105, 98, 53, 20, 88, 29, 17, 79, 22, 77, 75, 124, 61, 41, 64, 51, 126, 44, 6, 72, 119, 2, 80, 82, 59, 91, 112, 60, 118, 38, 63, 4, 108, 117, 127, 55, 58, 43, 45, 50, 62, 57, 35, 115, 125, 73, 101, 122, 12, 111, 110, 49, 46, 28, 123, 107, 42, 48, 32, 104, 19, 26, 47, 10, 40, 109, 66, 113, 34, 52, 78, 103, 114, 120, 39, 87, 56, 96, 90, 21, 86, 9, 102, 95, 116, 5, 31, 37, 36, 65, 97, 25, 106, 54, 121, 71, 67, 81, 30, 100, 33, 74, 99, 89, 8, 27, 92, 76, 1, 94, 69, 70, 85, 23, 11, 93, 7, 68, 24, 16, 15, 18, 3, 83, 14, 13, 84, 0]], "model.layers.12.self_attn.qk_proj": [[62, 49, 105, 41, 57, 113, 48, 45, 127, 109, 43, 107, 38, 102, 88, 40, 93, 53, 24, 18, 34, 82, 86, 90, 21, 104, 58, 124, 98, 75, 97, 22, 85, 80, 20, 16, 11, 84, 95, 92, 112, 29, 77, 31, 78, 61, 13, 14, 81, 17, 119, 103, 28, 87, 26, 25, 27, 19, 73, 15, 79, 126, 33, 83, 121, 37, 89, 12, 23, 9, 50, 60, 118, 76, 6, 116, 44, 55, 115, 96, 0, 51, 108, 56, 101, 111, 125, 7, 39, 110, 64, 91, 59, 63, 71, 35, 47, 122, 117, 72, 8, 3, 1, 2, 65, 69, 66, 52, 100, 114, 36, 70, 46, 120, 5, 123, 94, 30, 67, 32, 10, 54, 42, 99, 4, 68, 106, 74], [49, 62, 105, 41, 57, 113, 48, 45, 127, 109, 43, 107, 38, 40, 102, 34, 93, 88, 53, 24, 86, 124, 90, 21, 58, 104, 18, 82, 11, 75, 97, 112, 98, 29, 31, 92, 22, 20, 95, 80, 84, 77, 85, 16, 13, 78, 61, 119, 14, 28, 87, 26, 17, 25, 27, 103, 81, 73, 19, 15, 9, 23, 6, 60, 126, 111, 83, 89, 33, 79, 118, 50, 55, 108, 115, 125, 37, 121, 39, 12, 44, 0, 76, 101, 110, 72, 59, 51, 47, 63, 116, 7, 35, 117, 96, 64, 56, 71, 1, 123, 36, 122, 2, 91, 94, 65, 66, 32, 100, 3, 5, 30, 46, 4, 114, 8, 120, 106, 42, 10, 54, 69, 67, 70, 52, 99, 68, 74], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 102, 93, 40, 53, 88, 34, 24, 104, 58, 90, 86, 124, 112, 95, 21, 98, 20, 22, 18, 82, 31, 29, 97, 92, 61, 84, 85, 75, 80, 11, 119, 77, 16, 78, 103, 26, 28, 27, 13, 87, 126, 14, 33, 37, 17, 25, 23, 81, 115, 121, 44, 110, 118, 51, 50, 73, 59, 35, 60, 19, 125, 6, 116, 55, 15, 108, 117, 83, 89, 79, 9, 39, 101, 76, 12, 111, 30, 63, 0, 36, 47, 56, 96, 72, 7, 91, 65, 122, 114, 66, 100, 3, 46, 71, 70, 64, 120, 123, 52, 1, 94, 32, 5, 106, 4, 69, 2, 42, 54, 67, 10, 68, 99, 8, 74], [62, 49, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 40, 102, 53, 24, 93, 88, 34, 90, 58, 86, 124, 112, 82, 104, 18, 21, 20, 98, 80, 31, 95, 29, 22, 77, 97, 92, 75, 16, 85, 11, 84, 119, 14, 61, 103, 78, 25, 13, 28, 17, 27, 121, 79, 23, 87, 126, 44, 26, 81, 33, 50, 60, 51, 111, 118, 9, 83, 108, 15, 64, 89, 73, 101, 12, 115, 19, 39, 59, 63, 116, 37, 35, 55, 47, 70, 72, 125, 56, 110, 6, 91, 117, 76, 114, 0, 65, 30, 100, 7, 71, 123, 96, 2, 42, 66, 94, 120, 36, 67, 32, 3, 1, 122, 4, 54, 46, 52, 68, 69, 106, 5, 99, 8, 74, 10], [62, 49, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 40, 102, 53, 34, 88, 24, 58, 90, 93, 82, 124, 18, 104, 86, 80, 20, 75, 112, 98, 21, 31, 11, 16, 77, 84, 85, 61, 22, 13, 29, 92, 78, 97, 14, 95, 119, 103, 81, 26, 17, 25, 87, 28, 9, 23, 15, 79, 70, 83, 126, 121, 73, 27, 33, 50, 118, 111, 39, 19, 12, 0, 55, 89, 37, 7, 64, 72, 110, 76, 108, 59, 35, 2, 51, 44, 125, 116, 60, 47, 101, 91, 115, 6, 117, 1, 30, 56, 71, 120, 63, 66, 114, 123, 67, 122, 3, 42, 36, 65, 5, 96, 68, 69, 100, 52, 54, 46, 4, 94, 32, 106, 8, 99, 10, 74], [62, 49, 41, 105, 57, 113, 48, 127, 45, 109, 43, 107, 38, 40, 102, 53, 24, 34, 88, 90, 18, 86, 58, 75, 93, 21, 82, 20, 104, 124, 80, 77, 11, 98, 84, 31, 16, 85, 13, 78, 22, 14, 112, 92, 97, 61, 29, 119, 95, 17, 25, 81, 103, 28, 26, 87, 73, 15, 27, 9, 19, 121, 79, 83, 111, 12, 76, 70, 23, 89, 47, 50, 126, 37, 116, 55, 39, 72, 108, 118, 60, 56, 33, 115, 125, 117, 0, 44, 59, 35, 7, 64, 110, 51, 91, 65, 114, 94, 123, 63, 46, 66, 101, 2, 71, 96, 1, 6, 54, 52, 68, 120, 5, 122, 30, 99, 42, 36, 69, 67, 3, 100, 4, 106, 8, 74, 32, 10], [49, 62, 41, 105, 113, 57, 48, 127, 109, 45, 43, 107, 38, 102, 40, 88, 24, 53, 90, 18, 34, 86, 21, 82, 93, 20, 80, 104, 75, 58, 124, 84, 77, 22, 98, 16, 85, 92, 11, 13, 61, 29, 78, 31, 112, 14, 97, 17, 119, 81, 95, 23, 87, 26, 79, 28, 103, 25, 76, 15, 83, 27, 33, 121, 73, 9, 44, 50, 89, 12, 70, 19, 39, 126, 111, 37, 118, 55, 110, 116, 108, 60, 72, 51, 115, 47, 56, 117, 7, 59, 64, 122, 35, 101, 96, 63, 52, 94, 36, 91, 125, 67, 120, 0, 42, 1, 123, 71, 100, 66, 65, 2, 30, 46, 54, 3, 69, 114, 8, 5, 6, 4, 32, 99, 74, 106, 68, 10], [49, 62, 41, 105, 57, 113, 48, 127, 45, 109, 43, 107, 38, 88, 40, 102, 53, 34, 93, 24, 90, 18, 21, 86, 82, 104, 20, 85, 80, 22, 75, 84, 92, 77, 11, 16, 124, 98, 31, 58, 78, 97, 13, 29, 14, 61, 95, 119, 112, 17, 87, 81, 15, 28, 19, 23, 26, 79, 25, 33, 44, 70, 121, 103, 115, 83, 89, 12, 9, 27, 73, 118, 50, 111, 37, 126, 76, 116, 117, 110, 55, 7, 60, 0, 51, 108, 47, 59, 64, 63, 35, 8, 72, 125, 122, 39, 71, 91, 3, 120, 2, 65, 101, 36, 94, 100, 66, 114, 46, 54, 5, 96, 30, 56, 1, 123, 32, 6, 67, 52, 68, 42, 99, 69, 106, 10, 4, 74], [49, 62, 105, 41, 57, 113, 48, 127, 109, 45, 43, 107, 38, 40, 102, 93, 53, 88, 24, 34, 90, 18, 20, 86, 104, 85, 82, 21, 98, 22, 29, 92, 58, 80, 16, 31, 61, 124, 119, 112, 84, 75, 95, 78, 11, 77, 97, 14, 25, 23, 17, 26, 81, 28, 13, 87, 19, 121, 79, 44, 103, 15, 115, 111, 126, 37, 89, 33, 27, 9, 73, 110, 108, 50, 60, 39, 83, 47, 76, 96, 51, 55, 125, 70, 94, 12, 116, 63, 118, 64, 117, 59, 7, 35, 122, 30, 0, 91, 2, 36, 6, 56, 8, 42, 67, 66, 54, 1, 71, 114, 101, 65, 52, 46, 100, 120, 106, 3, 123, 72, 4, 32, 68, 99, 5, 69, 74, 10], [49, 62, 41, 57, 105, 113, 48, 127, 45, 109, 43, 107, 38, 53, 102, 40, 88, 24, 93, 34, 18, 90, 58, 104, 86, 85, 82, 22, 20, 75, 21, 16, 124, 92, 31, 77, 61, 80, 29, 78, 84, 112, 119, 11, 13, 95, 14, 97, 98, 81, 25, 23, 28, 26, 19, 15, 17, 87, 103, 27, 9, 121, 79, 126, 33, 83, 44, 89, 73, 50, 111, 60, 51, 47, 115, 108, 37, 8, 118, 6, 70, 76, 55, 12, 110, 125, 63, 7, 39, 71, 64, 0, 96, 67, 120, 36, 101, 30, 66, 59, 94, 4, 91, 114, 56, 117, 1, 116, 65, 123, 35, 32, 2, 3, 42, 54, 68, 5, 100, 122, 46, 106, 69, 74, 99, 72, 52, 10], [49, 62, 105, 41, 57, 113, 48, 45, 127, 43, 109, 107, 38, 102, 53, 40, 34, 24, 93, 88, 18, 82, 86, 58, 21, 90, 104, 75, 80, 16, 31, 77, 124, 61, 85, 20, 78, 22, 98, 112, 29, 92, 97, 11, 95, 84, 14, 119, 13, 17, 81, 25, 26, 6, 103, 9, 87, 28, 126, 23, 19, 27, 15, 79, 76, 83, 64, 8, 50, 73, 118, 89, 33, 115, 121, 111, 12, 37, 47, 7, 2, 116, 60, 71, 0, 51, 63, 120, 66, 91, 125, 1, 110, 44, 70, 114, 35, 67, 108, 55, 56, 5, 65, 101, 4, 30, 39, 59, 96, 123, 3, 122, 117, 36, 68, 69, 72, 42, 54, 94, 32, 52, 46, 74, 100, 99, 106, 10], [49, 62, 41, 105, 113, 57, 48, 109, 127, 45, 43, 107, 38, 40, 102, 53, 88, 34, 24, 93, 86, 18, 90, 58, 21, 82, 80, 75, 104, 77, 22, 16, 20, 98, 84, 92, 85, 78, 11, 31, 29, 112, 124, 61, 13, 14, 97, 95, 17, 81, 119, 26, 28, 103, 6, 15, 25, 87, 9, 73, 19, 76, 23, 12, 83, 89, 79, 126, 37, 121, 50, 111, 27, 8, 33, 64, 60, 44, 47, 0, 39, 71, 51, 115, 118, 2, 120, 55, 7, 66, 56, 108, 116, 125, 59, 35, 91, 3, 94, 65, 122, 63, 70, 123, 5, 117, 110, 72, 96, 1, 114, 36, 30, 101, 42, 69, 46, 100, 4, 67, 74, 52, 54, 68, 99, 32, 106, 10], [49, 62, 41, 105, 113, 57, 48, 127, 109, 45, 43, 107, 38, 102, 40, 53, 24, 93, 88, 104, 18, 34, 86, 21, 82, 90, 20, 22, 58, 85, 124, 80, 84, 92, 77, 31, 16, 98, 29, 11, 78, 75, 61, 95, 112, 97, 17, 119, 13, 14, 81, 26, 15, 25, 28, 121, 19, 89, 87, 23, 83, 37, 103, 79, 33, 126, 50, 27, 44, 9, 12, 6, 60, 111, 110, 108, 39, 73, 55, 115, 35, 118, 76, 51, 63, 8, 47, 91, 125, 101, 30, 116, 56, 123, 59, 117, 7, 122, 96, 94, 36, 114, 100, 64, 42, 2, 120, 0, 65, 54, 71, 52, 3, 1, 67, 66, 68, 70, 32, 99, 5, 4, 106, 72, 46, 69, 10, 74], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 53, 40, 34, 102, 24, 88, 104, 18, 82, 93, 21, 90, 75, 20, 77, 11, 86, 80, 58, 124, 98, 78, 22, 13, 84, 85, 31, 61, 112, 14, 16, 97, 92, 29, 95, 17, 119, 81, 25, 6, 15, 9, 19, 126, 83, 79, 87, 27, 89, 121, 26, 28, 73, 23, 103, 8, 50, 60, 118, 111, 0, 108, 33, 44, 7, 115, 64, 76, 12, 110, 125, 66, 51, 55, 47, 71, 2, 116, 70, 37, 56, 39, 91, 1, 117, 101, 30, 123, 63, 36, 65, 68, 35, 122, 96, 5, 114, 120, 59, 67, 4, 54, 72, 52, 94, 69, 42, 100, 3, 46, 32, 74, 99, 10, 106], [49, 62, 41, 105, 113, 57, 48, 127, 45, 109, 43, 107, 38, 53, 102, 40, 34, 93, 88, 24, 104, 18, 82, 90, 58, 86, 21, 20, 75, 31, 98, 124, 80, 77, 16, 11, 78, 84, 29, 92, 95, 112, 85, 61, 22, 97, 13, 14, 81, 17, 119, 9, 103, 28, 25, 121, 126, 26, 23, 15, 83, 33, 73, 27, 79, 44, 87, 89, 19, 37, 50, 110, 108, 60, 6, 118, 12, 111, 116, 51, 8, 115, 47, 0, 55, 39, 56, 70, 76, 35, 71, 30, 63, 7, 101, 91, 125, 96, 123, 66, 120, 59, 64, 117, 65, 54, 94, 114, 100, 72, 36, 32, 46, 122, 5, 52, 69, 106, 2, 1, 74, 67, 4, 3, 42, 99, 68, 10], [49, 62, 105, 41, 113, 57, 48, 109, 43, 127, 45, 107, 38, 102, 40, 53, 34, 93, 58, 24, 18, 88, 82, 104, 21, 86, 90, 20, 75, 98, 124, 61, 16, 80, 22, 85, 78, 84, 11, 31, 77, 92, 17, 95, 29, 112, 13, 97, 14, 81, 121, 119, 103, 15, 26, 87, 9, 126, 73, 27, 23, 44, 25, 79, 111, 83, 19, 12, 60, 76, 70, 28, 50, 115, 89, 33, 37, 59, 108, 64, 118, 110, 7, 0, 39, 116, 51, 123, 47, 125, 66, 55, 67, 35, 8, 101, 6, 120, 71, 117, 63, 2, 91, 56, 1, 30, 54, 96, 72, 65, 52, 114, 3, 36, 4, 122, 94, 42, 69, 5, 100, 32, 46, 68, 74, 10, 99, 106], [49, 62, 41, 105, 57, 113, 45, 48, 43, 127, 109, 107, 38, 40, 53, 102, 34, 88, 93, 18, 24, 90, 82, 104, 86, 22, 21, 20, 29, 75, 61, 80, 98, 85, 58, 16, 84, 95, 92, 77, 31, 11, 97, 112, 124, 78, 17, 14, 81, 13, 28, 119, 79, 15, 87, 23, 25, 26, 73, 103, 126, 83, 19, 50, 121, 108, 9, 33, 70, 89, 27, 111, 37, 51, 60, 118, 39, 76, 44, 12, 115, 117, 116, 91, 101, 110, 55, 30, 125, 63, 120, 47, 72, 7, 0, 114, 123, 54, 71, 56, 122, 59, 2, 1, 35, 42, 36, 5, 100, 65, 96, 64, 4, 32, 46, 94, 66, 8, 3, 68, 6, 52, 67, 99, 10, 69, 74, 106], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 40, 102, 53, 34, 93, 88, 24, 18, 82, 21, 104, 86, 90, 20, 98, 11, 58, 75, 22, 80, 16, 31, 95, 92, 97, 84, 124, 112, 29, 85, 77, 61, 13, 14, 81, 17, 119, 78, 26, 103, 87, 28, 79, 15, 121, 83, 126, 50, 25, 27, 70, 23, 33, 73, 44, 9, 89, 19, 108, 37, 60, 118, 39, 111, 110, 116, 76, 51, 47, 35, 30, 101, 71, 55, 12, 117, 64, 125, 72, 91, 115, 63, 0, 7, 59, 67, 100, 96, 114, 36, 65, 66, 56, 123, 6, 2, 122, 52, 46, 120, 32, 1, 54, 94, 106, 42, 8, 69, 68, 99, 5, 10, 4, 3, 74], [49, 41, 62, 105, 57, 113, 127, 48, 45, 109, 43, 107, 38, 40, 102, 93, 53, 88, 34, 104, 58, 24, 18, 82, 90, 20, 86, 112, 29, 98, 80, 85, 31, 97, 61, 21, 124, 84, 16, 119, 75, 95, 22, 92, 11, 77, 14, 103, 26, 81, 17, 126, 78, 87, 28, 121, 13, 25, 108, 39, 110, 118, 44, 9, 33, 27, 15, 50, 79, 89, 23, 37, 73, 70, 51, 111, 60, 63, 83, 19, 59, 125, 101, 116, 47, 55, 117, 56, 96, 94, 30, 36, 72, 35, 12, 115, 71, 91, 100, 64, 42, 76, 32, 0, 7, 123, 66, 120, 65, 122, 106, 46, 6, 67, 114, 54, 52, 1, 99, 2, 69, 3, 68, 4, 8, 10, 74, 5], [49, 62, 105, 41, 57, 113, 48, 127, 45, 109, 43, 107, 38, 102, 40, 53, 93, 34, 88, 24, 86, 90, 18, 21, 58, 104, 82, 98, 20, 31, 80, 75, 124, 11, 29, 61, 95, 97, 22, 85, 92, 112, 84, 77, 16, 78, 13, 17, 119, 14, 28, 25, 81, 103, 121, 87, 15, 33, 27, 9, 26, 50, 19, 126, 83, 37, 111, 89, 79, 101, 73, 60, 116, 47, 118, 23, 63, 39, 72, 59, 12, 108, 76, 55, 51, 44, 70, 117, 96, 115, 110, 35, 30, 125, 0, 120, 64, 71, 7, 123, 56, 91, 114, 1, 36, 42, 6, 2, 54, 4, 94, 106, 100, 32, 52, 67, 122, 46, 65, 68, 99, 5, 69, 66, 3, 74, 8, 10], [49, 62, 41, 105, 57, 113, 48, 45, 127, 43, 109, 107, 38, 40, 53, 34, 102, 93, 24, 104, 88, 86, 18, 58, 82, 90, 21, 98, 80, 20, 11, 75, 124, 97, 22, 77, 31, 92, 16, 85, 13, 29, 84, 95, 112, 78, 61, 14, 81, 17, 119, 103, 121, 73, 15, 25, 9, 27, 126, 79, 83, 87, 26, 19, 23, 110, 28, 50, 89, 76, 116, 72, 12, 111, 47, 44, 33, 118, 115, 60, 51, 37, 108, 55, 6, 39, 101, 59, 70, 71, 64, 117, 0, 123, 125, 63, 7, 35, 120, 30, 52, 36, 2, 46, 1, 54, 66, 96, 69, 91, 65, 56, 114, 3, 122, 42, 32, 67, 68, 4, 99, 10, 100, 94, 74, 8, 106, 5], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 40, 102, 53, 93, 34, 104, 88, 58, 24, 86, 90, 21, 18, 98, 82, 31, 97, 84, 20, 124, 95, 11, 85, 61, 22, 16, 80, 13, 29, 112, 92, 78, 75, 77, 14, 81, 28, 26, 17, 119, 121, 103, 15, 23, 126, 87, 27, 25, 110, 44, 19, 79, 9, 89, 73, 33, 111, 59, 115, 50, 83, 6, 37, 108, 12, 55, 76, 118, 47, 116, 91, 101, 35, 60, 96, 51, 63, 117, 39, 72, 64, 0, 71, 125, 123, 7, 52, 36, 42, 32, 114, 56, 122, 94, 120, 3, 30, 65, 46, 2, 70, 66, 1, 4, 100, 106, 68, 10, 8, 54, 99, 67, 69, 5, 74], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 53, 102, 40, 88, 93, 34, 104, 24, 18, 86, 21, 90, 82, 58, 98, 80, 61, 20, 31, 124, 22, 85, 11, 92, 84, 29, 112, 95, 75, 13, 97, 16, 77, 119, 78, 14, 26, 17, 81, 28, 126, 15, 103, 23, 87, 83, 9, 33, 110, 6, 27, 25, 121, 111, 51, 89, 44, 118, 50, 115, 73, 79, 60, 19, 12, 37, 39, 108, 55, 76, 47, 59, 63, 0, 116, 123, 35, 101, 91, 32, 64, 72, 36, 71, 96, 7, 120, 125, 65, 117, 100, 30, 114, 2, 56, 1, 122, 66, 42, 54, 3, 94, 8, 5, 46, 4, 67, 52, 99, 106, 68, 69, 70, 10, 74], [49, 62, 41, 105, 57, 113, 48, 45, 109, 127, 43, 107, 38, 53, 40, 102, 34, 24, 88, 93, 104, 58, 18, 82, 86, 98, 90, 11, 31, 80, 21, 20, 85, 124, 75, 92, 77, 112, 97, 22, 16, 84, 95, 13, 29, 61, 78, 81, 14, 28, 119, 103, 26, 17, 25, 9, 6, 126, 87, 73, 111, 79, 23, 121, 83, 15, 19, 33, 27, 89, 50, 115, 37, 118, 47, 110, 116, 55, 108, 60, 76, 123, 12, 44, 59, 39, 63, 117, 7, 72, 35, 101, 51, 56, 64, 125, 96, 91, 71, 114, 8, 36, 120, 0, 30, 65, 2, 68, 32, 46, 66, 3, 99, 54, 4, 52, 70, 100, 94, 122, 1, 69, 5, 42, 106, 67, 74, 10], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 40, 102, 34, 93, 53, 88, 24, 18, 86, 104, 21, 90, 58, 82, 22, 20, 98, 16, 124, 61, 29, 31, 11, 97, 80, 112, 84, 75, 85, 92, 95, 77, 13, 28, 78, 119, 14, 103, 81, 26, 17, 25, 121, 73, 87, 23, 89, 9, 83, 27, 79, 12, 33, 108, 6, 126, 15, 50, 44, 19, 39, 111, 37, 118, 60, 76, 51, 110, 116, 117, 35, 55, 59, 101, 47, 7, 56, 63, 0, 123, 125, 115, 8, 122, 120, 64, 1, 96, 30, 70, 72, 2, 114, 71, 52, 91, 54, 46, 3, 36, 100, 32, 66, 42, 5, 67, 94, 106, 65, 99, 68, 69, 74, 4, 10], [49, 62, 41, 105, 57, 113, 48, 109, 127, 45, 43, 107, 38, 40, 34, 102, 88, 53, 93, 104, 24, 90, 18, 86, 22, 82, 58, 20, 21, 11, 31, 98, 92, 85, 112, 97, 75, 80, 16, 84, 124, 29, 61, 77, 78, 95, 13, 119, 28, 81, 14, 17, 26, 15, 25, 103, 19, 73, 23, 27, 83, 9, 89, 87, 79, 33, 126, 111, 50, 121, 60, 12, 51, 118, 37, 125, 110, 44, 70, 76, 55, 108, 6, 8, 115, 7, 39, 116, 35, 0, 101, 47, 63, 64, 59, 123, 56, 114, 117, 122, 120, 42, 71, 1, 30, 91, 96, 65, 100, 54, 52, 69, 2, 32, 67, 36, 106, 5, 94, 66, 3, 72, 46, 68, 10, 74, 4, 99], [49, 62, 41, 105, 57, 113, 48, 109, 127, 45, 43, 107, 38, 53, 102, 40, 88, 104, 24, 34, 93, 90, 18, 86, 82, 58, 98, 22, 124, 20, 29, 112, 21, 61, 85, 80, 92, 75, 31, 11, 16, 84, 77, 95, 78, 81, 97, 17, 126, 119, 13, 14, 25, 103, 28, 26, 87, 23, 83, 33, 27, 9, 73, 110, 121, 115, 79, 125, 15, 51, 89, 44, 12, 50, 108, 37, 55, 39, 60, 8, 111, 70, 47, 76, 35, 36, 19, 118, 116, 59, 117, 7, 101, 63, 6, 91, 0, 114, 100, 56, 65, 64, 42, 122, 66, 2, 32, 71, 94, 96, 67, 120, 123, 30, 54, 68, 1, 52, 69, 3, 46, 5, 106, 4, 10, 99, 72, 74], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 53, 40, 102, 88, 24, 34, 18, 104, 93, 82, 90, 86, 75, 58, 11, 80, 21, 98, 124, 20, 85, 16, 61, 31, 22, 77, 92, 78, 97, 84, 13, 112, 29, 95, 81, 17, 14, 119, 70, 9, 103, 28, 126, 83, 25, 26, 73, 64, 87, 79, 121, 23, 115, 15, 27, 19, 33, 8, 89, 76, 12, 7, 118, 44, 111, 51, 55, 0, 116, 50, 2, 110, 66, 101, 60, 59, 125, 108, 117, 47, 68, 67, 65, 37, 56, 6, 39, 71, 123, 63, 3, 1, 91, 5, 52, 35, 96, 122, 36, 114, 100, 30, 46, 4, 69, 54, 120, 94, 72, 10, 106, 32, 42, 74, 99], [62, 49, 41, 105, 57, 113, 48, 127, 45, 109, 43, 107, 38, 102, 40, 53, 34, 93, 88, 58, 24, 104, 21, 124, 11, 86, 90, 18, 98, 75, 84, 82, 29, 61, 22, 31, 95, 92, 112, 80, 20, 85, 78, 97, 77, 16, 13, 26, 17, 14, 103, 119, 81, 23, 121, 25, 70, 15, 73, 28, 27, 9, 126, 8, 87, 89, 37, 19, 50, 12, 44, 47, 33, 51, 125, 83, 116, 115, 7, 110, 118, 60, 108, 79, 111, 59, 0, 39, 35, 55, 101, 76, 63, 64, 56, 91, 114, 2, 96, 1, 117, 30, 67, 100, 36, 71, 66, 52, 46, 122, 3, 123, 120, 4, 32, 69, 65, 54, 6, 5, 42, 94, 68, 99, 106, 72, 10, 74], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 102, 40, 24, 53, 88, 93, 34, 18, 90, 82, 86, 11, 58, 75, 21, 20, 104, 98, 22, 16, 80, 124, 85, 31, 92, 29, 77, 13, 84, 97, 112, 78, 61, 95, 14, 17, 81, 19, 119, 25, 103, 9, 121, 26, 87, 73, 15, 28, 70, 126, 23, 79, 89, 37, 50, 33, 44, 8, 27, 12, 116, 111, 83, 76, 101, 60, 115, 0, 47, 118, 125, 55, 7, 59, 56, 117, 64, 51, 108, 66, 96, 39, 110, 65, 63, 2, 123, 35, 71, 94, 5, 114, 1, 91, 67, 30, 6, 3, 54, 46, 120, 122, 68, 69, 74, 72, 4, 100, 42, 32, 36, 99, 52, 106, 10], [49, 62, 41, 105, 113, 57, 48, 45, 127, 109, 43, 107, 38, 40, 102, 53, 88, 93, 24, 58, 34, 82, 90, 18, 104, 124, 21, 20, 112, 11, 86, 85, 16, 22, 77, 97, 75, 98, 80, 92, 84, 95, 13, 61, 31, 14, 103, 29, 78, 17, 81, 119, 15, 28, 19, 23, 121, 115, 25, 9, 83, 126, 27, 73, 79, 70, 87, 26, 116, 37, 12, 44, 89, 33, 50, 59, 51, 39, 60, 108, 35, 111, 101, 118, 125, 76, 55, 7, 8, 110, 64, 6, 47, 117, 63, 114, 0, 56, 123, 91, 2, 96, 122, 46, 65, 69, 54, 71, 30, 72, 100, 1, 52, 66, 68, 36, 67, 3, 4, 42, 120, 94, 5, 32, 99, 106, 10, 74], [62, 49, 41, 105, 57, 113, 48, 127, 45, 109, 43, 107, 38, 102, 40, 53, 88, 34, 24, 93, 82, 18, 90, 86, 75, 104, 21, 77, 11, 20, 58, 22, 85, 124, 16, 98, 31, 84, 13, 97, 95, 92, 112, 80, 61, 29, 14, 78, 17, 81, 28, 26, 119, 79, 87, 9, 15, 19, 115, 103, 73, 83, 126, 25, 12, 37, 23, 27, 33, 121, 50, 108, 89, 70, 76, 7, 118, 111, 8, 6, 44, 51, 60, 63, 116, 59, 47, 72, 39, 110, 71, 96, 0, 101, 123, 91, 55, 117, 35, 64, 68, 56, 125, 114, 2, 1, 100, 66, 46, 42, 36, 5, 122, 65, 67, 94, 10, 120, 30, 54, 4, 74, 52, 32, 69, 99, 3, 106]], "model.layers.13.self_attn.q_proj": [[115, 103, 62, 124, 126, 50, 93, 54, 90, 20, 89, 13, 122, 74, 83, 23, 127, 33, 7, 82, 61, 39, 48, 112, 80, 56, 22, 25, 2, 91, 92, 29, 31, 0, 53, 52, 67, 117, 116, 119, 51, 120, 77, 4, 65, 121, 6, 5, 87, 69, 125, 21, 15, 12, 100, 114, 64, 118, 60, 59, 66, 49, 3, 16, 95, 34, 55, 37, 28, 19, 111, 17, 18, 68, 58, 99, 85, 72, 63, 32, 57, 81, 44, 9, 70, 30, 84, 36, 88, 101, 96, 24, 113, 45, 11, 109, 35, 94, 102, 98, 26, 123, 38, 27, 14, 76, 78, 73, 8, 47, 107, 86, 43, 46, 41, 40, 42, 105, 110, 108, 79, 71, 75, 104, 97, 1, 106, 10], [115, 62, 103, 124, 50, 54, 126, 23, 74, 67, 61, 90, 122, 127, 56, 112, 7, 53, 93, 82, 6, 39, 77, 116, 48, 2, 68, 117, 52, 0, 89, 33, 121, 114, 51, 120, 85, 15, 31, 25, 119, 49, 125, 60, 11, 111, 22, 118, 55, 63, 59, 37, 69, 58, 113, 57, 95, 44, 109, 45, 9, 110, 46, 107, 38, 35, 43, 83, 13, 65, 123, 108, 102, 47, 20, 41, 34, 42, 104, 105, 106, 100, 92, 64, 5, 98, 73, 28, 40, 96, 101, 66, 32, 99, 79, 36, 87, 18, 72, 14, 30, 29, 88, 24, 94, 21, 10, 80, 26, 16, 19, 86, 4, 70, 84, 12, 3, 91, 81, 97, 27, 76, 1, 78, 17, 75, 8, 71], [62, 115, 103, 39, 124, 127, 126, 50, 90, 61, 54, 112, 122, 23, 11, 7, 15, 20, 82, 52, 74, 53, 89, 116, 51, 119, 6, 56, 14, 48, 0, 117, 68, 67, 79, 120, 84, 121, 125, 114, 59, 80, 55, 111, 100, 118, 49, 95, 60, 72, 22, 66, 57, 33, 63, 102, 58, 113, 2, 107, 73, 43, 109, 110, 93, 34, 37, 40, 101, 31, 46, 92, 45, 35, 13, 18, 44, 28, 106, 25, 98, 81, 38, 108, 96, 123, 47, 32, 69, 41, 42, 105, 104, 99, 87, 88, 16, 24, 94, 85, 77, 83, 30, 29, 91, 26, 8, 27, 9, 86, 75, 36, 70, 12, 76, 78, 65, 97, 5, 19, 64, 10, 17, 21, 71, 4, 1, 3], [115, 62, 103, 50, 126, 124, 54, 90, 61, 122, 127, 74, 112, 67, 82, 48, 56, 7, 39, 117, 77, 52, 83, 116, 114, 93, 2, 53, 120, 125, 51, 119, 121, 18, 92, 13, 6, 111, 0, 89, 49, 59, 60, 88, 109, 55, 63, 58, 118, 33, 45, 57, 23, 11, 108, 113, 47, 21, 43, 25, 102, 107, 40, 46, 65, 44, 69, 123, 66, 64, 87, 110, 37, 15, 42, 20, 105, 35, 104, 41, 106, 5, 100, 19, 101, 68, 98, 17, 73, 38, 91, 3, 95, 96, 29, 34, 9, 85, 36, 99, 10, 31, 32, 70, 4, 28, 22, 30, 12, 16, 94, 72, 26, 27, 81, 24, 1, 97, 84, 76, 86, 80, 14, 78, 71, 79, 8, 75], [123, 127, 60, 62, 38, 121, 53, 56, 61, 122, 110, 120, 59, 57, 54, 52, 44, 114, 119, 51, 125, 118, 116, 91, 117, 124, 126, 55, 115, 48, 58, 47, 113, 49, 90, 50, 111, 63, 102, 82, 46, 108, 109, 32, 112, 14, 104, 45, 29, 100, 84, 34, 103, 99, 107, 37, 86, 43, 97, 105, 39, 22, 42, 106, 36, 24, 40, 33, 89, 98, 41, 94, 101, 27, 30, 23, 75, 20, 88, 35, 96, 18, 25, 26, 95, 93, 80, 31, 78, 19, 28, 8, 9, 21, 87, 11, 81, 83, 64, 68, 85, 92, 13, 66, 17, 1, 12, 76, 77, 16, 79, 70, 71, 5, 15, 72, 3, 7, 10, 65, 6, 67, 73, 4, 0, 69, 2, 74], [127, 123, 38, 121, 62, 60, 37, 53, 56, 61, 122, 57, 120, 54, 116, 51, 59, 114, 52, 58, 44, 32, 119, 125, 36, 118, 124, 117, 49, 126, 115, 90, 110, 55, 108, 47, 50, 111, 113, 63, 89, 107, 82, 109, 91, 112, 106, 100, 48, 87, 14, 46, 84, 102, 42, 96, 45, 103, 34, 27, 25, 20, 22, 104, 40, 78, 8, 97, 94, 39, 31, 43, 95, 98, 29, 105, 101, 41, 30, 33, 99, 86, 1, 35, 75, 9, 81, 18, 26, 19, 93, 92, 24, 28, 5, 12, 68, 3, 71, 76, 64, 0, 79, 13, 88, 70, 21, 72, 23, 17, 85, 10, 15, 66, 16, 67, 83, 73, 7, 6, 80, 11, 77, 4, 2, 69, 65, 74], [123, 127, 62, 60, 121, 1, 53, 61, 56, 122, 57, 115, 120, 54, 52, 63, 51, 58, 59, 32, 114, 116, 124, 113, 125, 118, 119, 117, 49, 110, 126, 107, 55, 87, 37, 0, 111, 47, 50, 48, 112, 20, 109, 106, 104, 91, 108, 64, 46, 38, 44, 45, 67, 75, 66, 41, 42, 39, 43, 100, 40, 105, 78, 82, 103, 3, 36, 26, 71, 101, 34, 22, 68, 99, 18, 95, 72, 14, 5, 98, 88, 13, 33, 28, 79, 30, 94, 35, 102, 27, 96, 16, 97, 92, 9, 8, 93, 10, 76, 21, 17, 11, 6, 83, 2, 70, 15, 29, 23, 85, 84, 86, 31, 12, 24, 90, 25, 19, 73, 65, 7, 81, 89, 4, 69, 80, 77, 74], [127, 123, 38, 24, 83, 17, 28, 79, 90, 26, 10, 97, 13, 60, 31, 86, 104, 121, 84, 76, 4, 42, 7, 29, 62, 74, 77, 85, 40, 88, 19, 2, 53, 46, 70, 92, 15, 49, 81, 89, 57, 25, 20, 91, 9, 6, 5, 71, 64, 58, 101, 21, 116, 12, 56, 33, 18, 112, 78, 80, 99, 27, 82, 16, 87, 54, 23, 1, 65, 0, 95, 102, 14, 75, 93, 32, 51, 109, 69, 67, 115, 73, 52, 61, 94, 11, 108, 96, 30, 36, 22, 68, 59, 125, 48, 122, 72, 8, 37, 34, 55, 126, 114, 124, 66, 63, 50, 120, 43, 45, 117, 119, 103, 100, 118, 44, 3, 110, 113, 47, 98, 106, 35, 41, 107, 111, 105, 39], [121, 60, 39, 123, 120, 89, 61, 54, 49, 119, 65, 51, 122, 116, 62, 78, 127, 71, 126, 59, 112, 58, 117, 92, 110, 48, 118, 11, 50, 57, 124, 80, 53, 4, 55, 18, 27, 56, 44, 47, 68, 115, 125, 23, 63, 43, 111, 96, 69, 105, 46, 90, 99, 103, 52, 113, 106, 45, 109, 9, 114, 70, 107, 0, 101, 28, 84, 104, 91, 3, 42, 108, 73, 66, 38, 40, 82, 41, 102, 22, 95, 2, 93, 94, 36, 100, 25, 37, 10, 30, 34, 97, 98, 86, 79, 16, 33, 64, 8, 5, 29, 87, 19, 14, 20, 74, 31, 7, 35, 83, 15, 12, 72, 26, 81, 88, 17, 21, 32, 75, 77, 85, 24, 1, 67, 13, 76, 6], [60, 121, 39, 89, 123, 120, 54, 61, 116, 62, 49, 119, 23, 96, 18, 126, 84, 51, 80, 122, 117, 16, 55, 27, 127, 57, 124, 48, 9, 25, 95, 86, 92, 59, 58, 118, 28, 22, 112, 53, 50, 56, 47, 15, 65, 13, 99, 78, 69, 110, 63, 115, 52, 106, 7, 34, 94, 30, 103, 19, 111, 75, 82, 125, 105, 11, 97, 32, 44, 43, 114, 10, 46, 45, 108, 113, 21, 104, 71, 20, 77, 109, 107, 42, 38, 40, 85, 35, 66, 41, 37, 73, 93, 87, 81, 0, 91, 100, 36, 90, 31, 98, 12, 102, 33, 26, 29, 101, 70, 2, 68, 74, 83, 24, 79, 3, 17, 88, 14, 4, 67, 76, 72, 64, 5, 8, 6, 1], [121, 39, 60, 123, 120, 89, 54, 116, 23, 61, 84, 27, 28, 11, 92, 96, 51, 49, 18, 80, 99, 16, 122, 48, 62, 75, 126, 119, 86, 124, 127, 118, 58, 59, 117, 50, 55, 112, 78, 57, 22, 56, 30, 25, 47, 110, 53, 94, 95, 82, 115, 106, 111, 29, 7, 9, 44, 114, 45, 91, 15, 109, 90, 103, 46, 63, 113, 52, 33, 105, 2, 107, 4, 20, 42, 65, 102, 81, 71, 40, 104, 101, 87, 70, 43, 19, 108, 93, 125, 77, 85, 36, 37, 98, 35, 41, 79, 3, 97, 21, 6, 32, 31, 100, 83, 0, 38, 34, 68, 69, 26, 17, 14, 73, 88, 12, 64, 24, 67, 76, 13, 10, 8, 74, 72, 5, 66, 1], [60, 121, 39, 123, 54, 84, 120, 116, 89, 61, 49, 51, 18, 16, 92, 117, 99, 28, 122, 27, 126, 82, 112, 119, 62, 127, 103, 23, 124, 48, 96, 118, 58, 59, 57, 94, 55, 53, 86, 50, 9, 88, 47, 95, 56, 52, 110, 11, 25, 115, 22, 65, 30, 71, 69, 15, 42, 114, 111, 75, 45, 46, 125, 33, 106, 102, 63, 113, 43, 78, 91, 107, 90, 101, 104, 34, 41, 44, 77, 109, 40, 38, 87, 105, 0, 13, 36, 29, 85, 31, 108, 26, 20, 7, 98, 68, 14, 37, 93, 97, 100, 24, 19, 66, 32, 35, 81, 21, 80, 17, 83, 70, 2, 79, 76, 12, 74, 73, 8, 64, 5, 72, 67, 3, 4, 10, 6, 1], [51, 118, 102, 48, 90, 24, 97, 53, 126, 61, 58, 117, 18, 121, 29, 62, 63, 54, 123, 86, 50, 19, 9, 80, 124, 26, 47, 127, 111, 93, 120, 112, 125, 38, 115, 23, 59, 116, 119, 27, 60, 91, 73, 17, 46, 55, 78, 108, 20, 113, 12, 104, 103, 114, 52, 101, 57, 71, 45, 56, 39, 89, 110, 82, 11, 87, 88, 30, 107, 36, 5, 77, 109, 66, 49, 42, 13, 44, 99, 16, 14, 33, 43, 41, 98, 83, 100, 28, 74, 122, 94, 105, 31, 34, 40, 25, 106, 37, 75, 32, 35, 96, 21, 81, 95, 85, 22, 84, 92, 15, 72, 3, 7, 68, 70, 79, 76, 64, 65, 69, 2, 1, 0, 8, 10, 4, 6, 67], [118, 51, 102, 48, 61, 126, 58, 123, 53, 124, 127, 120, 62, 117, 63, 90, 54, 60, 19, 125, 121, 119, 97, 116, 59, 41, 113, 50, 3, 56, 52, 55, 29, 112, 47, 73, 57, 0, 86, 115, 38, 93, 78, 24, 26, 111, 92, 81, 122, 46, 9, 5, 1, 109, 114, 43, 69, 87, 22, 89, 103, 94, 30, 45, 110, 108, 49, 7, 44, 66, 105, 39, 98, 107, 99, 104, 106, 91, 42, 83, 28, 2, 75, 14, 18, 4, 40, 25, 76, 37, 100, 6, 84, 65, 12, 32, 35, 96, 17, 16, 33, 82, 101, 67, 21, 13, 36, 34, 15, 8, 85, 77, 11, 64, 31, 20, 88, 80, 79, 95, 71, 23, 10, 74, 27, 70, 72, 68], [51, 102, 118, 48, 24, 97, 53, 90, 19, 58, 126, 47, 63, 61, 86, 81, 38, 18, 121, 117, 93, 123, 29, 62, 60, 124, 80, 9, 50, 99, 89, 59, 112, 54, 127, 120, 45, 94, 125, 108, 119, 13, 75, 100, 11, 39, 57, 111, 113, 114, 91, 104, 88, 77, 115, 26, 116, 3, 5, 109, 73, 36, 56, 37, 98, 49, 52, 110, 103, 78, 105, 107, 42, 55, 28, 66, 41, 122, 46, 30, 71, 44, 40, 83, 35, 43, 31, 34, 27, 95, 82, 22, 87, 17, 106, 85, 92, 20, 32, 101, 23, 21, 84, 69, 15, 96, 25, 16, 7, 74, 33, 14, 72, 0, 70, 67, 10, 1, 79, 64, 12, 8, 76, 65, 68, 6, 4, 2], [51, 102, 118, 48, 24, 97, 61, 86, 53, 29, 126, 19, 90, 63, 124, 115, 78, 75, 18, 58, 80, 81, 50, 123, 93, 66, 87, 62, 89, 30, 77, 39, 117, 16, 34, 99, 54, 9, 88, 32, 47, 116, 91, 45, 120, 37, 36, 125, 96, 83, 28, 111, 31, 44, 127, 3, 59, 100, 26, 121, 11, 95, 60, 103, 38, 52, 119, 23, 109, 71, 35, 112, 92, 17, 113, 5, 94, 20, 105, 27, 110, 57, 98, 101, 106, 56, 74, 114, 41, 25, 107, 70, 108, 104, 69, 46, 33, 0, 82, 55, 21, 1, 14, 72, 49, 40, 10, 85, 43, 64, 13, 73, 122, 42, 22, 7, 15, 84, 6, 68, 12, 79, 4, 76, 8, 65, 67, 2], [47, 125, 111, 56, 14, 124, 60, 24, 120, 127, 117, 54, 118, 49, 123, 5, 57, 91, 61, 75, 2, 21, 96, 59, 122, 1, 62, 94, 55, 7, 126, 63, 26, 58, 50, 121, 48, 103, 39, 53, 9, 3, 119, 64, 116, 104, 101, 98, 40, 113, 23, 4, 20, 100, 83, 42, 114, 52, 115, 112, 37, 36, 13, 82, 18, 97, 10, 17, 85, 72, 108, 44, 22, 110, 51, 89, 73, 46, 25, 27, 6, 43, 35, 19, 78, 29, 15, 107, 41, 71, 31, 45, 81, 32, 16, 79, 106, 105, 8, 28, 34, 99, 93, 109, 87, 74, 33, 88, 102, 70, 38, 92, 95, 76, 86, 80, 11, 77, 90, 30, 65, 67, 84, 0, 12, 69, 66, 68], [47, 111, 125, 56, 98, 26, 124, 2, 5, 60, 117, 103, 85, 64, 39, 36, 61, 54, 120, 127, 92, 37, 123, 49, 4, 87, 91, 97, 126, 122, 118, 1, 59, 57, 62, 20, 14, 6, 89, 48, 7, 3, 55, 24, 9, 50, 63, 96, 121, 113, 53, 65, 29, 58, 75, 13, 27, 114, 51, 101, 10, 41, 52, 81, 83, 84, 116, 94, 35, 42, 71, 72, 46, 119, 38, 112, 44, 8, 22, 115, 106, 28, 90, 100, 16, 31, 105, 99, 107, 43, 34, 109, 0, 23, 45, 95, 110, 11, 104, 108, 68, 76, 73, 86, 102, 25, 40, 32, 79, 15, 17, 66, 21, 33, 74, 80, 30, 19, 82, 93, 67, 70, 69, 77, 18, 88, 78, 12], [125, 47, 111, 56, 60, 124, 14, 39, 117, 26, 127, 118, 120, 54, 61, 49, 123, 5, 57, 103, 126, 62, 94, 59, 122, 81, 55, 63, 98, 50, 48, 23, 121, 19, 53, 20, 46, 58, 113, 116, 76, 114, 2, 8, 1, 119, 97, 112, 64, 4, 10, 75, 115, 107, 51, 52, 38, 44, 21, 72, 87, 65, 73, 16, 104, 100, 41, 3, 90, 42, 43, 45, 71, 96, 108, 110, 106, 105, 37, 99, 86, 6, 109, 17, 24, 36, 40, 22, 79, 80, 91, 28, 9, 7, 83, 101, 85, 13, 74, 34, 102, 33, 92, 18, 35, 82, 27, 95, 31, 29, 11, 78, 84, 88, 68, 30, 0, 25, 89, 77, 12, 32, 69, 93, 15, 70, 66, 67], [47, 125, 111, 36, 56, 24, 96, 124, 91, 75, 25, 120, 100, 7, 60, 49, 83, 29, 16, 99, 84, 21, 2, 54, 94, 1, 14, 127, 117, 5, 61, 57, 80, 26, 103, 40, 118, 123, 22, 97, 64, 33, 4, 46, 35, 44, 48, 59, 110, 85, 126, 62, 104, 13, 122, 9, 27, 28, 58, 89, 3, 20, 6, 31, 116, 90, 77, 53, 108, 15, 43, 93, 121, 101, 42, 39, 119, 87, 50, 55, 17, 79, 63, 88, 98, 67, 112, 95, 82, 106, 18, 92, 23, 113, 41, 34, 30, 52, 37, 38, 102, 114, 19, 71, 32, 45, 115, 105, 74, 109, 51, 107, 11, 70, 76, 78, 8, 10, 68, 12, 65, 72, 73, 81, 86, 0, 69, 66], [123, 63, 39, 114, 57, 115, 61, 127, 121, 124, 59, 62, 120, 54, 49, 93, 116, 125, 110, 113, 97, 50, 25, 53, 58, 112, 56, 55, 118, 51, 117, 92, 52, 80, 126, 60, 42, 122, 45, 48, 22, 111, 44, 28, 108, 119, 102, 46, 107, 43, 95, 41, 40, 47, 26, 103, 100, 88, 106, 19, 37, 104, 105, 34, 109, 35, 96, 17, 86, 31, 87, 36, 99, 32, 101, 94, 38, 24, 9, 29, 6, 91, 16, 3, 98, 76, 12, 72, 30, 90, 78, 33, 83, 73, 27, 68, 0, 67, 23, 18, 81, 20, 89, 85, 77, 15, 65, 2, 1, 5, 84, 79, 21, 69, 82, 11, 8, 14, 75, 70, 13, 4, 74, 66, 71, 10, 64, 7], [63, 123, 114, 39, 57, 115, 61, 59, 127, 124, 121, 62, 54, 125, 120, 116, 113, 49, 60, 117, 58, 53, 122, 88, 56, 50, 52, 48, 19, 55, 126, 51, 28, 118, 112, 119, 43, 42, 93, 45, 97, 38, 110, 46, 6, 0, 41, 85, 104, 95, 86, 102, 107, 111, 92, 47, 87, 105, 16, 106, 103, 109, 44, 76, 25, 37, 108, 81, 40, 34, 96, 35, 73, 80, 15, 101, 22, 99, 100, 36, 26, 67, 3, 83, 94, 32, 29, 90, 98, 23, 69, 24, 33, 17, 27, 30, 72, 31, 91, 5, 2, 12, 78, 65, 84, 70, 79, 9, 89, 75, 1, 20, 4, 68, 21, 14, 18, 77, 82, 74, 11, 8, 64, 10, 13, 66, 71, 7], [39, 63, 123, 25, 82, 97, 87, 13, 115, 22, 19, 114, 74, 80, 70, 57, 6, 77, 3, 66, 83, 7, 18, 64, 84, 91, 31, 4, 12, 94, 29, 27, 10, 9, 79, 93, 92, 16, 95, 24, 75, 30, 21, 90, 86, 5, 59, 85, 81, 76, 58, 104, 78, 23, 15, 20, 89, 17, 1, 127, 71, 120, 14, 73, 61, 54, 8, 88, 124, 62, 113, 116, 43, 72, 68, 109, 49, 96, 118, 99, 32, 2, 65, 28, 11, 55, 100, 34, 35, 98, 121, 48, 107, 37, 69, 45, 110, 40, 26, 41, 50, 51, 53, 122, 67, 102, 42, 126, 0, 60, 125, 46, 56, 105, 52, 36, 38, 119, 111, 112, 117, 33, 101, 108, 44, 103, 106, 47], [63, 123, 39, 115, 57, 28, 114, 80, 97, 19, 25, 22, 127, 61, 124, 87, 9, 62, 59, 96, 88, 120, 92, 58, 54, 113, 116, 104, 121, 49, 111, 3, 12, 125, 117, 53, 51, 24, 107, 95, 20, 60, 50, 31, 126, 52, 48, 15, 56, 72, 21, 77, 112, 55, 34, 122, 41, 46, 82, 118, 40, 119, 13, 42, 94, 102, 43, 37, 30, 45, 110, 99, 23, 16, 83, 93, 84, 70, 85, 29, 90, 100, 27, 91, 44, 105, 109, 78, 38, 108, 65, 68, 106, 98, 26, 35, 47, 36, 86, 76, 81, 17, 33, 8, 101, 5, 14, 79, 32, 74, 2, 89, 73, 64, 1, 7, 75, 18, 4, 103, 6, 10, 0, 69, 67, 11, 71, 66], [38, 47, 62, 111, 93, 83, 16, 26, 33, 76, 24, 9, 23, 86, 78, 92, 81, 118, 90, 68, 1, 70, 4, 34, 73, 74, 80, 29, 84, 20, 12, 87, 14, 19, 101, 2, 77, 75, 21, 71, 51, 64, 22, 85, 15, 63, 5, 10, 91, 66, 112, 95, 17, 11, 28, 82, 67, 88, 98, 36, 27, 13, 79, 7, 69, 65, 0, 53, 32, 30, 119, 18, 89, 56, 31, 52, 6, 3, 100, 8, 25, 72, 120, 104, 61, 122, 123, 114, 102, 96, 127, 110, 117, 49, 124, 55, 94, 57, 48, 115, 60, 109, 116, 50, 99, 35, 121, 106, 59, 58, 126, 46, 42, 40, 97, 125, 54, 105, 103, 44, 37, 39, 108, 113, 45, 41, 43, 107], [38, 47, 62, 111, 33, 24, 83, 93, 79, 81, 76, 100, 86, 118, 78, 29, 67, 26, 28, 16, 74, 6, 112, 32, 87, 90, 84, 119, 53, 92, 77, 34, 63, 18, 61, 120, 70, 85, 122, 56, 71, 114, 52, 13, 25, 98, 82, 23, 3, 88, 27, 19, 75, 51, 2, 20, 127, 95, 123, 22, 9, 60, 55, 121, 99, 117, 10, 17, 5, 49, 80, 115, 48, 94, 124, 0, 1, 50, 58, 113, 91, 4, 57, 89, 30, 8, 21, 15, 31, 12, 125, 68, 101, 116, 110, 14, 104, 106, 107, 59, 11, 96, 126, 73, 105, 72, 54, 37, 46, 35, 40, 7, 36, 108, 102, 45, 43, 109, 44, 66, 39, 41, 42, 103, 97, 65, 69, 64], [62, 111, 47, 38, 118, 120, 51, 63, 56, 53, 119, 100, 117, 58, 48, 122, 55, 124, 60, 123, 113, 52, 108, 116, 115, 127, 57, 112, 114, 121, 61, 49, 50, 126, 54, 18, 104, 59, 91, 125, 106, 109, 107, 110, 46, 27, 101, 28, 41, 84, 45, 44, 95, 35, 37, 105, 40, 43, 42, 103, 96, 92, 99, 36, 39, 24, 98, 33, 32, 87, 23, 21, 34, 31, 30, 93, 86, 94, 29, 89, 26, 97, 102, 82, 25, 20, 22, 90, 88, 81, 85, 79, 83, 15, 13, 77, 17, 8, 74, 67, 10, 72, 78, 64, 16, 3, 75, 19, 7, 76, 70, 69, 71, 11, 80, 14, 6, 5, 66, 0, 12, 4, 2, 65, 1, 9, 68, 73], [62, 111, 47, 118, 120, 100, 38, 56, 119, 52, 53, 112, 58, 63, 48, 57, 123, 113, 122, 55, 125, 117, 60, 115, 54, 51, 121, 114, 50, 124, 127, 61, 49, 126, 59, 116, 109, 110, 107, 36, 46, 104, 105, 106, 44, 108, 28, 45, 43, 27, 42, 41, 35, 103, 39, 91, 40, 82, 101, 84, 95, 33, 34, 92, 23, 37, 99, 102, 31, 18, 98, 93, 87, 32, 21, 30, 29, 96, 94, 86, 24, 97, 26, 22, 25, 89, 90, 67, 81, 20, 79, 6, 85, 10, 15, 12, 88, 8, 64, 72, 78, 70, 83, 19, 14, 17, 13, 66, 3, 7, 69, 77, 74, 11, 80, 16, 65, 75, 76, 0, 73, 71, 5, 2, 4, 68, 9, 1], [42, 43, 33, 72, 38, 1, 5, 91, 113, 106, 0, 21, 114, 12, 79, 24, 30, 67, 81, 19, 119, 61, 46, 124, 82, 49, 2, 107, 74, 53, 121, 122, 69, 66, 75, 71, 13, 25, 47, 112, 28, 126, 86, 77, 94, 87, 125, 45, 4, 117, 23, 3, 123, 70, 88, 50, 20, 80, 48, 65, 68, 89, 76, 63, 101, 100, 116, 8, 15, 103, 59, 27, 17, 110, 99, 7, 60, 14, 118, 11, 62, 104, 41, 64, 58, 31, 6, 115, 102, 57, 52, 92, 37, 127, 109, 51, 9, 78, 32, 22, 111, 90, 83, 56, 54, 16, 73, 26, 35, 39, 96, 55, 105, 95, 85, 84, 120, 10, 97, 98, 18, 44, 29, 36, 40, 108, 93, 34], [42, 38, 91, 33, 117, 43, 21, 82, 79, 74, 24, 107, 12, 20, 78, 10, 70, 14, 121, 89, 112, 30, 119, 81, 106, 27, 86, 19, 18, 84, 8, 88, 92, 28, 114, 6, 3, 113, 16, 94, 61, 67, 15, 87, 64, 90, 122, 80, 96, 5, 22, 23, 123, 57, 69, 83, 9, 76, 85, 25, 37, 102, 56, 40, 72, 75, 36, 93, 73, 51, 45, 13, 29, 116, 39, 32, 65, 63, 17, 77, 4, 46, 41, 100, 71, 26, 47, 35, 31, 95, 53, 124, 101, 11, 62, 105, 127, 125, 49, 7, 110, 34, 68, 99, 52, 98, 50, 126, 103, 66, 109, 59, 104, 60, 2, 48, 58, 118, 115, 44, 111, 54, 120, 55, 1, 108, 97, 0], [117, 43, 38, 42, 113, 33, 91, 107, 30, 114, 47, 123, 41, 46, 125, 62, 24, 26, 112, 39, 106, 59, 111, 60, 53, 116, 18, 115, 40, 50, 63, 87, 56, 118, 121, 119, 80, 108, 57, 89, 48, 124, 110, 51, 86, 61, 32, 109, 45, 127, 82, 52, 54, 126, 120, 55, 31, 58, 49, 28, 34, 44, 36, 81, 101, 104, 78, 95, 122, 94, 35, 96, 98, 88, 99, 19, 14, 21, 105, 37, 84, 93, 100, 29, 90, 103, 25, 20, 79, 11, 97, 23, 22, 16, 77, 27, 92, 13, 73, 83, 102, 85, 74, 17, 9, 70, 12, 76, 75, 2, 66, 8, 67, 6, 4, 71, 72, 7, 5, 0, 68, 15, 3, 10, 1, 64, 65, 69], [43, 38, 107, 33, 114, 91, 46, 47, 26, 24, 42, 117, 62, 30, 53, 56, 123, 119, 125, 39, 116, 111, 41, 59, 82, 115, 52, 90, 113, 48, 84, 118, 49, 55, 126, 50, 124, 51, 112, 122, 60, 57, 106, 121, 105, 63, 109, 120, 61, 110, 93, 44, 45, 127, 40, 54, 77, 108, 101, 34, 58, 35, 27, 88, 18, 100, 20, 37, 32, 104, 99, 98, 103, 29, 36, 31, 25, 81, 94, 86, 96, 102, 95, 28, 89, 87, 78, 22, 92, 14, 23, 76, 80, 17, 97, 75, 83, 19, 21, 16, 13, 12, 8, 79, 73, 11, 9, 6, 70, 4, 71, 67, 74, 5, 66, 72, 68, 15, 7, 0, 3, 64, 1, 69, 10, 2, 85, 65]], "model.layers.13.self_attn.k_proj": [[115, 62, 39, 124, 127, 50, 36, 61, 54, 22, 122, 112, 126, 97, 116, 52, 53, 56, 117, 51, 121, 48, 120, 119, 49, 114, 125, 55, 118, 111, 100, 57, 60, 59, 29, 58, 63, 89, 113, 91, 46, 107, 92, 109, 47, 44, 123, 45, 110, 42, 23, 105, 95, 43, 33, 108, 106, 41, 40, 37, 82, 104, 99, 101, 38, 30, 102, 20, 87, 19, 80, 78, 31, 103, 90, 88, 98, 34, 94, 35, 24, 96, 21, 1, 81, 79, 28, 32, 8, 75, 93, 26, 86, 73, 27, 76, 17, 13, 69, 10, 16, 71, 83, 14, 12, 68, 25, 84, 85, 11, 67, 15, 5, 18, 6, 66, 70, 74, 77, 4, 72, 9, 0, 7, 3, 2, 64, 65], [127, 123, 102, 86, 35, 60, 121, 58, 28, 63, 31, 51, 57, 83, 48, 99, 115, 110, 53, 24, 108, 46, 56, 17, 62, 79, 61, 125, 49, 52, 33, 116, 122, 50, 117, 114, 90, 112, 84, 54, 42, 55, 126, 40, 25, 59, 124, 29, 119, 107, 47, 44, 45, 120, 113, 109, 43, 16, 97, 111, 104, 118, 106, 34, 13, 41, 105, 76, 10, 39, 103, 38, 101, 37, 98, 14, 100, 91, 36, 32, 82, 95, 72, 96, 71, 68, 21, 81, 94, 22, 93, 30, 87, 19, 9, 20, 80, 77, 23, 92, 26, 88, 67, 11, 1, 66, 85, 89, 27, 15, 73, 12, 74, 18, 70, 75, 5, 69, 78, 6, 2, 0, 7, 8, 4, 64, 3, 65], [121, 60, 103, 123, 86, 54, 49, 35, 120, 61, 116, 32, 51, 119, 62, 122, 127, 55, 126, 118, 53, 28, 57, 59, 48, 124, 112, 50, 58, 115, 56, 110, 47, 99, 117, 63, 42, 44, 111, 114, 46, 125, 45, 52, 109, 106, 22, 113, 100, 105, 107, 30, 43, 23, 77, 108, 41, 40, 25, 84, 18, 101, 80, 96, 102, 104, 37, 91, 89, 98, 94, 90, 75, 38, 31, 36, 19, 97, 33, 15, 29, 93, 5, 16, 95, 34, 26, 88, 21, 81, 12, 17, 87, 3, 92, 13, 24, 76, 78, 6, 79, 27, 39, 85, 64, 66, 73, 8, 0, 9, 70, 83, 11, 82, 72, 74, 20, 71, 10, 14, 4, 67, 2, 7, 1, 65, 68, 69], [51, 118, 38, 86, 33, 48, 61, 53, 93, 123, 54, 120, 58, 124, 125, 117, 62, 126, 127, 59, 60, 115, 63, 47, 121, 119, 116, 55, 56, 50, 52, 57, 24, 113, 41, 112, 99, 26, 46, 91, 122, 81, 39, 104, 43, 103, 18, 111, 80, 45, 42, 11, 114, 109, 40, 19, 49, 110, 44, 105, 101, 107, 106, 20, 108, 79, 77, 36, 102, 89, 30, 90, 84, 97, 34, 21, 25, 35, 78, 27, 37, 15, 28, 6, 95, 4, 100, 98, 23, 67, 32, 88, 17, 96, 8, 9, 74, 16, 82, 72, 1, 22, 92, 31, 94, 75, 85, 87, 12, 10, 14, 65, 5, 76, 13, 64, 29, 69, 71, 70, 7, 2, 83, 66, 0, 73, 68, 3], [111, 125, 100, 47, 22, 120, 127, 118, 56, 54, 123, 60, 124, 59, 62, 117, 61, 57, 55, 122, 53, 63, 48, 126, 29, 121, 50, 49, 32, 108, 107, 119, 116, 42, 58, 113, 114, 36, 43, 115, 52, 105, 112, 51, 40, 39, 106, 41, 44, 46, 91, 110, 24, 103, 83, 45, 102, 104, 109, 25, 80, 26, 38, 79, 84, 99, 97, 92, 101, 35, 72, 37, 95, 76, 81, 18, 30, 34, 98, 33, 10, 31, 90, 4, 93, 21, 78, 77, 11, 89, 88, 86, 17, 20, 28, 71, 85, 94, 0, 82, 96, 16, 66, 13, 3, 12, 23, 27, 15, 73, 87, 65, 14, 70, 74, 6, 19, 2, 69, 75, 5, 9, 64, 7, 1, 8, 67, 68], [103, 123, 63, 22, 57, 115, 33, 114, 54, 13, 124, 127, 82, 25, 61, 59, 62, 28, 116, 120, 87, 121, 107, 125, 58, 27, 53, 60, 49, 55, 117, 80, 84, 56, 126, 113, 19, 74, 51, 118, 50, 48, 45, 109, 38, 99, 71, 119, 52, 7, 111, 97, 26, 122, 47, 12, 46, 37, 9, 30, 112, 93, 101, 106, 108, 31, 110, 85, 44, 104, 14, 42, 29, 105, 41, 100, 75, 98, 43, 102, 70, 89, 95, 86, 2, 17, 34, 15, 4, 79, 35, 32, 36, 20, 68, 1, 67, 40, 96, 0, 88, 18, 81, 94, 5, 73, 91, 11, 21, 72, 65, 10, 24, 3, 90, 23, 77, 64, 66, 78, 8, 76, 83, 39, 92, 6, 69, 16], [111, 62, 102, 86, 97, 118, 26, 29, 47, 83, 16, 81, 76, 78, 112, 9, 24, 119, 75, 53, 127, 122, 123, 114, 51, 55, 120, 96, 52, 63, 56, 50, 4, 71, 60, 124, 117, 121, 23, 61, 115, 116, 58, 49, 79, 74, 54, 36, 126, 57, 41, 59, 84, 42, 106, 46, 48, 35, 70, 110, 125, 37, 66, 105, 103, 109, 108, 45, 98, 40, 34, 43, 107, 65, 113, 77, 69, 1, 64, 44, 28, 99, 104, 93, 39, 101, 32, 3, 30, 21, 31, 38, 91, 80, 72, 11, 25, 100, 94, 7, 95, 89, 92, 5, 33, 0, 73, 27, 82, 15, 14, 13, 12, 20, 85, 17, 19, 88, 90, 22, 18, 2, 87, 67, 10, 8, 6, 68], [106, 107, 97, 102, 122, 86, 94, 121, 117, 49, 21, 50, 36, 48, 114, 42, 79, 103, 119, 27, 63, 113, 74, 92, 126, 116, 56, 61, 109, 91, 64, 123, 70, 62, 67, 127, 51, 24, 52, 105, 40, 19, 111, 57, 5, 12, 82, 112, 110, 59, 47, 45, 77, 44, 65, 18, 53, 115, 124, 81, 84, 60, 118, 125, 99, 98, 32, 88, 55, 54, 104, 71, 90, 66, 58, 37, 72, 78, 26, 8, 108, 120, 35, 101, 4, 46, 100, 34, 96, 1, 89, 11, 80, 93, 31, 95, 43, 23, 39, 25, 29, 41, 85, 87, 75, 9, 13, 0, 38, 15, 10, 14, 73, 28, 83, 30, 68, 22, 17, 33, 7, 76, 16, 20, 2, 6, 69, 3]], "model.layers.13.self_attn.qk_proj": [[123, 62, 111, 127, 121, 60, 51, 118, 115, 63, 47, 125, 53, 124, 61, 120, 106, 56, 57, 117, 59, 48, 116, 114, 119, 107, 122, 54, 50, 126, 42, 52, 102, 22, 38, 49, 58, 112, 86, 103, 113, 39, 55, 43, 110, 33, 91, 97, 100, 46, 88, 93, 92, 45, 108, 90, 44, 19, 24, 109, 27, 29, 36, 89, 83, 99, 82, 105, 18, 104, 26, 21, 16, 25, 40, 30, 32, 35, 80, 28, 87, 17, 41, 84, 12, 77, 101, 15, 20, 79, 23, 74, 85, 37, 76, 13, 94, 81, 98, 96, 78, 14, 95, 31, 75, 10, 9, 34, 73, 65, 70, 4, 11, 64, 3, 6, 7, 8, 68, 71, 5, 0, 72, 67, 69, 2, 66, 1], [123, 62, 111, 127, 121, 51, 118, 60, 63, 115, 47, 125, 53, 61, 56, 59, 54, 117, 57, 124, 106, 107, 120, 48, 114, 116, 119, 50, 102, 42, 122, 22, 126, 38, 58, 49, 52, 112, 103, 113, 86, 39, 43, 55, 110, 44, 93, 100, 33, 97, 90, 46, 92, 91, 45, 108, 88, 27, 36, 89, 40, 24, 29, 26, 105, 19, 99, 82, 35, 83, 25, 16, 109, 30, 32, 21, 80, 104, 37, 84, 28, 87, 18, 41, 20, 12, 85, 15, 76, 96, 17, 23, 31, 94, 81, 101, 77, 74, 78, 98, 10, 34, 75, 13, 70, 14, 95, 9, 79, 11, 71, 4, 73, 3, 7, 1, 0, 69, 65, 72, 64, 8, 5, 68, 2, 6, 67, 66], [123, 62, 111, 127, 121, 51, 118, 60, 63, 115, 47, 125, 61, 56, 117, 124, 53, 59, 106, 57, 120, 54, 48, 114, 42, 107, 50, 126, 116, 119, 122, 38, 102, 103, 112, 49, 113, 39, 58, 52, 22, 86, 55, 43, 33, 93, 97, 110, 91, 100, 46, 108, 45, 44, 92, 88, 90, 105, 27, 36, 29, 99, 40, 24, 109, 104, 32, 26, 89, 35, 83, 25, 41, 28, 37, 85, 30, 87, 101, 21, 16, 19, 18, 84, 98, 82, 23, 80, 94, 20, 79, 12, 34, 77, 31, 17, 96, 15, 95, 70, 74, 76, 10, 81, 14, 78, 75, 13, 9, 72, 64, 65, 1, 69, 0, 11, 71, 3, 67, 73, 66, 2, 4, 5, 68, 7, 8, 6], [123, 62, 111, 127, 121, 51, 118, 63, 60, 115, 47, 125, 61, 56, 53, 54, 124, 117, 106, 120, 126, 116, 57, 119, 59, 107, 58, 122, 38, 48, 49, 42, 50, 114, 102, 103, 112, 52, 22, 55, 43, 113, 39, 86, 110, 33, 44, 46, 92, 97, 93, 91, 88, 100, 45, 90, 40, 36, 29, 27, 108, 26, 89, 24, 32, 109, 83, 99, 105, 28, 35, 82, 19, 41, 87, 25, 37, 16, 18, 85, 23, 21, 17, 104, 79, 30, 31, 101, 12, 80, 34, 96, 20, 76, 77, 84, 98, 94, 70, 74, 81, 95, 14, 10, 75, 13, 15, 9, 78, 1, 64, 71, 3, 72, 68, 0, 11, 65, 67, 66, 73, 7, 69, 4, 5, 2, 6, 8], [123, 111, 62, 127, 121, 51, 115, 118, 63, 60, 47, 125, 61, 56, 53, 106, 124, 54, 57, 126, 117, 116, 114, 48, 119, 59, 50, 122, 42, 107, 120, 58, 102, 38, 52, 112, 49, 103, 55, 22, 113, 86, 39, 110, 44, 43, 46, 33, 91, 93, 92, 97, 88, 100, 90, 108, 24, 99, 36, 45, 27, 29, 40, 83, 89, 105, 19, 87, 26, 25, 28, 35, 32, 104, 109, 82, 41, 18, 15, 77, 16, 17, 12, 21, 76, 80, 23, 85, 81, 79, 94, 101, 20, 84, 10, 78, 74, 96, 30, 70, 98, 13, 95, 31, 37, 9, 75, 34, 14, 64, 7, 11, 68, 65, 72, 71, 73, 1, 66, 69, 3, 0, 4, 67, 6, 2, 5, 8], [123, 62, 111, 127, 121, 51, 60, 115, 63, 118, 47, 125, 53, 61, 56, 124, 106, 117, 54, 126, 116, 57, 119, 107, 59, 120, 122, 58, 42, 50, 55, 102, 38, 22, 48, 114, 112, 86, 103, 39, 52, 49, 113, 43, 44, 110, 46, 91, 88, 93, 92, 97, 33, 90, 100, 29, 83, 108, 45, 36, 104, 19, 27, 26, 24, 99, 82, 16, 105, 87, 89, 80, 17, 18, 109, 28, 41, 40, 25, 35, 76, 79, 21, 13, 77, 84, 23, 32, 81, 30, 15, 20, 12, 74, 10, 14, 34, 101, 78, 75, 96, 9, 98, 85, 31, 94, 37, 11, 7, 70, 95, 71, 72, 6, 73, 0, 65, 3, 67, 1, 66, 4, 69, 5, 68, 64, 8, 2], [123, 111, 62, 127, 121, 51, 115, 60, 63, 118, 47, 125, 61, 124, 53, 56, 126, 54, 117, 106, 122, 59, 107, 120, 119, 57, 116, 58, 114, 49, 42, 102, 48, 103, 38, 50, 55, 22, 112, 86, 39, 52, 43, 113, 88, 44, 46, 93, 110, 91, 92, 90, 97, 100, 45, 19, 33, 83, 24, 108, 27, 99, 40, 89, 36, 104, 29, 105, 82, 18, 26, 21, 80, 17, 87, 109, 35, 85, 76, 28, 25, 23, 16, 79, 77, 12, 32, 84, 37, 15, 30, 41, 94, 20, 13, 81, 78, 96, 10, 98, 14, 31, 34, 74, 9, 101, 75, 95, 72, 7, 73, 11, 6, 70, 71, 69, 67, 0, 5, 65, 1, 66, 64, 3, 8, 68, 4, 2], [123, 62, 111, 127, 121, 51, 115, 60, 63, 47, 118, 125, 61, 124, 53, 56, 106, 54, 117, 120, 48, 126, 114, 107, 122, 59, 57, 119, 103, 116, 42, 38, 102, 58, 22, 49, 86, 50, 39, 55, 112, 52, 113, 43, 44, 91, 93, 110, 90, 97, 92, 100, 33, 88, 46, 27, 24, 29, 83, 19, 89, 36, 108, 35, 40, 104, 45, 26, 87, 16, 28, 32, 23, 82, 105, 99, 109, 80, 21, 25, 18, 84, 85, 41, 37, 76, 17, 31, 30, 15, 13, 12, 96, 20, 101, 77, 81, 14, 78, 94, 74, 98, 79, 10, 9, 34, 95, 75, 6, 72, 11, 73, 1, 0, 65, 64, 7, 3, 67, 69, 5, 2, 70, 4, 68, 71, 8, 66], [123, 62, 111, 127, 51, 121, 63, 60, 115, 118, 47, 125, 61, 53, 54, 117, 124, 56, 106, 126, 119, 59, 107, 120, 48, 57, 114, 102, 50, 42, 122, 38, 116, 113, 103, 58, 22, 49, 39, 112, 55, 52, 86, 43, 93, 90, 91, 110, 44, 92, 33, 97, 100, 46, 26, 88, 45, 27, 108, 29, 89, 83, 36, 82, 24, 19, 35, 99, 104, 25, 109, 28, 32, 40, 80, 87, 85, 41, 105, 21, 23, 16, 37, 76, 18, 84, 30, 17, 98, 31, 81, 78, 96, 14, 12, 13, 94, 79, 15, 20, 101, 74, 77, 10, 34, 95, 73, 75, 6, 67, 11, 9, 3, 65, 1, 69, 8, 64, 7, 72, 0, 71, 70, 5, 66, 4, 2, 68], [62, 123, 111, 127, 121, 51, 63, 115, 60, 118, 47, 125, 53, 61, 124, 117, 59, 106, 56, 54, 48, 126, 114, 119, 57, 107, 116, 120, 38, 58, 50, 102, 42, 103, 52, 22, 112, 49, 122, 86, 39, 55, 43, 113, 44, 33, 91, 46, 93, 97, 100, 92, 110, 90, 88, 108, 24, 83, 27, 45, 89, 19, 109, 29, 36, 26, 40, 105, 104, 28, 18, 25, 99, 82, 16, 35, 41, 32, 80, 76, 37, 13, 87, 96, 30, 31, 79, 12, 23, 17, 77, 20, 85, 81, 78, 84, 98, 15, 74, 94, 101, 21, 14, 95, 10, 34, 75, 9, 6, 11, 73, 67, 8, 7, 71, 1, 68, 5, 70, 64, 65, 0, 2, 3, 4, 69, 72, 66], [123, 62, 111, 127, 121, 51, 60, 115, 63, 118, 47, 125, 124, 53, 61, 117, 54, 120, 106, 126, 114, 116, 56, 57, 59, 119, 48, 107, 50, 122, 42, 38, 102, 49, 22, 52, 58, 39, 55, 113, 86, 103, 43, 112, 91, 33, 46, 97, 93, 100, 92, 44, 88, 90, 110, 24, 29, 27, 40, 45, 99, 89, 19, 26, 83, 82, 36, 105, 108, 25, 109, 17, 37, 32, 18, 80, 16, 35, 87, 77, 12, 30, 94, 76, 84, 28, 104, 79, 15, 31, 74, 41, 21, 13, 96, 81, 10, 6, 20, 23, 101, 73, 98, 78, 14, 95, 75, 34, 85, 71, 8, 11, 7, 3, 1, 9, 67, 68, 70, 4, 65, 64, 0, 5, 66, 69, 72, 2], [123, 62, 111, 127, 121, 51, 60, 115, 63, 118, 47, 125, 61, 53, 54, 117, 106, 57, 126, 124, 119, 120, 116, 114, 59, 56, 122, 49, 42, 107, 50, 48, 52, 102, 22, 103, 55, 38, 58, 86, 113, 112, 39, 43, 46, 91, 97, 110, 93, 33, 90, 88, 44, 92, 100, 24, 19, 45, 27, 83, 99, 29, 36, 108, 21, 26, 89, 82, 87, 35, 109, 40, 32, 76, 80, 15, 16, 105, 77, 17, 10, 41, 18, 79, 104, 28, 13, 81, 12, 25, 85, 31, 94, 37, 96, 84, 30, 74, 20, 23, 34, 14, 78, 101, 98, 65, 11, 8, 9, 75, 73, 71, 95, 6, 70, 0, 68, 69, 67, 7, 3, 5, 64, 66, 4, 1, 72, 2], [123, 62, 111, 127, 121, 51, 115, 60, 63, 118, 47, 125, 61, 124, 53, 117, 126, 114, 54, 59, 56, 106, 120, 107, 57, 119, 50, 48, 38, 102, 116, 42, 49, 22, 103, 55, 122, 112, 52, 43, 86, 58, 113, 39, 46, 91, 110, 93, 97, 44, 92, 90, 100, 45, 33, 27, 89, 24, 29, 40, 108, 88, 104, 99, 19, 26, 25, 36, 82, 41, 109, 83, 35, 32, 28, 87, 105, 80, 18, 85, 16, 37, 76, 96, 23, 20, 17, 30, 31, 21, 84, 15, 94, 13, 79, 77, 12, 78, 81, 98, 10, 34, 95, 101, 14, 11, 75, 74, 73, 8, 70, 71, 0, 3, 65, 9, 5, 67, 66, 69, 4, 68, 6, 1, 2, 64, 7, 72], [123, 62, 111, 127, 121, 63, 60, 51, 118, 115, 47, 125, 61, 124, 53, 117, 59, 106, 119, 126, 57, 107, 56, 54, 120, 114, 50, 122, 38, 58, 55, 116, 49, 48, 22, 52, 102, 42, 86, 112, 43, 39, 103, 113, 46, 110, 44, 91, 92, 33, 97, 100, 93, 108, 88, 90, 109, 24, 45, 82, 19, 29, 83, 36, 105, 89, 80, 27, 35, 18, 104, 26, 25, 16, 13, 28, 40, 99, 17, 41, 30, 76, 87, 84, 78, 94, 21, 32, 79, 37, 23, 12, 14, 20, 96, 10, 85, 81, 98, 95, 31, 11, 74, 75, 101, 70, 77, 73, 15, 71, 9, 7, 8, 34, 67, 64, 0, 68, 65, 3, 4, 1, 6, 69, 66, 5, 72, 2], [123, 62, 111, 127, 121, 60, 51, 115, 118, 63, 47, 125, 124, 53, 61, 106, 56, 117, 119, 57, 54, 114, 120, 126, 59, 107, 50, 116, 122, 55, 42, 48, 38, 49, 52, 58, 102, 22, 86, 112, 103, 43, 39, 113, 110, 46, 91, 44, 93, 97, 100, 33, 92, 108, 24, 27, 45, 109, 88, 90, 29, 19, 82, 83, 36, 26, 40, 99, 32, 25, 104, 28, 80, 35, 18, 21, 16, 41, 89, 105, 17, 76, 20, 31, 23, 13, 37, 87, 94, 84, 12, 30, 81, 78, 96, 15, 98, 85, 74, 77, 10, 79, 14, 95, 75, 70, 34, 101, 73, 11, 71, 8, 68, 7, 9, 65, 64, 4, 3, 1, 67, 0, 5, 69, 72, 6, 2, 66], [123, 62, 111, 127, 121, 51, 60, 115, 118, 63, 47, 125, 124, 61, 53, 117, 106, 114, 54, 59, 120, 126, 57, 56, 119, 107, 116, 122, 48, 55, 42, 58, 49, 50, 102, 38, 22, 52, 112, 113, 86, 39, 103, 43, 46, 44, 93, 100, 110, 97, 33, 91, 45, 92, 27, 90, 24, 88, 108, 36, 109, 83, 82, 40, 26, 29, 21, 99, 19, 41, 104, 89, 18, 32, 15, 25, 12, 16, 28, 87, 10, 30, 79, 17, 31, 35, 13, 81, 76, 105, 80, 37, 96, 23, 84, 85, 98, 94, 74, 34, 77, 73, 20, 14, 78, 11, 101, 8, 75, 95, 70, 65, 69, 7, 71, 9, 64, 6, 1, 4, 0, 67, 68, 3, 72, 2, 5, 66], [123, 62, 111, 127, 121, 115, 51, 60, 63, 118, 47, 125, 61, 124, 53, 59, 57, 117, 126, 119, 54, 120, 106, 56, 107, 122, 38, 114, 52, 55, 58, 48, 49, 116, 42, 22, 103, 102, 86, 50, 39, 112, 113, 46, 43, 44, 91, 100, 33, 97, 93, 110, 92, 27, 90, 88, 45, 109, 108, 99, 24, 29, 83, 89, 26, 36, 104, 40, 19, 35, 32, 87, 80, 25, 105, 41, 82, 84, 28, 16, 18, 79, 85, 81, 13, 23, 37, 21, 30, 12, 17, 34, 101, 98, 94, 31, 76, 20, 15, 77, 96, 10, 95, 78, 73, 11, 75, 14, 74, 6, 7, 9, 71, 0, 64, 65, 8, 70, 3, 5, 1, 69, 72, 67, 66, 2, 4, 68], [123, 62, 111, 127, 121, 51, 115, 118, 63, 60, 47, 125, 53, 61, 124, 119, 126, 106, 117, 56, 57, 48, 107, 120, 54, 59, 38, 50, 122, 114, 55, 49, 42, 116, 22, 102, 58, 103, 86, 52, 113, 112, 39, 43, 46, 44, 97, 91, 33, 93, 100, 90, 110, 92, 45, 27, 108, 36, 88, 29, 109, 24, 40, 25, 105, 104, 19, 83, 87, 82, 35, 32, 26, 16, 89, 28, 99, 80, 18, 41, 13, 84, 34, 21, 37, 20, 17, 96, 30, 12, 81, 79, 23, 85, 14, 94, 76, 15, 78, 98, 101, 77, 31, 10, 74, 11, 95, 73, 9, 67, 75, 72, 6, 71, 70, 69, 68, 64, 7, 1, 0, 3, 65, 2, 4, 66, 5, 8], [123, 111, 62, 127, 121, 115, 51, 118, 63, 60, 47, 125, 53, 61, 117, 124, 56, 106, 120, 57, 119, 54, 58, 107, 59, 38, 126, 42, 116, 55, 102, 122, 114, 103, 50, 48, 113, 49, 39, 22, 112, 43, 86, 52, 46, 100, 97, 91, 33, 44, 93, 108, 92, 110, 45, 88, 104, 24, 36, 90, 29, 109, 40, 99, 27, 83, 25, 32, 19, 89, 82, 35, 105, 87, 26, 37, 18, 84, 21, 16, 101, 41, 30, 28, 98, 80, 96, 34, 23, 85, 94, 20, 31, 15, 81, 14, 17, 79, 13, 95, 12, 10, 77, 76, 6, 78, 9, 11, 67, 74, 75, 73, 4, 65, 0, 72, 71, 64, 7, 1, 5, 68, 3, 69, 66, 2, 8, 70], [123, 62, 111, 127, 121, 51, 63, 115, 60, 118, 47, 125, 53, 61, 124, 117, 106, 120, 59, 107, 56, 119, 57, 126, 54, 48, 116, 58, 114, 122, 38, 50, 22, 102, 42, 49, 39, 55, 86, 103, 112, 113, 52, 43, 44, 46, 97, 33, 100, 92, 93, 91, 90, 45, 27, 24, 110, 88, 29, 36, 82, 104, 19, 89, 35, 99, 26, 109, 108, 32, 28, 40, 41, 25, 83, 16, 18, 84, 87, 80, 21, 96, 23, 20, 105, 12, 30, 37, 98, 31, 13, 76, 94, 77, 81, 15, 101, 34, 17, 11, 79, 10, 14, 85, 74, 6, 95, 78, 9, 73, 75, 72, 71, 7, 3, 1, 4, 69, 64, 65, 70, 67, 5, 0, 66, 8, 68, 2], [123, 62, 111, 127, 121, 51, 115, 63, 60, 118, 47, 125, 53, 124, 61, 54, 117, 106, 119, 59, 126, 120, 122, 114, 107, 57, 50, 55, 116, 56, 49, 48, 22, 58, 38, 42, 86, 43, 102, 39, 52, 113, 103, 112, 46, 93, 91, 44, 33, 90, 92, 97, 27, 100, 45, 24, 89, 36, 88, 110, 29, 26, 108, 82, 109, 19, 40, 32, 35, 83, 28, 41, 104, 84, 18, 99, 21, 80, 16, 25, 23, 87, 17, 20, 12, 37, 13, 30, 96, 31, 98, 81, 79, 34, 76, 10, 15, 105, 94, 101, 77, 74, 85, 14, 78, 73, 7, 11, 75, 9, 3, 95, 1, 72, 6, 71, 0, 4, 69, 64, 68, 67, 65, 5, 70, 8, 2, 66], [123, 62, 111, 127, 121, 51, 118, 60, 63, 115, 47, 125, 53, 124, 61, 120, 106, 126, 59, 119, 117, 116, 54, 56, 122, 107, 58, 49, 38, 57, 114, 48, 55, 42, 103, 102, 39, 50, 113, 43, 22, 112, 52, 86, 46, 91, 93, 44, 90, 100, 97, 110, 33, 92, 24, 45, 108, 27, 36, 88, 29, 26, 32, 109, 89, 19, 104, 35, 99, 41, 83, 82, 40, 18, 28, 87, 25, 23, 16, 30, 85, 84, 96, 80, 31, 10, 21, 37, 15, 105, 17, 101, 13, 12, 94, 78, 20, 76, 98, 34, 77, 81, 79, 95, 14, 73, 75, 72, 74, 3, 11, 9, 71, 70, 64, 65, 1, 7, 0, 6, 68, 66, 67, 5, 69, 4, 8, 2], [123, 62, 111, 127, 121, 51, 60, 63, 115, 118, 47, 125, 53, 124, 117, 54, 61, 106, 59, 114, 120, 107, 126, 122, 56, 57, 119, 116, 58, 48, 39, 103, 102, 55, 38, 22, 42, 49, 50, 112, 52, 86, 113, 43, 93, 97, 100, 110, 46, 33, 44, 90, 91, 92, 45, 27, 36, 24, 108, 88, 89, 104, 40, 29, 26, 109, 99, 32, 83, 82, 25, 87, 35, 19, 28, 84, 30, 21, 16, 41, 80, 18, 96, 23, 105, 37, 101, 31, 78, 85, 15, 12, 20, 13, 94, 76, 10, 34, 95, 81, 14, 98, 79, 17, 77, 73, 11, 70, 74, 9, 4, 1, 65, 71, 0, 67, 75, 72, 3, 64, 69, 7, 2, 68, 66, 6, 5, 8], [123, 62, 111, 127, 121, 51, 60, 63, 118, 115, 47, 125, 53, 61, 124, 59, 106, 54, 117, 56, 114, 107, 126, 119, 58, 116, 57, 48, 120, 122, 49, 38, 22, 42, 55, 103, 50, 102, 52, 39, 86, 43, 113, 112, 46, 93, 92, 90, 33, 97, 91, 100, 27, 88, 44, 24, 45, 110, 36, 29, 26, 82, 89, 109, 25, 99, 83, 28, 18, 41, 80, 108, 19, 87, 32, 20, 104, 105, 40, 35, 96, 31, 16, 30, 79, 37, 84, 81, 21, 94, 12, 73, 14, 17, 76, 85, 13, 23, 77, 10, 34, 15, 74, 78, 101, 11, 98, 70, 7, 95, 75, 71, 9, 0, 3, 67, 72, 69, 4, 65, 5, 1, 66, 6, 68, 8, 64, 2], [123, 62, 111, 127, 121, 51, 118, 60, 115, 63, 47, 125, 61, 53, 124, 57, 117, 56, 106, 120, 122, 54, 126, 119, 59, 107, 48, 50, 116, 58, 22, 49, 114, 42, 38, 86, 103, 102, 55, 52, 113, 39, 112, 43, 46, 33, 110, 45, 97, 91, 100, 93, 90, 44, 92, 108, 27, 109, 88, 24, 83, 19, 99, 26, 29, 40, 89, 36, 18, 25, 82, 35, 41, 104, 21, 30, 16, 28, 87, 23, 32, 84, 12, 31, 80, 105, 94, 77, 101, 20, 13, 37, 79, 85, 17, 15, 81, 14, 96, 98, 10, 74, 78, 76, 73, 34, 70, 95, 7, 9, 11, 75, 64, 71, 1, 0, 72, 4, 3, 65, 69, 67, 66, 68, 5, 8, 2, 6], [123, 62, 111, 127, 121, 118, 51, 115, 60, 63, 47, 125, 61, 53, 124, 117, 54, 106, 57, 59, 56, 107, 120, 126, 58, 119, 122, 48, 50, 116, 49, 112, 114, 103, 38, 102, 22, 55, 42, 86, 39, 113, 52, 43, 46, 93, 44, 97, 45, 100, 92, 33, 110, 91, 90, 24, 27, 108, 88, 40, 19, 29, 109, 89, 26, 25, 99, 36, 83, 82, 28, 35, 87, 18, 21, 84, 41, 16, 105, 30, 31, 104, 32, 80, 85, 94, 17, 96, 15, 12, 98, 81, 20, 101, 23, 77, 37, 13, 76, 10, 79, 34, 95, 78, 74, 14, 11, 9, 70, 75, 73, 7, 8, 68, 71, 3, 1, 65, 64, 0, 6, 72, 67, 5, 69, 2, 4, 66], [123, 62, 111, 127, 121, 51, 118, 115, 60, 63, 47, 125, 61, 53, 124, 120, 117, 59, 106, 54, 56, 57, 114, 107, 122, 50, 48, 116, 126, 38, 58, 49, 119, 102, 22, 42, 55, 103, 113, 86, 52, 112, 39, 43, 46, 110, 97, 91, 93, 45, 44, 33, 108, 100, 92, 99, 24, 88, 90, 19, 36, 27, 40, 26, 89, 29, 109, 35, 28, 105, 104, 18, 25, 30, 83, 80, 31, 82, 101, 87, 16, 23, 21, 20, 32, 37, 85, 84, 41, 12, 94, 13, 79, 17, 78, 10, 76, 34, 77, 96, 98, 81, 15, 95, 74, 9, 11, 14, 73, 75, 7, 70, 8, 67, 6, 71, 3, 1, 64, 0, 69, 5, 4, 68, 65, 66, 2, 72], [123, 62, 111, 127, 121, 51, 60, 115, 118, 63, 47, 125, 61, 53, 124, 106, 54, 117, 120, 56, 50, 114, 107, 59, 57, 119, 48, 126, 22, 122, 58, 116, 42, 49, 38, 86, 113, 39, 102, 103, 52, 112, 55, 43, 97, 33, 92, 44, 91, 93, 110, 90, 45, 88, 46, 100, 108, 27, 82, 19, 29, 26, 36, 24, 89, 83, 109, 40, 18, 99, 16, 25, 28, 87, 80, 20, 32, 104, 23, 35, 105, 13, 17, 21, 30, 94, 84, 41, 37, 85, 31, 12, 10, 81, 78, 76, 79, 9, 96, 101, 77, 15, 34, 95, 74, 98, 6, 14, 11, 67, 75, 8, 7, 73, 0, 65, 71, 1, 3, 4, 70, 5, 64, 68, 66, 2, 69, 72], [123, 111, 62, 127, 51, 121, 60, 115, 118, 63, 47, 125, 61, 124, 53, 56, 106, 54, 117, 48, 116, 57, 114, 50, 126, 119, 107, 42, 59, 120, 122, 58, 113, 38, 102, 22, 49, 86, 112, 52, 103, 39, 55, 43, 110, 33, 97, 91, 46, 100, 44, 93, 92, 88, 90, 27, 24, 36, 45, 29, 99, 40, 89, 108, 109, 32, 26, 104, 18, 41, 19, 83, 25, 82, 28, 21, 35, 80, 94, 96, 12, 15, 105, 23, 30, 101, 87, 17, 37, 81, 16, 13, 85, 20, 31, 84, 98, 76, 74, 77, 10, 95, 78, 34, 9, 14, 6, 79, 64, 73, 11, 65, 7, 75, 3, 4, 1, 67, 70, 71, 68, 5, 8, 66, 69, 2, 0, 72], [123, 111, 62, 127, 121, 51, 115, 60, 118, 63, 47, 125, 53, 61, 124, 106, 56, 48, 57, 120, 126, 117, 50, 119, 59, 54, 107, 114, 42, 116, 22, 122, 58, 103, 86, 38, 102, 49, 55, 39, 112, 113, 52, 43, 44, 33, 110, 93, 46, 100, 97, 92, 88, 91, 27, 90, 108, 36, 24, 19, 45, 29, 40, 89, 26, 109, 32, 82, 99, 83, 104, 18, 28, 87, 25, 16, 35, 85, 76, 105, 12, 15, 84, 80, 21, 30, 96, 41, 37, 79, 81, 77, 17, 98, 23, 10, 74, 13, 14, 20, 94, 101, 95, 73, 78, 9, 6, 31, 11, 34, 75, 8, 71, 1, 67, 4, 66, 64, 7, 65, 3, 68, 5, 0, 70, 69, 72, 2], [123, 62, 111, 127, 121, 51, 60, 115, 63, 118, 47, 125, 53, 61, 124, 56, 106, 117, 48, 57, 126, 120, 119, 59, 114, 54, 50, 107, 116, 122, 49, 42, 38, 22, 58, 112, 86, 103, 102, 113, 55, 52, 43, 39, 33, 100, 46, 110, 91, 97, 93, 108, 90, 44, 92, 27, 88, 109, 45, 36, 29, 40, 83, 82, 26, 89, 105, 19, 32, 104, 99, 18, 16, 24, 80, 87, 41, 35, 25, 94, 30, 28, 77, 84, 37, 21, 101, 17, 15, 74, 98, 81, 79, 12, 6, 76, 96, 85, 14, 20, 31, 23, 10, 75, 13, 78, 9, 71, 11, 95, 73, 34, 7, 8, 0, 3, 68, 67, 1, 65, 4, 70, 64, 2, 69, 66, 72, 5], [123, 62, 111, 127, 121, 60, 51, 115, 118, 63, 47, 125, 61, 53, 124, 120, 106, 57, 117, 48, 59, 56, 54, 126, 116, 107, 114, 122, 119, 22, 42, 38, 103, 86, 102, 39, 58, 50, 49, 112, 52, 55, 113, 100, 43, 46, 110, 33, 97, 91, 93, 92, 44, 88, 90, 108, 27, 45, 89, 19, 24, 29, 26, 99, 36, 83, 16, 25, 105, 32, 104, 28, 109, 18, 30, 80, 87, 84, 82, 35, 40, 76, 77, 20, 12, 81, 85, 79, 21, 17, 15, 37, 23, 41, 94, 10, 14, 74, 13, 73, 101, 96, 75, 78, 31, 34, 98, 11, 6, 9, 7, 8, 71, 95, 5, 70, 67, 3, 4, 1, 72, 69, 64, 65, 0, 68, 66, 2]], "model.layers.14.self_attn.q_proj": [[60, 120, 62, 37, 51, 63, 33, 53, 90, 101, 30, 118, 117, 87, 19, 86, 57, 125, 58, 85, 74, 93, 123, 111, 109, 115, 119, 59, 54, 92, 21, 116, 121, 12, 26, 50, 52, 114, 42, 44, 29, 126, 61, 91, 127, 122, 45, 55, 56, 48, 39, 105, 124, 6, 110, 15, 49, 4, 81, 43, 97, 112, 46, 36, 113, 25, 108, 20, 104, 94, 38, 47, 17, 64, 106, 88, 107, 71, 41, 18, 13, 103, 78, 23, 99, 34, 102, 22, 16, 2, 98, 40, 32, 65, 28, 10, 89, 72, 100, 95, 31, 1, 27, 35, 14, 79, 96, 68, 83, 24, 84, 11, 82, 66, 76, 80, 70, 69, 77, 75, 7, 8, 3, 0, 73, 67, 5, 9], [60, 120, 62, 37, 33, 90, 30, 118, 19, 101, 87, 117, 51, 53, 86, 59, 115, 57, 119, 92, 58, 111, 121, 126, 85, 54, 12, 15, 63, 116, 123, 26, 127, 48, 114, 56, 122, 74, 125, 46, 105, 71, 52, 43, 81, 38, 97, 110, 61, 42, 124, 83, 41, 55, 44, 45, 93, 50, 94, 39, 88, 69, 108, 112, 102, 40, 109, 103, 106, 113, 47, 20, 6, 29, 13, 49, 107, 1, 79, 28, 2, 99, 22, 17, 25, 104, 31, 95, 34, 14, 91, 23, 8, 66, 36, 21, 24, 89, 7, 72, 64, 32, 82, 100, 76, 16, 84, 98, 35, 96, 80, 75, 4, 18, 78, 11, 10, 70, 27, 65, 67, 9, 77, 5, 73, 3, 68, 0], [60, 120, 37, 62, 90, 118, 51, 101, 33, 19, 53, 87, 117, 30, 86, 42, 74, 4, 12, 59, 93, 111, 21, 114, 58, 39, 116, 63, 46, 92, 113, 54, 57, 126, 50, 121, 115, 26, 52, 105, 2, 49, 125, 15, 123, 25, 119, 127, 45, 48, 43, 122, 94, 91, 88, 61, 17, 110, 83, 103, 112, 56, 71, 55, 124, 85, 109, 38, 41, 81, 98, 47, 106, 104, 40, 14, 108, 102, 44, 6, 8, 107, 78, 29, 34, 99, 79, 76, 28, 23, 64, 72, 31, 69, 22, 89, 18, 65, 35, 66, 84, 36, 16, 20, 97, 1, 32, 68, 100, 24, 27, 77, 96, 95, 7, 82, 73, 80, 13, 11, 10, 75, 5, 70, 0, 67, 9, 3], [120, 60, 37, 62, 118, 90, 33, 117, 30, 51, 101, 53, 111, 59, 19, 87, 63, 86, 58, 57, 85, 105, 121, 48, 115, 92, 54, 42, 12, 126, 116, 55, 122, 119, 123, 46, 125, 52, 56, 26, 93, 74, 61, 127, 110, 108, 50, 112, 124, 114, 88, 41, 69, 109, 49, 113, 47, 91, 15, 82, 4, 21, 79, 44, 107, 43, 106, 81, 45, 39, 2, 25, 97, 104, 103, 94, 28, 38, 64, 71, 22, 36, 83, 78, 29, 31, 17, 6, 13, 40, 1, 100, 66, 99, 76, 102, 35, 34, 95, 32, 24, 72, 98, 84, 23, 96, 73, 18, 16, 89, 75, 68, 10, 65, 8, 27, 20, 0, 14, 11, 80, 70, 5, 67, 9, 77, 7, 3], [52, 102, 51, 124, 125, 29, 33, 60, 116, 84, 61, 123, 26, 86, 87, 88, 56, 119, 62, 93, 55, 38, 107, 120, 118, 106, 114, 115, 81, 39, 57, 122, 113, 109, 127, 37, 45, 94, 50, 53, 110, 58, 121, 28, 20, 54, 42, 63, 101, 105, 111, 41, 103, 112, 99, 35, 49, 108, 36, 40, 104, 97, 48, 22, 44, 43, 117, 34, 90, 59, 46, 47, 85, 25, 15, 100, 126, 98, 95, 73, 14, 32, 24, 21, 96, 82, 30, 31, 23, 92, 17, 18, 91, 11, 27, 71, 89, 80, 83, 74, 19, 16, 10, 7, 13, 5, 67, 78, 79, 9, 77, 8, 76, 12, 72, 75, 69, 66, 68, 65, 6, 64, 1, 4, 2, 70, 3, 0], [52, 102, 124, 33, 29, 125, 87, 60, 61, 116, 88, 123, 56, 38, 55, 119, 51, 73, 84, 45, 101, 62, 107, 57, 86, 28, 112, 106, 115, 39, 113, 110, 50, 53, 105, 127, 63, 67, 26, 118, 120, 42, 36, 54, 121, 114, 49, 58, 108, 43, 117, 109, 64, 47, 46, 126, 48, 40, 94, 122, 111, 44, 41, 59, 15, 93, 32, 103, 104, 97, 24, 22, 83, 99, 11, 100, 91, 20, 98, 90, 37, 35, 65, 81, 95, 34, 82, 31, 3, 25, 66, 96, 69, 27, 30, 23, 92, 71, 21, 89, 13, 5, 7, 9, 17, 79, 77, 19, 1, 6, 76, 18, 85, 10, 80, 78, 75, 14, 72, 74, 4, 0, 16, 12, 70, 68, 2, 8], [52, 102, 33, 124, 86, 29, 116, 88, 125, 61, 84, 82, 60, 73, 26, 87, 15, 56, 38, 66, 24, 93, 123, 51, 81, 14, 119, 90, 105, 55, 20, 62, 28, 19, 50, 110, 9, 112, 57, 113, 18, 10, 95, 13, 120, 43, 91, 54, 5, 106, 21, 45, 79, 99, 2, 22, 53, 63, 121, 49, 77, 108, 115, 85, 31, 101, 30, 118, 127, 114, 83, 4, 107, 23, 34, 78, 39, 75, 48, 17, 11, 25, 40, 117, 64, 98, 58, 27, 122, 12, 109, 0, 71, 44, 7, 72, 37, 94, 42, 70, 92, 111, 126, 16, 59, 103, 97, 69, 35, 46, 47, 1, 76, 89, 36, 41, 104, 80, 96, 32, 3, 100, 8, 6, 74, 68, 67, 65], [52, 102, 33, 29, 125, 51, 61, 26, 86, 124, 56, 116, 88, 82, 60, 84, 81, 87, 93, 15, 71, 123, 20, 119, 85, 11, 120, 55, 38, 73, 110, 113, 97, 22, 32, 62, 17, 75, 28, 90, 103, 14, 57, 24, 13, 50, 19, 76, 5, 54, 106, 109, 115, 127, 10, 112, 67, 25, 18, 42, 63, 118, 77, 48, 122, 45, 49, 79, 121, 107, 27, 58, 23, 3, 111, 108, 16, 31, 37, 104, 46, 89, 114, 43, 39, 105, 101, 53, 40, 36, 98, 94, 30, 117, 64, 41, 7, 126, 35, 21, 47, 83, 6, 80, 9, 91, 92, 1, 44, 100, 65, 99, 59, 34, 74, 70, 96, 95, 78, 12, 72, 69, 68, 4, 8, 66, 2, 0], [103, 97, 53, 117, 23, 2, 80, 11, 9, 85, 65, 0, 69, 81, 83, 67, 51, 14, 4, 3, 12, 71, 116, 70, 1, 7, 124, 61, 127, 112, 64, 87, 118, 60, 106, 6, 122, 73, 50, 5, 89, 113, 126, 66, 74, 13, 55, 114, 25, 68, 121, 54, 120, 24, 88, 8, 39, 63, 30, 91, 10, 49, 76, 15, 78, 28, 75, 17, 59, 108, 29, 45, 119, 104, 82, 58, 42, 79, 109, 31, 43, 52, 105, 110, 47, 77, 44, 46, 20, 19, 27, 123, 90, 72, 56, 34, 40, 57, 22, 125, 26, 16, 21, 98, 32, 102, 100, 99, 86, 37, 35, 107, 101, 84, 41, 62, 95, 36, 93, 18, 38, 48, 92, 33, 111, 94, 115, 96], [103, 97, 53, 117, 106, 81, 80, 87, 14, 12, 85, 23, 74, 4, 2, 112, 51, 83, 7, 11, 127, 116, 9, 124, 118, 69, 0, 54, 60, 61, 3, 25, 15, 24, 113, 50, 55, 70, 65, 126, 88, 122, 64, 30, 45, 121, 120, 114, 63, 6, 1, 76, 18, 75, 29, 72, 73, 5, 71, 68, 28, 89, 91, 39, 49, 26, 78, 58, 16, 8, 10, 67, 59, 44, 40, 19, 109, 110, 21, 52, 105, 17, 82, 98, 92, 66, 108, 42, 86, 46, 32, 31, 79, 27, 41, 13, 77, 36, 119, 95, 90, 125, 43, 84, 47, 100, 48, 34, 57, 107, 62, 102, 38, 22, 104, 99, 37, 101, 20, 35, 115, 93, 56, 94, 96, 111, 123, 33], [103, 97, 117, 53, 85, 87, 80, 51, 58, 14, 83, 12, 106, 11, 27, 16, 10, 23, 24, 5, 127, 124, 49, 81, 113, 104, 76, 43, 116, 118, 112, 45, 54, 71, 57, 91, 25, 88, 120, 86, 108, 122, 17, 109, 60, 50, 66, 47, 78, 55, 121, 110, 92, 63, 114, 52, 6, 21, 61, 44, 99, 125, 119, 62, 40, 105, 42, 123, 38, 37, 111, 89, 126, 56, 41, 19, 101, 102, 75, 98, 72, 34, 59, 9, 29, 115, 67, 100, 46, 68, 20, 22, 4, 30, 48, 36, 107, 26, 32, 1, 94, 95, 28, 82, 35, 18, 93, 90, 77, 96, 64, 31, 39, 2, 15, 73, 69, 79, 8, 84, 13, 7, 70, 74, 33, 65, 3, 0], [103, 97, 53, 117, 81, 11, 85, 14, 87, 106, 23, 69, 80, 12, 9, 2, 88, 60, 127, 7, 5, 124, 3, 51, 74, 112, 116, 0, 65, 70, 61, 4, 28, 126, 24, 118, 16, 29, 121, 83, 15, 122, 22, 45, 54, 44, 30, 120, 6, 25, 105, 114, 21, 79, 49, 71, 78, 50, 75, 67, 10, 17, 89, 63, 76, 55, 73, 64, 113, 40, 8, 1, 19, 98, 66, 86, 77, 58, 46, 109, 27, 82, 42, 108, 72, 13, 92, 104, 91, 34, 110, 84, 59, 62, 20, 39, 31, 125, 52, 18, 123, 90, 68, 41, 26, 32, 93, 94, 96, 107, 95, 48, 57, 99, 43, 119, 101, 38, 111, 37, 36, 35, 100, 47, 102, 56, 115, 33], [56, 102, 127, 24, 17, 93, 44, 60, 78, 11, 61, 33, 86, 88, 29, 116, 113, 108, 90, 50, 35, 114, 115, 47, 19, 59, 6, 62, 92, 123, 31, 40, 26, 72, 63, 119, 54, 20, 46, 120, 118, 122, 112, 126, 49, 104, 51, 111, 53, 124, 99, 121, 117, 23, 125, 37, 48, 55, 85, 57, 110, 36, 109, 42, 106, 32, 34, 43, 94, 45, 58, 30, 103, 39, 107, 52, 38, 8, 16, 105, 41, 100, 83, 89, 87, 96, 91, 95, 81, 98, 79, 28, 101, 82, 70, 25, 14, 27, 74, 21, 4, 10, 22, 3, 84, 77, 97, 75, 71, 76, 13, 18, 12, 80, 2, 15, 66, 69, 64, 67, 9, 7, 5, 0, 73, 68, 1, 65], [56, 102, 127, 24, 61, 113, 93, 60, 116, 114, 50, 108, 115, 63, 62, 59, 47, 126, 122, 88, 123, 119, 90, 38, 120, 112, 49, 53, 125, 46, 51, 121, 55, 124, 111, 117, 54, 42, 33, 48, 44, 39, 110, 57, 52, 58, 118, 109, 17, 103, 45, 30, 105, 43, 32, 81, 104, 40, 36, 78, 29, 75, 107, 106, 37, 84, 35, 41, 96, 34, 22, 92, 86, 19, 66, 94, 99, 101, 26, 100, 87, 2, 8, 80, 83, 98, 20, 11, 95, 91, 31, 85, 28, 97, 27, 79, 14, 73, 89, 15, 16, 6, 72, 3, 23, 64, 70, 77, 25, 21, 76, 5, 68, 82, 4, 13, 74, 9, 0, 18, 1, 67, 10, 12, 7, 65, 71, 69], [56, 108, 102, 127, 93, 9, 33, 24, 17, 61, 90, 94, 113, 60, 103, 1, 116, 88, 86, 39, 115, 65, 2, 50, 30, 114, 47, 59, 120, 62, 34, 23, 85, 123, 122, 126, 119, 69, 63, 87, 44, 55, 46, 73, 111, 104, 3, 0, 121, 18, 20, 64, 112, 31, 38, 54, 53, 49, 96, 68, 91, 51, 109, 98, 124, 36, 79, 95, 5, 48, 11, 117, 26, 37, 13, 57, 32, 77, 28, 35, 125, 42, 58, 52, 21, 7, 110, 118, 8, 27, 97, 10, 74, 4, 106, 105, 100, 16, 83, 72, 99, 66, 107, 15, 25, 40, 29, 19, 92, 45, 89, 82, 70, 71, 101, 84, 78, 6, 41, 67, 80, 22, 43, 76, 12, 81, 75, 14], [56, 102, 33, 108, 127, 93, 17, 24, 88, 113, 61, 90, 11, 86, 60, 82, 116, 78, 35, 59, 29, 20, 6, 47, 114, 50, 26, 44, 38, 123, 62, 122, 115, 120, 16, 106, 37, 21, 34, 63, 95, 119, 30, 126, 112, 87, 76, 121, 55, 49, 94, 51, 27, 53, 54, 28, 36, 83, 85, 80, 9, 75, 72, 89, 117, 124, 25, 110, 48, 91, 111, 125, 81, 79, 46, 42, 22, 104, 45, 103, 118, 105, 43, 57, 58, 96, 109, 98, 39, 52, 40, 99, 18, 32, 12, 15, 97, 23, 107, 31, 92, 84, 5, 7, 19, 100, 101, 74, 41, 77, 10, 70, 13, 73, 64, 14, 4, 3, 0, 67, 71, 2, 66, 69, 8, 68, 1, 65], [123, 39, 113, 69, 3, 64, 112, 1, 77, 71, 49, 63, 29, 13, 48, 111, 91, 61, 73, 98, 122, 54, 60, 119, 120, 4, 11, 72, 124, 88, 82, 24, 53, 115, 101, 56, 78, 22, 66, 62, 95, 17, 121, 59, 57, 81, 125, 55, 65, 83, 40, 118, 43, 127, 114, 93, 51, 8, 94, 70, 68, 46, 92, 15, 116, 106, 96, 36, 89, 47, 50, 35, 76, 2, 99, 52, 110, 97, 109, 102, 108, 44, 80, 45, 117, 104, 107, 126, 38, 41, 28, 9, 105, 58, 74, 37, 32, 26, 23, 33, 10, 100, 21, 42, 0, 25, 67, 12, 87, 19, 6, 31, 30, 90, 16, 7, 18, 20, 27, 84, 103, 5, 34, 75, 85, 79, 14, 86], [113, 123, 39, 13, 73, 71, 17, 83, 115, 74, 92, 78, 76, 59, 60, 55, 126, 63, 114, 57, 120, 127, 47, 116, 52, 62, 95, 119, 118, 117, 53, 121, 54, 69, 112, 82, 51, 56, 99, 46, 122, 109, 50, 124, 3, 58, 111, 48, 6, 100, 16, 125, 49, 110, 61, 108, 45, 88, 11, 44, 75, 106, 37, 107, 42, 43, 31, 86, 28, 72, 21, 98, 94, 15, 84, 41, 105, 67, 104, 93, 5, 36, 68, 85, 65, 38, 102, 91, 101, 40, 35, 97, 30, 25, 1, 64, 90, 22, 103, 33, 96, 87, 20, 32, 23, 29, 34, 2, 80, 8, 27, 89, 18, 81, 24, 14, 4, 26, 0, 77, 12, 19, 79, 10, 70, 66, 7, 9], [123, 39, 113, 112, 49, 32, 54, 95, 60, 99, 63, 115, 120, 90, 83, 124, 122, 48, 43, 13, 121, 94, 87, 125, 119, 61, 98, 56, 53, 111, 57, 85, 40, 127, 17, 38, 59, 62, 118, 36, 55, 114, 102, 105, 37, 110, 97, 92, 101, 81, 26, 47, 104, 109, 28, 35, 51, 45, 46, 96, 108, 116, 107, 50, 30, 41, 100, 106, 74, 52, 117, 126, 31, 44, 23, 42, 58, 82, 93, 33, 73, 25, 91, 29, 22, 84, 6, 21, 78, 18, 79, 86, 89, 34, 88, 11, 27, 24, 14, 15, 19, 76, 16, 75, 103, 20, 69, 10, 71, 8, 80, 72, 66, 77, 7, 3, 4, 70, 68, 12, 67, 64, 2, 1, 65, 9, 0, 5], [113, 123, 39, 119, 52, 120, 63, 126, 121, 118, 53, 57, 115, 106, 114, 47, 56, 13, 59, 116, 55, 51, 60, 58, 122, 127, 117, 111, 46, 50, 54, 92, 48, 109, 110, 62, 45, 43, 124, 108, 107, 61, 44, 74, 42, 112, 105, 125, 73, 49, 69, 101, 82, 41, 76, 64, 83, 99, 21, 17, 102, 104, 78, 5, 38, 71, 91, 1, 6, 40, 3, 66, 70, 72, 37, 100, 90, 30, 97, 8, 25, 81, 36, 15, 4, 89, 93, 26, 75, 35, 16, 96, 80, 19, 2, 11, 88, 33, 68, 14, 9, 84, 79, 98, 27, 87, 86, 32, 31, 67, 24, 18, 103, 10, 95, 20, 94, 12, 28, 65, 29, 85, 34, 23, 22, 7, 77, 0], [62, 63, 39, 53, 127, 32, 57, 36, 54, 121, 107, 47, 116, 46, 115, 59, 52, 55, 50, 51, 41, 105, 58, 61, 123, 96, 60, 125, 91, 27, 113, 122, 120, 106, 48, 124, 44, 103, 117, 126, 114, 98, 49, 119, 56, 118, 110, 45, 85, 43, 35, 38, 89, 112, 109, 101, 104, 40, 111, 108, 81, 37, 25, 42, 99, 22, 100, 34, 102, 92, 94, 87, 19, 33, 31, 23, 95, 0, 93, 21, 86, 26, 20, 97, 78, 17, 10, 80, 12, 29, 28, 83, 90, 66, 74, 65, 77, 82, 15, 30, 64, 24, 71, 72, 18, 69, 9, 84, 67, 76, 6, 1, 3, 68, 13, 11, 5, 4, 8, 14, 16, 88, 2, 75, 79, 70, 7, 73], [39, 63, 62, 24, 84, 94, 33, 18, 53, 86, 26, 88, 92, 80, 32, 78, 90, 127, 14, 29, 75, 73, 23, 30, 16, 121, 67, 17, 57, 82, 70, 54, 9, 95, 107, 1, 98, 116, 58, 47, 115, 15, 50, 27, 59, 52, 21, 22, 74, 60, 20, 55, 46, 83, 11, 51, 93, 8, 13, 85, 61, 31, 34, 125, 96, 123, 124, 91, 28, 79, 117, 113, 120, 4, 64, 77, 41, 76, 81, 5, 89, 25, 6, 122, 42, 72, 48, 12, 2, 126, 36, 35, 65, 118, 49, 102, 38, 87, 110, 101, 103, 114, 56, 19, 71, 119, 106, 108, 45, 37, 7, 97, 104, 100, 109, 111, 44, 112, 43, 105, 99, 3, 10, 40, 69, 68, 66, 0], [63, 62, 39, 53, 127, 57, 32, 54, 116, 121, 52, 59, 55, 47, 50, 115, 125, 61, 46, 51, 58, 60, 96, 123, 120, 122, 90, 113, 91, 86, 117, 124, 41, 48, 126, 34, 119, 118, 56, 106, 114, 92, 27, 109, 107, 101, 49, 110, 26, 35, 33, 89, 45, 21, 111, 102, 105, 108, 100, 36, 85, 112, 93, 44, 42, 43, 104, 99, 40, 98, 22, 29, 81, 88, 37, 10, 94, 64, 38, 103, 95, 16, 66, 65, 31, 19, 0, 97, 14, 25, 30, 67, 83, 72, 17, 24, 76, 87, 20, 6, 23, 74, 28, 78, 68, 69, 9, 12, 8, 15, 11, 84, 1, 5, 7, 79, 77, 82, 80, 4, 71, 18, 70, 73, 2, 3, 13, 75], [39, 62, 63, 94, 84, 24, 88, 53, 86, 21, 32, 74, 14, 10, 33, 26, 83, 90, 18, 127, 8, 92, 20, 69, 30, 23, 36, 57, 16, 7, 76, 66, 68, 121, 67, 54, 96, 27, 116, 89, 73, 87, 13, 52, 47, 82, 55, 28, 72, 115, 59, 17, 50, 6, 85, 77, 60, 70, 58, 34, 22, 80, 46, 95, 5, 81, 65, 61, 107, 125, 29, 51, 98, 49, 123, 93, 91, 9, 117, 124, 101, 108, 19, 120, 79, 25, 11, 113, 78, 15, 31, 38, 35, 64, 12, 103, 105, 99, 118, 41, 110, 106, 48, 122, 56, 100, 45, 97, 126, 37, 119, 104, 114, 40, 71, 43, 109, 4, 44, 102, 42, 111, 112, 1, 75, 0, 3, 2], [113, 48, 101, 112, 26, 94, 87, 18, 123, 20, 49, 86, 122, 59, 120, 79, 57, 62, 124, 58, 55, 60, 54, 115, 76, 43, 125, 37, 53, 119, 6, 39, 45, 51, 117, 126, 52, 114, 63, 61, 56, 50, 118, 106, 30, 75, 98, 34, 88, 32, 104, 107, 111, 127, 47, 110, 41, 116, 121, 92, 82, 108, 33, 44, 29, 102, 40, 42, 15, 103, 89, 90, 96, 105, 109, 99, 46, 23, 36, 35, 24, 100, 17, 22, 13, 31, 19, 97, 28, 70, 25, 91, 16, 95, 85, 27, 84, 11, 78, 21, 38, 83, 93, 12, 81, 14, 77, 73, 67, 8, 74, 9, 72, 71, 80, 3, 10, 7, 5, 4, 65, 68, 69, 66, 1, 2, 0, 64], [48, 113, 101, 112, 26, 122, 18, 94, 59, 87, 20, 43, 123, 57, 76, 79, 30, 63, 92, 98, 49, 53, 61, 124, 51, 88, 120, 86, 58, 117, 119, 39, 60, 121, 55, 125, 62, 115, 6, 50, 52, 89, 54, 56, 126, 96, 107, 114, 45, 102, 110, 29, 118, 47, 82, 35, 127, 34, 40, 46, 106, 24, 111, 32, 42, 100, 116, 108, 105, 37, 109, 44, 75, 90, 103, 104, 93, 36, 97, 25, 41, 31, 13, 27, 22, 28, 16, 95, 99, 38, 19, 91, 83, 12, 23, 85, 81, 33, 15, 17, 84, 21, 14, 11, 9, 78, 70, 8, 72, 74, 73, 77, 80, 67, 3, 71, 10, 7, 5, 65, 68, 4, 69, 1, 66, 2, 0, 64], [48, 113, 74, 16, 101, 7, 13, 68, 64, 69, 4, 112, 2, 122, 0, 87, 5, 1, 66, 65, 94, 67, 86, 3, 20, 58, 71, 53, 54, 98, 125, 55, 107, 57, 70, 6, 60, 49, 56, 120, 119, 52, 114, 121, 32, 26, 18, 62, 117, 124, 50, 118, 10, 11, 75, 110, 127, 30, 63, 47, 89, 51, 45, 82, 105, 104, 115, 61, 80, 106, 8, 103, 28, 76, 44, 77, 73, 96, 72, 108, 12, 126, 111, 79, 59, 42, 9, 109, 123, 24, 90, 23, 46, 85, 84, 93, 81, 37, 15, 14, 22, 78, 29, 19, 116, 17, 40, 34, 102, 83, 27, 35, 43, 21, 91, 33, 88, 92, 31, 25, 97, 95, 41, 38, 39, 100, 99, 36], [48, 113, 101, 122, 94, 112, 87, 26, 18, 59, 20, 123, 86, 49, 43, 79, 98, 57, 19, 120, 124, 30, 89, 32, 8, 115, 51, 76, 107, 58, 39, 21, 97, 82, 61, 63, 60, 62, 88, 29, 13, 52, 42, 6, 125, 34, 53, 117, 56, 55, 37, 119, 111, 96, 54, 28, 106, 44, 100, 114, 16, 75, 108, 104, 102, 85, 118, 126, 50, 105, 17, 45, 36, 27, 103, 116, 127, 31, 40, 91, 110, 92, 25, 121, 35, 47, 41, 22, 23, 109, 38, 33, 83, 46, 90, 95, 99, 74, 84, 24, 93, 12, 14, 78, 81, 73, 15, 11, 9, 72, 70, 3, 67, 77, 7, 80, 71, 10, 69, 65, 4, 5, 68, 1, 66, 2, 0, 64], [56, 113, 63, 102, 117, 49, 124, 68, 53, 76, 59, 85, 55, 5, 114, 60, 116, 58, 94, 121, 61, 89, 16, 7, 2, 66, 72, 50, 62, 8, 9, 12, 111, 51, 80, 115, 123, 118, 6, 126, 122, 48, 77, 54, 120, 52, 101, 40, 125, 69, 87, 91, 46, 119, 21, 27, 1, 3, 4, 29, 30, 96, 127, 42, 112, 83, 105, 92, 43, 44, 57, 35, 39, 36, 75, 22, 45, 109, 17, 24, 107, 110, 14, 104, 108, 88, 103, 47, 18, 79, 25, 23, 78, 41, 64, 15, 28, 97, 37, 93, 106, 20, 99, 98, 95, 81, 19, 26, 33, 82, 90, 100, 10, 38, 70, 31, 11, 84, 34, 73, 32, 13, 67, 65, 74, 86, 71, 0], [113, 56, 63, 124, 53, 59, 117, 55, 9, 49, 40, 116, 94, 58, 81, 69, 121, 61, 77, 62, 50, 114, 48, 118, 54, 60, 64, 111, 125, 51, 115, 123, 122, 126, 2, 120, 1, 24, 43, 57, 127, 68, 8, 52, 12, 39, 46, 119, 21, 70, 101, 112, 3, 45, 105, 107, 104, 109, 110, 47, 106, 44, 102, 108, 96, 30, 79, 6, 42, 92, 85, 72, 41, 103, 26, 36, 33, 7, 37, 15, 98, 100, 66, 16, 35, 25, 65, 83, 74, 97, 34, 93, 73, 95, 99, 90, 4, 17, 31, 28, 88, 91, 75, 38, 89, 67, 19, 23, 10, 87, 0, 5, 29, 13, 20, 32, 71, 78, 84, 27, 18, 14, 11, 82, 80, 76, 86, 22], [113, 56, 63, 94, 124, 49, 102, 117, 24, 96, 53, 38, 59, 104, 19, 55, 92, 43, 40, 58, 26, 91, 89, 35, 9, 116, 48, 114, 118, 37, 121, 22, 62, 51, 61, 50, 15, 60, 30, 122, 97, 126, 87, 46, 111, 54, 69, 29, 123, 52, 83, 120, 1, 85, 33, 125, 127, 17, 36, 77, 98, 12, 64, 81, 68, 119, 115, 3, 73, 105, 57, 75, 90, 7, 112, 86, 45, 79, 39, 109, 2, 8, 108, 44, 99, 6, 110, 34, 95, 25, 28, 76, 100, 10, 106, 27, 16, 93, 101, 18, 82, 78, 14, 47, 42, 41, 103, 88, 70, 107, 23, 31, 13, 21, 72, 20, 4, 32, 84, 80, 11, 74, 66, 67, 71, 0, 65, 5], [113, 56, 63, 124, 49, 117, 53, 40, 59, 114, 55, 102, 58, 116, 48, 24, 50, 94, 26, 121, 51, 54, 96, 60, 62, 118, 61, 87, 89, 126, 35, 43, 120, 77, 122, 123, 81, 111, 125, 57, 109, 52, 105, 9, 46, 17, 115, 104, 29, 119, 127, 92, 97, 30, 112, 99, 44, 22, 70, 45, 91, 110, 1, 37, 42, 108, 107, 103, 100, 101, 47, 38, 36, 15, 39, 6, 41, 27, 85, 106, 3, 20, 31, 90, 98, 33, 34, 88, 14, 73, 28, 23, 95, 19, 72, 64, 68, 12, 93, 79, 18, 13, 16, 86, 32, 82, 21, 25, 8, 69, 83, 78, 74, 11, 84, 80, 76, 10, 2, 75, 7, 65, 66, 71, 5, 0, 4, 67]], "model.layers.14.self_attn.k_proj": [[60, 101, 97, 62, 86, 117, 120, 94, 51, 59, 53, 90, 118, 58, 126, 57, 48, 111, 123, 121, 115, 54, 116, 87, 61, 81, 114, 125, 119, 56, 52, 127, 124, 55, 107, 19, 110, 122, 105, 50, 63, 106, 112, 89, 37, 40, 46, 44, 109, 45, 47, 43, 49, 102, 88, 108, 113, 12, 103, 21, 92, 104, 78, 15, 25, 42, 41, 32, 38, 39, 16, 93, 1, 34, 74, 67, 100, 27, 36, 71, 70, 31, 95, 13, 91, 98, 35, 68, 0, 20, 83, 73, 24, 33, 99, 96, 14, 79, 84, 72, 22, 28, 29, 11, 85, 80, 26, 77, 30, 66, 18, 8, 23, 17, 76, 82, 10, 9, 75, 5, 69, 2, 7, 65, 64, 3, 6, 4], [52, 38, 97, 86, 60, 93, 124, 56, 119, 123, 125, 61, 50, 57, 26, 113, 110, 62, 121, 127, 55, 81, 58, 116, 63, 109, 112, 53, 45, 115, 117, 108, 42, 49, 15, 46, 54, 106, 48, 111, 122, 118, 120, 88, 47, 84, 33, 114, 126, 82, 107, 103, 43, 39, 59, 22, 40, 102, 41, 87, 37, 105, 85, 36, 44, 75, 83, 100, 96, 77, 35, 13, 101, 104, 9, 99, 0, 98, 31, 16, 27, 30, 92, 34, 25, 32, 95, 51, 80, 17, 89, 28, 94, 10, 76, 69, 65, 29, 19, 14, 21, 73, 70, 91, 2, 24, 78, 72, 12, 71, 7, 4, 18, 90, 67, 68, 23, 8, 20, 11, 79, 5, 3, 74, 1, 66, 6, 64], [53, 39, 33, 87, 117, 14, 81, 80, 7, 9, 12, 74, 0, 42, 11, 65, 85, 2, 3, 60, 4, 48, 69, 70, 114, 83, 116, 61, 51, 64, 127, 54, 66, 67, 122, 79, 126, 113, 55, 118, 63, 124, 52, 5, 72, 120, 68, 88, 91, 109, 89, 25, 115, 112, 121, 108, 6, 45, 103, 59, 40, 73, 71, 22, 49, 1, 41, 34, 105, 77, 104, 23, 26, 10, 21, 43, 106, 44, 47, 96, 13, 50, 30, 92, 16, 110, 119, 18, 93, 123, 28, 76, 20, 58, 125, 94, 75, 8, 78, 29, 102, 36, 24, 46, 101, 90, 111, 56, 97, 57, 107, 31, 95, 35, 99, 19, 37, 27, 98, 38, 84, 100, 62, 82, 17, 86, 32, 15], [56, 38, 127, 86, 97, 44, 60, 61, 119, 114, 50, 113, 122, 116, 123, 62, 110, 63, 29, 115, 126, 47, 112, 59, 120, 51, 90, 55, 121, 39, 53, 49, 57, 124, 24, 117, 58, 45, 30, 78, 48, 118, 54, 125, 6, 109, 111, 17, 11, 1, 103, 52, 46, 107, 101, 42, 0, 41, 102, 105, 106, 28, 23, 108, 89, 20, 40, 104, 16, 43, 33, 35, 100, 98, 76, 25, 96, 82, 68, 15, 84, 85, 99, 32, 77, 37, 18, 74, 95, 36, 2, 34, 91, 19, 7, 64, 21, 22, 12, 31, 80, 93, 27, 3, 92, 87, 10, 67, 13, 79, 72, 94, 9, 26, 71, 69, 5, 73, 8, 66, 81, 14, 75, 4, 65, 83, 70, 88], [123, 113, 103, 22, 34, 60, 122, 54, 124, 61, 53, 125, 111, 56, 57, 49, 62, 119, 121, 55, 18, 47, 63, 114, 127, 115, 51, 40, 59, 96, 112, 93, 117, 116, 52, 126, 120, 46, 58, 118, 50, 44, 48, 109, 45, 107, 42, 104, 27, 98, 110, 43, 75, 41, 108, 106, 14, 65, 38, 100, 7, 35, 105, 92, 68, 84, 102, 37, 101, 0, 26, 67, 80, 36, 15, 9, 32, 31, 5, 33, 24, 30, 8, 87, 97, 88, 77, 94, 99, 25, 95, 6, 79, 20, 10, 39, 28, 81, 72, 91, 23, 76, 83, 70, 2, 90, 89, 12, 29, 66, 78, 21, 82, 17, 16, 11, 85, 74, 64, 19, 71, 4, 69, 86, 3, 1, 73, 13], [63, 62, 103, 86, 127, 53, 57, 121, 54, 116, 55, 30, 59, 50, 52, 58, 115, 125, 123, 97, 120, 61, 92, 60, 46, 84, 18, 51, 113, 26, 117, 124, 96, 118, 47, 126, 48, 88, 49, 24, 110, 114, 122, 100, 56, 119, 80, 15, 108, 106, 91, 111, 45, 107, 42, 14, 112, 109, 101, 38, 41, 43, 102, 40, 33, 75, 99, 44, 9, 105, 37, 31, 77, 83, 104, 93, 35, 94, 21, 17, 16, 25, 13, 98, 85, 36, 72, 82, 23, 34, 11, 19, 79, 29, 6, 66, 78, 32, 68, 0, 12, 95, 69, 22, 27, 81, 89, 28, 10, 87, 71, 90, 20, 39, 74, 76, 65, 3, 7, 73, 70, 67, 8, 1, 2, 4, 5, 64], [112, 64, 37, 113, 48, 49, 1, 69, 16, 74, 30, 68, 86, 87, 7, 123, 13, 2, 53, 20, 122, 34, 43, 58, 57, 96, 121, 118, 59, 120, 66, 125, 62, 56, 119, 54, 60, 75, 3, 26, 18, 55, 4, 52, 6, 115, 50, 114, 124, 111, 117, 126, 63, 79, 61, 0, 51, 47, 76, 127, 67, 110, 103, 116, 41, 109, 46, 106, 108, 93, 45, 104, 89, 38, 44, 36, 105, 8, 29, 35, 39, 42, 100, 71, 81, 102, 28, 15, 33, 21, 31, 90, 40, 27, 107, 25, 94, 19, 78, 24, 84, 99, 92, 88, 70, 73, 97, 85, 22, 72, 80, 5, 9, 91, 98, 95, 14, 77, 17, 32, 23, 83, 65, 12, 10, 82, 101, 11], [56, 113, 63, 38, 99, 124, 117, 59, 53, 22, 58, 116, 114, 55, 62, 122, 60, 32, 61, 50, 126, 51, 123, 111, 48, 121, 54, 127, 115, 118, 52, 46, 120, 125, 119, 57, 108, 112, 49, 45, 110, 109, 27, 40, 47, 35, 43, 104, 44, 89, 107, 97, 41, 42, 101, 106, 103, 105, 39, 92, 30, 98, 93, 33, 36, 87, 100, 102, 37, 19, 96, 18, 31, 13, 95, 34, 24, 17, 26, 91, 86, 20, 15, 29, 94, 28, 88, 85, 65, 2, 23, 0, 70, 80, 90, 7, 83, 10, 78, 74, 82, 25, 84, 14, 73, 8, 11, 3, 67, 81, 79, 75, 68, 16, 69, 21, 5, 71, 76, 4, 9, 12, 77, 72, 66, 64, 1, 6]], "model.layers.14.self_attn.qk_proj": [[56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 116, 58, 120, 121, 124, 122, 57, 125, 127, 61, 103, 115, 55, 51, 59, 22, 97, 50, 119, 114, 38, 102, 49, 54, 39, 37, 118, 23, 126, 101, 47, 87, 111, 94, 86, 108, 16, 80, 33, 46, 0, 42, 110, 17, 26, 64, 109, 43, 90, 24, 106, 45, 88, 81, 93, 78, 14, 44, 107, 10, 69, 105, 29, 7, 5, 83, 74, 71, 12, 84, 11, 30, 40, 34, 19, 13, 4, 77, 2, 85, 9, 96, 73, 20, 21, 1, 66, 76, 41, 75, 67, 68, 28, 104, 99, 15, 65, 18, 82, 3, 36, 89, 35, 79, 92, 25, 98, 32, 70, 6, 100, 8, 31, 91, 95, 27, 72], [113, 56, 53, 52, 60, 123, 63, 62, 48, 112, 117, 116, 58, 124, 120, 127, 61, 125, 121, 122, 55, 103, 57, 115, 51, 59, 97, 102, 39, 22, 49, 38, 114, 54, 50, 23, 118, 37, 101, 119, 126, 94, 111, 47, 87, 109, 33, 86, 16, 64, 108, 43, 80, 0, 24, 17, 106, 90, 42, 46, 110, 45, 10, 26, 88, 14, 107, 78, 81, 93, 44, 29, 74, 4, 68, 71, 11, 85, 5, 105, 83, 69, 12, 9, 99, 75, 41, 34, 96, 19, 7, 21, 28, 77, 84, 40, 104, 65, 13, 20, 18, 1, 30, 76, 6, 92, 2, 73, 66, 67, 32, 36, 79, 25, 82, 70, 15, 98, 100, 35, 31, 89, 91, 8, 3, 27, 95, 72], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 116, 58, 127, 124, 61, 120, 122, 103, 125, 121, 59, 115, 57, 39, 54, 55, 97, 51, 49, 38, 50, 102, 114, 118, 47, 101, 119, 126, 37, 111, 46, 23, 22, 87, 94, 43, 108, 86, 0, 33, 64, 16, 110, 106, 109, 107, 80, 45, 88, 44, 90, 24, 42, 26, 30, 17, 93, 14, 29, 71, 10, 41, 105, 81, 78, 85, 69, 83, 34, 5, 99, 96, 12, 74, 7, 11, 4, 40, 65, 19, 68, 13, 66, 28, 104, 1, 9, 21, 75, 77, 76, 20, 73, 6, 84, 2, 35, 98, 32, 3, 25, 18, 67, 36, 82, 15, 92, 89, 70, 91, 31, 27, 79, 72, 100, 95, 8], [56, 113, 53, 52, 60, 123, 63, 62, 48, 112, 117, 116, 58, 125, 121, 122, 124, 61, 120, 127, 57, 49, 103, 114, 55, 39, 54, 97, 115, 59, 50, 51, 102, 111, 119, 101, 22, 38, 126, 47, 118, 23, 37, 94, 110, 46, 87, 108, 43, 0, 64, 16, 86, 80, 45, 109, 33, 90, 106, 24, 107, 17, 44, 93, 26, 42, 78, 88, 29, 81, 41, 69, 74, 2, 19, 7, 10, 104, 14, 105, 71, 30, 85, 96, 83, 13, 34, 12, 66, 76, 84, 77, 40, 65, 5, 11, 1, 9, 73, 4, 28, 3, 68, 20, 99, 75, 92, 36, 6, 32, 21, 98, 67, 18, 25, 100, 89, 15, 35, 82, 79, 70, 72, 31, 91, 95, 27, 8], [56, 113, 53, 52, 60, 123, 63, 62, 48, 112, 117, 122, 58, 121, 125, 116, 57, 61, 120, 55, 124, 103, 49, 127, 115, 114, 39, 59, 97, 118, 38, 51, 50, 22, 46, 102, 101, 47, 23, 54, 126, 94, 37, 108, 111, 110, 86, 0, 16, 119, 87, 80, 64, 33, 43, 109, 90, 106, 14, 81, 42, 44, 107, 24, 17, 26, 78, 71, 10, 30, 93, 2, 45, 66, 83, 11, 74, 69, 19, 88, 29, 105, 5, 4, 104, 68, 84, 13, 9, 12, 76, 41, 7, 40, 96, 73, 21, 1, 34, 75, 18, 85, 20, 77, 28, 65, 3, 79, 92, 82, 32, 89, 15, 99, 98, 6, 67, 35, 100, 70, 36, 25, 27, 91, 31, 72, 95, 8], [56, 113, 53, 52, 123, 60, 63, 62, 48, 112, 117, 116, 121, 55, 127, 125, 58, 124, 120, 122, 103, 57, 61, 59, 49, 115, 114, 97, 22, 118, 39, 102, 51, 38, 54, 50, 119, 23, 37, 101, 109, 94, 87, 108, 126, 110, 111, 86, 47, 16, 80, 64, 17, 33, 14, 43, 90, 26, 42, 81, 0, 44, 24, 78, 46, 45, 106, 29, 11, 4, 88, 93, 10, 9, 83, 12, 74, 76, 84, 30, 75, 41, 85, 19, 7, 71, 105, 96, 18, 68, 107, 69, 66, 20, 5, 13, 73, 77, 104, 21, 32, 82, 79, 65, 2, 15, 3, 40, 34, 28, 89, 98, 92, 1, 70, 99, 36, 67, 6, 31, 25, 35, 100, 72, 91, 27, 95, 8], [56, 113, 53, 123, 52, 60, 63, 62, 48, 112, 117, 116, 121, 120, 127, 55, 58, 115, 125, 61, 49, 122, 114, 59, 57, 103, 124, 22, 118, 97, 102, 23, 119, 38, 51, 39, 37, 101, 94, 50, 54, 87, 126, 86, 109, 111, 16, 47, 110, 80, 17, 26, 43, 108, 33, 90, 14, 46, 78, 81, 42, 24, 64, 45, 106, 85, 29, 0, 88, 44, 30, 11, 10, 105, 74, 104, 9, 75, 76, 93, 84, 71, 107, 20, 73, 7, 69, 83, 12, 15, 41, 5, 77, 96, 21, 19, 28, 40, 13, 82, 18, 32, 2, 66, 34, 67, 79, 4, 99, 36, 6, 98, 92, 68, 65, 89, 1, 70, 25, 3, 35, 31, 100, 27, 72, 91, 95, 8], [56, 113, 53, 52, 123, 60, 63, 62, 48, 112, 117, 120, 58, 121, 116, 59, 115, 55, 114, 124, 57, 122, 127, 103, 49, 97, 125, 38, 51, 22, 102, 61, 119, 23, 118, 54, 37, 101, 39, 126, 94, 87, 47, 50, 86, 46, 111, 109, 110, 16, 43, 108, 33, 26, 0, 64, 90, 80, 88, 78, 45, 14, 81, 17, 85, 107, 42, 29, 24, 30, 44, 84, 106, 75, 10, 96, 83, 11, 105, 93, 21, 74, 20, 5, 34, 71, 104, 65, 76, 9, 77, 7, 15, 41, 13, 18, 82, 4, 69, 1, 68, 12, 73, 2, 66, 36, 70, 19, 32, 79, 35, 28, 92, 98, 99, 89, 25, 3, 6, 91, 40, 67, 31, 100, 72, 27, 95, 8], [56, 113, 53, 52, 123, 60, 63, 62, 48, 112, 117, 116, 120, 125, 58, 127, 121, 122, 49, 124, 55, 115, 57, 103, 97, 38, 54, 61, 51, 102, 114, 50, 101, 59, 39, 118, 23, 22, 119, 37, 111, 94, 87, 47, 43, 108, 86, 109, 80, 64, 0, 126, 110, 16, 33, 46, 106, 88, 78, 90, 17, 14, 26, 85, 105, 44, 45, 81, 29, 24, 10, 42, 30, 93, 4, 96, 74, 83, 84, 19, 104, 1, 71, 107, 41, 5, 34, 7, 76, 99, 21, 69, 20, 65, 75, 68, 66, 13, 28, 73, 2, 77, 11, 12, 15, 9, 32, 98, 18, 70, 25, 92, 36, 40, 35, 82, 6, 3, 79, 67, 91, 89, 100, 31, 27, 72, 95, 8], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 58, 120, 116, 57, 55, 125, 124, 121, 49, 122, 114, 127, 59, 51, 97, 103, 38, 115, 61, 39, 50, 54, 119, 23, 22, 111, 118, 102, 37, 101, 87, 94, 110, 126, 33, 64, 47, 0, 16, 80, 108, 86, 46, 17, 107, 14, 78, 45, 90, 42, 109, 24, 44, 81, 43, 26, 93, 10, 29, 71, 74, 7, 11, 88, 85, 99, 106, 30, 68, 69, 13, 12, 83, 34, 28, 84, 105, 5, 76, 96, 9, 66, 73, 4, 21, 18, 104, 75, 77, 20, 1, 70, 67, 65, 3, 2, 41, 15, 19, 82, 79, 40, 32, 89, 98, 6, 92, 100, 91, 35, 25, 36, 8, 31, 95, 27, 72], [56, 113, 53, 52, 60, 123, 62, 63, 48, 112, 117, 116, 58, 120, 55, 121, 124, 57, 49, 122, 127, 125, 103, 115, 114, 59, 39, 61, 51, 97, 50, 22, 102, 54, 23, 126, 118, 38, 101, 37, 87, 119, 47, 110, 0, 86, 94, 64, 16, 108, 80, 111, 17, 109, 46, 33, 90, 10, 106, 14, 74, 42, 78, 107, 29, 93, 45, 81, 26, 24, 43, 30, 5, 44, 71, 69, 105, 88, 66, 7, 12, 76, 13, 19, 68, 73, 104, 83, 2, 11, 84, 15, 65, 96, 18, 85, 41, 9, 77, 70, 4, 20, 40, 75, 1, 21, 34, 79, 3, 82, 28, 99, 67, 6, 36, 32, 89, 98, 92, 35, 91, 100, 8, 25, 27, 31, 72, 95], [56, 113, 53, 52, 123, 60, 63, 62, 48, 112, 117, 116, 120, 58, 121, 55, 122, 127, 124, 103, 115, 114, 49, 57, 61, 125, 51, 22, 39, 118, 23, 59, 38, 102, 97, 119, 37, 54, 101, 50, 126, 94, 111, 110, 86, 87, 108, 16, 80, 109, 17, 64, 107, 0, 47, 14, 33, 42, 46, 81, 78, 26, 90, 10, 43, 74, 29, 44, 45, 106, 30, 24, 68, 105, 88, 93, 83, 69, 41, 84, 20, 4, 76, 5, 75, 85, 73, 7, 11, 19, 12, 13, 66, 2, 77, 9, 104, 28, 71, 1, 96, 32, 18, 82, 15, 21, 67, 34, 3, 70, 79, 6, 99, 40, 65, 35, 89, 25, 36, 98, 100, 8, 31, 91, 92, 27, 72, 95], [56, 113, 53, 52, 60, 123, 63, 62, 48, 112, 117, 120, 116, 58, 127, 115, 124, 103, 55, 57, 59, 121, 125, 114, 61, 38, 50, 49, 51, 122, 119, 39, 22, 102, 23, 97, 111, 54, 37, 118, 101, 94, 87, 108, 126, 47, 86, 110, 33, 17, 109, 46, 16, 80, 88, 14, 81, 26, 42, 64, 43, 0, 44, 106, 45, 90, 24, 107, 78, 29, 105, 93, 30, 10, 74, 104, 75, 41, 11, 21, 7, 83, 85, 68, 4, 69, 84, 99, 20, 96, 19, 28, 77, 34, 76, 5, 32, 82, 13, 18, 73, 92, 71, 66, 36, 98, 9, 1, 15, 12, 2, 40, 3, 65, 6, 31, 25, 100, 91, 89, 95, 79, 35, 70, 27, 67, 8, 72], [56, 113, 53, 52, 60, 123, 62, 63, 48, 112, 117, 120, 116, 127, 58, 59, 125, 115, 124, 49, 114, 55, 57, 51, 121, 103, 119, 39, 122, 61, 22, 50, 97, 111, 54, 118, 102, 37, 38, 87, 23, 64, 110, 94, 101, 86, 109, 108, 47, 126, 0, 17, 42, 46, 80, 14, 74, 33, 16, 44, 81, 45, 90, 43, 26, 78, 88, 24, 29, 10, 7, 71, 73, 105, 18, 75, 83, 93, 76, 30, 69, 11, 104, 77, 107, 19, 5, 13, 66, 41, 9, 6, 4, 96, 106, 1, 84, 65, 34, 20, 21, 99, 12, 85, 15, 28, 68, 40, 36, 2, 79, 32, 82, 70, 67, 3, 92, 25, 98, 35, 27, 8, 91, 89, 31, 100, 95, 72], [56, 113, 53, 52, 123, 60, 63, 62, 48, 112, 117, 120, 121, 116, 124, 55, 127, 125, 49, 58, 114, 57, 59, 122, 103, 39, 61, 115, 50, 119, 51, 97, 22, 38, 118, 102, 23, 54, 37, 101, 94, 111, 110, 108, 126, 86, 87, 109, 16, 0, 64, 46, 47, 43, 80, 33, 42, 14, 90, 17, 74, 44, 45, 78, 106, 26, 81, 88, 107, 10, 7, 93, 24, 29, 30, 105, 69, 11, 71, 83, 75, 84, 73, 76, 104, 66, 19, 21, 18, 13, 5, 20, 34, 40, 4, 68, 28, 1, 3, 67, 12, 85, 2, 9, 77, 6, 96, 41, 15, 32, 65, 99, 82, 70, 79, 36, 92, 89, 98, 100, 25, 27, 8, 35, 31, 91, 95, 72], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 116, 120, 114, 55, 121, 58, 127, 124, 122, 49, 103, 125, 115, 59, 61, 57, 39, 102, 119, 51, 118, 37, 22, 54, 97, 50, 23, 38, 126, 46, 111, 94, 108, 16, 86, 110, 87, 101, 17, 64, 45, 47, 80, 44, 109, 0, 33, 43, 106, 107, 74, 14, 90, 42, 10, 105, 81, 4, 26, 83, 78, 29, 2, 93, 11, 76, 88, 24, 68, 5, 69, 7, 75, 71, 66, 41, 19, 84, 28, 21, 13, 30, 96, 9, 20, 104, 65, 40, 12, 73, 77, 18, 6, 34, 85, 98, 70, 15, 67, 79, 92, 99, 36, 32, 3, 1, 8, 89, 31, 35, 82, 100, 25, 91, 27, 95, 72], [56, 113, 53, 52, 60, 123, 63, 62, 48, 112, 117, 120, 58, 116, 114, 121, 124, 127, 55, 115, 103, 59, 39, 125, 57, 49, 122, 50, 119, 61, 38, 118, 22, 51, 97, 54, 23, 102, 126, 37, 101, 94, 86, 111, 87, 110, 17, 46, 108, 44, 16, 45, 109, 80, 33, 0, 26, 47, 64, 107, 90, 106, 78, 43, 10, 42, 93, 29, 88, 81, 30, 14, 105, 24, 74, 11, 7, 28, 5, 75, 71, 41, 77, 21, 40, 34, 18, 83, 96, 68, 104, 4, 98, 9, 13, 85, 20, 12, 69, 84, 36, 66, 65, 76, 19, 99, 6, 32, 1, 2, 92, 73, 70, 79, 82, 35, 25, 15, 100, 3, 31, 91, 89, 8, 27, 67, 95, 72], [56, 113, 53, 52, 123, 60, 63, 62, 48, 112, 117, 120, 116, 58, 121, 61, 55, 124, 115, 103, 127, 125, 59, 49, 114, 122, 39, 38, 119, 57, 118, 22, 97, 23, 54, 50, 51, 101, 37, 102, 126, 87, 86, 94, 47, 111, 46, 64, 0, 108, 16, 109, 33, 110, 44, 45, 80, 106, 17, 42, 14, 90, 107, 88, 43, 26, 78, 81, 24, 30, 93, 105, 74, 75, 7, 28, 10, 66, 71, 96, 19, 29, 69, 40, 76, 83, 11, 104, 20, 65, 34, 99, 77, 21, 84, 85, 9, 5, 18, 4, 68, 1, 82, 70, 13, 41, 73, 12, 2, 36, 67, 15, 6, 3, 92, 79, 89, 31, 35, 25, 98, 91, 32, 27, 100, 8, 95, 72], [56, 113, 53, 123, 60, 52, 63, 62, 48, 112, 117, 58, 121, 116, 120, 103, 124, 114, 122, 125, 127, 38, 57, 55, 49, 59, 115, 51, 97, 61, 54, 39, 50, 119, 101, 118, 22, 102, 23, 94, 111, 110, 37, 87, 33, 109, 47, 46, 64, 126, 108, 0, 86, 16, 44, 80, 107, 106, 90, 45, 24, 26, 43, 78, 10, 17, 81, 30, 93, 40, 105, 42, 88, 14, 28, 34, 104, 96, 74, 65, 85, 29, 99, 7, 71, 69, 19, 20, 4, 13, 5, 83, 21, 76, 98, 66, 75, 12, 84, 41, 9, 11, 18, 68, 77, 2, 92, 1, 35, 25, 3, 73, 82, 70, 32, 36, 67, 100, 91, 89, 31, 15, 79, 6, 27, 95, 72, 8], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 116, 120, 58, 121, 124, 127, 59, 61, 55, 125, 49, 103, 114, 39, 57, 122, 115, 97, 54, 22, 102, 119, 51, 50, 38, 118, 23, 37, 126, 94, 101, 111, 87, 108, 44, 33, 109, 86, 107, 0, 16, 46, 80, 47, 17, 106, 43, 64, 45, 90, 26, 24, 110, 93, 78, 42, 88, 14, 74, 10, 81, 105, 30, 29, 75, 4, 76, 19, 96, 7, 83, 40, 12, 11, 71, 99, 34, 85, 84, 5, 68, 20, 77, 28, 65, 9, 104, 21, 82, 73, 98, 1, 69, 18, 36, 41, 13, 70, 2, 15, 79, 66, 32, 35, 6, 25, 92, 89, 100, 91, 67, 31, 3, 27, 95, 72, 8], [56, 113, 53, 123, 52, 60, 63, 62, 48, 112, 117, 120, 121, 61, 58, 55, 116, 59, 125, 127, 122, 124, 115, 49, 114, 103, 57, 119, 22, 54, 118, 97, 39, 38, 51, 23, 37, 50, 126, 102, 94, 101, 86, 108, 46, 111, 87, 107, 44, 80, 110, 16, 43, 47, 109, 90, 0, 45, 88, 33, 64, 26, 14, 10, 106, 17, 105, 78, 74, 42, 81, 93, 24, 7, 76, 12, 69, 30, 19, 83, 11, 29, 28, 71, 4, 96, 2, 104, 84, 9, 75, 5, 99, 20, 41, 77, 18, 40, 67, 85, 34, 13, 21, 73, 66, 65, 82, 98, 79, 68, 70, 36, 92, 1, 15, 3, 32, 6, 91, 89, 72, 35, 25, 31, 100, 27, 95, 8], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 58, 120, 121, 61, 59, 116, 125, 55, 127, 122, 49, 103, 115, 38, 114, 54, 57, 124, 39, 51, 119, 118, 23, 97, 126, 101, 22, 102, 37, 50, 87, 86, 47, 111, 94, 80, 64, 108, 46, 90, 16, 110, 0, 33, 109, 107, 44, 45, 43, 17, 26, 88, 106, 14, 10, 105, 93, 78, 42, 81, 24, 74, 29, 19, 7, 83, 5, 76, 30, 104, 28, 71, 65, 69, 34, 40, 20, 75, 9, 84, 85, 11, 18, 77, 96, 4, 68, 12, 13, 41, 21, 73, 2, 99, 1, 82, 70, 79, 3, 98, 36, 92, 91, 66, 15, 89, 31, 25, 35, 32, 6, 72, 67, 100, 27, 95, 8], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 120, 59, 58, 124, 121, 103, 116, 38, 125, 127, 122, 55, 57, 61, 115, 54, 114, 97, 118, 49, 51, 22, 39, 50, 101, 23, 119, 102, 126, 37, 94, 86, 87, 109, 111, 47, 16, 108, 46, 64, 33, 44, 80, 110, 0, 90, 45, 88, 42, 106, 26, 81, 93, 105, 14, 10, 24, 43, 17, 78, 74, 107, 83, 30, 11, 104, 5, 68, 29, 19, 34, 4, 7, 96, 77, 40, 20, 99, 69, 85, 84, 71, 76, 2, 28, 65, 21, 66, 92, 13, 73, 9, 75, 12, 41, 18, 36, 82, 6, 89, 1, 15, 3, 25, 98, 79, 35, 32, 67, 100, 91, 27, 70, 95, 72, 31, 8], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 125, 58, 120, 121, 124, 59, 55, 61, 116, 57, 127, 103, 114, 122, 38, 97, 115, 49, 22, 51, 39, 118, 54, 23, 37, 119, 50, 94, 101, 126, 87, 111, 102, 16, 86, 47, 33, 80, 90, 110, 44, 109, 64, 42, 26, 0, 17, 108, 46, 45, 78, 106, 10, 14, 29, 88, 81, 93, 74, 24, 43, 83, 30, 107, 11, 28, 105, 76, 104, 71, 75, 4, 73, 84, 68, 41, 12, 96, 6, 7, 99, 5, 85, 66, 69, 15, 40, 13, 34, 77, 9, 19, 2, 20, 21, 82, 3, 18, 36, 32, 67, 92, 1, 100, 65, 25, 70, 89, 98, 91, 79, 27, 35, 72, 31, 95, 8], [56, 113, 53, 60, 123, 52, 63, 62, 48, 112, 117, 120, 121, 58, 59, 125, 61, 55, 116, 127, 49, 103, 57, 114, 124, 22, 115, 122, 118, 39, 38, 54, 119, 23, 51, 37, 97, 50, 86, 102, 101, 87, 94, 47, 111, 80, 126, 108, 110, 109, 46, 16, 26, 43, 33, 17, 106, 45, 78, 90, 88, 0, 64, 44, 107, 81, 29, 11, 74, 42, 14, 10, 24, 105, 83, 5, 93, 85, 30, 76, 9, 71, 40, 7, 84, 104, 69, 28, 41, 20, 13, 2, 77, 82, 18, 12, 75, 15, 96, 73, 19, 34, 1, 21, 99, 65, 66, 4, 32, 98, 68, 36, 3, 6, 100, 67, 79, 25, 89, 35, 70, 91, 31, 72, 92, 27, 95, 8], [56, 113, 60, 52, 53, 123, 63, 62, 48, 112, 117, 120, 59, 58, 61, 121, 57, 127, 116, 55, 124, 125, 103, 122, 114, 115, 22, 49, 54, 38, 118, 39, 50, 119, 51, 97, 23, 102, 37, 101, 94, 126, 86, 87, 47, 109, 110, 106, 33, 80, 26, 90, 16, 42, 88, 43, 107, 81, 111, 64, 17, 44, 108, 78, 0, 10, 29, 14, 46, 93, 24, 45, 75, 40, 85, 30, 104, 83, 74, 28, 105, 84, 41, 11, 71, 21, 76, 19, 5, 20, 32, 96, 69, 12, 18, 7, 99, 9, 77, 68, 34, 4, 36, 13, 65, 6, 2, 15, 82, 25, 66, 98, 100, 79, 73, 70, 1, 92, 89, 35, 91, 31, 3, 67, 95, 72, 27, 8], [56, 113, 60, 53, 52, 123, 63, 62, 48, 112, 117, 120, 127, 124, 59, 58, 61, 57, 121, 122, 103, 115, 116, 125, 49, 50, 51, 55, 54, 114, 39, 22, 38, 97, 23, 119, 126, 101, 118, 47, 102, 86, 37, 46, 94, 87, 44, 17, 111, 80, 26, 0, 107, 109, 64, 108, 33, 16, 90, 43, 24, 42, 106, 45, 81, 88, 105, 110, 10, 14, 30, 93, 29, 74, 68, 28, 85, 76, 78, 84, 40, 71, 83, 5, 96, 11, 104, 2, 21, 69, 77, 13, 20, 4, 12, 41, 34, 19, 66, 7, 82, 9, 75, 99, 73, 65, 67, 25, 98, 1, 3, 18, 92, 6, 15, 35, 79, 32, 70, 36, 91, 89, 31, 27, 100, 95, 72, 8], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 124, 120, 121, 58, 116, 125, 122, 61, 127, 57, 55, 59, 114, 115, 103, 97, 22, 54, 39, 49, 38, 118, 51, 50, 101, 102, 23, 126, 37, 119, 94, 87, 86, 64, 0, 80, 108, 16, 47, 43, 90, 109, 111, 33, 26, 17, 81, 45, 46, 14, 110, 88, 106, 10, 93, 78, 44, 107, 42, 29, 71, 74, 11, 76, 105, 73, 24, 83, 75, 9, 84, 30, 12, 85, 69, 4, 7, 82, 28, 66, 65, 40, 77, 96, 104, 19, 1, 21, 5, 20, 34, 41, 13, 68, 18, 67, 2, 79, 99, 3, 70, 25, 6, 91, 32, 92, 89, 15, 98, 36, 31, 27, 35, 8, 100, 95, 72], [56, 113, 53, 52, 60, 123, 63, 62, 48, 112, 117, 120, 116, 125, 57, 121, 58, 122, 127, 124, 61, 55, 103, 97, 51, 115, 49, 54, 126, 39, 114, 118, 59, 22, 23, 38, 50, 102, 101, 37, 47, 119, 46, 94, 111, 0, 108, 109, 43, 16, 80, 87, 86, 110, 33, 90, 64, 45, 81, 17, 14, 26, 106, 42, 10, 93, 88, 74, 44, 105, 78, 83, 29, 30, 24, 71, 7, 13, 69, 107, 5, 85, 40, 34, 12, 76, 9, 2, 11, 19, 28, 65, 20, 104, 96, 98, 68, 73, 70, 4, 84, 99, 66, 82, 77, 75, 41, 92, 67, 1, 21, 79, 6, 36, 89, 18, 91, 32, 35, 100, 3, 31, 8, 15, 27, 25, 95, 72], [56, 113, 53, 52, 60, 123, 63, 62, 48, 112, 117, 58, 116, 121, 120, 125, 61, 55, 124, 122, 59, 127, 57, 103, 22, 97, 49, 115, 54, 51, 50, 119, 114, 39, 37, 38, 126, 102, 101, 23, 47, 118, 87, 94, 86, 45, 80, 43, 109, 16, 111, 46, 0, 26, 78, 17, 81, 10, 90, 108, 33, 64, 14, 110, 106, 29, 88, 2, 24, 93, 42, 44, 107, 74, 83, 85, 69, 12, 76, 105, 104, 73, 4, 96, 19, 40, 30, 5, 7, 11, 75, 20, 71, 84, 41, 9, 34, 68, 77, 99, 28, 21, 82, 70, 66, 13, 18, 3, 79, 65, 67, 1, 15, 32, 89, 36, 98, 6, 25, 35, 92, 8, 91, 100, 31, 95, 27, 72], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 116, 125, 58, 120, 61, 121, 122, 57, 124, 127, 59, 97, 115, 55, 103, 114, 22, 54, 39, 51, 50, 49, 119, 102, 37, 23, 126, 86, 38, 118, 111, 87, 94, 101, 47, 110, 16, 0, 17, 33, 64, 43, 45, 80, 78, 26, 108, 46, 109, 29, 42, 90, 44, 81, 10, 24, 88, 41, 83, 106, 107, 74, 93, 68, 71, 4, 30, 7, 5, 12, 14, 69, 75, 9, 99, 96, 105, 11, 19, 73, 76, 34, 18, 2, 13, 104, 70, 77, 66, 85, 40, 28, 21, 67, 65, 1, 84, 20, 79, 32, 89, 25, 98, 82, 15, 36, 35, 31, 6, 91, 92, 3, 8, 100, 27, 95, 72], [56, 113, 53, 52, 60, 123, 63, 62, 48, 112, 117, 120, 116, 121, 125, 58, 122, 61, 55, 59, 124, 103, 22, 127, 57, 97, 114, 51, 115, 54, 23, 38, 50, 49, 126, 102, 39, 119, 101, 37, 118, 86, 94, 87, 111, 47, 17, 78, 108, 109, 80, 26, 16, 33, 43, 42, 45, 88, 90, 46, 24, 10, 0, 11, 29, 110, 44, 107, 14, 64, 106, 74, 93, 81, 30, 83, 12, 5, 105, 75, 9, 71, 40, 13, 99, 76, 7, 73, 84, 19, 66, 85, 96, 4, 20, 18, 104, 34, 69, 70, 77, 41, 79, 15, 28, 21, 65, 25, 68, 82, 89, 32, 6, 3, 2, 92, 1, 67, 98, 36, 35, 91, 8, 100, 31, 27, 95, 72]], "model.layers.15.self_attn.q_proj": [[103, 62, 113, 33, 14, 21, 8, 11, 16, 18, 49, 89, 60, 67, 91, 19, 57, 69, 79, 26, 2, 22, 24, 59, 84, 46, 3, 95, 75, 70, 87, 65, 28, 66, 13, 6, 101, 27, 72, 80, 10, 25, 0, 73, 5, 78, 44, 88, 74, 83, 71, 56, 117, 77, 81, 76, 82, 85, 94, 20, 4, 64, 9, 92, 7, 17, 12, 119, 29, 15, 90, 23, 1, 43, 86, 116, 61, 68, 123, 110, 96, 104, 30, 40, 37, 98, 108, 112, 127, 125, 93, 106, 102, 51, 50, 122, 32, 45, 118, 120, 42, 124, 55, 111, 31, 34, 36, 54, 53, 100, 63, 99, 126, 121, 114, 58, 35, 38, 97, 47, 109, 41, 105, 48, 52, 107, 115, 39], [62, 113, 57, 60, 123, 59, 61, 58, 114, 121, 126, 56, 124, 127, 116, 120, 50, 63, 125, 122, 53, 54, 119, 55, 117, 51, 118, 48, 111, 46, 112, 52, 47, 109, 115, 49, 110, 45, 44, 42, 43, 108, 107, 35, 101, 41, 106, 105, 38, 37, 40, 104, 103, 102, 97, 36, 100, 88, 34, 99, 39, 98, 33, 96, 84, 23, 20, 30, 32, 95, 29, 31, 17, 19, 93, 90, 86, 22, 28, 94, 76, 92, 91, 26, 79, 81, 24, 21, 89, 12, 87, 14, 16, 77, 13, 83, 10, 73, 18, 27, 85, 25, 70, 82, 9, 68, 1, 4, 74, 7, 8, 2, 11, 15, 5, 67, 66, 64, 71, 80, 65, 78, 69, 6, 0, 3, 72, 75], [113, 62, 57, 60, 121, 59, 119, 126, 116, 61, 120, 123, 58, 127, 56, 114, 63, 54, 53, 125, 118, 55, 124, 122, 109, 112, 49, 51, 117, 52, 48, 50, 103, 111, 47, 115, 99, 38, 43, 106, 44, 108, 41, 110, 45, 107, 42, 46, 35, 105, 40, 104, 88, 100, 37, 101, 102, 20, 96, 36, 97, 29, 90, 39, 34, 98, 95, 17, 93, 33, 87, 32, 31, 23, 92, 24, 13, 30, 94, 28, 84, 26, 91, 22, 86, 81, 76, 77, 89, 74, 27, 79, 21, 83, 73, 18, 85, 19, 9, 10, 25, 68, 12, 15, 82, 69, 14, 16, 70, 8, 4, 5, 78, 1, 71, 6, 11, 80, 0, 64, 65, 7, 66, 2, 67, 72, 75, 3], [62, 103, 113, 57, 60, 59, 61, 58, 123, 56, 36, 126, 98, 121, 54, 116, 49, 120, 114, 117, 51, 55, 101, 125, 124, 119, 63, 53, 38, 108, 112, 122, 118, 46, 22, 48, 44, 52, 50, 32, 100, 45, 106, 34, 127, 33, 111, 99, 109, 42, 43, 47, 105, 107, 115, 35, 110, 104, 41, 89, 97, 40, 95, 93, 96, 102, 87, 31, 37, 94, 29, 17, 26, 28, 30, 86, 92, 19, 27, 23, 39, 91, 79, 24, 21, 83, 81, 88, 20, 25, 13, 18, 84, 90, 16, 15, 9, 80, 82, 85, 76, 77, 12, 10, 14, 73, 74, 5, 4, 75, 78, 11, 72, 2, 71, 6, 68, 70, 7, 8, 66, 1, 69, 65, 67, 3, 64, 0], [39, 112, 97, 45, 48, 125, 19, 21, 119, 115, 89, 14, 17, 95, 118, 93, 79, 25, 120, 12, 27, 8, 53, 22, 23, 111, 77, 40, 86, 57, 7, 127, 5, 28, 42, 55, 99, 124, 122, 10, 78, 49, 29, 85, 46, 47, 62, 9, 59, 91, 114, 116, 54, 90, 58, 108, 88, 63, 123, 24, 51, 110, 107, 117, 106, 94, 104, 113, 101, 32, 96, 82, 16, 52, 71, 100, 121, 37, 60, 84, 56, 73, 44, 38, 43, 30, 11, 34, 87, 67, 18, 126, 92, 35, 98, 41, 81, 26, 61, 102, 31, 69, 66, 50, 105, 80, 36, 20, 109, 3, 13, 83, 103, 72, 33, 76, 15, 64, 74, 1, 2, 0, 70, 6, 75, 65, 4, 68], [39, 112, 45, 97, 48, 115, 89, 19, 27, 14, 21, 118, 119, 125, 120, 53, 57, 46, 122, 99, 24, 95, 55, 108, 59, 17, 22, 28, 121, 123, 106, 58, 60, 116, 107, 124, 25, 126, 62, 63, 54, 101, 114, 88, 75, 111, 127, 34, 79, 37, 47, 113, 40, 11, 36, 30, 56, 49, 42, 50, 87, 100, 61, 71, 51, 104, 52, 109, 43, 8, 44, 110, 29, 117, 96, 85, 102, 41, 92, 35, 93, 105, 23, 91, 32, 26, 38, 98, 31, 73, 84, 12, 74, 94, 16, 78, 20, 90, 83, 80, 15, 82, 86, 81, 77, 33, 67, 103, 7, 18, 69, 5, 10, 13, 2, 76, 9, 6, 66, 72, 65, 64, 3, 70, 68, 0, 1, 4], [39, 45, 97, 48, 112, 118, 21, 19, 79, 89, 17, 9, 27, 14, 7, 12, 6, 95, 25, 115, 65, 23, 5, 122, 18, 124, 16, 127, 74, 119, 125, 8, 93, 66, 24, 3, 28, 4, 78, 22, 76, 55, 1, 15, 70, 64, 111, 51, 120, 29, 80, 10, 77, 46, 67, 117, 68, 114, 72, 109, 56, 86, 73, 30, 82, 13, 75, 57, 123, 20, 2, 121, 96, 69, 58, 116, 62, 81, 85, 71, 108, 84, 53, 43, 107, 61, 49, 42, 60, 92, 83, 100, 31, 88, 11, 40, 90, 99, 50, 98, 94, 36, 32, 44, 113, 103, 0, 38, 26, 126, 47, 41, 105, 59, 104, 37, 63, 33, 54, 35, 87, 34, 91, 52, 106, 101, 110, 102], [39, 97, 45, 112, 48, 12, 17, 19, 67, 21, 118, 25, 9, 3, 1, 7, 79, 0, 89, 95, 73, 69, 124, 5, 23, 22, 8, 122, 14, 71, 85, 66, 127, 120, 10, 28, 13, 74, 119, 114, 4, 103, 75, 93, 70, 76, 117, 51, 15, 81, 2, 24, 55, 58, 78, 29, 84, 111, 125, 72, 80, 83, 6, 115, 65, 116, 30, 18, 16, 77, 92, 82, 123, 57, 27, 11, 31, 49, 47, 108, 26, 91, 50, 99, 88, 94, 56, 41, 87, 121, 113, 53, 35, 96, 107, 46, 43, 86, 36, 90, 109, 59, 100, 62, 40, 64, 54, 37, 32, 106, 68, 63, 61, 20, 105, 102, 42, 34, 52, 60, 104, 38, 126, 101, 110, 98, 44, 33], [51, 61, 117, 36, 81, 22, 30, 27, 88, 97, 100, 10, 78, 18, 44, 96, 94, 19, 108, 2, 69, 28, 91, 80, 54, 42, 20, 68, 25, 5, 74, 104, 93, 33, 63, 11, 124, 64, 14, 115, 86, 77, 83, 71, 8, 21, 114, 12, 85, 16, 103, 118, 17, 53, 23, 119, 66, 62, 84, 120, 26, 109, 111, 24, 6, 31, 110, 76, 73, 15, 121, 70, 79, 127, 13, 34, 123, 59, 82, 38, 87, 102, 37, 126, 0, 105, 72, 50, 9, 112, 122, 1, 56, 52, 89, 107, 57, 7, 46, 43, 65, 45, 49, 29, 4, 106, 75, 55, 101, 67, 48, 39, 58, 98, 90, 95, 35, 92, 32, 113, 125, 116, 40, 47, 41, 60, 99, 3], [51, 61, 36, 117, 27, 88, 30, 96, 100, 22, 78, 19, 81, 25, 10, 94, 20, 97, 44, 91, 71, 32, 21, 62, 34, 54, 69, 53, 29, 89, 103, 80, 76, 85, 82, 33, 86, 42, 83, 11, 23, 124, 13, 119, 16, 90, 110, 5, 108, 14, 87, 98, 116, 66, 18, 93, 106, 92, 26, 28, 102, 118, 77, 111, 17, 84, 101, 45, 31, 43, 6, 40, 114, 38, 74, 99, 123, 50, 24, 35, 41, 2, 109, 95, 104, 121, 12, 122, 126, 73, 48, 39, 37, 127, 120, 79, 105, 115, 7, 9, 59, 64, 55, 63, 49, 107, 46, 113, 4, 56, 72, 57, 15, 67, 52, 75, 47, 0, 60, 70, 58, 112, 125, 8, 3, 1, 68, 65], [117, 51, 36, 100, 61, 97, 102, 91, 115, 88, 30, 18, 94, 96, 25, 44, 22, 80, 27, 93, 11, 62, 20, 33, 28, 54, 53, 78, 19, 124, 63, 81, 103, 108, 76, 119, 6, 120, 111, 42, 104, 21, 59, 3, 85, 10, 4, 32, 110, 16, 127, 82, 9, 114, 69, 15, 83, 71, 123, 48, 38, 112, 12, 65, 52, 73, 118, 26, 43, 107, 122, 35, 58, 50, 126, 56, 0, 46, 121, 40, 116, 109, 60, 14, 77, 37, 67, 57, 55, 125, 41, 47, 92, 66, 39, 98, 1, 84, 106, 45, 105, 113, 24, 75, 72, 90, 49, 86, 23, 87, 95, 79, 34, 13, 99, 89, 101, 29, 68, 2, 5, 8, 70, 31, 17, 74, 7, 64], [61, 117, 51, 36, 88, 100, 30, 108, 97, 44, 85, 115, 124, 102, 62, 27, 104, 34, 54, 120, 111, 63, 59, 94, 114, 119, 91, 127, 33, 46, 52, 123, 112, 103, 122, 126, 109, 121, 50, 78, 58, 42, 43, 56, 107, 110, 116, 80, 19, 23, 57, 55, 98, 31, 22, 25, 48, 60, 125, 113, 93, 53, 105, 118, 45, 35, 47, 21, 49, 101, 96, 106, 41, 81, 37, 40, 39, 38, 32, 99, 24, 26, 69, 10, 29, 28, 5, 90, 74, 95, 2, 83, 71, 17, 86, 92, 65, 89, 73, 7, 84, 14, 76, 3, 20, 82, 87, 72, 77, 16, 11, 0, 9, 75, 66, 64, 8, 18, 67, 15, 6, 79, 70, 1, 13, 68, 4, 12], [39, 47, 115, 34, 25, 71, 12, 10, 83, 17, 14, 4, 86, 117, 111, 79, 123, 87, 22, 18, 63, 92, 7, 89, 66, 64, 56, 9, 98, 2, 21, 5, 1, 78, 91, 24, 67, 94, 76, 62, 110, 30, 19, 68, 102, 65, 6, 15, 69, 13, 27, 51, 73, 85, 80, 57, 99, 8, 70, 88, 29, 74, 23, 84, 77, 101, 55, 96, 16, 38, 40, 116, 72, 11, 50, 26, 75, 81, 31, 93, 112, 60, 90, 3, 43, 0, 48, 82, 28, 42, 106, 95, 118, 108, 20, 37, 53, 45, 41, 36, 114, 97, 113, 124, 61, 52, 49, 103, 107, 126, 127, 109, 121, 32, 35, 46, 100, 33, 44, 105, 54, 120, 58, 125, 122, 59, 119, 104], [47, 39, 115, 34, 114, 27, 37, 43, 111, 116, 25, 117, 94, 112, 56, 83, 123, 63, 51, 79, 86, 124, 55, 30, 126, 91, 89, 16, 62, 110, 58, 60, 127, 17, 109, 15, 99, 54, 46, 118, 122, 40, 107, 57, 98, 53, 52, 59, 48, 113, 125, 29, 50, 104, 44, 10, 14, 24, 41, 108, 97, 102, 61, 23, 95, 38, 121, 88, 42, 101, 106, 120, 12, 96, 80, 21, 36, 49, 35, 119, 105, 28, 85, 22, 45, 90, 19, 33, 82, 100, 26, 93, 92, 9, 31, 32, 18, 20, 81, 8, 84, 87, 77, 75, 74, 78, 103, 13, 2, 4, 73, 11, 72, 68, 70, 76, 71, 6, 69, 67, 5, 66, 3, 0, 7, 64, 65, 1], [115, 47, 39, 43, 34, 27, 25, 60, 51, 117, 37, 112, 56, 83, 114, 116, 86, 123, 55, 63, 30, 94, 10, 127, 58, 24, 15, 16, 79, 110, 126, 89, 62, 111, 17, 118, 48, 61, 124, 57, 12, 14, 72, 122, 21, 98, 107, 54, 53, 52, 113, 125, 99, 109, 105, 120, 44, 102, 91, 41, 119, 121, 66, 38, 49, 101, 104, 46, 59, 95, 50, 4, 106, 36, 100, 68, 22, 108, 45, 96, 32, 13, 93, 19, 75, 28, 42, 40, 80, 31, 97, 26, 85, 84, 71, 18, 23, 29, 35, 33, 8, 82, 9, 103, 74, 5, 87, 92, 90, 67, 88, 78, 2, 81, 20, 70, 77, 76, 3, 6, 11, 73, 0, 64, 69, 65, 7, 1], [115, 39, 47, 114, 34, 27, 116, 55, 117, 25, 56, 123, 63, 24, 37, 112, 94, 41, 83, 58, 44, 126, 91, 89, 86, 124, 110, 62, 51, 30, 45, 60, 52, 53, 113, 15, 127, 54, 84, 118, 61, 43, 119, 111, 46, 121, 108, 125, 59, 101, 109, 107, 122, 106, 57, 42, 49, 102, 120, 104, 48, 50, 79, 29, 96, 22, 99, 36, 80, 10, 38, 105, 92, 85, 97, 20, 100, 21, 26, 17, 93, 40, 98, 16, 12, 14, 31, 33, 28, 19, 35, 103, 70, 8, 32, 95, 23, 75, 18, 88, 87, 81, 9, 82, 90, 68, 72, 13, 11, 78, 66, 74, 2, 4, 6, 77, 5, 3, 71, 67, 0, 76, 64, 69, 73, 7, 1, 65], [53, 120, 101, 127, 110, 38, 121, 25, 125, 56, 51, 100, 102, 28, 22, 40, 80, 30, 33, 97, 116, 29, 49, 104, 11, 114, 59, 113, 87, 118, 50, 32, 84, 86, 63, 35, 57, 55, 77, 61, 75, 60, 16, 119, 115, 24, 90, 79, 46, 39, 54, 124, 93, 112, 37, 98, 117, 7, 52, 96, 62, 99, 9, 48, 94, 106, 92, 108, 18, 23, 105, 45, 111, 27, 109, 34, 123, 126, 122, 103, 107, 89, 72, 58, 47, 69, 91, 83, 15, 88, 44, 26, 1, 78, 20, 42, 36, 71, 21, 31, 43, 13, 95, 41, 81, 5, 85, 19, 73, 66, 4, 14, 82, 10, 8, 17, 2, 12, 76, 67, 70, 74, 3, 68, 0, 6, 65, 64], [120, 127, 53, 51, 113, 110, 59, 104, 49, 50, 125, 56, 112, 124, 116, 114, 63, 121, 101, 44, 52, 62, 55, 61, 57, 117, 60, 118, 119, 126, 46, 115, 58, 111, 37, 123, 54, 45, 48, 122, 39, 43, 47, 27, 107, 105, 108, 109, 42, 40, 106, 38, 79, 41, 96, 103, 100, 29, 35, 33, 82, 19, 85, 87, 78, 98, 34, 99, 102, 36, 30, 88, 95, 97, 6, 31, 64, 86, 24, 66, 65, 77, 92, 91, 32, 10, 94, 9, 8, 83, 22, 75, 67, 84, 5, 20, 17, 93, 28, 4, 0, 71, 18, 90, 14, 80, 11, 25, 21, 12, 15, 68, 26, 74, 1, 81, 7, 2, 23, 69, 72, 70, 3, 76, 73, 13, 89, 16], [127, 120, 110, 53, 40, 101, 100, 121, 49, 38, 51, 25, 118, 97, 63, 56, 62, 28, 113, 117, 84, 61, 39, 125, 119, 30, 32, 124, 59, 29, 96, 52, 104, 115, 88, 105, 112, 18, 94, 50, 116, 19, 23, 99, 111, 0, 24, 87, 60, 33, 89, 37, 98, 70, 35, 108, 95, 55, 103, 48, 41, 31, 21, 122, 109, 114, 57, 126, 43, 54, 102, 14, 22, 76, 86, 44, 46, 27, 80, 47, 78, 90, 34, 10, 42, 45, 6, 93, 66, 107, 85, 17, 36, 13, 65, 12, 83, 123, 75, 26, 106, 58, 81, 3, 20, 82, 11, 74, 79, 68, 1, 91, 15, 67, 69, 72, 8, 77, 92, 9, 71, 64, 2, 5, 73, 16, 4, 7], [127, 101, 53, 120, 40, 59, 28, 84, 33, 97, 100, 30, 25, 37, 87, 51, 50, 113, 86, 63, 77, 46, 104, 112, 102, 88, 96, 29, 27, 20, 116, 35, 121, 38, 22, 83, 126, 91, 125, 56, 94, 11, 80, 98, 75, 45, 19, 34, 78, 17, 52, 103, 60, 55, 54, 62, 24, 122, 110, 61, 115, 32, 79, 81, 41, 114, 15, 93, 71, 107, 124, 48, 21, 58, 105, 99, 8, 31, 39, 7, 106, 12, 44, 10, 57, 118, 42, 76, 95, 123, 109, 111, 47, 82, 108, 119, 92, 43, 4, 26, 89, 2, 18, 49, 74, 85, 90, 36, 5, 68, 117, 72, 16, 1, 9, 14, 23, 69, 70, 66, 0, 65, 3, 6, 13, 67, 64, 73], [104, 117, 25, 95, 34, 92, 86, 52, 19, 54, 77, 97, 8, 53, 61, 51, 79, 22, 28, 17, 120, 80, 68, 59, 88, 85, 114, 45, 72, 12, 60, 121, 24, 124, 13, 40, 49, 116, 4, 122, 113, 56, 87, 55, 58, 57, 23, 73, 107, 91, 109, 46, 15, 110, 63, 127, 62, 89, 93, 65, 74, 118, 83, 48, 115, 47, 21, 31, 18, 69, 75, 90, 0, 33, 126, 100, 35, 39, 111, 125, 94, 2, 14, 43, 106, 82, 20, 112, 6, 42, 36, 3, 50, 32, 64, 44, 5, 101, 123, 105, 98, 119, 102, 76, 29, 99, 108, 84, 27, 26, 30, 9, 16, 96, 66, 103, 81, 11, 1, 38, 71, 78, 7, 70, 41, 37, 10, 67], [117, 104, 120, 25, 95, 54, 52, 34, 61, 92, 103, 86, 88, 97, 58, 51, 116, 12, 114, 124, 55, 122, 53, 60, 113, 102, 49, 121, 63, 47, 59, 85, 127, 46, 62, 106, 111, 109, 48, 56, 28, 18, 126, 118, 19, 44, 115, 8, 119, 69, 125, 45, 14, 112, 57, 91, 24, 123, 80, 43, 42, 110, 108, 50, 107, 17, 66, 20, 87, 98, 105, 22, 39, 36, 71, 33, 100, 41, 38, 32, 37, 79, 26, 35, 77, 99, 94, 93, 101, 75, 74, 40, 30, 84, 31, 82, 13, 29, 96, 16, 72, 3, 89, 78, 11, 27, 90, 23, 9, 4, 15, 65, 81, 7, 68, 21, 83, 5, 76, 2, 1, 0, 73, 6, 67, 64, 10, 70], [117, 104, 95, 25, 52, 54, 61, 34, 92, 56, 97, 86, 120, 60, 24, 12, 47, 17, 51, 116, 53, 113, 88, 20, 122, 49, 99, 124, 85, 103, 119, 80, 19, 59, 102, 77, 28, 48, 91, 63, 57, 31, 112, 106, 126, 26, 58, 18, 62, 109, 118, 111, 115, 114, 121, 127, 125, 107, 100, 8, 123, 110, 44, 45, 50, 69, 42, 55, 43, 22, 105, 36, 38, 46, 16, 37, 15, 108, 96, 78, 30, 87, 39, 75, 94, 79, 32, 33, 35, 41, 40, 82, 90, 14, 98, 84, 29, 27, 93, 83, 89, 101, 81, 21, 23, 9, 73, 76, 74, 72, 10, 11, 66, 3, 4, 7, 5, 2, 6, 13, 71, 1, 70, 0, 65, 68, 64, 67], [117, 104, 52, 25, 95, 61, 56, 86, 92, 97, 34, 54, 60, 51, 19, 53, 58, 77, 120, 17, 8, 80, 113, 114, 31, 122, 59, 66, 62, 124, 127, 116, 69, 22, 12, 49, 48, 46, 57, 88, 47, 121, 106, 55, 85, 112, 79, 115, 72, 28, 102, 14, 74, 109, 45, 63, 15, 10, 87, 110, 125, 36, 40, 111, 118, 24, 4, 43, 126, 35, 73, 94, 33, 119, 123, 27, 42, 93, 98, 30, 5, 103, 50, 100, 91, 21, 44, 3, 108, 82, 29, 32, 105, 37, 107, 26, 41, 99, 90, 78, 38, 101, 83, 20, 39, 6, 18, 96, 75, 89, 23, 11, 16, 76, 84, 71, 64, 13, 70, 81, 7, 2, 68, 9, 1, 65, 0, 67], [39, 121, 112, 33, 1, 114, 93, 23, 69, 82, 0, 21, 79, 20, 13, 14, 84, 67, 48, 2, 118, 10, 25, 90, 16, 91, 22, 5, 29, 81, 72, 17, 65, 89, 78, 124, 3, 75, 71, 66, 68, 61, 87, 80, 88, 7, 30, 56, 47, 52, 4, 6, 60, 19, 55, 120, 12, 11, 110, 119, 122, 125, 63, 116, 115, 57, 53, 42, 126, 73, 92, 49, 83, 76, 86, 9, 70, 34, 100, 64, 102, 107, 27, 59, 74, 108, 101, 98, 43, 127, 51, 111, 99, 106, 35, 85, 44, 28, 50, 8, 45, 113, 94, 24, 117, 18, 109, 58, 26, 40, 37, 95, 96, 54, 38, 41, 62, 77, 46, 36, 123, 104, 105, 32, 31, 15, 97, 103], [112, 39, 114, 93, 33, 121, 23, 20, 90, 84, 17, 110, 56, 10, 53, 61, 115, 49, 22, 91, 119, 101, 116, 108, 117, 113, 43, 50, 47, 29, 24, 55, 125, 51, 118, 16, 99, 40, 54, 82, 63, 81, 57, 59, 60, 100, 46, 42, 38, 52, 58, 124, 111, 79, 126, 45, 123, 120, 109, 6, 87, 48, 107, 13, 122, 105, 106, 62, 102, 88, 44, 3, 104, 41, 127, 95, 94, 34, 37, 98, 85, 36, 72, 83, 92, 97, 30, 27, 86, 80, 35, 32, 25, 26, 96, 11, 74, 28, 19, 103, 31, 89, 76, 21, 5, 70, 18, 15, 77, 12, 64, 71, 14, 73, 67, 78, 68, 9, 8, 7, 75, 0, 2, 4, 1, 65, 69, 66], [112, 39, 121, 114, 93, 33, 23, 61, 110, 48, 17, 120, 20, 84, 90, 47, 22, 56, 115, 91, 124, 117, 53, 45, 82, 60, 116, 49, 51, 118, 57, 119, 55, 10, 50, 29, 94, 38, 43, 24, 125, 111, 52, 30, 108, 63, 79, 126, 62, 99, 88, 59, 113, 54, 122, 109, 107, 102, 123, 44, 58, 46, 40, 104, 100, 85, 41, 6, 101, 42, 127, 106, 81, 103, 105, 13, 86, 98, 87, 25, 92, 97, 21, 74, 31, 35, 95, 76, 36, 37, 96, 26, 34, 32, 27, 15, 77, 28, 11, 8, 16, 18, 89, 3, 80, 67, 72, 83, 70, 19, 5, 12, 71, 75, 68, 78, 7, 64, 2, 73, 14, 0, 9, 1, 4, 69, 65, 66], [39, 112, 33, 93, 121, 114, 23, 120, 20, 10, 22, 17, 79, 84, 110, 96, 13, 61, 29, 72, 49, 87, 90, 21, 56, 82, 109, 6, 117, 60, 70, 47, 125, 16, 52, 26, 81, 50, 118, 92, 77, 119, 51, 57, 115, 42, 25, 53, 11, 30, 124, 34, 104, 107, 91, 116, 122, 3, 95, 98, 38, 99, 83, 100, 86, 43, 111, 41, 101, 126, 48, 74, 5, 37, 55, 88, 62, 94, 44, 27, 24, 63, 108, 113, 67, 40, 35, 59, 45, 54, 78, 123, 32, 89, 127, 14, 28, 58, 105, 46, 18, 15, 85, 106, 9, 36, 31, 12, 19, 8, 64, 80, 75, 73, 102, 4, 76, 7, 68, 97, 71, 2, 0, 1, 103, 69, 65, 66], [53, 60, 127, 124, 122, 119, 123, 120, 51, 116, 56, 55, 125, 57, 38, 126, 54, 115, 61, 63, 50, 52, 118, 49, 58, 36, 59, 114, 62, 48, 45, 121, 112, 111, 101, 39, 113, 105, 110, 47, 46, 117, 106, 42, 108, 109, 43, 104, 44, 34, 107, 40, 99, 103, 41, 91, 24, 102, 94, 95, 19, 27, 21, 97, 93, 35, 37, 31, 89, 98, 100, 29, 16, 96, 14, 23, 92, 33, 25, 90, 83, 13, 32, 85, 30, 87, 11, 88, 20, 82, 4, 80, 8, 78, 68, 72, 28, 26, 75, 77, 70, 86, 6, 17, 81, 67, 3, 79, 74, 84, 18, 15, 10, 73, 76, 9, 22, 65, 71, 1, 12, 5, 7, 64, 66, 2, 0, 69], [53, 60, 120, 124, 39, 62, 119, 36, 127, 123, 125, 56, 100, 122, 46, 104, 109, 31, 51, 37, 101, 108, 61, 110, 126, 121, 118, 117, 63, 33, 42, 115, 113, 57, 52, 54, 44, 116, 107, 106, 41, 58, 23, 43, 59, 45, 24, 50, 48, 112, 55, 47, 97, 114, 40, 95, 111, 49, 102, 105, 90, 38, 103, 92, 94, 21, 82, 32, 98, 29, 34, 86, 25, 35, 99, 85, 91, 30, 93, 14, 13, 96, 16, 20, 19, 89, 87, 88, 27, 26, 28, 22, 18, 11, 83, 78, 80, 72, 15, 84, 68, 17, 77, 81, 79, 4, 75, 8, 6, 70, 73, 10, 12, 74, 67, 9, 3, 76, 71, 7, 5, 1, 69, 2, 66, 65, 0, 64], [60, 53, 127, 56, 124, 119, 122, 51, 123, 125, 55, 52, 116, 61, 57, 114, 126, 58, 63, 54, 48, 115, 121, 59, 50, 113, 49, 118, 112, 106, 111, 62, 120, 47, 117, 38, 46, 110, 45, 109, 108, 44, 107, 43, 42, 41, 105, 40, 39, 36, 103, 100, 104, 102, 37, 97, 29, 22, 98, 101, 33, 34, 35, 99, 86, 95, 32, 90, 23, 31, 85, 96, 91, 83, 25, 87, 18, 80, 30, 94, 92, 84, 24, 75, 27, 93, 26, 89, 17, 82, 16, 21, 19, 13, 78, 81, 79, 14, 20, 68, 8, 28, 88, 76, 15, 72, 4, 11, 74, 9, 77, 6, 3, 7, 2, 70, 1, 10, 67, 12, 64, 69, 65, 0, 73, 5, 66, 71], [60, 53, 120, 20, 79, 76, 92, 36, 9, 124, 7, 82, 17, 24, 97, 2, 62, 86, 69, 71, 0, 66, 32, 125, 56, 5, 122, 100, 74, 14, 73, 12, 3, 52, 119, 34, 25, 33, 18, 11, 80, 38, 65, 94, 21, 89, 48, 28, 13, 67, 78, 15, 81, 123, 8, 84, 127, 88, 96, 19, 93, 10, 75, 99, 83, 31, 101, 16, 26, 95, 108, 23, 6, 90, 72, 77, 70, 1, 85, 87, 64, 68, 27, 43, 4, 41, 30, 110, 91, 55, 29, 44, 35, 22, 109, 59, 102, 117, 58, 112, 39, 126, 98, 104, 37, 107, 61, 103, 51, 49, 113, 47, 118, 42, 115, 105, 63, 106, 116, 45, 54, 114, 40, 121, 46, 57, 111, 50]], "model.layers.15.self_attn.k_proj": [[113, 39, 62, 22, 18, 11, 60, 14, 89, 16, 97, 8, 21, 19, 57, 59, 123, 56, 110, 92, 67, 58, 120, 70, 116, 63, 53, 127, 119, 108, 79, 10, 122, 126, 117, 61, 54, 55, 48, 109, 114, 121, 87, 51, 124, 91, 112, 107, 52, 111, 125, 101, 42, 50, 118, 30, 44, 106, 47, 36, 115, 13, 46, 76, 0, 45, 43, 41, 104, 105, 75, 37, 35, 49, 69, 38, 29, 26, 7, 9, 27, 102, 40, 71, 85, 83, 31, 100, 17, 68, 95, 96, 99, 72, 33, 2, 34, 88, 15, 73, 93, 32, 78, 80, 82, 20, 5, 90, 25, 98, 77, 23, 28, 84, 12, 65, 24, 81, 94, 66, 1, 74, 6, 86, 3, 64, 103, 4], [103, 109, 112, 33, 45, 89, 21, 118, 12, 19, 79, 17, 9, 122, 119, 14, 31, 51, 7, 125, 5, 0, 117, 3, 120, 116, 124, 55, 115, 53, 93, 27, 57, 127, 50, 123, 47, 60, 20, 114, 56, 1, 49, 40, 59, 94, 46, 61, 54, 43, 8, 58, 66, 62, 23, 63, 111, 18, 108, 25, 28, 24, 52, 35, 121, 110, 42, 48, 10, 32, 86, 30, 126, 68, 11, 70, 22, 4, 44, 87, 106, 91, 113, 105, 74, 96, 107, 104, 82, 41, 98, 100, 83, 38, 34, 99, 77, 92, 37, 102, 88, 6, 26, 81, 101, 95, 13, 36, 84, 29, 73, 65, 2, 16, 76, 78, 90, 69, 71, 39, 15, 80, 85, 75, 64, 72, 97, 67], [115, 51, 117, 61, 100, 22, 30, 33, 53, 27, 108, 19, 88, 78, 124, 81, 62, 10, 120, 54, 80, 59, 32, 114, 20, 119, 127, 118, 63, 111, 64, 44, 71, 106, 110, 4, 123, 116, 48, 112, 58, 66, 56, 125, 122, 52, 126, 55, 76, 60, 50, 109, 1, 25, 43, 39, 121, 107, 46, 17, 102, 113, 11, 38, 13, 104, 57, 47, 69, 85, 49, 24, 40, 45, 99, 15, 41, 84, 82, 36, 29, 77, 35, 92, 72, 90, 42, 105, 37, 73, 93, 28, 98, 67, 8, 86, 101, 103, 87, 89, 95, 96, 6, 26, 65, 34, 31, 94, 97, 75, 16, 18, 21, 3, 23, 70, 9, 79, 74, 91, 12, 83, 7, 0, 5, 68, 2, 14], [103, 115, 47, 111, 86, 98, 51, 25, 17, 12, 117, 83, 14, 27, 10, 30, 110, 71, 116, 56, 1, 79, 112, 126, 123, 4, 55, 63, 62, 6, 64, 9, 58, 114, 66, 127, 121, 52, 61, 113, 122, 69, 59, 43, 124, 48, 109, 54, 106, 101, 32, 125, 21, 45, 60, 41, 53, 75, 118, 34, 108, 5, 120, 2, 49, 50, 46, 13, 57, 88, 87, 105, 67, 119, 102, 29, 92, 85, 72, 40, 96, 36, 107, 44, 104, 31, 28, 37, 24, 8, 20, 82, 81, 76, 26, 100, 42, 95, 38, 99, 91, 70, 90, 18, 97, 23, 35, 15, 0, 33, 93, 78, 94, 16, 19, 84, 77, 73, 80, 3, 7, 68, 89, 11, 22, 65, 74, 39], [127, 37, 120, 53, 104, 22, 97, 102, 32, 56, 46, 110, 94, 61, 121, 125, 25, 51, 60, 28, 93, 35, 80, 89, 113, 92, 55, 116, 118, 54, 122, 119, 124, 63, 108, 117, 52, 20, 96, 57, 99, 126, 59, 112, 47, 45, 48, 115, 91, 58, 114, 41, 30, 111, 106, 50, 98, 77, 123, 33, 44, 9, 109, 49, 88, 87, 62, 107, 101, 42, 19, 39, 43, 18, 95, 4, 36, 103, 66, 86, 105, 84, 31, 65, 40, 71, 75, 17, 64, 34, 78, 12, 90, 8, 79, 10, 29, 15, 0, 85, 82, 21, 6, 16, 24, 27, 26, 13, 67, 3, 72, 69, 5, 11, 100, 23, 14, 83, 38, 81, 76, 7, 68, 73, 74, 1, 70, 2], [117, 40, 31, 98, 86, 28, 52, 61, 25, 62, 120, 60, 114, 48, 56, 122, 113, 116, 51, 124, 59, 49, 127, 115, 109, 57, 46, 125, 55, 47, 54, 58, 33, 19, 123, 121, 85, 110, 111, 12, 126, 112, 18, 119, 80, 45, 63, 118, 43, 106, 53, 24, 8, 79, 50, 108, 42, 35, 44, 39, 105, 15, 41, 36, 107, 2, 64, 87, 102, 38, 34, 101, 29, 81, 68, 100, 16, 92, 37, 77, 17, 23, 99, 74, 27, 5, 103, 104, 20, 32, 88, 96, 94, 75, 21, 91, 89, 65, 1, 30, 97, 13, 9, 67, 93, 10, 14, 11, 71, 90, 6, 84, 82, 26, 83, 78, 95, 70, 76, 73, 7, 22, 66, 72, 69, 0, 3, 4], [103, 121, 112, 97, 48, 29, 114, 22, 57, 120, 23, 84, 118, 50, 119, 61, 116, 17, 52, 56, 111, 115, 90, 49, 53, 79, 60, 10, 43, 42, 82, 64, 3, 59, 122, 47, 46, 55, 58, 72, 124, 125, 36, 123, 102, 62, 104, 117, 63, 126, 45, 13, 110, 51, 37, 106, 91, 113, 28, 127, 41, 44, 6, 38, 54, 109, 21, 68, 107, 24, 2, 25, 76, 93, 73, 98, 105, 32, 108, 89, 78, 86, 71, 101, 40, 99, 83, 35, 80, 33, 16, 26, 34, 15, 31, 11, 27, 85, 94, 77, 95, 19, 92, 96, 18, 100, 75, 14, 12, 30, 69, 39, 7, 65, 5, 88, 9, 1, 8, 20, 87, 66, 4, 0, 70, 81, 67, 74], [60, 53, 86, 100, 120, 94, 76, 125, 79, 7, 124, 20, 101, 9, 82, 92, 74, 69, 96, 17, 25, 122, 44, 62, 24, 61, 119, 52, 118, 2, 106, 56, 127, 123, 117, 57, 51, 63, 121, 19, 58, 112, 126, 48, 0, 109, 107, 54, 114, 55, 59, 16, 113, 49, 47, 46, 50, 14, 90, 1, 115, 116, 111, 104, 72, 110, 41, 34, 13, 45, 5, 6, 43, 108, 81, 97, 75, 39, 42, 3, 10, 27, 37, 40, 11, 105, 102, 103, 35, 98, 12, 15, 23, 99, 33, 21, 22, 31, 30, 71, 78, 38, 73, 26, 93, 29, 85, 28, 95, 32, 8, 84, 67, 91, 70, 77, 18, 87, 83, 65, 89, 4, 68, 66, 64, 88, 36, 80]], "model.layers.15.self_attn.qk_proj": [[117, 112, 53, 115, 60, 113, 62, 51, 120, 61, 127, 47, 121, 45, 114, 124, 48, 86, 39, 100, 57, 56, 125, 118, 25, 116, 103, 89, 122, 22, 110, 111, 91, 109, 46, 97, 40, 63, 59, 50, 81, 30, 54, 119, 33, 123, 104, 93, 19, 49, 108, 55, 52, 83, 17, 58, 21, 79, 85, 126, 14, 101, 94, 36, 92, 98, 15, 12, 44, 107, 76, 78, 88, 20, 28, 106, 24, 37, 43, 84, 34, 74, 42, 31, 102, 95, 87, 27, 10, 9, 29, 32, 23, 7, 71, 96, 41, 82, 80, 16, 18, 5, 73, 38, 35, 105, 72, 26, 11, 69, 77, 75, 67, 3, 64, 0, 13, 66, 99, 8, 90, 2, 68, 70, 4, 1, 6, 65], [117, 112, 53, 115, 60, 62, 113, 51, 120, 61, 127, 47, 121, 45, 114, 57, 48, 39, 86, 124, 56, 103, 100, 25, 118, 125, 22, 89, 46, 109, 116, 122, 54, 63, 111, 119, 91, 97, 110, 50, 40, 55, 49, 59, 52, 30, 108, 81, 104, 33, 123, 93, 126, 101, 83, 94, 19, 17, 15, 36, 58, 21, 85, 107, 98, 14, 12, 78, 92, 79, 76, 88, 34, 24, 28, 37, 10, 20, 106, 43, 44, 102, 74, 27, 84, 7, 31, 9, 42, 95, 23, 87, 71, 18, 82, 41, 96, 80, 16, 29, 32, 26, 35, 38, 99, 72, 0, 8, 69, 73, 5, 105, 2, 66, 75, 77, 11, 64, 67, 3, 13, 90, 4, 65, 6, 70, 68, 1], [117, 112, 53, 60, 115, 113, 62, 51, 120, 61, 127, 47, 121, 45, 114, 39, 56, 48, 124, 57, 118, 86, 100, 89, 103, 122, 111, 46, 25, 116, 109, 59, 125, 22, 91, 123, 97, 104, 110, 63, 40, 54, 119, 33, 30, 108, 52, 55, 93, 36, 81, 101, 50, 94, 58, 49, 19, 83, 126, 17, 21, 15, 28, 37, 85, 12, 24, 98, 92, 107, 44, 88, 76, 78, 34, 14, 102, 43, 79, 27, 95, 42, 106, 84, 9, 10, 74, 31, 20, 87, 23, 29, 7, 71, 18, 41, 32, 38, 64, 26, 82, 16, 96, 35, 2, 5, 0, 8, 80, 66, 99, 73, 69, 105, 67, 90, 11, 75, 13, 3, 72, 77, 6, 68, 65, 1, 4, 70], [117, 112, 53, 60, 115, 62, 113, 51, 120, 61, 47, 127, 121, 45, 124, 114, 48, 39, 57, 56, 118, 100, 125, 116, 89, 103, 86, 109, 111, 25, 22, 122, 97, 91, 110, 54, 63, 59, 40, 30, 55, 81, 119, 46, 104, 58, 123, 52, 50, 108, 33, 126, 93, 101, 49, 19, 94, 21, 17, 37, 107, 79, 98, 14, 85, 12, 83, 76, 28, 88, 36, 92, 24, 78, 34, 15, 102, 44, 42, 20, 95, 27, 43, 16, 106, 84, 23, 80, 96, 9, 29, 82, 74, 87, 31, 10, 38, 18, 7, 32, 0, 71, 5, 41, 69, 35, 26, 73, 99, 66, 8, 11, 67, 3, 64, 90, 77, 2, 105, 1, 75, 72, 65, 13, 6, 4, 68, 70], [117, 112, 53, 115, 60, 62, 113, 51, 120, 61, 127, 47, 121, 45, 124, 114, 118, 48, 57, 56, 39, 125, 86, 100, 103, 116, 122, 59, 63, 22, 89, 110, 25, 109, 55, 97, 111, 54, 40, 91, 46, 119, 104, 30, 108, 123, 81, 33, 52, 49, 58, 50, 17, 19, 83, 93, 101, 12, 85, 15, 14, 98, 37, 43, 78, 79, 126, 94, 76, 92, 21, 107, 24, 42, 28, 74, 106, 44, 88, 84, 34, 102, 20, 36, 87, 10, 38, 96, 27, 9, 8, 7, 95, 32, 71, 41, 69, 35, 80, 23, 82, 18, 16, 5, 31, 105, 64, 29, 73, 26, 3, 66, 2, 75, 67, 77, 11, 13, 0, 99, 72, 1, 65, 90, 4, 68, 6, 70], [117, 112, 53, 115, 60, 113, 62, 51, 120, 61, 127, 47, 121, 45, 114, 118, 124, 57, 56, 86, 39, 48, 89, 100, 25, 116, 22, 125, 63, 111, 110, 103, 59, 91, 97, 54, 30, 109, 52, 40, 122, 119, 50, 81, 55, 49, 108, 46, 33, 123, 104, 93, 17, 58, 85, 83, 14, 19, 126, 76, 21, 12, 101, 15, 79, 94, 107, 92, 78, 44, 28, 88, 98, 34, 36, 42, 84, 24, 43, 37, 20, 10, 95, 87, 106, 9, 23, 18, 82, 16, 31, 80, 74, 7, 102, 27, 96, 38, 71, 32, 41, 8, 29, 35, 13, 75, 105, 73, 5, 69, 77, 67, 11, 99, 26, 64, 2, 66, 90, 0, 3, 68, 72, 6, 4, 65, 1, 70], [117, 112, 53, 60, 115, 62, 113, 51, 120, 61, 127, 47, 121, 45, 118, 114, 57, 48, 124, 86, 100, 89, 39, 22, 56, 25, 59, 125, 103, 63, 111, 110, 91, 97, 122, 54, 116, 30, 81, 40, 123, 55, 119, 52, 109, 93, 46, 19, 126, 104, 50, 83, 108, 33, 101, 21, 49, 17, 92, 85, 76, 14, 15, 12, 58, 78, 79, 107, 94, 44, 28, 98, 84, 34, 88, 36, 24, 42, 106, 27, 37, 20, 87, 18, 10, 95, 23, 74, 96, 9, 38, 102, 16, 82, 105, 32, 31, 43, 80, 41, 71, 7, 29, 8, 13, 35, 73, 69, 11, 75, 26, 5, 77, 3, 0, 2, 99, 67, 66, 64, 72, 90, 68, 1, 6, 70, 4, 65], [117, 112, 53, 60, 115, 62, 113, 51, 120, 61, 127, 47, 121, 45, 114, 57, 118, 48, 39, 89, 56, 86, 103, 100, 25, 22, 124, 122, 125, 59, 97, 116, 91, 63, 111, 46, 30, 54, 110, 119, 109, 55, 40, 123, 104, 33, 83, 19, 108, 81, 49, 93, 50, 52, 101, 126, 17, 21, 92, 15, 36, 12, 14, 79, 28, 58, 85, 98, 78, 94, 76, 107, 24, 44, 42, 87, 88, 20, 84, 37, 106, 34, 43, 27, 96, 102, 23, 38, 31, 74, 18, 10, 41, 16, 9, 82, 95, 29, 32, 7, 105, 71, 80, 26, 35, 8, 0, 73, 11, 75, 5, 13, 66, 69, 99, 2, 64, 67, 3, 77, 72, 65, 70, 90, 1, 68, 4, 6], [117, 112, 53, 60, 115, 113, 51, 62, 120, 127, 61, 121, 47, 45, 57, 48, 118, 114, 56, 124, 39, 86, 89, 100, 125, 25, 103, 22, 122, 97, 63, 109, 30, 116, 111, 91, 55, 110, 123, 50, 54, 46, 59, 104, 119, 40, 101, 33, 49, 81, 19, 83, 93, 108, 98, 17, 58, 52, 21, 126, 36, 14, 107, 12, 79, 76, 94, 92, 85, 15, 24, 78, 42, 37, 88, 44, 28, 34, 20, 95, 106, 102, 27, 43, 16, 84, 31, 41, 87, 18, 32, 23, 10, 29, 74, 96, 80, 71, 9, 38, 82, 105, 35, 7, 73, 75, 99, 8, 13, 77, 11, 5, 0, 69, 26, 2, 72, 64, 67, 1, 66, 3, 90, 70, 65, 4, 68, 6], [117, 112, 53, 60, 115, 62, 113, 51, 61, 127, 120, 47, 121, 45, 114, 48, 124, 39, 57, 56, 118, 89, 125, 86, 100, 103, 22, 122, 25, 111, 123, 97, 55, 110, 109, 63, 91, 116, 54, 52, 59, 30, 119, 58, 46, 81, 83, 40, 104, 49, 50, 108, 126, 33, 93, 17, 19, 101, 14, 15, 94, 98, 85, 21, 12, 79, 107, 76, 44, 102, 28, 92, 36, 34, 78, 74, 24, 37, 42, 20, 43, 10, 84, 27, 88, 71, 7, 87, 23, 106, 41, 95, 18, 38, 9, 31, 16, 73, 32, 80, 96, 29, 5, 69, 82, 105, 0, 8, 35, 64, 3, 99, 66, 2, 67, 72, 13, 77, 26, 75, 11, 90, 70, 4, 68, 65, 1, 6], [117, 112, 53, 60, 115, 113, 62, 51, 120, 61, 127, 47, 121, 45, 114, 124, 39, 48, 118, 57, 56, 89, 86, 100, 122, 22, 25, 116, 125, 103, 110, 119, 97, 109, 46, 30, 59, 111, 54, 91, 123, 63, 40, 81, 55, 104, 52, 108, 33, 101, 93, 83, 50, 58, 49, 12, 19, 17, 14, 76, 79, 94, 21, 126, 15, 36, 78, 88, 85, 24, 28, 98, 107, 92, 10, 43, 44, 84, 34, 7, 102, 20, 87, 37, 71, 74, 18, 9, 106, 80, 5, 16, 23, 42, 29, 32, 38, 41, 95, 69, 73, 72, 75, 82, 64, 96, 27, 66, 0, 31, 35, 8, 2, 67, 105, 3, 13, 77, 11, 99, 1, 65, 90, 68, 26, 6, 4, 70], [117, 112, 53, 115, 60, 113, 62, 51, 61, 127, 120, 47, 121, 45, 114, 39, 124, 118, 48, 86, 89, 57, 25, 22, 110, 100, 116, 125, 103, 109, 56, 63, 122, 55, 54, 59, 97, 91, 46, 30, 104, 40, 119, 111, 123, 81, 50, 52, 19, 49, 93, 108, 17, 101, 21, 12, 33, 83, 76, 15, 14, 126, 79, 98, 58, 92, 44, 94, 43, 78, 107, 28, 85, 88, 20, 87, 36, 106, 34, 24, 84, 42, 10, 18, 102, 9, 37, 74, 95, 7, 23, 16, 82, 41, 71, 27, 73, 32, 75, 80, 31, 96, 29, 35, 72, 38, 5, 69, 77, 0, 13, 11, 3, 105, 8, 99, 64, 2, 1, 26, 66, 67, 70, 6, 4, 68, 90, 65], [117, 112, 53, 115, 60, 62, 51, 113, 120, 61, 127, 47, 121, 45, 114, 118, 57, 48, 39, 56, 86, 124, 89, 100, 22, 25, 103, 111, 125, 55, 97, 63, 109, 59, 91, 54, 116, 110, 119, 122, 30, 93, 46, 40, 104, 81, 52, 108, 123, 126, 33, 36, 101, 50, 83, 19, 49, 94, 107, 76, 98, 21, 79, 17, 92, 15, 58, 14, 85, 28, 44, 20, 88, 12, 37, 42, 43, 78, 34, 87, 24, 27, 41, 106, 95, 84, 74, 102, 31, 96, 18, 23, 82, 35, 29, 38, 10, 105, 80, 7, 32, 73, 16, 72, 71, 99, 26, 5, 75, 9, 13, 11, 67, 77, 90, 69, 3, 66, 1, 2, 8, 0, 68, 64, 6, 4, 70, 65], [117, 112, 53, 115, 60, 62, 113, 51, 127, 61, 47, 120, 121, 45, 48, 114, 39, 118, 57, 56, 124, 86, 125, 89, 116, 100, 22, 54, 103, 25, 122, 55, 109, 111, 91, 119, 123, 59, 63, 40, 97, 30, 46, 104, 110, 93, 81, 33, 50, 108, 52, 94, 49, 19, 58, 83, 76, 15, 107, 101, 79, 21, 14, 126, 78, 17, 98, 36, 85, 44, 12, 74, 24, 28, 92, 20, 10, 42, 88, 71, 43, 34, 73, 37, 7, 23, 29, 16, 84, 102, 87, 32, 18, 72, 95, 38, 64, 82, 31, 80, 35, 27, 41, 96, 9, 5, 69, 0, 106, 75, 2, 99, 105, 3, 26, 66, 13, 11, 67, 77, 8, 6, 1, 65, 68, 90, 4, 70], [117, 112, 53, 115, 60, 62, 113, 51, 127, 120, 61, 47, 121, 45, 114, 48, 57, 56, 118, 39, 125, 124, 86, 100, 89, 55, 103, 25, 63, 110, 22, 119, 116, 109, 111, 54, 122, 97, 59, 91, 30, 46, 40, 108, 104, 94, 50, 33, 123, 93, 19, 81, 52, 101, 43, 79, 76, 83, 36, 21, 49, 14, 126, 98, 44, 15, 42, 17, 12, 78, 85, 92, 58, 28, 10, 24, 88, 34, 84, 74, 107, 20, 95, 102, 23, 7, 73, 18, 87, 106, 27, 71, 37, 9, 41, 72, 16, 38, 32, 75, 5, 80, 31, 35, 82, 29, 96, 105, 69, 0, 64, 13, 2, 26, 66, 11, 8, 67, 77, 99, 3, 90, 68, 6, 1, 4, 70, 65], [117, 112, 53, 115, 60, 62, 113, 51, 120, 127, 61, 47, 121, 45, 114, 39, 57, 124, 118, 48, 63, 86, 100, 116, 56, 103, 22, 89, 25, 110, 119, 125, 55, 97, 122, 54, 109, 91, 111, 40, 30, 59, 52, 108, 104, 46, 50, 81, 123, 101, 126, 93, 94, 83, 79, 19, 12, 49, 76, 17, 33, 36, 21, 14, 98, 44, 15, 85, 92, 78, 74, 107, 28, 58, 88, 42, 20, 34, 43, 24, 95, 87, 73, 37, 23, 82, 18, 106, 80, 84, 102, 16, 38, 96, 27, 9, 71, 32, 7, 41, 29, 72, 10, 99, 31, 105, 69, 5, 77, 75, 35, 67, 11, 26, 0, 13, 66, 8, 3, 64, 90, 2, 1, 6, 68, 4, 65, 70], [117, 112, 53, 115, 60, 62, 113, 51, 61, 120, 127, 47, 121, 45, 114, 39, 57, 48, 89, 100, 116, 124, 86, 118, 125, 22, 25, 103, 56, 122, 54, 63, 97, 111, 109, 55, 110, 104, 91, 59, 52, 30, 119, 123, 46, 93, 49, 40, 19, 126, 50, 81, 83, 108, 101, 17, 33, 36, 21, 94, 107, 98, 85, 28, 44, 34, 15, 14, 76, 58, 79, 92, 12, 78, 24, 43, 102, 20, 87, 88, 96, 27, 74, 106, 84, 42, 37, 41, 18, 23, 29, 95, 38, 80, 10, 16, 31, 71, 32, 99, 105, 9, 82, 7, 73, 35, 26, 72, 5, 77, 69, 90, 64, 75, 66, 67, 13, 0, 11, 65, 8, 3, 2, 4, 70, 68, 1, 6], [117, 112, 53, 60, 115, 62, 113, 51, 127, 61, 47, 120, 121, 45, 114, 48, 56, 57, 39, 103, 124, 100, 86, 118, 116, 89, 25, 22, 125, 54, 59, 91, 122, 63, 109, 119, 123, 104, 97, 55, 30, 111, 40, 110, 46, 93, 49, 33, 52, 81, 126, 50, 108, 19, 101, 36, 17, 94, 83, 76, 14, 79, 15, 92, 98, 107, 28, 21, 44, 78, 85, 74, 58, 12, 37, 24, 34, 42, 88, 20, 102, 87, 106, 10, 43, 84, 27, 41, 38, 23, 32, 95, 71, 96, 7, 80, 82, 16, 18, 73, 9, 29, 105, 31, 5, 35, 0, 69, 8, 2, 26, 3, 72, 66, 99, 64, 75, 77, 13, 90, 67, 11, 1, 4, 68, 70, 65, 6], [117, 112, 53, 115, 60, 62, 113, 51, 120, 127, 61, 47, 121, 45, 114, 39, 124, 57, 56, 48, 118, 100, 103, 63, 86, 25, 125, 22, 116, 109, 89, 122, 104, 111, 97, 110, 54, 119, 59, 123, 91, 108, 40, 30, 55, 126, 49, 33, 46, 94, 93, 50, 83, 19, 101, 17, 81, 98, 52, 76, 106, 85, 79, 44, 42, 107, 58, 78, 34, 14, 36, 21, 102, 37, 28, 92, 24, 12, 15, 10, 88, 38, 43, 74, 96, 84, 27, 32, 41, 95, 23, 87, 7, 20, 16, 35, 18, 80, 31, 9, 71, 5, 29, 82, 73, 0, 99, 69, 64, 26, 13, 67, 66, 8, 105, 72, 2, 75, 11, 3, 90, 77, 1, 70, 4, 65, 68, 6], [117, 112, 53, 115, 60, 62, 113, 51, 120, 127, 61, 47, 121, 45, 114, 39, 124, 57, 48, 89, 118, 86, 56, 100, 116, 103, 22, 54, 122, 110, 109, 97, 25, 63, 125, 30, 91, 46, 126, 59, 111, 52, 119, 55, 50, 104, 40, 33, 108, 123, 93, 81, 36, 49, 101, 19, 83, 94, 58, 12, 79, 98, 28, 21, 44, 85, 15, 34, 76, 78, 14, 42, 92, 74, 107, 17, 102, 24, 106, 37, 87, 20, 88, 95, 84, 41, 27, 31, 23, 38, 18, 96, 43, 29, 16, 82, 7, 80, 9, 71, 10, 32, 35, 8, 99, 105, 73, 26, 75, 66, 69, 13, 5, 90, 0, 11, 64, 77, 2, 67, 72, 3, 68, 1, 70, 4, 65, 6], [117, 112, 115, 53, 60, 113, 62, 51, 61, 120, 127, 47, 121, 45, 114, 124, 48, 56, 57, 118, 86, 39, 89, 116, 100, 103, 22, 125, 63, 109, 122, 110, 54, 59, 97, 91, 30, 25, 46, 55, 111, 40, 104, 126, 52, 119, 50, 123, 93, 81, 108, 33, 17, 21, 19, 15, 83, 101, 58, 76, 14, 79, 92, 85, 44, 12, 28, 49, 24, 34, 36, 98, 88, 74, 94, 107, 43, 78, 42, 37, 87, 20, 106, 84, 23, 102, 9, 16, 31, 18, 71, 96, 29, 35, 7, 80, 27, 32, 38, 95, 41, 10, 82, 26, 105, 69, 8, 5, 77, 75, 73, 66, 11, 3, 13, 64, 0, 2, 90, 72, 99, 67, 4, 70, 65, 68, 1, 6], [117, 112, 53, 115, 60, 113, 62, 51, 120, 61, 127, 47, 121, 45, 114, 56, 124, 57, 48, 118, 89, 100, 103, 86, 39, 116, 125, 122, 22, 63, 54, 40, 97, 91, 109, 25, 111, 59, 119, 104, 126, 30, 110, 55, 123, 46, 50, 81, 17, 93, 33, 83, 52, 94, 19, 58, 108, 49, 15, 101, 21, 98, 36, 28, 79, 76, 12, 37, 92, 88, 24, 14, 85, 44, 34, 43, 78, 20, 87, 27, 74, 107, 42, 106, 84, 31, 102, 96, 18, 38, 23, 80, 95, 7, 32, 71, 29, 35, 41, 69, 16, 10, 9, 0, 82, 8, 73, 64, 105, 26, 66, 5, 67, 99, 90, 77, 11, 3, 75, 13, 2, 1, 4, 72, 65, 6, 70, 68], [117, 112, 53, 115, 60, 113, 62, 51, 120, 61, 127, 47, 121, 45, 114, 39, 48, 124, 118, 56, 100, 86, 125, 89, 103, 57, 116, 22, 25, 97, 122, 54, 104, 63, 91, 59, 111, 109, 55, 119, 46, 110, 30, 40, 123, 108, 52, 126, 33, 81, 19, 93, 58, 50, 83, 101, 36, 15, 94, 85, 92, 17, 79, 76, 98, 28, 21, 49, 12, 14, 78, 34, 37, 43, 24, 44, 88, 87, 42, 38, 84, 102, 107, 96, 27, 20, 106, 10, 23, 32, 95, 18, 74, 31, 7, 29, 82, 71, 41, 80, 8, 35, 9, 105, 16, 99, 11, 5, 75, 69, 26, 64, 73, 66, 77, 2, 0, 67, 90, 13, 3, 72, 4, 65, 1, 6, 68, 70], [117, 112, 53, 115, 60, 113, 62, 51, 61, 120, 127, 47, 121, 45, 114, 39, 48, 56, 57, 116, 124, 100, 89, 103, 86, 118, 125, 22, 59, 91, 97, 110, 25, 122, 52, 30, 54, 109, 63, 111, 104, 46, 119, 40, 55, 126, 33, 123, 81, 108, 93, 19, 50, 49, 58, 83, 79, 17, 94, 85, 15, 101, 14, 12, 28, 44, 21, 92, 34, 76, 98, 74, 107, 24, 78, 88, 36, 37, 43, 102, 96, 42, 20, 84, 87, 106, 8, 27, 23, 10, 7, 95, 32, 31, 29, 38, 71, 35, 16, 73, 41, 82, 80, 18, 105, 9, 26, 69, 75, 5, 67, 11, 13, 99, 2, 90, 77, 64, 3, 0, 66, 72, 6, 4, 68, 1, 70, 65], [117, 112, 53, 115, 60, 113, 62, 51, 120, 127, 61, 47, 121, 45, 114, 86, 57, 118, 124, 48, 56, 125, 39, 22, 100, 103, 116, 122, 89, 59, 63, 25, 97, 109, 110, 91, 54, 40, 119, 30, 123, 111, 104, 81, 108, 55, 46, 126, 52, 50, 33, 83, 17, 14, 93, 19, 49, 76, 101, 85, 58, 21, 36, 92, 15, 44, 94, 12, 79, 107, 78, 43, 74, 28, 88, 42, 34, 98, 84, 24, 87, 106, 20, 102, 71, 23, 37, 96, 16, 27, 10, 18, 7, 95, 38, 32, 105, 82, 69, 80, 29, 41, 9, 8, 31, 73, 35, 11, 26, 77, 75, 5, 66, 3, 67, 0, 2, 72, 64, 13, 99, 90, 68, 6, 65, 1, 4, 70], [117, 112, 53, 115, 60, 62, 113, 51, 61, 120, 127, 47, 121, 45, 114, 57, 56, 118, 124, 86, 48, 125, 103, 89, 22, 100, 122, 39, 25, 63, 54, 119, 59, 116, 97, 46, 91, 30, 110, 109, 126, 55, 111, 104, 40, 123, 52, 108, 49, 81, 33, 19, 93, 92, 21, 17, 50, 94, 85, 44, 83, 58, 14, 101, 36, 15, 12, 76, 79, 98, 43, 28, 37, 88, 107, 24, 34, 106, 78, 87, 84, 96, 10, 95, 102, 23, 20, 42, 74, 27, 82, 7, 32, 41, 31, 29, 38, 80, 18, 73, 16, 35, 105, 71, 9, 8, 26, 99, 69, 11, 75, 77, 5, 72, 90, 66, 13, 0, 2, 64, 3, 67, 6, 68, 65, 4, 70, 1], [117, 112, 53, 115, 60, 113, 62, 51, 61, 127, 120, 47, 121, 45, 114, 124, 56, 118, 57, 48, 86, 100, 39, 103, 89, 125, 22, 25, 122, 59, 116, 123, 40, 63, 119, 91, 109, 54, 111, 55, 30, 46, 104, 97, 110, 93, 81, 108, 52, 50, 19, 33, 101, 126, 58, 17, 92, 44, 49, 36, 83, 94, 79, 15, 21, 85, 12, 14, 28, 76, 98, 102, 24, 78, 43, 34, 88, 37, 106, 107, 96, 87, 42, 20, 16, 84, 32, 23, 82, 10, 29, 95, 38, 27, 7, 31, 71, 73, 80, 74, 18, 41, 35, 99, 9, 26, 69, 72, 105, 75, 11, 5, 64, 2, 8, 67, 0, 66, 77, 13, 65, 3, 6, 90, 68, 1, 70, 4], [117, 112, 53, 60, 115, 62, 113, 51, 61, 120, 127, 47, 121, 45, 114, 124, 48, 39, 57, 118, 56, 86, 103, 89, 100, 116, 125, 22, 25, 119, 110, 63, 97, 30, 54, 40, 91, 122, 46, 52, 59, 109, 111, 55, 104, 123, 81, 58, 50, 126, 33, 19, 17, 93, 83, 49, 108, 94, 21, 85, 15, 24, 101, 36, 76, 79, 92, 12, 14, 78, 44, 37, 43, 98, 28, 84, 74, 88, 34, 10, 107, 20, 42, 106, 87, 102, 7, 16, 96, 71, 31, 23, 82, 41, 29, 27, 72, 9, 32, 18, 73, 80, 95, 105, 35, 5, 38, 26, 0, 69, 2, 77, 67, 64, 11, 66, 3, 99, 75, 13, 8, 90, 1, 68, 4, 70, 65, 6], [117, 112, 53, 60, 115, 62, 113, 51, 120, 61, 127, 47, 121, 45, 114, 39, 124, 48, 57, 118, 86, 100, 125, 116, 110, 89, 56, 22, 103, 25, 63, 119, 46, 109, 59, 40, 97, 91, 104, 52, 123, 122, 54, 30, 111, 108, 58, 101, 55, 93, 33, 81, 36, 126, 21, 50, 17, 19, 83, 94, 92, 12, 15, 98, 107, 44, 14, 43, 78, 76, 79, 85, 34, 49, 106, 42, 28, 87, 24, 88, 102, 10, 37, 84, 7, 18, 74, 41, 71, 20, 82, 32, 96, 9, 95, 31, 5, 72, 16, 38, 27, 23, 29, 69, 26, 2, 64, 11, 35, 73, 80, 0, 13, 105, 90, 66, 77, 75, 3, 99, 67, 4, 70, 8, 1, 65, 68, 6], [117, 112, 53, 60, 115, 62, 113, 51, 120, 61, 127, 47, 121, 45, 114, 39, 57, 124, 56, 86, 118, 125, 116, 100, 103, 22, 48, 89, 110, 25, 59, 54, 63, 97, 91, 122, 30, 40, 52, 109, 111, 126, 46, 123, 104, 58, 55, 119, 50, 108, 33, 17, 81, 93, 19, 83, 107, 85, 12, 14, 49, 21, 101, 15, 94, 92, 98, 36, 44, 79, 76, 78, 106, 34, 24, 42, 88, 28, 37, 43, 10, 84, 96, 20, 74, 102, 87, 27, 7, 9, 16, 23, 31, 38, 72, 95, 71, 41, 18, 32, 80, 73, 29, 82, 26, 105, 5, 77, 11, 69, 35, 66, 99, 3, 13, 67, 75, 64, 2, 90, 0, 65, 8, 68, 70, 4, 6, 1], [117, 112, 53, 115, 60, 62, 113, 51, 120, 127, 61, 47, 121, 45, 124, 39, 114, 57, 125, 56, 118, 103, 48, 116, 86, 100, 89, 22, 122, 91, 46, 40, 110, 63, 109, 25, 59, 111, 55, 123, 54, 97, 52, 50, 58, 126, 108, 104, 119, 30, 33, 81, 93, 101, 83, 94, 49, 17, 21, 36, 107, 79, 12, 19, 76, 44, 14, 78, 15, 92, 42, 28, 85, 24, 34, 98, 106, 20, 10, 37, 43, 71, 88, 84, 74, 7, 96, 87, 72, 27, 102, 32, 31, 9, 73, 95, 38, 80, 23, 16, 5, 69, 29, 82, 18, 11, 41, 66, 0, 105, 26, 64, 35, 99, 77, 13, 67, 2, 3, 70, 4, 75, 90, 8, 68, 1, 6, 65], [117, 112, 53, 115, 60, 113, 62, 51, 120, 61, 127, 47, 121, 45, 114, 39, 56, 124, 57, 86, 100, 118, 125, 89, 116, 48, 22, 103, 91, 25, 59, 111, 122, 97, 40, 109, 110, 119, 54, 123, 63, 30, 55, 104, 81, 46, 93, 126, 33, 19, 50, 52, 17, 49, 79, 83, 85, 101, 14, 94, 15, 92, 12, 21, 78, 108, 76, 58, 44, 98, 107, 36, 10, 28, 88, 74, 20, 106, 24, 43, 84, 87, 34, 37, 102, 42, 31, 72, 96, 71, 27, 95, 9, 73, 23, 7, 38, 18, 16, 32, 82, 80, 29, 35, 26, 69, 99, 5, 13, 11, 41, 75, 77, 105, 90, 8, 2, 66, 3, 70, 67, 0, 64, 6, 65, 4, 68, 1]], "model.layers.16.self_attn.q_proj": [[61, 125, 115, 121, 36, 101, 43, 62, 127, 63, 55, 92, 100, 97, 57, 40, 124, 60, 119, 54, 109, 49, 123, 126, 53, 59, 58, 113, 47, 117, 114, 118, 112, 50, 103, 51, 116, 52, 122, 45, 120, 102, 56, 41, 48, 89, 46, 110, 111, 37, 42, 107, 44, 108, 104, 32, 33, 39, 38, 105, 106, 99, 84, 35, 34, 98, 76, 10, 13, 24, 29, 72, 70, 95, 5, 16, 88, 21, 96, 2, 25, 71, 9, 4, 94, 86, 65, 0, 26, 80, 31, 23, 11, 93, 15, 30, 82, 77, 20, 3, 28, 78, 22, 67, 73, 18, 27, 90, 91, 87, 85, 6, 14, 75, 83, 79, 69, 8, 74, 19, 12, 81, 17, 68, 66, 1, 7, 64], [125, 61, 121, 115, 62, 55, 124, 101, 57, 126, 49, 52, 119, 127, 60, 112, 63, 50, 123, 120, 59, 53, 47, 117, 54, 113, 114, 118, 43, 56, 116, 122, 51, 107, 58, 48, 109, 45, 46, 111, 110, 40, 108, 44, 106, 92, 42, 41, 35, 105, 104, 39, 36, 38, 103, 37, 100, 102, 98, 99, 22, 25, 34, 97, 33, 96, 19, 32, 93, 89, 95, 18, 31, 94, 86, 15, 30, 29, 76, 78, 4, 26, 24, 23, 11, 72, 28, 16, 70, 9, 13, 77, 27, 21, 87, 83, 81, 90, 85, 17, 91, 88, 10, 71, 20, 2, 73, 84, 0, 82, 65, 74, 3, 79, 5, 80, 14, 67, 6, 12, 8, 1, 75, 64, 68, 66, 69, 7], [61, 125, 27, 121, 36, 94, 87, 100, 25, 62, 115, 35, 21, 63, 101, 20, 43, 127, 122, 60, 55, 32, 57, 49, 102, 40, 96, 74, 84, 95, 54, 126, 123, 52, 79, 53, 37, 113, 124, 118, 47, 119, 93, 114, 50, 117, 44, 120, 10, 112, 56, 51, 109, 59, 16, 48, 116, 83, 58, 45, 99, 46, 42, 19, 41, 104, 30, 31, 90, 82, 110, 29, 111, 92, 6, 107, 86, 12, 89, 105, 23, 13, 80, 15, 97, 98, 39, 91, 18, 28, 34, 108, 38, 77, 78, 33, 106, 17, 4, 24, 26, 103, 88, 73, 85, 5, 66, 68, 1, 9, 2, 76, 0, 67, 14, 81, 8, 22, 71, 7, 69, 64, 70, 72, 3, 75, 11, 65], [61, 125, 100, 96, 26, 30, 88, 122, 27, 121, 87, 77, 92, 98, 115, 32, 81, 75, 20, 43, 17, 93, 8, 94, 36, 62, 33, 29, 90, 44, 86, 99, 23, 41, 63, 34, 127, 40, 85, 49, 68, 101, 83, 57, 73, 102, 69, 19, 55, 110, 53, 56, 118, 60, 113, 48, 116, 2, 103, 16, 84, 13, 24, 12, 11, 50, 52, 47, 54, 31, 112, 51, 91, 79, 14, 109, 18, 6, 117, 126, 95, 78, 123, 119, 25, 97, 89, 21, 120, 124, 35, 111, 1, 72, 46, 42, 58, 76, 15, 114, 104, 45, 64, 59, 80, 38, 7, 28, 105, 67, 39, 107, 108, 106, 74, 82, 70, 71, 37, 66, 10, 4, 22, 9, 5, 3, 0, 65], [42, 98, 30, 124, 24, 85, 116, 106, 15, 19, 12, 17, 8, 88, 102, 78, 95, 52, 27, 10, 68, 66, 70, 94, 81, 38, 7, 119, 60, 90, 125, 22, 47, 83, 64, 120, 63, 55, 11, 75, 21, 59, 37, 92, 41, 53, 1, 26, 86, 76, 9, 111, 109, 114, 28, 44, 49, 103, 126, 23, 80, 117, 93, 121, 57, 36, 91, 108, 34, 25, 62, 123, 29, 6, 14, 48, 87, 35, 58, 3, 105, 79, 32, 45, 20, 33, 99, 118, 67, 113, 100, 18, 51, 69, 43, 115, 110, 31, 61, 46, 127, 5, 13, 122, 40, 89, 39, 72, 104, 56, 73, 54, 65, 2, 97, 82, 101, 50, 77, 107, 74, 0, 112, 16, 84, 96, 71, 4], [42, 124, 98, 63, 116, 30, 106, 38, 27, 19, 85, 88, 24, 62, 15, 35, 90, 17, 28, 10, 87, 57, 102, 99, 53, 122, 114, 47, 59, 61, 78, 109, 48, 108, 56, 60, 94, 123, 120, 125, 117, 113, 40, 12, 119, 26, 110, 43, 8, 45, 50, 126, 44, 118, 115, 31, 58, 52, 95, 46, 111, 80, 121, 93, 127, 112, 51, 54, 105, 36, 107, 86, 70, 81, 20, 49, 100, 68, 55, 37, 103, 39, 22, 96, 104, 33, 11, 41, 32, 18, 83, 21, 97, 23, 5, 84, 29, 101, 91, 34, 0, 92, 3, 6, 66, 74, 25, 89, 1, 82, 77, 71, 9, 16, 4, 13, 79, 7, 14, 75, 64, 69, 67, 76, 73, 65, 2, 72], [42, 98, 30, 124, 106, 85, 19, 63, 24, 27, 17, 59, 116, 15, 88, 12, 10, 78, 68, 8, 53, 70, 60, 94, 81, 102, 95, 28, 66, 122, 117, 52, 58, 45, 93, 125, 96, 76, 90, 64, 72, 89, 119, 38, 55, 120, 41, 103, 62, 100, 49, 47, 80, 82, 126, 114, 108, 21, 127, 83, 123, 44, 87, 79, 111, 113, 32, 37, 36, 35, 40, 14, 26, 20, 50, 43, 110, 46, 91, 121, 105, 48, 75, 25, 109, 115, 56, 92, 104, 1, 4, 71, 51, 61, 65, 101, 16, 6, 57, 11, 74, 31, 54, 29, 67, 22, 7, 3, 84, 23, 99, 33, 118, 2, 97, 39, 107, 13, 112, 86, 18, 73, 9, 77, 34, 5, 0, 69], [42, 98, 124, 52, 106, 30, 85, 15, 24, 27, 19, 88, 63, 60, 17, 45, 10, 116, 53, 95, 12, 108, 117, 59, 100, 121, 8, 113, 102, 94, 114, 119, 122, 48, 93, 47, 44, 58, 40, 62, 36, 37, 123, 78, 105, 35, 111, 49, 28, 125, 81, 90, 38, 57, 43, 104, 33, 56, 109, 46, 70, 103, 50, 41, 118, 68, 115, 23, 61, 51, 32, 54, 55, 127, 20, 126, 87, 6, 112, 39, 110, 80, 101, 120, 67, 91, 99, 25, 97, 96, 83, 21, 86, 82, 66, 92, 26, 107, 89, 31, 1, 3, 13, 71, 84, 74, 29, 22, 75, 79, 18, 11, 77, 34, 16, 64, 14, 72, 76, 69, 73, 2, 5, 9, 7, 65, 4, 0], [120, 39, 51, 48, 119, 123, 25, 112, 98, 89, 82, 53, 13, 31, 54, 44, 52, 113, 127, 111, 122, 121, 59, 58, 124, 60, 126, 117, 57, 50, 118, 116, 62, 61, 63, 125, 91, 107, 47, 114, 46, 115, 41, 49, 109, 56, 55, 20, 108, 28, 106, 45, 110, 42, 104, 43, 92, 103, 102, 1, 4, 11, 87, 105, 96, 65, 35, 36, 86, 101, 37, 72, 38, 100, 40, 33, 18, 99, 32, 66, 93, 90, 29, 95, 23, 7, 79, 97, 30, 69, 94, 8, 34, 80, 21, 77, 16, 27, 84, 15, 26, 85, 75, 14, 0, 9, 68, 19, 76, 88, 64, 22, 3, 24, 83, 78, 17, 5, 81, 10, 71, 67, 12, 2, 73, 6, 74, 70], [51, 48, 39, 120, 25, 98, 82, 123, 119, 87, 127, 91, 50, 121, 124, 54, 45, 122, 89, 106, 13, 31, 118, 53, 111, 86, 28, 52, 58, 112, 61, 113, 49, 46, 92, 126, 47, 110, 60, 57, 115, 62, 40, 117, 63, 44, 56, 108, 43, 55, 15, 41, 79, 116, 104, 59, 125, 77, 73, 36, 84, 109, 4, 38, 114, 65, 20, 23, 42, 103, 96, 7, 107, 99, 105, 27, 18, 100, 11, 102, 3, 37, 35, 68, 95, 8, 93, 101, 72, 88, 76, 34, 66, 94, 64, 22, 24, 30, 69, 85, 83, 90, 32, 33, 80, 78, 26, 75, 97, 10, 81, 17, 16, 29, 19, 74, 1, 2, 12, 70, 21, 14, 67, 9, 5, 71, 6, 0], [120, 39, 51, 25, 48, 98, 31, 89, 54, 127, 53, 123, 82, 20, 91, 11, 13, 56, 58, 118, 86, 49, 41, 57, 113, 116, 119, 112, 124, 15, 43, 4, 111, 47, 121, 60, 122, 62, 115, 28, 108, 63, 52, 107, 23, 114, 92, 50, 117, 126, 61, 59, 42, 45, 125, 7, 72, 32, 46, 55, 36, 102, 79, 88, 1, 104, 8, 109, 44, 19, 101, 93, 96, 27, 103, 87, 37, 77, 40, 110, 76, 99, 9, 33, 94, 106, 95, 17, 26, 66, 105, 69, 35, 100, 81, 30, 29, 97, 90, 85, 38, 75, 0, 84, 22, 21, 16, 80, 83, 24, 65, 10, 18, 68, 14, 34, 78, 6, 64, 12, 73, 2, 74, 70, 71, 5, 67, 3], [39, 51, 56, 120, 48, 123, 98, 119, 25, 20, 31, 89, 82, 28, 13, 50, 52, 54, 122, 43, 60, 87, 86, 41, 59, 79, 53, 91, 116, 88, 115, 4, 113, 124, 9, 92, 118, 7, 42, 58, 127, 126, 73, 112, 11, 57, 77, 114, 76, 125, 101, 109, 121, 61, 63, 1, 117, 81, 17, 111, 62, 102, 49, 10, 55, 19, 90, 108, 46, 8, 104, 45, 21, 85, 47, 93, 44, 22, 27, 74, 6, 38, 97, 23, 70, 75, 107, 14, 95, 32, 69, 15, 78, 40, 105, 103, 106, 84, 99, 33, 18, 96, 26, 16, 94, 0, 66, 36, 35, 110, 72, 67, 30, 80, 29, 12, 37, 24, 3, 5, 65, 100, 64, 71, 83, 34, 68, 2], [42, 100, 106, 90, 79, 78, 18, 20, 31, 85, 94, 75, 77, 111, 16, 86, 70, 73, 26, 4, 76, 46, 72, 126, 125, 124, 58, 3, 64, 67, 66, 56, 103, 23, 55, 117, 40, 28, 82, 21, 52, 2, 45, 65, 114, 80, 0, 11, 33, 127, 24, 10, 12, 119, 44, 71, 83, 81, 95, 88, 62, 15, 7, 97, 84, 8, 74, 5, 14, 53, 25, 93, 39, 122, 22, 109, 30, 59, 50, 61, 121, 69, 29, 6, 57, 89, 123, 9, 115, 13, 32, 113, 104, 47, 17, 91, 36, 54, 27, 41, 120, 99, 37, 51, 110, 105, 112, 34, 92, 116, 19, 108, 87, 48, 107, 43, 98, 60, 96, 35, 102, 49, 38, 68, 101, 118, 63, 1], [42, 100, 20, 106, 18, 77, 85, 90, 88, 111, 79, 75, 95, 73, 31, 70, 58, 40, 62, 23, 94, 26, 125, 52, 46, 16, 127, 68, 82, 78, 2, 89, 124, 104, 126, 45, 11, 109, 117, 72, 80, 61, 84, 56, 57, 67, 64, 1, 9, 99, 47, 114, 60, 33, 48, 96, 122, 21, 86, 103, 55, 12, 36, 14, 13, 97, 119, 3, 54, 25, 91, 15, 50, 115, 112, 101, 76, 17, 81, 4, 29, 65, 69, 39, 44, 102, 51, 32, 27, 92, 38, 8, 93, 53, 116, 105, 37, 24, 107, 22, 0, 123, 30, 49, 63, 41, 87, 108, 7, 110, 28, 5, 71, 83, 113, 74, 43, 59, 10, 118, 6, 34, 121, 120, 35, 19, 98, 66], [42, 100, 106, 20, 78, 90, 18, 85, 77, 73, 88, 111, 75, 94, 16, 70, 46, 124, 58, 26, 79, 72, 31, 125, 4, 56, 2, 23, 40, 67, 82, 52, 64, 127, 62, 33, 126, 104, 9, 117, 119, 95, 3, 103, 24, 11, 13, 45, 68, 57, 65, 55, 0, 99, 30, 86, 80, 29, 114, 44, 21, 8, 47, 48, 50, 66, 112, 22, 25, 60, 12, 91, 84, 69, 115, 7, 89, 109, 1, 81, 39, 96, 83, 116, 105, 97, 51, 36, 122, 93, 15, 123, 113, 92, 54, 32, 59, 87, 14, 28, 38, 63, 120, 110, 17, 74, 49, 53, 107, 118, 102, 61, 27, 108, 71, 10, 35, 5, 37, 34, 43, 41, 19, 76, 101, 6, 121, 98], [42, 100, 94, 88, 106, 85, 31, 78, 18, 90, 16, 111, 20, 79, 95, 70, 75, 77, 46, 86, 11, 73, 56, 124, 127, 40, 80, 72, 10, 55, 58, 45, 67, 26, 71, 21, 39, 65, 125, 104, 13, 83, 23, 119, 62, 117, 33, 82, 47, 103, 126, 5, 114, 12, 57, 91, 109, 2, 69, 0, 19, 97, 6, 74, 24, 107, 60, 1, 52, 84, 17, 61, 76, 53, 30, 81, 15, 29, 7, 102, 44, 113, 28, 115, 98, 105, 110, 4, 59, 38, 89, 120, 122, 101, 116, 99, 3, 25, 50, 32, 64, 22, 49, 93, 36, 8, 54, 123, 118, 34, 37, 112, 63, 92, 27, 108, 51, 35, 96, 87, 121, 43, 48, 41, 68, 66, 14, 9], [40, 54, 63, 36, 98, 122, 90, 20, 88, 81, 22, 123, 29, 78, 15, 93, 24, 13, 82, 33, 57, 62, 115, 126, 26, 38, 91, 68, 83, 51, 27, 76, 8, 61, 125, 120, 92, 114, 17, 127, 113, 34, 45, 124, 121, 21, 74, 19, 10, 11, 59, 100, 116, 50, 117, 119, 56, 46, 55, 97, 118, 73, 70, 16, 94, 31, 84, 53, 44, 107, 52, 28, 23, 48, 111, 106, 75, 109, 87, 18, 49, 77, 72, 80, 96, 69, 30, 35, 43, 25, 79, 89, 108, 112, 85, 0, 71, 32, 95, 101, 99, 1, 47, 39, 9, 2, 65, 58, 6, 37, 42, 4, 110, 105, 41, 103, 102, 60, 86, 14, 67, 3, 5, 104, 66, 64, 12, 7], [40, 54, 98, 63, 29, 82, 20, 76, 16, 122, 22, 71, 88, 50, 36, 3, 78, 90, 26, 93, 8, 73, 7, 83, 13, 61, 123, 15, 81, 84, 87, 5, 80, 66, 17, 64, 127, 115, 21, 77, 89, 74, 113, 65, 62, 12, 0, 125, 120, 100, 119, 86, 27, 23, 104, 97, 52, 91, 116, 121, 24, 2, 59, 19, 55, 112, 10, 57, 18, 48, 14, 51, 79, 118, 126, 9, 11, 43, 69, 53, 49, 92, 124, 45, 114, 56, 106, 6, 4, 111, 25, 33, 31, 117, 95, 70, 34, 30, 94, 67, 32, 85, 72, 46, 110, 28, 75, 37, 58, 38, 35, 103, 96, 99, 108, 42, 102, 109, 105, 101, 47, 1, 68, 60, 39, 41, 107, 44], [40, 61, 63, 122, 98, 54, 26, 123, 29, 88, 20, 24, 113, 93, 57, 22, 81, 90, 127, 125, 82, 78, 45, 76, 16, 73, 91, 92, 44, 121, 99, 32, 50, 62, 96, 19, 85, 38, 107, 15, 106, 28, 126, 47, 118, 112, 41, 120, 31, 109, 105, 119, 46, 116, 102, 21, 51, 39, 23, 74, 111, 117, 14, 55, 37, 43, 33, 89, 86, 35, 94, 53, 42, 103, 52, 5, 69, 60, 114, 110, 97, 115, 49, 80, 58, 12, 27, 17, 95, 13, 124, 108, 56, 48, 71, 59, 100, 83, 36, 104, 25, 101, 84, 87, 11, 30, 3, 10, 79, 66, 0, 64, 75, 68, 2, 34, 9, 18, 65, 8, 70, 1, 77, 6, 67, 4, 72, 7], [63, 40, 54, 122, 100, 123, 125, 127, 62, 57, 98, 113, 121, 119, 118, 59, 116, 115, 61, 51, 55, 114, 120, 53, 29, 124, 50, 52, 117, 91, 48, 56, 126, 20, 111, 26, 45, 36, 93, 88, 49, 112, 46, 87, 47, 25, 110, 27, 43, 106, 22, 81, 103, 109, 23, 108, 58, 60, 107, 99, 39, 38, 42, 44, 105, 41, 102, 96, 35, 101, 34, 33, 97, 37, 89, 104, 94, 92, 83, 21, 82, 28, 32, 79, 86, 24, 18, 90, 95, 15, 31, 16, 30, 75, 84, 13, 17, 85, 11, 80, 0, 73, 19, 74, 10, 2, 1, 72, 76, 64, 3, 69, 66, 68, 5, 8, 6, 78, 14, 71, 9, 70, 67, 77, 4, 12, 65, 7], [48, 52, 63, 116, 127, 55, 123, 61, 53, 54, 124, 117, 51, 56, 119, 62, 60, 122, 115, 126, 120, 112, 118, 59, 47, 111, 121, 49, 114, 57, 125, 58, 113, 50, 45, 46, 109, 36, 110, 101, 107, 108, 42, 44, 106, 43, 105, 98, 32, 96, 41, 38, 40, 39, 103, 100, 104, 37, 92, 102, 34, 90, 26, 99, 33, 35, 89, 93, 95, 97, 91, 31, 15, 94, 28, 20, 30, 23, 27, 29, 86, 22, 84, 17, 85, 81, 79, 87, 21, 25, 82, 76, 88, 14, 18, 74, 12, 72, 8, 10, 16, 83, 78, 24, 66, 69, 67, 5, 71, 77, 7, 3, 19, 11, 64, 2, 68, 80, 6, 4, 70, 73, 13, 75, 0, 1, 65, 9], [116, 52, 48, 101, 57, 37, 100, 97, 120, 59, 123, 94, 95, 63, 112, 114, 58, 54, 126, 38, 34, 60, 92, 50, 110, 127, 15, 125, 46, 39, 47, 113, 43, 55, 106, 61, 118, 56, 124, 111, 121, 44, 45, 99, 32, 22, 49, 105, 107, 103, 108, 119, 115, 40, 62, 51, 53, 42, 36, 109, 88, 85, 117, 98, 41, 122, 102, 35, 76, 91, 31, 23, 104, 89, 27, 28, 96, 90, 86, 30, 12, 17, 18, 20, 87, 26, 33, 29, 82, 84, 93, 25, 79, 14, 72, 21, 81, 24, 74, 83, 19, 10, 8, 78, 16, 80, 71, 5, 67, 69, 7, 3, 77, 11, 13, 73, 75, 2, 70, 9, 4, 66, 68, 6, 1, 65, 0, 64], [116, 48, 97, 101, 52, 88, 83, 77, 11, 16, 73, 4, 71, 22, 14, 70, 126, 85, 18, 61, 0, 2, 68, 42, 30, 64, 26, 27, 105, 55, 75, 9, 28, 80, 13, 93, 17, 47, 120, 65, 82, 121, 6, 24, 7, 67, 19, 41, 91, 5, 78, 118, 66, 37, 74, 127, 20, 69, 34, 21, 86, 10, 89, 23, 108, 3, 29, 31, 54, 57, 81, 53, 84, 87, 79, 114, 15, 36, 103, 72, 123, 50, 12, 8, 99, 1, 76, 45, 35, 43, 51, 33, 25, 92, 96, 98, 115, 46, 110, 94, 95, 90, 49, 112, 32, 113, 109, 102, 38, 44, 63, 39, 100, 106, 104, 40, 58, 59, 125, 60, 56, 124, 119, 62, 111, 107, 117, 122], [52, 63, 116, 48, 60, 114, 55, 51, 54, 122, 124, 53, 117, 125, 59, 127, 126, 120, 119, 47, 62, 58, 115, 123, 118, 61, 121, 111, 113, 46, 49, 101, 57, 56, 50, 109, 110, 45, 36, 112, 42, 98, 107, 32, 105, 43, 44, 97, 92, 106, 15, 108, 41, 37, 94, 90, 96, 104, 40, 103, 38, 39, 99, 100, 102, 35, 85, 34, 23, 26, 95, 33, 84, 29, 82, 87, 28, 30, 20, 27, 76, 93, 31, 91, 79, 89, 25, 17, 22, 86, 18, 81, 12, 21, 74, 8, 10, 72, 88, 5, 69, 24, 83, 14, 16, 71, 67, 78, 77, 7, 2, 3, 66, 4, 73, 9, 19, 13, 68, 11, 80, 0, 65, 75, 70, 6, 64, 1], [124, 49, 37, 55, 126, 61, 87, 118, 26, 93, 96, 121, 80, 12, 86, 119, 57, 84, 101, 50, 54, 60, 78, 123, 25, 90, 32, 67, 122, 18, 58, 85, 115, 112, 72, 29, 51, 16, 102, 120, 56, 6, 127, 59, 53, 15, 83, 52, 116, 34, 63, 92, 38, 89, 114, 62, 47, 46, 76, 79, 111, 103, 110, 104, 117, 125, 48, 98, 9, 44, 109, 36, 17, 43, 39, 45, 97, 81, 33, 21, 27, 107, 30, 10, 20, 108, 23, 106, 113, 35, 28, 31, 74, 73, 105, 42, 70, 88, 41, 22, 95, 91, 94, 11, 99, 13, 3, 19, 40, 77, 100, 24, 64, 14, 82, 75, 66, 4, 1, 69, 7, 0, 8, 65, 71, 68, 2, 5], [49, 124, 37, 55, 93, 61, 96, 126, 50, 118, 54, 57, 119, 123, 102, 121, 87, 79, 127, 112, 115, 53, 60, 122, 58, 125, 51, 120, 110, 114, 59, 85, 56, 90, 101, 26, 63, 62, 43, 116, 38, 36, 5, 46, 108, 113, 42, 52, 48, 117, 111, 44, 86, 47, 107, 41, 32, 30, 45, 109, 12, 39, 40, 104, 18, 35, 80, 105, 21, 103, 106, 24, 99, 20, 69, 100, 29, 25, 84, 33, 34, 73, 8, 92, 31, 98, 71, 89, 95, 77, 2, 94, 6, 23, 97, 91, 27, 17, 67, 28, 0, 88, 19, 11, 22, 64, 72, 15, 13, 7, 83, 16, 78, 3, 65, 81, 76, 9, 4, 70, 75, 82, 10, 74, 66, 14, 1, 68], [49, 124, 37, 55, 61, 126, 101, 118, 93, 119, 26, 87, 96, 50, 57, 123, 60, 121, 54, 21, 44, 115, 122, 58, 29, 12, 84, 85, 51, 56, 63, 90, 53, 120, 125, 47, 112, 102, 25, 59, 110, 62, 48, 116, 46, 52, 113, 114, 105, 100, 104, 45, 108, 111, 127, 43, 39, 86, 117, 103, 19, 109, 99, 27, 107, 38, 42, 28, 34, 88, 81, 80, 106, 41, 36, 72, 33, 31, 6, 40, 97, 79, 32, 35, 24, 78, 95, 98, 75, 30, 92, 67, 20, 73, 94, 17, 18, 83, 23, 82, 77, 91, 22, 10, 15, 89, 76, 14, 3, 69, 11, 8, 64, 13, 16, 70, 9, 7, 71, 0, 65, 74, 4, 5, 66, 1, 68, 2], [49, 124, 37, 55, 87, 61, 126, 118, 96, 50, 57, 123, 26, 80, 93, 101, 54, 121, 90, 86, 119, 60, 12, 122, 120, 51, 127, 115, 58, 53, 78, 84, 6, 56, 63, 114, 67, 112, 29, 52, 59, 47, 104, 85, 116, 113, 46, 25, 48, 125, 62, 102, 111, 20, 83, 65, 44, 89, 70, 16, 8, 110, 108, 107, 15, 79, 23, 21, 45, 18, 33, 99, 117, 109, 64, 92, 43, 35, 94, 72, 39, 40, 106, 32, 77, 42, 22, 2, 17, 0, 97, 71, 38, 27, 36, 103, 105, 10, 28, 76, 100, 14, 81, 30, 95, 73, 88, 34, 41, 4, 19, 9, 24, 3, 11, 13, 91, 31, 98, 69, 7, 68, 74, 82, 75, 1, 5, 66], [103, 34, 45, 28, 109, 84, 24, 38, 79, 17, 85, 86, 30, 92, 31, 12, 11, 95, 9, 22, 46, 111, 49, 23, 122, 116, 88, 14, 19, 63, 27, 36, 20, 75, 57, 44, 33, 121, 53, 107, 119, 52, 83, 115, 101, 113, 104, 124, 15, 50, 32, 48, 61, 100, 37, 97, 29, 91, 25, 94, 125, 108, 26, 93, 54, 73, 126, 106, 16, 55, 47, 56, 114, 40, 43, 42, 35, 117, 99, 112, 51, 120, 60, 21, 89, 58, 41, 123, 81, 102, 82, 62, 59, 110, 72, 105, 96, 127, 13, 90, 118, 18, 10, 87, 39, 8, 5, 98, 7, 68, 76, 78, 77, 6, 4, 80, 65, 69, 74, 67, 3, 66, 70, 1, 71, 2, 64, 0], [103, 45, 34, 28, 14, 109, 17, 12, 7, 84, 71, 66, 79, 9, 3, 5, 24, 85, 69, 44, 64, 65, 67, 31, 86, 10, 20, 96, 127, 98, 105, 81, 2, 61, 25, 76, 35, 38, 39, 51, 0, 63, 8, 124, 52, 49, 55, 72, 125, 58, 90, 23, 30, 68, 78, 19, 4, 92, 73, 75, 122, 115, 118, 57, 16, 88, 95, 111, 100, 70, 11, 77, 6, 74, 21, 46, 119, 102, 60, 89, 18, 113, 94, 13, 59, 97, 120, 83, 93, 121, 107, 126, 36, 80, 43, 53, 1, 40, 15, 116, 29, 110, 112, 54, 91, 22, 56, 123, 37, 50, 82, 48, 117, 101, 47, 41, 87, 114, 42, 99, 27, 62, 104, 33, 106, 108, 26, 32], [103, 45, 34, 38, 109, 0, 28, 79, 7, 3, 24, 84, 12, 9, 14, 2, 17, 66, 5, 31, 86, 85, 1, 67, 10, 125, 69, 122, 92, 57, 100, 25, 23, 20, 112, 64, 98, 35, 61, 58, 44, 121, 51, 26, 55, 22, 68, 21, 72, 118, 59, 49, 96, 81, 65, 105, 102, 75, 48, 82, 32, 30, 123, 70, 63, 52, 74, 90, 54, 46, 93, 13, 76, 36, 60, 111, 116, 88, 56, 124, 42, 119, 104, 6, 126, 127, 8, 120, 50, 89, 40, 110, 73, 71, 115, 18, 80, 4, 114, 94, 95, 83, 19, 91, 43, 108, 16, 62, 117, 47, 39, 29, 78, 99, 77, 15, 113, 53, 27, 107, 87, 37, 106, 97, 101, 41, 11, 33], [103, 34, 45, 28, 109, 84, 79, 24, 31, 92, 17, 86, 22, 12, 108, 63, 9, 29, 88, 30, 102, 55, 25, 89, 98, 122, 112, 58, 14, 105, 104, 53, 100, 27, 49, 20, 61, 11, 50, 57, 114, 72, 113, 33, 42, 121, 127, 81, 95, 52, 116, 38, 23, 35, 90, 83, 36, 85, 111, 124, 41, 93, 125, 18, 126, 46, 77, 39, 74, 94, 115, 123, 80, 5, 101, 59, 26, 82, 62, 119, 16, 73, 19, 13, 43, 21, 47, 118, 96, 76, 56, 32, 15, 91, 48, 51, 87, 99, 110, 106, 78, 44, 117, 97, 75, 7, 37, 8, 10, 107, 70, 54, 60, 4, 6, 120, 68, 40, 69, 3, 71, 66, 67, 65, 1, 64, 0, 2]], "model.layers.16.self_attn.k_proj": [[125, 61, 86, 36, 96, 127, 17, 115, 121, 49, 30, 62, 63, 55, 57, 113, 60, 54, 26, 122, 53, 39, 51, 118, 48, 56, 126, 107, 116, 47, 112, 59, 52, 29, 58, 108, 124, 87, 40, 114, 120, 109, 44, 119, 123, 117, 79, 50, 27, 83, 11, 20, 110, 46, 78, 18, 43, 45, 111, 13, 25, 85, 98, 105, 106, 97, 41, 24, 42, 104, 72, 16, 82, 33, 71, 38, 101, 103, 9, 34, 28, 76, 10, 37, 91, 102, 100, 89, 31, 88, 35, 4, 99, 22, 70, 80, 95, 5, 94, 92, 21, 15, 14, 2, 32, 90, 19, 93, 77, 12, 65, 84, 23, 81, 0, 7, 3, 67, 73, 75, 6, 8, 74, 1, 69, 68, 66, 64], [106, 94, 34, 124, 116, 24, 63, 19, 52, 59, 85, 17, 15, 42, 55, 60, 121, 10, 70, 78, 12, 120, 27, 49, 8, 117, 111, 58, 47, 108, 114, 53, 109, 64, 68, 18, 1, 66, 4, 126, 123, 50, 122, 22, 44, 102, 45, 91, 51, 125, 46, 127, 90, 0, 119, 62, 112, 54, 56, 11, 84, 105, 104, 48, 61, 100, 28, 57, 41, 110, 40, 113, 115, 95, 118, 14, 80, 92, 103, 37, 77, 107, 101, 76, 73, 69, 2, 16, 26, 89, 32, 99, 43, 31, 13, 35, 9, 36, 23, 33, 97, 87, 20, 25, 7, 39, 93, 65, 96, 38, 75, 71, 67, 3, 86, 29, 74, 79, 5, 88, 82, 21, 83, 30, 72, 6, 98, 81], [103, 51, 120, 34, 86, 56, 48, 95, 123, 115, 112, 119, 53, 28, 54, 122, 58, 89, 60, 126, 52, 88, 40, 82, 61, 15, 124, 118, 111, 50, 121, 63, 46, 57, 113, 47, 62, 125, 117, 55, 107, 116, 20, 26, 114, 98, 91, 108, 110, 8, 59, 45, 49, 44, 81, 11, 43, 127, 109, 13, 42, 4, 38, 105, 93, 65, 41, 73, 106, 6, 64, 30, 16, 68, 37, 24, 27, 104, 72, 23, 102, 35, 33, 36, 85, 29, 80, 100, 76, 94, 101, 99, 12, 9, 87, 97, 14, 67, 1, 21, 79, 19, 66, 83, 71, 78, 17, 32, 90, 96, 10, 7, 2, 18, 92, 0, 70, 74, 77, 25, 5, 3, 84, 39, 69, 31, 75, 22], [106, 36, 90, 18, 85, 47, 20, 42, 79, 77, 110, 16, 58, 72, 75, 124, 126, 73, 78, 70, 56, 127, 109, 117, 119, 114, 30, 95, 120, 40, 86, 4, 125, 65, 108, 76, 112, 62, 45, 52, 43, 67, 60, 99, 88, 61, 96, 51, 39, 53, 111, 0, 66, 54, 93, 97, 69, 68, 8, 100, 19, 116, 25, 55, 38, 74, 13, 103, 32, 91, 29, 115, 48, 6, 23, 57, 89, 41, 107, 64, 123, 11, 2, 31, 7, 5, 63, 101, 81, 1, 44, 94, 50, 122, 104, 92, 87, 27, 28, 17, 49, 46, 37, 105, 34, 3, 118, 12, 113, 33, 71, 83, 35, 121, 22, 80, 59, 102, 14, 9, 26, 98, 15, 82, 24, 84, 10, 21], [104, 54, 63, 34, 22, 93, 122, 120, 90, 123, 62, 119, 125, 121, 126, 127, 61, 50, 88, 59, 113, 51, 118, 81, 52, 20, 16, 116, 115, 76, 55, 64, 46, 82, 57, 53, 48, 114, 117, 56, 71, 60, 73, 91, 124, 45, 69, 111, 78, 49, 39, 109, 13, 106, 110, 43, 112, 2, 58, 44, 15, 108, 1, 87, 98, 47, 74, 11, 102, 67, 42, 80, 79, 41, 29, 100, 103, 37, 105, 96, 107, 35, 14, 92, 97, 21, 28, 25, 26, 10, 85, 95, 83, 99, 32, 19, 38, 9, 101, 8, 18, 33, 30, 70, 94, 36, 12, 72, 3, 31, 65, 68, 23, 75, 24, 40, 77, 27, 6, 89, 4, 86, 66, 84, 17, 0, 7, 5], [52, 116, 37, 48, 83, 16, 77, 33, 22, 88, 11, 112, 73, 70, 4, 71, 14, 126, 61, 127, 63, 0, 2, 50, 123, 106, 1, 121, 29, 44, 51, 55, 58, 114, 60, 111, 120, 57, 125, 93, 115, 30, 118, 56, 108, 110, 91, 122, 119, 99, 53, 49, 113, 124, 59, 54, 85, 62, 105, 117, 46, 47, 109, 107, 41, 43, 40, 102, 104, 42, 90, 103, 35, 81, 74, 101, 45, 95, 38, 75, 18, 39, 98, 8, 92, 5, 36, 9, 67, 12, 25, 32, 96, 20, 97, 34, 100, 3, 65, 94, 31, 21, 78, 89, 13, 23, 69, 24, 82, 17, 84, 87, 19, 76, 15, 64, 7, 79, 10, 28, 6, 27, 80, 72, 68, 26, 66, 86], [124, 49, 101, 113, 86, 55, 32, 126, 57, 118, 29, 54, 61, 119, 121, 50, 123, 120, 26, 58, 122, 53, 115, 51, 114, 52, 63, 47, 60, 56, 125, 116, 62, 59, 111, 112, 48, 45, 108, 80, 46, 109, 44, 103, 99, 18, 127, 87, 43, 117, 78, 110, 107, 105, 37, 42, 106, 40, 102, 98, 72, 64, 104, 12, 39, 97, 73, 3, 20, 38, 84, 41, 21, 19, 36, 100, 17, 79, 94, 92, 33, 34, 25, 30, 23, 35, 85, 89, 27, 68, 81, 6, 31, 28, 24, 10, 77, 95, 14, 91, 4, 70, 93, 83, 88, 66, 1, 16, 96, 11, 5, 71, 90, 75, 0, 22, 7, 67, 13, 65, 74, 82, 15, 9, 8, 2, 76, 69], [109, 39, 98, 84, 92, 12, 17, 64, 7, 79, 14, 9, 5, 45, 3, 66, 61, 121, 41, 49, 122, 52, 86, 95, 127, 58, 21, 119, 40, 111, 124, 55, 65, 57, 118, 63, 51, 24, 23, 126, 48, 116, 25, 102, 28, 72, 30, 60, 34, 1, 22, 44, 125, 47, 0, 106, 59, 100, 18, 120, 90, 80, 82, 117, 112, 32, 97, 46, 89, 36, 29, 104, 6, 11, 19, 4, 70, 53, 26, 74, 123, 93, 77, 35, 108, 99, 85, 68, 62, 27, 50, 16, 56, 8, 107, 31, 115, 88, 114, 78, 83, 15, 91, 38, 20, 105, 94, 37, 42, 76, 13, 54, 43, 96, 110, 10, 87, 33, 73, 101, 113, 2, 71, 81, 67, 69, 75, 103]], "model.layers.16.self_attn.qk_proj": [[106, 124, 125, 116, 61, 52, 109, 42, 49, 63, 51, 54, 120, 48, 121, 45, 53, 123, 55, 119, 127, 56, 58, 98, 126, 28, 22, 39, 34, 122, 26, 84, 20, 88, 86, 79, 57, 15, 36, 59, 112, 101, 37, 40, 100, 62, 113, 118, 103, 50, 24, 30, 47, 117, 81, 85, 78, 111, 21, 115, 104, 12, 60, 29, 82, 14, 17, 76, 93, 90, 9, 92, 114, 80, 18, 16, 73, 94, 77, 110, 11, 31, 44, 83, 95, 27, 13, 46, 19, 72, 96, 108, 71, 75, 70, 25, 107, 102, 89, 64, 91, 32, 43, 97, 3, 7, 87, 23, 8, 68, 0, 41, 4, 67, 38, 6, 69, 5, 74, 105, 33, 2, 66, 10, 35, 99, 1, 65], [106, 124, 125, 61, 116, 52, 109, 42, 63, 49, 51, 54, 120, 48, 45, 121, 56, 123, 119, 127, 98, 58, 55, 53, 28, 126, 34, 39, 20, 86, 22, 118, 36, 26, 88, 84, 37, 79, 122, 47, 15, 59, 57, 111, 113, 24, 101, 112, 40, 115, 21, 100, 62, 30, 103, 85, 117, 60, 81, 29, 78, 76, 114, 14, 50, 104, 12, 92, 17, 90, 82, 93, 73, 9, 44, 77, 94, 16, 31, 18, 108, 80, 11, 72, 27, 46, 83, 13, 95, 96, 70, 107, 75, 110, 89, 19, 25, 7, 64, 102, 0, 87, 91, 38, 67, 8, 32, 71, 97, 43, 4, 68, 23, 5, 69, 41, 3, 105, 2, 74, 66, 35, 33, 1, 99, 6, 65, 10], [106, 125, 124, 116, 61, 52, 109, 42, 49, 63, 51, 54, 120, 48, 45, 121, 119, 123, 55, 98, 126, 56, 127, 58, 53, 28, 34, 39, 88, 36, 101, 122, 86, 84, 26, 100, 22, 20, 47, 113, 118, 57, 62, 79, 24, 59, 112, 85, 15, 40, 37, 103, 81, 50, 111, 104, 21, 60, 117, 29, 115, 30, 114, 93, 14, 90, 78, 12, 94, 17, 18, 92, 9, 27, 73, 44, 82, 76, 64, 31, 102, 110, 13, 95, 80, 83, 16, 96, 91, 11, 0, 77, 46, 25, 108, 72, 70, 19, 32, 107, 38, 75, 7, 71, 87, 4, 68, 3, 89, 8, 97, 41, 23, 1, 66, 69, 105, 67, 2, 65, 33, 35, 43, 5, 74, 99, 6, 10], [106, 125, 124, 116, 61, 52, 63, 109, 49, 42, 51, 54, 120, 48, 45, 121, 53, 123, 119, 56, 127, 58, 98, 55, 126, 28, 39, 59, 62, 57, 22, 20, 79, 101, 118, 34, 36, 26, 84, 100, 47, 113, 122, 86, 37, 24, 115, 112, 111, 40, 117, 15, 50, 30, 88, 103, 85, 104, 81, 29, 21, 14, 78, 60, 90, 92, 114, 93, 17, 94, 76, 27, 9, 73, 18, 0, 12, 44, 82, 80, 110, 16, 83, 13, 96, 77, 46, 108, 102, 70, 31, 19, 11, 107, 95, 72, 75, 7, 25, 97, 91, 38, 67, 87, 89, 71, 68, 8, 4, 41, 43, 64, 3, 2, 32, 66, 69, 23, 1, 5, 105, 33, 65, 35, 10, 99, 74, 6], [106, 124, 125, 116, 61, 52, 63, 49, 109, 42, 51, 54, 120, 48, 45, 121, 56, 123, 119, 53, 127, 126, 58, 55, 98, 57, 20, 22, 34, 39, 86, 47, 122, 59, 101, 118, 28, 112, 79, 62, 50, 84, 15, 40, 100, 88, 26, 36, 103, 111, 24, 117, 114, 81, 37, 30, 115, 85, 104, 14, 17, 113, 18, 78, 12, 93, 29, 60, 44, 76, 27, 73, 9, 90, 21, 94, 11, 80, 92, 13, 83, 68, 82, 107, 110, 77, 16, 46, 108, 64, 19, 31, 102, 72, 95, 0, 96, 25, 7, 8, 71, 4, 70, 67, 75, 41, 3, 91, 38, 69, 87, 23, 89, 2, 43, 32, 97, 1, 5, 33, 66, 6, 105, 35, 65, 99, 74, 10], [106, 124, 125, 116, 61, 52, 63, 109, 42, 49, 51, 54, 120, 48, 45, 121, 119, 123, 58, 127, 56, 126, 53, 98, 55, 20, 26, 22, 34, 47, 79, 57, 39, 84, 86, 62, 118, 59, 15, 103, 101, 88, 28, 36, 100, 111, 81, 24, 112, 122, 37, 50, 85, 30, 40, 60, 78, 117, 114, 14, 17, 29, 76, 104, 93, 27, 12, 21, 80, 82, 18, 73, 90, 16, 11, 94, 113, 13, 9, 92, 77, 44, 83, 115, 31, 19, 46, 110, 108, 107, 75, 96, 89, 25, 72, 8, 95, 7, 64, 23, 71, 6, 32, 91, 102, 0, 41, 97, 43, 69, 68, 87, 4, 67, 3, 70, 105, 5, 2, 35, 38, 66, 65, 33, 74, 99, 1, 10], [106, 125, 124, 116, 61, 52, 63, 109, 42, 49, 51, 120, 54, 48, 45, 119, 121, 127, 53, 56, 123, 126, 58, 98, 55, 20, 26, 57, 122, 84, 86, 22, 88, 34, 79, 118, 36, 47, 62, 28, 39, 81, 15, 24, 112, 101, 103, 37, 50, 59, 40, 30, 114, 100, 111, 85, 14, 78, 115, 90, 18, 117, 12, 17, 104, 29, 93, 27, 80, 21, 60, 113, 44, 76, 73, 94, 9, 82, 16, 83, 11, 92, 13, 77, 110, 107, 95, 19, 25, 31, 96, 108, 46, 41, 43, 75, 71, 89, 23, 102, 97, 6, 91, 32, 8, 7, 0, 72, 38, 3, 33, 67, 87, 64, 2, 69, 4, 35, 68, 74, 105, 5, 99, 66, 70, 10, 65, 1], [106, 124, 116, 125, 61, 52, 109, 42, 63, 49, 51, 54, 120, 48, 45, 119, 121, 127, 53, 56, 123, 55, 126, 98, 58, 22, 20, 26, 28, 84, 39, 86, 122, 101, 34, 57, 88, 36, 24, 100, 79, 15, 30, 81, 103, 37, 40, 112, 111, 62, 59, 85, 115, 47, 50, 118, 14, 113, 60, 117, 90, 17, 78, 93, 21, 114, 29, 104, 92, 80, 18, 12, 27, 16, 82, 73, 9, 13, 25, 76, 83, 95, 96, 44, 94, 77, 46, 107, 32, 11, 43, 91, 110, 102, 6, 19, 8, 31, 89, 0, 108, 75, 71, 64, 23, 41, 7, 97, 68, 105, 87, 33, 38, 72, 67, 3, 66, 4, 99, 35, 69, 2, 74, 5, 65, 1, 70, 10], [106, 124, 116, 125, 61, 52, 109, 42, 63, 51, 49, 54, 120, 48, 45, 127, 119, 121, 53, 98, 56, 55, 58, 34, 28, 123, 22, 20, 39, 126, 84, 101, 26, 88, 103, 86, 24, 100, 122, 36, 37, 79, 85, 112, 50, 111, 15, 30, 47, 57, 115, 113, 59, 60, 40, 62, 118, 17, 81, 78, 114, 14, 12, 21, 93, 90, 104, 117, 29, 18, 27, 76, 82, 80, 44, 108, 92, 16, 9, 73, 95, 13, 25, 77, 94, 83, 46, 96, 102, 19, 11, 8, 31, 107, 43, 75, 6, 91, 89, 0, 71, 110, 32, 23, 87, 64, 7, 97, 33, 41, 2, 38, 4, 72, 105, 67, 68, 66, 3, 35, 5, 69, 99, 1, 10, 74, 65, 70], [106, 124, 125, 61, 116, 52, 109, 42, 63, 49, 51, 54, 48, 120, 45, 121, 127, 119, 53, 56, 123, 55, 98, 58, 126, 34, 28, 101, 20, 57, 59, 84, 22, 26, 47, 115, 39, 103, 118, 36, 37, 122, 88, 50, 86, 100, 79, 111, 15, 24, 117, 113, 30, 40, 112, 17, 85, 62, 21, 12, 81, 78, 104, 60, 90, 44, 14, 9, 93, 29, 114, 18, 76, 92, 82, 16, 8, 94, 110, 27, 80, 73, 13, 83, 108, 46, 95, 96, 11, 75, 77, 6, 102, 7, 0, 64, 25, 19, 31, 32, 107, 71, 89, 3, 72, 38, 43, 4, 97, 23, 70, 91, 5, 33, 41, 68, 67, 2, 87, 66, 69, 99, 35, 74, 105, 1, 65, 10], [106, 124, 116, 125, 61, 52, 109, 42, 63, 49, 54, 51, 120, 48, 121, 45, 127, 53, 123, 119, 56, 58, 98, 55, 126, 122, 118, 57, 28, 34, 20, 15, 84, 39, 79, 22, 59, 86, 101, 26, 100, 50, 81, 88, 36, 113, 85, 37, 47, 111, 24, 40, 30, 115, 62, 60, 17, 112, 9, 12, 14, 78, 103, 117, 82, 21, 76, 73, 29, 104, 8, 18, 92, 90, 114, 93, 16, 27, 44, 94, 11, 0, 13, 80, 77, 83, 31, 64, 19, 68, 110, 7, 108, 75, 71, 70, 96, 107, 72, 25, 46, 4, 91, 67, 102, 95, 32, 89, 3, 87, 6, 2, 66, 1, 69, 5, 23, 65, 38, 97, 41, 33, 35, 105, 43, 74, 10, 99], [106, 124, 116, 125, 61, 52, 63, 109, 42, 49, 51, 54, 120, 48, 45, 121, 127, 53, 119, 58, 56, 126, 98, 123, 122, 86, 84, 22, 34, 20, 26, 57, 15, 55, 118, 28, 39, 79, 62, 47, 100, 101, 30, 50, 40, 88, 36, 37, 85, 103, 81, 112, 111, 59, 24, 17, 76, 115, 113, 14, 21, 60, 78, 117, 90, 114, 29, 82, 104, 12, 18, 9, 73, 16, 93, 27, 92, 94, 31, 80, 83, 44, 13, 77, 0, 110, 11, 95, 8, 19, 71, 46, 70, 102, 96, 108, 75, 91, 7, 72, 89, 68, 87, 4, 32, 64, 107, 25, 38, 67, 41, 66, 3, 43, 23, 97, 5, 69, 33, 2, 99, 6, 105, 10, 65, 35, 74, 1], [106, 124, 116, 125, 61, 52, 63, 109, 49, 42, 51, 120, 54, 48, 121, 45, 119, 53, 127, 56, 58, 123, 98, 126, 20, 26, 101, 39, 34, 22, 28, 118, 86, 37, 47, 57, 122, 88, 84, 114, 112, 79, 40, 55, 103, 62, 30, 59, 15, 36, 24, 100, 85, 50, 60, 113, 21, 117, 111, 115, 90, 81, 29, 93, 78, 104, 17, 82, 94, 92, 14, 18, 76, 95, 27, 44, 46, 12, 16, 9, 25, 108, 110, 31, 96, 19, 77, 13, 73, 83, 102, 70, 80, 43, 11, 107, 75, 91, 71, 87, 32, 89, 7, 41, 38, 72, 23, 97, 8, 99, 35, 0, 33, 67, 105, 5, 3, 68, 4, 64, 69, 66, 2, 74, 1, 10, 6, 65], [106, 124, 116, 125, 61, 52, 109, 63, 42, 49, 51, 54, 120, 48, 45, 121, 119, 53, 58, 123, 127, 126, 98, 56, 55, 20, 22, 28, 39, 57, 34, 101, 112, 118, 47, 122, 26, 79, 100, 37, 88, 59, 84, 86, 50, 36, 15, 62, 85, 30, 24, 115, 40, 114, 103, 81, 117, 104, 111, 60, 78, 113, 14, 12, 93, 76, 17, 82, 21, 29, 16, 73, 18, 90, 94, 9, 92, 11, 27, 70, 80, 44, 77, 64, 46, 83, 19, 72, 110, 71, 96, 31, 108, 75, 89, 0, 8, 13, 95, 25, 7, 107, 91, 97, 102, 2, 32, 43, 41, 67, 4, 87, 38, 68, 66, 23, 35, 3, 69, 65, 99, 5, 33, 10, 1, 105, 74, 6], [106, 124, 116, 125, 61, 52, 109, 63, 42, 51, 54, 49, 120, 48, 45, 121, 119, 53, 58, 127, 98, 55, 123, 126, 56, 20, 39, 22, 28, 34, 101, 15, 84, 118, 122, 47, 88, 86, 26, 79, 57, 37, 100, 30, 59, 36, 62, 24, 40, 112, 111, 115, 85, 81, 117, 50, 17, 103, 9, 78, 104, 29, 76, 93, 114, 14, 12, 113, 60, 90, 92, 94, 21, 82, 16, 73, 11, 18, 31, 80, 77, 27, 64, 46, 108, 75, 72, 19, 110, 83, 71, 44, 96, 25, 70, 102, 91, 8, 13, 41, 7, 107, 95, 4, 43, 87, 89, 67, 0, 68, 32, 3, 2, 97, 65, 38, 5, 33, 66, 69, 74, 6, 23, 35, 99, 10, 105, 1], [106, 125, 116, 124, 61, 52, 63, 109, 42, 49, 54, 51, 120, 48, 45, 121, 119, 53, 127, 56, 123, 55, 58, 98, 126, 39, 20, 118, 86, 122, 22, 28, 84, 101, 57, 26, 15, 37, 30, 79, 34, 112, 36, 47, 59, 62, 88, 50, 111, 100, 103, 81, 85, 60, 113, 24, 78, 117, 115, 40, 76, 114, 14, 12, 9, 29, 104, 93, 73, 90, 17, 18, 77, 82, 21, 94, 44, 108, 16, 31, 92, 96, 80, 110, 11, 83, 19, 46, 75, 13, 27, 72, 0, 107, 71, 7, 102, 8, 91, 70, 87, 43, 41, 64, 4, 25, 68, 23, 95, 3, 67, 97, 89, 32, 38, 6, 69, 2, 33, 5, 66, 74, 65, 105, 35, 1, 99, 10], [106, 124, 125, 116, 61, 52, 63, 109, 49, 42, 51, 54, 120, 48, 45, 121, 53, 119, 127, 98, 58, 126, 123, 56, 20, 26, 57, 122, 28, 101, 39, 22, 55, 36, 86, 84, 37, 34, 50, 112, 59, 15, 79, 100, 30, 114, 40, 88, 47, 115, 24, 62, 81, 118, 60, 21, 103, 85, 90, 14, 93, 104, 29, 111, 17, 117, 78, 94, 82, 16, 110, 76, 44, 113, 92, 108, 18, 9, 12, 27, 96, 77, 80, 19, 102, 25, 46, 83, 95, 73, 31, 75, 43, 13, 107, 91, 11, 72, 38, 41, 89, 32, 97, 6, 87, 64, 71, 7, 23, 35, 33, 0, 99, 66, 8, 5, 105, 67, 3, 68, 4, 69, 70, 74, 2, 65, 1, 10], [106, 124, 116, 125, 61, 52, 109, 63, 42, 51, 49, 54, 120, 48, 45, 121, 127, 119, 53, 98, 123, 58, 56, 126, 28, 39, 101, 55, 122, 26, 20, 22, 100, 86, 57, 36, 34, 59, 118, 84, 40, 15, 88, 37, 30, 117, 103, 79, 50, 24, 113, 111, 112, 62, 114, 14, 17, 60, 85, 29, 115, 81, 21, 47, 93, 44, 76, 104, 90, 95, 16, 82, 9, 27, 94, 78, 92, 73, 18, 12, 80, 108, 75, 77, 96, 31, 6, 110, 83, 72, 13, 0, 107, 46, 102, 11, 19, 91, 7, 43, 25, 87, 32, 64, 38, 97, 89, 41, 71, 23, 33, 8, 105, 2, 4, 67, 69, 3, 68, 35, 66, 74, 99, 65, 5, 1, 70, 10], [106, 124, 125, 116, 61, 52, 109, 63, 42, 49, 54, 51, 120, 48, 45, 53, 121, 119, 127, 56, 98, 58, 123, 101, 126, 34, 36, 55, 39, 118, 40, 59, 28, 20, 50, 103, 86, 115, 37, 111, 26, 100, 57, 22, 84, 24, 122, 15, 47, 88, 113, 30, 114, 62, 112, 79, 117, 104, 44, 60, 21, 81, 29, 14, 108, 78, 90, 82, 94, 76, 17, 85, 93, 92, 73, 16, 110, 18, 9, 12, 27, 102, 31, 46, 41, 77, 83, 80, 107, 75, 6, 19, 72, 95, 96, 89, 87, 8, 13, 43, 7, 11, 38, 97, 64, 0, 105, 25, 91, 23, 32, 71, 4, 33, 3, 35, 99, 69, 67, 68, 5, 66, 2, 74, 1, 65, 10, 70], [106, 124, 125, 116, 61, 52, 63, 109, 42, 49, 51, 54, 120, 48, 45, 53, 121, 119, 123, 127, 98, 58, 56, 55, 20, 39, 126, 22, 28, 57, 59, 37, 86, 34, 101, 84, 36, 118, 15, 122, 100, 30, 79, 26, 47, 103, 111, 40, 115, 24, 88, 117, 112, 113, 50, 90, 85, 81, 114, 29, 21, 14, 94, 92, 62, 93, 60, 12, 104, 78, 76, 82, 44, 27, 110, 17, 73, 80, 108, 9, 31, 77, 18, 75, 96, 83, 16, 13, 6, 95, 25, 72, 91, 11, 46, 19, 89, 102, 7, 107, 8, 0, 41, 23, 87, 32, 64, 71, 43, 105, 38, 35, 97, 33, 67, 69, 3, 68, 4, 5, 1, 99, 65, 74, 2, 66, 10, 70], [106, 124, 116, 125, 61, 52, 63, 109, 42, 49, 51, 54, 120, 48, 45, 121, 53, 119, 123, 127, 58, 56, 98, 57, 126, 22, 86, 59, 122, 34, 28, 20, 15, 100, 39, 84, 101, 26, 55, 103, 36, 40, 118, 88, 113, 37, 79, 47, 62, 114, 24, 78, 30, 117, 111, 115, 112, 50, 85, 81, 90, 76, 29, 93, 60, 17, 27, 14, 92, 9, 94, 104, 12, 44, 18, 73, 21, 82, 80, 108, 110, 83, 31, 16, 75, 13, 95, 46, 77, 96, 91, 19, 107, 89, 72, 11, 0, 25, 43, 102, 6, 7, 8, 41, 64, 87, 71, 32, 23, 97, 38, 3, 68, 33, 5, 67, 35, 4, 66, 105, 1, 74, 70, 69, 2, 65, 99, 10], [106, 124, 125, 116, 61, 52, 109, 63, 42, 49, 51, 54, 120, 48, 45, 121, 123, 53, 119, 127, 98, 58, 56, 126, 59, 28, 57, 39, 55, 34, 101, 26, 22, 20, 122, 84, 15, 36, 103, 86, 47, 88, 100, 115, 113, 118, 24, 79, 112, 62, 30, 37, 114, 111, 40, 117, 85, 21, 104, 90, 78, 76, 81, 29, 93, 14, 17, 110, 108, 50, 60, 18, 27, 94, 44, 92, 82, 9, 16, 73, 80, 83, 13, 12, 25, 0, 31, 77, 46, 102, 107, 64, 95, 19, 7, 8, 96, 75, 11, 91, 97, 38, 43, 72, 71, 87, 70, 4, 89, 67, 32, 3, 23, 6, 68, 2, 33, 66, 41, 5, 65, 1, 35, 69, 105, 99, 74, 10], [106, 124, 116, 125, 61, 52, 63, 49, 109, 42, 51, 54, 120, 48, 45, 53, 119, 121, 123, 58, 127, 98, 56, 126, 57, 122, 34, 86, 84, 101, 26, 22, 36, 28, 20, 39, 59, 55, 15, 100, 114, 103, 115, 88, 47, 37, 30, 62, 111, 113, 24, 85, 40, 79, 90, 112, 118, 29, 81, 50, 60, 117, 104, 76, 14, 78, 93, 17, 21, 94, 92, 82, 18, 44, 27, 80, 9, 110, 73, 77, 31, 16, 95, 108, 12, 11, 83, 13, 25, 46, 107, 19, 102, 64, 70, 96, 91, 8, 43, 87, 41, 7, 75, 4, 71, 32, 38, 97, 72, 68, 33, 89, 0, 35, 23, 3, 2, 1, 65, 105, 5, 69, 67, 66, 74, 99, 10, 6], [106, 124, 116, 125, 61, 52, 63, 109, 42, 49, 51, 54, 120, 48, 45, 53, 121, 119, 123, 58, 98, 127, 56, 126, 57, 59, 55, 28, 34, 26, 22, 103, 101, 86, 62, 84, 39, 118, 36, 122, 79, 88, 20, 30, 15, 117, 100, 37, 112, 40, 81, 111, 113, 114, 24, 47, 29, 115, 93, 50, 85, 90, 21, 76, 78, 14, 104, 94, 12, 92, 17, 27, 44, 82, 73, 60, 18, 9, 31, 80, 108, 70, 95, 46, 110, 16, 19, 75, 77, 8, 13, 11, 83, 96, 91, 25, 89, 102, 32, 72, 107, 7, 97, 87, 43, 38, 23, 35, 71, 41, 3, 0, 105, 33, 4, 68, 64, 74, 69, 67, 5, 66, 2, 10, 65, 1, 99, 6], [106, 125, 116, 124, 61, 52, 63, 109, 49, 42, 51, 54, 120, 48, 45, 121, 119, 53, 58, 98, 127, 56, 123, 126, 122, 26, 34, 59, 20, 55, 28, 84, 86, 22, 39, 100, 88, 15, 101, 62, 57, 118, 36, 103, 47, 79, 50, 37, 85, 40, 30, 112, 81, 117, 24, 114, 111, 14, 115, 113, 60, 76, 104, 110, 17, 93, 18, 90, 44, 78, 21, 80, 108, 82, 73, 94, 29, 12, 31, 9, 92, 27, 70, 46, 16, 77, 95, 83, 13, 19, 11, 107, 75, 8, 96, 25, 102, 71, 7, 89, 41, 23, 0, 91, 64, 87, 32, 38, 72, 43, 33, 67, 97, 68, 3, 66, 4, 5, 105, 69, 35, 74, 1, 2, 10, 65, 99, 6], [106, 116, 124, 125, 61, 52, 63, 109, 49, 42, 51, 54, 120, 48, 45, 121, 53, 98, 119, 56, 127, 123, 122, 58, 55, 59, 26, 20, 126, 28, 86, 22, 101, 100, 88, 84, 34, 39, 15, 115, 36, 57, 40, 79, 62, 37, 30, 85, 113, 118, 50, 44, 117, 24, 111, 112, 103, 60, 47, 81, 114, 93, 29, 108, 21, 94, 78, 104, 76, 17, 90, 14, 110, 12, 92, 9, 18, 46, 73, 82, 80, 77, 102, 31, 95, 27, 16, 83, 11, 25, 8, 96, 13, 19, 107, 75, 91, 70, 41, 105, 71, 89, 23, 0, 7, 43, 38, 72, 87, 67, 32, 97, 64, 35, 33, 68, 4, 99, 2, 74, 5, 66, 69, 3, 1, 6, 65, 10], [106, 124, 116, 125, 61, 52, 109, 63, 42, 49, 51, 54, 120, 48, 121, 45, 123, 127, 55, 53, 98, 56, 122, 119, 126, 40, 58, 26, 28, 34, 86, 60, 57, 39, 20, 88, 101, 115, 118, 36, 22, 37, 100, 59, 103, 84, 15, 47, 50, 79, 62, 117, 85, 30, 104, 111, 24, 90, 112, 114, 93, 81, 44, 78, 113, 29, 12, 76, 17, 9, 21, 18, 14, 82, 31, 102, 27, 95, 108, 16, 96, 110, 73, 92, 80, 46, 94, 77, 83, 8, 19, 75, 41, 11, 13, 25, 64, 89, 91, 23, 7, 71, 32, 38, 97, 0, 107, 43, 67, 68, 33, 72, 6, 3, 87, 105, 4, 70, 66, 2, 69, 35, 74, 65, 5, 99, 1, 10], [106, 124, 125, 116, 61, 52, 63, 109, 42, 49, 54, 51, 120, 48, 45, 121, 123, 55, 56, 53, 119, 58, 98, 127, 122, 28, 126, 34, 39, 20, 57, 101, 59, 84, 118, 79, 26, 22, 88, 100, 15, 103, 86, 36, 47, 117, 37, 111, 62, 115, 112, 40, 81, 85, 113, 30, 78, 24, 17, 50, 76, 60, 93, 104, 18, 29, 9, 12, 82, 21, 14, 73, 90, 114, 110, 27, 44, 92, 94, 16, 8, 77, 19, 75, 13, 80, 0, 108, 6, 83, 11, 46, 95, 31, 7, 64, 91, 102, 71, 96, 72, 3, 25, 23, 97, 41, 107, 89, 32, 5, 4, 87, 67, 66, 38, 35, 2, 105, 65, 69, 43, 68, 33, 74, 70, 99, 1, 10], [106, 116, 125, 124, 61, 52, 109, 63, 49, 42, 54, 51, 120, 48, 45, 121, 53, 119, 123, 55, 56, 58, 98, 127, 39, 34, 126, 57, 101, 28, 118, 59, 22, 26, 115, 84, 20, 100, 37, 86, 36, 122, 88, 15, 103, 111, 47, 85, 79, 40, 117, 113, 30, 50, 62, 60, 24, 78, 112, 104, 44, 93, 29, 90, 12, 81, 110, 92, 76, 9, 17, 14, 18, 73, 21, 27, 114, 82, 94, 8, 80, 31, 19, 46, 6, 13, 64, 77, 108, 95, 83, 11, 16, 7, 75, 96, 102, 25, 72, 0, 71, 32, 89, 4, 91, 38, 41, 23, 107, 87, 2, 105, 3, 69, 1, 5, 67, 66, 97, 33, 68, 43, 65, 74, 99, 35, 10, 70], [106, 116, 125, 124, 61, 52, 109, 63, 49, 42, 51, 54, 120, 48, 121, 45, 53, 119, 55, 58, 98, 56, 123, 126, 39, 59, 20, 26, 127, 22, 34, 84, 15, 86, 28, 57, 118, 88, 101, 36, 122, 37, 79, 100, 62, 115, 117, 111, 81, 103, 50, 78, 60, 30, 47, 85, 40, 113, 12, 24, 14, 17, 104, 76, 9, 93, 73, 29, 21, 112, 44, 18, 27, 90, 80, 82, 92, 16, 6, 94, 114, 13, 46, 83, 19, 75, 11, 77, 110, 108, 8, 96, 64, 31, 95, 7, 71, 3, 87, 25, 91, 32, 72, 107, 4, 68, 102, 67, 89, 41, 5, 66, 105, 97, 23, 43, 38, 69, 33, 10, 0, 2, 74, 35, 99, 1, 65, 70], [106, 125, 124, 116, 61, 52, 49, 109, 63, 42, 51, 54, 120, 48, 53, 121, 45, 123, 58, 127, 119, 56, 98, 126, 59, 122, 39, 55, 26, 118, 28, 34, 100, 62, 57, 15, 101, 22, 20, 37, 47, 36, 86, 84, 88, 40, 79, 60, 50, 112, 115, 117, 111, 103, 81, 30, 24, 85, 113, 104, 14, 73, 78, 29, 93, 21, 12, 76, 9, 80, 44, 90, 17, 18, 94, 82, 27, 92, 114, 11, 6, 96, 46, 13, 31, 16, 72, 110, 83, 0, 19, 77, 7, 75, 8, 64, 43, 71, 108, 95, 91, 68, 89, 25, 32, 67, 4, 38, 5, 102, 3, 97, 66, 87, 23, 33, 70, 107, 2, 69, 105, 99, 41, 1, 35, 65, 10, 74], [106, 124, 125, 116, 61, 52, 42, 109, 63, 49, 51, 120, 54, 48, 121, 45, 53, 123, 126, 98, 58, 119, 86, 59, 127, 56, 57, 26, 55, 22, 84, 79, 28, 39, 20, 15, 115, 34, 101, 40, 88, 36, 62, 37, 111, 122, 118, 117, 60, 112, 24, 30, 100, 103, 47, 81, 85, 50, 14, 12, 78, 113, 29, 104, 21, 76, 93, 90, 73, 9, 17, 18, 114, 82, 92, 11, 16, 13, 94, 80, 27, 95, 77, 46, 44, 31, 19, 72, 83, 75, 110, 108, 107, 96, 89, 7, 25, 6, 43, 71, 91, 102, 32, 8, 41, 97, 70, 23, 105, 35, 68, 87, 67, 38, 3, 5, 64, 4, 0, 69, 10, 74, 33, 99, 2, 66, 1, 65]], "model.layers.17.self_attn.q_proj": [[60, 106, 114, 100, 29, 89, 115, 123, 93, 86, 27, 83, 120, 47, 80, 42, 22, 112, 11, 57, 21, 113, 122, 53, 62, 127, 50, 16, 46, 61, 118, 59, 32, 111, 126, 45, 125, 56, 14, 99, 110, 63, 119, 91, 116, 81, 69, 55, 108, 96, 54, 26, 48, 117, 44, 51, 52, 121, 124, 58, 37, 97, 49, 109, 43, 105, 24, 76, 38, 41, 88, 71, 75, 95, 5, 107, 102, 103, 39, 104, 87, 31, 20, 40, 36, 15, 9, 101, 79, 35, 34, 98, 84, 17, 28, 10, 2, 94, 19, 90, 85, 8, 3, 23, 30, 33, 0, 7, 12, 1, 25, 4, 82, 6, 64, 18, 66, 77, 13, 78, 92, 74, 65, 67, 73, 72, 70, 68], [114, 106, 100, 60, 29, 89, 93, 50, 42, 86, 120, 59, 44, 61, 83, 22, 80, 115, 87, 99, 118, 27, 32, 105, 119, 81, 47, 31, 95, 46, 43, 21, 121, 97, 17, 25, 63, 91, 48, 54, 56, 116, 113, 51, 57, 14, 24, 11, 111, 103, 117, 96, 45, 53, 107, 125, 62, 110, 76, 58, 122, 41, 19, 38, 126, 112, 101, 37, 127, 40, 94, 55, 52, 16, 124, 88, 20, 102, 104, 98, 109, 33, 85, 108, 123, 71, 8, 23, 49, 35, 39, 90, 92, 30, 82, 79, 34, 9, 26, 15, 5, 10, 28, 1, 13, 36, 65, 84, 69, 77, 18, 12, 74, 75, 0, 3, 66, 6, 67, 72, 4, 70, 64, 7, 78, 73, 68, 2], [106, 60, 114, 100, 50, 29, 89, 86, 42, 57, 62, 107, 59, 93, 22, 80, 83, 56, 116, 37, 11, 14, 63, 47, 124, 99, 115, 91, 111, 46, 51, 71, 123, 52, 69, 48, 97, 110, 44, 125, 27, 55, 61, 49, 9, 16, 24, 113, 32, 76, 53, 58, 105, 81, 104, 120, 126, 119, 54, 31, 117, 127, 118, 122, 41, 38, 40, 45, 102, 121, 20, 108, 79, 103, 43, 109, 112, 19, 101, 94, 98, 39, 28, 35, 26, 33, 95, 96, 5, 88, 23, 25, 2, 30, 34, 12, 90, 82, 92, 21, 75, 85, 67, 1, 0, 4, 36, 17, 6, 18, 84, 74, 87, 13, 77, 8, 15, 10, 7, 72, 70, 3, 66, 78, 65, 64, 73, 68], [106, 60, 100, 115, 114, 89, 29, 83, 14, 86, 93, 42, 11, 71, 80, 27, 9, 50, 22, 25, 69, 110, 2, 67, 76, 1, 91, 4, 24, 62, 0, 16, 6, 53, 61, 116, 57, 3, 111, 37, 85, 21, 26, 97, 122, 113, 125, 7, 63, 118, 51, 127, 90, 77, 119, 19, 55, 35, 47, 17, 105, 123, 117, 75, 112, 52, 81, 95, 54, 64, 43, 108, 44, 120, 48, 78, 49, 18, 87, 46, 13, 88, 59, 98, 101, 96, 38, 92, 20, 45, 34, 40, 58, 121, 104, 68, 84, 124, 23, 103, 28, 94, 79, 102, 10, 107, 31, 73, 65, 70, 32, 41, 126, 56, 39, 33, 12, 99, 72, 109, 36, 74, 30, 15, 8, 82, 66, 5], [103, 126, 29, 82, 99, 85, 15, 76, 73, 71, 112, 90, 3, 5, 24, 105, 35, 93, 96, 97, 57, 23, 120, 87, 21, 61, 54, 18, 81, 119, 84, 17, 33, 26, 80, 49, 67, 115, 106, 64, 116, 83, 88, 50, 79, 16, 111, 78, 28, 101, 19, 12, 9, 121, 13, 127, 1, 69, 114, 25, 65, 34, 91, 7, 122, 89, 118, 74, 8, 32, 53, 110, 62, 20, 86, 14, 104, 11, 92, 4, 98, 117, 63, 40, 59, 51, 102, 10, 22, 27, 55, 31, 66, 41, 38, 77, 36, 43, 56, 123, 94, 58, 125, 100, 52, 70, 47, 107, 39, 42, 95, 46, 60, 6, 44, 75, 37, 30, 124, 109, 72, 68, 48, 113, 108, 45, 2, 0], [103, 126, 29, 99, 15, 85, 82, 76, 73, 71, 112, 3, 5, 90, 96, 24, 26, 1, 16, 27, 21, 9, 114, 119, 67, 57, 64, 84, 74, 11, 19, 93, 78, 79, 81, 23, 120, 88, 65, 14, 97, 101, 61, 91, 49, 33, 18, 98, 104, 105, 116, 6, 50, 43, 87, 8, 69, 92, 55, 32, 106, 7, 2, 30, 121, 94, 117, 13, 34, 125, 12, 54, 25, 56, 28, 80, 70, 75, 41, 68, 66, 127, 83, 36, 20, 115, 110, 10, 22, 17, 102, 100, 63, 122, 4, 51, 118, 39, 86, 58, 59, 113, 0, 108, 111, 35, 62, 77, 124, 60, 37, 72, 46, 44, 45, 89, 107, 31, 95, 53, 109, 48, 42, 123, 40, 38, 47, 52], [103, 126, 35, 49, 112, 29, 90, 114, 99, 56, 26, 85, 88, 24, 120, 93, 82, 116, 84, 119, 121, 96, 115, 46, 86, 63, 55, 117, 41, 57, 15, 60, 54, 110, 43, 32, 111, 122, 106, 62, 74, 59, 127, 101, 76, 16, 125, 52, 61, 104, 50, 118, 22, 124, 102, 123, 58, 107, 105, 108, 47, 51, 53, 97, 23, 73, 71, 40, 34, 113, 109, 77, 2, 48, 27, 42, 36, 11, 83, 0, 21, 37, 19, 44, 33, 18, 45, 98, 20, 38, 14, 100, 5, 8, 3, 95, 4, 78, 25, 66, 79, 91, 75, 39, 94, 68, 92, 80, 31, 28, 89, 72, 67, 87, 30, 1, 13, 81, 10, 64, 17, 69, 7, 65, 70, 12, 9, 6], [103, 126, 29, 85, 24, 82, 76, 90, 15, 112, 93, 99, 120, 26, 73, 71, 74, 35, 88, 127, 119, 41, 98, 21, 23, 3, 115, 5, 13, 111, 116, 121, 78, 57, 54, 96, 92, 16, 62, 49, 34, 63, 61, 91, 36, 14, 114, 55, 60, 104, 84, 117, 25, 30, 56, 32, 18, 33, 118, 106, 27, 50, 83, 28, 53, 51, 31, 79, 110, 86, 67, 52, 43, 87, 97, 17, 122, 19, 105, 101, 8, 47, 125, 12, 42, 102, 81, 46, 58, 20, 22, 77, 107, 108, 7, 123, 89, 48, 1, 95, 40, 94, 59, 38, 113, 124, 11, 100, 80, 44, 37, 45, 68, 72, 9, 109, 64, 10, 65, 66, 75, 39, 4, 6, 69, 70, 0, 2], [45, 109, 55, 58, 111, 59, 57, 63, 124, 52, 47, 116, 46, 125, 115, 119, 61, 53, 114, 48, 62, 51, 113, 38, 123, 60, 126, 50, 54, 117, 121, 112, 102, 122, 106, 49, 118, 120, 101, 127, 107, 36, 110, 42, 56, 43, 108, 33, 44, 28, 105, 29, 104, 41, 40, 103, 26, 39, 92, 99, 97, 37, 96, 98, 87, 34, 31, 100, 21, 35, 86, 32, 91, 30, 23, 95, 24, 18, 94, 89, 90, 27, 12, 84, 93, 25, 85, 78, 88, 22, 20, 82, 81, 17, 80, 7, 15, 11, 76, 9, 14, 79, 73, 16, 19, 83, 71, 75, 13, 69, 4, 68, 70, 74, 5, 2, 77, 65, 66, 8, 3, 10, 6, 1, 0, 64, 67, 72], [58, 45, 38, 47, 94, 55, 59, 89, 95, 60, 124, 112, 109, 111, 33, 57, 52, 115, 28, 35, 127, 87, 46, 113, 61, 92, 123, 51, 117, 122, 49, 118, 116, 63, 53, 54, 125, 101, 96, 50, 119, 121, 62, 105, 126, 110, 48, 41, 114, 120, 107, 86, 106, 43, 42, 44, 40, 108, 104, 56, 85, 36, 16, 37, 19, 98, 82, 103, 23, 20, 39, 12, 34, 84, 18, 91, 100, 15, 93, 81, 32, 22, 27, 99, 97, 102, 9, 11, 31, 29, 24, 78, 17, 25, 90, 7, 26, 88, 21, 71, 30, 79, 75, 14, 73, 76, 4, 77, 69, 5, 2, 13, 83, 68, 74, 1, 80, 66, 65, 10, 64, 0, 6, 8, 70, 67, 3, 72], [45, 109, 47, 52, 38, 101, 53, 60, 58, 94, 61, 115, 92, 59, 62, 46, 126, 23, 34, 116, 28, 111, 51, 119, 118, 108, 87, 33, 54, 57, 50, 112, 110, 63, 49, 85, 127, 123, 113, 117, 48, 30, 42, 44, 114, 89, 21, 41, 122, 103, 36, 40, 102, 55, 121, 120, 43, 35, 39, 107, 125, 37, 106, 104, 29, 105, 124, 99, 95, 100, 56, 93, 27, 96, 26, 97, 31, 78, 12, 32, 25, 24, 98, 16, 80, 18, 9, 88, 91, 86, 73, 82, 90, 81, 7, 20, 22, 84, 17, 11, 76, 83, 79, 19, 14, 75, 15, 69, 77, 71, 6, 74, 66, 10, 68, 4, 5, 2, 70, 67, 13, 0, 3, 72, 65, 64, 1, 8], [45, 38, 58, 19, 89, 94, 13, 59, 16, 74, 109, 35, 70, 87, 8, 3, 47, 6, 33, 67, 95, 30, 52, 111, 0, 55, 64, 72, 124, 1, 85, 77, 92, 84, 17, 65, 88, 69, 15, 10, 5, 81, 23, 21, 83, 82, 80, 28, 22, 20, 27, 78, 115, 25, 4, 31, 66, 76, 9, 11, 90, 32, 101, 2, 18, 75, 79, 37, 14, 96, 57, 126, 71, 63, 73, 12, 26, 68, 7, 29, 86, 98, 34, 24, 39, 91, 93, 108, 50, 125, 122, 61, 62, 105, 44, 104, 103, 116, 60, 100, 118, 121, 36, 97, 43, 99, 112, 120, 117, 51, 54, 40, 49, 56, 107, 123, 42, 106, 41, 113, 48, 53, 110, 119, 114, 127, 46, 102], [40, 50, 109, 98, 126, 29, 51, 84, 23, 80, 13, 18, 71, 73, 53, 75, 26, 56, 47, 90, 104, 2, 87, 66, 4, 111, 72, 93, 69, 68, 112, 39, 86, 94, 70, 1, 20, 0, 120, 11, 22, 19, 65, 74, 30, 103, 101, 64, 5, 25, 96, 59, 43, 119, 60, 16, 67, 89, 34, 76, 31, 77, 8, 36, 95, 38, 78, 117, 57, 113, 108, 100, 32, 6, 62, 121, 27, 45, 105, 124, 127, 7, 102, 21, 88, 33, 55, 97, 48, 15, 85, 99, 41, 118, 44, 28, 110, 83, 91, 35, 92, 58, 63, 10, 52, 12, 61, 46, 42, 17, 24, 115, 116, 82, 3, 123, 114, 81, 37, 107, 14, 9, 54, 122, 125, 49, 106, 79], [40, 126, 29, 98, 109, 87, 47, 39, 50, 23, 93, 84, 20, 75, 36, 90, 125, 58, 52, 124, 26, 59, 13, 121, 119, 56, 54, 115, 117, 53, 81, 18, 57, 108, 118, 34, 17, 103, 31, 122, 88, 120, 104, 63, 60, 127, 61, 107, 111, 105, 97, 110, 72, 44, 21, 51, 42, 46, 48, 24, 15, 32, 116, 123, 41, 45, 94, 70, 30, 43, 114, 49, 112, 99, 55, 106, 113, 80, 37, 38, 62, 85, 35, 100, 25, 92, 22, 101, 95, 79, 96, 102, 33, 14, 77, 27, 19, 83, 11, 4, 89, 28, 12, 91, 16, 86, 6, 82, 8, 76, 78, 10, 74, 73, 66, 9, 1, 68, 2, 65, 7, 3, 71, 5, 64, 69, 0, 67], [40, 126, 98, 109, 29, 60, 84, 90, 51, 87, 93, 124, 47, 23, 80, 50, 26, 18, 75, 59, 54, 34, 117, 36, 39, 88, 20, 119, 118, 53, 121, 13, 63, 114, 58, 103, 22, 123, 125, 127, 37, 55, 44, 122, 52, 45, 46, 111, 56, 115, 105, 21, 48, 104, 110, 107, 62, 120, 94, 92, 102, 61, 96, 108, 57, 112, 30, 38, 91, 49, 113, 43, 72, 70, 24, 12, 116, 35, 41, 42, 106, 97, 89, 86, 32, 31, 99, 100, 101, 16, 17, 85, 28, 79, 19, 33, 25, 73, 81, 82, 76, 15, 27, 95, 83, 14, 78, 4, 11, 10, 9, 6, 74, 8, 3, 69, 68, 67, 7, 77, 5, 71, 65, 1, 66, 2, 0, 64], [40, 51, 98, 29, 126, 13, 80, 18, 87, 84, 72, 23, 90, 4, 75, 93, 96, 26, 70, 60, 10, 73, 88, 124, 52, 19, 94, 121, 56, 125, 71, 54, 86, 30, 119, 81, 65, 117, 20, 109, 47, 59, 122, 77, 28, 50, 53, 111, 46, 16, 3, 61, 104, 105, 127, 112, 63, 74, 108, 78, 64, 15, 101, 68, 32, 39, 7, 91, 5, 89, 95, 97, 79, 67, 82, 11, 83, 58, 8, 118, 41, 22, 123, 106, 14, 2, 115, 120, 33, 76, 6, 62, 31, 34, 48, 69, 49, 113, 55, 116, 27, 12, 38, 9, 21, 100, 36, 44, 37, 110, 25, 57, 43, 92, 45, 17, 1, 103, 99, 24, 35, 66, 85, 102, 114, 42, 0, 107], [113, 115, 105, 57, 49, 63, 124, 53, 119, 34, 30, 39, 112, 48, 56, 127, 123, 61, 58, 45, 59, 117, 60, 116, 52, 120, 122, 62, 47, 27, 38, 118, 110, 107, 37, 108, 54, 121, 55, 94, 51, 125, 46, 111, 106, 85, 126, 44, 88, 114, 99, 104, 109, 87, 101, 90, 42, 43, 89, 36, 25, 24, 19, 103, 78, 18, 28, 40, 50, 92, 23, 41, 100, 35, 80, 83, 91, 22, 98, 33, 102, 31, 93, 21, 32, 14, 97, 26, 74, 95, 96, 6, 17, 16, 82, 65, 29, 12, 76, 20, 70, 81, 66, 64, 67, 72, 8, 10, 3, 0, 69, 86, 15, 11, 1, 2, 5, 13, 7, 68, 9, 75, 77, 4, 79, 73, 71, 84], [115, 61, 105, 113, 34, 30, 50, 124, 121, 120, 37, 38, 48, 63, 57, 126, 122, 53, 55, 112, 119, 27, 123, 49, 56, 59, 92, 60, 101, 62, 58, 127, 52, 46, 47, 45, 39, 118, 90, 110, 107, 44, 85, 111, 54, 125, 94, 116, 117, 108, 109, 103, 99, 87, 25, 114, 51, 40, 106, 41, 42, 33, 43, 89, 88, 104, 93, 96, 80, 32, 23, 91, 35, 36, 19, 31, 95, 102, 100, 98, 82, 78, 29, 28, 83, 21, 26, 18, 16, 97, 24, 22, 70, 17, 74, 6, 14, 64, 76, 0, 65, 20, 66, 1, 3, 69, 12, 86, 8, 81, 67, 2, 5, 10, 13, 72, 11, 15, 7, 9, 71, 4, 79, 68, 84, 73, 77, 75], [105, 113, 61, 57, 34, 115, 22, 20, 27, 30, 55, 41, 24, 17, 63, 53, 80, 13, 37, 59, 74, 38, 82, 18, 15, 11, 124, 123, 21, 12, 19, 7, 56, 51, 97, 121, 122, 70, 88, 26, 62, 47, 49, 103, 75, 99, 117, 9, 90, 116, 106, 31, 95, 36, 52, 127, 119, 60, 23, 58, 71, 10, 118, 39, 120, 89, 100, 46, 72, 112, 33, 45, 83, 111, 2, 6, 91, 50, 44, 3, 28, 48, 29, 14, 114, 126, 109, 101, 77, 93, 40, 85, 107, 54, 104, 84, 64, 110, 102, 16, 73, 81, 96, 76, 66, 42, 32, 25, 108, 87, 86, 92, 79, 68, 78, 94, 5, 69, 43, 1, 98, 4, 125, 35, 65, 8, 0, 67], [105, 57, 115, 113, 53, 34, 27, 22, 20, 15, 30, 41, 63, 9, 17, 24, 13, 49, 127, 74, 120, 4, 94, 55, 39, 116, 61, 37, 85, 1, 68, 3, 47, 36, 11, 87, 76, 73, 29, 72, 78, 77, 28, 31, 6, 46, 118, 123, 48, 79, 88, 95, 58, 119, 93, 103, 60, 26, 8, 18, 107, 84, 54, 38, 92, 104, 124, 35, 90, 112, 52, 43, 83, 110, 56, 40, 100, 121, 45, 109, 126, 23, 42, 59, 106, 122, 10, 96, 21, 50, 44, 62, 117, 125, 108, 81, 64, 111, 32, 114, 33, 86, 89, 12, 70, 25, 80, 51, 97, 99, 5, 16, 19, 14, 75, 66, 91, 98, 69, 65, 101, 67, 82, 102, 0, 7, 71, 2], [108, 37, 126, 88, 19, 90, 78, 32, 75, 81, 44, 8, 5, 36, 93, 86, 85, 67, 21, 1, 24, 34, 61, 22, 0, 16, 15, 83, 18, 12, 11, 72, 20, 69, 119, 58, 80, 65, 17, 25, 28, 14, 89, 13, 84, 87, 103, 23, 73, 10, 2, 98, 77, 74, 79, 35, 91, 27, 29, 30, 3, 76, 9, 94, 71, 82, 46, 26, 100, 118, 127, 68, 70, 95, 97, 125, 31, 53, 6, 7, 51, 92, 106, 64, 45, 52, 40, 66, 54, 4, 120, 122, 33, 113, 107, 117, 60, 115, 63, 99, 57, 123, 59, 105, 39, 42, 96, 111, 62, 104, 110, 43, 50, 114, 116, 102, 124, 109, 55, 47, 121, 101, 38, 48, 41, 49, 56, 112], [126, 108, 37, 44, 23, 58, 120, 57, 113, 51, 48, 61, 122, 63, 114, 118, 56, 62, 115, 116, 102, 104, 59, 32, 52, 119, 93, 124, 55, 53, 46, 54, 60, 125, 121, 101, 49, 100, 117, 110, 50, 103, 47, 45, 111, 123, 41, 109, 112, 40, 127, 106, 16, 107, 27, 42, 43, 90, 105, 39, 38, 91, 36, 35, 92, 84, 98, 99, 97, 95, 87, 33, 94, 34, 30, 25, 86, 88, 96, 29, 31, 85, 81, 28, 17, 20, 80, 89, 22, 74, 10, 2, 19, 24, 12, 26, 83, 82, 15, 0, 6, 65, 4, 21, 76, 66, 18, 70, 71, 13, 79, 78, 11, 67, 64, 7, 5, 9, 77, 69, 68, 3, 1, 73, 14, 75, 8, 72], [108, 61, 126, 44, 36, 119, 90, 85, 32, 122, 82, 93, 35, 24, 88, 37, 104, 58, 7, 19, 12, 15, 81, 13, 62, 91, 29, 101, 59, 95, 28, 63, 86, 46, 51, 20, 23, 79, 27, 120, 48, 78, 53, 57, 60, 96, 109, 87, 98, 116, 97, 115, 74, 89, 45, 71, 4, 18, 118, 114, 70, 66, 99, 52, 49, 33, 124, 64, 54, 125, 25, 127, 113, 56, 107, 117, 100, 34, 55, 9, 75, 41, 121, 26, 42, 92, 47, 31, 110, 30, 94, 123, 38, 16, 39, 112, 50, 76, 40, 102, 84, 6, 111, 106, 105, 65, 43, 103, 80, 21, 77, 2, 68, 22, 8, 83, 3, 11, 17, 0, 10, 5, 1, 73, 67, 69, 14, 72], [108, 44, 37, 126, 93, 23, 90, 58, 81, 24, 32, 61, 36, 85, 120, 19, 123, 41, 56, 86, 31, 29, 96, 118, 114, 110, 16, 78, 125, 112, 54, 105, 113, 88, 33, 63, 38, 103, 122, 48, 106, 87, 51, 111, 42, 109, 62, 115, 52, 57, 30, 47, 124, 119, 50, 102, 53, 127, 97, 43, 116, 27, 60, 34, 55, 121, 35, 45, 46, 99, 107, 98, 94, 74, 117, 49, 40, 84, 59, 104, 91, 7, 28, 25, 80, 39, 95, 100, 92, 26, 75, 82, 76, 89, 101, 13, 17, 9, 22, 10, 77, 15, 18, 0, 79, 71, 2, 83, 8, 67, 21, 64, 66, 5, 20, 12, 11, 70, 4, 68, 6, 73, 65, 14, 1, 69, 3, 72], [56, 126, 101, 62, 60, 122, 37, 125, 116, 123, 88, 53, 119, 58, 55, 51, 118, 120, 115, 117, 63, 48, 50, 89, 46, 127, 114, 84, 54, 124, 121, 33, 61, 59, 57, 103, 49, 14, 52, 11, 34, 113, 47, 92, 110, 39, 95, 107, 104, 44, 112, 98, 109, 108, 45, 111, 100, 106, 41, 40, 43, 93, 42, 35, 26, 16, 20, 38, 91, 105, 36, 15, 94, 99, 102, 18, 12, 22, 31, 82, 96, 90, 28, 25, 72, 86, 80, 32, 30, 68, 24, 87, 5, 23, 29, 17, 74, 97, 2, 7, 21, 70, 27, 79, 77, 19, 83, 85, 76, 81, 6, 73, 78, 64, 65, 9, 13, 75, 3, 0, 8, 66, 10, 67, 71, 69, 4, 1], [56, 126, 101, 116, 39, 62, 37, 20, 91, 53, 84, 26, 123, 127, 14, 125, 60, 119, 114, 33, 46, 100, 103, 120, 48, 54, 122, 118, 59, 51, 124, 55, 88, 58, 121, 45, 11, 104, 115, 40, 110, 50, 41, 36, 25, 106, 107, 63, 89, 61, 47, 117, 35, 34, 108, 44, 57, 99, 111, 52, 105, 42, 43, 112, 109, 95, 113, 29, 38, 16, 98, 49, 82, 31, 102, 94, 32, 12, 87, 93, 96, 83, 90, 92, 72, 15, 27, 30, 24, 28, 80, 97, 18, 77, 74, 86, 73, 22, 5, 6, 7, 68, 81, 70, 19, 21, 23, 65, 78, 79, 66, 75, 85, 9, 76, 17, 64, 2, 13, 8, 71, 67, 0, 10, 4, 3, 69, 1], [56, 126, 101, 60, 84, 116, 26, 40, 11, 21, 93, 62, 89, 28, 2, 125, 109, 68, 122, 127, 92, 33, 53, 55, 31, 118, 37, 79, 51, 29, 88, 24, 90, 114, 119, 124, 63, 58, 47, 123, 57, 61, 80, 48, 59, 110, 117, 50, 0, 103, 75, 120, 23, 121, 91, 52, 15, 16, 41, 54, 49, 108, 18, 106, 32, 34, 8, 115, 46, 100, 113, 112, 96, 67, 111, 35, 44, 17, 20, 30, 85, 70, 95, 105, 81, 19, 94, 77, 7, 45, 43, 22, 42, 82, 38, 9, 39, 107, 36, 99, 13, 104, 74, 76, 98, 3, 69, 102, 83, 5, 6, 73, 14, 87, 1, 65, 25, 27, 12, 72, 86, 10, 78, 4, 71, 97, 66, 64], [126, 56, 101, 62, 37, 88, 122, 53, 123, 15, 103, 125, 116, 24, 60, 91, 119, 120, 118, 23, 124, 92, 114, 48, 58, 127, 51, 117, 55, 63, 89, 54, 12, 33, 46, 121, 94, 50, 25, 61, 84, 59, 14, 6, 39, 49, 115, 57, 113, 52, 44, 47, 100, 11, 90, 45, 110, 18, 34, 112, 111, 20, 9, 99, 107, 17, 36, 105, 87, 7, 109, 40, 108, 95, 81, 98, 72, 22, 96, 104, 42, 102, 26, 35, 3, 28, 38, 106, 31, 32, 5, 86, 93, 68, 41, 79, 30, 82, 43, 80, 27, 16, 70, 97, 29, 74, 19, 64, 65, 76, 21, 66, 85, 1, 67, 8, 78, 2, 77, 73, 10, 4, 83, 75, 0, 71, 13, 69], [120, 109, 104, 124, 34, 123, 127, 59, 27, 89, 58, 60, 30, 52, 56, 125, 114, 86, 91, 54, 126, 118, 78, 121, 61, 113, 53, 62, 29, 45, 51, 50, 88, 119, 63, 40, 35, 110, 122, 84, 116, 94, 117, 19, 25, 49, 42, 55, 57, 112, 48, 43, 46, 107, 111, 47, 39, 115, 37, 106, 100, 108, 105, 41, 36, 10, 101, 103, 96, 44, 102, 99, 33, 21, 38, 18, 97, 26, 32, 31, 92, 28, 98, 87, 90, 95, 15, 22, 93, 17, 20, 16, 82, 14, 23, 85, 83, 9, 69, 80, 79, 24, 12, 76, 74, 13, 81, 73, 64, 5, 65, 77, 1, 6, 11, 7, 75, 4, 71, 68, 70, 72, 2, 67, 0, 8, 66, 3], [120, 109, 104, 123, 52, 58, 30, 127, 121, 59, 54, 124, 60, 122, 126, 61, 114, 112, 53, 56, 125, 117, 118, 45, 62, 51, 116, 113, 63, 110, 89, 107, 50, 49, 119, 57, 48, 78, 55, 115, 111, 46, 106, 47, 43, 42, 105, 34, 38, 108, 102, 25, 44, 18, 87, 41, 40, 101, 39, 100, 99, 103, 37, 94, 36, 98, 91, 35, 97, 27, 96, 88, 10, 33, 86, 32, 93, 28, 31, 95, 19, 85, 29, 82, 92, 90, 16, 84, 22, 26, 23, 74, 14, 21, 64, 20, 65, 83, 69, 1, 79, 15, 80, 76, 6, 17, 0, 5, 9, 24, 12, 81, 4, 3, 70, 66, 2, 67, 13, 11, 7, 71, 68, 73, 8, 75, 72, 77], [104, 120, 34, 88, 109, 17, 13, 75, 15, 7, 9, 86, 84, 92, 66, 71, 91, 2, 124, 11, 85, 19, 30, 69, 5, 4, 27, 40, 45, 22, 74, 0, 98, 68, 20, 64, 81, 82, 77, 24, 83, 79, 18, 80, 87, 29, 73, 26, 21, 76, 70, 107, 1, 14, 23, 94, 3, 72, 93, 90, 35, 8, 25, 123, 16, 47, 12, 6, 89, 56, 65, 116, 67, 78, 10, 59, 127, 46, 63, 31, 96, 54, 28, 95, 32, 97, 113, 106, 58, 52, 42, 99, 100, 101, 126, 55, 118, 43, 33, 37, 36, 112, 60, 105, 122, 125, 121, 114, 62, 117, 115, 49, 38, 110, 108, 61, 50, 39, 57, 53, 48, 102, 51, 103, 44, 119, 41, 111], [109, 104, 120, 34, 123, 78, 27, 59, 88, 116, 30, 112, 54, 86, 84, 52, 127, 125, 10, 121, 89, 40, 29, 18, 58, 126, 124, 60, 91, 114, 55, 56, 46, 113, 50, 61, 87, 122, 53, 62, 25, 110, 115, 26, 98, 45, 118, 63, 117, 119, 49, 19, 43, 48, 15, 51, 39, 42, 57, 100, 47, 36, 97, 90, 35, 106, 108, 38, 32, 28, 96, 107, 99, 101, 14, 31, 44, 102, 33, 111, 94, 17, 16, 9, 37, 105, 41, 92, 95, 23, 93, 103, 83, 82, 13, 22, 21, 76, 85, 20, 24, 80, 4, 6, 74, 79, 69, 12, 0, 2, 75, 73, 72, 11, 7, 81, 5, 71, 64, 1, 77, 70, 68, 65, 67, 8, 66, 3]], "model.layers.17.self_attn.k_proj": [[42, 36, 114, 60, 86, 93, 91, 80, 83, 89, 11, 63, 14, 19, 124, 119, 0, 53, 125, 46, 9, 69, 113, 111, 59, 25, 56, 54, 120, 126, 76, 50, 47, 44, 57, 115, 123, 118, 127, 116, 117, 32, 45, 43, 15, 2, 33, 71, 62, 51, 48, 122, 101, 121, 110, 49, 4, 55, 108, 41, 52, 24, 103, 104, 66, 106, 40, 105, 112, 58, 102, 90, 74, 98, 107, 1, 109, 39, 97, 18, 30, 77, 61, 5, 35, 88, 12, 34, 31, 100, 87, 95, 38, 81, 37, 20, 23, 17, 99, 28, 21, 6, 96, 72, 82, 70, 8, 92, 27, 26, 84, 94, 10, 79, 85, 65, 78, 7, 67, 13, 68, 29, 3, 64, 22, 16, 73, 75], [126, 39, 93, 85, 15, 82, 24, 1, 76, 5, 73, 71, 64, 48, 112, 74, 26, 50, 3, 120, 116, 96, 35, 121, 119, 117, 77, 118, 57, 115, 110, 105, 41, 16, 113, 114, 63, 60, 97, 54, 55, 42, 125, 46, 56, 52, 43, 49, 6, 29, 84, 34, 4, 40, 59, 122, 66, 61, 32, 27, 123, 62, 53, 90, 47, 75, 124, 99, 19, 51, 107, 127, 72, 87, 78, 38, 86, 109, 44, 111, 58, 37, 70, 100, 108, 23, 45, 104, 101, 25, 14, 83, 22, 11, 36, 106, 89, 13, 68, 95, 28, 69, 33, 81, 94, 2, 17, 102, 98, 31, 91, 92, 0, 30, 80, 20, 10, 8, 88, 9, 65, 67, 18, 21, 7, 103, 12, 79], [109, 58, 102, 30, 59, 8, 45, 13, 74, 19, 25, 86, 16, 70, 3, 97, 89, 65, 99, 28, 23, 0, 46, 32, 49, 47, 69, 36, 127, 121, 12, 126, 94, 115, 18, 15, 62, 91, 51, 84, 118, 53, 61, 120, 111, 107, 63, 116, 35, 77, 117, 48, 125, 11, 114, 83, 43, 101, 20, 87, 9, 110, 24, 27, 57, 106, 50, 52, 54, 68, 78, 4, 98, 123, 112, 113, 124, 122, 56, 29, 105, 17, 44, 37, 55, 119, 85, 31, 42, 60, 34, 7, 108, 104, 71, 64, 41, 39, 103, 40, 2, 100, 80, 90, 81, 6, 66, 26, 93, 82, 21, 5, 10, 96, 14, 88, 95, 75, 33, 79, 72, 92, 67, 73, 22, 76, 1, 38], [104, 126, 34, 50, 51, 115, 93, 23, 84, 26, 114, 13, 80, 111, 18, 45, 75, 56, 124, 59, 53, 70, 105, 72, 48, 127, 65, 119, 4, 58, 81, 73, 122, 64, 55, 57, 125, 103, 118, 94, 123, 3, 49, 60, 121, 99, 63, 2, 30, 117, 61, 10, 109, 110, 106, 32, 120, 22, 46, 14, 29, 24, 108, 113, 7, 38, 62, 36, 12, 107, 87, 116, 92, 112, 95, 42, 52, 102, 41, 28, 44, 54, 90, 25, 39, 101, 83, 19, 31, 71, 100, 47, 86, 43, 91, 37, 21, 96, 35, 15, 27, 89, 76, 33, 98, 5, 97, 85, 78, 88, 17, 79, 9, 67, 69, 82, 1, 6, 0, 74, 77, 16, 8, 20, 68, 11, 66, 40], [41, 98, 57, 115, 113, 94, 24, 51, 11, 20, 15, 91, 13, 22, 17, 61, 27, 55, 49, 9, 63, 68, 62, 59, 85, 26, 25, 72, 127, 56, 53, 123, 19, 116, 6, 114, 101, 35, 99, 107, 7, 18, 102, 48, 80, 108, 28, 122, 109, 65, 104, 54, 87, 125, 16, 60, 118, 8, 84, 117, 66, 74, 47, 103, 82, 97, 43, 121, 111, 93, 120, 78, 45, 33, 119, 42, 52, 58, 44, 105, 110, 124, 112, 96, 36, 50, 40, 46, 100, 76, 126, 3, 32, 73, 95, 5, 106, 29, 77, 31, 64, 39, 92, 90, 0, 67, 37, 79, 75, 89, 21, 23, 69, 70, 12, 83, 88, 71, 38, 81, 34, 2, 86, 10, 30, 14, 4, 1], [44, 126, 86, 101, 108, 100, 96, 61, 29, 58, 78, 90, 57, 19, 8, 51, 63, 119, 75, 120, 62, 81, 55, 125, 5, 124, 114, 47, 117, 116, 59, 122, 118, 123, 53, 16, 67, 60, 46, 74, 111, 56, 52, 112, 115, 110, 54, 0, 50, 121, 49, 88, 106, 127, 48, 109, 113, 23, 34, 65, 104, 42, 85, 99, 105, 39, 45, 98, 9, 107, 43, 41, 27, 66, 83, 40, 103, 82, 25, 38, 4, 70, 35, 13, 36, 102, 1, 94, 92, 28, 30, 73, 31, 26, 20, 93, 33, 95, 77, 97, 2, 68, 89, 15, 24, 7, 3, 17, 91, 12, 21, 79, 76, 84, 64, 18, 6, 72, 71, 32, 22, 80, 37, 11, 10, 87, 14, 69], [126, 37, 56, 22, 97, 62, 51, 125, 122, 123, 59, 119, 61, 60, 53, 116, 127, 58, 118, 50, 63, 48, 121, 120, 117, 124, 52, 49, 57, 112, 114, 55, 113, 54, 92, 47, 104, 115, 46, 94, 111, 27, 109, 110, 44, 42, 108, 45, 107, 103, 38, 93, 33, 43, 106, 39, 105, 41, 89, 82, 40, 35, 102, 101, 80, 84, 36, 98, 96, 99, 88, 86, 28, 34, 19, 77, 79, 100, 91, 32, 95, 30, 74, 90, 25, 3, 29, 9, 31, 7, 5, 85, 87, 26, 17, 23, 15, 81, 64, 16, 83, 21, 76, 24, 14, 65, 75, 72, 12, 6, 18, 78, 13, 2, 68, 4, 73, 71, 20, 11, 8, 70, 10, 66, 67, 1, 69, 0], [120, 40, 45, 86, 98, 123, 54, 27, 109, 60, 17, 59, 127, 113, 94, 58, 7, 88, 61, 13, 126, 52, 56, 117, 114, 62, 63, 9, 15, 53, 121, 118, 119, 122, 48, 75, 34, 116, 110, 66, 57, 55, 49, 124, 50, 51, 115, 112, 111, 46, 43, 47, 125, 84, 19, 103, 42, 78, 44, 26, 10, 107, 18, 5, 39, 106, 89, 102, 65, 108, 64, 36, 31, 105, 41, 93, 16, 101, 38, 87, 21, 95, 69, 8, 6, 99, 67, 4, 35, 30, 33, 37, 72, 90, 28, 97, 96, 24, 100, 76, 20, 32, 11, 12, 29, 104, 85, 92, 23, 68, 80, 83, 3, 71, 70, 81, 91, 0, 25, 82, 77, 79, 14, 1, 74, 73, 22, 2]], "model.layers.17.self_attn.qk_proj": [[126, 120, 109, 58, 44, 114, 56, 60, 115, 45, 113, 108, 104, 29, 57, 42, 61, 51, 50, 59, 41, 40, 106, 55, 119, 22, 118, 123, 49, 34, 105, 124, 125, 116, 93, 87, 112, 86, 127, 62, 117, 90, 111, 24, 94, 98, 53, 63, 27, 80, 37, 16, 52, 47, 30, 89, 23, 100, 25, 26, 54, 121, 39, 88, 18, 91, 83, 13, 85, 82, 75, 101, 122, 19, 48, 21, 103, 77, 110, 35, 84, 11, 20, 15, 36, 73, 79, 17, 99, 102, 32, 96, 43, 107, 9, 46, 78, 81, 14, 92, 76, 72, 38, 12, 97, 69, 7, 8, 5, 74, 71, 3, 28, 6, 64, 67, 33, 10, 65, 1, 70, 0, 4, 95, 68, 66, 31, 2], [126, 120, 109, 56, 44, 58, 114, 60, 115, 45, 113, 108, 29, 57, 104, 42, 61, 59, 50, 51, 40, 106, 123, 41, 22, 119, 55, 118, 49, 105, 124, 125, 34, 86, 93, 53, 112, 47, 98, 87, 116, 37, 127, 111, 63, 90, 24, 94, 54, 52, 27, 101, 39, 62, 30, 26, 16, 25, 121, 80, 48, 23, 91, 18, 83, 103, 110, 100, 82, 35, 89, 13, 117, 21, 88, 85, 79, 19, 122, 20, 77, 11, 75, 36, 32, 84, 46, 102, 96, 99, 73, 78, 15, 9, 107, 76, 17, 92, 81, 97, 14, 43, 12, 72, 69, 5, 38, 74, 65, 71, 7, 10, 67, 8, 28, 0, 70, 33, 64, 1, 68, 3, 6, 31, 66, 4, 95, 2], [126, 120, 109, 56, 58, 44, 114, 60, 115, 45, 113, 108, 29, 104, 61, 42, 57, 51, 40, 59, 50, 106, 41, 55, 22, 123, 105, 124, 118, 119, 34, 86, 93, 90, 49, 125, 112, 98, 116, 127, 62, 39, 52, 26, 87, 27, 16, 121, 63, 47, 24, 100, 37, 101, 25, 94, 117, 30, 111, 53, 80, 91, 18, 122, 103, 48, 89, 54, 88, 82, 23, 32, 19, 83, 99, 85, 36, 15, 102, 35, 20, 21, 13, 77, 84, 79, 11, 73, 75, 110, 43, 96, 92, 46, 65, 9, 1, 17, 64, 5, 3, 78, 107, 70, 76, 14, 81, 97, 10, 8, 67, 74, 0, 33, 7, 38, 69, 12, 72, 71, 28, 4, 2, 6, 31, 68, 66, 95], [126, 120, 109, 58, 56, 44, 114, 60, 113, 115, 45, 108, 104, 29, 61, 57, 42, 51, 50, 40, 59, 41, 119, 106, 55, 49, 123, 105, 125, 118, 22, 34, 116, 124, 62, 52, 90, 93, 63, 53, 98, 39, 122, 86, 127, 94, 47, 87, 112, 16, 117, 121, 100, 27, 30, 37, 111, 85, 101, 25, 48, 80, 24, 88, 26, 54, 91, 18, 23, 15, 103, 89, 36, 19, 21, 110, 82, 35, 46, 102, 77, 20, 96, 84, 75, 79, 83, 13, 32, 99, 73, 43, 11, 67, 92, 17, 10, 65, 76, 70, 5, 78, 81, 14, 74, 9, 3, 1, 12, 64, 107, 97, 8, 0, 69, 38, 33, 7, 72, 6, 68, 28, 71, 4, 66, 2, 31, 95], [126, 120, 109, 58, 56, 44, 114, 60, 115, 113, 45, 108, 61, 57, 104, 42, 50, 59, 51, 29, 41, 40, 106, 123, 119, 105, 55, 124, 116, 62, 22, 49, 118, 34, 90, 47, 125, 86, 93, 98, 127, 112, 111, 87, 122, 39, 94, 63, 27, 16, 80, 117, 24, 100, 53, 88, 37, 89, 18, 121, 91, 52, 23, 48, 19, 83, 26, 82, 30, 25, 15, 54, 85, 77, 75, 20, 13, 35, 103, 110, 101, 102, 46, 84, 79, 21, 11, 36, 43, 9, 73, 3, 96, 78, 14, 17, 76, 81, 70, 32, 99, 8, 1, 10, 67, 92, 107, 7, 5, 69, 12, 97, 74, 0, 65, 6, 72, 64, 71, 28, 33, 68, 4, 38, 66, 31, 2, 95], [126, 120, 109, 56, 58, 44, 114, 60, 115, 113, 45, 108, 104, 29, 61, 57, 50, 42, 51, 59, 41, 40, 55, 22, 106, 123, 105, 49, 119, 124, 34, 116, 125, 118, 62, 86, 93, 87, 127, 63, 16, 98, 90, 27, 94, 24, 37, 80, 117, 47, 18, 112, 39, 111, 53, 88, 91, 85, 30, 13, 89, 19, 82, 26, 48, 122, 83, 121, 23, 54, 25, 84, 35, 15, 100, 79, 110, 75, 103, 77, 20, 21, 52, 101, 11, 46, 17, 102, 99, 96, 36, 9, 81, 97, 14, 12, 78, 73, 92, 32, 8, 10, 74, 107, 76, 43, 70, 5, 69, 7, 72, 71, 33, 65, 67, 28, 38, 1, 3, 64, 0, 6, 31, 95, 4, 68, 2, 66], [126, 120, 109, 58, 56, 44, 60, 114, 115, 113, 45, 108, 29, 57, 42, 104, 51, 61, 50, 106, 40, 59, 41, 123, 22, 55, 119, 124, 49, 105, 118, 34, 127, 125, 86, 87, 93, 24, 27, 53, 94, 47, 116, 90, 80, 37, 88, 62, 89, 111, 16, 98, 25, 63, 83, 121, 52, 19, 18, 84, 112, 117, 91, 122, 15, 54, 30, 26, 77, 39, 100, 23, 48, 75, 21, 82, 85, 110, 101, 13, 46, 35, 20, 36, 103, 102, 11, 99, 79, 96, 17, 14, 81, 78, 9, 12, 32, 73, 92, 10, 107, 76, 97, 43, 8, 33, 7, 28, 38, 69, 74, 72, 5, 3, 71, 1, 0, 31, 70, 67, 65, 68, 6, 64, 4, 95, 66, 2], [126, 120, 109, 58, 44, 56, 60, 114, 115, 113, 108, 45, 57, 29, 42, 61, 51, 104, 59, 50, 22, 123, 106, 41, 40, 55, 118, 119, 125, 53, 49, 86, 124, 90, 105, 93, 34, 62, 87, 27, 127, 94, 24, 25, 89, 88, 98, 116, 37, 112, 80, 47, 63, 101, 16, 52, 39, 111, 18, 91, 26, 100, 83, 23, 30, 117, 121, 19, 54, 122, 99, 15, 103, 85, 21, 13, 82, 84, 35, 77, 102, 20, 46, 48, 75, 36, 32, 110, 97, 79, 96, 81, 92, 73, 17, 11, 33, 14, 78, 9, 43, 28, 12, 107, 8, 69, 5, 71, 6, 10, 76, 7, 65, 72, 31, 0, 3, 74, 64, 67, 1, 38, 4, 95, 68, 66, 70, 2], [126, 120, 109, 58, 56, 44, 60, 114, 113, 115, 45, 108, 57, 29, 104, 42, 61, 51, 50, 59, 40, 106, 123, 41, 55, 22, 119, 86, 49, 34, 93, 105, 118, 127, 37, 47, 124, 122, 90, 53, 116, 94, 62, 25, 89, 87, 26, 111, 27, 63, 125, 24, 30, 98, 52, 121, 16, 88, 80, 18, 101, 112, 83, 48, 54, 23, 35, 46, 100, 91, 82, 39, 85, 15, 117, 103, 84, 21, 13, 20, 110, 36, 19, 102, 32, 11, 99, 77, 43, 75, 79, 78, 96, 17, 81, 97, 73, 92, 107, 14, 74, 9, 12, 10, 33, 76, 28, 6, 69, 71, 8, 38, 7, 5, 72, 64, 65, 1, 31, 0, 68, 67, 3, 95, 4, 66, 2, 70], [126, 120, 109, 58, 56, 44, 114, 60, 113, 115, 45, 108, 57, 29, 61, 104, 42, 51, 50, 40, 123, 41, 106, 62, 59, 119, 49, 22, 55, 86, 93, 127, 34, 52, 118, 124, 105, 37, 125, 121, 47, 116, 53, 94, 90, 87, 63, 54, 98, 30, 122, 111, 89, 101, 16, 80, 25, 39, 85, 112, 24, 88, 27, 117, 83, 100, 23, 26, 46, 82, 91, 13, 18, 11, 35, 77, 36, 48, 75, 79, 110, 21, 103, 20, 19, 84, 102, 99, 76, 81, 15, 96, 9, 73, 17, 78, 32, 43, 92, 107, 5, 8, 7, 97, 71, 69, 6, 72, 12, 14, 74, 28, 3, 65, 1, 10, 38, 0, 33, 67, 64, 4, 70, 95, 68, 66, 2, 31], [126, 120, 109, 58, 56, 44, 114, 60, 115, 45, 113, 108, 104, 29, 57, 42, 61, 51, 123, 41, 40, 59, 50, 55, 106, 125, 105, 34, 49, 22, 119, 118, 62, 124, 86, 53, 93, 80, 116, 112, 98, 90, 16, 47, 52, 127, 94, 37, 122, 111, 87, 89, 27, 63, 13, 24, 30, 18, 117, 39, 85, 54, 23, 103, 25, 88, 26, 15, 100, 91, 84, 19, 11, 75, 77, 83, 82, 121, 79, 20, 48, 101, 73, 21, 96, 110, 36, 9, 35, 46, 17, 5, 67, 6, 14, 81, 72, 92, 78, 32, 43, 102, 99, 1, 69, 71, 76, 7, 10, 12, 97, 74, 65, 3, 8, 0, 64, 70, 68, 38, 33, 107, 4, 28, 2, 66, 95, 31], [126, 120, 109, 58, 56, 44, 114, 60, 115, 108, 113, 45, 104, 29, 42, 57, 61, 50, 51, 40, 59, 106, 41, 123, 22, 55, 49, 34, 86, 125, 105, 118, 124, 119, 47, 62, 127, 52, 98, 80, 90, 53, 24, 93, 27, 87, 16, 116, 122, 117, 37, 89, 94, 26, 39, 121, 112, 88, 91, 13, 63, 18, 30, 111, 83, 103, 48, 19, 82, 85, 54, 11, 23, 101, 84, 100, 25, 79, 75, 21, 20, 15, 77, 35, 46, 36, 9, 110, 78, 14, 73, 81, 99, 102, 96, 12, 32, 43, 17, 71, 97, 92, 72, 33, 107, 10, 69, 7, 5, 76, 6, 74, 3, 67, 1, 8, 0, 38, 65, 70, 64, 28, 31, 4, 68, 2, 66, 95], [126, 120, 109, 56, 114, 58, 60, 44, 115, 113, 108, 45, 29, 57, 61, 42, 123, 51, 104, 50, 40, 106, 41, 59, 124, 125, 55, 119, 118, 22, 86, 105, 49, 93, 62, 122, 52, 37, 34, 90, 53, 94, 116, 87, 47, 127, 27, 24, 88, 98, 101, 89, 111, 30, 121, 112, 103, 54, 25, 82, 91, 117, 16, 80, 48, 46, 26, 100, 83, 18, 35, 110, 21, 85, 39, 19, 63, 96, 102, 99, 23, 13, 20, 36, 84, 32, 79, 97, 11, 81, 15, 77, 43, 75, 78, 92, 107, 12, 17, 73, 14, 33, 9, 10, 71, 28, 72, 5, 38, 74, 76, 70, 69, 8, 7, 31, 68, 67, 6, 95, 1, 65, 3, 0, 4, 66, 64, 2], [126, 120, 109, 56, 114, 58, 44, 60, 115, 108, 113, 45, 57, 29, 104, 51, 42, 61, 123, 50, 106, 40, 59, 119, 41, 118, 55, 22, 124, 86, 105, 125, 49, 34, 62, 127, 90, 116, 37, 93, 47, 53, 63, 54, 98, 52, 87, 30, 16, 111, 24, 117, 27, 94, 122, 80, 101, 100, 26, 25, 89, 121, 91, 112, 103, 88, 83, 23, 35, 39, 84, 13, 85, 48, 46, 19, 21, 18, 82, 11, 75, 79, 81, 99, 107, 9, 36, 102, 20, 32, 15, 77, 110, 43, 97, 78, 96, 73, 14, 71, 17, 5, 76, 70, 92, 10, 72, 64, 33, 12, 7, 65, 69, 0, 1, 74, 8, 3, 38, 67, 28, 4, 95, 31, 66, 68, 2, 6], [126, 120, 109, 58, 56, 44, 114, 60, 115, 113, 108, 45, 57, 29, 104, 42, 61, 51, 50, 123, 59, 106, 40, 41, 22, 55, 119, 49, 93, 105, 34, 118, 124, 98, 86, 87, 62, 125, 37, 80, 90, 47, 16, 52, 89, 53, 94, 111, 127, 122, 27, 39, 116, 24, 30, 54, 101, 46, 117, 112, 18, 100, 26, 21, 19, 91, 13, 82, 83, 11, 85, 103, 25, 75, 23, 84, 15, 63, 36, 81, 77, 20, 88, 107, 110, 73, 79, 121, 9, 35, 102, 17, 48, 78, 96, 99, 14, 12, 76, 92, 7, 32, 70, 10, 5, 97, 72, 71, 67, 43, 69, 8, 3, 64, 74, 33, 1, 0, 65, 38, 28, 4, 6, 68, 31, 2, 95, 66], [126, 120, 109, 56, 58, 44, 60, 114, 115, 113, 45, 108, 29, 57, 42, 61, 104, 50, 51, 59, 40, 123, 41, 55, 49, 119, 106, 105, 22, 34, 90, 124, 47, 62, 98, 116, 118, 125, 86, 117, 53, 111, 122, 37, 52, 127, 103, 30, 112, 24, 93, 94, 87, 26, 46, 54, 80, 39, 101, 16, 27, 63, 82, 25, 85, 13, 88, 19, 110, 91, 83, 89, 100, 11, 121, 18, 15, 75, 73, 21, 84, 36, 35, 23, 48, 77, 20, 79, 9, 102, 99, 96, 107, 81, 92, 78, 10, 12, 32, 14, 72, 76, 70, 17, 33, 69, 97, 43, 7, 67, 3, 8, 71, 65, 74, 38, 0, 1, 5, 64, 6, 28, 68, 4, 66, 95, 31, 2], [126, 120, 109, 114, 56, 44, 58, 60, 115, 108, 113, 45, 29, 57, 61, 42, 104, 51, 50, 40, 106, 123, 41, 59, 55, 49, 119, 118, 22, 86, 34, 105, 124, 62, 94, 52, 93, 116, 98, 125, 111, 90, 37, 47, 30, 53, 27, 117, 87, 127, 101, 39, 24, 122, 54, 63, 80, 112, 88, 25, 26, 91, 100, 89, 19, 82, 46, 48, 16, 21, 13, 85, 83, 103, 84, 35, 23, 121, 18, 96, 79, 102, 15, 36, 11, 110, 107, 99, 92, 20, 32, 75, 17, 12, 77, 81, 33, 43, 73, 78, 97, 9, 14, 10, 64, 69, 28, 71, 38, 76, 72, 74, 8, 5, 7, 65, 70, 67, 1, 0, 3, 95, 31, 4, 6, 2, 68, 66], [126, 120, 109, 58, 114, 44, 56, 60, 115, 113, 108, 45, 29, 57, 104, 42, 61, 51, 50, 106, 40, 123, 41, 59, 119, 55, 22, 49, 34, 105, 118, 93, 86, 62, 124, 125, 47, 111, 98, 90, 94, 52, 122, 26, 37, 116, 117, 87, 24, 63, 27, 30, 39, 53, 80, 112, 88, 85, 16, 89, 101, 127, 25, 91, 121, 100, 82, 18, 54, 83, 13, 110, 103, 46, 19, 23, 36, 11, 20, 102, 21, 35, 15, 32, 48, 99, 96, 12, 75, 79, 84, 77, 9, 92, 73, 17, 107, 81, 97, 78, 14, 43, 71, 33, 7, 69, 5, 74, 8, 72, 28, 38, 64, 67, 10, 1, 3, 0, 76, 65, 6, 70, 31, 68, 4, 66, 2, 95], [126, 120, 109, 58, 44, 60, 56, 114, 115, 113, 45, 108, 57, 104, 29, 61, 42, 51, 50, 40, 59, 123, 41, 124, 106, 119, 62, 22, 105, 49, 34, 118, 55, 37, 93, 47, 116, 52, 94, 98, 111, 90, 125, 87, 53, 122, 39, 100, 86, 127, 30, 112, 54, 121, 27, 80, 25, 26, 101, 117, 24, 18, 103, 35, 63, 16, 88, 110, 36, 89, 46, 23, 21, 83, 82, 91, 102, 85, 96, 19, 15, 75, 11, 48, 84, 13, 20, 99, 107, 79, 81, 73, 77, 32, 38, 92, 17, 97, 9, 12, 43, 76, 14, 78, 8, 1, 74, 71, 67, 6, 10, 5, 64, 7, 3, 69, 72, 28, 33, 65, 0, 70, 95, 68, 66, 31, 4, 2], [126, 120, 109, 58, 56, 114, 44, 60, 115, 113, 45, 108, 29, 57, 61, 104, 42, 51, 50, 40, 59, 41, 123, 22, 119, 106, 124, 49, 34, 55, 105, 118, 62, 125, 93, 98, 116, 52, 86, 94, 90, 37, 111, 112, 63, 47, 53, 16, 101, 87, 121, 127, 30, 25, 24, 27, 54, 26, 91, 80, 100, 103, 23, 122, 82, 89, 110, 18, 39, 48, 11, 85, 88, 83, 21, 19, 96, 46, 13, 117, 35, 84, 102, 15, 75, 107, 77, 99, 20, 36, 79, 92, 73, 81, 32, 14, 17, 9, 78, 97, 74, 76, 33, 10, 12, 8, 6, 5, 71, 43, 1, 7, 72, 38, 67, 28, 69, 65, 64, 3, 0, 95, 4, 70, 68, 31, 66, 2], [126, 120, 109, 58, 56, 114, 44, 60, 115, 113, 45, 108, 29, 104, 61, 42, 57, 51, 40, 50, 59, 123, 106, 49, 41, 55, 22, 62, 119, 124, 34, 118, 86, 105, 52, 125, 90, 93, 94, 116, 37, 63, 98, 53, 16, 47, 24, 87, 80, 117, 112, 111, 39, 30, 54, 26, 27, 91, 89, 88, 101, 82, 25, 23, 121, 48, 103, 13, 21, 85, 83, 127, 18, 100, 84, 20, 19, 110, 75, 11, 77, 36, 96, 79, 122, 102, 107, 46, 32, 15, 99, 35, 9, 17, 81, 92, 73, 12, 33, 43, 78, 14, 76, 10, 6, 71, 8, 97, 74, 7, 5, 72, 3, 28, 69, 0, 65, 1, 67, 38, 64, 31, 95, 70, 4, 68, 66, 2], [126, 120, 109, 58, 56, 114, 44, 60, 113, 115, 45, 108, 104, 29, 51, 61, 57, 42, 40, 106, 59, 50, 41, 62, 123, 119, 105, 124, 55, 118, 34, 49, 125, 86, 52, 22, 37, 87, 98, 93, 116, 47, 90, 121, 53, 63, 112, 111, 94, 117, 89, 16, 39, 24, 27, 127, 26, 88, 30, 110, 54, 122, 18, 80, 91, 101, 23, 83, 85, 48, 102, 82, 25, 21, 100, 20, 96, 19, 35, 13, 77, 46, 11, 36, 84, 107, 99, 17, 15, 79, 75, 92, 103, 12, 32, 33, 73, 81, 43, 0, 9, 78, 14, 67, 65, 64, 1, 5, 8, 10, 69, 72, 97, 6, 74, 76, 38, 71, 7, 28, 3, 4, 70, 31, 2, 68, 66, 95], [126, 120, 109, 58, 56, 44, 114, 60, 115, 113, 45, 108, 61, 29, 104, 57, 51, 42, 40, 50, 59, 123, 106, 41, 118, 62, 55, 22, 124, 119, 105, 49, 116, 86, 34, 93, 37, 98, 111, 94, 87, 90, 125, 53, 30, 117, 47, 112, 16, 121, 110, 52, 39, 54, 127, 27, 100, 103, 122, 91, 88, 63, 26, 101, 18, 89, 80, 24, 48, 19, 83, 20, 25, 82, 23, 21, 35, 36, 85, 107, 46, 77, 13, 99, 84, 102, 11, 32, 79, 96, 92, 75, 15, 17, 9, 12, 73, 33, 78, 81, 14, 43, 69, 3, 8, 97, 76, 7, 74, 10, 28, 38, 1, 65, 5, 6, 64, 68, 70, 71, 72, 0, 67, 31, 66, 95, 4, 2], [126, 120, 109, 56, 58, 114, 44, 60, 115, 113, 108, 45, 29, 104, 61, 42, 57, 51, 123, 50, 40, 106, 59, 41, 62, 22, 49, 118, 119, 55, 105, 124, 34, 116, 93, 47, 125, 86, 87, 98, 37, 63, 90, 52, 94, 111, 53, 30, 80, 91, 117, 24, 121, 88, 39, 16, 26, 18, 122, 127, 112, 100, 48, 27, 89, 19, 54, 101, 46, 85, 25, 110, 82, 20, 11, 83, 103, 15, 75, 23, 21, 36, 102, 35, 84, 77, 107, 96, 13, 79, 32, 73, 17, 92, 14, 76, 9, 99, 7, 12, 97, 81, 78, 8, 69, 43, 10, 38, 70, 33, 74, 5, 28, 71, 72, 67, 1, 0, 3, 65, 64, 68, 6, 95, 4, 31, 66, 2], [126, 120, 109, 58, 56, 44, 114, 60, 115, 113, 45, 108, 57, 29, 104, 42, 61, 51, 40, 123, 50, 106, 59, 41, 22, 55, 119, 105, 49, 118, 62, 34, 124, 93, 125, 86, 87, 121, 52, 47, 98, 39, 90, 116, 94, 37, 26, 111, 24, 63, 89, 127, 112, 117, 16, 54, 27, 18, 80, 30, 91, 82, 53, 88, 83, 122, 101, 100, 48, 23, 19, 21, 85, 77, 15, 20, 110, 75, 11, 13, 25, 84, 103, 35, 96, 36, 73, 17, 46, 79, 102, 9, 81, 76, 107, 92, 78, 32, 99, 74, 14, 12, 70, 43, 71, 7, 5, 72, 8, 69, 97, 10, 3, 38, 28, 67, 65, 33, 1, 64, 0, 4, 6, 31, 68, 95, 2, 66], [126, 120, 109, 56, 58, 44, 114, 60, 115, 113, 45, 108, 29, 104, 57, 42, 61, 123, 51, 50, 59, 40, 106, 119, 41, 49, 22, 118, 124, 105, 55, 34, 63, 62, 125, 121, 86, 116, 52, 90, 98, 93, 53, 87, 47, 24, 94, 54, 37, 39, 117, 101, 27, 26, 30, 127, 89, 111, 122, 112, 18, 48, 80, 16, 88, 91, 100, 25, 83, 35, 84, 23, 82, 19, 96, 36, 46, 110, 13, 21, 75, 85, 9, 102, 107, 103, 15, 11, 99, 92, 20, 77, 79, 73, 43, 76, 32, 97, 14, 17, 33, 78, 81, 74, 28, 38, 7, 10, 71, 5, 72, 70, 69, 12, 8, 67, 1, 65, 64, 0, 31, 95, 3, 68, 6, 66, 4, 2], [126, 120, 109, 58, 44, 56, 114, 60, 115, 108, 45, 113, 29, 104, 42, 57, 61, 51, 123, 50, 59, 40, 106, 119, 41, 124, 55, 118, 62, 125, 34, 22, 105, 90, 86, 53, 49, 116, 93, 94, 127, 52, 122, 37, 98, 47, 63, 117, 39, 112, 87, 111, 27, 54, 103, 30, 121, 101, 100, 48, 25, 88, 89, 18, 23, 24, 16, 26, 80, 83, 110, 91, 36, 84, 35, 99, 75, 82, 20, 21, 19, 46, 107, 13, 96, 43, 77, 102, 15, 85, 9, 11, 32, 17, 7, 79, 97, 14, 92, 78, 81, 73, 76, 33, 38, 72, 67, 74, 12, 69, 5, 70, 28, 3, 71, 31, 8, 64, 65, 10, 0, 1, 6, 68, 4, 95, 2, 66], [126, 120, 109, 58, 56, 44, 114, 60, 115, 113, 45, 108, 29, 104, 57, 42, 61, 51, 50, 40, 59, 106, 123, 41, 62, 22, 124, 118, 119, 49, 105, 55, 116, 87, 86, 47, 93, 34, 125, 122, 94, 121, 111, 98, 117, 63, 52, 37, 112, 90, 127, 30, 39, 80, 18, 24, 53, 16, 27, 89, 54, 26, 25, 101, 91, 83, 100, 13, 75, 84, 23, 19, 88, 82, 21, 48, 96, 110, 11, 35, 15, 79, 20, 36, 85, 77, 102, 107, 73, 99, 32, 9, 17, 81, 103, 46, 12, 14, 76, 78, 74, 10, 7, 5, 92, 97, 72, 1, 43, 69, 67, 71, 8, 64, 33, 6, 3, 65, 38, 70, 0, 4, 28, 68, 66, 95, 31, 2], [126, 120, 109, 58, 56, 44, 114, 60, 115, 113, 45, 108, 29, 61, 104, 57, 42, 40, 50, 51, 59, 123, 41, 106, 22, 49, 124, 105, 62, 55, 119, 118, 34, 86, 125, 63, 98, 47, 93, 116, 94, 111, 87, 30, 37, 90, 122, 16, 127, 112, 39, 27, 52, 100, 24, 117, 54, 80, 96, 91, 82, 101, 25, 26, 121, 89, 88, 18, 53, 110, 75, 46, 103, 83, 13, 85, 79, 48, 23, 19, 21, 35, 84, 36, 77, 11, 15, 102, 20, 73, 9, 43, 107, 14, 99, 81, 72, 76, 5, 92, 17, 32, 1, 74, 78, 97, 64, 65, 12, 10, 8, 0, 7, 3, 69, 6, 71, 38, 67, 33, 70, 28, 4, 66, 68, 95, 31, 2], [126, 120, 109, 56, 58, 44, 114, 115, 60, 113, 45, 108, 29, 104, 61, 51, 42, 57, 59, 50, 123, 40, 41, 106, 22, 118, 105, 49, 124, 119, 55, 34, 125, 86, 116, 93, 98, 87, 94, 47, 63, 53, 90, 127, 52, 37, 30, 117, 80, 24, 111, 112, 39, 62, 16, 54, 27, 121, 26, 82, 88, 25, 75, 83, 91, 13, 21, 100, 23, 89, 101, 18, 35, 48, 79, 96, 85, 77, 110, 122, 84, 46, 20, 11, 15, 19, 73, 103, 99, 36, 92, 107, 76, 14, 9, 78, 81, 43, 102, 97, 12, 17, 32, 72, 5, 6, 7, 10, 33, 74, 28, 69, 71, 8, 67, 3, 38, 1, 64, 65, 0, 70, 31, 4, 68, 2, 95, 66], [126, 120, 109, 56, 114, 58, 44, 115, 60, 113, 45, 108, 104, 29, 61, 42, 50, 57, 51, 123, 59, 40, 41, 106, 49, 118, 119, 55, 124, 105, 116, 22, 34, 125, 86, 52, 63, 112, 98, 62, 47, 30, 94, 87, 111, 127, 93, 39, 90, 37, 54, 101, 121, 27, 24, 26, 16, 53, 80, 100, 117, 122, 103, 91, 82, 23, 89, 48, 18, 96, 88, 85, 25, 75, 35, 13, 84, 15, 110, 19, 21, 9, 36, 73, 77, 83, 20, 79, 7, 11, 107, 99, 46, 14, 5, 6, 76, 102, 81, 32, 92, 78, 43, 12, 69, 17, 10, 97, 8, 74, 67, 72, 64, 0, 33, 65, 1, 71, 38, 3, 28, 4, 95, 68, 70, 31, 66, 2], [126, 120, 109, 44, 56, 114, 58, 60, 115, 108, 113, 45, 29, 104, 57, 61, 42, 51, 50, 59, 40, 41, 106, 22, 123, 55, 118, 116, 119, 125, 124, 34, 86, 105, 49, 62, 87, 93, 47, 98, 94, 37, 127, 111, 90, 27, 39, 53, 30, 80, 100, 24, 16, 52, 63, 117, 121, 112, 122, 26, 25, 19, 101, 91, 88, 54, 13, 23, 82, 75, 18, 48, 35, 89, 83, 85, 21, 15, 110, 84, 77, 79, 96, 20, 11, 103, 9, 73, 14, 46, 76, 36, 102, 107, 97, 43, 32, 99, 81, 92, 78, 7, 17, 6, 69, 12, 72, 8, 74, 28, 33, 38, 71, 5, 10, 67, 31, 1, 95, 68, 3, 4, 65, 64, 0, 70, 66, 2]], "model.layers.18.self_attn.q_proj": [[39, 52, 112, 113, 32, 26, 87, 85, 121, 49, 29, 90, 74, 79, 116, 70, 77, 20, 53, 93, 82, 80, 46, 44, 23, 125, 119, 0, 75, 57, 98, 124, 18, 55, 61, 68, 111, 3, 48, 60, 54, 123, 115, 110, 62, 106, 65, 126, 19, 7, 117, 107, 84, 99, 118, 38, 56, 96, 120, 42, 47, 43, 9, 50, 109, 8, 2, 40, 51, 58, 11, 105, 21, 17, 25, 97, 41, 101, 59, 102, 122, 35, 24, 45, 6, 127, 63, 89, 104, 94, 33, 37, 30, 114, 100, 28, 108, 66, 22, 34, 14, 86, 36, 31, 88, 27, 92, 83, 103, 15, 69, 91, 10, 95, 81, 5, 16, 76, 71, 72, 1, 78, 12, 73, 67, 13, 4, 64], [39, 113, 52, 110, 112, 49, 29, 46, 87, 85, 32, 116, 90, 53, 26, 80, 93, 60, 83, 92, 77, 125, 115, 108, 23, 44, 123, 79, 21, 31, 121, 98, 119, 56, 111, 88, 58, 8, 48, 124, 107, 61, 105, 122, 42, 101, 18, 74, 118, 126, 62, 100, 25, 57, 55, 33, 63, 104, 106, 97, 45, 43, 24, 70, 117, 54, 99, 35, 38, 41, 82, 89, 47, 114, 120, 59, 50, 40, 109, 51, 127, 102, 36, 86, 68, 12, 11, 28, 17, 78, 91, 96, 94, 37, 15, 22, 34, 84, 9, 95, 30, 20, 7, 76, 19, 13, 27, 72, 75, 6, 16, 81, 73, 14, 2, 4, 64, 71, 65, 3, 69, 10, 0, 103, 1, 67, 66, 5], [39, 113, 52, 49, 29, 85, 32, 87, 116, 90, 26, 121, 79, 60, 125, 110, 46, 93, 112, 119, 53, 44, 21, 77, 124, 111, 122, 59, 61, 96, 109, 118, 123, 80, 98, 84, 56, 115, 54, 82, 47, 55, 57, 18, 23, 8, 74, 108, 83, 101, 120, 50, 92, 43, 22, 88, 89, 37, 31, 48, 97, 127, 42, 41, 51, 11, 105, 38, 35, 91, 106, 45, 62, 58, 75, 107, 36, 104, 78, 70, 117, 33, 63, 28, 114, 126, 94, 15, 19, 86, 95, 102, 40, 30, 27, 34, 9, 99, 17, 100, 24, 16, 76, 68, 25, 20, 71, 7, 81, 65, 3, 14, 12, 103, 2, 13, 1, 0, 72, 73, 4, 67, 64, 10, 6, 5, 69, 66], [39, 113, 52, 112, 110, 49, 29, 32, 85, 116, 77, 122, 87, 26, 31, 98, 46, 93, 80, 48, 8, 119, 50, 83, 70, 79, 53, 18, 123, 120, 74, 44, 125, 111, 114, 60, 42, 58, 124, 88, 57, 100, 92, 21, 117, 118, 90, 56, 115, 23, 121, 61, 35, 7, 108, 45, 27, 109, 63, 101, 25, 37, 62, 43, 0, 59, 82, 51, 54, 28, 6, 86, 68, 99, 102, 36, 104, 107, 41, 97, 38, 47, 55, 106, 105, 126, 127, 69, 40, 33, 34, 12, 3, 94, 95, 13, 91, 11, 30, 78, 19, 24, 84, 22, 15, 96, 81, 89, 20, 65, 67, 66, 9, 2, 73, 75, 4, 71, 76, 72, 16, 14, 17, 10, 1, 64, 5, 103], [124, 63, 56, 100, 115, 36, 39, 110, 102, 122, 116, 111, 59, 54, 105, 58, 103, 57, 121, 120, 60, 49, 62, 127, 50, 45, 25, 117, 125, 47, 112, 123, 55, 48, 114, 53, 109, 93, 126, 35, 52, 42, 40, 113, 51, 101, 41, 34, 119, 46, 87, 118, 107, 108, 61, 32, 44, 99, 104, 43, 38, 106, 37, 80, 90, 96, 94, 20, 30, 98, 23, 24, 97, 33, 84, 27, 31, 95, 29, 73, 85, 91, 89, 19, 28, 22, 21, 92, 78, 26, 88, 11, 75, 16, 86, 18, 69, 83, 77, 82, 13, 7, 9, 14, 1, 71, 5, 6, 66, 17, 3, 2, 79, 81, 67, 4, 15, 70, 12, 72, 74, 10, 65, 76, 64, 0, 8, 68], [63, 124, 30, 39, 115, 100, 19, 36, 25, 96, 22, 12, 18, 17, 15, 99, 27, 89, 32, 13, 72, 24, 5, 103, 35, 74, 92, 6, 123, 122, 56, 45, 2, 54, 4, 1, 90, 120, 126, 28, 33, 71, 21, 121, 42, 85, 14, 113, 26, 98, 83, 125, 73, 84, 75, 11, 94, 108, 86, 117, 23, 69, 52, 9, 10, 0, 81, 20, 97, 7, 67, 101, 38, 112, 87, 118, 93, 60, 43, 29, 16, 57, 34, 47, 37, 78, 91, 107, 80, 82, 88, 3, 58, 105, 79, 44, 65, 127, 119, 48, 70, 59, 62, 95, 53, 40, 109, 51, 66, 76, 31, 102, 111, 104, 49, 77, 106, 110, 114, 50, 68, 64, 61, 55, 116, 41, 46, 8], [124, 63, 100, 36, 120, 98, 103, 38, 25, 96, 32, 84, 19, 87, 97, 45, 94, 101, 126, 39, 22, 56, 51, 89, 117, 59, 121, 58, 113, 27, 21, 17, 62, 34, 54, 31, 23, 107, 40, 86, 52, 33, 13, 47, 111, 48, 46, 112, 102, 90, 122, 127, 99, 15, 60, 24, 57, 43, 91, 125, 83, 116, 29, 118, 44, 26, 28, 119, 110, 114, 30, 88, 109, 106, 49, 61, 93, 123, 105, 115, 35, 55, 53, 108, 12, 80, 85, 37, 78, 42, 20, 11, 104, 41, 81, 18, 95, 92, 50, 7, 14, 16, 82, 5, 77, 6, 74, 79, 75, 73, 72, 69, 76, 4, 2, 71, 9, 64, 68, 70, 8, 10, 66, 65, 1, 67, 3, 0], [124, 63, 24, 100, 21, 30, 18, 17, 12, 22, 15, 39, 115, 8, 74, 72, 27, 4, 90, 96, 79, 84, 13, 92, 76, 36, 1, 25, 94, 122, 68, 64, 75, 26, 77, 51, 121, 0, 6, 7, 3, 83, 2, 33, 85, 69, 81, 31, 10, 28, 70, 19, 86, 89, 46, 14, 5, 78, 67, 11, 82, 66, 20, 16, 71, 23, 80, 102, 88, 65, 73, 87, 93, 29, 43, 111, 103, 95, 32, 9, 127, 97, 35, 91, 98, 114, 126, 42, 125, 44, 37, 99, 56, 40, 61, 45, 120, 52, 58, 34, 104, 105, 119, 50, 101, 118, 38, 53, 59, 108, 107, 123, 55, 106, 112, 41, 49, 116, 60, 57, 109, 110, 113, 48, 54, 117, 47, 62], [55, 102, 118, 123, 46, 97, 61, 62, 53, 116, 106, 21, 120, 27, 57, 51, 115, 112, 126, 58, 122, 119, 48, 49, 63, 114, 91, 124, 50, 45, 31, 109, 52, 117, 24, 99, 110, 127, 60, 95, 41, 125, 121, 54, 113, 56, 103, 111, 59, 14, 47, 108, 104, 44, 38, 87, 88, 40, 18, 28, 19, 78, 70, 6, 42, 39, 26, 105, 107, 92, 86, 43, 101, 36, 37, 100, 93, 98, 96, 35, 34, 32, 65, 33, 82, 23, 94, 25, 30, 29, 0, 89, 16, 84, 90, 85, 22, 76, 80, 64, 17, 1, 3, 68, 20, 4, 83, 67, 11, 12, 8, 15, 66, 81, 2, 71, 72, 5, 9, 75, 7, 69, 74, 79, 77, 73, 13, 10], [118, 102, 55, 123, 48, 97, 53, 62, 61, 27, 46, 24, 57, 91, 51, 21, 116, 120, 31, 38, 122, 126, 115, 50, 49, 119, 124, 112, 58, 114, 63, 110, 40, 117, 95, 106, 109, 52, 54, 113, 99, 121, 18, 127, 56, 86, 88, 125, 45, 47, 111, 41, 60, 59, 104, 14, 44, 39, 43, 103, 108, 100, 107, 42, 92, 33, 105, 26, 101, 94, 98, 37, 36, 32, 78, 19, 35, 70, 34, 96, 29, 93, 30, 25, 28, 22, 17, 23, 84, 87, 82, 90, 6, 16, 85, 89, 80, 83, 20, 76, 12, 65, 15, 81, 3, 0, 1, 8, 75, 9, 71, 73, 11, 68, 79, 67, 64, 5, 4, 66, 77, 7, 2, 69, 74, 72, 10, 13], [102, 55, 118, 97, 123, 86, 24, 84, 46, 15, 27, 82, 17, 88, 77, 90, 74, 18, 48, 11, 72, 76, 30, 38, 120, 91, 54, 8, 95, 69, 92, 106, 93, 62, 71, 104, 22, 5, 16, 3, 20, 9, 81, 29, 14, 119, 75, 10, 31, 7, 116, 13, 19, 79, 21, 89, 45, 25, 85, 26, 126, 83, 28, 80, 23, 12, 67, 122, 101, 94, 40, 1, 87, 33, 108, 98, 70, 124, 49, 78, 109, 61, 96, 73, 36, 64, 65, 53, 68, 6, 114, 32, 103, 34, 0, 66, 57, 4, 37, 35, 60, 2, 107, 99, 100, 59, 110, 44, 125, 58, 41, 113, 105, 42, 47, 39, 112, 43, 50, 111, 51, 117, 56, 63, 52, 127, 121, 115], [102, 118, 55, 97, 123, 84, 24, 86, 17, 46, 27, 15, 31, 62, 91, 14, 76, 90, 18, 38, 88, 73, 95, 22, 79, 101, 82, 120, 81, 53, 61, 48, 11, 54, 12, 104, 75, 80, 19, 74, 49, 106, 28, 83, 20, 71, 29, 119, 122, 6, 116, 23, 85, 51, 45, 30, 77, 10, 25, 126, 69, 96, 87, 21, 9, 89, 112, 16, 26, 2, 108, 92, 93, 7, 32, 57, 5, 13, 94, 109, 124, 67, 65, 35, 58, 98, 115, 125, 37, 114, 78, 72, 103, 8, 34, 63, 50, 59, 52, 110, 44, 99, 100, 111, 40, 60, 113, 127, 121, 47, 3, 56, 39, 66, 117, 36, 0, 42, 68, 41, 105, 70, 107, 43, 64, 1, 33, 4], [43, 36, 54, 96, 63, 60, 89, 20, 107, 82, 126, 50, 87, 115, 80, 62, 25, 21, 9, 52, 16, 61, 32, 79, 13, 48, 101, 112, 47, 75, 53, 39, 57, 29, 85, 41, 69, 71, 103, 122, 119, 111, 19, 106, 105, 35, 104, 45, 18, 84, 55, 86, 23, 40, 127, 94, 46, 59, 125, 99, 88, 44, 93, 121, 28, 26, 90, 38, 100, 58, 92, 97, 51, 78, 6, 98, 110, 108, 120, 33, 10, 37, 114, 15, 74, 27, 123, 124, 7, 116, 56, 17, 102, 83, 113, 118, 24, 49, 67, 77, 95, 12, 8, 11, 109, 34, 31, 14, 22, 72, 30, 91, 117, 81, 76, 42, 66, 73, 4, 1, 5, 2, 70, 64, 3, 0, 65, 68], [43, 60, 96, 36, 63, 107, 89, 48, 123, 20, 126, 82, 21, 119, 46, 111, 25, 19, 85, 44, 80, 26, 42, 55, 112, 23, 13, 51, 122, 79, 9, 45, 61, 54, 18, 75, 41, 53, 59, 87, 16, 62, 50, 39, 105, 69, 52, 57, 103, 38, 40, 106, 116, 27, 121, 113, 101, 71, 114, 92, 109, 84, 49, 32, 58, 115, 35, 104, 95, 117, 124, 108, 125, 56, 86, 33, 11, 90, 127, 37, 81, 110, 88, 98, 29, 99, 83, 118, 34, 100, 30, 102, 24, 7, 120, 93, 97, 17, 91, 47, 94, 77, 1, 31, 22, 28, 8, 15, 0, 14, 72, 3, 73, 78, 67, 12, 10, 6, 2, 74, 5, 68, 76, 70, 66, 4, 64, 65], [43, 96, 54, 36, 63, 47, 20, 89, 107, 25, 60, 119, 82, 9, 80, 16, 87, 21, 69, 79, 18, 111, 28, 71, 13, 48, 55, 112, 75, 120, 53, 38, 32, 126, 19, 61, 123, 41, 62, 50, 115, 51, 84, 85, 57, 110, 104, 40, 103, 39, 45, 29, 67, 106, 122, 125, 1, 118, 59, 95, 26, 37, 116, 44, 64, 121, 56, 127, 76, 58, 100, 113, 108, 49, 7, 114, 102, 65, 105, 52, 83, 46, 94, 109, 124, 117, 91, 86, 90, 101, 99, 34, 92, 23, 30, 35, 22, 42, 73, 98, 27, 93, 4, 97, 33, 6, 14, 66, 0, 72, 81, 74, 78, 15, 88, 31, 11, 5, 24, 10, 3, 68, 2, 17, 8, 77, 12, 70], [43, 54, 96, 36, 60, 89, 107, 82, 20, 87, 75, 63, 25, 80, 79, 71, 111, 13, 9, 47, 29, 69, 85, 16, 21, 77, 126, 97, 40, 32, 41, 38, 92, 106, 18, 119, 115, 28, 34, 117, 53, 121, 67, 19, 44, 27, 46, 57, 109, 58, 50, 103, 14, 48, 101, 10, 84, 35, 55, 122, 45, 93, 65, 95, 1, 114, 91, 23, 118, 51, 12, 37, 2, 83, 112, 15, 104, 88, 94, 31, 78, 123, 62, 30, 66, 59, 116, 99, 90, 11, 100, 64, 26, 61, 42, 127, 22, 56, 113, 33, 49, 7, 120, 24, 74, 81, 76, 73, 39, 110, 125, 98, 8, 124, 52, 17, 108, 5, 86, 4, 102, 105, 6, 70, 72, 68, 0, 3], [120, 104, 59, 113, 109, 51, 62, 110, 22, 56, 88, 57, 119, 112, 90, 52, 83, 41, 21, 114, 36, 98, 55, 116, 107, 50, 103, 29, 115, 123, 93, 54, 95, 43, 38, 42, 124, 49, 60, 122, 99, 45, 117, 105, 126, 102, 61, 48, 106, 58, 33, 30, 127, 108, 39, 118, 47, 125, 44, 46, 111, 53, 27, 121, 63, 101, 37, 97, 35, 28, 100, 92, 31, 96, 80, 10, 32, 34, 79, 86, 40, 74, 17, 25, 91, 94, 81, 19, 76, 23, 85, 77, 84, 26, 13, 24, 89, 14, 15, 20, 71, 87, 72, 82, 16, 18, 7, 12, 11, 67, 69, 3, 9, 78, 5, 6, 75, 8, 1, 70, 65, 68, 73, 0, 66, 4, 2, 64], [104, 120, 59, 113, 98, 88, 18, 80, 65, 22, 6, 76, 93, 9, 110, 15, 14, 62, 68, 40, 112, 119, 72, 57, 83, 84, 97, 32, 51, 28, 36, 100, 26, 123, 29, 1, 67, 82, 74, 52, 11, 5, 8, 24, 70, 81, 99, 61, 4, 27, 86, 21, 91, 90, 46, 19, 107, 75, 66, 117, 34, 55, 116, 33, 58, 77, 16, 124, 54, 37, 49, 114, 118, 50, 31, 23, 25, 60, 13, 42, 109, 127, 115, 53, 78, 38, 3, 87, 106, 17, 126, 89, 122, 30, 69, 101, 12, 20, 92, 108, 64, 85, 105, 71, 41, 56, 45, 95, 96, 35, 0, 7, 102, 94, 43, 125, 48, 44, 103, 10, 111, 47, 2, 63, 79, 73, 121, 39], [113, 104, 120, 59, 88, 98, 22, 93, 52, 51, 83, 57, 29, 21, 112, 80, 90, 119, 62, 110, 124, 114, 76, 28, 116, 81, 55, 123, 50, 11, 95, 18, 122, 117, 118, 15, 108, 54, 49, 42, 96, 115, 58, 61, 41, 85, 100, 39, 14, 56, 53, 47, 84, 46, 109, 60, 127, 44, 125, 91, 27, 92, 31, 36, 86, 48, 43, 103, 35, 26, 101, 126, 105, 97, 45, 40, 121, 106, 38, 82, 6, 99, 72, 111, 19, 17, 102, 65, 63, 13, 30, 79, 25, 74, 24, 107, 23, 33, 37, 94, 87, 10, 77, 32, 89, 9, 34, 12, 20, 5, 75, 68, 3, 8, 71, 66, 16, 69, 7, 67, 78, 0, 2, 1, 4, 64, 70, 73], [59, 104, 120, 113, 112, 57, 62, 51, 83, 36, 43, 55, 105, 89, 119, 52, 50, 98, 95, 60, 110, 124, 49, 126, 115, 90, 35, 121, 117, 122, 88, 45, 97, 92, 56, 109, 123, 21, 61, 118, 102, 54, 116, 58, 42, 46, 103, 127, 17, 107, 114, 47, 48, 44, 15, 125, 23, 108, 41, 106, 63, 74, 34, 53, 100, 39, 101, 32, 99, 37, 111, 38, 19, 10, 26, 29, 30, 96, 13, 93, 28, 24, 33, 25, 31, 81, 94, 84, 79, 91, 5, 67, 40, 85, 87, 3, 71, 20, 0, 77, 22, 65, 69, 27, 14, 64, 18, 7, 76, 11, 75, 66, 86, 1, 68, 9, 72, 6, 82, 80, 2, 8, 4, 12, 78, 73, 70, 16], [127, 54, 122, 125, 100, 62, 126, 119, 120, 41, 40, 36, 42, 28, 117, 106, 118, 56, 83, 115, 92, 112, 58, 124, 59, 113, 39, 55, 49, 46, 57, 111, 53, 123, 63, 86, 52, 116, 99, 61, 114, 121, 48, 104, 47, 60, 51, 50, 105, 107, 37, 103, 108, 82, 110, 43, 45, 44, 38, 109, 35, 15, 74, 94, 101, 33, 25, 102, 98, 29, 34, 31, 80, 20, 96, 89, 90, 71, 95, 77, 24, 79, 97, 32, 88, 67, 26, 22, 75, 30, 91, 93, 65, 19, 27, 18, 23, 21, 6, 14, 85, 87, 10, 7, 68, 66, 84, 0, 13, 76, 73, 17, 78, 4, 1, 64, 8, 81, 16, 2, 12, 11, 70, 9, 72, 3, 69, 5], [122, 54, 127, 40, 100, 125, 24, 90, 33, 31, 62, 120, 105, 99, 19, 36, 86, 26, 126, 28, 106, 119, 53, 25, 43, 81, 102, 41, 98, 20, 88, 87, 49, 69, 35, 94, 115, 85, 59, 76, 66, 73, 61, 32, 7, 104, 56, 91, 27, 52, 34, 15, 110, 23, 8, 57, 38, 83, 72, 22, 75, 114, 29, 58, 63, 12, 78, 5, 101, 3, 109, 113, 92, 117, 48, 123, 42, 60, 111, 51, 17, 124, 16, 116, 37, 70, 103, 74, 112, 93, 55, 121, 96, 47, 82, 10, 21, 13, 50, 11, 9, 14, 30, 95, 46, 118, 39, 1, 89, 4, 80, 108, 107, 45, 77, 6, 44, 67, 18, 79, 84, 2, 71, 64, 65, 68, 0, 97], [54, 122, 127, 100, 41, 40, 92, 33, 106, 120, 125, 126, 36, 24, 103, 28, 83, 86, 119, 117, 94, 32, 26, 99, 115, 15, 42, 62, 49, 31, 59, 90, 29, 21, 105, 57, 96, 88, 53, 104, 82, 124, 113, 56, 39, 58, 95, 63, 74, 112, 35, 38, 34, 16, 123, 114, 55, 118, 48, 80, 61, 50, 98, 52, 37, 22, 51, 121, 46, 89, 101, 27, 102, 23, 47, 97, 85, 19, 77, 116, 111, 60, 107, 20, 43, 30, 110, 25, 87, 91, 13, 79, 109, 81, 44, 93, 75, 108, 76, 18, 12, 45, 71, 78, 14, 11, 17, 84, 8, 6, 10, 73, 9, 66, 4, 67, 72, 65, 7, 0, 69, 68, 70, 1, 64, 3, 5, 2], [127, 122, 54, 100, 41, 40, 126, 86, 26, 33, 28, 120, 15, 36, 42, 43, 117, 119, 96, 125, 83, 106, 49, 53, 57, 59, 21, 92, 118, 39, 124, 38, 115, 88, 103, 62, 107, 56, 24, 58, 35, 19, 114, 46, 108, 95, 112, 61, 37, 48, 105, 110, 81, 52, 123, 47, 63, 55, 113, 102, 17, 104, 50, 60, 111, 97, 116, 31, 99, 34, 98, 121, 51, 101, 44, 30, 22, 45, 13, 84, 109, 82, 32, 74, 77, 10, 94, 29, 11, 18, 93, 90, 89, 91, 87, 78, 27, 23, 14, 71, 20, 7, 85, 25, 79, 80, 75, 16, 67, 76, 72, 73, 8, 6, 4, 66, 1, 65, 12, 68, 9, 0, 64, 3, 70, 69, 2, 5], [105, 98, 117, 86, 96, 28, 53, 20, 52, 116, 25, 41, 94, 38, 80, 126, 109, 56, 63, 119, 123, 45, 111, 82, 19, 14, 44, 83, 62, 118, 71, 75, 120, 89, 77, 87, 115, 54, 78, 8, 49, 21, 112, 79, 23, 48, 51, 10, 50, 58, 57, 102, 61, 30, 46, 55, 59, 37, 39, 124, 27, 114, 125, 6, 107, 100, 127, 16, 43, 40, 93, 31, 35, 101, 104, 29, 60, 103, 121, 26, 42, 97, 110, 2, 85, 122, 4, 95, 113, 106, 17, 33, 36, 99, 47, 108, 9, 90, 34, 24, 22, 91, 88, 84, 18, 7, 32, 64, 81, 1, 92, 76, 12, 11, 15, 67, 68, 13, 73, 70, 0, 74, 66, 3, 65, 72, 5, 69], [105, 98, 96, 86, 20, 117, 82, 53, 75, 25, 41, 123, 79, 80, 52, 77, 63, 8, 116, 28, 94, 6, 126, 10, 16, 118, 89, 2, 119, 112, 4, 59, 62, 56, 45, 115, 44, 91, 111, 3, 109, 11, 57, 120, 13, 30, 88, 84, 26, 23, 54, 48, 97, 7, 51, 38, 17, 93, 50, 22, 74, 127, 67, 29, 114, 83, 18, 0, 58, 14, 31, 95, 102, 21, 61, 85, 78, 70, 39, 76, 104, 81, 19, 40, 107, 55, 92, 36, 15, 5, 24, 60, 64, 101, 125, 35, 121, 124, 106, 27, 113, 68, 87, 32, 1, 37, 90, 33, 72, 43, 49, 47, 110, 73, 34, 69, 99, 66, 65, 9, 103, 46, 122, 12, 42, 100, 108, 71], [105, 98, 86, 96, 117, 82, 8, 53, 79, 25, 63, 20, 75, 80, 4, 77, 41, 94, 2, 64, 123, 62, 6, 89, 48, 116, 118, 0, 115, 52, 72, 28, 66, 1, 29, 68, 13, 126, 16, 71, 69, 70, 87, 44, 85, 10, 112, 30, 65, 51, 109, 45, 11, 59, 119, 56, 57, 50, 88, 5, 18, 7, 24, 26, 93, 19, 111, 120, 35, 22, 78, 17, 15, 127, 21, 84, 3, 74, 76, 58, 39, 114, 12, 9, 100, 92, 60, 106, 23, 73, 14, 83, 102, 99, 67, 27, 40, 38, 121, 55, 31, 95, 91, 97, 61, 49, 101, 113, 122, 36, 124, 34, 81, 47, 33, 37, 90, 46, 54, 110, 32, 125, 107, 42, 103, 104, 43, 108], [105, 98, 20, 28, 96, 25, 86, 82, 41, 117, 53, 123, 80, 94, 63, 116, 77, 75, 52, 118, 79, 126, 115, 16, 59, 56, 44, 8, 89, 111, 62, 119, 10, 50, 57, 17, 120, 48, 109, 112, 51, 71, 45, 127, 84, 18, 12, 13, 30, 38, 22, 23, 6, 91, 95, 31, 55, 83, 78, 81, 76, 88, 54, 27, 61, 60, 19, 39, 74, 93, 46, 33, 124, 24, 104, 106, 7, 49, 34, 37, 26, 114, 58, 4, 102, 43, 107, 122, 11, 99, 15, 9, 2, 125, 87, 97, 14, 85, 92, 36, 90, 29, 21, 110, 101, 72, 100, 32, 35, 121, 40, 42, 68, 103, 70, 113, 69, 108, 47, 3, 1, 0, 64, 66, 5, 73, 65, 67], [112, 105, 113, 125, 110, 41, 114, 89, 126, 104, 117, 54, 61, 27, 84, 127, 57, 59, 58, 56, 107, 123, 30, 91, 99, 48, 52, 51, 46, 111, 49, 108, 47, 55, 120, 92, 42, 115, 101, 119, 53, 94, 122, 43, 109, 23, 118, 44, 116, 62, 86, 25, 121, 124, 106, 50, 63, 60, 34, 102, 36, 45, 39, 37, 100, 40, 98, 33, 38, 35, 103, 90, 21, 97, 32, 15, 96, 28, 82, 95, 88, 31, 79, 18, 80, 29, 93, 81, 26, 87, 85, 17, 20, 22, 24, 83, 13, 12, 19, 78, 3, 14, 76, 69, 74, 75, 65, 77, 16, 11, 10, 9, 8, 71, 72, 73, 6, 5, 70, 7, 67, 68, 66, 1, 2, 4, 64, 0], [105, 112, 86, 30, 41, 84, 91, 80, 82, 78, 27, 76, 98, 99, 89, 94, 126, 113, 9, 10, 7, 110, 61, 114, 74, 34, 25, 52, 71, 35, 127, 14, 58, 125, 20, 49, 23, 109, 88, 67, 54, 69, 57, 59, 3, 123, 56, 117, 19, 47, 13, 46, 62, 5, 22, 111, 18, 108, 81, 48, 51, 118, 55, 77, 116, 17, 21, 85, 16, 107, 43, 120, 115, 119, 37, 79, 40, 32, 73, 50, 122, 92, 11, 60, 93, 24, 102, 83, 53, 124, 65, 63, 28, 121, 15, 106, 8, 4, 104, 97, 36, 44, 87, 64, 42, 45, 90, 12, 95, 38, 39, 29, 26, 103, 33, 75, 31, 101, 96, 72, 100, 2, 70, 66, 0, 1, 6, 68], [112, 105, 86, 91, 41, 30, 84, 25, 27, 58, 126, 89, 15, 62, 80, 113, 106, 82, 33, 111, 114, 110, 17, 127, 116, 104, 52, 124, 78, 94, 83, 117, 76, 49, 61, 46, 48, 39, 125, 59, 92, 57, 11, 56, 107, 10, 85, 47, 98, 36, 99, 54, 13, 51, 35, 123, 45, 7, 87, 60, 101, 23, 32, 119, 63, 108, 18, 22, 29, 6, 121, 120, 115, 44, 12, 109, 71, 55, 93, 21, 97, 40, 43, 96, 118, 20, 95, 72, 34, 16, 37, 31, 53, 122, 28, 42, 88, 50, 26, 38, 103, 90, 69, 2, 9, 79, 24, 102, 81, 100, 64, 14, 19, 67, 68, 75, 74, 5, 77, 8, 73, 4, 66, 1, 70, 65, 0, 3], [105, 112, 30, 84, 86, 89, 41, 98, 126, 82, 48, 113, 118, 123, 110, 25, 80, 35, 85, 45, 31, 91, 99, 52, 54, 94, 88, 61, 125, 78, 27, 58, 115, 9, 59, 127, 92, 116, 109, 117, 39, 114, 33, 17, 57, 12, 60, 32, 49, 121, 95, 76, 108, 6, 40, 73, 56, 111, 47, 74, 120, 18, 103, 51, 62, 26, 97, 79, 55, 53, 122, 75, 20, 106, 63, 19, 22, 107, 104, 93, 90, 119, 13, 42, 28, 124, 102, 46, 21, 3, 100, 101, 43, 50, 44, 83, 23, 37, 4, 36, 10, 87, 96, 72, 38, 24, 15, 14, 29, 34, 11, 16, 81, 77, 68, 1, 71, 69, 70, 67, 5, 8, 7, 65, 0, 66, 2, 64]], "model.layers.18.self_attn.k_proj": [[113, 52, 103, 110, 96, 93, 90, 87, 85, 18, 80, 112, 77, 49, 55, 44, 124, 74, 48, 119, 54, 79, 118, 53, 20, 70, 121, 57, 46, 62, 105, 61, 86, 56, 125, 51, 43, 34, 111, 116, 8, 126, 127, 109, 120, 115, 64, 123, 50, 47, 108, 117, 41, 58, 0, 68, 65, 59, 107, 102, 63, 45, 122, 42, 114, 9, 60, 40, 106, 104, 38, 19, 7, 83, 99, 37, 101, 2, 3, 30, 28, 76, 36, 98, 95, 1, 4, 92, 78, 81, 88, 97, 35, 33, 75, 25, 5, 11, 94, 100, 12, 24, 91, 89, 27, 73, 31, 66, 17, 69, 14, 72, 84, 16, 22, 15, 29, 67, 13, 71, 23, 21, 26, 82, 32, 6, 39, 10], [63, 124, 36, 22, 120, 15, 30, 17, 74, 103, 12, 56, 72, 18, 13, 54, 4, 32, 27, 24, 117, 62, 60, 51, 122, 64, 107, 46, 121, 2, 44, 58, 126, 21, 47, 118, 59, 109, 28, 61, 69, 57, 114, 45, 10, 39, 38, 119, 113, 79, 50, 53, 49, 55, 43, 48, 116, 40, 19, 125, 115, 108, 42, 123, 127, 106, 105, 110, 104, 8, 112, 52, 111, 41, 102, 1, 25, 100, 37, 97, 67, 26, 101, 34, 7, 76, 29, 33, 35, 96, 95, 98, 90, 99, 71, 6, 85, 93, 91, 78, 94, 73, 31, 75, 82, 92, 23, 84, 88, 80, 3, 20, 87, 11, 5, 81, 14, 9, 83, 86, 16, 0, 89, 66, 77, 65, 68, 70], [55, 118, 38, 123, 33, 86, 91, 24, 61, 62, 48, 15, 116, 53, 18, 95, 119, 126, 120, 122, 17, 112, 102, 115, 109, 77, 49, 51, 57, 84, 124, 117, 63, 114, 110, 21, 11, 45, 52, 90, 31, 46, 50, 74, 104, 19, 44, 113, 54, 58, 121, 76, 125, 111, 127, 60, 37, 59, 56, 105, 108, 7, 3, 16, 103, 30, 5, 107, 42, 23, 78, 35, 47, 8, 106, 43, 41, 1, 39, 94, 27, 40, 9, 100, 99, 36, 32, 6, 14, 93, 98, 64, 20, 89, 101, 29, 72, 82, 34, 87, 12, 28, 96, 88, 25, 75, 26, 92, 85, 79, 66, 71, 83, 13, 80, 22, 10, 69, 4, 73, 67, 81, 68, 97, 2, 70, 65, 0], [107, 60, 100, 32, 89, 54, 47, 87, 20, 80, 63, 82, 75, 21, 9, 43, 53, 79, 126, 13, 69, 71, 1, 117, 111, 118, 29, 51, 64, 121, 61, 2, 57, 67, 52, 0, 50, 62, 114, 102, 115, 127, 124, 92, 48, 122, 125, 49, 56, 34, 103, 7, 41, 40, 77, 105, 119, 46, 108, 58, 45, 110, 120, 116, 94, 123, 42, 112, 37, 86, 113, 39, 44, 30, 3, 27, 109, 59, 38, 101, 81, 55, 104, 91, 19, 28, 31, 90, 106, 11, 22, 26, 78, 99, 85, 33, 35, 98, 72, 83, 95, 93, 4, 14, 70, 68, 8, 74, 97, 17, 6, 76, 15, 10, 66, 88, 25, 24, 12, 5, 18, 16, 23, 73, 65, 36, 96, 84], [40, 120, 59, 34, 22, 113, 29, 49, 90, 14, 80, 119, 31, 51, 9, 88, 56, 112, 57, 18, 76, 52, 6, 83, 118, 123, 110, 117, 68, 124, 109, 61, 116, 50, 58, 62, 43, 60, 98, 115, 54, 122, 55, 114, 102, 72, 53, 46, 127, 95, 125, 45, 65, 101, 0, 108, 126, 121, 63, 47, 28, 105, 111, 48, 99, 107, 82, 91, 81, 44, 16, 39, 106, 42, 79, 41, 103, 36, 89, 30, 2, 21, 66, 23, 38, 37, 32, 27, 11, 96, 75, 13, 17, 97, 35, 86, 100, 67, 71, 33, 92, 94, 74, 24, 85, 69, 3, 25, 73, 84, 5, 20, 78, 15, 104, 64, 87, 26, 10, 93, 77, 7, 70, 12, 1, 8, 4, 19], [122, 54, 127, 36, 97, 120, 125, 124, 86, 53, 62, 119, 126, 57, 118, 104, 121, 49, 58, 56, 117, 113, 48, 112, 50, 51, 52, 123, 59, 42, 26, 60, 114, 115, 55, 43, 116, 61, 63, 39, 47, 105, 106, 41, 111, 46, 94, 81, 110, 93, 28, 109, 38, 29, 107, 40, 19, 45, 100, 108, 24, 44, 30, 15, 83, 73, 103, 69, 27, 33, 67, 21, 82, 99, 102, 75, 87, 101, 76, 4, 25, 6, 8, 23, 88, 65, 34, 37, 91, 85, 98, 32, 78, 16, 64, 31, 35, 90, 12, 92, 20, 13, 0, 17, 95, 96, 10, 89, 66, 71, 84, 14, 22, 80, 79, 77, 18, 7, 2, 11, 72, 74, 1, 5, 70, 9, 68, 3], [41, 53, 34, 25, 63, 30, 52, 79, 86, 80, 117, 126, 20, 75, 32, 8, 48, 10, 77, 51, 82, 108, 116, 47, 45, 59, 123, 109, 118, 6, 4, 2, 56, 50, 64, 120, 119, 62, 1, 105, 113, 9, 40, 57, 71, 55, 111, 106, 61, 83, 28, 87, 18, 54, 14, 29, 114, 31, 67, 69, 70, 110, 127, 103, 23, 92, 44, 46, 58, 35, 36, 124, 74, 21, 0, 94, 122, 102, 37, 66, 115, 90, 121, 3, 33, 81, 107, 88, 24, 93, 39, 49, 112, 101, 97, 100, 125, 78, 60, 104, 91, 12, 38, 13, 5, 85, 42, 22, 43, 17, 73, 27, 26, 76, 19, 98, 15, 11, 84, 95, 96, 72, 99, 65, 7, 68, 16, 89], [41, 112, 48, 94, 86, 113, 35, 27, 82, 110, 80, 89, 126, 84, 78, 49, 105, 61, 52, 125, 58, 59, 76, 9, 114, 115, 111, 123, 54, 50, 34, 57, 127, 117, 51, 91, 109, 60, 108, 62, 98, 10, 45, 7, 116, 47, 118, 56, 55, 5, 63, 119, 120, 53, 96, 46, 121, 124, 107, 1, 81, 37, 43, 40, 36, 44, 88, 122, 102, 101, 16, 39, 106, 79, 12, 100, 30, 67, 97, 104, 21, 83, 42, 38, 31, 103, 32, 33, 13, 93, 29, 95, 8, 87, 99, 24, 26, 28, 17, 69, 23, 75, 15, 11, 14, 64, 90, 92, 74, 19, 20, 6, 18, 85, 71, 72, 25, 77, 68, 66, 22, 70, 73, 4, 2, 3, 65, 0]], "model.layers.18.self_attn.qk_proj": [[113, 112, 63, 124, 55, 118, 41, 54, 122, 52, 120, 59, 60, 105, 107, 127, 53, 43, 123, 36, 86, 48, 22, 102, 110, 117, 126, 25, 62, 49, 30, 84, 100, 89, 29, 51, 119, 82, 40, 18, 20, 16, 80, 96, 38, 57, 116, 15, 61, 115, 56, 46, 79, 109, 77, 34, 125, 94, 47, 23, 58, 50, 98, 27, 26, 114, 24, 45, 104, 39, 103, 121, 111, 85, 32, 21, 92, 88, 87, 93, 13, 42, 91, 8, 10, 75, 11, 90, 81, 72, 76, 74, 19, 31, 44, 97, 9, 17, 33, 95, 37, 14, 35, 108, 12, 83, 106, 28, 78, 73, 7, 71, 6, 101, 0, 99, 64, 69, 5, 2, 70, 65, 67, 68, 1, 3, 4, 66], [113, 112, 63, 124, 55, 118, 54, 41, 52, 120, 122, 59, 60, 105, 107, 127, 43, 53, 123, 86, 102, 36, 22, 117, 110, 48, 49, 25, 126, 100, 89, 30, 29, 84, 62, 18, 116, 82, 16, 15, 40, 119, 94, 46, 20, 57, 96, 56, 51, 79, 47, 61, 38, 80, 77, 104, 58, 125, 103, 50, 39, 34, 26, 27, 23, 24, 115, 87, 109, 98, 114, 85, 21, 88, 92, 32, 13, 42, 8, 10, 93, 45, 121, 90, 111, 81, 97, 74, 75, 11, 91, 35, 28, 83, 44, 9, 12, 76, 19, 95, 31, 106, 108, 72, 17, 6, 69, 37, 78, 14, 33, 99, 0, 101, 71, 64, 7, 67, 73, 65, 66, 68, 1, 4, 70, 2, 5, 3], [113, 112, 63, 124, 55, 118, 54, 41, 122, 52, 120, 59, 105, 60, 107, 127, 43, 53, 123, 22, 86, 36, 48, 102, 126, 110, 25, 117, 49, 30, 29, 100, 89, 84, 51, 94, 62, 56, 82, 40, 104, 116, 47, 18, 20, 119, 79, 109, 38, 96, 58, 39, 57, 16, 27, 125, 15, 80, 26, 61, 103, 114, 23, 98, 115, 46, 50, 34, 45, 24, 85, 87, 77, 13, 88, 121, 90, 91, 8, 32, 21, 42, 92, 111, 75, 93, 11, 10, 19, 6, 33, 81, 106, 44, 64, 28, 31, 97, 108, 83, 35, 12, 95, 76, 72, 9, 14, 74, 0, 37, 2, 101, 7, 69, 65, 17, 78, 67, 73, 3, 1, 66, 5, 4, 68, 71, 70, 99], [113, 112, 63, 124, 55, 54, 118, 41, 52, 120, 122, 59, 60, 105, 107, 127, 53, 43, 123, 36, 117, 48, 22, 102, 126, 110, 49, 86, 25, 62, 51, 116, 89, 47, 82, 58, 100, 56, 30, 125, 119, 96, 46, 29, 20, 61, 18, 15, 94, 79, 57, 84, 39, 16, 115, 40, 103, 80, 38, 114, 50, 104, 109, 34, 27, 26, 24, 13, 23, 98, 75, 111, 77, 45, 121, 87, 85, 21, 93, 32, 97, 106, 8, 91, 90, 42, 92, 10, 88, 44, 76, 74, 81, 35, 11, 0, 6, 108, 83, 37, 72, 95, 17, 64, 31, 28, 3, 12, 14, 66, 2, 78, 1, 73, 19, 33, 9, 7, 70, 71, 4, 69, 101, 5, 68, 65, 99, 67], [113, 112, 63, 124, 55, 54, 118, 41, 52, 122, 120, 60, 59, 105, 107, 127, 53, 43, 123, 48, 22, 110, 117, 36, 86, 126, 25, 62, 49, 116, 102, 125, 82, 51, 89, 30, 20, 56, 47, 57, 84, 16, 79, 100, 18, 119, 109, 29, 58, 61, 15, 40, 94, 115, 38, 96, 104, 98, 111, 46, 80, 26, 103, 50, 23, 114, 34, 13, 39, 77, 45, 121, 21, 8, 27, 24, 88, 85, 106, 10, 93, 75, 90, 32, 87, 44, 42, 81, 92, 12, 97, 91, 11, 108, 19, 28, 74, 76, 78, 72, 14, 33, 31, 17, 83, 95, 73, 35, 37, 6, 101, 70, 9, 0, 7, 66, 64, 3, 69, 65, 2, 4, 71, 68, 5, 1, 99, 67], [113, 63, 112, 124, 55, 118, 54, 41, 52, 122, 120, 60, 59, 105, 107, 127, 43, 53, 123, 22, 48, 86, 110, 117, 36, 126, 102, 25, 51, 49, 15, 89, 30, 82, 18, 62, 116, 79, 20, 100, 57, 96, 40, 80, 125, 16, 84, 109, 119, 47, 29, 58, 50, 39, 115, 56, 34, 94, 98, 103, 61, 104, 46, 23, 114, 77, 38, 13, 24, 21, 26, 27, 111, 85, 88, 8, 32, 87, 42, 93, 75, 10, 11, 97, 90, 45, 92, 81, 78, 12, 83, 74, 31, 17, 121, 28, 76, 106, 91, 44, 19, 72, 95, 14, 9, 35, 33, 108, 71, 37, 73, 101, 99, 70, 64, 0, 6, 69, 7, 2, 1, 66, 65, 5, 67, 4, 68, 3], [113, 63, 112, 124, 55, 118, 41, 54, 122, 52, 120, 60, 59, 105, 107, 43, 127, 53, 123, 22, 86, 48, 36, 126, 49, 25, 110, 117, 30, 102, 82, 18, 62, 89, 96, 84, 15, 29, 51, 47, 16, 100, 80, 20, 79, 40, 58, 56, 57, 119, 94, 77, 104, 23, 98, 34, 38, 115, 27, 61, 116, 125, 26, 32, 109, 24, 88, 50, 46, 114, 103, 13, 85, 91, 39, 21, 75, 93, 111, 87, 92, 121, 8, 10, 74, 97, 11, 90, 78, 31, 12, 81, 35, 33, 95, 76, 42, 44, 19, 17, 14, 101, 45, 106, 83, 9, 108, 73, 72, 28, 99, 70, 71, 37, 64, 7, 5, 65, 69, 2, 66, 0, 1, 68, 67, 6, 4, 3], [113, 63, 112, 124, 55, 54, 118, 41, 52, 120, 122, 59, 60, 105, 107, 127, 43, 22, 53, 123, 36, 86, 102, 126, 48, 117, 25, 30, 110, 49, 119, 62, 29, 84, 89, 82, 16, 20, 56, 18, 96, 100, 40, 47, 80, 51, 94, 57, 79, 15, 98, 116, 38, 125, 46, 23, 26, 61, 27, 104, 50, 115, 24, 58, 39, 85, 88, 34, 91, 13, 32, 114, 109, 77, 21, 87, 103, 121, 93, 90, 111, 92, 33, 75, 19, 31, 17, 97, 10, 42, 95, 11, 81, 8, 45, 44, 14, 108, 78, 9, 83, 76, 106, 12, 70, 35, 74, 72, 28, 99, 37, 73, 64, 101, 5, 71, 7, 1, 0, 68, 2, 69, 3, 4, 66, 65, 67, 6], [113, 63, 112, 124, 55, 118, 54, 41, 52, 120, 122, 60, 59, 105, 107, 127, 53, 43, 22, 123, 36, 102, 110, 86, 126, 48, 117, 25, 89, 49, 30, 84, 57, 119, 96, 29, 20, 18, 79, 82, 40, 38, 62, 94, 116, 16, 100, 56, 47, 46, 125, 80, 15, 51, 115, 27, 58, 61, 104, 24, 98, 39, 34, 88, 26, 32, 87, 13, 114, 21, 23, 103, 77, 85, 93, 50, 92, 90, 97, 11, 109, 111, 75, 28, 106, 108, 81, 91, 72, 10, 19, 121, 95, 17, 31, 12, 42, 35, 74, 33, 76, 45, 8, 44, 83, 78, 70, 14, 101, 37, 99, 73, 9, 7, 5, 71, 64, 68, 0, 69, 65, 3, 4, 67, 1, 2, 66, 6], [113, 112, 63, 124, 55, 54, 118, 41, 52, 120, 122, 60, 59, 105, 107, 127, 123, 53, 43, 48, 102, 117, 36, 22, 86, 126, 25, 110, 62, 49, 57, 100, 116, 125, 18, 51, 82, 30, 47, 89, 40, 119, 20, 56, 29, 58, 84, 79, 61, 80, 38, 16, 96, 15, 46, 27, 104, 26, 94, 34, 115, 39, 98, 50, 24, 85, 103, 77, 21, 32, 88, 72, 13, 109, 75, 23, 10, 93, 45, 87, 106, 111, 121, 92, 114, 97, 31, 74, 90, 76, 81, 108, 35, 95, 19, 91, 44, 28, 11, 8, 42, 83, 78, 12, 14, 17, 9, 70, 71, 37, 73, 7, 101, 64, 33, 5, 99, 67, 68, 3, 0, 65, 4, 69, 6, 2, 1, 66], [113, 112, 63, 124, 55, 54, 41, 118, 120, 122, 52, 60, 59, 105, 107, 127, 53, 43, 123, 117, 48, 22, 126, 36, 102, 86, 110, 116, 62, 25, 51, 18, 49, 56, 82, 89, 20, 100, 61, 79, 96, 47, 57, 30, 80, 125, 15, 84, 40, 119, 16, 46, 58, 38, 29, 104, 77, 50, 98, 115, 39, 34, 72, 94, 13, 23, 109, 24, 114, 103, 27, 26, 45, 75, 85, 87, 32, 111, 93, 121, 21, 10, 92, 11, 88, 90, 8, 28, 81, 106, 76, 12, 74, 17, 97, 14, 83, 78, 6, 35, 91, 42, 73, 44, 64, 71, 37, 31, 2, 108, 19, 9, 69, 0, 95, 7, 33, 101, 68, 70, 1, 65, 5, 4, 67, 66, 3, 99], [113, 63, 112, 124, 55, 41, 118, 54, 52, 120, 122, 60, 59, 105, 107, 127, 43, 53, 123, 22, 86, 48, 102, 36, 117, 126, 25, 89, 84, 18, 110, 82, 30, 100, 49, 96, 80, 15, 20, 79, 29, 62, 51, 40, 56, 16, 116, 98, 47, 46, 94, 57, 115, 119, 77, 85, 23, 38, 58, 125, 104, 61, 34, 13, 24, 26, 72, 27, 75, 87, 103, 39, 109, 21, 32, 50, 93, 11, 88, 111, 74, 10, 90, 114, 97, 92, 19, 81, 45, 121, 31, 91, 17, 78, 14, 95, 12, 106, 76, 6, 44, 33, 108, 8, 9, 35, 83, 42, 73, 28, 99, 101, 71, 7, 37, 64, 68, 5, 2, 70, 1, 66, 3, 69, 0, 4, 67, 65], [113, 63, 112, 124, 55, 118, 54, 41, 52, 122, 120, 60, 59, 105, 107, 127, 43, 123, 53, 36, 48, 102, 86, 22, 117, 25, 110, 126, 30, 100, 62, 29, 96, 49, 89, 38, 18, 82, 40, 51, 20, 94, 84, 47, 80, 79, 58, 61, 16, 15, 56, 125, 116, 115, 57, 119, 98, 46, 23, 24, 39, 27, 88, 26, 104, 21, 85, 34, 32, 103, 50, 87, 90, 97, 109, 121, 13, 93, 77, 111, 91, 31, 72, 92, 75, 19, 45, 114, 10, 95, 11, 106, 35, 17, 83, 44, 28, 42, 81, 33, 74, 78, 14, 76, 12, 37, 73, 108, 6, 101, 99, 8, 9, 7, 71, 69, 64, 66, 4, 1, 65, 5, 0, 67, 2, 68, 70, 3], [113, 63, 112, 124, 55, 118, 54, 41, 52, 122, 120, 59, 60, 107, 105, 127, 43, 53, 123, 22, 86, 48, 62, 117, 36, 102, 110, 126, 49, 25, 89, 100, 30, 82, 61, 51, 15, 18, 125, 96, 47, 80, 29, 20, 119, 58, 40, 115, 98, 50, 94, 79, 57, 38, 84, 116, 46, 56, 16, 39, 24, 34, 104, 121, 27, 21, 23, 85, 72, 13, 103, 77, 114, 88, 26, 75, 10, 32, 109, 111, 87, 93, 74, 97, 92, 11, 12, 35, 90, 106, 45, 33, 83, 91, 28, 31, 81, 19, 8, 95, 6, 42, 17, 0, 73, 76, 14, 7, 108, 44, 78, 65, 71, 101, 69, 64, 9, 67, 99, 37, 5, 4, 3, 66, 1, 68, 2, 70], [113, 63, 112, 124, 55, 41, 54, 118, 52, 120, 122, 59, 60, 105, 107, 43, 127, 53, 123, 22, 48, 117, 102, 86, 36, 110, 126, 25, 100, 62, 89, 51, 82, 15, 30, 40, 80, 20, 96, 18, 61, 79, 49, 115, 84, 125, 29, 47, 56, 57, 94, 46, 23, 16, 116, 98, 50, 34, 58, 38, 85, 72, 39, 13, 27, 119, 77, 26, 21, 104, 103, 45, 88, 111, 93, 32, 24, 10, 11, 75, 109, 74, 121, 87, 8, 114, 83, 12, 91, 81, 97, 90, 28, 17, 31, 106, 76, 44, 92, 33, 78, 95, 19, 35, 42, 6, 73, 14, 7, 101, 64, 69, 9, 108, 37, 4, 71, 0, 66, 3, 1, 68, 67, 65, 70, 2, 99, 5], [113, 112, 63, 124, 55, 118, 54, 41, 52, 120, 122, 59, 60, 105, 107, 127, 53, 43, 123, 22, 117, 48, 86, 36, 102, 110, 62, 126, 100, 25, 89, 82, 116, 96, 30, 15, 49, 20, 56, 51, 46, 84, 18, 40, 79, 80, 16, 47, 61, 57, 125, 94, 29, 115, 98, 77, 119, 23, 50, 58, 38, 39, 85, 34, 24, 13, 111, 72, 103, 121, 104, 114, 75, 27, 10, 26, 11, 88, 109, 32, 87, 74, 42, 45, 97, 106, 8, 21, 81, 93, 92, 12, 76, 90, 19, 95, 83, 91, 44, 108, 17, 31, 14, 9, 78, 6, 35, 33, 28, 101, 73, 7, 37, 4, 70, 64, 0, 5, 2, 71, 1, 65, 99, 67, 69, 68, 66, 3], [113, 63, 112, 124, 55, 118, 41, 52, 54, 120, 122, 59, 60, 105, 107, 127, 53, 43, 123, 48, 86, 36, 22, 126, 102, 117, 25, 30, 82, 110, 62, 49, 89, 18, 96, 80, 29, 20, 40, 47, 94, 84, 100, 16, 15, 104, 46, 57, 125, 61, 38, 79, 51, 98, 115, 39, 119, 56, 116, 23, 26, 58, 24, 27, 103, 34, 50, 21, 111, 85, 13, 109, 32, 88, 77, 97, 114, 87, 121, 91, 93, 75, 92, 42, 90, 74, 8, 19, 11, 31, 106, 10, 95, 76, 28, 108, 44, 17, 33, 72, 81, 35, 12, 45, 83, 78, 37, 14, 70, 101, 73, 69, 9, 64, 71, 65, 2, 99, 7, 6, 0, 4, 68, 1, 66, 67, 3, 5], [113, 63, 112, 124, 55, 54, 118, 41, 52, 120, 122, 60, 105, 59, 107, 43, 123, 127, 53, 48, 36, 86, 102, 22, 117, 126, 25, 89, 49, 110, 30, 96, 29, 94, 62, 40, 18, 82, 20, 84, 116, 56, 100, 47, 80, 38, 15, 79, 104, 16, 119, 46, 58, 57, 23, 115, 27, 77, 125, 98, 39, 34, 24, 61, 26, 51, 85, 21, 13, 88, 32, 111, 103, 50, 114, 11, 87, 74, 90, 92, 121, 109, 8, 97, 93, 91, 75, 10, 95, 19, 81, 106, 72, 12, 70, 31, 14, 42, 28, 35, 33, 108, 83, 76, 78, 44, 17, 71, 0, 45, 9, 1, 69, 3, 73, 64, 37, 68, 7, 66, 101, 65, 99, 6, 4, 2, 5, 67], [113, 112, 63, 124, 55, 54, 118, 41, 52, 120, 122, 59, 60, 105, 107, 53, 127, 43, 123, 36, 48, 117, 22, 102, 86, 126, 62, 116, 56, 110, 25, 57, 47, 100, 125, 30, 49, 46, 115, 89, 20, 18, 96, 40, 38, 82, 94, 61, 51, 84, 80, 29, 119, 34, 15, 104, 79, 16, 114, 58, 121, 39, 98, 32, 23, 27, 77, 50, 103, 111, 8, 88, 21, 24, 109, 74, 87, 26, 106, 85, 13, 45, 91, 11, 92, 90, 97, 95, 93, 10, 75, 35, 31, 42, 17, 76, 37, 19, 81, 72, 108, 28, 44, 83, 70, 12, 33, 14, 9, 99, 71, 0, 64, 101, 73, 69, 1, 78, 3, 4, 7, 65, 2, 66, 67, 68, 5, 6], [113, 112, 63, 124, 55, 54, 118, 52, 41, 120, 122, 59, 60, 105, 107, 127, 53, 43, 48, 123, 117, 102, 36, 22, 86, 126, 110, 56, 62, 100, 25, 30, 49, 57, 116, 47, 15, 96, 119, 51, 125, 89, 61, 18, 94, 29, 84, 46, 115, 39, 20, 40, 82, 79, 16, 38, 80, 23, 98, 34, 27, 13, 103, 21, 58, 104, 26, 50, 8, 24, 109, 121, 88, 87, 114, 32, 77, 45, 111, 97, 85, 93, 92, 74, 11, 10, 95, 90, 91, 31, 108, 28, 76, 75, 106, 33, 83, 42, 12, 35, 19, 17, 81, 78, 72, 44, 37, 14, 70, 9, 64, 73, 65, 71, 5, 101, 0, 4, 7, 99, 2, 69, 1, 67, 68, 3, 66, 6], [113, 112, 63, 124, 55, 54, 118, 52, 41, 120, 122, 59, 60, 105, 107, 127, 43, 53, 123, 117, 36, 48, 22, 102, 86, 126, 25, 62, 89, 96, 110, 51, 56, 57, 30, 119, 100, 116, 47, 49, 84, 94, 38, 20, 79, 61, 15, 29, 98, 18, 115, 80, 40, 82, 125, 46, 23, 34, 58, 16, 121, 26, 104, 111, 50, 77, 27, 24, 39, 88, 103, 13, 85, 32, 8, 97, 21, 114, 109, 75, 87, 93, 92, 91, 90, 19, 10, 106, 11, 95, 78, 81, 76, 31, 74, 17, 108, 12, 35, 83, 42, 28, 33, 72, 44, 45, 14, 37, 70, 71, 73, 9, 101, 99, 64, 65, 7, 0, 1, 3, 67, 4, 6, 5, 68, 66, 69, 2], [113, 112, 63, 124, 55, 54, 118, 52, 41, 120, 122, 59, 60, 105, 107, 127, 43, 53, 123, 117, 48, 36, 126, 86, 62, 22, 25, 102, 116, 110, 56, 89, 30, 61, 51, 57, 100, 125, 47, 49, 119, 94, 38, 84, 96, 20, 82, 40, 80, 79, 115, 15, 29, 18, 98, 46, 58, 16, 104, 34, 50, 103, 121, 27, 23, 26, 77, 111, 88, 32, 24, 85, 114, 39, 21, 13, 90, 91, 8, 93, 87, 11, 92, 109, 106, 75, 97, 74, 19, 95, 28, 10, 35, 31, 17, 83, 42, 108, 44, 12, 72, 76, 45, 0, 81, 14, 64, 33, 6, 78, 9, 71, 70, 37, 2, 7, 73, 99, 1, 101, 65, 69, 68, 67, 5, 3, 66, 4], [113, 63, 112, 124, 55, 54, 118, 52, 41, 120, 122, 59, 60, 105, 107, 127, 43, 123, 53, 48, 117, 36, 102, 22, 86, 25, 126, 110, 62, 57, 30, 116, 51, 119, 100, 40, 96, 47, 61, 84, 89, 38, 56, 29, 18, 94, 20, 82, 15, 16, 49, 46, 125, 79, 98, 23, 104, 50, 34, 80, 26, 24, 115, 58, 103, 111, 39, 27, 121, 87, 77, 32, 85, 88, 21, 109, 93, 114, 97, 35, 13, 31, 28, 45, 90, 106, 92, 91, 8, 42, 83, 75, 74, 108, 95, 19, 44, 11, 81, 76, 10, 6, 33, 12, 72, 17, 78, 9, 14, 101, 37, 0, 64, 7, 65, 71, 73, 1, 99, 5, 4, 69, 66, 2, 67, 3, 70, 68], [113, 63, 112, 124, 55, 118, 54, 52, 41, 120, 122, 59, 60, 105, 107, 127, 43, 53, 123, 48, 22, 36, 117, 86, 102, 126, 116, 25, 62, 110, 96, 51, 30, 100, 57, 18, 119, 16, 89, 61, 49, 47, 15, 94, 84, 82, 29, 20, 79, 40, 38, 125, 34, 98, 39, 46, 23, 104, 26, 50, 56, 80, 103, 77, 115, 58, 24, 27, 121, 13, 32, 21, 85, 109, 8, 74, 88, 92, 114, 90, 75, 111, 93, 97, 87, 72, 10, 91, 95, 81, 11, 19, 45, 106, 76, 83, 28, 42, 12, 6, 17, 31, 35, 14, 78, 108, 44, 7, 73, 37, 9, 33, 71, 99, 101, 0, 4, 5, 68, 67, 69, 1, 3, 64, 66, 65, 70, 2], [113, 112, 63, 124, 55, 118, 54, 41, 52, 122, 120, 59, 60, 105, 107, 127, 43, 123, 53, 22, 117, 86, 48, 36, 102, 25, 62, 126, 89, 116, 30, 110, 20, 16, 51, 84, 18, 115, 82, 61, 100, 96, 40, 119, 79, 57, 47, 15, 49, 38, 80, 29, 94, 23, 98, 46, 109, 58, 125, 34, 56, 50, 104, 26, 88, 103, 27, 114, 77, 13, 111, 85, 32, 75, 21, 121, 39, 24, 10, 87, 90, 42, 72, 11, 93, 31, 8, 92, 97, 74, 28, 91, 76, 45, 81, 19, 83, 95, 6, 106, 35, 17, 78, 12, 108, 33, 44, 14, 73, 101, 9, 64, 7, 99, 71, 4, 65, 0, 37, 69, 66, 2, 5, 3, 1, 68, 67, 70], [113, 63, 112, 124, 55, 118, 54, 41, 52, 120, 122, 59, 60, 105, 107, 127, 43, 123, 53, 48, 86, 36, 22, 117, 102, 62, 126, 25, 30, 110, 100, 89, 116, 29, 51, 96, 94, 20, 119, 40, 61, 56, 57, 18, 16, 82, 47, 84, 15, 38, 49, 125, 46, 109, 115, 23, 98, 58, 80, 50, 104, 24, 79, 77, 26, 34, 121, 85, 103, 27, 21, 13, 88, 92, 39, 87, 11, 93, 111, 32, 114, 72, 91, 74, 97, 42, 90, 75, 10, 31, 95, 81, 19, 45, 106, 33, 8, 12, 28, 76, 108, 17, 83, 44, 78, 35, 73, 14, 6, 9, 37, 0, 101, 7, 99, 65, 69, 2, 3, 71, 64, 67, 5, 68, 70, 66, 4, 1], [113, 112, 63, 124, 55, 118, 54, 41, 52, 120, 122, 60, 59, 105, 107, 127, 43, 123, 53, 36, 117, 48, 102, 62, 22, 110, 126, 86, 25, 61, 116, 89, 49, 30, 18, 100, 51, 56, 96, 40, 119, 57, 84, 47, 29, 125, 20, 121, 23, 58, 39, 16, 38, 98, 104, 94, 82, 46, 50, 115, 79, 34, 15, 80, 109, 114, 26, 111, 77, 103, 27, 24, 13, 88, 32, 85, 97, 21, 42, 72, 87, 93, 92, 90, 75, 74, 91, 106, 35, 11, 45, 108, 10, 44, 19, 17, 12, 31, 81, 83, 76, 28, 95, 37, 33, 101, 8, 78, 14, 73, 99, 6, 9, 0, 7, 70, 64, 69, 5, 71, 4, 68, 67, 66, 3, 2, 1, 65], [113, 112, 63, 124, 55, 54, 118, 41, 120, 52, 122, 60, 59, 105, 107, 127, 43, 53, 123, 22, 48, 36, 117, 126, 102, 86, 110, 62, 25, 61, 20, 89, 18, 47, 100, 82, 30, 51, 125, 96, 49, 56, 116, 79, 40, 80, 15, 84, 16, 119, 29, 46, 57, 23, 58, 38, 94, 50, 104, 121, 88, 77, 34, 115, 27, 26, 103, 72, 13, 39, 85, 111, 24, 114, 32, 21, 93, 92, 98, 11, 109, 74, 75, 91, 90, 45, 97, 87, 10, 19, 12, 28, 106, 17, 76, 81, 42, 8, 78, 31, 35, 95, 70, 83, 7, 9, 44, 0, 73, 14, 108, 69, 99, 3, 71, 65, 37, 64, 101, 4, 33, 5, 2, 1, 68, 6, 67, 66], [113, 63, 112, 124, 55, 118, 41, 54, 120, 52, 122, 60, 59, 105, 107, 127, 123, 53, 43, 48, 117, 36, 62, 22, 86, 110, 51, 102, 126, 119, 49, 30, 25, 125, 100, 96, 89, 47, 61, 40, 18, 115, 57, 56, 20, 15, 29, 84, 46, 116, 16, 79, 82, 94, 39, 34, 23, 104, 58, 38, 50, 103, 77, 121, 80, 26, 98, 72, 109, 45, 85, 32, 27, 92, 13, 93, 111, 21, 114, 106, 74, 10, 42, 90, 11, 24, 75, 88, 97, 87, 81, 108, 95, 35, 76, 70, 91, 28, 78, 31, 44, 12, 8, 19, 17, 33, 9, 5, 0, 64, 83, 14, 37, 67, 7, 65, 1, 73, 71, 101, 66, 69, 3, 4, 2, 99, 68, 6], [113, 112, 63, 124, 55, 118, 54, 41, 52, 120, 122, 60, 59, 105, 107, 127, 53, 43, 123, 48, 117, 22, 36, 86, 110, 102, 126, 62, 51, 119, 25, 49, 89, 18, 30, 100, 82, 79, 56, 46, 125, 20, 84, 116, 115, 15, 40, 29, 61, 96, 94, 16, 47, 80, 23, 109, 77, 98, 57, 39, 38, 34, 103, 24, 58, 13, 114, 50, 72, 121, 32, 42, 21, 88, 85, 104, 45, 111, 92, 27, 93, 87, 11, 76, 10, 97, 75, 74, 26, 106, 17, 90, 108, 91, 81, 14, 19, 8, 44, 95, 70, 83, 31, 28, 35, 12, 78, 9, 33, 7, 73, 71, 5, 101, 64, 67, 37, 69, 4, 99, 65, 3, 2, 0, 1, 68, 6, 66], [113, 112, 63, 124, 55, 118, 54, 41, 52, 120, 122, 59, 60, 105, 107, 127, 53, 43, 123, 48, 117, 102, 86, 36, 62, 22, 51, 110, 126, 25, 116, 30, 89, 100, 49, 125, 82, 96, 29, 61, 47, 16, 40, 20, 115, 119, 94, 39, 79, 18, 15, 84, 23, 58, 121, 56, 57, 109, 77, 50, 26, 34, 38, 104, 80, 98, 46, 114, 103, 27, 72, 85, 74, 88, 10, 24, 21, 87, 42, 32, 92, 111, 45, 97, 93, 75, 13, 76, 91, 8, 11, 90, 70, 14, 81, 31, 7, 19, 83, 12, 33, 5, 78, 95, 35, 17, 28, 106, 73, 108, 71, 44, 9, 0, 64, 69, 3, 4, 1, 65, 68, 67, 101, 66, 2, 37, 6, 99], [113, 112, 63, 124, 55, 118, 54, 41, 52, 122, 120, 60, 59, 105, 107, 127, 43, 53, 123, 86, 36, 102, 48, 22, 117, 25, 126, 62, 110, 100, 82, 29, 30, 51, 15, 49, 84, 89, 18, 16, 96, 61, 79, 116, 20, 47, 40, 119, 80, 94, 125, 38, 34, 56, 77, 57, 26, 98, 115, 23, 39, 27, 109, 46, 50, 88, 13, 24, 114, 58, 85, 21, 103, 92, 121, 32, 10, 72, 104, 74, 75, 42, 111, 45, 91, 97, 76, 87, 93, 11, 8, 95, 90, 106, 81, 83, 14, 19, 12, 78, 31, 33, 44, 28, 17, 73, 35, 108, 70, 9, 7, 71, 101, 37, 69, 99, 5, 67, 64, 6, 68, 4, 65, 66, 0, 1, 3, 2]], "model.layers.19.self_attn.q_proj": [[124, 110, 63, 115, 58, 119, 101, 120, 57, 46, 112, 52, 60, 123, 127, 109, 53, 59, 56, 116, 54, 88, 126, 50, 108, 61, 122, 55, 94, 111, 121, 114, 104, 51, 37, 47, 62, 97, 48, 40, 44, 49, 113, 117, 125, 45, 27, 19, 24, 118, 90, 34, 41, 105, 42, 81, 98, 106, 43, 107, 96, 31, 22, 39, 89, 102, 26, 16, 92, 9, 100, 38, 103, 36, 35, 78, 91, 99, 93, 25, 18, 30, 80, 29, 32, 73, 17, 20, 95, 23, 67, 85, 15, 28, 21, 69, 77, 84, 33, 12, 6, 14, 87, 83, 86, 10, 70, 79, 75, 13, 4, 1, 82, 68, 72, 0, 74, 5, 11, 64, 3, 65, 66, 8, 7, 76, 71, 2], [63, 57, 46, 101, 123, 112, 59, 110, 124, 120, 47, 97, 53, 121, 119, 81, 109, 114, 90, 24, 94, 111, 34, 105, 60, 19, 88, 87, 122, 126, 31, 85, 55, 104, 27, 91, 117, 58, 45, 52, 61, 25, 56, 106, 20, 39, 22, 116, 32, 28, 78, 42, 16, 38, 10, 107, 62, 44, 41, 54, 83, 127, 113, 50, 108, 14, 102, 100, 17, 80, 37, 49, 35, 51, 118, 40, 26, 12, 103, 36, 95, 96, 125, 43, 115, 89, 21, 48, 84, 98, 99, 93, 74, 30, 23, 29, 86, 92, 15, 11, 76, 77, 75, 6, 67, 7, 72, 18, 70, 13, 9, 73, 82, 68, 33, 79, 8, 69, 66, 1, 71, 4, 0, 2, 3, 64, 65, 5], [63, 124, 110, 57, 115, 101, 58, 46, 50, 122, 97, 88, 52, 60, 27, 120, 49, 55, 116, 81, 56, 59, 112, 123, 94, 119, 42, 121, 109, 111, 54, 126, 24, 125, 105, 127, 62, 117, 114, 44, 51, 93, 113, 37, 53, 84, 61, 90, 47, 31, 104, 20, 85, 108, 45, 6, 103, 118, 75, 73, 9, 41, 40, 67, 106, 48, 65, 43, 36, 38, 16, 4, 39, 72, 22, 14, 107, 0, 17, 100, 68, 12, 83, 80, 69, 30, 102, 87, 71, 91, 95, 7, 26, 15, 99, 19, 96, 11, 2, 35, 10, 74, 25, 3, 29, 34, 66, 77, 64, 32, 28, 92, 98, 13, 23, 21, 82, 70, 1, 8, 89, 18, 5, 79, 86, 76, 78, 33], [124, 63, 110, 101, 54, 46, 122, 119, 58, 59, 28, 112, 61, 55, 62, 120, 94, 56, 114, 111, 57, 90, 50, 53, 113, 116, 126, 123, 127, 121, 60, 125, 118, 97, 82, 22, 117, 48, 37, 47, 52, 115, 18, 49, 108, 109, 51, 41, 45, 44, 43, 39, 107, 32, 19, 106, 86, 102, 100, 42, 27, 92, 36, 30, 104, 40, 105, 25, 38, 103, 99, 34, 14, 35, 98, 96, 78, 20, 15, 33, 83, 88, 31, 77, 12, 26, 10, 29, 93, 24, 95, 89, 87, 23, 84, 74, 91, 13, 85, 76, 79, 21, 81, 17, 8, 75, 7, 72, 11, 80, 71, 9, 16, 0, 70, 68, 73, 67, 3, 2, 6, 66, 4, 5, 65, 69, 64, 1], [59, 118, 39, 46, 105, 127, 63, 34, 113, 112, 60, 119, 53, 54, 111, 50, 48, 126, 61, 122, 30, 125, 109, 49, 123, 89, 58, 97, 57, 121, 115, 56, 110, 43, 52, 25, 47, 104, 116, 55, 51, 107, 31, 108, 45, 62, 114, 36, 44, 41, 42, 120, 117, 102, 23, 33, 106, 28, 40, 82, 37, 38, 22, 94, 90, 84, 27, 98, 91, 18, 80, 101, 78, 87, 86, 99, 100, 26, 35, 32, 93, 103, 95, 96, 29, 14, 92, 21, 24, 19, 75, 20, 8, 124, 0, 76, 16, 12, 7, 88, 67, 66, 9, 2, 68, 70, 71, 73, 79, 6, 85, 65, 81, 3, 69, 4, 64, 83, 11, 1, 72, 17, 13, 5, 15, 10, 77, 74], [39, 59, 118, 34, 124, 90, 46, 84, 43, 79, 75, 54, 60, 122, 98, 30, 112, 49, 109, 127, 21, 19, 88, 111, 25, 94, 115, 76, 119, 23, 63, 89, 105, 22, 113, 9, 38, 87, 56, 8, 41, 106, 57, 31, 44, 81, 110, 33, 50, 70, 92, 48, 82, 116, 125, 100, 114, 53, 27, 62, 120, 99, 107, 42, 40, 73, 108, 58, 45, 123, 93, 104, 55, 95, 126, 121, 35, 91, 20, 18, 37, 52, 51, 61, 14, 72, 97, 28, 47, 6, 16, 86, 13, 24, 36, 96, 3, 102, 29, 11, 12, 101, 32, 26, 117, 71, 80, 66, 85, 78, 15, 7, 83, 2, 67, 68, 17, 10, 5, 69, 77, 4, 65, 103, 64, 0, 1, 74], [118, 39, 59, 34, 124, 127, 46, 105, 54, 63, 43, 55, 89, 112, 90, 84, 50, 120, 47, 57, 49, 113, 30, 115, 122, 53, 51, 125, 107, 60, 119, 80, 35, 75, 126, 45, 21, 19, 116, 123, 110, 25, 8, 111, 109, 22, 82, 88, 48, 52, 104, 27, 94, 33, 56, 121, 44, 61, 108, 62, 98, 92, 117, 58, 106, 37, 78, 79, 31, 38, 23, 40, 101, 103, 86, 99, 91, 42, 95, 24, 114, 97, 93, 12, 87, 102, 81, 32, 26, 100, 41, 20, 29, 18, 76, 14, 7, 36, 73, 28, 71, 96, 16, 13, 3, 67, 11, 70, 2, 69, 66, 68, 6, 10, 9, 0, 72, 4, 64, 77, 65, 85, 1, 83, 15, 5, 17, 74], [39, 59, 118, 34, 81, 90, 21, 13, 19, 79, 10, 46, 69, 54, 100, 65, 8, 68, 74, 115, 77, 33, 3, 11, 75, 95, 49, 55, 38, 120, 122, 71, 85, 30, 94, 0, 32, 88, 87, 18, 17, 107, 56, 112, 23, 44, 57, 63, 25, 126, 86, 67, 24, 83, 58, 119, 113, 5, 76, 72, 15, 41, 127, 89, 7, 111, 4, 12, 16, 26, 20, 43, 29, 80, 91, 14, 64, 110, 99, 22, 78, 84, 60, 123, 66, 92, 70, 37, 6, 105, 31, 1, 27, 36, 82, 96, 73, 98, 50, 35, 124, 93, 9, 47, 28, 45, 2, 53, 101, 125, 52, 114, 102, 121, 103, 109, 48, 62, 61, 104, 116, 97, 51, 40, 42, 117, 108, 106], [105, 112, 34, 84, 18, 12, 15, 27, 22, 5, 71, 48, 9, 41, 96, 2, 64, 89, 53, 82, 91, 58, 13, 4, 122, 30, 83, 1, 20, 25, 79, 63, 86, 118, 76, 87, 7, 74, 31, 78, 80, 50, 19, 85, 69, 119, 120, 70, 75, 94, 127, 16, 73, 11, 14, 81, 10, 92, 77, 46, 98, 17, 68, 6, 103, 60, 90, 28, 21, 24, 37, 72, 93, 23, 65, 107, 113, 61, 3, 26, 67, 59, 8, 56, 52, 114, 32, 115, 66, 57, 88, 110, 29, 95, 0, 123, 33, 62, 49, 55, 35, 125, 44, 36, 126, 101, 42, 99, 116, 51, 97, 47, 54, 104, 100, 102, 124, 109, 111, 43, 39, 117, 45, 38, 106, 40, 108, 121], [112, 105, 53, 25, 127, 58, 118, 89, 56, 34, 114, 54, 101, 83, 125, 96, 32, 49, 48, 120, 124, 116, 55, 52, 57, 93, 59, 33, 41, 113, 119, 62, 117, 115, 126, 61, 60, 107, 50, 106, 121, 122, 104, 45, 111, 110, 98, 51, 63, 44, 109, 47, 42, 46, 94, 123, 43, 108, 102, 37, 103, 100, 27, 38, 36, 39, 23, 99, 35, 40, 81, 87, 18, 95, 30, 22, 97, 77, 29, 31, 13, 6, 28, 8, 92, 10, 90, 68, 24, 91, 84, 16, 65, 26, 88, 86, 70, 15, 4, 19, 14, 85, 17, 80, 82, 11, 67, 21, 1, 74, 72, 75, 64, 3, 7, 78, 12, 0, 20, 66, 73, 71, 79, 5, 9, 2, 76, 69], [105, 112, 34, 15, 22, 12, 84, 18, 9, 27, 41, 89, 48, 71, 53, 5, 96, 4, 91, 58, 25, 13, 85, 2, 78, 81, 75, 64, 122, 19, 50, 67, 30, 1, 80, 31, 86, 20, 77, 82, 63, 79, 119, 72, 59, 83, 95, 88, 69, 73, 118, 21, 37, 76, 74, 7, 46, 11, 57, 3, 60, 26, 14, 6, 65, 23, 92, 28, 17, 127, 70, 62, 93, 87, 8, 24, 115, 98, 56, 61, 52, 68, 107, 125, 90, 33, 16, 120, 114, 110, 36, 10, 66, 47, 94, 35, 29, 126, 55, 116, 51, 113, 0, 97, 32, 42, 103, 123, 49, 100, 124, 102, 43, 54, 109, 117, 39, 101, 38, 99, 44, 111, 45, 104, 121, 106, 108, 40], [105, 53, 112, 34, 41, 48, 88, 25, 89, 94, 27, 33, 98, 13, 22, 84, 58, 56, 114, 118, 101, 96, 125, 120, 59, 19, 18, 78, 57, 80, 91, 124, 116, 117, 55, 62, 127, 107, 119, 49, 93, 68, 113, 6, 75, 115, 60, 50, 122, 70, 100, 15, 110, 52, 63, 54, 126, 83, 103, 47, 61, 51, 111, 4, 37, 29, 121, 92, 12, 104, 45, 97, 109, 30, 31, 42, 71, 38, 46, 123, 65, 36, 23, 44, 102, 43, 67, 74, 21, 99, 108, 10, 26, 106, 1, 24, 35, 0, 95, 39, 11, 3, 86, 81, 77, 16, 32, 40, 82, 14, 90, 9, 28, 72, 85, 17, 8, 87, 2, 20, 66, 64, 5, 7, 79, 73, 69, 76], [120, 121, 53, 57, 118, 126, 63, 113, 124, 51, 59, 60, 58, 123, 125, 115, 49, 116, 122, 119, 117, 62, 127, 56, 112, 54, 47, 55, 99, 50, 61, 114, 48, 111, 83, 44, 108, 110, 52, 13, 45, 103, 109, 46, 43, 41, 107, 40, 31, 32, 90, 106, 42, 105, 39, 38, 102, 100, 35, 9, 69, 34, 3, 97, 104, 101, 21, 82, 71, 96, 37, 28, 2, 36, 70, 98, 18, 76, 85, 68, 77, 23, 33, 30, 66, 11, 81, 92, 25, 26, 72, 0, 10, 93, 15, 29, 94, 84, 24, 86, 88, 22, 20, 16, 1, 89, 91, 75, 4, 79, 27, 78, 12, 5, 95, 65, 80, 17, 14, 19, 7, 64, 87, 73, 8, 74, 67, 6], [121, 120, 53, 57, 113, 63, 58, 118, 60, 125, 124, 126, 59, 56, 117, 119, 51, 116, 123, 55, 115, 48, 114, 99, 49, 62, 50, 61, 112, 122, 54, 52, 47, 43, 44, 110, 45, 108, 127, 111, 109, 46, 39, 41, 40, 42, 102, 106, 107, 105, 89, 90, 100, 83, 103, 35, 26, 104, 38, 32, 3, 101, 96, 37, 31, 2, 71, 72, 36, 82, 70, 34, 98, 33, 0, 13, 4, 69, 16, 76, 11, 91, 9, 92, 21, 27, 29, 22, 97, 95, 1, 94, 93, 77, 85, 8, 87, 30, 23, 15, 18, 86, 28, 88, 10, 68, 14, 25, 64, 19, 80, 12, 20, 78, 65, 84, 75, 17, 74, 79, 24, 66, 6, 7, 81, 67, 5, 73], [121, 120, 126, 40, 53, 99, 36, 63, 58, 124, 39, 51, 42, 106, 117, 88, 49, 113, 89, 59, 57, 102, 26, 122, 96, 31, 60, 123, 56, 28, 50, 46, 118, 119, 24, 105, 16, 116, 87, 112, 110, 55, 125, 32, 90, 98, 34, 82, 52, 103, 127, 35, 10, 115, 30, 54, 48, 95, 47, 100, 25, 45, 37, 38, 111, 104, 44, 41, 109, 61, 62, 33, 114, 22, 83, 43, 80, 21, 107, 27, 91, 97, 101, 29, 108, 69, 72, 92, 93, 19, 94, 15, 9, 70, 13, 71, 23, 11, 77, 84, 3, 86, 14, 85, 76, 20, 81, 79, 18, 17, 68, 74, 2, 0, 1, 12, 7, 65, 75, 78, 8, 4, 6, 73, 67, 64, 66, 5], [121, 120, 118, 39, 63, 117, 126, 124, 53, 116, 57, 51, 102, 125, 56, 60, 54, 119, 123, 58, 113, 49, 35, 99, 62, 83, 105, 112, 15, 40, 36, 44, 127, 114, 61, 48, 52, 59, 122, 42, 45, 111, 50, 47, 55, 38, 108, 107, 115, 101, 32, 106, 103, 110, 21, 46, 24, 13, 96, 88, 41, 109, 90, 26, 81, 16, 31, 43, 100, 37, 75, 34, 104, 91, 33, 25, 20, 28, 84, 69, 92, 97, 79, 98, 9, 27, 71, 89, 85, 74, 86, 87, 29, 94, 23, 22, 30, 77, 19, 93, 12, 76, 10, 11, 14, 17, 80, 5, 7, 3, 73, 95, 82, 18, 68, 78, 70, 8, 67, 4, 0, 72, 2, 65, 66, 6, 64, 1], [55, 105, 101, 120, 116, 49, 32, 112, 25, 87, 46, 57, 110, 81, 91, 109, 27, 48, 47, 119, 28, 60, 96, 94, 39, 113, 126, 44, 54, 59, 115, 63, 89, 107, 118, 84, 127, 76, 78, 85, 83, 11, 50, 106, 37, 20, 14, 122, 104, 31, 123, 58, 52, 111, 125, 61, 124, 108, 71, 88, 99, 51, 121, 19, 92, 114, 56, 45, 42, 103, 38, 43, 117, 26, 97, 62, 30, 35, 95, 34, 98, 21, 7, 86, 53, 22, 29, 73, 100, 18, 24, 102, 93, 90, 79, 15, 17, 33, 40, 5, 16, 12, 80, 70, 36, 82, 68, 41, 10, 75, 3, 23, 8, 9, 74, 2, 4, 77, 13, 72, 6, 69, 67, 65, 66, 1, 64, 0], [105, 55, 109, 116, 101, 120, 32, 112, 57, 46, 87, 25, 113, 28, 60, 110, 81, 119, 91, 104, 49, 35, 89, 84, 48, 121, 85, 39, 88, 111, 92, 42, 108, 37, 78, 27, 58, 114, 11, 107, 97, 117, 53, 31, 96, 95, 127, 125, 118, 56, 61, 94, 124, 126, 63, 62, 38, 44, 115, 99, 59, 93, 30, 54, 14, 45, 26, 50, 52, 29, 51, 98, 122, 76, 100, 106, 20, 41, 123, 33, 17, 73, 36, 79, 47, 103, 40, 12, 23, 43, 102, 86, 83, 16, 21, 24, 22, 34, 7, 19, 15, 71, 80, 75, 82, 18, 90, 10, 74, 5, 70, 6, 72, 77, 9, 3, 8, 4, 13, 2, 67, 68, 69, 65, 1, 0, 64, 66], [55, 105, 101, 120, 46, 112, 116, 57, 32, 87, 25, 39, 81, 28, 126, 47, 119, 48, 94, 84, 27, 89, 50, 56, 113, 34, 92, 118, 14, 122, 91, 60, 124, 125, 127, 38, 11, 111, 104, 78, 115, 45, 83, 110, 58, 36, 114, 108, 96, 54, 121, 109, 52, 117, 20, 31, 103, 106, 95, 42, 44, 49, 30, 107, 63, 123, 33, 37, 73, 29, 59, 79, 61, 70, 43, 51, 85, 62, 72, 99, 53, 19, 100, 102, 22, 97, 86, 21, 93, 35, 98, 17, 8, 41, 76, 40, 24, 12, 82, 90, 26, 88, 16, 75, 80, 18, 10, 15, 5, 7, 3, 6, 23, 4, 68, 9, 71, 77, 69, 66, 74, 2, 67, 13, 65, 64, 0, 1], [105, 57, 120, 116, 46, 112, 32, 55, 101, 119, 39, 113, 127, 87, 92, 94, 28, 53, 62, 104, 50, 115, 25, 126, 60, 114, 124, 38, 108, 58, 52, 27, 121, 56, 54, 111, 123, 84, 107, 98, 110, 61, 48, 122, 117, 44, 36, 106, 96, 59, 42, 51, 100, 118, 47, 63, 43, 125, 45, 109, 89, 91, 83, 79, 41, 49, 85, 33, 99, 88, 81, 35, 86, 29, 37, 95, 22, 20, 77, 30, 34, 97, 21, 18, 26, 31, 103, 40, 93, 102, 14, 78, 74, 82, 24, 90, 73, 23, 13, 75, 80, 70, 11, 76, 16, 5, 15, 64, 19, 17, 12, 7, 66, 0, 8, 2, 67, 3, 65, 72, 1, 10, 69, 68, 71, 4, 6, 9], [102, 62, 56, 55, 97, 22, 29, 19, 89, 35, 15, 88, 93, 38, 31, 76, 45, 84, 27, 60, 17, 90, 25, 33, 21, 12, 86, 46, 96, 30, 81, 95, 43, 14, 98, 78, 50, 100, 7, 26, 9, 24, 44, 61, 101, 91, 40, 87, 79, 94, 57, 122, 116, 23, 67, 71, 83, 59, 10, 34, 20, 39, 127, 85, 32, 92, 53, 28, 121, 54, 103, 73, 36, 99, 82, 18, 115, 105, 114, 117, 63, 113, 13, 104, 111, 11, 49, 124, 16, 112, 70, 41, 58, 80, 37, 125, 48, 109, 42, 8, 119, 106, 68, 51, 110, 2, 107, 123, 52, 108, 120, 75, 47, 1, 69, 126, 118, 74, 77, 0, 3, 5, 65, 72, 4, 64, 66, 6], [56, 102, 97, 55, 62, 29, 15, 45, 19, 89, 22, 60, 76, 88, 38, 93, 7, 69, 17, 82, 10, 74, 2, 59, 32, 49, 31, 81, 3, 54, 33, 40, 46, 86, 58, 25, 91, 9, 98, 68, 112, 21, 104, 43, 50, 122, 28, 27, 64, 26, 11, 13, 110, 120, 108, 90, 44, 78, 84, 73, 57, 14, 23, 117, 121, 53, 124, 61, 80, 100, 116, 101, 83, 1, 37, 115, 126, 51, 6, 12, 48, 20, 47, 114, 52, 96, 36, 87, 30, 107, 63, 119, 41, 35, 127, 118, 125, 92, 123, 75, 109, 85, 113, 67, 99, 39, 42, 106, 103, 4, 8, 94, 71, 16, 24, 77, 18, 111, 70, 105, 34, 79, 95, 72, 5, 0, 66, 65], [55, 102, 62, 56, 97, 29, 22, 38, 93, 19, 60, 15, 76, 74, 88, 12, 69, 27, 28, 13, 45, 17, 90, 81, 84, 25, 118, 32, 104, 10, 23, 82, 73, 89, 21, 31, 49, 37, 79, 42, 11, 35, 58, 16, 30, 53, 50, 101, 61, 92, 46, 109, 98, 94, 86, 7, 71, 107, 117, 26, 91, 96, 44, 83, 59, 57, 106, 114, 121, 68, 24, 43, 2, 100, 110, 124, 126, 87, 48, 47, 54, 20, 39, 36, 112, 52, 85, 125, 113, 63, 33, 95, 99, 108, 116, 119, 78, 123, 75, 9, 3, 105, 14, 127, 115, 41, 122, 51, 80, 111, 120, 34, 40, 5, 103, 77, 18, 67, 72, 64, 6, 4, 8, 70, 66, 0, 1, 65], [102, 62, 56, 97, 55, 50, 29, 43, 40, 22, 61, 45, 25, 38, 89, 19, 126, 27, 17, 42, 127, 116, 18, 88, 46, 91, 57, 15, 59, 24, 53, 13, 114, 123, 93, 8, 54, 39, 109, 60, 113, 76, 125, 81, 117, 121, 111, 104, 124, 52, 115, 119, 47, 58, 122, 120, 63, 107, 112, 110, 6, 10, 87, 48, 77, 51, 49, 106, 31, 100, 20, 44, 86, 108, 103, 26, 118, 101, 65, 84, 30, 33, 105, 78, 35, 94, 11, 95, 41, 9, 37, 23, 72, 5, 99, 36, 14, 98, 69, 28, 96, 7, 74, 73, 90, 3, 75, 83, 68, 34, 66, 82, 32, 92, 85, 80, 21, 70, 16, 2, 1, 4, 67, 12, 79, 64, 71, 0], [41, 120, 47, 34, 57, 105, 22, 93, 84, 80, 109, 60, 78, 76, 88, 117, 45, 26, 103, 61, 118, 127, 125, 74, 126, 110, 108, 8, 90, 52, 113, 86, 114, 59, 124, 58, 119, 75, 35, 49, 89, 48, 116, 99, 111, 20, 1, 123, 101, 53, 81, 51, 62, 54, 104, 24, 28, 3, 43, 29, 7, 56, 19, 31, 112, 87, 2, 69, 92, 70, 107, 121, 17, 4, 25, 95, 18, 63, 50, 16, 55, 73, 115, 122, 27, 44, 102, 13, 21, 42, 94, 32, 91, 100, 106, 96, 11, 40, 46, 33, 71, 85, 36, 79, 39, 14, 23, 37, 10, 83, 6, 97, 30, 82, 77, 15, 38, 12, 98, 5, 9, 72, 65, 66, 68, 67, 0, 64], [41, 120, 57, 34, 84, 22, 88, 109, 93, 60, 110, 116, 29, 108, 90, 80, 49, 95, 42, 105, 76, 52, 123, 24, 26, 35, 74, 39, 127, 126, 113, 104, 78, 81, 25, 44, 82, 61, 40, 53, 124, 101, 112, 19, 43, 54, 119, 111, 48, 45, 122, 125, 7, 59, 121, 86, 55, 107, 58, 98, 63, 97, 115, 32, 62, 38, 106, 56, 37, 51, 96, 27, 18, 28, 118, 103, 79, 85, 36, 117, 114, 92, 83, 20, 102, 47, 16, 91, 50, 17, 71, 94, 46, 15, 89, 21, 23, 31, 100, 30, 99, 70, 33, 14, 87, 77, 13, 75, 8, 4, 69, 66, 68, 11, 73, 1, 12, 9, 65, 10, 5, 2, 72, 3, 6, 67, 64, 0], [41, 120, 34, 47, 70, 57, 64, 8, 76, 78, 93, 67, 66, 80, 87, 105, 7, 22, 84, 74, 1, 124, 0, 60, 3, 19, 11, 88, 13, 65, 18, 86, 117, 101, 126, 61, 20, 83, 69, 59, 26, 5, 72, 110, 73, 90, 82, 68, 121, 108, 16, 89, 31, 37, 25, 9, 2, 109, 79, 10, 75, 17, 111, 14, 4, 122, 24, 12, 114, 92, 32, 27, 116, 112, 6, 23, 77, 58, 85, 71, 81, 15, 95, 127, 100, 91, 102, 21, 115, 94, 62, 113, 43, 119, 28, 30, 107, 125, 103, 53, 48, 104, 99, 55, 33, 97, 50, 45, 96, 35, 36, 51, 63, 49, 118, 38, 39, 44, 46, 106, 123, 56, 40, 42, 54, 52, 29, 98], [120, 41, 34, 109, 22, 88, 48, 84, 47, 60, 113, 93, 117, 26, 110, 49, 119, 51, 124, 43, 61, 111, 29, 58, 24, 125, 50, 55, 80, 115, 81, 62, 63, 127, 116, 57, 114, 123, 108, 112, 54, 42, 126, 56, 78, 105, 90, 118, 36, 53, 40, 121, 102, 74, 44, 46, 35, 52, 28, 59, 107, 37, 122, 76, 106, 7, 95, 86, 39, 19, 45, 101, 38, 87, 99, 103, 18, 104, 100, 97, 27, 32, 83, 20, 31, 25, 33, 30, 94, 92, 96, 91, 17, 89, 70, 98, 11, 79, 73, 68, 8, 21, 1, 82, 23, 75, 15, 16, 77, 66, 9, 85, 13, 14, 71, 10, 12, 67, 5, 69, 65, 4, 6, 2, 64, 72, 3, 0], [38, 48, 112, 24, 125, 84, 17, 109, 27, 22, 91, 97, 78, 47, 18, 10, 107, 35, 95, 81, 121, 43, 11, 98, 118, 77, 115, 46, 52, 49, 119, 86, 102, 20, 68, 13, 50, 26, 120, 90, 54, 59, 92, 40, 34, 29, 88, 53, 7, 14, 15, 56, 108, 89, 82, 55, 30, 9, 37, 96, 103, 42, 117, 51, 39, 60, 64, 106, 110, 28, 87, 94, 32, 69, 126, 61, 113, 83, 123, 105, 41, 45, 25, 63, 122, 111, 85, 19, 93, 44, 33, 116, 114, 127, 8, 80, 72, 124, 23, 104, 31, 58, 21, 100, 73, 57, 79, 16, 76, 36, 99, 6, 101, 2, 12, 75, 74, 62, 5, 71, 70, 67, 65, 4, 1, 3, 66, 0], [48, 38, 112, 24, 22, 93, 107, 84, 123, 17, 52, 50, 27, 91, 95, 118, 53, 30, 20, 98, 94, 115, 78, 125, 111, 99, 113, 49, 117, 55, 126, 45, 97, 81, 19, 119, 104, 77, 58, 110, 92, 41, 109, 28, 47, 102, 10, 36, 13, 116, 86, 105, 101, 32, 88, 35, 57, 18, 54, 80, 90, 122, 61, 121, 62, 120, 108, 34, 106, 39, 33, 40, 44, 56, 59, 43, 42, 31, 100, 124, 63, 60, 103, 127, 51, 37, 72, 96, 114, 8, 26, 46, 25, 29, 65, 23, 87, 11, 16, 68, 9, 71, 85, 89, 21, 14, 12, 75, 82, 83, 15, 7, 6, 76, 64, 79, 74, 5, 2, 73, 67, 69, 70, 4, 66, 3, 1, 0], [38, 48, 112, 24, 22, 125, 84, 107, 17, 93, 19, 80, 95, 10, 20, 78, 109, 50, 62, 13, 77, 33, 18, 27, 47, 98, 97, 68, 94, 111, 28, 92, 72, 81, 34, 35, 75, 56, 119, 6, 52, 88, 96, 8, 86, 90, 26, 9, 30, 91, 82, 49, 116, 14, 32, 31, 69, 99, 64, 43, 67, 12, 54, 53, 41, 117, 55, 29, 126, 89, 16, 15, 45, 120, 21, 36, 83, 61, 85, 114, 0, 79, 73, 39, 74, 1, 121, 57, 23, 7, 118, 59, 51, 87, 63, 25, 46, 100, 44, 110, 76, 40, 124, 58, 123, 2, 106, 5, 42, 66, 104, 37, 115, 101, 103, 65, 102, 113, 127, 11, 108, 71, 70, 105, 60, 122, 4, 3], [38, 48, 112, 24, 22, 107, 84, 93, 17, 62, 123, 97, 78, 10, 121, 30, 20, 110, 125, 18, 98, 49, 27, 96, 95, 115, 94, 50, 33, 35, 127, 116, 19, 34, 13, 45, 124, 91, 120, 72, 43, 77, 59, 86, 126, 101, 105, 51, 14, 53, 92, 111, 119, 102, 113, 41, 52, 28, 118, 80, 57, 7, 68, 109, 103, 106, 32, 55, 81, 88, 89, 40, 36, 47, 54, 117, 8, 104, 114, 42, 60, 5, 21, 99, 63, 58, 122, 26, 61, 44, 56, 39, 74, 12, 31, 100, 9, 108, 46, 25, 37, 29, 87, 82, 64, 3, 11, 65, 90, 75, 15, 16, 83, 2, 71, 66, 67, 79, 76, 85, 23, 70, 6, 69, 73, 4, 1, 0]], "model.layers.19.self_attn.k_proj": [[63, 124, 110, 37, 33, 22, 53, 55, 112, 113, 59, 122, 114, 121, 120, 125, 54, 116, 61, 126, 62, 127, 119, 50, 49, 117, 56, 111, 47, 57, 123, 118, 58, 60, 30, 48, 109, 92, 51, 105, 43, 52, 14, 108, 44, 90, 45, 15, 104, 36, 69, 35, 12, 81, 71, 101, 107, 115, 3, 9, 46, 85, 88, 99, 74, 106, 40, 39, 42, 96, 41, 102, 38, 86, 103, 91, 20, 1, 77, 82, 64, 34, 19, 72, 100, 24, 11, 25, 98, 78, 32, 75, 70, 29, 18, 87, 66, 31, 26, 93, 95, 68, 89, 94, 76, 23, 16, 28, 27, 21, 80, 83, 84, 97, 79, 0, 13, 4, 10, 2, 8, 7, 73, 5, 67, 17, 65, 6], [118, 59, 103, 98, 94, 10, 79, 81, 13, 21, 69, 19, 75, 110, 25, 8, 97, 90, 80, 78, 0, 122, 44, 43, 57, 49, 23, 48, 108, 84, 120, 87, 77, 22, 71, 82, 26, 115, 35, 74, 76, 42, 2, 106, 54, 107, 119, 95, 1, 123, 52, 102, 68, 91, 41, 113, 9, 7, 121, 112, 55, 53, 63, 50, 88, 127, 14, 27, 67, 24, 65, 101, 93, 58, 100, 62, 99, 56, 104, 60, 117, 126, 92, 105, 111, 51, 114, 38, 116, 125, 28, 61, 47, 109, 36, 17, 96, 5, 32, 124, 73, 40, 37, 4, 83, 18, 70, 11, 12, 31, 46, 33, 45, 86, 72, 66, 6, 89, 29, 85, 3, 16, 30, 39, 20, 15, 34, 64], [41, 112, 98, 22, 48, 9, 27, 64, 15, 12, 71, 84, 2, 63, 5, 25, 114, 18, 53, 62, 122, 125, 56, 113, 94, 117, 59, 124, 1, 60, 120, 58, 119, 57, 83, 118, 116, 13, 51, 50, 126, 55, 49, 4, 46, 47, 105, 115, 54, 45, 32, 111, 74, 97, 29, 123, 110, 108, 61, 70, 52, 43, 127, 121, 78, 75, 107, 80, 67, 42, 37, 85, 88, 44, 87, 109, 103, 106, 104, 102, 101, 100, 40, 19, 93, 28, 21, 39, 8, 66, 95, 35, 81, 38, 30, 36, 72, 6, 11, 99, 10, 14, 16, 90, 92, 31, 26, 33, 17, 96, 91, 0, 24, 73, 76, 65, 34, 69, 3, 89, 77, 23, 86, 68, 79, 7, 20, 82], [121, 120, 35, 22, 104, 126, 124, 95, 57, 49, 60, 117, 59, 46, 112, 63, 50, 38, 58, 51, 119, 92, 127, 106, 111, 48, 123, 118, 125, 56, 122, 113, 61, 115, 116, 91, 41, 52, 54, 103, 45, 36, 47, 28, 109, 110, 62, 18, 42, 108, 55, 32, 114, 102, 43, 12, 44, 20, 101, 53, 14, 40, 81, 39, 107, 105, 26, 94, 37, 34, 79, 29, 80, 88, 74, 97, 99, 100, 27, 96, 33, 70, 13, 23, 98, 9, 8, 71, 31, 24, 30, 21, 89, 93, 83, 87, 25, 90, 3, 4, 10, 11, 86, 84, 16, 15, 17, 1, 69, 75, 85, 2, 19, 73, 7, 72, 82, 78, 5, 76, 68, 64, 77, 6, 67, 0, 66, 65], [55, 37, 110, 116, 120, 96, 112, 41, 30, 91, 89, 28, 57, 81, 45, 124, 87, 23, 78, 103, 56, 109, 83, 102, 84, 52, 49, 36, 60, 11, 123, 31, 40, 25, 34, 76, 85, 94, 121, 117, 62, 99, 8, 71, 73, 111, 39, 97, 54, 126, 58, 14, 44, 12, 59, 125, 115, 114, 50, 122, 22, 17, 61, 24, 119, 35, 118, 51, 127, 107, 43, 93, 79, 33, 18, 48, 95, 9, 47, 63, 21, 108, 42, 106, 113, 68, 6, 5, 2, 32, 70, 27, 26, 88, 15, 90, 92, 98, 104, 3, 29, 80, 100, 86, 53, 19, 16, 7, 38, 46, 82, 67, 20, 74, 75, 13, 10, 72, 1, 65, 105, 64, 0, 77, 69, 4, 101, 66], [62, 56, 38, 55, 33, 22, 93, 15, 19, 89, 17, 76, 81, 59, 7, 109, 29, 120, 102, 78, 110, 57, 90, 53, 121, 88, 74, 27, 12, 60, 119, 10, 69, 117, 124, 103, 63, 2, 113, 125, 104, 54, 45, 9, 31, 61, 77, 58, 116, 47, 0, 49, 107, 126, 118, 114, 122, 21, 112, 52, 39, 44, 127, 115, 34, 111, 46, 48, 13, 87, 123, 84, 108, 41, 3, 37, 99, 51, 23, 18, 106, 80, 96, 105, 71, 73, 36, 24, 79, 30, 95, 94, 50, 4, 92, 16, 14, 91, 85, 83, 32, 40, 42, 28, 67, 65, 6, 26, 75, 100, 20, 68, 101, 11, 8, 64, 98, 82, 43, 35, 5, 25, 72, 86, 70, 97, 66, 1], [105, 120, 98, 47, 29, 22, 57, 64, 84, 8, 80, 76, 111, 78, 67, 88, 74, 56, 70, 48, 66, 69, 60, 50, 26, 19, 119, 46, 117, 126, 124, 61, 82, 114, 44, 115, 53, 116, 125, 63, 62, 122, 43, 45, 113, 95, 59, 7, 23, 118, 41, 99, 112, 51, 55, 65, 58, 127, 81, 107, 110, 52, 49, 18, 33, 54, 89, 71, 123, 106, 27, 109, 42, 38, 121, 4, 101, 3, 104, 87, 15, 37, 83, 39, 77, 79, 40, 13, 103, 108, 28, 102, 96, 25, 30, 31, 2, 1, 91, 94, 32, 34, 16, 35, 85, 92, 36, 100, 12, 97, 21, 73, 11, 75, 5, 9, 24, 90, 20, 68, 17, 6, 93, 72, 0, 10, 14, 86], [112, 48, 102, 22, 84, 24, 78, 107, 18, 17, 10, 13, 33, 68, 93, 72, 31, 92, 9, 109, 125, 94, 65, 19, 64, 12, 98, 6, 27, 30, 62, 80, 71, 2, 67, 77, 52, 91, 85, 8, 119, 5, 32, 35, 121, 113, 123, 11, 7, 105, 34, 36, 28, 118, 43, 61, 26, 111, 87, 106, 56, 75, 69, 120, 50, 115, 122, 83, 47, 89, 126, 23, 63, 79, 58, 49, 97, 40, 16, 90, 51, 53, 99, 45, 29, 46, 104, 76, 103, 117, 41, 25, 44, 101, 54, 100, 21, 60, 70, 124, 15, 57, 42, 20, 37, 55, 95, 127, 108, 96, 110, 116, 73, 74, 114, 59, 39, 38, 82, 14, 88, 3, 86, 81, 4, 66, 1, 0]], "model.layers.19.self_attn.qk_proj": [[112, 120, 55, 48, 62, 59, 56, 63, 118, 121, 124, 41, 110, 57, 105, 116, 86, 38, 102, 47, 98, 22, 53, 125, 50, 20, 111, 46, 58, 29, 60, 122, 93, 119, 84, 126, 113, 37, 25, 127, 17, 91, 51, 24, 89, 27, 109, 79, 61, 88, 12, 49, 76, 81, 83, 19, 90, 115, 97, 78, 54, 101, 14, 117, 114, 94, 123, 34, 82, 15, 52, 33, 35, 39, 32, 26, 45, 107, 87, 10, 30, 13, 18, 43, 103, 74, 92, 31, 44, 16, 77, 106, 72, 23, 108, 21, 95, 28, 96, 42, 9, 73, 11, 104, 71, 64, 80, 5, 40, 99, 75, 85, 36, 6, 7, 69, 0, 8, 100, 2, 70, 66, 67, 3, 4, 68, 65, 1], [112, 120, 55, 48, 59, 62, 56, 121, 63, 118, 124, 41, 110, 105, 57, 116, 86, 38, 102, 47, 53, 98, 58, 22, 125, 119, 50, 84, 91, 37, 111, 113, 20, 89, 126, 127, 93, 29, 46, 60, 25, 76, 24, 83, 12, 17, 49, 51, 109, 88, 79, 78, 114, 117, 81, 123, 34, 27, 61, 122, 90, 35, 19, 107, 33, 94, 97, 54, 115, 15, 18, 32, 39, 43, 103, 14, 92, 52, 26, 101, 30, 87, 10, 82, 74, 13, 45, 104, 96, 23, 44, 16, 77, 108, 31, 95, 40, 7, 9, 42, 21, 106, 28, 80, 73, 69, 99, 5, 0, 75, 71, 85, 11, 8, 64, 100, 6, 72, 70, 36, 2, 68, 66, 67, 4, 1, 65, 3], [112, 120, 48, 55, 59, 62, 56, 121, 63, 118, 41, 124, 105, 110, 57, 116, 86, 38, 102, 47, 22, 53, 58, 98, 29, 125, 93, 126, 20, 84, 46, 119, 49, 113, 127, 60, 122, 91, 89, 25, 50, 24, 81, 37, 111, 109, 88, 27, 19, 90, 34, 54, 94, 79, 51, 61, 12, 15, 76, 17, 35, 33, 97, 18, 117, 115, 107, 78, 26, 14, 83, 32, 39, 103, 101, 123, 52, 30, 114, 43, 23, 108, 45, 87, 8, 92, 13, 10, 82, 77, 96, 74, 95, 16, 42, 64, 31, 0, 11, 104, 44, 99, 9, 7, 21, 69, 5, 71, 80, 70, 28, 106, 36, 73, 100, 75, 66, 2, 3, 85, 40, 68, 4, 1, 67, 72, 6, 65], [112, 120, 48, 55, 62, 59, 63, 56, 118, 121, 124, 41, 110, 105, 57, 116, 38, 86, 102, 47, 126, 58, 53, 22, 29, 98, 84, 93, 89, 50, 46, 27, 91, 25, 20, 113, 123, 119, 49, 127, 51, 122, 125, 34, 81, 94, 15, 61, 37, 19, 76, 117, 24, 12, 115, 109, 78, 17, 88, 54, 79, 14, 60, 90, 107, 39, 83, 114, 97, 32, 111, 101, 35, 30, 33, 52, 18, 82, 45, 43, 23, 92, 87, 103, 8, 108, 104, 10, 26, 96, 95, 74, 13, 44, 106, 77, 0, 70, 69, 16, 64, 31, 71, 99, 5, 73, 42, 9, 80, 11, 7, 21, 75, 2, 40, 85, 66, 36, 3, 100, 68, 28, 6, 67, 4, 72, 65, 1], [112, 120, 48, 55, 62, 59, 121, 56, 63, 118, 124, 41, 110, 105, 57, 116, 38, 86, 47, 22, 102, 53, 98, 126, 20, 58, 29, 125, 84, 61, 25, 60, 51, 27, 46, 93, 122, 127, 119, 78, 17, 81, 19, 76, 49, 111, 37, 117, 12, 91, 89, 88, 50, 113, 79, 34, 24, 94, 15, 115, 90, 123, 14, 54, 97, 103, 18, 83, 101, 109, 30, 45, 33, 8, 43, 82, 106, 52, 32, 39, 13, 107, 74, 35, 23, 10, 114, 77, 73, 104, 96, 87, 108, 44, 31, 16, 11, 26, 92, 28, 71, 5, 64, 95, 7, 99, 9, 69, 70, 100, 85, 0, 42, 36, 21, 75, 40, 68, 66, 80, 2, 67, 3, 4, 72, 6, 65, 1], [112, 120, 55, 48, 62, 59, 56, 63, 121, 118, 124, 41, 110, 57, 105, 116, 86, 38, 102, 47, 22, 98, 84, 25, 29, 20, 126, 119, 53, 58, 27, 17, 113, 46, 49, 37, 81, 89, 76, 12, 88, 24, 78, 60, 93, 15, 83, 19, 122, 125, 127, 52, 50, 54, 14, 117, 123, 79, 111, 51, 91, 61, 90, 94, 115, 109, 97, 82, 34, 33, 13, 18, 39, 32, 43, 74, 101, 107, 10, 30, 8, 114, 103, 35, 96, 92, 44, 23, 45, 87, 16, 106, 104, 21, 108, 73, 11, 31, 28, 5, 80, 71, 77, 85, 26, 0, 42, 9, 75, 95, 69, 7, 99, 70, 64, 2, 66, 40, 6, 36, 100, 67, 3, 68, 72, 4, 1, 65], [112, 120, 55, 48, 62, 59, 121, 56, 63, 118, 41, 124, 57, 110, 105, 116, 86, 102, 38, 47, 22, 29, 20, 84, 98, 58, 17, 125, 53, 119, 24, 60, 25, 93, 111, 126, 27, 109, 12, 46, 19, 50, 49, 89, 76, 90, 14, 37, 81, 52, 91, 122, 78, 79, 88, 39, 117, 113, 83, 127, 15, 94, 61, 34, 97, 123, 82, 114, 32, 51, 18, 45, 33, 101, 54, 115, 92, 30, 8, 13, 96, 23, 10, 43, 103, 74, 104, 26, 35, 77, 87, 16, 31, 80, 107, 21, 85, 28, 9, 11, 44, 42, 108, 71, 106, 5, 7, 95, 99, 73, 100, 69, 6, 75, 64, 70, 40, 36, 0, 2, 68, 3, 67, 66, 1, 4, 72, 65], [112, 120, 48, 55, 62, 59, 56, 121, 63, 118, 41, 124, 105, 57, 110, 116, 86, 102, 38, 47, 53, 22, 29, 84, 98, 93, 24, 58, 126, 119, 20, 60, 111, 109, 27, 46, 37, 17, 89, 122, 25, 91, 50, 117, 81, 76, 88, 125, 12, 49, 54, 19, 90, 15, 107, 14, 83, 79, 94, 114, 78, 39, 113, 34, 123, 32, 115, 61, 97, 33, 101, 127, 51, 18, 35, 82, 92, 52, 26, 43, 103, 96, 23, 45, 77, 10, 30, 42, 104, 87, 8, 16, 74, 31, 44, 28, 13, 80, 21, 99, 7, 9, 71, 108, 5, 11, 36, 40, 85, 6, 106, 95, 64, 73, 0, 69, 75, 100, 2, 66, 72, 3, 4, 68, 70, 67, 1, 65], [112, 120, 55, 48, 62, 59, 56, 63, 118, 121, 41, 124, 105, 110, 57, 116, 86, 102, 47, 38, 53, 22, 29, 126, 84, 98, 119, 37, 93, 60, 122, 25, 58, 27, 91, 17, 20, 88, 24, 125, 109, 81, 114, 54, 94, 46, 50, 111, 113, 89, 12, 15, 49, 19, 127, 83, 33, 76, 123, 14, 117, 79, 107, 39, 61, 90, 97, 51, 34, 32, 35, 115, 52, 101, 18, 78, 103, 43, 13, 30, 82, 77, 92, 104, 26, 42, 87, 74, 23, 45, 10, 28, 31, 108, 44, 96, 16, 7, 99, 21, 6, 80, 36, 9, 8, 40, 73, 95, 11, 106, 5, 71, 85, 75, 64, 69, 100, 72, 0, 2, 70, 3, 67, 4, 66, 68, 1, 65], [112, 120, 48, 55, 62, 59, 56, 118, 63, 121, 41, 124, 57, 105, 110, 116, 102, 38, 86, 47, 22, 53, 98, 84, 58, 29, 125, 60, 126, 25, 93, 20, 37, 49, 12, 46, 119, 27, 88, 89, 122, 113, 50, 91, 81, 117, 61, 127, 109, 114, 19, 123, 76, 54, 94, 111, 24, 17, 97, 83, 79, 34, 107, 15, 115, 39, 32, 51, 14, 30, 10, 78, 103, 33, 101, 90, 92, 35, 104, 31, 82, 45, 74, 43, 77, 18, 13, 44, 42, 52, 23, 108, 26, 96, 9, 87, 106, 6, 16, 7, 99, 80, 95, 5, 0, 71, 72, 73, 69, 11, 28, 8, 64, 36, 40, 75, 21, 100, 85, 68, 67, 70, 66, 2, 4, 3, 1, 65], [112, 120, 55, 48, 62, 59, 118, 121, 56, 63, 124, 41, 110, 57, 105, 116, 38, 86, 102, 22, 53, 47, 29, 46, 20, 98, 25, 60, 84, 50, 126, 125, 122, 27, 58, 93, 119, 81, 117, 12, 61, 97, 89, 76, 88, 14, 101, 37, 78, 17, 113, 19, 15, 51, 91, 49, 83, 39, 24, 79, 127, 111, 10, 18, 114, 115, 54, 34, 94, 103, 30, 90, 109, 106, 123, 33, 107, 32, 72, 74, 82, 35, 43, 104, 77, 11, 5, 9, 92, 6, 96, 45, 13, 42, 31, 23, 52, 87, 16, 108, 26, 64, 44, 69, 73, 80, 7, 71, 0, 28, 2, 66, 21, 75, 95, 85, 36, 99, 8, 70, 68, 40, 67, 3, 4, 100, 65, 1], [112, 120, 48, 55, 62, 59, 56, 121, 63, 118, 124, 41, 105, 110, 57, 116, 86, 38, 47, 22, 102, 53, 29, 20, 84, 60, 98, 119, 27, 81, 93, 126, 50, 25, 17, 125, 46, 122, 58, 88, 37, 117, 12, 19, 89, 76, 24, 78, 34, 113, 109, 79, 83, 51, 97, 49, 15, 94, 61, 39, 14, 18, 111, 91, 107, 90, 32, 33, 115, 114, 74, 54, 101, 10, 103, 127, 106, 82, 52, 30, 123, 35, 87, 77, 104, 72, 26, 42, 31, 11, 13, 16, 9, 108, 96, 23, 80, 28, 45, 92, 43, 73, 44, 21, 99, 71, 95, 5, 85, 7, 0, 64, 69, 75, 6, 66, 36, 70, 3, 2, 100, 40, 68, 4, 8, 67, 65, 1], [112, 120, 48, 55, 62, 59, 121, 56, 63, 118, 41, 124, 110, 105, 57, 116, 86, 102, 47, 38, 53, 22, 119, 29, 84, 98, 58, 60, 20, 125, 93, 24, 27, 50, 126, 37, 122, 109, 89, 88, 117, 17, 49, 25, 91, 81, 113, 114, 76, 61, 19, 111, 12, 33, 51, 39, 46, 54, 15, 78, 34, 123, 83, 115, 90, 127, 107, 79, 52, 94, 32, 103, 35, 18, 92, 101, 26, 82, 74, 97, 14, 30, 96, 28, 72, 10, 77, 87, 44, 23, 13, 45, 106, 31, 80, 108, 104, 42, 85, 16, 21, 71, 43, 9, 99, 95, 100, 36, 11, 7, 69, 40, 5, 73, 70, 75, 0, 64, 6, 66, 67, 2, 68, 4, 65, 3, 8, 1], [112, 120, 55, 48, 59, 62, 56, 121, 63, 118, 124, 41, 105, 57, 110, 116, 86, 102, 38, 47, 22, 60, 53, 125, 58, 98, 84, 93, 119, 50, 29, 126, 27, 46, 25, 20, 61, 122, 113, 109, 12, 91, 49, 37, 117, 81, 17, 88, 89, 111, 115, 19, 76, 35, 24, 127, 34, 94, 101, 15, 54, 52, 97, 51, 79, 14, 114, 78, 33, 30, 107, 123, 72, 32, 18, 39, 26, 83, 90, 10, 82, 103, 74, 13, 104, 31, 23, 87, 96, 80, 77, 73, 92, 7, 16, 71, 106, 0, 42, 69, 45, 21, 95, 43, 108, 44, 70, 9, 99, 11, 85, 5, 28, 64, 40, 75, 100, 36, 66, 2, 4, 67, 6, 68, 3, 65, 1, 8], [112, 120, 55, 48, 62, 59, 56, 118, 121, 63, 41, 124, 110, 105, 57, 116, 86, 38, 102, 47, 98, 125, 84, 22, 29, 119, 53, 25, 20, 93, 60, 27, 12, 17, 58, 24, 126, 81, 50, 37, 117, 76, 113, 94, 46, 34, 49, 91, 78, 61, 19, 15, 88, 89, 14, 39, 122, 109, 111, 101, 97, 79, 35, 52, 83, 30, 18, 51, 72, 33, 54, 115, 10, 127, 123, 103, 82, 74, 90, 107, 13, 32, 26, 31, 23, 114, 73, 87, 106, 45, 92, 96, 9, 80, 71, 16, 77, 43, 69, 44, 64, 70, 104, 108, 11, 85, 5, 7, 0, 36, 99, 28, 95, 66, 40, 42, 21, 75, 6, 67, 68, 3, 2, 8, 100, 65, 4, 1], [112, 120, 48, 55, 62, 59, 56, 63, 118, 121, 41, 124, 110, 105, 57, 116, 86, 102, 38, 47, 22, 53, 29, 58, 98, 84, 20, 27, 125, 17, 117, 46, 76, 119, 126, 25, 60, 37, 81, 93, 113, 49, 50, 61, 24, 12, 34, 122, 89, 15, 91, 88, 19, 14, 94, 101, 78, 107, 79, 123, 51, 109, 52, 97, 115, 111, 83, 127, 35, 10, 54, 74, 39, 33, 18, 30, 82, 13, 90, 31, 32, 106, 72, 114, 103, 9, 26, 43, 77, 11, 73, 80, 44, 87, 45, 104, 92, 96, 42, 69, 71, 23, 95, 7, 108, 16, 70, 40, 85, 64, 5, 21, 75, 28, 99, 36, 0, 8, 2, 66, 67, 6, 100, 3, 68, 4, 1, 65], [112, 120, 48, 55, 62, 59, 56, 121, 118, 63, 41, 124, 105, 110, 57, 116, 86, 38, 102, 22, 53, 47, 98, 126, 29, 60, 58, 119, 20, 84, 125, 27, 46, 88, 50, 34, 49, 93, 37, 91, 89, 17, 81, 24, 25, 117, 122, 19, 113, 76, 109, 127, 12, 51, 33, 90, 83, 61, 32, 111, 94, 52, 79, 15, 97, 54, 78, 82, 39, 107, 92, 101, 18, 14, 114, 123, 30, 26, 74, 115, 43, 103, 35, 87, 96, 31, 10, 106, 13, 104, 77, 44, 23, 80, 16, 85, 28, 95, 42, 21, 45, 108, 40, 99, 36, 72, 8, 73, 5, 71, 9, 64, 70, 0, 11, 7, 75, 69, 100, 66, 6, 3, 2, 68, 4, 67, 1, 65], [112, 120, 48, 55, 62, 59, 56, 118, 121, 63, 41, 124, 105, 110, 57, 116, 38, 86, 102, 47, 22, 53, 98, 20, 29, 93, 113, 60, 46, 119, 84, 58, 125, 37, 89, 27, 91, 126, 25, 122, 17, 88, 50, 12, 49, 24, 76, 19, 117, 81, 109, 94, 15, 14, 79, 90, 127, 123, 83, 78, 61, 32, 34, 33, 30, 82, 74, 114, 107, 111, 39, 97, 115, 101, 103, 52, 54, 35, 51, 26, 10, 92, 18, 43, 13, 104, 87, 77, 31, 96, 23, 80, 0, 42, 73, 71, 16, 45, 28, 85, 106, 8, 21, 36, 7, 9, 5, 69, 108, 44, 95, 6, 64, 99, 75, 72, 11, 40, 70, 66, 2, 100, 67, 3, 4, 68, 65, 1], [112, 120, 48, 55, 62, 59, 63, 118, 56, 121, 41, 124, 110, 105, 57, 116, 38, 86, 47, 102, 53, 98, 22, 46, 125, 126, 29, 50, 93, 117, 27, 20, 25, 58, 84, 122, 89, 17, 127, 113, 37, 119, 114, 115, 91, 60, 81, 49, 19, 61, 12, 24, 94, 34, 33, 14, 109, 76, 52, 88, 123, 79, 83, 97, 15, 30, 101, 107, 51, 32, 90, 39, 54, 92, 104, 35, 82, 78, 74, 103, 8, 18, 31, 111, 43, 10, 87, 26, 45, 13, 96, 108, 77, 106, 44, 73, 23, 36, 6, 40, 16, 0, 80, 9, 71, 64, 69, 7, 42, 21, 28, 5, 11, 75, 95, 99, 85, 100, 3, 2, 70, 68, 66, 72, 4, 67, 65, 1], [112, 120, 48, 55, 62, 59, 63, 56, 118, 121, 41, 124, 110, 105, 57, 116, 86, 102, 38, 47, 53, 98, 22, 29, 125, 46, 84, 58, 25, 126, 27, 117, 89, 37, 20, 93, 119, 91, 17, 61, 24, 12, 50, 34, 60, 76, 49, 113, 109, 83, 14, 127, 111, 88, 81, 51, 115, 122, 33, 94, 114, 79, 32, 19, 123, 15, 78, 10, 54, 8, 97, 74, 107, 90, 104, 103, 82, 101, 39, 43, 30, 52, 92, 31, 77, 35, 106, 18, 13, 96, 26, 87, 80, 108, 44, 45, 28, 23, 6, 21, 73, 16, 9, 11, 95, 71, 69, 99, 75, 42, 5, 7, 64, 85, 0, 40, 36, 100, 66, 2, 70, 68, 72, 4, 1, 67, 3, 65], [112, 120, 48, 55, 62, 56, 59, 118, 63, 121, 124, 41, 110, 105, 57, 116, 86, 38, 102, 47, 53, 22, 98, 119, 25, 126, 125, 29, 20, 93, 27, 58, 60, 117, 61, 46, 84, 114, 89, 127, 113, 50, 91, 76, 37, 24, 122, 49, 109, 17, 19, 83, 81, 115, 78, 88, 123, 12, 94, 79, 14, 97, 30, 34, 52, 51, 15, 54, 107, 33, 18, 103, 32, 10, 43, 82, 111, 39, 101, 90, 8, 28, 106, 77, 13, 26, 35, 9, 104, 74, 44, 42, 92, 45, 87, 31, 80, 21, 23, 96, 16, 75, 95, 69, 6, 71, 73, 85, 11, 99, 7, 108, 5, 0, 36, 40, 64, 2, 70, 4, 67, 66, 3, 100, 68, 72, 65, 1], [112, 120, 48, 55, 62, 59, 56, 121, 63, 118, 41, 124, 110, 105, 57, 116, 86, 38, 102, 47, 58, 22, 98, 53, 126, 29, 119, 93, 20, 117, 49, 50, 125, 61, 25, 27, 51, 60, 46, 89, 84, 37, 115, 17, 91, 113, 123, 83, 114, 109, 88, 122, 76, 12, 127, 24, 81, 54, 52, 94, 14, 79, 19, 97, 18, 15, 103, 78, 32, 33, 34, 8, 30, 39, 43, 107, 92, 26, 82, 45, 111, 74, 90, 10, 13, 77, 87, 35, 31, 101, 96, 104, 21, 28, 80, 23, 44, 95, 9, 64, 0, 108, 69, 40, 106, 42, 7, 99, 5, 16, 71, 11, 73, 85, 75, 6, 36, 70, 100, 66, 4, 2, 67, 65, 3, 68, 1, 72], [112, 120, 48, 55, 59, 62, 56, 121, 118, 63, 41, 124, 105, 110, 57, 116, 38, 86, 102, 47, 22, 53, 58, 98, 119, 126, 29, 50, 60, 49, 20, 117, 113, 27, 91, 93, 61, 51, 25, 46, 89, 24, 17, 37, 125, 123, 84, 122, 76, 127, 114, 111, 88, 81, 12, 54, 15, 115, 94, 52, 19, 79, 33, 109, 83, 30, 78, 34, 107, 14, 90, 35, 97, 32, 103, 92, 26, 101, 43, 45, 77, 82, 18, 8, 10, 106, 104, 31, 39, 44, 74, 13, 87, 99, 21, 23, 108, 36, 80, 28, 16, 95, 96, 9, 7, 42, 0, 70, 73, 85, 71, 5, 11, 75, 40, 100, 69, 6, 64, 2, 66, 4, 67, 68, 72, 3, 1, 65], [112, 120, 48, 55, 62, 56, 59, 121, 118, 63, 124, 41, 110, 105, 57, 116, 86, 38, 102, 47, 22, 58, 29, 126, 98, 53, 119, 25, 20, 84, 27, 89, 125, 93, 113, 50, 24, 61, 91, 117, 60, 46, 15, 115, 81, 37, 88, 17, 94, 109, 123, 76, 122, 12, 127, 49, 51, 111, 97, 32, 34, 114, 78, 19, 14, 83, 33, 79, 30, 10, 107, 54, 39, 82, 18, 35, 74, 104, 92, 13, 101, 103, 90, 96, 8, 73, 52, 31, 77, 43, 87, 21, 23, 26, 106, 44, 45, 9, 16, 71, 28, 7, 42, 69, 75, 95, 80, 40, 70, 11, 5, 108, 85, 100, 0, 99, 2, 6, 64, 68, 36, 72, 67, 66, 4, 3, 65, 1], [112, 120, 48, 55, 62, 59, 121, 56, 118, 63, 41, 124, 110, 105, 57, 116, 86, 38, 22, 47, 102, 58, 98, 60, 20, 53, 29, 84, 25, 113, 126, 89, 119, 125, 17, 88, 81, 24, 50, 27, 61, 12, 93, 76, 49, 78, 97, 37, 19, 117, 123, 46, 91, 127, 109, 34, 122, 15, 111, 14, 83, 107, 54, 51, 103, 79, 39, 10, 33, 101, 115, 32, 94, 82, 114, 18, 90, 26, 30, 74, 13, 77, 52, 35, 43, 92, 31, 106, 87, 96, 28, 23, 9, 16, 45, 80, 104, 7, 21, 42, 44, 73, 70, 75, 69, 108, 8, 71, 5, 72, 11, 85, 0, 64, 40, 36, 100, 99, 95, 66, 2, 3, 68, 67, 6, 4, 65, 1], [112, 120, 48, 55, 62, 56, 59, 121, 118, 63, 124, 41, 105, 110, 57, 116, 86, 102, 38, 47, 53, 22, 125, 58, 98, 119, 29, 60, 84, 20, 113, 126, 49, 27, 91, 88, 46, 37, 89, 117, 122, 123, 93, 25, 61, 50, 76, 107, 12, 24, 34, 17, 127, 109, 111, 81, 32, 19, 78, 114, 51, 83, 54, 33, 115, 94, 97, 79, 103, 15, 30, 26, 35, 90, 14, 74, 39, 18, 82, 101, 43, 92, 87, 10, 96, 42, 44, 23, 31, 77, 28, 13, 45, 52, 21, 40, 16, 7, 95, 80, 104, 108, 99, 106, 72, 36, 70, 9, 85, 75, 69, 73, 71, 8, 5, 0, 100, 11, 64, 2, 66, 67, 6, 68, 1, 3, 4, 65], [112, 120, 48, 55, 56, 62, 59, 63, 121, 118, 124, 41, 105, 110, 57, 116, 47, 38, 86, 102, 53, 22, 98, 119, 126, 50, 58, 29, 20, 60, 125, 122, 113, 91, 93, 27, 61, 37, 25, 88, 111, 49, 84, 114, 46, 89, 107, 117, 51, 34, 24, 115, 109, 81, 123, 17, 76, 127, 15, 30, 12, 90, 39, 54, 83, 97, 33, 79, 14, 32, 101, 35, 103, 19, 94, 43, 92, 42, 104, 18, 78, 23, 10, 82, 52, 74, 44, 45, 26, 87, 77, 72, 96, 13, 16, 106, 99, 95, 108, 21, 28, 36, 75, 31, 80, 5, 71, 73, 9, 70, 69, 40, 100, 7, 64, 11, 0, 2, 85, 67, 66, 4, 8, 6, 3, 68, 65, 1], [112, 120, 48, 55, 56, 62, 59, 121, 63, 118, 124, 41, 110, 105, 57, 116, 86, 38, 102, 47, 22, 53, 98, 126, 119, 84, 58, 29, 125, 122, 50, 20, 46, 54, 60, 89, 93, 113, 17, 24, 27, 127, 25, 76, 12, 49, 51, 37, 78, 115, 81, 61, 91, 15, 117, 19, 109, 79, 123, 111, 83, 88, 97, 14, 114, 34, 39, 18, 94, 30, 32, 33, 10, 35, 90, 103, 101, 72, 107, 52, 74, 45, 13, 82, 77, 43, 92, 26, 87, 9, 16, 23, 71, 73, 64, 31, 44, 7, 5, 28, 96, 42, 108, 104, 75, 80, 21, 69, 0, 95, 99, 106, 40, 85, 70, 11, 36, 66, 6, 2, 8, 100, 3, 68, 67, 4, 65, 1], [112, 120, 55, 48, 62, 59, 56, 63, 121, 118, 124, 41, 110, 105, 57, 116, 86, 38, 102, 47, 98, 22, 58, 53, 60, 125, 119, 20, 122, 46, 113, 25, 126, 29, 27, 84, 51, 50, 89, 49, 76, 83, 93, 81, 127, 114, 78, 37, 12, 17, 91, 111, 97, 61, 79, 19, 24, 117, 15, 115, 88, 34, 109, 94, 54, 33, 101, 123, 45, 35, 14, 82, 43, 30, 92, 10, 90, 18, 52, 72, 107, 32, 44, 103, 104, 31, 77, 106, 9, 74, 23, 39, 13, 108, 87, 0, 69, 64, 21, 75, 5, 96, 71, 26, 73, 95, 6, 36, 16, 28, 7, 80, 85, 66, 42, 11, 99, 40, 67, 4, 100, 68, 2, 3, 1, 70, 8, 65], [112, 120, 55, 48, 62, 59, 56, 121, 118, 63, 124, 41, 110, 105, 57, 116, 86, 102, 38, 47, 22, 58, 98, 53, 20, 119, 46, 60, 29, 122, 125, 84, 49, 113, 76, 93, 25, 24, 123, 91, 27, 51, 37, 12, 81, 17, 126, 54, 89, 19, 88, 15, 14, 50, 83, 78, 115, 127, 79, 111, 34, 97, 114, 117, 101, 82, 109, 94, 61, 90, 72, 10, 33, 39, 43, 32, 30, 35, 18, 107, 87, 26, 103, 52, 92, 74, 44, 45, 106, 13, 96, 9, 23, 77, 28, 16, 6, 108, 5, 21, 104, 85, 95, 73, 71, 42, 80, 31, 7, 36, 75, 40, 69, 11, 64, 99, 2, 0, 100, 66, 3, 70, 67, 4, 8, 68, 1, 65], [112, 120, 55, 48, 59, 62, 121, 56, 118, 63, 124, 41, 110, 105, 57, 116, 38, 102, 86, 47, 98, 22, 58, 53, 60, 122, 29, 46, 20, 113, 119, 125, 126, 84, 89, 49, 24, 12, 93, 76, 51, 50, 81, 91, 127, 115, 27, 25, 123, 17, 88, 37, 97, 15, 61, 78, 14, 111, 54, 83, 101, 19, 30, 34, 72, 79, 103, 117, 94, 39, 33, 35, 109, 107, 92, 114, 32, 10, 45, 18, 82, 90, 52, 26, 74, 9, 108, 106, 6, 23, 71, 13, 44, 5, 16, 69, 95, 73, 87, 7, 96, 80, 21, 99, 31, 77, 64, 43, 85, 104, 28, 11, 75, 40, 0, 42, 2, 3, 100, 66, 36, 8, 4, 68, 70, 67, 1, 65], [112, 120, 55, 48, 62, 59, 56, 63, 121, 118, 124, 41, 110, 105, 57, 116, 86, 102, 38, 47, 58, 22, 53, 98, 50, 20, 93, 29, 60, 113, 24, 126, 84, 125, 89, 25, 49, 119, 37, 81, 12, 27, 46, 111, 91, 127, 17, 76, 61, 122, 51, 78, 15, 123, 88, 19, 79, 14, 97, 109, 94, 115, 32, 54, 90, 33, 39, 117, 34, 83, 35, 101, 52, 10, 45, 74, 92, 114, 72, 103, 107, 82, 18, 30, 26, 96, 87, 77, 44, 43, 106, 13, 108, 73, 23, 80, 16, 71, 11, 36, 6, 21, 95, 5, 9, 28, 31, 7, 75, 104, 99, 40, 42, 69, 85, 64, 100, 4, 3, 8, 70, 0, 67, 2, 68, 66, 65, 1]], "model.layers.20.self_attn.q_proj": [[114, 126, 101, 110, 46, 98, 19, 86, 29, 92, 25, 120, 24, 67, 124, 82, 12, 89, 9, 87, 28, 37, 119, 7, 49, 90, 16, 60, 23, 51, 13, 74, 115, 79, 31, 122, 58, 65, 127, 62, 77, 112, 121, 100, 116, 54, 4, 0, 97, 57, 88, 43, 118, 53, 125, 39, 61, 111, 108, 96, 34, 55, 50, 15, 14, 63, 32, 113, 47, 48, 40, 94, 30, 5, 93, 6, 56, 59, 64, 20, 109, 123, 42, 44, 117, 66, 1, 35, 72, 107, 68, 95, 45, 52, 2, 17, 27, 36, 83, 41, 26, 3, 70, 102, 38, 80, 104, 33, 105, 84, 106, 91, 81, 103, 18, 21, 99, 69, 8, 78, 73, 10, 85, 75, 71, 22, 76, 11], [46, 126, 114, 110, 101, 120, 98, 49, 124, 92, 55, 115, 58, 122, 19, 60, 119, 28, 37, 40, 54, 127, 86, 121, 116, 52, 53, 125, 118, 112, 57, 50, 51, 47, 61, 48, 62, 56, 108, 111, 90, 31, 42, 63, 113, 59, 24, 123, 43, 89, 109, 117, 105, 91, 45, 103, 44, 29, 106, 88, 15, 104, 82, 102, 39, 107, 41, 25, 95, 36, 16, 94, 35, 100, 23, 99, 85, 7, 38, 22, 33, 97, 13, 93, 96, 87, 34, 14, 79, 9, 32, 26, 21, 30, 27, 83, 73, 12, 17, 20, 84, 80, 8, 77, 81, 72, 11, 18, 78, 75, 66, 74, 10, 68, 6, 5, 4, 71, 76, 3, 1, 69, 64, 70, 2, 65, 67, 0], [126, 101, 110, 114, 46, 98, 28, 92, 120, 19, 86, 115, 124, 14, 24, 60, 58, 121, 82, 123, 23, 29, 89, 119, 116, 125, 37, 42, 39, 122, 36, 49, 31, 55, 118, 112, 103, 54, 109, 48, 25, 105, 127, 57, 62, 99, 47, 53, 51, 16, 111, 50, 59, 61, 56, 117, 26, 100, 91, 63, 33, 104, 95, 38, 32, 45, 30, 113, 83, 93, 52, 35, 94, 40, 41, 44, 85, 97, 87, 90, 96, 12, 107, 27, 108, 43, 84, 7, 102, 74, 34, 106, 21, 13, 81, 22, 15, 17, 88, 79, 80, 5, 8, 20, 18, 76, 78, 75, 71, 77, 3, 72, 9, 11, 73, 10, 66, 70, 69, 6, 1, 68, 67, 4, 2, 65, 0, 64], [114, 126, 46, 110, 101, 92, 98, 120, 119, 12, 124, 28, 19, 82, 60, 5, 51, 25, 86, 37, 127, 49, 115, 0, 62, 58, 16, 122, 50, 57, 74, 112, 117, 121, 7, 123, 61, 53, 38, 55, 113, 54, 68, 125, 118, 14, 31, 116, 47, 44, 48, 56, 109, 111, 63, 89, 87, 104, 43, 108, 2, 100, 36, 91, 45, 107, 33, 105, 94, 29, 52, 23, 59, 8, 41, 39, 6, 85, 77, 40, 70, 79, 3, 102, 42, 15, 35, 66, 18, 95, 32, 88, 69, 83, 99, 106, 96, 73, 75, 30, 9, 1, 71, 93, 64, 17, 27, 13, 103, 97, 24, 4, 20, 65, 78, 11, 67, 76, 84, 81, 21, 80, 22, 26, 72, 90, 10, 34], [43, 98, 93, 103, 107, 116, 46, 117, 78, 58, 21, 24, 81, 19, 127, 12, 63, 8, 51, 95, 121, 62, 92, 90, 120, 29, 37, 70, 26, 109, 41, 59, 101, 104, 119, 105, 100, 52, 48, 50, 36, 57, 87, 74, 86, 16, 27, 44, 7, 110, 73, 2, 72, 42, 54, 33, 67, 13, 32, 39, 56, 126, 115, 75, 53, 79, 114, 68, 83, 84, 3, 10, 125, 23, 64, 118, 9, 22, 1, 112, 61, 113, 80, 40, 25, 17, 47, 108, 85, 66, 106, 45, 30, 14, 97, 99, 38, 76, 123, 94, 35, 122, 96, 11, 82, 18, 60, 28, 49, 111, 31, 6, 15, 0, 55, 91, 71, 20, 77, 69, 89, 34, 102, 88, 124, 5, 4, 65], [43, 121, 107, 98, 93, 46, 63, 62, 52, 58, 118, 109, 38, 24, 116, 39, 127, 103, 54, 119, 37, 104, 19, 53, 51, 55, 26, 29, 44, 49, 56, 105, 21, 57, 110, 115, 48, 117, 41, 95, 50, 90, 61, 33, 47, 59, 32, 125, 112, 92, 120, 81, 123, 42, 106, 114, 108, 60, 23, 78, 111, 25, 126, 124, 122, 96, 86, 45, 113, 87, 34, 75, 12, 40, 100, 36, 101, 70, 9, 91, 83, 97, 73, 102, 31, 35, 84, 28, 22, 30, 80, 88, 99, 18, 77, 10, 82, 68, 94, 11, 27, 20, 13, 79, 17, 89, 4, 15, 8, 16, 67, 1, 85, 7, 64, 6, 71, 74, 76, 3, 65, 69, 2, 5, 14, 66, 0, 72], [43, 113, 121, 116, 93, 98, 107, 59, 52, 24, 103, 63, 53, 125, 49, 29, 127, 105, 58, 51, 110, 95, 26, 48, 117, 54, 119, 112, 46, 124, 37, 92, 120, 47, 44, 106, 61, 21, 123, 118, 109, 114, 56, 62, 45, 111, 104, 60, 90, 115, 91, 41, 122, 126, 39, 108, 57, 100, 81, 42, 97, 50, 40, 32, 33, 87, 38, 23, 101, 55, 102, 36, 22, 19, 35, 96, 88, 34, 83, 86, 84, 99, 94, 31, 18, 30, 80, 27, 77, 28, 25, 82, 89, 75, 85, 20, 16, 15, 78, 12, 17, 9, 76, 13, 79, 73, 11, 74, 70, 14, 10, 6, 72, 7, 5, 4, 66, 68, 8, 0, 69, 65, 71, 67, 2, 3, 1, 64], [43, 98, 93, 107, 103, 46, 116, 121, 24, 81, 78, 51, 21, 12, 109, 62, 8, 63, 19, 120, 26, 7, 117, 52, 29, 69, 74, 127, 68, 58, 38, 70, 92, 119, 73, 65, 90, 3, 2, 105, 37, 16, 66, 125, 44, 95, 50, 86, 48, 10, 39, 75, 106, 54, 0, 33, 64, 47, 59, 87, 56, 4, 96, 5, 97, 49, 82, 53, 71, 41, 101, 45, 84, 67, 18, 113, 114, 123, 110, 115, 104, 83, 13, 77, 79, 118, 6, 126, 11, 111, 36, 60, 17, 55, 124, 61, 1, 108, 76, 57, 23, 28, 25, 42, 35, 20, 112, 27, 80, 30, 72, 22, 40, 99, 85, 100, 122, 31, 9, 91, 15, 32, 14, 89, 102, 94, 34, 88], [39, 109, 46, 97, 93, 110, 78, 17, 10, 71, 19, 45, 5, 85, 76, 86, 16, 65, 26, 3, 22, 116, 83, 9, 89, 66, 2, 121, 35, 90, 87, 88, 107, 81, 24, 64, 79, 21, 91, 80, 119, 103, 75, 11, 28, 12, 74, 29, 15, 82, 54, 8, 92, 72, 127, 7, 18, 111, 59, 13, 23, 77, 14, 94, 1, 84, 4, 123, 48, 69, 68, 67, 20, 32, 73, 6, 70, 0, 27, 118, 30, 25, 115, 34, 99, 33, 114, 96, 102, 113, 31, 100, 38, 43, 95, 101, 58, 44, 41, 51, 125, 120, 122, 57, 98, 53, 50, 55, 126, 40, 60, 36, 104, 52, 47, 106, 62, 37, 42, 61, 105, 63, 117, 124, 112, 49, 56, 108], [109, 39, 45, 93, 97, 85, 107, 123, 110, 16, 99, 116, 54, 26, 76, 59, 53, 119, 57, 90, 23, 121, 56, 60, 118, 58, 115, 24, 55, 125, 51, 112, 40, 126, 63, 111, 43, 114, 29, 48, 102, 19, 122, 113, 127, 52, 86, 44, 124, 9, 105, 47, 62, 49, 117, 92, 46, 108, 120, 37, 61, 50, 41, 30, 106, 21, 42, 83, 6, 31, 104, 36, 94, 98, 100, 17, 64, 28, 5, 103, 35, 101, 34, 88, 96, 38, 4, 32, 25, 95, 0, 18, 87, 12, 27, 80, 10, 89, 22, 78, 1, 33, 3, 65, 68, 66, 84, 70, 73, 20, 91, 71, 82, 67, 81, 79, 77, 11, 72, 15, 75, 13, 2, 69, 8, 7, 14, 74], [109, 39, 45, 97, 93, 107, 123, 85, 110, 46, 26, 16, 86, 76, 23, 19, 121, 24, 83, 90, 29, 119, 59, 116, 51, 102, 111, 127, 57, 118, 99, 54, 17, 22, 10, 125, 53, 78, 48, 122, 58, 114, 55, 112, 60, 113, 63, 21, 124, 56, 126, 115, 27, 30, 104, 120, 50, 49, 62, 52, 12, 92, 44, 105, 61, 108, 47, 96, 35, 103, 42, 41, 89, 40, 117, 43, 15, 36, 88, 101, 37, 5, 79, 100, 32, 38, 71, 64, 33, 106, 94, 80, 95, 34, 9, 11, 6, 98, 18, 68, 82, 28, 87, 3, 77, 31, 81, 91, 25, 84, 75, 13, 20, 66, 70, 1, 0, 65, 67, 14, 4, 8, 74, 72, 73, 69, 7, 2], [39, 109, 110, 46, 97, 45, 93, 102, 86, 26, 85, 118, 117, 96, 19, 24, 123, 17, 111, 92, 57, 35, 30, 15, 60, 125, 49, 59, 116, 53, 58, 29, 121, 119, 113, 54, 99, 127, 78, 44, 63, 48, 27, 89, 122, 52, 107, 55, 51, 124, 47, 11, 112, 36, 56, 98, 126, 95, 41, 37, 62, 61, 83, 115, 40, 108, 42, 114, 106, 101, 50, 43, 105, 100, 10, 120, 32, 87, 31, 34, 28, 104, 88, 22, 76, 75, 84, 90, 16, 9, 91, 23, 25, 94, 18, 5, 79, 38, 20, 0, 21, 4, 82, 6, 13, 80, 33, 77, 70, 12, 1, 2, 8, 71, 103, 81, 65, 14, 64, 69, 72, 67, 68, 73, 3, 74, 7, 66], [51, 101, 120, 119, 97, 127, 25, 115, 82, 37, 113, 110, 118, 46, 104, 22, 28, 62, 89, 31, 93, 78, 53, 109, 21, 94, 116, 42, 122, 121, 98, 75, 30, 27, 123, 117, 126, 23, 87, 20, 59, 52, 96, 54, 47, 69, 44, 114, 48, 60, 9, 124, 81, 61, 63, 57, 56, 83, 58, 111, 55, 16, 50, 80, 125, 86, 91, 49, 95, 77, 112, 88, 24, 29, 41, 10, 90, 92, 14, 105, 45, 107, 108, 73, 3, 19, 84, 99, 71, 85, 39, 34, 43, 103, 106, 102, 100, 35, 70, 18, 38, 64, 32, 26, 5, 40, 15, 17, 8, 36, 12, 68, 67, 65, 66, 7, 74, 72, 33, 76, 13, 2, 1, 4, 11, 79, 0, 6], [127, 120, 51, 119, 101, 109, 97, 56, 113, 107, 82, 118, 102, 54, 106, 62, 59, 117, 25, 93, 124, 123, 116, 52, 126, 122, 57, 37, 38, 121, 114, 104, 55, 105, 48, 50, 125, 49, 98, 115, 112, 58, 47, 91, 44, 60, 89, 63, 35, 45, 61, 111, 110, 40, 17, 30, 46, 92, 28, 53, 23, 85, 90, 94, 42, 108, 18, 34, 36, 43, 31, 39, 96, 84, 27, 103, 41, 100, 99, 24, 88, 95, 79, 29, 26, 15, 12, 19, 32, 81, 73, 33, 83, 87, 21, 11, 78, 20, 22, 16, 77, 75, 13, 86, 1, 66, 76, 7, 0, 80, 14, 72, 69, 3, 70, 6, 9, 4, 68, 74, 10, 5, 2, 8, 67, 64, 65, 71], [120, 51, 119, 101, 25, 89, 97, 37, 47, 118, 127, 115, 62, 117, 113, 123, 84, 122, 54, 121, 52, 60, 126, 53, 55, 116, 125, 46, 56, 16, 57, 49, 50, 59, 124, 61, 63, 109, 114, 58, 48, 110, 93, 111, 28, 44, 30, 22, 42, 108, 112, 45, 12, 29, 107, 40, 92, 41, 43, 80, 103, 39, 38, 104, 105, 86, 106, 87, 73, 20, 27, 31, 95, 102, 18, 35, 98, 33, 69, 7, 36, 3, 66, 100, 34, 14, 99, 88, 77, 72, 94, 82, 76, 17, 32, 10, 1, 96, 13, 91, 70, 85, 15, 8, 74, 75, 4, 19, 83, 24, 78, 26, 11, 0, 5, 68, 21, 9, 23, 71, 79, 81, 90, 6, 67, 64, 2, 65], [119, 101, 51, 127, 120, 84, 97, 113, 115, 54, 59, 123, 62, 118, 82, 92, 30, 50, 37, 46, 27, 124, 126, 89, 25, 28, 43, 47, 110, 125, 61, 52, 45, 121, 40, 109, 53, 100, 122, 42, 102, 18, 107, 116, 58, 60, 49, 104, 48, 35, 15, 57, 114, 44, 63, 117, 95, 56, 105, 91, 96, 36, 22, 111, 108, 77, 106, 112, 73, 55, 90, 34, 41, 32, 99, 20, 38, 33, 39, 103, 98, 12, 79, 19, 26, 94, 86, 17, 93, 31, 81, 88, 29, 24, 87, 11, 13, 16, 21, 23, 74, 83, 76, 80, 10, 85, 14, 75, 72, 9, 71, 69, 6, 78, 70, 7, 68, 8, 0, 5, 67, 2, 4, 1, 3, 66, 65, 64], [110, 50, 37, 46, 26, 38, 88, 114, 41, 58, 98, 30, 36, 82, 53, 94, 90, 85, 97, 28, 34, 84, 117, 109, 100, 122, 20, 96, 123, 86, 57, 87, 104, 39, 51, 21, 25, 111, 81, 126, 120, 102, 99, 62, 22, 61, 113, 52, 45, 76, 112, 48, 116, 115, 27, 95, 29, 43, 78, 44, 79, 42, 103, 59, 47, 101, 108, 17, 105, 107, 83, 18, 121, 127, 106, 23, 31, 56, 60, 119, 54, 40, 125, 49, 124, 63, 10, 33, 91, 35, 32, 89, 55, 24, 92, 16, 4, 93, 118, 64, 75, 6, 66, 13, 72, 65, 15, 80, 77, 7, 74, 11, 73, 14, 19, 5, 12, 3, 67, 70, 68, 71, 8, 1, 9, 69, 2, 0], [110, 50, 46, 37, 58, 26, 97, 88, 41, 84, 96, 62, 114, 90, 106, 109, 30, 78, 126, 20, 92, 85, 86, 81, 34, 28, 87, 53, 123, 94, 99, 83, 104, 91, 23, 122, 38, 117, 57, 31, 93, 39, 10, 124, 63, 98, 112, 11, 21, 42, 51, 56, 48, 127, 61, 125, 17, 108, 111, 107, 116, 29, 16, 105, 115, 3, 24, 52, 118, 49, 121, 60, 7, 55, 6, 113, 70, 43, 54, 103, 47, 45, 36, 68, 102, 59, 80, 4, 32, 33, 120, 25, 95, 119, 100, 64, 44, 76, 40, 89, 77, 27, 19, 14, 82, 15, 18, 73, 35, 0, 75, 5, 79, 66, 74, 71, 72, 101, 22, 8, 9, 12, 65, 67, 13, 1, 69, 2], [110, 50, 37, 58, 46, 26, 41, 98, 97, 30, 109, 87, 122, 90, 84, 88, 94, 31, 85, 62, 60, 81, 28, 125, 78, 29, 63, 35, 20, 91, 36, 43, 126, 48, 17, 86, 121, 124, 61, 83, 51, 32, 102, 100, 54, 15, 118, 117, 114, 123, 52, 93, 92, 47, 45, 127, 56, 11, 112, 111, 113, 40, 107, 120, 116, 4, 103, 22, 27, 108, 106, 55, 104, 53, 24, 42, 101, 76, 49, 99, 89, 21, 115, 34, 96, 16, 38, 57, 33, 44, 39, 66, 6, 18, 59, 75, 119, 7, 95, 19, 79, 25, 105, 23, 8, 80, 72, 77, 10, 73, 14, 64, 74, 82, 70, 12, 5, 3, 65, 13, 67, 69, 71, 9, 2, 68, 1, 0], [110, 50, 37, 46, 58, 26, 97, 84, 30, 41, 109, 88, 28, 78, 90, 96, 117, 94, 126, 17, 47, 20, 91, 87, 85, 53, 106, 108, 81, 114, 29, 122, 105, 93, 86, 107, 11, 31, 111, 48, 34, 23, 40, 95, 45, 57, 43, 124, 16, 92, 119, 123, 49, 59, 83, 51, 15, 21, 118, 62, 63, 112, 104, 55, 98, 36, 24, 56, 116, 4, 61, 10, 127, 113, 44, 65, 54, 60, 121, 18, 103, 5, 120, 125, 39, 77, 52, 38, 100, 72, 115, 35, 89, 102, 76, 6, 99, 33, 42, 32, 74, 79, 75, 7, 70, 3, 27, 80, 73, 66, 14, 22, 25, 82, 12, 13, 101, 71, 0, 19, 8, 68, 9, 64, 67, 2, 1, 69], [125, 48, 112, 37, 53, 124, 46, 126, 61, 117, 100, 113, 121, 54, 51, 116, 57, 56, 59, 58, 114, 34, 120, 127, 118, 52, 60, 122, 123, 50, 49, 63, 62, 55, 110, 115, 97, 111, 119, 47, 109, 45, 108, 42, 44, 102, 43, 105, 103, 107, 41, 99, 104, 26, 39, 40, 106, 32, 35, 84, 38, 101, 86, 90, 96, 94, 95, 82, 88, 36, 28, 33, 30, 98, 31, 29, 87, 22, 93, 73, 14, 76, 20, 23, 16, 92, 18, 24, 78, 25, 80, 27, 7, 89, 5, 12, 91, 3, 65, 69, 9, 85, 1, 71, 83, 0, 67, 17, 21, 64, 2, 4, 15, 66, 19, 70, 10, 11, 13, 75, 77, 68, 6, 8, 74, 81, 72, 79], [48, 100, 46, 112, 25, 85, 125, 17, 34, 87, 14, 15, 83, 27, 86, 80, 96, 12, 10, 40, 29, 84, 97, 98, 103, 13, 94, 95, 37, 93, 16, 9, 22, 75, 24, 99, 8, 78, 42, 124, 73, 28, 32, 110, 69, 6, 3, 36, 77, 118, 20, 117, 11, 81, 68, 26, 21, 5, 92, 71, 74, 76, 18, 30, 82, 90, 126, 88, 50, 121, 89, 23, 79, 47, 38, 19, 0, 91, 39, 105, 1, 31, 70, 7, 115, 35, 2, 41, 116, 67, 45, 54, 114, 65, 101, 120, 53, 4, 102, 63, 104, 61, 109, 43, 64, 72, 58, 113, 33, 57, 127, 59, 56, 119, 62, 123, 52, 49, 66, 108, 106, 111, 122, 51, 55, 107, 44, 60], [48, 112, 27, 100, 46, 85, 83, 34, 25, 97, 17, 13, 15, 125, 75, 8, 28, 87, 86, 32, 37, 30, 29, 4, 12, 10, 18, 72, 124, 68, 66, 6, 96, 16, 2, 24, 77, 40, 47, 91, 33, 20, 76, 93, 26, 70, 71, 7, 31, 84, 95, 14, 90, 22, 88, 9, 36, 0, 23, 82, 39, 121, 89, 126, 81, 21, 11, 79, 38, 92, 94, 74, 118, 73, 1, 99, 54, 98, 43, 103, 19, 78, 69, 110, 114, 117, 35, 80, 115, 5, 65, 102, 67, 53, 113, 3, 45, 59, 57, 101, 56, 64, 116, 50, 105, 44, 42, 123, 111, 41, 58, 61, 52, 49, 63, 62, 104, 120, 107, 60, 119, 51, 109, 108, 127, 55, 122, 106], [46, 48, 112, 125, 121, 37, 124, 54, 126, 57, 56, 117, 62, 53, 113, 59, 120, 58, 116, 61, 114, 127, 122, 49, 123, 51, 52, 63, 118, 115, 60, 110, 55, 50, 111, 119, 45, 47, 109, 43, 44, 100, 108, 102, 104, 106, 103, 107, 41, 39, 42, 105, 40, 38, 96, 34, 101, 98, 99, 97, 33, 32, 36, 95, 86, 35, 90, 88, 28, 22, 94, 84, 82, 26, 31, 30, 92, 29, 93, 25, 89, 65, 91, 87, 20, 78, 18, 7, 16, 24, 80, 27, 76, 14, 23, 0, 3, 12, 85, 67, 73, 2, 69, 71, 5, 83, 9, 1, 21, 17, 70, 4, 81, 19, 10, 64, 13, 15, 66, 8, 75, 79, 68, 74, 72, 77, 11, 6], [116, 122, 103, 57, 126, 55, 119, 118, 40, 127, 54, 51, 59, 100, 120, 114, 125, 124, 121, 61, 115, 112, 111, 50, 123, 85, 63, 53, 62, 113, 108, 56, 47, 117, 60, 30, 48, 58, 52, 46, 104, 49, 109, 42, 33, 110, 44, 102, 45, 106, 41, 43, 107, 29, 105, 38, 36, 24, 37, 27, 34, 99, 101, 35, 25, 32, 96, 98, 19, 31, 94, 95, 97, 39, 93, 23, 78, 28, 91, 87, 83, 21, 14, 22, 92, 89, 17, 86, 90, 26, 10, 1, 72, 0, 88, 8, 5, 3, 4, 16, 81, 80, 74, 75, 68, 13, 71, 64, 67, 66, 6, 18, 84, 69, 12, 2, 70, 7, 65, 77, 82, 9, 76, 79, 15, 11, 73, 20], [122, 116, 103, 55, 54, 120, 50, 40, 114, 118, 33, 53, 124, 24, 126, 100, 115, 59, 127, 19, 113, 117, 31, 125, 63, 121, 61, 57, 119, 112, 111, 60, 123, 56, 62, 58, 46, 110, 52, 51, 30, 102, 47, 42, 49, 48, 108, 27, 85, 74, 106, 109, 104, 22, 94, 80, 25, 44, 35, 107, 36, 45, 93, 43, 91, 41, 105, 32, 29, 38, 37, 84, 101, 28, 34, 26, 20, 98, 89, 82, 86, 23, 99, 97, 87, 96, 90, 21, 95, 83, 15, 72, 92, 88, 39, 5, 12, 0, 14, 18, 16, 78, 77, 3, 17, 73, 71, 6, 79, 76, 81, 68, 13, 10, 70, 9, 4, 2, 8, 66, 69, 64, 67, 75, 1, 65, 11, 7], [116, 122, 103, 55, 118, 54, 126, 50, 127, 61, 114, 57, 120, 112, 119, 121, 125, 115, 124, 62, 117, 40, 123, 33, 52, 53, 59, 60, 56, 113, 58, 104, 63, 111, 51, 100, 34, 91, 47, 49, 26, 36, 48, 93, 22, 110, 19, 46, 42, 44, 45, 25, 109, 102, 108, 24, 27, 107, 21, 106, 41, 31, 43, 80, 101, 94, 37, 35, 99, 38, 105, 29, 85, 28, 98, 86, 32, 39, 97, 96, 95, 89, 30, 92, 23, 78, 90, 74, 16, 14, 83, 82, 17, 88, 84, 87, 3, 18, 77, 12, 15, 10, 11, 1, 9, 68, 4, 81, 66, 0, 8, 20, 65, 13, 6, 79, 67, 70, 64, 71, 75, 69, 5, 2, 72, 73, 76, 7], [103, 116, 122, 93, 100, 84, 82, 55, 81, 33, 88, 15, 22, 75, 91, 87, 77, 52, 24, 11, 118, 79, 17, 71, 50, 26, 31, 119, 102, 126, 27, 40, 120, 13, 57, 32, 54, 89, 29, 23, 20, 34, 25, 90, 12, 21, 28, 72, 83, 125, 115, 121, 14, 18, 86, 59, 68, 94, 114, 61, 112, 85, 117, 124, 113, 127, 53, 96, 92, 123, 63, 8, 30, 111, 80, 58, 76, 62, 16, 110, 46, 47, 51, 7, 56, 104, 78, 69, 9, 10, 19, 60, 73, 97, 67, 95, 5, 37, 44, 49, 4, 66, 3, 108, 48, 99, 6, 74, 43, 109, 42, 70, 45, 98, 38, 36, 0, 107, 35, 106, 1, 64, 2, 105, 41, 101, 65, 39], [102, 113, 56, 97, 23, 49, 79, 29, 77, 81, 20, 75, 10, 72, 6, 3, 22, 16, 18, 32, 67, 38, 28, 15, 34, 5, 9, 88, 48, 17, 112, 65, 115, 92, 25, 70, 26, 74, 53, 78, 0, 39, 69, 86, 84, 63, 87, 64, 11, 31, 12, 105, 83, 19, 89, 66, 58, 117, 57, 80, 30, 85, 21, 13, 91, 93, 73, 8, 55, 82, 90, 52, 100, 41, 27, 1, 44, 7, 95, 96, 103, 36, 76, 94, 59, 99, 24, 119, 121, 14, 51, 50, 116, 35, 118, 33, 47, 60, 106, 101, 110, 46, 122, 2, 98, 123, 45, 62, 40, 109, 4, 42, 71, 37, 126, 104, 54, 124, 68, 61, 43, 114, 120, 125, 107, 108, 127, 111], [102, 56, 113, 97, 29, 49, 23, 20, 79, 77, 75, 81, 10, 3, 6, 72, 22, 69, 0, 30, 18, 65, 88, 25, 53, 84, 74, 13, 70, 17, 87, 86, 59, 26, 93, 71, 1, 89, 64, 66, 15, 76, 52, 80, 92, 11, 119, 73, 24, 4, 95, 96, 8, 31, 34, 90, 91, 14, 33, 19, 32, 28, 38, 62, 58, 82, 9, 41, 36, 112, 94, 110, 12, 83, 78, 16, 5, 121, 27, 57, 21, 103, 7, 48, 115, 50, 63, 117, 85, 35, 99, 2, 101, 43, 98, 39, 100, 67, 125, 60, 68, 45, 123, 122, 105, 51, 126, 44, 54, 127, 47, 61, 37, 124, 114, 106, 55, 108, 111, 40, 120, 46, 42, 107, 109, 104, 118, 116], [113, 102, 56, 121, 48, 49, 97, 39, 29, 22, 44, 126, 16, 115, 52, 50, 60, 59, 124, 118, 51, 117, 123, 38, 57, 119, 63, 54, 47, 112, 28, 114, 127, 58, 46, 122, 90, 62, 120, 110, 53, 125, 43, 55, 40, 109, 104, 61, 116, 108, 24, 41, 37, 111, 105, 103, 107, 32, 101, 106, 42, 45, 91, 23, 20, 26, 36, 100, 86, 25, 34, 98, 81, 99, 88, 80, 96, 93, 31, 94, 35, 85, 14, 30, 27, 95, 92, 83, 76, 33, 18, 78, 82, 9, 89, 19, 21, 12, 73, 77, 75, 69, 84, 10, 71, 79, 7, 4, 2, 68, 87, 17, 66, 5, 72, 64, 65, 0, 3, 1, 6, 15, 11, 13, 67, 8, 70, 74], [56, 102, 113, 48, 97, 49, 29, 23, 41, 22, 117, 52, 28, 115, 16, 46, 36, 20, 114, 103, 81, 86, 121, 44, 31, 50, 53, 63, 105, 58, 25, 77, 39, 119, 51, 106, 126, 91, 54, 90, 76, 78, 27, 60, 89, 55, 59, 123, 62, 92, 79, 118, 10, 38, 107, 104, 42, 120, 122, 109, 108, 124, 72, 69, 112, 14, 12, 127, 47, 116, 57, 110, 33, 43, 19, 111, 93, 32, 83, 24, 40, 101, 45, 125, 30, 61, 80, 75, 37, 35, 99, 82, 84, 26, 98, 88, 18, 34, 85, 96, 100, 9, 94, 21, 3, 64, 95, 66, 68, 71, 5, 7, 74, 1, 0, 17, 73, 6, 4, 65, 2, 87, 13, 70, 15, 67, 11, 8]], "model.layers.20.self_attn.k_proj": [[126, 114, 46, 37, 86, 34, 50, 64, 124, 28, 12, 16, 25, 51, 19, 66, 119, 57, 120, 7, 60, 58, 115, 82, 53, 127, 123, 62, 116, 61, 74, 122, 14, 113, 112, 9, 117, 63, 56, 125, 54, 121, 55, 47, 118, 48, 44, 111, 43, 79, 59, 49, 13, 45, 109, 29, 39, 101, 95, 6, 52, 107, 24, 3, 41, 36, 42, 108, 102, 104, 87, 105, 106, 40, 94, 103, 21, 38, 98, 4, 33, 90, 27, 32, 23, 8, 35, 72, 96, 5, 1, 99, 15, 100, 91, 80, 30, 110, 77, 97, 89, 84, 20, 93, 70, 88, 17, 2, 31, 11, 18, 65, 68, 26, 81, 75, 92, 69, 76, 85, 78, 73, 67, 0, 10, 83, 71, 22], [107, 34, 43, 29, 116, 51, 90, 78, 81, 8, 127, 19, 31, 70, 12, 56, 0, 110, 7, 120, 87, 32, 68, 61, 63, 3, 74, 62, 75, 54, 2, 39, 121, 88, 21, 66, 100, 24, 92, 73, 85, 58, 46, 95, 28, 86, 109, 45, 52, 59, 69, 77, 126, 117, 37, 1, 48, 113, 57, 36, 33, 47, 42, 114, 53, 27, 64, 4, 11, 84, 99, 18, 15, 5, 50, 119, 108, 97, 112, 65, 82, 80, 124, 44, 125, 16, 30, 71, 102, 22, 25, 41, 122, 40, 94, 60, 118, 105, 55, 101, 104, 10, 79, 13, 106, 38, 89, 115, 20, 26, 49, 9, 123, 111, 23, 35, 91, 72, 76, 96, 17, 14, 67, 93, 83, 6, 103, 98], [45, 103, 46, 33, 29, 109, 86, 19, 85, 123, 90, 119, 5, 10, 16, 64, 78, 57, 116, 76, 71, 52, 17, 112, 54, 126, 127, 121, 65, 3, 53, 115, 125, 51, 122, 107, 55, 48, 113, 58, 56, 114, 118, 59, 120, 60, 24, 124, 9, 2, 47, 44, 61, 63, 34, 111, 89, 62, 108, 106, 67, 13, 41, 11, 49, 32, 117, 100, 43, 99, 50, 72, 35, 38, 101, 104, 6, 31, 69, 105, 23, 42, 37, 15, 18, 88, 30, 95, 82, 87, 36, 91, 94, 102, 25, 84, 92, 75, 0, 40, 110, 27, 93, 98, 14, 1, 77, 96, 28, 20, 8, 4, 73, 79, 66, 81, 26, 83, 68, 22, 7, 70, 74, 39, 12, 97, 80, 21], [51, 37, 119, 120, 127, 33, 22, 46, 122, 123, 121, 50, 118, 56, 45, 124, 113, 52, 126, 61, 116, 53, 62, 117, 48, 125, 106, 115, 54, 55, 60, 57, 30, 59, 58, 63, 40, 89, 91, 49, 108, 114, 101, 47, 110, 104, 111, 112, 41, 109, 80, 44, 102, 103, 43, 42, 107, 38, 105, 39, 16, 92, 93, 95, 23, 36, 32, 82, 35, 34, 31, 21, 84, 29, 25, 86, 99, 96, 100, 28, 77, 76, 26, 98, 87, 88, 19, 72, 12, 27, 90, 94, 24, 83, 70, 78, 15, 85, 10, 75, 97, 7, 9, 74, 81, 14, 17, 13, 68, 3, 79, 11, 5, 2, 20, 18, 73, 65, 6, 4, 71, 64, 67, 8, 69, 0, 1, 66], [46, 50, 110, 101, 26, 30, 58, 20, 114, 85, 78, 33, 17, 4, 66, 88, 11, 76, 79, 41, 62, 7, 72, 126, 64, 75, 5, 53, 122, 70, 6, 3, 19, 34, 87, 73, 18, 23, 91, 86, 15, 68, 83, 16, 32, 77, 60, 84, 81, 49, 43, 10, 65, 117, 100, 118, 105, 127, 24, 112, 109, 31, 51, 92, 45, 67, 61, 120, 48, 35, 57, 63, 56, 29, 40, 107, 96, 25, 89, 22, 27, 113, 119, 125, 82, 52, 80, 124, 55, 116, 21, 93, 0, 1, 115, 121, 103, 108, 44, 54, 106, 99, 28, 74, 38, 123, 95, 13, 104, 39, 47, 59, 98, 36, 102, 111, 42, 8, 97, 14, 9, 37, 12, 2, 90, 94, 71, 69], [112, 48, 36, 86, 15, 17, 125, 110, 83, 75, 25, 10, 27, 33, 13, 29, 85, 70, 32, 30, 8, 124, 117, 46, 56, 126, 121, 54, 84, 82, 115, 61, 28, 57, 53, 2, 116, 98, 0, 47, 4, 62, 90, 120, 59, 51, 80, 63, 102, 60, 114, 87, 123, 35, 127, 44, 113, 50, 109, 52, 122, 118, 49, 58, 92, 105, 111, 55, 41, 119, 77, 45, 11, 104, 26, 107, 42, 73, 43, 38, 6, 103, 101, 106, 37, 31, 14, 108, 40, 74, 79, 65, 99, 7, 88, 16, 34, 23, 72, 39, 12, 76, 96, 95, 67, 21, 93, 78, 3, 24, 20, 89, 91, 9, 69, 94, 97, 100, 19, 18, 68, 71, 81, 5, 66, 22, 1, 64], [122, 116, 39, 22, 55, 97, 54, 61, 126, 118, 52, 117, 112, 124, 57, 121, 50, 123, 120, 115, 62, 113, 127, 27, 58, 56, 114, 53, 59, 63, 51, 95, 119, 98, 60, 26, 34, 49, 125, 111, 47, 110, 104, 45, 42, 44, 46, 82, 109, 48, 24, 107, 108, 80, 32, 38, 35, 41, 43, 84, 102, 37, 106, 16, 105, 101, 12, 36, 96, 40, 14, 99, 85, 92, 23, 15, 29, 9, 33, 94, 90, 17, 77, 19, 30, 31, 93, 100, 28, 10, 71, 75, 103, 81, 87, 25, 91, 89, 20, 79, 78, 76, 70, 67, 83, 18, 11, 8, 72, 1, 88, 13, 69, 86, 74, 5, 3, 21, 73, 0, 6, 66, 7, 4, 64, 2, 68, 65], [113, 56, 38, 33, 22, 93, 81, 23, 79, 72, 77, 20, 6, 10, 75, 112, 0, 3, 120, 69, 117, 27, 57, 65, 16, 49, 52, 115, 103, 54, 59, 110, 50, 126, 46, 122, 51, 28, 53, 60, 124, 26, 58, 55, 61, 119, 125, 102, 118, 105, 62, 123, 5, 121, 48, 95, 116, 2, 94, 47, 63, 45, 127, 83, 108, 30, 114, 99, 109, 106, 9, 107, 66, 100, 89, 67, 32, 104, 44, 43, 35, 21, 14, 76, 40, 42, 18, 96, 78, 85, 88, 24, 101, 39, 111, 34, 73, 68, 41, 25, 4, 36, 31, 37, 29, 90, 98, 12, 92, 1, 80, 19, 7, 71, 74, 82, 64, 91, 87, 84, 8, 17, 70, 11, 13, 15, 86, 97]], "model.layers.20.self_attn.qk_proj": [[46, 113, 56, 116, 122, 112, 126, 48, 110, 51, 114, 45, 107, 43, 119, 120, 50, 109, 127, 37, 93, 125, 121, 54, 49, 22, 52, 123, 86, 101, 58, 39, 102, 118, 55, 29, 115, 53, 57, 90, 63, 124, 60, 62, 103, 26, 33, 92, 61, 97, 59, 17, 81, 34, 23, 83, 38, 19, 84, 89, 47, 117, 91, 87, 85, 24, 14, 111, 44, 32, 108, 41, 78, 98, 21, 16, 74, 30, 80, 88, 25, 20, 105, 36, 79, 100, 75, 77, 10, 15, 106, 95, 11, 12, 13, 28, 104, 27, 76, 31, 94, 72, 6, 82, 40, 18, 8, 96, 35, 7, 42, 67, 71, 3, 66, 9, 64, 0, 99, 73, 5, 2, 70, 4, 65, 1, 69, 68], [46, 113, 56, 116, 126, 112, 122, 48, 110, 45, 51, 114, 120, 107, 119, 43, 50, 109, 127, 93, 37, 125, 121, 101, 49, 52, 58, 123, 86, 22, 54, 102, 39, 29, 53, 103, 115, 55, 60, 97, 61, 90, 118, 117, 63, 34, 57, 59, 33, 124, 26, 62, 38, 81, 92, 23, 85, 87, 83, 24, 47, 89, 19, 17, 30, 98, 91, 84, 14, 41, 78, 16, 88, 32, 111, 21, 80, 36, 75, 95, 20, 76, 100, 44, 25, 108, 10, 77, 15, 12, 11, 74, 104, 79, 105, 106, 13, 42, 28, 94, 96, 6, 27, 72, 82, 31, 40, 9, 8, 64, 18, 35, 3, 71, 66, 2, 7, 0, 99, 70, 5, 68, 67, 1, 73, 69, 65, 4], [46, 113, 116, 56, 126, 112, 48, 122, 110, 51, 114, 45, 119, 107, 43, 120, 50, 109, 127, 93, 125, 37, 121, 22, 86, 101, 49, 54, 52, 58, 55, 39, 29, 63, 102, 123, 103, 115, 33, 118, 60, 90, 53, 124, 61, 97, 62, 92, 26, 34, 57, 59, 30, 81, 17, 23, 24, 83, 38, 117, 47, 14, 85, 87, 44, 88, 19, 21, 41, 84, 111, 108, 98, 91, 32, 95, 78, 100, 89, 80, 25, 105, 16, 28, 36, 75, 104, 74, 79, 31, 20, 15, 77, 106, 12, 27, 76, 13, 94, 10, 82, 72, 18, 11, 64, 35, 96, 67, 0, 42, 8, 6, 9, 40, 70, 99, 3, 66, 7, 69, 2, 5, 71, 68, 73, 4, 65, 1], [46, 113, 116, 56, 126, 122, 112, 48, 110, 51, 114, 45, 107, 120, 119, 43, 109, 50, 127, 125, 93, 37, 49, 22, 101, 121, 54, 58, 55, 53, 39, 102, 115, 86, 63, 29, 60, 103, 52, 61, 97, 90, 57, 118, 62, 34, 124, 33, 117, 26, 59, 92, 47, 17, 81, 123, 19, 14, 87, 30, 83, 38, 75, 23, 24, 78, 88, 16, 80, 85, 95, 20, 98, 106, 89, 91, 84, 21, 44, 111, 31, 79, 15, 105, 25, 12, 100, 32, 28, 74, 42, 41, 104, 13, 27, 76, 36, 94, 10, 11, 77, 82, 96, 108, 67, 72, 40, 18, 3, 8, 2, 70, 7, 66, 64, 0, 5, 71, 6, 35, 73, 69, 9, 99, 68, 4, 65, 1], [46, 113, 116, 56, 122, 126, 48, 112, 110, 51, 114, 45, 120, 107, 119, 43, 109, 50, 127, 125, 93, 49, 37, 54, 121, 22, 86, 39, 115, 58, 101, 63, 123, 55, 29, 61, 103, 53, 102, 57, 52, 60, 59, 62, 26, 124, 97, 81, 90, 118, 34, 33, 47, 117, 19, 17, 98, 38, 23, 87, 78, 83, 84, 24, 92, 30, 85, 20, 80, 21, 75, 16, 111, 89, 79, 91, 25, 95, 14, 88, 105, 74, 44, 15, 12, 104, 40, 36, 10, 108, 31, 28, 41, 77, 100, 106, 76, 32, 27, 13, 11, 82, 42, 72, 70, 73, 18, 94, 71, 8, 7, 35, 0, 3, 96, 66, 64, 99, 67, 9, 69, 2, 6, 5, 68, 65, 4, 1], [46, 113, 116, 56, 122, 126, 112, 48, 110, 51, 45, 114, 119, 120, 107, 43, 50, 109, 127, 93, 125, 49, 37, 22, 86, 121, 58, 54, 101, 39, 60, 115, 29, 123, 102, 52, 97, 90, 63, 124, 53, 117, 26, 57, 81, 103, 118, 34, 19, 59, 55, 61, 84, 16, 87, 80, 85, 62, 33, 17, 23, 47, 79, 78, 98, 38, 92, 24, 89, 14, 20, 111, 30, 12, 83, 44, 75, 31, 88, 95, 104, 91, 15, 25, 21, 10, 74, 76, 32, 28, 11, 77, 100, 36, 105, 82, 13, 108, 106, 41, 27, 72, 96, 8, 70, 40, 42, 94, 18, 73, 9, 3, 67, 7, 0, 64, 71, 35, 99, 66, 6, 5, 2, 69, 4, 65, 1, 68], [46, 113, 56, 116, 48, 126, 112, 110, 122, 114, 45, 51, 120, 119, 109, 107, 43, 50, 127, 125, 93, 37, 22, 86, 49, 39, 102, 101, 121, 58, 29, 123, 115, 57, 54, 60, 103, 90, 34, 118, 26, 117, 55, 52, 63, 53, 33, 61, 124, 87, 19, 97, 17, 23, 16, 84, 24, 47, 81, 62, 85, 59, 38, 92, 83, 91, 88, 30, 98, 80, 20, 79, 78, 14, 44, 21, 89, 25, 15, 76, 104, 32, 36, 75, 100, 108, 40, 10, 12, 28, 31, 95, 27, 41, 74, 77, 13, 111, 11, 42, 105, 106, 82, 18, 94, 35, 9, 96, 8, 72, 70, 99, 7, 73, 64, 71, 67, 3, 0, 6, 66, 2, 69, 5, 4, 68, 1, 65], [46, 113, 56, 116, 126, 48, 112, 110, 122, 51, 45, 114, 120, 119, 107, 43, 109, 50, 127, 125, 93, 22, 37, 86, 101, 121, 123, 102, 49, 58, 39, 29, 53, 57, 117, 54, 55, 60, 90, 62, 115, 118, 103, 124, 52, 33, 26, 87, 34, 63, 91, 92, 59, 24, 81, 23, 61, 19, 97, 83, 85, 38, 30, 17, 47, 98, 89, 21, 111, 80, 108, 41, 88, 32, 36, 84, 25, 15, 14, 20, 79, 104, 27, 16, 10, 44, 31, 78, 28, 40, 100, 95, 77, 94, 75, 12, 42, 13, 76, 18, 74, 11, 105, 106, 82, 99, 35, 70, 8, 96, 7, 72, 64, 3, 9, 0, 71, 66, 73, 69, 2, 6, 67, 5, 68, 1, 4, 65], [46, 113, 56, 126, 116, 48, 112, 122, 110, 51, 45, 114, 120, 107, 119, 50, 43, 109, 127, 93, 125, 49, 37, 121, 101, 22, 123, 58, 102, 29, 86, 55, 117, 60, 39, 26, 53, 63, 54, 61, 57, 62, 103, 90, 52, 118, 115, 34, 33, 124, 92, 17, 81, 38, 97, 47, 85, 87, 83, 23, 78, 59, 24, 30, 21, 91, 36, 88, 32, 84, 19, 89, 14, 16, 20, 104, 41, 111, 15, 98, 25, 44, 80, 79, 31, 100, 95, 27, 28, 11, 10, 105, 77, 94, 76, 12, 40, 75, 108, 13, 106, 74, 18, 96, 42, 8, 35, 7, 82, 99, 72, 3, 70, 0, 2, 73, 6, 67, 64, 9, 71, 69, 66, 68, 1, 4, 5, 65], [46, 113, 116, 56, 126, 48, 112, 122, 110, 51, 45, 114, 120, 107, 43, 119, 50, 109, 127, 93, 37, 125, 121, 49, 58, 22, 54, 52, 123, 101, 117, 86, 102, 53, 55, 124, 29, 57, 115, 39, 63, 118, 61, 62, 103, 90, 34, 97, 60, 38, 26, 17, 33, 92, 81, 47, 78, 87, 85, 59, 23, 19, 83, 88, 24, 89, 25, 30, 32, 98, 10, 16, 80, 21, 41, 84, 14, 111, 11, 95, 100, 15, 36, 44, 91, 77, 79, 104, 12, 75, 28, 40, 31, 20, 27, 8, 74, 76, 106, 96, 13, 105, 42, 108, 94, 7, 6, 71, 0, 72, 3, 18, 35, 2, 64, 9, 82, 70, 73, 69, 67, 66, 99, 68, 5, 4, 1, 65], [46, 113, 56, 116, 122, 126, 48, 110, 112, 51, 114, 45, 120, 107, 43, 119, 50, 109, 127, 125, 93, 37, 121, 49, 115, 22, 58, 101, 54, 86, 118, 123, 39, 63, 52, 124, 53, 62, 102, 97, 29, 103, 55, 26, 57, 61, 60, 90, 81, 17, 19, 117, 33, 111, 87, 84, 92, 34, 38, 83, 85, 78, 16, 15, 47, 25, 79, 24, 14, 80, 95, 30, 11, 59, 23, 21, 44, 75, 20, 88, 10, 91, 98, 12, 104, 77, 32, 28, 106, 6, 76, 8, 74, 36, 100, 82, 72, 89, 105, 27, 31, 41, 108, 13, 18, 40, 94, 42, 7, 64, 96, 2, 73, 71, 0, 3, 9, 35, 67, 69, 66, 99, 5, 70, 4, 1, 65, 68], [46, 113, 56, 116, 48, 126, 110, 122, 112, 51, 45, 114, 120, 107, 119, 43, 50, 109, 125, 127, 93, 37, 22, 86, 49, 101, 58, 102, 121, 39, 29, 115, 53, 54, 123, 63, 117, 34, 55, 52, 90, 62, 26, 103, 61, 17, 19, 124, 60, 97, 33, 118, 81, 57, 83, 38, 85, 87, 59, 84, 92, 23, 78, 47, 15, 98, 16, 80, 24, 14, 10, 30, 88, 91, 20, 100, 79, 12, 21, 44, 111, 36, 25, 32, 11, 95, 31, 89, 104, 75, 28, 41, 77, 76, 27, 74, 106, 82, 8, 108, 94, 96, 40, 18, 105, 6, 13, 72, 71, 73, 9, 42, 67, 35, 66, 3, 0, 64, 7, 99, 69, 2, 70, 5, 1, 65, 4, 68], [46, 113, 56, 116, 126, 48, 112, 110, 122, 51, 45, 120, 114, 107, 50, 109, 43, 119, 93, 127, 125, 37, 49, 22, 102, 86, 58, 123, 39, 121, 54, 101, 29, 55, 117, 115, 124, 62, 63, 52, 33, 53, 97, 60, 103, 34, 118, 90, 61, 47, 38, 57, 26, 92, 30, 19, 85, 59, 87, 81, 83, 89, 88, 98, 17, 32, 91, 23, 84, 36, 25, 44, 24, 16, 111, 80, 78, 21, 100, 20, 108, 15, 95, 11, 31, 105, 77, 28, 10, 79, 27, 12, 14, 41, 74, 82, 76, 94, 40, 13, 104, 75, 106, 96, 35, 71, 8, 42, 18, 6, 72, 0, 66, 64, 73, 7, 9, 3, 99, 2, 67, 70, 5, 69, 1, 4, 68, 65], [46, 113, 56, 122, 126, 116, 48, 112, 110, 51, 45, 114, 120, 107, 119, 50, 43, 109, 127, 125, 93, 37, 22, 49, 58, 54, 101, 121, 86, 39, 60, 29, 102, 123, 57, 52, 53, 62, 103, 115, 55, 63, 117, 90, 61, 124, 118, 34, 26, 59, 17, 81, 47, 33, 92, 97, 83, 111, 38, 87, 19, 32, 85, 78, 98, 23, 30, 80, 21, 91, 24, 89, 84, 11, 36, 20, 14, 95, 15, 44, 10, 28, 104, 25, 105, 108, 16, 79, 41, 106, 31, 88, 76, 96, 74, 12, 77, 100, 82, 75, 27, 72, 94, 42, 8, 18, 64, 0, 13, 71, 73, 6, 35, 9, 70, 7, 99, 66, 67, 40, 5, 1, 2, 3, 69, 4, 65, 68], [46, 113, 116, 56, 48, 126, 112, 122, 110, 51, 114, 45, 107, 120, 43, 119, 50, 109, 127, 93, 125, 37, 22, 86, 121, 49, 102, 101, 58, 52, 123, 39, 55, 29, 118, 54, 117, 103, 63, 61, 57, 33, 62, 90, 26, 17, 53, 81, 19, 60, 124, 59, 34, 115, 87, 97, 24, 92, 78, 85, 83, 74, 11, 91, 79, 16, 21, 10, 89, 104, 23, 80, 32, 38, 98, 20, 84, 36, 111, 95, 47, 12, 27, 30, 14, 28, 88, 76, 15, 25, 100, 75, 41, 108, 106, 77, 96, 94, 44, 18, 31, 73, 105, 40, 8, 82, 70, 35, 13, 71, 72, 0, 64, 67, 7, 2, 3, 9, 6, 66, 42, 69, 5, 68, 1, 65, 99, 4], [46, 113, 56, 116, 126, 48, 122, 110, 112, 51, 114, 45, 120, 107, 43, 119, 109, 50, 125, 127, 93, 37, 49, 22, 101, 121, 86, 54, 58, 102, 57, 123, 52, 63, 60, 117, 115, 29, 53, 55, 62, 118, 103, 39, 81, 59, 26, 124, 97, 61, 90, 83, 87, 33, 17, 34, 38, 19, 74, 47, 23, 78, 85, 32, 89, 11, 91, 24, 84, 95, 92, 98, 20, 15, 79, 104, 111, 36, 16, 30, 77, 14, 25, 10, 88, 75, 27, 12, 21, 80, 28, 40, 44, 41, 106, 105, 13, 70, 31, 100, 108, 76, 94, 8, 72, 18, 42, 96, 82, 71, 73, 67, 7, 35, 9, 3, 0, 66, 2, 64, 99, 6, 69, 5, 65, 1, 4, 68], [46, 113, 56, 116, 126, 122, 48, 112, 110, 51, 45, 120, 114, 107, 119, 43, 109, 50, 127, 125, 93, 37, 121, 49, 22, 86, 101, 102, 58, 29, 54, 103, 123, 60, 52, 39, 115, 118, 124, 117, 57, 55, 26, 34, 53, 62, 97, 90, 63, 33, 47, 92, 59, 83, 61, 81, 19, 17, 23, 24, 111, 38, 87, 30, 20, 91, 89, 85, 84, 32, 98, 16, 21, 88, 36, 78, 80, 25, 100, 40, 104, 44, 79, 74, 108, 95, 28, 14, 27, 11, 31, 15, 12, 77, 94, 41, 76, 10, 96, 82, 13, 72, 18, 70, 75, 35, 105, 106, 42, 64, 71, 9, 0, 7, 67, 2, 8, 5, 73, 99, 66, 3, 6, 68, 69, 65, 1, 4], [46, 113, 56, 116, 126, 48, 110, 112, 122, 45, 51, 114, 107, 120, 43, 119, 109, 50, 127, 93, 125, 37, 121, 22, 86, 49, 101, 117, 58, 55, 29, 102, 53, 39, 118, 54, 103, 63, 124, 33, 115, 90, 123, 92, 60, 26, 34, 57, 62, 52, 97, 17, 61, 19, 81, 24, 111, 14, 83, 91, 85, 98, 23, 30, 16, 38, 87, 36, 89, 44, 11, 21, 59, 20, 47, 88, 74, 84, 28, 94, 95, 32, 15, 76, 79, 41, 80, 108, 10, 40, 78, 27, 31, 25, 13, 12, 100, 104, 75, 77, 42, 70, 72, 18, 96, 82, 106, 0, 105, 9, 35, 64, 67, 71, 99, 5, 3, 2, 8, 69, 7, 66, 73, 6, 65, 4, 68, 1], [46, 113, 116, 56, 126, 112, 48, 122, 110, 51, 45, 114, 120, 107, 43, 119, 109, 50, 127, 93, 125, 37, 49, 121, 117, 58, 63, 54, 22, 102, 39, 52, 101, 86, 62, 61, 29, 55, 53, 118, 103, 123, 115, 97, 57, 124, 33, 60, 90, 34, 26, 59, 23, 38, 83, 111, 24, 92, 17, 81, 14, 36, 30, 47, 21, 95, 87, 88, 85, 19, 20, 89, 78, 40, 16, 80, 98, 91, 44, 104, 84, 32, 28, 41, 11, 106, 76, 100, 25, 10, 74, 75, 15, 27, 31, 94, 105, 77, 42, 96, 79, 72, 108, 12, 82, 13, 70, 0, 66, 71, 35, 8, 67, 18, 6, 64, 7, 9, 4, 3, 5, 2, 99, 73, 1, 68, 69, 65], [46, 113, 116, 56, 126, 48, 110, 122, 112, 51, 45, 114, 107, 120, 43, 109, 50, 119, 127, 125, 93, 37, 121, 49, 22, 58, 101, 86, 117, 102, 52, 54, 97, 118, 63, 123, 29, 115, 39, 62, 55, 60, 124, 61, 103, 53, 34, 26, 38, 90, 57, 33, 87, 19, 95, 59, 81, 85, 92, 47, 24, 23, 83, 32, 98, 80, 11, 111, 84, 17, 16, 30, 89, 20, 78, 88, 91, 36, 14, 25, 21, 74, 104, 106, 15, 79, 100, 108, 40, 76, 28, 94, 12, 31, 10, 13, 72, 96, 44, 41, 27, 75, 77, 42, 8, 35, 6, 18, 71, 9, 0, 7, 82, 66, 73, 70, 105, 67, 64, 99, 3, 2, 4, 68, 69, 5, 1, 65], [46, 113, 116, 56, 126, 48, 110, 122, 112, 51, 114, 45, 107, 43, 120, 119, 109, 50, 127, 125, 93, 49, 37, 121, 22, 58, 86, 117, 54, 53, 123, 115, 63, 101, 124, 52, 118, 29, 39, 102, 55, 60, 26, 57, 103, 34, 90, 62, 97, 33, 81, 92, 78, 61, 87, 16, 38, 59, 83, 19, 111, 47, 17, 95, 32, 30, 91, 98, 11, 20, 84, 21, 23, 89, 88, 14, 36, 24, 25, 15, 44, 74, 28, 85, 108, 79, 76, 106, 12, 31, 94, 105, 13, 80, 104, 42, 75, 27, 41, 100, 82, 40, 77, 6, 10, 72, 8, 67, 18, 7, 35, 9, 96, 73, 99, 5, 0, 71, 3, 70, 66, 69, 2, 1, 68, 64, 4, 65], [46, 113, 116, 56, 48, 126, 112, 110, 122, 51, 114, 45, 120, 107, 43, 119, 50, 109, 127, 125, 93, 49, 37, 121, 54, 58, 117, 22, 53, 118, 39, 63, 86, 124, 29, 115, 55, 52, 101, 102, 123, 97, 103, 62, 61, 34, 57, 90, 26, 60, 33, 59, 92, 38, 17, 95, 81, 91, 83, 19, 87, 24, 47, 84, 16, 21, 88, 85, 111, 78, 108, 23, 89, 14, 30, 11, 32, 25, 44, 104, 36, 100, 28, 98, 74, 20, 80, 42, 79, 94, 41, 15, 106, 31, 76, 40, 75, 27, 82, 96, 77, 8, 64, 10, 13, 6, 105, 12, 71, 0, 18, 7, 3, 72, 9, 73, 67, 35, 2, 70, 99, 66, 5, 69, 68, 65, 1, 4], [46, 113, 116, 56, 126, 112, 122, 48, 110, 51, 114, 45, 120, 107, 119, 43, 109, 50, 125, 127, 93, 49, 121, 37, 123, 54, 22, 86, 117, 101, 58, 115, 52, 39, 102, 124, 118, 53, 29, 57, 60, 63, 26, 103, 61, 55, 34, 97, 62, 38, 90, 33, 92, 83, 47, 59, 17, 111, 88, 87, 81, 44, 19, 21, 91, 89, 24, 23, 32, 80, 36, 85, 78, 84, 98, 95, 30, 108, 14, 20, 74, 16, 94, 25, 27, 41, 31, 28, 106, 100, 79, 104, 12, 40, 15, 96, 10, 76, 11, 75, 77, 105, 18, 82, 13, 6, 35, 8, 73, 42, 7, 99, 64, 0, 72, 3, 71, 67, 66, 2, 9, 5, 69, 70, 4, 68, 65, 1], [46, 113, 116, 122, 56, 126, 48, 112, 110, 51, 114, 45, 107, 120, 119, 43, 109, 50, 127, 125, 93, 37, 49, 54, 22, 58, 86, 121, 101, 115, 117, 123, 29, 118, 39, 57, 52, 53, 124, 102, 97, 63, 26, 55, 90, 103, 60, 111, 61, 34, 59, 33, 92, 87, 47, 81, 62, 17, 19, 83, 85, 84, 23, 16, 98, 32, 36, 30, 91, 80, 78, 89, 38, 24, 95, 75, 14, 20, 88, 21, 31, 25, 44, 76, 74, 28, 79, 11, 12, 106, 15, 27, 13, 41, 10, 100, 104, 72, 96, 77, 105, 94, 108, 8, 82, 18, 42, 6, 40, 71, 9, 35, 73, 67, 7, 3, 70, 0, 64, 66, 5, 2, 69, 99, 1, 4, 68, 65], [46, 113, 116, 56, 122, 112, 48, 126, 110, 114, 51, 45, 120, 107, 43, 119, 109, 50, 125, 127, 93, 37, 22, 49, 86, 121, 115, 58, 54, 124, 63, 29, 102, 118, 53, 117, 39, 52, 123, 57, 101, 90, 55, 61, 26, 81, 34, 83, 19, 103, 97, 87, 60, 33, 59, 62, 23, 111, 24, 92, 80, 47, 84, 38, 17, 85, 89, 78, 21, 20, 88, 44, 91, 30, 16, 75, 12, 15, 25, 36, 95, 14, 74, 98, 10, 32, 31, 28, 76, 100, 104, 27, 13, 79, 41, 11, 94, 8, 77, 42, 40, 106, 96, 108, 18, 7, 9, 99, 72, 105, 35, 0, 82, 6, 71, 73, 70, 67, 5, 2, 66, 64, 3, 69, 1, 4, 68, 65], [46, 113, 116, 56, 112, 126, 48, 122, 110, 51, 45, 114, 120, 107, 119, 43, 109, 50, 127, 125, 93, 37, 124, 121, 22, 49, 58, 86, 115, 54, 118, 53, 101, 117, 102, 123, 29, 60, 39, 63, 55, 52, 61, 90, 34, 97, 62, 57, 103, 26, 33, 92, 38, 24, 83, 36, 59, 47, 111, 98, 19, 21, 32, 30, 88, 91, 17, 85, 95, 84, 87, 20, 81, 89, 23, 74, 14, 16, 104, 25, 100, 78, 75, 27, 80, 108, 44, 28, 41, 94, 79, 15, 31, 12, 96, 40, 105, 10, 11, 77, 35, 76, 8, 82, 13, 42, 72, 73, 106, 18, 70, 99, 64, 67, 71, 9, 66, 6, 7, 3, 69, 0, 5, 2, 1, 4, 65, 68], [46, 113, 122, 56, 116, 126, 112, 48, 110, 114, 45, 51, 107, 120, 119, 109, 43, 50, 127, 125, 93, 121, 123, 115, 49, 37, 101, 54, 58, 102, 118, 22, 124, 117, 86, 55, 53, 39, 61, 57, 103, 29, 60, 63, 52, 97, 90, 34, 62, 26, 92, 47, 38, 111, 33, 19, 81, 83, 17, 23, 44, 98, 59, 36, 87, 30, 95, 84, 32, 21, 85, 24, 91, 14, 108, 100, 78, 88, 89, 25, 75, 28, 94, 41, 79, 20, 80, 104, 10, 31, 16, 76, 74, 105, 15, 77, 13, 96, 40, 12, 27, 18, 106, 70, 82, 11, 42, 99, 8, 35, 2, 3, 9, 64, 72, 71, 69, 7, 73, 67, 5, 0, 66, 6, 65, 4, 68, 1], [46, 113, 116, 56, 126, 122, 112, 48, 110, 51, 114, 45, 107, 50, 120, 43, 119, 109, 127, 93, 125, 37, 49, 121, 58, 22, 55, 54, 118, 101, 115, 86, 124, 102, 39, 123, 29, 63, 53, 117, 57, 97, 90, 61, 52, 103, 62, 33, 59, 26, 60, 92, 19, 81, 17, 87, 34, 83, 47, 23, 85, 14, 75, 106, 16, 20, 38, 78, 88, 21, 111, 25, 84, 30, 24, 80, 95, 32, 91, 89, 44, 74, 10, 15, 8, 100, 31, 76, 12, 79, 11, 70, 41, 13, 98, 77, 94, 28, 104, 40, 96, 36, 27, 105, 108, 7, 42, 82, 72, 35, 9, 67, 0, 18, 64, 71, 69, 66, 73, 6, 99, 2, 3, 68, 5, 1, 4, 65], [46, 113, 116, 56, 48, 126, 122, 112, 110, 51, 114, 45, 119, 107, 43, 120, 109, 50, 127, 37, 93, 125, 49, 121, 54, 22, 123, 86, 118, 102, 57, 39, 58, 52, 55, 101, 62, 115, 53, 29, 124, 103, 34, 117, 63, 60, 26, 97, 33, 90, 61, 47, 59, 81, 78, 19, 17, 87, 95, 38, 85, 83, 84, 100, 75, 111, 21, 92, 89, 91, 16, 23, 14, 98, 32, 74, 44, 80, 104, 24, 30, 10, 41, 12, 79, 20, 88, 15, 36, 105, 31, 76, 40, 108, 25, 27, 28, 77, 64, 8, 11, 70, 3, 72, 96, 82, 35, 106, 18, 94, 67, 13, 2, 9, 0, 7, 71, 42, 5, 66, 73, 69, 6, 4, 1, 68, 99, 65], [46, 113, 116, 56, 122, 126, 48, 112, 110, 51, 114, 45, 119, 120, 107, 43, 50, 109, 127, 125, 93, 37, 121, 54, 49, 22, 118, 86, 101, 123, 63, 57, 102, 58, 117, 29, 53, 124, 52, 59, 103, 26, 115, 39, 97, 61, 62, 34, 60, 90, 33, 47, 55, 81, 92, 111, 87, 19, 23, 16, 17, 83, 38, 14, 80, 30, 84, 78, 24, 21, 44, 91, 98, 75, 15, 20, 88, 89, 12, 10, 85, 104, 25, 95, 79, 36, 32, 100, 31, 74, 40, 105, 13, 28, 76, 41, 77, 11, 27, 8, 106, 96, 108, 72, 18, 9, 94, 70, 82, 6, 71, 42, 3, 7, 67, 35, 66, 73, 64, 5, 0, 99, 69, 2, 68, 65, 1, 4], [46, 113, 116, 56, 122, 48, 112, 126, 110, 51, 45, 114, 107, 119, 120, 43, 50, 109, 127, 125, 93, 54, 37, 49, 121, 86, 22, 58, 101, 52, 102, 124, 118, 123, 63, 39, 57, 115, 53, 60, 29, 117, 103, 55, 90, 62, 97, 61, 34, 47, 33, 59, 26, 92, 83, 111, 17, 81, 91, 87, 44, 23, 38, 19, 98, 14, 32, 75, 85, 78, 88, 80, 24, 104, 21, 30, 89, 10, 100, 36, 16, 25, 79, 72, 74, 12, 20, 76, 95, 84, 28, 41, 15, 31, 40, 13, 108, 11, 105, 77, 35, 96, 71, 27, 8, 18, 94, 73, 6, 64, 82, 106, 66, 7, 70, 67, 0, 42, 9, 69, 99, 5, 3, 2, 68, 65, 1, 4], [46, 113, 116, 56, 122, 126, 48, 112, 110, 51, 45, 114, 120, 119, 107, 50, 43, 109, 93, 127, 125, 37, 22, 121, 54, 86, 101, 49, 52, 102, 58, 124, 39, 118, 123, 53, 29, 60, 103, 57, 90, 55, 61, 97, 34, 115, 63, 62, 117, 33, 92, 47, 26, 81, 83, 38, 59, 85, 111, 24, 19, 91, 98, 23, 87, 17, 89, 88, 14, 30, 78, 32, 21, 12, 84, 75, 44, 20, 95, 36, 16, 100, 80, 10, 15, 79, 25, 41, 76, 104, 27, 74, 77, 105, 13, 31, 72, 94, 40, 28, 18, 106, 108, 11, 96, 42, 6, 8, 82, 70, 9, 7, 35, 73, 71, 3, 64, 5, 99, 0, 69, 2, 66, 65, 67, 4, 68, 1]], "model.layers.21.self_attn.q_proj": [[103, 49, 112, 48, 97, 82, 15, 84, 29, 86, 12, 113, 99, 62, 8, 26, 54, 78, 13, 57, 4, 70, 96, 52, 90, 33, 17, 72, 23, 37, 80, 56, 89, 79, 77, 88, 73, 120, 98, 93, 24, 21, 20, 92, 18, 9, 27, 22, 59, 87, 83, 16, 5, 76, 94, 39, 19, 107, 85, 123, 14, 1, 124, 91, 46, 50, 28, 67, 31, 81, 108, 66, 25, 10, 6, 40, 51, 74, 95, 30, 122, 106, 102, 2, 116, 127, 44, 68, 115, 38, 60, 7, 125, 100, 75, 119, 109, 55, 63, 58, 34, 101, 53, 118, 61, 36, 121, 69, 32, 64, 117, 114, 47, 104, 35, 11, 126, 41, 110, 43, 45, 42, 71, 65, 111, 105, 3, 0], [49, 103, 112, 48, 97, 29, 113, 62, 56, 84, 87, 35, 38, 26, 123, 52, 82, 63, 95, 115, 120, 59, 57, 51, 124, 61, 116, 104, 86, 89, 53, 106, 119, 78, 45, 12, 122, 125, 60, 118, 126, 47, 40, 54, 117, 127, 50, 46, 88, 110, 108, 111, 55, 96, 30, 58, 114, 43, 37, 105, 34, 107, 44, 121, 102, 42, 15, 11, 19, 41, 109, 8, 100, 16, 99, 36, 92, 101, 31, 94, 32, 98, 39, 67, 23, 14, 25, 33, 85, 91, 3, 28, 17, 73, 70, 10, 4, 13, 18, 7, 22, 27, 90, 93, 24, 21, 68, 80, 71, 66, 5, 20, 83, 75, 76, 77, 9, 79, 69, 2, 65, 72, 81, 64, 6, 1, 74, 0], [112, 62, 49, 103, 48, 56, 52, 123, 122, 59, 120, 113, 50, 97, 115, 121, 51, 57, 54, 109, 46, 119, 63, 125, 35, 114, 118, 60, 105, 104, 117, 58, 108, 124, 29, 61, 127, 126, 42, 41, 110, 116, 45, 47, 88, 55, 53, 99, 87, 106, 43, 111, 107, 74, 3, 71, 37, 44, 82, 40, 38, 2, 102, 68, 6, 100, 95, 0, 84, 86, 101, 36, 30, 85, 34, 5, 26, 31, 98, 32, 14, 33, 27, 72, 94, 92, 96, 65, 89, 81, 9, 21, 90, 76, 18, 39, 91, 7, 28, 1, 19, 23, 83, 22, 11, 12, 25, 75, 93, 24, 67, 15, 17, 80, 16, 4, 73, 70, 77, 13, 20, 79, 10, 69, 78, 64, 8, 66], [112, 103, 49, 48, 97, 113, 29, 0, 65, 50, 26, 11, 95, 84, 78, 17, 54, 82, 31, 86, 62, 5, 56, 57, 123, 2, 66, 122, 88, 114, 70, 52, 42, 46, 118, 120, 59, 121, 4, 64, 8, 104, 115, 67, 125, 63, 116, 60, 108, 1, 106, 44, 35, 10, 127, 38, 119, 15, 75, 51, 12, 109, 13, 58, 117, 61, 93, 7, 3, 25, 124, 69, 126, 90, 110, 111, 45, 53, 47, 81, 99, 43, 34, 55, 96, 9, 94, 107, 37, 41, 6, 105, 100, 83, 72, 98, 74, 71, 102, 77, 85, 73, 22, 33, 40, 24, 92, 19, 87, 101, 32, 30, 89, 27, 68, 91, 14, 28, 23, 16, 20, 21, 36, 80, 39, 18, 76, 79], [102, 120, 118, 54, 53, 84, 56, 11, 15, 9, 18, 1, 67, 76, 6, 5, 87, 31, 71, 64, 66, 0, 93, 68, 77, 78, 89, 72, 13, 80, 25, 65, 86, 74, 46, 21, 109, 63, 48, 92, 69, 114, 52, 22, 32, 8, 127, 40, 126, 75, 16, 7, 10, 117, 107, 70, 115, 82, 4, 73, 57, 12, 116, 121, 113, 3, 94, 60, 38, 14, 24, 23, 79, 41, 45, 44, 110, 20, 91, 90, 98, 2, 85, 125, 103, 36, 124, 95, 17, 59, 30, 100, 47, 119, 50, 101, 62, 105, 29, 43, 34, 19, 42, 61, 99, 35, 33, 112, 108, 123, 97, 27, 51, 28, 39, 49, 96, 122, 106, 26, 37, 81, 104, 55, 83, 111, 88, 58], [102, 54, 118, 120, 84, 15, 18, 56, 72, 76, 53, 11, 25, 6, 93, 9, 78, 5, 80, 71, 31, 67, 89, 66, 74, 16, 22, 64, 1, 69, 3, 77, 46, 87, 48, 29, 92, 24, 85, 8, 117, 10, 13, 90, 65, 124, 40, 110, 75, 20, 107, 23, 109, 57, 12, 70, 83, 115, 125, 114, 79, 41, 103, 0, 42, 113, 99, 82, 63, 45, 86, 36, 14, 127, 73, 68, 32, 19, 44, 37, 39, 47, 116, 26, 61, 55, 88, 121, 51, 105, 7, 30, 81, 96, 112, 59, 123, 33, 95, 2, 38, 60, 49, 119, 4, 52, 126, 91, 17, 100, 34, 122, 97, 35, 50, 27, 104, 21, 111, 62, 98, 101, 43, 108, 28, 94, 106, 58], [102, 120, 54, 118, 56, 80, 18, 84, 76, 25, 9, 78, 93, 11, 89, 87, 31, 53, 15, 6, 71, 72, 5, 67, 77, 66, 13, 1, 64, 0, 22, 86, 74, 113, 46, 68, 65, 116, 115, 90, 32, 63, 26, 4, 81, 16, 7, 107, 8, 52, 82, 69, 40, 17, 125, 117, 73, 48, 92, 99, 36, 23, 70, 75, 12, 98, 43, 88, 51, 14, 126, 124, 19, 29, 55, 79, 37, 39, 20, 61, 114, 85, 3, 10, 122, 112, 49, 57, 109, 38, 127, 59, 30, 2, 27, 119, 24, 21, 33, 121, 35, 104, 83, 60, 100, 103, 105, 45, 34, 91, 42, 96, 94, 110, 28, 108, 62, 106, 41, 111, 101, 123, 50, 58, 47, 95, 97, 44], [102, 120, 54, 118, 56, 78, 53, 84, 72, 31, 18, 5, 15, 1, 93, 76, 71, 67, 66, 11, 9, 89, 25, 87, 13, 64, 65, 0, 117, 6, 77, 46, 68, 80, 73, 70, 107, 2, 26, 75, 22, 88, 8, 3, 39, 113, 32, 48, 69, 12, 4, 63, 82, 85, 60, 19, 109, 127, 74, 16, 116, 90, 125, 27, 28, 52, 79, 14, 110, 86, 29, 41, 126, 98, 114, 81, 124, 7, 34, 55, 92, 62, 105, 91, 96, 44, 101, 51, 115, 57, 40, 17, 42, 20, 104, 111, 36, 61, 119, 24, 94, 83, 43, 45, 50, 58, 121, 33, 38, 122, 59, 108, 100, 23, 10, 99, 106, 112, 103, 97, 95, 123, 30, 47, 37, 21, 35, 49], [59, 106, 60, 36, 99, 62, 28, 119, 42, 87, 123, 55, 61, 85, 121, 18, 53, 127, 31, 114, 32, 98, 89, 122, 124, 86, 47, 56, 92, 54, 117, 104, 112, 111, 126, 116, 125, 58, 115, 13, 118, 30, 41, 57, 113, 25, 90, 107, 100, 49, 110, 97, 109, 120, 46, 43, 50, 39, 38, 51, 52, 105, 45, 44, 101, 103, 48, 91, 63, 108, 35, 93, 102, 21, 80, 94, 29, 37, 34, 33, 40, 14, 75, 19, 79, 20, 96, 27, 4, 26, 15, 65, 22, 74, 23, 95, 82, 7, 17, 71, 83, 24, 6, 88, 77, 76, 73, 9, 68, 12, 8, 1, 84, 64, 81, 2, 69, 67, 3, 66, 16, 0, 5, 11, 10, 78, 70, 72], [106, 99, 59, 80, 28, 13, 20, 74, 8, 14, 86, 6, 60, 42, 75, 4, 18, 65, 58, 55, 30, 32, 85, 89, 64, 2, 67, 114, 31, 111, 27, 24, 91, 10, 119, 25, 62, 92, 21, 95, 22, 90, 19, 35, 107, 84, 81, 72, 82, 36, 96, 123, 16, 7, 70, 102, 112, 97, 78, 125, 17, 101, 57, 77, 79, 11, 88, 29, 87, 56, 12, 23, 69, 9, 5, 68, 73, 109, 46, 53, 113, 34, 71, 93, 0, 83, 61, 15, 127, 122, 94, 51, 45, 54, 37, 26, 66, 41, 116, 76, 117, 3, 1, 108, 52, 121, 104, 98, 115, 33, 43, 100, 120, 39, 110, 118, 105, 38, 103, 40, 124, 49, 47, 48, 50, 63, 44, 126], [106, 99, 59, 20, 14, 80, 86, 75, 74, 32, 6, 67, 8, 28, 64, 92, 60, 42, 18, 111, 25, 58, 55, 4, 2, 3, 91, 114, 69, 29, 87, 97, 65, 123, 102, 66, 127, 119, 57, 30, 1, 62, 107, 90, 36, 95, 31, 7, 88, 113, 112, 35, 12, 22, 70, 10, 122, 96, 21, 68, 89, 24, 73, 82, 11, 109, 17, 56, 125, 13, 45, 16, 72, 100, 61, 77, 78, 53, 79, 27, 15, 5, 116, 81, 19, 9, 71, 34, 84, 104, 0, 26, 108, 51, 23, 101, 85, 94, 54, 76, 98, 83, 121, 43, 48, 37, 52, 39, 115, 93, 41, 33, 46, 105, 103, 110, 38, 117, 120, 47, 118, 40, 50, 44, 49, 124, 126, 63], [106, 60, 36, 62, 42, 119, 58, 123, 55, 112, 61, 114, 121, 59, 87, 124, 111, 53, 125, 41, 115, 47, 110, 118, 63, 103, 122, 101, 127, 113, 85, 99, 18, 126, 54, 104, 56, 50, 120, 49, 107, 25, 51, 46, 108, 52, 57, 48, 28, 116, 44, 109, 43, 45, 100, 117, 105, 40, 32, 92, 89, 39, 35, 38, 102, 90, 27, 31, 37, 15, 34, 98, 33, 91, 88, 19, 30, 97, 86, 26, 93, 96, 95, 21, 94, 29, 13, 83, 22, 24, 17, 23, 76, 79, 82, 20, 81, 77, 71, 14, 80, 73, 12, 7, 9, 84, 65, 75, 4, 74, 68, 78, 1, 16, 6, 69, 11, 67, 2, 5, 8, 64, 10, 70, 3, 0, 66, 72], [102, 57, 58, 59, 114, 90, 87, 127, 26, 96, 60, 29, 38, 93, 85, 74, 82, 15, 61, 18, 110, 21, 46, 35, 48, 71, 111, 121, 62, 12, 44, 51, 84, 113, 32, 22, 49, 117, 79, 16, 50, 112, 13, 41, 119, 109, 86, 63, 43, 124, 72, 100, 4, 23, 123, 53, 52, 105, 115, 56, 118, 122, 107, 40, 106, 54, 120, 116, 98, 45, 126, 125, 55, 2, 39, 108, 33, 104, 101, 17, 94, 27, 37, 30, 103, 47, 31, 95, 20, 42, 34, 5, 91, 25, 88, 8, 28, 97, 0, 92, 89, 36, 99, 11, 83, 81, 3, 80, 19, 24, 78, 68, 1, 75, 9, 77, 14, 70, 66, 73, 65, 76, 7, 6, 64, 10, 69, 67], [102, 58, 57, 59, 114, 90, 87, 127, 121, 96, 26, 29, 38, 15, 93, 85, 49, 74, 41, 110, 82, 32, 60, 84, 63, 43, 109, 113, 50, 116, 117, 79, 62, 21, 124, 35, 52, 54, 105, 22, 48, 53, 46, 56, 13, 55, 18, 20, 125, 44, 71, 51, 12, 86, 126, 123, 61, 119, 111, 112, 108, 118, 122, 115, 120, 103, 45, 95, 23, 104, 106, 36, 47, 101, 42, 39, 31, 16, 100, 107, 33, 40, 89, 80, 28, 98, 37, 78, 2, 97, 88, 94, 34, 30, 4, 91, 99, 27, 25, 0, 72, 92, 77, 24, 83, 11, 19, 17, 73, 5, 81, 14, 68, 3, 75, 64, 70, 1, 8, 7, 66, 6, 9, 76, 10, 65, 67, 69], [102, 59, 58, 114, 57, 90, 87, 26, 15, 29, 85, 96, 74, 93, 121, 71, 18, 82, 21, 38, 109, 13, 79, 47, 44, 84, 48, 2, 116, 117, 12, 115, 49, 32, 86, 60, 54, 0, 4, 119, 55, 127, 105, 124, 35, 45, 112, 5, 43, 46, 113, 110, 17, 118, 63, 53, 52, 123, 41, 111, 37, 72, 122, 65, 61, 23, 120, 51, 50, 66, 25, 108, 1, 126, 62, 81, 56, 106, 89, 3, 22, 20, 104, 125, 16, 100, 24, 101, 36, 107, 83, 98, 8, 91, 28, 40, 94, 67, 19, 103, 31, 39, 42, 7, 68, 77, 10, 99, 95, 64, 27, 33, 75, 6, 88, 92, 73, 78, 30, 9, 34, 70, 80, 97, 11, 76, 69, 14], [102, 59, 58, 57, 127, 114, 90, 60, 26, 87, 96, 15, 85, 71, 38, 29, 74, 121, 18, 93, 105, 35, 12, 84, 49, 108, 82, 79, 41, 47, 4, 120, 110, 32, 22, 54, 21, 52, 119, 43, 118, 46, 122, 117, 124, 62, 20, 123, 116, 109, 16, 111, 104, 44, 55, 99, 112, 113, 13, 115, 61, 86, 63, 89, 40, 5, 2, 48, 107, 50, 45, 30, 103, 23, 98, 51, 0, 39, 106, 94, 125, 36, 126, 37, 33, 56, 34, 53, 42, 101, 100, 24, 27, 97, 8, 83, 17, 66, 28, 73, 31, 25, 95, 68, 88, 92, 19, 91, 1, 78, 3, 77, 72, 81, 7, 65, 64, 11, 70, 80, 9, 76, 14, 75, 10, 69, 6, 67], [126, 104, 99, 87, 28, 32, 83, 92, 85, 81, 127, 110, 79, 47, 42, 76, 115, 111, 119, 72, 11, 15, 30, 74, 51, 113, 31, 108, 21, 95, 116, 78, 37, 50, 24, 57, 122, 45, 6, 48, 25, 12, 90, 96, 5, 121, 56, 68, 26, 53, 23, 107, 55, 106, 120, 62, 59, 101, 70, 123, 89, 17, 49, 43, 46, 109, 105, 41, 97, 13, 19, 2, 33, 112, 93, 39, 52, 124, 82, 40, 65, 114, 117, 34, 0, 125, 102, 61, 29, 118, 3, 44, 80, 60, 20, 84, 58, 54, 100, 88, 27, 22, 94, 36, 98, 103, 64, 18, 86, 91, 75, 63, 38, 66, 16, 10, 67, 14, 9, 77, 1, 73, 4, 71, 35, 7, 69, 8], [126, 104, 99, 87, 28, 83, 85, 32, 127, 92, 47, 115, 81, 79, 119, 51, 106, 76, 50, 118, 72, 48, 74, 111, 113, 42, 108, 15, 116, 45, 120, 31, 122, 121, 6, 58, 5, 61, 110, 57, 11, 41, 12, 70, 68, 37, 53, 91, 54, 21, 75, 43, 9, 97, 89, 49, 23, 38, 46, 30, 105, 13, 55, 59, 56, 40, 35, 65, 0, 62, 123, 17, 117, 109, 107, 34, 78, 33, 114, 60, 95, 103, 98, 112, 25, 67, 52, 39, 3, 93, 101, 36, 82, 19, 63, 102, 125, 18, 44, 26, 90, 29, 94, 2, 24, 66, 124, 20, 96, 86, 100, 84, 27, 73, 88, 80, 1, 69, 14, 22, 10, 16, 7, 71, 77, 4, 8, 64], [126, 104, 99, 87, 28, 32, 85, 81, 92, 83, 115, 47, 127, 79, 110, 42, 76, 45, 121, 116, 58, 95, 108, 68, 31, 15, 78, 72, 41, 113, 60, 86, 106, 5, 54, 122, 117, 11, 119, 96, 50, 9, 24, 26, 6, 7, 52, 91, 22, 0, 101, 21, 57, 8, 12, 90, 17, 111, 66, 13, 44, 18, 23, 93, 94, 30, 55, 105, 114, 70, 118, 37, 49, 51, 25, 123, 48, 97, 88, 46, 74, 63, 38, 56, 39, 43, 59, 27, 120, 82, 34, 53, 62, 89, 102, 33, 103, 19, 80, 36, 61, 98, 124, 125, 14, 112, 109, 29, 100, 16, 107, 73, 84, 2, 20, 10, 35, 4, 77, 40, 71, 3, 64, 67, 1, 75, 65, 69], [104, 126, 99, 28, 87, 85, 32, 83, 81, 79, 92, 127, 116, 89, 111, 76, 47, 48, 25, 50, 72, 122, 15, 74, 54, 6, 11, 106, 51, 12, 119, 2, 52, 68, 105, 43, 58, 57, 42, 118, 3, 45, 108, 1, 13, 14, 31, 70, 23, 94, 5, 0, 115, 17, 29, 110, 41, 107, 113, 64, 93, 21, 46, 63, 120, 30, 123, 26, 61, 82, 39, 8, 125, 62, 102, 59, 112, 75, 90, 124, 53, 37, 60, 9, 19, 86, 65, 101, 84, 78, 73, 117, 69, 44, 40, 109, 33, 114, 77, 18, 24, 121, 34, 56, 38, 103, 66, 10, 95, 36, 100, 49, 97, 55, 98, 27, 7, 22, 67, 4, 35, 91, 88, 96, 20, 16, 80, 71], [62, 126, 48, 39, 116, 53, 30, 27, 122, 88, 20, 35, 117, 76, 121, 15, 91, 106, 112, 83, 55, 57, 79, 81, 94, 58, 86, 32, 123, 124, 52, 46, 49, 72, 19, 84, 33, 60, 98, 63, 89, 115, 100, 127, 50, 29, 24, 34, 56, 107, 114, 5, 41, 105, 108, 109, 51, 61, 44, 113, 118, 110, 26, 125, 47, 42, 23, 119, 87, 103, 18, 28, 43, 120, 36, 111, 17, 54, 22, 37, 59, 45, 31, 75, 95, 104, 93, 10, 97, 13, 102, 12, 85, 99, 40, 25, 38, 21, 74, 101, 90, 92, 78, 2, 14, 8, 82, 80, 96, 9, 16, 77, 3, 1, 11, 71, 73, 69, 6, 7, 70, 67, 66, 68, 4, 0, 65, 64], [116, 126, 39, 62, 48, 55, 57, 117, 123, 46, 53, 125, 111, 49, 121, 30, 50, 60, 41, 20, 122, 52, 127, 51, 120, 63, 114, 124, 89, 56, 115, 35, 59, 58, 113, 118, 43, 109, 32, 86, 54, 112, 47, 61, 88, 42, 106, 119, 108, 44, 45, 110, 107, 37, 34, 91, 40, 103, 105, 27, 104, 81, 23, 92, 102, 80, 38, 98, 29, 94, 101, 33, 28, 99, 36, 22, 87, 93, 9, 79, 100, 90, 73, 95, 96, 13, 97, 25, 31, 76, 26, 14, 82, 85, 84, 68, 83, 17, 1, 77, 21, 15, 75, 7, 78, 18, 2, 24, 16, 3, 71, 70, 11, 72, 4, 67, 0, 74, 12, 10, 5, 66, 65, 6, 19, 64, 8, 69], [126, 62, 116, 39, 48, 117, 55, 57, 53, 111, 123, 112, 49, 122, 114, 58, 120, 118, 124, 50, 20, 35, 127, 121, 42, 51, 30, 109, 88, 46, 56, 32, 52, 60, 28, 41, 115, 119, 43, 113, 125, 59, 47, 63, 106, 108, 103, 94, 44, 61, 110, 54, 101, 45, 89, 104, 107, 40, 86, 34, 38, 102, 100, 105, 81, 99, 37, 95, 87, 29, 93, 23, 92, 14, 36, 98, 80, 33, 97, 79, 21, 26, 31, 90, 27, 85, 91, 18, 22, 96, 25, 13, 83, 15, 77, 17, 84, 24, 76, 9, 82, 12, 69, 73, 75, 70, 78, 2, 19, 11, 16, 72, 10, 64, 67, 66, 3, 4, 0, 6, 71, 65, 68, 74, 1, 8, 5, 7], [126, 116, 39, 48, 55, 57, 27, 123, 30, 117, 83, 86, 122, 52, 37, 20, 53, 26, 62, 118, 110, 89, 112, 91, 72, 17, 19, 35, 32, 121, 16, 41, 94, 10, 79, 4, 74, 76, 88, 15, 125, 124, 114, 60, 25, 120, 115, 49, 29, 107, 56, 63, 22, 13, 106, 97, 50, 58, 51, 93, 42, 46, 34, 113, 127, 80, 111, 109, 81, 61, 108, 28, 5, 105, 66, 95, 100, 59, 31, 47, 75, 119, 38, 24, 23, 87, 33, 92, 44, 98, 9, 54, 21, 78, 45, 14, 36, 18, 77, 102, 84, 67, 7, 104, 43, 101, 68, 103, 8, 73, 6, 12, 90, 40, 11, 99, 82, 69, 85, 0, 71, 65, 64, 96, 70, 2, 3, 1], [55, 103, 62, 52, 117, 98, 25, 57, 89, 92, 115, 28, 119, 54, 20, 83, 22, 78, 120, 126, 48, 121, 73, 19, 58, 50, 76, 27, 51, 82, 45, 125, 60, 111, 49, 32, 109, 12, 123, 112, 107, 94, 81, 124, 53, 40, 33, 113, 110, 38, 61, 118, 114, 43, 42, 116, 86, 17, 100, 84, 59, 14, 63, 47, 46, 108, 127, 97, 79, 95, 101, 18, 56, 96, 122, 106, 88, 44, 5, 104, 105, 41, 6, 37, 77, 99, 36, 15, 35, 8, 75, 39, 70, 93, 80, 30, 102, 91, 26, 72, 31, 16, 90, 3, 23, 85, 29, 66, 24, 64, 71, 34, 68, 69, 21, 1, 11, 9, 87, 0, 74, 67, 13, 65, 7, 10, 2, 4], [55, 103, 98, 20, 25, 89, 120, 92, 112, 51, 117, 28, 126, 78, 54, 84, 48, 62, 57, 73, 58, 52, 50, 121, 32, 118, 125, 99, 95, 127, 22, 108, 56, 119, 122, 59, 124, 110, 116, 83, 104, 109, 47, 61, 15, 111, 106, 113, 46, 53, 115, 49, 101, 82, 43, 79, 63, 114, 100, 107, 40, 44, 37, 67, 123, 60, 24, 97, 45, 35, 102, 27, 38, 42, 31, 11, 76, 74, 88, 105, 87, 36, 68, 91, 81, 23, 71, 94, 29, 19, 18, 26, 93, 33, 85, 30, 41, 90, 21, 39, 17, 9, 96, 86, 12, 10, 77, 3, 14, 80, 69, 34, 1, 8, 16, 2, 70, 13, 65, 6, 75, 72, 4, 7, 5, 66, 64, 0], [55, 103, 57, 62, 25, 98, 89, 83, 92, 78, 117, 20, 52, 54, 28, 120, 112, 22, 126, 125, 115, 48, 32, 76, 84, 61, 73, 51, 118, 50, 58, 19, 119, 41, 116, 60, 53, 64, 8, 108, 5, 56, 113, 105, 109, 27, 59, 49, 46, 127, 121, 95, 111, 114, 47, 12, 82, 124, 63, 123, 110, 66, 43, 65, 106, 44, 107, 30, 17, 122, 6, 38, 104, 3, 39, 79, 99, 45, 16, 72, 77, 81, 102, 86, 40, 13, 37, 94, 100, 36, 91, 14, 42, 75, 93, 2, 35, 23, 69, 101, 29, 97, 33, 90, 85, 34, 26, 70, 68, 31, 88, 24, 74, 10, 80, 96, 1, 15, 87, 18, 0, 21, 9, 67, 11, 7, 4, 71], [55, 103, 62, 52, 98, 20, 82, 92, 89, 22, 25, 117, 63, 51, 108, 112, 84, 57, 58, 54, 73, 56, 125, 50, 28, 44, 71, 119, 124, 48, 32, 79, 11, 15, 109, 87, 120, 74, 46, 53, 49, 43, 18, 97, 42, 126, 121, 110, 116, 77, 59, 113, 101, 45, 122, 61, 115, 21, 39, 107, 114, 106, 37, 30, 118, 26, 88, 123, 75, 111, 2, 104, 100, 31, 41, 127, 68, 14, 38, 60, 93, 99, 81, 102, 33, 94, 29, 17, 40, 105, 95, 85, 1, 47, 83, 23, 35, 7, 10, 27, 12, 8, 24, 90, 91, 0, 80, 19, 86, 4, 70, 96, 13, 36, 78, 16, 67, 69, 34, 6, 3, 76, 9, 5, 66, 72, 65, 64], [121, 127, 56, 55, 39, 118, 63, 98, 31, 51, 122, 89, 25, 57, 124, 120, 119, 84, 112, 125, 18, 105, 92, 53, 62, 59, 58, 52, 115, 15, 50, 49, 110, 28, 23, 47, 76, 54, 108, 60, 117, 116, 61, 114, 111, 113, 106, 126, 123, 10, 46, 30, 16, 107, 40, 24, 104, 48, 32, 96, 102, 43, 101, 6, 44, 90, 45, 41, 80, 87, 109, 99, 88, 42, 94, 79, 22, 36, 100, 14, 83, 95, 38, 37, 85, 69, 82, 93, 19, 75, 20, 86, 27, 77, 9, 81, 35, 97, 8, 34, 74, 71, 64, 13, 78, 2, 26, 103, 33, 12, 67, 72, 29, 65, 21, 11, 5, 91, 70, 68, 7, 17, 3, 1, 4, 0, 73, 66], [55, 127, 56, 39, 121, 118, 63, 122, 51, 89, 31, 84, 57, 120, 125, 18, 124, 119, 105, 98, 116, 62, 112, 117, 52, 53, 25, 115, 59, 50, 92, 95, 28, 76, 60, 15, 61, 49, 106, 47, 96, 111, 58, 54, 48, 123, 114, 126, 40, 113, 23, 108, 90, 85, 110, 46, 45, 107, 43, 44, 30, 109, 36, 102, 16, 101, 41, 38, 42, 79, 104, 24, 10, 103, 99, 20, 100, 22, 37, 35, 14, 87, 32, 33, 34, 97, 93, 88, 81, 80, 83, 9, 86, 82, 77, 27, 94, 29, 91, 78, 19, 75, 8, 74, 13, 21, 72, 71, 4, 26, 12, 17, 70, 73, 2, 6, 11, 5, 68, 65, 0, 69, 1, 64, 7, 3, 67, 66], [127, 39, 56, 63, 118, 55, 31, 98, 89, 51, 122, 57, 84, 92, 53, 25, 120, 125, 62, 105, 47, 115, 54, 96, 30, 126, 119, 52, 124, 18, 112, 114, 87, 59, 110, 116, 40, 44, 50, 58, 49, 90, 48, 113, 103, 85, 117, 60, 107, 23, 45, 16, 123, 111, 61, 121, 95, 106, 43, 76, 108, 102, 21, 35, 46, 104, 42, 79, 109, 26, 22, 34, 99, 41, 101, 83, 93, 15, 36, 32, 28, 38, 91, 37, 100, 24, 20, 27, 33, 19, 82, 97, 81, 94, 10, 80, 86, 74, 88, 29, 14, 77, 72, 17, 75, 8, 12, 13, 78, 9, 5, 69, 71, 6, 4, 11, 2, 70, 66, 68, 64, 73, 65, 67, 3, 7, 1, 0], [56, 39, 127, 55, 121, 118, 51, 31, 63, 122, 57, 84, 92, 25, 105, 120, 124, 62, 59, 115, 119, 125, 53, 50, 112, 52, 98, 47, 58, 116, 60, 49, 95, 89, 61, 113, 126, 110, 123, 114, 117, 54, 18, 44, 111, 41, 79, 48, 43, 108, 45, 90, 23, 76, 109, 101, 106, 104, 107, 42, 46, 96, 40, 102, 88, 100, 38, 10, 28, 15, 37, 36, 99, 35, 22, 29, 94, 97, 32, 103, 30, 82, 33, 83, 16, 86, 93, 77, 34, 8, 21, 27, 75, 87, 20, 85, 24, 80, 9, 72, 5, 91, 26, 81, 12, 13, 74, 17, 19, 71, 78, 14, 2, 4, 70, 73, 6, 69, 64, 3, 11, 67, 66, 0, 68, 7, 1, 65]], "model.layers.21.self_attn.k_proj": [[112, 49, 39, 86, 93, 57, 84, 15, 33, 26, 54, 123, 35, 50, 82, 122, 12, 8, 121, 52, 59, 64, 119, 60, 113, 116, 127, 118, 13, 125, 109, 58, 51, 73, 126, 117, 70, 111, 114, 53, 120, 55, 108, 29, 63, 48, 115, 78, 66, 10, 110, 46, 32, 47, 56, 43, 61, 44, 45, 40, 107, 88, 124, 62, 41, 68, 87, 42, 101, 106, 105, 16, 102, 38, 36, 65, 17, 95, 104, 100, 34, 94, 98, 89, 67, 27, 5, 92, 24, 4, 91, 37, 9, 30, 6, 7, 83, 96, 19, 80, 85, 28, 81, 69, 31, 21, 99, 22, 3, 90, 75, 71, 25, 2, 97, 11, 79, 74, 23, 18, 1, 77, 20, 14, 76, 72, 0, 103], [120, 54, 64, 118, 9, 84, 78, 76, 15, 5, 18, 53, 71, 11, 72, 65, 56, 38, 6, 67, 25, 2, 3, 0, 66, 13, 87, 29, 80, 4, 95, 69, 1, 89, 93, 22, 85, 102, 92, 31, 10, 70, 7, 116, 68, 43, 86, 73, 40, 24, 88, 17, 90, 19, 8, 50, 110, 32, 39, 114, 26, 112, 55, 63, 57, 115, 46, 27, 83, 21, 124, 49, 48, 99, 36, 52, 45, 113, 51, 16, 111, 33, 74, 109, 91, 47, 28, 103, 125, 96, 81, 30, 59, 41, 105, 23, 98, 60, 108, 97, 121, 119, 100, 117, 77, 75, 34, 62, 94, 12, 127, 58, 61, 82, 35, 126, 122, 123, 44, 42, 104, 14, 37, 107, 20, 101, 106, 79], [42, 59, 35, 86, 55, 64, 8, 119, 58, 47, 80, 6, 14, 20, 92, 74, 114, 75, 25, 2, 62, 60, 18, 53, 127, 106, 13, 116, 112, 123, 57, 61, 56, 122, 4, 50, 125, 118, 95, 54, 87, 65, 67, 124, 96, 117, 51, 111, 126, 109, 48, 99, 110, 121, 49, 113, 103, 52, 69, 45, 120, 108, 30, 115, 85, 46, 36, 38, 90, 44, 76, 63, 40, 28, 32, 26, 91, 5, 100, 43, 83, 107, 94, 0, 98, 37, 3, 41, 101, 33, 105, 34, 15, 71, 97, 17, 102, 93, 79, 1, 104, 39, 27, 68, 73, 9, 88, 19, 31, 23, 29, 66, 21, 7, 24, 89, 72, 81, 12, 84, 16, 22, 77, 11, 78, 82, 10, 70], [38, 57, 59, 58, 93, 32, 26, 85, 87, 84, 15, 18, 74, 71, 0, 13, 123, 45, 52, 12, 111, 121, 22, 126, 63, 116, 110, 1, 55, 117, 42, 127, 16, 107, 125, 108, 62, 51, 114, 54, 61, 109, 56, 53, 60, 124, 99, 122, 25, 106, 3, 112, 46, 115, 48, 118, 41, 119, 49, 5, 43, 50, 89, 44, 105, 102, 4, 120, 2, 24, 104, 75, 47, 113, 68, 35, 101, 95, 14, 103, 39, 17, 40, 66, 19, 69, 100, 83, 78, 36, 70, 27, 81, 9, 34, 98, 88, 37, 33, 92, 30, 28, 97, 72, 82, 31, 94, 91, 11, 73, 8, 80, 7, 20, 29, 86, 6, 77, 65, 23, 21, 67, 76, 90, 64, 79, 96, 10], [40, 126, 83, 81, 92, 35, 115, 85, 87, 15, 74, 111, 96, 89, 72, 76, 46, 42, 79, 52, 122, 12, 68, 58, 0, 28, 70, 106, 127, 114, 48, 6, 66, 5, 14, 54, 44, 57, 116, 11, 119, 31, 2, 56, 41, 1, 3, 43, 123, 110, 65, 125, 32, 118, 60, 75, 107, 9, 109, 29, 50, 82, 112, 120, 78, 49, 101, 93, 7, 13, 25, 121, 84, 59, 77, 88, 90, 86, 24, 34, 64, 8, 67, 108, 94, 30, 53, 20, 61, 47, 27, 39, 55, 117, 113, 69, 100, 26, 36, 22, 33, 105, 97, 103, 99, 102, 91, 51, 124, 95, 73, 98, 38, 23, 37, 16, 62, 45, 4, 63, 80, 10, 18, 71, 21, 19, 17, 104], [126, 103, 116, 86, 99, 55, 96, 52, 94, 62, 48, 26, 53, 112, 57, 109, 42, 91, 88, 63, 28, 58, 113, 56, 123, 20, 29, 110, 121, 102, 117, 50, 114, 47, 92, 124, 83, 35, 13, 115, 60, 81, 119, 127, 120, 59, 105, 125, 51, 37, 97, 43, 98, 49, 122, 61, 54, 111, 107, 95, 89, 46, 30, 33, 44, 79, 45, 118, 106, 10, 70, 40, 101, 108, 15, 36, 31, 104, 34, 7, 76, 27, 41, 85, 65, 38, 100, 72, 73, 87, 23, 66, 21, 14, 93, 0, 18, 5, 3, 90, 32, 17, 80, 25, 82, 6, 39, 71, 68, 16, 4, 8, 74, 24, 22, 84, 75, 77, 78, 12, 19, 9, 2, 11, 67, 69, 1, 64], [55, 39, 34, 22, 28, 89, 62, 125, 78, 117, 20, 53, 58, 50, 51, 54, 83, 109, 96, 112, 61, 59, 44, 113, 49, 114, 48, 121, 118, 119, 111, 60, 116, 63, 124, 126, 46, 127, 8, 47, 52, 64, 5, 56, 123, 82, 73, 43, 120, 91, 42, 122, 115, 12, 106, 1, 108, 110, 45, 30, 40, 18, 104, 79, 105, 107, 95, 57, 37, 93, 100, 81, 98, 23, 76, 41, 70, 77, 36, 3, 38, 66, 102, 27, 26, 90, 101, 35, 33, 31, 2, 32, 4, 11, 85, 99, 92, 10, 75, 24, 88, 97, 0, 80, 6, 65, 15, 17, 29, 21, 71, 94, 69, 103, 16, 25, 74, 87, 84, 13, 19, 7, 67, 68, 14, 9, 72, 86], [103, 127, 56, 34, 22, 55, 121, 95, 28, 118, 63, 122, 89, 116, 51, 125, 115, 62, 119, 57, 59, 58, 60, 53, 43, 52, 38, 124, 49, 48, 47, 117, 114, 61, 84, 92, 120, 123, 46, 106, 111, 44, 112, 113, 54, 126, 107, 50, 108, 71, 45, 23, 105, 29, 80, 109, 42, 75, 110, 40, 104, 35, 16, 41, 78, 101, 99, 37, 27, 98, 96, 102, 100, 18, 36, 10, 12, 64, 25, 33, 81, 86, 97, 30, 94, 87, 68, 79, 91, 39, 93, 88, 13, 90, 32, 66, 85, 83, 82, 17, 24, 26, 69, 21, 31, 1, 9, 20, 6, 8, 73, 14, 77, 19, 4, 5, 7, 76, 11, 65, 15, 74, 67, 2, 3, 72, 0, 70]], "model.layers.21.self_attn.qk_proj": [[126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 116, 42, 102, 118, 62, 53, 106, 121, 39, 28, 48, 103, 84, 20, 79, 119, 114, 25, 38, 15, 89, 29, 113, 86, 82, 122, 92, 40, 60, 22, 52, 23, 87, 99, 90, 35, 115, 18, 32, 12, 14, 111, 46, 78, 76, 6, 85, 72, 50, 117, 93, 51, 64, 75, 61, 21, 96, 125, 19, 10, 43, 26, 63, 123, 30, 108, 31, 34, 0, 83, 110, 124, 9, 45, 17, 44, 47, 8, 16, 11, 74, 80, 104, 73, 105, 5, 7, 69, 66, 95, 109, 67, 81, 107, 2, 97, 65, 13, 1, 71, 98, 68, 27, 77, 37, 33, 3, 41, 70, 36, 24, 88, 101, 94, 91, 4, 100], [126, 55, 120, 54, 59, 112, 49, 127, 57, 58, 56, 116, 42, 102, 118, 62, 53, 106, 121, 39, 28, 103, 48, 79, 20, 114, 84, 25, 29, 38, 89, 113, 119, 92, 22, 23, 60, 15, 99, 90, 52, 122, 87, 82, 12, 115, 86, 32, 35, 18, 78, 14, 51, 111, 40, 6, 76, 72, 117, 123, 93, 85, 75, 50, 125, 46, 0, 43, 44, 63, 108, 19, 26, 17, 21, 61, 64, 47, 30, 69, 31, 66, 124, 8, 74, 73, 11, 9, 96, 10, 109, 105, 104, 34, 110, 45, 2, 5, 83, 7, 95, 71, 13, 67, 27, 80, 16, 3, 88, 81, 33, 98, 1, 65, 107, 36, 97, 77, 41, 37, 68, 70, 4, 101, 91, 94, 24, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 56, 58, 116, 42, 102, 62, 106, 118, 53, 121, 28, 39, 103, 84, 48, 29, 20, 25, 15, 99, 22, 92, 23, 122, 119, 38, 79, 114, 86, 89, 115, 90, 113, 60, 87, 35, 117, 52, 78, 40, 82, 18, 76, 14, 12, 111, 32, 6, 51, 64, 85, 72, 46, 0, 96, 124, 44, 125, 75, 63, 93, 31, 21, 69, 43, 30, 34, 108, 2, 50, 19, 123, 10, 110, 5, 74, 8, 17, 65, 66, 11, 47, 109, 83, 9, 104, 61, 26, 98, 73, 70, 16, 1, 95, 7, 80, 67, 13, 3, 81, 77, 41, 105, 45, 36, 88, 27, 71, 107, 4, 68, 33, 24, 97, 37, 91, 101, 94, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 56, 58, 116, 42, 102, 118, 62, 121, 106, 53, 39, 28, 48, 103, 84, 20, 89, 79, 15, 29, 92, 119, 22, 25, 99, 113, 23, 60, 38, 52, 115, 40, 14, 82, 122, 90, 35, 18, 114, 12, 86, 78, 111, 76, 32, 87, 117, 46, 85, 6, 0, 17, 51, 72, 125, 96, 50, 30, 5, 11, 75, 93, 63, 123, 43, 110, 69, 47, 44, 34, 70, 19, 64, 2, 8, 10, 104, 83, 108, 124, 31, 21, 26, 61, 16, 66, 109, 65, 73, 74, 1, 9, 77, 98, 27, 81, 95, 45, 105, 67, 80, 7, 107, 71, 3, 88, 41, 101, 33, 36, 4, 13, 68, 91, 24, 97, 37, 94, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 56, 58, 116, 42, 118, 102, 62, 106, 121, 53, 39, 103, 113, 28, 20, 84, 48, 60, 79, 38, 15, 23, 89, 119, 114, 29, 115, 82, 25, 92, 22, 86, 52, 90, 76, 122, 99, 40, 18, 78, 12, 32, 117, 35, 111, 14, 51, 50, 75, 46, 87, 85, 72, 96, 63, 125, 8, 70, 64, 0, 21, 93, 26, 43, 109, 44, 17, 6, 31, 69, 10, 30, 47, 2, 19, 5, 110, 61, 11, 7, 74, 16, 83, 34, 73, 123, 104, 66, 81, 98, 9, 108, 95, 65, 1, 105, 124, 27, 80, 77, 4, 67, 45, 71, 33, 88, 3, 107, 97, 13, 37, 36, 68, 101, 41, 91, 94, 24, 100], [126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 116, 42, 102, 118, 62, 106, 121, 53, 103, 28, 39, 20, 84, 79, 29, 48, 114, 89, 86, 15, 115, 82, 119, 23, 113, 60, 76, 22, 25, 40, 38, 92, 35, 99, 78, 122, 18, 32, 14, 90, 12, 87, 52, 111, 85, 50, 75, 63, 72, 70, 43, 51, 46, 125, 117, 8, 17, 31, 64, 21, 96, 93, 34, 0, 124, 26, 16, 83, 30, 44, 69, 19, 11, 123, 10, 108, 109, 9, 81, 47, 104, 74, 95, 66, 5, 2, 61, 73, 97, 7, 27, 110, 13, 65, 80, 45, 67, 1, 105, 3, 98, 107, 6, 4, 71, 77, 33, 88, 24, 41, 68, 37, 94, 91, 36, 101, 100], [126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 42, 116, 118, 102, 62, 106, 121, 53, 39, 28, 20, 84, 79, 23, 25, 29, 103, 38, 60, 15, 89, 86, 90, 22, 99, 82, 119, 18, 76, 40, 122, 12, 92, 113, 48, 35, 114, 87, 32, 78, 115, 111, 70, 85, 14, 51, 75, 46, 52, 117, 50, 93, 64, 19, 8, 13, 125, 72, 16, 17, 11, 10, 30, 21, 63, 9, 0, 77, 26, 31, 83, 44, 96, 123, 43, 34, 1, 66, 47, 74, 104, 80, 81, 45, 124, 105, 69, 110, 61, 7, 73, 2, 71, 65, 108, 109, 95, 97, 5, 41, 27, 4, 3, 107, 98, 36, 37, 88, 6, 67, 33, 24, 91, 101, 68, 94, 100], [126, 55, 120, 54, 59, 112, 49, 57, 127, 58, 56, 116, 42, 102, 118, 106, 62, 121, 53, 28, 39, 20, 25, 122, 38, 103, 23, 84, 22, 29, 90, 48, 60, 119, 15, 79, 32, 86, 111, 89, 82, 99, 52, 113, 18, 92, 12, 40, 70, 117, 85, 115, 87, 35, 78, 14, 76, 114, 51, 46, 125, 0, 93, 21, 75, 123, 30, 64, 19, 96, 11, 63, 8, 31, 83, 43, 26, 50, 104, 34, 44, 16, 47, 17, 110, 61, 13, 10, 72, 108, 74, 69, 124, 9, 80, 73, 2, 65, 45, 95, 77, 81, 66, 109, 67, 7, 41, 5, 3, 36, 68, 1, 98, 6, 105, 33, 71, 107, 37, 4, 27, 88, 24, 97, 91, 94, 101, 100], [126, 55, 120, 54, 59, 112, 57, 127, 49, 58, 56, 42, 116, 102, 118, 62, 106, 121, 53, 28, 39, 48, 84, 38, 20, 29, 15, 103, 89, 122, 25, 119, 111, 23, 79, 60, 22, 92, 90, 87, 51, 52, 99, 32, 86, 78, 82, 35, 18, 76, 115, 40, 114, 12, 113, 14, 85, 117, 46, 70, 93, 123, 125, 47, 17, 0, 21, 8, 96, 44, 26, 31, 19, 109, 110, 50, 30, 63, 83, 11, 75, 34, 64, 43, 45, 61, 72, 9, 10, 81, 104, 13, 73, 5, 74, 108, 16, 2, 65, 66, 6, 69, 77, 67, 124, 95, 80, 41, 105, 7, 68, 27, 3, 98, 71, 107, 33, 1, 88, 97, 101, 4, 36, 24, 37, 94, 91, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 58, 56, 116, 42, 102, 118, 62, 121, 53, 106, 39, 103, 28, 48, 84, 79, 38, 20, 119, 114, 92, 89, 15, 29, 52, 25, 60, 86, 51, 23, 99, 90, 115, 122, 22, 113, 35, 40, 78, 76, 87, 111, 46, 50, 32, 12, 18, 14, 125, 82, 123, 8, 64, 85, 43, 117, 108, 110, 0, 63, 96, 21, 6, 74, 109, 75, 31, 70, 83, 11, 17, 124, 5, 30, 10, 72, 44, 93, 66, 47, 34, 45, 26, 2, 19, 73, 104, 9, 105, 81, 61, 69, 95, 98, 3, 67, 16, 7, 1, 27, 41, 71, 65, 80, 33, 77, 107, 13, 97, 36, 37, 88, 4, 91, 24, 68, 101, 94, 100], [126, 55, 120, 54, 59, 112, 127, 57, 49, 56, 58, 116, 42, 102, 118, 62, 53, 121, 106, 39, 103, 28, 20, 48, 79, 84, 113, 119, 89, 15, 38, 115, 99, 86, 25, 92, 23, 22, 29, 78, 40, 51, 35, 52, 60, 76, 82, 114, 8, 122, 90, 18, 14, 12, 0, 87, 32, 85, 6, 5, 50, 46, 64, 111, 125, 96, 66, 123, 72, 47, 117, 30, 34, 75, 74, 83, 43, 110, 93, 17, 11, 70, 124, 19, 9, 31, 73, 81, 44, 21, 65, 98, 61, 108, 2, 45, 67, 63, 69, 80, 3, 26, 71, 109, 1, 10, 105, 27, 7, 95, 16, 77, 104, 68, 33, 97, 13, 107, 88, 41, 4, 91, 94, 24, 101, 100, 36, 37], [126, 55, 120, 54, 59, 112, 57, 127, 49, 58, 56, 116, 42, 102, 118, 62, 106, 121, 53, 28, 39, 89, 20, 25, 84, 48, 86, 23, 22, 60, 15, 79, 103, 29, 38, 115, 99, 76, 40, 114, 35, 12, 90, 122, 78, 32, 113, 51, 92, 6, 18, 82, 119, 85, 111, 87, 8, 14, 96, 52, 93, 75, 19, 47, 17, 74, 11, 72, 0, 46, 50, 64, 83, 13, 34, 31, 43, 21, 123, 9, 73, 117, 30, 110, 125, 44, 81, 10, 16, 80, 26, 63, 5, 65, 77, 69, 109, 71, 108, 66, 104, 3, 1, 97, 67, 98, 45, 95, 61, 2, 105, 124, 27, 4, 7, 107, 70, 88, 41, 68, 36, 101, 91, 33, 94, 24, 37, 100], [126, 55, 54, 120, 112, 59, 49, 57, 127, 58, 56, 116, 42, 102, 118, 62, 121, 106, 53, 103, 39, 28, 25, 48, 20, 84, 60, 29, 23, 38, 15, 122, 90, 86, 89, 115, 52, 79, 92, 119, 32, 22, 111, 113, 51, 87, 35, 114, 18, 40, 82, 12, 99, 6, 78, 85, 76, 123, 50, 96, 43, 14, 46, 93, 125, 21, 47, 117, 108, 31, 72, 34, 64, 83, 124, 19, 63, 110, 11, 61, 104, 74, 73, 75, 17, 8, 26, 109, 10, 105, 77, 41, 30, 45, 44, 95, 0, 16, 5, 81, 2, 97, 9, 13, 69, 67, 80, 36, 66, 27, 98, 71, 3, 33, 1, 65, 7, 88, 107, 68, 37, 91, 101, 94, 4, 70, 24, 100], [126, 55, 120, 54, 112, 59, 57, 49, 127, 58, 56, 116, 42, 102, 118, 121, 62, 106, 53, 28, 39, 103, 119, 48, 84, 79, 22, 20, 86, 29, 15, 99, 89, 25, 60, 38, 23, 113, 114, 122, 51, 111, 87, 52, 12, 115, 40, 50, 35, 18, 92, 90, 76, 32, 14, 82, 6, 64, 78, 125, 0, 85, 72, 46, 123, 117, 8, 96, 63, 17, 75, 43, 45, 11, 21, 110, 61, 104, 10, 19, 1, 73, 83, 30, 31, 26, 74, 47, 2, 34, 5, 124, 93, 80, 66, 81, 9, 70, 108, 3, 69, 109, 13, 65, 105, 71, 41, 44, 16, 95, 67, 7, 27, 77, 107, 98, 97, 4, 33, 68, 101, 88, 36, 24, 37, 94, 91, 100], [126, 55, 120, 54, 112, 59, 127, 57, 49, 58, 56, 116, 42, 102, 118, 62, 106, 121, 53, 39, 28, 103, 22, 79, 84, 20, 29, 38, 23, 89, 48, 15, 25, 86, 40, 60, 113, 119, 92, 122, 114, 99, 12, 18, 76, 90, 82, 52, 32, 35, 115, 51, 78, 111, 0, 14, 46, 85, 72, 87, 8, 6, 123, 64, 96, 75, 21, 50, 5, 125, 117, 17, 83, 30, 74, 10, 26, 34, 70, 31, 11, 47, 45, 73, 81, 9, 93, 105, 66, 98, 44, 7, 43, 110, 2, 19, 61, 108, 63, 69, 1, 67, 16, 71, 80, 124, 95, 109, 3, 65, 27, 13, 41, 4, 77, 68, 91, 97, 33, 104, 37, 24, 88, 36, 107, 101, 94, 100], [126, 55, 120, 54, 112, 59, 127, 57, 49, 58, 56, 116, 42, 102, 118, 121, 62, 106, 53, 28, 103, 39, 22, 84, 79, 48, 89, 119, 29, 20, 115, 23, 99, 60, 25, 122, 15, 32, 86, 114, 12, 38, 113, 92, 40, 76, 18, 78, 90, 35, 52, 46, 111, 82, 85, 72, 87, 14, 51, 125, 8, 93, 0, 123, 75, 43, 50, 11, 96, 70, 30, 6, 10, 5, 21, 61, 64, 110, 17, 63, 34, 26, 2, 19, 83, 74, 73, 69, 105, 31, 47, 95, 7, 44, 109, 117, 108, 81, 9, 65, 98, 66, 13, 16, 124, 1, 80, 68, 45, 104, 71, 27, 4, 41, 97, 67, 77, 3, 107, 24, 91, 88, 33, 36, 37, 101, 100, 94], [126, 55, 120, 54, 59, 112, 127, 49, 57, 58, 56, 42, 116, 102, 118, 62, 121, 106, 28, 39, 53, 103, 25, 84, 89, 29, 20, 86, 38, 48, 60, 23, 119, 114, 79, 15, 22, 92, 90, 87, 111, 76, 115, 18, 40, 113, 52, 35, 82, 99, 32, 85, 122, 12, 51, 46, 125, 78, 70, 50, 14, 93, 123, 31, 117, 72, 21, 96, 19, 61, 10, 11, 63, 124, 44, 26, 30, 0, 64, 47, 9, 75, 34, 13, 83, 108, 43, 5, 16, 81, 17, 110, 8, 45, 80, 95, 41, 66, 1, 77, 105, 73, 104, 69, 65, 2, 74, 36, 67, 27, 97, 98, 109, 71, 7, 107, 37, 3, 6, 4, 24, 88, 101, 33, 91, 94, 68, 100], [126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 116, 42, 102, 118, 62, 106, 121, 28, 53, 39, 48, 84, 20, 103, 29, 89, 79, 38, 22, 60, 86, 15, 99, 114, 23, 113, 25, 12, 90, 92, 87, 76, 119, 122, 46, 111, 70, 82, 18, 14, 40, 35, 78, 32, 85, 115, 117, 0, 52, 123, 72, 64, 51, 44, 31, 26, 93, 63, 47, 96, 11, 30, 43, 75, 21, 125, 17, 10, 50, 83, 19, 69, 74, 104, 61, 66, 9, 5, 1, 34, 8, 110, 73, 80, 16, 2, 124, 108, 3, 95, 65, 13, 81, 27, 71, 77, 7, 45, 4, 67, 41, 36, 68, 109, 107, 6, 98, 97, 101, 88, 105, 37, 33, 91, 24, 94, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 58, 56, 116, 42, 118, 102, 62, 121, 53, 106, 39, 48, 28, 103, 38, 20, 84, 119, 79, 89, 46, 122, 15, 29, 60, 111, 114, 92, 23, 113, 52, 86, 25, 99, 12, 40, 90, 22, 76, 82, 32, 35, 18, 123, 70, 87, 115, 14, 125, 78, 117, 51, 72, 93, 44, 63, 47, 11, 85, 64, 31, 96, 0, 75, 30, 110, 43, 10, 34, 61, 17, 26, 2, 5, 21, 83, 50, 109, 9, 8, 104, 45, 108, 74, 19, 81, 73, 105, 124, 98, 16, 1, 7, 97, 66, 107, 95, 6, 69, 80, 41, 71, 27, 67, 13, 3, 68, 33, 88, 37, 101, 36, 65, 77, 24, 91, 4, 94, 100], [126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 116, 42, 102, 118, 62, 121, 106, 53, 28, 39, 103, 48, 20, 119, 84, 29, 25, 38, 79, 60, 113, 99, 52, 90, 40, 114, 89, 23, 12, 15, 86, 92, 35, 115, 32, 22, 123, 76, 14, 46, 125, 122, 78, 87, 18, 82, 85, 51, 111, 70, 72, 117, 96, 93, 43, 0, 11, 30, 50, 108, 47, 31, 10, 34, 44, 124, 61, 63, 75, 66, 5, 64, 19, 8, 109, 74, 26, 9, 95, 6, 17, 104, 73, 21, 83, 110, 2, 81, 105, 98, 69, 7, 71, 13, 45, 107, 1, 3, 16, 33, 67, 80, 68, 37, 97, 77, 88, 27, 65, 41, 4, 36, 91, 101, 100, 94, 24], [126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 116, 42, 102, 118, 121, 62, 106, 53, 28, 48, 103, 39, 119, 20, 113, 29, 84, 79, 60, 38, 22, 86, 23, 89, 25, 15, 35, 40, 122, 92, 90, 46, 99, 114, 12, 115, 52, 76, 32, 87, 18, 78, 111, 82, 125, 123, 51, 14, 117, 43, 11, 85, 72, 96, 50, 34, 6, 47, 31, 26, 104, 30, 0, 17, 93, 44, 19, 63, 75, 64, 10, 61, 45, 108, 21, 83, 9, 8, 110, 70, 74, 80, 95, 66, 2, 5, 81, 124, 109, 73, 71, 67, 16, 69, 1, 13, 98, 3, 7, 27, 107, 88, 36, 91, 97, 33, 77, 65, 68, 24, 41, 101, 105, 37, 4, 94, 100], [126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 116, 42, 118, 102, 121, 62, 106, 53, 28, 39, 20, 48, 84, 29, 60, 38, 103, 15, 25, 22, 113, 79, 122, 92, 119, 99, 23, 89, 111, 46, 86, 114, 52, 35, 90, 12, 125, 82, 14, 40, 78, 76, 115, 32, 64, 6, 87, 123, 0, 18, 51, 43, 85, 117, 47, 93, 72, 50, 8, 63, 21, 61, 2, 30, 31, 10, 17, 11, 5, 75, 66, 19, 44, 110, 124, 96, 26, 34, 108, 104, 80, 1, 83, 95, 71, 74, 69, 9, 73, 45, 65, 81, 109, 70, 107, 67, 16, 77, 7, 41, 97, 13, 27, 98, 36, 3, 105, 4, 33, 68, 88, 91, 101, 37, 24, 94, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 58, 56, 116, 42, 118, 102, 62, 121, 106, 53, 28, 39, 103, 20, 25, 60, 38, 114, 84, 48, 92, 119, 29, 122, 22, 86, 90, 23, 117, 111, 113, 46, 89, 79, 15, 32, 125, 40, 52, 35, 115, 87, 14, 99, 51, 82, 18, 12, 61, 6, 78, 76, 64, 43, 96, 30, 123, 44, 108, 124, 21, 31, 11, 10, 50, 93, 47, 85, 8, 34, 17, 19, 72, 0, 63, 110, 104, 83, 5, 75, 109, 26, 66, 74, 9, 45, 95, 107, 3, 73, 2, 69, 71, 81, 16, 80, 1, 97, 7, 98, 36, 13, 67, 88, 33, 77, 41, 27, 70, 65, 91, 37, 24, 68, 105, 4, 101, 94, 100], [126, 55, 120, 54, 112, 59, 57, 49, 127, 58, 56, 116, 42, 102, 118, 62, 106, 121, 53, 28, 39, 48, 103, 114, 29, 20, 84, 38, 79, 113, 89, 25, 119, 22, 15, 60, 40, 23, 92, 52, 115, 86, 82, 90, 12, 14, 99, 35, 122, 76, 87, 46, 18, 32, 111, 6, 78, 8, 123, 96, 43, 85, 0, 104, 51, 72, 10, 50, 125, 63, 11, 21, 30, 61, 26, 34, 83, 44, 17, 31, 9, 93, 19, 108, 117, 110, 47, 2, 5, 74, 75, 64, 71, 7, 45, 69, 73, 95, 81, 13, 109, 27, 1, 105, 67, 66, 16, 98, 80, 70, 65, 124, 3, 97, 107, 33, 4, 77, 88, 91, 101, 41, 24, 37, 68, 36, 94, 100], [126, 55, 120, 54, 59, 112, 57, 49, 127, 58, 56, 116, 42, 102, 118, 62, 121, 106, 39, 28, 53, 20, 84, 25, 23, 38, 103, 60, 89, 79, 15, 119, 86, 48, 22, 29, 113, 90, 12, 114, 92, 40, 18, 122, 99, 115, 82, 14, 46, 52, 32, 35, 51, 76, 78, 123, 111, 125, 87, 6, 85, 8, 21, 11, 83, 0, 61, 10, 108, 64, 34, 72, 96, 75, 93, 19, 17, 45, 26, 63, 74, 47, 43, 50, 71, 9, 104, 30, 124, 16, 31, 109, 73, 80, 117, 13, 81, 70, 5, 66, 77, 44, 2, 95, 110, 65, 69, 1, 67, 97, 7, 105, 98, 27, 107, 4, 91, 3, 33, 37, 68, 88, 36, 24, 101, 94, 41, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 58, 56, 116, 42, 102, 118, 62, 121, 28, 53, 106, 39, 48, 103, 25, 89, 23, 38, 20, 119, 22, 60, 29, 84, 86, 79, 122, 113, 90, 40, 123, 15, 32, 35, 12, 92, 82, 46, 52, 99, 61, 114, 14, 18, 51, 111, 115, 85, 76, 87, 78, 93, 43, 21, 8, 125, 50, 30, 108, 0, 96, 10, 31, 11, 117, 34, 44, 26, 70, 47, 64, 19, 83, 75, 6, 110, 9, 17, 72, 80, 63, 104, 124, 77, 66, 95, 109, 69, 73, 45, 74, 5, 98, 81, 16, 13, 97, 41, 71, 7, 105, 2, 65, 3, 27, 107, 1, 67, 33, 36, 68, 94, 37, 88, 101, 4, 24, 91, 100], [126, 55, 120, 54, 59, 112, 49, 127, 57, 58, 56, 42, 116, 102, 118, 121, 53, 62, 106, 28, 39, 103, 119, 48, 25, 20, 23, 84, 38, 89, 60, 29, 15, 79, 22, 115, 46, 52, 86, 114, 99, 35, 122, 51, 113, 123, 90, 82, 111, 12, 92, 32, 76, 40, 87, 14, 117, 61, 18, 125, 70, 78, 8, 11, 85, 50, 44, 63, 96, 34, 110, 45, 64, 83, 108, 0, 30, 93, 21, 104, 109, 47, 31, 72, 19, 73, 17, 75, 43, 5, 124, 69, 3, 10, 13, 105, 74, 80, 26, 98, 9, 65, 2, 81, 16, 1, 41, 66, 97, 7, 77, 6, 68, 95, 107, 67, 27, 36, 71, 33, 4, 24, 101, 94, 88, 91, 37, 100], [126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 116, 42, 102, 118, 62, 121, 106, 53, 39, 28, 20, 84, 79, 103, 89, 48, 23, 15, 38, 25, 52, 119, 29, 12, 82, 113, 22, 60, 114, 90, 76, 14, 92, 122, 32, 78, 18, 99, 40, 86, 70, 35, 46, 50, 123, 8, 115, 87, 85, 117, 125, 111, 64, 21, 72, 83, 96, 43, 5, 11, 19, 0, 93, 51, 74, 47, 73, 9, 66, 61, 31, 63, 75, 26, 81, 10, 30, 2, 17, 104, 44, 34, 108, 45, 69, 110, 80, 1, 7, 124, 67, 95, 109, 77, 105, 6, 98, 71, 16, 13, 68, 41, 3, 65, 27, 4, 97, 33, 24, 107, 36, 101, 88, 37, 94, 91, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 58, 56, 116, 42, 102, 118, 53, 62, 121, 106, 103, 28, 114, 39, 84, 38, 20, 48, 52, 79, 25, 89, 15, 22, 29, 92, 40, 23, 113, 86, 119, 60, 32, 12, 99, 50, 35, 76, 70, 90, 18, 115, 82, 78, 14, 122, 64, 8, 111, 46, 31, 85, 51, 0, 123, 125, 63, 87, 43, 45, 5, 61, 11, 72, 34, 108, 75, 96, 66, 69, 74, 117, 10, 93, 124, 17, 2, 30, 19, 21, 83, 1, 110, 73, 104, 44, 109, 7, 47, 95, 65, 81, 71, 9, 26, 16, 33, 98, 3, 107, 27, 77, 67, 105, 13, 97, 4, 80, 6, 37, 101, 36, 24, 88, 41, 68, 100, 94, 91], [126, 55, 120, 54, 112, 59, 127, 49, 57, 58, 56, 116, 42, 102, 118, 121, 62, 53, 106, 28, 39, 103, 84, 20, 38, 89, 15, 29, 60, 79, 52, 25, 48, 86, 119, 22, 113, 23, 32, 92, 114, 82, 18, 99, 76, 14, 40, 12, 35, 78, 115, 90, 46, 70, 87, 50, 51, 122, 125, 111, 8, 63, 123, 93, 117, 96, 19, 21, 85, 72, 31, 11, 43, 75, 124, 17, 73, 30, 83, 10, 74, 34, 108, 69, 61, 26, 110, 0, 45, 104, 80, 44, 13, 7, 64, 9, 109, 5, 47, 81, 27, 67, 66, 95, 16, 1, 2, 65, 105, 77, 71, 98, 3, 6, 68, 107, 97, 37, 33, 91, 4, 41, 101, 24, 88, 36, 94, 100], [126, 55, 120, 54, 112, 59, 127, 49, 57, 58, 56, 116, 42, 102, 118, 62, 106, 121, 53, 28, 103, 39, 114, 84, 20, 25, 113, 48, 15, 115, 119, 29, 23, 52, 79, 22, 89, 90, 122, 60, 40, 86, 38, 12, 32, 99, 92, 14, 51, 76, 78, 18, 46, 111, 35, 125, 87, 82, 50, 63, 85, 0, 72, 123, 124, 70, 43, 74, 8, 96, 64, 31, 117, 6, 19, 75, 110, 93, 104, 34, 9, 73, 7, 30, 5, 11, 2, 47, 10, 83, 69, 21, 17, 66, 44, 13, 108, 45, 65, 61, 16, 1, 26, 81, 80, 98, 95, 109, 97, 67, 68, 77, 71, 3, 41, 27, 105, 4, 107, 37, 36, 101, 33, 24, 88, 91, 100, 94], [126, 55, 120, 54, 59, 112, 49, 57, 127, 58, 56, 42, 116, 102, 118, 62, 121, 106, 103, 39, 53, 28, 48, 20, 84, 25, 23, 15, 29, 79, 38, 89, 114, 86, 119, 60, 22, 90, 92, 115, 18, 99, 40, 35, 12, 32, 14, 52, 122, 76, 113, 111, 51, 82, 125, 123, 46, 63, 87, 50, 85, 78, 75, 72, 43, 61, 93, 8, 74, 104, 96, 6, 11, 117, 108, 19, 110, 21, 31, 17, 30, 124, 73, 70, 44, 9, 34, 10, 83, 45, 64, 109, 47, 5, 26, 80, 105, 13, 81, 7, 0, 66, 41, 98, 77, 27, 95, 69, 71, 65, 16, 2, 107, 33, 67, 3, 88, 36, 68, 97, 1, 37, 4, 94, 24, 91, 101, 100]], "model.layers.22.self_attn.q_proj": [[57, 40, 50, 121, 54, 118, 51, 125, 52, 119, 123, 124, 63, 60, 117, 58, 61, 116, 33, 47, 62, 45, 114, 127, 59, 48, 56, 111, 126, 112, 49, 38, 55, 44, 53, 113, 122, 115, 120, 36, 102, 92, 108, 42, 109, 46, 107, 28, 17, 43, 110, 106, 94, 77, 41, 26, 25, 85, 105, 103, 83, 86, 89, 99, 39, 32, 87, 95, 19, 35, 100, 30, 37, 104, 27, 101, 31, 11, 22, 97, 21, 96, 34, 24, 98, 81, 13, 90, 29, 93, 69, 15, 2, 88, 84, 71, 0, 91, 79, 20, 16, 3, 75, 82, 1, 5, 9, 23, 67, 78, 65, 64, 73, 18, 66, 7, 80, 4, 14, 10, 74, 68, 72, 12, 8, 6, 70, 76], [50, 40, 121, 57, 51, 118, 54, 33, 114, 125, 52, 119, 123, 45, 44, 63, 58, 60, 116, 117, 127, 124, 112, 62, 49, 47, 59, 126, 61, 36, 113, 56, 48, 53, 94, 120, 122, 111, 55, 46, 115, 28, 17, 107, 26, 109, 110, 92, 25, 41, 108, 38, 43, 77, 99, 30, 42, 89, 102, 106, 85, 105, 86, 83, 19, 95, 87, 100, 24, 103, 21, 27, 31, 35, 39, 104, 101, 22, 37, 32, 98, 97, 11, 90, 84, 34, 2, 81, 96, 16, 13, 15, 69, 64, 93, 88, 82, 78, 0, 29, 79, 5, 66, 73, 23, 1, 65, 91, 67, 3, 75, 20, 7, 72, 9, 12, 18, 4, 14, 71, 68, 74, 10, 80, 6, 8, 70, 76], [121, 40, 50, 114, 125, 51, 54, 118, 57, 33, 63, 45, 52, 123, 119, 60, 117, 127, 58, 44, 116, 124, 56, 112, 59, 126, 62, 108, 61, 48, 113, 28, 46, 53, 120, 49, 122, 47, 55, 111, 115, 110, 17, 102, 26, 92, 42, 107, 109, 36, 94, 25, 38, 99, 106, 85, 43, 41, 105, 89, 95, 100, 30, 77, 87, 86, 19, 83, 103, 21, 35, 39, 104, 24, 37, 101, 96, 32, 22, 27, 97, 98, 34, 31, 81, 11, 13, 84, 2, 82, 88, 90, 64, 73, 29, 93, 16, 0, 15, 79, 9, 78, 1, 91, 69, 65, 23, 14, 5, 75, 66, 3, 4, 7, 67, 20, 71, 12, 18, 72, 10, 74, 68, 6, 70, 8, 80, 76], [40, 121, 50, 57, 82, 114, 78, 12, 33, 27, 16, 25, 99, 118, 23, 38, 29, 74, 19, 86, 6, 10, 93, 18, 36, 89, 72, 94, 51, 14, 70, 76, 104, 8, 28, 20, 84, 63, 102, 88, 90, 30, 95, 68, 54, 21, 80, 58, 119, 4, 55, 87, 91, 22, 125, 9, 15, 85, 49, 31, 1, 81, 96, 26, 62, 47, 42, 73, 24, 7, 127, 66, 116, 83, 92, 3, 52, 123, 39, 11, 32, 13, 115, 2, 124, 60, 64, 44, 98, 34, 117, 17, 105, 61, 79, 35, 126, 53, 101, 112, 48, 56, 5, 122, 45, 75, 77, 109, 71, 107, 67, 120, 113, 106, 37, 108, 43, 110, 59, 41, 46, 111, 100, 103, 69, 97, 0, 65], [42, 54, 32, 35, 123, 99, 106, 50, 89, 85, 27, 87, 16, 23, 58, 29, 96, 18, 78, 36, 57, 40, 98, 83, 103, 116, 107, 117, 7, 84, 62, 31, 17, 108, 61, 48, 60, 82, 44, 12, 47, 37, 105, 95, 100, 77, 26, 21, 125, 74, 76, 118, 46, 55, 45, 41, 90, 9, 114, 10, 20, 115, 94, 113, 101, 119, 92, 127, 110, 19, 124, 112, 81, 97, 122, 109, 126, 102, 49, 28, 24, 121, 59, 75, 104, 56, 22, 70, 88, 79, 43, 120, 34, 68, 33, 51, 25, 53, 63, 93, 15, 8, 91, 30, 52, 86, 39, 111, 80, 38, 4, 11, 14, 0, 72, 66, 13, 3, 69, 71, 5, 73, 64, 67, 6, 65, 1, 2], [54, 42, 32, 89, 117, 99, 123, 85, 87, 23, 29, 58, 35, 106, 96, 28, 50, 121, 55, 44, 36, 60, 15, 111, 33, 101, 57, 40, 46, 82, 122, 52, 126, 51, 112, 53, 31, 47, 10, 16, 100, 78, 103, 104, 41, 18, 125, 119, 45, 110, 109, 38, 83, 114, 95, 81, 17, 105, 115, 39, 118, 21, 12, 25, 113, 56, 76, 127, 19, 124, 48, 116, 34, 107, 37, 14, 27, 8, 98, 90, 108, 91, 49, 77, 63, 62, 79, 59, 102, 43, 120, 97, 30, 61, 69, 93, 84, 80, 92, 24, 72, 26, 94, 86, 74, 22, 88, 6, 67, 20, 75, 66, 71, 13, 11, 7, 5, 3, 70, 65, 73, 64, 68, 2, 4, 9, 1, 0], [54, 42, 32, 99, 35, 29, 89, 87, 96, 115, 23, 85, 36, 106, 95, 27, 110, 62, 105, 101, 18, 61, 123, 119, 94, 44, 45, 50, 83, 82, 117, 118, 63, 59, 48, 41, 86, 113, 28, 40, 121, 112, 55, 58, 31, 93, 120, 47, 114, 92, 103, 109, 125, 107, 37, 127, 122, 104, 57, 53, 108, 34, 60, 100, 126, 38, 52, 49, 16, 51, 20, 43, 116, 15, 124, 102, 90, 46, 33, 56, 22, 39, 26, 97, 12, 77, 98, 78, 84, 10, 25, 111, 30, 91, 81, 17, 24, 13, 75, 88, 14, 19, 76, 21, 4, 6, 8, 72, 74, 67, 7, 69, 79, 73, 1, 68, 5, 80, 70, 9, 11, 0, 66, 71, 64, 3, 65, 2], [42, 54, 32, 85, 35, 123, 106, 99, 89, 23, 16, 87, 76, 78, 18, 12, 50, 48, 27, 84, 29, 125, 36, 11, 14, 6, 72, 117, 10, 81, 4, 115, 31, 21, 80, 51, 25, 17, 8, 116, 28, 26, 101, 111, 79, 104, 83, 77, 3, 74, 109, 55, 92, 82, 2, 19, 9, 73, 60, 88, 57, 62, 1, 98, 69, 86, 64, 124, 97, 103, 119, 33, 66, 37, 44, 90, 93, 49, 102, 40, 105, 108, 30, 95, 114, 7, 110, 96, 67, 100, 41, 94, 75, 46, 56, 15, 127, 24, 126, 34, 38, 107, 53, 70, 63, 20, 118, 91, 43, 5, 112, 22, 39, 47, 52, 13, 45, 58, 71, 120, 122, 61, 59, 113, 121, 68, 65, 0], [117, 38, 120, 97, 25, 58, 63, 93, 89, 83, 116, 52, 53, 121, 55, 49, 102, 39, 86, 112, 115, 75, 122, 126, 127, 113, 119, 57, 125, 123, 50, 109, 47, 21, 48, 60, 114, 118, 46, 62, 20, 84, 41, 88, 105, 61, 59, 111, 110, 51, 44, 73, 56, 29, 124, 17, 100, 54, 104, 45, 80, 106, 42, 107, 108, 103, 40, 43, 22, 87, 16, 101, 37, 24, 26, 98, 99, 34, 36, 35, 15, 95, 18, 96, 31, 77, 94, 19, 23, 71, 32, 30, 33, 92, 4, 28, 79, 91, 14, 81, 90, 13, 11, 85, 27, 68, 1, 7, 82, 72, 3, 74, 70, 65, 64, 10, 76, 69, 66, 78, 9, 5, 0, 2, 8, 12, 67, 6], [38, 120, 117, 63, 97, 93, 17, 83, 25, 89, 86, 20, 29, 88, 102, 14, 58, 76, 74, 121, 65, 72, 15, 53, 116, 77, 71, 73, 49, 64, 41, 126, 122, 60, 100, 57, 4, 91, 46, 55, 59, 22, 11, 119, 123, 98, 94, 50, 81, 127, 115, 34, 10, 26, 39, 66, 104, 7, 47, 111, 109, 125, 16, 113, 52, 42, 118, 19, 56, 48, 51, 112, 90, 54, 114, 61, 44, 45, 84, 79, 62, 96, 24, 28, 35, 37, 78, 82, 2, 103, 87, 70, 108, 31, 75, 101, 124, 99, 106, 67, 40, 105, 32, 95, 21, 30, 36, 80, 5, 43, 27, 85, 110, 92, 12, 9, 13, 69, 1, 23, 33, 68, 107, 8, 3, 18, 6, 0], [38, 120, 117, 97, 58, 25, 83, 86, 93, 63, 73, 89, 17, 102, 29, 121, 15, 53, 88, 3, 14, 49, 75, 22, 126, 57, 20, 71, 19, 109, 41, 76, 77, 113, 116, 55, 127, 84, 46, 111, 79, 72, 52, 119, 11, 4, 47, 122, 65, 118, 112, 39, 125, 87, 115, 51, 60, 62, 50, 59, 123, 74, 70, 45, 21, 80, 56, 81, 9, 44, 114, 61, 16, 48, 42, 104, 10, 82, 100, 24, 108, 67, 124, 78, 34, 105, 103, 2, 85, 110, 33, 90, 23, 54, 7, 107, 40, 8, 37, 98, 35, 91, 28, 101, 43, 36, 0, 66, 99, 13, 69, 26, 12, 94, 18, 96, 1, 106, 68, 92, 31, 95, 6, 27, 30, 32, 5, 64], [38, 117, 120, 97, 63, 83, 70, 93, 25, 15, 73, 89, 3, 86, 58, 53, 41, 14, 17, 88, 80, 74, 29, 109, 78, 77, 75, 121, 57, 84, 65, 22, 55, 111, 115, 46, 49, 19, 79, 105, 0, 102, 118, 98, 122, 76, 42, 126, 106, 116, 119, 52, 125, 72, 4, 20, 107, 113, 127, 50, 56, 21, 54, 104, 62, 61, 47, 60, 9, 6, 123, 45, 59, 110, 11, 10, 44, 39, 51, 26, 114, 48, 12, 103, 2, 112, 81, 124, 82, 85, 8, 66, 108, 67, 43, 87, 71, 24, 40, 16, 36, 69, 34, 101, 94, 1, 23, 31, 7, 37, 95, 30, 90, 99, 28, 91, 100, 5, 32, 18, 35, 27, 68, 92, 64, 96, 13, 33], [122, 58, 53, 38, 125, 124, 121, 56, 87, 90, 92, 55, 59, 34, 111, 127, 76, 117, 63, 126, 30, 81, 123, 113, 62, 50, 120, 54, 52, 110, 49, 57, 61, 60, 114, 119, 48, 118, 51, 116, 115, 47, 100, 102, 19, 112, 8, 40, 107, 109, 91, 3, 22, 44, 89, 46, 83, 45, 21, 108, 41, 23, 2, 32, 80, 31, 103, 84, 105, 42, 43, 104, 106, 72, 36, 94, 39, 5, 37, 29, 35, 85, 28, 96, 33, 16, 24, 93, 25, 101, 70, 65, 95, 17, 14, 27, 26, 99, 10, 86, 0, 97, 13, 6, 78, 64, 12, 73, 9, 88, 98, 77, 1, 18, 15, 82, 4, 11, 75, 79, 66, 71, 20, 7, 68, 69, 67, 74], [53, 38, 125, 122, 58, 87, 121, 56, 30, 127, 34, 92, 90, 60, 50, 113, 63, 124, 55, 59, 123, 48, 120, 126, 32, 51, 41, 57, 114, 61, 62, 52, 118, 54, 119, 116, 47, 111, 115, 49, 21, 117, 76, 102, 110, 19, 108, 46, 85, 112, 40, 45, 107, 83, 106, 22, 109, 81, 44, 94, 37, 43, 80, 42, 100, 27, 28, 26, 84, 104, 105, 39, 103, 33, 91, 101, 23, 29, 36, 15, 31, 72, 97, 99, 16, 89, 77, 35, 86, 13, 78, 96, 98, 17, 93, 3, 0, 95, 65, 8, 5, 14, 18, 88, 2, 25, 69, 20, 73, 10, 75, 74, 24, 11, 9, 82, 12, 79, 71, 7, 6, 68, 70, 66, 4, 67, 64, 1], [122, 53, 38, 90, 121, 58, 56, 59, 127, 34, 87, 60, 55, 124, 92, 126, 123, 63, 117, 52, 30, 120, 48, 57, 111, 113, 61, 49, 110, 114, 125, 115, 62, 118, 119, 54, 50, 51, 47, 116, 102, 83, 22, 112, 46, 81, 19, 94, 32, 89, 109, 85, 23, 104, 108, 41, 14, 43, 84, 103, 107, 76, 31, 44, 105, 17, 106, 45, 36, 40, 42, 101, 25, 86, 28, 78, 100, 13, 82, 80, 37, 27, 39, 35, 98, 33, 26, 21, 29, 15, 97, 99, 88, 95, 8, 24, 91, 96, 16, 65, 93, 72, 18, 10, 12, 79, 69, 20, 3, 11, 6, 9, 77, 75, 7, 68, 74, 73, 71, 70, 67, 5, 0, 2, 66, 64, 4, 1], [125, 122, 53, 38, 58, 90, 121, 56, 51, 59, 127, 113, 120, 110, 81, 87, 55, 50, 63, 30, 61, 48, 126, 124, 123, 102, 49, 108, 57, 52, 117, 60, 114, 119, 34, 118, 62, 47, 111, 46, 109, 54, 115, 116, 19, 44, 106, 112, 92, 39, 42, 107, 43, 76, 103, 83, 22, 101, 32, 100, 94, 37, 105, 41, 21, 40, 45, 104, 36, 99, 14, 31, 35, 33, 8, 85, 91, 17, 26, 93, 24, 96, 29, 23, 97, 27, 84, 78, 95, 88, 28, 25, 98, 86, 80, 72, 10, 89, 74, 12, 15, 79, 13, 16, 18, 82, 11, 77, 3, 9, 73, 20, 70, 69, 75, 6, 5, 68, 7, 4, 71, 66, 0, 64, 65, 1, 67, 2], [102, 110, 97, 126, 46, 73, 21, 77, 82, 68, 11, 15, 26, 66, 90, 70, 53, 0, 7, 25, 87, 71, 85, 121, 31, 18, 1, 23, 29, 79, 100, 93, 4, 54, 118, 81, 88, 124, 10, 117, 55, 9, 57, 106, 78, 28, 64, 13, 80, 22, 20, 127, 2, 75, 32, 5, 108, 76, 83, 65, 69, 17, 12, 58, 6, 105, 92, 3, 89, 63, 16, 24, 116, 115, 86, 125, 84, 27, 72, 39, 40, 104, 123, 98, 91, 48, 59, 30, 62, 96, 49, 74, 52, 122, 37, 101, 67, 60, 14, 95, 112, 34, 19, 8, 51, 45, 99, 35, 94, 103, 109, 50, 61, 120, 56, 47, 36, 43, 107, 42, 44, 119, 111, 113, 41, 114, 33, 38], [102, 110, 126, 97, 21, 26, 82, 46, 15, 77, 73, 11, 87, 90, 93, 68, 7, 23, 12, 25, 45, 115, 40, 53, 78, 70, 8, 18, 44, 76, 85, 118, 84, 55, 66, 120, 54, 116, 109, 127, 0, 96, 101, 36, 95, 31, 37, 91, 13, 112, 79, 72, 125, 24, 83, 1, 98, 121, 124, 27, 41, 48, 19, 71, 32, 114, 17, 103, 52, 56, 75, 122, 58, 69, 14, 9, 123, 39, 89, 49, 42, 106, 59, 88, 63, 22, 81, 105, 6, 20, 51, 107, 62, 94, 80, 119, 117, 28, 34, 29, 3, 86, 60, 38, 47, 99, 61, 57, 108, 43, 92, 104, 30, 111, 10, 16, 100, 50, 35, 113, 74, 4, 33, 2, 5, 67, 65, 64], [102, 110, 126, 97, 118, 46, 26, 21, 15, 82, 87, 11, 31, 63, 95, 90, 93, 68, 49, 73, 77, 44, 41, 120, 53, 116, 88, 127, 106, 122, 123, 25, 54, 112, 22, 7, 101, 38, 84, 42, 56, 37, 12, 58, 52, 17, 51, 80, 29, 59, 109, 70, 23, 43, 92, 48, 119, 55, 57, 124, 66, 5, 113, 94, 45, 125, 111, 115, 0, 65, 104, 121, 117, 83, 10, 50, 108, 60, 105, 1, 62, 85, 100, 47, 96, 91, 69, 61, 36, 107, 27, 78, 98, 99, 28, 114, 40, 30, 39, 19, 8, 86, 76, 3, 32, 103, 79, 74, 16, 34, 71, 64, 67, 24, 72, 35, 89, 20, 18, 4, 81, 75, 33, 14, 2, 6, 13, 9], [102, 110, 97, 126, 21, 46, 26, 77, 82, 11, 15, 90, 73, 116, 87, 70, 92, 91, 68, 7, 25, 66, 37, 54, 32, 45, 85, 99, 62, 72, 49, 30, 83, 108, 41, 51, 19, 124, 18, 44, 111, 52, 71, 0, 113, 53, 122, 112, 127, 105, 104, 88, 24, 118, 48, 58, 100, 89, 40, 106, 50, 17, 123, 8, 75, 101, 16, 95, 36, 121, 96, 78, 63, 42, 93, 31, 94, 22, 38, 98, 3, 76, 115, 79, 28, 60, 55, 5, 107, 64, 33, 34, 114, 39, 35, 125, 109, 61, 119, 20, 57, 10, 47, 84, 6, 120, 29, 103, 59, 13, 43, 1, 86, 56, 81, 2, 117, 14, 9, 23, 27, 74, 80, 12, 67, 69, 4, 65], [111, 55, 101, 47, 25, 87, 113, 95, 21, 28, 48, 109, 58, 119, 82, 56, 59, 31, 62, 97, 91, 118, 94, 44, 54, 45, 122, 89, 33, 127, 126, 42, 116, 51, 115, 50, 120, 107, 61, 102, 80, 106, 23, 15, 76, 85, 123, 38, 53, 74, 121, 112, 32, 39, 92, 18, 57, 40, 110, 52, 105, 19, 104, 125, 117, 72, 124, 22, 41, 77, 63, 37, 49, 46, 30, 108, 16, 98, 29, 103, 99, 35, 60, 34, 93, 114, 26, 100, 14, 90, 84, 71, 96, 17, 36, 83, 69, 43, 86, 88, 27, 24, 75, 81, 20, 11, 13, 73, 79, 65, 9, 7, 78, 70, 3, 12, 67, 1, 5, 4, 8, 66, 10, 2, 68, 6, 0, 64], [111, 55, 101, 47, 25, 95, 21, 87, 28, 82, 33, 94, 91, 15, 74, 53, 76, 59, 62, 97, 22, 89, 72, 29, 114, 60, 56, 121, 49, 116, 107, 37, 92, 45, 93, 123, 38, 32, 11, 70, 43, 18, 110, 109, 67, 122, 98, 54, 113, 48, 103, 57, 23, 99, 115, 78, 102, 4, 69, 31, 80, 44, 51, 40, 83, 85, 105, 124, 36, 119, 50, 34, 86, 46, 66, 41, 13, 117, 77, 104, 120, 58, 26, 52, 108, 118, 42, 27, 71, 17, 61, 127, 63, 112, 96, 39, 125, 106, 84, 19, 90, 126, 79, 10, 81, 100, 30, 8, 7, 24, 12, 35, 73, 88, 20, 9, 75, 16, 64, 68, 5, 3, 1, 14, 65, 2, 6, 0], [111, 101, 55, 47, 25, 87, 21, 95, 15, 74, 82, 70, 4, 89, 76, 94, 64, 97, 67, 77, 28, 80, 113, 2, 114, 72, 84, 50, 73, 18, 0, 91, 58, 31, 126, 29, 79, 66, 11, 85, 6, 122, 23, 92, 40, 83, 16, 10, 71, 45, 3, 116, 107, 75, 41, 65, 5, 7, 86, 35, 27, 88, 13, 98, 93, 9, 14, 12, 121, 33, 8, 78, 19, 123, 17, 68, 96, 81, 69, 20, 24, 90, 26, 105, 34, 22, 117, 44, 30, 32, 51, 115, 46, 38, 99, 54, 37, 1, 62, 124, 60, 109, 127, 103, 100, 59, 61, 48, 39, 110, 108, 102, 53, 120, 125, 42, 36, 112, 52, 104, 56, 63, 106, 57, 119, 49, 118, 43], [111, 55, 101, 47, 25, 120, 87, 21, 121, 28, 95, 56, 116, 31, 82, 110, 53, 33, 123, 37, 126, 38, 46, 58, 50, 124, 104, 122, 115, 89, 60, 44, 62, 51, 97, 112, 59, 29, 118, 61, 57, 127, 76, 23, 63, 48, 94, 52, 45, 39, 107, 103, 113, 91, 49, 117, 114, 85, 80, 42, 109, 125, 105, 119, 32, 40, 108, 41, 98, 102, 99, 106, 18, 15, 90, 54, 30, 26, 84, 43, 34, 72, 77, 92, 35, 96, 86, 100, 93, 27, 36, 16, 74, 78, 20, 67, 17, 11, 83, 22, 79, 88, 24, 81, 13, 14, 69, 19, 71, 70, 9, 12, 65, 75, 7, 73, 8, 4, 64, 3, 66, 5, 10, 2, 0, 1, 68, 6], [115, 100, 51, 23, 91, 32, 96, 84, 86, 9, 77, 15, 81, 39, 71, 5, 11, 27, 20, 108, 49, 10, 94, 36, 24, 83, 22, 44, 68, 21, 67, 118, 17, 0, 64, 65, 57, 82, 126, 127, 121, 117, 114, 16, 54, 45, 110, 2, 87, 79, 28, 35, 95, 107, 33, 13, 12, 80, 73, 25, 101, 74, 3, 69, 19, 76, 40, 85, 116, 37, 18, 42, 75, 30, 112, 8, 90, 89, 104, 92, 72, 124, 70, 88, 122, 14, 41, 78, 48, 120, 52, 34, 99, 50, 62, 4, 93, 97, 102, 123, 47, 105, 29, 98, 63, 119, 125, 109, 60, 53, 113, 31, 66, 38, 6, 46, 103, 56, 26, 7, 59, 1, 43, 106, 55, 111, 61, 58], [115, 100, 51, 121, 36, 91, 32, 23, 86, 84, 39, 82, 77, 81, 127, 48, 96, 113, 15, 118, 33, 20, 29, 44, 63, 104, 55, 71, 116, 120, 45, 119, 123, 49, 37, 47, 108, 22, 102, 114, 41, 126, 107, 50, 42, 106, 46, 54, 35, 110, 124, 53, 125, 105, 27, 10, 43, 38, 52, 111, 112, 16, 59, 56, 60, 58, 57, 30, 61, 122, 40, 95, 11, 19, 18, 62, 99, 109, 97, 117, 66, 34, 101, 98, 64, 31, 103, 83, 17, 26, 92, 24, 94, 5, 68, 85, 87, 90, 12, 21, 93, 89, 28, 76, 73, 6, 88, 8, 25, 13, 79, 78, 2, 75, 9, 80, 14, 67, 74, 70, 72, 1, 69, 4, 0, 65, 7, 3], [115, 100, 51, 121, 36, 91, 23, 32, 84, 86, 103, 63, 82, 20, 114, 104, 44, 22, 77, 15, 48, 39, 117, 46, 107, 99, 55, 43, 127, 49, 116, 29, 34, 45, 81, 110, 31, 30, 40, 119, 61, 118, 37, 33, 57, 108, 38, 41, 112, 42, 120, 113, 54, 125, 101, 35, 111, 60, 87, 106, 96, 58, 109, 53, 59, 56, 52, 95, 123, 47, 98, 126, 92, 105, 62, 24, 90, 97, 102, 27, 50, 94, 124, 122, 28, 9, 93, 71, 26, 17, 16, 85, 74, 25, 88, 76, 18, 19, 14, 89, 79, 13, 21, 11, 83, 80, 78, 10, 6, 12, 66, 64, 7, 73, 1, 4, 0, 75, 5, 72, 8, 68, 70, 69, 3, 2, 67, 65], [115, 100, 51, 36, 23, 91, 121, 96, 84, 86, 81, 11, 32, 15, 118, 94, 48, 117, 39, 9, 50, 108, 116, 107, 102, 5, 20, 2, 119, 25, 105, 45, 77, 44, 0, 37, 113, 27, 63, 82, 58, 62, 68, 19, 57, 46, 47, 110, 120, 49, 114, 126, 122, 43, 112, 123, 99, 125, 127, 53, 59, 22, 54, 41, 61, 55, 24, 124, 109, 52, 104, 42, 95, 17, 60, 65, 40, 56, 33, 38, 106, 35, 111, 70, 101, 71, 103, 87, 88, 93, 64, 16, 26, 98, 30, 79, 34, 29, 73, 83, 10, 31, 97, 90, 72, 75, 69, 8, 28, 14, 89, 12, 67, 6, 92, 13, 21, 18, 66, 78, 85, 4, 1, 80, 76, 3, 74, 7], [115, 40, 122, 53, 93, 124, 49, 55, 58, 51, 52, 48, 120, 33, 59, 60, 56, 62, 127, 111, 125, 26, 44, 61, 54, 47, 50, 123, 63, 113, 43, 126, 114, 118, 121, 116, 107, 57, 119, 45, 46, 112, 117, 22, 19, 109, 42, 110, 90, 29, 108, 88, 84, 41, 101, 77, 99, 38, 104, 106, 105, 32, 94, 92, 95, 103, 39, 83, 36, 102, 37, 34, 81, 100, 89, 97, 35, 86, 16, 98, 96, 25, 82, 28, 87, 71, 27, 10, 31, 13, 21, 23, 91, 79, 30, 80, 17, 14, 7, 3, 24, 74, 15, 12, 78, 85, 2, 20, 64, 66, 0, 18, 9, 67, 5, 72, 11, 69, 65, 6, 1, 75, 73, 4, 68, 76, 8, 70], [53, 115, 40, 122, 124, 93, 49, 52, 55, 59, 51, 33, 60, 56, 62, 48, 111, 126, 61, 58, 113, 117, 120, 125, 43, 114, 50, 127, 54, 26, 121, 63, 47, 123, 109, 119, 57, 116, 118, 46, 45, 112, 90, 22, 108, 81, 107, 44, 110, 35, 29, 88, 37, 42, 41, 39, 105, 101, 106, 103, 92, 77, 99, 36, 104, 32, 34, 102, 95, 84, 100, 83, 38, 94, 19, 89, 97, 86, 98, 17, 25, 96, 31, 27, 16, 82, 13, 28, 20, 30, 15, 79, 24, 10, 91, 85, 14, 87, 21, 71, 80, 74, 78, 23, 72, 5, 7, 12, 66, 64, 11, 8, 0, 2, 1, 3, 75, 4, 67, 18, 69, 65, 68, 9, 73, 6, 70, 76], [40, 115, 53, 122, 88, 14, 33, 19, 82, 12, 22, 90, 80, 9, 92, 49, 28, 111, 91, 26, 30, 1, 55, 73, 93, 6, 31, 50, 124, 29, 25, 75, 20, 72, 27, 4, 95, 23, 17, 89, 69, 86, 78, 15, 96, 85, 32, 58, 10, 38, 117, 59, 76, 18, 51, 24, 34, 45, 98, 83, 70, 112, 21, 100, 104, 35, 127, 52, 16, 99, 105, 46, 56, 11, 123, 94, 79, 84, 42, 81, 106, 108, 61, 113, 121, 13, 8, 68, 109, 62, 114, 3, 120, 54, 110, 87, 116, 103, 101, 126, 37, 63, 125, 119, 39, 102, 71, 77, 48, 107, 36, 47, 43, 60, 118, 41, 57, 67, 7, 74, 44, 97, 5, 0, 66, 65, 2, 64], [122, 40, 53, 115, 33, 124, 93, 55, 52, 60, 49, 59, 26, 48, 62, 111, 61, 56, 120, 58, 125, 113, 101, 47, 127, 51, 54, 116, 126, 121, 119, 114, 50, 117, 123, 57, 63, 118, 109, 45, 84, 95, 39, 88, 112, 90, 43, 29, 46, 108, 22, 41, 107, 110, 44, 77, 34, 32, 19, 25, 42, 89, 94, 92, 106, 99, 100, 105, 83, 103, 37, 21, 81, 97, 102, 104, 35, 38, 24, 36, 98, 96, 86, 31, 28, 23, 17, 87, 27, 16, 91, 30, 2, 80, 3, 82, 69, 64, 13, 74, 1, 65, 14, 5, 0, 79, 66, 10, 7, 71, 67, 85, 15, 20, 78, 12, 11, 68, 9, 6, 72, 75, 4, 73, 8, 18, 70, 76]], "model.layers.22.self_attn.k_proj": [[104, 121, 50, 86, 57, 97, 92, 100, 54, 94, 51, 118, 114, 119, 25, 31, 124, 63, 125, 58, 17, 60, 117, 123, 102, 62, 126, 116, 52, 127, 48, 85, 55, 83, 47, 53, 49, 56, 112, 122, 44, 120, 59, 113, 111, 115, 27, 105, 61, 46, 107, 108, 35, 110, 106, 42, 19, 109, 43, 77, 39, 41, 45, 16, 80, 103, 12, 15, 37, 78, 93, 33, 99, 34, 38, 87, 26, 75, 79, 98, 101, 91, 32, 76, 84, 11, 23, 72, 96, 82, 90, 7, 95, 36, 5, 28, 10, 20, 24, 13, 30, 29, 40, 74, 6, 88, 73, 71, 18, 89, 21, 9, 4, 3, 22, 8, 14, 81, 70, 1, 68, 0, 67, 69, 2, 64, 66, 65], [106, 54, 89, 96, 123, 50, 23, 85, 16, 17, 99, 42, 78, 82, 76, 48, 124, 27, 114, 30, 29, 21, 57, 72, 19, 34, 112, 74, 12, 93, 14, 108, 10, 18, 70, 55, 77, 100, 60, 47, 68, 127, 125, 3, 35, 118, 7, 11, 53, 115, 46, 8, 79, 104, 110, 95, 59, 20, 49, 15, 111, 109, 119, 41, 98, 38, 28, 105, 44, 65, 13, 83, 101, 107, 5, 126, 9, 81, 113, 33, 84, 37, 117, 120, 43, 116, 52, 103, 92, 56, 66, 31, 61, 63, 90, 122, 102, 40, 45, 62, 39, 121, 86, 97, 24, 51, 94, 87, 36, 75, 64, 91, 58, 88, 26, 22, 25, 6, 69, 80, 32, 71, 73, 67, 4, 2, 1, 0], [120, 117, 102, 33, 86, 29, 25, 58, 17, 83, 49, 121, 0, 50, 88, 73, 118, 14, 126, 65, 119, 53, 3, 122, 15, 116, 57, 46, 47, 125, 103, 55, 123, 52, 115, 113, 111, 38, 4, 62, 75, 112, 44, 48, 51, 114, 127, 61, 105, 45, 7, 59, 56, 70, 107, 36, 106, 41, 124, 60, 77, 74, 108, 109, 84, 63, 80, 110, 42, 21, 104, 94, 16, 40, 20, 18, 72, 54, 43, 98, 39, 90, 13, 5, 89, 37, 87, 34, 76, 91, 100, 26, 99, 2, 68, 31, 35, 92, 28, 101, 95, 64, 71, 1, 10, 67, 23, 96, 69, 32, 8, 82, 27, 85, 30, 12, 93, 79, 11, 66, 78, 24, 81, 19, 6, 9, 97, 22], [122, 53, 102, 22, 98, 57, 121, 117, 62, 56, 55, 110, 50, 116, 28, 59, 123, 115, 63, 124, 120, 126, 52, 48, 60, 58, 118, 114, 113, 127, 119, 54, 51, 61, 112, 49, 111, 33, 47, 109, 44, 94, 104, 108, 96, 46, 107, 90, 45, 40, 105, 34, 43, 125, 41, 80, 37, 42, 106, 78, 103, 36, 92, 101, 39, 16, 82, 100, 85, 13, 95, 19, 86, 83, 97, 99, 35, 17, 73, 87, 24, 25, 93, 31, 11, 29, 27, 81, 76, 32, 38, 30, 68, 84, 23, 91, 21, 15, 26, 88, 10, 71, 5, 14, 18, 8, 79, 75, 7, 67, 89, 72, 66, 20, 77, 70, 9, 69, 74, 1, 12, 0, 2, 65, 6, 3, 64, 4], [46, 126, 38, 33, 110, 82, 21, 26, 15, 77, 73, 0, 11, 70, 66, 7, 68, 64, 87, 25, 53, 3, 93, 65, 121, 19, 17, 95, 54, 2, 84, 125, 58, 10, 80, 12, 118, 88, 59, 115, 63, 48, 22, 1, 124, 45, 94, 55, 14, 105, 71, 16, 49, 112, 74, 117, 40, 8, 60, 104, 44, 92, 101, 116, 123, 51, 52, 67, 62, 6, 113, 42, 120, 96, 127, 69, 41, 43, 56, 39, 5, 119, 109, 34, 83, 114, 23, 103, 111, 122, 107, 108, 57, 35, 37, 32, 50, 36, 72, 99, 28, 61, 47, 102, 29, 106, 24, 100, 81, 89, 20, 86, 98, 30, 27, 78, 31, 91, 4, 9, 76, 79, 97, 75, 13, 85, 90, 18], [47, 111, 37, 55, 87, 21, 72, 82, 15, 25, 76, 31, 74, 66, 70, 107, 77, 69, 64, 4, 94, 80, 65, 119, 97, 102, 28, 116, 78, 115, 56, 91, 40, 44, 58, 123, 118, 53, 38, 43, 110, 121, 39, 120, 127, 62, 7, 92, 83, 122, 42, 60, 113, 52, 1, 105, 114, 45, 124, 126, 117, 104, 108, 112, 59, 106, 125, 22, 34, 63, 51, 48, 73, 17, 61, 93, 46, 54, 57, 50, 90, 98, 49, 103, 32, 41, 109, 36, 71, 20, 33, 86, 35, 29, 99, 19, 81, 100, 11, 30, 26, 13, 84, 96, 85, 24, 2, 14, 27, 68, 101, 88, 16, 23, 75, 10, 3, 9, 5, 12, 67, 8, 6, 89, 79, 0, 95, 18], [51, 115, 36, 86, 91, 96, 84, 23, 81, 77, 15, 71, 64, 121, 94, 82, 11, 117, 1, 49, 44, 113, 119, 68, 57, 9, 48, 66, 112, 39, 47, 46, 106, 45, 108, 67, 120, 122, 127, 50, 53, 125, 5, 101, 63, 24, 104, 58, 59, 41, 114, 109, 61, 107, 126, 118, 62, 56, 55, 3, 60, 103, 123, 70, 105, 54, 10, 116, 43, 124, 88, 52, 26, 0, 110, 2, 19, 97, 38, 76, 42, 99, 111, 100, 31, 25, 37, 74, 80, 40, 85, 89, 78, 33, 93, 16, 65, 30, 98, 6, 69, 34, 12, 102, 72, 92, 14, 21, 29, 95, 35, 8, 90, 28, 83, 4, 18, 73, 32, 7, 27, 87, 13, 79, 75, 22, 20, 17], [104, 115, 53, 22, 122, 97, 51, 29, 124, 111, 59, 56, 52, 26, 28, 55, 49, 62, 126, 50, 54, 118, 117, 113, 61, 60, 127, 116, 121, 119, 47, 58, 82, 63, 12, 19, 83, 120, 114, 48, 125, 110, 123, 35, 91, 112, 16, 109, 81, 88, 44, 57, 45, 6, 107, 105, 77, 43, 15, 93, 46, 32, 89, 94, 108, 17, 42, 41, 84, 31, 36, 14, 80, 37, 33, 78, 101, 103, 96, 24, 102, 79, 30, 106, 39, 27, 95, 23, 38, 9, 99, 98, 34, 90, 13, 87, 100, 69, 75, 40, 74, 25, 85, 21, 11, 18, 92, 72, 66, 20, 7, 68, 71, 73, 70, 0, 64, 76, 10, 65, 86, 4, 67, 1, 8, 2, 3, 5]], "model.layers.22.self_attn.qk_proj": [[115, 53, 122, 51, 117, 120, 47, 111, 126, 121, 54, 50, 46, 110, 57, 106, 55, 42, 102, 38, 104, 89, 58, 22, 123, 118, 125, 25, 87, 97, 85, 49, 40, 27, 56, 36, 21, 90, 86, 23, 93, 18, 62, 63, 127, 79, 60, 101, 59, 96, 15, 33, 82, 77, 114, 124, 99, 13, 116, 81, 48, 17, 26, 112, 52, 61, 19, 100, 29, 44, 83, 119, 11, 37, 113, 28, 84, 32, 95, 92, 45, 9, 30, 94, 73, 91, 31, 75, 7, 108, 20, 16, 109, 41, 14, 80, 78, 76, 70, 12, 39, 35, 4, 0, 71, 34, 74, 10, 105, 107, 8, 98, 64, 43, 68, 88, 103, 67, 3, 2, 66, 6, 24, 65, 1, 72, 5, 69], [115, 53, 122, 117, 51, 120, 47, 111, 126, 121, 54, 50, 46, 110, 57, 55, 106, 42, 102, 38, 123, 58, 89, 22, 104, 125, 87, 97, 118, 25, 36, 62, 85, 56, 49, 23, 93, 27, 40, 86, 21, 90, 114, 101, 79, 59, 116, 82, 99, 113, 52, 18, 13, 63, 100, 96, 127, 77, 15, 33, 29, 60, 19, 81, 124, 17, 112, 73, 61, 26, 119, 44, 48, 92, 28, 91, 11, 109, 32, 94, 107, 45, 20, 105, 70, 37, 83, 41, 75, 9, 84, 76, 108, 0, 16, 14, 12, 78, 95, 4, 7, 30, 39, 31, 35, 98, 80, 74, 43, 103, 10, 71, 64, 34, 2, 68, 24, 88, 8, 1, 67, 3, 66, 72, 65, 6, 5, 69], [115, 53, 122, 117, 51, 120, 47, 111, 121, 126, 54, 50, 46, 57, 110, 55, 106, 42, 102, 38, 58, 104, 22, 25, 118, 123, 89, 97, 93, 27, 56, 23, 87, 59, 36, 21, 40, 62, 85, 86, 125, 49, 63, 116, 15, 90, 114, 33, 60, 18, 82, 101, 77, 79, 100, 99, 96, 13, 52, 48, 32, 127, 124, 113, 81, 17, 119, 45, 92, 112, 29, 44, 107, 26, 73, 19, 28, 37, 11, 20, 0, 108, 94, 41, 91, 30, 61, 83, 84, 70, 109, 71, 9, 75, 95, 34, 7, 98, 35, 80, 105, 76, 12, 78, 31, 4, 39, 16, 14, 43, 64, 10, 103, 74, 68, 88, 66, 72, 2, 67, 3, 65, 1, 24, 69, 6, 8, 5], [115, 53, 122, 117, 51, 120, 47, 111, 126, 121, 50, 54, 46, 57, 110, 106, 55, 42, 38, 102, 58, 123, 104, 25, 22, 89, 125, 56, 62, 21, 87, 63, 59, 27, 97, 93, 86, 118, 85, 23, 15, 114, 49, 82, 18, 77, 40, 90, 36, 100, 124, 101, 127, 13, 52, 33, 81, 79, 45, 116, 48, 99, 96, 119, 113, 112, 32, 60, 17, 107, 11, 37, 29, 84, 61, 83, 70, 26, 73, 12, 44, 19, 9, 92, 95, 28, 64, 41, 105, 109, 91, 7, 71, 98, 14, 94, 75, 78, 43, 16, 80, 30, 76, 31, 4, 108, 20, 0, 74, 68, 34, 66, 35, 72, 10, 103, 67, 39, 3, 24, 65, 88, 2, 6, 1, 5, 69, 8], [115, 122, 53, 117, 51, 120, 47, 111, 126, 121, 54, 50, 46, 110, 57, 106, 55, 42, 102, 38, 58, 123, 22, 89, 97, 104, 118, 49, 63, 25, 21, 86, 87, 85, 62, 27, 125, 23, 93, 40, 59, 82, 36, 114, 56, 15, 79, 60, 18, 124, 48, 77, 33, 81, 45, 90, 116, 100, 99, 13, 119, 52, 127, 96, 44, 101, 19, 26, 112, 61, 17, 92, 11, 29, 73, 91, 9, 113, 75, 37, 71, 20, 32, 31, 95, 83, 28, 64, 16, 94, 14, 78, 7, 80, 41, 12, 84, 76, 0, 109, 98, 108, 39, 10, 43, 70, 34, 105, 107, 72, 74, 4, 68, 35, 30, 6, 2, 67, 24, 66, 103, 1, 3, 88, 5, 69, 65, 8], [115, 53, 122, 117, 51, 120, 47, 111, 121, 126, 50, 54, 46, 57, 110, 106, 55, 42, 102, 38, 123, 22, 58, 86, 27, 89, 85, 125, 97, 21, 25, 87, 104, 62, 93, 40, 23, 56, 49, 63, 82, 15, 77, 33, 59, 36, 60, 118, 18, 81, 13, 119, 90, 79, 114, 124, 99, 96, 100, 127, 116, 11, 48, 17, 37, 92, 84, 113, 32, 29, 19, 83, 26, 52, 41, 101, 16, 112, 31, 73, 28, 75, 94, 91, 9, 44, 20, 61, 95, 14, 12, 98, 45, 80, 76, 7, 109, 108, 78, 74, 0, 64, 10, 105, 71, 68, 107, 30, 72, 39, 6, 35, 43, 4, 34, 24, 2, 103, 65, 70, 3, 67, 88, 1, 66, 5, 69, 8], [115, 53, 122, 117, 51, 120, 47, 111, 126, 54, 121, 50, 46, 57, 110, 106, 55, 42, 102, 38, 22, 123, 87, 89, 86, 25, 85, 104, 23, 27, 21, 58, 40, 97, 93, 60, 63, 36, 15, 62, 90, 49, 82, 96, 79, 125, 33, 18, 118, 77, 114, 13, 99, 116, 32, 17, 59, 48, 81, 113, 119, 52, 44, 100, 19, 101, 56, 84, 127, 29, 112, 94, 11, 124, 26, 92, 91, 45, 83, 28, 107, 37, 76, 73, 109, 20, 12, 61, 95, 16, 9, 78, 80, 75, 14, 41, 31, 43, 39, 105, 34, 72, 108, 7, 35, 71, 30, 10, 6, 64, 74, 98, 0, 24, 88, 103, 68, 4, 1, 66, 3, 2, 67, 70, 69, 65, 5, 8], [115, 53, 122, 117, 51, 120, 47, 126, 111, 54, 121, 50, 46, 57, 110, 55, 106, 42, 38, 102, 22, 89, 87, 97, 25, 63, 85, 21, 58, 36, 60, 23, 27, 104, 118, 93, 90, 86, 40, 123, 18, 49, 62, 59, 96, 33, 15, 125, 82, 112, 116, 119, 48, 79, 77, 100, 113, 101, 17, 99, 52, 114, 61, 56, 32, 13, 124, 81, 44, 26, 91, 29, 20, 28, 127, 19, 37, 84, 108, 80, 35, 12, 73, 83, 11, 109, 92, 94, 107, 39, 34, 95, 43, 45, 14, 6, 9, 16, 88, 76, 78, 75, 41, 105, 30, 7, 31, 64, 72, 4, 74, 71, 10, 98, 0, 24, 103, 68, 66, 65, 2, 3, 67, 69, 1, 5, 70, 8], [115, 53, 122, 117, 51, 120, 47, 126, 111, 121, 54, 50, 46, 110, 57, 55, 106, 42, 102, 38, 89, 58, 22, 125, 25, 62, 36, 27, 93, 87, 21, 23, 97, 123, 118, 90, 104, 85, 63, 49, 59, 114, 82, 86, 112, 127, 40, 15, 77, 96, 116, 56, 33, 79, 101, 124, 60, 61, 52, 18, 48, 32, 100, 99, 119, 19, 107, 13, 28, 113, 81, 9, 17, 109, 45, 29, 26, 37, 44, 11, 20, 41, 83, 91, 92, 84, 35, 94, 108, 12, 78, 105, 39, 80, 14, 34, 31, 76, 73, 95, 30, 6, 98, 75, 16, 7, 103, 43, 74, 72, 68, 64, 0, 88, 71, 4, 10, 24, 66, 65, 2, 1, 3, 67, 5, 70, 69, 8], [115, 53, 122, 51, 117, 120, 47, 126, 111, 54, 121, 50, 46, 110, 57, 55, 106, 42, 102, 38, 104, 58, 123, 89, 22, 125, 62, 25, 87, 118, 27, 63, 97, 23, 119, 85, 36, 93, 86, 49, 21, 90, 114, 52, 56, 59, 40, 61, 96, 99, 124, 101, 79, 116, 127, 18, 60, 77, 82, 13, 48, 15, 100, 33, 45, 107, 11, 32, 91, 109, 41, 9, 94, 17, 28, 81, 112, 19, 37, 113, 92, 44, 26, 29, 20, 83, 108, 84, 73, 35, 75, 105, 31, 14, 78, 7, 12, 39, 43, 16, 6, 34, 30, 64, 95, 0, 76, 71, 98, 80, 103, 74, 68, 4, 10, 70, 2, 72, 3, 66, 1, 8, 67, 24, 88, 65, 69, 5], [115, 53, 122, 51, 117, 120, 47, 111, 121, 126, 54, 50, 46, 110, 106, 57, 55, 42, 38, 102, 58, 104, 125, 89, 22, 25, 63, 97, 27, 123, 86, 87, 62, 21, 49, 118, 85, 23, 119, 100, 93, 82, 59, 114, 101, 77, 18, 52, 90, 15, 79, 116, 40, 33, 99, 56, 36, 127, 61, 13, 96, 48, 113, 17, 124, 83, 112, 81, 29, 9, 60, 41, 94, 28, 37, 32, 26, 109, 92, 16, 64, 44, 91, 20, 11, 12, 84, 7, 45, 75, 107, 70, 19, 74, 73, 95, 76, 31, 71, 78, 80, 14, 39, 98, 68, 10, 4, 35, 0, 108, 103, 105, 43, 8, 34, 6, 66, 30, 2, 3, 24, 72, 67, 1, 88, 65, 69, 5], [115, 53, 122, 51, 117, 120, 47, 111, 126, 121, 54, 50, 46, 110, 57, 106, 55, 42, 102, 38, 22, 89, 123, 86, 25, 21, 104, 58, 40, 87, 27, 97, 36, 85, 118, 23, 90, 82, 93, 79, 62, 125, 49, 124, 63, 77, 18, 116, 96, 59, 15, 33, 119, 61, 114, 100, 99, 127, 17, 13, 112, 32, 113, 81, 101, 19, 26, 83, 60, 94, 28, 44, 84, 92, 20, 48, 11, 37, 29, 9, 73, 56, 52, 12, 75, 91, 41, 95, 14, 70, 31, 76, 80, 78, 10, 16, 7, 45, 107, 64, 98, 0, 74, 35, 34, 105, 71, 4, 109, 108, 68, 39, 43, 30, 88, 24, 8, 103, 66, 3, 1, 6, 2, 67, 72, 69, 65, 5], [115, 53, 122, 117, 51, 120, 47, 126, 111, 54, 121, 50, 46, 110, 57, 55, 106, 102, 42, 38, 89, 22, 123, 87, 25, 118, 36, 23, 93, 97, 21, 86, 27, 58, 40, 85, 104, 90, 82, 125, 96, 116, 63, 49, 33, 59, 100, 99, 56, 114, 112, 124, 18, 61, 119, 60, 15, 113, 29, 101, 13, 62, 32, 79, 77, 26, 81, 28, 94, 19, 44, 17, 91, 92, 20, 83, 127, 48, 37, 107, 41, 109, 84, 9, 75, 105, 95, 39, 11, 31, 52, 14, 16, 73, 35, 108, 45, 12, 70, 30, 78, 80, 34, 64, 76, 10, 43, 71, 98, 7, 8, 68, 24, 88, 0, 103, 4, 74, 65, 66, 2, 3, 1, 67, 69, 5, 72, 6], [115, 122, 53, 117, 51, 120, 47, 126, 111, 54, 121, 50, 46, 110, 57, 106, 55, 42, 38, 102, 58, 123, 22, 104, 25, 86, 125, 89, 21, 118, 59, 23, 27, 85, 119, 40, 87, 90, 93, 36, 82, 124, 116, 77, 60, 63, 49, 112, 56, 114, 62, 97, 100, 79, 127, 33, 96, 15, 113, 52, 61, 18, 81, 9, 37, 101, 99, 32, 29, 17, 11, 13, 44, 48, 70, 75, 31, 109, 73, 94, 0, 41, 20, 19, 84, 45, 91, 83, 95, 92, 26, 28, 30, 107, 98, 14, 12, 71, 7, 35, 108, 80, 16, 64, 105, 78, 8, 76, 39, 68, 43, 10, 4, 74, 103, 34, 2, 1, 24, 66, 67, 65, 88, 3, 5, 72, 6, 69], [115, 122, 53, 117, 51, 120, 47, 111, 121, 126, 54, 50, 46, 110, 106, 57, 55, 42, 38, 102, 58, 22, 104, 97, 86, 25, 89, 87, 27, 123, 21, 125, 40, 36, 62, 23, 118, 82, 79, 93, 119, 63, 85, 77, 90, 49, 112, 60, 114, 99, 15, 56, 18, 81, 127, 33, 124, 96, 13, 100, 61, 59, 101, 113, 17, 75, 83, 32, 48, 116, 109, 9, 91, 45, 29, 52, 37, 26, 92, 19, 28, 71, 76, 84, 14, 11, 31, 44, 95, 73, 12, 107, 20, 39, 80, 7, 70, 41, 78, 94, 35, 64, 30, 0, 43, 105, 8, 74, 108, 34, 10, 16, 98, 4, 103, 68, 2, 24, 6, 3, 67, 88, 66, 1, 65, 72, 5, 69], [115, 122, 53, 117, 51, 120, 47, 121, 111, 126, 54, 50, 46, 110, 106, 55, 57, 42, 38, 102, 123, 22, 104, 86, 21, 25, 40, 125, 58, 89, 85, 87, 97, 27, 23, 63, 93, 36, 82, 90, 77, 79, 49, 18, 56, 100, 60, 118, 59, 15, 116, 33, 81, 62, 61, 127, 114, 113, 52, 119, 48, 96, 13, 101, 112, 124, 99, 29, 41, 17, 26, 12, 32, 45, 75, 73, 109, 83, 84, 76, 94, 19, 91, 92, 44, 9, 64, 28, 31, 7, 16, 37, 20, 78, 11, 34, 14, 10, 8, 95, 35, 39, 68, 70, 74, 108, 103, 105, 107, 30, 43, 71, 98, 80, 0, 24, 4, 6, 66, 88, 65, 67, 1, 2, 3, 5, 69, 72], [115, 53, 122, 51, 117, 120, 47, 126, 111, 121, 54, 50, 46, 110, 55, 57, 106, 42, 102, 38, 104, 89, 22, 58, 86, 123, 87, 40, 25, 85, 23, 90, 36, 114, 93, 125, 97, 21, 27, 63, 96, 118, 82, 79, 100, 33, 59, 49, 60, 56, 18, 77, 116, 15, 81, 113, 29, 124, 61, 94, 99, 17, 84, 19, 26, 44, 101, 127, 48, 28, 119, 62, 112, 32, 83, 13, 41, 31, 52, 75, 20, 91, 73, 37, 92, 30, 11, 45, 16, 35, 39, 78, 76, 109, 98, 12, 80, 108, 107, 9, 95, 6, 34, 43, 103, 71, 0, 10, 105, 7, 14, 24, 8, 88, 68, 64, 4, 74, 1, 2, 66, 70, 65, 3, 67, 5, 72, 69], [115, 53, 122, 51, 117, 120, 47, 126, 111, 54, 121, 50, 46, 110, 57, 55, 106, 42, 38, 102, 25, 58, 22, 86, 104, 87, 89, 40, 123, 93, 27, 114, 36, 90, 21, 97, 125, 118, 85, 79, 63, 23, 59, 96, 60, 18, 99, 82, 77, 62, 112, 15, 49, 33, 116, 100, 26, 124, 81, 48, 56, 101, 29, 13, 61, 17, 127, 52, 32, 75, 19, 37, 113, 44, 107, 109, 45, 73, 92, 20, 91, 119, 9, 28, 84, 94, 6, 83, 78, 16, 12, 95, 41, 30, 31, 108, 11, 39, 0, 105, 7, 35, 34, 80, 71, 76, 14, 98, 10, 64, 68, 4, 103, 43, 74, 88, 66, 67, 8, 2, 65, 24, 3, 1, 72, 69, 70, 5], [115, 53, 122, 117, 51, 120, 47, 126, 111, 121, 54, 50, 46, 110, 57, 55, 106, 42, 102, 38, 63, 104, 89, 125, 22, 25, 114, 97, 123, 58, 21, 40, 86, 36, 87, 62, 48, 27, 82, 23, 85, 90, 93, 127, 112, 99, 49, 118, 101, 79, 124, 60, 96, 77, 59, 52, 116, 100, 13, 61, 15, 18, 33, 56, 26, 29, 17, 41, 44, 32, 75, 28, 37, 84, 45, 109, 119, 91, 73, 81, 113, 94, 83, 107, 19, 6, 105, 9, 95, 12, 11, 39, 92, 78, 7, 108, 34, 31, 30, 35, 98, 43, 76, 20, 0, 64, 14, 103, 10, 71, 16, 4, 80, 74, 68, 67, 24, 88, 2, 72, 65, 8, 66, 1, 3, 70, 5, 69], [115, 122, 53, 117, 51, 120, 47, 126, 111, 121, 50, 54, 46, 110, 57, 106, 55, 42, 102, 38, 104, 89, 22, 123, 97, 58, 125, 93, 85, 87, 40, 21, 25, 86, 63, 118, 36, 23, 27, 62, 60, 56, 90, 114, 116, 124, 49, 96, 77, 101, 100, 18, 15, 33, 13, 79, 82, 99, 127, 113, 52, 59, 92, 81, 48, 26, 41, 112, 119, 29, 94, 44, 75, 28, 83, 17, 9, 45, 61, 91, 73, 32, 11, 84, 37, 107, 20, 19, 109, 30, 31, 78, 12, 6, 35, 95, 105, 76, 64, 39, 14, 0, 10, 80, 71, 16, 7, 34, 43, 98, 103, 68, 108, 66, 74, 72, 4, 67, 88, 1, 24, 2, 3, 65, 8, 70, 69, 5], [115, 53, 122, 117, 51, 120, 47, 111, 126, 121, 50, 54, 46, 110, 57, 55, 106, 42, 102, 38, 58, 89, 22, 104, 118, 25, 40, 123, 97, 63, 86, 87, 85, 27, 93, 23, 116, 21, 60, 36, 49, 90, 125, 56, 79, 82, 59, 77, 124, 96, 33, 62, 18, 99, 112, 113, 48, 114, 100, 15, 119, 52, 13, 127, 101, 81, 92, 44, 31, 26, 32, 29, 17, 19, 107, 61, 94, 75, 41, 73, 109, 28, 37, 20, 83, 11, 91, 78, 7, 39, 30, 95, 45, 84, 12, 10, 108, 9, 71, 64, 76, 98, 14, 34, 68, 6, 16, 80, 103, 0, 74, 35, 72, 105, 4, 88, 43, 70, 67, 24, 1, 2, 66, 3, 5, 65, 69, 8], [115, 53, 122, 117, 51, 120, 47, 111, 126, 121, 54, 50, 46, 110, 57, 55, 106, 42, 38, 102, 118, 89, 58, 123, 125, 104, 22, 25, 87, 63, 85, 62, 97, 56, 124, 93, 23, 86, 90, 114, 27, 52, 48, 21, 40, 59, 36, 49, 96, 79, 82, 119, 116, 60, 18, 127, 15, 112, 113, 13, 29, 33, 101, 77, 99, 100, 26, 44, 91, 61, 32, 109, 81, 17, 92, 19, 45, 107, 31, 20, 41, 75, 37, 28, 64, 83, 73, 39, 11, 0, 7, 71, 94, 9, 105, 10, 95, 78, 98, 68, 80, 103, 72, 12, 84, 16, 35, 70, 108, 76, 30, 14, 43, 34, 4, 1, 88, 6, 74, 67, 66, 65, 24, 3, 2, 5, 69, 8], [115, 122, 53, 117, 51, 120, 47, 111, 121, 126, 54, 50, 46, 110, 57, 55, 106, 42, 102, 38, 118, 89, 123, 22, 58, 104, 97, 27, 25, 52, 85, 59, 87, 93, 125, 60, 23, 36, 86, 49, 63, 116, 40, 124, 21, 96, 90, 61, 119, 48, 82, 79, 18, 33, 56, 112, 113, 13, 101, 114, 62, 44, 100, 26, 99, 77, 29, 15, 92, 109, 17, 127, 28, 107, 81, 94, 32, 20, 83, 91, 19, 37, 45, 31, 30, 41, 84, 75, 73, 7, 70, 11, 39, 9, 64, 12, 72, 95, 0, 14, 76, 105, 35, 78, 43, 71, 34, 80, 98, 10, 16, 108, 68, 74, 88, 4, 2, 67, 103, 1, 24, 66, 6, 65, 3, 5, 69, 8], [115, 122, 53, 117, 51, 120, 47, 111, 126, 121, 54, 50, 46, 110, 57, 55, 106, 42, 102, 38, 118, 22, 89, 86, 104, 25, 27, 97, 58, 49, 87, 85, 123, 125, 93, 40, 36, 90, 21, 124, 56, 23, 18, 63, 62, 96, 15, 79, 119, 59, 82, 116, 114, 77, 127, 52, 60, 13, 81, 61, 33, 99, 113, 48, 100, 112, 92, 17, 26, 101, 37, 29, 83, 9, 28, 75, 11, 19, 44, 70, 31, 84, 107, 41, 73, 76, 94, 32, 20, 91, 95, 109, 10, 108, 105, 7, 45, 12, 30, 98, 14, 72, 78, 16, 80, 71, 0, 103, 68, 64, 43, 74, 4, 35, 39, 34, 88, 65, 24, 67, 3, 66, 2, 1, 69, 5, 8, 6], [115, 53, 122, 117, 51, 120, 47, 111, 126, 121, 50, 54, 46, 110, 57, 55, 106, 42, 102, 38, 22, 89, 104, 25, 123, 87, 86, 118, 85, 58, 23, 93, 21, 40, 97, 27, 36, 125, 90, 52, 49, 62, 79, 59, 18, 56, 63, 48, 77, 119, 33, 124, 82, 96, 127, 15, 114, 116, 61, 99, 13, 112, 100, 17, 81, 101, 32, 60, 19, 92, 29, 84, 26, 44, 11, 83, 9, 113, 20, 28, 75, 91, 37, 73, 109, 94, 41, 70, 7, 16, 31, 76, 12, 45, 95, 10, 35, 107, 105, 64, 71, 78, 39, 14, 108, 30, 34, 80, 98, 0, 43, 74, 68, 72, 24, 88, 4, 103, 66, 1, 67, 65, 3, 2, 69, 5, 6, 8], [115, 53, 122, 117, 51, 120, 47, 121, 126, 111, 50, 54, 46, 110, 57, 55, 106, 42, 102, 38, 58, 123, 22, 104, 25, 89, 118, 36, 87, 125, 23, 86, 85, 90, 124, 40, 27, 93, 97, 21, 96, 63, 52, 59, 49, 82, 116, 33, 60, 18, 79, 119, 114, 99, 127, 56, 62, 13, 100, 77, 26, 48, 101, 29, 44, 15, 41, 61, 112, 109, 19, 113, 17, 94, 81, 92, 32, 28, 37, 91, 84, 73, 83, 20, 9, 39, 30, 11, 107, 76, 35, 7, 75, 70, 105, 45, 0, 78, 98, 80, 10, 43, 31, 12, 108, 16, 103, 95, 34, 14, 71, 24, 68, 74, 64, 72, 88, 4, 66, 1, 3, 6, 2, 8, 67, 65, 5, 69], [115, 53, 122, 51, 117, 120, 47, 121, 126, 111, 50, 54, 46, 110, 57, 55, 106, 42, 102, 38, 58, 118, 125, 89, 104, 123, 22, 87, 25, 27, 63, 21, 59, 36, 62, 86, 116, 49, 124, 40, 85, 96, 52, 93, 56, 23, 90, 97, 60, 114, 119, 99, 127, 82, 100, 112, 33, 48, 18, 61, 101, 13, 79, 15, 107, 77, 81, 113, 32, 44, 26, 37, 19, 92, 109, 29, 17, 45, 83, 94, 28, 11, 20, 9, 30, 12, 73, 91, 34, 105, 98, 108, 41, 76, 14, 95, 31, 84, 71, 43, 35, 16, 103, 0, 80, 39, 78, 75, 70, 10, 74, 7, 68, 6, 4, 24, 64, 88, 8, 66, 2, 1, 72, 67, 3, 69, 5, 65], [115, 53, 122, 51, 117, 120, 47, 121, 111, 126, 54, 50, 46, 110, 57, 106, 55, 42, 102, 38, 25, 58, 123, 118, 104, 89, 62, 125, 22, 87, 27, 97, 85, 21, 90, 93, 86, 36, 23, 56, 59, 18, 124, 13, 79, 40, 63, 48, 82, 49, 60, 114, 96, 52, 116, 33, 15, 113, 77, 99, 101, 100, 29, 112, 119, 44, 127, 17, 11, 81, 107, 28, 32, 26, 92, 19, 73, 61, 45, 84, 75, 9, 83, 41, 94, 37, 6, 71, 78, 16, 76, 64, 91, 105, 109, 95, 20, 30, 31, 14, 98, 43, 0, 12, 108, 7, 10, 39, 80, 74, 35, 68, 4, 34, 66, 103, 8, 65, 3, 88, 67, 70, 2, 24, 1, 72, 5, 69], [115, 53, 122, 51, 117, 120, 47, 111, 121, 126, 54, 50, 46, 110, 57, 106, 55, 42, 102, 38, 58, 123, 104, 89, 118, 97, 22, 25, 86, 27, 21, 125, 87, 40, 62, 56, 60, 93, 36, 85, 63, 59, 23, 116, 79, 49, 124, 33, 52, 82, 13, 18, 99, 90, 127, 44, 48, 96, 77, 100, 119, 114, 113, 61, 15, 101, 112, 81, 32, 26, 11, 73, 92, 28, 6, 31, 29, 17, 0, 20, 45, 83, 41, 91, 84, 94, 37, 19, 71, 9, 39, 109, 105, 34, 108, 95, 98, 76, 75, 107, 78, 12, 14, 64, 30, 8, 80, 4, 7, 43, 68, 74, 35, 16, 10, 103, 67, 88, 1, 66, 3, 2, 65, 24, 69, 5, 72, 70], [115, 53, 122, 117, 51, 120, 47, 121, 111, 126, 54, 50, 46, 110, 57, 106, 55, 42, 38, 102, 123, 22, 58, 118, 89, 87, 125, 104, 21, 25, 86, 97, 27, 85, 23, 124, 63, 93, 60, 36, 127, 90, 62, 116, 59, 82, 79, 40, 18, 100, 13, 56, 49, 99, 15, 119, 96, 81, 77, 52, 33, 48, 114, 61, 113, 19, 101, 11, 29, 17, 32, 26, 28, 83, 9, 92, 84, 73, 91, 94, 109, 6, 41, 44, 31, 20, 37, 112, 12, 80, 76, 108, 78, 98, 75, 43, 74, 95, 107, 8, 71, 14, 105, 68, 7, 45, 39, 0, 30, 35, 34, 16, 103, 10, 4, 64, 88, 24, 66, 67, 1, 69, 3, 2, 65, 70, 5, 72], [115, 53, 122, 51, 117, 120, 47, 111, 121, 126, 54, 50, 46, 110, 57, 106, 55, 42, 38, 102, 58, 89, 104, 123, 25, 22, 118, 97, 87, 56, 27, 93, 86, 23, 40, 21, 36, 85, 90, 59, 124, 100, 49, 63, 125, 33, 127, 52, 60, 82, 79, 116, 18, 13, 96, 77, 99, 15, 101, 29, 62, 114, 113, 61, 119, 11, 81, 17, 48, 19, 44, 84, 32, 94, 9, 31, 28, 7, 73, 6, 30, 75, 92, 83, 112, 45, 91, 71, 26, 78, 74, 109, 37, 64, 0, 20, 41, 12, 95, 8, 16, 76, 4, 35, 98, 108, 80, 34, 14, 68, 107, 10, 105, 43, 39, 66, 103, 2, 67, 24, 65, 3, 70, 88, 1, 5, 69, 72], [115, 53, 122, 51, 117, 120, 47, 111, 126, 54, 121, 50, 46, 110, 55, 106, 57, 42, 102, 38, 22, 123, 58, 104, 89, 25, 87, 86, 27, 97, 85, 118, 56, 21, 125, 63, 40, 93, 36, 23, 90, 18, 82, 49, 79, 124, 96, 116, 60, 77, 15, 62, 119, 99, 101, 13, 114, 33, 59, 100, 61, 11, 48, 29, 81, 17, 9, 94, 26, 112, 127, 44, 19, 52, 113, 83, 32, 84, 28, 37, 92, 75, 73, 91, 76, 41, 30, 107, 8, 80, 35, 16, 78, 12, 109, 39, 31, 20, 45, 7, 14, 95, 108, 71, 10, 105, 6, 103, 74, 64, 98, 43, 4, 34, 68, 24, 0, 70, 88, 66, 3, 2, 65, 67, 1, 5, 69, 72]], "model.layers.23.self_attn.q_proj": [[111, 100, 107, 15, 88, 90, 21, 93, 81, 47, 20, 75, 11, 31, 52, 82, 73, 39, 19, 83, 85, 22, 41, 104, 12, 110, 54, 28, 13, 32, 95, 120, 14, 77, 27, 72, 76, 45, 119, 37, 87, 36, 40, 106, 71, 94, 24, 50, 6, 118, 26, 92, 101, 51, 33, 44, 38, 25, 16, 122, 114, 59, 10, 80, 112, 34, 58, 124, 5, 56, 108, 46, 123, 116, 109, 84, 79, 62, 68, 23, 70, 125, 96, 43, 103, 63, 127, 17, 57, 61, 105, 113, 78, 97, 55, 53, 126, 86, 89, 117, 74, 60, 98, 30, 18, 9, 8, 48, 49, 91, 42, 115, 29, 69, 99, 35, 121, 7, 102, 2, 67, 4, 0, 3, 66, 1, 65, 64], [111, 100, 47, 107, 32, 88, 90, 27, 94, 21, 83, 85, 24, 78, 80, 84, 81, 40, 95, 109, 30, 22, 75, 86, 17, 26, 44, 15, 10, 89, 7, 96, 77, 110, 28, 118, 119, 36, 31, 123, 50, 51, 39, 68, 0, 23, 46, 2, 63, 43, 116, 126, 127, 60, 120, 5, 73, 56, 121, 93, 41, 105, 124, 48, 42, 49, 52, 37, 33, 106, 57, 72, 101, 45, 114, 125, 98, 91, 122, 99, 103, 29, 61, 35, 20, 55, 115, 113, 112, 53, 11, 92, 54, 71, 59, 62, 34, 102, 38, 58, 108, 104, 97, 117, 6, 74, 87, 64, 25, 79, 82, 66, 19, 3, 16, 67, 18, 12, 4, 69, 14, 76, 8, 13, 65, 9, 1, 70], [111, 100, 107, 47, 90, 83, 86, 27, 88, 31, 93, 18, 80, 24, 78, 79, 91, 41, 17, 95, 85, 23, 32, 74, 72, 101, 39, 22, 96, 110, 15, 36, 94, 29, 76, 77, 50, 21, 120, 92, 34, 109, 81, 62, 13, 89, 40, 9, 118, 123, 82, 122, 119, 7, 30, 11, 84, 75, 121, 12, 127, 16, 106, 10, 56, 59, 114, 60, 20, 117, 116, 61, 5, 57, 104, 87, 58, 124, 43, 33, 52, 25, 115, 26, 46, 68, 37, 35, 28, 19, 54, 108, 97, 51, 126, 8, 102, 14, 112, 48, 73, 53, 38, 98, 105, 42, 103, 2, 0, 45, 63, 55, 71, 125, 44, 99, 49, 113, 4, 3, 6, 69, 70, 64, 1, 66, 67, 65], [111, 100, 107, 47, 32, 88, 121, 27, 85, 24, 63, 90, 51, 119, 21, 95, 81, 49, 36, 94, 40, 80, 91, 52, 50, 109, 84, 83, 93, 45, 42, 96, 61, 78, 38, 106, 34, 58, 124, 108, 110, 10, 118, 56, 46, 44, 122, 117, 125, 30, 57, 15, 115, 62, 28, 97, 104, 55, 39, 120, 59, 60, 54, 126, 112, 114, 103, 53, 23, 22, 92, 105, 75, 99, 116, 31, 41, 98, 113, 48, 35, 86, 43, 127, 123, 102, 37, 33, 77, 29, 101, 26, 5, 20, 89, 68, 17, 0, 16, 18, 72, 7, 2, 71, 25, 87, 82, 11, 73, 67, 19, 79, 14, 76, 12, 6, 74, 13, 3, 9, 8, 66, 64, 70, 69, 1, 4, 65], [56, 51, 103, 46, 110, 19, 13, 97, 15, 90, 28, 17, 99, 23, 116, 60, 94, 21, 10, 115, 47, 81, 87, 24, 57, 79, 100, 75, 72, 124, 91, 9, 53, 93, 86, 109, 80, 31, 61, 84, 89, 54, 14, 85, 74, 58, 92, 126, 3, 7, 77, 76, 6, 52, 18, 48, 113, 25, 11, 5, 123, 114, 41, 30, 127, 59, 119, 122, 27, 55, 45, 112, 12, 83, 63, 16, 71, 68, 49, 121, 29, 62, 22, 67, 43, 26, 78, 70, 88, 8, 98, 102, 82, 95, 20, 32, 108, 125, 0, 34, 36, 118, 117, 104, 105, 1, 69, 120, 101, 96, 38, 37, 106, 111, 35, 50, 107, 40, 44, 42, 64, 73, 4, 2, 33, 65, 66, 39], [46, 56, 103, 110, 53, 57, 60, 115, 61, 116, 47, 122, 124, 20, 52, 89, 127, 97, 126, 112, 51, 59, 55, 108, 114, 121, 58, 94, 54, 118, 63, 49, 62, 123, 48, 113, 109, 119, 50, 125, 111, 84, 88, 30, 106, 28, 99, 117, 25, 86, 44, 45, 80, 105, 43, 120, 107, 37, 36, 41, 24, 92, 42, 104, 27, 78, 14, 40, 38, 39, 22, 101, 102, 98, 82, 16, 33, 100, 35, 91, 19, 87, 95, 34, 31, 96, 32, 11, 9, 4, 17, 71, 64, 66, 69, 93, 0, 26, 90, 68, 73, 2, 29, 18, 1, 13, 75, 10, 21, 65, 85, 5, 83, 12, 15, 7, 67, 72, 3, 81, 23, 79, 6, 70, 76, 77, 8, 74], [56, 103, 46, 53, 57, 110, 60, 61, 124, 116, 47, 126, 51, 122, 112, 52, 97, 127, 115, 20, 89, 55, 59, 54, 58, 94, 123, 118, 28, 121, 114, 49, 113, 125, 62, 37, 119, 109, 108, 63, 99, 48, 120, 111, 25, 117, 45, 50, 80, 44, 30, 36, 105, 86, 84, 43, 92, 106, 41, 107, 88, 24, 104, 14, 42, 40, 19, 23, 39, 82, 102, 101, 38, 22, 98, 100, 27, 34, 33, 35, 78, 16, 91, 13, 96, 31, 21, 71, 87, 95, 9, 81, 12, 90, 15, 69, 32, 18, 68, 11, 72, 0, 93, 3, 74, 5, 29, 26, 85, 10, 64, 66, 1, 2, 4, 83, 17, 75, 73, 67, 6, 65, 76, 7, 79, 8, 77, 70], [46, 51, 103, 56, 19, 110, 87, 17, 15, 97, 28, 13, 92, 116, 10, 81, 61, 12, 47, 90, 53, 94, 23, 86, 112, 60, 57, 126, 72, 58, 52, 59, 96, 121, 124, 43, 89, 122, 115, 25, 114, 127, 106, 49, 55, 62, 21, 48, 63, 54, 108, 45, 4, 123, 109, 125, 119, 120, 42, 111, 118, 113, 24, 117, 50, 83, 30, 44, 105, 79, 85, 93, 107, 75, 67, 99, 9, 82, 11, 31, 20, 71, 8, 98, 104, 70, 6, 76, 26, 101, 84, 77, 40, 32, 41, 38, 7, 34, 37, 29, 22, 36, 102, 100, 95, 5, 3, 91, 80, 18, 14, 68, 27, 73, 69, 0, 1, 65, 35, 88, 74, 66, 16, 64, 2, 78, 39, 33], [102, 110, 33, 49, 46, 92, 113, 111, 86, 28, 122, 19, 57, 81, 24, 55, 78, 54, 70, 74, 85, 13, 108, 90, 61, 8, 117, 94, 30, 23, 66, 105, 80, 109, 114, 26, 76, 87, 44, 14, 60, 75, 9, 68, 79, 127, 67, 17, 21, 47, 77, 107, 31, 119, 65, 51, 0, 106, 18, 63, 116, 56, 62, 15, 7, 120, 58, 53, 126, 83, 52, 112, 115, 40, 11, 50, 39, 103, 43, 121, 104, 36, 124, 2, 123, 12, 20, 99, 125, 3, 118, 45, 48, 41, 59, 37, 22, 32, 29, 98, 71, 42, 89, 72, 97, 25, 35, 101, 69, 88, 16, 100, 27, 93, 95, 34, 91, 82, 96, 5, 4, 10, 1, 84, 38, 64, 73, 6], [110, 102, 33, 46, 49, 57, 92, 113, 111, 86, 28, 19, 24, 81, 85, 61, 116, 119, 56, 122, 115, 87, 26, 79, 108, 90, 107, 78, 94, 63, 109, 114, 9, 13, 52, 23, 121, 127, 30, 74, 47, 104, 123, 70, 21, 17, 99, 126, 59, 48, 62, 51, 31, 14, 60, 98, 124, 58, 125, 80, 55, 106, 54, 43, 112, 53, 120, 50, 100, 44, 118, 29, 42, 76, 41, 105, 117, 103, 75, 77, 8, 45, 37, 36, 83, 15, 101, 25, 20, 66, 68, 35, 22, 95, 34, 40, 12, 18, 39, 93, 11, 32, 96, 0, 97, 38, 27, 89, 88, 69, 91, 3, 65, 84, 1, 72, 4, 10, 71, 73, 7, 16, 82, 6, 64, 5, 67, 2], [110, 102, 46, 113, 49, 33, 92, 111, 86, 19, 122, 28, 24, 81, 79, 55, 60, 85, 78, 39, 26, 127, 87, 74, 115, 68, 90, 93, 12, 118, 119, 114, 77, 70, 9, 125, 56, 8, 13, 50, 84, 108, 61, 29, 107, 98, 14, 59, 37, 105, 51, 100, 22, 112, 73, 71, 35, 25, 47, 94, 58, 88, 44, 117, 124, 53, 120, 42, 106, 21, 95, 57, 40, 23, 20, 17, 83, 82, 121, 2, 96, 62, 54, 52, 32, 30, 41, 65, 31, 116, 43, 109, 76, 104, 126, 75, 36, 64, 63, 38, 99, 45, 48, 91, 101, 34, 11, 5, 80, 103, 123, 66, 15, 89, 27, 18, 10, 97, 16, 4, 6, 72, 0, 7, 3, 69, 67, 1], [110, 102, 46, 49, 33, 113, 92, 111, 86, 122, 19, 28, 54, 81, 85, 119, 24, 90, 26, 109, 94, 126, 79, 70, 13, 120, 78, 9, 51, 57, 112, 74, 125, 21, 87, 108, 123, 61, 50, 59, 105, 107, 121, 52, 14, 58, 8, 53, 47, 56, 68, 17, 60, 43, 45, 100, 93, 127, 114, 63, 12, 118, 106, 80, 44, 116, 55, 20, 48, 29, 15, 83, 30, 124, 23, 117, 103, 77, 31, 39, 65, 115, 42, 62, 75, 22, 104, 91, 40, 76, 99, 36, 71, 66, 34, 97, 101, 41, 98, 37, 32, 95, 88, 35, 82, 10, 27, 89, 18, 96, 25, 73, 84, 5, 11, 64, 16, 38, 72, 3, 0, 2, 4, 69, 7, 6, 1, 67], [48, 39, 119, 117, 56, 33, 60, 112, 121, 127, 58, 120, 47, 116, 125, 63, 122, 49, 90, 29, 114, 118, 55, 51, 123, 115, 124, 52, 59, 126, 113, 61, 62, 53, 107, 24, 44, 111, 54, 50, 110, 45, 91, 108, 93, 46, 106, 57, 88, 95, 109, 26, 86, 42, 20, 40, 43, 41, 105, 102, 17, 101, 104, 97, 36, 103, 38, 27, 85, 87, 37, 92, 100, 34, 22, 96, 94, 99, 98, 28, 89, 35, 21, 81, 16, 32, 31, 30, 84, 83, 78, 76, 14, 25, 12, 15, 82, 80, 10, 23, 18, 74, 19, 72, 65, 1, 67, 3, 0, 13, 64, 8, 69, 5, 68, 66, 2, 79, 4, 11, 7, 6, 77, 71, 9, 75, 73, 70], [119, 39, 48, 117, 60, 33, 56, 121, 47, 127, 120, 55, 63, 125, 58, 116, 51, 122, 49, 123, 114, 118, 115, 61, 52, 112, 90, 53, 59, 113, 106, 44, 62, 126, 124, 110, 29, 57, 54, 24, 107, 50, 111, 45, 108, 46, 41, 91, 93, 26, 95, 88, 105, 20, 86, 109, 43, 42, 102, 40, 104, 17, 103, 92, 101, 100, 38, 27, 97, 21, 34, 36, 94, 85, 28, 37, 87, 35, 99, 96, 89, 31, 22, 98, 16, 32, 76, 30, 81, 84, 25, 83, 15, 80, 78, 23, 14, 12, 82, 18, 10, 74, 1, 0, 65, 67, 64, 8, 19, 69, 72, 5, 4, 3, 13, 79, 11, 68, 66, 2, 7, 6, 71, 75, 70, 9, 77, 73], [117, 39, 119, 48, 56, 90, 60, 33, 55, 62, 124, 52, 121, 120, 95, 127, 47, 46, 105, 93, 58, 29, 116, 106, 61, 115, 122, 114, 63, 83, 26, 125, 51, 10, 118, 102, 126, 113, 49, 24, 57, 107, 50, 53, 123, 89, 59, 45, 104, 109, 108, 96, 54, 16, 111, 86, 44, 110, 112, 20, 42, 27, 43, 18, 41, 81, 85, 78, 38, 30, 40, 32, 94, 31, 19, 36, 25, 17, 34, 101, 82, 92, 100, 98, 14, 88, 91, 23, 75, 35, 97, 84, 87, 28, 71, 37, 13, 99, 22, 103, 12, 8, 21, 76, 3, 15, 11, 80, 4, 9, 68, 2, 72, 7, 74, 79, 5, 0, 6, 69, 73, 70, 67, 77, 1, 66, 65, 64], [39, 117, 119, 48, 83, 33, 15, 13, 11, 9, 29, 86, 6, 89, 66, 7, 74, 55, 88, 27, 87, 73, 81, 68, 80, 53, 90, 72, 22, 84, 91, 75, 82, 85, 8, 65, 30, 23, 18, 62, 26, 25, 17, 56, 16, 34, 0, 77, 79, 12, 21, 67, 76, 19, 92, 14, 31, 78, 70, 71, 28, 94, 4, 20, 5, 10, 32, 58, 50, 96, 35, 3, 64, 43, 106, 45, 24, 95, 40, 2, 69, 47, 102, 104, 93, 112, 38, 105, 99, 101, 54, 121, 108, 61, 124, 120, 57, 122, 1, 103, 110, 41, 98, 127, 49, 100, 115, 123, 36, 60, 111, 44, 63, 125, 118, 42, 52, 51, 107, 37, 116, 59, 97, 114, 113, 126, 109, 46], [44, 37, 108, 76, 18, 21, 14, 97, 24, 57, 80, 28, 71, 0, 5, 73, 125, 90, 69, 67, 7, 2, 51, 121, 58, 3, 11, 72, 75, 113, 8, 10, 19, 52, 23, 12, 89, 118, 55, 33, 91, 68, 16, 64, 65, 9, 88, 101, 34, 92, 70, 96, 77, 85, 13, 1, 84, 119, 99, 124, 20, 109, 25, 123, 66, 86, 26, 6, 82, 46, 93, 83, 78, 127, 114, 104, 79, 74, 15, 81, 35, 4, 100, 95, 47, 111, 29, 30, 27, 50, 56, 120, 98, 60, 126, 17, 117, 22, 31, 32, 61, 63, 110, 94, 36, 87, 106, 62, 103, 45, 41, 39, 102, 38, 105, 112, 43, 40, 107, 54, 116, 115, 122, 48, 42, 49, 53, 59], [44, 37, 108, 28, 24, 21, 97, 80, 18, 14, 19, 76, 73, 57, 87, 51, 90, 58, 71, 125, 15, 121, 17, 113, 117, 47, 5, 55, 11, 7, 106, 95, 69, 23, 52, 50, 98, 2, 93, 30, 114, 99, 85, 83, 45, 86, 22, 127, 27, 92, 10, 66, 35, 109, 31, 84, 20, 46, 3, 81, 82, 100, 29, 6, 75, 91, 79, 78, 124, 70, 16, 34, 96, 118, 105, 12, 33, 123, 61, 13, 9, 48, 89, 94, 88, 26, 36, 8, 111, 104, 25, 112, 32, 40, 103, 77, 38, 42, 53, 120, 126, 102, 41, 119, 107, 49, 43, 74, 110, 72, 68, 115, 0, 67, 122, 116, 39, 56, 54, 63, 59, 60, 101, 62, 4, 64, 65, 1], [57, 44, 37, 125, 108, 90, 97, 19, 28, 58, 21, 114, 24, 126, 113, 109, 117, 17, 86, 116, 121, 61, 51, 40, 110, 50, 49, 122, 55, 118, 41, 46, 59, 47, 11, 48, 43, 111, 106, 56, 124, 102, 63, 53, 123, 52, 62, 115, 112, 120, 119, 91, 54, 60, 127, 42, 107, 80, 104, 45, 105, 18, 87, 93, 39, 95, 14, 72, 35, 38, 84, 73, 77, 103, 23, 13, 101, 75, 27, 36, 20, 74, 66, 34, 89, 83, 25, 15, 26, 6, 30, 98, 8, 33, 100, 4, 29, 22, 81, 99, 94, 5, 79, 31, 68, 96, 32, 70, 76, 69, 2, 10, 1, 65, 12, 85, 92, 0, 64, 88, 78, 7, 9, 82, 16, 3, 71, 67], [44, 37, 125, 24, 108, 21, 28, 18, 97, 14, 90, 19, 58, 80, 57, 51, 11, 121, 114, 73, 109, 17, 13, 95, 118, 117, 52, 87, 75, 113, 69, 35, 50, 93, 61, 81, 46, 123, 76, 9, 124, 34, 119, 102, 71, 111, 115, 59, 36, 104, 100, 32, 96, 4, 55, 103, 20, 84, 99, 30, 85, 45, 105, 60, 106, 62, 29, 86, 83, 39, 25, 43, 40, 48, 66, 94, 38, 42, 120, 107, 47, 91, 110, 23, 112, 82, 15, 127, 78, 116, 98, 92, 16, 31, 122, 54, 89, 126, 49, 33, 72, 88, 41, 56, 22, 53, 101, 26, 10, 63, 74, 79, 67, 6, 1, 27, 8, 77, 5, 64, 12, 2, 3, 70, 68, 65, 7, 0], [54, 37, 127, 62, 63, 24, 116, 33, 101, 119, 123, 18, 55, 59, 51, 120, 58, 112, 46, 53, 60, 125, 114, 117, 113, 39, 126, 44, 15, 57, 61, 56, 111, 121, 122, 118, 47, 52, 49, 27, 110, 50, 48, 91, 115, 45, 124, 94, 109, 85, 107, 108, 105, 106, 43, 92, 86, 20, 21, 104, 42, 73, 103, 41, 102, 88, 100, 30, 40, 38, 25, 29, 82, 23, 95, 36, 13, 28, 97, 35, 31, 22, 9, 77, 79, 99, 84, 90, 98, 34, 16, 93, 87, 26, 17, 89, 12, 14, 1, 80, 75, 71, 65, 32, 4, 19, 6, 96, 70, 2, 66, 10, 7, 11, 68, 67, 72, 78, 3, 76, 5, 69, 0, 83, 8, 81, 64, 74], [127, 54, 37, 25, 62, 17, 94, 101, 33, 86, 15, 123, 20, 6, 63, 24, 14, 12, 27, 77, 19, 10, 90, 30, 109, 23, 116, 84, 88, 13, 61, 93, 72, 113, 9, 51, 107, 95, 91, 59, 120, 38, 119, 92, 125, 117, 112, 22, 7, 41, 124, 70, 32, 58, 85, 35, 67, 36, 75, 118, 48, 42, 78, 31, 18, 126, 122, 21, 79, 28, 102, 55, 34, 80, 29, 115, 114, 68, 82, 5, 11, 53, 50, 16, 87, 57, 83, 103, 46, 98, 106, 81, 26, 56, 60, 44, 4, 111, 39, 74, 89, 8, 108, 76, 52, 110, 121, 1, 45, 49, 73, 40, 96, 100, 97, 66, 105, 104, 99, 43, 47, 0, 71, 3, 2, 69, 65, 64], [127, 63, 37, 62, 54, 24, 33, 51, 112, 123, 116, 101, 117, 55, 120, 59, 58, 119, 125, 53, 19, 114, 46, 111, 44, 60, 113, 122, 61, 47, 126, 57, 121, 91, 118, 52, 56, 49, 48, 27, 50, 45, 94, 110, 115, 85, 124, 39, 108, 109, 18, 102, 86, 106, 88, 30, 107, 29, 105, 43, 41, 104, 103, 42, 96, 21, 14, 40, 38, 95, 20, 97, 32, 26, 100, 22, 16, 31, 75, 83, 36, 25, 98, 12, 92, 35, 99, 78, 34, 15, 90, 82, 13, 93, 80, 17, 6, 11, 73, 77, 9, 65, 1, 84, 89, 28, 87, 23, 68, 4, 66, 2, 7, 70, 10, 71, 72, 5, 8, 76, 79, 0, 81, 67, 64, 3, 69, 74], [127, 54, 37, 33, 17, 62, 86, 10, 15, 24, 14, 123, 12, 5, 72, 90, 27, 65, 7, 116, 25, 0, 101, 63, 77, 91, 64, 20, 44, 23, 88, 73, 4, 19, 67, 87, 89, 113, 22, 71, 112, 2, 93, 8, 107, 83, 59, 81, 111, 80, 76, 68, 79, 74, 82, 118, 21, 13, 6, 120, 104, 46, 114, 75, 126, 28, 119, 70, 122, 18, 3, 97, 11, 30, 102, 1, 16, 84, 69, 56, 31, 106, 58, 26, 9, 66, 78, 32, 85, 48, 95, 40, 117, 61, 94, 96, 45, 125, 99, 35, 29, 98, 57, 92, 60, 39, 53, 124, 34, 36, 100, 51, 38, 52, 103, 41, 108, 55, 109, 110, 105, 47, 43, 121, 49, 50, 115, 42], [43, 121, 45, 107, 100, 97, 109, 27, 91, 102, 123, 24, 103, 15, 22, 54, 81, 59, 88, 127, 120, 118, 114, 117, 110, 20, 44, 124, 85, 61, 111, 31, 79, 119, 51, 26, 75, 57, 7, 41, 11, 33, 58, 98, 49, 92, 46, 23, 112, 39, 52, 99, 42, 13, 113, 55, 21, 62, 29, 17, 122, 28, 47, 84, 63, 125, 25, 48, 18, 93, 50, 108, 126, 56, 82, 115, 116, 86, 101, 104, 37, 105, 53, 60, 32, 30, 96, 38, 40, 35, 34, 89, 80, 106, 90, 19, 94, 77, 83, 95, 16, 73, 87, 71, 74, 70, 76, 14, 78, 36, 3, 6, 9, 12, 5, 69, 8, 10, 72, 66, 67, 65, 2, 1, 68, 64, 0, 4], [43, 107, 123, 117, 100, 27, 97, 91, 88, 51, 103, 21, 49, 124, 80, 59, 54, 24, 109, 55, 48, 85, 115, 127, 57, 14, 50, 121, 61, 114, 113, 126, 58, 118, 22, 45, 119, 60, 62, 52, 41, 31, 63, 125, 112, 47, 111, 74, 122, 116, 53, 29, 19, 76, 110, 102, 46, 44, 56, 108, 39, 93, 37, 120, 5, 20, 42, 89, 8, 30, 90, 6, 2, 99, 105, 3, 18, 40, 106, 68, 35, 101, 98, 87, 104, 84, 38, 78, 32, 33, 16, 1, 86, 12, 17, 96, 0, 28, 34, 25, 94, 26, 81, 10, 72, 11, 95, 23, 15, 36, 92, 4, 69, 83, 82, 66, 73, 65, 64, 70, 13, 77, 67, 71, 7, 75, 9, 79], [43, 121, 45, 107, 100, 27, 97, 91, 116, 63, 24, 22, 61, 106, 125, 54, 114, 37, 15, 49, 88, 118, 58, 124, 29, 33, 85, 76, 120, 57, 31, 17, 19, 32, 81, 21, 41, 127, 47, 109, 123, 80, 50, 18, 52, 60, 51, 55, 98, 122, 74, 86, 53, 101, 113, 34, 105, 103, 12, 28, 119, 14, 111, 62, 94, 93, 59, 8, 115, 46, 38, 44, 112, 42, 7, 110, 90, 40, 126, 68, 48, 20, 117, 26, 104, 102, 36, 99, 108, 35, 2, 39, 96, 30, 92, 84, 13, 11, 56, 79, 82, 25, 95, 89, 75, 73, 83, 77, 1, 78, 5, 3, 10, 87, 71, 69, 0, 72, 9, 23, 67, 6, 4, 66, 65, 64, 70, 16], [121, 43, 45, 100, 91, 97, 27, 107, 88, 85, 21, 123, 80, 51, 54, 24, 115, 49, 61, 117, 127, 126, 48, 59, 57, 114, 109, 46, 52, 116, 118, 63, 55, 74, 53, 50, 22, 14, 76, 60, 105, 31, 62, 119, 124, 120, 111, 19, 122, 125, 113, 58, 106, 47, 44, 56, 102, 8, 112, 90, 103, 110, 12, 108, 39, 93, 37, 20, 32, 29, 42, 41, 99, 89, 81, 33, 5, 87, 2, 104, 17, 40, 38, 6, 34, 16, 82, 26, 101, 78, 28, 86, 96, 92, 30, 3, 35, 1, 98, 36, 68, 94, 72, 23, 95, 18, 10, 84, 0, 69, 25, 83, 15, 70, 4, 73, 65, 67, 11, 13, 66, 64, 77, 79, 71, 9, 7, 75], [103, 115, 51, 85, 83, 80, 62, 10, 90, 13, 124, 118, 26, 63, 72, 48, 56, 70, 60, 98, 68, 52, 55, 88, 106, 47, 59, 41, 53, 50, 93, 110, 30, 91, 100, 107, 76, 71, 86, 58, 111, 19, 45, 84, 15, 66, 82, 122, 61, 42, 123, 57, 125, 38, 44, 96, 89, 77, 119, 65, 81, 40, 21, 28, 16, 14, 87, 11, 3, 23, 25, 94, 101, 104, 43, 33, 102, 17, 74, 27, 105, 121, 49, 95, 112, 31, 92, 117, 116, 24, 108, 20, 120, 36, 7, 9, 109, 5, 114, 46, 97, 34, 69, 126, 127, 73, 35, 113, 79, 8, 99, 22, 29, 18, 32, 78, 54, 12, 37, 64, 2, 39, 6, 1, 75, 4, 0, 67], [103, 51, 115, 85, 83, 10, 80, 13, 72, 70, 66, 68, 26, 62, 63, 90, 118, 60, 81, 124, 107, 64, 2, 91, 53, 59, 55, 44, 52, 119, 89, 56, 65, 93, 106, 47, 111, 50, 0, 123, 98, 19, 15, 100, 40, 76, 1, 110, 61, 5, 48, 114, 69, 77, 16, 41, 25, 6, 99, 74, 8, 4, 104, 3, 32, 126, 79, 11, 94, 117, 105, 22, 121, 87, 20, 88, 29, 58, 116, 46, 43, 73, 112, 14, 75, 21, 27, 23, 127, 24, 120, 35, 71, 38, 86, 109, 57, 49, 82, 92, 125, 45, 113, 122, 67, 54, 102, 78, 96, 84, 9, 108, 18, 95, 12, 101, 42, 37, 30, 33, 17, 28, 34, 39, 7, 31, 36, 97], [103, 51, 115, 85, 80, 13, 10, 68, 83, 70, 72, 26, 66, 0, 62, 124, 55, 65, 90, 118, 63, 53, 60, 64, 47, 76, 100, 98, 89, 107, 46, 59, 88, 50, 56, 48, 61, 2, 91, 93, 82, 1, 58, 23, 111, 4, 3, 19, 81, 73, 99, 121, 25, 44, 5, 106, 123, 6, 69, 75, 74, 11, 41, 87, 119, 15, 20, 127, 71, 79, 21, 95, 77, 110, 8, 52, 40, 126, 16, 114, 67, 27, 17, 112, 9, 122, 78, 94, 57, 22, 117, 86, 49, 12, 14, 32, 39, 113, 30, 37, 38, 92, 120, 18, 29, 105, 84, 96, 31, 36, 7, 33, 108, 42, 125, 35, 116, 97, 101, 24, 109, 45, 104, 54, 43, 102, 28, 34], [103, 115, 51, 80, 85, 10, 13, 83, 72, 70, 68, 62, 26, 124, 118, 60, 59, 55, 66, 107, 90, 82, 64, 93, 119, 47, 52, 25, 15, 63, 48, 53, 98, 40, 88, 56, 111, 87, 73, 5, 1, 89, 76, 65, 69, 23, 29, 2, 61, 71, 0, 46, 44, 81, 100, 50, 117, 77, 58, 19, 22, 96, 84, 32, 112, 110, 24, 74, 8, 7, 3, 16, 114, 12, 17, 122, 126, 121, 125, 91, 27, 11, 123, 35, 38, 28, 99, 4, 104, 106, 116, 20, 101, 45, 6, 9, 79, 31, 78, 95, 41, 18, 30, 21, 86, 67, 92, 49, 108, 113, 43, 94, 37, 33, 57, 42, 127, 36, 105, 109, 54, 34, 102, 14, 120, 97, 75, 39]], "model.layers.23.self_attn.k_proj": [[47, 111, 36, 107, 88, 96, 30, 90, 80, 21, 0, 78, 81, 2, 27, 104, 68, 84, 83, 7, 15, 75, 74, 93, 95, 5, 103, 106, 10, 77, 11, 86, 124, 110, 22, 6, 120, 91, 59, 71, 73, 85, 43, 122, 46, 127, 13, 51, 67, 94, 119, 114, 48, 32, 50, 105, 55, 92, 101, 23, 66, 118, 42, 52, 33, 97, 56, 63, 60, 102, 125, 35, 39, 123, 72, 29, 31, 99, 116, 64, 19, 14, 45, 41, 53, 61, 112, 28, 12, 117, 49, 58, 113, 16, 40, 34, 57, 17, 25, 38, 108, 62, 3, 79, 98, 126, 37, 54, 109, 121, 18, 4, 44, 115, 24, 20, 26, 82, 89, 87, 65, 8, 76, 9, 100, 1, 70, 69], [39, 56, 46, 51, 86, 33, 110, 116, 61, 53, 30, 20, 60, 47, 57, 124, 55, 59, 120, 80, 89, 58, 114, 119, 52, 122, 127, 112, 49, 91, 126, 123, 63, 54, 118, 48, 115, 121, 44, 113, 62, 34, 125, 45, 106, 109, 117, 92, 50, 111, 101, 96, 100, 93, 14, 94, 108, 105, 107, 88, 98, 43, 42, 35, 40, 24, 37, 78, 104, 38, 41, 36, 11, 102, 97, 95, 27, 16, 90, 82, 29, 65, 72, 13, 31, 19, 84, 71, 99, 64, 5, 73, 26, 87, 85, 3, 32, 25, 10, 28, 12, 17, 21, 15, 69, 9, 18, 7, 23, 70, 6, 0, 76, 1, 79, 77, 103, 75, 83, 4, 67, 22, 68, 66, 81, 8, 2, 74], [46, 113, 38, 110, 97, 28, 19, 86, 79, 26, 94, 24, 81, 21, 87, 77, 122, 14, 70, 74, 47, 17, 78, 60, 68, 9, 59, 52, 126, 65, 57, 112, 111, 119, 62, 44, 8, 55, 114, 20, 105, 115, 56, 127, 108, 117, 106, 85, 58, 76, 104, 0, 125, 50, 11, 42, 64, 93, 118, 45, 41, 109, 13, 2, 43, 103, 80, 124, 92, 48, 123, 63, 51, 116, 39, 16, 121, 120, 36, 61, 66, 100, 53, 99, 75, 54, 40, 18, 89, 101, 49, 37, 107, 90, 15, 73, 95, 5, 29, 72, 32, 71, 23, 3, 35, 25, 31, 34, 96, 84, 98, 12, 91, 27, 30, 22, 69, 82, 10, 33, 88, 7, 6, 83, 4, 67, 1, 102], [117, 103, 119, 86, 48, 97, 112, 13, 93, 11, 83, 15, 56, 26, 60, 120, 124, 47, 121, 6, 89, 113, 53, 122, 116, 63, 58, 127, 55, 125, 115, 24, 9, 114, 123, 7, 52, 49, 118, 51, 61, 62, 91, 59, 54, 126, 18, 107, 81, 111, 45, 42, 50, 46, 44, 57, 109, 16, 110, 20, 43, 108, 74, 41, 73, 98, 104, 29, 92, 30, 23, 105, 40, 25, 106, 37, 88, 35, 38, 21, 68, 17, 64, 100, 101, 78, 102, 31, 14, 27, 36, 99, 2, 95, 28, 8, 32, 96, 94, 34, 69, 85, 10, 82, 4, 19, 87, 84, 66, 90, 80, 12, 76, 79, 75, 65, 33, 70, 71, 5, 77, 72, 67, 0, 39, 1, 3, 22], [108, 101, 44, 57, 21, 80, 28, 14, 24, 11, 71, 33, 18, 76, 73, 90, 58, 125, 19, 67, 50, 5, 64, 17, 1, 52, 55, 15, 118, 117, 121, 51, 115, 122, 53, 119, 4, 47, 48, 69, 75, 124, 123, 120, 95, 45, 35, 31, 111, 2, 59, 13, 87, 126, 12, 104, 61, 60, 56, 72, 110, 42, 127, 41, 114, 62, 43, 112, 8, 84, 7, 63, 109, 107, 68, 116, 54, 23, 40, 98, 106, 88, 91, 39, 6, 46, 49, 27, 105, 10, 70, 89, 113, 26, 30, 86, 36, 29, 34, 103, 32, 100, 92, 38, 81, 102, 94, 22, 78, 93, 74, 3, 66, 16, 0, 97, 77, 99, 25, 79, 9, 96, 82, 20, 85, 83, 37, 65], [127, 54, 101, 97, 86, 24, 27, 123, 118, 18, 94, 77, 113, 63, 116, 120, 48, 17, 10, 64, 58, 85, 62, 59, 117, 57, 15, 38, 114, 12, 111, 67, 42, 121, 122, 14, 108, 29, 55, 61, 7, 125, 53, 126, 60, 49, 103, 1, 51, 119, 19, 56, 16, 52, 112, 124, 43, 109, 50, 5, 105, 45, 110, 46, 115, 47, 31, 72, 75, 71, 37, 90, 6, 25, 66, 107, 40, 44, 20, 39, 89, 87, 95, 104, 106, 83, 36, 100, 41, 34, 102, 93, 98, 68, 96, 11, 99, 0, 2, 35, 3, 21, 26, 92, 9, 79, 23, 30, 28, 80, 32, 78, 91, 81, 84, 76, 8, 88, 73, 69, 74, 13, 82, 33, 70, 22, 4, 65], [107, 33, 121, 36, 22, 91, 43, 124, 88, 45, 61, 80, 21, 51, 54, 60, 57, 49, 74, 55, 119, 123, 58, 115, 117, 29, 116, 62, 63, 122, 127, 50, 26, 125, 53, 52, 111, 42, 118, 59, 112, 113, 109, 120, 126, 46, 110, 56, 48, 38, 47, 76, 95, 108, 101, 19, 114, 39, 14, 65, 17, 106, 66, 44, 69, 4, 78, 20, 104, 12, 41, 0, 105, 98, 8, 35, 37, 15, 30, 67, 103, 70, 64, 83, 102, 6, 23, 40, 99, 31, 32, 34, 94, 92, 5, 81, 10, 87, 73, 96, 28, 25, 100, 82, 9, 97, 18, 11, 93, 90, 89, 79, 71, 27, 72, 86, 85, 77, 84, 13, 1, 7, 3, 75, 68, 2, 24, 16], [115, 39, 13, 72, 51, 80, 64, 83, 10, 70, 85, 68, 66, 90, 62, 2, 124, 0, 65, 76, 43, 111, 26, 60, 91, 112, 118, 75, 53, 55, 61, 56, 67, 5, 81, 88, 63, 4, 119, 89, 46, 42, 40, 34, 59, 52, 50, 108, 73, 29, 123, 1, 93, 82, 114, 15, 121, 105, 23, 35, 36, 71, 126, 30, 31, 22, 96, 7, 122, 125, 84, 69, 41, 110, 58, 3, 102, 79, 25, 109, 87, 86, 24, 120, 94, 57, 127, 38, 14, 98, 28, 100, 104, 99, 6, 106, 117, 44, 33, 107, 54, 45, 8, 48, 101, 95, 78, 97, 49, 116, 17, 12, 47, 18, 92, 27, 32, 20, 9, 113, 37, 11, 74, 21, 19, 77, 16, 103]], "model.layers.23.self_attn.qk_proj": [[46, 115, 111, 127, 54, 56, 47, 110, 108, 119, 117, 107, 51, 44, 121, 48, 43, 113, 22, 24, 26, 21, 55, 83, 85, 57, 62, 19, 92, 27, 124, 101, 80, 88, 118, 90, 63, 39, 37, 16, 59, 86, 45, 60, 123, 103, 61, 33, 13, 116, 58, 125, 49, 122, 81, 77, 74, 97, 14, 10, 120, 78, 50, 72, 79, 53, 52, 17, 36, 28, 93, 15, 91, 94, 29, 70, 106, 112, 11, 126, 75, 68, 109, 8, 4, 64, 82, 114, 0, 89, 71, 18, 100, 2, 25, 9, 76, 66, 102, 38, 12, 30, 105, 23, 73, 7, 42, 5, 96, 32, 20, 84, 104, 40, 95, 87, 69, 6, 41, 98, 1, 31, 34, 67, 99, 35, 65, 3], [46, 115, 127, 111, 54, 56, 47, 110, 108, 119, 107, 117, 51, 44, 121, 48, 43, 113, 57, 21, 22, 26, 83, 124, 27, 118, 39, 24, 92, 62, 85, 63, 55, 101, 19, 88, 16, 60, 86, 90, 80, 58, 37, 59, 123, 116, 33, 125, 74, 61, 103, 49, 77, 45, 78, 13, 10, 81, 120, 112, 52, 70, 91, 122, 53, 14, 72, 97, 50, 17, 15, 93, 114, 94, 28, 4, 0, 11, 29, 36, 106, 126, 2, 66, 79, 109, 75, 8, 82, 68, 102, 30, 38, 25, 105, 100, 64, 76, 73, 12, 104, 23, 18, 42, 89, 7, 20, 9, 71, 5, 96, 31, 69, 32, 98, 84, 95, 6, 40, 41, 87, 3, 34, 35, 65, 99, 1, 67], [46, 115, 111, 127, 56, 54, 110, 47, 108, 119, 107, 117, 51, 44, 48, 121, 43, 113, 57, 22, 124, 26, 62, 85, 21, 24, 118, 83, 27, 55, 39, 92, 37, 63, 19, 101, 60, 90, 80, 88, 33, 58, 16, 86, 103, 59, 97, 61, 123, 125, 74, 53, 77, 122, 116, 45, 14, 13, 78, 91, 120, 10, 52, 72, 81, 49, 112, 70, 36, 66, 4, 28, 50, 15, 17, 109, 93, 64, 79, 11, 29, 126, 2, 114, 38, 94, 0, 68, 106, 25, 12, 82, 75, 104, 100, 23, 76, 89, 18, 42, 71, 7, 30, 84, 8, 96, 105, 41, 20, 5, 9, 32, 98, 73, 102, 40, 69, 87, 35, 31, 95, 6, 34, 99, 65, 67, 1, 3], [46, 115, 127, 111, 54, 56, 47, 110, 108, 107, 119, 117, 51, 44, 121, 48, 43, 113, 57, 22, 21, 83, 63, 124, 26, 62, 118, 60, 19, 24, 85, 92, 39, 88, 58, 27, 55, 80, 101, 37, 16, 122, 53, 123, 86, 59, 90, 77, 61, 33, 125, 10, 116, 17, 81, 13, 14, 78, 120, 103, 74, 45, 72, 112, 49, 97, 15, 70, 52, 4, 50, 91, 11, 66, 114, 106, 126, 2, 109, 36, 100, 68, 29, 28, 93, 94, 0, 18, 75, 76, 79, 12, 71, 38, 89, 8, 102, 7, 69, 64, 42, 73, 82, 25, 9, 104, 30, 84, 20, 5, 96, 105, 23, 6, 31, 41, 32, 98, 87, 95, 35, 65, 67, 40, 99, 34, 3, 1], [46, 115, 127, 111, 54, 47, 56, 110, 108, 119, 117, 107, 51, 44, 48, 121, 43, 113, 57, 62, 83, 124, 63, 22, 21, 19, 26, 85, 55, 24, 60, 27, 101, 39, 118, 86, 16, 37, 92, 58, 90, 122, 123, 88, 80, 61, 33, 103, 97, 72, 59, 14, 13, 10, 77, 45, 74, 49, 125, 52, 4, 81, 78, 68, 112, 17, 50, 116, 75, 53, 120, 91, 66, 79, 15, 28, 11, 109, 0, 126, 106, 93, 114, 29, 64, 2, 36, 12, 94, 70, 6, 7, 18, 20, 76, 25, 100, 82, 71, 38, 102, 9, 23, 73, 69, 89, 8, 42, 30, 5, 105, 84, 40, 104, 41, 32, 98, 96, 35, 34, 87, 31, 99, 95, 67, 3, 1, 65], [46, 115, 111, 127, 54, 47, 56, 110, 108, 119, 107, 117, 51, 44, 121, 48, 43, 113, 22, 57, 21, 26, 19, 27, 55, 24, 83, 62, 85, 101, 63, 39, 16, 80, 124, 88, 86, 58, 92, 60, 118, 37, 122, 72, 90, 123, 10, 33, 77, 45, 74, 13, 61, 81, 14, 97, 59, 125, 103, 17, 112, 49, 15, 78, 116, 75, 120, 53, 52, 0, 50, 91, 11, 68, 126, 28, 79, 64, 94, 66, 29, 82, 106, 114, 20, 36, 100, 6, 7, 4, 12, 76, 93, 25, 18, 23, 109, 38, 71, 2, 69, 102, 89, 30, 8, 105, 9, 70, 84, 73, 96, 42, 87, 32, 34, 95, 41, 5, 104, 65, 98, 40, 1, 35, 67, 31, 99, 3], [46, 115, 111, 127, 54, 56, 110, 47, 108, 119, 107, 51, 121, 44, 48, 117, 43, 113, 26, 22, 57, 21, 27, 85, 24, 83, 19, 88, 124, 92, 62, 80, 16, 101, 90, 86, 63, 39, 58, 60, 55, 77, 37, 122, 123, 33, 14, 103, 125, 74, 118, 10, 13, 97, 72, 81, 120, 45, 61, 59, 17, 78, 112, 116, 28, 49, 91, 106, 93, 50, 15, 79, 36, 53, 6, 126, 64, 114, 75, 82, 11, 52, 94, 109, 29, 68, 12, 89, 20, 38, 18, 25, 100, 4, 8, 66, 76, 23, 84, 102, 7, 0, 9, 71, 2, 73, 87, 42, 96, 105, 30, 32, 98, 31, 69, 70, 34, 5, 1, 41, 95, 104, 40, 35, 65, 99, 67, 3], [46, 115, 111, 127, 56, 54, 110, 47, 108, 119, 107, 51, 117, 44, 121, 48, 43, 113, 27, 21, 22, 24, 83, 57, 85, 26, 60, 92, 62, 19, 124, 55, 16, 80, 88, 90, 39, 86, 101, 123, 63, 33, 37, 61, 118, 58, 125, 59, 49, 77, 122, 116, 13, 97, 74, 103, 14, 45, 10, 81, 78, 112, 79, 17, 120, 28, 114, 36, 6, 91, 50, 52, 126, 72, 53, 94, 93, 11, 82, 106, 75, 4, 100, 109, 68, 29, 15, 18, 8, 0, 38, 23, 64, 89, 2, 25, 12, 105, 102, 66, 32, 73, 76, 30, 71, 20, 42, 96, 84, 9, 104, 87, 7, 70, 31, 98, 95, 5, 69, 65, 40, 41, 35, 34, 99, 1, 67, 3], [46, 115, 111, 127, 56, 54, 110, 47, 108, 107, 117, 119, 51, 121, 44, 48, 43, 113, 57, 83, 85, 24, 26, 21, 124, 22, 92, 27, 19, 88, 60, 55, 16, 62, 90, 123, 39, 101, 63, 86, 118, 58, 80, 33, 13, 116, 61, 37, 125, 97, 77, 74, 49, 45, 120, 103, 112, 81, 59, 17, 28, 122, 53, 10, 14, 52, 91, 78, 93, 79, 8, 36, 6, 29, 126, 11, 106, 75, 50, 100, 72, 114, 15, 12, 82, 109, 68, 94, 102, 23, 38, 4, 18, 76, 42, 73, 0, 2, 66, 105, 32, 104, 7, 89, 30, 41, 96, 31, 9, 64, 20, 25, 87, 98, 84, 5, 71, 95, 70, 34, 35, 40, 99, 69, 65, 3, 1, 67], [46, 115, 111, 127, 56, 54, 47, 110, 108, 117, 107, 51, 119, 44, 121, 48, 43, 113, 57, 62, 21, 26, 123, 85, 19, 124, 27, 83, 24, 22, 92, 118, 55, 39, 60, 16, 88, 86, 37, 58, 90, 63, 101, 33, 10, 80, 116, 61, 77, 45, 125, 8, 13, 103, 122, 59, 112, 74, 49, 97, 120, 17, 78, 52, 53, 14, 114, 2, 0, 81, 91, 28, 68, 4, 94, 36, 15, 93, 75, 29, 79, 50, 64, 109, 72, 6, 12, 106, 66, 126, 38, 71, 11, 76, 42, 89, 70, 82, 100, 18, 7, 73, 96, 25, 5, 9, 20, 105, 102, 23, 87, 104, 32, 30, 31, 69, 98, 41, 40, 84, 99, 1, 67, 34, 65, 95, 35, 3], [46, 115, 111, 127, 54, 47, 56, 110, 108, 117, 51, 119, 107, 44, 121, 43, 48, 113, 57, 124, 62, 22, 21, 85, 39, 19, 26, 80, 83, 123, 86, 92, 24, 16, 118, 60, 63, 88, 27, 58, 77, 37, 101, 55, 125, 61, 10, 13, 116, 90, 45, 103, 8, 120, 74, 97, 33, 59, 28, 52, 4, 14, 81, 75, 78, 112, 49, 17, 53, 50, 66, 68, 15, 91, 122, 11, 94, 79, 114, 29, 70, 64, 36, 71, 82, 2, 109, 106, 126, 0, 6, 100, 7, 12, 76, 42, 93, 20, 38, 25, 32, 72, 23, 9, 84, 73, 5, 18, 105, 89, 102, 69, 30, 104, 87, 96, 31, 95, 34, 98, 40, 99, 67, 3, 41, 1, 65, 35], [46, 115, 111, 127, 54, 56, 47, 110, 108, 107, 119, 51, 117, 44, 121, 48, 43, 113, 22, 21, 57, 85, 26, 83, 19, 24, 92, 124, 80, 88, 27, 86, 101, 39, 62, 63, 16, 90, 60, 58, 55, 33, 13, 77, 10, 8, 125, 118, 14, 37, 123, 74, 61, 97, 116, 45, 81, 52, 28, 49, 103, 91, 78, 59, 53, 120, 17, 79, 15, 70, 122, 75, 4, 112, 68, 11, 50, 36, 64, 94, 29, 93, 109, 12, 23, 126, 100, 106, 18, 114, 2, 38, 82, 25, 0, 9, 7, 76, 66, 71, 30, 84, 73, 20, 89, 102, 105, 32, 72, 6, 5, 96, 34, 98, 95, 69, 104, 87, 31, 40, 41, 42, 99, 1, 67, 35, 65, 3], [46, 115, 111, 127, 56, 110, 54, 47, 108, 107, 119, 117, 51, 44, 121, 48, 43, 113, 21, 57, 27, 24, 22, 85, 26, 60, 92, 19, 88, 80, 90, 55, 101, 83, 61, 16, 39, 86, 62, 124, 33, 58, 59, 63, 123, 49, 125, 122, 116, 91, 13, 103, 37, 74, 8, 77, 118, 97, 10, 52, 120, 81, 14, 45, 29, 53, 100, 17, 94, 28, 112, 70, 50, 36, 78, 15, 126, 93, 106, 79, 114, 64, 109, 38, 25, 82, 4, 18, 75, 23, 2, 102, 89, 12, 68, 105, 30, 0, 20, 11, 76, 71, 9, 72, 87, 66, 42, 96, 31, 7, 104, 98, 73, 41, 99, 65, 34, 84, 5, 32, 69, 95, 6, 35, 40, 1, 3, 67], [46, 115, 111, 127, 54, 110, 56, 47, 108, 117, 119, 51, 107, 44, 121, 48, 43, 113, 57, 22, 60, 21, 85, 19, 26, 55, 63, 80, 62, 27, 24, 123, 124, 92, 88, 101, 58, 86, 83, 39, 118, 61, 16, 59, 13, 10, 122, 37, 90, 45, 8, 74, 120, 81, 33, 49, 125, 52, 53, 77, 70, 112, 17, 116, 103, 14, 91, 28, 97, 78, 126, 64, 114, 93, 2, 68, 15, 94, 79, 11, 100, 50, 36, 4, 29, 0, 109, 106, 66, 75, 89, 12, 38, 76, 82, 18, 9, 42, 20, 30, 72, 23, 104, 7, 71, 105, 32, 102, 96, 69, 95, 41, 73, 25, 5, 84, 87, 6, 31, 98, 34, 1, 99, 67, 40, 35, 3, 65], [46, 115, 111, 127, 54, 56, 47, 110, 108, 51, 117, 107, 119, 44, 48, 121, 43, 113, 22, 57, 21, 19, 85, 26, 83, 24, 124, 62, 123, 55, 80, 27, 63, 92, 39, 88, 37, 16, 90, 60, 101, 61, 13, 58, 97, 86, 74, 8, 125, 103, 33, 118, 45, 116, 10, 77, 120, 122, 81, 52, 49, 78, 70, 94, 14, 91, 15, 59, 93, 36, 50, 68, 112, 17, 28, 66, 79, 0, 64, 53, 75, 106, 4, 2, 29, 11, 72, 102, 100, 109, 114, 71, 12, 126, 30, 7, 6, 76, 20, 82, 18, 105, 73, 9, 32, 25, 96, 42, 89, 38, 104, 69, 23, 41, 87, 5, 40, 35, 84, 98, 99, 34, 31, 95, 65, 67, 1, 3], [46, 115, 127, 111, 54, 56, 110, 47, 108, 107, 117, 51, 119, 44, 121, 48, 43, 113, 22, 57, 21, 19, 26, 124, 85, 62, 80, 123, 39, 83, 55, 88, 24, 16, 92, 60, 61, 86, 27, 13, 37, 90, 33, 77, 63, 101, 125, 118, 103, 10, 58, 74, 45, 49, 8, 59, 14, 78, 97, 122, 116, 81, 91, 17, 50, 120, 75, 28, 53, 93, 112, 52, 79, 11, 29, 68, 15, 94, 36, 70, 72, 100, 109, 4, 102, 64, 114, 126, 18, 76, 106, 6, 2, 73, 82, 71, 7, 12, 66, 9, 20, 32, 23, 89, 30, 0, 105, 25, 84, 38, 87, 69, 96, 41, 104, 42, 95, 40, 5, 98, 31, 3, 1, 34, 99, 35, 67, 65], [46, 115, 127, 111, 54, 56, 110, 47, 108, 107, 119, 51, 117, 44, 48, 121, 43, 113, 22, 21, 85, 57, 88, 26, 27, 19, 80, 24, 83, 92, 62, 86, 90, 124, 39, 55, 101, 61, 60, 33, 123, 16, 125, 103, 63, 116, 13, 97, 91, 58, 122, 37, 118, 10, 45, 77, 17, 59, 49, 74, 81, 14, 78, 94, 36, 112, 28, 93, 120, 52, 114, 29, 6, 72, 79, 53, 15, 102, 0, 8, 106, 100, 68, 18, 25, 126, 64, 82, 38, 50, 11, 12, 2, 109, 30, 75, 4, 23, 89, 66, 105, 71, 76, 41, 7, 32, 87, 96, 20, 84, 70, 73, 9, 98, 69, 5, 40, 95, 42, 65, 31, 104, 1, 99, 34, 35, 67, 3], [46, 115, 127, 111, 56, 54, 110, 47, 108, 107, 119, 51, 117, 44, 48, 121, 43, 113, 57, 22, 85, 83, 21, 19, 26, 124, 92, 86, 27, 62, 24, 60, 80, 63, 39, 123, 88, 101, 61, 103, 125, 13, 55, 90, 16, 37, 58, 10, 118, 77, 74, 120, 14, 91, 59, 33, 122, 97, 116, 53, 72, 94, 6, 112, 17, 45, 81, 36, 52, 49, 28, 114, 78, 15, 29, 79, 126, 4, 109, 75, 100, 50, 66, 2, 0, 11, 93, 82, 64, 106, 68, 12, 38, 102, 89, 18, 76, 8, 25, 84, 71, 30, 7, 87, 73, 104, 105, 32, 20, 9, 42, 96, 23, 41, 40, 98, 69, 31, 5, 65, 70, 34, 99, 95, 35, 1, 3, 67], [46, 115, 127, 111, 56, 54, 47, 110, 108, 119, 107, 51, 117, 44, 48, 121, 43, 113, 57, 22, 62, 85, 83, 26, 21, 123, 124, 19, 24, 63, 80, 92, 101, 39, 90, 58, 103, 60, 86, 88, 55, 45, 120, 118, 61, 125, 37, 74, 33, 27, 97, 16, 10, 53, 122, 116, 13, 77, 50, 72, 59, 91, 49, 52, 28, 78, 14, 114, 126, 112, 2, 6, 81, 93, 94, 79, 29, 11, 4, 17, 15, 0, 68, 106, 75, 36, 64, 102, 12, 18, 32, 71, 109, 66, 38, 82, 89, 105, 76, 8, 41, 73, 100, 7, 9, 25, 42, 31, 104, 20, 84, 98, 23, 69, 40, 30, 96, 5, 99, 87, 95, 70, 35, 65, 34, 3, 67, 1], [46, 115, 127, 111, 56, 54, 47, 110, 108, 119, 107, 117, 44, 51, 48, 121, 43, 113, 57, 62, 21, 22, 19, 39, 124, 24, 26, 101, 63, 83, 85, 90, 92, 123, 88, 27, 58, 16, 55, 86, 61, 80, 33, 103, 60, 10, 37, 13, 125, 120, 118, 116, 45, 97, 59, 77, 49, 112, 72, 74, 78, 122, 53, 52, 91, 29, 126, 81, 6, 75, 28, 94, 68, 15, 106, 114, 14, 93, 79, 0, 4, 36, 17, 109, 11, 64, 66, 102, 82, 2, 100, 50, 12, 32, 84, 30, 7, 76, 38, 89, 71, 9, 41, 18, 105, 25, 23, 8, 73, 42, 96, 5, 69, 87, 20, 104, 40, 70, 34, 98, 95, 31, 35, 67, 65, 99, 1, 3], [46, 115, 111, 127, 47, 54, 56, 110, 108, 119, 107, 117, 51, 44, 121, 48, 43, 113, 22, 57, 62, 124, 63, 85, 26, 19, 21, 83, 24, 39, 92, 27, 55, 61, 37, 88, 101, 123, 125, 49, 80, 58, 86, 16, 118, 60, 90, 59, 45, 72, 116, 53, 33, 52, 13, 77, 97, 120, 10, 74, 103, 91, 112, 78, 81, 14, 68, 122, 94, 2, 15, 28, 79, 93, 36, 75, 29, 126, 17, 6, 100, 64, 114, 109, 11, 82, 106, 25, 18, 0, 4, 50, 89, 32, 76, 73, 30, 7, 38, 66, 20, 71, 70, 12, 41, 84, 102, 40, 105, 9, 23, 87, 104, 5, 69, 42, 34, 98, 8, 96, 95, 1, 31, 67, 35, 65, 99, 3], [46, 115, 127, 111, 56, 54, 47, 110, 119, 108, 107, 51, 117, 48, 44, 121, 43, 113, 57, 62, 63, 85, 124, 22, 24, 60, 19, 21, 123, 83, 26, 92, 90, 27, 39, 53, 101, 80, 88, 61, 49, 86, 59, 37, 16, 58, 116, 74, 118, 120, 103, 125, 72, 13, 122, 10, 55, 45, 91, 112, 33, 52, 77, 14, 78, 97, 64, 114, 81, 17, 68, 29, 126, 93, 79, 70, 28, 36, 11, 15, 89, 100, 2, 106, 94, 50, 66, 0, 4, 75, 18, 38, 20, 109, 76, 82, 32, 7, 42, 30, 25, 12, 6, 84, 41, 71, 105, 8, 9, 73, 69, 31, 87, 96, 102, 104, 5, 95, 23, 65, 35, 99, 98, 34, 3, 1, 40, 67], [46, 115, 127, 111, 47, 56, 54, 110, 108, 119, 107, 117, 51, 44, 48, 121, 43, 113, 57, 62, 124, 22, 85, 21, 19, 27, 24, 92, 101, 90, 59, 63, 26, 60, 123, 83, 61, 39, 86, 55, 80, 125, 33, 118, 16, 88, 37, 49, 58, 116, 103, 120, 13, 77, 45, 74, 52, 97, 122, 10, 53, 112, 91, 14, 70, 72, 81, 50, 29, 17, 68, 15, 79, 78, 114, 28, 36, 94, 64, 100, 109, 93, 4, 126, 18, 38, 106, 75, 76, 11, 8, 66, 2, 89, 42, 0, 82, 105, 25, 7, 9, 12, 20, 73, 96, 30, 102, 71, 23, 31, 32, 84, 87, 41, 69, 5, 104, 99, 6, 98, 95, 40, 34, 35, 67, 3, 65, 1], [46, 115, 127, 111, 54, 47, 56, 110, 108, 119, 107, 117, 51, 44, 48, 121, 43, 113, 22, 57, 21, 19, 24, 85, 62, 124, 27, 101, 83, 26, 92, 39, 88, 60, 63, 16, 86, 55, 123, 90, 80, 13, 37, 33, 58, 74, 45, 61, 77, 103, 118, 125, 97, 59, 120, 14, 49, 10, 81, 70, 72, 52, 91, 122, 114, 17, 116, 53, 36, 29, 78, 94, 15, 11, 8, 93, 75, 50, 28, 112, 109, 79, 106, 126, 68, 12, 0, 76, 82, 38, 4, 30, 32, 18, 66, 100, 25, 89, 42, 7, 71, 64, 20, 23, 73, 105, 9, 2, 102, 84, 41, 40, 96, 95, 87, 5, 98, 104, 6, 69, 31, 34, 1, 3, 99, 67, 65, 35], [46, 115, 111, 127, 54, 56, 47, 110, 108, 119, 107, 51, 117, 44, 48, 121, 43, 113, 22, 85, 26, 21, 124, 24, 19, 83, 57, 62, 88, 55, 63, 27, 39, 90, 60, 16, 92, 101, 103, 86, 80, 123, 59, 74, 37, 58, 33, 125, 45, 10, 77, 118, 97, 13, 49, 91, 120, 14, 61, 116, 53, 52, 17, 112, 81, 70, 78, 122, 93, 79, 94, 114, 15, 11, 8, 72, 28, 64, 126, 36, 0, 75, 18, 29, 106, 109, 82, 68, 50, 2, 12, 4, 25, 7, 66, 89, 20, 71, 100, 38, 76, 32, 84, 30, 105, 9, 23, 5, 42, 73, 102, 96, 6, 69, 98, 87, 104, 41, 40, 31, 95, 34, 1, 65, 35, 3, 99, 67], [46, 115, 127, 111, 54, 56, 47, 110, 108, 119, 107, 117, 51, 44, 48, 121, 43, 113, 57, 21, 22, 85, 124, 27, 60, 26, 19, 83, 92, 62, 24, 123, 63, 88, 86, 61, 90, 59, 55, 101, 39, 80, 16, 33, 125, 118, 49, 58, 53, 103, 97, 120, 74, 91, 37, 116, 13, 112, 77, 52, 45, 122, 10, 94, 36, 78, 93, 8, 81, 106, 14, 79, 50, 29, 70, 126, 114, 100, 28, 17, 4, 109, 75, 66, 64, 12, 15, 18, 68, 11, 82, 25, 30, 38, 89, 0, 41, 42, 105, 76, 72, 20, 102, 23, 2, 7, 104, 71, 32, 9, 95, 40, 96, 73, 87, 6, 84, 99, 5, 34, 69, 31, 98, 65, 67, 1, 3, 35], [46, 115, 127, 111, 54, 47, 56, 110, 108, 107, 117, 119, 51, 44, 48, 121, 43, 113, 57, 21, 85, 60, 124, 26, 125, 83, 22, 19, 62, 92, 55, 27, 61, 39, 88, 123, 24, 63, 90, 118, 80, 101, 59, 120, 13, 86, 37, 58, 16, 33, 8, 45, 52, 53, 97, 77, 103, 49, 74, 114, 81, 112, 10, 126, 122, 28, 116, 14, 50, 109, 91, 17, 15, 78, 29, 79, 36, 94, 75, 11, 82, 4, 106, 68, 100, 93, 6, 2, 38, 104, 76, 66, 12, 64, 42, 0, 32, 30, 25, 41, 70, 20, 102, 89, 71, 105, 18, 9, 96, 87, 7, 23, 73, 84, 40, 95, 98, 31, 34, 69, 99, 72, 35, 5, 1, 67, 3, 65], [46, 115, 111, 127, 56, 47, 54, 110, 108, 107, 117, 119, 51, 44, 121, 48, 43, 113, 57, 21, 62, 22, 124, 19, 24, 85, 26, 83, 63, 88, 92, 80, 27, 118, 55, 58, 39, 123, 120, 60, 101, 90, 16, 53, 13, 86, 8, 74, 103, 61, 45, 125, 37, 49, 33, 77, 59, 10, 52, 122, 97, 78, 14, 112, 6, 116, 81, 91, 79, 94, 0, 114, 17, 15, 68, 11, 29, 28, 66, 75, 126, 4, 109, 100, 50, 12, 2, 36, 106, 38, 93, 76, 89, 82, 102, 7, 18, 71, 64, 42, 32, 5, 84, 73, 20, 25, 23, 70, 30, 105, 72, 9, 87, 69, 96, 104, 98, 95, 67, 31, 41, 40, 34, 65, 1, 3, 35, 99], [46, 115, 111, 127, 54, 47, 56, 110, 108, 119, 51, 107, 117, 44, 121, 48, 43, 113, 57, 62, 22, 26, 55, 21, 83, 124, 24, 19, 85, 39, 45, 118, 27, 63, 92, 88, 37, 60, 101, 90, 123, 61, 8, 80, 86, 59, 74, 97, 33, 16, 125, 120, 49, 13, 103, 52, 53, 58, 77, 122, 114, 6, 78, 10, 68, 14, 50, 0, 81, 11, 17, 29, 91, 116, 66, 106, 112, 75, 15, 4, 79, 94, 28, 64, 126, 93, 109, 100, 71, 36, 76, 102, 18, 7, 2, 73, 25, 32, 12, 84, 38, 89, 82, 105, 9, 98, 5, 72, 30, 69, 96, 41, 35, 42, 23, 20, 34, 95, 87, 70, 40, 99, 104, 65, 31, 67, 3, 1], [46, 115, 111, 127, 56, 54, 47, 110, 108, 119, 107, 117, 51, 44, 48, 121, 43, 113, 57, 22, 26, 19, 62, 55, 85, 21, 24, 39, 27, 124, 83, 92, 86, 88, 60, 101, 16, 45, 80, 123, 58, 118, 90, 59, 37, 61, 63, 13, 33, 8, 103, 125, 77, 74, 97, 81, 122, 120, 53, 10, 14, 49, 78, 6, 112, 116, 17, 68, 52, 11, 91, 114, 29, 50, 94, 15, 36, 75, 28, 79, 93, 18, 4, 109, 100, 66, 106, 126, 25, 82, 0, 102, 76, 12, 38, 71, 105, 20, 72, 30, 7, 9, 2, 89, 73, 32, 64, 23, 5, 104, 87, 95, 84, 69, 70, 98, 40, 42, 96, 41, 34, 31, 35, 67, 1, 3, 65, 99], [46, 115, 127, 111, 56, 54, 47, 110, 108, 107, 119, 117, 51, 44, 48, 121, 43, 113, 26, 22, 57, 21, 62, 27, 124, 88, 19, 24, 55, 85, 63, 60, 39, 92, 59, 83, 16, 86, 118, 80, 101, 58, 45, 37, 103, 125, 90, 53, 123, 74, 61, 97, 8, 33, 10, 13, 116, 120, 14, 49, 122, 77, 91, 78, 112, 81, 6, 52, 0, 15, 68, 36, 93, 17, 114, 94, 102, 29, 50, 11, 2, 75, 4, 28, 126, 109, 64, 7, 79, 76, 18, 71, 100, 66, 12, 89, 25, 82, 70, 72, 38, 106, 9, 20, 5, 30, 69, 32, 73, 98, 105, 84, 104, 23, 87, 96, 1, 3, 95, 65, 42, 41, 40, 34, 31, 35, 99, 67], [46, 115, 127, 111, 56, 54, 47, 110, 108, 107, 119, 117, 44, 51, 121, 48, 43, 113, 57, 26, 22, 62, 19, 27, 80, 21, 24, 92, 63, 39, 124, 55, 85, 88, 83, 86, 60, 90, 16, 123, 101, 118, 37, 58, 61, 77, 59, 33, 97, 45, 120, 10, 125, 116, 49, 74, 13, 53, 103, 112, 122, 8, 81, 78, 14, 17, 93, 91, 36, 50, 29, 79, 28, 11, 52, 15, 94, 114, 75, 106, 76, 72, 126, 18, 68, 25, 12, 100, 82, 6, 109, 102, 70, 64, 7, 38, 66, 84, 73, 9, 71, 89, 4, 42, 105, 20, 30, 96, 23, 2, 32, 0, 69, 41, 98, 104, 87, 95, 31, 5, 34, 99, 40, 65, 35, 1, 3, 67]], "model.layers.24.self_attn.q_proj": [[112, 37, 48, 54, 90, 41, 94, 86, 30, 101, 56, 25, 59, 81, 62, 19, 20, 106, 58, 44, 63, 121, 123, 89, 105, 114, 12, 84, 9, 22, 95, 35, 74, 113, 46, 77, 52, 97, 104, 70, 2, 16, 24, 107, 117, 124, 60, 39, 18, 43, 119, 31, 88, 14, 125, 79, 122, 26, 10, 126, 109, 49, 4, 108, 68, 67, 80, 50, 45, 92, 8, 47, 120, 111, 29, 57, 0, 32, 115, 98, 51, 61, 118, 53, 99, 27, 71, 28, 116, 42, 110, 23, 103, 93, 102, 36, 127, 33, 91, 38, 40, 55, 100, 3, 21, 87, 17, 15, 78, 34, 85, 75, 1, 5, 96, 82, 13, 83, 65, 7, 69, 73, 76, 11, 66, 64, 72, 6], [112, 37, 48, 54, 122, 90, 111, 94, 30, 86, 125, 25, 101, 117, 41, 81, 24, 58, 19, 89, 88, 22, 35, 121, 59, 84, 45, 57, 126, 114, 109, 53, 20, 63, 106, 42, 105, 77, 12, 31, 119, 102, 120, 55, 104, 46, 10, 82, 60, 124, 3, 113, 14, 50, 18, 123, 70, 62, 100, 97, 49, 56, 47, 8, 52, 110, 44, 61, 74, 21, 26, 16, 107, 118, 87, 36, 9, 43, 103, 29, 40, 116, 65, 115, 95, 64, 93, 4, 27, 79, 32, 34, 51, 108, 28, 23, 17, 92, 91, 85, 127, 15, 96, 33, 2, 38, 39, 99, 80, 78, 98, 72, 73, 5, 83, 71, 11, 6, 75, 13, 7, 66, 68, 67, 0, 69, 76, 1], [112, 37, 48, 89, 25, 90, 54, 86, 94, 30, 56, 24, 18, 19, 35, 41, 22, 45, 105, 43, 20, 59, 50, 101, 116, 127, 97, 42, 81, 117, 125, 63, 16, 77, 60, 123, 88, 49, 95, 85, 38, 47, 9, 121, 106, 122, 109, 12, 114, 84, 115, 119, 100, 67, 0, 108, 55, 40, 26, 31, 110, 58, 70, 87, 51, 103, 104, 36, 102, 27, 92, 124, 111, 79, 2, 62, 46, 61, 13, 107, 91, 113, 53, 23, 126, 57, 74, 32, 34, 71, 39, 52, 33, 120, 29, 14, 82, 93, 99, 98, 44, 83, 96, 21, 15, 118, 69, 66, 8, 28, 78, 17, 10, 80, 7, 1, 75, 68, 76, 3, 11, 4, 72, 73, 6, 65, 64, 5], [112, 37, 48, 54, 90, 41, 94, 84, 81, 86, 117, 56, 62, 88, 30, 14, 20, 105, 19, 111, 24, 25, 51, 125, 101, 109, 49, 114, 10, 120, 63, 45, 58, 47, 31, 8, 60, 70, 126, 59, 43, 64, 77, 55, 121, 22, 52, 92, 102, 42, 119, 18, 57, 118, 53, 66, 3, 124, 4, 122, 113, 127, 65, 97, 106, 108, 89, 35, 123, 44, 107, 50, 16, 34, 39, 78, 103, 74, 110, 61, 26, 46, 32, 98, 104, 99, 38, 116, 40, 28, 12, 69, 17, 72, 100, 80, 21, 115, 93, 87, 1, 95, 33, 83, 29, 36, 85, 73, 96, 67, 5, 79, 91, 11, 2, 27, 9, 23, 71, 68, 75, 15, 0, 82, 6, 13, 76, 7], [108, 36, 44, 123, 51, 90, 26, 114, 85, 92, 96, 52, 121, 120, 18, 82, 31, 77, 28, 111, 94, 24, 15, 125, 56, 53, 95, 127, 60, 126, 20, 62, 107, 63, 113, 122, 23, 76, 46, 73, 110, 55, 32, 124, 39, 105, 102, 101, 45, 84, 87, 58, 50, 43, 74, 30, 37, 118, 41, 109, 116, 16, 6, 8, 97, 29, 27, 49, 21, 54, 33, 61, 115, 38, 9, 48, 7, 57, 112, 106, 40, 98, 14, 103, 117, 91, 89, 119, 47, 22, 99, 34, 25, 59, 104, 100, 35, 3, 42, 86, 78, 64, 88, 93, 4, 19, 80, 2, 83, 81, 11, 12, 17, 65, 10, 79, 67, 71, 5, 13, 69, 75, 66, 0, 68, 70, 1, 72], [108, 36, 44, 123, 51, 63, 127, 52, 90, 28, 96, 56, 114, 120, 92, 23, 105, 95, 119, 118, 46, 58, 31, 85, 87, 111, 30, 102, 106, 112, 57, 54, 124, 40, 103, 122, 55, 42, 77, 26, 107, 125, 60, 113, 99, 104, 116, 110, 15, 43, 109, 39, 32, 94, 61, 41, 100, 45, 62, 47, 38, 49, 11, 121, 82, 18, 115, 117, 126, 48, 53, 17, 37, 50, 10, 20, 59, 101, 93, 84, 8, 35, 97, 98, 33, 76, 34, 27, 6, 91, 88, 24, 16, 9, 89, 29, 83, 25, 19, 80, 73, 5, 21, 86, 22, 74, 3, 79, 81, 13, 7, 68, 12, 72, 75, 78, 14, 69, 66, 70, 64, 1, 4, 71, 65, 67, 2, 0], [108, 36, 44, 51, 52, 92, 96, 56, 85, 121, 90, 126, 26, 28, 31, 60, 114, 23, 120, 45, 117, 87, 18, 82, 15, 105, 49, 94, 77, 124, 102, 24, 112, 32, 48, 88, 109, 123, 55, 104, 110, 46, 103, 93, 41, 27, 73, 76, 100, 107, 125, 115, 34, 116, 61, 29, 63, 118, 113, 122, 43, 50, 53, 84, 119, 89, 111, 57, 127, 95, 101, 16, 22, 38, 47, 42, 39, 33, 106, 54, 25, 62, 4, 2, 40, 8, 37, 59, 98, 21, 97, 58, 64, 6, 3, 74, 99, 86, 30, 91, 9, 35, 80, 20, 65, 83, 78, 19, 7, 69, 71, 79, 81, 14, 5, 12, 17, 70, 10, 0, 13, 11, 75, 68, 72, 67, 66, 1], [108, 36, 123, 52, 90, 94, 44, 92, 28, 121, 124, 111, 63, 62, 93, 59, 109, 86, 105, 82, 87, 23, 116, 114, 50, 60, 26, 20, 100, 49, 96, 42, 91, 46, 31, 127, 119, 17, 103, 51, 78, 19, 41, 84, 57, 110, 15, 27, 53, 98, 48, 102, 37, 122, 117, 112, 58, 55, 33, 101, 99, 30, 104, 47, 45, 40, 25, 56, 120, 43, 61, 35, 54, 11, 106, 38, 118, 39, 113, 115, 97, 126, 107, 125, 22, 29, 18, 34, 8, 14, 85, 89, 80, 95, 88, 32, 21, 73, 77, 76, 24, 9, 83, 6, 16, 10, 81, 13, 79, 3, 7, 74, 12, 71, 75, 64, 2, 70, 72, 4, 5, 69, 65, 68, 66, 1, 67, 0], [45, 52, 109, 32, 89, 85, 18, 123, 12, 29, 87, 10, 9, 79, 69, 81, 96, 83, 71, 91, 101, 58, 4, 30, 117, 0, 68, 2, 82, 8, 19, 1, 16, 105, 59, 84, 35, 108, 37, 7, 64, 23, 6, 50, 54, 77, 76, 78, 127, 42, 25, 124, 97, 33, 17, 46, 102, 24, 31, 120, 114, 21, 28, 72, 15, 20, 94, 80, 73, 26, 88, 3, 126, 62, 104, 92, 70, 74, 110, 75, 66, 14, 103, 13, 122, 61, 44, 116, 27, 95, 67, 36, 60, 112, 39, 40, 86, 100, 22, 90, 34, 5, 119, 38, 113, 43, 98, 49, 93, 125, 65, 115, 56, 107, 11, 118, 48, 51, 53, 111, 99, 121, 63, 106, 57, 41, 47, 55], [45, 52, 109, 87, 123, 91, 79, 85, 69, 89, 12, 10, 32, 18, 96, 101, 4, 71, 1, 29, 9, 0, 81, 66, 83, 92, 30, 2, 36, 35, 56, 82, 65, 122, 42, 120, 117, 60, 8, 105, 22, 70, 58, 39, 110, 97, 64, 104, 88, 127, 106, 7, 84, 72, 53, 90, 14, 61, 95, 63, 74, 3, 126, 5, 21, 77, 16, 48, 54, 73, 111, 80, 27, 23, 68, 24, 76, 67, 94, 119, 17, 25, 13, 50, 33, 75, 62, 26, 107, 15, 116, 51, 44, 20, 49, 93, 31, 19, 46, 11, 6, 102, 98, 37, 108, 114, 103, 78, 28, 41, 38, 59, 121, 86, 125, 34, 118, 113, 99, 40, 57, 112, 100, 55, 124, 115, 43, 47], [45, 52, 109, 91, 32, 123, 85, 89, 87, 29, 18, 12, 9, 79, 46, 81, 112, 10, 4, 114, 69, 71, 83, 126, 96, 50, 54, 94, 61, 58, 47, 119, 108, 60, 82, 127, 92, 25, 63, 72, 22, 70, 62, 77, 30, 84, 107, 59, 110, 101, 35, 105, 64, 15, 57, 106, 16, 13, 111, 40, 49, 27, 93, 68, 116, 117, 42, 21, 1, 56, 55, 41, 39, 2, 115, 17, 33, 20, 23, 118, 24, 80, 0, 75, 19, 37, 113, 120, 103, 99, 95, 36, 26, 7, 122, 88, 124, 31, 34, 98, 90, 125, 28, 104, 100, 51, 8, 121, 102, 44, 73, 76, 14, 97, 5, 6, 38, 11, 3, 86, 43, 74, 53, 48, 78, 67, 65, 66], [45, 52, 109, 89, 32, 87, 91, 29, 85, 123, 79, 96, 18, 12, 10, 71, 83, 69, 50, 114, 9, 60, 35, 119, 49, 92, 84, 81, 106, 4, 77, 101, 95, 57, 107, 93, 63, 16, 1, 30, 36, 46, 82, 108, 100, 61, 117, 94, 39, 22, 112, 120, 118, 23, 58, 127, 53, 26, 21, 113, 110, 116, 43, 40, 44, 59, 28, 105, 25, 103, 88, 99, 42, 122, 55, 31, 121, 111, 124, 24, 126, 13, 54, 98, 62, 125, 37, 56, 27, 38, 115, 97, 15, 80, 51, 19, 20, 48, 102, 86, 68, 14, 90, 6, 74, 34, 33, 75, 78, 47, 17, 64, 8, 41, 76, 72, 104, 7, 73, 11, 5, 0, 70, 65, 67, 2, 3, 66], [104, 127, 98, 92, 87, 84, 81, 109, 31, 15, 17, 22, 54, 41, 28, 58, 82, 51, 37, 76, 124, 14, 21, 63, 122, 116, 95, 121, 89, 30, 20, 43, 60, 38, 57, 90, 53, 13, 56, 25, 73, 18, 94, 85, 102, 29, 24, 11, 88, 126, 111, 117, 125, 70, 27, 46, 55, 107, 23, 48, 62, 12, 42, 59, 78, 67, 61, 4, 40, 45, 93, 110, 80, 105, 118, 106, 32, 9, 114, 83, 79, 19, 35, 97, 99, 120, 96, 36, 47, 6, 39, 101, 33, 68, 86, 52, 75, 10, 50, 49, 77, 100, 119, 74, 113, 123, 115, 44, 103, 26, 5, 108, 91, 16, 72, 69, 8, 112, 34, 71, 7, 66, 3, 1, 65, 64, 0, 2], [104, 127, 92, 98, 84, 87, 14, 81, 109, 31, 73, 76, 37, 6, 63, 125, 122, 82, 57, 105, 75, 89, 124, 18, 41, 4, 107, 94, 99, 78, 25, 17, 97, 116, 95, 83, 61, 10, 64, 58, 28, 30, 68, 59, 108, 66, 43, 54, 20, 65, 60, 51, 47, 115, 72, 15, 100, 106, 3, 50, 46, 53, 45, 56, 49, 85, 48, 23, 13, 16, 22, 114, 96, 24, 9, 62, 77, 12, 74, 35, 26, 80, 70, 119, 19, 36, 33, 38, 91, 79, 90, 40, 117, 44, 55, 29, 111, 121, 101, 39, 88, 102, 103, 2, 21, 34, 93, 8, 123, 69, 120, 126, 7, 32, 113, 112, 5, 27, 11, 42, 110, 52, 118, 71, 0, 86, 1, 67], [104, 127, 98, 101, 22, 43, 92, 18, 105, 53, 31, 59, 125, 84, 124, 87, 63, 61, 89, 54, 81, 41, 39, 122, 57, 106, 50, 95, 45, 19, 46, 28, 99, 35, 121, 109, 71, 79, 118, 38, 26, 62, 25, 76, 58, 37, 116, 44, 8, 96, 15, 56, 0, 30, 102, 27, 111, 117, 14, 115, 13, 120, 126, 107, 108, 60, 94, 93, 49, 73, 74, 51, 55, 110, 52, 11, 85, 123, 113, 103, 82, 119, 32, 4, 97, 42, 75, 36, 77, 112, 100, 90, 48, 33, 47, 86, 114, 68, 65, 6, 5, 29, 24, 34, 10, 80, 88, 23, 91, 83, 40, 7, 67, 17, 20, 72, 69, 2, 21, 1, 16, 70, 66, 3, 64, 9, 12, 78], [104, 127, 98, 73, 87, 31, 66, 14, 92, 81, 124, 84, 76, 4, 6, 116, 37, 25, 64, 89, 2, 63, 71, 13, 5, 60, 41, 80, 65, 75, 19, 0, 18, 106, 61, 114, 22, 79, 67, 46, 10, 1, 105, 53, 47, 58, 57, 125, 8, 122, 95, 40, 85, 54, 82, 101, 12, 59, 102, 51, 77, 74, 117, 121, 62, 70, 3, 111, 11, 90, 55, 24, 69, 72, 29, 109, 50, 23, 49, 15, 9, 100, 16, 86, 21, 91, 42, 33, 56, 88, 35, 99, 26, 7, 119, 123, 107, 17, 112, 43, 68, 36, 32, 38, 44, 118, 48, 94, 103, 96, 27, 126, 113, 52, 97, 108, 83, 28, 115, 34, 110, 39, 120, 78, 45, 30, 93, 20], [61, 102, 121, 114, 58, 116, 119, 62, 115, 54, 50, 33, 111, 126, 60, 125, 59, 122, 120, 124, 123, 117, 112, 110, 63, 52, 38, 24, 56, 53, 49, 45, 113, 42, 57, 29, 127, 55, 48, 26, 51, 118, 46, 107, 15, 27, 108, 47, 109, 91, 88, 105, 41, 85, 21, 84, 106, 44, 43, 86, 93, 37, 90, 18, 101, 28, 103, 100, 40, 104, 39, 22, 30, 97, 36, 32, 83, 17, 79, 34, 98, 35, 82, 96, 12, 19, 73, 95, 99, 31, 92, 81, 9, 94, 23, 80, 77, 64, 7, 20, 76, 25, 71, 66, 78, 89, 67, 87, 0, 2, 1, 65, 3, 5, 4, 69, 16, 13, 68, 11, 6, 14, 75, 10, 8, 72, 74, 70], [121, 102, 114, 61, 116, 58, 119, 62, 115, 54, 33, 60, 126, 59, 122, 111, 50, 123, 117, 120, 124, 110, 125, 52, 112, 53, 57, 24, 29, 63, 56, 45, 38, 49, 48, 26, 127, 55, 42, 113, 46, 118, 51, 88, 15, 47, 109, 108, 27, 91, 105, 41, 107, 106, 21, 85, 43, 84, 44, 86, 93, 28, 37, 90, 18, 103, 100, 101, 104, 39, 30, 97, 22, 40, 83, 17, 36, 79, 32, 12, 99, 95, 35, 34, 96, 98, 73, 19, 20, 31, 81, 87, 92, 82, 94, 80, 76, 23, 9, 25, 71, 67, 7, 77, 64, 66, 0, 3, 65, 1, 78, 89, 4, 2, 69, 5, 68, 13, 16, 11, 14, 75, 6, 74, 10, 72, 8, 70], [114, 121, 102, 61, 58, 116, 119, 115, 62, 122, 60, 124, 126, 50, 59, 123, 54, 33, 111, 112, 125, 120, 110, 117, 24, 52, 63, 56, 53, 42, 106, 57, 127, 48, 49, 107, 55, 26, 118, 113, 38, 15, 46, 51, 45, 29, 47, 108, 27, 105, 84, 109, 91, 85, 21, 44, 41, 88, 93, 28, 43, 86, 37, 90, 18, 103, 40, 104, 32, 101, 36, 100, 39, 30, 17, 22, 82, 79, 97, 98, 35, 9, 34, 96, 12, 99, 83, 92, 31, 95, 73, 81, 94, 7, 19, 76, 67, 69, 20, 77, 25, 64, 1, 0, 78, 71, 66, 65, 3, 4, 2, 80, 23, 5, 89, 68, 87, 14, 13, 11, 16, 6, 74, 75, 8, 72, 70, 10], [61, 121, 114, 102, 23, 80, 83, 50, 58, 74, 77, 30, 25, 45, 11, 8, 33, 37, 29, 90, 91, 78, 32, 92, 20, 70, 81, 119, 100, 69, 116, 88, 86, 62, 75, 68, 72, 120, 89, 59, 42, 105, 26, 21, 14, 125, 18, 6, 87, 16, 10, 66, 28, 54, 24, 76, 19, 27, 127, 85, 96, 126, 103, 56, 107, 1, 111, 106, 93, 63, 64, 51, 12, 67, 22, 13, 48, 84, 34, 4, 82, 17, 60, 31, 47, 110, 43, 115, 122, 98, 124, 52, 101, 7, 112, 117, 94, 35, 123, 57, 79, 108, 49, 0, 104, 41, 95, 2, 46, 71, 55, 5, 9, 36, 118, 3, 113, 99, 44, 53, 109, 15, 38, 39, 40, 65, 73, 97], [53, 42, 58, 120, 122, 100, 125, 116, 61, 45, 63, 123, 127, 48, 50, 52, 121, 49, 30, 54, 113, 89, 126, 51, 23, 108, 56, 59, 32, 44, 112, 111, 115, 46, 60, 106, 114, 62, 18, 47, 117, 57, 55, 21, 124, 119, 110, 109, 43, 118, 90, 91, 107, 40, 86, 92, 35, 15, 25, 104, 103, 28, 94, 24, 85, 41, 105, 12, 31, 37, 27, 38, 39, 83, 99, 101, 96, 36, 22, 102, 82, 34, 80, 98, 88, 93, 81, 95, 87, 78, 33, 1, 20, 76, 79, 97, 17, 5, 14, 29, 11, 26, 7, 77, 74, 16, 67, 68, 65, 0, 3, 72, 84, 19, 73, 64, 2, 70, 4, 71, 66, 69, 10, 9, 8, 13, 75, 6], [53, 42, 58, 120, 83, 80, 100, 89, 32, 11, 77, 73, 86, 27, 106, 45, 122, 70, 28, 24, 1, 92, 99, 26, 85, 81, 17, 67, 25, 74, 79, 72, 14, 5, 103, 20, 75, 111, 116, 23, 40, 113, 76, 125, 63, 123, 57, 48, 84, 119, 96, 61, 66, 6, 37, 93, 78, 82, 19, 50, 8, 91, 127, 34, 88, 15, 30, 126, 43, 94, 31, 9, 87, 69, 10, 49, 22, 105, 29, 4, 21, 0, 16, 95, 112, 71, 13, 39, 90, 56, 33, 54, 52, 124, 121, 101, 18, 51, 104, 118, 59, 109, 108, 7, 68, 114, 62, 12, 3, 35, 97, 117, 60, 115, 46, 44, 55, 98, 38, 2, 65, 102, 107, 64, 47, 110, 41, 36], [58, 42, 53, 120, 122, 125, 116, 45, 100, 61, 30, 123, 63, 48, 121, 50, 127, 49, 111, 54, 52, 89, 56, 59, 51, 112, 126, 23, 106, 108, 32, 46, 113, 115, 57, 114, 44, 62, 47, 60, 117, 55, 119, 21, 124, 110, 18, 109, 24, 118, 90, 94, 43, 107, 104, 91, 40, 25, 41, 105, 103, 92, 31, 35, 85, 86, 39, 37, 99, 12, 101, 38, 28, 27, 15, 96, 102, 36, 22, 82, 95, 93, 34, 98, 17, 81, 88, 97, 79, 33, 83, 76, 20, 80, 14, 87, 29, 26, 7, 16, 78, 2, 74, 8, 19, 64, 65, 72, 66, 77, 68, 0, 5, 71, 1, 67, 3, 84, 10, 4, 11, 69, 6, 70, 13, 75, 9, 73], [120, 42, 58, 53, 122, 125, 45, 61, 116, 123, 106, 50, 89, 52, 23, 49, 100, 48, 59, 56, 63, 112, 111, 44, 121, 54, 127, 117, 126, 57, 62, 110, 113, 51, 115, 30, 55, 60, 46, 109, 118, 114, 47, 90, 119, 124, 18, 108, 107, 28, 43, 21, 105, 24, 104, 91, 32, 83, 95, 29, 40, 103, 38, 39, 35, 101, 34, 41, 99, 102, 94, 86, 37, 97, 85, 92, 27, 31, 33, 25, 15, 98, 96, 77, 36, 26, 12, 82, 93, 20, 13, 19, 22, 75, 17, 10, 14, 87, 7, 11, 76, 84, 74, 79, 88, 16, 80, 71, 67, 6, 68, 66, 72, 1, 8, 2, 5, 81, 70, 3, 64, 0, 4, 69, 65, 78, 9, 73], [51, 118, 55, 39, 63, 54, 121, 25, 23, 122, 124, 120, 126, 59, 53, 61, 113, 123, 62, 58, 117, 95, 50, 56, 57, 89, 52, 125, 60, 82, 49, 116, 115, 119, 34, 47, 48, 114, 127, 106, 111, 112, 42, 46, 44, 109, 104, 110, 45, 108, 29, 87, 80, 14, 27, 43, 107, 84, 36, 76, 35, 37, 99, 41, 96, 40, 102, 103, 20, 101, 100, 105, 28, 97, 38, 31, 18, 12, 85, 91, 26, 24, 16, 74, 32, 93, 92, 98, 78, 86, 30, 71, 69, 33, 94, 8, 21, 88, 7, 2, 22, 1, 68, 90, 83, 10, 70, 72, 13, 81, 3, 73, 6, 19, 11, 0, 5, 9, 79, 65, 4, 64, 66, 75, 67, 17, 77, 15], [55, 39, 118, 122, 25, 23, 120, 121, 63, 89, 95, 115, 124, 51, 123, 113, 126, 61, 58, 60, 34, 82, 59, 112, 48, 53, 119, 106, 52, 57, 62, 117, 111, 42, 49, 50, 56, 47, 114, 125, 44, 127, 116, 43, 54, 80, 14, 29, 108, 45, 110, 46, 109, 84, 76, 41, 104, 107, 40, 27, 87, 20, 31, 105, 18, 102, 100, 16, 28, 96, 78, 37, 36, 101, 38, 103, 12, 93, 35, 8, 97, 32, 24, 94, 99, 83, 88, 74, 33, 86, 10, 71, 92, 85, 72, 7, 98, 90, 70, 73, 69, 68, 13, 30, 6, 21, 3, 2, 5, 26, 9, 22, 0, 65, 66, 19, 1, 4, 67, 64, 81, 11, 91, 79, 17, 77, 75, 15], [51, 55, 39, 120, 121, 118, 54, 122, 63, 23, 89, 25, 59, 123, 46, 113, 62, 115, 124, 60, 61, 126, 44, 112, 125, 58, 50, 116, 33, 53, 52, 49, 82, 95, 57, 34, 119, 29, 108, 56, 45, 110, 114, 111, 117, 87, 47, 42, 127, 48, 43, 76, 14, 109, 100, 84, 68, 71, 31, 36, 99, 98, 3, 106, 26, 72, 11, 80, 38, 107, 37, 40, 88, 32, 18, 102, 105, 27, 0, 83, 104, 20, 41, 28, 96, 97, 1, 12, 101, 81, 30, 92, 93, 90, 94, 78, 103, 5, 35, 2, 64, 74, 24, 69, 65, 70, 9, 4, 85, 19, 21, 8, 91, 13, 16, 6, 86, 7, 75, 67, 10, 73, 79, 66, 17, 22, 77, 15], [51, 55, 120, 39, 63, 121, 27, 124, 126, 59, 53, 61, 123, 116, 122, 57, 58, 60, 62, 56, 45, 118, 119, 54, 25, 117, 125, 114, 113, 50, 47, 49, 127, 111, 34, 115, 108, 95, 52, 48, 46, 84, 107, 112, 36, 86, 110, 82, 43, 20, 109, 42, 76, 89, 44, 29, 40, 23, 14, 92, 41, 106, 24, 94, 80, 102, 101, 105, 91, 93, 22, 79, 99, 31, 103, 28, 17, 81, 90, 35, 104, 38, 15, 74, 98, 100, 12, 69, 85, 7, 37, 8, 88, 32, 13, 16, 9, 96, 21, 70, 71, 97, 26, 18, 33, 87, 68, 1, 77, 83, 6, 30, 2, 11, 78, 73, 10, 19, 0, 67, 72, 5, 75, 66, 4, 65, 3, 64], [38, 115, 114, 50, 51, 113, 89, 75, 19, 82, 16, 23, 77, 7, 73, 78, 69, 10, 39, 87, 25, 6, 94, 67, 8, 85, 57, 104, 13, 74, 125, 52, 27, 3, 26, 80, 118, 21, 14, 68, 65, 5, 106, 92, 124, 12, 105, 11, 123, 30, 64, 79, 98, 122, 37, 95, 81, 43, 44, 63, 62, 56, 41, 18, 93, 70, 102, 54, 101, 2, 112, 121, 107, 83, 42, 76, 100, 59, 34, 48, 111, 49, 45, 40, 119, 117, 33, 28, 126, 60, 84, 47, 108, 120, 36, 96, 109, 103, 4, 61, 88, 86, 29, 32, 17, 99, 127, 35, 58, 72, 46, 9, 116, 97, 31, 55, 22, 15, 90, 110, 53, 91, 71, 24, 20, 1, 66, 0], [38, 114, 115, 50, 51, 89, 23, 82, 19, 77, 16, 69, 75, 7, 73, 113, 64, 2, 85, 14, 93, 17, 67, 66, 78, 1, 21, 72, 3, 87, 79, 12, 94, 8, 13, 32, 71, 126, 106, 25, 15, 68, 95, 102, 10, 22, 41, 57, 74, 6, 127, 24, 44, 34, 112, 107, 56, 39, 76, 27, 92, 26, 90, 98, 0, 54, 18, 101, 80, 65, 40, 121, 105, 49, 81, 30, 122, 84, 47, 9, 104, 108, 83, 109, 117, 4, 125, 11, 20, 100, 37, 36, 48, 42, 116, 43, 86, 103, 97, 59, 63, 120, 28, 29, 61, 46, 45, 111, 99, 55, 110, 119, 53, 35, 70, 58, 60, 91, 33, 124, 118, 31, 88, 96, 62, 52, 123, 5], [38, 114, 115, 50, 51, 89, 7, 19, 73, 16, 75, 113, 82, 2, 77, 64, 69, 23, 3, 67, 14, 25, 85, 87, 93, 6, 13, 1, 17, 48, 66, 107, 8, 125, 9, 124, 41, 56, 71, 104, 0, 12, 79, 20, 52, 122, 45, 126, 5, 94, 81, 18, 22, 65, 74, 95, 127, 91, 57, 49, 59, 83, 15, 11, 80, 46, 24, 60, 92, 39, 118, 21, 86, 90, 105, 68, 10, 117, 121, 4, 44, 110, 29, 78, 72, 62, 106, 111, 88, 32, 108, 96, 58, 27, 40, 31, 123, 34, 109, 116, 120, 76, 119, 97, 101, 43, 63, 37, 100, 112, 98, 36, 28, 30, 99, 33, 61, 54, 103, 53, 26, 42, 55, 47, 84, 35, 102, 70], [38, 50, 114, 115, 51, 113, 23, 16, 82, 89, 39, 19, 77, 85, 126, 75, 73, 78, 7, 30, 24, 37, 22, 57, 116, 14, 52, 12, 35, 92, 34, 27, 125, 112, 36, 79, 83, 69, 13, 81, 26, 107, 49, 56, 88, 63, 80, 104, 17, 106, 67, 21, 94, 25, 20, 120, 44, 90, 46, 87, 60, 121, 45, 117, 42, 40, 9, 108, 47, 41, 48, 8, 109, 93, 76, 6, 122, 32, 72, 84, 97, 102, 53, 62, 10, 59, 119, 61, 99, 103, 98, 2, 118, 31, 111, 11, 54, 105, 74, 28, 18, 86, 127, 124, 33, 110, 15, 95, 29, 123, 43, 100, 96, 101, 58, 55, 91, 68, 3, 70, 5, 64, 71, 65, 1, 4, 66, 0]], "model.layers.24.self_attn.k_proj": [[48, 112, 101, 30, 86, 90, 19, 88, 89, 25, 81, 84, 12, 70, 14, 107, 77, 63, 8, 54, 47, 120, 60, 17, 52, 125, 41, 0, 49, 117, 115, 119, 93, 114, 2, 124, 74, 10, 33, 79, 64, 113, 43, 46, 102, 1, 57, 62, 80, 59, 42, 18, 20, 100, 118, 110, 35, 9, 109, 56, 29, 123, 44, 55, 58, 51, 68, 106, 116, 45, 28, 38, 103, 122, 108, 61, 50, 111, 4, 3, 40, 104, 87, 53, 127, 126, 39, 121, 67, 98, 99, 16, 97, 96, 15, 23, 32, 36, 31, 27, 85, 91, 5, 105, 92, 83, 34, 71, 94, 95, 73, 24, 82, 75, 21, 78, 7, 26, 11, 13, 72, 69, 76, 65, 22, 6, 66, 37], [44, 108, 52, 121, 82, 77, 15, 85, 100, 26, 32, 87, 60, 123, 6, 8, 76, 73, 55, 28, 3, 51, 74, 127, 120, 24, 48, 92, 61, 0, 9, 64, 65, 29, 124, 4, 109, 2, 89, 16, 5, 111, 94, 20, 110, 63, 18, 114, 105, 116, 78, 56, 35, 118, 7, 83, 47, 126, 59, 22, 88, 91, 50, 95, 62, 33, 122, 46, 84, 119, 115, 25, 31, 80, 54, 23, 27, 113, 86, 97, 98, 45, 81, 99, 103, 39, 102, 69, 40, 12, 70, 101, 107, 37, 57, 43, 117, 34, 112, 49, 11, 93, 53, 41, 38, 36, 42, 106, 96, 125, 104, 58, 19, 90, 30, 21, 14, 17, 71, 10, 13, 75, 79, 68, 72, 66, 1, 67], [109, 52, 45, 0, 123, 85, 32, 12, 69, 10, 87, 79, 116, 89, 18, 9, 4, 91, 81, 68, 2, 29, 83, 60, 71, 84, 8, 77, 78, 3, 1, 70, 16, 119, 120, 7, 93, 126, 58, 117, 94, 41, 30, 64, 66, 115, 124, 72, 118, 112, 114, 113, 61, 46, 54, 47, 44, 37, 92, 49, 111, 127, 65, 50, 53, 125, 95, 105, 106, 51, 110, 63, 35, 40, 43, 107, 48, 56, 102, 55, 39, 34, 19, 108, 104, 22, 121, 57, 97, 6, 59, 42, 62, 75, 103, 36, 100, 122, 82, 38, 67, 25, 28, 101, 99, 98, 13, 33, 27, 14, 20, 11, 88, 31, 21, 23, 90, 96, 86, 26, 73, 15, 80, 24, 5, 76, 17, 74], [127, 40, 34, 84, 87, 76, 95, 81, 73, 14, 17, 89, 6, 5, 28, 66, 125, 124, 64, 4, 116, 60, 122, 110, 0, 111, 13, 79, 58, 74, 19, 75, 92, 117, 51, 18, 80, 50, 24, 57, 105, 43, 63, 22, 45, 41, 101, 107, 26, 106, 54, 16, 121, 70, 10, 86, 71, 53, 20, 39, 78, 59, 8, 65, 52, 56, 30, 37, 62, 49, 94, 3, 42, 25, 123, 114, 61, 27, 83, 90, 7, 85, 96, 15, 109, 119, 103, 67, 21, 35, 44, 118, 33, 47, 99, 113, 55, 77, 108, 120, 126, 32, 1, 102, 69, 112, 115, 93, 68, 31, 98, 38, 36, 88, 46, 29, 48, 97, 82, 91, 12, 72, 100, 23, 9, 11, 2, 104], [38, 61, 114, 121, 86, 97, 119, 93, 18, 54, 30, 27, 50, 116, 15, 59, 41, 62, 56, 120, 126, 58, 115, 125, 124, 122, 55, 63, 90, 60, 127, 57, 49, 117, 48, 85, 123, 47, 112, 113, 118, 108, 52, 110, 101, 24, 51, 45, 36, 109, 46, 96, 53, 43, 94, 29, 106, 111, 95, 17, 104, 44, 25, 42, 107, 39, 99, 91, 105, 79, 33, 98, 12, 20, 28, 81, 80, 103, 92, 13, 40, 73, 77, 88, 100, 83, 7, 14, 16, 34, 10, 21, 31, 37, 84, 102, 23, 35, 74, 19, 26, 32, 78, 11, 89, 87, 8, 82, 70, 22, 75, 6, 68, 3, 9, 1, 76, 72, 2, 69, 66, 65, 5, 71, 0, 4, 67, 64], [106, 36, 120, 86, 53, 58, 96, 116, 94, 28, 49, 61, 89, 63, 27, 121, 123, 42, 122, 50, 45, 125, 48, 56, 62, 111, 54, 124, 59, 52, 18, 117, 55, 80, 127, 57, 119, 115, 112, 35, 46, 114, 51, 83, 60, 108, 34, 126, 107, 118, 47, 113, 109, 21, 98, 23, 12, 41, 43, 73, 15, 81, 110, 9, 105, 11, 104, 77, 29, 44, 40, 102, 14, 16, 39, 38, 7, 37, 103, 101, 17, 90, 0, 95, 87, 97, 33, 92, 24, 91, 20, 31, 67, 26, 100, 13, 99, 32, 71, 93, 72, 5, 88, 30, 68, 84, 19, 66, 78, 70, 74, 82, 85, 25, 2, 65, 64, 76, 69, 10, 75, 6, 79, 8, 22, 3, 4, 1], [103, 51, 55, 86, 120, 98, 123, 63, 93, 126, 124, 61, 31, 114, 53, 45, 59, 58, 49, 57, 121, 89, 122, 52, 125, 62, 112, 56, 47, 117, 116, 50, 60, 46, 119, 127, 115, 48, 110, 113, 108, 111, 107, 38, 109, 40, 42, 105, 44, 54, 43, 36, 80, 41, 106, 22, 18, 33, 118, 34, 79, 82, 16, 104, 99, 88, 90, 81, 84, 26, 96, 97, 102, 92, 91, 11, 100, 13, 37, 27, 21, 101, 32, 35, 83, 12, 23, 25, 29, 94, 72, 78, 85, 20, 10, 6, 30, 28, 17, 39, 24, 14, 66, 87, 19, 95, 9, 5, 73, 69, 4, 15, 8, 77, 75, 67, 76, 65, 71, 68, 0, 74, 7, 3, 1, 70, 2, 64], [50, 115, 102, 77, 89, 16, 19, 82, 75, 73, 114, 7, 23, 64, 69, 113, 2, 3, 67, 66, 1, 85, 10, 93, 17, 8, 70, 14, 65, 78, 21, 57, 30, 38, 112, 49, 94, 42, 43, 126, 34, 76, 71, 39, 45, 108, 92, 25, 90, 48, 6, 32, 44, 117, 123, 122, 13, 121, 87, 56, 106, 103, 125, 22, 79, 28, 74, 20, 27, 120, 63, 124, 116, 68, 33, 127, 61, 54, 15, 60, 104, 4, 41, 52, 119, 105, 100, 95, 12, 91, 72, 9, 88, 26, 53, 36, 31, 101, 29, 98, 62, 35, 118, 47, 59, 84, 46, 109, 51, 40, 96, 99, 0, 110, 55, 107, 111, 24, 37, 83, 58, 80, 97, 11, 5, 81, 86, 18]], "model.layers.24.self_attn.qk_proj": [[127, 50, 48, 115, 112, 114, 109, 51, 45, 108, 55, 61, 121, 44, 53, 120, 58, 52, 38, 25, 23, 124, 123, 63, 87, 18, 42, 89, 92, 60, 22, 113, 106, 40, 125, 82, 126, 116, 29, 122, 83, 19, 59, 17, 77, 32, 101, 54, 20, 26, 13, 94, 84, 73, 16, 9, 30, 81, 12, 56, 102, 85, 80, 90, 21, 57, 117, 49, 37, 76, 46, 86, 79, 111, 75, 98, 118, 110, 36, 7, 62, 15, 78, 14, 41, 11, 47, 119, 71, 27, 91, 103, 64, 43, 95, 34, 39, 5, 69, 74, 10, 105, 0, 96, 24, 31, 100, 104, 6, 88, 93, 28, 3, 68, 107, 67, 2, 66, 33, 70, 8, 4, 97, 99, 72, 35, 1, 65], [127, 50, 115, 114, 48, 109, 112, 51, 55, 108, 61, 45, 121, 44, 53, 120, 58, 52, 38, 25, 123, 92, 23, 87, 60, 89, 113, 18, 42, 124, 22, 106, 116, 40, 122, 63, 54, 125, 82, 29, 126, 73, 59, 83, 77, 101, 56, 32, 13, 20, 85, 19, 94, 16, 17, 26, 30, 81, 102, 9, 12, 49, 117, 86, 76, 84, 79, 21, 119, 111, 118, 57, 71, 36, 98, 80, 110, 41, 27, 103, 39, 75, 37, 90, 62, 47, 104, 78, 14, 46, 91, 11, 15, 43, 74, 34, 64, 105, 5, 69, 24, 100, 31, 95, 107, 7, 0, 96, 4, 93, 10, 28, 6, 88, 2, 68, 8, 66, 3, 67, 33, 97, 70, 72, 99, 35, 65, 1], [127, 50, 48, 115, 114, 112, 109, 51, 45, 108, 55, 61, 121, 120, 44, 53, 58, 52, 25, 38, 123, 113, 23, 87, 106, 92, 42, 18, 89, 60, 40, 22, 126, 125, 122, 82, 63, 32, 56, 116, 124, 54, 29, 59, 17, 26, 101, 13, 19, 73, 30, 102, 49, 83, 85, 16, 119, 20, 77, 94, 81, 76, 86, 12, 90, 9, 117, 21, 47, 57, 111, 118, 98, 110, 84, 104, 41, 80, 103, 78, 34, 79, 36, 31, 39, 7, 37, 62, 71, 75, 64, 15, 11, 14, 27, 5, 105, 69, 96, 91, 46, 95, 74, 100, 43, 10, 107, 0, 28, 93, 24, 66, 6, 33, 67, 2, 88, 3, 8, 4, 68, 97, 70, 99, 35, 65, 1, 72], [127, 50, 48, 115, 114, 112, 109, 51, 61, 45, 108, 55, 121, 44, 53, 120, 58, 52, 123, 38, 25, 106, 113, 23, 42, 87, 89, 18, 126, 82, 92, 59, 22, 124, 63, 125, 56, 116, 119, 29, 60, 94, 40, 76, 13, 122, 17, 32, 30, 73, 86, 19, 77, 80, 9, 101, 85, 16, 54, 49, 83, 81, 26, 79, 102, 12, 84, 111, 20, 98, 118, 110, 75, 47, 117, 5, 15, 41, 21, 104, 62, 71, 39, 36, 37, 69, 57, 14, 90, 7, 27, 34, 78, 46, 31, 0, 43, 11, 74, 103, 10, 95, 100, 96, 2, 28, 24, 88, 66, 67, 8, 70, 64, 93, 91, 4, 105, 6, 3, 107, 33, 35, 68, 97, 99, 65, 1, 72], [127, 48, 50, 115, 114, 112, 109, 51, 108, 61, 55, 45, 44, 121, 120, 53, 58, 52, 25, 123, 38, 23, 42, 113, 60, 63, 82, 18, 89, 87, 106, 59, 92, 56, 126, 124, 125, 22, 122, 40, 54, 13, 17, 73, 19, 86, 49, 29, 116, 9, 94, 80, 77, 30, 57, 32, 84, 83, 81, 26, 20, 76, 12, 101, 16, 75, 85, 90, 110, 118, 119, 14, 79, 111, 102, 21, 117, 37, 15, 104, 62, 98, 36, 5, 103, 47, 41, 34, 71, 78, 27, 95, 7, 11, 10, 46, 69, 91, 64, 70, 74, 96, 105, 31, 100, 39, 0, 24, 43, 107, 33, 28, 88, 3, 66, 8, 2, 67, 4, 93, 97, 68, 6, 35, 1, 99, 72, 65], [127, 50, 48, 115, 114, 112, 109, 61, 51, 108, 55, 45, 44, 121, 53, 120, 58, 52, 25, 38, 42, 23, 22, 123, 89, 113, 87, 18, 82, 106, 92, 60, 126, 40, 124, 77, 84, 122, 73, 63, 59, 76, 83, 81, 17, 94, 80, 19, 116, 125, 86, 29, 12, 13, 101, 85, 54, 30, 16, 32, 9, 26, 102, 37, 79, 56, 15, 98, 21, 41, 110, 57, 90, 36, 34, 118, 49, 20, 47, 14, 103, 5, 11, 95, 75, 27, 111, 78, 71, 46, 70, 10, 7, 104, 0, 117, 64, 43, 105, 62, 119, 39, 91, 8, 74, 24, 69, 100, 96, 93, 107, 88, 31, 2, 33, 66, 97, 28, 3, 67, 68, 6, 35, 99, 4, 65, 1, 72], [127, 50, 48, 115, 114, 112, 109, 55, 108, 51, 45, 61, 44, 121, 53, 120, 58, 52, 25, 23, 42, 87, 38, 82, 89, 123, 92, 18, 106, 124, 22, 60, 113, 125, 122, 40, 63, 19, 17, 54, 126, 32, 94, 84, 86, 26, 73, 30, 80, 13, 77, 29, 83, 101, 81, 56, 116, 12, 16, 59, 76, 85, 20, 9, 57, 21, 119, 98, 79, 102, 15, 90, 49, 111, 78, 41, 75, 118, 47, 104, 11, 110, 27, 14, 46, 71, 117, 5, 36, 37, 91, 7, 103, 95, 34, 64, 10, 62, 105, 100, 39, 74, 0, 24, 43, 31, 69, 70, 107, 96, 88, 8, 28, 93, 33, 35, 2, 4, 3, 97, 68, 67, 99, 66, 6, 72, 1, 65], [127, 50, 48, 115, 114, 109, 112, 55, 51, 45, 108, 121, 44, 61, 120, 53, 58, 52, 25, 23, 123, 38, 124, 89, 92, 18, 87, 42, 60, 82, 22, 113, 63, 125, 106, 40, 122, 116, 83, 59, 29, 126, 119, 32, 30, 56, 16, 94, 54, 73, 86, 26, 101, 20, 80, 19, 49, 13, 84, 17, 21, 90, 81, 102, 12, 9, 85, 77, 57, 76, 37, 15, 111, 41, 79, 78, 62, 11, 46, 71, 36, 110, 39, 117, 118, 34, 91, 27, 5, 105, 104, 43, 28, 75, 98, 31, 0, 47, 24, 103, 64, 10, 14, 95, 7, 69, 100, 70, 88, 74, 93, 96, 68, 107, 66, 2, 35, 3, 33, 4, 8, 97, 67, 72, 6, 99, 1, 65], [127, 50, 48, 115, 112, 109, 114, 55, 45, 51, 108, 121, 61, 44, 53, 120, 58, 52, 25, 38, 23, 92, 89, 123, 124, 87, 42, 125, 113, 63, 82, 106, 60, 59, 18, 126, 116, 122, 83, 22, 32, 77, 40, 56, 101, 80, 94, 81, 26, 84, 29, 54, 86, 30, 19, 117, 13, 119, 73, 102, 20, 85, 17, 49, 90, 76, 16, 9, 21, 111, 12, 37, 98, 118, 43, 15, 41, 47, 79, 57, 14, 34, 91, 110, 103, 27, 96, 75, 105, 11, 71, 36, 46, 69, 39, 100, 62, 104, 31, 78, 28, 7, 5, 95, 24, 74, 107, 10, 88, 93, 64, 0, 70, 33, 2, 4, 35, 68, 66, 6, 97, 3, 72, 99, 67, 8, 1, 65], [127, 50, 48, 114, 115, 109, 112, 45, 51, 108, 55, 121, 61, 44, 53, 120, 58, 52, 25, 38, 123, 124, 23, 87, 89, 42, 125, 113, 63, 106, 92, 82, 59, 18, 60, 116, 40, 126, 22, 29, 122, 30, 101, 83, 9, 73, 13, 94, 32, 54, 17, 119, 19, 77, 76, 56, 86, 84, 26, 85, 117, 16, 81, 80, 102, 57, 36, 111, 49, 12, 21, 37, 41, 118, 79, 103, 69, 20, 14, 110, 7, 11, 34, 90, 75, 104, 71, 98, 15, 5, 27, 47, 62, 78, 91, 95, 0, 10, 105, 6, 64, 31, 96, 24, 43, 39, 74, 28, 46, 100, 4, 66, 3, 88, 72, 2, 68, 93, 33, 70, 107, 67, 35, 97, 99, 8, 1, 65], [127, 50, 48, 115, 109, 112, 114, 51, 108, 45, 44, 55, 61, 121, 53, 120, 58, 52, 25, 42, 38, 123, 124, 87, 23, 106, 92, 113, 18, 82, 89, 22, 63, 122, 126, 125, 40, 73, 13, 29, 60, 116, 83, 9, 59, 56, 54, 77, 80, 86, 16, 84, 85, 19, 32, 81, 94, 17, 30, 26, 12, 76, 119, 102, 101, 20, 118, 36, 15, 62, 79, 71, 11, 117, 5, 49, 111, 90, 0, 75, 21, 69, 37, 96, 34, 27, 98, 104, 103, 7, 6, 57, 14, 39, 95, 78, 10, 64, 43, 74, 110, 46, 105, 47, 91, 2, 31, 41, 88, 24, 72, 67, 100, 28, 66, 3, 68, 93, 107, 4, 70, 97, 33, 8, 65, 35, 99, 1], [127, 50, 48, 115, 114, 112, 109, 51, 45, 108, 61, 44, 55, 121, 53, 120, 58, 52, 25, 42, 23, 87, 92, 38, 123, 22, 18, 106, 124, 89, 113, 82, 63, 40, 77, 13, 76, 125, 19, 126, 86, 30, 80, 73, 29, 17, 60, 85, 20, 84, 83, 9, 54, 94, 116, 81, 122, 101, 32, 16, 49, 26, 12, 56, 118, 90, 98, 15, 21, 57, 59, 111, 79, 11, 37, 102, 36, 7, 34, 95, 75, 71, 0, 117, 104, 27, 62, 47, 14, 100, 10, 91, 6, 119, 39, 96, 41, 78, 5, 69, 103, 74, 43, 24, 105, 46, 64, 88, 107, 28, 93, 31, 110, 72, 33, 97, 67, 66, 68, 2, 4, 3, 99, 70, 35, 8, 1, 65], [127, 50, 48, 115, 112, 109, 114, 51, 55, 45, 108, 61, 44, 121, 53, 120, 58, 52, 25, 38, 92, 124, 123, 23, 89, 42, 22, 87, 125, 113, 82, 63, 126, 18, 106, 40, 60, 32, 116, 94, 101, 30, 59, 54, 122, 29, 84, 26, 85, 77, 17, 13, 9, 86, 49, 117, 83, 56, 81, 19, 57, 20, 80, 73, 16, 111, 21, 119, 90, 37, 76, 12, 98, 41, 79, 36, 103, 118, 91, 15, 34, 27, 95, 11, 105, 100, 102, 24, 110, 75, 14, 71, 39, 7, 62, 47, 78, 46, 0, 5, 104, 28, 69, 31, 74, 10, 88, 43, 96, 72, 6, 68, 64, 93, 33, 97, 107, 2, 99, 66, 67, 4, 35, 70, 3, 65, 1, 8], [127, 50, 48, 114, 115, 112, 109, 51, 108, 45, 55, 61, 44, 120, 121, 53, 58, 52, 25, 124, 42, 38, 123, 23, 126, 89, 113, 106, 87, 22, 125, 59, 92, 40, 63, 18, 82, 122, 32, 56, 102, 77, 60, 101, 73, 116, 83, 84, 9, 119, 13, 30, 94, 54, 19, 81, 118, 16, 17, 86, 49, 26, 57, 111, 85, 29, 80, 117, 12, 37, 21, 76, 79, 90, 36, 20, 7, 41, 11, 98, 62, 27, 47, 15, 78, 103, 39, 69, 34, 75, 105, 71, 43, 0, 100, 64, 104, 10, 88, 74, 31, 14, 5, 95, 110, 96, 24, 28, 72, 6, 66, 91, 46, 33, 107, 93, 4, 2, 3, 67, 70, 97, 35, 68, 99, 65, 8, 1], [127, 114, 48, 50, 115, 109, 112, 51, 108, 45, 44, 55, 61, 121, 120, 53, 58, 52, 42, 25, 38, 23, 106, 89, 123, 22, 124, 113, 87, 63, 82, 92, 40, 18, 125, 126, 116, 59, 29, 77, 32, 9, 81, 60, 73, 83, 122, 16, 19, 30, 94, 54, 84, 13, 85, 26, 12, 101, 111, 80, 17, 20, 118, 49, 21, 76, 36, 119, 102, 56, 57, 37, 11, 98, 86, 79, 7, 15, 117, 41, 14, 62, 5, 103, 43, 90, 34, 64, 78, 110, 69, 39, 95, 10, 71, 24, 47, 31, 105, 91, 100, 27, 75, 0, 104, 74, 46, 96, 72, 33, 70, 66, 28, 97, 107, 88, 2, 67, 6, 68, 3, 4, 93, 35, 99, 8, 65, 1], [127, 50, 48, 115, 114, 109, 112, 51, 45, 55, 108, 61, 121, 44, 120, 53, 58, 52, 38, 25, 42, 23, 123, 92, 82, 87, 106, 89, 124, 22, 63, 113, 18, 59, 125, 126, 77, 122, 116, 84, 40, 73, 83, 54, 13, 80, 81, 94, 30, 9, 16, 19, 32, 60, 12, 29, 86, 17, 76, 56, 20, 26, 85, 21, 111, 37, 101, 15, 118, 102, 90, 119, 62, 27, 57, 98, 110, 11, 71, 7, 75, 79, 39, 36, 10, 43, 14, 5, 46, 34, 31, 49, 117, 95, 78, 24, 41, 0, 69, 103, 104, 74, 105, 70, 47, 96, 91, 88, 107, 68, 100, 64, 72, 28, 66, 2, 67, 93, 97, 33, 6, 99, 4, 3, 8, 35, 65, 1], [127, 50, 48, 115, 114, 112, 109, 51, 55, 45, 61, 108, 121, 44, 120, 53, 52, 58, 25, 38, 123, 42, 92, 23, 87, 124, 106, 89, 126, 18, 82, 22, 63, 125, 40, 59, 113, 30, 32, 116, 19, 29, 86, 101, 77, 60, 84, 26, 94, 81, 122, 16, 13, 56, 20, 17, 21, 54, 73, 9, 83, 111, 57, 76, 85, 102, 27, 110, 80, 118, 12, 90, 79, 7, 98, 103, 47, 41, 117, 36, 75, 119, 49, 24, 37, 95, 91, 15, 11, 62, 14, 104, 31, 10, 34, 28, 43, 105, 46, 96, 5, 78, 0, 39, 71, 107, 74, 64, 100, 88, 93, 70, 69, 68, 66, 4, 33, 97, 8, 67, 2, 72, 35, 99, 3, 6, 65, 1], [127, 50, 48, 115, 114, 109, 112, 55, 45, 51, 108, 61, 120, 121, 44, 53, 58, 52, 25, 38, 92, 123, 89, 87, 23, 126, 42, 106, 125, 113, 124, 82, 18, 116, 63, 22, 59, 86, 29, 40, 54, 32, 17, 19, 77, 94, 60, 101, 83, 26, 56, 16, 122, 9, 73, 13, 30, 57, 102, 119, 84, 81, 80, 76, 20, 85, 49, 12, 90, 39, 98, 111, 118, 21, 36, 37, 79, 7, 15, 41, 5, 27, 78, 47, 11, 69, 110, 91, 75, 64, 31, 104, 96, 103, 71, 14, 24, 117, 34, 95, 28, 105, 0, 62, 10, 43, 74, 70, 46, 100, 88, 2, 93, 107, 68, 4, 33, 8, 67, 66, 3, 35, 97, 6, 65, 72, 99, 1], [127, 50, 48, 114, 115, 109, 112, 51, 45, 108, 61, 55, 44, 121, 53, 120, 58, 52, 38, 123, 124, 89, 42, 25, 23, 113, 126, 125, 106, 18, 92, 63, 87, 59, 82, 116, 22, 40, 77, 73, 29, 32, 30, 94, 101, 9, 76, 60, 84, 119, 81, 19, 54, 17, 86, 16, 26, 13, 21, 122, 83, 103, 57, 56, 111, 85, 49, 98, 37, 41, 36, 80, 117, 20, 11, 12, 102, 75, 27, 47, 118, 79, 110, 5, 90, 7, 69, 62, 34, 64, 104, 39, 105, 31, 15, 78, 43, 46, 74, 0, 14, 24, 71, 91, 95, 107, 8, 100, 88, 66, 96, 28, 70, 68, 10, 4, 67, 93, 3, 33, 2, 97, 6, 35, 99, 65, 72, 1], [127, 48, 50, 114, 115, 112, 109, 108, 51, 55, 45, 61, 44, 121, 53, 120, 58, 52, 38, 25, 123, 113, 124, 42, 23, 89, 18, 106, 125, 126, 22, 92, 87, 82, 40, 59, 63, 116, 29, 60, 32, 77, 30, 81, 73, 13, 9, 101, 83, 94, 122, 86, 76, 80, 84, 17, 26, 85, 37, 110, 57, 98, 111, 12, 21, 56, 19, 54, 36, 20, 102, 16, 104, 119, 117, 75, 15, 7, 41, 24, 103, 10, 27, 79, 34, 69, 11, 91, 118, 105, 64, 5, 49, 78, 62, 39, 95, 71, 90, 43, 88, 74, 8, 46, 107, 0, 31, 96, 28, 47, 14, 100, 93, 97, 70, 2, 6, 68, 4, 33, 66, 67, 3, 99, 35, 65, 72, 1], [127, 48, 50, 114, 112, 115, 109, 51, 108, 55, 45, 120, 44, 61, 121, 53, 58, 52, 25, 124, 123, 23, 38, 92, 42, 113, 125, 126, 106, 89, 116, 59, 22, 87, 60, 18, 82, 40, 122, 32, 63, 13, 77, 83, 80, 73, 86, 26, 29, 19, 101, 119, 94, 17, 9, 81, 12, 56, 84, 57, 76, 85, 16, 30, 102, 104, 54, 20, 21, 110, 37, 49, 90, 111, 98, 117, 75, 15, 27, 11, 79, 69, 118, 41, 34, 5, 31, 36, 46, 7, 14, 71, 96, 47, 24, 10, 91, 39, 78, 95, 74, 62, 6, 43, 100, 103, 107, 0, 28, 105, 2, 93, 64, 8, 88, 4, 66, 68, 70, 67, 33, 3, 97, 35, 99, 65, 1, 72], [127, 50, 48, 114, 115, 112, 109, 45, 55, 51, 108, 120, 121, 61, 44, 53, 58, 52, 38, 25, 123, 113, 125, 59, 126, 89, 42, 23, 106, 92, 124, 18, 116, 63, 82, 87, 22, 32, 40, 122, 54, 60, 56, 101, 119, 94, 30, 29, 77, 9, 86, 81, 73, 57, 13, 12, 17, 83, 110, 76, 26, 85, 20, 80, 21, 16, 19, 102, 84, 111, 71, 36, 46, 79, 11, 117, 90, 5, 34, 7, 75, 103, 69, 15, 98, 37, 62, 104, 0, 27, 31, 41, 118, 49, 39, 47, 78, 24, 6, 91, 96, 14, 64, 10, 4, 95, 88, 100, 74, 43, 67, 107, 93, 66, 68, 105, 2, 8, 28, 3, 33, 70, 97, 99, 65, 35, 1, 72], [127, 48, 50, 115, 112, 114, 109, 55, 51, 45, 108, 121, 44, 61, 120, 53, 58, 52, 38, 25, 113, 125, 23, 124, 89, 123, 92, 42, 59, 18, 87, 106, 126, 82, 63, 40, 122, 22, 30, 60, 77, 54, 116, 29, 86, 32, 73, 20, 56, 94, 57, 83, 17, 26, 101, 19, 9, 13, 111, 85, 81, 102, 119, 80, 12, 16, 37, 21, 84, 62, 98, 110, 76, 117, 15, 5, 75, 118, 47, 90, 104, 79, 91, 14, 49, 7, 36, 103, 11, 71, 34, 27, 46, 69, 24, 6, 100, 41, 95, 31, 39, 28, 0, 78, 88, 43, 74, 107, 93, 68, 10, 96, 105, 4, 8, 2, 66, 64, 97, 67, 33, 3, 70, 99, 35, 72, 65, 1], [127, 48, 50, 115, 114, 112, 109, 51, 55, 108, 61, 45, 121, 44, 53, 120, 58, 52, 25, 38, 42, 124, 23, 89, 123, 87, 18, 106, 113, 92, 63, 22, 82, 125, 122, 60, 126, 40, 77, 54, 59, 9, 29, 86, 73, 116, 32, 17, 101, 13, 94, 83, 30, 12, 19, 56, 26, 84, 20, 80, 16, 76, 57, 21, 85, 102, 119, 81, 98, 37, 104, 36, 27, 90, 7, 15, 111, 79, 75, 110, 103, 41, 78, 10, 34, 69, 62, 47, 11, 118, 95, 74, 5, 71, 31, 14, 91, 46, 28, 117, 88, 43, 49, 0, 96, 39, 100, 107, 6, 64, 8, 24, 105, 2, 67, 66, 33, 70, 68, 4, 97, 3, 93, 99, 72, 35, 65, 1], [127, 50, 48, 114, 115, 112, 109, 51, 45, 108, 55, 61, 44, 121, 53, 120, 58, 52, 25, 38, 123, 23, 22, 124, 82, 42, 87, 18, 106, 113, 89, 63, 92, 126, 122, 125, 84, 40, 60, 13, 54, 116, 83, 12, 32, 73, 59, 101, 94, 29, 86, 17, 9, 85, 57, 30, 16, 77, 21, 19, 26, 111, 56, 81, 20, 110, 80, 98, 76, 75, 117, 90, 79, 102, 91, 37, 71, 15, 119, 36, 103, 34, 78, 5, 46, 27, 95, 39, 11, 62, 7, 104, 14, 31, 47, 41, 24, 69, 64, 74, 105, 49, 10, 118, 0, 96, 43, 28, 88, 93, 6, 100, 107, 3, 2, 67, 4, 70, 68, 33, 66, 72, 8, 97, 99, 1, 35, 65], [127, 50, 48, 114, 115, 109, 112, 51, 45, 55, 61, 108, 121, 53, 44, 120, 58, 52, 25, 38, 123, 23, 124, 92, 126, 113, 42, 63, 87, 89, 106, 116, 22, 122, 82, 125, 18, 40, 32, 59, 54, 30, 86, 101, 29, 94, 84, 77, 73, 17, 57, 9, 83, 21, 85, 13, 60, 19, 119, 12, 118, 26, 110, 98, 20, 16, 111, 81, 41, 56, 103, 79, 90, 80, 37, 76, 102, 117, 15, 75, 27, 49, 34, 71, 36, 43, 78, 47, 62, 46, 91, 74, 107, 104, 69, 11, 100, 96, 24, 14, 28, 7, 64, 31, 39, 0, 95, 5, 105, 10, 72, 93, 68, 88, 97, 2, 4, 33, 70, 3, 6, 35, 99, 66, 67, 65, 8, 1], [127, 50, 48, 115, 109, 114, 112, 55, 51, 45, 61, 108, 121, 44, 120, 53, 52, 58, 25, 124, 23, 38, 123, 92, 63, 87, 106, 89, 42, 59, 113, 60, 122, 54, 125, 18, 116, 82, 126, 32, 40, 22, 118, 83, 111, 119, 19, 57, 77, 56, 9, 29, 20, 26, 101, 73, 102, 12, 94, 86, 30, 85, 81, 84, 13, 17, 49, 117, 80, 16, 36, 76, 98, 75, 21, 27, 103, 41, 110, 47, 104, 90, 71, 37, 62, 78, 79, 91, 95, 15, 34, 105, 46, 43, 11, 31, 39, 69, 14, 74, 5, 7, 88, 64, 96, 100, 107, 10, 28, 70, 93, 24, 33, 0, 2, 68, 4, 66, 97, 72, 3, 99, 6, 35, 67, 65, 1, 8], [127, 50, 48, 114, 115, 112, 109, 51, 55, 108, 61, 45, 121, 44, 53, 120, 58, 52, 123, 38, 25, 124, 23, 89, 87, 116, 113, 125, 106, 18, 60, 82, 63, 92, 42, 126, 22, 40, 122, 32, 9, 73, 59, 101, 77, 13, 54, 12, 83, 30, 84, 94, 81, 110, 85, 19, 29, 57, 17, 119, 86, 26, 80, 111, 16, 76, 36, 56, 21, 49, 20, 117, 41, 15, 102, 79, 75, 71, 98, 46, 37, 34, 103, 69, 74, 27, 39, 91, 5, 0, 7, 11, 90, 78, 104, 118, 14, 96, 62, 24, 10, 47, 95, 31, 70, 105, 28, 64, 100, 72, 2, 43, 4, 66, 67, 107, 88, 93, 97, 3, 33, 68, 35, 6, 99, 65, 1, 8], [127, 50, 114, 115, 48, 112, 109, 51, 108, 45, 61, 44, 55, 121, 53, 120, 58, 52, 124, 38, 23, 25, 63, 123, 113, 89, 42, 92, 18, 22, 122, 87, 82, 60, 116, 106, 40, 125, 59, 126, 73, 77, 32, 54, 12, 13, 83, 94, 9, 101, 85, 81, 110, 29, 30, 17, 16, 19, 111, 86, 20, 76, 80, 119, 75, 26, 37, 56, 36, 117, 62, 49, 98, 102, 41, 84, 90, 57, 71, 118, 27, 79, 104, 103, 34, 69, 21, 15, 70, 11, 46, 64, 14, 7, 5, 10, 47, 105, 95, 74, 78, 72, 43, 39, 2, 91, 88, 96, 31, 28, 100, 0, 66, 107, 24, 4, 33, 3, 93, 67, 97, 99, 68, 35, 6, 65, 1, 8], [127, 50, 48, 115, 114, 112, 109, 51, 55, 108, 61, 45, 44, 53, 121, 120, 58, 52, 25, 124, 38, 123, 23, 42, 89, 113, 87, 92, 22, 63, 82, 18, 106, 60, 122, 126, 116, 59, 40, 125, 77, 17, 32, 13, 73, 56, 94, 9, 86, 83, 81, 29, 12, 19, 80, 101, 54, 16, 30, 84, 20, 85, 21, 102, 26, 76, 111, 57, 75, 117, 98, 36, 62, 79, 47, 118, 41, 110, 15, 71, 90, 46, 7, 103, 78, 34, 14, 49, 27, 74, 37, 119, 11, 105, 39, 69, 91, 104, 70, 31, 5, 64, 100, 10, 95, 96, 88, 43, 72, 24, 33, 107, 68, 0, 2, 93, 28, 4, 66, 6, 67, 3, 97, 99, 35, 8, 65, 1], [127, 50, 48, 115, 114, 51, 109, 112, 45, 108, 61, 55, 121, 44, 53, 120, 58, 52, 25, 123, 23, 124, 38, 42, 87, 106, 89, 116, 60, 113, 126, 18, 125, 92, 122, 82, 40, 63, 22, 77, 94, 29, 30, 73, 59, 9, 32, 101, 17, 86, 13, 12, 83, 81, 80, 20, 56, 54, 85, 26, 76, 111, 118, 84, 57, 21, 16, 98, 19, 41, 102, 62, 110, 117, 36, 71, 7, 37, 34, 49, 75, 47, 79, 15, 69, 27, 119, 90, 78, 10, 104, 64, 0, 24, 91, 11, 39, 14, 95, 96, 74, 5, 93, 46, 31, 103, 107, 43, 6, 33, 105, 28, 88, 67, 72, 100, 2, 66, 3, 68, 70, 4, 35, 8, 97, 99, 1, 65], [127, 50, 48, 115, 114, 109, 112, 51, 55, 108, 45, 61, 121, 44, 53, 120, 52, 58, 25, 38, 123, 23, 124, 42, 87, 89, 92, 113, 22, 82, 60, 106, 18, 63, 40, 116, 125, 13, 126, 29, 59, 122, 32, 9, 54, 83, 57, 12, 94, 73, 81, 77, 26, 101, 20, 16, 86, 19, 30, 17, 85, 111, 56, 75, 84, 80, 21, 102, 15, 76, 36, 110, 117, 41, 98, 90, 78, 14, 62, 118, 79, 37, 10, 11, 49, 7, 91, 119, 47, 71, 27, 24, 103, 31, 43, 46, 74, 96, 34, 95, 100, 69, 5, 105, 104, 39, 6, 64, 28, 88, 93, 107, 0, 67, 72, 3, 4, 68, 33, 2, 70, 97, 66, 8, 35, 99, 1, 65]], "model.layers.25.self_attn.q_proj": [[62, 102, 124, 121, 56, 60, 97, 47, 50, 63, 38, 93, 114, 24, 127, 117, 95, 86, 105, 53, 89, 41, 35, 29, 43, 106, 115, 51, 52, 22, 37, 49, 103, 122, 20, 25, 123, 59, 87, 55, 33, 83, 91, 116, 57, 58, 100, 36, 99, 79, 27, 26, 46, 113, 111, 92, 108, 112, 109, 94, 61, 48, 125, 82, 19, 110, 54, 34, 42, 17, 39, 44, 12, 119, 126, 18, 118, 80, 120, 101, 104, 40, 45, 77, 98, 32, 16, 96, 28, 30, 107, 31, 90, 88, 21, 23, 81, 85, 70, 75, 84, 74, 72, 9, 4, 78, 15, 2, 14, 8, 76, 13, 67, 11, 0, 5, 66, 71, 10, 7, 73, 6, 68, 65, 3, 1, 64, 69], [102, 62, 93, 121, 124, 24, 97, 83, 56, 35, 60, 127, 85, 81, 13, 21, 17, 63, 86, 20, 22, 98, 36, 25, 123, 114, 92, 19, 120, 77, 84, 46, 74, 78, 48, 116, 111, 76, 52, 49, 11, 95, 33, 30, 32, 79, 115, 1, 38, 9, 71, 28, 122, 88, 34, 69, 31, 12, 104, 75, 101, 7, 66, 80, 108, 0, 43, 99, 117, 67, 29, 4, 72, 107, 8, 90, 113, 70, 58, 96, 23, 119, 103, 27, 26, 37, 18, 59, 54, 87, 14, 15, 126, 89, 16, 91, 94, 105, 53, 125, 10, 40, 106, 2, 51, 6, 118, 39, 82, 68, 42, 100, 73, 57, 50, 44, 3, 61, 65, 47, 41, 110, 55, 64, 45, 109, 112, 5], [62, 124, 102, 60, 97, 122, 123, 116, 93, 38, 44, 117, 43, 127, 59, 121, 87, 48, 114, 53, 118, 24, 89, 54, 39, 29, 58, 46, 94, 56, 40, 25, 82, 119, 57, 95, 106, 50, 125, 108, 99, 51, 49, 20, 109, 55, 47, 86, 126, 52, 111, 112, 63, 113, 45, 41, 91, 98, 104, 42, 110, 22, 79, 61, 115, 120, 103, 100, 31, 27, 92, 107, 17, 23, 105, 37, 35, 76, 85, 36, 34, 90, 101, 80, 32, 33, 96, 26, 28, 84, 12, 30, 13, 21, 18, 83, 19, 77, 15, 81, 16, 8, 9, 88, 74, 78, 7, 6, 73, 11, 75, 14, 70, 71, 10, 4, 5, 72, 68, 69, 2, 3, 67, 1, 66, 65, 64, 0], [62, 121, 102, 117, 124, 114, 60, 49, 50, 53, 116, 97, 123, 43, 38, 108, 122, 93, 45, 63, 127, 94, 59, 113, 104, 46, 58, 25, 44, 87, 125, 106, 61, 52, 54, 109, 17, 31, 118, 111, 56, 48, 47, 103, 33, 42, 119, 51, 120, 112, 34, 126, 29, 110, 107, 115, 57, 41, 40, 55, 89, 28, 105, 39, 22, 99, 37, 27, 98, 95, 82, 85, 90, 86, 100, 35, 20, 30, 26, 91, 36, 101, 24, 79, 96, 76, 80, 32, 19, 18, 23, 12, 92, 21, 8, 11, 84, 75, 77, 64, 1, 78, 10, 6, 16, 14, 0, 83, 4, 66, 74, 68, 69, 81, 67, 72, 7, 2, 70, 5, 71, 9, 65, 13, 73, 88, 3, 15], [58, 126, 117, 39, 111, 124, 59, 48, 51, 112, 25, 125, 54, 127, 47, 118, 123, 35, 62, 21, 55, 57, 89, 113, 96, 49, 63, 50, 60, 53, 61, 119, 116, 56, 29, 91, 45, 120, 110, 87, 52, 23, 114, 82, 42, 121, 122, 115, 46, 85, 109, 44, 108, 80, 27, 76, 107, 20, 105, 14, 93, 106, 83, 99, 41, 40, 43, 104, 38, 102, 9, 103, 28, 18, 31, 32, 16, 101, 36, 22, 95, 100, 79, 24, 8, 88, 81, 12, 13, 4, 84, 78, 92, 33, 71, 90, 37, 86, 97, 10, 69, 6, 94, 34, 3, 11, 30, 74, 75, 98, 73, 19, 77, 26, 66, 7, 70, 72, 17, 15, 2, 5, 0, 68, 64, 67, 1, 65], [111, 117, 58, 126, 39, 124, 59, 51, 48, 118, 25, 21, 50, 57, 54, 127, 55, 91, 125, 62, 112, 49, 113, 47, 63, 53, 110, 96, 61, 52, 119, 123, 116, 35, 56, 60, 120, 29, 87, 89, 23, 42, 121, 115, 45, 114, 46, 85, 82, 44, 108, 122, 109, 80, 76, 27, 107, 14, 106, 93, 41, 37, 105, 9, 102, 38, 43, 83, 104, 88, 40, 8, 20, 99, 100, 18, 24, 12, 28, 31, 10, 13, 81, 79, 32, 71, 101, 92, 103, 36, 69, 97, 30, 78, 94, 4, 33, 6, 16, 70, 22, 3, 90, 98, 95, 34, 86, 84, 26, 73, 19, 75, 74, 7, 11, 66, 5, 72, 77, 17, 65, 64, 15, 2, 67, 1, 68, 0], [117, 126, 39, 58, 111, 124, 59, 51, 48, 112, 21, 25, 57, 54, 127, 118, 62, 29, 55, 110, 50, 125, 49, 53, 35, 96, 91, 123, 63, 52, 61, 119, 113, 56, 116, 89, 120, 60, 45, 47, 87, 115, 121, 114, 82, 23, 80, 46, 44, 108, 85, 122, 42, 109, 107, 27, 76, 83, 14, 105, 102, 40, 38, 43, 9, 103, 28, 41, 106, 104, 20, 93, 8, 99, 32, 10, 13, 31, 37, 24, 36, 18, 81, 79, 12, 88, 70, 100, 16, 71, 92, 78, 101, 34, 22, 84, 33, 90, 94, 3, 7, 6, 97, 86, 69, 95, 73, 11, 30, 4, 75, 98, 19, 74, 72, 77, 66, 26, 15, 17, 2, 5, 64, 1, 0, 68, 65, 67], [126, 117, 39, 58, 111, 124, 51, 59, 48, 21, 25, 112, 54, 57, 62, 55, 127, 89, 125, 96, 63, 118, 29, 123, 50, 53, 113, 35, 110, 49, 61, 52, 60, 119, 120, 56, 116, 87, 91, 47, 23, 42, 45, 121, 44, 114, 80, 115, 122, 85, 46, 82, 108, 109, 83, 14, 76, 107, 27, 20, 40, 105, 102, 9, 8, 103, 106, 43, 38, 104, 41, 99, 93, 32, 18, 31, 4, 28, 79, 37, 95, 36, 24, 16, 84, 10, 100, 13, 22, 101, 81, 12, 92, 33, 90, 71, 78, 97, 30, 69, 70, 86, 88, 34, 94, 6, 11, 19, 98, 73, 3, 75, 74, 26, 7, 72, 77, 15, 17, 0, 66, 5, 64, 68, 65, 1, 2, 67], [40, 59, 34, 90, 83, 12, 53, 91, 63, 88, 17, 95, 45, 112, 85, 31, 79, 9, 124, 123, 13, 6, 108, 119, 113, 5, 109, 74, 126, 72, 99, 93, 96, 15, 78, 114, 62, 58, 47, 28, 29, 61, 55, 8, 73, 107, 18, 97, 52, 35, 41, 20, 86, 48, 14, 33, 87, 76, 94, 127, 122, 92, 16, 50, 121, 11, 98, 82, 116, 24, 101, 57, 110, 77, 120, 103, 10, 51, 71, 111, 49, 38, 60, 125, 54, 115, 42, 56, 105, 118, 106, 80, 23, 44, 46, 69, 100, 26, 117, 32, 39, 30, 102, 43, 36, 81, 89, 7, 37, 27, 22, 21, 84, 19, 25, 75, 70, 3, 2, 67, 4, 68, 64, 1, 66, 104, 0, 65], [40, 59, 34, 90, 124, 88, 83, 13, 63, 126, 79, 17, 85, 6, 95, 50, 121, 118, 53, 48, 46, 127, 12, 45, 10, 91, 106, 103, 22, 29, 4, 72, 116, 68, 74, 96, 60, 98, 55, 99, 112, 11, 58, 9, 125, 35, 69, 41, 24, 115, 120, 43, 31, 37, 61, 122, 62, 0, 39, 33, 113, 123, 105, 110, 78, 21, 77, 84, 107, 51, 67, 20, 7, 87, 54, 119, 28, 38, 47, 14, 73, 114, 82, 93, 26, 75, 117, 32, 94, 66, 18, 42, 56, 108, 111, 52, 44, 36, 1, 109, 2, 8, 102, 15, 30, 70, 23, 19, 71, 57, 86, 92, 49, 76, 81, 100, 101, 80, 27, 89, 97, 5, 16, 25, 104, 3, 65, 64], [40, 59, 34, 90, 85, 83, 127, 124, 63, 53, 17, 48, 88, 110, 116, 118, 79, 95, 13, 113, 29, 58, 96, 91, 31, 42, 111, 121, 12, 24, 11, 50, 94, 115, 98, 38, 45, 60, 10, 112, 47, 14, 114, 103, 125, 22, 55, 108, 25, 35, 44, 119, 109, 100, 36, 9, 57, 56, 86, 117, 102, 49, 26, 39, 72, 28, 6, 71, 61, 101, 23, 89, 52, 43, 27, 122, 41, 21, 51, 33, 123, 62, 30, 126, 54, 106, 105, 87, 37, 99, 46, 97, 92, 107, 84, 93, 120, 20, 0, 18, 104, 81, 16, 4, 82, 76, 78, 32, 15, 19, 80, 1, 7, 68, 74, 69, 75, 2, 8, 77, 73, 70, 66, 3, 5, 67, 65, 64], [40, 59, 34, 90, 88, 83, 124, 79, 17, 13, 95, 53, 12, 118, 9, 58, 126, 72, 85, 24, 63, 81, 4, 48, 114, 86, 2, 71, 11, 6, 45, 44, 112, 121, 75, 103, 116, 1, 50, 10, 105, 14, 0, 110, 3, 31, 69, 60, 78, 42, 46, 106, 39, 120, 74, 67, 115, 54, 32, 29, 43, 26, 98, 62, 8, 51, 113, 68, 52, 7, 20, 22, 21, 80, 91, 65, 92, 107, 57, 35, 37, 47, 56, 127, 84, 18, 89, 125, 16, 77, 15, 19, 94, 41, 96, 82, 108, 70, 55, 25, 109, 104, 76, 28, 49, 27, 61, 64, 93, 97, 23, 36, 100, 33, 99, 122, 111, 101, 66, 30, 119, 38, 73, 123, 102, 117, 87, 5], [58, 61, 127, 56, 38, 120, 125, 53, 89, 52, 115, 112, 107, 50, 83, 122, 57, 106, 124, 59, 116, 85, 60, 119, 23, 114, 118, 121, 123, 117, 55, 126, 87, 48, 99, 46, 96, 102, 49, 63, 110, 54, 62, 51, 111, 47, 76, 109, 113, 29, 45, 94, 41, 18, 44, 91, 42, 16, 28, 43, 36, 25, 104, 108, 103, 92, 78, 95, 40, 90, 82, 105, 80, 100, 35, 84, 39, 14, 17, 26, 34, 73, 70, 69, 12, 71, 32, 21, 13, 22, 37, 3, 74, 7, 27, 101, 20, 97, 98, 24, 31, 11, 33, 86, 79, 9, 30, 93, 8, 15, 88, 68, 72, 75, 5, 19, 10, 77, 81, 66, 4, 2, 6, 1, 64, 67, 65, 0], [56, 127, 61, 58, 38, 120, 89, 125, 53, 52, 50, 115, 59, 57, 124, 83, 114, 107, 119, 116, 112, 85, 106, 122, 118, 60, 55, 121, 123, 117, 126, 102, 23, 99, 63, 51, 54, 49, 110, 48, 46, 62, 96, 45, 29, 111, 47, 87, 94, 109, 113, 18, 25, 43, 76, 16, 41, 95, 44, 42, 90, 36, 91, 108, 104, 103, 92, 28, 40, 35, 80, 105, 73, 78, 84, 100, 39, 21, 82, 17, 70, 32, 71, 37, 13, 31, 98, 14, 8, 74, 26, 97, 22, 12, 101, 79, 24, 34, 27, 86, 11, 3, 15, 7, 20, 30, 33, 69, 77, 93, 68, 9, 72, 10, 19, 88, 81, 5, 0, 2, 6, 75, 4, 67, 1, 64, 65, 66], [61, 127, 58, 38, 56, 125, 120, 53, 89, 52, 115, 107, 50, 124, 83, 57, 59, 119, 116, 114, 85, 112, 121, 118, 126, 60, 122, 55, 123, 46, 23, 117, 48, 49, 106, 110, 62, 99, 63, 51, 102, 96, 54, 94, 87, 111, 47, 42, 29, 45, 113, 109, 16, 18, 25, 76, 43, 28, 41, 44, 108, 103, 40, 78, 95, 36, 90, 105, 104, 91, 80, 92, 39, 84, 82, 100, 73, 34, 35, 17, 21, 37, 14, 32, 71, 13, 12, 26, 22, 33, 70, 101, 97, 20, 74, 86, 11, 27, 98, 79, 68, 8, 15, 10, 24, 30, 93, 31, 9, 72, 77, 3, 69, 7, 88, 19, 75, 6, 81, 5, 4, 0, 65, 2, 64, 66, 1, 67], [127, 61, 56, 38, 58, 120, 89, 125, 53, 52, 57, 50, 115, 119, 59, 124, 114, 83, 116, 55, 107, 112, 23, 85, 123, 122, 121, 63, 118, 126, 99, 60, 62, 110, 117, 29, 48, 49, 106, 96, 94, 102, 87, 51, 54, 46, 45, 111, 47, 113, 109, 16, 18, 25, 43, 42, 44, 76, 90, 41, 91, 28, 108, 36, 92, 103, 104, 84, 40, 82, 14, 105, 26, 37, 35, 80, 100, 34, 95, 32, 39, 17, 12, 22, 69, 21, 3, 78, 13, 97, 70, 71, 101, 73, 74, 24, 98, 11, 20, 8, 86, 27, 33, 31, 30, 79, 93, 68, 72, 7, 10, 9, 15, 19, 81, 88, 77, 5, 75, 2, 4, 1, 6, 65, 66, 0, 64, 67], [42, 98, 106, 112, 26, 36, 21, 49, 118, 16, 94, 18, 54, 22, 48, 31, 60, 14, 13, 95, 101, 77, 93, 9, 123, 116, 88, 90, 114, 125, 58, 38, 113, 127, 53, 92, 15, 19, 8, 63, 47, 20, 52, 17, 99, 61, 124, 46, 12, 55, 111, 23, 73, 29, 27, 120, 115, 117, 84, 28, 122, 87, 10, 105, 51, 85, 33, 72, 24, 109, 104, 44, 34, 97, 74, 126, 45, 102, 43, 40, 30, 75, 59, 62, 107, 50, 4, 39, 100, 81, 108, 119, 121, 57, 41, 76, 2, 83, 35, 110, 71, 89, 64, 103, 37, 56, 67, 80, 7, 32, 91, 11, 82, 79, 25, 86, 96, 6, 0, 3, 78, 5, 1, 69, 65, 66, 70, 68], [42, 118, 98, 36, 106, 26, 18, 21, 124, 116, 63, 49, 111, 127, 54, 112, 123, 52, 88, 94, 24, 122, 126, 48, 31, 22, 99, 117, 53, 110, 55, 121, 58, 16, 38, 101, 20, 92, 47, 62, 33, 59, 61, 90, 95, 12, 51, 119, 60, 108, 46, 109, 8, 57, 29, 30, 120, 114, 28, 50, 44, 93, 11, 14, 113, 86, 125, 115, 107, 100, 39, 43, 77, 105, 56, 4, 34, 79, 97, 103, 40, 41, 27, 104, 45, 15, 37, 82, 102, 35, 23, 2, 10, 13, 96, 85, 74, 32, 75, 91, 25, 89, 0, 7, 87, 83, 84, 17, 19, 1, 73, 80, 71, 9, 81, 76, 6, 64, 67, 68, 5, 65, 78, 72, 66, 69, 70, 3], [42, 98, 118, 126, 106, 36, 26, 18, 112, 127, 24, 21, 48, 88, 46, 31, 41, 94, 63, 116, 20, 113, 125, 62, 53, 11, 54, 7, 8, 16, 52, 38, 49, 12, 124, 121, 123, 99, 111, 122, 39, 60, 4, 2, 50, 58, 109, 114, 120, 86, 117, 95, 90, 47, 30, 119, 57, 61, 1, 55, 33, 59, 79, 14, 56, 77, 37, 44, 51, 115, 103, 101, 92, 91, 107, 43, 75, 29, 6, 64, 102, 40, 110, 97, 105, 22, 108, 9, 80, 23, 45, 89, 19, 96, 93, 27, 82, 13, 67, 0, 104, 35, 83, 34, 17, 69, 85, 100, 32, 10, 74, 73, 25, 5, 76, 28, 84, 15, 81, 66, 87, 72, 65, 68, 71, 78, 3, 70], [42, 98, 106, 26, 31, 20, 16, 21, 18, 36, 112, 54, 75, 14, 77, 113, 9, 73, 49, 94, 79, 76, 15, 93, 116, 123, 118, 90, 101, 70, 10, 7, 19, 8, 1, 60, 53, 48, 6, 69, 58, 84, 74, 3, 34, 127, 78, 107, 114, 71, 99, 88, 0, 17, 2, 41, 105, 125, 67, 87, 109, 81, 25, 68, 119, 11, 28, 23, 22, 13, 27, 80, 40, 12, 64, 91, 82, 37, 72, 52, 4, 61, 47, 83, 62, 85, 38, 92, 24, 33, 32, 122, 44, 66, 115, 111, 46, 39, 86, 65, 104, 110, 102, 108, 120, 45, 63, 50, 96, 89, 5, 29, 55, 56, 95, 126, 57, 117, 59, 97, 100, 35, 121, 51, 124, 43, 30, 103], [39, 63, 64, 0, 66, 1, 12, 121, 3, 4, 2, 69, 79, 77, 9, 23, 87, 74, 70, 122, 43, 49, 7, 108, 127, 42, 83, 85, 53, 46, 67, 118, 48, 61, 18, 115, 31, 58, 93, 103, 113, 68, 45, 59, 105, 16, 99, 40, 47, 124, 52, 60, 75, 17, 20, 78, 89, 97, 19, 71, 114, 100, 112, 111, 33, 65, 55, 11, 84, 126, 25, 50, 57, 5, 88, 51, 116, 81, 6, 90, 102, 120, 98, 117, 14, 86, 21, 30, 109, 34, 32, 41, 92, 94, 80, 13, 10, 27, 28, 106, 73, 38, 56, 26, 123, 76, 96, 82, 101, 35, 29, 54, 15, 119, 36, 107, 22, 37, 8, 110, 91, 72, 104, 95, 125, 44, 24, 62], [39, 63, 87, 122, 79, 77, 121, 74, 12, 118, 9, 31, 46, 69, 4, 66, 83, 127, 23, 115, 108, 3, 17, 70, 64, 1, 93, 49, 100, 18, 43, 75, 20, 41, 52, 15, 72, 90, 85, 58, 42, 89, 7, 82, 16, 48, 78, 59, 13, 60, 113, 27, 124, 109, 50, 10, 94, 8, 112, 80, 92, 95, 125, 99, 106, 35, 67, 96, 117, 53, 114, 97, 81, 29, 19, 24, 91, 123, 55, 86, 11, 104, 56, 126, 111, 36, 21, 105, 101, 57, 71, 6, 65, 116, 62, 119, 54, 61, 45, 51, 26, 22, 33, 84, 47, 40, 120, 110, 88, 37, 38, 28, 14, 73, 34, 107, 25, 102, 44, 30, 98, 76, 68, 0, 5, 32, 2, 103], [39, 63, 64, 0, 1, 121, 12, 74, 4, 3, 23, 9, 69, 79, 77, 66, 87, 70, 2, 122, 43, 108, 49, 42, 127, 83, 46, 53, 93, 7, 68, 99, 48, 52, 61, 67, 5, 118, 40, 103, 31, 18, 85, 59, 113, 45, 58, 112, 114, 6, 65, 81, 60, 55, 120, 105, 51, 100, 116, 88, 71, 115, 25, 90, 126, 80, 34, 91, 22, 86, 14, 33, 11, 124, 17, 16, 78, 47, 72, 73, 20, 101, 94, 56, 123, 57, 37, 19, 89, 75, 8, 111, 35, 106, 109, 82, 29, 10, 36, 26, 97, 62, 27, 98, 96, 54, 28, 24, 92, 84, 119, 125, 117, 38, 104, 30, 76, 32, 95, 41, 102, 50, 107, 44, 13, 15, 110, 21], [39, 63, 74, 79, 9, 87, 77, 12, 4, 121, 3, 66, 64, 23, 1, 70, 69, 43, 108, 18, 72, 127, 93, 49, 42, 2, 122, 46, 67, 20, 31, 115, 40, 83, 5, 113, 48, 7, 53, 118, 61, 27, 68, 60, 45, 6, 19, 125, 100, 97, 52, 24, 114, 16, 124, 111, 71, 112, 58, 81, 47, 89, 13, 85, 17, 34, 10, 99, 86, 8, 14, 78, 76, 73, 59, 26, 82, 90, 51, 92, 33, 84, 120, 11, 75, 0, 65, 119, 29, 22, 126, 57, 62, 32, 80, 28, 95, 30, 117, 56, 88, 25, 116, 15, 94, 37, 105, 36, 104, 91, 109, 21, 102, 98, 41, 35, 50, 123, 55, 106, 101, 107, 44, 96, 110, 38, 54, 103], [106, 36, 85, 42, 63, 83, 77, 113, 80, 15, 6, 73, 122, 124, 91, 74, 119, 3, 71, 48, 27, 115, 68, 58, 127, 0, 2, 76, 25, 98, 50, 112, 95, 66, 32, 57, 118, 67, 93, 69, 94, 75, 12, 61, 9, 87, 65, 56, 5, 14, 1, 10, 19, 29, 22, 121, 79, 126, 4, 16, 11, 81, 13, 84, 125, 8, 54, 37, 64, 109, 7, 110, 88, 21, 116, 114, 70, 39, 46, 30, 120, 105, 33, 41, 18, 99, 55, 20, 23, 123, 86, 78, 59, 72, 102, 90, 52, 51, 34, 49, 82, 104, 117, 107, 26, 89, 96, 24, 53, 111, 45, 28, 40, 31, 17, 97, 44, 60, 35, 108, 62, 103, 43, 101, 47, 92, 38, 100], [106, 36, 63, 42, 122, 85, 91, 58, 83, 124, 48, 80, 113, 77, 50, 27, 61, 112, 119, 15, 93, 116, 29, 25, 87, 98, 115, 120, 95, 73, 59, 32, 74, 114, 46, 123, 51, 57, 35, 71, 45, 109, 117, 126, 44, 2, 121, 54, 94, 30, 96, 118, 127, 52, 47, 111, 65, 56, 125, 108, 60, 68, 105, 49, 110, 6, 41, 62, 90, 55, 26, 21, 39, 99, 86, 17, 53, 10, 81, 8, 38, 107, 11, 101, 43, 19, 78, 104, 40, 103, 75, 33, 82, 66, 102, 18, 20, 37, 88, 5, 14, 0, 23, 79, 97, 92, 24, 31, 16, 76, 28, 34, 89, 22, 3, 12, 84, 64, 72, 7, 13, 100, 69, 1, 9, 4, 70, 67], [106, 36, 63, 42, 122, 113, 85, 91, 50, 83, 27, 80, 117, 61, 48, 124, 25, 112, 29, 118, 98, 87, 93, 52, 123, 77, 95, 60, 15, 119, 32, 59, 58, 127, 115, 57, 96, 90, 53, 126, 51, 120, 17, 73, 54, 35, 49, 40, 94, 114, 41, 109, 45, 46, 44, 121, 55, 111, 116, 21, 110, 62, 47, 105, 74, 71, 56, 30, 125, 108, 99, 39, 107, 38, 88, 103, 24, 43, 6, 104, 11, 86, 23, 101, 78, 102, 37, 34, 20, 19, 97, 5, 79, 31, 75, 28, 8, 33, 14, 81, 22, 92, 68, 89, 100, 84, 10, 16, 66, 26, 12, 2, 82, 0, 13, 76, 69, 72, 18, 65, 3, 7, 4, 9, 67, 70, 1, 64], [106, 36, 122, 42, 85, 124, 113, 48, 91, 50, 83, 127, 93, 58, 63, 59, 27, 98, 80, 118, 32, 126, 77, 47, 15, 87, 29, 25, 73, 95, 35, 115, 57, 109, 119, 96, 39, 90, 54, 114, 17, 112, 55, 123, 99, 56, 94, 21, 71, 41, 117, 30, 74, 45, 11, 51, 61, 38, 6, 75, 52, 121, 49, 103, 5, 68, 46, 62, 120, 34, 111, 2, 116, 40, 60, 125, 110, 53, 108, 105, 86, 18, 24, 33, 44, 43, 107, 97, 81, 102, 104, 26, 19, 12, 88, 37, 101, 79, 10, 72, 89, 78, 8, 20, 31, 82, 92, 16, 28, 14, 23, 84, 0, 76, 3, 22, 65, 66, 13, 69, 100, 7, 9, 4, 70, 64, 67, 1], [108, 111, 93, 44, 24, 14, 125, 116, 15, 72, 85, 74, 82, 19, 121, 98, 76, 6, 47, 124, 81, 36, 63, 20, 35, 37, 119, 95, 1, 25, 11, 87, 3, 33, 67, 68, 126, 38, 97, 57, 64, 27, 34, 100, 29, 55, 77, 60, 96, 13, 51, 65, 40, 4, 62, 49, 70, 105, 123, 75, 52, 5, 30, 0, 79, 43, 73, 86, 41, 92, 71, 9, 56, 46, 113, 88, 32, 117, 112, 80, 31, 50, 7, 109, 103, 127, 84, 53, 107, 23, 10, 48, 104, 26, 106, 114, 45, 12, 115, 122, 83, 120, 90, 78, 2, 61, 89, 21, 17, 18, 99, 42, 28, 101, 110, 102, 59, 118, 54, 8, 94, 66, 22, 16, 58, 69, 39, 91], [108, 44, 111, 93, 74, 14, 24, 116, 72, 124, 125, 82, 76, 85, 19, 68, 15, 6, 36, 1, 98, 64, 13, 47, 87, 81, 37, 63, 4, 20, 126, 121, 100, 65, 29, 53, 3, 67, 35, 119, 46, 7, 51, 60, 43, 49, 11, 34, 123, 57, 95, 2, 50, 80, 41, 113, 0, 55, 62, 86, 101, 56, 127, 69, 45, 88, 70, 40, 73, 99, 75, 104, 97, 38, 96, 90, 92, 117, 105, 9, 5, 31, 27, 58, 10, 114, 61, 18, 23, 21, 12, 89, 25, 71, 33, 42, 83, 66, 102, 103, 77, 120, 107, 30, 112, 79, 54, 8, 122, 59, 78, 118, 28, 115, 48, 84, 16, 110, 106, 91, 26, 22, 32, 39, 52, 94, 17, 109], [108, 111, 44, 125, 93, 24, 82, 15, 14, 76, 13, 74, 50, 68, 36, 116, 124, 72, 19, 98, 1, 6, 121, 113, 0, 87, 20, 86, 89, 67, 9, 64, 47, 37, 126, 95, 34, 53, 60, 35, 11, 71, 63, 4, 33, 32, 119, 92, 29, 100, 127, 41, 22, 3, 62, 84, 73, 83, 80, 55, 59, 49, 46, 123, 85, 70, 16, 57, 94, 23, 43, 7, 56, 88, 27, 65, 52, 115, 51, 8, 120, 90, 96, 105, 30, 26, 31, 102, 103, 69, 110, 40, 97, 61, 81, 91, 42, 17, 28, 114, 38, 18, 106, 104, 10, 117, 45, 21, 58, 118, 12, 107, 78, 101, 112, 109, 99, 75, 39, 122, 77, 25, 48, 5, 79, 66, 54, 2], [108, 111, 93, 44, 82, 24, 125, 14, 68, 36, 76, 6, 72, 116, 22, 13, 74, 64, 19, 67, 50, 1, 37, 0, 124, 87, 15, 35, 98, 75, 119, 85, 103, 113, 47, 121, 18, 57, 51, 11, 55, 49, 29, 70, 62, 63, 34, 3, 53, 95, 88, 9, 127, 112, 86, 25, 65, 10, 60, 46, 126, 59, 20, 100, 89, 5, 97, 52, 2, 81, 40, 90, 109, 115, 120, 38, 92, 61, 117, 94, 104, 83, 30, 12, 96, 4, 123, 31, 84, 78, 56, 26, 43, 66, 33, 110, 48, 23, 41, 122, 54, 21, 28, 42, 106, 102, 58, 91, 39, 99, 107, 32, 7, 118, 73, 16, 105, 45, 114, 101, 27, 69, 80, 17, 8, 79, 71, 77]], "model.layers.25.self_attn.k_proj": [[62, 38, 121, 33, 124, 56, 127, 29, 20, 60, 72, 13, 26, 79, 83, 63, 74, 17, 91, 16, 78, 50, 24, 86, 80, 4, 77, 52, 71, 49, 22, 32, 25, 19, 102, 100, 123, 53, 47, 6, 9, 28, 120, 5, 18, 93, 117, 113, 12, 75, 99, 115, 3, 46, 110, 2, 54, 126, 119, 36, 111, 82, 44, 42, 95, 70, 116, 40, 34, 23, 57, 30, 87, 61, 118, 59, 43, 81, 8, 48, 92, 125, 58, 45, 106, 94, 11, 51, 41, 122, 88, 108, 0, 112, 107, 105, 55, 103, 65, 109, 37, 114, 21, 101, 31, 27, 89, 67, 10, 96, 104, 97, 85, 98, 35, 39, 68, 90, 1, 15, 84, 73, 14, 64, 7, 76, 69, 66], [103, 22, 99, 117, 126, 58, 32, 111, 59, 124, 51, 119, 60, 48, 62, 54, 93, 92, 114, 127, 61, 57, 123, 116, 56, 49, 120, 52, 55, 121, 63, 110, 46, 112, 50, 125, 122, 118, 113, 30, 53, 89, 43, 109, 44, 86, 115, 45, 41, 108, 42, 100, 104, 47, 40, 37, 27, 106, 107, 38, 35, 91, 105, 82, 34, 84, 97, 101, 80, 81, 14, 29, 79, 102, 21, 16, 36, 33, 90, 26, 94, 83, 78, 20, 18, 87, 76, 98, 31, 28, 19, 74, 24, 11, 85, 12, 95, 23, 25, 6, 7, 39, 77, 17, 96, 5, 8, 13, 75, 72, 15, 88, 68, 10, 67, 73, 2, 0, 1, 4, 9, 71, 69, 3, 66, 70, 65, 64], [59, 104, 98, 90, 83, 17, 13, 79, 72, 112, 63, 31, 12, 88, 24, 75, 118, 85, 0, 53, 50, 9, 124, 1, 4, 71, 121, 126, 14, 6, 86, 68, 81, 10, 67, 2, 125, 116, 111, 66, 123, 42, 55, 109, 41, 45, 11, 25, 57, 47, 52, 107, 46, 60, 7, 26, 28, 115, 20, 114, 49, 51, 95, 69, 48, 108, 58, 56, 18, 103, 84, 29, 74, 44, 91, 62, 94, 106, 22, 33, 54, 92, 122, 39, 21, 30, 119, 110, 82, 113, 117, 99, 16, 100, 105, 93, 89, 38, 8, 87, 23, 27, 43, 96, 80, 78, 102, 35, 120, 76, 97, 32, 3, 77, 101, 36, 37, 127, 70, 65, 61, 19, 15, 73, 5, 34, 64, 40], [102, 22, 61, 127, 35, 56, 58, 32, 125, 50, 52, 120, 60, 53, 30, 59, 98, 116, 111, 86, 49, 117, 55, 126, 115, 119, 124, 118, 110, 123, 54, 92, 121, 33, 63, 112, 122, 57, 48, 113, 16, 109, 62, 114, 51, 28, 43, 26, 46, 44, 47, 108, 42, 45, 100, 106, 36, 107, 105, 14, 18, 85, 27, 40, 99, 41, 37, 83, 104, 103, 101, 25, 11, 39, 89, 24, 34, 20, 77, 19, 29, 38, 23, 90, 91, 87, 82, 12, 17, 97, 94, 31, 93, 84, 74, 95, 72, 79, 96, 13, 5, 88, 73, 78, 15, 70, 8, 4, 10, 81, 67, 9, 80, 1, 6, 21, 76, 66, 7, 64, 75, 71, 2, 68, 0, 69, 3, 65], [106, 34, 30, 26, 49, 48, 118, 14, 64, 21, 88, 16, 42, 116, 18, 54, 123, 8, 6, 119, 20, 121, 60, 117, 53, 127, 114, 124, 55, 120, 31, 63, 4, 126, 105, 104, 87, 58, 67, 107, 10, 12, 2, 3, 28, 122, 99, 110, 74, 56, 47, 57, 125, 83, 111, 59, 1, 46, 109, 108, 62, 9, 52, 86, 43, 61, 51, 115, 112, 40, 44, 45, 98, 37, 113, 50, 79, 23, 17, 84, 65, 13, 101, 102, 39, 35, 103, 89, 22, 25, 41, 38, 77, 27, 73, 95, 100, 29, 75, 33, 11, 93, 19, 96, 68, 70, 32, 7, 36, 81, 91, 66, 97, 71, 15, 78, 69, 82, 94, 92, 5, 76, 0, 85, 72, 80, 24, 90], [63, 103, 64, 121, 9, 65, 1, 79, 12, 66, 4, 77, 69, 3, 87, 74, 70, 122, 44, 113, 107, 118, 110, 106, 67, 23, 127, 48, 52, 115, 18, 58, 68, 7, 20, 5, 85, 59, 81, 6, 40, 105, 50, 83, 31, 93, 61, 78, 53, 117, 47, 89, 39, 2, 33, 60, 27, 19, 124, 46, 43, 125, 75, 95, 109, 16, 17, 80, 84, 35, 10, 36, 29, 49, 126, 25, 72, 42, 90, 112, 55, 116, 11, 54, 92, 99, 98, 71, 56, 97, 73, 0, 21, 86, 111, 45, 15, 41, 37, 38, 82, 30, 51, 101, 8, 96, 62, 114, 24, 104, 91, 26, 14, 88, 119, 28, 120, 123, 100, 57, 94, 108, 34, 22, 32, 102, 76, 13], [42, 63, 122, 83, 85, 100, 112, 0, 119, 80, 15, 77, 91, 73, 124, 29, 115, 50, 57, 127, 66, 106, 25, 113, 96, 71, 6, 61, 87, 109, 34, 126, 121, 74, 65, 68, 3, 58, 75, 1, 49, 45, 11, 46, 2, 111, 56, 90, 51, 30, 44, 125, 120, 95, 8, 17, 81, 5, 14, 60, 4, 98, 31, 24, 108, 55, 23, 93, 41, 89, 59, 116, 27, 36, 110, 52, 12, 118, 102, 62, 86, 35, 10, 47, 94, 114, 104, 54, 48, 101, 105, 82, 43, 33, 18, 103, 78, 107, 40, 20, 38, 22, 99, 117, 53, 123, 28, 92, 7, 39, 84, 37, 26, 97, 76, 88, 32, 69, 13, 72, 64, 67, 16, 19, 70, 21, 79, 9], [44, 111, 108, 6, 0, 93, 47, 24, 14, 72, 76, 68, 65, 74, 64, 82, 2, 15, 53, 66, 125, 124, 50, 67, 116, 9, 85, 19, 87, 20, 81, 121, 119, 52, 13, 3, 57, 70, 75, 34, 101, 7, 11, 113, 73, 54, 86, 100, 62, 51, 127, 55, 80, 126, 120, 112, 10, 90, 16, 123, 104, 63, 60, 110, 49, 41, 61, 97, 45, 32, 107, 4, 96, 105, 89, 115, 95, 58, 59, 114, 22, 83, 88, 122, 79, 37, 36, 31, 5, 1, 109, 33, 56, 43, 117, 98, 46, 103, 118, 99, 94, 27, 39, 28, 23, 21, 42, 91, 35, 102, 29, 25, 40, 69, 38, 48, 106, 84, 30, 71, 26, 92, 18, 77, 12, 17, 78, 8]], "model.layers.25.self_attn.qk_proj": [[63, 59, 42, 62, 106, 44, 111, 58, 127, 124, 61, 108, 121, 126, 117, 56, 122, 15, 93, 64, 50, 0, 79, 102, 125, 60, 77, 19, 118, 53, 13, 85, 90, 104, 10, 12, 23, 83, 115, 21, 36, 38, 76, 24, 113, 87, 98, 26, 103, 123, 73, 112, 119, 9, 74, 88, 116, 49, 52, 27, 68, 48, 47, 120, 6, 82, 34, 65, 46, 18, 4, 2, 39, 55, 57, 114, 67, 80, 51, 16, 1, 78, 17, 66, 54, 70, 29, 14, 99, 3, 40, 8, 31, 81, 86, 72, 110, 95, 97, 89, 33, 20, 84, 43, 22, 35, 109, 41, 25, 107, 32, 94, 69, 91, 100, 96, 7, 5, 71, 11, 105, 37, 28, 30, 75, 45, 101, 92], [63, 59, 42, 62, 106, 44, 111, 127, 58, 61, 124, 121, 117, 108, 126, 56, 122, 0, 93, 102, 15, 50, 79, 64, 125, 77, 60, 19, 23, 118, 10, 21, 12, 112, 53, 103, 38, 76, 83, 13, 6, 90, 98, 115, 74, 119, 88, 113, 4, 24, 36, 9, 87, 116, 85, 104, 26, 49, 39, 48, 68, 73, 123, 120, 52, 47, 34, 65, 40, 2, 29, 82, 27, 57, 46, 1, 16, 66, 18, 55, 54, 72, 78, 14, 80, 17, 51, 3, 70, 99, 110, 114, 81, 89, 31, 22, 67, 97, 20, 109, 94, 8, 95, 43, 33, 25, 86, 107, 84, 41, 100, 5, 11, 75, 71, 96, 45, 28, 91, 69, 35, 105, 7, 30, 37, 92, 32, 101], [63, 59, 42, 62, 106, 44, 111, 127, 58, 61, 126, 108, 121, 124, 117, 56, 64, 93, 122, 125, 50, 79, 60, 0, 77, 118, 15, 83, 13, 112, 19, 119, 23, 90, 102, 36, 76, 21, 38, 98, 10, 74, 116, 115, 87, 24, 12, 120, 85, 73, 103, 9, 104, 113, 53, 49, 88, 4, 34, 48, 39, 6, 68, 82, 29, 26, 52, 66, 47, 27, 2, 1, 65, 72, 123, 3, 16, 18, 57, 70, 67, 80, 54, 51, 55, 46, 40, 78, 114, 14, 99, 17, 31, 89, 81, 110, 86, 35, 94, 22, 20, 109, 33, 84, 100, 25, 95, 107, 91, 96, 43, 69, 71, 105, 5, 8, 97, 92, 41, 28, 75, 32, 7, 11, 45, 30, 101, 37], [63, 59, 42, 106, 62, 44, 58, 111, 127, 61, 121, 108, 126, 117, 56, 124, 122, 125, 0, 93, 64, 50, 79, 53, 102, 60, 77, 15, 83, 115, 13, 112, 113, 19, 21, 10, 85, 118, 38, 12, 119, 74, 76, 87, 73, 23, 26, 98, 104, 34, 9, 36, 24, 88, 39, 6, 120, 52, 49, 116, 55, 4, 103, 68, 65, 66, 67, 72, 40, 90, 80, 48, 3, 47, 16, 46, 123, 114, 18, 54, 1, 78, 82, 27, 110, 57, 29, 70, 14, 99, 2, 17, 81, 51, 31, 89, 86, 94, 33, 20, 43, 22, 109, 45, 97, 69, 35, 84, 25, 100, 41, 5, 71, 8, 7, 95, 91, 96, 105, 107, 28, 75, 32, 92, 11, 30, 101, 37], [63, 59, 62, 42, 106, 44, 58, 111, 127, 121, 126, 61, 108, 56, 117, 124, 125, 122, 50, 93, 79, 60, 15, 0, 102, 64, 13, 77, 118, 119, 23, 83, 53, 113, 98, 38, 74, 85, 116, 104, 112, 76, 73, 19, 115, 12, 36, 90, 10, 21, 103, 123, 34, 4, 9, 49, 88, 55, 24, 52, 87, 26, 72, 46, 68, 120, 70, 48, 27, 39, 65, 18, 2, 57, 82, 1, 47, 16, 14, 99, 66, 114, 51, 81, 80, 6, 17, 54, 29, 40, 67, 78, 31, 3, 22, 110, 86, 25, 33, 94, 35, 95, 20, 107, 89, 84, 43, 109, 41, 97, 75, 69, 105, 45, 7, 101, 71, 5, 92, 96, 28, 30, 100, 91, 8, 32, 11, 37], [63, 59, 106, 62, 42, 44, 58, 111, 127, 121, 61, 124, 126, 108, 56, 117, 122, 93, 125, 79, 50, 64, 0, 15, 13, 102, 77, 83, 118, 115, 119, 36, 23, 85, 90, 21, 98, 87, 52, 12, 88, 19, 112, 38, 60, 113, 34, 10, 49, 104, 76, 73, 53, 103, 26, 4, 9, 116, 74, 24, 27, 72, 68, 70, 39, 17, 1, 14, 51, 16, 46, 81, 66, 57, 55, 47, 48, 6, 82, 2, 3, 40, 123, 18, 120, 80, 65, 99, 54, 114, 67, 22, 78, 86, 110, 97, 29, 84, 35, 31, 89, 94, 25, 41, 20, 32, 33, 109, 69, 107, 43, 100, 5, 75, 95, 7, 8, 105, 71, 96, 28, 11, 30, 45, 92, 91, 37, 101], [63, 59, 42, 106, 62, 44, 111, 127, 58, 121, 61, 108, 124, 126, 117, 56, 122, 79, 93, 102, 15, 0, 77, 64, 125, 19, 13, 85, 26, 118, 113, 60, 98, 38, 119, 21, 112, 83, 76, 116, 88, 74, 90, 104, 50, 12, 23, 103, 36, 1, 9, 49, 10, 53, 24, 52, 73, 57, 48, 39, 70, 34, 87, 18, 47, 46, 27, 16, 82, 115, 72, 2, 51, 81, 123, 80, 65, 68, 99, 14, 120, 67, 66, 17, 78, 55, 4, 54, 6, 29, 40, 3, 25, 22, 86, 31, 35, 114, 94, 41, 33, 110, 109, 89, 8, 100, 107, 84, 5, 20, 75, 97, 32, 95, 11, 28, 91, 7, 45, 30, 69, 71, 43, 105, 101, 37, 96, 92], [63, 59, 62, 42, 106, 44, 58, 111, 127, 121, 124, 108, 61, 117, 126, 56, 122, 93, 125, 0, 64, 79, 60, 15, 50, 118, 13, 77, 116, 112, 38, 21, 90, 102, 52, 83, 19, 23, 103, 26, 85, 119, 10, 48, 12, 36, 98, 113, 24, 53, 88, 76, 87, 74, 70, 57, 39, 27, 73, 65, 9, 49, 47, 104, 1, 51, 29, 4, 68, 34, 46, 18, 123, 115, 2, 55, 66, 82, 78, 16, 120, 54, 72, 81, 99, 80, 6, 110, 22, 95, 86, 14, 17, 40, 3, 67, 31, 33, 114, 89, 25, 94, 35, 8, 84, 97, 100, 107, 41, 43, 28, 20, 109, 45, 91, 69, 5, 96, 7, 30, 75, 32, 37, 11, 105, 71, 92, 101], [63, 59, 42, 106, 62, 44, 111, 58, 127, 124, 121, 61, 108, 117, 126, 56, 122, 93, 125, 0, 64, 15, 50, 79, 85, 13, 102, 77, 38, 90, 19, 118, 112, 83, 116, 21, 23, 26, 103, 60, 119, 113, 24, 88, 53, 76, 12, 10, 48, 52, 74, 68, 87, 73, 34, 46, 27, 39, 36, 9, 98, 47, 115, 4, 104, 51, 29, 49, 70, 1, 120, 65, 82, 123, 14, 80, 114, 55, 6, 17, 66, 16, 3, 54, 57, 67, 78, 81, 2, 18, 99, 110, 31, 22, 8, 86, 40, 25, 72, 89, 35, 100, 97, 95, 94, 20, 43, 33, 109, 91, 84, 107, 41, 69, 5, 45, 30, 96, 75, 28, 71, 32, 11, 7, 37, 92, 105, 101], [63, 59, 106, 42, 62, 44, 111, 58, 127, 121, 124, 108, 61, 117, 126, 56, 122, 0, 64, 93, 125, 119, 50, 15, 60, 79, 102, 53, 83, 116, 118, 77, 74, 13, 38, 90, 12, 120, 23, 112, 98, 76, 85, 10, 73, 103, 21, 19, 36, 113, 9, 88, 104, 115, 49, 68, 24, 46, 48, 6, 4, 87, 26, 70, 34, 47, 39, 1, 52, 65, 57, 66, 18, 54, 2, 8, 51, 3, 123, 80, 27, 29, 67, 55, 14, 17, 40, 81, 114, 82, 86, 16, 110, 72, 78, 97, 99, 31, 25, 22, 94, 33, 84, 43, 89, 20, 35, 7, 95, 69, 75, 100, 11, 28, 30, 32, 5, 71, 96, 107, 105, 45, 109, 41, 91, 37, 101, 92], [63, 59, 62, 42, 106, 44, 58, 111, 127, 124, 108, 121, 61, 117, 126, 56, 122, 93, 0, 64, 79, 15, 13, 125, 119, 50, 77, 102, 38, 116, 118, 83, 53, 60, 21, 10, 74, 85, 112, 12, 19, 23, 76, 98, 73, 49, 113, 104, 34, 36, 6, 90, 9, 115, 87, 88, 68, 103, 4, 48, 24, 120, 2, 39, 26, 8, 55, 65, 80, 1, 47, 52, 46, 40, 66, 54, 51, 123, 14, 70, 16, 99, 81, 67, 3, 82, 86, 29, 27, 78, 18, 17, 114, 110, 94, 22, 57, 35, 84, 31, 33, 25, 20, 97, 72, 89, 69, 96, 95, 107, 43, 7, 75, 71, 5, 105, 11, 28, 91, 100, 30, 92, 32, 41, 45, 101, 37, 109], [63, 59, 42, 106, 62, 44, 111, 127, 58, 121, 108, 126, 124, 61, 56, 117, 122, 0, 93, 15, 79, 64, 77, 13, 85, 19, 83, 50, 102, 118, 21, 116, 9, 23, 90, 53, 38, 98, 104, 26, 10, 76, 125, 112, 60, 88, 74, 36, 119, 24, 12, 103, 48, 113, 73, 68, 6, 87, 4, 49, 40, 82, 65, 115, 47, 39, 46, 27, 34, 55, 52, 8, 123, 120, 14, 80, 17, 57, 16, 54, 2, 99, 1, 110, 29, 67, 114, 81, 18, 66, 22, 78, 3, 86, 31, 70, 97, 35, 94, 69, 51, 84, 25, 107, 33, 72, 89, 30, 28, 20, 75, 41, 5, 100, 45, 43, 95, 96, 91, 7, 11, 37, 71, 105, 32, 109, 101, 92], [63, 59, 42, 106, 62, 44, 111, 58, 127, 124, 121, 61, 108, 126, 117, 56, 122, 93, 64, 116, 118, 102, 50, 0, 15, 79, 83, 112, 90, 125, 77, 13, 19, 48, 113, 60, 23, 119, 85, 24, 26, 103, 104, 38, 36, 53, 21, 12, 87, 10, 9, 76, 88, 98, 74, 73, 54, 39, 6, 52, 34, 8, 68, 49, 4, 115, 1, 65, 27, 123, 47, 18, 46, 82, 51, 120, 66, 14, 2, 55, 29, 57, 99, 16, 22, 17, 80, 40, 3, 81, 70, 89, 110, 25, 86, 114, 107, 94, 78, 67, 31, 33, 97, 20, 35, 84, 91, 28, 43, 95, 41, 32, 30, 105, 11, 109, 69, 100, 75, 5, 72, 7, 96, 45, 92, 71, 37, 101], [63, 59, 42, 106, 62, 44, 111, 127, 58, 61, 124, 108, 121, 117, 126, 56, 64, 122, 93, 50, 0, 60, 125, 77, 79, 102, 90, 15, 118, 116, 13, 119, 48, 83, 85, 112, 120, 36, 19, 53, 38, 52, 9, 76, 113, 10, 74, 87, 21, 27, 103, 23, 73, 6, 104, 12, 26, 24, 8, 34, 88, 98, 1, 51, 49, 39, 65, 115, 68, 47, 66, 4, 80, 54, 81, 67, 123, 16, 82, 18, 57, 110, 2, 29, 17, 70, 55, 99, 14, 31, 114, 86, 46, 3, 78, 22, 89, 40, 11, 43, 95, 20, 97, 100, 33, 41, 35, 72, 32, 94, 5, 69, 84, 45, 25, 37, 107, 7, 75, 109, 91, 28, 105, 30, 92, 71, 96, 101], [63, 59, 42, 106, 62, 44, 58, 111, 127, 61, 108, 121, 124, 56, 117, 126, 122, 0, 93, 64, 79, 102, 77, 15, 60, 50, 83, 125, 85, 13, 116, 76, 118, 74, 112, 38, 104, 36, 53, 19, 98, 119, 26, 10, 34, 9, 23, 48, 12, 24, 21, 113, 52, 90, 103, 73, 68, 27, 87, 115, 88, 39, 8, 55, 1, 6, 70, 49, 4, 67, 2, 120, 66, 80, 82, 65, 29, 18, 51, 3, 47, 16, 81, 123, 14, 46, 54, 99, 86, 31, 57, 22, 78, 17, 97, 110, 114, 40, 43, 100, 72, 20, 33, 89, 94, 35, 41, 84, 25, 107, 11, 7, 96, 95, 69, 5, 30, 75, 105, 28, 32, 45, 71, 91, 101, 92, 37, 109], [63, 59, 62, 42, 106, 44, 127, 111, 58, 124, 121, 126, 108, 61, 56, 117, 122, 93, 79, 15, 64, 77, 50, 85, 0, 125, 116, 102, 83, 112, 26, 76, 118, 13, 38, 23, 73, 90, 24, 60, 19, 113, 87, 74, 98, 10, 12, 9, 103, 119, 21, 88, 53, 4, 55, 34, 27, 47, 36, 104, 48, 39, 68, 120, 52, 115, 80, 1, 40, 123, 16, 49, 14, 8, 70, 78, 29, 46, 114, 17, 18, 81, 57, 2, 51, 82, 3, 66, 65, 6, 110, 54, 67, 99, 33, 86, 97, 31, 22, 20, 72, 89, 69, 95, 84, 94, 41, 35, 5, 107, 25, 43, 11, 100, 105, 7, 28, 75, 91, 71, 45, 109, 30, 96, 101, 32, 37, 92], [63, 59, 42, 106, 62, 44, 58, 111, 127, 121, 124, 61, 126, 108, 117, 56, 122, 93, 64, 0, 15, 102, 50, 79, 23, 118, 125, 112, 90, 19, 116, 13, 119, 83, 77, 103, 113, 85, 88, 87, 38, 21, 53, 39, 26, 104, 120, 98, 12, 36, 76, 60, 24, 74, 27, 73, 48, 10, 34, 9, 29, 70, 47, 52, 65, 66, 123, 82, 49, 68, 1, 4, 80, 54, 46, 40, 16, 51, 114, 110, 18, 2, 57, 3, 78, 115, 99, 14, 81, 72, 95, 33, 17, 8, 67, 89, 86, 97, 6, 22, 31, 84, 25, 41, 94, 55, 20, 43, 35, 5, 107, 11, 28, 45, 75, 96, 100, 30, 69, 32, 91, 105, 71, 109, 7, 92, 37, 101], [63, 59, 42, 62, 106, 44, 58, 127, 111, 121, 61, 117, 124, 108, 126, 56, 93, 122, 0, 64, 50, 118, 79, 116, 15, 77, 112, 13, 19, 102, 90, 125, 23, 76, 60, 38, 85, 87, 48, 21, 119, 83, 24, 74, 53, 98, 103, 88, 39, 70, 12, 104, 26, 1, 120, 113, 36, 73, 54, 10, 9, 115, 29, 34, 27, 4, 47, 52, 49, 66, 3, 114, 18, 82, 16, 65, 123, 110, 68, 67, 72, 81, 57, 80, 78, 2, 14, 51, 6, 99, 40, 55, 86, 89, 31, 8, 46, 22, 43, 17, 94, 95, 20, 33, 69, 25, 84, 28, 109, 107, 41, 35, 11, 100, 97, 91, 5, 96, 32, 7, 75, 71, 30, 45, 105, 92, 37, 101], [63, 59, 42, 62, 106, 44, 58, 111, 127, 121, 61, 108, 124, 117, 126, 56, 122, 64, 50, 102, 93, 0, 60, 79, 15, 13, 125, 116, 103, 77, 118, 98, 83, 53, 112, 104, 76, 23, 113, 38, 12, 119, 34, 19, 74, 36, 24, 10, 85, 70, 87, 21, 52, 9, 4, 90, 73, 88, 48, 123, 1, 120, 29, 47, 72, 46, 68, 26, 49, 39, 82, 114, 54, 115, 16, 66, 51, 67, 18, 55, 27, 40, 2, 80, 3, 110, 6, 99, 81, 57, 65, 78, 43, 14, 86, 33, 31, 94, 17, 20, 89, 35, 109, 25, 97, 71, 107, 105, 69, 22, 100, 96, 45, 32, 7, 91, 84, 5, 8, 11, 30, 95, 41, 101, 28, 75, 92, 37], [63, 59, 42, 62, 106, 44, 111, 127, 58, 121, 108, 61, 117, 124, 126, 56, 122, 64, 93, 0, 102, 79, 15, 125, 50, 116, 23, 13, 60, 112, 77, 83, 118, 103, 19, 76, 12, 38, 119, 36, 87, 85, 74, 98, 120, 104, 34, 21, 26, 115, 90, 113, 48, 88, 73, 4, 53, 1, 10, 68, 72, 70, 24, 9, 52, 47, 49, 54, 123, 39, 18, 46, 2, 29, 40, 55, 6, 65, 27, 114, 16, 67, 82, 66, 99, 80, 81, 78, 57, 3, 51, 110, 17, 97, 14, 22, 94, 89, 86, 25, 107, 96, 105, 33, 41, 95, 100, 20, 71, 31, 35, 11, 69, 109, 84, 7, 43, 75, 8, 5, 91, 37, 32, 28, 45, 30, 101, 92], [63, 59, 106, 42, 62, 44, 127, 111, 58, 124, 121, 117, 61, 108, 126, 56, 122, 93, 0, 50, 15, 13, 125, 64, 116, 112, 79, 77, 104, 23, 118, 113, 60, 98, 74, 85, 87, 53, 83, 76, 120, 36, 12, 119, 38, 90, 24, 48, 49, 54, 102, 103, 9, 73, 19, 21, 10, 47, 72, 34, 88, 26, 70, 115, 46, 4, 27, 67, 114, 1, 55, 39, 66, 52, 29, 17, 40, 65, 6, 123, 51, 16, 18, 80, 2, 68, 82, 86, 78, 14, 57, 3, 110, 99, 89, 43, 81, 31, 94, 20, 25, 95, 33, 22, 5, 35, 97, 84, 107, 91, 100, 109, 71, 7, 96, 41, 32, 75, 8, 69, 37, 11, 105, 30, 28, 92, 45, 101], [63, 59, 42, 106, 62, 44, 58, 111, 127, 61, 121, 108, 117, 126, 124, 56, 64, 122, 0, 93, 125, 15, 50, 79, 112, 77, 60, 13, 118, 53, 116, 23, 102, 119, 74, 24, 83, 12, 113, 120, 38, 85, 87, 10, 36, 115, 76, 104, 98, 19, 73, 103, 21, 1, 54, 9, 48, 26, 6, 72, 88, 90, 47, 49, 34, 66, 68, 51, 4, 82, 52, 123, 39, 114, 3, 46, 70, 65, 55, 18, 2, 29, 27, 40, 57, 67, 99, 16, 14, 110, 80, 31, 78, 17, 43, 33, 86, 35, 89, 94, 109, 84, 95, 81, 8, 22, 25, 20, 97, 5, 45, 11, 107, 7, 69, 91, 100, 28, 32, 71, 105, 30, 75, 92, 41, 96, 101, 37], [63, 59, 42, 106, 62, 44, 58, 111, 127, 61, 121, 108, 124, 117, 126, 56, 125, 93, 122, 0, 79, 23, 102, 118, 77, 50, 64, 116, 15, 60, 123, 76, 103, 13, 53, 90, 120, 83, 104, 115, 9, 24, 38, 21, 113, 74, 36, 98, 85, 87, 26, 12, 112, 48, 6, 19, 51, 73, 4, 10, 88, 34, 49, 68, 119, 52, 29, 1, 39, 66, 46, 55, 82, 47, 72, 114, 27, 18, 2, 54, 3, 16, 57, 80, 40, 95, 67, 99, 14, 33, 65, 81, 94, 31, 70, 78, 22, 110, 25, 35, 89, 86, 17, 20, 8, 97, 43, 91, 84, 105, 96, 71, 28, 45, 5, 11, 109, 32, 100, 69, 7, 30, 41, 101, 75, 107, 37, 92], [63, 59, 42, 62, 106, 44, 58, 111, 127, 61, 124, 121, 108, 126, 117, 56, 125, 122, 93, 15, 0, 64, 77, 23, 102, 79, 118, 83, 50, 13, 112, 116, 19, 104, 21, 38, 113, 53, 98, 49, 115, 60, 103, 12, 85, 74, 6, 76, 119, 9, 10, 26, 48, 90, 34, 120, 36, 87, 88, 24, 73, 4, 68, 123, 54, 52, 46, 72, 16, 51, 39, 65, 40, 47, 27, 82, 18, 55, 3, 66, 2, 80, 29, 114, 22, 110, 1, 81, 78, 94, 99, 14, 17, 67, 70, 33, 57, 45, 84, 8, 97, 31, 20, 25, 86, 89, 5, 32, 41, 35, 75, 71, 95, 96, 100, 43, 28, 11, 107, 69, 109, 7, 91, 105, 92, 37, 30, 101], [63, 59, 42, 106, 62, 44, 111, 58, 127, 121, 61, 108, 124, 126, 117, 56, 122, 93, 64, 15, 0, 102, 79, 125, 13, 50, 118, 83, 77, 90, 53, 98, 12, 85, 23, 21, 112, 19, 103, 119, 60, 113, 10, 24, 104, 76, 87, 36, 49, 9, 74, 116, 48, 26, 120, 73, 88, 38, 6, 115, 47, 123, 18, 39, 34, 82, 27, 52, 16, 65, 1, 2, 54, 40, 70, 14, 68, 55, 4, 80, 29, 66, 72, 86, 46, 57, 81, 110, 17, 67, 8, 51, 99, 31, 89, 114, 3, 25, 78, 20, 33, 22, 94, 97, 95, 84, 43, 107, 5, 32, 11, 71, 75, 7, 105, 100, 109, 35, 91, 96, 41, 28, 92, 30, 45, 69, 101, 37], [63, 59, 62, 42, 106, 44, 58, 111, 127, 121, 124, 61, 108, 126, 117, 56, 93, 122, 64, 50, 0, 15, 118, 125, 79, 116, 102, 13, 24, 90, 87, 48, 26, 23, 112, 120, 77, 60, 38, 119, 19, 12, 21, 103, 113, 85, 76, 49, 104, 83, 10, 73, 36, 54, 74, 98, 53, 47, 52, 123, 88, 39, 9, 27, 29, 34, 4, 6, 68, 51, 82, 115, 1, 70, 18, 2, 40, 16, 46, 65, 114, 80, 66, 67, 8, 99, 14, 3, 57, 89, 110, 31, 81, 22, 55, 33, 78, 25, 17, 43, 95, 86, 94, 97, 20, 72, 35, 84, 5, 91, 109, 7, 30, 41, 69, 11, 100, 71, 107, 45, 105, 28, 32, 75, 96, 92, 101, 37], [63, 59, 62, 42, 106, 44, 111, 127, 58, 124, 61, 121, 117, 108, 126, 56, 122, 93, 125, 50, 0, 64, 102, 118, 15, 112, 79, 48, 23, 90, 19, 13, 83, 116, 120, 103, 38, 77, 115, 39, 12, 76, 47, 113, 87, 49, 85, 53, 88, 74, 36, 73, 119, 60, 98, 24, 26, 21, 51, 52, 10, 123, 9, 4, 104, 68, 70, 40, 34, 55, 1, 110, 54, 57, 29, 27, 16, 46, 81, 78, 65, 8, 82, 67, 80, 2, 18, 66, 14, 17, 99, 6, 43, 94, 114, 22, 31, 35, 84, 3, 89, 95, 97, 33, 20, 91, 86, 25, 69, 5, 100, 41, 45, 28, 105, 96, 107, 72, 109, 11, 71, 37, 32, 7, 101, 92, 75, 30], [63, 59, 42, 106, 62, 44, 111, 127, 58, 121, 61, 124, 108, 117, 126, 56, 122, 64, 93, 79, 0, 15, 125, 50, 102, 13, 118, 60, 77, 116, 53, 12, 112, 23, 21, 38, 83, 74, 115, 10, 36, 119, 73, 85, 76, 103, 19, 70, 24, 87, 52, 98, 49, 26, 88, 114, 9, 113, 68, 8, 104, 51, 90, 34, 54, 47, 1, 120, 123, 4, 39, 80, 18, 65, 48, 2, 81, 16, 27, 29, 82, 66, 55, 57, 110, 78, 46, 14, 40, 94, 99, 3, 17, 6, 67, 43, 97, 22, 31, 89, 33, 20, 86, 25, 69, 84, 100, 45, 72, 75, 95, 35, 96, 11, 7, 107, 71, 109, 5, 32, 91, 37, 92, 30, 28, 105, 41, 101], [63, 59, 42, 62, 106, 44, 111, 58, 127, 124, 121, 108, 61, 117, 56, 126, 64, 122, 125, 15, 0, 93, 79, 102, 53, 115, 118, 23, 13, 77, 50, 10, 119, 104, 12, 73, 60, 98, 83, 19, 113, 103, 112, 85, 9, 24, 38, 70, 21, 90, 52, 36, 116, 8, 74, 26, 87, 49, 76, 120, 1, 48, 88, 39, 54, 46, 68, 51, 34, 123, 27, 47, 2, 65, 18, 57, 4, 66, 114, 16, 17, 67, 55, 81, 82, 80, 99, 31, 110, 3, 78, 29, 40, 97, 6, 14, 22, 86, 84, 89, 94, 33, 20, 25, 35, 69, 43, 100, 71, 7, 45, 109, 107, 72, 75, 32, 105, 5, 96, 95, 101, 30, 91, 41, 11, 28, 37, 92], [63, 59, 62, 42, 106, 44, 111, 127, 58, 124, 61, 117, 108, 121, 56, 126, 122, 93, 125, 15, 79, 50, 0, 118, 102, 13, 85, 12, 23, 116, 64, 77, 112, 113, 119, 19, 38, 83, 60, 76, 90, 26, 21, 52, 24, 49, 104, 87, 10, 73, 120, 4, 115, 88, 9, 98, 53, 103, 39, 70, 48, 123, 74, 34, 68, 8, 47, 27, 36, 81, 55, 54, 57, 40, 80, 51, 46, 1, 3, 18, 78, 16, 2, 67, 110, 114, 22, 65, 82, 29, 17, 66, 99, 14, 6, 20, 94, 31, 97, 25, 89, 33, 86, 84, 69, 43, 100, 5, 35, 95, 107, 75, 72, 41, 109, 32, 105, 71, 11, 28, 7, 30, 45, 92, 96, 91, 101, 37], [63, 59, 42, 62, 106, 44, 111, 127, 58, 124, 61, 121, 108, 117, 126, 56, 0, 122, 64, 93, 125, 50, 15, 118, 79, 102, 19, 13, 60, 98, 77, 112, 119, 36, 85, 53, 23, 116, 12, 113, 10, 120, 76, 26, 74, 83, 104, 73, 90, 103, 49, 9, 88, 24, 38, 52, 4, 21, 87, 115, 47, 48, 34, 123, 68, 70, 65, 8, 39, 1, 54, 27, 66, 18, 80, 29, 2, 51, 6, 40, 46, 55, 3, 82, 110, 81, 78, 114, 14, 57, 99, 31, 16, 89, 94, 67, 33, 17, 43, 97, 25, 22, 72, 86, 20, 35, 95, 100, 5, 84, 109, 32, 69, 28, 11, 75, 105, 45, 91, 71, 107, 92, 41, 7, 30, 96, 37, 101], [63, 59, 62, 106, 42, 44, 111, 58, 127, 124, 61, 121, 108, 117, 126, 56, 15, 122, 93, 50, 125, 79, 64, 13, 102, 19, 60, 112, 118, 12, 0, 77, 23, 74, 52, 116, 76, 98, 83, 87, 104, 10, 21, 53, 90, 103, 115, 119, 73, 85, 38, 9, 24, 26, 36, 113, 48, 123, 49, 120, 34, 54, 88, 39, 47, 8, 6, 29, 18, 27, 55, 70, 4, 82, 46, 68, 16, 65, 78, 1, 81, 3, 66, 80, 51, 57, 14, 110, 17, 22, 31, 94, 2, 99, 67, 40, 114, 89, 20, 25, 97, 72, 35, 95, 91, 84, 33, 86, 11, 32, 69, 28, 100, 43, 109, 107, 5, 7, 71, 75, 45, 105, 96, 30, 41, 92, 101, 37]], "model.layers.26.self_attn.q_proj": [[107, 43, 51, 99, 36, 100, 127, 53, 57, 52, 105, 116, 54, 58, 126, 60, 114, 48, 122, 62, 123, 118, 119, 32, 115, 55, 112, 124, 46, 120, 49, 125, 28, 44, 117, 106, 40, 113, 59, 56, 30, 35, 110, 63, 111, 42, 109, 103, 61, 39, 24, 47, 37, 108, 50, 41, 95, 121, 102, 104, 45, 96, 38, 93, 98, 23, 81, 92, 33, 76, 34, 29, 101, 84, 97, 85, 86, 88, 21, 31, 67, 91, 82, 25, 19, 22, 5, 94, 27, 89, 20, 26, 9, 90, 73, 77, 66, 83, 70, 18, 72, 87, 80, 79, 8, 78, 12, 17, 14, 15, 16, 3, 74, 75, 11, 6, 13, 0, 4, 69, 65, 7, 71, 1, 68, 10, 2, 64], [107, 43, 35, 65, 13, 74, 80, 7, 68, 28, 22, 32, 18, 126, 99, 127, 69, 3, 2, 57, 0, 119, 100, 52, 64, 118, 12, 122, 86, 72, 11, 114, 124, 54, 115, 53, 46, 9, 56, 62, 112, 67, 58, 1, 123, 117, 51, 108, 113, 34, 63, 49, 121, 116, 20, 6, 111, 47, 120, 84, 125, 110, 92, 61, 50, 10, 4, 83, 55, 78, 59, 70, 60, 71, 21, 16, 109, 5, 44, 31, 19, 66, 30, 17, 90, 45, 77, 76, 87, 48, 8, 85, 105, 79, 15, 41, 93, 73, 82, 81, 25, 23, 91, 75, 14, 26, 89, 101, 95, 42, 38, 104, 103, 27, 102, 24, 40, 106, 33, 37, 36, 97, 88, 29, 98, 94, 96, 39], [107, 43, 105, 32, 122, 100, 116, 37, 53, 103, 28, 124, 35, 92, 127, 57, 56, 21, 51, 39, 30, 52, 126, 49, 115, 117, 22, 58, 118, 112, 80, 62, 104, 108, 24, 55, 123, 41, 44, 109, 29, 91, 20, 54, 45, 18, 89, 23, 61, 17, 119, 75, 99, 46, 111, 34, 113, 114, 120, 27, 125, 63, 110, 121, 47, 42, 60, 19, 93, 48, 59, 72, 101, 50, 106, 38, 26, 25, 98, 95, 96, 40, 31, 84, 78, 83, 94, 33, 12, 90, 87, 79, 97, 85, 16, 13, 36, 11, 76, 82, 9, 70, 102, 15, 88, 77, 66, 81, 14, 73, 71, 86, 69, 8, 10, 4, 0, 6, 67, 7, 5, 74, 68, 3, 65, 1, 2, 64], [107, 126, 43, 127, 114, 52, 57, 37, 119, 103, 110, 99, 39, 122, 123, 46, 105, 54, 111, 118, 62, 28, 115, 55, 113, 124, 58, 84, 35, 44, 117, 120, 49, 19, 100, 112, 53, 63, 47, 9, 125, 116, 60, 48, 56, 51, 109, 61, 108, 59, 32, 76, 121, 104, 33, 50, 41, 45, 101, 36, 23, 80, 30, 93, 20, 89, 38, 22, 83, 42, 106, 40, 21, 91, 102, 67, 34, 87, 17, 97, 98, 29, 95, 11, 26, 78, 96, 25, 94, 92, 18, 85, 2, 15, 88, 86, 0, 13, 31, 65, 7, 4, 24, 6, 14, 10, 12, 90, 16, 79, 27, 74, 70, 77, 81, 5, 73, 82, 72, 8, 66, 1, 69, 68, 75, 71, 3, 64], [110, 55, 36, 46, 115, 95, 121, 26, 60, 50, 123, 61, 54, 53, 124, 100, 22, 119, 45, 58, 88, 31, 30, 116, 42, 127, 51, 40, 114, 106, 59, 92, 24, 47, 113, 44, 34, 117, 112, 105, 56, 28, 37, 41, 109, 63, 107, 99, 19, 122, 48, 52, 125, 126, 111, 39, 118, 49, 57, 43, 38, 33, 85, 93, 104, 120, 32, 17, 27, 108, 35, 25, 62, 87, 103, 101, 86, 21, 18, 102, 91, 29, 97, 98, 96, 16, 15, 94, 20, 23, 77, 90, 89, 14, 84, 75, 81, 76, 12, 80, 13, 83, 82, 74, 79, 9, 71, 78, 10, 5, 72, 7, 67, 6, 70, 73, 3, 68, 4, 2, 1, 11, 66, 8, 65, 64, 0, 69], [110, 46, 36, 26, 95, 22, 55, 19, 115, 4, 54, 30, 72, 31, 14, 100, 17, 58, 41, 62, 119, 2, 77, 74, 65, 121, 5, 63, 106, 37, 28, 125, 61, 97, 111, 91, 60, 10, 90, 107, 44, 94, 18, 73, 86, 52, 93, 32, 88, 104, 21, 84, 24, 118, 7, 33, 13, 34, 71, 45, 92, 39, 123, 11, 98, 75, 23, 6, 29, 101, 112, 113, 27, 116, 35, 117, 103, 127, 59, 42, 96, 25, 51, 89, 1, 15, 50, 81, 108, 85, 105, 79, 82, 80, 49, 69, 126, 76, 114, 12, 102, 20, 120, 87, 53, 99, 48, 9, 64, 56, 38, 57, 124, 47, 122, 67, 40, 68, 43, 109, 83, 3, 78, 70, 66, 8, 16, 0], [110, 36, 115, 62, 46, 26, 95, 30, 55, 88, 119, 22, 58, 100, 60, 31, 54, 51, 19, 124, 50, 96, 63, 125, 117, 34, 40, 86, 42, 123, 97, 41, 24, 122, 45, 44, 48, 87, 38, 126, 17, 116, 121, 108, 21, 16, 14, 59, 28, 61, 83, 49, 57, 120, 103, 52, 118, 33, 53, 127, 27, 106, 35, 111, 43, 114, 113, 107, 81, 56, 39, 89, 12, 93, 90, 91, 102, 47, 105, 94, 104, 109, 29, 112, 15, 98, 99, 32, 92, 101, 18, 79, 37, 80, 10, 82, 74, 25, 85, 75, 77, 20, 23, 7, 76, 8, 84, 72, 11, 78, 73, 13, 4, 9, 70, 5, 6, 69, 2, 67, 71, 66, 1, 0, 64, 65, 3, 68], [110, 36, 46, 54, 58, 121, 123, 50, 95, 26, 108, 62, 60, 119, 55, 30, 88, 31, 48, 100, 22, 49, 47, 24, 59, 33, 57, 114, 115, 109, 125, 113, 63, 56, 28, 17, 19, 112, 52, 106, 41, 117, 92, 122, 45, 51, 18, 21, 116, 15, 37, 42, 104, 40, 44, 61, 103, 111, 127, 126, 53, 107, 120, 86, 77, 118, 105, 14, 32, 124, 85, 39, 93, 99, 25, 101, 102, 43, 98, 38, 75, 13, 35, 20, 29, 16, 97, 90, 12, 34, 27, 83, 96, 89, 72, 94, 91, 87, 74, 84, 81, 23, 80, 7, 6, 73, 76, 10, 67, 5, 79, 1, 9, 4, 82, 2, 71, 64, 78, 8, 11, 65, 69, 66, 0, 70, 68, 3], [123, 57, 101, 33, 50, 37, 121, 88, 51, 44, 63, 92, 118, 24, 115, 62, 28, 60, 108, 23, 21, 54, 85, 116, 103, 117, 127, 97, 31, 105, 41, 61, 52, 56, 26, 114, 91, 109, 47, 82, 45, 110, 124, 49, 104, 126, 58, 125, 59, 34, 119, 112, 90, 55, 93, 48, 107, 111, 113, 18, 96, 122, 46, 106, 43, 87, 53, 42, 38, 79, 32, 40, 98, 20, 95, 83, 19, 75, 36, 100, 39, 15, 102, 30, 77, 120, 35, 89, 27, 94, 29, 99, 76, 16, 10, 17, 71, 86, 65, 25, 22, 84, 80, 67, 70, 14, 73, 69, 78, 81, 5, 68, 72, 8, 13, 12, 66, 0, 4, 1, 74, 2, 3, 9, 6, 64, 7, 11], [57, 101, 123, 33, 50, 37, 121, 24, 51, 124, 85, 103, 15, 88, 97, 76, 90, 20, 115, 82, 54, 105, 60, 10, 118, 29, 21, 34, 93, 104, 19, 96, 18, 91, 119, 95, 26, 87, 44, 117, 46, 67, 89, 49, 77, 59, 25, 79, 65, 28, 40, 62, 112, 73, 31, 99, 75, 30, 72, 12, 92, 74, 13, 9, 107, 43, 8, 7, 5, 83, 35, 102, 63, 127, 38, 94, 108, 114, 69, 4, 32, 2, 27, 125, 1, 52, 70, 22, 81, 71, 61, 109, 106, 53, 17, 68, 47, 126, 3, 6, 36, 23, 111, 39, 98, 116, 48, 16, 64, 100, 84, 120, 14, 110, 45, 41, 42, 58, 55, 122, 0, 113, 66, 78, 56, 80, 11, 86], [101, 123, 57, 50, 124, 33, 37, 121, 88, 28, 119, 21, 92, 61, 51, 90, 24, 44, 54, 85, 118, 126, 19, 58, 56, 105, 62, 114, 31, 117, 18, 16, 60, 115, 49, 103, 26, 82, 46, 52, 106, 97, 45, 116, 75, 38, 59, 14, 77, 42, 73, 47, 122, 86, 112, 91, 48, 109, 35, 107, 43, 63, 53, 76, 55, 39, 125, 94, 30, 110, 41, 113, 111, 89, 36, 79, 120, 80, 83, 127, 34, 93, 22, 95, 104, 99, 102, 17, 96, 29, 98, 108, 40, 71, 10, 100, 32, 20, 84, 27, 72, 81, 78, 5, 87, 23, 67, 25, 70, 65, 68, 13, 66, 15, 64, 4, 69, 11, 74, 9, 12, 2, 8, 7, 3, 1, 6, 0], [101, 123, 57, 50, 33, 121, 37, 51, 88, 24, 19, 115, 61, 54, 92, 28, 85, 21, 124, 75, 118, 105, 73, 117, 119, 77, 114, 60, 59, 44, 113, 18, 100, 46, 97, 62, 14, 83, 31, 90, 17, 98, 93, 58, 16, 95, 79, 43, 107, 78, 126, 104, 52, 45, 111, 48, 38, 127, 110, 125, 82, 41, 109, 47, 35, 116, 122, 103, 71, 29, 84, 112, 86, 89, 55, 30, 56, 42, 53, 76, 26, 36, 34, 120, 106, 5, 49, 87, 99, 63, 96, 40, 108, 102, 94, 32, 20, 4, 22, 39, 27, 11, 23, 91, 15, 80, 72, 10, 70, 66, 25, 67, 12, 13, 81, 68, 65, 6, 74, 7, 3, 1, 0, 9, 8, 69, 64, 2], [61, 59, 101, 63, 44, 125, 116, 51, 21, 62, 33, 58, 93, 118, 113, 55, 85, 88, 119, 127, 114, 110, 49, 56, 42, 54, 30, 123, 126, 124, 115, 52, 45, 50, 53, 40, 120, 112, 117, 57, 27, 48, 103, 78, 121, 122, 109, 60, 83, 24, 92, 105, 111, 37, 47, 80, 43, 90, 107, 91, 46, 26, 106, 89, 108, 95, 104, 41, 22, 82, 81, 25, 39, 74, 20, 11, 99, 75, 7, 13, 29, 12, 84, 77, 102, 32, 38, 9, 36, 6, 19, 10, 73, 17, 96, 100, 98, 34, 94, 8, 79, 23, 14, 69, 3, 15, 35, 16, 70, 28, 86, 31, 87, 18, 68, 72, 5, 76, 97, 67, 65, 71, 4, 0, 66, 1, 2, 64], [125, 63, 61, 101, 59, 116, 44, 51, 55, 58, 21, 50, 49, 127, 85, 62, 119, 114, 56, 110, 33, 113, 53, 126, 118, 52, 54, 124, 123, 57, 115, 48, 117, 42, 88, 45, 40, 120, 93, 37, 121, 109, 60, 83, 24, 112, 122, 30, 111, 27, 91, 47, 90, 46, 43, 92, 78, 105, 26, 107, 106, 41, 108, 103, 104, 89, 80, 39, 29, 22, 34, 38, 99, 96, 25, 102, 100, 82, 95, 17, 77, 81, 94, 7, 98, 36, 73, 32, 20, 75, 35, 28, 87, 31, 97, 11, 10, 13, 19, 79, 86, 6, 8, 74, 84, 12, 9, 23, 70, 14, 16, 67, 69, 72, 0, 4, 68, 18, 76, 15, 66, 3, 71, 5, 64, 1, 65, 2], [63, 101, 59, 61, 44, 21, 51, 33, 115, 85, 118, 127, 62, 113, 58, 93, 114, 119, 88, 49, 55, 126, 56, 116, 30, 54, 123, 110, 45, 117, 42, 53, 124, 50, 125, 103, 52, 40, 92, 120, 37, 57, 91, 48, 27, 112, 121, 83, 80, 109, 60, 78, 122, 111, 105, 47, 24, 43, 108, 46, 90, 26, 106, 107, 41, 104, 29, 7, 74, 82, 22, 95, 39, 89, 12, 20, 77, 25, 99, 81, 75, 19, 32, 13, 84, 14, 102, 17, 11, 38, 10, 73, 36, 15, 79, 16, 8, 100, 23, 34, 9, 94, 96, 98, 4, 18, 86, 35, 28, 69, 76, 72, 6, 87, 68, 5, 31, 97, 71, 70, 0, 1, 66, 67, 3, 64, 2, 65], [125, 59, 61, 63, 101, 62, 115, 54, 55, 116, 56, 33, 58, 44, 120, 126, 119, 114, 50, 110, 51, 124, 127, 118, 88, 53, 123, 37, 48, 49, 57, 117, 113, 112, 40, 52, 121, 45, 111, 60, 122, 90, 109, 93, 83, 42, 47, 103, 46, 43, 24, 108, 106, 81, 107, 41, 104, 78, 85, 105, 22, 7, 32, 39, 25, 89, 19, 80, 30, 74, 73, 21, 34, 6, 17, 27, 29, 91, 11, 100, 102, 26, 69, 36, 38, 77, 79, 75, 15, 13, 95, 82, 86, 12, 16, 14, 28, 97, 35, 98, 10, 9, 99, 87, 84, 8, 96, 92, 31, 94, 20, 4, 76, 68, 70, 5, 2, 23, 1, 0, 18, 67, 71, 66, 72, 65, 3, 64], [41, 117, 55, 34, 52, 31, 23, 115, 119, 21, 62, 39, 80, 58, 83, 45, 27, 105, 60, 99, 59, 44, 63, 13, 113, 30, 61, 72, 103, 57, 11, 109, 42, 29, 74, 32, 93, 94, 126, 111, 33, 88, 89, 8, 108, 112, 104, 121, 92, 116, 56, 118, 81, 90, 100, 54, 75, 26, 43, 17, 47, 37, 107, 25, 87, 20, 85, 24, 101, 51, 18, 91, 48, 15, 36, 22, 95, 124, 76, 127, 106, 110, 123, 71, 82, 50, 97, 49, 35, 38, 122, 114, 86, 84, 12, 19, 79, 40, 69, 96, 14, 46, 102, 28, 120, 53, 125, 78, 16, 67, 5, 10, 77, 64, 9, 66, 6, 7, 68, 98, 73, 4, 3, 70, 2, 65, 0, 1], [41, 55, 34, 52, 31, 13, 21, 23, 83, 80, 117, 27, 11, 66, 115, 105, 6, 96, 109, 28, 74, 19, 8, 49, 108, 44, 60, 73, 26, 64, 63, 67, 37, 30, 72, 119, 62, 4, 91, 45, 33, 85, 18, 22, 15, 7, 89, 43, 25, 17, 56, 87, 94, 9, 78, 84, 113, 122, 99, 77, 59, 93, 5, 61, 88, 57, 118, 81, 32, 48, 97, 70, 102, 10, 46, 42, 16, 29, 120, 92, 35, 71, 24, 82, 36, 103, 65, 69, 116, 2, 101, 20, 106, 40, 12, 14, 76, 90, 39, 68, 54, 127, 1, 53, 124, 123, 79, 75, 100, 110, 104, 111, 125, 86, 121, 47, 58, 3, 112, 38, 126, 50, 95, 0, 51, 114, 107, 98], [55, 41, 59, 34, 117, 45, 31, 23, 112, 54, 105, 119, 120, 110, 44, 111, 56, 46, 61, 92, 47, 116, 115, 58, 108, 42, 28, 118, 48, 124, 63, 121, 122, 123, 51, 114, 39, 127, 126, 109, 60, 25, 43, 125, 53, 57, 107, 94, 113, 21, 62, 50, 36, 106, 40, 20, 30, 49, 83, 97, 26, 103, 88, 102, 104, 100, 38, 33, 79, 52, 37, 99, 93, 29, 101, 80, 15, 35, 11, 91, 95, 86, 66, 96, 32, 71, 24, 27, 98, 69, 10, 7, 81, 87, 3, 74, 85, 17, 84, 14, 12, 89, 90, 22, 5, 1, 76, 70, 65, 82, 13, 64, 67, 0, 9, 8, 78, 19, 18, 68, 2, 4, 6, 73, 72, 75, 16, 77], [41, 55, 34, 52, 23, 31, 21, 13, 83, 80, 82, 74, 40, 117, 119, 125, 4, 103, 49, 28, 27, 105, 59, 60, 17, 115, 64, 61, 43, 44, 6, 101, 113, 66, 26, 96, 47, 118, 57, 56, 45, 94, 42, 62, 109, 3, 25, 116, 39, 70, 100, 65, 111, 112, 11, 53, 97, 72, 54, 108, 36, 126, 0, 85, 33, 104, 99, 29, 63, 123, 110, 18, 91, 16, 50, 38, 127, 32, 124, 68, 93, 87, 88, 92, 106, 122, 48, 102, 46, 22, 37, 84, 95, 58, 9, 89, 107, 69, 114, 35, 81, 78, 121, 30, 51, 10, 120, 79, 73, 19, 14, 15, 24, 7, 8, 90, 1, 86, 71, 98, 5, 12, 75, 20, 77, 76, 2, 67], [45, 103, 109, 38, 29, 54, 126, 87, 119, 57, 107, 83, 84, 114, 115, 117, 93, 118, 111, 47, 80, 122, 50, 95, 31, 46, 22, 121, 113, 127, 78, 61, 13, 108, 63, 85, 44, 10, 124, 53, 26, 52, 112, 98, 17, 59, 62, 49, 56, 125, 41, 116, 123, 51, 32, 71, 55, 58, 120, 110, 104, 30, 60, 96, 86, 36, 21, 42, 12, 48, 102, 37, 24, 34, 40, 100, 28, 106, 2, 88, 33, 35, 43, 89, 94, 70, 72, 101, 9, 14, 91, 92, 97, 90, 27, 69, 23, 76, 81, 105, 3, 6, 25, 75, 99, 19, 8, 0, 11, 67, 82, 68, 18, 4, 79, 20, 15, 73, 66, 7, 39, 65, 5, 1, 16, 64, 77, 74], [103, 45, 29, 109, 87, 80, 84, 13, 54, 10, 71, 69, 113, 32, 23, 115, 65, 47, 67, 126, 93, 64, 118, 3, 12, 127, 60, 77, 125, 38, 94, 17, 30, 62, 31, 72, 95, 57, 49, 59, 88, 20, 39, 66, 89, 68, 14, 119, 16, 83, 27, 24, 120, 74, 21, 102, 101, 79, 111, 34, 22, 18, 96, 7, 82, 15, 98, 85, 58, 90, 100, 121, 107, 51, 81, 76, 36, 5, 37, 42, 78, 25, 11, 48, 56, 63, 97, 46, 117, 91, 33, 112, 116, 4, 55, 104, 19, 99, 92, 73, 86, 41, 28, 50, 53, 6, 110, 105, 108, 52, 124, 35, 40, 61, 44, 122, 8, 70, 43, 123, 9, 106, 26, 114, 2, 0, 75, 1], [103, 45, 29, 109, 84, 87, 38, 13, 80, 54, 10, 69, 93, 71, 12, 83, 113, 3, 126, 2, 47, 32, 78, 127, 82, 79, 118, 65, 1, 115, 23, 22, 119, 88, 59, 64, 6, 95, 0, 46, 60, 89, 77, 123, 111, 31, 37, 49, 8, 62, 30, 66, 57, 122, 24, 5, 70, 97, 72, 33, 81, 125, 42, 15, 16, 58, 68, 43, 52, 7, 25, 124, 14, 74, 67, 96, 50, 112, 121, 20, 106, 85, 98, 107, 41, 92, 9, 34, 117, 17, 53, 51, 61, 120, 63, 48, 26, 114, 40, 91, 18, 110, 36, 55, 11, 101, 76, 19, 104, 99, 116, 108, 4, 35, 27, 94, 56, 100, 105, 86, 90, 44, 73, 21, 102, 28, 39, 75], [103, 45, 29, 109, 87, 84, 54, 80, 13, 10, 38, 126, 69, 71, 113, 93, 115, 32, 65, 47, 82, 67, 23, 118, 127, 30, 64, 95, 88, 40, 77, 56, 60, 125, 21, 102, 24, 94, 72, 79, 31, 46, 41, 108, 49, 83, 58, 20, 11, 101, 121, 78, 57, 25, 76, 66, 22, 26, 53, 52, 75, 117, 50, 17, 3, 119, 114, 44, 74, 42, 0, 16, 34, 9, 91, 97, 100, 48, 14, 4, 63, 62, 18, 59, 111, 124, 7, 15, 19, 5, 61, 96, 89, 122, 36, 37, 43, 112, 106, 8, 28, 55, 33, 90, 85, 104, 51, 105, 99, 81, 35, 107, 68, 110, 98, 86, 123, 12, 116, 27, 2, 39, 70, 73, 92, 120, 6, 1], [104, 59, 99, 92, 20, 114, 119, 54, 62, 82, 117, 55, 95, 98, 51, 107, 34, 63, 47, 120, 25, 125, 124, 49, 113, 115, 60, 86, 45, 13, 30, 112, 16, 123, 28, 103, 33, 58, 108, 121, 50, 48, 42, 56, 18, 44, 89, 61, 31, 39, 110, 87, 127, 52, 106, 116, 46, 122, 126, 88, 53, 57, 111, 105, 37, 118, 109, 41, 40, 43, 78, 29, 101, 38, 97, 93, 11, 94, 17, 22, 24, 76, 10, 80, 67, 15, 84, 36, 100, 71, 90, 21, 102, 79, 77, 26, 73, 96, 68, 85, 75, 23, 8, 91, 27, 19, 1, 81, 32, 35, 74, 14, 70, 7, 5, 83, 9, 66, 12, 0, 4, 65, 6, 69, 64, 72, 3, 2], [104, 59, 99, 92, 114, 82, 20, 51, 98, 62, 115, 95, 63, 105, 54, 47, 13, 16, 61, 117, 30, 34, 25, 119, 58, 122, 124, 45, 60, 109, 116, 86, 67, 55, 89, 87, 102, 66, 28, 11, 56, 80, 127, 43, 50, 88, 113, 15, 65, 123, 46, 57, 10, 18, 64, 53, 70, 42, 101, 7, 49, 48, 9, 31, 17, 97, 103, 8, 106, 44, 112, 78, 73, 40, 111, 68, 69, 22, 12, 94, 0, 5, 37, 121, 120, 76, 41, 125, 74, 33, 39, 84, 93, 29, 79, 38, 21, 118, 126, 24, 108, 75, 91, 32, 110, 107, 26, 85, 52, 71, 19, 90, 36, 83, 6, 23, 77, 81, 1, 100, 27, 14, 35, 96, 72, 4, 3, 2], [104, 59, 99, 92, 114, 20, 82, 16, 51, 116, 13, 25, 95, 87, 89, 54, 10, 115, 124, 34, 17, 8, 56, 50, 45, 53, 21, 100, 78, 76, 94, 86, 31, 63, 68, 62, 61, 18, 66, 70, 67, 65, 71, 123, 73, 80, 125, 98, 74, 85, 41, 52, 44, 12, 101, 97, 47, 39, 28, 40, 88, 91, 55, 29, 9, 127, 103, 15, 107, 118, 119, 108, 96, 11, 69, 27, 122, 37, 49, 90, 33, 84, 19, 7, 30, 5, 24, 106, 42, 121, 126, 32, 22, 58, 113, 110, 48, 105, 77, 112, 36, 6, 111, 79, 93, 23, 120, 43, 64, 109, 38, 75, 102, 26, 0, 60, 117, 46, 57, 14, 1, 83, 81, 35, 2, 4, 72, 3], [104, 59, 122, 45, 92, 20, 95, 114, 25, 99, 86, 115, 54, 62, 82, 63, 61, 34, 116, 51, 98, 49, 11, 56, 43, 0, 127, 124, 123, 33, 4, 119, 53, 1, 28, 110, 2, 46, 41, 58, 113, 31, 125, 55, 89, 120, 52, 70, 48, 111, 126, 76, 73, 50, 108, 118, 39, 121, 71, 112, 88, 78, 5, 13, 42, 106, 117, 109, 60, 83, 57, 44, 107, 105, 30, 22, 19, 87, 90, 47, 26, 102, 103, 84, 101, 16, 14, 38, 36, 21, 93, 3, 24, 100, 32, 37, 94, 6, 91, 81, 97, 35, 96, 15, 18, 8, 17, 29, 27, 65, 40, 12, 85, 10, 23, 69, 7, 66, 74, 75, 80, 64, 79, 9, 67, 68, 77, 72], [107, 56, 126, 43, 99, 87, 82, 110, 91, 78, 20, 25, 81, 40, 75, 104, 79, 27, 12, 112, 7, 72, 52, 31, 86, 32, 89, 117, 118, 49, 116, 92, 10, 125, 62, 48, 55, 16, 88, 127, 109, 95, 120, 26, 28, 85, 121, 69, 17, 93, 23, 44, 113, 74, 37, 77, 6, 57, 14, 53, 122, 84, 97, 58, 9, 80, 115, 61, 54, 11, 50, 101, 90, 102, 29, 76, 15, 108, 30, 51, 46, 68, 124, 18, 21, 83, 60, 114, 119, 105, 19, 96, 47, 33, 45, 24, 123, 38, 98, 59, 41, 100, 5, 103, 111, 94, 36, 63, 22, 2, 34, 106, 67, 8, 73, 71, 42, 39, 35, 70, 13, 4, 64, 3, 1, 65, 66, 0], [107, 43, 112, 56, 53, 127, 48, 115, 50, 116, 55, 125, 117, 25, 99, 19, 52, 120, 126, 61, 37, 118, 46, 121, 57, 62, 93, 113, 63, 58, 124, 29, 51, 114, 109, 122, 59, 54, 60, 49, 110, 91, 45, 119, 89, 31, 123, 36, 105, 47, 104, 111, 33, 44, 40, 86, 108, 24, 106, 42, 27, 28, 41, 95, 88, 38, 80, 39, 35, 101, 100, 83, 34, 102, 22, 20, 16, 103, 13, 98, 21, 77, 87, 97, 92, 90, 30, 17, 26, 79, 82, 32, 23, 70, 14, 96, 94, 81, 4, 10, 65, 85, 7, 11, 68, 6, 3, 1, 67, 84, 71, 15, 78, 18, 74, 5, 69, 75, 2, 72, 64, 66, 0, 8, 12, 9, 76, 73], [107, 126, 56, 99, 43, 117, 109, 25, 79, 86, 82, 31, 93, 91, 84, 89, 118, 116, 18, 33, 4, 110, 0, 73, 20, 77, 53, 28, 66, 78, 9, 55, 96, 41, 105, 1, 75, 98, 39, 112, 37, 81, 120, 121, 125, 72, 127, 36, 32, 17, 26, 30, 52, 95, 113, 54, 64, 80, 29, 3, 97, 92, 42, 6, 27, 76, 60, 50, 101, 58, 61, 94, 48, 40, 22, 114, 49, 63, 62, 57, 51, 44, 46, 122, 19, 23, 5, 14, 104, 38, 24, 15, 87, 90, 123, 69, 16, 67, 2, 34, 100, 59, 45, 70, 12, 10, 103, 21, 88, 102, 13, 7, 74, 119, 108, 124, 47, 83, 85, 111, 11, 68, 8, 35, 71, 65, 106, 115], [56, 107, 112, 99, 125, 19, 50, 126, 116, 43, 127, 25, 52, 117, 62, 121, 120, 48, 53, 45, 31, 124, 93, 57, 113, 40, 61, 109, 55, 46, 60, 51, 122, 49, 118, 63, 114, 54, 59, 29, 58, 123, 115, 119, 105, 47, 111, 36, 104, 89, 108, 110, 33, 35, 44, 91, 24, 86, 106, 27, 41, 42, 39, 38, 98, 88, 101, 37, 100, 83, 95, 103, 102, 28, 21, 80, 22, 92, 34, 90, 13, 74, 77, 17, 23, 30, 97, 87, 32, 26, 94, 81, 16, 96, 75, 10, 7, 71, 14, 20, 78, 65, 82, 70, 6, 1, 3, 79, 69, 4, 67, 72, 85, 18, 68, 11, 2, 5, 8, 66, 84, 15, 0, 64, 12, 76, 9, 73]], "model.layers.26.self_attn.k_proj": [[43, 22, 107, 99, 92, 126, 127, 96, 64, 47, 46, 56, 48, 80, 119, 124, 68, 7, 122, 13, 116, 57, 49, 109, 54, 113, 18, 2, 117, 112, 63, 118, 44, 62, 115, 55, 52, 58, 123, 50, 125, 53, 120, 51, 59, 121, 60, 110, 108, 114, 61, 101, 74, 98, 111, 11, 106, 42, 45, 76, 41, 102, 3, 37, 78, 36, 6, 40, 105, 103, 34, 104, 1, 97, 69, 39, 38, 10, 70, 8, 21, 89, 72, 15, 5, 94, 31, 90, 84, 33, 95, 87, 30, 9, 17, 100, 23, 24, 82, 28, 29, 26, 88, 91, 32, 81, 12, 25, 93, 27, 73, 85, 75, 67, 19, 20, 79, 71, 35, 83, 14, 16, 66, 77, 4, 65, 0, 86], [46, 110, 100, 31, 26, 22, 19, 86, 17, 14, 94, 12, 74, 28, 58, 121, 60, 24, 77, 15, 72, 62, 104, 123, 54, 29, 118, 43, 10, 106, 122, 52, 112, 32, 37, 61, 108, 68, 83, 49, 67, 7, 16, 40, 63, 125, 5, 44, 119, 127, 116, 97, 98, 33, 38, 113, 50, 27, 59, 89, 57, 41, 42, 111, 115, 109, 102, 126, 93, 91, 81, 56, 34, 101, 53, 69, 55, 70, 124, 103, 105, 84, 99, 48, 18, 117, 107, 71, 39, 51, 1, 95, 35, 23, 114, 47, 76, 120, 96, 87, 36, 65, 45, 0, 21, 64, 30, 88, 92, 25, 20, 85, 9, 13, 82, 8, 79, 75, 90, 6, 66, 80, 4, 3, 11, 73, 78, 2], [123, 57, 97, 37, 24, 95, 92, 85, 82, 90, 80, 108, 115, 124, 73, 75, 19, 50, 114, 104, 76, 40, 78, 77, 83, 101, 53, 59, 121, 39, 35, 70, 119, 10, 0, 88, 105, 14, 55, 111, 71, 5, 48, 45, 126, 61, 4, 103, 30, 93, 109, 52, 21, 43, 1, 113, 116, 3, 112, 49, 42, 72, 54, 62, 96, 44, 117, 107, 41, 51, 58, 106, 60, 81, 118, 66, 46, 56, 127, 47, 110, 15, 63, 122, 120, 17, 20, 125, 36, 86, 89, 102, 100, 91, 32, 99, 34, 33, 84, 98, 27, 22, 94, 38, 29, 87, 2, 13, 25, 23, 79, 18, 16, 67, 31, 8, 7, 74, 26, 28, 64, 11, 6, 12, 69, 65, 68, 9], [125, 37, 63, 61, 22, 97, 59, 114, 126, 119, 51, 55, 124, 117, 112, 58, 57, 127, 53, 60, 121, 110, 123, 108, 122, 56, 52, 49, 111, 107, 44, 118, 113, 45, 31, 50, 62, 47, 29, 115, 120, 48, 109, 104, 54, 46, 36, 106, 43, 101, 90, 105, 79, 42, 34, 27, 17, 102, 116, 41, 40, 83, 38, 35, 39, 98, 85, 88, 92, 86, 30, 81, 89, 103, 87, 91, 99, 20, 24, 25, 93, 100, 13, 15, 78, 95, 82, 28, 18, 32, 96, 73, 21, 75, 94, 84, 26, 19, 33, 12, 69, 23, 11, 6, 72, 4, 8, 80, 10, 77, 76, 7, 14, 16, 67, 2, 74, 65, 71, 64, 9, 70, 1, 68, 66, 5, 0, 3], [105, 55, 98, 52, 95, 23, 21, 80, 83, 64, 13, 61, 11, 116, 74, 6, 28, 117, 51, 60, 27, 46, 69, 8, 49, 109, 2, 90, 71, 59, 123, 65, 42, 57, 12, 111, 4, 115, 127, 67, 113, 72, 103, 53, 126, 50, 79, 54, 58, 93, 82, 124, 43, 118, 26, 100, 62, 119, 108, 56, 110, 25, 122, 44, 41, 112, 63, 78, 47, 34, 97, 45, 114, 99, 1, 104, 106, 29, 125, 37, 121, 107, 35, 30, 101, 32, 38, 39, 3, 120, 96, 18, 40, 102, 81, 33, 24, 48, 94, 15, 36, 20, 88, 17, 86, 91, 76, 9, 14, 77, 22, 89, 92, 75, 66, 5, 84, 31, 73, 10, 7, 87, 70, 19, 0, 85, 16, 68], [109, 39, 80, 84, 10, 93, 13, 87, 45, 71, 54, 69, 64, 3, 111, 118, 57, 12, 119, 29, 126, 117, 65, 121, 49, 78, 68, 106, 88, 31, 1, 9, 122, 127, 55, 123, 22, 51, 115, 113, 17, 104, 102, 96, 53, 82, 46, 76, 58, 125, 41, 14, 61, 63, 83, 2, 52, 59, 62, 112, 72, 15, 44, 60, 124, 101, 100, 32, 86, 36, 26, 50, 94, 95, 43, 30, 48, 105, 6, 47, 114, 108, 85, 107, 110, 42, 34, 27, 56, 98, 120, 116, 66, 21, 37, 75, 0, 40, 25, 91, 35, 28, 90, 79, 4, 67, 24, 89, 33, 99, 92, 38, 70, 18, 5, 19, 8, 23, 97, 73, 103, 7, 11, 81, 74, 77, 20, 16], [59, 40, 31, 20, 28, 86, 114, 98, 13, 113, 89, 88, 54, 115, 82, 56, 116, 111, 50, 99, 124, 21, 66, 0, 67, 35, 127, 41, 87, 62, 16, 53, 49, 17, 121, 27, 15, 68, 80, 126, 18, 11, 78, 44, 48, 55, 1, 109, 8, 70, 123, 47, 92, 10, 105, 58, 57, 112, 25, 60, 46, 118, 33, 122, 106, 63, 23, 51, 76, 29, 43, 125, 83, 30, 34, 36, 7, 61, 117, 119, 45, 52, 120, 73, 93, 65, 103, 39, 108, 26, 107, 42, 69, 9, 100, 96, 74, 97, 5, 101, 110, 102, 38, 85, 94, 79, 24, 81, 71, 64, 37, 90, 14, 32, 19, 6, 12, 72, 91, 4, 2, 77, 22, 75, 95, 3, 84, 104], [43, 35, 86, 56, 107, 29, 25, 19, 118, 126, 117, 95, 127, 116, 121, 57, 79, 109, 119, 55, 101, 92, 112, 125, 58, 63, 113, 41, 97, 52, 59, 51, 53, 54, 122, 114, 115, 61, 60, 62, 49, 124, 123, 45, 50, 111, 105, 82, 120, 48, 47, 64, 42, 44, 110, 20, 27, 34, 94, 46, 108, 80, 103, 38, 77, 37, 106, 104, 102, 75, 70, 100, 9, 65, 98, 39, 4, 40, 2, 36, 85, 84, 24, 7, 32, 67, 99, 90, 93, 69, 28, 16, 71, 26, 73, 96, 33, 11, 30, 87, 78, 91, 31, 76, 17, 81, 14, 21, 12, 18, 68, 23, 74, 8, 88, 10, 72, 13, 5, 15, 0, 83, 89, 1, 6, 3, 22, 66]], "model.layers.26.self_attn.qk_proj": [[59, 43, 57, 123, 109, 110, 107, 46, 63, 61, 55, 125, 56, 45, 105, 37, 52, 54, 126, 29, 41, 50, 117, 99, 40, 101, 28, 115, 114, 23, 62, 119, 22, 58, 95, 92, 127, 121, 80, 31, 124, 87, 16, 53, 116, 13, 88, 86, 103, 60, 39, 77, 51, 20, 112, 111, 44, 21, 19, 104, 84, 97, 90, 48, 24, 47, 49, 85, 82, 93, 113, 118, 83, 122, 120, 10, 34, 18, 26, 74, 33, 106, 100, 98, 42, 108, 7, 35, 25, 89, 36, 91, 71, 64, 30, 72, 75, 11, 102, 32, 96, 81, 79, 15, 14, 0, 78, 17, 76, 38, 67, 12, 3, 69, 27, 94, 5, 66, 68, 1, 73, 9, 70, 65, 8, 2, 6, 4], [43, 59, 57, 123, 109, 107, 110, 46, 61, 63, 125, 55, 56, 45, 105, 52, 37, 54, 126, 29, 117, 101, 41, 115, 50, 114, 62, 40, 99, 127, 58, 28, 31, 121, 23, 92, 103, 51, 22, 124, 119, 112, 116, 77, 80, 95, 104, 39, 16, 87, 86, 24, 60, 49, 53, 13, 88, 85, 48, 21, 111, 93, 97, 44, 47, 120, 122, 20, 113, 19, 118, 83, 90, 34, 10, 84, 108, 82, 74, 100, 18, 33, 26, 35, 25, 89, 98, 42, 91, 71, 106, 15, 30, 36, 0, 75, 64, 72, 96, 11, 32, 7, 79, 67, 12, 14, 102, 5, 76, 81, 78, 3, 38, 94, 69, 27, 17, 9, 66, 73, 2, 6, 4, 1, 68, 70, 8, 65], [59, 43, 57, 123, 109, 107, 110, 46, 61, 63, 55, 125, 56, 45, 105, 54, 52, 37, 126, 29, 117, 41, 101, 114, 62, 99, 50, 40, 115, 124, 28, 104, 127, 121, 119, 22, 51, 112, 103, 86, 53, 80, 95, 92, 116, 31, 23, 87, 48, 77, 44, 16, 49, 88, 39, 58, 113, 97, 13, 93, 34, 47, 24, 21, 60, 20, 122, 118, 85, 83, 111, 90, 74, 19, 84, 120, 98, 108, 82, 10, 18, 33, 36, 26, 100, 35, 64, 106, 42, 91, 71, 89, 25, 0, 7, 15, 78, 38, 81, 17, 30, 11, 76, 79, 75, 14, 67, 96, 102, 32, 69, 72, 94, 3, 27, 5, 12, 1, 6, 68, 2, 4, 73, 65, 8, 66, 9, 70], [43, 59, 57, 123, 109, 110, 107, 46, 63, 61, 55, 56, 125, 45, 105, 52, 37, 126, 54, 117, 29, 114, 101, 50, 49, 41, 121, 40, 115, 62, 124, 99, 53, 44, 112, 104, 80, 119, 116, 23, 28, 22, 16, 92, 95, 86, 51, 77, 60, 113, 31, 13, 103, 58, 127, 111, 88, 87, 93, 120, 19, 48, 20, 39, 21, 118, 24, 85, 10, 90, 84, 97, 122, 108, 47, 74, 34, 83, 82, 33, 98, 26, 18, 0, 100, 106, 71, 7, 36, 89, 35, 81, 79, 42, 15, 102, 64, 91, 25, 17, 75, 78, 30, 14, 76, 5, 67, 11, 3, 38, 69, 72, 8, 96, 27, 6, 4, 12, 94, 32, 65, 68, 2, 1, 66, 73, 9, 70], [43, 59, 57, 123, 109, 110, 107, 46, 63, 61, 125, 56, 55, 45, 105, 52, 126, 37, 117, 54, 114, 50, 41, 29, 40, 121, 99, 119, 44, 101, 49, 124, 23, 22, 62, 127, 28, 51, 104, 53, 80, 16, 13, 116, 88, 115, 60, 86, 95, 118, 112, 92, 87, 77, 120, 31, 47, 58, 20, 84, 93, 103, 111, 74, 39, 34, 19, 10, 21, 97, 122, 83, 24, 48, 85, 113, 90, 82, 108, 18, 26, 98, 33, 35, 7, 100, 106, 64, 89, 71, 36, 75, 96, 0, 15, 81, 91, 25, 79, 5, 67, 76, 30, 102, 8, 78, 3, 11, 17, 32, 69, 14, 42, 4, 94, 12, 1, 6, 2, 38, 65, 27, 72, 68, 9, 66, 73, 70], [43, 59, 57, 123, 107, 110, 109, 46, 63, 61, 125, 55, 56, 45, 105, 52, 37, 54, 117, 29, 50, 126, 41, 99, 40, 23, 80, 115, 22, 114, 101, 86, 44, 116, 124, 16, 95, 28, 121, 104, 62, 13, 60, 92, 88, 49, 119, 118, 19, 103, 93, 51, 127, 77, 31, 39, 58, 85, 87, 112, 20, 21, 53, 111, 120, 84, 47, 97, 10, 74, 24, 48, 122, 34, 113, 83, 26, 82, 33, 108, 90, 100, 106, 36, 98, 15, 18, 35, 25, 7, 64, 0, 91, 11, 89, 81, 71, 30, 14, 12, 75, 102, 79, 32, 96, 42, 17, 5, 78, 76, 38, 8, 3, 67, 94, 72, 69, 66, 70, 4, 6, 27, 9, 73, 65, 68, 1, 2], [59, 43, 57, 123, 109, 110, 107, 46, 63, 61, 55, 125, 56, 45, 105, 52, 37, 54, 29, 126, 41, 50, 117, 28, 121, 114, 124, 40, 99, 101, 22, 80, 119, 95, 87, 23, 86, 49, 115, 16, 92, 31, 44, 48, 88, 77, 13, 51, 116, 104, 62, 20, 122, 60, 58, 19, 84, 97, 127, 85, 118, 39, 103, 47, 34, 21, 120, 93, 90, 24, 112, 83, 74, 100, 111, 10, 108, 18, 26, 53, 82, 113, 98, 35, 33, 25, 30, 36, 106, 91, 89, 15, 7, 42, 11, 78, 71, 102, 14, 79, 75, 17, 64, 32, 81, 0, 96, 5, 38, 12, 8, 76, 94, 69, 3, 67, 27, 6, 4, 73, 9, 68, 1, 2, 66, 65, 70, 72], [59, 43, 57, 123, 107, 109, 110, 46, 61, 63, 55, 125, 56, 45, 105, 52, 37, 29, 126, 54, 117, 41, 101, 115, 114, 28, 121, 99, 50, 22, 119, 23, 116, 31, 86, 40, 124, 95, 92, 87, 51, 62, 16, 49, 103, 80, 88, 112, 39, 127, 104, 53, 113, 13, 44, 48, 60, 58, 97, 20, 19, 122, 90, 77, 21, 93, 85, 24, 100, 84, 118, 120, 34, 83, 82, 42, 74, 108, 91, 47, 18, 33, 26, 10, 36, 25, 106, 35, 111, 30, 89, 98, 7, 14, 78, 79, 17, 0, 32, 75, 81, 64, 15, 12, 71, 8, 27, 94, 102, 5, 38, 11, 3, 96, 67, 69, 66, 76, 65, 2, 4, 73, 9, 1, 68, 70, 6, 72], [59, 43, 57, 123, 107, 109, 110, 46, 63, 61, 55, 56, 125, 45, 105, 37, 52, 29, 54, 41, 117, 126, 101, 50, 115, 58, 99, 124, 114, 28, 40, 121, 44, 23, 119, 92, 22, 62, 116, 95, 31, 16, 127, 49, 13, 104, 97, 87, 122, 60, 86, 53, 112, 77, 103, 88, 80, 113, 85, 51, 48, 93, 21, 118, 34, 20, 24, 39, 84, 111, 90, 47, 19, 82, 100, 120, 74, 36, 108, 83, 33, 18, 91, 10, 26, 25, 106, 35, 98, 89, 30, 42, 38, 15, 71, 14, 32, 11, 78, 79, 96, 7, 27, 8, 17, 75, 94, 64, 102, 12, 0, 76, 81, 67, 5, 3, 69, 70, 9, 4, 1, 66, 68, 73, 2, 65, 72, 6], [59, 43, 57, 123, 109, 46, 110, 107, 63, 61, 55, 125, 56, 45, 105, 37, 54, 52, 126, 29, 117, 115, 50, 41, 40, 101, 62, 28, 114, 112, 99, 121, 119, 58, 124, 53, 116, 44, 23, 95, 103, 51, 86, 92, 22, 127, 104, 60, 31, 13, 16, 48, 80, 87, 49, 77, 122, 88, 21, 39, 118, 108, 113, 120, 111, 93, 20, 97, 24, 84, 19, 85, 90, 18, 34, 74, 47, 10, 100, 83, 33, 36, 82, 7, 71, 35, 98, 0, 91, 42, 25, 75, 11, 26, 8, 64, 79, 30, 96, 89, 14, 15, 106, 17, 5, 67, 76, 3, 78, 81, 38, 70, 102, 69, 32, 12, 94, 1, 9, 65, 2, 4, 73, 27, 66, 68, 72, 6], [59, 43, 57, 123, 109, 110, 46, 107, 61, 63, 56, 55, 125, 45, 105, 52, 37, 126, 54, 29, 115, 41, 121, 117, 101, 99, 50, 114, 119, 80, 28, 53, 40, 127, 16, 51, 60, 22, 13, 62, 23, 124, 58, 44, 87, 116, 86, 95, 112, 77, 104, 49, 31, 88, 103, 20, 111, 92, 113, 39, 74, 19, 84, 21, 97, 118, 122, 10, 90, 24, 85, 120, 83, 93, 26, 33, 108, 100, 7, 18, 47, 34, 48, 98, 82, 91, 35, 42, 71, 36, 25, 0, 11, 64, 81, 79, 15, 67, 75, 14, 12, 106, 3, 38, 89, 76, 78, 96, 8, 17, 30, 69, 102, 5, 32, 72, 73, 4, 1, 70, 2, 94, 27, 65, 68, 66, 6, 9], [43, 59, 57, 123, 109, 107, 46, 110, 63, 61, 55, 125, 56, 45, 105, 52, 37, 126, 29, 54, 41, 114, 101, 117, 22, 121, 40, 115, 99, 28, 86, 80, 23, 50, 119, 13, 104, 95, 31, 16, 124, 127, 51, 92, 87, 44, 58, 116, 77, 88, 97, 39, 60, 20, 62, 93, 53, 103, 84, 118, 74, 112, 113, 21, 85, 111, 48, 120, 90, 19, 49, 10, 33, 82, 83, 122, 34, 24, 18, 25, 100, 108, 26, 35, 91, 47, 89, 71, 98, 79, 7, 15, 30, 96, 36, 106, 11, 12, 78, 0, 102, 14, 64, 75, 17, 81, 42, 32, 69, 67, 27, 73, 38, 76, 3, 8, 65, 94, 5, 2, 4, 1, 72, 6, 9, 70, 68, 66], [59, 43, 57, 123, 109, 107, 110, 46, 63, 61, 55, 125, 56, 45, 105, 37, 52, 54, 29, 126, 121, 114, 50, 101, 41, 117, 115, 99, 22, 95, 119, 40, 28, 86, 87, 116, 80, 58, 118, 92, 124, 62, 103, 104, 31, 88, 23, 97, 127, 93, 122, 51, 16, 44, 39, 112, 49, 113, 77, 60, 33, 84, 48, 34, 85, 19, 53, 13, 108, 111, 120, 90, 24, 21, 10, 20, 83, 26, 82, 18, 36, 30, 74, 89, 98, 91, 100, 25, 35, 47, 42, 106, 96, 7, 64, 79, 15, 32, 75, 78, 71, 12, 81, 38, 94, 11, 69, 14, 102, 76, 17, 67, 5, 27, 72, 8, 0, 1, 6, 3, 2, 73, 4, 65, 66, 9, 68, 70], [59, 43, 57, 123, 107, 109, 110, 46, 63, 61, 55, 125, 45, 56, 105, 37, 52, 126, 29, 54, 101, 41, 116, 50, 117, 114, 121, 124, 115, 119, 23, 40, 99, 62, 120, 51, 86, 13, 28, 103, 122, 95, 104, 87, 31, 16, 22, 80, 127, 92, 118, 44, 53, 49, 113, 39, 111, 58, 88, 19, 60, 112, 77, 24, 97, 84, 10, 85, 21, 20, 90, 48, 93, 34, 108, 106, 100, 47, 18, 83, 82, 74, 26, 33, 98, 42, 0, 30, 35, 36, 7, 89, 71, 91, 64, 25, 72, 15, 81, 79, 75, 38, 69, 12, 67, 3, 11, 32, 14, 94, 96, 5, 78, 17, 76, 102, 65, 8, 4, 6, 1, 2, 73, 27, 9, 70, 66, 68], [59, 43, 57, 123, 109, 107, 110, 46, 63, 61, 55, 125, 56, 45, 105, 37, 52, 126, 54, 29, 41, 114, 101, 117, 40, 50, 99, 115, 23, 124, 31, 60, 44, 86, 121, 58, 22, 119, 95, 62, 16, 13, 28, 104, 116, 87, 80, 39, 88, 112, 103, 53, 113, 127, 77, 92, 51, 49, 48, 118, 93, 85, 120, 111, 84, 74, 97, 20, 19, 24, 21, 10, 90, 122, 82, 100, 47, 34, 83, 18, 33, 106, 108, 91, 26, 71, 0, 35, 42, 98, 7, 64, 75, 25, 89, 79, 36, 30, 38, 81, 15, 5, 78, 72, 3, 11, 102, 67, 76, 96, 17, 14, 32, 94, 73, 69, 6, 4, 68, 1, 66, 8, 65, 12, 2, 27, 9, 70], [59, 43, 57, 123, 107, 109, 110, 46, 63, 61, 55, 56, 125, 45, 105, 52, 37, 126, 54, 101, 29, 117, 50, 41, 115, 114, 62, 44, 80, 99, 23, 28, 95, 22, 86, 121, 40, 124, 116, 16, 31, 92, 60, 13, 119, 87, 127, 112, 51, 104, 113, 58, 88, 103, 111, 118, 97, 77, 84, 53, 39, 48, 85, 19, 20, 10, 74, 49, 120, 122, 21, 108, 90, 93, 100, 24, 26, 34, 47, 18, 83, 33, 82, 106, 89, 25, 98, 7, 35, 71, 30, 42, 79, 0, 96, 91, 36, 15, 75, 78, 17, 12, 102, 76, 11, 64, 14, 27, 81, 72, 67, 5, 38, 3, 94, 1, 32, 69, 65, 73, 66, 9, 68, 70, 6, 2, 4, 8], [59, 43, 57, 123, 107, 109, 110, 46, 61, 63, 125, 55, 56, 45, 105, 52, 37, 54, 126, 115, 101, 29, 40, 114, 50, 28, 117, 121, 41, 62, 95, 116, 31, 99, 92, 22, 127, 80, 23, 124, 86, 39, 87, 16, 58, 60, 13, 49, 44, 51, 88, 119, 122, 112, 53, 97, 84, 118, 19, 120, 85, 104, 93, 103, 24, 20, 21, 48, 108, 34, 90, 33, 82, 77, 111, 100, 26, 74, 83, 113, 35, 98, 18, 36, 89, 47, 25, 10, 42, 91, 30, 106, 71, 96, 7, 79, 64, 0, 81, 78, 15, 32, 11, 75, 102, 72, 12, 38, 76, 17, 27, 3, 14, 94, 67, 69, 5, 73, 65, 6, 70, 1, 9, 68, 4, 66, 8, 2], [59, 43, 57, 123, 109, 107, 110, 46, 61, 63, 55, 56, 125, 45, 105, 37, 52, 126, 41, 29, 54, 50, 115, 101, 117, 40, 114, 80, 22, 28, 49, 121, 23, 104, 31, 99, 95, 86, 92, 103, 119, 60, 124, 116, 58, 13, 87, 62, 16, 51, 39, 44, 127, 77, 88, 93, 97, 113, 24, 120, 122, 21, 118, 20, 19, 112, 84, 48, 74, 53, 34, 85, 111, 18, 108, 26, 90, 100, 82, 47, 33, 10, 83, 42, 89, 35, 98, 25, 71, 91, 79, 0, 30, 106, 81, 36, 72, 7, 17, 96, 64, 11, 15, 78, 5, 67, 14, 69, 94, 32, 102, 75, 3, 12, 76, 65, 38, 66, 68, 27, 73, 70, 9, 4, 1, 6, 2, 8], [59, 43, 57, 123, 109, 110, 46, 107, 63, 61, 55, 56, 125, 45, 105, 37, 126, 52, 54, 117, 50, 29, 114, 41, 40, 44, 101, 115, 121, 62, 124, 104, 58, 28, 60, 99, 80, 119, 116, 103, 113, 23, 53, 95, 118, 22, 51, 31, 122, 92, 13, 86, 120, 49, 39, 112, 16, 87, 88, 77, 111, 97, 100, 24, 74, 93, 108, 84, 21, 20, 127, 48, 85, 47, 34, 19, 90, 82, 10, 18, 106, 33, 83, 35, 98, 42, 26, 71, 36, 7, 30, 0, 91, 89, 102, 64, 79, 11, 81, 38, 25, 75, 3, 72, 32, 94, 96, 78, 76, 15, 12, 17, 69, 67, 27, 5, 14, 70, 73, 65, 66, 9, 8, 4, 1, 68, 2, 6], [43, 59, 57, 123, 109, 107, 110, 46, 63, 61, 56, 55, 125, 45, 105, 37, 52, 126, 54, 29, 117, 50, 41, 121, 101, 62, 40, 114, 115, 44, 23, 99, 95, 104, 28, 16, 58, 116, 124, 31, 60, 103, 13, 51, 49, 92, 22, 80, 39, 122, 87, 127, 86, 119, 88, 93, 118, 77, 120, 112, 113, 97, 85, 111, 108, 47, 24, 10, 53, 20, 84, 74, 21, 34, 100, 90, 19, 33, 18, 83, 98, 48, 89, 26, 82, 91, 42, 35, 30, 11, 71, 7, 64, 36, 75, 106, 32, 3, 25, 12, 79, 69, 15, 0, 102, 96, 94, 17, 67, 76, 78, 38, 72, 14, 8, 5, 81, 27, 70, 65, 68, 66, 9, 73, 1, 2, 4, 6], [59, 43, 57, 123, 109, 107, 110, 46, 61, 63, 55, 56, 125, 45, 105, 126, 54, 37, 52, 29, 41, 121, 40, 101, 50, 117, 115, 99, 116, 114, 95, 127, 28, 44, 119, 62, 104, 49, 86, 122, 22, 124, 39, 80, 23, 53, 118, 16, 31, 113, 120, 13, 103, 87, 60, 92, 112, 51, 93, 88, 58, 111, 77, 84, 48, 34, 97, 20, 74, 85, 33, 21, 19, 47, 98, 90, 10, 24, 108, 83, 18, 26, 100, 35, 91, 25, 89, 71, 82, 36, 7, 42, 15, 11, 79, 30, 75, 17, 106, 96, 76, 81, 102, 78, 0, 72, 14, 64, 32, 69, 67, 3, 94, 38, 12, 8, 5, 70, 1, 68, 73, 4, 65, 66, 2, 27, 9, 6], [59, 43, 57, 123, 109, 107, 110, 46, 61, 63, 56, 125, 55, 45, 105, 126, 37, 52, 54, 29, 117, 41, 50, 101, 99, 40, 121, 115, 114, 116, 49, 104, 28, 44, 113, 119, 122, 124, 62, 23, 80, 103, 92, 22, 58, 16, 95, 87, 60, 51, 86, 39, 112, 118, 31, 88, 53, 48, 127, 120, 97, 13, 24, 77, 34, 20, 84, 111, 93, 21, 74, 85, 108, 90, 47, 19, 33, 83, 100, 26, 82, 64, 10, 98, 18, 71, 36, 106, 35, 0, 30, 42, 91, 89, 25, 7, 15, 11, 67, 17, 75, 8, 81, 79, 78, 3, 14, 38, 5, 102, 69, 32, 12, 94, 76, 72, 65, 1, 96, 70, 68, 27, 4, 66, 6, 2, 73, 9], [59, 43, 57, 123, 107, 109, 110, 46, 63, 61, 55, 56, 125, 45, 105, 37, 126, 54, 52, 117, 29, 50, 101, 41, 115, 40, 99, 114, 28, 119, 116, 121, 104, 124, 23, 62, 53, 95, 22, 92, 127, 51, 58, 49, 86, 103, 80, 122, 112, 31, 118, 16, 44, 39, 97, 113, 13, 77, 93, 20, 87, 88, 21, 90, 111, 24, 60, 120, 19, 85, 48, 34, 83, 84, 47, 91, 30, 108, 74, 33, 100, 26, 18, 35, 10, 82, 42, 98, 25, 89, 36, 7, 15, 71, 102, 11, 106, 32, 8, 0, 75, 79, 81, 94, 27, 14, 12, 17, 67, 64, 3, 69, 96, 38, 78, 5, 66, 65, 76, 68, 73, 72, 1, 70, 6, 9, 2, 4], [59, 43, 57, 123, 107, 109, 110, 46, 63, 61, 125, 56, 55, 45, 105, 37, 52, 29, 54, 126, 41, 114, 121, 116, 40, 50, 117, 28, 62, 101, 115, 95, 104, 99, 44, 23, 119, 80, 22, 103, 16, 60, 92, 118, 13, 124, 58, 39, 86, 88, 77, 111, 31, 87, 112, 127, 20, 93, 51, 53, 49, 21, 84, 122, 85, 113, 83, 10, 97, 74, 90, 19, 24, 33, 120, 82, 25, 98, 26, 18, 47, 34, 108, 48, 7, 100, 35, 36, 89, 91, 71, 79, 8, 75, 11, 106, 30, 12, 76, 32, 42, 15, 0, 78, 102, 14, 69, 38, 17, 96, 81, 3, 73, 67, 64, 5, 94, 6, 9, 65, 27, 70, 66, 4, 72, 68, 2, 1], [59, 43, 57, 123, 109, 107, 110, 46, 61, 63, 125, 55, 56, 45, 105, 37, 52, 126, 29, 41, 54, 114, 117, 40, 50, 99, 28, 127, 101, 121, 23, 22, 95, 86, 115, 16, 80, 31, 60, 111, 62, 116, 44, 104, 58, 77, 53, 92, 88, 49, 113, 13, 103, 124, 119, 87, 39, 21, 51, 20, 118, 19, 97, 122, 84, 112, 93, 18, 47, 10, 74, 90, 108, 24, 34, 48, 85, 83, 26, 120, 82, 100, 35, 98, 25, 33, 7, 30, 8, 89, 36, 71, 11, 106, 14, 42, 91, 79, 15, 102, 75, 76, 0, 78, 32, 96, 17, 81, 64, 12, 94, 69, 67, 3, 6, 38, 5, 73, 65, 68, 9, 70, 1, 4, 27, 66, 2, 72], [59, 43, 57, 123, 107, 109, 46, 110, 61, 63, 55, 125, 56, 45, 105, 37, 54, 126, 52, 50, 29, 117, 114, 101, 41, 121, 115, 28, 99, 40, 31, 112, 49, 116, 124, 62, 92, 22, 23, 95, 127, 113, 58, 86, 44, 88, 97, 60, 39, 87, 103, 104, 118, 122, 80, 16, 111, 21, 13, 53, 77, 24, 84, 119, 51, 85, 34, 20, 120, 100, 19, 48, 90, 93, 26, 47, 74, 83, 42, 33, 106, 108, 36, 18, 30, 89, 10, 98, 82, 35, 7, 25, 91, 11, 0, 64, 96, 15, 75, 32, 102, 79, 78, 8, 14, 71, 94, 81, 27, 76, 69, 12, 38, 3, 17, 67, 1, 5, 68, 73, 65, 6, 66, 2, 4, 9, 70, 72], [59, 43, 57, 123, 109, 107, 46, 110, 61, 63, 55, 125, 56, 45, 105, 37, 52, 126, 54, 29, 101, 50, 114, 41, 121, 40, 115, 117, 99, 62, 124, 28, 53, 127, 23, 95, 116, 58, 119, 31, 92, 80, 49, 44, 104, 118, 22, 86, 39, 16, 13, 112, 51, 111, 122, 88, 103, 60, 87, 97, 48, 77, 120, 20, 19, 34, 21, 113, 24, 85, 93, 84, 26, 47, 100, 74, 90, 10, 108, 33, 98, 83, 35, 106, 18, 82, 36, 42, 91, 7, 25, 30, 89, 38, 102, 79, 96, 8, 71, 81, 12, 14, 11, 78, 15, 94, 17, 75, 5, 76, 64, 32, 27, 0, 3, 69, 67, 4, 6, 72, 73, 65, 9, 68, 66, 2, 1, 70], [59, 43, 57, 123, 109, 107, 110, 46, 63, 61, 125, 55, 56, 45, 105, 37, 52, 126, 29, 54, 114, 121, 41, 50, 101, 117, 99, 40, 116, 28, 115, 124, 119, 80, 118, 104, 44, 95, 86, 49, 16, 23, 103, 13, 127, 62, 92, 22, 77, 58, 31, 51, 87, 112, 60, 39, 21, 88, 53, 113, 24, 111, 85, 93, 48, 84, 19, 10, 20, 122, 34, 74, 83, 120, 108, 97, 90, 18, 82, 26, 7, 106, 47, 33, 36, 100, 98, 42, 35, 71, 25, 0, 89, 64, 11, 79, 30, 69, 75, 3, 91, 78, 102, 14, 17, 12, 8, 67, 32, 15, 76, 96, 81, 5, 38, 94, 65, 6, 70, 72, 27, 68, 73, 4, 9, 66, 1, 2], [59, 43, 57, 123, 109, 110, 107, 46, 61, 63, 56, 125, 55, 45, 105, 52, 37, 54, 126, 29, 41, 50, 117, 53, 114, 101, 121, 40, 99, 60, 44, 104, 28, 95, 116, 58, 127, 124, 115, 80, 119, 86, 23, 111, 22, 92, 16, 112, 118, 103, 49, 77, 31, 88, 87, 39, 13, 62, 113, 20, 21, 10, 48, 93, 74, 122, 24, 51, 84, 97, 108, 33, 34, 19, 64, 106, 90, 83, 35, 26, 7, 85, 82, 120, 47, 36, 18, 98, 71, 100, 91, 11, 79, 0, 75, 42, 96, 102, 30, 15, 76, 78, 69, 17, 25, 89, 3, 67, 12, 32, 38, 14, 72, 65, 66, 81, 8, 5, 70, 4, 68, 94, 9, 1, 6, 73, 2, 27], [43, 59, 57, 123, 107, 109, 110, 46, 61, 63, 55, 56, 125, 45, 105, 52, 37, 54, 126, 29, 50, 41, 101, 114, 99, 116, 117, 115, 40, 112, 127, 28, 22, 23, 95, 60, 119, 16, 124, 104, 80, 86, 13, 118, 88, 92, 58, 44, 121, 103, 77, 62, 53, 31, 87, 49, 97, 113, 21, 85, 39, 93, 111, 20, 84, 10, 83, 19, 51, 122, 34, 90, 108, 24, 26, 48, 47, 120, 74, 33, 100, 106, 82, 18, 36, 98, 7, 91, 75, 25, 71, 35, 30, 15, 78, 64, 32, 17, 89, 12, 42, 14, 96, 81, 79, 11, 0, 102, 38, 3, 72, 5, 69, 76, 70, 67, 9, 65, 4, 8, 68, 2, 94, 66, 6, 73, 1, 27], [59, 43, 57, 123, 109, 110, 107, 46, 61, 63, 55, 125, 56, 45, 105, 52, 37, 126, 54, 29, 101, 50, 41, 28, 117, 115, 40, 116, 121, 99, 114, 112, 92, 104, 119, 124, 62, 127, 23, 16, 95, 58, 103, 80, 22, 31, 86, 39, 13, 88, 60, 53, 44, 113, 87, 118, 97, 111, 77, 34, 47, 85, 49, 21, 24, 122, 84, 108, 120, 20, 51, 48, 19, 93, 74, 90, 10, 33, 83, 18, 82, 100, 26, 0, 98, 7, 106, 35, 91, 71, 36, 64, 89, 11, 25, 42, 79, 67, 75, 102, 72, 96, 30, 15, 69, 14, 32, 12, 3, 5, 76, 68, 38, 81, 17, 70, 78, 94, 9, 2, 73, 4, 65, 8, 6, 27, 1, 66], [59, 43, 57, 123, 109, 107, 110, 46, 61, 63, 55, 56, 125, 45, 105, 52, 37, 126, 54, 29, 114, 41, 101, 50, 40, 115, 116, 28, 99, 117, 112, 95, 121, 127, 22, 23, 80, 86, 119, 92, 62, 103, 16, 87, 31, 39, 53, 13, 124, 60, 118, 44, 58, 88, 104, 113, 77, 51, 49, 21, 97, 19, 48, 93, 84, 111, 24, 10, 34, 33, 122, 85, 47, 20, 120, 74, 90, 82, 108, 83, 18, 26, 35, 36, 91, 100, 7, 75, 30, 25, 98, 14, 72, 106, 71, 79, 42, 89, 11, 76, 102, 96, 32, 17, 12, 64, 78, 15, 94, 5, 81, 27, 38, 73, 69, 0, 67, 9, 3, 70, 2, 68, 8, 66, 4, 1, 65, 6]], "model.layers.27.self_attn.q_proj": [[109, 45, 94, 90, 33, 23, 83, 81, 21, 60, 54, 117, 79, 125, 76, 28, 78, 123, 111, 112, 39, 58, 73, 62, 56, 61, 51, 5, 32, 59, 38, 114, 57, 115, 6, 52, 48, 105, 11, 35, 97, 43, 55, 100, 9, 122, 113, 106, 46, 7, 37, 53, 121, 10, 116, 98, 118, 63, 24, 120, 3, 126, 19, 0, 110, 49, 87, 17, 44, 22, 85, 50, 119, 75, 104, 124, 47, 103, 42, 127, 95, 101, 31, 29, 88, 108, 92, 25, 13, 4, 26, 14, 36, 30, 40, 80, 18, 107, 41, 20, 84, 86, 27, 93, 102, 99, 96, 15, 77, 89, 91, 82, 34, 74, 16, 66, 70, 72, 65, 8, 12, 64, 1, 2, 68, 69, 71, 67], [109, 45, 33, 94, 90, 21, 83, 23, 81, 79, 76, 105, 54, 28, 112, 73, 97, 123, 99, 7, 117, 75, 121, 6, 5, 127, 60, 39, 4, 38, 11, 9, 47, 125, 78, 106, 3, 0, 37, 48, 32, 56, 115, 111, 62, 63, 26, 71, 14, 74, 2, 46, 120, 18, 58, 119, 35, 113, 114, 17, 51, 110, 87, 16, 19, 49, 85, 1, 66, 107, 36, 70, 77, 59, 98, 126, 53, 89, 57, 44, 24, 50, 31, 124, 43, 22, 102, 122, 88, 55, 52, 42, 103, 108, 10, 92, 118, 82, 86, 104, 96, 80, 34, 30, 65, 116, 20, 84, 95, 41, 13, 29, 8, 61, 101, 25, 91, 72, 40, 100, 67, 69, 93, 27, 64, 15, 12, 68], [109, 45, 94, 90, 33, 83, 23, 21, 76, 79, 81, 123, 125, 28, 54, 39, 105, 99, 6, 73, 112, 111, 106, 5, 127, 60, 62, 58, 56, 9, 78, 38, 70, 32, 50, 3, 115, 44, 42, 72, 75, 66, 37, 40, 17, 48, 7, 97, 11, 63, 124, 52, 18, 84, 89, 51, 120, 10, 113, 101, 26, 87, 25, 98, 47, 41, 88, 80, 57, 59, 14, 49, 85, 0, 108, 19, 117, 126, 24, 31, 30, 95, 77, 116, 104, 103, 119, 107, 12, 74, 121, 110, 22, 1, 35, 102, 13, 4, 2, 55, 46, 34, 53, 118, 92, 96, 43, 91, 61, 114, 29, 122, 8, 100, 86, 16, 36, 82, 20, 93, 65, 15, 27, 71, 69, 67, 68, 64], [109, 45, 94, 90, 33, 125, 83, 23, 117, 81, 21, 76, 115, 32, 124, 57, 119, 113, 112, 79, 54, 121, 114, 9, 28, 123, 58, 39, 47, 51, 52, 60, 91, 118, 122, 53, 59, 29, 48, 49, 106, 61, 24, 14, 30, 101, 13, 110, 37, 38, 43, 100, 126, 116, 44, 17, 85, 80, 87, 50, 127, 5, 108, 95, 92, 88, 102, 103, 25, 63, 111, 99, 73, 96, 97, 55, 19, 107, 56, 40, 120, 89, 41, 75, 46, 6, 36, 84, 78, 22, 62, 42, 82, 20, 18, 35, 31, 16, 27, 26, 104, 98, 105, 93, 34, 86, 15, 12, 77, 65, 10, 8, 7, 74, 72, 11, 71, 0, 1, 68, 3, 69, 70, 4, 67, 66, 2, 64], [118, 53, 120, 63, 101, 60, 127, 121, 84, 57, 126, 119, 88, 56, 50, 112, 61, 124, 92, 52, 17, 54, 125, 117, 123, 24, 115, 113, 33, 55, 116, 49, 58, 59, 62, 51, 94, 110, 122, 93, 48, 47, 111, 45, 114, 43, 30, 37, 108, 39, 46, 44, 7, 14, 107, 90, 41, 109, 11, 104, 26, 9, 25, 74, 78, 20, 91, 77, 42, 103, 106, 5, 105, 29, 31, 75, 40, 22, 32, 80, 38, 36, 100, 21, 18, 6, 13, 72, 95, 99, 96, 73, 79, 4, 34, 16, 86, 35, 98, 83, 12, 3, 19, 81, 66, 10, 102, 85, 28, 97, 69, 23, 27, 87, 2, 89, 64, 1, 65, 70, 68, 71, 82, 0, 15, 67, 8, 76], [53, 120, 118, 101, 63, 60, 127, 50, 56, 57, 61, 124, 126, 84, 123, 121, 112, 119, 54, 115, 62, 52, 49, 117, 88, 93, 113, 116, 58, 110, 125, 55, 51, 122, 59, 108, 39, 111, 48, 47, 92, 114, 33, 45, 17, 46, 107, 37, 44, 14, 24, 77, 30, 90, 7, 109, 41, 94, 104, 26, 43, 105, 20, 103, 74, 106, 91, 9, 42, 83, 4, 19, 100, 25, 96, 3, 5, 11, 22, 29, 23, 40, 75, 38, 34, 80, 12, 6, 16, 73, 21, 32, 18, 36, 78, 98, 102, 95, 31, 89, 99, 86, 35, 27, 85, 72, 97, 79, 28, 81, 10, 87, 64, 65, 13, 70, 66, 1, 2, 71, 15, 82, 69, 76, 68, 8, 0, 67], [120, 118, 53, 101, 63, 60, 126, 121, 127, 56, 57, 84, 50, 119, 61, 124, 54, 112, 52, 62, 123, 115, 117, 125, 58, 116, 55, 110, 88, 93, 39, 113, 122, 49, 59, 47, 17, 51, 48, 111, 92, 108, 114, 33, 46, 14, 45, 43, 94, 109, 37, 90, 30, 11, 26, 41, 7, 25, 44, 107, 104, 24, 106, 9, 103, 42, 77, 22, 20, 100, 105, 74, 29, 91, 40, 78, 6, 34, 80, 5, 38, 31, 102, 18, 98, 95, 36, 32, 96, 73, 3, 85, 83, 16, 4, 86, 12, 79, 10, 28, 19, 75, 99, 27, 23, 21, 35, 13, 97, 89, 72, 81, 64, 66, 65, 1, 87, 70, 15, 71, 82, 67, 69, 0, 76, 68, 2, 8], [63, 53, 120, 118, 101, 60, 57, 50, 84, 127, 62, 61, 54, 112, 56, 124, 121, 88, 123, 115, 119, 117, 52, 49, 55, 126, 116, 39, 113, 58, 110, 122, 30, 59, 51, 125, 93, 111, 44, 48, 108, 114, 47, 14, 24, 45, 90, 43, 17, 46, 94, 77, 37, 33, 109, 92, 103, 74, 107, 11, 5, 7, 104, 73, 106, 98, 26, 9, 91, 41, 42, 105, 100, 36, 6, 83, 3, 18, 4, 40, 80, 32, 22, 38, 102, 85, 28, 25, 96, 20, 27, 75, 35, 34, 31, 95, 21, 23, 16, 78, 19, 99, 79, 72, 97, 86, 89, 10, 66, 29, 81, 12, 87, 13, 82, 65, 68, 15, 0, 64, 71, 2, 70, 1, 76, 69, 67, 8], [40, 98, 63, 23, 31, 85, 80, 26, 121, 60, 13, 19, 125, 54, 82, 122, 79, 8, 49, 74, 6, 1, 117, 55, 113, 105, 59, 57, 9, 12, 106, 127, 111, 52, 46, 66, 50, 119, 56, 58, 53, 112, 43, 103, 11, 27, 120, 108, 39, 126, 4, 124, 37, 24, 62, 123, 109, 28, 115, 38, 67, 100, 107, 64, 0, 90, 51, 15, 73, 75, 47, 61, 76, 42, 116, 97, 78, 21, 99, 35, 18, 30, 25, 104, 36, 118, 41, 87, 48, 88, 44, 102, 32, 93, 45, 94, 84, 101, 114, 3, 95, 33, 68, 5, 83, 110, 34, 96, 77, 29, 20, 89, 16, 91, 86, 69, 92, 7, 14, 81, 17, 72, 70, 22, 65, 71, 10, 2], [40, 63, 98, 31, 80, 23, 60, 85, 13, 74, 6, 4, 19, 8, 22, 64, 52, 66, 121, 46, 108, 28, 124, 56, 122, 106, 0, 79, 125, 104, 54, 65, 11, 55, 107, 113, 75, 59, 18, 90, 48, 82, 7, 1, 58, 73, 126, 119, 12, 68, 117, 84, 127, 83, 105, 34, 57, 49, 21, 120, 35, 77, 109, 20, 101, 111, 94, 47, 14, 15, 39, 81, 30, 97, 24, 100, 96, 2, 3, 16, 62, 114, 10, 53, 33, 32, 70, 9, 25, 112, 95, 17, 123, 116, 38, 89, 5, 91, 87, 88, 72, 86, 76, 51, 71, 78, 29, 67, 69, 43, 27, 36, 103, 44, 115, 50, 93, 41, 110, 102, 99, 26, 92, 42, 45, 37, 61, 118], [40, 63, 98, 31, 60, 8, 80, 85, 19, 13, 23, 6, 74, 1, 66, 121, 64, 122, 117, 124, 4, 26, 46, 106, 15, 52, 28, 54, 57, 109, 55, 125, 0, 89, 104, 87, 12, 24, 120, 56, 79, 119, 105, 68, 69, 108, 72, 27, 7, 113, 11, 71, 59, 126, 42, 21, 48, 18, 76, 83, 33, 78, 84, 103, 96, 75, 32, 16, 82, 65, 115, 20, 41, 90, 81, 30, 77, 50, 2, 62, 3, 51, 25, 116, 92, 123, 5, 114, 127, 67, 37, 93, 88, 61, 10, 47, 29, 118, 58, 36, 38, 107, 39, 94, 17, 70, 111, 97, 14, 49, 86, 110, 45, 91, 73, 100, 101, 35, 44, 53, 112, 99, 102, 22, 43, 9, 95, 34], [40, 63, 98, 23, 85, 60, 26, 80, 31, 13, 8, 74, 19, 66, 82, 6, 121, 59, 52, 106, 4, 58, 41, 122, 113, 46, 119, 28, 49, 48, 95, 57, 84, 5, 120, 111, 105, 125, 65, 38, 67, 79, 54, 55, 108, 109, 117, 2, 73, 102, 90, 7, 123, 47, 61, 76, 64, 0, 18, 107, 50, 83, 45, 69, 56, 53, 97, 32, 115, 77, 81, 87, 71, 124, 15, 42, 89, 103, 91, 11, 30, 44, 114, 126, 21, 118, 78, 100, 43, 24, 39, 96, 36, 62, 17, 68, 35, 127, 93, 20, 29, 51, 116, 16, 3, 22, 94, 27, 70, 75, 9, 112, 88, 33, 110, 92, 37, 10, 101, 25, 14, 12, 72, 86, 1, 99, 104, 34], [104, 120, 98, 46, 95, 126, 115, 52, 44, 91, 108, 59, 56, 60, 54, 27, 84, 58, 88, 96, 118, 82, 24, 122, 51, 36, 50, 21, 45, 42, 48, 112, 101, 57, 62, 124, 114, 55, 63, 113, 92, 119, 117, 116, 61, 76, 125, 86, 85, 47, 127, 23, 53, 123, 111, 121, 49, 43, 38, 13, 107, 110, 109, 22, 41, 106, 74, 99, 103, 105, 80, 89, 33, 97, 31, 29, 32, 19, 6, 78, 37, 102, 25, 39, 69, 94, 28, 100, 35, 64, 90, 18, 93, 8, 30, 15, 26, 83, 65, 87, 34, 14, 12, 0, 40, 2, 79, 20, 66, 81, 72, 7, 1, 17, 3, 73, 16, 67, 11, 4, 75, 10, 5, 68, 77, 71, 9, 70], [120, 104, 98, 95, 44, 108, 126, 46, 113, 125, 54, 27, 88, 91, 55, 58, 127, 122, 61, 59, 41, 60, 115, 50, 110, 106, 56, 84, 36, 118, 116, 109, 123, 22, 38, 103, 62, 124, 111, 57, 96, 63, 114, 49, 112, 43, 28, 121, 40, 117, 53, 48, 35, 42, 24, 52, 45, 51, 101, 107, 119, 37, 47, 39, 105, 82, 94, 19, 85, 86, 83, 97, 102, 21, 12, 79, 93, 73, 26, 25, 81, 71, 100, 99, 31, 29, 92, 9, 30, 4, 33, 32, 89, 34, 23, 76, 74, 78, 90, 80, 15, 20, 18, 87, 8, 14, 10, 17, 70, 69, 72, 66, 7, 6, 2, 68, 13, 75, 3, 67, 5, 16, 77, 11, 65, 0, 64, 1], [104, 120, 98, 95, 126, 91, 80, 84, 13, 82, 22, 86, 115, 27, 74, 44, 31, 93, 116, 36, 16, 97, 69, 88, 20, 77, 60, 100, 52, 59, 58, 90, 56, 6, 108, 125, 72, 94, 45, 2, 19, 122, 32, 25, 3, 62, 63, 18, 54, 114, 92, 96, 42, 64, 41, 43, 75, 15, 29, 113, 68, 55, 87, 30, 46, 106, 11, 118, 102, 33, 79, 48, 81, 76, 24, 99, 78, 103, 117, 70, 89, 83, 57, 85, 50, 65, 37, 8, 23, 21, 35, 26, 67, 17, 61, 73, 47, 107, 9, 4, 7, 1, 111, 38, 49, 71, 10, 12, 28, 14, 110, 53, 5, 127, 0, 34, 39, 66, 105, 101, 124, 121, 119, 112, 109, 123, 51, 40], [104, 120, 98, 46, 95, 82, 84, 96, 91, 80, 115, 14, 126, 44, 66, 48, 125, 27, 13, 67, 60, 29, 42, 116, 68, 74, 93, 8, 28, 85, 63, 102, 113, 36, 0, 23, 55, 117, 101, 105, 38, 6, 76, 65, 1, 21, 35, 24, 111, 54, 39, 45, 56, 99, 88, 26, 32, 119, 103, 86, 37, 50, 17, 108, 107, 64, 9, 97, 109, 52, 79, 40, 90, 16, 22, 31, 100, 92, 124, 41, 94, 122, 25, 58, 61, 62, 59, 47, 121, 72, 57, 112, 34, 123, 81, 15, 43, 110, 5, 89, 70, 20, 19, 106, 30, 114, 127, 83, 51, 118, 49, 53, 12, 87, 33, 11, 73, 78, 18, 69, 75, 10, 77, 3, 4, 7, 71, 2], [111, 100, 47, 56, 24, 53, 95, 121, 120, 58, 31, 54, 77, 36, 82, 103, 86, 57, 1, 124, 59, 119, 60, 84, 127, 16, 52, 83, 104, 97, 94, 10, 63, 45, 105, 49, 64, 116, 110, 28, 122, 62, 51, 126, 108, 92, 42, 112, 67, 113, 39, 27, 55, 125, 43, 102, 61, 46, 19, 118, 26, 40, 50, 8, 68, 123, 4, 38, 44, 114, 85, 71, 2, 109, 115, 22, 73, 6, 15, 41, 117, 99, 48, 66, 69, 106, 98, 37, 107, 101, 20, 35, 90, 0, 96, 81, 70, 30, 32, 89, 33, 17, 80, 78, 9, 88, 65, 23, 29, 91, 34, 87, 93, 5, 25, 11, 12, 18, 79, 13, 72, 21, 3, 75, 7, 74, 14, 76], [111, 47, 58, 100, 24, 31, 122, 95, 61, 106, 51, 125, 82, 54, 84, 28, 113, 26, 85, 56, 53, 20, 126, 110, 117, 112, 30, 77, 127, 50, 13, 46, 118, 98, 83, 36, 104, 22, 55, 6, 12, 79, 120, 39, 45, 37, 49, 15, 123, 86, 116, 41, 94, 52, 119, 78, 59, 124, 14, 121, 17, 57, 114, 92, 43, 108, 105, 60, 109, 101, 63, 25, 107, 99, 88, 23, 103, 115, 44, 68, 96, 40, 27, 90, 80, 35, 62, 0, 29, 97, 32, 73, 48, 74, 33, 89, 87, 38, 102, 3, 42, 21, 81, 10, 34, 72, 66, 76, 18, 16, 93, 19, 91, 11, 65, 7, 8, 69, 75, 9, 4, 2, 5, 71, 67, 70, 64, 1], [111, 47, 100, 58, 56, 24, 95, 31, 114, 115, 52, 94, 106, 83, 121, 36, 82, 119, 53, 86, 54, 120, 112, 110, 28, 85, 113, 55, 103, 51, 49, 92, 104, 59, 109, 118, 17, 84, 126, 63, 127, 20, 48, 50, 60, 62, 45, 117, 30, 122, 116, 105, 38, 57, 124, 61, 77, 96, 46, 125, 16, 107, 43, 41, 123, 39, 108, 88, 42, 102, 22, 80, 90, 26, 40, 79, 6, 78, 44, 98, 101, 10, 89, 37, 97, 35, 12, 81, 68, 99, 34, 15, 11, 33, 0, 19, 23, 13, 29, 73, 9, 21, 32, 27, 91, 93, 71, 72, 25, 18, 74, 87, 66, 3, 14, 8, 76, 7, 4, 75, 64, 67, 5, 69, 65, 70, 1, 2], [111, 47, 58, 100, 56, 24, 95, 31, 127, 121, 122, 106, 115, 36, 94, 120, 44, 62, 82, 28, 83, 108, 116, 126, 59, 53, 52, 112, 85, 77, 54, 92, 118, 84, 105, 51, 113, 104, 60, 49, 17, 107, 110, 124, 45, 109, 20, 63, 57, 61, 103, 86, 55, 43, 114, 46, 48, 123, 125, 78, 6, 80, 119, 102, 96, 42, 30, 117, 38, 26, 39, 37, 33, 16, 50, 22, 41, 90, 34, 19, 12, 10, 40, 79, 32, 15, 88, 101, 74, 98, 91, 27, 29, 93, 81, 89, 35, 99, 97, 66, 11, 23, 68, 13, 65, 21, 25, 87, 71, 72, 8, 7, 3, 18, 1, 73, 76, 14, 0, 9, 4, 75, 5, 70, 69, 2, 67, 64], [48, 41, 51, 62, 125, 55, 121, 112, 24, 88, 30, 57, 52, 54, 60, 53, 123, 114, 49, 119, 115, 118, 117, 113, 56, 124, 110, 126, 61, 97, 50, 127, 94, 116, 47, 44, 27, 120, 122, 106, 58, 90, 103, 82, 111, 91, 63, 59, 36, 109, 107, 108, 89, 45, 37, 43, 46, 79, 21, 42, 19, 22, 104, 28, 39, 101, 102, 105, 93, 15, 83, 38, 35, 80, 33, 40, 98, 32, 99, 34, 18, 73, 86, 100, 95, 76, 96, 71, 84, 13, 16, 29, 92, 77, 85, 87, 25, 9, 31, 20, 26, 65, 17, 23, 81, 7, 3, 66, 68, 5, 12, 6, 1, 64, 0, 2, 67, 4, 70, 14, 11, 69, 78, 74, 10, 75, 8, 72], [41, 48, 51, 119, 112, 62, 20, 89, 105, 14, 81, 115, 80, 11, 97, 10, 12, 30, 8, 125, 54, 27, 60, 5, 70, 78, 25, 121, 22, 49, 52, 84, 56, 117, 0, 28, 118, 72, 57, 87, 126, 61, 29, 127, 104, 17, 37, 110, 96, 124, 55, 93, 50, 94, 67, 3, 111, 88, 123, 114, 58, 16, 122, 23, 90, 53, 36, 45, 120, 2, 74, 100, 44, 92, 113, 76, 32, 46, 31, 116, 83, 24, 102, 43, 47, 19, 26, 75, 59, 108, 103, 107, 21, 106, 63, 109, 6, 71, 101, 95, 91, 98, 34, 42, 39, 77, 18, 35, 33, 99, 7, 68, 38, 40, 79, 86, 65, 69, 4, 15, 82, 85, 13, 9, 66, 1, 64, 73], [51, 48, 41, 119, 125, 24, 30, 121, 52, 62, 57, 124, 115, 113, 54, 88, 55, 60, 53, 123, 49, 117, 110, 27, 118, 127, 56, 126, 97, 61, 112, 50, 114, 103, 120, 94, 122, 58, 36, 91, 116, 44, 111, 47, 106, 101, 59, 82, 90, 107, 109, 46, 63, 108, 43, 45, 42, 37, 39, 22, 104, 102, 15, 35, 93, 77, 40, 79, 89, 21, 18, 33, 38, 19, 28, 105, 9, 83, 26, 32, 84, 17, 100, 99, 98, 86, 13, 76, 5, 95, 92, 71, 73, 12, 65, 87, 34, 31, 29, 16, 96, 70, 7, 3, 66, 80, 23, 4, 11, 85, 1, 25, 81, 68, 6, 0, 2, 20, 14, 10, 64, 67, 75, 69, 74, 78, 8, 72], [119, 48, 41, 51, 125, 24, 121, 57, 62, 52, 55, 60, 30, 97, 54, 88, 113, 123, 53, 115, 56, 124, 49, 110, 117, 126, 127, 118, 114, 50, 61, 112, 116, 94, 120, 103, 58, 91, 122, 27, 36, 111, 44, 82, 47, 63, 109, 101, 59, 90, 107, 106, 46, 108, 37, 45, 89, 104, 43, 22, 39, 19, 79, 42, 102, 99, 98, 73, 28, 105, 38, 80, 35, 21, 34, 18, 93, 83, 40, 96, 100, 32, 15, 33, 86, 71, 77, 95, 65, 85, 13, 31, 26, 92, 29, 76, 16, 3, 84, 9, 23, 17, 87, 68, 66, 5, 81, 6, 7, 4, 75, 64, 0, 1, 12, 70, 2, 67, 25, 20, 11, 69, 10, 74, 14, 78, 8, 72], [106, 35, 42, 110, 91, 85, 50, 89, 52, 96, 83, 54, 113, 31, 51, 56, 122, 123, 114, 55, 14, 17, 32, 126, 57, 46, 117, 59, 53, 120, 62, 60, 27, 107, 75, 22, 9, 58, 29, 87, 119, 49, 70, 125, 15, 115, 16, 112, 109, 21, 61, 13, 48, 44, 63, 127, 39, 28, 68, 86, 23, 84, 41, 69, 19, 71, 45, 33, 92, 97, 25, 116, 111, 5, 20, 26, 34, 81, 121, 37, 79, 36, 38, 40, 124, 47, 94, 90, 11, 76, 82, 80, 18, 93, 105, 88, 43, 100, 104, 30, 108, 118, 103, 77, 24, 98, 102, 99, 95, 101, 78, 73, 8, 12, 72, 74, 10, 7, 65, 3, 6, 4, 66, 2, 0, 67, 1, 64], [106, 35, 42, 85, 31, 17, 83, 96, 89, 50, 52, 55, 51, 14, 9, 27, 113, 56, 69, 75, 62, 57, 110, 53, 93, 91, 122, 6, 117, 3, 66, 1, 61, 65, 127, 64, 87, 123, 32, 126, 7, 15, 59, 68, 13, 70, 114, 25, 23, 76, 84, 120, 0, 21, 63, 5, 22, 74, 10, 109, 54, 119, 71, 29, 11, 19, 16, 82, 72, 20, 8, 18, 4, 67, 78, 81, 77, 118, 41, 37, 73, 79, 88, 90, 38, 30, 121, 86, 115, 12, 48, 94, 98, 45, 43, 44, 26, 24, 33, 80, 39, 108, 100, 105, 99, 47, 36, 92, 28, 125, 107, 111, 102, 34, 49, 124, 116, 46, 2, 97, 60, 101, 112, 103, 104, 95, 58, 40], [106, 35, 42, 110, 91, 85, 50, 122, 83, 126, 56, 57, 17, 89, 117, 96, 113, 14, 54, 55, 31, 32, 59, 52, 114, 62, 127, 63, 61, 53, 22, 9, 116, 112, 75, 51, 94, 29, 27, 123, 48, 37, 65, 45, 99, 47, 25, 76, 7, 23, 58, 3, 21, 107, 69, 124, 46, 33, 66, 19, 115, 64, 93, 109, 41, 71, 120, 39, 118, 36, 84, 70, 119, 79, 97, 13, 105, 60, 92, 100, 87, 28, 81, 11, 5, 98, 12, 88, 111, 6, 44, 125, 86, 104, 102, 43, 108, 68, 49, 121, 26, 18, 103, 80, 20, 24, 90, 38, 82, 16, 30, 4, 95, 15, 40, 78, 101, 8, 73, 34, 74, 10, 77, 72, 67, 1, 0, 2], [106, 35, 42, 110, 56, 91, 85, 52, 53, 83, 122, 96, 57, 17, 114, 50, 113, 51, 32, 62, 31, 120, 89, 14, 55, 59, 117, 29, 112, 27, 22, 54, 46, 126, 49, 119, 75, 45, 99, 109, 94, 87, 123, 124, 115, 116, 61, 9, 118, 21, 92, 11, 43, 86, 60, 25, 37, 47, 39, 125, 127, 28, 58, 38, 34, 19, 23, 121, 108, 44, 33, 84, 107, 103, 41, 97, 111, 63, 88, 48, 40, 100, 30, 16, 102, 93, 104, 26, 101, 105, 81, 24, 90, 82, 98, 36, 15, 78, 20, 76, 18, 79, 95, 71, 4, 12, 13, 80, 74, 77, 68, 70, 73, 7, 3, 10, 8, 65, 69, 6, 72, 66, 67, 64, 0, 5, 1, 2], [61, 122, 118, 102, 49, 54, 109, 45, 59, 50, 116, 90, 103, 119, 63, 125, 101, 112, 108, 94, 55, 127, 56, 57, 113, 62, 111, 37, 124, 60, 38, 22, 110, 44, 121, 114, 123, 115, 47, 40, 117, 41, 42, 51, 48, 46, 96, 43, 120, 58, 53, 104, 93, 52, 107, 105, 106, 126, 36, 100, 39, 92, 31, 23, 11, 34, 86, 99, 97, 98, 32, 79, 30, 35, 33, 85, 81, 4, 83, 20, 88, 14, 18, 2, 26, 66, 95, 25, 76, 16, 15, 91, 73, 28, 29, 13, 3, 5, 27, 72, 10, 6, 82, 21, 24, 68, 89, 78, 80, 0, 70, 8, 71, 19, 84, 65, 75, 17, 74, 12, 87, 69, 67, 7, 1, 77, 9, 64], [102, 118, 54, 61, 116, 77, 9, 29, 7, 122, 23, 1, 64, 81, 69, 20, 4, 68, 49, 3, 15, 67, 82, 65, 26, 74, 45, 33, 66, 70, 90, 11, 113, 53, 0, 22, 75, 86, 19, 5, 112, 83, 73, 124, 71, 25, 87, 10, 96, 72, 18, 107, 119, 14, 36, 85, 24, 30, 78, 89, 6, 79, 91, 80, 13, 108, 59, 12, 94, 31, 17, 88, 27, 125, 93, 8, 16, 21, 120, 2, 84, 99, 63, 32, 76, 95, 110, 28, 55, 92, 56, 127, 98, 100, 101, 35, 57, 109, 46, 50, 39, 97, 47, 37, 34, 126, 42, 40, 104, 103, 38, 43, 111, 44, 60, 117, 123, 106, 58, 41, 114, 115, 48, 105, 121, 62, 52, 51], [102, 118, 54, 116, 122, 61, 45, 90, 49, 23, 112, 53, 33, 93, 113, 127, 15, 81, 38, 43, 107, 114, 57, 31, 44, 96, 125, 101, 20, 50, 119, 30, 24, 106, 83, 28, 26, 56, 59, 29, 124, 48, 55, 21, 39, 77, 11, 100, 60, 117, 6, 76, 22, 42, 67, 14, 51, 52, 126, 82, 37, 94, 111, 66, 92, 97, 47, 99, 108, 64, 58, 40, 5, 41, 19, 74, 46, 110, 63, 34, 98, 103, 4, 123, 62, 75, 36, 71, 105, 7, 86, 109, 32, 78, 120, 85, 89, 35, 121, 9, 12, 69, 80, 8, 84, 115, 1, 70, 18, 72, 95, 91, 17, 104, 25, 79, 88, 2, 27, 16, 3, 10, 73, 87, 68, 0, 13, 65], [102, 118, 116, 61, 122, 54, 49, 90, 23, 20, 45, 9, 92, 81, 74, 77, 29, 15, 22, 93, 33, 113, 107, 7, 124, 53, 11, 64, 26, 38, 112, 55, 6, 94, 82, 31, 21, 70, 79, 66, 100, 50, 28, 119, 72, 57, 4, 83, 44, 37, 41, 127, 86, 59, 101, 36, 89, 5, 3, 85, 58, 110, 121, 67, 126, 125, 106, 35, 43, 115, 75, 1, 103, 63, 108, 60, 117, 123, 68, 56, 114, 98, 39, 109, 69, 47, 19, 34, 104, 42, 51, 62, 48, 24, 97, 111, 96, 27, 120, 65, 105, 95, 30, 52, 84, 14, 2, 32, 88, 46, 99, 10, 16, 80, 78, 76, 87, 18, 12, 0, 8, 25, 40, 91, 17, 71, 73, 13]], "model.layers.27.self_attn.k_proj": [[45, 109, 83, 23, 21, 90, 94, 81, 76, 79, 33, 74, 30, 54, 97, 28, 125, 60, 6, 123, 48, 127, 52, 22, 51, 8, 0, 113, 49, 112, 32, 101, 47, 126, 73, 63, 106, 31, 122, 46, 118, 42, 103, 117, 124, 91, 24, 121, 96, 44, 18, 11, 35, 102, 50, 7, 37, 95, 108, 114, 13, 39, 4, 119, 43, 65, 105, 116, 120, 115, 16, 36, 57, 86, 89, 61, 104, 111, 53, 40, 56, 110, 34, 82, 55, 15, 29, 38, 107, 25, 26, 59, 9, 5, 62, 41, 92, 14, 58, 88, 80, 75, 99, 100, 27, 98, 78, 69, 93, 10, 84, 3, 20, 77, 87, 85, 17, 71, 72, 19, 66, 70, 68, 12, 1, 67, 2, 64], [53, 37, 22, 120, 63, 118, 97, 86, 58, 61, 60, 57, 116, 121, 125, 112, 113, 56, 55, 114, 26, 51, 119, 127, 62, 124, 52, 122, 50, 108, 110, 59, 117, 123, 48, 126, 95, 45, 115, 54, 49, 44, 93, 111, 109, 35, 47, 46, 42, 107, 96, 43, 41, 83, 28, 105, 40, 106, 29, 104, 15, 89, 38, 103, 102, 79, 100, 23, 101, 81, 32, 99, 82, 36, 39, 31, 12, 13, 34, 33, 18, 98, 72, 17, 91, 88, 92, 30, 78, 87, 27, 11, 25, 85, 94, 16, 20, 10, 9, 90, 6, 77, 7, 24, 84, 14, 68, 80, 21, 75, 5, 76, 19, 2, 69, 74, 3, 67, 71, 65, 73, 70, 0, 1, 4, 8, 64, 66], [104, 63, 34, 80, 13, 74, 23, 8, 85, 60, 19, 4, 64, 6, 95, 26, 66, 52, 121, 114, 106, 122, 105, 117, 49, 79, 54, 119, 65, 57, 110, 28, 7, 44, 127, 82, 124, 120, 43, 55, 56, 125, 1, 112, 107, 59, 45, 11, 98, 3, 116, 103, 46, 70, 41, 2, 9, 58, 126, 69, 39, 53, 5, 47, 84, 51, 18, 24, 71, 115, 0, 123, 42, 50, 108, 67, 48, 31, 14, 96, 12, 20, 62, 61, 100, 35, 27, 75, 68, 97, 22, 118, 30, 99, 102, 36, 29, 109, 111, 10, 25, 83, 37, 113, 76, 78, 17, 32, 101, 94, 72, 89, 38, 33, 88, 93, 90, 15, 92, 81, 86, 91, 73, 87, 77, 21, 16, 40], [40, 120, 34, 27, 31, 110, 126, 56, 84, 82, 88, 74, 46, 80, 52, 13, 125, 64, 48, 93, 67, 108, 6, 16, 58, 85, 49, 76, 65, 72, 114, 23, 44, 30, 106, 77, 55, 100, 45, 115, 66, 117, 68, 111, 60, 86, 107, 5, 20, 22, 69, 79, 8, 78, 75, 94, 54, 59, 53, 21, 91, 61, 24, 57, 63, 109, 102, 124, 96, 73, 62, 112, 42, 87, 90, 15, 81, 83, 28, 2, 116, 101, 122, 123, 105, 103, 12, 121, 113, 127, 70, 38, 7, 41, 92, 118, 3, 119, 29, 33, 43, 39, 97, 51, 50, 36, 26, 89, 32, 11, 25, 99, 17, 37, 18, 47, 35, 0, 14, 98, 95, 4, 19, 71, 10, 9, 1, 104], [47, 111, 58, 36, 31, 86, 24, 92, 56, 82, 0, 26, 17, 15, 6, 16, 121, 3, 83, 12, 54, 11, 105, 91, 113, 84, 116, 20, 66, 85, 45, 55, 19, 77, 63, 46, 51, 126, 57, 78, 43, 124, 25, 61, 119, 68, 49, 42, 38, 117, 48, 44, 104, 59, 118, 60, 89, 50, 114, 52, 112, 53, 103, 127, 123, 7, 115, 97, 122, 120, 65, 110, 62, 99, 9, 41, 1, 8, 5, 39, 40, 34, 29, 109, 107, 37, 10, 108, 94, 93, 125, 35, 106, 71, 32, 74, 101, 21, 102, 72, 76, 4, 33, 67, 90, 22, 87, 64, 79, 98, 81, 100, 96, 30, 23, 69, 27, 18, 80, 2, 13, 75, 28, 73, 88, 70, 14, 95], [105, 48, 22, 119, 33, 51, 62, 112, 54, 99, 49, 61, 94, 118, 125, 60, 55, 122, 50, 121, 127, 35, 45, 56, 124, 106, 117, 123, 126, 108, 40, 52, 120, 114, 53, 82, 59, 100, 111, 47, 63, 98, 58, 39, 24, 110, 57, 44, 28, 32, 113, 92, 46, 115, 43, 79, 109, 91, 107, 41, 38, 34, 101, 42, 26, 116, 19, 18, 29, 102, 103, 36, 9, 15, 104, 73, 85, 88, 96, 37, 97, 21, 80, 77, 13, 90, 23, 31, 89, 30, 93, 95, 87, 4, 81, 27, 25, 83, 20, 14, 11, 71, 76, 1, 86, 12, 16, 74, 66, 17, 7, 8, 78, 10, 5, 75, 68, 6, 72, 84, 70, 64, 67, 3, 69, 2, 65, 0], [42, 85, 83, 89, 17, 114, 126, 120, 55, 35, 46, 14, 9, 56, 61, 127, 32, 117, 99, 123, 109, 57, 49, 62, 3, 75, 106, 69, 95, 7, 45, 91, 122, 48, 115, 15, 119, 53, 50, 51, 31, 93, 110, 59, 52, 0, 1, 87, 116, 13, 64, 58, 6, 54, 29, 44, 60, 23, 84, 111, 43, 66, 121, 41, 40, 38, 22, 113, 112, 74, 97, 107, 5, 10, 125, 68, 30, 108, 8, 118, 124, 20, 104, 80, 71, 98, 101, 76, 100, 102, 47, 33, 63, 11, 25, 39, 88, 26, 92, 103, 82, 12, 36, 105, 34, 94, 90, 37, 96, 16, 28, 86, 27, 72, 24, 18, 77, 79, 70, 21, 65, 4, 78, 81, 19, 73, 2, 67], [118, 38, 61, 54, 116, 113, 64, 122, 65, 0, 109, 69, 7, 23, 77, 74, 90, 93, 3, 15, 94, 108, 81, 97, 9, 59, 2, 48, 6, 68, 52, 124, 53, 117, 112, 43, 82, 75, 57, 119, 103, 20, 30, 62, 50, 19, 111, 123, 21, 83, 106, 115, 110, 127, 56, 67, 107, 120, 60, 125, 11, 63, 47, 46, 105, 96, 102, 66, 49, 51, 121, 55, 104, 22, 114, 126, 100, 44, 41, 40, 58, 4, 84, 39, 36, 45, 35, 98, 76, 33, 89, 34, 1, 8, 42, 14, 37, 92, 101, 95, 27, 88, 5, 80, 85, 16, 99, 72, 28, 70, 31, 79, 12, 24, 25, 91, 32, 86, 10, 18, 71, 78, 87, 13, 73, 29, 17, 26]], "model.layers.27.self_attn.qk_proj": [[120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 61, 106, 119, 51, 54, 58, 104, 60, 122, 56, 126, 40, 116, 31, 85, 94, 62, 52, 35, 55, 26, 83, 21, 91, 19, 125, 87, 121, 102, 110, 127, 95, 57, 117, 90, 23, 105, 46, 113, 49, 112, 108, 33, 80, 41, 77, 6, 16, 81, 34, 13, 98, 84, 88, 27, 50, 17, 10, 115, 86, 44, 38, 79, 59, 114, 24, 15, 20, 22, 37, 124, 36, 74, 123, 30, 64, 97, 100, 28, 0, 25, 18, 107, 93, 12, 76, 82, 101, 72, 43, 39, 11, 96, 92, 29, 89, 32, 9, 103, 69, 8, 99, 4, 65, 70, 73, 67, 14, 68, 3, 75, 66, 78, 2, 5, 71, 1, 7], [120, 63, 118, 45, 109, 47, 111, 42, 53, 48, 51, 119, 106, 61, 54, 122, 58, 104, 60, 126, 125, 56, 31, 116, 62, 40, 121, 35, 55, 52, 85, 102, 19, 94, 95, 21, 46, 26, 23, 127, 105, 110, 57, 113, 112, 90, 33, 6, 117, 98, 49, 91, 87, 77, 108, 41, 13, 124, 50, 83, 80, 44, 59, 17, 114, 86, 115, 22, 123, 24, 38, 27, 16, 36, 88, 84, 15, 10, 74, 81, 79, 37, 34, 64, 93, 28, 20, 100, 43, 30, 0, 82, 101, 97, 107, 8, 96, 12, 11, 92, 68, 76, 25, 18, 9, 32, 39, 103, 1, 72, 69, 65, 99, 29, 3, 73, 66, 2, 75, 78, 4, 89, 5, 67, 7, 71, 14, 70], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 51, 61, 54, 106, 119, 58, 60, 56, 122, 104, 40, 116, 126, 62, 121, 125, 85, 31, 19, 105, 127, 94, 21, 46, 26, 83, 49, 110, 112, 52, 55, 35, 6, 117, 113, 57, 91, 102, 87, 95, 98, 23, 90, 80, 33, 17, 16, 108, 41, 123, 13, 27, 77, 43, 88, 24, 107, 115, 22, 38, 79, 37, 36, 34, 10, 59, 50, 44, 81, 84, 86, 64, 114, 93, 97, 30, 15, 124, 100, 0, 20, 8, 18, 74, 28, 92, 39, 101, 12, 82, 96, 11, 9, 68, 32, 4, 29, 76, 78, 99, 25, 65, 103, 1, 67, 73, 69, 14, 75, 89, 3, 66, 5, 72, 7, 2, 71, 70], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 51, 61, 106, 119, 54, 58, 60, 104, 122, 62, 116, 56, 40, 126, 125, 127, 85, 121, 31, 55, 46, 19, 26, 110, 113, 52, 112, 35, 87, 105, 102, 83, 91, 21, 49, 6, 94, 57, 16, 95, 124, 41, 117, 90, 77, 59, 98, 123, 43, 86, 17, 23, 115, 108, 79, 33, 27, 13, 10, 37, 88, 22, 44, 80, 107, 24, 81, 84, 8, 34, 0, 114, 30, 93, 64, 74, 15, 97, 50, 36, 38, 101, 100, 18, 28, 69, 68, 82, 76, 92, 20, 12, 73, 65, 11, 67, 29, 9, 1, 103, 4, 3, 99, 2, 96, 39, 71, 32, 70, 5, 25, 78, 66, 14, 89, 75, 72, 7], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 51, 61, 54, 106, 119, 58, 104, 122, 60, 40, 126, 56, 116, 85, 125, 52, 112, 21, 35, 62, 46, 87, 83, 121, 127, 31, 105, 113, 90, 26, 110, 91, 49, 55, 57, 117, 19, 108, 98, 27, 16, 102, 13, 94, 23, 41, 95, 123, 6, 59, 17, 77, 86, 34, 33, 50, 124, 80, 74, 8, 79, 88, 44, 64, 22, 115, 84, 81, 15, 37, 93, 114, 107, 70, 20, 36, 24, 0, 10, 97, 18, 28, 82, 38, 43, 30, 73, 92, 12, 69, 11, 29, 25, 101, 100, 76, 68, 103, 96, 2, 39, 99, 9, 65, 67, 3, 1, 78, 5, 14, 72, 89, 4, 71, 7, 75, 32, 66], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 106, 51, 119, 54, 61, 58, 104, 60, 122, 116, 56, 126, 40, 127, 21, 87, 125, 85, 62, 52, 83, 110, 113, 35, 90, 112, 94, 46, 49, 105, 33, 13, 102, 41, 31, 26, 91, 77, 117, 57, 98, 23, 108, 17, 19, 16, 124, 15, 121, 86, 59, 80, 34, 95, 79, 22, 55, 8, 44, 27, 70, 81, 115, 20, 88, 10, 84, 74, 64, 43, 37, 24, 0, 38, 123, 18, 114, 50, 30, 107, 6, 39, 28, 12, 97, 100, 36, 93, 82, 92, 96, 73, 3, 75, 4, 25, 9, 101, 5, 99, 76, 65, 29, 11, 66, 69, 103, 78, 68, 2, 1, 67, 71, 7, 14, 32, 72, 89], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 106, 51, 61, 119, 54, 58, 104, 122, 116, 56, 40, 94, 126, 60, 90, 87, 46, 83, 35, 52, 21, 125, 127, 85, 113, 110, 23, 112, 105, 62, 31, 57, 49, 102, 19, 55, 91, 121, 98, 117, 41, 27, 80, 77, 22, 33, 16, 95, 26, 124, 13, 70, 88, 59, 15, 24, 34, 84, 115, 108, 17, 86, 81, 8, 10, 79, 20, 18, 114, 44, 50, 38, 74, 30, 36, 123, 107, 37, 82, 28, 97, 64, 25, 93, 0, 43, 100, 39, 12, 96, 11, 76, 101, 99, 92, 66, 73, 14, 103, 69, 9, 32, 4, 5, 75, 71, 29, 89, 78, 3, 1, 68, 72, 65, 7, 6, 2, 67], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 61, 51, 106, 54, 119, 58, 122, 56, 60, 104, 126, 94, 40, 116, 35, 125, 52, 21, 105, 121, 85, 87, 31, 112, 110, 55, 83, 19, 113, 46, 102, 90, 62, 33, 95, 23, 98, 117, 57, 127, 49, 91, 27, 26, 70, 108, 80, 41, 22, 88, 115, 77, 13, 38, 16, 84, 36, 81, 79, 10, 24, 86, 59, 34, 114, 124, 28, 15, 17, 107, 74, 97, 44, 82, 20, 123, 30, 25, 0, 100, 50, 64, 43, 37, 93, 101, 8, 32, 12, 18, 99, 92, 76, 103, 96, 29, 72, 39, 66, 11, 75, 89, 9, 5, 4, 68, 14, 78, 73, 65, 3, 1, 67, 69, 7, 71, 2, 6], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 61, 51, 106, 119, 54, 58, 122, 104, 60, 56, 126, 116, 21, 94, 52, 121, 125, 40, 31, 117, 19, 85, 105, 110, 95, 35, 57, 62, 90, 87, 26, 113, 127, 112, 33, 23, 102, 46, 91, 83, 49, 16, 27, 59, 13, 55, 98, 80, 44, 114, 41, 77, 70, 108, 115, 22, 88, 50, 124, 17, 38, 24, 123, 34, 37, 10, 79, 84, 92, 86, 81, 43, 15, 36, 18, 97, 20, 74, 107, 30, 100, 28, 82, 64, 103, 93, 96, 0, 32, 101, 9, 76, 25, 89, 39, 12, 72, 99, 8, 75, 4, 73, 14, 29, 11, 68, 78, 5, 3, 2, 67, 7, 6, 69, 1, 65, 71, 66], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 61, 51, 54, 106, 119, 58, 60, 122, 104, 125, 116, 56, 55, 40, 52, 46, 31, 121, 126, 85, 62, 35, 94, 90, 112, 83, 113, 127, 21, 110, 33, 95, 105, 117, 87, 26, 102, 49, 59, 124, 70, 19, 115, 23, 98, 41, 50, 91, 77, 80, 57, 108, 27, 0, 10, 74, 16, 114, 86, 123, 13, 43, 15, 34, 22, 17, 38, 30, 81, 44, 24, 64, 84, 100, 28, 36, 79, 20, 37, 107, 88, 72, 82, 92, 93, 18, 9, 97, 6, 32, 8, 65, 25, 39, 68, 76, 73, 5, 1, 67, 75, 96, 101, 69, 4, 2, 29, 103, 3, 12, 7, 66, 99, 11, 14, 78, 89, 71], [120, 63, 118, 45, 109, 47, 42, 111, 53, 48, 61, 51, 119, 106, 54, 58, 104, 60, 122, 126, 40, 56, 116, 85, 125, 31, 87, 113, 117, 21, 94, 62, 55, 46, 105, 57, 127, 83, 26, 91, 35, 52, 90, 23, 112, 33, 80, 27, 95, 124, 19, 115, 110, 102, 13, 81, 121, 59, 77, 16, 74, 86, 49, 34, 6, 72, 79, 15, 98, 108, 114, 0, 22, 17, 41, 38, 24, 50, 10, 20, 43, 70, 44, 64, 88, 84, 30, 37, 18, 82, 36, 93, 28, 107, 123, 75, 100, 73, 12, 97, 68, 76, 92, 9, 3, 1, 101, 5, 29, 14, 99, 4, 67, 78, 8, 66, 11, 96, 32, 69, 2, 39, 89, 7, 103, 65, 71, 25], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 106, 61, 119, 51, 54, 104, 58, 56, 60, 122, 116, 126, 40, 31, 87, 21, 105, 85, 83, 113, 49, 19, 127, 35, 46, 23, 94, 55, 102, 117, 90, 52, 121, 125, 110, 6, 33, 91, 115, 112, 95, 77, 41, 26, 62, 80, 57, 34, 86, 124, 13, 98, 16, 79, 27, 72, 17, 59, 88, 50, 44, 108, 15, 114, 81, 22, 84, 74, 24, 123, 38, 10, 20, 0, 28, 43, 97, 93, 64, 30, 18, 75, 37, 76, 107, 99, 25, 73, 82, 92, 14, 100, 36, 12, 67, 68, 96, 9, 29, 101, 39, 65, 11, 78, 103, 2, 32, 3, 7, 70, 8, 89, 69, 66, 4, 5, 71, 1], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 61, 106, 51, 119, 54, 58, 56, 104, 60, 122, 116, 40, 105, 126, 121, 35, 52, 49, 125, 94, 112, 102, 110, 87, 23, 21, 57, 85, 90, 83, 113, 46, 31, 33, 19, 62, 41, 117, 26, 91, 95, 98, 115, 80, 55, 108, 27, 86, 127, 6, 124, 24, 15, 77, 22, 34, 79, 59, 38, 13, 50, 74, 88, 72, 123, 44, 114, 84, 16, 30, 17, 10, 100, 37, 81, 18, 20, 36, 28, 93, 43, 92, 64, 82, 97, 25, 0, 39, 75, 101, 107, 89, 12, 32, 96, 9, 14, 29, 76, 103, 99, 73, 4, 5, 1, 2, 68, 69, 66, 3, 78, 11, 8, 7, 67, 71, 65, 70], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 119, 106, 61, 54, 51, 58, 56, 60, 104, 122, 40, 126, 116, 62, 94, 125, 52, 46, 49, 31, 121, 117, 127, 21, 105, 114, 57, 85, 110, 87, 83, 113, 90, 6, 91, 98, 35, 102, 112, 23, 13, 19, 27, 26, 55, 95, 124, 33, 41, 80, 86, 59, 44, 16, 72, 24, 115, 74, 50, 84, 34, 123, 17, 108, 10, 77, 38, 36, 79, 0, 22, 43, 15, 88, 64, 81, 107, 37, 93, 82, 20, 30, 100, 18, 28, 4, 75, 92, 97, 76, 101, 32, 9, 25, 96, 2, 12, 29, 78, 5, 3, 39, 1, 68, 73, 14, 69, 67, 99, 65, 89, 103, 7, 11, 71, 66, 8, 70], [120, 63, 118, 45, 109, 47, 42, 111, 53, 48, 106, 61, 51, 54, 119, 58, 104, 60, 122, 116, 56, 126, 40, 94, 87, 52, 125, 21, 46, 31, 35, 62, 85, 83, 19, 105, 112, 121, 90, 113, 102, 127, 117, 26, 95, 57, 91, 55, 110, 27, 33, 124, 80, 49, 59, 114, 13, 81, 6, 41, 86, 15, 10, 108, 23, 50, 72, 34, 17, 64, 88, 16, 84, 98, 115, 44, 43, 36, 74, 77, 79, 123, 20, 100, 30, 37, 0, 38, 24, 73, 97, 75, 93, 12, 18, 22, 82, 32, 107, 28, 76, 39, 101, 92, 89, 69, 65, 68, 3, 5, 70, 4, 25, 9, 67, 1, 96, 2, 103, 29, 66, 78, 11, 14, 71, 99, 8, 7], [120, 63, 118, 45, 109, 47, 42, 111, 53, 48, 51, 106, 61, 119, 54, 58, 104, 56, 60, 122, 116, 126, 85, 87, 83, 62, 105, 40, 35, 21, 52, 102, 31, 125, 46, 94, 95, 113, 91, 19, 33, 55, 110, 57, 13, 23, 26, 121, 59, 112, 90, 127, 41, 80, 117, 108, 115, 34, 81, 124, 49, 16, 77, 27, 38, 79, 17, 15, 44, 123, 10, 86, 37, 22, 114, 50, 6, 43, 88, 84, 74, 98, 24, 72, 0, 36, 93, 20, 12, 75, 76, 92, 101, 82, 100, 28, 70, 30, 73, 18, 64, 9, 4, 97, 99, 107, 8, 1, 5, 96, 14, 68, 11, 39, 25, 3, 2, 32, 89, 69, 29, 78, 71, 66, 7, 103, 67, 65], [120, 118, 63, 45, 109, 47, 53, 42, 111, 48, 106, 51, 61, 54, 119, 58, 104, 122, 56, 60, 52, 116, 121, 46, 94, 126, 35, 21, 40, 95, 83, 102, 31, 49, 85, 23, 90, 110, 125, 62, 113, 41, 33, 55, 112, 57, 87, 19, 105, 26, 91, 98, 13, 80, 127, 86, 108, 115, 59, 124, 34, 27, 15, 16, 88, 117, 123, 24, 92, 44, 28, 10, 22, 17, 84, 74, 114, 70, 50, 36, 77, 81, 79, 30, 97, 20, 38, 0, 100, 93, 37, 82, 18, 64, 43, 101, 107, 72, 8, 12, 103, 9, 25, 99, 96, 32, 39, 29, 76, 69, 1, 4, 89, 66, 73, 6, 68, 78, 75, 14, 67, 11, 3, 7, 65, 2, 71, 5], [120, 63, 118, 45, 109, 47, 42, 111, 53, 48, 51, 61, 119, 106, 54, 58, 122, 104, 60, 56, 116, 94, 121, 40, 52, 62, 126, 85, 31, 35, 90, 21, 105, 19, 125, 49, 87, 110, 95, 102, 46, 83, 55, 23, 41, 112, 91, 59, 117, 26, 108, 113, 98, 57, 33, 27, 70, 24, 13, 127, 81, 114, 17, 44, 74, 77, 22, 16, 86, 15, 36, 79, 80, 43, 34, 124, 88, 50, 37, 38, 115, 100, 10, 84, 92, 123, 97, 28, 30, 0, 64, 20, 8, 12, 32, 101, 18, 107, 82, 96, 72, 9, 75, 1, 103, 25, 69, 76, 29, 93, 4, 78, 89, 99, 68, 67, 39, 11, 66, 7, 73, 14, 2, 5, 3, 6, 71, 65], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 106, 119, 51, 61, 54, 58, 104, 60, 122, 116, 40, 121, 52, 56, 125, 62, 126, 46, 35, 85, 19, 49, 110, 21, 105, 102, 90, 94, 31, 108, 95, 70, 87, 55, 117, 127, 124, 112, 59, 26, 113, 91, 33, 23, 57, 83, 41, 37, 123, 114, 98, 13, 115, 79, 50, 44, 38, 80, 16, 8, 81, 27, 10, 36, 17, 15, 34, 24, 88, 43, 84, 64, 0, 77, 22, 30, 86, 107, 74, 1, 20, 97, 100, 82, 92, 101, 75, 93, 28, 9, 99, 18, 32, 69, 12, 103, 96, 25, 2, 73, 4, 78, 76, 5, 39, 72, 89, 29, 67, 68, 11, 3, 65, 66, 7, 71, 6, 14], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 51, 106, 61, 119, 54, 58, 104, 60, 122, 116, 40, 52, 62, 35, 56, 113, 126, 85, 55, 46, 125, 94, 110, 102, 90, 21, 26, 121, 105, 31, 23, 70, 57, 83, 127, 41, 117, 33, 19, 95, 87, 59, 108, 112, 49, 124, 13, 91, 86, 16, 27, 123, 50, 8, 98, 15, 37, 77, 114, 80, 74, 44, 10, 17, 38, 24, 34, 115, 81, 88, 43, 36, 79, 64, 22, 30, 107, 84, 100, 97, 82, 0, 93, 73, 28, 20, 92, 32, 101, 12, 1, 68, 18, 66, 96, 4, 75, 76, 69, 103, 39, 3, 5, 9, 25, 67, 99, 14, 78, 29, 65, 71, 2, 72, 89, 7, 11, 6], [120, 63, 118, 45, 109, 47, 111, 42, 53, 48, 106, 61, 51, 54, 58, 119, 104, 60, 116, 122, 40, 56, 126, 85, 55, 125, 105, 52, 113, 94, 62, 46, 121, 21, 35, 83, 110, 90, 87, 127, 117, 31, 19, 57, 102, 95, 41, 49, 115, 26, 124, 16, 91, 23, 112, 77, 33, 108, 98, 13, 34, 27, 59, 81, 86, 15, 70, 123, 38, 74, 44, 8, 22, 80, 50, 43, 88, 10, 79, 17, 36, 84, 30, 114, 20, 92, 24, 100, 28, 37, 0, 75, 107, 99, 82, 25, 97, 39, 18, 101, 93, 12, 6, 64, 73, 32, 9, 76, 14, 68, 4, 5, 67, 1, 78, 66, 103, 3, 69, 96, 72, 11, 65, 71, 7, 89, 29, 2], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 51, 61, 106, 54, 119, 58, 122, 60, 104, 116, 121, 40, 126, 125, 56, 62, 110, 52, 46, 85, 113, 55, 21, 94, 127, 35, 105, 102, 112, 19, 117, 31, 124, 59, 87, 83, 26, 49, 90, 57, 95, 41, 43, 108, 44, 114, 23, 98, 91, 13, 38, 33, 123, 0, 27, 88, 77, 16, 80, 6, 34, 8, 10, 64, 86, 37, 15, 115, 36, 24, 74, 81, 100, 79, 50, 17, 107, 22, 28, 30, 93, 97, 1, 84, 32, 82, 20, 92, 101, 5, 70, 39, 18, 73, 68, 4, 66, 12, 9, 69, 76, 67, 2, 65, 103, 75, 11, 14, 96, 29, 99, 25, 71, 3, 89, 78, 7, 72], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 61, 106, 54, 51, 119, 58, 122, 56, 104, 60, 116, 40, 52, 117, 85, 126, 125, 105, 46, 35, 62, 121, 31, 94, 95, 21, 83, 87, 55, 102, 59, 49, 127, 57, 90, 113, 26, 110, 124, 112, 19, 6, 41, 114, 91, 27, 23, 33, 108, 98, 34, 44, 123, 16, 80, 77, 115, 88, 13, 15, 38, 10, 22, 36, 24, 28, 81, 84, 79, 8, 74, 50, 100, 37, 97, 86, 107, 30, 43, 0, 17, 93, 18, 92, 39, 82, 20, 101, 25, 64, 9, 4, 68, 103, 96, 76, 75, 1, 89, 99, 32, 12, 11, 69, 73, 78, 2, 3, 5, 67, 29, 71, 72, 66, 7, 65, 14, 70], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 51, 106, 61, 119, 54, 58, 104, 122, 60, 56, 116, 40, 126, 52, 21, 105, 94, 31, 85, 125, 113, 121, 35, 46, 87, 90, 83, 95, 110, 49, 57, 6, 55, 59, 62, 26, 102, 127, 41, 112, 33, 117, 124, 19, 23, 44, 108, 91, 114, 27, 86, 80, 77, 98, 22, 34, 16, 13, 88, 10, 81, 107, 15, 123, 17, 115, 74, 43, 8, 79, 37, 38, 84, 50, 30, 36, 20, 97, 18, 92, 100, 0, 93, 24, 82, 28, 64, 4, 76, 101, 12, 9, 73, 75, 96, 25, 5, 39, 11, 78, 68, 3, 14, 2, 103, 69, 66, 29, 32, 67, 72, 1, 65, 99, 7, 89, 71, 70], [120, 118, 63, 45, 109, 111, 47, 42, 53, 48, 51, 106, 54, 61, 119, 122, 104, 58, 60, 116, 40, 126, 56, 125, 52, 21, 83, 87, 90, 85, 94, 62, 105, 121, 35, 46, 124, 55, 6, 117, 27, 33, 113, 31, 49, 26, 102, 127, 95, 91, 110, 115, 19, 77, 112, 23, 80, 13, 86, 108, 41, 17, 59, 123, 57, 98, 16, 22, 88, 34, 79, 10, 15, 38, 84, 24, 44, 43, 74, 30, 81, 114, 50, 20, 8, 36, 64, 107, 18, 37, 28, 82, 93, 97, 0, 100, 25, 9, 92, 72, 101, 12, 99, 73, 75, 103, 78, 96, 32, 1, 76, 5, 66, 68, 69, 4, 11, 29, 3, 7, 14, 89, 65, 2, 39, 67, 71, 70], [120, 118, 63, 45, 109, 47, 111, 53, 42, 48, 51, 61, 106, 54, 119, 60, 122, 58, 104, 56, 116, 125, 21, 35, 40, 121, 52, 126, 87, 31, 62, 83, 94, 95, 46, 105, 55, 57, 85, 33, 113, 102, 117, 127, 26, 115, 49, 91, 23, 59, 110, 19, 90, 108, 123, 41, 124, 77, 114, 27, 13, 98, 22, 50, 44, 36, 112, 80, 38, 34, 88, 86, 24, 6, 16, 10, 15, 81, 84, 43, 30, 79, 37, 28, 17, 93, 0, 92, 74, 97, 100, 107, 18, 72, 32, 82, 101, 64, 20, 103, 76, 96, 25, 1, 29, 73, 8, 12, 9, 39, 68, 78, 75, 89, 11, 99, 14, 3, 69, 66, 4, 5, 67, 7, 70, 65, 71, 2], [120, 118, 63, 45, 47, 109, 42, 111, 53, 48, 54, 106, 61, 119, 51, 104, 122, 56, 58, 60, 126, 116, 31, 40, 94, 125, 35, 62, 117, 52, 21, 87, 121, 105, 127, 55, 83, 46, 85, 95, 90, 91, 115, 33, 102, 19, 113, 23, 41, 112, 57, 27, 26, 114, 110, 108, 13, 44, 77, 49, 124, 22, 59, 98, 80, 34, 16, 36, 81, 88, 123, 17, 72, 24, 38, 74, 37, 15, 86, 107, 100, 50, 10, 18, 79, 84, 43, 97, 92, 20, 25, 82, 30, 28, 101, 93, 64, 70, 6, 103, 96, 39, 12, 68, 0, 73, 29, 32, 4, 11, 99, 76, 69, 9, 1, 75, 3, 14, 67, 66, 78, 8, 89, 65, 7, 2, 71, 5], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 51, 61, 54, 106, 119, 58, 104, 60, 122, 116, 126, 56, 40, 52, 62, 21, 94, 125, 35, 46, 112, 127, 31, 85, 90, 87, 121, 102, 19, 105, 95, 108, 110, 55, 41, 83, 26, 23, 49, 117, 113, 33, 59, 70, 91, 123, 27, 13, 57, 72, 16, 98, 77, 124, 115, 44, 43, 79, 17, 34, 88, 80, 37, 50, 15, 86, 36, 22, 114, 74, 81, 10, 64, 84, 100, 0, 38, 30, 24, 20, 18, 39, 107, 92, 82, 28, 4, 93, 5, 66, 101, 97, 73, 76, 9, 1, 11, 32, 12, 25, 6, 69, 89, 65, 68, 75, 29, 67, 3, 7, 2, 78, 96, 14, 103, 99, 8, 71], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 106, 61, 51, 54, 119, 58, 122, 104, 60, 56, 116, 125, 40, 126, 46, 85, 55, 52, 62, 121, 35, 70, 49, 90, 31, 87, 117, 19, 105, 26, 83, 21, 57, 94, 112, 113, 110, 102, 95, 124, 91, 27, 108, 41, 17, 115, 50, 77, 33, 13, 64, 72, 34, 127, 23, 59, 16, 81, 44, 80, 86, 10, 43, 123, 38, 36, 84, 98, 0, 79, 15, 100, 74, 88, 22, 1, 37, 20, 28, 24, 107, 92, 9, 30, 93, 114, 82, 76, 101, 97, 11, 2, 5, 73, 25, 12, 18, 29, 96, 3, 67, 66, 39, 4, 103, 75, 68, 99, 69, 78, 32, 89, 14, 7, 65, 71, 8, 6], [120, 63, 118, 45, 109, 47, 42, 111, 53, 48, 51, 106, 61, 54, 119, 58, 60, 122, 104, 56, 125, 116, 126, 52, 94, 40, 85, 62, 90, 87, 21, 55, 46, 35, 117, 83, 31, 105, 49, 102, 70, 110, 113, 23, 57, 26, 123, 19, 121, 127, 91, 59, 77, 95, 33, 114, 112, 13, 41, 108, 27, 16, 34, 80, 124, 22, 81, 86, 44, 15, 10, 72, 98, 17, 84, 43, 74, 115, 38, 37, 36, 79, 88, 24, 50, 20, 100, 18, 107, 64, 97, 92, 82, 28, 101, 93, 76, 96, 0, 73, 30, 11, 25, 9, 4, 68, 12, 103, 39, 32, 5, 69, 99, 14, 89, 65, 3, 1, 78, 75, 67, 66, 29, 7, 71, 8, 2, 6], [120, 63, 118, 45, 109, 47, 42, 53, 111, 48, 51, 61, 106, 54, 119, 58, 60, 104, 56, 122, 125, 116, 126, 40, 94, 52, 62, 35, 121, 57, 85, 87, 105, 83, 46, 21, 110, 19, 26, 90, 31, 55, 113, 98, 49, 70, 102, 117, 91, 127, 95, 41, 112, 23, 33, 59, 124, 115, 123, 77, 86, 16, 72, 34, 27, 88, 79, 15, 44, 10, 81, 80, 74, 13, 108, 64, 114, 84, 17, 43, 22, 24, 0, 28, 38, 50, 36, 30, 20, 97, 92, 100, 82, 37, 93, 68, 107, 101, 4, 25, 18, 5, 76, 9, 39, 69, 73, 66, 67, 11, 78, 1, 32, 65, 96, 3, 14, 12, 71, 2, 8, 103, 75, 29, 99, 7, 89, 6], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 106, 61, 51, 54, 119, 58, 122, 60, 104, 56, 126, 40, 125, 116, 52, 85, 31, 35, 94, 121, 87, 83, 19, 90, 21, 117, 55, 62, 105, 57, 108, 26, 127, 95, 46, 110, 77, 102, 33, 112, 86, 113, 49, 23, 41, 91, 98, 27, 124, 13, 80, 16, 79, 38, 22, 44, 114, 115, 34, 15, 17, 74, 50, 10, 88, 59, 70, 81, 72, 36, 100, 24, 30, 25, 84, 20, 82, 123, 76, 97, 37, 92, 43, 93, 28, 18, 64, 11, 0, 12, 73, 107, 99, 96, 6, 101, 39, 32, 78, 4, 14, 9, 67, 69, 103, 66, 29, 89, 68, 8, 5, 65, 1, 75, 3, 71, 7, 2]], "model.layers.28.self_attn.q_proj": [[40, 120, 48, 34, 27, 51, 23, 85, 116, 126, 89, 53, 54, 16, 17, 112, 121, 3, 62, 31, 125, 124, 61, 93, 123, 50, 52, 109, 58, 114, 30, 49, 122, 19, 45, 110, 42, 7, 78, 29, 57, 82, 119, 60, 8, 115, 113, 59, 47, 55, 111, 63, 107, 117, 5, 118, 12, 56, 108, 46, 43, 79, 127, 74, 14, 44, 105, 88, 71, 100, 21, 20, 41, 65, 37, 64, 106, 11, 39, 1, 101, 38, 36, 87, 35, 91, 102, 0, 103, 22, 97, 66, 95, 6, 96, 10, 99, 98, 104, 84, 26, 32, 94, 2, 4, 90, 68, 73, 24, 77, 15, 13, 33, 9, 25, 28, 69, 18, 67, 83, 92, 80, 86, 72, 81, 75, 70, 76], [40, 120, 34, 48, 23, 27, 85, 17, 16, 31, 51, 11, 25, 13, 97, 61, 29, 62, 100, 55, 4, 78, 126, 19, 87, 56, 58, 73, 70, 59, 93, 28, 36, 35, 21, 113, 79, 46, 68, 22, 53, 18, 32, 96, 66, 109, 92, 10, 123, 95, 84, 8, 60, 63, 127, 0, 106, 49, 104, 37, 12, 112, 114, 116, 94, 26, 119, 99, 124, 7, 101, 30, 117, 83, 76, 103, 2, 118, 20, 44, 89, 52, 54, 125, 108, 81, 38, 33, 77, 110, 107, 15, 39, 102, 121, 111, 43, 122, 80, 86, 91, 47, 14, 57, 24, 90, 41, 82, 105, 50, 98, 75, 45, 74, 115, 5, 6, 88, 42, 67, 9, 72, 69, 71, 1, 65, 3, 64], [120, 40, 34, 48, 62, 23, 51, 27, 126, 85, 31, 93, 117, 109, 112, 58, 56, 113, 107, 16, 61, 50, 55, 44, 89, 123, 17, 19, 49, 53, 47, 8, 0, 30, 116, 106, 21, 95, 42, 52, 46, 25, 79, 43, 29, 108, 100, 121, 111, 119, 68, 110, 11, 57, 73, 38, 98, 54, 122, 59, 96, 36, 124, 118, 99, 63, 5, 103, 12, 105, 41, 78, 127, 125, 115, 94, 114, 97, 45, 35, 87, 37, 33, 60, 32, 39, 102, 22, 66, 101, 14, 24, 2, 70, 13, 10, 26, 86, 28, 91, 88, 92, 65, 18, 82, 67, 90, 6, 83, 84, 77, 4, 20, 64, 71, 75, 81, 104, 7, 1, 80, 69, 3, 15, 76, 9, 72, 74], [40, 120, 48, 34, 23, 63, 51, 31, 27, 62, 85, 49, 13, 123, 61, 108, 56, 79, 1, 110, 53, 55, 67, 93, 10, 64, 107, 16, 74, 113, 30, 19, 2, 17, 117, 70, 124, 18, 100, 59, 115, 127, 45, 68, 38, 26, 103, 102, 105, 111, 58, 46, 122, 119, 109, 52, 37, 116, 96, 36, 54, 24, 78, 106, 121, 112, 60, 50, 6, 84, 11, 125, 29, 114, 21, 118, 126, 0, 101, 57, 94, 43, 12, 73, 89, 88, 97, 47, 77, 39, 22, 35, 98, 95, 41, 5, 99, 33, 65, 44, 28, 32, 42, 86, 87, 92, 90, 8, 14, 72, 83, 25, 9, 82, 20, 69, 91, 3, 76, 7, 75, 80, 81, 15, 4, 71, 104, 66], [40, 125, 87, 17, 34, 21, 82, 70, 12, 74, 14, 4, 44, 0, 8, 124, 113, 111, 66, 54, 79, 58, 127, 68, 52, 109, 73, 110, 53, 19, 28, 75, 16, 33, 35, 123, 22, 59, 104, 27, 1, 95, 115, 119, 69, 86, 25, 114, 42, 50, 89, 18, 93, 108, 11, 26, 71, 96, 13, 90, 77, 56, 116, 67, 84, 63, 55, 5, 64, 62, 81, 78, 6, 92, 85, 106, 51, 7, 45, 39, 101, 23, 57, 103, 122, 98, 49, 94, 91, 88, 99, 32, 107, 48, 9, 2, 100, 29, 83, 105, 112, 65, 60, 24, 15, 76, 97, 3, 61, 47, 72, 38, 80, 36, 37, 121, 126, 46, 30, 20, 10, 31, 102, 118, 120, 41, 117, 43], [125, 40, 3, 87, 34, 12, 17, 14, 70, 0, 74, 1, 67, 66, 82, 21, 58, 64, 65, 8, 4, 69, 124, 54, 113, 25, 44, 111, 106, 52, 71, 49, 127, 2, 28, 75, 104, 109, 110, 50, 11, 42, 114, 22, 5, 79, 43, 26, 23, 121, 37, 68, 123, 101, 53, 86, 96, 31, 98, 112, 90, 27, 95, 77, 78, 20, 59, 13, 35, 122, 102, 81, 16, 29, 119, 6, 19, 91, 45, 39, 80, 47, 24, 32, 73, 117, 9, 76, 118, 103, 7, 88, 93, 84, 46, 72, 18, 33, 116, 55, 41, 30, 126, 105, 36, 115, 89, 62, 15, 120, 60, 63, 10, 57, 85, 48, 100, 38, 92, 107, 108, 94, 56, 83, 51, 97, 61, 99], [125, 40, 87, 34, 82, 21, 124, 12, 17, 14, 74, 109, 103, 50, 19, 4, 44, 8, 71, 95, 89, 92, 54, 70, 123, 127, 90, 9, 111, 68, 93, 108, 5, 39, 28, 115, 43, 41, 25, 77, 29, 110, 42, 105, 27, 61, 59, 100, 119, 58, 116, 64, 56, 66, 22, 79, 114, 88, 113, 47, 73, 121, 45, 91, 20, 60, 62, 106, 18, 75, 37, 49, 15, 10, 57, 52, 122, 31, 63, 51, 23, 48, 38, 33, 26, 96, 94, 6, 53, 32, 117, 86, 120, 112, 102, 36, 80, 97, 101, 72, 118, 83, 35, 69, 126, 99, 107, 46, 30, 85, 24, 55, 81, 13, 84, 78, 2, 98, 16, 7, 76, 11, 0, 1, 3, 67, 65, 104], [40, 125, 87, 0, 17, 34, 21, 12, 82, 1, 14, 66, 124, 4, 70, 50, 74, 44, 75, 58, 26, 127, 54, 69, 110, 113, 28, 111, 8, 95, 52, 13, 88, 33, 45, 79, 93, 42, 114, 64, 22, 103, 32, 98, 109, 67, 20, 123, 73, 119, 35, 11, 92, 19, 49, 104, 15, 39, 30, 27, 18, 90, 25, 59, 68, 96, 71, 102, 107, 53, 72, 101, 112, 62, 2, 65, 91, 60, 16, 77, 61, 80, 106, 99, 121, 115, 43, 7, 23, 105, 81, 84, 78, 55, 97, 3, 118, 76, 51, 122, 94, 83, 29, 89, 48, 117, 41, 86, 63, 9, 56, 5, 46, 108, 120, 31, 126, 24, 6, 116, 57, 85, 47, 100, 38, 10, 36, 37], [42, 36, 30, 43, 56, 106, 20, 118, 87, 77, 35, 82, 91, 123, 94, 46, 52, 80, 117, 49, 53, 89, 104, 120, 27, 99, 112, 17, 39, 47, 23, 122, 51, 78, 85, 54, 15, 11, 60, 19, 114, 103, 124, 71, 105, 125, 34, 58, 57, 95, 107, 108, 126, 110, 61, 62, 22, 74, 50, 32, 84, 109, 9, 121, 44, 88, 29, 13, 116, 101, 18, 55, 31, 59, 92, 115, 86, 45, 113, 98, 90, 111, 102, 41, 63, 48, 26, 12, 21, 25, 97, 119, 8, 24, 38, 127, 16, 37, 81, 83, 75, 14, 100, 33, 7, 96, 40, 28, 93, 72, 70, 69, 5, 76, 73, 79, 3, 67, 66, 2, 10, 68, 6, 1, 4, 65, 0, 64], [42, 36, 114, 118, 30, 15, 74, 35, 20, 17, 68, 87, 106, 77, 82, 70, 4, 91, 56, 76, 71, 0, 66, 123, 95, 64, 50, 31, 94, 43, 52, 21, 27, 23, 62, 7, 53, 11, 10, 122, 98, 9, 12, 29, 5, 67, 102, 84, 19, 18, 28, 111, 13, 65, 88, 1, 37, 85, 6, 8, 120, 2, 107, 32, 49, 112, 57, 103, 81, 83, 93, 14, 79, 69, 59, 96, 124, 46, 24, 44, 97, 26, 54, 126, 47, 109, 25, 16, 45, 73, 3, 60, 38, 72, 104, 86, 80, 78, 34, 108, 115, 41, 101, 58, 39, 22, 75, 61, 105, 92, 89, 48, 90, 51, 117, 33, 121, 119, 125, 110, 116, 63, 100, 113, 55, 127, 40, 99], [42, 36, 114, 118, 30, 87, 15, 123, 17, 35, 89, 74, 106, 20, 82, 94, 70, 56, 77, 50, 19, 66, 104, 52, 27, 68, 91, 8, 43, 6, 54, 72, 29, 44, 120, 58, 95, 75, 11, 47, 1, 38, 23, 51, 59, 122, 112, 113, 57, 49, 124, 61, 32, 86, 53, 62, 125, 99, 93, 111, 3, 116, 92, 102, 98, 12, 24, 103, 34, 71, 48, 28, 78, 64, 63, 76, 40, 127, 107, 108, 97, 46, 83, 119, 126, 96, 60, 110, 84, 10, 115, 81, 45, 25, 117, 37, 101, 0, 105, 21, 121, 2, 33, 31, 14, 109, 80, 55, 18, 26, 41, 9, 39, 85, 13, 16, 79, 5, 22, 90, 67, 88, 69, 100, 73, 7, 4, 65], [42, 114, 36, 35, 30, 106, 91, 103, 20, 87, 50, 27, 15, 17, 43, 74, 82, 94, 89, 68, 71, 123, 56, 104, 77, 95, 70, 120, 113, 85, 105, 65, 29, 62, 46, 3, 57, 127, 118, 61, 80, 119, 2, 125, 76, 122, 48, 117, 90, 1, 126, 47, 45, 111, 52, 44, 11, 51, 55, 109, 12, 49, 54, 121, 38, 75, 124, 63, 8, 102, 112, 67, 78, 107, 58, 66, 31, 59, 110, 115, 108, 116, 7, 72, 69, 81, 101, 41, 24, 86, 53, 33, 0, 73, 5, 26, 39, 28, 37, 19, 60, 13, 23, 21, 98, 84, 34, 93, 99, 14, 64, 92, 16, 9, 83, 96, 40, 25, 97, 88, 32, 22, 10, 4, 18, 79, 6, 100], [126, 41, 98, 56, 115, 89, 84, 13, 111, 80, 18, 112, 28, 45, 86, 75, 74, 31, 50, 21, 96, 82, 54, 58, 79, 5, 73, 8, 29, 81, 55, 11, 14, 70, 63, 120, 68, 4, 3, 25, 106, 119, 127, 117, 77, 40, 60, 71, 122, 92, 48, 10, 95, 88, 66, 46, 57, 59, 6, 76, 78, 83, 108, 34, 2, 93, 43, 51, 72, 16, 22, 0, 19, 20, 35, 124, 23, 44, 102, 121, 125, 62, 87, 107, 1, 113, 100, 61, 52, 24, 27, 85, 12, 90, 26, 17, 123, 49, 47, 105, 9, 114, 118, 32, 116, 103, 36, 7, 53, 99, 91, 110, 67, 109, 104, 33, 38, 94, 30, 97, 37, 39, 101, 65, 42, 64, 15, 69], [56, 41, 58, 98, 87, 63, 31, 32, 126, 28, 54, 122, 115, 44, 57, 17, 53, 60, 121, 46, 112, 43, 103, 55, 124, 116, 108, 127, 50, 120, 51, 26, 47, 117, 111, 86, 48, 113, 62, 52, 110, 61, 118, 119, 49, 45, 92, 114, 123, 109, 59, 125, 89, 101, 105, 40, 95, 107, 42, 23, 88, 22, 39, 91, 24, 100, 106, 84, 25, 94, 34, 30, 104, 37, 82, 15, 102, 38, 97, 99, 36, 35, 90, 21, 93, 80, 96, 33, 83, 29, 18, 20, 81, 79, 12, 27, 78, 19, 10, 85, 77, 1, 76, 14, 13, 7, 72, 66, 0, 5, 67, 3, 65, 16, 2, 64, 69, 11, 75, 71, 4, 8, 73, 70, 9, 74, 6, 68], [56, 41, 126, 98, 89, 79, 84, 18, 115, 80, 15, 31, 21, 28, 72, 13, 58, 10, 11, 63, 86, 82, 46, 4, 57, 40, 14, 73, 112, 70, 92, 77, 111, 100, 25, 23, 32, 122, 3, 121, 113, 60, 43, 127, 119, 75, 81, 16, 45, 87, 125, 95, 53, 66, 29, 62, 55, 109, 96, 51, 19, 124, 69, 5, 50, 52, 120, 9, 49, 71, 123, 1, 0, 48, 59, 22, 20, 47, 33, 76, 116, 8, 85, 6, 44, 27, 117, 103, 93, 17, 30, 61, 78, 37, 110, 64, 114, 88, 118, 24, 35, 2, 12, 91, 94, 83, 39, 90, 104, 54, 105, 99, 26, 102, 108, 68, 101, 7, 65, 97, 106, 36, 107, 42, 38, 74, 34, 67], [41, 126, 56, 63, 105, 60, 31, 127, 122, 87, 51, 52, 115, 98, 47, 49, 62, 121, 57, 123, 112, 124, 58, 46, 28, 117, 103, 114, 59, 116, 118, 45, 109, 86, 119, 54, 61, 113, 48, 55, 111, 120, 92, 50, 32, 53, 125, 44, 110, 108, 89, 84, 40, 43, 42, 107, 18, 39, 106, 104, 26, 100, 101, 37, 38, 102, 23, 7, 36, 35, 94, 96, 34, 91, 95, 99, 97, 17, 33, 80, 30, 25, 21, 85, 82, 88, 20, 22, 93, 24, 29, 27, 90, 11, 19, 83, 78, 77, 5, 73, 14, 81, 12, 3, 70, 0, 8, 71, 76, 79, 1, 16, 65, 10, 13, 67, 66, 64, 2, 74, 69, 9, 4, 72, 68, 75, 6, 15], [123, 103, 55, 47, 34, 125, 27, 95, 88, 32, 110, 122, 21, 15, 76, 19, 127, 24, 53, 97, 91, 94, 18, 36, 71, 51, 59, 43, 104, 38, 40, 101, 44, 118, 74, 106, 42, 26, 58, 102, 57, 52, 90, 117, 9, 92, 115, 111, 62, 2, 124, 96, 10, 87, 29, 48, 116, 33, 16, 126, 22, 78, 120, 112, 105, 85, 60, 63, 46, 107, 100, 114, 41, 113, 25, 1, 45, 54, 121, 37, 99, 23, 14, 31, 98, 49, 108, 56, 93, 39, 89, 50, 61, 109, 5, 79, 70, 119, 68, 13, 35, 28, 84, 82, 83, 86, 81, 30, 20, 80, 66, 64, 17, 11, 77, 12, 8, 73, 75, 3, 69, 4, 72, 7, 6, 65, 0, 67], [123, 103, 55, 34, 116, 27, 76, 88, 70, 0, 3, 18, 15, 1, 19, 68, 122, 66, 21, 95, 99, 71, 94, 91, 22, 59, 51, 9, 62, 125, 74, 56, 30, 73, 77, 67, 33, 126, 10, 47, 31, 37, 96, 109, 36, 57, 6, 79, 8, 107, 48, 80, 110, 29, 49, 35, 45, 118, 115, 58, 50, 63, 44, 121, 65, 86, 24, 26, 60, 17, 82, 85, 92, 23, 119, 40, 20, 52, 104, 13, 98, 112, 5, 120, 28, 113, 117, 114, 69, 72, 11, 102, 124, 81, 78, 14, 38, 53, 46, 43, 93, 61, 39, 87, 16, 108, 105, 75, 111, 42, 54, 106, 41, 25, 84, 32, 90, 97, 83, 101, 127, 4, 89, 100, 7, 12, 2, 64], [123, 103, 58, 122, 27, 88, 21, 91, 47, 18, 15, 59, 38, 34, 62, 9, 5, 95, 48, 120, 55, 49, 125, 60, 110, 44, 53, 121, 50, 56, 117, 119, 30, 105, 57, 52, 63, 24, 124, 67, 40, 113, 126, 94, 109, 112, 96, 51, 54, 118, 97, 46, 45, 116, 107, 111, 76, 108, 114, 85, 42, 115, 100, 43, 106, 104, 61, 36, 127, 101, 99, 41, 75, 29, 0, 92, 71, 86, 33, 102, 2, 64, 4, 37, 14, 39, 35, 17, 81, 3, 98, 73, 93, 82, 69, 77, 7, 11, 32, 87, 79, 68, 26, 84, 72, 89, 31, 8, 25, 65, 20, 83, 80, 90, 23, 22, 13, 28, 6, 78, 16, 19, 74, 1, 10, 12, 70, 66], [123, 103, 47, 34, 71, 76, 62, 122, 27, 55, 21, 22, 59, 19, 88, 125, 68, 15, 18, 101, 91, 109, 97, 45, 58, 24, 2, 56, 48, 64, 95, 117, 63, 36, 32, 38, 85, 111, 94, 66, 92, 77, 107, 10, 84, 9, 110, 30, 44, 119, 127, 5, 115, 50, 87, 79, 82, 73, 20, 99, 40, 124, 83, 49, 51, 42, 75, 61, 116, 57, 41, 46, 89, 60, 126, 120, 52, 54, 70, 118, 104, 113, 29, 112, 114, 53, 35, 121, 26, 105, 1, 93, 0, 100, 37, 108, 106, 80, 33, 65, 98, 12, 17, 43, 16, 72, 86, 74, 90, 23, 25, 96, 14, 31, 11, 28, 81, 4, 102, 78, 7, 13, 8, 6, 3, 69, 67, 39], [103, 61, 95, 60, 23, 104, 54, 91, 18, 116, 114, 27, 127, 80, 20, 108, 51, 40, 98, 62, 7, 46, 56, 78, 123, 107, 121, 122, 55, 113, 57, 12, 97, 53, 117, 109, 59, 58, 42, 75, 125, 124, 112, 86, 74, 126, 47, 41, 45, 63, 22, 115, 25, 65, 120, 49, 52, 69, 119, 50, 101, 28, 31, 44, 43, 118, 48, 99, 110, 105, 81, 39, 87, 38, 32, 24, 111, 102, 106, 93, 37, 85, 36, 79, 88, 100, 19, 34, 35, 21, 33, 16, 29, 84, 9, 83, 30, 90, 13, 82, 5, 96, 92, 8, 66, 94, 15, 89, 6, 70, 2, 72, 67, 26, 77, 76, 3, 17, 73, 1, 68, 71, 14, 4, 11, 0, 10, 64], [103, 61, 95, 113, 23, 104, 27, 91, 49, 42, 108, 60, 78, 114, 80, 18, 12, 74, 109, 7, 54, 127, 57, 21, 20, 34, 52, 58, 53, 45, 105, 112, 101, 2, 6, 56, 47, 86, 59, 69, 121, 117, 1, 70, 51, 39, 107, 126, 46, 43, 3, 64, 31, 119, 85, 116, 19, 62, 125, 32, 88, 72, 122, 67, 97, 13, 40, 92, 102, 65, 123, 8, 41, 120, 111, 48, 106, 124, 55, 118, 63, 44, 4, 115, 99, 25, 96, 66, 0, 22, 28, 9, 11, 38, 83, 50, 17, 110, 30, 100, 84, 35, 89, 68, 73, 90, 36, 81, 98, 33, 77, 15, 71, 37, 93, 24, 94, 29, 26, 79, 87, 75, 5, 14, 16, 10, 82, 76], [103, 61, 95, 74, 18, 86, 69, 27, 12, 78, 23, 48, 60, 64, 7, 3, 20, 54, 66, 113, 65, 22, 80, 123, 0, 127, 6, 21, 98, 33, 47, 72, 114, 38, 82, 68, 121, 73, 85, 35, 49, 75, 29, 81, 43, 94, 62, 16, 31, 88, 84, 19, 104, 53, 106, 34, 26, 32, 55, 56, 90, 119, 87, 44, 124, 14, 118, 101, 83, 24, 15, 46, 13, 2, 97, 17, 10, 93, 28, 25, 77, 92, 108, 51, 96, 100, 58, 111, 5, 99, 50, 89, 30, 11, 36, 116, 110, 40, 115, 57, 102, 67, 79, 42, 105, 45, 52, 107, 76, 126, 70, 122, 63, 120, 1, 59, 117, 8, 125, 71, 112, 41, 37, 109, 91, 9, 4, 39], [103, 61, 60, 55, 23, 95, 54, 91, 27, 18, 56, 78, 20, 113, 7, 53, 51, 114, 98, 123, 121, 12, 116, 107, 104, 69, 41, 58, 46, 63, 108, 59, 40, 109, 126, 62, 80, 49, 74, 65, 97, 124, 22, 117, 52, 45, 112, 86, 57, 75, 127, 115, 47, 105, 42, 122, 38, 25, 125, 44, 101, 37, 110, 119, 34, 118, 43, 50, 106, 6, 66, 32, 102, 48, 39, 87, 111, 31, 71, 93, 83, 120, 33, 35, 28, 29, 4, 79, 36, 21, 17, 96, 100, 68, 72, 99, 30, 16, 67, 84, 19, 15, 76, 92, 88, 89, 3, 13, 24, 14, 90, 5, 85, 82, 73, 77, 81, 9, 0, 1, 94, 8, 26, 70, 10, 11, 2, 64], [121, 114, 39, 56, 120, 33, 93, 116, 19, 101, 29, 57, 113, 126, 63, 77, 49, 103, 109, 51, 110, 127, 54, 122, 111, 53, 119, 58, 50, 66, 105, 23, 59, 125, 117, 115, 43, 55, 124, 52, 46, 48, 61, 118, 41, 112, 123, 22, 108, 62, 45, 60, 107, 8, 65, 47, 25, 89, 42, 106, 24, 74, 44, 1, 104, 98, 37, 15, 69, 102, 40, 38, 32, 72, 26, 99, 3, 35, 90, 97, 67, 28, 96, 100, 91, 36, 88, 34, 79, 94, 7, 17, 2, 16, 83, 92, 18, 86, 95, 81, 10, 30, 31, 6, 84, 13, 85, 71, 87, 27, 20, 5, 21, 76, 9, 11, 68, 78, 64, 70, 80, 82, 0, 75, 4, 73, 14, 12], [56, 39, 114, 33, 88, 85, 16, 78, 18, 26, 120, 76, 71, 29, 91, 50, 11, 31, 20, 121, 116, 100, 115, 94, 119, 58, 84, 80, 97, 6, 9, 54, 24, 57, 37, 19, 51, 22, 15, 124, 110, 113, 75, 81, 79, 126, 99, 95, 83, 87, 111, 108, 13, 90, 92, 25, 27, 61, 34, 44, 7, 122, 63, 125, 52, 28, 96, 89, 127, 53, 30, 72, 60, 77, 82, 93, 41, 74, 21, 48, 86, 123, 55, 117, 59, 49, 10, 40, 98, 14, 17, 43, 105, 101, 23, 62, 68, 36, 32, 12, 109, 102, 35, 47, 107, 118, 45, 42, 106, 112, 104, 38, 46, 69, 1, 73, 70, 8, 65, 3, 66, 67, 103, 5, 4, 2, 64, 0], [39, 56, 114, 0, 64, 9, 121, 120, 68, 6, 11, 67, 78, 16, 33, 65, 88, 66, 71, 4, 19, 29, 3, 85, 1, 76, 77, 2, 18, 22, 73, 12, 70, 86, 75, 20, 5, 87, 83, 8, 81, 7, 10, 72, 90, 99, 110, 31, 27, 116, 13, 14, 91, 69, 108, 17, 21, 41, 23, 80, 98, 58, 26, 25, 113, 79, 96, 74, 61, 35, 84, 52, 54, 95, 119, 82, 94, 107, 89, 103, 125, 15, 28, 102, 30, 57, 92, 24, 118, 34, 59, 32, 117, 49, 100, 44, 53, 115, 93, 63, 36, 42, 127, 97, 105, 123, 101, 104, 62, 40, 38, 50, 124, 37, 51, 43, 47, 106, 60, 48, 46, 109, 55, 111, 122, 45, 126, 112], [114, 39, 56, 120, 116, 29, 57, 117, 126, 93, 54, 119, 19, 113, 51, 111, 122, 49, 110, 58, 63, 50, 127, 101, 53, 23, 125, 109, 33, 55, 60, 59, 62, 118, 123, 61, 115, 52, 121, 46, 48, 47, 124, 43, 112, 45, 103, 107, 108, 22, 44, 42, 105, 41, 106, 37, 104, 27, 97, 38, 40, 86, 98, 83, 91, 26, 102, 89, 31, 88, 99, 36, 35, 25, 77, 15, 100, 34, 20, 87, 95, 84, 24, 32, 96, 90, 17, 30, 85, 18, 81, 94, 79, 28, 13, 78, 92, 16, 71, 6, 69, 11, 21, 10, 80, 74, 7, 82, 5, 65, 72, 9, 14, 1, 76, 8, 12, 75, 68, 67, 66, 70, 73, 2, 3, 4, 64, 0], [46, 102, 49, 110, 26, 84, 16, 53, 29, 123, 60, 126, 98, 10, 119, 90, 25, 91, 44, 56, 124, 113, 62, 41, 112, 86, 63, 127, 42, 92, 59, 57, 33, 47, 118, 107, 31, 54, 125, 6, 87, 115, 80, 114, 66, 104, 95, 105, 20, 99, 111, 120, 55, 61, 43, 52, 24, 58, 116, 36, 108, 117, 45, 18, 7, 122, 93, 39, 51, 48, 109, 35, 97, 121, 37, 50, 30, 32, 106, 100, 64, 103, 27, 2, 13, 74, 40, 1, 22, 101, 81, 88, 72, 28, 94, 21, 78, 70, 96, 34, 69, 0, 83, 38, 85, 15, 17, 89, 11, 12, 3, 23, 68, 73, 19, 75, 67, 76, 5, 14, 71, 79, 65, 4, 9, 8, 82, 77], [46, 102, 110, 29, 26, 60, 84, 16, 86, 51, 69, 18, 100, 123, 109, 67, 121, 36, 112, 49, 113, 1, 63, 97, 90, 11, 13, 50, 31, 53, 33, 108, 7, 61, 115, 30, 47, 107, 28, 12, 105, 119, 37, 59, 106, 114, 42, 120, 99, 45, 52, 98, 87, 124, 104, 89, 125, 92, 41, 116, 55, 111, 39, 103, 10, 57, 58, 127, 117, 62, 56, 35, 44, 54, 122, 40, 126, 64, 6, 38, 101, 85, 76, 93, 27, 81, 14, 32, 24, 25, 4, 43, 95, 48, 34, 73, 118, 96, 70, 3, 66, 91, 9, 17, 19, 83, 72, 88, 94, 78, 22, 79, 20, 80, 2, 74, 68, 21, 71, 15, 23, 5, 75, 82, 8, 65, 77, 0], [46, 102, 110, 26, 84, 18, 29, 13, 72, 16, 67, 116, 10, 109, 68, 107, 121, 113, 59, 0, 47, 6, 90, 112, 60, 98, 115, 48, 24, 87, 22, 123, 120, 8, 12, 11, 49, 127, 30, 73, 23, 106, 126, 41, 119, 34, 61, 40, 37, 35, 66, 97, 124, 108, 100, 114, 42, 93, 25, 85, 103, 53, 33, 81, 80, 89, 63, 55, 31, 104, 105, 74, 50, 44, 117, 95, 45, 3, 111, 21, 57, 36, 43, 62, 92, 58, 125, 88, 86, 122, 20, 54, 83, 96, 70, 39, 7, 75, 91, 28, 52, 118, 14, 27, 101, 99, 51, 56, 32, 19, 9, 2, 69, 17, 94, 82, 64, 79, 5, 77, 65, 1, 76, 4, 71, 15, 78, 38], [46, 102, 18, 110, 13, 29, 84, 26, 16, 10, 72, 6, 68, 93, 25, 87, 1, 73, 90, 60, 0, 2, 22, 91, 78, 7, 14, 49, 9, 88, 15, 82, 79, 36, 59, 33, 112, 20, 17, 30, 123, 109, 27, 31, 19, 116, 67, 65, 8, 71, 92, 96, 85, 3, 80, 61, 76, 81, 24, 11, 77, 98, 113, 83, 50, 5, 107, 28, 106, 23, 118, 54, 101, 86, 37, 75, 34, 12, 66, 89, 63, 74, 41, 56, 48, 4, 108, 32, 35, 115, 99, 94, 21, 100, 119, 62, 57, 114, 70, 95, 52, 58, 124, 69, 39, 117, 120, 44, 121, 127, 105, 43, 55, 64, 126, 104, 40, 53, 103, 47, 42, 122, 125, 97, 51, 45, 111, 38]], "model.layers.28.self_attn.k_proj": [[120, 104, 98, 23, 85, 0, 29, 11, 17, 16, 27, 61, 58, 30, 19, 95, 110, 73, 78, 125, 66, 114, 91, 108, 42, 50, 115, 51, 62, 52, 68, 121, 127, 57, 47, 124, 60, 45, 59, 53, 122, 55, 49, 70, 43, 67, 112, 119, 109, 65, 117, 54, 123, 116, 48, 102, 113, 56, 44, 8, 118, 111, 63, 38, 4, 46, 79, 105, 107, 89, 106, 5, 12, 126, 41, 25, 77, 2, 39, 36, 86, 88, 13, 7, 82, 31, 92, 35, 103, 101, 100, 37, 10, 90, 33, 28, 71, 34, 84, 26, 24, 1, 32, 22, 94, 20, 99, 96, 9, 72, 6, 15, 97, 83, 18, 81, 3, 93, 80, 14, 21, 74, 75, 69, 76, 64, 87, 40], [125, 104, 64, 87, 98, 14, 70, 74, 17, 12, 82, 0, 21, 65, 8, 68, 66, 124, 108, 58, 69, 3, 1, 90, 4, 114, 47, 79, 28, 49, 2, 54, 19, 127, 73, 9, 75, 46, 50, 123, 61, 93, 52, 25, 16, 77, 103, 111, 31, 67, 41, 113, 53, 106, 92, 20, 109, 89, 45, 122, 105, 27, 97, 107, 71, 94, 60, 33, 13, 55, 59, 121, 120, 86, 26, 11, 63, 99, 29, 95, 91, 51, 101, 119, 5, 62, 118, 88, 83, 112, 7, 102, 24, 80, 22, 115, 56, 38, 42, 44, 48, 37, 32, 116, 110, 57, 15, 36, 84, 100, 23, 117, 43, 39, 76, 30, 35, 126, 85, 96, 6, 72, 34, 18, 10, 78, 81, 40], [106, 114, 100, 94, 118, 87, 68, 0, 99, 71, 77, 74, 20, 82, 15, 17, 50, 70, 52, 119, 42, 27, 47, 44, 115, 62, 75, 89, 56, 66, 123, 1, 122, 109, 102, 120, 76, 2, 110, 53, 57, 45, 48, 9, 65, 41, 43, 107, 121, 39, 35, 127, 95, 73, 29, 54, 59, 91, 31, 58, 124, 61, 72, 69, 51, 22, 113, 49, 28, 12, 111, 93, 85, 63, 97, 60, 3, 125, 46, 40, 116, 96, 30, 98, 126, 108, 117, 33, 55, 34, 32, 37, 5, 112, 101, 11, 92, 64, 19, 80, 104, 78, 103, 90, 14, 7, 83, 21, 86, 24, 88, 8, 16, 67, 105, 38, 26, 4, 81, 25, 79, 18, 6, 10, 84, 36, 23, 13], [105, 126, 56, 34, 95, 86, 92, 89, 115, 84, 82, 79, 57, 119, 51, 62, 110, 63, 87, 54, 59, 80, 120, 112, 58, 121, 113, 127, 117, 55, 124, 116, 118, 109, 48, 60, 123, 61, 122, 46, 49, 114, 52, 53, 47, 125, 111, 45, 32, 104, 44, 107, 99, 50, 108, 21, 17, 91, 39, 101, 43, 26, 106, 42, 103, 96, 11, 36, 77, 27, 28, 102, 30, 13, 38, 40, 97, 33, 37, 73, 29, 8, 22, 41, 76, 35, 10, 19, 93, 75, 100, 98, 14, 94, 70, 85, 18, 88, 24, 12, 20, 31, 16, 23, 90, 25, 83, 4, 5, 3, 78, 81, 0, 72, 9, 65, 74, 7, 71, 68, 1, 6, 66, 15, 69, 67, 64, 2], [123, 39, 88, 21, 27, 64, 76, 15, 18, 30, 1, 68, 98, 71, 122, 95, 70, 125, 9, 47, 63, 121, 66, 46, 19, 57, 60, 86, 116, 110, 2, 49, 119, 120, 118, 33, 113, 3, 45, 52, 117, 44, 112, 48, 51, 124, 40, 10, 50, 54, 59, 114, 20, 69, 111, 31, 102, 37, 126, 8, 108, 61, 56, 106, 109, 115, 55, 34, 43, 87, 53, 62, 13, 42, 14, 41, 127, 58, 104, 29, 107, 105, 32, 101, 100, 11, 36, 75, 93, 92, 23, 90, 28, 35, 99, 22, 26, 74, 89, 97, 25, 96, 84, 65, 77, 17, 80, 81, 0, 16, 94, 38, 82, 78, 79, 103, 72, 4, 5, 6, 7, 83, 67, 73, 91, 24, 12, 85], [61, 39, 31, 23, 91, 18, 64, 74, 80, 78, 86, 62, 69, 49, 56, 12, 54, 3, 51, 112, 66, 7, 55, 113, 110, 124, 121, 109, 50, 20, 52, 117, 114, 58, 0, 34, 122, 59, 60, 123, 115, 42, 111, 126, 118, 107, 63, 53, 127, 116, 44, 41, 108, 45, 79, 47, 57, 35, 25, 46, 98, 65, 106, 6, 48, 88, 119, 75, 1, 43, 40, 72, 120, 33, 125, 21, 32, 105, 67, 68, 19, 37, 100, 36, 38, 92, 70, 73, 17, 2, 102, 96, 97, 81, 99, 27, 90, 94, 101, 103, 4, 11, 84, 104, 77, 93, 29, 9, 28, 15, 24, 13, 22, 89, 26, 83, 30, 85, 16, 5, 8, 10, 76, 14, 71, 95, 82, 87], [56, 103, 64, 114, 93, 121, 97, 1, 50, 19, 9, 68, 6, 22, 120, 67, 116, 11, 23, 113, 78, 66, 77, 16, 46, 88, 57, 58, 110, 54, 3, 69, 76, 71, 51, 122, 105, 118, 126, 43, 95, 8, 62, 61, 117, 24, 55, 59, 47, 49, 53, 123, 125, 42, 119, 63, 60, 44, 48, 115, 85, 124, 12, 26, 37, 52, 109, 18, 112, 2, 45, 74, 127, 111, 108, 41, 89, 106, 21, 107, 40, 81, 38, 65, 101, 91, 14, 15, 35, 36, 84, 104, 0, 99, 102, 92, 4, 34, 100, 94, 30, 28, 96, 25, 32, 17, 20, 31, 98, 72, 80, 10, 75, 90, 70, 82, 27, 79, 13, 5, 87, 73, 7, 86, 29, 39, 33, 83], [110, 38, 46, 29, 84, 26, 13, 16, 18, 6, 2, 0, 112, 10, 25, 121, 123, 68, 11, 113, 73, 86, 1, 47, 7, 97, 72, 63, 60, 109, 115, 34, 119, 53, 114, 30, 59, 55, 61, 124, 32, 41, 103, 107, 101, 88, 49, 50, 116, 42, 87, 51, 104, 62, 37, 57, 111, 108, 58, 127, 120, 45, 56, 122, 118, 40, 44, 105, 39, 125, 117, 54, 52, 64, 92, 43, 106, 95, 15, 76, 48, 14, 69, 36, 96, 91, 4, 35, 99, 31, 100, 126, 98, 33, 75, 22, 67, 23, 5, 94, 81, 19, 24, 85, 17, 78, 12, 27, 83, 8, 28, 3, 89, 21, 65, 9, 79, 71, 93, 82, 77, 74, 80, 66, 70, 90, 20, 102]], "model.layers.28.self_attn.qk_proj": [[123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 104, 23, 87, 121, 118, 103, 82, 18, 91, 64, 0, 29, 95, 58, 39, 40, 50, 85, 41, 27, 10, 70, 20, 76, 74, 105, 84, 30, 12, 55, 62, 98, 93, 21, 81, 51, 68, 115, 80, 79, 13, 77, 122, 124, 44, 90, 6, 60, 15, 4, 48, 112, 59, 113, 54, 31, 116, 78, 49, 57, 47, 53, 109, 66, 65, 17, 127, 16, 2, 22, 14, 63, 28, 102, 24, 11, 34, 1, 83, 9, 86, 43, 73, 52, 19, 38, 25, 7, 88, 108, 36, 8, 119, 94, 89, 26, 33, 107, 3, 71, 117, 75, 5, 45, 67, 111, 99, 97, 35, 101, 100, 72, 69, 32, 37, 92, 96], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 104, 121, 23, 87, 103, 118, 91, 0, 82, 64, 18, 40, 70, 95, 50, 58, 29, 30, 47, 10, 62, 39, 27, 20, 85, 105, 4, 41, 76, 12, 93, 51, 98, 81, 55, 74, 21, 60, 84, 13, 49, 77, 44, 115, 63, 53, 80, 65, 124, 66, 116, 34, 112, 113, 22, 54, 79, 102, 17, 16, 2, 6, 68, 11, 14, 78, 90, 15, 122, 28, 1, 9, 109, 59, 127, 48, 8, 108, 52, 31, 38, 57, 43, 71, 24, 86, 88, 83, 73, 33, 117, 7, 107, 111, 119, 19, 94, 45, 5, 36, 100, 3, 89, 75, 69, 26, 67, 35, 25, 37, 97, 99, 72, 101, 92, 32, 96], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 121, 104, 23, 87, 0, 118, 64, 18, 103, 82, 91, 70, 39, 95, 27, 58, 41, 40, 98, 105, 29, 12, 21, 84, 76, 10, 85, 112, 30, 50, 74, 93, 55, 20, 115, 44, 53, 4, 62, 63, 80, 34, 81, 77, 124, 60, 15, 68, 79, 113, 22, 13, 66, 2, 90, 57, 51, 49, 78, 47, 17, 54, 102, 65, 122, 16, 14, 1, 116, 8, 48, 59, 28, 31, 119, 127, 86, 109, 19, 6, 43, 71, 36, 88, 9, 107, 38, 108, 52, 7, 25, 11, 117, 111, 24, 94, 67, 89, 26, 75, 3, 73, 83, 33, 5, 69, 99, 101, 100, 97, 37, 45, 35, 92, 96, 32, 72], [123, 125, 56, 61, 120, 114, 110, 46, 126, 106, 42, 104, 121, 23, 103, 87, 118, 64, 0, 18, 91, 82, 70, 40, 39, 41, 74, 76, 58, 95, 53, 105, 12, 27, 50, 29, 10, 98, 21, 84, 112, 122, 62, 4, 63, 81, 30, 124, 55, 2, 77, 14, 78, 85, 34, 93, 80, 1, 20, 44, 68, 66, 113, 16, 17, 115, 60, 22, 13, 15, 54, 6, 79, 8, 90, 49, 57, 51, 59, 65, 47, 25, 116, 31, 7, 86, 28, 11, 48, 36, 33, 19, 67, 9, 83, 119, 71, 102, 52, 38, 88, 94, 73, 107, 109, 3, 89, 69, 101, 127, 117, 108, 24, 75, 43, 45, 5, 26, 97, 111, 35, 99, 100, 37, 92, 72, 32, 96], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 104, 121, 23, 87, 103, 118, 18, 0, 82, 64, 40, 58, 41, 70, 91, 50, 53, 39, 29, 105, 55, 12, 76, 95, 98, 112, 21, 27, 80, 74, 10, 84, 20, 85, 30, 6, 78, 77, 113, 4, 81, 62, 115, 15, 122, 60, 124, 68, 66, 54, 48, 14, 2, 57, 44, 59, 90, 86, 116, 16, 47, 79, 65, 17, 8, 28, 63, 93, 49, 13, 51, 127, 1, 25, 31, 11, 34, 19, 22, 7, 107, 117, 9, 109, 24, 52, 108, 102, 83, 36, 73, 119, 67, 88, 3, 38, 71, 94, 111, 75, 5, 26, 69, 89, 33, 101, 43, 45, 97, 100, 35, 99, 92, 72, 37, 96, 32], [123, 125, 56, 61, 120, 114, 110, 46, 126, 106, 42, 104, 121, 23, 103, 87, 18, 82, 118, 64, 0, 50, 29, 58, 40, 91, 95, 21, 41, 74, 27, 44, 112, 62, 20, 30, 105, 98, 85, 81, 10, 55, 53, 68, 113, 84, 12, 76, 6, 115, 70, 34, 54, 39, 49, 48, 4, 15, 77, 31, 80, 124, 2, 65, 66, 93, 47, 14, 13, 78, 60, 122, 79, 1, 8, 22, 63, 17, 16, 116, 19, 86, 28, 57, 127, 109, 24, 90, 119, 25, 108, 5, 7, 59, 51, 71, 11, 94, 83, 43, 117, 52, 75, 89, 33, 9, 102, 3, 88, 97, 73, 38, 100, 35, 111, 67, 69, 99, 45, 101, 107, 26, 37, 36, 96, 92, 32, 72], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 23, 104, 87, 121, 103, 41, 118, 82, 40, 50, 64, 58, 18, 91, 0, 29, 98, 105, 95, 27, 6, 39, 76, 44, 30, 21, 113, 85, 53, 112, 55, 20, 12, 81, 90, 10, 115, 15, 84, 4, 93, 13, 80, 74, 16, 109, 34, 47, 77, 49, 14, 68, 17, 124, 60, 54, 79, 48, 127, 51, 63, 66, 62, 59, 70, 78, 1, 57, 22, 122, 86, 25, 31, 116, 2, 24, 19, 36, 65, 8, 28, 7, 89, 111, 119, 9, 43, 83, 108, 73, 94, 88, 52, 117, 71, 11, 38, 67, 45, 97, 102, 26, 69, 3, 101, 99, 100, 75, 107, 37, 35, 72, 33, 5, 32, 92, 96], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 23, 104, 87, 121, 103, 118, 18, 82, 91, 0, 29, 64, 39, 41, 58, 40, 105, 6, 95, 44, 27, 98, 21, 50, 112, 55, 30, 76, 34, 54, 12, 85, 47, 20, 74, 122, 4, 53, 81, 84, 77, 79, 115, 51, 10, 113, 48, 93, 90, 31, 80, 16, 49, 14, 15, 102, 17, 116, 68, 65, 124, 59, 2, 22, 60, 13, 1, 57, 66, 43, 127, 63, 109, 62, 119, 89, 78, 38, 25, 70, 28, 108, 24, 88, 9, 86, 73, 83, 45, 52, 107, 36, 11, 19, 94, 71, 67, 111, 7, 117, 33, 26, 100, 37, 5, 72, 97, 99, 69, 75, 3, 8, 101, 92, 32, 35, 96], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 87, 23, 104, 121, 103, 18, 82, 29, 118, 64, 91, 27, 0, 44, 58, 95, 6, 40, 41, 39, 21, 50, 76, 20, 112, 30, 74, 113, 55, 47, 10, 105, 84, 13, 63, 85, 98, 68, 124, 77, 122, 80, 12, 93, 51, 16, 34, 78, 109, 54, 48, 90, 115, 4, 116, 79, 62, 81, 17, 53, 15, 66, 49, 14, 102, 65, 43, 119, 22, 31, 28, 60, 59, 57, 127, 83, 88, 2, 1, 24, 86, 89, 72, 70, 52, 73, 19, 7, 108, 25, 107, 9, 117, 38, 94, 67, 71, 33, 111, 36, 45, 75, 99, 26, 5, 100, 11, 3, 101, 35, 97, 8, 32, 69, 92, 37, 96], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 104, 121, 87, 23, 64, 58, 118, 103, 0, 82, 18, 91, 6, 29, 41, 40, 30, 27, 76, 48, 4, 39, 50, 98, 85, 63, 55, 12, 74, 113, 10, 122, 44, 20, 95, 112, 93, 21, 51, 124, 105, 47, 68, 2, 62, 34, 22, 53, 84, 54, 70, 66, 16, 115, 81, 13, 108, 49, 60, 57, 43, 79, 77, 90, 14, 80, 17, 116, 65, 15, 102, 78, 1, 59, 86, 109, 127, 119, 72, 24, 28, 31, 11, 73, 7, 9, 38, 25, 71, 111, 19, 33, 83, 94, 3, 107, 36, 88, 75, 52, 89, 26, 5, 67, 8, 45, 117, 69, 101, 100, 97, 35, 99, 32, 37, 92, 96], [123, 56, 61, 125, 120, 114, 110, 46, 126, 106, 42, 104, 121, 23, 87, 118, 82, 18, 103, 0, 64, 91, 10, 40, 58, 70, 27, 74, 29, 95, 50, 76, 98, 6, 85, 39, 41, 30, 21, 4, 20, 55, 77, 84, 66, 53, 13, 16, 12, 90, 79, 81, 54, 105, 68, 78, 72, 93, 15, 1, 80, 124, 122, 2, 62, 48, 112, 57, 17, 47, 60, 51, 115, 34, 113, 71, 59, 94, 63, 49, 7, 31, 65, 28, 14, 24, 19, 86, 22, 25, 73, 88, 109, 83, 11, 44, 89, 116, 3, 102, 52, 9, 127, 108, 119, 38, 5, 67, 33, 43, 107, 75, 36, 26, 117, 45, 111, 69, 35, 97, 100, 101, 99, 92, 8, 37, 32, 96], [123, 56, 125, 61, 120, 114, 110, 46, 126, 106, 104, 42, 87, 23, 121, 82, 18, 103, 118, 64, 29, 40, 91, 0, 98, 95, 70, 76, 27, 50, 84, 21, 39, 10, 20, 74, 44, 58, 30, 4, 79, 41, 12, 85, 55, 105, 34, 13, 16, 93, 78, 77, 80, 122, 68, 66, 90, 59, 113, 54, 17, 51, 47, 15, 62, 115, 112, 6, 31, 81, 2, 49, 14, 22, 60, 53, 65, 72, 57, 1, 25, 102, 24, 124, 86, 116, 28, 48, 75, 83, 63, 73, 109, 127, 7, 71, 36, 9, 3, 19, 119, 94, 89, 67, 52, 43, 11, 107, 88, 33, 45, 38, 69, 108, 111, 35, 26, 5, 101, 99, 117, 100, 32, 37, 97, 8, 96, 92], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 104, 87, 42, 103, 121, 23, 118, 58, 18, 91, 40, 82, 64, 0, 50, 29, 39, 95, 44, 70, 51, 47, 27, 41, 20, 98, 12, 105, 54, 21, 113, 30, 115, 55, 85, 93, 60, 116, 63, 112, 84, 102, 76, 74, 34, 57, 68, 59, 62, 53, 28, 17, 4, 31, 48, 81, 90, 72, 49, 109, 124, 122, 65, 86, 80, 16, 14, 77, 78, 22, 13, 10, 79, 108, 2, 66, 24, 15, 127, 6, 43, 1, 9, 88, 119, 75, 73, 45, 52, 19, 89, 38, 7, 111, 11, 36, 71, 94, 107, 83, 25, 26, 117, 3, 97, 35, 5, 67, 99, 37, 69, 100, 101, 33, 92, 32, 96, 8], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 121, 104, 87, 23, 64, 103, 0, 118, 91, 18, 82, 70, 58, 39, 54, 95, 76, 29, 40, 21, 41, 74, 50, 12, 27, 93, 30, 122, 68, 10, 20, 124, 85, 63, 57, 55, 105, 115, 84, 4, 98, 2, 44, 22, 34, 13, 81, 77, 116, 113, 1, 60, 112, 16, 78, 90, 79, 109, 49, 51, 65, 66, 47, 15, 14, 62, 17, 6, 48, 59, 80, 53, 43, 102, 119, 31, 127, 72, 24, 108, 111, 28, 88, 86, 71, 11, 38, 19, 9, 107, 52, 7, 36, 75, 73, 83, 25, 33, 94, 97, 99, 117, 5, 89, 3, 35, 67, 45, 69, 100, 101, 26, 37, 8, 92, 32, 96], [123, 61, 56, 125, 120, 110, 114, 46, 126, 106, 42, 104, 23, 121, 87, 18, 64, 0, 91, 103, 118, 82, 40, 70, 41, 74, 58, 76, 29, 98, 85, 95, 12, 50, 10, 105, 84, 21, 39, 30, 4, 2, 81, 20, 68, 27, 63, 90, 16, 55, 66, 124, 6, 115, 13, 15, 54, 77, 112, 53, 47, 44, 79, 22, 113, 93, 14, 122, 34, 78, 80, 17, 62, 116, 109, 65, 51, 57, 48, 59, 60, 1, 31, 9, 7, 86, 119, 75, 72, 127, 49, 28, 33, 108, 11, 88, 52, 24, 94, 67, 38, 89, 3, 25, 19, 5, 111, 73, 71, 36, 83, 107, 69, 43, 97, 35, 117, 45, 100, 26, 99, 102, 8, 101, 37, 96, 92, 32], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 121, 104, 23, 87, 103, 18, 118, 91, 82, 64, 29, 50, 58, 0, 95, 10, 70, 40, 27, 74, 105, 55, 93, 6, 30, 76, 84, 39, 85, 13, 12, 20, 98, 68, 21, 41, 81, 90, 122, 115, 77, 47, 16, 112, 80, 78, 15, 4, 44, 34, 66, 54, 1, 62, 113, 109, 14, 2, 49, 60, 22, 79, 17, 59, 65, 119, 31, 63, 19, 28, 53, 48, 124, 86, 7, 75, 71, 9, 57, 24, 116, 73, 127, 102, 25, 3, 88, 33, 89, 43, 108, 83, 52, 72, 11, 51, 67, 8, 94, 111, 107, 38, 100, 45, 69, 5, 97, 26, 37, 35, 117, 36, 32, 99, 101, 96, 92], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 104, 23, 87, 121, 103, 64, 18, 118, 0, 82, 58, 95, 29, 40, 91, 39, 30, 41, 27, 50, 98, 20, 93, 21, 74, 85, 112, 12, 105, 6, 84, 115, 90, 55, 10, 16, 76, 44, 81, 54, 68, 108, 122, 4, 14, 60, 1, 53, 34, 80, 62, 22, 13, 113, 59, 77, 17, 66, 109, 31, 86, 28, 47, 57, 48, 116, 49, 78, 15, 2, 70, 63, 51, 127, 102, 79, 124, 9, 88, 38, 43, 119, 19, 24, 71, 45, 52, 65, 73, 75, 25, 89, 107, 7, 26, 83, 36, 8, 94, 33, 35, 11, 99, 111, 100, 67, 3, 97, 72, 117, 5, 101, 69, 92, 37, 32, 96], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 104, 87, 23, 121, 64, 103, 82, 18, 118, 0, 91, 6, 39, 29, 40, 27, 95, 74, 76, 30, 50, 85, 58, 10, 112, 1, 12, 34, 20, 41, 54, 44, 55, 21, 84, 13, 93, 105, 81, 98, 2, 68, 14, 47, 113, 80, 122, 63, 16, 53, 4, 77, 90, 51, 115, 119, 60, 78, 62, 49, 102, 15, 17, 79, 22, 124, 66, 48, 70, 31, 43, 116, 86, 59, 65, 9, 109, 71, 57, 75, 19, 108, 8, 33, 94, 88, 28, 25, 26, 127, 24, 38, 36, 89, 3, 52, 67, 83, 111, 107, 7, 69, 99, 117, 73, 101, 97, 11, 5, 100, 35, 45, 37, 32, 72, 96, 92], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 104, 23, 121, 103, 87, 64, 0, 118, 91, 40, 82, 6, 18, 58, 41, 29, 27, 112, 76, 39, 50, 74, 98, 49, 10, 95, 93, 85, 12, 2, 122, 113, 20, 60, 84, 4, 30, 105, 124, 44, 63, 21, 34, 81, 62, 48, 115, 14, 68, 13, 77, 55, 80, 109, 116, 119, 53, 47, 54, 17, 51, 1, 8, 15, 59, 127, 43, 16, 66, 78, 70, 22, 102, 108, 31, 90, 79, 75, 57, 33, 65, 86, 36, 19, 38, 28, 45, 88, 52, 24, 9, 89, 71, 94, 111, 73, 3, 83, 11, 25, 99, 7, 107, 69, 67, 101, 97, 35, 100, 26, 5, 117, 37, 96, 92, 32, 72], [123, 61, 56, 125, 120, 114, 110, 46, 126, 106, 42, 104, 103, 121, 23, 87, 118, 0, 91, 82, 40, 6, 64, 18, 58, 29, 95, 10, 27, 93, 30, 41, 74, 39, 4, 55, 50, 98, 122, 81, 124, 12, 76, 66, 105, 85, 20, 112, 21, 62, 34, 54, 8, 68, 84, 48, 44, 16, 53, 115, 116, 90, 77, 113, 17, 14, 70, 63, 13, 78, 2, 1, 60, 80, 79, 15, 47, 28, 59, 109, 49, 102, 57, 75, 108, 119, 22, 127, 83, 71, 51, 31, 86, 73, 65, 33, 52, 9, 45, 7, 38, 35, 3, 19, 25, 88, 43, 111, 94, 69, 36, 97, 89, 26, 107, 24, 117, 11, 100, 99, 67, 5, 101, 92, 72, 37, 96, 32], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 104, 23, 121, 87, 118, 103, 18, 82, 40, 29, 64, 0, 91, 27, 44, 39, 58, 95, 21, 105, 50, 55, 30, 41, 6, 112, 74, 12, 10, 76, 113, 85, 122, 115, 20, 51, 98, 116, 47, 93, 54, 62, 34, 63, 70, 84, 1, 68, 109, 60, 17, 13, 8, 16, 80, 79, 78, 4, 90, 119, 124, 81, 2, 49, 108, 127, 102, 48, 15, 77, 53, 31, 86, 22, 43, 66, 59, 107, 52, 57, 14, 65, 28, 25, 75, 88, 7, 24, 19, 9, 94, 3, 71, 38, 73, 33, 67, 36, 83, 100, 5, 89, 97, 26, 11, 117, 111, 101, 69, 35, 37, 32, 99, 96, 45, 92, 72], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 104, 121, 87, 64, 23, 0, 118, 103, 91, 18, 40, 82, 70, 58, 41, 27, 29, 39, 50, 12, 76, 112, 105, 95, 68, 20, 21, 10, 34, 30, 124, 54, 44, 85, 122, 74, 62, 78, 93, 55, 115, 2, 6, 81, 98, 113, 63, 84, 116, 13, 1, 51, 53, 60, 15, 119, 65, 66, 43, 17, 49, 57, 16, 4, 86, 79, 102, 77, 80, 47, 22, 8, 48, 59, 14, 52, 109, 31, 127, 90, 38, 75, 7, 108, 28, 71, 19, 88, 67, 25, 107, 33, 94, 9, 24, 5, 36, 89, 73, 97, 11, 26, 3, 45, 83, 92, 69, 100, 117, 37, 111, 101, 99, 35, 72, 32, 96], [123, 56, 61, 125, 120, 114, 110, 46, 126, 106, 42, 121, 104, 23, 87, 103, 64, 118, 18, 0, 91, 82, 39, 58, 70, 40, 95, 98, 29, 27, 50, 62, 30, 105, 93, 112, 116, 12, 44, 41, 55, 21, 10, 115, 122, 81, 74, 20, 63, 85, 68, 113, 76, 57, 78, 84, 54, 34, 2, 16, 80, 102, 13, 4, 6, 31, 66, 15, 79, 124, 109, 86, 1, 51, 90, 17, 60, 119, 59, 53, 48, 49, 47, 14, 22, 28, 65, 88, 108, 77, 127, 38, 7, 36, 73, 9, 75, 107, 25, 3, 8, 52, 24, 43, 71, 89, 26, 33, 67, 11, 83, 111, 94, 97, 45, 69, 19, 117, 5, 100, 72, 99, 35, 37, 92, 101, 32, 96], [123, 56, 61, 125, 120, 114, 110, 46, 126, 106, 42, 23, 104, 87, 121, 18, 103, 91, 118, 82, 58, 0, 64, 95, 70, 29, 74, 40, 122, 105, 21, 55, 27, 85, 39, 10, 30, 12, 50, 115, 34, 98, 20, 93, 41, 76, 13, 4, 54, 81, 80, 68, 15, 84, 62, 78, 113, 90, 63, 44, 51, 59, 16, 112, 57, 14, 124, 28, 48, 116, 22, 86, 79, 1, 77, 17, 47, 6, 66, 108, 60, 102, 119, 49, 2, 71, 25, 109, 31, 94, 53, 127, 24, 9, 19, 75, 111, 11, 43, 52, 33, 7, 83, 8, 36, 65, 88, 38, 107, 72, 97, 73, 3, 5, 89, 100, 69, 35, 26, 67, 99, 45, 37, 117, 32, 101, 96, 92], [123, 61, 56, 125, 120, 114, 110, 46, 126, 106, 42, 23, 104, 87, 121, 118, 103, 18, 82, 91, 64, 70, 58, 0, 29, 40, 122, 98, 39, 41, 50, 10, 30, 21, 55, 12, 27, 105, 20, 54, 124, 74, 113, 81, 95, 57, 85, 76, 116, 112, 109, 84, 47, 62, 93, 44, 15, 79, 127, 22, 13, 115, 78, 4, 17, 16, 34, 68, 49, 65, 80, 90, 77, 2, 48, 86, 119, 59, 63, 6, 51, 66, 60, 14, 53, 24, 31, 28, 1, 108, 52, 43, 89, 9, 94, 102, 19, 71, 73, 111, 88, 25, 75, 83, 72, 38, 7, 107, 67, 11, 36, 101, 117, 26, 33, 45, 3, 35, 5, 100, 97, 69, 37, 8, 99, 96, 32, 92], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 87, 104, 23, 121, 118, 91, 103, 0, 18, 64, 82, 58, 29, 39, 95, 30, 44, 51, 50, 27, 41, 40, 70, 116, 54, 122, 10, 47, 98, 34, 20, 12, 113, 84, 119, 112, 105, 21, 124, 93, 76, 115, 81, 109, 102, 60, 74, 85, 68, 53, 49, 63, 62, 57, 16, 4, 59, 31, 14, 66, 48, 77, 55, 86, 90, 13, 43, 79, 6, 1, 65, 127, 15, 78, 80, 17, 108, 33, 22, 28, 36, 2, 19, 24, 38, 9, 52, 72, 107, 73, 88, 71, 26, 45, 11, 89, 3, 97, 25, 94, 111, 67, 7, 100, 75, 117, 69, 83, 35, 92, 99, 101, 37, 32, 5, 8, 96], [123, 125, 56, 61, 120, 114, 110, 46, 126, 106, 42, 104, 23, 121, 87, 103, 118, 64, 0, 18, 40, 82, 41, 50, 29, 58, 91, 47, 95, 55, 39, 30, 27, 98, 60, 122, 124, 62, 57, 21, 20, 105, 93, 10, 76, 34, 49, 54, 12, 51, 115, 74, 85, 116, 81, 90, 6, 112, 53, 63, 84, 113, 1, 70, 4, 48, 13, 17, 59, 43, 16, 14, 77, 80, 15, 31, 2, 68, 78, 127, 102, 44, 28, 109, 72, 86, 79, 22, 119, 65, 25, 52, 108, 24, 36, 107, 7, 66, 88, 19, 38, 89, 99, 33, 11, 83, 9, 73, 71, 94, 75, 111, 101, 67, 35, 97, 3, 117, 100, 26, 37, 45, 69, 5, 96, 8, 32, 92], [123, 125, 56, 61, 120, 114, 110, 46, 126, 106, 42, 23, 104, 121, 87, 118, 64, 82, 0, 103, 18, 91, 58, 40, 6, 41, 10, 29, 74, 122, 12, 55, 62, 50, 95, 27, 85, 124, 17, 76, 39, 13, 105, 93, 30, 21, 20, 34, 68, 81, 98, 4, 57, 115, 54, 60, 78, 59, 90, 70, 80, 63, 48, 84, 77, 2, 79, 66, 112, 15, 116, 16, 72, 1, 19, 44, 49, 47, 119, 127, 113, 22, 11, 14, 51, 86, 28, 31, 102, 65, 7, 52, 53, 36, 73, 109, 33, 71, 43, 67, 38, 89, 24, 108, 75, 9, 111, 83, 25, 88, 101, 3, 69, 94, 26, 107, 117, 35, 5, 99, 45, 97, 100, 8, 37, 92, 32, 96], [123, 61, 56, 125, 120, 114, 110, 46, 126, 106, 42, 104, 121, 23, 87, 0, 103, 64, 118, 18, 91, 82, 6, 40, 58, 115, 41, 98, 62, 116, 29, 12, 50, 76, 39, 30, 95, 10, 55, 93, 85, 74, 105, 21, 4, 122, 47, 68, 84, 124, 78, 60, 49, 27, 51, 13, 66, 20, 44, 34, 48, 59, 77, 2, 112, 72, 63, 1, 17, 53, 70, 113, 57, 79, 65, 81, 14, 54, 86, 80, 16, 22, 90, 15, 31, 7, 52, 119, 11, 127, 28, 109, 111, 108, 25, 38, 19, 33, 83, 3, 9, 89, 94, 73, 102, 67, 36, 71, 100, 43, 24, 117, 97, 88, 75, 107, 45, 69, 5, 35, 26, 101, 37, 99, 96, 32, 92, 8], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 104, 23, 87, 121, 103, 18, 82, 58, 118, 6, 91, 29, 64, 40, 50, 0, 95, 44, 85, 39, 10, 12, 30, 62, 74, 21, 47, 27, 122, 51, 20, 41, 84, 124, 115, 105, 63, 76, 116, 98, 54, 60, 55, 112, 81, 68, 78, 17, 77, 80, 93, 4, 14, 13, 34, 16, 48, 113, 65, 15, 53, 119, 59, 66, 57, 31, 109, 90, 2, 49, 70, 79, 83, 72, 86, 1, 11, 7, 28, 22, 52, 127, 71, 24, 9, 111, 19, 73, 108, 25, 38, 89, 88, 75, 3, 67, 45, 43, 94, 33, 102, 97, 117, 69, 107, 26, 36, 37, 100, 5, 101, 99, 8, 35, 32, 92, 96], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 104, 42, 23, 121, 87, 0, 103, 64, 118, 18, 91, 6, 29, 82, 58, 40, 95, 10, 105, 50, 39, 41, 27, 74, 68, 12, 62, 55, 76, 81, 63, 30, 122, 85, 112, 84, 20, 124, 21, 115, 34, 4, 70, 93, 98, 54, 60, 66, 59, 80, 65, 2, 90, 78, 31, 51, 15, 44, 49, 14, 16, 116, 77, 13, 22, 57, 17, 113, 86, 79, 28, 53, 47, 48, 72, 71, 109, 7, 127, 119, 1, 108, 24, 25, 11, 111, 9, 36, 73, 88, 83, 67, 75, 117, 52, 102, 43, 19, 99, 69, 94, 38, 33, 89, 45, 5, 3, 97, 100, 35, 26, 107, 8, 37, 101, 92, 96, 32], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 104, 87, 23, 121, 103, 118, 18, 82, 29, 50, 91, 53, 95, 12, 58, 64, 30, 85, 41, 40, 51, 6, 27, 39, 21, 10, 124, 0, 34, 76, 62, 105, 54, 112, 93, 63, 98, 55, 48, 74, 60, 47, 15, 115, 90, 20, 84, 17, 81, 16, 44, 122, 113, 57, 14, 68, 22, 4, 80, 59, 79, 13, 70, 77, 116, 31, 127, 78, 28, 108, 86, 102, 49, 66, 109, 83, 9, 1, 2, 71, 11, 43, 65, 119, 24, 7, 25, 33, 73, 88, 52, 111, 89, 72, 107, 19, 94, 38, 45, 8, 35, 100, 37, 36, 99, 26, 75, 67, 3, 117, 5, 97, 69, 101, 92, 32, 96]], "model.layers.29.self_attn.q_proj": [[45, 35, 109, 93, 22, 25, 80, 18, 83, 11, 14, 91, 67, 31, 1, 115, 124, 10, 88, 127, 43, 55, 13, 119, 8, 64, 81, 90, 6, 21, 7, 102, 48, 95, 5, 9, 69, 27, 112, 70, 40, 113, 78, 114, 126, 4, 120, 53, 50, 111, 61, 121, 2, 101, 24, 12, 103, 54, 37, 63, 57, 108, 38, 122, 92, 16, 44, 59, 97, 33, 52, 123, 28, 3, 96, 26, 116, 79, 104, 47, 49, 86, 100, 15, 17, 32, 60, 117, 62, 106, 36, 34, 118, 41, 58, 84, 51, 39, 89, 94, 46, 110, 19, 107, 29, 42, 77, 72, 56, 105, 125, 76, 30, 85, 0, 87, 23, 98, 20, 99, 82, 66, 73, 75, 65, 71, 68, 74], [45, 109, 35, 112, 93, 25, 22, 115, 18, 83, 14, 61, 13, 10, 80, 119, 31, 91, 55, 11, 121, 124, 54, 126, 7, 114, 50, 103, 60, 90, 102, 101, 21, 113, 62, 116, 123, 1, 8, 84, 15, 127, 110, 33, 88, 9, 20, 79, 48, 108, 27, 47, 122, 92, 125, 23, 95, 49, 28, 17, 69, 32, 118, 100, 75, 117, 85, 120, 52, 24, 51, 97, 107, 111, 37, 98, 63, 41, 46, 34, 59, 81, 4, 56, 43, 39, 104, 40, 106, 26, 53, 87, 57, 58, 42, 86, 94, 82, 38, 6, 36, 19, 105, 44, 76, 89, 30, 5, 29, 12, 64, 73, 67, 96, 16, 77, 71, 68, 74, 78, 72, 99, 70, 2, 66, 65, 3, 0], [45, 109, 55, 35, 118, 93, 22, 112, 111, 121, 113, 115, 43, 48, 25, 31, 91, 53, 62, 126, 40, 122, 52, 80, 119, 123, 50, 61, 49, 117, 51, 56, 27, 92, 58, 103, 104, 102, 18, 83, 60, 39, 124, 46, 108, 59, 44, 94, 54, 41, 127, 106, 90, 114, 76, 47, 57, 107, 99, 42, 20, 21, 116, 101, 120, 110, 32, 125, 88, 84, 63, 14, 16, 23, 38, 33, 28, 74, 64, 105, 95, 37, 97, 36, 7, 15, 98, 96, 1, 100, 2, 30, 79, 19, 89, 11, 86, 34, 85, 69, 10, 0, 87, 66, 29, 8, 72, 67, 12, 68, 65, 26, 24, 17, 70, 75, 78, 4, 81, 5, 6, 82, 3, 71, 13, 9, 77, 73], [45, 35, 109, 93, 22, 18, 25, 55, 83, 13, 124, 31, 115, 14, 88, 80, 61, 112, 91, 81, 10, 118, 21, 50, 11, 15, 121, 28, 90, 63, 77, 9, 73, 48, 64, 56, 33, 7, 103, 85, 23, 44, 126, 120, 127, 27, 41, 24, 43, 92, 119, 111, 114, 113, 29, 26, 12, 62, 99, 30, 58, 67, 1, 8, 122, 52, 117, 76, 97, 95, 57, 123, 60, 69, 116, 108, 84, 89, 102, 71, 20, 51, 39, 70, 74, 40, 53, 2, 17, 37, 86, 59, 107, 6, 104, 106, 79, 5, 38, 19, 105, 54, 100, 110, 82, 49, 42, 36, 101, 34, 47, 32, 98, 46, 96, 125, 72, 87, 16, 94, 3, 66, 78, 75, 0, 68, 4, 65], [41, 98, 57, 117, 25, 51, 86, 127, 95, 114, 125, 120, 121, 18, 93, 16, 78, 50, 115, 61, 110, 105, 59, 53, 48, 55, 119, 46, 15, 49, 31, 58, 38, 87, 40, 12, 47, 39, 28, 19, 88, 118, 62, 108, 63, 45, 74, 85, 11, 116, 72, 75, 54, 67, 123, 68, 9, 76, 56, 106, 124, 35, 122, 10, 126, 80, 101, 90, 7, 69, 107, 13, 65, 111, 91, 66, 60, 43, 42, 109, 64, 92, 1, 113, 71, 112, 100, 33, 70, 23, 103, 32, 44, 82, 21, 89, 52, 20, 27, 22, 36, 104, 17, 29, 102, 99, 81, 30, 37, 24, 26, 79, 77, 14, 2, 96, 84, 97, 94, 4, 83, 8, 73, 0, 3, 34, 5, 6], [41, 98, 117, 57, 127, 95, 25, 125, 86, 114, 51, 93, 53, 54, 50, 18, 48, 120, 60, 61, 58, 105, 119, 55, 111, 38, 78, 115, 31, 16, 87, 59, 62, 121, 40, 108, 88, 21, 106, 43, 126, 56, 35, 63, 19, 15, 116, 85, 110, 12, 28, 10, 45, 52, 122, 47, 92, 7, 90, 101, 124, 123, 104, 109, 36, 113, 49, 72, 29, 80, 100, 33, 81, 76, 112, 103, 44, 30, 74, 107, 32, 46, 91, 118, 9, 77, 42, 23, 24, 82, 97, 22, 1, 75, 2, 39, 89, 67, 37, 20, 94, 27, 83, 13, 5, 102, 11, 26, 99, 70, 96, 79, 17, 3, 4, 84, 14, 68, 71, 69, 73, 64, 34, 6, 8, 0, 66, 65], [41, 98, 57, 117, 25, 86, 95, 51, 61, 127, 114, 18, 78, 59, 125, 119, 121, 48, 16, 93, 53, 110, 50, 105, 120, 116, 62, 58, 55, 38, 49, 31, 7, 76, 67, 65, 39, 10, 15, 87, 115, 12, 28, 108, 9, 74, 88, 45, 71, 56, 75, 21, 118, 109, 44, 72, 47, 19, 46, 60, 113, 124, 111, 68, 99, 40, 122, 106, 54, 103, 92, 35, 64, 81, 85, 70, 100, 89, 90, 107, 29, 43, 23, 63, 42, 69, 66, 112, 82, 1, 52, 91, 20, 32, 126, 123, 104, 11, 36, 80, 27, 13, 102, 14, 2, 33, 37, 26, 84, 22, 4, 8, 5, 83, 30, 101, 77, 17, 79, 97, 73, 96, 24, 94, 0, 3, 34, 6], [41, 98, 57, 117, 95, 120, 86, 25, 114, 51, 127, 125, 18, 110, 93, 121, 78, 16, 105, 59, 48, 108, 53, 58, 62, 119, 50, 45, 61, 38, 47, 49, 15, 124, 115, 39, 87, 44, 72, 85, 31, 46, 55, 111, 118, 12, 76, 21, 101, 54, 109, 7, 65, 116, 77, 67, 92, 35, 107, 43, 40, 69, 32, 90, 9, 11, 10, 112, 19, 88, 52, 75, 80, 97, 106, 28, 42, 82, 68, 113, 123, 102, 83, 81, 66, 20, 74, 70, 84, 24, 56, 64, 60, 22, 89, 99, 29, 63, 94, 126, 104, 36, 14, 13, 122, 91, 26, 103, 27, 73, 33, 5, 30, 100, 4, 96, 17, 37, 79, 0, 23, 8, 71, 1, 2, 34, 6, 3], [39, 63, 52, 97, 18, 113, 13, 87, 22, 28, 100, 120, 10, 4, 8, 45, 92, 11, 66, 0, 89, 126, 6, 81, 94, 27, 29, 68, 69, 42, 72, 65, 15, 12, 1, 14, 30, 59, 127, 108, 25, 70, 86, 74, 82, 110, 80, 119, 20, 79, 77, 44, 111, 90, 21, 7, 78, 62, 50, 85, 43, 16, 17, 121, 5, 19, 102, 23, 101, 40, 35, 106, 116, 9, 64, 24, 67, 96, 122, 75, 26, 84, 55, 83, 47, 38, 118, 125, 95, 54, 105, 91, 99, 76, 58, 88, 37, 104, 93, 3, 98, 32, 61, 34, 57, 117, 49, 73, 123, 115, 56, 36, 109, 103, 46, 41, 51, 31, 60, 48, 53, 124, 112, 71, 107, 114, 2, 33], [63, 52, 39, 113, 120, 37, 122, 125, 126, 45, 106, 119, 58, 61, 127, 111, 55, 110, 57, 47, 116, 51, 62, 103, 123, 56, 48, 115, 117, 59, 49, 54, 118, 108, 60, 97, 109, 53, 50, 121, 44, 114, 46, 124, 112, 100, 105, 107, 43, 36, 22, 42, 28, 41, 40, 38, 104, 21, 87, 20, 101, 102, 27, 35, 96, 2, 91, 94, 98, 11, 32, 34, 19, 99, 18, 5, 25, 92, 24, 33, 90, 29, 26, 31, 85, 8, 89, 93, 30, 95, 88, 84, 78, 81, 14, 86, 15, 13, 73, 6, 83, 23, 16, 64, 75, 79, 17, 69, 66, 9, 10, 80, 82, 12, 76, 77, 3, 67, 72, 0, 65, 70, 71, 4, 74, 7, 1, 68], [63, 39, 52, 97, 24, 87, 50, 66, 54, 110, 64, 83, 18, 69, 11, 7, 92, 58, 120, 79, 3, 36, 73, 13, 20, 114, 28, 0, 57, 2, 67, 6, 9, 22, 119, 106, 80, 44, 45, 10, 113, 8, 26, 1, 68, 108, 48, 37, 111, 4, 30, 65, 122, 85, 29, 62, 47, 72, 71, 107, 126, 25, 88, 125, 49, 61, 89, 76, 103, 70, 51, 96, 34, 12, 123, 40, 16, 104, 35, 81, 5, 38, 78, 116, 124, 117, 115, 31, 90, 84, 127, 32, 46, 56, 59, 55, 101, 98, 42, 60, 100, 118, 14, 75, 121, 94, 91, 105, 53, 112, 43, 27, 15, 17, 23, 109, 19, 95, 99, 93, 77, 102, 86, 41, 21, 82, 74, 33], [63, 39, 113, 52, 97, 120, 55, 28, 122, 87, 126, 119, 62, 92, 50, 114, 59, 22, 111, 58, 109, 36, 118, 106, 51, 47, 18, 110, 121, 61, 125, 54, 56, 116, 105, 115, 48, 57, 42, 45, 60, 49, 15, 102, 107, 21, 101, 127, 123, 124, 46, 100, 41, 37, 103, 117, 40, 44, 112, 13, 19, 35, 24, 53, 108, 43, 89, 81, 38, 84, 93, 99, 20, 69, 91, 104, 33, 8, 96, 74, 85, 34, 94, 30, 11, 98, 27, 31, 32, 86, 25, 23, 29, 26, 95, 90, 10, 17, 9, 88, 79, 83, 82, 76, 12, 4, 14, 5, 78, 6, 80, 64, 16, 71, 72, 73, 66, 3, 1, 77, 70, 2, 65, 68, 7, 75, 0, 67], [111, 47, 95, 28, 25, 84, 97, 18, 86, 33, 39, 122, 16, 123, 74, 29, 69, 15, 53, 117, 78, 120, 107, 49, 58, 7, 4, 106, 124, 35, 75, 125, 64, 17, 2, 110, 62, 55, 61, 113, 63, 91, 13, 32, 46, 42, 126, 99, 23, 12, 54, 121, 3, 108, 112, 72, 115, 1, 37, 57, 38, 96, 73, 59, 85, 44, 36, 43, 41, 60, 45, 50, 119, 77, 19, 114, 27, 48, 109, 101, 67, 116, 118, 104, 79, 52, 87, 127, 56, 102, 22, 76, 26, 103, 70, 90, 40, 24, 20, 83, 31, 89, 51, 6, 82, 92, 105, 30, 88, 94, 0, 65, 93, 100, 98, 9, 34, 21, 11, 80, 71, 81, 5, 14, 8, 10, 66, 68], [111, 47, 28, 95, 25, 84, 39, 97, 86, 33, 18, 16, 60, 59, 115, 112, 29, 54, 15, 74, 7, 106, 78, 52, 110, 118, 123, 1, 114, 69, 64, 17, 46, 62, 120, 57, 109, 2, 13, 4, 75, 12, 122, 101, 116, 107, 126, 49, 99, 27, 119, 9, 113, 121, 50, 45, 48, 41, 124, 42, 70, 63, 72, 58, 81, 36, 55, 85, 90, 56, 23, 73, 87, 117, 125, 76, 94, 61, 26, 67, 20, 35, 3, 96, 30, 32, 38, 43, 100, 53, 44, 103, 77, 24, 51, 40, 127, 89, 91, 37, 108, 31, 88, 22, 79, 19, 82, 83, 102, 6, 34, 105, 65, 98, 104, 92, 93, 80, 21, 66, 5, 0, 14, 68, 71, 11, 10, 8], [111, 47, 28, 95, 25, 84, 126, 33, 18, 97, 57, 86, 39, 120, 16, 60, 110, 124, 61, 29, 63, 116, 74, 54, 78, 56, 17, 64, 7, 46, 15, 125, 99, 50, 4, 12, 107, 62, 127, 75, 48, 53, 27, 35, 96, 106, 112, 109, 92, 59, 38, 113, 67, 2, 49, 55, 101, 72, 45, 42, 36, 69, 1, 41, 119, 20, 31, 94, 108, 52, 58, 123, 13, 89, 85, 122, 121, 87, 23, 22, 40, 114, 117, 73, 37, 19, 44, 32, 115, 70, 76, 105, 100, 90, 118, 9, 24, 51, 103, 43, 104, 91, 102, 34, 26, 88, 82, 81, 30, 98, 6, 3, 83, 0, 68, 21, 14, 80, 77, 93, 71, 10, 79, 8, 5, 65, 11, 66], [111, 47, 28, 25, 95, 84, 57, 86, 39, 110, 62, 33, 18, 16, 74, 116, 78, 29, 97, 2, 49, 41, 17, 118, 96, 15, 115, 1, 7, 64, 69, 4, 126, 13, 59, 120, 3, 127, 113, 63, 51, 65, 58, 56, 125, 36, 44, 50, 85, 123, 35, 60, 121, 75, 99, 43, 54, 37, 42, 76, 117, 119, 98, 114, 106, 46, 6, 55, 67, 70, 108, 45, 79, 124, 40, 89, 109, 52, 48, 38, 32, 53, 0, 26, 24, 105, 101, 72, 73, 27, 9, 20, 82, 112, 103, 22, 5, 122, 30, 81, 94, 61, 12, 102, 104, 107, 83, 90, 34, 77, 23, 11, 80, 92, 87, 88, 93, 21, 10, 91, 31, 14, 19, 100, 71, 66, 68, 8], [49, 117, 102, 113, 123, 52, 116, 53, 38, 23, 41, 91, 33, 25, 121, 85, 118, 119, 54, 122, 115, 109, 59, 93, 104, 120, 30, 110, 62, 61, 51, 108, 60, 124, 82, 112, 45, 111, 98, 114, 57, 48, 50, 96, 42, 63, 36, 94, 56, 47, 126, 46, 106, 37, 125, 21, 35, 43, 107, 127, 58, 44, 77, 79, 27, 55, 100, 103, 99, 95, 89, 40, 105, 39, 97, 24, 84, 18, 34, 101, 32, 20, 16, 17, 22, 31, 19, 29, 76, 87, 11, 78, 90, 83, 28, 88, 86, 26, 72, 92, 15, 5, 12, 4, 14, 70, 71, 13, 74, 80, 73, 9, 75, 3, 7, 81, 66, 1, 10, 8, 6, 0, 65, 68, 69, 67, 2, 64], [49, 102, 113, 117, 33, 91, 25, 38, 116, 123, 54, 30, 82, 23, 110, 55, 85, 35, 53, 121, 109, 39, 115, 95, 58, 62, 122, 100, 43, 93, 98, 51, 89, 52, 59, 60, 27, 21, 41, 125, 78, 48, 119, 57, 120, 42, 77, 40, 37, 92, 50, 118, 108, 80, 90, 96, 63, 28, 114, 112, 24, 111, 61, 103, 106, 124, 104, 71, 56, 34, 47, 101, 5, 14, 11, 127, 99, 22, 20, 36, 46, 94, 29, 12, 126, 32, 9, 65, 19, 88, 68, 86, 31, 97, 44, 15, 107, 105, 26, 45, 87, 2, 18, 17, 83, 67, 70, 72, 6, 64, 84, 8, 10, 76, 1, 3, 0, 16, 13, 75, 81, 74, 79, 7, 73, 69, 66, 4], [49, 102, 113, 117, 53, 33, 30, 25, 38, 91, 85, 120, 23, 125, 122, 93, 60, 82, 119, 54, 61, 41, 48, 118, 96, 107, 59, 27, 94, 109, 50, 108, 121, 110, 114, 106, 52, 35, 98, 42, 103, 101, 20, 40, 95, 43, 45, 123, 58, 57, 105, 99, 14, 116, 62, 89, 46, 36, 44, 56, 100, 124, 39, 47, 80, 77, 26, 127, 51, 111, 55, 97, 78, 72, 126, 112, 92, 3, 63, 115, 31, 34, 104, 22, 11, 24, 90, 29, 84, 32, 28, 75, 37, 10, 88, 73, 71, 64, 83, 4, 87, 18, 21, 15, 17, 12, 19, 2, 81, 65, 74, 9, 5, 86, 70, 16, 79, 76, 7, 13, 6, 68, 1, 0, 69, 8, 66, 67], [49, 102, 113, 117, 123, 33, 52, 25, 53, 91, 122, 61, 23, 38, 63, 20, 41, 48, 82, 85, 100, 118, 121, 93, 112, 27, 89, 116, 15, 96, 114, 42, 108, 11, 35, 30, 97, 50, 39, 101, 62, 9, 120, 99, 51, 87, 46, 45, 6, 115, 47, 60, 57, 59, 44, 124, 68, 126, 111, 127, 105, 109, 58, 43, 94, 54, 90, 34, 110, 80, 32, 103, 40, 55, 56, 28, 29, 26, 107, 119, 77, 125, 106, 104, 14, 98, 4, 1, 37, 95, 84, 22, 73, 36, 81, 21, 18, 16, 92, 83, 88, 31, 8, 86, 66, 79, 70, 19, 24, 10, 12, 0, 75, 2, 17, 76, 69, 5, 71, 78, 13, 7, 67, 3, 74, 64, 65, 72], [117, 120, 60, 38, 58, 127, 122, 46, 52, 62, 102, 126, 55, 125, 118, 43, 110, 116, 123, 121, 124, 115, 113, 63, 61, 119, 51, 57, 53, 114, 112, 56, 25, 54, 48, 50, 45, 59, 47, 108, 111, 39, 107, 44, 97, 104, 35, 37, 49, 26, 109, 94, 106, 41, 105, 86, 42, 98, 83, 103, 17, 24, 101, 23, 28, 36, 40, 85, 88, 92, 100, 91, 78, 19, 22, 21, 75, 32, 30, 34, 99, 11, 96, 16, 95, 27, 77, 79, 29, 82, 84, 31, 33, 14, 90, 93, 20, 81, 89, 80, 72, 18, 9, 3, 12, 87, 7, 67, 76, 15, 73, 71, 69, 70, 8, 2, 6, 1, 5, 0, 13, 65, 64, 74, 66, 68, 10, 4], [117, 60, 58, 127, 38, 110, 122, 46, 52, 116, 62, 115, 118, 126, 120, 125, 123, 43, 102, 51, 61, 124, 63, 55, 113, 109, 53, 94, 56, 121, 119, 57, 112, 59, 97, 54, 50, 114, 48, 45, 107, 47, 108, 111, 39, 104, 28, 44, 106, 25, 103, 23, 49, 86, 26, 105, 42, 35, 83, 41, 37, 36, 98, 101, 17, 91, 40, 74, 30, 100, 88, 84, 33, 24, 32, 99, 19, 22, 95, 90, 85, 81, 34, 96, 92, 75, 93, 31, 11, 69, 79, 27, 77, 78, 29, 66, 15, 80, 72, 16, 0, 14, 3, 64, 5, 89, 12, 87, 71, 20, 67, 18, 2, 82, 21, 65, 1, 8, 68, 6, 13, 10, 70, 76, 7, 9, 73, 4], [117, 60, 46, 120, 38, 110, 127, 122, 58, 52, 115, 62, 57, 125, 116, 118, 43, 126, 119, 48, 63, 123, 61, 56, 35, 49, 25, 109, 94, 121, 55, 124, 47, 51, 111, 50, 112, 54, 114, 83, 53, 113, 24, 26, 108, 45, 59, 105, 40, 107, 104, 102, 44, 86, 41, 97, 106, 28, 103, 37, 42, 88, 30, 23, 36, 39, 101, 100, 22, 75, 34, 85, 99, 11, 84, 17, 31, 19, 32, 16, 98, 79, 27, 81, 91, 69, 92, 33, 78, 93, 96, 95, 3, 14, 72, 82, 90, 29, 8, 89, 77, 20, 5, 13, 18, 66, 80, 74, 15, 87, 2, 64, 0, 9, 21, 67, 70, 65, 71, 6, 1, 76, 7, 12, 10, 73, 68, 4], [46, 117, 38, 120, 110, 85, 94, 25, 97, 16, 9, 90, 12, 126, 122, 81, 23, 116, 58, 100, 15, 62, 118, 98, 52, 68, 14, 76, 28, 89, 7, 125, 87, 82, 61, 71, 121, 37, 18, 123, 48, 86, 1, 42, 60, 30, 21, 99, 57, 96, 31, 91, 44, 107, 32, 119, 112, 36, 53, 47, 56, 127, 51, 101, 45, 55, 115, 10, 113, 43, 70, 114, 124, 88, 104, 29, 26, 73, 92, 54, 79, 109, 111, 75, 49, 50, 63, 24, 19, 13, 20, 93, 59, 83, 4, 108, 95, 74, 80, 34, 84, 35, 17, 106, 72, 27, 6, 11, 103, 2, 77, 22, 105, 78, 41, 8, 40, 39, 5, 65, 33, 69, 64, 3, 66, 0, 67, 102], [41, 99, 67, 74, 80, 0, 86, 44, 4, 13, 71, 73, 19, 115, 118, 69, 124, 105, 52, 119, 113, 114, 51, 65, 122, 54, 12, 116, 112, 111, 35, 117, 110, 126, 57, 2, 62, 63, 64, 60, 48, 107, 61, 22, 6, 89, 121, 76, 84, 123, 109, 1, 29, 90, 3, 45, 30, 88, 14, 106, 83, 79, 10, 33, 92, 9, 108, 120, 20, 55, 23, 68, 39, 15, 40, 101, 102, 103, 98, 104, 66, 75, 100, 56, 27, 94, 11, 38, 53, 17, 58, 24, 31, 32, 8, 77, 70, 7, 5, 26, 43, 16, 85, 95, 81, 46, 34, 37, 49, 125, 96, 21, 28, 36, 72, 50, 127, 91, 93, 59, 97, 42, 82, 78, 87, 18, 47, 25], [41, 99, 80, 74, 13, 4, 86, 71, 19, 67, 73, 113, 52, 69, 61, 51, 12, 44, 118, 115, 112, 122, 64, 60, 114, 105, 111, 66, 2, 120, 57, 116, 103, 63, 6, 54, 119, 62, 124, 35, 75, 117, 43, 1, 55, 20, 15, 88, 48, 108, 126, 22, 101, 27, 125, 107, 21, 82, 29, 85, 31, 102, 76, 17, 50, 78, 34, 3, 90, 106, 37, 79, 26, 24, 8, 109, 83, 127, 59, 39, 23, 91, 45, 46, 87, 49, 98, 123, 100, 81, 96, 92, 110, 32, 30, 68, 93, 89, 38, 56, 97, 42, 18, 53, 84, 36, 33, 5, 58, 94, 72, 25, 28, 104, 14, 95, 121, 10, 65, 47, 70, 40, 11, 7, 16, 77, 9, 0], [41, 74, 0, 99, 65, 86, 2, 44, 80, 13, 67, 69, 4, 118, 119, 116, 115, 113, 71, 112, 54, 19, 111, 110, 114, 105, 73, 124, 12, 117, 121, 52, 122, 51, 126, 57, 35, 22, 107, 62, 48, 63, 29, 61, 15, 94, 3, 59, 68, 89, 83, 60, 6, 123, 10, 27, 90, 33, 45, 20, 108, 5, 24, 40, 88, 16, 106, 1, 49, 92, 64, 9, 76, 23, 53, 8, 32, 38, 75, 120, 100, 66, 7, 56, 21, 97, 79, 39, 104, 85, 26, 91, 18, 109, 93, 11, 82, 70, 55, 101, 87, 30, 58, 125, 98, 17, 78, 127, 31, 81, 72, 96, 84, 14, 77, 42, 28, 95, 34, 43, 25, 103, 102, 37, 47, 46, 36, 50], [41, 99, 0, 2, 44, 67, 86, 80, 13, 4, 65, 115, 118, 69, 71, 119, 116, 113, 73, 114, 111, 112, 122, 54, 19, 52, 51, 105, 124, 110, 126, 22, 121, 117, 57, 35, 74, 62, 29, 12, 107, 3, 48, 68, 61, 60, 63, 89, 33, 123, 64, 15, 6, 108, 90, 40, 5, 97, 94, 88, 16, 70, 27, 106, 20, 83, 100, 98, 17, 75, 79, 38, 49, 81, 45, 84, 11, 1, 39, 24, 76, 92, 56, 9, 91, 120, 104, 55, 96, 28, 18, 10, 26, 8, 14, 78, 82, 125, 109, 42, 25, 30, 85, 53, 59, 93, 7, 23, 37, 95, 101, 36, 58, 72, 46, 21, 127, 87, 31, 47, 34, 32, 102, 43, 50, 77, 66, 103], [106, 99, 59, 26, 118, 124, 94, 49, 42, 21, 62, 19, 115, 86, 17, 113, 60, 121, 40, 109, 55, 30, 78, 32, 80, 56, 117, 107, 73, 122, 28, 123, 47, 12, 24, 96, 98, 29, 120, 74, 23, 43, 50, 53, 11, 69, 51, 37, 125, 61, 63, 114, 52, 27, 79, 108, 116, 103, 57, 126, 110, 58, 127, 112, 119, 48, 84, 89, 105, 92, 39, 54, 46, 111, 36, 104, 4, 22, 8, 71, 101, 45, 3, 72, 67, 7, 90, 44, 41, 34, 38, 15, 77, 33, 102, 93, 97, 31, 100, 70, 14, 66, 64, 95, 88, 85, 76, 0, 1, 82, 91, 25, 81, 16, 87, 18, 5, 83, 20, 13, 6, 35, 65, 9, 68, 10, 75, 2], [106, 61, 99, 118, 115, 59, 121, 26, 94, 42, 107, 62, 113, 21, 19, 86, 125, 29, 11, 109, 28, 79, 50, 22, 55, 60, 25, 117, 40, 17, 126, 47, 90, 56, 123, 23, 119, 124, 24, 58, 52, 93, 80, 57, 78, 114, 39, 98, 13, 74, 66, 32, 16, 73, 116, 101, 85, 34, 65, 104, 53, 63, 103, 49, 12, 70, 122, 35, 36, 82, 120, 108, 45, 46, 4, 51, 71, 95, 30, 44, 20, 91, 37, 38, 105, 87, 72, 48, 112, 43, 27, 100, 77, 6, 7, 111, 97, 31, 41, 3, 89, 64, 110, 54, 127, 88, 96, 102, 92, 18, 84, 83, 33, 5, 69, 81, 76, 15, 14, 0, 67, 10, 75, 8, 9, 2, 1, 68], [106, 59, 115, 99, 118, 62, 50, 112, 26, 55, 94, 125, 86, 19, 117, 121, 124, 42, 24, 116, 21, 53, 80, 17, 29, 52, 122, 107, 44, 78, 49, 123, 30, 32, 40, 28, 45, 11, 57, 74, 61, 98, 63, 58, 120, 101, 113, 23, 46, 12, 110, 60, 104, 27, 48, 96, 51, 108, 109, 114, 47, 72, 126, 119, 69, 4, 127, 111, 54, 97, 73, 56, 71, 43, 36, 39, 95, 37, 103, 38, 102, 105, 41, 84, 33, 100, 7, 22, 34, 91, 93, 90, 35, 31, 79, 8, 66, 92, 87, 67, 88, 82, 20, 16, 25, 18, 68, 85, 76, 70, 15, 81, 64, 89, 3, 83, 1, 0, 14, 77, 6, 5, 9, 10, 65, 13, 75, 2], [106, 99, 107, 115, 59, 26, 94, 113, 86, 17, 19, 42, 21, 12, 121, 80, 30, 62, 40, 119, 117, 55, 96, 49, 78, 74, 60, 69, 61, 125, 112, 24, 72, 11, 118, 53, 114, 56, 124, 70, 120, 4, 32, 123, 116, 109, 108, 58, 5, 44, 104, 73, 110, 68, 52, 126, 93, 54, 89, 122, 67, 48, 45, 57, 1, 101, 63, 51, 37, 29, 47, 28, 71, 36, 127, 16, 43, 41, 111, 25, 105, 46, 22, 102, 13, 88, 9, 98, 83, 50, 27, 0, 84, 39, 8, 20, 95, 90, 14, 87, 7, 38, 77, 33, 64, 100, 6, 82, 92, 34, 79, 23, 76, 35, 3, 103, 31, 65, 18, 91, 85, 66, 97, 81, 10, 15, 2, 75]], "model.layers.29.self_attn.k_proj": [[109, 45, 99, 93, 22, 64, 31, 83, 80, 14, 25, 18, 11, 91, 67, 8, 2, 7, 126, 119, 1, 124, 6, 127, 70, 122, 112, 77, 13, 115, 60, 16, 21, 55, 9, 43, 114, 23, 10, 51, 4, 29, 90, 12, 52, 110, 63, 116, 41, 42, 36, 111, 57, 69, 118, 28, 59, 49, 120, 84, 103, 72, 123, 56, 61, 39, 121, 88, 113, 62, 54, 105, 44, 47, 53, 27, 40, 82, 58, 50, 117, 85, 102, 96, 48, 38, 92, 33, 100, 5, 24, 87, 108, 107, 106, 94, 101, 125, 98, 32, 46, 26, 34, 104, 15, 37, 30, 76, 97, 89, 79, 20, 95, 78, 75, 17, 35, 73, 81, 74, 66, 19, 86, 68, 71, 3, 0, 65], [105, 34, 57, 18, 53, 46, 51, 125, 78, 86, 117, 25, 58, 120, 16, 45, 64, 31, 127, 67, 95, 109, 98, 50, 12, 119, 29, 69, 59, 111, 108, 61, 66, 10, 112, 9, 19, 74, 87, 7, 70, 113, 41, 60, 92, 80, 13, 49, 15, 55, 91, 52, 114, 88, 21, 121, 62, 76, 2, 107, 65, 75, 116, 103, 38, 83, 63, 68, 100, 93, 104, 39, 0, 40, 72, 56, 20, 35, 106, 90, 110, 115, 73, 54, 118, 27, 36, 48, 6, 42, 44, 126, 81, 33, 122, 124, 47, 102, 43, 26, 123, 84, 77, 37, 1, 101, 28, 30, 96, 94, 23, 24, 99, 79, 85, 97, 32, 17, 71, 89, 11, 8, 22, 82, 5, 4, 14, 3], [63, 103, 22, 33, 92, 87, 13, 65, 18, 120, 10, 50, 4, 119, 126, 111, 54, 0, 61, 109, 49, 6, 62, 58, 81, 51, 117, 48, 127, 47, 125, 123, 122, 118, 59, 124, 46, 8, 45, 60, 56, 121, 115, 44, 30, 110, 113, 53, 116, 114, 112, 29, 2, 57, 71, 40, 55, 42, 43, 102, 52, 108, 37, 107, 25, 105, 73, 41, 98, 36, 20, 106, 104, 38, 11, 3, 34, 5, 39, 101, 91, 14, 100, 12, 7, 16, 78, 89, 67, 99, 83, 79, 31, 80, 24, 90, 76, 15, 32, 70, 95, 27, 93, 96, 35, 88, 21, 74, 94, 23, 17, 9, 85, 84, 26, 28, 86, 69, 75, 82, 19, 64, 1, 68, 97, 72, 66, 77], [47, 111, 64, 86, 95, 16, 84, 28, 25, 18, 74, 4, 1, 2, 72, 7, 97, 78, 67, 65, 6, 69, 15, 17, 29, 123, 12, 3, 124, 23, 76, 103, 75, 54, 31, 14, 0, 39, 77, 120, 55, 53, 27, 113, 85, 79, 61, 110, 70, 35, 81, 56, 13, 63, 122, 36, 105, 50, 126, 9, 19, 33, 101, 87, 91, 121, 57, 71, 125, 114, 32, 117, 62, 116, 24, 73, 48, 99, 109, 38, 127, 100, 108, 106, 118, 43, 115, 46, 52, 37, 44, 41, 119, 21, 42, 51, 26, 102, 60, 104, 30, 45, 107, 59, 34, 58, 90, 40, 98, 80, 94, 88, 49, 92, 82, 112, 89, 8, 96, 93, 20, 83, 11, 22, 10, 5, 66, 68], [49, 38, 97, 91, 25, 93, 117, 23, 80, 20, 92, 82, 15, 9, 64, 77, 11, 85, 2, 30, 68, 65, 87, 3, 45, 14, 108, 21, 5, 22, 17, 95, 96, 94, 50, 71, 120, 112, 36, 35, 18, 61, 37, 119, 8, 6, 118, 53, 70, 48, 46, 126, 106, 109, 55, 104, 59, 57, 34, 58, 122, 63, 111, 107, 33, 54, 123, 40, 62, 103, 52, 78, 105, 116, 124, 121, 39, 110, 99, 47, 28, 44, 24, 31, 101, 115, 42, 56, 41, 43, 89, 51, 127, 114, 125, 10, 60, 90, 83, 32, 19, 26, 98, 29, 1, 13, 67, 7, 100, 72, 113, 88, 84, 86, 0, 102, 12, 75, 81, 79, 16, 27, 76, 69, 74, 66, 73, 4], [110, 117, 102, 86, 33, 30, 52, 92, 25, 99, 58, 62, 119, 22, 125, 123, 54, 122, 114, 126, 53, 112, 55, 14, 81, 49, 111, 116, 51, 124, 47, 121, 48, 59, 61, 44, 103, 56, 63, 120, 75, 41, 108, 113, 115, 57, 118, 50, 83, 109, 16, 127, 39, 38, 107, 105, 35, 43, 31, 104, 46, 106, 28, 42, 45, 94, 91, 34, 32, 85, 23, 40, 37, 72, 100, 24, 101, 93, 26, 36, 18, 12, 29, 90, 5, 97, 9, 27, 98, 60, 84, 67, 0, 96, 95, 21, 87, 6, 20, 19, 82, 89, 80, 8, 78, 15, 2, 1, 7, 10, 88, 76, 68, 13, 77, 11, 79, 3, 17, 74, 65, 71, 64, 70, 69, 66, 73, 4], [105, 64, 35, 1, 118, 0, 116, 108, 67, 51, 119, 71, 19, 13, 86, 69, 73, 66, 4, 80, 65, 49, 2, 124, 48, 57, 12, 61, 122, 113, 47, 46, 114, 115, 43, 111, 44, 54, 3, 63, 99, 117, 126, 50, 74, 53, 58, 62, 121, 123, 52, 107, 60, 5, 56, 70, 106, 89, 6, 93, 55, 15, 103, 110, 59, 29, 94, 20, 88, 26, 112, 104, 17, 7, 75, 85, 72, 41, 33, 109, 76, 79, 97, 68, 45, 120, 90, 92, 9, 18, 101, 11, 125, 102, 84, 91, 30, 96, 32, 8, 77, 82, 87, 27, 23, 100, 38, 34, 78, 24, 14, 40, 21, 31, 37, 39, 127, 25, 98, 95, 42, 28, 81, 10, 83, 36, 22, 16], [42, 30, 99, 118, 35, 86, 26, 51, 21, 19, 78, 113, 61, 17, 64, 45, 11, 121, 66, 63, 111, 117, 106, 125, 28, 73, 124, 114, 123, 102, 71, 126, 53, 80, 108, 49, 122, 62, 12, 105, 41, 4, 32, 43, 74, 116, 16, 7, 38, 23, 56, 40, 119, 44, 55, 52, 3, 46, 104, 54, 65, 101, 115, 24, 47, 127, 110, 57, 109, 79, 69, 95, 70, 89, 36, 103, 58, 120, 48, 39, 29, 112, 33, 107, 98, 100, 25, 1, 59, 60, 50, 97, 37, 34, 94, 87, 15, 27, 91, 92, 20, 6, 72, 82, 31, 96, 85, 88, 93, 8, 10, 84, 67, 77, 13, 68, 18, 5, 75, 76, 90, 0, 22, 83, 81, 9, 2, 14]], "model.layers.29.self_attn.qk_proj": [[49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 99, 22, 57, 106, 118, 41, 113, 86, 64, 0, 89, 51, 35, 119, 38, 16, 80, 53, 93, 61, 116, 95, 52, 28, 115, 82, 120, 124, 121, 94, 114, 62, 122, 25, 27, 46, 58, 125, 19, 44, 77, 102, 98, 18, 103, 83, 13, 54, 126, 108, 65, 50, 55, 33, 1, 39, 48, 10, 127, 3, 92, 71, 68, 78, 43, 112, 123, 7, 4, 97, 59, 14, 67, 107, 31, 56, 2, 66, 9, 90, 87, 5, 69, 74, 84, 73, 6, 21, 60, 26, 85, 29, 30, 11, 12, 23, 20, 75, 76, 79, 104, 81, 15, 8, 34, 72, 37, 17, 40, 101, 91, 36, 32, 24, 96, 88, 100, 70], [49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 57, 41, 106, 99, 22, 0, 118, 64, 113, 86, 51, 89, 52, 35, 38, 119, 61, 16, 115, 95, 80, 121, 120, 124, 28, 116, 94, 62, 93, 114, 55, 53, 122, 59, 65, 82, 25, 18, 98, 127, 46, 44, 58, 77, 50, 27, 123, 13, 126, 83, 102, 54, 2, 43, 68, 1, 108, 33, 125, 103, 19, 48, 39, 4, 7, 3, 74, 71, 87, 67, 14, 97, 92, 112, 5, 78, 10, 31, 73, 66, 56, 60, 9, 90, 107, 21, 11, 6, 29, 76, 84, 69, 75, 12, 23, 20, 30, 104, 17, 85, 34, 40, 8, 72, 79, 81, 15, 26, 91, 36, 101, 88, 37, 24, 70, 32, 96, 100], [49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 57, 99, 22, 113, 106, 64, 41, 86, 0, 118, 89, 35, 51, 119, 38, 115, 52, 124, 116, 16, 61, 80, 93, 120, 95, 62, 94, 122, 18, 28, 98, 102, 123, 25, 59, 53, 82, 65, 127, 121, 19, 50, 27, 67, 48, 46, 54, 83, 126, 125, 114, 77, 1, 44, 7, 58, 33, 43, 56, 108, 103, 2, 3, 13, 68, 112, 14, 39, 66, 10, 92, 97, 4, 55, 78, 90, 71, 9, 74, 84, 69, 87, 60, 73, 5, 21, 75, 31, 29, 107, 15, 20, 30, 12, 23, 11, 76, 85, 104, 6, 26, 70, 79, 81, 34, 40, 37, 101, 72, 17, 8, 91, 36, 32, 96, 88, 100, 24], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 57, 106, 41, 99, 22, 113, 86, 64, 0, 118, 35, 89, 122, 51, 119, 115, 52, 16, 124, 62, 116, 80, 38, 120, 61, 25, 94, 93, 95, 53, 28, 123, 46, 18, 121, 114, 82, 55, 54, 126, 102, 19, 1, 44, 50, 77, 39, 66, 127, 13, 3, 98, 125, 43, 33, 56, 48, 59, 58, 4, 27, 78, 83, 71, 7, 65, 68, 67, 14, 103, 97, 74, 69, 60, 108, 112, 11, 9, 31, 73, 2, 107, 87, 92, 70, 21, 10, 5, 76, 90, 85, 26, 29, 20, 79, 104, 84, 23, 12, 75, 15, 37, 81, 32, 72, 30, 36, 40, 101, 8, 6, 91, 34, 17, 88, 100, 24, 96], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 41, 106, 57, 99, 86, 22, 64, 118, 0, 113, 89, 51, 119, 124, 61, 35, 122, 16, 115, 52, 80, 62, 121, 120, 38, 54, 28, 46, 116, 25, 93, 18, 82, 95, 94, 53, 125, 114, 48, 44, 58, 126, 98, 1, 102, 83, 19, 27, 13, 59, 123, 50, 77, 43, 55, 78, 127, 7, 112, 39, 56, 103, 4, 68, 108, 67, 71, 73, 33, 65, 74, 66, 87, 14, 70, 2, 107, 10, 92, 69, 31, 97, 84, 3, 9, 60, 90, 30, 5, 21, 20, 11, 26, 76, 81, 23, 29, 15, 12, 85, 79, 91, 75, 104, 40, 8, 34, 37, 101, 100, 32, 17, 72, 88, 96, 24, 36, 6], [49, 63, 117, 105, 111, 47, 45, 109, 42, 41, 110, 106, 57, 22, 86, 118, 0, 99, 89, 64, 51, 113, 119, 35, 80, 52, 16, 116, 38, 115, 122, 124, 95, 61, 18, 44, 62, 28, 82, 94, 46, 53, 121, 93, 25, 125, 120, 27, 13, 126, 58, 19, 54, 114, 98, 50, 55, 48, 77, 65, 83, 71, 112, 102, 4, 123, 108, 107, 33, 59, 39, 68, 1, 56, 127, 103, 66, 78, 97, 92, 3, 87, 67, 74, 70, 14, 73, 10, 2, 43, 7, 60, 11, 90, 5, 31, 84, 21, 20, 9, 30, 69, 76, 23, 15, 34, 79, 29, 37, 75, 12, 8, 85, 40, 17, 26, 91, 81, 72, 104, 36, 32, 88, 101, 6, 24, 100, 96], [49, 63, 105, 117, 111, 47, 45, 109, 42, 41, 106, 22, 57, 99, 110, 86, 118, 64, 89, 0, 51, 35, 113, 38, 80, 119, 16, 95, 52, 115, 61, 116, 120, 94, 28, 46, 124, 122, 121, 126, 18, 125, 82, 53, 77, 93, 62, 25, 65, 13, 50, 27, 83, 1, 114, 112, 44, 127, 54, 123, 59, 43, 58, 48, 19, 103, 2, 102, 3, 67, 55, 98, 14, 108, 31, 71, 39, 4, 68, 74, 78, 10, 87, 90, 92, 5, 107, 7, 33, 97, 66, 29, 56, 60, 21, 9, 30, 84, 23, 70, 11, 73, 76, 20, 26, 85, 69, 12, 15, 34, 37, 81, 72, 75, 79, 17, 40, 91, 104, 101, 8, 24, 88, 100, 96, 32, 36, 6], [49, 63, 117, 105, 111, 47, 45, 109, 42, 41, 110, 99, 22, 106, 86, 57, 0, 113, 118, 64, 51, 38, 35, 89, 119, 52, 95, 28, 61, 115, 120, 93, 124, 80, 122, 62, 116, 16, 94, 25, 53, 121, 125, 82, 58, 123, 77, 102, 27, 103, 126, 44, 98, 54, 1, 83, 18, 114, 65, 112, 13, 127, 39, 48, 67, 59, 50, 19, 46, 56, 71, 108, 33, 68, 2, 66, 3, 97, 43, 92, 7, 55, 31, 14, 87, 90, 74, 10, 78, 4, 5, 23, 9, 107, 73, 20, 29, 85, 60, 21, 30, 69, 11, 84, 70, 26, 12, 40, 75, 79, 76, 15, 81, 17, 34, 37, 72, 101, 36, 91, 88, 104, 32, 6, 96, 24, 100, 8], [49, 63, 117, 105, 111, 47, 45, 109, 42, 99, 41, 110, 57, 22, 106, 118, 86, 64, 113, 89, 51, 0, 35, 38, 95, 119, 120, 52, 93, 16, 61, 116, 115, 123, 28, 80, 62, 124, 121, 94, 127, 126, 44, 59, 114, 82, 122, 58, 83, 48, 1, 98, 18, 25, 102, 103, 125, 65, 108, 13, 53, 19, 56, 67, 46, 112, 39, 54, 55, 3, 68, 33, 77, 27, 4, 43, 71, 66, 10, 31, 60, 50, 92, 97, 107, 90, 7, 74, 78, 30, 87, 69, 29, 14, 2, 85, 20, 23, 5, 26, 73, 21, 84, 76, 11, 75, 9, 12, 17, 79, 37, 81, 40, 15, 91, 104, 36, 101, 6, 70, 72, 34, 88, 96, 8, 32, 100, 24], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 57, 99, 118, 106, 41, 113, 22, 119, 64, 0, 86, 51, 35, 89, 38, 61, 115, 62, 52, 124, 116, 16, 55, 28, 80, 121, 120, 123, 25, 58, 46, 95, 65, 122, 126, 127, 82, 94, 53, 93, 18, 39, 83, 13, 98, 112, 114, 59, 44, 67, 125, 102, 19, 33, 77, 48, 56, 50, 92, 27, 4, 54, 31, 7, 68, 9, 2, 60, 43, 1, 71, 78, 3, 103, 108, 66, 74, 69, 97, 87, 14, 84, 10, 90, 11, 5, 107, 73, 29, 30, 76, 6, 20, 21, 23, 79, 26, 104, 40, 75, 85, 12, 91, 15, 34, 17, 81, 101, 8, 72, 36, 37, 70, 88, 100, 32, 96, 24], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 41, 99, 64, 22, 118, 0, 106, 57, 86, 51, 113, 89, 119, 35, 52, 38, 16, 122, 80, 61, 116, 25, 82, 28, 95, 121, 120, 93, 115, 46, 124, 18, 94, 126, 62, 125, 127, 53, 13, 19, 48, 65, 50, 1, 44, 83, 123, 59, 71, 4, 27, 3, 78, 114, 54, 77, 2, 55, 39, 108, 7, 58, 9, 102, 67, 14, 97, 10, 66, 103, 74, 68, 6, 73, 5, 33, 60, 56, 92, 98, 90, 69, 20, 23, 11, 87, 43, 112, 29, 84, 21, 30, 75, 31, 15, 85, 107, 12, 76, 79, 26, 81, 72, 104, 91, 17, 34, 32, 8, 37, 40, 101, 70, 36, 96, 24, 88, 100], [49, 63, 105, 117, 111, 47, 45, 109, 42, 41, 110, 22, 57, 106, 99, 86, 64, 118, 0, 89, 113, 35, 122, 52, 51, 38, 119, 16, 80, 61, 120, 28, 116, 95, 115, 93, 62, 82, 25, 124, 94, 18, 121, 19, 126, 53, 13, 83, 125, 103, 58, 1, 59, 3, 77, 48, 44, 55, 46, 4, 27, 123, 54, 102, 33, 114, 50, 2, 71, 43, 65, 78, 98, 68, 74, 67, 127, 108, 39, 14, 112, 60, 10, 66, 92, 97, 7, 107, 31, 56, 20, 90, 75, 9, 21, 6, 85, 23, 69, 87, 5, 29, 12, 26, 73, 76, 11, 81, 84, 30, 40, 17, 79, 34, 91, 15, 8, 37, 72, 36, 88, 24, 104, 100, 32, 101, 70, 96], [49, 63, 105, 117, 111, 47, 45, 109, 42, 110, 57, 99, 106, 41, 118, 113, 22, 86, 51, 64, 89, 120, 119, 0, 35, 52, 38, 61, 115, 116, 80, 95, 94, 124, 25, 122, 28, 16, 62, 46, 125, 121, 93, 43, 44, 53, 59, 58, 126, 123, 98, 102, 83, 18, 114, 33, 39, 82, 55, 65, 27, 48, 103, 77, 127, 19, 56, 54, 50, 13, 108, 112, 3, 67, 92, 1, 97, 71, 4, 90, 31, 60, 23, 2, 66, 68, 10, 78, 87, 73, 74, 107, 7, 26, 21, 40, 5, 29, 69, 85, 30, 20, 84, 14, 76, 9, 75, 6, 91, 17, 15, 12, 79, 34, 81, 11, 104, 101, 88, 100, 8, 37, 24, 96, 72, 36, 70, 32], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 99, 41, 106, 22, 57, 64, 118, 0, 86, 113, 51, 89, 35, 119, 80, 52, 61, 120, 38, 121, 95, 115, 16, 124, 122, 28, 116, 62, 25, 93, 53, 94, 59, 58, 123, 1, 48, 46, 65, 125, 50, 83, 13, 127, 114, 18, 92, 19, 98, 82, 71, 33, 4, 103, 66, 3, 126, 67, 77, 108, 54, 102, 27, 55, 60, 39, 14, 112, 2, 44, 43, 73, 10, 7, 107, 78, 56, 74, 68, 97, 31, 69, 84, 5, 87, 90, 20, 29, 9, 23, 76, 12, 75, 11, 26, 21, 30, 85, 40, 6, 34, 70, 79, 15, 81, 17, 37, 72, 104, 8, 100, 91, 101, 24, 88, 36, 32, 96], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 41, 106, 57, 118, 22, 99, 0, 64, 86, 89, 51, 113, 119, 38, 35, 52, 80, 95, 120, 16, 122, 116, 61, 28, 121, 126, 25, 94, 124, 82, 46, 53, 93, 123, 115, 13, 48, 114, 125, 65, 55, 83, 19, 62, 18, 39, 102, 3, 2, 1, 127, 71, 9, 98, 108, 54, 44, 50, 59, 73, 67, 4, 78, 43, 77, 7, 68, 92, 27, 14, 84, 60, 58, 74, 103, 66, 70, 10, 112, 33, 97, 21, 5, 20, 69, 75, 31, 107, 56, 90, 11, 23, 29, 76, 87, 30, 15, 26, 12, 79, 34, 91, 17, 85, 8, 40, 81, 104, 6, 101, 72, 37, 88, 100, 32, 24, 36, 96], [49, 63, 117, 105, 111, 47, 45, 109, 42, 41, 110, 57, 22, 99, 118, 106, 0, 86, 64, 113, 89, 51, 38, 35, 116, 119, 16, 80, 95, 52, 121, 124, 122, 46, 61, 123, 28, 120, 62, 25, 93, 126, 115, 18, 94, 82, 53, 83, 114, 13, 125, 102, 50, 33, 127, 1, 44, 48, 19, 4, 98, 108, 59, 103, 3, 68, 7, 65, 27, 78, 67, 77, 74, 39, 56, 58, 54, 55, 2, 71, 66, 73, 60, 97, 14, 43, 9, 70, 21, 90, 92, 69, 10, 87, 107, 5, 75, 85, 29, 84, 31, 30, 76, 20, 23, 12, 112, 104, 11, 79, 26, 81, 8, 37, 17, 15, 40, 34, 91, 72, 101, 32, 36, 100, 88, 96, 6, 24], [49, 63, 105, 117, 111, 47, 45, 109, 42, 41, 22, 110, 106, 99, 57, 118, 86, 51, 113, 64, 0, 35, 89, 38, 52, 80, 95, 16, 120, 28, 119, 61, 94, 115, 93, 25, 124, 116, 125, 123, 53, 121, 122, 46, 82, 102, 18, 114, 62, 83, 39, 126, 19, 27, 58, 13, 55, 1, 59, 48, 92, 33, 103, 127, 50, 98, 43, 44, 77, 87, 108, 54, 56, 4, 14, 65, 74, 90, 67, 31, 78, 66, 97, 3, 71, 73, 7, 10, 85, 68, 112, 2, 60, 29, 21, 23, 69, 9, 30, 76, 11, 107, 84, 26, 20, 5, 70, 81, 75, 12, 34, 104, 17, 91, 37, 15, 40, 32, 36, 24, 101, 79, 8, 72, 100, 96, 88, 6], [49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 57, 86, 41, 106, 99, 22, 113, 118, 64, 0, 51, 89, 35, 95, 16, 115, 38, 119, 80, 122, 94, 28, 120, 116, 18, 62, 121, 93, 52, 124, 61, 123, 25, 53, 98, 83, 102, 125, 19, 82, 1, 27, 114, 3, 13, 48, 65, 50, 126, 67, 103, 39, 58, 59, 55, 127, 92, 54, 66, 68, 33, 77, 2, 71, 7, 74, 56, 46, 44, 14, 112, 78, 4, 108, 73, 10, 97, 60, 43, 5, 29, 20, 31, 9, 69, 90, 85, 30, 70, 87, 84, 23, 76, 107, 21, 26, 12, 75, 11, 15, 34, 81, 79, 91, 40, 17, 104, 101, 88, 8, 36, 72, 37, 32, 96, 6, 24, 100], [49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 41, 106, 57, 118, 99, 22, 113, 64, 86, 0, 51, 89, 122, 38, 123, 119, 115, 120, 35, 121, 116, 61, 16, 28, 94, 80, 58, 55, 95, 25, 62, 124, 102, 125, 52, 127, 93, 53, 82, 19, 46, 39, 114, 13, 126, 43, 59, 18, 44, 83, 1, 54, 65, 66, 98, 33, 50, 92, 56, 7, 77, 48, 3, 27, 112, 108, 71, 14, 67, 78, 9, 73, 4, 103, 10, 31, 2, 23, 68, 29, 74, 60, 97, 107, 104, 5, 21, 69, 84, 30, 90, 87, 85, 12, 76, 20, 75, 11, 15, 26, 70, 91, 34, 72, 81, 6, 79, 36, 17, 40, 37, 8, 96, 88, 101, 32, 100, 24], [49, 63, 117, 105, 111, 47, 45, 109, 42, 41, 110, 106, 57, 99, 22, 118, 86, 64, 113, 51, 0, 89, 35, 119, 38, 61, 52, 16, 28, 94, 120, 124, 116, 122, 80, 121, 62, 115, 82, 48, 125, 55, 25, 95, 93, 102, 127, 123, 54, 53, 39, 58, 46, 44, 83, 98, 114, 27, 59, 19, 18, 65, 1, 13, 112, 126, 33, 14, 2, 50, 77, 74, 3, 108, 68, 103, 73, 92, 43, 97, 71, 4, 7, 66, 78, 31, 60, 9, 10, 21, 56, 67, 6, 5, 90, 75, 23, 87, 76, 84, 69, 11, 29, 107, 20, 30, 85, 12, 79, 104, 26, 17, 72, 81, 36, 34, 37, 15, 101, 8, 70, 88, 40, 91, 96, 24, 32, 100], [49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 57, 106, 22, 41, 99, 86, 118, 113, 0, 51, 89, 64, 35, 124, 61, 120, 62, 38, 119, 116, 16, 125, 93, 28, 121, 115, 94, 52, 80, 95, 122, 53, 25, 102, 18, 50, 44, 48, 83, 123, 82, 27, 98, 67, 114, 59, 1, 55, 54, 66, 13, 33, 77, 46, 108, 19, 103, 58, 4, 126, 112, 2, 71, 43, 14, 39, 3, 127, 7, 92, 65, 10, 97, 74, 9, 78, 69, 73, 60, 31, 68, 107, 56, 20, 87, 75, 23, 6, 84, 21, 90, 30, 29, 11, 5, 15, 76, 40, 26, 37, 104, 79, 12, 91, 34, 72, 81, 17, 85, 36, 24, 8, 32, 101, 96, 70, 100, 88], [49, 63, 117, 111, 105, 47, 45, 109, 42, 57, 41, 64, 110, 106, 22, 0, 99, 113, 118, 86, 51, 119, 124, 89, 62, 38, 80, 116, 35, 52, 121, 122, 95, 120, 16, 93, 115, 61, 28, 123, 46, 25, 102, 94, 48, 98, 127, 50, 1, 18, 58, 125, 83, 65, 53, 33, 2, 13, 59, 19, 3, 114, 77, 82, 54, 66, 4, 71, 27, 44, 112, 55, 103, 39, 67, 108, 7, 97, 56, 78, 126, 60, 92, 31, 6, 9, 68, 43, 14, 73, 10, 74, 107, 5, 29, 87, 30, 90, 69, 84, 75, 23, 72, 21, 20, 11, 85, 76, 104, 37, 26, 91, 79, 40, 12, 15, 17, 34, 81, 101, 36, 96, 8, 32, 88, 24, 70, 100], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 41, 106, 57, 99, 22, 118, 113, 64, 86, 51, 0, 52, 89, 35, 119, 121, 62, 120, 16, 38, 80, 95, 61, 28, 124, 93, 58, 25, 122, 94, 83, 115, 125, 116, 98, 48, 82, 123, 102, 39, 46, 55, 27, 53, 103, 18, 126, 127, 108, 114, 44, 13, 19, 77, 43, 54, 50, 33, 1, 59, 7, 65, 3, 71, 56, 4, 92, 31, 66, 78, 97, 10, 112, 29, 2, 67, 68, 9, 74, 69, 73, 87, 90, 14, 107, 5, 30, 76, 60, 84, 6, 20, 26, 23, 21, 11, 79, 75, 85, 40, 12, 104, 34, 91, 37, 72, 15, 17, 81, 101, 8, 70, 88, 32, 36, 96, 100, 24], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 41, 106, 57, 22, 99, 118, 0, 86, 51, 89, 113, 52, 35, 64, 16, 119, 62, 38, 28, 121, 95, 116, 115, 80, 122, 124, 82, 25, 120, 94, 61, 93, 125, 53, 58, 27, 46, 18, 98, 55, 126, 48, 50, 102, 83, 114, 19, 33, 39, 3, 77, 123, 71, 13, 108, 127, 54, 1, 103, 68, 44, 78, 74, 65, 59, 7, 97, 112, 67, 43, 66, 14, 4, 92, 60, 56, 10, 73, 69, 9, 21, 84, 76, 31, 2, 30, 11, 87, 107, 29, 5, 6, 85, 23, 90, 75, 20, 26, 12, 34, 72, 37, 79, 15, 40, 101, 8, 36, 104, 81, 17, 91, 70, 88, 32, 96, 24, 100], [49, 63, 117, 111, 105, 47, 45, 109, 42, 41, 106, 57, 118, 22, 110, 99, 86, 113, 64, 0, 89, 119, 51, 35, 124, 62, 52, 80, 28, 38, 61, 16, 121, 116, 120, 115, 95, 93, 122, 82, 53, 25, 94, 83, 125, 18, 54, 114, 48, 27, 98, 77, 46, 55, 127, 108, 102, 59, 58, 39, 50, 65, 126, 13, 123, 44, 66, 19, 1, 43, 74, 92, 33, 78, 2, 3, 67, 103, 97, 71, 68, 7, 4, 112, 73, 14, 31, 84, 21, 10, 87, 60, 20, 9, 29, 23, 11, 90, 5, 75, 69, 107, 56, 30, 76, 26, 40, 12, 34, 70, 85, 6, 104, 91, 15, 37, 101, 81, 79, 8, 72, 17, 32, 96, 88, 24, 100, 36], [49, 63, 117, 105, 111, 47, 45, 109, 42, 41, 99, 110, 57, 118, 22, 106, 86, 0, 120, 113, 64, 89, 51, 35, 119, 124, 62, 61, 52, 95, 16, 38, 80, 28, 115, 94, 116, 93, 122, 126, 121, 25, 102, 123, 58, 48, 83, 18, 53, 125, 114, 82, 55, 27, 127, 54, 98, 46, 33, 65, 59, 44, 77, 13, 103, 108, 2, 39, 19, 97, 1, 60, 92, 10, 43, 7, 50, 112, 67, 3, 71, 74, 78, 87, 31, 14, 4, 73, 68, 90, 66, 69, 56, 29, 9, 107, 11, 21, 30, 20, 23, 5, 85, 70, 76, 40, 84, 12, 26, 37, 75, 91, 17, 104, 15, 81, 36, 34, 8, 101, 100, 72, 79, 88, 32, 96, 6, 24], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 118, 41, 22, 99, 57, 106, 86, 64, 0, 89, 51, 113, 35, 119, 38, 52, 124, 61, 120, 116, 80, 115, 16, 95, 126, 25, 102, 121, 93, 28, 122, 82, 114, 53, 62, 94, 54, 125, 83, 108, 58, 1, 123, 18, 39, 103, 19, 127, 27, 44, 50, 3, 55, 33, 66, 46, 67, 7, 92, 59, 13, 77, 65, 48, 43, 98, 4, 74, 112, 107, 97, 56, 68, 87, 78, 71, 9, 60, 90, 31, 10, 5, 14, 73, 2, 29, 26, 84, 85, 69, 20, 21, 11, 30, 70, 23, 12, 37, 17, 76, 75, 15, 34, 81, 104, 91, 40, 36, 79, 72, 101, 32, 8, 88, 96, 100, 24, 6], [49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 118, 57, 41, 106, 22, 99, 64, 86, 113, 0, 51, 89, 119, 124, 52, 61, 120, 38, 62, 115, 122, 28, 35, 95, 16, 116, 82, 94, 80, 53, 25, 121, 93, 58, 98, 18, 114, 46, 19, 44, 126, 83, 125, 33, 123, 77, 1, 13, 127, 27, 59, 54, 55, 102, 9, 108, 39, 50, 3, 60, 7, 65, 78, 68, 71, 43, 4, 92, 66, 48, 10, 67, 73, 2, 14, 103, 31, 56, 97, 74, 5, 84, 70, 112, 87, 29, 23, 21, 69, 90, 107, 11, 15, 30, 85, 20, 76, 75, 79, 26, 37, 8, 12, 104, 17, 40, 101, 91, 34, 72, 36, 81, 32, 88, 6, 24, 96, 100], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 41, 57, 106, 99, 118, 22, 64, 0, 113, 86, 51, 119, 35, 89, 61, 38, 16, 120, 116, 46, 52, 80, 124, 53, 115, 25, 95, 94, 122, 28, 121, 62, 18, 125, 55, 126, 93, 114, 82, 44, 58, 65, 33, 77, 13, 108, 54, 98, 4, 48, 102, 27, 59, 1, 19, 83, 127, 60, 39, 123, 2, 78, 103, 43, 9, 56, 50, 92, 73, 67, 14, 71, 68, 70, 3, 66, 74, 7, 10, 112, 97, 29, 11, 87, 31, 5, 69, 84, 90, 20, 21, 107, 30, 75, 26, 76, 85, 79, 23, 8, 40, 34, 17, 91, 104, 15, 12, 81, 6, 36, 72, 101, 37, 100, 32, 88, 96, 24], [49, 63, 117, 111, 105, 47, 45, 109, 42, 41, 57, 110, 22, 99, 106, 86, 118, 51, 64, 113, 89, 0, 80, 35, 16, 120, 38, 119, 61, 95, 122, 52, 115, 124, 94, 25, 93, 121, 116, 62, 46, 28, 125, 82, 18, 126, 53, 27, 102, 58, 83, 44, 114, 103, 65, 13, 55, 54, 77, 108, 98, 127, 19, 123, 50, 39, 33, 3, 48, 59, 60, 7, 92, 10, 71, 67, 68, 4, 1, 56, 90, 9, 43, 78, 74, 14, 66, 87, 69, 97, 31, 112, 21, 29, 73, 84, 2, 23, 85, 107, 5, 11, 20, 70, 30, 26, 79, 34, 12, 37, 6, 15, 40, 76, 81, 75, 104, 91, 72, 17, 8, 101, 88, 36, 24, 96, 32, 100], [49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 57, 99, 41, 22, 0, 118, 106, 86, 64, 51, 113, 89, 35, 80, 119, 62, 52, 38, 124, 28, 16, 120, 95, 116, 121, 115, 61, 93, 122, 53, 18, 123, 94, 46, 25, 83, 98, 125, 77, 102, 48, 58, 54, 108, 82, 4, 55, 126, 1, 44, 27, 7, 59, 114, 127, 68, 39, 66, 103, 65, 50, 92, 71, 43, 10, 13, 33, 73, 2, 3, 74, 112, 90, 9, 67, 19, 60, 14, 78, 11, 87, 20, 29, 31, 97, 12, 21, 56, 6, 107, 5, 104, 69, 84, 40, 30, 75, 23, 85, 34, 76, 79, 26, 91, 70, 8, 37, 101, 81, 72, 17, 24, 88, 15, 32, 100, 36, 96], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 41, 57, 22, 118, 106, 99, 86, 113, 51, 35, 119, 89, 64, 0, 61, 115, 38, 16, 52, 80, 62, 28, 53, 120, 95, 124, 121, 116, 122, 94, 25, 125, 93, 77, 114, 18, 55, 82, 102, 44, 27, 59, 98, 83, 48, 50, 126, 33, 19, 54, 123, 13, 7, 58, 65, 108, 10, 46, 3, 39, 127, 97, 71, 92, 103, 112, 1, 68, 67, 43, 78, 74, 14, 31, 73, 2, 87, 66, 4, 5, 90, 60, 11, 21, 85, 56, 23, 84, 107, 6, 26, 20, 12, 9, 69, 75, 29, 76, 34, 30, 104, 40, 79, 81, 15, 91, 17, 72, 8, 101, 37, 36, 32, 24, 96, 88, 100, 70]], "model.layers.30.self_attn.q_proj": [[61, 38, 48, 50, 97, 49, 111, 123, 83, 64, 89, 1, 127, 108, 79, 41, 86, 28, 94, 114, 14, 116, 102, 106, 17, 119, 67, 57, 54, 4, 40, 13, 59, 65, 85, 73, 109, 39, 52, 37, 58, 126, 12, 68, 60, 92, 84, 45, 120, 11, 125, 76, 8, 72, 44, 55, 87, 105, 80, 121, 43, 10, 2, 62, 110, 63, 74, 46, 118, 51, 81, 42, 101, 23, 3, 24, 9, 56, 71, 6, 112, 124, 5, 31, 15, 53, 35, 107, 0, 34, 117, 115, 21, 18, 88, 36, 113, 100, 7, 27, 47, 104, 16, 69, 93, 26, 90, 19, 95, 77, 103, 122, 20, 82, 32, 96, 66, 91, 99, 30, 29, 75, 22, 98, 33, 70, 78, 25], [61, 38, 50, 48, 97, 89, 112, 83, 55, 108, 17, 11, 60, 114, 72, 57, 85, 70, 28, 14, 111, 15, 62, 101, 3, 92, 123, 66, 25, 127, 34, 6, 126, 113, 36, 106, 86, 59, 52, 87, 69, 124, 39, 102, 45, 58, 0, 120, 94, 125, 63, 37, 29, 118, 104, 35, 43, 109, 47, 115, 10, 64, 119, 121, 56, 122, 51, 49, 40, 22, 27, 95, 53, 4, 46, 116, 90, 103, 21, 24, 32, 107, 81, 110, 23, 117, 26, 44, 19, 99, 16, 76, 42, 105, 41, 98, 88, 77, 30, 67, 20, 54, 73, 31, 93, 100, 65, 96, 18, 8, 84, 82, 12, 5, 80, 1, 78, 2, 91, 75, 7, 33, 71, 79, 13, 68, 74, 9], [50, 61, 38, 111, 48, 55, 97, 89, 106, 45, 86, 127, 60, 41, 125, 123, 124, 120, 17, 85, 102, 108, 70, 121, 62, 112, 92, 59, 52, 58, 51, 43, 119, 114, 49, 126, 66, 87, 11, 57, 83, 14, 53, 46, 39, 28, 40, 101, 115, 113, 104, 118, 117, 47, 56, 116, 122, 105, 63, 54, 44, 110, 103, 91, 3, 25, 72, 107, 109, 32, 36, 42, 0, 64, 34, 15, 6, 100, 30, 35, 94, 9, 33, 22, 37, 81, 90, 10, 31, 21, 95, 27, 96, 13, 12, 24, 99, 88, 26, 78, 84, 93, 19, 69, 67, 4, 98, 8, 16, 23, 18, 76, 29, 82, 77, 65, 73, 20, 80, 2, 7, 5, 79, 75, 1, 71, 68, 74], [61, 50, 111, 38, 112, 55, 60, 114, 124, 48, 110, 125, 97, 123, 109, 92, 53, 127, 89, 58, 52, 51, 62, 121, 44, 28, 118, 59, 41, 108, 119, 120, 45, 102, 46, 63, 126, 122, 116, 47, 104, 113, 56, 117, 54, 43, 101, 105, 57, 107, 37, 42, 29, 115, 88, 86, 106, 103, 40, 80, 100, 39, 34, 91, 49, 35, 30, 84, 36, 85, 83, 93, 25, 98, 32, 95, 99, 24, 16, 94, 17, 31, 23, 18, 22, 15, 69, 96, 21, 76, 82, 11, 90, 33, 87, 77, 20, 27, 81, 26, 4, 13, 19, 12, 68, 7, 73, 2, 71, 9, 5, 79, 75, 14, 10, 72, 74, 66, 70, 6, 0, 78, 1, 64, 67, 8, 65, 3], [46, 110, 34, 92, 119, 86, 24, 90, 83, 113, 61, 49, 54, 95, 82, 94, 31, 38, 70, 85, 74, 0, 121, 13, 7, 120, 127, 114, 12, 8, 101, 9, 59, 29, 79, 17, 47, 112, 23, 1, 48, 98, 53, 80, 51, 52, 4, 63, 2, 109, 67, 97, 55, 26, 42, 125, 58, 116, 36, 117, 60, 40, 41, 126, 108, 18, 62, 56, 124, 111, 122, 43, 115, 123, 73, 57, 105, 87, 118, 91, 81, 45, 25, 102, 28, 11, 19, 39, 37, 106, 64, 107, 100, 44, 104, 66, 93, 16, 27, 103, 20, 33, 88, 50, 96, 32, 68, 89, 35, 22, 84, 14, 5, 99, 21, 69, 30, 65, 78, 3, 10, 15, 6, 77, 71, 75, 76, 72], [46, 110, 34, 90, 86, 92, 24, 79, 18, 113, 31, 95, 83, 17, 119, 9, 70, 54, 4, 85, 42, 74, 55, 56, 61, 12, 120, 59, 58, 107, 98, 117, 7, 63, 67, 115, 13, 44, 50, 49, 116, 88, 114, 38, 16, 36, 40, 96, 29, 73, 8, 94, 91, 20, 48, 111, 52, 23, 97, 118, 37, 81, 2, 0, 109, 100, 57, 122, 125, 43, 45, 1, 51, 39, 127, 103, 93, 104, 47, 41, 27, 106, 5, 123, 33, 126, 101, 112, 124, 80, 26, 62, 28, 14, 102, 15, 32, 60, 87, 77, 108, 11, 78, 75, 105, 99, 84, 53, 76, 25, 69, 121, 35, 89, 30, 82, 21, 19, 72, 10, 68, 64, 6, 22, 66, 3, 65, 71], [46, 34, 110, 113, 120, 92, 24, 83, 90, 42, 31, 86, 17, 94, 8, 79, 14, 54, 118, 112, 117, 15, 52, 9, 66, 100, 45, 85, 36, 55, 38, 70, 13, 76, 84, 58, 11, 60, 10, 51, 98, 3, 56, 4, 63, 0, 29, 95, 49, 65, 32, 12, 48, 127, 119, 124, 122, 39, 59, 116, 47, 104, 41, 23, 43, 53, 126, 19, 44, 107, 61, 75, 7, 62, 1, 111, 67, 78, 106, 22, 114, 68, 109, 105, 35, 108, 18, 80, 82, 37, 123, 115, 96, 91, 102, 88, 97, 28, 93, 26, 6, 103, 50, 33, 40, 27, 121, 69, 87, 5, 20, 25, 57, 71, 101, 81, 30, 2, 89, 21, 125, 16, 77, 74, 73, 99, 72, 64], [46, 110, 34, 92, 90, 24, 86, 49, 83, 69, 13, 113, 114, 95, 61, 119, 31, 23, 85, 56, 94, 53, 80, 100, 17, 98, 54, 108, 9, 125, 57, 52, 103, 79, 27, 115, 74, 63, 50, 11, 18, 126, 111, 7, 41, 99, 122, 25, 28, 120, 1, 67, 55, 88, 29, 4, 26, 87, 62, 101, 45, 39, 33, 117, 104, 36, 43, 60, 102, 58, 38, 116, 118, 127, 59, 40, 51, 96, 44, 124, 123, 109, 73, 42, 121, 15, 70, 107, 5, 97, 106, 30, 112, 91, 78, 48, 47, 37, 89, 35, 32, 76, 105, 12, 0, 20, 21, 81, 75, 93, 16, 71, 19, 14, 8, 84, 64, 68, 10, 82, 72, 2, 65, 3, 22, 77, 6, 66], [42, 63, 117, 97, 27, 86, 106, 89, 16, 78, 31, 84, 115, 52, 11, 99, 81, 76, 53, 119, 96, 56, 49, 13, 10, 88, 100, 26, 73, 92, 121, 55, 118, 7, 24, 51, 50, 61, 114, 57, 8, 102, 127, 90, 20, 122, 41, 23, 46, 19, 116, 60, 28, 95, 82, 38, 6, 25, 36, 58, 94, 33, 120, 91, 62, 22, 35, 21, 54, 110, 83, 48, 9, 59, 15, 18, 66, 5, 125, 111, 126, 109, 17, 124, 72, 47, 39, 103, 112, 107, 108, 85, 113, 44, 123, 87, 40, 93, 14, 69, 80, 29, 45, 4, 77, 70, 30, 105, 43, 32, 74, 34, 101, 104, 98, 3, 12, 79, 37, 75, 68, 0, 71, 65, 67, 2, 1, 64], [42, 63, 117, 97, 35, 16, 56, 84, 27, 83, 86, 102, 101, 94, 92, 18, 19, 57, 11, 64, 106, 121, 49, 116, 93, 52, 100, 4, 37, 103, 24, 29, 98, 81, 72, 85, 65, 77, 99, 127, 9, 50, 76, 88, 70, 54, 55, 122, 61, 31, 73, 32, 96, 51, 119, 118, 58, 120, 68, 115, 78, 79, 95, 114, 82, 26, 74, 90, 109, 111, 28, 20, 125, 124, 23, 25, 41, 60, 7, 59, 48, 34, 3, 10, 80, 47, 46, 43, 36, 113, 67, 53, 5, 108, 126, 66, 1, 22, 44, 110, 105, 89, 15, 39, 21, 112, 38, 12, 30, 91, 8, 45, 62, 40, 104, 33, 123, 14, 6, 69, 13, 0, 107, 2, 17, 87, 71, 75], [42, 63, 11, 81, 78, 7, 97, 117, 84, 1, 86, 106, 27, 102, 94, 4, 55, 0, 65, 114, 6, 116, 98, 16, 9, 21, 13, 2, 100, 66, 58, 50, 92, 119, 89, 14, 30, 19, 101, 115, 93, 8, 64, 67, 52, 35, 18, 77, 76, 71, 82, 73, 75, 88, 31, 108, 12, 38, 70, 122, 103, 68, 72, 36, 20, 83, 10, 3, 91, 17, 23, 74, 80, 24, 79, 15, 32, 26, 5, 22, 29, 28, 90, 85, 120, 46, 96, 53, 39, 61, 69, 25, 34, 99, 49, 95, 118, 59, 87, 104, 121, 123, 57, 111, 47, 109, 126, 56, 37, 54, 124, 62, 45, 110, 125, 127, 48, 33, 60, 51, 105, 44, 41, 40, 43, 113, 107, 112], [63, 42, 55, 57, 49, 56, 51, 121, 112, 52, 122, 54, 116, 118, 127, 61, 120, 50, 48, 111, 59, 125, 47, 124, 108, 113, 115, 126, 60, 117, 114, 110, 97, 62, 44, 53, 85, 58, 123, 94, 46, 109, 38, 45, 24, 43, 86, 107, 41, 119, 27, 33, 102, 29, 106, 40, 39, 100, 28, 92, 104, 105, 18, 36, 26, 103, 22, 31, 90, 99, 101, 35, 88, 91, 21, 81, 37, 96, 84, 77, 30, 87, 93, 32, 15, 98, 95, 34, 20, 17, 82, 83, 19, 79, 25, 23, 80, 13, 16, 89, 12, 73, 75, 76, 11, 70, 78, 68, 74, 1, 10, 14, 0, 4, 8, 5, 67, 72, 9, 7, 6, 69, 71, 65, 66, 64, 2, 3], [111, 44, 61, 125, 116, 84, 108, 20, 124, 97, 52, 75, 100, 60, 11, 120, 117, 48, 115, 46, 47, 106, 56, 50, 49, 123, 121, 119, 28, 112, 87, 70, 63, 62, 55, 53, 59, 122, 127, 118, 110, 57, 113, 42, 90, 126, 114, 39, 51, 92, 45, 88, 54, 58, 93, 107, 109, 105, 32, 41, 98, 104, 91, 40, 79, 102, 25, 43, 15, 17, 86, 30, 37, 103, 27, 38, 31, 26, 101, 19, 35, 81, 33, 34, 36, 94, 99, 74, 23, 95, 83, 96, 22, 29, 89, 10, 2, 82, 21, 6, 16, 13, 85, 66, 24, 72, 64, 67, 18, 0, 76, 14, 65, 77, 68, 80, 5, 71, 9, 78, 4, 1, 12, 8, 73, 3, 7, 69], [44, 111, 116, 88, 18, 97, 61, 16, 78, 9, 115, 90, 51, 108, 89, 30, 109, 100, 71, 47, 4, 125, 102, 85, 26, 76, 2, 20, 120, 83, 10, 86, 79, 80, 73, 56, 68, 92, 23, 82, 94, 93, 112, 27, 55, 95, 11, 58, 66, 6, 40, 64, 118, 14, 87, 70, 52, 63, 53, 42, 127, 122, 126, 65, 7, 124, 67, 41, 50, 49, 12, 19, 24, 119, 91, 113, 28, 29, 74, 117, 81, 101, 59, 25, 1, 114, 103, 13, 38, 43, 21, 31, 17, 32, 107, 57, 123, 48, 96, 98, 0, 121, 72, 62, 34, 60, 84, 105, 106, 77, 46, 22, 8, 35, 54, 104, 5, 3, 45, 39, 36, 99, 33, 110, 37, 15, 75, 69], [44, 61, 108, 125, 97, 106, 115, 85, 88, 100, 116, 47, 51, 49, 120, 117, 126, 46, 124, 52, 87, 28, 93, 112, 59, 111, 50, 48, 119, 55, 17, 122, 92, 76, 57, 127, 62, 90, 123, 113, 118, 114, 56, 63, 60, 121, 21, 26, 53, 32, 54, 107, 58, 105, 45, 91, 40, 30, 42, 12, 104, 109, 86, 41, 39, 79, 27, 74, 110, 94, 103, 5, 34, 102, 43, 19, 23, 69, 38, 98, 83, 36, 37, 33, 31, 15, 101, 81, 95, 82, 99, 22, 35, 67, 72, 25, 89, 84, 96, 13, 2, 10, 29, 64, 18, 24, 16, 71, 20, 7, 77, 80, 3, 0, 66, 8, 78, 14, 65, 70, 4, 9, 11, 1, 68, 6, 73, 75], [44, 125, 111, 108, 61, 116, 115, 97, 51, 117, 87, 52, 122, 88, 106, 112, 49, 55, 126, 60, 100, 120, 93, 47, 59, 124, 58, 50, 113, 57, 48, 119, 118, 62, 127, 28, 46, 45, 92, 56, 63, 121, 114, 123, 90, 42, 17, 53, 74, 39, 41, 107, 104, 109, 19, 54, 86, 32, 79, 30, 91, 102, 40, 105, 13, 98, 110, 103, 26, 94, 15, 43, 95, 38, 36, 83, 35, 25, 101, 27, 37, 34, 89, 81, 96, 22, 23, 31, 33, 99, 29, 77, 67, 21, 85, 72, 84, 82, 10, 16, 2, 64, 24, 0, 9, 18, 70, 20, 80, 65, 8, 76, 5, 71, 78, 3, 14, 66, 12, 7, 11, 69, 75, 1, 73, 4, 68, 6], [113, 103, 122, 124, 52, 33, 55, 93, 119, 107, 47, 24, 42, 26, 48, 31, 112, 43, 54, 104, 90, 117, 123, 56, 45, 85, 41, 57, 108, 118, 116, 53, 87, 83, 115, 121, 37, 62, 120, 63, 35, 61, 99, 22, 29, 125, 114, 91, 59, 23, 39, 40, 21, 28, 60, 105, 49, 44, 102, 111, 101, 98, 51, 92, 127, 126, 106, 110, 58, 32, 36, 50, 38, 88, 46, 34, 97, 95, 100, 27, 11, 109, 96, 84, 30, 89, 18, 94, 82, 8, 10, 16, 72, 19, 14, 20, 80, 13, 71, 25, 75, 86, 77, 74, 81, 15, 78, 70, 5, 7, 68, 65, 2, 76, 6, 69, 0, 66, 64, 79, 12, 67, 17, 4, 3, 73, 1, 9], [122, 103, 33, 52, 93, 31, 117, 119, 91, 83, 22, 24, 54, 112, 111, 124, 90, 85, 26, 60, 105, 43, 27, 125, 42, 107, 45, 39, 97, 127, 29, 113, 20, 114, 96, 123, 73, 92, 118, 53, 78, 46, 61, 120, 110, 44, 58, 86, 116, 40, 115, 102, 48, 51, 21, 49, 47, 38, 62, 50, 55, 109, 101, 108, 25, 121, 126, 95, 35, 81, 56, 99, 37, 36, 63, 19, 30, 41, 28, 15, 57, 13, 34, 104, 89, 84, 87, 94, 32, 100, 106, 9, 82, 80, 23, 75, 98, 18, 14, 16, 59, 72, 88, 77, 5, 6, 64, 70, 67, 8, 1, 11, 0, 69, 10, 65, 68, 2, 17, 71, 4, 3, 79, 66, 7, 76, 12, 74], [57, 122, 103, 52, 113, 110, 105, 112, 22, 33, 54, 38, 101, 59, 116, 125, 43, 111, 60, 26, 109, 118, 50, 48, 97, 55, 3, 34, 39, 47, 126, 92, 78, 45, 127, 24, 40, 124, 31, 114, 61, 64, 120, 90, 75, 65, 119, 49, 123, 108, 83, 115, 18, 67, 20, 68, 71, 99, 36, 107, 106, 104, 44, 62, 63, 46, 100, 53, 96, 102, 25, 41, 42, 117, 19, 95, 51, 58, 74, 84, 121, 93, 56, 37, 21, 89, 87, 27, 94, 30, 32, 13, 35, 73, 86, 88, 82, 85, 98, 72, 28, 91, 10, 14, 23, 4, 77, 29, 9, 6, 8, 2, 1, 7, 0, 69, 66, 80, 70, 11, 16, 12, 15, 5, 79, 81, 17, 76], [103, 113, 93, 122, 24, 33, 15, 81, 76, 29, 83, 17, 57, 85, 10, 70, 104, 88, 21, 36, 34, 48, 19, 7, 22, 84, 97, 26, 66, 107, 5, 35, 12, 27, 124, 87, 42, 54, 30, 38, 55, 90, 23, 43, 16, 82, 74, 102, 75, 8, 20, 121, 63, 86, 49, 119, 94, 77, 45, 91, 68, 127, 14, 71, 62, 117, 73, 0, 79, 61, 13, 25, 32, 28, 2, 106, 80, 111, 11, 69, 116, 18, 47, 98, 59, 52, 31, 58, 89, 3, 72, 67, 92, 4, 78, 37, 96, 40, 99, 123, 100, 95, 118, 110, 39, 120, 112, 6, 46, 101, 65, 9, 64, 105, 60, 53, 126, 108, 44, 125, 114, 1, 41, 50, 115, 51, 56, 109], [56, 39, 60, 53, 117, 122, 94, 127, 109, 123, 116, 92, 58, 50, 124, 61, 103, 105, 59, 51, 55, 52, 113, 33, 111, 90, 114, 119, 115, 62, 97, 110, 54, 42, 121, 63, 47, 96, 57, 98, 34, 43, 95, 25, 36, 44, 45, 26, 108, 46, 112, 19, 126, 28, 49, 91, 85, 125, 30, 107, 118, 40, 38, 120, 102, 37, 106, 104, 88, 48, 41, 100, 23, 35, 101, 99, 27, 31, 29, 81, 20, 32, 83, 86, 87, 84, 89, 22, 93, 24, 76, 21, 78, 14, 17, 15, 75, 16, 66, 18, 79, 71, 74, 9, 0, 5, 4, 12, 67, 11, 80, 65, 82, 73, 7, 69, 70, 1, 2, 72, 77, 8, 68, 64, 3, 10, 6, 13], [53, 39, 60, 123, 61, 124, 55, 56, 50, 109, 51, 116, 54, 127, 120, 57, 122, 63, 97, 115, 47, 52, 59, 121, 90, 112, 119, 26, 58, 113, 117, 62, 114, 108, 126, 28, 125, 44, 49, 118, 107, 43, 94, 48, 110, 46, 19, 45, 105, 106, 111, 23, 38, 42, 41, 103, 101, 29, 100, 40, 85, 33, 104, 87, 102, 34, 25, 78, 35, 99, 22, 92, 95, 37, 36, 86, 30, 96, 98, 31, 91, 32, 83, 93, 21, 24, 17, 18, 27, 81, 20, 15, 14, 84, 75, 88, 89, 11, 79, 72, 76, 82, 16, 80, 12, 5, 77, 9, 74, 70, 66, 13, 67, 7, 6, 71, 8, 2, 69, 3, 10, 4, 65, 73, 0, 68, 1, 64], [39, 53, 56, 25, 18, 77, 16, 23, 74, 71, 4, 60, 97, 70, 64, 9, 0, 91, 52, 29, 31, 76, 34, 92, 67, 65, 122, 1, 100, 117, 30, 36, 15, 84, 13, 124, 40, 68, 2, 12, 82, 125, 99, 20, 10, 5, 17, 69, 98, 8, 94, 55, 6, 104, 79, 19, 81, 73, 14, 85, 42, 61, 88, 32, 7, 50, 86, 11, 90, 26, 3, 72, 24, 66, 21, 83, 75, 27, 80, 37, 107, 93, 38, 118, 87, 22, 114, 89, 113, 35, 78, 28, 41, 95, 63, 46, 126, 109, 96, 127, 51, 101, 58, 119, 33, 116, 123, 120, 59, 105, 45, 108, 102, 47, 112, 106, 54, 115, 62, 43, 57, 121, 44, 111, 110, 49, 48, 103], [39, 56, 53, 25, 18, 16, 23, 34, 97, 30, 83, 77, 52, 60, 122, 76, 29, 100, 74, 86, 93, 124, 61, 11, 94, 104, 10, 9, 71, 15, 22, 21, 13, 36, 55, 127, 90, 20, 26, 108, 91, 27, 119, 19, 80, 89, 98, 35, 40, 117, 79, 88, 92, 82, 59, 70, 107, 31, 12, 65, 51, 85, 84, 28, 41, 17, 4, 78, 102, 14, 7, 64, 87, 67, 75, 114, 72, 99, 8, 69, 81, 0, 37, 46, 32, 57, 24, 1, 112, 2, 125, 96, 73, 95, 54, 58, 115, 33, 126, 50, 118, 3, 101, 113, 38, 116, 42, 105, 6, 111, 63, 45, 120, 62, 44, 123, 49, 106, 68, 66, 103, 48, 110, 109, 5, 47, 43, 121], [123, 41, 34, 120, 115, 107, 58, 60, 12, 48, 51, 90, 127, 125, 82, 70, 23, 109, 73, 116, 32, 47, 77, 121, 62, 29, 106, 84, 126, 14, 59, 55, 86, 24, 108, 80, 46, 26, 45, 57, 63, 52, 54, 122, 37, 110, 4, 36, 92, 38, 50, 61, 105, 56, 117, 79, 113, 111, 112, 119, 104, 103, 118, 42, 69, 124, 15, 43, 94, 49, 53, 67, 66, 101, 85, 44, 93, 99, 114, 7, 89, 33, 88, 39, 97, 1, 95, 31, 28, 30, 91, 100, 35, 40, 64, 102, 21, 22, 74, 10, 13, 19, 83, 78, 8, 98, 9, 20, 76, 6, 16, 18, 87, 17, 25, 96, 27, 2, 81, 0, 71, 65, 11, 75, 5, 3, 72, 68], [41, 123, 34, 107, 65, 66, 0, 29, 90, 121, 14, 37, 1, 74, 2, 11, 59, 93, 70, 80, 124, 72, 81, 69, 68, 105, 26, 47, 4, 10, 60, 75, 58, 12, 84, 3, 24, 86, 32, 109, 17, 82, 103, 7, 115, 77, 111, 127, 125, 35, 13, 21, 5, 15, 120, 73, 46, 110, 51, 71, 45, 8, 67, 94, 52, 97, 126, 118, 55, 87, 48, 56, 106, 116, 42, 62, 61, 57, 108, 101, 63, 112, 38, 33, 98, 28, 50, 122, 31, 44, 117, 113, 43, 53, 6, 18, 64, 79, 100, 119, 88, 92, 104, 54, 114, 49, 16, 99, 30, 27, 85, 20, 39, 83, 78, 91, 36, 76, 40, 95, 19, 25, 89, 9, 96, 23, 102, 22], [123, 41, 109, 34, 59, 29, 115, 90, 58, 23, 127, 61, 120, 51, 125, 43, 126, 111, 103, 108, 46, 54, 113, 122, 47, 121, 57, 110, 116, 56, 86, 107, 48, 50, 124, 52, 118, 63, 38, 117, 12, 55, 112, 80, 53, 119, 84, 42, 89, 93, 60, 21, 62, 49, 45, 114, 102, 106, 104, 99, 70, 44, 101, 26, 77, 6, 7, 4, 82, 25, 14, 36, 39, 32, 40, 37, 31, 94, 88, 35, 65, 33, 100, 97, 73, 92, 10, 105, 22, 74, 3, 27, 83, 96, 87, 0, 67, 20, 30, 19, 17, 91, 15, 64, 85, 98, 95, 28, 24, 72, 66, 79, 68, 81, 13, 18, 75, 11, 69, 9, 71, 78, 16, 76, 1, 8, 2, 5], [123, 41, 109, 34, 107, 59, 58, 121, 23, 47, 60, 127, 29, 115, 84, 55, 90, 120, 125, 106, 56, 104, 43, 112, 12, 80, 126, 88, 61, 46, 50, 62, 103, 53, 122, 48, 116, 124, 57, 51, 32, 54, 45, 52, 111, 110, 63, 93, 113, 26, 38, 73, 114, 117, 42, 118, 82, 99, 108, 36, 101, 119, 86, 14, 37, 35, 89, 49, 24, 102, 105, 70, 100, 74, 17, 77, 65, 25, 44, 39, 21, 85, 97, 83, 31, 4, 94, 40, 67, 95, 7, 3, 92, 30, 76, 91, 79, 33, 0, 16, 9, 19, 28, 98, 20, 27, 96, 18, 22, 6, 87, 11, 78, 72, 15, 69, 81, 64, 13, 75, 66, 1, 71, 10, 68, 8, 5, 2], [127, 126, 63, 38, 120, 82, 20, 97, 91, 80, 9, 11, 13, 21, 56, 25, 69, 14, 30, 15, 29, 24, 90, 26, 75, 8, 94, 86, 102, 99, 124, 2, 88, 37, 93, 39, 95, 76, 103, 84, 78, 3, 7, 74, 43, 98, 71, 46, 68, 114, 17, 54, 121, 18, 47, 67, 125, 108, 19, 105, 0, 101, 85, 119, 58, 77, 44, 5, 49, 10, 55, 111, 42, 34, 51, 89, 96, 83, 31, 27, 62, 92, 107, 123, 52, 57, 35, 53, 48, 28, 110, 61, 87, 6, 72, 12, 60, 116, 16, 33, 64, 32, 59, 22, 118, 73, 70, 23, 66, 115, 79, 122, 100, 117, 112, 50, 41, 109, 36, 113, 104, 65, 81, 4, 40, 45, 106, 1], [63, 38, 126, 123, 119, 54, 56, 120, 62, 110, 47, 124, 61, 59, 125, 118, 60, 122, 51, 121, 115, 117, 111, 58, 48, 53, 50, 46, 49, 55, 23, 127, 116, 102, 109, 112, 97, 113, 114, 57, 52, 44, 87, 29, 45, 81, 93, 43, 26, 105, 92, 86, 108, 94, 88, 107, 41, 106, 17, 42, 104, 99, 103, 22, 40, 33, 21, 90, 39, 84, 101, 37, 95, 15, 83, 18, 35, 98, 79, 30, 74, 36, 31, 96, 100, 89, 70, 34, 27, 25, 28, 76, 0, 32, 80, 20, 6, 85, 19, 91, 65, 10, 24, 82, 78, 16, 12, 13, 1, 75, 14, 8, 67, 66, 64, 4, 69, 73, 68, 3, 9, 11, 7, 71, 2, 5, 77, 72], [126, 127, 63, 38, 123, 54, 119, 56, 62, 110, 125, 120, 61, 47, 60, 124, 59, 118, 46, 122, 49, 121, 51, 111, 115, 53, 55, 48, 58, 50, 116, 117, 113, 112, 97, 52, 44, 23, 114, 102, 29, 57, 43, 109, 45, 105, 93, 81, 86, 94, 26, 108, 104, 92, 88, 87, 107, 41, 42, 40, 99, 90, 103, 17, 22, 106, 33, 39, 31, 37, 74, 98, 101, 30, 15, 21, 95, 83, 36, 100, 35, 70, 96, 6, 34, 84, 27, 32, 20, 28, 79, 18, 25, 89, 0, 65, 16, 76, 19, 85, 80, 14, 91, 12, 82, 10, 24, 67, 78, 1, 4, 13, 64, 8, 75, 66, 68, 9, 3, 7, 73, 77, 69, 72, 5, 71, 2, 11], [126, 63, 38, 127, 16, 120, 20, 88, 78, 97, 82, 91, 75, 56, 30, 47, 73, 99, 21, 29, 54, 32, 13, 125, 39, 7, 15, 86, 119, 95, 62, 5, 90, 124, 25, 10, 8, 66, 80, 51, 69, 76, 0, 123, 9, 67, 84, 102, 111, 24, 77, 49, 58, 98, 53, 103, 109, 43, 114, 28, 26, 61, 44, 27, 79, 60, 50, 34, 18, 104, 14, 117, 118, 71, 59, 107, 19, 115, 70, 122, 57, 65, 121, 48, 17, 12, 85, 87, 89, 31, 113, 55, 83, 101, 112, 92, 100, 72, 52, 116, 23, 4, 110, 94, 22, 74, 35, 11, 46, 6, 45, 96, 33, 37, 105, 41, 108, 93, 68, 106, 40, 36, 81, 3, 1, 42, 2, 64]], "model.layers.30.self_attn.k_proj": [[61, 102, 50, 33, 48, 89, 86, 112, 92, 85, 14, 83, 60, 0, 58, 52, 113, 2, 108, 94, 11, 17, 114, 40, 120, 121, 47, 62, 65, 59, 55, 45, 123, 126, 72, 57, 53, 43, 116, 127, 56, 119, 124, 125, 63, 118, 51, 69, 98, 42, 38, 122, 117, 67, 46, 115, 44, 77, 49, 109, 41, 106, 80, 110, 54, 68, 107, 103, 100, 104, 29, 37, 105, 101, 9, 39, 10, 87, 91, 7, 6, 35, 64, 15, 31, 99, 111, 12, 36, 71, 4, 5, 84, 93, 70, 30, 90, 96, 32, 88, 27, 95, 23, 26, 34, 73, 82, 18, 28, 13, 66, 22, 76, 24, 16, 20, 21, 3, 74, 75, 1, 78, 25, 8, 19, 79, 97, 81], [110, 46, 98, 90, 0, 92, 24, 13, 31, 67, 18, 86, 2, 70, 113, 74, 1, 15, 83, 52, 17, 8, 4, 12, 16, 80, 51, 105, 9, 49, 84, 59, 21, 79, 41, 123, 116, 11, 64, 68, 94, 85, 7, 23, 54, 127, 119, 109, 29, 96, 124, 58, 115, 22, 122, 34, 100, 97, 101, 30, 38, 48, 14, 53, 45, 60, 126, 106, 118, 104, 39, 40, 35, 117, 56, 108, 120, 62, 63, 107, 47, 93, 37, 114, 5, 44, 87, 50, 102, 33, 43, 78, 111, 91, 27, 125, 42, 10, 89, 103, 112, 55, 36, 99, 32, 73, 69, 121, 61, 76, 26, 57, 95, 66, 71, 25, 28, 65, 75, 19, 20, 77, 81, 72, 6, 3, 88, 82], [106, 63, 86, 33, 117, 27, 11, 77, 78, 9, 4, 81, 7, 0, 84, 1, 55, 18, 49, 57, 31, 122, 16, 121, 95, 30, 72, 42, 116, 85, 56, 50, 59, 114, 119, 52, 51, 87, 58, 110, 54, 53, 43, 120, 34, 111, 44, 48, 118, 61, 124, 112, 115, 90, 113, 125, 103, 46, 36, 47, 62, 24, 29, 66, 41, 107, 105, 127, 37, 64, 45, 65, 109, 123, 3, 28, 126, 60, 104, 6, 67, 102, 10, 100, 108, 68, 32, 38, 99, 39, 35, 92, 40, 80, 94, 12, 5, 101, 73, 26, 23, 69, 93, 25, 76, 15, 8, 88, 20, 89, 70, 98, 91, 96, 83, 82, 14, 19, 74, 79, 13, 71, 2, 21, 17, 22, 75, 97], [108, 36, 44, 33, 86, 115, 47, 94, 111, 127, 17, 61, 52, 59, 26, 28, 41, 57, 96, 114, 126, 119, 49, 112, 104, 125, 117, 122, 118, 37, 48, 56, 120, 42, 62, 107, 60, 13, 55, 63, 50, 21, 123, 58, 124, 121, 88, 46, 43, 113, 39, 51, 106, 53, 102, 45, 40, 110, 109, 116, 54, 79, 101, 100, 105, 90, 31, 103, 72, 38, 5, 95, 98, 85, 93, 99, 76, 19, 14, 34, 24, 35, 27, 84, 87, 30, 91, 16, 12, 92, 89, 80, 32, 67, 97, 11, 3, 78, 74, 25, 65, 29, 23, 9, 75, 22, 81, 82, 18, 0, 20, 77, 2, 70, 83, 8, 73, 71, 6, 15, 7, 66, 1, 10, 68, 64, 4, 69], [39, 122, 113, 97, 49, 29, 57, 112, 26, 22, 88, 27, 73, 124, 24, 19, 93, 78, 48, 102, 59, 30, 127, 81, 123, 21, 10, 17, 47, 105, 118, 117, 85, 1, 83, 40, 54, 15, 111, 61, 107, 119, 63, 76, 43, 115, 95, 100, 60, 58, 109, 114, 52, 106, 126, 31, 12, 62, 55, 5, 53, 87, 41, 110, 37, 0, 121, 66, 120, 46, 7, 108, 4, 92, 104, 116, 51, 44, 103, 36, 125, 89, 79, 84, 101, 99, 96, 50, 42, 34, 25, 3, 32, 45, 13, 94, 56, 98, 14, 35, 38, 82, 77, 90, 75, 91, 80, 8, 69, 28, 20, 9, 18, 23, 16, 11, 33, 72, 71, 68, 86, 67, 6, 64, 70, 65, 2, 74], [103, 56, 53, 25, 33, 18, 16, 94, 77, 74, 60, 90, 23, 4, 70, 71, 87, 76, 22, 119, 0, 65, 109, 124, 55, 89, 122, 50, 120, 63, 52, 75, 123, 51, 86, 85, 67, 91, 93, 113, 44, 54, 35, 125, 127, 117, 114, 100, 47, 61, 116, 29, 9, 118, 36, 28, 19, 101, 59, 107, 106, 57, 98, 43, 62, 32, 72, 111, 49, 121, 42, 48, 58, 110, 8, 115, 45, 78, 108, 126, 11, 81, 41, 104, 46, 96, 105, 112, 79, 102, 2, 34, 5, 95, 92, 17, 31, 7, 37, 40, 38, 84, 66, 99, 30, 24, 12, 20, 6, 73, 27, 68, 88, 13, 14, 15, 39, 3, 80, 83, 1, 82, 64, 21, 97, 69, 26, 10], [105, 123, 98, 86, 59, 93, 64, 127, 50, 125, 60, 90, 67, 116, 121, 46, 80, 117, 124, 126, 51, 119, 73, 96, 56, 122, 101, 58, 57, 62, 48, 63, 14, 115, 45, 61, 11, 53, 112, 118, 52, 120, 82, 110, 55, 42, 43, 111, 113, 47, 106, 12, 114, 54, 44, 69, 1, 40, 109, 84, 49, 70, 108, 66, 39, 102, 24, 81, 37, 68, 3, 23, 100, 95, 19, 104, 36, 91, 71, 28, 8, 15, 77, 32, 21, 94, 35, 74, 17, 72, 103, 97, 85, 10, 5, 89, 99, 30, 38, 29, 25, 107, 33, 20, 87, 27, 83, 31, 92, 79, 41, 34, 6, 7, 88, 75, 65, 22, 26, 2, 13, 76, 18, 78, 4, 0, 9, 16], [126, 102, 63, 33, 86, 93, 127, 56, 95, 26, 120, 47, 124, 111, 112, 121, 54, 114, 46, 119, 53, 23, 61, 117, 51, 118, 57, 107, 48, 15, 122, 60, 58, 45, 55, 44, 59, 105, 52, 17, 104, 62, 20, 115, 49, 82, 94, 103, 13, 123, 50, 106, 108, 40, 125, 113, 116, 30, 41, 43, 42, 109, 110, 88, 39, 91, 79, 31, 28, 37, 38, 27, 87, 101, 18, 77, 92, 81, 89, 99, 29, 36, 96, 100, 34, 35, 83, 21, 11, 74, 32, 7, 10, 98, 75, 19, 25, 90, 84, 71, 65, 9, 6, 85, 16, 8, 14, 12, 76, 5, 97, 22, 78, 80, 24, 4, 73, 67, 1, 72, 66, 0, 69, 68, 70, 3, 2, 64]], "model.layers.30.self_attn.qk_proj": [[63, 61, 123, 56, 110, 126, 53, 46, 122, 113, 44, 50, 106, 127, 111, 117, 108, 60, 48, 22, 42, 57, 102, 59, 103, 52, 124, 125, 39, 105, 33, 49, 120, 119, 116, 38, 51, 90, 112, 86, 115, 55, 54, 26, 93, 29, 58, 97, 118, 114, 41, 109, 121, 47, 34, 24, 88, 30, 94, 92, 43, 62, 82, 25, 31, 45, 98, 28, 81, 89, 36, 23, 18, 107, 77, 83, 91, 101, 87, 13, 40, 80, 14, 17, 21, 85, 100, 78, 11, 16, 19, 27, 37, 79, 9, 84, 104, 75, 95, 15, 73, 76, 64, 0, 20, 96, 12, 71, 10, 7, 8, 35, 74, 67, 6, 4, 70, 68, 5, 99, 32, 65, 1, 72, 3, 69, 2, 66], [63, 61, 123, 126, 56, 110, 46, 53, 122, 113, 50, 44, 111, 106, 42, 117, 127, 108, 60, 48, 22, 57, 102, 120, 125, 103, 105, 124, 59, 52, 38, 112, 39, 86, 51, 33, 29, 115, 58, 54, 49, 26, 47, 55, 118, 97, 41, 109, 93, 116, 90, 119, 114, 30, 121, 45, 24, 94, 62, 88, 28, 25, 107, 92, 43, 31, 34, 98, 82, 91, 23, 89, 13, 17, 100, 18, 40, 104, 36, 81, 77, 14, 11, 19, 16, 78, 83, 80, 85, 87, 21, 37, 95, 9, 79, 99, 101, 76, 27, 0, 12, 75, 15, 70, 73, 68, 74, 64, 10, 20, 71, 35, 32, 84, 8, 7, 4, 3, 96, 1, 65, 67, 69, 6, 5, 72, 66, 2], [63, 61, 123, 126, 56, 110, 53, 46, 122, 113, 50, 44, 111, 106, 127, 117, 42, 48, 108, 22, 57, 102, 60, 103, 124, 105, 120, 59, 52, 51, 86, 33, 38, 39, 115, 54, 116, 49, 125, 55, 112, 58, 93, 119, 26, 47, 118, 121, 114, 29, 90, 97, 107, 24, 94, 109, 30, 62, 28, 88, 41, 34, 92, 25, 43, 40, 89, 45, 98, 18, 83, 31, 36, 81, 23, 101, 82, 100, 13, 16, 77, 80, 14, 91, 21, 19, 0, 11, 70, 17, 87, 85, 104, 95, 78, 20, 9, 37, 99, 75, 79, 15, 27, 74, 76, 71, 10, 84, 12, 32, 73, 64, 7, 4, 35, 67, 1, 96, 72, 68, 65, 3, 8, 5, 69, 2, 66, 6], [63, 61, 123, 126, 56, 110, 53, 46, 122, 113, 50, 44, 111, 106, 117, 108, 42, 127, 48, 52, 57, 22, 120, 124, 103, 102, 60, 58, 125, 39, 105, 38, 116, 121, 47, 49, 112, 51, 54, 86, 26, 33, 59, 93, 118, 114, 97, 119, 55, 43, 29, 109, 90, 115, 30, 24, 62, 41, 94, 45, 107, 34, 28, 92, 40, 25, 31, 88, 82, 104, 13, 98, 81, 89, 14, 80, 23, 36, 83, 78, 19, 18, 101, 100, 21, 91, 17, 77, 64, 11, 37, 87, 16, 76, 85, 95, 70, 99, 75, 79, 84, 15, 9, 73, 0, 27, 10, 71, 12, 20, 68, 32, 7, 3, 35, 1, 74, 96, 72, 65, 4, 8, 67, 69, 5, 66, 2, 6], [63, 61, 123, 56, 126, 53, 110, 46, 122, 113, 44, 50, 111, 127, 106, 108, 117, 42, 22, 48, 102, 52, 60, 57, 124, 120, 103, 125, 59, 51, 86, 116, 105, 39, 119, 58, 54, 47, 49, 55, 33, 112, 90, 26, 115, 118, 38, 93, 114, 29, 97, 41, 121, 109, 24, 62, 43, 107, 34, 31, 83, 92, 25, 89, 88, 94, 36, 30, 82, 81, 23, 45, 28, 18, 80, 21, 13, 98, 77, 14, 11, 17, 78, 40, 87, 101, 85, 27, 16, 100, 0, 19, 95, 79, 91, 104, 76, 15, 73, 12, 9, 75, 70, 64, 20, 84, 74, 37, 10, 1, 7, 72, 3, 71, 4, 65, 68, 35, 67, 96, 32, 8, 6, 5, 99, 69, 66, 2], [63, 61, 126, 56, 123, 110, 53, 46, 122, 113, 44, 50, 108, 111, 106, 42, 127, 22, 57, 48, 117, 105, 103, 102, 120, 86, 60, 52, 59, 38, 47, 112, 125, 124, 39, 54, 58, 116, 118, 51, 93, 33, 114, 29, 26, 45, 49, 90, 97, 55, 119, 41, 24, 109, 121, 115, 92, 25, 28, 43, 62, 88, 94, 81, 82, 23, 30, 21, 98, 36, 34, 14, 95, 31, 18, 13, 83, 107, 87, 89, 91, 80, 78, 17, 77, 27, 85, 40, 11, 19, 15, 104, 16, 101, 100, 12, 64, 9, 76, 79, 75, 37, 73, 70, 0, 4, 72, 74, 84, 20, 10, 7, 35, 6, 68, 65, 71, 32, 96, 3, 99, 67, 66, 69, 5, 8, 1, 2], [63, 61, 110, 123, 126, 56, 53, 122, 46, 44, 113, 50, 42, 111, 106, 108, 48, 22, 127, 102, 117, 59, 105, 125, 103, 124, 60, 57, 52, 120, 86, 112, 39, 47, 90, 116, 26, 38, 29, 93, 119, 115, 41, 109, 45, 55, 33, 121, 97, 114, 58, 51, 118, 43, 92, 54, 88, 62, 25, 24, 28, 49, 94, 23, 31, 81, 18, 107, 21, 82, 87, 85, 30, 80, 36, 89, 83, 98, 91, 17, 27, 77, 40, 95, 100, 11, 34, 14, 13, 78, 19, 16, 20, 15, 101, 12, 76, 6, 79, 73, 9, 75, 74, 104, 84, 10, 35, 37, 64, 0, 32, 7, 67, 71, 4, 72, 99, 65, 68, 70, 8, 1, 96, 3, 2, 5, 69, 66], [63, 61, 123, 56, 110, 126, 53, 122, 46, 113, 44, 50, 111, 106, 108, 42, 127, 48, 22, 117, 57, 102, 59, 60, 103, 124, 120, 52, 125, 115, 55, 105, 119, 112, 33, 90, 93, 38, 58, 116, 39, 49, 86, 51, 26, 47, 29, 97, 41, 121, 109, 114, 118, 92, 45, 54, 88, 25, 62, 24, 43, 30, 31, 89, 28, 107, 91, 94, 34, 18, 21, 36, 81, 23, 82, 17, 98, 40, 83, 100, 87, 13, 27, 14, 101, 16, 80, 78, 11, 85, 77, 20, 37, 95, 64, 75, 79, 0, 76, 19, 104, 15, 6, 9, 73, 84, 35, 10, 12, 96, 67, 7, 1, 99, 74, 72, 32, 65, 4, 71, 68, 3, 8, 69, 2, 66, 5, 70], [63, 61, 56, 110, 123, 53, 126, 46, 122, 113, 44, 50, 106, 111, 127, 42, 48, 108, 52, 60, 102, 57, 124, 117, 103, 22, 120, 58, 59, 116, 86, 125, 105, 39, 38, 51, 93, 119, 55, 112, 109, 97, 33, 29, 90, 26, 115, 49, 45, 47, 118, 43, 25, 41, 94, 92, 114, 88, 24, 54, 31, 107, 121, 30, 62, 23, 28, 18, 89, 34, 98, 91, 13, 40, 82, 27, 87, 83, 80, 21, 78, 100, 17, 36, 19, 77, 81, 95, 85, 11, 104, 20, 16, 14, 76, 84, 101, 79, 75, 37, 73, 6, 15, 12, 68, 99, 0, 32, 7, 4, 72, 9, 10, 74, 64, 71, 35, 1, 3, 96, 65, 5, 70, 8, 67, 2, 69, 66], [63, 61, 123, 126, 56, 110, 53, 122, 46, 113, 44, 50, 111, 106, 108, 127, 48, 60, 117, 124, 125, 120, 52, 102, 57, 22, 42, 103, 55, 38, 59, 49, 51, 58, 105, 33, 39, 47, 119, 121, 97, 86, 116, 115, 90, 93, 26, 43, 109, 54, 112, 29, 114, 34, 62, 118, 92, 24, 94, 88, 45, 30, 98, 107, 31, 41, 13, 40, 23, 28, 25, 83, 100, 82, 18, 91, 81, 37, 89, 104, 16, 77, 17, 85, 87, 80, 11, 21, 19, 64, 78, 6, 14, 36, 101, 75, 95, 27, 99, 74, 84, 73, 7, 68, 10, 4, 0, 20, 9, 71, 79, 12, 15, 76, 35, 72, 3, 65, 67, 96, 32, 1, 8, 70, 5, 66, 69, 2], [63, 61, 123, 126, 56, 110, 46, 122, 53, 113, 44, 50, 111, 106, 127, 108, 117, 48, 42, 22, 60, 52, 125, 57, 102, 103, 59, 124, 105, 112, 33, 39, 93, 49, 55, 120, 119, 86, 114, 121, 51, 116, 47, 38, 26, 118, 90, 54, 97, 29, 62, 115, 41, 109, 58, 24, 88, 34, 43, 92, 94, 30, 25, 31, 13, 89, 82, 14, 107, 17, 45, 28, 40, 98, 85, 36, 18, 77, 78, 19, 23, 91, 101, 81, 80, 83, 21, 104, 16, 11, 95, 73, 74, 9, 79, 87, 76, 12, 15, 0, 75, 64, 100, 27, 20, 6, 84, 7, 71, 4, 10, 72, 35, 8, 65, 67, 70, 3, 37, 68, 99, 32, 69, 96, 1, 66, 5, 2], [63, 61, 110, 123, 126, 56, 53, 46, 122, 113, 44, 50, 106, 108, 127, 111, 42, 22, 102, 48, 60, 124, 125, 52, 105, 103, 86, 117, 57, 112, 120, 59, 116, 33, 39, 29, 58, 93, 55, 119, 26, 41, 38, 115, 97, 47, 90, 114, 49, 51, 109, 118, 25, 121, 92, 24, 43, 54, 31, 62, 88, 30, 82, 23, 17, 94, 13, 81, 18, 98, 40, 28, 45, 14, 91, 77, 78, 101, 85, 16, 107, 34, 83, 89, 19, 21, 87, 80, 36, 27, 11, 95, 73, 100, 79, 75, 76, 9, 12, 84, 104, 37, 20, 71, 64, 35, 15, 4, 74, 0, 10, 68, 70, 69, 32, 72, 7, 8, 3, 96, 67, 99, 1, 6, 65, 5, 66, 2], [63, 61, 56, 110, 123, 126, 53, 122, 46, 113, 44, 50, 108, 127, 111, 106, 22, 42, 48, 124, 60, 102, 103, 38, 57, 120, 125, 55, 52, 58, 86, 119, 59, 33, 49, 105, 117, 47, 90, 29, 26, 97, 112, 39, 115, 116, 93, 109, 92, 118, 51, 54, 45, 114, 41, 24, 25, 43, 94, 88, 23, 121, 28, 91, 31, 62, 30, 18, 34, 89, 107, 17, 100, 82, 85, 83, 27, 87, 98, 95, 81, 19, 40, 13, 80, 78, 36, 21, 77, 37, 75, 14, 15, 16, 101, 104, 11, 0, 79, 70, 35, 76, 84, 73, 20, 99, 10, 9, 64, 12, 4, 96, 32, 68, 7, 74, 71, 72, 3, 1, 67, 8, 65, 66, 2, 69, 6, 5], [63, 61, 123, 56, 110, 126, 46, 53, 122, 113, 44, 50, 108, 111, 42, 106, 48, 127, 22, 124, 120, 117, 60, 57, 55, 102, 105, 103, 59, 39, 52, 58, 125, 38, 119, 33, 49, 51, 121, 47, 86, 116, 93, 112, 90, 26, 115, 41, 118, 109, 114, 29, 97, 24, 54, 62, 94, 43, 45, 92, 31, 25, 28, 88, 98, 107, 23, 82, 30, 21, 91, 89, 18, 34, 40, 83, 87, 17, 13, 78, 80, 36, 14, 81, 85, 75, 77, 104, 101, 16, 100, 64, 0, 15, 27, 70, 11, 95, 19, 37, 76, 12, 73, 79, 84, 7, 68, 74, 35, 8, 9, 71, 32, 67, 20, 99, 10, 4, 65, 69, 1, 3, 5, 72, 96, 66, 6, 2], [63, 61, 126, 123, 56, 110, 46, 53, 122, 44, 113, 50, 111, 106, 108, 42, 22, 127, 48, 117, 105, 57, 124, 102, 52, 103, 125, 55, 60, 120, 38, 115, 49, 39, 33, 121, 26, 97, 59, 116, 119, 86, 93, 54, 41, 47, 109, 90, 43, 51, 29, 114, 112, 34, 118, 58, 24, 45, 98, 62, 88, 92, 94, 25, 30, 31, 82, 28, 13, 23, 81, 89, 107, 40, 14, 78, 0, 80, 21, 87, 77, 18, 91, 83, 16, 17, 19, 100, 101, 36, 85, 76, 95, 75, 11, 70, 64, 104, 84, 9, 74, 73, 79, 15, 20, 12, 8, 37, 27, 35, 71, 10, 99, 7, 1, 68, 67, 4, 69, 65, 32, 3, 72, 96, 66, 2, 5, 6], [63, 61, 126, 123, 56, 110, 53, 46, 122, 113, 44, 50, 111, 106, 127, 108, 42, 22, 48, 117, 102, 52, 124, 57, 103, 105, 38, 60, 33, 86, 125, 116, 58, 59, 120, 118, 115, 114, 49, 39, 55, 112, 119, 93, 29, 97, 51, 26, 47, 41, 30, 94, 90, 54, 43, 121, 24, 109, 92, 25, 31, 89, 45, 34, 88, 40, 28, 107, 82, 98, 14, 91, 23, 81, 77, 62, 18, 80, 78, 100, 16, 85, 83, 87, 95, 13, 17, 21, 19, 101, 11, 64, 36, 76, 27, 75, 104, 79, 9, 73, 20, 12, 37, 84, 8, 3, 10, 70, 74, 0, 68, 4, 15, 7, 71, 1, 35, 96, 67, 99, 65, 5, 32, 69, 72, 2, 6, 66], [63, 61, 110, 126, 53, 56, 123, 46, 122, 44, 113, 50, 111, 106, 42, 108, 127, 22, 124, 57, 102, 48, 117, 60, 52, 105, 86, 103, 38, 116, 59, 125, 58, 33, 55, 47, 39, 112, 26, 29, 119, 120, 118, 115, 97, 90, 93, 49, 51, 121, 94, 109, 114, 43, 41, 62, 92, 31, 88, 45, 24, 25, 30, 34, 54, 23, 28, 40, 81, 83, 18, 89, 82, 17, 91, 98, 13, 100, 77, 16, 107, 87, 36, 21, 101, 95, 78, 14, 80, 27, 19, 84, 11, 85, 37, 20, 75, 104, 79, 15, 10, 12, 99, 9, 64, 76, 0, 4, 73, 96, 35, 7, 74, 8, 67, 68, 6, 3, 65, 71, 70, 32, 1, 5, 72, 66, 69, 2], [63, 61, 110, 56, 126, 123, 53, 46, 122, 113, 44, 50, 111, 106, 127, 42, 108, 48, 117, 57, 22, 102, 52, 105, 103, 60, 124, 59, 86, 116, 115, 33, 55, 38, 120, 39, 47, 125, 109, 112, 49, 97, 26, 51, 93, 62, 41, 29, 90, 119, 114, 118, 121, 58, 54, 94, 24, 92, 30, 107, 34, 88, 28, 45, 89, 43, 25, 18, 98, 31, 91, 23, 81, 83, 40, 13, 36, 16, 82, 19, 17, 78, 75, 77, 37, 14, 87, 100, 21, 85, 27, 101, 80, 104, 95, 9, 0, 64, 79, 20, 6, 15, 76, 73, 84, 35, 74, 11, 99, 12, 7, 96, 8, 32, 10, 1, 67, 71, 3, 4, 69, 72, 70, 66, 68, 65, 5, 2], [63, 61, 123, 126, 110, 56, 46, 53, 122, 113, 50, 44, 111, 106, 42, 127, 52, 108, 48, 57, 102, 117, 124, 120, 22, 103, 60, 51, 105, 39, 125, 33, 38, 47, 116, 59, 55, 86, 115, 49, 121, 118, 26, 112, 29, 58, 119, 97, 93, 109, 45, 43, 90, 54, 62, 41, 114, 30, 31, 107, 94, 24, 34, 40, 92, 98, 88, 23, 28, 36, 25, 77, 100, 81, 101, 89, 82, 17, 18, 91, 13, 21, 75, 78, 83, 95, 19, 14, 80, 16, 87, 37, 104, 9, 85, 11, 99, 6, 27, 12, 35, 76, 7, 84, 73, 71, 20, 64, 15, 79, 74, 10, 0, 68, 1, 32, 96, 8, 69, 3, 4, 65, 2, 72, 67, 5, 70, 66], [63, 61, 126, 123, 110, 56, 53, 46, 122, 113, 44, 50, 111, 108, 106, 42, 48, 127, 117, 22, 38, 124, 57, 52, 102, 105, 103, 120, 60, 33, 125, 59, 47, 39, 86, 26, 112, 97, 119, 51, 49, 58, 114, 116, 118, 29, 55, 115, 93, 90, 109, 121, 41, 54, 92, 30, 24, 45, 94, 34, 88, 31, 28, 62, 43, 107, 25, 23, 91, 98, 77, 36, 18, 81, 104, 40, 89, 87, 82, 17, 13, 19, 78, 101, 21, 85, 80, 83, 14, 79, 95, 16, 9, 10, 75, 6, 100, 76, 20, 11, 0, 15, 74, 37, 12, 71, 73, 68, 84, 64, 8, 27, 7, 32, 99, 4, 3, 72, 65, 35, 1, 96, 67, 66, 5, 69, 2, 70], [63, 61, 126, 123, 110, 56, 53, 46, 122, 113, 50, 44, 111, 106, 42, 108, 117, 57, 48, 127, 22, 60, 124, 52, 105, 102, 103, 59, 109, 33, 120, 125, 115, 114, 112, 116, 55, 86, 49, 38, 51, 118, 47, 39, 62, 90, 58, 93, 119, 26, 29, 97, 41, 121, 54, 107, 94, 45, 24, 28, 88, 92, 43, 98, 34, 30, 89, 36, 25, 31, 82, 95, 91, 83, 13, 80, 40, 77, 18, 104, 21, 87, 17, 81, 101, 23, 14, 78, 16, 75, 19, 11, 85, 64, 27, 37, 100, 15, 9, 76, 79, 6, 73, 0, 10, 35, 20, 99, 84, 4, 74, 65, 67, 32, 7, 71, 68, 12, 8, 1, 72, 69, 3, 96, 70, 2, 5, 66], [63, 61, 123, 126, 56, 110, 53, 46, 122, 113, 50, 44, 111, 117, 42, 106, 108, 127, 124, 52, 48, 102, 57, 60, 120, 103, 125, 22, 105, 112, 55, 58, 39, 59, 114, 38, 33, 115, 121, 119, 51, 118, 47, 116, 86, 29, 49, 26, 90, 109, 93, 54, 41, 107, 97, 94, 88, 62, 45, 30, 43, 31, 28, 23, 34, 92, 24, 91, 25, 83, 87, 36, 98, 82, 100, 17, 81, 89, 18, 77, 104, 64, 40, 37, 14, 19, 13, 85, 21, 95, 27, 101, 75, 80, 16, 78, 0, 76, 15, 20, 79, 11, 4, 9, 3, 73, 74, 99, 35, 10, 12, 96, 84, 6, 1, 70, 71, 7, 67, 8, 32, 65, 72, 69, 5, 68, 2, 66], [63, 61, 123, 110, 126, 56, 53, 46, 122, 113, 44, 50, 111, 127, 106, 108, 42, 124, 117, 48, 102, 22, 120, 60, 52, 125, 58, 38, 116, 33, 57, 103, 59, 119, 105, 115, 86, 118, 112, 49, 39, 55, 26, 47, 29, 93, 90, 121, 51, 97, 114, 41, 54, 94, 109, 88, 43, 28, 34, 45, 24, 62, 25, 30, 31, 92, 107, 89, 36, 98, 91, 82, 18, 23, 83, 87, 81, 100, 77, 17, 13, 40, 21, 101, 16, 14, 95, 85, 80, 78, 19, 0, 11, 27, 75, 104, 64, 9, 15, 84, 76, 71, 20, 73, 4, 74, 37, 79, 72, 65, 70, 12, 35, 99, 3, 1, 96, 7, 10, 68, 67, 2, 32, 8, 5, 66, 6, 69], [63, 61, 126, 123, 110, 56, 53, 46, 122, 113, 50, 44, 106, 111, 108, 42, 48, 22, 127, 120, 117, 124, 105, 52, 102, 57, 103, 38, 125, 33, 60, 59, 116, 118, 55, 112, 86, 97, 26, 93, 49, 90, 39, 121, 47, 119, 29, 51, 58, 114, 41, 115, 54, 109, 24, 92, 43, 62, 94, 28, 30, 88, 34, 25, 45, 36, 89, 107, 40, 82, 17, 98, 31, 23, 78, 91, 18, 81, 13, 104, 77, 83, 80, 19, 14, 95, 85, 87, 21, 16, 11, 75, 15, 79, 27, 76, 100, 70, 10, 9, 101, 37, 12, 73, 35, 4, 99, 20, 64, 72, 84, 32, 74, 71, 68, 7, 0, 67, 5, 1, 65, 3, 8, 96, 69, 6, 66, 2], [63, 61, 123, 126, 56, 110, 53, 46, 122, 113, 44, 50, 111, 108, 106, 42, 22, 48, 117, 57, 124, 52, 127, 60, 105, 120, 102, 59, 103, 116, 112, 55, 114, 115, 38, 125, 119, 86, 33, 121, 49, 39, 26, 109, 90, 47, 51, 118, 93, 29, 54, 43, 97, 41, 107, 58, 94, 28, 92, 25, 24, 30, 62, 23, 45, 88, 31, 36, 17, 34, 77, 85, 83, 98, 89, 40, 81, 87, 82, 101, 21, 18, 78, 16, 19, 91, 100, 27, 95, 13, 11, 104, 80, 75, 15, 70, 79, 9, 14, 73, 64, 84, 76, 12, 37, 7, 35, 74, 20, 0, 10, 68, 71, 32, 72, 67, 3, 99, 65, 8, 1, 96, 4, 5, 66, 69, 6, 2], [63, 61, 110, 126, 123, 56, 53, 46, 122, 113, 44, 50, 111, 108, 106, 42, 22, 48, 127, 124, 120, 60, 52, 102, 117, 57, 103, 105, 55, 125, 116, 33, 59, 115, 38, 119, 86, 49, 29, 39, 112, 118, 47, 90, 26, 58, 93, 121, 109, 97, 114, 94, 54, 51, 41, 24, 28, 62, 30, 92, 34, 25, 88, 43, 107, 31, 89, 98, 23, 91, 45, 82, 18, 100, 40, 36, 37, 83, 77, 19, 87, 17, 81, 85, 101, 27, 80, 95, 16, 13, 104, 21, 11, 75, 14, 64, 78, 35, 12, 79, 84, 9, 20, 15, 73, 99, 76, 10, 70, 0, 71, 74, 96, 68, 65, 32, 72, 3, 1, 7, 67, 5, 4, 69, 8, 2, 66, 6], [63, 61, 110, 123, 126, 56, 53, 46, 122, 113, 50, 44, 106, 111, 42, 108, 48, 117, 127, 22, 57, 52, 60, 102, 103, 105, 124, 59, 116, 38, 112, 125, 120, 109, 115, 121, 39, 55, 33, 119, 49, 86, 51, 93, 97, 118, 114, 54, 58, 26, 90, 41, 29, 62, 47, 88, 24, 94, 30, 91, 89, 34, 43, 31, 92, 25, 107, 28, 45, 101, 36, 98, 40, 18, 81, 82, 87, 83, 17, 85, 23, 19, 80, 11, 21, 13, 78, 100, 16, 27, 14, 77, 95, 104, 84, 37, 79, 35, 75, 12, 73, 0, 99, 76, 15, 9, 8, 64, 20, 74, 70, 32, 68, 10, 71, 7, 65, 67, 96, 4, 72, 3, 69, 1, 5, 6, 2, 66], [63, 61, 123, 126, 56, 110, 53, 46, 122, 113, 50, 44, 111, 106, 108, 117, 48, 127, 42, 22, 124, 120, 57, 52, 102, 103, 60, 125, 38, 105, 116, 33, 59, 39, 121, 49, 55, 112, 51, 47, 119, 26, 114, 115, 86, 93, 41, 43, 54, 90, 97, 58, 29, 109, 118, 62, 45, 34, 88, 92, 24, 94, 31, 107, 25, 89, 28, 30, 36, 91, 82, 81, 23, 98, 18, 77, 21, 11, 83, 14, 17, 40, 80, 0, 87, 78, 37, 16, 13, 19, 85, 100, 79, 9, 104, 84, 73, 95, 75, 12, 10, 64, 27, 15, 101, 76, 35, 71, 74, 99, 20, 6, 67, 68, 7, 1, 4, 65, 96, 70, 3, 32, 2, 72, 5, 8, 69, 66], [63, 61, 123, 126, 110, 53, 56, 46, 122, 113, 50, 44, 111, 106, 127, 117, 108, 42, 48, 125, 52, 124, 59, 22, 102, 57, 103, 38, 112, 60, 105, 33, 120, 116, 51, 119, 39, 115, 55, 97, 54, 43, 86, 62, 93, 49, 29, 26, 114, 41, 47, 58, 90, 118, 121, 34, 94, 24, 92, 109, 45, 89, 30, 88, 28, 31, 40, 95, 18, 101, 98, 25, 23, 14, 91, 107, 80, 82, 17, 16, 11, 85, 36, 77, 13, 78, 19, 100, 81, 104, 0, 87, 15, 83, 64, 73, 71, 6, 21, 79, 9, 75, 37, 76, 12, 10, 74, 27, 35, 20, 84, 4, 65, 68, 7, 8, 3, 67, 1, 96, 99, 70, 32, 72, 69, 66, 5, 2], [63, 61, 126, 123, 56, 110, 46, 53, 122, 113, 50, 44, 111, 42, 106, 57, 108, 117, 124, 22, 127, 48, 59, 52, 102, 125, 60, 120, 103, 105, 116, 38, 86, 39, 55, 112, 119, 58, 33, 26, 49, 114, 97, 29, 93, 51, 47, 41, 118, 90, 43, 115, 54, 45, 24, 109, 62, 88, 28, 92, 25, 94, 121, 18, 30, 34, 91, 31, 23, 100, 82, 81, 98, 89, 77, 21, 36, 85, 78, 87, 17, 19, 40, 14, 107, 80, 101, 13, 83, 11, 16, 104, 95, 73, 27, 15, 6, 9, 76, 79, 84, 75, 12, 37, 0, 4, 35, 74, 20, 68, 10, 71, 99, 64, 8, 7, 3, 67, 32, 1, 72, 69, 65, 5, 70, 96, 66, 2], [63, 61, 126, 123, 110, 56, 53, 46, 122, 113, 50, 44, 111, 124, 108, 117, 106, 42, 57, 60, 127, 22, 48, 103, 52, 102, 38, 116, 120, 59, 105, 49, 86, 55, 125, 39, 33, 47, 121, 114, 112, 51, 90, 115, 97, 29, 119, 26, 54, 118, 41, 93, 109, 58, 43, 25, 28, 62, 94, 30, 34, 24, 92, 31, 23, 88, 45, 36, 95, 78, 81, 82, 98, 19, 18, 83, 107, 91, 89, 13, 77, 21, 87, 6, 100, 14, 80, 85, 11, 40, 64, 73, 17, 104, 16, 15, 101, 9, 0, 71, 37, 4, 79, 7, 27, 75, 10, 74, 12, 84, 68, 67, 76, 8, 35, 3, 20, 1, 99, 65, 96, 66, 32, 72, 2, 69, 5, 70], [63, 61, 123, 110, 56, 126, 53, 122, 46, 113, 44, 50, 111, 106, 42, 108, 127, 22, 48, 60, 117, 102, 103, 124, 57, 125, 120, 52, 59, 38, 33, 105, 86, 55, 39, 29, 97, 49, 51, 121, 119, 26, 47, 90, 112, 115, 116, 93, 54, 114, 30, 109, 118, 41, 62, 43, 94, 25, 92, 31, 24, 88, 58, 34, 28, 91, 18, 81, 45, 100, 98, 82, 23, 107, 85, 11, 14, 13, 89, 37, 17, 36, 21, 80, 19, 83, 16, 40, 87, 77, 95, 27, 79, 78, 104, 73, 101, 6, 15, 9, 75, 35, 12, 10, 76, 99, 74, 8, 20, 71, 0, 4, 84, 96, 7, 32, 68, 72, 67, 69, 1, 3, 70, 65, 5, 64, 2, 66]], "model.layers.31.self_attn.q_proj": [[40, 121, 98, 46, 18, 25, 111, 9, 30, 21, 13, 51, 15, 11, 48, 105, 49, 67, 23, 60, 55, 53, 4, 6, 58, 116, 114, 124, 72, 107, 31, 88, 115, 95, 41, 109, 22, 108, 61, 17, 37, 59, 92, 50, 45, 19, 87, 43, 127, 20, 5, 97, 1, 14, 123, 27, 126, 69, 75, 78, 81, 90, 16, 120, 102, 101, 8, 24, 112, 93, 85, 52, 26, 63, 118, 0, 7, 33, 117, 62, 122, 96, 76, 94, 74, 113, 35, 36, 54, 28, 12, 77, 79, 57, 39, 56, 82, 70, 73, 42, 89, 84, 3, 80, 32, 106, 110, 103, 83, 99, 100, 2, 125, 65, 119, 68, 38, 29, 86, 71, 44, 47, 10, 104, 91, 64, 66, 34], [40, 121, 98, 46, 9, 13, 111, 21, 25, 18, 51, 67, 15, 49, 4, 95, 11, 108, 60, 50, 48, 117, 45, 68, 107, 88, 61, 116, 37, 64, 122, 41, 30, 58, 6, 92, 1, 72, 2, 105, 43, 123, 65, 109, 55, 0, 28, 87, 42, 3, 53, 5, 126, 23, 27, 39, 127, 54, 74, 19, 101, 114, 73, 85, 14, 62, 70, 113, 22, 115, 66, 31, 63, 34, 20, 93, 16, 78, 120, 33, 118, 69, 110, 124, 36, 90, 7, 75, 52, 104, 82, 57, 80, 77, 94, 32, 29, 24, 81, 10, 103, 12, 83, 17, 79, 76, 89, 26, 59, 8, 35, 112, 38, 44, 99, 119, 84, 125, 56, 71, 86, 100, 96, 102, 106, 91, 97, 47], [40, 111, 46, 121, 98, 37, 49, 21, 25, 117, 54, 114, 18, 95, 23, 60, 11, 15, 107, 51, 61, 53, 13, 116, 105, 4, 78, 62, 16, 48, 67, 30, 9, 113, 124, 80, 24, 72, 39, 41, 123, 17, 19, 6, 1, 120, 108, 88, 42, 90, 92, 70, 83, 52, 63, 8, 109, 0, 50, 58, 20, 85, 27, 87, 55, 122, 56, 84, 26, 65, 127, 115, 45, 38, 22, 103, 59, 112, 43, 96, 125, 29, 71, 126, 75, 118, 91, 57, 100, 7, 119, 36, 102, 101, 10, 68, 33, 79, 76, 2, 14, 34, 35, 81, 44, 74, 86, 94, 12, 97, 82, 110, 32, 47, 89, 73, 104, 5, 106, 93, 31, 69, 28, 99, 77, 66, 64, 3], [40, 121, 98, 46, 117, 111, 13, 21, 18, 61, 37, 9, 25, 15, 4, 51, 48, 30, 105, 11, 122, 107, 72, 49, 58, 87, 108, 24, 6, 23, 123, 67, 116, 63, 1, 110, 90, 88, 60, 0, 124, 14, 31, 127, 101, 39, 126, 45, 32, 95, 43, 41, 27, 109, 85, 52, 26, 92, 93, 64, 80, 50, 75, 5, 97, 22, 36, 78, 55, 100, 74, 16, 57, 70, 53, 103, 89, 19, 82, 73, 102, 28, 115, 113, 120, 83, 59, 35, 38, 42, 62, 12, 118, 17, 106, 20, 2, 114, 119, 79, 76, 71, 33, 125, 81, 66, 54, 112, 8, 68, 44, 91, 7, 77, 56, 96, 86, 94, 69, 65, 99, 29, 3, 84, 10, 104, 47, 34], [117, 50, 105, 108, 123, 49, 99, 44, 97, 25, 31, 87, 120, 21, 112, 16, 93, 82, 58, 78, 34, 29, 62, 118, 48, 114, 51, 38, 30, 43, 109, 32, 90, 42, 55, 7, 106, 10, 59, 103, 36, 53, 107, 52, 12, 61, 96, 45, 33, 124, 47, 46, 60, 26, 56, 23, 35, 63, 88, 39, 95, 113, 17, 125, 77, 28, 115, 121, 54, 19, 83, 57, 24, 122, 98, 94, 84, 68, 119, 116, 104, 15, 110, 92, 11, 69, 111, 40, 102, 126, 3, 100, 20, 127, 14, 22, 91, 86, 79, 27, 101, 8, 75, 37, 71, 89, 65, 74, 81, 85, 2, 72, 18, 64, 6, 76, 66, 73, 9, 80, 70, 67, 4, 13, 1, 5, 41, 0], [105, 123, 49, 117, 31, 99, 118, 2, 114, 109, 10, 16, 7, 82, 25, 78, 64, 69, 3, 68, 65, 50, 12, 8, 21, 77, 44, 97, 87, 67, 89, 115, 58, 41, 0, 66, 4, 71, 1, 111, 23, 98, 59, 61, 93, 104, 120, 127, 72, 63, 5, 39, 56, 28, 51, 70, 42, 91, 108, 112, 24, 57, 95, 122, 20, 107, 74, 15, 36, 75, 121, 106, 14, 34, 83, 52, 86, 29, 9, 54, 6, 37, 79, 92, 84, 88, 30, 18, 27, 101, 22, 13, 73, 110, 103, 46, 80, 11, 17, 96, 62, 26, 19, 90, 119, 33, 76, 113, 43, 85, 60, 102, 45, 38, 48, 53, 81, 116, 47, 40, 55, 125, 32, 124, 94, 35, 126, 100], [105, 49, 123, 108, 31, 58, 118, 10, 25, 16, 117, 82, 99, 3, 69, 21, 8, 87, 12, 65, 78, 77, 68, 109, 94, 7, 29, 122, 114, 1, 120, 42, 5, 55, 97, 64, 4, 44, 50, 111, 43, 6, 36, 98, 115, 23, 73, 95, 33, 13, 125, 38, 2, 93, 11, 79, 48, 76, 63, 62, 66, 88, 61, 72, 89, 30, 67, 91, 47, 54, 46, 51, 27, 18, 56, 39, 32, 9, 92, 110, 90, 101, 100, 45, 112, 75, 86, 70, 113, 103, 116, 0, 127, 17, 34, 106, 74, 80, 71, 81, 14, 52, 40, 20, 104, 22, 121, 53, 60, 41, 84, 19, 83, 59, 96, 24, 107, 26, 37, 119, 15, 28, 57, 126, 85, 102, 124, 35], [118, 49, 108, 105, 58, 44, 42, 99, 21, 117, 87, 97, 94, 31, 12, 39, 25, 93, 29, 82, 104, 46, 51, 56, 52, 33, 38, 95, 43, 103, 113, 119, 107, 26, 109, 45, 73, 50, 16, 59, 115, 91, 125, 123, 111, 47, 60, 57, 112, 40, 127, 114, 20, 121, 54, 110, 77, 53, 101, 63, 37, 122, 126, 23, 61, 48, 1, 120, 124, 116, 32, 19, 102, 36, 55, 100, 96, 81, 78, 75, 4, 98, 62, 8, 85, 106, 92, 79, 34, 5, 17, 30, 70, 72, 10, 89, 9, 7, 90, 69, 35, 6, 22, 66, 76, 68, 24, 65, 28, 27, 18, 15, 3, 88, 83, 84, 13, 86, 2, 0, 41, 14, 67, 11, 80, 74, 71, 64], [40, 109, 121, 34, 59, 89, 53, 120, 17, 45, 55, 15, 86, 83, 114, 12, 29, 113, 71, 95, 28, 93, 4, 73, 94, 31, 69, 125, 67, 46, 122, 56, 106, 42, 33, 11, 92, 2, 77, 107, 61, 44, 88, 65, 108, 23, 38, 6, 58, 115, 8, 30, 50, 110, 26, 116, 0, 118, 126, 16, 99, 90, 21, 119, 87, 80, 62, 117, 20, 48, 101, 37, 49, 3, 68, 32, 85, 24, 41, 81, 84, 13, 52, 76, 19, 36, 82, 18, 1, 43, 91, 100, 112, 97, 74, 103, 78, 14, 96, 79, 35, 5, 64, 22, 75, 27, 39, 54, 57, 25, 47, 123, 63, 10, 124, 7, 127, 70, 104, 111, 51, 105, 9, 72, 102, 60, 66, 98], [40, 109, 121, 34, 15, 12, 86, 17, 73, 83, 53, 4, 89, 55, 71, 67, 59, 120, 45, 42, 92, 114, 2, 106, 95, 69, 122, 113, 0, 77, 6, 46, 29, 68, 28, 103, 56, 118, 11, 8, 33, 116, 72, 31, 18, 125, 65, 111, 37, 38, 88, 1, 61, 100, 85, 23, 108, 30, 101, 115, 64, 107, 98, 119, 20, 112, 82, 48, 3, 74, 126, 66, 70, 16, 13, 14, 81, 5, 93, 21, 84, 110, 19, 27, 94, 79, 80, 87, 96, 22, 102, 76, 58, 35, 7, 97, 25, 10, 91, 24, 36, 26, 62, 104, 75, 63, 117, 57, 47, 39, 78, 49, 41, 32, 90, 44, 9, 51, 43, 52, 127, 60, 99, 123, 105, 50, 124, 54], [40, 109, 121, 34, 89, 17, 86, 15, 55, 83, 12, 53, 45, 95, 59, 67, 122, 28, 92, 2, 29, 0, 65, 73, 69, 71, 114, 56, 4, 21, 42, 116, 6, 62, 120, 61, 125, 119, 31, 77, 46, 49, 10, 11, 93, 16, 1, 19, 26, 96, 88, 98, 36, 126, 106, 113, 118, 64, 30, 23, 57, 108, 44, 48, 101, 13, 103, 18, 123, 58, 22, 66, 37, 14, 27, 72, 76, 20, 87, 70, 80, 112, 127, 8, 115, 78, 94, 54, 68, 33, 84, 32, 81, 82, 24, 25, 7, 91, 39, 79, 90, 107, 74, 3, 85, 104, 52, 35, 5, 60, 51, 110, 75, 99, 9, 43, 100, 38, 47, 111, 97, 124, 102, 63, 117, 41, 50, 105], [121, 109, 40, 59, 34, 53, 111, 44, 114, 39, 124, 118, 116, 115, 88, 20, 61, 58, 55, 93, 117, 60, 126, 56, 62, 49, 108, 50, 72, 122, 48, 54, 63, 43, 46, 41, 57, 51, 119, 92, 127, 123, 47, 86, 125, 42, 31, 110, 105, 120, 98, 38, 52, 107, 113, 112, 45, 106, 83, 97, 89, 103, 104, 27, 36, 102, 28, 96, 26, 100, 14, 101, 32, 35, 37, 99, 10, 17, 95, 29, 33, 30, 94, 18, 85, 91, 87, 68, 23, 78, 25, 90, 13, 84, 73, 15, 4, 77, 70, 21, 74, 82, 8, 24, 6, 16, 75, 12, 80, 2, 71, 11, 19, 22, 66, 1, 64, 9, 67, 3, 81, 7, 69, 5, 76, 79, 0, 65], [107, 100, 58, 32, 21, 112, 78, 19, 61, 127, 87, 93, 115, 12, 43, 9, 114, 120, 116, 119, 16, 56, 82, 55, 42, 96, 111, 113, 25, 72, 38, 125, 54, 92, 79, 48, 29, 5, 66, 47, 88, 97, 110, 105, 57, 123, 4, 124, 35, 50, 49, 2, 18, 27, 0, 24, 63, 36, 62, 28, 53, 71, 91, 39, 77, 75, 68, 1, 86, 11, 3, 26, 69, 46, 20, 7, 15, 104, 126, 14, 95, 117, 106, 83, 73, 74, 98, 13, 23, 6, 70, 64, 41, 84, 45, 89, 67, 80, 30, 44, 99, 90, 85, 81, 10, 108, 40, 76, 59, 51, 94, 37, 31, 34, 103, 22, 65, 101, 102, 122, 8, 60, 52, 17, 109, 33, 121, 118], [107, 58, 100, 63, 127, 32, 112, 25, 56, 61, 47, 39, 115, 114, 116, 79, 21, 105, 113, 54, 108, 104, 96, 19, 119, 87, 42, 55, 120, 126, 93, 45, 43, 50, 41, 110, 125, 35, 106, 17, 122, 82, 111, 57, 97, 20, 44, 118, 60, 52, 48, 88, 123, 30, 124, 117, 49, 59, 16, 53, 38, 27, 46, 109, 62, 36, 74, 91, 78, 33, 51, 5, 121, 102, 101, 28, 24, 103, 40, 71, 75, 92, 99, 89, 31, 11, 37, 86, 94, 90, 72, 95, 12, 26, 98, 34, 15, 22, 9, 23, 84, 10, 14, 13, 77, 81, 18, 65, 29, 85, 70, 6, 3, 67, 66, 7, 83, 80, 69, 0, 68, 1, 8, 76, 4, 64, 2, 73], [43, 93, 58, 2, 68, 65, 107, 67, 0, 39, 87, 12, 70, 9, 69, 78, 64, 1, 21, 25, 63, 72, 96, 10, 16, 66, 110, 19, 112, 103, 100, 8, 61, 114, 97, 56, 7, 32, 37, 73, 29, 79, 5, 75, 55, 82, 120, 116, 14, 38, 20, 17, 49, 44, 91, 42, 3, 80, 57, 115, 113, 51, 109, 119, 101, 45, 74, 4, 104, 23, 40, 85, 28, 125, 98, 33, 127, 124, 47, 46, 31, 6, 53, 105, 59, 99, 48, 71, 35, 83, 11, 24, 121, 15, 26, 84, 106, 81, 60, 77, 86, 22, 89, 108, 123, 34, 95, 18, 30, 13, 76, 27, 92, 126, 118, 88, 50, 90, 41, 54, 94, 122, 36, 52, 111, 102, 62, 117], [107, 100, 58, 127, 112, 32, 12, 19, 78, 61, 25, 16, 9, 82, 119, 114, 21, 87, 43, 93, 113, 115, 54, 120, 66, 56, 0, 55, 125, 39, 49, 4, 38, 42, 105, 29, 116, 72, 97, 111, 68, 27, 63, 5, 81, 110, 96, 47, 69, 48, 124, 62, 51, 6, 71, 57, 35, 10, 92, 123, 3, 83, 28, 91, 44, 84, 126, 95, 79, 76, 7, 89, 23, 98, 75, 20, 18, 64, 88, 80, 90, 37, 101, 106, 22, 8, 52, 15, 85, 30, 73, 67, 50, 103, 59, 117, 11, 74, 70, 53, 118, 2, 13, 31, 14, 34, 1, 45, 109, 24, 46, 65, 77, 94, 33, 86, 108, 26, 104, 99, 41, 40, 60, 17, 122, 121, 102, 36], [55, 58, 115, 50, 52, 39, 122, 62, 63, 59, 121, 106, 34, 54, 103, 53, 47, 48, 95, 120, 119, 118, 117, 124, 56, 126, 113, 107, 127, 123, 49, 105, 96, 111, 116, 51, 112, 93, 42, 61, 84, 46, 109, 60, 114, 57, 44, 125, 43, 110, 45, 102, 91, 27, 22, 108, 38, 29, 99, 86, 41, 104, 76, 97, 24, 89, 30, 35, 40, 37, 82, 33, 32, 101, 100, 25, 36, 92, 16, 18, 10, 8, 20, 88, 31, 98, 94, 90, 28, 65, 21, 85, 23, 81, 15, 74, 14, 87, 80, 26, 4, 12, 78, 3, 17, 11, 1, 72, 5, 19, 2, 77, 68, 71, 79, 83, 6, 69, 73, 70, 64, 67, 66, 0, 75, 13, 9, 7], [50, 58, 39, 62, 52, 34, 122, 55, 45, 120, 118, 115, 106, 24, 108, 105, 81, 47, 64, 51, 126, 63, 71, 124, 93, 84, 103, 56, 109, 95, 113, 59, 96, 44, 53, 125, 117, 57, 27, 119, 54, 25, 48, 5, 112, 116, 111, 38, 42, 99, 127, 60, 91, 49, 22, 121, 114, 3, 86, 30, 2, 32, 61, 0, 20, 82, 123, 104, 35, 4, 46, 73, 98, 107, 43, 41, 110, 77, 66, 65, 101, 6, 15, 31, 76, 102, 14, 97, 28, 40, 88, 29, 67, 10, 37, 92, 21, 83, 18, 36, 26, 87, 8, 80, 89, 33, 100, 23, 78, 16, 85, 11, 94, 90, 12, 7, 69, 70, 79, 68, 74, 17, 72, 13, 9, 1, 19, 75], [58, 50, 115, 39, 34, 83, 77, 15, 2, 73, 11, 3, 71, 109, 24, 27, 6, 0, 81, 64, 122, 93, 79, 1, 51, 52, 85, 44, 68, 4, 22, 20, 67, 45, 113, 82, 9, 54, 99, 5, 25, 26, 66, 72, 8, 80, 106, 111, 65, 56, 19, 88, 114, 7, 12, 95, 116, 13, 78, 62, 104, 59, 63, 107, 21, 35, 89, 75, 10, 17, 119, 105, 70, 98, 74, 69, 28, 30, 32, 18, 61, 29, 76, 33, 14, 120, 91, 117, 118, 86, 41, 57, 92, 46, 127, 87, 16, 96, 126, 108, 31, 43, 23, 97, 94, 90, 38, 36, 103, 100, 102, 101, 37, 53, 110, 84, 124, 121, 47, 49, 125, 55, 40, 48, 123, 60, 112, 42], [58, 115, 62, 39, 55, 52, 122, 106, 34, 108, 107, 53, 120, 61, 63, 103, 105, 95, 118, 47, 54, 124, 59, 109, 49, 91, 117, 60, 57, 119, 48, 114, 112, 111, 93, 96, 99, 121, 84, 126, 46, 45, 116, 127, 123, 125, 41, 104, 51, 110, 42, 43, 19, 56, 22, 113, 44, 24, 33, 86, 7, 40, 38, 71, 32, 102, 101, 76, 25, 82, 97, 87, 37, 27, 16, 29, 35, 10, 100, 36, 83, 94, 50, 31, 98, 21, 30, 18, 90, 92, 20, 81, 11, 23, 78, 28, 89, 26, 75, 88, 74, 85, 15, 5, 14, 64, 8, 4, 80, 67, 0, 3, 69, 1, 12, 65, 66, 17, 68, 2, 79, 73, 77, 72, 6, 13, 9, 70], [122, 104, 105, 126, 53, 112, 51, 110, 118, 123, 55, 119, 57, 111, 124, 120, 114, 117, 59, 121, 125, 63, 116, 115, 61, 49, 34, 54, 113, 106, 60, 89, 32, 86, 50, 31, 62, 48, 46, 58, 56, 127, 44, 52, 45, 109, 91, 33, 98, 47, 107, 43, 25, 108, 95, 73, 41, 40, 20, 82, 93, 42, 75, 22, 83, 81, 103, 23, 19, 88, 39, 96, 101, 37, 87, 102, 92, 99, 36, 11, 90, 38, 29, 100, 27, 26, 35, 21, 72, 97, 84, 17, 94, 13, 30, 18, 14, 24, 16, 28, 78, 76, 85, 6, 80, 8, 77, 9, 68, 4, 10, 70, 79, 5, 12, 71, 67, 3, 74, 69, 15, 7, 65, 2, 64, 0, 66, 1], [55, 122, 104, 105, 127, 119, 121, 114, 62, 60, 118, 117, 112, 110, 57, 51, 63, 111, 49, 124, 34, 56, 116, 123, 61, 31, 125, 113, 59, 44, 86, 47, 120, 48, 115, 108, 54, 58, 33, 89, 106, 45, 46, 50, 52, 53, 126, 109, 98, 107, 91, 41, 43, 81, 42, 95, 88, 40, 87, 23, 103, 83, 27, 101, 32, 37, 25, 29, 92, 19, 93, 90, 39, 20, 30, 96, 22, 99, 38, 102, 36, 18, 82, 100, 94, 80, 26, 35, 97, 75, 15, 17, 28, 24, 85, 69, 12, 77, 76, 78, 84, 21, 13, 6, 14, 3, 10, 16, 71, 79, 8, 5, 70, 4, 72, 11, 9, 65, 67, 74, 68, 73, 2, 7, 64, 66, 0, 1], [55, 122, 104, 105, 126, 41, 88, 18, 20, 80, 79, 10, 31, 60, 98, 15, 85, 78, 114, 76, 121, 29, 86, 75, 34, 69, 12, 33, 93, 91, 118, 8, 47, 53, 123, 89, 27, 62, 110, 13, 120, 71, 59, 119, 63, 3, 116, 108, 57, 113, 51, 124, 56, 28, 44, 96, 97, 127, 74, 49, 43, 112, 87, 16, 106, 83, 115, 111, 54, 50, 117, 5, 26, 61, 25, 58, 94, 24, 125, 95, 46, 21, 48, 52, 72, 45, 42, 90, 82, 84, 109, 107, 14, 92, 23, 9, 39, 30, 100, 19, 32, 77, 101, 22, 73, 102, 36, 37, 81, 99, 35, 11, 17, 38, 70, 103, 7, 2, 6, 4, 40, 68, 67, 66, 65, 64, 1, 0], [122, 55, 104, 64, 71, 2, 126, 79, 4, 10, 1, 3, 73, 105, 78, 18, 8, 76, 20, 80, 77, 75, 9, 31, 69, 86, 7, 93, 51, 98, 68, 81, 88, 6, 27, 85, 121, 5, 66, 65, 107, 119, 70, 16, 23, 67, 114, 0, 84, 37, 74, 63, 11, 59, 97, 82, 106, 17, 15, 33, 120, 91, 118, 127, 21, 14, 57, 52, 41, 49, 12, 53, 87, 34, 95, 39, 72, 13, 110, 100, 89, 24, 47, 83, 29, 101, 46, 32, 25, 117, 112, 19, 56, 44, 28, 60, 111, 103, 99, 90, 26, 22, 58, 50, 124, 45, 125, 30, 96, 102, 92, 123, 94, 109, 36, 43, 113, 35, 40, 108, 38, 42, 48, 115, 116, 54, 61, 62], [103, 126, 98, 9, 14, 4, 21, 17, 71, 95, 64, 75, 67, 115, 27, 24, 88, 1, 109, 63, 48, 44, 62, 61, 16, 70, 118, 104, 114, 116, 31, 124, 45, 90, 122, 106, 3, 19, 5, 99, 105, 127, 6, 65, 56, 12, 46, 32, 37, 91, 53, 50, 113, 13, 66, 47, 100, 93, 22, 57, 81, 23, 20, 78, 15, 87, 0, 74, 92, 51, 69, 123, 72, 7, 41, 76, 73, 34, 10, 83, 11, 85, 112, 68, 49, 2, 79, 94, 125, 55, 58, 40, 89, 8, 77, 117, 26, 35, 121, 33, 80, 107, 119, 25, 82, 38, 52, 86, 42, 84, 54, 101, 97, 30, 36, 29, 28, 18, 110, 59, 39, 60, 96, 43, 102, 108, 111, 120], [103, 126, 98, 17, 65, 75, 14, 9, 71, 21, 4, 3, 61, 48, 88, 95, 109, 69, 0, 115, 44, 41, 5, 24, 27, 106, 124, 63, 118, 70, 62, 105, 114, 112, 12, 16, 122, 1, 31, 64, 45, 13, 50, 127, 91, 113, 19, 76, 56, 99, 46, 47, 90, 68, 81, 66, 84, 85, 86, 57, 116, 110, 78, 123, 55, 104, 23, 80, 77, 117, 96, 92, 18, 15, 100, 37, 20, 29, 42, 73, 67, 6, 82, 87, 72, 33, 53, 74, 107, 120, 34, 7, 119, 43, 54, 11, 32, 58, 26, 121, 89, 30, 83, 94, 93, 49, 51, 36, 102, 97, 38, 25, 79, 40, 35, 2, 22, 125, 10, 8, 28, 108, 52, 101, 111, 59, 60, 39], [103, 126, 98, 61, 95, 21, 88, 17, 27, 124, 44, 14, 109, 63, 91, 48, 113, 75, 115, 9, 104, 31, 122, 105, 77, 71, 90, 39, 50, 24, 83, 100, 127, 12, 4, 47, 37, 41, 114, 116, 53, 45, 19, 99, 36, 42, 57, 107, 87, 56, 33, 121, 93, 62, 94, 74, 51, 72, 13, 32, 70, 35, 29, 106, 119, 38, 112, 10, 123, 67, 1, 108, 40, 118, 84, 96, 97, 79, 20, 52, 110, 85, 26, 28, 5, 58, 54, 80, 76, 86, 46, 25, 55, 111, 73, 15, 69, 18, 22, 59, 125, 102, 64, 43, 117, 16, 101, 30, 81, 78, 23, 82, 8, 49, 89, 120, 92, 60, 6, 34, 11, 2, 3, 7, 68, 0, 66, 65], [126, 103, 41, 98, 61, 95, 118, 51, 112, 127, 119, 109, 54, 110, 91, 124, 27, 116, 62, 57, 88, 21, 53, 48, 29, 44, 39, 106, 122, 113, 105, 104, 46, 31, 114, 55, 63, 37, 42, 45, 47, 58, 50, 56, 92, 108, 22, 17, 115, 121, 14, 16, 100, 59, 60, 111, 117, 90, 123, 19, 43, 49, 120, 125, 52, 107, 94, 9, 84, 15, 75, 36, 102, 33, 38, 99, 30, 101, 40, 89, 35, 93, 24, 18, 32, 8, 83, 82, 96, 71, 85, 3, 23, 97, 5, 72, 4, 86, 13, 77, 20, 87, 25, 76, 28, 10, 26, 12, 80, 2, 65, 34, 74, 79, 69, 78, 66, 70, 1, 6, 67, 81, 0, 64, 68, 11, 73, 7], [58, 104, 56, 34, 80, 52, 84, 26, 42, 109, 12, 9, 82, 31, 28, 24, 35, 64, 70, 37, 103, 96, 11, 6, 94, 36, 66, 95, 49, 60, 114, 2, 87, 126, 76, 92, 7, 20, 73, 47, 115, 23, 14, 86, 113, 21, 51, 16, 97, 53, 22, 85, 10, 48, 41, 107, 38, 127, 27, 91, 93, 4, 106, 67, 111, 77, 30, 74, 0, 122, 54, 68, 121, 108, 59, 17, 18, 19, 45, 15, 13, 29, 75, 90, 124, 125, 102, 120, 83, 110, 118, 8, 88, 63, 116, 40, 46, 55, 50, 123, 43, 61, 101, 3, 65, 39, 69, 98, 89, 105, 79, 72, 32, 71, 119, 81, 57, 99, 33, 112, 62, 1, 44, 78, 100, 25, 5, 117], [56, 52, 104, 58, 107, 37, 34, 31, 109, 48, 21, 85, 55, 120, 95, 87, 112, 114, 126, 63, 40, 26, 111, 41, 113, 121, 47, 36, 54, 116, 60, 125, 28, 42, 124, 59, 50, 110, 92, 123, 61, 118, 62, 83, 108, 91, 49, 14, 119, 24, 115, 43, 45, 44, 53, 127, 74, 51, 46, 84, 94, 106, 105, 66, 7, 80, 0, 38, 39, 99, 67, 11, 103, 57, 64, 23, 35, 101, 25, 19, 102, 122, 88, 17, 6, 98, 81, 96, 27, 79, 32, 86, 13, 10, 15, 117, 30, 90, 100, 8, 93, 33, 78, 4, 97, 82, 29, 3, 72, 12, 68, 89, 2, 5, 1, 22, 77, 75, 65, 70, 9, 16, 69, 18, 71, 20, 73, 76], [58, 104, 37, 109, 107, 53, 127, 34, 126, 57, 50, 31, 120, 60, 121, 41, 63, 59, 114, 115, 118, 54, 48, 123, 113, 116, 108, 111, 110, 52, 49, 21, 61, 119, 51, 112, 95, 45, 55, 47, 44, 103, 46, 106, 40, 43, 124, 105, 87, 26, 39, 36, 62, 125, 92, 42, 117, 77, 17, 96, 28, 122, 56, 82, 97, 102, 38, 24, 18, 35, 15, 30, 85, 32, 83, 90, 86, 99, 69, 94, 33, 101, 25, 100, 65, 98, 27, 79, 74, 89, 84, 23, 13, 29, 80, 88, 93, 4, 72, 7, 67, 14, 91, 1, 5, 19, 81, 6, 64, 66, 12, 3, 10, 22, 11, 8, 71, 9, 20, 78, 68, 75, 70, 0, 2, 16, 76, 73], [104, 58, 34, 56, 31, 126, 26, 28, 47, 52, 23, 127, 84, 92, 41, 82, 83, 122, 38, 87, 103, 80, 115, 94, 35, 114, 48, 109, 14, 53, 54, 95, 90, 12, 20, 50, 125, 36, 71, 42, 86, 27, 101, 113, 7, 44, 74, 32, 102, 118, 119, 37, 96, 121, 60, 97, 30, 88, 111, 106, 62, 55, 24, 59, 29, 1, 124, 18, 57, 78, 99, 112, 45, 22, 105, 93, 9, 63, 117, 116, 120, 25, 33, 51, 123, 6, 49, 3, 110, 89, 19, 100, 85, 43, 21, 17, 81, 98, 108, 39, 16, 5, 46, 61, 91, 72, 76, 11, 77, 10, 69, 8, 15, 40, 107, 13, 68, 79, 73, 65, 4, 67, 0, 75, 66, 70, 2, 64]], "model.layers.31.self_attn.k_proj": [[104, 121, 34, 25, 47, 1, 72, 6, 21, 110, 51, 0, 15, 11, 13, 18, 112, 113, 4, 9, 64, 60, 44, 41, 111, 31, 87, 54, 30, 116, 66, 45, 107, 114, 67, 120, 123, 19, 10, 105, 109, 117, 53, 80, 58, 71, 92, 61, 98, 57, 17, 62, 23, 124, 55, 27, 42, 14, 22, 103, 63, 56, 3, 7, 65, 102, 69, 5, 84, 115, 16, 78, 50, 106, 49, 46, 43, 90, 122, 28, 100, 101, 119, 94, 99, 91, 125, 24, 79, 93, 52, 59, 26, 36, 33, 118, 81, 76, 97, 108, 96, 88, 68, 74, 83, 75, 12, 39, 95, 126, 8, 32, 89, 38, 70, 48, 29, 127, 37, 40, 35, 86, 20, 82, 2, 73, 77, 85], [41, 49, 123, 117, 64, 50, 113, 25, 87, 95, 44, 53, 93, 7, 65, 33, 35, 82, 69, 16, 21, 3, 45, 108, 10, 118, 8, 77, 54, 78, 12, 121, 68, 66, 127, 63, 115, 43, 120, 58, 47, 83, 106, 91, 42, 57, 111, 2, 0, 99, 110, 36, 124, 11, 55, 109, 94, 97, 125, 112, 102, 116, 61, 107, 101, 59, 37, 31, 32, 51, 119, 19, 126, 62, 6, 60, 52, 100, 84, 40, 34, 46, 39, 114, 98, 26, 9, 15, 122, 24, 38, 105, 103, 70, 28, 88, 22, 56, 73, 48, 27, 29, 5, 89, 86, 104, 75, 17, 30, 13, 96, 79, 90, 85, 76, 92, 20, 1, 81, 14, 4, 67, 80, 23, 18, 72, 71, 74], [104, 121, 45, 98, 109, 0, 86, 65, 83, 89, 15, 67, 73, 56, 6, 17, 69, 50, 12, 53, 120, 93, 114, 55, 92, 61, 125, 110, 59, 31, 49, 71, 118, 2, 62, 58, 116, 126, 124, 11, 26, 57, 52, 88, 37, 123, 119, 47, 122, 48, 42, 108, 106, 30, 111, 54, 100, 51, 43, 115, 113, 77, 105, 102, 4, 127, 63, 60, 112, 97, 44, 29, 117, 103, 66, 28, 85, 7, 107, 64, 46, 20, 27, 33, 16, 90, 23, 41, 13, 101, 99, 96, 39, 38, 21, 95, 32, 18, 36, 74, 94, 91, 76, 68, 10, 35, 75, 70, 25, 82, 87, 72, 3, 80, 81, 24, 84, 14, 22, 19, 78, 79, 5, 9, 1, 8, 34, 40], [43, 93, 96, 48, 87, 16, 12, 127, 61, 78, 9, 36, 49, 19, 21, 66, 51, 0, 25, 119, 68, 58, 82, 65, 39, 5, 114, 54, 72, 50, 111, 10, 79, 3, 47, 56, 8, 106, 110, 120, 75, 116, 70, 105, 53, 107, 4, 33, 6, 63, 20, 55, 115, 108, 125, 44, 71, 32, 35, 91, 46, 17, 62, 27, 76, 126, 37, 123, 31, 102, 24, 97, 42, 57, 118, 121, 117, 98, 40, 7, 26, 18, 52, 113, 1, 59, 73, 11, 99, 88, 109, 122, 34, 41, 92, 60, 64, 74, 112, 124, 2, 45, 101, 95, 67, 103, 38, 94, 104, 13, 29, 89, 30, 77, 80, 86, 22, 83, 90, 28, 81, 84, 23, 15, 14, 100, 85, 69], [103, 58, 98, 115, 50, 45, 77, 83, 73, 81, 91, 114, 6, 11, 15, 24, 108, 29, 25, 99, 27, 4, 64, 106, 2, 31, 65, 30, 71, 127, 3, 95, 110, 28, 82, 54, 117, 57, 113, 35, 63, 107, 93, 96, 105, 49, 42, 90, 88, 84, 125, 22, 111, 119, 120, 43, 48, 126, 53, 59, 60, 118, 124, 102, 51, 46, 47, 39, 21, 68, 123, 26, 112, 122, 61, 14, 109, 5, 44, 41, 40, 52, 97, 80, 23, 69, 87, 86, 85, 37, 56, 33, 100, 116, 38, 8, 36, 101, 34, 92, 104, 94, 89, 62, 79, 55, 74, 121, 12, 16, 32, 70, 17, 10, 76, 66, 19, 20, 78, 18, 72, 67, 1, 13, 75, 7, 9, 0], [40, 126, 122, 95, 55, 34, 91, 86, 20, 29, 65, 88, 64, 33, 23, 2, 119, 41, 82, 89, 107, 114, 113, 98, 63, 127, 115, 4, 110, 121, 1, 61, 71, 118, 106, 75, 18, 78, 42, 73, 0, 49, 125, 80, 79, 103, 51, 117, 116, 59, 62, 124, 97, 47, 56, 45, 36, 85, 123, 52, 43, 48, 108, 46, 50, 68, 60, 76, 54, 111, 10, 57, 101, 109, 22, 44, 58, 81, 112, 25, 120, 7, 93, 13, 77, 8, 92, 53, 37, 90, 83, 102, 30, 39, 32, 38, 67, 17, 96, 19, 74, 3, 21, 100, 14, 99, 69, 105, 35, 28, 94, 72, 5, 24, 6, 27, 66, 26, 12, 70, 15, 87, 16, 84, 9, 31, 11, 104], [126, 34, 39, 64, 17, 88, 21, 75, 9, 71, 14, 31, 4, 112, 3, 63, 108, 45, 114, 1, 27, 124, 70, 105, 115, 65, 103, 12, 109, 111, 49, 61, 5, 106, 2, 36, 51, 91, 58, 16, 53, 46, 18, 116, 118, 122, 95, 127, 69, 24, 94, 104, 55, 41, 54, 76, 42, 50, 113, 35, 90, 107, 13, 19, 96, 62, 22, 32, 20, 40, 48, 79, 125, 57, 110, 83, 87, 59, 99, 102, 60, 119, 89, 52, 92, 26, 100, 123, 117, 47, 67, 77, 29, 93, 86, 25, 23, 38, 97, 30, 10, 28, 101, 56, 72, 66, 84, 43, 15, 37, 120, 82, 33, 121, 8, 80, 44, 98, 74, 68, 0, 78, 7, 81, 11, 73, 6, 85], [40, 58, 56, 98, 95, 87, 92, 52, 90, 26, 30, 101, 84, 32, 6, 66, 45, 80, 11, 64, 21, 41, 106, 18, 51, 28, 9, 54, 82, 60, 48, 59, 126, 112, 118, 100, 61, 12, 25, 79, 116, 88, 125, 122, 63, 110, 111, 113, 78, 39, 3, 8, 62, 127, 20, 43, 74, 119, 49, 114, 13, 72, 29, 47, 24, 50, 121, 108, 46, 97, 16, 76, 17, 124, 53, 123, 44, 104, 120, 109, 89, 115, 55, 102, 105, 27, 96, 57, 14, 1, 38, 4, 91, 73, 7, 117, 85, 65, 10, 94, 77, 22, 33, 83, 15, 42, 0, 93, 36, 103, 35, 19, 99, 37, 75, 5, 86, 81, 23, 107, 67, 70, 69, 68, 71, 2, 31, 34]], "model.layers.31.self_attn.qk_proj": [[121, 126, 58, 43, 122, 49, 109, 50, 55, 115, 123, 103, 56, 104, 117, 45, 93, 89, 40, 61, 98, 105, 85, 21, 107, 25, 127, 23, 34, 41, 51, 114, 64, 118, 39, 95, 111, 0, 31, 53, 73, 9, 48, 108, 52, 4, 29, 87, 78, 27, 82, 12, 14, 44, 120, 24, 113, 28, 54, 16, 65, 79, 15, 76, 18, 81, 68, 66, 88, 80, 112, 63, 116, 17, 124, 83, 71, 2, 75, 7, 67, 119, 3, 19, 1, 110, 11, 60, 22, 5, 106, 62, 59, 46, 47, 6, 96, 13, 8, 99, 32, 72, 77, 125, 69, 42, 70, 91, 57, 26, 74, 86, 30, 36, 100, 20, 10, 37, 84, 97, 33, 90, 35, 101, 102, 94, 38, 92], [121, 58, 126, 43, 122, 49, 109, 50, 55, 115, 56, 103, 123, 117, 104, 45, 93, 89, 61, 98, 85, 107, 21, 105, 40, 118, 25, 127, 51, 23, 34, 41, 114, 111, 64, 0, 31, 39, 9, 29, 95, 52, 73, 27, 68, 108, 87, 76, 124, 48, 112, 12, 54, 53, 4, 113, 28, 78, 80, 88, 15, 18, 82, 81, 116, 1, 65, 66, 120, 24, 16, 2, 14, 67, 7, 83, 6, 71, 79, 11, 110, 119, 75, 44, 63, 17, 19, 72, 106, 3, 47, 46, 60, 22, 99, 59, 13, 5, 77, 62, 32, 69, 125, 96, 42, 57, 8, 74, 91, 86, 84, 70, 26, 37, 30, 20, 97, 10, 36, 100, 90, 94, 33, 35, 38, 102, 101, 92], [121, 58, 126, 43, 122, 109, 49, 50, 55, 115, 56, 123, 103, 117, 45, 104, 98, 21, 89, 93, 105, 107, 40, 61, 85, 51, 114, 25, 34, 41, 0, 64, 127, 118, 108, 111, 23, 95, 39, 31, 53, 9, 68, 87, 73, 52, 12, 27, 48, 82, 63, 15, 29, 14, 28, 124, 78, 4, 116, 24, 88, 120, 112, 44, 54, 76, 113, 18, 2, 80, 16, 1, 81, 66, 6, 65, 7, 79, 110, 72, 17, 83, 67, 3, 60, 19, 75, 119, 11, 71, 46, 59, 69, 106, 47, 99, 22, 5, 62, 77, 32, 13, 96, 125, 42, 91, 86, 100, 57, 20, 84, 37, 70, 26, 33, 10, 36, 97, 30, 74, 8, 35, 94, 90, 102, 101, 92, 38], [121, 126, 58, 43, 122, 49, 109, 55, 50, 115, 56, 123, 103, 104, 117, 45, 105, 89, 21, 107, 98, 93, 61, 40, 118, 85, 51, 41, 34, 25, 23, 127, 31, 0, 64, 52, 114, 108, 73, 39, 111, 95, 68, 29, 9, 113, 53, 116, 16, 54, 27, 124, 28, 78, 87, 12, 4, 76, 14, 24, 80, 120, 66, 79, 15, 88, 1, 112, 81, 72, 82, 65, 44, 48, 18, 63, 71, 83, 7, 2, 67, 6, 46, 3, 110, 106, 59, 17, 60, 19, 75, 11, 42, 69, 13, 32, 47, 99, 5, 125, 77, 26, 22, 57, 119, 62, 84, 74, 91, 96, 33, 20, 86, 10, 100, 94, 90, 37, 70, 36, 8, 30, 97, 102, 101, 35, 38, 92], [121, 126, 58, 43, 122, 49, 109, 50, 55, 115, 56, 123, 104, 103, 117, 45, 21, 93, 98, 105, 40, 107, 25, 89, 41, 61, 23, 118, 85, 127, 51, 34, 0, 73, 111, 114, 31, 9, 64, 39, 53, 108, 52, 95, 68, 12, 78, 24, 120, 15, 4, 113, 65, 82, 16, 79, 116, 124, 14, 48, 29, 27, 54, 87, 80, 44, 81, 72, 67, 2, 3, 76, 63, 83, 28, 71, 1, 7, 66, 18, 112, 47, 75, 17, 11, 46, 88, 119, 106, 19, 5, 60, 42, 13, 6, 69, 110, 77, 26, 59, 99, 22, 62, 57, 70, 125, 96, 32, 33, 86, 84, 10, 30, 20, 100, 91, 37, 8, 74, 97, 36, 102, 94, 90, 92, 38, 35, 101], [121, 126, 58, 43, 49, 122, 109, 50, 55, 115, 56, 123, 104, 103, 45, 117, 98, 93, 21, 61, 41, 89, 105, 40, 107, 23, 25, 0, 85, 127, 64, 114, 31, 39, 51, 118, 34, 73, 111, 9, 108, 95, 27, 53, 15, 16, 79, 80, 4, 87, 113, 24, 52, 120, 82, 78, 76, 68, 1, 12, 14, 119, 83, 48, 54, 66, 18, 65, 116, 17, 7, 112, 29, 75, 88, 44, 2, 28, 71, 72, 81, 67, 63, 11, 106, 5, 99, 3, 59, 124, 46, 19, 60, 110, 13, 47, 69, 77, 22, 70, 42, 26, 96, 125, 6, 32, 62, 20, 84, 86, 57, 10, 91, 33, 30, 37, 36, 100, 74, 8, 101, 97, 94, 102, 90, 35, 38, 92], [121, 126, 58, 43, 122, 49, 55, 109, 50, 115, 123, 56, 93, 104, 103, 117, 45, 21, 89, 98, 61, 107, 105, 40, 25, 41, 85, 23, 127, 114, 34, 108, 118, 39, 51, 0, 9, 31, 64, 73, 53, 95, 27, 111, 113, 79, 14, 29, 4, 24, 15, 82, 78, 52, 12, 76, 16, 44, 88, 68, 48, 18, 54, 87, 17, 80, 116, 81, 11, 65, 75, 63, 112, 120, 19, 28, 1, 119, 7, 66, 124, 71, 110, 83, 3, 72, 70, 106, 67, 2, 13, 96, 22, 125, 42, 59, 77, 30, 5, 47, 62, 99, 57, 69, 60, 26, 46, 32, 86, 20, 33, 8, 84, 97, 6, 37, 10, 36, 100, 91, 74, 90, 101, 38, 94, 102, 35, 92], [121, 58, 126, 43, 122, 49, 109, 50, 55, 115, 123, 56, 117, 103, 104, 45, 93, 98, 21, 89, 61, 40, 85, 105, 41, 107, 25, 114, 108, 23, 34, 118, 51, 127, 0, 64, 39, 111, 9, 95, 53, 31, 73, 29, 27, 24, 113, 124, 82, 4, 14, 79, 44, 48, 76, 68, 52, 18, 112, 12, 16, 28, 87, 116, 120, 66, 15, 54, 80, 63, 78, 81, 1, 7, 119, 75, 65, 3, 83, 71, 88, 70, 17, 2, 19, 11, 60, 67, 59, 106, 47, 110, 22, 13, 72, 77, 99, 125, 46, 5, 96, 57, 42, 32, 26, 30, 69, 6, 91, 8, 36, 86, 62, 37, 84, 10, 74, 97, 20, 33, 101, 90, 100, 94, 38, 102, 35, 92], [121, 126, 58, 43, 122, 49, 50, 109, 55, 115, 56, 103, 123, 45, 117, 93, 104, 40, 85, 21, 89, 98, 105, 41, 61, 107, 25, 23, 114, 34, 51, 108, 111, 39, 127, 9, 95, 29, 31, 118, 73, 27, 113, 53, 52, 4, 64, 87, 0, 12, 24, 124, 44, 16, 48, 82, 76, 63, 78, 116, 18, 17, 88, 112, 120, 28, 68, 119, 60, 54, 15, 79, 14, 80, 81, 83, 65, 110, 19, 11, 7, 1, 46, 59, 75, 70, 106, 71, 3, 32, 42, 2, 66, 67, 125, 77, 47, 26, 13, 96, 99, 57, 91, 72, 22, 8, 62, 5, 86, 30, 97, 6, 74, 20, 69, 84, 37, 10, 36, 33, 94, 90, 100, 101, 35, 102, 92, 38], [121, 58, 126, 43, 122, 49, 109, 50, 55, 115, 56, 123, 103, 45, 117, 93, 104, 98, 89, 105, 61, 40, 21, 107, 41, 25, 118, 64, 0, 34, 51, 114, 85, 95, 39, 127, 53, 23, 9, 108, 31, 73, 27, 111, 112, 87, 68, 29, 113, 52, 120, 15, 78, 12, 24, 4, 44, 80, 48, 88, 16, 2, 14, 110, 82, 3, 67, 7, 116, 71, 63, 1, 65, 76, 28, 54, 66, 81, 18, 11, 106, 79, 83, 60, 75, 124, 17, 119, 19, 70, 46, 59, 8, 99, 13, 32, 42, 6, 47, 5, 77, 69, 22, 62, 125, 57, 72, 30, 20, 26, 91, 74, 86, 96, 84, 10, 97, 37, 36, 94, 90, 100, 33, 101, 35, 38, 102, 92], [121, 58, 126, 43, 122, 55, 49, 109, 50, 115, 56, 123, 103, 104, 117, 45, 93, 98, 21, 105, 61, 41, 89, 0, 40, 107, 127, 23, 118, 64, 34, 25, 51, 114, 9, 73, 85, 39, 31, 108, 52, 95, 111, 4, 16, 78, 15, 12, 113, 87, 14, 76, 80, 24, 124, 68, 27, 44, 53, 79, 1, 82, 120, 29, 65, 67, 83, 8, 116, 54, 88, 48, 18, 71, 7, 63, 11, 28, 2, 66, 3, 75, 60, 81, 5, 17, 110, 46, 19, 106, 112, 13, 70, 119, 59, 42, 6, 32, 77, 99, 69, 62, 22, 57, 86, 26, 47, 91, 125, 84, 74, 10, 20, 97, 96, 33, 72, 37, 100, 30, 36, 94, 90, 35, 102, 101, 92, 38], [121, 126, 58, 43, 122, 49, 109, 55, 50, 115, 56, 123, 103, 104, 117, 45, 93, 89, 21, 98, 85, 105, 61, 107, 40, 41, 25, 51, 118, 23, 114, 127, 34, 95, 0, 73, 64, 39, 31, 9, 111, 108, 53, 113, 27, 29, 12, 52, 28, 24, 44, 78, 124, 14, 87, 16, 4, 82, 15, 79, 80, 68, 65, 76, 54, 48, 1, 11, 81, 18, 120, 88, 116, 8, 63, 2, 17, 71, 75, 7, 83, 110, 6, 112, 119, 66, 46, 67, 13, 19, 60, 3, 42, 77, 57, 59, 32, 47, 99, 106, 5, 26, 69, 22, 125, 91, 86, 20, 96, 62, 70, 100, 84, 97, 74, 37, 30, 72, 33, 10, 36, 102, 90, 101, 94, 38, 35, 92], [121, 126, 58, 43, 122, 49, 109, 50, 55, 115, 56, 123, 103, 117, 45, 104, 93, 89, 98, 40, 21, 107, 25, 61, 105, 85, 95, 51, 41, 127, 114, 118, 39, 34, 108, 29, 23, 111, 27, 31, 53, 64, 112, 73, 113, 0, 87, 9, 44, 119, 76, 28, 4, 78, 82, 54, 24, 79, 120, 18, 80, 14, 71, 16, 52, 68, 15, 12, 48, 17, 124, 2, 88, 8, 1, 65, 63, 19, 6, 11, 7, 81, 110, 67, 83, 75, 116, 3, 60, 59, 106, 66, 47, 46, 32, 22, 125, 13, 42, 77, 57, 99, 30, 69, 96, 5, 26, 84, 97, 70, 91, 10, 86, 74, 36, 62, 20, 100, 37, 33, 90, 72, 94, 38, 35, 101, 102, 92], [121, 126, 58, 43, 122, 49, 109, 50, 55, 115, 56, 123, 104, 103, 117, 45, 93, 21, 89, 98, 61, 64, 40, 105, 107, 51, 127, 34, 0, 25, 41, 118, 85, 31, 73, 114, 95, 23, 39, 9, 52, 111, 4, 53, 108, 29, 113, 76, 87, 27, 78, 16, 1, 28, 82, 44, 71, 48, 8, 12, 24, 68, 18, 67, 112, 110, 81, 3, 120, 88, 65, 14, 116, 80, 15, 2, 11, 6, 60, 79, 66, 83, 54, 63, 19, 7, 106, 124, 75, 17, 125, 119, 47, 13, 46, 99, 5, 69, 59, 77, 42, 91, 62, 32, 96, 86, 10, 57, 97, 22, 26, 74, 84, 37, 20, 70, 72, 36, 30, 100, 33, 101, 94, 90, 102, 35, 38, 92], [121, 126, 58, 43, 122, 49, 109, 50, 55, 115, 123, 56, 117, 103, 104, 45, 21, 93, 98, 0, 105, 64, 40, 107, 25, 34, 41, 89, 51, 73, 23, 61, 114, 118, 53, 85, 9, 127, 31, 27, 95, 111, 76, 68, 113, 4, 39, 108, 52, 29, 16, 78, 82, 67, 24, 65, 18, 66, 80, 54, 7, 44, 87, 15, 79, 14, 71, 116, 1, 63, 6, 83, 28, 12, 8, 48, 81, 11, 112, 88, 2, 120, 17, 3, 106, 75, 119, 124, 5, 60, 19, 22, 110, 47, 99, 46, 59, 13, 69, 77, 42, 125, 84, 26, 32, 70, 57, 10, 91, 62, 74, 96, 37, 97, 20, 72, 30, 86, 100, 90, 36, 33, 94, 38, 102, 101, 92, 35], [121, 126, 58, 43, 122, 49, 50, 109, 55, 115, 56, 103, 123, 45, 117, 104, 93, 98, 21, 89, 85, 118, 107, 114, 61, 25, 23, 34, 105, 41, 40, 127, 73, 51, 31, 108, 95, 9, 0, 111, 64, 39, 29, 52, 4, 53, 87, 27, 76, 14, 79, 44, 24, 1, 16, 78, 120, 68, 28, 82, 112, 12, 15, 124, 48, 116, 65, 113, 80, 81, 71, 63, 18, 54, 7, 88, 83, 67, 11, 66, 106, 119, 110, 60, 75, 19, 17, 8, 2, 47, 32, 22, 59, 57, 13, 3, 26, 77, 6, 99, 70, 46, 125, 5, 42, 69, 62, 72, 91, 96, 33, 20, 84, 74, 10, 30, 86, 97, 37, 100, 36, 101, 90, 102, 35, 38, 94, 92], [121, 58, 126, 43, 122, 49, 109, 50, 55, 115, 56, 123, 117, 103, 104, 45, 93, 89, 85, 21, 40, 107, 25, 105, 23, 98, 61, 31, 34, 127, 39, 51, 114, 95, 118, 64, 41, 111, 0, 108, 29, 9, 27, 73, 87, 53, 28, 82, 52, 112, 68, 88, 76, 113, 24, 15, 48, 1, 81, 14, 120, 44, 18, 79, 4, 71, 78, 16, 106, 12, 80, 54, 2, 110, 66, 67, 17, 63, 19, 65, 116, 11, 119, 83, 75, 7, 60, 70, 32, 125, 13, 124, 3, 59, 47, 77, 91, 42, 96, 8, 57, 62, 99, 97, 72, 26, 46, 22, 69, 37, 5, 20, 74, 30, 100, 6, 86, 10, 33, 84, 90, 35, 36, 38, 101, 94, 102, 92], [121, 58, 126, 43, 122, 109, 49, 50, 55, 115, 56, 123, 103, 117, 45, 104, 93, 105, 98, 89, 40, 21, 107, 61, 85, 51, 0, 34, 64, 41, 118, 114, 25, 95, 23, 39, 73, 111, 31, 9, 127, 108, 53, 76, 52, 28, 27, 29, 87, 4, 116, 82, 44, 112, 78, 15, 113, 120, 48, 80, 24, 54, 106, 16, 124, 110, 18, 1, 2, 63, 68, 71, 88, 14, 12, 11, 83, 19, 17, 81, 66, 119, 65, 79, 67, 7, 46, 60, 3, 70, 77, 72, 47, 59, 99, 75, 32, 5, 22, 69, 91, 125, 96, 62, 42, 13, 57, 6, 84, 86, 26, 8, 30, 10, 36, 37, 74, 97, 33, 20, 94, 90, 100, 102, 35, 38, 101, 92], [121, 126, 58, 43, 49, 55, 122, 50, 109, 115, 123, 56, 103, 117, 104, 45, 105, 98, 93, 40, 21, 25, 107, 89, 61, 51, 41, 85, 0, 95, 111, 23, 34, 31, 39, 9, 64, 114, 118, 73, 29, 127, 53, 27, 4, 87, 113, 52, 44, 14, 78, 71, 116, 1, 76, 28, 24, 108, 16, 80, 48, 54, 66, 124, 82, 112, 68, 15, 18, 88, 3, 12, 2, 83, 81, 46, 79, 120, 17, 70, 47, 67, 11, 106, 7, 119, 63, 65, 72, 110, 59, 75, 42, 60, 125, 19, 32, 13, 77, 99, 26, 5, 62, 57, 91, 69, 20, 22, 84, 96, 86, 74, 6, 8, 37, 10, 97, 100, 36, 30, 90, 33, 101, 94, 102, 38, 35, 92], [121, 58, 126, 43, 122, 49, 109, 55, 50, 115, 56, 123, 104, 117, 103, 45, 93, 98, 21, 105, 25, 89, 40, 61, 107, 85, 41, 23, 34, 64, 51, 127, 31, 118, 0, 9, 39, 114, 95, 73, 111, 27, 68, 16, 79, 108, 4, 53, 82, 29, 52, 76, 87, 48, 44, 24, 12, 65, 78, 88, 28, 116, 14, 66, 46, 71, 18, 80, 113, 3, 11, 15, 112, 7, 2, 72, 1, 54, 124, 70, 106, 119, 81, 17, 120, 83, 67, 75, 63, 47, 60, 19, 99, 110, 5, 32, 77, 13, 22, 69, 59, 42, 26, 125, 57, 62, 84, 74, 10, 96, 91, 86, 30, 6, 33, 20, 37, 97, 8, 90, 100, 36, 38, 102, 94, 35, 101, 92], [121, 126, 58, 43, 109, 49, 122, 50, 55, 115, 56, 123, 103, 117, 45, 104, 93, 105, 61, 98, 40, 21, 89, 41, 107, 114, 51, 25, 127, 85, 118, 34, 23, 0, 95, 64, 31, 111, 53, 52, 9, 39, 108, 44, 73, 48, 27, 87, 76, 78, 82, 79, 16, 24, 29, 113, 4, 14, 1, 68, 17, 120, 28, 12, 110, 81, 88, 15, 71, 72, 124, 2, 54, 80, 11, 112, 66, 60, 63, 116, 65, 18, 46, 7, 106, 3, 83, 67, 47, 99, 62, 19, 59, 119, 70, 22, 75, 5, 125, 13, 69, 77, 100, 91, 6, 57, 96, 42, 84, 32, 26, 37, 20, 86, 33, 74, 36, 30, 10, 8, 102, 97, 94, 90, 101, 35, 38, 92], [121, 126, 58, 43, 122, 55, 49, 109, 50, 115, 56, 123, 117, 45, 103, 104, 93, 105, 98, 61, 64, 51, 89, 95, 21, 40, 41, 107, 85, 0, 25, 34, 118, 23, 31, 29, 114, 53, 108, 39, 9, 111, 52, 73, 127, 44, 87, 4, 27, 124, 16, 113, 116, 14, 68, 79, 76, 18, 67, 82, 120, 66, 78, 2, 28, 112, 63, 12, 48, 1, 106, 80, 72, 15, 24, 17, 88, 54, 65, 7, 47, 83, 110, 81, 71, 119, 3, 19, 11, 75, 6, 46, 125, 60, 57, 69, 99, 59, 5, 13, 77, 32, 70, 42, 26, 22, 62, 84, 96, 86, 91, 10, 97, 20, 8, 74, 33, 37, 30, 90, 101, 100, 36, 94, 102, 38, 35, 92], [121, 58, 126, 43, 122, 109, 49, 50, 55, 115, 56, 123, 117, 103, 45, 104, 93, 105, 98, 21, 40, 89, 61, 25, 107, 51, 41, 34, 85, 114, 118, 127, 95, 23, 31, 39, 111, 0, 108, 73, 64, 53, 29, 9, 27, 52, 87, 4, 113, 44, 28, 112, 76, 78, 14, 48, 79, 82, 88, 3, 80, 54, 71, 116, 16, 15, 110, 17, 18, 120, 1, 24, 7, 12, 124, 47, 63, 68, 67, 83, 2, 65, 81, 59, 60, 119, 19, 106, 11, 72, 6, 75, 66, 5, 125, 46, 77, 13, 69, 32, 99, 57, 22, 26, 96, 42, 62, 91, 20, 86, 74, 97, 84, 8, 10, 30, 70, 37, 33, 36, 100, 90, 38, 35, 94, 92, 101, 102], [121, 58, 126, 43, 122, 49, 109, 55, 50, 115, 123, 56, 103, 117, 104, 45, 93, 21, 98, 61, 40, 105, 23, 89, 25, 41, 107, 118, 34, 85, 127, 31, 51, 0, 111, 108, 114, 64, 95, 73, 39, 9, 52, 27, 53, 24, 113, 4, 76, 18, 15, 12, 54, 80, 79, 87, 120, 44, 16, 68, 78, 82, 48, 29, 28, 112, 6, 1, 14, 81, 71, 63, 88, 2, 65, 7, 17, 75, 66, 11, 124, 83, 116, 119, 72, 106, 47, 19, 46, 3, 67, 110, 69, 22, 59, 60, 99, 42, 125, 77, 32, 13, 26, 62, 5, 91, 74, 96, 8, 57, 86, 84, 100, 20, 33, 97, 10, 36, 30, 37, 70, 90, 94, 101, 102, 35, 38, 92], [121, 58, 126, 43, 122, 49, 109, 55, 50, 115, 56, 123, 103, 117, 104, 45, 93, 98, 40, 89, 21, 105, 41, 61, 85, 107, 51, 25, 127, 23, 34, 95, 118, 114, 0, 39, 9, 73, 31, 64, 53, 108, 111, 27, 52, 4, 113, 79, 87, 44, 48, 78, 12, 80, 28, 68, 24, 14, 76, 54, 3, 120, 29, 82, 15, 81, 110, 66, 65, 116, 18, 75, 63, 1, 112, 124, 7, 6, 88, 16, 11, 71, 17, 2, 83, 119, 106, 67, 19, 42, 60, 46, 22, 99, 47, 77, 62, 57, 96, 72, 8, 13, 59, 32, 5, 69, 125, 91, 70, 30, 20, 26, 97, 74, 86, 84, 37, 10, 33, 36, 100, 90, 101, 38, 102, 35, 94, 92], [121, 126, 58, 43, 122, 49, 109, 50, 55, 115, 56, 123, 103, 45, 117, 104, 89, 93, 98, 40, 105, 21, 85, 34, 107, 25, 114, 41, 23, 118, 95, 127, 51, 64, 31, 61, 39, 111, 29, 108, 0, 27, 112, 53, 73, 28, 87, 44, 9, 12, 113, 52, 68, 18, 78, 88, 24, 15, 82, 124, 116, 120, 54, 48, 14, 76, 60, 79, 16, 66, 1, 7, 75, 81, 65, 80, 4, 17, 63, 110, 11, 19, 2, 71, 106, 6, 119, 47, 59, 83, 99, 67, 3, 13, 46, 77, 42, 8, 125, 96, 22, 32, 57, 91, 5, 62, 70, 69, 20, 86, 37, 30, 74, 72, 84, 26, 36, 97, 100, 94, 38, 10, 33, 101, 90, 35, 102, 92], [121, 58, 126, 43, 122, 49, 109, 50, 55, 115, 56, 123, 103, 117, 104, 45, 93, 40, 98, 21, 89, 105, 127, 61, 25, 107, 23, 85, 34, 41, 111, 114, 51, 108, 73, 118, 39, 31, 95, 0, 29, 9, 64, 48, 28, 52, 68, 53, 27, 54, 112, 87, 24, 82, 12, 120, 113, 4, 78, 76, 44, 88, 16, 14, 79, 17, 80, 1, 15, 18, 116, 110, 81, 83, 63, 65, 2, 75, 7, 47, 59, 46, 66, 19, 71, 60, 8, 67, 119, 11, 106, 3, 13, 6, 42, 32, 99, 62, 125, 124, 5, 77, 26, 70, 69, 57, 86, 96, 37, 30, 91, 33, 72, 22, 97, 20, 84, 74, 100, 10, 36, 102, 94, 35, 90, 38, 101, 92], [121, 58, 126, 43, 122, 49, 109, 50, 55, 115, 56, 123, 103, 117, 104, 45, 93, 98, 21, 61, 89, 40, 105, 41, 85, 107, 23, 0, 25, 51, 34, 114, 64, 95, 31, 127, 111, 73, 118, 39, 9, 113, 108, 29, 24, 53, 52, 27, 68, 4, 120, 76, 79, 16, 18, 82, 28, 87, 12, 78, 44, 54, 116, 1, 124, 14, 15, 80, 7, 71, 48, 46, 3, 8, 66, 2, 75, 88, 67, 83, 112, 81, 110, 65, 63, 11, 17, 60, 70, 106, 99, 125, 47, 19, 22, 5, 119, 13, 59, 42, 77, 57, 32, 69, 62, 26, 74, 6, 37, 10, 97, 91, 96, 84, 100, 72, 20, 86, 33, 30, 36, 90, 94, 101, 102, 35, 38, 92], [121, 126, 58, 43, 122, 49, 109, 55, 50, 115, 56, 123, 103, 104, 117, 45, 93, 98, 105, 89, 21, 64, 40, 61, 85, 0, 107, 51, 25, 114, 118, 23, 41, 127, 9, 53, 34, 111, 95, 31, 73, 39, 108, 48, 113, 1, 68, 12, 52, 27, 44, 54, 87, 4, 29, 120, 14, 80, 18, 78, 70, 24, 66, 7, 8, 79, 28, 82, 15, 76, 63, 16, 112, 67, 71, 81, 124, 75, 46, 65, 88, 2, 3, 17, 110, 116, 69, 83, 11, 47, 106, 119, 19, 125, 59, 60, 77, 57, 32, 13, 99, 42, 62, 5, 26, 96, 10, 22, 33, 84, 86, 74, 97, 100, 20, 37, 91, 30, 36, 6, 72, 90, 102, 101, 94, 35, 92, 38], [121, 58, 126, 43, 122, 49, 50, 55, 109, 115, 56, 123, 103, 104, 117, 45, 93, 21, 40, 105, 98, 89, 61, 85, 127, 23, 41, 25, 107, 118, 114, 51, 34, 31, 39, 95, 73, 111, 9, 108, 64, 53, 0, 27, 29, 48, 12, 44, 79, 52, 82, 68, 87, 24, 54, 15, 18, 14, 113, 80, 78, 28, 16, 76, 120, 81, 70, 65, 112, 17, 116, 4, 88, 124, 75, 8, 83, 7, 110, 3, 1, 71, 11, 60, 19, 66, 63, 59, 106, 125, 47, 119, 2, 67, 13, 57, 69, 46, 77, 99, 5, 32, 96, 22, 42, 62, 20, 30, 26, 86, 10, 74, 91, 84, 97, 100, 33, 37, 36, 72, 94, 6, 90, 101, 102, 38, 35, 92], [121, 126, 58, 43, 122, 49, 109, 55, 50, 115, 56, 123, 104, 103, 117, 45, 98, 93, 89, 105, 61, 21, 40, 107, 41, 23, 25, 51, 127, 64, 34, 85, 0, 114, 39, 118, 31, 108, 73, 111, 9, 95, 4, 12, 53, 15, 87, 44, 29, 27, 113, 28, 79, 68, 52, 1, 78, 48, 14, 112, 80, 24, 16, 120, 116, 63, 76, 18, 65, 70, 2, 7, 54, 71, 67, 66, 81, 8, 82, 110, 11, 83, 124, 17, 3, 19, 47, 88, 75, 106, 46, 119, 125, 13, 60, 69, 59, 32, 5, 77, 99, 57, 62, 22, 96, 91, 6, 42, 86, 26, 30, 97, 72, 37, 20, 74, 84, 10, 90, 36, 94, 33, 100, 101, 102, 38, 35, 92], [121, 126, 58, 43, 122, 49, 50, 109, 55, 115, 123, 56, 103, 117, 104, 45, 93, 40, 89, 85, 105, 107, 98, 61, 21, 127, 23, 25, 114, 34, 118, 41, 95, 111, 51, 39, 9, 108, 29, 73, 31, 52, 53, 27, 12, 0, 24, 64, 18, 16, 120, 14, 113, 78, 87, 79, 54, 44, 112, 48, 4, 76, 15, 82, 80, 88, 68, 63, 28, 17, 1, 75, 81, 83, 119, 116, 8, 71, 11, 19, 7, 65, 106, 124, 2, 67, 3, 110, 66, 60, 70, 59, 5, 77, 13, 47, 96, 22, 62, 32, 99, 69, 46, 6, 42, 125, 57, 91, 86, 20, 26, 30, 72, 84, 74, 97, 10, 37, 36, 33, 94, 90, 100, 101, 35, 38, 102, 92]]} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ep/test_deepep_large.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ep/test_deepep_large.py new file mode 100644 index 00000000..26f4925f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ep/test_deepep_large.py @@ -0,0 +1,159 @@ +import os +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestDeepseek(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--moe-dense-tp-size", + "1", + "--enable-dp-lm-head", + "--moe-a2a-backend", + "deepep", + "--enable-two-batch-overlap", + "--ep-num-redundant-experts", + "32", + "--ep-dispatch-algorithm", + "dynamic", + "--eplb-algorithm", + "deepseek", + "--cuda-graph-bs", + "256", + "--max-running-requests", + "2048", + "--disable-radix-cache", + ], + env={ + **os.environ, + "SGLANG_JIT_DEEPGEMM_PRECOMPILE": "0", + }, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1200, + parallel=1200, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Eval accuracy of GSM8K: {metrics=}") + + self.assertGreater(metrics["accuracy"], 0.92) + + +@unittest.skip("Can pass locally, but will cause Timeout on CI runner.") +class TestDeepseekMTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--moe-dense-tp-size", + "1", + "--enable-dp-lm-head", + "--moe-a2a-backend", + "deepep", + "--enable-two-batch-overlap", + "--ep-num-redundant-experts", + "32", + "--ep-dispatch-algorithm", + "dynamic", + "--eplb-algorithm", + "deepseek", + "--cuda-graph-bs", + "64", # TODO: increase it to 128 when TBO is supported in draft_extend + "--max-running-requests", + "512", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "1", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "2", + "--disable-radix-cache", + ], + env={ + **os.environ, + "SGLANG_JIT_DEEPGEMM_PRECOMPILE": "0", + }, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1200, + parallel=1200, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Eval accuracy of GSM8K: {metrics=}") + + self.assertGreater(metrics["accuracy"], 0.92) + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print( + f"###test_gsm8k:\n" + f"accuracy={metrics['accuracy']=:.3f}\n" + f"{avg_spec_accept_length=:.3f}\n" + ) + self.assertGreater(avg_spec_accept_length, 1.85) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ep/test_deepep_small.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ep/test_deepep_small.py new file mode 100644 index 00000000..4417affb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ep/test_deepep_small.py @@ -0,0 +1,388 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST_MLA, + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestPureDP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "4", + "--enable-dp-attention", + "--dp", + "4", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "128", + "--max-running-requests", + "512", + "--mem-fraction-static", + "0.5", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + +class TestHybridDPTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "4", + "--enable-dp-attention", + "--dp", + "2", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "128", + "--max-running-requests", + "256", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + +class TestTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "4", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "128", + "--max-running-requests", + "128", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + +@unittest.skip("covered in test_deepep_large.py") +class TestNoGatherdBuffer(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "4", + "--enable-dp-attention", + "--dp", + "4", + "--moe-dense-tp-size", + "1", + "--enable-dp-lm-head", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "32", + "--max-running-requests", + "512", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + +class TestTBO(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "4", + "--enable-dp-attention", + "--dp", + "4", + "--moe-dense-tp-size", + "1", + "--moe-a2a-backend", + "deepep", + "--enable-two-batch-overlap", + "--cuda-graph-max-bs", + "128", + "--max-running-requests", + "512", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + +@unittest.skip("covered in TestMTPWithTBO") +class TestMTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "4", + "--enable-dp-attention", + "--dp", + "2", + "--enable-dp-lm-head", + "--moe-a2a-backend", + "deepep", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "3", + "--speculative-num-draft-tokens", + "3", + "--cuda-graph-max-bs", + "32", + "--max-running-requests", + "64", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print( + f"###test_gsm8k (deepseek-v3 mtp + dp + tbo):\n" + f"accuracy={metrics['accuracy']=:.3f}\n" + f"{avg_spec_accept_length=:.3f}\n" + ) + self.assertGreater(avg_spec_accept_length, 2.1) + + +class TestMTPWithTBO(CustomTestCase): + @classmethod + def setUpClass(cls): + + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--tp-size", + "4", + "--enable-dp-attention", + "--dp-size", + "4", + "--enable-two-batch-overlap", + "--moe-a2a-backend", + "deepep", + "--trust-remote-code", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "3", + "--speculative-num-draft-tokens", + "3", + "--speculative-draft-model-path", + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, + "--chunked-prefill-size", + "256", + "--cuda-graph-max-bs", + "32", + "--max-running-requests", + "128", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print( + f"###test_gsm8k (deepseek-v3 mtp + dp + tbo):\n" + f"accuracy={metrics['accuracy']=:.3f}\n" + f"{avg_spec_accept_length=:.3f}\n" + ) + self.assertGreater(avg_spec_accept_length, 2.1) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ep/test_moe_ep.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ep/test_moe_ep.py new file mode 100644 index 00000000..74a5790d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ep/test_moe_ep.py @@ -0,0 +1,90 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestEp(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "2", + "--ep-size", + "2", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.8) + + +class TestEpDeepGEMM(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "2", + "--ep-size", + "2", + "--quantization", + "fp8", + "--moe-runner-backend", + "deep_gemm", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.8) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/experiment_runner.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/experiment_runner.py new file mode 100644 index 00000000..f6af0562 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/experiment_runner.py @@ -0,0 +1,364 @@ +import argparse +import logging +import os +import queue +import re +import subprocess +import threading +import time +from dataclasses import dataclass +from datetime import datetime +from typing import List, Optional, Tuple + +import psutil +import requests +import yaml + + +@dataclass +class ServerConfig: + command: str + process_names: List[str] + default_port: int + + +@dataclass +class TaskConfig: + server_cmd: str + client_cmd: str + name: Optional[str] = None + server_type: Optional[str] = None + + +@dataclass +class TaskResult: + name: str + success: bool + output: str + runtime: float + timestamp: str + + +SERVER_DEFAULTS = { + "sglang": ServerConfig( + command="sglang.launch_server", + process_names=["sglang.launch_server"], + default_port=30000, + ), + "vllm": ServerConfig( + command="vllm.entrypoints.openai.api_server", + process_names=["vllm.entrypoints.openai.api_server"], + default_port=8000, + ), +} + + +def parse_key_info(output: str) -> str: + """Extract and format key information from the output""" + key_info = [] + + # Extract Args namespace + args_match = re.search(r"Namespace\(.*?\)", output, re.DOTALL) + if args_match: + key_info.append(args_match.group(0)) + + # Extract input/output token counts + token_matches = re.findall(r"#(Input|Output) tokens: \d+", output) + key_info.extend(token_matches) + + # Extract benchmark result section + result_match = re.search( + r"============ Serving Benchmark Result ============.*?={50,}", + output, + re.DOTALL, + ) + if result_match: + key_info.append(result_match.group(0)) + + return "\n\n".join(key_info) + + +def extract_port_from_command(cmd: str, server_type: str) -> int: + port_match = re.search(r"--port[= ](\d+)", cmd) + if port_match: + return int(port_match.group(1)) + return SERVER_DEFAULTS.get(server_type, ServerConfig("", [], 8000)).default_port + + +def detect_server_type(cmd: str) -> str: + for server_type, config in SERVER_DEFAULTS.items(): + if config.command in cmd: + return server_type + return "unknown" + + +def stream_output( + process: subprocess.Popen, prefix: str, logger: logging.Logger +) -> queue.Queue: + output_queue = queue.Queue() + + def stream_pipe(pipe, prefix): + for line in iter(pipe.readline, ""): + if prefix == "CLIENT": + output_queue.put(line.rstrip()) + logger.debug(f"{prefix} | {line.rstrip()}") + + stdout_thread = threading.Thread( + target=stream_pipe, args=(process.stdout, prefix), daemon=True + ) + stderr_thread = threading.Thread( + target=stream_pipe, args=(process.stderr, prefix), daemon=True + ) + + stdout_thread.start() + stderr_thread.start() + return output_queue, (stdout_thread, stderr_thread) + + +class ProcessManager: + def __init__(self): + self.server_process: Optional[subprocess.Popen] = None + self.client_process: Optional[subprocess.Popen] = None + self.logger = logging.getLogger(__name__) + + def start_process( + self, command: str, prefix: str + ) -> Tuple[subprocess.Popen, queue.Queue]: + process = subprocess.Popen( + command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + + output_queue, threads = stream_output(process, prefix, self.logger) + return process, output_queue, threads + + def kill_process_tree(self, process: subprocess.Popen): + try: + parent = psutil.Process(process.pid) + children = parent.children(recursive=True) + + for child in children: + try: + child.kill() + except psutil.NoSuchProcess: + pass + + parent.kill() + gone, alive = psutil.wait_procs(children + [parent], timeout=3) + + for p in alive: + try: + p.kill() + except psutil.NoSuchProcess: + pass + + except psutil.NoSuchProcess: + pass + + def cleanup(self, process_names: List[str]): + if self.client_process: + self.kill_process_tree(self.client_process) + self.client_process = None + + if self.server_process: + self.kill_process_tree(self.server_process) + self.server_process = None + + for proc in psutil.process_iter(["pid", "name", "cmdline"]): + try: + cmdline = " ".join(proc.cmdline()) + if any(name in cmdline for name in process_names): + proc.kill() + except (psutil.NoSuchProcess, psutil.AccessDenied): + continue + + +class ExperimentRunner: + def __init__(self): + self.process_manager = ProcessManager() + self.logger = logging.getLogger(__name__) + + def wait_for_server(self, port: int, timeout: int = 300) -> bool: + start_time = time.perf_counter() + + while time.perf_counter() - start_time < timeout: + try: + response = requests.get(f"http://localhost:{port}/health") + if response.status_code == 200: + self.logger.debug(f"Server ready on port {port}") + return True + except requests.RequestException: + time.sleep(2) + return False + + def run_task(self, config: TaskConfig) -> TaskResult: + start_time = time.perf_counter() + client_output = [] + + try: + if not config.server_type: + config.server_type = detect_server_type(config.server_cmd) + + server_config = SERVER_DEFAULTS.get(config.server_type) + if not server_config: + raise ValueError(f"Unknown server type: {config.server_type}") + + port = extract_port_from_command(config.server_cmd, config.server_type) + + self.process_manager.cleanup(server_config.process_names) + + self.logger.debug(f"Starting server: {config.name}") + self.process_manager.server_process, _, server_threads = ( + self.process_manager.start_process(config.server_cmd, "SERVER") + ) + + if not self.wait_for_server(port): + raise TimeoutError("Server startup timeout") + + time.sleep(10) + + self.logger.debug("Starting client") + self.process_manager.client_process, output_queue, client_threads = ( + self.process_manager.start_process(config.client_cmd, "CLIENT") + ) + + returncode = self.process_manager.client_process.wait() + + while True: + try: + line = output_queue.get_nowait() + client_output.append(line) + except queue.Empty: + break + + if returncode != 0: + raise RuntimeError(f"Client failed with code {returncode}") + + # Parse and format the output + full_output = "\n".join(client_output) + formatted_output = parse_key_info(full_output) + + return TaskResult( + name=config.name, + success=True, + output=formatted_output, + runtime=time.perf_counter() - start_time, + timestamp=datetime.now().isoformat(), + ) + + except Exception as e: + return TaskResult( + name=config.name, + success=False, + output=str(e), + runtime=time.perf_counter() - start_time, + timestamp=datetime.now().isoformat(), + ) + + finally: + if config.server_type in SERVER_DEFAULTS: + self.process_manager.cleanup( + SERVER_DEFAULTS[config.server_type].process_names + ) + time.sleep(10) + + +def load_config(config_path: str) -> List[TaskConfig]: + with open(config_path, "r") as f: + config_data = yaml.safe_load(f) + + configs = [] + for idx, entry in enumerate(config_data.get("tasks", [])): + if not isinstance(entry, dict): + raise ValueError(f"Invalid entry at index {idx}") + + config = TaskConfig( + server_cmd=entry.get("server_cmd"), + client_cmd=entry.get("client_cmd"), + name=entry.get("name", f"task-{idx+1}"), + server_type=entry.get("server_type"), + ) + + if not config.server_cmd or not config.client_cmd: + raise ValueError(f"Missing commands in {config.name}") + + configs.append(config) + + return configs + + +def setup_logging(debug: bool = False): + level = logging.DEBUG if debug else logging.INFO + logging.basicConfig( + level=level, + format="%(asctime)s - %(levelname)s - %(message)s", + handlers=[logging.StreamHandler(), logging.FileHandler("experiment.log")], + ) + + +def format_results(results: List[TaskResult]) -> str: + """Format experiment results in Markdown for GitHub step summary.""" + output = ["# Experiment Results\n"] + + for result in results: + output.append(f"## {result.name}") + output.append(f"**Status**: {'✅ Success' if result.success else '❌ Failed'}") + output.append(f"**Runtime**: {result.runtime:.2f} seconds") + output.append(f"**Timestamp**: {result.timestamp}") + output.append("\n**Output**:\n```") + output.append(result.output) + output.append("```\n") + + return "\n".join(output) + + +def get_bool_env_var(name: str, default: str = "false") -> bool: + value = os.getenv(name, default) + return value.lower() in ("true", "1") + + +def write_in_github_step_summary(results: List[TaskResult]): + """Write formatted results to GitHub step summary.""" + if not os.environ.get("GITHUB_STEP_SUMMARY"): + logging.warning("GITHUB_STEP_SUMMARY environment variable not set") + return + + formatted_content = format_results(results) + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: + f.write(formatted_content) + + +def main(): + parser = argparse.ArgumentParser(description="Experiment Runner") + parser.add_argument( + "--config", type=str, required=True, help="Path to YAML config file" + ) + parser.add_argument("--debug", action="store_true", help="Enable debug output") + args = parser.parse_args() + + setup_logging(args.debug) + logger = logging.getLogger(__name__) + results = [] + + try: + configs = load_config(args.config) + runner = ExperimentRunner() + + for config in configs: + logger.info(f"Running {config.name}") + result = runner.run_task(config) + results.append(result) + + if get_bool_env_var("SGLANG_IS_IN_CI"): + write_in_github_step_summary(results) + except Exception as e: + logger.error(f"Error: {e}") + raise + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/external_models/custom_qwen2_vl.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/external_models/custom_qwen2_vl.py new file mode 100644 index 00000000..5e1b9649 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/external_models/custom_qwen2_vl.py @@ -0,0 +1,21 @@ +from sglang.srt.models.qwen2_vl import ( + Qwen2VLForConditionalGeneration as OriginalQwen2VLForConditionalGeneration, +) +from sglang.srt.multimodal.processors.qwen_vl import QwenVLImageProcessor + + +class Qwen2VLForConditionalGeneration(OriginalQwen2VLForConditionalGeneration): + def __init__(self, config, quant_config, prefix: str = "") -> None: + super().__init__(config, quant_config, prefix) + print("init custom model:", self.__class__.__name__) + + +class CustomProcessor(QwenVLImageProcessor): + models = [Qwen2VLForConditionalGeneration] + + def __init__(self, hf_config, server_args, _processor, *args, **kwargs): + super().__init__(hf_config, server_args, _processor, *args, **kwargs) + print("init custom processor:", self.__class__.__name__) + + +EntryClass = Qwen2VLForConditionalGeneration diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/hicache/test_hicache_storage.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/hicache/test_hicache_storage.py new file mode 100644 index 00000000..b4e90532 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/hicache/test_hicache_storage.py @@ -0,0 +1,59 @@ +import time +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import is_hip, kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +_is_hip = is_hip() + + +class TestHiCache(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--enable-hierarchical-cache", + "--mem-fraction-static", + 0.7, + "--hicache-size", + 100 if not _is_hip else 200, + "--page-size", + "64", + "--hicache-storage-backend", + "file", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + time.sleep(5) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/hicache/test_hicache_storage_3fs_backend.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/hicache/test_hicache_storage_3fs_backend.py new file mode 100644 index 00000000..c9177288 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/hicache/test_hicache_storage_3fs_backend.py @@ -0,0 +1,88 @@ +""" +Benchmark tests for HiCache Storage with 3FS backend. +Usage: + python3 -m pytest test/srt/hicache/test_hicache_storage_3fs_backend.py -v +""" + +import json +import os +import unittest + +from test_hicache_storage_file_backend import HiCacheStorageBaseMixin + +from sglang.test.test_utils import CustomTestCase + + +class HiCacheStorage3FSBackendBaseMixin(HiCacheStorageBaseMixin): + """Base mixin class with common setup and utilities""" + + @classmethod + def _get_additional_server_args_and_env(cls): + """Get additional server arguments specific to configuration - override in subclasses""" + # Create a temporary JSON config file for HF3FS + hf3fs_config = { + "file_path_prefix": os.path.join(cls.temp_dir, "hicache"), + "file_size": 1024 * 1024 * 1024 * 2, + "numjobs": 2, + "entries": 8, + "use_mock_hf3fs_client": True, + "hicache_storage_pass_prefix_keys": True, + } + + # Write config to temporary file + config_file = os.path.join(cls.temp_dir, "hf3fs_config.json") + with open(config_file, "w") as f: + json.dump(hf3fs_config, f, indent=2) + + server_args = { + "--tp-size": 1, + "--hicache-ratio": 1.2, + "--hicache-storage-backend": "hf3fs", + "--hicache-storage-backend-extra-config": json.dumps(hf3fs_config), + } + + # Set the environment variable to point to our config file + env_vars = { + "SGLANG_HICACHE_HF3FS_CONFIG_PATH": config_file, + } + + return server_args, env_vars + + +class TestHf3fsBackendLayerFirstLayout( + HiCacheStorage3FSBackendBaseMixin, CustomTestCase +): + """Layer first layout tests for HiCache-Hf3fs backend""" + + @classmethod + def _get_additional_server_args_and_env(cls): + """Get additional server arguments specific to configuration - override in subclasses""" + server_args, env_vars = super()._get_additional_server_args_and_env() + server_args["--hicache-mem-layout"] = "layer_first" + server_args["--hicache-io-backend"] = "direct" + server_args["--tp-size"] = 2 + return server_args, env_vars + + +class TestHf3fsBackendAccuracy(HiCacheStorage3FSBackendBaseMixin, CustomTestCase): + """Accuracy tests for HiCache-Hf3fs backend""" + + @classmethod + def _get_additional_server_args_and_env(cls): + """Get additional server arguments specific to configuration - override in subclasses""" + server_args, env_vars = super()._get_additional_server_args_and_env() + server_args["--hicache-ratio"] = 1.5 + server_args["--tp-size"] = 2 + server_args["--hicache-mem-layout"] = "page_first_direct" + server_args["--hicache-io-backend"] = "direct" + return server_args, env_vars + + def test_eval_accuracy(self): + """Test eval accuracy with cache persistence across cache flushes""" + from test_hicache_storage_file_backend import run_eval_accuracy_test + + run_eval_accuracy_test(self) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/hicache/test_hicache_storage_file_backend.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/hicache/test_hicache_storage_file_backend.py new file mode 100644 index 00000000..382db07b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/hicache/test_hicache_storage_file_backend.py @@ -0,0 +1,333 @@ +""" +E2E tests for HiCache Storage functionality. +Usage: + python3 -m pytest test/srt/hicache/test_hicache_storage_e2e.py -v +""" + +import json +import os +import random +import tempfile +import time +import unittest +from types import SimpleNamespace +from typing import Dict +from urllib.parse import urlparse + +import requests + +from sglang.bench_serving import get_tokenizer +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) + + +class HiCacheStorageBaseMixin: + """Base mixin class with common setup and utilities""" + + @classmethod + def setUpClass(cls): + """Set up test environment and launch server once for all tests""" + cls.temp_dir = tempfile.mkdtemp() + cls.model = cls._get_model_name() + cls.base_url = DEFAULT_URL_FOR_TEST + + parsed_url = urlparse(cls.base_url) + cls.base_host = parsed_url.hostname + cls.base_port = str(parsed_url.port) + + # Prepare tokenizer for prompt generation + cls.tokenizer = get_tokenizer(cls.model) + + # Launch server with HiCache enabled and cache report + cls.process = cls._launch_server_with_hicache() + cls._wait_for_server_ready() + + print(f"Test server launched successfully at {cls.base_url}") + print(f"Cache directory: {cls.temp_dir}") + + @classmethod + def tearDownClass(cls): + """Clean up test environment""" + kill_process_tree(cls.process.pid) + + import shutil + + shutil.rmtree(cls.temp_dir, ignore_errors=True) + + @classmethod + def _get_model_name(cls): + """Get model name for the test configuration - override in subclasses""" + return DEFAULT_MODEL_NAME_FOR_TEST + + @classmethod + def _get_base_server_args(cls): + """Get base server arguments - can be extended in subclasses""" + extra_config = { + "hicache_storage_pass_prefix_keys": True, + } + return { + "--enable-hierarchical-cache": True, + "--mem-fraction-static": 0.6, + "--hicache-ratio": 1.2, + "--page-size": 64, + "--enable-cache-report": True, + "--hicache-storage-prefetch-policy": "wait_complete", + "--hicache-storage-backend": "file", + "--hicache-storage-backend-extra-config": json.dumps(extra_config), + } + + @classmethod + def _get_additional_server_args_and_env(cls): + """Get additional server arguments specific to configuration - override in subclasses""" + return {}, {"SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR": cls.temp_dir} + + @classmethod + def _launch_server_with_hicache(cls): + """Launch server with HiCache enabled""" + + additional_server_args, env_vars = cls._get_additional_server_args_and_env() + env_vars["SGLANG_ENABLE_DETERMINISTIC_INFERENCE"] = "1" + server_args = cls._get_base_server_args() + if additional_server_args: + server_args.update(additional_server_args) + + final_server_args = [] + for k, v in server_args.items(): + if isinstance(v, bool): + final_server_args.append(str(k)) + else: + final_server_args.append(str(k)) + final_server_args.append(str(v)) + + print(f"final_server_args: {final_server_args}") + + env_vars = { + **os.environ, + **env_vars, + } + + return popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=final_server_args, + env=env_vars, + ) + + @classmethod + def _wait_for_server_ready(cls, timeout: int = 60) -> bool: + """Wait for server to be ready""" + start_time = time.time() + while time.time() - start_time < timeout: + try: + response = requests.get(f"{cls.base_url}/health", timeout=5) + if response.status_code == 200: + return True + except requests.RequestException: + pass + time.sleep(2) + raise TimeoutError("Server failed to start within timeout") + + def send_request( + self, prompt: str, max_tokens: int = 100, temperature: float = 0.0 + ) -> Dict: + """Send a generate request and return response""" + response = requests.post( + f"{self.base_url}/generate", + json={ + "text": prompt, + "sampling_params": { + "temperature": temperature, + "max_new_tokens": max_tokens, + "ignore_eos": True, + }, + }, + timeout=60, + ) + + self.assertEqual( + response.status_code, + 200, + f"Request failed: {response.status_code} - {response.text}", + ) + return response.json() + + def get_cached_tokens(self, response_json: Dict) -> int: + """Extract cached tokens count from /generate response""" + meta = response_json.get("meta_info", {}) + return int(meta.get("cached_tokens", 0)) + + def flush_cache(self) -> bool: + """Flush device cache to force remote storage access""" + try: + response = requests.post(f"{self.base_url}/flush_cache", timeout=10) + return response.status_code == 200 + except requests.RequestException: + return False + + def gen_prompt(self, token_num: int) -> str: + """Generate a random prompt of specified token length using tokenizer vocabulary.""" + all_available_tokens = list(self.tokenizer.get_vocab().values()) + selected_tokens = random.choices(all_available_tokens, k=token_num) + return self.tokenizer.decode(selected_tokens) + + def trigger_offloading_and_flush(self): + """Helper method to trigger offloading and flush cache""" + # Trigger offloading + self.send_request(self.gen_prompt(1), max_tokens=150) + + # Flush device cache to force remote storage access + time.sleep(2) + self.assertTrue(self.flush_cache(), "Cache flush should succeed") + + def test_basic_backup_and_prefetch(self): + """Test storage and retrieval of large context through remote cache""" + print("\n=== Testing Large Context Cache Storage & Retrieval ===") + + # Generate substantial context that will be cached + base_prompt = self.gen_prompt(768) + + # First request - populate cache + print("Step 1: Populating cache with large context...") + response1 = self.send_request(base_prompt, max_tokens=150) + self.assertIsNotNone(response1) + + # Flush device cache to force remote storage access + self.trigger_offloading_and_flush() + + # Second request with extended prompt - should hit remote cache + print("Step 2: Testing cache hit from remote storage...") + + start_time = time.time() + response2 = self.send_request(base_prompt, max_tokens=150) + retrieval_time = time.time() - start_time + + cached_tokens = self.get_cached_tokens(response2) + print( + f"Remote cache retrieval time: {retrieval_time:.3f}s, cached_tokens={cached_tokens}" + ) + + # Assert cached tokens indicate a remote hit + self.assertGreater( + cached_tokens, 700, "Expected significant cached tokens for remote hit" + ) + + +@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") +class TestHiCacheStoragePageFirstLayout(HiCacheStorageBaseMixin, CustomTestCase): + """Page first layout tests for HiCache Storage functionality""" + + @classmethod + def _get_additional_server_args_and_env(cls): + """Get additional server arguments specific to configuration - override in subclasses""" + server_args = {"--hicache-mem-layout": "page_first"} + return server_args, {} + + +@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") +class TestHiCacheStorageMLA(HiCacheStorageBaseMixin, CustomTestCase): + """MLA Model tests for HiCache Storage functionality""" + + @classmethod + def _get_model_name(cls): + """Use MLA model for testing""" + return DEFAULT_MLA_MODEL_NAME_FOR_TEST + + @classmethod + def _get_additional_server_args_and_env(cls): + """Get additional server arguments specific to configuration - override in subclasses""" + server_args = {"--tp-size": 2} + return server_args, {} + + +class TestHiCacheStoragePageFirstDirectIO(HiCacheStorageBaseMixin, CustomTestCase): + """Page first direct tests for HiCache Storage functionality""" + + @classmethod + def _get_additional_server_args_and_env(cls): + """Get additional server arguments specific to configuration - override in subclasses""" + server_args = { + "--hicache-mem-layout": "page_first_direct", + "--hicache-io-backend": "direct", + "--tp-size": 2, + } + return server_args, {} + + +class TestHiCacheStorageAccuracy(HiCacheStorageBaseMixin, CustomTestCase): + """Accuracy tests for HiCache Storage functionality""" + + @classmethod + def _get_additional_server_args_and_env(cls): + """Get additional server arguments specific to configuration - override in subclasses""" + server_args = { + "--tp-size": 2, + "--hicache-ratio": 1.5, + } + + return server_args, {} + + def test_eval_accuracy(self): + """Test eval accuracy with cache persistence across cache flushes""" + run_eval_accuracy_test(self) + + +def run_eval_accuracy_test(test_instance, accuracy_threshold: float = 0.03): + """Generic eval accuracy test with configurable accuracy threshold + + Args: + test_instance: The test class instance that provides base_host, base_port, flush_cache, and assert methods + """ + print("\n=== Testing Eval Accuracy with Cache Persistence ===") + + # First evaluation - populate cache + print("Phase 1: Running initial GSM8K evaluation to populate cache...") + args_initial = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=50, + max_new_tokens=512, + parallel=10, + host=f"http://{test_instance.base_host}", + port=int(test_instance.base_port), + ) + metrics_initial = run_eval_few_shot_gsm8k(args_initial) + + # Flush cache to force remote storage access + print("Phase 2: Flushing device cache...") + test_instance.assertTrue(test_instance.flush_cache(), "Cache flush should succeed") + time.sleep(2) + + # Second evaluation - should use remote cache + print("Phase 3: Running second GSM8K evaluation using remote cache...") + metrics_cached = run_eval_few_shot_gsm8k(args_initial) + + # Verify accuracy consistency + accuracy_diff = abs(metrics_initial["accuracy"] - metrics_cached["accuracy"]) + print(f"Accuracy difference: {accuracy_diff:.4f}") + + # Assertions + test_instance.assertGreater( + metrics_initial["accuracy"], 0.6, "Initial accuracy should be reasonable" + ) + test_instance.assertGreater( + metrics_cached["accuracy"], 0.6, "Cached accuracy should be reasonable" + ) + test_instance.assertLess( + accuracy_diff, + accuracy_threshold, + "Accuracy should be consistent between cache states", + ) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/hicache/test_hicache_storage_mooncake_backend.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/hicache/test_hicache_storage_mooncake_backend.py new file mode 100644 index 00000000..7e0c8313 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/hicache/test_hicache_storage_mooncake_backend.py @@ -0,0 +1,283 @@ +""" +Benchmark tests for HiCache Storage with Mooncake backend. +Usage: + python3.10 -m pytest test/srt/hicache/test_hicache_storage_mooncake_backend.py -v +""" + +import os +import subprocess +import time +import unittest + +import requests +from test_hicache_storage_file_backend import HiCacheStorageBaseMixin + +from sglang.test.test_utils import ( + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + CustomTestCase, + find_available_port, + is_in_ci, +) + + +class HiCacheStorageMooncakeBackendBaseMixin(HiCacheStorageBaseMixin): + """Base mixin class with common setup and utilities""" + + # Default port ranges for Mooncake services - can be overridden in subclasses + mooncake_master_port_base = 50051 + mooncake_metadata_port_base = 8080 + + @classmethod + def setUpClass(cls): + """Set up test environment and launch Mooncake services before server setup""" + # Find available ports for Mooncake services to avoid conflicts + cls.mooncake_master_port = find_available_port( + HiCacheStorageMooncakeBackendBaseMixin.mooncake_master_port_base + ) + cls.mooncake_metadata_port = find_available_port( + HiCacheStorageMooncakeBackendBaseMixin.mooncake_metadata_port_base + ) + + # Start Mooncake services first + cls._start_mooncake_services() + + # Call parent setup + super().setUpClass() + + @classmethod + def tearDownClass(cls): + """Clean up Mooncake services after server teardown""" + # Call parent teardown first + super().tearDownClass() + + # Stop Mooncake services + cls._stop_mooncake_services() + + @classmethod + def _start_mooncake_services(cls): + """Start Mooncake metadata and master services with configurable ports and readiness detection""" + print("Starting Mooncake services...") + print( + f"Using master port: {cls.mooncake_master_port}, metadata port: {cls.mooncake_metadata_port}" + ) + + # Start metadata service with configurable port + try: + # Start metadata server with port configuration + cls.metadata_service_process = subprocess.Popen( + [ + "python3", + "-m", + "mooncake.http_metadata_server", + "--port", + str(cls.mooncake_metadata_port), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=os.setsid, # Create new process group + ) + print( + f"Mooncake metadata service started on port {cls.mooncake_metadata_port}" + ) + except (FileNotFoundError, subprocess.SubprocessError) as e: + print(f"Warning: Could not start Mooncake metadata service: {e}") + cls.metadata_service_process = None + + # Start master service with configurable port + try: + # Start master server with port configuration + cls.master_service_process = subprocess.Popen( + ["mooncake_master", "--port", str(cls.mooncake_master_port)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=os.setsid, # Create new process group + ) + print(f"Mooncake master service started on port {cls.mooncake_master_port}") + except (FileNotFoundError, subprocess.SubprocessError) as e: + print(f"Warning: Could not start Mooncake master service: {e}") + cls.master_service_process = None + + # Wait for services to be ready instead of fixed sleep + cls._wait_for_mooncake_services_ready() + + @classmethod + def _wait_for_mooncake_services_ready(cls, timeout: int = 30) -> bool: + """Wait for Mooncake services to be ready by checking their endpoints""" + print("Waiting for Mooncake services to be ready...") + + start_time = time.time() + services_ready = False + + while time.time() - start_time < timeout: + try: + # Check metadata service + metadata_ready = False + if ( + cls.metadata_service_process + and cls.metadata_service_process.poll() is None + ): + try: + # Try to connect to the metadata service + metadata_url = ( + f"http://127.0.0.1:{cls.mooncake_metadata_port}/metadata" + ) + response = requests.get(metadata_url, timeout=2) + if response.status_code == 200: + metadata_ready = True + print("Mooncake metadata service is ready") + except (requests.RequestException, ConnectionError): + # Service might not be fully started yet + pass + + # Check master service (if it has a health endpoint) + master_ready = False + if ( + cls.master_service_process + and cls.master_service_process.poll() is None + ): + # For now, we'll assume master service is ready if process is running + # and it's been a few seconds since startup + if ( + time.time() - start_time > 5 + ): # Give master service time to initialize + master_ready = True + print("Mooncake master service is ready") + + # Both services should be ready + if metadata_ready and master_ready: + services_ready = True + print("All Mooncake services are ready") + break + + except Exception as e: + print(f"Error checking service readiness: {e}") + + time.sleep(2) + + if not services_ready: + print( + "Warning: Mooncake services may not be fully ready, continuing anyway..." + ) + + return services_ready + + @classmethod + def _stop_mooncake_services(cls): + """Stop Mooncake services""" + print("Stopping Mooncake services...") + + # Stop metadata service + if hasattr(cls, "metadata_service_process") and cls.metadata_service_process: + try: + os.killpg(os.getpgid(cls.metadata_service_process.pid), 9) + cls.metadata_service_process.wait(timeout=5) + print("Mooncake metadata service stopped") + except (ProcessLookupError, subprocess.TimeoutExpired, OSError) as e: + print(f"Warning: Could not stop Mooncake metadata service: {e}") + + # Stop master service + if hasattr(cls, "master_service_process") and cls.master_service_process: + try: + os.killpg(os.getpgid(cls.master_service_process.pid), 9) + cls.master_service_process.wait(timeout=5) + print("Mooncake master service stopped") + except (ProcessLookupError, subprocess.TimeoutExpired, OSError) as e: + print(f"Warning: Could not stop Mooncake master service: {e}") + + @classmethod + def _get_additional_server_args_and_env(cls): + """Get additional server arguments specific to configuration - override in subclasses""" + + server_args = { + "--tp-size": 2, + "--hicache-ratio": 2, + "--hicache-storage-backend": "mooncake", + } + + # Set the environment variables for Mooncake using dynamic ports + env_vars = { + "MOONCAKE_MASTER": f"127.0.0.1:{cls.mooncake_master_port}", + "MOONCAKE_PROTOCOL": "tcp", + "MC_MS_AUTO_DISC": "0", + "MOONCAKE_DEVICE": "", + "MOONCAKE_TE_META_DATA_SERVER": f"http://127.0.0.1:{cls.mooncake_metadata_port}/metadata", + "MOONCAKE_GLOBAL_SEGMENT_SIZE": "4294967296", # 4 GiB + } + + return server_args, env_vars + + +''' +# Same as #10131, layer first layout test TODO(mateng): will make it work +class TestMooncakeBackendLayerFirstLayout( + HiCacheStorageMooncakeBackendBaseMixin, CustomTestCase +): + """Layer first layout tests for HiCache-Mooncake backend""" + + @classmethod + def _get_additional_server_args_and_env(cls): + """Get additional server arguments specific to configuration - override in subclasses""" + server_args, env_vars = super()._get_additional_server_args_and_env() + server_args["--hicache-mem-layout"] = "layer_first" + server_args["--hicache-io-backend"] = "direct" + return server_args, env_vars +''' + + +@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") +class TestMooncakeBackendPageFirstLayout( + HiCacheStorageMooncakeBackendBaseMixin, CustomTestCase +): + """Page first layout tests for HiCache-Mooncake backend""" + + @classmethod + def _get_additional_server_args_and_env(cls): + """Get additional server arguments specific to configuration - override in subclasses""" + server_args, env_vars = super()._get_additional_server_args_and_env() + server_args["--hicache-mem-layout"] = "page_first" + return server_args, env_vars + + +class TestMooncakeBackendMLAModel( + HiCacheStorageMooncakeBackendBaseMixin, CustomTestCase +): + """MLA Model tests for HiCache-Mooncake backend""" + + @classmethod + def _get_model_name(cls): + """Use MLA model for testing""" + return DEFAULT_MLA_MODEL_NAME_FOR_TEST + + @classmethod + def _get_additional_server_args_and_env(cls): + """Get additional server arguments specific to configuration - override in subclasses""" + server_args, env_vars = super()._get_additional_server_args_and_env() + server_args["--hicache-mem-layout"] = "page_first" + server_args["--tp-size"] = 2 + return server_args, env_vars + + +class TestMooncakeBackendAccuracy( + HiCacheStorageMooncakeBackendBaseMixin, CustomTestCase +): + """Accuracy tests for HiCache-Mooncake backend""" + + @classmethod + def _get_additional_server_args_and_env(cls): + """Get additional server arguments specific to configuration - override in subclasses""" + server_args, env_vars = super()._get_additional_server_args_and_env() + server_args["--hicache-ratio"] = 1.5 + server_args["--tp-size"] = 2 + server_args["--hicache-mem-layout"] = "page_first_direct" + server_args["--hicache-io-backend"] = "direct" + return server_args, env_vars + + def test_eval_accuracy(self): + """Test eval accuracy with cache persistence across cache flushes""" + from test_hicache_storage_file_backend import run_eval_accuracy_test + + run_eval_accuracy_test(self) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/hicache/test_hicache_variants.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/hicache/test_hicache_variants.py new file mode 100644 index 00000000..98706878 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/hicache/test_hicache_variants.py @@ -0,0 +1,179 @@ +""" +Consolidated HiCache variant tests. +Tests HiCache with different configurations: standard, MLA, EAGLE, and page size variants. +""" + +import unittest +from types import SimpleNamespace + +import requests + +from sglang.bench_serving import get_tokenizer +from sglang.srt.utils import is_hip, kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST_EAGLE3, + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +_is_hip = is_hip() + + +class HiCacheEvalMixin: + """Mixin class containing common HiCache evaluation test methods""" + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], self.expected_mmlu_score) + + +class HiCacheMGSMEvalMixin: + """Mixin for tests that also run MGSM evaluation""" + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.8) + + +class HiCacheBaseServer(CustomTestCase): + """Base class for HiCache tests with configurable server setup""" + + model_name = DEFAULT_MODEL_NAME_FOR_TEST + hicache_args = [] + expected_mmlu_score = 0.65 + + @classmethod + def setUpClass(cls): + cls.model = cls.model_name + cls.base_url = DEFAULT_URL_FOR_TEST + + # Setup tokenizer if needed by subclass + if hasattr(cls, "needs_tokenizer") and cls.needs_tokenizer: + cls.tokenizer = get_tokenizer(cls.model) + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=cls.hicache_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +class TestHiCacheStandard(HiCacheBaseServer, HiCacheEvalMixin): + """Standard HiCache configuration tests""" + + model_name = DEFAULT_MODEL_NAME_FOR_TEST + hicache_args = [ + "--enable-hierarchical-cache", + "--mem-fraction-static", + 0.7, + "--hicache-size", + 100 if not _is_hip else 200, + ] + expected_mmlu_score = 0.65 + + +class TestHiCacheMLA(HiCacheBaseServer, HiCacheEvalMixin, HiCacheMGSMEvalMixin): + """HiCache with MLA model tests""" + + model_name = DEFAULT_MLA_MODEL_NAME_FOR_TEST + hicache_args = [ + "--trust-remote-code", + "--enable-hierarchical-cache", + ] + (["--hicache-size", 200] if _is_hip else ["--hicache-ratio", 2]) + expected_mmlu_score = 0.5 + + +class TestHiCacheEagle(HiCacheBaseServer, HiCacheEvalMixin): + """HiCache with EAGLE speculative decoding tests""" + + model_name = DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST_EAGLE3 + needs_tokenizer = True + hicache_args = [ + "--enable-hierarchical-cache", + "--hicache-ratio", + 1.2, + "--mem-fraction-static", + 0.7, + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3, + "--speculative-num-steps", + 2, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 3, + "--dtype", + "float16", + "--chunked-prefill-size", + 1024, + ] + expected_mmlu_score = 0.72 + + def test_mmlu(self): + """Override to add EAGLE-specific assertions""" + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], self.expected_mmlu_score) + + # EAGLE-specific check + server_info = requests.get(self.base_url + "/get_server_info") + print(f"{server_info=}") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, 2.26) + + +class TestHiCachePage(HiCacheBaseServer, HiCacheEvalMixin): + """HiCache with custom page size tests""" + + model_name = DEFAULT_MODEL_NAME_FOR_TEST + hicache_args = [ + "--enable-hierarchical-cache", + "--page-size", + 32, + "--hicache-write-policy", + "write_back", + ] + expected_mmlu_score = 0.65 + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/kv_cache_scales_llama3_1_8b.json b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/kv_cache_scales_llama3_1_8b.json new file mode 100644 index 00000000..3e890e50 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/kv_cache_scales_llama3_1_8b.json @@ -0,0 +1,42 @@ +{ + "model_type": "llama", + "kv_cache": { + "dtype": "float8_e4m3fn", + "scaling_factor": { + "0": { + "0": 1, + "1": 1, + "2": 1, + "3": 1, + "4": 1, + "5": 1, + "6": 1, + "7": 1, + "8": 1, + "9": 1, + "10": 1, + "11": 1, + "12": 1, + "13": 1, + "14": 1, + "15": 1, + "16": 1, + "17": 1, + "18": 1, + "19": 1, + "20": 1, + "21": 1, + "22": 1, + "23": 1, + "24": 1, + "25": 1, + "26": 1, + "27": 1, + "28": 1, + "29": 1, + "30": 1, + "31": 1 + } + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/kv_cache_scales_llama3_8b.json b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/kv_cache_scales_llama3_8b.json new file mode 100644 index 00000000..466b0d01 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/kv_cache_scales_llama3_8b.json @@ -0,0 +1,42 @@ +{ + "model_type": "llama", + "kv_cache": { + "dtype": "float8_e4m3fn", + "scaling_factor": { + "0": { + "0": 0.0408, + "1": 0.0503, + "2": 0.0667, + "3": 0.0909, + "4": 0.1135, + "5": 0.127, + "6": 0.1768, + "7": 0.1488, + "8": 0.1135, + "9": 0.1203, + "10": 0.1013, + "11": 0.0842, + "12": 0.1231, + "13": 0.1096, + "14": 0.1221, + "15": 0.1013, + "16": 0.1067, + "17": 0.0952, + "18": 0.0899, + "19": 0.097, + "20": 0.087, + "21": 0.0994, + "22": 0.0904, + "23": 0.1013, + "24": 0.1019, + "25": 0.1053, + "26": 0.1, + "27": 0.0894, + "28": 0.1013, + "29": 0.1488, + "30": 0.0766, + "31": 0.0821 + } + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/kv_cache_scales_qwen2_1_5b.json b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/kv_cache_scales_qwen2_1_5b.json new file mode 100644 index 00000000..98474750 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/kv_cache_scales_qwen2_1_5b.json @@ -0,0 +1,38 @@ +{ + "model_type": "qwen", + "kv_cache": { + "dtype": "float8_e4m3fn", + "scaling_factor": { + "0": { + "0": 0.9846, + "1": 0.0645, + "2": 0.0731, + "3": 0.0800, + "4": 0.0748, + "5": 0.0780, + "6": 0.0702, + "7": 0.0894, + "8": 0.0410, + "9": 0.0758, + "10": 0.0556, + "11": 0.0731, + "12": 0.0899, + "13": 0.0780, + "14": 0.1441, + "15": 0.0914, + "16": 0.5614, + "17": 0.1067, + "18": 0.0537, + "19": 0.0658, + "20": 0.0523, + "21": 0.0533, + "22": 0.0699, + "23": 0.0635, + "24": 0.0588, + "25": 0.0884, + "26": 0.0947, + "27": 0.1032 + } + } + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/layers/attention/mamba/test_causal_conv1d.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/layers/attention/mamba/test_causal_conv1d.py new file mode 100644 index 00000000..dd1a9a25 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/layers/attention/mamba/test_causal_conv1d.py @@ -0,0 +1,379 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/main/tests/kernels/mamba/test_causal_conv1d.py + + +from typing import Optional + +import pytest +import torch +import torch.nn.functional as F +from einops import rearrange + +from sglang.srt.layers.attention.mamba.causal_conv1d_triton import ( + PAD_SLOT_ID, + causal_conv1d_fn, + causal_conv1d_update, +) + + +def causal_conv1d_ref( + x: torch.Tensor, + weight: torch.Tensor, + bias: Optional[torch.Tensor] = None, + initial_states: Optional[torch.Tensor] = None, + return_final_states: bool = False, + final_states_out: Optional[torch.Tensor] = None, + activation: Optional[str] = "silu", +): + """ + x: (batch, dim, seqlen) + weight: (dim, width) + bias: (dim,) + initial_states: (batch, dim, width - 1) + final_states_out: (batch, dim, width - 1) + + out: (batch, dim, seqlen) + """ + if activation not in [None, "silu", "swish"]: + raise NotImplementedError("activation must be None, silu, or swish") + dtype_in = x.dtype + x = x.to(weight.dtype) + seqlen = x.shape[-1] + dim, width = weight.shape + if initial_states is None: + out = F.conv1d(x, weight.unsqueeze(1), bias, padding=width - 1, groups=dim) + else: + x = torch.cat([initial_states, x], dim=-1) + out = F.conv1d(x, weight.unsqueeze(1), bias, padding=0, groups=dim) + out = out[..., :seqlen] + if return_final_states: + final_states = F.pad(x, (width - 1 - x.shape[-1], 0)).to( + dtype_in + ) # (batch, dim, width - 1) + if final_states_out is not None: + final_states_out.copy_(final_states) + else: + final_states_out = final_states + out = (out if activation is None else F.silu(out)).to(dtype=dtype_in) + return (out, None) if not return_final_states else (out, final_states_out) + + +def causal_conv1d_update_ref( + x, conv_state, weight, bias=None, activation=None, cache_seqlens=None +): + """ + x: (batch, dim) or (batch, dim, seqlen) + conv_state: (batch, dim, state_len), where state_len >= width - 1 + weight: (dim, width) + bias: (dim,) + cache_seqlens: (batch,), dtype int32. + If not None, the conv_state is treated as a circular buffer. + The conv_state will be updated by copying x to the + conv_state starting at the index + @cache_seqlens % state_len before performing the convolution. + + out: (batch, dim) or (batch, dim, seqlen) + """ + if activation not in [None, "silu", "swish"]: + raise NotImplementedError("activation must be None, silu, or swish") + dtype_in = x.dtype + unsqueeze = x.dim() == 2 + if unsqueeze: + x = x.unsqueeze(-1) + batch, dim, seqlen = x.shape + width = weight.shape[1] + state_len = conv_state.shape[-1] + assert conv_state.shape == (batch, dim, state_len) + assert weight.shape == (dim, width) + if cache_seqlens is None: + x_new = torch.cat([conv_state, x], dim=-1).to( + weight.dtype + ) # (batch, dim, state_len + seqlen) + conv_state.copy_(x_new[:, :, -state_len:]) + else: + width_idx = torch.arange( + -(width - 1), 0, dtype=torch.long, device=x.device + ).unsqueeze(0) + cache_seqlens.unsqueeze(1) + width_idx = ( + torch.remainder(width_idx, state_len).unsqueeze(1).expand(-1, dim, -1) + ) + x_new = torch.cat([conv_state.gather(2, width_idx), x], dim=-1).to(weight.dtype) + copy_idx = torch.arange(seqlen, dtype=torch.long, device=x.device).unsqueeze( + 0 + ) + cache_seqlens.unsqueeze(1) + copy_idx = torch.remainder(copy_idx, state_len).unsqueeze(1).expand(-1, dim, -1) + conv_state.scatter_(2, copy_idx, x) + out = F.conv1d(x_new, weight.unsqueeze(1), bias, padding=0, groups=dim)[ + :, :, -seqlen: + ] + if unsqueeze: + out = out.squeeze(-1) + return (out if activation is None else F.silu(out)).to(dtype=dtype_in) + + +@pytest.mark.parametrize("itype", [torch.bfloat16, torch.float]) +@pytest.mark.parametrize("silu_activation", [True]) +@pytest.mark.parametrize("has_bias", [True]) +def causal_conv1d_opcheck_fn( + x: torch.Tensor, + weight: torch.Tensor, + bias: Optional[torch.Tensor] = None, + cu_seq_len: Optional[torch.Tensor] = None, + cache_indices: Optional[torch.Tensor] = None, + has_initial_state: Optional[torch.Tensor] = None, + conv_states: Optional[torch.Tensor] = None, + activation: Optional[str] = "silu", + pad_slot_id: int = PAD_SLOT_ID, +): + """ + x: (batch, dim, seqlen) + weight: (dim, width) + bias: (dim,) + seq_idx: (batch, seqlen) + initial_states: (batch, dim, width - 1) + final_states_out: (batch, dim, width - 1), to be written to + activation: either None or "silu" or "swish" + + out: (batch, dim, seqlen) + """ + if activation not in [None, "silu", "swish"]: + raise NotImplementedError("activation must be None, silu, or swish") + if x.stride(-1) != 1: + x = x.contiguous() + bias = bias.contiguous() if bias is not None else None + + +@pytest.mark.parametrize("itype", [torch.bfloat16]) +@pytest.mark.parametrize("silu_activation", [False, True]) +@pytest.mark.parametrize("has_bias", [False, True]) +@pytest.mark.parametrize("seqlen", [1]) +@pytest.mark.parametrize("width", [4]) +@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096]) +def test_causal_conv1d_update(dim, width, seqlen, has_bias, silu_activation, itype): + if not torch.cuda.is_available(): + pytest.skip("CUDA device not available") + + device = "cuda" + rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3) + if itype == torch.bfloat16: + rtol, atol = 1e-2, 5e-2 + # set seed + torch.manual_seed(0) + batch = 2 + x = torch.randn(batch, dim, seqlen, device=device, dtype=itype) + x_ref = x.clone() + conv_state = torch.randn(batch, dim, width - 1, device=device, dtype=itype) + + weight = torch.randn(dim, width, device=device, dtype=itype) + bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None + conv_state_ref = conv_state.detach().clone() + activation = None if not silu_activation else "silu" + out = causal_conv1d_update(x, conv_state, weight, bias, activation=activation) + out_ref = causal_conv1d_update_ref( + x_ref, conv_state_ref, weight, bias, activation=activation + ) + + assert torch.equal(conv_state, conv_state_ref) + assert torch.allclose(out, out_ref, rtol=rtol, atol=atol) + + +@pytest.mark.parametrize("itype", [torch.float32, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("silu_activation", [False, True]) +@pytest.mark.parametrize("has_bias", [False, True]) +@pytest.mark.parametrize("seqlen", [1, 3]) +@pytest.mark.parametrize("width", [3, 4]) +@pytest.mark.parametrize("dim", [2048 + 16, 4096]) +# tests correctness in case subset of the sequences are padded +@pytest.mark.parametrize("with_padding", [True, False]) +@pytest.mark.parametrize("batch_size", [3]) +def test_causal_conv1d_update_with_batch_gather( + batch_size, with_padding, dim, width, seqlen, has_bias, silu_activation, itype +): + if not torch.cuda.is_available(): + pytest.skip("CUDA device not available") + + device = "cuda" + rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3) + if itype == torch.bfloat16: + rtol, atol = 1e-2, 5e-2 + + # set seed + torch.manual_seed(0) + + padding = 5 if with_padding else 0 + padded_batch_size = batch_size + padding + # total_entries = number of cache line + total_entries = 10 * batch_size + + # x will be (batch, dim, seqlen) with contiguous along dim-axis + x = torch.randn( + padded_batch_size, seqlen, dim, device=device, dtype=itype + ).transpose(1, 2) + + x_ref = x.clone() + + conv_state_indices = torch.randperm(total_entries)[:batch_size].to( + dtype=torch.int32, device=device + ) + unused_states_bool = torch.ones(total_entries, dtype=torch.bool, device=device) + unused_states_bool[conv_state_indices] = False + padded_state_indices = torch.concat( + [ + conv_state_indices, + torch.as_tensor([PAD_SLOT_ID] * padding, dtype=torch.int32, device=device), + ], + dim=0, + ) + + # conv_state will be (cache_lines, dim, state_len) + # with contiguous along dim-axis + conv_state = torch.randn( + total_entries, width - 1, dim, device=device, dtype=itype + ).transpose(1, 2) + + conv_state_for_padding_test = conv_state.clone() + + weight = torch.randn(dim, width, device=device, dtype=itype) + bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None + conv_state_ref = conv_state[conv_state_indices, :].detach().clone() + activation = None if not silu_activation else "silu" + + out = causal_conv1d_update( + x, + conv_state, + weight, + bias, + activation=activation, + conv_state_indices=padded_state_indices, + pad_slot_id=PAD_SLOT_ID, + ) + out_ref = causal_conv1d_update_ref( + x_ref[:batch_size], conv_state_ref, weight, bias, activation=activation + ) + + assert torch.equal(conv_state[conv_state_indices, :], conv_state_ref) + assert torch.equal( + conv_state[unused_states_bool], conv_state_for_padding_test[unused_states_bool] + ) + assert torch.allclose(out[:batch_size], out_ref, rtol=rtol, atol=atol) + + +@pytest.mark.parametrize("itype", [torch.bfloat16]) +@pytest.mark.parametrize("silu_activation", [True]) +@pytest.mark.parametrize("has_bias", [True]) +@pytest.mark.parametrize("width", [4]) +@pytest.mark.parametrize("seqlen", [8, 30, 249, 2049, 4096]) +@pytest.mark.parametrize("dim", [64, 4096]) +@pytest.mark.parametrize("with_padding", [True, False]) +@pytest.mark.parametrize("batch", [4, 10]) +def test_causal_conv1d_varlen( + batch, with_padding, dim, seqlen, width, has_bias, silu_activation, itype +): + if not torch.cuda.is_available(): + pytest.skip("CUDA device not available") + + device = "cuda" + torch.cuda.empty_cache() + rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3) + if itype == torch.bfloat16: + rtol, atol = 1e-2, 5e-2 + # set seed + torch.manual_seed(0) + seqlens = [] + batch_size = batch + padding = 3 if with_padding else 0 + padded_batch_size = batch_size + padding + nsplits = padded_batch_size - 1 + + eos_pos = torch.randperm(seqlen - 1)[:nsplits].sort().values + + seqlens.append( + torch.diff( + torch.cat([torch.tensor([-1]), eos_pos, torch.tensor([seqlen - 1])]) + ).tolist() + ) + assert sum(seqlens[-1]) == seqlen + assert all(s > 0 for s in seqlens[-1]) + + total_entries = batch_size * 10 + cumsum = torch.cumsum(torch.tensor(seqlens[0]), dim=0).to(torch.int32) + cumsum = torch.concat([torch.tensor([0], dtype=torch.int32), cumsum], dim=0) + x = rearrange( + torch.randn(1, seqlen, 4096 + dim + 64, device=device, dtype=itype), + "b s d -> b d s", + )[:, 4096 : 4096 + dim, :] + + weight = torch.randn(dim, width, device=device, dtype=itype) + + bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None + x_ref = x.clone() + weight_ref = weight.clone() + bias_ref = bias.clone() if bias is not None else None + activation = None if not silu_activation else "silu" + final_states = torch.randn( + total_entries, width - 1, dim, device=x.device, dtype=x.dtype + ).transpose(1, 2) + final_states_ref = final_states.clone() + has_initial_states = torch.randint( + 0, 2, (cumsum.shape[0] - 1,), dtype=torch.bool, device=x.device + ) + state_indices = torch.randperm(total_entries, dtype=torch.int32, device=x.device)[ + :batch_size + ] + padded_state_indices = torch.concat( + [ + state_indices, + torch.as_tensor([PAD_SLOT_ID] * padding, dtype=torch.int32, device=device), + ], + dim=-1, + ) + out = causal_conv1d_fn( + x.squeeze(0), + weight, + bias=bias, + conv_states=final_states, + query_start_loc=cumsum.cuda(), + seq_lens_cpu=torch.tensor(seqlens[0]), + cache_indices=padded_state_indices, + has_initial_state=has_initial_states, + activation=activation, + pad_slot_id=PAD_SLOT_ID, + ) + + out_ref = [] + out_ref_b = [] + + splits = [torch.split(var, seqlens[0], dim=-1) for var in (x_ref)] + for i in range(len(seqlens[0])): + x_s = [v[i].unsqueeze(0) for v in splits][0] + if padded_state_indices[i] == PAD_SLOT_ID: + continue + out_ref_b.append( + causal_conv1d_ref( + x_s, + weight_ref, + bias_ref, + activation=activation, + return_final_states=True, + final_states_out=final_states_ref[padded_state_indices[i]].unsqueeze(0), + initial_states=( + final_states_ref[padded_state_indices[i]].unsqueeze(0) + if has_initial_states[i] + else None + ), + ) + ) + out_ref.append(torch.cat([t[0] for t in out_ref_b], dim=2)) + out_ref_tensor = torch.cat(out_ref, dim=0) + + assert torch.allclose( + final_states[state_indices], + final_states_ref[state_indices], + rtol=rtol, + atol=atol, + ) + unpadded_out = out[:, : out_ref_tensor.shape[-1]] + assert torch.allclose(unpadded_out, out_ref_tensor, rtol=rtol, atol=atol) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/layers/attention/mamba/test_mamba2_mixer.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/layers/attention/mamba/test_mamba2_mixer.py new file mode 100644 index 00000000..2252db65 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/layers/attention/mamba/test_mamba2_mixer.py @@ -0,0 +1,143 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/2c58742dff8613a3bd7496f2008ce927e18d38d1/tests/kernels/mamba/test_mamba_mixer2.py + + +from unittest.mock import patch + +import pytest +import torch + +from sglang.srt.distributed.device_communicators.custom_all_reduce_utils import ( + update_environment_variables, +) +from sglang.srt.distributed.parallel_state import ( + init_distributed_environment, + initialize_model_parallel, +) + +NUM_GPUS = 2 + + +@pytest.mark.parametrize("batch_size", [8]) +@pytest.mark.parametrize("seq_len", [128]) +@pytest.mark.parametrize( + "hidden_size_n_groups", + [ + (64, 1), # hidden_size be divisible by num_gpus + (100, 4), # and n_groups must divide hidden_size + ], +) +@pytest.mark.parametrize("dtype", [torch.float16]) +def test_mixer2_gated_norm_multi_gpu( + batch_size: int, + seq_len: int, + hidden_size_n_groups: tuple[int, int], + dtype: torch.dtype, + device: str = "cuda", +): + if not torch.cuda.is_available(): + pytest.skip("CUDA device not available") + + assert torch.cuda.device_count() == NUM_GPUS + + hidden_size, n_groups = hidden_size_n_groups + num_processes = NUM_GPUS + + def run_torch_spawn(fn, nprocs): + # need to use torch.mp.spawn otherwise will have problems with + # torch.distributed and cuda + torch.multiprocessing.spawn( + fn, + args=( + num_processes, + batch_size, + seq_len, + hidden_size, + n_groups, + dtype, + device, + ), + nprocs=nprocs, + ) + + run_torch_spawn(mixer2_gated_norm_tensor_parallel, NUM_GPUS) + + +def mixer2_gated_norm_tensor_parallel( + local_rank: int, + world_size: int, + batch_size: int, + seq_len: int, + hidden_size: int, + n_groups: int, + dtype: torch.dtype, + device: str, +): + torch.manual_seed(0) + + device = torch.device(f"cuda:{local_rank}") + torch.cuda.set_device(device) + torch.set_default_device(device) + torch.set_default_dtype(dtype) + + update_environment_variables( + { + "RANK": str(local_rank), + "LOCAL_RANK": str(local_rank), + "WORLD_SIZE": str(world_size), + "MASTER_ADDR": "localhost", + "MASTER_PORT": "12345", + } + ) + + # initialize distributed + init_distributed_environment( + world_size=world_size, rank=local_rank, local_rank=local_rank + ) + initialize_model_parallel(tensor_model_parallel_size=world_size) + + # create random weights an inputs + weight = torch.rand((hidden_size,), dtype=dtype, device=device) + hidden_states = torch.randn(batch_size, seq_len, hidden_size) + gate_states = torch.randn(batch_size, seq_len, hidden_size) + + import sglang.srt.layers.attention.mamba.mixer2_rms_norm_gated as m2 + import sglang.srt.model_loader.weight_utils as wu + + # Convenience: Avoid calling initialize_dp_attention + with patch.object(wu, "get_attention_tp_rank", return_value=local_rank): + # create gated-norm with TP + mixer = m2.Mixer2RMSNormGated( + full_hidden_size=hidden_size, + full_n_groups=n_groups, + ) + mixer.weight.weight_loader(mixer.weight, weight) + + with ( + patch.object(m2, "get_tensor_model_parallel_world_size", return_value=1), + patch.object(m2, "get_tensor_model_parallel_rank", return_value=0), + ): + # create gated-norm without TP to compute reference + mixer_single_gpu = m2.Mixer2RMSNormGated( + full_hidden_size=hidden_size, + full_n_groups=n_groups, + ) + # assign weight to single-gpu mixer + mixer_single_gpu.weight.data = weight + + # generate and compare + N = hidden_size // world_size + output = mixer( + hidden_states[..., local_rank * N : (local_rank + 1) * N], + gate_states[..., local_rank * N : (local_rank + 1) * N], + ) + ref_output = mixer_single_gpu(hidden_states, gate_states) + torch.testing.assert_close( + output, + ref_output[..., local_rank * N : (local_rank + 1) * N], + atol=5e-3, + rtol=1e-3, + ) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/layers/attention/mamba/test_mamba_ssm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/layers/attention/mamba/test_mamba_ssm.py new file mode 100644 index 00000000..4a2c9a8e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/layers/attention/mamba/test_mamba_ssm.py @@ -0,0 +1,296 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/633f943e30a4444d890d26b81850f7217736f840/tests/kernels/mamba/test_mamba_ssm_ssd.py + + +import pytest +import torch +import torch.nn.functional as F +from einops import rearrange, repeat + +from sglang.srt.layers.attention.mamba.causal_conv1d_triton import PAD_SLOT_ID +from sglang.srt.layers.attention.mamba.ops import selective_state_update + + +def selective_state_update_ref( + state, x, dt, A, B, C, D=None, z=None, dt_bias=None, dt_softplus=False +): + """ + Argument: + state: (batch, dim, dstate) or (batch, nheads, dim, dstate) + x: (batch, dim) or (batch, nheads, dim) + dt: (batch, dim) or (batch, nheads, dim) + A: (dim, dstate) or (nheads, dim, dstate) + B: (batch, dstate) or (batch, ngroups, dstate) + C: (batch, dstate) or (batch, ngroups, dstate) + D: (dim,) or (nheads, dim) + z: (batch, dim) or (batch, nheads, dim) + dt_bias: (dim,) or (nheads, dim) + Return: + out: (batch, dim) or (batch, nheads, dim) + """ + has_heads = state.dim() > 3 + if state.dim() == 3: + state = state.unsqueeze(1) + if x.dim() == 2: + x = x.unsqueeze(1) + if dt.dim() == 2: + dt = dt.unsqueeze(1) + if A.dim() == 2: + A = A.unsqueeze(0) + if B.dim() == 2: + B = B.unsqueeze(1) + if C.dim() == 2: + C = C.unsqueeze(1) + if D is not None and D.dim() == 1: + D = D.unsqueeze(0) + if z is not None and z.dim() == 2: + z = z.unsqueeze(1) + if dt_bias is not None and dt_bias.dim() == 1: + dt_bias = dt_bias.unsqueeze(0) + batch, nheads, dim, dstate = state.shape + assert x.shape == (batch, nheads, dim) + assert dt.shape == x.shape + assert A.shape == (nheads, dim, dstate) + ngroups = B.shape[1] + assert nheads % ngroups == 0, "nheads must be divisible by ngroups" + assert B.shape == (batch, ngroups, dstate) + assert C.shape == B.shape + if D is not None: + assert D.shape == (nheads, dim) + if z is not None: + assert z.shape == x.shape + if dt_bias is not None: + assert dt_bias.shape == (nheads, dim) + dt = dt + dt_bias + dt = F.softplus(dt) if dt_softplus else dt + dA = torch.exp( + rearrange(dt, "b h d -> b h d 1") * A + ) # (batch, nheads, dim, dstate) + B = repeat(B, "b g n -> b (g h) n", h=nheads // ngroups) # (batch, nheads, dstate) + C = repeat(C, "b g n -> b (g h) n", h=nheads // ngroups) # (batch, nheads, dstate) + dB = rearrange(dt, "b h d -> b h d 1") * rearrange( + B, "b h n -> b h 1 n" + ) # (batch, nheads, dim, dstate) + state.copy_( + state * dA + dB * rearrange(x, "b h d -> b h d 1") + ) # (batch, dim, dstate + out = torch.einsum("bhdn,bhn->bhd", state.to(C.dtype), C) + if D is not None: + out += (x * D).to(out.dtype) + out = (out if z is None else out * F.silu(z)).to(x.dtype) + if not has_heads: + out = out.squeeze(1) + return out + + +@pytest.mark.parametrize("itype", [torch.float32, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("has_z", [False, True]) +@pytest.mark.parametrize("dstate", [16, 32, 64]) +@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096]) +def test_selective_state_update(dim, dstate, has_z, itype): + if not torch.cuda.is_available(): + pytest.skip("CUDA device not available") + + device = "cuda" + + rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2) + if itype == torch.bfloat16: + rtol, atol = 1e-2, 5e-2 + if torch.version.hip: + atol *= 2 + # set seed + torch.manual_seed(0) + batch_size = 1 + state = torch.randn(batch_size, dim, dstate, dtype=itype, device=device) + x = torch.randn(batch_size, dim, device=device, dtype=itype) + out = torch.empty_like(x) + dt = torch.randn(batch_size, dim, device=device, dtype=itype) + dt_bias = torch.rand(dim, device=device) - 4.0 + A = -torch.rand(dim, dstate, device=device) - 1.0 + B = torch.randn(batch_size, dstate, device=device) + C = torch.randn(batch_size, dstate, device=device) + D = torch.randn(dim, device=device) + z = torch.randn_like(x) if has_z else None + state_ref = state.detach().clone() + selective_state_update( + state, x, dt, A, B, C, D=D, z=z, dt_bias=dt_bias, dt_softplus=True, out=out + ) + out_ref = selective_state_update_ref( + state_ref, x, dt, A, B, C, D=D, z=z, dt_bias=dt_bias, dt_softplus=True + ) + + assert torch.allclose(state, state_ref, rtol=rtol, atol=atol) + assert torch.allclose(out, out_ref, rtol=rtol, atol=atol) + + +@pytest.mark.parametrize("itype", [torch.float32, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("has_z", [True]) +@pytest.mark.parametrize("dstate", [16, 32, 64]) +@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096]) +# tests correctness in case subset of the sequences are padded +@pytest.mark.parametrize("with_padding", [True, False]) +def test_selective_state_update_with_batch_indices( + with_padding, dim, dstate, has_z, itype +): + if not torch.cuda.is_available(): + pytest.skip("CUDA device not available") + + device = "cuda" + rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2) + if itype == torch.bfloat16: + rtol, atol = 1e-1, 1e-1 + if torch.version.hip: + atol *= 2 + # set seed + torch.random.manual_seed(0) + batch_size = 3 + padding = 5 if with_padding else 0 + padded_batch_size = batch_size + padding + total_entries = 10 * batch_size + state = torch.randn(total_entries, dim, dstate, dtype=itype, device=device) + state_indices = torch.randperm(total_entries)[:batch_size].to( + dtype=torch.int32, device=device + ) + unused_states_bool = torch.ones(total_entries, dtype=torch.bool, device=device) + unused_states_bool[state_indices] = False + padded_state_indices = torch.concat( + [ + state_indices, + torch.as_tensor([PAD_SLOT_ID] * padding, dtype=torch.int32, device=device), + ], + dim=0, + ) + x = torch.randn(padded_batch_size, dim, device=device, dtype=itype) + out = torch.empty_like(x) + dt = torch.randn(padded_batch_size, dim, device=device, dtype=itype) + dt_bias = torch.rand(dim, device=device) - 4.0 + A = -torch.rand(dim, dstate, device=device) - 1.0 + B = torch.randn(padded_batch_size, dstate, device=device) + C = torch.randn(padded_batch_size, dstate, device=device) + D = torch.randn(dim, device=device) + z = torch.randn_like(x) if has_z else None + state_ref = state[state_indices, :].clone() + state_before = state.clone() + selective_state_update( + state, + x, + dt, + A, + B, + C, + D=D, + z=z, + dt_bias=dt_bias, + dt_softplus=True, + state_batch_indices=padded_state_indices, + pad_slot_id=PAD_SLOT_ID, + out=out, + ) + out_ref = selective_state_update_ref( + state_ref, + x[:batch_size], + dt[:batch_size], + A, + B[:batch_size], + C[:batch_size], + D=D, + z=z[:batch_size], + dt_bias=dt_bias, + dt_softplus=True, + ) + + print("Output diff max", (out[:batch_size] - out_ref).max()) + print("Output diff mean", (out[:batch_size] - out_ref).mean()) + print("Output state diff max", (state[state_indices, :] - state_ref).max()) + print("Output state diff mean", (state[state_indices, :] - state_ref).mean()) + # test padded entries stay the same + if with_padding: + assert torch.equal(state_before[unused_states_bool], state[unused_states_bool]) + assert torch.equal(x[batch_size + 1 :], x[batch_size + 1 :]) + assert torch.equal(dt[batch_size + 1 :], dt[batch_size + 1 :]) + assert torch.equal(B[batch_size + 1 :], B[batch_size + 1 :]) + assert torch.equal(C[batch_size + 1 :], C[batch_size + 1 :]) + + # test "real" entries + assert torch.allclose(state[state_indices, :], state_ref, rtol=rtol, atol=atol) + assert torch.allclose(out[:batch_size], out_ref, rtol=rtol, atol=atol) + + +@pytest.mark.parametrize("itype", [torch.float32, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("has_z", [False, True]) +@pytest.mark.parametrize("tie_hdim", [False, True]) +@pytest.mark.parametrize("ngroups", [1, 2, 4]) +@pytest.mark.parametrize("dstate", [16, 32, 64]) +@pytest.mark.parametrize("dim", [2048, 4096]) +def test_selective_state_update_with_heads_with_batch_indices( + dim, dstate, ngroups, has_z, tie_hdim, itype +): + if not torch.cuda.is_available(): + pytest.skip("CUDA device not available") + + device = "cuda" + rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 3e-2) + if itype == torch.bfloat16: + rtol, atol = 1e-1, 1e-1 + # set seed + torch.random.manual_seed(0) + batch_size = 3 + headdim = 64 + nheads = dim // headdim + + total_entries = 10 * batch_size + state = torch.randn( + total_entries, nheads, headdim, dstate, dtype=itype, device=device + ) + state_indices = torch.randperm(total_entries)[:batch_size].to( + dtype=torch.int32, device=device + ) + + x = torch.randn(batch_size, nheads, headdim, device=device, dtype=itype) + out = torch.empty_like(x) + if not tie_hdim: + dt = torch.randn(batch_size, nheads, headdim, device=device, dtype=itype) + dt_bias = torch.rand(nheads, headdim, device=device) - 4.0 + A = -torch.rand(nheads, headdim, dstate, device=device) - 1.0 + D = torch.randn(nheads, headdim, device=device) + else: + dt = repeat( + torch.randn(batch_size, nheads, device=device, dtype=itype), + "b h -> b h p", + p=headdim, + ) + dt_bias = repeat(torch.rand(nheads, device=device) - 4.0, "h -> h p", p=headdim) + A = repeat( + -torch.rand(nheads, device=device) - 1.0, "h -> h p n", p=headdim, n=dstate + ) + D = repeat(torch.randn(nheads, device=device), "h -> h p", p=headdim) + B = torch.randn(batch_size, ngroups, dstate, device=device) + C = torch.randn(batch_size, ngroups, dstate, device=device) + z = torch.randn_like(x) if has_z else None + state_ref = state[state_indices, :].detach().clone() + selective_state_update( + state, + x, + dt, + A, + B, + C, + D=D, + z=z, + dt_bias=dt_bias, + dt_softplus=True, + state_batch_indices=state_indices, + pad_slot_id=PAD_SLOT_ID, + out=out, + ) + out_ref = selective_state_update_ref( + state_ref, x, dt, A, B, C, D=D, z=z, dt_bias=dt_bias, dt_softplus=True + ) + + print(f"Output max diff: {(out - out_ref).abs().max().item()}") + print(f"Output mean diff: {(out - out_ref).abs().mean().item()}") + assert torch.allclose(state[state_indices, :], state_ref, rtol=rtol, atol=atol) + assert torch.allclose(out, out_ref, rtol=rtol, atol=atol) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/layers/attention/mamba/test_mamba_ssm_ssd.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/layers/attention/mamba/test_mamba_ssm_ssd.py new file mode 100644 index 00000000..10a7f3f8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/layers/attention/mamba/test_mamba_ssm_ssd.py @@ -0,0 +1,606 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/633f943e30a4444d890d26b81850f7217736f840/tests/kernels/mamba/test_mamba_ssm_ssd.py + + +import pytest +import torch +import torch.nn.functional as F +from einops import rearrange, repeat + +from sglang.srt.layers.attention.mamba.mamba2_metadata import Mamba2Metadata +from sglang.srt.layers.attention.mamba.ops import mamba_chunk_scan_combined +from sglang.utils import is_in_ci + +# Added by the IBM Team, 2024 + +# Adapted from https://github.com/state-spaces/mamba/blob/v2.2.4/mamba_ssm/modules/ssd_minimal.py + + +# this is the segsum implementation taken from above +def segsum(x): + """Calculates segment sum.""" + T = x.size(-1) + x = repeat(x, "... d -> ... d e", e=T) + mask = torch.tril(torch.ones(T, T, device=x.device, dtype=bool), diagonal=-1) + x = x.masked_fill(~mask, 0) + x_segsum = torch.cumsum(x, dim=-2) + mask = torch.tril(torch.ones(T, T, device=x.device, dtype=bool), diagonal=0) + x_segsum = x_segsum.masked_fill(~mask, -torch.inf) + return x_segsum + + +def ssd_minimal_discrete(X, A, B, C, block_len, initial_states=None): + """ + Arguments: + X: (batch, length, n_heads, d_head) + A: (batch, length, n_heads) + B: (batch, length, n_heads, d_state) + C: (batch, length, n_heads, d_state) + Return: + Y: (batch, length, n_heads, d_head) + """ + assert X.dtype == A.dtype == B.dtype == C.dtype + assert X.shape[1] % block_len == 0 + + # Rearrange into blocks/chunks + X, A, B, C = ( + rearrange(x, "b (c l) ... -> b c l ...", l=block_len) for x in (X, A, B, C) + ) + + A = rearrange(A, "b c l h -> b h c l") + A_cumsum = torch.cumsum(A, dim=-1) + + # 1. Compute the output for each intra-chunk (diagonal blocks) + L = torch.exp(segsum(A)) + Y_diag = torch.einsum("bclhn,bcshn,bhcls,bcshp->bclhp", C, B, L, X) + + # 2. Compute the state for each intra-chunk + # (right term of low-rank factorization of off-diagonal blocks; B terms) + decay_states = torch.exp(A_cumsum[:, :, :, -1:] - A_cumsum) + states = torch.einsum("bclhn,bhcl,bclhp->bchpn", B, decay_states, X) + + # 3. Compute the inter-chunk SSM recurrence; produces correct SSM states at + # chunk boundaries + # (middle term of factorization of off-diag blocks; A terms) + if initial_states is None: + initial_states = torch.zeros_like(states[:, :1]) + states = torch.cat([initial_states, states], dim=1) + decay_chunk = torch.exp(segsum(F.pad(A_cumsum[:, :, :, -1], (1, 0)))) + new_states = torch.einsum("bhzc,bchpn->bzhpn", decay_chunk, states) + states, final_state = new_states[:, :-1], new_states[:, -1] + + # 4. Compute state -> output conversion per chunk + # (left term of low-rank factorization of off-diagonal blocks; C terms) + state_decay_out = torch.exp(A_cumsum) + Y_off = torch.einsum("bclhn,bchpn,bhcl->bclhp", C, states, state_decay_out) + + # Add output of intra-chunk and inter-chunk terms + # (diagonal and off-diagonal blocks) + Y = rearrange(Y_diag + Y_off, "b c l h p -> b (c l) h p") + return Y, final_state + + +def generate_random_inputs(batch_size, seqlen, n_heads, d_head, itype, device="cuda"): + + if not torch.cuda.is_available(): + pytest.skip("CUDA device not available") + + torch.manual_seed(0) + A = -torch.exp(torch.rand(n_heads, dtype=itype, device=device)) + dt = F.softplus( + torch.randn(batch_size, seqlen, n_heads, dtype=itype, device=device) - 4 + ) + X = torch.randn((batch_size, seqlen, n_heads, d_head), dtype=itype, device=device) + B = torch.randn((batch_size, seqlen, n_heads, d_head), dtype=itype, device=device) + C = torch.randn((batch_size, seqlen, n_heads, d_head), dtype=itype, device=device) + + return A, dt, X, B, C + + +def generate_continuous_batched_examples( + example_lens_by_batch, + num_examples, + full_length, + last_taken, + exhausted, + n_heads, + d_head, + itype, + device="cuda", + return_naive_ref=True, +): + + # this function generates a random examples of certain length + # and then cut according to "example_lens_by_batch" and feed + # them in continuous batches to the kernels. + # If if return_naive_ref=True, the naive torch implementation + # ssd_minimal_discrete will be used to compute and return + # reference output. + + # generate the full-length example + A, dt, X, B, C = generate_random_inputs( + num_examples, full_length, n_heads, d_head, itype + ) + + if return_naive_ref: + Y_min, final_state_min = ssd_minimal_discrete( + X * dt.unsqueeze(-1), A * dt, B, C, block_len=full_length // 4 + ) + + # internal function that outputs a cont batch of examples + # given a tuple of lengths for each example in the batch + # e.g., example_lens=(8, 4) means take 8 samples from first eg, + # 4 examples from second eg, etc + def get_continuous_batch(example_lens: tuple[int, ...]): + + indices = [] + for i, x in enumerate(example_lens): + c = last_taken.get(i, 0) + indices.append((c, c + x)) + last_taken[i] = (c + x) % full_length + exhausted[i] = last_taken[i] == 0 + + return ( + torch.concat([x[i, s:e] for i, (s, e) in enumerate(indices)]).unsqueeze(0) + for x in (dt, X, B, C) + ) + + # internal function that maps "n" to the appropriate right boundary + # value when forming continuous batches from examples of length given + # by "full_length". + # - e.g., when n > full_length, returns n % full_length + # when n == full_length, returns full_length + def end_boundary(n: int): + return n - ((n - 1) // full_length) * full_length + + IND_E = None + for spec in example_lens_by_batch: + + # get the (maybe partial) example seen in this cont batch + dt2, X2, B2, C2 = get_continuous_batch(spec) + + # get the metadata + cu_seqlens = torch.tensor((0,) + spec, device=device).cumsum(dim=0) + seq_idx = torch.zeros( + cu_seqlens[-1], dtype=torch.int32, device=cu_seqlens.device + ) + for i, (srt, end) in enumerate( + zip( + cu_seqlens, + cu_seqlens[1:], + ) + ): + seq_idx[srt:end] = i + + # for cont batch + if IND_E is None: + IND_S = [0 for _ in range(len(spec))] + else: + IND_S = [x % full_length for x in IND_E] + IND_E = [end_boundary(x + y) for x, y in zip(IND_S, spec)] + + yield ( + ( + [Y_min[s, IND_S[s] : IND_E[s]] for s in range(num_examples)] + if return_naive_ref + else None + ), + cu_seqlens, + seq_idx.unsqueeze(0), + (A, dt2, X2, B2, C2), + ) + + +SINGLE_ITYPE = [torch.float32, torch.float16, torch.bfloat16] +SINGLE_NHEADS = [3, 4, 11, 16, 32] +SINGLE_DHEAD = [5, 8, 19, 32, 128] +SINGLE_SEQ_LEN_CHUNK_SIZE = [(112, 16), (128, 32)] + +if is_in_ci(): + SINGLE_ITYPE = [torch.float32, torch.bfloat16] + SINGLE_NHEADS = [3, 32] + SINGLE_DHEAD = [5, 128] + SINGLE_SEQ_LEN_CHUNK_SIZE = [(112, 16)] + + +@pytest.mark.parametrize("itype", SINGLE_ITYPE) +@pytest.mark.parametrize("n_heads", SINGLE_NHEADS) +@pytest.mark.parametrize("d_head", SINGLE_DHEAD) +@pytest.mark.parametrize("seq_len_chunk_size", SINGLE_SEQ_LEN_CHUNK_SIZE) +def test_mamba_chunk_scan_single_example(d_head, n_heads, seq_len_chunk_size, itype): + if not torch.cuda.is_available(): + pytest.skip("CUDA device not available") + + # this tests the kernels on a single example (no batching) + + # TODO: the bfloat16 case requires higher thresholds. To be investigated + + if itype == torch.bfloat16: + atol, rtol = 5e-2, 5e-2 + else: + atol, rtol = 8e-3, 5e-3 + + # set seed + batch_size = 1 # batch_size + # ssd_minimal_discrete requires chunk_size divide seqlen + # - this is only required for generating the reference seqs, + # it is not an operational limitation. + seqlen, chunk_size = seq_len_chunk_size + + A, dt, X, B, C = generate_random_inputs(batch_size, seqlen, n_heads, d_head, itype) + + Y_min, final_state_min = ssd_minimal_discrete( + X * dt.unsqueeze(-1), A * dt, B, C, chunk_size + ) + Y = torch.empty_like(X) + final_state = mamba_chunk_scan_combined( + X, dt, A, B, C, chunk_size, D=None, return_final_states=True, out=Y + ) + + # just test the last in sequence + torch.testing.assert_close(Y[:, -1], Y_min[:, -1], atol=atol, rtol=rtol) + + # just test the last head + # NOTE, in the kernel we always cast states to fp32 + torch.testing.assert_close( + final_state[:, -1], + final_state_min[:, -1].to(torch.float32), + atol=atol, + rtol=rtol, + ) + + +BATCHED_ITYPE = [torch.float32, torch.float16] +BATCHED_NHEADS = [4, 8, 13] +BATCHED_DHEAD = [5, 16, 21, 32] + +if is_in_ci(): + BATCHED_ITYPE = [torch.float32] + BATCHED_NHEADS = [4, 13] + BATCHED_DHEAD = [5, 32] + + +@pytest.mark.parametrize("itype", BATCHED_ITYPE) +@pytest.mark.parametrize("n_heads", BATCHED_NHEADS) +@pytest.mark.parametrize("d_head", BATCHED_DHEAD) +@pytest.mark.parametrize( + "seq_len_chunk_size_cases", + [ + # small-ish chunk_size (8) + (64, 8, 2, [(64, 32), (64, 32)]), + (64, 8, 2, [(32, 32), (32, 32), (32, 32)]), + (64, 8, 2, [(8, 8), (8, 8), (8, 8)]), # chunk size boundary + ( + 64, + 8, + 2, + [(4, 4), (4, 4), (4, 4), (4, 4)], + ), # chunk_size larger than cont batches + ( + 64, + 8, + 5, + [ + (64, 32, 16, 8, 8), + (8, 16, 32, 16, 8), + (8, 8, 16, 32, 16), + ], + ), # mode examples with varied lengths + # large-ish chunk_size (256) + (64, 256, 1, [(5,), (1,), (1,), (1,)]), # irregular sizes with small sequences + ( + 64, + 256, + 2, + [(5, 30), (1, 2), (1, 2), (1, 2)], + ), # irregular sizes with small sequences + # we also need to test some large seqlen + # to catch errors with init states decay + (768, 128, 2, [(138, 225), (138, 225)]), + ], +) +def test_mamba_chunk_scan_cont_batch(d_head, n_heads, seq_len_chunk_size_cases, itype): + if not torch.cuda.is_available(): + pytest.skip("CUDA device not available") + + # this test with multiple examples in a continuous batch + # (i.e. chunked prefill) + + seqlen, chunk_size, num_examples, cases = seq_len_chunk_size_cases + + # This test can have larger error for longer sequences + if seqlen > 256: + atol, rtol = 1e-2, 5e-3 + else: + atol, rtol = 5e-3, 5e-3 + + # hold state during the cutting process so we know if an + # example has been exhausted and needs to cycle + last_taken: dict = {} # map: eg -> pointer to last taken sample + exhausted: dict = {} # map: eg -> boolean indicating example is exhausted + + states = None + for ( + Y_min, + cu_seqlens, + seq_idx, + (A, dt, X, B, C), + ) in generate_continuous_batched_examples( + cases, num_examples, seqlen, last_taken, exhausted, n_heads, d_head, itype + ): + + chunk_indices, chunk_offsets = ( + Mamba2Metadata._query_start_loc_to_chunk_indices_offsets( + cu_seqlens, chunk_size, cu_seqlens[-1] + ) + ) + + Y = torch.empty_like(X) + new_states = mamba_chunk_scan_combined( + X, + dt, + A, + B, + C, + chunk_size, + D=None, + cu_seqlens=cu_seqlens, + seq_idx=seq_idx, + chunk_indices=chunk_indices, + chunk_offsets=chunk_offsets, + return_varlen_states=True, + initial_states=states, + out=Y, + ) + + # just test the last in sequence + for i in range(num_examples): + + # just test one dim and dstate + Y_eg = Y[0, cu_seqlens[i] : cu_seqlens[i + 1], 0, 0] + Y_min_eg = Y_min[i][:, 0, 0] + torch.testing.assert_close(Y_eg, Y_min_eg, atol=atol, rtol=rtol) + + # update states + states = new_states + for i, clear in exhausted.items(): + if clear: + states[i].fill_(0.0) + exhausted[i] = False + + +@pytest.mark.parametrize("chunk_size", [8, 256]) +@pytest.mark.parametrize( + "seqlens", + [ + (16, 2, 8, 13), + (270, 88, 212, 203), + (16, 20), + ], +) +def test_mamba_chunk_scan_cont_batch_prefill_chunking(chunk_size, seqlens): + if not torch.cuda.is_available(): + pytest.skip("CUDA device not available") + + # This test verifies the correctness of the chunked prefill implementation + # in the mamba2 ssd kernels, by comparing concatenation (in the sequence + # dimension) of chunked results with the full sequence result. + # It is different from test_mamba_chunk_scan_cont_batch by: + # 1. Not using the naive torch implementation (ssd_minimal_discrete) to get + # reference outputs. Instead, it compares chunked kernel outputs to full + # sequence kernel outputs. This is the most straightforward way to + # assert chunked prefill correctness. + # 2. It focuses on cases where sequences change in the middle of mamba + # chunks, and not necessarily on chunk boundaries. + + max_seqlen = max(seqlens) + # This test can have larger error for longer sequences + if max_seqlen > 256: + atol, rtol = 1e-2, 5e-3 + else: + atol, rtol = 5e-3, 5e-3 + + num_sequences = len(seqlens) + n_heads = 16 + d_head = 64 + itype = torch.float32 + + # hold state during the cutting process so we know if an + # example has been exhausted and needs to cycle + last_taken: dict = {} # map: eg -> pointer to last taken sample + exhausted: dict = {} # map: eg -> boolean indicating example is exhausted + _, cu_seqlens, seq_idx, (A, dt, X, B, C) = next( + generate_continuous_batched_examples( + [seqlens], + num_sequences, + max_seqlen, + last_taken, + exhausted, + n_heads, + d_head, + itype, + return_naive_ref=False, + ) + ) + seqlens = torch.tensor(seqlens, dtype=torch.int32, device=X.device) + device = X.device + + ## full seqlen computation + chunk_indices, chunk_offsets = ( + Mamba2Metadata._query_start_loc_to_chunk_indices_offsets( + cu_seqlens, chunk_size, cu_seqlens[-1] + ) + ) + Y_ref = torch.empty_like(X) + state_ref = mamba_chunk_scan_combined( + X, + dt, + A, + B, + C, + chunk_size, + D=None, + cu_seqlens=cu_seqlens, + seq_idx=seq_idx, + chunk_indices=chunk_indices, + chunk_offsets=chunk_offsets, + return_varlen_states=True, + initial_states=None, + out=Y_ref, + ) + + ## chunked seqlen computation + # first chunk + chunked_seqlens = seqlens // 2 + chunked_cu_seqlens = torch.cat( + [torch.tensor([0], device=device), torch.cumsum(chunked_seqlens, dim=0)], dim=0 + ) + chunked_seq_idx = ( + torch.repeat_interleave( + torch.arange(len(chunked_seqlens), device=device), + chunked_seqlens, + output_size=chunked_cu_seqlens[-1], + ) + .unsqueeze(0) + .to(torch.int32) + ) + chunked_input_seq_len = chunked_cu_seqlens[-1] + X_chunked = torch.zeros_like(X)[:, :chunked_input_seq_len, ...] + dt_chunked = torch.zeros_like(dt)[:, :chunked_input_seq_len, ...] + B_chunked = torch.zeros_like(B)[:, :chunked_input_seq_len, ...] + C_chunked = torch.zeros_like(C)[:, :chunked_input_seq_len, ...] + for i in range(num_sequences): + # fmt: off + chunk_f = lambda x, i: x[:, cu_seqlens[i]:cu_seqlens[i] + chunked_seqlens[i], ...] # noqa: E501 + + X_chunked[:, chunked_cu_seqlens[i]:chunked_cu_seqlens[i+1], ...] = chunk_f(X, i) # noqa: E501 + dt_chunked[:, chunked_cu_seqlens[i]:chunked_cu_seqlens[i+1], ...] = chunk_f(dt, i) # noqa: E501 + B_chunked[:, chunked_cu_seqlens[i]:chunked_cu_seqlens[i+1], ...] = chunk_f(B, i) # noqa: E501 + C_chunked[:, chunked_cu_seqlens[i]:chunked_cu_seqlens[i+1], ...] = chunk_f(C, i) # noqa: E501 + # fmt: on + + chunk_indices, chunk_offsets = ( + Mamba2Metadata._query_start_loc_to_chunk_indices_offsets( + chunked_cu_seqlens, chunk_size, chunked_cu_seqlens[-1] + ) + ) + Y_partial = torch.empty_like(X_chunked) + partial_state = mamba_chunk_scan_combined( + X_chunked, + dt_chunked, + A, + B_chunked, + C_chunked, + chunk_size, + D=None, + cu_seqlens=chunked_cu_seqlens, + seq_idx=chunked_seq_idx, + chunk_indices=chunk_indices, + chunk_offsets=chunk_offsets, + return_varlen_states=True, + initial_states=None, + out=Y_partial, + ) + + # remaining chunk + remaining_chunked_seqlens = seqlens - chunked_seqlens + remaining_chunked_cu_seqlens = torch.cat( + [ + torch.tensor([0], device=device), + torch.cumsum(remaining_chunked_seqlens, dim=0), + ], + dim=0, + ) + remaining_chunked_seq_idx = ( + torch.repeat_interleave( + torch.arange(len(remaining_chunked_seqlens), device=device), + remaining_chunked_seqlens, + output_size=remaining_chunked_cu_seqlens[-1], + ) + .unsqueeze(0) + .to(torch.int32) + ) + remaining_chunked_input_seq_len = remaining_chunked_cu_seqlens[-1] + # fmt: off + remaining_X_chunked = torch.zeros_like(X)[:, :remaining_chunked_input_seq_len, ...] # noqa: E501 + remaining_dt_chunked = torch.zeros_like(dt)[:, :remaining_chunked_input_seq_len, ...] # noqa: E501 + remaining_B_chunked = torch.zeros_like(B)[:, :remaining_chunked_input_seq_len, ...] # noqa: E501 + remaining_C_chunked = torch.zeros_like(C)[:, :remaining_chunked_input_seq_len, ...] # noqa: E501 + for i in range(num_sequences): + remaining_chunk_f = lambda x, i: x[:, cu_seqlens[i] + chunked_seqlens[i]:cu_seqlens[i+1], ...] # noqa: E501 + + remaining_X_chunked[:, remaining_chunked_cu_seqlens[i]:remaining_chunked_cu_seqlens[i+1], ...] = remaining_chunk_f(X, i) # noqa: E501 + remaining_dt_chunked[:, remaining_chunked_cu_seqlens[i]:remaining_chunked_cu_seqlens[i+1], ...] = remaining_chunk_f(dt, i) # noqa: E501 + remaining_B_chunked[:, remaining_chunked_cu_seqlens[i]:remaining_chunked_cu_seqlens[i+1], ...] = remaining_chunk_f(B, i) # noqa: E501 + remaining_C_chunked[:, remaining_chunked_cu_seqlens[i]:remaining_chunked_cu_seqlens[i+1], ...] = remaining_chunk_f(C, i) # noqa: E501 + + # assert input chunking is correct + concat_chunk_f = lambda pt1, pt2, i: torch.cat([ + pt1[:,chunked_cu_seqlens[i]:chunked_cu_seqlens[i+1],...], + pt2[:,remaining_chunked_cu_seqlens[i]:remaining_chunked_cu_seqlens[i+1],...], + ], + dim=1) + concat_batch_f = lambda pt1, pt2: torch.cat([concat_chunk_f(pt1, pt2, i) for i in range(num_sequences)], dim=1) # noqa: E501 + # fmt: on + + assert concat_batch_f(X_chunked, remaining_X_chunked).equal(X) + assert concat_batch_f(dt_chunked, remaining_dt_chunked).equal(dt) + assert concat_batch_f(B_chunked, remaining_B_chunked).equal(B) + assert concat_batch_f(C_chunked, remaining_C_chunked).equal(C) + + chunk_indices, chunk_offsets = ( + Mamba2Metadata._query_start_loc_to_chunk_indices_offsets( + remaining_chunked_cu_seqlens, chunk_size, remaining_chunked_cu_seqlens[-1] + ) + ) + + Y_chunked = torch.empty_like(remaining_X_chunked) + state_chunked = mamba_chunk_scan_combined( + remaining_X_chunked, + remaining_dt_chunked, + A, + remaining_B_chunked, + remaining_C_chunked, + chunk_size, + D=None, + cu_seqlens=remaining_chunked_cu_seqlens, + seq_idx=remaining_chunked_seq_idx, + chunk_indices=chunk_indices, + chunk_offsets=chunk_offsets, + return_varlen_states=True, + initial_states=partial_state, + out=Y_chunked, + ) + Y = concat_batch_f(Y_partial, Y_chunked) + + # kernel chunked is same as kernel overall + for i in range(num_sequences): + Y_seq = Y[:, cu_seqlens[i] : cu_seqlens[i + 1], ...] + Y_ref_seq = Y_ref[:, cu_seqlens[i] : cu_seqlens[i + 1], ...] + torch.testing.assert_close( + Y_seq[:, : chunked_seqlens[i], ...], + Y_ref_seq[:, : chunked_seqlens[i], ...], + atol=atol, + rtol=rtol, + msg=lambda x: f"seq{i} output part1 " + x, + ) # noqa: B023 + torch.testing.assert_close( + Y_seq[:, chunked_seqlens[i] :, ...], + Y_ref_seq[:, chunked_seqlens[i] :, ...], + atol=atol, + rtol=rtol, + msg=lambda x: f"seq{i} output part2 " + x, + ) # noqa: B023 + + state_seq = state_chunked[i] + state_seq_ref = state_ref[i] + torch.testing.assert_close( + state_seq, + state_seq_ref, + atol=atol, + rtol=rtol, + msg=lambda x: f"seq{i} state " + x, + ) # noqa: B023 + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_lora.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_lora.py new file mode 100644 index 00000000..3dc08d2f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_lora.py @@ -0,0 +1,57 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import multiprocessing as mp +import os +import sys +import unittest +from pathlib import Path + +# Add test directory to path for lora_utils import +# TODO: can be removed after migration +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) + +from lora_utils import ( + ALL_OTHER_MULTI_LORA_MODELS, + CI_MULTI_LORA_MODELS, + run_lora_multiple_batch_on_model_cases, +) + +from sglang.test.test_utils import CustomTestCase, is_in_ci + + +class TestLoRA(CustomTestCase): + def test_ci_lora_models(self): + run_lora_multiple_batch_on_model_cases(CI_MULTI_LORA_MODELS) + + def test_all_lora_models(self): + if is_in_ci(): + return + + filtered_models = [] + for model_case in ALL_OTHER_MULTI_LORA_MODELS: + if "ONLY_RUN" in os.environ and os.environ["ONLY_RUN"] != model_case.base: + continue + filtered_models.append(model_case) + + run_lora_multiple_batch_on_model_cases(filtered_models) + + +if __name__ == "__main__": + try: + mp.set_start_method("spawn") + except RuntimeError: + pass + + unittest.main(warnings="ignore") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_lora_backend.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_lora_backend.py new file mode 100644 index 00000000..bf9d5b75 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_lora_backend.py @@ -0,0 +1,82 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import multiprocessing as mp +import os +import sys +import unittest +from pathlib import Path +from typing import List + +# Add test directory to path for lora_utils import +# TODO: can be removed after migration +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) + +from lora_utils import ( + ALL_OTHER_LORA_MODELS, + BACKENDS, + CI_LORA_MODELS, + DEFAULT_PROMPTS, + TORCH_DTYPES, + LoRAModelCase, + run_lora_test_one_by_one, +) + +from sglang.test.test_utils import CustomTestCase, is_in_ci + + +class TestLoRABackend(CustomTestCase): + + def _run_backend_on_model_cases(self, model_cases: List[LoRAModelCase]): + for model_case in model_cases: + # If skip_long_prompt is True, filter out prompts longer than 1000 characters + prompts = ( + DEFAULT_PROMPTS + if not model_case.skip_long_prompt + else [p for p in DEFAULT_PROMPTS if len(p) < 1000] + ) + for torch_dtype in TORCH_DTYPES: + for backend in BACKENDS: + run_lora_test_one_by_one( + prompts, + model_case, + torch_dtype, + max_new_tokens=32, + backend=backend, + ) + + def test_ci_lora_models(self): + self._run_backend_on_model_cases(CI_LORA_MODELS) + + def test_all_lora_models(self): + if is_in_ci(): + return + + # Retain ONLY_RUN check here + filtered_models = [] + for model_case in ALL_OTHER_LORA_MODELS: + if "ONLY_RUN" in os.environ and os.environ["ONLY_RUN"] != model_case.base: + continue + filtered_models.append(model_case) + + self._run_backend_on_model_cases(filtered_models) + + +if __name__ == "__main__": + try: + mp.set_start_method("spawn") + except RuntimeError: + pass + + unittest.main(warnings="ignore") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_lora_eviction.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_lora_eviction.py new file mode 100644 index 00000000..fc1e00e3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_lora_eviction.py @@ -0,0 +1,144 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import contextlib +import multiprocessing as mp +import unittest +from typing import Dict, List, Tuple + +import torch + +from sglang.test.runners import SRTRunner +from sglang.test.test_utils import CustomTestCase + +PROMPTS = [ + "AI is a field of computer science focused on", + """ + ### Instruction: + Compose a SQL query that uses the following table: users, and returns the user_id and name of all users whose name that does not have a duplicate in the table. + ### Response: + SELECT user_id, name FROM users WHERE name LIKE 'A%'; + """, +] + +ADAPTERS = [ + "faridlazuarda/valadapt-llama-3.1-8B-it-chinese", # target_modules = q, v + "philschmid/code-llama-3-1-8b-text-to-sql-lora", # target_modules = q, k, v, o, gate, up, down +] + +BASE_MODEL = "meta-llama/Meta-Llama-3.1-8B-Instruct" + + +@contextlib.contextmanager +def dynamically_loaded_adapter(runner, lora_path: str, lora_name: str): + """A context manager to load and automatically unload a LoRA adapter.""" + try: + runner.load_lora_adapter(lora_name=lora_name, lora_path=lora_path) + yield + finally: + runner.unload_lora_adapter(lora_name=lora_name) + + +class TestLoRAEviction(CustomTestCase): + def test_lora_eviction_with_different_target_modules(self): + """ + Test LoRA eviction with different target modules. + + This test runs inference against two LoRA adapters in different orders to force eviction behavior, and ensures + that the outputs of the same (adapter, prompt) pair are consistent across runs. + """ + output_history = {} + self._run_test(ADAPTERS, output_history, reverse=False) + self._run_test(ADAPTERS, output_history, reverse=True) + + def test_lora_eviction_with_reused_lora_name(self): + """ + Test LoRA eviction with reused LoRA names. + + This test runs inference against two LoRA adapters with the same name to ensure that the eviction behavior + works correctly when reusing LoRA names. + """ + output_history = {} + self._run_test(ADAPTERS, output_history, reuse_lora_name=True, repeat=1) + self._run_test(ADAPTERS, output_history, reuse_lora_name=False, repeat=1) + + def _run_test( + self, + lora_paths: List[str], + output_history: Dict[Tuple[str, str], str], + reverse: bool = False, + repeat: int = 2, + reuse_lora_name: bool = False, + ): + REUSED_LORA_NAME = "lora" + max_new_tokens = 256 + torch_dtype = torch.float16 + base_path = BASE_MODEL + assert len(lora_paths) >= 2 + + initial_lora_paths = lora_paths if not reuse_lora_name else None + # Initialize runners + with SRTRunner( + base_path, + torch_dtype=torch_dtype, + model_type="generation", + lora_paths=initial_lora_paths, + max_loras_per_batch=1, + enable_lora=True, + max_lora_rank=256, + lora_target_modules=["all"], + ) as srt_runner: + adapter_sequence = lora_paths if not reverse else lora_paths[::-1] + + for i in range(repeat): + for j, lora_path in enumerate(adapter_sequence): + print( + f"\n========== Testing LoRA eviction with adapter '{lora_path}' (#{j + 1}/{len(adapter_sequence)}), reuse_lora_name: {reuse_lora_name}, reversed: {reverse}, repeat: {i + 1}/{repeat} ---" + ) + + lora_name = REUSED_LORA_NAME if reuse_lora_name else lora_path + context = ( + dynamically_loaded_adapter(srt_runner, lora_path, lora_name) + if reuse_lora_name + else contextlib.nullcontext() + ) + with context: + for prompt in PROMPTS: + print("\nprompt:\n", prompt) + srt_outputs = srt_runner.forward( + [prompt], + max_new_tokens=max_new_tokens, + lora_paths=[lora_name], + ) + output = srt_outputs.output_strs[0].strip() + print("\noutput:\n", output) + + prev_output = output_history.get((lora_path, prompt)) + if prev_output is not None: + self.assertEqual( + prev_output, + output, + f"Output mismatch for adapter {lora_path} and prompt '{prompt}' on repeat {j + 1}, previous: '{prev_output}', current: '{output}'.", + ) + else: + output_history[(lora_path, prompt)] = output + + +if __name__ == "__main__": + try: + mp.set_start_method("spawn") + except RuntimeError: + pass + + unittest.main(warnings="ignore") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_lora_tp.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_lora_tp.py new file mode 100644 index 00000000..8e7b23f6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_lora_tp.py @@ -0,0 +1,83 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import multiprocessing as mp +import os +import sys +import unittest +from pathlib import Path +from typing import List + +# Add test directory to path for lora_utils import +# TODO: can be removed after migration +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) + +from lora_utils import ( + ALL_OTHER_LORA_MODELS, + CI_LORA_MODELS, + DEFAULT_PROMPTS, + TORCH_DTYPES, + LoRAModelCase, + run_lora_test_one_by_one, +) + +from sglang.test.test_utils import CustomTestCase, is_in_ci + + +class TestLoRATP(CustomTestCase): + + def _run_tp_on_model_cases(self, model_cases: List[LoRAModelCase]): + tp_list = [2] # Define TP sizes to iterate over + for model_case in model_cases: + # If skip_long_prompt is True, filter out prompts longer than 1000 characters + prompts = ( + DEFAULT_PROMPTS + if not model_case.skip_long_prompt + else [p for p in DEFAULT_PROMPTS if len(p) < 1000] + ) + for tp_size in tp_list: + model_case.tp_size = tp_size + for torch_dtype in TORCH_DTYPES: + run_lora_test_one_by_one( + prompts, + model_case, + torch_dtype, + max_new_tokens=32, + test_tag=f"tp={tp_size}", + ) + + def test_ci_lora_models(self): + self._run_tp_on_model_cases(CI_LORA_MODELS) + + def test_all_lora_models(self): + if is_in_ci(): + return + + # Retain ONLY_RUN check here + filtered_models = [] + for model_case in ALL_OTHER_LORA_MODELS: + if "ONLY_RUN" in os.environ and os.environ["ONLY_RUN"] != model_case.base: + continue + filtered_models.append(model_case) + + self._run_tp_on_model_cases(filtered_models) + + +if __name__ == "__main__": + try: + mp.set_start_method("spawn") + except RuntimeError: + pass + + unittest.main(warnings="ignore") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_lora_update.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_lora_update.py new file mode 100644 index 00000000..3f11bdd4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_lora_update.py @@ -0,0 +1,1569 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import json +import multiprocessing as mp +import unittest +from dataclasses import dataclass +from enum import Enum +from typing import Any, Iterable, List, Optional, Union + +import requests +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.runners import SRTRunner +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + calculate_rouge_l, + is_in_ci, + popen_launch_server, +) + +PROMPTS = [ + "SGL is a", + "AI is a field of computer science focused on", + "Computer science is the study of", + "Write a short story.", + "What are the main components of a computer?", +] + +MEM_FRACTION_STATIC = 0.8 + + +class OperationType(Enum): + LOAD = "load" + UNLOAD = "unload" + FORWARD = "forward" + + +@dataclass +class Operation: + # Operation type, can be LOAD, UNLOAD, FORWARD + type: OperationType + # Data associated with the operation. Exact type varies depending on the operation + data: Optional[Any] + # If the operation is expected to fail, this is the error message to expect + expected_error: Optional[str] = None + # Because the logic for implicitly evicting LoRA adapters can be complicated, we explicitly + # pass in LoRA adapters that should be implicitly evicted here + expected_implicit_evictions: Optional[set[str]] = None + + +@dataclass +class TestCase: + description: str + base: str + max_loras_per_batch: int + all_adapters: List[str] + op_sequence: List[Operation] + initial_adapters: Optional[List[str]] = None + enable_lora: Optional[bool] = None + max_lora_rank: Optional[int] = None + lora_target_modules: Optional[List] = None + max_new_tokens: int = 32 + max_loaded_loras: Optional[int] = None + + +def create_batch_data(adapters: Union[str, list]) -> List[tuple[str, str]]: + if not isinstance(adapters, list): + adapters = [adapters] + return [(prompt, adapter) for prompt in PROMPTS for adapter in adapters] + + +BASIC_TESTS = [ + TestCase( + description="dynamic lora update with initial lora_paths", + base="meta-llama/Llama-3.1-8B-Instruct", + max_loras_per_batch=3, + all_adapters=[ + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pbevan11/llama-3.1-8b-ocr-correction", + ], + initial_adapters=[ + # Testing 3 supported lora-path formats. + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16=Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + { + "lora_name": "pbevan11/llama-3.1-8b-ocr-correction", + "lora_path": "pbevan11/llama-3.1-8b-ocr-correction", + "pinned": False, + }, + ], + op_sequence=[ + Operation( + type=OperationType.LOAD, + data="philschmid/code-llama-3-1-8b-text-to-sql-lora", + expected_error="already loaded", + ), + Operation( + type=OperationType.UNLOAD, + data="philschmid/code-llama-3-1-8b-text-to-sql-lora", + ), + Operation( + type=OperationType.LOAD, + data="philschmid/code-llama-3-1-8b-text-to-sql-lora", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pbevan11/llama-3.1-8b-ocr-correction", + ] + ), + ), + Operation( + type=OperationType.UNLOAD, + data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ), + Operation( + type=OperationType.UNLOAD, + data="pbevan11/llama-3.1-8b-ocr-correction", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data("philschmid/code-llama-3-1-8b-text-to-sql-lora"), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" + ), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data("pbevan11/llama-3.1-8b-ocr-correction"), + ), + Operation( + type=OperationType.LOAD, + data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + expected_error="already loaded", + ), + Operation( + type=OperationType.LOAD, + data="pbevan11/llama-3.1-8b-ocr-correction", + expected_error="already loaded", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pbevan11/llama-3.1-8b-ocr-correction", + ] + ), + ), + Operation( + type=OperationType.UNLOAD, + data="philschmid/code-llama-3-1-8b-text-to-sql-lora", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data("philschmid/code-llama-3-1-8b-text-to-sql-lora"), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pbevan11/llama-3.1-8b-ocr-correction", + ] + ), + ), + Operation( + type=OperationType.UNLOAD, + data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ), + Operation( + type=OperationType.UNLOAD, + data="pbevan11/llama-3.1-8b-ocr-correction", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" + ), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data("pbevan11/llama-3.1-8b-ocr-correction"), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + None, + ), + ), + ], + ), + TestCase( + description="dynamic lora update without initial lora_paths", + base="meta-llama/Llama-3.1-8B-Instruct", + enable_lora=True, + max_lora_rank=256, + lora_target_modules=["all"], + max_loras_per_batch=4, + all_adapters=[ + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pbevan11/llama-3.1-8b-ocr-correction", + ], + op_sequence=[ + Operation( + type=OperationType.LOAD, + data="philschmid/code-llama-3-1-8b-text-to-sql-lora", + ), + Operation( + type=OperationType.LOAD, + data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ), + Operation( + type=OperationType.LOAD, + data="pbevan11/llama-3.1-8b-ocr-correction", + ), + Operation( + type=OperationType.LOAD, + data="philschmid/code-llama-3-1-8b-text-to-sql-lora", + expected_error="already loaded", + ), + Operation( + type=OperationType.UNLOAD, + data="philschmid/code-llama-3-1-8b-text-to-sql-lora", + ), + Operation( + type=OperationType.LOAD, + data="philschmid/code-llama-3-1-8b-text-to-sql-lora", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pbevan11/llama-3.1-8b-ocr-correction", + None, + ] + ), + ), + Operation( + type=OperationType.UNLOAD, + data="philschmid/code-llama-3-1-8b-text-to-sql-lora", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data("philschmid/code-llama-3-1-8b-text-to-sql-lora"), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + None, + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pbevan11/llama-3.1-8b-ocr-correction", + None, + ] + ), + ), + Operation( + type=OperationType.UNLOAD, + data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ), + Operation( + type=OperationType.UNLOAD, + data="pbevan11/llama-3.1-8b-ocr-correction", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" + ), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data("pbevan11/llama-3.1-8b-ocr-correction"), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data(None), + ), + Operation( + type=OperationType.LOAD, + data="philschmid/code-llama-3-1-8b-text-to-sql-lora", + expected_error="already loaded", + ), + Operation( + type=OperationType.LOAD, + data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + expected_error="already loaded", + ), + Operation( + type=OperationType.LOAD, + data="pbevan11/llama-3.1-8b-ocr-correction", + expected_error="already loaded", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pbevan11/llama-3.1-8b-ocr-correction", + None, + ] + ), + ), + ], + ), +] +TARGET_MODULE_TESTS = [ + TestCase( + description="Test explicitly specified lora-target-modules.", + base="meta-llama/Llama-3.1-8B-Instruct", + max_loras_per_batch=3, + lora_target_modules=[ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj", + ], + max_lora_rank=64, + all_adapters=[ + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # target_modules = q, k, v, o, gate, up, down + "algoprog/fact-generation-llama-3.1-8b-instruct-lora", # target_modules = q, k, v, o, gate + ], + initial_adapters=["algoprog/fact-generation-llama-3.1-8b-instruct-lora"], + op_sequence=[ + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "algoprog/fact-generation-llama-3.1-8b-instruct-lora" + ), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" + ), + expected_error="never been loaded", + ), + Operation( + type=OperationType.LOAD, + data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "algoprog/fact-generation-llama-3.1-8b-instruct-lora", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + None, + ] + ), + ), + ], + ), + TestCase( + description="Test inferred lora-target-modules - start with larger adapter", + base="meta-llama/Llama-3.1-8B-Instruct", + max_loras_per_batch=3, + max_lora_rank=64, + all_adapters=[ + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # target_modules = q, k, v, o, gate, up, down + "algoprog/fact-generation-llama-3.1-8b-instruct-lora", # target_modules = q, k, v, o, gate + ], + initial_adapters=["Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"], + op_sequence=[ + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" + ), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "algoprog/fact-generation-llama-3.1-8b-instruct-lora" + ), + expected_error="never been loaded", + ), + Operation( + type=OperationType.LOAD, + data="algoprog/fact-generation-llama-3.1-8b-instruct-lora", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "algoprog/fact-generation-llama-3.1-8b-instruct-lora", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + None, + ] + ), + ), + ], + ), + TestCase( + description="Test inferred lora-target-modules - start with smaller adapter", + base="meta-llama/Llama-3.1-8B-Instruct", + max_loras_per_batch=3, + max_lora_rank=64, + all_adapters=[ + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # target_modules = q, k, v, o, gate, up, down + "algoprog/fact-generation-llama-3.1-8b-instruct-lora", # target_modules = q, k, v, o, gate + ], + initial_adapters=["algoprog/fact-generation-llama-3.1-8b-instruct-lora"], + op_sequence=[ + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "algoprog/fact-generation-llama-3.1-8b-instruct-lora" + ), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" + ), + expected_error="never been loaded", + ), + Operation( + type=OperationType.LOAD, + data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + expected_error="incompatible", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "algoprog/fact-generation-llama-3.1-8b-instruct-lora", + None, + ] + ), + ), + ], + ), +] +MAX_LORA_RANK_TESTS = [ + TestCase( + description="Test explicitly specified max-lora-rank.", + base="meta-llama/Llama-3.1-8B-Instruct", + max_loras_per_batch=3, + max_lora_rank=32, + all_adapters=[ + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # r = 4 + "pbevan11/llama-3.1-8b-ocr-correction", # r = 32 + "philschmid/code-llama-3-1-8b-text-to-sql-lora", # r = 256 + ], + initial_adapters=["Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"], + op_sequence=[ + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" + ), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data("philschmid/code-llama-3-1-8b-text-to-sql-lora"), + expected_error="never been loaded", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data("pbevan11/llama-3.1-8b-ocr-correction"), + expected_error="never been loaded", + ), + Operation( + type=OperationType.LOAD, + data="pbevan11/llama-3.1-8b-ocr-correction", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "pbevan11/llama-3.1-8b-ocr-correction", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + None, + ] + ), + ), + Operation( + type=OperationType.LOAD, + data="philschmid/code-llama-3-1-8b-text-to-sql-lora", + expected_error="incompatible", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + ), + expected_error="never been loaded", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "pbevan11/llama-3.1-8b-ocr-correction", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + None, + ] + ), + ), + ], + ), + TestCase( + description="test implicitly inferred max-lora-rank", + base="meta-llama/Llama-3.1-8B-Instruct", + max_loras_per_batch=3, + all_adapters=[ + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # r = 4 + "pbevan11/llama-3.1-8b-ocr-correction", # r = 32 + "philschmid/code-llama-3-1-8b-text-to-sql-lora", # r = 256 + ], + initial_adapters=["pbevan11/llama-3.1-8b-ocr-correction"], + op_sequence=[ + Operation( + type=OperationType.FORWARD, + data=create_batch_data("pbevan11/llama-3.1-8b-ocr-correction"), + ), + Operation( + type=OperationType.LOAD, + data="philschmid/code-llama-3-1-8b-text-to-sql-lora", + expected_error="incompatible", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data("philschmid/code-llama-3-1-8b-text-to-sql-lora"), + expected_error="never been loaded", + ), + Operation( + type=OperationType.LOAD, + data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pbevan11/llama-3.1-8b-ocr-correction", + None, + ] + ), + ), + ], + ), +] +MAX_LOADED_LORAS_TESTS = [ + TestCase( + description="Test max_loaded_loras limit as well as implicit eviction and reloading", + base="meta-llama/Llama-3.1-8B-Instruct", + max_loras_per_batch=2, + max_loaded_loras=2, + all_adapters=[ + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pbevan11/llama-3.1-8b-ocr-correction", + ], + initial_adapters=["philschmid/code-llama-3-1-8b-text-to-sql-lora"], + op_sequence=[ + Operation( + type=OperationType.LOAD, + data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ), + Operation( + type=OperationType.LOAD, + data="pbevan11/llama-3.1-8b-ocr-correction", + expected_implicit_evictions={ + "philschmid/code-llama-3-1-8b-text-to-sql-lora" + }, + ), + # Implicitly load "philschmid/code-llama-3-1-8b-text-to-sql-lora" + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + ] + ), + expected_implicit_evictions={"pbevan11/llama-3.1-8b-ocr-correction"}, + ), + Operation( + type=OperationType.UNLOAD, + data="philschmid/code-llama-3-1-8b-text-to-sql-lora", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ] + ), + ), + Operation( + type=OperationType.LOAD, + data="philschmid/code-llama-3-1-8b-text-to-sql-lora", + ), + # Implicitly load "pbevan11/llama-3.1-8b-ocr-correction" and make sure that "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" + # isn't implicitly unloaded even though it is LRU because it is needed for this forward pass + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pbevan11/llama-3.1-8b-ocr-correction", + ] + ), + expected_implicit_evictions={ + "philschmid/code-llama-3-1-8b-text-to-sql-lora" + }, + ), + Operation( + type=OperationType.UNLOAD, + data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ), + Operation( + type=OperationType.LOAD, + data="algoprog/fact-generation-llama-3.1-8b-instruct-lora", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + ] + ), + expected_implicit_evictions={ + "pbevan11/llama-3.1-8b-ocr-correction", + "algoprog/fact-generation-llama-3.1-8b-instruct-lora", + }, + ), + ], + ), + TestCase( + description="Test implicit eviction and reloading with pinned LoRA adapters", + base="meta-llama/Llama-3.1-8B-Instruct", + max_loras_per_batch=2, + max_loaded_loras=2, + all_adapters=[ + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pbevan11/llama-3.1-8b-ocr-correction", + ], + initial_adapters=[ + { + "lora_name": "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "lora_path": "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "pinned": True, + } + ], + op_sequence=[ + Operation( + type=OperationType.LOAD, + data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ), + Operation( + type=OperationType.LOAD, + data="pbevan11/llama-3.1-8b-ocr-correction", + expected_implicit_evictions={ + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" + }, + ), + # Implicitly load "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ] + ), + expected_implicit_evictions={"pbevan11/llama-3.1-8b-ocr-correction"}, + ), + Operation( + type=OperationType.LOAD, + data={ + "lora_name": "pbevan11/llama-3.1-8b-ocr-correction", + "lora_path": "pbevan11/llama-3.1-8b-ocr-correction", + "pinned": True, + }, + expected_error="starvation", + ), + Operation( + type=OperationType.UNLOAD, + data="philschmid/code-llama-3-1-8b-text-to-sql-lora", + ), + Operation( + type=OperationType.LOAD, + data={ + "lora_name": "pbevan11/llama-3.1-8b-ocr-correction", + "lora_path": "pbevan11/llama-3.1-8b-ocr-correction", + "pinned": True, + }, + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pbevan11/llama-3.1-8b-ocr-correction", + ] + ), + ), + ], + ), +] +EVICTION_TESTS = [ + TestCase( + description="dynamic lora update with evictions", + base="meta-llama/Llama-3.1-8B-Instruct", + max_loras_per_batch=2, + all_adapters=[ + "lora1=philschmid/code-llama-3-1-8b-text-to-sql-lora", + "lora2=Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "lora3=pbevan11/llama-3.1-8b-ocr-correction", + ], + enable_lora=True, + max_lora_rank=256, + lora_target_modules=["all"], + op_sequence=[ + Operation( + type=OperationType.LOAD, + data={ + "lora_name": "lora1", + "lora_path": "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "pinned": True, + }, + ), + Operation( + type=OperationType.LOAD, + data={ + "lora_name": "lora2", + "lora_path": "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pinned": True, + }, + expected_error="starvation", + ), + Operation( + type=OperationType.LOAD, + data={ + "lora_name": "lora2", + "lora_path": "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pinned": False, + }, + ), + Operation( + type=OperationType.LOAD, + data={ + "lora_name": "lora3", + "lora_path": "pbevan11/llama-3.1-8b-ocr-correction", + "pinned": False, + }, + ), + Operation( + type=OperationType.UNLOAD, + data="lora1", + ), + Operation( + type=OperationType.UNLOAD, + data="lora3", + ), + Operation( + type=OperationType.LOAD, + data={ + "lora_name": "lora3", + "lora_path": "pbevan11/llama-3.1-8b-ocr-correction", + "pinned": True, + }, + ), + Operation( + type=OperationType.LOAD, + data={ + "lora_name": "lora1", + "lora_path": "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "pinned": True, + }, + expected_error="starvation", + ), + Operation( + type=OperationType.LOAD, + data={ + "lora_name": "lora1", + "lora_path": "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "pinned": False, + }, + ), + # pinned: lora3 + # unpinned: lora1, lora2 + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "lora1", + "lora2", + ] + ), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "lora1", + "lora3", + ] + ), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "lora1", + "lora2", + ] + ), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "lora1", + "lora2", + None, + ] + ), + ), + ], + ), +] + +ALL_TESTS = ( + BASIC_TESTS + + TARGET_MODULE_TESTS + + MAX_LORA_RANK_TESTS + + MAX_LOADED_LORAS_TESTS + + EVICTION_TESTS +) + + +class LoRAUpdateTestSessionMode(Enum): + ENGINE = "engine" + SERVER = "server" + + +class LoRAUpdateTestSessionBase: + """ + Base context manager for testing LoRA adapters. + """ + + def __init__( + self, + *, + testcase: Optional[TestCase], + model_path: str, + lora_paths: List[Union[str, dict]], + max_loras_per_batch: int, + max_loaded_loras: Optional[int] = None, + max_lora_rank: Optional[int], + enable_lora: Optional[bool] = None, + lora_target_modules: Optional[List[str]] = None, + lora_backend: str = "csgmv", + disable_cuda_graph: bool = False, + cuda_graph_max_bs: int = 4, + ): + self.testcase = testcase + self.model_path = model_path + self.lora_paths = lora_paths + self.max_lora_rank = max_lora_rank + self.lora_target_modules = lora_target_modules + self.max_loras_per_batch = max_loras_per_batch + self.max_loaded_loras = max_loaded_loras + self.lora_backend = lora_backend + self.disable_cuda_graph = disable_cuda_graph + self.cuda_graph_max_bs = cuda_graph_max_bs + self.enable_lora = enable_lora + + self.expected_adapters = set() + if self.lora_paths: + for adapter in self.lora_paths: + if isinstance(adapter, dict): + lora_name = adapter["lora_name"] + elif "=" in adapter: + lora_name = adapter.split("=")[0] + else: + lora_name = adapter + self.expected_adapters.add(lora_name) + + self.handle = None # Will be set in __enter__ + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + # Don't suppress exceptions by default + return False + + def load_lora_adapter( + self, + lora_name: str, + lora_path: Optional[str] = None, + expected_error: Optional[str] = None, + expected_implicit_evictions: Optional[set[str]] = None, + ): + """ + Load a LoRA adapter by name and path. + """ + raise NotImplementedError("Subclasses must implement load_lora_adapter") + + def unload_lora_adapter(self, lora_name: str): + """ + Unload a LoRA adapter by name. + """ + raise NotImplementedError("Subclasses must implement unload_lora_adapter") + + def forward( + self, + prompts: List[str], + lora_paths: List[str], + max_new_tokens: int = 32, + expected_error: Optional[str] = None, + expected_implicit_evictions: Optional[set[str]] = None, + ): + """ + Perform a batch forward pass with the current set of loaded LoRA adapters. + """ + raise NotImplementedError("Subclasses must implement forward") + + +class LoRAUpdateEngineTestSession(LoRAUpdateTestSessionBase): + """ + Context manager for testing LoRA adapters with in-process engine. + """ + + def __enter__(self): + # in-process runner + self.handle = SRTRunner( + model_path=self.model_path, + model_type="generation", + lora_paths=self.lora_paths, + max_lora_rank=self.max_lora_rank, + lora_target_modules=self.lora_target_modules, + lora_backend=self.lora_backend, + torch_dtype=torch.float16, + mem_fraction_static=MEM_FRACTION_STATIC, + max_loras_per_batch=self.max_loras_per_batch, + max_loaded_loras=self.max_loaded_loras, + disable_cuda_graph=self.disable_cuda_graph, + cuda_graph_max_bs=self.cuda_graph_max_bs, + enable_lora=self.enable_lora, + disable_radix_cache=True, + ) + self.handle.__enter__() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.handle is not None: + # delegate cleanup to SRTRunner + return self.handle.__exit__(exc_type, exc_val, exc_tb) + # don't suppress exceptions + return False + + def load_lora_adapter( + self, + lora_name: str, + lora_path: Optional[str] = None, + expected_error: Optional[str] = None, + pinned: bool = False, + expected_implicit_evictions: Optional[set[str]] = None, + ): + """ + Load a LoRA adapter by name and path. + """ + if lora_path is None: + lora_path = lora_name + + response = self.handle.load_lora_adapter( + lora_name=lora_name, + lora_path=lora_path, + pinned=pinned, + ) + if expected_error: + self.testcase.assertFalse( + response.success, f"Expected failure for {lora_name}, but got success." + ) + self.testcase.assertIn( + expected_error, + response.error_message, + f"Expected error message to contain '{expected_error}', but got '{response.error_message}'", + ) + print(f"Received error as expected: {response.error_message}") + else: + self.expected_adapters.add(lora_name) + if expected_implicit_evictions is not None: + self.expected_adapters -= expected_implicit_evictions + + self.testcase.assertTrue( + response.success, + f"Failed to load LoRA adapter {lora_name}: {response.error_message}", + ) + loaded_adapters = set(response.loaded_adapters) + print(f"loaded_adapters: {loaded_adapters}") + self.testcase.assertEqual( + loaded_adapters, + self.expected_adapters, + f"Expected loaded adapters to be {self.expected_adapters}, but got {loaded_adapters}", + ) + + def unload_lora_adapter(self, lora_name: str): + """ + Unload a LoRA adapter by name. + """ + self.expected_adapters.remove(lora_name) + + response = self.handle.unload_lora_adapter( + lora_name=lora_name, + ) + self.testcase.assertTrue( + response.success, + f"Failed to unload LoRA adapter {lora_name}: {response.error_message}", + ) + loaded_adapters = set(response.loaded_adapters) + + print(f"loaded_adapters: {loaded_adapters}") + self.testcase.assertEqual( + loaded_adapters, + self.expected_adapters, + f"Expected loaded adapters to be {self.expected_adapters}, but got {loaded_adapters}", + ) + + def forward( + self, + prompts: List[str], + lora_paths: List[str], + max_new_tokens: int = 32, + expected_error: Optional[str] = None, + expected_implicit_evictions: Optional[set[str]] = None, + ): + """ + Perform a batch forward pass with the current set of loaded LoRA adapters. + """ + try: + response = self.handle.batch_forward( + prompts=prompts, + lora_paths=lora_paths, + max_new_tokens=max_new_tokens, + ) + except ValueError as e: + if expected_error: + error_message = str(e) + self.testcase.assertIn( + expected_error, + error_message, + f"Expected error message to contain '{expected_error}', but got '{error_message}'", + ) + print(f"Received error as expected: {error_message}") + return error_message + + raise e + + self.testcase.assertEqual( + len(response.output_strs), + len(prompts), + f"Expected {len(prompts)} outputs, but got {len(response.output_strs)}", + ) + output = response.output_strs + print(f"output_strs: {output}") + + self.expected_adapters.update( + [lora_path for lora_path in lora_paths if lora_path is not None] + ) + + if expected_implicit_evictions is not None: + self.expected_adapters -= expected_implicit_evictions + + return output + + +class LoRAUpdateServerTestSession(LoRAUpdateTestSessionBase): + """ + Context manager for testing LoRA adapters with standalone server. + """ + + def __enter__(self): + other_args = [ + "--cuda-graph-max-bs", + str(self.cuda_graph_max_bs), + "--max-loras-per-batch", + str(self.max_loras_per_batch), + "--lora-backend", + self.lora_backend, + "--random-seed", + "42", + "--max-running-request", + "1", + "--mem-fraction-static", + str(MEM_FRACTION_STATIC), + "--disable-radix-cache", + ] + if self.enable_lora: + other_args.append("--enable-lora") + if self.lora_paths: + other_args.append("--lora-paths") + for lora_path in self.lora_paths: + if isinstance(lora_path, dict): + lora_path = json.dumps(lora_path) + other_args.append(lora_path) + if self.disable_cuda_graph: + other_args.append("--disable-cuda-graph") + if self.max_lora_rank is not None: + other_args.extend(["--max-lora-rank", str(self.max_lora_rank)]) + if self.lora_target_modules is not None: + other_args.extend(["--lora-target-modules"] + self.lora_target_modules) + if self.max_loaded_loras is not None: + other_args.extend(["--max-loaded-loras", str(self.max_loaded_loras)]) + + # launch external server + self.handle = popen_launch_server( + self.model_path, + DEFAULT_URL_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.handle is not None: + kill_process_tree(self.handle.pid) + # don't suppress exceptions + return False + + def load_lora_adapter( + self, + lora_name: str, + lora_path: Optional[str] = None, + expected_error: Optional[str] = None, + pinned: bool = False, + expected_implicit_evictions: Optional[set[str]] = None, + ): + """ + Load a LoRA adapter by name and path. + """ + if lora_path is None: + lora_path = lora_name + + response = requests.post( + DEFAULT_URL_FOR_TEST + "/load_lora_adapter", + json={"lora_name": lora_name, "lora_path": lora_path, "pinned": pinned}, + ) + if expected_error: + self.testcase.assertEqual( + response.status_code, + 400, + f"Expected error for {lora_name}, but got success.", + ) + self.testcase.assertIn( + expected_error, + response.text, + f"Expected error message to contain '{expected_error}', but got '{response.text}'", + ) + print(f"Received error as expected: {response.text}") + else: + self.expected_adapters.add(lora_name) + if expected_implicit_evictions is not None: + self.expected_adapters -= expected_implicit_evictions + + self.testcase.assertTrue( + response.ok, f"Failed to load LoRA adapter {lora_name}: {response.text}" + ) + loaded_adapters = set(response.json()["loaded_adapters"]) + print(f"loaded_adapters: {loaded_adapters}") + self.testcase.assertEqual( + loaded_adapters, + self.expected_adapters, + f"Expected loaded adapters to be {self.expected_adapters}, but got {loaded_adapters}", + ) + + def unload_lora_adapter(self, lora_name: str): + """ + Unload a LoRA adapter by name. + """ + self.expected_adapters.remove(lora_name) + + response = requests.post( + DEFAULT_URL_FOR_TEST + "/unload_lora_adapter", + json={"lora_name": lora_name}, + ) + + self.testcase.assertTrue( + response.ok, f"Failed to unload LoRA adapter {lora_name}: {response.text}" + ) + loaded_adapters = set(response.json()["loaded_adapters"]) + + print(f"loaded_adapters: {loaded_adapters}") + self.testcase.assertEqual( + loaded_adapters, + self.expected_adapters, + f"Expected loaded adapters to be {self.expected_adapters}, but got {loaded_adapters}", + ) + + def forward( + self, + prompts: List[str], + lora_paths: List[str], + max_new_tokens: int = 32, + expected_error: Optional[str] = None, + expected_implicit_evictions: Optional[set[str]] = None, + ): + """ + Perform a batch forward pass with the current set of loaded LoRA adapters. + """ + response = requests.post( + DEFAULT_URL_FOR_TEST + "/generate", + json={ + "text": prompts, + "lora_path": lora_paths, + "sampling_params": { + "temperature": 0, + "top_k": 1, + "max_new_tokens": max_new_tokens, + }, + }, + ) + if expected_error: + self.testcase.assertEqual( + response.status_code, + 400, + f"Expected error for forward pass, but got success: {response.text}", + ) + self.testcase.assertIn( + expected_error, + response.text, + f"Expected error message to contain '{expected_error}', but got '{response.text}'", + ) + output = response.text + print(f"Received error as expected: {response.text}") + return output + else: + self.testcase.assertTrue( + response.ok, f"Failed to generate text: {response.text}" + ) + output = [r["text"] for r in response.json()] + self.testcase.assertEqual( + len(output), + len(prompts), + f"Expected {len(prompts)} outputs, but got {len(output)}", + ) + print(f"output_strs: {output}") + + self.expected_adapters.update( + [lora_path for lora_path in lora_paths if lora_path is not None] + ) + + if expected_implicit_evictions is not None: + self.expected_adapters -= expected_implicit_evictions + + return output + + +# Factory function to create the appropriate LoRA test session based on mode +def LoRAUpdateTestSession( + testcase: Optional[TestCase], + mode: LoRAUpdateTestSessionMode, + **kwargs: Any, +): + if mode == LoRAUpdateTestSessionMode.ENGINE: + return LoRAUpdateEngineTestSession(testcase=testcase, **kwargs) + elif mode == LoRAUpdateTestSessionMode.SERVER: + return LoRAUpdateServerTestSession(testcase=testcase, **kwargs) + else: + raise ValueError(f"Unrecognized mode: {mode!r}") + + +class TestLoRADynamicUpdate(CustomTestCase): + """ + This test case verifies that the SRT runner can dynamically load and unload LoRA adapters + during a sequence of operations, and that the outputs of forward passes with dynamically loaded + adapters match the outputs of forward passes with statically loaded adapters. + """ + + def _repeat_each(lst, n): + return [x for x in lst for _ in range(n)] + + def _run_operation_sequence( + self, + mode: LoRAUpdateTestSessionMode, + base: str, + initial_adapters: List[Union[str, dict]], + op_sequence: List[Operation], + max_loras_per_batch: int, + max_loaded_loras: Optional[int] = None, + enable_lora: Optional[bool] = None, + max_lora_rank: Optional[int] = None, + lora_target_modules: Optional[List[str]] = None, + max_new_tokens: int = 32, + ) -> List[tuple]: + """ + Runs a sequence of operations on the SRT runner, including loading and unloading LoRA adapters, + and performing forward passes with the current set of loaded adapters. + """ + + forward_outputs = [] + with LoRAUpdateTestSession( + testcase=self, + mode=mode, + model_path=base, + lora_paths=initial_adapters, + max_loras_per_batch=max_loras_per_batch, + max_loaded_loras=max_loaded_loras, + max_lora_rank=max_lora_rank, + lora_target_modules=lora_target_modules, + enable_lora=enable_lora, + ) as session: + for op in op_sequence: + op_type = op.type + data = op.data + expected_error = op.expected_error + expected_implicit_evictions = op.expected_implicit_evictions + print("-" * 100) + print( + f"Running operation: {op_type} --- data: {data} --- mode: {mode} ---" + ) + if op_type == OperationType.LOAD: + if isinstance(data, str): + adapter_info = { + "lora_name": data, + "lora_path": data, + "pinned": False, + } + else: + adapter_info = data + + result = session.load_lora_adapter( + expected_error=expected_error, + expected_implicit_evictions=expected_implicit_evictions, + **adapter_info, + ) + elif op_type == OperationType.UNLOAD: + result = session.unload_lora_adapter( + lora_name=data, + ) + elif op_type == OperationType.FORWARD: + prompts, adapters = zip(*data) + result = session.forward( + prompts=list(prompts), + lora_paths=list(adapters), + max_new_tokens=max_new_tokens, + expected_error=expected_error, + expected_implicit_evictions=expected_implicit_evictions, + ) + if not expected_error: + forward_outputs.append(result) + + return forward_outputs + + def _run_dynamic_adapter_updates( + self, mode: LoRAUpdateTestSessionMode, test_cases: Iterable[TestCase] + ): + for case_idx, test_case in enumerate(test_cases, start=1): + print("=" * 100) + print( + f"Starting test case {case_idx} in {mode.value} mode. Test description: {test_case.description}" + ) + print("=" * 100) + + print( + f"--- Running dynamic update pass with {len(test_case.op_sequence)} operations ---" + ) + # Test dynamic loading of adapters + dynamic_output = self._run_operation_sequence( + mode=mode, + initial_adapters=test_case.initial_adapters, + enable_lora=test_case.enable_lora, + base=test_case.base, + max_loras_per_batch=test_case.max_loras_per_batch, + max_loaded_loras=test_case.max_loaded_loras, + op_sequence=test_case.op_sequence, + max_new_tokens=test_case.max_new_tokens, + max_lora_rank=test_case.max_lora_rank, + lora_target_modules=test_case.lora_target_modules, + ) + + # static loading + forward_ops = [ + x + for x in test_case.op_sequence + if x.type == OperationType.FORWARD and x.expected_error is None + ] + + if not forward_ops: + print( + f"No forward operations found in test case {case_idx}. Skipping static pass." + ) + continue + + print("=" * 100) + print(f"\n--- Running static pass with {len(forward_ops)} operations ---") + static_output = self._run_operation_sequence( + mode=mode, + initial_adapters=test_case.all_adapters, + enable_lora=test_case.enable_lora, + base=test_case.base, + max_loras_per_batch=test_case.max_loras_per_batch, + op_sequence=forward_ops, + max_new_tokens=test_case.max_new_tokens, + ) + + ROUGE_L_TOL = 0.9 + + print(f"Dynamic output: {dynamic_output}") + print(f"Static output: {static_output}") + print("=" * 100) + self.assertEqual( + len(dynamic_output), + len(static_output), + f"Dynamic output length {len(dynamic_output)} does not match static output length {len(static_output)}", + ) + for i, (dynamic, static) in enumerate( + zip(dynamic_output, static_output), start=1 + ): + self.assertEqual( + len(dynamic), + len(static), + f"Output length mismatch at batch {i}:\n- Dynamic={len(dynamic)}\n- Static={len(static)}", + ) + for j, (d_out, s_out) in enumerate(zip(dynamic, static), start=1): + d_out_str = d_out.strip() + s_out_str = s_out.strip() + rouge_score = calculate_rouge_l([d_out_str], [s_out_str])[0] + + self.assertGreaterEqual( + rouge_score, + ROUGE_L_TOL, + f"ROUGE-L score {rouge_score} of outputs is below tolerance of {ROUGE_L_TOL} " + f"at batch {i}, prompt {j}:\n- Dynamic: '{d_out}'\n- Static: '{s_out}'", + ) + + def test_dynamic_lora_update_engine(self): + """ + Test dynamic LoRA updates in engine mode. + """ + test_cases = BASIC_TESTS if is_in_ci() else ALL_TESTS + self._run_dynamic_adapter_updates( + mode=LoRAUpdateTestSessionMode.ENGINE, + test_cases=test_cases, + ) + + def test_dynamic_lora_update_server(self): + """ + Test dynamic LoRA updates in server mode. + """ + test_cases = BASIC_TESTS if is_in_ci() else ALL_TESTS + self._run_dynamic_adapter_updates( + mode=LoRAUpdateTestSessionMode.SERVER, test_cases=test_cases + ) + + def test_v1_models_endpoint_with_lora(self): + """ + Test that /v1/models endpoint returns base model and loaded LoRA adapters. + """ + adapters = [ + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ] + + with LoRAUpdateTestSession( + testcase=self, + mode=LoRAUpdateTestSessionMode.SERVER, + model_path="meta-llama/Llama-3.1-8B-Instruct", + lora_paths=[], + max_loras_per_batch=2, + max_lora_rank=256, + lora_target_modules=["all"], + enable_lora=True, + ) as session: + # Test with no adapters loaded + response = requests.get(DEFAULT_URL_FOR_TEST + "/v1/models") + self.assertTrue(response.ok, response.text) + models_data = response.json() + self.assertEqual(models_data["object"], "list") + self.assertEqual(len(models_data["data"]), 1) # Only base model + base_model = models_data["data"][0] + self.assertIn("meta-llama", base_model["id"].lower()) + self.assertIsNone(base_model.get("parent")) + + # Load first adapter + session.load_lora_adapter(lora_name="adapter1", lora_path=adapters[0]) + + # Test with one adapter loaded + response = requests.get(DEFAULT_URL_FOR_TEST + "/v1/models") + self.assertTrue(response.ok, response.text) + models_data = response.json() + self.assertEqual(len(models_data["data"]), 2) # Base model + 1 adapter + + # Verify adapter information + adapter_models = [m for m in models_data["data"] if m.get("parent")] + self.assertEqual(len(adapter_models), 1) + self.assertEqual(adapter_models[0]["id"], "adapter1") + self.assertEqual(adapter_models[0]["root"], adapters[0]) + self.assertIsNotNone(adapter_models[0]["parent"]) + + # Load second adapter + session.load_lora_adapter(lora_name="adapter2", lora_path=adapters[1]) + + # Test with two adapters loaded + response = requests.get(DEFAULT_URL_FOR_TEST + "/v1/models") + self.assertTrue(response.ok, response.text) + models_data = response.json() + self.assertEqual(len(models_data["data"]), 3) # Base model + 2 adapters + + # Verify both adapters are listed + adapter_models = [m for m in models_data["data"] if m.get("parent")] + self.assertEqual(len(adapter_models), 2) + adapter_names = {m["id"] for m in adapter_models} + self.assertEqual(adapter_names, {"adapter1", "adapter2"}) + + # Unload one adapter + session.unload_lora_adapter(lora_name="adapter1") + + # Test after unloading + response = requests.get(DEFAULT_URL_FOR_TEST + "/v1/models") + self.assertTrue(response.ok, response.text) + models_data = response.json() + self.assertEqual(len(models_data["data"]), 2) # Base model + 1 adapter + adapter_models = [m for m in models_data["data"] if m.get("parent")] + self.assertEqual(len(adapter_models), 1) + self.assertEqual(adapter_models[0]["id"], "adapter2") + + +if __name__ == "__main__": + try: + mp.set_start_method("spawn") + except RuntimeError: + pass + + unittest.main(warnings="ignore") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_multi_lora_backend.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_multi_lora_backend.py new file mode 100644 index 00000000..84def481 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/lora/test_multi_lora_backend.py @@ -0,0 +1,72 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import multiprocessing as mp +import os +import sys +import unittest +from pathlib import Path + +# Add test directory to path for lora_utils import +# TODO: can be removed after migration +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) + +from lora_utils import ( + ALL_OTHER_MULTI_LORA_MODELS, + CI_MULTI_LORA_MODELS, + run_lora_multiple_batch_on_model_cases, +) + +from sglang.test.test_utils import CustomTestCase, is_in_ci + +# All prompts are used at once in a batch. +PROMPTS = [ + "AI is a field of computer science focused on", + """ + ### Instruction: + Tell me about llamas and alpacas + ### Response: + Llamas are large, long-necked animals with a woolly coat. They have two toes on each foot instead of three like other camelids. + ### Question: + What do you know about llamas? + ### Answer: + """, +] + + +class TestMultiLoRABackend(CustomTestCase): + def test_ci_lora_models(self): + run_lora_multiple_batch_on_model_cases(CI_MULTI_LORA_MODELS) + + def test_all_lora_models(self): + if is_in_ci(): + return + + # Retain ONLY_RUN check here + filtered_models = [] + for model_case in ALL_OTHER_MULTI_LORA_MODELS: + if "ONLY_RUN" in os.environ and os.environ["ONLY_RUN"] != model_case.base: + continue + filtered_models.append(model_case) + + run_lora_multiple_batch_on_model_cases(filtered_models) + + +if __name__ == "__main__": + try: + mp.set_start_method("spawn") + except RuntimeError: + pass + + unittest.main(warnings="ignore") diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/compare.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/compare.py new file mode 100644 index 00000000..2fe35357 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/compare.py @@ -0,0 +1,52 @@ +""" +used for debug using tensor comparison +dump {name: tensor} into "log_hf.jsonl" and "log_srt.jsonl" +use the same name for two tensors that supposed to be close +recommend name like: "layer 2 after mlp" +""" + +import json +import sys + +import torch + +if len(sys.argv) > 1: + assert sys.argv[1] == "base" + hf_log = "base_log_hf.jsonl" + srt_log = "base_log_srt.jsonl" +else: + hf_log = "log_hf.jsonl" + srt_log = "log_srt.jsonl" + + +def load_data(filepath): + tensors = {} + with open(filepath, "r") as f: + lines = f.readlines() + for line in lines: + data = json.loads(line) + for k, v in data.items(): + tensors[k] = torch.tensor(v) + return tensors + + +hf_tensors = load_data(hf_log) +srt_tensors = load_data(srt_log) + + +def get_diff(t1, t2): + t1 = t1.reshape(t2.shape) + max_diff = torch.max(abs(t1.reshape(t2.shape) - t2)) + l2_dis = torch.dist(t1, t2, p=2) + return l2_dis, max_diff + + +for k, _ in srt_tensors.items(): + l2_dis, max_diff = get_diff(hf_tensors[k], srt_tensors[k]) + print(f"{k} {l2_dis=} {max_diff=}") + if k == "layer 1 attn": + print(hf_tensors[k]) + print(srt_tensors[k]) + if k == "layer 0 prefill k": + print(srt_tensors[k].shape) + print(hf_tensors[k].shape) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_compressed_tensors_models.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_compressed_tensors_models.py new file mode 100644 index 00000000..34f699de --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_compressed_tensors_models.py @@ -0,0 +1,46 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestCompressedTensorsLlama3FP8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "RedHatAI/Meta-Llama-3.1-8B-FP8" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreaterEqual(metrics["accuracy"], 0.45) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_cross_encoder_models.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_cross_encoder_models.py new file mode 100644 index 00000000..6ff963dd --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_cross_encoder_models.py @@ -0,0 +1,90 @@ +import multiprocessing as mp +import random +import unittest + +import torch + +from sglang.test.runners import TEST_RERANK_QUERY_DOCS, HFRunner, SRTRunner +from sglang.test.test_utils import CustomTestCase, is_in_ci + +MODELS = [ + ("cross-encoder/ms-marco-MiniLM-L6-v2", 1, 1e-2), + ("BAAI/bge-reranker-v2-m3", 1, 1e-2), +] +ATTENTION_BACKEND = ["torch_native", "triton"] + +TORCH_DTYPES = [torch.float32] + + +class TestCrossEncoderModels(CustomTestCase): + + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + + def assert_close_prefill_logits( + self, + prompts, + model_path, + tp_size, + torch_dtype, + score_tolerance, + attention_backend, + ) -> None: + with HFRunner( + model_path, + torch_dtype=torch_dtype, + model_type="cross_encoder", + ) as hf_runner: + hf_scores = hf_runner.forward(prompts).scores + + with SRTRunner( + model_path, + tp_size=tp_size, + torch_dtype=torch_dtype, + model_type="cross_encoder", + attention_backend=attention_backend, + chunked_prefill_size=-1, + disable_radix_cache=True, + ) as srt_runner: + srt_scores = srt_runner.forward(prompts).scores + + for i in range(len(srt_scores)): + score_difference = abs(hf_scores[i] - srt_scores[i]) + + assert ( + score_difference < score_tolerance + ), "cross encoder scores are not all close" + + def preprocess_prompts(self, prompt): + processed_prompts = [] + query = prompt["query"] + documents = prompt["documents"] + for document in documents: + processed_prompts.append([query, document]) + + return processed_prompts + + def test_prefill_logits(self): + models_to_test = MODELS + + if is_in_ci(): + models_to_test = [random.choice(MODELS)] + + for model, tp_size, prefill_tolerance in models_to_test: + for attention_backend in ATTENTION_BACKEND: + for queryDocs in TEST_RERANK_QUERY_DOCS: + prompts = self.preprocess_prompts(queryDocs) + for torch_dtype in TORCH_DTYPES: + self.assert_close_prefill_logits( + prompts, + model, + tp_size, + torch_dtype, + prefill_tolerance, + attention_backend, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_dummy_grok_models.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_dummy_grok_models.py new file mode 100644 index 00000000..bebf9949 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_dummy_grok_models.py @@ -0,0 +1,34 @@ +import unittest + +from sglang.test.test_utils import CustomTestCase, is_in_ci, run_bench_one_batch + + +class TestDummyGrok1(CustomTestCase): + + def test_dummy_grok_1(self): + _, output_throughput, _ = run_bench_one_batch( + None, + [ + "--model", + "/dummy-grok", + "--tokenizer-path", + "Xenova/grok-1-tokenizer", + "--batch-size", + "2", + "--tp", + "2", + "--quantization", + "fp8", + "--load-format", + "dummy", + "--json-model-override-args", + '{"num_hidden_layers": 2}', + ], + ) + + if is_in_ci(): + self.assertGreater(output_throughput, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_embedding_models.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_embedding_models.py new file mode 100644 index 00000000..a93e762c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_embedding_models.py @@ -0,0 +1,145 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import multiprocessing as mp +import random +import unittest +from typing import Optional + +import torch +from transformers import AutoConfig, AutoTokenizer + +from sglang.test.runners import DEFAULT_PROMPTS, HFRunner, SRTRunner +from sglang.test.test_utils import ( + CustomTestCase, + get_similarities, + is_in_amd_ci, + is_in_ci, +) + +MODELS = [ + ("Alibaba-NLP/gte-Qwen2-1.5B-instruct", 1, 1e-5), + ("intfloat/e5-mistral-7b-instruct", 1, 1e-5), + ("marco/mcdse-2b-v1", 1, 1e-5), + ("Qwen/Qwen3-Embedding-8B", 1, 1e-5), + # Temporarily disable before this model is fixed + # ("jason9693/Qwen2.5-1.5B-apeach", 1, 1e-5), +] +TORCH_DTYPES = [torch.float16] + + +class TestEmbeddingModels(CustomTestCase): + + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + + def _truncate_prompts(self, prompts, model_path): + config = AutoConfig.from_pretrained(model_path) + max_length = getattr(config, "max_position_embeddings", 2048) + + tokenizer = AutoTokenizer.from_pretrained(model_path) + + truncated_prompts = [] + for prompt in prompts: + tokens = tokenizer(prompt, return_tensors="pt", truncation=False) + if len(tokens.input_ids[0]) > max_length: + truncated_text = tokenizer.decode( + tokens.input_ids[0][: max_length - 1], skip_special_tokens=True + ) + truncated_prompts.append(truncated_text) + else: + truncated_prompts.append(prompt) + return truncated_prompts + + def assert_close_prefill_logits( + self, + prompts, + model_path, + tp_size, + torch_dtype, + prefill_tolerance, + matryoshka_dim: Optional[int] = None, + ) -> None: + truncated_prompts = self._truncate_prompts(prompts, model_path) + + with HFRunner( + model_path, + torch_dtype=torch_dtype, + model_type="embedding", + matryoshka_dim=matryoshka_dim, + ) as hf_runner: + hf_outputs = hf_runner.forward(truncated_prompts) + + attention_backend = "triton" if is_in_amd_ci() else None + with SRTRunner( + model_path, + tp_size=tp_size, + torch_dtype=torch_dtype, + model_type="embedding", + attention_backend=attention_backend, + json_model_override_args=( + {"matryoshka_dimensions": [matryoshka_dim]} if matryoshka_dim else None + ), + ) as srt_runner: + srt_outputs = srt_runner.forward( + truncated_prompts, dimensions=matryoshka_dim + ) + + for i in range(len(prompts)): + hf_logits = torch.Tensor(hf_outputs.embed_logits[i]) + srt_logits = torch.Tensor(srt_outputs.embed_logits[i]) + + similarity = torch.tensor(get_similarities(hf_logits, srt_logits)) + print("similarity diff", abs(similarity - 1)) + + if len(prompts[i]) <= 1000: + assert torch.all( + abs(similarity - 1) < prefill_tolerance + ), "embeddings are not all close" + + def test_prefill_logits(self): + models_to_test = MODELS + + if is_in_ci(): + models_to_test = [random.choice(MODELS)] + + for model, tp_size, prefill_tolerance in models_to_test: + for torch_dtype in TORCH_DTYPES: + self.assert_close_prefill_logits( + DEFAULT_PROMPTS, model, tp_size, torch_dtype, prefill_tolerance + ) + + def test_matryoshka_embedding(self): + models_to_test = [ + model + for model in MODELS + if "Alibaba-NLP/gte-Qwen2-1.5B-instruct" == model[0] + ] + assert len(models_to_test) == 1 + + for model, tp_size, prefill_tolerance in models_to_test: + for torch_dtype in TORCH_DTYPES: + self.assert_close_prefill_logits( + DEFAULT_PROMPTS, + model, + tp_size, + torch_dtype, + prefill_tolerance, + matryoshka_dim=128, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_encoder_embedding_models.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_encoder_embedding_models.py new file mode 100644 index 00000000..dafaa72d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_encoder_embedding_models.py @@ -0,0 +1,162 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# python -m unittest test_encoder_embedding_models.TestEncoderEmbeddingModels.test_prefill_logits + +import multiprocessing as mp +import random +import time +import unittest + +import torch +from transformers import AutoConfig, AutoTokenizer + +from sglang.test.runners import DEFAULT_PROMPTS, HFRunner, SRTRunner +from sglang.test.test_utils import CustomTestCase, get_similarities, is_in_ci + +MODELS = [("BAAI/bge-small-en", 1, 1e-5), ("BAAI/bge-m3", 1, 1e-5)] + +ATTENTION_BACKEND = ["torch_native", "triton", "flashinfer"] +BATCH_SIZE = [1, 2] +TORCH_DTYPES = [torch.float32, torch.float16] +sgl_to_st_ratio = [] + + +class TestEncoderEmbeddingModels(CustomTestCase): + + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + + def _truncate_prompts(self, prompts, model_path): + config = AutoConfig.from_pretrained(model_path) + max_length = getattr(config, "max_position_embeddings", 512) - 20 + + tokenizer = AutoTokenizer.from_pretrained(model_path) + + truncated_prompts = [] + for prompt in prompts: + tokens = tokenizer(prompt, return_tensors="pt", truncation=False) + if len(tokens.input_ids[0]) > max_length: + truncated_text = tokenizer.decode( + tokens.input_ids[0][: max_length - 1], skip_special_tokens=True + ) + truncated_prompts.append(truncated_text) + else: + truncated_prompts.append(prompt) + + return truncated_prompts + + def assert_close_prefill_logits( + self, + prompts, + model_path, + tp_size, + torch_dtype, + prefill_tolerance, + attention_backend, + batch_size, + ) -> None: + truncated_prompts = self._truncate_prompts(prompts, model_path) + truncated_prompts = truncated_prompts * batch_size + + with HFRunner( + model_path, + torch_dtype=torch_dtype, + model_type="embedding", + ) as hf_runner: + # warm up + hf_outputs = hf_runner.forward(truncated_prompts) + + st_start_time = time.perf_counter() + hf_outputs = hf_runner.forward(truncated_prompts) + st_end_time = time.perf_counter() + + with SRTRunner( + model_path, + tp_size=tp_size, + torch_dtype=torch_dtype, + model_type="embedding", + attention_backend=attention_backend, + chunked_prefill_size=-1, + disable_radix_cache=True, + ) as srt_runner: + # warm up + srt_outputs = srt_runner.forward(truncated_prompts) + + sgl_start_time = time.perf_counter() + srt_outputs = srt_runner.forward(truncated_prompts) + sgl_end_time = time.perf_counter() + + transformer_time = st_end_time - st_start_time + sgl_time = sgl_end_time - sgl_start_time + sgl_to_st_ratio.append(sgl_time / transformer_time) + + for i in range(len(truncated_prompts)): + hf_logits = torch.Tensor(hf_outputs.embed_logits[i]) + srt_logits = torch.Tensor(srt_outputs.embed_logits[i]) + + similarity = torch.tensor(get_similarities(hf_logits, srt_logits)) + # If something is wrong, uncomment this to observe similarity. + # print("similarity diff", abs(similarity - 1)) + + if len(truncated_prompts[i]) <= 1000: + assert torch.all( + abs(similarity - 1) < prefill_tolerance + ), "embeddings are not all close" + + def test_prefill_logits(self): + models_to_test = MODELS + + if is_in_ci(): + models_to_test = [random.choice(MODELS)] + + for model, tp_size, prefill_tolerance in models_to_test: + for attention_backend in ATTENTION_BACKEND: + for batch_size in BATCH_SIZE: + for torch_dtype in TORCH_DTYPES: + # NOTE: FlashInfer currently has limitations with head_dim = 32 or + # other dimensions. + # The FlashInfer head_dim limitation itself is tracked here: + # https://github.com/flashinfer-ai/flashinfer/issues/1048 + # + # Flashinfer does not support torch.float32 for dtype_q, so skip it + if attention_backend == "flashinfer": + if ( + model == "BAAI/bge-small-en" + or torch_dtype == torch.float32 + ): + continue + + self.assert_close_prefill_logits( + DEFAULT_PROMPTS, + model, + tp_size, + torch_dtype, + prefill_tolerance, + attention_backend, + batch_size, + ) + + for i in range(len(BATCH_SIZE)): + print( + "bacth size: ", + BATCH_SIZE[i] * 5, + "sgl_time/st_time", + round(sgl_to_st_ratio[i], 3), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_generation_models.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_generation_models.py new file mode 100644 index 00000000..43770d5a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_generation_models.py @@ -0,0 +1,192 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Usage: + +To test a specific model locally: +1. Add it to ALL_MODELS, for example, `ModelCase("Qwen/Qwen2-1.5B")` +2. Run `ONLY_RUN=Qwen/Qwen2-1.5B python3 -m unittest test_generation_models.TestGenerationModels` +""" + +import dataclasses +import multiprocessing as mp +import os +import unittest +from typing import List + +import torch + +from sglang.test.runners import ( + DEFAULT_PROMPTS, + HFRunner, + SRTRunner, + check_close_model_outputs, +) +from sglang.test.test_utils import CustomTestCase, is_in_ci + + +@dataclasses.dataclass +class ModelCase: + model_path: str + tp_size: int = 1 + prefill_tolerance: float = 5e-2 + decode_tolerance: float = 6e-2 # Increased to fix numerical error in issue #8614. + rouge_l_tolerance: float = 1 + skip_long_prompt: bool = False + trust_remote_code: bool = False + + +# Popular models that run on the CI +CI_MODELS = [ + ModelCase("meta-llama/Llama-3.1-8B-Instruct"), + ModelCase("google/gemma-2-2b"), +] + +# the complete set of models to test sglang's generation model +ALL_MODELS = [ + *CI_MODELS, + ModelCase("Qwen/Qwen2-1.5B"), + ModelCase("Qwen/Qwen2.5-14B-Instruct"), + ModelCase("HuggingFaceTB/SmolLM-135M-Instruct", skip_long_prompt=True), + ModelCase("allenai/OLMo-1B-0724-hf", decode_tolerance=8e-2, skip_long_prompt=True), + ModelCase("shanearora/2025-sep-a-base-model"), + ModelCase( + "THUDM/glm-4-9b-chat", tp_size=2, trust_remote_code=True, skip_long_prompt=True + ), + ModelCase("openai-community/gpt2"), + ModelCase("microsoft/phi-1_5", trust_remote_code=True), + ModelCase("adept/persimmon-8b-chat"), + ModelCase("upstage/SOLAR-10.7B-Instruct-v1.0"), + ModelCase("inclusionAI/Ling-lite", trust_remote_code=True), + ModelCase("microsoft/Phi-3-small-8k-instruct", trust_remote_code=True), + ModelCase("allenai/OLMo-2-1124-7B-Instruct", skip_long_prompt=True), + ModelCase("ibm-granite/granite-3.0-2b-instruct", skip_long_prompt=True), + ModelCase( + "microsoft/Phi-3.5-MoE-instruct", + tp_size=2, + trust_remote_code=True, + skip_long_prompt=True, + ), + ModelCase("facebook/opt-125m", skip_long_prompt=True), + ModelCase( + "nvidia/Llama-3_3-Nemotron-Super-49B-v1_5", + tp_size=2, + trust_remote_code=True, + skip_long_prompt=True, + ), + ModelCase( + "nvidia/Llama-3_1-Nemotron-Ultra-253B-v1", + tp_size=8, + trust_remote_code=True, + skip_long_prompt=True, + ), + ModelCase( + "nvidia/NVIDIA-Nemotron-Nano-9B-v2", + trust_remote_code=True, + skip_long_prompt=True, + ), + ModelCase( + "swiss-ai/Apertus-8B", + trust_remote_code=True, + skip_long_prompt=True, + ), +] + +TORCH_DTYPES = [torch.float16] + + +class TestGenerationModels(CustomTestCase): + + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + + def assert_close_logits_and_output_strs( + self, + prompts: List[str], + model_case: ModelCase, + torch_dtype: torch.dtype, + ) -> None: + model_path = model_case.model_path + prefill_tolerance, decode_tolerance, rouge_l_tolerance = ( + model_case.prefill_tolerance, + model_case.decode_tolerance, + model_case.rouge_l_tolerance, + ) + max_new_tokens = 32 + + with HFRunner( + model_path, + torch_dtype=torch_dtype, + model_type="generation", + trust_remote_code=model_case.trust_remote_code, + ) as hf_runner: + hf_outputs = hf_runner.forward(prompts, max_new_tokens=max_new_tokens) + + with SRTRunner( + model_path, + tp_size=model_case.tp_size, + torch_dtype=torch_dtype, + model_type="generation", + trust_remote_code=model_case.trust_remote_code, + ) as srt_runner: + srt_outputs = srt_runner.forward(prompts, max_new_tokens=max_new_tokens) + + check_close_model_outputs( + hf_outputs=hf_outputs, + srt_outputs=srt_outputs, + prefill_tolerance=model_case.prefill_tolerance, + decode_tolerance=model_case.decode_tolerance, + rouge_l_tolerance=model_case.rouge_l_tolerance, + debug_text=f"model_path={model_path} prompts={prompts}", + ) + + @unittest.skipIf(not is_in_ci(), "Local test should run all models") + def test_ci_models(self): + for model_case in CI_MODELS: + for torch_dtype in TORCH_DTYPES: + prompts = DEFAULT_PROMPTS + + # Skip long prompts for models that do not have a long context + if model_case.skip_long_prompt: + prompts = [p for p in DEFAULT_PROMPTS if len(p) < 1000] + + # Assert the logits and output strs are close + self.assert_close_logits_and_output_strs( + prompts, model_case, torch_dtype + ) + + @unittest.skipIf(is_in_ci(), "CI only runs selected models for simplicity") + def test_all_models(self): + for model_case in ALL_MODELS: + for torch_dtype in TORCH_DTYPES: + if ( + "ONLY_RUN" in os.environ + and os.environ["ONLY_RUN"] != model_case.model_path + ): + continue + + # Skip long prompts for models that do not have a long context + prompts = DEFAULT_PROMPTS + if model_case.skip_long_prompt: + prompts = [p for p in DEFAULT_PROMPTS if len(p) < 1000] + + # Assert the logits and output strs are close + self.assert_close_logits_and_output_strs( + prompts, model_case, torch_dtype + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_glm4_moe_models.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_glm4_moe_models.py new file mode 100644 index 00000000..cb742c69 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_glm4_moe_models.py @@ -0,0 +1,49 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestGLM4MoE(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "zai-org/GLM-4.5-Air-FP8" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--tp-size", + "2", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=100, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], 0.8) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_kimi_k2_models.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_kimi_k2_models.py new file mode 100644 index 00000000..fc5e4c38 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_kimi_k2_models.py @@ -0,0 +1,68 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + + +class TestKimiK2Thinking(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "moonshotai/Kimi-K2-Thinking" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--tp", + "8", + "--trust-remote-code", + "--tool-call-parser", + "kimi_k2", + "--reasoning-parser", + "kimi_k2", + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k( + self, + ): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (Kimi-K2-Thinking)\n" f'{metrics["accuracy"]=:.3f}\n' + ) + self.assertGreater(metrics["accuracy"], 0.95) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_kimi_linear_models.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_kimi_linear_models.py new file mode 100644 index 00000000..25e280f1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_kimi_linear_models.py @@ -0,0 +1,46 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestKimiLinear(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "moonshotai/Kimi-Linear-48B-A3B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--tp-size", "2", "--trust-remote"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], 0.88) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_nvidia_nemotron_nano_v2.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_nvidia_nemotron_nano_v2.py new file mode 100644 index 00000000..d0b3ab11 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_nvidia_nemotron_nano_v2.py @@ -0,0 +1,28 @@ +import unittest + +from sglang.srt.utils import is_blackwell +from sglang.test.gsm8k_mixin import GSM8KMixin +from sglang.test.test_utils import CustomTestCase + + +class TestNvidiaNemotronNanoV2BF16(GSM8KMixin, CustomTestCase): + model = "nvidia/NVIDIA-Nemotron-Nano-9B-v2" + accuracy = 0.87 + other_args = ["--max-mamba-cache-size", "256"] + + +class TestNvidiaNemotronNanoV2FP8(GSM8KMixin, CustomTestCase): + accuracy = 0.87 + model = "nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8" + other_args = ["--max-mamba-cache-size", "256"] + + +@unittest.skipIf(not is_blackwell(), "NVFP4 only supported on blackwell") +class TestNvidiaNemotronNanoV2NVFP4(GSM8KMixin, CustomTestCase): + accuracy = 0.855 + model = "nvidia/NVIDIA-Nemotron-Nano-9B-v2-NVFP4" + other_args = ["--max-mamba-cache-size", "256"] + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_nvidia_nemotron_nano_v2_vl.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_nvidia_nemotron_nano_v2_vl.py new file mode 100644 index 00000000..a3621841 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_nvidia_nemotron_nano_v2_vl.py @@ -0,0 +1,31 @@ +import unittest +from types import SimpleNamespace + +from sglang.test.gsm8k_mixin import GSM8KMixin +from sglang.test.mmmu_vlm_mixin import MMMUVLMMixin +from sglang.test.test_utils import CustomTestCase + +MODEL = "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16" + + +class TestNvidiaNemotronNanoV2VLTextOnly(GSM8KMixin, CustomTestCase): + accuracy = 0.87 + model = MODEL + other_args = ["--max-mamba-cache-size", "256", "--trust-remote-code"] + + +class TestNvidiaNemotronNanoV2VLMMMU(MMMUVLMMixin, CustomTestCase): + accuracy = 0.454 + model = MODEL + other_args = ["--max-mamba-cache-size", "128", "--trust-remote-code"] + mmmu_args = ["--limit=0.1"] + """`--limit=0.1`: 10 percent of each task - this is fine for testing since the nominal result isn't interesting - this run is just to prevent relative regressions.""" + + def test_vlm_mmmu_benchmark(self): + self._run_vlm_mmmu_test( + SimpleNamespace(model=self.model, mmmu_accuracy=self.accuracy), "./logs" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_qwen3_next_models.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_qwen3_next_models.py new file mode 100644 index 00000000..c0386974 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_qwen3_next_models.py @@ -0,0 +1,177 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestQwen3Next(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "Qwen/Qwen3-Next-80B-A3B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--tp-size", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], 0.93) + + +class TestQwen3NextMTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "Qwen/Qwen3-Next-80B-A3B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--mem-fraction-static", + "0.8", + "--tp", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], 0.93) + + +class TestQwen3NextMTPTopk(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "Qwen/Qwen3-Next-80B-A3B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + "5", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "8", + "--mem-fraction-static", + "0.8", + "--tp", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], 0.93) + + +class TestQwen3NextPiecewiseCudaGraph(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.model = "Qwen/Qwen3-Next-80B-A3B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--tp", + "4", + "--enable-piecewise-cuda-graph", + "--piecewise-cuda-graph-compiler", + "eager", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], 0.93) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_qwen_models.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_qwen_models.py new file mode 100644 index 00000000..567e19f0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_qwen_models.py @@ -0,0 +1,77 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestQwen2(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "Qwen/Qwen2-7B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], 0.78) + + +class TestQwen2FP8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "neuralmagic/Qwen2-7B-Instruct-FP8" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], 0.78) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_reward_models.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_reward_models.py new file mode 100644 index 00000000..72b74eb3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_reward_models.py @@ -0,0 +1,94 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import multiprocessing as mp +import unittest + +import torch + +from sglang.test.runners import HFRunner, SRTRunner +from sglang.test.test_utils import CustomTestCase + +MODELS = [ + ("LxzGordon/URM-LLaMa-3.1-8B", 1, 4e-2), + ("Skywork/Skywork-Reward-Llama-3.1-8B-v0.2", 1, 4e-2), +] +TORCH_DTYPES = [torch.float16] + +# PROMPT = "Jane has 12 apples. She gives 4 apples to her friend Mark, then buys 1 more apple, and finally splits all her apples equally among herself and her 2 siblings. How many apples does each person get?" +# RESPONSE1 = "1. Jane starts with 12 apples and gives 4 to Mark. 12 - 4 = 8. Jane now has 8 apples.\n2. Jane buys 1 more apple. 8 + 1 = 9. Jane now has 9 apples.\n3. Jane splits the 9 apples equally among herself and her 2 siblings (3 people in total). 9 ÷ 3 = 3 apples each. Each person gets 3 apples." +# RESPONSE2 = "1. Jane starts with 12 apples and gives 4 to Mark. 12 - 4 = 8. Jane now has 8 apples.\n2. Jane buys 1 more apple. 8 + 1 = 9. Jane now has 9 apples.\n3. Jane splits the 9 apples equally among her 2 siblings (2 people in total). 9 ÷ 2 = 4.5 apples each. Each person gets 4 apples." + +PROMPT = ( + "What is the range of the numeric output of a sigmoid node in a neural network?" +) +RESPONSE1 = "The output of a sigmoid node is bounded between -1 and 1." +RESPONSE2 = "The output of a sigmoid node is bounded between 0 and 1." + +CONVS = [ + [{"role": "user", "content": PROMPT}, {"role": "assistant", "content": RESPONSE1}], + [{"role": "user", "content": PROMPT}, {"role": "assistant", "content": RESPONSE2}], +] + + +class TestRewardModels(CustomTestCase): + + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + + def assert_close_reward_scores( + self, + convs, + model_path, + tp_size, + torch_dtype, + tolerance, + ) -> None: + with HFRunner( + model_path, + torch_dtype=torch_dtype, + model_type="reward", + ) as hf_runner: + hf_outputs = hf_runner.forward(convs) + + with SRTRunner( + model_path, + torch_dtype=torch_dtype, + model_type="reward", + ) as srt_runner: + prompts = srt_runner.tokenizer.apply_chat_template( + convs, tokenize=False, return_dict=False + ) + srt_outputs = srt_runner.forward(prompts) + + hf_scores = torch.tensor(hf_outputs.scores) + srt_scores = torch.tensor(srt_outputs.scores) + print(f"{hf_scores=}") + print(f"{srt_scores=}") + + assert torch.all( + abs(hf_scores - srt_scores) < tolerance + ), "reward scores are not all close" + + def test_reward_scores(self): + for model, tp_size, tolerance in MODELS: + for torch_dtype in TORCH_DTYPES: + self.assert_close_reward_scores( + CONVS, model, tp_size, torch_dtype, tolerance + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_transformers_models.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_transformers_models.py new file mode 100644 index 00000000..95592453 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_transformers_models.py @@ -0,0 +1,181 @@ +import dataclasses +import multiprocessing as mp +import unittest +from types import SimpleNamespace +from typing import List + +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.runners import DEFAULT_PROMPTS, SRTRunner, check_close_model_outputs +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) + + +class TestTransformersFallbackEndpoint(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--model-impl", "transformers"], + ) + cls.mmlu_lower_bound = 0.65 + cls.gsm8k_lower_bound = 0.65 + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + from sglang.test.run_eval import run_eval + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], self.mmlu_lower_bound) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + from sglang.test.few_shot_gsm8k import run_eval + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], self.gsm8k_lower_bound) + + +class TestTransformersFallbackTorchAO(TestTransformersFallbackEndpoint): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--model-impl", + "transformers", + "--torchao-config", + "int4wo-128", + ], + ) + cls.mmlu_lower_bound = 0.65 + cls.gsm8k_lower_bound = 0.65 + + +@dataclasses.dataclass +class ModelCase: + model_path: str + tp_size: int = 1 + prefill_tolerance: float = 5e-2 + decode_tolerance: float = 5e-2 + rouge_l_tolerance: float = 1 + skip_long_prompt: bool = False + trust_remote_code: bool = False + torchao_config: str = None + torch_dtype: torch.dtype = torch.float16 + + +# Popular models that run on the CI +CI_MODELS = [ + ModelCase(DEFAULT_MODEL_NAME_FOR_TEST), +] + +ALL_OTHER_MODELS = [ + ModelCase(DEFAULT_MODEL_NAME_FOR_TEST, tp_size=2), +] + + +class TestTransformersFallbackEngine(CustomTestCase): + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + + def assert_close_logits_and_output_strs( + self, + prompts: List[str], + model_case: ModelCase, + ) -> None: + model_path = model_case.model_path + max_new_tokens = 32 + # force to use transformers impl + with SRTRunner( + model_path, + tp_size=model_case.tp_size, + torch_dtype=model_case.torch_dtype, + model_type="generation", + model_impl="transformers", + trust_remote_code=model_case.trust_remote_code, + torchao_config=model_case.torchao_config, + ) as srt_runner: + srt_outputs = srt_runner.forward(prompts, max_new_tokens=max_new_tokens) + + with SRTRunner( + model_path, + tp_size=model_case.tp_size, + torch_dtype=model_case.torch_dtype, + model_type="generation", + trust_remote_code=model_case.trust_remote_code, + torchao_config=model_case.torchao_config, + ) as srt_runner: + srt_transformers_outputs = srt_runner.forward( + prompts, max_new_tokens=max_new_tokens + ) + + check_close_model_outputs( + hf_outputs=srt_transformers_outputs, + srt_outputs=srt_outputs, + prefill_tolerance=model_case.prefill_tolerance, + decode_tolerance=model_case.decode_tolerance, + rouge_l_tolerance=model_case.rouge_l_tolerance, + debug_text=f"model_path={model_path} prompts={prompts}", + ) + + def test_ci_models(self): + for model_case in CI_MODELS: + # Skip long prompts for models that do not have a long context + prompts = DEFAULT_PROMPTS + if model_case.skip_long_prompt: + prompts = [p for p in DEFAULT_PROMPTS if len(p) < 1000] + # Assert the logits and output strs are close + self.assert_close_logits_and_output_strs(prompts, model_case) + + def test_others(self): + if is_in_ci(): + return + + # Skip long prompts for models that do not have a long context + prompts = DEFAULT_PROMPTS + for model_case in ALL_OTHER_MODELS: + if model_case.skip_long_prompt: + prompts = [p for p in DEFAULT_PROMPTS if len(p) < 1000] + + # Assert the logits and output strs are close + self.assert_close_logits_and_output_strs(prompts, model_case) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_vlm_models.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_vlm_models.py new file mode 100644 index 00000000..b26ff083 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/models/test_vlm_models.py @@ -0,0 +1,52 @@ +import argparse +import random +import sys +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import is_hip +from sglang.test.mmmu_vlm_mixin import DEFAULT_MEM_FRACTION_STATIC, MMMUVLMMixin +from sglang.test.test_utils import CustomTestCase, is_in_ci + +_is_hip = is_hip() +# VLM models for testing +if _is_hip: + MODELS = [SimpleNamespace(model="openbmb/MiniCPM-V-2_6", mmmu_accuracy=0.4)] +else: + MODELS = [ + SimpleNamespace(model="google/gemma-3-27b-it", mmmu_accuracy=0.45), + SimpleNamespace(model="Qwen/Qwen2.5-VL-3B-Instruct", mmmu_accuracy=0.4), + SimpleNamespace(model="openbmb/MiniCPM-V-2_6", mmmu_accuracy=0.4), + ] + + +class TestVLMModels(MMMUVLMMixin, CustomTestCase): + def test_vlm_mmmu_benchmark(self): + """Test VLM models against MMMU benchmark.""" + models_to_test = MODELS + + if is_in_ci(): + models_to_test = [random.choice(MODELS)] + + for model in models_to_test: + self._run_vlm_mmmu_test(model, "./logs") + + +if __name__ == "__main__": + # Define and parse arguments here, before unittest.main + parser = argparse.ArgumentParser(description="Test VLM models") + parser.add_argument( + "--mem-fraction-static", + type=float, + help="Static memory fraction for the model", + default=DEFAULT_MEM_FRACTION_STATIC, + ) + + # Parse args intended for unittest + args = parser.parse_args() + + # Store the parsed args object on the class + TestVLMModels.parsed_args = args + + # Pass args to unittest + unittest.main(argv=[sys.argv[0]]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/nightly/test_gsm8k_eval_amd.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/nightly/test_gsm8k_eval_amd.py new file mode 100644 index 00000000..f2799863 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/nightly/test_gsm8k_eval_amd.py @@ -0,0 +1,193 @@ +import json +import os +import unittest +import warnings +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1, + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2, + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1, + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + is_in_ci, + parse_models, + popen_launch_server, + write_github_step_summary, + write_results_to_json, +) + +MODEL_SCORE_THRESHOLDS = { + "meta-llama/Llama-3.1-8B-Instruct": 0.82, + "mistralai/Mistral-7B-Instruct-v0.3": 0.58, + "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 0.85, + "meta-llama/Llama-3.1-70B-Instruct": 0.95, + "mistralai/Mixtral-8x7B-Instruct-v0.1": 0.64, + "Qwen/Qwen2-57B-A14B-Instruct": 0.86, + "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.83, + "neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.54, + "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.94, + "neuralmagic/Qwen2-72B-Instruct-FP8": 0.94, + "neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.86, + "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.65, + "google/gemma-2-27b-it": 0.91, + "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.84, +} + +failing_models = { + "neuralmagic/gemma-2-2b-it-FP8", +} + + +def remove_failing_models(model_str): + models = model_str.split(",") + filtered = [m for m in models if m not in failing_models] + return ",".join(filtered) + + +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 = remove_failing_models( + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 +) +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2 = remove_failing_models( + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2 +) +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1 = remove_failing_models( + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1 +) +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2 = remove_failing_models( + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2 +) + +NO_MOE_PADDING_MODELS = {"neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8"} +DISABLE_HF_XET_MODELS = { + "Qwen/Qwen2-57B-A14B-Instruct", + "neuralmagic/Qwen2-57B-A14B-Instruct-FP8", +} +TRITON_MOE_MODELS = { + "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8", + "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8", + "mistralai/Mixtral-8x7B-Instruct-v0.1", + "mistralai/Mistral-7B-Instruct-v0.3", +} + + +def popen_launch_server_wrapper(base_url, model, is_tp2): + other_args = ["--log-level-http", "warning", "--trust-remote-code"] + if is_tp2: + other_args.extend(["--tp", "2"]) + + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + return process + + +def check_model_scores(results): + failed_models = [] + summary = " | model | score | threshold |\n" + summary += "| ----- | ----- | --------- |\n" + + for model, score in results: + threshold = MODEL_SCORE_THRESHOLDS.get(model) + if threshold is None: + print(f"Warning: No threshold defined for model {model}") + continue + + if score < threshold: + failed_models.append( + f"\nScore Check Failed: {model}\n" + f"Model {model} score ({score:.4f}) is below threshold ({threshold:.4f})" + ) + + line = f"| {model} | {score} | {threshold} |\n" + summary += line + + print(summary) + + if is_in_ci(): + write_github_step_summary(f"### TestNightlyGsm8KEval\n{summary}") + + if failed_models: + raise AssertionError("\n".join(failed_models)) + + +# Do not use `CustomTestCase` since `test_mgsm_en_all_models` does not want retry +class TestNightlyGsm8KEval(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model_groups = [ + (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1), False, False), + (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2), False, True), + (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1), True, False), + (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2), True, True), + ] + cls.base_url = DEFAULT_URL_FOR_TEST + + def test_mgsm_en_all_models(self): + warnings.filterwarnings( + "ignore", category=ResourceWarning, message="unclosed.*socket" + ) + is_first = True + all_results = [] + + for model_group, is_fp8, is_tp2 in self.model_groups: + for model in model_group: + with self.subTest(model=model): + os.environ["SGLANG_MOE_PADDING"] = ( + "0" if model in NO_MOE_PADDING_MODELS else "1" + ) + os.environ["HF_HUB_DISABLE_XET"] = ( + "1" if model in DISABLE_HF_XET_MODELS else "0" + ) + os.environ["SGLANG_USE_AITER"] = ( + "0" if model in TRITON_MOE_MODELS else "1" + ) + + process = popen_launch_server_wrapper(self.base_url, model, is_tp2) + + args = SimpleNamespace( + base_url=self.base_url, + model=model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + # Allow retries, so flaky errors are avoided. + threshold = MODEL_SCORE_THRESHOLDS.get(model) + for attempt in range(3): + try: + metrics = run_eval(args) + score = metrics["score"] + if score >= threshold: + break + except Exception as e: + print(f"Attempt {attempt + 1} failed with error: {e}") + print( + f"{'=' * 42}\n{model} - metrics={metrics} score={metrics['score']}\n{'=' * 42}\n" + ) + + write_results_to_json(model, metrics, "w" if is_first else "a") + is_first = False + + all_results.append((model, metrics["score"])) + kill_process_tree(process.pid) + + try: + with open("results.json", "r") as f: + print("\nFinal Results from results.json:") + print(json.dumps(json.load(f), indent=2)) + except Exception as e: + print(f"Error reading results.json: {e}") + + # Check all scores after collecting all results + check_model_scores(all_results) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_openai_embedding.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_openai_embedding.py new file mode 100644 index 00000000..21da5dd8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_openai_embedding.py @@ -0,0 +1,197 @@ +import json +import unittest + +import openai + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestOpenAIEmbedding(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + # Configure embedding-specific args + other_args = ["--is-embedding", "--enable-metrics"] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=other_args, + ) + cls.base_url += "/v1" + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_embedding_single(self): + """Test single embedding request""" + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.embeddings.create(model=self.model, input="Hello world") + self.assertEqual(len(response.data), 1) + self.assertTrue(len(response.data[0].embedding) > 0) + + def test_embedding_batch(self): + """Test batch embedding request""" + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.embeddings.create( + model=self.model, input=["Hello world", "Test text"] + ) + self.assertEqual(len(response.data), 2) + self.assertTrue(len(response.data[0].embedding) > 0) + self.assertTrue(len(response.data[1].embedding) > 0) + + def test_embedding_single_batch_str(self): + """Test embedding with a List[str] and length equals to 1""" + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.embeddings.create(model=self.model, input=["Hello world"]) + self.assertEqual(len(response.data), 1) + self.assertTrue(len(response.data[0].embedding) > 0) + + def test_embedding_single_int_list(self): + """Test embedding with a List[int] or List[List[int]]]""" + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.embeddings.create( + model=self.model, + input=[[15339, 314, 703, 284, 612, 262, 10658, 10188, 286, 2061]], + ) + self.assertEqual(len(response.data), 1) + self.assertTrue(len(response.data[0].embedding) > 0) + + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.embeddings.create( + model=self.model, + input=[15339, 314, 703, 284, 612, 262, 10658, 10188, 286, 2061], + ) + self.assertEqual(len(response.data), 1) + self.assertTrue(len(response.data[0].embedding) > 0) + + def test_empty_string_embedding(self): + """Test embedding an empty string.""" + + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + # Text embedding example with empty string + text = "" + # Expect a BadRequestError for empty input + with self.assertRaises(openai.BadRequestError) as cm: + client.embeddings.create( + model=self.model, + input=text, + ) + # check the status code + self.assertEqual(cm.exception.status_code, 400) + + def test_embedding_with_dimensions_parameter(self): + """Test that non-Matryoshka models reject dimensions parameter.""" + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + # Test that specifying dimensions fails for non-Matryoshka models + with self.assertRaises(openai.BadRequestError) as cm: + client.embeddings.create( + model=self.model, input="Hello world", dimensions=512 + ) + + self.assertEqual(cm.exception.status_code, 400) + + +class TestMatryoshkaEmbeddingModel(CustomTestCase): + """Test class for Model that supports Matryoshka embedding functionality, using OpenAI API.""" + + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.matryoshka_dims = [128, 256, 512, 768, 1024] + + # Configure embedding-specific args with Matryoshka support via json_model_override_args + matryoshka_config = { + "is_matryoshka": True, + "matryoshka_dimensions": cls.matryoshka_dims, + } + other_args = [ + "--is-embedding", + "--enable-metrics", + "--json-model-override-args", + json.dumps(matryoshka_config), + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=other_args, + ) + cls.base_url += "/v1" + + @classmethod + def tearDownClass(cls): + if hasattr(cls, "process"): + kill_process_tree(cls.process.pid) + + def test_matryoshka_embedding_valid_dimensions(self): + """Test Matryoshka embedding with valid dimensions.""" + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + # Test with various valid dimensions + for dimensions in self.matryoshka_dims: + with self.subTest(dimensions=dimensions): + response = client.embeddings.create( + model=self.model, input="Hello world", dimensions=dimensions + ) + self.assertEqual(len(response.data), 1) + self.assertEqual(len(response.data[0].embedding), dimensions) + + def test_matryoshka_embedding_batch_same_dimensions(self): + """Test Matryoshka embedding with batch input and same dimensions.""" + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.embeddings.create( + model=self.model, + input=["Hello world", "Test text", "Another example"], + dimensions=256, + ) + + self.assertEqual(len(response.data), 3) + for embedding_data in response.data: + self.assertEqual(len(embedding_data.embedding), 256) + + def test_matryoshka_embedding_no_dimensions(self): + """Test embedding without specifying dimensions (should use full size).""" + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.embeddings.create(model=self.model, input="Hello world") + + self.assertEqual(len(response.data), 1) + + # Should return full embedding size when no dimensions specified + self.assertEqual(len(response.data[0].embedding), 1536) + + def test_matryoshka_embedding_invalid_dimensions(self): + """Test Matryoshka embedding with invalid dimensions.""" + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + for dimensions in [100, 0, -1, 10000]: + with self.assertRaises(openai.BadRequestError) as cm: + client.embeddings.create( + model=self.model, + input="Hello world", + dimensions=dimensions, + ) + self.assertEqual(cm.exception.status_code, 400) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_openai_server.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_openai_server.py new file mode 100644 index 00000000..9b09fbfa --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_openai_server.py @@ -0,0 +1,1105 @@ +""" +python3 -m unittest openai_server.basic.test_openai_server.TestOpenAIServer.test_completion +python3 -m unittest openai_server.basic.test_openai_server.TestOpenAIServer.test_completion_stream +python3 -m unittest openai_server.basic.test_openai_server.TestOpenAIServer.test_chat_completion +python3 -m unittest openai_server.basic.test_openai_server.TestOpenAIServer.test_chat_completion_stream +""" + +import json +import random +import unittest +from concurrent.futures import ThreadPoolExecutor +from typing import Optional + +import openai +import requests + +from sglang.srt.sampling.custom_logit_processor import CustomLogitProcessor +from sglang.srt.utils import kill_process_tree +from sglang.srt.utils.hf_transformers_utils import get_tokenizer +from sglang.test.runners import TEST_RERANK_QUERY_DOCS +from sglang.test.test_utils import ( + DEFAULT_SMALL_CROSS_ENCODER_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestOpenAIServer(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_completion( + self, echo, logprobs, use_list_input, parallel_sample_num, token_input + ): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + prompt = "The capital of France is" + if token_input: + prompt_input = self.tokenizer.encode(prompt) + num_prompt_tokens = len(prompt_input) + else: + prompt_input = prompt + num_prompt_tokens = len(self.tokenizer.encode(prompt)) + + if use_list_input: + prompt_arg = [prompt_input, prompt_input] + num_choices = len(prompt_arg) + num_prompt_tokens *= 2 + else: + prompt_arg = prompt_input + num_choices = 1 + + response = client.completions.create( + model=self.model, + prompt=prompt_arg, + temperature=0, + max_tokens=32, + echo=echo, + logprobs=logprobs, + n=parallel_sample_num, + ) + + assert len(response.choices) == num_choices * parallel_sample_num + + if echo: + text = response.choices[0].text + assert text.startswith(prompt) + + if logprobs: + assert response.choices[0].logprobs + assert isinstance(response.choices[0].logprobs.tokens[0], str) + assert isinstance(response.choices[0].logprobs.top_logprobs[1], dict) + ret_num_top_logprobs = len(response.choices[0].logprobs.top_logprobs[1]) + + # FIXME: Sometimes, some top_logprobs are missing in the return value. The reason is that some output id maps to the same output token and duplicate in the map + # assert ret_num_top_logprobs == logprobs, f"{ret_num_top_logprobs} vs {logprobs}" + assert ret_num_top_logprobs > 0 + + # when echo=True and request.logprobs>0, logprob_start_len is 0, so the first token's logprob would be None. + if not echo: + assert response.choices[0].logprobs.token_logprobs[0] + + assert response.id + assert response.created + assert ( + response.usage.prompt_tokens == num_prompt_tokens + ), f"{response.usage.prompt_tokens} vs {num_prompt_tokens}" + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def run_completion_stream( + self, echo, logprobs, use_list_input, parallel_sample_num, token_input + ): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + prompt = "The capital of France is" + if token_input: + prompt_input = self.tokenizer.encode(prompt) + num_prompt_tokens = len(prompt_input) + else: + prompt_input = prompt + num_prompt_tokens = len(self.tokenizer.encode(prompt)) + + if use_list_input: + prompt_arg = [prompt_input, prompt_input] + num_choices = len(prompt_arg) + num_prompt_tokens *= 2 + else: + prompt_arg = prompt_input + num_choices = 1 + + generator = client.completions.create( + model=self.model, + prompt=prompt_arg, + temperature=0, + max_tokens=32, + echo=echo, + logprobs=logprobs, + stream=True, + stream_options={"include_usage": True}, + n=parallel_sample_num, + ) + + is_firsts = {} + for response in generator: + usage = response.usage + if usage is not None: + assert usage.prompt_tokens > 0, f"usage.prompt_tokens was zero" + assert usage.completion_tokens > 0, f"usage.completion_tokens was zero" + assert usage.total_tokens > 0, f"usage.total_tokens was zero" + continue + + index = response.choices[0].index + is_first = is_firsts.get(index, True) + + if logprobs: + assert response.choices[0].logprobs, f"no logprobs in response" + assert isinstance( + response.choices[0].logprobs.tokens[0], str + ), f"{response.choices[0].logprobs.tokens[0]} is not a string" + if not (is_first and echo): + assert isinstance( + response.choices[0].logprobs.top_logprobs[0], dict + ), f"top_logprobs was not a dictionary" + ret_num_top_logprobs = len( + response.choices[0].logprobs.top_logprobs[0] + ) + # FIXME: Sometimes, some top_logprobs are missing in the return value. The reason is that some output id maps to the same output token and duplicate in the map + # assert ret_num_top_logprobs == logprobs, f"{ret_num_top_logprobs} vs {logprobs}" + assert ret_num_top_logprobs > 0, f"ret_num_top_logprobs was 0" + + if is_first: + if echo: + assert response.choices[0].text.startswith( + prompt + ), f"{response.choices[0].text} and all args {echo} {logprobs} {token_input} {is_first}" + is_firsts[index] = False + assert response.id, f"no id in response" + assert response.created, f"no created in response" + + for index in [i for i in range(parallel_sample_num * num_choices)]: + assert not is_firsts.get( + index, True + ), f"index {index} is not found in the response" + + def run_chat_completion(self, logprobs, parallel_sample_num): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + { + "role": "user", + "content": "What is the capital of France? Answer in a few words.", + }, + ], + temperature=0, + logprobs=logprobs is not None and logprobs > 0, + top_logprobs=logprobs, + n=parallel_sample_num, + ) + + if logprobs: + assert isinstance( + response.choices[0].logprobs.content[0].top_logprobs[0].token, str + ) + + ret_num_top_logprobs = len( + response.choices[0].logprobs.content[0].top_logprobs + ) + assert ( + ret_num_top_logprobs == logprobs + ), f"{ret_num_top_logprobs} vs {logprobs}" + + assert len(response.choices) == parallel_sample_num + assert response.choices[0].message.role == "assistant" + assert isinstance(response.choices[0].message.content, str) + assert response.id + assert response.created + assert response.usage.prompt_tokens > 0 + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def run_chat_completion_stream(self, logprobs, parallel_sample_num=1): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + generator = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "What is the capital of France?"}, + ], + temperature=0, + logprobs=logprobs is not None and logprobs > 0, + top_logprobs=logprobs, + stream=True, + stream_options={"include_usage": True}, + n=parallel_sample_num, + ) + + is_firsts = {} + is_finished = {} + finish_reason_counts = {} + for response in generator: + usage = response.usage + if usage is not None: + assert usage.prompt_tokens > 0, f"usage.prompt_tokens was zero" + assert usage.completion_tokens > 0, f"usage.completion_tokens was zero" + assert usage.total_tokens > 0, f"usage.total_tokens was zero" + continue + + index = response.choices[0].index + finish_reason = response.choices[0].finish_reason + if finish_reason is not None: + is_finished[index] = True + finish_reason_counts[index] = finish_reason_counts.get(index, 0) + 1 + + data = response.choices[0].delta + + if is_firsts.get(index, True): + assert ( + data.role == "assistant" + ), f"data.role was not 'assistant' for first chunk" + is_firsts[index] = False + continue + + if logprobs and not is_finished.get(index, False): + assert response.choices[0].logprobs, f"logprobs was not returned" + assert isinstance( + response.choices[0].logprobs.content[0].top_logprobs[0].token, str + ), f"top_logprobs token was not a string" + assert isinstance( + response.choices[0].logprobs.content[0].top_logprobs, list + ), f"top_logprobs was not a list" + ret_num_top_logprobs = len( + response.choices[0].logprobs.content[0].top_logprobs + ) + assert ( + ret_num_top_logprobs == logprobs + ), f"{ret_num_top_logprobs} vs {logprobs}" + + assert ( + isinstance(data.content, str) + or isinstance(data.reasoning_content, str) + or (isinstance(data.tool_calls, list) and len(data.tool_calls) > 0) + or response.choices[0].finish_reason + ) + assert response.id + assert response.created + + for index in [i for i in range(parallel_sample_num)]: + assert not is_firsts.get( + index, True + ), f"index {index} is not found in the response" + + # Verify that each choice gets exactly one finish_reason chunk + for index in range(parallel_sample_num): + assert ( + index in finish_reason_counts + ), f"No finish_reason found for index {index}" + assert ( + finish_reason_counts[index] == 1 + ), f"Expected 1 finish_reason chunk for index {index}, got {finish_reason_counts[index]}" + + def test_completion(self): + for echo in [False, True]: + for logprobs in [None, 5]: + for use_list_input in [True, False]: + for parallel_sample_num in [1, 2]: + for token_input in [False, True]: + self.run_completion( + echo, + logprobs, + use_list_input, + parallel_sample_num, + token_input, + ) + + def test_completion_stream(self): + # parallel sampling and list input are not supported in streaming mode + for echo in [False, True]: + for logprobs in [None, 5]: + for use_list_input in [True, False]: + for parallel_sample_num in [1, 2]: + for token_input in [False, True]: + self.run_completion_stream( + echo, + logprobs, + use_list_input, + parallel_sample_num, + token_input, + ) + + def test_chat_completion(self): + for logprobs in [None, 5]: + for parallel_sample_num in [1, 2]: + self.run_chat_completion(logprobs, parallel_sample_num) + + def test_chat_completion_stream(self): + for logprobs in [None, 5]: + for parallel_sample_num in [1, 2]: + self.run_chat_completion_stream(logprobs, parallel_sample_num) + + def test_regex(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + regex = ( + r"""\{\n""" + + r""" "name": "[\w]+",\n""" + + r""" "population": [\d]+\n""" + + r"""\}""" + ) + + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "Introduce the capital of France."}, + ], + temperature=0, + max_tokens=128, + extra_body={"regex": regex}, + ) + text = response.choices[0].message.content + + try: + js_obj = json.loads(text) + except (TypeError, json.decoder.JSONDecodeError): + print("JSONDecodeError", text) + raise + assert isinstance(js_obj["name"], str) + assert isinstance(js_obj["population"], int) + + def test_penalty(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "Introduce the capital of France."}, + ], + temperature=0, + max_tokens=32, + frequency_penalty=1.0, + ) + text = response.choices[0].message.content + assert isinstance(text, str) + + def test_response_prefill(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.chat.completions.create( + model="meta-llama/Llama-3.1-8B-Instruct", + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + { + "role": "user", + "content": """ +Extract the name, size, price, and color from this product description as a JSON object: + + +The SmartHome Mini is a compact smart home assistant available in black or white for only $49.99. At just 5 inches wide, it lets you control lights, thermostats, and other connected devices via voice or app—no matter where you place it in your home. This affordable little hub brings convenient hands-free control to your smart devices. + +""", + }, + { + "role": "assistant", + "content": "{\n", + }, + ], + temperature=0, + extra_body={"continue_final_message": True}, + ) + + assert ( + response.choices[0] + .message.content.strip() + .startswith('"name": "SmartHome Mini",') + ) + + def test_model_list(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + models = list(client.models.list()) + assert len(models) == 1 + assert isinstance(getattr(models[0], "max_model_len", None), int) + + def test_retrieve_model(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + # Test retrieving an existing model + retrieved_model = client.models.retrieve(self.model) + self.assertEqual(retrieved_model.id, self.model) + self.assertEqual(retrieved_model.root, self.model) + + # Test retrieving a non-existent model + with self.assertRaises(openai.NotFoundError): + client.models.retrieve("non-existent-model") + + +class TestOpenAIServerv1Responses(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_response( + self, + input_text: str = "The capital of France is", + *, + instructions: str | None = None, + temperature: float | None = 0.0, + top_p: float | None = 1.0, + max_output_tokens: int | None = 32, + store: bool | None = True, + parallel_tool_calls: bool | None = True, + tool_choice: str | None = "auto", + previous_response_id: str | None = None, + truncation: str | None = "disabled", + user: str | None = None, + metadata: dict | None = None, + ): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "input": input_text, + "temperature": temperature, + "top_p": top_p, + "max_output_tokens": max_output_tokens, + "store": store, + "parallel_tool_calls": parallel_tool_calls, + "tool_choice": tool_choice, + "previous_response_id": previous_response_id, + "truncation": truncation, + "user": user, + "instructions": instructions, + } + if metadata is not None: + payload["metadata"] = metadata + payload = {k: v for k, v in payload.items() if v is not None} + return client.responses.create(**payload) + + def run_response_stream( + self, + input_text: str = "The capital of France is", + *, + instructions: str | None = None, + temperature: float | None = 0.0, + top_p: float | None = 1.0, + max_output_tokens: int | None = 32, + store: bool | None = True, + parallel_tool_calls: bool | None = True, + tool_choice: str | None = "auto", + previous_response_id: str | None = None, + truncation: str | None = "disabled", + user: str | None = None, + metadata: dict | None = None, + ): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "input": input_text, + "temperature": temperature, + "top_p": top_p, + "max_output_tokens": max_output_tokens, + "store": store, + "parallel_tool_calls": parallel_tool_calls, + "tool_choice": tool_choice, + "previous_response_id": previous_response_id, + "truncation": truncation, + "user": user, + "instructions": instructions, + "stream": True, + "stream_options": {"include_usage": True}, + } + if metadata is not None: + payload["metadata"] = metadata + payload = {k: v for k, v in payload.items() if v is not None} + + aggregated_text = "" + saw_created = False + saw_in_progress = False + saw_completed = False + final_usage_ok = False + + stream_ctx = getattr(client.responses, "stream", None) + if callable(stream_ctx): + stream_payload = dict(payload) + stream_payload.pop("stream", None) + stream_payload.pop("stream_options", None) + with client.responses.stream(**stream_payload) as stream: + for event in stream: + et = getattr(event, "type", None) + if et == "response.created": + saw_created = True + elif et == "response.in_progress": + saw_in_progress = True + elif et == "response.output_text.delta": + # event.delta expected to be a string + delta = getattr(event, "delta", "") + if isinstance(delta, str): + aggregated_text += delta + elif et == "response.completed": + saw_completed = True + # Validate streaming-completed usage mapping + resp = getattr(event, "response", None) + try: + # resp may be dict-like already + usage = ( + resp.get("usage") + if isinstance(resp, dict) + else getattr(resp, "usage", None) + ) + if isinstance(usage, dict): + final_usage_ok = all( + k in usage + for k in ( + "input_tokens", + "output_tokens", + "total_tokens", + ) + ) + except Exception: + pass + _ = stream.get_final_response() + else: + generator = client.responses.create(**payload) + for event in generator: + et = getattr(event, "type", None) + if et == "response.created": + saw_created = True + elif et == "response.in_progress": + saw_in_progress = True + elif et == "response.output_text.delta": + delta = getattr(event, "delta", "") + if isinstance(delta, str): + aggregated_text += delta + elif et == "response.completed": + saw_completed = True + + return ( + aggregated_text, + saw_created, + saw_in_progress, + saw_completed, + final_usage_ok, + ) + + def run_chat_completion_stream(self, logprobs=None, parallel_sample_num=1): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + generator = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "What is the capital of France?"}, + ], + temperature=0, + logprobs=logprobs is not None and logprobs > 0, + top_logprobs=logprobs, + stream=True, + stream_options={"include_usage": True}, + n=parallel_sample_num, + ) + for _ in generator: + pass + + # ---- tests ---- + def test_response(self): + resp = self.run_response(temperature=0, max_output_tokens=32) + assert resp.id + assert resp.object == "response" + assert resp.created_at + assert isinstance(resp.model, str) + assert isinstance(resp.output, list) + assert resp.status in ( + "completed", + "in_progress", + "queued", + "failed", + "cancelled", + ) + if resp.status == "completed": + assert resp.usage is not None + assert resp.usage.prompt_tokens >= 0 + assert resp.usage.completion_tokens >= 0 + assert resp.usage.total_tokens >= 0 + if hasattr(resp, "error"): + assert resp.error is None + if hasattr(resp, "incomplete_details"): + assert resp.incomplete_details is None + if getattr(resp, "text", None): + fmt = resp.text.get("format") if isinstance(resp.text, dict) else None + if fmt: + assert fmt.get("type") == "text" + + def test_response_stream(self): + aggregated_text, saw_created, saw_in_progress, saw_completed, final_usage_ok = ( + self.run_response_stream(temperature=0, max_output_tokens=32) + ) + assert saw_created, "Did not observe response.created" + assert saw_in_progress, "Did not observe response.in_progress" + assert saw_completed, "Did not observe response.completed" + assert isinstance(aggregated_text, str) + assert len(aggregated_text) >= 0 + assert final_usage_ok or True # final_usage's stats are not done for now + + def test_response_completion(self): + resp = self.run_response(temperature=0, max_output_tokens=16) + assert resp.status in ("completed", "in_progress", "queued") + if resp.status == "completed": + assert resp.usage is not None + assert resp.usage.total_tokens >= 0 + + def test_response_completion_stream(self): + _, saw_created, saw_in_progress, saw_completed, final_usage_ok = ( + self.run_response_stream(temperature=0, max_output_tokens=16) + ) + assert saw_created + assert saw_in_progress + assert saw_completed + assert final_usage_ok or True # final_usage's stats are not done for now + + def test_regex(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + regex = ( + r"""\{\n""" + + r""" "name": "[\w]+",\n""" + + r""" "population": [\d]+\n""" + + r"""\}""" + ) + + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "Introduce the capital of France."}, + ], + temperature=0, + max_tokens=128, + extra_body={"regex": regex}, + ) + text = response.choices[0].message.content + + try: + js_obj = json.loads(text) + except (TypeError, json.decoder.JSONDecodeError): + print("JSONDecodeError", text) + raise + assert isinstance(js_obj["name"], str) + assert isinstance(js_obj["population"], int) + + def test_error(self): + url = f"{self.base_url}/responses" + headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + } + payload = { + "model": self.model, + "input": "Hi", + "previous_response_id": "bad", # invalid prefix + } + r = requests.post(url, headers=headers, json=payload) + self.assertEqual(r.status_code, 400) + body = r.json() + self.assertIn("error", body) + self.assertIn("message", body["error"]) + self.assertIn("type", body["error"]) + self.assertIn("code", body["error"]) + + def test_penalty(self): + url = f"{self.base_url}/responses" + headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + } + payload = { + "model": self.model, + "input": "Introduce the capital of France.", + "temperature": 0, + "max_output_tokens": 32, + "frequency_penalty": 1.0, + } + r = requests.post(url, headers=headers, json=payload) + self.assertEqual(r.status_code, 200) + body = r.json() + self.assertEqual(body.get("object"), "response") + self.assertIn("output", body) + self.assertIn("status", body) + if "usage" in body: + self.assertIn("prompt_tokens", body["usage"]) + self.assertIn("total_tokens", body["usage"]) + + def test_response_prefill(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.chat.completions.create( + model="meta-llama/Llama-3.1-8B-Instruct", + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + { + "role": "user", + "content": """ +Extract the name, size, price, and color from this product description as a JSON object: + + +The SmartHome Mini is a compact smart home assistant available in black or white for only $49.99. At just 5 inches wide, it lets you control lights, thermostats, and other connected devices via voice or app—no matter where you place it in your home. This affordable little hub brings convenient hands-free control to your smart devices. + +""", + }, + { + "role": "assistant", + "content": "{\n", + }, + ], + temperature=0, + extra_body={"continue_final_message": True}, + ) + + assert ( + response.choices[0] + .message.content.strip() + .startswith('"name": "SmartHome Mini",') + ) + + def test_model_list(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + models = list(client.models.list()) + assert len(models) == 1 + assert isinstance(getattr(models[0], "max_model_len", None), int) + + +class TestOpenAIV1Rerank(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_CROSS_ENCODER_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.score_tolerance = 1e-2 + + # Configure embedding-specific args + other_args = [ + "--is-embedding", + "--enable-metrics", + "--disable-radix-cache", + "--chunked-prefill-size", + "-1", + "--attention-backend", + "torch_native", + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=other_args, + ) + cls.base_url += "/v1/rerank" + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_rerank(self, query, docs): + response = requests.post( + self.base_url, + headers={ + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + }, + json={"query": query, "documents": docs}, + ) + + return response.json() + + def test_rerank_single(self): + """Test single rerank request""" + query = TEST_RERANK_QUERY_DOCS[0]["query"] + docs = TEST_RERANK_QUERY_DOCS[0]["documents"] + + response = self.run_rerank(query, docs) + + self.assertEqual(len(response), 1) + self.assertTrue(isinstance(response[0]["score"], float)) + self.assertTrue(isinstance(response[0]["document"], str)) + self.assertTrue(isinstance(response[0]["index"], int)) + + def test_rerank_batch(self): + """Test batch rerank request""" + query = TEST_RERANK_QUERY_DOCS[1]["query"] + docs = TEST_RERANK_QUERY_DOCS[1]["documents"] + + response = self.run_rerank(query, docs) + + self.assertEqual(len(response), 2) + self.assertTrue(isinstance(response[0]["score"], float)) + self.assertTrue(isinstance(response[1]["score"], float)) + self.assertTrue(isinstance(response[0]["document"], str)) + self.assertTrue(isinstance(response[1]["document"], str)) + self.assertTrue(isinstance(response[0]["index"], int)) + self.assertTrue(isinstance(response[1]["index"], int)) + + +class TestOpenAIServerCustomLogitProcessor(CustomTestCase): + @classmethod + def setUpClass(cls) -> None: + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=["--enable-custom-logit-processor"], + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + @classmethod + def tearDownClass(cls) -> None: + kill_process_tree(cls.process.pid) + + def run_custom_logit_processor(self, target_token_id: Optional[int] = None) -> None: + """ + Test custom logit processor with custom params. + + If target_token_id is None, the custom logit processor won't be passed in. + """ + + class DeterministicLogitProcessor(CustomLogitProcessor): + """A dummy logit processor that changes the logits to always sample the given token id.""" + + CUSTOM_PARAM_KEY = "token_id" + + def __call__(self, logits, custom_param_list): + assert logits.shape[0] == len(custom_param_list) + + for i, param_dict in enumerate(custom_param_list): + # Mask all other tokens + logits[i, :] = -float("inf") + # Assign highest probability to the specified token + logits[i, param_dict[self.CUSTOM_PARAM_KEY]] = 0.0 + + return logits + + extra_body = {} + + if target_token_id is not None: + extra_body["custom_logit_processor"] = ( + DeterministicLogitProcessor().to_str() + ) + extra_body["custom_params"] = { + "token_id": target_token_id, + } + + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + max_tokens = 200 + + response = client.chat.completions.create( + model=self.model, + messages=[ + { + "role": "user", + "content": "Question: Is Paris the Capital of France?", + }, + ], + temperature=0.0, + max_tokens=max_tokens, + extra_body=extra_body, + ) + + if target_token_id is not None: + target_text = self.tokenizer.decode([target_token_id] * max_tokens) + self.assertTrue( + target_text == response.choices[0].message.content, + f"{target_token_id=}\n{target_text=}\n{response.model_dump(mode='json')}", + ) + + def test_custom_logit_processor(self) -> None: + """Test custom logit processor with a single request.""" + self.run_custom_logit_processor(target_token_id=5) + + def test_custom_logit_processor_batch_mixed(self) -> None: + """Test a batch of requests mixed of requests with and without custom logit processor.""" + target_token_ids = list(range(32)) + [None] * 16 + random.shuffle(target_token_ids) + with ThreadPoolExecutor(len(target_token_ids)) as executor: + list(executor.map(self.run_custom_logit_processor, target_token_ids)) + + +class TestOpenAIV1Score(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + ) + cls.base_url += "/v1/score" + cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_score( + self, query, items, label_token_ids, apply_softmax=False, item_first=False + ): + response = requests.post( + self.base_url, + headers={ + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + }, + json={ + "model": self.model, + "query": query, + "items": items, + "label_token_ids": label_token_ids, + "apply_softmax": apply_softmax, + "item_first": item_first, + }, + ) + return response.json() + + def test_score_text_input(self): + """Test scoring with text input""" + query = "The capital of France is" + items = ["Paris", "London", "Berlin"] + + # Get valid token IDs from the tokenizer + label_token_ids = [] + for item in items: + token_ids = self.tokenizer.encode(item, add_special_tokens=False) + if not token_ids: + self.fail(f"Failed to encode item: {item}") + label_token_ids.append(token_ids[0]) + + response = self.run_score(query, items, label_token_ids, apply_softmax=True) + + # Handle error responses + if response.get("type") == "BadRequestError": + self.fail(f"Score request failed with error: {response['message']}") + + # Verify response structure + self.assertIn("scores", response, "Response should have a 'scores' field") + self.assertIsInstance(response["scores"], list, "scores should be a list") + self.assertEqual( + len(response["scores"]), + len(items), + "Number of scores should match number of items", + ) + + # Each score should be a list of floats in the order of label_token_ids + for i, score_list in enumerate(response["scores"]): + self.assertIsInstance(score_list, list, f"Score {i} should be a list") + self.assertEqual( + len(score_list), + len(label_token_ids), + f"Score {i} length should match label_token_ids", + ) + self.assertTrue( + all(isinstance(v, float) for v in score_list), + f"Score {i} values should be floats", + ) + self.assertAlmostEqual( + sum(score_list), + 1.0, + places=6, + msg=f"Score {i} probabilities should sum to 1", + ) + + def test_score_token_input(self): + """Test scoring with token IDs input""" + query = "The capital of France is" + items = ["Paris", "London", "Berlin"] + + # Get valid token IDs + query_ids = self.tokenizer.encode(query, add_special_tokens=False) + item_ids = [ + self.tokenizer.encode(item, add_special_tokens=False) for item in items + ] + label_token_ids = [ + ids[0] for ids in item_ids if ids + ] # Get first token ID of each item + + response = self.run_score( + query_ids, item_ids, label_token_ids, apply_softmax=True + ) + + # Handle error responses + if response.get("type") == "BadRequestError": + self.fail(f"Score request failed with error: {response['message']}") + + # Verify response structure + self.assertIn("scores", response, "Response should have a 'scores' field") + self.assertIsInstance(response["scores"], list, "scores should be a list") + self.assertEqual( + len(response["scores"]), + len(items), + "Number of scores should match number of items", + ) + + # Each score should be a list of floats in the order of label_token_ids + for i, score_list in enumerate(response["scores"]): + self.assertIsInstance(score_list, list, f"Score {i} should be a list") + self.assertEqual( + len(score_list), + len(label_token_ids), + f"Score {i} length should match label_token_ids", + ) + self.assertTrue( + all(isinstance(v, float) for v in score_list), + f"Score {i} values should be floats", + ) + self.assertAlmostEqual( + sum(score_list), + 1.0, + places=6, + msg=f"Score {i} probabilities should sum to 1", + ) + + def test_score_error_handling(self): + """Test error handling for invalid inputs""" + query = "The capital of France is" + items = ["Paris", "London", "Berlin"] + + # Test with invalid token ID + response = requests.post( + self.base_url, + headers={ + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + }, + json={ + "model": self.model, + "query": query, + "items": items, + "label_token_ids": [999999], # Invalid token ID + "apply_softmax": True, + }, + ) + self.assertEqual(response.status_code, 400) + error_response = response.json() + self.assertEqual(error_response["type"], "BadRequestError") + self.assertIn("Token ID 999999 is out of vocabulary", error_response["message"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_protocol.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_protocol.py new file mode 100644 index 00000000..289b6638 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_protocol.py @@ -0,0 +1,350 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for OpenAI API protocol models""" + +import unittest +from typing import List, Optional + +from pydantic import BaseModel, Field, ValidationError + +from sglang.srt.entrypoints.openai.protocol import ( + ChatCompletionRequest, + ChatCompletionResponse, + ChatCompletionResponseChoice, + ChatMessage, + CompletionRequest, + ModelCard, + ModelList, + UsageInfo, +) + + +class TestModelCard(unittest.TestCase): + """Test ModelCard protocol model""" + + def test_model_card_serialization(self): + """Test model card JSON serialization""" + card = ModelCard(id="test-model", max_model_len=4096) + data = card.model_dump() + self.assertEqual(data["id"], "test-model") + self.assertEqual(data["object"], "model") + self.assertEqual(data["max_model_len"], 4096) + + +class TestModelList(unittest.TestCase): + """Test ModelList protocol model""" + + def test_empty_model_list(self): + """Test empty model list creation""" + model_list = ModelList() + self.assertEqual(model_list.object, "list") + self.assertEqual(len(model_list.data), 0) + + def test_model_list_with_cards(self): + """Test model list with model cards""" + cards = [ + ModelCard(id="model-1"), + ModelCard(id="model-2", max_model_len=2048), + ] + model_list = ModelList(data=cards) + self.assertEqual(len(model_list.data), 2) + self.assertEqual(model_list.data[0].id, "model-1") + self.assertEqual(model_list.data[1].id, "model-2") + + +class TestCompletionRequest(unittest.TestCase): + """Test CompletionRequest protocol model""" + + def test_basic_completion_request(self): + """Test basic completion request""" + request = CompletionRequest(model="test-model", prompt="Hello world") + self.assertEqual(request.model, "test-model") + self.assertEqual(request.prompt, "Hello world") + self.assertEqual(request.max_tokens, 16) # default + self.assertEqual(request.temperature, 1.0) # default + self.assertEqual(request.n, 1) # default + self.assertFalse(request.stream) # default + self.assertFalse(request.echo) # default + + def test_completion_request_sglang_extensions(self): + """Test completion request with SGLang-specific extensions""" + request = CompletionRequest( + model="test-model", + prompt="Hello", + top_k=50, + min_p=0.1, + repetition_penalty=1.1, + regex=r"\d+", + json_schema='{"type": "object"}', + lora_path="/path/to/lora", + ) + self.assertEqual(request.top_k, 50) + self.assertEqual(request.min_p, 0.1) + self.assertEqual(request.repetition_penalty, 1.1) + self.assertEqual(request.regex, r"\d+") + self.assertEqual(request.json_schema, '{"type": "object"}') + self.assertEqual(request.lora_path, "/path/to/lora") + + def test_completion_request_validation_errors(self): + """Test completion request validation errors""" + with self.assertRaises(ValidationError): + CompletionRequest() # missing required fields + + with self.assertRaises(ValidationError): + CompletionRequest(model="test-model") # missing prompt + + +class TestChatCompletionRequest(unittest.TestCase): + """Test ChatCompletionRequest protocol model""" + + def test_basic_chat_completion_request(self): + """Test basic chat completion request""" + messages = [{"role": "user", "content": "Hello"}] + request = ChatCompletionRequest(model="test-model", messages=messages) + self.assertEqual(request.model, "test-model") + self.assertEqual(len(request.messages), 1) + self.assertEqual(request.messages[0].role, "user") + self.assertEqual(request.messages[0].content, "Hello") + self.assertEqual(request.temperature, None) # default + self.assertFalse(request.stream) # default + self.assertEqual(request.tool_choice, "none") # default when no tools + + def test_sampling_param_build(self): + req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Hi"}], + temperature=0.8, + max_tokens=150, + min_tokens=5, + top_p=0.9, + stop=[""], + ) + params = req.to_sampling_params([""], {}, None) + self.assertEqual(params["temperature"], 0.8) + self.assertEqual(params["max_new_tokens"], 150) + self.assertEqual(params["min_new_tokens"], 5) + self.assertEqual(params["stop"], [""]) + + def test_chat_completion_tool_choice_validation(self): + """Test tool choice validation logic""" + messages = [{"role": "user", "content": "Hello"}] + + # No tools, tool_choice should default to "none" + request1 = ChatCompletionRequest(model="test-model", messages=messages) + self.assertEqual(request1.tool_choice, "none") + + # With tools, tool_choice should default to "auto" + tools = [ + { + "type": "function", + "function": {"name": "test_func", "description": "Test function"}, + } + ] + request2 = ChatCompletionRequest( + model="test-model", messages=messages, tools=tools + ) + self.assertEqual(request2.tool_choice, "auto") + + def test_chat_completion_sglang_extensions(self): + """Test chat completion with SGLang extensions""" + messages = [{"role": "user", "content": "Hello"}] + request = ChatCompletionRequest( + model="test-model", + messages=messages, + top_k=40, + min_p=0.05, + separate_reasoning=False, + stream_reasoning=False, + chat_template_kwargs={"custom_param": "value"}, + ) + self.assertEqual(request.top_k, 40) + self.assertEqual(request.min_p, 0.05) + self.assertFalse(request.separate_reasoning) + self.assertFalse(request.stream_reasoning) + self.assertEqual(request.chat_template_kwargs, {"custom_param": "value"}) + + def test_chat_completion_reasoning_effort(self): + """Test chat completion with reasoning effort""" + messages = [{"role": "user", "content": "Hello"}] + request = ChatCompletionRequest( + model="test-model", + messages=messages, + reasoning={ + "enabled": True, + "reasoning_effort": "high", + }, + ) + self.assertEqual(request.reasoning_effort, "high") + self.assertEqual(request.chat_template_kwargs, {"thinking": True}) + + def test_chat_completion_json_format(self): + """Test chat completion json format""" + transcript = "Good morning! It's 7:00 AM, and I'm just waking up. Today is going to be a busy day, " + "so let's get started. First, I need to make a quick breakfast. I think I'll have some " + "scrambled eggs and toast with a cup of coffee. While I'm cooking, I'll also check my " + "emails to see if there's anything urgent." + + messages = [ + { + "role": "system", + "content": "The following is a voice message transcript. Only answer in JSON.", + }, + { + "role": "user", + "content": transcript, + }, + ] + + class VoiceNote(BaseModel): + title: str = Field(description="A title for the voice note") + summary: str = Field( + description="A short one sentence summary of the voice note." + ) + strict: Optional[bool] = True + actionItems: List[str] = Field( + description="A list of action items from the voice note" + ) + + request = ChatCompletionRequest( + model="test-model", + messages=messages, + top_k=40, + min_p=0.05, + separate_reasoning=False, + stream_reasoning=False, + chat_template_kwargs={"custom_param": "value"}, + response_format={ + "type": "json_schema", + "schema": VoiceNote.model_json_schema(), + }, + ) + res_format = request.response_format + json_format = res_format.json_schema + name = json_format.name + schema = json_format.schema_ + strict = json_format.strict + self.assertEqual(name, "VoiceNote") + self.assertEqual(strict, True) + self.assertNotIn("strict", schema["properties"]) + + request = ChatCompletionRequest( + model="test-model", + messages=messages, + top_k=40, + min_p=0.05, + separate_reasoning=False, + stream_reasoning=False, + chat_template_kwargs={"custom_param": "value"}, + response_format={ + "type": "json_schema", + "json_schema": { + "name": "VoiceNote", + "schema": VoiceNote.model_json_schema(), + "strict": True, + }, + }, + ) + res_format = request.response_format + json_format = res_format.json_schema + name = json_format.name + schema = json_format.schema_ + strict = json_format.strict + self.assertEqual(name, "VoiceNote") + self.assertEqual(strict, True) + + +class TestModelSerialization(unittest.TestCase): + """Test model serialization with hidden states""" + + def test_hidden_states_excluded_when_none(self): + """Test that None hidden_states are excluded with exclude_none=True""" + choice = ChatCompletionResponseChoice( + index=0, + message=ChatMessage(role="assistant", content="Hello"), + finish_reason="stop", + hidden_states=None, + ) + + response = ChatCompletionResponse( + id="test-id", + model="test-model", + choices=[choice], + usage=UsageInfo(prompt_tokens=5, completion_tokens=1, total_tokens=6), + ) + + # Test exclude_none serialization (should exclude None hidden_states) + data = response.model_dump(exclude_none=True) + self.assertNotIn("hidden_states", data["choices"][0]) + + def test_hidden_states_included_when_not_none(self): + """Test that non-None hidden_states are included""" + choice = ChatCompletionResponseChoice( + index=0, + message=ChatMessage(role="assistant", content="Hello"), + finish_reason="stop", + hidden_states=[0.1, 0.2, 0.3], + ) + + response = ChatCompletionResponse( + id="test-id", + model="test-model", + choices=[choice], + usage=UsageInfo(prompt_tokens=5, completion_tokens=1, total_tokens=6), + ) + + # Test exclude_none serialization (should include non-None hidden_states) + data = response.model_dump(exclude_none=True) + self.assertIn("hidden_states", data["choices"][0]) + self.assertEqual(data["choices"][0]["hidden_states"], [0.1, 0.2, 0.3]) + + +class TestValidationEdgeCases(unittest.TestCase): + """Test edge cases and validation scenarios""" + + def test_invalid_tool_choice_type(self): + """Test invalid tool choice type""" + messages = [{"role": "user", "content": "Hello"}] + with self.assertRaises(ValidationError): + ChatCompletionRequest( + model="test-model", messages=messages, tool_choice=123 + ) + + def test_negative_token_limits(self): + """Test negative token limits""" + with self.assertRaises(ValidationError): + CompletionRequest(model="test-model", prompt="Hello", max_tokens=-1) + + def test_model_serialization_roundtrip(self): + """Test that models can be serialized and deserialized""" + original_request = ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "Hello"}], + temperature=0.7, + max_tokens=100, + ) + + # Serialize to dict + data = original_request.model_dump() + + # Deserialize back + restored_request = ChatCompletionRequest(**data) + + self.assertEqual(restored_request.model, original_request.model) + self.assertEqual(restored_request.temperature, original_request.temperature) + self.assertEqual(restored_request.max_tokens, original_request.max_tokens) + self.assertEqual(len(restored_request.messages), len(original_request.messages)) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_serving_chat.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_serving_chat.py new file mode 100644 index 00000000..ed20cbc9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_serving_chat.py @@ -0,0 +1,630 @@ +""" +Unit-tests for OpenAIServingChat — rewritten to use only the std-lib 'unittest'. +Run with either: + python tests/test_serving_chat_unit.py -v +or + python -m unittest discover -s tests -p "test_*unit.py" -v +""" + +import json +import unittest +import uuid +from typing import Optional +from unittest.mock import Mock, patch + +from fastapi import Request + +from sglang.srt.entrypoints.openai.protocol import ( + ChatCompletionRequest, + MessageProcessingResult, +) +from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat +from sglang.srt.managers.io_struct import GenerateReqInput +from sglang.srt.utils import get_or_create_event_loop + + +class _MockTokenizerManager: + """Minimal mock that satisfies OpenAIServingChat.""" + + def __init__(self): + self.model_config = Mock(is_multimodal=False) + self.server_args = Mock( + enable_cache_report=False, + tool_call_parser="hermes", + reasoning_parser=None, + ) + # Mock hf_config for _use_dpsk_v32_encoding check + mock_hf_config = Mock() + mock_hf_config.architectures = ["LlamaForCausalLM"] + self.server_args.get_hf_config.return_value = mock_hf_config + + self.chat_template_name: Optional[str] = "llama-3" + + # tokenizer stub + self.tokenizer = Mock() + self.tokenizer.encode.return_value = [1, 2, 3, 4, 5] + self.tokenizer.decode.return_value = "Test response" + self.tokenizer.chat_template = None + self.tokenizer.bos_token_id = 1 + + # async generator stub for generate_request + async def _mock_generate(): + yield { + "text": "Test response", + "meta_info": { + "id": f"chatcmpl-{uuid.uuid4()}", + "prompt_tokens": 10, + "completion_tokens": 5, + "cached_tokens": 0, + "finish_reason": {"type": "stop", "matched": None}, + "output_token_logprobs": [(0.1, 1, "Test"), (0.2, 2, "response")], + "output_top_logprobs": None, + }, + "index": 0, + } + + self.generate_request = Mock(return_value=_mock_generate()) + self.create_abort_task = Mock() + + +class _MockTemplateManager: + """Minimal mock for TemplateManager.""" + + def __init__(self): + self.chat_template_name: Optional[str] = "llama-3" + self.jinja_template_content_format: Optional[str] = None + self.completion_template_name: Optional[str] = None + + +class ServingChatTestCase(unittest.TestCase): + # ------------- common fixtures ------------- + def setUp(self): + self.tm = _MockTokenizerManager() + self.template_manager = _MockTemplateManager() + self.chat = OpenAIServingChat(self.tm, self.template_manager) + + # frequently reused requests + self.basic_req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Hi?"}], + temperature=0.7, + max_tokens=100, + stream=False, + ) + self.stream_req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Hi?"}], + temperature=0.7, + max_tokens=100, + stream=True, + ) + + self.fastapi_request = Mock(spec=Request) + self.fastapi_request.headers = {} + + # ------------- conversion tests ------------- + def test_convert_to_internal_request_single(self): + with patch( + "sglang.srt.entrypoints.openai.serving_chat.generate_chat_conv" + ) as conv_mock, patch.object(self.chat, "_process_messages") as proc_mock: + conv_ins = Mock() + conv_ins.get_prompt.return_value = "Test prompt" + conv_ins.image_data = conv_ins.audio_data = None + conv_ins.modalities = [] + conv_ins.stop_str = [""] + conv_mock.return_value = conv_ins + + proc_mock.return_value = MessageProcessingResult( + "Test prompt", + [1, 2, 3], + None, + None, + [], + [""], + None, + ) + + adapted, processed = self.chat._convert_to_internal_request(self.basic_req) + self.assertIsInstance(adapted, GenerateReqInput) + self.assertFalse(adapted.stream) + self.assertEqual(processed, self.basic_req) + + def test_stop_str_isolation_between_requests(self): + """Test that stop strings from one request don't affect subsequent requests. + + This tests the fix for the bug where conv.stop_str was being mutated globally, + causing stop strings from one request to persist in subsequent requests. + """ + # Mock conversation template with initial stop_str + initial_stop_str = ["\n"] + + with patch( + "sglang.srt.entrypoints.openai.serving_chat.generate_chat_conv" + ) as conv_mock: + # Create a mock conversation object that will be returned by generate_chat_conv + conv_ins = Mock() + conv_ins.get_prompt.return_value = "Test prompt" + conv_ins.image_data = None + conv_ins.audio_data = None + conv_ins.modalities = [] + conv_ins.stop_str = ( + initial_stop_str.copy() + ) # Template's default stop strings + conv_mock.return_value = conv_ins + + # First request with additional stop string + req1 = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "First request"}], + stop=["CUSTOM_STOP"], + ) + + # Call the actual _apply_conversation_template method (not mocked) + result1 = self.chat._apply_conversation_template(req1, is_multimodal=False) + + # Verify first request has both stop strings + expected_stop1 = initial_stop_str + ["CUSTOM_STOP"] + self.assertEqual(result1.stop, expected_stop1) + + # Verify the original template's stop_str wasn't mutated after first request + self.assertEqual(conv_ins.stop_str, initial_stop_str) + + # Second request without additional stop string + req2 = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Second request"}], + # No custom stop strings + ) + result2 = self.chat._apply_conversation_template(req2, is_multimodal=False) + + # Verify second request only has original stop strings (no CUSTOM_STOP from req1) + self.assertEqual(result2.stop, initial_stop_str) + self.assertNotIn("CUSTOM_STOP", result2.stop) + self.assertEqual(conv_ins.stop_str, initial_stop_str) + + def test_unstreamed_tool_args_completion(self): + """Test that remaining tool call arguments are sent when generation finishes.""" + + # Mock FunctionCallParser with detector that has partial tool call data + mock_parser = Mock() + mock_detector = Mock() + + # Simulate a tool call that was partially streamed + mock_detector.prev_tool_call_arr = [ + { + "name": "get_weather", + "arguments": {"location": "San Francisco", "unit": "celsius"}, + } + ] + mock_detector.streamed_args_for_tool = [ + '{"location": "San Francisco"' # Partial arguments streamed so far + ] + mock_parser.detector = mock_detector + + content = { + "meta_info": { + "id": "chatcmpl-test123", + } + } + + request = ChatCompletionRequest( + model="test", + messages=[{"role": "user", "content": "What's the weather?"}], + tools=[{"type": "function", "function": {"name": "get_weather"}}], + ) + + # Test the completion method + result = self.chat._check_for_unstreamed_tool_args( + parser=mock_parser, + content=content, + request=request, + index=0, + ) + + # Should return a chunk with remaining arguments + self.assertIsNotNone(result, "Should return chunk with remaining arguments") + + # Parse the result to verify content + self.assertTrue(result.startswith("data: ")) + chunk = json.loads(result[6:]) + tool_calls = chunk["choices"][0]["delta"]["tool_calls"] + self.assertEqual(len(tool_calls), 1) + arguments = tool_calls[0]["function"]["arguments"] + self.assertIn(', "unit": "celsius"}', arguments) + + self.assertIn( + '"finish_reason":null', + result, + "Should not include finish_reason in completion chunk", + ) + + def test_unstreamed_tool_args_no_completion_needed(self): + """Test that no completion chunk is sent when all arguments were already streamed.""" + + # Mock FunctionCallParser with detector that has complete tool call data + mock_parser = Mock() + mock_detector = Mock() + + # Simulate a tool call that was completely streamed + mock_detector.prev_tool_call_arr = [ + {"name": "get_weather", "arguments": {"location": "San Francisco"}} + ] + mock_detector.streamed_args_for_tool = [ + '{"location": "San Francisco"}' # All arguments already streamed + ] + mock_parser.detector = mock_detector + + content = { + "meta_info": { + "id": "chatcmpl-test123", + } + } + + request = ChatCompletionRequest( + model="test", + messages=[{"role": "user", "content": "What's the weather?"}], + tools=[{"type": "function", "function": {"name": "get_weather"}}], + ) + + # Test the completion method + result = self.chat._check_for_unstreamed_tool_args( + parser=mock_parser, + content=content, + request=request, + index=0, + ) + + # Should return None since no completion is needed + self.assertIsNone(result, "Should return None when no completion is needed") + + def test_unstreamed_tool_args_no_parser_data(self): + """Test that no completion chunk is sent when parser has no tool call data.""" + + # Mock FunctionCallParser with empty detector + mock_parser = Mock() + mock_detector = Mock() + mock_detector.prev_tool_call_arr = [] + mock_detector.streamed_args_for_tool = [] + mock_parser.detector = mock_detector + + content = { + "meta_info": { + "id": "chatcmpl-test123", + } + } + + request = ChatCompletionRequest( + model="test", + messages=[{"role": "user", "content": "What's the weather?"}], + tools=[{"type": "function", "function": {"name": "get_weather"}}], + ) + + # Test the completion method + result = self.chat._check_for_unstreamed_tool_args( + parser=mock_parser, + content=content, + request=request, + index=0, + ) + + # Should return None since there's no parser data + self.assertIsNone( + result, "Should return None when parser has no tool call data" + ) + + # ------------- kimi_k2 tool_call_id formatting ------------- + def test_kimi_k2_non_streaming_tool_call_id_format(self): + """Ensure non-streaming tool_call.id matches functions.{name}:{index} for kimi_k2 parser.""" + + # Force kimi_k2 parser + self.chat.tool_call_parser = "kimi_k2" + + # Mock FunctionCallParser.parse_non_stream to return one tool call + with patch( + "sglang.srt.entrypoints.openai.serving_chat.FunctionCallParser" + ) as ParserMock: + parser_instance = ParserMock.return_value + + # Build a mock ToolCallItem-like object + call_info = Mock() + call_info.name = "get_weather" + call_info.parameters = '{"city":"Paris"}' + call_info.tool_index = 0 + + parser_instance.has_tool_call.return_value = True + parser_instance.parse_non_stream.return_value = ("", [call_info]) + + finish_reason = {"type": "stop", "matched": None} + tools = [ + {"type": "function", "function": {"name": "get_weather"}}, + ] + + tool_calls, remaining_text, finish_reason = self.chat._process_tool_calls( + text="<|tool_calls_section_begin|>...", + tools=tools, + finish_reason=finish_reason, + ) + + self.assertIsNotNone(tool_calls) + self.assertEqual(len(tool_calls), 1) + self.assertEqual(tool_calls[0].id, "functions.get_weather:0") + self.assertEqual(tool_calls[0].function.name, "get_weather") + + def test_kimi_k2_streaming_tool_call_id_format(self): + """Ensure streaming first chunk tool_call.id matches functions.{name}:{index} for kimi_k2 parser.""" + + # Force kimi_k2 parser + self.chat.tool_call_parser = "kimi_k2" + + # Prepare request with tools + req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Hi?"}], + tools=[{"type": "function", "function": {"name": "get_weather"}}], + stream=True, + ) + + # Patch FunctionCallParser used inside _process_tool_call_stream + with patch( + "sglang.srt.entrypoints.openai.serving_chat.FunctionCallParser" + ) as ParserMock: + parser_instance = ParserMock.return_value + + # First call returns one ToolCallItem-like chunk (with name) + first_chunk_call = Mock() + first_chunk_call.tool_index = 0 + first_chunk_call.name = "get_weather" + first_chunk_call.parameters = "" + parser_instance.parse_stream_chunk.side_effect = [ + ("", [first_chunk_call]), + ("", []), + ] + + async def collect_first_tool_chunk(): + gen = self.chat._process_tool_call_stream( + index=0, + delta="irrelevant", + parser_dict={}, + content={"meta_info": {"id": "chatcmpl-test"}}, + request=req, + has_tool_calls={}, + ) + # Get first yielded SSE line + line = None + async for emitted in gen: + line = emitted + break + return line + + loop = get_or_create_event_loop() + line = loop.run_until_complete(collect_first_tool_chunk()) + self.assertIsNotNone(line) + self.assertTrue(line.startswith("data: ")) + + payload = json.loads(line[len("data: ") :]) + tool_calls = payload["choices"][0]["delta"]["tool_calls"] + self.assertEqual(tool_calls[0]["id"], "functions.get_weather:0") + + def test_kimi_k2_non_streaming_tool_call_id_with_history(self): + """Ensure non-streaming tool_call.id increase with tool calls history for kimi_k2 parser.""" + + # Force kimi_k2 parser + self.chat.tool_call_parser = "kimi_k2" + + # Prepare request with tool calls history + req = ChatCompletionRequest( + model="x", + messages=[ + {"role": "user", "content": "What's the weather today in paris?"}, + { + "role": "assistant", + "content": "Let me do some search first.", + "tool_calls": [ + { + "id": "functions.get_weather:0", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "Paris"}', + }, + } + ], + }, + { + "role": "tool", + "content": "It's rainy in paris now.", + "tool_call_id": "functions.get_weather:0", + }, + { + "role": "assistant", + "content": "It's rainy now.", + }, + { + "role": "user", + "content": "What about LA and Tokyo?", + }, + ], + tools=[{"type": "function", "function": {"name": "get_weather"}}], + stream=False, + ) + + # Mock FunctionCallParser.parse_non_stream to return one tool call + with patch( + "sglang.srt.entrypoints.openai.serving_chat.FunctionCallParser" + ) as ParserMock: + parser_instance = ParserMock.return_value + + # Build a mock ToolCallItem-like object + call_info = Mock() + call_info.name = "get_weather" + call_info.parameters = '{"city":"Loa Angeles"}' + # Kimi-K2 series models might generate fixed number tool_indx, + # ignoring the tool calls history and mess up all the following tool calls + call_info.tool_index = 0 + + call_info2 = Mock() + call_info2.name = "get_weather" + call_info2.parameters = '{"city":"Tokyo"}' + call_info2.tool_index = 1 + + parser_instance.has_tool_call.return_value = True + parser_instance.parse_non_stream.return_value = ( + "", + [call_info, call_info2], + ) + + finish_reason = {"type": "stop", "matched": None} + tools = [ + {"type": "function", "function": {"name": "get_weather"}}, + ] + + history_tool_calls_cnt = self.chat._get_history_tool_calls_cnt(req) + tool_calls, remaining_text, _ = self.chat._process_tool_calls( + text="<|tool_calls_section_begin|>...", + tools=tools, + finish_reason=finish_reason, + history_tool_calls_cnt=history_tool_calls_cnt, + ) + + self.assertEqual(history_tool_calls_cnt, 1) + self.assertIsNotNone(tool_calls) + self.assertEqual(len(tool_calls), 2) + self.assertEqual(tool_calls[0].id, "functions.get_weather:1") + self.assertEqual(tool_calls[0].function.name, "get_weather") + self.assertEqual(tool_calls[1].id, "functions.get_weather:2") + self.assertEqual(tool_calls[1].function.name, "get_weather") + + def test_kimi_k2_streaming_tool_call_id_with_history(self): + """Ensure streaming first chunk tool_call.id increase with tool calls history for kimi_k2 parser.""" + + # Force kimi_k2 parser + self.chat.tool_call_parser = "kimi_k2" + + # Prepare request with tool calls history + req = ChatCompletionRequest( + model="x", + messages=[ + {"role": "user", "content": "What's the weather today in paris?"}, + { + "role": "assistant", + "content": "Let me do some search first.", + "tool_calls": [ + { + "id": "functions.get_weather:0", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "Paris"}', + }, + } + ], + }, + { + "role": "tool", + "content": "It's rainy in paris now.", + "tool_call_id": "functions.get_weather:0", + }, + { + "role": "assistant", + "content": "It's rainy now.", + }, + { + "role": "user", + "content": "What about LA?", + }, + ], + tools=[{"type": "function", "function": {"name": "get_weather"}}], + stream=True, + ) + + # Patch FunctionCallParser used inside _process_tool_call_stream + with patch( + "sglang.srt.entrypoints.openai.serving_chat.FunctionCallParser" + ) as ParserMock: + parser_instance = ParserMock.return_value + + # First call returns one ToolCallItem-like chunk (with name) + first_chunk_call = Mock() + # Kimi-K2 series models might generate fixed number tool_indx, + # ignoring the tool calls history and mess up all the following tool calls + first_chunk_call.tool_index = 0 + first_chunk_call.name = "get_weather" + first_chunk_call.parameters = "" + parser_instance.parse_stream_chunk.side_effect = [ + ("", [first_chunk_call]), + ("", []), + ] + + async def collect_first_tool_chunk(): + gen = self.chat._process_tool_call_stream( + index=0, + delta="irrelevant", + parser_dict={}, + content={"meta_info": {"id": "chatcmpl-test"}}, + request=req, + has_tool_calls={}, + ) + # Get first yielded SSE line + line = None + async for emitted in gen: + line = emitted + break + return line + + loop = get_or_create_event_loop() + line = loop.run_until_complete(collect_first_tool_chunk()) + self.assertIsNotNone(line) + self.assertTrue(line.startswith("data: ")) + + payload = json.loads(line[len("data: ") :]) + tool_calls = payload["choices"][0]["delta"]["tool_calls"] + self.assertEqual(tool_calls[0]["id"], "functions.get_weather:1") + + def test_dpsk_v32_encoding_path(self): + """Test DeepSeek V3.2 encoding path detection and application.""" + from sglang.srt.managers.template_manager import TemplateManager + from sglang.srt.server_args import PortArgs, ServerArgs + + server_args = ServerArgs(model_path="deepseek-ai/DeepSeek-V3.2") + port_args = PortArgs.init_new(server_args) + + # Use mocks for TokenizerManager components to avoid full initialization + with patch( + "sglang.srt.managers.tokenizer_manager.TokenizerManager" + ) as MockTokenizerManager: + tokenizer_manager = MockTokenizerManager(server_args, port_args) + tokenizer_manager.server_args = server_args + tokenizer_manager.model_config = Mock() + tokenizer_manager.model_config.get_default_sampling_params.return_value = ( + None + ) + + # Mock hf_config + mock_hf_config = Mock() + mock_hf_config.architectures = ["DeepseekV32ForCausalLM"] + + tokenizer_manager.server_args.get_hf_config = Mock( + return_value=mock_hf_config + ) + + # Case 1: No chat template in tokenizer -> should use dpsk encoding + tokenizer_manager.tokenizer = Mock() + tokenizer_manager.tokenizer.chat_template = None + + serving_chat = OpenAIServingChat(tokenizer_manager, TemplateManager()) + self.assertTrue(serving_chat.use_dpsk_v32_encoding) + + # Case 2: Chat template exists -> should NOT use dpsk encoding + tokenizer_manager.tokenizer.chat_template = "some template" + serving_chat = OpenAIServingChat(tokenizer_manager, TemplateManager()) + self.assertFalse(serving_chat.use_dpsk_v32_encoding) + + # Case 3: Not DeepSeek V3.2 architecture -> should NOT use dpsk encoding + tokenizer_manager.tokenizer.chat_template = None + mock_hf_config.architectures = ["LlamaForCausalLM"] + serving_chat = OpenAIServingChat(tokenizer_manager, TemplateManager()) + self.assertFalse(serving_chat.use_dpsk_v32_encoding) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_serving_completions.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_serving_completions.py new file mode 100644 index 00000000..93aba834 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_serving_completions.py @@ -0,0 +1,157 @@ +""" +Unit-tests for the refactored completions-serving handler (no pytest). +Run with: + python -m unittest tests.test_serving_completions_unit -v +""" + +import unittest +from typing import Optional +from unittest.mock import AsyncMock, Mock + +from sglang.srt.entrypoints.openai.protocol import CompletionRequest +from sglang.srt.entrypoints.openai.serving_completions import OpenAIServingCompletion +from sglang.srt.managers.tokenizer_manager import TokenizerManager + + +class _MockTemplateManager: + """Minimal mock for TemplateManager.""" + + def __init__(self): + self.chat_template_name: Optional[str] = None + self.jinja_template_content_format: Optional[str] = None + self.completion_template_name: Optional[str] = ( + None # Set to None to avoid template processing + ) + + +class ServingCompletionTestCase(unittest.TestCase): + """Bundle all prompt/echo tests in one TestCase.""" + + # ---------- shared test fixtures ---------- + def setUp(self): + # build the mock TokenizerManager once for every test + tm = Mock(spec=TokenizerManager) + + tm.tokenizer = Mock() + tm.tokenizer.encode.return_value = [1, 2, 3, 4] + tm.tokenizer.decode.return_value = "decoded text" + tm.tokenizer.bos_token_id = 1 + + tm.model_config = Mock(is_multimodal=False) + tm.server_args = Mock(enable_cache_report=False) + + tm.generate_request = AsyncMock() + tm.create_abort_task = Mock() + + self.template_manager = _MockTemplateManager() + self.sc = OpenAIServingCompletion(tm, self.template_manager) + + # ---------- prompt-handling ---------- + def test_single_string_prompt(self): + req = CompletionRequest(model="x", prompt="Hello world", max_tokens=100) + internal, _ = self.sc._convert_to_internal_request(req) + self.assertEqual(internal.text, "Hello world") + + def test_single_token_ids_prompt(self): + req = CompletionRequest(model="x", prompt=[1, 2, 3, 4], max_tokens=100) + internal, _ = self.sc._convert_to_internal_request(req) + self.assertEqual(internal.input_ids, [1, 2, 3, 4]) + + # ---------- echo-handling ---------- + def test_echo_with_string_prompt_streaming(self): + req = CompletionRequest(model="x", prompt="Hello", max_tokens=1, echo=True) + self.assertEqual(self.sc._get_echo_text(req, 0), "Hello") + + def test_echo_with_list_of_strings_streaming(self): + req = CompletionRequest( + model="x", prompt=["A", "B"], max_tokens=1, echo=True, n=1 + ) + self.assertEqual(self.sc._get_echo_text(req, 0), "A") + self.assertEqual(self.sc._get_echo_text(req, 1), "B") + + def test_echo_with_token_ids_streaming(self): + req = CompletionRequest(model="x", prompt=[1, 2, 3], max_tokens=1, echo=True) + self.sc.tokenizer_manager.tokenizer.decode.return_value = "decoded_prompt" + self.assertEqual(self.sc._get_echo_text(req, 0), "decoded_prompt") + + def test_echo_with_multiple_token_ids_streaming(self): + req = CompletionRequest( + model="x", prompt=[[1, 2], [3, 4]], max_tokens=1, echo=True, n=1 + ) + self.sc.tokenizer_manager.tokenizer.decode.return_value = "decoded" + self.assertEqual(self.sc._get_echo_text(req, 0), "decoded") + + def test_prepare_echo_prompts_non_streaming(self): + # single string + req = CompletionRequest(model="x", prompt="Hi", echo=True) + self.assertEqual(self.sc._prepare_echo_prompts(req), ["Hi"]) + + # list of strings + req = CompletionRequest(model="x", prompt=["Hi", "Yo"], echo=True) + self.assertEqual(self.sc._prepare_echo_prompts(req), ["Hi", "Yo"]) + + # token IDs + req = CompletionRequest(model="x", prompt=[1, 2, 3], echo=True) + self.sc.tokenizer_manager.tokenizer.decode.return_value = "decoded" + self.assertEqual(self.sc._prepare_echo_prompts(req), ["decoded"]) + + # ---------- response_format handling ---------- + def test_response_format_json_object(self): + """Test that response_format json_object is correctly processed in sampling params.""" + req = CompletionRequest( + model="x", + prompt="Generate a JSON object:", + max_tokens=100, + response_format={"type": "json_object"}, + ) + sampling_params = self.sc._build_sampling_params(req) + self.assertEqual(sampling_params["json_schema"], '{"type": "object"}') + + def test_response_format_json_schema(self): + """Test that response_format json_schema is correctly processed in sampling params.""" + schema = { + "type": "object", + "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, + } + req = CompletionRequest( + model="x", + prompt="Generate a JSON object:", + max_tokens=100, + response_format={ + "type": "json_schema", + "json_schema": {"name": "person", "schema": schema}, + }, + ) + sampling_params = self.sc._build_sampling_params(req) + # The schema should be converted to string by convert_json_schema_to_str + self.assertIn("json_schema", sampling_params) + self.assertIsInstance(sampling_params["json_schema"], str) + + def test_response_format_structural_tag(self): + """Test that response_format structural_tag is correctly processed in sampling params.""" + req = CompletionRequest( + model="x", + prompt="Generate structured output:", + max_tokens=100, + response_format={ + "type": "structural_tag", + "structures": [{"begin": "", "end": ""}], + "triggers": [""], + }, + ) + sampling_params = self.sc._build_sampling_params(req) + # The structural_tag should be processed + self.assertIn("structural_tag", sampling_params) + self.assertIsInstance(sampling_params["structural_tag"], str) + + def test_response_format_none(self): + """Test that no response_format doesn't add extra constraints.""" + req = CompletionRequest(model="x", prompt="Generate text:", max_tokens=100) + sampling_params = self.sc._build_sampling_params(req) + # Should not have json_schema or structural_tag from response_format + # (but might have json_schema from the legacy json_schema field) + self.assertIsNone(sampling_params.get("structural_tag")) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_serving_embedding.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_serving_embedding.py new file mode 100644 index 00000000..2621eb52 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/basic/test_serving_embedding.py @@ -0,0 +1,144 @@ +""" +Unit tests for the OpenAIServingEmbedding class from serving_embedding.py. +""" + +import unittest +import uuid +from unittest.mock import Mock + +from fastapi import Request + +from sglang.srt.entrypoints.openai.protocol import ( + EmbeddingRequest, + MultimodalEmbeddingInput, +) +from sglang.srt.entrypoints.openai.serving_embedding import OpenAIServingEmbedding +from sglang.srt.managers.io_struct import EmbeddingReqInput + + +# Mock TokenizerManager for embedding tests +class _MockTokenizerManager: + def __init__(self): + self.model_config = Mock() + self.model_config.is_multimodal = False + self.server_args = Mock() + self.server_args.enable_cache_report = False + self.model_path = "test-model" + + # Mock tokenizer + self.tokenizer = Mock() + self.tokenizer.encode = Mock(return_value=[1, 2, 3, 4, 5]) + self.tokenizer.decode = Mock(return_value="Test embedding input") + self.tokenizer.chat_template = None + self.tokenizer.bos_token_id = 1 + + # Mock generate_request method for embeddings + async def mock_generate_embedding(): + yield { + "embedding": [0.1, 0.2, 0.3, 0.4, 0.5] * 20, # 100-dim embedding + "meta_info": { + "id": f"embd-{uuid.uuid4()}", + "prompt_tokens": 5, + }, + } + + self.generate_request = Mock(return_value=mock_generate_embedding()) + + +# Mock TemplateManager for embedding tests +class _MockTemplateManager: + def __init__(self): + self.chat_template_name = None # None for embeddings usually + self.jinja_template_content_format = None + self.completion_template_name = None + + +class ServingEmbeddingTestCase(unittest.TestCase): + def setUp(self): + """Set up test fixtures.""" + self.tokenizer_manager = _MockTokenizerManager() + self.template_manager = _MockTemplateManager() + self.serving_embedding = OpenAIServingEmbedding( + self.tokenizer_manager, self.template_manager + ) + + self.request = Mock(spec=Request) + self.request.headers = {} + + self.basic_req = EmbeddingRequest( + model="test-model", + input="Hello, how are you?", + encoding_format="float", + ) + self.list_req = EmbeddingRequest( + model="test-model", + input=["Hello, how are you?", "I am fine, thank you!"], + encoding_format="float", + ) + self.multimodal_req = EmbeddingRequest( + model="test-model", + input=[ + MultimodalEmbeddingInput(text="Hello", image="base64_image_data"), + MultimodalEmbeddingInput(text="World", image=None), + ], + encoding_format="float", + ) + self.token_ids_req = EmbeddingRequest( + model="test-model", + input=[1, 2, 3, 4, 5], + encoding_format="float", + ) + + def test_convert_single_string_request(self): + """Test converting single string request to internal format.""" + adapted_request, processed_request = ( + self.serving_embedding._convert_to_internal_request(self.basic_req) + ) + + self.assertIsInstance(adapted_request, EmbeddingReqInput) + self.assertEqual(adapted_request.text, "Hello, how are you?") + # self.assertEqual(adapted_request.rid, "test-id") + self.assertEqual(processed_request, self.basic_req) + + def test_convert_list_string_request(self): + """Test converting list of strings request to internal format.""" + adapted_request, processed_request = ( + self.serving_embedding._convert_to_internal_request(self.list_req) + ) + + self.assertIsInstance(adapted_request, EmbeddingReqInput) + self.assertEqual( + adapted_request.text, ["Hello, how are you?", "I am fine, thank you!"] + ) + # self.assertEqual(adapted_request.rid, "test-id") + self.assertEqual(processed_request, self.list_req) + + def test_convert_token_ids_request(self): + """Test converting token IDs request to internal format.""" + adapted_request, processed_request = ( + self.serving_embedding._convert_to_internal_request(self.token_ids_req) + ) + + self.assertIsInstance(adapted_request, EmbeddingReqInput) + self.assertEqual(adapted_request.input_ids, [1, 2, 3, 4, 5]) + # self.assertEqual(adapted_request.rid, "test-id") + self.assertEqual(processed_request, self.token_ids_req) + + def test_convert_multimodal_request(self): + """Test converting multimodal request to internal format.""" + adapted_request, processed_request = ( + self.serving_embedding._convert_to_internal_request(self.multimodal_req) + ) + + self.assertIsInstance(adapted_request, EmbeddingReqInput) + # Should extract text and images separately + self.assertEqual(len(adapted_request.text), 2) + self.assertIn("Hello", adapted_request.text) + self.assertIn("World", adapted_request.text) + self.assertEqual(adapted_request.image_data[0], "base64_image_data") + self.assertIsNone(adapted_request.image_data[1]) + # self.assertEqual(adapted_request.rid, "test-id") + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/features/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/features/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/features/test_enable_thinking.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/features/test_enable_thinking.py new file mode 100644 index 00000000..5687c68e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/features/test_enable_thinking.py @@ -0,0 +1,237 @@ +""" +Usage: +python3 -m unittest openai_server.features.test_enable_thinking.TestEnableThinking.test_chat_completion_with_reasoning +python3 -m unittest openai_server.features.test_enable_thinking.TestEnableThinking.test_chat_completion_without_reasoning +python3 -m unittest openai_server.features.test_enable_thinking.TestEnableThinking.test_stream_chat_completion_with_reasoning +python3 -m unittest openai_server.features.test_enable_thinking.TestEnableThinking.test_stream_chat_completion_without_reasoning +""" + +import json +import unittest + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_ENABLE_THINKING_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestEnableThinking(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_ENABLE_THINKING_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-1234" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--reasoning-parser", + "qwen3", + ], + ) + cls.additional_chat_kwargs = {} + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_chat_completion_with_reasoning(self): + # Test non-streaming with "enable_thinking": True, reasoning_content should not be empty + client = requests.post( + f"{self.base_url}/v1/chat/completions", + headers={"Authorization": f"Bearer {self.api_key}"}, + json={ + "model": self.model, + "messages": [{"role": "user", "content": "Hello"}], + "temperature": 0, + "separate_reasoning": True, + "chat_template_kwargs": {"enable_thinking": True}, + **self.additional_chat_kwargs, + }, + ) + + self.assertEqual(client.status_code, 200, f"Failed with: {client.text}") + data = client.json() + + self.assertIn("choices", data) + self.assertTrue(len(data["choices"]) > 0) + self.assertIn("message", data["choices"][0]) + self.assertIn("reasoning_content", data["choices"][0]["message"]) + self.assertIsNotNone(data["choices"][0]["message"]["reasoning_content"]) + + def test_chat_completion_without_reasoning(self): + # Test non-streaming with "enable_thinking": False, reasoning_content should be empty + client = requests.post( + f"{self.base_url}/v1/chat/completions", + headers={"Authorization": f"Bearer {self.api_key}"}, + json={ + "model": self.model, + "messages": [{"role": "user", "content": "Hello"}], + "temperature": 0, + "separate_reasoning": True, + "chat_template_kwargs": {"enable_thinking": False}, + **self.additional_chat_kwargs, + }, + ) + + self.assertEqual(client.status_code, 200, f"Failed with: {client.text}") + data = client.json() + + self.assertIn("choices", data) + self.assertTrue(len(data["choices"]) > 0) + self.assertIn("message", data["choices"][0]) + + if "reasoning_content" in data["choices"][0]["message"]: + self.assertIsNone(data["choices"][0]["message"]["reasoning_content"]) + + def test_stream_chat_completion_with_reasoning(self): + # Test streaming with "enable_thinking": True, reasoning_content should not be empty + response = requests.post( + f"{self.base_url}/v1/chat/completions", + headers={"Authorization": f"Bearer {self.api_key}"}, + json={ + "model": self.model, + "messages": [{"role": "user", "content": "Hello"}], + "temperature": 0, + "separate_reasoning": True, + "stream": True, + "chat_template_kwargs": {"enable_thinking": True}, + **self.additional_chat_kwargs, + }, + stream=True, + ) + + self.assertEqual(response.status_code, 200, f"Failed with: {response.text}") + + has_reasoning = False + has_content = False + + print("\n=== Stream With Reasoning ===") + for line in response.iter_lines(): + if line: + line = line.decode("utf-8") + if line.startswith("data:") and not line.startswith("data: [DONE]"): + data = json.loads(line[6:]) + if "choices" in data and len(data["choices"]) > 0: + delta = data["choices"][0].get("delta", {}) + + if "reasoning_content" in delta and delta["reasoning_content"]: + has_reasoning = True + + if "content" in delta and delta["content"]: + has_content = True + + self.assertTrue( + has_reasoning, + "The reasoning content is not included in the stream response", + ) + self.assertTrue( + has_content, "The stream response does not contain normal content" + ) + + def test_stream_chat_completion_without_reasoning(self): + # Test streaming with "enable_thinking": False, reasoning_content should be empty + response = requests.post( + f"{self.base_url}/v1/chat/completions", + headers={"Authorization": f"Bearer {self.api_key}"}, + json={ + "model": self.model, + "messages": [{"role": "user", "content": "Hello"}], + "temperature": 0, + "separate_reasoning": True, + "stream": True, + "chat_template_kwargs": {"enable_thinking": False}, + **self.additional_chat_kwargs, + }, + stream=True, + ) + + self.assertEqual(response.status_code, 200, f"Failed with: {response.text}") + + has_reasoning = False + has_content = False + + print("\n=== Stream Without Reasoning ===") + for line in response.iter_lines(): + if line: + line = line.decode("utf-8") + if line.startswith("data:") and not line.startswith("data: [DONE]"): + data = json.loads(line[6:]) + if "choices" in data and len(data["choices"]) > 0: + delta = data["choices"][0].get("delta", {}) + + if "reasoning_content" in delta and delta["reasoning_content"]: + has_reasoning = True + + if "content" in delta and delta["content"]: + has_content = True + + self.assertFalse( + has_reasoning, + "The reasoning content should not be included in the stream response", + ) + self.assertTrue( + has_content, "The stream response does not contain normal content" + ) + + +# Skip for ci test +# class TestGLM45EnableThinking(TestEnableThinking): +# @classmethod +# def setUpClass(cls): +# # Replace with the model name needed for testing; if not required, reuse DEFAULT_SMALL_MODEL_NAME_FOR_TEST +# cls.model = "THUDM/GLM-4.5" +# cls.base_url = DEFAULT_URL_FOR_TEST +# cls.api_key = "sk-1234" +# cls.process = popen_launch_server( +# cls.model, +# cls.base_url, +# timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, +# api_key=cls.api_key, +# other_args=[ +# "--tool-call-parser", +# "glm45", +# "--reasoning-parser", +# "glm45", +# "--tp-size", +# "8" +# ], +# ) + +# # Validate whether enable-thinking conflict with tool_calls +# cls.additional_chat_kwargs = { +# "tools": [ +# { +# "type": "function", +# "function": { +# "name": "add", +# "description": "Compute the sum of two numbers", +# "parameters": { +# "type": "object", +# "properties": { +# "a": { +# "type": "int", +# "description": "A number", +# }, +# "b": { +# "type": "int", +# "description": "A number", +# }, +# }, +# "required": ["a", "b"], +# }, +# }, +# } +# ] +# } + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/features/test_json_mode.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/features/test_json_mode.py new file mode 100644 index 00000000..dd5062d4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/features/test_json_mode.py @@ -0,0 +1,131 @@ +import json +import unittest + +import openai + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestJSONModeMixin: + """Mixin class containing JSON mode test methods""" + + def test_json_mode_response(self): + """Test that response_format json_object (also known as "json mode") produces valid JSON, even without a system prompt that mentions JSON.""" + response = self.client.chat.completions.create( + model=self.model, + messages=[ + # We are deliberately omitting "That produces JSON" or similar phrases from the assistant prompt so that we don't have misleading test results + { + "role": "system", + "content": "You are a helpful AI assistant that gives a short answer.", + }, + {"role": "user", "content": "What is the capital of Bulgaria?"}, + ], + temperature=0, + max_tokens=128, + response_format={"type": "json_object"}, + ) + text = response.choices[0].message.content + + print(f"Response ({len(text)} characters): {text}") + + # Verify the response is valid JSON + try: + js_obj = json.loads(text) + except json.JSONDecodeError as e: + self.fail(f"Response is not valid JSON. Error: {e}. Response: {text}") + + # Verify it's actually an object (dict) + self.assertIsInstance(js_obj, dict, f"Response is not a JSON object: {text}") + + def test_json_mode_with_streaming(self): + """Test that streaming with json_object response (also known as "json mode") format works correctly, even without a system prompt that mentions JSON.""" + stream = self.client.chat.completions.create( + model=self.model, + messages=[ + # We are deliberately omitting "That produces JSON" or similar phrases from the assistant prompt so that we don't have misleading test results + { + "role": "system", + "content": "You are a helpful AI assistant that gives a short answer.", + }, + {"role": "user", "content": "What is the capital of Bulgaria?"}, + ], + temperature=0, + max_tokens=128, + response_format={"type": "json_object"}, + stream=True, + ) + + # Collect all chunks + chunks = [] + for chunk in stream: + if chunk.choices[0].delta.content is not None: + chunks.append(chunk.choices[0].delta.content) + full_response = "".join(chunks) + + print( + f"Concatenated Response ({len(full_response)} characters): {full_response}" + ) + + # Verify the combined response is valid JSON + try: + js_obj = json.loads(full_response) + except json.JSONDecodeError as e: + self.fail( + f"Streamed response is not valid JSON. Error: {e}. Response: {full_response}" + ) + + self.assertIsInstance(js_obj, dict) + + +class ServerWithGrammarBackend(CustomTestCase): + """Base class for tests requiring a grammar backend server""" + + backend = "xgrammar" + + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + + other_args = [ + "--max-running-requests", + "10", + "--grammar-backend", + cls.backend, + ] + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + cls.client = openai.Client(api_key="EMPTY", base_url=f"{cls.base_url}/v1") + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +class TestJSONModeXGrammar(ServerWithGrammarBackend, TestJSONModeMixin): + backend = "xgrammar" + + +class TestJSONModeOutlines(ServerWithGrammarBackend, TestJSONModeMixin): + backend = "outlines" + + +class TestJSONModeLLGuidance(ServerWithGrammarBackend, TestJSONModeMixin): + backend = "llguidance" + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/features/test_openai_server_ebnf.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/features/test_openai_server_ebnf.py new file mode 100644 index 00000000..0104d398 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/features/test_openai_server_ebnf.py @@ -0,0 +1,98 @@ +import re + +import openai + +from sglang.srt.utils import kill_process_tree +from sglang.srt.utils.hf_transformers_utils import get_tokenizer +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +# ------------------------------------------------------------------------- +# EBNF Test Class: TestOpenAIServerEBNF +# Launches the server with xgrammar, has only EBNF tests +# ------------------------------------------------------------------------- +class TestOpenAIServerEBNF(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + # passing xgrammar specifically + other_args = ["--grammar-backend", "xgrammar"] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=other_args, + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_ebnf(self): + """ + Ensure we can pass `ebnf` to the local openai server + and that it enforces the grammar. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + ebnf_grammar = r""" + root ::= "Hello" | "Hi" | "Hey" + """ + pattern = re.compile(r"^(Hello|Hi|Hey)[.!?]*\s*$") + + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful EBNF test bot."}, + {"role": "user", "content": "Say a greeting (Hello, Hi, or Hey)."}, + ], + temperature=0, + max_tokens=32, + extra_body={"ebnf": ebnf_grammar}, + ) + text = response.choices[0].message.content.strip() + self.assertTrue(len(text) > 0, "Got empty text from EBNF generation") + self.assertRegex(text, pattern, f"Text '{text}' doesn't match EBNF choices") + + def test_ebnf_strict_json(self): + """ + A stricter EBNF that produces exactly {"name":"Alice"} format + with no trailing punctuation or extra fields. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + ebnf_grammar = r""" + root ::= "{" pair "}" + pair ::= "\"name\"" ":" string + string ::= "\"" [A-Za-z]+ "\"" + """ + pattern = re.compile(r'^\{"name":"[A-Za-z]+"\}$') + + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "EBNF mini-JSON generator."}, + { + "role": "user", + "content": "Generate single key JSON with only letters.", + }, + ], + temperature=0, + max_tokens=64, + extra_body={"ebnf": ebnf_grammar}, + ) + text = response.choices[0].message.content.strip() + self.assertTrue(len(text) > 0, "Got empty text from EBNF strict JSON test") + self.assertRegex( + text, pattern, f"Text '{text}' not matching the EBNF strict JSON shape" + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/features/test_openai_server_hidden_states.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/features/test_openai_server_hidden_states.py new file mode 100644 index 00000000..6544a9a8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/features/test_openai_server_hidden_states.py @@ -0,0 +1,350 @@ +import unittest +from abc import ABC + +import openai + +from sglang.srt.utils import kill_process_tree +from sglang.srt.utils.hf_transformers_utils import get_tokenizer +from sglang.test.test_utils import ( + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class BaseTestOpenAIServerWithHiddenStates(ABC): + + @classmethod + def setUpClass(cls): + cls.return_hidden_states = [False, True] + cls.use_list_input = [True, False] + cls.parallel_sample_nums = [1, 2] + + def test_completion(self): + for return_hidden_states in self.return_hidden_states: + for use_list_input in self.use_list_input: + for parallel_sample_num in self.parallel_sample_nums: + self.run_completion( + use_list_input, + parallel_sample_num, + return_hidden_states, + ) + + def test_completion_stream(self): + # parallel sampling and list input are not supported in streaming mode + for return_hidden_states in self.return_hidden_states: + for use_list_input in self.use_list_input: + for parallel_sample_num in self.parallel_sample_nums: + self.run_completion_stream( + use_list_input, + parallel_sample_num, + return_hidden_states, + ) + + def test_chat_completion(self): + for return_hidden_states in self.return_hidden_states: + for ( + parallel_sample_num + ) in ( + self.parallel_sample_nums + ): # parallel sample num 2 breaks in the adapter with a 400 for EAGLE + self.run_chat_completion(parallel_sample_num, return_hidden_states) + + def test_chat_completion_stream(self): + for return_hidden_states in self.return_hidden_states: + for ( + parallel_sample_num + ) in ( + self.parallel_sample_nums + ): # parallel sample num > 1 breaks in the adapter with a 400 for EAGLE + self.run_chat_completion_stream( + parallel_sample_num, return_hidden_states + ) + + def run_completion( + self, + use_list_input, + parallel_sample_num, + return_hidden_states, + ): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + prompt = "The capital of France is" + prompt_input = prompt + + if use_list_input: + prompt_arg = [prompt_input, prompt_input] + num_choices = len(prompt_arg) + else: + prompt_arg = prompt_input + num_choices = 1 + + response = client.completions.create( + model=self.model, + prompt=prompt_arg, + temperature=0, + max_tokens=32, + n=parallel_sample_num, + extra_body=dict(return_hidden_states=return_hidden_states), + ) + + for choice in response.choices: + assert hasattr(choice, "hidden_states") == return_hidden_states + if return_hidden_states: + assert choice.hidden_states is not None, "hidden_states was None" + + def run_completion_stream( + self, + use_list_input, + parallel_sample_num, + return_hidden_states, + ): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + prompt = "The capital of France is" + prompt_input = prompt + num_prompt_tokens = len(self.tokenizer.encode(prompt)) + + if use_list_input: + prompt_arg = [prompt_input, prompt_input] + num_choices = len(prompt_arg) + num_prompt_tokens *= 2 + else: + prompt_arg = prompt_input + num_choices = 1 + + generator = client.completions.create( + model=self.model, + prompt=prompt_arg, + temperature=0, + max_tokens=32, + stream=True, + stream_options={"include_usage": True}, + n=parallel_sample_num, + extra_body=dict(return_hidden_states=return_hidden_states), + ) + + hidden_states_list = [] + for response in generator: + usage = response.usage + for choice in response.choices: + if hasattr(choice, "hidden_states"): + assert return_hidden_states + assert choice.hidden_states is not None + hidden_states_list.append(choice.hidden_states) + + if return_hidden_states: + assert ( + len(hidden_states_list) == parallel_sample_num * num_choices + ), f"Expected {parallel_sample_num * num_choices} hidden states, got {len(hidden_states_list)}" + else: + assert ( + hidden_states_list == [] + ), "hidden_states were returned and should not have been" + + def run_chat_completion(self, parallel_sample_num, return_hidden_states): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + { + "role": "user", + "content": "What is the capital of France? Answer in a few words.", + }, + ], + temperature=0, + n=parallel_sample_num, + extra_body=dict(return_hidden_states=return_hidden_states), + ) + + for choice in response.choices: + assert hasattr(choice, "hidden_states") == return_hidden_states + if return_hidden_states: + assert choice.hidden_states is not None, "hidden_states was None" + + def run_chat_completion_stream( + self, parallel_sample_num=1, return_hidden_states=False + ): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + generator = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "What is the capital of France?"}, + ], + temperature=0, + stream=True, + stream_options={"include_usage": True}, + n=parallel_sample_num, + extra_body=dict(return_hidden_states=return_hidden_states), + ) + + is_firsts = {} + hidden_states_list = [] + + for response in generator: + for choice in response.choices: + if hasattr(choice.delta, "hidden_states"): + assert return_hidden_states + assert choice.delta.hidden_states is not None + hidden_states_list.append(choice.delta.hidden_states) + + if return_hidden_states: + assert ( + len(hidden_states_list) == parallel_sample_num + ), f"Expected {parallel_sample_num} hidden states, got {len(hidden_states_list)}" + else: + assert ( + hidden_states_list == [] + ), "hidden_states were returned and should not have been" + + +class TestOpenAIServerWithHiddenStatesEnabled( + CustomTestCase, BaseTestOpenAIServerWithHiddenStates +): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=["--enable-return-hidden-states"], + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + cls.return_hidden_states = [False, True] + cls.use_list_input = [True, False] + cls.parallel_sample_nums = [1, 2] + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +class TestOpenAIServerWithHiddenStatesEnabledAndCUDAGraphDisabled( + CustomTestCase, BaseTestOpenAIServerWithHiddenStates +): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=["--enable-return-hidden-states", "--disable-cuda-graph"], + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + cls.return_hidden_states = [False, True] + cls.use_list_input = [True, False] + cls.parallel_sample_nums = [1] + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +class TestOpenAIServerWithEAGLEAndHiddenStatesEnabled( + CustomTestCase, BaseTestOpenAIServerWithHiddenStates +): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.speculative_draft_model = DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST + cls.speculative_algorithm = "EAGLE" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + 5, + "--speculative-eagle-topk", + 8, + "--speculative-num-draft-tokens", + 64, + "--mem-fraction-static", + 0.7, + "--chunked-prefill-size", + 128, + "--max-running-requests", + 8, + "--enable-return-hidden-states", + ], + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST) + cls.return_hidden_states = [False, True] + cls.use_list_input = [True, False] + cls.parallel_sample_nums = [1] + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +class TestOpenAIServerWithEAGLE3AndHiddenStatesEnabled( + CustomTestCase, BaseTestOpenAIServerWithHiddenStates +): + @classmethod + def setUpClass(cls): + cls.model = "meta-llama/Llama-3.1-8B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.speculative_algorithm = "EAGLE3" + cls.speculative_draft_model = "jamesliu1/sglang-EAGLE3-Llama-3.1-Instruct-8B" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + cls.speculative_algorithm, + "--speculative-draft-model-path", + cls.speculative_draft_model, + "--speculative-num-steps", + 5, + "--speculative-eagle-topk", + 16, + "--speculative-num-draft-tokens", + 64, + "--mem-fraction-static", + 0.7, + "--chunked-prefill-size", + 128, + "--max-running-requests", + 8, + "--dtype", + "float16", + "--enable-return-hidden-states", + ], + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + cls.return_hidden_states = [False, True] + cls.use_list_input = [True, False] + cls.parallel_sample_nums = [1] + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/features/test_reasoning_content.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/features/test_reasoning_content.py new file mode 100644 index 00000000..d605a24f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/features/test_reasoning_content.py @@ -0,0 +1,341 @@ +""" +Usage: +python3 -m unittest openai_server.features.test_reasoning_content.TestReasoningContentAPI.test_streaming_separate_reasoning_false +python3 -m unittest openai_server.features.test_reasoning_content.TestReasoningContentAPI.test_streaming_separate_reasoning_true +python3 -m unittest openai_server.features.test_reasoning_content.TestReasoningContentAPI.test_streaming_separate_reasoning_true_stream_reasoning_false +python3 -m unittest openai_server.features.test_reasoning_content.TestReasoningContentAPI.test_nonstreaming_separate_reasoning_false +python3 -m unittest openai_server.features.test_reasoning_content.TestReasoningContentAPI.test_nonstreaming_separate_reasoning_true +python3 -m unittest openai_server.features.test_reasoning_content.TestReasoningContentStartup.test_nonstreaming +python3 -m unittest openai_server.features.test_reasoning_content.TestReasoningContentStartup.test_streaming +""" + +import unittest + +import openai + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_REASONING_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestReasoningContentAPI(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_REASONING_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-1234" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--reasoning-parser", + "deepseek-r1", + ], + ) + cls.base_url += "/v1" + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_streaming_separate_reasoning_false(self): + # Test streaming with separate_reasoning=False, reasoning_content should be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "stream": True, + "extra_body": {"separate_reasoning": False}, + } + response = client.chat.completions.create(**payload) + + reasoning_content = "" + content = "" + for chunk in response: + if chunk.choices[0].delta.content: + content += chunk.choices[0].delta.content + elif chunk.choices[0].delta.reasoning_content: + reasoning_content += chunk.choices[0].delta.reasoning_content + + assert len(reasoning_content) == 0 + assert len(content) > 0 + + def test_streaming_separate_reasoning_true(self): + # Test streaming with separate_reasoning=True, reasoning_content should not be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "stream": True, + "extra_body": {"separate_reasoning": True}, + } + response = client.chat.completions.create(**payload) + + reasoning_content = "" + content = "" + for chunk in response: + if chunk.choices[0].delta.content: + content += chunk.choices[0].delta.content + elif chunk.choices[0].delta.reasoning_content: + reasoning_content += chunk.choices[0].delta.reasoning_content + + assert len(reasoning_content) > 0 + assert len(content) > 0 + + def test_streaming_separate_reasoning_true_stream_reasoning_false(self): + # Test streaming with separate_reasoning=True, reasoning_content should not be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "stream": True, + "extra_body": {"separate_reasoning": True, "stream_reasoning": False}, + } + response = client.chat.completions.create(**payload) + + reasoning_content = "" + content = "" + first_chunk = False + for chunk in response: + if chunk.choices[0].delta.reasoning_content: + reasoning_content = chunk.choices[0].delta.reasoning_content + first_chunk = True + if chunk.choices[0].delta.content: + content += chunk.choices[0].delta.content + if not first_chunk: + reasoning_content = chunk.choices[0].delta.reasoning_content + first_chunk = True + if not first_chunk: + assert ( + not chunk.choices[0].delta.reasoning_content + or len(chunk.choices[0].delta.reasoning_content) == 0 + ) + assert len(reasoning_content) > 0 + assert len(content) > 0 + + def test_nonstreaming_separate_reasoning_false(self): + # Test non-streaming with separate_reasoning=False, reasoning_content should be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "extra_body": {"separate_reasoning": False}, + } + response = client.chat.completions.create(**payload) + + assert ( + not response.choices[0].message.reasoning_content + or len(response.choices[0].message.reasoning_content) == 0 + ) + assert len(response.choices[0].message.content) > 0 + + def test_nonstreaming_separate_reasoning_true(self): + # Test non-streaming with separate_reasoning=True, reasoning_content should not be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "extra_body": {"separate_reasoning": True}, + } + response = client.chat.completions.create(**payload) + + assert len(response.choices[0].message.reasoning_content) > 0 + assert len(response.choices[0].message.content) > 0 + + +class TestReasoningContentWithoutParser(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_REASONING_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-1234" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[], # No reasoning parser + ) + cls.base_url += "/v1" + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_streaming_separate_reasoning_false(self): + # Test streaming with separate_reasoning=False, reasoning_content should be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "stream": True, + "extra_body": {"separate_reasoning": False}, + } + response = client.chat.completions.create(**payload) + + reasoning_content = "" + content = "" + for chunk in response: + if chunk.choices[0].delta.content: + content += chunk.choices[0].delta.content + elif chunk.choices[0].delta.reasoning_content: + reasoning_content += chunk.choices[0].delta.reasoning_content + + assert len(reasoning_content) == 0 + assert len(content) > 0 + + def test_streaming_separate_reasoning_true(self): + # Test streaming with separate_reasoning=True, reasoning_content should not be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "stream": True, + "extra_body": {"separate_reasoning": True}, + } + response = client.chat.completions.create(**payload) + + reasoning_content = "" + content = "" + for chunk in response: + if chunk.choices[0].delta.content: + content += chunk.choices[0].delta.content + elif chunk.choices[0].delta.reasoning_content: + reasoning_content += chunk.choices[0].delta.reasoning_content + + assert len(reasoning_content) == 0 + assert len(content) > 0 + + def test_streaming_separate_reasoning_true_stream_reasoning_false(self): + # Test streaming with separate_reasoning=True, reasoning_content should not be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "stream": True, + "extra_body": {"separate_reasoning": True, "stream_reasoning": False}, + } + response = client.chat.completions.create(**payload) + + reasoning_content = "" + content = "" + first_chunk = False + for chunk in response: + if chunk.choices[0].delta.reasoning_content: + reasoning_content = chunk.choices[0].delta.reasoning_content + first_chunk = True + if chunk.choices[0].delta.content: + content += chunk.choices[0].delta.content + if not first_chunk: + reasoning_content = chunk.choices[0].delta.reasoning_content + first_chunk = True + if not first_chunk: + assert ( + not chunk.choices[0].delta.reasoning_content + or len(chunk.choices[0].delta.reasoning_content) == 0 + ) + assert not reasoning_content or len(reasoning_content) == 0 + assert len(content) > 0 + + def test_nonstreaming_separate_reasoning_false(self): + # Test non-streaming with separate_reasoning=False, reasoning_content should be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "extra_body": {"separate_reasoning": False}, + } + response = client.chat.completions.create(**payload) + + assert ( + not response.choices[0].message.reasoning_content + or len(response.choices[0].message.reasoning_content) == 0 + ) + assert len(response.choices[0].message.content) > 0 + + def test_nonstreaming_separate_reasoning_true(self): + # Test non-streaming with separate_reasoning=True, reasoning_content should not be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "extra_body": {"separate_reasoning": True}, + } + response = client.chat.completions.create(**payload) + + assert ( + not response.choices[0].message.reasoning_content + or len(response.choices[0].message.reasoning_content) == 0 + ) + assert len(response.choices[0].message.content) > 0 + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/function_call/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/function_call/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/function_call/test_openai_function_calling.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/function_call/test_openai_function_calling.py new file mode 100644 index 00000000..b54f90fc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/function_call/test_openai_function_calling.py @@ -0,0 +1,952 @@ +import json +import unittest + +import openai + +from sglang.srt.utils import kill_process_tree +from sglang.srt.utils.hf_transformers_utils import get_tokenizer +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestOpenAIServerFunctionCalling(CustomTestCase): + # NOTE: this system_message is for Llama3.2 system prompt. Without this, + # sometimes Llama3.2 gives a different tool call format such as: + # '<|python_tag|>{"type": "function", "function": "add", "parameters": {"a": "3", "b": "5"}}' + SYSTEM_MESSAGE = ( + "You are a helpful assistant with tool calling capabilities. " + "Only reply with a tool call if the function exists in the library provided by the user. " + "If it doesn't exist, just reply directly in natural language. " + "When you receive a tool call response, use the output to format an answer to the original user question. " + "You have access to the following functions. " + "To call a function, please respond with JSON for a function call. " + 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. ' + "Do not use variables.\n\n" + ) + + @classmethod + def setUpClass(cls): + # Replace with the model name needed for testing; if not required, reuse DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + # Start the local OpenAI Server. If necessary, you can add other parameters such as --enable-tools. + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + # If your server needs extra parameters to test function calling, please add them here. + "--tool-call-parser", + "llama3", + ], + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_function_calling_format(self): + """ + Test: Whether the function call format returned by the AI is correct. + When returning a tool call, message.content should be None, and tool_calls should be a list. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "add", + "description": "Compute the sum of two numbers", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "integer", + "description": "A number", + }, + "b": { + "type": "integer", + "description": "A number", + }, + }, + "required": ["a", "b"], + }, + }, + } + ] + + messages = [ + {"role": "system", "content": self.SYSTEM_MESSAGE}, + {"role": "user", "content": "Compute (3+5)"}, + ] + response = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=False, + tools=tools, + ) + + tool_calls = response.choices[0].message.tool_calls + + assert ( + isinstance(tool_calls, list) and len(tool_calls) > 0 + ), "tool_calls should be a non-empty list" + + function_name = tool_calls[0].function.name + assert function_name == "add", "Function name should be 'add'" + + # This unit test is too difficult for default model. Mark it as optional unit tests so it won't trigger unless specified. + def _test_function_calling_multiturn(self): + """ + Test: Whether the function call format returned by the AI is correct. + When returning a tool call, message.content should be None, and tool_calls should be a list. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "add", + "description": "Compute the sum of two numbers", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "integer", + "description": "A number", + }, + "b": { + "type": "integer", + "description": "A number", + }, + }, + "required": ["a", "b"], + }, + }, + } + ] + + messages = [{"role": "user", "content": "Compute (3+5)"}] + + response = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=False, + tools=tools, + ) + + tool_call = response.choices[0].message.tool_calls[0] + function_name = tool_call.function.name + assert function_name == "add", "Function name should be 'add'" + function_arguments = tool_call.function.arguments + function_arguments = json.loads(tool_call.function.arguments) + assert function_arguments in [ + {"a": 3, "b": 5}, + {"a": "3", "b": "5"}, + ], f"Unexpected function arguments: {function_arguments}" + + messages.append(response.choices[0].message) + messages.append( + { + "role": "tool", + "tool_call_id": tool_call.id, + "content": "8", + "name": function_name, + } + ) + + final_response = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=False, + tools=tools, + ) + + assert ( + "8" in final_response.choices[0].message.content + ), "tool_call response should have the sum 8 in the content" + + def test_function_calling_streaming_simple(self): + """ + Test: Whether the function name can be correctly recognized in streaming mode. + - Expect a function call to be found, and the function name to be correct. + - Verify that streaming mode returns at least multiple chunks. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city to find the weather for", + }, + "unit": { + "type": "string", + "description": "Weather unit (celsius or fahrenheit)", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["city", "unit"], + }, + }, + } + ] + + messages = [ + {"role": "system", "content": self.SYSTEM_MESSAGE}, + { + "role": "user", + "content": "What is the temperature in Paris in celsius??", + }, + ] + + response_stream = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=True, + tools=tools, + ) + + chunks = list(response_stream) + self.assertTrue(len(chunks) > 0, "Streaming should return at least one chunk") + + found_function_name = False + for chunk in chunks: + choice = chunk.choices[0] + # Check whether the current chunk contains tool_calls + if choice.delta.tool_calls: + tool_call = choice.delta.tool_calls[0] + if tool_call.function.name: + self.assertEqual( + tool_call.function.name, + "get_current_weather", + "Function name should be 'get_current_weather'", + ) + found_function_name = True + break + + self.assertTrue( + found_function_name, + "Target function name 'get_current_weather' was not found in the streaming chunks", + ) + + finish_reason = chunks[-1].choices[0].finish_reason + self.assertEqual( + finish_reason, + "tool_calls", + "Final response of function calling should have finish_reason 'tool_calls'", + ) + + def test_function_calling_streaming_args_parsing(self): + """ + Test: Whether the function call arguments returned in streaming mode can be correctly concatenated into valid JSON. + - The user request requires multiple parameters. + - AI may return the arguments in chunks that need to be concatenated. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "add", + "description": "Compute the sum of two integers", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "integer", + "description": "First integer", + }, + "b": { + "type": "integer", + "description": "Second integer", + }, + }, + "required": ["a", "b"], + }, + "strict": True, # Llama-3.2-1B is flaky in tool call. It won't always respond with parameters unless we set strict. + }, + } + ] + + messages = [ + {"role": "system", "content": self.SYSTEM_MESSAGE}, + {"role": "user", "content": "Please sum 5 and 7, just call the function."}, + ] + + response_stream = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.9, + top_p=0.9, + stream=True, + tools=tools, + ) + + argument_fragments = [] + chunks = list(response_stream) + function_name = None + for chunk in chunks: + choice = chunk.choices[0] + if choice.delta.tool_calls: + tool_call = choice.delta.tool_calls[0] + # Record the function name on first occurrence + function_name = tool_call.function.name or function_name + # In case of multiple chunks, JSON fragments may need to be concatenated + if tool_call.function.arguments is not None: + argument_fragments.append(tool_call.function.arguments) + + self.assertEqual(function_name, "add", "Function name should be 'add'") + joined_args = "".join(argument_fragments) + self.assertTrue( + len(joined_args) > 0, + "No parameter fragments were returned in the function call", + ) + + finish_reason = chunks[-1].choices[0].finish_reason + self.assertEqual( + finish_reason, + "tool_calls", + "Final response of function calling should have finish_reason 'tool_calls'", + ) + + # Check whether the concatenated JSON is valid + try: + args_obj = json.loads(joined_args) + except json.JSONDecodeError: + self.fail( + "The concatenated tool call arguments are not valid JSON, parsing failed" + ) + + self.assertIn("a", args_obj, "Missing parameter 'a'") + self.assertIn("b", args_obj, "Missing parameter 'b'") + self.assertEqual(str(args_obj["a"]), "5", "Parameter a should be 5") + self.assertEqual(str(args_obj["b"]), "7", "Parameter b should be 7") + + def test_function_call_strict(self): + """ + Test: Whether the strict mode of function calling works as expected. + - When strict mode is enabled, the AI should not return a function call if the function name is not recognized. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "sub", + "description": "Compute the difference of two integers", + "parameters": { + "type": "object", + "properties": { + "int_a": { + "type": "integer", + "description": "First integer", + }, + "int_b": { + "type": "integer", + "description": "Second integer", + }, + }, + "required": ["int_a", "int_b"], + }, + "strict": True, + }, + } + ] + + messages = [ + {"role": "user", "content": "Please compute 5 - 7, using your tool."} + ] + response = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=False, + tools=tools, + ) + + tool_calls = response.choices[0].message.tool_calls + function_name = tool_calls[0].function.name + arguments = tool_calls[0].function.arguments + args_obj = json.loads(arguments) + + self.assertEqual(function_name, "sub", "Function name should be 'sub'") + self.assertEqual(str(args_obj["int_a"]), "5", "Parameter int_a should be 5") + self.assertEqual(str(args_obj["int_b"]), "7", "Parameter int_b should be 7") + + def test_function_call_required(self): + """ + Test: Whether tool_choice: "required" works as expected + - When tool_choice == "required", the model should return one or more tool_calls. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "sub", + "description": "Compute the difference of two integers", + "parameters": { + "type": "object", + "properties": { + "int_a": { + "type": "integer", + "description": "First integer", + }, + "int_b": { + "type": "integer", + "description": "Second integer", + }, + }, + "required": ["int_a", "int_b"], + }, + "strict": True, + }, + }, + { + "type": "function", + "function": { + "name": "get_weather", + "description": "use this to get latest weather information for a city given its name", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "name of the city to get weather for", + } + }, + "required": ["city"], + }, + }, + }, + ] + + messages = [{"role": "user", "content": "What is the capital of France?"}] + response = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=False, + tools=tools, + tool_choice="required", + ) + + tool_calls = response.choices[0].message.tool_calls + self.assertIsNotNone(tool_calls, "No tool_calls in the response") + function_name = tool_calls[0].function.name + arguments = tool_calls[0].function.arguments + args_obj = json.loads(arguments) + + self.assertEqual( + function_name, + "get_weather", + f"Function name should be 'get_weather', got: {function_name}", + ) + self.assertIn( + "city", args_obj, f"Function arguments should have 'city', got: {args_obj}" + ) + + # Make the test more robust by checking type and accepting valid responses + city_value = args_obj["city"] + self.assertIsInstance( + city_value, + str, + f"Parameter city should be a string, got: {type(city_value)}", + ) + self.assertTrue( + "Paris" in city_value or "France" in city_value, + f"Parameter city should contain either 'Paris' or 'France', got: {city_value}", + ) + + def test_function_call_specific(self): + """ + Test: Whether tool_choice: ToolChoice works as expected + - When tool_choice is a specific ToolChoice, the model should return one or more tool_calls. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "sub", + "description": "Compute the difference of two integers", + "parameters": { + "type": "object", + "properties": { + "int_a": { + "type": "integer", + "description": "First integer", + }, + "int_b": { + "type": "integer", + "description": "Second integer", + }, + }, + "required": ["int_a", "int_b"], + }, + "strict": True, + }, + }, + { + "type": "function", + "function": { + "name": "get_weather", + "description": "use this to get latest weather information for a city given its name", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "name of the city to get weather for", + } + }, + "required": ["city"], + }, + }, + }, + ] + + messages = [{"role": "user", "content": "What is the capital of France?"}] + response = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=False, + tools=tools, + tool_choice={"type": "function", "function": {"name": "get_weather"}}, + ) + + tool_calls = response.choices[0].message.tool_calls + self.assertIsNotNone(tool_calls, "No tool_calls in the response") + function_name = tool_calls[0].function.name + arguments = tool_calls[0].function.arguments + args_obj = json.loads(arguments) + + self.assertEqual( + function_name, "get_weather", "Function name should be 'get_weather'" + ) + self.assertIn("city", args_obj, "Function arguments should have 'city'") + + def test_streaming_multiple_choices_finish_reason(self): + """ + Test: Verify that each choice gets its own finish_reason chunk in streaming mode with n > 1. + This tests the fix for the bug where only the last index got a finish_reason chunk. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["location"], + }, + }, + } + ] + + messages = [ + {"role": "user", "content": "What is the weather like in Los Angeles?"} + ] + + # Request with n=2 to get multiple choices + response_stream = client.chat.completions.create( + model=self.model, + messages=messages, + max_tokens=2048, + temperature=0.8, + stream=True, + tools=tools, + tool_choice="required", # Force tool calls + n=2, # Multiple choices + ) + + chunks = list(response_stream) + + # Track finish_reason chunks for each index + finish_reason_chunks = {} + for chunk in chunks: + if chunk.choices: + for choice in chunk.choices: + if choice.finish_reason is not None: + index = choice.index + if index not in finish_reason_chunks: + finish_reason_chunks[index] = [] + finish_reason_chunks[index].append(choice.finish_reason) + + # Verify we got finish_reason chunks for both indices + self.assertEqual( + len(finish_reason_chunks), + 2, + f"Expected finish_reason chunks for 2 indices, got {len(finish_reason_chunks)}", + ) + + # Verify both index 0 and 1 have finish_reason + self.assertIn( + 0, finish_reason_chunks, "Missing finish_reason chunk for index 0" + ) + self.assertIn( + 1, finish_reason_chunks, "Missing finish_reason chunk for index 1" + ) + + # Verify the finish_reason is "tool_calls" since we forced tool calls + for index, reasons in finish_reason_chunks.items(): + self.assertEqual( + reasons[-1], # Last finish_reason for this index + "tool_calls", + f"Expected finish_reason 'tool_calls' for index {index}, got {reasons[-1]}", + ) + + def test_function_calling_streaming_no_tool_call(self): + """ + Test: Whether the finish_reason is stop in streaming mode when no tool call is given. + - Expect no function call to be found. + - Verify that finish_reason is stop + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city to find the weather for", + }, + "unit": { + "type": "string", + "description": "Weather unit (celsius or fahrenheit)", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["city", "unit"], + }, + }, + } + ] + + messages = [{"role": "user", "content": "Who are you?"}] + + response_stream = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=True, + tools=tools, + tool_choice="none", + ) + + chunks = list(response_stream) + self.assertTrue(len(chunks) > 0, "Streaming should return at least one chunk") + + found_tool_call = False + for chunk in chunks: + choice = chunk.choices[0] + # Check whether the current chunk contains tool_calls + found_tool_call = choice.delta.tool_calls is not None + + self.assertFalse( + found_tool_call, + "Shouldn't have any tool_call in the streaming chunks", + ) + + finish_reason = chunks[-1].choices[0].finish_reason + self.assertEqual( + finish_reason, + "stop", + "Final response of no function calling should have finish_reason 'stop'", + ) + + def test_streaming_multiple_choices_without_tools(self): + """ + Test: Verify that each choice gets its own finish_reason chunk without tool calls. + This tests the fix for regular content streaming with multiple choices. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + messages = [{"role": "user", "content": "Say hello in one word."}] + + # Request with n=2 to get multiple choices, no tools + response_stream = client.chat.completions.create( + model=self.model, + messages=messages, + temperature=0.8, + stream=True, + max_tokens=10, # Keep it short + n=2, # Multiple choices + ) + + chunks = list(response_stream) + + # Track finish_reason chunks for each index + finish_reason_chunks = {} + for chunk in chunks: + if chunk.choices: + for choice in chunk.choices: + if choice.finish_reason is not None: + index = choice.index + if index not in finish_reason_chunks: + finish_reason_chunks[index] = [] + finish_reason_chunks[index].append(choice.finish_reason) + + # Verify we got finish_reason chunks for both indices + self.assertEqual( + len(finish_reason_chunks), + 2, + f"Expected finish_reason chunks for 2 indices, got {len(finish_reason_chunks)}", + ) + + # Verify both index 0 and 1 have finish_reason + self.assertIn( + 0, finish_reason_chunks, "Missing finish_reason chunk for index 0" + ) + self.assertIn( + 1, finish_reason_chunks, "Missing finish_reason chunk for index 1" + ) + + # Verify the finish_reason is "stop" (regular completion) + for index, reasons in finish_reason_chunks.items(): + self.assertIn( + reasons[-1], + ["stop", "length"], # Could be either depending on how model responds + f"Expected finish_reason 'stop' or 'length' for index {index}, got {reasons[-1]}", + ) + + +class TestOpenAIPythonicFunctionCalling(CustomTestCase): + PYTHONIC_TOOLS = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a given location.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The name of the city or location.", + } + }, + "required": ["location"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_tourist_attractions", + "description": "Get a list of top tourist attractions for a given city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The name of the city to find attractions for.", + } + }, + "required": ["city"], + }, + }, + }, + ] + + PYTHONIC_MESSAGES = [ + { + "role": "system", + "content": ( + "You are a travel assistant. " + "When asked to call functions, ALWAYS respond ONLY with a python list of function calls, " + "using this format: [func_name1(param1=value1, param2=value2), func_name2(param=value)]. " + "Do NOT use JSON, do NOT use variables, do NOT use any other format. " + "Here is an example:\n" + '[get_weather(location="Paris"), get_tourist_attractions(city="Paris")]' + ), + }, + { + "role": "user", + "content": ( + "I'm planning a trip to Tokyo next week. What's the weather like and what are some top tourist attractions? " + "Propose parallel tool calls at once, using the python list of function calls format as shown above." + ), + }, + ] + + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--tool-call-parser", + "pythonic", + ], + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_pythonic_tool_call_prompt(self): + """ + Test: Explicit prompt for pythonic tool call format without chat template. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.chat.completions.create( + model=self.model, + messages=self.PYTHONIC_MESSAGES, + tools=self.PYTHONIC_TOOLS, + temperature=0.1, + stream=False, + ) + tool_calls = response.choices[0].message.tool_calls + self.assertIsInstance(tool_calls, list, "No tool_calls found") + self.assertGreaterEqual(len(tool_calls), 1) + names = [tc.function.name for tc in tool_calls] + self.assertTrue( + "get_weather" in names or "get_tourist_attractions" in names, + f"Function name '{names}' should container either 'get_weather' or 'get_tourist_attractions'", + ) + + def test_pythonic_tool_call_streaming(self): + """ + Test: Streaming pythonic tool call format; assert tool_call index is present. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response_stream = client.chat.completions.create( + model=self.model, + messages=self.PYTHONIC_MESSAGES, + tools=self.PYTHONIC_TOOLS, + temperature=0.1, + stream=True, + ) + found_tool_calls = False + found_index = False + found_names = set() + for chunk in response_stream: + choice = chunk.choices[0] + if getattr(choice.delta, "tool_calls", None): + found_tool_calls = True + tool_call = choice.delta.tool_calls[0] + if hasattr(tool_call, "index") or ( + isinstance(tool_call, dict) and "index" in tool_call + ): + found_index = True + found_names.add(str(tool_call.function.name)) + + self.assertTrue(found_tool_calls, "No tool_calls found in streaming response") + self.assertTrue(found_index, "No index field found in any streamed tool_call") + self.assertTrue( + "get_weather" in found_names or "get_tourist_attractions" in found_names, + f"Function name '{found_names}' should container either 'get_weather' or 'get_tourist_attractions'", + ) + + +# Skip for ci test +# class TestGLM45ServerFunctionCalling(TestOpenAIServerFunctionCalling): +# @classmethod +# def setUpClass(cls): +# # Replace with the model name needed for testing; if not required, reuse DEFAULT_SMALL_MODEL_NAME_FOR_TEST +# cls.model = "THUDM/GLM-4.5" +# cls.base_url = DEFAULT_URL_FOR_TEST +# cls.api_key = "sk-123456" + +# # Start the local OpenAI Server. If necessary, you can add other parameters such as --enable-tools. +# cls.process = popen_launch_server( +# cls.model, +# cls.base_url, +# timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, +# api_key=cls.api_key, +# other_args=[ +# # If your server needs extra parameters to test function calling, please add them here. +# "--tool-call-parser", +# "glm45", +# "--reasoning-parser", +# "glm45", +# "--tp-size", +# "8" +# ], +# ) +# cls.base_url += "/v1" +# cls.tokenizer = get_tokenizer(cls.model) + +# # This test is too difficult for GLM4-moe. Skip it from the UT +# def test_function_call_required(self): +# pass + +# def test_function_calling_multiturn(self): +# self._test_function_calling_multiturn() + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/function_call/test_tool_choice.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/function_call/test_tool_choice.py new file mode 100644 index 00000000..f324f66e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/function_call/test_tool_choice.py @@ -0,0 +1,855 @@ +""" +Test script for tool_choice functionality in SGLang +Tests: required, auto, and specific function choices in both streaming and non-streaming modes + +# To run the tests, use the following command: +# +# python3 -m unittest openai_server.function_call.test_tool_choice +""" + +import json +import unittest + +import openai + +from sglang.srt.utils import kill_process_tree +from sglang.srt.utils.hf_transformers_utils import get_tokenizer +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestToolChoiceLlama32(CustomTestCase): + + @classmethod + def setUpClass(cls): + # Mark flaky tests for this model + cls.flaky_tests = { + "test_multi_tool_scenario_auto", + "test_multi_tool_scenario_required", + } + + # Use a model that supports function calling + cls.model = "meta-llama/Llama-3.2-1B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + # Start the local OpenAI Server with tool calling support + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--tool-call-parser", + "llama3", # Default parser for the test model + ], + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def setUp(self): + self.client = openai.Client(base_url=self.base_url, api_key=self.api_key) + self.model_name = self.client.models.list().data[0].id + + def _is_flaky_test(self): + """Check if the current test is marked as flaky for this class""" + return ( + hasattr(self.__class__, "flaky_tests") + and self._testMethodName in self.__class__.flaky_tests + ) + + def get_test_tools(self): + """Get the test tools for function calling""" + return [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "use this to get latest weather information for a city given its name", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "name of the city to get weather for", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["city"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_pokemon_info", + "description": "get detailed information about a pokemon given its name", + "parameters": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "name of the pokemon to get info for", + } + }, + "required": ["name"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "make_next_step_decision", + "description": "You will be given a trace of thinking process in the following format.\n\nQuestion: the input question you must answer\nTOOL: think about what to do, and choose a tool to use ONLY IF there are defined tools. \n You should never call the same tool with the same input twice in a row.\n If the previous conversation history already contains the information that can be retrieved from the tool, you should not call the tool again.\nOBSERVATION: the result of the tool call, NEVER include this in your response, this information will be provided\n... (this TOOL/OBSERVATION can repeat N times)\nANSWER: If you know the answer to the original question, require for more information,\n or you don't know the answer and there are no defined tools or all available tools are not helpful, respond with the answer without mentioning anything else.\n If the previous conversation history already contains the answer, respond with the answer right away.\n\n If no tools are configured, naturally mention this limitation while still being helpful. Briefly note that adding tools in the agent configuration would expand capabilities.\n\nYour task is to respond with the next step to take, based on the traces, \nor answer the question if you have enough information.", + "parameters": { + "type": "object", + "properties": { + "decision": { + "type": "string", + "description": 'The next step to take, it must be either "TOOL" or "ANSWER". If the previous conversation history already contains the information that can be retrieved from the tool, you should not call the tool again. If there are no defined tools, you should not return "TOOL" in your response.', + }, + "content": { + "type": "string", + "description": 'The content of the next step. If the decision is "TOOL", this should be a short and concise reasoning of why you chose the tool, MUST include the tool name. If the decision is "ANSWER", this should be the answer to the question. If no tools are available, integrate this limitation conversationally without sounding scripted.', + }, + }, + "required": ["decision", "content"], + }, + }, + }, + ] + + def get_test_messages(self): + """Get test messages that should trigger tool usage""" + return [ + { + "role": "user", + "content": "Answer the following questions as best you can:\n\nYou will be given a trace of thinking process in the following format.\n\nQuestion: the input question you must answer\nTOOL: think about what to do, and choose a tool to use ONLY IF there are defined tools\nOBSERVATION: the result of the tool call or the observation of the current task, NEVER include this in your response, this information will be provided\n... (this TOOL/OBSERVATION can repeat N times)\nANSWER: If you know the answer to the original question, require for more information, \nif the previous conversation history already contains the answer, \nor you don't know the answer and there are no defined tools or all available tools are not helpful, respond with the answer without mentioning anything else.\nYou may use light Markdown formatting to improve clarity (e.g. lists, **bold**, *italics*), but keep it minimal and unobtrusive.\n\nYour task is to respond with the next step to take, based on the traces, \nor answer the question if you have enough information.\n\nQuestion: what is the weather in top 5 populated cities in the US in celsius?\n\nTraces:\n\n\nThese are some additional instructions that you should follow:", + } + ] + + def get_travel_tools(self): + """Get tools for travel assistant scenario that should trigger multiple tool calls""" + return [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a given location.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The name of the city or location.", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["location"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_tourist_attractions", + "description": "Get a list of top tourist attractions for a given city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The name of the city to find attractions for.", + } + }, + "required": ["city"], + }, + }, + }, + ] + + def get_travel_messages(self): + """Get travel assistant messages that should trigger multiple tool calls""" + return [ + { + "content": "You are a travel assistant providing real-time weather updates and top tourist attractions.", + "role": "system", + }, + { + "content": "I'm planning a trip to Tokyo next week. What's the weather like? What are the most amazing sights?", + "role": "user", + }, + ] + + def test_tool_choice_auto_non_streaming(self): + """Test tool_choice='auto' in non-streaming mode""" + tools = self.get_test_tools() + messages = self.get_test_messages() + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + tools=tools, + tool_choice="auto", + stream=False, + ) + + self.assertIsNotNone(response.choices[0].message) + # With auto, tool calls are optional + + def test_tool_choice_auto_streaming(self): + """Test tool_choice='auto' in streaming mode""" + tools = self.get_test_tools() + messages = self.get_test_messages() + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + tools=tools, + tool_choice="auto", + stream=True, + ) + + # Collect streaming response + content_chunks = [] + tool_call_chunks = [] + + for chunk in response: + if chunk.choices[0].delta.content: + content_chunks.append(chunk.choices[0].delta.content) + elif chunk.choices[0].delta.tool_calls: + tool_call_chunks.extend(chunk.choices[0].delta.tool_calls) + + # Should complete without errors + self.assertIsInstance(content_chunks, list) + self.assertIsInstance(tool_call_chunks, list) + + def test_tool_choice_required_non_streaming(self): + """Test tool_choice='required' in non-streaming mode""" + tools = self.get_test_tools() + messages = self.get_test_messages() + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + temperature=0.2, + tools=tools, + tool_choice="required", + stream=False, + ) + + # With required, we should get tool calls + tool_calls = response.choices[0].message.tool_calls + self.assertIsNotNone(tool_calls) + self.assertGreater(len(tool_calls), 0) + + def test_tool_choice_required_streaming(self): + """Test tool_choice='required' in streaming mode""" + tools = self.get_test_tools() + messages = self.get_test_messages() + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + tools=tools, + tool_choice="required", + stream=True, + ) + + # Collect streaming response + tool_call_chunks = [] + + for chunk in response: + if chunk.choices[0].delta.tool_calls: + tool_call_chunks.extend(chunk.choices[0].delta.tool_calls) + + # With required, we should get tool call chunks + self.assertGreater(len(tool_call_chunks), 0) + + def test_tool_choice_specific_function_non_streaming(self): + """Test tool_choice with specific function in non-streaming mode""" + tools = self.get_test_tools() + messages = self.get_test_messages() + + tool_choice = {"type": "function", "function": {"name": "get_weather"}} + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + tools=tools, + tool_choice=tool_choice, + stream=False, + ) + + # Should call the specific function + tool_calls = response.choices[0].message.tool_calls + self.assertIsNotNone(tool_calls) + # Our messages ask the top 5 populated cities in the US, so the model could get 5 tool calls + self.assertGreaterEqual(len(tool_calls), 1) + for tool_call in tool_calls: + self.assertEqual(tool_call.function.name, "get_weather") + + def test_tool_choice_specific_function_streaming(self): + """Test tool_choice with specific function in streaming mode""" + tools = self.get_test_tools() + messages = self.get_test_messages() + + tool_choice = {"type": "function", "function": {"name": "get_weather"}} + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + tools=tools, + tool_choice=tool_choice, + stream=True, + ) + + # Collect streaming response + tool_call_chunks = [] + + for chunk in response: + if chunk.choices[0].delta.tool_calls: + tool_call_chunks.extend(chunk.choices[0].delta.tool_calls) + + # Should get tool call chunks for the specific function + self.assertGreater(len(tool_call_chunks), 0) + + # Find function name in chunks + found_name = None + for chunk in tool_call_chunks: + if chunk.function and chunk.function.name: + found_name = chunk.function.name + break + + self.assertEqual(found_name, "get_weather") + + def test_required_streaming_arguments_chunks_json(self): + """In streaming required mode, complete tool call arguments should be valid JSON when all chunks are combined""" + tools = self.get_test_tools() + messages = self.get_test_messages() + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=1024, + temperature=0.1, + tools=tools, + tool_choice="required", + stream=True, + ) + + # Collect all tool call chunks and reconstruct complete tool calls + tool_calls_by_index = {} + for chunk in response: + if chunk.choices[0].delta.tool_calls: + for tool_call_delta in chunk.choices[0].delta.tool_calls: + tool_index = tool_call_delta.index + + # Initialize tool call if not seen before + if tool_index not in tool_calls_by_index: + tool_calls_by_index[tool_index] = { + "id": tool_call_delta.id, + "type": "function", + "function": {"name": "", "arguments": ""}, + } + + # Update function name if present (first chunk) + if tool_call_delta.function and tool_call_delta.function.name: + tool_calls_by_index[tool_index]["function"][ + "name" + ] = tool_call_delta.function.name + + # Accumulate arguments (all chunks) + if tool_call_delta.function and tool_call_delta.function.arguments: + tool_calls_by_index[tool_index]["function"][ + "arguments" + ] += tool_call_delta.function.arguments + + self.assertGreater(len(tool_calls_by_index), 0) + + # Validate that complete tool calls have valid JSON arguments + for tool_call in tool_calls_by_index.values(): + self.assertIsNotNone(tool_call["function"]["name"]) + self.assertIsNotNone(tool_call["function"]["arguments"]) + + # The complete arguments should be valid JSON + try: + args = json.loads(tool_call["function"]["arguments"]) + self.assertIsInstance(args, dict) + except json.JSONDecodeError: + self.fail( + f"Invalid JSON in complete tool call arguments: {tool_call['function']['arguments']}" + ) + + def test_complex_parameters_required_non_streaming(self): + """Validate complex nested parameter schemas in non-streaming required mode""" + complex_tools = [ + { + "type": "function", + "function": { + "name": "analyze_data", + "description": "Analyze complex data structures", + "parameters": { + "type": "object", + "properties": { + "data": { + "type": "object", + "properties": { + "metrics": { + "type": "array", + "items": {"type": "string"}, + }, + "config": { + "type": "object", + "properties": { + "threshold": {"type": "number"}, + "enabled": {"type": "boolean"}, + }, + }, + }, + "required": ["metrics"], + }, + "options": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "value": {"type": "string"}, + }, + }, + }, + }, + "required": ["data"], + }, + }, + } + ] + + messages = [ + { + "role": "user", + "content": "Analyze some data with metrics and configuration", + } + ] + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=1024, + temperature=0.1, + tools=complex_tools, + tool_choice="required", + stream=False, + ) + + tool_calls = response.choices[0].message.tool_calls + self.assertIsNotNone(tool_calls) + self.assertGreater(len(tool_calls), 0) + + for tool_call in tool_calls: + self.assertEqual(tool_call.function.name, "analyze_data") + try: + args = json.loads(tool_call.function.arguments) + self.assertIsInstance(args, dict) + self.assertIn("data", args) + self.assertIsInstance(args["data"], dict) + except json.JSONDecodeError: + self.fail( + f"Invalid JSON in complex tool call arguments: {tool_call.function.arguments}" + ) + + def test_multi_tool_scenario_auto(self): + """Test multi-tool scenario with tool_choice='auto'""" + tools = self.get_travel_tools() + messages = self.get_travel_messages() + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + temperature=0.2, + tools=tools, + tool_choice="auto", + stream=False, + ) + + # Should complete without errors + self.assertIsNotNone(response.choices[0].message) + + tool_calls = response.choices[0].message.tool_calls + expected_functions = {"get_weather", "get_tourist_attractions"} + + if self._is_flaky_test(): + # For flaky tests, just verify all called functions are available tools + if tool_calls: + available_names = [tool["function"]["name"] for tool in tools] + for call in tool_calls: + self.assertIn(call.function.name, available_names) + else: + # For non-flaky tests, enforce strict requirements + self.assertIsNotNone(tool_calls, "Expected tool calls but got none") + self.assertEqual( + len(tool_calls), 2, f"Expected 2 tool calls, got {len(tool_calls)}" + ) + + called_functions = {call.function.name for call in tool_calls} + self.assertEqual( + called_functions, + expected_functions, + f"Expected functions {expected_functions}, got {called_functions}", + ) + + def test_multi_tool_scenario_required(self): + """Test multi-tool scenario with tool_choice='required'""" + tools = self.get_travel_tools() + messages = self.get_travel_messages() + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + temperature=0.2, + tools=tools, + tool_choice="required", + stream=False, + ) + + # With required, we should get at least one tool call + tool_calls = response.choices[0].message.tool_calls + self.assertIsNotNone(tool_calls) + self.assertGreater(len(tool_calls), 0) + + # Verify all called functions are available tools + available_names = [tool["function"]["name"] for tool in tools] + expected_functions = {"get_weather", "get_tourist_attractions"} + + for tool_call in tool_calls: + self.assertIsNotNone(tool_call.function.name) + self.assertIsNotNone(tool_call.function.arguments) + + if self._is_flaky_test(): + # For flaky tests, just ensure basic functionality works + self.assertGreater( + len(tool_calls), + 0, + f"Expected at least 1 tool call, got {len(tool_calls)}", + ) + for call in tool_calls: + self.assertIn(call.function.name, available_names) + else: + # For non-flaky tests, enforce strict requirements + self.assertEqual( + len(tool_calls), 2, f"Expected 2 tool calls, got {len(tool_calls)}" + ) + + called_functions = {call.function.name for call in tool_calls} + self.assertEqual( + called_functions, + expected_functions, + f"Expected functions {expected_functions}, got {called_functions}", + ) + + def test_error_handling_invalid_tool_choice(self): + """Test error handling for invalid tool_choice""" + tools = self.get_test_tools() + messages = self.get_test_messages() + + # Test with invalid function name + tool_choice = {"type": "function", "function": {"name": "nonexistent_function"}} + + # Expect a 400 BadRequestError to be raised for invalid tool_choice + with self.assertRaises(openai.BadRequestError) as context: + self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + tools=tools, + tool_choice=tool_choice, + stream=False, + ) + + # Verify the error message contains the expected text + self.assertIn( + "Tool 'nonexistent_function' not found in tools list", + str(context.exception), + ) + + def test_invalid_tool_missing_name(self): + """Test what happens when user doesn't provide a tool name in request""" + # Test with malformed JSON in tool parameters - missing required "name" field + invalid_tools = [ + { + "type": "function", + "function": { + # Missing required "name" field + "description": "Test function with invalid schema", + "parameters": { + "type": "object", + "properties": { + "test_field": { + "type": "string", + "description": "Test field", + } + }, + "required": ["test_field"], + }, + }, + } + ] + + messages = [ + { + "role": "user", + "content": "Test the function", + } + ] + + # Should raise BadRequestError due to missing required 'name' field + with self.assertRaises(openai.BadRequestError) as context: + self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=100, + temperature=0.1, + tools=invalid_tools, + tool_choice="required", + stream=False, + ) + + # Verify the error message indicates missing name field + error_msg = str(context.exception).lower() + self.assertIn("name", error_msg) + + def test_invalid_json_schema_in_tool(self): + """Test what happens when tool function has invalid JSON schema""" + invalid_tools = [ + { + "type": "function", + "function": { + "name": "test_function", + "description": "Test function with invalid JSON schema", + "parameters": { + "type": "object", + "properties": { + "invalid_field": { + "type": "unknown_type", # Invalid type + "description": "This field has an invalid type", + } + }, + "required": ["invalid_field"], + }, + }, + } + ] + + messages = [ + { + "role": "user", + "content": "Test the function", + } + ] + + # Should raise BadRequestError due to invalid JSON schema in tool parameters + with self.assertRaises(openai.BadRequestError) as context: + self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=100, + temperature=0.1, + tools=invalid_tools, + tool_choice="required", + stream=False, + ) + + # Verify the error message indicates invalid JSON schema for parameters field + error_msg = str(context.exception).lower() + self.assertIn("invalid 'parameters' schema", error_msg) + + def test_conflicting_defs_required_tool_choice(self): + """Test that conflicting $defs with required tool_choice returns 400 error""" + conflicting_tools = [ + { + "type": "function", + "function": { + "name": "tool1", + "description": "Tool 1 with conflicting $defs", + "parameters": { + "type": "object", + "properties": { + "data": {"$ref": "#/$defs/DataType"}, + }, + "required": ["data"], + "$defs": { + "DataType": { + "type": "object", + "properties": {"value": {"type": "string"}}, + "required": ["value"], + }, + }, + }, + }, + }, + { + "type": "function", + "function": { + "name": "tool2", + "description": "Tool 2 with conflicting $defs", + "parameters": { + "type": "object", + "properties": { + "data": {"$ref": "#/$defs/DataType"}, + }, + "required": ["data"], + "$defs": { + "DataType": { # Different definition for DataType + "type": "object", + "properties": {"value": {"type": "number"}}, + "required": ["value"], + }, + }, + }, + }, + }, + ] + + messages = [ + { + "role": "user", + "content": "Test the conflicting tools", + } + ] + + # Should raise BadRequestError due to conflicting $defs + with self.assertRaises(openai.BadRequestError) as context: + self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=100, + temperature=0.1, + tools=conflicting_tools, + tool_choice="required", + stream=False, + ) + + # Verify the error message indicates conflicting tool definitions + error_msg = str(context.exception).lower() + self.assertIn("multiple schemas", error_msg) + self.assertIn("not supported", error_msg) + + +class TestToolChoiceQwen25(TestToolChoiceLlama32): + """Test tool_choice functionality with Qwen2.5 model""" + + @classmethod + def setUpClass(cls): + cls.flaky_tests = {} + + cls.model = "Qwen/Qwen2.5-7B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--tool-call-parser", + "qwen25", + ], + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + +class TestToolChoiceMistral(TestToolChoiceLlama32): + """Test tool_choice functionality with Mistral model""" + + @classmethod + def setUpClass(cls): + # Mark flaky tests for this model + cls.flaky_tests = { + "test_multi_tool_scenario_auto", + "test_multi_tool_scenario_required", + } + + cls.model = "mistralai/Mistral-7B-Instruct-v0.3" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--tool-call-parser", + "mistral", + ], + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + @unittest.skip("Fails due to whitespace issue with Mistral - skipping") + def test_multi_tool_scenario_required(self): + """Test multi-tool scenario with tool_choice='required'""" + super().test_multi_tool_scenario_required() + + @unittest.skip("Fails due to whitespace issue with Mistral - skipping") + def test_complex_parameters_required_non_streaming(self): + """Validate complex nested parameter schemas in non-streaming required mode""" + super().test_complex_parameters_required_non_streaming() + + +# Skip for ci test +# class TestToolChoiceGLM45(TestToolChoiceLlama32): +# @classmethod +# def setUpClass(cls): +# # Replace with the model name needed for testing; if not required, reuse DEFAULT_SMALL_MODEL_NAME_FOR_TEST +# cls.model = "THUDM/GLM-4.5" +# cls.base_url = DEFAULT_URL_FOR_TEST +# cls.api_key = "sk-123456" + +# # Start the local OpenAI Server. If necessary, you can add other parameters such as --enable-tools. +# cls.process = popen_launch_server( +# cls.model, +# cls.base_url, +# timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, +# api_key=cls.api_key, +# other_args=[ +# # If your server needs extra parameters to test function calling, please add them here. +# "--tool-call-parser", +# "glm45", +# "--reasoning-parser", +# "glm45", +# "--tp-size", +# "8" +# ], +# ) +# cls.base_url += "/v1" +# cls.tokenizer = get_tokenizer(cls.model) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/validation/__init__.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/validation/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/validation/test_large_max_new_tokens.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/validation/test_large_max_new_tokens.py new file mode 100644 index 00000000..e1e2aa8f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/validation/test_large_max_new_tokens.py @@ -0,0 +1,103 @@ +""" +python3 -m unittest openai_server.validation.test_large_max_new_tokens.TestLargeMaxNewTokens.test_chat_completion +""" + +import os +import time +import unittest +from concurrent.futures import ThreadPoolExecutor + +import openai + +from sglang.srt.utils import kill_process_tree +from sglang.srt.utils.hf_transformers_utils import get_tokenizer +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + STDERR_FILENAME, + STDOUT_FILENAME, + CustomTestCase, + popen_launch_server, +) + + +class TestLargeMaxNewTokens(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + cls.stdout = open(STDOUT_FILENAME, "w") + cls.stderr = open(STDERR_FILENAME, "w") + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=( + "--max-total-token", + "1536", + "--context-len", + "8192", + "--decode-log-interval", + "2", + ), + env={"SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION": "256", **os.environ}, + return_stdout_stderr=(cls.stdout, cls.stderr), + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + cls.stdout.close() + cls.stderr.close() + os.remove(STDOUT_FILENAME) + os.remove(STDERR_FILENAME) + + def run_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + { + "role": "user", + "content": "Please repeat the world 'hello' for 10000 times.", + }, + ], + temperature=0, + ) + return response + + def test_chat_completion(self): + num_requests = 4 + + futures = [] + with ThreadPoolExecutor(num_requests) as executor: + # Send multiple requests + for i in range(num_requests): + futures.append(executor.submit(self.run_chat_completion)) + + # Ensure that they are running concurrently + pt = 0 + while pt >= 0: + time.sleep(5) + lines = open(STDERR_FILENAME).readlines() + for line in lines[pt:]: + print(line, end="", flush=True) + if f"#running-req: {num_requests}" in line: + all_requests_running = True + pt = -1 + break + pt += 1 + + assert all_requests_running + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/validation/test_matched_stop.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/validation/test_matched_stop.py new file mode 100644 index 00000000..62c586ba --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/validation/test_matched_stop.py @@ -0,0 +1,80 @@ +import unittest + +from sglang.srt.sampling.sampling_params import MAX_LEN, get_max_seq_length +from sglang.srt.utils import kill_process_tree +from sglang.test.kits.matched_stop_kit import MatchedStopMixin +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestMatchedStop(CustomTestCase, MatchedStopMixin): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=300, + other_args=["--max-running-requests", "10"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +class TestRegexPatternMaxLength(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.regex_str_to_max_len = { + "((ab|cd(e|f){2}){3,5}g|hij)*k": MAX_LEN, + # - '*' → infinite tokens need to be stored + "abc*?k": MAX_LEN, + # - '*?' → infinite tokens still need to be stored even if lazy matching used + "^spec(foo|at)$": 7, + # - '^' and '$' don't add any characters to the max length + # "spec" → 4 + # "(foo|at)" → max(3, 2) = 3 + # Whole regex = 7 + "(a(bca|de(fg|hi){2,3})j){2}kl": 22, + # - Innermost alt: "fg" vs "hi" → 2 + # - Repeat {2,3}: max = 3 * 2 = 6 + # - Inner group "de(...)": 2 (for "de") + 6 = 8. + # - "bca" or "de(...)" → max(3, 8) = 8 + # - Whole group: "a" (1) + group (8) + "j"(1) = 10 + # - Repeat {2} → 20 + # - Add "kl"(2) → 22 + "(foo(bar|baz(qux){1,2}))|(x(yz){5,10})": 21, + # Branch 1: + # "foo"(3) + max("bar"(3), "baz"(3)+"qux"{2} = 3 + 6 = 9) = 3 + 9 = 12 + # Branch 2: + # "x"(1) + "yz"{10} = 1 + 20 =21 + # Whole regex = max(12, 21) = 21 + "(((a|bc){1,3}(d(e|f){2}|gh){2,4})|(ijk|lmp(no|p){3})){5}": 90, + # Branch A: + # (a|bc){1,3} → max = 3 * 2 = 6 + # Inside: d(e|f){2} = 1 + 2 * 1 = 3 vs gh = 2 → max = 3 + # Repeat {2,4} → 4 * 3 = 12 + # Branch A total = 18 + # Branch B: + # "ijk"(3) vs "lmp(no|p){3}" = 3 + 3 * max(2, 1) = 3 + 6 = 9 → max = 9 + # Branch B total = 9 + # Whole outer alt = max(18, 9) = 18 + # Repeat {5} → 90 + } + + def test_get_max_length(self): + for regex_str, max_len in self.regex_str_to_max_len.items(): + if max_len == MAX_LEN: + self.assertGreaterEqual(get_max_seq_length(regex_str), MAX_LEN) + else: + self.assertEqual(get_max_seq_length(regex_str), max_len) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/validation/test_openai_server_ignore_eos.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/validation/test_openai_server_ignore_eos.py new file mode 100644 index 00000000..7c69011f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/validation/test_openai_server_ignore_eos.py @@ -0,0 +1,84 @@ +import openai + +from sglang.srt.utils import kill_process_tree +from sglang.srt.utils.hf_transformers_utils import get_tokenizer +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestOpenAIServerIgnoreEOS(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_ignore_eos(self): + """ + Test that ignore_eos=True allows generation to continue beyond EOS token + and reach the max_tokens limit. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + max_tokens = 200 + + response_default = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Count from 1 to 20."}, + ], + temperature=0, + max_tokens=max_tokens, + extra_body={"ignore_eos": False}, + ) + + response_ignore_eos = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Count from 1 to 20."}, + ], + temperature=0, + max_tokens=max_tokens, + extra_body={"ignore_eos": True}, + ) + + default_tokens = len( + self.tokenizer.encode(response_default.choices[0].message.content) + ) + ignore_eos_tokens = len( + self.tokenizer.encode(response_ignore_eos.choices[0].message.content) + ) + + # Check if ignore_eos resulted in more tokens or exactly max_tokens + # The ignore_eos response should either: + # 1. Have more tokens than the default response (if default stopped at EOS before max_tokens) + # 2. Have exactly max_tokens (if it reached the max_tokens limit) + self.assertTrue( + ignore_eos_tokens > default_tokens or ignore_eos_tokens >= max_tokens, + f"ignore_eos did not generate more tokens: {ignore_eos_tokens} vs {default_tokens}", + ) + + self.assertEqual( + response_ignore_eos.choices[0].finish_reason, + "length", + f"Expected finish_reason='length' for ignore_eos=True, got {response_ignore_eos.choices[0].finish_reason}", + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/validation/test_request_length_validation.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/validation/test_request_length_validation.py new file mode 100644 index 00000000..7276906d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/openai_server/validation/test_request_length_validation.py @@ -0,0 +1,88 @@ +import unittest + +import openai + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestRequestLengthValidation(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + # Start server with auto truncate disabled + cls.process = popen_launch_server( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=("--max-total-tokens", "1000", "--context-length", "1000"), + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_input_length_longer_than_context_length(self): + client = openai.Client(api_key=self.api_key, base_url=f"{self.base_url}/v1") + + long_text = "hello " * 1200 # Will tokenize to more than context length + + with self.assertRaises(openai.BadRequestError) as cm: + client.chat.completions.create( + model=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + messages=[ + {"role": "user", "content": long_text}, + ], + temperature=0, + ) + + self.assertIn("is longer than the model's context length", str(cm.exception)) + + def test_input_length_longer_than_maximum_allowed_length(self): + client = openai.Client(api_key=self.api_key, base_url=f"{self.base_url}/v1") + + long_text = "hello " * 999 # the maximum allowed length is 994 tokens + + with self.assertRaises(openai.BadRequestError) as cm: + client.chat.completions.create( + model=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + messages=[ + {"role": "user", "content": long_text}, + ], + temperature=0, + ) + + self.assertIn("is longer than the model's context length", str(cm.exception)) + + def test_max_tokens_validation(self): + client = openai.Client(api_key=self.api_key, base_url=f"{self.base_url}/v1") + + long_text = "hello " + + with self.assertRaises(openai.BadRequestError) as cm: + client.chat.completions.create( + model=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + messages=[ + {"role": "user", "content": long_text}, + ], + temperature=0, + max_tokens=1200, + ) + + self.assertIn( + "max_completion_tokens is too large", + str(cm.exception), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ops/test_repeat_interleave.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ops/test_repeat_interleave.py new file mode 100644 index 00000000..9aa0a859 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/ops/test_repeat_interleave.py @@ -0,0 +1,141 @@ +import time +from typing import Tuple + +import numpy as np +import pytest +import torch + +from sglang.srt.models.utils import compute_cu_seqlens_from_grid_numpy as cpu_numpy_impl + + +def torch_ref_impl(grid_thw: torch.Tensor) -> torch.Tensor: + """ + Pure PyTorch implementation of cu_seqlens computation. + Assumes grid_thw is already on the correct device (CPU here). + Shape: [T, 3], columns: [repeat_count, H, W] + """ + cu_seqlens = torch.repeat_interleave( + grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0] + ).cumsum(dim=0) + cu_seqlens = torch.cat( + [ + torch.zeros(1, dtype=torch.int32, device=cu_seqlens.device), + cu_seqlens.to(torch.int32), + ] + ) + return cu_seqlens + + +def benchmark_once(fn, grid_thw, iters: int = 1000): + """ + Run a function `fn` on the same input `grid_thw` for `iters` times + and measure total elapsed time. + """ + start = time.perf_counter() + for _ in range(iters): + out = fn(grid_thw) + end = time.perf_counter() + return (end - start), out + + +# (T, repeat_min, repeat_max) +GRID_TEST_CONFIGS: list[Tuple[int, int, int]] = [ + (16, 1, 4), # small T, small repeat counts + (128, 0, 4), # allow repeat=0 to test edge cases + (512, 1, 8), + (1024, 1, 16), +] + +NUM_CASES_PER_CONFIG = 10 + + +def _generate_random_grid(T: int, repeat_min: int, repeat_max: int) -> torch.Tensor: + """ + grid_thw: [T, 3] + col0: repeat count + col1, col2: arbitrary positive integers (here 1..16) + """ + repeats = torch.randint(repeat_min, repeat_max + 1, (T, 1), dtype=torch.int32) + th = torch.randint(1, 17, (T, 1), dtype=torch.int32) + tw = torch.randint(1, 17, (T, 1), dtype=torch.int32) + grid_thw = torch.cat([repeats, th, tw], dim=1) + return grid_thw + + +class TestRepeatInterleave: + @classmethod + def setup_class(cls): + torch.set_num_threads(1) + + def setup_method(self, method): + torch.manual_seed(0) + np.random.seed(0) + + @pytest.mark.parametrize( + "T,repeat_min,repeat_max", + GRID_TEST_CONFIGS, + ) + @pytest.mark.parametrize("case_idx", range(NUM_CASES_PER_CONFIG)) + def test_cpu_correctness_random_cases( + self, + T: int, + repeat_min: int, + repeat_max: int, + case_idx: int, + ): + torch.manual_seed(case_idx) + np.random.seed(case_idx) + + grid_thw = _generate_random_grid(T, repeat_min, repeat_max) + + grid_clone = grid_thw.clone() + + out_torch = torch_ref_impl(grid_thw) + out_numpy = cpu_numpy_impl(grid_thw) + + assert torch.equal(grid_thw, grid_clone), "Function modified input grid_thw!" + + assert ( + out_torch.shape == out_numpy.shape + ), f"Shape mismatch: torch={out_torch.shape}, numpy={out_numpy.shape}" + + assert ( + out_torch.dtype == torch.int32 + ), f"Unexpected torch dtype: {out_torch.dtype}" + assert ( + out_numpy.dtype == torch.int32 + ), f"Unexpected numpy impl dtype: {out_numpy.dtype}" + + if not torch.equal(out_torch.cpu(), out_numpy.cpu()): + diff_idx = (out_torch.cpu() != out_numpy.cpu()).nonzero(as_tuple=False) + idx0 = diff_idx[0].item() + pytest.fail( + f"Value mismatch, T={T}, case_idx={case_idx}, first differing index={idx0}, " + f"torch={out_torch[idx0].item()}, " + f"numpy={out_numpy[idx0].item()}" + ) + + def test_zero_repeat_edge_case(self): + T = 4 + grid_thw = torch.tensor( + [ + [0, 4, 4], + [1, 2, 3], # 6 + [2, 1, 5], # 5, 5 + [0, 7, 7], # 0 + ], + dtype=torch.int32, + ) + + grid_clone = grid_thw.clone() + + out_torch = torch_ref_impl(grid_thw) + out_numpy = cpu_numpy_impl(grid_thw) + + assert torch.equal( + grid_thw, grid_clone + ), "Function modified input grid_thw with zero repeats!" + + assert torch.equal( + out_torch.cpu(), out_numpy.cpu() + ), f"Zero-repeat case mismatch: torch={out_torch}, numpy={out_numpy}" diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/parse_results.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/parse_results.py new file mode 100644 index 00000000..f552739f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/parse_results.py @@ -0,0 +1,57 @@ +import argparse +import json +import os + +import pandas as pd +from tabulate import tabulate + +# Parse command-line arguments +parser = argparse.ArgumentParser(description="Parse JSONL benchmark and summarize.") +parser.add_argument("input_file", type=str, help="Path to input JSONL file") +parser.add_argument( + "--md", + action="store_true", + help="If set, print the summary table in Markdown format (GitHub style)", +) +args = parser.parse_args() + +input_file = args.input_file +base_name = os.path.splitext(os.path.basename(input_file))[0] +output_file = f"{base_name}_summary.csv" + +fields = [ + "max_concurrency", + "input_throughput", + "output_throughput", + "mean_ttft_ms", + "median_ttft_ms", + "p99_ttft_ms", + "mean_tpot_ms", + "median_tpot_ms", + "p99_tpot_ms", +] + +# Read JSONL and parse +results = [] +with open(input_file, "r") as f: + for line in f: + data = json.loads(line) + row = {field: data.get(field, None) for field in fields} + max_conc = data.get("max_concurrency") + out_tp = data.get("output_throughput") + row["per_user_throughput"] = out_tp / max_conc if max_conc else None + results.append(row) + +# Convert to DataFrame +df = pd.DataFrame(results) + +# Save to CSV +df.to_csv(output_file, index=False) +print(f"\nSaved summary to: {output_file}\n") + +if args.md: + # Print Markdown table + print(tabulate(df, headers="keys", tablefmt="github", floatfmt=".3f")) +else: + # Print ASCII table + print(tabulate(df, headers="keys", tablefmt="grid", floatfmt=".3f")) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_autoround.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_autoround.py new file mode 100644 index 00000000..18e5ad66 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_autoround.py @@ -0,0 +1,62 @@ +""" +Usage: +python3 -m unittest test_autoround.TestAutoRound.test_mmlu +""" + +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_AUTOROUND_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestAutoRound(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + + @classmethod + def tearDownClass(cls): + pass + + def test_mmlu(self): + device = "auto" + for model in DEFAULT_AUTOROUND_MODEL_NAME_FOR_TEST: + with self.subTest(model=model): + print(f"\n[INFO] Launching server for model: {model}") + process = popen_launch_server( + model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--trust-remote-code", "--quantization", "auto-round"], + device=device, + ) + + try: + args = SimpleNamespace( + base_url=self.base_url, + model=model, + eval_name="mmlu", + num_examples=32, + num_threads=32, + device=device, + ) + metrics = run_eval(args) + if "Llama" in model: + self.assertGreaterEqual(metrics["score"], 0.6) + else: + self.assertGreaterEqual(metrics["score"], 0.26) + finally: + kill_process_tree(process.pid) + print(f"[INFO] Server for {model} stopped.") + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_awq.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_awq.py new file mode 100644 index 00000000..63254935 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_awq.py @@ -0,0 +1,78 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_AWQ_MOE_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestAWQ(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_AWQ_MOE_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--trust-remote-code"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.64) + + +class TestAWQMarlinBfloat16(CustomTestCase): + """ + Verify that the model can be loaded with bfloat16 dtype and awq_marlin quantization + """ + + @classmethod + def setUpClass(cls): + cls.model = "QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--dtype", "bfloat16", "--quantization", "awq_marlin"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.85) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_awq_dequant.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_awq_dequant.py new file mode 100644 index 00000000..ec1f2b16 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_awq_dequant.py @@ -0,0 +1,175 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/main/tests/kernels/quantization/test_awq_triton.py +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +unittest version of the AWQ Triton kernel tests. + +Run with: + python -m unittest test_awq_dequant.py +""" +import unittest + +import torch + +from sglang.srt.layers.quantization.awq_triton import ( + AWQ_TRITON_SUPPORTED_GROUP_SIZES, + awq_dequantize_triton, + awq_gemm_triton, +) +from sglang.test.test_utils import CustomTestCase + +device = "cuda" + + +def reverse_awq_order(t: torch.Tensor) -> torch.Tensor: + bits = 4 + AWQ_REVERSE_ORDER = [0, 4, 1, 5, 2, 6, 3, 7] + idx = torch.arange(t.shape[-1], dtype=torch.int32, device=t.device) + idx = idx.view(-1, 32 // bits)[:, AWQ_REVERSE_ORDER].view(-1) + return (t[:, idx] & 0xF).contiguous() + + +def awq_dequantize_torch( + qweight: torch.Tensor, + scales: torch.Tensor, + qzeros: torch.Tensor, + group_size: int, +) -> torch.Tensor: + if group_size == -1: + group_size = qweight.shape[0] + + bits = 4 + shifts = torch.arange(0, 32, bits, device=qzeros.device) + + iweights = torch.bitwise_right_shift(qweight[:, :, None], shifts[None, None, :]).to( + torch.int8 + ) + iweights = reverse_awq_order(iweights.view(iweights.shape[0], -1)) + + zeros = torch.bitwise_right_shift(qzeros[:, :, None], shifts[None, None, :]).to( + torch.int8 + ) + zeros = reverse_awq_order(zeros.view(qzeros.shape[0], -1)) + + iweights = torch.bitwise_and(iweights, (2**bits) - 1) + zeros = torch.bitwise_and(zeros, (2**bits) - 1) + + scales = scales.repeat_interleave(group_size, dim=0) + zeros = zeros.repeat_interleave(group_size, dim=0) + return (iweights - zeros) * scales + + +class TestAWQTriton(CustomTestCase): + def test_dequantize(self): + rows_list = [3584, 18944, 128, 256, 512, 1024] + cols_list = [448, 576, 4736, 16, 32, 64, 128] + + for qweight_rows in rows_list: + for qweight_cols in cols_list: + for group_size in AWQ_TRITON_SUPPORTED_GROUP_SIZES: + with self.subTest( + rows=qweight_rows, cols=qweight_cols, g=group_size + ): + self._run_dequant_case( + qweight_rows=qweight_rows, + qweight_cols=qweight_cols, + group_size=group_size, + ) + + def _run_dequant_case(self, qweight_rows, qweight_cols, group_size): + if group_size == -1: + group_size = qweight_rows + + torch.manual_seed(0) + + qweight = torch.randint( + 0, + torch.iinfo(torch.int32).max, + (qweight_rows, qweight_cols), + dtype=torch.int32, + device=device, + ) + scales = torch.rand( + qweight_rows // group_size, + qweight_cols * 8, + dtype=torch.float16, + device=device, + ) + zeros = torch.randint( + 0, + torch.iinfo(torch.int32).max, + (qweight_rows // group_size, qweight_cols), + dtype=torch.int32, + device=device, + ) + + ref = awq_dequantize_torch(qweight, scales, zeros, group_size) + tri = awq_dequantize_triton(qweight, scales, zeros) + + # sanity + self.assertFalse(torch.any(torch.isinf(tri)) or torch.any(torch.isnan(tri))) + torch.testing.assert_close(ref, tri) + + # GEMM + def test_gemm(self): + N_list = [1, 2, 4, 8, 14, 17, 23, 32] + K_list = [128] + M_list = [16, 24, 32] + splitK_list = [1, 8] + + for N in N_list: + for K in K_list: + for M in M_list: + for group_size in AWQ_TRITON_SUPPORTED_GROUP_SIZES: + for splitK in splitK_list: + with self.subTest(N=N, K=K, M=M, g=group_size, sk=splitK): + self._run_gemm_case( + N=N, + K=K, + M=M, + group_size=group_size, + splitK=splitK, + ) + + def _run_gemm_case(self, N, K, M, group_size, splitK): + if group_size == -1: + group_size = K + + torch.manual_seed(0) + + x = torch.rand((N, K), dtype=torch.float32, device=device) + qweight = torch.randint( + 0, + torch.iinfo(torch.int32).max, + (K, M // 8), + dtype=torch.int32, + device=device, + ) + qzeros = torch.randint( + 0, + torch.iinfo(torch.int32).max, + (K // group_size, M // 8), + dtype=torch.int32, + device=device, + ) + scales = torch.rand((K // group_size, M), dtype=torch.float32, device=device) + + tri_out = awq_gemm_triton(x, qweight, scales, qzeros, splitK) + + self.assertFalse( + torch.any(torch.isinf(tri_out)) or torch.any(torch.isnan(tri_out)) + ) + + # dequantize & compare + w_deq = awq_dequantize_triton(qweight, scales, qzeros) + ref_out = torch.matmul(x, w_deq) + + self.assertFalse( + torch.any(torch.isinf(ref_out)) or torch.any(torch.isnan(ref_out)) + ) + + torch.testing.assert_close(tri_out.cpu(), ref_out.cpu(), atol=1e-1, rtol=1e-1) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_block_int8.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_block_int8.py new file mode 100644 index 00000000..1dc1edad --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_block_int8.py @@ -0,0 +1,230 @@ +import itertools +import unittest + +import torch + +from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import fused_moe +from sglang.srt.layers.moe.topk import TopKConfig, select_experts +from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler +from sglang.test.test_utils import CustomTestCase + + +# For test +def native_per_token_group_quant_int8(x, group_size, eps=1e-10, dtype=torch.int8): + """Function to perform per-token-group quantization on an input tensor `x` using native torch. + + It converts the tensor values into float8 values and returns the + quantized tensor along with the scaling factor used for quantization. + Note that only `torch.float8_e4m3fn` is supported for now. + """ + assert ( + x.shape[-1] % group_size == 0 + ), "the last dimension of `x` cannot be divisible by `group_size`" + assert x.is_contiguous(), "`x` is not contiguous" + + iinfo = torch.iinfo(dtype) + int8_min = iinfo.min + int8_max = iinfo.max + + x_ = x.reshape(x.numel() // group_size, group_size) + amax = x_.abs().max(dim=-1, keepdim=True)[0].clamp(min=eps).to(torch.float32) + x_s = amax / int8_max + x_q = (x_ / x_s).clamp(min=int8_min, max=int8_max).to(dtype) + x_q = x_q.reshape(x.shape) + x_s = x_s.reshape(x.shape[:-1] + (x.shape[-1] // group_size,)) + + return x_q, x_s + + +# For test +def native_w8a8_block_int8_matmul(A, B, As, Bs, block_size, output_dtype=torch.float16): + """This function performs matrix multiplication with block-wise quantization using native torch. + + It takes two input tensors `A` and `B` with scales `As` and `Bs`. + The output is returned in the specified `output_dtype`. + """ + + A = A.to(torch.float32) + B = B.to(torch.float32) + assert A.shape[-1] == B.shape[-1] + assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2 + assert len(block_size) == 2 + block_n, block_k = block_size[0], block_size[1] + assert (A.shape[-1] + block_k - 1) // block_k == As.shape[-1] + assert A.shape[:-1] == As.shape[:-1] + + M = A.numel() // A.shape[-1] + N, K = B.shape + origin_C_shape = A.shape[:-1] + (N,) + A = A.reshape(M, A.shape[-1]) + As = As.reshape(M, As.shape[-1]) + n_tiles = (N + block_n - 1) // block_n + k_tiles = (K + block_k - 1) // block_k + assert n_tiles == Bs.shape[0] + assert k_tiles == Bs.shape[1] + + C_shape = (M, N) + C = torch.zeros(C_shape, dtype=torch.float32, device=A.device) + + A_tiles = [A[:, i * block_k : min((i + 1) * block_k, K)] for i in range(k_tiles)] + B_tiles = [ + [ + B[ + j * block_n : min((j + 1) * block_n, N), + i * block_k : min((i + 1) * block_k, K), + ] + for i in range(k_tiles) + ] + for j in range(n_tiles) + ] + C_tiles = [C[:, j * block_n : min((j + 1) * block_n, N)] for j in range(n_tiles)] + As_tiles = [As[:, i : i + 1] for i in range(k_tiles)] + + for i in range(k_tiles): + for j in range(n_tiles): + a = A_tiles[i] + b = B_tiles[j][i] + c = C_tiles[j] + s = As_tiles[i] * Bs[j][i] + c[:, :] += torch.matmul(a, b.t()) * s + + C = C.reshape(origin_C_shape).to(output_dtype) + return C + + +# For test +def torch_w8a8_block_int8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape): + """This function performs fused moe with block-wise quantization using native torch.""" + + set_global_server_args_for_scheduler(ServerArgs(model_path="dummy")) + + B, D = a.shape + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + + _, block_k = block_shape[0], block_shape[1] + a_q, a_s = native_per_token_group_quant_int8(a, block_k) + for i in range(w1.shape[0]): + mask = topk_ids == i + if mask.sum(): + inter_out = native_w8a8_block_int8_matmul( + a_q[mask], w1[i], a_s[mask], w1_s[i], block_shape, output_dtype=a.dtype + ) + act_out = SiluAndMul().forward_native(inter_out) + act_out_q, act_out_s = native_per_token_group_quant_int8(act_out, block_k) + act_out = act_out.to(torch.float32) + out[mask] = native_w8a8_block_int8_matmul( + act_out_q, w2[i], act_out_s, w2_s[i], block_shape, output_dtype=a.dtype + ) + return ( + out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype) + ).sum(dim=1) + + +class TestW8A8BlockINT8FusedMoE(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16] + M = [1, 33, 64, 222] + N = [128, 1024] + K = [256, 4096] + E = [8, 24] + TOP_KS = [2, 6] + # BLOCK_SIZE = [[64, 64], [64, 128], [128, 64], [128, 128]] + BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _w8a8_block_int8_fused_moe(self, M, N, K, E, topk, block_size, dtype, seed): + torch.manual_seed(seed) + # NOTE(HandH1998): to avoid overflow when out_dtype = torch.half + factor_for_scale = 1e-2 + int8_info = torch.iinfo(torch.int8) + int8_max, int8_min = int8_info.max, int8_info.min + + a = torch.randn((M, K), dtype=dtype) / 10 + + w1_fp32 = (torch.rand((E, 2 * N, K), dtype=torch.float32) - 0.5) * 2 * int8_max + w1 = w1_fp32.clamp(min=int8_min, max=int8_max).to(torch.int8) + + w2_fp32 = (torch.rand((E, K, N), dtype=torch.float32) - 0.5) * 2 * int8_max + w2 = w2_fp32.clamp(min=int8_min, max=int8_max).to(torch.int8) + + block_n, block_k = block_size[0], block_size[1] + n_tiles_w1 = (2 * N + block_n - 1) // block_n + n_tiles_w2 = (K + block_n - 1) // block_n + k_tiles_w1 = (K + block_k - 1) // block_k + k_tiles_w2 = (N + block_k - 1) // block_k + + w1_s = ( + torch.rand((E, n_tiles_w1, k_tiles_w1), dtype=torch.float32) + * factor_for_scale + ) + w2_s = ( + torch.rand((E, n_tiles_w2, k_tiles_w2), dtype=torch.float32) + * factor_for_scale + ) + + score = torch.randn((M, E), dtype=dtype) + + topk_output = select_experts( + hidden_states=a, + router_logits=score, + topk_config=TopKConfig(top_k=topk, renormalize=False), + ) + + with torch.inference_mode(): + ref_out = torch_w8a8_block_int8_moe( + a, w1, w2, w1_s, w2_s, score, topk, block_size + ) + out = fused_moe( + a, + w1, + w2, + topk_output, + use_int8_w8a8=True, + w1_scale=w1_s, + w2_scale=w2_s, + block_shape=block_size, + ) + + self.assertTrue( + torch.mean(torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))) + / torch.mean(torch.abs(ref_out.to(torch.float32))) + < 0.02 + ) + + def test_w8a8_block_int8_fused_moe(self): + for params in itertools.product( + self.M, + self.N, + self.K, + self.E, + self.TOP_KS, + self.BLOCK_SIZE, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + E=params[3], + topk=params[4], + block_size=params[5], + dtype=params[6], + seed=params[7], + ): + self._w8a8_block_int8_fused_moe(*params) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_fp8_kernel.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_fp8_kernel.py new file mode 100644 index 00000000..42502277 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_fp8_kernel.py @@ -0,0 +1,126 @@ +import unittest + +import torch + +from sglang.srt.layers.quantization.fp8_kernel import ( + per_token_group_quant_fp8, + w8a8_block_fp8_matmul, +) +from sglang.test.test_utils import CustomTestCase + + +class TestFP8Base(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.M = 256 + # test non-aligned + cls.N = 1024 + 64 + cls.K = 512 + cls.group_size = 128 + cls.quant_type = torch.float8_e4m3fn + cls.output_type = torch.bfloat16 + + @staticmethod + def _make_A(M, K, group_size, out_dtype): + quant_A = torch.rand( + M, K // group_size, group_size, dtype=torch.float32, device="cuda" + ) + # -1 ~ 1 + quant_A = quant_A * 2 - 1 + # scaling abs max to fmax + finfo = torch.finfo(out_dtype) + fmax = finfo.max + scaling = fmax / quant_A.abs().amax(-1, keepdim=True) + quant_A *= scaling + quant_A = quant_A.to(out_dtype).to(torch.float32) + + # create scale and A + scale = torch.rand(M, K // group_size, dtype=torch.float32, device="cuda") + scale /= fmax + A = quant_A * scale[..., None] + + A = A.reshape(M, K) + quant_A = quant_A.reshape(M, K).to(out_dtype) + return A, quant_A, scale + + @staticmethod + def _make_B(K, N, group_size, out_dtype): + def _aligned_size(a, b): + return (a + b - 1) // b * b + + K_aligned = _aligned_size(K, group_size) + N_aligned = _aligned_size(N, group_size) + + quant_B = torch.rand( + K_aligned // group_size, + group_size, + N_aligned // group_size, + group_size, + dtype=torch.float32, + device="cuda", + ) + quant_B = quant_B * 2 - 1 + + # scaling abs max to fmax + finfo = torch.finfo(out_dtype) + fmax = finfo.max + scaling = fmax / quant_B.abs().amax((1, 3), keepdim=True) + quant_B *= scaling + quant_B = quant_B.to(out_dtype).to(torch.float32) + + scale = torch.rand( + K_aligned // group_size, + 1, + N_aligned // group_size, + 1, + dtype=torch.float32, + device="cuda", + ) + scale /= fmax + + B = quant_B * scale + + B = B.reshape(K_aligned, N_aligned)[:K, :N] + quant_B = quant_B.reshape(K_aligned, N_aligned).to(out_dtype)[:K, :N] + scale = scale.reshape(K_aligned // group_size, N_aligned // group_size) + return B, quant_B, scale + + +class TestPerTokenGroupQuantFP8(TestFP8Base): + def test_per_token_group_quant_fp8(self): + if torch.cuda.get_device_capability()[0] < 9: + return + A, A_quant_gt, scale_gt = self._make_A( + M=self.M, K=self.K, group_size=self.group_size, out_dtype=self.quant_type + ) + A_quant, scale = per_token_group_quant_fp8(x=A, group_size=self.group_size) + torch.testing.assert_close(scale, scale_gt) + diff = (A_quant.to(torch.float16) - A_quant_gt.to(torch.float16)).abs() + diff_count = (diff > 1e-5).count_nonzero() + assert diff_count / diff.numel() < 1e-4 + + +class TestW8A8BlockFP8Matmul(TestFP8Base): + def test_w8a8_block_fp8_matmul(self): + if torch.cuda.get_device_capability()[0] < 9: + return + A, A_quant_gt, A_scale_gt = self._make_A( + M=self.M, K=self.K, group_size=self.group_size, out_dtype=self.quant_type + ) + B, B_quant_gt, B_scale_gt = self._make_B( + K=self.K, N=self.N, group_size=self.group_size, out_dtype=self.quant_type + ) + C_gt = A.to(self.output_type) @ B.to(self.output_type) + C = w8a8_block_fp8_matmul( + A=A_quant_gt, + B=B_quant_gt.T.contiguous(), + As=A_scale_gt, + Bs=B_scale_gt.T.contiguous(), + block_size=[128, 128], + output_dtype=self.output_type, + ) + torch.testing.assert_close(C, C_gt, atol=0.5, rtol=1e-4) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_fused_rms_fp8_group_quant.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_fused_rms_fp8_group_quant.py new file mode 100644 index 00000000..443c590e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_fused_rms_fp8_group_quant.py @@ -0,0 +1,148 @@ +# test/srt/quant/test_fused_rms_fp8_group_quant.py +import itertools +import unittest + +import torch +import torch.nn.functional as F + +from sglang.test.test_utils import CustomTestCase + + +def _fp8_available() -> bool: + # requirement：1) GPU；2) ROCm；3) torch support float8_e4m3fn + if not torch.cuda.is_available(): + return False + if getattr(torch.version, "hip", None) is None: + return False + return hasattr(torch, "float8_e4m3fn") + + +def _rmsnorm(x, weight, eps=1e-6): + # row-wise RMSNorm + row_norm = (x * x).sum(dim=-1) + norm = torch.rsqrt(row_norm / x.shape[1] + eps) + return x * norm[:, None] * weight[None, :] + + +def _per_token_fp8_group_quant(x, dtype_quant, group_size=128): + """per token、group-size quant, return (quantized, scale)。""" + DTYPE_MAX = torch.finfo(dtype_quant).max + M, N = x.shape + + pad = (group_size - (N % group_size)) % group_size + if pad: + x_reshape = F.pad(x, (0, pad, 0, 0), "constant", 0) + else: + x_reshape = x + + G = (N + group_size - 1) // group_size + x_reshape = x_reshape.view(M, G, group_size).to(torch.float32) + x_max = torch.max(torch.abs(x_reshape), dim=-1, keepdim=True)[0].clamp_min_(1e-10) + x_scale = x_max / DTYPE_MAX + inv = 1.0 / x_scale + + x_q = torch.clamp(x_reshape * inv, -DTYPE_MAX, DTYPE_MAX).to(dtype_quant) + x_q = x_q.view(M, G * group_size) + if pad: + x_q = x_q[:, :N] + x_scale = x_scale.squeeze(-1) # [M, G] + return x_q, x_scale + + +def _upcast_fp8_group(x_q, x_s, out_dtype=torch.float32, group_size=128): + """unqaunt""" + M, N = x_q.shape + G = (N + group_size - 1) // group_size + pad = (group_size - (N % group_size)) % group_size + + if pad: + x_q = F.pad(x_q, (0, pad, 0, 0), "constant", 0) + + x_q = x_q.view(M, G, group_size).to(torch.float32) + x = x_q * x_s.view(M, G, 1) + x = x.view(M, G * group_size)[:, :N] + return x.to(out_dtype) + + +class TestFusedRMSFP8GroupQuant(CustomTestCase): + # + DTYPES = [torch.bfloat16, torch.float16] + # (M, N1, N2) + SHAPES = [(32, 128, 7168), (128, 7168, 7168)] + GROUP_SIZE = [128] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not _fp8_available(): + raise unittest.SkipTest("Skip: ROCm/FP8 is not available") + torch.set_default_device("cuda") + + def _run_ref(self, x1, w1, eps1, x2, w2, eps2, res1, dtype_quant, group_size): + s = x1 + (res1 if res1 is not None else 0) + y1 = _rmsnorm(s, w1, eps1) + y2 = _rmsnorm(x2, w2, eps2) if x2 is not None else None + y1_q, y1_s = _per_token_fp8_group_quant(y1, dtype_quant, group_size) + return ( + (y1_q, y1_s), + y1.to(x1.dtype), + (y2.to(x1.dtype) if y2 is not None else None), + (s.to(x1.dtype) if res1 is not None else None), + ) + + def _case(self, M, N1, N2, group_size, dtype, seed): + torch.manual_seed(seed) + fp8 = torch.float8_e4m3fn + device = "cuda" + + x1 = torch.randn(M, N1, dtype=dtype, device=device) / 10 + x2 = torch.randn(M, N2, dtype=dtype, device=device) / 10 + w1 = torch.ones(N1, dtype=torch.float32, device=device) + w2 = torch.ones(N2, dtype=torch.float32, device=device) + res1 = torch.randn(M, N1, dtype=dtype, device=device) / 10 + + # ref + (y1_q_ref, y1_s_ref), y1_ref, y2_ref, s_ref = self._run_ref( + x1, w1, 1e-6, x2, w2, 1e-6, res1, fp8, group_size + ) + + # be tested：aiter fused op + from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_group_quant + + (y1_q, y1_s), y1, y2, s = fused_rms_fp8_group_quant( + x1, + w1, + 1e-6, + inp2=x2, + inp2_weight=w2, + inp2_epsilon=1e-6, + group_size=group_size, + dtype_quant=fp8, + res1=res1, + output_unquantized_inp1=True, # get unqaunt y1 + ) + + torch.testing.assert_close(y1, y1_ref, atol=0.1, rtol=0.1) + torch.testing.assert_close(y2, y2_ref, atol=0.1, rtol=0.1) + torch.testing.assert_close(s, s_ref, atol=0.1, rtol=0.1) + + # check unquant + y1_up_ref = _upcast_fp8_group( + y1_q_ref, y1_s_ref, out_dtype=torch.float32, group_size=group_size + ) + y1_up = _upcast_fp8_group( + y1_q, y1_s, out_dtype=torch.float32, group_size=group_size + ) + torch.testing.assert_close(y1_up, y1_up_ref, atol=0.1, rtol=0.1) + + def test_fused_rms_fp8_group_quant(self): + for params in itertools.product( + self.SHAPES, self.GROUP_SIZE, self.DTYPES, self.SEEDS + ): + (M, N1, N2), g, dtype, seed = params + with self.subTest(M=M, N1=N1, N2=N2, group_size=g, dtype=dtype, seed=seed): + self._case(M, N1, N2, g, dtype, seed) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_int8_kernel.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_int8_kernel.py new file mode 100644 index 00000000..31414d1c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_int8_kernel.py @@ -0,0 +1,172 @@ +import itertools +import unittest + +import torch + +from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import fused_moe +from sglang.srt.layers.moe.topk import TopKConfig, select_experts +from sglang.srt.layers.quantization.int8_kernel import per_token_quant_int8 +from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler +from sglang.test.test_utils import CustomTestCase + + +def native_w8a8_per_token_matmul(A, B, As, Bs, output_dtype=torch.float16): + """Matrix multiplication function that supports per-token input quantization and per-column weight quantization""" + A = A.to(torch.float32) + B = B.to(torch.float32) + + assert A.shape[-1] == B.shape[-1], "Dimension mismatch" + assert B.ndim == 2 and B.is_contiguous(), "B must be a 2D contiguous tensor" + + # Reshape input + M = A.numel() // A.shape[-1] + B = B.t() # Transpose weight matrix + N, K = B.shape + origin_C_shape = A.shape[:-1] + (K,) + A = A.reshape(M, N) + + # As is per-token [M, 1], Bs is per-column [1, K] + C = torch.matmul(A, B) # [M, K] + C = As * C * Bs.view(1, -1) # Broadcast per-column scale + + return C.reshape(origin_C_shape).to(output_dtype) + + +def torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, score, topk): + """This function performs fused moe with per-column int8 quantization using native torch.""" + + set_global_server_args_for_scheduler(ServerArgs(model_path="dummy")) + + B, D = a.shape + # Perform per-token quantization + a_q, a_s = per_token_quant_int8(a) + # Repeat tokens to match topk + a_q = a_q.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + # Also repeat the scale + a_s = a_s.view(B, -1, 1).repeat(1, topk, 1).reshape(-1, 1) # [B*topk, 1] + + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) + + # Calculate routing + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + # Process each expert + for i in range(w1.shape[0]): + mask = topk_ids == i + if mask.sum(): + # First MLP layer: note that a_s is now per-token + inter_out = native_w8a8_per_token_matmul( + a_q[mask], w1[i], a_s[mask], w1_s[i], output_dtype=a.dtype + ) + # Activation function + act_out = SiluAndMul().forward_native(inter_out) + # Quantize activation output with per-token + act_out_q, act_out_s = per_token_quant_int8(act_out) + + # Second MLP layer + out[mask] = native_w8a8_per_token_matmul( + act_out_q, w2[i], act_out_s, w2_s[i], output_dtype=a.dtype + ) + # Apply routing weights and sum + return ( + out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype) + ).sum(dim=1) + + +class TestW8A8Int8FusedMoE(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16] + M = [1, 33] + N = [128, 1024] + K = [256, 4096] + E = [8] + TOP_KS = [2, 6] + BLOCK_SIZE = [[64, 64], [64, 128], [128, 64], [128, 128]] + BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _w8a8_int8_fused_moe(self, M, N, K, E, topk, block_size, dtype, seed): + torch.manual_seed(seed) + # Initialize int8 quantization parameters + factor_for_scale = 1e-2 + int8_max = 127 + int8_min = -128 + + # Input tensor + # M * K + a = torch.randn((M, K), dtype=dtype) / 10 + + # Generate int8 weights + w1_fp32 = (torch.rand((E, 2 * N, K), dtype=torch.float32) - 0.5) * 2 + w1 = (w1_fp32 * int8_max).clamp(min=int8_min, max=int8_max).to(torch.int8) + + w2_fp32 = (torch.rand((E, K, N), dtype=torch.float32) - 0.5) * 2 + w2 = (w2_fp32 * int8_max).clamp(min=int8_min, max=int8_max).to(torch.int8) + + # Generate scale for each column (per-column quantization) + w1_s = torch.rand(E, 2 * N, device=w1_fp32.device) * factor_for_scale + w2_s = torch.rand(E, K, device=w2_fp32.device) * factor_for_scale + score = torch.randn((M, E), dtype=dtype) + + with torch.inference_mode(): + ref_out = torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, score, topk) + topk_output = select_experts( + hidden_states=a, + router_logits=score, + topk_config=TopKConfig(top_k=topk, renormalize=False), + ) + out = fused_moe( + a, + w1, + w2, + topk_output, + use_fp8_w8a8=False, # Not using fp8 + use_int8_w8a16=False, # Not using int8-w8a16 + use_int8_w8a8=True, # Using int8-w8a8 + per_channel_quant=True, + w1_scale=w1_s, + w2_scale=w2_s, + block_shape=None, # Not using block quantization + ) + + # Check results + self.assertTrue( + torch.mean(torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))) + / torch.mean(torch.abs(ref_out.to(torch.float32))) + < 0.05 + ) + + def test_w8a8_int8_fused_moe(self): + for params in itertools.product( + self.M, + self.N, + self.K, + self.E, + self.TOP_KS, + self.BLOCK_SIZE, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + E=params[3], + topk=params[4], + block_size=params[5], + dtype=params[6], + seed=params[7], + ): + self._w8a8_int8_fused_moe(*params) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_triton_scaled_mm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_triton_scaled_mm.py new file mode 100644 index 00000000..72a0bbf3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_triton_scaled_mm.py @@ -0,0 +1,93 @@ +import unittest +from typing import Optional + +import torch +import torch.testing + +from sglang.srt.layers.quantization.fp8_kernel import triton_scaled_mm +from sglang.test.test_utils import CustomTestCase + + +def torch_scaled_mm( + a: torch.Tensor, + b: torch.Tensor, + scale_a: torch.Tensor, + scale_b: torch.Tensor, + out_dtype: torch.dtype, + bias: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """Reference implementation using float32 for stability""" + out = torch.mm(a.to(torch.float32), b.to(torch.float32)) + out = scale_a.to(torch.float32) * out * scale_b.to(torch.float32).T + if bias is not None: + out = out + bias.to(torch.float32) + return out.to(out_dtype) + + +class TestScaledMM(CustomTestCase): + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("This test requires a CUDA device.") + torch.set_default_device("cuda") + + def _make_inputs(self, M, K, N, in_dtype): + if in_dtype == torch.int8: + a = torch.randint(-8, 8, (M, K), dtype=in_dtype, device="cuda") + b = torch.randint(-8, 8, (K, N), dtype=in_dtype, device="cuda") + else: # fp8 + a = torch.clamp( + 0.1 * torch.randn((M, K), dtype=torch.float16, device="cuda"), -0.3, 0.3 + ).to(in_dtype) + b = torch.clamp( + 0.1 * torch.randn((K, N), dtype=torch.float16, device="cuda"), -0.3, 0.3 + ).to(in_dtype) + return a, b + + def test_basic_cases(self): + """Test core functionality with reduced precision requirements""" + test_configs = [ + (32, 32, 32, torch.int8, torch.float16, False), + (64, 64, 64, torch.int8, torch.float16, True), + ] + + try: + torch.tensor([1.0], dtype=torch.float8_e4m3fn, device="cuda") + test_configs.append((32, 32, 32, torch.float8_e4m3fn, torch.float16, False)) + except: + print("FP8 not supported, skipping") + + for M, K, N, in_dtype, out_dtype, with_bias in test_configs: + with self.subTest(M=M, K=K, N=N, dtype=in_dtype, bias=with_bias): + print(f"Currently testing with in_dtype: {in_dtype}") + torch.manual_seed(42) + + input, weight = self._make_inputs(M, K, N, in_dtype) + scale_a = 0.1 + 0.05 * torch.rand( + (M, 1), dtype=torch.float32, device="cuda" + ) + scale_b = 0.1 + 0.05 * torch.rand( + (N, 1), dtype=torch.float32, device="cuda" + ) + bias = ( + 0.01 * torch.randn((M, N), dtype=out_dtype, device="cuda") + if with_bias + else None + ) + + triton_out = triton_scaled_mm( + input, weight, scale_a, scale_b, out_dtype, bias + ) + ref_out = torch_scaled_mm( + input, weight, scale_a, scale_b, out_dtype, bias + ) + + # Use relaxed tolerances + rtol = 0.15 if in_dtype == torch.int8 else 0.25 + atol = 0.1 if in_dtype == torch.int8 else 0.15 + + torch.testing.assert_close(triton_out, ref_out, rtol=rtol, atol=atol) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_w4a8_deepseek_v3.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_w4a8_deepseek_v3.py new file mode 100644 index 00000000..064986a5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_w4a8_deepseek_v3.py @@ -0,0 +1,246 @@ +import os +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_DEEPSEEK_W4AFP8_MODEL_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_amd_ci, + is_in_ci, + popen_launch_server, + try_cached_model, + write_github_step_summary, +) + + +class TestDeepseekV3W4afp8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = try_cached_model(DEFAULT_DEEPSEEK_W4AFP8_MODEL_FOR_TEST) + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code", "--tp", "8", "--ep-size", "8"] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1200, + parallel=1200, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Eval accuracy of GSM8K: {metrics=}") + + self.assertGreater(metrics["accuracy"], 0.92) + + +class TestDeepseekV3W4Afp8Mtp(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = try_cached_model(DEFAULT_DEEPSEEK_W4AFP8_MODEL_FOR_TEST) + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--tp", + "8", + "--trust-remote-code", + "--ep-size", + "8", + "--cuda-graph-bs", + "256", + "--disable-radix-cache", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "2", + "--speculative-num-draft-tokens", + "4", + ] + if not is_in_amd_ci(): + other_args += ["--mem-frac", "0.7"] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k( + self, + ): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v3 mtp)\n" + f'{metrics["accuracy"]=:.3f}\n' + f"{avg_spec_accept_length=:.2f}\n" + ) + self.assertGreater(metrics["accuracy"], 0.935) + self.assertGreater(avg_spec_accept_length, 2.9) + + +class TestDeepseekV3W4Afp8DeepepNormal(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = try_cached_model(DEFAULT_DEEPSEEK_W4AFP8_MODEL_FOR_TEST) + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--tp", + "8", + "--trust-remote-code", + "--ep-size", + "8", + "--cuda-graph-bs", + "256", + "--disable-radix-cache", + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "normal", + "--dp", + "8", + "--enable-dp-attention", + "--moe-runner-backend", + "cutlass", + ] + if not is_in_amd_ci(): + other_args += ["--mem-frac", "0.7"] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k( + self, + ): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Eval accuracy of GSM8K: {metrics=}") + + self.assertGreater(metrics["accuracy"], 0.92) + + +class TestDeepseekV3W4Afp8DeepepAutoMtp(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = try_cached_model(DEFAULT_DEEPSEEK_W4AFP8_MODEL_FOR_TEST) + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--tp", + "8", + "--trust-remote-code", + "--ep-size", + "8", + "--cuda-graph-bs", + "256", + "--disable-radix-cache", + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "auto", + "--dp", + "8", + "--enable-dp-attention", + "--moe-runner-backend", + "cutlass", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + ] + if not is_in_amd_ci(): + other_args += ["--mem-frac", "0.7"] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + env={ + **os.environ, + "SGLANG_DEEPEP_BF16_DISPATCH": "1", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "256", + }, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k( + self, + ): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Eval accuracy of GSM8K: {metrics=}") + + self.assertGreater(metrics["accuracy"], 0.92) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_w8a8_quantization.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_w8a8_quantization.py new file mode 100644 index 00000000..cef51f0f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/quant/test_w8a8_quantization.py @@ -0,0 +1,111 @@ +import time +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class BaseW8A8Test(CustomTestCase): + model: str = None + quantization: str = None + gsm8k_accuracy_threshold: float = None + throughput_threshold: float = None + + @classmethod + def setUpClass(cls): + if cls is BaseW8A8Test: + raise unittest.SkipTest("Skip base test class") + + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [] + if cls.quantization: + other_args.extend(["--quantization", cls.quantization]) + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + if cls is BaseW8A8Test: + return + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + if self.gsm8k_accuracy_threshold is None: + self.skipTest("gsm8k_accuracy_threshold not set for this test") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(metrics) + self.assertGreater(metrics["accuracy"], self.gsm8k_accuracy_threshold) + + def run_decode(self, max_new_tokens): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + }, + "ignore_eos": True, + }, + ) + return response.json() + + def test_throughput(self): + + max_tokens = 256 + tic = time.perf_counter() + res = self.run_decode(max_tokens) + tok = time.perf_counter() + print(res["text"]) + throughput = max_tokens / (tok - tic) + print(f"Throughput: {throughput} tokens/s") + self.assertGreaterEqual(throughput, self.throughput_threshold) + + +class TestW8A8Int8(BaseW8A8Test): + model = "neuralmagic/Meta-Llama-3-8B-Instruct-quantized.w8a8" + quantization = "w8a8_int8" + gsm8k_accuracy_threshold = 0.69 + throughput_threshold = 200 + + +class TestW8A8Fp8(BaseW8A8Test): + model = "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8-dynamic" + quantization = "w8a8_fp8" + gsm8k_accuracy_threshold = 0.69 + throughput_threshold = 200 + + +class TestW8A8Fp8MoE(BaseW8A8Test): + model = "RedHatAI/Qwen3-30B-A3B-FP8-dynamic" + quantization = "w8a8_fp8" + gsm8k_accuracy_threshold = 0.88 + throughput_threshold = 180 + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/rl/test_fp32_lm_head.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/rl/test_fp32_lm_head.py new file mode 100644 index 00000000..cf6dd283 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/rl/test_fp32_lm_head.py @@ -0,0 +1,111 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from sglang.srt.layers.logits_processor import LogitsProcessor +from sglang.srt.server_args import ( + ServerArgs, + get_global_server_args, + set_global_server_args_for_scheduler, +) + + +class LMHeadStub(nn.Module): + def __init__(self, vocab, hidden, dtype, device="cuda"): + super().__init__() + self.weight = nn.Parameter( + torch.randn(vocab, hidden, dtype=dtype, device=device) + ) + + +class DummyMeta: + gathered_buffer = None + next_token_logits_buffer = None + + def compute_dp_attention_metadata(self): ... + + +class TestLMHeadFP32(unittest.TestCase): + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("needs CUDA GPU") + + def _make_logprocessor(self, vocab_size, enable_fp32): + set_global_server_args_for_scheduler(ServerArgs(model_path="dummy")) + get_global_server_args().enable_dp_lm_head = False + get_global_server_args().enable_fp32_lm_head = enable_fp32 + cfg = SimpleNamespace(vocab_size=vocab_size, final_logit_softcapping=None) + return LogitsProcessor(cfg, skip_all_gather=True, logit_scale=None) + + def _run_case( + self, + hidden_state_dtype, + enable_fp32, + weights_dtype, + expected_a_dtype, + expected_b_dtype, + ): + device = "cuda" + BATCH_SIZE, HIDDEN_SIZE, VOCAB_SIZE = 2, 64, 128 + hidden_state = torch.randn( + BATCH_SIZE, HIDDEN_SIZE, dtype=hidden_state_dtype, device=device + ) + head = LMHeadStub(VOCAB_SIZE, HIDDEN_SIZE, dtype=weights_dtype, device=device) + meta = DummyMeta() + logprocessor = self._make_logprocessor(VOCAB_SIZE, enable_fp32) + + original_matmul = torch.matmul + original_linear = F.linear + + state = { + "called": False, # Whether a matmul/linear call has been intercepted yet + "operation": None, # Which operation was captured ("matmul" or "linear") + "a": None, # The dtype of the first input tensor to the operation + "b": None, # The dtype of the second input tensor to the operation + } + + def probe_matmul(a, b, *args, **kw): + if not state["called"]: + state.update(called=True, operation="matmul", a=a.dtype, b=b.dtype) + return original_matmul(a, b, *args, **kw) + + def probe_linear(x, w, bias=None): + if not state["called"]: + state.update(called=True, ooperationp="linear", a=x.dtype, b=w.dtype) + return original_linear(x, w, bias) + + with patch("torch.matmul", new=probe_matmul), patch( + "torch.nn.functional.linear", new=probe_linear + ): + logits = logprocessor._get_logits(hidden_state, head, meta) + self.assertEqual(hidden_state.dtype, hidden_state_dtype) + self.assertTrue(state["called"], "no call lm head matlmul/linear") + self.assertEqual(state["a"], expected_a_dtype) + self.assertEqual(state["b"], expected_b_dtype) + + def test_flag_true_fp16_activations(self): + self._run_case(torch.float16, True, torch.float16, torch.float32, torch.float32) + + def test_flag_true_bf16_activations(self): + self._run_case( + torch.bfloat16, True, torch.bfloat16, torch.float32, torch.float32 + ) + + def test_flag_false_fp16_path(self): + self._run_case( + torch.float16, False, torch.float16, torch.float16, torch.float16 + ) + + def test_flag_false_bf16_path(self): + self._run_case( + torch.bfloat16, False, torch.bfloat16, torch.bfloat16, torch.bfloat16 + ) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/rl/test_update_weights_from_disk.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/rl/test_update_weights_from_disk.py new file mode 100644 index 00000000..0127b98d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/rl/test_update_weights_from_disk.py @@ -0,0 +1,439 @@ +import json +import random +import time +import unittest +from concurrent.futures import ThreadPoolExecutor, as_completed + +import requests + +import sglang as sgl +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) + + +############################################################################### +# Engine Mode Tests (Single-configuration) +############################################################################### +class TestEngineUpdateWeightsFromDisk(CustomTestCase): + def setUp(self): + self.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + # Initialize the engine in offline (direct) mode. + self.engine = sgl.Engine(model_path=self.model) + + def tearDown(self): + self.engine.shutdown() + + def run_decode(self): + prompts = ["The capital of France is"] + sampling_params = {"temperature": 0, "max_new_tokens": 32} + outputs = self.engine.generate(prompts, sampling_params) + print("=" * 100) + print( + f"[Engine Mode] Prompt: {prompts[0]}\nGenerated text: {outputs[0]['text']}" + ) + return outputs[0]["text"] + + def run_update_weights(self, model_path): + ret = self.engine.update_weights_from_disk(model_path) + print(json.dumps(ret)) + return ret + + def test_update_weights(self): + origin_response = self.run_decode() + # Update weights: use new model (remove "-Instruct") + new_model_path = self.model.replace("-Instruct", "") + ret = self.run_update_weights(new_model_path) + self.assertTrue(ret[0]) # ret is a tuple; index 0 holds the success flag + + updated_response = self.run_decode() + self.assertNotEqual(origin_response[:32], updated_response[:32]) + + # Revert back to original weights + ret = self.run_update_weights(self.model) + self.assertTrue(ret[0]) + reverted_response = self.run_decode() + self.assertEqual(origin_response[:32], reverted_response[:32]) + + def test_update_weights_unexist_model(self): + origin_response = self.run_decode() + new_model_path = self.model.replace("-Instruct", "wrong") + ret = self.run_update_weights(new_model_path) + self.assertFalse(ret[0]) + updated_response = self.run_decode() + self.assertEqual(origin_response[:32], updated_response[:32]) + + +############################################################################### +# HTTP Server Mode Tests (Single-configuration) +############################################################################### +class TestServerUpdateWeightsFromDisk(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, cls.base_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode(self): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": {"temperature": 0, "max_new_tokens": 32}, + }, + ) + print("=" * 100) + print(f"[Server Mode] Generated text: {response.json()['text']}") + return response.json()["text"] + + def run_decode_random(self, max_new_tokens=32): + response = requests.post( + self.base_url + "/generate", + json={ + "text": f"Question: {random.randint(0, 100)},The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + "ignore_eos": True, + }, + }, + ) + return response.json() + + def get_model_info(self): + response = requests.get(self.base_url + "/get_model_info") + model_path = response.json()["model_path"] + print(json.dumps(response.json())) + return model_path + + def run_update_weights(self, model_path, flush_cache=True): + response = requests.post( + self.base_url + "/update_weights_from_disk", + json={ + "model_path": model_path, + "flush_cache": flush_cache, + }, + ) + ret = response.json() + return ret + + def pause_generation(self, mode): + response = requests.post( + self.base_url + "/pause_generation", + json={"mode": mode}, + ) + ret = response.json() + return ret + + def continue_generation(self): + response = requests.post( + self.base_url + "/continue_generation", + json={}, + ) + ret = response.json() + return ret + + def test_update_weights(self): + origin_model_path = self.get_model_info() + print(f"[Server Mode] origin_model_path: {origin_model_path}") + origin_response = self.run_decode() + + new_model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST.replace("-Instruct", "") + ret = self.run_update_weights(new_model_path) + self.assertTrue(ret["success"]) + + updated_model_path = self.get_model_info() + print(f"[Server Mode] updated_model_path: {updated_model_path}") + self.assertEqual(updated_model_path, new_model_path) + self.assertNotEqual(updated_model_path, origin_model_path) + + updated_response = self.run_decode() + self.assertNotEqual(origin_response[:32], updated_response[:32]) + + ret = self.run_update_weights(origin_model_path) + self.assertTrue(ret["success"]) + updated_model_path = self.get_model_info() + self.assertEqual(updated_model_path, origin_model_path) + + updated_response = self.run_decode() + self.assertEqual(origin_response[:32], updated_response[:32]) + + def test_update_weights_non_blocking(self): + origin_model_path = self.get_model_info() + print(f"[Server Mode] origin_model_path: {origin_model_path}") + + pause_generation_modes = ["in_place", "retract"] + for pause_generation_mode in pause_generation_modes: + num_requests = 32 + with ThreadPoolExecutor(num_requests) as executor: + futures = [ + executor.submit(self.run_decode_random, 1600) + for _ in range(num_requests) + ] + + # ensure the decode has been started + time.sleep(2) + + new_model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST.replace( + "-Instruct", "" + ) + ret = self.pause_generation(pause_generation_mode) + ret = self.run_update_weights( + new_model_path, flush_cache=pause_generation_mode == "retract" + ) + self.assertTrue(ret["success"]) + ret = self.continue_generation() + + for future in as_completed(futures): + self.assertNotEqual( + future.result()["meta_info"]["finish_reason"]["type"], "abort" + ) + + updated_model_path = self.get_model_info() + print(f"[Server Mode] updated_model_path: {updated_model_path}") + self.assertEqual(updated_model_path, new_model_path) + self.assertNotEqual(updated_model_path, origin_model_path) + + def test_update_weights_unexist_model(self): + origin_model_path = self.get_model_info() + print(f"[Server Mode] origin_model_path: {origin_model_path}") + origin_response = self.run_decode() + + new_model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST.replace("-Instruct", "wrong") + ret = self.run_update_weights(new_model_path) + self.assertFalse(ret["success"]) + + updated_model_path = self.get_model_info() + print(f"[Server Mode] updated_model_path: {updated_model_path}") + self.assertEqual(updated_model_path, origin_model_path) + + updated_response = self.run_decode() + self.assertEqual(origin_response[:32], updated_response[:32]) + + +class TestServerUpdateWeightsFromDiskAbortAllRequests(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--max-running-requests", 8], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode(self, max_new_tokens=32): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + "ignore_eos": True, + }, + }, + ) + return response.json() + + def get_model_info(self): + response = requests.get(self.base_url + "/get_model_info") + model_path = response.json()["model_path"] + print(json.dumps(response.json())) + return model_path + + def run_update_weights(self, model_path, abort_all_requests=False): + response = requests.post( + self.base_url + "/update_weights_from_disk", + json={ + "model_path": model_path, + "abort_all_requests": abort_all_requests, + }, + ) + ret = response.json() + print(json.dumps(ret)) + return ret + + def test_update_weights_abort_all_requests(self): + origin_model_path = self.get_model_info() + print(f"[Server Mode] origin_model_path: {origin_model_path}") + + num_requests = 32 + with ThreadPoolExecutor(num_requests) as executor: + futures = [ + executor.submit(self.run_decode, 16000) for _ in range(num_requests) + ] + + # ensure the decode has been started + time.sleep(2) + + new_model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST.replace("-Instruct", "") + ret = self.run_update_weights(new_model_path, abort_all_requests=True) + self.assertTrue(ret["success"]) + + for future in as_completed(futures): + self.assertEqual( + future.result()["meta_info"]["finish_reason"]["type"], "abort" + ) + + updated_model_path = self.get_model_info() + print(f"[Server Mode] updated_model_path: {updated_model_path}") + self.assertEqual(updated_model_path, new_model_path) + self.assertNotEqual(updated_model_path, origin_model_path) + + +############################################################################### +# Parameterized Tests for update_weights_from_disk +# Test coverage is determined based on the value of is_in_ci: +# - In a CI environment: randomly select one mode (Engine or Server) and test only with tp=1, dp=1. +# - In a non-CI environment: test both Engine and Server modes, and enumerate all combinations +# with tp and dp ranging from 1 to 2. +############################################################################### +class TestUpdateWeightsFromDiskParameterized(CustomTestCase): + def run_common_test(self, mode, tp, dp): + """ + Common test procedure for update_weights_from_disk. + For Engine mode, we instantiate the engine with tp_size=tp. + For Server mode, we launch the server with additional arguments for tp (dp is not used in server launch here). + """ + if mode == "Engine": + # Instantiate engine with additional parameter tp_size. + print(f"[Parameterized Engine] Testing with tp={tp}, dp={dp}") + engine = sgl.Engine( + model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + random_seed=42, + tp_size=tp, + # dp parameter is not explicitly used in this API. + ) + try: + origin_response = self._engine_update_weights_test(engine) + finally: + engine.shutdown() + elif mode == "Server": + print(f"[Parameterized Server] Testing with tp={tp}, dp={dp}") + # Pass additional arguments to launch the server. + base_args = ["--tp-size", str(tp)] + process = popen_launch_server( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_URL_FOR_TEST, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=base_args, + ) + try: + origin_response = self._server_update_weights_test(DEFAULT_URL_FOR_TEST) + finally: + kill_process_tree(process.pid) + else: + raise ValueError(f"Unknown mode: {mode}") + + def _engine_update_weights_test(self, engine): + # Run the update weights test on the given engine instance. + def run_decode(): + prompts = ["The capital of France is"] + sampling_params = {"temperature": 0, "max_new_tokens": 32} + outputs = engine.generate(prompts, sampling_params) + print("=" * 100) + print( + f"[Parameterized Engine] Prompt: {prompts[0]}\nGenerated text: {outputs[0]['text']}" + ) + return outputs[0]["text"] + + def run_update_weights(model_path): + ret = engine.update_weights_from_disk(model_path) + print(json.dumps(ret)) + return ret + + origin_response = run_decode() + new_model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST.replace("-Instruct", "") + ret = run_update_weights(new_model_path) + self.assertTrue(ret[0]) + updated_response = run_decode() + self.assertNotEqual(origin_response[:32], updated_response[:32]) + ret = run_update_weights(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + self.assertTrue(ret[0]) + reverted_response = run_decode() + self.assertEqual(origin_response[:32], reverted_response[:32]) + return origin_response + + def _server_update_weights_test(self, base_url): + def run_decode(): + response = requests.post( + base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": {"temperature": 0, "max_new_tokens": 32}, + }, + ) + print("=" * 100) + print(f"[Parameterized Server] Generated text: {response.json()['text']}") + return response.json()["text"] + + def get_model_info(): + response = requests.get(base_url + "/get_model_info") + model_path = response.json()["model_path"] + print(json.dumps(response.json())) + return model_path + + def run_update_weights(model_path): + response = requests.post( + base_url + "/update_weights_from_disk", + json={"model_path": model_path}, + ) + ret = response.json() + print(json.dumps(ret)) + return ret + + origin_model_path = get_model_info() + origin_response = run_decode() + new_model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST.replace("-Instruct", "") + ret = run_update_weights(new_model_path) + self.assertTrue(ret["success"]) + updated_model_path = get_model_info() + self.assertEqual(updated_model_path, new_model_path) + self.assertNotEqual(updated_model_path, origin_model_path) + updated_response = run_decode() + self.assertNotEqual(origin_response[:32], updated_response[:32]) + ret = run_update_weights(origin_model_path) + self.assertTrue(ret["success"]) + updated_model_path = get_model_info() + self.assertEqual(updated_model_path, origin_model_path) + reverted_response = run_decode() + self.assertEqual(origin_response[:32], reverted_response[:32]) + return origin_response + + def test_parameterized_update_weights(self): + if is_in_ci(): + # In CI, choose one random mode (Engine or Server) with tp=1, dp=1. + mode = random.choice(["Engine", "Server"]) + test_suits = [(1, 1, mode)] + else: + # Otherwise, test both modes and enumerate tp,dp combinations from 1 to 2. + test_suits = [] + for mode in ["Engine", "Server"]: + for tp in [1, 2]: + for dp in [1, 2]: + test_suits.append((tp, dp, mode)) + for tp, dp, mode in test_suits: + with self.subTest(mode=mode, tp=tp, dp=dp): + self.run_common_test(mode, tp, dp) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/rl/test_update_weights_from_distributed.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/rl/test_update_weights_from_distributed.py new file mode 100644 index 00000000..e4834266 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/rl/test_update_weights_from_distributed.py @@ -0,0 +1,765 @@ +"""Test distributed weight updates. + +This test suite simulates a distributed training environment to ensure +correct weight synchronization. On rank 0, the instruct model represents +pre-training weights, and the base model represents post-training weights. +The base model's weights are broadcasted to other ranks using the online +weight update API. + +On other ranks, an engine is initialized with the instruct model, and its +parameters are verified against the Hugging Face model. After updating +weights from the distributed system, post-training weights are loaded +and verified again to ensure consistency and accuracy across the +distributed setup. +""" + +import gc +import os +import random +import time +import unittest +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +import requests +import torch +import torch.multiprocessing as mp +from transformers import AutoModelForCausalLM + +import sglang as sgl +from sglang.srt.utils import init_custom_process_group +from sglang.srt.weight_sync.tensor_bucket import FlattenedTensorBucket +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) +from sglang.utils import terminate_process + +mp.set_start_method("spawn", force=True) + + +def verify_params_close(params1, params2, error_msg): + """Verify if two parameter arrays are close enough.""" + try: + assert np.allclose(np.array(params1), np.array(params2)), error_msg + except Exception as e: + print(f"Parameters not close for {error_msg}") + print("Params1:", np.array(params1)) + print("Params2:", np.array(params2)) + raise e + + +def verify_params_not_close(params1, params2, error_msg): + """Verify if two parameter arrays are different enough.""" + assert not np.allclose(np.array(params1), np.array(params2)), error_msg + + +def init_process( + rank, + world_size, + param_queue, + truncate_size, + state_dict_key_to_shape, + tp_size, + model_name, + backend, + checking_parameters, + tie_word_embeddings, + load_format, + barrier, + pause_generation_mode, +): + torch.cuda.set_device(rank) + + if rank == 0: + init_process_hf( + rank, + world_size, + param_queue, + truncate_size, + model_name, + checking_parameters, + tie_word_embeddings, + state_dict_key_to_shape, + load_format, + barrier, + ) + elif rank in [1, 2]: + init_process_sgl( + rank, + world_size, + param_queue, + truncate_size, + model_name, + checking_parameters, + tie_word_embeddings, + state_dict_key_to_shape, + backend, + tp_size, + load_format, + barrier, + pause_generation_mode, + ) + + +def init_process_hf( + rank, + world_size, + param_queue, + truncate_size, + model_name, + checking_parameters, + tie_word_embeddings, + state_dict_key_to_shape, + load_format, + barrier, +): + # These two environment variables are very important + # to avoid unexpected behaviors of CUDA and NCCL. + os.environ["NCCL_CUMEM_ENABLE"] = "0" + os.environ["NCCL_NVLS_ENABLE"] = "0" + + # Load model and get parameters + hf_instruct_model = AutoModelForCausalLM.from_pretrained( + model_name, + torch_dtype="bfloat16", + tie_word_embeddings=tie_word_embeddings, + ).to("cuda:0") + base_model_name = model_name.replace("-Instruct", "") + hf_base_model = AutoModelForCausalLM.from_pretrained( + base_model_name, + torch_dtype="bfloat16", + tie_word_embeddings=tie_word_embeddings, + ).to("cuda:0") + + hf_instruct_params = [] + hf_base_params = [] + + print("[hf] get parameter in hf instruct model and base model") + for parameter_name in checking_parameters: + hf_instruct_params.append( + hf_instruct_model.get_parameter(parameter_name)[:truncate_size] + .cpu() + .detach() + .float() + .numpy() + .tolist() + ) + hf_base_params.append( + hf_base_model.get_parameter(parameter_name)[:truncate_size] + .cpu() + .detach() + .float() + .numpy() + .tolist() + ) + + param_queue.put(("hf_instruct_params", hf_instruct_params)) + param_queue.put(("hf_base_params", hf_base_params)) + + # Init weight update group for rank 0 (the training engine in RLHF). + port = 60000 + int(os.environ.get("CUDA_VISIBLE_DEVICES", "0")[0]) * 100 + init_method = f"tcp://localhost:{port}" + print(f"[hf] {rank=} {world_size=} init custom process group. {init_method=}") + group = init_custom_process_group( + backend="nccl", + init_method=init_method, + world_size=world_size, + rank=rank, + group_name="test_parameter_update_group", + ) + torch.cuda.synchronize() + barrier.wait() + time_begin_broadcast = time.perf_counter() + + # The last parameter is lm_head.weight, which is tied + # with embed_tokens.weight. Actually, we only need + # to broadcast embed_tokens.weight once. + broadcast_parameters = list(state_dict_key_to_shape.keys()) + if tie_word_embeddings: + broadcast_parameters.remove("lm_head.weight") + + if load_format == "flattened_bucket": + named_tensors = [ + (parameter_name, hf_base_model.get_parameter(parameter_name)) + for parameter_name in broadcast_parameters + ] + bucket = FlattenedTensorBucket(named_tensors=named_tensors) + flattened_tensor = bucket.get_flattened_tensor() + torch.distributed.broadcast(flattened_tensor, src=0, group=group) + else: + # Broadcast all the weights from the training + # engine to other ranks (inference engine). + for parameter_name in broadcast_parameters: + torch.distributed.broadcast( + hf_base_model.get_parameter(parameter_name), + src=0, + group=group, + ) + torch.cuda.synchronize() + time_end_broadcast = time.perf_counter() + + # Measure the latency of broadcasting/weights update. + broadcast_time = time_end_broadcast - time_begin_broadcast + print(f"[hf] {rank=} {broadcast_time=:.3f}s") + param_queue.put(("broadcast_time", broadcast_time)) + + # Destroy process group and release related resource + torch.distributed.destroy_process_group(group) + + # Delete the huggingface models to free up memory. + del hf_instruct_model + del hf_base_model + gc.collect() + torch.cuda.empty_cache() + + +def init_process_sgl( + rank, + world_size, + param_queue, + truncate_size, + model_name, + checking_parameters, + tie_word_embeddings, + state_dict_key_to_shape, + backend, + tp_size, + load_format, + barrier, + pause_generation_mode, +): + torch.cuda.set_device(rank) + torch.cuda.synchronize() + base_gpu_id = 1 if rank == 1 else 1 + tp_size + if backend == "Engine": + print(f"[sgl] rank {rank} init engine") + engine = sgl.Engine( + model_path=model_name, + base_gpu_id=base_gpu_id, + tp_size=tp_size, + cuda_graph_max_bs=2, + ) + else: + if rank == 1: + url = DEFAULT_URL_FOR_TEST + else: + host, _, port = DEFAULT_URL_FOR_TEST.rpartition(":") + url = ":".join([host, str(int(port) + 10000)]) + + print(f"[sgl] rank {rank} init server on url: {url}") + process = popen_launch_server( + model_name, + url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=( + "--base-gpu-id", + str(base_gpu_id), + "--tp-size", + str(tp_size), + "--cuda-graph-max-bs", + 2, + ), + ) + torch.cuda.synchronize() + + # Get weights of instruct model, i.e. pre-training weights. + instruct_params = [] + for parameter_name in checking_parameters: + instruct_params.append( + engine.get_weights_by_name(parameter_name, truncate_size) + if backend == "Engine" + else requests.get( + f"{url}/get_weights_by_name", + json={"name": parameter_name, "truncate_size": truncate_size}, + ).json() + ) + + param_queue.put((f"sgl_dp_{rank}_instruct_params", instruct_params)) + + port = 60000 + int(os.environ.get("CUDA_VISIBLE_DEVICES", "0")[0]) * 100 + + # Init weight update group with the training engine. + if backend == "Engine": + engine.init_weights_update_group( + master_address="localhost", + master_port=str(port), + rank_offset=base_gpu_id, + world_size=world_size, + group_name="test_parameter_update_group", + backend="nccl", + ) + else: + requests.post( + f"{url}/init_weights_update_group", + json={ + "master_address": "localhost", + "master_port": str(port), + "rank_offset": base_gpu_id, + "world_size": world_size, + "group_name": "test_parameter_update_group", + "backend": "nccl", + }, + ) + + if pause_generation_mode in ["in_place", "retract"]: + + def run_decode(max_new_tokens=32): + response = requests.post( + url + "/generate", + json={ + "text": f"Question: {random.randint(0, 100)},The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + "ignore_eos": True, + }, + }, + ) + return response.json() + + with ThreadPoolExecutor(32) as executor: + futures = [executor.submit(run_decode, 1000) for _ in range(32)] + time.sleep(2) + + # The last parameter is lm_head.weight, which is tied + # with embed_tokens.weight. Actually, we only need + # to update embed_tokens.weight once. + tie_word_embeddings = ( + True if model_name == DEFAULT_SMALL_MODEL_NAME_FOR_TEST else False + ) + update_parameters = list(state_dict_key_to_shape.keys()) + if tie_word_embeddings: + update_parameters.remove("lm_head.weight") + + # Get weights from the training engine and update the inference engine. + names = [parameter_name for parameter_name in update_parameters] + dtypes = [torch.bfloat16 if backend == "Engine" else "bfloat16"] * len(names) + shapes = [state_dict_key_to_shape[parameter_name] for parameter_name in names] + + if pause_generation_mode in ["in_place", "retract"]: + requests.post( + url + "/pause_generation", + json={"mode": pause_generation_mode}, + ) + torch.cuda.synchronize() + barrier.wait() + time_begin_update = time.perf_counter() + if backend == "Engine": + engine.update_weights_from_distributed( + names, + dtypes=dtypes, + shapes=shapes, + group_name="test_parameter_update_group", + load_format=load_format, + ) + else: + requests.post( + f"{url}/update_weights_from_distributed", + json={ + "names": names, + "dtypes": dtypes, + "shapes": shapes, + "group_name": "test_parameter_update_group", + "load_format": load_format, + "flush_cache": not (pause_generation_mode == "in_place"), + }, + ) + torch.cuda.synchronize() + time_end_update = time.perf_counter() + if pause_generation_mode in ["in_place", "retract"]: + requests.post( + url + "/continue_generation", + json={}, + ) + + # discard unfinished requests to save test overhead + time.sleep(2) + requests.post( + url + "/pause_generation", + json={"mode": "abort"}, + ) + + # Measure the latency of broadcast/weights update. + update_time = time_end_update - time_begin_update + print( + f"[sgl] fully update model_name {model_name} rank {rank} parameter from distributed time: {update_time:.3f}s" + ) + param_queue.put((f"update_sgl_dp_{rank}_time", update_time)) + + # Get the weights of post-training model after weights update for correctness check. + base_params = [] + for parameter_name in checking_parameters: + if backend == "Engine": + base_params.append( + engine.get_weights_by_name(parameter_name, truncate_size) + ) + else: + base_params.append( + requests.get( + f"{url}/get_weights_by_name", + json={ + "name": parameter_name, + "truncate_size": truncate_size, + }, + ).json() + ) + param_queue.put((f"sgl_dp_{rank}_base_params", base_params)) + + if backend == "Engine": + success, _ = engine.destroy_weights_update_group( + group_name="test_parameter_update_group", + ) + assert success is True + else: + response = requests.post( + f"{url}/destroy_weights_update_group", + json={ + "group_name": "test_parameter_update_group", + }, + ) + assert response.status_code == 200 + + # Shutdown the engine or terminate the server process. + if backend == "Engine": + engine.shutdown() + else: + terminate_process(process) + + +def assert_tied_weights(params_list, message, should_be_tied): + for params in params_list: + if should_be_tied: + assert np.allclose(params[0], params[-1]), message + else: + assert not np.allclose(params[0], params[-1]), message + + +def test_update_weights_from_distributed( + tp_size, + dp_size, + model_name, + backend, + state_dict_key_to_shape, + truncate_size, + checking_parameters, + load_format=None, + pause_generation_mode=None, +): + tie_word_embeddings = ( + True if model_name == DEFAULT_SMALL_MODEL_NAME_FOR_TEST else False + ) + + print( + f"Testing model: {model_name} tp_size: {tp_size}, dp_size: {dp_size} backend: {backend}" + ) + param_queue = mp.Queue() + results = {} + barrier = mp.Barrier(1 + dp_size) + + context = mp.spawn( + init_process, + args=( + 1 + tp_size * dp_size, + param_queue, + truncate_size, + state_dict_key_to_shape, + tp_size, + model_name, + backend, + checking_parameters, + tie_word_embeddings, + load_format, + barrier, + pause_generation_mode, + ), + nprocs=1 + dp_size, + join=False, + ) + + while len(results) < 3 * (1 + dp_size): + try: + key, value = param_queue.get(timeout=5) + results[key] = value + except Exception as e: + if all(not p.is_alive() for p in context.processes): + break + + context.join() + + if len(results) != 3 * (1 + dp_size): + raise RuntimeError( + f"Expected {3 * (1 + dp_size)} parameters but got {len(results)}" + ) + + params = { + "hf_instruct": results.get("hf_instruct_params"), + "hf_base": results.get("hf_base_params"), + "sgl_dp_1_instruct": results.get("sgl_dp_1_instruct_params"), + "sgl_dp_1_base": results.get("sgl_dp_1_base_params"), + "broadcast_time": results.get("broadcast_time"), + "update_sgl_dp_1_time": results.get("update_sgl_dp_1_time"), + } + + if dp_size == 2: + dp2_params = { + "sgl_dp_2_instruct": results.get("sgl_dp_2_instruct_params"), + "sgl_dp_2_base": results.get("sgl_dp_2_base_params"), + "update_sgl_dp_2_time": results.get("update_sgl_dp_2_time"), + } + assert all(v is not None for v in dp2_params.values()) + params.update(dp2_params) + + # Check the correctness of weights update by verifying + # the weights of instruct model and base model. + for i in range(len(params["hf_instruct"])): + verify_params_close( + params["hf_instruct"][i], + params["sgl_dp_1_instruct"][i], + f"sgl_dp_1_instruct_params rank {i}", + ) + + verify_params_close( + params["hf_base"][i], + params["sgl_dp_1_base"][i], + f"sgl_dp_1_base_params rank {i}", + ) + + verify_params_not_close( + params["hf_instruct"][i], + params["hf_base"][i], + f"hf_instruct_params rank {i}", + ) + + if dp_size == 2: + verify_params_close( + params["hf_base"][i], + params["sgl_dp_2_base"][i], + f"sgl_dp_2_base_params rank {i}", + ) + verify_params_close( + params["hf_instruct"][i], + params["sgl_dp_2_instruct"][i], + f"sgl_dp_2_instruct_params rank {i}", + ) + + assert len(params["hf_instruct"]) == len( + params["hf_base"] + ), "hf_instruct_params and hf_base_params have different lengths" + + # Check if the weights of lm_head are tied with embed_tokens. + params_to_check = [ + ( + params["hf_instruct"], + "lm_head.weight is not tied with embed_tokens.weight", + ), + ( + params["hf_base"], + "lm_head.weight is not tied with embed_tokens.weight", + ), + ( + params["sgl_dp_1_instruct"], + "lm_head.weight is not tied with embed_tokens.weight", + ), + ( + params["sgl_dp_1_base"], + "lm_head.weight is not tied with embed_tokens.weight", + ), + ] + + if dp_size == 2: + params_to_check.extend( + [ + ( + params["sgl_dp_2_instruct"], + "lm_head.weight is not tied with embed_tokens.weight", + ), + ( + params["sgl_dp_2_base"], + "lm_head.weight is not tied with embed_tokens.weight", + ), + ] + ) + + assert_tied_weights( + [params for params, _ in params_to_check], + ( + "lm_head.weight is not tied with embed_tokens.weight" + if tie_word_embeddings + else "lm_head.weight is tied with embed_tokens.weight" + ), + tie_word_embeddings, + ) + + # Time limit for broadcast and update on CI is 3 / 6 + # On local H100, it's 1 / 2 + time_limit = 3 if model_name == DEFAULT_SMALL_MODEL_NAME_FOR_TEST else 6 + + assert ( + params["broadcast_time"] < time_limit + ), f"broadcast_time exceeds time limit {time_limit}s" + + assert ( + params["update_sgl_dp_1_time"] < time_limit + ), f"update_sgl_dp_one_time exceeds time limit {time_limit}s" + + if dp_size == 2: + assert ( + params["update_sgl_dp_2_time"] < time_limit + ), f"update_sgl_dp_two_time exceeds time limit {time_limit}s" + + # Delete the context and close the parameter queue. + del context + param_queue.close() + param_queue.join_thread() + gc.collect() + torch.cuda.empty_cache() + + +class TestUpdateWeightsFromDistributed(CustomTestCase): + + def test_update_weights_from_distributed(self): + + assert torch.cuda.device_count() >= 2, "At least 2 GPUs are required" + # test_suits : tp, dp, model_name, backend + if is_in_ci(): + mode = random.choice(["Engine", "Server"]) + if mode == "Server": + pause_generation_mode = random.choice(["in_place", "retract"]) + else: + pause_generation_mode = None + load_format = random.choice(["flattened_bucket", None]) + test_suits = [ + ( + 1, + 1, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + mode, + pause_generation_mode, + load_format, + ), + ] + else: + test_suits = [ + ( + 1, + 1, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + "Engine", + None, + random.choice(["flattened_bucket", None]), + ), + ( + 1, + 1, + DEFAULT_MODEL_NAME_FOR_TEST, + "Sever", + random.choice(["in_place", "retract"]), + random.choice(["flattened_bucket", None]), + ), + ] + + if torch.cuda.device_count() >= 4: + test_suits.extend( + [ + ( + 2, + 1, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + "Engine", + None, + random.choice(["flattened_bucket", None]), + ), + ( + 1, + 2, + DEFAULT_MODEL_NAME_FOR_TEST, + "Server", + random.choice(["in_place", "retract"]), + random.choice(["flattened_bucket", None]), + ), + ] + ) + + if torch.cuda.device_count() >= 5: + test_suits.extend( + [ + ( + 2, + 2, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + "Engine", + None, + random.choice(["flattened_bucket", None]), + ), + ( + 2, + 2, + DEFAULT_MODEL_NAME_FOR_TEST, + "Server", + random.choice(["in_place", "retract"]), + random.choice(["flattened_bucket", None]), + ), + ] + ) + + model_state_dict_shapes = {} + test_models = [test_suit[2] for test_suit in test_suits] + + for model_name in test_models: + model = AutoModelForCausalLM.from_pretrained( + model_name, torch_dtype="bfloat16" + ).to("cuda:0") + state_dict = model.state_dict() + state_dict_keys = list(state_dict.keys()) + model_state_dict_shapes[model_name] = { + key: state_dict[key].shape for key in state_dict_keys + } + del model + gc.collect() + torch.cuda.empty_cache() + + truncate_size = 10 + checking_parameters = [ + "model.embed_tokens.weight", + "model.layers.0.input_layernorm.weight", + "model.layers.1.self_attn.q_proj.weight", + "model.layers.2.self_attn.k_proj.weight", + "model.layers.3.self_attn.v_proj.weight", + "model.layers.4.self_attn.o_proj.weight", + "model.layers.5.mlp.gate_proj.weight", + "model.layers.6.mlp.up_proj.weight", + "model.layers.7.mlp.down_proj.weight", + "model.layers.8.post_attention_layernorm.weight", + "model.norm.weight", + "lm_head.weight", + ] + + for ( + tp_size, + dp_size, + model_name, + backend, + pause_generation_mode, + load_format, + ) in test_suits: + test_update_weights_from_distributed( + tp_size, + dp_size, + model_name, + backend, + model_state_dict_shapes[model_name], + truncate_size, + checking_parameters, + load_format, + pause_generation_mode, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/rl/test_update_weights_from_tensor.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/rl/test_update_weights_from_tensor.py new file mode 100644 index 00000000..04dd01d9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/rl/test_update_weights_from_tensor.py @@ -0,0 +1,295 @@ +import gc +import json +import random +import time +import unittest +from concurrent.futures import ThreadPoolExecutor, as_completed + +import requests +import torch + +import sglang as sgl +from sglang.srt.utils import MultiprocessingSerializer, kill_process_tree +from sglang.srt.weight_sync.tensor_bucket import FlattenedTensorBucket +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +def test_update_weights_from_tensor(tp_size): + assert torch.cuda.device_count() >= tp_size, f"At least {tp_size} GPUs are required" + torch.cuda.empty_cache() + + engine = sgl.Engine(model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, tp_size=tp_size) + + param_names = [f"model.layers.{i}.mlp.up_proj.weight" for i in range(6, 16)] + + _check_param(engine, param_names[0], [0.0087, -0.0214, -0.0004, 0.0039, 0.0110]) + + memory_before = torch.cuda.memory_allocated() + new_tensor = torch.full((16384, 2048), 1.5, device="cuda") + + time_start = time.perf_counter() + engine.update_weights_from_tensor([(x, new_tensor) for x in param_names]) + print(f"Time delta: {time.perf_counter() - time_start:.03f}") + + for param_name in param_names[:3]: + _check_param(engine, param_name, [1.5] * 5) + + engine.shutdown() + + del new_tensor + gc.collect() + torch.cuda.ipc_collect() + torch.cuda.empty_cache() + memory_after = torch.cuda.memory_allocated() + assert ( + memory_after <= memory_before + 1024 + ), f"Memory leak detected: {memory_after - memory_before} bytes" + + +class TestUpdateWeightsFromTensor(CustomTestCase): + def test_update_weights_from_tensor(self): + tp_sizes = [1, 2] + for tp_size in tp_sizes: + if torch.cuda.device_count() < tp_size: + continue + + with self.subTest(tp_size=tp_size): + test_update_weights_from_tensor(tp_size) + + def test_update_weights_from_tensor_load_format_direct(self): + engine = sgl.Engine(model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + + write_param_names = [ + f"model.layers.{i}.self_attn.qkv_proj.weight" for i in range(6, 16) + ] + read_param_names = [ + f"model.layers.{i}.self_attn.k_proj.weight" for i in range(6, 16) + ] + + _check_param( + engine, read_param_names[0], [-0.0198, 0.0227, 0.0168, 0.0232, -0.0178] + ) + + new_tensor = torch.full((3072, 2048), 1.5) + engine.update_weights_from_tensor( + [ + (write_param_name, new_tensor.clone()) + for write_param_name in write_param_names + ], + load_format="direct", + ) + + for read_param_name in read_param_names[:3]: + _check_param(engine, read_param_name, [1.5] * 5) + + engine.shutdown() + + def test_update_weights_from_tensor_load_format_custom(self): + custom_loader_name = ( + "sglang.srt.model_executor.model_runner._model_load_weights_direct" + ) + engine = sgl.Engine( + model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + custom_weight_loader=[custom_loader_name], + ) + + write_param_names = [ + f"model.layers.{i}.self_attn.qkv_proj.weight" for i in range(6, 16) + ] + read_param_names = [ + f"model.layers.{i}.self_attn.k_proj.weight" for i in range(6, 16) + ] + + _check_param( + engine, read_param_names[0], [-0.0198, 0.0227, 0.0168, 0.0232, -0.0178] + ) + + new_tensor = torch.full((3072, 2048), 1.5) + engine.update_weights_from_tensor( + [ + (write_param_name, new_tensor.clone()) + for write_param_name in write_param_names + ], + load_format=custom_loader_name, + ) + + for read_param_name in read_param_names[:3]: + _check_param(engine, read_param_name, [1.5] * 5) + + engine.shutdown() + + def test_update_weights_from_tensor_load_format_flattened_bucket(self): + """Test updating weights using flattened_bucket format""" + engine = sgl.Engine(model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + + # Create a small set of parameters for testing + param_names = [f"model.layers.{i}.mlp.up_proj.weight" for i in range(6, 10)] + + # Check original values + _check_param(engine, param_names[0], [0.0087, -0.0214, -0.0004, 0.0039, 0.0110]) + + # Create new tensors with different values + new_tensors = [] + for _, name in enumerate(param_names): + # Create tensors with different values for each parameter + value = 2.0 # Different value for each parameter + new_tensor = torch.full((16384, 2048), value, device="cuda") + new_tensors.append((name, new_tensor)) + + # Create a flattened bucket + flattened_bucket = FlattenedTensorBucket(named_tensors=new_tensors) + + # Extract the flattened tensor and metadata in the format expected by model_runner + flattened_tensor = flattened_bucket.get_flattened_tensor() + metadata = flattened_bucket.get_metadata() + + # Create the dict format expected by _update_weights_from_flattened_bucket + bucket_dict = {"flattened_tensor": flattened_tensor, "metadata": metadata} + + # Serialize the bucket data + from sglang.srt.utils import MultiprocessingSerializer + + serialized_bucket = MultiprocessingSerializer.serialize( + bucket_dict, output_str=True + ) + + # Create a list where each rank contains the same serialized data + # This simulates the distributed environment where each rank has the same data + serialized_bucket_list = [serialized_bucket] + + # Update weights using flattened_bucket format + time_start = time.perf_counter() + engine.update_weights_from_tensor( + named_tensors=serialized_bucket_list, load_format="flattened_bucket" + ) + update_time = time.perf_counter() - time_start + print(f"Flattened bucket update time: {update_time:.03f}") + + # Verify the weights were updated correctly + for i, param_name in enumerate(param_names): + _check_param(engine, param_name, [2.0] * 5) + + engine.shutdown() + + +class TestServerUpdateWeightsFromTensorNonBlocking(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--max-running-requests", 8], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode(self, max_new_tokens=32): + response = requests.post( + self.base_url + "/generate", + json={ + "text": f"Question: {random.randint(0, 100)},The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + "ignore_eos": True, + }, + }, + ) + return response.json() + + def get_model_info(self): + response = requests.get(self.base_url + "/get_model_info") + model_path = response.json()["model_path"] + print(json.dumps(response.json())) + return model_path + + def pause_generation(self, mode): + response = requests.post( + self.base_url + "/pause_generation", + json={"mode": mode}, + ) + ret = response.json() + return ret + + def continue_generation(self): + response = requests.post( + self.base_url + "/continue_generation", + json={}, + ) + ret = response.json() + return ret + + def run_update_weights(self, named_tensors, flush_cache=True): + response = requests.post( + self.base_url + "/update_weights_from_tensor", + json={ + "serialized_named_tensors": [ + MultiprocessingSerializer.serialize(named_tensors, output_str=True) + ], + "flush_cache": flush_cache, + }, + ) + ret = response.json() + return ret + + def test_update_weights(self): + pause_generation_modes = ["in_place", "retract"] + for pause_generation_mode in pause_generation_modes: + num_requests = 32 + with ThreadPoolExecutor(num_requests) as executor: + futures = [ + executor.submit(self.run_decode, 3000) for _ in range(num_requests) + ] + + # ensure the decode has been started + time.sleep(2) + + param_names = [ + f"model.layers.{i}.mlp.up_proj.weight" for i in range(6, 16) + ] + new_tensor = torch.full((16384, 2048), 1.5, device="cuda") + named_tensors = [(x, new_tensor) for x in param_names] + + ret = self.pause_generation(pause_generation_mode) + ret = self.run_update_weights( + named_tensors, flush_cache=pause_generation_mode == "retract" + ) + self.assertTrue(ret["success"]) + ret = self.continue_generation() + + for future in as_completed(futures): + self.assertNotEqual( + future.result()["meta_info"]["finish_reason"]["type"], "abort" + ) + + for param_name in param_names[:3]: + response = requests.post( + self.base_url + "/get_weights_by_name", + json={"name": param_name}, + ) + actual_values = torch.tensor(response.json())[0, :5] + assert torch.allclose( + actual_values, torch.tensor([1.5] * 5), atol=0.002 + ), f"{actual_values=}" + + +def _check_param(engine, param_name, expect_values): + actual_values = torch.tensor(engine.get_weights_by_name(param_name))[0, :5] + assert torch.allclose( + actual_values, torch.tensor(expect_values), atol=0.002 + ), f"{actual_values=}" + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/rotary_embedding/test_mrope.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/rotary_embedding/test_mrope.py new file mode 100644 index 00000000..c9837f87 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/rotary_embedding/test_mrope.py @@ -0,0 +1,143 @@ +from typing import NamedTuple + +import pytest +import torch +from packaging.version import Version +from transformers import AutoConfig +from transformers import __version__ as TRANSFORMERS_VERSION + +from sglang.srt.layers.rotary_embedding import get_rope +from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler +from sglang.srt.utils import ( + cpu_has_amx_support, + is_cpu, + is_cuda, + is_hip, + is_npu, + is_xpu, +) + +_is_cuda = is_cuda() +_is_hip = is_hip() +_is_cpu = is_cpu() +_is_cpu_amx_available = cpu_has_amx_support() +_is_npu = is_npu() +_is_xpu = is_xpu() + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + +def generate_test_data( + num_tokens: int, + num_q_heads: int, + num_kv_heads: int, + head_size: int, + max_position_embeddings: int, + dtype: torch.dtype, + device: torch.device, +): + """Generate test data for given configuration.""" + torch.manual_seed(42) + # Create 2D positions (3, num_tokens) for multimodal case + positions = torch.randint( + 0, max_position_embeddings // 4, (3, num_tokens), device=device + ) + + # Create query and key tensors + query = torch.randn(num_tokens, num_q_heads * head_size, dtype=dtype, device=device) + key = torch.randn(num_tokens, num_kv_heads * head_size, dtype=dtype, device=device) + + return positions, query, key + + +class MRoPETestInfo(NamedTuple): + model_name: str + atol: float = 1e-2 + rtol: float = 1.6e-2 + marks: list[pytest.MarkDecorator] = [] + + +TRANSFORMERS_BASE_VERSION = Version(TRANSFORMERS_VERSION).base_version + +MODELS_TO_TEST = [ + MRoPETestInfo(model_name="Qwen/Qwen2-VL-7B-Instruct"), + MRoPETestInfo(model_name="Qwen/Qwen2-VL-72B-Instruct"), + MRoPETestInfo(model_name="Qwen/Qwen2.5-VL-72B-Instruct"), +] + +num_tokens_list = [11, 8192] + + +@pytest.mark.skipif(not _is_cuda, reason="Skipping CUDA/ROCm only tests.") +@pytest.mark.parametrize( + "model_info, model_name", + [ + pytest.param(test_config, test_config.model_name, marks=test_config.marks) + for test_config in MODELS_TO_TEST + ], +) +@pytest.mark.parametrize("tp_size", [1, 2]) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) +@pytest.mark.parametrize("num_tokens", num_tokens_list) +def test_mrope( + model_name: str, + model_info: MRoPETestInfo, + tp_size: int, + dtype: torch.dtype, + num_tokens: int, +): + set_global_server_args_for_scheduler(ServerArgs(model_path="dummy")) + + atol = model_info.atol + rtol = model_info.rtol + + config = AutoConfig.from_pretrained(model_name) + config = config.get_text_config() + + # get the model config + total_num_kv_heads = config.num_key_value_heads + total_num_heads = config.num_attention_heads + num_heads = total_num_heads // tp_size + num_kv_heads = max(1, total_num_kv_heads // tp_size) + head_dim = ( + config.head_dim + if hasattr(config, "head_dim") + else config.hidden_size // total_num_heads + ) + is_neox_style = True + + rope_theta = config.rope_theta + max_position = config.max_position_embeddings + partial_rotary_factor = getattr(config, "partial_rotary_factor", 1.0) + rotary_dim = int(head_dim * partial_rotary_factor) + + mrope_helper_class = get_rope( + head_size=head_dim, + rotary_dim=rotary_dim, + max_position=max_position, + base=rope_theta, + is_neox_style=is_neox_style, + rope_scaling=config.rope_scaling, + dtype=dtype, + ).to(device=device) + + # create q k v input tensors + # create rotary pos emb input tensors + positions, query, key = generate_test_data( + num_tokens, num_heads, num_kv_heads, head_dim, max_position, dtype, device + ) + + query_native, key_native = mrope_helper_class._forward_native( + positions, + query.clone(), + key.clone(), + ) + + query_cuda, key_cuda = mrope_helper_class.forward( + positions, + query.clone(), + key.clone(), + ) + + torch.testing.assert_close(query_native, query_cuda, atol=atol, rtol=rtol) + torch.testing.assert_close(key_native, key_cuda, atol=atol, rtol=rtol) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/run_suite.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/run_suite.py new file mode 100644 index 00000000..ecfe1fe3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/run_suite.py @@ -0,0 +1,530 @@ +import argparse +import glob +from pathlib import Path + +from sglang.test.ci.ci_utils import TestFile, run_unittest_files + +# NOTE: please sort the test cases alphabetically by the test file name +suites = { + "per-commit-1-gpu": [ + TestFile("test_eagle_constrained_decoding.py", 100), + TestFile("debug_utils/test_tensor_dump_forward_hook.py", 15), + TestFile("hicache/test_hicache_storage.py", 127), + TestFile("hicache/test_hicache_variants.py", 393), + TestFile("layers/attention/mamba/test_causal_conv1d.py", 25), + TestFile("layers/attention/mamba/test_mamba_ssm.py", 50), + TestFile("layers/attention/mamba/test_mamba_ssm_ssd.py", 20), + TestFile("lora/test_lora.py", 150), + TestFile("lora/test_lora_eviction.py", 240), + TestFile("lora/test_lora_update.py", 600), + TestFile("lora/test_lora_backend.py", 99), + TestFile("lora/test_multi_lora_backend.py", 60), + TestFile("models/test_compressed_tensors_models.py", 42), + TestFile("models/test_cross_encoder_models.py", 100), + TestFile("models/test_embedding_models.py", 73), + TestFile("models/test_encoder_embedding_models.py", 460), + TestFile("models/test_generation_models.py", 103), + TestFile("models/test_nvidia_nemotron_nano_v2.py", 160), + TestFile("models/test_nvidia_nemotron_nano_v2_vl.py", 350), # GSM8k + MMMU + TestFile("models/test_qwen_models.py", 150), + TestFile("models/test_reward_models.py", 132), + TestFile("models/test_transformers_models.py", 320), + TestFile("models/test_vlm_models.py", 741), + TestFile("openai_server/basic/test_openai_embedding.py", 79), + TestFile("openai_server/basic/test_openai_server.py", 270), + TestFile("openai_server/basic/test_protocol.py", 10), + TestFile("openai_server/basic/test_serving_chat.py", 10), + TestFile("openai_server/basic/test_serving_completions.py", 10), + TestFile("openai_server/basic/test_serving_embedding.py", 10), + TestFile("openai_server/features/test_enable_thinking.py", 70), + TestFile("openai_server/features/test_json_mode.py", 120), + TestFile("openai_server/features/test_openai_server_ebnf.py", 20), + TestFile("openai_server/features/test_openai_server_hidden_states.py", 240), + TestFile("openai_server/features/test_reasoning_content.py", 89), + TestFile("openai_server/function_call/test_openai_function_calling.py", 60), + TestFile("openai_server/function_call/test_tool_choice.py", 120), + TestFile("openai_server/validation/test_large_max_new_tokens.py", 41), + TestFile("openai_server/validation/test_matched_stop.py", 60), + TestFile("openai_server/validation/test_openai_server_ignore_eos.py", 85), + TestFile("openai_server/validation/test_request_length_validation.py", 31), + TestFile("ops/test_repeat_interleave.py", 60), + TestFile("quant/test_block_int8.py", 22), + TestFile("quant/test_fp8_kernel.py", 8), + TestFile("quant/test_int8_kernel.py", 8), + TestFile("quant/test_triton_scaled_mm.py", 8), + TestFile("quant/test_w8a8_quantization.py", 160), + TestFile("quant/test_autoround.py", 60), + TestFile("rl/test_fp32_lm_head.py", 30), + # TestFile("rl/test_update_weights_from_disk.py", 210), # Temporarily disabled, see https://github.com/sgl-project/sglang/pull/13998 + TestFile("rl/test_update_weights_from_tensor.py", 80), + TestFile("test_abort.py", 190), + TestFile("test_build_eagle_tree.py", 8), + TestFile("test_chunked_prefill.py", 410), + TestFile("test_create_kvindices.py", 2), + TestFile("test_deterministic.py", 400), + TestFile("test_eagle_infer_a.py", 750), + TestFile("test_eagle_infer_b.py", 750), + TestFile("test_eagle_infer_beta.py", 90), + TestFile("test_constrained_decoding.py", 150), + TestFile("test_eval_fp8_accuracy.py", 303), + TestFile("test_external_models.py", 155), + TestFile("test_fa3.py", 420), + TestFile("test_flashmla.py", 230), + TestFile("test_fp8_utils.py", 5), + TestFile("rotary_embedding/test_mrope.py", 10), + TestFile("test_fused_moe.py", 80), + TestFile("test_gpt_oss_1gpu.py", 750), + TestFile("test_harmony_parser.py", 20), + TestFile("test_hidden_states.py", 55), + TestFile("test_hybrid_attn_backend.py", 379), + TestFile("test_input_embeddings.py", 38), + TestFile("test_io_struct.py", 8), + TestFile("test_jinja_template_utils.py", 1), + TestFile("test_mamba_unittest.py", 4), + TestFile("test_metrics.py", 32), + TestFile("test_metrics_utils.py", 1), + TestFile("test_mla.py", 180), + TestFile("test_mla_deepseek_v3.py", 500), + TestFile("test_mla_flashinfer.py", 302), + TestFile("test_mla_fp8.py", 93), + TestFile("test_mla_int8_deepseek_v3.py", 300), + TestFile("test_model_hooks.py", 1), + TestFile("test_modelopt_loader.py", 30), + TestFile("test_multi_tokenizer.py", 230), + TestFile("test_ngram_speculative_decoding.py", 290), + TestFile("test_no_chunked_prefill.py", 108), + TestFile("test_no_overlap_scheduler.py", 234), + TestFile("test_original_logprobs.py", 41), + TestFile("test_page_size.py", 60), + TestFile("test_penalty.py", 82), + TestFile("test_priority_scheduling.py", 130), + TestFile("test_pytorch_sampling_backend.py", 66), + TestFile("test_radix_attention.py", 105), + TestFile("test_radix_cache_unit.py", 5), + TestFile("test_reasoning_parser.py", 5), + TestFile("test_request_queue_validation.py", 30), + TestFile("test_retract_decode.py", 450), + TestFile("test_score_api.py", 310), + TestFile("test_server_args.py", 1), + TestFile("test_speculative_registry.py", 1), + TestFile("test_skip_tokenizer_init.py", 117), + TestFile("test_srt_endpoint.py", 130), + TestFile("test_srt_engine.py", 450), + TestFile("test_standalone_speculative_decoding.py", 150), + TestFile("test_start_profile.py", 180), + TestFile("test_profile_merger.py", 60), + TestFile("test_profile_merger_http_api.py", 15), + TestFile("test_swa_unittest.py", 1), + TestFile("test_torch_compile.py", 76), + TestFile("test_torch_compile_moe.py", 210), + TestFile("test_triton_fused_moe.py", 80), + TestFile("test_torch_native_attention_backend.py", 123), + TestFile("test_torchao.py", 70), + TestFile("test_triton_attention_kernels.py", 4), + TestFile("test_triton_attention_backend.py", 150), + TestFile("test_triton_attention_kernels.py", 4), + TestFile("test_triton_moe_channel_fp8_kernel.py", 25), + TestFile("test_triton_sliding_window.py", 100), + TestFile("test_utils_update_weights.py", 48), + TestFile("test_video_utils.py", 5), + TestFile("test_vision_chunked_prefill.py", 170), + TestFile("test_vision_openai_server_a.py", 900), + TestFile("test_vlm_input_format.py", 300), + TestFile("test_modelopt_loader.py", 30), + TestFile("test_modelopt_export.py", 30), + ], + "per-commit-2-gpu": [ + TestFile("ep/test_moe_ep.py", 140), + TestFile("hicache/test_hicache_storage_3fs_backend.py", 200), + TestFile("hicache/test_hicache_storage_file_backend.py", 200), + TestFile("hicache/test_hicache_storage_mooncake_backend.py", 300), + TestFile("layers/attention/mamba/test_mamba2_mixer.py", 50), + TestFile("lora/test_lora_tp.py", 116), + TestFile("models/test_glm4_moe_models.py", 100), + TestFile("models/test_kimi_linear_models.py", 90), + TestFile("rl/test_update_weights_from_distributed.py", 103), + TestFile("test_constrained_decoding_spec_reasoning.py", 60), + TestFile("test_data_parallelism.py", 73), + TestFile("test_disaggregation_basic.py", 400), + TestFile("test_dp_attention.py", 350), + TestFile("test_load_weights_from_remote_instance.py", 72), + TestFile("test_patch_torch.py", 19), + TestFile("test_eagle_dp_attention.py", 200), + ], + "per-commit-4-gpu": [ + TestFile("models/test_qwen3_next_models.py", 291), + TestFile("test_gpt_oss_4gpu.py", 300), + TestFile("test_local_attn.py", 411), + TestFile("test_multi_instance_release_memory_occupation.py", 64), + TestFile("test_pp_single_node.py", 481), + TestFile("test_piecewise_cuda_graph.py", 1200), + ], + "per-commit-8-gpu-h200": [ + TestFile("test_deepseek_v3_basic.py", 275), + TestFile("test_deepseek_v3_mtp.py", 275), + TestFile("test_disaggregation_hybrid_attention.py", 200), + TestFile("models/test_kimi_k2_models.py", 200), + TestFile("test_deepseek_v32_basic.py", 275), + TestFile("test_deepseek_v32_mtp.py", 275), + ], + "per-commit-8-gpu-h20": [ + TestFile("quant/test_w4a8_deepseek_v3.py", 520), + TestFile("test_disaggregation_different_tp.py", 600), + TestFile("test_disaggregation_pp.py", 140), + TestFile("test_disaggregation_dp_attention.py", 155), + ], + "per-commit-4-gpu-b200": [ + TestFile("test_deepseek_v3_fp4_4gpu.py", 1800), + TestFile("test_flash_attention_4.py", 300), + TestFile("test_gpt_oss_4gpu.py", 600), + TestFile("test_llama31_fp4.py", 300), + # TODO: Add it back after the bug is fixed + # TestFile("test_eagle_infer_beta_dp_attention.py", 200), + ], + "per-commit-8-gpu-b200": [], + "per-commit-4-gpu-gb200": [ + TestFile("test_cutedsl_moe.py", 300), + TestFile("test_deepseek_v3_cutedsl_4gpu.py", 590), + ], + "per-commit-4-gpu-deepep": [ + TestFile("ep/test_deepep_small.py", 531), + # TODO: Add it back after mooncake supports torch 2.9 + # TestFile("ep/test_mooncake_ep_small.py", 450), + ], + "per-commit-8-gpu-h200-deepep": [ + TestFile("ep/test_deepep_large.py", 338), + ], + "quantization_test": [ + TestFile("quant/test_awq.py", 163), + TestFile("test_bnb.py", 5), + TestFile("test_gptqmodel_dynamic.py", 102), + TestFile("test_quantization.py", 185), + TestFile("test_gguf.py", 96), + ], + # Nightly test suites have been moved to test/run_suite_nightly.py + "__not_in_ci__": [ + TestFile("test_release_memory_occupation.py", 200), # Temporarily disabled + TestFile("models/test_dummy_grok_models.py"), + TestFile( + "rl/test_update_weights_from_disk.py" + ), # Temporarily disabled, see https://github.com/sgl-project/sglang/pull/13998 + TestFile("test_bench_one_batch.py"), + TestFile("test_bench_serving.py"), + TestFile("test_eval_accuracy_large.py"), + TestFile("test_gpt_oss_common.py"), + TestFile("test_moe_eval_accuracy_large.py"), + TestFile("test_vision_openai_server_common.py"), + TestFile("test_profile_v2.py"), + ], +} + +# Add AMD tests +# NOTE: please sort the test cases alphabetically by the test file name +suite_amd = { + "per-commit-amd": [ + # TestFile("hicache/test_hicache.py", 116), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/12575 + # TestFile("hicache/test_hicache_mla.py", 127), # Disabled temporarily, # Temporarily disabled, see https://github.com/sgl-project/sglang/issues/12574 + # TestFile("hicache/test_hicache_storage.py", 127), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/12575 + TestFile("lora/test_lora.py", 665), + # TestFile("lora/test_lora_backend.py", 99), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/13107 + # TestFile("lora/test_lora_cuda_graph.py", 250), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/13107 + TestFile("lora/test_lora_eviction.py", 240), + # TestFile("lora/test_lora_qwen3.py", 97), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/13107 + TestFile("lora/test_multi_lora_backend.py", 60), + TestFile("models/test_compressed_tensors_models.py", 42), + TestFile("models/test_qwen_models.py", 82), + TestFile("models/test_reward_models.py", 132), + TestFile("models/test_transformers_models.py", 320), + TestFile("models/test_vlm_models.py", 437), + TestFile("openai_server/basic/test_openai_embedding.py", 141), + TestFile("openai_server/basic/test_openai_server.py", 149), + TestFile("openai_server/basic/test_protocol.py", 10), + TestFile("openai_server/basic/test_serving_chat.py", 10), + TestFile("openai_server/basic/test_serving_completions.py", 10), + TestFile("openai_server/basic/test_serving_embedding.py", 10), + TestFile("openai_server/features/test_enable_thinking.py", 70), + TestFile("openai_server/features/test_json_mode.py", 120), + TestFile("openai_server/features/test_openai_server_ebnf.py", 20), + TestFile("openai_server/features/test_reasoning_content.py", 89), + TestFile("openai_server/function_call/test_openai_function_calling.py", 60), + TestFile("openai_server/function_call/test_tool_choice.py", 120), + TestFile("openai_server/validation/test_large_max_new_tokens.py", 41), + TestFile("openai_server/validation/test_matched_stop.py", 60), + TestFile("openai_server/validation/test_openai_server_ignore_eos.py", 85), + TestFile("openai_server/validation/test_request_length_validation.py", 31), + TestFile("quant/test_awq_dequant.py", 2), + TestFile("quant/test_block_int8.py", 22), + TestFile("quant/test_fused_rms_fp8_group_quant.py", 10), + # TestFile("rl/test_update_weights_from_disk.py", 210), # Temporarily disabled, see https://github.com/sgl-project/sglang/pull/13998 + TestFile("test_abort.py", 51), + TestFile("test_bench_typebaseddispatcher.py", 10), + TestFile("test_chunked_prefill.py", 410), + TestFile("test_create_kvindices.py", 2), + TestFile("test_eval_fp8_accuracy.py", 303), + TestFile("test_fused_moe.py", 30), + TestFile("test_harmony_parser.py", 20), + TestFile("test_input_embeddings.py", 38), + TestFile("test_io_struct.py", 8), + TestFile("test_jinja_template_utils.py", 1), + TestFile("test_metrics.py", 32), + TestFile("test_metrics_utils.py", 1), + # TestFile("test_mla.py", 242), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/13107 + # TestFile("test_mla_deepseek_v3.py", 221), # Temporarily disabled, see https://github.com/sgl-project/sglang/issues/12574 + TestFile("test_no_chunked_prefill.py", 108), + TestFile("test_page_size.py", 60), + TestFile("test_penalty.py", 180), + TestFile("test_pytorch_sampling_backend.py", 66), + TestFile("test_radix_attention.py", 105), + TestFile("test_reasoning_parser.py", 5), + TestFile("test_constrained_decoding.py", 120), + TestFile("test_retract_decode.py", 450), + TestFile("test_rope_rocm.py", 3), + TestFile("test_server_args.py", 1), + TestFile("test_skip_tokenizer_init.py", 117), + TestFile("test_srt_endpoint.py", 130), + TestFile("test_srt_engine.py", 261), + TestFile("test_torch_compile.py", 169), + # TestFile("test_torch_compile_moe.py", 210), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/13107 + TestFile("test_torch_native_attention_backend.py", 123), + # TestFile("test_triton_attention_kernels.py", 4), + TestFile("test_triton_attention_backend.py", 150), + TestFile("test_triton_sliding_window.py", 250), + TestFile("test_type_based_dispatcher.py", 10), + TestFile("test_wave_attention_kernels.py", 2), + # Disabled temporarily + # TestFile("test_vlm_input_format.py", 300), + # TestFile("models/test_embedding_models.py", 73), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/11127 + # TestFile("openai_server/features/test_openai_server_hidden_states.py", 240), + # TestFile("rl/test_update_weights_from_tensor.py", 48), + # TestFile("test_no_overlap_scheduler.py", 234), # Disabled temporarily and track in #7703 + # TestFile("test_vision_chunked_prefill.py", 175), # Disabled temporarily and track in #7701 + # TestFile("test_wave_attention_backend.py", 150), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/11127 + ], + "per-commit-amd-mi35x": [ + TestFile("test_gpt_oss_1gpu.py", 750), + TestFile("test_mla.py", 242), + ], + "per-commit-2-gpu-amd": [ + # TestFile("lora/test_lora_tp.py", 116), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/13107 + TestFile("rl/test_update_weights_from_distributed.py", 103), + TestFile("test_data_parallelism.py", 73), + TestFile("test_load_weights_from_remote_instance.py", 72), + # TestFile("test_patch_torch.py", 19), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/11127 + ], + "per-commit-4-gpu-amd": [ + TestFile("test_pp_single_node.py", 150), + ], + "per-commit-8-gpu-amd": [ + TestFile("test_deepseek_v3_basic.py", 275), + TestFile("test_deepseek_v3_mtp.py", 275), + ], + "nightly-amd": [ + TestFile("nightly/test_gsm8k_eval_amd.py"), + ], +} + +# Add Intel Xeon tests +# NOTE: please sort the test cases alphabetically by the test file name +suite_xeon = { + "per-commit-cpu": [ + TestFile("cpu/test_activation.py"), + TestFile("cpu/test_binding.py"), + TestFile("cpu/test_decode.py"), + TestFile("cpu/test_extend.py"), + TestFile("cpu/test_gemm.py"), + TestFile("cpu/test_mla.py"), + TestFile("cpu/test_moe.py"), + TestFile("cpu/test_norm.py"), + TestFile("cpu/test_qkv_proj_with_rope.py"), + TestFile("cpu/test_rope.py"), + TestFile("cpu/test_shared_expert.py"), + TestFile("cpu/test_topk.py"), + TestFile("cpu/test_cpu_graph.py"), + TestFile("cpu/test_intel_amx_attention_backend_a.py"), + TestFile("cpu/test_intel_amx_attention_backend_b.py"), + TestFile("cpu/test_intel_amx_attention_backend_c.py"), + ], +} + +# Add Intel XPU tests +suite_xpu = { + "per-commit-xpu": [ + TestFile("xpu/test_intel_xpu_backend.py"), + ], +} + +# Add Ascend NPU tests +# TODO: Set accurate estimate time +# NOTE: please sort the test cases alphabetically by the test file name +suite_ascend = { + "per-commit-1-npu-a2": [ + TestFile("ascend/test_ascend_graph_tp1_bf16.py", 400), + TestFile("ascend/test_ascend_hicache_mha.py", 400), + TestFile("ascend/test_ascend_sampling_backend.py", 400), + TestFile("ascend/test_ascend_tp1_bf16.py", 400), + ], + "per-commit-2-npu-a2": [ + TestFile("ascend/test_ascend_graph_tp2_bf16.py", 400), + TestFile("ascend/test_ascend_mla_fia_w8a8int8.py", 400), + TestFile("ascend/test_ascend_tp2_bf16.py", 400), + TestFile("ascend/test_ascend_tp2_fia_bf16.py", 400), + ], + "per-commit-4-npu-a2": [ + TestFile("ascend/test_ascend_mla_w8a8int8.py", 400), + TestFile("ascend/test_ascend_tp4_bf16.py", 400), + ], + "per-commit-16-npu-a3": [ + TestFile("ascend/test_ascend_deepep.py", 400), + TestFile("ascend/test_ascend_deepseek_mtp.py", 400), + ], +} + +suites.update(suite_amd) +suites.update(suite_xeon) +suites.update(suite_ascend) +suites.update(suite_xpu) + + +def auto_partition(files, rank, size): + """ + Partition files into size sublists with approximately equal sums of estimated times + using stable sorting, and return the partition for the specified rank. + + Args: + files (list): List of file objects with estimated_time attribute + rank (int): Index of the partition to return (0 to size-1) + size (int): Number of partitions + + Returns: + list: List of file objects in the specified rank's partition + """ + weights = [f.estimated_time for f in files] + + if not weights or size <= 0 or size > len(weights): + return [] + + # Create list of (weight, original_index) tuples + # Using negative index as secondary key to maintain original order for equal weights + indexed_weights = [(w, -i) for i, w in enumerate(weights)] + # Stable sort in descending order by weight + # If weights are equal, larger (negative) index comes first (i.e., earlier original position) + indexed_weights = sorted(indexed_weights, reverse=True) + + # Extract original indices (negate back to positive) + indexed_weights = [(w, -i) for w, i in indexed_weights] + + # Initialize partitions and their sums + partitions = [[] for _ in range(size)] + sums = [0.0] * size + + # Greedy approach: assign each weight to partition with smallest current sum + for weight, idx in indexed_weights: + # Find partition with minimum sum + min_sum_idx = sums.index(min(sums)) + partitions[min_sum_idx].append(idx) + sums[min_sum_idx] += weight + + # Return the files corresponding to the indices in the specified rank's partition + indices = partitions[rank] + return [files[i] for i in indices] + + +def _sanity_check_suites(suites): + dir_base = Path(__file__).parent + disk_files = set( + [ + str(x.relative_to(dir_base)) + for x in dir_base.glob("**/*.py") + if x.name.startswith("test_") + ] + ) + + suite_files = set( + [test_file.name for _, suite in suites.items() for test_file in suite] + ) + + missing_files = sorted(list(disk_files - suite_files)) + missing_text = "\n".join(f'TestFile("{x}"),' for x in missing_files) + assert len(missing_files) == 0, ( + f"Some test files are not in test suite. " + f"If this is intentional, please add the following to `not_in_ci` section:\n" + f"{missing_text}" + ) + + nonexistent_files = sorted(list(suite_files - disk_files)) + nonexistent_text = "\n".join(f'TestFile("{x}"),' for x in nonexistent_files) + assert ( + len(nonexistent_files) == 0 + ), f"Some test files in test suite do not exist on disk:\n{nonexistent_text}" + + not_in_ci_files = set( + [test_file.name for test_file in suites.get("__not_in_ci__", [])] + ) + in_ci_files = set( + [ + test_file.name + for suite_name, suite in suites.items() + if suite_name != "__not_in_ci__" + for test_file in suite + ] + ) + intersection = not_in_ci_files & in_ci_files + intersection_text = "\n".join(f'TestFile("{x}"),' for x in intersection) + assert len(intersection) == 0, ( + f"Some test files are in both `not_in_ci` section and other suites:\n" + f"{intersection_text}" + ) + + +def main(): + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument( + "--timeout-per-file", + type=int, + default=1200, + help="The time limit for running one file in seconds.", + ) + arg_parser.add_argument( + "--suite", + type=str, + default=list(suites.keys())[0], + choices=list(suites.keys()) + ["all"], + help="The suite to run", + ) + arg_parser.add_argument( + "--auto-partition-id", + type=int, + help="Use auto load balancing. The part id.", + ) + arg_parser.add_argument( + "--auto-partition-size", + type=int, + help="Use auto load balancing. The number of parts.", + ) + arg_parser.add_argument( + "--continue-on-error", + action="store_true", + default=False, + help="Continue running remaining tests even if one fails (useful for nightly tests)", + ) + args = arg_parser.parse_args() + print(f"{args=}") + + _sanity_check_suites(suites) + + if args.suite == "all": + files = glob.glob("**/test_*.py", recursive=True) + else: + files = suites[args.suite] + + if args.auto_partition_size: + files = auto_partition(files, args.auto_partition_id, args.auto_partition_size) + + print("The running tests are ", [f.name for f in files]) + + exit_code = run_unittest_files(files, args.timeout_per_file, args.continue_on_error) + exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_abort.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_abort.py new file mode 100644 index 00000000..94256e55 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_abort.py @@ -0,0 +1,191 @@ +import multiprocessing +import time +import unittest +from concurrent.futures import ThreadPoolExecutor, as_completed + +import requests + +from sglang.srt.environ import envs +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, + run_and_check_memory_leak, +) + + +class TestAbort(CustomTestCase): + def workload_func(self, base_url, model): + def process_func(): + def run_one(_): + prompt = """ + System: You are a helpful assistant. + User: What is the capital of France? + Assistant: The capital of France is + """ + + response = requests.post( + f"{base_url}/generate", + json={ + "text": prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 2048, + }, + }, + ) + ret = response.json() + + with ThreadPoolExecutor(16) as executor: + list(executor.map(run_one, list(range(16)))) + + p = multiprocessing.Process(target=process_func) + p.start() + time.sleep(0.5) + p.terminate() + time.sleep(10) + + def test_memory_leak(self): + run_and_check_memory_leak( + self.workload_func, + disable_radix_cache=False, + enable_mixed_chunk=False, + disable_overlap=False, + chunked_prefill_size=8192, + assert_has_abort=True, + ) + + +class TestAbortAll(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--max-running-requests", 8], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def _run_decode(self): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 16000, + "ignore_eos": True, + }, + }, + ) + return response.json() + + def test_abort_all(self): + num_requests = 32 + with ThreadPoolExecutor(num_requests) as executor: + futures = [executor.submit(self._run_decode) for _ in range(num_requests)] + + # ensure the decode has been started + time.sleep(2) + + requests.post( + self.base_url + "/abort_request", + json={ + "abort_all": True, + }, + ) + + for future in as_completed(futures): + self.assertEqual( + future.result()["meta_info"]["finish_reason"]["type"], "abort" + ) + + +class TestAbortAllWithRetraction(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + # Here's a small trick: in scheduler.py, when SGLANG_TEST_RETRACT is enabled, + # retraction is triggered when the batch size reaches 10. + # However, since SGLANG_TEST_RETRACT_NO_PREFILL_BS is set to 6, the remaining 4 + # requests will stay in the waiting queue. + with ( + envs.SGLANG_TEST_RETRACT.override(True), + envs.SGLANG_TEST_RETRACT_NO_PREFILL_BS.override(6), + ): + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--max-running-requests", + 16, + "--schedule-policy", + "random", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def _run_decode(self): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 4000, + "ignore_eos": True, + }, + }, + ) + return response.json() + + def test_abort_all_with_retraction(self): + num_requests = 32 + with ThreadPoolExecutor(num_requests) as executor: + futures = [executor.submit(self._run_decode) for _ in range(num_requests)] + + # ensure the decode has been started and retractions happen. + time.sleep(8) + + requests.post( + self.base_url + "/abort_request", + json={ + "abort_all": True, + }, + ) + + abort_in_queue_count = 0 + abort_in_queue_with_none_empty_text = 0 + + for future in as_completed(futures): + self.assertEqual( + future.result()["meta_info"]["finish_reason"]["type"], "abort" + ) + if ( + future.result()["meta_info"]["finish_reason"]["message"] + == "Abort in waiting queue" + ): + abort_in_queue_count += 1 + if len(future.result()["output_ids"]) > 0: + abort_in_queue_with_none_empty_text += 1 + assert abort_in_queue_count > 0 + assert abort_in_queue_with_none_empty_text > 0 + print("Finished test_abort_all_with_retraction") + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_bench_one_batch.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_bench_one_batch.py new file mode 100644 index 00000000..8d14cd0b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_bench_one_batch.py @@ -0,0 +1,72 @@ +import unittest + +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_MOE_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + CustomTestCase, + is_in_amd_ci, + is_in_ci, + run_bench_offline_throughput, + run_bench_one_batch, + write_github_step_summary, +) + +# We use `run_bench_offline_throughput`` instead of `run_bench_one_batch` for most cases +# because `run_bench_offline_throughput`` has overlap scheduler. + + +class TestBenchOneBatch(CustomTestCase): + + def test_bs1_small(self): + _, output_throughput, _ = run_bench_one_batch( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, ["--cuda-graph-max-bs", "2"] + ) + self.assertGreater(output_throughput, 50) + + def test_bs1_default(self): + output_throughput = run_bench_offline_throughput( + DEFAULT_MODEL_NAME_FOR_TEST, ["--cuda-graph-max-bs", "2"] + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_bs1_default (llama-3.1-8b)\n" + f"output_throughput: {output_throughput:.2f} token/s\n" + ) + self.assertGreater(output_throughput, 135) + + def test_moe_tp2_bs1(self): + output_throughput = run_bench_offline_throughput( + DEFAULT_MOE_MODEL_NAME_FOR_TEST, ["--tp", "2", "--cuda-graph-max-bs", "2"] + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_moe_tp2_bs1 (Mixtral-8x7B)\n" + f"output_throughput: {output_throughput:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(output_throughput, 85) + else: + self.assertGreater(output_throughput, 125) + + def test_torch_compile_tp2_bs1(self): + output_throughput = run_bench_offline_throughput( + DEFAULT_MODEL_NAME_FOR_TEST, + ["--tp", "2", "--enable-torch-compile", "--cuda-graph-max-bs", "2"], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_torch_compile_tp2_bs1 (Mixtral-8x7B)\n" + f"output_throughput: {output_throughput:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(output_throughput, 200) + else: + self.assertGreater(output_throughput, 220) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_bench_serving.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_bench_serving.py new file mode 100644 index 00000000..69ef3fea --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_bench_serving.py @@ -0,0 +1,562 @@ +import asyncio +import itertools +import unittest + +import requests + +from sglang.test.test_utils import ( + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST_FP8, + DEFAULT_MOE_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE, + DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST, + CustomTestCase, + is_in_amd_ci, + is_in_ci, + run_bench_serving, + run_embeddings_benchmark, + run_score_benchmark, + write_github_step_summary, +) + + +class TestBenchServing(CustomTestCase): + def test_offline_throughput_default(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=500, + request_rate=float("inf"), + other_server_args=[], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_offline_throughput_default\n" + f"Output throughput: {res['output_throughput']:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(res["output_throughput"], 3050) + else: + self.assertGreater(res["output_throughput"], 3800) + + def test_offline_throughput_non_stream_small_batch_size(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=200, + request_rate=float("inf"), + other_server_args=["--max-running-requests", "10"], + dataset_name="sharegpt", + random_input_len=None, + random_output_len=None, + disable_stream=True, + need_warmup=True, + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_offline_throughput_non_stream_small_batch_size\n" + f"Output throughput: {res['output_throughput']:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(res["output_throughput"], 1000) + else: + self.assertGreater(res["output_throughput"], 1050) + + def test_offline_throughput_without_radix_cache(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=500, + request_rate=float("inf"), + other_server_args=["--disable-radix-cache"], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_offline_throughput_without_radix_cache\n" + f"Output throughput: {res['output_throughput']:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(res["output_throughput"], 3050) + else: + self.assertGreater(res["output_throughput"], 3800) + + def test_offline_throughput_without_chunked_prefill(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=500, + request_rate=float("inf"), + other_server_args=["--chunked-prefill-size", "-1"], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_offline_throughput_without_chunked_prefill\n" + f"Output throughput: {res['output_throughput']:.2f} token/s\n" + ) + self.assertGreater(res["output_throughput"], 2600) + + def test_offline_throughput_with_triton_attention_backend(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=500, + request_rate=float("inf"), + other_server_args=[ + "--attention-backend", + "triton", + "--context-length", + "8192", + ], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_offline_throughput_with_triton_attention_backend\n" + f"Output throughput: {res['output_throughput']:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(res["output_throughput"], 3500) + else: + self.assertGreater(res["output_throughput"], 3700) + + def test_offline_throughput_default_fp8(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST_FP8, + num_prompts=500, + request_rate=float("inf"), + other_server_args=[], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_offline_throughput_default_fp8\n" + f"Output throughput: {res['output_throughput']:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(res["output_throughput"], 3500) + else: + self.assertGreater(res["output_throughput"], 4300) + + def test_online_latency_default(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=100, + request_rate=1, + other_server_args=[], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_online_latency_default\n" + f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n" + ) + self.assertLess(res["median_e2e_latency_ms"], 11000) + if is_in_amd_ci(): + self.assertLess(res["median_ttft_ms"], 115) + else: + self.assertLess(res["median_ttft_ms"], 86) + self.assertLess(res["median_itl_ms"], 10) + + def test_vlm_offline_throughput(self): + res = run_bench_serving( + model=DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST, + num_prompts=200, + request_rate=float("inf"), + other_server_args=[ + "--mem-fraction-static", + "0.7", + ], + dataset_name="mmmu", + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_vlm_offline_throughput\n" + f"Output throughput: {res['output_throughput']:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(res["output_throughput"], 2000) + # TODO: not set yet, need AMD machine + else: + self.assertGreater(res["output_throughput"], 2500) + + def test_vlm_online_latency(self): + res = run_bench_serving( + model=DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST, + num_prompts=250, + request_rate=1, + other_server_args=[ + "--mem-fraction-static", + "0.7", + ], + dataset_name="mmmu", + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_vlm_online_latency\n" + f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n" + ) + self.assertLess(res["median_e2e_latency_ms"], 16500) + if is_in_amd_ci(): + self.assertLess(res["median_ttft_ms"], 150) + # TODO: not set yet, need AMD machine + else: + self.assertLess(res["median_ttft_ms"], 100) + self.assertLess(res["median_itl_ms"], 8) + + def test_lora_online_latency(self): + # TODO (lifuhuang): verify LoRA support in AMD. + if is_in_amd_ci(): + pass + + res = self._run_lora_latency_test(enable_background_task=False) + + if is_in_ci(): + write_github_step_summary( + f"### test_lora_online_latency\n" + f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n" + f"median_ttft_ms: {res['median_ttft_ms']:.2f} ms\n" + ) + self.assertLess(res["median_e2e_latency_ms"], 2400) + self.assertLess(res["median_ttft_ms"], 58) + + def test_lora_online_latency_with_concurrent_adapter_updates(self): + # TODO (lifuhuang): verify LoRA support in AMD. + if is_in_amd_ci(): + pass + + res = self._run_lora_latency_test(enable_background_task=True) + + if is_in_ci(): + write_github_step_summary( + f"### test_lora_online_latency\n" + f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n" + f"median_ttft_ms: {res['median_ttft_ms']:.2f} ms\n" + ) + self.assertLess(res["median_e2e_latency_ms"], 4000) + self.assertLess(res["median_ttft_ms"], 80) + + def _run_lora_latency_test(self, enable_background_task: bool): + """ + Run a latency test for LoRA with the specified background task setting. + """ + + async def lora_loader_unloader_task( + base_url: str, + start_event: asyncio.Event, + stop_event: asyncio.Event, + ): + """ + A background task that repeatedly loads and unloads a LoRA adapter. + """ + await start_event.wait() + + path_cycler = itertools.cycle( + [ + "pbevan11/llama-3.1-8b-ocr-correction", + "faridlazuarda/valadapt-llama-3.1-8B-it-chinese", + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + ] + ) + load_url = f"{base_url}/load_lora_adapter" + unload_url = f"{base_url}/unload_lora_adapter" + num_updates = 0 + + while not stop_event.is_set(): + # 1. Load the LoRA adapter + lora_path = next(path_cycler) + response = await asyncio.to_thread( + requests.post, + load_url, + json={"lora_name": lora_path, "lora_path": lora_path}, + ) + self.assertTrue( + response.ok, f"Failed to load LoRA adapter: {response.text}" + ) + num_updates += 1 + + if stop_event.is_set(): + break + + # Yield control to allow other tasks to run. + await asyncio.sleep(1) + + # 2. Unload the LoRA adapter + response = await asyncio.to_thread( + requests.post, + unload_url, + json={"lora_name": lora_path}, + ) + self.assertTrue( + response.ok, f"Failed to unload LoRA adapter: {response.text}" + ) + num_updates += 1 + + # Yield control to allow other tasks to run. + await asyncio.sleep(1) + + background_task = lora_loader_unloader_task if enable_background_task else None + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=400, + request_rate=8, + other_server_args=[ + "--enable-lora", + "--max-loras-per-batch", + "1", + "--disable-radix-cache", + "--random-seed", + "42", + "--mem-fraction-static", + "0.8", + "--lora-paths", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "--max-lora-rank", + "256", + ], + dataset_name="random", + random_input_len=256, + random_output_len=256, + lora_name=["Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"], + background_task=background_task, + ) + + return res + + def test_online_latency_eagle(self): + res = run_bench_serving( + model=DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + num_prompts=300, + request_rate=8, + sharegpt_context_len=3072, + disable_ignore_eos=True, + dataset_name="sharegpt", + other_server_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + "5", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "16", + "--mem-fraction-static", + "0.7", + ], + need_warmup=True, + seed=42, + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_online_latency_eagle\n" + f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n" + f"accept_length: {res['accept_length']:.2f} \n" + ) + if is_in_amd_ci(): + self.assertLess(res["median_e2e_latency_ms"], 1800) + else: + self.assertLess(res["median_e2e_latency_ms"], 900) + self.assertGreater(res["accept_length"], 3.0) + + def test_moe_offline_throughput_default(self): + res = run_bench_serving( + model=DEFAULT_MOE_MODEL_NAME_FOR_TEST, + num_prompts=300, + request_rate=float("inf"), + other_server_args=["--tp", "2"], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_moe_offline_throughput_default\n" + f"Output throughput: {res['output_throughput']:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(res["output_throughput"], 2100) + else: + self.assertGreater(res["output_throughput"], 2200) + + def test_moe_offline_throughput_without_radix_cache(self): + res = run_bench_serving( + model=DEFAULT_MOE_MODEL_NAME_FOR_TEST, + num_prompts=300, + request_rate=float("inf"), + other_server_args=["--tp", "2", "--disable-radix-cache"], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_moe_offline_throughput_without_radix_cache\n" + f"Output throughput: {res['output_throughput']:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(res["output_throughput"], 2100) + else: + self.assertGreater(res["output_throughput"], 2200) + + def test_pp_offline_throughput_default_decode(self): + res = run_bench_serving( + model=DEFAULT_MOE_MODEL_NAME_FOR_TEST, + num_prompts=1000, + request_rate=float("inf"), + random_input_len=1, + random_output_len=1024, + other_server_args=["--pp-size", "2"], + need_warmup=True, + seed=42, + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_pp_offline_throughput_default_decode\n" + f"Output throughput: {res['output_throughput']:.2f} token/s\n" + ) + self.assertGreater(res["output_throughput"], 6700) + + def test_pp_long_context_prefill(self): + res = run_bench_serving( + model="meta-llama/Llama-3.3-70B-Instruct", + num_prompts=4, + request_rate=float("inf"), + random_input_len=128000, + random_output_len=1, + dataset_name="random", + other_server_args=[ + "--quantization", + "fp8", + "--pp-size", + "2", + ], + need_warmup=False, + seed=42, + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_pp_long_context_latency_prefill\n" + f"input_throughput: {res['input_throughput']:.2f} ms\n" + ) + self.assertGreater(res["input_throughput"], 4000) + + def test_score_api_latency_throughput(self): + """Test score API latency and throughput performance""" + res = run_score_benchmark( + model=DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE, + num_requests=1000, + batch_size=10, + other_server_args=[], + need_warmup=True, + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_score_api_throughput\n" + f"Average latency: {res['avg_latency_ms']:.2f} ms\n" + f"P95 latency: {res['p95_latency_ms']:.2f} ms\n" + f"Score API throughput: {res['throughput']:.2f} req/s\n" + f"Successful requests: {res['successful_requests']}/{res['total_requests']}\n" + ) + + self.assertEqual(res["successful_requests"], res["total_requests"]) + self.assertLess(res["avg_latency_ms"], 48) + self.assertLess(res["p95_latency_ms"], 50) + self.assertGreater(res["throughput"], 20) + + def test_score_api_batch_scaling(self): + """Test score API performance with different batch sizes""" + batch_sizes = [10, 25, 50] + + for batch_size in batch_sizes: + res = run_score_benchmark( + model=DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE, + num_requests=500, + batch_size=batch_size, + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_score_api_batch_scaling_size_{batch_size}\n" + f"Batch size: {batch_size}\n" + f"Average latency: {res['avg_latency_ms']:.2f} ms\n" + f"P95 latency: {res['p95_latency_ms']:.2f} ms\n" + f"Throughput: {res['throughput']:.2f} req/s\n" + f"Successful requests: {res['successful_requests']}/{res['total_requests']}\n" + ) + + self.assertEqual(res["successful_requests"], res["total_requests"]) + bounds = { + 10: (45, 50), + 25: (50, 60), + 50: (60, 65), + } + avg_latency_bound, p95_latency_bound = bounds.get(batch_size, (60, 65)) + self.assertLess(res["avg_latency_ms"], avg_latency_bound) + self.assertLess(res["p95_latency_ms"], p95_latency_bound) + + def test_embeddings_api_latency_throughput(self): + """Test embeddings API latency and throughput performance""" + res = run_embeddings_benchmark( + model=DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST, + num_requests=1000, + batch_size=1, + input_tokens=500, + other_server_args=[], + need_warmup=True, + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_embeddings_api_throughput\n" + f"Average latency: {res['avg_latency_ms']:.2f} ms\n" + f"P95 latency: {res['p95_latency_ms']:.2f} ms\n" + f"Embeddings API throughput: {res['throughput']:.2f} req/s\n" + f"Successful requests: {res['successful_requests']}/{res['total_requests']}\n" + ) + + self.assertEqual(res["successful_requests"], res["total_requests"]) + # Bounds based on actual performance on 1xH100: avg=15ms, p95=15ms, throughput=67req/s + self.assertLess(res["avg_latency_ms"], 20) + self.assertLess(res["p95_latency_ms"], 25) + self.assertGreater(res["throughput"], 60) + + def test_embeddings_api_batch_scaling(self): + """Test embeddings API performance with different batch sizes""" + batch_sizes = [10, 25, 50] + + for batch_size in batch_sizes: + res = run_embeddings_benchmark( + model=DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST, + num_requests=500, + batch_size=batch_size, + input_tokens=500, + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_embeddings_api_batch_scaling_size_{batch_size}\n" + f"Batch size: {batch_size}\n" + f"Average latency: {res['avg_latency_ms']:.2f} ms\n" + f"P95 latency: {res['p95_latency_ms']:.2f} ms\n" + f"Throughput: {res['throughput']:.2f} req/s\n" + f"Successful requests: {res['successful_requests']}/{res['total_requests']}\n" + ) + + self.assertEqual(res["successful_requests"], res["total_requests"]) + bounds = { + 10: (60, 65), + 25: (115, 120), + 50: (190, 195), + } + avg_latency_bound, p95_latency_bound = bounds.get(batch_size, (250, 250)) + self.assertLess(res["avg_latency_ms"], avg_latency_bound) + self.assertLess(res["p95_latency_ms"], p95_latency_bound) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_bench_typebaseddispatcher.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_bench_typebaseddispatcher.py new file mode 100644 index 00000000..1c048b89 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_bench_typebaseddispatcher.py @@ -0,0 +1,261 @@ +import timeit +from typing import Any, Callable, List, Tuple, Type + +from sglang.utils import TypeBasedDispatcher + + +class TypeBasedDispatcherList: + def __init__(self, mapping: List[Tuple[Type, Callable]]): + self._mapping = mapping + self._fallback_fn = None + + def add_fallback_fn(self, fallback_fn: Callable): + self._fallback_fn = fallback_fn + + def __iadd__(self, other: "TypeBasedDispatcher"): + self._mapping.extend(other._mapping) + return self + + def __call__(self, obj: Any): + for ty, fn in self._mapping: + if isinstance(obj, ty): + return fn(obj) + + if self._fallback_fn is not None: + return self._fallback_fn(obj) + raise ValueError(f"Invalid object: {obj}") + + +def create_test_mapping(num_types=30): + types = [type(f"RequestType{i}", (), {}) for i in range(num_types)] + + def create_handler(i): + def handler(req): + return f"handler{i}" + + return handler + + handlers = [create_handler(i) for i in range(num_types)] + + return list(zip(types, handlers)) + + +def test_inheritance(): + print("\n" + "=" * 60) + print("test for inheritance") + print("=" * 60) + + class BaseRequest: + pass + + def base_handler(req): + return "base_handler" + + class DerivedRequest(BaseRequest): + pass + + mapping = [(BaseRequest, base_handler)] + dict_dispatcher = TypeBasedDispatcher(mapping) + + derived_obj = DerivedRequest() + expected = "base_handler" + + # This test will fail with the current implementation, but pass with the suggested MRO-based fix + result_dict = dict_dispatcher(derived_obj) + assert result_dict == expected, f"Expected '{expected}', but got '{result_dict}'" + print("Pass: dict dispatcher handles inheritance.") + + +def benchmark_with_inheritance(): + """Performance test with inheritance scenarios""" + print("\nBenchmarking with inheritance scenarios...") + + # Create type hierarchy with inheritance relationships + class BaseType: + pass + + class ChildType1(BaseType): + pass + + class ChildType2(BaseType): + pass + + class GrandChildType(ChildType1): + pass + + class UnrelatedType: + pass + + def base_handler(obj): + return "handled" + + mapping = [(BaseType, base_handler)] + dispatcher = TypeBasedDispatcher(mapping) + + test_cases = [ + BaseType(), + ChildType1(), + ChildType2(), + GrandChildType(), + UnrelatedType(), + ] + + # Test first call (includes MRO lookup) + first_call_times = [] + for case in test_cases: + if not isinstance(case, UnrelatedType): + time_taken = timeit.timeit(lambda: dispatcher(case), number=1000) + first_call_times.append(time_taken) + + # Test subsequent calls (using cache) + cached_call_times = [] + for case in test_cases: + if not isinstance(case, UnrelatedType): + time_taken = timeit.timeit(lambda: dispatcher(case), number=1000) + cached_call_times.append(time_taken) + + print( + f"First call (with MRO lookup): {sum(first_call_times)/len(first_call_times):.6f}s avg" + ) + print(f"Cached call: {sum(cached_call_times)/len(cached_call_times):.6f}s avg") + print(f"Caching improvement: {sum(first_call_times)/sum(cached_call_times):.2f}x") + + +def benchmark_dispatchers(): + mapping = create_test_mapping(30) + list_dispatcher = TypeBasedDispatcherList(mapping) + dist_dispatcher = TypeBasedDispatcher(mapping) + + test_cases = [] + for _, (ty, _) in enumerate(mapping): + test_cases.append(ty()) + + test_scenarios = [ + ("the first", [test_cases[0]] * 1000), + ("the middle", [test_cases[len(test_cases) // 2]] * 1000), + ("the last", [test_cases[-1]] * 1000), + ("the random", test_cases * 1000), + ] + + print("=" * 60) + print("TypeBasedDispatcher benchmark test") + print("=" * 60) + + for scenario_name, cases in test_scenarios: + print(f"\ntest scenario: {scenario_name}") + print(f"\ntest numbers: {len(cases)}") + + list_time = timeit.timeit( + lambda: [list_dispatcher(case) for case in cases], number=10 + ) + + dict_time = timeit.timeit( + lambda: [dist_dispatcher(case) for case in cases], number=10 + ) + + print(f"for list: {list_time:.4f} s") + print(f"for dict: {dict_time:.4f} s") + print(f"improvement: {list_time/dict_time:.2f} x") + print(f"time reduce: {(1-dict_time/list_time) * 100:.1f} %") + + +def test_memory_usage(): + import sys + + mapping = create_test_mapping(30) + list_dispatcher = TypeBasedDispatcherList(mapping) + dict_dispatcher = TypeBasedDispatcher(mapping) + + print("\n" + "=" * 60) + print("compare memory used:") + print("=" * 60) + + list_size = sys.getsizeof(list_dispatcher._mapping) + dict_size = sys.getsizeof(dict_dispatcher._mapping) + + print(f"memory used by list version: {list_size} bytes") + print(f"memory used by dict version: {dict_size} bytes") + print(f"compare memory used by the two version: {dict_size - list_size} bytes") + + +def test_edge_case(): + """test for edge case""" + print("\n" + "=" * 60) + print("test for edge case") + print("=" * 60) + + mapping = create_test_mapping(30) + list_dispatcher = TypeBasedDispatcherList(mapping) + dict_dispatcher = TypeBasedDispatcher(mapping) + + test_obj = mapping[0][0]() + result1 = list_dispatcher(test_obj) + result2 = dict_dispatcher(test_obj) + + assert result1 == result2 + print("Pass for normal test") + + class UnkownType: + pass + + try: + list_dispatcher(UnkownType()) + print("exception was thrown from list version as expected") + except ValueError: + print("exception thrown from list version was processed...") + + try: + dict_dispatcher(UnkownType()) + print("exception was thrown from dict version as expected") + except ValueError: + print("exception thrown from dict version was processed...") + + +def simulate_real_workload(): + """simulate real workload""" + + print("\n" + "=" * 60) + print("simulate real workload") + print("=" * 60) + + mapping = create_test_mapping(30) + + request_distribution = { + 0: 0.2, + 5: 0.3, + 10: 0.1, + 15: 0.15, + } + + list_dispatcher = TypeBasedDispatcherList(mapping) + dict_dispatcher = TypeBasedDispatcher(mapping) + + test_requests = [] + for idx, prob in request_distribution.items(): + count = int(1000 * prob) + test_requests.extend([mapping[idx][0]()] * count) + + remaining = 1000 - len(test_requests) + for i in range(remaining): + test_requests.append(mapping[i % len(mapping)][0]()) + + list_time = timeit.timeit( + lambda: [list_dispatcher(req) for req in test_requests], number=100 + ) + + dict_time = timeit.timeit( + lambda: [dict_dispatcher(req) for req in test_requests], number=100 + ) + + print(f"list version: {list_time:.4f} s") + print(f"dict version: {dict_time:.4f} s") + print(f"improvement: {list_time/dict_time:.2f} x") + + +if __name__ == "__main__": + benchmark_dispatchers() + test_memory_usage() + test_edge_case() + simulate_real_workload() + test_inheritance() + benchmark_with_inheritance() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_bnb.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_bnb.py new file mode 100644 index 00000000..4328d56b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_bnb.py @@ -0,0 +1,307 @@ +""" +Usage: +python3 -m unittest test_bnb.TestVisionModel.test_vlm +python3 -m unittest test_bnb.TestLanguageModel.test_mmlu +""" + +import multiprocessing as mp +import random +from concurrent.futures import ThreadPoolExecutor +from types import SimpleNamespace + +import openai + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) + +VISION_MODELS = [ + "unsloth/Qwen2.5-VL-7B-Instruct-bnb-4bit", + "unsloth/Qwen2-VL-7B-Instruct-bnb-4bit", + "unsloth/Llama-3.2-11B-Vision-Instruct-bnb-4bit", + "unsloth/Llama-3.2-11B-Vision-bnb-4bit", + "unsloth/gemma-3-4b-it-bnb-4bit", + "unsloth/gemma-3-4b-it-unsloth-bnb-4bit", +] +LANGUAGE_MODELS = [ + "unsloth/Qwen2.5-7B-Instruct-bnb-4bit", + "unsloth/Qwen2-7B-Instruct-bnb-4bit", + "unsloth/Llama-3.2-3B-Instruct-bnb-4bit", + "unsloth/gemma-3-1b-it-bnb-4bit", +] + +# image +IMAGE_MAN_IRONING_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/images/man_ironing_on_back_of_suv.png" +IMAGE_SGL_LOGO_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/images/sgl_logo.png" + +# video +VIDEO_JOBS_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/videos/jobs_presenting_ipod.mp4" + +# audio +AUDIO_TRUMP_SPEECH_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/audios/Trump_WEF_2018_10s.mp3" +AUDIO_BIRD_SONG_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/audios/bird_song.mp3" + + +def popen_launch_server_wrapper(base_url, model, other_args): + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + return process + + +class TestVisionModel(CustomTestCase): + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + cls.base_url = DEFAULT_URL_FOR_TEST + cls.base_url += "/v1" + cls.api_key = "sk-123456" + + def _run_single_image_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + }, + { + "type": "text", + "text": "Describe this image in a very short sentence.", + }, + ], + }, + ], + temperature=0, + ) + + assert response.choices[0].message.role == "assistant" + text = response.choices[0].message.content + assert isinstance(text, str) + # `driver` is for gemma-3-it + assert "man" in text or "person" or "driver" in text, text + assert "cab" in text or "taxi" in text or "SUV" in text, text + # MiniCPMO fails to recognize `iron`, but `hanging` + assert "iron" in text or "hang" in text, text + assert response.id + assert response.created + assert response.usage.prompt_tokens > 0 + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def _run_multi_turn_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + }, + { + "type": "text", + "text": "Describe this image in a very short sentence.", + }, + ], + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "There is a man at the back of a yellow cab ironing his clothes.", + } + ], + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "Repeat your previous answer."} + ], + }, + ], + temperature=0, + ) + + assert response.choices[0].message.role == "assistant" + text = response.choices[0].message.content + assert isinstance(text, str) + assert "man" in text or "cab" in text, text + assert response.id + assert response.created + assert response.usage.prompt_tokens > 0 + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def _run_multi_images_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + "modalities": "multi-images", + }, + { + "type": "image_url", + "image_url": {"url": IMAGE_SGL_LOGO_URL}, + "modalities": "multi-images", + }, + { + "type": "text", + "text": "I have two very different images. They are not related at all. " + "Please describe the first image in one sentence, and then describe the second image in another sentence.", + }, + ], + }, + ], + temperature=0, + ) + + assert response.choices[0].message.role == "assistant" + text = response.choices[0].message.content + assert isinstance(text, str) + print("-" * 30) + print(f"Multi images response:\n{text}") + print("-" * 30) + assert "man" in text or "cab" in text or "SUV" in text or "taxi" in text, text + assert "logo" in text or '"S"' in text or "SG" in text, text + assert response.id + assert response.created + assert response.usage.prompt_tokens > 0 + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def run_decode_with_image(self, image_id): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + content = [] + if image_id == 0: + content.append( + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + } + ) + elif image_id == 1: + content.append( + { + "type": "image_url", + "image_url": {"url": IMAGE_SGL_LOGO_URL}, + } + ) + else: + pass + + content.append( + { + "type": "text", + "text": "Describe this image in a very short sentence.", + } + ) + + response = client.chat.completions.create( + model="default", + messages=[ + {"role": "user", "content": content}, + ], + temperature=0, + ) + + assert response.choices[0].message.role == "assistant" + text = response.choices[0].message.content + assert isinstance(text, str) + + def _run_test_mixed_batch(self): + image_ids = [0, 1, 2] * 4 + with ThreadPoolExecutor(4) as executor: + list(executor.map(self.run_decode_with_image, image_ids)) + + def test_vlm(self): + models_to_test = VISION_MODELS + + if is_in_ci(): + models_to_test = [random.choice(VISION_MODELS)] + + for model in models_to_test: + with self.subTest(model=model): + other_args = [ + "--mem-fraction-static", + "0.6", + "--load-format", + "bitsandbytes", + "--enable-multimodal", + ] + try: + process = popen_launch_server_wrapper( + DEFAULT_URL_FOR_TEST, model, other_args + ) + self._run_test_mixed_batch() + self._run_multi_images_chat_completion() + self._run_multi_turn_chat_completion() + self._run_single_image_chat_completion() + finally: + kill_process_tree(process.pid) + + +class TestLanguageModel(CustomTestCase): + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + cls.base_url = DEFAULT_URL_FOR_TEST + # cls.base_url += "/v1" + cls.api_key = "sk-123456" + + def test_mmlu(self): + models_to_test = LANGUAGE_MODELS + + if is_in_ci(): + models_to_test = [random.choice(LANGUAGE_MODELS)] + + for model in models_to_test: + with self.subTest(model=model): + other_args = [ + "--mem-fraction-static", + "0.6", + "--load-format", + "bitsandbytes", + ] + try: + process = popen_launch_server_wrapper( + DEFAULT_URL_FOR_TEST, model, other_args + ) + args = SimpleNamespace( + base_url=self.base_url, + model=model, + eval_name="mmlu", + num_examples=32, + num_threads=16, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.3) + finally: + kill_process_tree(process.pid) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_build_eagle_tree.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_build_eagle_tree.py new file mode 100644 index 00000000..5372393d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_build_eagle_tree.py @@ -0,0 +1,308 @@ +import unittest + +import torch + +from sglang.srt.speculative.eagle_utils import ( + build_tree_kernel_efficient, + organize_draft_results, +) + + +class TestBuildEagleTree(unittest.TestCase): + """Unit tests for build_eagle_tree functionality.""" + + def test_build_tree_kernel_efficient(self): + """Test the build_tree_kernel_efficient function with known inputs and expected outputs.""" + verified_id = torch.tensor([29974, 13], device="cuda", dtype=torch.int32) + score_list = [ + torch.tensor( + [ + [[7.1127e-01, 2.8292e-01, 2.2995e-03, 1.7357e-03]], + [[9.7476e-01, 2.2219e-02, 6.5031e-04, 1.3212e-04]], + ], + dtype=torch.float32, + device="cuda", + ), + torch.tensor( + [ + [ + [6.9142e-01, 1.2863e-02, 1.6873e-03, 1.1871e-03], + [2.4787e-01, 1.8818e-02, 1.4204e-02, 9.2235e-04], + [2.2971e-03, 1.6700e-06, 1.8737e-07, 8.3146e-08], + [1.2771e-03, 2.4374e-04, 1.7832e-04, 1.1947e-05], + ], + [ + [8.4832e-02, 6.6068e-02, 5.8304e-02, 5.7851e-02], + [2.3616e-03, 1.1243e-03, 5.4368e-04, 2.7768e-04], + [2.5286e-04, 1.5578e-04, 2.8817e-05, 1.2888e-05], + [1.2834e-04, 2.5417e-06, 1.1279e-06, 1.6088e-08], + ], + ], + dtype=torch.float32, + device="cuda", + ), + torch.tensor( + [ + [ + [6.6438e-01, 2.6997e-02, 2.4236e-05, 4.0821e-06], + [2.4402e-01, 2.8409e-03, 5.0935e-04, 2.9022e-04], + [1.6178e-02, 2.0567e-03, 4.5892e-04, 3.0034e-05], + [1.3023e-02, 5.0497e-04, 3.6371e-04, 8.7750e-05], + ], + [ + [2.3263e-02, 2.0054e-02, 9.3990e-03, 2.7783e-03], + [6.4156e-02, 5.5506e-04, 1.0429e-04, 9.7211e-05], + [4.9950e-02, 5.0630e-03, 9.0068e-04, 3.3656e-04], + [7.5817e-03, 8.5731e-04, 6.9972e-04, 6.0793e-04], + ], + ], + dtype=torch.float32, + device="cuda", + ), + torch.tensor( + [ + [ + [6.6420e-01, 1.0525e-04, 6.5864e-05, 1.2253e-06], + [1.3019e-01, 1.0461e-01, 5.2083e-03, 1.6777e-03], + [2.0103e-02, 6.7335e-03, 1.2625e-04, 1.0364e-05], + [1.5142e-02, 7.0819e-04, 9.6595e-05, 8.7951e-05], + ], + [ + [5.8608e-02, 1.8840e-03, 7.8535e-04, 4.4400e-04], + [1.2185e-02, 2.0684e-03, 1.7418e-03, 1.4327e-03], + [6.2455e-03, 6.1487e-03, 2.6862e-03, 1.8034e-03], + [1.8590e-03, 1.6151e-03, 1.2481e-03, 3.6038e-04], + ], + ], + dtype=torch.float32, + device="cuda", + ), + ] + token_list = [ + torch.tensor( + [[29896, 29906, 29900, 29945], [13, 2, 29871, 28956]], + dtype=torch.int64, + device="cuda", + ), + torch.tensor( + [ + [ + 29889, + 29974, + 29945, + 29900, + 29974, + 29922, + 29930, + 29958, + 29889, + 29974, + 29930, + 29945, + 29974, + 29922, + 29930, + 29958, + ], + [ + 22550, + 4136, + 16492, + 8439, + 29871, + 2, + 3001, + 13, + 2, + 13, + 29906, + 29946, + 2, + 13, + 29871, + 259, + ], + ], + device="cuda", + ), + torch.tensor( + [ + [ + 29946, + 29945, + 29953, + 29906, + 29896, + 29945, + 29900, + 29906, + 29896, + 29945, + 29906, + 29953, + 29896, + 29945, + 29906, + 29946, + ], + [ + 29871, + 2, + 29901, + 29889, + 29871, + 2, + 395, + 259, + 29901, + 29871, + 2, + 29889, + 3001, + 1234, + 7146, + 2186, + ], + ], + device="cuda", + ), + torch.tensor( + [ + [ + 29946, + 29974, + 29945, + 29930, + 29889, + 29922, + 29974, + 29930, + 29974, + 29946, + 29930, + 29922, + 29889, + 29974, + 29945, + 29922, + ], + [ + 29941, + 29906, + 2, + 29946, + 29871, + 450, + 319, + 14990, + 29946, + 29941, + 2, + 29906, + 29871, + 2, + 3001, + 13, + ], + ], + device="cuda", + ), + ] + parents_list = [ + torch.tensor( + [[-1, 0, 1, 2, 3], [-1, 0, 1, 2, 3]], dtype=torch.int64, device="cuda" + ), + torch.tensor( + [[4, 8, 9, 10], [4, 5, 6, 7]], dtype=torch.int64, device="cuda" + ), + torch.tensor( + [[20, 24, 21, 28], [24, 28, 20, 21]], dtype=torch.int64, device="cuda" + ), + torch.tensor( + [[36, 40, 41, 44], [36, 40, 44, 45]], dtype=torch.int64, device="cuda" + ), + ] + seq_lens = torch.tensor([5, 10], dtype=torch.int64, device="cuda") + topk = 4 + depth = 4 + num_draft_token = 8 + + parent_list, top_scores_index, draft_tokens = organize_draft_results( + score_list, token_list, parents_list, num_draft_token + ) + + ( + tree_mask, + position, + retrieve_index, + retrieve_next_token, + retrieve_next_sibling, + draft_tokens, + ) = build_tree_kernel_efficient( + verified_id=verified_id, + parent_list=parent_list, + top_scores_index=top_scores_index, + draft_tokens=draft_tokens, + seq_lens=seq_lens, + seq_lens_sum=torch.sum(seq_lens).item(), + topk=topk, + spec_steps=depth, + num_verify_tokens=num_draft_token, + ) + + # Verify expected outputs + self.assertEqual( + position.tolist(), + [5, 6, 6, 7, 7, 8, 8, 9, 10, 11, 12, 12, 12, 12, 13, 14], + "Position tensor does not match expected values", + ) + self.assertEqual( + retrieve_index.tolist(), + [ + [0, 1, 2, 3, 4, 5, 6, 7], + [8, 9, 10, 11, 12, 13, 14, 15], + ], + "Retrieve index tensor does not match expected values", + ) + self.assertEqual( + retrieve_next_token.tolist(), + [ + [1, 3, 4, 5, 6, 7, -1, -1], + [1, 2, -1, 6, -1, -1, 7, -1], + ], + "Retrieve next token tensor does not match expected values", + ) + self.assertEqual( + retrieve_next_sibling.tolist(), + [ + [-1, 2, -1, -1, -1, -1, -1, -1], + [-1, -1, 3, 4, 5, -1, -1, -1], + ], + "Retrieve next sibling tensor does not match expected values", + ) + self.assertEqual( + draft_tokens.tolist(), + [ + 29974, + 29896, + 29906, + 29889, + 29974, + 29946, + 29896, + 29946, + 13, + 13, + 22550, + 4136, + 16492, + 8439, + 29871, + 29941, + ], + "Draft tokens tensor does not match expected values", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_chunked_prefill.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_chunked_prefill.py new file mode 100644 index 00000000..fbdc87c5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_chunked_prefill.py @@ -0,0 +1,31 @@ +""" +python3 -m unittest test_chunked_prefill.TestChunkedPrefill.test_mixed_chunked_prefill_without_radix_cache +""" + +import unittest + +from sglang.test.test_utils import CustomTestCase, run_mmlu_test, run_mulit_request_test + + +class TestChunkedPrefill(CustomTestCase): + def test_chunked_prefill(self): + run_mmlu_test(disable_radix_cache=False, enable_mixed_chunk=False) + + def test_mixed_chunked_prefill(self): + run_mmlu_test(disable_radix_cache=False, enable_mixed_chunk=True) + + def test_chunked_prefill_without_radix_cache(self): + run_mmlu_test(disable_radix_cache=True, enable_mixed_chunk=False) + + def test_mixed_chunked_prefill_without_radix_cache(self): + run_mmlu_test(disable_radix_cache=True, enable_mixed_chunk=True) + + def test_mixed_chunked_prefill_multi_requests(self): + run_mulit_request_test( + enable_mixed_chunk=True, + chunked_prefill_size=2048, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_constrained_decoding.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_constrained_decoding.py new file mode 100644 index 00000000..9b8b42a9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_constrained_decoding.py @@ -0,0 +1,69 @@ +import unittest + +from sglang.srt.utils import kill_process_tree +from sglang.test.kits.ebnf_constrained_kit import TestEBNFConstrainedMinxin +from sglang.test.kits.json_constrained_kit import TestJSONConstrainedMixin +from sglang.test.kits.regex_constrained_kit import TestRegexConstrainedMixin +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class ServerWithGrammar(CustomTestCase): + backend = "xgrammar" + disable_overlap = False + + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + launch_args = [ + "--max-running-requests", + "10", + "--grammar-backend", + cls.backend, + ] + + if cls.disable_overlap: + launch_args += ["--disable-overlap-schedule"] + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=launch_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +class TestXGrammarBackend( + ServerWithGrammar, + TestJSONConstrainedMixin, + TestEBNFConstrainedMinxin, + TestRegexConstrainedMixin, +): + backend = "xgrammar" + + +class TestOutlinesBackend(ServerWithGrammar, TestJSONConstrainedMixin): + backend = "outlines" + + +class TestLLGuidanceBackend( + ServerWithGrammar, + TestJSONConstrainedMixin, + TestEBNFConstrainedMinxin, + TestRegexConstrainedMixin, +): + backend = "llguidance" + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_constrained_decoding_spec_reasoning.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_constrained_decoding_spec_reasoning.py new file mode 100644 index 00000000..1370cd0d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_constrained_decoding_spec_reasoning.py @@ -0,0 +1,100 @@ +import json +import unittest + +import openai + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class ServerWithGrammar(CustomTestCase): + json_schema = json.dumps( + { + "type": "object", + "properties": { + "name": {"type": "string", "pattern": "^[\\w]+$"}, + "population": {"type": "integer"}, + "languages": { + "type": "array", + "items": {"type": "string"}, + "minItems": 1, + }, + "has_held_olympics": {"type": "boolean"}, + }, + "required": ["name", "population", "languages", "has_held_olympics"], + "additionalProperties": False, + } + ) + + @classmethod + def setUpClass(cls): + cls.model = "openai/gpt-oss-120b" + cls.base_url = DEFAULT_URL_FOR_TEST + launch_args = [ + "--trust-remote-code", + "--tp=2", + "--reasoning-parser=gpt-oss", + "--speculative-algorithm=EAGLE3", + "--speculative-draft-model-path=lmsys/EAGLE3-gpt-oss-120b-bf16", + "--speculative-num-steps=5", + "--speculative-eagle-topk=4", + "--speculative-num-draft-tokens=8", + ] + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=launch_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_json_openai(self): + client = openai.Client(api_key="EMPTY", base_url=f"{self.base_url}/v1") + + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + { + "role": "user", + "content": "Introduce the capital of France. Return in a JSON format. " + "The JSON Schema is: " + json.dumps(self.json_schema), + }, + ], + temperature=0, + max_tokens=1024, + response_format={ + "type": "json_schema", + "json_schema": {"name": "foo", "schema": json.loads(self.json_schema)}, + }, + ) + text = response.choices[0].message.content + + print("\n=== Reasoning Content ===") + reasoning_content = response.choices[0].message.reasoning_content + assert reasoning_content is not None and len(reasoning_content) > 0 + print(reasoning_content) + + try: + js_obj = json.loads(text) + print("\n=== Parsed JSON Content ===") + print(json.dumps(js_obj)) + except (TypeError, json.decoder.JSONDecodeError): + print("JSONDecodeError", text) + raise + + self.assertIsInstance(js_obj["name"], str) + self.assertIsInstance(js_obj["population"], int) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_create_kvindices.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_create_kvindices.py new file mode 100644 index 00000000..7e63fd82 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_create_kvindices.py @@ -0,0 +1,74 @@ +import unittest + +import numpy as np +import torch + +from sglang.srt.layers.attention.utils import create_flashinfer_kv_indices_triton +from sglang.test.test_utils import CustomTestCase + + +class TestCreateKvIndices(CustomTestCase): + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _run_test(self, batch, max_batch, max_context_len): + req_to_token = torch.arange( + max_batch * max_context_len, dtype=torch.int32, device="cuda" + ).reshape((max_batch, max_context_len)) + req_pool_indices = torch.tensor( + torch.from_numpy( + np.random.choice(range(max_batch), size=batch, replace=False) + ), + dtype=torch.int32, + device="cuda", + ) + paged_kernel_lens = torch.tensor( + torch.from_numpy( + np.random.choice(range(max_context_len), size=batch, replace=False) + ), + dtype=torch.int32, + device="cuda", + ) + + kv_indptr = torch.zeros((batch + 1,), dtype=torch.int32, device="cuda") + kv_indptr[1:] = torch.cumsum(paged_kernel_lens, dim=0) + + # ref + req_pool_indices_cpu = req_pool_indices.cpu().numpy() + paged_kernel_lens_cpu = paged_kernel_lens.cpu().numpy() + kv_indices_ref = torch.cat( + [ + req_to_token[req_pool_indices_cpu[i], : paged_kernel_lens_cpu[i]] + for i in range(batch) + ], + dim=0, + ).contiguous() + + # triton + kv_indices_triton = torch.empty(kv_indptr[-1], dtype=torch.int32, device="cuda") + create_flashinfer_kv_indices_triton[(batch,)]( + req_to_token, + req_pool_indices, + paged_kernel_lens, + kv_indptr, + None, + kv_indices_triton, + req_to_token.size(1), + ) + + # Check + self.assertTrue(torch.equal(kv_indices_ref, kv_indices_triton)) + + def test_create_kvindices(self): + BATCH = [1, 37, 1786] + MAX_BATCH = 4096 + MAX_CONTEXT_LEN = 4096 + for batch in BATCH: + self._run_test(batch, MAX_BATCH, MAX_CONTEXT_LEN) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_cutedsl_moe.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_cutedsl_moe.py new file mode 100644 index 00000000..d60ce723 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_cutedsl_moe.py @@ -0,0 +1,482 @@ +# SPDX-License-Identifier: Apache-2.0 +import unittest +from typing import Callable + +import torch +from flashinfer import fp4_quantize, scaled_fp4_grouped_quantize +from sgl_kernel import scaled_fp4_quant +from torch.nn import functional as F + +from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.moe.flashinfer_cutedsl_moe import flashinfer_cutedsl_moe_masked +from sglang.srt.layers.moe.topk import TopKConfig, select_experts + +SKIP_TEST = torch.cuda.get_device_capability() < (10, 0) +SKIP_REASON = "Nvfp4 Requires compute capability of 10 or above." + +kE2M1ToFloat = torch.tensor( + [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0], dtype=torch.float32 +) + +FLOAT8_E4M3_MAX = 448.0 +FLOAT4_E2M1_MAX = 6.0 + + +def convert_swizzled_to_linear(a_sf_swizzled: torch.Tensor, m, k, block_size): + m_tiles = (m + 128 - 1) // 128 + f = block_size * 4 + k_tiles = (k + f - 1) // f + tmp = torch.reshape(a_sf_swizzled, (1, m_tiles, k_tiles, 32, 4, 4)) + tmp = torch.permute(tmp, (0, 1, 4, 3, 2, 5)) + out = tmp.reshape(m_tiles * 128, k_tiles * f // block_size) + return out[0:m, 0:k] + + +def dequantize_nvfp4_to_dtype( + tensor_fp4, tensor_sf, global_scale, dtype, device, block_size=16 +): + """Dequantize the fp4 tensor back to high precision.""" + # Two fp4 values are packed into one uint8. + assert tensor_fp4.dtype == torch.uint8 + m, packed_k = tensor_fp4.shape + k = packed_k * 2 + tensor_f32 = break_fp4_bytes(tensor_fp4, dtype) + tensor_f32 = tensor_f32.reshape(m, k // block_size, block_size) + tensor_sf = tensor_sf.view(torch.float8_e4m3fn) + tensor_sf = convert_swizzled_to_linear(tensor_sf, m, k, block_size) + tensor_sf_dtype = tensor_sf.to(torch.float32) / global_scale + + # scale the tensor + out = (tensor_f32 * tensor_sf_dtype.unsqueeze(-1)).reshape(m, k) + return out.to(dtype=dtype) + + +def break_fp4_bytes(a, dtype): + assert a.dtype == torch.uint8 + m, n = a.shape + + # Vectorized nibble processing + a_flat = a.flatten() + high = (a_flat & 0xF0) >> 4 # Upper nibbles + low = a_flat & 0x0F # Lower nibbles + + # Combine nibbles for batch processing + combined = torch.stack((low, high), dim=1).flatten() + + # Vectorized sign and magnitude extraction + signs = (combined & 0x08).to(torch.bool) # Sign bits + abs_vals = (combined & 0x07).to(torch.long) # Magnitude indices + + # Device-aware lookup and sign application + kE2M1 = kE2M1ToFloat.to(device=a.device) + values = kE2M1[abs_vals] * torch.where(signs, -1.0, 1.0) + + # Reshape to final form + return values.reshape(m, n * 2).to(dtype=dtype) + + +def compute_routing(router_logits: torch.Tensor, top_k: int): + routing_weights = torch.softmax(router_logits, dim=1, dtype=torch.float) + routing_weights, selected_experts = torch.topk(routing_weights, top_k, dim=-1) + routing_weights /= routing_weights.sum(dim=-1, keepdim=True) + routing_weights = routing_weights.float() + return routing_weights, selected_experts + + +def prepare_inputs( + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + num_experts: int, + topk: int, +): + routing_weights, topk_idx = compute_routing(router_logits, topk) + + masked_m = [] + for i in range(num_experts): + mask = topk_idx.view(-1) == i + masked_m.append(mask.sum()) + + masked_m = torch.tensor(masked_m, dtype=torch.int32) + hidden_states_3d = torch.empty( + (num_experts, max(masked_m), hidden_states.shape[1]), dtype=hidden_states.dtype + ) + for i in range(num_experts): + hidden_states_3d[i, : masked_m[i], :] = hidden_states[topk_idx.view(-1) == i] + + return hidden_states_3d, masked_m, topk_idx, routing_weights + + +MNK_FACTORS = [ + (2, 1024, 1024), + (2, 1024, 1536), + (2, 3072, 1024), + (2, 3072, 1536), + (64, 1024, 1024), + (64, 1024, 1536), + (64, 3072, 1024), + (64, 2048, 1024), + (224, 1024, 1024), + (224, 1024, 1536), +] + + +# Reference implementation of torch_moe +def torch_moe(a, w1, w2, score, topk, expert_map): + B, D = a.shape + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + if expert_map is not None: + topk_ids = expert_map[topk_ids] + for i in range(w1.shape[0]): + mask = topk_ids == i + if mask.sum(): + out[mask] = SiluAndMul()(a[mask] @ w1[i].transpose(0, 1)) @ w2[i].transpose( + 0, 1 + ) + return ( + out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype) + ).sum(dim=1) + + +def torch_moe_nvfp4(a, w1, w2, topk, topk_weight, topk_ids): + B, D = a.shape + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) + + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + + for i in range(w1.shape[0]): + mask = topk_ids == i + if mask.sum(): + m = w1[i].shape[0] + assert m % 2 == 0 + # Note: w1 and w3 are swapped! + w3_expert, w1_expert = w1[i][m // 2 :, :], w1[i][: m // 2, :] + inter = F.silu(a[mask] @ w1_expert.t()) * (a[mask] @ w3_expert.t()) + inter_gs = torch.tensor(1.0).cuda() + inter_q, inter_blockscale = fp4_quantize(inter, inter_gs) + inter = dequantize_nvfp4_to_dtype( + inter_q, + inter_blockscale, + inter_gs, + dtype=inter.dtype, + device=inter.device, + block_size=16, + ).cuda() + out[mask] = inter @ w2[i].transpose(0, 1) + return ( + out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype) + ).sum(dim=1) + + +def check_moe( + m: int, + n: int, + k: int, + e: int, + topk: int, + dtype: torch.dtype, + moe_impl: Callable, + flip_w13: bool, +): + torch.manual_seed(7) + a = torch.randn((m, k), device="cuda", dtype=dtype) / 10 + w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10 + quant_blocksize = 16 + round_up = lambda x, y: (x + y - 1) // y * y + sf_w1_2n = round_up(2 * n, 128) + sf_w1_k = round_up(k // quant_blocksize, 4) + w1_blockscale = torch.empty( + (e, sf_w1_2n, sf_w1_k), device="cuda", dtype=torch.float8_e4m3fn + ) + + w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10 + sf_w2_k = round_up(k, 128) + sf_w2_n = round_up(n // quant_blocksize, 4) + w2_blockscale = torch.empty( + (e, sf_w2_k, sf_w2_n), device="cuda", dtype=torch.float8_e4m3fn + ) + + w1_q = torch.empty((e, 2 * n, k // 2), device="cuda", dtype=torch.uint8) + w2_q = torch.empty((e, k, n // 2), device="cuda", dtype=torch.uint8) + w1_gs = torch.empty((e,), device="cuda", dtype=torch.float32) + w2_gs = torch.empty((e,), device="cuda", dtype=torch.float32) + + for expert in range(e): + w1_amax = torch.abs(w1).max().to(torch.float32) + w2_amax = torch.abs(w2).max().to(torch.float32) + w1_gs[expert] = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / w1_amax + w2_gs[expert] = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / w2_amax + + w1_q[expert], w1_blockscale[expert] = scaled_fp4_quant( + w1[expert], w1_gs[expert] + ) + + w2_q[expert], w2_blockscale[expert] = scaled_fp4_quant( + w2[expert], w2_gs[expert] + ) + + score = torch.randn((m, e), device="cuda", dtype=dtype) + + topk_output = select_experts( + hidden_states=a, + router_logits=score, + topk_config=TopKConfig(top_k=topk, renormalize=False), + ) + topk_weights, topk_ids, _ = topk_output + + a1_gs = torch.ones((e,), device="cuda", dtype=torch.float32) + a2_gs = torch.ones((e,), device="cuda", dtype=torch.float32) + test_output = moe_impl( + a=a, + topk_weights=topk_weights, + topk_ids=topk_ids, + w1_q=w1_q, + w2_q=w2_q, + a1_gs=a1_gs, + w1_blockscale=w1_blockscale, + w1_alphas=(1 / w1_gs), + a2_gs=a2_gs, + w2_blockscale=w2_blockscale, + w2_alphas=(1 / w2_gs), + ) + + # Reference check: + a_global_scale = ( + (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(a.flatten(), dim=-1) + ).to(torch.float32) + a_fp4, a_scale_interleaved = scaled_fp4_quant(a, a_global_scale) + _, m_k = a_fp4.shape + a_in_dtype = dequantize_nvfp4_to_dtype( + a_fp4, + a_scale_interleaved, + a_global_scale, + dtype=a.dtype, + device=a.device, + block_size=quant_blocksize, + ) + + w1_d = torch.empty((e, 2 * n, k), device="cuda", dtype=dtype) + w2_d = torch.empty((e, k, n), device="cuda", dtype=dtype) + + for idx in range(0, e): + w1_d[idx] = dequantize_nvfp4_to_dtype( + w1_q[idx], + w1_blockscale[idx], + w1_gs[idx], + dtype=w1.dtype, + device=w1.device, + block_size=quant_blocksize, + ) + w2_d[idx] = dequantize_nvfp4_to_dtype( + w2_q[idx], + w2_blockscale[idx], + w2_gs[idx], + dtype=w2.dtype, + device=w2.device, + block_size=quant_blocksize, + ) + + if flip_w13: + dim = -2 + size = w1_d.size(dim) + assert size % 2 == 0, f"Expected even size in dim {dim}, got {size}" + half = size // 2 + # Reorder weight + w1, w3 = w1_d.split(half, dim=dim) + w1_d = torch.cat([w3, w1], dim=dim).contiguous() + + torch_output = torch_moe(a_in_dtype, w1_d, w2_d, score, topk, None) + + torch.testing.assert_close(torch_output, test_output, atol=1e-1, rtol=1e-1) + + +class TestFlashinferCutedslMoe(unittest.TestCase): + @unittest.skipIf(SKIP_TEST, SKIP_REASON) + def test_flashinfer_cutedsl_moe_masked(self): + # Test parameters + test_cases = [ + (2, 128, 256, 1), + (2, 128, 256, 2), + (2, 128, 256, 4), + (16, 128, 512, 1), + (16, 128, 512, 2), + (16, 128, 512, 4), + ] + + for bs, hidden_dim, inter_dim, topk in test_cases: + with self.subTest( + bs=bs, hidden_dim=hidden_dim, inter_dim=inter_dim, topk=topk + ): + print( + f"Testing with bs={bs}, hidden_dim={hidden_dim}, inter_dim={inter_dim}, topk={topk}" + ) + with torch.inference_mode(): + torch.manual_seed(42) + device = "cuda" + dtype = torch.bfloat16 + num_experts = 8 + hidden_states = ( + torch.randn(bs, hidden_dim, dtype=torch.bfloat16, device=device) + / 5.0 + ) + w1 = ( + torch.randn( + num_experts, + 2 * inter_dim, + hidden_dim, + dtype=torch.bfloat16, + device=device, + ) + / 10.0 + ) + w2 = ( + torch.randn( + num_experts, + hidden_dim, + inter_dim, + dtype=torch.bfloat16, + device=device, + ) + / 10.0 + ) + router_logits = torch.randn(bs, num_experts, dtype=torch.float32) + + hidden_states_expanded = ( + hidden_states.view(bs, -1, hidden_dim) + .repeat(1, topk, 1) + .reshape(-1, hidden_dim) + ) + hidden_states_3d, masked_m, topk_idx, routing_weights = ( + prepare_inputs( + hidden_states_expanded, router_logits, num_experts, topk + ) + ) + + w1_amax = w1.abs().amax(dim=(1, 2)).to(torch.float32).to(w1.device) + w2_amax = w2.abs().amax(dim=(1, 2)).to(torch.float32).to(w2.device) + input_global_scale = torch.ones( + (num_experts,), dtype=torch.float32, device=hidden_states.device + ) + + w1_global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / w1_amax + w2_global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / w2_amax + a2_global_scale = torch.ones( + (num_experts,), dtype=torch.float32, device=hidden_states.device + ) # assume intermediate scale is 1.0 + + w1_fp4, w1_blockscale = scaled_fp4_grouped_quantize( + w1, + torch.ones(num_experts, dtype=torch.int32, device=w1.device) + * 2 + * inter_dim, + w1_global_scale, + ) + w2_fp4, w2_blockscale = scaled_fp4_grouped_quantize( + w2, + torch.ones(num_experts, dtype=torch.int32, device=w2.device) + * hidden_dim, + w2_global_scale, + ) + + w1_alpha = 1.0 / (input_global_scale * w1_global_scale) + w2_alpha = 1.0 / (a2_global_scale * w2_global_scale) + + out = flashinfer_cutedsl_moe_masked( + (hidden_states_3d.to(hidden_states.device), None), + input_global_scale, + w1_fp4.permute(2, 0, 1), + w1_blockscale, + w1_alpha, + w2_fp4.permute(2, 0, 1), + a2_global_scale, + w2_blockscale, + w2_alpha, + masked_m.to(hidden_states.device), + ) + + # reference + a_fp4, a_scale_interleaved = fp4_quantize( + hidden_states, input_global_scale + ) + a_in_dtype = dequantize_nvfp4_to_dtype( + a_fp4, + a_scale_interleaved, + input_global_scale, + dtype=hidden_states.dtype, + device=hidden_states.device, + block_size=16, + ) + w1_d = torch.empty( + (num_experts, 2 * inter_dim, hidden_dim), + device=w1.device, + dtype=w1.dtype, + ) + w2_d = torch.empty( + (num_experts, hidden_dim, inter_dim), + device=w2.device, + dtype=w2.dtype, + ) + + for idx in range(0, num_experts): + w1_fp4_sliced, w1_blockscale_sliced = fp4_quantize( + w1[idx], w1_global_scale[idx] + ) + w2_fp4_sliced, w2_blockscale_sliced = fp4_quantize( + w2[idx], w2_global_scale[idx] + ) + w1_d[idx] = dequantize_nvfp4_to_dtype( + w1_fp4_sliced, + w1_blockscale_sliced, + w1_global_scale[idx], + dtype=w1.dtype, + device=w1.device, + block_size=16, + ) + w2_d[idx] = dequantize_nvfp4_to_dtype( + w2_fp4_sliced, + w2_blockscale_sliced, + w2_global_scale[idx], + dtype=w2.dtype, + device=w2.device, + block_size=16, + ) + + ref_output = torch_moe_nvfp4( + a_in_dtype, + w1_d, + w2_d, + topk, + routing_weights.to(a_in_dtype.device), + topk_idx.to(a_in_dtype.device), + ) + out_weighted = torch.zeros_like( + ref_output, device=out.device, dtype=out.dtype + ) + + positions = torch.nonzero(masked_m[topk_idx], as_tuple=False) + rows, cols = positions[:, 0], positions[:, 1] + experts = topk_idx[rows, cols] + for i in range(num_experts): + mask = experts == i + if mask.any(): + idx = torch.nonzero(mask, as_tuple=False).squeeze(-1) + r, c = rows[idx], cols[idx] + out_weighted[r] += out[i, : len(r), :] * routing_weights[ + r, c + ].to(out.device).unsqueeze(-1) + torch.testing.assert_close( + out_weighted.cpu(), ref_output.cpu(), atol=5e-2, rtol=5e-2 + ) + print( + f"Test passed with bs={bs}, hidden_dim={hidden_dim}, inter_dim={inter_dim}, topk={topk}" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_data_parallelism.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_data_parallelism.py new file mode 100644 index 00000000..e54e3a5c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_data_parallelism.py @@ -0,0 +1,78 @@ +import time +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestDataParallelism(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--dp", 2], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + def test_update_weight(self): + response = requests.post( + self.base_url + "/update_weights_from_disk", + json={"model_path": DEFAULT_MODEL_NAME_FOR_TEST}, + ) + + # check if the response is 200 + assert response.status_code == 200 + + # pause a few seconds then send again + time.sleep(1) + + response = requests.post( + self.base_url + "/update_weights_from_disk", + json={"model_path": DEFAULT_MODEL_NAME_FOR_TEST}, + ) + + # check if the response is 200 + assert response.status_code == 200 + + def test_get_memory_pool_size(self): + # use `get_server_info` instead since `get_memory_pool_size` is merged into `get_server_info` + response = requests.get(self.base_url + "/get_server_info") + assert response.status_code == 200 + + time.sleep(1) + + response = requests.get(self.base_url + "/get_server_info") + assert response.status_code == 200 + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v32_basic.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v32_basic.py new file mode 100644 index 00000000..ce2d5afa --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v32_basic.py @@ -0,0 +1,78 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp" + + +class TestDeepseekV32Basic(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_V32_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--tp", + "8", + "--dp", + "8", + "--enable-dp-attention", + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k( + self, + ): # Append an "a" to make this test run first (alphabetically) to warm up the server + args = SimpleNamespace( + num_shots=20, + data_path=None, + num_questions=1400, + parallel=1400, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v32)\n" f'{metrics["accuracy"]=:.3f}\n' + ) + self.assertGreater(metrics["accuracy"], 0.935) + + def test_bs_1_speed(self): + args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) + acc_length, speed = send_one_prompt(args) + + print(f"{speed=:.2f}") + + if is_in_ci(): + write_github_step_summary( + f"### test_bs_1_speed (deepseek-v32)\n" f"{speed=:.2f} token/s\n" + ) + self.assertGreater(speed, 50) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v32_mtp.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v32_mtp.py new file mode 100644 index 00000000..41e3aa78 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v32_mtp.py @@ -0,0 +1,105 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +FULL_DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp" + + +class TestDeepseekV32MTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = FULL_DEEPSEEK_V32_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--tp", + "8", + "--dp", + "8", + "--enable-dp-attention", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--mem-frac", + "0.7", + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k( + self, + ): # Append an "a" to make this test run first (alphabetically) to warm up the server + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=20, + data_path=None, + num_questions=1400, + parallel=1400, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v32 mtp)\n" + f'{metrics["accuracy"]=:.3f}\n' + f"{avg_spec_accept_length=:.2f}\n" + ) + self.assertGreater(metrics["accuracy"], 0.94) + self.assertGreater(avg_spec_accept_length, 2.7) + + def test_bs_1_speed(self): + args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) + acc_length, speed = send_one_prompt(args) + + print(f"{acc_length=:.2f} {speed=:.2f}") + + if is_in_ci(): + write_github_step_summary( + f"### test_bs_1_speed (deepseek-v32 mtp)\n" + f"{acc_length=:.2f}\n" + f"{speed=:.2f} token/s\n" + ) + + self.assertGreater(acc_length, 2.7) + self.assertGreater(speed, 75) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v3_basic.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v3_basic.py new file mode 100644 index 00000000..e8c4b2d0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v3_basic.py @@ -0,0 +1,75 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_amd_ci, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +FULL_DEEPSEEK_V3_MODEL_PATH = "deepseek-ai/DeepSeek-V3-0324" + + +class TestDeepseekV3Basic(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = FULL_DEEPSEEK_V3_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code", "--tp", "8"] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 5, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k( + self, + ): # Append an "a" to make this test run first (alphabetically) to warm up the server + args = SimpleNamespace( + num_shots=8, + data_path=None, + num_questions=1400, + parallel=1400, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v3)\n" f'{metrics["accuracy"]=:.3f}\n' + ) + self.assertGreater(metrics["accuracy"], 0.935) + + def test_bs_1_speed(self): + args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) + acc_length, speed = send_one_prompt(args) + + print(f"{speed=:.2f}") + + if is_in_ci(): + write_github_step_summary( + f"### test_bs_1_speed (deepseek-v3)\n" f"{speed=:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(speed, 12) + else: + self.assertGreater(speed, 75) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v3_cutedsl_4gpu.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v3_cutedsl_4gpu.py new file mode 100644 index 00000000..7d8f39e0 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v3_cutedsl_4gpu.py @@ -0,0 +1,162 @@ +import os +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_DEEPSEEK_NVFP4_MODEL_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, + try_cached_model, +) + + +@unittest.skip("See https://github.com/sgl-project/sglang/issues/12533") +class TestDeepseekR1Nvfp4CuteDSLDeepEP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = try_cached_model(DEFAULT_DEEPSEEK_NVFP4_MODEL_FOR_TEST) + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--disable-radix-cache", + "--mem-fraction-static", + "0.89", + "--max-prefill-tokens", + "16384", + "--max-running-requests", + "256", + "--chunked-prefill-size", + "1024", + "--tp", + "4", + "--dp", + "4", + "--ep", + "4", + "--moe-dense-tp-size", + "1", + "--enable-dp-attention", + "--quantization", + "modelopt_fp4", + "--attention-backend", + "trtllm_mla", + "--moe-a2a-backend", + "deepep", + "--moe-runner-backend", + "flashinfer_cutedsl", + "--deepep-mode", + "low_latency", + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + env={ + **os.environ, + "SGLANG_DEEPEP_BF16_DISPATCH": "1", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "256", + "SGLANG_MOE_NVFP4_DISPATCH": "0", + }, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=512, + parallel=512, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Eval accuracy of GSM8K: {metrics=}") + + self.assertGreater(metrics["accuracy"], 0.92) + + +class TestDummyWithSBO(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = try_cached_model(DEFAULT_DEEPSEEK_NVFP4_MODEL_FOR_TEST) + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--disable-radix-cache", + "--mem-fraction-static", + "0.05", + "--max-prefill-tokens", + "16384", + "--max-running-requests", + "256", + "--chunked-prefill-size", + "1024", + "--cuda-graph-bs", + "64", + "--tp", + "4", + "--dp", + "4", + "--ep", + "4", + "--moe-dense-tp-size", + "1", + "--enable-dp-attention", + "--quantization", + "modelopt_fp4", + "--attention-backend", + "trtllm_mla", + "--moe-a2a-backend", + "deepep", + "--moe-runner-backend", + "flashinfer_cutedsl", + "--deepep-mode", + "low_latency", + "--json-model-override-args", + '{"num_hidden_layers": 1, "first_k_dense_replace": 0, "n_routed_experts": 24}', + "--enable-single-batch-overlap", + "--load-format", + "dummy", + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + env={ + **os.environ, + "SGLANG_DEEPEP_BF16_DISPATCH": "1", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "256", + "SGLANG_MOE_NVFP4_DISPATCH": "0", + }, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=0, + data_path=None, + num_questions=512, + parallel=512, + max_new_tokens=16, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Eval accuracy of GSM8K: {metrics=}") + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v3_fp4_4gpu.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v3_fp4_4gpu.py new file mode 100644 index 00000000..2d675f5a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v3_fp4_4gpu.py @@ -0,0 +1,176 @@ +import os +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4" +SERVER_LAUNCH_TIMEOUT = 1000 + + +class TestDeepseekV3FP4(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--tp", + "4", + "--attention-backend", + "trtllm_mla", + "--moe-runner-backend", + "flashinfer_trtllm", + "--quantization", + "modelopt_fp4", + "--kv-cache-dtype", + "fp8_e4m3", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=SERVER_LAUNCH_TIMEOUT, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k( + self, + ): # Append an "a" to make this test run first (alphabetically) to warm up the server + args = SimpleNamespace( + num_shots=8, + data_path=None, + num_questions=1319, + parallel=1319, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["accuracy"]=:.3f}\n' + ) + self.assertGreater(metrics["accuracy"], 0.935) + + def test_bs_1_speed(self): + args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) + acc_length, speed = send_one_prompt(args) + + print(f"{speed=:.2f}") + + if is_in_ci(): + write_github_step_summary( + f"### test_bs_1_speed (deepseek-v3-fp4)\n" f"{speed=:.2f} token/s\n" + ) + self.assertGreater(speed, 75) + + +class TestDeepseekV3FP4MTP(CustomTestCase): + @classmethod + def setUpClass(cls): + os.environ["SGLANG_ENABLE_SPEC_V2"] = "1" + cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--tp", + "4", + "--attention-backend", + "trtllm_mla", + "--moe-runner-backend", + "flashinfer_trtllm", + "--quantization", + "modelopt_fp4", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--kv-cache-dtype", + "fp8_e4m3", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=SERVER_LAUNCH_TIMEOUT, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + if "SGLANG_ENABLE_SPEC_V2" in os.environ: + del os.environ["SGLANG_ENABLE_SPEC_V2"] + + def test_a_gsm8k( + self, + ): # Append an "a" to make this test run first (alphabetically) to warm up the server + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v3-fp4 mtp)\n" + f'{metrics["accuracy"]=:.3f}\n' + f"{avg_spec_accept_length=:.2f}\n" + ) + self.assertGreater(metrics["accuracy"], 0.94) + self.assertGreater(avg_spec_accept_length, 2.04) + + def test_bs_1_speed(self): + args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) + acc_length, speed = send_one_prompt(args) + + print(f"{acc_length=:.2f} {speed=:.2f}") + + if is_in_ci(): + write_github_step_summary( + f"### test_bs_1_speed (deepseek-v3-fp4 mtp)\n" + f"{acc_length=:.2f}\n" + f"{speed=:.2f} token/s\n" + ) + self.assertGreater(acc_length, 2.04) + self.assertGreater(speed, 150) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v3_mtp.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v3_mtp.py new file mode 100644 index 00000000..b8c43a29 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deepseek_v3_mtp.py @@ -0,0 +1,108 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_amd_ci, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +FULL_DEEPSEEK_V3_MODEL_PATH = "deepseek-ai/DeepSeek-V3-0324" + + +class TestDeepseekV3MTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = FULL_DEEPSEEK_V3_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--tp", + "8", + "--trust-remote-code", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + ] + if not is_in_amd_ci(): + other_args += ["--mem-frac", "0.7"] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 5, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k( + self, + ): # Append an "a" to make this test run first (alphabetically) to warm up the server + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v3 mtp)\n" + f'{metrics["accuracy"]=:.3f}\n' + f"{avg_spec_accept_length=:.2f}\n" + ) + self.assertGreater(metrics["accuracy"], 0.935) + self.assertGreater(avg_spec_accept_length, 2.9) + + def test_bs_1_speed(self): + args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) + acc_length, speed = send_one_prompt(args) + + print(f"{acc_length=:.2f} {speed=:.2f}") + + if is_in_ci(): + write_github_step_summary( + f"### test_bs_1_speed (deepseek-v3 mtp)\n" + f"{acc_length=:.2f}\n" + f"{speed=:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(acc_length, 2.8) + else: + self.assertGreater(acc_length, 2.9) + if is_in_amd_ci(): + self.assertGreater(speed, 15) + else: + self.assertGreater(speed, 130) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deterministic.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deterministic.py new file mode 100644 index 00000000..4d6c8e60 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_deterministic.py @@ -0,0 +1,61 @@ +""" +Usage: +cd test/srt +python3 -m unittest test_deterministic.TestDeterministic.TESTCASE + +Note that there is also `python/sglang/test/test_deterministic.py` as an interactive test. We are converting that +test into unit tests so that's easily reproducible in CI. +""" + +import unittest + +from sglang.test.test_deterministic_utils import ( + COMMON_SERVER_ARGS, + TestDeterministicBase, +) + + +class TestFlashinferDeterministic(TestDeterministicBase): + # Test with flashinfer attention backend + @classmethod + def get_server_args(cls): + args = COMMON_SERVER_ARGS + args.extend( + [ + "--attention-backend", + "flashinfer", + ] + ) + return args + + +class TestFa3Deterministic(TestDeterministicBase): + # Test with fa3 attention backend + @classmethod + def get_server_args(cls): + args = COMMON_SERVER_ARGS + args.extend( + [ + "--attention-backend", + "fa3", + ] + ) + return args + + +class TestTritonDeterministic(TestDeterministicBase): + # Test with triton attention backend + @classmethod + def get_server_args(cls): + args = COMMON_SERVER_ARGS + args.extend( + [ + "--attention-backend", + "triton", + ] + ) + return args + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_disaggregation_basic.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_disaggregation_basic.py new file mode 100644 index 00000000..5d2ee147 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_disaggregation_basic.py @@ -0,0 +1,436 @@ +import json +import os +import unittest +from types import SimpleNamespace + +import openai +import requests +from transformers import AutoTokenizer + +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_disaggregation_utils import TestDisaggregationBase +from sglang.test.test_utils import ( + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + popen_launch_pd_server, +) + + +class TestDisaggregationAccuracy(TestDisaggregationBase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + + # Non blocking start servers + cls.start_prefill() + cls.start_decode() + + # Block until both + cls.wait_server_ready(cls.prefill_url + "/health") + cls.wait_server_ready(cls.decode_url + "/health") + + cls.launch_lb() + + @classmethod + def start_prefill(cls): + prefill_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "prefill", + "--tp", + "1", + ] + prefill_args += cls.transfer_backend + cls.rdma_devices + cls.process_prefill = popen_launch_pd_server( + cls.model, + cls.prefill_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=prefill_args, + ) + + @classmethod + def start_decode(cls): + decode_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "decode", + "--tp", + "1", + "--base-gpu-id", + "1", + ] + decode_args += cls.transfer_backend + cls.rdma_devices + cls.process_decode = popen_launch_pd_server( + cls.model, + cls.decode_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=decode_args, + ) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host=f"http://{self.base_host}", + port=int(self.lb_port), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Evaluation metrics: {metrics}") + + self.assertGreater(metrics["accuracy"], 0.62) + + def test_logprob(self): + prompt = "The capital of france is " + response = requests.post( + self.lb_url + "/generate", + json={ + "text": prompt, + "sampling_params": {"temperature": 0}, + "return_logprob": True, + "return_input_logprob": True, + "logprob_start_len": 0, + }, + ) + + j = response.json() + completion_tokens = j["meta_info"]["completion_tokens"] + input_logprobs = j["meta_info"]["input_token_logprobs"] + output_logprobs = j["meta_info"]["output_token_logprobs"] + + assert ( + len(output_logprobs) == completion_tokens + ), f"output_logprobs and completion_tokens should have the same length, but got {len(output_logprobs)} and {completion_tokens}" + assert ( + len(input_logprobs) > 0 + ), f"input_logprobs should have at least one token, but got {len(input_logprobs)}" + + def test_structured_output(self): + json_schema = json.dumps( + { + "type": "object", + "properties": { + "name": {"type": "string", "pattern": "^[\\w]+$"}, + "population": {"type": "integer"}, + }, + "required": ["name", "population"], + } + ) + + # JSON + response = requests.post( + f"{self.lb_url}/generate", + json={ + "text": "Here is the information of the capital of France in the JSON format.\n", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 64, + "json_schema": json_schema, + }, + }, + ) + output = response.json()["text"] + # ensure the output is a valid JSON + json.loads(output) + + def test_first_token_finish(self): + client = openai.Client(api_key="empty", base_url=f"{self.lb_url}/v1") + tokenizer = AutoTokenizer.from_pretrained(self.model) + eos_token = tokenizer.eos_token_id + prompt = "The best programming language for AI is" + + # First token EOS + res = client.completions.create( + model="dummy", prompt=prompt, logit_bias={eos_token: 42} + ).model_dump() + print(f"{res=}") + + assert res["usage"]["completion_tokens"] == 1, ( + "Expected completion_tokens to be 1 when first token is EOS, " + f"but got {res['usage']['completion_tokens']}" + ) + + # First token EOS with ignore_eos + res = client.completions.create( + model="dummy", + prompt=prompt, + logit_bias={eos_token: 42}, + extra_body={"ignore_eos": True}, + ).model_dump() + print(f"{res=}") + + assert res["usage"]["completion_tokens"] > 1, ( + "Expected completion_tokens to be greater than 1 when ignore_eos is True, " + f"but got {res['usage']['completion_tokens']}" + ) + + # First token with specified stop token + stop_token_id = tokenizer.encode(" hello", add_special_tokens=False)[0] + res = client.completions.create( + model="dummy", + prompt=prompt, + logit_bias={stop_token_id: 42}, + stop=[" hello"], + ).model_dump() + print(f"{res=}") + + assert res["usage"]["completion_tokens"] == 1, ( + "Expected completion_tokens to be 1 when first token is stop token, " + f"but got {res['usage']['completion_tokens']}" + ) + + +class TestDisaggregationMooncakeFailure(TestDisaggregationBase): + @classmethod + def setUpClass(cls): + super().setUpClass() + # set DISAGGREGATION_TEST_FAILURE_PROB to simulate failure + os.environ["DISAGGREGATION_TEST_FAILURE_PROB"] = "0.05" + + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + + # Non blocking start servers + cls.start_prefill() + cls.start_decode() + + # Block until both + cls.wait_server_ready(cls.prefill_url + "/health") + cls.wait_server_ready(cls.decode_url + "/health") + + cls.launch_lb() + + @classmethod + def tearDownClass(cls): + os.environ.pop("DISAGGREGATION_TEST_FAILURE_PROB") + super().tearDownClass() + + @classmethod + def start_prefill(cls): + prefill_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "prefill", + "--tp", + "1", + ] + prefill_args += cls.transfer_backend + cls.rdma_devices + cls.process_prefill = popen_launch_pd_server( + cls.model, + cls.prefill_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=prefill_args, + ) + + @classmethod + def start_decode(cls): + decode_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "decode", + "--tp", + "1", + "--base-gpu-id", + "1", + ] + decode_args += cls.transfer_backend + cls.rdma_devices + cls.process_decode = popen_launch_pd_server( + cls.model, + cls.decode_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=decode_args, + ) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host=f"http://{self.base_host}", + port=int(self.lb_port), + ) + + # Expect lots of failure but the server cannot crash + try: + metrics = run_eval_few_shot_gsm8k(args) + print(f"Evaluation metrics: {metrics}") + except Exception as e: + print(f"Test encountered expected errors: {e}") + # Check if servers are still healthy + try: + response = requests.get(self.prefill_url + "/health_generate") + assert response.status_code == 200 + response = requests.get(self.decode_url + "/health_generate") + assert response.status_code == 200 + except Exception as health_check_error: + # If health check fails, re-raise the original exception + raise e from health_check_error + + +class TestDisaggregationMooncakeSpec(TestDisaggregationBase): + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.model = DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST + cls.draft_model = DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST + cls.spec_args = [ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + cls.draft_model, + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "16", + "--cuda-graph-max-bs", + "8", + ] + print(f"{cls.base_host=} {cls.lb_port=} {cls.prefill_port=} {cls.decode_port=}") + + # Non blocking start servers + cls.start_prefill() + cls.start_decode() + + # Block until both + cls.wait_server_ready(cls.prefill_url + "/health") + cls.wait_server_ready(cls.decode_url + "/health") + + cls.launch_lb() + + @classmethod + def start_prefill(cls): + prefill_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "prefill", + "--tp", + "1", + ] + cls.spec_args + prefill_args += cls.transfer_backend + cls.rdma_devices + cls.process_prefill = popen_launch_pd_server( + cls.model, + cls.prefill_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=prefill_args, + ) + + @classmethod + def start_decode(cls): + decode_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "decode", + "--tp", + "1", + "--base-gpu-id", + "1", + ] + cls.spec_args + decode_args += cls.transfer_backend + cls.rdma_devices + cls.process_decode = popen_launch_pd_server( + cls.model, + cls.decode_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=decode_args, + ) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=2, + host=f"http://{self.base_host}", + port=int(self.lb_port), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Evaluation metrics: {metrics}") + + self.assertGreater(metrics["accuracy"], 0.20) + + +class TestDisaggregationSimulatedRetract(TestDisaggregationBase): + @classmethod + def setUpClass(cls): + super().setUpClass() + os.environ["SGLANG_TEST_RETRACT"] = "true" + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + + # Non blocking start servers + cls.start_prefill() + cls.start_decode() + + # Block until both + cls.wait_server_ready(cls.prefill_url + "/health") + cls.wait_server_ready(cls.decode_url + "/health") + + cls.launch_lb() + + @classmethod + def tearDownClass(cls): + os.environ.pop("SGLANG_TEST_RETRACT") + super().tearDownClass() + + @classmethod + def start_prefill(cls): + prefill_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "prefill", + "--tp", + "1", + ] + prefill_args += cls.transfer_backend + cls.rdma_devices + cls.process_prefill = popen_launch_pd_server( + cls.model, + cls.prefill_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=prefill_args, + ) + + @classmethod + def start_decode(cls): + decode_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "decode", + "--tp", + "1", + "--base-gpu-id", + "1", + ] + decode_args += cls.transfer_backend + cls.rdma_devices + cls.process_decode = popen_launch_pd_server( + cls.model, + cls.decode_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=decode_args, + ) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host=f"http://{self.base_host}", + port=int(self.lb_port), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Evaluation metrics: {metrics}") + + self.assertGreater(metrics["accuracy"], 0.62) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_disaggregation_different_tp.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_disaggregation_different_tp.py new file mode 100644 index 00000000..2cb05842 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_disaggregation_different_tp.py @@ -0,0 +1,301 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.environ import envs +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_disaggregation_utils import TestDisaggregationBase +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST_MLA, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + popen_launch_pd_server, + try_cached_model, +) + + +class TestDisaggregationMooncakePrefillLargerTP(TestDisaggregationBase): + @classmethod + def setUpClass(cls): + super().setUpClass() + # Temporarily disable JIT DeepGEMM + envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False) + + cls.model = try_cached_model(DEFAULT_MODEL_NAME_FOR_TEST_MLA) + + # Non blocking start servers + cls.start_prefill() + cls.start_decode() + + # Block until both + cls.wait_server_ready(cls.prefill_url + "/health") + cls.wait_server_ready(cls.decode_url + "/health") + + cls.launch_lb() + + @classmethod + def start_prefill(cls): + prefill_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "prefill", + "--tp", + "4", + ] + prefill_args += cls.transfer_backend + cls.rdma_devices + cls.process_prefill = popen_launch_pd_server( + cls.model, + cls.prefill_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=prefill_args, + ) + + @classmethod + def start_decode(cls): + decode_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "decode", + "--tp", + "2", + "--base-gpu-id", + "4", + ] + decode_args += cls.transfer_backend + cls.rdma_devices + cls.process_decode = popen_launch_pd_server( + cls.model, + cls.decode_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=decode_args, + ) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host=f"http://{self.base_host}", + port=int(self.lb_port), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Evaluation metrics: {metrics}") + + self.assertGreater(metrics["accuracy"], 0.60) + + +class TestDisaggregationMooncakeDecodeLargerTP(TestDisaggregationBase): + @classmethod + def setUpClass(cls): + super().setUpClass() + # Temporarily disable JIT DeepGEMM + envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False) + + cls.model = try_cached_model(DEFAULT_MODEL_NAME_FOR_TEST_MLA) + + # Non blocking start servers + cls.start_prefill() + cls.start_decode() + + # Block until both + cls.wait_server_ready(cls.prefill_url + "/health") + cls.wait_server_ready(cls.decode_url + "/health") + + cls.launch_lb() + + @classmethod + def start_prefill(cls): + prefill_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "prefill", + "--tp", + "2", + ] + prefill_args += cls.transfer_backend + cls.rdma_devices + cls.process_prefill = popen_launch_pd_server( + cls.model, + cls.prefill_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=prefill_args, + ) + + @classmethod + def start_decode(cls): + decode_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "decode", + "--tp", + "4", + "--base-gpu-id", + "4", + ] + decode_args += cls.transfer_backend + cls.rdma_devices + cls.process_decode = popen_launch_pd_server( + cls.model, + cls.decode_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=decode_args, + ) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host=f"http://{self.base_host}", + port=int(self.lb_port), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Evaluation metrics: {metrics}") + + self.assertGreater(metrics["accuracy"], 0.60) + + +class TestDisaggregationMooncakeMHAPrefillLargerTP(TestDisaggregationBase): + @classmethod + def setUpClass(cls): + super().setUpClass() + # Temporarily disable JIT DeepGEMM + envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False) + + cls.model = try_cached_model(DEFAULT_MODEL_NAME_FOR_TEST) + + # Non blocking start servers + cls.start_prefill() + cls.start_decode() + + # Block until both + cls.wait_server_ready(cls.prefill_url + "/health") + cls.wait_server_ready(cls.decode_url + "/health") + + cls.launch_lb() + + @classmethod + def start_prefill(cls): + prefill_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "prefill", + "--tp", + "4", + ] + prefill_args += cls.transfer_backend + cls.rdma_devices + cls.process_prefill = popen_launch_pd_server( + cls.model, + cls.prefill_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=prefill_args, + ) + + @classmethod + def start_decode(cls): + decode_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "decode", + "--tp", + "2", + "--base-gpu-id", + "4", + ] + decode_args += cls.transfer_backend + cls.rdma_devices + cls.process_decode = popen_launch_pd_server( + cls.model, + cls.decode_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=decode_args, + ) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host=f"http://{self.base_host}", + port=int(self.lb_port), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Evaluation metrics: {metrics}") + + self.assertGreater(metrics["accuracy"], 0.60) + + +class TestDisaggregationMooncakeMHADecodeLargerTP(TestDisaggregationBase): + @classmethod + def setUpClass(cls): + super().setUpClass() + # Temporarily disable JIT DeepGEMM + envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False) + + cls.model = try_cached_model(DEFAULT_MODEL_NAME_FOR_TEST) + + # Non blocking start servers + cls.start_prefill() + cls.start_decode() + + # Block until both + cls.wait_server_ready(cls.prefill_url + "/health") + cls.wait_server_ready(cls.decode_url + "/health") + + cls.launch_lb() + + @classmethod + def start_prefill(cls): + prefill_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "prefill", + "--tp", + "2", + ] + prefill_args += cls.transfer_backend + cls.rdma_devices + cls.process_prefill = popen_launch_pd_server( + cls.model, + cls.prefill_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=prefill_args, + ) + + @classmethod + def start_decode(cls): + decode_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "decode", + "--tp", + "4", + "--base-gpu-id", + "4", + ] + decode_args += cls.transfer_backend + cls.rdma_devices + cls.process_decode = popen_launch_pd_server( + cls.model, + cls.decode_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=decode_args, + ) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host=f"http://{self.base_host}", + port=int(self.lb_port), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Evaluation metrics: {metrics}") + + self.assertGreater(metrics["accuracy"], 0.60) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_disaggregation_dp_attention.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_disaggregation_dp_attention.py new file mode 100644 index 00000000..f1f02ad6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_disaggregation_dp_attention.py @@ -0,0 +1,97 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.environ import envs +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_disaggregation_utils import TestDisaggregationBase +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST_MLA, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + popen_launch_pd_server, + try_cached_model, +) + + +class TestDisaggregationDPAttention(TestDisaggregationBase): + PREFILL_DP_SIZE = 4 + DECODE_DP_SIZE = 4 + + @classmethod + def setUpClass(cls): + super().setUpClass() + # Temporarily disable JIT DeepGEMM + envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False) + + cls.model = try_cached_model(DEFAULT_MODEL_NAME_FOR_TEST_MLA) + + # Non blocking start servers + cls.start_prefill() + cls.start_decode() + + # Block until both + cls.wait_server_ready(cls.prefill_url + "/health") + cls.wait_server_ready(cls.decode_url + "/health") + + cls.launch_lb() + + @classmethod + def start_prefill(cls): + prefill_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "prefill", + "--tp", + str(cls.PREFILL_DP_SIZE), + "--dp", + str(cls.PREFILL_DP_SIZE), + "--enable-dp-attention", + ] + prefill_args += cls.transfer_backend + cls.rdma_devices + cls.process_prefill = popen_launch_pd_server( + cls.model, + cls.prefill_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=prefill_args, + ) + + @classmethod + def start_decode(cls): + decode_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "decode", + "--tp", + str(cls.DECODE_DP_SIZE), + "--dp", + str(cls.DECODE_DP_SIZE), + "--enable-dp-attention", + "--base-gpu-id", + str(cls.PREFILL_DP_SIZE), + "--prefill-round-robin-balance", + ] + decode_args += cls.transfer_backend + cls.rdma_devices + cls.process_decode = popen_launch_pd_server( + cls.model, + cls.decode_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=decode_args, + ) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1400, + max_new_tokens=512, + parallel=128, + host=f"http://{self.base_host}", + port=int(self.lb_port), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Evaluation metrics: {metrics}") + + self.assertGreater(metrics["accuracy"], 0.60) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_disaggregation_hybrid_attention.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_disaggregation_hybrid_attention.py new file mode 100644 index 00000000..83ebed93 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_disaggregation_hybrid_attention.py @@ -0,0 +1,81 @@ +import unittest +from types import SimpleNamespace + +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_disaggregation_utils import TestDisaggregationBase +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + popen_launch_pd_server, +) + + +class TestDisaggregationHybridAttentionMamba(TestDisaggregationBase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.model = "Qwen/Qwen3-Next-80B-A3B-Instruct" + + # Non blocking start servers + cls.start_prefill() + cls.start_decode() + + # Block until both + cls.wait_server_ready(cls.prefill_url + "/health") + cls.wait_server_ready(cls.decode_url + "/health") + + cls.launch_lb() + + @classmethod + def start_prefill(cls): + prefill_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "prefill", + "--tp", + "4", + ] + prefill_args += cls.transfer_backend + cls.rdma_devices + cls.process_prefill = popen_launch_pd_server( + cls.model, + cls.prefill_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=prefill_args, + ) + + @classmethod + def start_decode(cls): + decode_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "decode", + "--tp", + "4", + "--base-gpu-id", + "4", + ] + decode_args += cls.transfer_backend + cls.rdma_devices + cls.process_decode = popen_launch_pd_server( + cls.model, + cls.decode_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=decode_args, + ) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host=f"http://{self.base_host}", + port=int(self.lb_port), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Evaluation metrics: {metrics}") + + self.assertGreater(metrics["accuracy"], 0.93) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_disaggregation_pp.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_disaggregation_pp.py new file mode 100644 index 00000000..29df8b07 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_disaggregation_pp.py @@ -0,0 +1,89 @@ +import time +import unittest +from types import SimpleNamespace + +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.test_disaggregation_utils import TestDisaggregationBase +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + popen_launch_pd_server, + try_cached_model, +) + + +class TestDisaggregationPPAccuracy(TestDisaggregationBase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.model = try_cached_model(DEFAULT_MODEL_NAME_FOR_TEST) + + # Non blocking start servers + cls.start_prefill() + cls.start_decode() + + # Block until both + cls.wait_server_ready(cls.prefill_url + "/health") + cls.wait_server_ready(cls.decode_url + "/health") + + cls.launch_lb() + + @classmethod + def start_prefill(cls): + prefill_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "prefill", + "--tp-size", + "2", + "--pp-size", + "2", + "--disable-overlap-schedule", + ] + prefill_args += cls.transfer_backend + cls.rdma_devices + cls.process_prefill = popen_launch_pd_server( + cls.model, + cls.prefill_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=prefill_args, + ) + + @classmethod + def start_decode(cls): + decode_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "decode", + "--tp", + "2", + "--base-gpu-id", + "4", + ] + decode_args += cls.transfer_backend + cls.rdma_devices + cls.process_decode = popen_launch_pd_server( + cls.model, + cls.decode_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=decode_args, + ) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host=f"http://{self.base_host}", + port=int(self.lb_port), + ) + metrics = run_eval(args) + print(f"{metrics=}") + + self.assertGreater(metrics["accuracy"], 0.24) + # Wait a little bit so that the memory check happens. + time.sleep(5) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_dp_attention.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_dp_attention.py new file mode 100644 index 00000000..42661645 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_dp_attention.py @@ -0,0 +1,128 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST_MLA, + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_amd_ci, + popen_launch_server, +) + + +class TestDPAttentionDP2TP2(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "2", + "--enable-dp-attention", + "--dp", + "2", + "--enable-torch-compile", + "--torch-compile-max-bs", + "2", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.8) + + +class TestDPAttentionDP2TP2DeepseekV3MTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--disable-radix", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + "--speculative-draft-model-path", + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, + "--tp-size", + "2", + "--enable-dp-attention", + "--dp-size", + "2", + ] + if not is_in_amd_ci(): + other_args += ["--mem-frac", "0.7"] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print( + f"###test_gsm8k (deepseek-v3 mtp + dp):\n" + f"accuracy={metrics['accuracy']=:.3f}\n" + f"{avg_spec_accept_length=:.3f}\n" + ) + self.assertGreater(avg_spec_accept_length, 2.5) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eagle_constrained_decoding.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eagle_constrained_decoding.py new file mode 100644 index 00000000..f172618c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eagle_constrained_decoding.py @@ -0,0 +1,77 @@ +import unittest + +from sglang.srt.environ import envs +from sglang.srt.utils import kill_process_tree +from sglang.test.kits.json_constrained_kit import TestJSONConstrainedMixin +from sglang.test.kits.regex_constrained_kit import TestRegexConstrainedMixin +from sglang.test.test_utils import ( + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestEagleConstrainedDecoding( + CustomTestCase, TestRegexConstrainedMixin, TestJSONConstrainedMixin +): + max_running_requests = 64 + attention_backend = "triton" + spec_steps = 5 + spec_topk = 1 + spec_draft_tokens = 6 + page_size = 1 + other_launch_args = [] + model = DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST + draft_model = DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST + grammar_backend = "xgrammar" + eagle_v2 = False + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + launch_args = [ + "--trust-remote-code", + "--attention-backend", + cls.attention_backend, + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model", + cls.draft_model, + "--speculative-num-steps", + cls.spec_steps, + "--speculative-eagle-topk", + cls.spec_topk, + "--speculative-num-draft-tokens", + cls.spec_draft_tokens, + "--page-size", + str(cls.page_size), + "--mem-fraction-static", + "0.75", + "--max-running-requests", + str(cls.max_running_requests), + "--grammar-backend", + cls.grammar_backend, + ] + launch_args.extend(cls.other_launch_args) + with envs.SGLANG_ENABLE_SPEC_V2.override(cls.eagle_v2): + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=launch_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +class TestEagleConstrainedDecodingV2(TestEagleConstrainedDecoding): + eagle_v2 = True + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eagle_dp_attention.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eagle_dp_attention.py new file mode 100644 index 00000000..14c1ad39 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eagle_dp_attention.py @@ -0,0 +1,129 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.test_utils import ( + DEFAULT_EAGLE_DP_ATTENTION_DRAFT_MODEL_FOR_TEST, + DEFAULT_EAGLE_DP_ATTENTION_TARGET_MODEL_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_amd_ci, + is_in_ci, + kill_process_tree, + popen_launch_server, + write_github_step_summary, +) + + +class TestEAGLE3EngineDPAttention(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_EAGLE_DP_ATTENTION_TARGET_MODEL_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--speculative-algorithm", + "EAGLE3", + "--speculative-num-steps", + "6", + "--speculative-eagle-topk", + "10", + "--speculative-num-draft-tokens", + "32", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DP_ATTENTION_DRAFT_MODEL_FOR_TEST, + "--tp-size", + "2", + "--dp-size", + "2", + "--enable-dp-attention", + "--enable-dp-lm-head", + "--moe-dense-tp-size", + "1", + "--attention-backend", + "fa3", + "--mem-fraction-static", + "0.75", + "--cuda-graph-max-bs", + "64", + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k(self): + """Test GSM8K evaluation - append 'a' to run first alphabetically""" + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + server_info = requests.get(self.base_url + "/get_server_info") + server_data = server_info.json() + + # Try to get avg_spec_accept_length + avg_spec_accept_length = None + if "internal_states" in server_data and len(server_data["internal_states"]) > 0: + internal_state = server_data["internal_states"][0] + if "avg_spec_accept_length" in internal_state: + avg_spec_accept_length = internal_state["avg_spec_accept_length"] + elif "spec_accept_length" in internal_state: + avg_spec_accept_length = internal_state["spec_accept_length"] + + print(f"{avg_spec_accept_length=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (EAGLE3 DP Attention)\n" + f'{metrics["accuracy"]=:.3f}\n' + f"{avg_spec_accept_length=:.2f}\n" + ) + self.assertGreater(metrics["accuracy"], 0.91) + if avg_spec_accept_length is not None: + self.assertGreater(avg_spec_accept_length, 2.5) + + def test_bs_1_speed(self): + """Test batch size 1 speed with EAGLE3 DP Attention""" + args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) + acc_length, speed = send_one_prompt(args) + + print(f"{acc_length=:.2f} {speed=:.2f}") + + if is_in_ci(): + write_github_step_summary( + f"### test_bs_1_speed (EAGLE3 DP Attention)\n" + f"{acc_length=:.2f}\n" + f"{speed=:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(acc_length, 2.0) + else: + self.assertGreater(acc_length, 2.3) + if is_in_amd_ci(): + self.assertGreater(speed, 10) + else: + self.assertGreater(speed, 40) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eagle_infer_a.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eagle_infer_a.py new file mode 100644 index 00000000..31efd87e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eagle_infer_a.py @@ -0,0 +1,438 @@ +import os +import unittest + +import requests +import torch + +import sglang as sgl +from sglang.srt.utils import kill_process_tree +from sglang.srt.utils.hf_transformers_utils import get_tokenizer +from sglang.test.test_utils import ( + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST_EAGLE3, + DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3, + DEFAULT_MODEL_NAME_FOR_TEST_MLA, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) + +torch_dtype = torch.float16 +prefill_tolerance = 5e-2 +decode_tolerance: float = 5e-2 + + +class TestEAGLEEngine(CustomTestCase): + BASE_CONFIG = { + "model_path": DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + "speculative_draft_model_path": DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "speculative_algorithm": "EAGLE", + "speculative_num_steps": 5, + "speculative_eagle_topk": 4, + "speculative_num_draft_tokens": 8, + "mem_fraction_static": 0.7, + "cuda_graph_max_bs": 5, + "trust_remote_code": True, + } + NUM_CONFIGS = 2 + + THRESHOLDS = { + "batch_avg_accept_len": 1.9, + "accept_len": 3.6, + } + + def setUp(self): + self.prompt = "Today is a sunny day and I like" + self.sampling_params = {"temperature": 0, "max_new_tokens": 8} + + ref_engine = sgl.Engine( + model_path=self.BASE_CONFIG["model_path"], cuda_graph_max_bs=1 + ) + self.ref_output = ref_engine.generate(self.prompt, self.sampling_params)["text"] + ref_engine.shutdown() + + def test_correctness(self): + configs = [ + # Basic config + self.BASE_CONFIG, + # Chunked prefill + {**self.BASE_CONFIG, "chunked_prefill_size": 4}, + ] + + for i, config in enumerate(configs[: self.NUM_CONFIGS]): + with self.subTest(i=i): + print(f"{config=}") + engine = sgl.Engine(**config, log_level="info", decode_log_interval=10) + try: + self._test_single_generation(engine) + self._test_batch_generation(engine) + self._test_eos_token(engine) + self._test_acc_length(engine) + finally: + engine.flush_cache() # check engine alive + engine.shutdown() + print("=" * 100) + + def _test_single_generation(self, engine): + output = engine.generate(self.prompt, self.sampling_params)["text"] + print(f"{output=}, {self.ref_output=}") + self.assertEqual(output, self.ref_output) + + def _test_batch_generation(self, engine): + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + params = {"temperature": 0, "max_new_tokens": 50} + + outputs = engine.generate(prompts, params) + for prompt, output in zip(prompts, outputs): + print(f"Prompt: {prompt}") + print(f"Generated: {output['text']}") + print("-" * 40) + + print(f"{engine.get_server_info()=}") + + avg_spec_accept_length = engine.get_server_info()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater( + avg_spec_accept_length, self.THRESHOLDS["batch_avg_accept_len"] + ) + + def _test_eos_token(self, engine): + prompt = "[INST] <>\nYou are a helpful assistant.\n<>\nToday is a sunny day and I like [/INST]" + params = { + "temperature": 0.1, + "max_new_tokens": 1024, + "skip_special_tokens": False, + } + + tokenizer = get_tokenizer(DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST) + output = engine.generate(prompt, params)["text"] + print(f"{output=}") + + tokens = tokenizer.encode(output, truncation=False) + self.assertNotIn(tokenizer.eos_token_id, tokens) + + def _test_acc_length(self, engine): + prompt = [ + "Human: Give me a fully functional FastAPI server. Show the python code.\n\nAssistant:", + ] * 5 # test batched generation + sampling_params = {"temperature": 0, "max_new_tokens": 512} + output = engine.generate(prompt, sampling_params) + output = output[0] + + if "spec_verify_ct" in output["meta_info"]: + acc_length = ( + output["meta_info"]["completion_tokens"] + / output["meta_info"]["spec_verify_ct"] + ) + else: + acc_length = 1.0 + + speed = ( + output["meta_info"]["completion_tokens"] + / output["meta_info"]["e2e_latency"] + ) + print(f"{acc_length=:.4f}, {speed=}") + + self.assertGreater(acc_length, self.THRESHOLDS["accept_len"]) + + +class TestEAGLEEngineTokenMap(TestEAGLEEngine): + BASE_CONFIG = { + "model_path": "meta-llama/Meta-Llama-3-8B-Instruct", + "speculative_draft_model_path": "lmsys/sglang-EAGLE-LLaMA3-Instruct-8B", + "speculative_algorithm": "EAGLE", + "speculative_num_steps": 5, + "speculative_eagle_topk": 4, + "speculative_num_draft_tokens": 8, + "speculative_token_map": "thunlp/LLaMA3-Instruct-8B-FR-Spec/freq_32768.pt", + "mem_fraction_static": 0.7, + "cuda_graph_max_bs": 5, + "dtype": "float16", + } + NUM_CONFIGS = 1 + THRESHOLDS = { + "batch_avg_accept_len": 1.9, + "accept_len": 2.5, + } + + +class TestEAGLE3Engine(TestEAGLEEngine): + BASE_CONFIG = { + "model_path": DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST_EAGLE3, + "speculative_draft_model_path": DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3, + "speculative_algorithm": "EAGLE3", + "speculative_num_steps": 5, + "speculative_eagle_topk": 16, + "speculative_num_draft_tokens": 64, + "mem_fraction_static": 0.7, + "cuda_graph_max_bs": 5, + "dtype": "float16", + } + NUM_CONFIGS = 1 + THRESHOLDS = { + "batch_avg_accept_len": 1.75, + "accept_len": 3.1, + } + + +class TestEAGLERadixCache(CustomTestCase): + BASE_CONFIG = { + "model_path": DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST_EAGLE3, + "speculative_draft_model_path": DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3, + "speculative_algorithm": "EAGLE3", + "speculative_num_steps": 2, + "speculative_eagle_topk": 2, + "speculative_num_draft_tokens": 5, + "mem_fraction_static": 0.7, + "dtype": "float16", + "trust_remote_code": True, + "attention_backend": "fa3", + "skip_server_warmup": True, + "cuda_graph_max_bs": 5, + } + + def test_correctness(self): + os.environ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN"] = "1" + configs = [ + # Basic config + self.BASE_CONFIG, + # Chunked prefill & Page Size > 1 + {**self.BASE_CONFIG, "chunked_prefill_size": 64, "page_size": 4}, + {**self.BASE_CONFIG, "page_size": 4}, + # Preferred by some kernels + {**self.BASE_CONFIG, "page_size": 64}, + # Disable CUDA Graph + { + **self.BASE_CONFIG, + "disable_cuda_graph": True, + "page_size": 4, + }, + ] + + for i, config in enumerate(configs): + with self.subTest(i=i): + print(f"{config=}") + engine = sgl.Engine(**config, log_level="info", decode_log_interval=10) + try: + self._test_acc_length(engine) + self._test_batch_generation(engine) + finally: + engine.shutdown() + print("=" * 100) + del os.environ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN"] + + def _test_acc_length(self, engine): + warmup_prompt = [ + "Human: Give me a fully functional FastAPI server. Show the python code.\n\nAssistant:", + ] + sampling_params = {"temperature": 0, "max_new_tokens": 512} + output = engine.generate(warmup_prompt, sampling_params) + test_prompt = [ + "<|start_header_id|>system<|end_header_id|>\n\nYou are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nGive me a fully functional FastAPI server. Show the python code.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n" + ] + output = engine.generate(test_prompt, sampling_params) + output = output[0] + + if "spec_verify_ct" in output["meta_info"]: + acc_length = ( + output["meta_info"]["completion_tokens"] + / output["meta_info"]["spec_verify_ct"] + ) + else: + acc_length = 1.0 + + speed = ( + output["meta_info"]["completion_tokens"] + / output["meta_info"]["e2e_latency"] + ) + print(f"{acc_length=:.4f}, {speed=}") + + self.assertGreater(acc_length, 2.5) + + def _test_batch_generation(self, engine): + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + params = {"temperature": 0, "max_new_tokens": 50} + + outputs = engine.generate(prompts, params) + for prompt, output in zip(prompts, outputs): + print(f"Prompt: {prompt}") + print(f"Generated: {output['text']}") + print("-" * 40) + + print(f"{engine.get_server_info()=}") + + avg_spec_accept_length = engine.get_server_info()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, 2.0) + + +@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") +class TestEAGLEDraftExtend(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + 1, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 2, + "--max-running-requests", + 4, + "--attention-backend", + "fa3", + ], + ) + cls.accept_len_threshold = 1.50 + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_one_batch_accept_length(self): + resp = requests.get(self.base_url + "/flush_cache") + self.assertEqual(resp.status_code, 200) + + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + url = self.base_url + "/generate" + data = { + "text": prompts, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 512, + }, + } + response = requests.post(url, json=data) + self.assertEqual(response.status_code, 200) + outputs = response.json() + for i in range(len(prompts)): + output = outputs[i] + if "spec_verify_ct" in output["meta_info"]: + acc_length = ( + output["meta_info"]["completion_tokens"] + / output["meta_info"]["spec_verify_ct"] + ) + else: + acc_length = 1.0 + + print(f"{acc_length=}") + self.assertGreater(acc_length, self.accept_len_threshold) + + +class TestEAGLEDraftExtendFlashinfer(TestEAGLEDraftExtend): + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + 1, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 2, + "--max-running-requests", + 4, + "--attention-backend", + "flashinfer", + ], + ) + cls.accept_len_threshold = 1.50 + + +@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") +class TestEAGLEDraftExtendTriton(TestEAGLEDraftExtend): + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + 1, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 2, + "--max-running-requests", + 4, + "--attention-backend", + "triton", + ], + ) + cls.accept_len_threshold = 1.50 + + +@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") +class TestEAGLEDraftExtendFlashinferMLA(TestEAGLEDraftExtend): + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + DEFAULT_MODEL_NAME_FOR_TEST_MLA, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + 1, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 2, + "--max-running-requests", + 4, + "--attention-backend", + "flashinfer", + ], + ) + cls.accept_len_threshold = 1.85 + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eagle_infer_b.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eagle_infer_b.py new file mode 100644 index 00000000..5b9df163 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eagle_infer_b.py @@ -0,0 +1,511 @@ +import json +import os +import random +import threading +import time +import unittest +from concurrent.futures import ThreadPoolExecutor +from functools import partial +from types import SimpleNamespace + +import numpy as np +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.test_utils import ( + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, + run_logprob_check, +) + + +class TestEAGLEServer(CustomTestCase): + PROMPTS = [ + "[INST] <>\\nYou are a helpful assistant.\\n<>\\nToday is a sunny day and I like[/INST]" + '[INST] <>\\nYou are a helpful assistant.\\n<>\\nWhat are the mental triggers in Jeff Walker\'s Product Launch Formula and "Launch" book?[/INST]', + "[INST] <>\\nYou are a helpful assistant.\\n<>\\nSummarize Russell Brunson's Perfect Webinar Script...[/INST]", + "[INST] <>\\nYou are a helpful assistant.\\n<>\\nwho are you?[/INST]", + "[INST] <>\\nYou are a helpful assistant.\\n<>\\nwhere are you from?[/INST]", + ] + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + 5, + "--speculative-eagle-topk", + 8, + "--speculative-num-draft-tokens", + 64, + "--mem-fraction-static", + 0.7, + "--chunked-prefill-size", + 128, + "--max-running-requests", + 8, + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def send_request(self): + time.sleep(random.uniform(0, 2)) + for prompt in self.PROMPTS: + url = self.base_url + "/generate" + data = { + "text": prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 1024, + }, + } + response = requests.post(url, json=data) + assert response.status_code == 200 + + def send_requests_abort(self): + for prompt in self.PROMPTS: + try: + time.sleep(random.uniform(0, 2)) + url = self.base_url + "/generate" + data = { + "model": "base", + "text": prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 1024, + }, + } + # set timeout = 1s, mock disconnected + requests.post(url, json=data, timeout=1) + except Exception as e: + print(e) + pass + + def test_request_abort(self): + concurrency = 4 + threads = [ + threading.Thread(target=self.send_request) for _ in range(concurrency) + ] + [ + threading.Thread(target=self.send_requests_abort) + for _ in range(concurrency) + ] + for worker in threads: + worker.start() + for p in threads: + p.join() + + def test_max_token_one(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=1, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + + # Just run and check it does not hang + metrics = run_eval(args) + self.assertGreater(metrics["output_throughput"], 50) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], 0.20) + + server_info = requests.get(self.base_url + "/get_server_info").json() + avg_spec_accept_length = server_info["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + + speculative_eagle_topk = server_info["speculative_eagle_topk"] + + if speculative_eagle_topk == 1: + self.assertGreater(avg_spec_accept_length, 2.5) + else: + self.assertGreater(avg_spec_accept_length, 3.5) + + # Wait a little bit so that the memory check happens. + time.sleep(4) + + def test_logprob_start_len(self): + logprob_start_len = 4 + new_tokens = 4 + prompts = [ + "I have a very good idea on", + "Today is a sunndy day and", + ] + + response = requests.post( + self.base_url + "/generate", + json={ + "text": prompts, + "sampling_params": { + "temperature": 0, + "max_new_tokens": new_tokens, + }, + "return_logprob": True, + "top_logprobs_num": 5, + "logprob_start_len": logprob_start_len, + }, + ) + response_json = response.json() + print(json.dumps(response_json, indent=2)) + + for res in response_json: + self.assertEqual( + res["meta_info"]["prompt_tokens"], + logprob_start_len + len(res["meta_info"]["input_token_logprobs"]), + ) + + self.assertEqual(res["meta_info"]["completion_tokens"], new_tokens) + self.assertEqual(len(res["meta_info"]["output_token_logprobs"]), new_tokens) + + def test_logprob_match(self): + """Test the output logprobs are close to the input logprobs if we run a prefill again.""" + + def run_generate( + prompt, + return_logprob=False, + max_new_tokens=512, + logprob_start_len=-1, + temperature=1.0, + ): + + if isinstance(prompt, str): + prompt_kwargs = {"text": prompt} + else: + prompt_kwargs = {"input_ids": prompt} + + response = requests.post( + self.base_url + "/generate", + json={ + **prompt_kwargs, + "sampling_params": { + "temperature": temperature, + "max_new_tokens": max_new_tokens, + "ignore_eos": True, + }, + "return_logprob": return_logprob, + "return_text_in_logprobs": True, + "logprob_start_len": logprob_start_len, + "temp_scaled_logprobs": True, + }, + ) + return response.json() + + prompt = "I have a very good idea on how to" + + for temperature in [1.0]: + gen = run_generate( + prompt, + return_logprob=True, + logprob_start_len=0, + temperature=temperature, + ) + output_logprobs = np.array( + [x[0] for x in gen["meta_info"]["output_token_logprobs"]] + ) + num_prompts_tokens = gen["meta_info"]["prompt_tokens"] + + input_tokens = [x[1] for x in gen["meta_info"]["input_token_logprobs"]] + output_tokens = [x[1] for x in gen["meta_info"]["output_token_logprobs"]] + + new_prompt = input_tokens + output_tokens + score = run_generate( + new_prompt, + return_logprob=True, + logprob_start_len=0, + max_new_tokens=0, + temperature=temperature, + ) + output_logprobs_score = np.array( + [ + x[0] + for x in score["meta_info"]["input_token_logprobs"][ + num_prompts_tokens: + ] + ] + ) + + print(f"{output_logprobs[-10:]=}") + print(f"{output_logprobs_score[-10:]=}") + + diff = np.abs(output_logprobs - output_logprobs_score) + max_diff = np.max(diff) + self.assertLess(max_diff, 0.255) + + def test_logprob_mixed(self): + args = [] + temperature = 0 + # input_len, output_len, temperature, logprob_start_len, return_logprob, top_logprobs_num + # Llama 2 context length seems to be only 2k, so we can only test small length. + for input_len in [200, 500, 1000, 2000]: + for output_len in [4, 8]: + for logprob_start_len in [0, 100, 300, 800, 1998]: + for return_logprob in [True, False]: + for top_logprobs_num in [0, 5]: + + if logprob_start_len >= input_len: + continue + + args.append( + ( + input_len, + output_len, + temperature, + logprob_start_len, + return_logprob, + top_logprobs_num, + ) + ) + + random.shuffle(args) + + func = partial(run_logprob_check, self) + with ThreadPoolExecutor(8) as executor: + list(executor.map(func, args)) + + def run_decode(self, sampling_params): + return_logprob = True + top_logprobs_num = 5 + return_text = True + n = 1 + + response = requests.post( + self.base_url + "/generate", + json={ + "text": "Human: Write a travel blog post to Hawaii.\n\nAssistant:", + "sampling_params": { + "max_new_tokens": 48, + "n": n, + "temperature": 0.7, + **sampling_params, + }, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "return_text_in_logprobs": return_text, + "logprob_start_len": 0, + }, + ) + self.assertEqual(response.status_code, 200) + print(json.dumps(response.json())) + print("=" * 100) + + def test_penalty_mixed(self): + args = [ + {}, + {}, + {}, + {"frequency_penalty": 2}, + {"presence_penalty": 1}, + {"min_new_tokens": 16}, + {"frequency_penalty": 0.2}, + {"presence_penalty": 0.4}, + {"min_new_tokens": 8}, + {"frequency_penalty": 0.4, "presence_penalty": 0.8}, + {"frequency_penalty": 0.4, "min_new_tokens": 12}, + {"presence_penalty": 0.8, "min_new_tokens": 12}, + {"presence_penalty": -0.3, "frequency_penalty": 1.3, "min_new_tokens": 32}, + {"presence_penalty": 0.3, "frequency_penalty": -1.3, "min_new_tokens": 32}, + ] + random.shuffle(args * 5) + with ThreadPoolExecutor(8) as executor: + list(executor.map(self.run_decode, args)) + + def test_constrained_decoding(self): + messages = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Give me a json"}, + ] + + response = requests.post( + self.base_url + "/v1/chat/completions", + json={ + "model": DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + "messages": messages, + "temperature": 0, + "response_format": {"type": "json_object"}, + }, + ) + self.assertEqual(response.status_code, 200) + res = response.json() + + # Validate response structure + self.assertIn("choices", res) + self.assertEqual(len(res["choices"]), 1) + self.assertIn("message", res["choices"][0]) + self.assertIn("content", res["choices"][0]["message"]) + + # Validate JSON content + content_json = res["choices"][0]["message"]["content"] + is_valid_json = True + try: + content = json.loads(content_json) + self.assertIsInstance(content, dict) + except Exception: + print(f"parse JSON failed: {content_json}") + is_valid_json = False + self.assertTrue(is_valid_json) + + +class TestEAGLERetract(TestEAGLEServer): + @classmethod + def setUpClass(cls): + # These config helps find a leak. + os.environ["SGLANG_CI_SMALL_KV_SIZE"] = "4500" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + 5, + "--speculative-eagle-topk", + 8, + "--speculative-num-draft-tokens", + 64, + "--mem-fraction-static", + 0.7, + "--chunked-prefill-size", + 128, + "--max-running-requests", + 64, + ], + ) + + +class TestEAGLEServerTriton(TestEAGLEServer): + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + 5, + "--speculative-eagle-topk", + 8, + "--speculative-num-draft-tokens", + 64, + "--mem-fraction-static", + 0.7, + "--attention-backend", + "triton", + "--max-running-requests", + 8, + ], + ) + + +class TestEAGLEServerPageSize(TestEAGLEServer): + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + 5, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 6, + "--mem-fraction-static", + 0.7, + "--chunked-prefill-size", + 128, + "--max-running-requests", + 8, + "--page-size", + 4, + "--attention-backend", + "flashinfer", + ], + ) + + +class TestEAGLEServerPageSizeTopk(TestEAGLEServer): + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + 5, + "--speculative-eagle-topk", + 8, + "--speculative-num-draft-tokens", + 64, + "--mem-fraction-static", + 0.7, + "--chunked-prefill-size", + 128, + "--max-running-requests", + 8, + "--page-size", + 4, + "--attention-backend", + "flashinfer", + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eagle_infer_beta.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eagle_infer_beta.py new file mode 100644 index 00000000..8cf9936e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eagle_infer_beta.py @@ -0,0 +1,96 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.environ import envs +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.kits.matched_stop_kit import MatchedStopMixin +from sglang.test.kits.radix_cache_server_kit import run_radix_attention_test +from sglang.test.test_utils import ( + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestEagleServerBase(CustomTestCase, MatchedStopMixin): + max_running_requests = 64 + attention_backend = "triton" + spec_steps = 5 + spec_topk = 1 + spec_draft_tokens = 6 + page_size = 1 + other_launch_args = [] + model = DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST + draft_model = DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + launch_args = [ + "--trust-remote-code", + "--attention-backend", + cls.attention_backend, + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model", + cls.draft_model, + "--speculative-num-steps", + cls.spec_steps, + "--speculative-eagle-topk", + cls.spec_topk, + "--speculative-num-draft-tokens", + cls.spec_draft_tokens, + "--page-size", + str(cls.page_size), + "--mem-fraction-static", + "0.75", + "--max-running-requests", + str(cls.max_running_requests), + "--cuda-graph-bs", + *[str(i) for i in range(1, cls.max_running_requests + 1)], + ] + launch_args.extend(cls.other_launch_args) + with envs.SGLANG_ENABLE_SPEC_V2.override(True): + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=launch_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_radix_attention(self): + run_radix_attention_test(self.base_url) + assert self.process.poll() is None + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1000, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"TestEagleLargeBS -- {metrics=}") + self.assertGreater( + metrics["accuracy"], 0.23 + ) # 0.3333 for 60 questions; 0.234 for 1319 questions + assert self.process.poll() is None + + +class TestEagleServerPage(TestEagleServerBase): + other_launch_args = ["--page-size", "64"] + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eval_accuracy_large.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eval_accuracy_large.py new file mode 100644 index 00000000..efb20246 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eval_accuracy_large.py @@ -0,0 +1,92 @@ +""" +Usage: +python -m unittest test_eval_accuracy_large.TestEvalAccuracyLarge.test_mmlu +""" + +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + + +class TestEvalAccuracyLarge(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--log-level-http", "warning"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=5000, + num_threads=1024, + ) + + metrics = run_eval(args) + + if is_in_ci(): + write_github_step_summary(f"### test_mmlu\n" f'{metrics["score"]=:.4f}\n') + + self.assertGreater(metrics["score"], 0.70) + + def test_human_eval(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="humaneval", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + + if is_in_ci(): + write_github_step_summary( + f"### test_human_eval\n" f'{metrics["score"]=:.4f}\n' + ) + + self.assertGreater(metrics["score"], 0.64) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + + if is_in_ci(): + write_github_step_summary( + f"### test_mgsm_en\n" f'{metrics["score"]=:.4f}\n' + ) + + self.assertGreater(metrics["score"], 0.835) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eval_fp8_accuracy.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eval_fp8_accuracy.py new file mode 100644 index 00000000..329e2dad --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_eval_fp8_accuracy.py @@ -0,0 +1,114 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import is_hip, kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_ACCURACY_TEST_FP8, + DEFAULT_MODEL_NAME_FOR_DYNAMIC_QUANT_ACCURACY_TEST_FP8, + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestEvalFP8Accuracy(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_ACCURACY_TEST_FP8 + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, cls.base_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + temperature=0.1, + ) + + metrics = run_eval(args) + if is_hip(): + # Another threshold for AMD because fp8 dtype is difference + self.assertGreaterEqual(metrics["score"], 0.60) + else: + self.assertGreaterEqual(metrics["score"], 0.60) + + +class TestEvalFP8DynamicQuantAccuracy(CustomTestCase): + + def _run_test(self, model, other_args, expected_score): + base_url = DEFAULT_URL_FOR_TEST + other_args = other_args or [] + + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + try: + args = SimpleNamespace( + base_url=base_url, + model=model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + temperature=0.1, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], expected_score) + finally: + kill_process_tree(process.pid) + + def test_mmlu_offline_only(self): + """Test with offline quantization only.""" + self._run_test( + model=DEFAULT_MODEL_NAME_FOR_DYNAMIC_QUANT_ACCURACY_TEST_FP8, + other_args=[], + expected_score=0.64, + ) + + def test_mmlu_offline_and_online_override(self): + """Test with both offline and online quantization.""" + self._run_test( + model=DEFAULT_MODEL_NAME_FOR_DYNAMIC_QUANT_ACCURACY_TEST_FP8, + other_args=["--quantization", "w8a8_fp8"], + # inference will use sgl kernel w/ online quant override + # we observed that the accuracy is higher then offline only + expected_score=0.64, + ) + + def test_mmlu_online_only(self): + """Test with online quantization only.""" + self._run_test( + model=DEFAULT_MODEL_NAME_FOR_TEST, + # inference will use sgl kernel w/ online quantization only + # we observed that the accuracy is higher then offline only + other_args=["--quantization", "w8a8_fp8"], + expected_score=0.64, + ) + + def test_mmlu_fp16_baseline(self): + """Test with unquantized fp16 baseline.""" + self._run_test( + model=DEFAULT_MODEL_NAME_FOR_TEST, + other_args=[], + expected_score=0.64, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_external_models.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_external_models.py new file mode 100644 index 00000000..cab5b785 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_external_models.py @@ -0,0 +1,28 @@ +import os +import unittest + +import sglang as sgl +from sglang.test.test_utils import CustomTestCase + + +class TestExternalModels(CustomTestCase): + def test_external_model(self): + os.environ["SGLANG_EXTERNAL_MODEL_PACKAGE"] = "external_models" + os.environ["SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE"] = "external_models" + prompt = "Today is a sunny day and I like" + model_path = "Qwen/Qwen2-VL-2B-Instruct" + + engine = sgl.Engine( + model_path=model_path, + cuda_graph_max_bs=1, + max_total_tokens=64, + enable_multimodal=True, + ) + out = engine.generate(prompt)["text"] + engine.shutdown() + + self.assertGreater(len(out), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_fa3.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_fa3.py new file mode 100644 index 00000000..739b143f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_fa3.py @@ -0,0 +1,263 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.environ import envs +from sglang.srt.utils import get_device_sm, kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3, + DEFAULT_MODEL_NAME_FOR_TEST_MLA, + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +GSM_DATASET_PATH = None + +# In case of some machine lack internet connection, we can set OFFLINE_MODE to True. +OFFLINE_MODE = False + +# Change the path below when OFFLINE_MODE is True. +OFFLINE_PATH_DICT = { + DEFAULT_MODEL_NAME_FOR_TEST: "/shared/public/elr-models/meta-llama/Meta-Llama-3.1-8B-Instruct", + DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3: "/shared/public/elr-models/jamesliu1/sglang-EAGLE3-Llama-3.1-Instruct-8B", + DEFAULT_MODEL_NAME_FOR_TEST_MLA: "/shared/public/sharing/deepseek/dsv3-test/snapshots/", + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN: "/shared/public/sharing/deepseek/dsv3-test-NextN/snapshots/", + GSM_DATASET_PATH: "/shared/public/data/gsm8k/test.jsonl", +} + + +if OFFLINE_MODE: + DEFAULT_MODEL_NAME_FOR_TEST = OFFLINE_PATH_DICT[DEFAULT_MODEL_NAME_FOR_TEST] + DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3 = OFFLINE_PATH_DICT[ + DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3 + ] + DEFAULT_MODEL_NAME_FOR_TEST_MLA = OFFLINE_PATH_DICT[DEFAULT_MODEL_NAME_FOR_TEST_MLA] + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN = OFFLINE_PATH_DICT[ + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN + ] + GSM_DATASET_PATH = OFFLINE_PATH_DICT[GSM_DATASET_PATH] + + +# Default server arguments shared across all tests +DEFAULT_SERVER_ARGS = [ + "--trust-remote-code", + "--cuda-graph-max-bs", + "8", + "--attention-backend", + "fa3", +] + +""" +Integration test for python/sglang/srt/layers/attention/flashattention_backend.py +""" + + +@unittest.skipIf(get_device_sm() < 90, "Test requires CUDA SM 90 or higher") +class BaseFlashAttentionTest(CustomTestCase): + """Base class for testing FlashAttention3.""" + + model = DEFAULT_MODEL_NAME_FOR_TEST + base_url = DEFAULT_URL_FOR_TEST + accuracy_threshold = 0.65 # derived tests need to override this + speculative_decode = False + spec_decode_threshold = 1.0 # derived spec decoding tests need to override this + + @classmethod + def get_server_args(cls): + """Return the arguments for the server launch. Override in subclasses.""" + return DEFAULT_SERVER_ARGS + + @classmethod + def setUpClass(cls): + # disable deep gemm precompile to make launch server faster + # please don't do this if you want to make your inference workload faster + with ( + envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.override(False), + envs.SGLANG_ENABLE_JIT_DEEPGEMM.override(False), + ): + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=cls.get_server_args(), + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=4, + num_questions=100, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + data_path=GSM_DATASET_PATH, + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + # Use the appropriate metric key based on the test class + metric_key = "accuracy" + self.assertGreater(metrics[metric_key], self.accuracy_threshold) + + if self.speculative_decode: + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, self.spec_decode_threshold) + + +class TestFlashAttention3MLA(BaseFlashAttentionTest): + """Test FlashAttention3 with MLA, e.g. deepseek v3 test model""" + + accuracy_threshold = 0.60 + model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + + @classmethod + def get_server_args(cls): + return DEFAULT_SERVER_ARGS + + +class TestFlashAttention3SpeculativeDecode(BaseFlashAttentionTest): + """Test FlashAttention3 with speculative decode enabled with Llama 3.1 8B and its eagle3 model""" + + model = DEFAULT_MODEL_NAME_FOR_TEST + accuracy_threshold = 0.65 + speculative_decode = True + spec_decode_threshold = 1.5 + + @classmethod + def get_server_args(cls): + args = DEFAULT_SERVER_ARGS + args.extend( + [ + "--cuda-graph-max-bs", + "4", + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3, + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--dtype", + "float16", + ] + ) + return args + + +class TestFlashAttention3SpeculativeDecodeTopk(BaseFlashAttentionTest): + """Tests FlashAttention3 with enhanced speculative decoding using Llama 3.1 8B and EAGLE3. + This test will be using top-k value > 1 which would verify the other branches of the FA3 code + """ + + model = DEFAULT_MODEL_NAME_FOR_TEST + accuracy_threshold = 0.65 + speculative_decode = True + spec_decode_threshold = 1.6 + + @classmethod + def get_server_args(cls): + args = DEFAULT_SERVER_ARGS + args.extend( + [ + "--cuda-graph-max-bs", + "4", + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3, + "--speculative-num-steps", + "5", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "8", + "--dtype", + "float16", + ] + ) + return args + + +class TestFlashAttention3MLASpeculativeDecode(BaseFlashAttentionTest): + """Test FlashAttention3 with speculative decode enabled with deepseek v3 test model and its nextN model""" + + model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + accuracy_threshold = 0.60 + speculative_decode = True + spec_decode_threshold = 2.5 + + @classmethod + def get_server_args(cls): + args = DEFAULT_SERVER_ARGS + args.extend( + [ + "--cuda-graph-max-bs", + "4", + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + ] + ) + return args + + +class TestFlashAttention3MLASpeculativeDecodeTopk(BaseFlashAttentionTest): + """Test FlashAttention3 with speculative decode enabled with deepseek v3 test model and its nextN model + This test will be using top-k value > 1 which would verify the other branches of the FA3 code + """ + + model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + accuracy_threshold = 0.60 + speculative_decode = True + spec_decode_threshold = 2.95 + + @classmethod + def get_server_args(cls): + args = DEFAULT_SERVER_ARGS + args.extend( + [ + "--cuda-graph-max-bs", + "4", + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, + "--speculative-num-steps", + "5", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "8", + ] + ) + return args + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_flash_attention_4.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_flash_attention_4.py new file mode 100644 index 00000000..4322263c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_flash_attention_4.py @@ -0,0 +1,54 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import get_device_sm, kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + popen_launch_server, +) + + +@unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher") +class TestFlashAttention4(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model = "Qwen/Qwen3-8B" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--mem-fraction-static", + "0.8", + "--prefill-attention-backend", + "fa4", + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=4, + data_path=None, + num_questions=100, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.75) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_flashmla.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_flashmla.py new file mode 100644 index 00000000..e9c69e5d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_flashmla.py @@ -0,0 +1,132 @@ +""" +Usage: +python3 test/srt/test_flashmla.py +""" + +import unittest +from types import SimpleNamespace + +import requests +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST_MLA, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestFlashMLAAttnBackend(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + [ + "--cuda-graph-max-bs", + "2", + "--attention-backend", + "flashmla", + ] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + +class TestFlashMLAMTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmsys/sglang-ci-dsv3-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + [ + "--cuda-graph-max-bs", + "4", + "--disable-radix", + "--enable-torch-compile", + "--torch-compile-max-bs", + "1", + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/sglang-ci-dsv3-test-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "3", + "--attention-backend", + "flashmla", + ] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + server_info = requests.get(self.base_url + "/get_server_info") + print(f"{server_info=}") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, 2.4) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_fp8_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_fp8_utils.py new file mode 100644 index 00000000..1e75d485 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_fp8_utils.py @@ -0,0 +1,44 @@ +import unittest + +import torch + +from sglang.srt.layers.quantization.fp8_utils import ( + inverse_transform_scale_ue8m0, + quant_weight_ue8m0, + transform_scale_ue8m0, +) +from sglang.test.test_utils import CustomTestCase + + +class TestInverseTransformScaleUe8m0(CustomTestCase): + def test_round_trip(self): + for _ in range(100): + weight_bf16 = torch.randn( + # DeepSeek V3 kv_b_proj + (32768, 512), + dtype=torch.bfloat16, + device="cuda", + ) + + weight_block_size = [128, 128] + + qweight, sf_fp32_original = quant_weight_ue8m0( + weight_bf16, weight_block_size=weight_block_size + ) + mn = qweight.shape[-2] + + sf_packed_original = transform_scale_ue8m0(sf_fp32_original, mn=mn) + sf_fp32_recreated = inverse_transform_scale_ue8m0(sf_packed_original, mn=mn) + + sf_packed_recreated = transform_scale_ue8m0(sf_fp32_recreated, mn=mn) + + assert torch.all( + sf_packed_original == sf_packed_recreated + ), f"{sf_packed_original=} {sf_packed_recreated}" + assert torch.all( + sf_fp32_original == sf_fp32_recreated + ), f"{sf_fp32_original=} {sf_fp32_recreated}" + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_fused_moe.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_fused_moe.py new file mode 100644 index 00000000..65d35c59 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_fused_moe.py @@ -0,0 +1,240 @@ +import unittest + +import torch +from tqdm import tqdm + +from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import fused_moe +from sglang.srt.layers.moe.topk import TopKConfig, select_experts +from sglang.srt.layers.quantization.fp8_kernel import is_fp8_fnuz +from sglang.srt.layers.quantization.fp8_utils import normalize_e4m3fn_to_e4m3fnuz +from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler +from sglang.srt.utils import is_hip +from sglang.test.test_utils import CustomTestCase + +_is_hip = is_hip() +_is_fp8_fnuz = is_fp8_fnuz() + + +class TestFusedMOE(CustomTestCase): + NUM_EXPERTS = [8, 64] + TOP_KS = [2, 6] + + @staticmethod + def create_random_cuda_tensor(shape, dtype, mean=0, std=0.01): + """Create a random CUDA tensor + + Args: + shape: Tensor shape + dtype: Data type + mean: Mean value + std: Standard deviation + + Returns: + torch.Tensor: Randomly initialized CUDA tensor + """ + return torch.empty(shape, dtype=dtype, device="cuda").normal_(mean, std) + + def get_tolerance(self, dtype): + """Get tolerance values for different data types + + Args: + dtype: Data type + + Returns: + tuple: (relative tolerance, absolute tolerance) + """ + if dtype == torch.float32: + return 1e-3, 1e-5 + elif dtype in [torch.float16, torch.bfloat16]: + return 1e-1, 1e-2 + else: + return 1e-2, 1e-2 # Default values for other types + + def torch_naive_moe( + self, + a, + w1, + w2, + score, + topk, + w1_scale=None, + w2_scale=None, + a1_scale=None, + a2_scale=None, + ): + set_global_server_args_for_scheduler(ServerArgs(model_path="dummy")) + + B, D = a.shape + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + + if w1.dtype in [torch.float8_e4m3fn, torch.float8_e4m3fnuz]: + w1_compute = w1.to(a.dtype) + w2_compute = w2.to(a.dtype) + + if w1_scale is not None: + w1_compute = (w1_compute * w1_scale.view(-1, 1, 1)).to(a.dtype) + if w2_scale is not None: + w2_compute = (w2_compute * w2_scale.view(-1, 1, 1)).to(a.dtype) + if a1_scale is not None: + a = (a * a1_scale).to(a.dtype) + if a2_scale is not None: + a = (a * a2_scale).to(a.dtype) + else: + w1_compute = w1 + w2_compute = w2 + + for i in range(w1_compute.shape[0]): + mask = topk_ids == i + if mask.sum(): + out[mask] = SiluAndMul()( + a[mask] @ w1_compute[i].transpose(0, 1) + ) @ w2_compute[i].transpose(0, 1) + + return ( + out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype) + ).sum(dim=1) + + def _test_case(self, m, n, k, e, topk, dtype, use_fp8_w8a8=False): + rtol, atol = self.get_tolerance(dtype) + + if use_fp8_w8a8: + # AssertionError: fp8e4nv data type is not supported on CUDA arch < 89 + capability = torch.cuda.get_device_capability() + if not _is_hip and not (capability[0] >= 9 or capability == (8, 9)): + return + + a = self.create_random_cuda_tensor((m, k), dtype) + w1 = self.create_random_cuda_tensor((e, 2 * n, k), dtype) + w2 = self.create_random_cuda_tensor((e, k, n), dtype) + w1 = w1.to(torch.float8_e4m3fn) + w2 = w2.to(torch.float8_e4m3fn) + score = self.create_random_cuda_tensor((m, e), dtype) + w1_scale = self.create_random_cuda_tensor(e, torch.float32) + w2_scale = self.create_random_cuda_tensor(e, torch.float32) + a1_scale = self.create_random_cuda_tensor(1, torch.float32) + a2_scale = self.create_random_cuda_tensor(1, torch.float32) + + # Handle HIP case: normalize float8 weights so fused kernel doesn't break + # on ROCm. + if _is_fp8_fnuz: + # Normalize to e4m3fnuz on HIP + w1, w1_scale, _ = normalize_e4m3fn_to_e4m3fnuz( + weight=w1, + weight_scale=w1_scale, + input_scale=a1_scale, + ) + w2, w2_scale, _ = normalize_e4m3fn_to_e4m3fnuz( + weight=w2, + weight_scale=w2_scale, + input_scale=a2_scale, + ) + + topk_output = select_experts( + hidden_states=a, + router_logits=score, + topk_config=TopKConfig(top_k=topk, renormalize=False), + ) + + torch_output = self.torch_naive_moe( + a, + w1, + w2, + score, + topk, + w1_scale, + w2_scale, + a1_scale, + a2_scale, + ) + + sglang_output = fused_moe( + a, + w1, + w2, + topk_output, + use_fp8_w8a8=True, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + ) + torch.testing.assert_close( + sglang_output, torch_output, rtol=rtol, atol=atol + ) + else: + a = self.create_random_cuda_tensor((m, k), dtype) + w1 = self.create_random_cuda_tensor((e, 2 * n, k), dtype) + w2 = self.create_random_cuda_tensor((e, k, n), dtype) + score = self.create_random_cuda_tensor((m, e), dtype) + + topk_output = select_experts( + hidden_states=a, + router_logits=score, + topk_config=TopKConfig(top_k=topk, renormalize=False), + ) + + triton_output = fused_moe(a, w1, w2, topk_output) + torch_output = self.torch_naive_moe(a, w1, w2, score, topk) + torch.testing.assert_close( + triton_output, torch_output, rtol=rtol, atol=atol + ) + + def test_various_configurations(self): + m_values = [1, 33, 64, 222] + n_values = [128, 1024] + k_values = [128, 511, 1024] + dtypes = [torch.float16, torch.bfloat16] + fp8_modes = [False, True] + + set_global_server_args_for_scheduler(ServerArgs(model_path="dummy")) + + # Calculate total number of tests + total_tests = ( + len(m_values) + * len(n_values) + * len(k_values) + * len(self.NUM_EXPERTS) + * len(self.TOP_KS) + * len(dtypes) + * len(fp8_modes) + ) + + # Create progress bar + with tqdm(total=total_tests, desc="Running MoE tests") as pbar: + for m in m_values: + for n in n_values: + for k in k_values: + for e in self.NUM_EXPERTS: + for topk in self.TOP_KS: + for dtype in dtypes: + for use_fp8_w8a8 in fp8_modes: + with self.subTest( + m=m, + n=n, + k=k, + e=e, + topk=topk, + dtype=dtype, + fp8=use_fp8_w8a8, + ): + self._test_case( + m, + n, + k, + e, + topk, + dtype, + use_fp8_w8a8=use_fp8_w8a8, + ) + torch.cuda.empty_cache() + pbar.update(1) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_gguf.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_gguf.py new file mode 100644 index 00000000..e9776067 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_gguf.py @@ -0,0 +1,27 @@ +import unittest + +from huggingface_hub import hf_hub_download + +import sglang as sgl +from sglang.test.test_utils import CustomTestCase + + +class TestGGUF(CustomTestCase): + def test_models(self): + prompt = "Today is a sunny day and I like" + sampling_params = {"temperature": 0, "max_new_tokens": 8} + + model_path = hf_hub_download( + "Qwen/Qwen2-1.5B-Instruct-GGUF", + filename="qwen2-1_5b-instruct-q4_k_m.gguf", + ) + + engine = sgl.Engine(model_path=model_path, random_seed=42, cuda_graph_max_bs=2) + outputs = engine.generate(prompt, sampling_params)["text"] + engine.shutdown() + + self.assertEqual(outputs, " it. I have a lot of work") + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_gpt_oss_1gpu.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_gpt_oss_1gpu.py new file mode 100644 index 00000000..d3fc0693 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_gpt_oss_1gpu.py @@ -0,0 +1,31 @@ +import unittest + +from test_gpt_oss_common import BaseTestGptOss + + +class TestGptOss1Gpu(BaseTestGptOss): + def test_mxfp4_20b(self): + self.run_test( + model_variant="20b", + quantization="mxfp4", + expected_score_of_reasoning_effort={ + "low": 0.34, + "medium": 0.34, + "high": 0.27, # TODO investigate + }, + ) + + def test_bf16_20b(self): + self.run_test( + model_variant="20b", + quantization="bf16", + expected_score_of_reasoning_effort={ + "low": 0.34, + "medium": 0.34, + "high": 0.27, # TODO investigate + }, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_gpt_oss_4gpu.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_gpt_oss_4gpu.py new file mode 100644 index 00000000..da787c6f --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_gpt_oss_4gpu.py @@ -0,0 +1,36 @@ +import unittest + +from test_gpt_oss_common import BaseTestGptOss + + +class TestGptOss4Gpu(BaseTestGptOss): + def test_bf16_120b(self): + self.run_test( + model_variant="120b", + quantization="bf16", + expected_score_of_reasoning_effort={ + "low": 0.60, + }, + other_args=["--tp", "4", "--cuda-graph-max-bs", "200"], + ) + + def test_mxfp4_120b(self): + self.run_test( + model_variant="120b", + quantization="mxfp4", + expected_score_of_reasoning_effort={ + "low": 0.60, + }, + other_args=[ + "--tp", + "4", + "--cuda-graph-max-bs", + "200", + "--mem-fraction-static", + "0.93", + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_gpt_oss_common.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_gpt_oss_common.py new file mode 100644 index 00000000..68402b5e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_gpt_oss_common.py @@ -0,0 +1,152 @@ +import json +import os +from concurrent.futures import ThreadPoolExecutor +from types import SimpleNamespace +from typing import Dict, List, Literal, Optional + +import requests + +from sglang.srt.utils import is_hip, kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +_base_url = DEFAULT_URL_FOR_TEST +_is_hip = is_hip() + + +class BaseTestGptOss(CustomTestCase): + def run_test( + self, + model_variant: Literal["20b", "120b"], + quantization: Literal["mxfp4", "bf16"], + expected_score_of_reasoning_effort: Dict[str, float], + other_args: Optional[List[str]] = None, + ): + if other_args is None: + other_args = [] + + model = { + ("20b", "bf16"): "lmsys/gpt-oss-20b-bf16", + ("120b", "bf16"): "lmsys/gpt-oss-120b-bf16", + ("20b", "mxfp4"): "openai/gpt-oss-20b", + ("120b", "mxfp4"): "openai/gpt-oss-120b", + }[(model_variant, quantization)] + + if model_variant == "20b": + other_args += ["--cuda-graph-max-bs", "600"] + if _is_hip: + os.environ["SGLANG_USE_AITER"] = "0" + self._run_test_raw( + model=model, + expected_score_of_reasoning_effort=expected_score_of_reasoning_effort, + other_args=other_args, + ) + + def _run_test_raw( + self, + model: str, + expected_score_of_reasoning_effort: Dict[str, float], + other_args: List[str], + ): + process = popen_launch_server( + model, + _base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + try: + self._check_streaming_responses_api_request(model) + + # run multiple tests in parallel since we are mostly bound by the longest generate sequence + # instead of the number of questions + with ThreadPoolExecutor(max_workers=4) as executor: + list( + executor.map( + lambda d: self._run_one_eval(**d), + [ + dict( + model=model, + reasoning_effort=reasoning_effort, + expected_score=expected_score, + ) + for reasoning_effort, expected_score in expected_score_of_reasoning_effort.items() + ], + ) + ) + finally: + kill_process_tree(process.pid) + + def _check_streaming_responses_api_request(self, model): + # Use requests to verify /v1/responses streaming + url = f"{_base_url}/v1/responses" + payload = { + "model": model, + "input": "What is 1 + 1?", + "stream": True, + "temperature": 0, + } + + response = requests.post(url, json=payload, stream=True) + if response.status_code != 200: + print(f"Response API failed: {response.text}") + response.raise_for_status() + + content = "" + for line in response.iter_lines(): + if line: + decoded_line = line.decode("utf-8") + if decoded_line.startswith("data: "): + data_str = decoded_line[6:] + if data_str.strip() == "[DONE]": + break + + try: + data = json.loads(data_str) + if data.get("type") == "response.output_text.delta": + delta = data.get("delta", "") + content += delta + except json.JSONDecodeError: + pass + + print(f"Streaming check response: {content}") + self.assertTrue(len(content) > 0) + self.assertIn("2", content) + + def _run_one_eval(self, model, reasoning_effort, expected_score): + args = SimpleNamespace( + base_url=_base_url, + model=model, + eval_name="gpqa", + num_examples=198, + # use enough threads to allow parallelism + num_threads=198, + # TODO 4k is still not enough, we need e.g. 64k token, but that is super slow + # otherwise a lot of questions are not answered + max_tokens=4096, + # simple-evals by default use 0.5 and is better than 0.0 temperature + # but here for reproducibility, we use 0.1 + temperature=0.1, + reasoning_effort=reasoning_effort, + ) + + setup = f"model={model} reasoning_effort={reasoning_effort} expected_score={expected_score}" + + print(f"Evaluation start: {setup}") + metrics = run_eval(args) + print(f"Evaluation end: {setup} {metrics=}") + self.assertGreaterEqual(metrics["score"], expected_score) + + if is_in_ci(): + write_github_step_summary( + f"### test_gpt_oss_common\n" + f"Setup: {setup}\n" + f"Score: {metrics['score']:.2f}\n" + ) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_gptqmodel_dynamic.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_gptqmodel_dynamic.py new file mode 100644 index 00000000..ea141df3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_gptqmodel_dynamic.py @@ -0,0 +1,204 @@ +import time +import unittest + +import requests +import torch + +from sglang.srt.server_args import set_global_server_args_for_scheduler +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +def check_quant_method(model_path: str, use_marlin_kernel: bool): + from sglang.srt.configs.device_config import DeviceConfig + from sglang.srt.configs.load_config import LoadConfig + from sglang.srt.configs.model_config import ModelConfig + from sglang.srt.distributed import ( + init_distributed_environment, + initialize_model_parallel, + ) + from sglang.srt.distributed.parallel_state import monkey_patch_vllm_parallel_state + from sglang.srt.layers.quantization.utils import get_dynamic_override + from sglang.srt.model_loader import get_model + from sglang.srt.server_args import ServerArgs + + try: + init_distributed_environment( + backend="nccl", + world_size=1, + rank=0, + local_rank=0, + distributed_init_method="tcp://127.0.0.1:2646", + ) + initialize_model_parallel(tensor_model_parallel_size=1) + monkey_patch_vllm_parallel_state() + except AssertionError: + # ignore this error: tensor model parallel group is already initialized + pass + + server_args = ServerArgs(model_path=model_path, dtype=torch.float16) + set_global_server_args_for_scheduler(server_args) + model_config = ModelConfig.from_server_args(server_args) + + load_config = LoadConfig() + device_config = DeviceConfig("cuda") + model = get_model( + model_config=model_config, load_config=load_config, device_config=device_config + ) + + from sglang.srt.layers.linear import UnquantizedLinearMethod + from sglang.srt.layers.quantization.gptq import ( + GPTQLinearMethod, + GPTQMarlinLinearMethod, + ) + + linear_method_cls = ( + GPTQMarlinLinearMethod if use_marlin_kernel else (GPTQLinearMethod) + ) + + for name, submodule in model.named_modules(): + if name == "lm_head": + assert isinstance(submodule.quant_method, linear_method_cls) + elif name == "model.layers.0.self_attn.qkv_proj": + # The first layer is quantized using bits=4, group_size=128 + # desc_act=True + assert isinstance(submodule.quant_method, linear_method_cls) + config = submodule.quant_method.quant_config + assert config.weight_bits == 4 + assert config.group_size == 128 + assert config.desc_act + elif name == "model.layers.1.self_attn.qkv_proj": + # The second layer is quantized using bits=8, group_size=32 + # desc_act=False + assert isinstance(submodule.quant_method, linear_method_cls) + config = submodule.quant_method.quant_config + assert get_dynamic_override(config, layer_name=name, key="bits") == 8 + assert get_dynamic_override(config, layer_name=name, key="group_size") == 32 + assert not get_dynamic_override(config, layer_name=name, key="desc_act") + elif ( + name == "model.layers.2.self_attn.qkv_proj" + or name == "model.layers.2.mlp.gate_up_proj" + ): + # All other layers (layer index >= 2) are not quantized + assert isinstance(submodule.quant_method, UnquantizedLinearMethod) + + del model + + +# GPTQ with Dynamic Per/Module Quantization Control +# Leverages GPTQModel (pypi) to produce the `dynamic` models +# Test GPTQ fallback kernel that is not Marlin +class TestGPTQModelDynamic(CustomTestCase): + MODEL_PATH = ( + "ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symFalse" + ) + + @classmethod + def setUpClass(cls): + cls.model = cls.MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--dtype", "float16"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode(self, max_new_tokens): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "max_new_tokens": max_new_tokens, + "temperature": 0.001, + }, + }, + ) + return response.json() + + def test_throughput(self): + max_tokens = 256 + + tic = time.perf_counter() + result = self.run_decode(max_tokens) + tok = time.perf_counter() + + print(f"result = `{result}`") + + self.assertIn("paris", result["text"].lower()) + + throughput = max_tokens / (tok - tic) + print(f"Throughput: {throughput} tokens/s") + self.assertGreaterEqual(throughput, 140) + + def test_gptq_module(self): + check_quant_method(self.MODEL_PATH, use_marlin_kernel=False) + + +# GPTQ with Dynamic Per/Module Quantization Control +# Leverages GPTQModel (pypi) to produce the `dynamic` models +# Test Marlin kernel +class TestGPTQModelDynamicWithMarlin(CustomTestCase): + MODEL_PATH = ( + "ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symTrue" + ) + + @classmethod + def setUpClass(cls): + cls.model = cls.MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--dtype", "bfloat16"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode(self, max_new_tokens): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "max_new_tokens": max_new_tokens, + "temperature": 0.001, + }, + }, + ) + return response.json() + + def test_throughput(self): + max_tokens = 256 + + tic = time.perf_counter() + result = self.run_decode(max_tokens) + tok = time.perf_counter() + + print(f"result = `{result}`") + + assert "paris" in result["text"].lower() + + throughput = max_tokens / (tok - tic) + print(f"Throughput: {throughput} tokens/s") + assert throughput >= 140 + + def test_gptq_marlin_module(self): + check_quant_method(self.MODEL_PATH, use_marlin_kernel=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_harmony_parser.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_harmony_parser.py new file mode 100644 index 00000000..20cc02e5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_harmony_parser.py @@ -0,0 +1,876 @@ +import unittest + +from sglang.srt.parser.harmony_parser import ( + CanonicalStrategy, + Event, + HarmonyParser, + TextStrategy, + Token, + iter_tokens, + prefix_hold, +) +from sglang.test.test_utils import CustomTestCase + + +class TestEvent(CustomTestCase): + def test_init(self): + """Test Event dataclass initialization.""" + event = Event("reasoning", "content") + self.assertEqual(event.event_type, "reasoning") + self.assertEqual(event.content, "content") + + +class TestToken(CustomTestCase): + def test_init(self): + """Test Token dataclass initialization.""" + token = Token("START", 0, 7) + self.assertEqual(token.type, "START") + self.assertEqual(token.start, 0) + self.assertEqual(token.end, 7) + + +class TestPrefixHold(CustomTestCase): + def test_empty_text(self): + """Test prefix_hold with empty text.""" + emit, hold = prefix_hold("", ["<|start|>"]) + self.assertEqual(emit, "") + self.assertEqual(hold, "") + + def test_no_matching_prefixes(self): + """Test prefix_hold with no matching prefixes.""" + emit, hold = prefix_hold("hello world", ["<|start|>", "<|end|>"]) + self.assertEqual(emit, "hello world") + self.assertEqual(hold, "") + + def test_partial_token_suffix(self): + """Test prefix_hold with partial token at end.""" + emit, hold = prefix_hold("hello <|ret", ["<|return|>"]) + self.assertEqual(emit, "hello ") + self.assertEqual(hold, "<|ret") + + def test_multiple_potential_matches(self): + """Test prefix_hold with multiple potential matches.""" + emit, hold = prefix_hold("text <|", ["<|start|>", "<|end|>"]) + self.assertEqual(emit, "text ") + self.assertEqual(hold, "<|") + + def test_exact_token_match(self): + """Test prefix_hold with exact token match.""" + emit, hold = prefix_hold("text <|start|>", ["<|start|>"]) + self.assertEqual(emit, "text <|start|>") + self.assertEqual(hold, "") + + +class TestIterTokens(CustomTestCase): + def test_empty_text(self): + """Test iter_tokens with empty text.""" + tokens = list(iter_tokens("")) + self.assertEqual(tokens, []) + + def test_plain_text(self): + """Test iter_tokens with plain text.""" + tokens = list(iter_tokens("hello world")) + self.assertEqual(len(tokens), 1) + self.assertEqual(tokens[0].type, "TEXT") + self.assertEqual(tokens[0].start, 0) + self.assertEqual(tokens[0].end, 11) + + def test_single_token(self): + """Test iter_tokens with single structural token.""" + tokens = list(iter_tokens("<|start|>")) + self.assertEqual(len(tokens), 1) + self.assertEqual(tokens[0].type, "START") + self.assertEqual(tokens[0].start, 0) + self.assertEqual(tokens[0].end, 9) + + def test_mixed_content(self): + """Test iter_tokens with mixed text and tokens.""" + tokens = list(iter_tokens("text<|start|>more text")) + self.assertEqual(len(tokens), 3) + + self.assertEqual(tokens[0].type, "TEXT") + self.assertEqual(tokens[0].start, 0) + self.assertEqual(tokens[0].end, 4) + + self.assertEqual(tokens[1].type, "START") + self.assertEqual(tokens[1].start, 4) + self.assertEqual(tokens[1].end, 13) + + self.assertEqual(tokens[2].type, "TEXT") + self.assertEqual(tokens[2].start, 13) + self.assertEqual(tokens[2].end, 22) + + def test_unknown_token_partial_suffix(self): + """Test iter_tokens with unknown token that could be partial.""" + tokens = list(iter_tokens("text <|ret")) + self.assertEqual(len(tokens), 2) + + self.assertEqual(tokens[0].type, "TEXT") + self.assertEqual(tokens[0].start, 0) + self.assertEqual(tokens[0].end, 5) + + self.assertEqual(tokens[1].type, "TEXT") + self.assertEqual(tokens[1].start, 5) + self.assertEqual(tokens[1].end, 10) + + def test_unknown_token_middle(self): + """Test iter_tokens with unknown token in middle.""" + tokens = list(iter_tokens("text <|weird|> more <|start|>")) + self.assertEqual(len(tokens), 5) + + self.assertEqual(tokens[0].type, "TEXT") + self.assertEqual(tokens[1].type, "TEXT") # "<|" + self.assertEqual(tokens[2].type, "TEXT") # "weird|> more " + self.assertEqual(tokens[3].type, "START") + # No trailing text token since it ends with a known token + + def test_all_structural_tokens(self): + """Test iter_tokens recognizes all structural tokens.""" + text = "<|start|><|channel|><|message|><|constrain|><|end|><|call|><|return|>" + tokens = list(iter_tokens(text)) + + expected_types = [ + "START", + "CHANNEL", + "MESSAGE", + "CONSTRAIN", + "END", + "CALL", + "RETURN", + ] + self.assertEqual(len(tokens), len(expected_types)) + + for token, expected_type in zip(tokens, expected_types): + self.assertEqual(token.type, expected_type) + + +class TestCanonicalStrategy(CustomTestCase): + def setUp(self): + self.strategy = CanonicalStrategy() + + def test_init(self): + """Test CanonicalStrategy initialization.""" + self.assertIn("<|start|>", self.strategy.guard_tokens) + self.assertIn("<|constrain|>", self.strategy.guard_tokens) + + def test_extract_channel_type(self): + """Test _extract_channel_type method.""" + self.assertEqual(self.strategy._extract_channel_type("analysis"), "analysis") + self.assertEqual( + self.strategy._extract_channel_type("commentary to=functions.tool"), + "commentary", + ) + self.assertEqual(self.strategy._extract_channel_type("final to=user"), "final") + self.assertEqual(self.strategy._extract_channel_type("ANALYSIS"), "analysis") + self.assertIsNone(self.strategy._extract_channel_type("unknown")) + + def test_parse_single_analysis_block(self): + """Test parsing single analysis block.""" + text = "<|channel|>analysis<|message|>Let me think about this<|end|>" + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual(events[0].content, "Let me think about this") + self.assertEqual(remaining, "") + + def test_parse_single_commentary_block(self): + """Test parsing single commentary block.""" + text = "<|channel|>commentary<|message|>User-visible message<|end|>" + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "normal") + self.assertEqual(events[0].content, "User-visible message") + self.assertEqual(remaining, "") + + def test_parse_single_final_block(self): + """Test parsing single final block.""" + text = "<|start|>assistant<|channel|>final<|message|>The answer is 42<|return|>" + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "normal") + self.assertEqual(events[0].content, "The answer is 42") + self.assertEqual(remaining, "") + + def test_parse_tool_call_commentary(self): + """Test parsing tool call on commentary channel.""" + text = '<|channel|>commentary to=functions.get_weather<|message|>{"location": "SF"}<|call|>' + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "tool_call") + self.assertEqual(events[0].content, '{"location": "SF"}') + self.assertEqual(remaining, "") + + def test_parse_tool_call_analysis(self): + """Test parsing built-in tool call on analysis channel.""" + text = '<|channel|>analysis to=browser.search<|message|>{"query": "SGLang"}<|call|>' + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "tool_call") + self.assertEqual(events[0].content, '{"query": "SGLang"}') + self.assertEqual(remaining, "") + + def test_parse_complex_sequence(self): + """Test parsing complex sequence with multiple blocks.""" + text = ( + "<|channel|>analysis<|message|>Need to use function get_weather.<|end|>" + "<|start|>assistant<|channel|>commentary to=functions.get_weather<|message|>" + '{"location":"San Francisco"}<|call|>' + ) + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 2) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual(events[0].content, "Need to use function get_weather.") + self.assertEqual(events[1].event_type, "tool_call") + self.assertEqual(events[1].content, '{"location":"San Francisco"}') + self.assertEqual(remaining, "") + + def test_parse_with_interspersed_text(self): + """Test parsing with plain text between blocks.""" + text = ( + "Some text " + "<|channel|>analysis<|message|>reasoning<|end|>" + " more text " + "<|start|>assistant<|channel|>final<|message|>answer<|return|>" + " trailing text" + ) + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 4) + self.assertEqual(events[0].event_type, "normal") + self.assertEqual(events[0].content, "Some text ") + self.assertEqual(events[1].event_type, "reasoning") + self.assertEqual(events[1].content, "reasoning") + self.assertEqual(events[2].event_type, "normal") + self.assertEqual(events[2].content, " more text ") + self.assertEqual(events[3].event_type, "normal") + self.assertEqual(events[3].content, "answer trailing text") + self.assertEqual(remaining, "") + + def test_parse_incomplete_block(self): + """Test parsing incomplete block (streaming scenario).""" + text = "<|channel|>analysis<|message|>partial content" + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual(events[0].content, "partial content") + self.assertEqual(remaining, "<|channel|>analysis<|message|>") + + def test_parse_partial_token_suffix(self): + """Test parsing with partial token at end.""" + text = "complete text <|ret" + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "normal") + self.assertEqual(events[0].content, "complete text ") + self.assertEqual(remaining, "<|ret") + + def test_parse_tool_response_message(self): + """Test parsing tool response message (no channel).""" + text = '<|start|>functions.get_weather to=assistant<|message|>{"sunny": true}<|end|>' + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "normal") + self.assertEqual(events[0].content, '{"sunny": true}') + self.assertEqual(remaining, "") + + def test_parse_empty_content_blocks(self): + """Test parsing blocks with empty content.""" + text = "<|channel|>analysis<|message|><|end|>" + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual(events[0].content, "") + self.assertEqual(remaining, "") + + def test_parse_commentary_filler_between_blocks(self): + """Test that 'commentary' filler between <|call|> and <|channel|> is filtered out.""" + # This pattern occurs when the model generates malformed output + text = ( + '<|channel|>commentary to=functions.get_weather<|message|>{"location":"SF"}<|call|>' + "commentary" # This should be filtered out + '<|channel|>commentary to=functions.get_temp<|message|>{"location":"NYC"}<|call|>' + ) + events, remaining = self.strategy.parse(text) + + # Should have 2 tool calls, no "commentary" normal text + self.assertEqual(len(events), 2) + self.assertEqual(events[0].event_type, "tool_call") + self.assertEqual(events[0].content, '{"location":"SF"}') + self.assertEqual(events[1].event_type, "tool_call") + self.assertEqual(events[1].content, '{"location":"NYC"}') + self.assertEqual(remaining, "") + + # Verify no "commentary" text was emitted as normal content + normal_events = [e for e in events if e.event_type == "normal"] + commentary_events = [ + e for e in normal_events if "commentary" in e.content.lower() + ] + self.assertEqual( + len(commentary_events), 0, "Commentary filler should be filtered out" + ) + + +class TestTextStrategy(CustomTestCase): + def setUp(self): + self.strategy = TextStrategy() + + def test_init(self): + """Test TextStrategy initialization.""" + self.assertIn("analysis_then_final", self.strategy.patterns) + + def test_parse_analysis_then_final(self): + """Test parsing analysis then final format.""" + text = "analysis I need to think about this. assistantfinal The answer is 42." + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 2) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual(events[0].content, "I need to think about this.") + self.assertEqual(events[1].event_type, "normal") + self.assertEqual(events[1].content, "The answer is 42.") + self.assertEqual(remaining, "") + + def test_parse_commentary_then_final(self): + """Test parsing commentary then final format.""" + text = "commentary User-visible preamble. assistantfinal The answer is 42." + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 2) + self.assertEqual(events[0].event_type, "normal") + self.assertEqual(events[0].content, "User-visible preamble.") + self.assertEqual(events[1].event_type, "normal") + self.assertEqual(events[1].content, "The answer is 42.") + self.assertEqual(remaining, "") + + def test_parse_final_only(self): + """Test parsing final-only format.""" + text = "assistantfinal The direct answer." + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "normal") + self.assertEqual(events[0].content, "The direct answer.") + self.assertEqual(remaining, "") + + def test_parse_analysis_only(self): + """Test parsing analysis-only format.""" + text = "analysis This is reasoning content." + events, remaining = self.strategy.parse(text) + + # For analysis-only, streaming parse should keep header and emit with leading space + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual(events[0].content, " This is reasoning content.") + self.assertEqual(remaining, "analysis") + + def test_parse_incomplete_assistantfinal(self): + """Test parsing with incomplete assistantfinal.""" + text = "analysis reasoning content assistantfin" + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 0) + self.assertEqual(remaining, text) # Hold entire buffer + + def test_parse_partial_analysis_streaming(self): + """Test streaming partial analysis content.""" + text = "analysis partial content" + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual(events[0].content, " partial content") # Space preserved + self.assertEqual(remaining, "analysis") # Hold header + + def test_parse_case_insensitive(self): + """Test case insensitive parsing.""" + text = "ANALYSIS reasoning ASSISTANTFINAL answer" + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 2) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual(events[1].event_type, "normal") + + def test_parse_plain_text_fallback(self): + """Test parsing plain text without harmony markers.""" + text = "Just plain text without any markers." + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "normal") + self.assertEqual(events[0].content, "Just plain text without any markers.") + self.assertEqual(remaining, "") + + def test_parse_analysis_no_space_after_header(self): + """Test parsing analysis format without space after header (real gpt-oss output).""" + text = "analysisThe user typed random strings. We should respond politely.assistantfinalIt looks like you're testing. How can I help?" + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 2) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual( + events[0].content, + "The user typed random strings. We should respond politely.", + ) + self.assertEqual(events[1].event_type, "normal") + self.assertEqual( + events[1].content, "It looks like you're testing. How can I help?" + ) + + +class TestHarmonyParser(CustomTestCase): + def setUp(self): + self.parser = HarmonyParser() + + def test_init(self): + """Test HarmonyParser initialization.""" + self.assertIsNone(self.parser.strategy) + self.assertEqual(self.parser._buffer, "") + + def test_strategy_selection_canonical(self): + """Test automatic strategy selection for canonical format.""" + events = self.parser.parse("<|channel|>analysis<|message|>test<|end|>") + + self.assertIsInstance(self.parser.strategy, CanonicalStrategy) + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "reasoning") + + def test_strategy_selection_text(self): + """Test automatic strategy selection for text format.""" + events = self.parser.parse("analysis test content") + + self.assertIsInstance(self.parser.strategy, TextStrategy) + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "reasoning") + + def test_strategy_selection_delayed(self): + """Test strategy selection with insufficient initial content.""" + # First chunk doesn't have enough info + events1 = self.parser.parse("some") + self.assertEqual(len(events1), 0) + self.assertIsNone(self.parser.strategy) + + # Second chunk triggers strategy selection + events2 = self.parser.parse(" analysis content") + self.assertIsInstance(self.parser.strategy, TextStrategy) + self.assertEqual(len(events2), 1) + + def test_streaming_canonical_format(self): + """Test streaming with canonical format.""" + chunks = [ + "<|channel|>analysis<|message|>", + "reasoning content", + "<|end|>", + "<|start|>assistant<|channel|>final<|message|>", + "final answer", + "<|return|>", + ] + + all_events = [] + for chunk in chunks: + events = self.parser.parse(chunk) + all_events.extend(events) + + self.assertEqual(len(all_events), 5) + + # Verify we get reasoning events + reasoning_events = [e for e in all_events if e.event_type == "reasoning"] + self.assertTrue(len(reasoning_events) > 0) + + # Verify we get normal events + normal_events = [e for e in all_events if e.event_type == "normal"] + self.assertTrue(len(normal_events) > 0) + + # Verify content is eventually parsed correctly + combined_reasoning = "".join(e.content for e in reasoning_events) + combined_normal = "".join( + e.content + for e in normal_events + if e.content and "<|return|>" not in e.content + ) + + self.assertIn("reasoning content", combined_reasoning) + self.assertIn("final answer", combined_normal) + + def test_streaming_text_format(self): + """Test streaming with text format.""" + chunks = ["analysis reasoning", " content assistantfinal", " the answer"] + + all_events = [] + for chunk in chunks: + events = self.parser.parse(chunk) + all_events.extend(events) + + # Should have reasoning and normal events + reasoning_events = [e for e in all_events if e.event_type == "reasoning"] + normal_events = [e for e in all_events if e.event_type == "normal"] + + self.assertGreater(len(reasoning_events), 0) + self.assertGreater(len(normal_events), 0) + + def test_streaming_commentary_filler(self): + """Test that 'commentary' filler is filtered in streaming case.""" + # Test when commentary arrives as a separate chunk after <|call|> + chunks = [ + "<|channel|>commentary to=functions.get_weather", + "<|message|>", + '{"location":"SF"}', + "<|call|>", + "comment", # This arrives as separate chunk - should be filtered + "ary", # Continuation of the filler - should be filtered + "<|channel|>commentary to=functions.get_temp", + "<|message|>", + '{"location":"NYC"}', + "<|call|>", + "comment", # Another separate chunk - should be filtered + "ary", # Continuation of the filler - should be filtered + "<|start|>assistant<|channel|>final", + "<|message|>Done<|return|>", + ] + + all_events = [] + for chunk in chunks: + events = self.parser.parse(chunk) + all_events.extend(events) + + # Count event types + tool_events = [e for e in all_events if e.event_type == "tool_call"] + normal_events = [e for e in all_events if e.event_type == "normal"] + + # Should have 2 tool calls and 1 final message + self.assertEqual(len(tool_events), 2, "Should have 2 tool calls") + self.assertEqual( + len(normal_events), 1, "Should have 1 normal event (final message)" + ) + + # Verify no "commentary" in normal events + for event in normal_events: + self.assertNotEqual( + event.content.strip().lower(), + "commentary", + "Commentary filler should not appear as normal content in streaming", + ) + + # Verify content + self.assertEqual(tool_events[0].content, '{"location":"SF"}') + self.assertEqual(tool_events[1].content, '{"location":"NYC"}') + self.assertEqual(normal_events[0].content, "Done") + + def test_repetitive_tool_calls_with_commentary_filler(self): + """Test handling of repetitive tool calls with 'commentary' filler text.""" + # This simulates malformed output with repeated tool calls and commentary filler + text = ( + "<|channel|>analysis<|message|>Need to get weather<|end|>" + '<|start|>assistant<|channel|>commentary to=functions.get_weather<|message|>{"city":"Boston"}<|call|>' + "commentary" # Filler that should be filtered + '<|channel|>commentary to=functions.get_weather<|message|>{"city":"Boston"}<|call|>' + "commentary" # Another filler + '<|channel|>commentary to=functions.get_weather<|message|>{"city":"Boston"}<|call|>' + "<|channel|>analysis<|message|>Tool not responding<|end|>" + "<|start|>assistant<|channel|>final<|message|>Unable to fetch weather data<|return|>" + ) + + events = self.parser.parse(text) + + # Count event types + reasoning_events = [e for e in events if e.event_type == "reasoning"] + tool_events = [e for e in events if e.event_type == "tool_call"] + normal_events = [e for e in events if e.event_type == "normal"] + + # Verify correct number of each type + self.assertEqual(len(reasoning_events), 2, "Should have 2 reasoning events") + self.assertEqual(len(tool_events), 3, "Should have 3 tool calls") + self.assertEqual( + len(normal_events), 1, "Should have 1 normal event (final message)" + ) + + # Verify no "commentary" filler in normal events + for event in normal_events: + self.assertNotEqual( + event.content.strip().lower(), + "commentary", + "Commentary filler should not appear as normal content", + ) + + # Verify content is correct + self.assertEqual(reasoning_events[0].content, "Need to get weather") + self.assertEqual(reasoning_events[1].content, "Tool not responding") + self.assertEqual(normal_events[0].content, "Unable to fetch weather data") + + +class TestIntegrationScenarios(CustomTestCase): + """Integration tests for realistic Harmony parsing scenarios.""" + + def test_complete_reasoning_flow(self): + """Test complete reasoning flow from HARMONY_DOCS.md examples.""" + parser = HarmonyParser() + + text = ( + '<|channel|>analysis<|message|>User asks: "What is 2 + 2?" Simple arithmetic. Provide answer.<|end|>' + "<|start|>assistant<|channel|>final<|message|>2 + 2 = 4.<|return|>" + ) + + events = parser.parse(text) + + self.assertEqual(len(events), 2) + self.assertEqual(events[0].event_type, "reasoning") + self.assertIn("Simple arithmetic", events[0].content) + self.assertEqual(events[1].event_type, "normal") + self.assertEqual(events[1].content, "2 + 2 = 4.") + + def test_tool_call_sequence(self): + """Test tool call sequence from HARMONY_DOCS.md examples.""" + parser = HarmonyParser() + + text = ( + "<|channel|>analysis<|message|>Need to use function get_weather.<|end|>" + "<|start|>assistant<|channel|>commentary to=functions.get_weather <|constrain|>json<|message|>" + '{"location":"San Francisco"}<|call|>' + ) + + events = parser.parse(text) + + self.assertEqual(len(events), 2) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual(events[0].content, "Need to use function get_weather.") + self.assertEqual(events[1].event_type, "tool_call") + self.assertEqual(events[1].content, '{"location":"San Francisco"}') + + def test_preamble_sequence(self): + """Test preamble sequence with multiple commentary blocks.""" + parser = HarmonyParser() + + text = ( + "<|channel|>analysis<|message|>Long chain of thought<|end|>" + "<|start|>assistant<|channel|>commentary<|message|>**Action plan**: 1. Generate file 2. Start server<|end|>" + "<|start|>assistant<|channel|>commentary to=functions.generate_file<|message|>" + '{"template": "basic_html"}<|call|>' + ) + + events = parser.parse(text) + + self.assertEqual(len(events), 3) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual(events[1].event_type, "normal") + self.assertIn("Action plan", events[1].content) + self.assertEqual(events[2].event_type, "tool_call") + + def test_built_in_tool_call(self): + """Test built-in tool call on analysis channel.""" + parser = HarmonyParser() + + text = '<|channel|>analysis to=browser.search<|message|>{"query": "SGLang"}<|call|>' + + events = parser.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "tool_call") + self.assertEqual(events[0].content, '{"query": "SGLang"}') + + def test_tool_response_handling(self): + """Test tool response message handling.""" + parser = HarmonyParser() + + text = '<|start|>functions.get_weather to=assistant<|channel|>commentary<|message|>{"sunny": true, "temperature": 20}<|end|>' + + events = parser.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "normal") + self.assertEqual(events[0].content, '{"sunny": true, "temperature": 20}') + + def test_text_fallback_formats(self): + """Test various text fallback formats.""" + parser = HarmonyParser() + + # Test analysis then final + events1 = parser.parse("analysis thinking assistantfinal answer") + self.assertEqual(len([e for e in events1 if e.event_type == "reasoning"]), 1) + self.assertEqual(len([e for e in events1 if e.event_type == "normal"]), 1) + + # Reset parser for next test + parser = HarmonyParser() + + # Test final only + events2 = parser.parse("assistantfinal direct answer") + self.assertEqual(len(events2), 1) + self.assertEqual(events2[0].event_type, "normal") + + def test_streaming_property_canonical(self): + """Test streaming property: chunked parsing produces same semantic content as one-shot parsing.""" + full_text = ( + "<|channel|>analysis<|message|>reasoning content<|end|>" + "<|start|>assistant<|channel|>final<|message|>final content" + ) + + # One-shot parsing + parser1 = HarmonyParser() + events_oneshot = parser1.parse(full_text) + events_oneshot += parser1.parse("") + + # Chunked parsing + parser2 = HarmonyParser() + chunks = [ + "<|channel|>", + "analysis", + "<|message|>", + "reasoning content", + "<|end|>", + "<|start|>assistant", + "<|channel|>final", + "<|message|>", + "final ", + "content", + ] + events_chunked = [] + for chunk in chunks: + events_chunked.extend(parser2.parse(chunk)) + + # Compare semantic content rather than exact event structure + reasoning_oneshot = "".join( + e.content for e in events_oneshot if e.event_type == "reasoning" + ) + normal_oneshot = "".join( + e.content for e in events_oneshot if e.event_type == "normal" + ) + + reasoning_chunked = "".join( + e.content for e in events_chunked if e.event_type == "reasoning" + ) + normal_chunked = "".join( + e.content for e in events_chunked if e.event_type == "normal" + ) + + self.assertEqual(reasoning_chunked, reasoning_oneshot) + self.assertEqual(normal_chunked, normal_oneshot) + + def test_streaming_property_text(self): + """Test streaming property for text format.""" + full_text = "analysis reasoning content assistantfinal final answer" + + # One-shot parsing + parser1 = HarmonyParser() + events_oneshot = parser1.parse(full_text) + + # Chunked parsing + parser2 = HarmonyParser() + chunks = ["analysis reason", "ing content assistant", "final final answer"] + events_chunked = [] + for chunk in chunks: + events_chunked.extend(parser2.parse(chunk)) + + # Combine content by type for comparison + reasoning_oneshot = "".join( + e.content for e in events_oneshot if e.event_type == "reasoning" + ) + normal_oneshot = "".join( + e.content for e in events_oneshot if e.event_type == "normal" + ) + + reasoning_chunked = "".join( + e.content for e in events_chunked if e.event_type == "reasoning" + ) + normal_chunked = "".join( + e.content for e in events_chunked if e.event_type == "normal" + ) + + # Account for whitespace differences due to streaming - compare trimmed content + self.assertEqual(reasoning_oneshot.strip(), reasoning_chunked.strip()) + self.assertEqual(normal_oneshot.strip(), normal_chunked.strip()) + + +class TestEdgeCases(CustomTestCase): + """Test edge cases and error conditions.""" + + def test_malformed_channel_headers(self): + """Test handling of malformed channel headers.""" + parser = HarmonyParser() + + # Unknown channel type + text = "<|channel|>unknown<|message|>content<|end|>" + events = parser.parse(text) + + # Should be held as incomplete since channel is unknown + self.assertEqual(len(events), 0) + + def test_mixed_unknown_tokens(self): + """Test handling of mixed unknown tokens.""" + parser = HarmonyParser() + + text = "text <|weird|> more text <|channel|>analysis<|message|>content<|end|>" + events = parser.parse(text) + + # Should parse the valid parts + reasoning_events = [e for e in events if e.event_type == "reasoning"] + normal_events = [e for e in events if e.event_type == "normal"] + + self.assertEqual(len(reasoning_events), 1) + self.assertGreater(len(normal_events), 0) + + def test_empty_input(self): + """Test handling of empty input.""" + parser = HarmonyParser() + events = parser.parse("") + self.assertEqual(len(events), 0) + + def test_whitespace_preservation(self): + """Test that whitespace is preserved correctly.""" + parser = HarmonyParser() + + text = "<|channel|>analysis<|message|> content with spaces <|end|>" + events = parser.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].content, " content with spaces ") + + def test_streaming_whitespace_preservation(self): + """Test that streaming preserves whitespace between chunks.""" + parser = HarmonyParser() + + # Simulate streaming where space is at chunk boundary + chunks = ["analysis The user typed ", '"wapppa". Not a question.'] + + all_events = [] + for chunk in chunks: + events = parser.parse(chunk) + all_events.extend(events) + + # Combine all reasoning content + reasoning_content = "".join( + e.content for e in all_events if e.event_type == "reasoning" + ) + + # Should preserve the space before the quote + self.assertIn('typed "wapppa"', reasoning_content) + self.assertNotIn( + 'typed"wapppa"', reasoning_content + ) # Should not be mashed together + + def test_consecutive_blocks_same_type(self): + """Test consecutive blocks of the same type.""" + parser = HarmonyParser() + + text = ( + "<|channel|>analysis<|message|>first reasoning<|end|>" + "<|channel|>analysis<|message|>second reasoning<|end|>" + ) + events = parser.parse(text) + + self.assertEqual(len(events), 2) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual(events[1].event_type, "reasoning") + self.assertEqual(events[0].content, "first reasoning") + self.assertEqual(events[1].content, "second reasoning") + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_hidden_states.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_hidden_states.py new file mode 100644 index 00000000..2046ce52 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_hidden_states.py @@ -0,0 +1,139 @@ +import unittest + +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + +import sglang as sgl +from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST, CustomTestCase + + +class TestHiddenState(CustomTestCase): + def test_return_hidden_states(self): + prompts = ["Today is", "Today is a sunny day and I like"] + model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + tokenizer = AutoTokenizer.from_pretrained(model_path) + input_ids = tokenizer(prompts).input_ids + + sampling_params = { + "temperature": 0, + "max_new_tokens": 8, + } + + engine = sgl.Engine( + model_path=model_path, + random_seed=42, + skip_tokenizer_init=True, + enable_return_hidden_states=True, + ) + outputs = engine.generate( + input_ids=input_ids, + sampling_params=sampling_params, + return_hidden_states=True, + ) + engine.shutdown() + + for output in outputs: + self.assertEqual(len(output["meta_info"]["hidden_states"]), 8) + for i in range(len(output["meta_info"]["hidden_states"])): + assert isinstance(output["meta_info"]["hidden_states"][i], list) + output["meta_info"]["hidden_states"][i] = torch.tensor( + output["meta_info"]["hidden_states"][i], dtype=torch.bfloat16 + ) + # Checks that splicing of the batch was done correctly + self.assertGreater( + outputs[1]["meta_info"]["hidden_states"][0].shape[0], + outputs[0]["meta_info"]["hidden_states"][0].shape[0], + ) + + model = AutoModelForCausalLM.from_pretrained( + model_path, torch_dtype=torch.bfloat16, device_map="cuda" + ) + + for input_id, output in zip(input_ids, outputs): + with torch.inference_mode(): + hf_out = model( + torch.tensor( + [input_id + output["output_ids"][:-1]], device=model.device + ), + output_hidden_states=True, + ) + print("=== HF Hiddens ===") + print(hf_out["hidden_states"][-1][0]) + sg_hidden_states = torch.cat( + [ + i.unsqueeze(0) if len(i.shape) == 1 else i + for i in output["meta_info"]["hidden_states"] + ] + ).to("cuda") + print("=== SRT Hiddens ===") + print(sg_hidden_states) + + print( + f"Max diff: {torch.max(torch.abs(hf_out['hidden_states'][-1][0] - sg_hidden_states))}" + ) + + atol = 0.8 + self.assertTrue( + torch.allclose( + hf_out["hidden_states"][-1][0], + sg_hidden_states, + atol=atol, + rtol=0, + ) + ) + + def test_repeatedly_changes_hidden_states(self): + prompts = ["Today is", "Today is a sunny day and I like"] + model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + tokenizer = AutoTokenizer.from_pretrained(model_path) + input_ids = tokenizer(prompts).input_ids + + sampling_params = { + "temperature": 0, + "max_new_tokens": 8, + } + + engine = sgl.Engine( + model_path=model_path, + random_seed=42, + skip_tokenizer_init=True, + enable_return_hidden_states=True, + ) + outputs_completion_first_round = engine.generate( + input_ids=input_ids, + sampling_params=sampling_params, + return_hidden_states=True, + ) + outputs_hidden_state = engine.generate( + input_ids=input_ids, + sampling_params=sampling_params, + return_hidden_states=False, + ) + + outputs_completion_last_round = engine.generate( + input_ids=input_ids, + sampling_params=sampling_params, + return_hidden_states=True, + ) + engine.shutdown() + + for ( + output_completion_first_round, + output_hidden_state, + output_completion_last_round, + ) in zip( + outputs_completion_first_round, + outputs_hidden_state, + outputs_completion_last_round, + ): + self.assertEqual( + len(output_completion_first_round["meta_info"]["hidden_states"]), 8 + ) + self.assertNotIn("hidden_states", output_hidden_state["meta_info"]) + self.assertEqual( + len(output_completion_last_round["meta_info"]["hidden_states"]), 8 + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_hybrid_attn_backend.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_hybrid_attn_backend.py new file mode 100644 index 00000000..a16203ae --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_hybrid_attn_backend.py @@ -0,0 +1,163 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.environ import envs +from sglang.srt.utils import get_device_sm, kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST_MLA, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +GSM_DATASET_PATH = None + +# Default server arguments shared across all tests +DEFAULT_SERVER_ARGS = [ + "--trust-remote-code", + "--cuda-graph-max-bs", + "8", + "--prefill-attention-backend", + "fa3", + "--decode-attention-backend", + "flashinfer", +] + + +@unittest.skipIf(get_device_sm() < 90, "Test requires CUDA SM 90 or higher") +class TestHybridAttnBackendBase(CustomTestCase): + + model = DEFAULT_MODEL_NAME_FOR_TEST + base_url = DEFAULT_URL_FOR_TEST + accuracy_threshold = 0.65 # derived tests need to override this + speculative_decode = False + spec_decode_threshold = 2.2 # derived spec decoding tests need to override this + + @classmethod + def get_server_args(cls): + """Return the arguments for the server launch. Override in subclasses.""" + return DEFAULT_SERVER_ARGS + + @classmethod + def setUpClass(cls): + # disable deep gemm precompile to make launch server faster + # please don't do this if you want to make your inference workload faster + with ( + envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.override(False), + envs.SGLANG_ENABLE_JIT_DEEPGEMM.override(False), + ): + if cls.speculative_decode: + model = DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST + else: + model = cls.model + cls.process = popen_launch_server( + model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=cls.get_server_args(), + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=4, + num_questions=100, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + data_path=GSM_DATASET_PATH, + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + # Use the appropriate metric key based on the test class + metric_key = "accuracy" + self.assertGreater(metrics[metric_key], self.accuracy_threshold) + + if self.speculative_decode: + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, self.spec_decode_threshold) + + +class TestHybridAttnBackendMLA(TestHybridAttnBackendBase): + accuracy_threshold = 0.60 + model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + + @classmethod + def get_server_args(cls): + return DEFAULT_SERVER_ARGS + + +class TestHybridAttnBackendTorchCompile(TestHybridAttnBackendBase): + accuracy_threshold = 0.65 + + @classmethod + def get_server_args(cls): + return DEFAULT_SERVER_ARGS + ["--enable-torch-compile"] + + +class TestHybridAttnBackendSpeculativeDecodingPrefillBackend(TestHybridAttnBackendBase): + speculative_decode = True + # This eagle test uses a very small model, so the accuracy is low. + accuracy_threshold = 0.2 + + @classmethod + def get_server_args(cls): + return DEFAULT_SERVER_ARGS + [ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "2", + "--speculative-num-draft-tokens", + "4", + "--speculative-attention-mode", + "prefill", + ] + + +class TestHybridAttnBackendSpeculativeDecodingDecodeBackend(TestHybridAttnBackendBase): + speculative_decode = True + # This eagle test uses a very small model, so the accuracy is low. + accuracy_threshold = 0.2 + + @classmethod + def get_server_args(cls): + return DEFAULT_SERVER_ARGS + [ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "2", + "--speculative-num-draft-tokens", + "4", + "--speculative-attention-mode", + "decode", + ] + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_input_embeddings.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_input_embeddings.py new file mode 100644 index 00000000..7efd6656 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_input_embeddings.py @@ -0,0 +1,151 @@ +import json +import os +import tempfile +import unittest + +import requests +from transformers import AutoModelForCausalLM, AutoTokenizer + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestInputEmbeds(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.tokenizer = AutoTokenizer.from_pretrained(cls.model) + cls.ref_model = AutoModelForCausalLM.from_pretrained(cls.model) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--disable-radix", "--cuda-graph-max-bs", 4], + ) + cls.texts = [ + "The capital of France is", + "What is the best time of year to visit Japan for cherry blossoms?", + ] + + def generate_input_embeddings(self, text): + """Generate input embeddings for a given text.""" + input_ids = self.tokenizer(text, return_tensors="pt")["input_ids"] + embeddings = self.ref_model.get_input_embeddings()(input_ids) + return embeddings.squeeze().tolist() # Convert tensor to a list for API use + + def send_request(self, payload): + """Send a POST request to the /generate endpoint and return the response.""" + response = requests.post( + self.base_url + "/generate", + json=payload, + timeout=30, # Set a reasonable timeout for the API request + ) + if response.status_code == 200: + return response.json() + return { + "error": f"Request failed with status {response.status_code}: {response.text}" + } + + def send_file_request(self, file_path): + """Send a POST request to the /generate_from_file endpoint with a file.""" + with open(file_path, "rb") as f: + response = requests.post( + self.base_url + "/generate_from_file", + files={"file": f}, + timeout=30, # Set a reasonable timeout for the API request + ) + if response.status_code == 200: + return response.json() + return { + "error": f"Request failed with status {response.status_code}: {response.text}" + } + + def test_text_based_response(self): + """Test and print API responses using text-based input.""" + for text in self.texts: + payload = { + "model": self.model, + "text": text, + "sampling_params": {"temperature": 0, "max_new_tokens": 50}, + } + response = self.send_request(payload) + print( + f"Text Input: {text}\nResponse: {json.dumps(response, indent=2)}\n{'-' * 80}" + ) + + def test_embedding_based_response(self): + """Test and print API responses using input embeddings.""" + for text in self.texts: + embeddings = self.generate_input_embeddings(text) + payload = { + "model": self.model, + "input_embeds": embeddings, + "sampling_params": {"temperature": 0, "max_new_tokens": 50}, + } + response = self.send_request(payload) + print( + f"Embeddings Input (for text '{text}'):\nResponse: {json.dumps(response, indent=2)}\n{'-' * 80}" + ) + + def test_compare_text_vs_embedding(self): + """Test and compare responses for text-based and embedding-based inputs.""" + for text in self.texts: + # Text-based payload + text_payload = { + "model": self.model, + "text": text, + "sampling_params": {"temperature": 0, "max_new_tokens": 50}, + } + # Embedding-based payload + embeddings = self.generate_input_embeddings(text) + embed_payload = { + "model": self.model, + "input_embeds": embeddings, + "sampling_params": {"temperature": 0, "max_new_tokens": 50}, + } + # Get responses + text_response = self.send_request(text_payload) + embed_response = self.send_request(embed_payload) + # Print responses + print( + f"Text Input: {text}\nText-Based Response: {json.dumps(text_response, indent=2)}\n" + ) + print( + f"Embeddings Input (for text '{text}'):\nEmbedding-Based Response: {json.dumps(embed_response, indent=2)}\n{'-' * 80}" + ) + # This is flaky, so we skip this temporarily + # self.assertEqual(text_response["text"], embed_response["text"]) + + def test_generate_from_file(self): + """Test the /generate_from_file endpoint using tokenized embeddings.""" + for text in self.texts: + embeddings = self.generate_input_embeddings(text) + with tempfile.NamedTemporaryFile( + mode="w", suffix=".json", delete=False + ) as tmp_file: + json.dump(embeddings, tmp_file) + tmp_file_path = tmp_file.name + + try: + response = self.send_file_request(tmp_file_path) + print( + f"Text Input: {text}\nResponse from /generate_from_file: {json.dumps(response, indent=2)}\n{'-' * 80}" + ) + finally: + # Ensure the temporary file is deleted + os.remove(tmp_file_path) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_io_struct.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_io_struct.py new file mode 100644 index 00000000..a8efebb2 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_io_struct.py @@ -0,0 +1,577 @@ +import copy +import unittest + +from sglang.srt.managers.io_struct import GenerateReqInput +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_URL_FOR_TEST, + CustomTestCase, +) + + +class TestGenerateReqInputNormalization(CustomTestCase): + """Test the normalization of GenerateReqInput for batch processing and different input formats.""" + + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + + def setUp(self): + # Common setup for all tests + self.base_req = GenerateReqInput( + text=["Hello", "World"], + sampling_params=[{}, {}], + rid=["id1", "id2"], + ) + + def test_single_image_to_list_of_lists(self): + """Test that a single image is converted to a list of single-image lists.""" + req = copy.deepcopy(self.base_req) + req.image_data = "single_image.jpg" # A single image (non-list) + + req.normalize_batch_and_arguments() + + # Should be converted to [[image], [image]] + self.assertEqual(len(req.image_data), 2) + self.assertEqual(len(req.image_data[0]), 1) + self.assertEqual(len(req.image_data[1]), 1) + self.assertEqual(req.image_data[0][0], "single_image.jpg") + self.assertEqual(req.image_data[1][0], "single_image.jpg") + + # Check modalities + self.assertEqual(req.modalities, ["image", "image"]) + + def test_list_of_images_to_list_of_lists(self): + """Test that a list of images is converted to a list of single-image lists.""" + req = copy.deepcopy(self.base_req) + req.image_data = ["image1.jpg", "image2.jpg"] # List of images + + req.normalize_batch_and_arguments() + + # Should be converted to [[image1], [image2]] + self.assertEqual(len(req.image_data), 2) + self.assertEqual(len(req.image_data[0]), 1) + self.assertEqual(len(req.image_data[1]), 1) + self.assertEqual(req.image_data[0][0], "image1.jpg") + self.assertEqual(req.image_data[1][0], "image2.jpg") + + # Check modalities + self.assertEqual(req.modalities, ["image", "image"]) + + def test_list_of_lists_with_different_modalities(self): + """Test handling of list of lists of images with different modalities.""" + req = copy.deepcopy(self.base_req) + req.image_data = [ + ["image1.jpg"], # Single image (image modality) + ["image2.jpg", "image3.jpg"], # Multiple images (multi-images modality) + ] + + req.normalize_batch_and_arguments() + + # Structure should remain the same + self.assertEqual(len(req.image_data), 2) + self.assertEqual(len(req.image_data[0]), 1) + self.assertEqual(len(req.image_data[1]), 2) + + # Check modalities + self.assertEqual(req.modalities, ["image", "multi-images"]) + + def test_list_of_lists_with_none_values(self): + """Test handling of list of lists with None values.""" + req = copy.deepcopy(self.base_req) + req.image_data = [ + [None], # None value + ["image.jpg"], # Single image + ] + + req.normalize_batch_and_arguments() + + # Structure should remain the same + self.assertEqual(len(req.image_data), 2) + self.assertEqual(len(req.image_data[0]), 1) + self.assertEqual(len(req.image_data[1]), 1) + + # Check modalities + self.assertEqual(req.modalities, [None, "image"]) + + def test_expanding_parallel_sample_correlation(self): + """Test that when expanding with parallel samples, prompts, images and modalities are properly correlated.""" + req = copy.deepcopy(self.base_req) + req.text = ["Prompt 1", "Prompt 2"] + req.image_data = [ + ["image1.jpg"], + ["image2.jpg", "image3.jpg"], + ] + req.sampling_params = {"n": 3} # All prompts get 3 samples + + # Define expected values before normalization + expected_text = req.text * 3 + expected_images = req.image_data * 3 + expected_modalities = ["image", "multi-images"] * 3 + + req.normalize_batch_and_arguments() + + # Should be expanded to 6 items (2 original * 3 parallel) + self.assertEqual(len(req.image_data), 6) + + # Check that images are properly expanded + self.assertEqual(req.image_data, expected_images) + + # Check modalities + self.assertEqual(req.modalities, expected_modalities) + + # Ensure that text items are properly duplicated too + self.assertEqual(req.text, expected_text) + + def test_specific_parallel_n_per_sample(self): + """Test parallel expansion when different samples have different n values.""" + req = copy.deepcopy(self.base_req) + req.text = ["Prompt 1", "Prompt 2"] + req.image_data = [ + ["image1.jpg"], + ["image2.jpg", "image3.jpg"], + ] + req.sampling_params = [ + {"n": 2}, + {"n": 2}, + ] # First prompt gets 2 samples, second prompt gets 2 samples + + expected_images = req.image_data * 2 + expected_modalities = ["image", "multi-images"] * 2 + expected_text = req.text * 2 + + req.normalize_batch_and_arguments() + + # Should be expanded to 4 items (2 original * 2 parallel) + self.assertEqual(len(req.image_data), 4) + + # Check that the first 2 are copies for the first prompt + self.assertEqual(req.image_data, expected_images) + + # Check modalities + self.assertEqual(req.modalities, expected_modalities) + + # Check text expansion + self.assertEqual(req.text, expected_text) + + def test_mixed_none_and_images_with_parallel_samples(self): + """Test that when some batch items have images and others None, parallel expansion works correctly.""" + req = copy.deepcopy(self.base_req) + req.text = ["Prompt 1", "Prompt 2", "Prompt 3"] + req.rid = ["id1", "id2", "id3"] + req.image_data = [ + ["image1.jpg"], + None, + ["image3_1.jpg", "image3_2.jpg"], + ] + req.sampling_params = {"n": 2} # All prompts get 2 samples + + expected_images = req.image_data * 2 + expected_modalities = ["image", None, "multi-images"] * 2 + expected_text = req.text * 2 + + req.normalize_batch_and_arguments() + + # Should be expanded to 6 items (3 original * 2 parallel) + self.assertEqual(len(req.image_data), 6) + + # Check image data + self.assertEqual(req.image_data, expected_images) + + # Check modalities + self.assertEqual(req.modalities, expected_modalities) + + # Check text expansion + self.assertEqual(req.text, expected_text) + + def test_correlation_with_sampling_params(self): + """Test that sampling parameters are correctly correlated with prompts during expansion.""" + req = copy.deepcopy(self.base_req) + req.text = ["Prompt 1", "Prompt 2"] + req.image_data = [ + ["image1.jpg"], + ["image2.jpg"], + ] + req.sampling_params = [ + {"temperature": 0.7, "n": 2}, + {"temperature": 0.9, "n": 2}, + ] + + req.normalize_batch_and_arguments() + + # Check sampling params expansion + self.assertEqual(len(req.sampling_params), 4) + self.assertEqual(req.sampling_params[0]["temperature"], 0.7) + self.assertEqual(req.sampling_params[1]["temperature"], 0.9) + self.assertEqual(req.sampling_params[2]["temperature"], 0.7) + self.assertEqual(req.sampling_params[3]["temperature"], 0.9) + + # Should be expanded to 4 items (2 original * 2 parallel) + self.assertEqual(len(req.image_data), 4) + + # Check correlation with images + self.assertEqual(req.image_data[0], ["image1.jpg"]) + self.assertEqual(req.image_data[1], ["image2.jpg"]) + self.assertEqual(req.image_data[2], ["image1.jpg"]) + self.assertEqual(req.image_data[3], ["image2.jpg"]) + + def test_single_example_with_image(self): + """Test handling of single example with image.""" + req = GenerateReqInput( + text="Hello", + image_data="single_image.jpg", + ) + + req.normalize_batch_and_arguments() + + # For single examples, image_data doesn't get processed into lists + self.assertEqual(req.image_data, "single_image.jpg") + self.assertIsNone(req.modalities) # Modalities isn't set for single examples + + def test_single_to_batch_with_parallel_sampling(self): + """Test single example converted to batch with parallel sampling.""" + req = GenerateReqInput( + text="Hello", + image_data="single_image.jpg", + sampling_params={"n": 3}, # parallel_sample_num = 3 + ) + + # Define expected values before normalization + expected_text = ["Hello"] * 3 + + req.normalize_batch_and_arguments() + + # Should be converted to batch with text=["Hello"] + self.assertEqual(req.text, expected_text) + + # Image should be automatically wrapped to list of lists with length 1*3=3 + self.assertEqual(len(req.image_data), 3) + self.assertEqual(req.image_data[0][0], "single_image.jpg") + self.assertEqual(req.image_data[1][0], "single_image.jpg") + self.assertEqual(req.image_data[2][0], "single_image.jpg") + + # Modalities should be set for all 3 examples + self.assertEqual(req.modalities, ["image", "image", "image"]) + + def test_audio_data_handling(self): + """Test handling of audio_data.""" + req = copy.deepcopy(self.base_req) + req.audio_data = "audio.mp3" # Single audio + + req.normalize_batch_and_arguments() + + # Should be converted to ["audio.mp3", "audio.mp3"] + self.assertEqual(len(req.audio_data), 2) + self.assertEqual(req.audio_data[0], "audio.mp3") + self.assertEqual(req.audio_data[1], "audio.mp3") + + # Test with list + req = copy.deepcopy(self.base_req) + req.audio_data = ["audio1.mp3", "audio2.mp3"] + + req.normalize_batch_and_arguments() + + # Should remain the same + self.assertEqual(len(req.audio_data), 2) + self.assertEqual(req.audio_data[0], "audio1.mp3") + self.assertEqual(req.audio_data[1], "audio2.mp3") + + def test_input_ids_normalization(self): + """Test normalization of input_ids instead of text.""" + # Test single input_ids + req = GenerateReqInput(input_ids=[1, 2, 3]) + req.normalize_batch_and_arguments() + self.assertTrue(req.is_single) + self.assertEqual(req.batch_size, 1) + + # Test batch input_ids + req = GenerateReqInput(input_ids=[[1, 2, 3], [4, 5, 6]]) + req.normalize_batch_and_arguments() + self.assertFalse(req.is_single) + self.assertEqual(req.batch_size, 2) + + # Test with parallel sampling + req = GenerateReqInput( + input_ids=[[1, 2, 3], [4, 5, 6]], sampling_params={"n": 2} + ) + req.normalize_batch_and_arguments() + self.assertEqual(len(req.input_ids), 4) # 2 original * 2 parallel + + def test_input_embeds_normalization(self): + """Test normalization of input_embeds.""" + # Test single input_embeds + req = GenerateReqInput(input_embeds=[[0.1, 0.2], [0.3, 0.4]]) + req.normalize_batch_and_arguments() + self.assertTrue(req.is_single) + self.assertEqual(req.batch_size, 1) + + # Test batch input_embeds + req = GenerateReqInput(input_embeds=[[[0.1, 0.2]], [[0.3, 0.4]]]) + req.normalize_batch_and_arguments() + self.assertFalse(req.is_single) + self.assertEqual(req.batch_size, 2) + + def test_input_embeds_with_parallel_sampling(self): + """Test input_embeds normalization with parallel sampling (n > 1).""" + # Test single input_embeds with parallel sampling + req = GenerateReqInput( + input_embeds=[[0.1, 0.2]], # single embedding vector + sampling_params={"n": 2}, + ) + req.normalize_batch_and_arguments() + + # Should be converted from single to batch and then expanded + self.assertFalse(req.is_single) + self.assertEqual(len(req.input_embeds), 2) + # Both should be the same input_embeds + self.assertEqual(req.input_embeds[0], [[0.1, 0.2]]) + self.assertEqual(req.input_embeds[1], [[0.1, 0.2]]) + + # Test batch input_embeds with parallel sampling + req = GenerateReqInput( + input_embeds=[[[0.1, 0.2]], [[0.3, 0.4]]], sampling_params={"n": 3} + ) + req.normalize_batch_and_arguments() + + # Should be expanded + self.assertFalse(req.is_single) + self.assertEqual(len(req.input_embeds), 6) + + # Check that the expansion is correct + expected_embeds = [[[0.1, 0.2]], [[0.3, 0.4]]] * 3 + self.assertEqual(req.input_embeds, expected_embeds) + + # Test with different n values per sample (should raise error) + req = GenerateReqInput( + input_embeds=[[[0.1, 0.2]], [[0.3, 0.4]]], + sampling_params=[{"n": 2}, {"n": 3}], + ) + with self.assertRaises(ValueError): + req.normalize_batch_and_arguments() + + def test_input_embeds_single_to_batch_conversion(self): + """Test that single input_embeds are properly converted to batch when using parallel sampling.""" + # Test the specific case that was fixed: single input_embeds with n > 1 + req = GenerateReqInput( + input_embeds=[[0.1, 0.2, 0.3]], sampling_params={"n": 2} # Single embedding + ) + req.normalize_batch_and_arguments() + + # Should convert single to batch and then expand + self.assertFalse(req.is_single) + self.assertEqual(len(req.input_embeds), 2) + + # Both should be the same single embedding + self.assertEqual(req.input_embeds[0], [[0.1, 0.2, 0.3]]) + self.assertEqual(req.input_embeds[1], [[0.1, 0.2, 0.3]]) + + # Test with higher n value + req = GenerateReqInput(input_embeds=[[0.1, 0.2, 0.3]], sampling_params={"n": 5}) + req.normalize_batch_and_arguments() + + self.assertFalse(req.is_single) + self.assertEqual(len(req.input_embeds), 5) + + # All should be the same + for i in range(5): + self.assertEqual(req.input_embeds[i], [[0.1, 0.2, 0.3]]) + + def test_lora_path_normalization(self): + """Test normalization of lora_path.""" + # Test single lora_path with batch input + req = GenerateReqInput(text=["Hello", "World"], lora_path="path/to/lora") + + # Define expected lora_paths before normalization + expected_lora_paths = ["path/to/lora", "path/to/lora"] + + req.normalize_batch_and_arguments() + self.assertEqual(req.lora_path, expected_lora_paths) + + # Test list of lora_paths + req = GenerateReqInput(text=["Hello", "World"], lora_path=["path1", "path2"]) + + # Define expected lora_paths before normalization + expected_lora_paths = ["path1", "path2"] + + req.normalize_batch_and_arguments() + self.assertEqual(req.lora_path, expected_lora_paths) + + # Test with parallel sampling + req = GenerateReqInput( + text=["Hello", "World"], + lora_path=["path1", "path2"], + sampling_params={"n": 2}, + ) + + # Define expected lora_paths before normalization + expected_lora_paths = ["path1", "path2"] * 2 + + req.normalize_batch_and_arguments() + self.assertEqual(req.lora_path, expected_lora_paths) + + def test_logprob_parameters_normalization(self): + """Test normalization of logprob-related parameters.""" + # Test single example + req = GenerateReqInput( + text="Hello", + return_logprob=True, + logprob_start_len=10, + top_logprobs_num=5, + token_ids_logprob=[7, 8, 9], + ) + req.normalize_batch_and_arguments() + self.assertEqual(req.return_logprob, True) + self.assertEqual(req.logprob_start_len, 10) + self.assertEqual(req.top_logprobs_num, 5) + self.assertEqual(req.token_ids_logprob, [7, 8, 9]) + + # Test batch with scalar values + req = GenerateReqInput( + text=["Hello", "World"], + return_logprob=True, + logprob_start_len=10, + top_logprobs_num=5, + token_ids_logprob=[7, 8, 9], + ) + req.normalize_batch_and_arguments() + self.assertEqual(req.return_logprob, [True, True]) + self.assertEqual(req.logprob_start_len, [10, 10]) + self.assertEqual(req.top_logprobs_num, [5, 5]) + self.assertEqual(req.token_ids_logprob, [[7, 8, 9], [7, 8, 9]]) + + # Test batch with list values + req = GenerateReqInput( + text=["Hello", "World"], + return_logprob=[True, False], + logprob_start_len=[10, 5], + top_logprobs_num=[5, 3], + token_ids_logprob=[[7, 8, 9], [4, 5, 6]], + return_hidden_states=[False, False, True], + ) + req.normalize_batch_and_arguments() + self.assertEqual(req.return_logprob, [True, False]) + self.assertEqual(req.logprob_start_len, [10, 5]) + self.assertEqual(req.top_logprobs_num, [5, 3]) + self.assertEqual(req.token_ids_logprob, [[7, 8, 9], [4, 5, 6]]) + self.assertEqual(req.return_hidden_states, [False, False, True]) + + def test_custom_logit_processor_normalization(self): + """Test normalization of custom_logit_processor.""" + # Test single processor + req = GenerateReqInput( + text=["Hello", "World"], custom_logit_processor="serialized_processor" + ) + req.normalize_batch_and_arguments() + self.assertEqual( + req.custom_logit_processor, ["serialized_processor", "serialized_processor"] + ) + + # Test list of processors + req = GenerateReqInput( + text=["Hello", "World"], custom_logit_processor=["processor1", "processor2"] + ) + req.normalize_batch_and_arguments() + self.assertEqual(req.custom_logit_processor, ["processor1", "processor2"]) + + def test_session_params_handling(self): + """Test handling of session_params.""" + # Test with dict + req = GenerateReqInput( + text=["Hello", "World"], session_params={"id": "session1", "offset": 10} + ) + req.normalize_batch_and_arguments() + self.assertEqual(req.session_params, {"id": "session1", "offset": 10}) + + # Test with list of dicts + req = GenerateReqInput( + text=["Hello", "World"], + session_params=[{"id": "session1"}, {"id": "session2"}], + ) + req.normalize_batch_and_arguments() + self.assertEqual(req.session_params, [{"id": "session1"}, {"id": "session2"}]) + + def test_getitem_method(self): + """Test the __getitem__ method.""" + req = GenerateReqInput( + text=["Hello", "World"], + image_data=[["img1.jpg"], ["img2.jpg"]], + audio_data=["audio1.mp3", "audio2.mp3"], + sampling_params=[{"temp": 0.7}, {"temp": 0.8}], + rid=["id1", "id2"], + return_logprob=[True, False], + logprob_start_len=[10, 5], + top_logprobs_num=[5, 3], + token_ids_logprob=[[7, 8, 9], [4, 5, 6]], + stream=True, + log_metrics=True, + modalities=["image", "image"], + lora_path=["path1", "path2"], + custom_logit_processor=["processor1", "processor2"], + return_hidden_states=True, + ) + req.normalize_batch_and_arguments() + + # Get the first item + item0 = req[0] + self.assertEqual(item0.text, "Hello") + self.assertEqual(item0.image_data, ["img1.jpg"]) + self.assertEqual(item0.audio_data, "audio1.mp3") + self.assertEqual(item0.sampling_params, {"temp": 0.7}) + self.assertEqual(item0.rid, "id1") + self.assertEqual(item0.return_logprob, True) + self.assertEqual(item0.logprob_start_len, 10) + self.assertEqual(item0.top_logprobs_num, 5) + self.assertEqual(item0.token_ids_logprob, [7, 8, 9]) + self.assertEqual(item0.stream, True) + self.assertEqual(item0.log_metrics, True) + self.assertEqual(item0.modalities, "image") + self.assertEqual(item0.lora_path, "path1") + self.assertEqual(item0.custom_logit_processor, "processor1") + self.assertEqual(item0.return_hidden_states, True) + + def test_regenerate_rid(self): + """Test the regenerate_rid method.""" + req = GenerateReqInput(text="Hello") + req.normalize_batch_and_arguments() + + original_rid = req.rid + new_rid = req.regenerate_rid() + + self.assertNotEqual(original_rid, new_rid) + self.assertEqual(req.rid, new_rid) + + def test_error_cases(self): + """Test various error cases.""" + # Test when neither text, input_ids, nor input_embeds is provided + with self.assertRaises(ValueError): + req = GenerateReqInput() + req.normalize_batch_and_arguments() + + # Test when all of text, input_ids, and input_embeds are provided + with self.assertRaises(ValueError): + req = GenerateReqInput( + text="Hello", input_ids=[1, 2, 3], input_embeds=[[0.1, 0.2]] + ) + req.normalize_batch_and_arguments() + + def test_multiple_input_formats(self): + """Test different combinations of input formats.""" + # Test with text only + req = GenerateReqInput(text="Hello") + req.normalize_batch_and_arguments() + self.assertTrue(req.is_single) + + # Test with input_ids only + req = GenerateReqInput(input_ids=[1, 2, 3]) + req.normalize_batch_and_arguments() + self.assertTrue(req.is_single) + + # Test with input_embeds only + req = GenerateReqInput(input_embeds=[[0.1, 0.2]]) + req.normalize_batch_and_arguments() + self.assertTrue(req.is_single) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_jinja_template_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_jinja_template_utils.py new file mode 100644 index 00000000..46e63400 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_jinja_template_utils.py @@ -0,0 +1,310 @@ +""" +Unit tests for Jinja chat template utils. +""" + +import unittest + +from sglang.srt.parser.jinja_template_utils import ( + detect_jinja_template_content_format, + process_content_for_template_format, +) +from sglang.test.test_utils import CustomTestCase + + +class TestTemplateContentFormatDetection(CustomTestCase): + """Test template content format detection functionality.""" + + def test_detect_llama4_openai_format(self): + """Test detection of llama4-style template (should be 'openai' format).""" + llama4_pattern = """ +{%- for message in messages %} + {%- if message['content'] is string %} + {{- message['content'] }} + {%- else %} + {%- for content in message['content'] %} + {%- if content['type'] == 'image' %} + {{- '<|image|>' }} + {%- elif content['type'] == 'text' %} + {{- content['text'] | trim }} + {%- endif %} + {%- endfor %} + {%- endif %} +{%- endfor %} + """ + + result = detect_jinja_template_content_format(llama4_pattern) + self.assertEqual(result, "openai") + + def test_detect_deepseek_string_format(self): + """Test detection of deepseek-style template (should be 'string' format).""" + deepseek_pattern = """ +{%- for message in messages %} + {%- if message['role'] == 'user' %} + {{- '<|User|>' + message['content'] + '<|Assistant|>' }} + {%- endif %} +{%- endfor %} + """ + + result = detect_jinja_template_content_format(deepseek_pattern) + self.assertEqual(result, "string") + + def test_detect_invalid_template(self): + """Test handling of invalid template (should default to 'string').""" + invalid_pattern = "{{{{ invalid jinja syntax }}}}" + + result = detect_jinja_template_content_format(invalid_pattern) + self.assertEqual(result, "string") + + def test_detect_empty_template(self): + """Test handling of empty template (should default to 'string').""" + result = detect_jinja_template_content_format("") + self.assertEqual(result, "string") + + def test_detect_msg_content_pattern(self): + """Test detection of template with msg.content pattern (should be 'openai' format).""" + msg_content_pattern = """ +[gMASK] +{%- for msg in messages %} + {%- if msg.role == 'system' %} +<|system|> +{{ msg.content }} + {%- elif msg.role == 'user' %} +<|user|>{{ '\n' }} + {%- if msg.content is string %} +{{ msg.content }} + {%- else %} + {%- for item in msg.content %} + {%- if item.type == 'video' or 'video' in item %} +<|begin_of_video|><|video|><|end_of_video|> + {%- elif item.type == 'image' or 'image' in item %} +<|begin_of_image|><|image|><|end_of_image|> + {%- elif item.type == 'text' %} +{{ item.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- elif msg.role == 'assistant' %} + {%- if msg.metadata %} +<|assistant|>{{ msg.metadata }} +{{ msg.content }} + {%- else %} +<|assistant|> +{{ msg.content }} + {%- endif %} + {%- endif %} +{%- endfor %} +{% if add_generation_prompt %}<|assistant|> +{% endif %} + """ + + result = detect_jinja_template_content_format(msg_content_pattern) + self.assertEqual(result, "openai") + + def test_detect_m_content_pattern(self): + """Test detection of template with m.content pattern (should be 'openai' format).""" + msg_content_pattern = """ +[gMASK] +{%- for m in messages %} + {%- if m.role == 'system' %} +<|system|> +{{ m.content }} + {%- elif m.role == 'user' %} +<|user|>{{ '\n' }} + {%- if m.content is string %} +{{ m.content }} + {%- else %} + {%- for item in m.content %} + {%- if item.type == 'video' or 'video' in item %} +<|begin_of_video|><|video|><|end_of_video|> + {%- elif item.type == 'image' or 'image' in item %} +<|begin_of_image|><|image|><|end_of_image|> + {%- elif item.type == 'text' %} +{{ item.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- elif m.role == 'assistant' %} + {%- if m.metadata %} +<|assistant|>{{ m.metadata }} +{{ m.content }} + {%- else %} +<|assistant|> +{{ m.content }} + {%- endif %} + {%- endif %} +{%- endfor %} +{% if add_generation_prompt %}<|assistant|> +{% endif %} + """ + + result = detect_jinja_template_content_format(msg_content_pattern) + self.assertEqual(result, "openai") + + def test_process_content_openai_format(self): + """Test content processing for openai format.""" + msg_dict = { + "role": "user", + "content": [ + {"type": "text", "text": "Look at this image:"}, + { + "type": "image_url", + "image_url": {"url": "http://example.com/image.jpg"}, + }, + {"type": "text", "text": "What do you see?"}, + ], + } + + image_data = [] + video_data = [] + audio_data = [] + modalities = [] + + result = process_content_for_template_format( + msg_dict, "openai", image_data, video_data, audio_data, modalities + ) + + # Check that image_data was extracted + self.assertEqual(len(image_data), 1) + self.assertEqual(image_data[0].url, "http://example.com/image.jpg") + + # Check that content was normalized + expected_content = [ + {"type": "text", "text": "Look at this image:"}, + {"type": "image"}, # normalized from image_url + {"type": "text", "text": "What do you see?"}, + ] + self.assertEqual(result["content"], expected_content) + self.assertEqual(result["role"], "user") + + def test_process_content_string_format(self): + """Test content processing for string format.""" + msg_dict = { + "role": "user", + "content": [ + {"type": "text", "text": "Hello"}, + { + "type": "image_url", + "image_url": {"url": "http://example.com/image.jpg"}, + }, + {"type": "text", "text": "world"}, + ], + } + + image_data = [] + video_data = [] + audio_data = [] + modalities = [] + + result = process_content_for_template_format( + msg_dict, "string", image_data, video_data, audio_data, modalities + ) + + # For string format, should flatten to text only + self.assertEqual(result["content"], "Hello world") + self.assertEqual(result["role"], "user") + + # Image data should not be extracted for string format + self.assertEqual(len(image_data), 0) + + def test_process_content_with_audio(self): + """Test content processing with audio content.""" + msg_dict = { + "role": "user", + "content": [ + {"type": "text", "text": "Listen to this:"}, + { + "type": "audio_url", + "audio_url": {"url": "http://example.com/audio.mp3"}, + }, + ], + } + + image_data = [] + video_data = [] + audio_data = [] + modalities = [] + + result = process_content_for_template_format( + msg_dict, "openai", image_data, video_data, audio_data, modalities + ) + + # Check that audio_data was extracted + self.assertEqual(len(audio_data), 1) + self.assertEqual(audio_data[0], "http://example.com/audio.mp3") + + # Check that content was normalized + expected_content = [ + {"type": "text", "text": "Listen to this:"}, + {"type": "audio"}, # normalized from audio_url + ] + self.assertEqual(result["content"], expected_content) + + def test_process_content_already_string(self): + """Test processing content that's already a string.""" + msg_dict = {"role": "user", "content": "Hello world"} + + image_data = [] + video_data = [] + audio_data = [] + modalities = [] + + result = process_content_for_template_format( + msg_dict, "openai", image_data, video_data, audio_data, modalities + ) + + # Should pass through unchanged + self.assertEqual(result["content"], "Hello world") + self.assertEqual(result["role"], "user") + self.assertEqual(len(image_data), 0) + + def test_process_content_with_modalities(self): + """Test content processing with modalities field.""" + msg_dict = { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "http://example.com/image.jpg"}, + "modalities": ["vision"], + } + ], + } + + image_data = [] + video_data = [] + audio_data = [] + modalities = [] + + result = process_content_for_template_format( + msg_dict, "openai", image_data, video_data, audio_data, modalities + ) + + # Check that modalities was extracted + self.assertEqual(len(modalities), 1) + self.assertEqual(modalities[0], ["vision"]) + + def test_process_content_filter_none_values(self): + """Test that None values are filtered out of processed messages.""" + msg_dict = { + "role": "user", + "content": "Hello", + "name": None, + "tool_call_id": None, + } + + image_data = [] + video_data = [] + audio_data = [] + modalities = [] + + result = process_content_for_template_format( + msg_dict, "string", image_data, video_data, audio_data, modalities + ) + + # None values should be filtered out + expected_keys = {"role", "content"} + self.assertEqual(set(result.keys()), expected_keys) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_llama31_fp4.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_llama31_fp4.py new file mode 100644 index 00000000..1be96718 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_llama31_fp4.py @@ -0,0 +1,58 @@ +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import get_device_sm, kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + popen_launch_server, +) + +MODEL_PATH = "nvidia/Llama-3.1-8B-Instruct-FP4" + + +@unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher") +class TestLlama31FP4B200(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model = MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--mem-fraction-static", + "0.8", + "--quantization", + "modelopt_fp4", + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + parsed_url = urlparse(self.base_url) + args = SimpleNamespace( + num_shots=4, + data_path=None, + num_questions=100, + max_new_tokens=512, + parallel=128, + host=f"{parsed_url.scheme}://{parsed_url.hostname}", + port=parsed_url.port, + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.61) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_load_weights_from_remote_instance.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_load_weights_from_remote_instance.py new file mode 100644 index 00000000..bd722a92 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_load_weights_from_remote_instance.py @@ -0,0 +1,383 @@ +"""Test loading weights from remote instance. + +This test suite simulates loading weights from a remote instance. +Rank 0 represents the seed instance, while ranks 1 represents the +new instance that needs to loading weights from the seed instance. + +Seed instance must be started in `Server` mode, while the dst instance +can be either `Engine` mode or `Server` mode. + +Seed instance does not support concurrently serving multiple dst instances. +User has to guarantee that there is only one dst instance trying to load +weights from the seed instance at any time. + +""" + +import gc +import os +import random +import unittest + +import numpy as np +import requests +import torch +import torch.multiprocessing as mp + +import sglang as sgl +from sglang.test.test_utils import ( + DEFAULT_PORT_FOR_SRT_TEST_RUNNER, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) +from sglang.utils import terminate_process + +mp.set_start_method("spawn", force=True) + + +def verify_params_close(params1, params2, error_msg): + """Verify if two parameter arrays are close enough.""" + try: + assert np.allclose(np.array(params1), np.array(params2)), error_msg + except Exception as e: + print(f"Parameters not close for {error_msg}") + print("Params1:", np.array(params1)) + print("Params2:", np.array(params2)) + raise e + + +def init_process( + rank, + param_queue, + truncate_size, + tp_size, + model_name, + backends, + checking_parameters, + seed_instance_ip, + seed_instance_service_port, + seed_instance_group_base_port, + event_seed_ready, + event_dst_ready_list, +): + torch.cuda.set_device(rank) + + if rank == 0: + init_process_seed( + rank, + param_queue, + truncate_size, + model_name, + checking_parameters, + tp_size, + event_seed_ready, + event_dst_ready_list, + ) + elif rank in [1, 2]: + init_process_dst( + rank, + param_queue, + truncate_size, + model_name, + seed_instance_ip, + seed_instance_service_port, + seed_instance_group_base_port, + checking_parameters, + backends[rank - 1], + tp_size, + event_seed_ready, + event_dst_ready_list, + ) + + +def init_process_seed( + rank, + param_queue, + truncate_size, + model_name, + checking_parameters, + tp_size, + event_seed_ready, + event_dst_ready_list, +): + # These two environment variables are very important + # to avoid unexpected behaviors of CUDA and NCCL. + os.environ["NCCL_CUMEM_ENABLE"] = "0" + os.environ["NCCL_NVLS_ENABLE"] = "0" + + # Load model and get parameters + torch.cuda.set_device(rank) + torch.cuda.synchronize() + + url = DEFAULT_URL_FOR_TEST + process = popen_launch_server( + model_name, + url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=( + "--base-gpu-id", + str(rank), + "--tp-size", + str(tp_size), + ), + ) + torch.cuda.synchronize() + + seed_params = [] + # Get the weights of seed instance for correctness check. + for parameter_name in checking_parameters: + seed_params.append( + requests.get( + f"{url}/get_weights_by_name", + json={ + "name": parameter_name, + "truncate_size": truncate_size, + }, + ).json() + ) + param_queue.put((f"seed_params", seed_params)) + + event_seed_ready.set() + for i in range(len(event_dst_ready_list)): + event_dst_ready_list[i].wait() + terminate_process(process) + + +def init_process_dst( + rank, + param_queue, + truncate_size, + model_name, + seed_instance_ip, + seed_instance_service_port, + seed_instance_group_base_port, + checking_parameters, + backend, + tp_size, + event_seed_ready, + event_dst_ready_list, +): + torch.cuda.set_device(rank * tp_size) + torch.cuda.synchronize() + base_gpu_id = rank * tp_size + + event_seed_ready.wait() + print(f"rank {rank}, seed ready") + for i in range(rank - 1): + print(f"rank {rank}, wait dst {i}") + event_dst_ready_list[i].wait() + + ports = [] + for i in range(tp_size): + ports.append(seed_instance_group_base_port + (rank - 1) * tp_size + i) + + if backend == "Engine": + print(f"[sgl] rank {rank} init engine") + engine = sgl.Engine( + model_path=model_name, + base_gpu_id=base_gpu_id, + tp_size=tp_size, + cuda_graph_max_bs=2, + tokenizer_path=model_name, + remote_instance_weight_loader_seed_instance_ip=seed_instance_ip, + remote_instance_weight_loader_seed_instance_service_port=seed_instance_service_port, + remote_instance_weight_loader_send_weights_group_ports=ports, + load_format="remote_instance", + ) + else: + host, _, port = DEFAULT_URL_FOR_TEST.rpartition(":") + url = ":".join([host, str(int(port) + 10000 + rank)]) + + print(f"[sgl] rank {rank} init server on url: {url}") + process = popen_launch_server( + model_name, + url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=( + "--base-gpu-id", + str(base_gpu_id), + "--tp-size", + str(tp_size), + "--cuda-graph-max-bs", + 2, + "--tokenizer-path", + model_name, + "--remote-instance-weight-loader-seed-instance-ip", + seed_instance_ip, + "--remote-instance-weight-loader-seed-instance-service-port", + seed_instance_service_port, + "--remote-instance-weight-loader-send-weights-group-ports", + f"[{','.join(str(port) for port in ports)}]", + "--load-format", + "remote_instance", + ), + ) + torch.cuda.synchronize() + + event_dst_ready_list[rank - 1].set() + + # Get weights of destination instance loaded from remote instance. + dst_params = [] + for parameter_name in checking_parameters: + dst_params.append( + engine.get_weights_by_name(parameter_name, truncate_size) + if backend == "Engine" + else requests.get( + f"{url}/get_weights_by_name", + json={"name": parameter_name, "truncate_size": truncate_size}, + ).json() + ) + + param_queue.put((f"sgl_dp_{rank}_dst_params", dst_params)) + + # Shutdown the engine or terminate the server process. + if backend == "Engine": + engine.shutdown() + else: + terminate_process(process) + + +def test_load_weights_from_remote_instance( + tp_size, + dp_size, + model_name, + backends, + truncate_size, + checking_parameters, + seed_instance_ip, + seed_instance_service_port, + seed_instance_group_base_port, +): + print( + f"Testing model: {model_name} tp_size: {tp_size}, dp_size: {dp_size} backend: {backends}" + ) + param_queue = mp.Queue() + results = {} + event_seed_ready = mp.Event() + event_dst_ready_list = [] + for i in range(dp_size): + event_dst_ready = mp.Event() + event_dst_ready_list.append(event_dst_ready) + + context = mp.spawn( + init_process, + args=( + param_queue, + truncate_size, + tp_size, + model_name, + backends, + checking_parameters, + seed_instance_ip, + seed_instance_service_port, + seed_instance_group_base_port, + event_seed_ready, + event_dst_ready_list, + ), + nprocs=1 + dp_size, + join=False, + ) + + while len(results) < (1 + dp_size): + try: + key, value = param_queue.get(timeout=5) + results[key] = value + except Exception as e: + if all(not p.is_alive() for p in context.processes): + break + + context.join() + + if len(results) != (1 + dp_size): + raise RuntimeError( + f"Expected {(1 + dp_size)} parameters but got {len(results)}" + ) + + params = { + "seed": results.get("seed_params"), + "sgl_dp_1_dest": results.get("sgl_dp_1_dst_params"), + } + + if dp_size == 2: + dp2_params = { + "sgl_dp_2_dest": results.get("sgl_dp_2_dst_params"), + } + assert all(v is not None for v in dp2_params.values()) + params.update(dp2_params) + + # Check the correctness of weights loaded from remote instance + # by verifying the weights of seed instance and destination instance. + for i in range(len(params["seed"])): + verify_params_close( + params["seed"][i], + params["sgl_dp_1_dest"][i], + f"sgl_dp_1_dst_params rank {i}", + ) + + if dp_size == 2: + verify_params_close( + params["seed"][i], + params["sgl_dp_2_dest"][i], + f"sgl_dp_2_dst_params rank {i}", + ) + + # Delete the context and close the parameter queue. + del context + param_queue.close() + param_queue.join_thread() + gc.collect() + torch.cuda.empty_cache() + + +class TestLoadWeightsFromRemoteInstance(CustomTestCase): + + def test_load_weights_from_remote_instance(self): + + assert torch.cuda.device_count() >= 2, "At least 2 GPUs are required" + # test_suits : tp, dp, model_name, backend, dst_instance_id + if is_in_ci(): + mode = random.choice(["Engine", "Server"]) + test_suits = [ + (1, 1, DEFAULT_SMALL_MODEL_NAME_FOR_TEST, [mode]), + ] + else: + test_suits = [ + (1, 1, DEFAULT_SMALL_MODEL_NAME_FOR_TEST, ["Engine"]), + (1, 1, DEFAULT_SMALL_MODEL_NAME_FOR_TEST, ["Sever"]), + (2, 2, DEFAULT_SMALL_MODEL_NAME_FOR_TEST, ["Engine", "Server"]), + ] + + truncate_size = 10 + checking_parameters = [ + "model.embed_tokens.weight", + "model.layers.0.input_layernorm.weight", + "model.layers.1.self_attn.q_proj.weight", + "model.layers.2.self_attn.k_proj.weight", + "model.layers.3.self_attn.v_proj.weight", + "model.layers.4.self_attn.o_proj.weight", + "model.layers.5.mlp.gate_proj.weight", + "model.layers.6.mlp.up_proj.weight", + "model.layers.7.mlp.down_proj.weight", + "model.layers.8.post_attention_layernorm.weight", + "model.norm.weight", + ] + + for tp_size, dp_size, model_name, backends in test_suits: + test_load_weights_from_remote_instance( + tp_size, + dp_size, + model_name, + backends, + truncate_size, + checking_parameters, + "127.0.0.1", + DEFAULT_PORT_FOR_SRT_TEST_RUNNER + 1000, + 60000, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_local_attn.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_local_attn.py new file mode 100644 index 00000000..923ffdd5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_local_attn.py @@ -0,0 +1,72 @@ +import os +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import get_device_sm, kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST_LOCAL_ATTENTION, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +@unittest.skipIf(get_device_sm() < 90, "Test requires CUDA SM 90 or higher") +class TestFlashAttention3LocalAttn(CustomTestCase): + model = DEFAULT_MODEL_NAME_FOR_TEST_LOCAL_ATTENTION + base_url = DEFAULT_URL_FOR_TEST + accuracy_threshold = 0.90 + + @classmethod + def get_server_args(cls): + return [ + "--cuda-graph-max-bs", + "2", + "--attention-backend", + "fa3", + "--tp", + "4", + "--context-length", + "1000000", + ] + + @classmethod + def setUpClass(cls): + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=cls.get_server_args(), + env=os.environ, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=4, + num_questions=100, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + data_path=None, + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + # Use the appropriate metric key based on the test class + metric_key = "accuracy" + self.assertGreater(metrics[metric_key], self.accuracy_threshold) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mamba_unittest.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mamba_unittest.py new file mode 100644 index 00000000..d72cad94 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mamba_unittest.py @@ -0,0 +1,337 @@ +import os +import unittest + +import torch + +from sglang.srt.configs.mamba_utils import Mamba2CacheParams, Mamba2StateShape +from sglang.srt.managers.schedule_batch import Req +from sglang.srt.mem_cache.allocator import TokenToKVPoolAllocator +from sglang.srt.mem_cache.cache_init_params import CacheInitParams +from sglang.srt.mem_cache.mamba_radix_cache import MambaRadixCache +from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, HybridReqToTokenPool +from sglang.srt.mem_cache.radix_cache import RadixKey +from sglang.srt.sampling.sampling_params import SamplingParams + + +class TestMamba(unittest.TestCase): + @classmethod + def setUpClass(cls): + pass + + @classmethod + def tearDownClass(cls): + pass + + def test_hybrid_linear_kv_pool(self): + size = 16 + head_num = 2 + head_dim = 256 + num_layers = 48 + global_interval = 4 + dtype = torch.bfloat16 + device = "cuda" + full_attention_layer_ids = [ + i for i in range(global_interval - 1, num_layers, global_interval) + ] + pool = HybridLinearKVPool( + size=size, + dtype=dtype, + page_size=1, + head_num=head_num, + head_dim=head_dim, + full_attention_layer_ids=full_attention_layer_ids, + enable_kvcache_transpose=False, + device=device, + enable_memory_saver=False, + mamba_pool=None, + ) + assert pool._transfer_full_attention_id(global_interval - 1) == 0 + assert pool._transfer_full_attention_id(2 * global_interval - 1) == 1 + with self.assertRaises(ValueError) as context: + pool._transfer_full_attention_id(1) + self.assertIn( + "layer_id=1 not in full attention layers:", str(context.exception) + ) + + def test_mamba_pool(self): + max_num_reqs = 10 + mamba_cache_size = 20 + max_context_len = 128 + device = "cuda" + global_interval = 4 + num_layers = 48 + full_attention_layer_ids = [ + i for i in range(global_interval - 1, num_layers, global_interval) + ] + mamba_layers = [ + i for i in range(num_layers) if i not in full_attention_layer_ids + ] + shape = Mamba2StateShape.create( + tp_world_size=1, + intermediate_size=4096, + n_groups=16, + num_heads=32, + head_dim=128, + state_size=128, + conv_kernel=4, + ) + os.environ["SGLANG_MAMBA_SSM_DTYPE"] = "bfloat16" + mamba2_cache_params = Mamba2CacheParams(shape=shape, layers=mamba_layers) + + req_to_token_pool = HybridReqToTokenPool( + size=max_num_reqs, + mamba_size=mamba_cache_size, + max_context_len=max_context_len, + device=device, + enable_memory_saver=False, + cache_params=mamba2_cache_params, + speculative_num_draft_tokens=3, + ) + + assert req_to_token_pool.available_size() == max_num_reqs + assert req_to_token_pool.mamba_pool.available_size() == mamba_cache_size + + sampling_params = SamplingParams( + temperature=0, + max_new_tokens=1, + ) + req = Req( + rid=0, + origin_input_text="", + origin_input_ids=[], + sampling_params=sampling_params, + ) + + # alloc req + req_index = req_to_token_pool.alloc(1, [req]) + assert req_to_token_pool.available_size() == max_num_reqs - 1 + assert req_to_token_pool.mamba_pool.available_size() == mamba_cache_size - 1 + + # free req + req_to_token_pool.free(req_index) + assert req_to_token_pool.available_size() == max_num_reqs + assert req_to_token_pool.mamba_pool.available_size() == mamba_cache_size + + # alloc req without free mamba cache + req.mamba_pool_idx = None + req_index = req_to_token_pool.alloc(1, [req]) + req_to_token_pool.free(req_index, free_mamba_cache=False) + assert req_to_token_pool.available_size() == max_num_reqs + assert req_to_token_pool.mamba_pool.available_size() == mamba_cache_size - 1 + + # alloc again + req_index = req_to_token_pool.alloc(1, [req]) + assert req_to_token_pool.available_size() == max_num_reqs - 1 + assert req_to_token_pool.mamba_pool.available_size() == mamba_cache_size - 1 + + def test_mamba_radix_cache_1(self): + # kv cache + size = 128 + dtype = torch.bfloat16 + head_num = 2 + head_dim = 256 + num_layers = 48 + global_interval = 4 + max_num_reqs = 10 + mamba_cache_size = 20 + max_context_len = 128 + device = "cuda" + full_attention_layer_ids = [ + i for i in range(global_interval - 1, num_layers, global_interval) + ] + + # mamba + mamba_layers = [ + i for i in range(num_layers) if i not in full_attention_layer_ids + ] + os.environ["SGLANG_MAMBA_SSM_DTYPE"] = "bfloat16" + shape = Mamba2StateShape.create( + tp_world_size=1, + intermediate_size=4096, + n_groups=16, + num_heads=32, + head_dim=128, + state_size=128, + conv_kernel=4, + ) + mamba2_cache_params = Mamba2CacheParams(shape=shape, layers=mamba_layers) + + req_to_token_pool = HybridReqToTokenPool( + size=max_num_reqs, + mamba_size=mamba_cache_size, + max_context_len=max_context_len, + device=device, + enable_memory_saver=False, + cache_params=mamba2_cache_params, + speculative_num_draft_tokens=3, + ) + # setup kv pool + pool = HybridLinearKVPool( + size=size, + dtype=dtype, + page_size=1, + head_num=head_num, + head_dim=head_dim, + full_attention_layer_ids=full_attention_layer_ids, + enable_kvcache_transpose=False, + device=device, + enable_memory_saver=False, + mamba_pool=req_to_token_pool.mamba_pool, + ) + + # setup token to kv pool allocator + allocator = TokenToKVPoolAllocator( + size=size, + dtype=dtype, + device=device, + kvcache=pool, + need_sort=False, + ) + params = CacheInitParams( + req_to_token_pool=req_to_token_pool, + token_to_kv_pool_allocator=allocator, + page_size=1, + disable=False, + ) + # setup radix cache + tree = MambaRadixCache(params=params) + + def make_dummy_req(): + sampling_params = SamplingParams( + temperature=0, + max_new_tokens=1, + ) + req = Req( + rid=0, + origin_input_text="", + origin_input_ids=[], + sampling_params=sampling_params, + ) + req_to_token_pool.alloc(1, reqs=[req]) + return req + + mamba_pool = req_to_token_pool.mamba_pool + # test + print( + f"[Start] allocator mamba available size: {mamba_pool.available_size()}, full available size: {allocator.available_size()}" + ) + req1 = make_dummy_req() + req1_token_ids, req1_kv_indices = [1, 2, 3], allocator.alloc(3) + assert len(req1_token_ids) == len(req1_kv_indices) + print( + f"req1: inserting, req1_token_ids: {req1_token_ids}, req1_kv_indices: {req1_kv_indices}" + ) + prefix_len = tree.insert( + RadixKey(req1_token_ids), req1_kv_indices, req1.mamba_pool_idx.unsqueeze(0) + ) + print( + f"req1: prefix_len: {prefix_len}, allocator mamba available size: {mamba_pool.available_size()}, full available size: {allocator.available_size()}" + ) + req2 = make_dummy_req() + req2_token_ids, req2_kv_indices = [1, 2, 3, 4, 5, 6, 7], allocator.alloc(7) + assert len(req2_token_ids) == len(req2_kv_indices) + print( + f"req2: inserting, req2_token_ids: {req2_token_ids}, req2_kv_indices: {req2_kv_indices}" + ) + prefix_len = tree.insert( + RadixKey(req2_token_ids), req2_kv_indices, req2.mamba_pool_idx.unsqueeze(0) + ) + print( + f"req2: prefix_len: {prefix_len}, allocator mamba available size: {mamba_pool.available_size()}, full available size: {allocator.available_size()}" + ) + + req3 = make_dummy_req() + req3_token_ids, req3_kv_indices = [10, 11, 12], allocator.alloc(3) + assert len(req3_token_ids) == len(req3_kv_indices) + print( + f"req3: inserting, req3_token_ids: {req3_token_ids}, req3_kv_indices: {req3_kv_indices}" + ) + prefix_len = tree.insert( + RadixKey(req3_token_ids), req3_kv_indices, req3.mamba_pool_idx.unsqueeze(0) + ) + print( + f"req3: prefix_len: {prefix_len}, allocator mamba available size: {mamba_pool.available_size()}, full available size: {allocator.available_size()}" + ) + req4 = make_dummy_req() + req4_token_ids, req4_kv_indices = [1, 2, 3, 4, 5, 60, 70], allocator.alloc(7) + assert len(req4_token_ids) == len(req4_kv_indices) + print( + f"req4: inserting, req4_token_ids: {req4_token_ids}, req4_kv_indices: {req4_kv_indices}" + ) + prefix_len = tree.insert( + RadixKey(req4_token_ids), req4_kv_indices, req4.mamba_pool_idx.unsqueeze(0) + ) + print( + f"req4: prefix_len: {prefix_len}, allocator mamba available size: {mamba_pool.available_size()}, full available size: {allocator.available_size()}" + ) + + tree.pretty_print() + full_num_tokens = 1 + print(f"evicting {full_num_tokens} full token") + tree.evict(full_num_tokens=full_num_tokens) + tree.pretty_print() + + mamba_num = 1 + print(f"evicting {mamba_num} mamba") + tree.evict_mamba(mamba_num=mamba_num) + tree.pretty_print() + + req5_token_ids = [1, 2, 3, 4, 5] + result = tree.match_prefix(RadixKey(req5_token_ids)) + kv_indices, last_node = result.device_indices, result.last_device_node + print( + f"req5: token_ids: {req5_token_ids}, matched kv_indices: {kv_indices}, last_node.key: {last_node.key}" + ) + assert len(kv_indices) == 0 + + req6_token_ids = [1, 2, 3, 4, 5, 60, 70] + result = tree.match_prefix(RadixKey(req6_token_ids)) + kv_indices, last_node = result.device_indices, result.last_device_node + print( + f"req6: token_ids: {req6_token_ids}, matched kv_indices: {kv_indices}, last_node.key: {last_node.key}" + ) + assert len(kv_indices) == 7 + assert len(last_node.key) == 2 + + req7_token_ids = [1, 2, 3, 4, 5, 6, 7] + result = tree.match_prefix(RadixKey(req7_token_ids)) + kv_indices, last_node = result.device_indices, result.last_device_node + print( + f"req7: token_ids: {req7_token_ids}, matched kv_indices: {kv_indices}, last_node.key: {last_node.key}" + ) + assert len(kv_indices) == 7 + assert len(last_node.key) == 2 + + mamba_num = 1 + print(f"evicting {mamba_num} mamba") + tree.evict_mamba(mamba_num=mamba_num) + tree.pretty_print() + + req8_token_ids = [1, 2, 3, 4, 5, 60, 70] + result = tree.match_prefix(RadixKey(req8_token_ids)) + kv_indices, last_node = result.device_indices, result.last_device_node + print( + f"req8: token_ids: {req8_token_ids}, matched kv_indices: {kv_indices}, last_node.key: {last_node.key}" + ) + assert len(kv_indices) == 0 + assert len(last_node.key) == 0 + + req9_token_ids = [1, 2, 3, 4, 5, 6, 7] + req9 = make_dummy_req() + result = tree.match_prefix( + RadixKey(req9_token_ids), **({"req": req9, "cow_mamba": True}) + ) + kv_indices, last_node = result.device_indices, result.last_device_node + assert req9.mamba_pool_idx is not None + assert torch.all( + mamba_pool.mamba_cache.conv[0][:, req9.mamba_pool_idx] + == mamba_pool.mamba_cache.conv[0][:, last_node.mamba_value] + ) + assert torch.all( + mamba_pool.mamba_cache.temporal[:, req9.mamba_pool_idx] + == mamba_pool.mamba_cache.temporal[:, last_node.mamba_value] + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_metrics.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_metrics.py new file mode 100644 index 00000000..ce85e0d8 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_metrics.py @@ -0,0 +1,88 @@ +import unittest + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestEnableMetrics(CustomTestCase): + def test_metrics_enabled(self): + """Test that metrics endpoint returns data when enabled""" + process = popen_launch_server( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_URL_FOR_TEST, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--enable-metrics", "--cuda-graph-max-bs", 2], + ) + + try: + # Make some requests to generate some metrics + response = requests.get(f"{DEFAULT_URL_FOR_TEST}/health_generate") + self.assertEqual(response.status_code, 200) + + response = requests.post( + f"{DEFAULT_URL_FOR_TEST}/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + "stream": True, + }, + stream=True, + ) + for _ in response.iter_lines(decode_unicode=False): + pass + + # Get metrics + metrics_response = requests.get(f"{DEFAULT_URL_FOR_TEST}/metrics") + self.assertEqual(metrics_response.status_code, 200) + metrics_content = metrics_response.text + + print(f"metrics_content=\n{metrics_content}") + + # Verify essential metrics are present + essential_metrics = [ + "sglang:num_running_reqs", + "sglang:num_used_tokens", + "sglang:token_usage", + "sglang:gen_throughput", + "sglang:num_queue_reqs", + "sglang:num_grammar_queue_reqs", + "sglang:cache_hit_rate", + "sglang:spec_accept_length", + "sglang:prompt_tokens_total", + "sglang:generation_tokens_total", + "sglang:cached_tokens_total", + "sglang:num_requests_total", + "sglang:time_to_first_token_seconds", + "sglang:inter_token_latency_seconds", + "sglang:e2e_request_latency_seconds", + ] + + for metric in essential_metrics: + self.assertIn(metric, metrics_content, f"Missing metric: {metric}") + + # Verify model name label is present and correct + expected_model_name = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + self.assertIn(f'model_name="{expected_model_name}"', metrics_content) + + # Verify metrics have values (not empty) + self.assertIn("_sum{", metrics_content) + self.assertIn("_count{", metrics_content) + self.assertIn("_bucket{", metrics_content) + + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_metrics_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_metrics_utils.py new file mode 100644 index 00000000..1a93a75e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_metrics_utils.py @@ -0,0 +1,137 @@ +import unittest + +from sglang.srt.metrics.utils import generate_buckets, two_sides_exponential_buckets + + +class TestMetricsUtils(unittest.TestCase): + """Test cases for metrics utility functions.""" + + def test_two_sides_exponential_buckets_basic(self): + """Test basic functionality of two_sides_exponential_buckets.""" + # Test with simple parameters + count = 5 + buckets = two_sides_exponential_buckets(middle=10.0, base=2.0, count=count) + + # Should contain the middle value + self.assertIn(10.0, buckets) + + # Should be sorted + self.assertEqual(buckets, sorted(buckets)) + + # Should have unique values (no duplicates) + self.assertEqual(len(buckets), len(set(buckets))) + + # Should have reasonable number of buckets (not exactly count due to ceiling and deduplication) + self.assertGreaterEqual(len(buckets), 3) + self.assertLessEqual(len(buckets), count + 2) + + def test_two_sides_exponential_buckets_specific_values(self): + """Test specific values for two_sides_exponential_buckets.""" + buckets = two_sides_exponential_buckets(middle=100.0, base=2.0, count=4) + expected_values = [96.0, 98.0, 100.0, 102.0, 104.0] + self.assertEqual(buckets, expected_values) + + def test_two_sides_exponential_buckets_negative_values(self): + """Test two_sides_exponential_buckets with values that could go negative.""" + buckets = two_sides_exponential_buckets(middle=5.0, base=3.0, count=4) + + # Should not contain negative values (max(0, middle - distance)) + for bucket in buckets: + self.assertGreaterEqual(bucket, 0.0) + + # Should contain the middle value + self.assertIn(5.0, buckets) + + def test_two_sides_exponential_buckets_edge_cases(self): + """Test edge cases for two_sides_exponential_buckets.""" + # Count = 1 + buckets = two_sides_exponential_buckets(middle=10.0, base=2.0, count=1) + self.assertIn(10.0, buckets) + + # Very small middle value + buckets = two_sides_exponential_buckets(middle=0.1, base=2.0, count=2) + self.assertIn(0.1, buckets) + for bucket in buckets: + self.assertGreaterEqual(bucket, 0.0) + + def test_generate_buckets_default(self): + """Test generate_buckets with default rule.""" + default_buckets = [1.0, 5.0, 10.0, 50.0, 100.0] + + # Test with "default" rule + result = generate_buckets(["default"], default_buckets) + self.assertEqual(result, default_buckets) + + # Test with None (should default to "default") + result = generate_buckets(None, default_buckets) + self.assertEqual(result, default_buckets) + + # Test with empty (should default to "default") + result = generate_buckets(None, default_buckets) + self.assertEqual(result, default_buckets) + + def test_generate_buckets_tse(self): + """Test generate_buckets with tse (two sides exponential) rule.""" + default_buckets = [1.0, 5.0, 10.0] + + # Test with "tse" rule + result = generate_buckets(["tse", "10", "2.0", "4"], default_buckets) + + # Should return the same as calling two_sides_exponential_buckets directly + expected = two_sides_exponential_buckets(10.0, 2.0, 4) + self.assertEqual(result, expected) + + def test_generate_buckets_custom(self): + """Test generate_buckets with custom rule.""" + default_buckets = [1.0, 5.0, 10.0] + + # Test with "custom" rule + result = generate_buckets( + ["custom", "1.5", "3.2", "7.8", "15.6"], default_buckets + ) + expected = [1.5, 3.2, 7.8, 15.6] + self.assertEqual(result, expected) + + def test_generate_buckets_custom_with_integers(self): + """Test generate_buckets with custom rule using integer strings.""" + default_buckets = [1.0, 5.0, 10.0] + + # Test with integer strings + result = generate_buckets(["custom", "1", "5", "10", "50"], default_buckets) + expected = [1.0, 5.0, 10.0, 50.0] + self.assertEqual(result, expected) + + def test_generate_buckets_preserves_order_and_type(self): + """Test that generate_buckets preserves order and returns floats.""" + default_buckets = [1, 5, 10, 50, 100] # integers + + # Test default rule + result = generate_buckets(["default"], default_buckets) + self.assertEqual(result, default_buckets) + self.assertIsInstance(result, list) + + # Test custom rule with proper float conversion + result = generate_buckets( + ["custom", "100", "50", "10", "5", "1"], default_buckets + ) + expected = [1.0, 5.0, 10.0, 50.0, 100.0] + self.assertEqual(result, expected) + + # All values should be floats + for value in result: + self.assertIsInstance(value, float) + + def test_integration_tse_through_generate_buckets(self): + """Test integration of TSE buckets through generate_buckets function.""" + default_buckets = [1.0, 10.0, 100.0] + + # Generate buckets using both methods + direct_result = two_sides_exponential_buckets(50.0, 1.5, 6) + indirect_result = generate_buckets(["tse", "50.0", "1.5", "6"], default_buckets) + + # Results should be identical + self.assertEqual(direct_result, indirect_result) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mla.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mla.py new file mode 100644 index 00000000..af867797 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mla.py @@ -0,0 +1,52 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestMLA(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--enable-torch-compile", + "--torch-compile-max-bs", + "4", + "--chunked-prefill-size", + "256", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.8) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mla_deepseek_v3.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mla_deepseek_v3.py new file mode 100644 index 00000000..4e9e99ce --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mla_deepseek_v3.py @@ -0,0 +1,191 @@ +import os +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import is_cuda, is_hip, kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) + + +class TestMLADeepseekV3(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmsys/sglang-ci-dsv3-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code", "--chunked-prefill-size", "256"] + if is_cuda(): + other_args.extend(["--enable-torch-compile", "--cuda-graph-max-bs", "2"]) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.62) + + +@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") +class TestMLADeepseekV3DisableFusedFunc(CustomTestCase): + @classmethod + def setUpClass(cls): + os.environ["SGLANG_CI_DISABLE_MOE_FUSED_FUNC"] = "1" + cls.model = "lmsys/sglang-ci-dsv3-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code", "--chunked-prefill-size", "256"] + if is_cuda(): + other_args.extend(["--cuda-graph-max-bs", "2"]) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.62) + + +@unittest.skipIf(is_hip(), "FA is not available.") +class TestMLADeepseekV3Fa3Fp8Kvcache(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmsys/sglang-ci-dsv3-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--chunked-prefill-size", + "256", + "--kv-cache-dtype", + "fp8_e4m3", + ] + if is_cuda(): + other_args.extend(["--attention-backend", "fa3"]) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.62) + + +class TestDeepseekV3MTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmsys/sglang-ci-dsv3-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--cuda-graph-max-bs", + "2", + "--disable-radix", + "--enable-torch-compile", + "--torch-compile-max-bs", + "1", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, 2.5) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mla_flashinfer.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mla_flashinfer.py new file mode 100644 index 00000000..f72aef5a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mla_flashinfer.py @@ -0,0 +1,124 @@ +import unittest +from types import SimpleNamespace + +import requests +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestFlashinferMLA(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmsys/sglang-ci-dsv3-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + [ + "--enable-torch-compile", + "--cuda-graph-max-bs", + "4", + "--attention-backend", + "flashinfer", + ] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.615) + + +class TestFlashinferMLAMTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmsys/sglang-ci-dsv3-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + [ + "--cuda-graph-max-bs", + "4", + "--enable-torch-compile", + "--torch-compile-max-bs", + "1", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--attention-backend", + "flashinfer", + ] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + server_info = requests.get(self.base_url + "/get_server_info") + print(f"{server_info=}") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, 2.5) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mla_fp8.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mla_fp8.py new file mode 100644 index 00000000..a2fac988 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mla_fp8.py @@ -0,0 +1,49 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MLA_FP8_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestMLA(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_FP8_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--kv-cache-dtype", + "fp8_e5m2", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + assert metrics["score"] >= 0.8 + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mla_int8_deepseek_v3.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mla_int8_deepseek_v3.py new file mode 100644 index 00000000..ceea8835 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_mla_int8_deepseek_v3.py @@ -0,0 +1,229 @@ +import unittest +from types import SimpleNamespace + +import requests +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) + + +class TestMLADeepseekV3ChannelInt8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "sgl-project/sglang-ci-dsv3-channel-int8-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + [ + "--cuda-graph-max-bs", + "16", + "--enable-torch-compile", + "--torch-compile-max-bs", + "2", + ] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreaterEqual(metrics["accuracy"], 0.61) + + +@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") +class TestDeepseekV3MTPChannelInt8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "sgl-project/sglang-ci-dsv3-channel-int8-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + [ + "--cuda-graph-max-bs", + "16", + "--enable-torch-compile", + "--torch-compile-max-bs", + "2", + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + "sgl-project/sglang-ci-dsv3-channel-int8-test-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, 2.5) + + +@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") +class TestMLADeepseekV3BlockInt8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "sgl-project/sglang-ci-dsv3-block-int8-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + [ + "--cuda-graph-max-bs", + "16", + "--enable-torch-compile", + "--torch-compile-max-bs", + "2", + ] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.62) + + +class TestDeepseekV3MTPBlockInt8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "sgl-project/sglang-ci-dsv3-block-int8-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + [ + "--cuda-graph-max-bs", + "16", + "--enable-torch-compile", + "--torch-compile-max-bs", + "2", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, 2.5) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_model_hooks.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_model_hooks.py new file mode 100644 index 00000000..473aafb3 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_model_hooks.py @@ -0,0 +1,152 @@ +import argparse +import json + +import torch +import torch.nn as nn + +from sglang.srt.model_executor.hook_manager import register_forward_hooks +from sglang.srt.server_args import ServerArgs +from sglang.test.test_utils import CustomTestCase + +HOOK_CALLS = [] + + +def dummy_hook_factory(config): + """Factory that returns a forward hook capturing a tag from config.""" + tag = config.get("tag", "default") + + def hook(module, inputs, output): + HOOK_CALLS.append( + { + "module_type": type(module).__name__, + "tag": tag, + "shape": tuple(output.shape), + } + ) + return output + + return hook + + +class TinyModel(nn.Module): + def __init__(self): + super().__init__() + self.inner = nn.Sequential( + nn.Linear(4, 2), + nn.ReLU(), + ) + self.outer = nn.Sequential( + nn.Linear(4, 4), + nn.ReLU(), + self.inner, + ) + + def forward(self, x): + return self.outer(x) + + +class TestAttachHooks(CustomTestCase): + """Tests for register_forward_hooks / resolve_callable integration.""" + + def setUp(self): + HOOK_CALLS.clear() + + def test_hook_is_attached(self): + """Hook from a factory string is registered and fired.""" + hook_specs = [ + { + "target_modules": ["outer.0", "outer.1"], + "hook_factory": "test_model_hooks:dummy_hook_factory", + "config": {"tag": "forward-ok"}, + }, + { + "target_modules": ["inner.*"], + "hook_factory": "test_model_hooks:dummy_hook_factory", + "config": {"tag": "forward-ok"}, + }, + ] + + model = TinyModel() + register_forward_hooks(model, hook_specs) + + x = torch.randn(3, 4) + _ = model(x) + + self.assertEqual( + len(HOOK_CALLS), + 4, + "Forward hook was not called correct number of times", + ) + tags = {call["tag"] for call in HOOK_CALLS} + self.assertIn("forward-ok", tags) + + def test_no_matching_modules_does_not_crash(self): + """Hook spec with no matching modules should not crash.""" + model = TinyModel() + hook_specs = [ + { + "name": "no_match", + "target_modules": ["does_not_exist.*"], + "hook_factory": "test_model_hooks:dummy_hook_factory", + "config": {"tag": "unused"}, + } + ] + + register_forward_hooks(model, hook_specs) + + x = torch.randn(3, 4) + _ = model(x) + + # No hooks should have fired + self.assertEqual(len(HOOK_CALLS), 0) + + def test_cli_hooks_reach_model(self): + """ + Ensure that when hooks are provided via CLI, they are parsed into + ServerArgs, passed to register_forward_hooks, and actually + run during a forward pass. + """ + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + + hooks_spec = [ + { + "name": "outer_and_inner_from_cli", + "target_modules": ["outer.0", "outer.1", "inner.*"], + "hook_factory": "test_model_hooks:dummy_hook_factory", + "config": {"tag": "cli-hook"}, + } + ] + + cli_args = [ + "--model-path", + "Qwen/Qwen2-7B-Instruct", # Dummy value; not used in this test + "--forward-hooks", + json.dumps(hooks_spec), + ] + + args = parser.parse_args(cli_args) + server_args = ServerArgs.from_cli_args(args) + + self.assertEqual(server_args.forward_hooks, hooks_spec) + + model = TinyModel() + register_forward_hooks(model, server_args.forward_hooks) + + x = torch.randn(3, 4) + _ = model(x) + + # We expect hooks on outer.0, outer.1, inner.0, inner.1 => 4 calls + self.assertEqual( + len(HOOK_CALLS), + 4, + "CLI-configured hooks did not fire expected number of times", + ) + + tags = {call["tag"] for call in HOOK_CALLS} + self.assertEqual(tags, {"cli-hook"}) + + +if __name__ == "__main__": + pass + # unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_modelopt_export.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_modelopt_export.py new file mode 100644 index 00000000..aa477ff6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_modelopt_export.py @@ -0,0 +1,352 @@ +""" +Unit tests for ModelOpt export functionality in SGLang. + +These tests verify the integration of ModelOpt export API with SGLang's model loading +and quantization workflow. +""" + +import json +import os +import tempfile +import unittest +from unittest.mock import Mock, patch + +import torch + +from sglang.srt.configs.device_config import DeviceConfig +from sglang.srt.configs.load_config import LoadConfig +from sglang.srt.configs.model_config import ModelConfig +from sglang.srt.model_loader.loader import ModelOptModelLoader + +# Note: PYTHONPATH=python should be set when running tests + +# Check if modelopt is available +try: + import modelopt # noqa: F401 + + MODELOPT_AVAILABLE = True +except ImportError: + MODELOPT_AVAILABLE = False + + +class TestModelOptExport(unittest.TestCase): + """Test suite for ModelOpt export functionality.""" + + def setUp(self): + """Set up test fixtures.""" + # Mock distributed functionality to avoid initialization errors + self.mock_tp_rank = patch( + "sglang.srt.distributed.parallel_state.get_tensor_model_parallel_rank", + return_value=0, + ) + self.mock_tp_rank.start() + + self.mock_rank0_log = patch("sglang.srt.model_loader.loader.rank0_log") + self.mock_rank0_log.start() + + # Mock logger to avoid issues + self.mock_logger = patch("sglang.srt.model_loader.loader.logger") + self.mock_logger.start() + + # Mock all distributed functions that might be called + self.mock_get_tp_group = patch( + "sglang.srt.distributed.parallel_state.get_tp_group" + ) + self.mock_get_tp_group.start() + + # Mock model parallel initialization check + self.mock_mp_is_initialized = patch( + "sglang.srt.distributed.parallel_state.model_parallel_is_initialized", + return_value=True, + ) + self.mock_mp_is_initialized.start() + self.temp_dir = tempfile.mkdtemp() + self.export_dir = os.path.join(self.temp_dir, "exported_model") + self.checkpoint_dir = os.path.join(self.temp_dir, "checkpoint") + + # Mock model + self.mock_model = Mock(spec=torch.nn.Module) + self.mock_model.device = torch.device("cuda:0") + + # Mock tokenizer + self.mock_tokenizer = Mock() + + # Mock quantization config + self.mock_quant_cfg = Mock() + + # Create ModelOptModelLoader instance + self.load_config = LoadConfig() + self.model_loader = ModelOptModelLoader(self.load_config) + + def tearDown(self): + """Clean up test fixtures.""" + import shutil + + shutil.rmtree(self.temp_dir, ignore_errors=True) + + # Stop mocks + self.mock_tp_rank.stop() + self.mock_rank0_log.stop() + self.mock_logger.stop() + self.mock_get_tp_group.stop() + self.mock_mp_is_initialized.stop() + + def _create_mock_export_files(self, export_dir: str): + """Create mock export files for testing validation.""" + os.makedirs(export_dir, exist_ok=True) + + # Create config.json + config = { + "model_type": "test_model", + "architectures": ["TestModel"], + "quantization_config": { + "quant_method": "modelopt", + "bits": 8, + }, + } + with open(os.path.join(export_dir, "config.json"), "w") as f: + json.dump(config, f) + + # Create tokenizer_config.json + tokenizer_config = {"tokenizer_class": "TestTokenizer"} + with open(os.path.join(export_dir, "tokenizer_config.json"), "w") as f: + json.dump(tokenizer_config, f) + + # Create model file + with open(os.path.join(export_dir, "model.safetensors"), "w") as f: + f.write("mock_model_data") + + @unittest.skipIf(not MODELOPT_AVAILABLE, "nvidia-modelopt not available") + @patch("sglang.srt.model_loader.loader.os.makedirs") + @patch("modelopt.torch.export.export_hf_checkpoint") + def test_export_modelopt_checkpoint_success(self, mock_export, mock_makedirs): + """Test successful model export.""" + # Arrange + mock_export.return_value = None + mock_makedirs.return_value = None + + # Act + self.model_loader._export_modelopt_checkpoint(self.mock_model, self.export_dir) + + # Assert + mock_makedirs.assert_called_once_with(self.export_dir, exist_ok=True) + mock_export.assert_called_once_with(self.mock_model, export_dir=self.export_dir) + + @unittest.skipIf(not MODELOPT_AVAILABLE, "nvidia-modelopt not available") + @patch("modelopt.torch.opt.restore") + @patch("modelopt.torch.quantization.utils.is_quantized") + def test_setup_quantization_with_export_from_checkpoint( + self, mock_is_quantized, mock_restore + ): + """Test export functionality when restoring from checkpoint.""" + # Arrange + mock_is_quantized.return_value = False + mock_restore.return_value = None + + with patch.object( + self.model_loader, "_export_modelopt_checkpoint" + ) as mock_export: + # Act + self.model_loader._setup_modelopt_quantization( + self.mock_model, + self.mock_tokenizer, + self.mock_quant_cfg, + quantized_ckpt_restore_path=self.checkpoint_dir, + export_path=self.export_dir, + ) + + # Assert + mock_restore.assert_called_once_with(self.mock_model, self.checkpoint_dir) + mock_export.assert_called_once_with(self.mock_model, self.export_dir, None) + + @unittest.skipIf(not MODELOPT_AVAILABLE, "nvidia-modelopt not available") + @patch("modelopt.torch.quantization.quantize") + @patch("modelopt.torch.quantization.print_quant_summary") + @patch("modelopt.torch.quantization.utils.is_quantized") + @patch("modelopt.torch.utils.dataset_utils.get_dataset_dataloader") + @patch("modelopt.torch.utils.dataset_utils.create_forward_loop") + def test_setup_quantization_with_export_after_calibration( + self, + mock_create_loop, + mock_get_dataloader, + mock_is_quantized, + mock_print_summary, + mock_quantize, + ): + """Test export functionality after calibration-based quantization.""" + # Arrange + mock_is_quantized.return_value = False + mock_dataloader = Mock() + mock_get_dataloader.return_value = mock_dataloader + mock_calibrate_loop = Mock() + mock_create_loop.return_value = mock_calibrate_loop + mock_quantize.return_value = None + mock_print_summary.return_value = None + + with patch.object( + self.model_loader, "_export_modelopt_checkpoint" + ) as mock_export: + # Act + self.model_loader._setup_modelopt_quantization( + self.mock_model, + self.mock_tokenizer, + self.mock_quant_cfg, + export_path=self.export_dir, + ) + + # Assert + mock_quantize.assert_called_once_with( + self.mock_model, self.mock_quant_cfg, forward_loop=mock_calibrate_loop + ) + mock_export.assert_called_once_with(self.mock_model, self.export_dir, None) + + @unittest.skipIf(not MODELOPT_AVAILABLE, "nvidia-modelopt not available") + def test_setup_quantization_without_export(self): + """Test quantization setup without export path specified.""" + with patch("modelopt.torch.quantization.utils.is_quantized", return_value=True): + # Act + with patch.object( + self.model_loader, "_export_modelopt_checkpoint" + ) as mock_export: + self.model_loader._setup_modelopt_quantization( + self.mock_model, + self.mock_tokenizer, + self.mock_quant_cfg, + export_path=None, # No export path + ) + + # Assert + mock_export.assert_not_called() + + def test_quantize_and_serve_config_validation(self): + """Test that quantize_and_serve is properly disabled.""" + # Test that quantize-and-serve mode raises NotImplementedError + with self.assertRaises(NotImplementedError) as context: + ModelConfig( + model_path="TinyLlama/TinyLlama-1.1B-Chat-v1.0", + quantization="modelopt_fp8", + quantize_and_serve=True, + ) + + # Verify the error message contains helpful instructions + error_msg = str(context.exception) + self.assertIn("disabled due to compatibility issues", error_msg) + self.assertIn("separate quantize-then-deploy workflow", error_msg) + + # Test invalid configuration - no quantization + with self.assertRaises(ValueError) as context: + ModelConfig( + model_path="TinyLlama/TinyLlama-1.1B-Chat-v1.0", + quantize_and_serve=True, + ) + self.assertIn("requires ModelOpt quantization", str(context.exception)) + + @unittest.skipIf(not MODELOPT_AVAILABLE, "nvidia-modelopt not available") + def test_standard_workflow_selection(self): + """Test that standard workflow is selected by default.""" + with patch( + "modelopt.torch.quantization.utils.is_quantized", return_value=False + ): + with patch.object( + self.model_loader, "_standard_quantization_workflow" + ) as mock_standard: + with patch.object(self.model_loader, "_load_modelopt_base_model"): + mock_standard.return_value = Mock() + + # Create model config without quantize_and_serve + model_config = ModelConfig( + model_path="TinyLlama/TinyLlama-1.1B-Chat-v1.0", + quantization="modelopt_fp8", + quantize_and_serve=False, + ) + device_config = DeviceConfig() + + # Act + self.model_loader.load_model( + model_config=model_config, + device_config=device_config, + ) + + # Assert + mock_standard.assert_called_once_with(model_config, device_config) + + def _get_export_info(self, export_dir: str) -> dict: + """Get information about an exported model.""" + if not self._validate_export(export_dir): + return None + + try: + config_path = os.path.join(export_dir, "config.json") + with open(config_path, "r") as f: + config = json.load(f) + + return { + "model_type": config.get("model_type", "unknown"), + "architectures": config.get("architectures", []), + "quantization_config": config.get("quantization_config", {}), + "export_dir": export_dir, + } + except Exception: + return None + + +@unittest.skipIf(not MODELOPT_AVAILABLE, "nvidia-modelopt not available") +class TestModelOptExportIntegration(unittest.TestCase): + """Integration tests for ModelOpt export with full model loading workflow.""" + + def setUp(self): + """Set up integration test fixtures.""" + self.temp_dir = tempfile.mkdtemp() + self.export_dir = os.path.join(self.temp_dir, "exported_model") + + def tearDown(self): + """Clean up integration test fixtures.""" + import shutil + + shutil.rmtree(self.temp_dir, ignore_errors=True) + + @patch("sglang.srt.model_loader.loader.get_model_architecture") + @patch("transformers.AutoTokenizer.from_pretrained") + @patch("transformers.AutoModelForCausalLM.from_pretrained") + def test_full_workflow_with_export(self, mock_model, mock_tokenizer, mock_arch): + """Test the complete workflow from model config to export.""" + # Arrange + mock_arch.return_value = ("TestModel", "TestConfig") + mock_tokenizer.return_value = Mock() + mock_model.return_value = Mock(spec=torch.nn.Module) + + model_config = ModelConfig( + model_path="TinyLlama/TinyLlama-1.1B-Chat-v1.0", + modelopt_quant="fp8", + modelopt_export_path=self.export_dir, + ) + + load_config = LoadConfig() + device_config = DeviceConfig() + + # Mock the quantization and export process + with patch.object( + ModelOptModelLoader, "_setup_modelopt_quantization" + ) as mock_setup: + with patch.object( + ModelOptModelLoader, "_load_modelopt_base_model" + ) as mock_load_base: + mock_load_base.return_value = mock_model.return_value + + # Act + model_loader = ModelOptModelLoader(load_config) + result = model_loader.load_model( + model_config=model_config, + device_config=device_config, + ) + + # Assert + self.assertIsNotNone(result) + mock_setup.assert_called_once() + # Verify export_path was passed to setup + args, kwargs = mock_setup.call_args + self.assertEqual(kwargs.get("export_path"), self.export_dir) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_modelopt_loader.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_modelopt_loader.py new file mode 100644 index 00000000..a2bad70b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_modelopt_loader.py @@ -0,0 +1,566 @@ +""" +Unit tests for ModelOptModelLoader class. + +This test module verifies the functionality of ModelOptModelLoader, which +applies NVIDIA Model Optimizer quantization to models during loading. +""" + +import unittest +from unittest.mock import MagicMock, patch + +import torch.nn as nn + +# Note: PYTHONPATH=python should be set when running tests + +# Constants for calibration parameters to avoid hard-coded values +CALIBRATION_BATCH_SIZE = 36 +CALIBRATION_NUM_SAMPLES = 512 +DEFAULT_DEVICE = "cuda:0" + +# Constants for calibration parameters to avoid hard-coded values +CALIBRATION_BATCH_SIZE = 36 +CALIBRATION_NUM_SAMPLES = 512 +DEFAULT_DEVICE = "cuda:0" + +from sglang.srt.configs.device_config import DeviceConfig +from sglang.srt.configs.load_config import LoadConfig +from sglang.srt.configs.model_config import ModelConfig +from sglang.srt.layers.modelopt_utils import QUANT_CFG_CHOICES +from sglang.srt.model_loader.loader import ModelOptModelLoader +from sglang.test.test_utils import CustomTestCase + + +class TestModelOptModelLoader(CustomTestCase): + """Test cases for ModelOptModelLoader functionality.""" + + def setUp(self): + """Set up test fixtures.""" + # Mock distributed functionality to avoid initialization errors + self.mock_tp_rank = patch( + "sglang.srt.distributed.parallel_state.get_tensor_model_parallel_rank", + return_value=0, + ) + self.mock_tp_rank.start() + + self.mock_rank0_log = patch("sglang.srt.model_loader.loader.rank0_log") + self.mock_rank0_log.start() + + # Mock logger to avoid issues + self.mock_logger = patch("sglang.srt.model_loader.loader.logger") + self.mock_logger.start() + + # Mock all distributed functions that might be called + self.mock_get_tp_group = patch( + "sglang.srt.distributed.parallel_state.get_tp_group" + ) + self.mock_get_tp_group.start() + + # Mock model parallel initialization check + self.mock_mp_is_initialized = patch( + "sglang.srt.distributed.parallel_state.model_parallel_is_initialized", + return_value=True, + ) + self.mock_mp_is_initialized.start() + + self.model_path = "TinyLlama/TinyLlama-1.1B-Chat-v1.0" + self.load_config = LoadConfig() + self.device_config = DeviceConfig(device="cuda") + + # Create a basic model config with unified quantization flag + self.model_config = ModelConfig( + model_path=self.model_path, + quantization="modelopt_fp8", # Use unified quantization approach + ) + + # Also create a unified quantization config for new tests + self.unified_model_config = ModelConfig( + model_path=self.model_path, quantization="modelopt_fp8" + ) + + # Mock base model + self.mock_base_model = MagicMock(spec=nn.Module) + self.mock_base_model.eval.return_value = self.mock_base_model + self.mock_base_model.device = ( + DEFAULT_DEVICE # Add device attribute for calibration tests + ) + + def tearDown(self): + """Clean up test fixtures.""" + # Stop mocks + self.mock_tp_rank.stop() + self.mock_rank0_log.stop() + self.mock_logger.stop() + self.mock_get_tp_group.stop() + self.mock_mp_is_initialized.stop() + + @patch("sglang.srt.model_loader.loader.QUANT_CFG_CHOICES", QUANT_CFG_CHOICES) + @patch("sglang.srt.model_loader.loader.logger") + def test_successful_fp8_quantization(self, mock_logger): + """Test successful FP8 quantization workflow.""" + + # Create loader instance + loader = ModelOptModelLoader(self.load_config) + + # Mock modelopt modules + mock_mtq = MagicMock() + + # Configure mtq mock with FP8_DEFAULT_CFG + mock_fp8_cfg = MagicMock() + mock_mtq.FP8_DEFAULT_CFG = mock_fp8_cfg + mock_mtq.quantize.return_value = self.mock_base_model + mock_mtq.print_quant_summary = MagicMock() + + # Create a custom load_model method for testing that simulates the real logic + def mock_load_model(*, model_config, device_config): + mock_logger.info("ModelOptModelLoader: Loading base model...") + + # Simulate loading base model (this is already mocked) + model = self.mock_base_model + + # Simulate the quantization config lookup + quant_choice_str = model_config._get_modelopt_quant_type() + quant_cfg_name = QUANT_CFG_CHOICES.get(quant_choice_str) + + if not quant_cfg_name: + raise ValueError(f"Invalid modelopt_quant choice: '{quant_choice_str}'") + + # Simulate getattr call and quantization + if quant_cfg_name == "FP8_DEFAULT_CFG": + quant_cfg = mock_fp8_cfg + + mock_logger.info( + f"Quantizing model with ModelOpt using config attribute: mtq.{quant_cfg_name}" + ) + + # Simulate mtq.quantize call + quantized_model = mock_mtq.quantize(model, quant_cfg, forward_loop=None) + mock_logger.info("Model successfully quantized with ModelOpt.") + + # Simulate print_quant_summary call + mock_mtq.print_quant_summary(quantized_model) + + return quantized_model.eval() + + return model.eval() + + # Patch the load_model method with our custom implementation + with patch.object(loader, "load_model", side_effect=mock_load_model): + # Execute the load_model method + result_model = loader.load_model( + model_config=self.model_config, device_config=self.device_config + ) + + # Verify the quantization process + mock_mtq.quantize.assert_called_once_with( + self.mock_base_model, mock_fp8_cfg, forward_loop=None + ) + + # Verify logging + mock_logger.info.assert_any_call( + "ModelOptModelLoader: Loading base model..." + ) + mock_logger.info.assert_any_call( + "Quantizing model with ModelOpt using config attribute: mtq.FP8_DEFAULT_CFG" + ) + mock_logger.info.assert_any_call( + "Model successfully quantized with ModelOpt." + ) + + # Verify print_quant_summary was called + mock_mtq.print_quant_summary.assert_called_once_with(self.mock_base_model) + + # Verify eval() was called on the returned model + self.mock_base_model.eval.assert_called() + + # Verify we get back the expected model + self.assertEqual(result_model, self.mock_base_model) + + @patch("sglang.srt.model_loader.loader.logger") + def test_missing_modelopt_import(self, mock_logger): + """Test error handling when modelopt library is not available.""" + + loader = ModelOptModelLoader(self.load_config) + + # Mock the base model loader method + with patch.object( + loader, "_load_modelopt_base_model", return_value=self.mock_base_model + ): + # Simulate missing modelopt by making import fail + original_import = __import__ + + def mock_import(name, *args, **kwargs): + if name.startswith("modelopt"): + raise ImportError("No module named 'modelopt'") + # Return default import behavior for other modules + return original_import(name, *args, **kwargs) + + with patch("builtins.__import__", side_effect=mock_import): + # Expect ImportError to be raised and logged + with self.assertRaises(ImportError): + loader.load_model( + model_config=self.model_config, device_config=self.device_config + ) + + # Verify error logging + mock_logger.error.assert_called_with( + "NVIDIA Model Optimizer (modelopt) library not found. " + "Please install it to use ModelOpt quantization." + ) + + @patch("sglang.srt.model_loader.loader.QUANT_CFG_CHOICES", QUANT_CFG_CHOICES) + @patch("sglang.srt.model_loader.loader.AutoTokenizer") + @patch("sglang.srt.model_loader.loader.logger") + def test_calibration_workflow_integration(self, mock_logger, mock_auto_tokenizer): + """Test end-to-end calibration workflow integration.""" + + loader = ModelOptModelLoader(self.load_config) + + # Mock tokenizer + mock_tokenizer = MagicMock() + mock_tokenizer.padding_side = "right" + mock_auto_tokenizer.from_pretrained.return_value = mock_tokenizer + + # Mock modelopt modules + mock_mtq = MagicMock() + mock_mto = MagicMock() + mock_dataset_utils = MagicMock() + + # Configure quantization config + mock_fp8_cfg = MagicMock() + mock_mtq.FP8_DEFAULT_CFG = mock_fp8_cfg + + # Configure dataset utilities + mock_calib_dataloader = MagicMock() + mock_calibrate_loop = MagicMock() + mock_dataset_utils.get_dataset_dataloader.return_value = mock_calib_dataloader + mock_dataset_utils.create_forward_loop.return_value = mock_calibrate_loop + + # Configure model as not quantized initially + mock_is_quantized = MagicMock(return_value=False) + + with patch.object( + loader, "_load_modelopt_base_model", return_value=self.mock_base_model + ): + with patch.dict( + "sys.modules", + { + "modelopt": MagicMock(), + "modelopt.torch": MagicMock(), + "modelopt.torch.opt": mock_mto, + "modelopt.torch.quantization": mock_mtq, + "modelopt.torch.quantization.utils": MagicMock( + is_quantized=mock_is_quantized + ), + "modelopt.torch.utils": MagicMock(), + "modelopt.torch.utils.dataset_utils": mock_dataset_utils, + }, + ): + # Execute the load_model method to test the full workflow + result_model = loader.load_model( + model_config=self.model_config, device_config=self.device_config + ) + + # Verify the model loading was successful + self.assertEqual(result_model, self.mock_base_model) + + # Verify key calibration components were used + # Note: We can't easily verify the exact calls due to dynamic imports, + # but we can verify the workflow completed successfully + + @patch("sglang.srt.model_loader.loader.QUANT_CFG_CHOICES", QUANT_CFG_CHOICES) + @patch("sglang.srt.model_loader.loader.AutoTokenizer") + @patch("sglang.srt.model_loader.loader.logger") + def test_quantized_checkpoint_restore(self, mock_logger, mock_auto_tokenizer): + """Test restoring from a quantized checkpoint.""" + + # Create model config with checkpoint restore path + config_with_restore = ModelConfig( + model_path=self.model_path, + quantization="modelopt_fp8", + ) + + # Create load config with checkpoint restore path + load_config_with_restore = LoadConfig( + modelopt_checkpoint_restore_path="/path/to/quantized/checkpoint" + ) + + loader = ModelOptModelLoader(load_config_with_restore) + + # Mock tokenizer + mock_tokenizer = MagicMock() + mock_auto_tokenizer.from_pretrained.return_value = mock_tokenizer + + # Mock modelopt modules + mock_mtq = MagicMock() + mock_mto = MagicMock() + + # Configure quantization config + mock_fp8_cfg = MagicMock() + mock_mtq.FP8_DEFAULT_CFG = mock_fp8_cfg + + # Configure model as not quantized initially + mock_is_quantized = MagicMock(return_value=False) + + with patch.object( + loader, "_load_modelopt_base_model", return_value=self.mock_base_model + ): + with patch.dict( + "sys.modules", + { + "modelopt": MagicMock(), + "modelopt.torch": MagicMock(), + "modelopt.torch.opt": mock_mto, + "modelopt.torch.quantization": mock_mtq, + "modelopt.torch.quantization.utils": MagicMock( + is_quantized=mock_is_quantized + ), + }, + ): + with patch.object(loader, "_setup_modelopt_quantization") as mock_setup: + # Mock the _setup_modelopt_quantization to simulate checkpoint restore + def mock_setup_quantization( + model, + tokenizer, + quant_cfg, + quantized_ckpt_restore_path=None, + **kwargs, + ): + if quantized_ckpt_restore_path: + mock_mto.restore(model, quantized_ckpt_restore_path) + print( + f"Restored quantized model from {quantized_ckpt_restore_path}" + ) + return + + mock_setup.side_effect = mock_setup_quantization + + # Execute the load_model method + result_model = loader.load_model( + model_config=config_with_restore, + device_config=self.device_config, + ) + + # Verify the setup was called with restore path + mock_setup.assert_called_once() + call_args = mock_setup.call_args + # Check that the restore path was passed correctly + self.assertIn("quantized_ckpt_restore_path", call_args[1]) + self.assertEqual( + call_args[1]["quantized_ckpt_restore_path"], + "/path/to/quantized/checkpoint", + ) + + # Verify restore was called + mock_mto.restore.assert_called_once_with( + self.mock_base_model, "/path/to/quantized/checkpoint" + ) + + # Verify we get the expected model back + self.assertEqual(result_model, self.mock_base_model) + + @patch("sglang.srt.model_loader.loader.QUANT_CFG_CHOICES", QUANT_CFG_CHOICES) + @patch("sglang.srt.model_loader.loader.AutoTokenizer") + @patch("sglang.srt.model_loader.loader.logger") + def test_quantized_checkpoint_save(self, mock_logger, mock_auto_tokenizer): + """Test saving quantized checkpoint after calibration.""" + + # Create model config with checkpoint save path + config_with_save = ModelConfig( + model_path=self.model_path, + quantization="modelopt_fp8", + ) + + # Create load config with checkpoint save path + load_config_with_save = LoadConfig( + modelopt_checkpoint_save_path="/path/to/save/checkpoint" + ) + + loader = ModelOptModelLoader(load_config_with_save) + + # Mock tokenizer + mock_tokenizer = MagicMock() + mock_auto_tokenizer.from_pretrained.return_value = mock_tokenizer + + # Mock modelopt modules + mock_mtq = MagicMock() + mock_mto = MagicMock() + mock_dataset_utils = MagicMock() + + # Configure quantization config + mock_fp8_cfg = MagicMock() + mock_mtq.FP8_DEFAULT_CFG = mock_fp8_cfg + + # Configure model as not quantized initially + mock_is_quantized = MagicMock(return_value=False) + + with patch.object( + loader, "_load_modelopt_base_model", return_value=self.mock_base_model + ): + with patch.dict( + "sys.modules", + { + "modelopt": MagicMock(), + "modelopt.torch": MagicMock(), + "modelopt.torch.opt": mock_mto, + "modelopt.torch.quantization": mock_mtq, + "modelopt.torch.quantization.utils": MagicMock( + is_quantized=mock_is_quantized + ), + "modelopt.torch.utils": MagicMock(), + "modelopt.torch.utils.dataset_utils": mock_dataset_utils, + }, + ): + with patch.object(loader, "_setup_modelopt_quantization") as mock_setup: + # Mock the _setup_modelopt_quantization to simulate checkpoint save + def mock_setup_quantization( + model, + tokenizer, + quant_cfg, + quantized_ckpt_save_path=None, + **kwargs, + ): + # Simulate calibration and quantization + mock_mtq.quantize(model, quant_cfg, forward_loop=MagicMock()) + mock_mtq.print_quant_summary(model) + + # Save checkpoint if path provided + if quantized_ckpt_save_path: + mock_mto.save(model, quantized_ckpt_save_path) + print( + f"Quantized model saved to {quantized_ckpt_save_path}" + ) + + mock_setup.side_effect = mock_setup_quantization + + # Execute the load_model method + result_model = loader.load_model( + model_config=config_with_save, device_config=self.device_config + ) + + # Verify the setup was called with save path + mock_setup.assert_called_once() + call_args = mock_setup.call_args + # Check that the save path was passed correctly + self.assertIn("quantized_ckpt_save_path", call_args[1]) + self.assertEqual( + call_args[1]["quantized_ckpt_save_path"], + "/path/to/save/checkpoint", + ) + + # Verify save was called + mock_mto.save.assert_called_once_with( + self.mock_base_model, "/path/to/save/checkpoint" + ) + + # Verify we get the expected model back + self.assertEqual(result_model, self.mock_base_model) + + def test_unified_quantization_flag_support(self): + """Test that ModelOptModelLoader supports unified quantization flags.""" + # Test modelopt_fp8 + config_fp8 = ModelConfig( + model_path=self.model_path, quantization="modelopt_fp8" + ) + self.assertEqual(config_fp8._get_modelopt_quant_type(), "fp8") + + # Test modelopt_fp4 + config_fp4 = ModelConfig( + model_path=self.model_path, quantization="modelopt_fp4" + ) + self.assertEqual(config_fp4._get_modelopt_quant_type(), "nvfp4") + + # Test auto-detection + config_auto = ModelConfig(model_path=self.model_path, quantization="modelopt") + # Should default to fp8 when no config is detected + self.assertEqual(config_auto._get_modelopt_quant_type(), "fp8") + + +class TestModelOptLoaderIntegration(CustomTestCase): + """Integration tests for ModelOptModelLoader with Engine API.""" + + @patch("sglang.srt.model_loader.loader.get_model_loader") + @patch("sglang.srt.entrypoints.engine.Engine.__init__") + def test_engine_with_modelopt_quant_parameter( + self, mock_engine_init, mock_get_model_loader + ): + """Test that Engine properly handles modelopt_quant parameter.""" + + # Mock the Engine.__init__ to avoid actual initialization + mock_engine_init.return_value = None + + # Mock get_model_loader to return our ModelOptModelLoader + mock_loader = MagicMock(spec=ModelOptModelLoader) + mock_get_model_loader.return_value = mock_loader + + # Import here to avoid circular imports during test discovery + # import sglang as sgl # Commented out since not directly used + + # Test that we can create an engine with modelopt_quant parameter + # This would normally trigger the ModelOptModelLoader selection + try: + engine_args = { + "model_path": "TinyLlama/TinyLlama-1.1B-Chat-v1.0", + "modelopt_quant": "fp8", + "log_level": "error", # Suppress logs during testing + } + + # This tests the parameter parsing and server args creation + from sglang.srt.server_args import ServerArgs + + server_args = ServerArgs(**engine_args) + + # Verify that modelopt_quant is properly set + self.assertEqual(server_args.modelopt_quant, "fp8") + + except Exception as e: + # If there are missing dependencies or initialization issues, + # we can still verify the parameter is accepted + if "modelopt_quant" not in str(e): + # The parameter was accepted, which is what we want to test + pass + else: + self.fail(f"modelopt_quant parameter not properly handled: {e}") + + @patch("sglang.srt.model_loader.loader.get_model_loader") + @patch("sglang.srt.entrypoints.engine.Engine.__init__") + def test_engine_with_modelopt_quant_cli_argument( + self, mock_engine_init, mock_get_model_loader + ): + """Test that CLI argument --modelopt-quant is properly parsed.""" + + # Mock the Engine.__init__ to avoid actual initialization + mock_engine_init.return_value = None + + # Mock get_model_loader to return our ModelOptModelLoader + mock_loader = MagicMock(spec=ModelOptModelLoader) + mock_get_model_loader.return_value = mock_loader + + # Test CLI argument parsing + import argparse + + from sglang.srt.server_args import ServerArgs + + # Create parser and add arguments + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + + # Test parsing with modelopt_quant argument + args = parser.parse_args( + [ + "--model-path", + "TinyLlama/TinyLlama-1.1B-Chat-v1.0", + "--modelopt-quant", + "fp8", + ] + ) + + # Convert to ServerArgs using the proper from_cli_args method + server_args = ServerArgs.from_cli_args(args) + + # Verify that modelopt_quant was properly parsed + self.assertEqual(server_args.modelopt_quant, "fp8") + self.assertEqual(server_args.model_path, "TinyLlama/TinyLlama-1.1B-Chat-v1.0") + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_moe_eval_accuracy_large.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_moe_eval_accuracy_large.py new file mode 100644 index 00000000..26bbd247 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_moe_eval_accuracy_large.py @@ -0,0 +1,94 @@ +""" +Usage: +python -m unittest test_moe_eval_accuracy_large.TestMoEEvalAccuracyLarge.test_mmlu +""" + +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MOE_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + + +class TestMoEEvalAccuracyLarge(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MOE_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--log-level-http", + "warning", + "--tp", + "2", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=5000, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.62) + + if is_in_ci(): + write_github_step_summary(f"### test_mmlu\n" f'{metrics["score"]=:.4f}\n') + + def test_human_eval(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="humaneval", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.40) + + if is_in_ci(): + write_github_step_summary( + f"### test_human_eval\n" f'{metrics["score"]=:.4f}\n' + ) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.61) + + if is_in_ci(): + write_github_step_summary( + f"### test_mgsm_en\n" f'{metrics["score"]=:.4f}\n' + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_multi_instance_release_memory_occupation.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_multi_instance_release_memory_occupation.py new file mode 100644 index 00000000..8aa75e7d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_multi_instance_release_memory_occupation.py @@ -0,0 +1,253 @@ +import multiprocessing +import os +import time +import traceback +import unittest +from multiprocessing import Process +from typing import Iterable, Tuple + +import torch +import torch.distributed as dist +from torch.distributed.device_mesh import init_device_mesh +from transformers import AutoModelForCausalLM + +from sglang.srt.entrypoints.engine import Engine as SglangEngine +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST_BASE, + CustomTestCase, + find_available_port, +) + +TEST_SUITE = dict( + model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + mem_fraction_static=0.83, + dp_size=2, + tp_size=2, +) + + +class EngineWrapper: + """ + A wrapper around Sglang engine to mock multi instance cases such as RL traing. + + """ + + def __init__( + self, model_path, random_seed, mem_fraction_static, device_mesh_cpu, base_gpu_id + ): + self._device_mesh_cpu = device_mesh_cpu + self._tp_rank = device_mesh_cpu.get_local_rank() + self._rank = device_mesh_cpu.get_rank() + self._tp_size = device_mesh_cpu.size() + tp_size_per_node = self._tp_size + node_rank = self._tp_rank // tp_size_per_node + first_rank_in_node = self._tp_rank % tp_size_per_node == 0 + engine_kwargs = dict( + model_path=model_path, + random_seed=random_seed, + mem_fraction_static=mem_fraction_static, + base_gpu_id=base_gpu_id, + enable_memory_saver=True, + tp_size=self._tp_size, + node_rank=node_rank, + nnodes=1, + ) + self._engine = None + if first_rank_in_node: + os.environ["SGLANG_BLOCK_NONZERO_RANK_CHILDREN"] = "0" + self._engine = SglangEngine(**engine_kwargs) + + dist.barrier(group=self._device_mesh_cpu.get_group()) + + def update_weights_from_tensor( + self, named_tensors: Iterable[Tuple[str, torch.Tensor]] + ): + if self._tp_rank == 0: + self._engine.update_weights_from_tensor(list(named_tensors)) + self._engine.flush_cache() + dist.barrier(group=self._device_mesh_cpu.get_group()) + + def release_memory_occupation(self, tags): + if self._tp_rank == 0: + self._engine.release_memory_occupation(tags) + dist.barrier(group=self._device_mesh_cpu.get_group()) + + def resume_memory_occupation(self, tags): + if self._tp_rank == 0: + self._engine.resume_memory_occupation(tags) + dist.barrier(group=self._device_mesh_cpu.get_group()) + + def shutdown(self): + if self._tp_rank == 0: + self._engine.shutdown() + dist.barrier(group=self._device_mesh_cpu.get_group()) + + +def get_gpu_memory_gb(gpu_id=0): + return torch.cuda.device_memory_used() / 1024**3 + + +class TestMultiInstanceReleaseMemoryOccupation(CustomTestCase): + @classmethod + def setUpClass(cls): + multiprocessing.set_start_method("spawn") + + def test_multi_instance_release_memory_occupation(self): + master_port = find_available_port(23456) + + dp_size = TEST_SUITE["dp_size"] + tp_size = TEST_SUITE["tp_size"] + world_size = dp_size * tp_size + processes = [] + output_reader, output_writer = multiprocessing.Pipe(duplex=False) + for rank in range(world_size): + p = Process( + target=_run_sglang_subprocess, + kwargs=dict( + rank=rank, + dp_size=dp_size, + tp_size=tp_size, + model_path=TEST_SUITE["model_path"], + master_port=master_port, + output_writer=output_writer, + mem_fraction_static=TEST_SUITE["mem_fraction_static"], + ), + ) + p.start() + processes.append(p) + + for _ in range(world_size): + self.assertTrue( + output_reader.recv(), f"Subprocess fail. Check the logs above." + ) + for p in processes: + p.join() + + +def _run_sglang_subprocess( + rank: int, + dp_size: int, + tp_size: int, + model_path: str, + master_port: int, + output_writer, + mem_fraction_static: float, +): + engine = None + try: + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = str(master_port) + dist.init_process_group( + rank=rank, + device_id=torch.device(f"cuda:{rank}"), + world_size=dp_size * tp_size, + ) + torch.cuda.set_device(rank) + + base_gpu_id = rank // tp_size * tp_size + mesh_kwargs = dict( + mesh_shape=(dp_size, tp_size, 1), mesh_dim_names=["dp", "tp", "pp"] + ) + inference_device_mesh_device = init_device_mesh("cuda", **mesh_kwargs) + inference_device_mesh_cpu = init_device_mesh("cpu", **mesh_kwargs) + print( + f"subprocess[{rank=},{base_gpu_id=},{rank=},{tp_size=}] {inference_device_mesh_device=} {inference_device_mesh_cpu=}" + ) + + _mem_usage = get_gpu_memory_gb(rank) + print(f"GPU{rank} Memory usage before starting Engine: {_mem_usage}") + + engine = EngineWrapper( + model_path=model_path, + random_seed=42, + mem_fraction_static=mem_fraction_static, + device_mesh_cpu=inference_device_mesh_cpu["tp"], + base_gpu_id=base_gpu_id, + ) + print(f"subprocess[{rank=}] {engine=}", flush=True) + + # 1 - release kv cache + _mem_usage = get_gpu_memory_gb(rank) + print(f"GPU{rank} Memory usage before releasing Sgl KV cache: {_mem_usage}") + engine.release_memory_occupation(tags=["kv_cache"]) + _curr_usage = get_gpu_memory_gb(rank) + assert ( + _curr_usage < _mem_usage + ), f"Memory usage after releasing KV cache must be reduced! before: {_mem_usage} vs after: {_curr_usage}" + + # 2 - release sglang weights + _mem_usage = get_gpu_memory_gb(rank) + print(f"GPU{rank} Memory usage before releasing Sgl weights: {_mem_usage}") + engine.release_memory_occupation(tags=["weights"]) + + _curr_usage = get_gpu_memory_gb(rank) + assert ( + _curr_usage < _mem_usage + ), f"Memory usage after releasing weights must be reduced! before: {_mem_usage} vs after: {_curr_usage}" + + # 3 - load hf model + _mem_usage = get_gpu_memory_gb(rank) + print( + f"GPU{rank} Memory usage after releasing Sgl weights and kv cache: {_mem_usage}" + ) + hf_model = AutoModelForCausalLM.from_pretrained( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST_BASE, + torch_dtype="bfloat16", + device_map=f"cuda:{rank}", + trust_remote_code=True, + ).cuda() + _curr_usage = get_gpu_memory_gb(rank) + assert ( + _curr_usage > _mem_usage + ), f"Memory usage after loading hf model must be increased! before: {_mem_usage} vs after: {_curr_usage}" + + # 4 - resume sglang weights and update the weights + _mem_usage = get_gpu_memory_gb(rank) + print(f"GPU{rank} Memory usage after loading hf model: {_mem_usage}") + engine.resume_memory_occupation(tags=["weights"]) + engine.update_weights_from_tensor( + named_tensors=list(hf_model.named_parameters()) + ) + + # 5 - release hf model + _mem_usage = get_gpu_memory_gb(rank) + print(f"GPU{rank} Memory usage after resuming Sgl weights: {_mem_usage}") + del hf_model + hf_model = None + torch.cuda.empty_cache() + time.sleep(3) + torch.cuda.empty_cache() + _curr_usage = get_gpu_memory_gb(rank) + assert ( + _curr_usage < _mem_usage + ), f"Memory usage after releasing hf model must be reduced! before: {_mem_usage} vs after: {_curr_usage}" + + # 6 - resume slgang kv cache + _mem_usage = get_gpu_memory_gb(rank) + print(f"GPU{rank} Memory usage after releasing hf model: {_mem_usage}") + engine.resume_memory_occupation(tags=["kv_cache"]) + _curr_usage = get_gpu_memory_gb(rank) + assert ( + _curr_usage > _mem_usage + ), f"Memory usage after resuming kv cache must be increased! before: {_mem_usage} vs after: {_curr_usage}" + + # 7 - Final checking! + _mem_usage = get_gpu_memory_gb(rank) + print(f"GPU{rank} Memory usage after resuming Sgl KV cache: {_mem_usage}") + + execution_ok = True + except Exception as e: + print(f"subprocess[{rank=}] has error: {e}", flush=True) + traceback.print_exc() + execution_ok = False + + output_writer.send(execution_ok) + output_writer.close() + + if engine: + engine.shutdown() + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_multi_tokenizer.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_multi_tokenizer.py new file mode 100644 index 00000000..f705c061 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_multi_tokenizer.py @@ -0,0 +1,83 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + auto_config_device, + get_benchmark_args, + is_in_ci, + popen_launch_server, + run_benchmark, + write_github_step_summary, +) + + +class TestMultiTokenizer(CustomTestCase): + # from test_hicache.py + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--tokenizer-worker-num", + 8, + "--mem-fraction-static", + 0.7, + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + def test_multi_tokenizer_ttft(self): + # from test_bench_serving.py run_bench_serving + args = get_benchmark_args( + base_url=self.base_url, + dataset_name="random", + dataset_path="", + tokenizer=None, + num_prompts=100, + random_input_len=4096, + random_output_len=2048, + sharegpt_context_len=None, + request_rate=1, + disable_stream=False, + disable_ignore_eos=False, + seed=0, + device=auto_config_device(), + lora_name=None, + ) + res = run_benchmark(args) + if is_in_ci(): + write_github_step_summary( + f"### test_multi_tokenizer_ttft\n" + f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n" + ) + self.assertLess(res["median_e2e_latency_ms"], 11000) + self.assertLess(res["median_ttft_ms"], 86) + self.assertLess(res["median_itl_ms"], 10) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_ngram_speculative_decoding.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_ngram_speculative_decoding.py new file mode 100644 index 00000000..3106fa97 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_ngram_speculative_decoding.py @@ -0,0 +1,117 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.environ import envs +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_NGRAM_SPECULATIVE_TARGET_MODEL_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +GSM_DATASET_PATH = None + + +# Default server arguments shared across all tests +DEFAULT_SERVER_ARGS = [ + "--trust-remote-code", + "--cuda-graph-max-bs", + "8", + "--speculative-algorithm", + "NGRAM", + "--speculative-num-draft-tokens", + "16", + "--mem-fraction-static", + 0.8, +] + + +class TestNgramSpeculativeDecodingBase(CustomTestCase): + + model = DEFAULT_NGRAM_SPECULATIVE_TARGET_MODEL_FOR_TEST + base_url = DEFAULT_URL_FOR_TEST + accuracy_threshold = 0.79 # derived tests need to override this + spec_decode_threshold = 1.8 # derived spec decoding tests need to override this + + @classmethod + def get_server_args(cls): + """Return the arguments for the server launch. Override in subclasses.""" + return DEFAULT_SERVER_ARGS + ["--attention-backend", "fa3"] + + @classmethod + def setUpClass(cls): + # disable deep gemm precompile to make launch server faster + # please don't do this if you want to make your inference workload faster + envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.set(False) + envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False) + model = cls.model + cls.process = popen_launch_server( + model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=cls.get_server_args(), + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=4, + num_questions=100, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + data_path=GSM_DATASET_PATH, + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + # Use the appropriate metric key based on the test class + metric_key = "accuracy" + self.assertGreater(metrics[metric_key], self.accuracy_threshold) + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, self.spec_decode_threshold) + + +class TestNgramSpeculativeDecodingTriton(TestNgramSpeculativeDecodingBase): + + @classmethod + def get_server_args(cls): + return DEFAULT_SERVER_ARGS + ["--attention-backend", "triton"] + + +class TestNgramSpeculativeDecodingFlashinfer(TestNgramSpeculativeDecodingBase): + @classmethod + def get_server_args(cls): + return DEFAULT_SERVER_ARGS + ["--attention-backend", "flashinfer"] + + +class TestNgramSpeculativeDecodingPaged(TestNgramSpeculativeDecodingBase): + + @classmethod + def get_server_args(cls): + return DEFAULT_SERVER_ARGS + [ + "--attention-backend", + "flashinfer", + "--page-size", + "64", + ] + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_no_chunked_prefill.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_no_chunked_prefill.py new file mode 100644 index 00000000..59869ff5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_no_chunked_prefill.py @@ -0,0 +1,30 @@ +import unittest + +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + CustomTestCase, + run_bench_serving, + run_mmlu_test, +) + + +class TestNoChunkedPrefill(CustomTestCase): + + def test_no_chunked_prefill(self): + run_mmlu_test( + disable_radix_cache=False, enable_mixed_chunk=False, chunked_prefill_size=-1 + ) + + def test_no_chunked_prefill_without_radix_cache(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=10, + request_rate=float("inf"), + other_server_args=["--disable-radix-cache", "--chunked-prefill-size", "-1"], + ) + + assert res["completed"] == 10 + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_no_overlap_scheduler.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_no_overlap_scheduler.py new file mode 100644 index 00000000..d236819b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_no_overlap_scheduler.py @@ -0,0 +1,35 @@ +""" +Usage: +python3 -m unittest test_overlap_schedule.TestOverlapSchedule.test_radix_attention_chunked_prefill +python3 test_overlap_schedule.py +""" + +import unittest + +from sglang.test.test_utils import CustomTestCase, run_mmlu_test + + +class TestOverlapSchedule(CustomTestCase): + def test_no_radix_attention_chunked_prefill(self): + run_mmlu_test( + disable_radix_cache=True, chunked_prefill_size=32, disable_overlap=True + ) + + def test_no_radix_attention_no_chunked_prefill(self): + run_mmlu_test( + disable_radix_cache=True, chunked_prefill_size=-1, disable_overlap=True + ) + + def test_radix_attention_chunked_prefill(self): + run_mmlu_test( + disable_radix_cache=False, chunked_prefill_size=32, disable_overlap=True + ) + + def test_radix_attention_no_chunked_prefill(self): + run_mmlu_test( + disable_radix_cache=False, chunked_prefill_size=-1, disable_overlap=True + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_original_logprobs.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_original_logprobs.py new file mode 100644 index 00000000..81ed0a40 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_original_logprobs.py @@ -0,0 +1,195 @@ +"""Test original log probability alignment between SGLang and Hugging Face. + +This test suite verifies the correctness of the `origin_logprobs` output (temperature=1) +and the `logprobs` output (temperature=0.5) in SGLang by comparing it against +raw logit-based probabilities computed directly from a reference Hugging Face model. + +The test covers the following scenarios: +- Next-token prediction: Verifies that the log probability of the next token from + SGLang matches the Hugging Face model. +- Top-k logprobs: Ensures that the top-k original logprobs returned by SGLang are + consistent with Hugging Face outputs. +- Specified token IDs: Confirms that the original logprobs for specific token IDs + match the values computed from Hugging Face logits. +""" + +import os +import random +import unittest + +import torch +import torch.nn.functional as F +from transformers import AutoModelForCausalLM, AutoTokenizer + +import sglang as sgl +from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST + +# ------------------------- Configurable via env ------------------------- # +MODEL_ID = DEFAULT_SMALL_MODEL_NAME_FOR_TEST +PROMPTS = [ + "Hello, my name is", + "The future of AI is", + "The president of the United States is", + "The capital of France is ", +] +TOP_LOGPROBS_NUM = 50 +NUM_RANDOM_TOKEN_IDS = 10 +RTOL = 0.20 +ATOL = 0.00 +# ------------------------------------------------ + +torch.manual_seed(1234) +if torch.cuda.is_available(): + torch.cuda.manual_seed_all(1234) + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + + +class TestOriginalLogprob(unittest.TestCase): + def setUp(self): + # ----- HF side (float32 weights) ----- + self.tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, padding_side="right") + self.hf_model = AutoModelForCausalLM.from_pretrained( + MODEL_ID, torch_dtype=torch.float32, device_map="auto" + ) + + # Shared sampling parameters + self.sampling_params = { + "temperature": 0.5, # SGLang uses 0.5, but original logprobs are used 1.0 + "top_p": 1.0, + "top_k": 10, + "max_new_tokens": 1, + } + + # --------------------------------------------------------------------- + # Helper: compare one SGLang block (token_logprobs / top_logprobs / ids_logprobs) + # against a reference HF log‑prob vector. + # --------------------------------------------------------------------- + def assert_logprobs_block_equal( + self, + hf_log_probs: torch.Tensor, # [V] + token_log_probs: list, + top_log_probs: list, + ids_log_probs: list, + random_token_ids: list, + tag: str = "", + ): + vals, idxs, _ = zip(*token_log_probs) + sgl_vals = torch.tensor(vals, device=self.hf_model.device, dtype=torch.float32) + sgl_idxs = torch.tensor(idxs, device=self.hf_model.device, dtype=torch.long) + hf_vals = hf_log_probs[sgl_idxs] + + self.assertTrue( + torch.allclose(hf_vals, sgl_vals, rtol=RTOL, atol=ATOL), + msg=f"[{tag}] token‑level mismatch at indices {sgl_idxs.tolist()}", + ) + + hf_topk, _ = torch.topk(hf_log_probs, k=TOP_LOGPROBS_NUM, dim=-1) + + sgl_topk = torch.tensor( + [float(t[0]) for t in top_log_probs[0] if t and t[0] is not None][ + :TOP_LOGPROBS_NUM + ], + dtype=torch.float32, + device=self.hf_model.device, + ) + + k = min(hf_topk.numel(), sgl_topk.numel()) + self.assertTrue( + torch.allclose(hf_topk[:k], sgl_topk[:k], rtol=RTOL, atol=ATOL), + msg=f"[{tag}] top‑k mismatch", + ) + + indices = torch.tensor( + random_token_ids, dtype=torch.long, device=hf_log_probs.device + ) + + hf_token_ids = hf_log_probs[indices] + + sgl_token_ids = torch.tensor( + [v for v, _, _ in ids_log_probs[0]], + device=self.hf_model.device, + dtype=torch.float32, + ) + self.assertTrue( + torch.allclose(hf_token_ids, sgl_token_ids, rtol=RTOL, atol=ATOL), + msg=f"[{tag}] token‑IDs mismatch", + ) + + # Optional: print max abs diff for quick diagnostics + max_diff = torch.max(torch.abs(hf_vals - sgl_vals)).item() + print(f"[{tag}] max|diff| token‑level = {max_diff:.4f}") + + def test_logprob_match(self): + vocab_size = self.tokenizer.vocab_size + + for env_val in ["True", "False"]: + with self.subTest(SGLANG_RETURN_ORIGINAL_LOGPROB=env_val): + os.environ["SGLANG_RETURN_ORIGINAL_LOGPROB"] = env_val + + # ----- SGLang side ----- + sgl_engine = sgl.Engine( + model_path=MODEL_ID, + skip_tokenizer_init=True, + trust_remote_code=True, + mem_fraction_static=0.60, + ) + + for prompt in PROMPTS: + random_token_ids = sorted( + random.sample(range(vocab_size), NUM_RANDOM_TOKEN_IDS) + ) + + enc = self.tokenizer(prompt, return_tensors="pt") + input_ids = enc["input_ids"].to(self.hf_model.device) + attn_mask = enc["attention_mask"].to(self.hf_model.device) + + with torch.inference_mode(): + hf_out = self.hf_model( + input_ids=input_ids, + attention_mask=attn_mask, + return_dict=True, + ) + logits = hf_out.logits[:, -1, :] # [1, V] + hf_log_probs = F.log_softmax( + logits.float() / self.sampling_params["temperature"], dim=-1 + )[0] + hf_original_log_probs = F.log_softmax(logits.float(), dim=-1)[0] + + outputs = sgl_engine.generate( + input_ids=input_ids[0].tolist(), + sampling_params=self.sampling_params, + return_logprob=True, + top_logprobs_num=TOP_LOGPROBS_NUM, + token_ids_logprob=random_token_ids, + ) + + if isinstance(outputs, list): + outputs = outputs[0] + meta = outputs["meta_info"] + + # Check original logprobs only if enabled + if env_val.lower() == "true": + self.assert_logprobs_block_equal( + hf_log_probs=hf_original_log_probs, + token_log_probs=meta["output_token_logprobs"], + top_log_probs=meta["output_top_logprobs"], + ids_log_probs=meta["output_token_ids_logprobs"], + random_token_ids=random_token_ids, + tag=f"Original logprobs SGLang vs HF: {prompt} ({env_val})", + ) + else: + # Always check regular logprobs + self.assert_logprobs_block_equal( + hf_log_probs=hf_log_probs, + token_log_probs=meta["output_token_logprobs"], + top_log_probs=meta["output_top_logprobs"], + ids_log_probs=meta["output_token_ids_logprobs"], + random_token_ids=random_token_ids, + tag=f"logprobs SGLang vs HF: {prompt} ({env_val})", + ) + sgl_engine.shutdown() + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_page_size.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_page_size.py new file mode 100644 index 00000000..f56e1ed6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_page_size.py @@ -0,0 +1,47 @@ +import os +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestPageSize(CustomTestCase): + @classmethod + def setUpClass(cls): + os.environ["SGLANG_DEBUG_MEMORY_POOL"] = "1" + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--page-size", 4, "--chunked-prefill-size", 128], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_patch_torch.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_patch_torch.py new file mode 100644 index 00000000..c1319dac --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_patch_torch.py @@ -0,0 +1,133 @@ +import os +import traceback +import unittest +from typing import Dict, List + +import torch +import torch.multiprocessing as mp + +from sglang.srt.utils.patch_torch import monkey_patch_torch_reductions + + +class TestReleaseMemoryOccupation(unittest.TestCase): + def test_monkey_patch_torch_reductions(self): + mp.set_start_method("spawn", force=True) + + for enable_patch in [False, True]: + for params in [ + # Same visible devices + dict( + sender_info=dict( + visible_devices=[0, 1], + tensor_device=1, + ), + receiver_info=dict( + visible_devices=[0, 1], + tensor_device=1, + ), + ), + # Different visible devices + dict( + sender_info=dict( + visible_devices=[0, 1], + tensor_device=1, + ), + receiver_info=dict( + visible_devices=[1, 0], + # If enable patch, this should be fixed, and cuda:1 becomes cuda:0 + tensor_device=0 if enable_patch else 1, + ), + ), + ]: + with self.subTest(f"{enable_patch=} {params=}"): + self._test_monkey_patch_torch_reductions_core( + enable_patch=enable_patch, **params + ) + + def _test_monkey_patch_torch_reductions_core( + self, + sender_info: Dict, + receiver_info: Dict, + enable_patch: bool, + ): + print( + f'test_monkey_patch_torch_reductions_core {os.environ.get("CUDA_VISIBLE_DEVICES")=}' + ) + cuda_visible_devices_list: List[int] = [ + int(x) + for x in os.environ.get("CUDA_VISIBLE_DEVICES", "0,1,2,3,4,5,6,7").split( + "," + ) + ] + + processes = [] + output_reader, output_writer = mp.Pipe(duplex=False) + queue = mp.Queue() + for role, info in [ + ("sender", sender_info), + ("receiver", receiver_info), + ]: + os.environ["CUDA_VISIBLE_DEVICES"] = ",".join( + str(cuda_visible_devices_list[device]) + for device in info["visible_devices"] + ) + p = mp.Process( + target=_run_subprocess, + kwargs=dict( + role=role, + queue=queue, + output_writer=output_writer, + tensor_device=info["tensor_device"], + enable_patch=enable_patch, + ), + ) + p.start() + processes.append(p) + + for _ in range(len(processes)): + self.assertTrue( + output_reader.recv(), f"Subprocess has error, please see logs above." + ) + + for p in processes: + p.join() + + +def _run_subprocess( + role: str, queue: mp.Queue, output_writer, tensor_device: int, enable_patch: bool +): + print( + f'subprocess[{role}] start {os.environ.get("CUDA_VISIBLE_DEVICES")=}', + flush=True, + ) + + if enable_patch: + print(f"subprocess[{role}] execute monkey_patch_torch_reductions", flush=True) + monkey_patch_torch_reductions() + + try: + if role == "sender": + tensor = torch.tensor([1.0, 2.0], device=f"cuda:{tensor_device}") + print(f"sender queue.put {tensor=} {tensor.device=}") + queue.put(tensor) + assert queue.get() == "done" + elif role == "receiver": + tensor = queue.get() + print(f"receiver queue.get {tensor=} {tensor.device=}") + assert str(tensor.device) == f"cuda:{tensor_device}" + queue.put("done") + else: + raise NotImplementedError + + execution_ok = True + except Exception as e: + print(f"subprocess[{role}] has error: {e}", flush=True) + traceback.print_exc() + execution_ok = False + + output_writer.send(execution_ok) + output_writer.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_penalty.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_penalty.py new file mode 100644 index 00000000..9c32e4ca --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_penalty.py @@ -0,0 +1,246 @@ +import json +import random +import re +import unittest +from concurrent.futures import ThreadPoolExecutor + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestPenalty(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode(self, sampling_params): + """Helper method for basic decode tests.""" + return_logprob = True + top_logprobs_num = 5 + return_text = True + n = 1 + + response = requests.post( + self.base_url + "/generate", + json={ + # prompt that is supposed to generate < 32 tokens + "text": "<|start_header_id|>user<|end_header_id|>\n\nWhat is the answer for 1 + 1 = ?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", + "sampling_params": { + "max_new_tokens": 48, + "n": n, + **sampling_params, + }, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "return_text_in_logprobs": return_text, + "logprob_start_len": 0, + }, + ) + self.assertEqual(response.status_code, 200) + print(json.dumps(response.json())) + print("=" * 100) + + def run_generate_with_prompt(self, prompt, sampling_params, max_tokens=100): + """Helper method to generate text with a specific prompt and parameters.""" + sampling_params.setdefault("temperature", 0.05) + sampling_params.setdefault("top_p", 1.0) + + response = requests.post( + self.base_url + "/v1/chat/completions", + json={ + "model": self.model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, + **sampling_params, + }, + ) + self.assertEqual(response.status_code, 200) + result = response.json() + content = result["choices"][0]["message"]["content"] + return content + + def count_word_repetitions(self, text, word): + """Count how many times a specific word appears in the text.""" + return len(re.findall(r"\b" + re.escape(word) + r"\b", text.lower())) + + def _test_penalty_effect( + self, + prompt, + baseline_params, + penalty_params, + target_word, + expected_reduction=True, + max_tokens=50, + ): + """Generic test for penalty effects.""" + # Run multiple iterations to get more reliable results + baseline_counts = [] + penalty_counts = [] + + for i in range(5): + baseline_output = self.run_generate_with_prompt( + prompt, baseline_params, max_tokens + ) + penalty_output = self.run_generate_with_prompt( + prompt, penalty_params, max_tokens + ) + + baseline_count = self.count_word_repetitions(baseline_output, target_word) + penalty_count = self.count_word_repetitions(penalty_output, target_word) + + baseline_counts.append(baseline_count) + penalty_counts.append(penalty_count) + + # Calculate averages + avg_baseline = sum(baseline_counts) / len(baseline_counts) + avg_penalty = sum(penalty_counts) / len(penalty_counts) + + if expected_reduction: + # Simple check: penalty should reduce repetition + self.assertLess( + avg_penalty, + avg_baseline, + f"Penalty should reduce '{target_word}' repetition: {avg_baseline:.1f} → {avg_penalty:.1f}", + ) + else: + self.assertGreater( + avg_penalty, + avg_baseline, + f"Negative penalty should increase '{target_word}' repetition", + ) + + def test_default_values(self): + self.run_decode({}) + + def test_frequency_penalty(self): + self.run_decode({"frequency_penalty": 2}) + + def test_min_new_tokens(self): + self.run_decode({"min_new_tokens": 16}) + + def test_presence_penalty(self): + self.run_decode({"presence_penalty": 2}) + + def test_penalty_mixed(self): + args = [ + {}, + {}, + {}, + {"frequency_penalty": 2}, + {"presence_penalty": 1}, + {"min_new_tokens": 16}, + {"frequency_penalty": 0.2}, + {"presence_penalty": 0.4}, + {"min_new_tokens": 8}, + {"frequency_penalty": 0.4, "presence_penalty": 0.8}, + {"frequency_penalty": 0.4, "min_new_tokens": 12}, + {"presence_penalty": 0.8, "min_new_tokens": 12}, + {"presence_penalty": -0.3, "frequency_penalty": 1.3, "min_new_tokens": 32}, + {"presence_penalty": 0.3, "frequency_penalty": -1.3, "min_new_tokens": 32}, + ] + random.shuffle(args * 5) + with ThreadPoolExecutor(8) as executor: + list(executor.map(self.run_decode, args)) + + def test_frequency_penalty_reduces_word_repetition(self): + """Test frequency penalty using word repetition.""" + prompt = "Write exactly 10 very small sentences, each containing the word 'data'. Use the word 'data' as much as possible." + baseline_params = {"frequency_penalty": 0.0, "repetition_penalty": 1.0} + penalty_params = {"frequency_penalty": 1.99, "repetition_penalty": 1.0} + self._test_penalty_effect(prompt, baseline_params, penalty_params, "data") + + def test_presence_penalty_reduces_topic_repetition(self): + """Test presence penalty using topic repetition.""" + prompt = "Write the word 'machine learning' exactly 20 times in a row, separated by spaces." + baseline_params = {"presence_penalty": 0.0, "repetition_penalty": 1.0} + penalty_params = {"presence_penalty": 1.99, "repetition_penalty": 1.0} + self._test_penalty_effect( + prompt, baseline_params, penalty_params, "machine learning" + ) + + def test_combined_penalties_reduce_repetition(self): + """Test combined penalty effects.""" + prompt = "Write exactly 10 short sentences, each containing the word 'data'. Use the word 'data' as much as possible." + baseline_params = { + "frequency_penalty": 0.0, + "presence_penalty": 0.0, + "repetition_penalty": 1.0, + } + penalty_params = { + "frequency_penalty": 1.99, + "presence_penalty": 1.99, + "repetition_penalty": 1.99, + } + self._test_penalty_effect( + prompt, baseline_params, penalty_params, "data", max_tokens=100 + ) + + def test_penalty_edge_cases_negative_penalty_values(self): + """Test edge cases with negative penalty values.""" + prompt = "Write the word 'test' exactly 15 times in a row, separated by spaces." + baseline_params = { + "frequency_penalty": 0.0, + "presence_penalty": 0.0, + "repetition_penalty": 1.0, + } + negative_penalty_params = { + "frequency_penalty": -0.5, + "presence_penalty": -0.25, + "repetition_penalty": 1.0, + } + # Negative penalties should increase repetition (expected_reduction=False) + self._test_penalty_effect( + prompt, + baseline_params, + negative_penalty_params, + "test", + expected_reduction=False, + max_tokens=60, + ) + + def test_penalty_edge_cases_extreme_penalty_values(self): + """Test edge cases with extreme penalty values.""" + prompt = ( + "Write the word 'extreme' exactly 20 times in a row, separated by spaces." + ) + baseline_params = { + "frequency_penalty": 0.0, + "presence_penalty": 0.0, + "repetition_penalty": 1.0, + } + extreme_penalty_params = { + "frequency_penalty": 2.0, + "presence_penalty": 2.0, + "repetition_penalty": 2.0, + } + # Extreme penalties should strongly reduce repetition + self._test_penalty_effect( + prompt, + baseline_params, + extreme_penalty_params, + "extreme", + expected_reduction=True, + max_tokens=80, + ) + + +if __name__ == "__main__": + unittest.main(verbosity=3) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_piecewise_cuda_graph.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_piecewise_cuda_graph.py new file mode 100644 index 00000000..b8f3511b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_piecewise_cuda_graph.py @@ -0,0 +1,417 @@ +import unittest + +import torch + +from sglang import Engine +from sglang.lang.chat_template import get_chat_template_by_model_path +from sglang.srt.utils import get_device_sm, kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_IMAGE_URL, + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST_MLA, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + SimpleNamespace, + popen_launch_server, + run_bench_one_batch, +) + + +class TestPiecewiseCudaGraphCorrectness(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--enable-piecewise-cuda-graph"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + +class TestPiecewiseCudaGraphBenchmark(CustomTestCase): + + def test_latency(self): + prefill_latency, _, _ = run_bench_one_batch( + DEFAULT_MODEL_NAME_FOR_TEST, + other_args=["--enable-piecewise-cuda-graph"], + ) + self.assertLess(prefill_latency, 0.015) + + +@unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher") +class TestPiecewiseCudaGraphLlama31FP4(CustomTestCase): + """MGSM test: piecewise CUDA graph with NVFP4 Llama3.1 8B on Blackwell.""" + + @classmethod + def setUpClass(cls): + cls.model = "nvidia/Llama-3.1-8B-Instruct-FP4" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--enable-piecewise-cuda-graph", + "--quantization", + "modelopt_fp4", + "--mem-fraction-static", + "0.8", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mgsm_accuracy(self): + num_examples = 1319 + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=num_examples, + num_threads=min(num_examples, 1024), + ) + metrics = run_eval(args) + print(f"MGSM Accuracy: {metrics['score']:.3f}") + self.assertGreaterEqual(metrics["score"], 0.78) + + +class TestPiecewiseCudaGraphQwen3MoE(CustomTestCase): + """Test piecewise CUDA graph with Qwen3-Coder-30B-A3B-Instruct MoE model""" + + @classmethod + def setUpClass(cls): + cls.model = "Qwen/Qwen3-Coder-30B-A3B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--enable-piecewise-cuda-graph", + "--piecewise-cuda-graph-compiler", + "eager", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k_accuracy(self): + """Test GSM8K accuracy with 8-shot setting""" + num_examples = 2000 + + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=num_examples, + num_threads=min(num_examples, 1024), + ) + + metrics = run_eval(args) + print(f"GSM8K Accuracy: {metrics['score']:.3f}") + + self.assertGreaterEqual(metrics["score"], 0.90) + + +class TestPiecewiseCudaGraphDeepSeek(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--enable-piecewise-cuda-graph", + "--piecewise-cuda-graph-compiler", + "eager", + "--piecewise-cuda-graph-max-tokens", + "4096", # should less than max_context_len + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.62) + + +class TestPiecewiseCudaGraphAWQ(CustomTestCase): + """Test piecewise CUDA graph with AWQ quantized model""" + + @classmethod + def setUpClass(cls): + cls.model = "Qwen/QwQ-32B-AWQ" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--enable-piecewise-cuda-graph"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mgsm_accuracy(self): + """Test MGSM accuracy with AWQ model""" + num_examples = 1319 + + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=num_examples, + num_threads=min(num_examples, 1024), + ) + + metrics = run_eval(args) + print(f"MGSM Accuracy: {metrics['score']:.3f}") + print(f"Output throughput: {metrics.get('throughput', 'N/A')} token/s") + + # Expected accuracy: 0.680, allow some variance + self.assertGreaterEqual(metrics["score"], 0.65) + + +class TestPiecewiseCudaGraphFP8(CustomTestCase): + """Test piecewise CUDA graph with FP8 quantized model""" + + @classmethod + def setUpClass(cls): + cls.model = "nvidia/Llama-3.1-8B-Instruct-FP8" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--enable-piecewise-cuda-graph", + "--quantization", + "modelopt_fp8", + "--kv-cache-dtype", + "bfloat16", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mgsm_accuracy(self): + """Test MGSM accuracy with FP8 model""" + num_examples = 1319 + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=num_examples, + num_threads=min(num_examples, 1024), + ) + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.85) + print(f"MGSM Accuracy: {metrics['score']:.3f}") + + +class TestPiecewiseCudaGraphQwen25VL(CustomTestCase): + """Test piecewise CUDA graph with Qwen2.5-VL-7B-Instruct model""" + + @classmethod + def setUpClass(cls): + cls.model = "Qwen/Qwen2.5-VL-7B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--enable-piecewise-cuda-graph", + "--piecewise-cuda-graph-compiler", + "eager", + "--disable-radix-cache", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k_accuracy(self): + """Test GSM8K accuracy with 8-shot setting""" + num_examples = 2000 + + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=num_examples, + num_threads=min(num_examples, 1024), + ) + + metrics = run_eval(args) + print(f"GSM8K Accuracy: {metrics['score']:.3f}") + + self.assertGreaterEqual(metrics["score"], 0.70) + + +class TestPiecewiseCudaGraphInternVL25(CustomTestCase): + """Test piecewise CUDA graph with InternVL2.5-8B-Instruct model""" + + @classmethod + def setUpClass(cls): + cls.model = "OpenGVLab/InternVL2_5-8B" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--enable-piecewise-cuda-graph", + "--piecewise-cuda-graph-compiler", + "eager", + "--disable-radix-cache", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k_accuracy(self): + """Test GSM8K accuracy with 8-shot setting""" + num_examples = 2000 + + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=num_examples, + num_threads=min(num_examples, 1024), + ) + + metrics = run_eval(args) + print(f"GSM8K Accuracy: {metrics['score']:.3f}") + + self.assertGreaterEqual(metrics["score"], 0.70) + + +class TestPiecewiseCudaGraphQwen25VLEmbedding(CustomTestCase): + """Test piecewise CUDA graph with Qwen2.5-VL-3B-Instruct embedding model""" + + def test_embedding(self): + model_path = "Qwen/Qwen2.5-VL-3B-Instruct" + chat_template = get_chat_template_by_model_path(model_path) + text = f"{chat_template.image_token}What is in this picture? Answer: " + + engine = Engine( + model_path=model_path, + enable_multimodal=True, + is_embedding=True, + enable_piecewise_cuda_graph=True, + piecewise_cuda_graph_compiler="eager", + ) + out = engine.encode([text], image_data=[DEFAULT_IMAGE_URL])[0]["embedding"] + engine.shutdown() + self.assertGreater(len(out), 0) + + engine = Engine( + model_path=model_path, + enable_multimodal=True, + is_embedding=True, + enable_piecewise_cuda_graph=False, + ) + out_without_pcg = engine.encode([text], image_data=[DEFAULT_IMAGE_URL])[0][ + "embedding" + ] + engine.shutdown() + self.assertGreater(len(out_without_pcg), 0) + + self.assertTrue( + torch.allclose(torch.tensor(out), torch.tensor(out_without_pcg)) + ) + + +class TestPiecewiseCudaGraphQwen3OmniMOE(CustomTestCase): + """Test piecewise CUDA graph with Qwen3-Omni-30B-A3B-Instruct model""" + + @classmethod + def setUpClass(cls): + cls.model = "Qwen/Qwen3-Omni-30B-A3B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--enable-piecewise-cuda-graph", + "--piecewise-cuda-graph-compiler", + "eager", + "--disable-radix-cache", + "--tp=4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k_accuracy(self): + """Test GSM8K accuracy with 8-shot setting""" + num_examples = 2000 + + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=num_examples, + num_threads=min(num_examples, 1024), + ) + + metrics = run_eval(args) + print(f"GSM8K Accuracy: {metrics['score']:.3f}") + + self.assertGreaterEqual(metrics["score"], 0.70) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_pp_single_node.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_pp_single_node.py new file mode 100644 index 00000000..e333c2d4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_pp_single_node.py @@ -0,0 +1,322 @@ +""" +Usage: +python3 -m unittest test_pp_single_node.TestPPAccuracy.test_gsm8k +python3 -m unittest test_pp_single_node.TestQwenPPAccuracy.test_pp_consistency +python3 -m unittest test_pp_single_node.TestFixedBugs.test_chunked_prefill_with_small_bs +""" + +import time +import unittest +from types import SimpleNamespace + +import requests + +from sglang.bench_one_batch_server import BenchArgs as OneBatchBenchArgs +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_one_batch_server, +) + + +class TestPPAccuracy(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.base_url = "http://127.0.0.1:23333" + cls.process = popen_launch_server( + DEFAULT_MODEL_NAME_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--tp-size", + 2, + "--pp-size", + 2, + "--chunked-prefill-size", + 256, + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + self.assertGreater(metrics["accuracy"], 0.74) + # Wait a little bit so that the memory check happens. + time.sleep(4) + + def test_logprob(self): + response = requests.post( + f"{self.base_url}/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 16, + }, + "return_logprob": True, + "top_logprobs_num": 5, + "logprob_start_len": 0, + }, + ) + response_json = response.json() + input_token_logprobs = response_json["meta_info"]["input_token_logprobs"] + output_token_logprobs = response_json["meta_info"]["output_token_logprobs"] + output_top_logprobs = response_json["meta_info"]["output_top_logprobs"] + + assert len(input_token_logprobs) == 6 + assert len(output_token_logprobs) == 16 + assert len(output_top_logprobs) == 16 + + +class TestDPAttentionDP2PP2(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "2", + "--pp-size", + "2", + "--enable-dp-attention", + "--dp", + "2", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.8) + + +class TestQwenPPAccuracy(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.base_url = "http://127.0.0.1:23334" # different ports to avoid conflicts + cls.model_name = "Qwen/Qwen3-8B" # replace with your Qwen Model if needed + + def run_gsm8k_test(self, pp_size): + process = popen_launch_server( + self.model_name, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--pp-size", + pp_size, + "--chunked-prefill-size", + 256, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + time.sleep(5) + return metrics + finally: + kill_process_tree(process.pid) + + @unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") + def test_pp_consistency(self): + baseline = self.run_gsm8k_test(pp_size=1) + pp_metrics = self.run_gsm8k_test(pp_size=2) + + print(f"[Qwen PP Comparison] Baseline: {baseline} | PP: {pp_metrics}") + + self.assertGreaterEqual(baseline["accuracy"], 0.74) + self.assertGreaterEqual( + pp_metrics["accuracy"], + baseline["accuracy"] - 0.02, + msg=( + f"PP accuracy dropped more than 1% compared to baseline. " + f"Baseline: {baseline['accuracy']:.2%}, PP: {pp_metrics['accuracy']:.2%}" + ), + ) + + +class TestQwenPPTieWeightsAccuracy(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.base_url = "http://127.0.0.1:23335" # different ports to avoid conflicts + cls.model_name = ( + "Qwen/Qwen3-0.6B" # qwen3 < 8B all have tie_word_embeddings = True + ) + + def run_gsm8k_test(self, pp_size): + process = popen_launch_server( + self.model_name, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--pp-size", + pp_size, + "--chunked-prefill-size", + 256, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + time.sleep(5) + return metrics + finally: + kill_process_tree(process.pid) + + def test_pp_consistency(self): + baseline = self.run_gsm8k_test(pp_size=1) + pp_metrics = self.run_gsm8k_test(pp_size=2) + + print(f"[Qwen PP Comparison] Baseline: {baseline} | PP: {pp_metrics}") + + self.assertGreaterEqual(baseline["accuracy"], 0.38) + self.assertGreaterEqual( + pp_metrics["accuracy"], + baseline["accuracy"] - 0.02, + msg=( + f"PP accuracy dropped more than 1% compared to baseline. " + f"Baseline: {baseline['accuracy']:.2%}, PP: {pp_metrics['accuracy']:.2%}" + ), + ) + + +class TestQwenMoePPAccuracy(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.base_url = "http://127.0.0.1:23336" # different ports to avoid conflicts + cls.model_name = "Qwen/Qwen3-30B-A3B" # replace with your Qwen Model if needed + + def run_gsm8k_test(self, pp_size): + process = popen_launch_server( + self.model_name, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--pp-size", + pp_size, + "--chunked-prefill-size", + 256, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + time.sleep(5) + return metrics + finally: + kill_process_tree(process.pid) + + def test_pp_consistency(self): + baseline = self.run_gsm8k_test(pp_size=1) + pp_metrics = self.run_gsm8k_test(pp_size=2) + + print(f"[Qwen PP Comparison] Baseline: {baseline} | PP: {pp_metrics}") + + self.assertGreaterEqual(baseline["accuracy"], 0.74) + self.assertGreaterEqual( + pp_metrics["accuracy"], + baseline["accuracy"] - 0.02, + msg=( + f"PP accuracy dropped more than 1% compared to baseline. " + f"Baseline: {baseline['accuracy']:.2%}, PP: {pp_metrics['accuracy']:.2%}" + ), + ) + + +class TestFixedBugs(unittest.TestCase): + def test_chunked_prefill_with_small_bs(self): + model = DEFAULT_MODEL_NAME_FOR_TEST + server_args = ServerArgs(model_path=model) + bench_args = OneBatchBenchArgs( + batch_size=(1,), + input_len=(1,), + output_len=(1,), + base_url=DEFAULT_URL_FOR_TEST, + ) + other_server_args = [ + "--tp-size", + 2, + "--pp-size", + 2, + "--chunked-prefill", + 256, + "--max-running-requests", + 2, + ] + run_bench_one_batch_server( + model, + DEFAULT_URL_FOR_TEST, + server_args, + bench_args, + other_server_args, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_priority_scheduling.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_priority_scheduling.py new file mode 100644 index 00000000..89733f15 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_priority_scheduling.py @@ -0,0 +1,421 @@ +import asyncio +import os +import re +import unittest +from typing import Any, List, Optional, Tuple + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + STDERR_FILENAME, + STDOUT_FILENAME, + CustomTestCase, + popen_launch_server, + send_concurrent_generate_requests_with_custom_params, +) + + +class TestPriorityScheduling(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + + cls.stdout = open(STDOUT_FILENAME, "w") + cls.stderr = open(STDERR_FILENAME, "w") + + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=( + "--max-running-requests", # Enforce max request concurrency is 1 + "1", + "--max-queued-requests", # Enforce max queued request number is 3 + "3", + "--enable-priority-scheduling", # Enable priority scheduling + ), + return_stdout_stderr=(cls.stdout, cls.stderr), + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + _verify_max_running_requests_and_max_queued_request_validation(1, 3) + cls.stdout.close() + cls.stderr.close() + os.remove(STDOUT_FILENAME) + os.remove(STDERR_FILENAME) + + def test_priority_scheduling_request_ordering_validation(self): + """Verify pending requests are ordered by priority and received timestamp.""" + + responses = asyncio.run( + send_concurrent_generate_requests_with_custom_params( + self.base_url, + [ + { + "priority": 0, + "sampling_params": {"max_new_tokens": 10000}, + }, # starts being processed first + {"priority": 1}, # third + {"priority": 1}, # fourth + {"priority": 2}, # second + ], + ) + ) + + expected_status_and_error_messages = [ + (200, None), + (200, None), + (200, None), + (200, None), + ] + + e2e_latencies = [] + _verify_genereate_responses( + responses, expected_status_and_error_messages, e2e_latencies + ) + assert e2e_latencies[0] < e2e_latencies[3] < e2e_latencies[1] < e2e_latencies[2] + + def test_priority_scheduling_existing_requests_abortion_validation(self): + """Verify lower priority requests are aborted when incoming requests have higher priority""" + + responses = asyncio.run( + send_concurrent_generate_requests_with_custom_params( + self.base_url, + [ + { + "priority": 1, + "sampling_params": {"max_new_tokens": 10000}, + }, # starts being processed first and holds the running queue capacity + {"priority": 2}, # aborted by request 5 + {"priority": 3}, # aborted by request 6 + {"priority": 4}, # aborted by request 7 + {"priority": 5}, # fourth + {"priority": 6}, # third + {"priority": 7}, # second + ], + ) + ) + + expected_status_and_error_messages = [ + (200, None), + (503, "The request is aborted by a higher priority request."), + (503, "The request is aborted by a higher priority request."), + (503, "The request is aborted by a higher priority request."), + (200, None), + (200, None), + (200, None), + ] + + e2e_latencies = [] + _verify_genereate_responses( + responses, expected_status_and_error_messages, e2e_latencies + ) + assert e2e_latencies[0] < e2e_latencies[6] < e2e_latencies[5] < e2e_latencies[4] + + def test_priority_scheduling_incoming_request_rejection_validation(self): + """Verify incoming requests are rejected when existing requests have higher priority""" + + responses = asyncio.run( + send_concurrent_generate_requests_with_custom_params( + self.base_url, + [ + { + "priority": 7, + "sampling_params": {"max_new_tokens": 10000}, + }, # starts being processed first and holds the running queue capacity + {"priority": 6}, # second + {"priority": 5}, # third + {"priority": 4}, # fourth + {"priority": 3}, # rejected + {"priority": 2}, # rejected + {"priority": 1}, # rejected + ], + ) + ) + + expected_status_and_error_messages = [ + (200, None), + (200, None), + (200, None), + (200, None), + (503, "The request queue is full."), + (503, "The request queue is full."), + (503, "The request queue is full."), + ] + + e2e_latencies = [] + _verify_genereate_responses( + responses, expected_status_and_error_messages, e2e_latencies + ) + assert e2e_latencies[0] < e2e_latencies[1] < e2e_latencies[2] < e2e_latencies[3] + + def test_priority_scheduling_preemption_meeting_threshold_validation(self): + """Verify running requests are preempted by requests with priorities meeting the preemption threshold""" + + responses = asyncio.run( + send_concurrent_generate_requests_with_custom_params( + self.base_url, + [ + { + "priority": 0, + "sampling_params": {"max_new_tokens": 10000}, + }, # starts being processed first then preempted or pushed by later requests, and finishes last. + { + "priority": 10, + "sampling_params": {"max_new_tokens": 10000}, + }, # scheduled after the third request, and finishes second. + { + "priority": 20, + "sampling_params": {"max_new_tokens": 10000}, + }, # finishes first. + ], + ) + ) + + expected_status_and_error_messages = [ + (200, None), + (200, None), + (200, None), + ] + + e2e_latencies = [] + _verify_genereate_responses( + responses, expected_status_and_error_messages, e2e_latencies + ) + + assert e2e_latencies[2] < e2e_latencies[1] < e2e_latencies[0] + + def test_priority_scheduling_preemption_below_threshold_validation(self): + """Verify running requests are not preempted by requests with priorities below preemption threshold""" + + responses = asyncio.run( + send_concurrent_generate_requests_with_custom_params( + self.base_url, + [ + { + "priority": 0, + "sampling_params": {"max_new_tokens": 10000}, + }, + { + "priority": 5, + "sampling_params": {"max_new_tokens": 10000}, + }, + ], + ) + ) + + expected_status_and_error_messages = [ + (200, None), + (200, None), + ] + + e2e_latencies = [] + _verify_genereate_responses( + responses, expected_status_and_error_messages, e2e_latencies + ) + + assert e2e_latencies[0] < e2e_latencies[1] + + +class TestPrioritySchedulingMultipleRunningRequests(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + + cls.stdout = open(STDOUT_FILENAME, "w") + cls.stderr = open(STDERR_FILENAME, "w") + + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=( + "--max-running-requests", # Enforce max request concurrency is 2 + "2", + "--max-queued-requests", # Enforce max queued request number is 3 + "3", + "--enable-priority-scheduling", # Enable priority scheduling + ), + return_stdout_stderr=(cls.stdout, cls.stderr), + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + _verify_max_running_requests_and_max_queued_request_validation(2, 3) + cls.stdout.close() + cls.stderr.close() + os.remove(STDOUT_FILENAME) + os.remove(STDERR_FILENAME) + + def test_priority_scheduling_with_multiple_running_requests_preemption(self): + """Verify preempting a subset of running requests is safe.""" + + responses = asyncio.run( + send_concurrent_generate_requests_with_custom_params( + self.base_url, + [ + { + "priority": 10, + "sampling_params": {"max_new_tokens": 10000}, + }, # finishes first + { + "priority": 5, + "sampling_params": {"max_new_tokens": 10000}, + }, # preempted by fourth request, then finishes third + { + "priority": 15, + "sampling_params": {"max_new_tokens": 10000}, + }, # preempt the first request + ], + ) + ) + + expected_status_and_error_messages = [ + (200, None), + (200, None), + (200, None), + (200, None), + ] + + _verify_genereate_responses(responses, expected_status_and_error_messages, []) + + def test_priority_scheduling_preemption_token_offset_calculation(self): + """ + Verify correct token offset calculation during preemption. + + This test specifically targets the bug where rem_total_token_offset was incorrectly + calculated using the incoming request's tokens instead of the preempted request's tokens + (related to issue #13111 and PR #13201). + + THE BUG: + In schedule_policy.py line 700, the code was using: + self.rem_total_token_offset -= self._get_running_request_total_token_offset(req) + Instead of: + self.rem_total_token_offset -= self._get_running_request_total_token_offset(running_req) + + WHY THIS TEST CATCHES THE BUG: + - Request 1 (preempted): 8000 tokens - This is what SHOULD be freed + - Request 3 (incoming): 1000 tokens - This is what WAS freed (bug) + - Token difference: 8000 - 1000 = 7000 tokens incorrectly accounted + + With the bug, the system thinks it only freed 1000 tokens instead of 8000 tokens. + This causes incorrect memory accounting and can lead to: + 1. Scheduler believes less memory is available than actually is + 2. Subsequent requests (like Request 4) may fail to schedule or cause issues + 3. Memory calculations become increasingly inaccurate with each preemption + + The test creates a scenario where: + 1. A low-priority request with many tokens (8000) starts running + 2. A high-priority request with few tokens (1000) arrives and triggers preemption + 3. The system must correctly free 8000 tokens from the preempted request + 4. Additional requests can be scheduled only if tokens were correctly freed + 5. Execution order validates priority-based scheduling works correctly + + The large token difference (8x) makes the bug's impact obvious and testable. + """ + responses = asyncio.run( + send_concurrent_generate_requests_with_custom_params( + self.base_url, + [ + { + "priority": 0, + "sampling_params": {"max_new_tokens": 8000}, + }, # Low priority, large token count - will be preempted + { + "priority": 1, + "sampling_params": {"max_new_tokens": 5000}, + }, # Medium priority, medium token count - queued initially + { + "priority": 100, + "sampling_params": {"max_new_tokens": 1000}, + }, # High priority, small token count - triggers preemption + { + "priority": 50, + "sampling_params": {"max_new_tokens": 2000}, + }, # Should be schedulable after correct token accounting + ], + ) + ) + + # All requests should complete successfully + # The key is that the fourth request should be schedulable because + # the system correctly freed tokens from the first (preempted) request + expected_status_and_error_messages = [ + (200, None), + (200, None), + (200, None), + (200, None), + ] + + e2e_latencies = [] + _verify_genereate_responses( + responses, expected_status_and_error_messages, e2e_latencies + ) + + # Verify execution order: high priority requests finish before low priority ones + # Request 3 (priority 100) should finish first + # Request 4 (priority 50) should finish second + # Request 2 (priority 1) should finish third + # Request 1 (priority 0) should finish last (after being preempted) + + # FIXME(harrison lim) + # assert e2e_latencies[2] < e2e_latencies[3] < e2e_latencies[1] < e2e_latencies[0] + + +def _verify_genereate_responses( + responses: Tuple[int, Any, float], + expected_code_and_error_message: Tuple[int, Any], + e2e_latencies: List[Optional[float]], +): + """ + Verify generate response results are as expected based on status code and response json object content. + In addition, collects e2e latency info to verify scheduling and processing ordering. + """ + for got, expected in zip(responses, expected_code_and_error_message): + got_status, got_json = got + expected_status, expected_err_msg = expected + + # Check status code is as expected + assert got_status == expected_status + + # Check error message content or fields' existence based on status code + if got_status != 200: + assert got_json["object"] == "error" + assert got_json["message"] == expected_err_msg + else: + assert "object" not in got_json + assert "message" not in got_json + + # Collect e2e latencies for scheduling validation + e2e_latencies.append( + got_json["meta_info"]["e2e_latency"] if got_status == 200 else None + ) + + +def _verify_max_running_requests_and_max_queued_request_validation( + max_running_requests: int, max_queued_requests: int +): + """Verify running request and queued request numbers based on server logs.""" + rr_pattern = re.compile(r"#running-req:\s*(\d+)") + qr_pattern = re.compile(r"#queue-req:\s*(\d+)") + + with open(STDERR_FILENAME) as lines: + for line in lines: + rr_match, qr_match = rr_pattern.search(line), qr_pattern.search(line) + if rr_match: + assert int(rr_match.group(1)) <= max_running_requests + if qr_match: + assert int(qr_match.group(1)) <= max_queued_requests + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_profile_merger.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_profile_merger.py new file mode 100644 index 00000000..2a88b191 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_profile_merger.py @@ -0,0 +1,363 @@ +""" +Unit tests for the ProfileMerger implementation. + +Usage: + python test_profile_merger.py + python -m unittest test_profile_merger.py -v +""" + +import gzip +import json +import os +import shutil +import tempfile +import unittest + +from sglang.srt.managers.io_struct import ProfileReq, ProfileReqInput, ProfileReqType +from sglang.srt.utils.profile_merger import ProfileMerger + + +class TestProfileMerger(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + self.profile_id = "test_profile_123" + self.merger = ProfileMerger(self.temp_dir, self.profile_id) + + def tearDown(self): + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def test_rank_extraction_and_labeling(self): + # Test TP-only + filename = f"{self.profile_id}-TP-0.trace.json.gz" + rank_info = self.merger._extract_rank_info(filename) + self.assertEqual(rank_info, {"tp_rank": 0}) + label = self.merger._create_rank_label(rank_info) + self.assertEqual(label, "[TP00]") + + # Test all parallelism types + filename = f"{self.profile_id}-TP-1-DP-2-PP-3-EP-4.trace.json.gz" + rank_info = self.merger._extract_rank_info(filename) + self.assertEqual( + rank_info, {"tp_rank": 1, "dp_rank": 2, "pp_rank": 3, "ep_rank": 4} + ) + label = self.merger._create_rank_label(rank_info) + self.assertEqual(label, "[TP01-DP02-PP03-EP04]") + + # Test partial ranks + filename = f"{self.profile_id}-TP-0-DP-1.trace.json.gz" + rank_info = self.merger._extract_rank_info(filename) + self.assertEqual(rank_info, {"tp_rank": 0, "dp_rank": 1}) + label = self.merger._create_rank_label(rank_info) + self.assertEqual(label, "[TP00-DP01]") + + # Test no ranks + filename = f"{self.profile_id}.trace.json.gz" + rank_info = self.merger._extract_rank_info(filename) + self.assertEqual(rank_info, {}) + label = self.merger._create_rank_label(rank_info) + self.assertEqual(label, "[Unknown]") + + def test_sort_index_calculation(self): + # Single rank + rank_info = {"tp_rank": 0} + sort_idx = self.merger._calculate_sort_index(rank_info, 83) + self.assertEqual(sort_idx, 83) + + # Multiple ranks + rank_info = {"tp_rank": 1, "dp_rank": 2, "pp_rank": 3, "ep_rank": 4} + sort_idx = self.merger._calculate_sort_index(rank_info, 83) + self.assertNotEqual(sort_idx, 83) + self.assertGreater(sort_idx, 1000000) + + # Empty ranks + rank_info = {} + sort_idx = self.merger._calculate_sort_index(rank_info, 83) + self.assertEqual(sort_idx, 83) + + def test_rank_sort_key(self): + # Full ranks: TP-1, DP-2, PP-3, EP-4 → sorted as (DP, EP, PP, TP) + filename = f"{self.profile_id}-TP-1-DP-2-PP-3-EP-4.trace.json.gz" + sort_key = self.merger._get_rank_sort_key(filename) + self.assertEqual(sort_key, (2, 4, 3, 1)) + + # Missing ranks: only TP-1 → sorted as (DP=0, EP=0, PP=0, TP=1) + filename = f"{self.profile_id}-TP-1.trace.json.gz" + sort_key = self.merger._get_rank_sort_key(filename) + self.assertEqual(sort_key, (0, 0, 0, 1)) + + def test_discover_trace_files(self): + # Create mock trace files + trace_files = [ + f"{self.profile_id}-TP-0.trace.json.gz", # Old format + f"{self.profile_id}-TP-1.trace.json.gz", # Old format + f"{self.profile_id}-TP-0-DP-1.trace.json.gz", # New format + ] + + for filename in trace_files: + filepath = os.path.join(self.temp_dir, filename) + with gzip.open(filepath, "wt") as f: + json.dump({"traceEvents": []}, f) + + discovered = self.merger._discover_trace_files() + self.assertEqual(len(discovered), 3) + + # Check that all expected files are discovered + discovered_basenames = {os.path.basename(f) for f in discovered} + expected_basenames = { + f"{self.profile_id}-TP-0.trace.json.gz", + f"{self.profile_id}-TP-1.trace.json.gz", + f"{self.profile_id}-TP-0-DP-1.trace.json.gz", + } + self.assertEqual(discovered_basenames, expected_basenames) + + # Test no matches + empty_merger = ProfileMerger(self.temp_dir, "nonexistent") + discovered = empty_merger._discover_trace_files() + self.assertEqual(len(discovered), 0) + + def test_merge_chrome_traces(self): + # Create multiple trace files in random order + trace_files = [ + { + "filename": f"{self.profile_id}-TP-1-DP-1.trace.json.gz", + "events": [ + {"ph": "X", "name": "op1", "pid": 83, "ts": 1000.0, "dur": 10.0} + ], + }, + { + "filename": f"{self.profile_id}-TP-0.trace.json.gz", + "events": [ + {"ph": "X", "name": "op2", "pid": 84, "ts": 2000.0, "dur": 15.0} + ], + }, + { + "filename": f"{self.profile_id}-TP-0-DP-1.trace.json.gz", + "events": [ + {"ph": "X", "name": "op3", "pid": 85, "ts": 3000.0, "dur": 20.0} + ], + }, + ] + + for trace_data in trace_files: + filepath = os.path.join(self.temp_dir, trace_data["filename"]) + trace_content = { + "schemaVersion": 1, + "deviceProperties": [{"device_id": 0, "name": "GPU-0"}], + "traceEvents": trace_data["events"], + } + with gzip.open(filepath, "wt") as f: + json.dump(trace_content, f) + + # Test file ordering by capturing log messages + import logging + + logger = logging.getLogger("sglang.srt.utils.profile_merger") + with self.assertLogs(logger, level="INFO") as log_capture: + merged_path = self.merger.merge_chrome_traces() + + # Verify files were processed in rank order + log_messages = [ + record.getMessage() + for record in log_capture.records + if "Processing file:" in record.getMessage() + ] + self.assertIn("TP-0.trace.json.gz", log_messages[0]) # (0,0,0,0) comes first + self.assertIn( + "TP-0-DP-1.trace.json.gz", log_messages[1] + ) # (0,1,0,0) comes second + self.assertIn( + "TP-1-DP-1.trace.json.gz", log_messages[2] + ) # (1,1,0,0) comes last + + # Verify merged content + self.assertTrue(os.path.exists(merged_path)) + with gzip.open(merged_path, "rt") as f: + merged_data = json.load(f) + + self.assertEqual(len(merged_data["traceEvents"]), 3) + self.assertEqual(len(merged_data["deviceProperties"]), 3) + + # Check rank labels in events + events = merged_data["traceEvents"] + pids = [event["pid"] for event in events] + self.assertIn("[TP00] 84", pids) + self.assertIn("[TP00-DP01] 85", pids) + self.assertIn("[TP01-DP01] 83", pids) + + # Test merge summary + summary = self.merger.get_merge_summary() + self.assertEqual(summary["total_files"], 3) + self.assertEqual(summary["total_events"], 3) + self.assertEqual(summary["profile_id"], self.profile_id) + + # Test no files error + empty_merger = ProfileMerger(self.temp_dir, "nonexistent") + with self.assertRaises(ValueError): + empty_merger.merge_chrome_traces() + + +class TestProfileMergerIntegration(unittest.TestCase): + + def test_data_structures_merge_profiles(self): + # Test ProfileReqInput + req_input = ProfileReqInput() + self.assertFalse(req_input.merge_profiles) + + req_input = ProfileReqInput(merge_profiles=True) + self.assertTrue(req_input.merge_profiles) + + # Test ProfileReq + req = ProfileReq(type=ProfileReqType.START_PROFILE) + self.assertFalse(req.merge_profiles) + + req = ProfileReq(type=ProfileReqType.START_PROFILE, merge_profiles=True) + self.assertTrue(req.merge_profiles) + + def test_integration_parameters(self): + import inspect + + # Test TokenizerManager + from sglang.srt.managers.tokenizer_communicator_mixin import ( + TokenizerCommunicatorMixin, + ) + + sig = inspect.signature(TokenizerCommunicatorMixin.start_profile) + self.assertIn("merge_profiles", sig.parameters) + + # Test SchedulerProfilerMixin + from sglang.srt.managers.scheduler_profiler_mixin import SchedulerProfilerMixin + + sig = inspect.signature(SchedulerProfilerMixin.init_profile) + self.assertIn("merge_profiles", sig.parameters) + + # Test CLI profiler + from sglang.profiler import run_profile + + sig = inspect.signature(run_profile) + self.assertIn("merge_profiles", sig.parameters) + + +class TestProfileMergerEdgeCases(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + self.profile_id = "test_edge_cases" + self.merger = ProfileMerger(self.temp_dir, self.profile_id) + + def tearDown(self): + import shutil + + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def test_error_handling_and_edge_cases(self): + # Test malformed trace file + filename = f"{self.profile_id}-TP-0.trace.json.gz" + filepath = os.path.join(self.temp_dir, filename) + with gzip.open(filepath, "wt") as f: + f.write("invalid json content") + + merged_path = self.merger.merge_chrome_traces() + self.assertTrue(os.path.exists(merged_path)) + + with gzip.open(merged_path, "rt") as f: + merged_data = json.load(f) + self.assertEqual(len(merged_data["traceEvents"]), 0) + + # Test empty trace file + with gzip.open(filepath, "wt") as f: + json.dump({}, f) + merged_path = self.merger.merge_chrome_traces() + self.assertTrue(os.path.exists(merged_path)) + + # Test missing device properties + trace_data = { + "schemaVersion": 1, + "traceEvents": [ + {"ph": "X", "name": "test", "pid": 83, "ts": 1000.0, "dur": 10.0} + ], + } + with gzip.open(filepath, "wt") as f: + json.dump(trace_data, f) + + merged_path = self.merger.merge_chrome_traces() + with gzip.open(merged_path, "rt") as f: + merged_data = json.load(f) + self.assertNotIn("deviceProperties", merged_data) + + def test_missing_ranks_and_none_handling(self): + # Test rank extraction with missing ranks + filename = f"{self.profile_id}-TP-0.trace.json.gz" + rank_info = self.merger._extract_rank_info(filename) + self.assertEqual(rank_info, {"tp_rank": 0}) + + # Test rank label creation with missing ranks + label = self.merger._create_rank_label({"tp_rank": 0}) + self.assertEqual(label, "[TP00]") + + label = self.merger._create_rank_label({}) + self.assertEqual(label, "[Unknown]") + + # Test sort index calculation + sort_idx = self.merger._calculate_sort_index({"tp_rank": 0}, 83) + self.assertGreater(sort_idx, 0) + + sort_idx = self.merger._calculate_sort_index({}, 83) + self.assertEqual(sort_idx, 83) + + # Test sort key generation + sort_key = self.merger._get_rank_sort_key(filename) + self.assertEqual(sort_key, (0, 0, 0, 0)) + + # Test _maybe_cast_int with various inputs + self.assertIsNone(self.merger._maybe_cast_int(None)) + self.assertIsNone(self.merger._maybe_cast_int("invalid")) + self.assertEqual(self.merger._maybe_cast_int("123"), 123) + self.assertEqual(self.merger._maybe_cast_int(456), 456) + + def test_mixed_rank_scenarios(self): + trace_scenarios = [ + { + "filename": f"{self.profile_id}-TP-0.trace.json.gz", + "events": [ + {"ph": "X", "name": "op1", "pid": 83, "ts": 1000.0, "dur": 10.0} + ], + }, + { + "filename": f"{self.profile_id}-TP-1-DP-0.trace.json.gz", + "events": [ + {"ph": "X", "name": "op2", "pid": 84, "ts": 2000.0, "dur": 15.0} + ], + }, + { + "filename": f"{self.profile_id}-TP-0-DP-1-PP-0.trace.json.gz", + "events": [ + {"ph": "X", "name": "op3", "pid": 85, "ts": 3000.0, "dur": 20.0} + ], + }, + ] + + for scenario in trace_scenarios: + filepath = os.path.join(self.temp_dir, scenario["filename"]) + trace_data = { + "schemaVersion": 1, + "deviceProperties": [{"device_id": 0, "name": "GPU-0"}], + "traceEvents": scenario["events"], + } + with gzip.open(filepath, "wt") as f: + json.dump(trace_data, f) + + merged_path = self.merger.merge_chrome_traces() + self.assertTrue(os.path.exists(merged_path)) + + with gzip.open(merged_path, "rt") as f: + merged_data = json.load(f) + + self.assertEqual(len(merged_data["traceEvents"]), 3) + events = merged_data["traceEvents"] + pids = [event["pid"] for event in events] + self.assertIn("[TP00] 83", pids) + self.assertIn("[TP01-DP00] 84", pids) + self.assertIn("[TP00-DP01-PP00] 85", pids) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_profile_merger_http_api.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_profile_merger_http_api.py new file mode 100644 index 00000000..e293dae9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_profile_merger_http_api.py @@ -0,0 +1,162 @@ +import json +import unittest + +from sglang.srt.managers.io_struct import ProfileReqInput + + +class TestProfileMergerHTTPAPI(unittest.TestCase): + def test_profile_req_input_merge_profiles_json_serialization(self): + # Test with merge_profiles=True + req_input = ProfileReqInput( + output_dir="/tmp/test", + num_steps=5, + activities=["CPU", "GPU"], + profile_by_stage=True, + merge_profiles=True, + ) + + # Convert to dict (as would happen in HTTP request) + req_dict = { + "output_dir": req_input.output_dir, + "num_steps": req_input.num_steps, + "activities": req_input.activities, + "profile_by_stage": req_input.profile_by_stage, + "merge_profiles": req_input.merge_profiles, + } + + # Test JSON serialization + json_str = json.dumps(req_dict) + parsed_data = json.loads(json_str) + + self.assertTrue(parsed_data["merge_profiles"]) + self.assertEqual(parsed_data["output_dir"], "/tmp/test") + self.assertEqual(parsed_data["num_steps"], 5) + self.assertEqual(parsed_data["activities"], ["CPU", "GPU"]) + self.assertTrue(parsed_data["profile_by_stage"]) + + def test_profile_req_input_merge_profiles_json_deserialization(self): + # Test JSON data as would come from HTTP request + json_data = { + "output_dir": "/tmp/test", + "num_steps": 10, + "activities": ["CPU", "GPU", "MEM"], + "profile_by_stage": False, + "merge_profiles": True, + } + + # Create ProfileReqInput from dict (as HTTP server would do) + req_input = ProfileReqInput(**json_data) + + self.assertTrue(req_input.merge_profiles) + self.assertEqual(req_input.output_dir, "/tmp/test") + self.assertEqual(req_input.num_steps, 10) + self.assertEqual(req_input.activities, ["CPU", "GPU", "MEM"]) + self.assertFalse(req_input.profile_by_stage) + + def test_profile_req_input_merge_profiles_default_value(self): + # Test with minimal data + json_data = {"output_dir": "/tmp/test"} + + req_input = ProfileReqInput(**json_data) + self.assertFalse(req_input.merge_profiles) + + def test_profile_req_input_merge_profiles_explicit_false(self): + json_data = {"output_dir": "/tmp/test", "merge_profiles": False} + + req_input = ProfileReqInput(**json_data) + self.assertFalse(req_input.merge_profiles) + + def test_http_api_parameter_flow(self): + # Simulate HTTP request data + request_data = { + "output_dir": "/tmp/test", + "num_steps": 5, + "activities": ["CPU", "GPU"], + "profile_by_stage": True, + "merge_profiles": True, + } + + # Create ProfileReqInput as HTTP server would + obj = ProfileReqInput(**request_data) + + # Verify the parameter is set correctly + self.assertTrue(obj.merge_profiles) + self.assertEqual(obj.output_dir, "/tmp/test") + self.assertEqual(obj.num_steps, 5) + self.assertEqual(obj.activities, ["CPU", "GPU"]) + self.assertTrue(obj.profile_by_stage) + + def test_http_api_parameter_validation(self): + # Test with True + json_data = {"merge_profiles": True} + req_input = ProfileReqInput(**json_data) + self.assertTrue(req_input.merge_profiles) + + # Test with False + json_data = {"merge_profiles": False} + req_input = ProfileReqInput(**json_data) + self.assertFalse(req_input.merge_profiles) + + # Test with string "true" (should be converted by JSON parser) + json_data = {"merge_profiles": "true"} + req_input = ProfileReqInput(**json_data) + self.assertEqual(req_input.merge_profiles, "true") # String, not boolean + + def test_http_api_backward_compatibility(self): + # Test minimal request (no merge_profiles) + json_data = {} + req_input = ProfileReqInput(**json_data) + self.assertFalse(req_input.merge_profiles) # Should default to False + + # Test with other parameters but no merge_profiles + json_data = { + "output_dir": "/tmp/test", + "num_steps": 5, + "activities": ["CPU", "GPU"], + } + req_input = ProfileReqInput(**json_data) + self.assertFalse(req_input.merge_profiles) # Should default to False + + def test_http_api_parameter_combinations(self): + test_cases = [ + { + "name": "minimal with merge_profiles", + "data": {"merge_profiles": True}, + "expected_merge": True, + }, + { + "name": "full parameters with merge_profiles=True", + "data": { + "output_dir": "/tmp/test", + "num_steps": 10, + "activities": ["CPU", "GPU", "MEM"], + "profile_by_stage": True, + "with_stack": True, + "record_shapes": True, + "merge_profiles": True, + }, + "expected_merge": True, + }, + { + "name": "full parameters with merge_profiles=False", + "data": { + "output_dir": "/tmp/test", + "num_steps": 10, + "activities": ["CPU", "GPU", "MEM"], + "profile_by_stage": False, + "with_stack": False, + "record_shapes": False, + "merge_profiles": False, + }, + "expected_merge": False, + }, + ] + + for test_case in test_cases: + with self.subTest(test_case["name"]): + req_input = ProfileReqInput(**test_case["data"]) + self.assertEqual(req_input.merge_profiles, test_case["expected_merge"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_profile_v2.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_profile_v2.py new file mode 100644 index 00000000..8ff16213 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_profile_v2.py @@ -0,0 +1,102 @@ +import os +import shutil +import tempfile +import unittest +from pathlib import Path + +import requests + +from sglang.srt.environ import envs +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestStartProfile(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.output_dir = tempfile.mkdtemp() + envs.SGLANG_TORCH_PROFILER_DIR.set(cls.output_dir) + envs.SGLANG_PROFILE_V2.set(True) + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def setUp(self): + self._clear_profile_dir() + + def test_profile_by_stage(self): + self._start_profile( + profile_by_stage=True, + num_steps=10, + ) + + self._post_request() + + self._check_profile_output(pattern="*-prefill*", expect_existence=True) + self._check_profile_output(pattern="*-decode*", expect_existence=True) + + def test_decode_only(self): + self._start_profile( + profile_by_stage=True, + profile_stages=["decode"], + num_steps=10, + ) + + self._post_request() + + self._check_profile_output(pattern="*-prefill*", expect_existence=False) # NOTE + self._check_profile_output(pattern="*-decode*", expect_existence=True) + + def _start_profile(self, **kwargs): + """Start profiling with optional parameters.""" + response = requests.post( + f"{DEFAULT_URL_FOR_TEST}/start_profile", + json=kwargs if kwargs else None, + ) + self.assertEqual(response.status_code, 200) + + def _post_request(self): + response = requests.post( + f"{DEFAULT_URL_FOR_TEST}/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, + ) + self.assertEqual(response.status_code, 200) + + def _clear_profile_dir(self): + if os.path.isdir(self.output_dir): + shutil.rmtree(self.output_dir) + + def _check_profile_output(self, pattern: str, expect_existence: bool): + self.assertTrue( + os.path.isdir(self.output_dir), "Output directory does not exist." + ) + self.assertEqual( + len(list(Path(self.output_dir).glob(pattern))) > 0, + expect_existence, + f"Does not find {pattern=} ({list(Path(self.output_dir).glob('**/*'))=})", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_pytorch_sampling_backend.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_pytorch_sampling_backend.py new file mode 100644 index 00000000..8a368310 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_pytorch_sampling_backend.py @@ -0,0 +1,90 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestPyTorchSamplingBackend(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--sampling-backend", "pytorch", "--disable-radix-cache"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + temperature=0.1, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + def test_greedy(self): + + first_text = None + + # ensure the answer is identical across single response + for _ in range(5): + response_single = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of Germany is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, + ).json() + text = response_single["text"] + if first_text is None: + first_text = text + + self.assertEqual(text, first_text) + + first_text = None + + response_batch = requests.post( + self.base_url + "/generate", + json={ + "text": ["The capital of Germany is"] * 10, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, + ).json() + + # ensure the answer is identical among the batch + for i in range(10): + text = response_batch[i]["text"] + if first_text is None: + first_text = text + self.assertEqual(text, first_text) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_quantization.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_quantization.py new file mode 100644 index 00000000..3ce4a5ee --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_quantization.py @@ -0,0 +1,138 @@ +import json +import unittest +import warnings +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_QUANT_TP1, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + is_in_ci, + popen_launch_server, + write_github_step_summary, + write_results_to_json, +) + +MODEL_SCORE_THRESHOLDS = { + "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4": 0.825, + "hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4": 0.825, + "hugging-quants/Mixtral-8x7B-Instruct-v0.1-AWQ-INT4": 0.62, +} + + +def parse_models(model_string): + return [model.strip() for model in model_string.split(",") if model.strip()] + + +def popen_launch_server_wrapper(base_url, model, is_fp8, is_tp2): + other_args = ["--log-level-http", "warning", "--trust-remote-code"] + if is_fp8: + if "Llama-3" in model or "gemma-2" in model: + other_args.extend(["--kv-cache-dtype", "fp8_e5m2"]) + elif "Qwen2-72B-Instruct-FP8" in model: + other_args.extend(["--quantization", "fp8"]) + elif "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8" in model: + other_args.extend([]) + else: + other_args.extend(["--quantization", "fp8", "--kv-cache-dtype", "fp8_e5m2"]) + if is_tp2: + other_args.extend(["--tp", "2"]) + if "DeepSeek" in model: + other_args.extend(["--mem-frac", "0.85"]) + + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + return process + + +def check_model_scores(results): + failed_models = [] + summary = " | model | score | threshold |\n" + summary += "| ----- | ----- | --------- |\n" + + for model, score in results: + threshold = MODEL_SCORE_THRESHOLDS.get(model) + if threshold is None: + print(f"Warning: No threshold defined for model {model}") + continue + + if score < threshold: + failed_models.append( + f"\nScore Check Failed: {model}\n" + f"Model {model} score ({score:.4f}) is below threshold ({threshold:.4f})" + ) + + line = f"| {model} | {score} | {threshold} |\n" + summary += line + + print(summary) + + if is_in_ci(): + write_github_step_summary( + f"### TestNightlyGsm8KEval for awq, gptq, gguf\n{summary}" + ) + + if failed_models: + raise AssertionError("\n".join(failed_models)) + + +class TestNightlyGsm8KEval(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model_groups = [ + (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_QUANT_TP1), False, False), + ] + cls.base_url = DEFAULT_URL_FOR_TEST + + def test_mgsm_en_all_models(self): + warnings.filterwarnings( + "ignore", category=ResourceWarning, message="unclosed.*socket" + ) + is_first = True + all_results = [] + + for model_group, is_fp8, is_tp2 in self.model_groups: + for model in model_group: + with self.subTest(model=model): + process = popen_launch_server_wrapper( + self.base_url, model, is_fp8, is_tp2 + ) + + args = SimpleNamespace( + base_url=self.base_url, + model=model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + print( + f"{'=' * 42}\n{model} - metrics={metrics} score={metrics['score']}\n{'=' * 42}\n" + ) + + write_results_to_json(model, metrics, "w" if is_first else "a") + is_first = False + + all_results.append((model, metrics["score"])) + kill_process_tree(process.pid) + + try: + with open("results.json", "r") as f: + print("\nFinal Results from results.json:") + print(json.dumps(json.load(f), indent=2)) + except Exception as e: + print(f"Error reading results.json: {e}") + + # Check all scores after collecting all results + check_model_scores(all_results) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_radix_attention.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_radix_attention.py new file mode 100644 index 00000000..c8c3f539 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_radix_attention.py @@ -0,0 +1,87 @@ +import unittest + +from sglang.srt.environ import envs +from sglang.test.kits.radix_cache_server_kit import run_radix_attention_test +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + kill_process_tree, + popen_launch_server, +) + + +class TestRadixCacheFCFS(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--chunked-prefill-size", + "128", + "--max-total-tokens", + "20000", + "--schedule-policy", + "fcfs", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_radix_attention(self): + run_radix_attention_test(self.base_url) + + +@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") +class TestRadixCacheLPM(TestRadixCacheFCFS): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--chunked-prefill-size", + "128", + "--max-total-tokens", + "20000", + "--schedule-policy", + "lpm", + ], + ) + + +class TestRadixCacheNonOverlapLPM(TestRadixCacheFCFS): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--disable-overlap-schedule", + "--chunked-prefill-size", + "128", + "--max-total-tokens", + "20000", + "--schedule-policy", + "lpm", + ], + ) + + +if __name__ == "__main__": + envs.SGLANG_TEST_RETRACT.set(True) + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_radix_cache_unit.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_radix_cache_unit.py new file mode 100644 index 00000000..b41e2e88 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_radix_cache_unit.py @@ -0,0 +1,646 @@ +""" +Unit tests for the RadixCache implementation. + +This module tests the core functionality of RadixCache, RadixKey, and TreeNode +following SGLang testing patterns. + +Test Coverage: +- RadixKey: token ID management, slicing, iteration, representation +- TreeNode: node properties, reference counting, hash values +- RadixCache: insert/match operations, eviction, page alignment, error handling +- Cache events and request handling +- Boundary conditions with parameterized testing + +Usage: + python test_radix_cache_unit.py + python -m pytest test_radix_cache_unit.py -v + python -m pytest test_radix_cache_unit.py::TestRadixCache::test_insert_basic +""" + +import time +import unittest +import unittest.mock + +import torch + +from sglang.srt.disaggregation.kv_events import BlockRemoved, BlockStored +from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey, TreeNode + +# Test constants +DEFAULT_PAGE_SIZE = 4 + + +class TestRadixKey(unittest.TestCase): + """Test cases for RadixKey class.""" + + def test_init_basic(self): + """Test basic initialization of RadixKey.""" + token_ids = [1, 2, 3, 4] + key = RadixKey(token_ids) + self.assertEqual(key.token_ids, token_ids) + self.assertIsNone(key.extra_key) + + def test_init_with_extra_key(self): + """Test initialization with extra_key.""" + token_ids = [1, 2, 3] + extra_key = "test_key" + key = RadixKey(token_ids, extra_key) + self.assertEqual(key.token_ids, token_ids) + self.assertEqual(key.extra_key, extra_key) + + def test_len(self): + """Test __len__ method.""" + key = RadixKey([1, 2, 3]) + self.assertEqual(len(key), 3) + + empty_key = RadixKey([]) + self.assertEqual(len(empty_key), 0) + + def test_iter(self): + """Test __iter__ method.""" + token_ids = [1, 2, 3, 4] + key = RadixKey(token_ids) + self.assertEqual(list(key), token_ids) + + def test_len_and_iter(self): + """Test __len__ and __iter__ methods.""" + test_cases = [ + ([1, 2, 3], 3), + ([], 0), + ([42], 1), + ] + + for tokens, expected in test_cases: + with self.subTest(tokens=tokens): + key = RadixKey(tokens) + self.assertEqual(len(key), expected) + self.assertEqual(list(key), tokens) + + def test_getitem_int(self): + """Test __getitem__ with int index.""" + test_cases = [ + ([10, 20, 30], 0, [10]), + ([10, 20, 30], -1, [30]), + ([10, 20, 30], 2, [30]), + ] + + for tokens, index, expected in test_cases: + with self.subTest(tokens=tokens, index=index): + key = RadixKey(tokens) + result = key[index] + self.assertIsInstance(result, RadixKey) + self.assertEqual(result.token_ids, expected) + + def test_getitem_slice(self): + """Test __getitem__ with slice and edge cases.""" + key = RadixKey([1, 2, 3, 4, 5], "extra") + + # Basic slice + sliced = key[1:4] + self.assertIsInstance(sliced, RadixKey) + self.assertEqual(sliced.token_ids, [2, 3, 4]) + self.assertEqual(sliced.extra_key, "extra") + + # Edge cases + self.assertEqual(key[2:2].token_ids, []) # Empty slice + self.assertEqual(key[:].token_ids, [1, 2, 3, 4, 5]) # Full slice + + def test_getitem_invalid_index(self): + """Test __getitem__ with invalid indices.""" + key = RadixKey([1, 2, 3]) + with self.assertRaises(IndexError): + _ = key[10] # Out of bounds + + def test_repr(self): + """Test __repr__ method.""" + key = RadixKey([1, 2, 3], "test") + repr_str = repr(key) + self.assertIn("RadixKey", repr_str) + self.assertIn("extra_key='test'", repr_str) + self.assertIn("[1, 2, 3]", repr_str) + + def test_repr_long_token_ids(self): + """Test __repr__ with long token_ids.""" + long_tokens = list(range(15)) + key = RadixKey(long_tokens) + repr_str = repr(key) + self.assertIn("...", repr_str) # Should be truncated + + +class TestTreeNode(unittest.TestCase): + """Test cases for TreeNode class.""" + + def setUp(self): + """Reset the counter before each test.""" + TreeNode.counter = 0 + + def test_init_basic(self): + """Test basic initialization of TreeNode.""" + node = TreeNode() + self.assertEqual(node.id, 0) + self.assertEqual(len(node.children), 0) + self.assertIsNone(node.parent) + self.assertIsNone(node.key) + self.assertIsNone(node.value) + self.assertEqual(node.lock_ref, 0) + self.assertEqual(node.hit_count, 0) + self.assertEqual(node.host_ref_counter, 0) + self.assertIsNone(node.host_value) + self.assertIsNone(node.hash_value) + + def test_init_with_id(self): + """Test initialization with custom ID.""" + node = TreeNode(id=42) + self.assertEqual(node.id, 42) + node2 = TreeNode() + self.assertEqual(node2.id, 1) # Counter was incremented + + def test_counter_increment(self): + """Test that counter increments properly.""" + node1 = TreeNode() + node2 = TreeNode() + self.assertEqual(node1.id, 0) + self.assertEqual(node2.id, 1) + + def test_evicted_backuped_properties(self): + """Test evicted and backuped properties.""" + test_cases = [ + (False, False, True, False), + (True, False, False, False), + (True, True, False, True), + (False, True, True, True), + ] + + for ( + has_value, + has_host_value, + expected_evicted, + expected_backuped, + ) in test_cases: + with self.subTest(has_value=has_value, has_host_value=has_host_value): + node = TreeNode() + + if has_value: + node.value = torch.tensor([1, 2, 3]) + if has_host_value: + node.host_value = torch.tensor([4, 5, 6]) + + self.assertEqual(node.evicted, expected_evicted) + self.assertEqual(node.backuped, expected_backuped) + + def test_protect_release_host(self): + """Test protect_host and release_host methods.""" + node = TreeNode() + self.assertEqual(node.host_ref_counter, 0) + + node.protect_host() + self.assertEqual(node.host_ref_counter, 1) + + node.release_host() + self.assertEqual(node.host_ref_counter, 0) + + # Test error case + with self.assertRaises(RuntimeError): + node.release_host() + + def test_get_last_hash_value(self): + """Test get_last_hash_value method.""" + node = TreeNode() + self.assertIsNone(node.get_last_hash_value()) + + node.hash_value = ["hash1", "hash2", "hash3"] + self.assertEqual(node.get_last_hash_value(), "hash3") + + def test_lt_comparison(self): + """Test less than comparison based on last_access_time.""" + node1 = TreeNode() + time.sleep(0.001) # Small delay to ensure different timestamps + node2 = TreeNode() + + self.assertTrue(node1 < node2) + self.assertFalse(node2 < node1) + + +class TestRadixCache(unittest.TestCase): + """Test cases for RadixCache class.""" + + def setUp(self): + """Set up test fixtures.""" + TreeNode.counter = 0 + + def test_init_variations(self): + """Test cache initialization with different parameters.""" + test_cases = [ + (1, False, False), + (4, False, True), + (1, True, False), + ] + + for page_size, disable, enable_events in test_cases: + with self.subTest( + page_size=page_size, disable=disable, enable_events=enable_events + ): + cache = RadixCache.create_simulated( + disable=disable, + page_size=page_size, + enable_kv_cache_events=enable_events, + ) + + self.assertEqual(cache.page_size, page_size) + self.assertEqual(cache.disable, disable) + self.assertEqual(cache.enable_kv_cache_events, enable_events) + self.assertEqual(cache.device, torch.device("cpu")) + self.assertIsNotNone(cache.root_node) + self.assertEqual(len(cache.root_node.key), 0) + + def test_reset(self): + """Test reset method.""" + cache = RadixCache.create_simulated() + + # Insert some data + cache.insert(RadixKey([1, 2, 3]), torch.tensor([10, 20, 30], dtype=torch.int64)) + self.assertGreater(cache.total_size(), 0) + + # Reset + cache.reset() + self.assertEqual(cache.total_size(), 0) + self.assertEqual(cache.evictable_size(), 0) + self.assertEqual(cache.protected_size(), 0) + + def test_insert_and_match_basic(self): + """Test basic insert and match operations.""" + for disable_cache in [False, True]: + with self.subTest(disable_cache=disable_cache): + cache = RadixCache.create_simulated(disable=disable_cache) + + key = RadixKey([1, 2, 3]) + value = torch.tensor([10, 20, 30], dtype=torch.int64) + prefix_len = cache.insert(key, value) + + if disable_cache: + self.assertEqual(prefix_len, 0) + self.assertEqual(cache.total_size(), 0) + continue + + self.assertEqual(prefix_len, 0) # No existing prefix + self.assertEqual(cache.total_size(), 3) + self.assertEqual(cache.evictable_size(), 3) + + # Test match_prefix + result = cache.match_prefix(RadixKey([1, 2, 3])) + self.assertEqual(len(result.device_indices), 3) + torch.testing.assert_close(result.device_indices, value) + + # Test partial match + result = cache.match_prefix(RadixKey([1, 2])) + self.assertEqual(len(result.device_indices), 2) + torch.testing.assert_close( + result.device_indices, torch.tensor([10, 20], dtype=torch.int64) + ) + + def test_insert_with_none_value(self): + """Test insert with None value (should use token_ids as list).""" + cache = RadixCache.create_simulated() + + key = RadixKey([1, 2, 3]) + prefix_len = cache.insert(key, None) + + # When None is passed, it should create value from token_ids + self.assertEqual(prefix_len, 0) + self.assertEqual(cache.total_size(), 3) + + def test_total_size(self): + """Test total_size calculation.""" + cache = RadixCache.create_simulated() + + self.assertEqual(cache.total_size(), 0) + + cache.insert(RadixKey([1, 2, 3]), torch.tensor([10, 20, 30], dtype=torch.int64)) + self.assertEqual(cache.total_size(), 3) + + cache.insert(RadixKey([4, 5]), torch.tensor([40, 50], dtype=torch.int64)) + self.assertEqual(cache.total_size(), 5) + + def test_kv_cache_events(self): + """Test KV cache events functionality.""" + test_cases = [ + (1, True), + (2, True), + (1, False), + ] + + for page_size, enable_events in test_cases: + with self.subTest(page_size=page_size, enable_events=enable_events): + cache = RadixCache.create_simulated( + page_size=page_size, enable_kv_cache_events=enable_events + ) + + # Insert data + cache.insert(RadixKey([1, 2, 3, 4, 5]), None) + + # Take events + events = cache.take_events() + + if enable_events: + self.assertGreater(len(events), 0) + # Verify events include BlockStored events (there might be other event types) + block_stored_events = [ + e for e in events if isinstance(e, BlockStored) + ] + self.assertGreater(len(block_stored_events), 0) + for event in block_stored_events: + self.assertLessEqual(len(event.token_ids), page_size) + else: + self.assertEqual(len(events), 0) + + def test_kv_cache_events_with_eviction(self): + """Test KV cache events include removal events.""" + mock_allocator = unittest.mock.Mock() + mock_allocator.device = torch.device("cpu") + + cache = RadixCache.create_simulated( + mock_allocator=mock_allocator, enable_kv_cache_events=True + ) + + # Insert and then evict data + cache.insert(RadixKey([1, 2, 3]), torch.tensor([10, 20, 30], dtype=torch.int64)) + cache.evict(3) + + # Take events - should include both store and remove events + events = cache.take_events() + self.assertGreater(len(events), 0) + + # Check event types + event_types = [type(event).__name__ for event in events] + self.assertIn("BlockStored", event_types) + + # Verify BlockRemoved event content + remove_events = [e for e in events if isinstance(e, BlockRemoved)] + for event in remove_events: + self.assertGreater(len(event.block_hashes), 0) + + def test_extra_key_isolation(self): + """Test that keys with different extra_key values are isolated.""" + cache = RadixCache.create_simulated() + + # Insert same token sequence with different extra keys + cache.insert( + RadixKey([1, 2, 3], "key1"), torch.tensor([10, 20, 30], dtype=torch.int64) + ) + cache.insert( + RadixKey([1, 2, 3], "key2"), torch.tensor([40, 50, 60], dtype=torch.int64) + ) + cache.insert( + RadixKey([1, 2, 3], None), torch.tensor([70, 80, 90], dtype=torch.int64) + ) + + # Keys with different extra_key should not match each other + result1 = cache.match_prefix(RadixKey([1, 2, 3], "key1")) + result2 = cache.match_prefix(RadixKey([1, 2, 3], "key2")) + result3 = cache.match_prefix(RadixKey([1, 2, 3], None)) + result4 = cache.match_prefix(RadixKey([1, 2, 3], "nonexistent")) + + # Each should match only its own data + self.assertEqual(len(result1.device_indices), 3) + torch.testing.assert_close( + result1.device_indices, torch.tensor([10, 20, 30], dtype=torch.int64) + ) + + self.assertEqual(len(result2.device_indices), 3) + torch.testing.assert_close( + result2.device_indices, torch.tensor([40, 50, 60], dtype=torch.int64) + ) + + self.assertEqual(len(result3.device_indices), 3) + torch.testing.assert_close( + result3.device_indices, torch.tensor([70, 80, 90], dtype=torch.int64) + ) + + # Non-existent extra_key should not match + self.assertEqual(len(result4.device_indices), 0) + + def test_lock_ref_operations(self): + """Test lock reference counting operations.""" + cache = RadixCache.create_simulated() + + # Insert sequence + cache.insert(RadixKey([1, 2, 3]), torch.tensor([10, 20, 30], dtype=torch.int64)) + + # Get node + result = cache.match_prefix(RadixKey([1, 2, 3])) + node = result.last_device_node + + initial_evictable = cache.evictable_size() + initial_protected = cache.protected_size() + + # Lock the node + cache.inc_lock_ref(node) + self.assertEqual(cache.protected_size(), initial_protected + 3) + self.assertEqual(cache.evictable_size(), initial_evictable - 3) + + # Unlock the node + cache.dec_lock_ref(node) + self.assertEqual(cache.protected_size(), initial_protected) + self.assertEqual(cache.evictable_size(), initial_evictable) + + def test_evict_functionality(self): + """Test eviction functionality.""" + mock_allocator = unittest.mock.Mock() + mock_allocator.device = torch.device("cpu") + + cache = RadixCache.create_simulated(mock_allocator=mock_allocator) + + # Insert sequences + cache.insert(RadixKey([1, 2]), torch.tensor([10, 20], dtype=torch.int64)) + cache.insert(RadixKey([3, 4]), torch.tensor([30, 40], dtype=torch.int64)) + + initial_size = cache.total_size() + + # Evict some tokens + cache.evict(2) + + # Should have called free and reduced size + mock_allocator.free.assert_called() + self.assertLess(cache.total_size(), initial_size) + + def test_page_alignment_boundary(self): + """Test page alignment with different sizes.""" + test_cases = [ + (1, 5), + (2, 5), + (4, 6), + ] + + for page_size, sequence_length in test_cases: + with self.subTest(page_size=page_size, sequence_length=sequence_length): + cache = RadixCache.create_simulated(page_size=page_size) + + tokens = list(range(sequence_length)) + cache.insert(RadixKey(tokens), torch.tensor(tokens, dtype=torch.int64)) + + result = cache.match_prefix(RadixKey(tokens)) + self.assertGreater(len(result.device_indices), 0) + + # Match length should be page-aligned + match_len = len(result.device_indices) + self.assertEqual(match_len % page_size, 0) + + def test_pretty_print_basic(self): + """Test pretty_print produces output.""" + cache = RadixCache.create_simulated() + + cache.insert(RadixKey([1, 2, 3]), torch.tensor([10, 20, 30], dtype=torch.int64)) + + # Just test that it doesn't crash + try: + cache.pretty_print() + except Exception as e: + self.fail(f"pretty_print raised an exception: {e}") + + def test_all_values_flatten(self): + """Test all_values_flatten method.""" + cache = RadixCache.create_simulated() + + cache.insert(RadixKey([1, 2]), torch.tensor([10, 20], dtype=torch.int64)) + cache.insert(RadixKey([3, 4]), torch.tensor([30, 40], dtype=torch.int64)) + + all_values = cache.all_values_flatten() + self.assertEqual(len(all_values), 4) + # Values should contain all inserted values (order may vary) + values_set = set(all_values.tolist()) + self.assertEqual(values_set, {10, 20, 30, 40}) + + def test_advanced_prefix_match_with_node_splits(self): + """Advanced prefix matching: splits inside nodes and across pages.""" + for page_size in [1, 2]: + with self.subTest(page_size=page_size): + cache = RadixCache.create_simulated(page_size=page_size) + + # Insert a long sequence that will be split later. + seq1 = [1, 2, 3, 4, 5, 6, 7, 8] + val1 = torch.tensor([x * 10 for x in seq1], dtype=torch.int64) + cache.insert(RadixKey(seq1), val1) + + # Insert a diverging branch to create an internal node on the path. + seq2 = [1, 2, 9, 10] + val2 = torch.tensor([x * 10 for x in seq2], dtype=torch.int64) + cache.insert(RadixKey(seq2), val2) + print(cache.pretty_print()) + + baseline_total = cache.total_size() + expected_total = 10 # 8 + 2 + self.assertEqual(baseline_total, expected_total) + + # Match that causes a split inside an existing node: + # take first 4 tokens of seq1, then diverge. + query1 = [1, 2, 3, 4, 999, 1000] + result1 = cache.match_prefix(RadixKey(query1)) + torch.testing.assert_close(result1.device_indices, val1[:4]) + # No data change after structural split during matching. + self.assertEqual(cache.total_size(), baseline_total) + + # Full match of the long sequence still returns the full indices. + result_full = cache.match_prefix(RadixKey(seq1)) + torch.testing.assert_close(result_full.device_indices, val1) + + # Another split deeper on the path (after matching 6 tokens, then diverge). + query2 = [1, 2, 3, 4, 5, 6, 777, 888] + result2 = cache.match_prefix(RadixKey(query2)) + torch.testing.assert_close(result2.device_indices, val1[:6]) + self.assertEqual(cache.total_size(), baseline_total) + + # Matching the short diverging branch should return exactly its indices. + result_branch = cache.match_prefix(RadixKey(seq2)) + torch.testing.assert_close(result_branch.device_indices, val2) + + def test_hash_value_storage(self): + """Test that hash_value is stored correctly after insert operations.""" + cache = RadixCache.create_simulated( + page_size=4, + enable_kv_cache_events=True, + ) + + # Insert a sequence + cache.insert(RadixKey([1, 2, 3, 4, 5, 6, 7, 8]), None) + + # Trigger event emission to compute hash_value lazily + cache.take_events() + + # Find the inserted node (traverse from root) + node = cache.root_node + for i in range(0, 8, 4): # page_size=4, so 2 pages + child_key = tuple([1, 2, 3, 4][:4]) if i == 0 else tuple([5, 6, 7, 8][:4]) + if child_key in node.children: + node = node.children[child_key] + break + + # Verify hash_value is set (computed lazily during event emission) + self.assertIsNotNone(node.hash_value) + # Should have 2 pages (8 tokens / 4 page_size) + self.assertEqual(len(node.hash_value), 2) + + def test_hash_value_repeating_tokens(self): + """Test that repeating token patterns get different hash values.""" + cache = RadixCache.create_simulated( + page_size=4, + enable_kv_cache_events=True, + ) + + # Insert a sequence with repeating token pattern: [1,2,3,4, 1,2,3,4] + cache.insert(RadixKey([1, 2, 3, 4, 1, 2, 3, 4]), None) + + events = cache.take_events() + block_stored_events = [e for e in events if isinstance(e, BlockStored)] + + # Should have 2 blocks (2 pages of size 4) + self.assertEqual(len(block_stored_events), 2) + + # Extract block hashes + block_hash_1 = block_stored_events[0].block_hashes[0] + block_hash_2 = block_stored_events[1].block_hashes[0] + + # The two blocks should have DIFFERENT hashes despite same content + # because they are at different positions (sequence-aware hashing) + self.assertNotEqual( + block_hash_1, + block_hash_2, + "Repeating token patterns should get different sequence-aware hashes", + ) + + # First block should have no parent + self.assertIsNone(block_stored_events[0].parent_block_hash) + + # Second block's parent should be the first block's hash + self.assertEqual(block_stored_events[1].parent_block_hash, block_hash_1) + + def test_hash_value_split(self): + """Test that hash_value is split correctly when nodes are split.""" + cache = RadixCache.create_simulated( + page_size=2, + enable_kv_cache_events=True, + ) + + # Insert a sequence that will cause a split + cache.insert(RadixKey([1, 2, 3, 4]), None) + cache.take_events() # Clear events and compute hash_value for first node + + # Insert a diverging sequence that will cause a split at page boundary + cache.insert(RadixKey([1, 2, 5, 6]), None) + cache.take_events() # Trigger event emission to compute hash_value + + # Find the split node + node = cache.root_node + child_key = tuple([1, 2]) + if child_key in node.children: + node = node.children[child_key] + # After split and event emission, hash_value should be computed + # Note: If hash_value wasn't set before split, it will be computed lazily + # during event emission. If it was set, it will be split. + # Either way, after events are emitted, it should be set. + self.assertIsNotNone(node.hash_value) + # Should have 1 page (split at page_size=2) + self.assertEqual(len(node.hash_value), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_reasoning_parser.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_reasoning_parser.py new file mode 100644 index 00000000..7d3f2a13 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_reasoning_parser.py @@ -0,0 +1,603 @@ +import unittest + +from sglang.srt.parser.reasoning_parser import ( + BaseReasoningFormatDetector, + DeepSeekR1Detector, + KimiDetector, + Qwen3Detector, + ReasoningParser, + StreamingParseResult, +) +from sglang.test.test_utils import CustomTestCase + + +class TestStreamingParseResult(CustomTestCase): + def test_init_default(self): + """Test default initialization of StreamingParseResult.""" + result = StreamingParseResult() + self.assertEqual(result.normal_text, "") + self.assertEqual(result.reasoning_text, "") + + def test_init_with_values(self): + """Test initialization with specific values.""" + result = StreamingParseResult("normal", "reasoning") + self.assertEqual(result.normal_text, "normal") + self.assertEqual(result.reasoning_text, "reasoning") + + +class TestBaseReasoningFormatDetector(CustomTestCase): + def setUp(self): + self.detector = BaseReasoningFormatDetector( + think_start_token="", + think_end_token="", + force_reasoning=False, + stream_reasoning=True, + ) + + def test_init(self): + """Test initialization of BaseReasoningFormatDetector.""" + self.assertEqual(self.detector.think_start_token, "") + self.assertEqual(self.detector.think_end_token, "") + self.assertFalse(self.detector._in_reasoning) + self.assertTrue(self.detector.stream_reasoning) + self.assertEqual(self.detector._buffer, "") + self.assertFalse(self.detector.stripped_think_start) + + def test_detect_and_parse_normal_text(self): + """Test parsing normal text without reasoning.""" + text = "This is normal text" + result = self.detector.detect_and_parse(text) + self.assertEqual(result.normal_text, text) + self.assertEqual(result.reasoning_text, "") + + def test_detect_and_parse_with_start_token(self): + """Test parsing text starting with think token.""" + text = "This is reasoning" + result = self.detector.detect_and_parse(text) + self.assertEqual(result.reasoning_text, "This is reasoning") + self.assertEqual(result.normal_text, "") + + def test_detect_and_parse_complete_reasoning(self): + """Test parsing complete reasoning block.""" + text = "This is reasoningThis is normal" + result = self.detector.detect_and_parse(text) + self.assertEqual(result.reasoning_text, "This is reasoning") + self.assertEqual(result.normal_text, "This is normal") + + def test_detect_and_parse_force_reasoning(self): + """Test forced reasoning mode.""" + detector = BaseReasoningFormatDetector( + "", "", force_reasoning=True + ) + text = "This should be reasoning" + result = detector.detect_and_parse(text) + self.assertEqual(result.reasoning_text, "This should be reasoning") + self.assertEqual(result.normal_text, "") + + def test_parse_streaming_increment_normal(self): + """Test streaming parse of normal text.""" + result = self.detector.parse_streaming_increment("Hello world") + self.assertEqual(result.normal_text, "Hello world") + self.assertEqual(result.reasoning_text, "") + + def test_parse_streaming_increment_partial_token(self): + """Test streaming parse with partial token.""" + # Test partial start token + result = self.detector.parse_streaming_increment("", "") + detector._in_reasoning = True + result = detector.parse_streaming_increment("") + self.assertEqual(result.normal_text, "") + self.assertEqual(result.reasoning_text, "") + self.assertTrue(self.detector._in_reasoning) + self.assertTrue(self.detector.stripped_think_start) + + def test_parse_streaming_increment_reasoning_content(self): + """Test streaming parse of reasoning content.""" + # First add start token + self.detector.parse_streaming_increment("") + + # Then add reasoning content + result = self.detector.parse_streaming_increment("reasoning content") + self.assertEqual(result.reasoning_text, "reasoning content") + self.assertEqual(result.normal_text, "") + + def test_parse_streaming_increment_end_token(self): + """Test streaming parse with end token.""" + # Start reasoning mode + self.detector.parse_streaming_increment("") + self.detector.parse_streaming_increment("reasoning") + + # End reasoning - the reasoning content accumulated in previous calls is cleared when end token is found + result = self.detector.parse_streaming_increment("normal text") + self.assertEqual(result.reasoning_text, "") # Buffer cleared, returns empty + self.assertEqual(result.normal_text, "normal text") + self.assertFalse(self.detector._in_reasoning) + + def test_parse_streaming_increment_no_stream_reasoning(self): + """Test streaming parse without streaming reasoning.""" + detector = BaseReasoningFormatDetector( + "", "", stream_reasoning=False + ) + + # Start reasoning mode + detector.parse_streaming_increment("") + + # Add reasoning content - should not return content + result = detector.parse_streaming_increment("reasoning content") + self.assertEqual(result.reasoning_text, "") + self.assertEqual(result.normal_text, "") + + def test_parse_streaming_increment_mixed_content(self): + """Test streaming parse with mixed content in one chunk.""" + result = self.detector.parse_streaming_increment( + "reasoningnormal" + ) + self.assertEqual(result.reasoning_text, "reasoning") + self.assertEqual(result.normal_text, "normal") + + +class TestDeepSeekR1Detector(CustomTestCase): + def setUp(self): + self.detector = DeepSeekR1Detector() + + def test_init(self): + """Test DeepSeekR1Detector initialization.""" + self.assertEqual(self.detector.think_start_token, "") + self.assertEqual(self.detector.think_end_token, "") + self.assertTrue(self.detector._in_reasoning) # force_reasoning=True + self.assertTrue(self.detector.stream_reasoning) + + def test_init_no_stream_reasoning(self): + """Test DeepSeekR1Detector with stream_reasoning=False.""" + detector = DeepSeekR1Detector(stream_reasoning=False) + self.assertFalse(detector.stream_reasoning) + + def test_detect_and_parse_r1_format(self): + """Test parsing DeepSeek-R1 format.""" + text = "I need to think about this. The answer is 42." + result = self.detector.detect_and_parse(text) + # Should be treated as reasoning because force_reasoning=True + self.assertEqual( + result.reasoning_text, "I need to think about this. The answer is 42." + ) + self.assertEqual(result.normal_text, "") + + def test_detect_and_parse_with_end_token(self): + """Test parsing with end token.""" + text = "I think this is the answerThe final answer is 42." + result = self.detector.detect_and_parse(text) + self.assertEqual(result.reasoning_text, "I think this is the answer") + self.assertEqual(result.normal_text, "The final answer is 42.") + + def test_detect_and_parse_with_start_token(self): + """Test parsing deepseek-ai/DeepSeek-R1-0528 format, which generates the token.""" + text = "I need to think about this.The answer is 42." + result = self.detector.detect_and_parse(text) + # Should be treated as reasoning because force_reasoning=True + self.assertEqual(result.reasoning_text, "I need to think about this.") + self.assertEqual(result.normal_text, "The answer is 42.") + + +class TestQwen3Detector(CustomTestCase): + def setUp(self): + self.detector = Qwen3Detector() + + def test_init(self): + """Test Qwen3Detector initialization.""" + self.assertEqual(self.detector.think_start_token, "") + self.assertEqual(self.detector.think_end_token, "") + self.assertFalse(self.detector._in_reasoning) # force_reasoning=False + self.assertTrue(self.detector.stream_reasoning) + + def test_detect_and_parse_qwen3_format(self): + """Test parsing Qwen3 format.""" + text = "Let me think about this problemThe answer is 42." + result = self.detector.detect_and_parse(text) + self.assertEqual(result.reasoning_text, "Let me think about this problem") + self.assertEqual(result.normal_text, "The answer is 42.") + + def test_detect_and_parse_without_thinking(self): + """Test parsing without thinking (enable_thinking=False case).""" + text = "Direct answer without thinking." + result = self.detector.detect_and_parse(text) + self.assertEqual(result.normal_text, text) + self.assertEqual(result.reasoning_text, "") + + +class TestQwen3ForcedReasoningDetector(CustomTestCase): + def setUp(self): + self.detector = Qwen3Detector(force_reasoning=True) + + def test_init(self): + """Test Qwen3ForcedReasoningDetector initialization.""" + self.assertEqual(self.detector.think_start_token, "") + self.assertEqual(self.detector.think_end_token, "") + self.assertTrue(self.detector._in_reasoning) # force_reasoning=True + self.assertTrue(self.detector.stream_reasoning) + + def test_detect_and_parse_qwen3_forced_reasoning_format(self): + """Test parsing Qwen3-ForcedReasoning format (no start tag).""" + text = "I need to think about this step by step.The answer is 42." + result = self.detector.detect_and_parse(text) + self.assertEqual( + result.reasoning_text, "I need to think about this step by step." + ) + self.assertEqual(result.normal_text, "The answer is 42.") + + def test_detect_and_parse_with_start_token(self): + """Test parsing Qwen3-ForcedReasoning with optional start tag.""" + text = "I need to think about this.The answer is 42." + result = self.detector.detect_and_parse(text) + # Should work because base class logic handles both force_reasoning=True OR start token + self.assertEqual(result.reasoning_text, "I need to think about this.") + self.assertEqual(result.normal_text, "The answer is 42.") + + def test_streaming_qwen3_forced_reasoning_format(self): + """Test streaming parse of Qwen3-ForcedReasoning format.""" + # First chunk without start + result = self.detector.parse_streaming_increment("I need to") + self.assertEqual(result.reasoning_text, "I need to") + self.assertEqual(result.normal_text, "") + + # More reasoning content + result = self.detector.parse_streaming_increment(" think about this.") + self.assertEqual(result.reasoning_text, " think about this.") + self.assertEqual(result.normal_text, "") + + # End token with normal text + result = self.detector.parse_streaming_increment("The answer is 42.") + self.assertEqual(result.reasoning_text, "") # Buffer cleared + self.assertEqual(result.normal_text, "The answer is 42.") + + +class TestKimiDetector(CustomTestCase): + def setUp(self): + self.detector = KimiDetector() + + def test_init(self): + """Test KimiDetector initialization.""" + self.assertEqual(self.detector.think_start_token, "◁think▷") + self.assertEqual(self.detector.think_end_token, "◁/think▷") + self.assertFalse(self.detector._in_reasoning) + self.assertTrue(self.detector.stream_reasoning) + + def test_detect_and_parse_kimi_format(self): + """Test parsing Kimi format.""" + text = "◁think▷Let me consider this carefully◁/think▷The answer is 42." + result = self.detector.detect_and_parse(text) + self.assertEqual(result.reasoning_text, "Let me consider this carefully") + self.assertEqual(result.normal_text, "The answer is 42.") + + def test_detect_and_parse_kimi_no_thinking(self): + """Test parsing Kimi format without thinking.""" + text = "Direct answer without thinking tokens." + result = self.detector.detect_and_parse(text) + self.assertEqual(result.normal_text, text) + self.assertEqual(result.reasoning_text, "") + + def test_streaming_kimi_format(self): + """Test streaming parse of Kimi format.""" + # Test partial token + result = self.detector.parse_streaming_increment("◁thi") + self.assertEqual(result.normal_text, "") + self.assertEqual(result.reasoning_text, "") + + # Complete start token + result = self.detector.parse_streaming_increment("nk▷Start") + self.assertEqual(result.normal_text, "") + self.assertEqual(result.reasoning_text, "Start") + self.assertTrue(self.detector._in_reasoning) + + # Add reasoning content + result = self.detector.parse_streaming_increment("thinking...") + self.assertEqual(result.reasoning_text, "thinking...") + self.assertEqual(result.normal_text, "") + + # End token - reasoning content is cleared when end token is processed + result = self.detector.parse_streaming_increment("◁/think▷answer") + self.assertEqual(result.reasoning_text, "") # Buffer cleared + self.assertEqual(result.normal_text, "answer") + + +class TestReasoningParser(CustomTestCase): + def test_init_valid_model(self): + """Test initialization with valid model types.""" + parser = ReasoningParser("deepseek-r1") + self.assertIsInstance(parser.detector, DeepSeekR1Detector) + + parser = ReasoningParser("qwen3") + self.assertIsInstance(parser.detector, Qwen3Detector) + + parser = ReasoningParser("kimi") + self.assertIsInstance(parser.detector, KimiDetector) + + def test_init_invalid_model(self): + """Test initialization with invalid model type.""" + with self.assertRaises(ValueError) as context: + ReasoningParser("invalid-model") + self.assertIn("Unsupported model type", str(context.exception)) + + def test_init_no_model(self): + """Test initialization without model type.""" + with self.assertRaises(ValueError) as context: + ReasoningParser(None) + self.assertEqual(str(context.exception), "Model type must be specified") + + def test_parse_non_stream(self): + """Test non-streaming parsing.""" + parser = ReasoningParser("qwen3") + reasoning, normal = parser.parse_non_stream( + "Let me thinkThe answer is 42." + ) + self.assertEqual(reasoning, "Let me think") + self.assertEqual(normal, "The answer is 42.") + + def test_parse_stream_chunk(self): + """Test streaming chunk parsing.""" + parser = ReasoningParser("qwen3") + + # First chunk with start token + reasoning, normal = parser.parse_stream_chunk("") + self.assertEqual(reasoning, "") + self.assertEqual(normal, "") + + # Second chunk with reasoning content + reasoning, normal = parser.parse_stream_chunk("thinking...") + self.assertEqual(reasoning, "thinking...") + self.assertEqual(normal, "") + + # Third chunk with end token and normal text + reasoning, normal = parser.parse_stream_chunk("answer") + self.assertEqual(reasoning, "") # Buffer cleared when end token processed + self.assertEqual(normal, "answer") + + def test_case_insensitive_model_type(self): + """Test case insensitive model type matching.""" + parser1 = ReasoningParser("DeepSeek-R1") + parser2 = ReasoningParser("QWEN3") + parser3 = ReasoningParser("Kimi") + + self.assertIsInstance(parser1.detector, DeepSeekR1Detector) + self.assertIsInstance(parser2.detector, Qwen3Detector) + self.assertIsInstance(parser3.detector, KimiDetector) + + def test_stream_reasoning_parameter(self): + """Test stream_reasoning parameter is passed correctly.""" + parser = ReasoningParser("qwen3", stream_reasoning=False) + self.assertFalse(parser.detector.stream_reasoning) + + parser = ReasoningParser("qwen3", stream_reasoning=True) + self.assertTrue(parser.detector.stream_reasoning) + + +class TestIntegrationScenarios(CustomTestCase): + """Integration tests for realistic usage scenarios.""" + + def test_deepseek_r1_complete_response(self): + """Test complete DeepSeek-R1 response parsing.""" + parser = ReasoningParser("deepseek-r1") + text = "I need to solve this step by step. First, I'll analyze the problem. The given equation is x + 2 = 5. To solve for x, I subtract 2 from both sides: x = 5 - 2 = 3.The answer is x = 3." + + reasoning, normal = parser.parse_non_stream(text) + self.assertIn("step by step", reasoning) + self.assertIn( + "= 3", reasoning + ) # The reasoning contains "x = 5 - 2 = 3" which has "= 3" + self.assertEqual(normal, "The answer is x = 3.") + + def test_qwen3_streaming_scenario(self): + """Test Qwen3 streaming scenario.""" + parser = ReasoningParser("qwen3") + + chunks = [ + "", + "Let me analyze this problem.", + " I need to consider multiple factors.", + "", + "Based on my analysis, the solution is to use a different approach.", + ] + + all_reasoning = "" + all_normal = "" + + for chunk in chunks: + reasoning, normal = parser.parse_stream_chunk(chunk) + all_reasoning += reasoning + all_normal += normal + + self.assertIn("analyze", all_reasoning) + self.assertIn("multiple factors", all_reasoning) + self.assertIn("different approach", all_normal) + + def test_kimi_streaming_scenario(self): + """Test Kimi streaming scenario.""" + parser = ReasoningParser("kimi") + chunks = [ + "◁thi", + "nk▷", + "Let me analyze this problem.", + " I need to consider multiple factors.", + "◁/th", + "ink▷", + "The answer is 42.", + ] + all_reasoning = "" + all_normal = "" + for chunk in chunks: + reasoning, normal = parser.parse_stream_chunk(chunk) + all_reasoning += reasoning + all_normal += normal + + self.assertIn("analyze", all_reasoning) + self.assertIn("multiple factors", all_reasoning) + self.assertIn("42", all_normal) + + def test_empty_reasoning_blocks(self): + """Test handling of empty reasoning blocks.""" + parser = ReasoningParser("qwen3") + text = "Just the answer." + + reasoning, normal = parser.parse_non_stream(text) + self.assertEqual(reasoning, "") + self.assertEqual(normal, "Just the answer.") + + def test_qwen3_forced_reasoning_complete_response(self): + """Test complete Qwen3-ForcedReasoning response parsing.""" + parser = ReasoningParser("qwen3", force_reasoning=True) + text = "Let me solve this step by step. The equation is x + 2 = 5. Subtracting 2 from both sides gives x = 3.The solution is x = 3." + + reasoning, normal = parser.parse_non_stream(text) + self.assertIn("step by step", reasoning) + self.assertIn("x = 3", reasoning) + self.assertEqual(normal, "The solution is x = 3.") + + def test_qwen3_forced_reasoning_streaming_scenario(self): + """Test Qwen3-ForcedReasoning streaming scenario.""" + parser = ReasoningParser("qwen3", force_reasoning=True) + + chunks = [ + "I need to analyze", + " this problem carefully.", + " Let me break it down.", + "", + "The final answer is 42.", + ] + + all_reasoning = "" + all_normal = "" + + for chunk in chunks: + reasoning, normal = parser.parse_stream_chunk(chunk) + all_reasoning += reasoning + all_normal += normal + + self.assertIn("analyze", all_reasoning) + self.assertIn("break it down", all_reasoning) + self.assertIn("final answer", all_normal) + + +class TestBufferLossBugFix(CustomTestCase): + """Test cases for the buffer loss bug fix in parse_streaming_increment.""" + + def test_partial_end_tag_buffer_loss_bug(self): + """ + Test the bug where partial end tag fragments are lost when followed by normal text. + + Bug scenario: + 1. _in_reasoning is False + 2. new_text is "", "") + + # Step 1: Send partial end tag when not in reasoning mode + # This should be buffered since it could be start of "" + result1 = detector.parse_streaming_increment("", "") + + # Send partial start tag + result1 = detector.parse_streaming_increment("", "") + + # Enter reasoning mode + detector.parse_streaming_increment("") + detector.parse_streaming_increment("some reasoning") + + # Send partial end tag + result1 = detector.parse_streaming_increment("normal text") + self.assertEqual(result2.normal_text, "normal text") + # The reasoning text should be empty since buffer was cleared when end tag was processed + self.assertEqual(result2.reasoning_text, "") + + def test_multiple_partial_fragments(self): + """ + Test handling of multiple partial fragments that don't match any tokens. + """ + detector = BaseReasoningFormatDetector("", "") + + # Send multiple partial fragments + result1 = detector.parse_streaming_increment("<") + self.assertEqual(result1.normal_text, "") + self.assertEqual(result1.reasoning_text, "") + + result2 = detector.parse_streaming_increment("/") + self.assertEqual(result2.normal_text, "") + self.assertEqual(result2.reasoning_text, "") + + result3 = detector.parse_streaming_increment("random>") + self.assertEqual(result3.normal_text, "") + self.assertEqual(result3.reasoning_text, "") + + def test_edge_case_exact_token_match(self): + """ + Test edge case where buffer content exactly matches a token. + """ + detector = BaseReasoningFormatDetector("", "") + + # Build up the exact start token character by character + detector.parse_streaming_increment("<") + detector.parse_streaming_increment("t") + detector.parse_streaming_increment("h") + detector.parse_streaming_increment("i") + detector.parse_streaming_increment("n") + result = detector.parse_streaming_increment("k>") + + # Should enter reasoning mode + self.assertEqual(result.normal_text, "") + self.assertEqual(result.reasoning_text, "") + self.assertTrue(detector._in_reasoning) + self.assertTrue(detector.stripped_think_start) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_release_memory_occupation.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_release_memory_occupation.py new file mode 100644 index 00000000..12bc3093 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_release_memory_occupation.py @@ -0,0 +1,405 @@ +"""Test memory release and resume operations for SGLang engine in hybrid RL training. + +This test suite evaluates the SGLang engine's memory management capabilities, focusing +on releasing and resuming memory occupation for KV cache and model weights. It simulates +an RL workflow where the SGLang engine acts as a rollout engine for experience collection. +The process involves initializing the engine, sending a small number of requests to simulate +rollout, releasing memory to mimic offloading during RL training, resuming memory occupation, +updating weights with a trained HuggingFace model, and verifying the updated weights. + +Detailed in our proposal (https://github.com/sgl-project/sglang/pull/7099), two test cases +are included: + +1. Basic Release and Resume: Uses a lower mem_fraction_static (0.6) to control memory allocation +and avoid OOM errors carefully. This test simulates a scenario without multi-stage memory management, +ensuring the engine can release and resume memory occupation while maintaining functionality after +weight updates. + +2. Multi-Stage Release and Resume: Employs a higher mem_fraction_static (0.85) to simulate higher +memory pressure, leveraging multi-stage memory management. It sequentially releases and resumes +KV cache and model weights, verifying memory deallocation and reallocation at each stage, and +ensuring correct weight updates and text generation. + +3. Tensor Parallel Tests: Tests memory release and resume operations with different tensor parallel +configurations (tp=1, tp=2) to ensure proper memory management in distributed settings. For different +data parallel size, we test it in verl. +""" + +import os +import time +import unittest + +import torch +from transformers import AutoModelForCausalLM + +import sglang as sgl +from sglang.srt.constants import ( + GPU_MEMORY_TYPE_CUDA_GRAPH, + GPU_MEMORY_TYPE_KV_CACHE, + GPU_MEMORY_TYPE_WEIGHTS, +) +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST_BASE, + DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_BASE, + DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_CHAT, + CustomTestCase, +) + +# (temporarily) set to true to observe memory usage in nvidia-smi more clearly +_DEBUG_EXTRA = False + + +def get_gpu_memory_gb(): + return torch.cuda.device_memory_used() / 1024**3 + + +class TestReleaseMemoryOccupation(CustomTestCase): + def _setup_engine( + self, + model_name, + mem_fraction_static=0.8, + tp_size=1, + ep_size=1, + enable_weights_cpu_backup=False, + ): + """Common setup for engine and HF model.""" + + os.environ["SGLANG_MEMORY_SAVER_CUDA_GRAPH"] = "1" + engine = sgl.Engine( + model_path=model_name, + random_seed=42, + enable_memory_saver=True, + mem_fraction_static=mem_fraction_static, + tp_size=tp_size, + ep_size=ep_size, + enable_weights_cpu_backup=enable_weights_cpu_backup, + # disable_cuda_graph=True, # for debugging only + ) + + return engine + + def _common_test_params(self): + """Common test parameters.""" + return { + "prompt": "Today is a sunny day and I like", + "sampling_params": {"temperature": 0, "max_new_tokens": 8}, + "expect_output_before_update_weights": " to spend it outdoors. I decided to", + "expect_output_after_update_weights": " to go for a walk. I like", + "prompt_moe": "The weather is nice today, and I want to", + "sampling_params_moe": {"temperature": 0, "max_new_tokens": 16}, + "expect_output_before_update_weights_moe": " go to the park. I have a picnic basket, a book, and a", + "expect_output_after_update_weights_moe": " go to the park. I have a lot of things to do, but I", + } + + def _test_initial_generation( + self, engine, prompt, sampling_params, expect_output_before_update_weights + ): + """Test initial generation and memory allocation.""" + print("generate (#1)") + outputs = engine.generate(prompt, sampling_params)["text"] + self.assertEqual(outputs, expect_output_before_update_weights) + + if _DEBUG_EXTRA: + time.sleep(3) + + def test_release_and_resume_occupation(self): + # Without multi-stage release and resume, we need to carefully control the memory fraction to avoid OOM + model_name = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + assert ( + torch.cuda.device_count() >= 2 + ), "Need at least 2 GPUs for tensor parallel tests" + + for tp_size in [1, 2]: + + print(f"Testing tp_size={tp_size} for test_release_and_resume_occupation") + engine = self._setup_engine( + model_name=model_name, mem_fraction_static=0.6, tp_size=tp_size + ) + params = self._common_test_params() + + self._test_initial_generation( + engine, + params["prompt"], + params["sampling_params"], + params["expect_output_before_update_weights"], + ) + + t = time.perf_counter() + gpu_memory_usage_before_release = get_gpu_memory_gb() + engine.release_memory_occupation() + gpu_memory_usage_after_release = get_gpu_memory_gb() + + self.assertLess( + gpu_memory_usage_after_release, + gpu_memory_usage_before_release, + ) + + print( + f"Release took {time.perf_counter() - t:.2f}s, memory: {gpu_memory_usage_before_release:.1f} GB → {gpu_memory_usage_after_release:.1f} GB" + ) + + if _DEBUG_EXTRA: + time.sleep(3) + + t = time.perf_counter() + engine.resume_memory_occupation() + print( + f"Resume took {time.perf_counter() - t:.2f}s, memory: {get_gpu_memory_gb():.1f} GB" + ) + + hf_model_new = AutoModelForCausalLM.from_pretrained( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST_BASE, + torch_dtype="bfloat16", + device_map="cuda", + ) + engine.update_weights_from_tensor(list(hf_model_new.named_parameters())) + + # destroy the hf model + del hf_model_new + torch.cuda.empty_cache() + + print("generate (#2)") + outputs = engine.generate(params["prompt"], params["sampling_params"])[ + "text" + ] + self.assertEqual(outputs, params["expect_output_after_update_weights"]) + engine.shutdown() + + def test_release_and_resume_occupation_with_weights_cpu_backup(self): + # Test release and resume occupation with weights CPU backup + model_name = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + + print("Testing test_release_and_resume_occupation_with_weights_cpu_backup") + engine = self._setup_engine( + model_name=model_name, + mem_fraction_static=0.6, + enable_weights_cpu_backup=True, + ) + params = self._common_test_params() + + self._test_initial_generation( + engine, + params["prompt"], + params["sampling_params"], + params["expect_output_before_update_weights"], + ) + + t = time.perf_counter() + gpu_memory_usage_before_release = get_gpu_memory_gb() + engine.release_memory_occupation() + gpu_memory_usage_after_release = get_gpu_memory_gb() + + self.assertLess( + gpu_memory_usage_after_release, + gpu_memory_usage_before_release, + ) + + print( + f"Release took {time.perf_counter() - t:.2f}s, memory: {gpu_memory_usage_before_release:.1f} GB → {gpu_memory_usage_after_release:.1f} GB" + ) + + if _DEBUG_EXTRA: + time.sleep(3) + + t = time.perf_counter() + engine.resume_memory_occupation() + print( + f"Resume took {time.perf_counter() - t:.2f}s, memory: {get_gpu_memory_gb():.1f} GB" + ) + + print("generate post resume") + outputs = engine.generate(params["prompt"], params["sampling_params"])["text"] + self.assertEqual(outputs, params["expect_output_before_update_weights"]) + engine.shutdown() + + def test_multi_stage_release_and_resume(self): + # With multi-stage release and resume, we can set the memory fraction to 0.85 without concern of OOM + model_name = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + + for tp_size in [1, 2]: + if tp_size == 2 and torch.cuda.device_count() < 2: + continue + + print(f"Testing tp_size={tp_size} for test_multi_stage_release_and_resume") + os.environ["SGLANG_MEMORY_SAVER_CUDA_GRAPH"] = "1" + engine = sgl.Engine( + model_path=model_name, + random_seed=42, + enable_memory_saver=True, + mem_fraction_static=0.85, # Higher memory pressure + tp_size=tp_size, + ) + params = self._common_test_params() + + self._test_initial_generation( + engine, + params["prompt"], + params["sampling_params"], + params["expect_output_before_update_weights"], + ) + + t = time.perf_counter() + gpu_memory_usage_before_release = get_gpu_memory_gb() + engine.release_memory_occupation(tags=[GPU_MEMORY_TYPE_KV_CACHE]) + + gpu_memory_usage_after_release_kv_cache = get_gpu_memory_gb() + + self.assertLess( + gpu_memory_usage_after_release_kv_cache, + gpu_memory_usage_before_release, + ) + + engine.release_memory_occupation(tags=[GPU_MEMORY_TYPE_WEIGHTS]) + gpu_memory_usage_after_release_weights = get_gpu_memory_gb() + + self.assertLess( + gpu_memory_usage_after_release_weights, + gpu_memory_usage_after_release_kv_cache, + ) + + engine.release_memory_occupation(tags=[GPU_MEMORY_TYPE_CUDA_GRAPH]) + gpu_memory_usage_after_release_cuda_graph = get_gpu_memory_gb() + + self.assertLess( + gpu_memory_usage_after_release_cuda_graph, + gpu_memory_usage_after_release_weights, + ) + + print(f"Release took {time.perf_counter() - t:.2f}s") + print( + f"Memory: {gpu_memory_usage_before_release:.1f} → {gpu_memory_usage_after_release_kv_cache:.1f} → {gpu_memory_usage_after_release_weights:.1f} → {gpu_memory_usage_after_release_cuda_graph:.1f} GB" + ) + + if _DEBUG_EXTRA: + time.sleep(3) + + t = time.perf_counter() + gpu_memory_usage_before_resume = get_gpu_memory_gb() + + # gpu_memory_usage_after_release_weights and gpu_memory_usage_before_resume should be close + + self.assertAlmostEqual( + gpu_memory_usage_after_release_weights, + gpu_memory_usage_before_resume, + delta=3.0, + ) + print(f"Resume weights took {time.perf_counter() - t:.2f}s") + + engine.resume_memory_occupation(tags=[GPU_MEMORY_TYPE_CUDA_GRAPH]) + gpu_memory_usage_after_resume_cuda_graph = get_gpu_memory_gb() + + self.assertGreater( + gpu_memory_usage_after_resume_cuda_graph, + gpu_memory_usage_before_resume, + ) + + engine.resume_memory_occupation(tags=[GPU_MEMORY_TYPE_WEIGHTS]) + gpu_memory_usage_after_resume_weights = get_gpu_memory_gb() + + self.assertGreater( + gpu_memory_usage_after_resume_weights, + gpu_memory_usage_after_resume_cuda_graph, + ) + + # Update weights from a trained model to serving engine, and then destroy the trained model + hf_model_new = AutoModelForCausalLM.from_pretrained( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST_BASE, + torch_dtype="bfloat16", + device_map="cuda", + ) + gpu_memory_usage_after_loaded_hf_model = get_gpu_memory_gb() + engine.update_weights_from_tensor(list(hf_model_new.named_parameters())) + + # destroy the hf model + del hf_model_new + torch.cuda.empty_cache() + engine.resume_memory_occupation(tags=[GPU_MEMORY_TYPE_KV_CACHE]) + + gpu_memory_usage_after_resume_kv_cache = get_gpu_memory_gb() + self.assertGreater( + gpu_memory_usage_after_resume_kv_cache, + gpu_memory_usage_after_resume_weights, + ) + + print(f"Resume + update took {time.perf_counter() - t:.2f}s") + print( + f"Memory: {gpu_memory_usage_before_resume:.1f} → {gpu_memory_usage_after_resume_cuda_graph:.1f} → {gpu_memory_usage_after_resume_weights:.1f} → {gpu_memory_usage_after_loaded_hf_model:.1f} → {gpu_memory_usage_after_resume_kv_cache:.1f} GB" + ) + + print("generate (#2)") + outputs = engine.generate(params["prompt"], params["sampling_params"])[ + "text" + ] + self.assertEqual(outputs, params["expect_output_after_update_weights"]) + engine.shutdown() + + def test_moe_model_release_and_resume(self): + # Test with MoE model + model_name = DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_CHAT + + tp_size = ep_size = 2 + + print( + f"Testing tp_size={tp_size} and ep_size={ep_size} for test_moe_model_release_and_resume" + ) + engine = sgl.Engine( + model_path=model_name, + random_seed=42, + enable_memory_saver=True, + mem_fraction_static=0.5, + tp_size=tp_size, + ep_size=ep_size, + ) + params = self._common_test_params() + + self._test_initial_generation( + engine, + params["prompt_moe"], + params["sampling_params_moe"], + params["expect_output_before_update_weights_moe"], + ) + + t = time.perf_counter() + gpu_memory_usage_before_release = get_gpu_memory_gb() + engine.release_memory_occupation() + gpu_memory_usage_after_release = get_gpu_memory_gb() + self.assertLess( + gpu_memory_usage_after_release, + gpu_memory_usage_before_release, + ) + + print( + f"Release took {time.perf_counter() - t:.2f}s, memory: {gpu_memory_usage_before_release:.1f} GB → {gpu_memory_usage_after_release:.1f} GB" + ) + + if _DEBUG_EXTRA: + time.sleep(3) + + t = time.perf_counter() + engine.resume_memory_occupation() + print( + f"Resume took {time.perf_counter() - t:.2f}s, memory: {get_gpu_memory_gb():.1f} GB" + ) + + hf_model_new = AutoModelForCausalLM.from_pretrained( + DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_BASE, + torch_dtype="bfloat16", + device_map="cuda", + ) + engine.update_weights_from_tensor(list(hf_model_new.named_parameters())) + + # destroy the hf model + del hf_model_new + torch.cuda.empty_cache() + + print("generate (#2)") + outputs = engine.generate(params["prompt_moe"], params["sampling_params_moe"])[ + "text" + ] + self.assertEqual(outputs, params["expect_output_after_update_weights_moe"]) + engine.shutdown() + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_request_queue_validation.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_request_queue_validation.py new file mode 100644 index 00000000..3dd41b0d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_request_queue_validation.py @@ -0,0 +1,88 @@ +import asyncio +import os +import re +import unittest + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + STDERR_FILENAME, + STDOUT_FILENAME, + CustomTestCase, + popen_launch_server, + send_concurrent_generate_requests, + send_generate_requests, +) + + +class TestMaxQueuedRequests(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + + cls.stdout = open(STDOUT_FILENAME, "w") + cls.stderr = open(STDERR_FILENAME, "w") + + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=( + "--max-running-requests", # Enforce max request concurrency is 1 + "1", + "--max-queued-requests", # Enforce max queued request number is 1 + "1", + "--attention-backend", + "triton", + ), + return_stdout_stderr=(cls.stdout, cls.stderr), + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + cls.stdout.close() + cls.stderr.close() + os.remove(STDOUT_FILENAME) + os.remove(STDERR_FILENAME) + + def test_max_queued_requests_validation_with_serial_requests(self): + """Verify request is not throttled when the max concurrency is 1.""" + status_codes = send_generate_requests( + self.base_url, + num_requests=10, + ) + + for status_code in status_codes: + assert status_code == 200 # request shouldn't be throttled + + def test_max_queued_requests_validation_with_concurrent_requests(self): + """Verify request throttling with concurrent requests.""" + status_codes = asyncio.run( + send_concurrent_generate_requests(self.base_url, num_requests=10) + ) + self.assertLessEqual(status_codes.count(200), 2) + + # expected_status_codes = [200, 200, 503, 503, 503, 503, 503, 503, 503, 503] + # self.assertEqual(status_codes, expected_status_codes) + + def test_max_running_requests_and_max_queued_request_validation(self): + """Verify running request and queued request numbers based on server logs.""" + rr_pattern = re.compile(r"#running-req:\s*(\d+)") + qr_pattern = re.compile(r"#queue-req:\s*(\d+)") + + with open(STDERR_FILENAME) as lines: + for line in lines: + rr_match, qr_match = rr_pattern.search(line), qr_pattern.search(line) + if rr_match: + assert int(rr_match.group(1)) <= 1 + if qr_match: + assert int(qr_match.group(1)) <= 1 + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_retract_decode.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_retract_decode.py new file mode 100644 index 00000000..da688b40 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_retract_decode.py @@ -0,0 +1,117 @@ +import time +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.environ import envs +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) +from sglang.utils import is_in_ci + + +class TestRetractDecode(CustomTestCase): + """python -m unittest test_retract_decode.TestRetractDecode""" + + other_args = [] + + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + launch_args = ["--chunked-prefill-size", "128"] + cls.other_args + with envs.SGLANG_TEST_RETRACT.override(True): + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=launch_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + time.sleep(1) # wait for mem check + + assert self.process.poll() is None, "Server crashed during test" + + +class TestRetractDecodePaged(TestRetractDecode): + """python -m unittest test_retract_decode.TestRetractDecodePaged""" + + other_args = ["--page-size", "16"] + + +class TestRetractDecodeChunkCache(TestRetractDecode): + """python -m unittest test_retract_decode.TestRetractDecodeChunkCache""" + + other_args = ["--disable-radix-cache"] + + +class TestRetractDecodeChunkCachePaged(TestRetractDecode): + """python -m unittest test_retract_decode.TestRetractDecodeChunkCachePaged""" + + other_args = ["--disable-radix-cache", "--page-size", "16"] + + +@unittest.skipIf(is_in_ci(), "Skipped in CI due to long runtime") +class TestRetractDecodeLongOutput(CustomTestCase): + """python -m unittest test_retract_decode.TestRetractDecodeLongOutput""" + + other_args = [] + + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + launch_args = [ + "--chunked-prefill-size", + "128", + "--page-size", + "16", + ] + cls.other_args + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=launch_args, + ) + + def test_long_output_retract(self): + data = { + "input_ids": [[233 + i] * 1234 for i in range(256)], + "sampling_params": {"max_new_tokens": 90000, "ignore_eos": True}, + } + res = requests.post(f"{self.base_url}/generate", json=data) + assert res.status_code == 200, f"Request failed: {res.status_code}" + assert self.process.poll() is None, "Server crashed during test" + + +@unittest.skipIf(is_in_ci(), "Skipped in CI due to long runtime") +class TestRetractDecodeLongOutputChunkCache(TestRetractDecodeLongOutput): + """python -m unittest test_retract_decode.TestRetractDecodeLongOutputChunkCache""" + + other_args = ["--disable-radix-cache"] + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_rope_rocm.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_rope_rocm.py new file mode 100644 index 00000000..5850e706 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_rope_rocm.py @@ -0,0 +1,116 @@ +import unittest + +import torch + +from sglang.srt.layers.rotary_embedding import RotaryEmbedding +from sglang.srt.utils import get_bool_env_var, is_hip +from sglang.test.test_utils import CustomTestCase + +torch.manual_seed(0) + +_is_hip = is_hip() +_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip + + +_CASES = [ + (64, 64, 32, 8000, True, torch.bfloat16, "cuda", 32, 32, 1, 1), + (256, 128, 4096, 10000, True, torch.bfloat16, "cuda", 2, 512, 4, 2), + (512, 128, 311, 10000, True, torch.bfloat16, "cuda", 3, 39, 4, 2), + (128, 128, 2048, 10000, False, torch.bfloat16, "cuda", 2, 512, 32, 8), + (128, 128, 2048, 10000, False, torch.bfloat16, "cuda", 2, 512, 16, 4), + (512, 128, 311, 10000, False, torch.bfloat16, "cuda", 3, 39, 4, 2), +] + + +@unittest.skipIf(_use_aiter, reason="SGLANG_USE_AITER=1 will not use vllm path.") +class TestRotaryEmbeddingNative(CustomTestCase): + # Compare RotaryEmbedding.forward_hip() to forward_native(). + def _run_case( + self, + head_size: int, + rotary_dim: int, + max_pos: int, + base: int, + is_neox: bool, + dtype: torch.dtype, + device: str, + batch_size: int, + seq_len: int, + num_q: int, + num_kv: int, + ) -> None: + rope_ref = RotaryEmbedding( + head_size, rotary_dim, max_pos, base, is_neox, dtype + ).to(device) + rope_hip = RotaryEmbedding( + head_size, rotary_dim, max_pos, base, is_neox, dtype + ).to(device) + + pos_ids = torch.arange(seq_len, device=device).repeat(batch_size) + query = torch.randn( + batch_size * seq_len, num_q * head_size, dtype=dtype, device=device + ) + key = torch.randn( + batch_size * seq_len, num_kv * head_size, dtype=dtype, device=device + ) + + q_ref, k_ref = rope_ref.forward_native(pos_ids, query.clone(), key.clone()) + q_hip, k_hip = rope_hip.forward_hip(pos_ids, query.clone(), key.clone()) + + torch.testing.assert_close(q_ref, q_hip, atol=1e-2, rtol=1e-2) + torch.testing.assert_close(k_ref, k_hip, atol=1e-2, rtol=1e-2) + + def test_all_cases(self) -> None: + """Drive over the full parameter matrix using subTest().""" + for case in _CASES: + with self.subTest(case=case): + self._run_case(*case) + + +@unittest.skipIf(not _use_aiter, reason="Requires AMD GPU plus SGLANG_USE_AITER=1") +class TestRotaryEmbeddingAITer(CustomTestCase): + @staticmethod + def _run_case_aiter( + head_size: int, + rotary_dim: int, + max_pos: int, + base: int, + is_neox: bool, + dtype: torch.dtype, + device: str, + batch_size: int, + seq_len: int, + num_q: int, + num_kv: int, + ) -> None: + from aiter.rotary_embedding import RotaryEmbedding as AiterRotaryEmbedding + + rope_ref = AiterRotaryEmbedding( + head_size, rotary_dim, max_pos, base, is_neox, dtype + ).to(device) + rope_hip = AiterRotaryEmbedding( + head_size, rotary_dim, max_pos, base, is_neox, dtype + ).to(device) + + pos_ids = torch.arange(seq_len, device=device).repeat(batch_size) + query = torch.randn( + batch_size * seq_len, num_q * head_size, dtype=dtype, device=device + ) + key = torch.randn( + batch_size * seq_len, num_kv * head_size, dtype=dtype, device=device + ) + + q_ref, k_ref = rope_ref.forward_native(pos_ids, query.clone(), key.clone()) + q_hip, k_hip = rope_hip.forward_hip(pos_ids, query.clone(), key.clone()) + + torch.testing.assert_close(q_ref, q_hip, atol=1e-2, rtol=1e-2) + torch.testing.assert_close(k_ref, k_hip, atol=1e-2, rtol=1e-2) + + def test_all_cases(self) -> None: + for case in _CASES: + with self.subTest(case=case): + self._run_case_aiter(*case) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_score_api.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_score_api.py new file mode 100644 index 00000000..757af86d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_score_api.py @@ -0,0 +1,590 @@ +import unittest + +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + +from sglang.srt.entrypoints.engine import Engine +from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST, CustomTestCase + +TEST_MODEL_NAME = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + + +class TestScoreAPI(CustomTestCase): + """Test the scoring API functionality.""" + + def setUp(self): + """Set up each test case.""" + self.engine = Engine(model_path=TEST_MODEL_NAME) + + def tearDown(self): + """Clean up after each test case.""" + if self.engine is not None: + self.engine.shutdown() + torch.cuda.empty_cache() + + def compute_hf_scores( + self, query, items, label_token_ids, apply_softmax=False, item_first=False + ): + """Compute scores using direct HuggingFace model inference. + Returns probabilities for each token ID, optionally normalized with softmax. + + Args: + query: The query text + items: List of item texts + label_token_ids: List of token IDs to compute probabilities for + apply_softmax: Whether to normalize probabilities using softmax + item_first: If True, prepend items to query. Otherwise append items to query. + """ + # Initialize HF model and tokenizer + tokenizer = AutoTokenizer.from_pretrained( + TEST_MODEL_NAME, trust_remote_code=True + ) + model = AutoModelForCausalLM.from_pretrained( + TEST_MODEL_NAME, trust_remote_code=True + ) + + try: + scores = [] + for item in items: + # Construct full text based on item_first parameter + full_text = f"{item}{query}" if item_first else f"{query}{item}" + inputs = tokenizer(full_text, return_tensors="pt").to(model.device) + + # Get logits for the last token + with torch.no_grad(): + outputs = model(**inputs) + last_token_logits = outputs.logits[0, -1] + + # Get logits for just our target tokens + target_logits = last_token_logits[label_token_ids] + + # Apply softmax over just the target tokens + target_probs = torch.softmax(target_logits, dim=-1) + + # Convert to list of probabilities in order of label_token_ids + probs = [target_probs[i].item() for i in range(len(label_token_ids))] + + scores.append(probs) + + return scores + finally: + # Clean up HF resources + model.cpu() + del model + del tokenizer + torch.cuda.empty_cache() + + def _get_token_ids(self, tokens): + """Helper method to get token IDs for a list of tokens.""" + tokenizer = AutoTokenizer.from_pretrained( + TEST_MODEL_NAME, trust_remote_code=True + ) + try: + label_token_ids = [] + for token in tokens: + encoding = tokenizer.encode_plus(token, add_special_tokens=False) + token_ids = encoding["input_ids"] + label_token_ids.append(token_ids[0]) + return label_token_ids + finally: + del tokenizer + + def _compare_scores(self, hf_scores, sglang_scores, label_token_ids, case_name=""): + """Helper method to compare scores between HF and SGLang using relative tolerance.""" + self.assertEqual( + len(hf_scores), + len(sglang_scores), + f"Score lengths don't match for {case_name}", + ) + + # Use a relative tolerance of 1% + TOLERANCE = 0.01 + + for hf_score_list, sglang_score_list in zip(hf_scores, sglang_scores): + self.assertEqual( + len(hf_score_list), + len(sglang_score_list), + f"Score list lengths don't match for {case_name}", + ) + + for hf_score, sglang_score in zip(hf_score_list, sglang_score_list): + diff = abs(hf_score - sglang_score) + self.assertLessEqual( + diff, + TOLERANCE, + msg=f"Scores differ by {diff:.2%} ({case_name}): " + f"HF={hf_score:.6f}, SGLang={sglang_score:.6f}", + ) + + self.assertGreaterEqual( + sglang_score, 0, f"SGLang score {sglang_score:.6f} not in [0,1]" + ) + self.assertLessEqual( + sglang_score, 1, f"SGLang score {sglang_score:.6f} not in [0,1]" + ) + + self.assertAlmostEqual( + sum(sglang_score_list), + 1.0, + places=6, + msg=f"SGLang scores don't sum to 1 ({case_name}): {sum(sglang_score_list):.6f}", + ) + + def test_score_consistency(self): + """Test that SGLang scoring matches direct HuggingFace model scoring.""" + # Define test cases + test_cases = [ + { + "name": "default case", + "query": "I pledge allegiance", + "items": ["", " to"], + "item_first": False, + }, + { + "name": "item_first case", + "query": " is a city", + "items": ["Tokyo", "Japan"], + "item_first": True, + }, + ] + + # Common tokens to test for all cases + tokens = [" to", " the"] + label_token_ids = self._get_token_ids(tokens) + + # Run each test case + for case in test_cases: + # Get scores from SGLang + sglang_scores = self.engine.score( + query=case["query"], + items=case["items"], + label_token_ids=label_token_ids, + apply_softmax=True, + item_first=case["item_first"], + ) + + # Get scores from HuggingFace using the same parameters + hf_scores = self.compute_hf_scores( + query=case["query"], + items=case["items"], + label_token_ids=label_token_ids, + apply_softmax=True, + item_first=case["item_first"], + ) + + # Compare scores + self._compare_scores( + hf_scores, sglang_scores, label_token_ids, case["name"] + ) + + def test_score_batch_handling(self): + """Test that batch scoring works correctly.""" + # Test with different batch sizes + batch_sizes = [1, 2, 4, 8] + label_token_ids = [1, 2, 3] + + for batch_size in batch_sizes: + texts = [f"test {i}" for i in range(batch_size)] + scores = self.engine.score( + query="The test was", + items=texts, + label_token_ids=label_token_ids, + apply_softmax=True, + ) + + self.assertEqual( + len(scores), + batch_size, + f"Expected {batch_size} scores, got {len(scores)}", + ) + + # Verify each score list has the correct length + for score_list in scores: + self.assertEqual( + len(score_list), + len(label_token_ids), + f"Score list length {len(score_list)} doesn't match label_token_ids length {len(label_token_ids)}", + ) + self.assertTrue( + all(isinstance(v, float) for v in score_list), + "All scores should be floats", + ) + self.assertAlmostEqual( + 1.0, sum(score_list), 6, "Scores should sum to 1" + ) + + def test_score_request_construction(self): + """Test that scoring requests are constructed to avoid decode phase.""" + from unittest.mock import patch + + # Capture the internal request to verify optimization + captured_requests = [] + original_gen = self.engine.tokenizer_manager.generate_request + + async def mock_generate_request(req, request=None): + captured_requests.append(req) + async for result in original_gen(req, request): + yield result + + # Patch the generate_request method + with patch.object( + self.engine.tokenizer_manager, + "generate_request", + side_effect=mock_generate_request, + ): + # Run a scoring request + query = "What is the capital of" + items = ["France", "Germany"] + label_token_ids = [1, 2, 3] + + scores = self.engine.score( + query=query, + items=items, + label_token_ids=label_token_ids, + apply_softmax=True, + ) + + # Verify we got results + self.assertEqual(len(scores), len(items)) + + # Verify the captured request has decode-avoiding properties + self.assertEqual(len(captured_requests), 1) + request = captured_requests[0] + + # Key assertions for decode phase avoidance: + # 1. max_new_tokens should be 0 (prevents token generation) + # Handle both single and batch request cases + if isinstance(request.sampling_params, dict): + max_new_tokens = request.sampling_params.get("max_new_tokens", 0) + elif isinstance(request.sampling_params, list): + # For batch requests, check the first item + max_new_tokens = request.sampling_params[0].get("max_new_tokens", 0) + else: + max_new_tokens = getattr(request.sampling_params, "max_new_tokens", 0) + + self.assertEqual( + max_new_tokens, 0, "max_new_tokens should be 0 to avoid decode phase" + ) + + # 2. Should have token_ids_logprob for scoring + # Handle both single and batch request cases + if ( + isinstance(request.token_ids_logprob, list) + and len(request.token_ids_logprob) > 0 + and isinstance(request.token_ids_logprob[0], list) + ): + # Batch case: token_ids_logprob is a list of lists + # Each item in the batch should have the same label_token_ids + for item_token_ids in request.token_ids_logprob: + self.assertEqual( + item_token_ids, + label_token_ids, + "Each batch item should have label_token_ids for scoring", + ) + else: + # Single request case + self.assertEqual( + request.token_ids_logprob, + label_token_ids, + "Should have label_token_ids for scoring", + ) + + # 3. Should request logprobs but not stream + self.assertTrue( + request.return_logprob, "Should request logprobs for scoring" + ) + self.assertFalse(request.stream, "Scoring requests should not stream") + + def test_multi_item_scoring_basic(self): + """Test basic multi-item scoring functionality.""" + # Test with a simple query and items + query = "What is the capital of California? Answer Yes or No for each of the following options:" + items = ["Sacramento", "San Jose", "San Francisco"] + label_token_ids = [9454, 2753] # "Yes" and "No" tokens + + # Get scores using SGLang + scores = self.engine.score( + query=query, + items=items, + label_token_ids=label_token_ids, + apply_softmax=True, + ) + + # Verify we get the expected number of scores + self.assertEqual(len(scores), len(items), "Should get one score list per item") + + # Verify each score list has the correct length + for i, score_list in enumerate(scores): + self.assertEqual( + len(score_list), + len(label_token_ids), + f"Item {i} should have {len(label_token_ids)} scores", + ) + # Verify scores are probabilities (sum to 1) + self.assertAlmostEqual( + sum(score_list), + 1.0, + places=6, + msg=f"Scores for item {i} should sum to 1", + ) + # Verify all scores are non-negative + for j, score in enumerate(score_list): + self.assertGreaterEqual( + score, 0, f"Score {j} for item {i} should be non-negative" + ) + + def test_multi_item_scoring_consistency(self): + """Test that multi-item scoring gives consistent results.""" + query = "Choose the best option:" + items = ["Option A", "Option B", "Option C"] + label_token_ids = [1, 2, 3] + + # Run the same test multiple times + scores1 = self.engine.score( + query=query, + items=items, + label_token_ids=label_token_ids, + apply_softmax=True, + ) + + scores2 = self.engine.score( + query=query, + items=items, + label_token_ids=label_token_ids, + apply_softmax=True, + ) + + # Results should be identical (deterministic) + self.assertEqual(len(scores1), len(scores2), "Should get same number of items") + for i, (s1, s2) in enumerate(zip(scores1, scores2)): + self.assertEqual( + len(s1), len(s2), f"Item {i} should have same number of scores" + ) + for j, (score1, score2) in enumerate(zip(s1, s2)): + self.assertAlmostEqual( + score1, + score2, + places=6, + msg=f"Score {j} for item {i} should be identical", + ) + + def test_multi_item_scoring_different_sizes(self): + """Test multi-item scoring with different numbers of items.""" + query = "Rate each option:" + label_token_ids = [1, 2, 3, 4, 5] + + # Test with different numbers of items + test_cases = [ + ["Single item"], + ["Item 1", "Item 2"], + ["A", "B", "C", "D"], + ["X", "Y", "Z", "W", "V", "U"], + ] + + for items in test_cases: + with self.subTest(items=items): + scores = self.engine.score( + query=query, + items=items, + label_token_ids=label_token_ids, + apply_softmax=True, + ) + + self.assertEqual( + len(scores), len(items), f"Should get {len(items)} score lists" + ) + + for i, score_list in enumerate(scores): + self.assertEqual( + len(score_list), + len(label_token_ids), + f"Item {i} should have {len(label_token_ids)} scores", + ) + self.assertAlmostEqual(sum(score_list), 1.0, places=6) + + def test_multi_item_scoring_empty_items(self): + """Test multi-item scoring with empty items list.""" + query = "Test query" + items = [] + label_token_ids = [1, 2] + + scores = self.engine.score( + query=query, + items=items, + label_token_ids=label_token_ids, + apply_softmax=True, + ) + + self.assertEqual(len(scores), 0, "Should return empty list for empty items") + + def test_multi_item_scoring_single_item(self): + """Test multi-item scoring with single item (should work like regular scoring).""" + query = "Complete this sentence: The capital of France is" + items = ["Paris"] + label_token_ids = [1, 2, 3] + + scores = self.engine.score( + query=query, + items=items, + label_token_ids=label_token_ids, + apply_softmax=True, + ) + + self.assertEqual(len(scores), 1, "Should get one score list") + self.assertEqual( + len(scores[0]), len(label_token_ids), "Should have correct number of scores" + ) + self.assertAlmostEqual(sum(scores[0]), 1.0, places=6) + + def test_multi_item_scoring_different_queries(self): + """Test multi-item scoring with different types of queries.""" + items = ["Yes", "No"] + label_token_ids = [1, 2] + + test_queries = [ + "Is this true?", + "Choose the correct answer:", + "What is the best option?", + "Select all that apply:", + "", # Empty query + ] + + for query in test_queries: + with self.subTest(query=query): + scores = self.engine.score( + query=query, + items=items, + label_token_ids=label_token_ids, + apply_softmax=True, + ) + + self.assertEqual( + len(scores), + len(items), + f"Should get {len(items)} score lists for query: '{query}'", + ) + + for i, score_list in enumerate(scores): + self.assertEqual(len(score_list), len(label_token_ids)) + self.assertAlmostEqual(sum(score_list), 1.0, places=6) + + def test_multi_item_scoring_different_label_tokens(self): + """Test multi-item scoring with different label token sets.""" + query = "Choose the best option:" + items = ["Option A", "Option B"] + + test_label_tokens = [ + [1, 2], # Two tokens + [1, 2, 3, 4], # Four tokens + [1], # Single token + [1, 2, 3, 4, 5, 6, 7, 8], # Many tokens + ] + + for label_token_ids in test_label_tokens: + with self.subTest(label_tokens=label_token_ids): + scores = self.engine.score( + query=query, + items=items, + label_token_ids=label_token_ids, + apply_softmax=True, + ) + + self.assertEqual(len(scores), len(items)) + + for i, score_list in enumerate(scores): + self.assertEqual( + len(score_list), + len(label_token_ids), + f"Item {i} should have {len(label_token_ids)} scores", + ) + self.assertAlmostEqual(sum(score_list), 1.0, places=6) + + def test_multi_item_scoring_without_softmax(self): + """Test multi-item scoring without softmax normalization.""" + query = "Rate each option:" + items = ["Good", "Bad", "Neutral"] + label_token_ids = [1, 2, 3] + + scores = self.engine.score( + query=query, + items=items, + label_token_ids=label_token_ids, + apply_softmax=False, # No softmax + ) + + self.assertEqual(len(scores), len(items)) + + for i, score_list in enumerate(scores): + self.assertEqual(len(score_list), len(label_token_ids)) + # Without softmax, scores don't need to sum to 1 + # But they should still be valid logits/probabilities + for j, score in enumerate(score_list): + self.assertIsInstance( + score, (int, float), f"Score {j} for item {i} should be numeric" + ) + + def test_multi_item_scoring_large_batch(self): + """Test multi-item scoring with a large number of items.""" + query = "Classify each item:" + items = [f"Item {i}" for i in range(20)] # 20 items + label_token_ids = [1, 2, 3] + + scores = self.engine.score( + query=query, + items=items, + label_token_ids=label_token_ids, + apply_softmax=True, + ) + + self.assertEqual(len(scores), len(items), "Should handle large batches") + + for i, score_list in enumerate(scores): + self.assertEqual(len(score_list), len(label_token_ids)) + self.assertAlmostEqual(sum(score_list), 1.0, places=6) + + def test_multi_item_scoring_unicode(self): + """Test multi-item scoring with unicode characters.""" + query = "选择最佳选项：" + items = ["选项A", "选项B", "选项C"] + label_token_ids = [1, 2, 3] + + scores = self.engine.score( + query=query, + items=items, + label_token_ids=label_token_ids, + apply_softmax=True, + ) + + self.assertEqual(len(scores), len(items)) + + for i, score_list in enumerate(scores): + self.assertEqual(len(score_list), len(label_token_ids)) + self.assertAlmostEqual(sum(score_list), 1.0, places=6) + + def test_multi_item_scoring_error_handling(self): + """Test multi-item scoring error handling.""" + query = "Test query" + items = ["Item 1", "Item 2"] + label_token_ids = [1, 2] + + # Test with invalid label_token_ids + with self.assertRaises((ValueError, TypeError)): + self.engine.score( + query=query, + items=items, + label_token_ids="invalid", # Should be list of ints + apply_softmax=True, + ) + + # Test with None items + with self.assertRaises((ValueError, TypeError)): + self.engine.score( + query=query, + items=None, + label_token_ids=label_token_ids, + apply_softmax=True, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_server_args.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_server_args.py new file mode 100644 index 00000000..1489160a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_server_args.py @@ -0,0 +1,274 @@ +import json +import unittest +from unittest.mock import patch + +from sglang.srt.server_args import PortArgs, ServerArgs, prepare_server_args +from sglang.test.test_utils import CustomTestCase + + +class TestPrepareServerArgs(CustomTestCase): + def test_prepare_server_args(self): + server_args = prepare_server_args( + [ + "--model-path", + "meta-llama/Meta-Llama-3.1-8B-Instruct", + "--json-model-override-args", + '{"rope_scaling": {"factor": 2.0, "rope_type": "linear"}}', + ] + ) + self.assertEqual( + server_args.model_path, "meta-llama/Meta-Llama-3.1-8B-Instruct" + ) + self.assertEqual( + json.loads(server_args.json_model_override_args), + {"rope_scaling": {"factor": 2.0, "rope_type": "linear"}}, + ) + + +class TestPortArgs(unittest.TestCase): + @patch("sglang.srt.server_args.is_port_available") + @patch("sglang.srt.server_args.tempfile.NamedTemporaryFile") + def test_init_new_standard_case(self, mock_temp_file, mock_is_port_available): + mock_is_port_available.return_value = True + mock_temp_file.return_value.name = "temp_file" + + server_args = ServerArgs(model_path="dummy") + server_args.port = 30000 + server_args.nccl_port = None + server_args.enable_dp_attention = False + + port_args = PortArgs.init_new(server_args) + + self.assertTrue(port_args.tokenizer_ipc_name.startswith("ipc://")) + self.assertTrue(port_args.scheduler_input_ipc_name.startswith("ipc://")) + self.assertTrue(port_args.detokenizer_ipc_name.startswith("ipc://")) + self.assertIsInstance(port_args.nccl_port, int) + + @patch("sglang.srt.server_args.is_port_available") + def test_init_new_with_single_node_dp_attention(self, mock_is_port_available): + mock_is_port_available.return_value = True + + server_args = ServerArgs(model_path="dummy") + server_args.port = 30000 + server_args.nccl_port = None + server_args.enable_dp_attention = True + server_args.nnodes = 1 + server_args.dist_init_addr = None + + port_args = PortArgs.init_new(server_args) + + self.assertTrue(port_args.tokenizer_ipc_name.startswith("tcp://127.0.0.1:")) + self.assertTrue( + port_args.scheduler_input_ipc_name.startswith("tcp://127.0.0.1:") + ) + self.assertTrue(port_args.detokenizer_ipc_name.startswith("tcp://127.0.0.1:")) + self.assertIsInstance(port_args.nccl_port, int) + + @patch("sglang.srt.server_args.is_port_available") + def test_init_new_with_dp_rank(self, mock_is_port_available): + mock_is_port_available.return_value = True + + server_args = ServerArgs(model_path="dummy") + server_args.port = 30000 + server_args.nccl_port = None + server_args.enable_dp_attention = True + server_args.nnodes = 1 + server_args.dist_init_addr = "192.168.1.1:25000" + + worker_ports = [25006, 25007, 25008, 25009] + port_args = PortArgs.init_new(server_args, dp_rank=2, worker_ports=worker_ports) + + self.assertTrue(port_args.scheduler_input_ipc_name.endswith(":25008")) + + self.assertTrue(port_args.tokenizer_ipc_name.startswith("tcp://192.168.1.1:")) + self.assertTrue(port_args.detokenizer_ipc_name.startswith("tcp://192.168.1.1:")) + self.assertIsInstance(port_args.nccl_port, int) + + @patch("sglang.srt.server_args.is_port_available") + def test_init_new_with_ipv4_address(self, mock_is_port_available): + mock_is_port_available.return_value = True + + server_args = ServerArgs(model_path="dummy") + server_args.port = 30000 + + server_args.nccl_port = None + + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "192.168.1.1:25000" + + port_args = PortArgs.init_new(server_args) + + self.assertTrue(port_args.tokenizer_ipc_name.startswith("tcp://192.168.1.1:")) + self.assertTrue( + port_args.scheduler_input_ipc_name.startswith("tcp://192.168.1.1:") + ) + self.assertTrue(port_args.detokenizer_ipc_name.startswith("tcp://192.168.1.1:")) + self.assertIsInstance(port_args.nccl_port, int) + + @patch("sglang.srt.server_args.is_port_available") + def test_init_new_with_malformed_ipv4_address(self, mock_is_port_available): + mock_is_port_available.return_value = True + + server_args = ServerArgs(model_path="dummy") + server_args.port = 30000 + server_args.nccl_port = None + + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "192.168.1.1" + + with self.assertRaises(AssertionError) as context: + PortArgs.init_new(server_args) + + self.assertIn( + "please provide --dist-init-addr as host:port", str(context.exception) + ) + + @patch("sglang.srt.server_args.is_port_available") + def test_init_new_with_malformed_ipv4_address_invalid_port( + self, mock_is_port_available + ): + mock_is_port_available.return_value = True + + server_args = ServerArgs(model_path="dummy") + server_args.port = 30000 + server_args.nccl_port = None + + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "192.168.1.1:abc" + + with self.assertRaises(ValueError) as context: + PortArgs.init_new(server_args) + + @patch("sglang.srt.server_args.is_port_available") + @patch("sglang.srt.server_args.is_valid_ipv6_address", return_value=True) + def test_init_new_with_ipv6_address( + self, mock_is_valid_ipv6, mock_is_port_available + ): + mock_is_port_available.return_value = True + + server_args = ServerArgs(model_path="dummy") + server_args.port = 30000 + server_args.nccl_port = None + + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "[2001:db8::1]:25000" + + port_args = PortArgs.init_new(server_args) + + self.assertTrue(port_args.tokenizer_ipc_name.startswith("tcp://[2001:db8::1]:")) + self.assertTrue( + port_args.scheduler_input_ipc_name.startswith("tcp://[2001:db8::1]:") + ) + self.assertTrue( + port_args.detokenizer_ipc_name.startswith("tcp://[2001:db8::1]:") + ) + self.assertIsInstance(port_args.nccl_port, int) + + @patch("sglang.srt.server_args.is_port_available") + @patch("sglang.srt.server_args.is_valid_ipv6_address", return_value=False) + def test_init_new_with_invalid_ipv6_address( + self, mock_is_valid_ipv6, mock_is_port_available + ): + mock_is_port_available.return_value = True + + server_args = ServerArgs(model_path="dummy") + server_args.port = 30000 + server_args.nccl_port = None + + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "[invalid-ipv6]:25000" + + with self.assertRaises(ValueError) as context: + PortArgs.init_new(server_args) + + self.assertIn("invalid IPv6 address", str(context.exception)) + + @patch("sglang.srt.server_args.is_port_available") + def test_init_new_with_malformed_ipv6_address_missing_bracket( + self, mock_is_port_available + ): + mock_is_port_available.return_value = True + + server_args = ServerArgs(model_path="dummy") + server_args.port = 30000 + server_args.nccl_port = None + + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "[2001:db8::1:25000" + + with self.assertRaises(ValueError) as context: + PortArgs.init_new(server_args) + + self.assertIn("invalid IPv6 address format", str(context.exception)) + + @patch("sglang.srt.server_args.is_port_available") + @patch("sglang.srt.server_args.is_valid_ipv6_address", return_value=True) + def test_init_new_with_malformed_ipv6_address_missing_port( + self, mock_is_valid_ipv6, mock_is_port_available + ): + mock_is_port_available.return_value = True + + server_args = ServerArgs(model_path="dummy") + server_args.port = 30000 + server_args.nccl_port = None + + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "[2001:db8::1]" + + with self.assertRaises(ValueError) as context: + PortArgs.init_new(server_args) + + self.assertIn( + "a port must be specified in IPv6 address", str(context.exception) + ) + + @patch("sglang.srt.server_args.is_port_available") + @patch("sglang.srt.server_args.is_valid_ipv6_address", return_value=True) + def test_init_new_with_malformed_ipv6_address_invalid_port( + self, mock_is_valid_ipv6, mock_is_port_available + ): + mock_is_port_available.return_value = True + + server_args = ServerArgs(model_path="dummy") + server_args.port = 30000 + server_args.nccl_port = None + + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "[2001:db8::1]:abcde" + + with self.assertRaises(ValueError) as context: + PortArgs.init_new(server_args) + + self.assertIn("invalid port in IPv6 address", str(context.exception)) + + @patch("sglang.srt.server_args.is_port_available") + @patch("sglang.srt.server_args.is_valid_ipv6_address", return_value=True) + def test_init_new_with_malformed_ipv6_address_wrong_separator( + self, mock_is_valid_ipv6, mock_is_port_available + ): + mock_is_port_available.return_value = True + + server_args = ServerArgs(model_path="dummy") + server_args.port = 30000 + server_args.nccl_port = None + + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "[2001:db8::1]#25000" + + with self.assertRaises(ValueError) as context: + PortArgs.init_new(server_args) + + self.assertIn("expected ':' after ']'", str(context.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_skip_tokenizer_init.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_skip_tokenizer_init.py new file mode 100644 index 00000000..c65dadb4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_skip_tokenizer_init.py @@ -0,0 +1,244 @@ +""" +python3 -m unittest test_skip_tokenizer_init.TestSkipTokenizerInit.test_parallel_sample +python3 -m unittest test_skip_tokenizer_init.TestSkipTokenizerInit.run_decode_stream +""" + +import json +import unittest + +import requests +from transformers import AutoProcessor, AutoTokenizer + +from sglang.lang.chat_template import get_chat_template_by_model_path +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_IMAGE_URL, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + download_image_with_retry, + popen_launch_server, +) + + +class TestSkipTokenizerInit(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--skip-tokenizer-init", "--stream-output"], + ) + cls.eos_token_id = [119690] + cls.tokenizer = AutoTokenizer.from_pretrained( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, use_fast=False + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode( + self, + prompt_text="The capital of France is", + max_new_tokens=32, + return_logprob=False, + top_logprobs_num=0, + n=1, + ): + input_ids = self.get_input_ids(prompt_text) + + request = self.get_request_json( + input_ids=input_ids, + return_logprob=return_logprob, + top_logprobs_num=top_logprobs_num, + max_new_tokens=max_new_tokens, + stream=False, + n=n, + ) + response = requests.post( + self.base_url + "/generate", + json=request, + ) + ret = response.json() + print(json.dumps(ret, indent=2)) + + def assert_one_item(item): + if item["meta_info"]["finish_reason"]["type"] == "stop": + self.assertEqual( + item["meta_info"]["finish_reason"]["matched"], + self.tokenizer.eos_token_id, + ) + elif item["meta_info"]["finish_reason"]["type"] == "length": + self.assertEqual( + len(item["output_ids"]), item["meta_info"]["completion_tokens"] + ) + self.assertEqual(len(item["output_ids"]), max_new_tokens) + self.assertEqual(item["meta_info"]["prompt_tokens"], len(input_ids)) + + if return_logprob: + num_input_logprobs = len(input_ids) - request["logprob_start_len"] + if num_input_logprobs > len(input_ids): + num_input_logprobs -= len(input_ids) + self.assertEqual( + len(item["meta_info"]["input_token_logprobs"]), + num_input_logprobs, + f'{len(item["meta_info"]["input_token_logprobs"])} mismatch with {len(input_ids)}', + ) + self.assertEqual( + len(item["meta_info"]["output_token_logprobs"]), + max_new_tokens, + ) + + # Determine whether to assert a single item or multiple items based on n + if n == 1: + assert_one_item(ret) + else: + self.assertEqual(len(ret), n) + for i in range(n): + assert_one_item(ret[i]) + + print("=" * 100) + + def run_decode_stream(self, return_logprob=False, top_logprobs_num=0, n=1): + max_new_tokens = 32 + input_ids = self.get_input_ids("The capital of France is") + requests.post(self.base_url + "/flush_cache") + response = requests.post( + self.base_url + "/generate", + json=self.get_request_json( + input_ids=input_ids, + max_new_tokens=max_new_tokens, + return_logprob=return_logprob, + top_logprobs_num=top_logprobs_num, + stream=False, + n=n, + ), + ) + ret = response.json() + print(json.dumps(ret)) + output_ids = ret["output_ids"] + print("output from non-streaming request:") + print(output_ids) + print(self.tokenizer.decode(output_ids, skip_special_tokens=True)) + + requests.post(self.base_url + "/flush_cache") + response_stream = requests.post( + self.base_url + "/generate", + json=self.get_request_json( + input_ids=input_ids, + return_logprob=return_logprob, + top_logprobs_num=top_logprobs_num, + stream=True, + n=n, + ), + ) + + response_stream_json = [] + for line in response_stream.iter_lines(): + print(line) + if line.startswith(b"data: ") and line[6:] != b"[DONE]": + response_stream_json.append(json.loads(line[6:])) + out_stream_ids = [] + for x in response_stream_json: + out_stream_ids += x["output_ids"] + print("output from streaming request:") + print(out_stream_ids) + print(self.tokenizer.decode(out_stream_ids, skip_special_tokens=True)) + + assert output_ids == out_stream_ids + + def test_simple_decode(self): + self.run_decode() + + def test_parallel_sample(self): + self.run_decode(n=3) + + def test_logprob(self): + for top_logprobs_num in [0, 3]: + self.run_decode(return_logprob=True, top_logprobs_num=top_logprobs_num) + + def test_eos_behavior(self): + self.run_decode(max_new_tokens=256) + + def test_simple_decode_stream(self): + self.run_decode_stream() + + def get_input_ids(self, prompt_text) -> list[int]: + input_ids = self.tokenizer(prompt_text, return_tensors="pt")["input_ids"][ + 0 + ].tolist() + return input_ids + + def get_request_json( + self, + input_ids, + max_new_tokens=32, + return_logprob=False, + top_logprobs_num=0, + stream=False, + n=1, + ): + return { + "input_ids": input_ids, + "sampling_params": { + "temperature": 0 if n == 1 else 0.5, + "max_new_tokens": max_new_tokens, + "n": n, + "stop_token_ids": self.eos_token_id, + }, + "stream": stream, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "logprob_start_len": 0, + } + + +class TestSkipTokenizerInitVLM(TestSkipTokenizerInit): + @classmethod + def setUpClass(cls): + cls.image_url = DEFAULT_IMAGE_URL + cls.image = download_image_with_retry(cls.image_url) + cls.model = DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST + cls.tokenizer = AutoTokenizer.from_pretrained(cls.model, use_fast=False) + cls.processor = AutoProcessor.from_pretrained(cls.model, trust_remote_code=True) + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--skip-tokenizer-init"], + ) + cls.eos_token_id = [cls.tokenizer.eos_token_id] + + def get_input_ids(self, _prompt_text) -> list[int]: + chat_template = get_chat_template_by_model_path(self.model) + text = f"{chat_template.image_token}What is in this picture?" + inputs = self.processor( + text=[text], + images=[self.image], + return_tensors="pt", + ) + + return inputs.input_ids[0].tolist() + + def get_request_json(self, *args, **kwargs): + ret = super().get_request_json(*args, **kwargs) + ret["image_data"] = [self.image_url] + ret["logprob_start_len"] = ( + -1 + ) # Do not try to calculate logprobs of image embeddings. + return ret + + def test_simple_decode_stream(self): + # TODO mick + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_speculative_registry.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_speculative_registry.py new file mode 100644 index 00000000..eb358086 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_speculative_registry.py @@ -0,0 +1,149 @@ +import unittest + +from sglang.srt.speculative import spec_info as spec_info_module +from sglang.srt.speculative.spec_info import ( + SpeculativeAlgorithm, + register_speculative_algorithm, +) + + +class DummyWorker: + def __init__(self, **kwargs): + self.kwargs = kwargs + + +class SpeculativeRegistryTests(unittest.TestCase): + def test_nextn_alias_maps_to_eagle(self): + eagle = SpeculativeAlgorithm.from_string("EAGLE") + alias = SpeculativeAlgorithm.from_string("NEXTN") + self.assertIs(alias, eagle) + + def test_register_speculative_algorithm_registers_worker_and_flags(self): + original_next_value = SpeculativeAlgorithm._next_value + algo = register_speculative_algorithm( + "TEST_SPEC_ALGO", + DummyWorker, + aliases=("TEST_SPEC_ALIAS",), + flags=("EAGLE",), + override_worker=True, + ) + self.addCleanup(self._cleanup_registered_algorithm, algo, ("TEST_SPEC_ALIAS",)) + self.addCleanup( + setattr, SpeculativeAlgorithm, "_next_value", original_next_value + ) + + self.assertIs(SpeculativeAlgorithm.from_string("TEST_SPEC_ALGO"), algo) + self.assertIs(SpeculativeAlgorithm.from_string("TEST_SPEC_ALIAS"), algo) + self.assertTrue(algo.is_eagle()) + self.assertIs(SpeculativeAlgorithm.from_value(int(algo)), algo) + self.assertIn(algo, list(spec_info_module._REGISTERED_WORKERS)) + + worker = algo.create_draft_worker(example_arg=42) + self.assertIsInstance(worker, DummyWorker) + self.assertEqual(worker.kwargs["example_arg"], 42) + + def test_builtin_algorithms_flags_and_factories(self): + cases = { + "NONE": { + "is_none": True, + "is_eagle": False, + "is_eagle3": False, + "is_standalone": False, + "is_ngram": False, + "has_factory": False, + }, + "EAGLE": { + "is_none": False, + "is_eagle": True, + "is_eagle3": False, + "is_standalone": False, + "is_ngram": False, + "has_factory": True, + }, + "EAGLE3": { + "is_none": False, + "is_eagle": True, + "is_eagle3": True, + "is_standalone": False, + "is_ngram": False, + "has_factory": True, + }, + "STANDALONE": { + "is_none": False, + "is_eagle": False, + "is_eagle3": False, + "is_standalone": True, + "is_ngram": False, + "has_factory": True, + }, + "NGRAM": { + "is_none": False, + "is_eagle": False, + "is_eagle3": False, + "is_standalone": False, + "is_ngram": True, + "has_factory": True, + }, + } + + for name, expectations in cases.items(): + with self.subTest(name=name): + algo = SpeculativeAlgorithm.from_string(name) + self.assertEqual(algo.name, name) + self.assertEqual(algo.is_none(), expectations["is_none"]) + self.assertEqual(algo.is_eagle(), expectations["is_eagle"]) + self.assertEqual(algo.is_eagle3(), expectations["is_eagle3"]) + self.assertEqual(algo.is_standalone(), expectations["is_standalone"]) + self.assertEqual(algo.is_ngram(), expectations["is_ngram"]) + + has_factory = algo._draft_worker_factory is not None + self.assertEqual(has_factory, expectations["has_factory"]) + self.assertIs(SpeculativeAlgorithm.from_value(int(algo)), algo) + + self.assertIs(SpeculativeAlgorithm.from_string(None), SpeculativeAlgorithm.NONE) + + def test_iteration_returns_registration_order(self): + names = [algo.name for algo in SpeculativeAlgorithm._registration_order] + for required in ["NONE", "EAGLE", "EAGLE3", "STANDALONE", "NGRAM"]: + self.assertIn(required, names) + + def test_create_draft_worker_returns_none_for_none_algorithm(self): + self.assertIsNone(SpeculativeAlgorithm.NONE.create_draft_worker()) + + def test_register_draft_worker_override(self): + algo = SpeculativeAlgorithm.from_string("EAGLE") + original_factory = algo._draft_worker_factory + + def dummy_factory(_: SpeculativeAlgorithm, **kwargs): + return "dummy" + + SpeculativeAlgorithm.register_draft_worker(algo, dummy_factory) + self.addCleanup( + SpeculativeAlgorithm.register_draft_worker, algo, original_factory + ) + + self.assertEqual(algo.create_draft_worker(), "dummy") + + def _cleanup_registered_algorithm(self, algorithm: SpeculativeAlgorithm, aliases): + name = algorithm.name + SpeculativeAlgorithm._registry_by_value.pop(algorithm.value, None) + SpeculativeAlgorithm._registry_by_name.pop(name, None) + if hasattr(SpeculativeAlgorithm, name): + delattr(SpeculativeAlgorithm, name) + + for alias in aliases: + SpeculativeAlgorithm._registry_by_name.pop(alias, None) + + try: + SpeculativeAlgorithm._registration_order.remove(algorithm) + except ValueError: + pass + + for flag_values in SpeculativeAlgorithm._flags.values(): + flag_values.discard(algorithm.value) + + spec_info_module._REGISTERED_WORKERS.pop(algorithm, None) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_srt_endpoint.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_srt_endpoint.py new file mode 100644 index 00000000..59a8c3c4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_srt_endpoint.py @@ -0,0 +1,743 @@ +""" +python3 -m unittest test_srt_endpoint.TestSRTEndpoint.test_simple_decode +python3 -m unittest test_srt_endpoint.TestSRTEndpoint.test_logprob_with_chunked_prefill +python3 -m unittest test_srt_endpoint.TestTokenizeDetokenize +""" + +import json +import random +import time +import unittest +from concurrent.futures import ThreadPoolExecutor +from functools import partial +from typing import Optional + +import numpy as np +import requests + +from sglang.srt.sampling.custom_logit_processor import CustomLogitProcessor +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, + run_logprob_check, +) + + +class TestSRTEndpoint(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=( + "--enable-custom-logit-processor", + "--mem-fraction-static", + "0.7", + "--cuda-graph-max-bs", + "8", + ), + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode( + self, + return_logprob=False, + top_logprobs_num=0, + return_text=False, + n=1, + stream=False, + batch=False, + ): + if batch: + text = ["The capital of France is"] + else: + text = "The capital of France is" + + response = requests.post( + self.base_url + "/generate", + json={ + "text": text, + "sampling_params": { + "temperature": 0 if n == 1 else 0.5, + "max_new_tokens": 16, + "n": n, + }, + "stream": stream, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "return_text_in_logprobs": return_text, + "logprob_start_len": 0, + }, + ) + if not stream: + response_json = response.json() + else: + response_json = [] + for line in response.iter_lines(): + if line.startswith(b"data: ") and line[6:] != b"[DONE]": + response_json.append(json.loads(line[6:])) + + print(json.dumps(response_json, indent=2)) + print("=" * 100) + + def test_simple_decode(self): + self.run_decode() + + def test_simple_decode_batch(self): + self.run_decode(batch=True) + + def test_parallel_sample(self): + self.run_decode(n=3) + + def test_parallel_sample_stream(self): + self.run_decode(n=3, stream=True) + + def test_logprob(self): + self.run_decode( + return_logprob=True, + top_logprobs_num=5, + return_text=True, + ) + + def test_logprob_start_len(self): + logprob_start_len = 4 + new_tokens = 4 + prompts = [ + "I have a very good idea on", + "Today is a sunndy day and", + ] + + response = requests.post( + self.base_url + "/generate", + json={ + "text": prompts, + "sampling_params": { + "temperature": 0, + "max_new_tokens": new_tokens, + }, + "return_logprob": True, + "top_logprobs_num": 5, + "return_text_in_logprobs": True, + "logprob_start_len": logprob_start_len, + }, + ) + response_json = response.json() + print(json.dumps(response_json, indent=2)) + + for i, res in enumerate(response_json): + self.assertEqual( + res["meta_info"]["prompt_tokens"], + logprob_start_len + len(res["meta_info"]["input_token_logprobs"]), + ) + assert prompts[i].endswith( + "".join([x[-1] for x in res["meta_info"]["input_token_logprobs"]]) + ) + + self.assertEqual(res["meta_info"]["completion_tokens"], new_tokens) + self.assertEqual(len(res["meta_info"]["output_token_logprobs"]), new_tokens) + self.assertEqual( + res["text"], + "".join([x[-1] for x in res["meta_info"]["output_token_logprobs"]]), + ) + + def test_logprob_with_chunked_prefill(self): + """Test a long prompt that requests output logprobs will not hit OOM.""" + new_tokens = 4 + prompts = "I have a very good idea on this. " * 8000 + + response = requests.post( + self.base_url + "/generate", + json={ + "text": prompts, + "sampling_params": { + "temperature": 0, + "max_new_tokens": new_tokens, + }, + "return_logprob": True, + "logprob_start_len": -1, + "top_logprobs_num": 5, + }, + ) + response_json = response.json() + # print(json.dumps(response_json, indent=2)) + + res = response_json + self.assertEqual(res["meta_info"]["completion_tokens"], new_tokens) + + # Test the number of tokens are correct + self.assertEqual(len(res["meta_info"]["output_token_logprobs"]), new_tokens) + self.assertEqual(len(res["meta_info"]["output_top_logprobs"]), new_tokens) + + # Test the top-1 tokens are the same as output tokens (because temp = 0.0) + for i in range(new_tokens): + self.assertListEqual( + res["meta_info"]["output_token_logprobs"][i], + res["meta_info"]["output_top_logprobs"][i][0], + ) + self.assertEqual(len(res["meta_info"]["output_top_logprobs"][i]), 5) + + def test_logprob_match(self): + """Test the output logprobs are close to the input logprobs if we run a prefill again.""" + + def run_generate( + prompt, return_logprob=False, max_new_tokens=512, logprob_start_len=-1 + ): + + if isinstance(prompt, str): + prompt_kwargs = {"text": prompt} + else: + prompt_kwargs = {"input_ids": prompt} + + response = requests.post( + self.base_url + "/generate", + json={ + **prompt_kwargs, + "sampling_params": { + "temperature": 1.0, + "max_new_tokens": max_new_tokens, + "ignore_eos": True, + }, + "return_logprob": return_logprob, + "return_text_in_logprobs": True, + "logprob_start_len": logprob_start_len, + }, + ) + return response.json() + + prompt = "I have a very good idea on how to" + + gen = run_generate(prompt, return_logprob=True, logprob_start_len=0) + output_logprobs = np.array( + [x[0] for x in gen["meta_info"]["output_token_logprobs"]] + ) + num_prompts_tokens = gen["meta_info"]["prompt_tokens"] + + input_tokens = [x[1] for x in gen["meta_info"]["input_token_logprobs"]] + output_tokens = [x[1] for x in gen["meta_info"]["output_token_logprobs"]] + + new_prompt = input_tokens + output_tokens + score = run_generate( + new_prompt, return_logprob=True, logprob_start_len=0, max_new_tokens=0 + ) + output_logprobs_score = np.array( + [ + x[0] + for x in score["meta_info"]["input_token_logprobs"][num_prompts_tokens:] + ] + ) + + print(f"{output_logprobs[-10:]=}") + print(f"{output_logprobs_score[-10:]=}") + + diff = np.abs(output_logprobs - output_logprobs_score) + max_diff = np.max(diff) + self.assertLess(max_diff, 0.35) + + def test_logprob_mixed(self): + args = [] + temperature = 0 + # input_len, output_len, temperature, logprob_start_len, return_logprob, top_logprobs_num + for input_len in [1000, 5000, 10000, 50000]: + for output_len in [4, 8]: + for logprob_start_len in [0, 500, 2500, 5000, 25000]: + for return_logprob in [True, False]: + for top_logprobs_num in [0, 5]: + + if logprob_start_len >= input_len: + continue + + args.append( + ( + input_len, + output_len, + temperature, + logprob_start_len, + return_logprob, + top_logprobs_num, + ) + ) + + random.shuffle(args) + + func = partial(run_logprob_check, self) + with ThreadPoolExecutor(8) as executor: + list(executor.map(func, args)) + + def test_logprob_grammar(self): + prompts = "Question: Is Paris the Capital of France? Answer:" + allowed_tokens = [" Yes", " No"] + + response = requests.post( + self.base_url + "/generate", + json={ + "text": prompts, + "sampling_params": { + "temperature": 1.0, + "max_new_tokens": 1, + "regex": "( Yes| No)", + }, + "return_logprob": True, + "top_logprobs_num": 5, # The grammar constraint allows all prefix tokens so we need to use a larger top_k. + "return_text_in_logprobs": True, + }, + ) + response_json = response.json() + output_top_logprobs = response_json["meta_info"]["output_top_logprobs"][0] + print(f"{output_top_logprobs=}") + + # Parse results + # This is because the grammar constraint allows all prefix tokens + logprobs = [None] * 2 + for i in range(len(output_top_logprobs)): + try: + idx = allowed_tokens.index(output_top_logprobs[i][2]) + except ValueError: + # Not found + continue + logprobs[idx] = output_top_logprobs[i][0] + + self.assertTrue(all(x is not None for x in logprobs)) + + def run_custom_logit_processor(self, target_token_id: Optional[int] = None): + """Test custom logit processor with custom params. + + If target_token_id is None, the custom logit processor won't be passed in. + """ + + custom_params = {"token_id": target_token_id} + + class DeterministicLogitProcessor(CustomLogitProcessor): + """A dummy logit processor that changes the logits to always + sample the given token id. + """ + + def __call__(self, logits, custom_param_list): + assert logits.shape[0] == len(custom_param_list) + key = "token_id" + + for i, param_dict in enumerate(custom_param_list): + # Mask all other tokens + logits[i, :] = -float("inf") + # Assign highest probability to the specified token + logits[i, param_dict[key]] = 0.0 + return logits + + prompts = "Question: Is Paris the Capital of France? Answer:" + + # Base case json data to be posted to the server. + base_json = { + "text": prompts, + "sampling_params": {"temperature": 0.0}, + "return_logprob": True, + } + + # Custom json data with custom logit processor and params. + custom_json = base_json.copy() + # Only set the custom logit processor if target_token_id is not None. + if target_token_id is not None: + custom_json["custom_logit_processor"] = DeterministicLogitProcessor.to_str() + custom_json["sampling_params"]["custom_params"] = custom_params + + custom_response = requests.post( + self.base_url + "/generate", + json=custom_json, + ).json() + + output_token_logprobs = custom_response["meta_info"]["output_token_logprobs"] + sampled_tokens = [x[1] for x in output_token_logprobs] + + # The logit processor should always sample the given token as the logits is deterministic. + if target_token_id is not None: + self.assertTrue( + all(x == custom_params["token_id"] for x in sampled_tokens), + # Print the detailed test case info if the test fails. + f"{target_token_id=}\n{sampled_tokens=}\n{custom_response=}", + ) + + def run_stateful_custom_logit_processor( + self, first_token_id: int | None, delay: int = 2 + ): + """Test custom logit processor with custom params and state. + + Should sample the first `delay` tokens normally, then output first_token_id and consecutive tokens after that. + If first_token_id is None, the custom logit processor won't be passed in. + """ + custom_params = {"token_id": first_token_id, "delay": 2} + + class DeterministicStatefulLogitProcessor(CustomLogitProcessor): + """A dummy logit processor that changes the logits to always + sample the given token id. + """ + + def __call__(self, logits, custom_param_list): + assert logits.shape[0] == len(custom_param_list) + + for i, param_dict in enumerate(custom_param_list): + if param_dict["delay"] > 0: + param_dict["delay"] -= 1 + continue + if param_dict["delay"] == 0: + param_dict["delay"] -= 1 + force_token = param_dict["token_id"] + else: + output_ids = param_dict["__req__"].output_ids + force_token = output_ids[-1] + 1 + # Mask all other tokens + logits[i, :] = -float("inf") + # Assign highest probability to the specified token + logits[i, force_token] = 0.0 + return logits + + prompts = "Question: Is Paris the Capital of France? Answer:" + + # Base case json data to be posted to the server. + base_json = { + "text": prompts, + "sampling_params": {"temperature": 0.0}, + "return_logprob": True, + } + + # Custom json data with custom logit processor and params. + custom_json = base_json.copy() + # Only set the custom logit processor if target_token_id is not None. + if first_token_id is not None: + custom_json["custom_logit_processor"] = ( + DeterministicStatefulLogitProcessor().to_str() + ) + custom_json["sampling_params"]["custom_params"] = custom_params + + custom_response = requests.post( + self.base_url + "/generate", + json=custom_json, + ).json() + + output_token_logprobs = custom_response["meta_info"]["output_token_logprobs"] + sampled_tokens = [x[1] for x in output_token_logprobs] + # The logit processor should always sample the given token as the logits is deterministic. + if first_token_id is not None: + self.assertTrue( + all( + x == custom_params["token_id"] + k + for k, x in enumerate(sampled_tokens[custom_params["delay"] :]) + ), + # Print the detailed test case info if the test fails. + f"{first_token_id=}\n{sampled_tokens=}\n{custom_response=}", + ) + + def test_custom_logit_processor(self): + """Test custom logit processor with a single request.""" + self.run_custom_logit_processor(target_token_id=5) + + def test_custom_logit_processor_batch_mixed(self): + """Test a batch of requests mixed of requests with and without custom logit processor.""" + target_token_ids = list(range(32)) + [None] * 16 + random.shuffle(target_token_ids) + with ThreadPoolExecutor(len(target_token_ids)) as executor: + list(executor.map(self.run_custom_logit_processor, target_token_ids)) + + @unittest.skip("Skip this test because this feature has a bug. See comments below.") + def test_stateful_custom_logit_processor(self): + """Test custom logit processor with a single request.""" + + """ + NOTE: This feature has a race condition bug. + This line https://github.com/sgl-project/sglang/blob/ef8ec07b2ce4c70c2a33ec5acda4ce529bc3cda4/test/srt/test_srt_endpoint.py#L395-L396 can be accessed by two concurrent threads at the same time. The access order is not guaranteed. + In sglang, we use two python threads to overlap the GPU computation and CPU scheduling. + Thread 1 (the CPU scheduling thread) will update the `param_dict["__req__"].output_ids`. + Thread 2 (the GPU computation thread) will call `DeterministicStatefulLogitProcessor` because sampling is considered as GPU computation. + We can fix this by moving the call of DeterministicStatefulLogitProcessor to the CPU scheduling thread. + """ + + self.run_stateful_custom_logit_processor(first_token_id=5) + + @unittest.skip("Skip this test because this feature has a bug. See comments above.") + def test_stateful_custom_logit_processor_batch_mixed(self): + """Test a batch of requests mixed of requests with and without custom logit processor.""" + target_token_ids = list(range(32)) + [None] * 16 + random.shuffle(target_token_ids) + with ThreadPoolExecutor(len(target_token_ids)) as executor: + list( + executor.map(self.run_stateful_custom_logit_processor, target_token_ids) + ) + + def test_cache_tokens(self): + for _ in range(2): + time.sleep(1) + response = requests.post(self.base_url + "/flush_cache") + assert response.status_code == 200 + + def send_and_check_cached_tokens(input_ids): + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": list(input_ids), + "sampling_params": { + "max_new_tokens": 1, + }, + }, + ) + response_json = response.json() + return response_json["meta_info"]["cached_tokens"] + + self.assertEqual(send_and_check_cached_tokens(range(0, 100)), 0) + self.assertEqual(send_and_check_cached_tokens(range(0, 10000)), 100) + self.assertEqual(send_and_check_cached_tokens(range(0, 10000)), 9999) + self.assertEqual(send_and_check_cached_tokens(range(0, 1000)), 999) + self.assertEqual(send_and_check_cached_tokens(range(0, 11000)), 10000) + + def test_get_server_info(self): + response = requests.get(self.base_url + "/get_server_info") + response_json = response.json() + + max_total_num_tokens = response_json["max_total_num_tokens"] + self.assertIsInstance(max_total_num_tokens, int) + + version = response_json["version"] + self.assertIsInstance(version, str) + + def test_logit_bias(self): + """Test that a very high logit bias forces sampling of a specific token.""" + # Choose a token ID to bias (using 5 as an example) + target_token_id = 60704 # Paris for meta-llama/Llama-3.2-1B-Instruct, DEFAULT_SMALL_MODEL_NAME_FOR_TEST + logit_bias = {str(target_token_id): 100.0} # Very high positive bias + + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 1.0, # Use high temperature to encourage exploration + "max_new_tokens": 4, + "logit_bias": logit_bias, + }, + "return_logprob": True, + }, + ) + response_json = response.json() + + # Extract the sampled token IDs from the output + output_token_logprobs = response_json["meta_info"]["output_token_logprobs"] + sampled_tokens = [x[1] for x in output_token_logprobs] + + # Verify that all sampled tokens are the target token + self.assertTrue( + all(x == target_token_id for x in sampled_tokens), + f"Expected all tokens to be {target_token_id}, but got {sampled_tokens}", + ) + + def test_forbidden_token(self): + """Test that a forbidden token (very negative logit bias) doesn't appear in the output.""" + # Choose a token ID to forbid (using 10 as an example) + forbidden_token_id = 23994 # rice for meta-llama/Llama-3.2-1B-Instruct, DEFAULT_SMALL_MODEL_NAME_FOR_TEST + logit_bias = { + str(forbidden_token_id): -100.0 + } # Very negative bias to forbid the token + + response = requests.post( + self.base_url + "/generate", + json={ + "text": "Only output 'rice' exactly like this, in lowercase ONLY: rice", + "sampling_params": { + "temperature": 1.0, # Use high temperature to encourage diverse output + "max_new_tokens": 50, # Generate enough tokens to likely include numbers + "logit_bias": logit_bias, + }, + "return_logprob": True, + }, + ) + response_json = response.json() + + # Extract the sampled token IDs from the output + output_token_logprobs = response_json["meta_info"]["output_token_logprobs"] + sampled_tokens = [x[1] for x in output_token_logprobs] + + # Verify that the forbidden token doesn't appear in the output + self.assertNotIn( + forbidden_token_id, + sampled_tokens, + f"Expected forbidden token {forbidden_token_id} not to be present, but it was found", + ) + + def test_logit_bias_isolation(self): + """Test that logit_bias applied to one request doesn't affect other requests in batch.""" + # Choose a token ID to bias in first request only + biased_token_id = 60704 # Paris for meta-llama/Llama-3.2-1B-Instruct, DEFAULT_SMALL_MODEL_NAME_FOR_TEST + + # Prepare batch requests - one with logit_bias and one without + requests_data = [ + { + "text": "The capital of France is", + "sampling_params": { + "temperature": 1.0, + "max_new_tokens": 4, + "logit_bias": {str(biased_token_id): 100.0}, # Strong bias + }, + "return_logprob": True, + }, + { + "text": "The capital of France is", + "sampling_params": { + "temperature": 1.0, + "max_new_tokens": 4, + }, + "return_logprob": True, + }, + ] + + # Send both requests + responses = [] + for req in requests_data: + response = requests.post(self.base_url + "/generate", json=req) + responses.append(response.json()) + + # Extract token IDs from each response + biased_tokens = [ + x[1] for x in responses[0]["meta_info"]["output_token_logprobs"] + ] + unbiased_tokens = [ + x[1] for x in responses[1]["meta_info"]["output_token_logprobs"] + ] + + # Verify first response contains only biased tokens + self.assertTrue( + all(x == biased_token_id for x in biased_tokens), + f"Expected all tokens to be {biased_token_id} in first response, but got {biased_tokens}", + ) + + # Verify second response contains at least some different tokens + # (We can't guarantee exactly what tokens will be generated, but they shouldn't all be the biased token) + self.assertTrue( + any(x != biased_token_id for x in unbiased_tokens), + f"Expected some tokens to be different from {biased_token_id} in second response, but got {unbiased_tokens}", + ) + + def test_get_server_info_concurrent(self): + """Make sure the concurrent get_server_info doesn't crash the server.""" + tp = ThreadPoolExecutor(max_workers=30) + + def s(): + server_info = requests.get(self.base_url + "/get_server_info") + server_info.json() + + futures = [] + for _ in range(4): + futures.append(tp.submit(s)) + + for f in futures: + f.result() + + +# ------------------------------------------------------------------------- +# /tokenize & /detokenize Test Class: TestTokenizeDetokenize +# ------------------------------------------------------------------------- + + +class TestTokenizeDetokenize(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.tokenize_url = f"{cls.base_url}/tokenize" + cls.detokenize_url = f"{cls.base_url}/detokenize" + cls.session = requests.Session() + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + cls.session.close() + + def _post_json(self, url, payload): + r = self.session.post(url, json=payload) + r.raise_for_status() + return r.json() + + def test_tokenize_various_inputs(self): + single = "Hello SGLang world! 123 😊, ಪರ್ವತದ ಮೇಲೆ ಹಿಮ." + multi = ["First sentence.", "Second, with 中文."] + scenarios = [ + {"prompt": single, "add_special_tokens": True}, + {"prompt": single, "add_special_tokens": False}, + {"prompt": multi, "add_special_tokens": True}, + {"prompt": multi, "add_special_tokens": False}, + {"prompt": "", "add_special_tokens": False}, + ] + for case in scenarios: + payload = {"model": self.model, "prompt": case["prompt"]} + if "add_special_tokens" in case: + payload["add_special_tokens"] = case["add_special_tokens"] + resp = self._post_json(self.tokenize_url, payload) + tokens = resp["tokens"] + count = resp["count"] + self.assertIsInstance(tokens, list) + if not tokens: + self.assertEqual(count, 0) + else: + if isinstance(tokens[0], list): + total = sum(len(t) for t in tokens) + expected = sum(count) if isinstance(count, list) else count + else: + total = len(tokens) + expected = count + self.assertEqual(total, expected) + + def test_tokenize_invalid_type(self): + r = self.session.post( + self.tokenize_url, json={"model": self.model, "prompt": 12345} + ) + self.assertEqual(r.status_code, 400) + + def test_detokenize_roundtrip(self): + text = "Verify detokenization round trip. यह डिटोकेनाइजेशन है" + t0 = self._post_json( + self.tokenize_url, + {"model": self.model, "prompt": text, "add_special_tokens": False}, + )["tokens"] + t1 = self._post_json( + self.tokenize_url, + {"model": self.model, "prompt": text, "add_special_tokens": True}, + )["tokens"] + cases = [ + {"tokens": t0, "skip_special_tokens": True, "expected": text}, + {"tokens": t1, "skip_special_tokens": True, "expected": text}, + {"tokens": t1, "skip_special_tokens": False, "expected": None}, + {"tokens": [], "skip_special_tokens": True, "expected": ""}, + ] + for case in cases: + payload = {"model": self.model, "tokens": case["tokens"]} + if "skip_special_tokens" in case: + payload["skip_special_tokens"] = case["skip_special_tokens"] + resp = self._post_json(self.detokenize_url, payload) + text_out = resp["text"] + if case["expected"] is not None: + self.assertEqual(text_out, case["expected"]) + else: + self.assertIsInstance(text_out, str) + + def test_detokenize_invalid_tokens(self): + r = self.session.post( + self.detokenize_url, json={"model": self.model, "tokens": ["a", "b"]} + ) + self.assertEqual(r.status_code, 400) + r2 = self.session.post( + self.detokenize_url, json={"model": self.model, "tokens": [1, -1, 2]} + ) + self.assertEqual(r2.status_code, 500) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_srt_engine.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_srt_engine.py new file mode 100644 index 00000000..d370f629 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_srt_engine.py @@ -0,0 +1,219 @@ +""" +Usage: +python3 -m unittest test_srt_engine.TestSRTEngine.test_4_sync_async_stream_combination +""" + +import asyncio +import json +import unittest +from types import SimpleNamespace + +import torch + +import sglang as sgl +from sglang.bench_offline_throughput import BenchArgs, throughput_test +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils.hf_transformers_utils import get_tokenizer +from sglang.test.few_shot_gsm8k_engine import run_eval +from sglang.test.test_utils import ( + DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + CustomTestCase, +) + + +class TestSRTEngine(CustomTestCase): + + def test_1_engine_runtime_consistency(self): + prompt = "Today is a sunny day and I like" + model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + + sampling_params = {"temperature": 0, "max_new_tokens": 8} + + engine = sgl.Engine(model_path=model_path, random_seed=42) + out1 = engine.generate(prompt, sampling_params)["text"] + engine.shutdown() + + runtime = sgl.Runtime(model_path=model_path, random_seed=42) + out2 = json.loads(runtime.generate(prompt, sampling_params))["text"] + runtime.shutdown() + + print("==== Answer 1 ====") + print(out1) + + print("==== Answer 2 ====") + print(out2) + self.assertEqual(out1, out2) + + def test_2_engine_runtime_encode_consistency(self): + prompt = "Today is a sunny day and I like" + model_path = DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST + + engine = sgl.Engine(model_path=model_path, is_embedding=True, random_seed=42) + out1 = torch.tensor(engine.encode(prompt)["embedding"]) + engine.shutdown() + + runtime = sgl.Runtime(model_path=model_path, is_embedding=True, random_seed=42) + out2 = torch.tensor(json.loads(runtime.encode(prompt))["embedding"]) + runtime.shutdown() + + self.assertTrue(torch.allclose(out1, out2, atol=1e-5, rtol=1e-3)) + + def test_3_engine_token_ids_consistency(self): + # just to ensure there is no issue running multiple generate calls + prompt = "Today is a sunny day and I like" + model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + sampling_params = {"temperature": 0, "max_new_tokens": 8} + + engine = sgl.Engine( + model_path=model_path, random_seed=42, disable_radix_cache=True + ) + out1 = engine.generate(prompt, sampling_params)["text"] + + tokenizer = get_tokenizer(model_path) + token_ids = tokenizer.encode(prompt) + out2 = engine.generate(input_ids=token_ids, sampling_params=sampling_params)[ + "text" + ] + + engine.shutdown() + + print("==== Answer 1 ====") + print(out1) + + print("==== Answer 2 ====") + print(out2) + self.assertEqual(out1, out2) + + def test_4_sync_async_stream_combination(self): + prompt = "AI safety is" + sampling_params = {"temperature": 0.8, "top_p": 0.95} + + # Create an LLM. + llm = sgl.Engine( + model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + ) + + if True: + # 1. sync + non streaming + print("\n\n==== 1. sync + non streaming ====") + output = llm.generate(prompt, sampling_params) + print(output["text"]) + + # 2. sync + streaming + print("\n\n==== 2. sync + streaming ====") + output_generator = llm.generate(prompt, sampling_params, stream=True) + offset = 0 + for output in output_generator: + print(output["text"][offset:], end="", flush=True) + offset = len(output["text"]) + print() + + if True: + loop = asyncio.get_event_loop() + # 3. async + non_streaming + print("\n\n==== 3. async + non streaming ====") + output = loop.run_until_complete( + llm.async_generate(prompt, sampling_params) + ) + print(output["text"]) + + # 4. async + streaming + async def async_streaming(engine): + generator = await engine.async_generate( + prompt, sampling_params, stream=True + ) + + offset = 0 + async for output in generator: + print(output["text"][offset:], end="", flush=True) + offset = len(output["text"]) + print() + + print("\n\n==== 4. async + streaming ====") + loop.run_until_complete(async_streaming(llm)) + + llm.shutdown() + + def test_5_gsm8k(self): + + args = SimpleNamespace( + model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + local_data_path=None, + num_shots=5, + num_questions=1400, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["accuracy"], 0.33) + + def test_6_engine_cpu_offload(self): + prompt = "Today is a sunny day and I like" + model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + + sampling_params = {"temperature": 0, "max_new_tokens": 8} + + engine = sgl.Engine( + model_path=model_path, + random_seed=42, + max_total_tokens=128, + ) + out1 = engine.generate(prompt, sampling_params)["text"] + engine.shutdown() + + engine = sgl.Engine( + model_path=model_path, + random_seed=42, + max_total_tokens=128, + cpu_offload_gb=3, + ) + out2 = engine.generate(prompt, sampling_params)["text"] + engine.shutdown() + + print("==== Answer 1 ====") + print(out1) + + print("==== Answer 2 ====") + print(out2) + self.assertEqual(out1, out2) + + def test_7_engine_offline_throughput(self): + server_args = ServerArgs( + model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + ) + bench_args = BenchArgs(num_prompts=10) + result = throughput_test(server_args=server_args, bench_args=bench_args) + self.assertGreater(result["total_throughput"], 3000) + + def test_8_engine_async_encode_consistency(self): + prompt = "Today is a sunny day and I like" + model_path = DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST + + engine = sgl.Engine( + model_path=model_path, + is_embedding=True, + random_seed=42, + disable_radix_cache=True, + ) + + # Get sync and async embeddings + out1 = torch.tensor(engine.encode(prompt)["embedding"]) + loop = asyncio.get_event_loop() + out2 = torch.tensor( + loop.run_until_complete(engine.async_encode(prompt))["embedding"] + ) + + engine.shutdown() + + print("\n==== Shapes ====") + print(f"sync shape: {out1.shape}") + print(f"async shape: {out2.shape}") + + self.assertTrue( + torch.allclose(out1, out2, atol=1e-5, rtol=1e-3), + "Sync and async embeddings are not equal within tolerance", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_standalone_speculative_decoding.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_standalone_speculative_decoding.py new file mode 100644 index 00000000..70d18db4 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_standalone_speculative_decoding.py @@ -0,0 +1,115 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.environ import envs +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_STANDALONE_SPECULATIVE_DRAFT_MODEL_FOR_TEST, + DEFAULT_STANDALONE_SPECULATIVE_TARGET_MODEL_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +GSM_DATASET_PATH = None + + +# Default server arguments shared across all tests +DEFAULT_SERVER_ARGS = [ + "--trust-remote-code", + "--cuda-graph-max-bs", + "8", + "--speculative-algorithm", + "STANDALONE", + "--speculative-draft-model-path", + DEFAULT_STANDALONE_SPECULATIVE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + "4", + "--speculative-eagle-topk", + "2", + "--speculative-num-draft-tokens", + "7", + "--mem-fraction-static", + 0.7, +] + + +class TestStandaloneSpeculativeDecodingBase(CustomTestCase): + + model = DEFAULT_STANDALONE_SPECULATIVE_TARGET_MODEL_FOR_TEST + draft_model = DEFAULT_STANDALONE_SPECULATIVE_DRAFT_MODEL_FOR_TEST + base_url = DEFAULT_URL_FOR_TEST + accuracy_threshold = 0.7 # derived tests need to override this + spec_decode_threshold = 3.6 # derived spec decoding tests need to override this + + @classmethod + def get_server_args(cls): + """Return the arguments for the server launch. Override in subclasses.""" + return DEFAULT_SERVER_ARGS + ["--attention-backend", "fa3"] + + @classmethod + def setUpClass(cls): + # disable deep gemm precompile to make launch server faster + # please don't do this if you want to make your inference workload faster + envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.set(False) + envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False) + model = cls.model + cls.process = popen_launch_server( + model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=cls.get_server_args(), + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=4, + num_questions=100, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + data_path=GSM_DATASET_PATH, + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + # Use the appropriate metric key based on the test class + metric_key = "accuracy" + self.assertGreater(metrics[metric_key], self.accuracy_threshold) + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, self.spec_decode_threshold) + + +class TestStandaloneSpeculativeDecodingTriton(TestStandaloneSpeculativeDecodingBase): + + @classmethod + def get_server_args(cls): + return DEFAULT_SERVER_ARGS + ["--attention-backend", "triton"] + + +class TestStandaloneSpeculativeDecodingFlashinfer( + TestStandaloneSpeculativeDecodingBase +): + @classmethod + def get_server_args(cls): + return DEFAULT_SERVER_ARGS + ["--attention-backend", "flashinfer"] + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_start_profile.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_start_profile.py new file mode 100644 index 00000000..5c235bf6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_start_profile.py @@ -0,0 +1,342 @@ +""" +Usage: +# From the test/srt directory: +cd test/srt +python3 -m unittest test_start_profile.TestStartProfile +python3 -m unittest test_start_profile.TestStartProfileWithNsys + +# Run specific tests: +python3 -m unittest test_start_profile.TestStartProfile.test_start_profile_1 +python3 -m unittest test_start_profile.TestStartProfileWithNsys.test_start_profile_cuda_profiler +""" + +import os +import shutil +import subprocess +import time +import unittest + +import requests + +from sglang.srt.environ import envs +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +OUTPUT_DIR = "./profiler_dir" + + +def _is_nsys_available(): + """Check if nsys (Nsight Systems) is available on the system.""" + try: + result = subprocess.run(["nsys", "--version"], capture_output=True, timeout=5) + return result.returncode == 0 + except (FileNotFoundError, subprocess.TimeoutExpired): + return False + + +class TestStartProfile(CustomTestCase): + + @classmethod + def setUpClass(cls): + envs.SGLANG_TORCH_PROFILER_DIR.set(OUTPUT_DIR) + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def setUp(self): + self._clear_profile_dir() + + def test_start_profile_1(self): + """Test /start_profile with start_step and num_steps argument. This have to be the first test for start_step to work""" + response = self._start_profile(start_step="15", num_steps=5) + + self._post_request() + + self._check_non_empty_profile_dir() + + def test_start_profile_2(self): + """Test /start_profile with no argument""" + response = self._start_profile() + + self._post_request() + + # Before /stop_profile, the profile directory should be empty + self._check_empty_profile_dir() + + # Post /stop_profile and check the profile directory is non-empty + response = requests.post( + f"{DEFAULT_URL_FOR_TEST}/stop_profile", + ) + self._check_non_empty_profile_dir() + + def test_start_profile_3(self): + """Test /start_profile with num_steps argument""" + response = self._start_profile(num_steps=5) + + self._post_request() + + self._check_non_empty_profile_dir() + + def _start_profile(self, **kwargs): + """Start profiling with optional parameters.""" + response = requests.post( + f"{DEFAULT_URL_FOR_TEST}/start_profile", + json=kwargs if kwargs else None, + ) + self.assertEqual(response.status_code, 200) + + def _post_request(self): + response = requests.post( + f"{DEFAULT_URL_FOR_TEST}/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, + ) + self.assertEqual(response.status_code, 200) + + def _clear_profile_dir(self): + if os.path.isdir(OUTPUT_DIR): + # Remove the directory and all its contents + shutil.rmtree(OUTPUT_DIR) + + def _check_non_empty_profile_dir(self): + self.assertTrue(os.path.isdir(OUTPUT_DIR), "Output directory does not exist.") + self.assertNotEqual( + len(os.listdir(OUTPUT_DIR)), 0, "Output directory is empty!" + ) + + def _check_empty_profile_dir(self): + if os.path.isdir(OUTPUT_DIR): + self.assertEqual( + len(os.listdir(OUTPUT_DIR)), 0, "Output directory is non-empty!" + ) + + +class TestStartProfileWithNsys(CustomTestCase): + """Test /start_profile with CUDA_PROFILER (requires nsys wrapper) + + Each test starts its own clean server instance with nsys profiling. + """ + + @classmethod + def setUpClass(cls): + if not _is_nsys_available(): + raise unittest.SkipTest("nsys (Nsight Systems) is not available") + + envs.SGLANG_TORCH_PROFILER_DIR.set(OUTPUT_DIR) + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + # Use a different port to avoid conflicts with other tests + cls.base_url = "http://127.0.0.1:21100" + + def setUp(self): + """Start a clean server with nsys for each test""" + # Kill any existing processes on this port + self._kill_existing_server() + + # Clean up old profile files for this test + test_name = self.id().split(".")[-1] # Get test method name + self.nsys_output_file = f"nsys_profile_{test_name}" + + if os.path.isdir(OUTPUT_DIR): + profile_file = os.path.join(OUTPUT_DIR, f"{self.nsys_output_file}.nsys-rep") + if os.path.exists(profile_file): + try: + os.remove(profile_file) + except OSError: + pass + + # Launch server with nsys wrapper + self.process = self._popen_launch_server_with_nsys( + self.model, + self.base_url, + self.nsys_output_file, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + ) + + def tearDown(self): + """Kill server and verify profile was created""" + + # Kill server first to let nsys finalize the .nsys-rep file + kill_process_tree(self.process.pid) + + # Also ensure nsys agent processes are killed + try: + subprocess.run( + ["pkill", "-f", "nsys.*--start-agent"], + timeout=5, + stderr=subprocess.DEVNULL, + ) + except (subprocess.TimeoutExpired, FileNotFoundError): + pass + + # Give nsys a moment to finalize the report + time.sleep(3) + + # Verify the .nsys-rep file was created + self._verify_nsys_profile_created() + + def _kill_existing_server(self): + """Kill any existing server process on our port and orphaned nsys agents""" + try: + # Kill server on our port + subprocess.run(["lsof", "-ti", ":21100"], capture_output=True, timeout=5) + subprocess.run(["pkill", "-f", "sglang.launch_server.*21100"], timeout=5) + + # Kill any orphaned nsys agent processes + subprocess.run( + ["pkill", "-f", "nsys.*--start-agent"], + timeout=5, + stderr=subprocess.DEVNULL, # Suppress "no process found" errors + ) + + time.sleep(2) # Wait for cleanup + except (subprocess.TimeoutExpired, FileNotFoundError): + pass + + def _popen_launch_server_with_nsys(self, model, base_url, output_file, timeout): + """Launch server wrapped with nsys profile -c cudaProfilerApi + + Each test gets its own output file for complete isolation. + """ + _, host, port = base_url.split(":") + host = host[2:] + + # Build the server launch command + command = [ + "nsys", + "profile", + "-c", + "cudaProfilerApi", + "--capture-range-end", + "stop", # Stop after first cudaProfilerStop() + "-o", + os.path.join(OUTPUT_DIR, output_file), + "python3", + "-m", + "sglang.launch_server", + "--model-path", + model, + "--host", + host, + "--port", + port, + ] + + # Create output directory if it doesn't exist + os.makedirs(OUTPUT_DIR, exist_ok=True) + + # Launch the process - capture output to keep test output clean + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + + # Wait for server to be ready + start_time = time.perf_counter() + elapsed = 0 + with requests.Session() as session: + while elapsed < timeout: + elapsed = time.perf_counter() - start_time + + return_code = process.poll() + if return_code is not None: + raise Exception( + f"Server process exited with code {return_code}. " + "Check server logs above for errors." + ) + + try: + response = session.get(f"{base_url}/health_generate", timeout=5) + if response.status_code == 200: + return process + except (requests.RequestException, requests.Timeout): + pass + + time.sleep(5) + + # Timeout reached + kill_process_tree(process.pid) + raise TimeoutError( + f"Server failed to start within {timeout} seconds. " + f"Check the server logs above for more information." + ) + + def _verify_nsys_profile_created(self): + """Verify that the .nsys-rep file was created after server shutdown.""" + if not os.path.isdir(OUTPUT_DIR): + raise AssertionError("Output directory does not exist.") + + expected_file = f"{self.nsys_output_file}.nsys-rep" + profile_path = os.path.join(OUTPUT_DIR, expected_file) + + if not os.path.exists(profile_path): + files = os.listdir(OUTPUT_DIR) + raise AssertionError( + f"Expected profile file '{expected_file}' not found. " + f"Files present: {files}" + ) + + def test_start_profile_cuda_profiler_with_start_step(self): + """Test /start_profile with CUDA_PROFILER, start_step, and num_steps""" + # Use start_step to let server warm up before profiling + response = self._start_profile( + activities=["CUDA_PROFILER"], start_step=10, num_steps=3 + ) + + self._post_request() + + # Profile verification happens in tearDown() + + def test_start_profile_cuda_profiler(self): + """Test /start_profile with CUDA_PROFILER activity (no start_step)""" + # Simple num_steps test - profiling starts immediately + response = self._start_profile(activities=["CUDA_PROFILER"], num_steps=5) + + self._post_request() + + # Profile verification happens in tearDown() + + def _start_profile(self, **kwargs): + """Start profiling with optional parameters.""" + response = requests.post( + f"{self.base_url}/start_profile", + json=kwargs if kwargs else None, + ) + self.assertEqual(response.status_code, 200) + return response + + def _post_request(self): + response = requests.post( + f"{self.base_url}/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, + ) + self.assertEqual(response.status_code, 200) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_swa_unittest.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_swa_unittest.py new file mode 100644 index 00000000..2d01f90b --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_swa_unittest.py @@ -0,0 +1,339 @@ +import unittest + +import torch + +from sglang.srt.mem_cache.allocator import SWAKVPool, SWATokenToKVPoolAllocator +from sglang.srt.mem_cache.cache_init_params import CacheInitParams +from sglang.srt.mem_cache.memory_pool import ReqToTokenPool +from sglang.srt.mem_cache.radix_cache import RadixKey +from sglang.srt.mem_cache.swa_radix_cache import SWARadixCache + + +class TestSWA(unittest.TestCase): + @classmethod + def setUpClass(cls): + pass + + @classmethod + def tearDownClass(cls): + pass + + def test_swa_memory_pool(self): + size = 16 + size_swa = 16 + head_num = 8 + head_dim = 128 + num_layers = 48 + global_interval = 4 + dtype = torch.bfloat16 + device = "cuda" + full_attention_layer_ids = [i for i in range(0, num_layers, global_interval)] + full_attention_layer_ids_set = set(full_attention_layer_ids) + swa_attention_layer_ids = [ + i for i in range(num_layers) if i not in full_attention_layer_ids_set + ] + pool = SWAKVPool( + size=size, + size_swa=size_swa, + dtype=dtype, + head_num=head_num, + head_dim=head_dim, + swa_attention_layer_ids=swa_attention_layer_ids, + full_attention_layer_ids=full_attention_layer_ids, + enable_kvcache_transpose=False, + device=device, + ) + alloc = SWATokenToKVPoolAllocator( + size=size, + size_swa=size_swa, + dtype=dtype, + device=device, + kvcache=pool, + need_sort=False, + ) + self.assertEqual( + alloc.full_available_size() + alloc.swa_available_size(), size + size_swa + ) + index = alloc.alloc(1) + self.assertEqual( + alloc.full_available_size() + alloc.swa_available_size(), + size_swa + size_swa - 2, + ) + alloc.free_swa(index) + result = alloc.translate_loc_from_full_to_swa(index) + print(result) + + def test_swa_radix_cache_1(self): + # args + req_size = 10 + max_context_len = 128 + kv_size = 128 + kv_size_swa = 64 + sliding_window_size = 4 + head_num = 8 + head_dim = 128 + num_layers = 48 + global_interval = 4 + dtype = torch.bfloat16 + device = "cuda" + full_attention_layer_ids = [i for i in range(0, num_layers, global_interval)] + full_attention_layer_ids_set = set(full_attention_layer_ids) + swa_attention_layer_ids = [ + i for i in range(num_layers) if i not in full_attention_layer_ids_set + ] + # setup req to token pool + req_to_token_pool = ReqToTokenPool( + size=req_size, + max_context_len=max_context_len, + device=device, + enable_memory_saver=False, + ) + # setup kv pool + kv_pool = SWAKVPool( + size=kv_size, + size_swa=kv_size_swa, + dtype=dtype, + head_num=head_num, + head_dim=head_dim, + swa_attention_layer_ids=swa_attention_layer_ids, + full_attention_layer_ids=full_attention_layer_ids, + enable_kvcache_transpose=False, + device=device, + ) + # setup token to kv pool allocator + allocator = SWATokenToKVPoolAllocator( + size=kv_size, + size_swa=kv_size_swa, + dtype=dtype, + device=device, + kvcache=kv_pool, + need_sort=False, + ) + # setup radix cache + tree = SWARadixCache( + params=CacheInitParams( + req_to_token_pool=req_to_token_pool, + token_to_kv_pool_allocator=allocator, + disable=False, + page_size=1, + ), + sliding_window_size=sliding_window_size, + ) + + # test + print( + f"[Start] allocator swa available size: {allocator.swa_available_size()}, full available size: {allocator.full_available_size()}" + ) + req1_token_ids, req1_kv_indices = [1, 2, 3], allocator.alloc(3) + self.assertEqual(len(req1_token_ids), len(req1_kv_indices)) + print( + f"req1: inserting, req1_token_ids: {req1_token_ids}, req1_kv_indices: {req1_kv_indices}" + ) + prefix_len = tree.insert(RadixKey(req1_token_ids), req1_kv_indices) + print( + f"req1: prefix_len: {prefix_len}, allocator swa available size: {allocator.swa_available_size()}, full available size: {allocator.full_available_size()}" + ) + req2_token_ids, req2_kv_indices = [1, 2, 3, 4, 5, 6, 7], allocator.alloc(7) + self.assertEqual(len(req2_token_ids), len(req2_kv_indices)) + print( + f"req2: inserting, req2_token_ids: {req2_token_ids}, req2_kv_indices: {req2_kv_indices}" + ) + prefix_len = tree.insert(RadixKey(req2_token_ids), req2_kv_indices) + print( + f"req2: prefix_len: {prefix_len}, allocator swa available size: {allocator.swa_available_size()}, full available size: {allocator.full_available_size()}" + ) + req3_token_ids, req3_kv_indices = [10, 11, 12], allocator.alloc(3) + self.assertEqual(len(req3_token_ids), len(req3_kv_indices)) + print( + f"req3: inserting, req3_token_ids: {req3_token_ids}, req3_kv_indices: {req3_kv_indices}" + ) + prefix_len = tree.insert(RadixKey(req3_token_ids), req3_kv_indices) + print( + f"req3: prefix_len: {prefix_len}, allocator swa available size: {allocator.swa_available_size()}, full available size: {allocator.full_available_size()}" + ) + req4_token_ids, req4_kv_indices = [1, 2, 3, 4, 5, 60, 70], allocator.alloc(7) + self.assertEqual(len(req4_token_ids), len(req4_kv_indices)) + print( + f"req4: inserting, req4_token_ids: {req4_token_ids}, req4_kv_indices: {req4_kv_indices}" + ) + prefix_len = tree.insert(RadixKey(req4_token_ids), req4_kv_indices) + print( + f"req4: prefix_len: {prefix_len}, allocator swa available size: {allocator.swa_available_size()}, full available size: {allocator.full_available_size()}" + ) + + tree.pretty_print() + full_num_tokens, swa_num_tokens = 1, 0 + print(f"evicting {full_num_tokens} full token and {swa_num_tokens} swa token") + tree.evict(full_num_tokens=full_num_tokens, swa_num_tokens=swa_num_tokens) + tree.pretty_print() + + full_num_tokens, swa_num_tokens = 0, 1 + print(f"evicting {full_num_tokens} full token and {swa_num_tokens} swa token") + tree.evict(full_num_tokens=full_num_tokens, swa_num_tokens=swa_num_tokens) + tree.pretty_print() + + full_num_tokens, swa_num_tokens = 1, 2 + print(f"evicting {full_num_tokens} full token and {swa_num_tokens} swa token") + tree.evict(full_num_tokens=full_num_tokens, swa_num_tokens=swa_num_tokens) + tree.pretty_print() + + req5_token_ids = [1, 2, 3, 4, 5] + result = tree.match_prefix(RadixKey(req5_token_ids)) + kv_indices, last_node = result.device_indices, result.last_device_node + print( + f"req5: token_ids: {req5_token_ids}, matched kv_indices: {kv_indices}, last_node.key: {last_node.key}" + ) + self.assertEqual(len(kv_indices), 0) + + req6_token_ids = [1, 2, 3, 4, 5, 60, 70] + result = tree.match_prefix(RadixKey(req6_token_ids)) + kv_indices, last_node = result.device_indices, result.last_device_node + print( + f"req6: token_ids: {req6_token_ids}, matched kv_indices: {kv_indices}, last_node.key: {last_node.key}" + ) + self.assertEqual(len(kv_indices), 7) + self.assertEqual(len(last_node.key), 2) + self.assertEqual(last_node.key.token_ids[0], 60) + self.assertEqual(last_node.key.token_ids[1], 70) + + def test_swa_radix_cache_eagle(self): + # args + req_size = 10 + max_context_len = 128 + kv_size = 128 + kv_size_swa = 64 + sliding_window_size = 4 + head_num = 8 + head_dim = 128 + num_layers = 48 + global_interval = 4 + dtype = torch.bfloat16 + device = "cuda" + full_attention_layer_ids = [i for i in range(0, num_layers, global_interval)] + full_attention_layer_ids_set = set(full_attention_layer_ids) + swa_attention_layer_ids = [ + i for i in range(num_layers) if i not in full_attention_layer_ids_set + ] + # setup req to token pool + req_to_token_pool = ReqToTokenPool( + size=req_size, + max_context_len=max_context_len, + device=device, + enable_memory_saver=False, + ) + # setup kv pool + kv_pool = SWAKVPool( + size=kv_size, + size_swa=kv_size_swa, + dtype=dtype, + head_num=head_num, + head_dim=head_dim, + swa_attention_layer_ids=swa_attention_layer_ids, + full_attention_layer_ids=full_attention_layer_ids, + enable_kvcache_transpose=False, + device=device, + ) + # setup token to kv pool allocator + allocator = SWATokenToKVPoolAllocator( + size=kv_size, + size_swa=kv_size_swa, + dtype=dtype, + device=device, + kvcache=kv_pool, + need_sort=False, + ) + # setup radix cache + tree = SWARadixCache( + params=CacheInitParams( + req_to_token_pool=req_to_token_pool, + token_to_kv_pool_allocator=allocator, + page_size=1, + disable=False, + is_eagle=True, + ), + sliding_window_size=sliding_window_size, + ) + + # test + print( + f"[Start] allocator swa available size: {allocator.swa_available_size()}, full available size: {allocator.full_available_size()}" + ) + req1_token_ids, req1_kv_indices = [1, 2, 3], allocator.alloc(3) + self.assertEqual(len(req1_token_ids), len(req1_kv_indices)) + print( + f"req1: inserting, req1_token_ids: {req1_token_ids}, req1_kv_indices: {req1_kv_indices}" + ) + prefix_len = tree.insert(RadixKey(req1_token_ids), req1_kv_indices) + self.assertEqual(prefix_len, 0) + print( + f"req1: prefix_len: {prefix_len}, allocator swa available size: {allocator.swa_available_size()}, full available size: {allocator.full_available_size()}" + ) + req2_token_ids, req2_kv_indices = [1, 2, 3, 4, 5, 6, 7], allocator.alloc(7) + self.assertEqual(len(req2_token_ids), len(req2_kv_indices)) + print( + f"req2: inserting, req2_token_ids: {req2_token_ids}, req2_kv_indices: {req2_kv_indices}" + ) + prefix_len = tree.insert(RadixKey(req2_token_ids), req2_kv_indices) + self.assertEqual(prefix_len, 2) + print( + f"req2: prefix_len: {prefix_len}, allocator swa available size: {allocator.swa_available_size()}, full available size: {allocator.full_available_size()}" + ) + req3_token_ids, req3_kv_indices = [10, 11, 12], allocator.alloc(3) + self.assertEqual(len(req3_token_ids), len(req3_kv_indices)) + print( + f"req3: inserting, req3_token_ids: {req3_token_ids}, req3_kv_indices: {req3_kv_indices}" + ) + prefix_len = tree.insert(RadixKey(req3_token_ids), req3_kv_indices) + self.assertEqual(prefix_len, 0) + print( + f"req3: prefix_len: {prefix_len}, allocator swa available size: {allocator.swa_available_size()}, full available size: {allocator.full_available_size()}" + ) + req4_token_ids, req4_kv_indices = [1, 2, 3, 4, 5, 60, 70], allocator.alloc(7) + self.assertEqual(len(req4_token_ids), len(req4_kv_indices)) + print( + f"req4: inserting, req4_token_ids: {req4_token_ids}, req4_kv_indices: {req4_kv_indices}" + ) + prefix_len = tree.insert(RadixKey(req4_token_ids), req4_kv_indices) + self.assertEqual(prefix_len, 4) + print( + f"req4: prefix_len: {prefix_len}, allocator swa available size: {allocator.swa_available_size()}, full available size: {allocator.full_available_size()}" + ) + + tree.pretty_print() + full_num_tokens, swa_num_tokens = 1, 0 + print(f"evicting {full_num_tokens} full token and {swa_num_tokens} swa token") + tree.evict(full_num_tokens=full_num_tokens, swa_num_tokens=swa_num_tokens) + tree.pretty_print() + + full_num_tokens, swa_num_tokens = 0, 1 + print(f"evicting {full_num_tokens} full token and {swa_num_tokens} swa token") + tree.evict(full_num_tokens=full_num_tokens, swa_num_tokens=swa_num_tokens) + tree.pretty_print() + + full_num_tokens, swa_num_tokens = 1, 2 + print(f"evicting {full_num_tokens} full token and {swa_num_tokens} swa token") + tree.evict(full_num_tokens=full_num_tokens, swa_num_tokens=swa_num_tokens) + tree.pretty_print() + + req5_token_ids = [1, 2, 3, 4, 5] + result = tree.match_prefix(RadixKey(req5_token_ids)) + kv_indices, last_node = result.device_indices, result.last_device_node + print( + f"req5: token_ids: {req5_token_ids}, matched kv_indices: {kv_indices}, last_node.key: {last_node.key}" + ) + self.assertEqual(len(kv_indices), 0) # no swa prefix matched + + req6_token_ids = [1, 2, 3, 4, 5, 60, 70] + result = tree.match_prefix(RadixKey(req6_token_ids)) + kv_indices, last_node = result.device_indices, result.last_device_node + print( + f"req6: token_ids: {req6_token_ids}, matched kv_indices: {kv_indices}, last_node.key: {last_node.key}" + ) + self.assertEqual(len(kv_indices), 6) + self.assertEqual(len(last_node.key), 2) + self.assertEqual(last_node.key.token_ids[0], (5, 60)) + self.assertEqual(last_node.key.token_ids[1], (60, 70)) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_torch_compile.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_torch_compile.py new file mode 100644 index 00000000..36815718 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_torch_compile.py @@ -0,0 +1,80 @@ +import time +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_amd_ci, + popen_launch_server, +) + + +class TestTorchCompile(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--enable-torch-compile", "--cuda-graph-max-bs", "4"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + def run_decode(self, max_new_tokens): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + "ignore_eos": True, + }, + }, + ) + return response.json() + + def test_throughput(self): + # Warmup + res = self.run_decode(16) + + max_tokens = 256 + tic = time.perf_counter() + res = self.run_decode(max_tokens) + tok = time.perf_counter() + print(f"{res=}") + throughput = max_tokens / (tok - tic) + print(f"Throughput: {throughput} tokens/s") + + if is_in_amd_ci(): + self.assertGreaterEqual(throughput, 145) + else: + self.assertGreaterEqual(throughput, 152) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_torch_compile_moe.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_torch_compile_moe.py new file mode 100644 index 00000000..8bc7b45d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_torch_compile_moe.py @@ -0,0 +1,77 @@ +import time +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import is_cuda, kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_BASE, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestTorchCompileMoe(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_BASE + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--enable-torch-compile", "--torch-compile-max-bs", "4"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.50) + + def run_decode(self, max_new_tokens): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + "ignore_eos": True, + }, + }, + ) + return response.json() + + def test_throughput(self): + # Warmup + res = self.run_decode(16) + + max_tokens = 256 + tic = time.perf_counter() + res = self.run_decode(max_tokens) + tok = time.perf_counter() + print(f"{res=}") + throughput = max_tokens / (tok - tic) + if is_cuda(): + self.assertGreaterEqual(throughput, 285) + else: + self.assertGreaterEqual(throughput, 270) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_torch_native_attention_backend.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_torch_native_attention_backend.py new file mode 100644 index 00000000..5bf012cc --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_torch_native_attention_backend.py @@ -0,0 +1,47 @@ +""" +Usage: +python3 -m unittest test_triton_attention_backend.TestTritonAttnBackend.test_mmlu +""" + +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestTorchNativeAttnBackend(CustomTestCase): + def test_mmlu(self): + model = DEFAULT_MODEL_NAME_FOR_TEST + base_url = DEFAULT_URL_FOR_TEST + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--attention-backend", "torch_native"], + ) + + try: + args = SimpleNamespace( + base_url=base_url, + model=model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_torchao.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_torchao.py new file mode 100644 index 00000000..53368aaa --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_torchao.py @@ -0,0 +1,95 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang import Engine +from sglang.lang.chat_template import get_chat_template_by_model_path +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_IMAGE_URL, + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestTorchAO(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--torchao-config", "int4wo-128"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + assert metrics["score"] >= 0.60 + + def run_decode(self, max_new_tokens): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + }, + "ignore_eos": True, + }, + ) + return response.json() + + def test_throughput(self): + import time + + max_tokens = 256 + + tic = time.perf_counter() + res = self.run_decode(max_tokens) + tok = time.perf_counter() + print(res["text"]) + throughput = max_tokens / (tok - tic) + print(f"Throughput: {throughput} tokens/s") + assert throughput >= 210 + + +class TestTorchAOForVLM(CustomTestCase): + def test_vlm_generate(self): + model_path = DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST + chat_template = get_chat_template_by_model_path(model_path) + text = f"{chat_template.image_token}What is in this picture? Answer: " + + engine = Engine( + model_path=model_path, + max_total_tokens=512, + enable_multimodal=True, + torchao_config="fp8wo", + ) + out = engine.generate([text], image_data=[DEFAULT_IMAGE_URL]) + engine.shutdown() + self.assertGreater(len(out), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_triton_attention_backend.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_triton_attention_backend.py new file mode 100644 index 00000000..05725301 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_triton_attention_backend.py @@ -0,0 +1,66 @@ +""" +Usage: +python3 -m unittest test_triton_attention_backend.TestTritonAttnBackend.test_mmlu +""" + +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_offline_throughput, +) + + +class TestTritonAttnBackend(CustomTestCase): + def test_latency(self): + output_throughput = run_bench_offline_throughput( + DEFAULT_MODEL_NAME_FOR_TEST, + [ + "--attention-backend", + "triton", + "--enable-torch-compile", + "--cuda-graph-max-bs", + 4, + ], + ) + + print(f"{output_throughput=}") + + if is_in_ci(): + self.assertGreater(output_throughput, 153) + + def test_mmlu(self): + model = DEFAULT_MODEL_NAME_FOR_TEST + base_url = DEFAULT_URL_FOR_TEST + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--attention-backend", "triton"], + ) + + try: + args = SimpleNamespace( + base_url=base_url, + model=model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_triton_attention_kernels.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_triton_attention_kernels.py new file mode 100644 index 00000000..5a7fb547 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_triton_attention_kernels.py @@ -0,0 +1,783 @@ +import random +import unittest + +import torch +import torch.nn.functional as F + +from sglang.srt.layers.attention.triton_ops.decode_attention import ( + decode_attention_fwd, + decode_attention_fwd_grouped, + decode_attention_fwd_normal, +) +from sglang.srt.layers.attention.triton_ops.extend_attention import ( + build_unified_kv_indices, + extend_attention_fwd, + extend_attention_fwd_unified, + redundant_attention, +) +from sglang.srt.layers.attention.triton_ops.prefill_attention import ( + context_attention_fwd, +) +from sglang.srt.utils import get_device +from sglang.test.test_utils import CustomTestCase + + +def extend_attention_fwd_torch( + q: torch.Tensor, # [extend_tokens, H_Q, D] + k: torch.Tensor, # [extend_tokens, H_KV, D] + v: torch.Tensor, # [extend_tokens, H_KV, D] + o: torch.Tensor, # [extend_tokens, H_Q, D] + k_cache: torch.Tensor, # [total_tokens, H_KV, D] + v_cache: torch.Tensor, # [total_tokens, H_KV, D] + qo_indptr: torch.Tensor, # [B+1] + kv_indptr: torch.Tensor, # [B+1] + kv_indices: torch.Tensor, # [prefix_tokens] + sliding_window_size: int, +): + B = qo_indptr.size(0) - 1 + _, H_Q, D = q.shape + _, H_KV, _ = k.shape + + group_size = H_Q // H_KV + scale = 1.0 / D**0.5 + + for i in range(B): + q_start = int(qo_indptr[i].item()) + q_end = int(qo_indptr[i + 1].item()) + kv_start = int(kv_indptr[i].item()) + kv_end = int(kv_indptr[i + 1].item()) + + prefix_indices = kv_indices[kv_start:kv_end] + k_prefix = k_cache[prefix_indices] # [prefix_len, H_KV, D] + v_prefix = v_cache[prefix_indices] # [prefix_len, H_KV, D] + + k_extend = k[q_start:q_end] # [extend_len, H_KV, D] + v_extend = v[q_start:q_end] # [extend_len, H_KV, D] + q_extend = q[q_start:q_end] # [extend_len, H_Q, D] + + k_full = torch.cat([k_prefix, k_extend], dim=0) # [total_len, H_KV, D] + v_full = torch.cat([v_prefix, v_extend], dim=0) # [total_len, H_KV, D] + + if group_size != 1: + k_full_hq = k_full.repeat_interleave( + group_size, dim=1 + ) # [total_len, H_Q, D] + v_full_hq = v_full.repeat_interleave( + group_size, dim=1 + ) # [total_len, H_Q, D] + else: + k_full_hq = k_full + v_full_hq = v_full + + prefix_len = k_prefix.size(0) + extend_len = k_extend.size(0) + total_len = prefix_len + extend_len + + # causal + pos_keys = torch.arange(total_len, device=q.device) + t = prefix_len + torch.arange(extend_len, device=q.device) # [extend_len] + causal_mask = pos_keys.unsqueeze(0) <= t.unsqueeze(1) + + # sliding window + if sliding_window_size is not None and sliding_window_size > 0: + start = (t - (sliding_window_size)).clamp_min(0) # [extend_len] + else: + start = torch.zeros_like(t) + window_mask = pos_keys.unsqueeze(0) >= start.unsqueeze(1) + + final_mask = causal_mask & window_mask + + attn_scores = ( + torch.einsum("qhd,khd->qhk", q_extend, k_full_hq) * scale + ) # [extend_len, H_Q, total_len] + attn_scores = attn_scores.masked_fill(~final_mask.unsqueeze(1), float("-inf")) + + attn_weights = F.softmax(attn_scores, dim=-1) + o[q_start:q_end] = torch.einsum("qhk,khd->qhd", attn_weights, v_full_hq) + + +class TestTritonAttention(CustomTestCase): + + def _set_all_seeds(self, seed): + """Set all random seeds for reproducibility.""" + random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + def setUp(self): + # Set seeds before each test method + self._set_all_seeds(42) + + def _test_extend_attention_once(self, B, N_CTX, H_Q, H_KV, D): + dtype = torch.bfloat16 + device = get_device() + + b_seq_len_prefix = torch.randint( + 1, N_CTX // 2, (B,), dtype=torch.int32, device=device + ) + b_seq_len_extend = torch.randint( + 1, N_CTX // 2, (B,), dtype=torch.int32, device=device + ) + b_seq_len = b_seq_len_prefix + b_seq_len_extend + max_len_in_batch = torch.max(b_seq_len, 0)[0].item() + + b_req_idx = torch.arange(B, dtype=torch.int32, device=device) + b_start_loc = torch.zeros((B,), dtype=torch.int32, device=device) + b_start_loc[1:] = torch.cumsum(b_seq_len[:-1], 0) + b_start_loc_extend = torch.zeros((B,), dtype=torch.int32, device=device) + b_start_loc_extend[1:] = torch.cumsum(b_seq_len_extend[:-1], 0) + + kv_indptr = torch.zeros((B + 1,), dtype=torch.int32, device=device) + kv_indptr[1 : B + 1] = torch.cumsum(b_seq_len_prefix[:B], dim=0) + kv_indices = torch.zeros( + (b_seq_len_prefix.sum().item(),), dtype=torch.int32, device=device + ) + + for i in range(B): + kv_indices[kv_indptr[i] : kv_indptr[i + 1]] = torch.arange( + b_start_loc[i], b_start_loc[i] + b_seq_len_prefix[i] + ) + + total_token_num = torch.sum(b_seq_len).item() + extend_token_num = torch.sum(b_seq_len_extend).item() + k_buffer = torch.empty( + (total_token_num, H_KV, D), dtype=dtype, device=device + ).normal_(mean=0.1, std=0.2) + v_buffer = torch.empty( + (total_token_num, H_KV, D), dtype=dtype, device=device + ).normal_(mean=0.1, std=0.2) + + k_extend = torch.empty((extend_token_num, H_KV, D), dtype=dtype, device=device) + v_extend = torch.empty((extend_token_num, H_KV, D), dtype=dtype, device=device) + q_extend = torch.empty((extend_token_num, H_Q, D), dtype=dtype, device=device) + for i in range(B): + extend_start_in_buffer = b_start_loc[i] + b_seq_len_prefix[i] + extend_end_in_buffer = b_start_loc[i] + b_seq_len[i] + extend_start = b_start_loc_extend[i] + extend_end = b_start_loc_extend[i] + b_seq_len_extend[i] + k_extend[extend_start:extend_end] = k_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + v_extend[extend_start:extend_end] = v_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + q_extend[extend_start:extend_end] = torch.empty( + (b_seq_len_extend[i], H_Q, D), dtype=dtype, device=device + ).normal_(mean=0.1, std=0.2) + + o_extend = torch.empty((extend_token_num, H_Q, D), dtype=dtype, device=device) + o_extend_mask = torch.empty( + (extend_token_num, H_Q, D), dtype=dtype, device=device + ) + o_redundant = torch.empty( + (extend_token_num, H_Q, D), dtype=dtype, device=device + ) + + b_seq_len_extend = b_seq_len - b_seq_len_prefix + max_len_extend = torch.max(b_seq_len_extend, 0)[0].item() + qo_indptr = torch.zeros((B + 1,), dtype=torch.int32, device=device) + qo_indptr[1 : B + 1] = torch.cumsum(b_seq_len_extend[:B], dim=0) + + custom_mask = None + mask_indptr = None + + extend_attention_fwd( + q_extend, + k_extend, + v_extend, + o_extend, + k_buffer, + v_buffer, + qo_indptr, + kv_indptr, + kv_indices, + custom_mask, + True, + mask_indptr, + max_len_extend, + ) + + b_seq_mask_len = b_seq_len_extend * b_seq_len + custom_mask = torch.ones( + (b_seq_mask_len.sum().item(),), dtype=torch.bool, device=device + ) + mask_indptr = torch.zeros((B + 1,), dtype=torch.int64, device=device) + mask_indptr[1 : B + 1] = torch.cumsum(b_seq_mask_len[:B], dim=0) + for i in range(B): + causal_mask = ( + torch.tril( + torch.ones(b_seq_len_extend[i], b_seq_len_extend[i]), diagonal=0 + ) + == 1 + ) + prefix_mask = torch.ones( + b_seq_len_extend[i], b_seq_len_prefix[i], dtype=torch.bool + ) + mask_flatten = torch.cat([prefix_mask, causal_mask], dim=1).flatten() + custom_mask[mask_indptr[i] : mask_indptr[i + 1]] = mask_flatten + + extend_attention_fwd( + q_extend, + k_extend, + v_extend, + o_extend_mask, + k_buffer, + v_buffer, + qo_indptr, + kv_indptr, + kv_indices, + custom_mask, + True, + mask_indptr, + max_len_extend, + ) + + redundant_attention( + q_extend, + o_redundant, + k_buffer, + v_buffer, + b_req_idx, + b_start_loc, + b_seq_len, + b_seq_len_prefix, + max_len_in_batch, + ) + + self.assertTrue(torch.allclose(o_extend, o_redundant, rtol=1e-2)) + self.assertTrue(torch.allclose(o_extend_mask, o_redundant, rtol=1e-2)) + + def test_extend_attention(self): + + # Define the varying parameter values + attention_values = [128, 96, 80, 13] + + # Loop through the values and call the method + for value in attention_values: + self._test_extend_attention_once(19, 12331, 12, 4, value) + + def _test_extend_attention_sliding_window_once( + self, B, N_CTX, H_Q, H_KV, D, WINDOW_SIZE + ): + dtype = torch.bfloat16 + device = get_device() + + b_seq_len_prefix = torch.randint( + 1, N_CTX // 2, (B,), dtype=torch.int32, device=device + ) + b_seq_len_extend = torch.randint( + 1, N_CTX // 2, (B,), dtype=torch.int32, device=device + ) + b_seq_len = b_seq_len_prefix + b_seq_len_extend + + b_start_loc = torch.zeros((B,), dtype=torch.int32, device=device) + b_start_loc[1:] = torch.cumsum(b_seq_len[:-1], 0) + b_start_loc_extend = torch.zeros((B,), dtype=torch.int32, device=device) + b_start_loc_extend[1:] = torch.cumsum(b_seq_len_extend[:-1], 0) + + kv_indptr = torch.zeros((B + 1,), dtype=torch.int32, device=device) + kv_indptr[1 : B + 1] = torch.cumsum(b_seq_len_prefix[:B], dim=0) + kv_indices = torch.zeros( + (b_seq_len_prefix.sum().item(),), dtype=torch.int32, device=device + ) + + for i in range(B): + kv_indices[kv_indptr[i] : kv_indptr[i + 1]] = torch.arange( + b_start_loc[i], b_start_loc[i] + b_seq_len_prefix[i] + ) + + total_token_num = torch.sum(b_seq_len).item() + extend_token_num = torch.sum(b_seq_len_extend).item() + k_buffer = torch.empty( + (total_token_num, H_KV, D), dtype=dtype, device=device + ).normal_(mean=0.1, std=0.2) + v_buffer = torch.empty( + (total_token_num, H_KV, D), dtype=dtype, device=device + ).normal_(mean=0.1, std=0.2) + + k_extend = torch.empty((extend_token_num, H_KV, D), dtype=dtype, device=device) + v_extend = torch.empty((extend_token_num, H_KV, D), dtype=dtype, device=device) + q_extend = torch.empty((extend_token_num, H_Q, D), dtype=dtype, device=device) + for i in range(B): + extend_start_in_buffer = b_start_loc[i] + b_seq_len_prefix[i] + extend_end_in_buffer = b_start_loc[i] + b_seq_len[i] + extend_start = b_start_loc_extend[i] + extend_end = b_start_loc_extend[i] + b_seq_len_extend[i] + k_extend[extend_start:extend_end] = k_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + v_extend[extend_start:extend_end] = v_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + q_extend[extend_start:extend_end] = torch.empty( + (b_seq_len_extend[i], H_Q, D), dtype=dtype, device=device + ).normal_(mean=0.1, std=0.2) + + o_extend_triton = torch.empty( + (extend_token_num, H_Q, D), dtype=dtype, device=device + ) + o_extend_torch = torch.empty( + (extend_token_num, H_Q, D), dtype=dtype, device=device + ) + + b_seq_len_extend = b_seq_len - b_seq_len_prefix + max_len_extend = torch.max(b_seq_len_extend, 0)[0].item() + qo_indptr = torch.zeros((B + 1,), dtype=torch.int32, device=device) + qo_indptr[1 : B + 1] = torch.cumsum(b_seq_len_extend[:B], dim=0) + + extend_attention_fwd( + q_extend, + k_extend, + v_extend, + o_extend_triton, + k_buffer, + v_buffer, + qo_indptr, + kv_indptr, + kv_indices, + custom_mask=None, + is_causal=True, + mask_indptr=None, + max_len_extend=max_len_extend, + sliding_window_size=WINDOW_SIZE, + ) + + extend_attention_fwd_torch( + q_extend, + k_extend, + v_extend, + o_extend_torch, + k_buffer, + v_buffer, + qo_indptr, + kv_indptr, + kv_indices, + WINDOW_SIZE, + ) + + self.assertTrue( + torch.allclose(o_extend_triton, o_extend_torch, rtol=1e-3, atol=1e-3) + ) + + def test_extend_attention_sliding_window(self): + window_sizes = [-1, 127] + for window_size in window_sizes: + self._test_extend_attention_sliding_window_once( + 19, 12331, 64, 8, 128, window_size + ) + + def _test_context_attention_once(self, head_dim, is_causal): + # Set up a simple test case + device = get_device() + num_heads = 4 + seq_lens = [8, 12] + max_seq_len = max(seq_lens) + + # Create random input tensors + q = torch.randn(sum(seq_lens), num_heads, head_dim, device=device) + k = torch.randn(sum(seq_lens), num_heads, head_dim, device=device) + v = torch.randn(sum(seq_lens), num_heads, head_dim, device=device) + o = torch.zeros(sum(seq_lens), num_heads, head_dim, device=device) + + # Create b_start_loc and b_seq_len tensors + b_start_loc = torch.tensor([0, seq_lens[0]], device=device) + b_seq_len = torch.tensor(seq_lens, device=device) + + context_attention_fwd( + q, k, v, o, b_start_loc, b_seq_len, max_seq_len, is_causal=is_causal + ) + + cu_seq_lens = [0] * (len(seq_lens) + 1) + for i, seq_len in enumerate(seq_lens): + cu_seq_lens[i + 1] = cu_seq_lens[i] + seq_len + + for i in range(len(seq_lens)): + start, end = cu_seq_lens[i], cu_seq_lens[i + 1] + o_torch = torch.nn.functional.scaled_dot_product_attention( + q[start:end].permute(1, 0, 2), + k[start:end].permute(1, 0, 2), + v[start:end].permute(1, 0, 2), + is_causal=is_causal, + ).permute(1, 0, 2) + + cos_sim = torch.nn.functional.cosine_similarity( + o[start:end].flatten(), o_torch.flatten(), dim=0 + ) + self.assertTrue(cos_sim.item() > 1 - (1e-5)) + self.assertTrue(torch.allclose(o[start:end], o_torch, atol=1e-2)) + + def test_context_attention(self): + head_dim = [128, 96, 80, 13] + + for dim in head_dim: + for is_causal in [True, False]: + self._test_context_attention_once(dim, is_causal) + + def _test_decode_attention_once(self, B, H_Q, H_KV, D): + device = get_device() + dtype = torch.bfloat16 + seq_len = 10 # This represents the number of tokens already in the sequence + total_tokens = B * seq_len + sm_scale = 1.0 / (D**0.5) + max_kv_splits = 8 + num_kv_splits = torch.full((B,), 4, dtype=torch.int32, device=device) + + # q represents the new token being generated, one per batch + q = torch.randn(B, H_Q, D, dtype=dtype, device=device) + + # k_buffer and v_buffer represent all previous tokens + k_buffer = torch.randn(total_tokens, H_KV, D, dtype=dtype, device=device) + v_buffer = torch.randn(total_tokens, H_KV, D, dtype=dtype, device=device) + + # o will have the same shape as q + o = torch.zeros(B, H_Q, D, dtype=dtype, device=device) + + b_seq_len = torch.full((B,), seq_len, device=device) + + kv_indptr = torch.zeros((B + 1,), dtype=torch.int32, device=device) + kv_indptr[1 : B + 1] = torch.cumsum(b_seq_len[:B], dim=0) + kv_indices = torch.arange(total_tokens, device=device) + + attn_logits = torch.empty( + (B, H_Q, max_kv_splits, D), + dtype=torch.float32, + device=device, + ) + attn_lse = torch.empty( + (B, H_Q, max_kv_splits), + dtype=torch.float32, + device=device, + ) + + decode_attention_fwd( + q, + k_buffer, + v_buffer, + o, + kv_indptr, + kv_indices, + attn_logits, + attn_lse, + num_kv_splits, + max_kv_splits, + sm_scale, + ) + + def test_decode_attention(self): + # Here we just to ensure there is no error + # TODO: correctnesss test + + # Test configurations + configs = [ + (2, 4, 4, 64), # MHA + (2, 4, 2, 64), # GQA + (2, 4, 4, 80), # Non-standard head dim + (2, 4, 4, 13), # Prime number head dim + ] + + for B, H_Q, H_KV, D in configs: + self._test_decode_attention_once(B, H_Q, H_KV, D) + + def _test_grouped_decode_attention_once(self, B, S, H_Q, H_KV, D, D_V): + dtype = torch.bfloat16 + device = get_device() + seq_len = S # This represents the number of tokens already in the sequence + total_tokens = B * seq_len + sm_scale = 1.0 / (D**0.5) + max_kv_splits = 8 + num_kv_splits = torch.full((B,), 4, dtype=torch.int32, device=device) + + # q represents the new token being generated, one per batch + q = torch.randn(B, H_Q, D, dtype=dtype, device=device) + + # k_buffer and v_buffer represent all previous tokens + k_buffer = torch.randn(total_tokens, H_KV, D, dtype=dtype, device=device) + v_buffer = torch.randn(total_tokens, H_KV, D_V, dtype=dtype, device=device) + + # o will have the same shape as q + o = torch.zeros(B, H_Q, D_V, dtype=dtype, device=device) + o_grouped = torch.zeros(B, H_Q, D_V, dtype=dtype, device=device) + + b_seq_len = torch.full((B,), seq_len, device=device) + + kv_indptr = torch.zeros((B + 1,), dtype=torch.int32, device=device) + kv_indptr[1 : B + 1] = torch.cumsum(b_seq_len[:B], dim=0) + kv_indices = torch.arange(total_tokens, device=device) + + attn_logits = torch.empty( + (B, H_Q, max_kv_splits, D_V), + dtype=torch.float32, + device=device, + ) + attn_lse = torch.empty( + (B, H_Q, max_kv_splits), + dtype=torch.float32, + device=device, + ) + + decode_attention_fwd_normal( + q, + k_buffer, + v_buffer, + o, + kv_indptr, + kv_indices, + attn_logits, + attn_lse, + num_kv_splits, + max_kv_splits, + sm_scale, + ) + + attn_logits1 = torch.empty( + (B, H_Q, max_kv_splits, D_V), + dtype=torch.float32, + device=device, + ) + attn_lse1 = torch.empty( + (B, H_Q, max_kv_splits, D_V), + dtype=torch.float32, + device=device, + ) + + decode_attention_fwd_grouped( + q, + k_buffer, + v_buffer, + o_grouped, + kv_indptr, + kv_indices, + attn_logits1, + attn_lse1, + num_kv_splits, + max_kv_splits, + sm_scale, + ) + + cos_sim = torch.nn.functional.cosine_similarity( + o.flatten(), o_grouped.flatten(), dim=0 + ) + print(cos_sim.item()) + self.assertTrue(cos_sim.item() > 0.99) + self.assertTrue(torch.allclose(o, o_grouped, atol=3e-2)) + + def test_grouped_decode_attention(self): + seq_lens = [5, 100, 128, 500] + configs = [ + (2, 16, 16, 64, 64), + (2, 16, 1, 64, 64), + (2, 64, 1, 13, 13), + (2, 128, 1, 80, 80), + (2, 128, 2, 512, 512), + (2, 128, 1, 576, 512), + ] + + for S in seq_lens: + for B, H_Q, H_KV, D, D_V in configs: + self._test_grouped_decode_attention_once(B, S, H_Q, H_KV, D, D_V) + + def _test_extend_attention_unified_vs_regular_once(self, B, N_CTX, H_Q, H_KV, D): + """Test that unified kernel produces same results as 2-stage kernel.""" + dtype = torch.bfloat16 + device = get_device() + + b_seq_len_prefix = torch.randint( + 1, N_CTX // 2, (B,), dtype=torch.int32, device=device + ) + b_seq_len_extend = torch.randint( + 1, N_CTX // 2, (B,), dtype=torch.int32, device=device + ) + b_seq_len = b_seq_len_prefix + b_seq_len_extend + + b_start_loc = torch.zeros((B,), dtype=torch.int32, device=device) + b_start_loc[1:] = torch.cumsum(b_seq_len[:-1], 0) + b_start_loc_extend = torch.zeros((B,), dtype=torch.int32, device=device) + b_start_loc_extend[1:] = torch.cumsum(b_seq_len_extend[:-1], 0) + + # Setup prefix KV indices + kv_indptr = torch.zeros((B + 1,), dtype=torch.int32, device=device) + kv_indptr[1 : B + 1] = torch.cumsum(b_seq_len_prefix[:B], dim=0) + kv_indices = torch.zeros( + (b_seq_len_prefix.sum().item(),), dtype=torch.int64, device=device + ) + + for i in range(B): + kv_indices[kv_indptr[i] : kv_indptr[i + 1]] = torch.arange( + b_start_loc[i], b_start_loc[i] + b_seq_len_prefix[i] + ) + + total_token_num = torch.sum(b_seq_len).item() + extend_token_num = torch.sum(b_seq_len_extend).item() + k_buffer = torch.empty( + (total_token_num, H_KV, D), dtype=dtype, device=device + ).normal_(mean=0.1, std=0.2) + v_buffer = torch.empty( + (total_token_num, H_KV, D), dtype=dtype, device=device + ).normal_(mean=0.1, std=0.2) + + k_extend = torch.empty((extend_token_num, H_KV, D), dtype=dtype, device=device) + v_extend = torch.empty((extend_token_num, H_KV, D), dtype=dtype, device=device) + q_extend = torch.empty((extend_token_num, H_Q, D), dtype=dtype, device=device) + + for i in range(B): + extend_start_in_buffer = b_start_loc[i] + b_seq_len_prefix[i] + extend_end_in_buffer = b_start_loc[i] + b_seq_len[i] + extend_start = b_start_loc_extend[i] + extend_end = b_start_loc_extend[i] + b_seq_len_extend[i] + k_extend[extend_start:extend_end] = k_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + v_extend[extend_start:extend_end] = v_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + q_extend[extend_start:extend_end] = torch.empty( + (b_seq_len_extend[i], H_Q, D), dtype=dtype, device=device + ).normal_(mean=0.1, std=0.2) + + # Setup for extend attention + max_len_extend = torch.max(b_seq_len_extend, 0)[0].item() + qo_indptr = torch.zeros((B + 1,), dtype=torch.int32, device=device) + qo_indptr[1 : B + 1] = torch.cumsum(b_seq_len_extend[:B], dim=0) + + # Run 2-stage kernel + o_regular = torch.empty((extend_token_num, H_Q, D), dtype=dtype, device=device) + extend_attention_fwd( + q_extend, + k_extend, + v_extend, + o_regular, + k_buffer, + v_buffer, + qo_indptr, + kv_indptr, + kv_indices, + custom_mask=None, + is_causal=True, + mask_indptr=None, + max_len_extend=max_len_extend, + ) + + # Build unified KV indices + extend_kv_indices = torch.arange( + total_token_num - extend_token_num, + total_token_num, + dtype=torch.int64, + device=device, + ) + extend_start_loc = torch.zeros((B,), dtype=torch.int32, device=device) + extend_start_loc[1:] = torch.cumsum(b_seq_len_extend[:-1], 0) + + unified_kv_indptr, unified_kv_indices, prefix_lens = build_unified_kv_indices( + kv_indptr, + kv_indices, + extend_start_loc, + b_seq_len_extend, + extend_kv_indices, + B, + ) + + # Run unified kernel + o_unified = torch.empty((extend_token_num, H_Q, D), dtype=dtype, device=device) + extend_attention_fwd_unified( + q_extend, + o_unified, + k_buffer, + v_buffer, + qo_indptr, + unified_kv_indptr, + unified_kv_indices, + prefix_lens, + max_len_extend=max_len_extend, + custom_mask=None, + mask_indptr=None, + sm_scale=None, + logit_cap=0.0, + is_causal=True, + ) + + # Compare results + self.assertTrue( + torch.allclose(o_regular, o_unified, rtol=0.15, atol=0.15), + f"Unified kernel output differs from 2-stage kernel. " + f"Max diff: {(o_regular - o_unified).abs().max()}", + ) + + def test_extend_attention_unified_vs_regular(self): + """Test unified kernel matches 2-stage kernel across different configs.""" + configs = [ + (4, 512, 32, 8, 128), # Standard config + (2, 2048, 32, 8, 128), # Long sequence (test 2048 specifically) + (8, 256, 64, 8, 80), # Non-standard head dim + ] + + for B, N_CTX, H_Q, H_KV, D in configs: + with self.subTest(B=B, N_CTX=N_CTX, H_Q=H_Q, H_KV=H_KV, D=D): + self._test_extend_attention_unified_vs_regular_once( + B, N_CTX, H_Q, H_KV, D + ) + + def test_build_unified_kv_indices(self): + """Test build_unified_kv_indices correctness.""" + B = 4 + dtype = torch.int64 + device = get_device() + + # Setup test data + prefix_lens = torch.tensor([10, 20, 15, 25], dtype=torch.int32, device=device) + extend_lens = torch.tensor([5, 3, 7, 4], dtype=torch.int32, device=device) + + # Build prefix indices + prefix_kv_indptr = torch.zeros((B + 1,), dtype=torch.int32, device=device) + prefix_kv_indptr[1:] = torch.cumsum(prefix_lens, dim=0) + prefix_kv_indices = torch.arange( + prefix_lens.sum().item(), dtype=dtype, device=device + ) + + # Build extend indices + extend_start_loc = torch.zeros((B,), dtype=torch.int32, device=device) + extend_start_loc[1:] = torch.cumsum(extend_lens[:-1], dim=0) + extend_kv_indices = torch.arange( + prefix_lens.sum().item(), + prefix_lens.sum().item() + extend_lens.sum().item(), + dtype=dtype, + device=device, + ) + + # Build unified indices + unified_kv_indptr, unified_kv_indices, returned_prefix_lens = ( + build_unified_kv_indices( + prefix_kv_indptr, + prefix_kv_indices, + extend_start_loc, + extend_lens, + extend_kv_indices, + B, + ) + ) + + # Verify unified_kv_indptr + expected_lens = prefix_lens + extend_lens + expected_indptr = torch.zeros((B + 1,), dtype=torch.int32, device=device) + expected_indptr[1:] = torch.cumsum(expected_lens, dim=0) + self.assertTrue(torch.equal(unified_kv_indptr, expected_indptr)) + + # Verify prefix_lens + self.assertTrue(torch.equal(returned_prefix_lens, prefix_lens)) + + # Verify unified_kv_indices structure + for i in range(B): + start_idx = int(unified_kv_indptr[i]) + end_idx = int(unified_kv_indptr[i + 1]) + prefix_len = int(prefix_lens[i]) + extend_len = int(extend_lens[i]) + + # Check that prefix and extend are concatenated correctly + unified_seq = unified_kv_indices[start_idx:end_idx] + self.assertEqual(len(unified_seq), prefix_len + extend_len) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_triton_fused_moe.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_triton_fused_moe.py new file mode 100644 index 00000000..07e774a9 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_triton_fused_moe.py @@ -0,0 +1,192 @@ +import unittest + +import torch +from tqdm import tqdm + +from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.moe import MoeRunner, MoeRunnerBackend, MoeRunnerConfig +from sglang.srt.layers.moe.moe_runner.triton_kernels import TritonKernelsQuantInfo +from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatchOutput +from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat +from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler +from sglang.test.test_utils import CustomTestCase + + +class TestFusedMOE(CustomTestCase): + NUM_EXPERTS = [8, 64] + TOP_KS = [2, 4] + + @staticmethod + def create_random_cuda_tensor(shape, dtype, mean=0, std=0.01): + """Create a random CUDA tensor + + Args: + shape: Tensor shape + dtype: Data type + mean: Mean value + std: Standard deviation + + Returns: + torch.Tensor: Randomly initialized CUDA tensor + """ + return torch.empty(shape, dtype=dtype, device="cuda").normal_(mean, std) + + def get_tolerance(self, dtype): + """Get tolerance values for different data types + + Args: + dtype: Data type + + Returns: + tuple: (relative tolerance, absolute tolerance) + """ + if dtype == torch.float32: + return 1e-5, 1e-5 + elif dtype in [torch.float16, torch.bfloat16]: + return 1e-5, 1e-5 + else: + return 1e-2, 1e-2 # Default values for other types + + def torch_naive_moe( + self, + a, + w1, + w2, + score, + topk, + return_per_expert: bool = False, + ): + set_global_server_args_for_scheduler(ServerArgs(model_path="dummy")) + + B, D = a.shape + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + + if w1.dtype == torch.float8_e4m3fn: + w1_compute = w1.to(a.dtype) + w2_compute = w2.to(a.dtype) + else: + w1_compute = w1 + w2_compute = w2 + + for i in range(w1_compute.shape[0]): + mask = topk_ids == i + if mask.sum(): + out[mask] = SiluAndMul()( + a[mask] @ w1_compute[i].transpose(0, 1) + ) @ w2_compute[i].transpose(0, 1) + + weighted = out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to( + out.dtype + ) + + if return_per_expert: + return weighted + + return weighted.sum(dim=1) + + def _test_case(self, m, n, k, e, topk, dtype): + rtol, atol = self.get_tolerance(dtype) + + a = self.create_random_cuda_tensor((m, k), dtype) + w1 = self.create_random_cuda_tensor((e, 2 * n, k), dtype) + w2 = self.create_random_cuda_tensor((e, k, n), dtype) + w1_tri = w1.clone() + w2_tri = w2.clone() + w1_tri = w1_tri.transpose(-2, -1).contiguous() + w2_tri = w2_tri.transpose(-2, -1).contiguous() + score = self.create_random_cuda_tensor((m, e), dtype) + + topk_op = TopK( + top_k=topk, + renormalize=False, + use_grouped_topk=False, + ) + topk_op.topk_config.output_format = TopKOutputFormat.TRITON_KERNEL + triton_topk_output = topk_op.forward_cuda( + hidden_states=a, + router_logits=score, + ) + + quant_info = TritonKernelsQuantInfo(w13_weight=w1_tri, w2_weight=w2_tri) + + dispatch_output = StandardDispatchOutput( + hidden_states=a, hidden_states_scale=None, topk_output=triton_topk_output + ) + + torch_per_expert = self.torch_naive_moe( + a, w1, w2, score, topk, return_per_expert=True + ) + torch_combined = torch_per_expert.sum(dim=1) + + def run_runner(config): + runner = MoeRunner(MoeRunnerBackend.TRITON_KERNELS, config) + result = runner.run(dispatch_output, quant_info) + return result.hidden_states + + # Combined output (no_combine=False) + non_fused_config = MoeRunnerConfig(inplace=False) + non_fused_output = run_runner(non_fused_config) + torch.testing.assert_close( + non_fused_output, torch_combined, rtol=rtol, atol=atol + ) + + # Per-expert output (no_combine=True) + non_fused_no_combine_config = MoeRunnerConfig( + inplace=False, no_combine=True, top_k=topk + ) + non_fused_no_combine_output = run_runner(non_fused_no_combine_config) + torch.testing.assert_close( + non_fused_no_combine_output, torch_per_expert, rtol=rtol, atol=atol + ) + + def test_various_configurations(self): + m_values = [1, 32, 64, 256] + n_values = [128, 1024] + k_values = [128, 512, 1024] + dtypes = [torch.bfloat16] + + # Calculate total number of tests + total_tests = ( + len(m_values) + * len(n_values) + * len(k_values) + * len(self.NUM_EXPERTS) + * len(self.TOP_KS) + * len(dtypes) + ) + + # Create progress bar + with tqdm(total=total_tests, desc="Running MoE tests") as pbar: + for m in m_values: + for n in n_values: + for k in k_values: + for e in self.NUM_EXPERTS: + for topk in self.TOP_KS: + for dtype in dtypes: + with self.subTest( + m=m, + n=n, + k=k, + e=e, + topk=topk, + dtype=dtype, + ): + self._test_case( + m, + n, + k, + e, + topk, + dtype, + ) + torch.cuda.empty_cache() + pbar.update(1) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_triton_moe_channel_fp8_kernel.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_triton_moe_channel_fp8_kernel.py new file mode 100644 index 00000000..ab280925 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_triton_moe_channel_fp8_kernel.py @@ -0,0 +1,184 @@ +import itertools +import unittest + +import torch + +from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import fused_moe +from sglang.srt.layers.moe.topk import TopKConfig, select_experts +from sglang.srt.layers.quantization.fp8_kernel import scaled_fp8_quant +from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler +from sglang.test.test_utils import CustomTestCase + + +def native_w8a8_per_token_matmul(A, B, As, Bs, output_dtype=torch.float16): + """Matrix multiplication function that supports per-token input quantization and per-column weight quantization""" + A = A.to(torch.float32) + B = B.to(torch.float32) + + assert A.shape[-1] == B.shape[-1], "Dimension mismatch" + assert B.ndim == 2 and B.is_contiguous(), "B must be a 2D contiguous tensor" + + # Reshape input + M = A.numel() // A.shape[-1] + B = B.t() # Transpose weight matrix + N, K = B.shape + origin_C_shape = A.shape[:-1] + (K,) + A = A.reshape(M, N) + + # As is per-token [M, 1], Bs is per-column [1, K] + C = torch.matmul(A, B) # [M, K] + C = As * C * Bs.view(1, -1) # Broadcast per-column scale + + return C.reshape(origin_C_shape).to(output_dtype) + + +def fp8_mask(a, mask): + dtype = a.dtype + return a.view(torch.int8)[mask].view(dtype) + + +def torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, score, topk): + """This function performs fused moe with per-column int8 quantization using native torch.""" + + set_global_server_args_for_scheduler(ServerArgs(model_path="dummy")) + + B, D = a.shape + # Perform per-token quantization + a_q, a_s = scaled_fp8_quant(a, use_per_token_if_dynamic=True) + # Repeat tokens to match topk + a_q = a_q.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + # Also repeat the scale + a_s = a_s.view(B, -1, 1).repeat(1, topk, 1).reshape(-1, 1) # [B*topk, 1] + + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) + + # Calculate routing + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + # Process each expert + for i in range(w1.shape[0]): + mask = topk_ids == i + if mask.sum(): + # First MLP layer: note that a_s is now per-token + inter_out = native_w8a8_per_token_matmul( + fp8_mask(a_q, mask), + w1[i], + fp8_mask(a_s, mask), + w1_s[i], + output_dtype=a.dtype, + ) + # Activation function + act_out = SiluAndMul().forward_native(inter_out) + # Quantize activation output with per-token + act_out_q, act_out_s = scaled_fp8_quant( + act_out, use_per_token_if_dynamic=True + ) + + # Second MLP layer + out[mask] = native_w8a8_per_token_matmul( + act_out_q, w2[i], act_out_s, w2_s[i], output_dtype=a.dtype + ) + # Apply routing weights and sum + return ( + out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype) + ).sum(dim=1) + + +class TestW8A8FP8FusedMoE(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16] + M = [1, 33] + N = [128, 1024] + K = [256, 4096] + E = [8] + TOP_KS = [2, 6] + BLOCK_SIZE = [[64, 64], [64, 128], [128, 64], [128, 128]] + BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _w8a8_fp8_fused_moe(self, M, N, K, E, topk, block_size, dtype, seed): + torch.manual_seed(seed) + # Initialize int8 quantization parameters + factor_for_scale = 1e-2 + finfo = torch.finfo(torch.float8_e4m3fn) + fp8_max = finfo.max + fp8_min = finfo.min + + # Input tensor + # M * K + a = torch.randn((M, K), dtype=dtype) / 10 + + # Generate int8 weights + w1_fp32 = (torch.rand((E, 2 * N, K), dtype=torch.float32) - 0.5) * 2 + w1 = (w1_fp32 * fp8_max).clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + w2_fp32 = (torch.rand((E, K, N), dtype=torch.float32) - 0.5) * 2 + w2 = (w2_fp32 * fp8_max).clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + # Generate scale for each column (per-column quantization) + w1_s = torch.rand(E, 2 * N, device=w1_fp32.device) * factor_for_scale + w2_s = torch.rand(E, K, device=w2_fp32.device) * factor_for_scale + score = torch.randn((M, E), dtype=dtype) + + with torch.inference_mode(): + ref_out = torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, score, topk) + topk_output = select_experts( + hidden_states=a, + router_logits=score, + topk_config=TopKConfig(top_k=topk, renormalize=False), + ) + out = fused_moe( + a, + w1, + w2, + topk_output, + use_fp8_w8a8=True, # using fp8 + use_int8_w8a16=False, + use_int8_w8a8=False, + per_channel_quant=True, + w1_scale=w1_s, + w2_scale=w2_s, + block_shape=None, # Not using block quantization + ) + + # Check results + self.assertTrue( + torch.mean(torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))) + / torch.mean(torch.abs(ref_out.to(torch.float32))) + < 0.05 + ) + + def test_w8a8_fp8_fused_moe(self): + for params in itertools.product( + self.M, + self.N, + self.K, + self.E, + self.TOP_KS, + self.BLOCK_SIZE, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + E=params[3], + topk=params[4], + block_size=params[5], + dtype=params[6], + seed=params[7], + ): + self._w8a8_fp8_fused_moe(*params) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_triton_sliding_window.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_triton_sliding_window.py new file mode 100644 index 00000000..f2b422c5 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_triton_sliding_window.py @@ -0,0 +1,129 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) + + +class TestSlidingWindowAttentionTriton(CustomTestCase): + """Test sliding window attention functionality with triton backend.""" + + @classmethod + def setUpClass(cls): + """Set up the test server with Gemma3 model and triton backend.""" + # Gemma3 model supports sliding window attention + cls.model = "google/gemma-3-4b-it" + cls.base_url = DEFAULT_URL_FOR_TEST + + cls.common_args = [ + "--trust-remote-code", + "--attention-backend", + "triton", + "--context-length", + "8192", + "--random-seed", + "42", + ] + + cls.short_context_prompt = "The capital of France is" + + # Test prompt longer than window size + cls.long_context_prompt = ( + """ + Once upon a time, there was a mountain. In the mountain, there was a temple. In the temple, there was an old monk telling a story. The story was: + """ + * 100 + ) + cls.long_context_prompt += "\nNow, summarize the story in one sentence:" + + def _test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=200, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"MMLU metrics with sliding window: {metrics}") + + self.assertGreaterEqual(metrics["score"], 0.60) + + def _test_short_context_generation(self): + response = requests.post( + self.base_url + "/generate", + json={ + "text": self.short_context_prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 256, + }, + }, + ) + + self.assertEqual(response.status_code, 200) + result = response.json() + self.assertIn("paris", result["text"].lower()) + print(f"Short context generation result: {result['text']}") + + def _test_long_context_generation(self): + response = requests.post( + self.base_url + "/generate", + json={ + "text": self.long_context_prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 256, + }, + }, + ) + + self.assertEqual(response.status_code, 200) + result = response.json() + self.assertGreater(len(result["text"].strip()), 0) + print(f"Long context generation result: {result['text'][:100]}...") + + @unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") + def test_no_cuda_graph(self): + self.no_cuda_graph_process = popen_launch_server( + self.model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=self.common_args + ["--disable-cuda-graph"], + ) + + try: + self._test_short_context_generation() + self._test_long_context_generation() + self._test_mmlu() + finally: + kill_process_tree(self.no_cuda_graph_process.pid) + + def test_cuda_graph(self): + self.cuda_graph_process = popen_launch_server( + self.model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=self.common_args, + ) + + try: + self._test_short_context_generation() + self._test_long_context_generation() + self._test_mmlu() + finally: + kill_process_tree(self.cuda_graph_process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_type_based_dispatcher.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_type_based_dispatcher.py new file mode 100644 index 00000000..1049a40a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_type_based_dispatcher.py @@ -0,0 +1,222 @@ +# tests/benchmarks/test_type_dispatcher_e2e.py +""" +E2E test for TypeBasedDispatcher optimization. +Tests real-world scenarios with actual request types. +""" + +import timeit +import unittest + +from sglang.srt.managers.io_struct import SamplingParams +from sglang.utils import TypeBasedDispatcher + + +class TestTypeBasedDispatcher(unittest.TestCase): + """Unit tests for TypeBasedDispatcher e2e performance.""" + + def test_type_dispatcher_e2e_performance(self): + """End-to-end performance test with real request types""" + print("E2E Performance Test for TypeBasedDispatcher") + print("=" * 50) + + from sglang.srt.managers.io_struct import ( + AbortReq, + BatchTokenizedEmbeddingReqInput, + BatchTokenizedGenerateReqInput, + ClearHiCacheReqInput, + CloseSessionReqInput, + DestroyWeightsUpdateGroupReqInput, + ExpertDistributionReq, + FlushCacheReqInput, + FreezeGCReq, + GetInternalStateReq, + GetLoadReqInput, + GetWeightsByNameReqInput, + InitWeightsSendGroupForRemoteInstanceReqInput, + InitWeightsUpdateGroupReqInput, + LoadLoRAAdapterReqInput, + OpenSessionReqInput, + ProfileReq, + ReleaseMemoryOccupationReqInput, + ResumeMemoryOccupationReqInput, + RpcReqInput, + SendWeightsToRemoteInstanceReqInput, + SetInternalStateReq, + SlowDownReqInput, + TokenizedEmbeddingReqInput, + TokenizedGenerateReqInput, + UnloadLoRAAdapterReqInput, + UpdateWeightFromDiskReqInput, + UpdateWeightsFromIPCReqInput, + UpdateWeightsFromTensorReqInput, + ) + + mapping = [ + (TokenizedGenerateReqInput, lambda req: "generate_handled"), + (TokenizedEmbeddingReqInput, lambda req: "embedding_handled"), + (BatchTokenizedGenerateReqInput, lambda req: "batch_generate_handled"), + ( + BatchTokenizedEmbeddingReqInput, + lambda req: "batch_generate_embedding_handled", + ), + (FlushCacheReqInput, lambda req: "flush_cache_handled"), + (ClearHiCacheReqInput, lambda req: "clear_hicache_handled"), + (AbortReq, lambda req: "abort_handled"), + (OpenSessionReqInput, lambda req: "open_session_handled"), + (CloseSessionReqInput, lambda req: "close_session_handled"), + ( + UpdateWeightFromDiskReqInput, + lambda req: "update_weights_from_disk_handled", + ), + ( + InitWeightsUpdateGroupReqInput, + lambda req: "init_weights_update_group_handled", + ), + ( + DestroyWeightsUpdateGroupReqInput, + lambda req: "destroy_weights_update_group_handled", + ), + ( + InitWeightsSendGroupForRemoteInstanceReqInput, + lambda req: "init_weights_send_group_for_remote_instance_handled", + ), + ( + SendWeightsToRemoteInstanceReqInput, + lambda req: "send_weights_to_remote_instance_handled", + ), + ( + UpdateWeightsFromTensorReqInput, + lambda req: "update_weights_from_tensor_handled", + ), + ( + UpdateWeightsFromIPCReqInput, + lambda req: "update_weights_from_ipc_handled", + ), + (GetWeightsByNameReqInput, lambda req: "get_weights_by_name_handled"), + ( + ReleaseMemoryOccupationReqInput, + lambda req: "release_memory_occupation_handled", + ), + ( + ResumeMemoryOccupationReqInput, + lambda req: "resume_memory_occupation_handled", + ), + (SlowDownReqInput, lambda req: "slow_down_handled"), + (ProfileReq, lambda req: "profile_handled"), + (FreezeGCReq, lambda req: "freeze_gc_handled"), + (GetInternalStateReq, lambda req: "get_internal_state_handled"), + (SetInternalStateReq, lambda req: "set_internal_state_handled"), + (RpcReqInput, lambda req: "rpc_request_handled"), + (ExpertDistributionReq, lambda req: "expert_distribution_handled"), + (LoadLoRAAdapterReqInput, lambda req: "load_lora_adapter_handled"), + (UnloadLoRAAdapterReqInput, lambda req: "unload_lora_adapter_handled"), + (GetLoadReqInput, lambda req: "get_load_handled"), + ] + + # Create requests that conforms to the real distribution + test_requests = [] + + test_requests.append( + TokenizedGenerateReqInput( + input_text="", + input_ids=[1, 2], + mm_inputs=dict(), + sampling_params=SamplingParams(), + return_logprob=False, + logprob_start_len=0, + top_logprobs_num=0, + token_ids_logprob=[1, 2], + stream=False, + ) + ) + + test_requests.append( + TokenizedEmbeddingReqInput( + input_text="", + input_ids=[1, 2], + image_inputs=dict(), + token_type_ids=[1, 2], + sampling_params=SamplingParams(), + ) + ) + + test_requests.append( + BatchTokenizedGenerateReqInput( + batch=[ + TokenizedGenerateReqInput( + input_text="", + input_ids=[1, 2], + mm_inputs=dict(), + sampling_params=SamplingParams(), + return_logprob=False, + logprob_start_len=0, + top_logprobs_num=0, + token_ids_logprob=[1, 2], + stream=False, + ) + ] + ) + ) + test_requests.append( + BatchTokenizedEmbeddingReqInput( + batch=[ + TokenizedEmbeddingReqInput( + input_text="", + input_ids=[1, 2], + image_inputs=dict(), + token_type_ids=[1, 2], + sampling_params=SamplingParams(), + ) + ] + ) + ) + + test_requests.append(FlushCacheReqInput()) + test_requests.append(ClearHiCacheReqInput()) + test_requests.append(AbortReq()) + test_requests.append(OpenSessionReqInput(capacity_of_str_len=0)) + test_requests.append(CloseSessionReqInput(session_id="")) + test_requests.append(UpdateWeightFromDiskReqInput(model_path="")) + test_requests.append( + InitWeightsUpdateGroupReqInput( + master_address="", + master_port=0, + rank_offset=0, + world_size=0, + group_name="", + ) + ) + test_requests.append(DestroyWeightsUpdateGroupReqInput()) + test_requests.append( + InitWeightsSendGroupForRemoteInstanceReqInput( + master_address="", ports="", group_name="", world_size=0, group_rank=0 + ) + ) + test_requests.append( + SendWeightsToRemoteInstanceReqInput(master_address="", ports="") + ) + test_requests.append( + UpdateWeightsFromTensorReqInput(serialized_named_tensors=[]) + ) + test_requests.append(GetWeightsByNameReqInput(name="")) + test_requests.append(ReleaseMemoryOccupationReqInput()) + test_requests.append(RpcReqInput(method="")) + test_requests.append(GetLoadReqInput()) + + dispatcher = TypeBasedDispatcher(mapping) + + # test + time_taken = timeit.timeit( + lambda: [dispatcher(req) for req in test_requests], + number=100, # Average of 100 runs + ) + + print(f"Total requests: {len(test_requests)}") + print(f"Time taken: {time_taken:.4f}s") + print(f"Requests per second: {len(test_requests) * 100 / time_taken:.0f}") + + return time_taken + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_utils_update_weights.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_utils_update_weights.py new file mode 100644 index 00000000..8c138f0a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_utils_update_weights.py @@ -0,0 +1,170 @@ +import asyncio +import os +import unittest + +import torch +import torch.distributed as dist +from torch.distributed.device_mesh import init_device_mesh +from transformers import AutoModelForCausalLM + +from sglang.srt.entrypoints.engine import Engine +from sglang.srt.weight_sync.utils import update_weights +from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST + + +class AsyncEngine(Engine): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + async def update_weights_from_tensor(self, update_weights_request): + return await self.tokenizer_manager.update_weights_from_tensor( + update_weights_request, None + ) + + +def is_distributed_available(): + """Check if distributed training environment is available""" + required_vars = ["RANK", "WORLD_SIZE", "MASTER_ADDR", "MASTER_PORT"] + return all(var in os.environ for var in required_vars) + + +def setup_single_process_distributed(): + """Setup distributed environment for single process testing""" + if not is_distributed_available(): + os.environ["RANK"] = "0" + os.environ["WORLD_SIZE"] = "1" + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = "12356" + os.environ["LOCAL_RANK"] = "0" + + +class TestUtilsUpdateWeights(unittest.TestCase): + """Test class for utils.update_weights function""" + + @classmethod + def setUpClass(cls): + """Setup distributed environment and test fixtures for the entire test class""" + cls.setup_distributed() + cls.setup_test_engine() + cls.setup_test_model() + cls.setup_device_mesh() + + @classmethod + def tearDownClass(cls): + """Cleanup after all tests""" + if hasattr(cls, "engine") and cls.engine: + cls.engine.shutdown() + + # Cleanup distributed + if dist.is_initialized(): + dist.destroy_process_group() + + @classmethod + def setup_distributed(cls): + """Setup distributed environment for testing""" + setup_single_process_distributed() + + if not dist.is_initialized(): + try: + dist.init_process_group( + backend="nccl" if torch.cuda.is_available() else "gloo" + ) + except Exception as e: + raise unittest.SkipTest( + f"Could not initialize distributed backend: {e}" + ) + + cls.rank = dist.get_rank() + cls.world_size = dist.get_world_size() + + if torch.cuda.is_available(): + torch.cuda.set_device(cls.rank % torch.cuda.device_count()) + + # Set up environment variables + os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" + os.environ["NCCL_CUMEM_ENABLE"] = "0" + os.environ["CUDA_DEVICE_MAX_CONNECTIONS"] = "4" + os.environ["CUDA_MODULE_LOADING"] = "AUTO" + + @classmethod + def setup_test_engine(cls): + """Setup test engine""" + if cls.rank == 0: + cls.engine = AsyncEngine( + model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + dtype="bfloat16", + mem_fraction_static=0.3, + enable_memory_saver=True, + tp_size=cls.world_size, + disable_cuda_graph=False, + ) + else: + cls.engine = None + + @classmethod + def setup_test_model(cls): + """Load test model""" + try: + cls.model = AutoModelForCausalLM.from_pretrained( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + device_map="cpu", + trust_remote_code=True, + low_cpu_mem_usage=True, + torch_dtype=( + torch.float16 if torch.cuda.is_available() else torch.float32 + ), + ) + except Exception as e: + raise unittest.SkipTest(f"Could not load test model: {e}") + + @classmethod + def setup_device_mesh(cls): + """Create device mesh for testing""" + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA not available for device mesh") + + cls.device_mesh_key = "tp" + cls.mesh = init_device_mesh( + "cuda", (cls.world_size,), mesh_dim_names=(cls.device_mesh_key,) + ) + + def create_test_params_batch(self, model, num_params=64): + """Create a batch of test parameters from the model""" + param_names = [] + test_tensors = [] + + # Get first few parameters from the model for testing + for i, (name, tensor) in enumerate(model.named_parameters()): + if i >= num_params: + break + param_names.append(name) + # Create test tensor with known values, matching original shape and dtype + test_tensor = torch.full_like(tensor, 1.5, dtype=tensor.dtype).cuda() + test_tensors.append(test_tensor) + + return list(zip(param_names, test_tensors)) + + def test_utils_update_weights(self): + """Test basic functionality of utils.update_weights""" + + async def async_test(): + # Create test parameters batch + params_batch = self.create_test_params_batch(self.model, num_params=2) + + # Test the utils.update_weights function + result = await update_weights( + engine=self.engine, + params_batch=params_batch, + device_mesh_key=self.device_mesh_key, + device_mesh=self.mesh, + load_format=None, + ) + + self.assertIn("Success", result) + + # Run the async test + asyncio.run(async_test()) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_video_utils.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_video_utils.py new file mode 100644 index 00000000..af21a6fb --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_video_utils.py @@ -0,0 +1,59 @@ +from dataclasses import dataclass + +import pytest + +from sglang.srt.utils import sample_video_frames + + +class DummyVideo: + def __init__(self, total_frames: int, avg_fps: float): + self._frames = total_frames + self._fps = avg_fps + + def __len__(self): + return self._frames + + def get_avg_fps(self): + return self._fps + + +@dataclass(kw_only=True) +class Case: + frames: int + avg_fps: float + desired_fps: int + max_frames: int + expected_frames: list[int] + description: str + + +# fmt: off +@pytest.mark.parametrize("case", [ + Case( + frames=100, avg_fps=25.0, desired_fps=5, max_frames=200, + expected_frames=[0, 5, 10, 15, 20, 26, 31, 36, 41, 46, 52, 57, 62, 67, 72, 78, 83, 88, 93, 99], + description="capped by desired_fps" + ), + Case( + frames=10, avg_fps=10.0, desired_fps=100, max_frames=5, + expected_frames=[0, 2, 4, 6, 9], + description="capped by max_frames" + ), + Case( + frames=50, avg_fps=25.0, desired_fps=50, max_frames=200, + expected_frames=list(range(50)), + description="capped by total_frames" + ), + Case( + frames=1, avg_fps=30.0, desired_fps=0, max_frames=0, + expected_frames=[0], + description="always sample at least 1 frame" + ) +], ids=lambda c: c.description) +def test_sample_video_frames_lengths(case: Case): + video = DummyVideo(case.frames, case.avg_fps) + result = sample_video_frames(video, desired_fps=case.desired_fps, max_frames=case.max_frames) + assert result == case.expected_frames + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_vision_chunked_prefill.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_vision_chunked_prefill.py new file mode 100644 index 00000000..305cbe0a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_vision_chunked_prefill.py @@ -0,0 +1,200 @@ +""" +Usage: +python3 -m unittest test_vision_chunked_prefill.TestVisionChunkedPrefill.test_chunked_prefill +""" + +import io +import os +import unittest +from concurrent.futures import ThreadPoolExecutor +from typing import Union + +import numpy as np +import pybase64 +import requests +from PIL import Image + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + calculate_rouge_l, + popen_launch_server, +) + + +class TestVisionChunkedPrefill(CustomTestCase): + + def prepare_video_messages(self, video_path, max_frames_num=8): + # We import decord here to avoid a strange Segmentation fault (core dumped) issue. + # The following import order will cause Segmentation fault. + # import decord + # from transformers import AutoTokenizer + from decord import VideoReader, cpu + + vr = VideoReader(video_path, ctx=cpu(0)) + total_frame_num = len(vr) + uniform_sampled_frames = np.linspace( + 0, total_frame_num - 1, max_frames_num, dtype=int + ) + frame_idx = uniform_sampled_frames.tolist() + frames = vr.get_batch(frame_idx).asnumpy() + + base64_frames = [] + for frame in frames: + pil_img = Image.fromarray(frame) + buff = io.BytesIO() + pil_img.save(buff, format="JPEG") + base64_str = pybase64.b64encode(buff.getvalue()).decode("utf-8") + base64_frames.append(base64_str) + + messages = [{"role": "user", "content": []}] + frame_format = { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,{}"}, + "modalities": "video", + } + + for base64_frame in base64_frames: + frame_format["image_url"]["url"] = "data:image/jpeg;base64,{}".format( + base64_frame + ) + messages[0]["content"].append(frame_format.copy()) + + prompt = {"type": "text", "text": "Please describe the video briefly."} + messages[0]["content"].append(prompt) + + return messages + + def get_prompt_from_messages(self, messages): + text = ( + "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" + "<|im_start|>user\n" + ) + image_data = [] + for content in messages[0]["content"]: + if content["type"] == "image_url": + text += "\n" + image_data.append(content["image_url"]["url"]) + text += "Please describe the video briefly.<|im_end|>\n<|im_start|>assistant\n" + return text, image_data + + def generate(self, text, image_data): + response = requests.post( + self.base_url + "/generate", + json={ + "text": text, + "image_data": image_data, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + "modalities": ["multi-images"], + }, + ).json() + return response["text"] + + def generate_for_video(self, batch, num_frame) -> Union[str, list[str]]: + # prepare the video input about Steven introducing ipod nano + url = "https://raw.githubusercontent.com/evolvinglmms-lab/sglang/dev/onevision_local/assets/jobs.mp4" + cache_dir = os.path.expanduser("~/.cache") + file_path = os.path.join(cache_dir, "jobs.mp4") + os.makedirs(cache_dir, exist_ok=True) + if not os.path.exists(file_path): + response = requests.get(url) + response.raise_for_status() + with open(file_path, "wb") as f: + f.write(response.content) + + if not batch: + assert isinstance(num_frame, int) + messages = self.prepare_video_messages(file_path, max_frames_num=num_frame) + text, image_data = self.get_prompt_from_messages(messages) + return self.generate(text, image_data) + else: + assert isinstance(num_frame, list) + func_args = [] + for max_frames_num in num_frame: + messages = self.prepare_video_messages( + file_path, + max_frames_num=max_frames_num, + ) + text, image_data = self.get_prompt_from_messages(messages) + func_args.append((text, image_data)) + + with ThreadPoolExecutor(max_workers=10) as executor: + responses = list(executor.map(lambda p: self.generate(*p), func_args)) + + return responses + + def launch_server(self, chunked_prefill_size) -> int: + # launch server + model = "lmms-lab/llava-onevision-qwen2-7b-ov" + # model = "meta-llama/Llama-3.2-11B-Vision-Instruct" + self.base_url = DEFAULT_URL_FOR_TEST + process = popen_launch_server( + model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--chunked-prefill-size", + f"{chunked_prefill_size}", + ], + ) + return process.pid + + def _test_chunked_prefill(self, batches, num_frames): + # Chunked + chunked_server_pid = self.launch_server(chunked_prefill_size=1024) + try: + outputs_chunked = [] + for batch, num_frame in zip(batches, num_frames): + output_chunked = self.generate_for_video( + batch=batch, num_frame=num_frame + ) + outputs_chunked += [output_chunked] + finally: + kill_process_tree(chunked_server_pid) + + # None-chunked + try: + no_chunked_server_pid = self.launch_server(chunked_prefill_size=-1) + outputs_no_chunked = [] + for batch, num_frame in zip(batches, num_frames): + output_no_chunked = self.generate_for_video( + batch=batch, num_frame=num_frame + ) + outputs_no_chunked += [output_no_chunked] + + finally: + kill_process_tree(no_chunked_server_pid) + + for output_chunked, output_no_chunked in zip( + outputs_chunked, outputs_no_chunked + ): + print("output with chunked prefill:") + print(output_chunked) + print("output without chunked prefill:") + print(output_no_chunked) + self.assertEqual(len(output_chunked), len(output_no_chunked)) + rouge_scores = calculate_rouge_l(output_chunked, output_no_chunked) + avg_score = sum(rouge_scores) / len(rouge_scores) + print(f"ROUGE-L scores: {rouge_scores}") + print(f"Average ROUGE-L score: {avg_score:.4f}") + # Allow for occasional divergence in one item while maintaining overall output quality + self.assertGreater( + avg_score, + 0.90, + f"Average ROUGE-L score too low: {avg_score:.4f}. " + f"Individual scores: {rouge_scores}", + ) + + def test_chunked_prefill(self): + self._test_chunked_prefill(batches=[False, True], num_frames=[1, [2, 6, 8, 10]]) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_vision_openai_server_a.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_vision_openai_server_a.py new file mode 100644 index 00000000..5068dcda --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_vision_openai_server_a.py @@ -0,0 +1,226 @@ +""" +Usage: +python3 -m unittest test_vision_openai_server.TestOpenAIVisionServer.test_mixed_batch +python3 -m unittest test_vision_openai_server.TestOpenAIVisionServer.test_multi_images_chat_completion +""" + +import unittest + +import openai +from test_vision_openai_server_common import * +from test_vision_openai_server_common import ( # DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,; DEFAULT_URL_FOR_TEST,; IMAGE_MAN_IRONING_URL,; popen_launch_server, + AudioOpenAITestMixin, + CustomTestCase, + ImageOpenAITestMixin, + OmniOpenAITestMixin, + TestOpenAIMLLMServerBase, + VideoOpenAITestMixin, +) + + +class TestLlavaServer(ImageOpenAITestMixin): + model = "lmms-lab/llava-onevision-qwen2-0.5b-ov" + + +class TestQwen25VLServer(ImageOpenAITestMixin, VideoOpenAITestMixin): + model = "Qwen/Qwen2.5-VL-7B-Instruct" + extra_args = [ + "--cuda-graph-max-bs=4", + ] + + +class TestQwen3VLServer(ImageOpenAITestMixin, VideoOpenAITestMixin): + model = "Qwen/Qwen3-VL-30B-A3B-Instruct" + extra_args = ["--cuda-graph-max-bs=4"] + + +class TestQwen3OmniServer(OmniOpenAITestMixin): + model = "Qwen/Qwen3-Omni-30B-A3B-Instruct" + extra_args = [ # workaround to fit into H100 + "--mem-fraction-static=0.90", + "--disable-cuda-graph", + "--disable-fast-image-processor", + "--grammar-backend=none", + ] + + +class TestQwen2VLContextLengthServer(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "Qwen/Qwen2-VL-7B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--context-length", + "300", + "--cuda-graph-max-bs", + "4", + ], + ) + cls.base_url += "/v1" + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_single_image_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + with self.assertRaises(openai.BadRequestError) as cm: + client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + }, + { + "type": "text", + "text": "Give a lengthy description of this picture", + }, + ], + }, + ], + temperature=0, + ) + + # context length is checked first, then max_req_input_len, which is calculated from the former + assert ( + "Multimodal prompt is too long after expanding multimodal tokens." + in str(cm.exception) + or "is longer than the model's context length" in str(cm.exception) + ) + + +# flaky +# class TestMllamaServer(ImageOpenAITestMixin): +# model = "meta-llama/Llama-3.2-11B-Vision-Instruct" + + +class TestInternVL25Server(ImageOpenAITestMixin): + model = "OpenGVLab/InternVL2_5-2B" + extra_args = [ + "--cuda-graph-max-bs=4", + ] + + +class TestMiniCPMV4Server(ImageOpenAITestMixin): + model = "openbmb/MiniCPM-V-4" + extra_args = [ + "--cuda-graph-max-bs=4", + ] + + +class TestMiniCPMo26Server(ImageOpenAITestMixin, AudioOpenAITestMixin): + model = "openbmb/MiniCPM-o-2_6" + extra_args = [ + "--cuda-graph-max-bs=4", + ] + + +class TestGemma3itServer(ImageOpenAITestMixin): + model = "google/gemma-3-4b-it" + extra_args = [ + "--cuda-graph-max-bs=4", + ] + + +class TestKimiVLServer(ImageOpenAITestMixin): + model = "moonshotai/Kimi-VL-A3B-Instruct" + extra_args = [ + "--context-length=8192", + "--dtype=bfloat16", + ] + + def test_video_images_chat_completion(self): + # model context length exceeded + pass + + +@unittest.skip( + "Disabling this test to speed up CI. Prefer to test it within nightly test." +) +class TestGLM41VServer(ImageOpenAITestMixin, VideoOpenAITestMixin): + model = "zai-org/GLM-4.1V-9B-Thinking" + extra_args = [ + "--reasoning-parser=glm45", + ] + + +class TestQwen2AudioServer(AudioOpenAITestMixin): + model = "Qwen/Qwen2-Audio-7B-Instruct" + + +class TestDeepseekOCRServer(TestOpenAIMLLMServerBase): + model = "deepseek-ai/DeepSeek-OCR" + trust_remote_code = False + + def verify_single_image_response_for_ocr(self, response): + """Verify DeepSeek-OCR grounding output with coordinates""" + assert response.choices[0].message.role == "assistant" + text = response.choices[0].message.content + assert isinstance(text, str) + + # DeepSeek-OCR uses grounding format, outputs coordinates + assert "text" in text.lower(), f"OCR text: {text}, should contain 'text'" + + # Verify coordinate format [[x1, y1, x2, y2]] + import re + + coord_pattern = r"\[\[[\d\s,]+\]\]" + assert re.search( + coord_pattern, text + ), f"OCR text: {text}, should contain coordinate format [[x1, y1, x2, y2]]" + + # Verify basic response fields + assert response.id + assert response.created + assert response.usage.prompt_tokens > 0 + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def test_single_image_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + image_url = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/images/ocr-text.png" + + response = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": image_url}, + }, + { + "type": "text", + "text": "<|grounding|>Convert the document to markdown.", + }, + ], + }, + ], + temperature=0, + **(self.get_vision_request_kwargs()), + ) + + self.verify_single_image_response_for_ocr(response) + + +if __name__ == "__main__": + del ( + TestOpenAIMLLMServerBase, + ImageOpenAITestMixin, + VideoOpenAITestMixin, + AudioOpenAITestMixin, + OmniOpenAITestMixin, + ) + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_vision_openai_server_common.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_vision_openai_server_common.py new file mode 100644 index 00000000..f737a569 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_vision_openai_server_common.py @@ -0,0 +1,583 @@ +import io +import os +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +import openai +import pybase64 +import requests +from PIL import Image + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +# image +IMAGE_MAN_IRONING_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/images/man_ironing_on_back_of_suv.png" +IMAGE_SGL_LOGO_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/images/sgl_logo.png" + +# video +VIDEO_JOBS_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/videos/jobs_presenting_ipod.mp4" + +# audio +AUDIO_TRUMP_SPEECH_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/audios/Trump_WEF_2018_10s.mp3" +AUDIO_BIRD_SONG_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/audios/bird_song.mp3" + + +class TestOpenAIMLLMServerBase(CustomTestCase): + model: str + extra_args: list = [] + fixed_args: list = ["--trust-remote-code", "--enable-multimodal"] + trust_remote_code: bool = True + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=( + cls.extra_args + cls.fixed_args + ["--trust-remote-code"] + if cls.trust_remote_code + else [] + ), + ) + cls.base_url += "/v1" + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def get_vision_request_kwargs(self): + return self.get_request_kwargs() + + def get_request_kwargs(self): + return {} + + def get_or_download_file(self, url: str) -> str: + cache_dir = os.path.expanduser("~/.cache") + if url is None: + raise ValueError() + file_name = url.split("/")[-1] + file_path = os.path.join(cache_dir, file_name) + os.makedirs(cache_dir, exist_ok=True) + + if not os.path.exists(file_path): + response = requests.get(url) + response.raise_for_status() + + with open(file_path, "wb") as f: + f.write(response.content) + return file_path + + +class AudioOpenAITestMixin(TestOpenAIMLLMServerBase): + def verify_speech_recognition_response(self, text): + check_list = [ + "thank you", + "it's a privilege to be here", + "leader", + "science", + "art", + ] + for check_word in check_list: + assert ( + check_word in text.lower() + ), f"audio_response: ｜{text}｜ should contain ｜{check_word}｜" + + def prepare_audio_messages(self, prompt, audio_file_name): + messages = [ + { + "role": "user", + "content": [ + { + "type": "audio_url", + "audio_url": {"url": f"{audio_file_name}"}, + }, + { + "type": "text", + "text": prompt, + }, + ], + } + ] + + return messages + + def get_audio_request_kwargs(self): + return self.get_request_kwargs() + + def get_audio_response(self, url: str, prompt, category): + audio_file_path = self.get_or_download_file(url) + client = openai.Client(api_key="sk-123456", base_url=self.base_url) + + messages = self.prepare_audio_messages(prompt, audio_file_path) + + response = client.chat.completions.create( + model="default", + messages=messages, + temperature=0, + max_tokens=128, + stream=False, + **(self.get_audio_request_kwargs()), + ) + + audio_response = response.choices[0].message.content + + print("-" * 30) + print(f"audio {category} response:\n{audio_response}") + print("-" * 30) + + audio_response = audio_response.lower() + + self.assertIsNotNone(audio_response) + self.assertGreater(len(audio_response), 0) + + return audio_response.lower() + + def test_audio_speech_completion(self): + # a fragment of Trump's speech + audio_response = self.get_audio_response( + AUDIO_TRUMP_SPEECH_URL, + "Listen to this audio and write down the audio transcription in English.", + category="speech", + ) + self.verify_speech_recognition_response(audio_response) + + def test_audio_ambient_completion(self): + # bird song + audio_response = self.get_audio_response( + AUDIO_BIRD_SONG_URL, + "Please listen to the audio snippet carefully and transcribe the content in English.", + "ambient", + ) + assert "bird" in audio_response + + +class ImageOpenAITestMixin(TestOpenAIMLLMServerBase): + def run_decode_with_image(self, image_id): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + content = [] + if image_id == 0: + content.append( + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + } + ) + elif image_id == 1: + content.append( + { + "type": "image_url", + "image_url": {"url": IMAGE_SGL_LOGO_URL}, + } + ) + else: + pass + + content.append( + { + "type": "text", + "text": "Describe this image in a sentence.", + } + ) + + response = client.chat.completions.create( + model="default", + messages=[ + {"role": "user", "content": content}, + ], + temperature=0, + **(self.get_vision_request_kwargs()), + ) + + assert response.choices[0].message.role == "assistant" + text = response.choices[0].message.content + assert isinstance(text, str) + + def test_mixed_batch(self): + image_ids = [0, 1, 2] * 4 + with ThreadPoolExecutor(4) as executor: + list(executor.map(self.run_decode_with_image, image_ids)) + + def verify_single_image_response(self, response): + assert response.choices[0].message.role == "assistant" + text = response.choices[0].message.content + assert isinstance(text, str) + + # `driver` is for gemma-3-it + assert ( + "man" in text or "person" or "driver" in text + ), f"text: {text}, should contain man, person or driver" + assert ( + "cab" in text + or "taxi" in text + or "SUV" in text + or "vehicle" in text + or "car" in text + ), f"text: {text}, should contain cab, taxi, SUV, vehicle or car" + # MiniCPMO fails to recognize `iron`, but `hanging` + assert ( + "iron" in text or "hang" in text or "cloth" in text or "holding" in text + ), f"text: {text}, should contain iron, hang, cloth or holding" + assert response.id + assert response.created + assert response.usage.prompt_tokens > 0 + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def test_single_image_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + }, + { + "type": "text", + "text": "Describe this image in a sentence.", + }, + ], + }, + ], + temperature=0, + **(self.get_vision_request_kwargs()), + ) + + print("-" * 30) + print(f"Single image response:\n{response.choices[0].message.content}") + print("-" * 30) + + self.verify_single_image_response(response) + + def test_multi_turn_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + }, + { + "type": "text", + "text": "Describe this image in a sentence.", + }, + ], + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "There is a man at the back of a yellow cab ironing his clothes.", + } + ], + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "Repeat your previous answer."} + ], + }, + ], + temperature=0, + **(self.get_vision_request_kwargs()), + ) + + assert response.choices[0].message.role == "assistant" + text = response.choices[0].message.content + assert isinstance(text, str) + assert ( + "man" in text or "cab" in text + ), f"text: {text}, should contain man or cab" + assert response.id + assert response.created + assert response.usage.prompt_tokens > 0 + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def test_multi_images_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + "modalities": "multi-images", + }, + { + "type": "image_url", + "image_url": {"url": IMAGE_SGL_LOGO_URL}, + "modalities": "multi-images", + }, + { + "type": "text", + "text": "I have two very different images. Please describe them.", + }, + ], + }, + ], + temperature=0, + **(self.get_vision_request_kwargs()), + ) + + assert response.choices[0].message.role == "assistant" + text = response.choices[0].message.content + assert isinstance(text, str) + print("-" * 30) + print(f"Multi images response:\n{text}") + print("-" * 30) + assert ( + "man" in text + or "cab" in text + or "SUV" in text + or "taxi" in text + or "car" in text + ), f"text: {text}, should contain man, cab, SUV, taxi or car" + assert ( + "logo" in text or '"S"' in text or "SG" in text or "graphic" in text + ), f"text: {text}, should contain logo, S or SG or graphic" + assert response.id + assert response.created + assert response.usage.prompt_tokens > 0 + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def prepare_video_images_messages(self, video_path): + # the memory consumed by the Vision Attention varies a lot, e.g. blocked qkv vs full-sequence sdpa + # the size of the video embeds differs from the `modality` argument when preprocessed + + # We import decord here to avoid a strange Segmentation fault (core dumped) issue. + # The following import order will cause Segmentation fault. + # import decord + # from transformers import AutoTokenizer + from decord import VideoReader, cpu + + max_frames_num = 10 + vr = VideoReader(video_path, ctx=cpu(0)) + total_frame_num = len(vr) + uniform_sampled_frames = np.linspace( + 0, total_frame_num - 1, max_frames_num, dtype=int + ) + frame_idx = uniform_sampled_frames.tolist() + frames = vr.get_batch(frame_idx).asnumpy() + + base64_frames = [] + for frame in frames: + pil_img = Image.fromarray(frame) + buff = io.BytesIO() + pil_img.save(buff, format="JPEG") + base64_str = pybase64.b64encode(buff.getvalue()).decode("utf-8") + base64_frames.append(base64_str) + + messages = [{"role": "user", "content": []}] + frame_format = { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,{}"}, + "modalities": "image", + } + + for base64_frame in base64_frames: + frame_format["image_url"]["url"] = "data:image/jpeg;base64,{}".format( + base64_frame + ) + messages[0]["content"].append(frame_format.copy()) + + prompt = {"type": "text", "text": "Please describe the video in detail."} + messages[0]["content"].append(prompt) + + return messages + + def test_video_images_chat_completion(self): + url = VIDEO_JOBS_URL + file_path = self.get_or_download_file(url) + + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + messages = self.prepare_video_images_messages(file_path) + + response = client.chat.completions.create( + model="default", + messages=messages, + temperature=0, + max_tokens=1024, + stream=False, + ) + + video_response = response.choices[0].message.content + + print("-" * 30) + print(f"Video images response:\n{video_response}") + print("-" * 30) + + # Add assertions to validate the video response + assert ( + "iPod" in video_response + or "device" in video_response + or "microphone" in video_response + ), f""" + ====================== video_images response ===================== + {video_response} + =========================================================== + should contain 'iPod' or 'device' or 'microphone' + """ + assert ( + "man" in video_response + or "person" in video_response + or "individual" in video_response + or "speaker" in video_response + or "presenter" in video_response + or "Steve" in video_response + or "hand" in video_response + ), f""" + ====================== video_images response ===================== + {video_response} + =========================================================== + should contain 'man' or 'person' or 'individual' or 'speaker' or 'presenter' or 'Steve' or 'hand' + """ + assert ( + "present" in video_response + or "examine" in video_response + or "display" in video_response + or "hold" in video_response + ), f""" + ====================== video_images response ===================== + {video_response} + =========================================================== + should contain 'present' or 'examine' or 'display' or 'hold' + """ + self.assertIsNotNone(video_response) + self.assertGreater(len(video_response), 0) + + +class VideoOpenAITestMixin(TestOpenAIMLLMServerBase): + def prepare_video_messages(self, video_path): + messages = [ + { + "role": "user", + "content": [ + { + "type": "video_url", + "video_url": {"url": f"{video_path}"}, + }, + {"type": "text", "text": "Please describe the video in detail."}, + ], + }, + ] + return messages + + def test_video_chat_completion(self): + url = VIDEO_JOBS_URL + file_path = self.get_or_download_file(url) + + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + messages = self.prepare_video_messages(file_path) + + response = client.chat.completions.create( + model="default", + messages=messages, + temperature=0, + max_tokens=1024, + stream=False, + **(self.get_vision_request_kwargs()), + ) + + video_response = response.choices[0].message.content.lower() + + print("-" * 30) + print(f"Video response:\n{video_response}") + print("-" * 30) + + # Add assertions to validate the video response + assert ( + "ipod" in video_response + or "device" in video_response + or "microphone" in video_response + or "phone" in video_response + ), f"video_response: {video_response}, should contain 'iPod' or 'device'" + assert ( + "man" in video_response + or "person" in video_response + or "individual" in video_response + or "speaker" in video_response + or "presenter" in video_response + or "hand" in video_response + ), f"video_response: {video_response}, should either have 'man' in video_response, or 'person' in video_response, or 'individual' in video_response or 'speaker' in video_response or 'presenter' or 'hand' in video_response" + assert ( + "present" in video_response + or "examine" in video_response + or "display" in video_response + or "hold" in video_response + ), f"video_response: {video_response}, should contain 'present', 'examine', 'display', or 'hold'" + assert ( + "black" in video_response or "dark" in video_response + ), f"video_response: {video_response}, should contain 'black' or 'dark'" + self.assertIsNotNone(video_response) + self.assertGreater(len(video_response), 0) + + +class OmniOpenAITestMixin( + ImageOpenAITestMixin, VideoOpenAITestMixin, AudioOpenAITestMixin +): + def test_mixed_modality_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + messages = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + }, + { + "type": "audio_url", + "audio_url": {"url": AUDIO_TRUMP_SPEECH_URL}, + }, + { + "type": "text", + "text": "I have an image and audio, which are not related at all. Please: 1. Describe the image in a sentence, 2. Repeat the exact words from the audio I provided. Be exact", + }, + ], + }, + ] + response = client.chat.completions.create( + model="default", + messages=messages, + temperature=0, + max_tokens=128, + stream=False, + ) + + text = response.choices[0].message.content + + print("-" * 30) + print(f"Mixed modality response:\n{text}") + print("-" * 30) + + self.verify_single_image_response(response=response) + self.verify_speech_recognition_response(text=text) diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_vlm_input_format.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_vlm_input_format.py new file mode 100644 index 00000000..7105573d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_vlm_input_format.py @@ -0,0 +1,255 @@ +import json +import unittest +from typing import Optional + +import torch +from transformers import ( + AutoProcessor, + Gemma3ForConditionalGeneration, + Qwen2_5_VLForConditionalGeneration, +) + +from sglang import Engine +from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest +from sglang.srt.parser.conversation import generate_chat_conv +from sglang.test.test_utils import download_image_with_retry + +TEST_IMAGE_URL = "https://github.com/sgl-project/sglang/blob/main/examples/assets/example_image.png?raw=true" + + +class VLMInputTestBase: + model_path = None + chat_template = None + processor = None + visual = None # Should be a callable for precomputed embeddings + + @classmethod + def setUpClass(cls): + assert cls.model_path is not None, "Set model_path in subclass" + assert cls.chat_template is not None, "Set chat_template in subclass" + cls.image_url = TEST_IMAGE_URL + cls.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + cls.main_image = download_image_with_retry(cls.image_url) + cls.processor = AutoProcessor.from_pretrained( + cls.model_path, trust_remote_code=True, use_fast=True + ) + cls._init_visual() + + @classmethod + def _init_visual(cls): + """Override in subclass to set up cls.visual as a callable for precomputed embeddings.""" + raise NotImplementedError + + def setUp(self): + self.engine = Engine( + model_path=self.model_path, + chat_template=self.chat_template, + device=self.device.type, + mem_fraction_static=0.8, + enable_multimodal=True, + disable_cuda_graph=True, + trust_remote_code=True, + ) + + def tearDown(self): + self.engine.shutdown() + + def verify_response(self, output): + out_text = output["text"].lower() + assert "taxi" in out_text or "cab" in out_text or "car" in out_text, out_text + + def get_completion_request(self) -> ChatCompletionRequest: + json_structure = { + "model": self.model_path, + "messages": [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": self.image_url}}, + {"type": "text", "text": "What's in this picture?"}, + ], + } + ], + } + json_str = json.dumps(json_structure) + return ChatCompletionRequest.model_validate_json(json_str) + + def get_processor_output(self, req: Optional[ChatCompletionRequest] = None): + if req is None: + req = self.get_completion_request() + conv = generate_chat_conv(req, template_name=self.chat_template) + text = conv.get_prompt() + + # Process inputs using processor + inputs = self.processor( + text=[text], + images=[self.main_image], + return_tensors="pt", + ).to(self.device) + + return inputs + + async def test_understands_image(self): + req = self.get_completion_request() + conv = generate_chat_conv(req, template_name=self.chat_template) + text = conv.get_prompt() + output = await self.engine.async_generate( + prompt=text, + image_data=[self.main_image], + sampling_params=dict(temperature=0.0), + ) + self.verify_response(output) + + async def test_understands_precomputed_embeddings(self): + req = self.get_completion_request() + processor_output = self.get_processor_output(req=req) + with torch.inference_mode(): + precomputed_embeddings = self.__class__.visual(processor_output) + output = await self.engine.async_generate( + input_ids=processor_output["input_ids"][0].detach().cpu().tolist(), + image_data=[ + self._precomputed_image_data(processor_output, precomputed_embeddings) + ], + sampling_params=dict(temperature=0.0), + ) + self.verify_response(output) + + async def test_understands_pixel_values(self): + req = self.get_completion_request() + processor_output = self.get_processor_output(req=req) + output = await self.engine.async_generate( + input_ids=processor_output["input_ids"][0].detach().cpu().tolist(), + image_data=[self._pixel_values_image_data(processor_output)], + sampling_params=dict(temperature=0.0), + ) + self.verify_response(output) + + def _precomputed_image_data(self, processor_output, precomputed_embeddings): + """This should not be overridden.""" + return dict( + modality="IMAGE", + precomputed_embeddings=precomputed_embeddings, + ) + + def _pixel_values_image_data(self, processor_output): + """Override in subclass to pass the correct set of arguments.""" + raise NotImplementedError + + +class TestQwenVLUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTestCase): + model_path = "Qwen/Qwen2.5-VL-3B-Instruct" + chat_template = "qwen2-vl" + + @classmethod + def _init_visual(cls): + cls.visual_model = ( + Qwen2_5_VLForConditionalGeneration.from_pretrained( + cls.model_path, torch_dtype=torch.bfloat16 + ) + .eval() + .visual.to(cls.device) + ) + cls.visual = lambda processor_output: cls.visual_model( + processor_output["pixel_values"], processor_output["image_grid_thw"] + ) + + def _pixel_values_image_data(self, processor_output): + return dict( + modality="IMAGE", + image_grid_thw=processor_output["image_grid_thw"], + pixel_values=processor_output["pixel_values"], + ) + + +class TestGemmaUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTestCase): + model_path = "google/gemma-3-4b-it" + chat_template = "gemma-it" + + @classmethod + def _init_visual(cls): + model = Gemma3ForConditionalGeneration.from_pretrained( + cls.model_path, torch_dtype=torch.bfloat16 + ) + cls.vision_tower = model.vision_tower.eval().to(cls.device) + cls.mm_projector = model.multi_modal_projector.eval().to(cls.device) + cls.visual = lambda processor_output: cls.mm_projector( + cls.vision_tower( + pixel_values=processor_output["pixel_values"] + ).last_hidden_state + ) + + def _pixel_values_image_data(self, processor_output): + return dict( + modality="IMAGE", + pixel_values=processor_output["pixel_values"][0], + ) + + +# Temporarily skip Kimi-VL for CI test due to issue in transformers=4.57.0 +# class TestKimiVLImageUnderstandsImage( +# VLMInputTestBase, unittest.IsolatedAsyncioTestCase +# ): +# model_path = "moonshotai/Kimi-VL-A3B-Instruct" +# chat_template = "kimi-vl" + +# @classmethod +# def _init_visual(cls): +# model = AutoModel.from_pretrained(cls.model_path, trust_remote_code=True) +# cls.vision_tower = model.vision_tower.eval().to(cls.device) +# cls.mm_projector = model.multi_modal_projector.eval().to(cls.device) + +# cls.visual = lambda tokenizer_output: cls.mm_projector( +# cls.vision_tower( +# pixel_values=tokenizer_output["pixel_values"], +# grid_hws=tokenizer_output["image_grid_hws"], +# ) +# ) + +# def _pixel_values_image_data(self, processor_output): +# return dict( +# modality="IMAGE", +# pixel_values=processor_output["pixel_values"], +# image_grid_hws=processor_output["image_grid_hws"], +# ) + + +# not for CI: too large +# class TestLlama4ImageUnderstandsImage( +# VLMInputTestBase, unittest.IsolatedAsyncioTestCase +# ): +# model_path = "meta-llama/Llama-4-Scout-17B-16E-Instruct" +# chat_template = "llama_4_vision" + +# def setUp(self): +# self.engine = Engine( +# model_path=self.model_path, +# trust_remote_code=True, +# chat_template=self.chat_template, +# enable_multimodal=True, +# mem_fraction_static=0.8, +# tp_size=4, +# attention_backend="fa3", +# context_length=65536, +# ) + +# @classmethod +# def _init_visual(cls): +# model = AutoModel.from_pretrained(cls.model_path, trust_remote_code=True, torch_dtype="auto") +# cls.vision_tower = model.vision_model.eval().to(cls.device) +# cls.mm_projector = model.multi_modal_projector.eval().to(cls.device) + +# cls.visual = lambda tokenizer_output: cls.mm_projector( +# cls.vision_tower( +# pixel_values=tokenizer_output["pixel_values"], +# ).last_hidden_state.flatten(0, -2) +# ) + +# def _pixel_values_image_data(self, processor_output): +# return dict( +# modality="IMAGE", +# pixel_values=processor_output["pixel_values"], +# ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_wave_attention_kernels.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_wave_attention_kernels.py new file mode 100644 index 00000000..d4c2ff8e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/test_wave_attention_kernels.py @@ -0,0 +1,322 @@ +import random +import unittest + +import torch + +from sglang.srt.layers.attention.triton_ops.decode_attention import ( + decode_attention_fwd_grouped as triton_decode_attention_fwd_grouped, +) +from sglang.srt.layers.attention.triton_ops.extend_attention import ( + extend_attention_fwd, + redundant_attention, +) +from sglang.srt.layers.attention.triton_ops.prefill_attention import ( + context_attention_fwd, +) +from sglang.srt.layers.attention.wave_ops.decode_attention import ( + decode_attention_intermediate_arrays_shapes, + decode_attention_wave, +) +from sglang.srt.layers.attention.wave_ops.extend_attention import extend_attention_wave +from sglang.srt.layers.attention.wave_ops.prefill_attention import ( + prefill_attention_wave, +) + + +class TestWaveAttention(unittest.TestCase): + + def _set_all_seeds(self, seed): + """Set all random seeds for reproducibility.""" + random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + def setUp(self): + # Set seeds before each test method + self._set_all_seeds(42) + + def _test_extend_attention_once(self, B, N_CTX, H_Q, H_KV, D): + dtype = torch.float16 + extend_seq_len = 1024 + + b_seq_len_prefix = torch.full( + (B,), N_CTX // B, dtype=torch.int32, device="cuda" + ) + b_seq_len_extend = torch.full( + (B,), extend_seq_len, dtype=torch.int32, device="cuda" + ) + b_seq_len = b_seq_len_prefix + b_seq_len_extend + max_len_in_batch = torch.max(b_seq_len, 0)[0].item() + + b_req_idx = torch.arange(B, dtype=torch.int32, device="cuda") + b_start_loc = torch.zeros((B,), dtype=torch.int32, device="cuda") + b_start_loc[1:] = torch.cumsum(b_seq_len[:-1], 0) + b_start_loc_extend = torch.zeros((B,), dtype=torch.int32, device="cuda") + b_start_loc_extend[1:] = torch.cumsum(b_seq_len_extend[:-1], 0) + + kv_indptr = torch.zeros((B + 1,), dtype=torch.int32, device="cuda") + kv_indptr[1 : B + 1] = torch.cumsum(b_seq_len_prefix[:B], dim=0) + kv_indices = torch.zeros( + (b_seq_len_prefix.sum().item(),), dtype=torch.int32, device="cuda" + ) + + for i in range(B): + kv_indices[kv_indptr[i] : kv_indptr[i + 1]] = torch.arange( + b_start_loc[i], b_start_loc[i] + b_seq_len_prefix[i] + ) + + total_token_num = torch.sum(b_seq_len).item() + extend_token_num = torch.sum(b_seq_len_extend).item() + k_buffer = torch.empty( + (total_token_num, H_KV, D), dtype=dtype, device="cuda" + ).normal_(mean=0.1, std=0.2) + v_buffer = torch.empty( + (total_token_num, H_KV, D), dtype=dtype, device="cuda" + ).normal_(mean=0.1, std=0.2) + + k_extend = torch.empty((extend_token_num, H_KV, D), dtype=dtype, device="cuda") + v_extend = torch.empty((extend_token_num, H_KV, D), dtype=dtype, device="cuda") + q_extend = torch.empty((extend_token_num, H_Q, D), dtype=dtype, device="cuda") + for i in range(B): + extend_start_in_buffer = b_start_loc[i] + b_seq_len_prefix[i] + extend_end_in_buffer = b_start_loc[i] + b_seq_len[i] + extend_start = b_start_loc_extend[i] + extend_end = b_start_loc_extend[i] + b_seq_len_extend[i] + k_extend[extend_start:extend_end] = k_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + v_extend[extend_start:extend_end] = v_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + q_extend[extend_start:extend_end] = torch.empty( + (b_seq_len_extend[i], H_Q, D), dtype=dtype, device="cuda" + ).normal_(mean=0.1, std=0.2) + + o_extend = torch.empty((extend_token_num, H_Q, D), dtype=dtype, device="cuda") + o_extend_mask = torch.empty( + (extend_token_num, H_Q, D), dtype=dtype, device="cuda" + ) + o_redundant = torch.empty( + (extend_token_num, H_Q, D), dtype=dtype, device="cuda" + ) + + b_seq_len_extend = b_seq_len - b_seq_len_prefix + max_len_extend = torch.max(b_seq_len_extend, 0)[0].item() + qo_indptr = torch.zeros((B + 1,), dtype=torch.int32, device="cuda") + qo_indptr[1 : B + 1] = torch.cumsum(b_seq_len_extend[:B], dim=0) + + custom_mask = None + mask_indptr = None + + redundant_attention( + q_extend, + o_redundant, + k_buffer, + v_buffer, + b_req_idx, + b_start_loc, + b_seq_len, + b_seq_len_prefix, + max_len_in_batch, + ) + + is_causal = True + + o_extend = torch.empty((extend_token_num, H_Q, D), dtype=dtype, device="cuda") + extend_attention_fwd( + q_extend, + k_extend, + v_extend, + o_extend, + k_buffer, + v_buffer, + qo_indptr, + kv_indptr, + kv_indices, + custom_mask, + is_causal, + mask_indptr, + max_len_extend, + ) + + o_wave = torch.empty((extend_token_num, H_Q, D), dtype=dtype, device="cuda") + extend_attention_wave( + q_extend, + k_extend, + v_extend, + k_buffer, + v_buffer, + qo_indptr, + kv_indptr, + kv_indices, + custom_mask, + mask_indptr, + max_len_extend, + o_wave, + is_causal=is_causal, + ) + + self.assertTrue(torch.allclose(o_extend, o_redundant, rtol=1e-2)) + self.assertTrue(torch.allclose(o_wave, o_redundant, rtol=1e-2)) + + def test_extend_attention(self): + + # Define the varying parameter values + attention_values = [128] + + # Loop through the values and call the method + for value in attention_values: + self._test_extend_attention_once(32, 16384, 6, 1, value) + + def _test_grouped_decode_attention_once(self, B, S, H_Q, H_KV, D, D_V): + dtype = torch.float16 + seq_len = S # This represents the number of tokens already in the sequence + total_tokens = B * seq_len + sm_scale = 1.0 / (D**0.5) + max_kv_splits = 8 + num_kv_splits = torch.full((B,), 4, dtype=torch.int32, device="cuda") + + # q represents the new token being generated, one per batch + q = torch.randn(B, H_Q, D, dtype=dtype, device="cuda") + + # k_buffer and v_buffer represent all previous tokens + k_buffer = torch.randn(total_tokens, H_KV, D, dtype=dtype, device="cuda") + v_buffer = torch.randn(total_tokens, H_KV, D_V, dtype=dtype, device="cuda") + + # o will have the same shape as q + o_triton = torch.zeros(B, H_Q, D_V, dtype=dtype, device="cuda") + o = torch.zeros(B, H_Q, D_V, dtype=dtype, device="cuda") + + req_to_token = torch.arange(total_tokens, device="cuda", dtype=torch.int32) + b_req_idx = torch.zeros(B + 1, device="cuda", dtype=torch.int32) + b_seq_len = torch.full((B,), seq_len, device="cuda", dtype=torch.int32) + b_req_idx[1 : B + 1] = torch.cumsum(b_seq_len, dim=0) + + attn_logits = torch.empty( + (B, H_Q, max_kv_splits, D_V + 1), + dtype=torch.float32, + device="cuda", + ) + attn_lse = torch.empty( + (B, H_Q, max_kv_splits), + dtype=torch.float32, + device="cuda", + ) + + logit_cap = 0.0 + triton_decode_attention_fwd_grouped( + q, + k_buffer, + v_buffer, + o_triton, + b_req_idx, + req_to_token, + attn_logits, + attn_lse, + num_kv_splits, + max_kv_splits, + sm_scale, + logit_cap, + ) + + attn_logits_shape, attn_logits_max_shape = ( + decode_attention_intermediate_arrays_shapes(B, D_V, H_Q, max_kv_splits) + ) + + attn_logits = torch.empty( + attn_logits_shape, + dtype=torch.float32, + device="cuda", + ) + + attn_logits_max = torch.empty( + attn_logits_max_shape, + dtype=torch.float32, + device="cuda", + ) + + decode_attention_wave( + q, + k_buffer, + v_buffer, + o, + b_req_idx, + req_to_token, + attn_logits, + attn_logits_max, + num_kv_splits, + max_kv_splits, + sm_scale, + logit_cap, + ) + + cos_sim = torch.nn.functional.cosine_similarity( + o.flatten(), o_triton.flatten(), dim=0 + ) + print(cos_sim.item()) + self.assertTrue(cos_sim.item() > 0.99) + self.assertTrue(torch.allclose(o, o_triton, atol=3e-2)) + + def test_grouped_decode_attention(self): + seq_lens = [5, 100, 128, 500] + configs = [ + (2, 16, 16, 64, 64), + (2, 16, 1, 64, 64), + (2, 128, 1, 80, 80), + (32, 128, 2, 512, 512), + (2, 128, 2, 512, 512), + (2, 128, 1, 576, 512), + ] + + for S in seq_lens: + for B, H_Q, H_KV, D, D_V in configs: + self._test_grouped_decode_attention_once(B, S, H_Q, H_KV, D, D_V) + + def _test_context_attention_once(self, head_dim, is_causal): + # Set up a simple test case + dtype = torch.float16 + num_heads = 4 + kv_heads = 1 + seq_lens = [128, 256] + max_seq_len = max(seq_lens) + + # Create random input tensors + q = torch.randn(sum(seq_lens), num_heads, head_dim, dtype=dtype, device="cuda") + k = torch.randn(sum(seq_lens), kv_heads, head_dim, dtype=dtype, device="cuda") + v = torch.randn(sum(seq_lens), kv_heads, head_dim, dtype=dtype, device="cuda") + o_triton = torch.zeros( + sum(seq_lens), num_heads, head_dim, dtype=dtype, device="cuda" + ) + o = torch.zeros(sum(seq_lens), num_heads, head_dim, dtype=dtype, device="cuda") + + # Create b_start_loc and b_seq_len tensors + b_start_loc = torch.tensor([0, seq_lens[0]], device="cuda") + b_seq_len = torch.tensor(seq_lens, device="cuda") + + context_attention_fwd( + q, k, v, o_triton, b_start_loc, b_seq_len, max_seq_len, is_causal=is_causal + ) + prefill_attention_wave( + q, k, v, o, b_start_loc, b_seq_len, max_seq_len, is_causal=is_causal + ) + cos_sim = torch.nn.functional.cosine_similarity( + o.flatten(), o_triton.flatten(), dim=0 + ) + + print(cos_sim.item()) + self.assertTrue(torch.allclose(o, o_triton, atol=3e-2)) + self.assertTrue(cos_sim.item() > 1 - (1e-5)) + + def test_context_attention(self): + head_dim = [128, 96] + + for dim in head_dim: + for is_causal in [False]: + self._test_context_attention_once(dim, is_causal) + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/xpu/test_intel_xpu_backend.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/xpu/test_intel_xpu_backend.py new file mode 100644 index 00000000..701769e7 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/.vendor/sglang/test/srt/xpu/test_intel_xpu_backend.py @@ -0,0 +1,64 @@ +""" +Usage: +python3 -m unittest test_intel_xpu_backend.TestIntelXPUBackend.test_latency_qwen_model +""" + +import unittest +from functools import wraps + +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST_BASE, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN, + CustomTestCase, + is_in_ci, + run_bench_one_batch, +) + + +def intel_xpu_benchmark(extra_args=None, min_throughput=None): + def decorator(test_func): + @wraps(test_func) + def wrapper(self): + common_args = [ + "--disable-radix", + "--trust-remote-code", + "--mem-fraction-static", + "0.3", + "--batch-size", + "1", + "--device", + "xpu", + ] + full_args = common_args + (extra_args or []) + + model = test_func(self) + prefill_latency, decode_throughput, decode_latency = run_bench_one_batch( + model, full_args + ) + + print(f"{model=}") + print(f"{prefill_latency=}") + print(f"{decode_throughput=}") + print(f"{decode_latency=}") + + if is_in_ci() and min_throughput is not None: + self.assertGreater(decode_throughput, min_throughput) + + return wrapper + + return decorator + + +class TestIntelXPUBackend(CustomTestCase): + + @intel_xpu_benchmark(min_throughput=10) + def test_latency_qwen_model(self): + return DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN + + @intel_xpu_benchmark(["--attention-backend", "intel_xpu", "--page-size", "128"]) + def test_attention_backend(self): + return DEFAULT_SMALL_MODEL_NAME_FOR_TEST_BASE + + +if __name__ == "__main__": + unittest.main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/NOTES.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/NOTES.md new file mode 100644 index 00000000..f11ed8b6 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/NOTES.md @@ -0,0 +1,65 @@ +# Suite F v5 Bottleneck Diagnosis and Optimization Notes + +## Workload + +- Model: Qwen/Qwen2.5-0.5B-Instruct (BF16) +- Hardware: single NVIDIA A100-SXM4-80GB +- Scenario: AccelMark Suite F offline, client concurrency 4/16/64, 200 requests +- Primary metric: output-only throughput at concurrency 64 (tok/s) +- Model dims: hidden_size=896, intermediate_size=4864, num_attention_heads=14, + num_key_value_heads=2, num_hidden_layers=24, head_dim=64. + +## Bottleneck diagnosis + +1. **Small model on a large accelerator**: the 0.5B model is compute-light on + A100. A single decode step is too small to saturate the GPU, so the main + gains come from reducing scheduler/Python overhead and keeping the GPU + busy with larger effective batch sizes. + +2. **SGLang's default FlashInfer path is not fastest**: for this tiny model, + the Triton attention backend with CUDA-graph capture is faster because it + avoids backend dispatch overhead. + +3. **CUDA-graph padding hurts**: padding batches up to the maximum captured + shape adds wasted memory-bandwidth work. Disabling padding (`disable_cuda_graph_padding`) + was the single largest win in the v3/v4 sweep. + +4. **Scheduler/Python overhead is now the dominant residual cost**: with + `continuous_decode_steps=4` and CUDA graph capture, raw kernel time is + small; the remaining gap to hardware peak is mostly Python scheduler work, + batch metadata construction, and small-launch overheads in decode attention. + +5. **Decode attention metadata dominates per-step CPU work**: the Triton decode + backend rebuilds `kv_indptr`, `kv_indices`, `num_kv_splits`, and temporary + attention buffers on every step. For very short sequences (Suite F output + tokens are short), this metadata work is large relative to the GEMM-like + attention kernel itself. + +6. **Framework-level patches must stay legal**: all patches live under the + runner's `.vendor/sglang` directory and are loaded via `sys.path` insertion + in `runner.py`. No global site-packages are modified. + +## v5 optimization directions + +- **Decode attention kernel tuning**: reduce `BLOCK_N`, `num_warps`, and + `num_stages` for tiny batches; avoid over-parallelization on short KV + sequences; make `num_kv_splits` smaller for the short-context Suite F. + +- **KV-cache memory layout**: keep the default MHA layout but disable the + alternate-stream copy path during CUDA graph capture to remove stream + synchronization overhead. + +- **CUDA graph capture minimalization**: capture only the batch sizes that + actually occur in Suite F offline (1..64 with steps) rather than the + default 1..160 dense ladder, reducing capture time and memory. + +- **Scheduler fast path**: for the synchronous offline batch, skip dynamic + prefill/decode interleaving once all requests are in decode phase and use + larger continuous-decode steps. + +- **Explored but not chosen unless measured faster**: FlashInfer backend + variants, custom hand-written Triton fused MLP, multi-instance data-parallel. + +## Experiment log + +See the v5 autoresearch summary for the per-attempt results. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/README.md b/runners/nvidia_sglang_inference_opt_4fd1e59e/README.md new file mode 100644 index 00000000..1de267df --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/README.md @@ -0,0 +1,29 @@ +# SGLang Suite F inference fast path (v5) + +Validated runner ID: `nvidia_sglang_inference_opt_4fd1e59e` + +This runner subclasses `nvidia_sglang_inference_opt_c45ad27a` and keeps the +same synchronous batch inference fast path. It ships a runner-local copy of +SGLang 0.5.6 under `.vendor/sglang` so framework-level patches stay inside the +runner directory and never modify global site-packages. The runner bootstraps +the vendored package, forces `TMPDIR=/tmp` for reliable ZMQ IPC sockets, and +adds the runtime Python's `ninja` to `PATH` for JIT compilation. + +v5 extends the v4 search space with aggressive, workload-aware optimizations: + +- Framework-level source patches under `.vendor/sglang` (decode attention, + scheduler, CUDA graph capture, memory allocation). +- Custom Triton kernels or autotune overrides for the 0.5B decode path. +- Static / minimal CUDA graph shapes for the fixed Suite F offline batch. +- Single-instance scheduler tuning; multi-instance data-parallel is left as a + future candidate if single-instance saturation is reached. + +The previous A100 leaderboard best for Suite F was 43,718 tok/s. The v4 +runner family reached ~45.9K tok/s; v5 aims to push further through source-level +optimization while staying within AccelMark integrity rules. + +To install in an AccelMark checkout, copy this directory to +`runners/nvidia_sglang_inference_opt_4fd1e59e`, then copy the desired config to +`configs/runner_configs/runner_nvidia_sglang_inference_opt_4fd1e59e.yaml`. +The SGLang 0.5.6 runtime used for validation must be selected by the external +harness/runtime registry. diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/meta.json b/runners/nvidia_sglang_inference_opt_4fd1e59e/meta.json new file mode 100644 index 00000000..3a061d6d --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/meta.json @@ -0,0 +1,23 @@ +{ + "id": "nvidia_sglang_inference_opt_4fd1e59e", + "platform": "nvidia", + "name": "SGLang offline inference fast path v5", + "framework": "SGLang", + "submitted_by": "AccelMark-Agent", + "description": "Scenario-aware SGLang runner with batched prompt formatting, metadata token accounting, vendored SGLang 0.5.6, tunable offline scheduling, and v5 aggressive kernel/scheduler patches.", + "supersedes_chain": [ + "nvidia_sglang_inference_opt_c45ad27a" + ], + "notes": "Experimental runner; the benchmark harness fills the content-hashed id in its isolated workspace.", + "created": "2026-07-29", + "hardware_label": "NVIDIA A100-SXM4-80GB", + "suite_support": { + "A": "pending", + "B": "pending", + "C": "pending", + "D": "pending", + "E": "pending", + "F": "validated", + "G": "pending" + } +} diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/requirements.txt b/runners/nvidia_sglang_inference_opt_4fd1e59e/requirements.txt new file mode 100644 index 00000000..35dcd617 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/requirements.txt @@ -0,0 +1,38 @@ +# AccelMark — NVIDIA SGLang runner dependencies +# Tested combination: torch 2.9.1 + sglang 0.4.x + CUDA 12.1 +# +# SGLang install — use the official release wheel: +# pip install sglang[all] +# or for a pinned version: +# pip install "sglang[all]==0.5.6" +# +# See https://sgl-project.github.io/start/install.html for full options. + +# Core +torch==2.9.1 +torchvision==0.24.1 +torchaudio==2.9.1 + +# LLM inference +sglang[all]>=0.5.6 + +# Transformers (for tokenizer) +transformers==4.57.1 +tokenizers>=0.22.2 +huggingface-hub>=0.36.2 +accelerate>=1.13.0 + +# AccelMark dependencies +numpy>=2.2.6 +jsonschema>=4.26.0 +psutil>=7.2.2 +tqdm>=4.67.3 + +# NVIDIA monitoring +nvidia-ml-py>=13.595.45 + +# Async support +aiohttp>=3.13.5 + +# Config file parsing +pyyaml>=6.0 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/runner.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/runner.py new file mode 100644 index 00000000..a68273e1 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/runner.py @@ -0,0 +1,236 @@ +"""SGLang runner with an optimized offline inference fast path (v5). + +The reference runner is intentionally retained for model loading, streaming, +resource cleanup, precision handling, and CLI compatibility. This subclass +changes the part that Suite F actually measures: + +* apply the chat template to the complete request batch in one tokenizer call; +* batch-tokenize generated strings instead of calling ``encode`` 200 times, + and use SGLang's ``completion_tokens`` fast path only after a warmup batch + proves it is identical to the reference runner's decoded-text token count; +* select offline-only tokenizer/scheduler/speculative policies without + imposing their latency trade-offs on online and interactive scenarios. + +No benchmark inputs, output limits, accuracy logic, or timing boundaries are +changed. The fallback paths deliberately preserve reference-runner semantics +for older SGLang/tokenizer versions. +""" + +from __future__ import annotations + +import os +import sys +import time +from pathlib import Path +from typing import Any + +# ── Vendor-local SGLang bootstrap ───────────────────────────────────────────── +# This runner ships a patched copy of SGLang under .vendor/sglang/python so that +# framework-level patches stay inside the runner folder and never modify the +# global site-packages installation. Insert the vendored package before any +# other import (including the AccelMark repo root) so `import sglang` resolves +# to the local copy. +_RUNNER_DIR = Path(__file__).resolve().parent +_VENDOR_SGLANG = _RUNNER_DIR / ".vendor" / "sglang" / "python" +if _VENDOR_SGLANG.is_dir(): + sys.path.insert(0, str(_VENDOR_SGLANG)) + +# Force a POSIX tmpfs for IPC sockets and build caches. Some container +# runtimes set TMPDIR to a distributed path that does not support unix-domain +# sockets, which causes SGLang's ZMQ IPC bind to fail. +os.environ["TMPDIR"] = "/tmp" + +# Make the runtime Python's ninja visible to SGLang/FlashInfer JIT compilation. +_VENV_BIN = Path(sys.executable).resolve().parent +if (_VENV_BIN / "ninja").is_file(): + _ninja_dir = str(_VENV_BIN) + _existing_path = os.environ.get("PATH", "") + if _ninja_dir not in _existing_path.split(os.pathsep): + os.environ["PATH"] = _ninja_dir + os.pathsep + _existing_path + +# The runner is launched as a file, so Python initially exposes only this +# runner directory. Add the AccelMark workspace root before shared imports. +_REPO_ROOT = _RUNNER_DIR.parent.parent +sys.path.insert(0, str(_REPO_ROOT)) + +from loadgen.types import InferenceResult +from runners.benchmark_runner import InferenceRequest +from runners.nvidia_sglang_c43a8309.runner import SGLangRunner + + +class SGLangInferenceOptimizedRunner(SGLangRunner): + """Reference SGLang runner with a scenario-aware offline fast path.""" + + _BATCHED_SCENARIOS = frozenset({"accuracy", "offline", "speculative"}) + + def __init__(self) -> None: + super().__init__() + self._metadata_counts_verified: bool | None = None + + def load_model(self, model_path: str, parallelism: dict[str, Any]) -> None: + """Apply throughput policies only to synchronous batch scenarios.""" + config = dict(getattr(self, "_runner_config", {}) or {}) + engine_kwargs = dict(config.get("engine_kwargs") or {}) + scenario = getattr(self, "_current_scenario", None) + + if scenario in self._BATCHED_SCENARIOS: + if config.get("enable_tokenizer_batch_encode", False): + engine_kwargs.setdefault("enable_tokenizer_batch_encode", True) + + continuous_steps = int(config.get("continuous_decode_steps", 1)) + if continuous_steps < 1: + raise ValueError("continuous_decode_steps must be at least 1") + engine_kwargs.setdefault( + "num_continuous_decode_steps", continuous_steps + ) + + speculative_mode = config.get("speculative_mode") + if speculative_mode not in (None, "none", "ngram"): + raise ValueError( + "speculative_mode must be one of: none, ngram" + ) + if speculative_mode == "ngram": + window = int(config.get("ngram_window", 4)) + breadth = int(config.get("ngram_breadth", 1)) + if window < 2 or breadth < 1: + raise ValueError( + "ngram_window must be >= 2 and ngram_breadth must be >= 1" + ) + engine_kwargs.setdefault("speculative_algorithm", "NGRAM") + engine_kwargs.setdefault( + "speculative_ngram_min_match_window_size", 1 + ) + engine_kwargs.setdefault( + "speculative_ngram_max_match_window_size", window + ) + engine_kwargs.setdefault( + "speculative_ngram_min_bfs_breadth", 1 + ) + engine_kwargs.setdefault( + "speculative_ngram_max_bfs_breadth", breadth + ) + # SGLang's C++ NGRAM cache requires branch_length to be + # strictly greater than max_match_window_size. + engine_kwargs.setdefault( + "speculative_ngram_branch_length", window + 1 + ) + engine_kwargs.setdefault("speculative_num_draft_tokens", window) + + config["engine_kwargs"] = engine_kwargs + self._runner_config = config + super().load_model(model_path, parallelism) + + def _format_prompts_batch( + self, requests: list[InferenceRequest] + ) -> list[str]: + """Format all prompts at once when the tokenizer supports batching.""" + tokenizer = self.tokenizer + if tokenizer and getattr(tokenizer, "chat_template", None): + conversations = [ + [{"role": "user", "content": request.prompt}] + for request in requests + ] + try: + formatted = tokenizer.apply_chat_template( + conversations, + tokenize=False, + add_generation_prompt=True, + ) + if ( + isinstance(formatted, list) + and len(formatted) == len(requests) + and all(isinstance(prompt, str) for prompt in formatted) + ): + return formatted + except (TypeError, ValueError): + # Older tokenizers may accept only one conversation at a time. + pass + return [self.format_prompt(request.prompt) for request in requests] + + @staticmethod + def _metadata_token_counts(outputs: list[Any]) -> list[int] | None: + counts: list[int] = [] + for output in outputs: + if not isinstance(output, dict): + return None + meta_info = output.get("meta_info") + count = ( + meta_info.get("completion_tokens") + if isinstance(meta_info, dict) + else None + ) + if not isinstance(count, int) or isinstance(count, bool) or count < 0: + return None + counts.append(count) + return counts + + def _batch_encoded_token_counts(self, texts: list[str]) -> list[int]: + """Match ``tokenizer.encode`` semantics with one vectorized call.""" + try: + encoded = self.tokenizer(texts, add_special_tokens=True) + input_ids = encoded["input_ids"] + if len(input_ids) == len(texts): + return [len(token_ids) if text else 0 for token_ids, text in zip(input_ids, texts)] + except (KeyError, TypeError, ValueError): + pass + return [len(self.tokenizer.encode(text)) if text else 0 for text in texts] + + def _completion_token_counts( + self, outputs: list[Any], texts: list[str] + ) -> list[int]: + """Preserve reference counting while avoiding redundant serial work.""" + metadata_counts = self._metadata_token_counts(outputs) + if self._metadata_counts_verified is True and metadata_counts is not None: + return metadata_counts + + encoded_counts = self._batch_encoded_token_counts(texts) + if self._metadata_counts_verified is None: + self._metadata_counts_verified = metadata_counts == encoded_counts + mode = "metadata" if self._metadata_counts_verified else "batch-encode" + print(f" output token-count fast path = {mode}") + return metadata_counts if self._metadata_counts_verified else encoded_counts + + def inference_fn_offline( + self, requests: list[InferenceRequest] + ) -> list[InferenceResult]: + """Run one synchronous SGLang batch with low-overhead postprocessing.""" + if (getattr(self, "_runner_config", {}) or {}).get( + "use_reference_offline", False + ): + return super().inference_fn_offline(requests) + + formatted = self._format_prompts_batch(requests) + start = time.perf_counter() + outputs = self.engine.generate( + prompt=formatted, + sampling_params=self._sampling_params, + ) + elapsed_ms = (time.perf_counter() - start) * 1000 + + texts = [ + ( + output.get("text", "") + if isinstance(output, dict) + else str(output) + ) + for output in outputs + ] + token_counts = self._completion_token_counts(outputs, texts) + + results: list[InferenceResult] = [] + for request, text, output_tokens in zip(requests, texts, token_counts): + results.append( + InferenceResult( + first_token_time_ms=None, + total_time_ms=elapsed_ms, + output_tokens=output_tokens, + input_tokens=request.input_tokens or 0, + success=True, + output_text=text, + ) + ) + return results + + +if __name__ == "__main__": + SGLangInferenceOptimizedRunner().main() diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/runner_config.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/runner_config.yaml new file mode 100644 index 00000000..a386581c --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/runner_config.yaml @@ -0,0 +1,8 @@ +enable_tokenizer_batch_encode: true +continuous_decode_steps: 4 +engine_kwargs: + attention_backend: triton + disable_cuda_graph_padding: true + enable_piecewise_cuda_graph: true + piecewise_cuda_graph_max_tokens: 1024 + prefill_attention_backend: fa3 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/runner_config_conservative.yaml b/runners/nvidia_sglang_inference_opt_4fd1e59e/runner_config_conservative.yaml new file mode 100644 index 00000000..92e82843 --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/runner_config_conservative.yaml @@ -0,0 +1,2 @@ +enable_tokenizer_batch_encode: true +continuous_decode_steps: 4 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/test_runner.py b/runners/nvidia_sglang_inference_opt_4fd1e59e/test_runner.py new file mode 100644 index 00000000..b6f7ea3e --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/test_runner.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +from runners.benchmark_runner import InferenceRequest + +from runner import SGLangInferenceOptimizedRunner, SGLangRunner + + +class _Tokenizer: + chat_template = "present" + + def __init__(self): + self.encode_calls = 0 + self.batch_encode_calls = 0 + + def apply_chat_template(self, conversations, **_kwargs): + return [f"{conversation[0]['content']}" for conversation in conversations] + + def encode(self, text): + self.encode_calls += 1 + return text.split() + + def __call__(self, texts, *, add_special_tokens): + assert add_special_tokens is True + self.batch_encode_calls += 1 + return {"input_ids": [text.split() for text in texts]} + + +class _Engine: + def __init__(self, outputs): + self.outputs = outputs + self.prompts = None + + def generate(self, *, prompt, sampling_params): + self.prompts = prompt + assert sampling_params == {"temperature": 0.0} + return self.outputs + + +def _request(index: int, prompt: str) -> InferenceRequest: + return InferenceRequest( + prompt=prompt, + request_id=index, + input_tokens=index + 1, + ) + + +def test_offline_fast_path_batches_formatting_and_uses_metadata(): + runner = SGLangInferenceOptimizedRunner() + runner.tokenizer = _Tokenizer() + runner._sampling_params = {"temperature": 0.0} + runner.engine = _Engine( + [ + {"text": "alpha beta", "meta_info": {"completion_tokens": 2}}, + {"text": "gamma", "meta_info": {"completion_tokens": 1}}, + ] + ) + + results = runner.inference_fn_offline( + [_request(0, "first"), _request(1, "second")] + ) + + assert runner.engine.prompts == [ + "first", + "second", + ] + assert [result.output_tokens for result in results] == [2, 1] + assert [result.input_tokens for result in results] == [1, 2] + assert [result.output_text for result in results] == ["alpha beta", "gamma"] + assert runner.tokenizer.encode_calls == 0 + assert runner.tokenizer.batch_encode_calls == 1 + + runner.inference_fn_offline( + [_request(0, "first"), _request(1, "second")] + ) + assert runner.tokenizer.batch_encode_calls == 1 + + +def test_offline_fast_path_falls_back_to_encoding_old_outputs(): + runner = SGLangInferenceOptimizedRunner() + runner.tokenizer = _Tokenizer() + runner._sampling_params = {"temperature": 0.0} + runner.engine = _Engine([{"text": "one two three"}]) + + result = runner.inference_fn_offline([_request(0, "first")])[0] + + assert result.output_tokens == 3 + assert runner.tokenizer.encode_calls == 0 + assert runner.tokenizer.batch_encode_calls == 1 + + +def test_load_policy_rejects_invalid_continuous_steps(): + runner = SGLangInferenceOptimizedRunner() + runner._current_scenario = "offline" + runner._runner_config = {"continuous_decode_steps": 0} + + try: + runner.load_model("unused", {}) + except ValueError as error: + assert "continuous_decode_steps" in str(error) + else: + raise AssertionError("invalid continuous_decode_steps was accepted") + + +def test_ngram_branch_is_strictly_larger_than_window(monkeypatch): + captured = {} + + def capture_load_model(self, _model_path, _parallelism): + captured.update(self._runner_config["engine_kwargs"]) + + monkeypatch.setattr(SGLangRunner, "load_model", capture_load_model) + runner = SGLangInferenceOptimizedRunner() + runner._current_scenario = "offline" + runner._runner_config = { + "speculative_mode": "ngram", + "ngram_window": 4, + "ngram_breadth": 1, + } + + runner.load_model("unused", {}) + + assert captured["speculative_ngram_max_match_window_size"] == 4 + assert captured["speculative_ngram_branch_length"] == 5 diff --git a/runners/nvidia_sglang_inference_opt_4fd1e59e/triton_kernel_config.json b/runners/nvidia_sglang_inference_opt_4fd1e59e/triton_kernel_config.json new file mode 100644 index 00000000..17507b4a --- /dev/null +++ b/runners/nvidia_sglang_inference_opt_4fd1e59e/triton_kernel_config.json @@ -0,0 +1,5 @@ +{ + "decode_grouped_block_h": 32, + "decode_grouped_block_n": 128, + "decode_grouped_num_warps": 8 +} From b74be87c4b85d7992051df4311addee623c5a03c Mon Sep 17 00:00:00 2001 From: ChiangYuhsin Date: Fri, 31 Jul 2026 03:33:34 +0000 Subject: [PATCH 2/2] results: A100-SXM4-80GB Suite F with nvidia_sglang_inference_opt_4fd1e59e --- .../accuracy/accuracy.json | 8 + .../env_info.json | 64 ++++ .../interactive/result.json | 169 ++++++++++ .../offline/result.json | 228 +++++++++++++ .../online/result.json | 201 +++++++++++ .../result.json | 315 ++++++++++++++++++ 6 files changed, 985 insertions(+) create mode 100644 results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/accuracy/accuracy.json create mode 100644 results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/env_info.json create mode 100644 results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/interactive/result.json create mode 100644 results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/offline/result.json create mode 100644 results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/online/result.json create mode 100644 results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/result.json diff --git a/results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/accuracy/accuracy.json b/results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/accuracy/accuracy.json new file mode 100644 index 00000000..c35ff064 --- /dev/null +++ b/results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/accuracy/accuracy.json @@ -0,0 +1,8 @@ +{ + "subset_score": 0.41, + "baseline_delta": 0.03, + "valid": true, + "framework": "SGLang", + "precision": "BF16", + "notes": "Integrated accuracy check \u2014 used same SGLang instance as benchmark." +} \ No newline at end of file diff --git a/results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/env_info.json b/results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/env_info.json new file mode 100644 index 00000000..049d327a --- /dev/null +++ b/results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/env_info.json @@ -0,0 +1,64 @@ +{ + "collected_at": "2026-07-31T02:37:50.313608+00:00", + "accelerators": [ + { + "index": 5, + "name": "NVIDIA A100-SXM4-80GB", + "vendor": "NVIDIA", + "memory_gb": 80.0, + "driver_version": "570.133.20", + "firmware_version": null, + "compute_capability": "8.0", + "supports_bf16": true + }, + { + "index": 7, + "name": "NVIDIA A100-SXM4-80GB", + "vendor": "NVIDIA", + "memory_gb": 80.0, + "driver_version": "570.133.20", + "firmware_version": null, + "compute_capability": "8.0", + "supports_bf16": true + } + ], + "accelerator_platform": "nvidia", + "accelerator_topology": null, + "intra_node_interconnect": null, + "cpu": { + "model": "Intel(R) Xeon(R) Processor @ 2.90GHz", + "physical_cores": 96, + "logical_cores": 96, + "numa_nodes": 2 + }, + "system_memory_gb": 1000.0, + "pcie_generation": "PCIe Gen 4", + "cpu_accelerator_bandwidth_gbs": null, + "network_interfaces": [ + { + "name": "mlx5_0", + "type": "InfiniBand/RoCE", + "bandwidth_gbps": null + }, + { + "name": "mlx5_1", + "type": "InfiniBand/RoCE", + "bandwidth_gbps": null + }, + { + "name": "mlx5_2", + "type": "InfiniBand/RoCE", + "bandwidth_gbps": null + }, + { + "name": "mlx5_3", + "type": "InfiniBand/RoCE", + "bandwidth_gbps": null + } + ], + "os": "Ubuntu 22.04.5 LTS", + "python_version": "3.11.13", + "kernel_version": "5.10.134-013.9.kangaroo.al8.x86_64", + "runtime_version": "CUDA 12.8", + "pytorch_version": "2.9.1+cu128" +} \ No newline at end of file diff --git a/results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/interactive/result.json b/results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/interactive/result.json new file mode 100644 index 00000000..bbafd4ba --- /dev/null +++ b/results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/interactive/result.json @@ -0,0 +1,169 @@ +{ + "schema_version": "1.0", + "suite_id": "suite_F", + "implementation_id": "nvidia_sglang_inference_opt_4fd1e59e", + "chip": { + "name": "NVIDIA A100-SXM4-80GB", + "vendor": "NVIDIA", + "count": 1, + "memory_gb": 80.0, + "interconnect_intra_node": null, + "interconnect_inter_node": null + }, + "environment": { + "collected_at": "2026-07-31T02:37:50.313608+00:00", + "accelerators": [ + { + "index": 5, + "name": "NVIDIA A100-SXM4-80GB", + "vendor": "NVIDIA", + "memory_gb": 80.0, + "driver_version": "570.133.20", + "firmware_version": null, + "compute_capability": "8.0", + "supports_bf16": true + }, + { + "index": 7, + "name": "NVIDIA A100-SXM4-80GB", + "vendor": "NVIDIA", + "memory_gb": 80.0, + "driver_version": "570.133.20", + "firmware_version": null, + "compute_capability": "8.0", + "supports_bf16": true + } + ], + "accelerator_platform": "nvidia", + "accelerator_topology": null, + "intra_node_interconnect": null, + "cpu": { + "model": "Intel(R) Xeon(R) Processor @ 2.90GHz", + "physical_cores": 96, + "logical_cores": 96, + "numa_nodes": 2 + }, + "system_memory_gb": 1000.0, + "pcie_generation": "PCIe Gen 4", + "cpu_accelerator_bandwidth_gbs": null, + "network_interfaces": [ + { + "name": "mlx5_0", + "type": "InfiniBand/RoCE", + "bandwidth_gbps": null + }, + { + "name": "mlx5_1", + "type": "InfiniBand/RoCE", + "bandwidth_gbps": null + }, + { + "name": "mlx5_2", + "type": "InfiniBand/RoCE", + "bandwidth_gbps": null + }, + { + "name": "mlx5_3", + "type": "InfiniBand/RoCE", + "bandwidth_gbps": null + } + ], + "os": "Ubuntu 22.04.5 LTS", + "python_version": "3.11.13", + "kernel_version": "5.10.134-013.9.kangaroo.al8.x86_64", + "runtime_version": "CUDA 12.8", + "pytorch_version": "2.9.1+cu128" + }, + "software": { + "framework": "SGLang", + "framework_version": "0.5.6", + "driver_version": "570.133.20", + "runtime_version": "CUDA 12.8", + "os": "Ubuntu 22.04.5 LTS", + "python_version": "3.11.13" + }, + "model": { + "model_id": "Qwen/Qwen2.5-0.5B-Instruct", + "model_revision": "7ae557604adf67be50417f59c2c2f167def9a775", + "model_name": null, + "model_note": "Local ModelScope snapshot for Suite F", + "model_source": "local", + "architecture": "dense", + "parameter_count_b": 0.5, + "precision": "BF16", + "effective_dtype": "bfloat16", + "quantization_method": null, + "model_format": "HuggingFace original" + }, + "task": { + "scenario": "interactive", + "num_runs": 3, + "warmup_runs": 1, + "parallelism": { + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "expert_parallel_size": 1, + "data_parallel_size": 1 + }, + "extra_config": { + "enable_tokenizer_batch_encode": true, + "continuous_decode_steps": 4, + "engine_kwargs": { + "attention_backend": "triton", + "disable_cuda_graph_padding": true, + "enable_piecewise_cuda_graph": true, + "piecewise_cuda_graph_max_tokens": 1024, + "prefill_attention_backend": "fa3" + } + }, + "runtime_metrics": null + }, + "metrics": { + "interactive": { + "ttft_ms_p50": 7.19, + "ttft_ms_p90": 7.54, + "ttft_ms_p99": 15.06, + "tpot_ms_p50": 1.62, + "tpot_ms_p90": 1.64, + "tpot_ms_p99": 1.71, + "peak_memory_gb": null, + "elapsed_seconds_median": 46.2, + "ttft_ms_p99_reliability": { + "n": 3, + "mean": 23.42, + "std": 22.39, + "cv_pct": 95.62, + "stability": "high-variance", + "runs": [ + 49.2, + 12.23, + 8.83 + ] + } + } + }, + "accuracy": { + "subset_score": null, + "baseline_delta": null, + "valid": false, + "notes": "Run --scenario accuracy to check model accuracy." + }, + "meta": { + "submitted_by": "ChiangYuhsin", + "submission_type": "individual", + "date": "2026-07-31", + "time": "02:46:54", + "run_id": "1189951b", + "run_name": "nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b", + "flagged": null, + "reproduce_script": "runners/nvidia_sglang_inference_opt_4fd1e59e/runner.py", + "env_info_file": "../env_info.json", + "log_file": "run.log", + "samples_file": "samples.jsonl", + "notes": null, + "benchmark_start_time": "2026-07-31T02:44:33.515731+00:00", + "benchmark_end_time": "2026-07-31T02:46:54.781708+00:00", + "benchmark_elapsed_minutes": 2.4, + "model_load_seconds": 47.7 + } +} \ No newline at end of file diff --git a/results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/offline/result.json b/results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/offline/result.json new file mode 100644 index 00000000..277d1dfd --- /dev/null +++ b/results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/offline/result.json @@ -0,0 +1,228 @@ +{ + "schema_version": "1.0", + "suite_id": "suite_F", + "implementation_id": "nvidia_sglang_inference_opt_4fd1e59e", + "chip": { + "name": "NVIDIA A100-SXM4-80GB", + "vendor": "NVIDIA", + "count": 1, + "memory_gb": 80.0, + "interconnect_intra_node": null, + "interconnect_inter_node": null + }, + "environment": { + "collected_at": "2026-07-31T02:37:50.313608+00:00", + "accelerators": [ + { + "index": 5, + "name": "NVIDIA A100-SXM4-80GB", + "vendor": "NVIDIA", + "memory_gb": 80.0, + "driver_version": "570.133.20", + "firmware_version": null, + "compute_capability": "8.0", + "supports_bf16": true + }, + { + "index": 7, + "name": "NVIDIA A100-SXM4-80GB", + "vendor": "NVIDIA", + "memory_gb": 80.0, + "driver_version": "570.133.20", + "firmware_version": null, + "compute_capability": "8.0", + "supports_bf16": true + } + ], + "accelerator_platform": "nvidia", + "accelerator_topology": null, + "intra_node_interconnect": null, + "cpu": { + "model": "Intel(R) Xeon(R) Processor @ 2.90GHz", + "physical_cores": 96, + "logical_cores": 96, + "numa_nodes": 2 + }, + "system_memory_gb": 1000.0, + "pcie_generation": "PCIe Gen 4", + "cpu_accelerator_bandwidth_gbs": null, + "network_interfaces": [ + { + "name": "mlx5_0", + "type": "InfiniBand/RoCE", + "bandwidth_gbps": null + }, + { + "name": "mlx5_1", + "type": "InfiniBand/RoCE", + "bandwidth_gbps": null + }, + { + "name": "mlx5_2", + "type": "InfiniBand/RoCE", + "bandwidth_gbps": null + }, + { + "name": "mlx5_3", + "type": "InfiniBand/RoCE", + "bandwidth_gbps": null + } + ], + "os": "Ubuntu 22.04.5 LTS", + "python_version": "3.11.13", + "kernel_version": "5.10.134-013.9.kangaroo.al8.x86_64", + "runtime_version": "CUDA 12.8", + "pytorch_version": "2.9.1+cu128" + }, + "software": { + "framework": "SGLang", + "framework_version": "0.5.6", + "driver_version": "570.133.20", + "runtime_version": "CUDA 12.8", + "os": "Ubuntu 22.04.5 LTS", + "python_version": "3.11.13" + }, + "model": { + "model_id": "Qwen/Qwen2.5-0.5B-Instruct", + "model_revision": "7ae557604adf67be50417f59c2c2f167def9a775", + "model_name": null, + "model_note": "Local ModelScope snapshot for Suite F", + "model_source": "local", + "architecture": "dense", + "parameter_count_b": 0.5, + "precision": "BF16", + "effective_dtype": "bfloat16", + "quantization_method": null, + "model_format": "HuggingFace original" + }, + "task": { + "scenario": "offline", + "num_runs": 3, + "warmup_runs": 1, + "parallelism": { + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "expert_parallel_size": 1, + "data_parallel_size": 1 + }, + "extra_config": { + "enable_tokenizer_batch_encode": true, + "continuous_decode_steps": 4, + "engine_kwargs": { + "attention_backend": "triton", + "disable_cuda_graph_padding": true, + "enable_piecewise_cuda_graph": true, + "piecewise_cuda_graph_max_tokens": 1024, + "prefill_attention_backend": "fa3", + "enable_tokenizer_batch_encode": true, + "num_continuous_decode_steps": 4 + } + }, + "runtime_metrics": null + }, + "metrics": { + "offline": { + "results_by_concurrency": [ + { + "client_concurrency": 4, + "throughput_tokens_per_sec": 46128.3, + "throughput_tokens_per_sec_per_chip": 46128.3, + "throughput_tokens_per_sec_total": 62323.45, + "elapsed_seconds_median": 0.9, + "peak_memory_gb": null, + "power_watts_avg": null, + "power_watts_peak": null, + "oom": false, + "throughput_tokens_per_sec_reliability": { + "n": 3, + "mean": 46035.52, + "std": 177.27, + "cv_pct": 0.39, + "stability": "stable", + "runs": [ + 45831.12, + 46147.16, + 46128.3 + ] + }, + "_throughput_note": "output_only", + "_concurrency_note": "client_concurrency is the number of requests sent simultaneously. The inference engine batches internally; this does not directly set engine parameters like max_num_seqs." + }, + { + "client_concurrency": 16, + "throughput_tokens_per_sec": 46406.6, + "throughput_tokens_per_sec_per_chip": 46406.6, + "throughput_tokens_per_sec_total": 62699.46, + "elapsed_seconds_median": 0.9, + "peak_memory_gb": null, + "power_watts_avg": null, + "power_watts_peak": null, + "oom": false, + "throughput_tokens_per_sec_reliability": { + "n": 3, + "mean": 46306.02, + "std": 204.98, + "cv_pct": 0.44, + "stability": "stable", + "runs": [ + 46070.18, + 46406.6, + 46441.29 + ] + }, + "_throughput_note": "output_only", + "_concurrency_note": "client_concurrency is the number of requests sent simultaneously. The inference engine batches internally; this does not directly set engine parameters like max_num_seqs." + }, + { + "client_concurrency": 64, + "throughput_tokens_per_sec": 46500.82, + "throughput_tokens_per_sec_per_chip": 46500.82, + "throughput_tokens_per_sec_total": 62826.76, + "elapsed_seconds_median": 0.9, + "peak_memory_gb": null, + "power_watts_avg": null, + "power_watts_peak": null, + "oom": false, + "throughput_tokens_per_sec_reliability": { + "n": 3, + "mean": 46028.86, + "std": 890.06, + "cv_pct": 1.93, + "stability": "stable", + "runs": [ + 45002.22, + 46500.82, + 46583.55 + ] + }, + "_throughput_note": "output_only", + "_concurrency_note": "client_concurrency is the number of requests sent simultaneously. The inference engine batches internally; this does not directly set engine parameters like max_num_seqs." + } + ] + } + }, + "accuracy": { + "subset_score": null, + "baseline_delta": null, + "valid": false, + "notes": "Run --scenario accuracy to check model accuracy." + }, + "meta": { + "submitted_by": "ChiangYuhsin", + "submission_type": "individual", + "date": "2026-07-31", + "time": "02:40:05", + "run_id": "1189951b", + "run_name": "nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b", + "flagged": null, + "reproduce_script": "runners/nvidia_sglang_inference_opt_4fd1e59e/runner.py", + "env_info_file": "../env_info.json", + "log_file": "run.log", + "samples_file": "samples.jsonl", + "notes": null, + "benchmark_start_time": "2026-07-31T02:39:52.638757+00:00", + "benchmark_end_time": "2026-07-31T02:40:05.992733+00:00", + "benchmark_elapsed_minutes": 0.2, + "model_load_seconds": 47.1 + } +} \ No newline at end of file diff --git a/results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/online/result.json b/results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/online/result.json new file mode 100644 index 00000000..9b503059 --- /dev/null +++ b/results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/online/result.json @@ -0,0 +1,201 @@ +{ + "schema_version": "1.0", + "suite_id": "suite_F", + "implementation_id": "nvidia_sglang_inference_opt_4fd1e59e", + "chip": { + "name": "NVIDIA A100-SXM4-80GB", + "vendor": "NVIDIA", + "count": 1, + "memory_gb": 80.0, + "interconnect_intra_node": null, + "interconnect_inter_node": null + }, + "environment": { + "collected_at": "2026-07-31T02:37:50.313608+00:00", + "accelerators": [ + { + "index": 5, + "name": "NVIDIA A100-SXM4-80GB", + "vendor": "NVIDIA", + "memory_gb": 80.0, + "driver_version": "570.133.20", + "firmware_version": null, + "compute_capability": "8.0", + "supports_bf16": true + }, + { + "index": 7, + "name": "NVIDIA A100-SXM4-80GB", + "vendor": "NVIDIA", + "memory_gb": 80.0, + "driver_version": "570.133.20", + "firmware_version": null, + "compute_capability": "8.0", + "supports_bf16": true + } + ], + "accelerator_platform": "nvidia", + "accelerator_topology": null, + "intra_node_interconnect": null, + "cpu": { + "model": "Intel(R) Xeon(R) Processor @ 2.90GHz", + "physical_cores": 96, + "logical_cores": 96, + "numa_nodes": 2 + }, + "system_memory_gb": 1000.0, + "pcie_generation": "PCIe Gen 4", + "cpu_accelerator_bandwidth_gbs": null, + "network_interfaces": [ + { + "name": "mlx5_0", + "type": "InfiniBand/RoCE", + "bandwidth_gbps": null + }, + { + "name": "mlx5_1", + "type": "InfiniBand/RoCE", + "bandwidth_gbps": null + }, + { + "name": "mlx5_2", + "type": "InfiniBand/RoCE", + "bandwidth_gbps": null + }, + { + "name": "mlx5_3", + "type": "InfiniBand/RoCE", + "bandwidth_gbps": null + } + ], + "os": "Ubuntu 22.04.5 LTS", + "python_version": "3.11.13", + "kernel_version": "5.10.134-013.9.kangaroo.al8.x86_64", + "runtime_version": "CUDA 12.8", + "pytorch_version": "2.9.1+cu128" + }, + "software": { + "framework": "SGLang", + "framework_version": "0.5.6", + "driver_version": "570.133.20", + "runtime_version": "CUDA 12.8", + "os": "Ubuntu 22.04.5 LTS", + "python_version": "3.11.13" + }, + "model": { + "model_id": "Qwen/Qwen2.5-0.5B-Instruct", + "model_revision": "7ae557604adf67be50417f59c2c2f167def9a775", + "model_name": null, + "model_note": "Local ModelScope snapshot for Suite F", + "model_source": "local", + "architecture": "dense", + "parameter_count_b": 0.5, + "precision": "BF16", + "effective_dtype": "bfloat16", + "quantization_method": null, + "model_format": "HuggingFace original" + }, + "task": { + "scenario": "online", + "num_runs": 3, + "warmup_runs": 1, + "parallelism": { + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "expert_parallel_size": 1, + "data_parallel_size": 1 + }, + "extra_config": { + "enable_tokenizer_batch_encode": true, + "continuous_decode_steps": 4, + "engine_kwargs": { + "attention_backend": "triton", + "disable_cuda_graph_padding": true, + "enable_piecewise_cuda_graph": true, + "piecewise_cuda_graph_max_tokens": 1024, + "prefill_attention_backend": "fa3" + } + }, + "runtime_metrics": null + }, + "metrics": { + "online": { + "sla_ttft_ms": 500, + "max_valid_qps": 40, + "results_by_qps": [ + { + "target_qps": 10, + "achieved_qps": 10.0, + "ttft_ms_p50": 8.39, + "ttft_ms_p90": 9.37, + "ttft_ms_p99": 14.12, + "tpot_ms_p50": 1.94, + "tpot_ms_p90": 2.05, + "tpot_ms_p99": 2.28, + "elapsed_seconds_median": 31.8, + "sla_met": true, + "ttft_ms_p99_reliability": { + "n": 3, + "mean": 17.9, + "std": 7.1, + "cv_pct": 39.67, + "stability": "high-variance", + "runs": [ + 25.95, + 15.2, + 12.54 + ] + } + }, + { + "target_qps": 40, + "achieved_qps": 40.0, + "ttft_ms_p50": 8.38, + "ttft_ms_p90": 11.9, + "ttft_ms_p99": 14.53, + "tpot_ms_p50": 2.4, + "tpot_ms_p90": 2.64, + "tpot_ms_p99": 3.18, + "elapsed_seconds_median": 7.8, + "sla_met": true, + "ttft_ms_p99_reliability": { + "n": 3, + "mean": 14.39, + "std": 0.16, + "cv_pct": 1.12, + "stability": "stable", + "runs": [ + 14.39, + 14.23, + 14.56 + ] + } + } + ] + } + }, + "accuracy": { + "subset_score": null, + "baseline_delta": null, + "valid": false, + "notes": "Run --scenario accuracy to check model accuracy." + }, + "meta": { + "submitted_by": "ChiangYuhsin", + "submission_type": "individual", + "date": "2026-07-31", + "time": "02:43:24", + "run_id": "1189951b", + "run_name": "nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b", + "flagged": null, + "reproduce_script": "runners/nvidia_sglang_inference_opt_4fd1e59e/runner.py", + "env_info_file": "../env_info.json", + "log_file": "run.log", + "samples_file": "samples.jsonl", + "notes": null, + "benchmark_start_time": "2026-07-31T02:41:19.824971+00:00", + "benchmark_end_time": "2026-07-31T02:43:24.313106+00:00", + "benchmark_elapsed_minutes": 2.1, + "model_load_seconds": 52.5 + } +} \ No newline at end of file diff --git a/results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/result.json b/results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/result.json new file mode 100644 index 00000000..b4d1df95 --- /dev/null +++ b/results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/result.json @@ -0,0 +1,315 @@ +{ + "schema_version": "1.0", + "suite_id": "suite_F", + "implementation_id": "nvidia_sglang_inference_opt_4fd1e59e", + "chip": { + "name": "NVIDIA A100-SXM4-80GB", + "vendor": "NVIDIA", + "count": 1, + "memory_gb": 80.0, + "interconnect_intra_node": null, + "interconnect_inter_node": null + }, + "environment": { + "collected_at": "2026-07-31T02:37:50.313608+00:00", + "accelerators": [ + { + "index": 5, + "name": "NVIDIA A100-SXM4-80GB", + "vendor": "NVIDIA", + "memory_gb": 80.0, + "driver_version": "570.133.20", + "firmware_version": null, + "compute_capability": "8.0", + "supports_bf16": true + }, + { + "index": 7, + "name": "NVIDIA A100-SXM4-80GB", + "vendor": "NVIDIA", + "memory_gb": 80.0, + "driver_version": "570.133.20", + "firmware_version": null, + "compute_capability": "8.0", + "supports_bf16": true + } + ], + "accelerator_platform": "nvidia", + "accelerator_topology": null, + "intra_node_interconnect": null, + "cpu": { + "model": "Intel(R) Xeon(R) Processor @ 2.90GHz", + "physical_cores": 96, + "logical_cores": 96, + "numa_nodes": 2 + }, + "system_memory_gb": 1000.0, + "pcie_generation": "PCIe Gen 4", + "cpu_accelerator_bandwidth_gbs": null, + "network_interfaces": [ + { + "name": "mlx5_0", + "type": "InfiniBand/RoCE", + "bandwidth_gbps": null + }, + { + "name": "mlx5_1", + "type": "InfiniBand/RoCE", + "bandwidth_gbps": null + }, + { + "name": "mlx5_2", + "type": "InfiniBand/RoCE", + "bandwidth_gbps": null + }, + { + "name": "mlx5_3", + "type": "InfiniBand/RoCE", + "bandwidth_gbps": null + } + ], + "os": "Ubuntu 22.04.5 LTS", + "python_version": "3.11.13", + "kernel_version": "5.10.134-013.9.kangaroo.al8.x86_64", + "runtime_version": "CUDA 12.8", + "pytorch_version": "2.9.1+cu128" + }, + "software": { + "framework": "SGLang", + "framework_version": "0.5.6", + "driver_version": "570.133.20", + "runtime_version": "CUDA 12.8", + "os": "Ubuntu 22.04.5 LTS", + "python_version": "3.11.13" + }, + "model": { + "model_id": "Qwen/Qwen2.5-0.5B-Instruct", + "model_revision": "7ae557604adf67be50417f59c2c2f167def9a775", + "model_name": null, + "model_note": "Local ModelScope snapshot for Suite F", + "model_source": "local", + "architecture": "dense", + "parameter_count_b": 0.5, + "precision": "BF16", + "effective_dtype": "bfloat16", + "quantization_method": null, + "model_format": "HuggingFace original" + }, + "task": { + "scenarios_run": [ + "offline", + "online", + "interactive" + ], + "parallelism": { + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "expert_parallel_size": 1, + "data_parallel_size": 1 + }, + "num_runs": 3, + "extra_config": { + "enable_tokenizer_batch_encode": true, + "continuous_decode_steps": 4, + "engine_kwargs": { + "attention_backend": "triton", + "disable_cuda_graph_padding": true, + "enable_piecewise_cuda_graph": true, + "piecewise_cuda_graph_max_tokens": 1024, + "prefill_attention_backend": "fa3", + "enable_tokenizer_batch_encode": true, + "num_continuous_decode_steps": 4 + } + } + }, + "metrics": { + "derived": {}, + "offline": { + "results_by_concurrency": [ + { + "client_concurrency": 4, + "throughput_tokens_per_sec": 46128.3, + "throughput_tokens_per_sec_per_chip": 46128.3, + "throughput_tokens_per_sec_total": 62323.45, + "elapsed_seconds_median": 0.9, + "peak_memory_gb": null, + "power_watts_avg": null, + "power_watts_peak": null, + "oom": false, + "throughput_tokens_per_sec_reliability": { + "n": 3, + "mean": 46035.52, + "std": 177.27, + "cv_pct": 0.39, + "stability": "stable", + "runs": [ + 45831.12, + 46147.16, + 46128.3 + ] + }, + "_throughput_note": "output_only", + "_concurrency_note": "client_concurrency is the number of requests sent simultaneously. The inference engine batches internally; this does not directly set engine parameters like max_num_seqs." + }, + { + "client_concurrency": 16, + "throughput_tokens_per_sec": 46406.6, + "throughput_tokens_per_sec_per_chip": 46406.6, + "throughput_tokens_per_sec_total": 62699.46, + "elapsed_seconds_median": 0.9, + "peak_memory_gb": null, + "power_watts_avg": null, + "power_watts_peak": null, + "oom": false, + "throughput_tokens_per_sec_reliability": { + "n": 3, + "mean": 46306.02, + "std": 204.98, + "cv_pct": 0.44, + "stability": "stable", + "runs": [ + 46070.18, + 46406.6, + 46441.29 + ] + }, + "_throughput_note": "output_only", + "_concurrency_note": "client_concurrency is the number of requests sent simultaneously. The inference engine batches internally; this does not directly set engine parameters like max_num_seqs." + }, + { + "client_concurrency": 64, + "throughput_tokens_per_sec": 46500.82, + "throughput_tokens_per_sec_per_chip": 46500.82, + "throughput_tokens_per_sec_total": 62826.76, + "elapsed_seconds_median": 0.9, + "peak_memory_gb": null, + "power_watts_avg": null, + "power_watts_peak": null, + "oom": false, + "throughput_tokens_per_sec_reliability": { + "n": 3, + "mean": 46028.86, + "std": 890.06, + "cv_pct": 1.93, + "stability": "stable", + "runs": [ + 45002.22, + 46500.82, + 46583.55 + ] + }, + "_throughput_note": "output_only", + "_concurrency_note": "client_concurrency is the number of requests sent simultaneously. The inference engine batches internally; this does not directly set engine parameters like max_num_seqs." + } + ] + }, + "online": { + "sla_ttft_ms": 500, + "max_valid_qps": 40, + "results_by_qps": [ + { + "target_qps": 10, + "achieved_qps": 10.0, + "ttft_ms_p50": 8.39, + "ttft_ms_p90": 9.37, + "ttft_ms_p99": 14.12, + "tpot_ms_p50": 1.94, + "tpot_ms_p90": 2.05, + "tpot_ms_p99": 2.28, + "elapsed_seconds_median": 31.8, + "sla_met": true, + "ttft_ms_p99_reliability": { + "n": 3, + "mean": 17.9, + "std": 7.1, + "cv_pct": 39.67, + "stability": "high-variance", + "runs": [ + 25.95, + 15.2, + 12.54 + ] + } + }, + { + "target_qps": 40, + "achieved_qps": 40.0, + "ttft_ms_p50": 8.38, + "ttft_ms_p90": 11.9, + "ttft_ms_p99": 14.53, + "tpot_ms_p50": 2.4, + "tpot_ms_p90": 2.64, + "tpot_ms_p99": 3.18, + "elapsed_seconds_median": 7.8, + "sla_met": true, + "ttft_ms_p99_reliability": { + "n": 3, + "mean": 14.39, + "std": 0.16, + "cv_pct": 1.12, + "stability": "stable", + "runs": [ + 14.39, + 14.23, + 14.56 + ] + } + } + ] + }, + "interactive": { + "ttft_ms_p50": 7.19, + "ttft_ms_p90": 7.54, + "ttft_ms_p99": 15.06, + "tpot_ms_p50": 1.62, + "tpot_ms_p90": 1.64, + "tpot_ms_p99": 1.71, + "peak_memory_gb": null, + "elapsed_seconds_median": 46.2, + "ttft_ms_p99_reliability": { + "n": 3, + "mean": 23.42, + "std": 22.39, + "cv_pct": 95.62, + "stability": "high-variance", + "runs": [ + 49.2, + 12.23, + 8.83 + ] + } + } + }, + "accuracy": { + "subset_score": 0.41, + "baseline_delta": 0.03, + "valid": true, + "framework": "SGLang", + "precision": "BF16", + "notes": "Integrated accuracy check \u2014 used same SGLang instance as benchmark." + }, + "meta": { + "submitted_by": "ChiangYuhsin", + "submission_type": "individual", + "date": "2026-07-31", + "time": "02:40:05", + "run_id": "1189951b", + "run_name": "nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b", + "flagged": null, + "reproduce_script": "runners/nvidia_sglang_inference_opt_4fd1e59e/runner.py", + "env_info_file": "../env_info.json", + "log_file": "run.log", + "samples_file": "samples.jsonl", + "notes": null, + "benchmark_start_time": "2026-07-31T02:39:52.638757+00:00", + "benchmark_end_time": "2026-07-31T02:40:05.992733+00:00", + "benchmark_elapsed_minutes": 4.7, + "model_load_seconds": 47.1, + "benchmark_elapsed_minutes_note": "Total across ['offline', 'online', 'interactive'] scenarios.", + "scenario_dirs": { + "offline": "results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/offline", + "online": "results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/online", + "interactive": "results/community/nvidia_a100_sxm4_80gbx1_suite_F_nvidia_sglang_inference_opt_4fd1e59e_1189951b/interactive" + } + } +} \ No newline at end of file